4942 files changed, 316830 insertions, 113452 deletions

diff --git a/drivers/Kconfig b/drivers/Kconfig
index 968bd0a6fd78..514ae6b24cb2 100644
--- a/drivers/Kconfig
+++ b/drivers/Kconfig
@@ -241,4 +241,6 @@ source "drivers/peci/Kconfig"
 
 source "drivers/hte/Kconfig"
 
+source "drivers/cdx/Kconfig"
+
 endmenu
diff --git a/drivers/Makefile b/drivers/Makefile
index 20b118dca999..7241d80a7b29 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -194,3 +194,4 @@ obj-$(CONFIG_MOST)		+= most/
 obj-$(CONFIG_PECI)		+= peci/
 obj-$(CONFIG_HTE)		+= hte/
 obj-$(CONFIG_DRM_ACCEL)		+= accel/
+obj-$(CONFIG_CDX_BUS)		+= cdx/
diff --git a/drivers/accel/Kconfig b/drivers/accel/Kconfig
index c437206aa3f1..64065fb8922b 100644
--- a/drivers/accel/Kconfig
+++ b/drivers/accel/Kconfig
@@ -26,5 +26,6 @@ menuconfig DRM_ACCEL
 
 source "drivers/accel/habanalabs/Kconfig"
 source "drivers/accel/ivpu/Kconfig"
+source "drivers/accel/qaic/Kconfig"
 
 endif
diff --git a/drivers/accel/Makefile b/drivers/accel/Makefile
index f22fd44d586b..ab3df932937f 100644
--- a/drivers/accel/Makefile
+++ b/drivers/accel/Makefile
@@ -2,3 +2,4 @@
 
 obj-$(CONFIG_DRM_ACCEL_HABANALABS)	+= habanalabs/
 obj-$(CONFIG_DRM_ACCEL_IVPU)		+= ivpu/
+obj-$(CONFIG_DRM_ACCEL_QAIC)		+= qaic/
diff --git a/drivers/accel/drm_accel.c b/drivers/accel/drm_accel.c
index 1b69824286fd..4a9baf02439e 100644
--- a/drivers/accel/drm_accel.c
+++ b/drivers/accel/drm_accel.c
@@ -34,7 +34,7 @@ static char *accel_devnode(const struct device *dev, umode_t *mode)
 
 static int accel_sysfs_init(void)
 {
-	accel_class = class_create(THIS_MODULE, "accel");
+	accel_class = class_create("accel");
 	if (IS_ERR(accel_class))
 		return PTR_ERR(accel_class);
 
diff --git a/drivers/accel/habanalabs/common/command_buffer.c b/drivers/accel/habanalabs/common/command_buffer.c
index 3a0535ac28b1..6e09f48750a0 100644
--- a/drivers/accel/habanalabs/common/command_buffer.c
+++ b/drivers/accel/habanalabs/common/command_buffer.c
@@ -45,20 +45,29 @@ static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb)
 	}
 
 	mutex_lock(&hdev->mmu_lock);
+
 	rc = hl_mmu_map_contiguous(ctx, cb->virtual_addr, cb->bus_address, cb->roundup_size);
 	if (rc) {
 		dev_err(hdev->dev, "Failed to map VA %#llx to CB\n", cb->virtual_addr);
-		goto err_va_umap;
+		goto err_va_pool_free;
 	}
+
 	rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV);
+	if (rc)
+		goto err_mmu_unmap;
+
 	mutex_unlock(&hdev->mmu_lock);
 
 	cb->is_mmu_mapped = true;
-	return rc;
 
-err_va_umap:
+	return 0;
+
+err_mmu_unmap:
+	hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size);
+err_va_pool_free:
 	mutex_unlock(&hdev->mmu_lock);
 	gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size);
+
 	return rc;
 }
 
diff --git a/drivers/accel/habanalabs/common/command_submission.c b/drivers/accel/habanalabs/common/command_submission.c
index 8270db0a72a2..af9d2e22c6e7 100644
--- a/drivers/accel/habanalabs/common/command_submission.c
+++ b/drivers/accel/habanalabs/common/command_submission.c
@@ -14,10 +14,10 @@
 #define HL_CS_FLAGS_TYPE_MASK	(HL_CS_FLAGS_SIGNAL | HL_CS_FLAGS_WAIT | \
 			HL_CS_FLAGS_COLLECTIVE_WAIT | HL_CS_FLAGS_RESERVE_SIGNALS_ONLY | \
 			HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY | HL_CS_FLAGS_ENGINE_CORE_COMMAND | \
-			HL_CS_FLAGS_FLUSH_PCI_HBW_WRITES)
+			HL_CS_FLAGS_ENGINES_COMMAND | HL_CS_FLAGS_FLUSH_PCI_HBW_WRITES)
 
 
-#define MAX_TS_ITER_NUM 10
+#define MAX_TS_ITER_NUM 100
 
 /**
  * enum hl_cs_wait_status - cs wait status
@@ -657,7 +657,7 @@ static inline void cs_release_sob_reset_handler(struct hl_device *hdev,
 	/*
 	 * we get refcount upon reservation of signals or signal/wait cs for the
 	 * hw_sob object, and need to put it when the first staged cs
-	 * (which cotains the encaps signals) or cs signal/wait is completed.
+	 * (which contains the encaps signals) or cs signal/wait is completed.
 	 */
 	if ((hl_cs_cmpl->type == CS_TYPE_SIGNAL) ||
 			(hl_cs_cmpl->type == CS_TYPE_WAIT) ||
@@ -1082,9 +1082,8 @@ static void
 wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt)
 {
 	struct hl_user_pending_interrupt *pend, *temp;
-	unsigned long flags;
 
-	spin_lock_irqsave(&interrupt->wait_list_lock, flags);
+	spin_lock(&interrupt->wait_list_lock);
 	list_for_each_entry_safe(pend, temp, &interrupt->wait_list_head, wait_list_node) {
 		if (pend->ts_reg_info.buf) {
 			list_del(&pend->wait_list_node);
@@ -1095,7 +1094,7 @@ wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt)
 			complete_all(&pend->fence.completion);
 		}
 	}
-	spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
+	spin_unlock(&interrupt->wait_list_lock);
 }
 
 void hl_release_pending_user_interrupts(struct hl_device *hdev)
@@ -1168,6 +1167,22 @@ static void cs_completion(struct work_struct *work)
 		hl_complete_job(hdev, job);
 }
 
+u32 hl_get_active_cs_num(struct hl_device *hdev)
+{
+	u32 active_cs_num = 0;
+	struct hl_cs *cs;
+
+	spin_lock(&hdev->cs_mirror_lock);
+
+	list_for_each_entry(cs, &hdev->cs_mirror_list, mirror_node)
+		if (!cs->completed)
+			active_cs_num++;
+
+	spin_unlock(&hdev->cs_mirror_lock);
+
+	return active_cs_num;
+}
+
 static int validate_queue_index(struct hl_device *hdev,
 				struct hl_cs_chunk *chunk,
 				enum hl_queue_type *queue_type,
@@ -1304,6 +1319,8 @@ static enum hl_cs_type hl_cs_get_cs_type(u32 cs_type_flags)
 		return CS_UNRESERVE_SIGNALS;
 	else if (cs_type_flags & HL_CS_FLAGS_ENGINE_CORE_COMMAND)
 		return CS_TYPE_ENGINE_CORE;
+	else if (cs_type_flags & HL_CS_FLAGS_ENGINES_COMMAND)
+		return CS_TYPE_ENGINES;
 	else if (cs_type_flags & HL_CS_FLAGS_FLUSH_PCI_HBW_WRITES)
 		return CS_TYPE_FLUSH_PCI_HBW_WRITES;
 	else
@@ -2429,10 +2446,13 @@ out:
 static int cs_ioctl_engine_cores(struct hl_fpriv *hpriv, u64 engine_cores,
 						u32 num_engine_cores, u32 core_command)
 {
-	int rc;
 	struct hl_device *hdev = hpriv->hdev;
 	void __user *engine_cores_arr;
 	u32 *cores;
+	int rc;
+
+	if (!hdev->asic_prop.supports_engine_modes)
+		return -EPERM;
 
 	if (!num_engine_cores || num_engine_cores > hdev->asic_prop.num_engine_cores) {
 		dev_err(hdev->dev, "Number of engine cores %d is invalid\n", num_engine_cores);
@@ -2461,6 +2481,48 @@ static int cs_ioctl_engine_cores(struct hl_fpriv *hpriv, u64 engine_cores,
 	return rc;
 }
 
+static int cs_ioctl_engines(struct hl_fpriv *hpriv, u64 engines_arr_user_addr,
+						u32 num_engines, enum hl_engine_command command)
+{
+	struct hl_device *hdev = hpriv->hdev;
+	u32 *engines, max_num_of_engines;
+	void __user *engines_arr;
+	int rc;
+
+	if (!hdev->asic_prop.supports_engine_modes)
+		return -EPERM;
+
+	if (command >= HL_ENGINE_COMMAND_MAX) {
+		dev_err(hdev->dev, "Engine command is invalid\n");
+		return -EINVAL;
+	}
+
+	max_num_of_engines = hdev->asic_prop.max_num_of_engines;
+	if (command == HL_ENGINE_CORE_RUN || command == HL_ENGINE_CORE_HALT)
+		max_num_of_engines = hdev->asic_prop.num_engine_cores;
+
+	if (!num_engines || num_engines > max_num_of_engines) {
+		dev_err(hdev->dev, "Number of engines %d is invalid\n", num_engines);
+		return -EINVAL;
+	}
+
+	engines_arr = (void __user *) (uintptr_t) engines_arr_user_addr;
+	engines = kmalloc_array(num_engines, sizeof(u32), GFP_KERNEL);
+	if (!engines)
+		return -ENOMEM;
+
+	if (copy_from_user(engines, engines_arr, num_engines * sizeof(u32))) {
+		dev_err(hdev->dev, "Failed to copy engine-ids array from user\n");
+		kfree(engines);
+		return -EFAULT;
+	}
+
+	rc = hdev->asic_funcs->set_engines(hdev, engines, num_engines, command);
+	kfree(engines);
+
+	return rc;
+}
+
 static int cs_ioctl_flush_pci_hbw_writes(struct hl_fpriv *hpriv)
 {
 	struct hl_device *hdev = hpriv->hdev;
@@ -2532,6 +2594,10 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
 		rc = cs_ioctl_engine_cores(hpriv, args->in.engine_cores,
 				args->in.num_engine_cores, args->in.core_command);
 		break;
+	case CS_TYPE_ENGINES:
+		rc = cs_ioctl_engines(hpriv, args->in.engines,
+				args->in.num_engines, args->in.engine_command);
+		break;
 	case CS_TYPE_FLUSH_PCI_HBW_WRITES:
 		rc = cs_ioctl_flush_pci_hbw_writes(hpriv);
 		break;
@@ -3143,8 +3209,9 @@ static int ts_buff_get_kernel_ts_record(struct hl_mmap_mem_buf *buf,
 	struct hl_user_pending_interrupt *cb_last =
 			(struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address +
 			(ts_buff->kernel_buff_size / sizeof(struct hl_user_pending_interrupt));
-	unsigned long flags, iter_counter = 0;
+	unsigned long iter_counter = 0;
 	u64 current_cq_counter;
+	ktime_t timestamp;
 
 	/* Validate ts_offset not exceeding last max */
 	if (requested_offset_record >= cb_last) {
@@ -3153,8 +3220,10 @@ static int ts_buff_get_kernel_ts_record(struct hl_mmap_mem_buf *buf,
 		return -EINVAL;
 	}
 
+	timestamp = ktime_get();
+
 start_over:
-	spin_lock_irqsave(wait_list_lock, flags);
+	spin_lock(wait_list_lock);
 
 	/* Unregister only if we didn't reach the target value
 	 * since in this case there will be no handling in irq context
@@ -3165,7 +3234,7 @@ start_over:
 		current_cq_counter = *requested_offset_record->cq_kernel_addr;
 		if (current_cq_counter < requested_offset_record->cq_target_value) {
 			list_del(&requested_offset_record->wait_list_node);
-			spin_unlock_irqrestore(wait_list_lock, flags);
+			spin_unlock(wait_list_lock);
 
 			hl_mmap_mem_buf_put(requested_offset_record->ts_reg_info.buf);
 			hl_cb_put(requested_offset_record->ts_reg_info.cq_cb);
@@ -3176,13 +3245,14 @@ start_over:
 			dev_dbg(buf->mmg->dev,
 				"ts node in middle of irq handling\n");
 
-			/* irq handling in the middle give it time to finish */
-			spin_unlock_irqrestore(wait_list_lock, flags);
-			usleep_range(1, 10);
+			/* irq thread handling in the middle give it time to finish */
+			spin_unlock(wait_list_lock);
+			usleep_range(100, 1000);
 			if (++iter_counter == MAX_TS_ITER_NUM) {
 				dev_err(buf->mmg->dev,
-					"handling registration interrupt took too long!!\n");
-				return -EINVAL;
+					"Timestamp offset processing reached timeout of %lld ms\n",
+					ktime_ms_delta(ktime_get(), timestamp));
+				return -EAGAIN;
 			}
 
 			goto start_over;
@@ -3197,7 +3267,7 @@ start_over:
 				(u64 *) cq_cb->kernel_address + cq_offset;
 		requested_offset_record->cq_target_value = target_value;
 
-		spin_unlock_irqrestore(wait_list_lock, flags);
+		spin_unlock(wait_list_lock);
 	}
 
 	*pend = requested_offset_record;
@@ -3217,7 +3287,7 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
 	struct hl_user_pending_interrupt *pend;
 	struct hl_mmap_mem_buf *buf;
 	struct hl_cb *cq_cb;
-	unsigned long timeout, flags;
+	unsigned long timeout;
 	long completion_rc;
 	int rc = 0;
 
@@ -3264,7 +3334,7 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
 		pend->cq_target_value = target_value;
 	}
 
-	spin_lock_irqsave(&interrupt->wait_list_lock, flags);
+	spin_lock(&interrupt->wait_list_lock);
 
 	/* We check for completion value as interrupt could have been received
 	 * before we added the node to the wait list
@@ -3272,7 +3342,7 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
 	if (*pend->cq_kernel_addr >= target_value) {
 		if (register_ts_record)
 			pend->ts_reg_info.in_use = 0;
-		spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
+		spin_unlock(&interrupt->wait_list_lock);
 
 		*status = HL_WAIT_CS_STATUS_COMPLETED;
 
@@ -3284,7 +3354,7 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
 			goto set_timestamp;
 		}
 	} else if (!timeout_us) {
-		spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
+		spin_unlock(&interrupt->wait_list_lock);
 		*status = HL_WAIT_CS_STATUS_BUSY;
 		pend->fence.timestamp = ktime_get();
 		goto set_timestamp;
@@ -3309,7 +3379,7 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
 		pend->ts_reg_info.in_use = 1;
 
 	list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head);
-	spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
+	spin_unlock(&interrupt->wait_list_lock);
 
 	if (register_ts_record) {
 		rc = *status = HL_WAIT_CS_STATUS_COMPLETED;
@@ -3353,9 +3423,9 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
 	 * for ts record, the node will be deleted in the irq handler after
 	 * we reach the target value.
 	 */
-	spin_lock_irqsave(&interrupt->wait_list_lock, flags);
+	spin_lock(&interrupt->wait_list_lock);
 	list_del(&pend->wait_list_node);
-	spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
+	spin_unlock(&interrupt->wait_list_lock);
 
 set_timestamp:
 	*timestamp = ktime_to_ns(pend->fence.timestamp);
@@ -3383,7 +3453,7 @@ static int _hl_interrupt_wait_ioctl_user_addr(struct hl_device *hdev, struct hl_
 				u64 *timestamp)
 {
 	struct hl_user_pending_interrupt *pend;
-	unsigned long timeout, flags;
+	unsigned long timeout;
 	u64 completion_value;
 	long completion_rc;
 	int rc = 0;
@@ -3403,9 +3473,9 @@ static int _hl_interrupt_wait_ioctl_user_addr(struct hl_device *hdev, struct hl_
 	/* Add pending user interrupt to relevant list for the interrupt
 	 * handler to monitor
 	 */
-	spin_lock_irqsave(&interrupt->wait_list_lock, flags);
+	spin_lock(&interrupt->wait_list_lock);
 	list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head);
-	spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
+	spin_unlock(&interrupt->wait_list_lock);
 
 	/* We check for completion value as interrupt could have been received
 	 * before we added the node to the wait list
@@ -3436,14 +3506,14 @@ wait_again:
 	 * If comparison fails, keep waiting until timeout expires
 	 */
 	if (completion_rc > 0) {
-		spin_lock_irqsave(&interrupt->wait_list_lock, flags);
+		spin_lock(&interrupt->wait_list_lock);
 		/* reinit_completion must be called before we check for user
 		 * completion value, otherwise, if interrupt is received after
 		 * the comparison and before the next wait_for_completion,
 		 * we will reach timeout and fail
 		 */
 		reinit_completion(&pend->fence.completion);
-		spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
+		spin_unlock(&interrupt->wait_list_lock);
 
 		if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 8)) {
 			dev_err(hdev->dev, "Failed to copy completion value from user\n");
@@ -3480,9 +3550,9 @@ wait_again:
 	}
 
 remove_pending_user_interrupt:
-	spin_lock_irqsave(&interrupt->wait_list_lock, flags);
+	spin_lock(&interrupt->wait_list_lock);
 	list_del(&pend->wait_list_node);
-	spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
+	spin_unlock(&interrupt->wait_list_lock);
 
 	*timestamp = ktime_to_ns(pend->fence.timestamp);
 
diff --git a/drivers/accel/habanalabs/common/debugfs.c b/drivers/accel/habanalabs/common/debugfs.c
index 945c0e6758ca..22dd17c077c0 100644
--- a/drivers/accel/habanalabs/common/debugfs.c
+++ b/drivers/accel/habanalabs/common/debugfs.c
@@ -258,7 +258,7 @@ static int vm_show(struct seq_file *s, void *data)
 	if (!dev_entry->hdev->mmu_enable)
 		return 0;
 
-	spin_lock(&dev_entry->ctx_mem_hash_spinlock);
+	mutex_lock(&dev_entry->ctx_mem_hash_mutex);
 
 	list_for_each_entry(ctx, &dev_entry->ctx_mem_hash_list, debugfs_list) {
 		once = false;
@@ -329,7 +329,7 @@ static int vm_show(struct seq_file *s, void *data)
 
 	}
 
-	spin_unlock(&dev_entry->ctx_mem_hash_spinlock);
+	mutex_unlock(&dev_entry->ctx_mem_hash_mutex);
 
 	ctx = hl_get_compute_ctx(dev_entry->hdev);
 	if (ctx) {
@@ -1583,209 +1583,216 @@ static const struct file_operations hl_debugfs_fops = {
 	.release = single_release,
 };
 
-static void add_secured_nodes(struct hl_dbg_device_entry *dev_entry)
+static void add_secured_nodes(struct hl_dbg_device_entry *dev_entry, struct dentry *root)
 {
 	debugfs_create_u8("i2c_bus",
 				0644,
-				dev_entry->root,
+				root,
 				&dev_entry->i2c_bus);
 
 	debugfs_create_u8("i2c_addr",
 				0644,
-				dev_entry->root,
+				root,
 				&dev_entry->i2c_addr);
 
 	debugfs_create_u8("i2c_reg",
 				0644,
-				dev_entry->root,
+				root,
 				&dev_entry->i2c_reg);
 
 	debugfs_create_u8("i2c_len",
 				0644,
-				dev_entry->root,
+				root,
 				&dev_entry->i2c_len);
 
 	debugfs_create_file("i2c_data",
 				0644,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_i2c_data_fops);
 
 	debugfs_create_file("led0",
 				0200,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_led0_fops);
 
 	debugfs_create_file("led1",
 				0200,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_led1_fops);
 
 	debugfs_create_file("led2",
 				0200,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_led2_fops);
 }
 
-void hl_debugfs_add_device(struct hl_device *hdev)
+static void add_files_to_device(struct hl_device *hdev, struct hl_dbg_device_entry *dev_entry,
+				struct dentry *root)
 {
-	struct hl_dbg_device_entry *dev_entry = &hdev->hl_debugfs;
 	int count = ARRAY_SIZE(hl_debugfs_list);
 	struct hl_debugfs_entry *entry;
 	int i;
 
-	dev_entry->hdev = hdev;
-	dev_entry->entry_arr = kmalloc_array(count,
-					sizeof(struct hl_debugfs_entry),
-					GFP_KERNEL);
-	if (!dev_entry->entry_arr)
-		return;
-
-	dev_entry->data_dma_blob_desc.size = 0;
-	dev_entry->data_dma_blob_desc.data = NULL;
-	dev_entry->mon_dump_blob_desc.size = 0;
-	dev_entry->mon_dump_blob_desc.data = NULL;
-
-	INIT_LIST_HEAD(&dev_entry->file_list);
-	INIT_LIST_HEAD(&dev_entry->cb_list);
-	INIT_LIST_HEAD(&dev_entry->cs_list);
-	INIT_LIST_HEAD(&dev_entry->cs_job_list);
-	INIT_LIST_HEAD(&dev_entry->userptr_list);
-	INIT_LIST_HEAD(&dev_entry->ctx_mem_hash_list);
-	mutex_init(&dev_entry->file_mutex);
-	init_rwsem(&dev_entry->state_dump_sem);
-	spin_lock_init(&dev_entry->cb_spinlock);
-	spin_lock_init(&dev_entry->cs_spinlock);
-	spin_lock_init(&dev_entry->cs_job_spinlock);
-	spin_lock_init(&dev_entry->userptr_spinlock);
-	spin_lock_init(&dev_entry->ctx_mem_hash_spinlock);
-
-	dev_entry->root = debugfs_create_dir(dev_name(hdev->dev),
-						hl_debug_root);
-
 	debugfs_create_x64("memory_scrub_val",
 				0644,
-				dev_entry->root,
+				root,
 				&hdev->memory_scrub_val);
 
 	debugfs_create_file("memory_scrub",
 				0200,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_mem_scrub_fops);
 
 	debugfs_create_x64("addr",
 				0644,
-				dev_entry->root,
+				root,
 				&dev_entry->addr);
 
 	debugfs_create_file("data32",
 				0644,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_data32b_fops);
 
 	debugfs_create_file("data64",
 				0644,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_data64b_fops);
 
 	debugfs_create_file("set_power_state",
 				0200,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_power_fops);
 
 	debugfs_create_file("device",
 				0200,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_device_fops);
 
 	debugfs_create_file("clk_gate",
 				0200,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_clk_gate_fops);
 
 	debugfs_create_file("stop_on_err",
 				0644,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_stop_on_err_fops);
 
 	debugfs_create_file("dump_security_violations",
 				0644,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_security_violations_fops);
 
 	debugfs_create_file("dump_razwi_events",
 				0644,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_razwi_check_fops);
 
 	debugfs_create_file("dma_size",
 				0200,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_dma_size_fops);
 
 	debugfs_create_blob("data_dma",
 				0400,
-				dev_entry->root,
+				root,
 				&dev_entry->data_dma_blob_desc);
 
 	debugfs_create_file("monitor_dump_trig",
 				0200,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_monitor_dump_fops);
 
 	debugfs_create_blob("monitor_dump",
 				0400,
-				dev_entry->root,
+				root,
 				&dev_entry->mon_dump_blob_desc);
 
 	debugfs_create_x8("skip_reset_on_timeout",
 				0644,
-				dev_entry->root,
+				root,
 				&hdev->reset_info.skip_reset_on_timeout);
 
 	debugfs_create_file("state_dump",
 				0600,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_state_dump_fops);
 
 	debugfs_create_file("timeout_locked",
 				0644,
-				dev_entry->root,
+				root,
 				dev_entry,
 				&hl_timeout_locked_fops);
 
 	debugfs_create_u32("device_release_watchdog_timeout",
 				0644,
-				dev_entry->root,
+				root,
 				&hdev->device_release_watchdog_timeout_sec);
 
 	for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) {
 		debugfs_create_file(hl_debugfs_list[i].name,
 					0444,
-					dev_entry->root,
+					root,
 					entry,
 					&hl_debugfs_fops);
 		entry->info_ent = &hl_debugfs_list[i];
 		entry->dev_entry = dev_entry;
 	}
+}
+
+void hl_debugfs_add_device(struct hl_device *hdev)
+{
+	struct hl_dbg_device_entry *dev_entry = &hdev->hl_debugfs;
+	int count = ARRAY_SIZE(hl_debugfs_list);
+
+	dev_entry->hdev = hdev;
+	dev_entry->entry_arr = kmalloc_array(count,
+					sizeof(struct hl_debugfs_entry),
+					GFP_KERNEL);
+	if (!dev_entry->entry_arr)
+		return;
+
+	dev_entry->data_dma_blob_desc.size = 0;
+	dev_entry->data_dma_blob_desc.data = NULL;
+	dev_entry->mon_dump_blob_desc.size = 0;
+	dev_entry->mon_dump_blob_desc.data = NULL;
+
+	INIT_LIST_HEAD(&dev_entry->file_list);
+	INIT_LIST_HEAD(&dev_entry->cb_list);
+	INIT_LIST_HEAD(&dev_entry->cs_list);
+	INIT_LIST_HEAD(&dev_entry->cs_job_list);
+	INIT_LIST_HEAD(&dev_entry->userptr_list);
+	INIT_LIST_HEAD(&dev_entry->ctx_mem_hash_list);
+	mutex_init(&dev_entry->file_mutex);
+	init_rwsem(&dev_entry->state_dump_sem);
+	spin_lock_init(&dev_entry->cb_spinlock);
+	spin_lock_init(&dev_entry->cs_spinlock);
+	spin_lock_init(&dev_entry->cs_job_spinlock);
+	spin_lock_init(&dev_entry->userptr_spinlock);
+	mutex_init(&dev_entry->ctx_mem_hash_mutex);
+
+	dev_entry->root = debugfs_create_dir(dev_name(hdev->dev),
+						hl_debug_root);
 
+	add_files_to_device(hdev, dev_entry, dev_entry->root);
 	if (!hdev->asic_prop.fw_security_enabled)
-		add_secured_nodes(dev_entry);
+		add_secured_nodes(dev_entry, dev_entry->root);
 }
 
 void hl_debugfs_remove_device(struct hl_device *hdev)
@@ -1795,6 +1802,7 @@ void hl_debugfs_remove_device(struct hl_device *hdev)
 
 	debugfs_remove_recursive(entry->root);
 
+	mutex_destroy(&entry->ctx_mem_hash_mutex);
 	mutex_destroy(&entry->file_mutex);
 
 	vfree(entry->data_dma_blob_desc.data);
@@ -1901,18 +1909,18 @@ void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx)
 {
 	struct hl_dbg_device_entry *dev_entry = &hdev->hl_debugfs;
 
-	spin_lock(&dev_entry->ctx_mem_hash_spinlock);
+	mutex_lock(&dev_entry->ctx_mem_hash_mutex);
 	list_add(&ctx->debugfs_list, &dev_entry->ctx_mem_hash_list);
-	spin_unlock(&dev_entry->ctx_mem_hash_spinlock);
+	mutex_unlock(&dev_entry->ctx_mem_hash_mutex);
 }
 
 void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx)
 {
 	struct hl_dbg_device_entry *dev_entry = &hdev->hl_debugfs;
 
-	spin_lock(&dev_entry->ctx_mem_hash_spinlock);
+	mutex_lock(&dev_entry->ctx_mem_hash_mutex);
 	list_del(&ctx->debugfs_list);
-	spin_unlock(&dev_entry->ctx_mem_hash_spinlock);
+	mutex_unlock(&dev_entry->ctx_mem_hash_mutex);
 }
 
 /**
diff --git a/drivers/accel/habanalabs/common/decoder.c b/drivers/accel/habanalabs/common/decoder.c
index 2aab14d74b53..c03a6da45d00 100644
--- a/drivers/accel/habanalabs/common/decoder.c
+++ b/drivers/accel/habanalabs/common/decoder.c
@@ -43,36 +43,44 @@ static void dec_print_abnrm_intr_source(struct hl_device *hdev, u32 irq_status)
 		intr_source[2], intr_source[3], intr_source[4], intr_source[5]);
 }
 
-static void dec_error_intr_work(struct hl_device *hdev, u32 base_addr, u32 core_id)
+static void dec_abnrm_intr_work(struct work_struct *work)
 {
+	struct hl_dec *dec = container_of(work, struct hl_dec, abnrm_intr_work);
+	struct hl_device *hdev = dec->hdev;
+	u32 irq_status, event_mask = 0;
 	bool reset_required = false;
-	u32 irq_status;
 
-	irq_status = RREG32(base_addr + VCMD_IRQ_STATUS_OFFSET);
+	irq_status = RREG32(dec->base_addr + VCMD_IRQ_STATUS_OFFSET);
 
-	dev_err(hdev->dev, "Decoder abnormal interrupt %#x, core %d\n", irq_status, core_id);
+	dev_err(hdev->dev, "Decoder abnormal interrupt %#x, core %d\n", irq_status, dec->core_id);
 
 	dec_print_abnrm_intr_source(hdev, irq_status);
 
-	if (irq_status & VCMD_IRQ_STATUS_TIMEOUT_MASK)
-		reset_required = true;
-
 	/* Clear the interrupt */
-	WREG32(base_addr + VCMD_IRQ_STATUS_OFFSET, irq_status);
+	WREG32(dec->base_addr + VCMD_IRQ_STATUS_OFFSET, irq_status);
 
 	/* Flush the interrupt clear */
-	RREG32(base_addr + VCMD_IRQ_STATUS_OFFSET);
-
-	if (reset_required)
-		hl_device_reset(hdev, HL_DRV_RESET_HARD);
-}
+	RREG32(dec->base_addr + VCMD_IRQ_STATUS_OFFSET);
 
-static void dec_completion_abnrm(struct work_struct *work)
-{
-	struct hl_dec *dec = container_of(work, struct hl_dec, completion_abnrm_work);
-	struct hl_device *hdev = dec->hdev;
+	if (irq_status & VCMD_IRQ_STATUS_TIMEOUT_MASK) {
+		reset_required = true;
+		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
+	}
 
-	dec_error_intr_work(hdev, dec->base_addr, dec->core_id);
+	if (irq_status & VCMD_IRQ_STATUS_CMDERR_MASK)
+		event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE;
+
+	if (irq_status & (VCMD_IRQ_STATUS_ENDCMD_MASK |
+				VCMD_IRQ_STATUS_BUSERR_MASK |
+				VCMD_IRQ_STATUS_ABORT_MASK))
+		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
+
+	if (reset_required) {
+		event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET;
+		hl_device_cond_reset(hdev, 0, event_mask);
+	} else if (event_mask) {
+		hl_notifier_event_send_all(hdev, event_mask);
+	}
 }
 
 void hl_dec_fini(struct hl_device *hdev)
@@ -98,7 +106,7 @@ int hl_dec_init(struct hl_device *hdev)
 		dec = hdev->dec + j;
 
 		dec->hdev = hdev;
-		INIT_WORK(&dec->completion_abnrm_work, dec_completion_abnrm);
+		INIT_WORK(&dec->abnrm_intr_work, dec_abnrm_intr_work);
 		dec->core_id = j;
 		dec->base_addr = hdev->asic_funcs->get_dec_base_addr(hdev, j);
 		if (!dec->base_addr) {
diff --git a/drivers/accel/habanalabs/common/device.c b/drivers/accel/habanalabs/common/device.c
index 9933e5858a36..fabfc501ef54 100644
--- a/drivers/accel/habanalabs/common/device.c
+++ b/drivers/accel/habanalabs/common/device.c
@@ -22,7 +22,6 @@
 
 enum dma_alloc_type {
 	DMA_ALLOC_COHERENT,
-	DMA_ALLOC_CPU_ACCESSIBLE,
 	DMA_ALLOC_POOL,
 };
 
@@ -121,9 +120,6 @@ static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t
 	case DMA_ALLOC_COHERENT:
 		ptr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, size, dma_handle, flag);
 		break;
-	case DMA_ALLOC_CPU_ACCESSIBLE:
-		ptr = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
-		break;
 	case DMA_ALLOC_POOL:
 		ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, size, flag, dma_handle);
 		break;
@@ -147,9 +143,6 @@ static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *c
 	case DMA_ALLOC_COHERENT:
 		hdev->asic_funcs->asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle);
 		break;
-	case DMA_ALLOC_CPU_ACCESSIBLE:
-		hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, cpu_addr);
-		break;
 	case DMA_ALLOC_POOL:
 		hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle);
 		break;
@@ -170,18 +163,6 @@ void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void
 	hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller);
 }
 
-void *hl_cpu_accessible_dma_pool_alloc_caller(struct hl_device *hdev, size_t size,
-						dma_addr_t *dma_handle, const char *caller)
-{
-	return hl_dma_alloc_common(hdev, size, dma_handle, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller);
-}
-
-void hl_cpu_accessible_dma_pool_free_caller(struct hl_device *hdev, size_t size, void *vaddr,
-						const char *caller)
-{
-	hl_asic_dma_free_common(hdev, size, vaddr, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller);
-}
-
 void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags,
 					dma_addr_t *dma_handle, const char *caller)
 {
@@ -194,6 +175,16 @@ void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_
 	hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller);
 }
 
+void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle)
+{
+	return hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
+}
+
+void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr)
+{
+	hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, vaddr);
+}
+
 int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir)
 {
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
@@ -389,18 +380,17 @@ bool hl_ctrl_device_operational(struct hl_device *hdev,
 static void print_idle_status_mask(struct hl_device *hdev, const char *message,
 					u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE])
 {
-	u32 pad_width[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {};
-
-	BUILD_BUG_ON(HL_BUSY_ENGINES_MASK_EXT_SIZE != 4);
-
-	pad_width[3] = idle_mask[3] ? 16 : 0;
-	pad_width[2] = idle_mask[2] || pad_width[3] ? 16 : 0;
-	pad_width[1] = idle_mask[1] || pad_width[2] ? 16 : 0;
-	pad_width[0] = idle_mask[0] || pad_width[1] ? 16 : 0;
-
-	dev_err(hdev->dev, "%s (mask %0*llx_%0*llx_%0*llx_%0*llx)\n",
-		message, pad_width[3], idle_mask[3], pad_width[2], idle_mask[2],
-		pad_width[1], idle_mask[1], pad_width[0], idle_mask[0]);
+	if (idle_mask[3])
+		dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx_%016llx)\n",
+			message, idle_mask[3], idle_mask[2], idle_mask[1], idle_mask[0]);
+	else if (idle_mask[2])
+		dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx)\n",
+			message, idle_mask[2], idle_mask[1], idle_mask[0]);
+	else if (idle_mask[1])
+		dev_err(hdev->dev, "%s (mask %#llx_%016llx)\n",
+			message, idle_mask[1], idle_mask[0]);
+	else
+		dev_err(hdev->dev, "%s (mask %#llx)\n", message, idle_mask[0]);
 }
 
 static void hpriv_release(struct kref *ref)
@@ -423,6 +413,9 @@ static void hpriv_release(struct kref *ref)
 	mutex_destroy(&hpriv->ctx_lock);
 	mutex_destroy(&hpriv->restore_phase_mutex);
 
+	/* There should be no memory buffers at this point and handles IDR can be destroyed */
+	hl_mem_mgr_idr_destroy(&hpriv->mem_mgr);
+
 	/* Device should be reset if reset-upon-device-release is enabled, or if there is a pending
 	 * reset that waits for device release.
 	 */
@@ -492,6 +485,36 @@ int hl_hpriv_put(struct hl_fpriv *hpriv)
 	return kref_put(&hpriv->refcount, hpriv_release);
 }
 
+static void print_device_in_use_info(struct hl_device *hdev, const char *message)
+{
+	u32 active_cs_num, dmabuf_export_cnt;
+	bool unknown_reason = true;
+	char buf[128];
+	size_t size;
+	int offset;
+
+	size = sizeof(buf);
+	offset = 0;
+
+	active_cs_num = hl_get_active_cs_num(hdev);
+	if (active_cs_num) {
+		unknown_reason = false;
+		offset += scnprintf(buf + offset, size - offset, " [%u active CS]", active_cs_num);
+	}
+
+	dmabuf_export_cnt = atomic_read(&hdev->dmabuf_export_cnt);
+	if (dmabuf_export_cnt) {
+		unknown_reason = false;
+		offset += scnprintf(buf + offset, size - offset, " [%u exported dma-buf]",
+					dmabuf_export_cnt);
+	}
+
+	if (unknown_reason)
+		scnprintf(buf + offset, size - offset, " [unknown reason]");
+
+	dev_notice(hdev->dev, "%s%s\n", message, buf);
+}
+
 /*
  * hl_device_release - release function for habanalabs device
  *
@@ -514,17 +537,20 @@ static int hl_device_release(struct inode *inode, struct file *filp)
 	}
 
 	hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
+
+	/* Memory buffers might be still in use at this point and thus the handles IDR destruction
+	 * is postponed to hpriv_release().
+	 */
 	hl_mem_mgr_fini(&hpriv->mem_mgr);
 
 	hdev->compute_ctx_in_release = 1;
 
 	if (!hl_hpriv_put(hpriv)) {
-		dev_notice(hdev->dev, "User process closed FD but device still in use\n");
+		print_device_in_use_info(hdev, "User process closed FD but device still in use");
 		hl_device_reset(hdev, HL_DRV_RESET_HARD);
 	}
 
-	hdev->last_open_session_duration_jif =
-		jiffies - hdev->last_successful_open_jif;
+	hdev->last_open_session_duration_jif = jiffies - hdev->last_successful_open_jif;
 
 	return 0;
 }
@@ -617,7 +643,7 @@ static void device_release_func(struct device *dev)
  * device_init_cdev - Initialize cdev and device for habanalabs device
  *
  * @hdev: pointer to habanalabs device structure
- * @hclass: pointer to the class object of the device
+ * @class: pointer to the class object of the device
  * @minor: minor number of the specific device
  * @fpos: file operations to install for this device
  * @name: name of the device as it will appear in the filesystem
@@ -626,7 +652,7 @@ static void device_release_func(struct device *dev)
  *
  * Initialize a cdev and a Linux device for habanalabs's device.
  */
-static int device_init_cdev(struct hl_device *hdev, struct class *hclass,
+static int device_init_cdev(struct hl_device *hdev, struct class *class,
 				int minor, const struct file_operations *fops,
 				char *name, struct cdev *cdev,
 				struct device **dev)
@@ -640,7 +666,7 @@ static int device_init_cdev(struct hl_device *hdev, struct class *hclass,
 
 	device_initialize(*dev);
 	(*dev)->devt = MKDEV(hdev->major, minor);
-	(*dev)->class = hclass;
+	(*dev)->class = class;
 	(*dev)->release = device_release_func;
 	dev_set_drvdata(*dev, hdev);
 	dev_set_name(*dev, "%s", name);
@@ -733,14 +759,14 @@ static void device_hard_reset_pending(struct work_struct *work)
 
 static void device_release_watchdog_func(struct work_struct *work)
 {
-	struct hl_device_reset_work *device_release_watchdog_work =
-				container_of(work, struct hl_device_reset_work, reset_work.work);
-	struct hl_device *hdev = device_release_watchdog_work->hdev;
+	struct hl_device_reset_work *watchdog_work =
+			container_of(work, struct hl_device_reset_work, reset_work.work);
+	struct hl_device *hdev = watchdog_work->hdev;
 	u32 flags;
 
-	dev_dbg(hdev->dev, "Device wasn't released in time. Initiate device reset.\n");
+	dev_dbg(hdev->dev, "Device wasn't released in time. Initiate hard-reset.\n");
 
-	flags = device_release_watchdog_work->flags | HL_DRV_RESET_FROM_WD_THR;
+	flags = watchdog_work->flags | HL_DRV_RESET_HARD | HL_DRV_RESET_FROM_WD_THR;
 
 	hl_device_reset(hdev, flags);
 }
@@ -805,7 +831,7 @@ static int device_early_init(struct hl_device *hdev)
 	}
 
 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
-		snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i);
+		snprintf(workq_name, 32, "hl%u-free-jobs-%u", hdev->cdev_idx, (u32) i);
 		hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
 		if (hdev->cq_wq[i] == NULL) {
 			dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
@@ -814,14 +840,16 @@ static int device_early_init(struct hl_device *hdev)
 		}
 	}
 
-	hdev->eq_wq = create_singlethread_workqueue("hl-events");
+	snprintf(workq_name, 32, "hl%u-events", hdev->cdev_idx);
+	hdev->eq_wq = create_singlethread_workqueue(workq_name);
 	if (hdev->eq_wq == NULL) {
 		dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
 		rc = -ENOMEM;
 		goto free_cq_wq;
 	}
 
-	hdev->cs_cmplt_wq = alloc_workqueue("hl-cs-completions", WQ_UNBOUND, 0);
+	snprintf(workq_name, 32, "hl%u-cs-completions", hdev->cdev_idx);
+	hdev->cs_cmplt_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
 	if (!hdev->cs_cmplt_wq) {
 		dev_err(hdev->dev,
 			"Failed to allocate CS completions workqueue\n");
@@ -829,7 +857,8 @@ static int device_early_init(struct hl_device *hdev)
 		goto free_eq_wq;
 	}
 
-	hdev->ts_free_obj_wq = alloc_workqueue("hl-ts-free-obj", WQ_UNBOUND, 0);
+	snprintf(workq_name, 32, "hl%u-ts-free-obj", hdev->cdev_idx);
+	hdev->ts_free_obj_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
 	if (!hdev->ts_free_obj_wq) {
 		dev_err(hdev->dev,
 			"Failed to allocate Timestamp registration free workqueue\n");
@@ -837,15 +866,15 @@ static int device_early_init(struct hl_device *hdev)
 		goto free_cs_cmplt_wq;
 	}
 
-	hdev->prefetch_wq = alloc_workqueue("hl-prefetch", WQ_UNBOUND, 0);
+	snprintf(workq_name, 32, "hl%u-prefetch", hdev->cdev_idx);
+	hdev->prefetch_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
 	if (!hdev->prefetch_wq) {
 		dev_err(hdev->dev, "Failed to allocate MMU prefetch workqueue\n");
 		rc = -ENOMEM;
 		goto free_ts_free_wq;
 	}
 
-	hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
-					GFP_KERNEL);
+	hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), GFP_KERNEL);
 	if (!hdev->hl_chip_info) {
 		rc = -ENOMEM;
 		goto free_prefetch_wq;
@@ -857,7 +886,8 @@ static int device_early_init(struct hl_device *hdev)
 
 	hl_mem_mgr_init(hdev->dev, &hdev->kernel_mem_mgr);
 
-	hdev->reset_wq = create_singlethread_workqueue("hl_device_reset");
+	snprintf(workq_name, 32, "hl%u_device_reset", hdev->cdev_idx);
+	hdev->reset_wq = create_singlethread_workqueue(workq_name);
 	if (!hdev->reset_wq) {
 		rc = -ENOMEM;
 		dev_err(hdev->dev, "Failed to create device reset WQ\n");
@@ -887,6 +917,7 @@ static int device_early_init(struct hl_device *hdev)
 
 free_cb_mgr:
 	hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
+	hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr);
 free_chip_info:
 	kfree(hdev->hl_chip_info);
 free_prefetch_wq:
@@ -930,6 +961,7 @@ static void device_early_fini(struct hl_device *hdev)
 	mutex_destroy(&hdev->clk_throttling.lock);
 
 	hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
+	hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr);
 
 	kfree(hdev->hl_chip_info);
 
@@ -953,6 +985,8 @@ static void hl_device_heartbeat(struct work_struct *work)
 {
 	struct hl_device *hdev = container_of(work, struct hl_device,
 						work_heartbeat.work);
+	struct hl_info_fw_err_info info = {0};
+	u64 event_mask = HL_NOTIFIER_EVENT_DEVICE_RESET | HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
 
 	if (!hl_device_operational(hdev, NULL))
 		goto reschedule;
@@ -963,7 +997,10 @@ static void hl_device_heartbeat(struct work_struct *work)
 	if (hl_device_operational(hdev, NULL))
 		dev_err(hdev->dev, "Device heartbeat failed!\n");
 
-	hl_device_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT);
+	info.err_type = HL_INFO_FW_HEARTBEAT_ERR;
+	info.event_mask = &event_mask;
+	hl_handle_fw_err(hdev, &info);
+	hl_device_cond_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT, event_mask);
 
 	return;
 
@@ -1234,7 +1271,6 @@ int hl_device_resume(struct hl_device *hdev)
 	return 0;
 
 disable_device:
-	pci_clear_master(hdev->pdev);
 	pci_disable_device(hdev->pdev);
 
 	return rc;
@@ -1344,6 +1380,34 @@ static void device_disable_open_processes(struct hl_device *hdev, bool control_d
 	mutex_unlock(fd_lock);
 }
 
+static void send_disable_pci_access(struct hl_device *hdev, u32 flags)
+{
+	/* If reset is due to heartbeat, device CPU is no responsive in
+	 * which case no point sending PCI disable message to it.
+	 */
+	if ((flags & HL_DRV_RESET_HARD) &&
+			!(flags & (HL_DRV_RESET_HEARTBEAT | HL_DRV_RESET_BYPASS_REQ_TO_FW))) {
+		/* Disable PCI access from device F/W so he won't send
+		 * us additional interrupts. We disable MSI/MSI-X at
+		 * the halt_engines function and we can't have the F/W
+		 * sending us interrupts after that. We need to disable
+		 * the access here because if the device is marked
+		 * disable, the message won't be send. Also, in case
+		 * of heartbeat, the device CPU is marked as disable
+		 * so this message won't be sent
+		 */
+		if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0)) {
+			dev_warn(hdev->dev, "Failed to disable FW's PCI access\n");
+			return;
+		}
+
+		/* verify that last EQs are handled before disabled is set */
+		if (hdev->cpu_queues_enable)
+			synchronize_irq(pci_irq_vector(hdev->pdev,
+					hdev->asic_prop.eq_interrupt_id));
+	}
+}
+
 static void handle_reset_trigger(struct hl_device *hdev, u32 flags)
 {
 	u32 cur_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
@@ -1382,28 +1446,6 @@ static void handle_reset_trigger(struct hl_device *hdev, u32 flags)
 	} else {
 		hdev->reset_info.reset_trigger_repeated = 1;
 	}
-
-	/* If reset is due to heartbeat, device CPU is no responsive in
-	 * which case no point sending PCI disable message to it.
-	 *
-	 * If F/W is performing the reset, no need to send it a message to disable
-	 * PCI access
-	 */
-	if ((flags & HL_DRV_RESET_HARD) &&
-			!(flags & (HL_DRV_RESET_HEARTBEAT | HL_DRV_RESET_BYPASS_REQ_TO_FW))) {
-		/* Disable PCI access from device F/W so he won't send
-		 * us additional interrupts. We disable MSI/MSI-X at
-		 * the halt_engines function and we can't have the F/W
-		 * sending us interrupts after that. We need to disable
-		 * the access here because if the device is marked
-		 * disable, the message won't be send. Also, in case
-		 * of heartbeat, the device CPU is marked as disable
-		 * so this message won't be sent
-		 */
-		if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0))
-			dev_warn(hdev->dev,
-				"Failed to disable PCI access by F/W\n");
-	}
 }
 
 /*
@@ -1424,12 +1466,11 @@ static void handle_reset_trigger(struct hl_device *hdev, u32 flags)
  */
 int hl_device_reset(struct hl_device *hdev, u32 flags)
 {
-	bool hard_reset, from_hard_reset_thread, fw_reset, hard_instead_soft = false,
-			reset_upon_device_release = false, schedule_hard_reset = false,
-			delay_reset, from_dev_release, from_watchdog_thread;
+	bool hard_reset, from_hard_reset_thread, fw_reset, reset_upon_device_release,
+		schedule_hard_reset = false, delay_reset, from_dev_release, from_watchdog_thread;
 	u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
 	struct hl_ctx *ctx;
-	int i, rc;
+	int i, rc, hw_fini_rc;
 
 	if (!hdev->init_done) {
 		dev_err(hdev->dev, "Can't reset before initialization is done\n");
@@ -1442,6 +1483,7 @@ int hl_device_reset(struct hl_device *hdev, u32 flags)
 	from_dev_release = !!(flags & HL_DRV_RESET_DEV_RELEASE);
 	delay_reset = !!(flags & HL_DRV_RESET_DELAY);
 	from_watchdog_thread = !!(flags & HL_DRV_RESET_FROM_WD_THR);
+	reset_upon_device_release = hdev->reset_upon_device_release && from_dev_release;
 
 	if (!hard_reset && (hl_device_status(hdev) == HL_DEVICE_STATUS_MALFUNCTION)) {
 		dev_dbg(hdev->dev, "soft-reset isn't supported on a malfunctioning device\n");
@@ -1449,30 +1491,26 @@ int hl_device_reset(struct hl_device *hdev, u32 flags)
 	}
 
 	if (!hard_reset && !hdev->asic_prop.supports_compute_reset) {
-		hard_instead_soft = true;
+		dev_dbg(hdev->dev, "asic doesn't support compute reset - do hard-reset instead\n");
 		hard_reset = true;
 	}
 
-	if (hdev->reset_upon_device_release && from_dev_release) {
+	if (reset_upon_device_release) {
 		if (hard_reset) {
 			dev_crit(hdev->dev,
 				"Aborting reset because hard-reset is mutually exclusive with reset-on-device-release\n");
 			return -EINVAL;
 		}
 
-		reset_upon_device_release = true;
-
 		goto do_reset;
 	}
 
 	if (!hard_reset && !hdev->asic_prop.allow_inference_soft_reset) {
-		hard_instead_soft = true;
+		dev_dbg(hdev->dev,
+			"asic doesn't allow inference soft reset - do hard-reset instead\n");
 		hard_reset = true;
 	}
 
-	if (hard_instead_soft)
-		dev_dbg(hdev->dev, "Doing hard-reset instead of compute reset\n");
-
 do_reset:
 	/* Re-entry of reset thread */
 	if (from_hard_reset_thread && hdev->process_kill_trial_cnt)
@@ -1480,14 +1518,14 @@ do_reset:
 
 	/*
 	 * Prevent concurrency in this function - only one reset should be
-	 * done at any given time. Only need to perform this if we didn't
-	 * get from the dedicated hard reset thread
+	 * done at any given time. We need to perform this only if we didn't
+	 * get here from a dedicated hard reset thread.
 	 */
 	if (!from_hard_reset_thread) {
 		/* Block future CS/VM/JOB completion operations */
 		spin_lock(&hdev->reset_info.lock);
 		if (hdev->reset_info.in_reset) {
-			/* We only allow scheduling of a hard reset during compute reset */
+			/* We allow scheduling of a hard reset only during a compute reset */
 			if (hard_reset && hdev->reset_info.in_compute_reset)
 				hdev->reset_info.hard_reset_schedule_flags = flags;
 			spin_unlock(&hdev->reset_info.lock);
@@ -1505,15 +1543,17 @@ do_reset:
 
 		/* Cancel the device release watchdog work if required.
 		 * In case of reset-upon-device-release while the release watchdog work is
-		 * scheduled, do hard-reset instead of compute-reset.
+		 * scheduled due to a hard-reset, do hard-reset instead of compute-reset.
 		 */
 		if ((hard_reset || from_dev_release) && hdev->reset_info.watchdog_active) {
+			struct hl_device_reset_work *watchdog_work =
+					&hdev->device_release_watchdog_work;
+
 			hdev->reset_info.watchdog_active = 0;
 			if (!from_watchdog_thread)
-				cancel_delayed_work_sync(
-						&hdev->device_release_watchdog_work.reset_work);
+				cancel_delayed_work_sync(&watchdog_work->reset_work);
 
-			if (from_dev_release) {
+			if (from_dev_release && (watchdog_work->flags & HL_DRV_RESET_HARD)) {
 				hdev->reset_info.in_compute_reset = 0;
 				flags |= HL_DRV_RESET_HARD;
 				flags &= ~HL_DRV_RESET_DEV_RELEASE;
@@ -1524,7 +1564,9 @@ do_reset:
 		if (delay_reset)
 			usleep_range(HL_RESET_DELAY_USEC, HL_RESET_DELAY_USEC << 1);
 
+escalate_reset_flow:
 		handle_reset_trigger(hdev, flags);
+		send_disable_pci_access(hdev, flags);
 
 		/* This also blocks future CS/VM/JOB completion operations */
 		hdev->disabled = true;
@@ -1539,7 +1581,6 @@ do_reset:
 			dev_dbg(hdev->dev, "Going to reset engines of inference device\n");
 	}
 
-again:
 	if ((hard_reset) && (!from_hard_reset_thread)) {
 		hdev->reset_info.hard_reset_pending = true;
 
@@ -1592,7 +1633,7 @@ kill_processes:
 	}
 
 	/* Reset the H/W. It will be in idle state after this returns */
-	hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset);
+	hw_fini_rc = hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset);
 
 	if (hard_reset) {
 		hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
@@ -1619,6 +1660,10 @@ kill_processes:
 		hl_ctx_put(ctx);
 	}
 
+	if (hw_fini_rc) {
+		rc = hw_fini_rc;
+		goto out_err;
+	}
 	/* Finished tear-down, starting to re-initialize */
 
 	if (hard_reset) {
@@ -1784,10 +1829,8 @@ kill_processes:
 			dev_info(hdev->dev, "Performing hard reset scheduled during compute reset\n");
 			flags = hdev->reset_info.hard_reset_schedule_flags;
 			hdev->reset_info.hard_reset_schedule_flags = 0;
-			hdev->disabled = true;
 			hard_reset = true;
-			handle_reset_trigger(hdev, flags);
-			goto again;
+			goto escalate_reset_flow;
 		}
 	}
 
@@ -1804,20 +1847,19 @@ out_err:
 			"%s Failed to reset! Device is NOT usable\n",
 			dev_name(&(hdev)->pdev->dev));
 		hdev->reset_info.hard_reset_cnt++;
-	} else if (reset_upon_device_release) {
-		spin_unlock(&hdev->reset_info.lock);
-		dev_err(hdev->dev, "Failed to reset device after user release\n");
-		flags |= HL_DRV_RESET_HARD;
-		flags &= ~HL_DRV_RESET_DEV_RELEASE;
-		hard_reset = true;
-		goto again;
 	} else {
+		if (reset_upon_device_release) {
+			dev_err(hdev->dev, "Failed to reset device after user release\n");
+			flags &= ~HL_DRV_RESET_DEV_RELEASE;
+		} else {
+			dev_err(hdev->dev, "Failed to do compute reset\n");
+			hdev->reset_info.compute_reset_cnt++;
+		}
+
 		spin_unlock(&hdev->reset_info.lock);
-		dev_err(hdev->dev, "Failed to do compute reset\n");
-		hdev->reset_info.compute_reset_cnt++;
 		flags |= HL_DRV_RESET_HARD;
 		hard_reset = true;
-		goto again;
+		goto escalate_reset_flow;
 	}
 
 	hdev->reset_info.in_reset = 0;
@@ -1840,10 +1882,6 @@ int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask)
 {
 	struct hl_ctx *ctx = NULL;
 
-	/* Device release watchdog is only for hard reset */
-	if (!(flags & HL_DRV_RESET_HARD) && hdev->asic_prop.allow_inference_soft_reset)
-		goto device_reset;
-
 	/* F/W reset cannot be postponed */
 	if (flags & HL_DRV_RESET_BYPASS_REQ_TO_FW)
 		goto device_reset;
@@ -1871,7 +1909,7 @@ int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask)
 		goto out;
 
 	hdev->device_release_watchdog_work.flags = flags;
-	dev_dbg(hdev->dev, "Device is going to be reset in %u sec unless being released\n",
+	dev_dbg(hdev->dev, "Device is going to be hard-reset in %u sec unless being released\n",
 		hdev->device_release_watchdog_timeout_sec);
 	schedule_delayed_work(&hdev->device_release_watchdog_work.reset_work,
 				msecs_to_jiffies(hdev->device_release_watchdog_timeout_sec * 1000));
@@ -1939,37 +1977,27 @@ void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask)
 	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
 }
 
-/*
- * hl_device_init - main initialization function for habanalabs device
- *
- * @hdev: pointer to habanalabs device structure
- *
- * Allocate an id for the device, do early initialization and then call the
- * ASIC specific initialization functions. Finally, create the cdev and the
- * Linux device to expose it to the user
- */
-int hl_device_init(struct hl_device *hdev, struct class *hclass)
+static int create_cdev(struct hl_device *hdev)
 {
-	int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt;
 	char *name;
-	bool add_cdev_sysfs_on_err = false;
+	int rc;
 
 	hdev->cdev_idx = hdev->id / 2;
 
 	name = kasprintf(GFP_KERNEL, "hl%d", hdev->cdev_idx);
 	if (!name) {
 		rc = -ENOMEM;
-		goto out_disabled;
+		goto out_err;
 	}
 
 	/* Initialize cdev and device structures */
-	rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name,
+	rc = device_init_cdev(hdev, hdev->hclass, hdev->id, &hl_ops, name,
 				&hdev->cdev, &hdev->dev);
 
 	kfree(name);
 
 	if (rc)
-		goto out_disabled;
+		goto out_err;
 
 	name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->cdev_idx);
 	if (!name) {
@@ -1978,7 +2006,7 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass)
 	}
 
 	/* Initialize cdev and device structures for control device */
-	rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,
+	rc = device_init_cdev(hdev, hdev->hclass, hdev->id_control, &hl_ctrl_ops,
 				name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
 
 	kfree(name);
@@ -1986,10 +2014,36 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass)
 	if (rc)
 		goto free_dev;
 
+	return 0;
+
+free_dev:
+	put_device(hdev->dev);
+out_err:
+	return rc;
+}
+
+/*
+ * hl_device_init - main initialization function for habanalabs device
+ *
+ * @hdev: pointer to habanalabs device structure
+ *
+ * Allocate an id for the device, do early initialization and then call the
+ * ASIC specific initialization functions. Finally, create the cdev and the
+ * Linux device to expose it to the user
+ */
+int hl_device_init(struct hl_device *hdev)
+{
+	int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt;
+	bool add_cdev_sysfs_on_err = false;
+
+	rc = create_cdev(hdev);
+	if (rc)
+		goto out_disabled;
+
 	/* Initialize ASIC function pointers and perform early init */
 	rc = device_early_init(hdev);
 	if (rc)
-		goto free_dev_ctrl;
+		goto free_dev;
 
 	user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count +
 				hdev->asic_prop.user_interrupt_count;
@@ -2241,9 +2295,8 @@ free_usr_intr_mem:
 	kfree(hdev->user_interrupt);
 early_fini:
 	device_early_fini(hdev);
-free_dev_ctrl:
-	put_device(hdev->dev_ctrl);
 free_dev:
+	put_device(hdev->dev_ctrl);
 	put_device(hdev->dev);
 out_disabled:
 	hdev->disabled = true;
@@ -2364,7 +2417,9 @@ void hl_device_fini(struct hl_device *hdev)
 	hl_cb_pool_fini(hdev);
 
 	/* Reset the H/W. It will be in idle state after this returns */
-	hdev->asic_funcs->hw_fini(hdev, true, false);
+	rc = hdev->asic_funcs->hw_fini(hdev, true, false);
+	if (rc)
+		dev_err(hdev->dev, "hw_fini failed in device fini while removing device %d\n", rc);
 
 	hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
 
@@ -2566,3 +2621,49 @@ void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_
 	if (event_mask)
 		*event_mask |=  HL_NOTIFIER_EVENT_PAGE_FAULT;
 }
+
+static void hl_capture_hw_err(struct hl_device *hdev, u16 event_id)
+{
+	struct hw_err_info *info = &hdev->captured_err_info.hw_err;
+
+	/* Capture only the first HW err */
+	if (atomic_cmpxchg(&info->event_detected, 0, 1))
+		return;
+
+	info->event.timestamp = ktime_to_ns(ktime_get());
+	info->event.event_id = event_id;
+
+	info->event_info_available = true;
+}
+
+void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask)
+{
+	hl_capture_hw_err(hdev, event_id);
+
+	if (event_mask)
+		*event_mask |= HL_NOTIFIER_EVENT_CRITICL_HW_ERR;
+}
+
+static void hl_capture_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *fw_info)
+{
+	struct fw_err_info *info = &hdev->captured_err_info.fw_err;
+
+	/* Capture only the first FW error */
+	if (atomic_cmpxchg(&info->event_detected, 0, 1))
+		return;
+
+	info->event.timestamp = ktime_to_ns(ktime_get());
+	info->event.err_type = fw_info->err_type;
+	if (fw_info->err_type == HL_INFO_FW_REPORTED_ERR)
+		info->event.event_id = fw_info->event_id;
+
+	info->event_info_available = true;
+}
+
+void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info)
+{
+	hl_capture_fw_err(hdev, info);
+
+	if (info->event_mask)
+		*info->event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR;
+}
diff --git a/drivers/accel/habanalabs/common/firmware_if.c b/drivers/accel/habanalabs/common/firmware_if.c
index da892d8fb3d6..59f61ec66445 100644
--- a/drivers/accel/habanalabs/common/firmware_if.c
+++ b/drivers/accel/habanalabs/common/firmware_if.c
@@ -71,7 +71,7 @@ free_fw_ver:
 	return NULL;
 }
 
-static int extract_fw_sub_versions(struct hl_device *hdev, char *preboot_ver)
+static int hl_get_preboot_major_minor(struct hl_device *hdev, char *preboot_ver)
 {
 	char major[8], minor[8], *first_dot, *second_dot;
 	int rc;
@@ -86,7 +86,7 @@ static int extract_fw_sub_versions(struct hl_device *hdev, char *preboot_ver)
 
 	if (rc) {
 		dev_err(hdev->dev, "Error %d parsing preboot major version\n", rc);
-		goto out;
+		return rc;
 	}
 
 	/* skip the first dot */
@@ -102,9 +102,6 @@ static int extract_fw_sub_versions(struct hl_device *hdev, char *preboot_ver)
 
 	if (rc)
 		dev_err(hdev->dev, "Error %d parsing preboot minor version\n", rc);
-
-out:
-	kfree(preboot_ver);
 	return rc;
 }
 
@@ -1263,7 +1260,7 @@ void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
 				COMMS_RST_DEV, 0, false,
 				hdev->fw_loader.cpu_timeout);
 		if (rc)
-			dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n");
+			dev_err(hdev->dev, "Failed sending COMMS_RST_DEV\n");
 	} else {
 		WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
 	}
@@ -1281,10 +1278,10 @@ void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
 	/* Stop device CPU to make sure nothing bad happens */
 	if (hdev->asic_prop.dynamic_fw_load) {
 		rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
-				COMMS_GOTO_WFE, 0, true,
+				COMMS_GOTO_WFE, 0, false,
 				hdev->fw_loader.cpu_timeout);
 		if (rc)
-			dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
+			dev_err(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
 	} else {
 		WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
 		msleep(static_loader->cpu_reset_wait_msec);
@@ -2181,8 +2178,8 @@ static int hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
 
 			dev_info(hdev->dev, "preboot version %s\n", preboot_ver);
 
-			/* This function takes care of freeing preboot_ver */
-			rc = extract_fw_sub_versions(hdev, preboot_ver);
+			rc = hl_get_preboot_major_minor(hdev, preboot_ver);
+			kfree(preboot_ver);
 			if (rc)
 				return rc;
 		}
@@ -3152,7 +3149,7 @@ int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_in
 int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode,
 						dma_addr_t buff, u32 *size)
 {
-	struct cpucp_packet pkt = {0};
+	struct cpucp_packet pkt = {};
 	u64 result;
 	int rc = 0;
 
diff --git a/drivers/accel/habanalabs/common/habanalabs.h b/drivers/accel/habanalabs/common/habanalabs.h
index fa05e76d3d21..eaae69a9f817 100644
--- a/drivers/accel/habanalabs/common/habanalabs.h
+++ b/drivers/accel/habanalabs/common/habanalabs.h
@@ -155,18 +155,12 @@ enum hl_mmu_enablement {
 #define hl_asic_dma_alloc_coherent(hdev, size, dma_handle, flags) \
 	hl_asic_dma_alloc_coherent_caller(hdev, size, dma_handle, flags, __func__)
 
-#define hl_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle) \
-	hl_cpu_accessible_dma_pool_alloc_caller(hdev, size, dma_handle, __func__)
-
 #define hl_asic_dma_pool_zalloc(hdev, size, mem_flags, dma_handle) \
 	hl_asic_dma_pool_zalloc_caller(hdev, size, mem_flags, dma_handle, __func__)
 
 #define hl_asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle) \
 	hl_asic_dma_free_coherent_caller(hdev, size, cpu_addr, dma_handle, __func__)
 
-#define hl_cpu_accessible_dma_pool_free(hdev, size, vaddr) \
-	hl_cpu_accessible_dma_pool_free_caller(hdev, size, vaddr, __func__)
-
 #define hl_asic_dma_pool_free(hdev, vaddr, dma_addr) \
 	hl_asic_dma_pool_free_caller(hdev, vaddr, dma_addr, __func__)
 
@@ -378,6 +372,7 @@ enum hl_cs_type {
 	CS_RESERVE_SIGNALS,
 	CS_UNRESERVE_SIGNALS,
 	CS_TYPE_ENGINE_CORE,
+	CS_TYPE_ENGINES,
 	CS_TYPE_FLUSH_PCI_HBW_WRITES,
 };
 
@@ -592,6 +587,8 @@ struct hl_hints_range {
  * @host_base_address: host physical start address for host DMA from device
  * @host_end_address: host physical end address for host DMA from device
  * @max_freq_value: current max clk frequency.
+ * @engine_core_interrupt_reg_addr: interrupt register address for engine core to use
+ *                                  in order to raise events toward FW.
  * @clk_pll_index: clock PLL index that specify which PLL determines the clock
  *                 we display to the user
  * @mmu_pgt_size: MMU page tables total size.
@@ -612,8 +609,8 @@ struct hl_hints_range {
  * @cb_pool_cb_cnt: number of CBs in the CB pool.
  * @cb_pool_cb_size: size of each CB in the CB pool.
  * @decoder_enabled_mask: which decoders are enabled.
- * @decoder_binning_mask: which decoders are binned, 0 means usable and 1
- *                        means binned (at most one binned decoder per dcore).
+ * @decoder_binning_mask: which decoders are binned, 0 means usable and 1 means binned.
+ * @rotator_enabled_mask: which rotators are enabled.
  * @edma_enabled_mask: which EDMAs are enabled.
  * @edma_binning_mask: which EDMAs are binned, 0 means usable and 1 means
  *                     binned (at most one binned DMA).
@@ -648,7 +645,8 @@ struct hl_hints_range {
  *                                      which the property supports_user_set_page_size is true
  *                                      (i.e. the DRAM supports multiple page sizes), otherwise
  *                                      it will shall  be equal to dram_page_size.
- * @num_engine_cores: number of engine cpu cores
+ * @num_engine_cores: number of engine cpu cores.
+ * @max_num_of_engines: maximum number of all engines in the ASIC.
  * @num_of_special_blocks: special_blocks array size.
  * @glbl_err_cause_num: global err cause number.
  * @hbw_flush_reg: register to read to generate HBW flush. value of 0 means HBW flush is
@@ -663,6 +661,8 @@ struct hl_hints_range {
  * @first_available_cq: first available CQ for the user.
  * @user_interrupt_count: number of user interrupts.
  * @user_dec_intr_count: number of decoder interrupts exposed to user.
+ * @tpc_interrupt_id: interrupt id for TPC to use in order to raise events towards the host.
+ * @eq_interrupt_id: interrupt id for EQ, uses to synchronize EQ interrupts in hard-reset.
  * @cache_line_size: device cache line size.
  * @server_type: Server type that the ASIC is currently installed in.
  *               The value is according to enum hl_server_type in uapi file.
@@ -698,6 +698,7 @@ struct hl_hints_range {
  * @supports_user_set_page_size: true if user can set the allocation page size.
  * @dma_mask: the dma mask to be set for this device
  * @supports_advanced_cpucp_rc: true if new cpucp opcodes are supported.
+ * @supports_engine_modes: true if changing engines/engine_cores modes is supported.
  */
 struct asic_fixed_properties {
 	struct hw_queue_properties	*hw_queues_props;
@@ -739,6 +740,7 @@ struct asic_fixed_properties {
 	u64				host_base_address;
 	u64				host_end_address;
 	u64				max_freq_value;
+	u64				engine_core_interrupt_reg_addr;
 	u32				clk_pll_index;
 	u32				mmu_pgt_size;
 	u32				mmu_pte_size;
@@ -759,6 +761,7 @@ struct asic_fixed_properties {
 	u32				cb_pool_cb_size;
 	u32				decoder_enabled_mask;
 	u32				decoder_binning_mask;
+	u32				rotator_enabled_mask;
 	u32				edma_enabled_mask;
 	u32				edma_binning_mask;
 	u32				max_pending_cs;
@@ -775,6 +778,7 @@ struct asic_fixed_properties {
 	u32				xbar_edge_enabled_mask;
 	u32				device_mem_alloc_default_page_size;
 	u32				num_engine_cores;
+	u32				max_num_of_engines;
 	u32				num_of_special_blocks;
 	u32				glbl_err_cause_num;
 	u32				hbw_flush_reg;
@@ -788,6 +792,8 @@ struct asic_fixed_properties {
 	u16				first_available_cq[HL_MAX_DCORES];
 	u16				user_interrupt_count;
 	u16				user_dec_intr_count;
+	u16				tpc_interrupt_id;
+	u16				eq_interrupt_id;
 	u16				cache_line_size;
 	u16				server_type;
 	u8				completion_queues_count;
@@ -811,6 +817,7 @@ struct asic_fixed_properties {
 	u8				supports_user_set_page_size;
 	u8				dma_mask;
 	u8				supports_advanced_cpucp_rc;
+	u8				supports_engine_modes;
 };
 
 /**
@@ -1096,6 +1103,8 @@ struct hl_cq {
 enum hl_user_interrupt_type {
 	HL_USR_INTERRUPT_CQ = 0,
 	HL_USR_INTERRUPT_DECODER,
+	HL_USR_INTERRUPT_TPC,
+	HL_USR_INTERRUPT_UNEXPECTED
 };
 
 /**
@@ -1104,6 +1113,7 @@ enum hl_user_interrupt_type {
  * @type: user interrupt type
  * @wait_list_head: head to the list of user threads pending on this interrupt
  * @wait_list_lock: protects wait_list_head
+ * @timestamp: last timestamp taken upon interrupt
  * @interrupt_id: msix interrupt id
  */
 struct hl_user_interrupt {
@@ -1111,6 +1121,7 @@ struct hl_user_interrupt {
 	enum hl_user_interrupt_type	type;
 	struct list_head		wait_list_head;
 	spinlock_t			wait_list_lock;
+	ktime_t				timestamp;
 	u32				interrupt_id;
 };
 
@@ -1200,15 +1211,15 @@ struct hl_eq {
 /**
  * struct hl_dec - describes a decoder sw instance.
  * @hdev: pointer to the device structure.
- * @completion_abnrm_work: workqueue object to run when decoder generates an error interrupt
+ * @abnrm_intr_work: workqueue work item to run when decoder generates an error interrupt.
  * @core_id: ID of the decoder.
  * @base_addr: base address of the decoder.
  */
 struct hl_dec {
-	struct hl_device		*hdev;
-	struct work_struct		completion_abnrm_work;
-	u32				core_id;
-	u32				base_addr;
+	struct hl_device	*hdev;
+	struct work_struct	abnrm_intr_work;
+	u32			core_id;
+	u32			base_addr;
 };
 
 /**
@@ -1562,6 +1573,7 @@ struct engines_data {
  * @access_dev_mem: access device memory
  * @set_dram_bar_base: set the base of the DRAM BAR
  * @set_engine_cores: set a config command to engine cores
+ * @set_engines: set a config command to user engines
  * @send_device_activity: indication to FW about device availability
  * @set_dram_properties: set DRAM related properties.
  * @set_binning_masks: set binning/enable masks for all relevant components.
@@ -1574,7 +1586,7 @@ struct hl_asic_funcs {
 	int (*sw_init)(struct hl_device *hdev);
 	int (*sw_fini)(struct hl_device *hdev);
 	int (*hw_init)(struct hl_device *hdev);
-	void (*hw_fini)(struct hl_device *hdev, bool hard_reset, bool fw_reset);
+	int (*hw_fini)(struct hl_device *hdev, bool hard_reset, bool fw_reset);
 	void (*halt_engines)(struct hl_device *hdev, bool hard_reset, bool fw_reset);
 	int (*suspend)(struct hl_device *hdev);
 	int (*resume)(struct hl_device *hdev);
@@ -1701,6 +1713,8 @@ struct hl_asic_funcs {
 	u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr);
 	int (*set_engine_cores)(struct hl_device *hdev, u32 *core_ids,
 					u32 num_cores, u32 core_command);
+	int (*set_engines)(struct hl_device *hdev, u32 *engine_ids,
+					u32 num_engines, u32 engine_command);
 	int (*send_device_activity)(struct hl_device *hdev, bool open);
 	int (*set_dram_properties)(struct hl_device *hdev);
 	int (*set_binning_masks)(struct hl_device *hdev);
@@ -1824,7 +1838,7 @@ struct hl_cs_outcome_store {
  * @hpriv: pointer to the private (Kernel Driver) data of the process (fd).
  * @hdev: pointer to the device structure.
  * @refcount: reference counter for the context. Context is released only when
- *		this hits 0l. It is incremented on CS and CS_WAIT.
+ *		this hits 0. It is incremented on CS and CS_WAIT.
  * @cs_pending: array of hl fence objects representing pending CS.
  * @outcome_store: storage data structure used to remember outcomes of completed
  *                 command submissions for a long time after CS id wraparound.
@@ -2318,7 +2332,7 @@ struct hl_debugfs_entry {
  * @userptr_list: list of available userptrs (virtual memory chunk descriptor).
  * @userptr_spinlock: protects userptr_list.
  * @ctx_mem_hash_list: list of available contexts with MMU mappings.
- * @ctx_mem_hash_spinlock: protects cb_list.
+ * @ctx_mem_hash_mutex: protects list of available contexts with MMU mappings.
  * @data_dma_blob_desc: data DMA descriptor of blob.
  * @mon_dump_blob_desc: monitor dump descriptor of blob.
  * @state_dump: data of the system states in case of a bad cs.
@@ -2349,7 +2363,7 @@ struct hl_dbg_device_entry {
 	struct list_head		userptr_list;
 	spinlock_t			userptr_spinlock;
 	struct list_head		ctx_mem_hash_list;
-	spinlock_t			ctx_mem_hash_spinlock;
+	struct mutex			ctx_mem_hash_mutex;
 	struct debugfs_blob_wrapper	data_dma_blob_desc;
 	struct debugfs_blob_wrapper	mon_dump_blob_desc;
 	char				*state_dump[HL_STATE_DUMP_HIST_LEN];
@@ -2974,8 +2988,8 @@ struct cs_timeout_info {
  * @cq_addr: the address of the current handled command buffer
  * @cq_size: the size of the current handled command buffer
  * @cb_addr_streams_len: num of streams - actual len of cb_addr_streams array.
- *                       should be equal to 1 incase of undefined opcode
- *                       in Upper-CP (specific stream) and equal to 4 incase
+ *                       should be equal to 1 in case of undefined opcode
+ *                       in Upper-CP (specific stream) and equal to 4 in case
  *                       of undefined opcode in Lower-CP.
  * @engine_id: engine-id that the error occurred on
  * @stream_id: the stream id the error occurred on. In case the stream equals to
@@ -3032,17 +3046,55 @@ struct razwi_info {
 };
 
 /**
+ * struct hw_err_info - HW error information.
+ * @event: holds information on the event.
+ * @event_detected: if set as 1, then a HW event was discovered for the
+ *                  first time after the driver has finished booting-up.
+ *                  currently we assume that only fatal events (that require hard-reset) are
+ *                  reported so we don't care of the others that might follow it.
+ *                  so once changed to 1, it will remain that way.
+ *                  TODO: support multiple events.
+ * @event_info_available: indicates that a HW event info is now available.
+ */
+struct hw_err_info {
+	struct hl_info_hw_err_event	event;
+	atomic_t			event_detected;
+	bool				event_info_available;
+};
+
+/**
+ * struct fw_err_info - FW error information.
+ * @event: holds information on the event.
+ * @event_detected: if set as 1, then a FW event was discovered for the
+ *                  first time after the driver has finished booting-up.
+ *                  currently we assume that only fatal events (that require hard-reset) are
+ *                  reported so we don't care of the others that might follow it.
+ *                  so once changed to 1, it will remain that way.
+ *                  TODO: support multiple events.
+ * @event_info_available: indicates that a HW event info is now available.
+ */
+struct fw_err_info {
+	struct hl_info_fw_err_event	event;
+	atomic_t			event_detected;
+	bool				event_info_available;
+};
+
+/**
  * struct hl_error_info - holds information collected during an error.
  * @cs_timeout: CS timeout error information.
  * @razwi_info: RAZWI information.
  * @undef_opcode: undefined opcode information.
  * @page_fault_info: page fault information.
+ * @hw_err: (fatal) hardware error information.
+ * @fw_err: firmware error information.
  */
 struct hl_error_info {
 	struct cs_timeout_info		cs_timeout;
 	struct razwi_info		razwi_info;
 	struct undefined_opcode_info	undef_opcode;
 	struct page_fault_info		page_fault_info;
+	struct hw_err_info		hw_err;
+	struct fw_err_info		fw_err;
 };
 
 /**
@@ -3090,6 +3142,7 @@ struct hl_reset_info {
  *		   (required only for PCI address match mode)
  * @pcie_bar: array of available PCIe bars virtual addresses.
  * @rmmio: configuration area address on SRAM.
+ * @hclass: pointer to the habanalabs class.
  * @cdev: related char device.
  * @cdev_ctrl: char device for control operations only (INFO IOCTL)
  * @dev: related kernel basic device structure.
@@ -3104,6 +3157,8 @@ struct hl_reset_info {
  * @user_interrupt: array of hl_user_interrupt. upon the corresponding user
  *                  interrupt, driver will monitor the list of fences
  *                  registered to this interrupt.
+ * @tpc_interrupt: single TPC interrupt for all TPCs.
+ * @unexpected_error_interrupt: single interrupt for unexpected user error indication.
  * @common_user_cq_interrupt: common user CQ interrupt for all user CQ interrupts.
  *                         upon any user CQ interrupt, driver will monitor the
  *                         list of fences registered to this common structure.
@@ -3199,6 +3254,7 @@ struct hl_reset_info {
  *                drams are binned-out
  * @tpc_binning: contains mask of tpc engines that is received from the f/w which indicates which
  *               tpc engines are binned-out
+ * @dmabuf_export_cnt: number of dma-buf exporting.
  * @card_type: Various ASICs have several card types. This indicates the card
  *             type of the current device.
  * @major: habanalabs kernel driver major.
@@ -3253,6 +3309,8 @@ struct hl_reset_info {
  * @supports_mmu_prefetch: true if prefetch is supported, otherwise false.
  * @reset_upon_device_release: reset the device when the user closes the file descriptor of the
  *                             device.
+ * @supports_ctx_switch: true if a ctx switch is required upon first submission.
+ * @support_preboot_binning: true if we support read binning info from preboot.
  * @nic_ports_mask: Controls which NIC ports are enabled. Used only for testing.
  * @fw_components: Controls which f/w components to load to the device. There are multiple f/w
  *                 stages and sometimes we want to stop at a certain stage. Used only for testing.
@@ -3266,14 +3324,13 @@ struct hl_reset_info {
  *                         Used only for testing.
  * @heartbeat: Controls if we want to enable the heartbeat mechanism vs. the f/w, which verifies
  *             that the f/w is always alive. Used only for testing.
- * @supports_ctx_switch: true if a ctx switch is required upon first submission.
- * @support_preboot_binning: true if we support read binning info from preboot.
  */
 struct hl_device {
 	struct pci_dev			*pdev;
 	u64				pcie_bar_phys[HL_PCI_NUM_BARS];
 	void __iomem			*pcie_bar[HL_PCI_NUM_BARS];
 	void __iomem			*rmmio;
+	struct class			*hclass;
 	struct cdev			cdev;
 	struct cdev			cdev_ctrl;
 	struct device			*dev;
@@ -3286,6 +3343,8 @@ struct hl_device {
 	enum hl_asic_type		asic_type;
 	struct hl_cq			*completion_queue;
 	struct hl_user_interrupt	*user_interrupt;
+	struct hl_user_interrupt	tpc_interrupt;
+	struct hl_user_interrupt	unexpected_error_interrupt;
 	struct hl_user_interrupt	common_user_cq_interrupt;
 	struct hl_user_interrupt	common_decoder_interrupt;
 	struct hl_cs			**shadow_cs_queue;
@@ -3369,7 +3428,7 @@ struct hl_device {
 	u64				fw_comms_poll_interval_usec;
 	u64				dram_binning;
 	u64				tpc_binning;
-
+	atomic_t			dmabuf_export_cnt;
 	enum cpucp_card_types		card_type;
 	u32				major;
 	u32				high_pll;
@@ -3412,7 +3471,7 @@ struct hl_device {
 	u8				supports_ctx_switch;
 	u8				support_preboot_binning;
 
-	/* Parameters for bring-up */
+	/* Parameters for bring-up to be upstreamed */
 	u64				nic_ports_mask;
 	u64				fw_components;
 	u8				mmu_enable;
@@ -3450,6 +3509,20 @@ struct hl_cs_encaps_sig_handle {
 	u32  count;
 };
 
+/**
+ * struct hl_info_fw_err_info - firmware error information structure
+ * @err_type: The type of error detected (or reported).
+ * @event_mask: Pointer to the event mask to be modified with the detected error flag
+ *              (can be NULL)
+ * @event_id: The id of the event that reported the error
+ *            (applicable when err_type is HL_INFO_FW_REPORTED_ERR).
+ */
+struct hl_info_fw_err_info {
+	enum hl_info_fw_err_type err_type;
+	u64 *event_mask;
+	u16 event_id;
+};
+
 /*
  * IOCTLs
  */
@@ -3474,6 +3547,10 @@ struct hl_ioctl_desc {
 	hl_ioctl_t *func;
 };
 
+static inline bool hl_is_fw_ver_below_1_9(struct hl_device *hdev)
+{
+	return (hdev->fw_major_version < 42);
+}
 
 /*
  * Kernel module functions that can be accessed by entire module
@@ -3537,14 +3614,12 @@ static inline bool hl_mem_area_crosses_range(u64 address, u32 size,
 }
 
 uint64_t hl_set_dram_bar_default(struct hl_device *hdev, u64 addr);
+void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle);
+void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr);
 void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
 					gfp_t flag, const char *caller);
 void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr,
 					dma_addr_t dma_handle, const char *caller);
-void *hl_cpu_accessible_dma_pool_alloc_caller(struct hl_device *hdev, size_t size,
-						dma_addr_t *dma_handle, const char *caller);
-void hl_cpu_accessible_dma_pool_free_caller(struct hl_device *hdev, size_t size, void *vaddr,
-						const char *caller);
 void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags,
 					dma_addr_t *dma_handle, const char *caller);
 void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr,
@@ -3591,7 +3666,7 @@ irqreturn_t hl_irq_handler_cq(int irq, void *arg);
 irqreturn_t hl_irq_handler_eq(int irq, void *arg);
 irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg);
 irqreturn_t hl_irq_handler_user_interrupt(int irq, void *arg);
-irqreturn_t hl_irq_handler_default(int irq, void *arg);
+irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg);
 u32 hl_cq_inc_ptr(u32 ptr);
 
 int hl_asid_init(struct hl_device *hdev);
@@ -3612,7 +3687,7 @@ int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr,
 void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
 void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);
 
-int hl_device_init(struct hl_device *hdev, struct class *hclass);
+int hl_device_init(struct hl_device *hdev);
 void hl_device_fini(struct hl_device *hdev);
 int hl_device_suspend(struct hl_device *hdev);
 int hl_device_resume(struct hl_device *hdev);
@@ -3662,6 +3737,7 @@ bool cs_needs_timeout(struct hl_cs *cs);
 bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs);
 struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq);
 void hl_multi_cs_completion_init(struct hl_device *hdev);
+u32 hl_get_active_cs_num(struct hl_device *hdev);
 
 void goya_set_asic_funcs(struct hl_device *hdev);
 void gaudi_set_asic_funcs(struct hl_device *hdev);
@@ -3861,6 +3937,7 @@ const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type);
 
 void hl_mem_mgr_init(struct device *dev, struct hl_mem_mgr *mmg);
 void hl_mem_mgr_fini(struct hl_mem_mgr *mmg);
+void hl_mem_mgr_idr_destroy(struct hl_mem_mgr *mmg);
 int hl_mem_mgr_mmap(struct hl_mem_mgr *mmg, struct vm_area_struct *vma,
 		    void *args);
 struct hl_mmap_mem_buf *hl_mmap_mem_buf_get(struct hl_mem_mgr *mmg,
@@ -3879,6 +3956,8 @@ void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_o
 void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu);
 void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu,
 				u64 *event_mask);
+void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask);
+void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info);
 
 #ifdef CONFIG_DEBUG_FS
 
diff --git a/drivers/accel/habanalabs/common/habanalabs_drv.c b/drivers/accel/habanalabs/common/habanalabs_drv.c
index 03dae57dc838..d9df64e75f33 100644
--- a/drivers/accel/habanalabs/common/habanalabs_drv.c
+++ b/drivers/accel/habanalabs/common/habanalabs_drv.c
@@ -12,7 +12,6 @@
 #include "../include/hw_ip/pci/pci_general.h"
 
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/module.h>
 
 #define CREATE_TRACE_POINTS
@@ -221,12 +220,9 @@ int hl_device_open(struct inode *inode, struct file *filp)
 
 	hl_debugfs_add_file(hpriv);
 
+	memset(&hdev->captured_err_info, 0, sizeof(hdev->captured_err_info));
 	atomic_set(&hdev->captured_err_info.cs_timeout.write_enable, 1);
-	atomic_set(&hdev->captured_err_info.razwi_info.razwi_detected, 0);
-	atomic_set(&hdev->captured_err_info.page_fault_info.page_fault_detected, 0);
 	hdev->captured_err_info.undef_opcode.write_enable = true;
-	hdev->captured_err_info.razwi_info.razwi_info_available = false;
-	hdev->captured_err_info.page_fault_info.page_fault_info_available = false;
 
 	hdev->open_counter++;
 	hdev->last_successful_open_jif = jiffies;
@@ -237,6 +233,7 @@ int hl_device_open(struct inode *inode, struct file *filp)
 out_err:
 	mutex_unlock(&hdev->fpriv_list_lock);
 	hl_mem_mgr_fini(&hpriv->mem_mgr);
+	hl_mem_mgr_idr_destroy(&hpriv->mem_mgr);
 	hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
 	filp->private_data = NULL;
 	mutex_destroy(&hpriv->ctx_lock);
@@ -324,6 +321,7 @@ static void copy_kernel_module_params_to_device(struct hl_device *hdev)
 	hdev->asic_prop.fw_security_enabled = is_asic_secured(hdev->asic_type);
 
 	hdev->major = hl_major;
+	hdev->hclass = hl_class;
 	hdev->memory_scrub = memory_scrub;
 	hdev->reset_on_lockup = reset_on_lockup;
 	hdev->boot_error_status_mask = boot_error_status_mask;
@@ -550,9 +548,7 @@ static int hl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	pci_set_drvdata(pdev, hdev);
 
-	pci_enable_pcie_error_reporting(pdev);
-
-	rc = hl_device_init(hdev, hl_class);
+	rc = hl_device_init(hdev);
 	if (rc) {
 		dev_err(&pdev->dev, "Fatal error during habanalabs device init\n");
 		rc = -ENODEV;
@@ -562,7 +558,6 @@ static int hl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	return 0;
 
 disable_device:
-	pci_disable_pcie_error_reporting(pdev);
 	pci_set_drvdata(pdev, NULL);
 	destroy_hdev(hdev);
 
@@ -585,7 +580,6 @@ static void hl_pci_remove(struct pci_dev *pdev)
 		return;
 
 	hl_device_fini(hdev);
-	pci_disable_pcie_error_reporting(pdev);
 	pci_set_drvdata(pdev, NULL);
 	destroy_hdev(hdev);
 }
@@ -702,7 +696,7 @@ static int __init hl_init(void)
 
 	hl_major = MAJOR(dev);
 
-	hl_class = class_create(THIS_MODULE, HL_NAME);
+	hl_class = class_create(HL_NAME);
 	if (IS_ERR(hl_class)) {
 		pr_err("failed to allocate class\n");
 		rc = PTR_ERR(hl_class);
diff --git a/drivers/accel/habanalabs/common/habanalabs_ioctl.c b/drivers/accel/habanalabs/common/habanalabs_ioctl.c
index 5005e6fca691..203ee857810c 100644
--- a/drivers/accel/habanalabs/common/habanalabs_ioctl.c
+++ b/drivers/accel/habanalabs/common/habanalabs_ioctl.c
@@ -102,11 +102,15 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
 	hw_ip.mme_master_slave_mode = prop->mme_master_slave_mode;
 	hw_ip.first_available_interrupt_id = prop->first_available_user_interrupt;
 	hw_ip.number_of_user_interrupts = prop->user_interrupt_count;
+	hw_ip.tpc_interrupt_id = prop->tpc_interrupt_id;
 
 	hw_ip.edma_enabled_mask = prop->edma_enabled_mask;
 	hw_ip.server_type = prop->server_type;
 	hw_ip.security_enabled = prop->fw_security_enabled;
 	hw_ip.revision_id = hdev->pdev->revision;
+	hw_ip.rotator_enabled_mask = prop->rotator_enabled_mask;
+	hw_ip.engine_core_interrupt_reg_addr = prop->engine_core_interrupt_reg_addr;
+	hw_ip.reserved_dram_size = dram_kmd_size;
 
 	return copy_to_user(out, &hw_ip,
 		min((size_t) size, sizeof(hw_ip))) ? -EFAULT : 0;
@@ -830,6 +834,50 @@ static int user_mappings_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 	return copy_to_user(out, pgf_info->user_mappings, actual_size) ? -EFAULT : 0;
 }
 
+static int hw_err_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
+{
+	void __user *user_buf = (void __user *) (uintptr_t) args->return_pointer;
+	struct hl_device *hdev = hpriv->hdev;
+	u32 user_buf_size = args->return_size;
+	struct hw_err_info *info;
+	int rc;
+
+	if ((!user_buf_size) || (!user_buf))
+		return -EINVAL;
+
+	if (user_buf_size < sizeof(struct hl_info_hw_err_event))
+		return -ENOMEM;
+
+	info = &hdev->captured_err_info.hw_err;
+	if (!info->event_info_available)
+		return -ENOENT;
+
+	rc = copy_to_user(user_buf, &info->event, sizeof(struct hl_info_hw_err_event));
+	return rc ? -EFAULT : 0;
+}
+
+static int fw_err_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
+{
+	void __user *user_buf = (void __user *) (uintptr_t) args->return_pointer;
+	struct hl_device *hdev = hpriv->hdev;
+	u32 user_buf_size = args->return_size;
+	struct fw_err_info *info;
+	int rc;
+
+	if ((!user_buf_size) || (!user_buf))
+		return -EINVAL;
+
+	if (user_buf_size < sizeof(struct hl_info_fw_err_event))
+		return -ENOMEM;
+
+	info = &hdev->captured_err_info.fw_err;
+	if (!info->event_info_available)
+		return -ENOENT;
+
+	rc = copy_to_user(user_buf, &info->event, sizeof(struct hl_info_fw_err_event));
+	return rc ? -EFAULT : 0;
+}
+
 static int send_fw_generic_request(struct hl_device *hdev, struct hl_info_args *info_args)
 {
 	void __user *buff = (void __user *) (uintptr_t) info_args->return_pointer;
@@ -950,6 +998,14 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
 	case HL_INFO_UNREGISTER_EVENTFD:
 		return eventfd_unregister(hpriv, args);
 
+	case HL_INFO_HW_ERR_EVENT:
+		return hw_err_info(hpriv, args);
+
+	case HL_INFO_FW_ERR_EVENT:
+		return fw_err_info(hpriv, args);
+
+	case HL_INFO_DRAM_USAGE:
+		return dram_usage_info(hpriv, args);
 	default:
 		break;
 	}
@@ -962,10 +1018,6 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
 	}
 
 	switch (args->op) {
-	case HL_INFO_DRAM_USAGE:
-		rc = dram_usage_info(hpriv, args);
-		break;
-
 	case HL_INFO_HW_IDLE:
 		rc = hw_idle(hdev, args);
 		break;
diff --git a/drivers/accel/habanalabs/common/hwmon.c b/drivers/accel/habanalabs/common/hwmon.c
index 55eb0203817f..8598056216e7 100644
--- a/drivers/accel/habanalabs/common/hwmon.c
+++ b/drivers/accel/habanalabs/common/hwmon.c
@@ -914,7 +914,7 @@ void hl_hwmon_fini(struct hl_device *hdev)
 
 void hl_hwmon_release_resources(struct hl_device *hdev)
 {
-	const struct hwmon_channel_info **channel_info_arr;
+	const struct hwmon_channel_info * const *channel_info_arr;
 	int i = 0;
 
 	if (!hdev->hl_chip_info->info)
diff --git a/drivers/accel/habanalabs/common/irq.c b/drivers/accel/habanalabs/common/irq.c
index 04844e843a7b..c67895b1cdeb 100644
--- a/drivers/accel/habanalabs/common/irq.c
+++ b/drivers/accel/habanalabs/common/irq.c
@@ -280,7 +280,6 @@ static void handle_user_interrupt(struct hl_device *hdev, struct hl_user_interru
 	struct list_head *ts_reg_free_list_head = NULL;
 	struct timestamp_reg_work_obj *job;
 	bool reg_node_handle_fail = false;
-	ktime_t now = ktime_get();
 	int rc;
 
 	/* For registration nodes:
@@ -303,13 +302,13 @@ static void handle_user_interrupt(struct hl_device *hdev, struct hl_user_interru
 			if (pend->ts_reg_info.buf) {
 				if (!reg_node_handle_fail) {
 					rc = handle_registration_node(hdev, pend,
-								&ts_reg_free_list_head, now);
+							&ts_reg_free_list_head, intr->timestamp);
 					if (rc)
 						reg_node_handle_fail = true;
 				}
 			} else {
 				/* Handle wait target value node */
-				pend->fence.timestamp = now;
+				pend->fence.timestamp = intr->timestamp;
 				complete_all(&pend->fence.completion);
 			}
 		}
@@ -326,6 +325,26 @@ static void handle_user_interrupt(struct hl_device *hdev, struct hl_user_interru
 	}
 }
 
+static void handle_tpc_interrupt(struct hl_device *hdev)
+{
+	u64 event_mask;
+	u32 flags;
+
+	event_mask = HL_NOTIFIER_EVENT_TPC_ASSERT |
+		HL_NOTIFIER_EVENT_USER_ENGINE_ERR |
+		HL_NOTIFIER_EVENT_DEVICE_RESET;
+
+	flags = HL_DRV_RESET_DELAY;
+
+	dev_err_ratelimited(hdev->dev, "Received TPC assert\n");
+	hl_device_cond_reset(hdev, flags, event_mask);
+}
+
+static void handle_unexpected_user_interrupt(struct hl_device *hdev)
+{
+	dev_err_ratelimited(hdev->dev, "Received unexpected user error interrupt\n");
+}
+
 /**
  * hl_irq_handler_user_interrupt - irq handler for user interrupts
  *
@@ -336,6 +355,23 @@ static void handle_user_interrupt(struct hl_device *hdev, struct hl_user_interru
 irqreturn_t hl_irq_handler_user_interrupt(int irq, void *arg)
 {
 	struct hl_user_interrupt *user_int = arg;
+
+	user_int->timestamp = ktime_get();
+
+	return IRQ_WAKE_THREAD;
+}
+
+/**
+ * hl_irq_user_interrupt_thread_handler - irq thread handler for user interrupts.
+ * This function is invoked by threaded irq mechanism
+ *
+ * @irq: irq number
+ * @arg: pointer to user interrupt structure
+ *
+ */
+irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg)
+{
+	struct hl_user_interrupt *user_int = arg;
 	struct hl_device *hdev = user_int->hdev;
 
 	switch (user_int->type) {
@@ -351,6 +387,12 @@ irqreturn_t hl_irq_handler_user_interrupt(int irq, void *arg)
 		/* Handle decoder interrupt registered on this specific irq */
 		handle_user_interrupt(hdev, user_int);
 		break;
+	case HL_USR_INTERRUPT_TPC:
+		handle_tpc_interrupt(hdev);
+		break;
+	case HL_USR_INTERRUPT_UNEXPECTED:
+		handle_unexpected_user_interrupt(hdev);
+		break;
 	default:
 		break;
 	}
@@ -359,24 +401,6 @@ irqreturn_t hl_irq_handler_user_interrupt(int irq, void *arg)
 }
 
 /**
- * hl_irq_handler_default - default irq handler
- *
- * @irq: irq number
- * @arg: pointer to user interrupt structure
- *
- */
-irqreturn_t hl_irq_handler_default(int irq, void *arg)
-{
-	struct hl_user_interrupt *user_interrupt = arg;
-	struct hl_device *hdev = user_interrupt->hdev;
-	u32 interrupt_id = user_interrupt->interrupt_id;
-
-	dev_err(hdev->dev, "got invalid user interrupt %u", interrupt_id);
-
-	return IRQ_HANDLED;
-}
-
-/**
  * hl_irq_handler_eq - irq handler for event queue
  *
  * @irq: irq number
@@ -391,8 +415,8 @@ irqreturn_t hl_irq_handler_eq(int irq, void *arg)
 	struct hl_eq_entry *eq_base;
 	struct hl_eqe_work *handle_eqe_work;
 	bool entry_ready;
-	u32 cur_eqe;
-	u16 cur_eqe_index;
+	u32 cur_eqe, ctl;
+	u16 cur_eqe_index, event_type;
 
 	eq_base = eq->kernel_address;
 
@@ -405,11 +429,10 @@ irqreturn_t hl_irq_handler_eq(int irq, void *arg)
 
 		cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
 		if ((hdev->event_queue.check_eqe_index) &&
-				(((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK)
-							!= cur_eqe_index)) {
+				(((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK) != cur_eqe_index)) {
 			dev_dbg(hdev->dev,
-				"EQE 0x%x in queue is ready but index does not match %d!=%d",
-				eq_base[eq->ci].hdr.ctl,
+				"EQE %#x in queue is ready but index does not match %d!=%d",
+				cur_eqe,
 				((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
 				cur_eqe_index);
 			break;
@@ -426,7 +449,10 @@ irqreturn_t hl_irq_handler_eq(int irq, void *arg)
 		dma_rmb();
 
 		if (hdev->disabled && !hdev->reset_info.in_compute_reset) {
-			dev_warn(hdev->dev, "Device disabled but received an EQ event\n");
+			ctl = le32_to_cpu(eq_entry->hdr.ctl);
+			event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT);
+			dev_warn(hdev->dev,
+				"Device disabled but received an EQ event (%u)\n", event_type);
 			goto skip_irq;
 		}
 
@@ -463,7 +489,7 @@ irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg)
 {
 	struct hl_dec *dec = arg;
 
-	schedule_work(&dec->completion_abnrm_work);
+	schedule_work(&dec->abnrm_intr_work);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/accel/habanalabs/common/memory.c b/drivers/accel/habanalabs/common/memory.c
index 761a47e89b00..a7b6a273ce21 100644
--- a/drivers/accel/habanalabs/common/memory.c
+++ b/drivers/accel/habanalabs/common/memory.c
@@ -235,10 +235,8 @@ static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size,
 	}
 
 	rc = hl_pin_host_memory(hdev, addr, size, userptr);
-	if (rc) {
-		dev_err(hdev->dev, "Failed to pin host memory\n");
+	if (rc)
 		goto pin_err;
-	}
 
 	userptr->dma_mapped = true;
 	userptr->dir = DMA_BIDIRECTIONAL;
@@ -607,6 +605,7 @@ static u64 get_va_block(struct hl_device *hdev,
 	bool is_align_pow_2  = is_power_of_2(va_range->page_size);
 	bool is_hint_dram_addr = hl_is_dram_va(hdev, hint_addr);
 	bool force_hint = flags & HL_MEM_FORCE_HINT;
+	int rc;
 
 	if (is_align_pow_2)
 		align_mask = ~((u64)va_block_align - 1);
@@ -724,9 +723,13 @@ static u64 get_va_block(struct hl_device *hdev,
 		kfree(new_va_block);
 	}
 
-	if (add_prev)
-		add_va_block_locked(hdev, &va_range->list, prev_start,
-				prev_end);
+	if (add_prev) {
+		rc = add_va_block_locked(hdev, &va_range->list, prev_start, prev_end);
+		if (rc) {
+			reserved_valid_start = 0;
+			goto out;
+		}
+	}
 
 	print_va_list_locked(hdev, &va_range->list);
 out:
@@ -1097,10 +1100,8 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, u64 *device
 			huge_page_size = hdev->asic_prop.pmmu_huge.page_size;
 
 		rc = dma_map_host_va(hdev, addr, size, &userptr);
-		if (rc) {
-			dev_err(hdev->dev, "failed to get userptr from va\n");
+		if (rc)
 			return rc;
-		}
 
 		rc = init_phys_pg_pack_from_userptr(ctx, userptr,
 				&phys_pg_pack, false);
@@ -1270,6 +1271,18 @@ init_page_pack_err:
 	return rc;
 }
 
+/* Should be called while the context's mem_hash_lock is taken */
+static struct hl_vm_hash_node *get_vm_hash_node_locked(struct hl_ctx *ctx, u64 vaddr)
+{
+	struct hl_vm_hash_node *hnode;
+
+	hash_for_each_possible(ctx->mem_hash, hnode, node, vaddr)
+		if (vaddr == hnode->vaddr)
+			return hnode;
+
+	return NULL;
+}
+
 /**
  * unmap_device_va() - unmap the given device virtual address.
  * @ctx: pointer to the context structure.
@@ -1285,10 +1298,10 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
 {
 	struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
 	u64 vaddr = args->unmap.device_virt_addr;
-	struct hl_vm_hash_node *hnode = NULL;
 	struct asic_fixed_properties *prop;
 	struct hl_device *hdev = ctx->hdev;
 	struct hl_userptr *userptr = NULL;
+	struct hl_vm_hash_node *hnode;
 	struct hl_va_range *va_range;
 	enum vm_type *vm_type;
 	bool is_userptr;
@@ -1298,15 +1311,10 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
 
 	/* protect from double entrance */
 	mutex_lock(&ctx->mem_hash_lock);
-	hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr)
-		if (vaddr == hnode->vaddr)
-			break;
-
+	hnode = get_vm_hash_node_locked(ctx, vaddr);
 	if (!hnode) {
 		mutex_unlock(&ctx->mem_hash_lock);
-		dev_err(hdev->dev,
-			"unmap failed, no mem hnode for vaddr 0x%llx\n",
-			vaddr);
+		dev_err(hdev->dev, "unmap failed, no mem hnode for vaddr 0x%llx\n", vaddr);
 		return -EINVAL;
 	}
 
@@ -1779,6 +1787,44 @@ static void hl_unmap_dmabuf(struct dma_buf_attachment *attachment,
 	kfree(sgt);
 }
 
+static struct hl_vm_hash_node *memhash_node_export_get(struct hl_ctx *ctx, u64 addr)
+{
+	struct hl_device *hdev = ctx->hdev;
+	struct hl_vm_hash_node *hnode;
+
+	/* get the memory handle */
+	mutex_lock(&ctx->mem_hash_lock);
+	hnode = get_vm_hash_node_locked(ctx, addr);
+	if (!hnode) {
+		mutex_unlock(&ctx->mem_hash_lock);
+		dev_dbg(hdev->dev, "map address %#llx not found\n", addr);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (upper_32_bits(hnode->handle)) {
+		mutex_unlock(&ctx->mem_hash_lock);
+		dev_dbg(hdev->dev, "invalid handle %#llx for map address %#llx\n",
+				hnode->handle, addr);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/*
+	 * node found, increase export count so this memory cannot be unmapped
+	 * and the hash node cannot be deleted.
+	 */
+	hnode->export_cnt++;
+	mutex_unlock(&ctx->mem_hash_lock);
+
+	return hnode;
+}
+
+static void memhash_node_export_put(struct hl_ctx *ctx, struct hl_vm_hash_node *hnode)
+{
+	mutex_lock(&ctx->mem_hash_lock);
+	hnode->export_cnt--;
+	mutex_unlock(&ctx->mem_hash_lock);
+}
+
 static void hl_release_dmabuf(struct dma_buf *dmabuf)
 {
 	struct hl_dmabuf_priv *hl_dmabuf = dmabuf->priv;
@@ -1789,13 +1835,15 @@ static void hl_release_dmabuf(struct dma_buf *dmabuf)
 
 	ctx = hl_dmabuf->ctx;
 
-	if (hl_dmabuf->memhash_hnode) {
-		mutex_lock(&ctx->mem_hash_lock);
-		hl_dmabuf->memhash_hnode->export_cnt--;
-		mutex_unlock(&ctx->mem_hash_lock);
-	}
+	if (hl_dmabuf->memhash_hnode)
+		memhash_node_export_put(ctx, hl_dmabuf->memhash_hnode);
 
+	atomic_dec(&ctx->hdev->dmabuf_export_cnt);
 	hl_ctx_put(ctx);
+
+	/* Paired with get_file() in export_dmabuf() */
+	fput(ctx->hpriv->filp);
+
 	kfree(hl_dmabuf);
 }
 
@@ -1834,6 +1882,13 @@ static int export_dmabuf(struct hl_ctx *ctx,
 
 	hl_dmabuf->ctx = ctx;
 	hl_ctx_get(hl_dmabuf->ctx);
+	atomic_inc(&ctx->hdev->dmabuf_export_cnt);
+
+	/* Get compute device file to enforce release order, such that all exported dma-buf will be
+	 * released first and only then the compute device.
+	 * Paired with fput() in hl_release_dmabuf().
+	 */
+	get_file(ctx->hpriv->filp);
 
 	*dmabuf_fd = fd;
 
@@ -1933,47 +1988,6 @@ static int validate_export_params(struct hl_device *hdev, u64 device_addr, u64 s
 	return 0;
 }
 
-static struct hl_vm_hash_node *memhash_node_export_get(struct hl_ctx *ctx, u64 addr)
-{
-	struct hl_device *hdev = ctx->hdev;
-	struct hl_vm_hash_node *hnode;
-
-	/* get the memory handle */
-	mutex_lock(&ctx->mem_hash_lock);
-	hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)addr)
-		if (addr == hnode->vaddr)
-			break;
-
-	if (!hnode) {
-		mutex_unlock(&ctx->mem_hash_lock);
-		dev_dbg(hdev->dev, "map address %#llx not found\n", addr);
-		return ERR_PTR(-EINVAL);
-	}
-
-	if (upper_32_bits(hnode->handle)) {
-		mutex_unlock(&ctx->mem_hash_lock);
-		dev_dbg(hdev->dev, "invalid handle %#llx for map address %#llx\n",
-				hnode->handle, addr);
-		return ERR_PTR(-EINVAL);
-	}
-
-	/*
-	 * node found, increase export count so this memory cannot be unmapped
-	 * and the hash node cannot be deleted.
-	 */
-	hnode->export_cnt++;
-	mutex_unlock(&ctx->mem_hash_lock);
-
-	return hnode;
-}
-
-static void memhash_node_export_put(struct hl_ctx *ctx, struct hl_vm_hash_node *hnode)
-{
-	mutex_lock(&ctx->mem_hash_lock);
-	hnode->export_cnt--;
-	mutex_unlock(&ctx->mem_hash_lock);
-}
-
 static struct hl_vm_phys_pg_pack *get_phys_pg_pack_from_hash_node(struct hl_device *hdev,
 							struct hl_vm_hash_node *hnode)
 {
@@ -2221,11 +2235,11 @@ static struct hl_mmap_mem_buf_behavior hl_ts_behavior = {
  * allocate_timestamps_buffers() - allocate timestamps buffers
  * This function will allocate ts buffer that will later on be mapped to the user
  * in order to be able to read the timestamp.
- * in additon it'll allocate an extra buffer for registration management.
+ * in addition it'll allocate an extra buffer for registration management.
  * since we cannot fail during registration for out-of-memory situation, so
  * we'll prepare a pool which will be used as user interrupt nodes and instead
  * of dynamically allocating nodes while registration we'll pick the node from
- * this pool. in addtion it'll add node to the mapping hash which will be used
+ * this pool. in addition it'll add node to the mapping hash which will be used
  * to map user ts buffer to the internal kernel ts buffer.
  * @hpriv: pointer to the private data of the fd
  * @args: ioctl input
diff --git a/drivers/accel/habanalabs/common/memory_mgr.c b/drivers/accel/habanalabs/common/memory_mgr.c
index 0f2759e26547..c4d84df355b0 100644
--- a/drivers/accel/habanalabs/common/memory_mgr.c
+++ b/drivers/accel/habanalabs/common/memory_mgr.c
@@ -275,7 +275,7 @@ int hl_mem_mgr_mmap(struct hl_mem_mgr *mmg, struct vm_area_struct *vma,
 
 	if (atomic_cmpxchg(&buf->mmap, 0, 1)) {
 		dev_err(mmg->dev,
-			"%s, Memory mmap failed, already mmaped to user\n",
+			"%s, Memory mmap failed, already mapped to user\n",
 			buf->behavior->topic);
 		rc = -EINVAL;
 		goto put_mem;
@@ -341,8 +341,19 @@ void hl_mem_mgr_fini(struct hl_mem_mgr *mmg)
 				"%s: Buff handle %u for CTX is still alive\n",
 				topic, id);
 	}
+}
 
-	/* TODO: can it happen that some buffer is still in use at this point? */
+/**
+ * hl_mem_mgr_idr_destroy() - destroy memory manager IDR.
+ * @mmg: parent unified memory manager
+ *
+ * Destroy the memory manager IDR.
+ * Shall be called when IDR is empty and no memory buffers are in use.
+ */
+void hl_mem_mgr_idr_destroy(struct hl_mem_mgr *mmg)
+{
+	if (!idr_is_empty(&mmg->handles))
+		dev_crit(mmg->dev, "memory manager IDR is destroyed while it is not empty!\n");
 
 	idr_destroy(&mmg->handles);
 }
diff --git a/drivers/accel/habanalabs/common/mmu/mmu.c b/drivers/accel/habanalabs/common/mmu/mmu.c
index a42ae8bc61e8..f379e5b461a6 100644
--- a/drivers/accel/habanalabs/common/mmu/mmu.c
+++ b/drivers/accel/habanalabs/common/mmu/mmu.c
@@ -540,8 +540,8 @@ static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr,
 		u32 page_off;
 
 		/*
-		 * Bit arithmetics cannot be used for non power of two page
-		 * sizes. In addition, since bit arithmetics is not used,
+		 * Bit arithmetic cannot be used for non power of two page
+		 * sizes. In addition, since bit arithmetic is not used,
 		 * we cannot ignore dram base. All that shall be considered.
 		 */
 
@@ -679,7 +679,9 @@ int hl_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags)
 
 	rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags);
 	if (rc)
-		dev_err_ratelimited(hdev->dev, "MMU cache invalidation failed\n");
+		dev_err_ratelimited(hdev->dev,
+				"%s cache invalidation failed, rc=%d\n",
+				flags == VM_TYPE_USERPTR ? "PMMU" : "HMMU", rc);
 
 	return rc;
 }
@@ -692,7 +694,9 @@ int hl_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard,
 	rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, is_hard, flags,
 								asid, va, size);
 	if (rc)
-		dev_err_ratelimited(hdev->dev, "MMU cache range invalidation failed\n");
+		dev_err_ratelimited(hdev->dev,
+				"%s cache range invalidation failed: va=%#llx, size=%llu, rc=%d",
+				flags == VM_TYPE_USERPTR ? "PMMU" : "HMMU", va, size, rc);
 
 	return rc;
 }
@@ -757,7 +761,7 @@ u64 hl_mmu_get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte)
  * @mmu_prop: MMU properties.
  * @hop_idx: HOP index.
  * @hop_addr: HOP address.
- * @virt_addr: virtual address fro the translation.
+ * @virt_addr: virtual address for the translation.
  *
  * @return the matching PTE value on success, otherwise U64_MAX.
  */
diff --git a/drivers/accel/habanalabs/common/pci/pci.c b/drivers/accel/habanalabs/common/pci/pci.c
index d1f4c695baf2..191e0e3cf3a5 100644
--- a/drivers/accel/habanalabs/common/pci/pci.c
+++ b/drivers/accel/habanalabs/common/pci/pci.c
@@ -420,7 +420,6 @@ int hl_pci_init(struct hl_device *hdev)
 unmap_pci_bars:
 	hl_pci_bars_unmap(hdev);
 disable_device:
-	pci_clear_master(pdev);
 	pci_disable_device(pdev);
 
 	return rc;
@@ -436,6 +435,5 @@ void hl_pci_fini(struct hl_device *hdev)
 {
 	hl_pci_bars_unmap(hdev);
 
-	pci_clear_master(hdev->pdev);
 	pci_disable_device(hdev->pdev);
 }
diff --git a/drivers/accel/habanalabs/common/security.c b/drivers/accel/habanalabs/common/security.c
index 5f03ade07ead..297e6e44fd0c 100644
--- a/drivers/accel/habanalabs/common/security.c
+++ b/drivers/accel/habanalabs/common/security.c
@@ -502,7 +502,7 @@ free_glbl_sec:
 int hl_init_pb_ranges_single_dcore(struct hl_device *hdev, u32 dcore_offset,
 		u32 num_instances, u32 instance_offset,
 		const u32 pb_blocks[], u32 blocks_array_size,
-		const struct range *regs_range_array, u32 regs_range_array_size)
+		const struct range *user_regs_range_array, u32 user_regs_range_array_size)
 {
 	int i;
 	struct hl_block_glbl_sec *glbl_sec;
@@ -514,8 +514,8 @@ int hl_init_pb_ranges_single_dcore(struct hl_device *hdev, u32 dcore_offset,
 		return -ENOMEM;
 
 	hl_secure_block(hdev, glbl_sec, blocks_array_size);
-	hl_unsecure_registers_range(hdev, regs_range_array,
-			regs_range_array_size, 0, pb_blocks, glbl_sec,
+	hl_unsecure_registers_range(hdev, user_regs_range_array,
+			user_regs_range_array_size, 0, pb_blocks, glbl_sec,
 			blocks_array_size);
 
 	/* Fill all blocks with the same configuration */
diff --git a/drivers/accel/habanalabs/common/security.h b/drivers/accel/habanalabs/common/security.h
index 234b4a6ed8bc..d7a3b3e82ea4 100644
--- a/drivers/accel/habanalabs/common/security.h
+++ b/drivers/accel/habanalabs/common/security.h
@@ -10,7 +10,7 @@
 
 #include <linux/io-64-nonatomic-lo-hi.h>
 
-extern struct hl_device *hdev;
+struct hl_device;
 
 /* special blocks */
 #define HL_MAX_NUM_OF_GLBL_ERR_CAUSE		10
diff --git a/drivers/accel/habanalabs/common/sysfs.c b/drivers/accel/habanalabs/common/sysfs.c
index 735d8bed0066..01f89f029355 100644
--- a/drivers/accel/habanalabs/common/sysfs.c
+++ b/drivers/accel/habanalabs/common/sysfs.c
@@ -497,10 +497,14 @@ int hl_sysfs_init(struct hl_device *hdev)
 	if (rc) {
 		dev_err(hdev->dev,
 			"Failed to add groups to device, error %d\n", rc);
-		return rc;
+		goto remove_groups;
 	}
 
 	return 0;
+
+remove_groups:
+	device_remove_groups(hdev->dev, hl_dev_attr_groups);
+	return rc;
 }
 
 void hl_sysfs_fini(struct hl_device *hdev)
diff --git a/drivers/accel/habanalabs/gaudi/gaudi.c b/drivers/accel/habanalabs/gaudi/gaudi.c
index bb858b94e1e8..a29aa8f7b6f3 100644
--- a/drivers/accel/habanalabs/gaudi/gaudi.c
+++ b/drivers/accel/habanalabs/gaudi/gaudi.c
@@ -656,6 +656,7 @@ static int gaudi_set_fixed_properties(struct hl_device *hdev)
 	prop->cfg_size = CFG_SIZE;
 	prop->max_asid = MAX_ASID;
 	prop->num_of_events = GAUDI_EVENT_SIZE;
+	prop->max_num_of_engines = GAUDI_ENGINE_ID_SIZE;
 	prop->tpc_enabled_mask = TPC_ENABLED_MASK;
 
 	set_default_power_values(hdev);
@@ -679,6 +680,10 @@ static int gaudi_set_fixed_properties(struct hl_device *hdev)
 			(num_sync_stream_queues * HL_RSVD_MONS);
 
 	prop->first_available_user_interrupt = USHRT_MAX;
+	prop->tpc_interrupt_id = USHRT_MAX;
+
+	/* single msi */
+	prop->eq_interrupt_id = 0;
 
 	for (i = 0 ; i < HL_MAX_DCORES ; i++)
 		prop->first_available_cq[i] = USHRT_MAX;
@@ -867,13 +872,18 @@ pci_init:
 	rc = hl_fw_read_preboot_status(hdev);
 	if (rc) {
 		if (hdev->reset_on_preboot_fail)
+			/* we are already on failure flow, so don't check if hw_fini fails. */
 			hdev->asic_funcs->hw_fini(hdev, true, false);
 		goto pci_fini;
 	}
 
 	if (gaudi_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
 		dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n");
-		hdev->asic_funcs->hw_fini(hdev, true, false);
+		rc = hdev->asic_funcs->hw_fini(hdev, true, false);
+		if (rc) {
+			dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc);
+			goto pci_fini;
+		}
 	}
 
 	return 0;
@@ -2010,38 +2020,6 @@ static int gaudi_enable_msi_single(struct hl_device *hdev)
 	return rc;
 }
 
-static int gaudi_enable_msi_multi(struct hl_device *hdev)
-{
-	int cq_cnt = hdev->asic_prop.completion_queues_count;
-	int rc, i, irq_cnt_init, irq;
-
-	for (i = 0, irq_cnt_init = 0 ; i < cq_cnt ; i++, irq_cnt_init++) {
-		irq = gaudi_pci_irq_vector(hdev, i, false);
-		rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi_irq_name[i],
-				&hdev->completion_queue[i]);
-		if (rc) {
-			dev_err(hdev->dev, "Failed to request IRQ %d", irq);
-			goto free_irqs;
-		}
-	}
-
-	irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX, true);
-	rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi_irq_name[cq_cnt],
-				&hdev->event_queue);
-	if (rc) {
-		dev_err(hdev->dev, "Failed to request IRQ %d", irq);
-		goto free_irqs;
-	}
-
-	return 0;
-
-free_irqs:
-	for (i = 0 ; i < irq_cnt_init ; i++)
-		free_irq(gaudi_pci_irq_vector(hdev, i, false),
-				&hdev->completion_queue[i]);
-	return rc;
-}
-
 static int gaudi_enable_msi(struct hl_device *hdev)
 {
 	struct gaudi_device *gaudi = hdev->asic_specific;
@@ -2056,14 +2034,7 @@ static int gaudi_enable_msi(struct hl_device *hdev)
 		return rc;
 	}
 
-	if (rc < NUMBER_OF_INTERRUPTS) {
-		gaudi->multi_msi_mode = false;
-		rc = gaudi_enable_msi_single(hdev);
-	} else {
-		gaudi->multi_msi_mode = true;
-		rc = gaudi_enable_msi_multi(hdev);
-	}
-
+	rc = gaudi_enable_msi_single(hdev);
 	if (rc)
 		goto free_pci_irq_vectors;
 
@@ -2079,47 +2050,23 @@ free_pci_irq_vectors:
 static void gaudi_sync_irqs(struct hl_device *hdev)
 {
 	struct gaudi_device *gaudi = hdev->asic_specific;
-	int i, cq_cnt = hdev->asic_prop.completion_queues_count;
 
 	if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))
 		return;
 
 	/* Wait for all pending IRQs to be finished */
-	if (gaudi->multi_msi_mode) {
-		for (i = 0 ; i < cq_cnt ; i++)
-			synchronize_irq(gaudi_pci_irq_vector(hdev, i, false));
-
-		synchronize_irq(gaudi_pci_irq_vector(hdev,
-						GAUDI_EVENT_QUEUE_MSI_IDX,
-						true));
-	} else {
-		synchronize_irq(gaudi_pci_irq_vector(hdev, 0, false));
-	}
+	synchronize_irq(gaudi_pci_irq_vector(hdev, 0, false));
 }
 
 static void gaudi_disable_msi(struct hl_device *hdev)
 {
 	struct gaudi_device *gaudi = hdev->asic_specific;
-	int i, irq, cq_cnt = hdev->asic_prop.completion_queues_count;
 
 	if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))
 		return;
 
 	gaudi_sync_irqs(hdev);
-
-	if (gaudi->multi_msi_mode) {
-		irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX,
-						true);
-		free_irq(irq, &hdev->event_queue);
-
-		for (i = 0 ; i < cq_cnt ; i++) {
-			irq = gaudi_pci_irq_vector(hdev, i, false);
-			free_irq(irq, &hdev->completion_queue[i]);
-		}
-	} else {
-		free_irq(gaudi_pci_irq_vector(hdev, 0, false), hdev);
-	}
-
+	free_irq(gaudi_pci_irq_vector(hdev, 0, false), hdev);
 	pci_free_irq_vectors(hdev->pdev);
 
 	gaudi->hw_cap_initialized &= ~HW_CAP_MSI;
@@ -3718,7 +3665,7 @@ static int gaudi_mmu_init(struct hl_device *hdev)
 		if (rc) {
 			dev_err(hdev->dev,
 				"failed to set hop0 addr for asid %d\n", i);
-			goto err;
+			return rc;
 		}
 	}
 
@@ -3729,7 +3676,9 @@ static int gaudi_mmu_init(struct hl_device *hdev)
 	/* mem cache invalidation */
 	WREG32(mmSTLB_MEM_CACHE_INVALIDATION, 1);
 
-	hl_mmu_invalidate_cache(hdev, true, 0);
+	rc = hl_mmu_invalidate_cache(hdev, true, 0);
+	if (rc)
+		return rc;
 
 	WREG32(mmMMU_UP_MMU_ENABLE, 1);
 	WREG32(mmMMU_UP_SPI_MASK, 0xF);
@@ -3745,9 +3694,6 @@ static int gaudi_mmu_init(struct hl_device *hdev)
 	gaudi->hw_cap_initialized |= HW_CAP_MMU;
 
 	return 0;
-
-err:
-	return rc;
 }
 
 static int gaudi_load_firmware_to_device(struct hl_device *hdev)
@@ -3915,11 +3861,7 @@ static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
 
 	WREG32(mmCPU_IF_PF_PQ_PI, 0);
 
-	if (gaudi->multi_msi_mode)
-		WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP);
-	else
-		WREG32(mmCPU_IF_QUEUE_INIT,
-			PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI);
+	WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI);
 
 	irq_handler_offset = prop->gic_interrupts_enable ?
 			mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
@@ -4068,7 +4010,7 @@ disable_queues:
 	return rc;
 }
 
-static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
+static int gaudi_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
 {
 	struct cpu_dyn_regs *dyn_regs =
 			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
@@ -4078,7 +4020,7 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset
 
 	if (!hard_reset) {
 		dev_err(hdev->dev, "GAUDI doesn't support soft-reset\n");
-		return;
+		return 0;
 	}
 
 	if (hdev->pldm) {
@@ -4199,10 +4141,10 @@ skip_reset:
 	msleep(reset_timeout_ms);
 
 	status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM);
-	if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK)
-		dev_err(hdev->dev,
-			"Timeout while waiting for device to reset 0x%x\n",
-			status);
+	if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK) {
+		dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", status);
+		return -ETIMEDOUT;
+	}
 
 	if (gaudi) {
 		gaudi->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q | HW_CAP_HBM |
@@ -4215,6 +4157,7 @@ skip_reset:
 
 		hdev->device_cpu_is_halted = false;
 	}
+	return 0;
 }
 
 static int gaudi_suspend(struct hl_device *hdev)
@@ -5595,7 +5538,6 @@ static void gaudi_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_add
 				u32 len, u32 original_len, u64 cq_addr, u32 cq_val,
 				u32 msi_vec, bool eb)
 {
-	struct gaudi_device *gaudi = hdev->asic_specific;
 	struct packet_msg_prot *cq_pkt;
 	struct packet_nop *cq_padding;
 	u64 msi_addr;
@@ -5625,12 +5567,7 @@ static void gaudi_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_add
 	tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);
 	cq_pkt->ctl = cpu_to_le32(tmp);
 	cq_pkt->value = cpu_to_le32(1);
-
-	if (gaudi->multi_msi_mode)
-		msi_addr = mmPCIE_MSI_INTR_0 + msi_vec * 4;
-	else
-		msi_addr = mmPCIE_CORE_MSI_REQ;
-
+	msi_addr = hdev->pdev ? mmPCIE_CORE_MSI_REQ : mmPCIE_MSI_INTR_0 + msi_vec * 4;
 	cq_pkt->addr = cpu_to_le64(CFG_BASE + msi_addr);
 }
 
@@ -7297,7 +7234,7 @@ static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *e
 }
 
 static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type,
-					bool razwi, u64 *event_mask)
+					bool check_razwi, u64 *event_mask)
 {
 	bool is_read = false, is_write = false;
 	u16 engine_id[2], num_of_razwi_eng = 0;
@@ -7316,7 +7253,7 @@ static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type,
 	dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
 		event_type, desc);
 
-	if (razwi) {
+	if (check_razwi) {
 		gaudi_print_and_get_razwi_info(hdev, &engine_id[0], &engine_id[1], &is_read,
 						&is_write);
 		gaudi_print_and_get_mmu_error_info(hdev, &razwi_addr, event_mask);
@@ -7333,8 +7270,9 @@ static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type,
 				num_of_razwi_eng = 1;
 		}
 
-		hl_handle_razwi(hdev, razwi_addr, engine_id, num_of_razwi_eng, razwi_flags,
-				event_mask);
+		if (razwi_flags)
+			hl_handle_razwi(hdev, razwi_addr, engine_id, num_of_razwi_eng,
+					razwi_flags, event_mask);
 	}
 }
 
@@ -7633,6 +7571,7 @@ static void gaudi_print_clk_change_info(struct hl_device *hdev, u16 event_type,
 static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
 {
 	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct hl_info_fw_err_info fw_err_info;
 	u64 data = le64_to_cpu(eq_entry->data[0]), event_mask = 0;
 	u32 ctl = le32_to_cpu(eq_entry->hdr.ctl);
 	u32 fw_fatal_err_flag = 0, flags = 0;
@@ -7911,7 +7850,10 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr
 	case GAUDI_EVENT_FW_ALIVE_S:
 		gaudi_print_irq_info(hdev, event_type, false, &event_mask);
 		gaudi_print_fw_alive_info(hdev, &eq_entry->fw_alive);
-		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
+		fw_err_info.err_type = HL_INFO_FW_REPORTED_ERR;
+		fw_err_info.event_id = event_type;
+		fw_err_info.event_mask = &event_mask;
+		hl_handle_fw_err(hdev, &fw_err_info);
 		goto reset_device;
 
 	default:
@@ -7942,6 +7884,10 @@ reset_device:
 	}
 
 	if (reset_required) {
+		/* escalate general hw errors to critical/fatal error */
+		if (event_mask & HL_NOTIFIER_EVENT_GENERAL_HW_ERR)
+			hl_handle_critical_hw_err(hdev, event_type, &event_mask);
+
 		hl_device_cond_reset(hdev, flags, event_mask);
 	} else {
 		hl_fw_unmask_irq(hdev, event_type);
@@ -8403,19 +8349,26 @@ static int gaudi_internal_cb_pool_init(struct hl_device *hdev,
 	}
 
 	mutex_lock(&hdev->mmu_lock);
+
 	rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base,
 			hdev->internal_cb_pool_dma_addr,
 			HOST_SPACE_INTERNAL_CB_SZ);
-
-	hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
-	mutex_unlock(&hdev->mmu_lock);
-
 	if (rc)
 		goto unreserve_internal_cb_pool;
 
+	rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
+	if (rc)
+		goto unmap_internal_cb_pool;
+
+	mutex_unlock(&hdev->mmu_lock);
+
 	return 0;
 
+unmap_internal_cb_pool:
+	hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base,
+			HOST_SPACE_INTERNAL_CB_SZ);
 unreserve_internal_cb_pool:
+	mutex_unlock(&hdev->mmu_lock);
 	hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base,
 			HOST_SPACE_INTERNAL_CB_SZ);
 destroy_internal_cb_pool:
diff --git a/drivers/accel/habanalabs/gaudi/gaudiP.h b/drivers/accel/habanalabs/gaudi/gaudiP.h
index 3d88d56c8eb3..b8fa724be5a1 100644
--- a/drivers/accel/habanalabs/gaudi/gaudiP.h
+++ b/drivers/accel/habanalabs/gaudi/gaudiP.h
@@ -28,20 +28,8 @@
 #define NUMBER_OF_COLLECTIVE_QUEUES	12
 #define NUMBER_OF_SOBS_IN_GRP		11
 
-/*
- * Number of MSI interrupts IDS:
- * Each completion queue has 1 ID
- * The event queue has 1 ID
- */
-#define NUMBER_OF_INTERRUPTS		(NUMBER_OF_CMPLT_QUEUES + \
-						NUMBER_OF_CPU_HW_QUEUES)
-
 #define GAUDI_STREAM_MASTER_ARR_SIZE	8
 
-#if (NUMBER_OF_INTERRUPTS > GAUDI_MSI_ENTRIES)
-#error "Number of MSI interrupts must be smaller or equal to GAUDI_MSI_ENTRIES"
-#endif
-
 #define CORESIGHT_TIMEOUT_USEC		100000		/* 100 ms */
 
 #define GAUDI_MAX_CLK_FREQ		2200000000ull	/* 2200 MHz */
@@ -324,8 +312,6 @@ struct gaudi_internal_qman_info {
  *                      signal we can use this engine in later code paths.
  *                      Each bit is cleared upon reset of its corresponding H/W
  *                      engine.
- * @multi_msi_mode: whether we are working in multi MSI single MSI mode.
- *                  Multi MSI is possible only with IOMMU enabled.
  * @mmu_cache_inv_pi: PI for MMU cache invalidation flow. The H/W expects an
  *                    8-bit value so use u8.
  */
@@ -345,7 +331,6 @@ struct gaudi_device {
 	u32				events_stat[GAUDI_EVENT_SIZE];
 	u32				events_stat_aggregate[GAUDI_EVENT_SIZE];
 	u32				hw_cap_initialized;
-	u8				multi_msi_mode;
 	u8				mmu_cache_inv_pi;
 };
 
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2.c b/drivers/accel/habanalabs/gaudi2/gaudi2.c
index 6f415fa94eee..b778cf764a68 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2.c
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2.c
@@ -23,7 +23,8 @@
 #define GAUDI2_DMA_POOL_BLK_SIZE		SZ_256		/* 256 bytes */
 
 #define GAUDI2_RESET_TIMEOUT_MSEC		2000		/* 2000ms */
-#define GAUDI2_RESET_POLL_TIMEOUT_USEC		50000		/* 50ms */
+
+#define GAUDI2_RESET_POLL_TIMEOUT_USEC		500000		/* 500ms */
 #define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC		25000		/* 25s */
 #define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC		25000		/* 25s */
 #define GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC	3000000		/* 3s */
@@ -86,10 +87,11 @@
 
 #define KDMA_TIMEOUT_USEC			USEC_PER_SEC
 
-#define IS_DMA_IDLE(dma_core_idle_ind_mask)	\
-	(!((dma_core_idle_ind_mask) &		\
-	((DCORE0_EDMA0_CORE_IDLE_IND_MASK_DESC_CNT_STS_MASK) | \
-	(DCORE0_EDMA0_CORE_IDLE_IND_MASK_COMP_MASK))))
+#define IS_DMA_IDLE(dma_core_sts0)	\
+	(!((dma_core_sts0) & (DCORE0_EDMA0_CORE_STS0_BUSY_MASK)))
+
+#define IS_DMA_HALTED(dma_core_sts1)	\
+	((dma_core_sts1) & (DCORE0_EDMA0_CORE_STS1_IS_HALT_MASK))
 
 #define IS_MME_IDLE(mme_arch_sts) (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK)
 
@@ -132,6 +134,282 @@
 
 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)
 
+/* RAZWI initiator coordinates */
+#define RAZWI_GET_AXUSER_XY(x) \
+	((x & 0xF8001FF0) >> 4)
+
+#define RAZWI_GET_AXUSER_LOW_XY(x) \
+	((x & 0x00001FF0) >> 4)
+
+#define RAZWI_INITIATOR_AXUER_L_X_SHIFT		0
+#define RAZWI_INITIATOR_AXUER_L_X_MASK		0x1F
+#define RAZWI_INITIATOR_AXUER_L_Y_SHIFT		5
+#define RAZWI_INITIATOR_AXUER_L_Y_MASK		0xF
+
+#define RAZWI_INITIATOR_AXUER_H_X_SHIFT		23
+#define RAZWI_INITIATOR_AXUER_H_X_MASK		0x1F
+
+#define RAZWI_INITIATOR_ID_X_Y_LOW(x, y) \
+	((((y) & RAZWI_INITIATOR_AXUER_L_Y_MASK) << RAZWI_INITIATOR_AXUER_L_Y_SHIFT) | \
+		(((x) & RAZWI_INITIATOR_AXUER_L_X_MASK) << RAZWI_INITIATOR_AXUER_L_X_SHIFT))
+
+#define RAZWI_INITIATOR_ID_X_HIGH(x) \
+		(((x) & RAZWI_INITIATOR_AXUER_H_X_MASK) << RAZWI_INITIATOR_AXUER_H_X_SHIFT)
+
+#define RAZWI_INITIATOR_ID_X_Y(xl, yl, xh) \
+	(RAZWI_INITIATOR_ID_X_Y_LOW(xl, yl) | RAZWI_INITIATOR_ID_X_HIGH(xh))
+
+#define PSOC_RAZWI_ENG_STR_SIZE 128
+#define PSOC_RAZWI_MAX_ENG_PER_RTR 5
+
+struct gaudi2_razwi_info {
+	u32 axuser_xy;
+	u32 rtr_ctrl;
+	u16 eng_id;
+	char *eng_name;
+};
+
+static struct gaudi2_razwi_info common_razwi_info[] = {
+		{RAZWI_INITIATOR_ID_X_Y(2, 4, 0), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_DEC_0, "DEC0"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_DEC_1, "DEC1"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 4, 18), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_DEC_0, "DEC2"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_DEC_1, "DEC3"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 11, 0), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_DEC_0, "DEC4"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_DEC_1, "DEC5"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 11, 18), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_DEC_0, "DEC6"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_DEC_1, "DEC7"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 4, 6), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC8"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 4, 7), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC9"},
+		{RAZWI_INITIATOR_ID_X_Y(3, 4, 2), mmDCORE0_RTR1_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_TPC_0, "TPC0"},
+		{RAZWI_INITIATOR_ID_X_Y(3, 4, 4), mmDCORE0_RTR1_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_TPC_1, "TPC1"},
+		{RAZWI_INITIATOR_ID_X_Y(4, 4, 2), mmDCORE0_RTR2_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_TPC_2, "TPC2"},
+		{RAZWI_INITIATOR_ID_X_Y(4, 4, 4), mmDCORE0_RTR2_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_TPC_3, "TPC3"},
+		{RAZWI_INITIATOR_ID_X_Y(5, 4, 2), mmDCORE0_RTR3_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_TPC_4, "TPC4"},
+		{RAZWI_INITIATOR_ID_X_Y(5, 4, 4), mmDCORE0_RTR3_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_TPC_5, "TPC5"},
+		{RAZWI_INITIATOR_ID_X_Y(16, 4, 14), mmDCORE1_RTR6_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_TPC_0, "TPC6"},
+		{RAZWI_INITIATOR_ID_X_Y(16, 4, 16), mmDCORE1_RTR6_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_TPC_1, "TPC7"},
+		{RAZWI_INITIATOR_ID_X_Y(15, 4, 14), mmDCORE1_RTR5_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_TPC_2, "TPC8"},
+		{RAZWI_INITIATOR_ID_X_Y(15, 4, 16), mmDCORE1_RTR5_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_TPC_3, "TPC9"},
+		{RAZWI_INITIATOR_ID_X_Y(14, 4, 14), mmDCORE1_RTR4_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_TPC_4, "TPC10"},
+		{RAZWI_INITIATOR_ID_X_Y(14, 4, 16), mmDCORE1_RTR4_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_TPC_5, "TPC11"},
+		{RAZWI_INITIATOR_ID_X_Y(5, 11, 2), mmDCORE2_RTR3_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_TPC_0, "TPC12"},
+		{RAZWI_INITIATOR_ID_X_Y(5, 11, 4), mmDCORE2_RTR3_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_TPC_1, "TPC13"},
+		{RAZWI_INITIATOR_ID_X_Y(4, 11, 2), mmDCORE2_RTR2_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_TPC_2, "TPC14"},
+		{RAZWI_INITIATOR_ID_X_Y(4, 11, 4), mmDCORE2_RTR2_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_TPC_3, "TPC15"},
+		{RAZWI_INITIATOR_ID_X_Y(3, 11, 2), mmDCORE2_RTR1_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_TPC_4, "TPC16"},
+		{RAZWI_INITIATOR_ID_X_Y(3, 11, 4), mmDCORE2_RTR1_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_TPC_5, "TPC17"},
+		{RAZWI_INITIATOR_ID_X_Y(14, 11, 14), mmDCORE3_RTR4_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_TPC_0, "TPC18"},
+		{RAZWI_INITIATOR_ID_X_Y(14, 11, 16), mmDCORE3_RTR4_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_TPC_1, "TPC19"},
+		{RAZWI_INITIATOR_ID_X_Y(15, 11, 14), mmDCORE3_RTR5_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_TPC_2, "TPC20"},
+		{RAZWI_INITIATOR_ID_X_Y(15, 11, 16), mmDCORE3_RTR5_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_TPC_3, "TPC21"},
+		{RAZWI_INITIATOR_ID_X_Y(16, 11, 14), mmDCORE3_RTR6_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_TPC_4, "TPC22"},
+		{RAZWI_INITIATOR_ID_X_Y(16, 11, 16), mmDCORE3_RTR6_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC23"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC24"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 4, 8), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC0_0, "NIC0"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 4, 10), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC0_1, "NIC1"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 4, 12), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC1_0, "NIC2"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC1_1, "NIC3"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 4, 15), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC2_0, "NIC4"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC2_1, "NIC5"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC3_0, "NIC6"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 11, 6), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC3_1, "NIC7"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 11, 8), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC4_0, "NIC8"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 11, 12), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC4_1, "NIC9"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC5_0, "NIC10"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_NIC5_1, "NIC11"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_PDMA_0, "PDMA0"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 4, 3), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_PDMA_1, "PDMA1"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "PMMU"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 4, 5), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "PCIE"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 4, 16), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_ARC_FARM, "ARC_FARM"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 4, 17), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_KDMA, "KDMA"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF1_RTR_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_EDMA_0, "EDMA0"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_EDMA_1, "EDMA1"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF1_RTR_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_EDMA_0, "EDMA2"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF0_RTR_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_EDMA_1, "EDMA3"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_EDMA_0, "EDMA4"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_EDMA_1, "EDMA5"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_EDMA_0, "EDMA6"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_EDMA_1, "EDMA7"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU0"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU1"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU2"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU3"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU4"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU5"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU6"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU7"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU8"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU9"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU10"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU11"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU12"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU13"},
+		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU14"},
+		{RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_SIZE, "HMMU15"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_ROT_0, "ROT0"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_ROT_1, "ROT1"},
+		{RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE,
+				GAUDI2_ENGINE_ID_PSOC, "CPU"},
+		{RAZWI_INITIATOR_ID_X_Y(17, 11, 11), mmDCORE3_RTR7_CTRL_BASE,
+				GAUDI2_ENGINE_ID_PSOC, "PSOC"}
+};
+
+static struct gaudi2_razwi_info mme_razwi_info[] = {
+		/* MME X high coordinate is N/A, hence using only low coordinates */
+		{RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP0"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP1"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_WR"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_RD"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE0"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE1"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE2"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE3"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE,
+				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE4"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP0"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP1"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_WR"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_RD"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE0"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE1"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE2"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE3"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE,
+				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE4"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP0"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP1"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_WR"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_RD"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE0"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE1"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE2"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE3"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE,
+				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE4"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP0"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP1"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_WR"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_RD"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE0"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE1"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE2"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE3"},
+		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE,
+				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE4"}
+};
+
 enum hl_pmmu_fatal_cause {
 	LATENCY_RD_OUT_FIFO_OVERRUN,
 	LATENCY_WR_OUT_FIFO_OVERRUN,
@@ -1437,6 +1715,34 @@ static const u32 gaudi2_tpc_cfg_blocks_bases[TPC_ID_SIZE] = {
 	[TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_CFG_BASE,
 };
 
+static const u32 gaudi2_tpc_eml_cfg_blocks_bases[TPC_ID_SIZE] = {
+	[TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_EML_CFG_BASE,
+	[TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_EML_CFG_BASE,
+	[TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_EML_CFG_BASE,
+	[TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_EML_CFG_BASE,
+	[TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_EML_CFG_BASE,
+	[TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_EML_CFG_BASE,
+	[TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_EML_CFG_BASE,
+	[TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_EML_CFG_BASE,
+	[TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_EML_CFG_BASE,
+	[TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_EML_CFG_BASE,
+	[TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_EML_CFG_BASE,
+	[TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_EML_CFG_BASE,
+	[TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_EML_CFG_BASE,
+	[TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_EML_CFG_BASE,
+	[TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_EML_CFG_BASE,
+	[TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_EML_CFG_BASE,
+	[TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_EML_CFG_BASE,
+	[TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_EML_CFG_BASE,
+	[TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_EML_CFG_BASE,
+	[TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_EML_CFG_BASE,
+	[TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_EML_CFG_BASE,
+	[TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_EML_CFG_BASE,
+	[TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_EML_CFG_BASE,
+	[TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_EML_CFG_BASE,
+	[TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_EML_CFG_BASE,
+};
+
 const u32 gaudi2_rot_blocks_bases[ROTATOR_ID_SIZE] = {
 	[ROTATOR_ID_0] = mmROT0_BASE,
 	[ROTATOR_ID_1] = mmROT1_BASE
@@ -1475,6 +1781,56 @@ static const u32 gaudi2_rot_id_to_queue_id[ROTATOR_ID_SIZE] = {
 	[ROTATOR_ID_1] = GAUDI2_QUEUE_ID_ROT_1_0,
 };
 
+static const u32 gaudi2_tpc_engine_id_to_tpc_id[] = {
+	[GAUDI2_DCORE0_ENGINE_ID_TPC_0] = TPC_ID_DCORE0_TPC0,
+	[GAUDI2_DCORE0_ENGINE_ID_TPC_1] = TPC_ID_DCORE0_TPC1,
+	[GAUDI2_DCORE0_ENGINE_ID_TPC_2] = TPC_ID_DCORE0_TPC2,
+	[GAUDI2_DCORE0_ENGINE_ID_TPC_3] = TPC_ID_DCORE0_TPC3,
+	[GAUDI2_DCORE0_ENGINE_ID_TPC_4] = TPC_ID_DCORE0_TPC4,
+	[GAUDI2_DCORE0_ENGINE_ID_TPC_5] = TPC_ID_DCORE0_TPC5,
+	[GAUDI2_DCORE1_ENGINE_ID_TPC_0] = TPC_ID_DCORE1_TPC0,
+	[GAUDI2_DCORE1_ENGINE_ID_TPC_1] = TPC_ID_DCORE1_TPC1,
+	[GAUDI2_DCORE1_ENGINE_ID_TPC_2] = TPC_ID_DCORE1_TPC2,
+	[GAUDI2_DCORE1_ENGINE_ID_TPC_3] = TPC_ID_DCORE1_TPC3,
+	[GAUDI2_DCORE1_ENGINE_ID_TPC_4] = TPC_ID_DCORE1_TPC4,
+	[GAUDI2_DCORE1_ENGINE_ID_TPC_5] = TPC_ID_DCORE1_TPC5,
+	[GAUDI2_DCORE2_ENGINE_ID_TPC_0] = TPC_ID_DCORE2_TPC0,
+	[GAUDI2_DCORE2_ENGINE_ID_TPC_1] = TPC_ID_DCORE2_TPC1,
+	[GAUDI2_DCORE2_ENGINE_ID_TPC_2] = TPC_ID_DCORE2_TPC2,
+	[GAUDI2_DCORE2_ENGINE_ID_TPC_3] = TPC_ID_DCORE2_TPC3,
+	[GAUDI2_DCORE2_ENGINE_ID_TPC_4] = TPC_ID_DCORE2_TPC4,
+	[GAUDI2_DCORE2_ENGINE_ID_TPC_5] = TPC_ID_DCORE2_TPC5,
+	[GAUDI2_DCORE3_ENGINE_ID_TPC_0] = TPC_ID_DCORE3_TPC0,
+	[GAUDI2_DCORE3_ENGINE_ID_TPC_1] = TPC_ID_DCORE3_TPC1,
+	[GAUDI2_DCORE3_ENGINE_ID_TPC_2] = TPC_ID_DCORE3_TPC2,
+	[GAUDI2_DCORE3_ENGINE_ID_TPC_3] = TPC_ID_DCORE3_TPC3,
+	[GAUDI2_DCORE3_ENGINE_ID_TPC_4] = TPC_ID_DCORE3_TPC4,
+	[GAUDI2_DCORE3_ENGINE_ID_TPC_5] = TPC_ID_DCORE3_TPC5,
+	/* the PCI TPC is placed last (mapped liked HW) */
+	[GAUDI2_DCORE0_ENGINE_ID_TPC_6] = TPC_ID_DCORE0_TPC6,
+};
+
+static const u32 gaudi2_mme_engine_id_to_mme_id[] = {
+	[GAUDI2_DCORE0_ENGINE_ID_MME] = MME_ID_DCORE0,
+	[GAUDI2_DCORE1_ENGINE_ID_MME] = MME_ID_DCORE1,
+	[GAUDI2_DCORE2_ENGINE_ID_MME] = MME_ID_DCORE2,
+	[GAUDI2_DCORE3_ENGINE_ID_MME] = MME_ID_DCORE3,
+};
+
+static const u32 gaudi2_edma_engine_id_to_edma_id[] = {
+	[GAUDI2_ENGINE_ID_PDMA_0] = DMA_CORE_ID_PDMA0,
+	[GAUDI2_ENGINE_ID_PDMA_1] = DMA_CORE_ID_PDMA1,
+	[GAUDI2_DCORE0_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA0,
+	[GAUDI2_DCORE0_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA1,
+	[GAUDI2_DCORE1_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA2,
+	[GAUDI2_DCORE1_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA3,
+	[GAUDI2_DCORE2_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA4,
+	[GAUDI2_DCORE2_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA5,
+	[GAUDI2_DCORE3_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA6,
+	[GAUDI2_DCORE3_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA7,
+	[GAUDI2_ENGINE_ID_KDMA] = DMA_CORE_ID_KDMA,
+};
+
 const u32 edma_stream_base[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = {
 	GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0,
 	GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0,
@@ -1499,41 +1855,6 @@ static const char gaudi2_vdec_irq_name[GAUDI2_VDEC_MSIX_ENTRIES][GAUDI2_MAX_STRI
 	"gaudi2 vdec s_1", "gaudi2 vdec s_1 abnormal"
 };
 
-static const u32 rtr_coordinates_to_rtr_id[NUM_OF_RTR_PER_DCORE * NUM_OF_DCORES] = {
-	RTR_ID_X_Y(2, 4),
-	RTR_ID_X_Y(3, 4),
-	RTR_ID_X_Y(4, 4),
-	RTR_ID_X_Y(5, 4),
-	RTR_ID_X_Y(6, 4),
-	RTR_ID_X_Y(7, 4),
-	RTR_ID_X_Y(8, 4),
-	RTR_ID_X_Y(9, 4),
-	RTR_ID_X_Y(10, 4),
-	RTR_ID_X_Y(11, 4),
-	RTR_ID_X_Y(12, 4),
-	RTR_ID_X_Y(13, 4),
-	RTR_ID_X_Y(14, 4),
-	RTR_ID_X_Y(15, 4),
-	RTR_ID_X_Y(16, 4),
-	RTR_ID_X_Y(17, 4),
-	RTR_ID_X_Y(2, 11),
-	RTR_ID_X_Y(3, 11),
-	RTR_ID_X_Y(4, 11),
-	RTR_ID_X_Y(5, 11),
-	RTR_ID_X_Y(6, 11),
-	RTR_ID_X_Y(7, 11),
-	RTR_ID_X_Y(8, 11),
-	RTR_ID_X_Y(9, 11),
-	RTR_ID_X_Y(0, 0),/* 24 no id */
-	RTR_ID_X_Y(0, 0),/* 25 no id */
-	RTR_ID_X_Y(0, 0),/* 26 no id */
-	RTR_ID_X_Y(0, 0),/* 27 no id */
-	RTR_ID_X_Y(14, 11),
-	RTR_ID_X_Y(15, 11),
-	RTR_ID_X_Y(16, 11),
-	RTR_ID_X_Y(17, 11)
-};
-
 enum rtr_id {
 	DCORE0_RTR0,
 	DCORE0_RTR1,
@@ -1784,7 +2105,14 @@ static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id);
 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val);
 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size,
 										bool is_memset);
+static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+		struct engines_data *e);
+static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+		struct engines_data *e);
+static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+		struct engines_data *e);
 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr);
+static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr);
 
 static void gaudi2_init_scrambler_hbm(struct hl_device *hdev)
 {
@@ -1988,6 +2316,8 @@ static int gaudi2_set_fixed_properties(struct hl_device *hdev)
 
 	prop->hints_range_reservation = true;
 
+	prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1;
+
 	if (hdev->pldm)
 		prop->mmu_pgt_size = 0x800000; /* 8MB */
 	else
@@ -2011,7 +2341,6 @@ static int gaudi2_set_fixed_properties(struct hl_device *hdev)
 	prop->dmmu.num_hops = MMU_ARCH_6_HOPS;
 	prop->dmmu.last_mask = LAST_MASK;
 	prop->dmmu.host_resident = 1;
-	/* TODO: will be duplicated until implementing per-MMU props */
 	prop->dmmu.hop_table_size = prop->mmu_hop_table_size;
 	prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size;
 
@@ -2027,7 +2356,6 @@ static int gaudi2_set_fixed_properties(struct hl_device *hdev)
 	prop->pmmu.host_resident = 1;
 	prop->pmmu.num_hops = MMU_ARCH_6_HOPS;
 	prop->pmmu.last_mask = LAST_MASK;
-	/* TODO: will be duplicated until implementing per-MMU props */
 	prop->pmmu.hop_table_size = prop->mmu_hop_table_size;
 	prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size;
 
@@ -2084,11 +2412,14 @@ static int gaudi2_set_fixed_properties(struct hl_device *hdev)
 		prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END;
 	}
 
+	prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE;
 	prop->num_engine_cores = CPU_ID_MAX;
 	prop->cfg_size = CFG_SIZE;
 	prop->max_asid = MAX_ASID;
 	prop->num_of_events = GAUDI2_EVENT_SIZE;
 
+	prop->supports_engine_modes = true;
+
 	prop->dc_power_default = DC_POWER_DEFAULT;
 
 	prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT;
@@ -2107,6 +2438,8 @@ static int gaudi2_set_fixed_properties(struct hl_device *hdev)
 					(num_sync_stream_queues * HL_RSVD_MONS);
 
 	prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST;
+	prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT;
+	prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE;
 
 	prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER;
 
@@ -2555,6 +2888,10 @@ static int gaudi2_cpucp_info_get(struct hl_device *hdev)
 	hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask);
 	hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask));
 
+	dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n",
+			hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning,
+			hdev->decoder_binning);
+
 	/*
 	 * at this point the DRAM parameters need to be updated according to data obtained
 	 * from the FW
@@ -2644,13 +2981,18 @@ static int gaudi2_early_init(struct hl_device *hdev)
 	rc = hl_fw_read_preboot_status(hdev);
 	if (rc) {
 		if (hdev->reset_on_preboot_fail)
+			/* we are already on failure flow, so don't check if hw_fini fails. */
 			hdev->asic_funcs->hw_fini(hdev, true, false);
 		goto pci_fini;
 	}
 
 	if (gaudi2_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
 		dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n");
-		hdev->asic_funcs->hw_fini(hdev, true, false);
+		rc = hdev->asic_funcs->hw_fini(hdev, true, false);
+		if (rc) {
+			dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc);
+			goto pci_fini;
+		}
 	}
 
 	return 0;
@@ -2692,6 +3034,7 @@ static bool gaudi2_is_arc_tpc_owned(u64 arc_id)
 
 static void gaudi2_init_arcs(struct hl_device *hdev)
 {
+	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
 	u64 arc_id;
 	u32 i;
@@ -2721,6 +3064,10 @@ static void gaudi2_init_arcs(struct hl_device *hdev)
 
 		gaudi2_set_arc_id_cap(hdev, arc_id);
 	}
+
+	/* Fetch ARC scratchpad address */
+	hdev->asic_prop.engine_core_interrupt_reg_addr =
+		CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl);
 }
 
 static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id)
@@ -2772,16 +3119,21 @@ static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id)
 	return 0;
 }
 
-static void gaudi2_scrub_arcs_dccm(struct hl_device *hdev)
+static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev)
 {
 	u16 arc_id;
+	int rc;
 
 	for (arc_id = CPU_ID_SCHED_ARC0 ; arc_id < CPU_ID_MAX ; arc_id++) {
 		if (!gaudi2_is_arc_enabled(hdev, arc_id))
 			continue;
 
-		gaudi2_scrub_arc_dccm(hdev, arc_id);
+		rc = gaudi2_scrub_arc_dccm(hdev, arc_id);
+		if (rc)
+			return rc;
 	}
+
+	return 0;
 }
 
 static int gaudi2_late_init(struct hl_device *hdev)
@@ -2805,7 +3157,13 @@ static int gaudi2_late_init(struct hl_device *hdev)
 	}
 
 	gaudi2_init_arcs(hdev);
-	gaudi2_scrub_arcs_dccm(hdev);
+
+	rc = gaudi2_scrub_arcs_dccm(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to scrub arcs DCCM\n");
+		goto disable_pci_access;
+	}
+
 	gaudi2_init_security(hdev);
 
 	return 0;
@@ -2989,6 +3347,13 @@ static void gaudi2_user_interrupt_setup(struct hl_device *hdev)
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
 	int i, j, k;
 
+	/* Initialize TPC interrupt */
+	HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC);
+
+	/* Initialize unexpected error interrupt */
+	HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0,
+						HL_USR_INTERRUPT_UNEXPECTED);
+
 	/* Initialize common user CQ interrupt */
 	HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev,
 				HL_COMMON_USER_CQ_INTERRUPT_ID, HL_USR_INTERRUPT_CQ);
@@ -3115,6 +3480,48 @@ static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev)
 	return gaudi2_special_blocks_config(hdev);
 }
 
+static void gaudi2_test_queues_msgs_free(struct hl_device *hdev)
+{
+	struct gaudi2_device *gaudi2 = hdev->asic_specific;
+	struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info;
+	int i;
+
+	for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) {
+		/* bail-out if this is an allocation failure point */
+		if (!msg_info[i].kern_addr)
+			break;
+
+		hl_asic_dma_pool_free(hdev, msg_info[i].kern_addr, msg_info[i].dma_addr);
+		msg_info[i].kern_addr = NULL;
+	}
+}
+
+static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev)
+{
+	struct gaudi2_device *gaudi2 = hdev->asic_specific;
+	struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info;
+	int i, rc;
+
+	/* allocate a message-short buf for each Q we intend to test */
+	for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) {
+		msg_info[i].kern_addr =
+			(void *)hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_short),
+							GFP_KERNEL, &msg_info[i].dma_addr);
+		if (!msg_info[i].kern_addr) {
+			dev_err(hdev->dev,
+				"Failed to allocate dma memory for H/W queue %d testing\n", i);
+			rc = -ENOMEM;
+			goto err_exit;
+		}
+	}
+
+	return 0;
+
+err_exit:
+	gaudi2_test_queues_msgs_free(hdev);
+	return rc;
+}
+
 static int gaudi2_sw_init(struct hl_device *hdev)
 {
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
@@ -3214,8 +3621,14 @@ static int gaudi2_sw_init(struct hl_device *hdev)
 	if (rc)
 		goto free_scratchpad_mem;
 
+	rc = gaudi2_test_queues_msgs_alloc(hdev);
+	if (rc)
+		goto special_blocks_free;
+
 	return 0;
 
+special_blocks_free:
+	gaudi2_special_blocks_iterator_free(hdev);
 free_scratchpad_mem:
 	hl_asic_dma_pool_free(hdev, gaudi2->scratchpad_kernel_address,
 				gaudi2->scratchpad_bus_address);
@@ -3238,6 +3651,8 @@ static int gaudi2_sw_fini(struct hl_device *hdev)
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
 
+	gaudi2_test_queues_msgs_free(hdev);
+
 	gaudi2_special_blocks_iterator_free(hdev);
 
 	hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);
@@ -3646,6 +4061,10 @@ static const char *gaudi2_irq_name(u16 irq_number)
 		return "gaudi2 completion";
 	case GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ... GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM:
 		return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM];
+	case GAUDI2_IRQ_NUM_TPC_ASSERT:
+		return "gaudi2 tpc assert";
+	case GAUDI2_IRQ_NUM_UNEXPECTED_ERROR:
+		return "gaudi2 unexpected error";
 	case GAUDI2_IRQ_NUM_USER_FIRST ... GAUDI2_IRQ_NUM_USER_LAST:
 		return "gaudi2 user completion";
 	default:
@@ -3677,7 +4096,6 @@ static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num)
 static int gaudi2_dec_enable_msix(struct hl_device *hdev)
 {
 	int rc, i, irq_init_cnt, irq, relative_idx;
-	irq_handler_t irq_handler;
 	struct hl_dec *dec;
 
 	for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, irq_init_cnt = 0;
@@ -3687,20 +4105,24 @@ static int gaudi2_dec_enable_msix(struct hl_device *hdev)
 		irq = pci_irq_vector(hdev->pdev, i);
 		relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM;
 
-		irq_handler = (relative_idx % 2) ?
-				hl_irq_handler_dec_abnrm :
-				hl_irq_handler_user_interrupt;
-
-		dec = hdev->dec + relative_idx / 2;
-
 		/* We pass different structures depending on the irq handler. For the abnormal
 		 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant
 		 * user_interrupt entry
+		 *
+		 * TODO: change the dec abnrm to threaded irq
 		 */
-		rc = request_irq(irq, irq_handler, 0, gaudi2_irq_name(i),
-				((relative_idx % 2) ?
-				(void *) dec :
-				(void *) &hdev->user_interrupt[dec->core_id]));
+
+		dec = hdev->dec + relative_idx / 2;
+		if (relative_idx % 2) {
+			rc = request_irq(irq, hl_irq_handler_dec_abnrm, 0,
+						gaudi2_irq_name(i), (void *) dec);
+		} else {
+			rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
+					hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
+					gaudi2_irq_name(i),
+					(void *) &hdev->user_interrupt[dec->core_id]);
+		}
+
 		if (rc) {
 			dev_err(hdev->dev, "Failed to request IRQ %d", irq);
 			goto free_dec_irqs;
@@ -3719,7 +4141,6 @@ static int gaudi2_enable_msix(struct hl_device *hdev)
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
 	int rc, irq, i, j, user_irq_init_cnt;
-	irq_handler_t irq_handler;
 	struct hl_cq *cq;
 
 	if (gaudi2->hw_cap_initialized & HW_CAP_MSIX)
@@ -3755,14 +4176,33 @@ static int gaudi2_enable_msix(struct hl_device *hdev)
 		goto free_event_irq;
 	}
 
+	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
+	rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
+			hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
+			gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), &hdev->tpc_interrupt);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to request IRQ %d", irq);
+		goto free_dec_irq;
+	}
+
+	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
+	rc = request_irq(irq, hl_irq_handler_user_interrupt, 0,
+			gaudi2_irq_name(GAUDI2_IRQ_NUM_UNEXPECTED_ERROR),
+					&hdev->unexpected_error_interrupt);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to request IRQ %d", irq);
+		goto free_tpc_irq;
+	}
+
 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0;
 			user_irq_init_cnt < prop->user_interrupt_count;
 			i++, j++, user_irq_init_cnt++) {
 
 		irq = pci_irq_vector(hdev->pdev, i);
-		irq_handler = hl_irq_handler_user_interrupt;
+		rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
+						hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
+						gaudi2_irq_name(i), &hdev->user_interrupt[j]);
 
-		rc = request_irq(irq, irq_handler, 0, gaudi2_irq_name(i), &hdev->user_interrupt[j]);
 		if (rc) {
 			dev_err(hdev->dev, "Failed to request IRQ %d", irq);
 			goto free_user_irq;
@@ -3780,9 +4220,13 @@ free_user_irq:
 		irq = pci_irq_vector(hdev->pdev, i);
 		free_irq(irq, &hdev->user_interrupt[j]);
 	}
-
-	gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1);
-
+	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
+	free_irq(irq, &hdev->unexpected_error_interrupt);
+free_tpc_irq:
+	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
+	free_irq(irq, &hdev->tpc_interrupt);
+free_dec_irq:
+	gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1);
 free_event_irq:
 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
 	free_irq(irq, cq);
@@ -3814,6 +4258,9 @@ static void gaudi2_sync_irqs(struct hl_device *hdev)
 		synchronize_irq(irq);
 	}
 
+	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT));
+	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR));
+
 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count;
 										i++, j++) {
 		irq = pci_irq_vector(hdev->pdev, i);
@@ -3840,6 +4287,12 @@ static void gaudi2_disable_msix(struct hl_device *hdev)
 
 	gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1);
 
+	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
+	free_irq(irq, &hdev->tpc_interrupt);
+
+	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
+	free_irq(irq, &hdev->unexpected_error_interrupt);
+
 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0;
 			k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) {
 
@@ -4037,7 +4490,6 @@ static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids,
 {
 	int i, rc;
 
-
 	for (i = 0 ; i < num_cores ; i++) {
 		if (gaudi2_is_arc_enabled(hdev, core_ids[i]))
 			gaudi2_set_arc_running_mode(hdev, core_ids[i], core_command);
@@ -4059,6 +4511,139 @@ static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids,
 	return 0;
 }
 
+static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
+{
+	struct gaudi2_device *gaudi2 = hdev->asic_specific;
+	u32 reg_base, reg_addr, reg_val, tpc_id;
+
+	if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
+		return 0;
+
+	tpc_id = gaudi2_tpc_engine_id_to_tpc_id[engine_id];
+	if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id)))
+		return 0;
+
+	reg_base = gaudi2_tpc_cfg_blocks_bases[tpc_id];
+	reg_addr = reg_base + TPC_CFG_STALL_OFFSET;
+	reg_val = FIELD_PREP(DCORE0_TPC0_CFG_TPC_STALL_V_MASK,
+			!!(engine_command == HL_ENGINE_STALL));
+	WREG32(reg_addr, reg_val);
+
+	if (engine_command == HL_ENGINE_RESUME) {
+		reg_base = gaudi2_tpc_eml_cfg_blocks_bases[tpc_id];
+		reg_addr = reg_base + TPC_EML_CFG_DBG_CNT_OFFSET;
+		RMWREG32(reg_addr, 0x1, DCORE0_TPC0_EML_CFG_DBG_CNT_DBG_EXIT_MASK);
+	}
+
+	return 0;
+}
+
+static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
+{
+	struct gaudi2_device *gaudi2 = hdev->asic_specific;
+	u32 reg_base, reg_addr, reg_val, mme_id;
+
+	mme_id = gaudi2_mme_engine_id_to_mme_id[engine_id];
+	if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id)))
+		return 0;
+
+	reg_base = gaudi2_mme_ctrl_lo_blocks_bases[mme_id];
+	reg_addr = reg_base + MME_CTRL_LO_QM_STALL_OFFSET;
+	reg_val = FIELD_PREP(DCORE0_MME_CTRL_LO_QM_STALL_V_MASK,
+			!!(engine_command == HL_ENGINE_STALL));
+	WREG32(reg_addr, reg_val);
+
+	return 0;
+}
+
+static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
+{
+	struct gaudi2_device *gaudi2 = hdev->asic_specific;
+	u32 reg_base, reg_addr, reg_val, edma_id;
+
+	if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
+		return 0;
+
+	edma_id = gaudi2_edma_engine_id_to_edma_id[engine_id];
+	if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id)))
+		return 0;
+
+	reg_base = gaudi2_dma_core_blocks_bases[edma_id];
+	reg_addr = reg_base + EDMA_CORE_CFG_STALL_OFFSET;
+	reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK,
+			!!(engine_command == HL_ENGINE_STALL));
+	WREG32(reg_addr, reg_val);
+
+	if (engine_command == HL_ENGINE_STALL) {
+		reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 0x1) |
+				FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_FLUSH_MASK, 0x1);
+		WREG32(reg_addr, reg_val);
+	}
+
+	return 0;
+}
+
+static int gaudi2_set_engine_modes(struct hl_device *hdev,
+		u32 *engine_ids, u32 num_engines, u32 engine_command)
+{
+	int i, rc;
+
+	for (i = 0 ; i < num_engines ; ++i) {
+		switch (engine_ids[i]) {
+		case GAUDI2_DCORE0_ENGINE_ID_TPC_0 ... GAUDI2_DCORE0_ENGINE_ID_TPC_5:
+		case GAUDI2_DCORE1_ENGINE_ID_TPC_0 ... GAUDI2_DCORE1_ENGINE_ID_TPC_5:
+		case GAUDI2_DCORE2_ENGINE_ID_TPC_0 ... GAUDI2_DCORE2_ENGINE_ID_TPC_5:
+		case GAUDI2_DCORE3_ENGINE_ID_TPC_0 ... GAUDI2_DCORE3_ENGINE_ID_TPC_5:
+			rc = gaudi2_set_tpc_engine_mode(hdev, engine_ids[i], engine_command);
+			if (rc)
+				return rc;
+
+			break;
+		case GAUDI2_DCORE0_ENGINE_ID_MME:
+		case GAUDI2_DCORE1_ENGINE_ID_MME:
+		case GAUDI2_DCORE2_ENGINE_ID_MME:
+		case GAUDI2_DCORE3_ENGINE_ID_MME:
+			rc = gaudi2_set_mme_engine_mode(hdev, engine_ids[i], engine_command);
+			if (rc)
+				return rc;
+
+			break;
+		case GAUDI2_DCORE0_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE0_ENGINE_ID_EDMA_1:
+		case GAUDI2_DCORE1_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE1_ENGINE_ID_EDMA_1:
+		case GAUDI2_DCORE2_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE2_ENGINE_ID_EDMA_1:
+		case GAUDI2_DCORE3_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE3_ENGINE_ID_EDMA_1:
+			rc = gaudi2_set_edma_engine_mode(hdev, engine_ids[i], engine_command);
+			if (rc)
+				return rc;
+
+			break;
+		default:
+			dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids,
+					u32 num_engines, u32 engine_command)
+{
+	switch (engine_command) {
+	case HL_ENGINE_CORE_HALT:
+	case HL_ENGINE_CORE_RUN:
+		return gaudi2_set_engine_cores(hdev, engine_ids, num_engines, engine_command);
+
+	case HL_ENGINE_STALL:
+	case HL_ENGINE_RESUME:
+		return gaudi2_set_engine_modes(hdev, engine_ids, num_engines, engine_command);
+
+	default:
+		dev_err(hdev->dev, "failed to execute command id %u\n", engine_command);
+		return -EINVAL;
+	}
+}
+
 static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)
 {
 	u32 wait_timeout_ms;
@@ -5509,11 +6094,10 @@ static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev)
  * gaudi2_execute_hard_reset - execute hard reset by driver/FW
  *
  * @hdev: pointer to the habanalabs device structure
- * @reset_sleep_ms: sleep time in msec after reset
  *
  * This function executes hard reset based on if driver/FW should do the reset
  */
-static void gaudi2_execute_hard_reset(struct hl_device *hdev, u32 reset_sleep_ms)
+static void gaudi2_execute_hard_reset(struct hl_device *hdev)
 {
 	if (hdev->asic_prop.hard_reset_done_by_fw) {
 		gaudi2_send_hard_reset_cmd(hdev);
@@ -5531,17 +6115,37 @@ static void gaudi2_execute_hard_reset(struct hl_device *hdev, u32 reset_sleep_ms
 	WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1);
 }
 
+static int gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_timeout_us)
+{
+	int i, rc = 0;
+	u32 reg_val;
+
+	for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++)
+		rc = hl_poll_timeout(
+			hdev,
+			mmCPU_RST_STATUS_TO_HOST,
+			reg_val,
+			reg_val == CPU_RST_STATUS_SOFT_RST_DONE,
+			1000,
+			poll_timeout_us);
+
+	if (rc)
+		dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n",
+				reg_val);
+	return rc;
+}
+
 /**
  * gaudi2_execute_soft_reset - execute soft reset by driver/FW
  *
  * @hdev: pointer to the habanalabs device structure
- * @reset_sleep_ms: sleep time in msec after reset
  * @driver_performs_reset: true if driver should perform reset instead of f/w.
+ * @poll_timeout_us: time to wait for response from f/w.
  *
  * This function executes soft reset based on if driver/FW should do the reset
  */
-static void gaudi2_execute_soft_reset(struct hl_device *hdev, u32 reset_sleep_ms,
-						bool driver_performs_reset)
+static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset,
+						u32 poll_timeout_us)
 {
 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
 
@@ -5554,7 +6158,8 @@ static void gaudi2_execute_soft_reset(struct hl_device *hdev, u32 reset_sleep_ms
 
 		WREG32(le32_to_cpu(dyn_regs->gic_host_soft_rst_irq),
 			gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET].cpu_id);
-		return;
+
+		return gaudi2_get_soft_rst_done_indication(hdev, poll_timeout_us);
 	}
 
 	/* Block access to engines, QMANs and SM during reset, these
@@ -5569,17 +6174,14 @@ static void gaudi2_execute_soft_reset(struct hl_device *hdev, u32 reset_sleep_ms
 				mmPCIE_VDEC1_MSTR_IF_RR_SHRD_HBW_BASE + HL_BLOCK_SIZE);
 
 	WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1);
+	return 0;
 }
 
-static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 reset_sleep_ms,
-								u32 poll_timeout_us)
+static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us)
 {
 	int i, rc = 0;
 	u32 reg_val;
 
-	/* without this sleep reset will not work */
-	msleep(reset_sleep_ms);
-
 	/* We poll the BTM done indication multiple times after reset due to
 	 * a HW errata 'GAUDI2_0300'
 	 */
@@ -5596,30 +6198,12 @@ static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 reset_sleep_m
 		dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val);
 }
 
-static void gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_timeout_us)
-{
-	int i, rc = 0;
-	u32 reg_val;
-
-	for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++)
-		rc = hl_poll_timeout(
-			hdev,
-			mmCPU_RST_STATUS_TO_HOST,
-			reg_val,
-			reg_val == CPU_RST_STATUS_SOFT_RST_DONE,
-			1000,
-			poll_timeout_us);
-
-	if (rc)
-		dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n",
-				reg_val);
-}
-
-static void gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
+static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
 {
 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
 	u32 poll_timeout_us, reset_sleep_ms;
 	bool driver_performs_reset = false;
+	int rc;
 
 	if (hdev->pldm) {
 		reset_sleep_ms = hard_reset ? GAUDI2_PLDM_HRESET_TIMEOUT_MSEC :
@@ -5637,7 +6221,7 @@ static void gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_rese
 
 	if (hard_reset) {
 		driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw;
-		gaudi2_execute_hard_reset(hdev, reset_sleep_ms);
+		gaudi2_execute_hard_reset(hdev);
 	} else {
 		/*
 		 * As we have to support also work with preboot only (which does not supports
@@ -5647,11 +6231,13 @@ static void gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_rese
 		 */
 		driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU &&
 							!hdev->asic_prop.fw_security_enabled);
-		gaudi2_execute_soft_reset(hdev, reset_sleep_ms, driver_performs_reset);
+		rc = gaudi2_execute_soft_reset(hdev, driver_performs_reset, poll_timeout_us);
+		if (rc)
+			return rc;
 	}
 
 skip_reset:
-	if (driver_performs_reset || hard_reset)
+	if (driver_performs_reset || hard_reset) {
 		/*
 		 * Instead of waiting for BTM indication we should wait for preboot ready:
 		 * Consider the below scenario:
@@ -5671,17 +6257,18 @@ skip_reset:
 		 * communicate with FW that is during reset.
 		 * to overcome this we will always wait to preboot ready indication
 		 */
-		if ((hdev->fw_components & FW_TYPE_PREBOOT_CPU)) {
-			msleep(reset_sleep_ms);
+
+		/* without this sleep reset will not work */
+		msleep(reset_sleep_ms);
+
+		if (hdev->fw_components & FW_TYPE_PREBOOT_CPU)
 			hl_fw_wait_preboot_ready(hdev);
-		} else {
-			gaudi2_poll_btm_indication(hdev, reset_sleep_ms, poll_timeout_us);
-		}
-	else
-		gaudi2_get_soft_rst_done_indication(hdev, poll_timeout_us);
+		else
+			gaudi2_poll_btm_indication(hdev, poll_timeout_us);
+	}
 
 	if (!gaudi2)
-		return;
+		return 0;
 
 	gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK);
 	gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK);
@@ -5708,6 +6295,7 @@ skip_reset:
 			HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_MME_MASK |
 			HW_CAP_ROT_MASK);
 	}
+	return 0;
 }
 
 static int gaudi2_suspend(struct hl_device *hdev)
@@ -6259,28 +6847,29 @@ static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, b
 	}
 }
 
-static int gaudi2_test_queue(struct hl_device *hdev, u32 hw_queue_id)
+static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id)
 {
-	u32 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4;
+	return hdev->asic_prop.first_available_user_sob[0] +
+				hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0;
+}
+
+static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id)
+{
+	u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
 	u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
-	u32 timeout_usec, tmp, sob_base = 1, sob_val = 0x5a5a;
-	struct packet_msg_short *msg_short_pkt;
-	dma_addr_t pkt_dma_addr;
-	size_t pkt_size;
-	int rc;
 
-	if (hdev->pldm)
-		timeout_usec = GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC;
-	else
-		timeout_usec = GAUDI2_TEST_QUEUE_WAIT_USEC;
+	/* Reset the SOB value */
+	WREG32(sob_addr, 0);
+}
 
-	pkt_size = sizeof(*msg_short_pkt);
-	msg_short_pkt = hl_asic_dma_pool_zalloc(hdev, pkt_size, GFP_KERNEL, &pkt_dma_addr);
-	if (!msg_short_pkt) {
-		dev_err(hdev->dev, "Failed to allocate packet for H/W queue %d testing\n",
-			hw_queue_id);
-		return -ENOMEM;
-	}
+static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val,
+					    struct gaudi2_queues_test_info *msg_info)
+{
+	u32 sob_offset =  gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
+	u32 tmp, sob_base = 1;
+	struct packet_msg_short *msg_short_pkt = msg_info->kern_addr;
+	size_t pkt_size = sizeof(struct packet_msg_short);
+	int rc;
 
 	tmp = (PACKET_MSG_SHORT << GAUDI2_PKT_CTL_OPCODE_SHIFT) |
 		(1 << GAUDI2_PKT_CTL_EB_SHIFT) |
@@ -6291,15 +6880,25 @@ static int gaudi2_test_queue(struct hl_device *hdev, u32 hw_queue_id)
 	msg_short_pkt->value = cpu_to_le32(sob_val);
 	msg_short_pkt->ctl = cpu_to_le32(tmp);
 
-	/* Reset the SOB value */
-	WREG32(sob_addr, 0);
+	rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr);
+	if (rc)
+		dev_err(hdev->dev,
+			"Failed to send msg_short packet to H/W queue %d\n", hw_queue_id);
 
-	rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, pkt_dma_addr);
-	if (rc) {
-		dev_err(hdev->dev, "Failed to send msg_short packet to H/W queue %d\n",
-			hw_queue_id);
-		goto free_pkt;
-	}
+	return rc;
+}
+
+static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val)
+{
+	u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
+	u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
+	u32 timeout_usec, tmp;
+	int rc;
+
+	if (hdev->pldm)
+		timeout_usec = GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC;
+	else
+		timeout_usec = GAUDI2_TEST_QUEUE_WAIT_USEC;
 
 	rc = hl_poll_timeout(
 			hdev,
@@ -6315,11 +6914,6 @@ static int gaudi2_test_queue(struct hl_device *hdev, u32 hw_queue_id)
 		rc = -EIO;
 	}
 
-	/* Reset the SOB value */
-	WREG32(sob_addr, 0);
-
-free_pkt:
-	hl_asic_dma_pool_free(hdev, (void *) msg_short_pkt, pkt_dma_addr);
 	return rc;
 }
 
@@ -6339,42 +6933,60 @@ static int gaudi2_test_cpu_queue(struct hl_device *hdev)
 
 static int gaudi2_test_queues(struct hl_device *hdev)
 {
-	int i, rc, ret_val = 0;
+	struct gaudi2_device *gaudi2 = hdev->asic_specific;
+	struct gaudi2_queues_test_info *msg_info;
+	u32 sob_val = 0x5a5a;
+	int i, rc;
 
+	/* send test message on all enabled Qs */
 	for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) {
 		if (!gaudi2_is_queue_enabled(hdev, i))
 			continue;
 
+		msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0];
 		gaudi2_qman_set_test_mode(hdev, i, true);
-		rc = gaudi2_test_queue(hdev, i);
-		gaudi2_qman_set_test_mode(hdev, i, false);
-
-		if (rc) {
-			ret_val = -EINVAL;
+		gaudi2_test_queue_clear(hdev, i);
+		rc = gaudi2_test_queue_send_msg_short(hdev, i, sob_val, msg_info);
+		if (rc)
 			goto done;
-		}
 	}
 
 	rc = gaudi2_test_cpu_queue(hdev);
-	if (rc) {
-		ret_val = -EINVAL;
+	if (rc)
 		goto done;
+
+	/* verify that all messages were processed */
+	for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) {
+		if (!gaudi2_is_queue_enabled(hdev, i))
+			continue;
+
+		rc = gaudi2_test_queue_wait_completion(hdev, i, sob_val);
+		if (rc)
+			/* chip is not usable, no need for cleanups, just bail-out with error */
+			goto done;
+
+		gaudi2_test_queue_clear(hdev, i);
+		gaudi2_qman_set_test_mode(hdev, i, false);
 	}
 
 done:
-	return ret_val;
+	return rc;
 }
 
 static int gaudi2_compute_reset_late_init(struct hl_device *hdev)
 {
 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
 	size_t irq_arr_size;
+	int rc;
 
-	/* TODO: missing gaudi2_nic_resume.
-	 * Until implemented nic_hw_cap_initialized will remain zeroed
-	 */
 	gaudi2_init_arcs(hdev);
-	gaudi2_scrub_arcs_dccm(hdev);
+
+	rc = gaudi2_scrub_arcs_dccm(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to scrub arcs DCCM\n");
+		return rc;
+	}
+
 	gaudi2_init_security(hdev);
 
 	/* Unmask all IRQs since some could have been received during the soft reset */
@@ -6382,74 +6994,21 @@ static int gaudi2_compute_reset_late_init(struct hl_device *hdev)
 	return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size);
 }
 
-static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset,
-					struct iterate_module_ctx *ctx)
-{
-	struct gaudi2_tpc_idle_data *idle_data = ctx->data;
-	u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
-	bool is_eng_idle;
-	int engine_idx;
-
-	if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1)))
-		engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_6;
-	else
-		engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_0 +
-				dcore * GAUDI2_ENGINE_ID_DCORE_OFFSET + inst;
-
-	tpc_cfg_sts = RREG32(mmDCORE0_TPC0_CFG_STATUS + offset);
-	qm_glbl_sts0 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS0 + offset);
-	qm_glbl_sts1 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS1 + offset);
-	qm_cgm_sts = RREG32(mmDCORE0_TPC0_QM_CGM_STS + offset);
-
-	is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
-						IS_TPC_IDLE(tpc_cfg_sts);
-	*(idle_data->is_idle) &= is_eng_idle;
-
-	if (idle_data->mask && !is_eng_idle)
-		set_bit(engine_idx, idle_data->mask);
-
-	if (idle_data->e)
-		hl_engine_data_sprintf(idle_data->e,
-					idle_data->tpc_fmt, dcore, inst,
-					is_eng_idle ? "Y" : "N",
-					qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts);
-}
-
-static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
-					struct engines_data *e)
+static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+		struct engines_data *e)
 {
-	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_idle_ind_mask,
-		mme_arch_sts, dec_swreg15, dec_enabled_bit;
+	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1;
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
-	const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-12x%s\n";
 	unsigned long *mask = (unsigned long *) mask_arr;
-	const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#x\n";
-	const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n";
-	const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n";
-	const char *pdma_fmt = "%-6d%-9s%#-14x%#x\n";
-	const char *pcie_dec_fmt = "%-10d%-9s%#x\n";
-	const char *dec_fmt = "%-6d%-5d%-9s%#x\n";
+	const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n";
 	bool is_idle = true, is_eng_idle;
-	u64 offset;
-
-	struct gaudi2_tpc_idle_data tpc_idle_data = {
-		.tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n",
-		.e = e,
-		.mask = mask,
-		.is_idle = &is_idle,
-	};
-	struct iterate_module_ctx tpc_iter = {
-		.fn = &gaudi2_is_tpc_engine_idle,
-		.data = &tpc_idle_data,
-	};
-
 	int engine_idx, i, j;
+	u64 offset;
 
-	/* EDMA, Two engines per Dcore */
 	if (e)
 		hl_engine_data_sprintf(e,
-			"\nCORE  EDMA  is_idle  QM_GLBL_STS0  DMA_CORE_IDLE_IND_MASK\n"
-			"----  ----  -------  ------------  ----------------------\n");
+			"\nCORE  EDMA  is_idle  QM_GLBL_STS0  DMA_CORE_STS0  DMA_CORE_STS1\n"
+			"----  ----  -------  ------------  -------------  -------------\n");
 
 	for (i = 0; i < NUM_OF_DCORES; i++) {
 		for (j = 0 ; j < NUM_OF_EDMA_PER_DCORE ; j++) {
@@ -6462,45 +7021,56 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask
 					i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j;
 			offset = i * DCORE_OFFSET + j * DCORE_EDMA_OFFSET;
 
-			dma_core_idle_ind_mask =
-			RREG32(mmDCORE0_EDMA0_CORE_IDLE_IND_MASK + offset);
+			dma_core_sts0 = RREG32(mmDCORE0_EDMA0_CORE_STS0 + offset);
+			dma_core_sts1 = RREG32(mmDCORE0_EDMA0_CORE_STS1 + offset);
 
 			qm_glbl_sts0 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS0 + offset);
 			qm_glbl_sts1 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS1 + offset);
 			qm_cgm_sts = RREG32(mmDCORE0_EDMA0_QM_CGM_STS + offset);
 
 			is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
-					IS_DMA_IDLE(dma_core_idle_ind_mask);
+					IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1);
 			is_idle &= is_eng_idle;
 
 			if (mask && !is_eng_idle)
 				set_bit(engine_idx, mask);
 
 			if (e)
-				hl_engine_data_sprintf(e, edma_fmt, i, j,
-							is_eng_idle ? "Y" : "N",
-							qm_glbl_sts0,
-							dma_core_idle_ind_mask);
+				hl_engine_data_sprintf(e, edma_fmt, i, j, is_eng_idle ? "Y" : "N",
+							qm_glbl_sts0, dma_core_sts0, dma_core_sts1);
 		}
 	}
 
-	/* PDMA, Two engines in Full chip */
+	return is_idle;
+}
+
+static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+		struct engines_data *e)
+{
+	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1;
+	unsigned long *mask = (unsigned long *) mask_arr;
+	const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n";
+	bool is_idle = true, is_eng_idle;
+	int engine_idx, i;
+	u64 offset;
+
 	if (e)
 		hl_engine_data_sprintf(e,
-					"\nPDMA  is_idle  QM_GLBL_STS0  DMA_CORE_IDLE_IND_MASK\n"
-					"----  -------  ------------  ----------------------\n");
+					"\nPDMA  is_idle  QM_GLBL_STS0  DMA_CORE_STS0  DMA_CORE_STS1\n"
+					"----  -------  ------------  -------------  -------------\n");
 
 	for (i = 0 ; i < NUM_OF_PDMA ; i++) {
 		engine_idx = GAUDI2_ENGINE_ID_PDMA_0 + i;
 		offset = i * PDMA_OFFSET;
-		dma_core_idle_ind_mask = RREG32(mmPDMA0_CORE_IDLE_IND_MASK + offset);
+		dma_core_sts0 = RREG32(mmPDMA0_CORE_STS0 + offset);
+		dma_core_sts1 = RREG32(mmPDMA0_CORE_STS1 + offset);
 
 		qm_glbl_sts0 = RREG32(mmPDMA0_QM_GLBL_STS0 + offset);
 		qm_glbl_sts1 = RREG32(mmPDMA0_QM_GLBL_STS1 + offset);
 		qm_cgm_sts = RREG32(mmPDMA0_QM_CGM_STS + offset);
 
 		is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
-				IS_DMA_IDLE(dma_core_idle_ind_mask);
+				IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1);
 		is_idle &= is_eng_idle;
 
 		if (mask && !is_eng_idle)
@@ -6508,9 +7078,22 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask
 
 		if (e)
 			hl_engine_data_sprintf(e, pdma_fmt, i, is_eng_idle ? "Y" : "N",
-						qm_glbl_sts0, dma_core_idle_ind_mask);
+						qm_glbl_sts0, dma_core_sts0, dma_core_sts1);
 	}
 
+	return is_idle;
+}
+
+static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+		struct engines_data *e)
+{
+	unsigned long *mask = (unsigned long *) mask_arr;
+	const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n";
+	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
+	bool is_idle = true, is_eng_idle;
+	int engine_idx, i;
+	u64 offset = 0;
+
 	/* NIC, twelve macros in Full chip */
 	if (e && hdev->nic_ports_mask)
 		hl_engine_data_sprintf(e,
@@ -6544,6 +7127,19 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask
 						qm_glbl_sts0, qm_cgm_sts);
 	}
 
+	return is_idle;
+}
+
+static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+		struct engines_data *e)
+{
+	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, mme_arch_sts;
+	unsigned long *mask = (unsigned long *) mask_arr;
+	const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n";
+	bool is_idle = true, is_eng_idle;
+	int engine_idx, i;
+	u64 offset;
+
 	if (e)
 		hl_engine_data_sprintf(e,
 					"\nMME  Stub  is_idle  QM_GLBL_STS0  MME_ARCH_STATUS\n"
@@ -6574,16 +7170,82 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask
 			set_bit(engine_idx, mask);
 	}
 
-	/*
-	 * TPC
-	 */
+	return is_idle;
+}
+
+static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset,
+					struct iterate_module_ctx *ctx)
+{
+	struct gaudi2_tpc_idle_data *idle_data = ctx->data;
+	u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
+	bool is_eng_idle;
+	int engine_idx;
+
+	if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1)))
+		engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_6;
+	else
+		engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_0 +
+				dcore * GAUDI2_ENGINE_ID_DCORE_OFFSET + inst;
+
+	tpc_cfg_sts = RREG32(mmDCORE0_TPC0_CFG_STATUS + offset);
+	qm_glbl_sts0 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS0 + offset);
+	qm_glbl_sts1 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS1 + offset);
+	qm_cgm_sts = RREG32(mmDCORE0_TPC0_QM_CGM_STS + offset);
+
+	is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
+						IS_TPC_IDLE(tpc_cfg_sts);
+	*(idle_data->is_idle) &= is_eng_idle;
+
+	if (idle_data->mask && !is_eng_idle)
+		set_bit(engine_idx, idle_data->mask);
+
+	if (idle_data->e)
+		hl_engine_data_sprintf(idle_data->e,
+					idle_data->tpc_fmt, dcore, inst,
+					is_eng_idle ? "Y" : "N",
+					qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts);
+}
+
+static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+		struct engines_data *e)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	unsigned long *mask = (unsigned long *) mask_arr;
+	bool is_idle = true;
+
+	struct gaudi2_tpc_idle_data tpc_idle_data = {
+		.tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n",
+		.e = e,
+		.mask = mask,
+		.is_idle = &is_idle,
+	};
+	struct iterate_module_ctx tpc_iter = {
+		.fn = &gaudi2_is_tpc_engine_idle,
+		.data = &tpc_idle_data,
+	};
+
 	if (e && prop->tpc_enabled_mask)
 		hl_engine_data_sprintf(e,
-			"\nCORE  TPC   is_idle  QM_GLBL_STS0  QM_CGM_STS  DMA_CORE_IDLE_IND_MASK\n"
-			"----  ---  --------  ------------  ----------  ----------------------\n");
+			"\nCORE  TPC  is_idle  QM_GLBL_STS0  QM_CGM_STS  STATUS\n"
+			"----  ---  -------  ------------  ----------  ------\n");
 
 	gaudi2_iterate_tpcs(hdev, &tpc_iter);
 
+	return tpc_idle_data.is_idle;
+}
+
+static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+		struct engines_data *e)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	unsigned long *mask = (unsigned long *) mask_arr;
+	const char *pcie_dec_fmt = "%-10d%-9s%#x\n";
+	const char *dec_fmt = "%-6d%-5d%-9s%#x\n";
+	bool is_idle = true, is_eng_idle;
+	u32 dec_swreg15, dec_enabled_bit;
+	int engine_idx, i, j;
+	u64 offset;
+
 	/* Decoders, two each Dcore and two shared PCIe decoders */
 	if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK)))
 		hl_engine_data_sprintf(e,
@@ -6638,10 +7300,23 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask
 						is_eng_idle ? "Y" : "N", dec_swreg15);
 	}
 
+	return is_idle;
+}
+
+static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+		struct engines_data *e)
+{
+	const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n";
+	unsigned long *mask = (unsigned long *) mask_arr;
+	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
+	bool is_idle = true, is_eng_idle;
+	int engine_idx, i;
+	u64 offset;
+
 	if (e)
 		hl_engine_data_sprintf(e,
-			"\nCORE  ROT  is_idle  QM_GLBL_STS0  QM_CGM_STS  DMA_CORE_STS0\n"
-			"----  ----  -------  ------------  ----------  -------------\n");
+			"\nCORE  ROT  is_idle  QM_GLBL_STS0  QM_GLBL_STS1  QM_CGM_STS\n"
+			"----  ---  -------  ------------  ------------  ----------\n");
 
 	for (i = 0 ; i < NUM_OF_ROT ; i++) {
 		engine_idx = GAUDI2_ENGINE_ID_ROT_0 + i;
@@ -6660,12 +7335,28 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask
 
 		if (e)
 			hl_engine_data_sprintf(e, rot_fmt, i, 0, is_eng_idle ? "Y" : "N",
-					qm_glbl_sts0, qm_cgm_sts, "-");
+						qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
 	}
 
 	return is_idle;
 }
 
+static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
+					struct engines_data *e)
+{
+	bool is_idle = true;
+
+	is_idle &= gaudi2_get_edma_idle_status(hdev, mask_arr, mask_len, e);
+	is_idle &= gaudi2_get_pdma_idle_status(hdev, mask_arr, mask_len, e);
+	is_idle &= gaudi2_get_nic_idle_status(hdev, mask_arr, mask_len, e);
+	is_idle &= gaudi2_get_mme_idle_status(hdev, mask_arr, mask_len, e);
+	is_idle &= gaudi2_get_tpc_idle_status(hdev, mask_arr, mask_len, e);
+	is_idle &= gaudi2_get_decoder_idle_status(hdev, mask_arr, mask_len, e);
+	is_idle &= gaudi2_get_rotator_idle_status(hdev, mask_arr, mask_len, e);
+
+	return is_idle;
+}
+
 static void gaudi2_hw_queues_lock(struct hl_device *hdev)
 	__acquires(&gaudi2->hw_queues_lock)
 {
@@ -7040,7 +7731,7 @@ static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type,
 	memory_wrapper_idx = ecc_data->memory_wrapper_idx;
 
 	gaudi2_print_event(hdev, event_type, !ecc_data->is_critical,
-		"ECC error detected. address: %#llx. Syndrom: %#llx. block id %u. critical %u.\n",
+		"ECC error detected. address: %#llx. Syndrom: %#llx. block id %u. critical %u.",
 		ecc_address, ecc_syndrom, memory_wrapper_idx, ecc_data->is_critical);
 
 	return !!ecc_data->is_critical;
@@ -7352,10 +8043,8 @@ static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev,
 	case RAZWI_TPC:
 		hbw_rtr_id = gaudi2_tpc_initiator_hbw_rtr_id[module_idx];
 
-		/* TODO : remove this check and depend only on tpc routers table
-		 * when SW-118828 is resolved
-		 */
-		if (!hdev->asic_prop.fw_security_enabled &&
+		if (hl_is_fw_ver_below_1_9(hdev) &&
+				!hdev->asic_prop.fw_security_enabled &&
 				((module_idx == 0) || (module_idx == 1)))
 			lbw_rtr_id = DCORE0_RTR0;
 		else
@@ -7526,297 +8215,115 @@ static void gaudi2_check_if_razwi_happened(struct hl_device *hdev)
 		gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL);
 }
 
-static const char *gaudi2_get_initiators_name(u32 rtr_id)
-{
-	switch (rtr_id) {
-	case DCORE0_RTR0:
-		return "DEC0/1/8/9, TPC24, PDMA0/1, PMMU, PCIE_IF, EDMA0/2, HMMU0/2/4/6, CPU";
-	case DCORE0_RTR1:
-		return "TPC0/1";
-	case DCORE0_RTR2:
-		return "TPC2/3";
-	case DCORE0_RTR3:
-		return "TPC4/5";
-	case DCORE0_RTR4:
-		return "MME0_SBTE0/1";
-	case DCORE0_RTR5:
-		return "MME0_WAP0/SBTE2";
-	case DCORE0_RTR6:
-		return "MME0_CTRL_WR/SBTE3";
-	case DCORE0_RTR7:
-		return "MME0_WAP1/CTRL_RD/SBTE4";
-	case DCORE1_RTR0:
-		return "MME1_WAP1/CTRL_RD/SBTE4";
-	case DCORE1_RTR1:
-		return "MME1_CTRL_WR/SBTE3";
-	case DCORE1_RTR2:
-		return "MME1_WAP0/SBTE2";
-	case DCORE1_RTR3:
-		return "MME1_SBTE0/1";
-	case DCORE1_RTR4:
-		return "TPC10/11";
-	case DCORE1_RTR5:
-		return "TPC8/9";
-	case DCORE1_RTR6:
-		return "TPC6/7";
-	case DCORE1_RTR7:
-		return "DEC2/3, NIC0/1/2/3/4, ARC_FARM, KDMA, EDMA1/3, HMMU1/3/5/7";
-	case DCORE2_RTR0:
-		return "DEC4/5, NIC5/6/7/8, EDMA4/6, HMMU8/10/12/14, ROT0";
-	case DCORE2_RTR1:
-		return "TPC16/17";
-	case DCORE2_RTR2:
-		return "TPC14/15";
-	case DCORE2_RTR3:
-		return "TPC12/13";
-	case DCORE2_RTR4:
-		return "MME2_SBTE0/1";
-	case DCORE2_RTR5:
-		return "MME2_WAP0/SBTE2";
-	case DCORE2_RTR6:
-		return "MME2_CTRL_WR/SBTE3";
-	case DCORE2_RTR7:
-		return "MME2_WAP1/CTRL_RD/SBTE4";
-	case DCORE3_RTR0:
-		return "MME3_WAP1/CTRL_RD/SBTE4";
-	case DCORE3_RTR1:
-		return "MME3_CTRL_WR/SBTE3";
-	case DCORE3_RTR2:
-		return "MME3_WAP0/SBTE2";
-	case DCORE3_RTR3:
-		return "MME3_SBTE0/1";
-	case DCORE3_RTR4:
-		return "TPC18/19";
-	case DCORE3_RTR5:
-		return "TPC20/21";
-	case DCORE3_RTR6:
-		return "TPC22/23";
-	case DCORE3_RTR7:
-		return "DEC6/7, NIC9/10/11, EDMA5/7, HMMU9/11/13/15, ROT1, PSOC";
-	default:
-	return "N/A";
-	}
-}
-
-static u16 gaudi2_get_razwi_initiators(u32 rtr_id, u16 *engines)
-{
-	switch (rtr_id) {
-	case DCORE0_RTR0:
-		engines[0] = GAUDI2_DCORE0_ENGINE_ID_DEC_0;
-		engines[1] = GAUDI2_DCORE0_ENGINE_ID_DEC_1;
-		engines[2] = GAUDI2_PCIE_ENGINE_ID_DEC_0;
-		engines[3] = GAUDI2_PCIE_ENGINE_ID_DEC_1;
-		engines[4] = GAUDI2_DCORE0_ENGINE_ID_TPC_6;
-		engines[5] = GAUDI2_ENGINE_ID_PDMA_0;
-		engines[6] = GAUDI2_ENGINE_ID_PDMA_1;
-		engines[7] = GAUDI2_ENGINE_ID_PCIE;
-		engines[8] = GAUDI2_DCORE0_ENGINE_ID_EDMA_0;
-		engines[9] = GAUDI2_DCORE1_ENGINE_ID_EDMA_0;
-		engines[10] = GAUDI2_ENGINE_ID_PSOC;
-		return 11;
-
-	case DCORE0_RTR1:
-		engines[0] = GAUDI2_DCORE0_ENGINE_ID_TPC_0;
-		engines[1] = GAUDI2_DCORE0_ENGINE_ID_TPC_1;
-		return 2;
-
-	case DCORE0_RTR2:
-		engines[0] = GAUDI2_DCORE0_ENGINE_ID_TPC_2;
-		engines[1] = GAUDI2_DCORE0_ENGINE_ID_TPC_3;
-		return 2;
-
-	case DCORE0_RTR3:
-		engines[0] = GAUDI2_DCORE0_ENGINE_ID_TPC_4;
-		engines[1] = GAUDI2_DCORE0_ENGINE_ID_TPC_5;
-		return 2;
-
-	case DCORE0_RTR4:
-	case DCORE0_RTR5:
-	case DCORE0_RTR6:
-	case DCORE0_RTR7:
-		engines[0] = GAUDI2_DCORE0_ENGINE_ID_MME;
-		return 1;
-
-	case DCORE1_RTR0:
-	case DCORE1_RTR1:
-	case DCORE1_RTR2:
-	case DCORE1_RTR3:
-		engines[0] = GAUDI2_DCORE1_ENGINE_ID_MME;
-		return 1;
-
-	case DCORE1_RTR4:
-		engines[0] = GAUDI2_DCORE1_ENGINE_ID_TPC_4;
-		engines[1] = GAUDI2_DCORE1_ENGINE_ID_TPC_5;
-		return 2;
-
-	case DCORE1_RTR5:
-		engines[0] = GAUDI2_DCORE1_ENGINE_ID_TPC_2;
-		engines[1] = GAUDI2_DCORE1_ENGINE_ID_TPC_3;
-		return 2;
-
-	case DCORE1_RTR6:
-		engines[0] = GAUDI2_DCORE1_ENGINE_ID_TPC_0;
-		engines[1] = GAUDI2_DCORE1_ENGINE_ID_TPC_1;
-		return 2;
-
-	case DCORE1_RTR7:
-		engines[0] = GAUDI2_DCORE1_ENGINE_ID_DEC_0;
-		engines[1] = GAUDI2_DCORE1_ENGINE_ID_DEC_1;
-		engines[2] = GAUDI2_ENGINE_ID_NIC0_0;
-		engines[3] = GAUDI2_ENGINE_ID_NIC1_0;
-		engines[4] = GAUDI2_ENGINE_ID_NIC2_0;
-		engines[5] = GAUDI2_ENGINE_ID_NIC3_0;
-		engines[6] = GAUDI2_ENGINE_ID_NIC4_0;
-		engines[7] = GAUDI2_ENGINE_ID_ARC_FARM;
-		engines[8] = GAUDI2_ENGINE_ID_KDMA;
-		engines[9] = GAUDI2_DCORE0_ENGINE_ID_EDMA_1;
-		engines[10] = GAUDI2_DCORE1_ENGINE_ID_EDMA_1;
-		return 11;
-
-	case DCORE2_RTR0:
-		engines[0] = GAUDI2_DCORE2_ENGINE_ID_DEC_0;
-		engines[1] = GAUDI2_DCORE2_ENGINE_ID_DEC_1;
-		engines[2] = GAUDI2_ENGINE_ID_NIC5_0;
-		engines[3] = GAUDI2_ENGINE_ID_NIC6_0;
-		engines[4] = GAUDI2_ENGINE_ID_NIC7_0;
-		engines[5] = GAUDI2_ENGINE_ID_NIC8_0;
-		engines[6] = GAUDI2_DCORE2_ENGINE_ID_EDMA_0;
-		engines[7] = GAUDI2_DCORE3_ENGINE_ID_EDMA_0;
-		engines[8] = GAUDI2_ENGINE_ID_ROT_0;
-		return 9;
-
-	case DCORE2_RTR1:
-		engines[0] = GAUDI2_DCORE2_ENGINE_ID_TPC_4;
-		engines[1] = GAUDI2_DCORE2_ENGINE_ID_TPC_5;
-		return 2;
-
-	case DCORE2_RTR2:
-		engines[0] = GAUDI2_DCORE2_ENGINE_ID_TPC_2;
-		engines[1] = GAUDI2_DCORE2_ENGINE_ID_TPC_3;
-		return 2;
-
-	case DCORE2_RTR3:
-		engines[0] = GAUDI2_DCORE2_ENGINE_ID_TPC_0;
-		engines[1] = GAUDI2_DCORE2_ENGINE_ID_TPC_1;
-		return 2;
-
-	case DCORE2_RTR4:
-	case DCORE2_RTR5:
-	case DCORE2_RTR6:
-	case DCORE2_RTR7:
-		engines[0] = GAUDI2_DCORE2_ENGINE_ID_MME;
-		return 1;
-	case DCORE3_RTR0:
-	case DCORE3_RTR1:
-	case DCORE3_RTR2:
-	case DCORE3_RTR3:
-		engines[0] = GAUDI2_DCORE3_ENGINE_ID_MME;
-		return 1;
-	case DCORE3_RTR4:
-		engines[0] = GAUDI2_DCORE3_ENGINE_ID_TPC_0;
-		engines[1] = GAUDI2_DCORE3_ENGINE_ID_TPC_1;
-		return 2;
-	case DCORE3_RTR5:
-		engines[0] = GAUDI2_DCORE3_ENGINE_ID_TPC_2;
-		engines[1] = GAUDI2_DCORE3_ENGINE_ID_TPC_3;
-		return 2;
-	case DCORE3_RTR6:
-		engines[0] = GAUDI2_DCORE3_ENGINE_ID_TPC_4;
-		engines[1] = GAUDI2_DCORE3_ENGINE_ID_TPC_5;
-		return 2;
-	case DCORE3_RTR7:
-		engines[0] = GAUDI2_DCORE3_ENGINE_ID_DEC_0;
-		engines[1] = GAUDI2_DCORE3_ENGINE_ID_DEC_1;
-		engines[2] = GAUDI2_ENGINE_ID_NIC9_0;
-		engines[3] = GAUDI2_ENGINE_ID_NIC10_0;
-		engines[4] = GAUDI2_ENGINE_ID_NIC11_0;
-		engines[5] = GAUDI2_DCORE2_ENGINE_ID_EDMA_1;
-		engines[6] = GAUDI2_DCORE3_ENGINE_ID_EDMA_1;
-		engines[7] = GAUDI2_ENGINE_ID_ROT_1;
-		engines[8] = GAUDI2_ENGINE_ID_ROT_0;
-		return 9;
-	default:
-		return 0;
-	}
-}
-
-static void gaudi2_razwi_unmapped_addr_hbw_printf_info(struct hl_device *hdev, u32 rtr_id,
-							u64 rtr_ctrl_base_addr, bool is_write,
-							u64 *event_mask)
+static int gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info *razwi_info, u32 array_size,
+						u32 axuser_xy, u32 *base, u16 *eng_id,
+						char *eng_name)
 {
-	u16 engines[HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR], num_of_eng;
-	u32 razwi_hi, razwi_lo;
-	u8 rd_wr_flag;
 
-	num_of_eng = gaudi2_get_razwi_initiators(rtr_id, &engines[0]);
+	int i, num_of_eng = 0;
+	u16 str_size = 0;
 
-	if (is_write) {
-		razwi_hi = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_ADDR_HI);
-		razwi_lo = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_ADDR_LO);
-		rd_wr_flag = HL_RAZWI_WRITE;
-
-		/* Clear set indication */
-		WREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_SET, 0x1);
-	} else {
-		razwi_hi = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_ADDR_HI);
-		razwi_lo = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_ADDR_LO);
-		rd_wr_flag = HL_RAZWI_READ;
+	for (i = 0 ; i < array_size ; i++) {
+		if (axuser_xy != razwi_info[i].axuser_xy)
+			continue;
 
-		/* Clear set indication */
-		WREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_SET, 0x1);
+		eng_id[num_of_eng] = razwi_info[i].eng_id;
+		base[num_of_eng] = razwi_info[i].rtr_ctrl;
+		if (!num_of_eng)
+			str_size += snprintf(eng_name + str_size,
+						PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s",
+						razwi_info[i].eng_name);
+		else
+			str_size += snprintf(eng_name + str_size,
+						PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s",
+						razwi_info[i].eng_name);
+		num_of_eng++;
 	}
 
-	hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &engines[0], num_of_eng,
-				rd_wr_flag | HL_RAZWI_HBW, event_mask);
-	dev_err_ratelimited(hdev->dev,
-		"RAZWI PSOC unmapped HBW %s error, rtr id %u, address %#llx\n",
-		is_write ? "WR" : "RD", rtr_id, (u64)razwi_hi << 32 | razwi_lo);
-
-	dev_err_ratelimited(hdev->dev,
-		"Initiators: %s\n", gaudi2_get_initiators_name(rtr_id));
+	return num_of_eng;
 }
 
-static void gaudi2_razwi_unmapped_addr_lbw_printf_info(struct hl_device *hdev, u32 rtr_id,
-							u64 rtr_ctrl_base_addr, bool is_write,
-							u64 *event_mask)
+static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg,
+						u64 *event_mask)
 {
-	u16 engines[HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR], num_of_eng;
-	u64 razwi_addr = CFG_BASE;
-	u8 rd_wr_flag;
+	u32 axuser_xy = RAZWI_GET_AXUSER_XY(razwi_reg), addr_hi = 0, addr_lo = 0;
+	u32 base[PSOC_RAZWI_MAX_ENG_PER_RTR];
+	u16 num_of_eng, eng_id[PSOC_RAZWI_MAX_ENG_PER_RTR];
+	char eng_name_str[PSOC_RAZWI_ENG_STR_SIZE];
+	bool razwi_happened = false;
+	int i;
 
-	num_of_eng = gaudi2_get_razwi_initiators(rtr_id, &engines[0]);
+	num_of_eng = gaudi2_psoc_razwi_get_engines(common_razwi_info, ARRAY_SIZE(common_razwi_info),
+							axuser_xy, base, eng_id, eng_name_str);
 
-	if (is_write) {
-		razwi_addr += RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AW_ADDR);
-		rd_wr_flag = HL_RAZWI_WRITE;
+	/* If no match for XY coordinates, try to find it in MME razwi table */
+	if (!num_of_eng) {
+		axuser_xy = RAZWI_GET_AXUSER_LOW_XY(razwi_reg);
+		num_of_eng = gaudi2_psoc_razwi_get_engines(mme_razwi_info,
+								ARRAY_SIZE(mme_razwi_info),
+								axuser_xy, base, eng_id,
+								eng_name_str);
+	}
 
-		/* Clear set indication */
-		WREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AW_SET, 0x1);
-	} else {
-		razwi_addr += RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_ADDR);
-		rd_wr_flag = HL_RAZWI_READ;
+	for  (i = 0 ; i < num_of_eng ; i++) {
+		if (RREG32(base[i] + DEC_RAZWI_HBW_AW_SET)) {
+			addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_HI);
+			addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_LO);
+			dev_err(hdev->dev,
+					"PSOC HBW AW RAZWI: %s, address (aligned to 128 byte): 0x%llX\n",
+					eng_name_str, ((u64)addr_hi << 32) + addr_lo);
+			hl_handle_razwi(hdev, ((u64)addr_hi << 32) + addr_lo, &eng_id[0],
+					num_of_eng, HL_RAZWI_HBW | HL_RAZWI_WRITE, event_mask);
+			razwi_happened = true;
+		}
 
-		/* Clear set indication */
-		WREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_SET, 0x1);
-	}
+		if (RREG32(base[i] + DEC_RAZWI_HBW_AR_SET)) {
+			addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_HI);
+			addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_LO);
+			dev_err(hdev->dev,
+					"PSOC HBW AR RAZWI: %s, address (aligned to 128 byte): 0x%llX\n",
+					eng_name_str, ((u64)addr_hi << 32) + addr_lo);
+			hl_handle_razwi(hdev, ((u64)addr_hi << 32) + addr_lo, &eng_id[0],
+					num_of_eng, HL_RAZWI_HBW | HL_RAZWI_READ, event_mask);
+			razwi_happened = true;
+		}
 
-	hl_handle_razwi(hdev, razwi_addr, &engines[0], num_of_eng, rd_wr_flag | HL_RAZWI_LBW,
-			event_mask);
-	dev_err_ratelimited(hdev->dev,
-		"RAZWI PSOC unmapped LBW %s error, rtr id %u, address 0x%llX\n",
-		is_write ? "WR" : "RD", rtr_id, razwi_addr);
+		if (RREG32(base[i] + DEC_RAZWI_LBW_AW_SET)) {
+			addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AW_ADDR);
+			dev_err(hdev->dev,
+					"PSOC LBW AW RAZWI: %s, address (aligned to 128 byte): 0x%X\n",
+					eng_name_str, addr_lo);
+			hl_handle_razwi(hdev, addr_lo, &eng_id[0],
+					num_of_eng, HL_RAZWI_LBW | HL_RAZWI_WRITE, event_mask);
+			razwi_happened = true;
+		}
 
-	dev_err_ratelimited(hdev->dev,
-		"Initiators: %s\n", gaudi2_get_initiators_name(rtr_id));
+		if (RREG32(base[i] + DEC_RAZWI_LBW_AR_SET)) {
+			addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AR_ADDR);
+			dev_err(hdev->dev,
+					"PSOC LBW AR RAZWI: %s, address (aligned to 128 byte): 0x%X\n",
+					eng_name_str, addr_lo);
+			hl_handle_razwi(hdev, addr_lo, &eng_id[0],
+					num_of_eng, HL_RAZWI_LBW | HL_RAZWI_READ, event_mask);
+			razwi_happened = true;
+		}
+		/* In common case the loop will break, when there is only one engine id, or
+		 * several engines with the same router. The exceptional case is with psoc razwi
+		 * from EDMA, where it's possible to get axuser id which fits 2 routers (2
+		 * interfaces of sft router). In this case, maybe the first router won't hold info
+		 * and we will need to iterate on the other router.
+		 */
+		if (razwi_happened)
+			break;
+	}
+
+	return razwi_happened;
 }
 
 /* PSOC RAZWI interrupt occurs only when trying to access a bad address */
 static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask)
 {
-	u32 hbw_aw_set, hbw_ar_set, lbw_aw_set, lbw_ar_set, rtr_id, dcore_id, dcore_rtr_id, xy,
-						razwi_mask_info, razwi_intr = 0, error_count = 0;
-	int rtr_map_arr_len = NUM_OF_RTR_PER_DCORE * NUM_OF_DCORES;
-	u64 rtr_ctrl_base_addr;
+	u32 razwi_mask_info, razwi_intr = 0, error_count = 0;
 
 	if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) {
 		razwi_intr = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT);
@@ -7825,63 +8332,22 @@ static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *even
 	}
 
 	razwi_mask_info = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_MASK_INFO);
-	xy = FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK, razwi_mask_info);
 
 	dev_err_ratelimited(hdev->dev,
 		"PSOC RAZWI interrupt: Mask %d, AR %d, AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n",
 		FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK, razwi_mask_info),
 		FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK, razwi_mask_info),
 		FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK, razwi_mask_info),
-		xy,
+		FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK, razwi_mask_info),
 		FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK, razwi_mask_info));
 
-	if (xy == 0) {
-		dev_err_ratelimited(hdev->dev,
-				"PSOC RAZWI interrupt: received event from 0 rtr coordinates\n");
-		goto clear;
-	}
-
-	/* Find router id by router coordinates */
-	for (rtr_id = 0 ; rtr_id < rtr_map_arr_len ; rtr_id++)
-		if (rtr_coordinates_to_rtr_id[rtr_id] == xy)
-			break;
-
-	if (rtr_id == rtr_map_arr_len) {
+	if (gaudi2_handle_psoc_razwi_happened(hdev, razwi_mask_info, event_mask))
+		error_count++;
+	else
 		dev_err_ratelimited(hdev->dev,
-				"PSOC RAZWI interrupt: invalid rtr coordinates (0x%x)\n", xy);
-		goto clear;
-	}
-
-	/* Find router mstr_if register base */
-	dcore_id = rtr_id / NUM_OF_RTR_PER_DCORE;
-	dcore_rtr_id = rtr_id % NUM_OF_RTR_PER_DCORE;
-	rtr_ctrl_base_addr = mmDCORE0_RTR0_CTRL_BASE + dcore_id * DCORE_OFFSET +
-				dcore_rtr_id * DCORE_RTR_OFFSET;
-
-	hbw_aw_set = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_SET);
-	hbw_ar_set = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_SET);
-	lbw_aw_set = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AW_SET);
-	lbw_ar_set = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_SET);
-
-	if (hbw_aw_set)
-		gaudi2_razwi_unmapped_addr_hbw_printf_info(hdev, rtr_id,
-						rtr_ctrl_base_addr, true, event_mask);
-
-	if (hbw_ar_set)
-		gaudi2_razwi_unmapped_addr_hbw_printf_info(hdev, rtr_id,
-						rtr_ctrl_base_addr, false, event_mask);
-
-	if (lbw_aw_set)
-		gaudi2_razwi_unmapped_addr_lbw_printf_info(hdev, rtr_id,
-						rtr_ctrl_base_addr, true, event_mask);
-
-	if (lbw_ar_set)
-		gaudi2_razwi_unmapped_addr_lbw_printf_info(hdev, rtr_id,
-						rtr_ctrl_base_addr, false, event_mask);
+				"PSOC RAZWI interrupt: invalid razwi info (0x%x)\n",
+				razwi_mask_info);
 
-	error_count++;
-
-clear:
 	/* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */
 	if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX))
 		WREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT, razwi_intr);
@@ -7976,7 +8442,7 @@ static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *e
 {
 	u32 qid_base, error_count = 0;
 	u64 qman_base;
-	u8 index;
+	u8 index = 0;
 
 	switch (event_type) {
 	case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC5_QM:
@@ -8094,23 +8560,28 @@ static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *e
 
 static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type)
 {
-	u32 i, sts_val, sts_clr_val = 0, error_count = 0;
+	u32 i, sts_val, sts_clr_val, error_count = 0, arc_farm;
 
-	sts_val = RREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_STS);
+	for (arc_farm = 0 ; arc_farm < NUM_OF_ARC_FARMS_ARC ; arc_farm++) {
+		sts_clr_val = 0;
+		sts_val = RREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_STS +
+				(arc_farm * ARC_FARM_OFFSET));
 
-	for (i = 0 ; i < GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE ; i++) {
-		if (sts_val & BIT(i)) {
-			gaudi2_print_event(hdev, event_type, true,
-				"err cause: %s", gaudi2_arc_sei_error_cause[i]);
-			sts_clr_val |= BIT(i);
-			error_count++;
+		for (i = 0 ; i < GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE ; i++) {
+			if (sts_val & BIT(i)) {
+				gaudi2_print_event(hdev, event_type, true,
+						"ARC FARM ARC %u err cause: %s",
+						arc_farm, gaudi2_arc_sei_error_cause[i]);
+				sts_clr_val |= BIT(i);
+				error_count++;
+			}
 		}
+		WREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_CLR + (arc_farm * ARC_FARM_OFFSET),
+				sts_clr_val);
 	}
 
 	hl_check_for_glbl_errors(hdev);
 
-	WREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_CLR, sts_clr_val);
-
 	return error_count;
 }
 
@@ -8318,14 +8789,13 @@ static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type,
 	return error_count;
 }
 
-static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type,
-					u64 intr_cause_data)
+static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, int sts_addr)
 {
-	u32 error_count = 0;
+	u32 error_count = 0, sts_val = RREG32(sts_addr);
 	int i;
 
 	for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++)
-		if (intr_cause_data & BIT(i)) {
+		if (sts_val & BIT(i)) {
 			gaudi2_print_event(hdev, event_type, true,
 				"err cause: %s", gaudi2_dma_core_interrupts_cause[i]);
 			error_count++;
@@ -8336,6 +8806,27 @@ static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type,
 	return error_count;
 }
 
+static int gaudi2_handle_pdma_core_event(struct hl_device *hdev, u16 event_type, int pdma_idx)
+{
+	u32 sts_addr;
+
+	sts_addr = mmPDMA0_CORE_ERR_CAUSE + pdma_idx * PDMA_OFFSET;
+	return gaudi2_handle_dma_core_event(hdev, event_type, sts_addr);
+}
+
+static int gaudi2_handle_edma_core_event(struct hl_device *hdev, u16 event_type, int edma_idx)
+{
+	static const int edma_event_index_map[] = {2, 3, 0, 1, 6, 7, 4, 5};
+	u32 sts_addr, index;
+
+	index = edma_event_index_map[edma_idx];
+
+	sts_addr = mmDCORE0_EDMA0_CORE_ERR_CAUSE +
+				DCORE_OFFSET * (index / NUM_OF_EDMA_PER_DCORE) +
+				DCORE_EDMA_OFFSET * (index % NUM_OF_EDMA_PER_DCORE);
+	return gaudi2_handle_dma_core_event(hdev, event_type, sts_addr);
+}
+
 static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask)
 {
 	u32 mstr_if_base_addr = mmPCIE_MSTR_RR_MSTR_IF_RR_SHRD_HBW_BASE, razwi_happened_addr;
@@ -8433,7 +8924,7 @@ static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 i
 static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu,
 					u64 *event_mask)
 {
-	u32 valid, val, axid_l, axid_h;
+	u32 valid, val;
 	u64 addr;
 
 	valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID));
@@ -8446,14 +8937,14 @@ static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool
 	addr <<= 32;
 	addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA));
 
-	axid_l = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_FAULT_ID_LSB));
-	axid_h = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_FAULT_ID_MSB));
+	if (!is_pmmu)
+		addr = gaudi2_mmu_descramble_addr(hdev, addr);
 
-	dev_err_ratelimited(hdev->dev, "%s page fault on va 0x%llx, transaction id 0x%llX\n",
-				is_pmmu ? "PMMU" : "HMMU", addr, ((u64)axid_h << 32) + axid_l);
+	dev_err_ratelimited(hdev->dev, "%s page fault on va 0x%llx\n",
+				is_pmmu ? "PMMU" : "HMMU", addr);
 	hl_handle_page_fault(hdev, addr, 0, is_pmmu, event_mask);
 
-	WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE), 0);
+	WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0);
 }
 
 static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu)
@@ -8471,9 +8962,12 @@ static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, boo
 	addr <<= 32;
 	addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA));
 
+	if (!is_pmmu)
+		addr = gaudi2_mmu_descramble_addr(hdev, addr);
+
 	dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n",
 				is_pmmu ? "PMMU" : "HMMU", addr);
-	WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE), 0);
+	WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0);
 }
 
 static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type,
@@ -8534,7 +9028,7 @@ static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_in
 				continue;
 
 			gaudi2_print_event(hdev, event_type, true,
-				"err cause: %s. %s: 0x%X\n",
+				"err cause: %s. %s: 0x%X",
 				gaudi2_sm_sei_cause[i].cause_name,
 				gaudi2_sm_sei_cause[i].log_name,
 				sei_cause_log);
@@ -8565,46 +9059,110 @@ static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_in
 	return error_count;
 }
 
+static u64 get_hmmu_base(u16 event_type)
+{
+	u8 dcore, index_in_dcore;
+
+	switch (event_type) {
+	case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU0_SECURITY_ERROR:
+		dcore = 0;
+		index_in_dcore = 0;
+	break;
+	case GAUDI2_EVENT_HMMU_1_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU1_SPI_BASE ... GAUDI2_EVENT_HMMU1_SECURITY_ERROR:
+		dcore = 1;
+		index_in_dcore = 0;
+	break;
+	case GAUDI2_EVENT_HMMU_2_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU2_SPI_BASE ... GAUDI2_EVENT_HMMU2_SECURITY_ERROR:
+		dcore = 0;
+		index_in_dcore = 1;
+	break;
+	case GAUDI2_EVENT_HMMU_3_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU3_SPI_BASE ... GAUDI2_EVENT_HMMU3_SECURITY_ERROR:
+		dcore = 1;
+		index_in_dcore = 1;
+	break;
+	case GAUDI2_EVENT_HMMU_4_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU4_SPI_BASE ... GAUDI2_EVENT_HMMU4_SECURITY_ERROR:
+		dcore = 3;
+		index_in_dcore = 2;
+	break;
+	case GAUDI2_EVENT_HMMU_5_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU5_SPI_BASE ... GAUDI2_EVENT_HMMU5_SECURITY_ERROR:
+		dcore = 2;
+		index_in_dcore = 2;
+	break;
+	case GAUDI2_EVENT_HMMU_6_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU6_SPI_BASE ... GAUDI2_EVENT_HMMU6_SECURITY_ERROR:
+		dcore = 3;
+		index_in_dcore = 3;
+	break;
+	case GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU7_SPI_BASE ... GAUDI2_EVENT_HMMU7_SECURITY_ERROR:
+		dcore = 2;
+		index_in_dcore = 3;
+	break;
+	case GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU8_SPI_BASE ... GAUDI2_EVENT_HMMU8_SECURITY_ERROR:
+		dcore = 0;
+		index_in_dcore = 2;
+	break;
+	case GAUDI2_EVENT_HMMU_9_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU9_SPI_BASE ... GAUDI2_EVENT_HMMU9_SECURITY_ERROR:
+		dcore = 1;
+		index_in_dcore = 2;
+	break;
+	case GAUDI2_EVENT_HMMU_10_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU10_SPI_BASE ... GAUDI2_EVENT_HMMU10_SECURITY_ERROR:
+		dcore = 0;
+		index_in_dcore = 3;
+	break;
+	case GAUDI2_EVENT_HMMU_11_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU11_SPI_BASE ... GAUDI2_EVENT_HMMU11_SECURITY_ERROR:
+		dcore = 1;
+		index_in_dcore = 3;
+	break;
+	case GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU12_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
+		dcore = 3;
+		index_in_dcore = 0;
+	break;
+	case GAUDI2_EVENT_HMMU_13_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU13_SPI_BASE ... GAUDI2_EVENT_HMMU13_SECURITY_ERROR:
+		dcore = 2;
+		index_in_dcore = 0;
+	break;
+	case GAUDI2_EVENT_HMMU_14_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU14_SPI_BASE ... GAUDI2_EVENT_HMMU14_SECURITY_ERROR:
+		dcore = 3;
+		index_in_dcore = 1;
+	break;
+	case GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU15_SPI_BASE ... GAUDI2_EVENT_HMMU15_SECURITY_ERROR:
+		dcore = 2;
+		index_in_dcore = 1;
+	break;
+	default:
+		return ULONG_MAX;
+	}
+
+	return mmDCORE0_HMMU0_MMU_BASE + dcore * DCORE_OFFSET + index_in_dcore * DCORE_HMMU_OFFSET;
+}
+
 static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)
 {
 	bool is_pmmu = false;
 	u32 error_count = 0;
 	u64 mmu_base;
-	u8 index;
 
 	switch (event_type) {
-	case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU3_SECURITY_ERROR:
-		index = (event_type - GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM) / 3;
-		mmu_base = mmDCORE0_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
-		break;
-	case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_3_AXI_ERR_RSP:
-		index = (event_type - GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP);
-		mmu_base = mmDCORE0_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
-		break;
-	case GAUDI2_EVENT_HMMU8_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_HMMU11_SECURITY_ERROR:
-		index = (event_type - GAUDI2_EVENT_HMMU8_PAGE_FAULT_WR_PERM) / 3;
-		mmu_base = mmDCORE1_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
-		break;
-	case GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_11_AXI_ERR_RSP:
-		index = (event_type - GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP);
-		mmu_base = mmDCORE1_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
-		break;
-	case GAUDI2_EVENT_HMMU7_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_HMMU4_SECURITY_ERROR:
-		index = (event_type - GAUDI2_EVENT_HMMU7_PAGE_FAULT_WR_PERM) / 3;
-		mmu_base = mmDCORE2_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
-		break;
-	case GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_4_AXI_ERR_RSP:
-		index = (event_type - GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP);
-		mmu_base = mmDCORE2_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
-		break;
-	case GAUDI2_EVENT_HMMU15_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
-		index = (event_type - GAUDI2_EVENT_HMMU15_PAGE_FAULT_WR_PERM) / 3;
-		mmu_base = mmDCORE3_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
-		break;
-	case GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
-		index = (event_type - GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP);
-		mmu_base = mmDCORE3_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
+	case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
+	case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
+		mmu_base = get_hmmu_base(event_type);
 		break;
+
 	case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR:
 	case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0:
 		is_pmmu = true;
@@ -8614,6 +9172,9 @@ static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type,
 		return 0;
 	}
 
+	if (mmu_base == ULONG_MAX)
+		return 0;
+
 	error_count = gaudi2_handle_mmu_spi_sei_generic(hdev, event_type, mmu_base,
 							is_pmmu, event_mask);
 	hl_check_for_glbl_errors(hdev);
@@ -8740,12 +9301,12 @@ static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type,
 	if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) {
 		gaudi2_print_event(hdev, event_type, true,
 			"err cause: %s",
-			"Invalid HBM SEI event cause (%d) provided by FW\n", cause_idx);
+			"Invalid HBM SEI event cause (%d) provided by FW", cause_idx);
 		return true;
 	}
 
 	gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical,
-		"System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s\n",
+		"System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s",
 		sei_data->hdr.is_critical ? "Critical" : "Non-critical",
 		hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel,
 		hbm_mc_sei_cause[cause_idx]);
@@ -8869,7 +9430,7 @@ static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type
 	struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
 
 	gaudi2_print_event(hdev, event_type, false,
-		"FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n",
+		"FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d",
 		le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci),
 		q->pi, atomic_read(&q->ci));
 }
@@ -8883,7 +9444,7 @@ static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type)
 
 	if (p2p_intr) {
 		gaudi2_print_event(hdev, event_type, true,
-			"pcie p2p transaction terminated due to security, req_id(0x%x)\n",
+			"pcie p2p transaction terminated due to security, req_id(0x%x)",
 			RREG32(mmPCIE_WRAP_P2P_REQ_ID));
 
 		WREG32(mmPCIE_WRAP_P2P_INTR, 0x1);
@@ -8892,7 +9453,7 @@ static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type)
 
 	if (msix_gw_intr) {
 		gaudi2_print_event(hdev, event_type, true,
-			"pcie msi-x gen denied due to vector num check failure, vec(0x%X)\n",
+			"pcie msi-x gen denied due to vector num check failure, vec(0x%X)",
 			RREG32(mmPCIE_WRAP_MSIX_GW_VEC));
 
 		WREG32(mmPCIE_WRAP_MSIX_GW_INTR, 0x1);
@@ -8954,7 +9515,7 @@ static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_
 	struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
 
 	gaudi2_print_event(hdev, event_type, false,
-		"FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n",
+		"FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d",
 		le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci));
 }
 
@@ -8974,11 +9535,11 @@ static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type,
 		q = (struct hl_engine_arc_dccm_queue_full_irq *) &payload;
 
 		gaudi2_print_event(hdev, event_type, true,
-				"ARC DCCM Full event: EngId: %u, Intr_type: %u, Qidx: %u\n",
+				"ARC DCCM Full event: EngId: %u, Intr_type: %u, Qidx: %u",
 				engine_id, intr_type, q->queue_index);
 		return 1;
 	default:
-		gaudi2_print_event(hdev, event_type, true, "Unknown ARC event type\n");
+		gaudi2_print_event(hdev, event_type, true, "Unknown ARC event type");
 		return 0;
 	}
 }
@@ -8987,7 +9548,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 {
 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
 	bool reset_required = false, is_critical = false;
-	u32 index, ctl, reset_flags = HL_DRV_RESET_HARD, error_count = 0;
+	u32 index, ctl, reset_flags = 0, error_count = 0;
 	u64 event_mask = 0;
 	u16 event_type;
 
@@ -9024,19 +9585,18 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 		break;
 
 	case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0:
-		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
 		error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type);
-		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
+		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
 		break;
 
 	case GAUDI2_EVENT_CPU_AXI_ERR_RSP:
 		error_count = gaudi2_handle_cpu_sei_err(hdev, event_type);
-		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
+		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
+		event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR;
 		break;
 
 	case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP:
 	case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP:
-		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
 		error_count = gaudi2_handle_qm_sei_err(hdev, event_type, true, &event_mask);
 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
 		break;
@@ -9153,9 +9713,15 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
 		break;
 
-	case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_PDMA1_CORE:
-		error_count = gaudi2_handle_dma_core_event(hdev, event_type,
-					le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
+	case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_HDMA5_CORE:
+		index = event_type - GAUDI2_EVENT_HDMA2_CORE;
+		error_count = gaudi2_handle_edma_core_event(hdev, event_type, index);
+		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
+		break;
+
+	case GAUDI2_EVENT_PDMA0_CORE ... GAUDI2_EVENT_PDMA1_CORE:
+		index = event_type - GAUDI2_EVENT_PDMA0_CORE;
+		error_count = gaudi2_handle_pdma_core_event(hdev, event_type, index);
 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
 		break;
 
@@ -9217,12 +9783,14 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 
 	case GAUDI2_EVENT_PCIE_DRAIN_COMPLETE:
 		error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data);
+		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
 		break;
 
 	case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN:
 		error_count = gaudi2_handle_psoc_drain(hdev,
 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
+		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
 		break;
 
@@ -9251,6 +9819,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 		break;
 	case GAUDI2_EVENT_PSOC_AXI_ERR_RSP:
 		error_count = GAUDI2_NA_EVENT_CAUSE;
+		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
 		break;
 	case GAUDI2_EVENT_PSOC_PRSTN_FALL:
@@ -9264,6 +9833,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 		break;
 	case GAUDI2_EVENT_PCIE_FATAL_ERR:
 		error_count = GAUDI2_NA_EVENT_CAUSE;
+		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
 		break;
 	case GAUDI2_EVENT_TPC0_BMON_SPMU:
@@ -9331,6 +9901,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 	case GAUDI2_EVENT_CPU_PKT_QUEUE_OUT_SYNC:
 		gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err);
 		error_count = GAUDI2_NA_EVENT_CAUSE;
+		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
 		break;
 
@@ -9372,6 +9943,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 	case GAUDI2_EVENT_CPU_PKT_SANITY_FAILED:
 		gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err);
 		error_count = GAUDI2_NA_EVENT_CAUSE;
+		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
 		break;
 
@@ -9381,7 +9953,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 		break;
 
 	case GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED:
-	case GAUDI2_EVENT_DEV_RESET_REQ:
+	case GAUDI2_EVENT_CPU_DEV_RESET_REQ:
 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
 		error_count = GAUDI2_NA_EVENT_CAUSE;
 		is_critical = true;
@@ -9403,12 +9975,18 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 		gaudi2_print_event(hdev, event_type, true, "%d", event_type);
 	else if (error_count == 0)
 		gaudi2_print_event(hdev, event_type, true,
-				"No error cause for H/W event %u\n", event_type);
+				"No error cause for H/W event %u", event_type);
 
-	if ((gaudi2_irq_map_table[event_type].reset || reset_required) &&
-				(hdev->hard_reset_on_fw_events ||
-				(hdev->asic_prop.fw_security_enabled && is_critical)))
-		goto reset_device;
+	if ((gaudi2_irq_map_table[event_type].reset != EVENT_RESET_TYPE_NONE) ||
+				reset_required) {
+		if (reset_required ||
+				(gaudi2_irq_map_table[event_type].reset == EVENT_RESET_TYPE_HARD))
+			reset_flags |= HL_DRV_RESET_HARD;
+
+		if (hdev->hard_reset_on_fw_events ||
+				(hdev->asic_prop.fw_security_enabled && is_critical))
+			goto reset_device;
+	}
 
 	/* Send unmask irq only for interrupts not classified as MSG */
 	if (!gaudi2_irq_map_table[event_type].msg)
@@ -9426,6 +10004,10 @@ reset_device:
 	} else {
 		reset_flags |= HL_DRV_RESET_DELAY;
 	}
+	/* escalate general hw errors to critical/fatal error */
+	if (event_mask & HL_NOTIFIER_EVENT_GENERAL_HW_ERR)
+		hl_handle_critical_hw_err(hdev, event_type, &event_mask);
+
 	event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET;
 	hl_device_cond_reset(hdev, reset_flags, event_mask);
 }
@@ -9832,16 +10414,23 @@ static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, v
 
 	/* Create mapping on asic side */
 	mutex_lock(&hdev->mmu_lock);
+
 	rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M);
-	hl_mmu_invalidate_cache_range(hdev, false,
+	if (rc) {
+		dev_err(hdev->dev, "Failed to create mapping on asic mmu\n");
+		goto unreserve_va;
+	}
+
+	rc = hl_mmu_invalidate_cache_range(hdev, false,
 				      MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV,
 				      ctx->asid, reserved_va_base, SZ_2M);
-	mutex_unlock(&hdev->mmu_lock);
 	if (rc) {
-		dev_err(hdev->dev, "Failed to create mapping on asic mmu\n");
+		hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M);
 		goto unreserve_va;
 	}
 
+	mutex_unlock(&hdev->mmu_lock);
+
 	/* Enable MMU on KDMA */
 	gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid);
 
@@ -9870,11 +10459,16 @@ static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, v
 	gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID);
 
 	mutex_lock(&hdev->mmu_lock);
-	hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M);
-	hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR,
+
+	rc = hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M);
+	if (rc)
+		goto unreserve_va;
+
+	rc = hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR,
 				      ctx->asid, reserved_va_base, SZ_2M);
-	mutex_unlock(&hdev->mmu_lock);
+
 unreserve_va:
+	mutex_unlock(&hdev->mmu_lock);
 	hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M);
 free_data_buffer:
 	hl_asic_dma_free_coherent(hdev, SZ_2M, host_mem_virtual_addr, host_mem_dma_addr);
@@ -9927,17 +10521,24 @@ static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *c
 	}
 
 	mutex_lock(&hdev->mmu_lock);
+
 	rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr,
 					HOST_SPACE_INTERNAL_CB_SZ);
-	hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
-	mutex_unlock(&hdev->mmu_lock);
-
 	if (rc)
 		goto unreserve_internal_cb_pool;
 
+	rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
+	if (rc)
+		goto unmap_internal_cb_pool;
+
+	mutex_unlock(&hdev->mmu_lock);
+
 	return 0;
 
+unmap_internal_cb_pool:
+	hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
 unreserve_internal_cb_pool:
+	mutex_unlock(&hdev->mmu_lock);
 	hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
 destroy_internal_cb_pool:
 	gen_pool_destroy(hdev->internal_cb_pool);
@@ -10724,6 +11325,7 @@ static const struct hl_asic_funcs gaudi2_funcs = {
 	.access_dev_mem = hl_access_dev_mem,
 	.set_dram_bar_base = gaudi2_set_hbm_bar_base,
 	.set_engine_cores = gaudi2_set_engine_cores,
+	.set_engines = gaudi2_set_engines,
 	.send_device_activity = gaudi2_send_device_activity,
 	.set_dram_properties = gaudi2_set_dram_properties,
 	.set_binning_masks = gaudi2_set_binning_masks,
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2P.h b/drivers/accel/habanalabs/gaudi2/gaudi2P.h
index 2687404d9d21..1cebe707772e 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2P.h
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2P.h
@@ -240,6 +240,8 @@
 #define GAUDI2_SOB_INCREMENT_BY_ONE	(FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1) | \
 					FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1))
 
+#define GAUDI2_NUM_TESTED_QS (GAUDI2_QUEUE_ID_CPU_PQ - GAUDI2_QUEUE_ID_PDMA_0_0)
+
 #define GAUDI2_NUM_OF_GLBL_ERR_CAUSE		8
 
 enum gaudi2_reserved_sob_id {
@@ -387,6 +389,8 @@ enum gaudi2_edma_id {
  * We have 64 CQ's per dcore, CQ0 in dcore 0 is reserved for legacy mode
  */
 #define GAUDI2_NUM_USER_INTERRUPTS 255
+#define GAUDI2_NUM_RESERVED_INTERRUPTS 1
+#define GAUDI2_TOTAL_USER_INTERRUPTS (GAUDI2_NUM_USER_INTERRUPTS + GAUDI2_NUM_RESERVED_INTERRUPTS)
 
 enum gaudi2_irq_num {
 	GAUDI2_IRQ_NUM_EVENT_QUEUE = GAUDI2_EVENT_QUEUE_MSIX_IDX,
@@ -410,12 +414,15 @@ enum gaudi2_irq_num {
 	GAUDI2_IRQ_NUM_SHARED_DEC0_ABNRM,
 	GAUDI2_IRQ_NUM_SHARED_DEC1_NRM,
 	GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM,
+	GAUDI2_IRQ_NUM_DEC_LAST = GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM,
 	GAUDI2_IRQ_NUM_COMPLETION,
 	GAUDI2_IRQ_NUM_NIC_PORT_FIRST,
 	GAUDI2_IRQ_NUM_NIC_PORT_LAST = (GAUDI2_IRQ_NUM_NIC_PORT_FIRST + NIC_NUMBER_OF_PORTS - 1),
+	GAUDI2_IRQ_NUM_TPC_ASSERT,
 	GAUDI2_IRQ_NUM_RESERVED_FIRST,
-	GAUDI2_IRQ_NUM_RESERVED_LAST = (GAUDI2_MSIX_ENTRIES - GAUDI2_NUM_USER_INTERRUPTS - 1),
-	GAUDI2_IRQ_NUM_USER_FIRST,
+	GAUDI2_IRQ_NUM_RESERVED_LAST = (GAUDI2_MSIX_ENTRIES - GAUDI2_TOTAL_USER_INTERRUPTS - 1),
+	GAUDI2_IRQ_NUM_UNEXPECTED_ERROR = RESERVED_MSIX_UNEXPECTED_USER_ERROR_INTERRUPT,
+	GAUDI2_IRQ_NUM_USER_FIRST = GAUDI2_IRQ_NUM_UNEXPECTED_ERROR + 1,
 	GAUDI2_IRQ_NUM_USER_LAST = (GAUDI2_IRQ_NUM_USER_FIRST + GAUDI2_NUM_USER_INTERRUPTS - 1),
 	GAUDI2_IRQ_NUM_LAST = (GAUDI2_MSIX_ENTRIES - 1)
 };
@@ -448,6 +455,17 @@ struct dup_block_ctx {
 };
 
 /**
+ * struct gaudi2_queues_test_info - Holds the address of a the messages used for testing the
+ *                                  device queues.
+ * @dma_addr: the address used by the HW for accessing the message.
+ * @kern_addr: The address used by the driver for accessing the message.
+ */
+struct gaudi2_queues_test_info {
+	dma_addr_t dma_addr;
+	void *kern_addr;
+};
+
+/**
  * struct gaudi2_device - ASIC specific manage structure.
  * @cpucp_info_get: get information on device from CPU-CP
  * @mapped_blocks: array that holds the base address and size of all blocks
@@ -505,6 +523,7 @@ struct dup_block_ctx {
  * @flush_db_fifo: flag to force flush DB FIFO after a write.
  * @hbm_cfg: HBM subsystem settings
  * @hw_queues_lock_mutex: used by simulator instead of hw_queues_lock.
+ * @queues_test_info: information used by the driver when testing the HW queues.
  */
 struct gaudi2_device {
 	int (*cpucp_info_get)(struct hl_device *hdev);
@@ -532,6 +551,9 @@ struct gaudi2_device {
 	u32				events_stat[GAUDI2_EVENT_SIZE];
 	u32				events_stat_aggregate[GAUDI2_EVENT_SIZE];
 	u32				num_of_valid_hw_events;
+
+	/* Queue testing */
+	struct gaudi2_queues_test_info	queues_test_info[GAUDI2_NUM_TESTED_QS];
 };
 
 /*
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c b/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c
index 1dfbe293ecec..25b5368f37dd 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c
@@ -2657,7 +2657,7 @@ int gaudi2_coresight_init(struct hl_device *hdev)
 	/*
 	 * Mask out all the disabled binned offsets.
 	 * so when user request to configure a binned or masked out component,
-	 * driver will ignore programing it ( happens when offset value is set to 0x0 )
+	 * driver will ignore programming it ( happens when offset value is set to 0x0 )
 	 * this is being set in gaudi2_coresight_set_disabled_components
 	 */
 
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2_masks.h b/drivers/accel/habanalabs/gaudi2/gaudi2_masks.h
index e9ac87828221..e6664c4a2cf5 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2_masks.h
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2_masks.h
@@ -79,7 +79,6 @@
 			DCORE0_MME_CTRL_LO_ARCH_STATUS_QM_RDY_MASK)
 
 #define TPC_IDLE_MASK	(DCORE0_TPC0_CFG_STATUS_SCALAR_PIPE_EMPTY_MASK | \
-			DCORE0_TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK | \
 			DCORE0_TPC0_CFG_STATUS_IQ_EMPTY_MASK | \
 			DCORE0_TPC0_CFG_STATUS_SB_EMPTY_MASK | \
 			DCORE0_TPC0_CFG_STATUS_QM_IDLE_MASK | \
@@ -87,6 +86,8 @@
 
 #define DCORE0_TPC0_QM_CGM_STS_AGENT_IDLE_MASK 0x100
 
+#define DCORE0_TPC0_EML_CFG_DBG_CNT_DBG_EXIT_MASK 0x40
+
 /* CGM_IDLE_MASK is valid for all engines CGM idle check */
 #define CGM_IDLE_MASK	DCORE0_TPC0_QM_CGM_STS_AGENT_IDLE_MASK
 
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2_security.c b/drivers/accel/habanalabs/gaudi2/gaudi2_security.c
index a212f82e6604..694735f9e6e6 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2_security.c
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2_security.c
@@ -1595,6 +1595,7 @@ static const u32 gaudi2_pb_dcr0_tpc0_unsecured_regs[] = {
 	mmDCORE0_TPC0_CFG_KERNEL_SRF_30,
 	mmDCORE0_TPC0_CFG_KERNEL_SRF_31,
 	mmDCORE0_TPC0_CFG_TPC_SB_L0CD,
+	mmDCORE0_TPC0_CFG_TPC_ID,
 	mmDCORE0_TPC0_CFG_QM_KERNEL_ID_INC,
 	mmDCORE0_TPC0_CFG_QM_TID_BASE_SIZE_HIGH_DIM_0,
 	mmDCORE0_TPC0_CFG_QM_TID_BASE_SIZE_HIGH_DIM_1,
diff --git a/drivers/accel/habanalabs/goya/goya.c b/drivers/accel/habanalabs/goya/goya.c
index df65e9bdc18a..fb0ac9df841a 100644
--- a/drivers/accel/habanalabs/goya/goya.c
+++ b/drivers/accel/habanalabs/goya/goya.c
@@ -472,6 +472,8 @@ int goya_set_fixed_properties(struct hl_device *hdev)
 	prop->max_pending_cs = GOYA_MAX_PENDING_CS;
 
 	prop->first_available_user_interrupt = USHRT_MAX;
+	prop->tpc_interrupt_id = USHRT_MAX;
+	prop->eq_interrupt_id = GOYA_EVENT_QUEUE_MSIX_IDX;
 
 	for (i = 0 ; i < HL_MAX_DCORES ; i++)
 		prop->first_available_cq[i] = USHRT_MAX;
@@ -668,13 +670,18 @@ pci_init:
 	rc = hl_fw_read_preboot_status(hdev);
 	if (rc) {
 		if (hdev->reset_on_preboot_fail)
+			/* we are already on failure flow, so don't check if hw_fini fails. */
 			hdev->asic_funcs->hw_fini(hdev, true, false);
 		goto pci_fini;
 	}
 
 	if (goya_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
 		dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n");
-		hdev->asic_funcs->hw_fini(hdev, true, false);
+		rc = hdev->asic_funcs->hw_fini(hdev, true, false);
+		if (rc) {
+			dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc);
+			goto pci_fini;
+		}
 	}
 
 	if (!hdev->pldm) {
@@ -2782,7 +2789,7 @@ disable_queues:
 	return rc;
 }
 
-static void goya_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
+static int goya_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
 {
 	struct goya_device *goya = hdev->asic_specific;
 	u32 reset_timeout_ms, cpu_timeout_ms, status;
@@ -2828,17 +2835,17 @@ static void goya_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
 	msleep(reset_timeout_ms);
 
 	status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM);
-	if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK)
-		dev_err(hdev->dev,
-			"Timeout while waiting for device to reset 0x%x\n",
-			status);
+	if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK) {
+		dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", status);
+		return -ETIMEDOUT;
+	}
 
 	if (!hard_reset && goya) {
 		goya->hw_cap_initialized &= ~(HW_CAP_DMA | HW_CAP_MME |
 						HW_CAP_GOLDEN | HW_CAP_TPC);
 		WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
 				GOYA_ASYNC_EVENT_ID_SOFT_RESET);
-		return;
+		return 0;
 	}
 
 	/* Chicken bit to re-initiate boot sequencer flow */
@@ -2857,6 +2864,7 @@ static void goya_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
 
 		memset(goya->events_stat, 0, sizeof(goya->events_stat));
 	}
+	return 0;
 }
 
 int goya_suspend(struct hl_device *hdev)
diff --git a/drivers/accel/habanalabs/include/common/cpucp_if.h b/drivers/accel/habanalabs/include/common/cpucp_if.h
index d713252a4f13..8bbe685458c4 100644
--- a/drivers/accel/habanalabs/include/common/cpucp_if.h
+++ b/drivers/accel/habanalabs/include/common/cpucp_if.h
@@ -357,6 +357,7 @@ struct hl_eq_addr_dec_intr_data {
 struct hl_eq_entry {
 	struct hl_eq_header hdr;
 	union {
+		__le64 data_placeholder;
 		struct hl_eq_ecc_data ecc_data;
 		struct hl_eq_hbm_ecc_data hbm_ecc_data;	/* Gaudi1 HBM */
 		struct hl_eq_sm_sei_data sm_sei_data;
@@ -661,6 +662,9 @@ enum pq_init_status {
  * CPUCP_PACKET_ACTIVE_STATUS_SET -
  *       LKD sends FW indication whether device is free or in use, this indication is reported
  *       also to the BMC.
+ *
+ * CPUCP_PACKET_REGISTER_INTERRUPTS -
+ *       Packet to register interrupts indicating LKD is ready to receive events from FW.
  */
 
 enum cpucp_packet_id {
@@ -725,6 +729,8 @@ enum cpucp_packet_id {
 	CPUCP_PACKET_RESERVED9,			/* not used */
 	CPUCP_PACKET_RESERVED10,		/* not used */
 	CPUCP_PACKET_RESERVED11,		/* not used */
+	CPUCP_PACKET_RESERVED12,		/* internal */
+	CPUCP_PACKET_REGISTER_INTERRUPTS,	/* internal */
 	CPUCP_PACKET_ID_MAX			/* must be last */
 };
 
@@ -1127,6 +1133,7 @@ struct cpucp_security_info {
  *                     (0 = functional 1 = binned)
  * @interposer_version: Interposer version programmed in eFuse
  * @substrate_version: Substrate version programmed in eFuse
+ * @fw_hbm_region_size: Size in bytes of FW reserved region in HBM.
  * @fw_os_version: Firmware OS Version
  */
 struct cpucp_info {
@@ -1154,7 +1161,7 @@ struct cpucp_info {
 	__u8 substrate_version;
 	__u8 reserved2;
 	struct cpucp_security_info sec_info;
-	__le32 reserved3;
+	__le32 fw_hbm_region_size;
 	__u8 pll_map[PLL_MAP_LEN];
 	__le64 mme_binning_mask;
 	__u8 fw_os_version[VERSION_MAX_LEN];
diff --git a/drivers/accel/habanalabs/include/common/hl_boot_if.h b/drivers/accel/habanalabs/include/common/hl_boot_if.h
index 2256add057c5..c58d76a2705c 100644
--- a/drivers/accel/habanalabs/include/common/hl_boot_if.h
+++ b/drivers/accel/habanalabs/include/common/hl_boot_if.h
@@ -770,15 +770,23 @@ enum hl_components {
 	HL_COMPONENTS_ARMCP,
 	HL_COMPONENTS_CPLD,
 	HL_COMPONENTS_UBOOT,
+	HL_COMPONENTS_FUSE,
 	HL_COMPONENTS_MAX_NUM = 16
 };
 
+#define NAME_MAX_LEN	32 /* bytes */
+struct hl_module_data {
+	__u8 name[NAME_MAX_LEN];
+	__u8 version[VERSION_MAX_LEN];
+};
+
 /**
  * struct hl_component_versions - versions associated with hl component.
  * @struct_size: size of all the struct (including dynamic size of modules).
  * @modules_offset: offset of the modules field in this struct.
  * @component: version of the component itself.
  * @fw_os: Firmware OS Version.
+ * @comp_name: Name of the component.
  * @modules_mask: i'th bit (from LSB) is a flag - on if module i in enum
  *              hl_modules is used.
  * @modules_counter: number of set bits in modules_mask.
@@ -791,45 +799,14 @@ struct hl_component_versions {
 	__le16 modules_offset;
 	__u8 component[VERSION_MAX_LEN];
 	__u8 fw_os[VERSION_MAX_LEN];
+	__u8 comp_name[NAME_MAX_LEN];
 	__le16 modules_mask;
 	__u8 modules_counter;
 	__u8 reserved[1];
-	__u8 modules[][VERSION_MAX_LEN];
-};
-
-/**
- * struct hl_fw_versions - all versions (fuse, cpucp's components with their
- *              modules)
- * @struct_size: size of all the struct (including dynamic size of components).
- * @components_offset: offset of the components field in this struct.
- * @fuse: silicon production FUSE information.
- * @components_mask: i'th bit (from LSB) is a flag - on if component i in enum
- *              hl_components is used.
- * @components_counter: number of set bits in components_mask.
- * @reserved: reserved for future use.
- * @components: versions of hl components. Index i corresponds to the i'th bit
- *              that is *on* in components_mask. For example, if
- *              components_mask=0b101, then *components represents arcpid and
- *              *(hl_component_versions*)((char*)components + 1') represents
- *              preboot, where 1' = components[0].struct_size.
- */
-struct hl_fw_versions {
-	__le16 struct_size;
-	__le16 components_offset;
-	__u8 fuse[VERSION_MAX_LEN];
-	__le16 components_mask;
-	__u8 components_counter;
-	__u8 reserved[1];
-	struct hl_component_versions components[];
+	struct hl_module_data modules[];
 };
 
-/* Max size of struct hl_component_versions */
-#define HL_COMPONENT_VERSIONS_MAX_SIZE \
-	(sizeof(struct hl_component_versions) + HL_MODULES_MAX_NUM * \
-	 VERSION_MAX_LEN)
-
-/* Max size of struct hl_fw_versions */
-#define HL_FW_VERSIONS_MAX_SIZE (sizeof(struct hl_fw_versions) + \
-		HL_COMPONENTS_MAX_NUM * HL_COMPONENT_VERSIONS_MAX_SIZE)
+/* Max size of fit size */
+#define HL_FW_VERSIONS_FIT_SIZE	4096
 
 #endif /* HL_BOOT_IF_H */
diff --git a/drivers/accel/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h b/drivers/accel/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h
index 0bf3092bfeea..6c58af614236 100644
--- a/drivers/accel/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h
+++ b/drivers/accel/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0
  *
- * Copyright 2020-2022 HabanaLabs, Ltd.
+ * Copyright 2020-2023 HabanaLabs, Ltd.
  * All Rights Reserved.
  *
  */
@@ -164,6 +164,8 @@
 
 #define mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR	0x4800040
 
+#define mmDCORE0_TPC0_EML_CFG_DBG_CNT		0x40000
+
 #define SM_OBJS_PROT_BITS_OFFS			0x14000
 
 #define DCORE_OFFSET			(mmDCORE1_TPC0_QM_BASE - mmDCORE0_TPC0_QM_BASE)
@@ -185,7 +187,10 @@
 #define TPC_CFG_STALL_ON_ERR_OFFSET	(mmDCORE0_TPC0_CFG_STALL_ON_ERR - mmDCORE0_TPC0_CFG_BASE)
 #define TPC_CFG_TPC_INTR_MASK_OFFSET	(mmDCORE0_TPC0_CFG_TPC_INTR_MASK - mmDCORE0_TPC0_CFG_BASE)
 #define TPC_CFG_MSS_CONFIG_OFFSET	(mmDCORE0_TPC0_CFG_MSS_CONFIG - mmDCORE0_TPC0_CFG_BASE)
+#define TPC_EML_CFG_DBG_CNT_OFFSET	(mmDCORE0_TPC0_EML_CFG_DBG_CNT - mmDCORE0_TPC0_EML_CFG_BASE)
 
+#define EDMA_CORE_CFG_STALL_OFFSET	(mmDCORE0_EDMA0_CORE_CFG_1 - mmDCORE0_EDMA0_CORE_BASE)
+#define MME_CTRL_LO_QM_STALL_OFFSET	(mmDCORE0_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_BASE)
 #define MME_ACC_INTR_MASK_OFFSET	(mmDCORE0_MME_ACC_INTR_MASK - mmDCORE0_MME_ACC_BASE)
 #define MME_ACC_WR_AXI_AGG_COUT0_OFFSET	(mmDCORE0_MME_ACC_WR_AXI_AGG_COUT0 - mmDCORE0_MME_ACC_BASE)
 #define MME_ACC_WR_AXI_AGG_COUT1_OFFSET	(mmDCORE0_MME_ACC_WR_AXI_AGG_COUT1 - mmDCORE0_MME_ACC_BASE)
@@ -538,6 +543,8 @@
 #define HBM_MC_SPI_IEEE1500_COMP_MASK		BIT(3)
 #define HBM_MC_SPI_IEEE1500_PAUSED_MASK		BIT(4)
 
+#define ARC_FARM_OFFSET (mmARC_FARM_ARC1_AUX_BASE - mmARC_FARM_ARC0_AUX_BASE)
+
 #include "nic0_qpc0_regs.h"
 #include "nic0_qm0_regs.h"
 #include "nic0_qm_arc_aux0_regs.h"
diff --git a/drivers/accel/habanalabs/include/gaudi2/gaudi2.h b/drivers/accel/habanalabs/include/gaudi2/gaudi2.h
index 5b4f9e108798..0231d6c55b4a 100644
--- a/drivers/accel/habanalabs/include/gaudi2/gaudi2.h
+++ b/drivers/accel/habanalabs/include/gaudi2/gaudi2.h
@@ -63,6 +63,8 @@
 #define RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_START	0xFFF0F80000000000ull
 #define RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_END	0xFFF0FFFFFFFFFFFFull
 
+#define RESERVED_MSIX_UNEXPECTED_USER_ERROR_INTERRUPT	256
+
 #define GAUDI2_MSIX_ENTRIES	512
 
 #define QMAN_PQ_ENTRY_SIZE	16			/* Bytes */
diff --git a/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_events.h b/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_events.h
index 50852cc80373..f661068d0c5f 100644
--- a/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_events.h
+++ b/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_events.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0
  *
- * Copyright 2018-2021 HabanaLabs, Ltd.
+ * Copyright 2018-2022 HabanaLabs, Ltd.
  * All Rights Reserved.
  *
  */
@@ -958,7 +958,7 @@ enum gaudi2_async_event_id {
 	GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1 = 1318,
 	GAUDI2_EVENT_ARC_DCCM_FULL = 1319,
 	GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED = 1320,
-	GAUDI2_EVENT_DEV_RESET_REQ = 1321,
+	GAUDI2_EVENT_CPU_DEV_RESET_REQ = 1321,
 	GAUDI2_EVENT_SIZE,
 };
 
diff --git a/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h b/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h
index 82be01bea98e..ad01fc4e9940 100644
--- a/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h
+++ b/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h
@@ -10,6 +10,12 @@
  **       DO NOT EDIT BELOW        **
  ************************************/
 
+enum event_reset_type {
+	EVENT_RESET_TYPE_NONE,
+	EVENT_RESET_TYPE_COMPUTE,
+	EVENT_RESET_TYPE_HARD,
+};
+
 #ifndef __GAUDI2_ASYNC_IDS_MAP_EVENTS_EXT_H_
 #define __GAUDI2_ASYNC_IDS_MAP_EVENTS_EXT_H_
 
@@ -23,2650 +29,2650 @@ struct gaudi2_async_events_ids_map {
 };
 
 static struct gaudi2_async_events_ids_map gaudi2_irq_map_table[] = {
-	{ .fc_id = 0, .cpu_id = 0, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1, .cpu_id = 1, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 2, .cpu_id = 2, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 3, .cpu_id = 3, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 4, .cpu_id = 4, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 5, .cpu_id = 5, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 6, .cpu_id = 6, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 7, .cpu_id = 7, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 8, .cpu_id = 8, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 9, .cpu_id = 9, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 10, .cpu_id = 10, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 11, .cpu_id = 11, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 12, .cpu_id = 12, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 13, .cpu_id = 13, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 14, .cpu_id = 14, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 15, .cpu_id = 15, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 16, .cpu_id = 16, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 17, .cpu_id = 17, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 18, .cpu_id = 18, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 19, .cpu_id = 19, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 20, .cpu_id = 20, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 21, .cpu_id = 21, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 22, .cpu_id = 22, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 23, .cpu_id = 23, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 24, .cpu_id = 24, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 25, .cpu_id = 25, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 26, .cpu_id = 26, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 27, .cpu_id = 27, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 28, .cpu_id = 28, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 29, .cpu_id = 29, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 30, .cpu_id = 30, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 31, .cpu_id = 31, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 32, .cpu_id = 32, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PCIE_CORE_SERR" },
-	{ .fc_id = 33, .cpu_id = 33, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PCIE_CORE_DERR" },
-	{ .fc_id = 34, .cpu_id = 34, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PCIE_IF_SERR" },
-	{ .fc_id = 35, .cpu_id = 35, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PCIE_IF_DERR" },
-	{ .fc_id = 36, .cpu_id = 36, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PCIE_PHY_SERR" },
-	{ .fc_id = 37, .cpu_id = 37, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PCIE_PHY_DERR" },
-	{ .fc_id = 38, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC0_ECC_SERR" },
-	{ .fc_id = 39, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC1_ECC_SERR" },
-	{ .fc_id = 40, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC2_ECC_SERR" },
-	{ .fc_id = 41, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC3_ECC_SERR" },
-	{ .fc_id = 42, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC4_ECC_SERR" },
-	{ .fc_id = 43, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC5_ECC_SERR" },
-	{ .fc_id = 44, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC6_ECC_SERR" },
-	{ .fc_id = 45, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC7_ECC_SERR" },
-	{ .fc_id = 46, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC8_ECC_SERR" },
-	{ .fc_id = 47, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC9_ECC_SERR" },
-	{ .fc_id = 48, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC10_ECC_SERR" },
-	{ .fc_id = 49, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC11_ECC_SERR" },
-	{ .fc_id = 50, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC12_ECC_SERR" },
-	{ .fc_id = 51, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC13_ECC_SERR" },
-	{ .fc_id = 52, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC14_ECC_SERR" },
-	{ .fc_id = 53, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC15_ECC_SERR" },
-	{ .fc_id = 54, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC16_ECC_SERR" },
-	{ .fc_id = 55, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC17_ECC_SERR" },
-	{ .fc_id = 56, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC18_ECC_SERR" },
-	{ .fc_id = 57, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC19_ECC_SERR" },
-	{ .fc_id = 58, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC20_ECC_SERR" },
-	{ .fc_id = 59, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC21_ECC_SERR" },
-	{ .fc_id = 60, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC22_ECC_SERR" },
-	{ .fc_id = 61, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC23_ECC_SERR" },
-	{ .fc_id = 62, .cpu_id = 38, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC24_ECC_SERR" },
-	{ .fc_id = 63, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC0_ECC_DERR" },
-	{ .fc_id = 64, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC1_ECC_DERR" },
-	{ .fc_id = 65, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC2_ECC_DERR" },
-	{ .fc_id = 66, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC3_ECC_DERR" },
-	{ .fc_id = 67, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC4_ECC_DERR" },
-	{ .fc_id = 68, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC5_ECC_DERR" },
-	{ .fc_id = 69, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC6_ECC_DERR" },
-	{ .fc_id = 70, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC7_ECC_DERR" },
-	{ .fc_id = 71, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC8_ECC_DERR" },
-	{ .fc_id = 72, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC9_ECC_DERR" },
-	{ .fc_id = 73, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC10_ECC_DERR" },
-	{ .fc_id = 74, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC11_ECC_DERR" },
-	{ .fc_id = 75, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC12_ECC_DERR" },
-	{ .fc_id = 76, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC13_ECC_DERR" },
-	{ .fc_id = 77, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC14_ECC_DERR" },
-	{ .fc_id = 78, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC15_ECC_DERR" },
-	{ .fc_id = 79, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC16_ECC_DERR" },
-	{ .fc_id = 80, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC17_ECC_DERR" },
-	{ .fc_id = 81, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC18_ECC_DERR" },
-	{ .fc_id = 82, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC19_ECC_DERR" },
-	{ .fc_id = 83, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC20_ECC_DERR" },
-	{ .fc_id = 84, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC21_ECC_DERR" },
-	{ .fc_id = 85, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC22_ECC_DERR" },
-	{ .fc_id = 86, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC23_ECC_DERR" },
-	{ .fc_id = 87, .cpu_id = 39, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC24_ECC_DERR" },
-	{ .fc_id = 88, .cpu_id = 40, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_SBTE0_ECC_SERR" },
-	{ .fc_id = 89, .cpu_id = 40, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_SBTE1_ECC_SERR" },
-	{ .fc_id = 90, .cpu_id = 40, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_SBTE2_ECC_SERR" },
-	{ .fc_id = 91, .cpu_id = 40, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_SBTE3_ECC_SERR" },
-	{ .fc_id = 92, .cpu_id = 40, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_SBTE4_ECC_SERR" },
-	{ .fc_id = 93, .cpu_id = 40, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_CTRL_ECC_SERR" },
-	{ .fc_id = 94, .cpu_id = 40, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_WAP_ECC_SERR" },
-	{ .fc_id = 95, .cpu_id = 41, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_SBTE0_ECC_SERR" },
-	{ .fc_id = 96, .cpu_id = 41, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_SBTE1_ECC_SERR" },
-	{ .fc_id = 97, .cpu_id = 41, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_SBTE2_ECC_SERR" },
-	{ .fc_id = 98, .cpu_id = 41, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_SBTE3_ECC_SERR" },
-	{ .fc_id = 99, .cpu_id = 41, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_SBTE4_ECC_SERR" },
-	{ .fc_id = 100, .cpu_id = 41, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_CTRL_ECC_SERR" },
-	{ .fc_id = 101, .cpu_id = 41, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_WAP_ECC_SERR" },
-	{ .fc_id = 102, .cpu_id = 42, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_SBTE0_ECC_SERR" },
-	{ .fc_id = 103, .cpu_id = 42, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_SBTE1_ECC_SERR" },
-	{ .fc_id = 104, .cpu_id = 42, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_SBTE2_ECC_SERR" },
-	{ .fc_id = 105, .cpu_id = 42, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_SBTE3_ECC_SERR" },
-	{ .fc_id = 106, .cpu_id = 42, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_SBTE4_ECC_SERR" },
-	{ .fc_id = 107, .cpu_id = 42, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_CTRL_ECC_SERR" },
-	{ .fc_id = 108, .cpu_id = 42, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_WAP_ECC_SERR" },
-	{ .fc_id = 109, .cpu_id = 43, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_SBTE0_ECC_SERR" },
-	{ .fc_id = 110, .cpu_id = 43, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_SBTE1_ECC_SERR" },
-	{ .fc_id = 111, .cpu_id = 43, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_SBTE2_ECC_SERR" },
-	{ .fc_id = 112, .cpu_id = 43, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_SBTE3_ECC_SERR" },
-	{ .fc_id = 113, .cpu_id = 43, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_SBTE4_ECC_SERR" },
-	{ .fc_id = 114, .cpu_id = 43, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_CTRL_ECC_SERR" },
-	{ .fc_id = 115, .cpu_id = 43, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_WAP_ECC_SERR" },
-	{ .fc_id = 116, .cpu_id = 44, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_SBTE0_ECC_DERR" },
-	{ .fc_id = 117, .cpu_id = 44, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_SBTE1_ECC_DERR" },
-	{ .fc_id = 118, .cpu_id = 44, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_SBTE2_ECC_DERR" },
-	{ .fc_id = 119, .cpu_id = 44, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_SBTE3_ECC_DERR" },
-	{ .fc_id = 120, .cpu_id = 44, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_SBTE4_ECC_DERR" },
-	{ .fc_id = 121, .cpu_id = 44, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_CTRL_ECC_DERR" },
-	{ .fc_id = 122, .cpu_id = 44, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_WAP_ECC_DERR" },
-	{ .fc_id = 123, .cpu_id = 45, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_SBTE0_ECC_DERR" },
-	{ .fc_id = 124, .cpu_id = 45, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_SBTE1_ECC_DERR" },
-	{ .fc_id = 125, .cpu_id = 45, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_SBTE2_ECC_DERR" },
-	{ .fc_id = 126, .cpu_id = 45, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_SBTE3_ECC_DERR" },
-	{ .fc_id = 127, .cpu_id = 45, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_SBTE4_ECC_DERR" },
-	{ .fc_id = 128, .cpu_id = 45, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_CTRL_ECC_DERR" },
-	{ .fc_id = 129, .cpu_id = 45, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_WAP_ECC_DERR" },
-	{ .fc_id = 130, .cpu_id = 46, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_SBTE0_ECC_DERR" },
-	{ .fc_id = 131, .cpu_id = 46, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_SBTE1_ECC_DERR" },
-	{ .fc_id = 132, .cpu_id = 46, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_SBTE2_ECC_DERR" },
-	{ .fc_id = 133, .cpu_id = 46, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_SBTE3_ECC_DERR" },
-	{ .fc_id = 134, .cpu_id = 46, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_SBTE4_ECC_DERR" },
-	{ .fc_id = 135, .cpu_id = 46, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_CTRL_ECC_DERR" },
-	{ .fc_id = 136, .cpu_id = 46, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_WAP_ECC_DERR" },
-	{ .fc_id = 137, .cpu_id = 47, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_SBTE0_ECC_DERR" },
-	{ .fc_id = 138, .cpu_id = 47, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_SBTE1_ECC_DERR" },
-	{ .fc_id = 139, .cpu_id = 47, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_SBTE2_ECC_DERR" },
-	{ .fc_id = 140, .cpu_id = 47, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_SBTE3_ECC_DERR" },
-	{ .fc_id = 141, .cpu_id = 47, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_SBTE4_ECC_DERR" },
-	{ .fc_id = 142, .cpu_id = 47, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_CTRL_ECC_DERR" },
-	{ .fc_id = 143, .cpu_id = 47, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_WAP_ECC_DERR" },
-	{ .fc_id = 144, .cpu_id = 48, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA2_ECC_SERR" },
-	{ .fc_id = 145, .cpu_id = 48, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA3_ECC_SERR" },
-	{ .fc_id = 146, .cpu_id = 48, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA0_ECC_SERR" },
-	{ .fc_id = 147, .cpu_id = 48, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA1_ECC_SERR" },
-	{ .fc_id = 148, .cpu_id = 48, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA6_ECC_SERR" },
-	{ .fc_id = 149, .cpu_id = 48, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA7_ECC_SERR" },
-	{ .fc_id = 150, .cpu_id = 48, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA4_ECC_SERR" },
-	{ .fc_id = 151, .cpu_id = 48, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA5_ECC_SERR" },
-	{ .fc_id = 152, .cpu_id = 49, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HDMA2_ECC_DERR" },
-	{ .fc_id = 153, .cpu_id = 49, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HDMA3_ECC_DERR" },
-	{ .fc_id = 154, .cpu_id = 49, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HDMA0_ECC_DERR" },
-	{ .fc_id = 155, .cpu_id = 49, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HDMA1_ECC_DERR" },
-	{ .fc_id = 156, .cpu_id = 49, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HDMA6_ECC_DERR" },
-	{ .fc_id = 157, .cpu_id = 49, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HDMA7_ECC_DERR" },
-	{ .fc_id = 158, .cpu_id = 49, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HDMA4_ECC_DERR" },
-	{ .fc_id = 159, .cpu_id = 49, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HDMA5_ECC_DERR" },
-	{ .fc_id = 160, .cpu_id = 50, .valid = 1,
-		.msg = 0, .reset = 0, .name = "KDMA0_ECC_SERR" },
-	{ .fc_id = 161, .cpu_id = 51, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PDMA0_ECC_SERR" },
-	{ .fc_id = 162, .cpu_id = 51, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PDMA1_ECC_SERR" },
-	{ .fc_id = 163, .cpu_id = 52, .valid = 1,
-		.msg = 0, .reset = 1, .name = "KDMA0_ECC_DERR" },
-	{ .fc_id = 164, .cpu_id = 53, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PDMA0_ECC_DERR" },
-	{ .fc_id = 165, .cpu_id = 53, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PDMA1_ECC_DERR" },
-	{ .fc_id = 166, .cpu_id = 54, .valid = 1,
-		.msg = 0, .reset = 0, .name = "CPU_IF_ECC_SERR" },
-	{ .fc_id = 167, .cpu_id = 55, .valid = 1,
-		.msg = 0, .reset = 1, .name = "CPU_IF_ECC_DERR" },
-	{ .fc_id = 168, .cpu_id = 56, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PSOC_MEM_SERR" },
-	{ .fc_id = 169, .cpu_id = 57, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PSOC_MEM_DERR" },
-	{ .fc_id = 170, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM0_ECC_SERR" },
-	{ .fc_id = 171, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM1_ECC_SERR" },
-	{ .fc_id = 172, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM2_ECC_SERR" },
-	{ .fc_id = 173, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM3_ECC_SERR" },
-	{ .fc_id = 174, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM4_ECC_SERR" },
-	{ .fc_id = 175, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM5_ECC_SERR" },
-	{ .fc_id = 176, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM6_ECC_SERR" },
-	{ .fc_id = 177, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM7_ECC_SERR" },
-	{ .fc_id = 178, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM8_ECC_SERR" },
-	{ .fc_id = 179, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM9_ECC_SERR" },
-	{ .fc_id = 180, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM10_ECC_SERR" },
-	{ .fc_id = 181, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM11_ECC_SERR" },
-	{ .fc_id = 182, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM12_ECC_SERR" },
-	{ .fc_id = 183, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM13_ECC_SERR" },
-	{ .fc_id = 184, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM14_ECC_SERR" },
-	{ .fc_id = 185, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM15_ECC_SERR" },
-	{ .fc_id = 186, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM16_ECC_SERR" },
-	{ .fc_id = 187, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM17_ECC_SERR" },
-	{ .fc_id = 188, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM18_ECC_SERR" },
-	{ .fc_id = 189, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM19_ECC_SERR" },
-	{ .fc_id = 190, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM20_ECC_SERR" },
-	{ .fc_id = 191, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM21_ECC_SERR" },
-	{ .fc_id = 192, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM22_ECC_SERR" },
-	{ .fc_id = 193, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM23_ECC_SERR" },
-	{ .fc_id = 194, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM24_ECC_SERR" },
-	{ .fc_id = 195, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM25_ECC_SERR" },
-	{ .fc_id = 196, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM26_ECC_SERR" },
-	{ .fc_id = 197, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM27_ECC_SERR" },
-	{ .fc_id = 198, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM28_ECC_SERR" },
-	{ .fc_id = 199, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM29_ECC_SERR" },
-	{ .fc_id = 200, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM30_ECC_SERR" },
-	{ .fc_id = 201, .cpu_id = 58, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SRAM31_ECC_SERR" },
-	{ .fc_id = 202, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM0_ECC_DERR" },
-	{ .fc_id = 203, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM1_ECC_DERR" },
-	{ .fc_id = 204, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM2_ECC_DERR" },
-	{ .fc_id = 205, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM3_ECC_DERR" },
-	{ .fc_id = 206, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM4_ECC_DERR" },
-	{ .fc_id = 207, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM5_ECC_DERR" },
-	{ .fc_id = 208, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM6_ECC_DERR" },
-	{ .fc_id = 209, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM7_ECC_DERR" },
-	{ .fc_id = 210, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM8_ECC_DERR" },
-	{ .fc_id = 211, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM9_ECC_DERR" },
-	{ .fc_id = 212, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM10_ECC_DERR" },
-	{ .fc_id = 213, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM11_ECC_DERR" },
-	{ .fc_id = 214, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM12_ECC_DERR" },
-	{ .fc_id = 215, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM13_ECC_DERR" },
-	{ .fc_id = 216, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM14_ECC_DERR" },
-	{ .fc_id = 217, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM15_ECC_DERR" },
-	{ .fc_id = 218, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM16_ECC_DERR" },
-	{ .fc_id = 219, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM17_ECC_DERR" },
-	{ .fc_id = 220, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM18_ECC_DERR" },
-	{ .fc_id = 221, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM19_ECC_DERR" },
-	{ .fc_id = 222, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM20_ECC_DERR" },
-	{ .fc_id = 223, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM21_ECC_DERR" },
-	{ .fc_id = 224, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM22_ECC_DERR" },
-	{ .fc_id = 225, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM23_ECC_DERR" },
-	{ .fc_id = 226, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM24_ECC_DERR" },
-	{ .fc_id = 227, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM25_ECC_DERR" },
-	{ .fc_id = 228, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM26_ECC_DERR" },
-	{ .fc_id = 229, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM27_ECC_DERR" },
-	{ .fc_id = 230, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM28_ECC_DERR" },
-	{ .fc_id = 231, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM29_ECC_DERR" },
-	{ .fc_id = 232, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM30_ECC_DERR" },
-	{ .fc_id = 233, .cpu_id = 59, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SRAM31_ECC_DERR" },
-	{ .fc_id = 234, .cpu_id = 60, .valid = 1,
-		.msg = 0, .reset = 1, .name = "GIC500" },
-	{ .fc_id = 235, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_0_MC0_ECC_SERR" },
-	{ .fc_id = 236, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_1_MC0_ECC_SERR" },
-	{ .fc_id = 237, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_2_MC0_ECC_SERR" },
-	{ .fc_id = 238, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_3_MC0_ECC_SERR" },
-	{ .fc_id = 239, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_4_MC0_ECC_SERR" },
-	{ .fc_id = 240, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_5_MC0_ECC_SERR" },
-	{ .fc_id = 241, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_0_MC1_ECC_SERR" },
-	{ .fc_id = 242, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_1_MC1_ECC_SERR" },
-	{ .fc_id = 243, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_2_MC1_ECC_SERR" },
-	{ .fc_id = 244, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_3_MC1_ECC_SERR" },
-	{ .fc_id = 245, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_4_MC1_ECC_SERR" },
-	{ .fc_id = 246, .cpu_id = 61, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM_5_MC1_ECC_SERR" },
-	{ .fc_id = 247, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_0_MC0_ECC_DERR" },
-	{ .fc_id = 248, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_1_MC0_ECC_DERR" },
-	{ .fc_id = 249, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_2_MC0_ECC_DERR" },
-	{ .fc_id = 250, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_3_MC0_ECC_DERR" },
-	{ .fc_id = 251, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_4_MC0_ECC_DERR" },
-	{ .fc_id = 252, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_5_MC0_ECC_DERR" },
-	{ .fc_id = 253, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_0_MC1_ECC_DERR" },
-	{ .fc_id = 254, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_1_MC1_ECC_DERR" },
-	{ .fc_id = 255, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_2_MC1_ECC_DERR" },
-	{ .fc_id = 256, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_3_MC1_ECC_DERR" },
-	{ .fc_id = 257, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_4_MC1_ECC_DERR" },
-	{ .fc_id = 258, .cpu_id = 62, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_5_MC1_ECC_DERR" },
-	{ .fc_id = 259, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_0_ECC_SERR" },
-	{ .fc_id = 260, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_1_ECC_SERR" },
-	{ .fc_id = 261, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_2_ECC_SERR" },
-	{ .fc_id = 262, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_3_ECC_SERR" },
-	{ .fc_id = 263, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_8_ECC_SERR" },
-	{ .fc_id = 264, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_9_ECC_SERR" },
-	{ .fc_id = 265, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_10_ECC_SERR" },
-	{ .fc_id = 266, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_11_ECC_SERR" },
-	{ .fc_id = 267, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_7_ECC_SERR" },
-	{ .fc_id = 268, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_6_ECC_SERR" },
-	{ .fc_id = 269, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_5_ECC_SERR" },
-	{ .fc_id = 270, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_4_ECC_SERR" },
-	{ .fc_id = 271, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_15_ECC_SERR" },
-	{ .fc_id = 272, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_14_ECC_SERR" },
-	{ .fc_id = 273, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_13_ECC_SERR" },
-	{ .fc_id = 274, .cpu_id = 63, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HMMU_12_ECC_SERR" },
-	{ .fc_id = 275, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_0_ECC_DERR" },
-	{ .fc_id = 276, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_1_ECC_DERR" },
-	{ .fc_id = 277, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_2_ECC_DERR" },
-	{ .fc_id = 278, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_3_ECC_DERR" },
-	{ .fc_id = 279, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_8_ECC_DERR" },
-	{ .fc_id = 280, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_9_ECC_DERR" },
-	{ .fc_id = 281, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_10_ECC_DERR" },
-	{ .fc_id = 282, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_11_ECC_DERR" },
-	{ .fc_id = 283, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_7_ECC_DERR" },
-	{ .fc_id = 284, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_6_ECC_DERR" },
-	{ .fc_id = 285, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_5_ECC_DERR" },
-	{ .fc_id = 286, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_4_ECC_DERR" },
-	{ .fc_id = 287, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_15_ECC_DERR" },
-	{ .fc_id = 288, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_14_ECC_DERR" },
-	{ .fc_id = 289, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_13_ECC_DERR" },
-	{ .fc_id = 290, .cpu_id = 64, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_12_ECC_DERR" },
-	{ .fc_id = 291, .cpu_id = 65, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PMMU_ECC_SERR" },
-	{ .fc_id = 292, .cpu_id = 66, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PMMU_ECC_DERR" },
-	{ .fc_id = 293, .cpu_id = 67, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 294, .cpu_id = 68, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 295, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC0_VCD_ECC_SERR" },
-	{ .fc_id = 296, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC1_VCD_ECC_SERR" },
-	{ .fc_id = 297, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC2_VCD_ECC_SERR" },
-	{ .fc_id = 298, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC3_VCD_ECC_SERR" },
-	{ .fc_id = 299, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC4_VCD_ECC_SERR" },
-	{ .fc_id = 300, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC5_VCD_ECC_SERR" },
-	{ .fc_id = 301, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC6_VCD_ECC_SERR" },
-	{ .fc_id = 302, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC7_VCD_ECC_SERR" },
-	{ .fc_id = 303, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC8_VCD_ECC_SERR" },
-	{ .fc_id = 304, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC9_VCD_ECC_SERR" },
-	{ .fc_id = 305, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC0_L2C_ECC_SERR" },
-	{ .fc_id = 306, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC1_L2C_ECC_SERR" },
-	{ .fc_id = 307, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC2_L2C_ECC_SERR" },
-	{ .fc_id = 308, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC3_L2C_ECC_SERR" },
-	{ .fc_id = 309, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC4_L2C_ECC_SERR" },
-	{ .fc_id = 310, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC5_L2C_ECC_SERR" },
-	{ .fc_id = 311, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC6_L2C_ECC_SERR" },
-	{ .fc_id = 312, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC7_L2C_ECC_SERR" },
-	{ .fc_id = 313, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC8_L2C_ECC_SERR" },
-	{ .fc_id = 314, .cpu_id = 69, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC9_L2C_ECC_SERR" },
-	{ .fc_id = 315, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC0_VCD_ECC_DERR" },
-	{ .fc_id = 316, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC1_VCD_ECC_DERR" },
-	{ .fc_id = 317, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC2_VCD_ECC_DERR" },
-	{ .fc_id = 318, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC3_VCD_ECC_DERR" },
-	{ .fc_id = 319, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC4_VCD_ECC_DERR" },
-	{ .fc_id = 320, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC5_VCD_ECC_DERR" },
-	{ .fc_id = 321, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC6_VCD_ECC_DERR" },
-	{ .fc_id = 322, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC7_VCD_ECC_DERR" },
-	{ .fc_id = 323, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC8_VCD_ECC_DERR" },
-	{ .fc_id = 324, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC9_VCD_ECC_DERR" },
-	{ .fc_id = 325, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC0_L2C_ECC_DERR" },
-	{ .fc_id = 326, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC1_L2C_ECC_DERR" },
-	{ .fc_id = 327, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC2_L2C_ECC_DERR" },
-	{ .fc_id = 328, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC3_L2C_ECC_DERR" },
-	{ .fc_id = 329, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC4_L2C_ECC_DERR" },
-	{ .fc_id = 330, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC5_L2C_ECC_DERR" },
-	{ .fc_id = 331, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC6_L2C_ECC_DERR" },
-	{ .fc_id = 332, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC7_L2C_ECC_DERR" },
-	{ .fc_id = 333, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC8_L2C_ECC_DERR" },
-	{ .fc_id = 334, .cpu_id = 70, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC9_L2C_ECC_DERR" },
-	{ .fc_id = 335, .cpu_id = 71, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 336, .cpu_id = 72, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 337, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF0_ECC_SERR" },
-	{ .fc_id = 338, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF1_ECC_SERR" },
-	{ .fc_id = 339, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF2_ECC_SERR" },
-	{ .fc_id = 340, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF3_ECC_SERR" },
-	{ .fc_id = 341, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF8_ECC_SERR" },
-	{ .fc_id = 342, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF9_ECC_SERR" },
-	{ .fc_id = 343, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF10_ECC_SERR" },
-	{ .fc_id = 344, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF11_ECC_SERR" },
-	{ .fc_id = 345, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF7_ECC_SERR" },
-	{ .fc_id = 346, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF6_ECC_SERR" },
-	{ .fc_id = 347, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF5_ECC_SERR" },
-	{ .fc_id = 348, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF4_ECC_SERR" },
-	{ .fc_id = 349, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF15_ECC_SERR" },
-	{ .fc_id = 350, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF14_ECC_SERR" },
-	{ .fc_id = 351, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF13_ECC_SERR" },
-	{ .fc_id = 352, .cpu_id = 73, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HIF12_ECC_SERR" },
-	{ .fc_id = 353, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF0_ECC_DERR" },
-	{ .fc_id = 354, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF1_ECC_DERR" },
-	{ .fc_id = 355, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF2_ECC_DERR" },
-	{ .fc_id = 356, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF3_ECC_DERR" },
-	{ .fc_id = 357, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF8_ECC_DERR" },
-	{ .fc_id = 358, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF9_ECC_DERR" },
-	{ .fc_id = 359, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF10_ECC_DERR" },
-	{ .fc_id = 360, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF11_ECC_DERR" },
-	{ .fc_id = 361, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF7_ECC_DERR" },
-	{ .fc_id = 362, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF6_ECC_DERR" },
-	{ .fc_id = 363, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF5_ECC_DERR" },
-	{ .fc_id = 364, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF4_ECC_DERR" },
-	{ .fc_id = 365, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF15_ECC_DERR" },
-	{ .fc_id = 366, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF14_ECC_DERR" },
-	{ .fc_id = 367, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF13_ECC_DERR" },
-	{ .fc_id = 368, .cpu_id = 74, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF12_ECC_DERR" },
-	{ .fc_id = 369, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC0_ECC_SERR" },
-	{ .fc_id = 370, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC1_ECC_SERR" },
-	{ .fc_id = 371, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC2_ECC_SERR" },
-	{ .fc_id = 372, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC3_ECC_SERR" },
-	{ .fc_id = 373, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC4_ECC_SERR" },
-	{ .fc_id = 374, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC5_ECC_SERR" },
-	{ .fc_id = 375, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC6_ECC_SERR" },
-	{ .fc_id = 376, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC7_ECC_SERR" },
-	{ .fc_id = 377, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC8_ECC_SERR" },
-	{ .fc_id = 378, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC9_ECC_SERR" },
-	{ .fc_id = 379, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC10_ECC_SERR" },
-	{ .fc_id = 380, .cpu_id = 75, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC11_ECC_SERR" },
-	{ .fc_id = 381, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC0_ECC_DERR" },
-	{ .fc_id = 382, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC1_ECC_DERR" },
-	{ .fc_id = 383, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC2_ECC_DERR" },
-	{ .fc_id = 384, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC3_ECC_DERR" },
-	{ .fc_id = 385, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC4_ECC_DERR" },
-	{ .fc_id = 386, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC5_ECC_DERR" },
-	{ .fc_id = 387, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC6_ECC_DERR" },
-	{ .fc_id = 388, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC7_ECC_DERR" },
-	{ .fc_id = 389, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC8_ECC_DERR" },
-	{ .fc_id = 390, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC9_ECC_DERR" },
-	{ .fc_id = 391, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC10_ECC_DERR" },
-	{ .fc_id = 392, .cpu_id = 76, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC11_ECC_DERR" },
-	{ .fc_id = 393, .cpu_id = 77, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SM0_ECC_DERR" },
-	{ .fc_id = 394, .cpu_id = 77, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SM1_ECC_DERR" },
-	{ .fc_id = 395, .cpu_id = 77, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SM2_ECC_DERR" },
-	{ .fc_id = 396, .cpu_id = 77, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SM3_ECC_DERR" },
-	{ .fc_id = 397, .cpu_id = 78, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SM0_ECC_SERR" },
-	{ .fc_id = 398, .cpu_id = 78, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SM1_ECC_SERR" },
-	{ .fc_id = 399, .cpu_id = 78, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SM2_ECC_SERR" },
-	{ .fc_id = 400, .cpu_id = 78, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SM3_ECC_SERR" },
-	{ .fc_id = 401, .cpu_id = 79, .valid = 1,
-		.msg = 0, .reset = 0, .name = "XBAR0_ECC_SERR" },
-	{ .fc_id = 402, .cpu_id = 79, .valid = 1,
-		.msg = 0, .reset = 0, .name = "XBAR1_ECC_SERR" },
-	{ .fc_id = 403, .cpu_id = 79, .valid = 1,
-		.msg = 0, .reset = 0, .name = "XBAR2_ECC_SERR" },
-	{ .fc_id = 404, .cpu_id = 79, .valid = 1,
-		.msg = 0, .reset = 0, .name = "XBAR3_ECC_SERR" },
-	{ .fc_id = 405, .cpu_id = 80, .valid = 1,
-		.msg = 0, .reset = 1, .name = "XBAR0_ECC_DERR" },
-	{ .fc_id = 406, .cpu_id = 80, .valid = 1,
-		.msg = 0, .reset = 1, .name = "XBAR1_ECC_DERR" },
-	{ .fc_id = 407, .cpu_id = 80, .valid = 1,
-		.msg = 0, .reset = 1, .name = "XBAR2_ECC_DERR" },
-	{ .fc_id = 408, .cpu_id = 80, .valid = 1,
-		.msg = 0, .reset = 1, .name = "XBAR3_ECC_DERR" },
-	{ .fc_id = 409, .cpu_id = 81, .valid = 1,
-		.msg = 0, .reset = 0, .name = "ARC0_ECC_SERR" },
-	{ .fc_id = 410, .cpu_id = 82, .valid = 1,
-		.msg = 0, .reset = 1, .name = "ARC0_ECC_DERR" },
-	{ .fc_id = 411, .cpu_id = 83, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 412, .cpu_id = 84, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PCIE_ADDR_DEC_ERR" },
-	{ .fc_id = 413, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC0_AXI_ERR_RSP" },
-	{ .fc_id = 414, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC1_AXI_ERR_RSP" },
-	{ .fc_id = 415, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC2_AXI_ERR_RSP" },
-	{ .fc_id = 416, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC3_AXI_ERR_RSP" },
-	{ .fc_id = 417, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC4_AXI_ERR_RSP" },
-	{ .fc_id = 418, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC5_AXI_ERR_RSP" },
-	{ .fc_id = 419, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC6_AXI_ERR_RSP" },
-	{ .fc_id = 420, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC7_AXI_ERR_RSP" },
-	{ .fc_id = 421, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC8_AXI_ERR_RSP" },
-	{ .fc_id = 422, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC9_AXI_ERR_RSP" },
-	{ .fc_id = 423, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC10_AXI_ERR_RSP" },
-	{ .fc_id = 424, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC11_AXI_ERR_RSP" },
-	{ .fc_id = 425, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC12_AXI_ERR_RSP" },
-	{ .fc_id = 426, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC13_AXI_ERR_RSP" },
-	{ .fc_id = 427, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC14_AXI_ERR_RSP" },
-	{ .fc_id = 428, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC15_AXI_ERR_RSP" },
-	{ .fc_id = 429, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC16_AXI_ERR_RSP" },
-	{ .fc_id = 430, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC17_AXI_ERR_RSP" },
-	{ .fc_id = 431, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC18_AXI_ERR_RSP" },
-	{ .fc_id = 432, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC19_AXI_ERR_RSP" },
-	{ .fc_id = 433, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC20_AXI_ERR_RSP" },
-	{ .fc_id = 434, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC21_AXI_ERR_RSP" },
-	{ .fc_id = 435, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC22_AXI_ERR_RSP" },
-	{ .fc_id = 436, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC23_AXI_ERR_RSP" },
-	{ .fc_id = 437, .cpu_id = 85, .valid = 1,
-		.msg = 0, .reset = 1, .name = "TPC24_AXI_ERR_RSP" },
-	{ .fc_id = 438, .cpu_id = 86, .valid = 1,
-		.msg = 0, .reset = 1, .name = "AXI_ECC" },
-	{ .fc_id = 439, .cpu_id = 87, .valid = 1,
-		.msg = 0, .reset = 1, .name = "L2_RAM_ECC" },
-	{ .fc_id = 440, .cpu_id = 88, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_SBTE0_AXI_ERR_RSP" },
-	{ .fc_id = 441, .cpu_id = 88, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_SBTE1_AXI_ERR_RSP" },
-	{ .fc_id = 442, .cpu_id = 88, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_SBTE2_AXI_ERR_RSP" },
-	{ .fc_id = 443, .cpu_id = 88, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_SBTE3_AXI_ERR_RSP" },
-	{ .fc_id = 444, .cpu_id = 88, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_SBTE4_AXI_ERR_RSP" },
-	{ .fc_id = 445, .cpu_id = 88, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_CTRL_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 446, .cpu_id = 88, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME0_QMAN_SW_ERROR" },
-	{ .fc_id = 447, .cpu_id = 89, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_SBTE0_AXI_ERR_RSP" },
-	{ .fc_id = 448, .cpu_id = 89, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_SBTE1_AXI_ERR_RSP" },
-	{ .fc_id = 449, .cpu_id = 89, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_SBTE2_AXI_ERR_RSP" },
-	{ .fc_id = 450, .cpu_id = 89, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_SBTE3_AXI_ERR_RSP" },
-	{ .fc_id = 451, .cpu_id = 89, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_SBTE4_AXI_ERR_RSP" },
-	{ .fc_id = 452, .cpu_id = 89, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_CTRL_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 453, .cpu_id = 89, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME1_QMAN_SW_ERROR" },
-	{ .fc_id = 454, .cpu_id = 90, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_SBTE0_AXI_ERR_RSP" },
-	{ .fc_id = 455, .cpu_id = 90, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_SBTE1_AXI_ERR_RSP" },
-	{ .fc_id = 456, .cpu_id = 90, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_SBTE2_AXI_ERR_RSP" },
-	{ .fc_id = 457, .cpu_id = 90, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_SBTE3_AXI_ERR_RSP" },
-	{ .fc_id = 458, .cpu_id = 90, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_SBTE4_AXI_ERR_RSP" },
-	{ .fc_id = 459, .cpu_id = 90, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_CTRL_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 460, .cpu_id = 90, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME2_QMAN_SW_ERROR" },
-	{ .fc_id = 461, .cpu_id = 91, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_SBTE0_AXI_ERR_RSP" },
-	{ .fc_id = 462, .cpu_id = 91, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_SBTE1_AXI_ERR_RSP" },
-	{ .fc_id = 463, .cpu_id = 91, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_SBTE2_AXI_ERR_RSP" },
-	{ .fc_id = 464, .cpu_id = 91, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_SBTE3_AXI_ERR_RSP" },
-	{ .fc_id = 465, .cpu_id = 91, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_SBTE4_AXI_ERR_RSP" },
-	{ .fc_id = 466, .cpu_id = 91, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_CTRL_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 467, .cpu_id = 91, .valid = 1,
-		.msg = 0, .reset = 1, .name = "MME3_QMAN_SW_ERROR" },
-	{ .fc_id = 468, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PSOC_MME_PLL_LOCK_ERR" },
-	{ .fc_id = 469, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PSOC_CPU_PLL_LOCK_ERR" },
-	{ .fc_id = 470, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE3_TPC_PLL_LOCK_ERR" },
-	{ .fc_id = 471, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE3_NIC_PLL_LOCK_ERR" },
-	{ .fc_id = 472, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE3_XBAR_MMU_PLL_LOCK_ERR" },
-	{ .fc_id = 473, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE3_XBAR_DMA_PLL_LOCK_ERR" },
-	{ .fc_id = 474, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE3_XBAR_IF_PLL_LOCK_ERR" },
-	{ .fc_id = 475, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE3_XBAR_BANK_PLL_LOCK_ERR" },
-	{ .fc_id = 476, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE1_XBAR_MMU_PLL_LOCK_ERR" },
-	{ .fc_id = 477, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE1_XBAR_DMA_PLL_LOCK_ERR" },
-	{ .fc_id = 478, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE1_XBAR_IF_PLL_LOCK_ERR" },
-	{ .fc_id = 479, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE1_XBAR_MESH_PLL_LOCK_ERR" },
-	{ .fc_id = 480, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE1_TPC_PLL_LOCK_ERR" },
-	{ .fc_id = 481, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE1_NIC_PLL_LOCK_ERR" },
-	{ .fc_id = 482, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PMMU_MME_PLL_LOCK_ERR" },
-	{ .fc_id = 483, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE0_TPC_PLL_LOCK_ERR" },
-	{ .fc_id = 484, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE0_PCI_PLL_LOCK_ERR" },
-	{ .fc_id = 485, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE0_XBAR_MMU_PLL_LOCK_ERR" },
-	{ .fc_id = 486, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE0_XBAR_DMA_PLL_LOCK_ERR" },
-	{ .fc_id = 487, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE0_XBAR_IF_PLL_LOCK_ERR" },
-	{ .fc_id = 488, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE0_XBAR_MESH_PLL_LOCK_ERR" },
-	{ .fc_id = 489, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE2_XBAR_MMU_PLL_LOCK_ERR" },
-	{ .fc_id = 490, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE2_XBAR_DMA_PLL_LOCK_ERR" },
-	{ .fc_id = 491, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE2_XBAR_IF_PLL_LOCK_ERR" },
-	{ .fc_id = 492, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE2_XBAR_BANK_PLL_LOCK_ERR" },
-	{ .fc_id = 493, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE2_TPC_PLL_LOCK_ERR" },
-	{ .fc_id = 494, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PSOC_VID_PLL_LOCK_ERR" },
-	{ .fc_id = 495, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PMMU_VID_PLL_LOCK_ERR" },
-	{ .fc_id = 496, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE3_HBM_PLL_LOCK_ERR" },
-	{ .fc_id = 497, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE1_XBAR_HBM_PLL_LOCK_ERR" },
-	{ .fc_id = 498, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE1_HBM_PLL_LOCK_ERR" },
-	{ .fc_id = 499, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE0_HBM_PLL_LOCK_ERR" },
-	{ .fc_id = 500, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE2_XBAR_HBM_PLL_LOCK_ERR" },
-	{ .fc_id = 501, .cpu_id = 92, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DCORE2_HBM_PLL_LOCK_ERR" },
-	{ .fc_id = 502, .cpu_id = 93, .valid = 1,
-		.msg = 0, .reset = 1, .name = "CPU_AXI_ERR_RSP" },
-	{ .fc_id = 503, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_0_AXI_ERR_RSP" },
-	{ .fc_id = 504, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_1_AXI_ERR_RSP" },
-	{ .fc_id = 505, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_2_AXI_ERR_RSP" },
-	{ .fc_id = 506, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_3_AXI_ERR_RSP" },
-	{ .fc_id = 507, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_8_AXI_ERR_RSP" },
-	{ .fc_id = 508, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_9_AXI_ERR_RSP" },
-	{ .fc_id = 509, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_10_AXI_ERR_RSP" },
-	{ .fc_id = 510, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_11_AXI_ERR_RSP" },
-	{ .fc_id = 511, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_7_AXI_ERR_RSP" },
-	{ .fc_id = 512, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_6_AXI_ERR_RSP" },
-	{ .fc_id = 513, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_5_AXI_ERR_RSP" },
-	{ .fc_id = 514, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_4_AXI_ERR_RSP" },
-	{ .fc_id = 515, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_15_AXI_ERR_RSP" },
-	{ .fc_id = 516, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_14_AXI_ERR_RSP" },
-	{ .fc_id = 517, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_13_AXI_ERR_RSP" },
-	{ .fc_id = 518, .cpu_id = 94, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU_12_AXI_ERR_RSP" },
-	{ .fc_id = 519, .cpu_id = 95, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PMMU_FATAL" },
-	{ .fc_id = 520, .cpu_id = 96, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PMMU_AXI_ERR_RSP" },
-	{ .fc_id = 521, .cpu_id = 97, .valid = 1,
-		.msg = 0, .reset = 0, .name = "VM0_ALARM_A" },
-	{ .fc_id = 522, .cpu_id = 98, .valid = 1,
-		.msg = 0, .reset = 0, .name = "VM0_ALARM_B" },
-	{ .fc_id = 523, .cpu_id = 99, .valid = 1,
-		.msg = 0, .reset = 0, .name = "VM1_ALARM_A" },
-	{ .fc_id = 524, .cpu_id = 100, .valid = 1,
-		.msg = 0, .reset = 0, .name = "VM1_ALARM_B" },
-	{ .fc_id = 525, .cpu_id = 101, .valid = 1,
-		.msg = 0, .reset = 0, .name = "VM2_ALARM_A" },
-	{ .fc_id = 526, .cpu_id = 102, .valid = 1,
-		.msg = 0, .reset = 0, .name = "VM2_ALARM_B" },
-	{ .fc_id = 527, .cpu_id = 103, .valid = 1,
-		.msg = 0, .reset = 0, .name = "VM3_ALARM_A" },
-	{ .fc_id = 528, .cpu_id = 104, .valid = 1,
-		.msg = 0, .reset = 0, .name = "VM3_ALARM_B" },
-	{ .fc_id = 529, .cpu_id = 105, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PSOC_AXI_ERR_RSP" },
-	{ .fc_id = 530, .cpu_id = 106, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PSOC_PRSTN_FALL" },
-	{ .fc_id = 531, .cpu_id = 107, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 532, .cpu_id = 107, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 533, .cpu_id = 107, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 534, .cpu_id = 107, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 535, .cpu_id = 107, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 536, .cpu_id = 107, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 537, .cpu_id = 107, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 538, .cpu_id = 107, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 539, .cpu_id = 108, .valid = 1,
-		.msg = 0, .reset = 1, .name = "KDMA_CH0_AXI_ERR_RSP" },
-	{ .fc_id = 540, .cpu_id = 109, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PDMA_CH0_AXI_ERR_RSP" },
-	{ .fc_id = 541, .cpu_id = 109, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PDMA_CH1_AXI_ERR_RSP" },
-	{ .fc_id = 542, .cpu_id = 110, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_CATTRIP_0" },
-	{ .fc_id = 543, .cpu_id = 111, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_CATTRIP_1" },
-	{ .fc_id = 544, .cpu_id = 112, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_CATTRIP_2" },
-	{ .fc_id = 545, .cpu_id = 113, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_CATTRIP_3" },
-	{ .fc_id = 546, .cpu_id = 114, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_CATTRIP_4" },
-	{ .fc_id = 547, .cpu_id = 115, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM_CATTRIP_5" },
-	{ .fc_id = 548, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM0_MC0_SEI_SEVERE" },
-	{ .fc_id = 549, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM0_MC0_SEI_NON_SEVERE" },
-	{ .fc_id = 550, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM0_MC1_SEI_SEVERE" },
-	{ .fc_id = 551, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM0_MC1_SEI_NON_SEVERE" },
-	{ .fc_id = 552, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM1_MC0_SEI_SEVERE" },
-	{ .fc_id = 553, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM1_MC0_SEI_NON_SEVERE" },
-	{ .fc_id = 554, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM1_MC1_SEI_SEVERE" },
-	{ .fc_id = 555, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM1_MC1_SEI_NON_SEVERE" },
-	{ .fc_id = 556, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM2_MC0_SEI_SEVERE" },
-	{ .fc_id = 557, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM2_MC0_SEI_NON_SEVERE" },
-	{ .fc_id = 558, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM2_MC1_SEI_SEVERE" },
-	{ .fc_id = 559, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM2_MC1_SEI_NON_SEVERE" },
-	{ .fc_id = 560, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM3_MC0_SEI_SEVERE" },
-	{ .fc_id = 561, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM3_MC0_SEI_NON_SEVERE" },
-	{ .fc_id = 562, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM3_MC1_SEI_SEVERE" },
-	{ .fc_id = 563, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM3_MC1_SEI_NON_SEVERE" },
-	{ .fc_id = 564, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM4_MC0_SEI_SEVERE" },
-	{ .fc_id = 565, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM4_MC0_SEI_NON_SEVERE" },
-	{ .fc_id = 566, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM4_MC1_SEI_SEVERE" },
-	{ .fc_id = 567, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM4_MC1_SEI_NON_SEVERE" },
-	{ .fc_id = 568, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM5_MC0_SEI_SEVERE" },
-	{ .fc_id = 569, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM5_MC0_SEI_NON_SEVERE" },
-	{ .fc_id = 570, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HBM5_MC1_SEI_SEVERE" },
-	{ .fc_id = 571, .cpu_id = 116, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM5_MC1_SEI_NON_SEVERE" },
-	{ .fc_id = 572, .cpu_id = 117, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC0_AXI_ERR_RSPONSE" },
-	{ .fc_id = 573, .cpu_id = 117, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC1_AXI_ERR_RSPONSE" },
-	{ .fc_id = 574, .cpu_id = 117, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC2_AXI_ERR_RSPONSE" },
-	{ .fc_id = 575, .cpu_id = 117, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC3_AXI_ERR_RSPONSE" },
-	{ .fc_id = 576, .cpu_id = 117, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC4_AXI_ERR_RSPONSE" },
-	{ .fc_id = 577, .cpu_id = 117, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC5_AXI_ERR_RSPONSE" },
-	{ .fc_id = 578, .cpu_id = 117, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC6_AXI_ERR_RSPONSE" },
-	{ .fc_id = 579, .cpu_id = 117, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC7_AXI_ERR_RSPONSE" },
-	{ .fc_id = 580, .cpu_id = 117, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC8_AXI_ERR_RSPONSE" },
-	{ .fc_id = 581, .cpu_id = 117, .valid = 1,
-		.msg = 0, .reset = 1, .name = "DEC9_AXI_ERR_RSPONSE" },
-	{ .fc_id = 582, .cpu_id = 118, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 583, .cpu_id = 119, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 584, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF0_FATAL" },
-	{ .fc_id = 585, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF1_FATAL" },
-	{ .fc_id = 586, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF2_FATAL" },
-	{ .fc_id = 587, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF3_FATAL" },
-	{ .fc_id = 588, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF8_FATAL" },
-	{ .fc_id = 589, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF9_FATAL" },
-	{ .fc_id = 590, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF10_FATAL" },
-	{ .fc_id = 591, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF11_FATAL" },
-	{ .fc_id = 592, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF7_FATAL" },
-	{ .fc_id = 593, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF6_FATAL" },
-	{ .fc_id = 594, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF5_FATAL" },
-	{ .fc_id = 595, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF4_FATAL" },
-	{ .fc_id = 596, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF15_FATAL" },
-	{ .fc_id = 597, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF14_FATAL" },
-	{ .fc_id = 598, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF13_FATAL" },
-	{ .fc_id = 599, .cpu_id = 120, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HIF12_FATAL" },
-	{ .fc_id = 600, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC0_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 601, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC1_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 602, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC2_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 603, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC3_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 604, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC4_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 605, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC5_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 606, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC6_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 607, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC7_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 608, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC8_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 609, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC9_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 610, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC10_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 611, .cpu_id = 121, .valid = 1,
-		.msg = 0, .reset = 1, .name = "NIC11_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 612, .cpu_id = 122, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SM0_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 613, .cpu_id = 122, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SM1_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 614, .cpu_id = 122, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SM2_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 615, .cpu_id = 122, .valid = 1,
-		.msg = 0, .reset = 1, .name = "SM3_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 616, .cpu_id = 123, .valid = 1,
-		.msg = 0, .reset = 1, .name = "ARC_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 617, .cpu_id = 124, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 618, .cpu_id = 125, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 619, .cpu_id = 125, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PCIE_FLR_REQUESTED" },
-	{ .fc_id = 620, .cpu_id = 125, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 621, .cpu_id = 125, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 622, .cpu_id = 125, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PCIE_APB_TIMEOUT" },
-	{ .fc_id = 623, .cpu_id = 125, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 624, .cpu_id = 125, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 625, .cpu_id = 125, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 626, .cpu_id = 125, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 627, .cpu_id = 125, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PCIE_FATAL_ERR" },
-	{ .fc_id = 628, .cpu_id = 125, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 629, .cpu_id = 126, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 630, .cpu_id = 127, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 631, .cpu_id = 128, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PCIE_P2P_MSIX" },
-	{ .fc_id = 632, .cpu_id = 129, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PCIE_DRAIN_COMPLETE" },
-	{ .fc_id = 633, .cpu_id = 130, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC0_BMON_SPMU" },
-	{ .fc_id = 634, .cpu_id = 131, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC0_KERNEL_ERR" },
-	{ .fc_id = 635, .cpu_id = 132, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC1_BMON_SPMU" },
-	{ .fc_id = 636, .cpu_id = 133, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC1_KERNEL_ERR" },
-	{ .fc_id = 637, .cpu_id = 134, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC2_BMON_SPMU" },
-	{ .fc_id = 638, .cpu_id = 135, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC2_KERNEL_ERR" },
-	{ .fc_id = 639, .cpu_id = 136, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC3_BMON_SPMU" },
-	{ .fc_id = 640, .cpu_id = 137, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC3_KERNEL_ERR" },
-	{ .fc_id = 641, .cpu_id = 138, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC4_BMON_SPMU" },
-	{ .fc_id = 642, .cpu_id = 139, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC4_KERNEL_ERR" },
-	{ .fc_id = 643, .cpu_id = 140, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC5_BMON_SPMU" },
-	{ .fc_id = 644, .cpu_id = 141, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC5_KERNEL_ERR" },
-	{ .fc_id = 645, .cpu_id = 150, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC6_BMON_SPMU" },
-	{ .fc_id = 646, .cpu_id = 151, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC6_KERNEL_ERR" },
-	{ .fc_id = 647, .cpu_id = 152, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC7_BMON_SPMU" },
-	{ .fc_id = 648, .cpu_id = 153, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC7_KERNEL_ERR" },
-	{ .fc_id = 649, .cpu_id = 146, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC8_BMON_SPMU" },
-	{ .fc_id = 650, .cpu_id = 147, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC8_KERNEL_ERR" },
-	{ .fc_id = 651, .cpu_id = 148, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC9_BMON_SPMU" },
-	{ .fc_id = 652, .cpu_id = 149, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC9_KERNEL_ERR" },
-	{ .fc_id = 653, .cpu_id = 142, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC10_BMON_SPMU" },
-	{ .fc_id = 654, .cpu_id = 143, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC10_KERNEL_ERR" },
-	{ .fc_id = 655, .cpu_id = 144, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC11_BMON_SPMU" },
-	{ .fc_id = 656, .cpu_id = 145, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC11_KERNEL_ERR" },
-	{ .fc_id = 657, .cpu_id = 162, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC12_BMON_SPMU" },
-	{ .fc_id = 658, .cpu_id = 163, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC12_KERNEL_ERR" },
-	{ .fc_id = 659, .cpu_id = 164, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC13_BMON_SPMU" },
-	{ .fc_id = 660, .cpu_id = 165, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC13_KERNEL_ERR" },
-	{ .fc_id = 661, .cpu_id = 158, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC14_BMON_SPMU" },
-	{ .fc_id = 662, .cpu_id = 159, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC14_KERNEL_ERR" },
-	{ .fc_id = 663, .cpu_id = 160, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC15_BMON_SPMU" },
-	{ .fc_id = 664, .cpu_id = 161, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC15_KERNEL_ERR" },
-	{ .fc_id = 665, .cpu_id = 154, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC16_BMON_SPMU" },
-	{ .fc_id = 666, .cpu_id = 155, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC16_KERNEL_ERR" },
-	{ .fc_id = 667, .cpu_id = 156, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC17_BMON_SPMU" },
-	{ .fc_id = 668, .cpu_id = 157, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC17_KERNEL_ERR" },
-	{ .fc_id = 669, .cpu_id = 166, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC18_BMON_SPMU" },
-	{ .fc_id = 670, .cpu_id = 167, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC18_KERNEL_ERR" },
-	{ .fc_id = 671, .cpu_id = 168, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC19_BMON_SPMU" },
-	{ .fc_id = 672, .cpu_id = 169, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC19_KERNEL_ERR" },
-	{ .fc_id = 673, .cpu_id = 170, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC20_BMON_SPMU" },
-	{ .fc_id = 674, .cpu_id = 171, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC20_KERNEL_ERR" },
-	{ .fc_id = 675, .cpu_id = 172, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC21_BMON_SPMU" },
-	{ .fc_id = 676, .cpu_id = 173, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC21_KERNEL_ERR" },
-	{ .fc_id = 677, .cpu_id = 174, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC22_BMON_SPMU" },
-	{ .fc_id = 678, .cpu_id = 175, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC22_KERNEL_ERR" },
-	{ .fc_id = 679, .cpu_id = 176, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC23_BMON_SPMU" },
-	{ .fc_id = 680, .cpu_id = 177, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC23_KERNEL_ERR" },
-	{ .fc_id = 681, .cpu_id = 178, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC24_BMON_SPMU" },
-	{ .fc_id = 682, .cpu_id = 179, .valid = 1,
-		.msg = 0, .reset = 0, .name = "TPC24_KERNEL_ERR" },
-	{ .fc_id = 683, .cpu_id = 180, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 684, .cpu_id = 180, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 685, .cpu_id = 180, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 686, .cpu_id = 180, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 687, .cpu_id = 180, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 688, .cpu_id = 180, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_CTRL_BMON_SPMU" },
-	{ .fc_id = 689, .cpu_id = 180, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_SBTE_BMON_SPMU" },
-	{ .fc_id = 690, .cpu_id = 180, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_WAP_BMON_SPMU" },
-	{ .fc_id = 691, .cpu_id = 180, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME0_WAP_SOURCE_RESULT_INVALID" },
-	{ .fc_id = 692, .cpu_id = 181, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 693, .cpu_id = 181, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 694, .cpu_id = 181, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 695, .cpu_id = 181, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 696, .cpu_id = 181, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 697, .cpu_id = 181, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_CTRL_BMON_SPMU" },
-	{ .fc_id = 698, .cpu_id = 181, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_SBTE_BMON_SPMU" },
-	{ .fc_id = 699, .cpu_id = 181, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_WAP_BMON_SPMU" },
-	{ .fc_id = 700, .cpu_id = 181, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME1_WAP_SOURCE_RESULT_INVALID" },
-	{ .fc_id = 701, .cpu_id = 182, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 702, .cpu_id = 182, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 703, .cpu_id = 182, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 704, .cpu_id = 182, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 705, .cpu_id = 182, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 706, .cpu_id = 182, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_CTRL_BMON_SPMU" },
-	{ .fc_id = 707, .cpu_id = 182, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_SBTE_BMON_SPMU" },
-	{ .fc_id = 708, .cpu_id = 182, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_WAP_BMON_SPMU" },
-	{ .fc_id = 709, .cpu_id = 182, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME2_WAP_SOURCE_RESULT_INVALID" },
-	{ .fc_id = 710, .cpu_id = 183, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 711, .cpu_id = 183, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 712, .cpu_id = 183, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 713, .cpu_id = 183, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 714, .cpu_id = 183, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 715, .cpu_id = 183, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_CTRL_BMON_SPMU" },
-	{ .fc_id = 716, .cpu_id = 183, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_SBTE_BMON_SPMU" },
-	{ .fc_id = 717, .cpu_id = 183, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_WAP_BMON_SPMU" },
-	{ .fc_id = 718, .cpu_id = 183, .valid = 1,
-		.msg = 0, .reset = 0, .name = "MME3_WAP_SOURCE_RESULT_INVALID" },
-	{ .fc_id = 719, .cpu_id = 184, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 720, .cpu_id = 184, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU0_PAGE_FAULT_OR_WR_PERM" },
-	{ .fc_id = 721, .cpu_id = 184, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU0_SECURITY_ERROR" },
-	{ .fc_id = 722, .cpu_id = 185, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 723, .cpu_id = 185, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU1_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 724, .cpu_id = 185, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU1_SECURITY_ERROR" },
-	{ .fc_id = 725, .cpu_id = 186, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 726, .cpu_id = 186, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU2_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 727, .cpu_id = 186, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU2_SECURITY_ERROR" },
-	{ .fc_id = 728, .cpu_id = 187, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 729, .cpu_id = 187, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU3_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 730, .cpu_id = 187, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU3_SECURITY_ERROR" },
-	{ .fc_id = 731, .cpu_id = 188, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 732, .cpu_id = 188, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU8_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 733, .cpu_id = 188, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU8_SECURITY_ERROR" },
-	{ .fc_id = 734, .cpu_id = 189, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 735, .cpu_id = 189, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU9_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 736, .cpu_id = 189, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU9_SECURITY_ERROR" },
-	{ .fc_id = 737, .cpu_id = 190, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 738, .cpu_id = 190, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU10_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 739, .cpu_id = 190, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU10_SECURITY_ERROR" },
-	{ .fc_id = 740, .cpu_id = 191, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 741, .cpu_id = 191, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU11_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 742, .cpu_id = 191, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU11_SECURITY_ERROR" },
-	{ .fc_id = 743, .cpu_id = 192, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 744, .cpu_id = 192, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU7_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 745, .cpu_id = 192, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU7_SECURITY_ERROR" },
-	{ .fc_id = 746, .cpu_id = 193, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 747, .cpu_id = 193, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU6_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 748, .cpu_id = 193, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU6_SECURITY_ERROR" },
-	{ .fc_id = 749, .cpu_id = 194, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 750, .cpu_id = 194, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU5_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 751, .cpu_id = 194, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU5_SECURITY_ERROR" },
-	{ .fc_id = 752, .cpu_id = 195, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 753, .cpu_id = 195, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU4_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 754, .cpu_id = 195, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU4_SECURITY_ERROR" },
-	{ .fc_id = 755, .cpu_id = 196, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 756, .cpu_id = 196, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU15_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 757, .cpu_id = 196, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU15_SECURITY_ERROR" },
-	{ .fc_id = 758, .cpu_id = 197, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 759, .cpu_id = 197, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU14_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 760, .cpu_id = 197, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU14_SECURITY_ERROR" },
-	{ .fc_id = 761, .cpu_id = 198, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 762, .cpu_id = 198, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU13_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 763, .cpu_id = 198, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU13_SECURITY_ERROR" },
-	{ .fc_id = 764, .cpu_id = 199, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 765, .cpu_id = 199, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU12_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 766, .cpu_id = 199, .valid = 1,
-		.msg = 0, .reset = 1, .name = "HMMU12_SECURITY_ERROR" },
-	{ .fc_id = 767, .cpu_id = 200, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 768, .cpu_id = 201, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PMMU0_PAGE_FAULT_WR_PERM" },
-	{ .fc_id = 769, .cpu_id = 202, .valid = 1,
-		.msg = 0, .reset = 1, .name = "PMMU0_SECURITY_ERROR" },
-	{ .fc_id = 770, .cpu_id = 203, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA2_BM_SPMU" },
-	{ .fc_id = 771, .cpu_id = 204, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 772, .cpu_id = 205, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA3_BM_SPMU" },
-	{ .fc_id = 773, .cpu_id = 206, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 774, .cpu_id = 207, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA0_BM_SPMU" },
-	{ .fc_id = 775, .cpu_id = 208, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 776, .cpu_id = 209, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA1_BM_SPMU" },
-	{ .fc_id = 777, .cpu_id = 210, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 778, .cpu_id = 211, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA6_BM_SPMU" },
-	{ .fc_id = 779, .cpu_id = 212, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 780, .cpu_id = 213, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA7_BM_SPMU" },
-	{ .fc_id = 781, .cpu_id = 214, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 782, .cpu_id = 215, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA4_BM_SPMU" },
-	{ .fc_id = 783, .cpu_id = 216, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 784, .cpu_id = 217, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HDMA5_BM_SPMU" },
-	{ .fc_id = 785, .cpu_id = 218, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 786, .cpu_id = 219, .valid = 1,
-		.msg = 0, .reset = 0, .name = "KDMA_BM_SPMU" },
-	{ .fc_id = 787, .cpu_id = 220, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 788, .cpu_id = 221, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PDMA0_BM_SPMU" },
-	{ .fc_id = 789, .cpu_id = 222, .valid = 1,
-		.msg = 0, .reset = 0, .name = "PDMA1_BM_SPMU" },
-	{ .fc_id = 790, .cpu_id = 223, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM0_MC0_SPI" },
-	{ .fc_id = 791, .cpu_id = 224, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM0_MC1_SPI" },
-	{ .fc_id = 792, .cpu_id = 225, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM1_MC0_SPI" },
-	{ .fc_id = 793, .cpu_id = 226, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM1_MC1_SPI" },
-	{ .fc_id = 794, .cpu_id = 227, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM2_MC0_SPI" },
-	{ .fc_id = 795, .cpu_id = 228, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM2_MC1_SPI" },
-	{ .fc_id = 796, .cpu_id = 229, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM3_MC0_SPI" },
-	{ .fc_id = 797, .cpu_id = 230, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM3_MC1_SPI" },
-	{ .fc_id = 798, .cpu_id = 231, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM4_MC0_SPI" },
-	{ .fc_id = 799, .cpu_id = 232, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM4_MC1_SPI" },
-	{ .fc_id = 800, .cpu_id = 233, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM5_MC0_SPI" },
-	{ .fc_id = 801, .cpu_id = 234, .valid = 1,
-		.msg = 0, .reset = 0, .name = "HBM5_MC1_SPI" },
-	{ .fc_id = 802, .cpu_id = 235, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 803, .cpu_id = 236, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 804, .cpu_id = 237, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 805, .cpu_id = 238, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 806, .cpu_id = 239, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 807, .cpu_id = 240, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 808, .cpu_id = 241, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 809, .cpu_id = 242, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 810, .cpu_id = 243, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 811, .cpu_id = 244, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 812, .cpu_id = 245, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 813, .cpu_id = 246, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 814, .cpu_id = 247, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 815, .cpu_id = 248, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 816, .cpu_id = 249, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 817, .cpu_id = 250, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 818, .cpu_id = 251, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 819, .cpu_id = 252, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 820, .cpu_id = 253, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 821, .cpu_id = 254, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 822, .cpu_id = 255, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 823, .cpu_id = 256, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 824, .cpu_id = 257, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 825, .cpu_id = 258, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 826, .cpu_id = 259, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 827, .cpu_id = 260, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 828, .cpu_id = 261, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 829, .cpu_id = 262, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 830, .cpu_id = 263, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 831, .cpu_id = 264, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 832, .cpu_id = 265, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 833, .cpu_id = 266, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 834, .cpu_id = 267, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 835, .cpu_id = 268, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 836, .cpu_id = 269, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 837, .cpu_id = 270, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 838, .cpu_id = 271, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 839, .cpu_id = 272, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 840, .cpu_id = 273, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 841, .cpu_id = 274, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 842, .cpu_id = 275, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 843, .cpu_id = 276, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 844, .cpu_id = 277, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 845, .cpu_id = 278, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 846, .cpu_id = 279, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 847, .cpu_id = 280, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 848, .cpu_id = 281, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 849, .cpu_id = 282, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 850, .cpu_id = 283, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 851, .cpu_id = 284, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 852, .cpu_id = 285, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 853, .cpu_id = 286, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 854, .cpu_id = 287, .valid = 0,
-		.msg = 0, .reset = 1, .name = "" },
-	{ .fc_id = 855, .cpu_id = 288, .valid = 0,
-		.msg = 0, .reset = 1, .name = "" },
-	{ .fc_id = 856, .cpu_id = 289, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 857, .cpu_id = 290, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 858, .cpu_id = 291, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 859, .cpu_id = 292, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 860, .cpu_id = 293, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 861, .cpu_id = 294, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 862, .cpu_id = 295, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 863, .cpu_id = 296, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 864, .cpu_id = 297, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 865, .cpu_id = 298, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 866, .cpu_id = 299, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 867, .cpu_id = 300, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 868, .cpu_id = 301, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 869, .cpu_id = 302, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 870, .cpu_id = 303, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 871, .cpu_id = 304, .valid = 1,
-		.msg = 0, .reset = 1, .name = "RPM_ERROR_OR_DRAIN" },
-	{ .fc_id = 872, .cpu_id = 305, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 873, .cpu_id = 306, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 874, .cpu_id = 307, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 875, .cpu_id = 308, .valid = 1,
-		.msg = 0, .reset = 0, .name = "RAZWI_OR_PID_MIN_MAX_INTERRUPT" },
-	{ .fc_id = 876, .cpu_id = 309, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 877, .cpu_id = 310, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 878, .cpu_id = 311, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 879, .cpu_id = 312, .valid = 0,
-		.msg = 0, .reset = 1, .name = "" },
-	{ .fc_id = 880, .cpu_id = 313, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 881, .cpu_id = 314, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 882, .cpu_id = 315, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 883, .cpu_id = 316, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 884, .cpu_id = 317, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 885, .cpu_id = 318, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 886, .cpu_id = 319, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 887, .cpu_id = 320, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 888, .cpu_id = 321, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 889, .cpu_id = 322, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 890, .cpu_id = 323, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 891, .cpu_id = 324, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 892, .cpu_id = 325, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 893, .cpu_id = 326, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 894, .cpu_id = 327, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 895, .cpu_id = 328, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 896, .cpu_id = 329, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC0_SPI" },
-	{ .fc_id = 897, .cpu_id = 329, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC0_BMON_SPMU" },
-	{ .fc_id = 898, .cpu_id = 330, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC1_SPI" },
-	{ .fc_id = 899, .cpu_id = 330, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC1_BMON_SPMU" },
-	{ .fc_id = 900, .cpu_id = 331, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC2_SPI" },
-	{ .fc_id = 901, .cpu_id = 331, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC2_BMON_SPMU" },
-	{ .fc_id = 902, .cpu_id = 332, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC3_SPI" },
-	{ .fc_id = 903, .cpu_id = 332, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC3_BMON_SPMU" },
-	{ .fc_id = 904, .cpu_id = 333, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC4_SPI" },
-	{ .fc_id = 905, .cpu_id = 333, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC4_BMON_SPMU" },
-	{ .fc_id = 906, .cpu_id = 334, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC5_SPI" },
-	{ .fc_id = 907, .cpu_id = 334, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC5_BMON_SPMU" },
-	{ .fc_id = 908, .cpu_id = 335, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC6_SPI" },
-	{ .fc_id = 909, .cpu_id = 335, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC6_BMON_SPMU" },
-	{ .fc_id = 910, .cpu_id = 336, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC7_SPI" },
-	{ .fc_id = 911, .cpu_id = 336, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC7_BMON_SPMU" },
-	{ .fc_id = 912, .cpu_id = 337, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC8_SPI" },
-	{ .fc_id = 913, .cpu_id = 337, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC8_BMON_SPMU" },
-	{ .fc_id = 914, .cpu_id = 338, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC9_SPI" },
-	{ .fc_id = 915, .cpu_id = 338, .valid = 1,
-		.msg = 0, .reset = 0, .name = "DEC9_BMON_SPMU" },
-	{ .fc_id = 916, .cpu_id = 339, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 917, .cpu_id = 340, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 918, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 919, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 920, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 921, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 922, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 923, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 924, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 925, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 926, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 927, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 928, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 929, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 930, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 931, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 932, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 933, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 934, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 935, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 936, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 937, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 938, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 939, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 940, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 941, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 942, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 943, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 944, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 945, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 946, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 947, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 948, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 949, .cpu_id = 341, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 950, .cpu_id = 342, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 951, .cpu_id = 343, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC0_BMON_SPMU" },
-	{ .fc_id = 952, .cpu_id = 343, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC0_SW_ERROR" },
-	{ .fc_id = 953, .cpu_id = 343, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 954, .cpu_id = 343, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 955, .cpu_id = 344, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC1_BMON_SPMU" },
-	{ .fc_id = 956, .cpu_id = 344, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC1_SW_ERROR" },
-	{ .fc_id = 957, .cpu_id = 344, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 958, .cpu_id = 344, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 959, .cpu_id = 345, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC2_BMON_SPMU" },
-	{ .fc_id = 960, .cpu_id = 345, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC2_SW_ERROR" },
-	{ .fc_id = 961, .cpu_id = 345, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 962, .cpu_id = 345, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 963, .cpu_id = 346, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC3_BMON_SPMU" },
-	{ .fc_id = 964, .cpu_id = 346, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC3_SW_ERROR" },
-	{ .fc_id = 965, .cpu_id = 346, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 966, .cpu_id = 346, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 967, .cpu_id = 347, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC4_BMON_SPMU" },
-	{ .fc_id = 968, .cpu_id = 347, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC4_SW_ERROR" },
-	{ .fc_id = 969, .cpu_id = 347, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 970, .cpu_id = 347, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 971, .cpu_id = 348, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC5_BMON_SPMU" },
-	{ .fc_id = 972, .cpu_id = 348, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC5_SW_ERROR" },
-	{ .fc_id = 973, .cpu_id = 348, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 974, .cpu_id = 348, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 975, .cpu_id = 349, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC6_BMON_SPMU" },
-	{ .fc_id = 976, .cpu_id = 349, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC6_SW_ERROR" },
-	{ .fc_id = 977, .cpu_id = 349, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 978, .cpu_id = 349, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 979, .cpu_id = 350, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC7_BMON_SPMU" },
-	{ .fc_id = 980, .cpu_id = 350, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC7_SW_ERROR" },
-	{ .fc_id = 981, .cpu_id = 350, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 982, .cpu_id = 350, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 983, .cpu_id = 351, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC8_BMON_SPMU" },
-	{ .fc_id = 984, .cpu_id = 351, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC8_SW_ERROR" },
-	{ .fc_id = 985, .cpu_id = 351, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 986, .cpu_id = 351, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 987, .cpu_id = 352, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC9_BMON_SPMU" },
-	{ .fc_id = 988, .cpu_id = 352, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC9_SW_ERROR" },
-	{ .fc_id = 989, .cpu_id = 352, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 990, .cpu_id = 352, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 991, .cpu_id = 353, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC10_BMON_SPMU" },
-	{ .fc_id = 992, .cpu_id = 353, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC10_SW_ERROR" },
-	{ .fc_id = 993, .cpu_id = 353, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 994, .cpu_id = 353, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 995, .cpu_id = 354, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC11_BMON_SPMU" },
-	{ .fc_id = 996, .cpu_id = 354, .valid = 1,
-		.msg = 0, .reset = 0, .name = "NIC11_SW_ERROR" },
-	{ .fc_id = 997, .cpu_id = 354, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 998, .cpu_id = 354, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 999, .cpu_id = 355, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1000, .cpu_id = 356, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1001, .cpu_id = 357, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1002, .cpu_id = 358, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1003, .cpu_id = 359, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1004, .cpu_id = 360, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1005, .cpu_id = 361, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1006, .cpu_id = 362, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1007, .cpu_id = 363, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1008, .cpu_id = 368, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1009, .cpu_id = 369, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1010, .cpu_id = 366, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1011, .cpu_id = 367, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1012, .cpu_id = 364, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1013, .cpu_id = 365, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1014, .cpu_id = 374, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1015, .cpu_id = 375, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1016, .cpu_id = 372, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1017, .cpu_id = 373, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1018, .cpu_id = 370, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1019, .cpu_id = 371, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1020, .cpu_id = 376, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1021, .cpu_id = 377, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1022, .cpu_id = 378, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1023, .cpu_id = 379, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1024, .cpu_id = 380, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1025, .cpu_id = 381, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1026, .cpu_id = 382, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1027, .cpu_id = 383, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1028, .cpu_id = 384, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1029, .cpu_id = 385, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1030, .cpu_id = 386, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1031, .cpu_id = 387, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1032, .cpu_id = 388, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1033, .cpu_id = 389, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1034, .cpu_id = 390, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1035, .cpu_id = 391, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1036, .cpu_id = 392, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1037, .cpu_id = 393, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1038, .cpu_id = 394, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1039, .cpu_id = 395, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1040, .cpu_id = 396, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1041, .cpu_id = 397, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1042, .cpu_id = 398, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1043, .cpu_id = 399, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1044, .cpu_id = 400, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1045, .cpu_id = 401, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1046, .cpu_id = 402, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1047, .cpu_id = 403, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1048, .cpu_id = 404, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1049, .cpu_id = 405, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1050, .cpu_id = 406, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1051, .cpu_id = 407, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1052, .cpu_id = 408, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1053, .cpu_id = 409, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1054, .cpu_id = 410, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1055, .cpu_id = 411, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1056, .cpu_id = 412, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1057, .cpu_id = 413, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1058, .cpu_id = 414, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1059, .cpu_id = 414, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1060, .cpu_id = 414, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1061, .cpu_id = 414, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1062, .cpu_id = 414, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1063, .cpu_id = 414, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1064, .cpu_id = 414, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1065, .cpu_id = 414, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1066, .cpu_id = 414, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1067, .cpu_id = 414, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1068, .cpu_id = 415, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1069, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1070, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1071, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1072, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1073, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1074, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1075, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1076, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1077, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1078, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1079, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1080, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1081, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1082, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1083, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1084, .cpu_id = 416, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1085, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1086, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1087, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1088, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1089, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1090, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1091, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1092, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1093, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1094, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1095, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1096, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1097, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1098, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1099, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1100, .cpu_id = 417, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1101, .cpu_id = 418, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1102, .cpu_id = 419, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1103, .cpu_id = 420, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1104, .cpu_id = 421, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1105, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1106, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1107, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1108, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1109, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1110, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1111, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1112, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1113, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1114, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1115, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1116, .cpu_id = 422, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1117, .cpu_id = 423, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1118, .cpu_id = 424, .valid = 1,
-		.msg = 0, .reset = 0, .name = "ROTATOR0_SERR" },
-	{ .fc_id = 1119, .cpu_id = 425, .valid = 1,
-		.msg = 0, .reset = 0, .name = "ROTATOR1_SERR" },
-	{ .fc_id = 1120, .cpu_id = 426, .valid = 1,
-		.msg = 0, .reset = 1, .name = "ROTATOR0_DERR" },
-	{ .fc_id = 1121, .cpu_id = 427, .valid = 1,
-		.msg = 0, .reset = 1, .name = "ROTATOR1_DERR" },
-	{ .fc_id = 1122, .cpu_id = 428, .valid = 1,
-		.msg = 0, .reset = 1, .name = "ROTATOR0_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 1123, .cpu_id = 429, .valid = 1,
-		.msg = 0, .reset = 1, .name = "ROTATOR1_AXI_ERROR_RESPONSE" },
-	{ .fc_id = 1124, .cpu_id = 430, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1125, .cpu_id = 431, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1126, .cpu_id = 432, .valid = 1,
-		.msg = 0, .reset = 0, .name = "ROTATOR0_BMON_SPMU" },
-	{ .fc_id = 1127, .cpu_id = 433, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1128, .cpu_id = 434, .valid = 1,
-		.msg = 0, .reset = 0, .name = "ROTATOR1_BMON_SPMU" },
-	{ .fc_id = 1129, .cpu_id = 435, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1130, .cpu_id = 436, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SM0_BMON_SPMU" },
-	{ .fc_id = 1131, .cpu_id = 437, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SM1_BMON_SPMU" },
-	{ .fc_id = 1132, .cpu_id = 438, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SM2_BMON_SPMU" },
-	{ .fc_id = 1133, .cpu_id = 439, .valid = 1,
-		.msg = 0, .reset = 0, .name = "SM3_BMON_SPMU" },
-	{ .fc_id = 1134, .cpu_id = 440, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1135, .cpu_id = 441, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1136, .cpu_id = 442, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1137, .cpu_id = 443, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1138, .cpu_id = 444, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1139, .cpu_id = 445, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1140, .cpu_id = 446, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1141, .cpu_id = 447, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1142, .cpu_id = 448, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1143, .cpu_id = 449, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1144, .cpu_id = 450, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1145, .cpu_id = 451, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1146, .cpu_id = 452, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1147, .cpu_id = 453, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1148, .cpu_id = 454, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1149, .cpu_id = 455, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1150, .cpu_id = 456, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1151, .cpu_id = 457, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1152, .cpu_id = 458, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1153, .cpu_id = 459, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1154, .cpu_id = 460, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1155, .cpu_id = 461, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1156, .cpu_id = 462, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1157, .cpu_id = 463, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1158, .cpu_id = 464, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1159, .cpu_id = 465, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1160, .cpu_id = 466, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1161, .cpu_id = 467, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1162, .cpu_id = 468, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1163, .cpu_id = 469, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1164, .cpu_id = 470, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1165, .cpu_id = 471, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1166, .cpu_id = 472, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1167, .cpu_id = 473, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1168, .cpu_id = 474, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1169, .cpu_id = 475, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1170, .cpu_id = 476, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1171, .cpu_id = 477, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1172, .cpu_id = 478, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1173, .cpu_id = 479, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1174, .cpu_id = 480, .valid = 1,
-		.msg = 1, .reset = 0, .name = "PSOC_DMA_QM" },
-	{ .fc_id = 1175, .cpu_id = 481, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1176, .cpu_id = 482, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1177, .cpu_id = 483, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1178, .cpu_id = 484, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1179, .cpu_id = 485, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1180, .cpu_id = 486, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1181, .cpu_id = 487, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1182, .cpu_id = 488, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1183, .cpu_id = 489, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1184, .cpu_id = 490, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1185, .cpu_id = 491, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1186, .cpu_id = 492, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1187, .cpu_id = 493, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1188, .cpu_id = 494, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1189, .cpu_id = 495, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1190, .cpu_id = 496, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1191, .cpu_id = 497, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1192, .cpu_id = 498, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1193, .cpu_id = 499, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1194, .cpu_id = 500, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1195, .cpu_id = 501, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1196, .cpu_id = 502, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1197, .cpu_id = 503, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1198, .cpu_id = 504, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1199, .cpu_id = 505, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1200, .cpu_id = 506, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1201, .cpu_id = 507, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1202, .cpu_id = 508, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1203, .cpu_id = 509, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1204, .cpu_id = 510, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1205, .cpu_id = 511, .valid = 0,
-		.msg = 0, .reset = 0, .name = "" },
-	{ .fc_id = 1206, .cpu_id = 512, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC0_QM" },
-	{ .fc_id = 1207, .cpu_id = 513, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC1_QM" },
-	{ .fc_id = 1208, .cpu_id = 514, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC2_QM" },
-	{ .fc_id = 1209, .cpu_id = 515, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC3_QM" },
-	{ .fc_id = 1210, .cpu_id = 516, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC4_QM" },
-	{ .fc_id = 1211, .cpu_id = 517, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC5_QM" },
-	{ .fc_id = 1212, .cpu_id = 518, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC6_QM" },
-	{ .fc_id = 1213, .cpu_id = 519, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC7_QM" },
-	{ .fc_id = 1214, .cpu_id = 520, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC8_QM" },
-	{ .fc_id = 1215, .cpu_id = 521, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC9_QM" },
-	{ .fc_id = 1216, .cpu_id = 522, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC10_QM" },
-	{ .fc_id = 1217, .cpu_id = 523, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC11_QM" },
-	{ .fc_id = 1218, .cpu_id = 524, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC12_QM" },
-	{ .fc_id = 1219, .cpu_id = 525, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC13_QM" },
-	{ .fc_id = 1220, .cpu_id = 526, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC14_QM" },
-	{ .fc_id = 1221, .cpu_id = 527, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC15_QM" },
-	{ .fc_id = 1222, .cpu_id = 528, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC16_QM" },
-	{ .fc_id = 1223, .cpu_id = 529, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC17_QM" },
-	{ .fc_id = 1224, .cpu_id = 530, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC18_QM" },
-	{ .fc_id = 1225, .cpu_id = 531, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC19_QM" },
-	{ .fc_id = 1226, .cpu_id = 532, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC20_QM" },
-	{ .fc_id = 1227, .cpu_id = 533, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC21_QM" },
-	{ .fc_id = 1228, .cpu_id = 534, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC22_QM" },
-	{ .fc_id = 1229, .cpu_id = 535, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC23_QM" },
-	{ .fc_id = 1230, .cpu_id = 536, .valid = 1,
-		.msg = 1, .reset = 0, .name = "TPC24_QM" },
-	{ .fc_id = 1231, .cpu_id = 537, .valid = 0,
-		.msg = 1, .reset = 0, .name = "" },
-	{ .fc_id = 1232, .cpu_id = 538, .valid = 1,
-		.msg = 1, .reset = 0, .name = "MME0_QM" },
-	{ .fc_id = 1233, .cpu_id = 539, .valid = 1,
-		.msg = 1, .reset = 0, .name = "MME1_QM" },
-	{ .fc_id = 1234, .cpu_id = 540, .valid = 1,
-		.msg = 1, .reset = 0, .name = "MME2_QM" },
-	{ .fc_id = 1235, .cpu_id = 541, .valid = 1,
-		.msg = 1, .reset = 0, .name = "MME3_QM" },
-	{ .fc_id = 1236, .cpu_id = 542, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA2_QM" },
-	{ .fc_id = 1237, .cpu_id = 543, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA3_QM" },
-	{ .fc_id = 1238, .cpu_id = 544, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA0_QM" },
-	{ .fc_id = 1239, .cpu_id = 545, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA1_QM" },
-	{ .fc_id = 1240, .cpu_id = 546, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA6_QM" },
-	{ .fc_id = 1241, .cpu_id = 547, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA7_QM" },
-	{ .fc_id = 1242, .cpu_id = 548, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA4_QM" },
-	{ .fc_id = 1243, .cpu_id = 549, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA5_QM" },
-	{ .fc_id = 1244, .cpu_id = 550, .valid = 1,
-		.msg = 1, .reset = 0, .name = "PDMA0_QM" },
-	{ .fc_id = 1245, .cpu_id = 551, .valid = 1,
-		.msg = 1, .reset = 0, .name = "PDMA1_QM" },
-	{ .fc_id = 1246, .cpu_id = 552, .valid = 1,
-		.msg = 1, .reset = 0, .name = "PI_UPDATE" },
-	{ .fc_id = 1247, .cpu_id = 553, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HALT_MACHINE" },
-	{ .fc_id = 1248, .cpu_id = 554, .valid = 1,
-		.msg = 1, .reset = 0, .name = "INTS_REGISTER" },
-	{ .fc_id = 1249, .cpu_id = 555, .valid = 1,
-		.msg = 1, .reset = 0, .name = "ROT0_QM" },
-	{ .fc_id = 1250, .cpu_id = 556, .valid = 1,
-		.msg = 1, .reset = 0, .name = "ROT1_QM" },
-	{ .fc_id = 1251, .cpu_id = 557, .valid = 1,
-		.msg = 1, .reset = 0, .name = "SOFT_RESET" },
-	{ .fc_id = 1252, .cpu_id = 558, .valid = 1,
-		.msg = 1, .reset = 0, .name = "CPLD_SHUTDOWN_CAUSE" },
-	{ .fc_id = 1253, .cpu_id = 559, .valid = 1,
-		.msg = 1, .reset = 0, .name = "FIX_POWER_ENV_S" },
-	{ .fc_id = 1254, .cpu_id = 560, .valid = 1,
-		.msg = 1, .reset = 0, .name = "FIX_POWER_ENV_E" },
-	{ .fc_id = 1255, .cpu_id = 561, .valid = 1,
-		.msg = 1, .reset = 0, .name = "FIX_THERMAL_ENV_S" },
-	{ .fc_id = 1256, .cpu_id = 562, .valid = 1,
-		.msg = 1, .reset = 0, .name = "FIX_THERMAL_ENV_E" },
-	{ .fc_id = 1257, .cpu_id = 563, .valid = 1,
-		.msg = 1, .reset = 0, .name = "CPLD_SHUTDOWN_EVENT" },
-	{ .fc_id = 1258, .cpu_id = 564, .valid = 1,
-		.msg = 1, .reset = 0, .name = "PKT_QUEUE_OUT_SYNC" },
-	{ .fc_id = 1259, .cpu_id = 565, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA2_CORE" },
-	{ .fc_id = 1260, .cpu_id = 566, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA3_CORE" },
-	{ .fc_id = 1261, .cpu_id = 567, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA0_CORE" },
-	{ .fc_id = 1262, .cpu_id = 568, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA1_CORE" },
-	{ .fc_id = 1263, .cpu_id = 569, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA6_CORE" },
-	{ .fc_id = 1264, .cpu_id = 570, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA7_CORE" },
-	{ .fc_id = 1265, .cpu_id = 571, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA4_CORE" },
-	{ .fc_id = 1266, .cpu_id = 572, .valid = 1,
-		.msg = 1, .reset = 0, .name = "HDMA5_CORE" },
-	{ .fc_id = 1267, .cpu_id = 573, .valid = 1,
-		.msg = 1, .reset = 0, .name = "PDMA0_CORE" },
-	{ .fc_id = 1268, .cpu_id = 574, .valid = 1,
-		.msg = 1, .reset = 0, .name = "PDMA1_CORE" },
-	{ .fc_id = 1269, .cpu_id = 575, .valid = 1,
-		.msg = 1, .reset = 0, .name = "KDMA0_CORE" },
-	{ .fc_id = 1270, .cpu_id = 576, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC0_QM0" },
-	{ .fc_id = 1271, .cpu_id = 577, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC0_QM1" },
-	{ .fc_id = 1272, .cpu_id = 578, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC1_QM0" },
-	{ .fc_id = 1273, .cpu_id = 579, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC1_QM1" },
-	{ .fc_id = 1274, .cpu_id = 580, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC2_QM0" },
-	{ .fc_id = 1275, .cpu_id = 581, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC2_QM1" },
-	{ .fc_id = 1276, .cpu_id = 582, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC3_QM0" },
-	{ .fc_id = 1277, .cpu_id = 583, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC3_QM1" },
-	{ .fc_id = 1278, .cpu_id = 584, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC4_QM0" },
-	{ .fc_id = 1279, .cpu_id = 585, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC4_QM1" },
-	{ .fc_id = 1280, .cpu_id = 586, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC5_QM0" },
-	{ .fc_id = 1281, .cpu_id = 587, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC5_QM1" },
-	{ .fc_id = 1282, .cpu_id = 588, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC6_QM0" },
-	{ .fc_id = 1283, .cpu_id = 589, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC6_QM1" },
-	{ .fc_id = 1284, .cpu_id = 590, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC7_QM0" },
-	{ .fc_id = 1285, .cpu_id = 591, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC7_QM1" },
-	{ .fc_id = 1286, .cpu_id = 592, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC8_QM0" },
-	{ .fc_id = 1287, .cpu_id = 593, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC8_QM1" },
-	{ .fc_id = 1288, .cpu_id = 594, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC9_QM0" },
-	{ .fc_id = 1289, .cpu_id = 595, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC9_QM1" },
-	{ .fc_id = 1290, .cpu_id = 596, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC10_QM0" },
-	{ .fc_id = 1291, .cpu_id = 597, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC10_QM1" },
-	{ .fc_id = 1292, .cpu_id = 598, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC11_QM0" },
-	{ .fc_id = 1293, .cpu_id = 599, .valid = 1,
-		.msg = 1, .reset = 0, .name = "NIC11_QM1" },
-	{ .fc_id = 1294, .cpu_id = 600, .valid = 1,
-		.msg = 1, .reset = 0, .name = "CPU_PKT_SANITY_FAILED" },
-	{ .fc_id = 1295, .cpu_id = 601, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC0_ENG0" },
-	{ .fc_id = 1296, .cpu_id = 602, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC0_ENG1" },
-	{ .fc_id = 1297, .cpu_id = 603, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC1_ENG0" },
-	{ .fc_id = 1298, .cpu_id = 604, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC1_ENG1" },
-	{ .fc_id = 1299, .cpu_id = 605, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC2_ENG0" },
-	{ .fc_id = 1300, .cpu_id = 606, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC2_ENG1" },
-	{ .fc_id = 1301, .cpu_id = 607, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC3_ENG0" },
-	{ .fc_id = 1302, .cpu_id = 608, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC3_ENG1" },
-	{ .fc_id = 1303, .cpu_id = 609, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC4_ENG0" },
-	{ .fc_id = 1304, .cpu_id = 610, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC4_ENG1" },
-	{ .fc_id = 1305, .cpu_id = 611, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC5_ENG0" },
-	{ .fc_id = 1306, .cpu_id = 612, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC5_ENG1" },
-	{ .fc_id = 1307, .cpu_id = 613, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC6_ENG0" },
-	{ .fc_id = 1308, .cpu_id = 614, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC6_ENG1" },
-	{ .fc_id = 1309, .cpu_id = 615, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC7_ENG0" },
-	{ .fc_id = 1310, .cpu_id = 616, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC7_ENG1" },
-	{ .fc_id = 1311, .cpu_id = 617, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC8_ENG0" },
-	{ .fc_id = 1312, .cpu_id = 618, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC8_ENG1" },
-	{ .fc_id = 1313, .cpu_id = 619, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC9_ENG0" },
-	{ .fc_id = 1314, .cpu_id = 620, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC9_ENG1" },
-	{ .fc_id = 1315, .cpu_id = 621, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC10_ENG0" },
-	{ .fc_id = 1316, .cpu_id = 622, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC10_ENG1" },
-	{ .fc_id = 1317, .cpu_id = 623, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC11_ENG0" },
-	{ .fc_id = 1318, .cpu_id = 624, .valid = 1,
-		.msg = 1, .reset = 0, .name = "STATUS_NIC11_ENG1" },
-	{ .fc_id = 1319, .cpu_id = 625, .valid = 1,
-		.msg = 1, .reset = 0, .name = "ARC_DCCM_FULL" },
-	{ .fc_id = 1320, .cpu_id = 626, .valid = 1,
-		.msg = 1, .reset = 1, .name = "FP32_NOT_SUPPORTED" },
-	{ .fc_id = 1321, .cpu_id = 627, .valid = 1,
-		.msg = 1, .reset = 1, .name = "DEV_RESET_REQ" },
+	{ .fc_id = 0, .cpu_id = 0, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1, .cpu_id = 1, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 2, .cpu_id = 2, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 3, .cpu_id = 3, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 4, .cpu_id = 4, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 5, .cpu_id = 5, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 6, .cpu_id = 6, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 7, .cpu_id = 7, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 8, .cpu_id = 8, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 9, .cpu_id = 9, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 10, .cpu_id = 10, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 11, .cpu_id = 11, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 12, .cpu_id = 12, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 13, .cpu_id = 13, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 14, .cpu_id = 14, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 15, .cpu_id = 15, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 16, .cpu_id = 16, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 17, .cpu_id = 17, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 18, .cpu_id = 18, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 19, .cpu_id = 19, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 20, .cpu_id = 20, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 21, .cpu_id = 21, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 22, .cpu_id = 22, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 23, .cpu_id = 23, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 24, .cpu_id = 24, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 25, .cpu_id = 25, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 26, .cpu_id = 26, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 27, .cpu_id = 27, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 28, .cpu_id = 28, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 29, .cpu_id = 29, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 30, .cpu_id = 30, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 31, .cpu_id = 31, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 32, .cpu_id = 32, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PCIE_CORE_SERR" },
+	{ .fc_id = 33, .cpu_id = 33, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PCIE_CORE_DERR" },
+	{ .fc_id = 34, .cpu_id = 34, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PCIE_IF_SERR" },
+	{ .fc_id = 35, .cpu_id = 35, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PCIE_IF_DERR" },
+	{ .fc_id = 36, .cpu_id = 36, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PCIE_PHY_SERR" },
+	{ .fc_id = 37, .cpu_id = 37, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PCIE_PHY_DERR" },
+	{ .fc_id = 38, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC0_ECC_SERR" },
+	{ .fc_id = 39, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC1_ECC_SERR" },
+	{ .fc_id = 40, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC2_ECC_SERR" },
+	{ .fc_id = 41, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC3_ECC_SERR" },
+	{ .fc_id = 42, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC4_ECC_SERR" },
+	{ .fc_id = 43, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC5_ECC_SERR" },
+	{ .fc_id = 44, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC6_ECC_SERR" },
+	{ .fc_id = 45, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC7_ECC_SERR" },
+	{ .fc_id = 46, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC8_ECC_SERR" },
+	{ .fc_id = 47, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC9_ECC_SERR" },
+	{ .fc_id = 48, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC10_ECC_SERR" },
+	{ .fc_id = 49, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC11_ECC_SERR" },
+	{ .fc_id = 50, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC12_ECC_SERR" },
+	{ .fc_id = 51, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC13_ECC_SERR" },
+	{ .fc_id = 52, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC14_ECC_SERR" },
+	{ .fc_id = 53, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC15_ECC_SERR" },
+	{ .fc_id = 54, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC16_ECC_SERR" },
+	{ .fc_id = 55, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC17_ECC_SERR" },
+	{ .fc_id = 56, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC18_ECC_SERR" },
+	{ .fc_id = 57, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC19_ECC_SERR" },
+	{ .fc_id = 58, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC20_ECC_SERR" },
+	{ .fc_id = 59, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC21_ECC_SERR" },
+	{ .fc_id = 60, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC22_ECC_SERR" },
+	{ .fc_id = 61, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC23_ECC_SERR" },
+	{ .fc_id = 62, .cpu_id = 38, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC24_ECC_SERR" },
+	{ .fc_id = 63, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC0_ECC_DERR" },
+	{ .fc_id = 64, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC1_ECC_DERR" },
+	{ .fc_id = 65, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC2_ECC_DERR" },
+	{ .fc_id = 66, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC3_ECC_DERR" },
+	{ .fc_id = 67, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC4_ECC_DERR" },
+	{ .fc_id = 68, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC5_ECC_DERR" },
+	{ .fc_id = 69, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC6_ECC_DERR" },
+	{ .fc_id = 70, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC7_ECC_DERR" },
+	{ .fc_id = 71, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC8_ECC_DERR" },
+	{ .fc_id = 72, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC9_ECC_DERR" },
+	{ .fc_id = 73, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC10_ECC_DERR" },
+	{ .fc_id = 74, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC11_ECC_DERR" },
+	{ .fc_id = 75, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC12_ECC_DERR" },
+	{ .fc_id = 76, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC13_ECC_DERR" },
+	{ .fc_id = 77, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC14_ECC_DERR" },
+	{ .fc_id = 78, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC15_ECC_DERR" },
+	{ .fc_id = 79, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC16_ECC_DERR" },
+	{ .fc_id = 80, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC17_ECC_DERR" },
+	{ .fc_id = 81, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC18_ECC_DERR" },
+	{ .fc_id = 82, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC19_ECC_DERR" },
+	{ .fc_id = 83, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC20_ECC_DERR" },
+	{ .fc_id = 84, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC21_ECC_DERR" },
+	{ .fc_id = 85, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC22_ECC_DERR" },
+	{ .fc_id = 86, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC23_ECC_DERR" },
+	{ .fc_id = 87, .cpu_id = 39, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "TPC24_ECC_DERR" },
+	{ .fc_id = 88, .cpu_id = 40, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME0_SBTE0_ECC_SERR" },
+	{ .fc_id = 89, .cpu_id = 40, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME0_SBTE1_ECC_SERR" },
+	{ .fc_id = 90, .cpu_id = 40, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME0_SBTE2_ECC_SERR" },
+	{ .fc_id = 91, .cpu_id = 40, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME0_SBTE3_ECC_SERR" },
+	{ .fc_id = 92, .cpu_id = 40, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME0_SBTE4_ECC_SERR" },
+	{ .fc_id = 93, .cpu_id = 40, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME0_CTRL_ECC_SERR" },
+	{ .fc_id = 94, .cpu_id = 40, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME0_WAP_ECC_SERR" },
+	{ .fc_id = 95, .cpu_id = 41, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME1_SBTE0_ECC_SERR" },
+	{ .fc_id = 96, .cpu_id = 41, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME1_SBTE1_ECC_SERR" },
+	{ .fc_id = 97, .cpu_id = 41, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME1_SBTE2_ECC_SERR" },
+	{ .fc_id = 98, .cpu_id = 41, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME1_SBTE3_ECC_SERR" },
+	{ .fc_id = 99, .cpu_id = 41, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME1_SBTE4_ECC_SERR" },
+	{ .fc_id = 100, .cpu_id = 41, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME1_CTRL_ECC_SERR" },
+	{ .fc_id = 101, .cpu_id = 41, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME1_WAP_ECC_SERR" },
+	{ .fc_id = 102, .cpu_id = 42, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME2_SBTE0_ECC_SERR" },
+	{ .fc_id = 103, .cpu_id = 42, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME2_SBTE1_ECC_SERR" },
+	{ .fc_id = 104, .cpu_id = 42, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME2_SBTE2_ECC_SERR" },
+	{ .fc_id = 105, .cpu_id = 42, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME2_SBTE3_ECC_SERR" },
+	{ .fc_id = 106, .cpu_id = 42, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME2_SBTE4_ECC_SERR" },
+	{ .fc_id = 107, .cpu_id = 42, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME2_CTRL_ECC_SERR" },
+	{ .fc_id = 108, .cpu_id = 42, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME2_WAP_ECC_SERR" },
+	{ .fc_id = 109, .cpu_id = 43, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME3_SBTE0_ECC_SERR" },
+	{ .fc_id = 110, .cpu_id = 43, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME3_SBTE1_ECC_SERR" },
+	{ .fc_id = 111, .cpu_id = 43, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME3_SBTE2_ECC_SERR" },
+	{ .fc_id = 112, .cpu_id = 43, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME3_SBTE3_ECC_SERR" },
+	{ .fc_id = 113, .cpu_id = 43, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME3_SBTE4_ECC_SERR" },
+	{ .fc_id = 114, .cpu_id = 43, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME3_CTRL_ECC_SERR" },
+	{ .fc_id = 115, .cpu_id = 43, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME3_WAP_ECC_SERR" },
+	{ .fc_id = 116, .cpu_id = 44, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME0_SBTE0_ECC_DERR" },
+	{ .fc_id = 117, .cpu_id = 44, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME0_SBTE1_ECC_DERR" },
+	{ .fc_id = 118, .cpu_id = 44, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME0_SBTE2_ECC_DERR" },
+	{ .fc_id = 119, .cpu_id = 44, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME0_SBTE3_ECC_DERR" },
+	{ .fc_id = 120, .cpu_id = 44, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME0_SBTE4_ECC_DERR" },
+	{ .fc_id = 121, .cpu_id = 44, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME0_CTRL_ECC_DERR" },
+	{ .fc_id = 122, .cpu_id = 44, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME0_WAP_ECC_DERR" },
+	{ .fc_id = 123, .cpu_id = 45, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME1_SBTE0_ECC_DERR" },
+	{ .fc_id = 124, .cpu_id = 45, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME1_SBTE1_ECC_DERR" },
+	{ .fc_id = 125, .cpu_id = 45, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME1_SBTE2_ECC_DERR" },
+	{ .fc_id = 126, .cpu_id = 45, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME1_SBTE3_ECC_DERR" },
+	{ .fc_id = 127, .cpu_id = 45, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME1_SBTE4_ECC_DERR" },
+	{ .fc_id = 128, .cpu_id = 45, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME1_CTRL_ECC_DERR" },
+	{ .fc_id = 129, .cpu_id = 45, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME1_WAP_ECC_DERR" },
+	{ .fc_id = 130, .cpu_id = 46, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME2_SBTE0_ECC_DERR" },
+	{ .fc_id = 131, .cpu_id = 46, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME2_SBTE1_ECC_DERR" },
+	{ .fc_id = 132, .cpu_id = 46, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME2_SBTE2_ECC_DERR" },
+	{ .fc_id = 133, .cpu_id = 46, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME2_SBTE3_ECC_DERR" },
+	{ .fc_id = 134, .cpu_id = 46, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME2_SBTE4_ECC_DERR" },
+	{ .fc_id = 135, .cpu_id = 46, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME2_CTRL_ECC_DERR" },
+	{ .fc_id = 136, .cpu_id = 46, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME2_WAP_ECC_DERR" },
+	{ .fc_id = 137, .cpu_id = 47, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME3_SBTE0_ECC_DERR" },
+	{ .fc_id = 138, .cpu_id = 47, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME3_SBTE1_ECC_DERR" },
+	{ .fc_id = 139, .cpu_id = 47, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME3_SBTE2_ECC_DERR" },
+	{ .fc_id = 140, .cpu_id = 47, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME3_SBTE3_ECC_DERR" },
+	{ .fc_id = 141, .cpu_id = 47, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME3_SBTE4_ECC_DERR" },
+	{ .fc_id = 142, .cpu_id = 47, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME3_CTRL_ECC_DERR" },
+	{ .fc_id = 143, .cpu_id = 47, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "MME3_WAP_ECC_DERR" },
+	{ .fc_id = 144, .cpu_id = 48, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA2_ECC_SERR" },
+	{ .fc_id = 145, .cpu_id = 48, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA3_ECC_SERR" },
+	{ .fc_id = 146, .cpu_id = 48, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA0_ECC_SERR" },
+	{ .fc_id = 147, .cpu_id = 48, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA1_ECC_SERR" },
+	{ .fc_id = 148, .cpu_id = 48, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA6_ECC_SERR" },
+	{ .fc_id = 149, .cpu_id = 48, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA7_ECC_SERR" },
+	{ .fc_id = 150, .cpu_id = 48, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HDMA4_ECC_SERR" },
+	{ .fc_id = 151, .cpu_id = 48, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HDMA5_ECC_SERR" },
+	{ .fc_id = 152, .cpu_id = 49, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "EDMA2_ECC_DERR" },
+	{ .fc_id = 153, .cpu_id = 49, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "EDMA3_ECC_DERR" },
+	{ .fc_id = 154, .cpu_id = 49, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "EDMA0_ECC_DERR" },
+	{ .fc_id = 155, .cpu_id = 49, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "EDMA1_ECC_DERR" },
+	{ .fc_id = 156, .cpu_id = 49, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "EDMA6_ECC_DERR" },
+	{ .fc_id = 157, .cpu_id = 49, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "EDMA7_ECC_DERR" },
+	{ .fc_id = 158, .cpu_id = 49, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "EDMA4_ECC_DERR" },
+	{ .fc_id = 159, .cpu_id = 49, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "EDMA5_ECC_DERR" },
+	{ .fc_id = 160, .cpu_id = 50, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "KDMA0_ECC_SERR" },
+	{ .fc_id = 161, .cpu_id = 51, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PDMA0_ECC_SERR" },
+	{ .fc_id = 162, .cpu_id = 51, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PDMA1_ECC_SERR" },
+	{ .fc_id = 163, .cpu_id = 52, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "KDMA0_ECC_DERR" },
+	{ .fc_id = 164, .cpu_id = 53, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PDMA0_ECC_DERR" },
+	{ .fc_id = 165, .cpu_id = 53, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PDMA1_ECC_DERR" },
+	{ .fc_id = 166, .cpu_id = 54, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "CPU_IF_ECC_SERR" },
+	{ .fc_id = 167, .cpu_id = 55, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "CPU_IF_ECC_DERR" },
+	{ .fc_id = 168, .cpu_id = 56, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PSOC_MEM_SERR" },
+	{ .fc_id = 169, .cpu_id = 57, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PSOC_MEM_DERR" },
+	{ .fc_id = 170, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM0_ECC_SERR" },
+	{ .fc_id = 171, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM1_ECC_SERR" },
+	{ .fc_id = 172, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM2_ECC_SERR" },
+	{ .fc_id = 173, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM3_ECC_SERR" },
+	{ .fc_id = 174, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM4_ECC_SERR" },
+	{ .fc_id = 175, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM5_ECC_SERR" },
+	{ .fc_id = 176, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM6_ECC_SERR" },
+	{ .fc_id = 177, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM7_ECC_SERR" },
+	{ .fc_id = 178, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM8_ECC_SERR" },
+	{ .fc_id = 179, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM9_ECC_SERR" },
+	{ .fc_id = 180, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM10_ECC_SERR" },
+	{ .fc_id = 181, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM11_ECC_SERR" },
+	{ .fc_id = 182, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM12_ECC_SERR" },
+	{ .fc_id = 183, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM13_ECC_SERR" },
+	{ .fc_id = 184, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM14_ECC_SERR" },
+	{ .fc_id = 185, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM15_ECC_SERR" },
+	{ .fc_id = 186, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM16_ECC_SERR" },
+	{ .fc_id = 187, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM17_ECC_SERR" },
+	{ .fc_id = 188, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM18_ECC_SERR" },
+	{ .fc_id = 189, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM19_ECC_SERR" },
+	{ .fc_id = 190, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM20_ECC_SERR" },
+	{ .fc_id = 191, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM21_ECC_SERR" },
+	{ .fc_id = 192, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM22_ECC_SERR" },
+	{ .fc_id = 193, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM23_ECC_SERR" },
+	{ .fc_id = 194, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM24_ECC_SERR" },
+	{ .fc_id = 195, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM25_ECC_SERR" },
+	{ .fc_id = 196, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM26_ECC_SERR" },
+	{ .fc_id = 197, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM27_ECC_SERR" },
+	{ .fc_id = 198, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM28_ECC_SERR" },
+	{ .fc_id = 199, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM29_ECC_SERR" },
+	{ .fc_id = 200, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM30_ECC_SERR" },
+	{ .fc_id = 201, .cpu_id = 58, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SRAM31_ECC_SERR" },
+	{ .fc_id = 202, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM0_ECC_DERR" },
+	{ .fc_id = 203, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM1_ECC_DERR" },
+	{ .fc_id = 204, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM2_ECC_DERR" },
+	{ .fc_id = 205, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM3_ECC_DERR" },
+	{ .fc_id = 206, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM4_ECC_DERR" },
+	{ .fc_id = 207, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM5_ECC_DERR" },
+	{ .fc_id = 208, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM6_ECC_DERR" },
+	{ .fc_id = 209, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM7_ECC_DERR" },
+	{ .fc_id = 210, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM8_ECC_DERR" },
+	{ .fc_id = 211, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM9_ECC_DERR" },
+	{ .fc_id = 212, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM10_ECC_DERR" },
+	{ .fc_id = 213, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM11_ECC_DERR" },
+	{ .fc_id = 214, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM12_ECC_DERR" },
+	{ .fc_id = 215, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM13_ECC_DERR" },
+	{ .fc_id = 216, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM14_ECC_DERR" },
+	{ .fc_id = 217, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM15_ECC_DERR" },
+	{ .fc_id = 218, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM16_ECC_DERR" },
+	{ .fc_id = 219, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM17_ECC_DERR" },
+	{ .fc_id = 220, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM18_ECC_DERR" },
+	{ .fc_id = 221, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM19_ECC_DERR" },
+	{ .fc_id = 222, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM20_ECC_DERR" },
+	{ .fc_id = 223, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM21_ECC_DERR" },
+	{ .fc_id = 224, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM22_ECC_DERR" },
+	{ .fc_id = 225, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM23_ECC_DERR" },
+	{ .fc_id = 226, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM24_ECC_DERR" },
+	{ .fc_id = 227, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM25_ECC_DERR" },
+	{ .fc_id = 228, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM26_ECC_DERR" },
+	{ .fc_id = 229, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM27_ECC_DERR" },
+	{ .fc_id = 230, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM28_ECC_DERR" },
+	{ .fc_id = 231, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM29_ECC_DERR" },
+	{ .fc_id = 232, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM30_ECC_DERR" },
+	{ .fc_id = 233, .cpu_id = 59, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SRAM31_ECC_DERR" },
+	{ .fc_id = 234, .cpu_id = 60, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "GIC500" },
+	{ .fc_id = 235, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_0_MC0_ECC_SERR" },
+	{ .fc_id = 236, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_1_MC0_ECC_SERR" },
+	{ .fc_id = 237, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_2_MC0_ECC_SERR" },
+	{ .fc_id = 238, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_3_MC0_ECC_SERR" },
+	{ .fc_id = 239, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_4_MC0_ECC_SERR" },
+	{ .fc_id = 240, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_5_MC0_ECC_SERR" },
+	{ .fc_id = 241, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_0_MC1_ECC_SERR" },
+	{ .fc_id = 242, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_1_MC1_ECC_SERR" },
+	{ .fc_id = 243, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_2_MC1_ECC_SERR" },
+	{ .fc_id = 244, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_3_MC1_ECC_SERR" },
+	{ .fc_id = 245, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_4_MC1_ECC_SERR" },
+	{ .fc_id = 246, .cpu_id = 61, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM_5_MC1_ECC_SERR" },
+	{ .fc_id = 247, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_0_MC0_ECC_DERR" },
+	{ .fc_id = 248, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_1_MC0_ECC_DERR" },
+	{ .fc_id = 249, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_2_MC0_ECC_DERR" },
+	{ .fc_id = 250, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_3_MC0_ECC_DERR" },
+	{ .fc_id = 251, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_4_MC0_ECC_DERR" },
+	{ .fc_id = 252, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_5_MC0_ECC_DERR" },
+	{ .fc_id = 253, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_0_MC1_ECC_DERR" },
+	{ .fc_id = 254, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_1_MC1_ECC_DERR" },
+	{ .fc_id = 255, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_2_MC1_ECC_DERR" },
+	{ .fc_id = 256, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_3_MC1_ECC_DERR" },
+	{ .fc_id = 257, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_4_MC1_ECC_DERR" },
+	{ .fc_id = 258, .cpu_id = 62, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_5_MC1_ECC_DERR" },
+	{ .fc_id = 259, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_0_ECC_SERR" },
+	{ .fc_id = 260, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_1_ECC_SERR" },
+	{ .fc_id = 261, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_2_ECC_SERR" },
+	{ .fc_id = 262, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_3_ECC_SERR" },
+	{ .fc_id = 263, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_8_ECC_SERR" },
+	{ .fc_id = 264, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_9_ECC_SERR" },
+	{ .fc_id = 265, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_10_ECC_SERR" },
+	{ .fc_id = 266, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_11_ECC_SERR" },
+	{ .fc_id = 267, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_7_ECC_SERR" },
+	{ .fc_id = 268, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_6_ECC_SERR" },
+	{ .fc_id = 269, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_5_ECC_SERR" },
+	{ .fc_id = 270, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_4_ECC_SERR" },
+	{ .fc_id = 271, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_15_ECC_SERR" },
+	{ .fc_id = 272, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_14_ECC_SERR" },
+	{ .fc_id = 273, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_13_ECC_SERR" },
+	{ .fc_id = 274, .cpu_id = 63, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU_12_ECC_SERR" },
+	{ .fc_id = 275, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_0_ECC_DERR" },
+	{ .fc_id = 276, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_1_ECC_DERR" },
+	{ .fc_id = 277, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_2_ECC_DERR" },
+	{ .fc_id = 278, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_3_ECC_DERR" },
+	{ .fc_id = 279, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_8_ECC_DERR" },
+	{ .fc_id = 280, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_9_ECC_DERR" },
+	{ .fc_id = 281, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_10_ECC_DERR" },
+	{ .fc_id = 282, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_11_ECC_DERR" },
+	{ .fc_id = 283, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_7_ECC_DERR" },
+	{ .fc_id = 284, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_6_ECC_DERR" },
+	{ .fc_id = 285, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_5_ECC_DERR" },
+	{ .fc_id = 286, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_4_ECC_DERR" },
+	{ .fc_id = 287, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_15_ECC_DERR" },
+	{ .fc_id = 288, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_14_ECC_DERR" },
+	{ .fc_id = 289, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_13_ECC_DERR" },
+	{ .fc_id = 290, .cpu_id = 64, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_12_ECC_DERR" },
+	{ .fc_id = 291, .cpu_id = 65, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PMMU_ECC_SERR" },
+	{ .fc_id = 292, .cpu_id = 66, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PMMU_ECC_DERR" },
+	{ .fc_id = 293, .cpu_id = 67, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 294, .cpu_id = 68, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 295, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC0_VCD_ECC_SERR" },
+	{ .fc_id = 296, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC1_VCD_ECC_SERR" },
+	{ .fc_id = 297, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC2_VCD_ECC_SERR" },
+	{ .fc_id = 298, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC3_VCD_ECC_SERR" },
+	{ .fc_id = 299, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC4_VCD_ECC_SERR" },
+	{ .fc_id = 300, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC5_VCD_ECC_SERR" },
+	{ .fc_id = 301, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC6_VCD_ECC_SERR" },
+	{ .fc_id = 302, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC7_VCD_ECC_SERR" },
+	{ .fc_id = 303, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC8_VCD_ECC_SERR" },
+	{ .fc_id = 304, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC9_VCD_ECC_SERR" },
+	{ .fc_id = 305, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC0_L2C_ECC_SERR" },
+	{ .fc_id = 306, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC1_L2C_ECC_SERR" },
+	{ .fc_id = 307, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC2_L2C_ECC_SERR" },
+	{ .fc_id = 308, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC3_L2C_ECC_SERR" },
+	{ .fc_id = 309, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC4_L2C_ECC_SERR" },
+	{ .fc_id = 310, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC5_L2C_ECC_SERR" },
+	{ .fc_id = 311, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC6_L2C_ECC_SERR" },
+	{ .fc_id = 312, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC7_L2C_ECC_SERR" },
+	{ .fc_id = 313, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC8_L2C_ECC_SERR" },
+	{ .fc_id = 314, .cpu_id = 69, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC9_L2C_ECC_SERR" },
+	{ .fc_id = 315, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC0_VCD_ECC_DERR" },
+	{ .fc_id = 316, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC1_VCD_ECC_DERR" },
+	{ .fc_id = 317, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC2_VCD_ECC_DERR" },
+	{ .fc_id = 318, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC3_VCD_ECC_DERR" },
+	{ .fc_id = 319, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC4_VCD_ECC_DERR" },
+	{ .fc_id = 320, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC5_VCD_ECC_DERR" },
+	{ .fc_id = 321, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC6_VCD_ECC_DERR" },
+	{ .fc_id = 322, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC7_VCD_ECC_DERR" },
+	{ .fc_id = 323, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC8_VCD_ECC_DERR" },
+	{ .fc_id = 324, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC9_VCD_ECC_DERR" },
+	{ .fc_id = 325, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC0_L2C_ECC_DERR" },
+	{ .fc_id = 326, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC1_L2C_ECC_DERR" },
+	{ .fc_id = 327, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC2_L2C_ECC_DERR" },
+	{ .fc_id = 328, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC3_L2C_ECC_DERR" },
+	{ .fc_id = 329, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC4_L2C_ECC_DERR" },
+	{ .fc_id = 330, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC5_L2C_ECC_DERR" },
+	{ .fc_id = 331, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC6_L2C_ECC_DERR" },
+	{ .fc_id = 332, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC7_L2C_ECC_DERR" },
+	{ .fc_id = 333, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC8_L2C_ECC_DERR" },
+	{ .fc_id = 334, .cpu_id = 70, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEC9_L2C_ECC_DERR" },
+	{ .fc_id = 335, .cpu_id = 71, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 336, .cpu_id = 72, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 337, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF0_ECC_SERR" },
+	{ .fc_id = 338, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF1_ECC_SERR" },
+	{ .fc_id = 339, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF2_ECC_SERR" },
+	{ .fc_id = 340, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF3_ECC_SERR" },
+	{ .fc_id = 341, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF8_ECC_SERR" },
+	{ .fc_id = 342, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF9_ECC_SERR" },
+	{ .fc_id = 343, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF10_ECC_SERR" },
+	{ .fc_id = 344, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF11_ECC_SERR" },
+	{ .fc_id = 345, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF7_ECC_SERR" },
+	{ .fc_id = 346, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF6_ECC_SERR" },
+	{ .fc_id = 347, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF5_ECC_SERR" },
+	{ .fc_id = 348, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF4_ECC_SERR" },
+	{ .fc_id = 349, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF15_ECC_SERR" },
+	{ .fc_id = 350, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF14_ECC_SERR" },
+	{ .fc_id = 351, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF13_ECC_SERR" },
+	{ .fc_id = 352, .cpu_id = 73, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HIF12_ECC_SERR" },
+	{ .fc_id = 353, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF0_ECC_DERR" },
+	{ .fc_id = 354, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF1_ECC_DERR" },
+	{ .fc_id = 355, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF2_ECC_DERR" },
+	{ .fc_id = 356, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF3_ECC_DERR" },
+	{ .fc_id = 357, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF8_ECC_DERR" },
+	{ .fc_id = 358, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF9_ECC_DERR" },
+	{ .fc_id = 359, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF10_ECC_DERR" },
+	{ .fc_id = 360, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF11_ECC_DERR" },
+	{ .fc_id = 361, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF7_ECC_DERR" },
+	{ .fc_id = 362, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF6_ECC_DERR" },
+	{ .fc_id = 363, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF5_ECC_DERR" },
+	{ .fc_id = 364, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF4_ECC_DERR" },
+	{ .fc_id = 365, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF15_ECC_DERR" },
+	{ .fc_id = 366, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF14_ECC_DERR" },
+	{ .fc_id = 367, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF13_ECC_DERR" },
+	{ .fc_id = 368, .cpu_id = 74, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF12_ECC_DERR" },
+	{ .fc_id = 369, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC0_ECC_SERR" },
+	{ .fc_id = 370, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC1_ECC_SERR" },
+	{ .fc_id = 371, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC2_ECC_SERR" },
+	{ .fc_id = 372, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC3_ECC_SERR" },
+	{ .fc_id = 373, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC4_ECC_SERR" },
+	{ .fc_id = 374, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC5_ECC_SERR" },
+	{ .fc_id = 375, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC6_ECC_SERR" },
+	{ .fc_id = 376, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC7_ECC_SERR" },
+	{ .fc_id = 377, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC8_ECC_SERR" },
+	{ .fc_id = 378, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC9_ECC_SERR" },
+	{ .fc_id = 379, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC10_ECC_SERR" },
+	{ .fc_id = 380, .cpu_id = 75, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC11_ECC_SERR" },
+	{ .fc_id = 381, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC0_ECC_DERR" },
+	{ .fc_id = 382, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC1_ECC_DERR" },
+	{ .fc_id = 383, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC2_ECC_DERR" },
+	{ .fc_id = 384, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC3_ECC_DERR" },
+	{ .fc_id = 385, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC4_ECC_DERR" },
+	{ .fc_id = 386, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC5_ECC_DERR" },
+	{ .fc_id = 387, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC6_ECC_DERR" },
+	{ .fc_id = 388, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC7_ECC_DERR" },
+	{ .fc_id = 389, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC8_ECC_DERR" },
+	{ .fc_id = 390, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC9_ECC_DERR" },
+	{ .fc_id = 391, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC10_ECC_DERR" },
+	{ .fc_id = 392, .cpu_id = 76, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC11_ECC_DERR" },
+	{ .fc_id = 393, .cpu_id = 77, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SM0_ECC_DERR" },
+	{ .fc_id = 394, .cpu_id = 77, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SM1_ECC_DERR" },
+	{ .fc_id = 395, .cpu_id = 77, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SM2_ECC_DERR" },
+	{ .fc_id = 396, .cpu_id = 77, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "SM3_ECC_DERR" },
+	{ .fc_id = 397, .cpu_id = 78, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SM0_ECC_SERR" },
+	{ .fc_id = 398, .cpu_id = 78, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SM1_ECC_SERR" },
+	{ .fc_id = 399, .cpu_id = 78, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SM2_ECC_SERR" },
+	{ .fc_id = 400, .cpu_id = 78, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SM3_ECC_SERR" },
+	{ .fc_id = 401, .cpu_id = 79, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "XBAR0_ECC_SERR" },
+	{ .fc_id = 402, .cpu_id = 79, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "XBAR1_ECC_SERR" },
+	{ .fc_id = 403, .cpu_id = 79, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "XBAR2_ECC_SERR" },
+	{ .fc_id = 404, .cpu_id = 79, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "XBAR3_ECC_SERR" },
+	{ .fc_id = 405, .cpu_id = 80, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "XBAR0_ECC_DERR" },
+	{ .fc_id = 406, .cpu_id = 80, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "XBAR1_ECC_DERR" },
+	{ .fc_id = 407, .cpu_id = 80, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "XBAR2_ECC_DERR" },
+	{ .fc_id = 408, .cpu_id = 80, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "XBAR3_ECC_DERR" },
+	{ .fc_id = 409, .cpu_id = 81, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "ARC0_ECC_SERR" },
+	{ .fc_id = 410, .cpu_id = 82, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "ARC0_ECC_DERR" },
+	{ .fc_id = 411, .cpu_id = 83, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 412, .cpu_id = 84, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PCIE_ADDR_DEC_ERR" },
+	{ .fc_id = 413, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC0_AXI_ERR_RSP" },
+	{ .fc_id = 414, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC1_AXI_ERR_RSP" },
+	{ .fc_id = 415, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC2_AXI_ERR_RSP" },
+	{ .fc_id = 416, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC3_AXI_ERR_RSP" },
+	{ .fc_id = 417, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC4_AXI_ERR_RSP" },
+	{ .fc_id = 418, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC5_AXI_ERR_RSP" },
+	{ .fc_id = 419, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC6_AXI_ERR_RSP" },
+	{ .fc_id = 420, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC7_AXI_ERR_RSP" },
+	{ .fc_id = 421, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC8_AXI_ERR_RSP" },
+	{ .fc_id = 422, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC9_AXI_ERR_RSP" },
+	{ .fc_id = 423, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC10_AXI_ERR_RSP" },
+	{ .fc_id = 424, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC11_AXI_ERR_RSP" },
+	{ .fc_id = 425, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC12_AXI_ERR_RSP" },
+	{ .fc_id = 426, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC13_AXI_ERR_RSP" },
+	{ .fc_id = 427, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC14_AXI_ERR_RSP" },
+	{ .fc_id = 428, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC15_AXI_ERR_RSP" },
+	{ .fc_id = 429, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC16_AXI_ERR_RSP" },
+	{ .fc_id = 430, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC17_AXI_ERR_RSP" },
+	{ .fc_id = 431, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC18_AXI_ERR_RSP" },
+	{ .fc_id = 432, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC19_AXI_ERR_RSP" },
+	{ .fc_id = 433, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC20_AXI_ERR_RSP" },
+	{ .fc_id = 434, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC21_AXI_ERR_RSP" },
+	{ .fc_id = 435, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC22_AXI_ERR_RSP" },
+	{ .fc_id = 436, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC23_AXI_ERR_RSP" },
+	{ .fc_id = 437, .cpu_id = 85, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC24_AXI_ERR_RSP" },
+	{ .fc_id = 438, .cpu_id = 86, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "AXI_ECC" },
+	{ .fc_id = 439, .cpu_id = 87, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "L2_RAM_ECC" },
+	{ .fc_id = 440, .cpu_id = 88, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME0_SBTE0_AXI_ERR_RSP" },
+	{ .fc_id = 441, .cpu_id = 88, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME0_SBTE1_AXI_ERR_RSP" },
+	{ .fc_id = 442, .cpu_id = 88, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME0_SBTE2_AXI_ERR_RSP" },
+	{ .fc_id = 443, .cpu_id = 88, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME0_SBTE3_AXI_ERR_RSP" },
+	{ .fc_id = 444, .cpu_id = 88, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME0_SBTE4_AXI_ERR_RSP" },
+	{ .fc_id = 445, .cpu_id = 88, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME0_CTRL_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 446, .cpu_id = 88, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME0_QMAN_SW_ERROR" },
+	{ .fc_id = 447, .cpu_id = 89, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME1_SBTE0_AXI_ERR_RSP" },
+	{ .fc_id = 448, .cpu_id = 89, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME1_SBTE1_AXI_ERR_RSP" },
+	{ .fc_id = 449, .cpu_id = 89, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME1_SBTE2_AXI_ERR_RSP" },
+	{ .fc_id = 450, .cpu_id = 89, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME1_SBTE3_AXI_ERR_RSP" },
+	{ .fc_id = 451, .cpu_id = 89, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME1_SBTE4_AXI_ERR_RSP" },
+	{ .fc_id = 452, .cpu_id = 89, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME1_CTRL_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 453, .cpu_id = 89, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME1_QMAN_SW_ERROR" },
+	{ .fc_id = 454, .cpu_id = 90, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME2_SBTE0_AXI_ERR_RSP" },
+	{ .fc_id = 455, .cpu_id = 90, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME2_SBTE1_AXI_ERR_RSP" },
+	{ .fc_id = 456, .cpu_id = 90, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME2_SBTE2_AXI_ERR_RSP" },
+	{ .fc_id = 457, .cpu_id = 90, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME2_SBTE3_AXI_ERR_RSP" },
+	{ .fc_id = 458, .cpu_id = 90, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME2_SBTE4_AXI_ERR_RSP" },
+	{ .fc_id = 459, .cpu_id = 90, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME2_CTRL_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 460, .cpu_id = 90, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME2_QMAN_SW_ERROR" },
+	{ .fc_id = 461, .cpu_id = 91, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME3_SBTE0_AXI_ERR_RSP" },
+	{ .fc_id = 462, .cpu_id = 91, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME3_SBTE1_AXI_ERR_RSP" },
+	{ .fc_id = 463, .cpu_id = 91, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME3_SBTE2_AXI_ERR_RSP" },
+	{ .fc_id = 464, .cpu_id = 91, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME3_SBTE3_AXI_ERR_RSP" },
+	{ .fc_id = 465, .cpu_id = 91, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME3_SBTE4_AXI_ERR_RSP" },
+	{ .fc_id = 466, .cpu_id = 91, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME3_CTRL_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 467, .cpu_id = 91, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME3_QMAN_SW_ERROR" },
+	{ .fc_id = 468, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PSOC_MME_PLL_LOCK_ERR" },
+	{ .fc_id = 469, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PSOC_CPU_PLL_LOCK_ERR" },
+	{ .fc_id = 470, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE3_TPC_PLL_LOCK_ERR" },
+	{ .fc_id = 471, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE3_NIC_PLL_LOCK_ERR" },
+	{ .fc_id = 472, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE3_XBAR_MMU_PLL_LOCK_ERR" },
+	{ .fc_id = 473, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE3_XBAR_DMA_PLL_LOCK_ERR" },
+	{ .fc_id = 474, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE3_XBAR_IF_PLL_LOCK_ERR" },
+	{ .fc_id = 475, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE3_XBAR_BANK_PLL_LOCK_ERR" },
+	{ .fc_id = 476, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE1_XBAR_MMU_PLL_LOCK_ERR" },
+	{ .fc_id = 477, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE1_XBAR_DMA_PLL_LOCK_ERR" },
+	{ .fc_id = 478, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE1_XBAR_IF_PLL_LOCK_ERR" },
+	{ .fc_id = 479, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE1_XBAR_MESH_PLL_LOCK_ERR" },
+	{ .fc_id = 480, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE1_TPC_PLL_LOCK_ERR" },
+	{ .fc_id = 481, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE1_NIC_PLL_LOCK_ERR" },
+	{ .fc_id = 482, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PMMU_MME_PLL_LOCK_ERR" },
+	{ .fc_id = 483, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE0_TPC_PLL_LOCK_ERR" },
+	{ .fc_id = 484, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE0_PCI_PLL_LOCK_ERR" },
+	{ .fc_id = 485, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE0_XBAR_MMU_PLL_LOCK_ERR" },
+	{ .fc_id = 486, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE0_XBAR_DMA_PLL_LOCK_ERR" },
+	{ .fc_id = 487, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE0_XBAR_IF_PLL_LOCK_ERR" },
+	{ .fc_id = 488, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE0_XBAR_MESH_PLL_LOCK_ERR" },
+	{ .fc_id = 489, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE2_XBAR_MMU_PLL_LOCK_ERR" },
+	{ .fc_id = 490, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE2_XBAR_DMA_PLL_LOCK_ERR" },
+	{ .fc_id = 491, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE2_XBAR_IF_PLL_LOCK_ERR" },
+	{ .fc_id = 492, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE2_XBAR_BANK_PLL_LOCK_ERR" },
+	{ .fc_id = 493, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE2_TPC_PLL_LOCK_ERR" },
+	{ .fc_id = 494, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PSOC_VID_PLL_LOCK_ERR" },
+	{ .fc_id = 495, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PMMU_VID_PLL_LOCK_ERR" },
+	{ .fc_id = 496, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE3_HBM_PLL_LOCK_ERR" },
+	{ .fc_id = 497, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE1_XBAR_HBM_PLL_LOCK_ERR" },
+	{ .fc_id = 498, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE1_HBM_PLL_LOCK_ERR" },
+	{ .fc_id = 499, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE0_HBM_PLL_LOCK_ERR" },
+	{ .fc_id = 500, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE2_XBAR_HBM_PLL_LOCK_ERR" },
+	{ .fc_id = 501, .cpu_id = 92, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DCORE2_HBM_PLL_LOCK_ERR" },
+	{ .fc_id = 502, .cpu_id = 93, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "CPU_AXI_ERR_RSP" },
+	{ .fc_id = 503, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_0_AXI_ERR_RSP" },
+	{ .fc_id = 504, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_1_AXI_ERR_RSP" },
+	{ .fc_id = 505, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_2_AXI_ERR_RSP" },
+	{ .fc_id = 506, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_3_AXI_ERR_RSP" },
+	{ .fc_id = 507, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_8_AXI_ERR_RSP" },
+	{ .fc_id = 508, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_9_AXI_ERR_RSP" },
+	{ .fc_id = 509, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_10_AXI_ERR_RSP" },
+	{ .fc_id = 510, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_11_AXI_ERR_RSP" },
+	{ .fc_id = 511, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_7_AXI_ERR_RSP" },
+	{ .fc_id = 512, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_6_AXI_ERR_RSP" },
+	{ .fc_id = 513, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_5_AXI_ERR_RSP" },
+	{ .fc_id = 514, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_4_AXI_ERR_RSP" },
+	{ .fc_id = 515, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_15_AXI_ERR_RSP" },
+	{ .fc_id = 516, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_14_AXI_ERR_RSP" },
+	{ .fc_id = 517, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_13_AXI_ERR_RSP" },
+	{ .fc_id = 518, .cpu_id = 94, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU_12_AXI_ERR_RSP" },
+	{ .fc_id = 519, .cpu_id = 95, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PMMU_FATAL" },
+	{ .fc_id = 520, .cpu_id = 96, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PMMU_AXI_ERR_RSP" },
+	{ .fc_id = 521, .cpu_id = 97, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "VM0_ALARM_A" },
+	{ .fc_id = 522, .cpu_id = 98, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "VM0_ALARM_B" },
+	{ .fc_id = 523, .cpu_id = 99, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "VM1_ALARM_A" },
+	{ .fc_id = 524, .cpu_id = 100, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "VM1_ALARM_B" },
+	{ .fc_id = 525, .cpu_id = 101, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "VM2_ALARM_A" },
+	{ .fc_id = 526, .cpu_id = 102, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "VM2_ALARM_B" },
+	{ .fc_id = 527, .cpu_id = 103, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "VM3_ALARM_A" },
+	{ .fc_id = 528, .cpu_id = 104, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "VM3_ALARM_B" },
+	{ .fc_id = 529, .cpu_id = 105, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PSOC_AXI_ERR_RSP" },
+	{ .fc_id = 530, .cpu_id = 106, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PSOC_PRSTN_FALL" },
+	{ .fc_id = 531, .cpu_id = 107, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 532, .cpu_id = 107, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 533, .cpu_id = 107, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 534, .cpu_id = 107, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 535, .cpu_id = 107, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 536, .cpu_id = 107, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 537, .cpu_id = 107, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 538, .cpu_id = 107, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 539, .cpu_id = 108, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "KDMA_CH0_AXI_ERR_RSP" },
+	{ .fc_id = 540, .cpu_id = 109, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "PDMA_CH0_AXI_ERR_RSP" },
+	{ .fc_id = 541, .cpu_id = 109, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "PDMA_CH1_AXI_ERR_RSP" },
+	{ .fc_id = 542, .cpu_id = 110, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_CATTRIP_0" },
+	{ .fc_id = 543, .cpu_id = 111, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_CATTRIP_1" },
+	{ .fc_id = 544, .cpu_id = 112, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_CATTRIP_2" },
+	{ .fc_id = 545, .cpu_id = 113, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_CATTRIP_3" },
+	{ .fc_id = 546, .cpu_id = 114, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_CATTRIP_4" },
+	{ .fc_id = 547, .cpu_id = 115, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM_CATTRIP_5" },
+	{ .fc_id = 548, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM0_MC0_SEI_SEVERE" },
+	{ .fc_id = 549, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM0_MC0_SEI_NON_SEVERE" },
+	{ .fc_id = 550, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM0_MC1_SEI_SEVERE" },
+	{ .fc_id = 551, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM0_MC1_SEI_NON_SEVERE" },
+	{ .fc_id = 552, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM1_MC0_SEI_SEVERE" },
+	{ .fc_id = 553, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM1_MC0_SEI_NON_SEVERE" },
+	{ .fc_id = 554, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM1_MC1_SEI_SEVERE" },
+	{ .fc_id = 555, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM1_MC1_SEI_NON_SEVERE" },
+	{ .fc_id = 556, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM2_MC0_SEI_SEVERE" },
+	{ .fc_id = 557, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM2_MC0_SEI_NON_SEVERE" },
+	{ .fc_id = 558, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM2_MC1_SEI_SEVERE" },
+	{ .fc_id = 559, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM2_MC1_SEI_NON_SEVERE" },
+	{ .fc_id = 560, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM3_MC0_SEI_SEVERE" },
+	{ .fc_id = 561, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM3_MC0_SEI_NON_SEVERE" },
+	{ .fc_id = 562, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM3_MC1_SEI_SEVERE" },
+	{ .fc_id = 563, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM3_MC1_SEI_NON_SEVERE" },
+	{ .fc_id = 564, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM4_MC0_SEI_SEVERE" },
+	{ .fc_id = 565, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM4_MC0_SEI_NON_SEVERE" },
+	{ .fc_id = 566, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM4_MC1_SEI_SEVERE" },
+	{ .fc_id = 567, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM4_MC1_SEI_NON_SEVERE" },
+	{ .fc_id = 568, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM5_MC0_SEI_SEVERE" },
+	{ .fc_id = 569, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM5_MC0_SEI_NON_SEVERE" },
+	{ .fc_id = 570, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HBM5_MC1_SEI_SEVERE" },
+	{ .fc_id = 571, .cpu_id = 116, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM5_MC1_SEI_NON_SEVERE" },
+	{ .fc_id = 572, .cpu_id = 117, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC0_AXI_ERR_RSPONSE" },
+	{ .fc_id = 573, .cpu_id = 117, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC1_AXI_ERR_RSPONSE" },
+	{ .fc_id = 574, .cpu_id = 117, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC2_AXI_ERR_RSPONSE" },
+	{ .fc_id = 575, .cpu_id = 117, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC3_AXI_ERR_RSPONSE" },
+	{ .fc_id = 576, .cpu_id = 117, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC4_AXI_ERR_RSPONSE" },
+	{ .fc_id = 577, .cpu_id = 117, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC5_AXI_ERR_RSPONSE" },
+	{ .fc_id = 578, .cpu_id = 117, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC6_AXI_ERR_RSPONSE" },
+	{ .fc_id = 579, .cpu_id = 117, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC7_AXI_ERR_RSPONSE" },
+	{ .fc_id = 580, .cpu_id = 117, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC8_AXI_ERR_RSPONSE" },
+	{ .fc_id = 581, .cpu_id = 117, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC9_AXI_ERR_RSPONSE" },
+	{ .fc_id = 582, .cpu_id = 118, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 583, .cpu_id = 119, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 584, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF0_FATAL" },
+	{ .fc_id = 585, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF1_FATAL" },
+	{ .fc_id = 586, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF2_FATAL" },
+	{ .fc_id = 587, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF3_FATAL" },
+	{ .fc_id = 588, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF8_FATAL" },
+	{ .fc_id = 589, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF9_FATAL" },
+	{ .fc_id = 590, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF10_FATAL" },
+	{ .fc_id = 591, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF11_FATAL" },
+	{ .fc_id = 592, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF7_FATAL" },
+	{ .fc_id = 593, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF6_FATAL" },
+	{ .fc_id = 594, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF5_FATAL" },
+	{ .fc_id = 595, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF4_FATAL" },
+	{ .fc_id = 596, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF15_FATAL" },
+	{ .fc_id = 597, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF14_FATAL" },
+	{ .fc_id = 598, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF13_FATAL" },
+	{ .fc_id = 599, .cpu_id = 120, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HIF12_FATAL" },
+	{ .fc_id = 600, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC0_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 601, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC1_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 602, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC2_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 603, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC3_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 604, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC4_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 605, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC5_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 606, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC6_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 607, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC7_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 608, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC8_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 609, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC9_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 610, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC10_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 611, .cpu_id = 121, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC11_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 612, .cpu_id = 122, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "SM0_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 613, .cpu_id = 122, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "SM1_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 614, .cpu_id = 122, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "SM2_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 615, .cpu_id = 122, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "SM3_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 616, .cpu_id = 123, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "ARC_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 617, .cpu_id = 124, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 618, .cpu_id = 125, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 619, .cpu_id = 125, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PCIE_FLR_REQUESTED" },
+	{ .fc_id = 620, .cpu_id = 125, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 621, .cpu_id = 125, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 622, .cpu_id = 125, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PCIE_APB_TIMEOUT" },
+	{ .fc_id = 623, .cpu_id = 125, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 624, .cpu_id = 125, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 625, .cpu_id = 125, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 626, .cpu_id = 125, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 627, .cpu_id = 125, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PCIE_FATAL_ERR" },
+	{ .fc_id = 628, .cpu_id = 125, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 629, .cpu_id = 126, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 630, .cpu_id = 127, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 631, .cpu_id = 128, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PCIE_P2P_MSIX" },
+	{ .fc_id = 632, .cpu_id = 129, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PCIE_DRAIN_COMPLETE" },
+	{ .fc_id = 633, .cpu_id = 130, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC0_BMON_SPMU" },
+	{ .fc_id = 634, .cpu_id = 131, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC0_KERNEL_ERR" },
+	{ .fc_id = 635, .cpu_id = 132, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC1_BMON_SPMU" },
+	{ .fc_id = 636, .cpu_id = 133, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC1_KERNEL_ERR" },
+	{ .fc_id = 637, .cpu_id = 134, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC2_BMON_SPMU" },
+	{ .fc_id = 638, .cpu_id = 135, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC2_KERNEL_ERR" },
+	{ .fc_id = 639, .cpu_id = 136, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC3_BMON_SPMU" },
+	{ .fc_id = 640, .cpu_id = 137, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC3_KERNEL_ERR" },
+	{ .fc_id = 641, .cpu_id = 138, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC4_BMON_SPMU" },
+	{ .fc_id = 642, .cpu_id = 139, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC4_KERNEL_ERR" },
+	{ .fc_id = 643, .cpu_id = 140, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC5_BMON_SPMU" },
+	{ .fc_id = 644, .cpu_id = 141, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC5_KERNEL_ERR" },
+	{ .fc_id = 645, .cpu_id = 150, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC6_BMON_SPMU" },
+	{ .fc_id = 646, .cpu_id = 151, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC6_KERNEL_ERR" },
+	{ .fc_id = 647, .cpu_id = 152, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC7_BMON_SPMU" },
+	{ .fc_id = 648, .cpu_id = 153, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC7_KERNEL_ERR" },
+	{ .fc_id = 649, .cpu_id = 146, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC8_BMON_SPMU" },
+	{ .fc_id = 650, .cpu_id = 147, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC8_KERNEL_ERR" },
+	{ .fc_id = 651, .cpu_id = 148, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC9_BMON_SPMU" },
+	{ .fc_id = 652, .cpu_id = 149, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC9_KERNEL_ERR" },
+	{ .fc_id = 653, .cpu_id = 142, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC10_BMON_SPMU" },
+	{ .fc_id = 654, .cpu_id = 143, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC10_KERNEL_ERR" },
+	{ .fc_id = 655, .cpu_id = 144, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC11_BMON_SPMU" },
+	{ .fc_id = 656, .cpu_id = 145, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC11_KERNEL_ERR" },
+	{ .fc_id = 657, .cpu_id = 162, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC12_BMON_SPMU" },
+	{ .fc_id = 658, .cpu_id = 163, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC12_KERNEL_ERR" },
+	{ .fc_id = 659, .cpu_id = 164, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC13_BMON_SPMU" },
+	{ .fc_id = 660, .cpu_id = 165, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC13_KERNEL_ERR" },
+	{ .fc_id = 661, .cpu_id = 158, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC14_BMON_SPMU" },
+	{ .fc_id = 662, .cpu_id = 159, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC14_KERNEL_ERR" },
+	{ .fc_id = 663, .cpu_id = 160, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC15_BMON_SPMU" },
+	{ .fc_id = 664, .cpu_id = 161, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC15_KERNEL_ERR" },
+	{ .fc_id = 665, .cpu_id = 154, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC16_BMON_SPMU" },
+	{ .fc_id = 666, .cpu_id = 155, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC16_KERNEL_ERR" },
+	{ .fc_id = 667, .cpu_id = 156, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC17_BMON_SPMU" },
+	{ .fc_id = 668, .cpu_id = 157, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC17_KERNEL_ERR" },
+	{ .fc_id = 669, .cpu_id = 166, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC18_BMON_SPMU" },
+	{ .fc_id = 670, .cpu_id = 167, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC18_KERNEL_ERR" },
+	{ .fc_id = 671, .cpu_id = 168, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC19_BMON_SPMU" },
+	{ .fc_id = 672, .cpu_id = 169, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC19_KERNEL_ERR" },
+	{ .fc_id = 673, .cpu_id = 170, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC20_BMON_SPMU" },
+	{ .fc_id = 674, .cpu_id = 171, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC20_KERNEL_ERR" },
+	{ .fc_id = 675, .cpu_id = 172, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC21_BMON_SPMU" },
+	{ .fc_id = 676, .cpu_id = 173, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC21_KERNEL_ERR" },
+	{ .fc_id = 677, .cpu_id = 174, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC22_BMON_SPMU" },
+	{ .fc_id = 678, .cpu_id = 175, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC22_KERNEL_ERR" },
+	{ .fc_id = 679, .cpu_id = 176, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC23_BMON_SPMU" },
+	{ .fc_id = 680, .cpu_id = 177, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC23_KERNEL_ERR" },
+	{ .fc_id = 681, .cpu_id = 178, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "TPC24_BMON_SPMU" },
+	{ .fc_id = 682, .cpu_id = 179, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC24_KERNEL_ERR" },
+	{ .fc_id = 683, .cpu_id = 180, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 684, .cpu_id = 180, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 685, .cpu_id = 180, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 686, .cpu_id = 180, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 687, .cpu_id = 180, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 688, .cpu_id = 180, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME0_CTRL_BMON_SPMU" },
+	{ .fc_id = 689, .cpu_id = 180, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME0_SBTE_BMON_SPMU" },
+	{ .fc_id = 690, .cpu_id = 180, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME0_WAP_BMON_SPMU" },
+	{ .fc_id = 691, .cpu_id = 180, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME0_WAP_SOURCE_RESULT_INVALID" },
+	{ .fc_id = 692, .cpu_id = 181, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 693, .cpu_id = 181, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 694, .cpu_id = 181, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 695, .cpu_id = 181, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 696, .cpu_id = 181, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 697, .cpu_id = 181, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME1_CTRL_BMON_SPMU" },
+	{ .fc_id = 698, .cpu_id = 181, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME1_SBTE_BMON_SPMU" },
+	{ .fc_id = 699, .cpu_id = 181, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME1_WAP_BMON_SPMU" },
+	{ .fc_id = 700, .cpu_id = 181, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME1_WAP_SOURCE_RESULT_INVALID" },
+	{ .fc_id = 701, .cpu_id = 182, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 702, .cpu_id = 182, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 703, .cpu_id = 182, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 704, .cpu_id = 182, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 705, .cpu_id = 182, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 706, .cpu_id = 182, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME2_CTRL_BMON_SPMU" },
+	{ .fc_id = 707, .cpu_id = 182, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME2_SBTE_BMON_SPMU" },
+	{ .fc_id = 708, .cpu_id = 182, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME2_WAP_BMON_SPMU" },
+	{ .fc_id = 709, .cpu_id = 182, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME2_WAP_SOURCE_RESULT_INVALID" },
+	{ .fc_id = 710, .cpu_id = 183, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 711, .cpu_id = 183, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 712, .cpu_id = 183, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 713, .cpu_id = 183, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 714, .cpu_id = 183, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 715, .cpu_id = 183, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME3_CTRL_BMON_SPMU" },
+	{ .fc_id = 716, .cpu_id = 183, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME3_SBTE_BMON_SPMU" },
+	{ .fc_id = 717, .cpu_id = 183, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "MME3_WAP_BMON_SPMU" },
+	{ .fc_id = 718, .cpu_id = 183, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME3_WAP_SOURCE_RESULT_INVALID" },
+	{ .fc_id = 719, .cpu_id = 184, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 720, .cpu_id = 184, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU0_PAGE_FAULT_OR_WR_PERM" },
+	{ .fc_id = 721, .cpu_id = 184, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU0_SECURITY_ERROR" },
+	{ .fc_id = 722, .cpu_id = 185, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 723, .cpu_id = 185, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU1_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 724, .cpu_id = 185, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU1_SECURITY_ERROR" },
+	{ .fc_id = 725, .cpu_id = 186, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 726, .cpu_id = 186, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU2_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 727, .cpu_id = 186, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU2_SECURITY_ERROR" },
+	{ .fc_id = 728, .cpu_id = 187, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 729, .cpu_id = 187, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU3_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 730, .cpu_id = 187, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU3_SECURITY_ERROR" },
+	{ .fc_id = 731, .cpu_id = 188, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 732, .cpu_id = 188, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU8_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 733, .cpu_id = 188, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU8_SECURITY_ERROR" },
+	{ .fc_id = 734, .cpu_id = 189, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 735, .cpu_id = 189, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU9_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 736, .cpu_id = 189, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU9_SECURITY_ERROR" },
+	{ .fc_id = 737, .cpu_id = 190, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 738, .cpu_id = 190, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU10_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 739, .cpu_id = 190, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU10_SECURITY_ERROR" },
+	{ .fc_id = 740, .cpu_id = 191, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 741, .cpu_id = 191, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU11_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 742, .cpu_id = 191, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU11_SECURITY_ERROR" },
+	{ .fc_id = 743, .cpu_id = 192, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 744, .cpu_id = 192, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU7_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 745, .cpu_id = 192, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU7_SECURITY_ERROR" },
+	{ .fc_id = 746, .cpu_id = 193, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 747, .cpu_id = 193, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU6_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 748, .cpu_id = 193, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU6_SECURITY_ERROR" },
+	{ .fc_id = 749, .cpu_id = 194, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 750, .cpu_id = 194, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU5_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 751, .cpu_id = 194, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU5_SECURITY_ERROR" },
+	{ .fc_id = 752, .cpu_id = 195, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 753, .cpu_id = 195, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU4_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 754, .cpu_id = 195, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU4_SECURITY_ERROR" },
+	{ .fc_id = 755, .cpu_id = 196, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 756, .cpu_id = 196, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU15_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 757, .cpu_id = 196, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU15_SECURITY_ERROR" },
+	{ .fc_id = 758, .cpu_id = 197, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 759, .cpu_id = 197, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU14_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 760, .cpu_id = 197, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU14_SECURITY_ERROR" },
+	{ .fc_id = 761, .cpu_id = 198, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 762, .cpu_id = 198, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU13_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 763, .cpu_id = 198, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU13_SECURITY_ERROR" },
+	{ .fc_id = 764, .cpu_id = 199, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 765, .cpu_id = 199, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HMMU12_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 766, .cpu_id = 199, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "HMMU12_SECURITY_ERROR" },
+	{ .fc_id = 767, .cpu_id = 200, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 768, .cpu_id = 201, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PMMU0_PAGE_FAULT_WR_PERM" },
+	{ .fc_id = 769, .cpu_id = 202, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PMMU0_SECURITY_ERROR" },
+	{ .fc_id = 770, .cpu_id = 203, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA2_BM_SPMU" },
+	{ .fc_id = 771, .cpu_id = 204, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 772, .cpu_id = 205, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA3_BM_SPMU" },
+	{ .fc_id = 773, .cpu_id = 206, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 774, .cpu_id = 207, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA0_BM_SPMU" },
+	{ .fc_id = 775, .cpu_id = 208, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 776, .cpu_id = 209, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA1_BM_SPMU" },
+	{ .fc_id = 777, .cpu_id = 210, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 778, .cpu_id = 211, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA6_BM_SPMU" },
+	{ .fc_id = 779, .cpu_id = 212, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 780, .cpu_id = 213, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA7_BM_SPMU" },
+	{ .fc_id = 781, .cpu_id = 214, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 782, .cpu_id = 215, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA4_BM_SPMU" },
+	{ .fc_id = 783, .cpu_id = 216, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 784, .cpu_id = 217, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "EDMA5_BM_SPMU" },
+	{ .fc_id = 785, .cpu_id = 218, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 786, .cpu_id = 219, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "KDMA_BM_SPMU" },
+	{ .fc_id = 787, .cpu_id = 220, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 788, .cpu_id = 221, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PDMA0_BM_SPMU" },
+	{ .fc_id = 789, .cpu_id = 222, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PDMA1_BM_SPMU" },
+	{ .fc_id = 790, .cpu_id = 223, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM0_MC0_SPI" },
+	{ .fc_id = 791, .cpu_id = 224, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM0_MC1_SPI" },
+	{ .fc_id = 792, .cpu_id = 225, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM1_MC0_SPI" },
+	{ .fc_id = 793, .cpu_id = 226, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM1_MC1_SPI" },
+	{ .fc_id = 794, .cpu_id = 227, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM2_MC0_SPI" },
+	{ .fc_id = 795, .cpu_id = 228, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM2_MC1_SPI" },
+	{ .fc_id = 796, .cpu_id = 229, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM3_MC0_SPI" },
+	{ .fc_id = 797, .cpu_id = 230, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM3_MC1_SPI" },
+	{ .fc_id = 798, .cpu_id = 231, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM4_MC0_SPI" },
+	{ .fc_id = 799, .cpu_id = 232, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM4_MC1_SPI" },
+	{ .fc_id = 800, .cpu_id = 233, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM5_MC0_SPI" },
+	{ .fc_id = 801, .cpu_id = 234, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HBM5_MC1_SPI" },
+	{ .fc_id = 802, .cpu_id = 235, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 803, .cpu_id = 236, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 804, .cpu_id = 237, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 805, .cpu_id = 238, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 806, .cpu_id = 239, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 807, .cpu_id = 240, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 808, .cpu_id = 241, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 809, .cpu_id = 242, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 810, .cpu_id = 243, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 811, .cpu_id = 244, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 812, .cpu_id = 245, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 813, .cpu_id = 246, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 814, .cpu_id = 247, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 815, .cpu_id = 248, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 816, .cpu_id = 249, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 817, .cpu_id = 250, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 818, .cpu_id = 251, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 819, .cpu_id = 252, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 820, .cpu_id = 253, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 821, .cpu_id = 254, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 822, .cpu_id = 255, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 823, .cpu_id = 256, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 824, .cpu_id = 257, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 825, .cpu_id = 258, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 826, .cpu_id = 259, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 827, .cpu_id = 260, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 828, .cpu_id = 261, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 829, .cpu_id = 262, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 830, .cpu_id = 263, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 831, .cpu_id = 264, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 832, .cpu_id = 265, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 833, .cpu_id = 266, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 834, .cpu_id = 267, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 835, .cpu_id = 268, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 836, .cpu_id = 269, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 837, .cpu_id = 270, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 838, .cpu_id = 271, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 839, .cpu_id = 272, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 840, .cpu_id = 273, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 841, .cpu_id = 274, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 842, .cpu_id = 275, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 843, .cpu_id = 276, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 844, .cpu_id = 277, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 845, .cpu_id = 278, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 846, .cpu_id = 279, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 847, .cpu_id = 280, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 848, .cpu_id = 281, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 849, .cpu_id = 282, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 850, .cpu_id = 283, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 851, .cpu_id = 284, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 852, .cpu_id = 285, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 853, .cpu_id = 286, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 854, .cpu_id = 287, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "" },
+	{ .fc_id = 855, .cpu_id = 288, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "" },
+	{ .fc_id = 856, .cpu_id = 289, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 857, .cpu_id = 290, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 858, .cpu_id = 291, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 859, .cpu_id = 292, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 860, .cpu_id = 293, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 861, .cpu_id = 294, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 862, .cpu_id = 295, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 863, .cpu_id = 296, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 864, .cpu_id = 297, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 865, .cpu_id = 298, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 866, .cpu_id = 299, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 867, .cpu_id = 300, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 868, .cpu_id = 301, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 869, .cpu_id = 302, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 870, .cpu_id = 303, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 871, .cpu_id = 304, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "RPM_ERROR_OR_DRAIN" },
+	{ .fc_id = 872, .cpu_id = 305, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 873, .cpu_id = 306, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 874, .cpu_id = 307, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 875, .cpu_id = 308, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "RAZWI_OR_PID_MIN_MAX_INTERRUPT" },
+	{ .fc_id = 876, .cpu_id = 309, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 877, .cpu_id = 310, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 878, .cpu_id = 311, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 879, .cpu_id = 312, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "" },
+	{ .fc_id = 880, .cpu_id = 313, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 881, .cpu_id = 314, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 882, .cpu_id = 315, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 883, .cpu_id = 316, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 884, .cpu_id = 317, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 885, .cpu_id = 318, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 886, .cpu_id = 319, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 887, .cpu_id = 320, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 888, .cpu_id = 321, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 889, .cpu_id = 322, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 890, .cpu_id = 323, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 891, .cpu_id = 324, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 892, .cpu_id = 325, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 893, .cpu_id = 326, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 894, .cpu_id = 327, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 895, .cpu_id = 328, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 896, .cpu_id = 329, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC0_SPI" },
+	{ .fc_id = 897, .cpu_id = 329, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC0_BMON_SPMU" },
+	{ .fc_id = 898, .cpu_id = 330, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC1_SPI" },
+	{ .fc_id = 899, .cpu_id = 330, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC1_SPI" },
+	{ .fc_id = 900, .cpu_id = 331, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC2_SPI" },
+	{ .fc_id = 901, .cpu_id = 331, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC2_BMON_SPMU" },
+	{ .fc_id = 902, .cpu_id = 332, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC3_SPI" },
+	{ .fc_id = 903, .cpu_id = 332, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC3_BMON_SPMU" },
+	{ .fc_id = 904, .cpu_id = 333, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC4_SPI" },
+	{ .fc_id = 905, .cpu_id = 333, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC4_BMON_SPMU" },
+	{ .fc_id = 906, .cpu_id = 334, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC5_SPI" },
+	{ .fc_id = 907, .cpu_id = 334, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC5_BMON_SPMU" },
+	{ .fc_id = 908, .cpu_id = 335, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC6_SPI" },
+	{ .fc_id = 909, .cpu_id = 335, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC6_BMON_SPMU" },
+	{ .fc_id = 910, .cpu_id = 336, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC7_SPI" },
+	{ .fc_id = 911, .cpu_id = 336, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC7_BMON_SPMU" },
+	{ .fc_id = 912, .cpu_id = 337, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC8_SPI" },
+	{ .fc_id = 913, .cpu_id = 337, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC8_BMON_SPMU" },
+	{ .fc_id = 914, .cpu_id = 338, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "DEC9_SPI" },
+	{ .fc_id = 915, .cpu_id = 338, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "DEC9_BMON_SPMU" },
+	{ .fc_id = 916, .cpu_id = 339, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 917, .cpu_id = 340, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 918, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 919, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 920, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 921, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 922, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 923, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 924, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 925, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 926, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 927, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 928, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 929, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 930, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 931, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 932, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 933, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 934, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 935, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 936, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 937, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 938, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 939, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 940, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 941, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 942, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 943, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 944, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 945, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 946, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 947, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 948, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 949, .cpu_id = 341, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 950, .cpu_id = 342, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 951, .cpu_id = 343, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC0_BMON_SPMU" },
+	{ .fc_id = 952, .cpu_id = 343, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC0_SW_ERROR" },
+	{ .fc_id = 953, .cpu_id = 343, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 954, .cpu_id = 343, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 955, .cpu_id = 344, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC1_BMON_SPMU" },
+	{ .fc_id = 956, .cpu_id = 344, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC1_SW_ERROR" },
+	{ .fc_id = 957, .cpu_id = 344, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 958, .cpu_id = 344, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 959, .cpu_id = 345, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC2_BMON_SPMU" },
+	{ .fc_id = 960, .cpu_id = 345, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC2_SW_ERROR" },
+	{ .fc_id = 961, .cpu_id = 345, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 962, .cpu_id = 345, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 963, .cpu_id = 346, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC3_BMON_SPMU" },
+	{ .fc_id = 964, .cpu_id = 346, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC3_SW_ERROR" },
+	{ .fc_id = 965, .cpu_id = 346, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 966, .cpu_id = 346, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 967, .cpu_id = 347, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC4_BMON_SPMU" },
+	{ .fc_id = 968, .cpu_id = 347, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC4_SW_ERROR" },
+	{ .fc_id = 969, .cpu_id = 347, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 970, .cpu_id = 347, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 971, .cpu_id = 348, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC5_BMON_SPMU" },
+	{ .fc_id = 972, .cpu_id = 348, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC5_SW_ERROR" },
+	{ .fc_id = 973, .cpu_id = 348, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 974, .cpu_id = 348, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 975, .cpu_id = 349, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC6_BMON_SPMU" },
+	{ .fc_id = 976, .cpu_id = 349, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC6_SW_ERROR" },
+	{ .fc_id = 977, .cpu_id = 349, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 978, .cpu_id = 349, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 979, .cpu_id = 350, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC7_BMON_SPMU" },
+	{ .fc_id = 980, .cpu_id = 350, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC7_SW_ERROR" },
+	{ .fc_id = 981, .cpu_id = 350, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 982, .cpu_id = 350, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 983, .cpu_id = 351, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC8_BMON_SPMU" },
+	{ .fc_id = 984, .cpu_id = 351, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC8_SW_ERROR" },
+	{ .fc_id = 985, .cpu_id = 351, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 986, .cpu_id = 351, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 987, .cpu_id = 352, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC9_BMON_SPMU" },
+	{ .fc_id = 988, .cpu_id = 352, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC9_SW_ERROR" },
+	{ .fc_id = 989, .cpu_id = 352, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 990, .cpu_id = 352, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 991, .cpu_id = 353, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC10_BMON_SPMU" },
+	{ .fc_id = 992, .cpu_id = 353, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC10_SW_ERROR" },
+	{ .fc_id = 993, .cpu_id = 353, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 994, .cpu_id = 353, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 995, .cpu_id = 354, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC11_BMON_SPMU" },
+	{ .fc_id = 996, .cpu_id = 354, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "NIC11_SW_ERROR" },
+	{ .fc_id = 997, .cpu_id = 354, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 998, .cpu_id = 354, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 999, .cpu_id = 355, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1000, .cpu_id = 356, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1001, .cpu_id = 357, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1002, .cpu_id = 358, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1003, .cpu_id = 359, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1004, .cpu_id = 360, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1005, .cpu_id = 361, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1006, .cpu_id = 362, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1007, .cpu_id = 363, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1008, .cpu_id = 368, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1009, .cpu_id = 369, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1010, .cpu_id = 366, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1011, .cpu_id = 367, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1012, .cpu_id = 364, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1013, .cpu_id = 365, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1014, .cpu_id = 374, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1015, .cpu_id = 375, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1016, .cpu_id = 372, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1017, .cpu_id = 373, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1018, .cpu_id = 370, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1019, .cpu_id = 371, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1020, .cpu_id = 376, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1021, .cpu_id = 377, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1022, .cpu_id = 378, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1023, .cpu_id = 379, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1024, .cpu_id = 380, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1025, .cpu_id = 381, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1026, .cpu_id = 382, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1027, .cpu_id = 383, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1028, .cpu_id = 384, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1029, .cpu_id = 385, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1030, .cpu_id = 386, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1031, .cpu_id = 387, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1032, .cpu_id = 388, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1033, .cpu_id = 389, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1034, .cpu_id = 390, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1035, .cpu_id = 391, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1036, .cpu_id = 392, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1037, .cpu_id = 393, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1038, .cpu_id = 394, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1039, .cpu_id = 395, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1040, .cpu_id = 396, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1041, .cpu_id = 397, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1042, .cpu_id = 398, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1043, .cpu_id = 399, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1044, .cpu_id = 400, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1045, .cpu_id = 401, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1046, .cpu_id = 402, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1047, .cpu_id = 403, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1048, .cpu_id = 404, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1049, .cpu_id = 405, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1050, .cpu_id = 406, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1051, .cpu_id = 407, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1052, .cpu_id = 408, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1053, .cpu_id = 409, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1054, .cpu_id = 410, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1055, .cpu_id = 411, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1056, .cpu_id = 412, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1057, .cpu_id = 413, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1058, .cpu_id = 414, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1059, .cpu_id = 414, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1060, .cpu_id = 414, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1061, .cpu_id = 414, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1062, .cpu_id = 414, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1063, .cpu_id = 414, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1064, .cpu_id = 414, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1065, .cpu_id = 414, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1066, .cpu_id = 414, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1067, .cpu_id = 414, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1068, .cpu_id = 415, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1069, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1070, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1071, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1072, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1073, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1074, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1075, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1076, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1077, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1078, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1079, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1080, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1081, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1082, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1083, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1084, .cpu_id = 416, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1085, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1086, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1087, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1088, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1089, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1090, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1091, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1092, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1093, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1094, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1095, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1096, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1097, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1098, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1099, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1100, .cpu_id = 417, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1101, .cpu_id = 418, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1102, .cpu_id = 419, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1103, .cpu_id = 420, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1104, .cpu_id = 421, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1105, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1106, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1107, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1108, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1109, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1110, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1111, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1112, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1113, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1114, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1115, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1116, .cpu_id = 422, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1117, .cpu_id = 423, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1118, .cpu_id = 424, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "ROTATOR0_SERR" },
+	{ .fc_id = 1119, .cpu_id = 425, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "ROTATOR1_SERR" },
+	{ .fc_id = 1120, .cpu_id = 426, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "ROTATOR0_DERR" },
+	{ .fc_id = 1121, .cpu_id = 427, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "ROTATOR1_DERR" },
+	{ .fc_id = 1122, .cpu_id = 428, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "ROTATOR0_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 1123, .cpu_id = 429, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "ROTATOR1_AXI_ERROR_RESPONSE" },
+	{ .fc_id = 1124, .cpu_id = 430, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1125, .cpu_id = 431, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1126, .cpu_id = 432, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "ROTATOR0_BMON_SPMU" },
+	{ .fc_id = 1127, .cpu_id = 433, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1128, .cpu_id = 434, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "ROTATOR1_BMON_SPMU" },
+	{ .fc_id = 1129, .cpu_id = 435, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1130, .cpu_id = 436, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SM0_BMON_SPMU" },
+	{ .fc_id = 1131, .cpu_id = 437, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SM1_BMON_SPMU" },
+	{ .fc_id = 1132, .cpu_id = 438, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SM2_BMON_SPMU" },
+	{ .fc_id = 1133, .cpu_id = 439, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SM3_BMON_SPMU" },
+	{ .fc_id = 1134, .cpu_id = 440, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1135, .cpu_id = 441, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1136, .cpu_id = 442, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1137, .cpu_id = 443, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1138, .cpu_id = 444, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1139, .cpu_id = 445, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1140, .cpu_id = 446, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1141, .cpu_id = 447, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1142, .cpu_id = 448, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1143, .cpu_id = 449, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1144, .cpu_id = 450, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1145, .cpu_id = 451, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1146, .cpu_id = 452, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1147, .cpu_id = 453, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1148, .cpu_id = 454, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1149, .cpu_id = 455, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1150, .cpu_id = 456, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1151, .cpu_id = 457, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1152, .cpu_id = 458, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1153, .cpu_id = 459, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1154, .cpu_id = 460, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1155, .cpu_id = 461, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1156, .cpu_id = 462, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1157, .cpu_id = 463, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1158, .cpu_id = 464, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1159, .cpu_id = 465, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1160, .cpu_id = 466, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1161, .cpu_id = 467, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1162, .cpu_id = 468, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1163, .cpu_id = 469, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1164, .cpu_id = 470, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1165, .cpu_id = 471, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1166, .cpu_id = 472, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1167, .cpu_id = 473, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1168, .cpu_id = 474, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1169, .cpu_id = 475, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1170, .cpu_id = 476, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1171, .cpu_id = 477, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1172, .cpu_id = 478, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1173, .cpu_id = 479, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1174, .cpu_id = 480, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1175, .cpu_id = 481, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1176, .cpu_id = 482, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1177, .cpu_id = 483, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1178, .cpu_id = 484, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1179, .cpu_id = 485, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1180, .cpu_id = 486, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1181, .cpu_id = 487, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1182, .cpu_id = 488, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1183, .cpu_id = 489, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1184, .cpu_id = 490, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1185, .cpu_id = 491, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1186, .cpu_id = 492, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1187, .cpu_id = 493, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1188, .cpu_id = 494, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1189, .cpu_id = 495, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1190, .cpu_id = 496, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1191, .cpu_id = 497, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1192, .cpu_id = 498, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1193, .cpu_id = 499, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1194, .cpu_id = 500, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1195, .cpu_id = 501, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1196, .cpu_id = 502, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1197, .cpu_id = 503, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1198, .cpu_id = 504, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1199, .cpu_id = 505, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1200, .cpu_id = 506, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1201, .cpu_id = 507, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1202, .cpu_id = 508, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1203, .cpu_id = 509, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1204, .cpu_id = 510, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1205, .cpu_id = 511, .valid = 0, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1206, .cpu_id = 512, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC0_QM" },
+	{ .fc_id = 1207, .cpu_id = 513, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC1_QM" },
+	{ .fc_id = 1208, .cpu_id = 514, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC2_QM" },
+	{ .fc_id = 1209, .cpu_id = 515, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC3_QM" },
+	{ .fc_id = 1210, .cpu_id = 516, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC4_QM" },
+	{ .fc_id = 1211, .cpu_id = 517, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC5_QM" },
+	{ .fc_id = 1212, .cpu_id = 518, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC6_QM" },
+	{ .fc_id = 1213, .cpu_id = 519, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC7_QM" },
+	{ .fc_id = 1214, .cpu_id = 520, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC8_QM" },
+	{ .fc_id = 1215, .cpu_id = 521, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC9_QM" },
+	{ .fc_id = 1216, .cpu_id = 522, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC10_QM" },
+	{ .fc_id = 1217, .cpu_id = 523, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC11_QM" },
+	{ .fc_id = 1218, .cpu_id = 524, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC12_QM" },
+	{ .fc_id = 1219, .cpu_id = 525, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC13_QM" },
+	{ .fc_id = 1220, .cpu_id = 526, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC14_QM" },
+	{ .fc_id = 1221, .cpu_id = 527, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC15_QM" },
+	{ .fc_id = 1222, .cpu_id = 528, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC16_QM" },
+	{ .fc_id = 1223, .cpu_id = 529, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC17_QM" },
+	{ .fc_id = 1224, .cpu_id = 530, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC18_QM" },
+	{ .fc_id = 1225, .cpu_id = 531, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC19_QM" },
+	{ .fc_id = 1226, .cpu_id = 532, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC20_QM" },
+	{ .fc_id = 1227, .cpu_id = 533, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC21_QM" },
+	{ .fc_id = 1228, .cpu_id = 534, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC22_QM" },
+	{ .fc_id = 1229, .cpu_id = 535, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC23_QM" },
+	{ .fc_id = 1230, .cpu_id = 536, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "TPC24_QM" },
+	{ .fc_id = 1231, .cpu_id = 537, .valid = 0, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "" },
+	{ .fc_id = 1232, .cpu_id = 538, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME0_QM" },
+	{ .fc_id = 1233, .cpu_id = 539, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME1_QM" },
+	{ .fc_id = 1234, .cpu_id = 540, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME2_QM" },
+	{ .fc_id = 1235, .cpu_id = 541, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "MME3_QM" },
+	{ .fc_id = 1236, .cpu_id = 542, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA2_QM" },
+	{ .fc_id = 1237, .cpu_id = 543, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA3_QM" },
+	{ .fc_id = 1238, .cpu_id = 544, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA0_QM" },
+	{ .fc_id = 1239, .cpu_id = 545, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA1_QM" },
+	{ .fc_id = 1240, .cpu_id = 546, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA6_QM" },
+	{ .fc_id = 1241, .cpu_id = 547, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA7_QM" },
+	{ .fc_id = 1242, .cpu_id = 548, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA4_QM" },
+	{ .fc_id = 1243, .cpu_id = 549, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA5_QM" },
+	{ .fc_id = 1244, .cpu_id = 550, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "PDMA0_QM" },
+	{ .fc_id = 1245, .cpu_id = 551, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "PDMA1_QM" },
+	{ .fc_id = 1246, .cpu_id = 552, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "PI_UPDATE" },
+	{ .fc_id = 1247, .cpu_id = 553, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "HALT_MACHINE" },
+	{ .fc_id = 1248, .cpu_id = 554, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "INTS_REGISTER" },
+	{ .fc_id = 1249, .cpu_id = 555, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "ROT0_QM" },
+	{ .fc_id = 1250, .cpu_id = 556, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "ROT1_QM" },
+	{ .fc_id = 1251, .cpu_id = 557, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "SOFT_RESET" },
+	{ .fc_id = 1252, .cpu_id = 558, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "CPLD_SHUTDOWN_CAUSE" },
+	{ .fc_id = 1253, .cpu_id = 559, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "FIX_POWER_ENV_S" },
+	{ .fc_id = 1254, .cpu_id = 560, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "FIX_POWER_ENV_E" },
+	{ .fc_id = 1255, .cpu_id = 561, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "FIX_THERMAL_ENV_S" },
+	{ .fc_id = 1256, .cpu_id = 562, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "FIX_THERMAL_ENV_E" },
+	{ .fc_id = 1257, .cpu_id = 563, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "CPLD_SHUTDOWN_EVENT" },
+	{ .fc_id = 1258, .cpu_id = 564, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "PKT_QUEUE_OUT_SYNC" },
+	{ .fc_id = 1259, .cpu_id = 565, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA2_CORE" },
+	{ .fc_id = 1260, .cpu_id = 566, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA3_CORE" },
+	{ .fc_id = 1261, .cpu_id = 567, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA0_CORE" },
+	{ .fc_id = 1262, .cpu_id = 568, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA1_CORE" },
+	{ .fc_id = 1263, .cpu_id = 569, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA6_CORE" },
+	{ .fc_id = 1264, .cpu_id = 570, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA7_CORE" },
+	{ .fc_id = 1265, .cpu_id = 571, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA4_CORE" },
+	{ .fc_id = 1266, .cpu_id = 572, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "EDMA5_CORE" },
+	{ .fc_id = 1267, .cpu_id = 573, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "PDMA0_CORE" },
+	{ .fc_id = 1268, .cpu_id = 574, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "PDMA1_CORE" },
+	{ .fc_id = 1269, .cpu_id = 575, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "KDMA0_CORE" },
+	{ .fc_id = 1270, .cpu_id = 576, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC0_QM0" },
+	{ .fc_id = 1271, .cpu_id = 577, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC0_QM1" },
+	{ .fc_id = 1272, .cpu_id = 578, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC1_QM0" },
+	{ .fc_id = 1273, .cpu_id = 579, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC1_QM1" },
+	{ .fc_id = 1274, .cpu_id = 580, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC2_QM0" },
+	{ .fc_id = 1275, .cpu_id = 581, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC2_QM1" },
+	{ .fc_id = 1276, .cpu_id = 582, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC3_QM0" },
+	{ .fc_id = 1277, .cpu_id = 583, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC3_QM1" },
+	{ .fc_id = 1278, .cpu_id = 584, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC4_QM0" },
+	{ .fc_id = 1279, .cpu_id = 585, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC4_QM1" },
+	{ .fc_id = 1280, .cpu_id = 586, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC5_QM0" },
+	{ .fc_id = 1281, .cpu_id = 587, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC5_QM1" },
+	{ .fc_id = 1282, .cpu_id = 588, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC6_QM0" },
+	{ .fc_id = 1283, .cpu_id = 589, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC6_QM1" },
+	{ .fc_id = 1284, .cpu_id = 590, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC7_QM0" },
+	{ .fc_id = 1285, .cpu_id = 591, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC7_QM1" },
+	{ .fc_id = 1286, .cpu_id = 592, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC8_QM0" },
+	{ .fc_id = 1287, .cpu_id = 593, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC8_QM1" },
+	{ .fc_id = 1288, .cpu_id = 594, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC9_QM0" },
+	{ .fc_id = 1289, .cpu_id = 595, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC9_QM1" },
+	{ .fc_id = 1290, .cpu_id = 596, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC10_QM0" },
+	{ .fc_id = 1291, .cpu_id = 597, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC10_QM1" },
+	{ .fc_id = 1292, .cpu_id = 598, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC11_QM0" },
+	{ .fc_id = 1293, .cpu_id = 599, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "NIC11_QM1" },
+	{ .fc_id = 1294, .cpu_id = 600, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "CPU_PKT_SANITY_FAILED" },
+	{ .fc_id = 1295, .cpu_id = 601, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC0_ENG0" },
+	{ .fc_id = 1296, .cpu_id = 602, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC0_ENG1" },
+	{ .fc_id = 1297, .cpu_id = 603, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC1_ENG0" },
+	{ .fc_id = 1298, .cpu_id = 604, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC1_ENG1" },
+	{ .fc_id = 1299, .cpu_id = 605, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC2_ENG0" },
+	{ .fc_id = 1300, .cpu_id = 606, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC2_ENG1" },
+	{ .fc_id = 1301, .cpu_id = 607, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC3_ENG0" },
+	{ .fc_id = 1302, .cpu_id = 608, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC3_ENG1" },
+	{ .fc_id = 1303, .cpu_id = 609, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC4_ENG0" },
+	{ .fc_id = 1304, .cpu_id = 610, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC4_ENG1" },
+	{ .fc_id = 1305, .cpu_id = 611, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC5_ENG0" },
+	{ .fc_id = 1306, .cpu_id = 612, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC5_ENG1" },
+	{ .fc_id = 1307, .cpu_id = 613, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC6_ENG0" },
+	{ .fc_id = 1308, .cpu_id = 614, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC6_ENG1" },
+	{ .fc_id = 1309, .cpu_id = 615, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC7_ENG0" },
+	{ .fc_id = 1310, .cpu_id = 616, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC7_ENG1" },
+	{ .fc_id = 1311, .cpu_id = 617, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC8_ENG0" },
+	{ .fc_id = 1312, .cpu_id = 618, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC8_ENG1" },
+	{ .fc_id = 1313, .cpu_id = 619, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC9_ENG0" },
+	{ .fc_id = 1314, .cpu_id = 620, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC9_ENG1" },
+	{ .fc_id = 1315, .cpu_id = 621, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC10_ENG0" },
+	{ .fc_id = 1316, .cpu_id = 622, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC10_ENG1" },
+	{ .fc_id = 1317, .cpu_id = 623, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC11_ENG0" },
+	{ .fc_id = 1318, .cpu_id = 624, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+		 .name = "STATUS_NIC11_ENG1" },
+	{ .fc_id = 1319, .cpu_id = 625, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_COMPUTE,
+		 .name = "ARC_DCCM_FULL" },
+	{ .fc_id = 1320, .cpu_id = 626, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "FP32_NOT_SUPPORTED" },
+	{ .fc_id = 1321, .cpu_id = 627, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
+		 .name = "DEV_RESET_REQ" },
 };
 
 #endif /* __GAUDI2_ASYNC_IDS_MAP_EVENTS_EXT_H_ */
diff --git a/drivers/accel/habanalabs/include/gaudi2/gaudi2_fw_if.h b/drivers/accel/habanalabs/include/gaudi2/gaudi2_fw_if.h
index 82f3ca2a3966..8522f24deac0 100644
--- a/drivers/accel/habanalabs/include/gaudi2/gaudi2_fw_if.h
+++ b/drivers/accel/habanalabs/include/gaudi2/gaudi2_fw_if.h
@@ -63,7 +63,10 @@ struct gaudi2_cold_rst_data {
 			u32 fake_sig_validation_en : 1;
 			u32 bist_skip_enable : 1;
 			u32 bist_need_iatu_config : 1;
-			u32 reserved : 24;
+			u32 fake_bis_compliant : 1;
+			u32 wd_rst_cause_arm : 1;
+			u32 wd_rst_cause_arcpid : 1;
+			u32 reserved : 21;
 		};
 		__le32 data;
 	};
diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 6a320a73e3cc..8396db2b5203 100644
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -437,6 +437,10 @@ static int ivpu_pci_init(struct ivpu_device *vdev)
 	/* Clear any pending errors */
 	pcie_capability_clear_word(pdev, PCI_EXP_DEVSTA, 0x3f);
 
+	/* VPU MTL does not require PCI spec 10m D3hot delay */
+	if (ivpu_is_mtl(vdev))
+		pdev->d3hot_delay = 0;
+
 	ret = pcim_enable_device(pdev);
 	if (ret) {
 		ivpu_err(vdev, "Failed to enable PCI device: %d\n", ret);
diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c
index bde42d6383da..aa4d56dc52b3 100644
--- a/drivers/accel/ivpu/ivpu_pm.c
+++ b/drivers/accel/ivpu/ivpu_pm.c
@@ -239,8 +239,6 @@ int ivpu_rpm_get(struct ivpu_device *vdev)
 {
 	int ret;
 
-	ivpu_dbg(vdev, RPM, "rpm_get count %d\n", atomic_read(&vdev->drm.dev->power.usage_count));
-
 	ret = pm_runtime_resume_and_get(vdev->drm.dev);
 	if (!drm_WARN_ON(&vdev->drm, ret < 0))
 		vdev->pm->suspend_reschedule_counter = PM_RESCHEDULE_LIMIT;
@@ -250,8 +248,6 @@ int ivpu_rpm_get(struct ivpu_device *vdev)
 
 void ivpu_rpm_put(struct ivpu_device *vdev)
 {
-	ivpu_dbg(vdev, RPM, "rpm_put count %d\n", atomic_read(&vdev->drm.dev->power.usage_count));
-
 	pm_runtime_mark_last_busy(vdev->drm.dev);
 	pm_runtime_put_autosuspend(vdev->drm.dev);
 }
@@ -321,16 +317,10 @@ void ivpu_pm_enable(struct ivpu_device *vdev)
 	pm_runtime_allow(dev);
 	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
-
-	ivpu_dbg(vdev, RPM, "Enable RPM count %d\n", atomic_read(&dev->power.usage_count));
 }
 
 void ivpu_pm_disable(struct ivpu_device *vdev)
 {
-	struct device *dev = vdev->drm.dev;
-
-	ivpu_dbg(vdev, RPM, "Disable RPM count %d\n", atomic_read(&dev->power.usage_count));
-
 	pm_runtime_get_noresume(vdev->drm.dev);
 	pm_runtime_forbid(vdev->drm.dev);
 }
diff --git a/drivers/accel/qaic/Kconfig b/drivers/accel/qaic/Kconfig
new file mode 100644
index 000000000000..a9f866230058
--- /dev/null
+++ b/drivers/accel/qaic/Kconfig
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Qualcomm Cloud AI accelerators driver
+#
+
+config DRM_ACCEL_QAIC
+	tristate "Qualcomm Cloud AI accelerators"
+	depends on DRM_ACCEL
+	depends on PCI && HAS_IOMEM
+	depends on MHI_BUS
+	depends on MMU
+	select CRC32
+	help
+	  Enables driver for Qualcomm's Cloud AI accelerator PCIe cards that are
+	  designed to accelerate Deep Learning inference workloads.
+
+	  The driver manages the PCIe devices and provides an IOCTL interface
+	  for users to submit workloads to the devices.
+
+	  If unsure, say N.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called qaic.
diff --git a/drivers/accel/qaic/Makefile b/drivers/accel/qaic/Makefile
new file mode 100644
index 000000000000..2418418f7a50
--- /dev/null
+++ b/drivers/accel/qaic/Makefile
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for Qualcomm Cloud AI accelerators driver
+#
+
+obj-$(CONFIG_DRM_ACCEL_QAIC)	:= qaic.o
+
+qaic-y := \
+	mhi_controller.o \
+	qaic_control.o \
+	qaic_data.o \
+	qaic_drv.o
diff --git a/drivers/accel/qaic/mhi_controller.c b/drivers/accel/qaic/mhi_controller.c
new file mode 100644
index 000000000000..5036e58e7235
--- /dev/null
+++ b/drivers/accel/qaic/mhi_controller.c
@@ -0,0 +1,563 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
+/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/memblock.h>
+#include <linux/mhi.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/sizes.h>
+
+#include "mhi_controller.h"
+#include "qaic.h"
+
+#define MAX_RESET_TIME_SEC 25
+
+static unsigned int mhi_timeout_ms = 2000; /* 2 sec default */
+module_param(mhi_timeout_ms, uint, 0600);
+MODULE_PARM_DESC(mhi_timeout_ms, "MHI controller timeout value");
+
+static struct mhi_channel_config aic100_channels[] = {
+	{
+		.name = "QAIC_LOOPBACK",
+		.num = 0,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_LOOPBACK",
+		.num = 1,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_SAHARA",
+		.num = 2,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_SBL,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_SAHARA",
+		.num = 3,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_SBL,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_DIAG",
+		.num = 4,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_DIAG",
+		.num = 5,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_SSR",
+		.num = 6,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_SSR",
+		.num = 7,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_QDSS",
+		.num = 8,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_QDSS",
+		.num = 9,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_CONTROL",
+		.num = 10,
+		.num_elements = 128,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_CONTROL",
+		.num = 11,
+		.num_elements = 128,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_LOGGING",
+		.num = 12,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_SBL,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_LOGGING",
+		.num = 13,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_SBL,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_STATUS",
+		.num = 14,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_STATUS",
+		.num = 15,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_TELEMETRY",
+		.num = 16,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_TELEMETRY",
+		.num = 17,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_DEBUG",
+		.num = 18,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_DEBUG",
+		.num = 19,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_TIMESYNC",
+		.num = 20,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_SBL | MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.num = 21,
+		.name = "QAIC_TIMESYNC",
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_SBL | MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+};
+
+static struct mhi_event_config aic100_events[] = {
+	{
+		.num_elements = 32,
+		.irq_moderation_ms = 0,
+		.irq = 0,
+		.channel = U32_MAX,
+		.priority = 1,
+		.mode = MHI_DB_BRST_DISABLE,
+		.data_type = MHI_ER_CTRL,
+		.hardware_event = false,
+		.client_managed = false,
+		.offload_channel = false,
+	},
+};
+
+static struct mhi_controller_config aic100_config = {
+	.max_channels = 128,
+	.timeout_ms = 0, /* controlled by mhi_timeout */
+	.buf_len = 0,
+	.num_channels = ARRAY_SIZE(aic100_channels),
+	.ch_cfg = aic100_channels,
+	.num_events = ARRAY_SIZE(aic100_events),
+	.event_cfg = aic100_events,
+	.use_bounce_buf = false,
+	.m2_no_db = false,
+};
+
+static int mhi_read_reg(struct mhi_controller *mhi_cntrl, void __iomem *addr, u32 *out)
+{
+	u32 tmp = readl_relaxed(addr);
+
+	if (tmp == U32_MAX)
+		return -EIO;
+
+	*out = tmp;
+
+	return 0;
+}
+
+static void mhi_write_reg(struct mhi_controller *mhi_cntrl, void __iomem *addr, u32 val)
+{
+	writel_relaxed(val, addr);
+}
+
+static int mhi_runtime_get(struct mhi_controller *mhi_cntrl)
+{
+	return 0;
+}
+
+static void mhi_runtime_put(struct mhi_controller *mhi_cntrl)
+{
+}
+
+static void mhi_status_cb(struct mhi_controller *mhi_cntrl, enum mhi_callback reason)
+{
+	struct qaic_device *qdev = pci_get_drvdata(to_pci_dev(mhi_cntrl->cntrl_dev));
+
+	/* this event occurs in atomic context */
+	if (reason == MHI_CB_FATAL_ERROR)
+		pci_err(qdev->pdev, "Fatal error received from device. Attempting to recover\n");
+	/* this event occurs in non-atomic context */
+	if (reason == MHI_CB_SYS_ERROR)
+		qaic_dev_reset_clean_local_state(qdev, true);
+}
+
+static int mhi_reset_and_async_power_up(struct mhi_controller *mhi_cntrl)
+{
+	u8 time_sec = 1;
+	int current_ee;
+	int ret;
+
+	/* Reset the device to bring the device in PBL EE */
+	mhi_soc_reset(mhi_cntrl);
+
+	/*
+	 * Keep checking the execution environment(EE) after every 1 second
+	 * interval.
+	 */
+	do {
+		msleep(1000);
+		current_ee = mhi_get_exec_env(mhi_cntrl);
+	} while (current_ee != MHI_EE_PBL && time_sec++ <= MAX_RESET_TIME_SEC);
+
+	/* If the device is in PBL EE retry power up */
+	if (current_ee == MHI_EE_PBL)
+		ret = mhi_async_power_up(mhi_cntrl);
+	else
+		ret = -EIO;
+
+	return ret;
+}
+
+struct mhi_controller *qaic_mhi_register_controller(struct pci_dev *pci_dev, void __iomem *mhi_bar,
+						    int mhi_irq)
+{
+	struct mhi_controller *mhi_cntrl;
+	int ret;
+
+	mhi_cntrl = devm_kzalloc(&pci_dev->dev, sizeof(*mhi_cntrl), GFP_KERNEL);
+	if (!mhi_cntrl)
+		return ERR_PTR(-ENOMEM);
+
+	mhi_cntrl->cntrl_dev = &pci_dev->dev;
+
+	/*
+	 * Covers the entire possible physical ram region. Remote side is
+	 * going to calculate a size of this range, so subtract 1 to prevent
+	 * rollover.
+	 */
+	mhi_cntrl->iova_start = 0;
+	mhi_cntrl->iova_stop = PHYS_ADDR_MAX - 1;
+	mhi_cntrl->status_cb = mhi_status_cb;
+	mhi_cntrl->runtime_get = mhi_runtime_get;
+	mhi_cntrl->runtime_put = mhi_runtime_put;
+	mhi_cntrl->read_reg = mhi_read_reg;
+	mhi_cntrl->write_reg = mhi_write_reg;
+	mhi_cntrl->regs = mhi_bar;
+	mhi_cntrl->reg_len = SZ_4K;
+	mhi_cntrl->nr_irqs = 1;
+	mhi_cntrl->irq = devm_kmalloc(&pci_dev->dev, sizeof(*mhi_cntrl->irq), GFP_KERNEL);
+
+	if (!mhi_cntrl->irq)
+		return ERR_PTR(-ENOMEM);
+
+	mhi_cntrl->irq[0] = mhi_irq;
+	mhi_cntrl->fw_image = "qcom/aic100/sbl.bin";
+
+	/* use latest configured timeout */
+	aic100_config.timeout_ms = mhi_timeout_ms;
+	ret = mhi_register_controller(mhi_cntrl, &aic100_config);
+	if (ret) {
+		pci_err(pci_dev, "mhi_register_controller failed %d\n", ret);
+		return ERR_PTR(ret);
+	}
+
+	ret = mhi_prepare_for_power_up(mhi_cntrl);
+	if (ret) {
+		pci_err(pci_dev, "mhi_prepare_for_power_up failed %d\n", ret);
+		goto prepare_power_up_fail;
+	}
+
+	ret = mhi_async_power_up(mhi_cntrl);
+	/*
+	 * If EIO is returned it is possible that device is in SBL EE, which is
+	 * undesired. SOC reset the device and try to power up again.
+	 */
+	if (ret == -EIO && MHI_EE_SBL == mhi_get_exec_env(mhi_cntrl)) {
+		pci_err(pci_dev, "Found device in SBL at MHI init. Attempting a reset.\n");
+		ret = mhi_reset_and_async_power_up(mhi_cntrl);
+	}
+
+	if (ret) {
+		pci_err(pci_dev, "mhi_async_power_up failed %d\n", ret);
+		goto power_up_fail;
+	}
+
+	return mhi_cntrl;
+
+power_up_fail:
+	mhi_unprepare_after_power_down(mhi_cntrl);
+prepare_power_up_fail:
+	mhi_unregister_controller(mhi_cntrl);
+	return ERR_PTR(ret);
+}
+
+void qaic_mhi_free_controller(struct mhi_controller *mhi_cntrl, bool link_up)
+{
+	mhi_power_down(mhi_cntrl, link_up);
+	mhi_unprepare_after_power_down(mhi_cntrl);
+	mhi_unregister_controller(mhi_cntrl);
+}
+
+void qaic_mhi_start_reset(struct mhi_controller *mhi_cntrl)
+{
+	mhi_power_down(mhi_cntrl, true);
+}
+
+void qaic_mhi_reset_done(struct mhi_controller *mhi_cntrl)
+{
+	struct pci_dev *pci_dev = container_of(mhi_cntrl->cntrl_dev, struct pci_dev, dev);
+	int ret;
+
+	ret = mhi_async_power_up(mhi_cntrl);
+	if (ret)
+		pci_err(pci_dev, "mhi_async_power_up failed after reset %d\n", ret);
+}
diff --git a/drivers/accel/qaic/mhi_controller.h b/drivers/accel/qaic/mhi_controller.h
new file mode 100644
index 000000000000..2ae45d768e24
--- /dev/null
+++ b/drivers/accel/qaic/mhi_controller.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-only
+ *
+ * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#ifndef MHICONTROLLERQAIC_H_
+#define MHICONTROLLERQAIC_H_
+
+struct mhi_controller *qaic_mhi_register_controller(struct pci_dev *pci_dev, void __iomem *mhi_bar,
+						    int mhi_irq);
+void qaic_mhi_free_controller(struct mhi_controller *mhi_cntrl, bool link_up);
+void qaic_mhi_start_reset(struct mhi_controller *mhi_cntrl);
+void qaic_mhi_reset_done(struct mhi_controller *mhi_cntrl);
+
+#endif /* MHICONTROLLERQAIC_H_ */
diff --git a/drivers/accel/qaic/qaic.h b/drivers/accel/qaic/qaic.h
new file mode 100644
index 000000000000..f2bd637a0d4e
--- /dev/null
+++ b/drivers/accel/qaic/qaic.h
@@ -0,0 +1,282 @@
+/* SPDX-License-Identifier: GPL-2.0-only
+ *
+ * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#ifndef _QAIC_H_
+#define _QAIC_H_
+
+#include <linux/interrupt.h>
+#include <linux/kref.h>
+#include <linux/mhi.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/srcu.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+#include <drm/drm_device.h>
+#include <drm/drm_gem.h>
+
+#define QAIC_DBC_BASE		SZ_128K
+#define QAIC_DBC_SIZE		SZ_4K
+
+#define QAIC_NO_PARTITION	-1
+
+#define QAIC_DBC_OFF(i)		((i) * QAIC_DBC_SIZE + QAIC_DBC_BASE)
+
+#define to_qaic_bo(obj) container_of(obj, struct qaic_bo, base)
+
+extern bool datapath_polling;
+
+struct qaic_user {
+	/* Uniquely identifies this user for the device */
+	int			handle;
+	struct kref		ref_count;
+	/* Char device opened by this user */
+	struct qaic_drm_device	*qddev;
+	/* Node in list of users that opened this drm device */
+	struct list_head	node;
+	/* SRCU used to synchronize this user during cleanup */
+	struct srcu_struct	qddev_lock;
+	atomic_t		chunk_id;
+};
+
+struct dma_bridge_chan {
+	/* Pointer to device strcut maintained by driver */
+	struct qaic_device	*qdev;
+	/* ID of this DMA bridge channel(DBC) */
+	unsigned int		id;
+	/* Synchronizes access to xfer_list */
+	spinlock_t		xfer_lock;
+	/* Base address of request queue */
+	void			*req_q_base;
+	/* Base address of response queue */
+	void			*rsp_q_base;
+	/*
+	 * Base bus address of request queue. Response queue bus address can be
+	 * calculated by adding request queue size to this variable
+	 */
+	dma_addr_t		dma_addr;
+	/* Total size of request and response queue in byte */
+	u32			total_size;
+	/* Capacity of request/response queue */
+	u32			nelem;
+	/* The user that opened this DBC */
+	struct qaic_user	*usr;
+	/*
+	 * Request ID of next memory handle that goes in request queue. One
+	 * memory handle can enqueue more than one request elements, all
+	 * this requests that belong to same memory handle have same request ID
+	 */
+	u16			next_req_id;
+	/* true: DBC is in use; false: DBC not in use */
+	bool			in_use;
+	/*
+	 * Base address of device registers. Used to read/write request and
+	 * response queue's head and tail pointer of this DBC.
+	 */
+	void __iomem		*dbc_base;
+	/* Head of list where each node is a memory handle queued in request queue */
+	struct list_head	xfer_list;
+	/* Synchronizes DBC readers during cleanup */
+	struct srcu_struct	ch_lock;
+	/*
+	 * When this DBC is released, any thread waiting on this wait queue is
+	 * woken up
+	 */
+	wait_queue_head_t	dbc_release;
+	/* Head of list where each node is a bo associated with this DBC */
+	struct list_head	bo_lists;
+	/* The irq line for this DBC. Used for polling */
+	unsigned int		irq;
+	/* Polling work item to simulate interrupts */
+	struct work_struct	poll_work;
+};
+
+struct qaic_device {
+	/* Pointer to base PCI device struct of our physical device */
+	struct pci_dev		*pdev;
+	/* Req. ID of request that will be queued next in MHI control device */
+	u32			next_seq_num;
+	/* Base address of bar 0 */
+	void __iomem		*bar_0;
+	/* Base address of bar 2 */
+	void __iomem		*bar_2;
+	/* Controller structure for MHI devices */
+	struct mhi_controller	*mhi_cntrl;
+	/* MHI control channel device */
+	struct mhi_device	*cntl_ch;
+	/* List of requests queued in MHI control device */
+	struct list_head	cntl_xfer_list;
+	/* Synchronizes MHI control device transactions and its xfer list */
+	struct mutex		cntl_mutex;
+	/* Array of DBC struct of this device */
+	struct dma_bridge_chan	*dbc;
+	/* Work queue for tasks related to MHI control device */
+	struct workqueue_struct	*cntl_wq;
+	/* Synchronizes all the users of device during cleanup */
+	struct srcu_struct	dev_lock;
+	/* true: Device under reset; false: Device not under reset */
+	bool			in_reset;
+	/*
+	 * true: A tx MHI transaction has failed and a rx buffer is still queued
+	 * in control device. Such a buffer is considered lost rx buffer
+	 * false: No rx buffer is lost in control device
+	 */
+	bool			cntl_lost_buf;
+	/* Maximum number of DBC supported by this device */
+	u32			num_dbc;
+	/* Reference to the drm_device for this device when it is created */
+	struct qaic_drm_device	*qddev;
+	/* Generate the CRC of a control message */
+	u32 (*gen_crc)(void *msg);
+	/* Validate the CRC of a control message */
+	bool (*valid_crc)(void *msg);
+};
+
+struct qaic_drm_device {
+	/* Pointer to the root device struct driven by this driver */
+	struct qaic_device	*qdev;
+	/*
+	 * The physical device can be partition in number of logical devices.
+	 * And each logical device is given a partition id. This member stores
+	 * that id. QAIC_NO_PARTITION is a sentinel used to mark that this drm
+	 * device is the actual physical device
+	 */
+	s32			partition_id;
+	/* Pointer to the drm device struct of this drm device */
+	struct drm_device	*ddev;
+	/* Head in list of users who have opened this drm device */
+	struct list_head	users;
+	/* Synchronizes access to users list */
+	struct mutex		users_mutex;
+};
+
+struct qaic_bo {
+	struct drm_gem_object	base;
+	/* Scatter/gather table for allocate/imported BO */
+	struct sg_table		*sgt;
+	/* BO size requested by user. GEM object might be bigger in size. */
+	u64			size;
+	/* Head in list of slices of this BO */
+	struct list_head	slices;
+	/* Total nents, for all slices of this BO */
+	int			total_slice_nents;
+	/*
+	 * Direction of transfer. It can assume only two value DMA_TO_DEVICE and
+	 * DMA_FROM_DEVICE.
+	 */
+	int			dir;
+	/* The pointer of the DBC which operates on this BO */
+	struct dma_bridge_chan	*dbc;
+	/* Number of slice that belongs to this buffer */
+	u32			nr_slice;
+	/* Number of slice that have been transferred by DMA engine */
+	u32			nr_slice_xfer_done;
+	/* true = BO is queued for execution, true = BO is not queued */
+	bool			queued;
+	/*
+	 * If true then user has attached slicing information to this BO by
+	 * calling DRM_IOCTL_QAIC_ATTACH_SLICE_BO ioctl.
+	 */
+	bool			sliced;
+	/* Request ID of this BO if it is queued for execution */
+	u16			req_id;
+	/* Handle assigned to this BO */
+	u32			handle;
+	/* Wait on this for completion of DMA transfer of this BO */
+	struct completion	xfer_done;
+	/*
+	 * Node in linked list where head is dbc->xfer_list.
+	 * This link list contain BO's that are queued for DMA transfer.
+	 */
+	struct list_head	xfer_list;
+	/*
+	 * Node in linked list where head is dbc->bo_lists.
+	 * This link list contain BO's that are associated with the DBC it is
+	 * linked to.
+	 */
+	struct list_head	bo_list;
+	struct {
+		/*
+		 * Latest timestamp(ns) at which kernel received a request to
+		 * execute this BO
+		 */
+		u64		req_received_ts;
+		/*
+		 * Latest timestamp(ns) at which kernel enqueued requests of
+		 * this BO for execution in DMA queue
+		 */
+		u64		req_submit_ts;
+		/*
+		 * Latest timestamp(ns) at which kernel received a completion
+		 * interrupt for requests of this BO
+		 */
+		u64		req_processed_ts;
+		/*
+		 * Number of elements already enqueued in DMA queue before
+		 * enqueuing requests of this BO
+		 */
+		u32		queue_level_before;
+	} perf_stats;
+
+};
+
+struct bo_slice {
+	/* Mapped pages */
+	struct sg_table		*sgt;
+	/* Number of requests required to queue in DMA queue */
+	int			nents;
+	/* See enum dma_data_direction */
+	int			dir;
+	/* Actual requests that will be copied in DMA queue */
+	struct dbc_req		*reqs;
+	struct kref		ref_count;
+	/* true: No DMA transfer required */
+	bool			no_xfer;
+	/* Pointer to the parent BO handle */
+	struct qaic_bo		*bo;
+	/* Node in list of slices maintained by parent BO */
+	struct list_head	slice;
+	/* Size of this slice in bytes */
+	u64			size;
+	/* Offset of this slice in buffer */
+	u64			offset;
+};
+
+int get_dbc_req_elem_size(void);
+int get_dbc_rsp_elem_size(void);
+int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr, u16 *major, u16 *minor);
+int qaic_manage_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv);
+void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result);
+
+void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result);
+
+int qaic_control_open(struct qaic_device *qdev);
+void qaic_control_close(struct qaic_device *qdev);
+void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr);
+
+irqreturn_t dbc_irq_threaded_fn(int irq, void *data);
+irqreturn_t dbc_irq_handler(int irq, void *data);
+int disable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr);
+void enable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr);
+void wakeup_dbc(struct qaic_device *qdev, u32 dbc_id);
+void release_dbc(struct qaic_device *qdev, u32 dbc_id);
+
+void wake_all_cntl(struct qaic_device *qdev);
+void qaic_dev_reset_clean_local_state(struct qaic_device *qdev, bool exit_reset);
+
+struct drm_gem_object *qaic_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf);
+
+int qaic_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv);
+int qaic_mmap_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv);
+int qaic_attach_slice_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv);
+int qaic_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv);
+int qaic_partial_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv);
+int qaic_wait_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv);
+int qaic_perf_stats_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv);
+void irq_polling_work(struct work_struct *work);
+
+#endif /* _QAIC_H_ */
diff --git a/drivers/accel/qaic/qaic_control.c b/drivers/accel/qaic/qaic_control.c
new file mode 100644
index 000000000000..9f216eb6f76e
--- /dev/null
+++ b/drivers/accel/qaic/qaic_control.c
@@ -0,0 +1,1526 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
+/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
+
+#include <asm/byteorder.h>
+#include <linux/completion.h>
+#include <linux/crc32.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/mhi.h>
+#include <linux/mm.h>
+#include <linux/moduleparam.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/scatterlist.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/workqueue.h>
+#include <linux/wait.h>
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+#include <uapi/drm/qaic_accel.h>
+
+#include "qaic.h"
+
+#define MANAGE_MAGIC_NUMBER		((__force __le32)0x43494151) /* "QAIC" in little endian */
+#define QAIC_DBC_Q_GAP			SZ_256
+#define QAIC_DBC_Q_BUF_ALIGN		SZ_4K
+#define QAIC_MANAGE_EXT_MSG_LENGTH	SZ_64K /* Max DMA message length */
+#define QAIC_WRAPPER_MAX_SIZE		SZ_4K
+#define QAIC_MHI_RETRY_WAIT_MS		100
+#define QAIC_MHI_RETRY_MAX		20
+
+static unsigned int control_resp_timeout_s = 60; /* 60 sec default */
+module_param(control_resp_timeout_s, uint, 0600);
+MODULE_PARM_DESC(control_resp_timeout_s, "Timeout for NNC responses from QSM");
+
+struct manage_msg {
+	u32 len;
+	u32 count;
+	u8 data[];
+};
+
+/*
+ * wire encoding structures for the manage protocol.
+ * All fields are little endian on the wire
+ */
+struct wire_msg_hdr {
+	__le32 crc32; /* crc of everything following this field in the message */
+	__le32 magic_number;
+	__le32 sequence_number;
+	__le32 len; /* length of this message */
+	__le32 count; /* number of transactions in this message */
+	__le32 handle; /* unique id to track the resources consumed */
+	__le32 partition_id; /* partition id for the request (signed) */
+	__le32 padding; /* must be 0 */
+} __packed;
+
+struct wire_msg {
+	struct wire_msg_hdr hdr;
+	u8 data[];
+} __packed;
+
+struct wire_trans_hdr {
+	__le32 type;
+	__le32 len;
+} __packed;
+
+/* Each message sent from driver to device are organized in a list of wrapper_msg */
+struct wrapper_msg {
+	struct list_head list;
+	struct kref ref_count;
+	u32 len; /* length of data to transfer */
+	struct wrapper_list *head;
+	union {
+		struct wire_msg msg;
+		struct wire_trans_hdr trans;
+	};
+};
+
+struct wrapper_list {
+	struct list_head list;
+	spinlock_t lock; /* Protects the list state during additions and removals */
+};
+
+struct wire_trans_passthrough {
+	struct wire_trans_hdr hdr;
+	u8 data[];
+} __packed;
+
+struct wire_addr_size_pair {
+	__le64 addr;
+	__le64 size;
+} __packed;
+
+struct wire_trans_dma_xfer {
+	struct wire_trans_hdr hdr;
+	__le32 tag;
+	__le32 count;
+	__le32 dma_chunk_id;
+	__le32 padding;
+	struct wire_addr_size_pair data[];
+} __packed;
+
+/* Initiated by device to continue the DMA xfer of a large piece of data */
+struct wire_trans_dma_xfer_cont {
+	struct wire_trans_hdr hdr;
+	__le32 dma_chunk_id;
+	__le32 padding;
+	__le64 xferred_size;
+} __packed;
+
+struct wire_trans_activate_to_dev {
+	struct wire_trans_hdr hdr;
+	__le64 req_q_addr;
+	__le64 rsp_q_addr;
+	__le32 req_q_size;
+	__le32 rsp_q_size;
+	__le32 buf_len;
+	__le32 options; /* unused, but BIT(16) has meaning to the device */
+} __packed;
+
+struct wire_trans_activate_from_dev {
+	struct wire_trans_hdr hdr;
+	__le32 status;
+	__le32 dbc_id;
+	__le64 options; /* unused */
+} __packed;
+
+struct wire_trans_deactivate_from_dev {
+	struct wire_trans_hdr hdr;
+	__le32 status;
+	__le32 dbc_id;
+} __packed;
+
+struct wire_trans_terminate_to_dev {
+	struct wire_trans_hdr hdr;
+	__le32 handle;
+	__le32 padding;
+} __packed;
+
+struct wire_trans_terminate_from_dev {
+	struct wire_trans_hdr hdr;
+	__le32 status;
+	__le32 padding;
+} __packed;
+
+struct wire_trans_status_to_dev {
+	struct wire_trans_hdr hdr;
+} __packed;
+
+struct wire_trans_status_from_dev {
+	struct wire_trans_hdr hdr;
+	__le16 major;
+	__le16 minor;
+	__le32 status;
+	__le64 status_flags;
+} __packed;
+
+struct wire_trans_validate_part_to_dev {
+	struct wire_trans_hdr hdr;
+	__le32 part_id;
+	__le32 padding;
+} __packed;
+
+struct wire_trans_validate_part_from_dev {
+	struct wire_trans_hdr hdr;
+	__le32 status;
+	__le32 padding;
+} __packed;
+
+struct xfer_queue_elem {
+	/*
+	 * Node in list of ongoing transfer request on control channel.
+	 * Maintained by root device struct.
+	 */
+	struct list_head list;
+	/* Sequence number of this transfer request */
+	u32 seq_num;
+	/* This is used to wait on until completion of transfer request */
+	struct completion xfer_done;
+	/* Received data from device */
+	void *buf;
+};
+
+struct dma_xfer {
+	/* Node in list of DMA transfers which is used for cleanup */
+	struct list_head list;
+	/* SG table of memory used for DMA */
+	struct sg_table *sgt;
+	/* Array pages used for DMA */
+	struct page **page_list;
+	/* Number of pages used for DMA */
+	unsigned long nr_pages;
+};
+
+struct ioctl_resources {
+	/* List of all DMA transfers which is used later for cleanup */
+	struct list_head dma_xfers;
+	/* Base address of request queue which belongs to a DBC */
+	void *buf;
+	/*
+	 * Base bus address of request queue which belongs to a DBC. Response
+	 * queue base bus address can be calculated by adding size of request
+	 * queue to base bus address of request queue.
+	 */
+	dma_addr_t dma_addr;
+	/* Total size of request queue and response queue in byte */
+	u32 total_size;
+	/* Total number of elements that can be queued in each of request and response queue */
+	u32 nelem;
+	/* Base address of response queue which belongs to a DBC */
+	void *rsp_q_base;
+	/* Status of the NNC message received */
+	u32 status;
+	/* DBC id of the DBC received from device */
+	u32 dbc_id;
+	/*
+	 * DMA transfer request messages can be big in size and it may not be
+	 * possible to send them in one shot. In such cases the messages are
+	 * broken into chunks, this field stores ID of such chunks.
+	 */
+	u32 dma_chunk_id;
+	/* Total number of bytes transferred for a DMA xfer request */
+	u64 xferred_dma_size;
+	/* Header of transaction message received from user. Used during DMA xfer request. */
+	void *trans_hdr;
+};
+
+struct resp_work {
+	struct work_struct work;
+	struct qaic_device *qdev;
+	void *buf;
+};
+
+/*
+ * Since we're working with little endian messages, its useful to be able to
+ * increment without filling a whole line with conversions back and forth just
+ * to add one(1) to a message count.
+ */
+static __le32 incr_le32(__le32 val)
+{
+	return cpu_to_le32(le32_to_cpu(val) + 1);
+}
+
+static u32 gen_crc(void *msg)
+{
+	struct wrapper_list *wrappers = msg;
+	struct wrapper_msg *w;
+	u32 crc = ~0;
+
+	list_for_each_entry(w, &wrappers->list, list)
+		crc = crc32(crc, &w->msg, w->len);
+
+	return crc ^ ~0;
+}
+
+static u32 gen_crc_stub(void *msg)
+{
+	return 0;
+}
+
+static bool valid_crc(void *msg)
+{
+	struct wire_msg_hdr *hdr = msg;
+	bool ret;
+	u32 crc;
+
+	/*
+	 * The output of this algorithm is always converted to the native
+	 * endianness.
+	 */
+	crc = le32_to_cpu(hdr->crc32);
+	hdr->crc32 = 0;
+	ret = (crc32(~0, msg, le32_to_cpu(hdr->len)) ^ ~0) == crc;
+	hdr->crc32 = cpu_to_le32(crc);
+	return ret;
+}
+
+static bool valid_crc_stub(void *msg)
+{
+	return true;
+}
+
+static void free_wrapper(struct kref *ref)
+{
+	struct wrapper_msg *wrapper = container_of(ref, struct wrapper_msg, ref_count);
+
+	list_del(&wrapper->list);
+	kfree(wrapper);
+}
+
+static void save_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources,
+			 struct qaic_user *usr)
+{
+	u32 dbc_id = resources->dbc_id;
+
+	if (resources->buf) {
+		wait_event_interruptible(qdev->dbc[dbc_id].dbc_release, !qdev->dbc[dbc_id].in_use);
+		qdev->dbc[dbc_id].req_q_base = resources->buf;
+		qdev->dbc[dbc_id].rsp_q_base = resources->rsp_q_base;
+		qdev->dbc[dbc_id].dma_addr = resources->dma_addr;
+		qdev->dbc[dbc_id].total_size = resources->total_size;
+		qdev->dbc[dbc_id].nelem = resources->nelem;
+		enable_dbc(qdev, dbc_id, usr);
+		qdev->dbc[dbc_id].in_use = true;
+		resources->buf = NULL;
+	}
+}
+
+static void free_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources)
+{
+	if (resources->buf)
+		dma_free_coherent(&qdev->pdev->dev, resources->total_size, resources->buf,
+				  resources->dma_addr);
+	resources->buf = NULL;
+}
+
+static void free_dma_xfers(struct qaic_device *qdev, struct ioctl_resources *resources)
+{
+	struct dma_xfer *xfer;
+	struct dma_xfer *x;
+	int i;
+
+	list_for_each_entry_safe(xfer, x, &resources->dma_xfers, list) {
+		dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);
+		sg_free_table(xfer->sgt);
+		kfree(xfer->sgt);
+		for (i = 0; i < xfer->nr_pages; ++i)
+			put_page(xfer->page_list[i]);
+		kfree(xfer->page_list);
+		list_del(&xfer->list);
+		kfree(xfer);
+	}
+}
+
+static struct wrapper_msg *add_wrapper(struct wrapper_list *wrappers, u32 size)
+{
+	struct wrapper_msg *w = kzalloc(size, GFP_KERNEL);
+
+	if (!w)
+		return NULL;
+	list_add_tail(&w->list, &wrappers->list);
+	kref_init(&w->ref_count);
+	w->head = wrappers;
+	return w;
+}
+
+static int encode_passthrough(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
+			      u32 *user_len)
+{
+	struct qaic_manage_trans_passthrough *in_trans = trans;
+	struct wire_trans_passthrough *out_trans;
+	struct wrapper_msg *trans_wrapper;
+	struct wrapper_msg *wrapper;
+	struct wire_msg *msg;
+	u32 msg_hdr_len;
+
+	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
+	msg = &wrapper->msg;
+	msg_hdr_len = le32_to_cpu(msg->hdr.len);
+
+	if (in_trans->hdr.len % 8 != 0)
+		return -EINVAL;
+
+	if (msg_hdr_len + in_trans->hdr.len > QAIC_MANAGE_EXT_MSG_LENGTH)
+		return -ENOSPC;
+
+	trans_wrapper = add_wrapper(wrappers,
+				    offsetof(struct wrapper_msg, trans) + in_trans->hdr.len);
+	if (!trans_wrapper)
+		return -ENOMEM;
+	trans_wrapper->len = in_trans->hdr.len;
+	out_trans = (struct wire_trans_passthrough *)&trans_wrapper->trans;
+
+	memcpy(out_trans->data, in_trans->data, in_trans->hdr.len - sizeof(in_trans->hdr));
+	msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
+	msg->hdr.count = incr_le32(msg->hdr.count);
+	*user_len += in_trans->hdr.len;
+	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_PASSTHROUGH_TO_DEV);
+	out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
+
+	return 0;
+}
+
+/* returns error code for failure, 0 if enough pages alloc'd, 1 if dma_cont is needed */
+static int find_and_map_user_pages(struct qaic_device *qdev,
+				   struct qaic_manage_trans_dma_xfer *in_trans,
+				   struct ioctl_resources *resources, struct dma_xfer *xfer)
+{
+	unsigned long need_pages;
+	struct page **page_list;
+	unsigned long nr_pages;
+	struct sg_table *sgt;
+	u64 xfer_start_addr;
+	int ret;
+	int i;
+
+	xfer_start_addr = in_trans->addr + resources->xferred_dma_size;
+
+	need_pages = DIV_ROUND_UP(in_trans->size + offset_in_page(xfer_start_addr) -
+				  resources->xferred_dma_size, PAGE_SIZE);
+
+	nr_pages = need_pages;
+
+	while (1) {
+		page_list = kmalloc_array(nr_pages, sizeof(*page_list), GFP_KERNEL | __GFP_NOWARN);
+		if (!page_list) {
+			nr_pages = nr_pages / 2;
+			if (!nr_pages)
+				return -ENOMEM;
+		} else {
+			break;
+		}
+	}
+
+	ret = get_user_pages_fast(xfer_start_addr, nr_pages, 0, page_list);
+	if (ret < 0 || ret != nr_pages) {
+		ret = -EFAULT;
+		goto free_page_list;
+	}
+
+	sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
+	if (!sgt) {
+		ret = -ENOMEM;
+		goto put_pages;
+	}
+
+	ret = sg_alloc_table_from_pages(sgt, page_list, nr_pages,
+					offset_in_page(xfer_start_addr),
+					in_trans->size - resources->xferred_dma_size, GFP_KERNEL);
+	if (ret) {
+		ret = -ENOMEM;
+		goto free_sgt;
+	}
+
+	ret = dma_map_sgtable(&qdev->pdev->dev, sgt, DMA_TO_DEVICE, 0);
+	if (ret)
+		goto free_table;
+
+	xfer->sgt = sgt;
+	xfer->page_list = page_list;
+	xfer->nr_pages = nr_pages;
+
+	return need_pages > nr_pages ? 1 : 0;
+
+free_table:
+	sg_free_table(sgt);
+free_sgt:
+	kfree(sgt);
+put_pages:
+	for (i = 0; i < nr_pages; ++i)
+		put_page(page_list[i]);
+free_page_list:
+	kfree(page_list);
+	return ret;
+}
+
+/* returns error code for failure, 0 if everything was encoded, 1 if dma_cont is needed */
+static int encode_addr_size_pairs(struct dma_xfer *xfer, struct wrapper_list *wrappers,
+				  struct ioctl_resources *resources, u32 msg_hdr_len, u32 *size,
+				  struct wire_trans_dma_xfer **out_trans)
+{
+	struct wrapper_msg *trans_wrapper;
+	struct sg_table *sgt = xfer->sgt;
+	struct wire_addr_size_pair *asp;
+	struct scatterlist *sg;
+	struct wrapper_msg *w;
+	unsigned int dma_len;
+	u64 dma_chunk_len;
+	void *boundary;
+	int nents_dma;
+	int nents;
+	int i;
+
+	nents = sgt->nents;
+	nents_dma = nents;
+	*size = QAIC_MANAGE_EXT_MSG_LENGTH - msg_hdr_len - sizeof(**out_trans);
+	for_each_sgtable_sg(sgt, sg, i) {
+		*size -= sizeof(*asp);
+		/* Save 1K for possible follow-up transactions. */
+		if (*size < SZ_1K) {
+			nents_dma = i;
+			break;
+		}
+	}
+
+	trans_wrapper = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
+	if (!trans_wrapper)
+		return -ENOMEM;
+	*out_trans = (struct wire_trans_dma_xfer *)&trans_wrapper->trans;
+
+	asp = (*out_trans)->data;
+	boundary = (void *)trans_wrapper + QAIC_WRAPPER_MAX_SIZE;
+	*size = 0;
+
+	dma_len = 0;
+	w = trans_wrapper;
+	dma_chunk_len = 0;
+	for_each_sg(sgt->sgl, sg, nents_dma, i) {
+		asp->size = cpu_to_le64(dma_len);
+		dma_chunk_len += dma_len;
+		if (dma_len) {
+			asp++;
+			if ((void *)asp + sizeof(*asp) > boundary) {
+				w->len = (void *)asp - (void *)&w->msg;
+				*size += w->len;
+				w = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
+				if (!w)
+					return -ENOMEM;
+				boundary = (void *)w + QAIC_WRAPPER_MAX_SIZE;
+				asp = (struct wire_addr_size_pair *)&w->msg;
+			}
+		}
+		asp->addr = cpu_to_le64(sg_dma_address(sg));
+		dma_len = sg_dma_len(sg);
+	}
+	/* finalize the last segment */
+	asp->size = cpu_to_le64(dma_len);
+	w->len = (void *)asp + sizeof(*asp) - (void *)&w->msg;
+	*size += w->len;
+	dma_chunk_len += dma_len;
+	resources->xferred_dma_size += dma_chunk_len;
+
+	return nents_dma < nents ? 1 : 0;
+}
+
+static void cleanup_xfer(struct qaic_device *qdev, struct dma_xfer *xfer)
+{
+	int i;
+
+	dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);
+	sg_free_table(xfer->sgt);
+	kfree(xfer->sgt);
+	for (i = 0; i < xfer->nr_pages; ++i)
+		put_page(xfer->page_list[i]);
+	kfree(xfer->page_list);
+}
+
+static int encode_dma(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
+		      u32 *user_len, struct ioctl_resources *resources, struct qaic_user *usr)
+{
+	struct qaic_manage_trans_dma_xfer *in_trans = trans;
+	struct wire_trans_dma_xfer *out_trans;
+	struct wrapper_msg *wrapper;
+	struct dma_xfer *xfer;
+	struct wire_msg *msg;
+	bool need_cont_dma;
+	u32 msg_hdr_len;
+	u32 size;
+	int ret;
+
+	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
+	msg = &wrapper->msg;
+	msg_hdr_len = le32_to_cpu(msg->hdr.len);
+
+	if (msg_hdr_len > (UINT_MAX - QAIC_MANAGE_EXT_MSG_LENGTH))
+		return -EINVAL;
+
+	/* There should be enough space to hold at least one ASP entry. */
+	if (msg_hdr_len + sizeof(*out_trans) + sizeof(struct wire_addr_size_pair) >
+	    QAIC_MANAGE_EXT_MSG_LENGTH)
+		return -ENOMEM;
+
+	if (in_trans->addr + in_trans->size < in_trans->addr || !in_trans->size)
+		return -EINVAL;
+
+	xfer = kmalloc(sizeof(*xfer), GFP_KERNEL);
+	if (!xfer)
+		return -ENOMEM;
+
+	ret = find_and_map_user_pages(qdev, in_trans, resources, xfer);
+	if (ret < 0)
+		goto free_xfer;
+
+	need_cont_dma = (bool)ret;
+
+	ret = encode_addr_size_pairs(xfer, wrappers, resources, msg_hdr_len, &size, &out_trans);
+	if (ret < 0)
+		goto cleanup_xfer;
+
+	need_cont_dma = need_cont_dma || (bool)ret;
+
+	msg->hdr.len = cpu_to_le32(msg_hdr_len + size);
+	msg->hdr.count = incr_le32(msg->hdr.count);
+
+	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
+	out_trans->hdr.len = cpu_to_le32(size);
+	out_trans->tag = cpu_to_le32(in_trans->tag);
+	out_trans->count = cpu_to_le32((size - sizeof(*out_trans)) /
+								sizeof(struct wire_addr_size_pair));
+
+	*user_len += in_trans->hdr.len;
+
+	if (resources->dma_chunk_id) {
+		out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
+	} else if (need_cont_dma) {
+		while (resources->dma_chunk_id == 0)
+			resources->dma_chunk_id = atomic_inc_return(&usr->chunk_id);
+
+		out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
+	}
+	resources->trans_hdr = trans;
+
+	list_add(&xfer->list, &resources->dma_xfers);
+	return 0;
+
+cleanup_xfer:
+	cleanup_xfer(qdev, xfer);
+free_xfer:
+	kfree(xfer);
+	return ret;
+}
+
+static int encode_activate(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
+			   u32 *user_len, struct ioctl_resources *resources)
+{
+	struct qaic_manage_trans_activate_to_dev *in_trans = trans;
+	struct wire_trans_activate_to_dev *out_trans;
+	struct wrapper_msg *trans_wrapper;
+	struct wrapper_msg *wrapper;
+	struct wire_msg *msg;
+	dma_addr_t dma_addr;
+	u32 msg_hdr_len;
+	void *buf;
+	u32 nelem;
+	u32 size;
+	int ret;
+
+	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
+	msg = &wrapper->msg;
+	msg_hdr_len = le32_to_cpu(msg->hdr.len);
+
+	if (msg_hdr_len + sizeof(*out_trans) > QAIC_MANAGE_MAX_MSG_LENGTH)
+		return -ENOSPC;
+
+	if (!in_trans->queue_size)
+		return -EINVAL;
+
+	if (in_trans->pad)
+		return -EINVAL;
+
+	nelem = in_trans->queue_size;
+	size = (get_dbc_req_elem_size() + get_dbc_rsp_elem_size()) * nelem;
+	if (size / nelem != get_dbc_req_elem_size() + get_dbc_rsp_elem_size())
+		return -EINVAL;
+
+	if (size + QAIC_DBC_Q_GAP + QAIC_DBC_Q_BUF_ALIGN < size)
+		return -EINVAL;
+
+	size = ALIGN((size + QAIC_DBC_Q_GAP), QAIC_DBC_Q_BUF_ALIGN);
+
+	buf = dma_alloc_coherent(&qdev->pdev->dev, size, &dma_addr, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	trans_wrapper = add_wrapper(wrappers,
+				    offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
+	if (!trans_wrapper) {
+		ret = -ENOMEM;
+		goto free_dma;
+	}
+	trans_wrapper->len = sizeof(*out_trans);
+	out_trans = (struct wire_trans_activate_to_dev *)&trans_wrapper->trans;
+
+	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_ACTIVATE_TO_DEV);
+	out_trans->hdr.len = cpu_to_le32(sizeof(*out_trans));
+	out_trans->buf_len = cpu_to_le32(size);
+	out_trans->req_q_addr = cpu_to_le64(dma_addr);
+	out_trans->req_q_size = cpu_to_le32(nelem);
+	out_trans->rsp_q_addr = cpu_to_le64(dma_addr + size - nelem * get_dbc_rsp_elem_size());
+	out_trans->rsp_q_size = cpu_to_le32(nelem);
+	out_trans->options = cpu_to_le32(in_trans->options);
+
+	*user_len += in_trans->hdr.len;
+	msg->hdr.len = cpu_to_le32(msg_hdr_len + sizeof(*out_trans));
+	msg->hdr.count = incr_le32(msg->hdr.count);
+
+	resources->buf = buf;
+	resources->dma_addr = dma_addr;
+	resources->total_size = size;
+	resources->nelem = nelem;
+	resources->rsp_q_base = buf + size - nelem * get_dbc_rsp_elem_size();
+	return 0;
+
+free_dma:
+	dma_free_coherent(&qdev->pdev->dev, size, buf, dma_addr);
+	return ret;
+}
+
+static int encode_deactivate(struct qaic_device *qdev, void *trans,
+			     u32 *user_len, struct qaic_user *usr)
+{
+	struct qaic_manage_trans_deactivate *in_trans = trans;
+
+	if (in_trans->dbc_id >= qdev->num_dbc || in_trans->pad)
+		return -EINVAL;
+
+	*user_len += in_trans->hdr.len;
+
+	return disable_dbc(qdev, in_trans->dbc_id, usr);
+}
+
+static int encode_status(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
+			 u32 *user_len)
+{
+	struct qaic_manage_trans_status_to_dev *in_trans = trans;
+	struct wire_trans_status_to_dev *out_trans;
+	struct wrapper_msg *trans_wrapper;
+	struct wrapper_msg *wrapper;
+	struct wire_msg *msg;
+	u32 msg_hdr_len;
+
+	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
+	msg = &wrapper->msg;
+	msg_hdr_len = le32_to_cpu(msg->hdr.len);
+
+	if (msg_hdr_len + in_trans->hdr.len > QAIC_MANAGE_MAX_MSG_LENGTH)
+		return -ENOSPC;
+
+	trans_wrapper = add_wrapper(wrappers, sizeof(*trans_wrapper));
+	if (!trans_wrapper)
+		return -ENOMEM;
+
+	trans_wrapper->len = sizeof(*out_trans);
+	out_trans = (struct wire_trans_status_to_dev *)&trans_wrapper->trans;
+
+	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_STATUS_TO_DEV);
+	out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
+	msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
+	msg->hdr.count = incr_le32(msg->hdr.count);
+	*user_len += in_trans->hdr.len;
+
+	return 0;
+}
+
+static int encode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
+			  struct wrapper_list *wrappers, struct ioctl_resources *resources,
+			  struct qaic_user *usr)
+{
+	struct qaic_manage_trans_hdr *trans_hdr;
+	struct wrapper_msg *wrapper;
+	struct wire_msg *msg;
+	u32 user_len = 0;
+	int ret;
+	int i;
+
+	if (!user_msg->count) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
+	msg = &wrapper->msg;
+
+	msg->hdr.len = cpu_to_le32(sizeof(msg->hdr));
+
+	if (resources->dma_chunk_id) {
+		ret = encode_dma(qdev, resources->trans_hdr, wrappers, &user_len, resources, usr);
+		msg->hdr.count = cpu_to_le32(1);
+		goto out;
+	}
+
+	for (i = 0; i < user_msg->count; ++i) {
+		if (user_len >= user_msg->len) {
+			ret = -EINVAL;
+			break;
+		}
+		trans_hdr = (struct qaic_manage_trans_hdr *)(user_msg->data + user_len);
+		if (user_len + trans_hdr->len > user_msg->len) {
+			ret = -EINVAL;
+			break;
+		}
+
+		switch (trans_hdr->type) {
+		case QAIC_TRANS_PASSTHROUGH_FROM_USR:
+			ret = encode_passthrough(qdev, trans_hdr, wrappers, &user_len);
+			break;
+		case QAIC_TRANS_DMA_XFER_FROM_USR:
+			ret = encode_dma(qdev, trans_hdr, wrappers, &user_len, resources, usr);
+			break;
+		case QAIC_TRANS_ACTIVATE_FROM_USR:
+			ret = encode_activate(qdev, trans_hdr, wrappers, &user_len, resources);
+			break;
+		case QAIC_TRANS_DEACTIVATE_FROM_USR:
+			ret = encode_deactivate(qdev, trans_hdr, &user_len, usr);
+			break;
+		case QAIC_TRANS_STATUS_FROM_USR:
+			ret = encode_status(qdev, trans_hdr, wrappers, &user_len);
+			break;
+		default:
+			ret = -EINVAL;
+			break;
+		}
+
+		if (ret)
+			break;
+	}
+
+	if (user_len != user_msg->len)
+		ret = -EINVAL;
+out:
+	if (ret) {
+		free_dma_xfers(qdev, resources);
+		free_dbc_buf(qdev, resources);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int decode_passthrough(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
+			      u32 *msg_len)
+{
+	struct qaic_manage_trans_passthrough *out_trans;
+	struct wire_trans_passthrough *in_trans = trans;
+	u32 len;
+
+	out_trans = (void *)user_msg->data + user_msg->len;
+
+	len = le32_to_cpu(in_trans->hdr.len);
+	if (len % 8 != 0)
+		return -EINVAL;
+
+	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
+		return -ENOSPC;
+
+	memcpy(out_trans->data, in_trans->data, len - sizeof(in_trans->hdr));
+	user_msg->len += len;
+	*msg_len += len;
+	out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
+	out_trans->hdr.len = len;
+
+	return 0;
+}
+
+static int decode_activate(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
+			   u32 *msg_len, struct ioctl_resources *resources, struct qaic_user *usr)
+{
+	struct qaic_manage_trans_activate_from_dev *out_trans;
+	struct wire_trans_activate_from_dev *in_trans = trans;
+	u32 len;
+
+	out_trans = (void *)user_msg->data + user_msg->len;
+
+	len = le32_to_cpu(in_trans->hdr.len);
+	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
+		return -ENOSPC;
+
+	user_msg->len += len;
+	*msg_len += len;
+	out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
+	out_trans->hdr.len = len;
+	out_trans->status = le32_to_cpu(in_trans->status);
+	out_trans->dbc_id = le32_to_cpu(in_trans->dbc_id);
+	out_trans->options = le64_to_cpu(in_trans->options);
+
+	if (!resources->buf)
+		/* how did we get an activate response without a request? */
+		return -EINVAL;
+
+	if (out_trans->dbc_id >= qdev->num_dbc)
+		/*
+		 * The device assigned an invalid resource, which should never
+		 * happen. Return an error so the user can try to recover.
+		 */
+		return -ENODEV;
+
+	if (out_trans->status)
+		/*
+		 * Allocating resources failed on device side. This is not an
+		 * expected behaviour, user is expected to handle this situation.
+		 */
+		return -ECANCELED;
+
+	resources->status = out_trans->status;
+	resources->dbc_id = out_trans->dbc_id;
+	save_dbc_buf(qdev, resources, usr);
+
+	return 0;
+}
+
+static int decode_deactivate(struct qaic_device *qdev, void *trans, u32 *msg_len,
+			     struct qaic_user *usr)
+{
+	struct wire_trans_deactivate_from_dev *in_trans = trans;
+	u32 dbc_id = le32_to_cpu(in_trans->dbc_id);
+	u32 status = le32_to_cpu(in_trans->status);
+
+	if (dbc_id >= qdev->num_dbc)
+		/*
+		 * The device assigned an invalid resource, which should never
+		 * happen. Inject an error so the user can try to recover.
+		 */
+		return -ENODEV;
+
+	if (status) {
+		/*
+		 * Releasing resources failed on the device side, which puts
+		 * us in a bind since they may still be in use, so enable the
+		 * dbc. User is expected to retry deactivation.
+		 */
+		enable_dbc(qdev, dbc_id, usr);
+		return -ECANCELED;
+	}
+
+	release_dbc(qdev, dbc_id);
+	*msg_len += sizeof(*in_trans);
+
+	return 0;
+}
+
+static int decode_status(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
+			 u32 *user_len, struct wire_msg *msg)
+{
+	struct qaic_manage_trans_status_from_dev *out_trans;
+	struct wire_trans_status_from_dev *in_trans = trans;
+	u32 len;
+
+	out_trans = (void *)user_msg->data + user_msg->len;
+
+	len = le32_to_cpu(in_trans->hdr.len);
+	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
+		return -ENOSPC;
+
+	out_trans->hdr.type = QAIC_TRANS_STATUS_FROM_DEV;
+	out_trans->hdr.len = len;
+	out_trans->major = le16_to_cpu(in_trans->major);
+	out_trans->minor = le16_to_cpu(in_trans->minor);
+	out_trans->status_flags = le64_to_cpu(in_trans->status_flags);
+	out_trans->status = le32_to_cpu(in_trans->status);
+	*user_len += le32_to_cpu(in_trans->hdr.len);
+	user_msg->len += len;
+
+	if (out_trans->status)
+		return -ECANCELED;
+	if (out_trans->status_flags & BIT(0) && !valid_crc(msg))
+		return -EPIPE;
+
+	return 0;
+}
+
+static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
+			  struct wire_msg *msg, struct ioctl_resources *resources,
+			  struct qaic_user *usr)
+{
+	u32 msg_hdr_len = le32_to_cpu(msg->hdr.len);
+	struct wire_trans_hdr *trans_hdr;
+	u32 msg_len = 0;
+	int ret;
+	int i;
+
+	if (msg_hdr_len > QAIC_MANAGE_MAX_MSG_LENGTH)
+		return -EINVAL;
+
+	user_msg->len = 0;
+	user_msg->count = le32_to_cpu(msg->hdr.count);
+
+	for (i = 0; i < user_msg->count; ++i) {
+		trans_hdr = (struct wire_trans_hdr *)(msg->data + msg_len);
+		if (msg_len + le32_to_cpu(trans_hdr->len) > msg_hdr_len)
+			return -EINVAL;
+
+		switch (le32_to_cpu(trans_hdr->type)) {
+		case QAIC_TRANS_PASSTHROUGH_FROM_DEV:
+			ret = decode_passthrough(qdev, trans_hdr, user_msg, &msg_len);
+			break;
+		case QAIC_TRANS_ACTIVATE_FROM_DEV:
+			ret = decode_activate(qdev, trans_hdr, user_msg, &msg_len, resources, usr);
+			break;
+		case QAIC_TRANS_DEACTIVATE_FROM_DEV:
+			ret = decode_deactivate(qdev, trans_hdr, &msg_len, usr);
+			break;
+		case QAIC_TRANS_STATUS_FROM_DEV:
+			ret = decode_status(qdev, trans_hdr, user_msg, &msg_len, msg);
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		if (ret)
+			return ret;
+	}
+
+	if (msg_len != (msg_hdr_len - sizeof(msg->hdr)))
+		return -EINVAL;
+
+	return 0;
+}
+
+static void *msg_xfer(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 seq_num,
+		      bool ignore_signal)
+{
+	struct xfer_queue_elem elem;
+	struct wire_msg *out_buf;
+	struct wrapper_msg *w;
+	int retry_count;
+	long ret;
+
+	if (qdev->in_reset) {
+		mutex_unlock(&qdev->cntl_mutex);
+		return ERR_PTR(-ENODEV);
+	}
+
+	elem.seq_num = seq_num;
+	elem.buf = NULL;
+	init_completion(&elem.xfer_done);
+	if (likely(!qdev->cntl_lost_buf)) {
+		/*
+		 * The max size of request to device is QAIC_MANAGE_EXT_MSG_LENGTH.
+		 * The max size of response from device is QAIC_MANAGE_MAX_MSG_LENGTH.
+		 */
+		out_buf = kmalloc(QAIC_MANAGE_MAX_MSG_LENGTH, GFP_KERNEL);
+		if (!out_buf) {
+			mutex_unlock(&qdev->cntl_mutex);
+			return ERR_PTR(-ENOMEM);
+		}
+
+		ret = mhi_queue_buf(qdev->cntl_ch, DMA_FROM_DEVICE, out_buf,
+				    QAIC_MANAGE_MAX_MSG_LENGTH, MHI_EOT);
+		if (ret) {
+			mutex_unlock(&qdev->cntl_mutex);
+			return ERR_PTR(ret);
+		}
+	} else {
+		/*
+		 * we lost a buffer because we queued a recv buf, but then
+		 * queuing the corresponding tx buf failed. To try to avoid
+		 * a memory leak, lets reclaim it and use it for this
+		 * transaction.
+		 */
+		qdev->cntl_lost_buf = false;
+	}
+
+	list_for_each_entry(w, &wrappers->list, list) {
+		kref_get(&w->ref_count);
+		retry_count = 0;
+retry:
+		ret = mhi_queue_buf(qdev->cntl_ch, DMA_TO_DEVICE, &w->msg, w->len,
+				    list_is_last(&w->list, &wrappers->list) ? MHI_EOT : MHI_CHAIN);
+		if (ret) {
+			if (ret == -EAGAIN && retry_count++ < QAIC_MHI_RETRY_MAX) {
+				msleep_interruptible(QAIC_MHI_RETRY_WAIT_MS);
+				if (!signal_pending(current))
+					goto retry;
+			}
+
+			qdev->cntl_lost_buf = true;
+			kref_put(&w->ref_count, free_wrapper);
+			mutex_unlock(&qdev->cntl_mutex);
+			return ERR_PTR(ret);
+		}
+	}
+
+	list_add_tail(&elem.list, &qdev->cntl_xfer_list);
+	mutex_unlock(&qdev->cntl_mutex);
+
+	if (ignore_signal)
+		ret = wait_for_completion_timeout(&elem.xfer_done, control_resp_timeout_s * HZ);
+	else
+		ret = wait_for_completion_interruptible_timeout(&elem.xfer_done,
+								control_resp_timeout_s * HZ);
+	/*
+	 * not using _interruptable because we have to cleanup or we'll
+	 * likely cause memory corruption
+	 */
+	mutex_lock(&qdev->cntl_mutex);
+	if (!list_empty(&elem.list))
+		list_del(&elem.list);
+	if (!ret && !elem.buf)
+		ret = -ETIMEDOUT;
+	else if (ret > 0 && !elem.buf)
+		ret = -EIO;
+	mutex_unlock(&qdev->cntl_mutex);
+
+	if (ret < 0) {
+		kfree(elem.buf);
+		return ERR_PTR(ret);
+	} else if (!qdev->valid_crc(elem.buf)) {
+		kfree(elem.buf);
+		return ERR_PTR(-EPIPE);
+	}
+
+	return elem.buf;
+}
+
+/* Add a transaction to abort the outstanding DMA continuation */
+static int abort_dma_cont(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 dma_chunk_id)
+{
+	struct wire_trans_dma_xfer *out_trans;
+	u32 size = sizeof(*out_trans);
+	struct wrapper_msg *wrapper;
+	struct wrapper_msg *w;
+	struct wire_msg *msg;
+
+	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
+	msg = &wrapper->msg;
+
+	/* Remove all but the first wrapper which has the msg header */
+	list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
+		if (!list_is_first(&wrapper->list, &wrappers->list))
+			kref_put(&wrapper->ref_count, free_wrapper);
+
+	wrapper = add_wrapper(wrappers, offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
+
+	if (!wrapper)
+		return -ENOMEM;
+
+	out_trans = (struct wire_trans_dma_xfer *)&wrapper->trans;
+	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
+	out_trans->hdr.len = cpu_to_le32(size);
+	out_trans->tag = cpu_to_le32(0);
+	out_trans->count = cpu_to_le32(0);
+	out_trans->dma_chunk_id = cpu_to_le32(dma_chunk_id);
+
+	msg->hdr.len = cpu_to_le32(size + sizeof(*msg));
+	msg->hdr.count = cpu_to_le32(1);
+	wrapper->len = size;
+
+	return 0;
+}
+
+static struct wrapper_list *alloc_wrapper_list(void)
+{
+	struct wrapper_list *wrappers;
+
+	wrappers = kmalloc(sizeof(*wrappers), GFP_KERNEL);
+	if (!wrappers)
+		return NULL;
+	INIT_LIST_HEAD(&wrappers->list);
+	spin_lock_init(&wrappers->lock);
+
+	return wrappers;
+}
+
+static int qaic_manage_msg_xfer(struct qaic_device *qdev, struct qaic_user *usr,
+				struct manage_msg *user_msg, struct ioctl_resources *resources,
+				struct wire_msg **rsp)
+{
+	struct wrapper_list *wrappers;
+	struct wrapper_msg *wrapper;
+	struct wrapper_msg *w;
+	bool all_done = false;
+	struct wire_msg *msg;
+	int ret;
+
+	wrappers = alloc_wrapper_list();
+	if (!wrappers)
+		return -ENOMEM;
+
+	wrapper = add_wrapper(wrappers, sizeof(*wrapper));
+	if (!wrapper) {
+		kfree(wrappers);
+		return -ENOMEM;
+	}
+
+	msg = &wrapper->msg;
+	wrapper->len = sizeof(*msg);
+
+	ret = encode_message(qdev, user_msg, wrappers, resources, usr);
+	if (ret && resources->dma_chunk_id)
+		ret = abort_dma_cont(qdev, wrappers, resources->dma_chunk_id);
+	if (ret)
+		goto encode_failed;
+
+	ret = mutex_lock_interruptible(&qdev->cntl_mutex);
+	if (ret)
+		goto lock_failed;
+
+	msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
+	msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
+
+	if (usr) {
+		msg->hdr.handle = cpu_to_le32(usr->handle);
+		msg->hdr.partition_id = cpu_to_le32(usr->qddev->partition_id);
+	} else {
+		msg->hdr.handle = 0;
+		msg->hdr.partition_id = cpu_to_le32(QAIC_NO_PARTITION);
+	}
+
+	msg->hdr.padding = cpu_to_le32(0);
+	msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
+
+	/* msg_xfer releases the mutex */
+	*rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, false);
+	if (IS_ERR(*rsp))
+		ret = PTR_ERR(*rsp);
+
+lock_failed:
+	free_dma_xfers(qdev, resources);
+encode_failed:
+	spin_lock(&wrappers->lock);
+	list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
+		kref_put(&wrapper->ref_count, free_wrapper);
+	all_done = list_empty(&wrappers->list);
+	spin_unlock(&wrappers->lock);
+	if (all_done)
+		kfree(wrappers);
+
+	return ret;
+}
+
+static int qaic_manage(struct qaic_device *qdev, struct qaic_user *usr, struct manage_msg *user_msg)
+{
+	struct wire_trans_dma_xfer_cont *dma_cont = NULL;
+	struct ioctl_resources resources;
+	struct wire_msg *rsp = NULL;
+	int ret;
+
+	memset(&resources, 0, sizeof(struct ioctl_resources));
+
+	INIT_LIST_HEAD(&resources.dma_xfers);
+
+	if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH ||
+	    user_msg->count > QAIC_MANAGE_MAX_MSG_LENGTH / sizeof(struct qaic_manage_trans_hdr))
+		return -EINVAL;
+
+dma_xfer_continue:
+	ret = qaic_manage_msg_xfer(qdev, usr, user_msg, &resources, &rsp);
+	if (ret)
+		return ret;
+	/* dma_cont should be the only transaction if present */
+	if (le32_to_cpu(rsp->hdr.count) == 1) {
+		dma_cont = (struct wire_trans_dma_xfer_cont *)rsp->data;
+		if (le32_to_cpu(dma_cont->hdr.type) != QAIC_TRANS_DMA_XFER_CONT)
+			dma_cont = NULL;
+	}
+	if (dma_cont) {
+		if (le32_to_cpu(dma_cont->dma_chunk_id) == resources.dma_chunk_id &&
+		    le64_to_cpu(dma_cont->xferred_size) == resources.xferred_dma_size) {
+			kfree(rsp);
+			goto dma_xfer_continue;
+		}
+
+		ret = -EINVAL;
+		goto dma_cont_failed;
+	}
+
+	ret = decode_message(qdev, user_msg, rsp, &resources, usr);
+
+dma_cont_failed:
+	free_dbc_buf(qdev, &resources);
+	kfree(rsp);
+	return ret;
+}
+
+int qaic_manage_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	struct qaic_manage_msg *user_msg;
+	struct qaic_device *qdev;
+	struct manage_msg *msg;
+	struct qaic_user *usr;
+	u8 __user *user_data;
+	int qdev_rcu_id;
+	int usr_rcu_id;
+	int ret;
+
+	usr = file_priv->driver_priv;
+
+	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
+	if (!usr->qddev) {
+		srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+		return -ENODEV;
+	}
+
+	qdev = usr->qddev->qdev;
+
+	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
+	if (qdev->in_reset) {
+		srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+		srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+		return -ENODEV;
+	}
+
+	user_msg = data;
+
+	if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	msg = kzalloc(QAIC_MANAGE_MAX_MSG_LENGTH + sizeof(*msg), GFP_KERNEL);
+	if (!msg) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	msg->len = user_msg->len;
+	msg->count = user_msg->count;
+
+	user_data = u64_to_user_ptr(user_msg->data);
+
+	if (copy_from_user(msg->data, user_data, user_msg->len)) {
+		ret = -EFAULT;
+		goto free_msg;
+	}
+
+	ret = qaic_manage(qdev, usr, msg);
+
+	/*
+	 * If the qaic_manage() is successful then we copy the message onto
+	 * userspace memory but we have an exception for -ECANCELED.
+	 * For -ECANCELED, it means that device has NACKed the message with a
+	 * status error code which userspace would like to know.
+	 */
+	if (ret == -ECANCELED || !ret) {
+		if (copy_to_user(user_data, msg->data, msg->len)) {
+			ret = -EFAULT;
+		} else {
+			user_msg->len = msg->len;
+			user_msg->count = msg->count;
+		}
+	}
+
+free_msg:
+	kfree(msg);
+out:
+	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+	return ret;
+}
+
+int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr, u16 *major, u16 *minor)
+{
+	struct qaic_manage_trans_status_from_dev *status_result;
+	struct qaic_manage_trans_status_to_dev *status_query;
+	struct manage_msg *user_msg;
+	int ret;
+
+	user_msg = kmalloc(sizeof(*user_msg) + sizeof(*status_result), GFP_KERNEL);
+	if (!user_msg) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	user_msg->len = sizeof(*status_query);
+	user_msg->count = 1;
+
+	status_query = (struct qaic_manage_trans_status_to_dev *)user_msg->data;
+	status_query->hdr.type = QAIC_TRANS_STATUS_FROM_USR;
+	status_query->hdr.len = sizeof(status_query->hdr);
+
+	ret = qaic_manage(qdev, usr, user_msg);
+	if (ret)
+		goto kfree_user_msg;
+	status_result = (struct qaic_manage_trans_status_from_dev *)user_msg->data;
+	*major = status_result->major;
+	*minor = status_result->minor;
+
+	if (status_result->status_flags & BIT(0)) { /* device is using CRC */
+		/* By default qdev->gen_crc is programmed to generate CRC */
+		qdev->valid_crc = valid_crc;
+	} else {
+		/* By default qdev->valid_crc is programmed to bypass CRC */
+		qdev->gen_crc = gen_crc_stub;
+	}
+
+kfree_user_msg:
+	kfree(user_msg);
+out:
+	return ret;
+}
+
+static void resp_worker(struct work_struct *work)
+{
+	struct resp_work *resp = container_of(work, struct resp_work, work);
+	struct qaic_device *qdev = resp->qdev;
+	struct wire_msg *msg = resp->buf;
+	struct xfer_queue_elem *elem;
+	struct xfer_queue_elem *i;
+	bool found = false;
+
+	mutex_lock(&qdev->cntl_mutex);
+	list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
+		if (elem->seq_num == le32_to_cpu(msg->hdr.sequence_number)) {
+			found = true;
+			list_del_init(&elem->list);
+			elem->buf = msg;
+			complete_all(&elem->xfer_done);
+			break;
+		}
+	}
+	mutex_unlock(&qdev->cntl_mutex);
+
+	if (!found)
+		/* request must have timed out, drop packet */
+		kfree(msg);
+
+	kfree(resp);
+}
+
+static void free_wrapper_from_list(struct wrapper_list *wrappers, struct wrapper_msg *wrapper)
+{
+	bool all_done = false;
+
+	spin_lock(&wrappers->lock);
+	kref_put(&wrapper->ref_count, free_wrapper);
+	all_done = list_empty(&wrappers->list);
+	spin_unlock(&wrappers->lock);
+
+	if (all_done)
+		kfree(wrappers);
+}
+
+void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
+{
+	struct wire_msg *msg = mhi_result->buf_addr;
+	struct wrapper_msg *wrapper = container_of(msg, struct wrapper_msg, msg);
+
+	free_wrapper_from_list(wrapper->head, wrapper);
+}
+
+void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
+{
+	struct qaic_device *qdev = dev_get_drvdata(&mhi_dev->dev);
+	struct wire_msg *msg = mhi_result->buf_addr;
+	struct resp_work *resp;
+
+	if (mhi_result->transaction_status || msg->hdr.magic_number != MANAGE_MAGIC_NUMBER) {
+		kfree(msg);
+		return;
+	}
+
+	resp = kmalloc(sizeof(*resp), GFP_ATOMIC);
+	if (!resp) {
+		kfree(msg);
+		return;
+	}
+
+	INIT_WORK(&resp->work, resp_worker);
+	resp->qdev = qdev;
+	resp->buf = msg;
+	queue_work(qdev->cntl_wq, &resp->work);
+}
+
+int qaic_control_open(struct qaic_device *qdev)
+{
+	if (!qdev->cntl_ch)
+		return -ENODEV;
+
+	qdev->cntl_lost_buf = false;
+	/*
+	 * By default qaic should assume that device has CRC enabled.
+	 * Qaic comes to know if device has CRC enabled or disabled during the
+	 * device status transaction, which is the first transaction performed
+	 * on control channel.
+	 *
+	 * So CRC validation of first device status transaction response is
+	 * ignored (by calling valid_crc_stub) and is done later during decoding
+	 * if device has CRC enabled.
+	 * Now that qaic knows whether device has CRC enabled or not it acts
+	 * accordingly.
+	 */
+	qdev->gen_crc = gen_crc;
+	qdev->valid_crc = valid_crc_stub;
+
+	return mhi_prepare_for_transfer(qdev->cntl_ch);
+}
+
+void qaic_control_close(struct qaic_device *qdev)
+{
+	mhi_unprepare_from_transfer(qdev->cntl_ch);
+}
+
+void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr)
+{
+	struct wire_trans_terminate_to_dev *trans;
+	struct wrapper_list *wrappers;
+	struct wrapper_msg *wrapper;
+	struct wire_msg *msg;
+	struct wire_msg *rsp;
+
+	wrappers = alloc_wrapper_list();
+	if (!wrappers)
+		return;
+
+	wrapper = add_wrapper(wrappers, sizeof(*wrapper) + sizeof(*msg) + sizeof(*trans));
+	if (!wrapper)
+		return;
+
+	msg = &wrapper->msg;
+
+	trans = (struct wire_trans_terminate_to_dev *)msg->data;
+
+	trans->hdr.type = cpu_to_le32(QAIC_TRANS_TERMINATE_TO_DEV);
+	trans->hdr.len = cpu_to_le32(sizeof(*trans));
+	trans->handle = cpu_to_le32(usr->handle);
+
+	mutex_lock(&qdev->cntl_mutex);
+	wrapper->len = sizeof(msg->hdr) + sizeof(*trans);
+	msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
+	msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
+	msg->hdr.len = cpu_to_le32(wrapper->len);
+	msg->hdr.count = cpu_to_le32(1);
+	msg->hdr.handle = cpu_to_le32(usr->handle);
+	msg->hdr.padding = cpu_to_le32(0);
+	msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
+
+	/*
+	 * msg_xfer releases the mutex
+	 * We don't care about the return of msg_xfer since we will not do
+	 * anything different based on what happens.
+	 * We ignore pending signals since one will be set if the user is
+	 * killed, and we need give the device a chance to cleanup, otherwise
+	 * DMA may still be in progress when we return.
+	 */
+	rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, true);
+	if (!IS_ERR(rsp))
+		kfree(rsp);
+	free_wrapper_from_list(wrappers, wrapper);
+}
+
+void wake_all_cntl(struct qaic_device *qdev)
+{
+	struct xfer_queue_elem *elem;
+	struct xfer_queue_elem *i;
+
+	mutex_lock(&qdev->cntl_mutex);
+	list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
+		list_del_init(&elem->list);
+		complete_all(&elem->xfer_done);
+	}
+	mutex_unlock(&qdev->cntl_mutex);
+}
diff --git a/drivers/accel/qaic/qaic_data.c b/drivers/accel/qaic/qaic_data.c
new file mode 100644
index 000000000000..c0a574cd1b35
--- /dev/null
+++ b/drivers/accel/qaic/qaic_data.c
@@ -0,0 +1,1902 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
+/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/math64.h>
+#include <linux/mm.h>
+#include <linux/moduleparam.h>
+#include <linux/scatterlist.h>
+#include <linux/spinlock.h>
+#include <linux/srcu.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/wait.h>
+#include <drm/drm_file.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_print.h>
+#include <uapi/drm/qaic_accel.h>
+
+#include "qaic.h"
+
+#define SEM_VAL_MASK	GENMASK_ULL(11, 0)
+#define SEM_INDEX_MASK	GENMASK_ULL(4, 0)
+#define BULK_XFER	BIT(3)
+#define GEN_COMPLETION	BIT(4)
+#define INBOUND_XFER	1
+#define OUTBOUND_XFER	2
+#define REQHP_OFF	0x0 /* we read this */
+#define REQTP_OFF	0x4 /* we write this */
+#define RSPHP_OFF	0x8 /* we write this */
+#define RSPTP_OFF	0xc /* we read this */
+
+#define ENCODE_SEM(val, index, sync, cmd, flags)			\
+		({							\
+			FIELD_PREP(GENMASK(11, 0), (val)) |		\
+			FIELD_PREP(GENMASK(20, 16), (index)) |		\
+			FIELD_PREP(BIT(22), (sync)) |			\
+			FIELD_PREP(GENMASK(26, 24), (cmd)) |		\
+			FIELD_PREP(GENMASK(30, 29), (flags)) |		\
+			FIELD_PREP(BIT(31), (cmd) ? 1 : 0);		\
+		})
+#define NUM_EVENTS	128
+#define NUM_DELAYS	10
+
+static unsigned int wait_exec_default_timeout_ms = 5000; /* 5 sec default */
+module_param(wait_exec_default_timeout_ms, uint, 0600);
+MODULE_PARM_DESC(wait_exec_default_timeout_ms, "Default timeout for DRM_IOCTL_QAIC_WAIT_BO");
+
+static unsigned int datapath_poll_interval_us = 100; /* 100 usec default */
+module_param(datapath_poll_interval_us, uint, 0600);
+MODULE_PARM_DESC(datapath_poll_interval_us,
+		 "Amount of time to sleep between activity when datapath polling is enabled");
+
+struct dbc_req {
+	/*
+	 * A request ID is assigned to each memory handle going in DMA queue.
+	 * As a single memory handle can enqueue multiple elements in DMA queue
+	 * all of them will have the same request ID.
+	 */
+	__le16	req_id;
+	/* Future use */
+	__u8	seq_id;
+	/*
+	 * Special encoded variable
+	 * 7	0 - Do not force to generate MSI after DMA is completed
+	 *	1 - Force to generate MSI after DMA is completed
+	 * 6:5	Reserved
+	 * 4	1 - Generate completion element in the response queue
+	 *	0 - No Completion Code
+	 * 3	0 - DMA request is a Link list transfer
+	 *	1 - DMA request is a Bulk transfer
+	 * 2	Reserved
+	 * 1:0	00 - No DMA transfer involved
+	 *	01 - DMA transfer is part of inbound transfer
+	 *	10 - DMA transfer has outbound transfer
+	 *	11 - NA
+	 */
+	__u8	cmd;
+	__le32	resv;
+	/* Source address for the transfer */
+	__le64	src_addr;
+	/* Destination address for the transfer */
+	__le64	dest_addr;
+	/* Length of transfer request */
+	__le32	len;
+	__le32	resv2;
+	/* Doorbell address */
+	__le64	db_addr;
+	/*
+	 * Special encoded variable
+	 * 7	1 - Doorbell(db) write
+	 *	0 - No doorbell write
+	 * 6:2	Reserved
+	 * 1:0	00 - 32 bit access, db address must be aligned to 32bit-boundary
+	 *	01 - 16 bit access, db address must be aligned to 16bit-boundary
+	 *	10 - 8 bit access, db address must be aligned to 8bit-boundary
+	 *	11 - Reserved
+	 */
+	__u8	db_len;
+	__u8	resv3;
+	__le16	resv4;
+	/* 32 bit data written to doorbell address */
+	__le32	db_data;
+	/*
+	 * Special encoded variable
+	 * All the fields of sem_cmdX are passed from user and all are ORed
+	 * together to form sem_cmd.
+	 * 0:11		Semaphore value
+	 * 15:12	Reserved
+	 * 20:16	Semaphore index
+	 * 21		Reserved
+	 * 22		Semaphore Sync
+	 * 23		Reserved
+	 * 26:24	Semaphore command
+	 * 28:27	Reserved
+	 * 29		Semaphore DMA out bound sync fence
+	 * 30		Semaphore DMA in bound sync fence
+	 * 31		Enable semaphore command
+	 */
+	__le32	sem_cmd0;
+	__le32	sem_cmd1;
+	__le32	sem_cmd2;
+	__le32	sem_cmd3;
+} __packed;
+
+struct dbc_rsp {
+	/* Request ID of the memory handle whose DMA transaction is completed */
+	__le16	req_id;
+	/* Status of the DMA transaction. 0 : Success otherwise failure */
+	__le16	status;
+} __packed;
+
+inline int get_dbc_req_elem_size(void)
+{
+	return sizeof(struct dbc_req);
+}
+
+inline int get_dbc_rsp_elem_size(void)
+{
+	return sizeof(struct dbc_rsp);
+}
+
+static void free_slice(struct kref *kref)
+{
+	struct bo_slice *slice = container_of(kref, struct bo_slice, ref_count);
+
+	list_del(&slice->slice);
+	drm_gem_object_put(&slice->bo->base);
+	sg_free_table(slice->sgt);
+	kfree(slice->sgt);
+	kfree(slice->reqs);
+	kfree(slice);
+}
+
+static int clone_range_of_sgt_for_slice(struct qaic_device *qdev, struct sg_table **sgt_out,
+					struct sg_table *sgt_in, u64 size, u64 offset)
+{
+	int total_len, len, nents, offf = 0, offl = 0;
+	struct scatterlist *sg, *sgn, *sgf, *sgl;
+	struct sg_table *sgt;
+	int ret, j;
+
+	/* find out number of relevant nents needed for this mem */
+	total_len = 0;
+	sgf = NULL;
+	sgl = NULL;
+	nents = 0;
+
+	size = size ? size : PAGE_SIZE;
+	for (sg = sgt_in->sgl; sg; sg = sg_next(sg)) {
+		len = sg_dma_len(sg);
+
+		if (!len)
+			continue;
+		if (offset >= total_len && offset < total_len + len) {
+			sgf = sg;
+			offf = offset - total_len;
+		}
+		if (sgf)
+			nents++;
+		if (offset + size >= total_len &&
+		    offset + size <= total_len + len) {
+			sgl = sg;
+			offl = offset + size - total_len;
+			break;
+		}
+		total_len += len;
+	}
+
+	if (!sgf || !sgl) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+	if (!sgt) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = sg_alloc_table(sgt, nents, GFP_KERNEL);
+	if (ret)
+		goto free_sgt;
+
+	/* copy relevant sg node and fix page and length */
+	sgn = sgf;
+	for_each_sgtable_sg(sgt, sg, j) {
+		memcpy(sg, sgn, sizeof(*sg));
+		if (sgn == sgf) {
+			sg_dma_address(sg) += offf;
+			sg_dma_len(sg) -= offf;
+			sg_set_page(sg, sg_page(sgn), sg_dma_len(sg), offf);
+		} else {
+			offf = 0;
+		}
+		if (sgn == sgl) {
+			sg_dma_len(sg) = offl - offf;
+			sg_set_page(sg, sg_page(sgn), offl - offf, offf);
+			sg_mark_end(sg);
+			break;
+		}
+		sgn = sg_next(sgn);
+	}
+
+	*sgt_out = sgt;
+	return ret;
+
+free_sgt:
+	kfree(sgt);
+out:
+	*sgt_out = NULL;
+	return ret;
+}
+
+static int encode_reqs(struct qaic_device *qdev, struct bo_slice *slice,
+		       struct qaic_attach_slice_entry *req)
+{
+	__le64 db_addr = cpu_to_le64(req->db_addr);
+	__le32 db_data = cpu_to_le32(req->db_data);
+	struct scatterlist *sg;
+	__u8 cmd = BULK_XFER;
+	int presync_sem;
+	u64 dev_addr;
+	__u8 db_len;
+	int i;
+
+	if (!slice->no_xfer)
+		cmd |= (slice->dir == DMA_TO_DEVICE ? INBOUND_XFER : OUTBOUND_XFER);
+
+	if (req->db_len && !IS_ALIGNED(req->db_addr, req->db_len / 8))
+		return -EINVAL;
+
+	presync_sem = req->sem0.presync + req->sem1.presync + req->sem2.presync + req->sem3.presync;
+	if (presync_sem > 1)
+		return -EINVAL;
+
+	presync_sem = req->sem0.presync << 0 | req->sem1.presync << 1 |
+		      req->sem2.presync << 2 | req->sem3.presync << 3;
+
+	switch (req->db_len) {
+	case 32:
+		db_len = BIT(7);
+		break;
+	case 16:
+		db_len = BIT(7) | 1;
+		break;
+	case 8:
+		db_len = BIT(7) | 2;
+		break;
+	case 0:
+		db_len = 0; /* doorbell is not active for this command */
+		break;
+	default:
+		return -EINVAL; /* should never hit this */
+	}
+
+	/*
+	 * When we end up splitting up a single request (ie a buf slice) into
+	 * multiple DMA requests, we have to manage the sync data carefully.
+	 * There can only be one presync sem. That needs to be on every xfer
+	 * so that the DMA engine doesn't transfer data before the receiver is
+	 * ready. We only do the doorbell and postsync sems after the xfer.
+	 * To guarantee previous xfers for the request are complete, we use a
+	 * fence.
+	 */
+	dev_addr = req->dev_addr;
+	for_each_sgtable_sg(slice->sgt, sg, i) {
+		slice->reqs[i].cmd = cmd;
+		slice->reqs[i].src_addr = cpu_to_le64(slice->dir == DMA_TO_DEVICE ?
+						      sg_dma_address(sg) : dev_addr);
+		slice->reqs[i].dest_addr = cpu_to_le64(slice->dir == DMA_TO_DEVICE ?
+						       dev_addr : sg_dma_address(sg));
+		/*
+		 * sg_dma_len(sg) returns size of a DMA segment, maximum DMA
+		 * segment size is set to UINT_MAX by qaic and hence return
+		 * values of sg_dma_len(sg) can never exceed u32 range. So,
+		 * by down sizing we are not corrupting the value.
+		 */
+		slice->reqs[i].len = cpu_to_le32((u32)sg_dma_len(sg));
+		switch (presync_sem) {
+		case BIT(0):
+			slice->reqs[i].sem_cmd0 = cpu_to_le32(ENCODE_SEM(req->sem0.val,
+									 req->sem0.index,
+									 req->sem0.presync,
+									 req->sem0.cmd,
+									 req->sem0.flags));
+			break;
+		case BIT(1):
+			slice->reqs[i].sem_cmd1 = cpu_to_le32(ENCODE_SEM(req->sem1.val,
+									 req->sem1.index,
+									 req->sem1.presync,
+									 req->sem1.cmd,
+									 req->sem1.flags));
+			break;
+		case BIT(2):
+			slice->reqs[i].sem_cmd2 = cpu_to_le32(ENCODE_SEM(req->sem2.val,
+									 req->sem2.index,
+									 req->sem2.presync,
+									 req->sem2.cmd,
+									 req->sem2.flags));
+			break;
+		case BIT(3):
+			slice->reqs[i].sem_cmd3 = cpu_to_le32(ENCODE_SEM(req->sem3.val,
+									 req->sem3.index,
+									 req->sem3.presync,
+									 req->sem3.cmd,
+									 req->sem3.flags));
+			break;
+		}
+		dev_addr += sg_dma_len(sg);
+	}
+	/* add post transfer stuff to last segment */
+	i--;
+	slice->reqs[i].cmd |= GEN_COMPLETION;
+	slice->reqs[i].db_addr = db_addr;
+	slice->reqs[i].db_len = db_len;
+	slice->reqs[i].db_data = db_data;
+	/*
+	 * Add a fence if we have more than one request going to the hardware
+	 * representing the entirety of the user request, and the user request
+	 * has no presync condition.
+	 * Fences are expensive, so we try to avoid them. We rely on the
+	 * hardware behavior to avoid needing one when there is a presync
+	 * condition. When a presync exists, all requests for that same
+	 * presync will be queued into a fifo. Thus, since we queue the
+	 * post xfer activity only on the last request we queue, the hardware
+	 * will ensure that the last queued request is processed last, thus
+	 * making sure the post xfer activity happens at the right time without
+	 * a fence.
+	 */
+	if (i && !presync_sem)
+		req->sem0.flags |= (slice->dir == DMA_TO_DEVICE ?
+				    QAIC_SEM_INSYNCFENCE : QAIC_SEM_OUTSYNCFENCE);
+	slice->reqs[i].sem_cmd0 = cpu_to_le32(ENCODE_SEM(req->sem0.val, req->sem0.index,
+							 req->sem0.presync, req->sem0.cmd,
+							 req->sem0.flags));
+	slice->reqs[i].sem_cmd1 = cpu_to_le32(ENCODE_SEM(req->sem1.val, req->sem1.index,
+							 req->sem1.presync, req->sem1.cmd,
+							 req->sem1.flags));
+	slice->reqs[i].sem_cmd2 = cpu_to_le32(ENCODE_SEM(req->sem2.val, req->sem2.index,
+							 req->sem2.presync, req->sem2.cmd,
+							 req->sem2.flags));
+	slice->reqs[i].sem_cmd3 = cpu_to_le32(ENCODE_SEM(req->sem3.val, req->sem3.index,
+							 req->sem3.presync, req->sem3.cmd,
+							 req->sem3.flags));
+
+	return 0;
+}
+
+static int qaic_map_one_slice(struct qaic_device *qdev, struct qaic_bo *bo,
+			      struct qaic_attach_slice_entry *slice_ent)
+{
+	struct sg_table *sgt = NULL;
+	struct bo_slice *slice;
+	int ret;
+
+	ret = clone_range_of_sgt_for_slice(qdev, &sgt, bo->sgt, slice_ent->size, slice_ent->offset);
+	if (ret)
+		goto out;
+
+	slice = kmalloc(sizeof(*slice), GFP_KERNEL);
+	if (!slice) {
+		ret = -ENOMEM;
+		goto free_sgt;
+	}
+
+	slice->reqs = kcalloc(sgt->nents, sizeof(*slice->reqs), GFP_KERNEL);
+	if (!slice->reqs) {
+		ret = -ENOMEM;
+		goto free_slice;
+	}
+
+	slice->no_xfer = !slice_ent->size;
+	slice->sgt = sgt;
+	slice->nents = sgt->nents;
+	slice->dir = bo->dir;
+	slice->bo = bo;
+	slice->size = slice_ent->size;
+	slice->offset = slice_ent->offset;
+
+	ret = encode_reqs(qdev, slice, slice_ent);
+	if (ret)
+		goto free_req;
+
+	bo->total_slice_nents += sgt->nents;
+	kref_init(&slice->ref_count);
+	drm_gem_object_get(&bo->base);
+	list_add_tail(&slice->slice, &bo->slices);
+
+	return 0;
+
+free_req:
+	kfree(slice->reqs);
+free_slice:
+	kfree(slice);
+free_sgt:
+	sg_free_table(sgt);
+	kfree(sgt);
+out:
+	return ret;
+}
+
+static int create_sgt(struct qaic_device *qdev, struct sg_table **sgt_out, u64 size)
+{
+	struct scatterlist *sg;
+	struct sg_table *sgt;
+	struct page **pages;
+	int *pages_order;
+	int buf_extra;
+	int max_order;
+	int nr_pages;
+	int ret = 0;
+	int i, j, k;
+	int order;
+
+	if (size) {
+		nr_pages = DIV_ROUND_UP(size, PAGE_SIZE);
+		/*
+		 * calculate how much extra we are going to allocate, to remove
+		 * later
+		 */
+		buf_extra = (PAGE_SIZE - size % PAGE_SIZE) % PAGE_SIZE;
+		max_order = min(MAX_ORDER - 1, get_order(size));
+	} else {
+		/* allocate a single page for book keeping */
+		nr_pages = 1;
+		buf_extra = 0;
+		max_order = 0;
+	}
+
+	pages = kvmalloc_array(nr_pages, sizeof(*pages) + sizeof(*pages_order), GFP_KERNEL);
+	if (!pages) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	pages_order = (void *)pages + sizeof(*pages) * nr_pages;
+
+	/*
+	 * Allocate requested memory using alloc_pages. It is possible to allocate
+	 * the requested memory in multiple chunks by calling alloc_pages
+	 * multiple times. Use SG table to handle multiple allocated pages.
+	 */
+	i = 0;
+	while (nr_pages > 0) {
+		order = min(get_order(nr_pages * PAGE_SIZE), max_order);
+		while (1) {
+			pages[i] = alloc_pages(GFP_KERNEL | GFP_HIGHUSER |
+					       __GFP_NOWARN | __GFP_ZERO |
+					       (order ? __GFP_NORETRY : __GFP_RETRY_MAYFAIL),
+					       order);
+			if (pages[i])
+				break;
+			if (!order--) {
+				ret = -ENOMEM;
+				goto free_partial_alloc;
+			}
+		}
+
+		max_order = order;
+		pages_order[i] = order;
+
+		nr_pages -= 1 << order;
+		if (nr_pages <= 0)
+			/* account for over allocation */
+			buf_extra += abs(nr_pages) * PAGE_SIZE;
+		i++;
+	}
+
+	sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
+	if (!sgt) {
+		ret = -ENOMEM;
+		goto free_partial_alloc;
+	}
+
+	if (sg_alloc_table(sgt, i, GFP_KERNEL)) {
+		ret = -ENOMEM;
+		goto free_sgt;
+	}
+
+	/* Populate the SG table with the allocated memory pages */
+	sg = sgt->sgl;
+	for (k = 0; k < i; k++, sg = sg_next(sg)) {
+		/* Last entry requires special handling */
+		if (k < i - 1) {
+			sg_set_page(sg, pages[k], PAGE_SIZE << pages_order[k], 0);
+		} else {
+			sg_set_page(sg, pages[k], (PAGE_SIZE << pages_order[k]) - buf_extra, 0);
+			sg_mark_end(sg);
+		}
+	}
+
+	kvfree(pages);
+	*sgt_out = sgt;
+	return ret;
+
+free_sgt:
+	kfree(sgt);
+free_partial_alloc:
+	for (j = 0; j < i; j++)
+		__free_pages(pages[j], pages_order[j]);
+	kvfree(pages);
+out:
+	*sgt_out = NULL;
+	return ret;
+}
+
+static bool invalid_sem(struct qaic_sem *sem)
+{
+	if (sem->val & ~SEM_VAL_MASK || sem->index & ~SEM_INDEX_MASK ||
+	    !(sem->presync == 0 || sem->presync == 1) || sem->pad ||
+	    sem->flags & ~(QAIC_SEM_INSYNCFENCE | QAIC_SEM_OUTSYNCFENCE) ||
+	    sem->cmd > QAIC_SEM_WAIT_GT_0)
+		return true;
+	return false;
+}
+
+static int qaic_validate_req(struct qaic_device *qdev, struct qaic_attach_slice_entry *slice_ent,
+			     u32 count, u64 total_size)
+{
+	int i;
+
+	for (i = 0; i < count; i++) {
+		if (!(slice_ent[i].db_len == 32 || slice_ent[i].db_len == 16 ||
+		      slice_ent[i].db_len == 8 || slice_ent[i].db_len == 0) ||
+		      invalid_sem(&slice_ent[i].sem0) || invalid_sem(&slice_ent[i].sem1) ||
+		      invalid_sem(&slice_ent[i].sem2) || invalid_sem(&slice_ent[i].sem3))
+			return -EINVAL;
+
+		if (slice_ent[i].offset + slice_ent[i].size > total_size)
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void qaic_free_sgt(struct sg_table *sgt)
+{
+	struct scatterlist *sg;
+
+	for (sg = sgt->sgl; sg; sg = sg_next(sg))
+		if (sg_page(sg))
+			__free_pages(sg_page(sg), get_order(sg->length));
+	sg_free_table(sgt);
+	kfree(sgt);
+}
+
+static void qaic_gem_print_info(struct drm_printer *p, unsigned int indent,
+				const struct drm_gem_object *obj)
+{
+	struct qaic_bo *bo = to_qaic_bo(obj);
+
+	drm_printf_indent(p, indent, "user requested size=%llu\n", bo->size);
+}
+
+static const struct vm_operations_struct drm_vm_ops = {
+	.open = drm_gem_vm_open,
+	.close = drm_gem_vm_close,
+};
+
+static int qaic_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+	struct qaic_bo *bo = to_qaic_bo(obj);
+	unsigned long offset = 0;
+	struct scatterlist *sg;
+	int ret;
+
+	if (obj->import_attach)
+		return -EINVAL;
+
+	for (sg = bo->sgt->sgl; sg; sg = sg_next(sg)) {
+		if (sg_page(sg)) {
+			ret = remap_pfn_range(vma, vma->vm_start + offset, page_to_pfn(sg_page(sg)),
+					      sg->length, vma->vm_page_prot);
+			if (ret)
+				goto out;
+			offset += sg->length;
+		}
+	}
+
+out:
+	return ret;
+}
+
+static void qaic_free_object(struct drm_gem_object *obj)
+{
+	struct qaic_bo *bo = to_qaic_bo(obj);
+
+	if (obj->import_attach) {
+		/* DMABUF/PRIME Path */
+		dma_buf_detach(obj->import_attach->dmabuf, obj->import_attach);
+		dma_buf_put(obj->import_attach->dmabuf);
+	} else {
+		/* Private buffer allocation path */
+		qaic_free_sgt(bo->sgt);
+	}
+
+	drm_gem_object_release(obj);
+	kfree(bo);
+}
+
+static const struct drm_gem_object_funcs qaic_gem_funcs = {
+	.free = qaic_free_object,
+	.print_info = qaic_gem_print_info,
+	.mmap = qaic_gem_object_mmap,
+	.vm_ops = &drm_vm_ops,
+};
+
+static struct qaic_bo *qaic_alloc_init_bo(void)
+{
+	struct qaic_bo *bo;
+
+	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
+	if (!bo)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&bo->slices);
+	init_completion(&bo->xfer_done);
+	complete_all(&bo->xfer_done);
+
+	return bo;
+}
+
+int qaic_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	struct qaic_create_bo *args = data;
+	int usr_rcu_id, qdev_rcu_id;
+	struct drm_gem_object *obj;
+	struct qaic_device *qdev;
+	struct qaic_user *usr;
+	struct qaic_bo *bo;
+	size_t size;
+	int ret;
+
+	if (args->pad)
+		return -EINVAL;
+
+	usr = file_priv->driver_priv;
+	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
+	if (!usr->qddev) {
+		ret = -ENODEV;
+		goto unlock_usr_srcu;
+	}
+
+	qdev = usr->qddev->qdev;
+	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
+	if (qdev->in_reset) {
+		ret = -ENODEV;
+		goto unlock_dev_srcu;
+	}
+
+	size = PAGE_ALIGN(args->size);
+	if (size == 0) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	bo = qaic_alloc_init_bo();
+	if (IS_ERR(bo)) {
+		ret = PTR_ERR(bo);
+		goto unlock_dev_srcu;
+	}
+	obj = &bo->base;
+
+	drm_gem_private_object_init(dev, obj, size);
+
+	obj->funcs = &qaic_gem_funcs;
+	ret = create_sgt(qdev, &bo->sgt, size);
+	if (ret)
+		goto free_bo;
+
+	bo->size = args->size;
+
+	ret = drm_gem_handle_create(file_priv, obj, &args->handle);
+	if (ret)
+		goto free_sgt;
+
+	bo->handle = args->handle;
+	drm_gem_object_put(obj);
+	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+
+	return 0;
+
+free_sgt:
+	qaic_free_sgt(bo->sgt);
+free_bo:
+	kfree(bo);
+unlock_dev_srcu:
+	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+unlock_usr_srcu:
+	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+	return ret;
+}
+
+int qaic_mmap_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	struct qaic_mmap_bo *args = data;
+	int usr_rcu_id, qdev_rcu_id;
+	struct drm_gem_object *obj;
+	struct qaic_device *qdev;
+	struct qaic_user *usr;
+	int ret;
+
+	usr = file_priv->driver_priv;
+	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
+	if (!usr->qddev) {
+		ret = -ENODEV;
+		goto unlock_usr_srcu;
+	}
+
+	qdev = usr->qddev->qdev;
+	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
+	if (qdev->in_reset) {
+		ret = -ENODEV;
+		goto unlock_dev_srcu;
+	}
+
+	obj = drm_gem_object_lookup(file_priv, args->handle);
+	if (!obj) {
+		ret = -ENOENT;
+		goto unlock_dev_srcu;
+	}
+
+	ret = drm_gem_create_mmap_offset(obj);
+	if (ret == 0)
+		args->offset = drm_vma_node_offset_addr(&obj->vma_node);
+
+	drm_gem_object_put(obj);
+
+unlock_dev_srcu:
+	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+unlock_usr_srcu:
+	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+	return ret;
+}
+
+struct drm_gem_object *qaic_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf)
+{
+	struct dma_buf_attachment *attach;
+	struct drm_gem_object *obj;
+	struct qaic_bo *bo;
+	size_t size;
+	int ret;
+
+	bo = qaic_alloc_init_bo();
+	if (IS_ERR(bo)) {
+		ret = PTR_ERR(bo);
+		goto out;
+	}
+
+	obj = &bo->base;
+	get_dma_buf(dma_buf);
+
+	attach = dma_buf_attach(dma_buf, dev->dev);
+	if (IS_ERR(attach)) {
+		ret = PTR_ERR(attach);
+		goto attach_fail;
+	}
+
+	size = PAGE_ALIGN(attach->dmabuf->size);
+	if (size == 0) {
+		ret = -EINVAL;
+		goto size_align_fail;
+	}
+
+	drm_gem_private_object_init(dev, obj, size);
+	/*
+	 * skipping dma_buf_map_attachment() as we do not know the direction
+	 * just yet. Once the direction is known in the subsequent IOCTL to
+	 * attach slicing, we can do it then.
+	 */
+
+	obj->funcs = &qaic_gem_funcs;
+	obj->import_attach = attach;
+	obj->resv = dma_buf->resv;
+
+	return obj;
+
+size_align_fail:
+	dma_buf_detach(dma_buf, attach);
+attach_fail:
+	dma_buf_put(dma_buf);
+	kfree(bo);
+out:
+	return ERR_PTR(ret);
+}
+
+static int qaic_prepare_import_bo(struct qaic_bo *bo, struct qaic_attach_slice_hdr *hdr)
+{
+	struct drm_gem_object *obj = &bo->base;
+	struct sg_table *sgt;
+	int ret;
+
+	if (obj->import_attach->dmabuf->size < hdr->size)
+		return -EINVAL;
+
+	sgt = dma_buf_map_attachment(obj->import_attach, hdr->dir);
+	if (IS_ERR(sgt)) {
+		ret = PTR_ERR(sgt);
+		return ret;
+	}
+
+	bo->sgt = sgt;
+	bo->size = hdr->size;
+
+	return 0;
+}
+
+static int qaic_prepare_export_bo(struct qaic_device *qdev, struct qaic_bo *bo,
+				  struct qaic_attach_slice_hdr *hdr)
+{
+	int ret;
+
+	if (bo->size != hdr->size)
+		return -EINVAL;
+
+	ret = dma_map_sgtable(&qdev->pdev->dev, bo->sgt, hdr->dir, 0);
+	if (ret)
+		return -EFAULT;
+
+	return 0;
+}
+
+static int qaic_prepare_bo(struct qaic_device *qdev, struct qaic_bo *bo,
+			   struct qaic_attach_slice_hdr *hdr)
+{
+	int ret;
+
+	if (bo->base.import_attach)
+		ret = qaic_prepare_import_bo(bo, hdr);
+	else
+		ret = qaic_prepare_export_bo(qdev, bo, hdr);
+
+	if (ret == 0)
+		bo->dir = hdr->dir;
+
+	return ret;
+}
+
+static void qaic_unprepare_import_bo(struct qaic_bo *bo)
+{
+	dma_buf_unmap_attachment(bo->base.import_attach, bo->sgt, bo->dir);
+	bo->sgt = NULL;
+	bo->size = 0;
+}
+
+static void qaic_unprepare_export_bo(struct qaic_device *qdev, struct qaic_bo *bo)
+{
+	dma_unmap_sgtable(&qdev->pdev->dev, bo->sgt, bo->dir, 0);
+}
+
+static void qaic_unprepare_bo(struct qaic_device *qdev, struct qaic_bo *bo)
+{
+	if (bo->base.import_attach)
+		qaic_unprepare_import_bo(bo);
+	else
+		qaic_unprepare_export_bo(qdev, bo);
+
+	bo->dir = 0;
+}
+
+static void qaic_free_slices_bo(struct qaic_bo *bo)
+{
+	struct bo_slice *slice, *temp;
+
+	list_for_each_entry_safe(slice, temp, &bo->slices, slice)
+		kref_put(&slice->ref_count, free_slice);
+}
+
+static int qaic_attach_slicing_bo(struct qaic_device *qdev, struct qaic_bo *bo,
+				  struct qaic_attach_slice_hdr *hdr,
+				  struct qaic_attach_slice_entry *slice_ent)
+{
+	int ret, i;
+
+	for (i = 0; i < hdr->count; i++) {
+		ret = qaic_map_one_slice(qdev, bo, &slice_ent[i]);
+		if (ret) {
+			qaic_free_slices_bo(bo);
+			return ret;
+		}
+	}
+
+	if (bo->total_slice_nents > qdev->dbc[hdr->dbc_id].nelem) {
+		qaic_free_slices_bo(bo);
+		return -ENOSPC;
+	}
+
+	bo->sliced = true;
+	bo->nr_slice = hdr->count;
+	list_add_tail(&bo->bo_list, &qdev->dbc[hdr->dbc_id].bo_lists);
+
+	return 0;
+}
+
+int qaic_attach_slice_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	struct qaic_attach_slice_entry *slice_ent;
+	struct qaic_attach_slice *args = data;
+	struct dma_bridge_chan	*dbc;
+	int usr_rcu_id, qdev_rcu_id;
+	struct drm_gem_object *obj;
+	struct qaic_device *qdev;
+	unsigned long arg_size;
+	struct qaic_user *usr;
+	u8 __user *user_data;
+	struct qaic_bo *bo;
+	int ret;
+
+	usr = file_priv->driver_priv;
+	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
+	if (!usr->qddev) {
+		ret = -ENODEV;
+		goto unlock_usr_srcu;
+	}
+
+	qdev = usr->qddev->qdev;
+	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
+	if (qdev->in_reset) {
+		ret = -ENODEV;
+		goto unlock_dev_srcu;
+	}
+
+	if (args->hdr.count == 0) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	arg_size = args->hdr.count * sizeof(*slice_ent);
+	if (arg_size / args->hdr.count != sizeof(*slice_ent)) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	if (args->hdr.dbc_id >= qdev->num_dbc) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	if (args->hdr.size == 0) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	if (!(args->hdr.dir == DMA_TO_DEVICE  || args->hdr.dir == DMA_FROM_DEVICE)) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	dbc = &qdev->dbc[args->hdr.dbc_id];
+	if (dbc->usr != usr) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	if (args->data == 0) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	user_data = u64_to_user_ptr(args->data);
+
+	slice_ent = kzalloc(arg_size, GFP_KERNEL);
+	if (!slice_ent) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	ret = copy_from_user(slice_ent, user_data, arg_size);
+	if (ret) {
+		ret = -EFAULT;
+		goto free_slice_ent;
+	}
+
+	ret = qaic_validate_req(qdev, slice_ent, args->hdr.count, args->hdr.size);
+	if (ret)
+		goto free_slice_ent;
+
+	obj = drm_gem_object_lookup(file_priv, args->hdr.handle);
+	if (!obj) {
+		ret = -ENOENT;
+		goto free_slice_ent;
+	}
+
+	bo = to_qaic_bo(obj);
+
+	ret = qaic_prepare_bo(qdev, bo, &args->hdr);
+	if (ret)
+		goto put_bo;
+
+	ret = qaic_attach_slicing_bo(qdev, bo, &args->hdr, slice_ent);
+	if (ret)
+		goto unprepare_bo;
+
+	if (args->hdr.dir == DMA_TO_DEVICE)
+		dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, args->hdr.dir);
+
+	bo->dbc = dbc;
+	drm_gem_object_put(obj);
+	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+
+	return 0;
+
+unprepare_bo:
+	qaic_unprepare_bo(qdev, bo);
+put_bo:
+	drm_gem_object_put(obj);
+free_slice_ent:
+	kfree(slice_ent);
+unlock_dev_srcu:
+	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+unlock_usr_srcu:
+	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+	return ret;
+}
+
+static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, u32 dbc_id,
+				 u32 head, u32 *ptail)
+{
+	struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];
+	struct dbc_req *reqs = slice->reqs;
+	u32 tail = *ptail;
+	u32 avail;
+
+	avail = head - tail;
+	if (head <= tail)
+		avail += dbc->nelem;
+
+	--avail;
+
+	if (avail < slice->nents)
+		return -EAGAIN;
+
+	if (tail + slice->nents > dbc->nelem) {
+		avail = dbc->nelem - tail;
+		avail = min_t(u32, avail, slice->nents);
+		memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
+		       sizeof(*reqs) * avail);
+		reqs += avail;
+		avail = slice->nents - avail;
+		if (avail)
+			memcpy(dbc->req_q_base, reqs, sizeof(*reqs) * avail);
+	} else {
+		memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
+		       sizeof(*reqs) * slice->nents);
+	}
+
+	*ptail = (tail + slice->nents) % dbc->nelem;
+
+	return 0;
+}
+
+/*
+ * Based on the value of resize we may only need to transmit first_n
+ * entries and the last entry, with last_bytes to send from the last entry.
+ * Note that first_n could be 0.
+ */
+static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice,
+					 u64 resize, u32 dbc_id, u32 head, u32 *ptail)
+{
+	struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];
+	struct dbc_req *reqs = slice->reqs;
+	struct dbc_req *last_req;
+	u32 tail = *ptail;
+	u64 total_bytes;
+	u64 last_bytes;
+	u32 first_n;
+	u32 avail;
+	int ret;
+	int i;
+
+	avail = head - tail;
+	if (head <= tail)
+		avail += dbc->nelem;
+
+	--avail;
+
+	total_bytes = 0;
+	for (i = 0; i < slice->nents; i++) {
+		total_bytes += le32_to_cpu(reqs[i].len);
+		if (total_bytes >= resize)
+			break;
+	}
+
+	if (total_bytes < resize) {
+		/* User space should have used the full buffer path. */
+		ret = -EINVAL;
+		return ret;
+	}
+
+	first_n = i;
+	last_bytes = i ? resize + le32_to_cpu(reqs[i].len) - total_bytes : resize;
+
+	if (avail < (first_n + 1))
+		return -EAGAIN;
+
+	if (first_n) {
+		if (tail + first_n > dbc->nelem) {
+			avail = dbc->nelem - tail;
+			avail = min_t(u32, avail, first_n);
+			memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
+			       sizeof(*reqs) * avail);
+			last_req = reqs + avail;
+			avail = first_n - avail;
+			if (avail)
+				memcpy(dbc->req_q_base, last_req, sizeof(*reqs) * avail);
+		} else {
+			memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
+			       sizeof(*reqs) * first_n);
+		}
+	}
+
+	/* Copy over the last entry. Here we need to adjust len to the left over
+	 * size, and set src and dst to the entry it is copied to.
+	 */
+	last_req = dbc->req_q_base + (tail + first_n) % dbc->nelem * get_dbc_req_elem_size();
+	memcpy(last_req, reqs + slice->nents - 1, sizeof(*reqs));
+
+	/*
+	 * last_bytes holds size of a DMA segment, maximum DMA segment size is
+	 * set to UINT_MAX by qaic and hence last_bytes can never exceed u32
+	 * range. So, by down sizing we are not corrupting the value.
+	 */
+	last_req->len = cpu_to_le32((u32)last_bytes);
+	last_req->src_addr = reqs[first_n].src_addr;
+	last_req->dest_addr = reqs[first_n].dest_addr;
+
+	*ptail = (tail + first_n + 1) % dbc->nelem;
+
+	return 0;
+}
+
+static int send_bo_list_to_device(struct qaic_device *qdev, struct drm_file *file_priv,
+				  struct qaic_execute_entry *exec, unsigned int count,
+				  bool is_partial, struct dma_bridge_chan *dbc, u32 head,
+				  u32 *tail)
+{
+	struct qaic_partial_execute_entry *pexec = (struct qaic_partial_execute_entry *)exec;
+	struct drm_gem_object *obj;
+	struct bo_slice *slice;
+	unsigned long flags;
+	struct qaic_bo *bo;
+	bool queued;
+	int i, j;
+	int ret;
+
+	for (i = 0; i < count; i++) {
+		/*
+		 * ref count will be decremented when the transfer of this
+		 * buffer is complete. It is inside dbc_irq_threaded_fn().
+		 */
+		obj = drm_gem_object_lookup(file_priv,
+					    is_partial ? pexec[i].handle : exec[i].handle);
+		if (!obj) {
+			ret = -ENOENT;
+			goto failed_to_send_bo;
+		}
+
+		bo = to_qaic_bo(obj);
+
+		if (!bo->sliced) {
+			ret = -EINVAL;
+			goto failed_to_send_bo;
+		}
+
+		if (is_partial && pexec[i].resize > bo->size) {
+			ret = -EINVAL;
+			goto failed_to_send_bo;
+		}
+
+		spin_lock_irqsave(&dbc->xfer_lock, flags);
+		queued = bo->queued;
+		bo->queued = true;
+		if (queued) {
+			spin_unlock_irqrestore(&dbc->xfer_lock, flags);
+			ret = -EINVAL;
+			goto failed_to_send_bo;
+		}
+
+		bo->req_id = dbc->next_req_id++;
+
+		list_for_each_entry(slice, &bo->slices, slice) {
+			/*
+			 * If this slice does not fall under the given
+			 * resize then skip this slice and continue the loop
+			 */
+			if (is_partial && pexec[i].resize && pexec[i].resize <= slice->offset)
+				continue;
+
+			for (j = 0; j < slice->nents; j++)
+				slice->reqs[j].req_id = cpu_to_le16(bo->req_id);
+
+			/*
+			 * If it is a partial execute ioctl call then check if
+			 * resize has cut this slice short then do a partial copy
+			 * else do complete copy
+			 */
+			if (is_partial && pexec[i].resize &&
+			    pexec[i].resize < slice->offset + slice->size)
+				ret = copy_partial_exec_reqs(qdev, slice,
+							     pexec[i].resize - slice->offset,
+							     dbc->id, head, tail);
+			else
+				ret = copy_exec_reqs(qdev, slice, dbc->id, head, tail);
+			if (ret) {
+				bo->queued = false;
+				spin_unlock_irqrestore(&dbc->xfer_lock, flags);
+				goto failed_to_send_bo;
+			}
+		}
+		reinit_completion(&bo->xfer_done);
+		list_add_tail(&bo->xfer_list, &dbc->xfer_list);
+		spin_unlock_irqrestore(&dbc->xfer_lock, flags);
+		dma_sync_sgtable_for_device(&qdev->pdev->dev, bo->sgt, bo->dir);
+	}
+
+	return 0;
+
+failed_to_send_bo:
+	if (likely(obj))
+		drm_gem_object_put(obj);
+	for (j = 0; j < i; j++) {
+		spin_lock_irqsave(&dbc->xfer_lock, flags);
+		bo = list_last_entry(&dbc->xfer_list, struct qaic_bo, xfer_list);
+		obj = &bo->base;
+		bo->queued = false;
+		list_del(&bo->xfer_list);
+		spin_unlock_irqrestore(&dbc->xfer_lock, flags);
+		dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir);
+		drm_gem_object_put(obj);
+	}
+	return ret;
+}
+
+static void update_profiling_data(struct drm_file *file_priv,
+				  struct qaic_execute_entry *exec, unsigned int count,
+				  bool is_partial, u64 received_ts, u64 submit_ts, u32 queue_level)
+{
+	struct qaic_partial_execute_entry *pexec = (struct qaic_partial_execute_entry *)exec;
+	struct drm_gem_object *obj;
+	struct qaic_bo *bo;
+	int i;
+
+	for (i = 0; i < count; i++) {
+		/*
+		 * Since we already committed the BO to hardware, the only way
+		 * this should fail is a pending signal. We can't cancel the
+		 * submit to hardware, so we have to just skip the profiling
+		 * data. In case the signal is not fatal to the process, we
+		 * return success so that the user doesn't try to resubmit.
+		 */
+		obj = drm_gem_object_lookup(file_priv,
+					    is_partial ? pexec[i].handle : exec[i].handle);
+		if (!obj)
+			break;
+		bo = to_qaic_bo(obj);
+		bo->perf_stats.req_received_ts = received_ts;
+		bo->perf_stats.req_submit_ts = submit_ts;
+		bo->perf_stats.queue_level_before = queue_level;
+		queue_level += bo->total_slice_nents;
+		drm_gem_object_put(obj);
+	}
+}
+
+static int __qaic_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv,
+				   bool is_partial)
+{
+	struct qaic_partial_execute_entry *pexec;
+	struct qaic_execute *args = data;
+	struct qaic_execute_entry *exec;
+	struct dma_bridge_chan *dbc;
+	int usr_rcu_id, qdev_rcu_id;
+	struct qaic_device *qdev;
+	struct qaic_user *usr;
+	u8 __user *user_data;
+	unsigned long n;
+	u64 received_ts;
+	u32 queue_level;
+	u64 submit_ts;
+	int rcu_id;
+	u32 head;
+	u32 tail;
+	u64 size;
+	int ret;
+
+	received_ts = ktime_get_ns();
+
+	size = is_partial ? sizeof(*pexec) : sizeof(*exec);
+
+	n = (unsigned long)size * args->hdr.count;
+	if (args->hdr.count == 0 || n / args->hdr.count != size)
+		return -EINVAL;
+
+	user_data = u64_to_user_ptr(args->data);
+
+	exec = kcalloc(args->hdr.count, size, GFP_KERNEL);
+	pexec = (struct qaic_partial_execute_entry *)exec;
+	if (!exec)
+		return -ENOMEM;
+
+	if (copy_from_user(exec, user_data, n)) {
+		ret = -EFAULT;
+		goto free_exec;
+	}
+
+	usr = file_priv->driver_priv;
+	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
+	if (!usr->qddev) {
+		ret = -ENODEV;
+		goto unlock_usr_srcu;
+	}
+
+	qdev = usr->qddev->qdev;
+	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
+	if (qdev->in_reset) {
+		ret = -ENODEV;
+		goto unlock_dev_srcu;
+	}
+
+	if (args->hdr.dbc_id >= qdev->num_dbc) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	dbc = &qdev->dbc[args->hdr.dbc_id];
+
+	rcu_id = srcu_read_lock(&dbc->ch_lock);
+	if (!dbc->usr || dbc->usr->handle != usr->handle) {
+		ret = -EPERM;
+		goto release_ch_rcu;
+	}
+
+	head = readl(dbc->dbc_base + REQHP_OFF);
+	tail = readl(dbc->dbc_base + REQTP_OFF);
+
+	if (head == U32_MAX || tail == U32_MAX) {
+		/* PCI link error */
+		ret = -ENODEV;
+		goto release_ch_rcu;
+	}
+
+	queue_level = head <= tail ? tail - head : dbc->nelem - (head - tail);
+
+	ret = send_bo_list_to_device(qdev, file_priv, exec, args->hdr.count, is_partial, dbc,
+				     head, &tail);
+	if (ret)
+		goto release_ch_rcu;
+
+	/* Finalize commit to hardware */
+	submit_ts = ktime_get_ns();
+	writel(tail, dbc->dbc_base + REQTP_OFF);
+
+	update_profiling_data(file_priv, exec, args->hdr.count, is_partial, received_ts,
+			      submit_ts, queue_level);
+
+	if (datapath_polling)
+		schedule_work(&dbc->poll_work);
+
+release_ch_rcu:
+	srcu_read_unlock(&dbc->ch_lock, rcu_id);
+unlock_dev_srcu:
+	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+unlock_usr_srcu:
+	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+free_exec:
+	kfree(exec);
+	return ret;
+}
+
+int qaic_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	return __qaic_execute_bo_ioctl(dev, data, file_priv, false);
+}
+
+int qaic_partial_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	return __qaic_execute_bo_ioctl(dev, data, file_priv, true);
+}
+
+/*
+ * Our interrupt handling is a bit more complicated than a simple ideal, but
+ * sadly necessary.
+ *
+ * Each dbc has a completion queue. Entries in the queue correspond to DMA
+ * requests which the device has processed. The hardware already has a built
+ * in irq mitigation. When the device puts an entry into the queue, it will
+ * only trigger an interrupt if the queue was empty. Therefore, when adding
+ * the Nth event to a non-empty queue, the hardware doesn't trigger an
+ * interrupt. This means the host doesn't get additional interrupts signaling
+ * the same thing - the queue has something to process.
+ * This behavior can be overridden in the DMA request.
+ * This means that when the host receives an interrupt, it is required to
+ * drain the queue.
+ *
+ * This behavior is what NAPI attempts to accomplish, although we can't use
+ * NAPI as we don't have a netdev. We use threaded irqs instead.
+ *
+ * However, there is a situation where the host drains the queue fast enough
+ * that every event causes an interrupt. Typically this is not a problem as
+ * the rate of events would be low. However, that is not the case with
+ * lprnet for example. On an Intel Xeon D-2191 where we run 8 instances of
+ * lprnet, the host receives roughly 80k interrupts per second from the device
+ * (per /proc/interrupts). While NAPI documentation indicates the host should
+ * just chug along, sadly that behavior causes instability in some hosts.
+ *
+ * Therefore, we implement an interrupt disable scheme similar to NAPI. The
+ * key difference is that we will delay after draining the queue for a small
+ * time to allow additional events to come in via polling. Using the above
+ * lprnet workload, this reduces the number of interrupts processed from
+ * ~80k/sec to about 64 in 5 minutes and appears to solve the system
+ * instability.
+ */
+irqreturn_t dbc_irq_handler(int irq, void *data)
+{
+	struct dma_bridge_chan *dbc = data;
+	int rcu_id;
+	u32 head;
+	u32 tail;
+
+	rcu_id = srcu_read_lock(&dbc->ch_lock);
+
+	if (!dbc->usr) {
+		srcu_read_unlock(&dbc->ch_lock, rcu_id);
+		return IRQ_HANDLED;
+	}
+
+	head = readl(dbc->dbc_base + RSPHP_OFF);
+	if (head == U32_MAX) { /* PCI link error */
+		srcu_read_unlock(&dbc->ch_lock, rcu_id);
+		return IRQ_NONE;
+	}
+
+	tail = readl(dbc->dbc_base + RSPTP_OFF);
+	if (tail == U32_MAX) { /* PCI link error */
+		srcu_read_unlock(&dbc->ch_lock, rcu_id);
+		return IRQ_NONE;
+	}
+
+	if (head == tail) { /* queue empty */
+		srcu_read_unlock(&dbc->ch_lock, rcu_id);
+		return IRQ_NONE;
+	}
+
+	disable_irq_nosync(irq);
+	srcu_read_unlock(&dbc->ch_lock, rcu_id);
+	return IRQ_WAKE_THREAD;
+}
+
+void irq_polling_work(struct work_struct *work)
+{
+	struct dma_bridge_chan *dbc = container_of(work, struct dma_bridge_chan,  poll_work);
+	unsigned long flags;
+	int rcu_id;
+	u32 head;
+	u32 tail;
+
+	rcu_id = srcu_read_lock(&dbc->ch_lock);
+
+	while (1) {
+		if (dbc->qdev->in_reset) {
+			srcu_read_unlock(&dbc->ch_lock, rcu_id);
+			return;
+		}
+		if (!dbc->usr) {
+			srcu_read_unlock(&dbc->ch_lock, rcu_id);
+			return;
+		}
+		spin_lock_irqsave(&dbc->xfer_lock, flags);
+		if (list_empty(&dbc->xfer_list)) {
+			spin_unlock_irqrestore(&dbc->xfer_lock, flags);
+			srcu_read_unlock(&dbc->ch_lock, rcu_id);
+			return;
+		}
+		spin_unlock_irqrestore(&dbc->xfer_lock, flags);
+
+		head = readl(dbc->dbc_base + RSPHP_OFF);
+		if (head == U32_MAX) { /* PCI link error */
+			srcu_read_unlock(&dbc->ch_lock, rcu_id);
+			return;
+		}
+
+		tail = readl(dbc->dbc_base + RSPTP_OFF);
+		if (tail == U32_MAX) { /* PCI link error */
+			srcu_read_unlock(&dbc->ch_lock, rcu_id);
+			return;
+		}
+
+		if (head != tail) {
+			irq_wake_thread(dbc->irq, dbc);
+			srcu_read_unlock(&dbc->ch_lock, rcu_id);
+			return;
+		}
+
+		cond_resched();
+		usleep_range(datapath_poll_interval_us, 2 * datapath_poll_interval_us);
+	}
+}
+
+irqreturn_t dbc_irq_threaded_fn(int irq, void *data)
+{
+	struct dma_bridge_chan *dbc = data;
+	int event_count = NUM_EVENTS;
+	int delay_count = NUM_DELAYS;
+	struct qaic_device *qdev;
+	struct qaic_bo *bo, *i;
+	struct dbc_rsp *rsp;
+	unsigned long flags;
+	int rcu_id;
+	u16 status;
+	u16 req_id;
+	u32 head;
+	u32 tail;
+
+	rcu_id = srcu_read_lock(&dbc->ch_lock);
+
+	head = readl(dbc->dbc_base + RSPHP_OFF);
+	if (head == U32_MAX) /* PCI link error */
+		goto error_out;
+
+	qdev = dbc->qdev;
+read_fifo:
+
+	if (!event_count) {
+		event_count = NUM_EVENTS;
+		cond_resched();
+	}
+
+	/*
+	 * if this channel isn't assigned or gets unassigned during processing
+	 * we have nothing further to do
+	 */
+	if (!dbc->usr)
+		goto error_out;
+
+	tail = readl(dbc->dbc_base + RSPTP_OFF);
+	if (tail == U32_MAX) /* PCI link error */
+		goto error_out;
+
+	if (head == tail) { /* queue empty */
+		if (delay_count) {
+			--delay_count;
+			usleep_range(100, 200);
+			goto read_fifo; /* check for a new event */
+		}
+		goto normal_out;
+	}
+
+	delay_count = NUM_DELAYS;
+	while (head != tail) {
+		if (!event_count)
+			break;
+		--event_count;
+		rsp = dbc->rsp_q_base + head * sizeof(*rsp);
+		req_id = le16_to_cpu(rsp->req_id);
+		status = le16_to_cpu(rsp->status);
+		if (status)
+			pci_dbg(qdev->pdev, "req_id %d failed with status %d\n", req_id, status);
+		spin_lock_irqsave(&dbc->xfer_lock, flags);
+		/*
+		 * A BO can receive multiple interrupts, since a BO can be
+		 * divided into multiple slices and a buffer receives as many
+		 * interrupts as slices. So until it receives interrupts for
+		 * all the slices we cannot mark that buffer complete.
+		 */
+		list_for_each_entry_safe(bo, i, &dbc->xfer_list, xfer_list) {
+			if (bo->req_id == req_id)
+				bo->nr_slice_xfer_done++;
+			else
+				continue;
+
+			if (bo->nr_slice_xfer_done < bo->nr_slice)
+				break;
+
+			/*
+			 * At this point we have received all the interrupts for
+			 * BO, which means BO execution is complete.
+			 */
+			dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir);
+			bo->nr_slice_xfer_done = 0;
+			bo->queued = false;
+			list_del(&bo->xfer_list);
+			bo->perf_stats.req_processed_ts = ktime_get_ns();
+			complete_all(&bo->xfer_done);
+			drm_gem_object_put(&bo->base);
+			break;
+		}
+		spin_unlock_irqrestore(&dbc->xfer_lock, flags);
+		head = (head + 1) % dbc->nelem;
+	}
+
+	/*
+	 * Update the head pointer of response queue and let the device know
+	 * that we have consumed elements from the queue.
+	 */
+	writel(head, dbc->dbc_base + RSPHP_OFF);
+
+	/* elements might have been put in the queue while we were processing */
+	goto read_fifo;
+
+normal_out:
+	if (likely(!datapath_polling))
+		enable_irq(irq);
+	else
+		schedule_work(&dbc->poll_work);
+	/* checking the fifo and enabling irqs is a race, missed event check */
+	tail = readl(dbc->dbc_base + RSPTP_OFF);
+	if (tail != U32_MAX && head != tail) {
+		if (likely(!datapath_polling))
+			disable_irq_nosync(irq);
+		goto read_fifo;
+	}
+	srcu_read_unlock(&dbc->ch_lock, rcu_id);
+	return IRQ_HANDLED;
+
+error_out:
+	srcu_read_unlock(&dbc->ch_lock, rcu_id);
+	if (likely(!datapath_polling))
+		enable_irq(irq);
+	else
+		schedule_work(&dbc->poll_work);
+
+	return IRQ_HANDLED;
+}
+
+int qaic_wait_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	struct qaic_wait *args = data;
+	int usr_rcu_id, qdev_rcu_id;
+	struct dma_bridge_chan *dbc;
+	struct drm_gem_object *obj;
+	struct qaic_device *qdev;
+	unsigned long timeout;
+	struct qaic_user *usr;
+	struct qaic_bo *bo;
+	int rcu_id;
+	int ret;
+
+	usr = file_priv->driver_priv;
+	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
+	if (!usr->qddev) {
+		ret = -ENODEV;
+		goto unlock_usr_srcu;
+	}
+
+	qdev = usr->qddev->qdev;
+	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
+	if (qdev->in_reset) {
+		ret = -ENODEV;
+		goto unlock_dev_srcu;
+	}
+
+	if (args->pad != 0) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	if (args->dbc_id >= qdev->num_dbc) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	dbc = &qdev->dbc[args->dbc_id];
+
+	rcu_id = srcu_read_lock(&dbc->ch_lock);
+	if (dbc->usr != usr) {
+		ret = -EPERM;
+		goto unlock_ch_srcu;
+	}
+
+	obj = drm_gem_object_lookup(file_priv, args->handle);
+	if (!obj) {
+		ret = -ENOENT;
+		goto unlock_ch_srcu;
+	}
+
+	bo = to_qaic_bo(obj);
+	timeout = args->timeout ? args->timeout : wait_exec_default_timeout_ms;
+	timeout = msecs_to_jiffies(timeout);
+	ret = wait_for_completion_interruptible_timeout(&bo->xfer_done, timeout);
+	if (!ret) {
+		ret = -ETIMEDOUT;
+		goto put_obj;
+	}
+	if (ret > 0)
+		ret = 0;
+
+	if (!dbc->usr)
+		ret = -EPERM;
+
+put_obj:
+	drm_gem_object_put(obj);
+unlock_ch_srcu:
+	srcu_read_unlock(&dbc->ch_lock, rcu_id);
+unlock_dev_srcu:
+	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+unlock_usr_srcu:
+	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+	return ret;
+}
+
+int qaic_perf_stats_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	struct qaic_perf_stats_entry *ent = NULL;
+	struct qaic_perf_stats *args = data;
+	int usr_rcu_id, qdev_rcu_id;
+	struct drm_gem_object *obj;
+	struct qaic_device *qdev;
+	struct qaic_user *usr;
+	struct qaic_bo *bo;
+	int ret, i;
+
+	usr = file_priv->driver_priv;
+	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
+	if (!usr->qddev) {
+		ret = -ENODEV;
+		goto unlock_usr_srcu;
+	}
+
+	qdev = usr->qddev->qdev;
+	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
+	if (qdev->in_reset) {
+		ret = -ENODEV;
+		goto unlock_dev_srcu;
+	}
+
+	if (args->hdr.dbc_id >= qdev->num_dbc) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	ent = kcalloc(args->hdr.count, sizeof(*ent), GFP_KERNEL);
+	if (!ent) {
+		ret = -EINVAL;
+		goto unlock_dev_srcu;
+	}
+
+	ret = copy_from_user(ent, u64_to_user_ptr(args->data), args->hdr.count * sizeof(*ent));
+	if (ret) {
+		ret = -EFAULT;
+		goto free_ent;
+	}
+
+	for (i = 0; i < args->hdr.count; i++) {
+		obj = drm_gem_object_lookup(file_priv, ent[i].handle);
+		if (!obj) {
+			ret = -ENOENT;
+			goto free_ent;
+		}
+		bo = to_qaic_bo(obj);
+		/*
+		 * perf stats ioctl is called before wait ioctl is complete then
+		 * the latency information is invalid.
+		 */
+		if (bo->perf_stats.req_processed_ts < bo->perf_stats.req_submit_ts) {
+			ent[i].device_latency_us = 0;
+		} else {
+			ent[i].device_latency_us = div_u64((bo->perf_stats.req_processed_ts -
+							    bo->perf_stats.req_submit_ts), 1000);
+		}
+		ent[i].submit_latency_us = div_u64((bo->perf_stats.req_submit_ts -
+						    bo->perf_stats.req_received_ts), 1000);
+		ent[i].queue_level_before = bo->perf_stats.queue_level_before;
+		ent[i].num_queue_element = bo->total_slice_nents;
+		drm_gem_object_put(obj);
+	}
+
+	if (copy_to_user(u64_to_user_ptr(args->data), ent, args->hdr.count * sizeof(*ent)))
+		ret = -EFAULT;
+
+free_ent:
+	kfree(ent);
+unlock_dev_srcu:
+	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+unlock_usr_srcu:
+	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+	return ret;
+}
+
+static void empty_xfer_list(struct qaic_device *qdev, struct dma_bridge_chan *dbc)
+{
+	unsigned long flags;
+	struct qaic_bo *bo;
+
+	spin_lock_irqsave(&dbc->xfer_lock, flags);
+	while (!list_empty(&dbc->xfer_list)) {
+		bo = list_first_entry(&dbc->xfer_list, typeof(*bo), xfer_list);
+		bo->queued = false;
+		list_del(&bo->xfer_list);
+		spin_unlock_irqrestore(&dbc->xfer_lock, flags);
+		dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir);
+		complete_all(&bo->xfer_done);
+		drm_gem_object_put(&bo->base);
+		spin_lock_irqsave(&dbc->xfer_lock, flags);
+	}
+	spin_unlock_irqrestore(&dbc->xfer_lock, flags);
+}
+
+int disable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr)
+{
+	if (!qdev->dbc[dbc_id].usr || qdev->dbc[dbc_id].usr->handle != usr->handle)
+		return -EPERM;
+
+	qdev->dbc[dbc_id].usr = NULL;
+	synchronize_srcu(&qdev->dbc[dbc_id].ch_lock);
+	return 0;
+}
+
+/**
+ * enable_dbc - Enable the DBC. DBCs are disabled by removing the context of
+ * user. Add user context back to DBC to enable it. This function trusts the
+ * DBC ID passed and expects the DBC to be disabled.
+ * @qdev: Qranium device handle
+ * @dbc_id: ID of the DBC
+ * @usr: User context
+ */
+void enable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr)
+{
+	qdev->dbc[dbc_id].usr = usr;
+}
+
+void wakeup_dbc(struct qaic_device *qdev, u32 dbc_id)
+{
+	struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];
+
+	dbc->usr = NULL;
+	empty_xfer_list(qdev, dbc);
+	synchronize_srcu(&dbc->ch_lock);
+}
+
+void release_dbc(struct qaic_device *qdev, u32 dbc_id)
+{
+	struct bo_slice *slice, *slice_temp;
+	struct qaic_bo *bo, *bo_temp;
+	struct dma_bridge_chan *dbc;
+
+	dbc = &qdev->dbc[dbc_id];
+	if (!dbc->in_use)
+		return;
+
+	wakeup_dbc(qdev, dbc_id);
+
+	dma_free_coherent(&qdev->pdev->dev, dbc->total_size, dbc->req_q_base, dbc->dma_addr);
+	dbc->total_size = 0;
+	dbc->req_q_base = NULL;
+	dbc->dma_addr = 0;
+	dbc->nelem = 0;
+	dbc->usr = NULL;
+
+	list_for_each_entry_safe(bo, bo_temp, &dbc->bo_lists, bo_list) {
+		list_for_each_entry_safe(slice, slice_temp, &bo->slices, slice)
+			kref_put(&slice->ref_count, free_slice);
+		bo->sliced = false;
+		INIT_LIST_HEAD(&bo->slices);
+		bo->total_slice_nents = 0;
+		bo->dir = 0;
+		bo->dbc = NULL;
+		bo->nr_slice = 0;
+		bo->nr_slice_xfer_done = 0;
+		bo->queued = false;
+		bo->req_id = 0;
+		init_completion(&bo->xfer_done);
+		complete_all(&bo->xfer_done);
+		list_del(&bo->bo_list);
+		bo->perf_stats.req_received_ts = 0;
+		bo->perf_stats.req_submit_ts = 0;
+		bo->perf_stats.req_processed_ts = 0;
+		bo->perf_stats.queue_level_before = 0;
+	}
+
+	dbc->in_use = false;
+	wake_up(&dbc->dbc_release);
+}
diff --git a/drivers/accel/qaic/qaic_drv.c b/drivers/accel/qaic/qaic_drv.c
new file mode 100644
index 000000000000..ff80eb571729
--- /dev/null
+++ b/drivers/accel/qaic/qaic_drv.c
@@ -0,0 +1,637 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
+/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/idr.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/kref.h>
+#include <linux/mhi.h>
+#include <linux/module.h>
+#include <linux/msi.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/wait.h>
+#include <drm/drm_accel.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_file.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_ioctl.h>
+#include <uapi/drm/qaic_accel.h>
+
+#include "mhi_controller.h"
+#include "qaic.h"
+
+MODULE_IMPORT_NS(DMA_BUF);
+
+#define PCI_DEV_AIC100			0xa100
+#define QAIC_NAME			"qaic"
+#define QAIC_DESC			"Qualcomm Cloud AI Accelerators"
+#define CNTL_MAJOR			5
+#define CNTL_MINOR			0
+
+bool datapath_polling;
+module_param(datapath_polling, bool, 0400);
+MODULE_PARM_DESC(datapath_polling, "Operate the datapath in polling mode");
+static bool link_up;
+static DEFINE_IDA(qaic_usrs);
+
+static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id);
+static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id);
+
+static void free_usr(struct kref *kref)
+{
+	struct qaic_user *usr = container_of(kref, struct qaic_user, ref_count);
+
+	cleanup_srcu_struct(&usr->qddev_lock);
+	ida_free(&qaic_usrs, usr->handle);
+	kfree(usr);
+}
+
+static int qaic_open(struct drm_device *dev, struct drm_file *file)
+{
+	struct qaic_drm_device *qddev = dev->dev_private;
+	struct qaic_device *qdev = qddev->qdev;
+	struct qaic_user *usr;
+	int rcu_id;
+	int ret;
+
+	rcu_id = srcu_read_lock(&qdev->dev_lock);
+	if (qdev->in_reset) {
+		ret = -ENODEV;
+		goto dev_unlock;
+	}
+
+	usr = kmalloc(sizeof(*usr), GFP_KERNEL);
+	if (!usr) {
+		ret = -ENOMEM;
+		goto dev_unlock;
+	}
+
+	usr->handle = ida_alloc(&qaic_usrs, GFP_KERNEL);
+	if (usr->handle < 0) {
+		ret = usr->handle;
+		goto free_usr;
+	}
+	usr->qddev = qddev;
+	atomic_set(&usr->chunk_id, 0);
+	init_srcu_struct(&usr->qddev_lock);
+	kref_init(&usr->ref_count);
+
+	ret = mutex_lock_interruptible(&qddev->users_mutex);
+	if (ret)
+		goto cleanup_usr;
+
+	list_add(&usr->node, &qddev->users);
+	mutex_unlock(&qddev->users_mutex);
+
+	file->driver_priv = usr;
+
+	srcu_read_unlock(&qdev->dev_lock, rcu_id);
+	return 0;
+
+cleanup_usr:
+	cleanup_srcu_struct(&usr->qddev_lock);
+free_usr:
+	kfree(usr);
+dev_unlock:
+	srcu_read_unlock(&qdev->dev_lock, rcu_id);
+	return ret;
+}
+
+static void qaic_postclose(struct drm_device *dev, struct drm_file *file)
+{
+	struct qaic_user *usr = file->driver_priv;
+	struct qaic_drm_device *qddev;
+	struct qaic_device *qdev;
+	int qdev_rcu_id;
+	int usr_rcu_id;
+	int i;
+
+	qddev = usr->qddev;
+	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
+	if (qddev) {
+		qdev = qddev->qdev;
+		qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
+		if (!qdev->in_reset) {
+			qaic_release_usr(qdev, usr);
+			for (i = 0; i < qdev->num_dbc; ++i)
+				if (qdev->dbc[i].usr && qdev->dbc[i].usr->handle == usr->handle)
+					release_dbc(qdev, i);
+		}
+		srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+
+		mutex_lock(&qddev->users_mutex);
+		if (!list_empty(&usr->node))
+			list_del_init(&usr->node);
+		mutex_unlock(&qddev->users_mutex);
+	}
+
+	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
+	kref_put(&usr->ref_count, free_usr);
+
+	file->driver_priv = NULL;
+}
+
+DEFINE_DRM_ACCEL_FOPS(qaic_accel_fops);
+
+static const struct drm_ioctl_desc qaic_drm_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(QAIC_MANAGE, qaic_manage_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(QAIC_CREATE_BO, qaic_create_bo_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(QAIC_MMAP_BO, qaic_mmap_bo_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(QAIC_ATTACH_SLICE_BO, qaic_attach_slice_bo_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(QAIC_EXECUTE_BO, qaic_execute_bo_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(QAIC_PARTIAL_EXECUTE_BO, qaic_partial_execute_bo_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(QAIC_WAIT_BO, qaic_wait_bo_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(QAIC_PERF_STATS_BO, qaic_perf_stats_bo_ioctl, 0),
+};
+
+static const struct drm_driver qaic_accel_driver = {
+	.driver_features	= DRIVER_GEM | DRIVER_COMPUTE_ACCEL,
+
+	.name			= QAIC_NAME,
+	.desc			= QAIC_DESC,
+	.date			= "20190618",
+
+	.fops			= &qaic_accel_fops,
+	.open			= qaic_open,
+	.postclose		= qaic_postclose,
+
+	.ioctls			= qaic_drm_ioctls,
+	.num_ioctls		= ARRAY_SIZE(qaic_drm_ioctls),
+	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
+	.gem_prime_import	= qaic_gem_prime_import,
+};
+
+static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id)
+{
+	struct qaic_drm_device *qddev;
+	struct drm_device *ddev;
+	struct device *pdev;
+	int ret;
+
+	/* Hold off implementing partitions until the uapi is determined */
+	if (partition_id != QAIC_NO_PARTITION)
+		return -EINVAL;
+
+	pdev = &qdev->pdev->dev;
+
+	qddev = kzalloc(sizeof(*qddev), GFP_KERNEL);
+	if (!qddev)
+		return -ENOMEM;
+
+	ddev = drm_dev_alloc(&qaic_accel_driver, pdev);
+	if (IS_ERR(ddev)) {
+		ret = PTR_ERR(ddev);
+		goto ddev_fail;
+	}
+
+	ddev->dev_private = qddev;
+	qddev->ddev = ddev;
+
+	qddev->qdev = qdev;
+	qddev->partition_id = partition_id;
+	INIT_LIST_HEAD(&qddev->users);
+	mutex_init(&qddev->users_mutex);
+
+	qdev->qddev = qddev;
+
+	ret = drm_dev_register(ddev, 0);
+	if (ret) {
+		pci_dbg(qdev->pdev, "%s: drm_dev_register failed %d\n", __func__, ret);
+		goto drm_reg_fail;
+	}
+
+	return 0;
+
+drm_reg_fail:
+	mutex_destroy(&qddev->users_mutex);
+	qdev->qddev = NULL;
+	drm_dev_put(ddev);
+ddev_fail:
+	kfree(qddev);
+	return ret;
+}
+
+static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id)
+{
+	struct qaic_drm_device *qddev;
+	struct qaic_user *usr;
+
+	qddev = qdev->qddev;
+
+	/*
+	 * Existing users get unresolvable errors till they close FDs.
+	 * Need to sync carefully with users calling close(). The
+	 * list of users can be modified elsewhere when the lock isn't
+	 * held here, but the sync'ing the srcu with the mutex held
+	 * could deadlock. Grab the mutex so that the list will be
+	 * unmodified. The user we get will exist as long as the
+	 * lock is held. Signal that the qcdev is going away, and
+	 * grab a reference to the user so they don't go away for
+	 * synchronize_srcu(). Then release the mutex to avoid
+	 * deadlock and make sure the user has observed the signal.
+	 * With the lock released, we cannot maintain any state of the
+	 * user list.
+	 */
+	mutex_lock(&qddev->users_mutex);
+	while (!list_empty(&qddev->users)) {
+		usr = list_first_entry(&qddev->users, struct qaic_user, node);
+		list_del_init(&usr->node);
+		kref_get(&usr->ref_count);
+		usr->qddev = NULL;
+		mutex_unlock(&qddev->users_mutex);
+		synchronize_srcu(&usr->qddev_lock);
+		kref_put(&usr->ref_count, free_usr);
+		mutex_lock(&qddev->users_mutex);
+	}
+	mutex_unlock(&qddev->users_mutex);
+
+	if (qddev->ddev) {
+		drm_dev_unregister(qddev->ddev);
+		drm_dev_put(qddev->ddev);
+	}
+
+	kfree(qddev);
+}
+
+static int qaic_mhi_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
+{
+	struct qaic_device *qdev;
+	u16 major, minor;
+	int ret;
+
+	/*
+	 * Invoking this function indicates that the control channel to the
+	 * device is available. We use that as a signal to indicate that
+	 * the device side firmware has booted. The device side firmware
+	 * manages the device resources, so we need to communicate with it
+	 * via the control channel in order to utilize the device. Therefore
+	 * we wait until this signal to create the drm dev that userspace will
+	 * use to control the device, because without the device side firmware,
+	 * userspace can't do anything useful.
+	 */
+
+	qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
+
+	qdev->in_reset = false;
+
+	dev_set_drvdata(&mhi_dev->dev, qdev);
+	qdev->cntl_ch = mhi_dev;
+
+	ret = qaic_control_open(qdev);
+	if (ret) {
+		pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret);
+		return ret;
+	}
+
+	ret = get_cntl_version(qdev, NULL, &major, &minor);
+	if (ret || major != CNTL_MAJOR || minor > CNTL_MINOR) {
+		pci_err(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n",
+			__func__, major, minor, CNTL_MAJOR, CNTL_MINOR, ret);
+		ret = -EINVAL;
+		goto close_control;
+	}
+
+	ret = qaic_create_drm_device(qdev, QAIC_NO_PARTITION);
+
+	return ret;
+
+close_control:
+	qaic_control_close(qdev);
+	return ret;
+}
+
+static void qaic_mhi_remove(struct mhi_device *mhi_dev)
+{
+/* This is redundant since we have already observed the device crash */
+}
+
+static void qaic_notify_reset(struct qaic_device *qdev)
+{
+	int i;
+
+	qdev->in_reset = true;
+	/* wake up any waiters to avoid waiting for timeouts at sync */
+	wake_all_cntl(qdev);
+	for (i = 0; i < qdev->num_dbc; ++i)
+		wakeup_dbc(qdev, i);
+	synchronize_srcu(&qdev->dev_lock);
+}
+
+void qaic_dev_reset_clean_local_state(struct qaic_device *qdev, bool exit_reset)
+{
+	int i;
+
+	qaic_notify_reset(qdev);
+
+	/* remove drmdevs to prevent new users from coming in */
+	qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
+
+	/* start tearing things down */
+	for (i = 0; i < qdev->num_dbc; ++i)
+		release_dbc(qdev, i);
+
+	if (exit_reset)
+		qdev->in_reset = false;
+}
+
+static struct qaic_device *create_qdev(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct qaic_device *qdev;
+	int i;
+
+	qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL);
+	if (!qdev)
+		return NULL;
+
+	if (id->device == PCI_DEV_AIC100) {
+		qdev->num_dbc = 16;
+		qdev->dbc = devm_kcalloc(&pdev->dev, qdev->num_dbc, sizeof(*qdev->dbc), GFP_KERNEL);
+		if (!qdev->dbc)
+			return NULL;
+	}
+
+	qdev->cntl_wq = alloc_workqueue("qaic_cntl", WQ_UNBOUND, 0);
+	if (!qdev->cntl_wq)
+		return NULL;
+
+	pci_set_drvdata(pdev, qdev);
+	qdev->pdev = pdev;
+
+	mutex_init(&qdev->cntl_mutex);
+	INIT_LIST_HEAD(&qdev->cntl_xfer_list);
+	init_srcu_struct(&qdev->dev_lock);
+
+	for (i = 0; i < qdev->num_dbc; ++i) {
+		spin_lock_init(&qdev->dbc[i].xfer_lock);
+		qdev->dbc[i].qdev = qdev;
+		qdev->dbc[i].id = i;
+		INIT_LIST_HEAD(&qdev->dbc[i].xfer_list);
+		init_srcu_struct(&qdev->dbc[i].ch_lock);
+		init_waitqueue_head(&qdev->dbc[i].dbc_release);
+		INIT_LIST_HEAD(&qdev->dbc[i].bo_lists);
+	}
+
+	return qdev;
+}
+
+static void cleanup_qdev(struct qaic_device *qdev)
+{
+	int i;
+
+	for (i = 0; i < qdev->num_dbc; ++i)
+		cleanup_srcu_struct(&qdev->dbc[i].ch_lock);
+	cleanup_srcu_struct(&qdev->dev_lock);
+	pci_set_drvdata(qdev->pdev, NULL);
+	destroy_workqueue(qdev->cntl_wq);
+}
+
+static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev)
+{
+	int bars;
+	int ret;
+
+	bars = pci_select_bars(pdev, IORESOURCE_MEM);
+
+	/* make sure the device has the expected BARs */
+	if (bars != (BIT(0) | BIT(2) | BIT(4))) {
+		pci_dbg(pdev, "%s: expected BARs 0, 2, and 4 not found in device. Found 0x%x\n",
+			__func__, bars);
+		return -EINVAL;
+	}
+
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return ret;
+
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret)
+		return ret;
+	ret = dma_set_max_seg_size(&pdev->dev, UINT_MAX);
+	if (ret)
+		return ret;
+
+	qdev->bar_0 = devm_ioremap_resource(&pdev->dev, &pdev->resource[0]);
+	if (IS_ERR(qdev->bar_0))
+		return PTR_ERR(qdev->bar_0);
+
+	qdev->bar_2 = devm_ioremap_resource(&pdev->dev, &pdev->resource[2]);
+	if (IS_ERR(qdev->bar_2))
+		return PTR_ERR(qdev->bar_2);
+
+	/* Managed release since we use pcim_enable_device above */
+	pci_set_master(pdev);
+
+	return 0;
+}
+
+static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
+{
+	int mhi_irq;
+	int ret;
+	int i;
+
+	/* Managed release since we use pcim_enable_device */
+	ret = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
+	if (ret < 0)
+		return ret;
+
+	if (ret < 32) {
+		pci_err(pdev, "%s: Requested 32 MSIs. Obtained %d MSIs which is less than the 32 required.\n",
+			__func__, ret);
+		return -ENODEV;
+	}
+
+	mhi_irq = pci_irq_vector(pdev, 0);
+	if (mhi_irq < 0)
+		return mhi_irq;
+
+	for (i = 0; i < qdev->num_dbc; ++i) {
+		ret = devm_request_threaded_irq(&pdev->dev, pci_irq_vector(pdev, i + 1),
+						dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED,
+						"qaic_dbc", &qdev->dbc[i]);
+		if (ret)
+			return ret;
+
+		if (datapath_polling) {
+			qdev->dbc[i].irq = pci_irq_vector(pdev, i + 1);
+			disable_irq_nosync(qdev->dbc[i].irq);
+			INIT_WORK(&qdev->dbc[i].poll_work, irq_polling_work);
+		}
+	}
+
+	return mhi_irq;
+}
+
+static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct qaic_device *qdev;
+	int mhi_irq;
+	int ret;
+	int i;
+
+	qdev = create_qdev(pdev, id);
+	if (!qdev)
+		return -ENOMEM;
+
+	ret = init_pci(qdev, pdev);
+	if (ret)
+		goto cleanup_qdev;
+
+	for (i = 0; i < qdev->num_dbc; ++i)
+		qdev->dbc[i].dbc_base = qdev->bar_2 + QAIC_DBC_OFF(i);
+
+	mhi_irq = init_msi(qdev, pdev);
+	if (mhi_irq < 0) {
+		ret = mhi_irq;
+		goto cleanup_qdev;
+	}
+
+	qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_0, mhi_irq);
+	if (IS_ERR(qdev->mhi_cntrl)) {
+		ret = PTR_ERR(qdev->mhi_cntrl);
+		goto cleanup_qdev;
+	}
+
+	return 0;
+
+cleanup_qdev:
+	cleanup_qdev(qdev);
+	return ret;
+}
+
+static void qaic_pci_remove(struct pci_dev *pdev)
+{
+	struct qaic_device *qdev = pci_get_drvdata(pdev);
+
+	if (!qdev)
+		return;
+
+	qaic_dev_reset_clean_local_state(qdev, false);
+	qaic_mhi_free_controller(qdev->mhi_cntrl, link_up);
+	cleanup_qdev(qdev);
+}
+
+static void qaic_pci_shutdown(struct pci_dev *pdev)
+{
+	/* see qaic_exit for what link_up is doing */
+	link_up = true;
+	qaic_pci_remove(pdev);
+}
+
+static pci_ers_result_t qaic_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error)
+{
+	return PCI_ERS_RESULT_NEED_RESET;
+}
+
+static void qaic_pci_reset_prepare(struct pci_dev *pdev)
+{
+	struct qaic_device *qdev = pci_get_drvdata(pdev);
+
+	qaic_notify_reset(qdev);
+	qaic_mhi_start_reset(qdev->mhi_cntrl);
+	qaic_dev_reset_clean_local_state(qdev, false);
+}
+
+static void qaic_pci_reset_done(struct pci_dev *pdev)
+{
+	struct qaic_device *qdev = pci_get_drvdata(pdev);
+
+	qdev->in_reset = false;
+	qaic_mhi_reset_done(qdev->mhi_cntrl);
+}
+
+static const struct mhi_device_id qaic_mhi_match_table[] = {
+	{ .chan = "QAIC_CONTROL", },
+	{},
+};
+
+static struct mhi_driver qaic_mhi_driver = {
+	.id_table = qaic_mhi_match_table,
+	.remove = qaic_mhi_remove,
+	.probe = qaic_mhi_probe,
+	.ul_xfer_cb = qaic_mhi_ul_xfer_cb,
+	.dl_xfer_cb = qaic_mhi_dl_xfer_cb,
+	.driver = {
+		.name = "qaic_mhi",
+	},
+};
+
+static const struct pci_device_id qaic_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, PCI_DEV_AIC100), },
+	{ }
+};
+MODULE_DEVICE_TABLE(pci, qaic_ids);
+
+static const struct pci_error_handlers qaic_pci_err_handler = {
+	.error_detected = qaic_pci_error_detected,
+	.reset_prepare = qaic_pci_reset_prepare,
+	.reset_done = qaic_pci_reset_done,
+};
+
+static struct pci_driver qaic_pci_driver = {
+	.name = QAIC_NAME,
+	.id_table = qaic_ids,
+	.probe = qaic_pci_probe,
+	.remove = qaic_pci_remove,
+	.shutdown = qaic_pci_shutdown,
+	.err_handler = &qaic_pci_err_handler,
+};
+
+static int __init qaic_init(void)
+{
+	int ret;
+
+	ret = mhi_driver_register(&qaic_mhi_driver);
+	if (ret) {
+		pr_debug("qaic: mhi_driver_register failed %d\n", ret);
+		return ret;
+	}
+
+	ret = pci_register_driver(&qaic_pci_driver);
+	if (ret) {
+		pr_debug("qaic: pci_register_driver failed %d\n", ret);
+		goto free_mhi;
+	}
+
+	return 0;
+
+free_mhi:
+	mhi_driver_unregister(&qaic_mhi_driver);
+	return ret;
+}
+
+static void __exit qaic_exit(void)
+{
+	/*
+	 * We assume that qaic_pci_remove() is called due to a hotplug event
+	 * which would mean that the link is down, and thus
+	 * qaic_mhi_free_controller() should not try to access the device during
+	 * cleanup.
+	 * We call pci_unregister_driver() below, which also triggers
+	 * qaic_pci_remove(), but since this is module exit, we expect the link
+	 * to the device to be up, in which case qaic_mhi_free_controller()
+	 * should try to access the device during cleanup to put the device in
+	 * a sane state.
+	 * For that reason, we set link_up here to let qaic_mhi_free_controller
+	 * know the expected link state. Since the module is going to be
+	 * removed at the end of this, we don't need to worry about
+	 * reinitializing the link_up state after the cleanup is done.
+	 */
+	link_up = true;
+	pci_unregister_driver(&qaic_pci_driver);
+	mhi_driver_unregister(&qaic_mhi_driver);
+}
+
+module_init(qaic_init);
+module_exit(qaic_exit);
+
+MODULE_AUTHOR(QAIC_DESC " Kernel Driver Team");
+MODULE_DESCRIPTION(QAIC_DESC " Accel Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/accessibility/braille/braille_console.c b/drivers/accessibility/braille/braille_console.c
index c4d54a5326b1..06b43b678d6e 100644
--- a/drivers/accessibility/braille/braille_console.c
+++ b/drivers/accessibility/braille/braille_console.c
@@ -24,7 +24,6 @@
 
 MODULE_AUTHOR("samuel.thibault@ens-lyon.org");
 MODULE_DESCRIPTION("braille device");
-MODULE_LICENSE("GPL");
 
 /*
  * Braille device support part.
diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c
index 3bbe2276cac7..80f945cbec8a 100644
--- a/drivers/acpi/acpi_apd.c
+++ b/drivers/acpi/acpi_apd.c
@@ -83,6 +83,8 @@ static int fch_misc_setup(struct apd_private_data *pdata)
 	if (!acpi_dev_get_property(adev, "clk-name", ACPI_TYPE_STRING, &obj)) {
 		clk_data->name = devm_kzalloc(&adev->dev, obj->string.length,
 					      GFP_KERNEL);
+		if (!clk_data->name)
+			return -ENOMEM;
 
 		strcpy(clk_data->name, obj->string.pointer);
 	} else {
diff --git a/drivers/acpi/acpi_lpit.c b/drivers/acpi/acpi_lpit.c
index 3843d2576d3f..c5598b6d5db8 100644
--- a/drivers/acpi/acpi_lpit.c
+++ b/drivers/acpi/acpi_lpit.c
@@ -98,6 +98,12 @@ EXPORT_SYMBOL_GPL(lpit_read_residency_count_address);
 static void lpit_update_residency(struct lpit_residency_info *info,
 				 struct acpi_lpit_native *lpit_native)
 {
+	struct device *dev_root = bus_get_dev_root(&cpu_subsys);
+
+	/* Silently fail, if cpuidle attribute group is not present */
+	if (!dev_root)
+		return;
+
 	info->frequency = lpit_native->counter_frequency ?
 				lpit_native->counter_frequency : tsc_khz * 1000;
 	if (!info->frequency)
@@ -108,18 +114,18 @@ static void lpit_update_residency(struct lpit_residency_info *info,
 		info->iomem_addr = ioremap(info->gaddr.address,
 						   info->gaddr.bit_width / 8);
 		if (!info->iomem_addr)
-			return;
+			goto exit;
 
-		/* Silently fail, if cpuidle attribute group is not present */
-		sysfs_add_file_to_group(&cpu_subsys.dev_root->kobj,
+		sysfs_add_file_to_group(&dev_root->kobj,
 					&dev_attr_low_power_idle_system_residency_us.attr,
 					"cpuidle");
 	} else if (info->gaddr.space_id == ACPI_ADR_SPACE_FIXED_HARDWARE) {
-		/* Silently fail, if cpuidle attribute group is not present */
-		sysfs_add_file_to_group(&cpu_subsys.dev_root->kobj,
+		sysfs_add_file_to_group(&dev_root->kobj,
 					&dev_attr_low_power_idle_cpu_residency_us.attr,
 					"cpuidle");
 	}
+exit:
+	put_device(dev_root);
 }
 
 static void lpit_process(u64 begin, u64 end)
diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c
index f08ffa75f4a7..77186f084d3a 100644
--- a/drivers/acpi/acpi_lpss.c
+++ b/drivers/acpi/acpi_lpss.c
@@ -271,6 +271,12 @@ static const struct lpss_device_desc bsw_pwm_dev_desc = {
 	.resume_from_noirq = true,
 };
 
+static const struct lpss_device_desc bsw_pwm2_dev_desc = {
+	.flags = LPSS_SAVE_CTX_ONCE | LPSS_NO_D3_DELAY,
+	.prv_offset = 0x800,
+	.resume_from_noirq = true,
+};
+
 static const struct lpss_device_desc byt_uart_dev_desc = {
 	.flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_SAVE_CTX,
 	.clk_con_id = "baudclk",
@@ -368,6 +374,7 @@ static const struct acpi_device_id acpi_lpss_device_ids[] = {
 	/* Braswell LPSS devices */
 	{ "80862286", LPSS_ADDR(lpss_dma_desc) },
 	{ "80862288", LPSS_ADDR(bsw_pwm_dev_desc) },
+	{ "80862289", LPSS_ADDR(bsw_pwm2_dev_desc) },
 	{ "8086228A", LPSS_ADDR(bsw_uart_dev_desc) },
 	{ "8086228E", LPSS_ADDR(bsw_spi_dev_desc) },
 	{ "808622C0", LPSS_ADDR(lpss_dma_desc) },
diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c
index 6737b1cbf6d6..f9aa02cac6d1 100644
--- a/drivers/acpi/acpi_processor.c
+++ b/drivers/acpi/acpi_processor.c
@@ -15,6 +15,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/platform_device.h>
 
 #include <acpi/processor.h>
 
@@ -148,6 +149,34 @@ static int acpi_processor_errata(void)
 	return result;
 }
 
+/* Create a platform device to represent a CPU frequency control mechanism. */
+static void cpufreq_add_device(const char *name)
+{
+	struct platform_device *pdev;
+
+	pdev = platform_device_register_simple(name, PLATFORM_DEVID_NONE, NULL, 0);
+	if (IS_ERR(pdev))
+		pr_info("%s device creation failed: %ld\n", name, PTR_ERR(pdev));
+}
+
+#ifdef CONFIG_X86
+/* Check presence of Processor Clocking Control by searching for \_SB.PCCH. */
+static void __init acpi_pcc_cpufreq_init(void)
+{
+	acpi_status status;
+	acpi_handle handle;
+
+	status = acpi_get_handle(NULL, "\\_SB", &handle);
+	if (ACPI_FAILURE(status))
+		return;
+
+	if (acpi_has_method(handle, "PCCH"))
+		cpufreq_add_device("pcc-cpufreq");
+}
+#else
+static void __init acpi_pcc_cpufreq_init(void) {}
+#endif /* CONFIG_X86 */
+
 /* Initialization */
 #ifdef CONFIG_ACPI_HOTPLUG_CPU
 int __weak acpi_map_cpu(acpi_handle handle,
@@ -280,14 +309,22 @@ static int acpi_processor_get_info(struct acpi_device *device)
 		dev_dbg(&device->dev, "Failed to get CPU physical ID.\n");
 
 	pr->id = acpi_map_cpuid(pr->phys_id, pr->acpi_id);
-	if (!cpu0_initialized && !acpi_has_cpu_in_madt()) {
+	if (!cpu0_initialized) {
 		cpu0_initialized = 1;
 		/*
 		 * Handle UP system running SMP kernel, with no CPU
 		 * entry in MADT
 		 */
-		if (invalid_logical_cpuid(pr->id) && (num_online_cpus() == 1))
+		if (!acpi_has_cpu_in_madt() && invalid_logical_cpuid(pr->id) &&
+		    (num_online_cpus() == 1))
 			pr->id = 0;
+		/*
+		 * Check availability of Processor Performance Control by
+		 * looking at the presence of the _PCT object under the first
+		 * processor definition.
+		 */
+		if (acpi_has_method(pr->handle, "_PCT"))
+			cpufreq_add_device("acpi-cpufreq");
 	}
 
 	/*
@@ -686,6 +723,7 @@ void __init acpi_processor_init(void)
 	acpi_processor_check_duplicates();
 	acpi_scan_add_handler_with_hotplug(&processor_handler, "processor");
 	acpi_scan_add_handler(&processor_container_handler);
+	acpi_pcc_cpufreq_init();
 }
 
 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE
diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c
index c7a6d0b69dab..62f4364e4460 100644
--- a/drivers/acpi/acpi_video.c
+++ b/drivers/acpi/acpi_video.c
@@ -70,12 +70,6 @@ module_param(device_id_scheme, bool, 0444);
 static int only_lcd = -1;
 module_param(only_lcd, int, 0444);
 
-static int register_backlight_delay;
-module_param(register_backlight_delay, int, 0444);
-MODULE_PARM_DESC(register_backlight_delay,
-	"Delay in seconds before doing fallback (non GPU driver triggered) "
-	"backlight registration, set to 0 to disable.");
-
 static bool may_report_brightness_keys;
 static int register_count;
 static DEFINE_MUTEX(register_count_mutex);
@@ -84,9 +78,6 @@ static LIST_HEAD(video_bus_head);
 static int acpi_video_bus_add(struct acpi_device *device);
 static void acpi_video_bus_remove(struct acpi_device *device);
 static void acpi_video_bus_notify(struct acpi_device *device, u32 event);
-static void acpi_video_bus_register_backlight_work(struct work_struct *ignored);
-static DECLARE_DELAYED_WORK(video_bus_register_backlight_work,
-			    acpi_video_bus_register_backlight_work);
 
 /*
  * Indices in the _BCL method response: the first two items are special,
@@ -2096,11 +2087,6 @@ static void acpi_video_bus_remove(struct acpi_device *device)
 	kfree(video);
 }
 
-static void acpi_video_bus_register_backlight_work(struct work_struct *ignored)
-{
-	acpi_video_register_backlight();
-}
-
 static int __init is_i740(struct pci_dev *dev)
 {
 	if (dev->device == 0x00D1)
@@ -2183,17 +2169,6 @@ static bool should_check_lcd_flag(void)
 	return false;
 }
 
-/*
- * At least one graphics driver has reported that no LCD is connected
- * via the native interface. cancel the registration for fallback acpi_video0.
- * If another driver still deems this necessary, it can explicitly register it.
- */
-void acpi_video_report_nolcd(void)
-{
-	cancel_delayed_work(&video_bus_register_backlight_work);
-}
-EXPORT_SYMBOL(acpi_video_report_nolcd);
-
 int acpi_video_register(void)
 {
 	int ret = 0;
@@ -2222,18 +2197,6 @@ int acpi_video_register(void)
 	 */
 	register_count = 1;
 
-	/*
-	 * acpi_video_bus_add() skips registering the userspace visible
-	 * backlight_device. The intend is for this to be registered by the
-	 * drm/kms driver calling acpi_video_register_backlight() *after* it is
-	 * done setting up its own native backlight device. The delayed work
-	 * ensures that acpi_video_register_backlight() always gets called
-	 * eventually, in case there is no drm/kms driver or it is disabled.
-	 */
-	if (register_backlight_delay)
-		schedule_delayed_work(&video_bus_register_backlight_work,
-				      register_backlight_delay * HZ);
-
 leave:
 	mutex_unlock(&register_count_mutex);
 	return ret;
@@ -2244,7 +2207,6 @@ void acpi_video_unregister(void)
 {
 	mutex_lock(&register_count_mutex);
 	if (register_count) {
-		cancel_delayed_work_sync(&video_bus_register_backlight_work);
 		acpi_bus_unregister_driver(&acpi_video_bus);
 		register_count = 0;
 		may_report_brightness_keys = false;
diff --git a/drivers/acpi/acpica/acapps.h b/drivers/acpi/acpica/acapps.h
index 0a50b4912515..9d4cbd956627 100644
--- a/drivers/acpi/acpica/acapps.h
+++ b/drivers/acpi/acpica/acapps.h
@@ -3,7 +3,7 @@
  *
  * Module Name: acapps - common include for ACPI applications/tools
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/accommon.h b/drivers/acpi/acpica/accommon.h
index bb329e34ee7d..4536dc9d3979 100644
--- a/drivers/acpi/acpica/accommon.h
+++ b/drivers/acpi/acpica/accommon.h
@@ -3,7 +3,7 @@
  *
  * Name: accommon.h - Common include files for generation of ACPICA source
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acconvert.h b/drivers/acpi/acpica/acconvert.h
index 476d21e67767..c6ba6a36cfb5 100644
--- a/drivers/acpi/acpica/acconvert.h
+++ b/drivers/acpi/acpica/acconvert.h
@@ -3,7 +3,7 @@
  *
  * Module Name: acapps - common include for ACPI applications/tools
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acdebug.h b/drivers/acpi/acpica/acdebug.h
index d629716aa5b2..22f1f7a9e5a3 100644
--- a/drivers/acpi/acpica/acdebug.h
+++ b/drivers/acpi/acpica/acdebug.h
@@ -3,7 +3,7 @@
  *
  * Name: acdebug.h - ACPI/AML debugger
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acdispat.h b/drivers/acpi/acpica/acdispat.h
index fe2c3630a38d..73eecbf62f06 100644
--- a/drivers/acpi/acpica/acdispat.h
+++ b/drivers/acpi/acpica/acdispat.h
@@ -3,7 +3,7 @@
  *
  * Name: acdispat.h - dispatcher (parser to interpreter interface)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acevents.h b/drivers/acpi/acpica/acevents.h
index 922f559a3e59..ddd072cbc738 100644
--- a/drivers/acpi/acpica/acevents.h
+++ b/drivers/acpi/acpica/acevents.h
@@ -3,7 +3,7 @@
  *
  * Name: acevents.h - Event subcomponent prototypes and defines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acglobal.h b/drivers/acpi/acpica/acglobal.h
index 777457a58340..778241173ed4 100644
--- a/drivers/acpi/acpica/acglobal.h
+++ b/drivers/acpi/acpica/acglobal.h
@@ -3,7 +3,7 @@
  *
  * Name: acglobal.h - Declarations for global variables
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/achware.h b/drivers/acpi/acpica/achware.h
index 6f2787506b50..ebf8fd373cf7 100644
--- a/drivers/acpi/acpica/achware.h
+++ b/drivers/acpi/acpica/achware.h
@@ -3,7 +3,7 @@
  *
  * Name: achware.h -- hardware specific interfaces
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acinterp.h b/drivers/acpi/acpica/acinterp.h
index 6bdf133a2767..955114c926bd 100644
--- a/drivers/acpi/acpica/acinterp.h
+++ b/drivers/acpi/acpica/acinterp.h
@@ -3,7 +3,7 @@
  *
  * Name: acinterp.h - Interpreter subcomponent prototypes and defines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/aclocal.h b/drivers/acpi/acpica/aclocal.h
index 901b1543b869..12d4a024f029 100644
--- a/drivers/acpi/acpica/aclocal.h
+++ b/drivers/acpi/acpica/aclocal.h
@@ -3,7 +3,7 @@
  *
  * Name: aclocal.h - Internal data types used across the ACPI subsystem
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -1122,7 +1122,8 @@ struct acpi_port_info {
 #define ACPI_RESOURCE_NAME_PIN_GROUP            0x90
 #define ACPI_RESOURCE_NAME_PIN_GROUP_FUNCTION   0x91
 #define ACPI_RESOURCE_NAME_PIN_GROUP_CONFIG     0x92
-#define ACPI_RESOURCE_NAME_LARGE_MAX            0x92
+#define ACPI_RESOURCE_NAME_CLOCK_INPUT          0x93
+#define ACPI_RESOURCE_NAME_LARGE_MAX            0x94
 
 /*****************************************************************************
  *
diff --git a/drivers/acpi/acpica/acmacros.h b/drivers/acpi/acpica/acmacros.h
index 2f3e609df47d..de83dd22292b 100644
--- a/drivers/acpi/acpica/acmacros.h
+++ b/drivers/acpi/acpica/acmacros.h
@@ -3,7 +3,7 @@
  *
  * Name: acmacros.h - C macros for the entire subsystem.
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acnamesp.h b/drivers/acpi/acpica/acnamesp.h
index 7b27b9cc5916..9448bc026b9b 100644
--- a/drivers/acpi/acpica/acnamesp.h
+++ b/drivers/acpi/acpica/acnamesp.h
@@ -3,7 +3,7 @@
  *
  * Name: acnamesp.h - Namespace subcomponent prototypes and defines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acobject.h b/drivers/acpi/acpica/acobject.h
index 6af5dc995085..1bdfeee5d7c5 100644
--- a/drivers/acpi/acpica/acobject.h
+++ b/drivers/acpi/acpica/acobject.h
@@ -3,7 +3,7 @@
  *
  * Name: acobject.h - Definition of union acpi_operand_object  (Internal object only)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acopcode.h b/drivers/acpi/acpica/acopcode.h
index a224926bd9c8..da96d80e6b3a 100644
--- a/drivers/acpi/acpica/acopcode.h
+++ b/drivers/acpi/acpica/acopcode.h
@@ -3,7 +3,7 @@
  *
  * Name: acopcode.h - AML opcode information for the AML parser and interpreter
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acparser.h b/drivers/acpi/acpica/acparser.h
index 4511c2bd8bc3..6dad786a382c 100644
--- a/drivers/acpi/acpica/acparser.h
+++ b/drivers/acpi/acpica/acparser.h
@@ -3,7 +3,7 @@
  *
  * Module Name: acparser.h - AML Parser subcomponent prototypes and defines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acpredef.h b/drivers/acpi/acpica/acpredef.h
index f7d65a20026b..e64aabe3d33a 100644
--- a/drivers/acpi/acpica/acpredef.h
+++ b/drivers/acpi/acpica/acpredef.h
@@ -3,7 +3,7 @@
  *
  * Name: acpredef - Information table for ACPI predefined methods and objects
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acresrc.h b/drivers/acpi/acpica/acresrc.h
index f7749c63d277..d772ff9ca07d 100644
--- a/drivers/acpi/acpica/acresrc.h
+++ b/drivers/acpi/acpica/acresrc.h
@@ -3,7 +3,7 @@
  *
  * Name: acresrc.h - Resource Manager function prototypes
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -306,6 +306,7 @@ extern struct acpi_rsconvert_info acpi_rs_convert_pin_config[];
 extern struct acpi_rsconvert_info acpi_rs_convert_pin_group[];
 extern struct acpi_rsconvert_info acpi_rs_convert_pin_group_function[];
 extern struct acpi_rsconvert_info acpi_rs_convert_pin_group_config[];
+extern struct acpi_rsconvert_info acpi_rs_convert_clock_input[];
 
 /* These resources require separate get/set tables */
 
@@ -361,6 +362,7 @@ extern struct acpi_rsdump_info acpi_rs_dump_pin_config[];
 extern struct acpi_rsdump_info acpi_rs_dump_pin_group[];
 extern struct acpi_rsdump_info acpi_rs_dump_pin_group_function[];
 extern struct acpi_rsdump_info acpi_rs_dump_pin_group_config[];
+extern struct acpi_rsdump_info acpi_rs_dump_clock_input[];
 #endif
 
 #endif				/* __ACRESRC_H__ */
diff --git a/drivers/acpi/acpica/acstruct.h b/drivers/acpi/acpica/acstruct.h
index b859de96a1e4..f8fee94ba708 100644
--- a/drivers/acpi/acpica/acstruct.h
+++ b/drivers/acpi/acpica/acstruct.h
@@ -3,7 +3,7 @@
  *
  * Name: acstruct.h - Internal structs
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/actables.h b/drivers/acpi/acpica/actables.h
index 1c29325e4c7f..b6ae979b01b6 100644
--- a/drivers/acpi/acpica/actables.h
+++ b/drivers/acpi/acpica/actables.h
@@ -3,7 +3,7 @@
  *
  * Name: actables.h - ACPI table management
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/acutils.h b/drivers/acpi/acpica/acutils.h
index 71175b664f49..edfdbbef81c1 100644
--- a/drivers/acpi/acpica/acutils.h
+++ b/drivers/acpi/acpica/acutils.h
@@ -3,7 +3,7 @@
  *
  * Name: acutils.h -- prototypes for the common (subsystem-wide) procedures
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -53,6 +53,8 @@ extern const char *acpi_gbl_sb_decode[];
 extern const char *acpi_gbl_fc_decode[];
 extern const char *acpi_gbl_pt_decode[];
 extern const char *acpi_gbl_ptyp_decode[];
+extern const char *acpi_gbl_clock_input_mode[];
+extern const char *acpi_gbl_clock_input_scale[];
 #endif
 
 /*
diff --git a/drivers/acpi/acpica/amlcode.h b/drivers/acpi/acpica/amlcode.h
index 62a7ec277513..effe52b40dce 100644
--- a/drivers/acpi/acpica/amlcode.h
+++ b/drivers/acpi/acpica/amlcode.h
@@ -5,7 +5,7 @@
  *                   Declarations and definitions contained herein are derived
  *                   directly from the ACPI specification.
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/amlresrc.h b/drivers/acpi/acpica/amlresrc.h
index b31779ce204a..4e88f9fc2a28 100644
--- a/drivers/acpi/acpica/amlresrc.h
+++ b/drivers/acpi/acpica/amlresrc.h
@@ -3,7 +3,7 @@
  *
  * Module Name: amlresrc.h - AML resource descriptors
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -70,6 +70,8 @@
 #define ACPI_RESTAG_TYPE                        "_TTP"	/* Translation(1), Static (0) */
 #define ACPI_RESTAG_XFERTYPE                    "_SIZ"	/* 8(0), 8And16(1), 16(2) */
 #define ACPI_RESTAG_VENDORDATA                  "_VEN"
+#define ACPI_RESTAG_FQN                         "_FQN"
+#define ACPI_RESTAG_FQD                         "_FQD"
 
 /* Default sizes for "small" resource descriptors */
 
@@ -259,7 +261,10 @@ struct aml_resource_address16 {
 struct aml_resource_extended_irq {
 	AML_RESOURCE_LARGE_HEADER_COMMON u8 flags;
 	u8 interrupt_count;
-	u32 interrupts[1];
+	union {
+		u32 interrupt;
+		 ACPI_FLEX_ARRAY(u32, interrupts);
+	};
 	/* res_source_index, res_source optional fields follow */
 };
 
@@ -427,6 +432,20 @@ struct aml_resource_pin_config {
 	 */
 };
 
+#define AML_RESOURCE_CLOCK_INPUT_REVISION      1	/* ACPI 6.5 */
+
+struct aml_resource_clock_input {
+	AML_RESOURCE_LARGE_HEADER_COMMON u8 revision_id;
+	u16 flags;
+	u16 frequency_divisor;
+	u32 frequency_numerator;
+	/*
+	 * Optional fields follow immediately:
+	 * 1) Resource Source index
+	 * 2) Resource Source String
+	 */
+};
+
 #define AML_RESOURCE_PIN_CONFIG_REVISION      1	/* ACPI 6.2 */
 
 struct aml_resource_pin_group {
@@ -533,6 +552,7 @@ union aml_resource {
 	struct aml_resource_pin_group pin_group;
 	struct aml_resource_pin_group_function pin_group_function;
 	struct aml_resource_pin_group_config pin_group_config;
+	struct aml_resource_clock_input clock_input;
 
 	/* Utility overlays */
 
diff --git a/drivers/acpi/acpica/dbhistry.c b/drivers/acpi/acpica/dbhistry.c
index 105e6ceaa887..e874c1dddefa 100644
--- a/drivers/acpi/acpica/dbhistry.c
+++ b/drivers/acpi/acpica/dbhistry.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dbhistry - debugger HISTORY command
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dbnames.c b/drivers/acpi/acpica/dbnames.c
index 3615e1a6efd8..b91155ea9c34 100644
--- a/drivers/acpi/acpica/dbnames.c
+++ b/drivers/acpi/acpica/dbnames.c
@@ -652,6 +652,9 @@ acpi_status acpi_db_display_objects(char *obj_type_arg, char *display_count_arg)
 		object_info =
 		    ACPI_ALLOCATE_ZEROED(sizeof(struct acpi_object_info));
 
+		if (!object_info)
+			return (AE_NO_MEMORY);
+
 		/* Walk the namespace from the root */
 
 		(void)acpi_walk_namespace(ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
diff --git a/drivers/acpi/acpica/dsargs.c b/drivers/acpi/acpica/dsargs.c
index 2963d1579c05..4354c175e12e 100644
--- a/drivers/acpi/acpica/dsargs.c
+++ b/drivers/acpi/acpica/dsargs.c
@@ -4,7 +4,7 @@
  * Module Name: dsargs - Support for execution of dynamic arguments for static
  *                       objects (regions, fields, buffer fields, etc.)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dscontrol.c b/drivers/acpi/acpica/dscontrol.c
index 8492619149d1..80c69af06948 100644
--- a/drivers/acpi/acpica/dscontrol.c
+++ b/drivers/acpi/acpica/dscontrol.c
@@ -4,7 +4,7 @@
  * Module Name: dscontrol - Support for execution control opcodes -
  *                          if/else/while/return
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dsdebug.c b/drivers/acpi/acpica/dsdebug.c
index 2d99ccf5bde7..c5c8380a3114 100644
--- a/drivers/acpi/acpica/dsdebug.c
+++ b/drivers/acpi/acpica/dsdebug.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dsdebug - Parser/Interpreter interface - debugging
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dsfield.c b/drivers/acpi/acpica/dsfield.c
index de175f1b4beb..532401ecdab0 100644
--- a/drivers/acpi/acpica/dsfield.c
+++ b/drivers/acpi/acpica/dsfield.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dsfield - Dispatcher field routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dsinit.c b/drivers/acpi/acpica/dsinit.c
index dffd54fdbd51..6e0e362e461f 100644
--- a/drivers/acpi/acpica/dsinit.c
+++ b/drivers/acpi/acpica/dsinit.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dsinit - Object initialization namespace walk
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dsmethod.c b/drivers/acpi/acpica/dsmethod.c
index 9332bc688713..e809c2aed78a 100644
--- a/drivers/acpi/acpica/dsmethod.c
+++ b/drivers/acpi/acpica/dsmethod.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dsmethod - Parser/Interpreter interface - control method parsing
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dsobject.c b/drivers/acpi/acpica/dsobject.c
index e3dfc734ace9..555f148d666b 100644
--- a/drivers/acpi/acpica/dsobject.c
+++ b/drivers/acpi/acpica/dsobject.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dsobject - Dispatcher object management routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dsopcode.c b/drivers/acpi/acpica/dsopcode.c
index 2b9b6a974ca9..dd3059000885 100644
--- a/drivers/acpi/acpica/dsopcode.c
+++ b/drivers/acpi/acpica/dsopcode.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dsopcode - Dispatcher support for regions and fields
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dspkginit.c b/drivers/acpi/acpica/dspkginit.c
index 1624d6e7dc46..ecf793fe9919 100644
--- a/drivers/acpi/acpica/dspkginit.c
+++ b/drivers/acpi/acpica/dspkginit.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dspkginit - Completion of deferred package initialization
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dswexec.c b/drivers/acpi/acpica/dswexec.c
index b082eb942a0f..a43336f05206 100644
--- a/drivers/acpi/acpica/dswexec.c
+++ b/drivers/acpi/acpica/dswexec.c
@@ -4,7 +4,7 @@
  * Module Name: dswexec - Dispatcher method execution callbacks;
  *                        dispatch to interpreter.
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dswload.c b/drivers/acpi/acpica/dswload.c
index 9f6573646ab5..f7b8496c8bdd 100644
--- a/drivers/acpi/acpica/dswload.c
+++ b/drivers/acpi/acpica/dswload.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dswload - Dispatcher first pass namespace load callbacks
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dswload2.c b/drivers/acpi/acpica/dswload2.c
index 778df616aaa0..541235f498c2 100644
--- a/drivers/acpi/acpica/dswload2.c
+++ b/drivers/acpi/acpica/dswload2.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dswload2 - Dispatcher second pass namespace load callbacks
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dswscope.c b/drivers/acpi/acpica/dswscope.c
index 634b9100f674..1fdd07ae862c 100644
--- a/drivers/acpi/acpica/dswscope.c
+++ b/drivers/acpi/acpica/dswscope.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dswscope - Scope stack manipulation
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/dswstate.c b/drivers/acpi/acpica/dswstate.c
index 0aa735d3b93c..d3841ded3a81 100644
--- a/drivers/acpi/acpica/dswstate.c
+++ b/drivers/acpi/acpica/dswstate.c
@@ -3,7 +3,7 @@
  *
  * Module Name: dswstate - Dispatcher parse tree walk management routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -576,9 +576,14 @@ acpi_ds_init_aml_walk(struct acpi_walk_state *walk_state,
 	ACPI_FUNCTION_TRACE(ds_init_aml_walk);
 
 	walk_state->parser_state.aml =
-	    walk_state->parser_state.aml_start = aml_start;
-	walk_state->parser_state.aml_end =
-	    walk_state->parser_state.pkg_end = aml_start + aml_length;
+	    walk_state->parser_state.aml_start =
+	    walk_state->parser_state.aml_end =
+	    walk_state->parser_state.pkg_end = aml_start;
+	/* Avoid undefined behavior: applying zero offset to null pointer */
+	if (aml_length != 0) {
+		walk_state->parser_state.aml_end += aml_length;
+		walk_state->parser_state.pkg_end += aml_length;
+	}
 
 	/* The next_op of the next_walk will be the beginning of the method */
 
diff --git a/drivers/acpi/acpica/evevent.c b/drivers/acpi/acpica/evevent.c
index 82d1728b9bc6..9e78c5b9ad52 100644
--- a/drivers/acpi/acpica/evevent.c
+++ b/drivers/acpi/acpica/evevent.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evevent - Fixed Event handling and dispatch
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -142,9 +142,6 @@ static acpi_status acpi_ev_fixed_event_initialize(void)
 			status =
 			    acpi_write_bit_register(acpi_gbl_fixed_event_info
 						    [i].enable_register_id,
-						    (i ==
-						     ACPI_EVENT_PCIE_WAKE) ?
-						    ACPI_ENABLE_EVENT :
 						    ACPI_DISABLE_EVENT);
 			if (ACPI_FAILURE(status)) {
 				return (status);
@@ -188,11 +185,6 @@ u32 acpi_ev_fixed_event_detect(void)
 		return (int_status);
 	}
 
-	if (fixed_enable & ACPI_BITMASK_PCIEXP_WAKE_DISABLE)
-		fixed_enable &= ~ACPI_BITMASK_PCIEXP_WAKE_DISABLE;
-	else
-		fixed_enable |= ACPI_BITMASK_PCIEXP_WAKE_DISABLE;
-
 	ACPI_DEBUG_PRINT((ACPI_DB_INTERRUPTS,
 			  "Fixed Event Block: Enable %08X Status %08X\n",
 			  fixed_enable, fixed_status));
@@ -258,9 +250,6 @@ static u32 acpi_ev_fixed_event_dispatch(u32 event)
 	if (!acpi_gbl_fixed_event_handlers[event].handler) {
 		(void)acpi_write_bit_register(acpi_gbl_fixed_event_info[event].
 					      enable_register_id,
-					      (event ==
-					       ACPI_EVENT_PCIE_WAKE) ?
-					      ACPI_ENABLE_EVENT :
 					      ACPI_DISABLE_EVENT);
 
 		ACPI_ERROR((AE_INFO,
diff --git a/drivers/acpi/acpica/evglock.c b/drivers/acpi/acpica/evglock.c
index 9aab54797ded..989dc01af03f 100644
--- a/drivers/acpi/acpica/evglock.c
+++ b/drivers/acpi/acpica/evglock.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evglock - Global Lock support
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evgpe.c b/drivers/acpi/acpica/evgpe.c
index a6bb480d631c..934b201d3820 100644
--- a/drivers/acpi/acpica/evgpe.c
+++ b/drivers/acpi/acpica/evgpe.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evgpe - General Purpose Event handling and dispatch
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evgpeblk.c b/drivers/acpi/acpica/evgpeblk.c
index 39fe4566310b..58e1890ab25b 100644
--- a/drivers/acpi/acpica/evgpeblk.c
+++ b/drivers/acpi/acpica/evgpeblk.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evgpeblk - GPE block creation and initialization.
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evgpeinit.c b/drivers/acpi/acpica/evgpeinit.c
index 2f1a75fee61c..0dbc4d88919a 100644
--- a/drivers/acpi/acpica/evgpeinit.c
+++ b/drivers/acpi/acpica/evgpeinit.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evgpeinit - System GPE initialization and update
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evgpeutil.c b/drivers/acpi/acpica/evgpeutil.c
index c32eb57aa21d..ee3b1ea656d4 100644
--- a/drivers/acpi/acpica/evgpeutil.c
+++ b/drivers/acpi/acpica/evgpeutil.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evgpeutil - GPE utilities
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evhandler.c b/drivers/acpi/acpica/evhandler.c
index be9a05498adc..1c8cb6d924df 100644
--- a/drivers/acpi/acpica/evhandler.c
+++ b/drivers/acpi/acpica/evhandler.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evhandler - Support for Address Space handlers
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evmisc.c b/drivers/acpi/acpica/evmisc.c
index 6172cddc1b39..e68e876d3b84 100644
--- a/drivers/acpi/acpica/evmisc.c
+++ b/drivers/acpi/acpica/evmisc.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evmisc - Miscellaneous event manager support functions
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c
index d035092799eb..18fdf2bc2d49 100644
--- a/drivers/acpi/acpica/evregion.c
+++ b/drivers/acpi/acpica/evregion.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evregion - Operation Region support
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evrgnini.c b/drivers/acpi/acpica/evrgnini.c
index ca4ba6b351fe..46d1b3f5582d 100644
--- a/drivers/acpi/acpica/evrgnini.c
+++ b/drivers/acpi/acpica/evrgnini.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evrgnini- ACPI address_space (op_region) init
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -430,7 +430,7 @@ acpi_ev_data_table_region_setup(acpi_handle handle,
 {
 	union acpi_operand_object *region_desc =
 	    (union acpi_operand_object *)handle;
-	struct acpi_data_table_space_context *local_region_context;
+	struct acpi_data_table_mapping *local_region_context;
 
 	ACPI_FUNCTION_TRACE(ev_data_table_region_setup);
 
@@ -445,7 +445,7 @@ acpi_ev_data_table_region_setup(acpi_handle handle,
 	/* Create a new context */
 
 	local_region_context =
-	    ACPI_ALLOCATE_ZEROED(sizeof(struct acpi_data_table_space_context));
+	    ACPI_ALLOCATE_ZEROED(sizeof(struct acpi_data_table_mapping));
 	if (!(local_region_context)) {
 		return_ACPI_STATUS(AE_NO_MEMORY);
 	}
diff --git a/drivers/acpi/acpica/evxface.c b/drivers/acpi/acpica/evxface.c
index 18219abba108..24fa6433d562 100644
--- a/drivers/acpi/acpica/evxface.c
+++ b/drivers/acpi/acpica/evxface.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evxface - External interfaces for ACPI events
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evxfevnt.c b/drivers/acpi/acpica/evxfevnt.c
index 8187b081e0a6..48bf845191d2 100644
--- a/drivers/acpi/acpica/evxfevnt.c
+++ b/drivers/acpi/acpica/evxfevnt.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evxfevnt - External Interfaces, ACPI event disable/enable
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evxfgpe.c b/drivers/acpi/acpica/evxfgpe.c
index 340947e412bb..4eeeb3b7ab7e 100644
--- a/drivers/acpi/acpica/evxfgpe.c
+++ b/drivers/acpi/acpica/evxfgpe.c
@@ -3,7 +3,7 @@
  *
  * Module Name: evxfgpe - External Interfaces for General Purpose Events (GPEs)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/evxfregn.c b/drivers/acpi/acpica/evxfregn.c
index a5c19f46ec17..3197e6303c5b 100644
--- a/drivers/acpi/acpica/evxfregn.c
+++ b/drivers/acpi/acpica/evxfregn.c
@@ -4,7 +4,7 @@
  * Module Name: evxfregn - External Interfaces, ACPI Operation Regions and
  *                         Address Spaces.
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exconcat.c b/drivers/acpi/acpica/exconcat.c
index 66201742f499..2fb78b35565b 100644
--- a/drivers/acpi/acpica/exconcat.c
+++ b/drivers/acpi/acpica/exconcat.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exconcat - Concatenate-type AML operators
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exconfig.c b/drivers/acpi/acpica/exconfig.c
index e82faabdf907..473115309860 100644
--- a/drivers/acpi/acpica/exconfig.c
+++ b/drivers/acpi/acpica/exconfig.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exconfig - Namespace reconfiguration (Load/Unload opcodes)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exconvrt.c b/drivers/acpi/acpica/exconvrt.c
index 8de5d47ad485..3729bf3b74f7 100644
--- a/drivers/acpi/acpica/exconvrt.c
+++ b/drivers/acpi/acpica/exconvrt.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exconvrt - Object conversion routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/excreate.c b/drivers/acpi/acpica/excreate.c
index fb2453fa9442..1bea9d97652c 100644
--- a/drivers/acpi/acpica/excreate.c
+++ b/drivers/acpi/acpica/excreate.c
@@ -3,7 +3,7 @@
  *
  * Module Name: excreate - Named object creation
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exdebug.c b/drivers/acpi/acpica/exdebug.c
index 8a99aadb9d15..3f86bfada510 100644
--- a/drivers/acpi/acpica/exdebug.c
+++ b/drivers/acpi/acpica/exdebug.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exdebug - Support for stores to the AML Debug Object
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exdump.c b/drivers/acpi/acpica/exdump.c
index 24b3d041b3e5..2e2da8790224 100644
--- a/drivers/acpi/acpica/exdump.c
+++ b/drivers/acpi/acpica/exdump.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exdump - Interpreter debug output routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exfield.c b/drivers/acpi/acpica/exfield.c
index 657f4002f9dc..61ff36189ace 100644
--- a/drivers/acpi/acpica/exfield.c
+++ b/drivers/acpi/acpica/exfield.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exfield - AML execution - field_unit read/write
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exfldio.c b/drivers/acpi/acpica/exfldio.c
index d769cea1468b..cf6c812a8b6d 100644
--- a/drivers/acpi/acpica/exfldio.c
+++ b/drivers/acpi/acpica/exfldio.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exfldio - Aml Field I/O
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exmisc.c b/drivers/acpi/acpica/exmisc.c
index b4bac8c60a13..c6f2a9166ac0 100644
--- a/drivers/acpi/acpica/exmisc.c
+++ b/drivers/acpi/acpica/exmisc.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exmisc - ACPI AML (p-code) execution - specific opcodes
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exmutex.c b/drivers/acpi/acpica/exmutex.c
index e9dcfa1e93eb..65c487facdda 100644
--- a/drivers/acpi/acpica/exmutex.c
+++ b/drivers/acpi/acpica/exmutex.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exmutex - ASL Mutex Acquire/Release functions
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exnames.c b/drivers/acpi/acpica/exnames.c
index 318eb769058d..9a448165bfeb 100644
--- a/drivers/acpi/acpica/exnames.c
+++ b/drivers/acpi/acpica/exnames.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exnames - interpreter/scanner name load/execute
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exoparg1.c b/drivers/acpi/acpica/exoparg1.c
index d108a1a86f12..20fb34b68bee 100644
--- a/drivers/acpi/acpica/exoparg1.c
+++ b/drivers/acpi/acpica/exoparg1.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exoparg1 - AML execution - opcodes with 1 argument
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exoparg2.c b/drivers/acpi/acpica/exoparg2.c
index ebf7c89d52d9..743c258bf2e8 100644
--- a/drivers/acpi/acpica/exoparg2.c
+++ b/drivers/acpi/acpica/exoparg2.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exoparg2 - AML execution - opcodes with 2 arguments
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exoparg3.c b/drivers/acpi/acpica/exoparg3.c
index 4b069bd6bc71..d3091f619909 100644
--- a/drivers/acpi/acpica/exoparg3.c
+++ b/drivers/acpi/acpica/exoparg3.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exoparg3 - AML execution - opcodes with 3 arguments
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exoparg6.c b/drivers/acpi/acpica/exoparg6.c
index 2a506ef386cf..1af35e143ba9 100644
--- a/drivers/acpi/acpica/exoparg6.c
+++ b/drivers/acpi/acpica/exoparg6.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exoparg6 - AML execution - opcodes with 6 arguments
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exprep.c b/drivers/acpi/acpica/exprep.c
index 08f06797386a..08196fa17080 100644
--- a/drivers/acpi/acpica/exprep.c
+++ b/drivers/acpi/acpica/exprep.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exprep - ACPI AML field prep utilities
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exregion.c b/drivers/acpi/acpica/exregion.c
index 4ff35852c0b3..8907b8bf4267 100644
--- a/drivers/acpi/acpica/exregion.c
+++ b/drivers/acpi/acpica/exregion.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exregion - ACPI default op_region (address space) handlers
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -509,12 +509,12 @@ acpi_ex_data_table_space_handler(u32 function,
 				 u64 *value,
 				 void *handler_context, void *region_context)
 {
-	struct acpi_data_table_space_context *mapping;
+	struct acpi_data_table_mapping *mapping;
 	char *pointer;
 
 	ACPI_FUNCTION_TRACE(ex_data_table_space_handler);
 
-	mapping = (struct acpi_data_table_space_context *) region_context;
+	mapping = (struct acpi_data_table_mapping *) region_context;
 	pointer = ACPI_CAST_PTR(char, mapping->pointer) +
 	    (address - ACPI_PTR_TO_PHYSADDR(mapping->pointer));
 
diff --git a/drivers/acpi/acpica/exresnte.c b/drivers/acpi/acpica/exresnte.c
index b81506d73447..873de01b8ad2 100644
--- a/drivers/acpi/acpica/exresnte.c
+++ b/drivers/acpi/acpica/exresnte.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exresnte - AML Interpreter object resolution
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exresolv.c b/drivers/acpi/acpica/exresolv.c
index 61ee7fb46006..24a78b5e266c 100644
--- a/drivers/acpi/acpica/exresolv.c
+++ b/drivers/acpi/acpica/exresolv.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exresolv - AML Interpreter object resolution
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exresop.c b/drivers/acpi/acpica/exresop.c
index 3342780230af..3a437e6ace5c 100644
--- a/drivers/acpi/acpica/exresop.c
+++ b/drivers/acpi/acpica/exresop.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exresop - AML Interpreter operand/object resolution
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exserial.c b/drivers/acpi/acpica/exserial.c
index fd63f2042514..5d99b1a76c83 100644
--- a/drivers/acpi/acpica/exserial.c
+++ b/drivers/acpi/acpica/exserial.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exserial - field_unit support for serial address spaces
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exstore.c b/drivers/acpi/acpica/exstore.c
index f99e8cf27a6c..575c7a39f1aa 100644
--- a/drivers/acpi/acpica/exstore.c
+++ b/drivers/acpi/acpica/exstore.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exstore - AML Interpreter object store support
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exstoren.c b/drivers/acpi/acpica/exstoren.c
index c848b328e760..b01ae015e1b5 100644
--- a/drivers/acpi/acpica/exstoren.c
+++ b/drivers/acpi/acpica/exstoren.c
@@ -4,7 +4,7 @@
  * Module Name: exstoren - AML Interpreter object store support,
  *                        Store to Node (namespace object)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exstorob.c b/drivers/acpi/acpica/exstorob.c
index 45c757bbf9a9..37c3131a82fa 100644
--- a/drivers/acpi/acpica/exstorob.c
+++ b/drivers/acpi/acpica/exstorob.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exstorob - AML object store support, store to object
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exsystem.c b/drivers/acpi/acpica/exsystem.c
index 7b5470f404f3..f665ffd9a396 100644
--- a/drivers/acpi/acpica/exsystem.c
+++ b/drivers/acpi/acpica/exsystem.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exsystem - Interface to OS services
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/extrace.c b/drivers/acpi/acpica/extrace.c
index b570d7a7e134..f1730221ff13 100644
--- a/drivers/acpi/acpica/extrace.c
+++ b/drivers/acpi/acpica/extrace.c
@@ -3,7 +3,7 @@
  *
  * Module Name: extrace - Support for interpreter execution tracing
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/exutils.c b/drivers/acpi/acpica/exutils.c
index 87f01ce1c1aa..f4d4a033f166 100644
--- a/drivers/acpi/acpica/exutils.c
+++ b/drivers/acpi/acpica/exutils.c
@@ -3,7 +3,7 @@
  *
  * Module Name: exutils - interpreter/scanner utilities
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/hwacpi.c b/drivers/acpi/acpica/hwacpi.c
index 2f1c2fc8bd2a..790f342dcd25 100644
--- a/drivers/acpi/acpica/hwacpi.c
+++ b/drivers/acpi/acpica/hwacpi.c
@@ -3,7 +3,7 @@
  *
  * Module Name: hwacpi - ACPI Hardware Initialization/Mode Interface
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/hwesleep.c b/drivers/acpi/acpica/hwesleep.c
index d8597e052912..a9ba9190408b 100644
--- a/drivers/acpi/acpica/hwesleep.c
+++ b/drivers/acpi/acpica/hwesleep.c
@@ -4,7 +4,7 @@
  * Name: hwesleep.c - ACPI Hardware Sleep/Wake Support functions for the
  *                    extended FADT-V5 sleep registers.
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/hwgpe.c b/drivers/acpi/acpica/hwgpe.c
index 13d54a48e6e9..e0c847ab8324 100644
--- a/drivers/acpi/acpica/hwgpe.c
+++ b/drivers/acpi/acpica/hwgpe.c
@@ -3,7 +3,7 @@
  *
  * Module Name: hwgpe - Low level GPE enable/disable/clear functions
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/hwsleep.c b/drivers/acpi/acpica/hwsleep.c
index 37b3f641feaa..e0921f08b71a 100644
--- a/drivers/acpi/acpica/hwsleep.c
+++ b/drivers/acpi/acpica/hwsleep.c
@@ -4,7 +4,7 @@
  * Name: hwsleep.c - ACPI Hardware Sleep/Wake Support functions for the
  *                   original/legacy sleep/PM registers.
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -311,20 +311,6 @@ acpi_status acpi_hw_legacy_wake(u8 sleep_state)
 				    [ACPI_EVENT_SLEEP_BUTTON].
 				    status_register_id, ACPI_CLEAR_STATUS);
 
-	/* Enable pcie wake event if support */
-	if ((acpi_gbl_FADT.flags & ACPI_FADT_PCI_EXPRESS_WAKE)) {
-		(void)
-		    acpi_write_bit_register(acpi_gbl_fixed_event_info
-					    [ACPI_EVENT_PCIE_WAKE].
-					    enable_register_id,
-					    ACPI_DISABLE_EVENT);
-		(void)
-		    acpi_write_bit_register(acpi_gbl_fixed_event_info
-					    [ACPI_EVENT_PCIE_WAKE].
-					    status_register_id,
-					    ACPI_CLEAR_STATUS);
-	}
-
 	acpi_hw_execute_sleep_method(METHOD_PATHNAME__SST, ACPI_SST_WORKING);
 	return_ACPI_STATUS(status);
 }
diff --git a/drivers/acpi/acpica/hwtimer.c b/drivers/acpi/acpica/hwtimer.c
index 46f3ae03ab99..192c04b5a599 100644
--- a/drivers/acpi/acpica/hwtimer.c
+++ b/drivers/acpi/acpica/hwtimer.c
@@ -3,7 +3,7 @@
  *
  * Name: hwtimer.c - ACPI Power Management Timer Interface
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/hwvalid.c b/drivers/acpi/acpica/hwvalid.c
index 0d392e7b0747..b8de458f0368 100644
--- a/drivers/acpi/acpica/hwvalid.c
+++ b/drivers/acpi/acpica/hwvalid.c
@@ -3,7 +3,7 @@
  *
  * Module Name: hwvalid - I/O request validation
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/hwxface.c b/drivers/acpi/acpica/hwxface.c
index 55d9b897e70f..c31f803995c6 100644
--- a/drivers/acpi/acpica/hwxface.c
+++ b/drivers/acpi/acpica/hwxface.c
@@ -3,7 +3,7 @@
  *
  * Module Name: hwxface - Public ACPICA hardware interfaces
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/hwxfsleep.c b/drivers/acpi/acpica/hwxfsleep.c
index aff51ceea02c..36ea48f64110 100644
--- a/drivers/acpi/acpica/hwxfsleep.c
+++ b/drivers/acpi/acpica/hwxfsleep.c
@@ -3,7 +3,7 @@
  *
  * Name: hwxfsleep.c - ACPI Hardware Sleep/Wake External Interfaces
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsarguments.c b/drivers/acpi/acpica/nsarguments.c
index 22586b90e532..3efb46f0dc54 100644
--- a/drivers/acpi/acpica/nsarguments.c
+++ b/drivers/acpi/acpica/nsarguments.c
@@ -3,7 +3,7 @@
  *
  * Module Name: nsarguments - Validation of args for ACPI predefined methods
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsconvert.c b/drivers/acpi/acpica/nsconvert.c
index b02555fe38f0..7e5a683ae957 100644
--- a/drivers/acpi/acpica/nsconvert.c
+++ b/drivers/acpi/acpica/nsconvert.c
@@ -4,7 +4,7 @@
  * Module Name: nsconvert - Object conversions for objects returned by
  *                          predefined methods
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsdump.c b/drivers/acpi/acpica/nsdump.c
index f154824d4eb6..90a26cb0c472 100644
--- a/drivers/acpi/acpica/nsdump.c
+++ b/drivers/acpi/acpica/nsdump.c
@@ -3,7 +3,7 @@
  *
  * Module Name: nsdump - table dumping routines for debug
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsdumpdv.c b/drivers/acpi/acpica/nsdumpdv.c
index b9a88b7b518b..fa116ebe49a3 100644
--- a/drivers/acpi/acpica/nsdumpdv.c
+++ b/drivers/acpi/acpica/nsdumpdv.c
@@ -3,7 +3,7 @@
  *
  * Module Name: nsdump - table dumping routines for debug
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsinit.c b/drivers/acpi/acpica/nsinit.c
index 3e6207ad18d8..86d126fdb27d 100644
--- a/drivers/acpi/acpica/nsinit.c
+++ b/drivers/acpi/acpica/nsinit.c
@@ -3,7 +3,7 @@
  *
  * Module Name: nsinit - namespace initialization
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsload.c b/drivers/acpi/acpica/nsload.c
index 880260a30c0c..fcb9de0f77a2 100644
--- a/drivers/acpi/acpica/nsload.c
+++ b/drivers/acpi/acpica/nsload.c
@@ -3,7 +3,7 @@
  *
  * Module Name: nsload - namespace loading/expanding/contracting procedures
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsparse.c b/drivers/acpi/acpica/nsparse.c
index 4b893676ab5c..31e551cf4ea6 100644
--- a/drivers/acpi/acpica/nsparse.c
+++ b/drivers/acpi/acpica/nsparse.c
@@ -3,7 +3,7 @@
  *
  * Module Name: nsparse - namespace interface to AML parser
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nspredef.c b/drivers/acpi/acpica/nspredef.c
index c0db6690bb32..cf57bd69616d 100644
--- a/drivers/acpi/acpica/nspredef.c
+++ b/drivers/acpi/acpica/nspredef.c
@@ -3,7 +3,7 @@
  *
  * Module Name: nspredef - Validation of ACPI predefined methods and objects
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsprepkg.c b/drivers/acpi/acpica/nsprepkg.c
index 82932c9a774b..dd37fc108fce 100644
--- a/drivers/acpi/acpica/nsprepkg.c
+++ b/drivers/acpi/acpica/nsprepkg.c
@@ -3,7 +3,7 @@
  *
  * Module Name: nsprepkg - Validation of package objects for predefined names
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsrepair.c b/drivers/acpi/acpica/nsrepair.c
index ec512e06a48e..b8657004190d 100644
--- a/drivers/acpi/acpica/nsrepair.c
+++ b/drivers/acpi/acpica/nsrepair.c
@@ -3,7 +3,7 @@
  *
  * Module Name: nsrepair - Repair for objects returned by predefined methods
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsrepair2.c b/drivers/acpi/acpica/nsrepair2.c
index dd533c887e3a..1bb7b71f07f1 100644
--- a/drivers/acpi/acpica/nsrepair2.c
+++ b/drivers/acpi/acpica/nsrepair2.c
@@ -4,7 +4,7 @@
  * Module Name: nsrepair2 - Repair for objects returned by specific
  *                          predefined methods
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -499,7 +499,7 @@ acpi_ns_repair_HID(struct acpi_evaluate_info *info,
 	char *source;
 	char *dest;
 
-	ACPI_FUNCTION_NAME(ns_repair_HID);
+	ACPI_FUNCTION_TRACE(ns_repair_HID);
 
 	/* We only care about string _HID objects (not integers) */
 
diff --git a/drivers/acpi/acpica/nsutils.c b/drivers/acpi/acpica/nsutils.c
index ef531b145add..06ffdb6808f5 100644
--- a/drivers/acpi/acpica/nsutils.c
+++ b/drivers/acpi/acpica/nsutils.c
@@ -4,7 +4,7 @@
  * Module Name: nsutils - Utilities for accessing ACPI namespace, accessing
  *                        parents and siblings and Scope manipulation
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nswalk.c b/drivers/acpi/acpica/nswalk.c
index 82a0dae349e2..eee396a77bae 100644
--- a/drivers/acpi/acpica/nswalk.c
+++ b/drivers/acpi/acpica/nswalk.c
@@ -3,7 +3,7 @@
  *
  * Module Name: nswalk - Functions for walking the ACPI namespace
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/nsxfname.c b/drivers/acpi/acpica/nsxfname.c
index a0592d15dd37..5d5bcf165298 100644
--- a/drivers/acpi/acpica/nsxfname.c
+++ b/drivers/acpi/acpica/nsxfname.c
@@ -4,7 +4,7 @@
  * Module Name: nsxfname - Public interfaces to the ACPI subsystem
  *                         ACPI Namespace oriented interfaces
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/psargs.c b/drivers/acpi/acpica/psargs.c
index f7ec5606098c..422c074ed289 100644
--- a/drivers/acpi/acpica/psargs.c
+++ b/drivers/acpi/acpica/psargs.c
@@ -3,7 +3,7 @@
  *
  * Module Name: psargs - Parse AML opcode arguments
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/psloop.c b/drivers/acpi/acpica/psloop.c
index 840512fa9fc6..d0fd55636129 100644
--- a/drivers/acpi/acpica/psloop.c
+++ b/drivers/acpi/acpica/psloop.c
@@ -3,7 +3,7 @@
  *
  * Module Name: psloop - Main AML parse loop
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/psobject.c b/drivers/acpi/acpica/psobject.c
index bca249e67c6b..54471083ba54 100644
--- a/drivers/acpi/acpica/psobject.c
+++ b/drivers/acpi/acpica/psobject.c
@@ -3,7 +3,7 @@
  *
  * Module Name: psobject - Support for parse objects
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/psopcode.c b/drivers/acpi/acpica/psopcode.c
index bef69e87a0a2..09029fe545f1 100644
--- a/drivers/acpi/acpica/psopcode.c
+++ b/drivers/acpi/acpica/psopcode.c
@@ -3,7 +3,7 @@
  *
  * Module Name: psopcode - Parser/Interpreter opcode information table
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/psopinfo.c b/drivers/acpi/acpica/psopinfo.c
index f10afe699ad7..bccf606e08b4 100644
--- a/drivers/acpi/acpica/psopinfo.c
+++ b/drivers/acpi/acpica/psopinfo.c
@@ -3,7 +3,7 @@
  *
  * Module Name: psopinfo - AML opcode information functions and dispatch tables
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/psparse.c b/drivers/acpi/acpica/psparse.c
index ba93f359760a..10a072953d78 100644
--- a/drivers/acpi/acpica/psparse.c
+++ b/drivers/acpi/acpica/psparse.c
@@ -3,7 +3,7 @@
  *
  * Module Name: psparse - Parser top level AML parse routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/psscope.c b/drivers/acpi/acpica/psscope.c
index 400f001631ea..a0035bde7556 100644
--- a/drivers/acpi/acpica/psscope.c
+++ b/drivers/acpi/acpica/psscope.c
@@ -3,7 +3,7 @@
  *
  * Module Name: psscope - Parser scope stack management routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/pstree.c b/drivers/acpi/acpica/pstree.c
index 3012a9342367..7f7f5ecd4011 100644
--- a/drivers/acpi/acpica/pstree.c
+++ b/drivers/acpi/acpica/pstree.c
@@ -3,7 +3,7 @@
  *
  * Module Name: pstree - Parser op tree manipulation/traversal/search
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/psutils.c b/drivers/acpi/acpica/psutils.c
index 49b39aeded12..d550c4af4702 100644
--- a/drivers/acpi/acpica/psutils.c
+++ b/drivers/acpi/acpica/psutils.c
@@ -3,7 +3,7 @@
  *
  * Module Name: psutils - Parser miscellaneous utilities (Parser only)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/pswalk.c b/drivers/acpi/acpica/pswalk.c
index 7735a01dab90..d92817c72b8d 100644
--- a/drivers/acpi/acpica/pswalk.c
+++ b/drivers/acpi/acpica/pswalk.c
@@ -3,7 +3,7 @@
  *
  * Module Name: pswalk - Parser routines to walk parsed op tree(s)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/psxface.c b/drivers/acpi/acpica/psxface.c
index a6509aeb2955..6f4eace0ba69 100644
--- a/drivers/acpi/acpica/psxface.c
+++ b/drivers/acpi/acpica/psxface.c
@@ -3,7 +3,7 @@
  *
  * Module Name: psxface - Parser external interfaces
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/rsaddr.c b/drivers/acpi/acpica/rsaddr.c
index 5737c3af1902..fff48001d7ef 100644
--- a/drivers/acpi/acpica/rsaddr.c
+++ b/drivers/acpi/acpica/rsaddr.c
@@ -272,12 +272,17 @@ u8
 acpi_rs_get_address_common(struct acpi_resource *resource,
 			   union aml_resource *aml)
 {
+	struct aml_resource_address address;
+
 	ACPI_FUNCTION_ENTRY();
 
+	/* Avoid undefined behavior: member access within misaligned address */
+
+	memcpy(&address, aml, sizeof(address));
+
 	/* Validate the Resource Type */
 
-	if ((aml->address.resource_type > 2) &&
-	    (aml->address.resource_type < 0xC0)) {
+	if ((address.resource_type > 2) && (address.resource_type < 0xC0)) {
 		return (FALSE);
 	}
 
@@ -298,7 +303,7 @@ acpi_rs_get_address_common(struct acpi_resource *resource,
 		/* Generic resource type, just grab the type_specific byte */
 
 		resource->data.address.info.type_specific =
-		    aml->address.specific_flags;
+		    address.specific_flags;
 	}
 
 	return (TRUE);
diff --git a/drivers/acpi/acpica/rscalc.c b/drivers/acpi/acpica/rscalc.c
index 90583db459a2..6e7a152d6459 100644
--- a/drivers/acpi/acpica/rscalc.c
+++ b/drivers/acpi/acpica/rscalc.c
@@ -320,6 +320,16 @@ acpi_rs_get_aml_length(struct acpi_resource *resource,
 
 			break;
 
+		case ACPI_RESOURCE_TYPE_CLOCK_INPUT:
+
+			total_size = (acpi_rs_length)(total_size +
+						      resource->data.
+						      clock_input.
+						      resource_source.
+						      string_length);
+
+			break;
+
 		case ACPI_RESOURCE_TYPE_SERIAL_BUS:
 
 			total_size =
@@ -596,15 +606,23 @@ acpi_rs_get_list_length(u8 *aml_buffer,
 			}
 			break;
 
-		case ACPI_RESOURCE_NAME_SERIAL_BUS:
+		case ACPI_RESOURCE_NAME_SERIAL_BUS:{
 
-			minimum_aml_resource_length =
-			    acpi_gbl_resource_aml_serial_bus_sizes
-			    [aml_resource->common_serial_bus.type];
-			extra_struct_bytes +=
-			    aml_resource->common_serial_bus.resource_length -
-			    minimum_aml_resource_length;
-			break;
+				/* Avoid undefined behavior: member access within misaligned address */
+
+				struct aml_resource_common_serialbus
+				    common_serial_bus;
+				memcpy(&common_serial_bus, aml_resource,
+				       sizeof(common_serial_bus));
+
+				minimum_aml_resource_length =
+				    acpi_gbl_resource_aml_serial_bus_sizes
+				    [common_serial_bus.type];
+				extra_struct_bytes +=
+				    common_serial_bus.resource_length -
+				    minimum_aml_resource_length;
+				break;
+			}
 
 		case ACPI_RESOURCE_NAME_PIN_CONFIG:
 
@@ -650,6 +668,13 @@ acpi_rs_get_list_length(u8 *aml_buffer,
 
 			break;
 
+		case ACPI_RESOURCE_NAME_CLOCK_INPUT:
+			extra_struct_bytes =
+			    acpi_rs_stream_option_length(resource_length,
+							 minimum_aml_resource_length);
+
+			break;
+
 		default:
 
 			break;
@@ -663,10 +688,16 @@ acpi_rs_get_list_length(u8 *aml_buffer,
 		 */
 		if (acpi_ut_get_resource_type(aml_buffer) ==
 		    ACPI_RESOURCE_NAME_SERIAL_BUS) {
+
+			/* Avoid undefined behavior: member access within misaligned address */
+
+			struct aml_resource_common_serialbus common_serial_bus;
+			memcpy(&common_serial_bus, aml_resource,
+			       sizeof(common_serial_bus));
+
 			buffer_size =
 			    acpi_gbl_resource_struct_serial_bus_sizes
-			    [aml_resource->common_serial_bus.type] +
-			    extra_struct_bytes;
+			    [common_serial_bus.type] + extra_struct_bytes;
 		} else {
 			buffer_size =
 			    acpi_gbl_resource_struct_sizes[resource_index] +
diff --git a/drivers/acpi/acpica/rsdumpinfo.c b/drivers/acpi/acpica/rsdumpinfo.c
index b8b37449011b..998a79cc09c2 100644
--- a/drivers/acpi/acpica/rsdumpinfo.c
+++ b/drivers/acpi/acpica/rsdumpinfo.c
@@ -301,6 +301,23 @@ struct acpi_rsdump_info acpi_rs_dump_pin_function[10] = {
 	 "VendorData", NULL},
 };
 
+struct acpi_rsdump_info acpi_rs_dump_clock_input[7] = {
+	{ACPI_RSD_TITLE, ACPI_RSD_TABLE_SIZE(acpi_rs_dump_clock_input),
+	 "ClockInput", NULL},
+	{ACPI_RSD_UINT8, ACPI_RSD_OFFSET(clock_input.revision_id), "RevisionId",
+	 NULL},
+	{ACPI_RSD_UINT32, ACPI_RSD_OFFSET(clock_input.frequency_numerator),
+	 "FrequencyNumerator", NULL},
+	{ACPI_RSD_UINT32, ACPI_RSD_OFFSET(clock_input.frequency_divisor),
+	 "FrequencyDivisor", NULL},
+	{ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET(clock_input.scale), "Scale",
+	 acpi_gbl_clock_input_scale},
+	{ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET(clock_input.mode), "Mode",
+	 acpi_gbl_clock_input_mode},
+	{ACPI_RSD_SOURCE, ACPI_RSD_OFFSET(clock_input.resource_source),
+	 "ResourceSource", NULL},
+};
+
 struct acpi_rsdump_info acpi_rs_dump_pin_config[11] = {
 	{ACPI_RSD_TITLE, ACPI_RSD_TABLE_SIZE(acpi_rs_dump_pin_config),
 	 "PinConfig", NULL},
diff --git a/drivers/acpi/acpica/rsinfo.c b/drivers/acpi/acpica/rsinfo.c
index eaeb7ab58c2a..ad7465ddfe13 100644
--- a/drivers/acpi/acpica/rsinfo.c
+++ b/drivers/acpi/acpica/rsinfo.c
@@ -49,6 +49,7 @@ struct acpi_rsconvert_info *acpi_gbl_set_resource_dispatch[] = {
 	acpi_rs_convert_pin_group,	/* 0x16, ACPI_RESOURCE_TYPE_PIN_GROUP */
 	acpi_rs_convert_pin_group_function,	/* 0x17, ACPI_RESOURCE_TYPE_PIN_GROUP_FUNCTION */
 	acpi_rs_convert_pin_group_config,	/* 0x18, ACPI_RESOURCE_TYPE_PIN_GROUP_CONFIG */
+	acpi_rs_convert_clock_input,	/* 0x19, ACPI_RESOURCE_TYPE_CLOCK_INPUT */
 };
 
 /* Dispatch tables for AML-to-resource (Get Resource) conversion functions */
@@ -94,6 +95,7 @@ struct acpi_rsconvert_info *acpi_gbl_get_resource_dispatch[] = {
 	acpi_rs_convert_pin_group,	/* 0x10, ACPI_RESOURCE_NAME_PIN_GROUP */
 	acpi_rs_convert_pin_group_function,	/* 0x11, ACPI_RESOURCE_NAME_PIN_GROUP_FUNCTION */
 	acpi_rs_convert_pin_group_config,	/* 0x12, ACPI_RESOURCE_NAME_PIN_GROUP_CONFIG */
+	acpi_rs_convert_clock_input,	/* 0x13, ACPI_RESOURCE_NAME_CLOCK_INPUT */
 };
 
 /* Subtype table for serial_bus -- I2C, SPI, UART, and CSI2 */
@@ -136,6 +138,7 @@ struct acpi_rsdump_info *acpi_gbl_dump_resource_dispatch[] = {
 	acpi_rs_dump_pin_group,	/* ACPI_RESOURCE_TYPE_PIN_GROUP */
 	acpi_rs_dump_pin_group_function,	/* ACPI_RESOURCE_TYPE_PIN_GROUP_FUNCTION */
 	acpi_rs_dump_pin_group_config,	/* ACPI_RESOURCE_TYPE_PIN_GROUP_CONFIG */
+	acpi_rs_dump_clock_input,	/* ACPI_RESOURCE_TYPE_CLOCK_INPUT */
 };
 
 struct acpi_rsdump_info *acpi_gbl_dump_serial_bus_dispatch[] = {
@@ -178,6 +181,7 @@ const u8 acpi_gbl_aml_resource_sizes[] = {
 	sizeof(struct aml_resource_pin_group),	/* ACPI_RESOURCE_TYPE_PIN_GROUP */
 	sizeof(struct aml_resource_pin_group_function),	/* ACPI_RESOURCE_TYPE_PIN_GROUP_FUNCTION */
 	sizeof(struct aml_resource_pin_group_config),	/* ACPI_RESOURCE_TYPE_PIN_GROUP_CONFIG */
+	sizeof(struct aml_resource_clock_input),	/* ACPI_RESOURCE_TYPE_CLOCK_INPUT */
 };
 
 const u8 acpi_gbl_resource_struct_sizes[] = {
@@ -221,6 +225,7 @@ const u8 acpi_gbl_resource_struct_sizes[] = {
 	ACPI_RS_SIZE(struct acpi_resource_pin_group),
 	ACPI_RS_SIZE(struct acpi_resource_pin_group_function),
 	ACPI_RS_SIZE(struct acpi_resource_pin_group_config),
+	ACPI_RS_SIZE(struct acpi_resource_clock_input),
 };
 
 const u8 acpi_gbl_aml_resource_serial_bus_sizes[] = {
diff --git a/drivers/acpi/acpica/rslist.c b/drivers/acpi/acpica/rslist.c
index e46efaa889cd..164c96e063c6 100644
--- a/drivers/acpi/acpica/rslist.c
+++ b/drivers/acpi/acpica/rslist.c
@@ -55,15 +55,21 @@ acpi_rs_convert_aml_to_resources(u8 * aml,
 	aml_resource = ACPI_CAST_PTR(union aml_resource, aml);
 
 	if (acpi_ut_get_resource_type(aml) == ACPI_RESOURCE_NAME_SERIAL_BUS) {
-		if (aml_resource->common_serial_bus.type >
-		    AML_RESOURCE_MAX_SERIALBUSTYPE) {
+
+		/* Avoid undefined behavior: member access within misaligned address */
+
+		struct aml_resource_common_serialbus common_serial_bus;
+		memcpy(&common_serial_bus, aml_resource,
+		       sizeof(common_serial_bus));
+
+		if (common_serial_bus.type > AML_RESOURCE_MAX_SERIALBUSTYPE) {
 			conversion_table = NULL;
 		} else {
 			/* This is an I2C, SPI, UART, or CSI2 serial_bus descriptor */
 
 			conversion_table =
 			    acpi_gbl_convert_resource_serial_bus_dispatch
-			    [aml_resource->common_serial_bus.type];
+			    [common_serial_bus.type];
 		}
 	} else {
 		conversion_table =
diff --git a/drivers/acpi/acpica/rsmisc.c b/drivers/acpi/acpica/rsmisc.c
index c2dd9aae4745..6e8e98cf598d 100644
--- a/drivers/acpi/acpica/rsmisc.c
+++ b/drivers/acpi/acpica/rsmisc.c
@@ -194,7 +194,8 @@ acpi_rs_convert_aml_to_resource(struct acpi_resource *resource,
 
 		case ACPI_RSC_COUNT_SERIAL_VEN:
 
-			item_count = ACPI_GET16(source) - info->value;
+			ACPI_MOVE_16_TO_16(&temp16, source);
+			item_count = temp16 - info->value;
 
 			resource->length = resource->length + item_count;
 			ACPI_SET16(destination, item_count);
@@ -202,9 +203,10 @@ acpi_rs_convert_aml_to_resource(struct acpi_resource *resource,
 
 		case ACPI_RSC_COUNT_SERIAL_RES:
 
+			ACPI_MOVE_16_TO_16(&temp16, source);
 			item_count = (aml_resource_length +
 				      sizeof(struct aml_resource_large_header))
-			    - ACPI_GET16(source) - info->value;
+			    - temp16 - info->value;
 
 			resource->length = resource->length + item_count;
 			ACPI_SET16(destination, item_count);
@@ -289,9 +291,9 @@ acpi_rs_convert_aml_to_resource(struct acpi_resource *resource,
 
 			/* Copy the resource_source string */
 
+			ACPI_MOVE_16_TO_16(&temp16, source);
 			source =
-			    ACPI_ADD_PTR(void, aml,
-					 (ACPI_GET16(source) + info->value));
+			    ACPI_ADD_PTR(void, aml, (temp16 + info->value));
 			acpi_rs_move_data(target, source, item_count,
 					  info->opcode);
 			break;
diff --git a/drivers/acpi/acpica/rsserial.c b/drivers/acpi/acpica/rsserial.c
index f9267956535c..279bfa27da94 100644
--- a/drivers/acpi/acpica/rsserial.c
+++ b/drivers/acpi/acpica/rsserial.c
@@ -111,6 +111,55 @@ struct acpi_rsconvert_info acpi_rs_convert_gpio[18] = {
 
 /*******************************************************************************
  *
+ * acpi_rs_convert_clock_input
+ *
+ ******************************************************************************/
+
+struct acpi_rsconvert_info acpi_rs_convert_clock_input[8] = {
+	{ACPI_RSC_INITGET, ACPI_RESOURCE_TYPE_CLOCK_INPUT,
+	 ACPI_RS_SIZE(struct acpi_resource_clock_input),
+	 ACPI_RSC_TABLE_SIZE(acpi_rs_convert_clock_input)},
+
+	{ACPI_RSC_INITSET, ACPI_RESOURCE_NAME_CLOCK_INPUT,
+	 sizeof(struct aml_resource_clock_input),
+	 0}
+	,
+
+	{ACPI_RSC_MOVE8, ACPI_RS_OFFSET(data.clock_input.revision_id),
+	 AML_OFFSET(clock_input.revision_id),
+	 1}
+	,
+
+	{ACPI_RSC_1BITFLAG, ACPI_RS_OFFSET(data.clock_input.mode),
+	 AML_OFFSET(clock_input.flags),
+	 0}
+	,
+
+	{ACPI_RSC_2BITFLAG, ACPI_RS_OFFSET(data.clock_input.scale),
+	 AML_OFFSET(clock_input.flags),
+	 1}
+	,
+
+	{ACPI_RSC_MOVE16, ACPI_RS_OFFSET(data.clock_input.frequency_divisor),
+	 AML_OFFSET(clock_input.frequency_divisor),
+	 2}
+	,
+
+	{ACPI_RSC_MOVE32, ACPI_RS_OFFSET(data.clock_input.frequency_numerator),
+	 AML_OFFSET(clock_input.frequency_numerator),
+	 4}
+	,
+
+	/* Resource Source */
+	{ACPI_RSC_SOURCE, ACPI_RS_OFFSET(data.clock_input.resource_source),
+	 0,
+	 sizeof(struct aml_resource_clock_input)}
+	,
+
+};
+
+/*******************************************************************************
+ *
  * acpi_rs_convert_pinfunction
  *
  ******************************************************************************/
diff --git a/drivers/acpi/acpica/tbdata.c b/drivers/acpi/acpica/tbdata.c
index 1f7677e0dbbe..a1f10e4409a3 100644
--- a/drivers/acpi/acpica/tbdata.c
+++ b/drivers/acpi/acpica/tbdata.c
@@ -3,7 +3,7 @@
  *
  * Module Name: tbdata - Table manager data structure functions
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/tbfadt.c b/drivers/acpi/acpica/tbfadt.c
index f04dc6051320..44267a92bce5 100644
--- a/drivers/acpi/acpica/tbfadt.c
+++ b/drivers/acpi/acpica/tbfadt.c
@@ -3,7 +3,7 @@
  *
  * Module Name: tbfadt   - FADT table utilities
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/tbfind.c b/drivers/acpi/acpica/tbfind.c
index c31a5ddb0ffd..1c1b2e284bd9 100644
--- a/drivers/acpi/acpica/tbfind.c
+++ b/drivers/acpi/acpica/tbfind.c
@@ -3,7 +3,7 @@
  *
  * Module Name: tbfind   - find table
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/tbinstal.c b/drivers/acpi/acpica/tbinstal.c
index 499efcaf798d..0dc003c20e4d 100644
--- a/drivers/acpi/acpica/tbinstal.c
+++ b/drivers/acpi/acpica/tbinstal.c
@@ -3,7 +3,7 @@
  *
  * Module Name: tbinstal - ACPI table installation and removal
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/tbprint.c b/drivers/acpi/acpica/tbprint.c
index f07aa9b46f3f..58b02e4b254b 100644
--- a/drivers/acpi/acpica/tbprint.c
+++ b/drivers/acpi/acpica/tbprint.c
@@ -3,7 +3,7 @@
  *
  * Module Name: tbprint - Table output utilities
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/tbutils.c b/drivers/acpi/acpica/tbutils.c
index 17ad9c227d42..bb4a56e5673a 100644
--- a/drivers/acpi/acpica/tbutils.c
+++ b/drivers/acpi/acpica/tbutils.c
@@ -3,7 +3,7 @@
  *
  * Module Name: tbutils - ACPI Table utilities
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -165,6 +165,7 @@ struct acpi_table_header *acpi_tb_copy_dsdt(u32 table_index)
 static acpi_physical_address
 acpi_tb_get_root_table_entry(u8 *table_entry, u32 table_entry_size)
 {
+	u32 address32;
 	u64 address64;
 
 	/*
@@ -176,8 +177,8 @@ acpi_tb_get_root_table_entry(u8 *table_entry, u32 table_entry_size)
 		 * 32-bit platform, RSDT: Return 32-bit table entry
 		 * 64-bit platform, RSDT: Expand 32-bit to 64-bit and return
 		 */
-		return ((acpi_physical_address)
-			(*ACPI_CAST_PTR(u32, table_entry)));
+		ACPI_MOVE_32_TO_32(&address32, table_entry);
+		return address32;
 	} else {
 		/*
 		 * 32-bit platform, XSDT: Truncate 64-bit to 32-bit and return
diff --git a/drivers/acpi/acpica/tbxface.c b/drivers/acpi/acpica/tbxface.c
index 37da09dca940..275b52dc42e9 100644
--- a/drivers/acpi/acpica/tbxface.c
+++ b/drivers/acpi/acpica/tbxface.c
@@ -3,7 +3,7 @@
  *
  * Module Name: tbxface - ACPI table-oriented external interfaces
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/tbxfload.c b/drivers/acpi/acpica/tbxfload.c
index 258796e02be1..0f2a7343de3a 100644
--- a/drivers/acpi/acpica/tbxfload.c
+++ b/drivers/acpi/acpica/tbxfload.c
@@ -3,7 +3,7 @@
  *
  * Module Name: tbxfload - Table load/unload external interfaces
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/tbxfroot.c b/drivers/acpi/acpica/tbxfroot.c
index 53afd75bbc06..5b413bbab338 100644
--- a/drivers/acpi/acpica/tbxfroot.c
+++ b/drivers/acpi/acpica/tbxfroot.c
@@ -3,7 +3,7 @@
  *
  * Module Name: tbxfroot - Find the root ACPI table (RSDT)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utaddress.c b/drivers/acpi/acpica/utaddress.c
index 915321806cd7..be94d2fd99a7 100644
--- a/drivers/acpi/acpica/utaddress.c
+++ b/drivers/acpi/acpica/utaddress.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utaddress - op_region address range check
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utalloc.c b/drivers/acpi/acpica/utalloc.c
index 2bab6017d827..c1fb70457e20 100644
--- a/drivers/acpi/acpica/utalloc.c
+++ b/drivers/acpi/acpica/utalloc.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utalloc - local memory allocation routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utascii.c b/drivers/acpi/acpica/utascii.c
index 72fb7e9ec485..2be37676edd7 100644
--- a/drivers/acpi/acpica/utascii.c
+++ b/drivers/acpi/acpica/utascii.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utascii - Utility ascii functions
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utbuffer.c b/drivers/acpi/acpica/utbuffer.c
index 59c4050b8e91..b054bb5eeaf0 100644
--- a/drivers/acpi/acpica/utbuffer.c
+++ b/drivers/acpi/acpica/utbuffer.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utbuffer - Buffer dump routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utcache.c b/drivers/acpi/acpica/utcache.c
index 5425968dd2a8..85a85f7cf750 100644
--- a/drivers/acpi/acpica/utcache.c
+++ b/drivers/acpi/acpica/utcache.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utcache - local cache allocation routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utcksum.c b/drivers/acpi/acpica/utcksum.c
index c166e4c05ab6..b483894c3629 100644
--- a/drivers/acpi/acpica/utcksum.c
+++ b/drivers/acpi/acpica/utcksum.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utcksum - Support generating table checksums
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utcopy.c b/drivers/acpi/acpica/utcopy.c
index 63c17f420fb8..2e17e657dfa4 100644
--- a/drivers/acpi/acpica/utcopy.c
+++ b/drivers/acpi/acpica/utcopy.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utcopy - Internal to external object translation utilities
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utdebug.c b/drivers/acpi/acpica/utdebug.c
index 64ed546cf19c..1bbba8585fa6 100644
--- a/drivers/acpi/acpica/utdebug.c
+++ b/drivers/acpi/acpica/utdebug.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utdebug - Debug print/trace routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utdecode.c b/drivers/acpi/acpica/utdecode.c
index 3176393a729d..95a4b7509e01 100644
--- a/drivers/acpi/acpica/utdecode.c
+++ b/drivers/acpi/acpica/utdecode.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utdecode - Utility decoding routines (value-to-string)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/uteval.c b/drivers/acpi/acpica/uteval.c
index df20d46ed8b7..3e5173d03953 100644
--- a/drivers/acpi/acpica/uteval.c
+++ b/drivers/acpi/acpica/uteval.c
@@ -3,7 +3,7 @@
  *
  * Module Name: uteval - Object evaluation
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utglobal.c b/drivers/acpi/acpica/utglobal.c
index 53afa5edb6ec..820820ea8119 100644
--- a/drivers/acpi/acpica/utglobal.c
+++ b/drivers/acpi/acpica/utglobal.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utglobal - Global variables for the ACPI subsystem
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
@@ -186,10 +186,6 @@ struct acpi_fixed_event_info acpi_gbl_fixed_event_info[ACPI_NUM_FIXED_EVENTS] =
 					ACPI_BITREG_RT_CLOCK_ENABLE,
 					ACPI_BITMASK_RT_CLOCK_STATUS,
 					ACPI_BITMASK_RT_CLOCK_ENABLE},
-	/* ACPI_EVENT_PCIE_WAKE     */ {ACPI_BITREG_PCIEXP_WAKE_STATUS,
-					ACPI_BITREG_PCIEXP_WAKE_DISABLE,
-					ACPI_BITMASK_PCIEXP_WAKE_STATUS,
-					ACPI_BITMASK_PCIEXP_WAKE_DISABLE},
 };
 #endif				/* !ACPI_REDUCED_HARDWARE */
 
diff --git a/drivers/acpi/acpica/uthex.c b/drivers/acpi/acpica/uthex.c
index c811ee2a8160..e62802791dcf 100644
--- a/drivers/acpi/acpica/uthex.c
+++ b/drivers/acpi/acpica/uthex.c
@@ -3,7 +3,7 @@
  *
  * Module Name: uthex -- Hex/ASCII support functions
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utids.c b/drivers/acpi/acpica/utids.c
index b6caab49f1bd..15c2ce91d403 100644
--- a/drivers/acpi/acpica/utids.c
+++ b/drivers/acpi/acpica/utids.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utids - support for device Ids - HID, UID, CID, SUB, CLS
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utinit.c b/drivers/acpi/acpica/utinit.c
index 18177e4f26f7..92fbaef161a7 100644
--- a/drivers/acpi/acpica/utinit.c
+++ b/drivers/acpi/acpica/utinit.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utinit - Common ACPI subsystem initialization
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utlock.c b/drivers/acpi/acpica/utlock.c
index 84abdbf5cfca..ee6d72385c5c 100644
--- a/drivers/acpi/acpica/utlock.c
+++ b/drivers/acpi/acpica/utlock.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utlock - Reader/Writer lock interfaces
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utobject.c b/drivers/acpi/acpica/utobject.c
index d3667bfff401..f4aae8f0d3a8 100644
--- a/drivers/acpi/acpica/utobject.c
+++ b/drivers/acpi/acpica/utobject.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utobject - ACPI object create/delete/size/cache routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utosi.c b/drivers/acpi/acpica/utosi.c
index b8ab0a3cb5b9..251bd396c6fd 100644
--- a/drivers/acpi/acpica/utosi.c
+++ b/drivers/acpi/acpica/utosi.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utosi - Support for the _OSI predefined control method
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utpredef.c b/drivers/acpi/acpica/utpredef.c
index 2524f013be7a..29d2977d0746 100644
--- a/drivers/acpi/acpica/utpredef.c
+++ b/drivers/acpi/acpica/utpredef.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utpredef - support functions for predefined names
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utprint.c b/drivers/acpi/acpica/utprint.c
index d5aa2109847f..42b30b9f9312 100644
--- a/drivers/acpi/acpica/utprint.c
+++ b/drivers/acpi/acpica/utprint.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utprint - Formatted printing routines
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utresdecode.c b/drivers/acpi/acpica/utresdecode.c
index 85730fcd7d00..d801d9069841 100644
--- a/drivers/acpi/acpica/utresdecode.c
+++ b/drivers/acpi/acpica/utresdecode.c
@@ -284,4 +284,15 @@ const char *acpi_gbl_ptyp_decode[] = {
 	"Input Schmitt Trigger",
 };
 
+const char *acpi_gbl_clock_input_mode[] = {
+	"Fixed",
+	"Variable",
+};
+
+const char *acpi_gbl_clock_input_scale[] = {
+	"Hz",
+	"KHz",
+	"MHz",
+};
+
 #endif
diff --git a/drivers/acpi/acpica/utresrc.c b/drivers/acpi/acpica/utresrc.c
index 16f9a7035b39..cff7901f7866 100644
--- a/drivers/acpi/acpica/utresrc.c
+++ b/drivers/acpi/acpica/utresrc.c
@@ -57,6 +57,8 @@ const u8 acpi_gbl_resource_aml_sizes[] = {
 	ACPI_AML_SIZE_LARGE(struct aml_resource_pin_group),
 	ACPI_AML_SIZE_LARGE(struct aml_resource_pin_group_function),
 	ACPI_AML_SIZE_LARGE(struct aml_resource_pin_group_config),
+	ACPI_AML_SIZE_LARGE(struct aml_resource_clock_input),
+
 };
 
 const u8 acpi_gbl_resource_aml_serial_bus_sizes[] = {
@@ -114,6 +116,7 @@ static const u8 acpi_gbl_resource_types[] = {
 	ACPI_VARIABLE_LENGTH,	/* 10 pin_group */
 	ACPI_VARIABLE_LENGTH,	/* 11 pin_group_function */
 	ACPI_VARIABLE_LENGTH,	/* 12 pin_group_config */
+	ACPI_VARIABLE_LENGTH,	/* 13 clock_input */
 };
 
 /*******************************************************************************
@@ -358,16 +361,20 @@ acpi_ut_validate_resource(struct acpi_walk_state *walk_state,
 	aml_resource = ACPI_CAST_PTR(union aml_resource, aml);
 	if (resource_type == ACPI_RESOURCE_NAME_SERIAL_BUS) {
 
+		/* Avoid undefined behavior: member access within misaligned address */
+
+		struct aml_resource_common_serialbus common_serial_bus;
+		memcpy(&common_serial_bus, aml_resource,
+		       sizeof(common_serial_bus));
+
 		/* Validate the bus_type field */
 
-		if ((aml_resource->common_serial_bus.type == 0) ||
-		    (aml_resource->common_serial_bus.type >
-		     AML_RESOURCE_MAX_SERIALBUSTYPE)) {
+		if ((common_serial_bus.type == 0) ||
+		    (common_serial_bus.type > AML_RESOURCE_MAX_SERIALBUSTYPE)) {
 			if (walk_state) {
 				ACPI_ERROR((AE_INFO,
 					    "Invalid/unsupported SerialBus resource descriptor: BusType 0x%2.2X",
-					    aml_resource->common_serial_bus.
-					    type));
+					    common_serial_bus.type));
 			}
 			return (AE_AML_INVALID_RESOURCE_TYPE);
 		}
diff --git a/drivers/acpi/acpica/uttrack.c b/drivers/acpi/acpica/uttrack.c
index a06988ac409d..f5f5da441458 100644
--- a/drivers/acpi/acpica/uttrack.c
+++ b/drivers/acpi/acpica/uttrack.c
@@ -3,7 +3,7 @@
  *
  * Module Name: uttrack - Memory allocation tracking routines (debug only)
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utuuid.c b/drivers/acpi/acpica/utuuid.c
index e24bc670b53e..8f10b413e928 100644
--- a/drivers/acpi/acpica/utuuid.c
+++ b/drivers/acpi/acpica/utuuid.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utuuid -- UUID support functions
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utxface.c b/drivers/acpi/acpica/utxface.c
index 86e76b443da7..aa2e923462b7 100644
--- a/drivers/acpi/acpica/utxface.c
+++ b/drivers/acpi/acpica/utxface.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utxface - External interfaces, miscellaneous utility functions
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/acpica/utxfinit.c b/drivers/acpi/acpica/utxfinit.c
index f2acec3a0ee3..1915bec2b279 100644
--- a/drivers/acpi/acpica/utxfinit.c
+++ b/drivers/acpi/acpica/utxfinit.c
@@ -3,7 +3,7 @@
  *
  * Module Name: utxfinit - External interfaces for ACPICA initialization
  *
- * Copyright (C) 2000 - 2022, Intel Corp.
+ * Copyright (C) 2000 - 2023, Intel Corp.
  *
  *****************************************************************************/
 
diff --git a/drivers/acpi/apei/einj.c b/drivers/acpi/apei/einj.c
index b4373e575660..013eb621dc92 100644
--- a/drivers/acpi/apei/einj.c
+++ b/drivers/acpi/apei/einj.c
@@ -489,9 +489,15 @@ static int __einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2,
 	if (rc)
 		return rc;
 	val = apei_exec_ctx_get_output(&ctx);
-	if (val != EINJ_STATUS_SUCCESS)
+	if (val == EINJ_STATUS_FAIL)
 		return -EBUSY;
+	else if (val == EINJ_STATUS_INVAL)
+		return -EINVAL;
 
+	/*
+	 * The error is injected into the platform successfully, then it needs
+	 * to trigger the error.
+	 */
 	rc = apei_exec_run(&ctx, ACPI_EINJ_GET_TRIGGER_TABLE);
 	if (rc)
 		return rc;
@@ -584,6 +590,12 @@ static const char * const einj_error_type_string[] = {
 	"0x00000200\tPlatform Correctable\n",
 	"0x00000400\tPlatform Uncorrectable non-fatal\n",
 	"0x00000800\tPlatform Uncorrectable fatal\n",
+	"0x00001000\tCXL.cache Protocol Correctable\n",
+	"0x00002000\tCXL.cache Protocol Uncorrectable non-fatal\n",
+	"0x00004000\tCXL.cache Protocol Uncorrectable fatal\n",
+	"0x00008000\tCXL.mem Protocol Correctable\n",
+	"0x00010000\tCXL.mem Protocol Uncorrectable non-fatal\n",
+	"0x00020000\tCXL.mem Protocol Uncorrectable fatal\n",
 };
 
 static int available_error_type_show(struct seq_file *m, void *v)
diff --git a/drivers/acpi/arm64/agdi.c b/drivers/acpi/arm64/agdi.c
index cf31abd0ed1b..f605302395c3 100644
--- a/drivers/acpi/arm64/agdi.c
+++ b/drivers/acpi/arm64/agdi.c
@@ -64,8 +64,11 @@ static int agdi_remove(struct platform_device *pdev)
 	int err, i;
 
 	err = sdei_event_disable(adata->sdei_event);
-	if (err)
-		return err;
+	if (err) {
+		dev_err(&pdev->dev, "Failed to disable sdei-event #%d (%pe)\n",
+			adata->sdei_event, ERR_PTR(err));
+		return 0;
+	}
 
 	for (i = 0; i < 3; i++) {
 		err = sdei_event_unregister(adata->sdei_event);
@@ -75,7 +78,11 @@ static int agdi_remove(struct platform_device *pdev)
 		schedule();
 	}
 
-	return err;
+	if (err)
+		dev_err(&pdev->dev, "Failed to unregister sdei-event #%d (%pe)\n",
+			adata->sdei_event, ERR_PTR(err));
+
+	return 0;
 }
 
 static struct platform_driver agdi_driver = {
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index a96da65057b1..d161ff707de4 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -589,6 +589,7 @@ static void acpi_device_remove_notify_handler(struct acpi_device *device,
 		acpi_remove_notify_handler(device->handle, type,
 					   acpi_notify_device);
 	}
+	acpi_os_wait_events_complete();
 }
 
 /* Handle events targeting \_SB device (at present only graceful shutdown) */
@@ -623,8 +624,9 @@ static void acpi_sb_notify(acpi_handle handle, u32 event, void *data)
 	if (event == ACPI_SB_NOTIFY_SHUTDOWN_REQUEST) {
 		if (!work_busy(&acpi_sb_work))
 			schedule_work(&acpi_sb_work);
-	} else
+	} else {
 		pr_warn("event %x is not supported by \\_SB device\n", event);
+	}
 }
 
 static int __init acpi_setup_sb_notify_handler(void)
@@ -808,9 +810,10 @@ static bool acpi_of_modalias(struct acpi_device *adev,
  * @modalias:   Pointer to buffer that modalias value will be copied into
  * @len:	Length of modalias buffer
  *
- * This is a counterpart of of_modalias_node() for struct acpi_device objects.
- * If there is a compatible string for @adev, it will be copied to @modalias
- * with the vendor prefix stripped; otherwise, @default_id will be used.
+ * This is a counterpart of of_alias_from_compatible() for struct acpi_device
+ * objects. If there is a compatible string for @adev, it will be copied to
+ * @modalias with the vendor prefix stripped; otherwise, @default_id will be
+ * used.
  */
 void acpi_set_modalias(struct acpi_device *adev, const char *default_id,
 		       char *modalias, size_t len)
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index c51d3ccb4cca..7ff269a78c20 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -1434,6 +1434,102 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable)
 EXPORT_SYMBOL_GPL(cppc_set_epp_perf);
 
 /**
+ * cppc_get_auto_sel_caps - Read autonomous selection register.
+ * @cpunum : CPU from which to read register.
+ * @perf_caps : struct where autonomous selection register value is updated.
+ */
+int cppc_get_auto_sel_caps(int cpunum, struct cppc_perf_caps *perf_caps)
+{
+	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum);
+	struct cpc_register_resource *auto_sel_reg;
+	u64  auto_sel;
+
+	if (!cpc_desc) {
+		pr_debug("No CPC descriptor for CPU:%d\n", cpunum);
+		return -ENODEV;
+	}
+
+	auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE];
+
+	if (!CPC_SUPPORTED(auto_sel_reg))
+		pr_warn_once("Autonomous mode is not unsupported!\n");
+
+	if (CPC_IN_PCC(auto_sel_reg)) {
+		int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum);
+		struct cppc_pcc_data *pcc_ss_data = NULL;
+		int ret = 0;
+
+		if (pcc_ss_id < 0)
+			return -ENODEV;
+
+		pcc_ss_data = pcc_data[pcc_ss_id];
+
+		down_write(&pcc_ss_data->pcc_lock);
+
+		if (send_pcc_cmd(pcc_ss_id, CMD_READ) >= 0) {
+			cpc_read(cpunum, auto_sel_reg, &auto_sel);
+			perf_caps->auto_sel = (bool)auto_sel;
+		} else {
+			ret = -EIO;
+		}
+
+		up_write(&pcc_ss_data->pcc_lock);
+
+		return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cppc_get_auto_sel_caps);
+
+/**
+ * cppc_set_auto_sel - Write autonomous selection register.
+ * @cpu    : CPU to which to write register.
+ * @enable : the desired value of autonomous selection resiter to be updated.
+ */
+int cppc_set_auto_sel(int cpu, bool enable)
+{
+	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu);
+	struct cpc_register_resource *auto_sel_reg;
+	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu);
+	struct cppc_pcc_data *pcc_ss_data = NULL;
+	int ret = -EINVAL;
+
+	if (!cpc_desc) {
+		pr_debug("No CPC descriptor for CPU:%d\n", cpu);
+		return -ENODEV;
+	}
+
+	auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE];
+
+	if (CPC_IN_PCC(auto_sel_reg)) {
+		if (pcc_ss_id < 0) {
+			pr_debug("Invalid pcc_ss_id\n");
+			return -ENODEV;
+		}
+
+		if (CPC_SUPPORTED(auto_sel_reg)) {
+			ret = cpc_write(cpu, auto_sel_reg, enable);
+			if (ret)
+				return ret;
+		}
+
+		pcc_ss_data = pcc_data[pcc_ss_id];
+
+		down_write(&pcc_ss_data->pcc_lock);
+		/* after writing CPC, transfer the ownership of PCC to platform */
+		ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE);
+		up_write(&pcc_ss_data->pcc_lock);
+	} else {
+		ret = -ENOTSUPP;
+		pr_debug("_CPC in PCC is not supported\n");
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(cppc_set_auto_sel);
+
+/**
  * cppc_set_enable - Set to enable CPPC on the processor by writing the
  * Continuous Performance Control package EnableRegister field.
  * @cpu: CPU for which to enable CPPC register.
@@ -1488,7 +1584,7 @@ EXPORT_SYMBOL_GPL(cppc_set_enable);
 int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 {
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu);
-	struct cpc_register_resource *desired_reg;
+	struct cpc_register_resource *desired_reg, *min_perf_reg, *max_perf_reg;
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu);
 	struct cppc_pcc_data *pcc_ss_data = NULL;
 	int ret = 0;
@@ -1499,6 +1595,8 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 	}
 
 	desired_reg = &cpc_desc->cpc_regs[DESIRED_PERF];
+	min_perf_reg = &cpc_desc->cpc_regs[MIN_PERF];
+	max_perf_reg = &cpc_desc->cpc_regs[MAX_PERF];
 
 	/*
 	 * This is Phase-I where we want to write to CPC registers
@@ -1507,7 +1605,7 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 	 * Since read_lock can be acquired by multiple CPUs simultaneously we
 	 * achieve that goal here
 	 */
-	if (CPC_IN_PCC(desired_reg)) {
+	if (CPC_IN_PCC(desired_reg) || CPC_IN_PCC(min_perf_reg) || CPC_IN_PCC(max_perf_reg)) {
 		if (pcc_ss_id < 0) {
 			pr_debug("Invalid pcc_ss_id\n");
 			return -ENODEV;
@@ -1530,13 +1628,19 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 		cpc_desc->write_cmd_status = 0;
 	}
 
+	cpc_write(cpu, desired_reg, perf_ctrls->desired_perf);
+
 	/*
-	 * Skip writing MIN/MAX until Linux knows how to come up with
-	 * useful values.
+	 * Only write if min_perf and max_perf not zero. Some drivers pass zero
+	 * value to min and max perf, but they don't mean to set the zero value,
+	 * they just don't want to write to those registers.
 	 */
-	cpc_write(cpu, desired_reg, perf_ctrls->desired_perf);
+	if (perf_ctrls->min_perf)
+		cpc_write(cpu, min_perf_reg, perf_ctrls->min_perf);
+	if (perf_ctrls->max_perf)
+		cpc_write(cpu, max_perf_reg, perf_ctrls->max_perf);
 
-	if (CPC_IN_PCC(desired_reg))
+	if (CPC_IN_PCC(desired_reg) || CPC_IN_PCC(min_perf_reg) || CPC_IN_PCC(max_perf_reg))
 		up_read(&pcc_ss_data->pcc_lock);	/* END Phase-I */
 	/*
 	 * This is Phase-II where we transfer the ownership of PCC to Platform
@@ -1584,7 +1688,7 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 	 * case during a CMD_READ and if there are pending writes it delivers
 	 * the write command before servicing the read command
 	 */
-	if (CPC_IN_PCC(desired_reg)) {
+	if (CPC_IN_PCC(desired_reg) || CPC_IN_PCC(min_perf_reg) || CPC_IN_PCC(max_perf_reg)) {
 		if (down_write_trylock(&pcc_ss_data->pcc_lock)) {/* BEGIN Phase-II */
 			/* Update only if there are pending write commands */
 			if (pcc_ss_data->pending_pcc_write_cmd)
diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c
index 105d2e795afa..928899ab9502 100644
--- a/drivers/acpi/ec.c
+++ b/drivers/acpi/ec.c
@@ -1083,9 +1083,12 @@ int acpi_ec_add_query_handler(struct acpi_ec *ec, u8 query_bit,
 			      acpi_handle handle, acpi_ec_query_func func,
 			      void *data)
 {
-	struct acpi_ec_query_handler *handler =
-	    kzalloc(sizeof(struct acpi_ec_query_handler), GFP_KERNEL);
+	struct acpi_ec_query_handler *handler;
+
+	if (!handle && !func)
+		return -EINVAL;
 
+	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
 	if (!handler)
 		return -ENOMEM;
 
@@ -1097,6 +1100,7 @@ int acpi_ec_add_query_handler(struct acpi_ec *ec, u8 query_bit,
 	kref_init(&handler->kref);
 	list_add(&handler->node, &ec->list);
 	mutex_unlock(&ec->mutex);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(acpi_ec_add_query_handler);
@@ -1109,9 +1113,16 @@ static void acpi_ec_remove_query_handlers(struct acpi_ec *ec,
 
 	mutex_lock(&ec->mutex);
 	list_for_each_entry_safe(handler, tmp, &ec->list, node) {
-		if (remove_all || query_bit == handler->query_bit) {
+		/*
+		 * When remove_all is false, only remove custom query handlers
+		 * which have handler->func set. This is done to preserve query
+		 * handlers discovered thru ACPI, as they should continue handling
+		 * EC queries.
+		 */
+		if (remove_all || (handler->func && handler->query_bit == query_bit)) {
 			list_del_init(&handler->node);
 			list_add(&handler->node, &free_list);
+
 		}
 	}
 	mutex_unlock(&ec->mutex);
@@ -1122,6 +1133,7 @@ static void acpi_ec_remove_query_handlers(struct acpi_ec *ec,
 void acpi_ec_remove_query_handler(struct acpi_ec *ec, u8 query_bit)
 {
 	acpi_ec_remove_query_handlers(ec, false, query_bit);
+	flush_workqueue(ec_query_wq);
 }
 EXPORT_SYMBOL_GPL(acpi_ec_remove_query_handler);
 
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index 4e48d6db05eb..07204d482968 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -894,7 +894,7 @@ static size_t sizeof_flush(struct acpi_nfit_flush_address *flush)
 {
 	if (flush->header.length < sizeof(*flush))
 		return 0;
-	return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1);
+	return struct_size(flush, hint_address, flush->hint_count);
 }
 
 static bool add_flush(struct acpi_nfit_desc *acpi_desc,
@@ -3476,8 +3476,8 @@ static __init int nfit_init(void)
 	BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
 	BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 64);
 	BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
-	BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
-	BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
+	BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 16);
+	BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 8);
 	BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
 	BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
 	BUILD_BUG_ON(sizeof(struct acpi_nfit_capabilities) != 16);
diff --git a/drivers/acpi/processor_pdc.c b/drivers/acpi/processor_pdc.c
index 8c3f82c9fff3..18fb04523f93 100644
--- a/drivers/acpi/processor_pdc.c
+++ b/drivers/acpi/processor_pdc.c
@@ -14,6 +14,8 @@
 #include <linux/acpi.h>
 #include <acpi/processor.h>
 
+#include <xen/xen.h>
+
 #include "internal.h"
 
 static bool __init processor_physically_present(acpi_handle handle)
@@ -47,6 +49,15 @@ static bool __init processor_physically_present(acpi_handle handle)
 		return false;
 	}
 
+	if (xen_initial_domain())
+		/*
+		 * When running as a Xen dom0 the number of processors Linux
+		 * sees can be different from the real number of processors on
+		 * the system, and we still need to execute _PDC for all of
+		 * them.
+		 */
+		return xen_processor_present(acpi_id);
+
 	type = (acpi_type == ACPI_TYPE_DEVICE) ? 1 : 0;
 	cpuid = acpi_get_cpuid(handle, type, acpi_id);
 
diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c
index b8d9eb9a433e..413e4fcadcaf 100644
--- a/drivers/acpi/property.c
+++ b/drivers/acpi/property.c
@@ -971,60 +971,48 @@ static int acpi_data_prop_read_single(const struct acpi_device_data *data,
 				      enum dev_prop_type proptype, void *val)
 {
 	const union acpi_object *obj;
-	int ret;
+	int ret = 0;
 
-	if (proptype >= DEV_PROP_U8 && proptype <= DEV_PROP_U64) {
+	if (proptype >= DEV_PROP_U8 && proptype <= DEV_PROP_U64)
 		ret = acpi_data_get_property(data, propname, ACPI_TYPE_INTEGER, &obj);
-		if (ret)
-			return ret;
-
-		switch (proptype) {
-		case DEV_PROP_U8:
-			if (obj->integer.value > U8_MAX)
-				return -EOVERFLOW;
-
-			if (val)
-				*(u8 *)val = obj->integer.value;
-
-			break;
-		case DEV_PROP_U16:
-			if (obj->integer.value > U16_MAX)
-				return -EOVERFLOW;
-
-			if (val)
-				*(u16 *)val = obj->integer.value;
-
-			break;
-		case DEV_PROP_U32:
-			if (obj->integer.value > U32_MAX)
-				return -EOVERFLOW;
-
-			if (val)
-				*(u32 *)val = obj->integer.value;
-
-			break;
-		default:
-			if (val)
-				*(u64 *)val = obj->integer.value;
-
-			break;
-		}
-
-		if (!val)
-			return 1;
-	} else if (proptype == DEV_PROP_STRING) {
+	else if (proptype == DEV_PROP_STRING)
 		ret = acpi_data_get_property(data, propname, ACPI_TYPE_STRING, &obj);
-		if (ret)
-			return ret;
+	if (ret)
+		return ret;
 
+	switch (proptype) {
+	case DEV_PROP_U8:
+		if (obj->integer.value > U8_MAX)
+			return -EOVERFLOW;
+		if (val)
+			*(u8 *)val = obj->integer.value;
+		break;
+	case DEV_PROP_U16:
+		if (obj->integer.value > U16_MAX)
+			return -EOVERFLOW;
+		if (val)
+			*(u16 *)val = obj->integer.value;
+		break;
+	case DEV_PROP_U32:
+		if (obj->integer.value > U32_MAX)
+			return -EOVERFLOW;
+		if (val)
+			*(u32 *)val = obj->integer.value;
+		break;
+	case DEV_PROP_U64:
+		if (val)
+			*(u64 *)val = obj->integer.value;
+		break;
+	case DEV_PROP_STRING:
 		if (val)
 			*(char **)val = obj->string.pointer;
-
 		return 1;
-	} else {
-		ret = -EINVAL;
+	default:
+		return -EINVAL;
 	}
-	return ret;
+
+	/* When no storage provided return number of available values */
+	return val ? 0 : 1;
 }
 
 #define acpi_copy_property_array_uint(items, val, nval)			\
diff --git a/drivers/acpi/sbs.c b/drivers/acpi/sbs.c
index e90752d4f488..94e3c000df2e 100644
--- a/drivers/acpi/sbs.c
+++ b/drivers/acpi/sbs.c
@@ -473,23 +473,32 @@ static const struct device_attribute alarm_attr = {
    -------------------------------------------------------------------------- */
 static int acpi_battery_read(struct acpi_battery *battery)
 {
-	int result = 0, saved_present = battery->present;
+	int result, saved_present = battery->present;
 	u16 state;
 
 	if (battery->sbs->manager_present) {
 		result = acpi_smbus_read(battery->sbs->hc, SMBUS_READ_WORD,
 				ACPI_SBS_MANAGER, 0x01, (u8 *)&state);
-		if (!result)
-			battery->present = state & (1 << battery->id);
-		state &= 0x0fff;
+		if (result)
+			return result;
+
+		battery->present = state & (1 << battery->id);
+		if (!battery->present)
+			return 0;
+
+		/* Masking necessary for Smart Battery Selectors */
+		state = 0x0fff;
 		state |= 1 << (battery->id + 12);
 		acpi_smbus_write(battery->sbs->hc, SMBUS_WRITE_WORD,
 				  ACPI_SBS_MANAGER, 0x01, (u8 *)&state, 2);
-	} else if (battery->id == 0)
-		battery->present = 1;
-
-	if (result || !battery->present)
-		return result;
+	} else {
+		if (battery->id == 0) {
+			battery->present = 1;
+		} else {
+			if (!battery->present)
+				return 0;
+		}
+	}
 
 	if (saved_present != battery->present) {
 		battery->update_time = 0;
diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c
index 4ca667251272..72470b9f16c4 100644
--- a/drivers/acpi/sleep.c
+++ b/drivers/acpi/sleep.c
@@ -714,7 +714,13 @@ int acpi_s2idle_begin(void)
 int acpi_s2idle_prepare(void)
 {
 	if (acpi_sci_irq_valid()) {
-		enable_irq_wake(acpi_sci_irq);
+		int error;
+
+		error = enable_irq_wake(acpi_sci_irq);
+		if (error)
+			pr_warn("Warning: Failed to enable wakeup from IRQ %d: %d\n",
+				acpi_sci_irq, error);
+
 		acpi_ec_set_gpe_wake_mask(ACPI_GPE_ENABLE);
 	}
 
diff --git a/drivers/acpi/spcr.c b/drivers/acpi/spcr.c
index 1eabfcd122ee..cd36a97b0ea2 100644
--- a/drivers/acpi/spcr.c
+++ b/drivers/acpi/spcr.c
@@ -71,7 +71,6 @@ static bool xgene_8250_erratum_present(struct acpi_table_spcr *tb)
 
 /**
  * acpi_parse_spcr() - parse ACPI SPCR table and add preferred console
- *
  * @enable_earlycon: set up earlycon for the console specified by the table
  * @enable_console: setup the console specified by the table.
  *
@@ -82,7 +81,6 @@ static bool xgene_8250_erratum_present(struct acpi_table_spcr *tb)
  *
  * When CONFIG_ACPI_SPCR_TABLE is defined, this function should be called
  * from arch initialization code as soon as the DT/ACPI decision is made.
- *
  */
 int __init acpi_parse_spcr(bool enable_earlycon, bool enable_console)
 {
@@ -97,9 +95,7 @@ int __init acpi_parse_spcr(bool enable_earlycon, bool enable_console)
 	if (acpi_disabled)
 		return -ENODEV;
 
-	status = acpi_get_table(ACPI_SIG_SPCR, 0,
-				(struct acpi_table_header **)&table);
-
+	status = acpi_get_table(ACPI_SIG_SPCR, 0, (struct acpi_table_header **)&table);
 	if (ACPI_FAILURE(status))
 		return -ENOENT;
 
@@ -110,12 +106,12 @@ int __init acpi_parse_spcr(bool enable_earlycon, bool enable_console)
 		u32 bit_width = table->serial_port.access_width;
 
 		if (bit_width > ACPI_ACCESS_BIT_MAX) {
-			pr_err("Unacceptable wide SPCR Access Width.  Defaulting to byte size\n");
+			pr_err(FW_BUG "Unacceptable wide SPCR Access Width. Defaulting to byte size\n");
 			bit_width = ACPI_ACCESS_BIT_DEFAULT;
 		}
 		switch (ACPI_ACCESS_BIT_WIDTH((bit_width))) {
 		default:
-			pr_err("Unexpected SPCR Access Width.  Defaulting to byte size\n");
+			pr_err(FW_BUG "Unexpected SPCR Access Width. Defaulting to byte size\n");
 			fallthrough;
 		case 8:
 			iotype = "mmio";
@@ -202,7 +198,8 @@ int __init acpi_parse_spcr(bool enable_earlycon, bool enable_console)
 	if (xgene_8250_erratum_present(table)) {
 		iotype = "mmio32";
 
-		/* for xgene v1 and v2 we don't know the clock rate of the
+		/*
+		 * For xgene v1 and v2 we don't know the clock rate of the
 		 * UART so don't attempt to change to the baud rate state
 		 * in the table because driver cannot calculate the dividers
 		 */
diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c
index 7f4ff56c9d42..687524b50085 100644
--- a/drivers/acpi/sysfs.c
+++ b/drivers/acpi/sysfs.c
@@ -458,11 +458,28 @@ static int acpi_bert_data_init(void *th, struct acpi_data_attr *data_attr)
 	return sysfs_create_bin_file(tables_data_kobj, &data_attr->attr);
 }
 
+static int acpi_ccel_data_init(void *th, struct acpi_data_attr *data_attr)
+{
+	struct acpi_table_ccel *ccel = th;
+
+	if (ccel->header.length < sizeof(struct acpi_table_ccel) ||
+	    !ccel->log_area_start_address || !ccel->log_area_minimum_length) {
+		kfree(data_attr);
+		return -EINVAL;
+	}
+	data_attr->addr = ccel->log_area_start_address;
+	data_attr->attr.size = ccel->log_area_minimum_length;
+	data_attr->attr.attr.name = "CCEL";
+
+	return sysfs_create_bin_file(tables_data_kobj, &data_attr->attr);
+}
+
 static struct acpi_data_obj {
 	char *name;
 	int (*fn)(void *, struct acpi_data_attr *);
 } acpi_data_objs[] = {
 	{ ACPI_SIG_BERT, acpi_bert_data_init },
+	{ ACPI_SIG_CCEL, acpi_ccel_data_init },
 };
 
 #define NUM_ACPI_DATA_OBJS ARRAY_SIZE(acpi_data_objs)
diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c
index 0b4b844f9d4c..ec002ecfe4cf 100644
--- a/drivers/acpi/thermal.c
+++ b/drivers/acpi/thermal.c
@@ -419,10 +419,9 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag)
 					 * the next higher trip point
 					 */
 					tz->trips.active[i-1].temperature =
-						(tz->trips.active[i-2].temperature <
-						celsius_to_deci_kelvin(act) ?
-						tz->trips.active[i-2].temperature :
-						celsius_to_deci_kelvin(act));
+						min_t(unsigned long,
+						      tz->trips.active[i-2].temperature,
+						      celsius_to_deci_kelvin(act));
 
 				break;
 			} else {
@@ -498,7 +497,7 @@ static int acpi_thermal_get_trip_points(struct acpi_thermal *tz)
 
 static int thermal_get_temp(struct thermal_zone_device *thermal, int *temp)
 {
-	struct acpi_thermal *tz = thermal->devdata;
+	struct acpi_thermal *tz = thermal_zone_device_priv(thermal);
 	int result;
 
 	if (!tz)
@@ -516,7 +515,7 @@ static int thermal_get_temp(struct thermal_zone_device *thermal, int *temp)
 static int thermal_get_trip_type(struct thermal_zone_device *thermal,
 				 int trip, enum thermal_trip_type *type)
 {
-	struct acpi_thermal *tz = thermal->devdata;
+	struct acpi_thermal *tz = thermal_zone_device_priv(thermal);
 	int i;
 
 	if (!tz || trip < 0)
@@ -560,7 +559,7 @@ static int thermal_get_trip_type(struct thermal_zone_device *thermal,
 static int thermal_get_trip_temp(struct thermal_zone_device *thermal,
 				 int trip, int *temp)
 {
-	struct acpi_thermal *tz = thermal->devdata;
+	struct acpi_thermal *tz = thermal_zone_device_priv(thermal);
 	int i;
 
 	if (!tz || trip < 0)
@@ -613,7 +612,7 @@ static int thermal_get_trip_temp(struct thermal_zone_device *thermal,
 static int thermal_get_crit_temp(struct thermal_zone_device *thermal,
 				int *temperature)
 {
-	struct acpi_thermal *tz = thermal->devdata;
+	struct acpi_thermal *tz = thermal_zone_device_priv(thermal);
 
 	if (tz->trips.critical.flags.valid) {
 		*temperature = deci_kelvin_to_millicelsius_with_offset(
@@ -628,7 +627,7 @@ static int thermal_get_crit_temp(struct thermal_zone_device *thermal,
 static int thermal_get_trend(struct thermal_zone_device *thermal,
 			     int trip, enum thermal_trend *trend)
 {
-	struct acpi_thermal *tz = thermal->devdata;
+	struct acpi_thermal *tz = thermal_zone_device_priv(thermal);
 	enum thermal_trip_type type;
 	int i;
 
@@ -670,7 +669,7 @@ static int thermal_get_trend(struct thermal_zone_device *thermal,
 
 static void acpi_thermal_zone_device_hot(struct thermal_zone_device *thermal)
 {
-	struct acpi_thermal *tz = thermal->devdata;
+	struct acpi_thermal *tz = thermal_zone_device_priv(thermal);
 
 	acpi_bus_generate_netlink_event(tz->device->pnp.device_class,
 					dev_name(&tz->device->dev),
@@ -679,7 +678,7 @@ static void acpi_thermal_zone_device_hot(struct thermal_zone_device *thermal)
 
 static void acpi_thermal_zone_device_critical(struct thermal_zone_device *thermal)
 {
-	struct acpi_thermal *tz = thermal->devdata;
+	struct acpi_thermal *tz = thermal_zone_device_priv(thermal);
 
 	acpi_bus_generate_netlink_event(tz->device->pnp.device_class,
 					dev_name(&tz->device->dev),
@@ -693,7 +692,7 @@ static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal,
 					  bool bind)
 {
 	struct acpi_device *device = cdev->devdata;
-	struct acpi_thermal *tz = thermal->devdata;
+	struct acpi_thermal *tz = thermal_zone_device_priv(thermal);
 	struct acpi_device *dev;
 	acpi_handle handle;
 	int i;
@@ -842,7 +841,7 @@ static int acpi_thermal_register_thermal_zone(struct acpi_thermal *tz)
 		goto acpi_bus_detach;
 
 	dev_info(&tz->device->dev, "registered as thermal_zone%d\n",
-		 tz->thermal_zone->id);
+		 thermal_zone_device_id(tz->thermal_zone));
 
 	return 0;
 
diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c
index e85729fc481f..69ef2d9710c2 100644
--- a/drivers/acpi/video_detect.c
+++ b/drivers/acpi/video_detect.c
@@ -130,12 +130,6 @@ static int video_detect_force_native(const struct dmi_system_id *d)
 	return 0;
 }
 
-static int video_detect_force_none(const struct dmi_system_id *d)
-{
-	acpi_backlight_dmi = acpi_backlight_none;
-	return 0;
-}
-
 static const struct dmi_system_id video_detect_dmi_table[] = {
 	/*
 	 * Models which should use the vendor backlight interface,
@@ -768,35 +762,6 @@ static const struct dmi_system_id video_detect_dmi_table[] = {
 		DMI_MATCH(DMI_PRODUCT_NAME, "Vostro 15 3535"),
 		},
 	},
-
-	/*
-	 * Desktops which falsely report a backlight and which our heuristics
-	 * for this do not catch.
-	 */
-	{
-	 .callback = video_detect_force_none,
-	 /* Dell OptiPlex 9020M */
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
-		DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex 9020M"),
-		},
-	},
-	{
-	 .callback = video_detect_force_none,
-	 /* GIGABYTE GB-BXBT-2807 */
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "GIGABYTE"),
-		DMI_MATCH(DMI_PRODUCT_NAME, "GB-BXBT-2807"),
-		},
-	},
-	{
-	 .callback = video_detect_force_none,
-	 /* MSI MS-7721 */
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "MSI"),
-		DMI_MATCH(DMI_PRODUCT_NAME, "MS-7721"),
-		},
-	},
 	{ },
 };
 
diff --git a/drivers/acpi/viot.c b/drivers/acpi/viot.c
index ed752cbbe636..c8025921c129 100644
--- a/drivers/acpi/viot.c
+++ b/drivers/acpi/viot.c
@@ -328,6 +328,7 @@ static int viot_pci_dev_iommu_init(struct pci_dev *pdev, u16 dev_id, void *data)
 {
 	u32 epid;
 	struct viot_endpoint *ep;
+	struct device *aliased_dev = data;
 	u32 domain_nr = pci_domain_nr(pdev->bus);
 
 	list_for_each_entry(ep, &viot_pci_ranges, list) {
@@ -338,7 +339,7 @@ static int viot_pci_dev_iommu_init(struct pci_dev *pdev, u16 dev_id, void *data)
 			epid = ((domain_nr - ep->segment_start) << 16) +
 				dev_id - ep->bdf_start + ep->endpoint_id;
 
-			return viot_dev_iommu_init(&pdev->dev, ep->viommu,
+			return viot_dev_iommu_init(aliased_dev, ep->viommu,
 						   epid);
 		}
 	}
@@ -372,7 +373,7 @@ int viot_iommu_configure(struct device *dev)
 {
 	if (dev_is_pci(dev))
 		return pci_for_each_dma_alias(to_pci_dev(dev),
-					      viot_pci_dev_iommu_init, NULL);
+					      viot_pci_dev_iommu_init, dev);
 	else if (dev_is_platform(dev))
 		return viot_mmio_dev_iommu_init(to_platform_device(dev));
 	return -ENODEV;
diff --git a/drivers/amba/tegra-ahb.c b/drivers/amba/tegra-ahb.c
index 0b2c20fddb7c..c0e8b765522d 100644
--- a/drivers/amba/tegra-ahb.c
+++ b/drivers/amba/tegra-ahb.c
@@ -285,5 +285,4 @@ module_platform_driver(tegra_ahb_driver);
 
 MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
 MODULE_DESCRIPTION("Tegra AHB driver");
-MODULE_LICENSE("GPL v2");
 MODULE_ALIAS("platform:" DRV_NAME);
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig
index b56fba76b43f..42b51c9812a0 100644
--- a/drivers/ata/Kconfig
+++ b/drivers/ata/Kconfig
@@ -276,6 +276,7 @@ config AHCI_XGENE
 config AHCI_QORIQ
 	tristate "Freescale QorIQ AHCI SATA support"
 	depends on OF
+	depends on SOC_LS1021A || ARCH_LAYERSCAPE || COMPILE_TEST
 	select SATA_HOST
 	help
 	  This option enables support for the Freescale QorIQ AHCI SoC's
diff --git a/drivers/ata/acard-ahci.c b/drivers/ata/acard-ahci.c
index 993eadd173da..547f56341705 100644
--- a/drivers/ata/acard-ahci.c
+++ b/drivers/ata/acard-ahci.c
@@ -66,7 +66,7 @@ static int acard_ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg
 static int acard_ahci_pci_device_resume(struct pci_dev *pdev);
 #endif
 
-static struct scsi_host_template acard_ahci_sht = {
+static const struct scsi_host_template acard_ahci_sht = {
 	AHCI_SHT("acard-ahci"),
 };
 
diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index 14a1c0d14916..addba109406b 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -101,7 +101,7 @@ static int ahci_pci_device_resume(struct device *dev);
 #endif
 #endif /* CONFIG_PM */
 
-static struct scsi_host_template ahci_sht = {
+static const struct scsi_host_template ahci_sht = {
 	AHCI_SHT("ahci"),
 };
 
diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h
index ff8e6ae1c636..4bae95b06ae3 100644
--- a/drivers/ata/ahci.h
+++ b/drivers/ata/ahci.h
@@ -430,7 +430,7 @@ void ahci_set_em_messages(struct ahci_host_priv *hpriv,
 			  struct ata_port_info *pi);
 int ahci_reset_em(struct ata_host *host);
 void ahci_print_info(struct ata_host *host, const char *scc_s);
-int ahci_host_activate(struct ata_host *host, struct scsi_host_template *sht);
+int ahci_host_activate(struct ata_host *host, const struct scsi_host_template *sht);
 void ahci_error_handler(struct ata_port *ap);
 u32 ahci_handle_port_intr(struct ata_host *host, u32 irq_masked);
 
diff --git a/drivers/ata/ahci_brcm.c b/drivers/ata/ahci_brcm.c
index 6f216eb25610..4e3dc2b6d67f 100644
--- a/drivers/ata/ahci_brcm.c
+++ b/drivers/ata/ahci_brcm.c
@@ -417,7 +417,7 @@ out_disable_clks:
 	return ret;
 }
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_ceva.c b/drivers/ata/ahci_ceva.c
index cb24ecf36faf..bc027468decb 100644
--- a/drivers/ata/ahci_ceva.c
+++ b/drivers/ata/ahci_ceva.c
@@ -185,7 +185,7 @@ static void ahci_ceva_setup(struct ahci_host_priv *hpriv)
 	}
 }
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_da850.c b/drivers/ata/ahci_da850.c
index dc8a019b8340..ca0924dc5bd2 100644
--- a/drivers/ata/ahci_da850.c
+++ b/drivers/ata/ahci_da850.c
@@ -153,7 +153,7 @@ static const struct ata_port_info ahci_da850_port_info = {
 	.port_ops	= &ahci_da850_port_ops,
 };
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_dm816.c b/drivers/ata/ahci_dm816.c
index d26efcd20f64..b08547b877a1 100644
--- a/drivers/ata/ahci_dm816.c
+++ b/drivers/ata/ahci_dm816.c
@@ -134,7 +134,7 @@ static const struct ata_port_info ahci_dm816_port_info = {
 	.port_ops	= &ahci_dm816_port_ops,
 };
 
-static struct scsi_host_template ahci_dm816_platform_sht = {
+static const struct scsi_host_template ahci_dm816_platform_sht = {
 	AHCI_SHT(AHCI_DM816_DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_dwc.c b/drivers/ata/ahci_dwc.c
index 8fb66860db31..4bfbb09cdc02 100644
--- a/drivers/ata/ahci_dwc.c
+++ b/drivers/ata/ahci_dwc.c
@@ -398,7 +398,7 @@ static const struct ata_port_info ahci_dwc_port_info = {
 	.port_ops	= &ahci_dwc_port_ops,
 };
 
-static struct scsi_host_template ahci_dwc_scsi_info = {
+static const struct scsi_host_template ahci_dwc_scsi_info = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_imx.c b/drivers/ata/ahci_imx.c
index a950767f7948..3a8c248e7c0e 100644
--- a/drivers/ata/ahci_imx.c
+++ b/drivers/ata/ahci_imx.c
@@ -418,7 +418,7 @@ static int __sata_ahci_read_temperature(void *dev, int *temp)
 
 static int sata_ahci_read_temperature(struct thermal_zone_device *tz, int *temp)
 {
-	return __sata_ahci_read_temperature(tz->devdata, temp);
+	return __sata_ahci_read_temperature(thermal_zone_device_priv(tz), temp);
 }
 
 static ssize_t sata_ahci_show_temp(struct device *dev,
@@ -979,7 +979,7 @@ static u32 imx_ahci_parse_props(struct device *dev,
 	return reg_value;
 }
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_mtk.c b/drivers/ata/ahci_mtk.c
index c056378e3e72..0bf83a297091 100644
--- a/drivers/ata/ahci_mtk.c
+++ b/drivers/ata/ahci_mtk.c
@@ -37,7 +37,7 @@ static const struct ata_port_info ahci_port_info = {
 	.port_ops	= &ahci_platform_ops,
 };
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
@@ -106,7 +106,7 @@ static int mtk_ahci_parse_property(struct ahci_host_priv *hpriv,
 	struct device_node *np = dev->of_node;
 
 	/* enable SATA function if needed */
-	if (of_find_property(np, "mediatek,phy-mode", NULL)) {
+	if (of_property_present(np, "mediatek,phy-mode")) {
 		plat->mode = syscon_regmap_lookup_by_phandle(
 					np, "mediatek,phy-mode");
 		if (IS_ERR(plat->mode)) {
diff --git a/drivers/ata/ahci_mvebu.c b/drivers/ata/ahci_mvebu.c
index 22ecc4f3ae79..596cf017f427 100644
--- a/drivers/ata/ahci_mvebu.c
+++ b/drivers/ata/ahci_mvebu.c
@@ -178,7 +178,7 @@ static const struct ata_port_info ahci_mvebu_port_info = {
 	.port_ops  = &ahci_platform_ops,
 };
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_platform.c b/drivers/ata/ahci_platform.c
index 8f5572a9f8f1..299ee686ac49 100644
--- a/drivers/ata/ahci_platform.c
+++ b/drivers/ata/ahci_platform.c
@@ -36,7 +36,7 @@ static const struct ata_port_info ahci_port_info_nolpm = {
 	.port_ops	= &ahci_platform_ops,
 };
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_qoriq.c b/drivers/ata/ahci_qoriq.c
index 9cf9bf36a874..0ba764d283c8 100644
--- a/drivers/ata/ahci_qoriq.c
+++ b/drivers/ata/ahci_qoriq.c
@@ -159,7 +159,7 @@ static const struct ata_port_info ahci_qoriq_port_info = {
 	.port_ops	= &ahci_qoriq_ops,
 };
 
-static struct scsi_host_template ahci_qoriq_sht = {
+static const struct scsi_host_template ahci_qoriq_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_seattle.c b/drivers/ata/ahci_seattle.c
index ced12705ed9d..9eda7bbd2151 100644
--- a/drivers/ata/ahci_seattle.c
+++ b/drivers/ata/ahci_seattle.c
@@ -72,7 +72,7 @@ static const struct ata_port_info ahci_port_seattle_info = {
 	.port_ops	= &ahci_seattle_ops,
 };
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_st.c b/drivers/ata/ahci_st.c
index 8607b68eee53..f2c1edb36986 100644
--- a/drivers/ata/ahci_st.c
+++ b/drivers/ata/ahci_st.c
@@ -138,7 +138,7 @@ static const struct ata_port_info st_ahci_port_info = {
 	.port_ops       = &st_ahci_port_ops,
 };
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_sunxi.c b/drivers/ata/ahci_sunxi.c
index c7273c1cb0c7..076c12b4ba08 100644
--- a/drivers/ata/ahci_sunxi.c
+++ b/drivers/ata/ahci_sunxi.c
@@ -206,7 +206,7 @@ static const struct ata_port_info ahci_sunxi_port_info = {
 	.port_ops	= &ahci_platform_ops,
 };
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_tegra.c b/drivers/ata/ahci_tegra.c
index 4fb94db1217d..8e5e2b359f2d 100644
--- a/drivers/ata/ahci_tegra.c
+++ b/drivers/ata/ahci_tegra.c
@@ -506,7 +506,7 @@ static const struct of_device_id tegra_ahci_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, tegra_ahci_of_match);
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ahci_xgene.c b/drivers/ata/ahci_xgene.c
index 1e08704d5117..83f5ff54ef5b 100644
--- a/drivers/ata/ahci_xgene.c
+++ b/drivers/ata/ahci_xgene.c
@@ -710,7 +710,7 @@ static int xgene_ahci_mux_select(struct xgene_ahci_context *ctx)
 	return val & CFG_SATA_ENET_SELECT_MASK ? -1 : 0;
 }
 
-static struct scsi_host_template ahci_platform_sht = {
+static const struct scsi_host_template ahci_platform_sht = {
 	AHCI_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ata_generic.c b/drivers/ata/ata_generic.c
index 20a32e4d501d..2f57ec00ab82 100644
--- a/drivers/ata/ata_generic.c
+++ b/drivers/ata/ata_generic.c
@@ -95,7 +95,7 @@ static int generic_set_mode(struct ata_link *link, struct ata_device **unused)
 	return 0;
 }
 
-static struct scsi_host_template generic_sht = {
+static const struct scsi_host_template generic_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/ata_piix.c b/drivers/ata/ata_piix.c
index ade5e894563b..ec3c5bd1f813 100644
--- a/drivers/ata/ata_piix.c
+++ b/drivers/ata/ata_piix.c
@@ -1059,7 +1059,7 @@ static u8 piix_vmw_bmdma_status(struct ata_port *ap)
 	return ata_bmdma_status(ap) & ~ATA_DMA_ERR;
 }
 
-static struct scsi_host_template piix_sht = {
+static const struct scsi_host_template piix_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
@@ -1095,7 +1095,7 @@ static struct attribute *piix_sidpr_shost_attrs[] = {
 
 ATTRIBUTE_GROUPS(piix_sidpr_shost);
 
-static struct scsi_host_template piix_sidpr_sht = {
+static const struct scsi_host_template piix_sidpr_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 	.shost_groups		= piix_sidpr_shost_groups,
 };
@@ -1645,7 +1645,7 @@ static int piix_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct device *dev = &pdev->dev;
 	struct ata_port_info port_info[2];
 	const struct ata_port_info *ppi[] = { &port_info[0], &port_info[1] };
-	struct scsi_host_template *sht = &piix_sht;
+	const struct scsi_host_template *sht = &piix_sht;
 	unsigned long port_flags;
 	struct ata_host *host;
 	struct piix_host_priv *hpriv;
diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
index 8f216de76648..9c2cb6cbea76 100644
--- a/drivers/ata/libahci.c
+++ b/drivers/ata/libahci.c
@@ -2692,7 +2692,7 @@ void ahci_set_em_messages(struct ahci_host_priv *hpriv,
 EXPORT_SYMBOL_GPL(ahci_set_em_messages);
 
 static int ahci_host_activate_multi_irqs(struct ata_host *host,
-					 struct scsi_host_template *sht)
+					 const struct scsi_host_template *sht)
 {
 	struct ahci_host_priv *hpriv = host->private_data;
 	int i, rc;
@@ -2736,7 +2736,7 @@ static int ahci_host_activate_multi_irqs(struct ata_host *host,
  *	RETURNS:
  *	0 on success, -errno otherwise.
  */
-int ahci_host_activate(struct ata_host *host, struct scsi_host_template *sht)
+int ahci_host_activate(struct ata_host *host, const struct scsi_host_template *sht)
 {
 	struct ahci_host_priv *hpriv = host->private_data;
 	int irq = hpriv->irq;
diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c
index b9e336bacf17..9a8d43f54adc 100644
--- a/drivers/ata/libahci_platform.c
+++ b/drivers/ata/libahci_platform.c
@@ -363,7 +363,7 @@ static int ahci_platform_get_phy(struct ahci_host_priv *hpriv, u32 port,
 	switch (rc) {
 	case -ENOSYS:
 		/* No PHY support. Check if PHY is required. */
-		if (of_find_property(node, "phys", NULL)) {
+		if (of_property_present(node, "phys")) {
 			dev_err(dev,
 				"couldn't get PHY in node %pOFn: ENOSYS\n",
 				node);
@@ -680,7 +680,7 @@ EXPORT_SYMBOL_GPL(ahci_platform_get_resources);
 int ahci_platform_init_host(struct platform_device *pdev,
 			    struct ahci_host_priv *hpriv,
 			    const struct ata_port_info *pi_template,
-			    struct scsi_host_template *sht)
+			    const struct scsi_host_template *sht)
 {
 	struct device *dev = &pdev->dev;
 	struct ata_port_info pi = *pi_template;
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index 14c17c3bda4e..8bf612bdd61a 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -5775,7 +5775,7 @@ static void async_port_probe(void *data, async_cookie_t cookie)
  *	RETURNS:
  *	0 on success, -errno otherwise.
  */
-int ata_host_register(struct ata_host *host, struct scsi_host_template *sht)
+int ata_host_register(struct ata_host *host, const struct scsi_host_template *sht)
 {
 	int i, rc;
 
@@ -5883,7 +5883,7 @@ EXPORT_SYMBOL_GPL(ata_host_register);
  */
 int ata_host_activate(struct ata_host *host, int irq,
 		      irq_handler_t irq_handler, unsigned long irq_flags,
-		      struct scsi_host_template *sht)
+		      const struct scsi_host_template *sht)
 {
 	int i, rc;
 	char *irq_desc;
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index e093c7a7deeb..7bb12deab70c 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -4186,7 +4186,7 @@ void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd)
 	scsi_done(cmd);
 }
 
-int ata_scsi_add_hosts(struct ata_host *host, struct scsi_host_template *sht)
+int ata_scsi_add_hosts(struct ata_host *host, const struct scsi_host_template *sht)
 {
 	int i, rc;
 
diff --git a/drivers/ata/libata-sff.c b/drivers/ata/libata-sff.c
index cd82d3b5ed14..9d28badfe41d 100644
--- a/drivers/ata/libata-sff.c
+++ b/drivers/ata/libata-sff.c
@@ -2281,7 +2281,7 @@ EXPORT_SYMBOL_GPL(ata_pci_sff_prepare_host);
  */
 int ata_pci_sff_activate_host(struct ata_host *host,
 			      irq_handler_t irq_handler,
-			      struct scsi_host_template *sht)
+			      const struct scsi_host_template *sht)
 {
 	struct device *dev = host->dev;
 	struct pci_dev *pdev = to_pci_dev(dev);
@@ -2378,7 +2378,7 @@ static const struct ata_port_info *ata_sff_find_valid_pi(
 
 static int ata_pci_init_one(struct pci_dev *pdev,
 		const struct ata_port_info * const *ppi,
-		struct scsi_host_template *sht, void *host_priv,
+		const struct scsi_host_template *sht, void *host_priv,
 		int hflags, bool bmdma)
 {
 	struct device *dev = &pdev->dev;
@@ -2452,7 +2452,7 @@ out:
  */
 int ata_pci_sff_init_one(struct pci_dev *pdev,
 		 const struct ata_port_info * const *ppi,
-		 struct scsi_host_template *sht, void *host_priv, int hflag)
+		 const struct scsi_host_template *sht, void *host_priv, int hflag)
 {
 	return ata_pci_init_one(pdev, ppi, sht, host_priv, hflag, 0);
 }
@@ -3175,7 +3175,7 @@ EXPORT_SYMBOL_GPL(ata_pci_bmdma_prepare_host);
  */
 int ata_pci_bmdma_init_one(struct pci_dev *pdev,
 			   const struct ata_port_info * const * ppi,
-			   struct scsi_host_template *sht, void *host_priv,
+			   const struct scsi_host_template *sht, void *host_priv,
 			   int hflags)
 {
 	return ata_pci_init_one(pdev, ppi, sht, host_priv, hflags, 1);
diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h
index 2cd6124a01e8..926d0d33cd29 100644
--- a/drivers/ata/libata.h
+++ b/drivers/ata/libata.h
@@ -111,7 +111,7 @@ static inline void ata_acpi_bind_dev(struct ata_device *dev) {}
 extern struct ata_device *ata_scsi_find_dev(struct ata_port *ap,
 					    const struct scsi_device *scsidev);
 extern int ata_scsi_add_hosts(struct ata_host *host,
-			      struct scsi_host_template *sht);
+			      const struct scsi_host_template *sht);
 extern void ata_scsi_scan_host(struct ata_port *ap, int sync);
 extern int ata_scsi_offline_dev(struct ata_device *dev);
 extern bool ata_scsi_sense_is_valid(u8 sk, u8 asc, u8 ascq);
diff --git a/drivers/ata/pata_acpi.c b/drivers/ata/pata_acpi.c
index f8706ee427d2..ab38871b5e00 100644
--- a/drivers/ata/pata_acpi.c
+++ b/drivers/ata/pata_acpi.c
@@ -205,7 +205,7 @@ static int pacpi_port_start(struct ata_port *ap)
 	return ata_bmdma_port_start(ap);
 }
 
-static struct scsi_host_template pacpi_sht = {
+static const struct scsi_host_template pacpi_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_ali.c b/drivers/ata/pata_ali.c
index 76ad0e73fe2a..bb790edd6036 100644
--- a/drivers/ata/pata_ali.c
+++ b/drivers/ata/pata_ali.c
@@ -355,7 +355,7 @@ static void ali_c2_c3_postreset(struct ata_link *link, unsigned int *classes)
 	ata_sff_postreset(link, classes);
 }
 
-static struct scsi_host_template ali_sht = {
+static const struct scsi_host_template ali_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_amd.c b/drivers/ata/pata_amd.c
index f216f9d7b9ec..5b02b89748b7 100644
--- a/drivers/ata/pata_amd.c
+++ b/drivers/ata/pata_amd.c
@@ -388,7 +388,7 @@ static void nv_host_stop(struct ata_host *host)
 	pci_write_config_dword(to_pci_dev(host->dev), 0x60, udma);
 }
 
-static struct scsi_host_template amd_sht = {
+static const struct scsi_host_template amd_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c
index e89617ed9175..6ab294322e79 100644
--- a/drivers/ata/pata_arasan_cf.c
+++ b/drivers/ata/pata_arasan_cf.c
@@ -218,7 +218,7 @@ struct arasan_cf_dev {
 	struct ata_queued_cmd *qc;
 };
 
-static struct scsi_host_template arasan_cf_sht = {
+static const struct scsi_host_template arasan_cf_sht = {
 	ATA_BASE_SHT(DRIVER_NAME),
 	.dma_boundary = 0xFFFFFFFFUL,
 };
diff --git a/drivers/ata/pata_artop.c b/drivers/ata/pata_artop.c
index 20a8f31a3f57..40544282f455 100644
--- a/drivers/ata/pata_artop.c
+++ b/drivers/ata/pata_artop.c
@@ -292,7 +292,7 @@ static int artop6210_qc_defer(struct ata_queued_cmd *qc)
 	return 0;
 }
 
-static struct scsi_host_template artop_sht = {
+static const struct scsi_host_template artop_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_atiixp.c b/drivers/ata/pata_atiixp.c
index efdb94cff68b..8c5cc803aab3 100644
--- a/drivers/ata/pata_atiixp.c
+++ b/drivers/ata/pata_atiixp.c
@@ -251,7 +251,7 @@ static void atiixp_bmdma_stop(struct ata_queued_cmd *qc)
 	ata_bmdma_stop(qc);
 }
 
-static struct scsi_host_template atiixp_sht = {
+static const struct scsi_host_template atiixp_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= LIBATA_DUMB_MAX_PRD,
 	.dma_boundary		= ATA_DMA_BOUNDARY,
diff --git a/drivers/ata/pata_atp867x.c b/drivers/ata/pata_atp867x.c
index 779d660415c8..aaef5924f636 100644
--- a/drivers/ata/pata_atp867x.c
+++ b/drivers/ata/pata_atp867x.c
@@ -259,7 +259,7 @@ static int atp867x_cable_detect(struct ata_port *ap)
 	return ATA_CBL_PATA_UNK;
 }
 
-static struct scsi_host_template atp867x_sht = {
+static const struct scsi_host_template atp867x_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_buddha.c b/drivers/ata/pata_buddha.c
index 27d4c417fc60..49bc619b83e2 100644
--- a/drivers/ata/pata_buddha.c
+++ b/drivers/ata/pata_buddha.c
@@ -57,7 +57,7 @@ static unsigned int xsurf_bases[2] = {
 	XSURF_BASE1, XSURF_BASE2
 };
 
-static struct scsi_host_template pata_buddha_sht = {
+static const struct scsi_host_template pata_buddha_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_cmd640.c b/drivers/ata/pata_cmd640.c
index 1a3372a72213..45a7217b136e 100644
--- a/drivers/ata/pata_cmd640.c
+++ b/drivers/ata/pata_cmd640.c
@@ -172,7 +172,7 @@ static bool cmd640_sff_irq_check(struct ata_port *ap)
 	return irq_stat & irq_mask;
 }
 
-static struct scsi_host_template cmd640_sht = {
+static const struct scsi_host_template cmd640_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_cmd64x.c b/drivers/ata/pata_cmd64x.c
index 5baa4a7819c1..fafea2b79145 100644
--- a/drivers/ata/pata_cmd64x.c
+++ b/drivers/ata/pata_cmd64x.c
@@ -319,7 +319,7 @@ static void cmd646r1_bmdma_stop(struct ata_queued_cmd *qc)
 	ata_bmdma_stop(qc);
 }
 
-static struct scsi_host_template cmd64x_sht = {
+static const struct scsi_host_template cmd64x_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_cs5520.c b/drivers/ata/pata_cs5520.c
index f4289a532f87..422d42761a1d 100644
--- a/drivers/ata/pata_cs5520.c
+++ b/drivers/ata/pata_cs5520.c
@@ -94,7 +94,7 @@ static void cs5520_set_piomode(struct ata_port *ap, struct ata_device *adev)
 	cs5520_set_timings(ap, adev, adev->pio_mode);
 }
 
-static struct scsi_host_template cs5520_sht = {
+static const struct scsi_host_template cs5520_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= LIBATA_DUMB_MAX_PRD,
 	.dma_boundary		= ATA_DMA_BOUNDARY,
diff --git a/drivers/ata/pata_cs5530.c b/drivers/ata/pata_cs5530.c
index d5b7ac14e78f..1e67b0f8db43 100644
--- a/drivers/ata/pata_cs5530.c
+++ b/drivers/ata/pata_cs5530.c
@@ -146,7 +146,7 @@ static unsigned int cs5530_qc_issue(struct ata_queued_cmd *qc)
 	return ata_bmdma_qc_issue(qc);
 }
 
-static struct scsi_host_template cs5530_sht = {
+static const struct scsi_host_template cs5530_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize	= LIBATA_DUMB_MAX_PRD,
 	.dma_boundary	= ATA_DMA_BOUNDARY,
diff --git a/drivers/ata/pata_cs5535.c b/drivers/ata/pata_cs5535.c
index c2c3238ff84b..d793fc441b46 100644
--- a/drivers/ata/pata_cs5535.c
+++ b/drivers/ata/pata_cs5535.c
@@ -141,7 +141,7 @@ static void cs5535_set_dmamode(struct ata_port *ap, struct ata_device *adev)
 	wrmsr(ATAC_CH0D0_DMA + 2 * adev->devno, reg, 0);
 }
 
-static struct scsi_host_template cs5535_sht = {
+static const struct scsi_host_template cs5535_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_cs5536.c b/drivers/ata/pata_cs5536.c
index ab47aeb5587f..b811efd2cc34 100644
--- a/drivers/ata/pata_cs5536.c
+++ b/drivers/ata/pata_cs5536.c
@@ -217,7 +217,7 @@ static void cs5536_set_dmamode(struct ata_port *ap, struct ata_device *adev)
 	cs5536_write(pdev, ETC, etc);
 }
 
-static struct scsi_host_template cs5536_sht = {
+static const struct scsi_host_template cs5536_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_cypress.c b/drivers/ata/pata_cypress.c
index 3be5d52a777b..ae347b5c2871 100644
--- a/drivers/ata/pata_cypress.c
+++ b/drivers/ata/pata_cypress.c
@@ -115,7 +115,7 @@ static void cy82c693_set_dmamode(struct ata_port *ap, struct ata_device *adev)
 	outb(0x50, 0x23);
 }
 
-static struct scsi_host_template cy82c693_sht = {
+static const struct scsi_host_template cy82c693_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_efar.c b/drivers/ata/pata_efar.c
index 21da59f35b41..2e6eccf2902f 100644
--- a/drivers/ata/pata_efar.c
+++ b/drivers/ata/pata_efar.c
@@ -234,7 +234,7 @@ static void efar_set_dmamode (struct ata_port *ap, struct ata_device *adev)
 	spin_unlock_irqrestore(&efar_lock, flags);
 }
 
-static struct scsi_host_template efar_sht = {
+static const struct scsi_host_template efar_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_ep93xx.c b/drivers/ata/pata_ep93xx.c
index 47845d920075..c6e043e05d43 100644
--- a/drivers/ata/pata_ep93xx.c
+++ b/drivers/ata/pata_ep93xx.c
@@ -872,7 +872,7 @@ static int ep93xx_pata_port_start(struct ata_port *ap)
 	return 0;
 }
 
-static struct scsi_host_template ep93xx_pata_sht = {
+static const struct scsi_host_template ep93xx_pata_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	/* ep93xx dma implementation limit */
 	.sg_tablesize		= 32,
diff --git a/drivers/ata/pata_falcon.c b/drivers/ata/pata_falcon.c
index 823c88622e34..996516e64f13 100644
--- a/drivers/ata/pata_falcon.c
+++ b/drivers/ata/pata_falcon.c
@@ -33,7 +33,7 @@
 #define DRV_NAME "pata_falcon"
 #define DRV_VERSION "0.1.0"
 
-static struct scsi_host_template pata_falcon_sht = {
+static const struct scsi_host_template pata_falcon_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_ftide010.c b/drivers/ata/pata_ftide010.c
index 88924b5daa1a..6f6734c09b11 100644
--- a/drivers/ata/pata_ftide010.c
+++ b/drivers/ata/pata_ftide010.c
@@ -84,7 +84,7 @@ struct ftide010 {
 #define FTIDE010_CLK_MOD_DEV0_UDMA_EN	BIT(4)
 #define FTIDE010_CLK_MOD_DEV1_UDMA_EN	BIT(5)
 
-static struct scsi_host_template pata_ftide010_sht = {
+static const struct scsi_host_template pata_ftide010_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_gayle.c b/drivers/ata/pata_gayle.c
index 65bc9f3042ce..e5aa07f92106 100644
--- a/drivers/ata/pata_gayle.c
+++ b/drivers/ata/pata_gayle.c
@@ -35,7 +35,7 @@
 
 #define GAYLE_CONTROL	0x101a
 
-static struct scsi_host_template pata_gayle_sht = {
+static const struct scsi_host_template pata_gayle_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_hpt366.c b/drivers/ata/pata_hpt366.c
index 7e441fb304d3..bdccd1ba1524 100644
--- a/drivers/ata/pata_hpt366.c
+++ b/drivers/ata/pata_hpt366.c
@@ -312,7 +312,7 @@ static int hpt366_prereset(struct ata_link *link, unsigned long deadline)
 	return ata_sff_prereset(link, deadline);
 }
 
-static struct scsi_host_template hpt36x_sht = {
+static const struct scsi_host_template hpt36x_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_hpt37x.c b/drivers/ata/pata_hpt37x.c
index ce3c5eaa7e76..c0329cf01135 100644
--- a/drivers/ata/pata_hpt37x.c
+++ b/drivers/ata/pata_hpt37x.c
@@ -526,7 +526,7 @@ static void hpt37x_bmdma_stop(struct ata_queued_cmd *qc)
 }
 
 
-static struct scsi_host_template hpt37x_sht = {
+static const struct scsi_host_template hpt37x_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_hpt3x2n.c b/drivers/ata/pata_hpt3x2n.c
index 617c95522f43..5b1ecccf3c83 100644
--- a/drivers/ata/pata_hpt3x2n.c
+++ b/drivers/ata/pata_hpt3x2n.c
@@ -337,7 +337,7 @@ static unsigned int hpt3x2n_qc_issue(struct ata_queued_cmd *qc)
 	return ata_bmdma_qc_issue(qc);
 }
 
-static struct scsi_host_template hpt3x2n_sht = {
+static const struct scsi_host_template hpt3x2n_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_hpt3x3.c b/drivers/ata/pata_hpt3x3.c
index 83974d5eb387..d65c586b5ad0 100644
--- a/drivers/ata/pata_hpt3x3.c
+++ b/drivers/ata/pata_hpt3x3.c
@@ -136,7 +136,7 @@ static int hpt3x3_atapi_dma(struct ata_queued_cmd *qc)
 
 #endif /* CONFIG_PATA_HPT3X3_DMA */
 
-static struct scsi_host_template hpt3x3_sht = {
+static const struct scsi_host_template hpt3x3_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_icside.c b/drivers/ata/pata_icside.c
index 498383cb6e29..9cfb064782c3 100644
--- a/drivers/ata/pata_icside.c
+++ b/drivers/ata/pata_icside.c
@@ -298,7 +298,7 @@ static int icside_dma_init(struct pata_icside_info *info)
 }
 
 
-static struct scsi_host_template pata_icside_sht = {
+static const struct scsi_host_template pata_icside_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= SG_MAX_SEGMENTS,
 	.dma_boundary		= IOMD_DMA_BOUNDARY,
diff --git a/drivers/ata/pata_imx.c b/drivers/ata/pata_imx.c
index 150939275b1b..4013f28679a9 100644
--- a/drivers/ata/pata_imx.c
+++ b/drivers/ata/pata_imx.c
@@ -97,7 +97,7 @@ static void pata_imx_set_piomode(struct ata_port *ap, struct ata_device *adev)
 	__raw_writel(val, priv->host_regs + PATA_IMX_ATA_CONTROL);
 }
 
-static struct scsi_host_template pata_imx_sht = {
+static const struct scsi_host_template pata_imx_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_isapnp.c b/drivers/ata/pata_isapnp.c
index 43bb224430d3..25a63d043c8e 100644
--- a/drivers/ata/pata_isapnp.c
+++ b/drivers/ata/pata_isapnp.c
@@ -20,7 +20,7 @@
 #define DRV_NAME "pata_isapnp"
 #define DRV_VERSION "0.2.5"
 
-static struct scsi_host_template isapnp_sht = {
+static const struct scsi_host_template isapnp_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_it8213.c b/drivers/ata/pata_it8213.c
index 8a3e8778163c..b7ac56103c8a 100644
--- a/drivers/ata/pata_it8213.c
+++ b/drivers/ata/pata_it8213.c
@@ -228,7 +228,7 @@ static void it8213_set_dmamode (struct ata_port *ap, struct ata_device *adev)
 	pci_write_config_byte(dev, 0x48, udma_enable);
 }
 
-static struct scsi_host_template it8213_sht = {
+static const struct scsi_host_template it8213_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_it821x.c b/drivers/ata/pata_it821x.c
index 8a5b4e0079ab..2fe3fb6102ce 100644
--- a/drivers/ata/pata_it821x.c
+++ b/drivers/ata/pata_it821x.c
@@ -800,7 +800,7 @@ static int it821x_rdc_cable(struct ata_port *ap)
 	return ATA_CBL_PATA80;
 }
 
-static struct scsi_host_template it821x_sht = {
+static const struct scsi_host_template it821x_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_ixp4xx_cf.c b/drivers/ata/pata_ixp4xx_cf.c
index e225913a619d..99a2ce723495 100644
--- a/drivers/ata/pata_ixp4xx_cf.c
+++ b/drivers/ata/pata_ixp4xx_cf.c
@@ -17,6 +17,7 @@
 #include <linux/module.h>
 #include <linux/libata.h>
 #include <linux/irq.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 #include <scsi/scsi_host.h>
@@ -173,7 +174,7 @@ static unsigned int ixp4xx_mmio_data_xfer(struct ata_queued_cmd *qc,
 	return words << 1;
 }
 
-static struct scsi_host_template ixp4xx_sht = {
+static const struct scsi_host_template ixp4xx_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_jmicron.c b/drivers/ata/pata_jmicron.c
index d1b3ce8958dd..f51fb8219762 100644
--- a/drivers/ata/pata_jmicron.c
+++ b/drivers/ata/pata_jmicron.c
@@ -107,7 +107,7 @@ static int jmicron_pre_reset(struct ata_link *link, unsigned long deadline)
 
 /* No PIO or DMA methods needed for this device */
 
-static struct scsi_host_template jmicron_sht = {
+static const struct scsi_host_template jmicron_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_legacy.c b/drivers/ata/pata_legacy.c
index 03c580625c2c..448a511cbc17 100644
--- a/drivers/ata/pata_legacy.c
+++ b/drivers/ata/pata_legacy.c
@@ -250,7 +250,7 @@ static int legacy_set_mode(struct ata_link *link, struct ata_device **unused)
 	return 0;
 }
 
-static struct scsi_host_template legacy_sht = {
+static const struct scsi_host_template legacy_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_macio.c b/drivers/ata/pata_macio.c
index 9ccaac9e2bc3..17f6ccee53c7 100644
--- a/drivers/ata/pata_macio.c
+++ b/drivers/ata/pata_macio.c
@@ -21,6 +21,7 @@
 #include <linux/adb.h>
 #include <linux/pmu.h>
 #include <linux/scatterlist.h>
+#include <linux/irqdomain.h>
 #include <linux/of.h>
 #include <linux/gfp.h>
 #include <linux/pci.h>
@@ -908,7 +909,7 @@ static int pata_macio_do_resume(struct pata_macio_priv *priv)
 }
 #endif /* CONFIG_PM_SLEEP */
 
-static struct scsi_host_template pata_macio_sht = {
+static const struct scsi_host_template pata_macio_sht = {
 	__ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= MAX_DCMDS,
 	/* We may not need that strict one */
diff --git a/drivers/ata/pata_marvell.c b/drivers/ata/pata_marvell.c
index 014ccb0f45dc..8119caaad605 100644
--- a/drivers/ata/pata_marvell.c
+++ b/drivers/ata/pata_marvell.c
@@ -92,7 +92,7 @@ static int marvell_cable_detect(struct ata_port *ap)
 
 /* No PIO or DMA methods needed for this device */
 
-static struct scsi_host_template marvell_sht = {
+static const struct scsi_host_template marvell_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_mpc52xx.c b/drivers/ata/pata_mpc52xx.c
index 3ebd6522a1fd..66c9dea4ea6e 100644
--- a/drivers/ata/pata_mpc52xx.c
+++ b/drivers/ata/pata_mpc52xx.c
@@ -606,7 +606,7 @@ mpc52xx_ata_task_irq(int irq, void *vpriv)
 	return IRQ_HANDLED;
 }
 
-static struct scsi_host_template mpc52xx_ata_sht = {
+static const struct scsi_host_template mpc52xx_ata_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_mpiix.c b/drivers/ata/pata_mpiix.c
index 8fda0e32c1ab..69e4baf27d72 100644
--- a/drivers/ata/pata_mpiix.c
+++ b/drivers/ata/pata_mpiix.c
@@ -136,7 +136,7 @@ static unsigned int mpiix_qc_issue(struct ata_queued_cmd *qc)
 	return ata_sff_qc_issue(qc);
 }
 
-static struct scsi_host_template mpiix_sht = {
+static const struct scsi_host_template mpiix_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_netcell.c b/drivers/ata/pata_netcell.c
index 06929e77c491..c0b2897fcf40 100644
--- a/drivers/ata/pata_netcell.c
+++ b/drivers/ata/pata_netcell.c
@@ -31,7 +31,7 @@ static unsigned int netcell_read_id(struct ata_device *adev,
 	return err_mask;
 }
 
-static struct scsi_host_template netcell_sht = {
+static const struct scsi_host_template netcell_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_ninja32.c b/drivers/ata/pata_ninja32.c
index f9255d6fd194..76a91013d27d 100644
--- a/drivers/ata/pata_ninja32.c
+++ b/drivers/ata/pata_ninja32.c
@@ -77,7 +77,7 @@ static void ninja32_dev_select(struct ata_port *ap, unsigned int device)
 	}
 }
 
-static struct scsi_host_template ninja32_sht = {
+static const struct scsi_host_template ninja32_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_ns87410.c b/drivers/ata/pata_ns87410.c
index ca3ab2736fef..44cc24d21d5f 100644
--- a/drivers/ata/pata_ns87410.c
+++ b/drivers/ata/pata_ns87410.c
@@ -114,7 +114,7 @@ static unsigned int ns87410_qc_issue(struct ata_queued_cmd *qc)
 	return ata_sff_qc_issue(qc);
 }
 
-static struct scsi_host_template ns87410_sht = {
+static const struct scsi_host_template ns87410_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_ns87415.c b/drivers/ata/pata_ns87415.c
index 9dd6bffefb48..d60e1f69d7b0 100644
--- a/drivers/ata/pata_ns87415.c
+++ b/drivers/ata/pata_ns87415.c
@@ -320,7 +320,7 @@ static struct ata_port_operations ns87560_pata_ops = {
 };
 #endif
 
-static struct scsi_host_template ns87415_sht = {
+static const struct scsi_host_template ns87415_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_octeon_cf.c b/drivers/ata/pata_octeon_cf.c
index 4cbcdc5da038..b1ce9f1761af 100644
--- a/drivers/ata/pata_octeon_cf.c
+++ b/drivers/ata/pata_octeon_cf.c
@@ -58,7 +58,7 @@ struct octeon_cf_port {
 	u64 dma_base;
 };
 
-static struct scsi_host_template octeon_cf_sht = {
+static const struct scsi_host_template octeon_cf_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_of_platform.c b/drivers/ata/pata_of_platform.c
index ac5a633c00a5..178b28eff170 100644
--- a/drivers/ata/pata_of_platform.c
+++ b/drivers/ata/pata_of_platform.c
@@ -15,7 +15,7 @@
 
 #define DRV_NAME "pata_of_platform"
 
-static struct scsi_host_template pata_platform_sht = {
+static const struct scsi_host_template pata_platform_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_oldpiix.c b/drivers/ata/pata_oldpiix.c
index 22a020374410..dca82d92b004 100644
--- a/drivers/ata/pata_oldpiix.c
+++ b/drivers/ata/pata_oldpiix.c
@@ -204,7 +204,7 @@ static unsigned int oldpiix_qc_issue(struct ata_queued_cmd *qc)
 }
 
 
-static struct scsi_host_template oldpiix_sht = {
+static const struct scsi_host_template oldpiix_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_opti.c b/drivers/ata/pata_opti.c
index 01976c4e4033..3d23f57eb128 100644
--- a/drivers/ata/pata_opti.c
+++ b/drivers/ata/pata_opti.c
@@ -148,7 +148,7 @@ static void opti_set_piomode(struct ata_port *ap, struct ata_device *adev)
 	opti_write_reg(ap, 0x85, CNTRL_REG);
 }
 
-static struct scsi_host_template opti_sht = {
+static const struct scsi_host_template opti_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_optidma.c b/drivers/ata/pata_optidma.c
index ad1090b90e52..dfc36b4ec9c6 100644
--- a/drivers/ata/pata_optidma.c
+++ b/drivers/ata/pata_optidma.c
@@ -334,7 +334,7 @@ static int optidma_set_mode(struct ata_link *link, struct ata_device **r_failed)
 	return rc;
 }
 
-static struct scsi_host_template optidma_sht = {
+static const struct scsi_host_template optidma_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_parport/Kconfig b/drivers/ata/pata_parport/Kconfig
index 0893a13e7979..2c953f5d1396 100644
--- a/drivers/ata/pata_parport/Kconfig
+++ b/drivers/ata/pata_parport/Kconfig
@@ -25,7 +25,7 @@ config PATA_PARPORT_BPCK
 
 config PATA_PARPORT_BPCK6
 	tristate "MicroSolutions backpack (Series 6) protocol"
-	depends on (PATA_PARPORT) && !64BIT
+	depends on PATA_PARPORT
 	help
 	  This option enables support for the Micro Solutions BACKPACK
 	  parallel port Series 6 IDE protocol.  (Most BACKPACK drives made
diff --git a/drivers/ata/pata_parport/aten.c b/drivers/ata/pata_parport/aten.c
index b66508bedbd0..1bd248c42f8b 100644
--- a/drivers/ata/pata_parport/aten.c
+++ b/drivers/ata/pata_parport/aten.c
@@ -9,14 +9,6 @@
 
 */
 
-/* Changes:
-
-	1.01	GRG 1998.05.05	init_proto, release_proto
-
-*/
-
-#define ATEN_VERSION      "1.01"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -24,8 +16,7 @@
 #include <linux/wait.h>
 #include <linux/types.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #define j44(a,b)                ((((a>>4)&0x0f)|(b&0xf0))^0x88)
 
@@ -35,7 +26,7 @@
 
 static int  cont_map[2] = { 0x08, 0x20 };
 
-static void  aten_write_regr( PIA *pi, int cont, int regr, int val)
+static void aten_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {	int r;
 
@@ -44,7 +35,7 @@ static void  aten_write_regr( PIA *pi, int cont, int regr, int val)
 	w0(r); w2(0xe); w2(6); w0(val); w2(7); w2(6); w2(0xc);
 }
 
-static int aten_read_regr( PIA *pi, int cont, int regr )
+static int aten_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {	int  a, b, r;
 
@@ -67,7 +58,7 @@ static int aten_read_regr( PIA *pi, int cont, int regr )
 	return -1;
 }
 
-static void aten_read_block( PIA *pi, char * buf, int count )
+static void aten_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int  k, a, b, c, d;
 
@@ -95,7 +86,7 @@ static void aten_read_block( PIA *pi, char * buf, int count )
 	}
 }
 
-static void aten_write_block( PIA *pi, char * buf, int count )
+static void aten_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int k;
 
@@ -107,28 +98,25 @@ static void aten_write_block( PIA *pi, char * buf, int count )
 	w2(0xc);
 }
 
-static void aten_connect ( PIA *pi  )
+static void aten_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
         pi->saved_r2 = r2();
 	w2(0xc);	
 }
 
-static void aten_disconnect ( PIA *pi )
+static void aten_disconnect(struct pi_adapter *pi)
 
 {       w0(pi->saved_r0);
         w2(pi->saved_r2);
 } 
 
-static void aten_log_adapter( PIA *pi, char * scratch, int verbose )
+static void aten_log_adapter(struct pi_adapter *pi)
 
 {       char    *mode_string[2] = {"4-bit","8-bit"};
 
-        printk("%s: aten %s, ATEN EH-100 at 0x%x, ",
-                pi->device,ATEN_VERSION,pi->port);
-        printk("mode %d (%s), delay %d\n",pi->mode,
-		mode_string[pi->mode],pi->delay);
-
+	dev_info(&pi->dev, "ATEN EH-100 at 0x%x, mode %d (%s), delay %d\n",
+		pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol aten = {
@@ -147,16 +135,5 @@ static struct pi_protocol aten = {
 	.log_adapter	= aten_log_adapter,
 };
 
-static int __init aten_init(void)
-{
-	return paride_register(&aten);
-}
-
-static void __exit aten_exit(void)
-{
-	paride_unregister( &aten );
-}
-
 MODULE_LICENSE("GPL");
-module_init(aten_init)
-module_exit(aten_exit)
+module_pata_parport_driver(aten);
diff --git a/drivers/ata/pata_parport/bpck.c b/drivers/ata/pata_parport/bpck.c
index 5fb3cf9ba11d..1c5035a09554 100644
--- a/drivers/ata/pata_parport/bpck.c
+++ b/drivers/ata/pata_parport/bpck.c
@@ -7,15 +7,6 @@
 
 */
 
-/* Changes:
-
-	1.01	GRG 1998.05.05 init_proto, release_proto, pi->delay 
-	1.02    GRG 1998.08.15 default pi->delay returned to 4
-
-*/
-
-#define	BPCK_VERSION	"1.02" 
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -23,8 +14,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #undef r2
 #undef w2
@@ -46,7 +36,7 @@
 
 static int  cont_map[3] = { 0x40, 0x48, 0 };
 
-static int bpck_read_regr( PIA *pi, int cont, int regr )
+static int bpck_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {       int r, l, h;
 
@@ -77,7 +67,7 @@ static int bpck_read_regr( PIA *pi, int cont, int regr )
 	return -1;
 }	
 
-static void bpck_write_regr( PIA *pi, int cont, int regr, int val )
+static void bpck_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {	int	r;
 
@@ -106,7 +96,7 @@ static void bpck_write_regr( PIA *pi, int cont, int regr, int val )
 #define WR(r,v)		bpck_write_regr(pi,2,r,v)
 #define RR(r)		(bpck_read_regr(pi,2,r))
 
-static void bpck_write_block( PIA *pi, char * buf, int count )
+static void bpck_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int i;
 
@@ -147,7 +137,7 @@ static void bpck_write_block( PIA *pi, char * buf, int count )
  	}
 }
 
-static void bpck_read_block( PIA *pi, char * buf, int count )
+static void bpck_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int i, l, h;
 
@@ -194,7 +184,7 @@ static void bpck_read_block( PIA *pi, char * buf, int count )
 	}
 }
 
-static int bpck_probe_unit ( PIA *pi )
+static int bpck_probe_unit(struct pi_adapter *pi)
 
 {	int o1, o0, f7, id;
 	int t, s;
@@ -217,7 +207,7 @@ static int bpck_probe_unit ( PIA *pi )
 	return 1;
 }
 	
-static void bpck_connect ( PIA *pi  )
+static void bpck_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
 	w0(0xff-pi->unit); w2(4); w0(pi->unit);
@@ -241,24 +231,24 @@ static void bpck_connect ( PIA *pi  )
 
 	WR(5,8);
 
-	if (pi->devtype == PI_PCD) {
+/*	if (pi->devtype == PI_PCD) {	possibly wrong, purpose unknown */
 		WR(0x46,0x10);		/* fiddle with ESS logic ??? */
 		WR(0x4c,0x38);
 		WR(0x4d,0x88);
 		WR(0x46,0xa0);
 		WR(0x41,0);
 		WR(0x4e,8);
-		}
+/*	}*/
 }
 
-static void bpck_disconnect ( PIA *pi )
+static void bpck_disconnect(struct pi_adapter *pi)
 
 {	w0(0); 
 	if (pi->mode >= 2) { w2(9); w2(0); } else t2(2);
 	w2(0x4c); w0(pi->saved_r0);
 } 
 
-static void bpck_force_spp ( PIA *pi )
+static void bpck_force_spp(struct pi_adapter *pi)
 
 /* This fakes the EPP protocol to turn off EPP ... */
 
@@ -276,7 +266,7 @@ static void bpck_force_spp ( PIA *pi )
 
 #define TEST_LEN  16
 
-static int bpck_test_proto( PIA *pi, char * scratch, int verbose )
+static int bpck_test_proto(struct pi_adapter *pi)
 
 {	int i, e, l, h, om;
 	char buf[TEST_LEN];
@@ -334,19 +324,16 @@ static int bpck_test_proto( PIA *pi, char * scratch, int verbose )
 
 	}
 
-	if (verbose) {
-	    printk("%s: bpck: 0x%x unit %d mode %d: ",
-		   pi->device,pi->port,pi->unit,pi->mode);
-	    for (i=0;i<TEST_LEN;i++) printk("%3d",buf[i]);
-	    printk("\n");
-	}
+	dev_dbg(&pi->dev, "bpck: 0x%x unit %d mode %d: ",
+		pi->port, pi->unit, pi->mode);
+	print_hex_dump_debug("bpck: ", DUMP_PREFIX_NONE, TEST_LEN, 1, buf, TEST_LEN, false);
 
 	e = 0;
 	for (i=0;i<TEST_LEN;i++) if (buf[i] != (i+1)) e++;
 	return e;
 }
 
-static void bpck_read_eeprom ( PIA *pi, char * buf )
+static void bpck_read_eeprom(struct pi_adapter *pi, char *buf)
 
 {       int i, j, k, p, v, f, om, od;
 
@@ -397,7 +384,7 @@ static void bpck_read_eeprom ( PIA *pi, char * buf )
 	pi->mode = om; pi->delay = od;
 }
 
-static int bpck_test_port ( PIA *pi ) 	/* check for 8-bit port */
+static int bpck_test_port(struct pi_adapter *pi)	/* check for 8-bit port */
 
 {	int	i, r, m;
 
@@ -416,31 +403,17 @@ static int bpck_test_port ( PIA *pi ) 	/* check for 8-bit port */
 	return 5;
 }
 
-static void bpck_log_adapter( PIA *pi, char * scratch, int verbose )
+static void bpck_log_adapter(struct pi_adapter *pi)
 
 {	char	*mode_string[5] = { "4-bit","8-bit","EPP-8",
 				    "EPP-16","EPP-32" };
-
-#ifdef DUMP_EEPROM
-	int i;
-#endif
+	char scratch[128];
 
 	bpck_read_eeprom(pi,scratch);
-
-#ifdef DUMP_EEPROM
-	if (verbose) {
-	   for(i=0;i<128;i++)
-		if ((scratch[i] < ' ') || (scratch[i] > '~'))
-		    scratch[i] = '.';
-	   printk("%s: bpck EEPROM: %64.64s\n",pi->device,scratch);
-	   printk("%s:              %64.64s\n",pi->device,&scratch[64]);
-	}
-#endif
-
-	printk("%s: bpck %s, backpack %8.8s unit %d",
-		pi->device,BPCK_VERSION,&scratch[110],pi->unit);
-	printk(" at 0x%x, mode %d (%s), delay %d\n",pi->port,
-		pi->mode,mode_string[pi->mode],pi->delay);
+	print_hex_dump_bytes("bpck EEPROM: ", DUMP_PREFIX_NONE, scratch, 128);
+	dev_info(&pi->dev, "backpack %8.8s unit %d at 0x%x, mode %d (%s), delay %d\n",
+		 &scratch[110], pi->unit, pi->port, pi->mode,
+		 mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol bpck = {
@@ -462,16 +435,5 @@ static struct pi_protocol bpck = {
 	.log_adapter	= bpck_log_adapter,
 };
 
-static int __init bpck_init(void)
-{
-	return paride_register(&bpck);
-}
-
-static void __exit bpck_exit(void)
-{
-	paride_unregister(&bpck);
-}
-
 MODULE_LICENSE("GPL");
-module_init(bpck_init)
-module_exit(bpck_exit)
+module_pata_parport_driver(bpck);
diff --git a/drivers/ata/pata_parport/bpck6.c b/drivers/ata/pata_parport/bpck6.c
index d897e2a28efe..76febd07a9bb 100644
--- a/drivers/ata/pata_parport/bpck6.c
+++ b/drivers/ata/pata_parport/bpck6.c
@@ -11,219 +11,430 @@
 
 */
 
-/*
-   This is Ken's linux wrapper for the PPC library
-   Version 1.0.0 is the backpack driver for which source is not available
-   Version 2.0.0 is the first to have source released 
-   Version 2.0.1 is the "Cox-ified" source code 
-   Version 2.0.2 - fixed version string usage, and made ppc functions static 
-*/
-
-
-#define BACKPACK_VERSION "2.0.2"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
-#include <linux/slab.h>
 #include <linux/types.h>
-#include <asm/io.h>
 #include <linux/parport.h>
-
-#include "ppc6lnx.c"
-#include <linux/pata_parport.h>
-
-/* PARAMETERS */
-static bool verbose; /* set this to 1 to see debugging messages and whatnot */
- 
-
-#define PPCSTRUCT(pi) ((Interface *)(pi->private))
-
-/****************************************************************/
-/*
- ATAPI CDROM DRIVE REGISTERS
-*/
-#define ATAPI_DATA       0      /* data port                  */
-#define ATAPI_ERROR      1      /* error register (read)      */
-#define ATAPI_FEATURES   1      /* feature register (write)   */
-#define ATAPI_INT_REASON 2      /* interrupt reason register  */
-#define ATAPI_COUNT_LOW  4      /* byte count register (low)  */
-#define ATAPI_COUNT_HIGH 5      /* byte count register (high) */
-#define ATAPI_DRIVE_SEL  6      /* drive select register      */
-#define ATAPI_STATUS     7      /* status port (read)         */
-#define ATAPI_COMMAND    7      /* command port (write)       */
-#define ATAPI_ALT_STATUS 0x0e /* alternate status reg (read) */
-#define ATAPI_DEVICE_CONTROL 0x0e /* device control (write)   */
-/****************************************************************/
-
-static int bpck6_read_regr(PIA *pi, int cont, int reg)
+#include "pata_parport.h"
+
+/* 60772 Commands */
+#define ACCESS_REG		0x00
+#define ACCESS_PORT		0x40
+
+#define ACCESS_READ		0x00
+#define ACCESS_WRITE		0x20
+
+/* 60772 Command Prefix */
+#define CMD_PREFIX_SET		0xe0	// Special command that modifies next command's operation
+#define CMD_PREFIX_RESET	0xc0	// Resets current cmd modifier reg bits
+ #define PREFIX_IO16		0x01	// perform 16-bit wide I/O
+ #define PREFIX_FASTWR		0x04	// enable PPC mode fast-write
+ #define PREFIX_BLK		0x08	// enable block transfer mode
+
+/* 60772 Registers */
+#define REG_STATUS		0x00	// status register
+ #define STATUS_IRQA		0x01	// Peripheral IRQA line
+ #define STATUS_EEPROM_DO	0x40	// Serial EEPROM data bit
+#define REG_VERSION		0x01	// PPC version register (read)
+#define REG_HWCFG		0x02	// Hardware Config register
+#define REG_RAMSIZE		0x03	// Size of RAM Buffer
+ #define RAMSIZE_128K		0x02
+#define REG_EEPROM		0x06	// EEPROM control register
+ #define EEPROM_SK		0x01	// eeprom SK bit
+ #define EEPROM_DI		0x02	// eeprom DI bit
+ #define EEPROM_CS		0x04	// eeprom CS bit
+ #define EEPROM_EN		0x08	// eeprom output enable
+#define REG_BLKSIZE		0x08	// Block transfer len (24 bit)
+
+/* flags */
+#define fifo_wait		0x10
+
+/* DONT CHANGE THESE LEST YOU BREAK EVERYTHING - BIT FIELD DEPENDENCIES */
+#define PPCMODE_UNI_SW		0
+#define PPCMODE_UNI_FW		1
+#define PPCMODE_BI_SW		2
+#define PPCMODE_BI_FW		3
+#define PPCMODE_EPP_BYTE	4
+#define PPCMODE_EPP_WORD	5
+#define PPCMODE_EPP_DWORD	6
+
+static int mode_map[] = { PPCMODE_UNI_FW, PPCMODE_BI_FW, PPCMODE_EPP_BYTE,
+			  PPCMODE_EPP_WORD, PPCMODE_EPP_DWORD };
+
+static void bpck6_send_cmd(struct pi_adapter *pi, u8 cmd)
 {
-	unsigned int out;
+	switch (mode_map[pi->mode]) {
+	case PPCMODE_UNI_SW:
+	case PPCMODE_UNI_FW:
+	case PPCMODE_BI_SW:
+	case PPCMODE_BI_FW:
+		parport_write_data(pi->pardev->port, cmd);
+		parport_frob_control(pi->pardev->port, 0, PARPORT_CONTROL_AUTOFD);
+		break;
+	case PPCMODE_EPP_BYTE:
+	case PPCMODE_EPP_WORD:
+	case PPCMODE_EPP_DWORD:
+		pi->pardev->port->ops->epp_write_addr(pi->pardev->port, &cmd, 1, 0);
+		break;
+	}
+}
 
-	/* check for bad settings */
-	if (reg<0 || reg>7 || cont<0 || cont>2)
-	{
-		return(-1);
+static u8 bpck6_rd_data_byte(struct pi_adapter *pi)
+{
+	u8 data = 0;
+
+	switch (mode_map[pi->mode]) {
+	case PPCMODE_UNI_SW:
+	case PPCMODE_UNI_FW:
+		parport_frob_control(pi->pardev->port, PARPORT_CONTROL_STROBE,
+							PARPORT_CONTROL_INIT);
+		data = parport_read_status(pi->pardev->port);
+		data = ((data & 0x80) >> 1) | ((data & 0x38) >> 3);
+		parport_frob_control(pi->pardev->port, PARPORT_CONTROL_STROBE,
+							PARPORT_CONTROL_STROBE);
+		data |= parport_read_status(pi->pardev->port) & 0xB8;
+		break;
+	case PPCMODE_BI_SW:
+	case PPCMODE_BI_FW:
+		parport_data_reverse(pi->pardev->port);
+		parport_frob_control(pi->pardev->port, PARPORT_CONTROL_STROBE,
+				PARPORT_CONTROL_STROBE | PARPORT_CONTROL_INIT);
+		data = parport_read_data(pi->pardev->port);
+		parport_frob_control(pi->pardev->port, PARPORT_CONTROL_STROBE, 0);
+		parport_data_forward(pi->pardev->port);
+		break;
+	case PPCMODE_EPP_BYTE:
+	case PPCMODE_EPP_WORD:
+	case PPCMODE_EPP_DWORD:
+		pi->pardev->port->ops->epp_read_data(pi->pardev->port, &data, 1, 0);
+		break;
 	}
-	out=ppc6_rd_port(PPCSTRUCT(pi),cont?reg|8:reg);
-	return(out);
+
+	return data;
 }
 
-static void bpck6_write_regr(PIA *pi, int cont, int reg, int val)
+static void bpck6_wr_data_byte(struct pi_adapter *pi, u8 data)
 {
-	/* check for bad settings */
-	if (reg>=0 && reg<=7 && cont>=0 && cont<=1)
-	{
-		ppc6_wr_port(PPCSTRUCT(pi),cont?reg|8:reg,(u8)val);
+	switch (mode_map[pi->mode]) {
+	case PPCMODE_UNI_SW:
+	case PPCMODE_UNI_FW:
+	case PPCMODE_BI_SW:
+	case PPCMODE_BI_FW:
+		parport_write_data(pi->pardev->port, data);
+		parport_frob_control(pi->pardev->port, 0, PARPORT_CONTROL_INIT);
+		break;
+	case PPCMODE_EPP_BYTE:
+	case PPCMODE_EPP_WORD:
+	case PPCMODE_EPP_DWORD:
+		pi->pardev->port->ops->epp_write_data(pi->pardev->port, &data, 1, 0);
+		break;
 	}
 }
 
-static void bpck6_write_block( PIA *pi, char * buf, int len )
+static int bpck6_read_regr(struct pi_adapter *pi, int cont, int reg)
 {
-	ppc6_wr_port16_blk(PPCSTRUCT(pi),ATAPI_DATA,buf,(u32)len>>1); 
+	u8 port = cont ? reg | 8 : reg;
+
+	bpck6_send_cmd(pi, port | ACCESS_PORT | ACCESS_READ);
+	return bpck6_rd_data_byte(pi);
 }
 
-static void bpck6_read_block( PIA *pi, char * buf, int len )
+static void bpck6_write_regr(struct pi_adapter *pi, int cont, int reg, int val)
 {
-	ppc6_rd_port16_blk(PPCSTRUCT(pi),ATAPI_DATA,buf,(u32)len>>1);
+	u8 port = cont ? reg | 8 : reg;
+
+	bpck6_send_cmd(pi, port | ACCESS_PORT | ACCESS_WRITE);
+	bpck6_wr_data_byte(pi, val);
 }
 
-static void bpck6_connect ( PIA *pi  )
+static void bpck6_wait_for_fifo(struct pi_adapter *pi)
 {
-	if(verbose)
-	{
-		printk(KERN_DEBUG "connect\n");
-	}
+	int i;
 
-	if(pi->mode >=2)
-  	{
-		PPCSTRUCT(pi)->mode=4+pi->mode-2;	
-	}
-	else if(pi->mode==1)
-	{
-		PPCSTRUCT(pi)->mode=3;	
+	if (pi->private & fifo_wait) {
+		for (i = 0; i < 20; i++)
+			parport_read_status(pi->pardev->port);
 	}
-	else
-	{
-		PPCSTRUCT(pi)->mode=1;		
+}
+
+static void bpck6_write_block(struct pi_adapter *pi, char *buf, int len)
+{
+	u8 this, last;
+
+	bpck6_send_cmd(pi, REG_BLKSIZE | ACCESS_REG | ACCESS_WRITE);
+	bpck6_wr_data_byte(pi, (u8)len);
+	bpck6_wr_data_byte(pi, (u8)(len >> 8));
+	bpck6_wr_data_byte(pi, 0);
+
+	bpck6_send_cmd(pi, CMD_PREFIX_SET | PREFIX_IO16 | PREFIX_BLK);
+	bpck6_send_cmd(pi, ATA_REG_DATA | ACCESS_PORT | ACCESS_WRITE);
+
+	switch (mode_map[pi->mode]) {
+	case PPCMODE_UNI_SW:
+	case PPCMODE_BI_SW:
+		while (len--) {
+			parport_write_data(pi->pardev->port, *buf++);
+			parport_frob_control(pi->pardev->port, 0,
+							PARPORT_CONTROL_INIT);
+		}
+		break;
+	case PPCMODE_UNI_FW:
+	case PPCMODE_BI_FW:
+		bpck6_send_cmd(pi, CMD_PREFIX_SET | PREFIX_FASTWR);
+
+		parport_frob_control(pi->pardev->port, PARPORT_CONTROL_STROBE,
+							PARPORT_CONTROL_STROBE);
+
+		last = *buf;
+
+		parport_write_data(pi->pardev->port, last);
+
+		while (len) {
+			this = *buf++;
+			len--;
+
+			if (this == last) {
+				parport_frob_control(pi->pardev->port, 0,
+							PARPORT_CONTROL_INIT);
+			} else {
+				parport_write_data(pi->pardev->port, this);
+				last = this;
+			}
+		}
+
+		parport_frob_control(pi->pardev->port, PARPORT_CONTROL_STROBE,
+							0);
+		bpck6_send_cmd(pi, CMD_PREFIX_RESET | PREFIX_FASTWR);
+		break;
+	case PPCMODE_EPP_BYTE:
+		pi->pardev->port->ops->epp_write_data(pi->pardev->port, buf,
+						len, PARPORT_EPP_FAST_8);
+		bpck6_wait_for_fifo(pi);
+		break;
+	case PPCMODE_EPP_WORD:
+		pi->pardev->port->ops->epp_write_data(pi->pardev->port, buf,
+						len, PARPORT_EPP_FAST_16);
+		bpck6_wait_for_fifo(pi);
+		break;
+	case PPCMODE_EPP_DWORD:
+		pi->pardev->port->ops->epp_write_data(pi->pardev->port, buf,
+						len, PARPORT_EPP_FAST_32);
+		bpck6_wait_for_fifo(pi);
+		break;
 	}
 
-	ppc6_open(PPCSTRUCT(pi));  
-	ppc6_wr_extout(PPCSTRUCT(pi),0x3);
+	bpck6_send_cmd(pi, CMD_PREFIX_RESET | PREFIX_IO16 | PREFIX_BLK);
 }
 
-static void bpck6_disconnect ( PIA *pi )
+static void bpck6_read_block(struct pi_adapter *pi, char *buf, int len)
 {
-	if(verbose)
-	{
-		printk("disconnect\n");
+	bpck6_send_cmd(pi, REG_BLKSIZE | ACCESS_REG | ACCESS_WRITE);
+	bpck6_wr_data_byte(pi, (u8)len);
+	bpck6_wr_data_byte(pi, (u8)(len >> 8));
+	bpck6_wr_data_byte(pi, 0);
+
+	bpck6_send_cmd(pi, CMD_PREFIX_SET | PREFIX_IO16 | PREFIX_BLK);
+	bpck6_send_cmd(pi, ATA_REG_DATA | ACCESS_PORT | ACCESS_READ);
+
+	switch (mode_map[pi->mode]) {
+	case PPCMODE_UNI_SW:
+	case PPCMODE_UNI_FW:
+		while (len) {
+			u8 d;
+
+			parport_frob_control(pi->pardev->port,
+					PARPORT_CONTROL_STROBE,
+					PARPORT_CONTROL_INIT); /* DATA STROBE */
+			d = parport_read_status(pi->pardev->port);
+			d = ((d & 0x80) >> 1) | ((d & 0x38) >> 3);
+			parport_frob_control(pi->pardev->port,
+					PARPORT_CONTROL_STROBE,
+					PARPORT_CONTROL_STROBE);
+			d |= parport_read_status(pi->pardev->port) & 0xB8;
+			*buf++ = d;
+			len--;
+		}
+		break;
+	case PPCMODE_BI_SW:
+	case PPCMODE_BI_FW:
+		parport_data_reverse(pi->pardev->port);
+		while (len) {
+			parport_frob_control(pi->pardev->port,
+				PARPORT_CONTROL_STROBE,
+				PARPORT_CONTROL_STROBE | PARPORT_CONTROL_INIT);
+			*buf++ = parport_read_data(pi->pardev->port);
+			len--;
+		}
+		parport_frob_control(pi->pardev->port, PARPORT_CONTROL_STROBE,
+					0);
+		parport_data_forward(pi->pardev->port);
+		break;
+	case PPCMODE_EPP_BYTE:
+		pi->pardev->port->ops->epp_read_data(pi->pardev->port, buf, len,
+						PARPORT_EPP_FAST_8);
+		break;
+	case PPCMODE_EPP_WORD:
+		pi->pardev->port->ops->epp_read_data(pi->pardev->port, buf, len,
+						PARPORT_EPP_FAST_16);
+		break;
+	case PPCMODE_EPP_DWORD:
+		pi->pardev->port->ops->epp_read_data(pi->pardev->port, buf, len,
+						PARPORT_EPP_FAST_32);
+		break;
 	}
-	ppc6_wr_extout(PPCSTRUCT(pi),0x0);
-	ppc6_close(PPCSTRUCT(pi));
+
+	bpck6_send_cmd(pi, CMD_PREFIX_RESET | PREFIX_IO16 | PREFIX_BLK);
 }
 
-static int bpck6_test_port ( PIA *pi )   /* check for 8-bit port */
+static int bpck6_open(struct pi_adapter *pi)
 {
-	if(verbose)
-	{
-		printk(KERN_DEBUG "PARPORT indicates modes=%x for lp=0x%lx\n",
-               		((struct pardevice*)(pi->pardev))->port->modes,
-			((struct pardevice *)(pi->pardev))->port->base); 
-	}
+	u8 i, j, k;
 
-	/*copy over duplicate stuff.. initialize state info*/
-	PPCSTRUCT(pi)->ppc_id=pi->unit;
-	PPCSTRUCT(pi)->lpt_addr=pi->port;
+	pi->saved_r0 = parport_read_data(pi->pardev->port);
+	pi->saved_r2 = parport_read_control(pi->pardev->port) & 0x5F;
 
-	/* look at the parport device to see if what modes we can use */
-	if(((struct pardevice *)(pi->pardev))->port->modes & 
-		(PARPORT_MODE_EPP)
-          )
-	{
-		return 5; /* Can do EPP*/
-	}
-	else if(((struct pardevice *)(pi->pardev))->port->modes & 
-			(PARPORT_MODE_TRISTATE)
-               )
-	{
-		return 2;
-	}
-	else /*Just flat SPP*/
-	{
-		return 1;
-	}
+	parport_frob_control(pi->pardev->port, PARPORT_CONTROL_SELECT,
+						PARPORT_CONTROL_SELECT);
+	if (pi->saved_r0 == 'b')
+		parport_write_data(pi->pardev->port, 'x');
+	parport_write_data(pi->pardev->port, 'b');
+	parport_write_data(pi->pardev->port, 'p');
+	parport_write_data(pi->pardev->port, pi->unit);
+	parport_write_data(pi->pardev->port, ~pi->unit);
+
+	parport_frob_control(pi->pardev->port, PARPORT_CONTROL_SELECT, 0);
+	parport_write_control(pi->pardev->port, PARPORT_CONTROL_INIT);
+
+	i = mode_map[pi->mode] & 0x0C;
+	if (i == 0)
+		i = (mode_map[pi->mode] & 2) | 1;
+	parport_write_data(pi->pardev->port, i);
+
+	parport_frob_control(pi->pardev->port, PARPORT_CONTROL_SELECT,
+						PARPORT_CONTROL_SELECT);
+	parport_frob_control(pi->pardev->port, PARPORT_CONTROL_AUTOFD,
+						PARPORT_CONTROL_AUTOFD);
+
+	j = ((i & 0x08) << 4) | ((i & 0x07) << 3);
+	k = parport_read_status(pi->pardev->port) & 0xB8;
+	if (j != k)
+		goto fail;
+
+	parport_frob_control(pi->pardev->port, PARPORT_CONTROL_AUTOFD, 0);
+	k = (parport_read_status(pi->pardev->port) & 0xB8) ^ 0xB8;
+	if (j != k)
+		goto fail;
+
+	if (i & 4)	// EPP
+		parport_frob_control(pi->pardev->port,
+			PARPORT_CONTROL_SELECT | PARPORT_CONTROL_INIT, 0);
+	else				// PPC/ECP
+		parport_frob_control(pi->pardev->port, PARPORT_CONTROL_SELECT, 0);
+
+	pi->private = 0;
+
+	bpck6_send_cmd(pi, ACCESS_REG | ACCESS_WRITE | REG_RAMSIZE);
+	bpck6_wr_data_byte(pi, RAMSIZE_128K);
+
+	bpck6_send_cmd(pi, ACCESS_REG | ACCESS_READ | REG_VERSION);
+	if ((bpck6_rd_data_byte(pi) & 0x3F) == 0x0C)
+		pi->private |= fifo_wait;
+
+	return 1;
+
+fail:
+	parport_write_control(pi->pardev->port, pi->saved_r2);
+	parport_write_data(pi->pardev->port, pi->saved_r0);
+
+	return 0; // FAIL
 }
 
-static int bpck6_probe_unit ( PIA *pi )
+static void bpck6_deselect(struct pi_adapter *pi)
 {
-	int out;
+	if (mode_map[pi->mode] & 4)	// EPP
+		parport_frob_control(pi->pardev->port, PARPORT_CONTROL_INIT,
+							PARPORT_CONTROL_INIT);
+	else								// PPC/ECP
+		parport_frob_control(pi->pardev->port, PARPORT_CONTROL_SELECT,
+							PARPORT_CONTROL_SELECT);
+
+	parport_write_data(pi->pardev->port, pi->saved_r0);
+	parport_write_control(pi->pardev->port,
+			pi->saved_r2 | PARPORT_CONTROL_SELECT);
+	parport_write_control(pi->pardev->port, pi->saved_r2);
+}
 
-	if(verbose)
-	{
-		printk(KERN_DEBUG "PROBE UNIT %x on port:%x\n",pi->unit,pi->port);
-	}
+static void bpck6_wr_extout(struct pi_adapter *pi, u8 regdata)
+{
+	bpck6_send_cmd(pi, REG_VERSION | ACCESS_REG | ACCESS_WRITE);
+	bpck6_wr_data_byte(pi, (u8)((regdata & 0x03) << 6));
+}
 
-	/*SET PPC UNIT NUMBER*/
-	PPCSTRUCT(pi)->ppc_id=pi->unit;
+static void bpck6_connect(struct pi_adapter *pi)
+{
+	dev_dbg(&pi->dev, "connect\n");
+
+	bpck6_open(pi);
+	bpck6_wr_extout(pi, 0x3);
+}
+
+static void bpck6_disconnect(struct pi_adapter *pi)
+{
+	dev_dbg(&pi->dev, "disconnect\n");
+	bpck6_wr_extout(pi, 0x0);
+	bpck6_deselect(pi);
+}
+
+static int bpck6_test_port(struct pi_adapter *pi)   /* check for 8-bit port */
+{
+	dev_dbg(&pi->dev, "PARPORT indicates modes=%x for lp=0x%lx\n",
+		pi->pardev->port->modes, pi->pardev->port->base);
+
+	/* look at the parport device to see what modes we can use */
+	if (pi->pardev->port->modes & PARPORT_MODE_EPP)
+		return 5; /* Can do EPP */
+	if (pi->pardev->port->modes & PARPORT_MODE_TRISTATE)
+		return 2;
+	return 1; /* Just flat SPP */
+}
 
+static int bpck6_probe_unit(struct pi_adapter *pi)
+{
+	int out, saved_mode;
+
+	dev_dbg(&pi->dev, "PROBE UNIT %x on port:%x\n", pi->unit, pi->port);
+
+	saved_mode = pi->mode;
 	/*LOWER DOWN TO UNIDIRECTIONAL*/
-	PPCSTRUCT(pi)->mode=1;		
+	pi->mode = 0;
 
-	out=ppc6_open(PPCSTRUCT(pi));
+	out = bpck6_open(pi);
 
-	if(verbose)
-	{
-		printk(KERN_DEBUG "ppc_open returned %2x\n",out);
-	}
+	dev_dbg(&pi->dev, "ppc_open returned %2x\n", out);
 
   	if(out)
  	{
-		ppc6_close(PPCSTRUCT(pi));
-		if(verbose)
-		{
-			printk(KERN_DEBUG "leaving probe\n");
-		}
+		bpck6_deselect(pi);
+		dev_dbg(&pi->dev, "leaving probe\n");
+		pi->mode = saved_mode;
                return(1);
 	}
   	else
   	{
-		if(verbose)
-		{
-			printk(KERN_DEBUG "Failed open\n");
-		}
+		dev_dbg(&pi->dev, "Failed open\n");
+		pi->mode = saved_mode;
     		return(0);
   	}
 }
 
-static void bpck6_log_adapter( PIA *pi, char * scratch, int verbose )
+static void bpck6_log_adapter(struct pi_adapter *pi)
 {
 	char *mode_string[5]=
 		{"4-bit","8-bit","EPP-8","EPP-16","EPP-32"};
 
-	printk("%s: BACKPACK Protocol Driver V"BACKPACK_VERSION"\n",pi->device);
-	printk("%s: Copyright 2001 by Micro Solutions, Inc., DeKalb IL.\n",pi->device);
-	printk("%s: BACKPACK %s, Micro Solutions BACKPACK Drive at 0x%x\n",
-		pi->device,BACKPACK_VERSION,pi->port);
-	printk("%s: Unit: %d Mode:%d (%s) Delay %d\n",pi->device,
-		pi->unit,pi->mode,mode_string[pi->mode],pi->delay);
-}
-
-static int bpck6_init_proto(PIA *pi)
-{
-	Interface *p = kzalloc(sizeof(Interface), GFP_KERNEL);
-
-	if (p) {
-		pi->private = (unsigned long)p;
-		return 0;
-	}
-
-	printk(KERN_ERR "%s: ERROR COULDN'T ALLOCATE MEMORY\n", pi->device); 
-	return -1;
-}
-
-static void bpck6_release_proto(PIA *pi)
-{
-	kfree((void *)(pi->private)); 
+	dev_info(&pi->dev, "Micro Solutions BACKPACK Drive unit %d at 0x%x, mode:%d (%s), delay %d\n",
+		pi->unit, pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol bpck6 = {
@@ -241,27 +452,9 @@ static struct pi_protocol bpck6 = {
 	.test_port	= bpck6_test_port,
 	.probe_unit	= bpck6_probe_unit,
 	.log_adapter	= bpck6_log_adapter,
-	.init_proto	= bpck6_init_proto,
-	.release_proto	= bpck6_release_proto,
 };
 
-static int __init bpck6_init(void)
-{
-	printk(KERN_INFO "bpck6: BACKPACK Protocol Driver V"BACKPACK_VERSION"\n");
-	printk(KERN_INFO "bpck6: Copyright 2001 by Micro Solutions, Inc., DeKalb IL. USA\n");
-	if(verbose)
-		printk(KERN_DEBUG "bpck6: verbose debug enabled.\n");
-	return paride_register(&bpck6);
-}
-
-static void __exit bpck6_exit(void)
-{
-	paride_unregister(&bpck6);
-}
-
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Micro Solutions Inc.");
 MODULE_DESCRIPTION("BACKPACK Protocol module, compatible with PARIDE");
-module_param(verbose, bool, 0644);
-module_init(bpck6_init)
-module_exit(bpck6_exit)
+module_pata_parport_driver(bpck6);
diff --git a/drivers/ata/pata_parport/comm.c b/drivers/ata/pata_parport/comm.c
index 1775e7ed9336..4c2f9ad60ad8 100644
--- a/drivers/ata/pata_parport/comm.c
+++ b/drivers/ata/pata_parport/comm.c
@@ -8,14 +8,6 @@
 	use this adapter.
 */
 
-/* Changes:
-
-	1.01	GRG 1998.05.05  init_proto, release_proto
-
-*/
-
-#define COMM_VERSION      "1.01"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -23,8 +15,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 /* mode codes:  0  nybble reads, 8-bit writes
                 1  8-bit reads and writes
@@ -42,7 +33,7 @@
 
 static int  cont_map[2] = { 0x08, 0x10 };
 
-static int comm_read_regr( PIA *pi, int cont, int regr )
+static int comm_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {       int     l, h, r;
 
@@ -68,7 +59,7 @@ static int comm_read_regr( PIA *pi, int cont, int regr )
         return -1;
 }       
 
-static void comm_write_regr( PIA *pi, int cont, int regr, int val )
+static void comm_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {       int  r;
 
@@ -87,7 +78,7 @@ static void comm_write_regr( PIA *pi, int cont, int regr, int val )
         }
 }
 
-static void comm_connect ( PIA *pi  )
+static void comm_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
         pi->saved_r2 = r2();
@@ -98,14 +89,14 @@ static void comm_connect ( PIA *pi  )
         w2(4); w0(0xe0); w2(0xc); w2(0xc); w2(4);
 }
 
-static void comm_disconnect ( PIA *pi )
+static void comm_disconnect(struct pi_adapter *pi)
 
 {       w2(0); w2(0); w2(0); w2(4); 
 	w0(pi->saved_r0);
         w2(pi->saved_r2);
 } 
 
-static void comm_read_block( PIA *pi, char * buf, int count )
+static void comm_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {       int     i, l, h;
 
@@ -146,7 +137,7 @@ static void comm_read_block( PIA *pi, char * buf, int count )
 
 /* NB: Watch out for the byte swapped writes ! */
 
-static void comm_write_block( PIA *pi, char * buf, int count )
+static void comm_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {       int	k;
 
@@ -165,26 +156,26 @@ static void comm_write_block( PIA *pi, char * buf, int count )
                 break;
 
         case 3: w3(0x48); (void)r1();
-                for (k=0;k<count/2;k++) w4w(pi_swab16(buf,k));
+		for (k = 0; k < count / 2; k++)
+			w4w(swab16(((u16 *)buf)[k]));
                 break;
 
         case 4: w3(0x48); (void)r1();
-                for (k=0;k<count/4;k++) w4l(pi_swab32(buf,k));
+		for (k = 0; k < count / 4; k++)
+			w4l(swab16(((u16 *)buf)[2 * k]) |
+			    swab16(((u16 *)buf)[2 * k + 1]) << 16);
                 break;
 
 
         }
 }
 
-static void comm_log_adapter( PIA *pi, char * scratch, int verbose )
+static void comm_log_adapter(struct pi_adapter *pi)
 
 {       char    *mode_string[5] = {"4-bit","8-bit","EPP-8","EPP-16","EPP-32"};
 
-        printk("%s: comm %s, DataStor Commuter at 0x%x, ",
-                pi->device,COMM_VERSION,pi->port);
-        printk("mode %d (%s), delay %d\n",pi->mode,
-		mode_string[pi->mode],pi->delay);
-
+	dev_info(&pi->dev, "DataStor Commuter at 0x%x, mode %d (%s), delay %d\n",
+		pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol comm = {
@@ -203,16 +194,5 @@ static struct pi_protocol comm = {
 	.log_adapter	= comm_log_adapter,
 };
 
-static int __init comm_init(void)
-{
-	return paride_register(&comm);
-}
-
-static void __exit comm_exit(void)
-{
-	paride_unregister(&comm);
-}
-
 MODULE_LICENSE("GPL");
-module_init(comm_init)
-module_exit(comm_exit)
+module_pata_parport_driver(comm);
diff --git a/drivers/ata/pata_parport/dstr.c b/drivers/ata/pata_parport/dstr.c
index edf414d186a6..2524684be206 100644
--- a/drivers/ata/pata_parport/dstr.c
+++ b/drivers/ata/pata_parport/dstr.c
@@ -7,14 +7,6 @@
 
 */
 
-/* Changes:
-
-        1.01    GRG 1998.05.06 init_proto, release_proto
-
-*/
-
-#define DSTR_VERSION      "1.01"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -22,8 +14,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 /* mode codes:  0  nybble reads, 8-bit writes
                 1  8-bit reads and writes
@@ -44,7 +35,7 @@
 
 static int  cont_map[2] = { 0x20, 0x40 };
 
-static int dstr_read_regr( PIA *pi, int cont, int regr )
+static int dstr_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {       int     a, b, r;
 
@@ -71,7 +62,7 @@ static int dstr_read_regr( PIA *pi, int cont, int regr )
         return -1;
 }       
 
-static void dstr_write_regr(  PIA *pi, int cont, int regr, int val )
+static void dstr_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {       int  r;
 
@@ -98,21 +89,21 @@ static void dstr_write_regr(  PIA *pi, int cont, int regr, int val )
 		 w0(0xaa);w0(0x55);w0(0);w0(0xff);w0(0x87);w0(0x78);\
 		 w0(x);w2(5);w2(4);
 
-static void dstr_connect ( PIA *pi  )
+static void dstr_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
         pi->saved_r2 = r2();
         w2(4); CCP(0xe0); w0(0xff);
 }
 
-static void dstr_disconnect ( PIA *pi )
+static void dstr_disconnect(struct pi_adapter *pi)
 
 {       CCP(0x30);
         w0(pi->saved_r0);
         w2(pi->saved_r2);
 } 
 
-static void dstr_read_block( PIA *pi, char * buf, int count )
+static void dstr_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {       int     k, a, b;
 
@@ -154,7 +145,7 @@ static void dstr_read_block( PIA *pi, char * buf, int count )
         }
 }
 
-static void dstr_write_block( PIA *pi, char * buf, int count )
+static void dstr_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {       int	k;
 
@@ -190,16 +181,13 @@ static void dstr_write_block( PIA *pi, char * buf, int count )
 }
 
 
-static void dstr_log_adapter( PIA *pi, char * scratch, int verbose )
+static void dstr_log_adapter(struct pi_adapter *pi)
 
 {       char    *mode_string[5] = {"4-bit","8-bit","EPP-8",
 				   "EPP-16","EPP-32"};
 
-        printk("%s: dstr %s, DataStor EP2000 at 0x%x, ",
-                pi->device,DSTR_VERSION,pi->port);
-        printk("mode %d (%s), delay %d\n",pi->mode,
-		mode_string[pi->mode],pi->delay);
-
+	dev_info(&pi->dev, "DataStor EP2000 at 0x%x, mode %d (%s), delay %d\n",
+		pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol dstr = {
@@ -218,16 +206,5 @@ static struct pi_protocol dstr = {
 	.log_adapter	= dstr_log_adapter,
 };
 
-static int __init dstr_init(void)
-{
-	return paride_register(&dstr);
-}
-
-static void __exit dstr_exit(void)
-{
-	paride_unregister(&dstr);
-}
-
 MODULE_LICENSE("GPL");
-module_init(dstr_init)
-module_exit(dstr_exit)
+module_pata_parport_driver(dstr);
diff --git a/drivers/ata/pata_parport/epat.c b/drivers/ata/pata_parport/epat.c
index 6ce2dee7657f..b146999368ae 100644
--- a/drivers/ata/pata_parport/epat.c
+++ b/drivers/ata/pata_parport/epat.c
@@ -9,15 +9,6 @@
 	
 */
 
-/* Changes:
-
-        1.01    GRG 1998.05.06 init_proto, release_proto
-        1.02    Joshua b. Jore CPP(renamed), epat_connect, epat_disconnect
-
-*/
-
-#define EPAT_VERSION      "1.02"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -25,8 +16,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #define j44(a,b)		(((a>>4)&0x0f)+(b&0xf0))
 #define j53(a,b)		(((a>>3)&0x1f)+((b<<4)&0xe0))
@@ -44,7 +34,7 @@ MODULE_PARM_DESC(epatc8, "support for the Shuttle EP1284 chip, "
 
 static int cont_map[3] = { 0x18, 0x10, 0 };
 
-static void epat_write_regr( PIA *pi, int cont, int regr, int val)
+static void epat_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {	int r;
 
@@ -65,7 +55,7 @@ static void epat_write_regr( PIA *pi, int cont, int regr, int val)
 	}
 }
 
-static int epat_read_regr( PIA *pi, int cont, int regr )
+static int epat_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {	int  a, b, r;
 
@@ -94,7 +84,7 @@ static int epat_read_regr( PIA *pi, int cont, int regr )
 	return -1;	/* never gets here */
 }
 
-static void epat_read_block( PIA *pi, char * buf, int count )
+static void epat_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int  k, ph, a, b;
 
@@ -159,7 +149,7 @@ static void epat_read_block( PIA *pi, char * buf, int count )
 	}
 }
 
-static void epat_write_block( PIA *pi, char * buf, int count )   
+static void epat_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int ph, k;
 
@@ -210,7 +200,7 @@ static void epat_write_block( PIA *pi, char * buf, int count )
 #define CPP(x) 	w2(4);w0(0x22);w0(0xaa);w0(0x55);w0(0);w0(0xff);\
                 w0(0x87);w0(0x78);w0(x);w2(4);w2(5);w2(4);w0(0xff);
 
-static void epat_connect ( PIA *pi )
+static void epat_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
         pi->saved_r2 = r2();
@@ -242,16 +232,17 @@ static void epat_connect ( PIA *pi )
 	}
 }
 
-static void epat_disconnect (PIA *pi)
+static void epat_disconnect(struct pi_adapter *pi)
 {	CPP(0x30);
 	w0(pi->saved_r0);
 	w2(pi->saved_r2);
 }
 
-static int epat_test_proto( PIA *pi, char * scratch, int verbose )
+static int epat_test_proto(struct pi_adapter *pi)
 
 {       int     k, j, f, cc;
 	int	e[2] = {0,0};
+	char scratch[512];
 
         epat_connect(pi);
 	cc = RR(0xd);
@@ -279,15 +270,13 @@ static int epat_test_proto( PIA *pi, char * scratch, int verbose )
         }
         epat_disconnect(pi);
 
-        if (verbose)  {
-            printk("%s: epat: port 0x%x, mode %d, ccr %x, test=(%d,%d,%d)\n",
-		   pi->device,pi->port,pi->mode,cc,e[0],e[1],f);
-	}
+	dev_dbg(&pi->dev, "epat: port 0x%x, mode %d, ccr %x, test=(%d,%d,%d)\n",
+	       pi->port, pi->mode, cc, e[0], e[1], f);
 	
         return (e[0] && e[1]) || f;
 }
 
-static void epat_log_adapter( PIA *pi, char * scratch, int verbose )
+static void epat_log_adapter(struct pi_adapter *pi)
 
 {	int	ver;
         char    *mode_string[6] = 
@@ -298,11 +287,8 @@ static void epat_log_adapter( PIA *pi, char * scratch, int verbose )
         ver = RR(0xb);
         epat_disconnect(pi);
 
-	printk("%s: epat %s, Shuttle EPAT chip %x at 0x%x, ",
-		pi->device,EPAT_VERSION,ver,pi->port);
-	printk("mode %d (%s), delay %d\n",pi->mode,
-		mode_string[pi->mode],pi->delay);
-
+	dev_info(&pi->dev, "Shuttle EPAT chip %x at 0x%x, mode %d (%s), delay %d\n",
+		 ver, pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol epat = {
@@ -324,15 +310,15 @@ static struct pi_protocol epat = {
 
 static int __init epat_init(void)
 {
-#ifdef CONFIG_PARIDE_EPATC8
+#ifdef CONFIG_PATA_PARPORT_EPATC8
 	epatc8 = 1;
 #endif
-	return paride_register(&epat);
+	return pata_parport_register_driver(&epat);
 }
 
 static void __exit epat_exit(void)
 {
-	paride_unregister(&epat);
+	pata_parport_unregister_driver(&epat);
 }
 
 MODULE_LICENSE("GPL");
diff --git a/drivers/ata/pata_parport/epia.c b/drivers/ata/pata_parport/epia.c
index 417d5a3c7f72..f6db2f79fe99 100644
--- a/drivers/ata/pata_parport/epia.c
+++ b/drivers/ata/pata_parport/epia.c
@@ -10,15 +10,6 @@
 
 */
 
-/* Changes:
-
-        1.01    GRG 1998.05.06 init_proto, release_proto
-	1.02    GRG 1998.06.17 support older versions of EPIA
-
-*/
-
-#define EPIA_VERSION      "1.02"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -26,8 +17,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 /* mode codes:  0  nybble reads on port 1, 8-bit writes
                 1  5/3 reads on ports 1 & 2, 8-bit writes
@@ -46,7 +36,7 @@
 
 static int cont_map[2] = { 0, 0x80 };
 
-static int epia_read_regr( PIA *pi, int cont, int regr )
+static int epia_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {       int     a, b, r;
 
@@ -79,7 +69,7 @@ static int epia_read_regr( PIA *pi, int cont, int regr )
         return -1;
 }       
 
-static void epia_write_regr( PIA *pi, int cont, int regr, int val)
+static void epia_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {       int  r;
 
@@ -110,7 +100,7 @@ static void epia_write_regr( PIA *pi, int cont, int regr, int val)
    2048 byte reads (the last two being used in the CDrom drivers.
 */
 
-static void epia_connect ( PIA *pi  )
+static void epia_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
         pi->saved_r2 = r2();
@@ -124,7 +114,7 @@ static void epia_connect ( PIA *pi  )
         WR(0x86,8);  
 }
 
-static void epia_disconnect ( PIA *pi )
+static void epia_disconnect(struct pi_adapter *pi)
 
 {       /* WR(0x84,0x10); */
         w0(pi->saved_r0);
@@ -133,7 +123,7 @@ static void epia_disconnect ( PIA *pi )
         w2(pi->saved_r2);
 } 
 
-static void epia_read_block( PIA *pi, char * buf, int count )
+static void epia_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {       int     k, ph, a, b;
 
@@ -193,7 +183,7 @@ static void epia_read_block( PIA *pi, char * buf, int count )
         }
 }
 
-static void epia_write_block( PIA *pi, char * buf, int count )
+static void epia_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {       int     ph, k, last, d;
 
@@ -234,10 +224,11 @@ static void epia_write_block( PIA *pi, char * buf, int count )
 
 }
 
-static int epia_test_proto( PIA *pi, char * scratch, int verbose )
+static int epia_test_proto(struct pi_adapter *pi)
 
 {       int     j, k, f;
 	int	e[2] = {0,0};
+	char scratch[512];
 
         epia_connect(pi);
         for (j=0;j<2;j++) {
@@ -262,26 +253,21 @@ static int epia_test_proto( PIA *pi, char * scratch, int verbose )
         WR(0x84,0);
         epia_disconnect(pi);
 
-        if (verbose)  {
-            printk("%s: epia: port 0x%x, mode %d, test=(%d,%d,%d)\n",
-                   pi->device,pi->port,pi->mode,e[0],e[1],f);
-        }
+	dev_dbg(&pi->dev, "epia: port 0x%x, mode %d, test=(%d,%d,%d)\n",
+	       pi->port, pi->mode, e[0], e[1], f);
         
         return (e[0] && e[1]) || f;
 
 }
 
 
-static void epia_log_adapter( PIA *pi, char * scratch, int verbose )
+static void epia_log_adapter(struct pi_adapter *pi)
 
 {       char    *mode_string[6] = {"4-bit","5/3","8-bit",
 				   "EPP-8","EPP-16","EPP-32"};
 
-        printk("%s: epia %s, Shuttle EPIA at 0x%x, ",
-                pi->device,EPIA_VERSION,pi->port);
-        printk("mode %d (%s), delay %d\n",pi->mode,
-		mode_string[pi->mode],pi->delay);
-
+	dev_info(&pi->dev, "Shuttle EPIA at 0x%x, mode %d (%s), delay %d\n",
+		pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol epia = {
@@ -301,16 +287,5 @@ static struct pi_protocol epia = {
 	.log_adapter	= epia_log_adapter,
 };
 
-static int __init epia_init(void)
-{
-	return paride_register(&epia);
-}
-
-static void __exit epia_exit(void)
-{
-	paride_unregister(&epia);
-}
-
 MODULE_LICENSE("GPL");
-module_init(epia_init)
-module_exit(epia_exit)
+module_pata_parport_driver(epia);
diff --git a/drivers/ata/pata_parport/fit2.c b/drivers/ata/pata_parport/fit2.c
index 3c7a1069b026..fd3b2ce426a5 100644
--- a/drivers/ata/pata_parport/fit2.c
+++ b/drivers/ata/pata_parport/fit2.c
@@ -13,8 +13,6 @@
 
 */
 
-#define FIT2_VERSION      "1.0"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -22,8 +20,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #define j44(a,b)                (((a>>4)&0x0f)|(b&0xf0))
 
@@ -37,13 +34,13 @@ devices.
 
 */
 
-static void  fit2_write_regr( PIA *pi, int cont, int regr, int val)
+static void fit2_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {	if (cont == 1) return;
 	w2(0xc); w0(regr); w2(4); w0(val); w2(5); w0(0); w2(4);
 }
 
-static int fit2_read_regr( PIA *pi, int cont, int regr )
+static int fit2_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {	int  a, b, r;
 
@@ -61,7 +58,7 @@ static int fit2_read_regr( PIA *pi, int cont, int regr )
 
 }
 
-static void fit2_read_block( PIA *pi, char * buf, int count )
+static void fit2_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int  k, a, b, c, d;
 
@@ -87,7 +84,7 @@ static void fit2_read_block( PIA *pi, char * buf, int count )
 
 }
 
-static void fit2_write_block( PIA *pi, char * buf, int count )
+static void fit2_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int k;
 
@@ -100,23 +97,24 @@ static void fit2_write_block( PIA *pi, char * buf, int count )
 	w2(4);
 }
 
-static void fit2_connect ( PIA *pi  )
+static void fit2_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
         pi->saved_r2 = r2();
 	w2(0xcc); 
 }
 
-static void fit2_disconnect ( PIA *pi )
+static void fit2_disconnect(struct pi_adapter *pi)
 
 {       w0(pi->saved_r0);
         w2(pi->saved_r2);
 } 
 
-static void fit2_log_adapter( PIA *pi, char * scratch, int verbose )
+static void fit2_log_adapter(struct pi_adapter *pi)
 
-{       printk("%s: fit2 %s, FIT 2000 adapter at 0x%x, delay %d\n",
-                pi->device,FIT2_VERSION,pi->port,pi->delay);
+{
+	dev_info(&pi->dev, "FIT 2000 adapter at 0x%x, delay %d\n",
+		pi->port, pi->delay);
 
 }
 
@@ -136,16 +134,5 @@ static struct pi_protocol fit2 = {
 	.log_adapter	= fit2_log_adapter,
 };
 
-static int __init fit2_init(void)
-{
-	return paride_register(&fit2);
-}
-
-static void __exit fit2_exit(void)
-{
-	paride_unregister(&fit2);
-}
-
 MODULE_LICENSE("GPL");
-module_init(fit2_init)
-module_exit(fit2_exit)
+module_pata_parport_driver(fit2);
diff --git a/drivers/ata/pata_parport/fit3.c b/drivers/ata/pata_parport/fit3.c
index cd95f4f0edc2..75df656ac472 100644
--- a/drivers/ata/pata_parport/fit3.c
+++ b/drivers/ata/pata_parport/fit3.c
@@ -17,8 +17,6 @@
 
 */
 
-#define FIT3_VERSION      "1.0"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -26,8 +24,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #define j44(a,b)                (((a>>3)&0x0f)|((b<<1)&0xf0))
 
@@ -39,7 +36,7 @@
 
 */
 
-static void  fit3_write_regr( PIA *pi, int cont, int regr, int val)
+static void fit3_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {	if (cont == 1) return;
 
@@ -59,7 +56,7 @@ static void  fit3_write_regr( PIA *pi, int cont, int regr, int val)
 	}
 }
 
-static int fit3_read_regr( PIA *pi, int cont, int regr )
+static int fit3_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {	int  a, b;
 
@@ -92,7 +89,7 @@ static int fit3_read_regr( PIA *pi, int cont, int regr )
 
 }
 
-static void fit3_read_block( PIA *pi, char * buf, int count )
+static void fit3_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int  k, a, b, c, d;
 
@@ -131,7 +128,7 @@ static void fit3_read_block( PIA *pi, char * buf, int count )
 	}
 }
 
-static void fit3_write_block( PIA *pi, char * buf, int count )
+static void fit3_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int k;
 
@@ -152,7 +149,7 @@ static void fit3_write_block( PIA *pi, char * buf, int count )
 	}
 }
 
-static void fit3_connect ( PIA *pi  )
+static void fit3_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
         pi->saved_r2 = r2();
@@ -162,22 +159,19 @@ static void fit3_connect ( PIA *pi  )
 		}
 }
 
-static void fit3_disconnect ( PIA *pi )
+static void fit3_disconnect(struct pi_adapter *pi)
 
 {       w2(0xc); w0(0xa); w2(0x8); w2(0xc);
 	w0(pi->saved_r0);
         w2(pi->saved_r2);
 } 
 
-static void fit3_log_adapter( PIA *pi, char * scratch, int verbose )
+static void fit3_log_adapter(struct pi_adapter *pi)
 
 {       char    *mode_string[3] = {"4-bit","8-bit","EPP"};
 
-	printk("%s: fit3 %s, FIT 3000 adapter at 0x%x, "
-	       "mode %d (%s), delay %d\n",
-                pi->device,FIT3_VERSION,pi->port,
-		pi->mode,mode_string[pi->mode],pi->delay);
-
+	dev_info(&pi->dev, "FIT 3000 adapter at 0x%x, mode %d (%s), delay %d\n",
+		pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol fit3 = {
@@ -196,16 +190,5 @@ static struct pi_protocol fit3 = {
 	.log_adapter	= fit3_log_adapter,
 };
 
-static int __init fit3_init(void)
-{
-	return paride_register(&fit3);
-}
-
-static void __exit fit3_exit(void)
-{
-	paride_unregister(&fit3);
-}
-
 MODULE_LICENSE("GPL");
-module_init(fit3_init)
-module_exit(fit3_exit)
+module_pata_parport_driver(fit3);
diff --git a/drivers/ata/pata_parport/friq.c b/drivers/ata/pata_parport/friq.c
index da1d0cb016d6..1647264cd9a8 100644
--- a/drivers/ata/pata_parport/friq.c
+++ b/drivers/ata/pata_parport/friq.c
@@ -20,13 +20,6 @@
 
 */
 
-/* Changes:
-
-	1.01	GRG 1998.12.20	 Added support for soft power switch
-*/
-
-#define	FRIQ_VERSION	"1.01" 
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -34,8 +27,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #define CMD(x)		w2(4);w0(0xff);w0(0xff);w0(0x73);w0(0x73);\
 			w0(0xc9);w0(0xc9);w0(0x26);w0(0x26);w0(x);w0(x);
@@ -48,7 +40,7 @@
 
 static int  cont_map[2] = { 0x08, 0x10 };
 
-static int friq_read_regr( PIA *pi, int cont, int regr )
+static int friq_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {	int	h,l,r;
 
@@ -63,7 +55,7 @@ static int friq_read_regr( PIA *pi, int cont, int regr )
 
 }
 
-static void friq_write_regr( PIA *pi, int cont, int regr, int val)
+static void friq_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {	int r;
 
@@ -74,7 +66,7 @@ static void friq_write_regr( PIA *pi, int cont, int regr, int val)
 	w2(5);w2(7);w2(5);w2(4);
 }
 
-static void friq_read_block_int( PIA *pi, char * buf, int count, int regr )
+static void friq_read_block_int(struct pi_adapter *pi, char *buf, int count, int regr)
 
 {       int     h, l, k, ph;
 
@@ -129,12 +121,12 @@ static void friq_read_block_int( PIA *pi, char * buf, int count, int regr )
         }
 }
 
-static void friq_read_block( PIA *pi, char * buf, int count)
+static void friq_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {	friq_read_block_int(pi,buf,count,0x08);
 }
 
-static void friq_write_block( PIA *pi, char * buf, int count )
+static void friq_write_block(struct pi_adapter *pi, char *buf, int count)
  
 {	int	k;
 
@@ -166,24 +158,25 @@ static void friq_write_block( PIA *pi, char * buf, int count )
 	}
 }
 
-static void friq_connect ( PIA *pi  )
+static void friq_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
         pi->saved_r2 = r2();
 	w2(4);
 }
 
-static void friq_disconnect ( PIA *pi )
+static void friq_disconnect(struct pi_adapter *pi)
 
 {       CMD(0x20);
 	w0(pi->saved_r0);
         w2(pi->saved_r2);
 } 
 
-static int friq_test_proto( PIA *pi, char * scratch, int verbose )
+static int friq_test_proto(struct pi_adapter *pi)
 
 {       int     j, k, r;
 	int	e[2] = {0,0};
+	char scratch[512];
 
 	pi->saved_r0 = r0();	
 	w0(0xff); udelay(20); CMD(0x3d); /* turn the power on */
@@ -207,24 +200,20 @@ static int friq_test_proto( PIA *pi, char * scratch, int verbose )
         for (k=0;k<128;k++) if (scratch[k] != k) r++;
 	friq_disconnect(pi);
 
-        if (verbose)  {
-            printk("%s: friq: port 0x%x, mode %d, test=(%d,%d,%d)\n",
-                   pi->device,pi->port,pi->mode,e[0],e[1],r);
-        }
+	dev_dbg(&pi->dev, "friq: port 0x%x, mode %d, test=(%d,%d,%d)\n",
+	       pi->port, pi->mode, e[0], e[1], r);
 
         return (r || (e[0] && e[1]));
 }
 
 
-static void friq_log_adapter( PIA *pi, char * scratch, int verbose )
+static void friq_log_adapter(struct pi_adapter *pi)
 
 {       char    *mode_string[6] = {"4-bit","8-bit",
 				   "EPP-8","EPP-16","EPP-32"};
 
-        printk("%s: friq %s, Freecom IQ ASIC-2 adapter at 0x%x, ", pi->device,
-		FRIQ_VERSION,pi->port);
-        printk("mode %d (%s), delay %d\n",pi->mode,
-		mode_string[pi->mode],pi->delay);
+	dev_info(&pi->dev, "Freecom IQ ASIC-2 adapter at 0x%x, mode %d (%s), delay %d\n",
+		pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 
 	pi->private = 1;
 	friq_connect(pi);
@@ -233,7 +222,7 @@ static void friq_log_adapter( PIA *pi, char * scratch, int verbose )
 
 }
 
-static void friq_release_proto( PIA *pi)
+static void friq_release_proto(struct pi_adapter *pi)
 {
 	if (pi->private) {		/* turn off the power */
 		friq_connect(pi);
@@ -261,16 +250,5 @@ static struct pi_protocol friq = {
 	.release_proto	= friq_release_proto,
 };
 
-static int __init friq_init(void)
-{
-	return paride_register(&friq);
-}
-
-static void __exit friq_exit(void)
-{
-	paride_unregister(&friq);
-}
-
 MODULE_LICENSE("GPL");
-module_init(friq_init)
-module_exit(friq_exit)
+module_pata_parport_driver(friq);
diff --git a/drivers/ata/pata_parport/frpw.c b/drivers/ata/pata_parport/frpw.c
index 7bc8fa16d5d8..3ec0abf16fa6 100644
--- a/drivers/ata/pata_parport/frpw.c
+++ b/drivers/ata/pata_parport/frpw.c
@@ -13,18 +13,6 @@
 
 */
 
-/* Changes:
-
-        1.01    GRG 1998.05.06 init_proto, release_proto
-			       fix chip detect
-			       added EPP-16 and EPP-32
-	1.02    GRG 1998.09.23 added hard reset to initialisation process
-	1.03    GRG 1998.12.14 made hard reset conditional
-
-*/
-
-#define	FRPW_VERSION	"1.03" 
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -32,8 +20,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #define cec4		w2(0xc);w2(0xe);w2(0xe);w2(0xc);w2(4);w2(4);w2(4);
 #define j44(l,h)	(((l>>4)&0x0f)|(h&0xf0))
@@ -44,7 +31,7 @@
 
 static int  cont_map[2] = { 0x08, 0x10 };
 
-static int frpw_read_regr( PIA *pi, int cont, int regr )
+static int frpw_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {	int	h,l,r;
 
@@ -60,7 +47,7 @@ static int frpw_read_regr( PIA *pi, int cont, int regr )
 
 }
 
-static void frpw_write_regr( PIA *pi, int cont, int regr, int val)
+static void frpw_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {	int r;
 
@@ -71,7 +58,7 @@ static void frpw_write_regr( PIA *pi, int cont, int regr, int val)
 	w2(5);w2(7);w2(5);w2(4);
 }
 
-static void frpw_read_block_int( PIA *pi, char * buf, int count, int regr )
+static void frpw_read_block_int(struct pi_adapter *pi, char *buf, int count, int regr)
 
 {       int     h, l, k, ph;
 
@@ -132,12 +119,12 @@ static void frpw_read_block_int( PIA *pi, char * buf, int count, int regr )
         }
 }
 
-static void frpw_read_block( PIA *pi, char * buf, int count)
+static void frpw_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {	frpw_read_block_int(pi,buf,count,0x08);
 }
 
-static void frpw_write_block( PIA *pi, char * buf, int count )
+static void frpw_write_block(struct pi_adapter *pi, char *buf, int count)
  
 {	int	k;
 
@@ -170,14 +157,14 @@ static void frpw_write_block( PIA *pi, char * buf, int count )
 	}
 }
 
-static void frpw_connect ( PIA *pi  )
+static void frpw_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
         pi->saved_r2 = r2();
 	w2(4);
 }
 
-static void frpw_disconnect ( PIA *pi )
+static void frpw_disconnect(struct pi_adapter *pi)
 
 {       w2(4); w0(0x20); cec4;
 	w0(pi->saved_r0);
@@ -188,7 +175,7 @@ static void frpw_disconnect ( PIA *pi )
    between the Xilinx and ASIC implementations of the Freecom adapter.
 */
 
-static int frpw_test_pnp ( PIA *pi )
+static int frpw_test_pnp(struct pi_adapter *pi)
 
 /*  returns chip_type:   0 = Xilinx, 1 = ASIC   */
 
@@ -221,25 +208,22 @@ static int frpw_test_pnp ( PIA *pi )
    a hack :-(
 */
 
-static int frpw_test_proto( PIA *pi, char * scratch, int verbose )
+static int frpw_test_proto(struct pi_adapter *pi)
 
 {       int     j, k, r;
 	int	e[2] = {0,0};
+	char scratch[512];
 
 	if ((pi->private>>1) != pi->port)
 	   pi->private = frpw_test_pnp(pi) + 2*pi->port;
 
 	if (((pi->private%2) == 0) && (pi->mode > 2)) {
-	   if (verbose) 
-		printk("%s: frpw: Xilinx does not support mode %d\n",
-			pi->device, pi->mode);
+		dev_dbg(&pi->dev, "frpw: Xilinx does not support mode %d\n", pi->mode);
 	   return 1;
 	}
 
 	if (((pi->private%2) == 1) && (pi->mode == 2)) {
-	   if (verbose)
-		printk("%s: frpw: ASIC does not support mode 2\n",
-			pi->device);
+		dev_dbg(&pi->dev, "frpw: ASIC does not support mode 2\n");
 	   return 1;
 	}
 
@@ -260,25 +244,21 @@ static int frpw_test_proto( PIA *pi, char * scratch, int verbose )
         for (k=0;k<128;k++) if (scratch[k] != k) r++;
 	frpw_disconnect(pi);
 
-        if (verbose)  {
-            printk("%s: frpw: port 0x%x, chip %ld, mode %d, test=(%d,%d,%d)\n",
-                   pi->device,pi->port,(pi->private%2),pi->mode,e[0],e[1],r);
-        }
+	dev_dbg(&pi->dev, "frpw: port 0x%x, chip %ld, mode %d, test=(%d,%d,%d)\n",
+	       pi->port, (pi->private%2), pi->mode, e[0], e[1], r);
 
         return (r || (e[0] && e[1]));
 }
 
 
-static void frpw_log_adapter( PIA *pi, char * scratch, int verbose )
+static void frpw_log_adapter(struct pi_adapter *pi)
 
 {       char    *mode_string[6] = {"4-bit","8-bit","EPP",
 				   "EPP-8","EPP-16","EPP-32"};
 
-        printk("%s: frpw %s, Freecom (%s) adapter at 0x%x, ", pi->device,
-		FRPW_VERSION,((pi->private%2) == 0)?"Xilinx":"ASIC",pi->port);
-        printk("mode %d (%s), delay %d\n",pi->mode,
-		mode_string[pi->mode],pi->delay);
-
+	dev_info(&pi->dev, "Freecom (%s) adapter at 0x%x, mode %d (%s), delay %d\n",
+		((pi->private % 2) == 0) ? "Xilinx" : "ASIC",
+		pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol frpw = {
@@ -298,16 +278,5 @@ static struct pi_protocol frpw = {
 	.log_adapter	= frpw_log_adapter,
 };
 
-static int __init frpw_init(void)
-{
-	return paride_register(&frpw);
-}
-
-static void __exit frpw_exit(void)
-{
-	paride_unregister(&frpw);
-}
-
 MODULE_LICENSE("GPL");
-module_init(frpw_init)
-module_exit(frpw_exit)
+module_pata_parport_driver(frpw);
diff --git a/drivers/ata/pata_parport/kbic.c b/drivers/ata/pata_parport/kbic.c
index f0960eb68635..8213e62f8f00 100644
--- a/drivers/ata/pata_parport/kbic.c
+++ b/drivers/ata/pata_parport/kbic.c
@@ -12,14 +12,6 @@
 
 */
 
-/* Changes:
-
-        1.01    GRG 1998.05.06 init_proto, release_proto
-
-*/
-
-#define KBIC_VERSION      "1.01"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -27,8 +19,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #define r12w()			(delay_p,inw(pi->port+1)&0xffff) 
 
@@ -42,7 +33,7 @@
 
 static int  cont_map[2] = { 0x80, 0x40 };
 
-static int kbic_read_regr( PIA *pi, int cont, int regr )
+static int kbic_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {       int     a, b, s;
 
@@ -72,7 +63,7 @@ static int kbic_read_regr( PIA *pi, int cont, int regr )
 	return -1;
 }       
 
-static void  kbic_write_regr( PIA *pi, int cont, int regr, int val)
+static void kbic_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {       int  s;
 
@@ -96,14 +87,14 @@ static void  kbic_write_regr( PIA *pi, int cont, int regr, int val)
 	}
 }
 
-static void k951_connect ( PIA *pi  )
+static void k951_connect(struct pi_adapter *pi)
 
 { 	pi->saved_r0 = r0();
         pi->saved_r2 = r2();
         w2(4); 
 }
 
-static void k951_disconnect ( PIA *pi )
+static void k951_disconnect(struct pi_adapter *pi)
 
 {      	w0(pi->saved_r0);
         w2(pi->saved_r2);
@@ -112,7 +103,7 @@ static void k951_disconnect ( PIA *pi )
 #define	CCP(x)	w2(0xc4);w0(0xaa);w0(0x55);w0(0);w0(0xff);w0(0x87);\
 		w0(0x78);w0(x);w2(0xc5);w2(0xc4);w0(0xff);
 
-static void k971_connect ( PIA *pi  )
+static void k971_connect(struct pi_adapter *pi)
 
 { 	pi->saved_r0 = r0();
         pi->saved_r2 = r2();
@@ -120,7 +111,7 @@ static void k971_connect ( PIA *pi  )
         w2(4); 
 }
 
-static void k971_disconnect ( PIA *pi )
+static void k971_disconnect(struct pi_adapter *pi)
 
 {       CCP(0x30);
 	w0(pi->saved_r0);
@@ -131,7 +122,7 @@ static void k971_disconnect ( PIA *pi )
    have this property.
 */
 
-static void kbic_read_block( PIA *pi, char * buf, int count )
+static void kbic_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {       int     k, a, b;
 
@@ -189,7 +180,7 @@ static void kbic_read_block( PIA *pi, char * buf, int count )
         }
 }
 
-static void kbic_write_block( PIA *pi, char * buf, int count )
+static void kbic_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {       int     k;
 
@@ -213,12 +204,15 @@ static void kbic_write_block( PIA *pi, char * buf, int count )
 		break;
 
 	case 4: w0(0xa0); w2(4); w2(6); w2(4); w3(0);
-                for(k=0;k<count/2;k++) w4w(pi_swab16(buf,k));
+		for (k = 0; k < count / 2; k++)
+			w4w(swab16(((u16 *)buf)[k]));
                 w2(4); w2(0); w2(4);
                 break;
 
         case 5: w0(0xa0); w2(4); w2(6); w2(4); w3(0);
-                for(k=0;k<count/4;k++) w4l(pi_swab32(buf,k));
+		for (k = 0; k < count / 4; k++)
+			w4l(swab16(((u16 *)buf)[2 * k]) |
+			    swab16(((u16 *)buf)[2 * k + 1]) << 16);
                 w2(4); w2(0); w2(4);
                 break;
 
@@ -226,27 +220,23 @@ static void kbic_write_block( PIA *pi, char * buf, int count )
 
 }
 
-static void kbic_log_adapter( PIA *pi, char * scratch, 
-			      int verbose, char * chip )
+static void kbic_log_adapter(struct pi_adapter *pi, char *chip)
 
 {       char    *mode_string[6] = {"4-bit","5/3","8-bit",
 				   "EPP-8","EPP_16","EPP-32"};
 
-        printk("%s: kbic %s, KingByte %s at 0x%x, ",
-                pi->device,KBIC_VERSION,chip,pi->port);
-        printk("mode %d (%s), delay %d\n",pi->mode,
-		mode_string[pi->mode],pi->delay);
-
+	dev_info(&pi->dev, "KingByte %s at 0x%x, mode %d (%s), delay %d\n",
+		 chip, pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
-static void k951_log_adapter( PIA *pi, char * scratch, int verbose )
-
-{	kbic_log_adapter(pi,scratch,verbose,"KBIC-951A");
+static void k951_log_adapter(struct pi_adapter *pi)
+{
+	kbic_log_adapter(pi, "KBIC-951A");
 }
 
-static void k971_log_adapter( PIA *pi, char * scratch, int verbose )
-
-{       kbic_log_adapter(pi,scratch,verbose,"KBIC-971A");
+static void k971_log_adapter(struct pi_adapter *pi)
+{
+	kbic_log_adapter(pi, "KBIC-971A");
 }
 
 static struct pi_protocol k951 = {
@@ -285,19 +275,19 @@ static int __init kbic_init(void)
 {
 	int rv;
 
-	rv = paride_register(&k951);
+	rv = pata_parport_register_driver(&k951);
 	if (rv < 0)
 		return rv;
-	rv = paride_register(&k971);
+	rv = pata_parport_register_driver(&k971);
 	if (rv < 0)
-		paride_unregister(&k951);
+		pata_parport_unregister_driver(&k951);
 	return rv;
 }
 
 static void __exit kbic_exit(void)
 {
-	paride_unregister(&k951);
-	paride_unregister(&k971);
+	pata_parport_unregister_driver(&k951);
+	pata_parport_unregister_driver(&k971);
 }
 
 MODULE_LICENSE("GPL");
diff --git a/drivers/ata/pata_parport/ktti.c b/drivers/ata/pata_parport/ktti.c
index fc4f707fed1f..4890b1f12348 100644
--- a/drivers/ata/pata_parport/ktti.c
+++ b/drivers/ata/pata_parport/ktti.c
@@ -9,8 +9,6 @@
 
 */
 
-#define KTTI_VERSION      "1.0"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -18,8 +16,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #define j44(a,b)                (((a>>4)&0x0f)|(b&0xf0))
 
@@ -29,7 +26,7 @@
 
 static int  cont_map[2] = { 0x10, 0x08 };
 
-static void  ktti_write_regr( PIA *pi, int cont, int regr, int val)
+static void ktti_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {	int r;
 
@@ -39,7 +36,7 @@ static void  ktti_write_regr( PIA *pi, int cont, int regr, int val)
 	w0(val); w2(3); w0(0); w2(6); w2(0xb);
 }
 
-static int ktti_read_regr( PIA *pi, int cont, int regr )
+static int ktti_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {	int  a, b, r;
 
@@ -51,7 +48,7 @@ static int ktti_read_regr( PIA *pi, int cont, int regr )
 
 }
 
-static void ktti_read_block( PIA *pi, char * buf, int count )
+static void ktti_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int  k, a, b;
 
@@ -64,7 +61,7 @@ static void ktti_read_block( PIA *pi, char * buf, int count )
 	}
 }
 
-static void ktti_write_block( PIA *pi, char * buf, int count )
+static void ktti_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int k;
 
@@ -76,25 +73,25 @@ static void ktti_write_block( PIA *pi, char * buf, int count )
 	}
 }
 
-static void ktti_connect ( PIA *pi  )
+static void ktti_connect(struct pi_adapter *pi)
 
 {       pi->saved_r0 = r0();
         pi->saved_r2 = r2();
 	w2(0xb); w2(0xa); w0(0); w2(3); w2(6);	
 }
 
-static void ktti_disconnect ( PIA *pi )
+static void ktti_disconnect(struct pi_adapter *pi)
 
 {       w2(0xb); w2(0xa); w0(0xa0); w2(3); w2(4);
 	w0(pi->saved_r0);
         w2(pi->saved_r2);
 } 
 
-static void ktti_log_adapter( PIA *pi, char * scratch, int verbose )
-
-{       printk("%s: ktti %s, KT adapter at 0x%x, delay %d\n",
-                pi->device,KTTI_VERSION,pi->port,pi->delay);
+static void ktti_log_adapter(struct pi_adapter *pi)
 
+{
+	dev_info(&pi->dev, "KT adapter at 0x%x, delay %d\n",
+		pi->port, pi->delay);
 }
 
 static struct pi_protocol ktti = {
@@ -113,16 +110,5 @@ static struct pi_protocol ktti = {
 	.log_adapter	= ktti_log_adapter,
 };
 
-static int __init ktti_init(void)
-{
-	return paride_register(&ktti);
-}
-
-static void __exit ktti_exit(void)
-{
-	paride_unregister(&ktti);
-}
-
 MODULE_LICENSE("GPL");
-module_init(ktti_init)
-module_exit(ktti_exit)
+module_pata_parport_driver(ktti);
diff --git a/drivers/ata/pata_parport/on20.c b/drivers/ata/pata_parport/on20.c
index 995fc41e3122..276ace12d490 100644
--- a/drivers/ata/pata_parport/on20.c
+++ b/drivers/ata/pata_parport/on20.c
@@ -6,14 +6,6 @@
         Onspec 90c20 parallel to IDE adapter. 
 */
 
-/* Changes:
-
-        1.01    GRG 1998.05.06 init_proto, release_proto
-
-*/
-
-#define	ON20_VERSION	"1.01"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -21,8 +13,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #define op(f)	w2(4);w0(f);w2(5);w2(0xd);w2(5);w2(0xd);w2(5);w2(4);
 #define vl(v)	w2(4);w0(v);w2(5);w2(7);w2(5);w2(4);
@@ -33,7 +24,7 @@
    cont = 1 - access the IDE command set 
 */
 
-static int on20_read_regr( PIA *pi, int cont, int regr )
+static int on20_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {	int h,l, r ;
 
@@ -56,7 +47,7 @@ static int on20_read_regr( PIA *pi, int cont, int regr )
 	return -1;
 }	
 
-static void on20_write_regr( PIA *pi, int cont, int regr, int val )
+static void on20_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {	int r;
 
@@ -67,7 +58,7 @@ static void on20_write_regr( PIA *pi, int cont, int regr, int val )
 	op(0); vl(val);
 }
 
-static void on20_connect ( PIA *pi)
+static void on20_connect(struct pi_adapter *pi)
 
 {	pi->saved_r0 = r0();
         pi->saved_r2 = r2();
@@ -77,14 +68,14 @@ static void on20_connect ( PIA *pi)
 	       else   { op(2); vl(0); op(2); vl(8); }
 }
 
-static void on20_disconnect ( PIA *pi )
+static void on20_disconnect(struct pi_adapter *pi)
 
 {	w2(4);w0(7);w2(4);w2(0xc);w2(4);
         w0(pi->saved_r0);
         w2(pi->saved_r2);
 } 
 
-static void on20_read_block( PIA *pi, char * buf, int count )
+static void on20_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int     k, l, h; 
 
@@ -101,7 +92,7 @@ static void on20_read_block( PIA *pi, char * buf, int count )
 	w2(4);
 }
 
-static void on20_write_block(  PIA *pi, char * buf, int count )
+static void on20_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {	int	k;
 
@@ -111,15 +102,12 @@ static void on20_write_block(  PIA *pi, char * buf, int count )
 	w2(4);
 }
 
-static void on20_log_adapter( PIA *pi, char * scratch, int verbose )
+static void on20_log_adapter(struct pi_adapter *pi)
 
 {       char    *mode_string[2] = {"4-bit","8-bit"};
 
-        printk("%s: on20 %s, OnSpec 90c20 at 0x%x, ",
-                pi->device,ON20_VERSION,pi->port);
-        printk("mode %d (%s), delay %d\n",pi->mode,
-		mode_string[pi->mode],pi->delay);
-
+	dev_info(&pi->dev, "OnSpec 90c20 at 0x%x, mode %d (%s), delay %d\n",
+		pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol on20 = {
@@ -138,16 +126,5 @@ static struct pi_protocol on20 = {
 	.log_adapter	= on20_log_adapter,
 };
 
-static int __init on20_init(void)
-{
-	return paride_register(&on20);
-}
-
-static void __exit on20_exit(void)
-{
-	paride_unregister(&on20);
-}
-
 MODULE_LICENSE("GPL");
-module_init(on20_init)
-module_exit(on20_exit)
+module_pata_parport_driver(on20);
diff --git a/drivers/ata/pata_parport/on26.c b/drivers/ata/pata_parport/on26.c
index 35f1c481a782..dc47a54b121f 100644
--- a/drivers/ata/pata_parport/on26.c
+++ b/drivers/ata/pata_parport/on26.c
@@ -7,17 +7,6 @@
 
 */
 
-/* Changes:
-
-        1.01    GRG 1998.05.06 init_proto, release_proto
-	1.02    GRG 1998.09.23 updates for the -E rev chip
-	1.03    GRG 1998.12.14 fix for slave drives
-	1.04    GRG 1998.12.20 yet another bug fix
-
-*/
-
-#define ON26_VERSION      "1.04"
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -25,8 +14,7 @@
 #include <linux/types.h>
 #include <linux/wait.h>
 #include <asm/io.h>
-
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 /* mode codes:  0  nybble reads, 8-bit writes
                 1  8-bit reads and writes
@@ -44,7 +32,7 @@
    cont = 1 - access the IDE command set 
 */
 
-static int on26_read_regr( PIA *pi, int cont, int regr )
+static int on26_read_regr(struct pi_adapter *pi, int cont, int regr)
 
 {       int     a, b, r;
 
@@ -73,7 +61,7 @@ static int on26_read_regr( PIA *pi, int cont, int regr )
         return -1;
 }       
 
-static void on26_write_regr( PIA *pi, int cont, int regr, int val )
+static void on26_write_regr(struct pi_adapter *pi, int cont, int regr, int val)
 
 {       int  r;
 
@@ -99,7 +87,7 @@ static void on26_write_regr( PIA *pi, int cont, int regr, int val )
 #define  CCP(x)  w0(0xfe);w0(0xaa);w0(0x55);w0(0);w0(0xff);\
 		 w0(0x87);w0(0x78);w0(x);w2(4);w2(5);w2(4);w0(0xff);
 
-static void on26_connect ( PIA *pi )
+static void on26_connect(struct pi_adapter *pi)
 
 {       int	x;
 
@@ -113,7 +101,7 @@ static void on26_connect ( PIA *pi )
 	w0(2); P1; w0(x); P2;
 }
 
-static void on26_disconnect ( PIA *pi )
+static void on26_disconnect(struct pi_adapter *pi)
 
 {       if (pi->mode >= 2) { w3(4); w3(4); w3(4); w3(4); }
 	              else { w0(4); P1; w0(4); P1; }
@@ -124,7 +112,7 @@ static void on26_disconnect ( PIA *pi )
 
 #define	RESET_WAIT  200
 
-static int on26_test_port( PIA *pi)  /* hard reset */
+static int on26_test_port(struct pi_adapter *pi)  /* hard reset */
 
 {       int     i, m, d, x=0, y=0;
 
@@ -167,7 +155,7 @@ static int on26_test_port( PIA *pi)  /* hard reset */
             }
 
 	    if (i == RESET_WAIT) 
-		printk("on26: Device reset failed (%x,%x)\n",x,y);
+		dev_err(&pi->dev, "on26: Device reset failed (%x,%x)\n", x, y);
 
             w0(4); P1; w0(4); P1;
         }
@@ -183,7 +171,7 @@ static int on26_test_port( PIA *pi)  /* hard reset */
 }
 
 
-static void on26_read_block( PIA *pi, char * buf, int count )
+static void on26_read_block(struct pi_adapter *pi, char *buf, int count)
 
 {       int     k, a, b;
 
@@ -232,7 +220,7 @@ static void on26_read_block( PIA *pi, char * buf, int count )
         }
 }
 
-static void on26_write_block( PIA *pi, char * buf, int count )
+static void on26_write_block(struct pi_adapter *pi, char *buf, int count)
 
 {       int	k;
 
@@ -275,16 +263,13 @@ static void on26_write_block( PIA *pi, char * buf, int count )
 
 }
 
-static void on26_log_adapter( PIA *pi, char * scratch, int verbose )
+static void on26_log_adapter(struct pi_adapter *pi)
 
 {       char    *mode_string[5] = {"4-bit","8-bit","EPP-8",
 				   "EPP-16","EPP-32"};
 
-        printk("%s: on26 %s, OnSpec 90c26 at 0x%x, ",
-                pi->device,ON26_VERSION,pi->port);
-        printk("mode %d (%s), delay %d\n",pi->mode,
-		mode_string[pi->mode],pi->delay);
-
+	dev_info(&pi->dev, "OnSpec 90c26 at 0x%x, mode %d (%s), delay %d\n",
+		pi->port, pi->mode, mode_string[pi->mode], pi->delay);
 }
 
 static struct pi_protocol on26 = {
@@ -304,16 +289,5 @@ static struct pi_protocol on26 = {
 	.log_adapter	= on26_log_adapter,
 };
 
-static int __init on26_init(void)
-{
-	return paride_register(&on26);
-}
-
-static void __exit on26_exit(void)
-{
-	paride_unregister(&on26);
-}
-
 MODULE_LICENSE("GPL");
-module_init(on26_init)
-module_exit(on26_exit)
+module_pata_parport_driver(on26);
diff --git a/drivers/ata/pata_parport/pata_parport.c b/drivers/ata/pata_parport/pata_parport.c
index c1576d943b43..1af64d435d3c 100644
--- a/drivers/ata/pata_parport/pata_parport.c
+++ b/drivers/ata/pata_parport/pata_parport.c
@@ -6,7 +6,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/parport.h>
-#include <linux/pata_parport.h>
+#include "pata_parport.h"
 
 #define DRV_NAME "pata_parport"
 
@@ -276,7 +276,7 @@ static void pi_release(struct pi_adapter *pi)
 	module_put(pi->proto->owner);
 }
 
-static int default_test_proto(struct pi_adapter *pi, char *scratch)
+static int default_test_proto(struct pi_adapter *pi)
 {
 	int j, k;
 	int e[2] = { 0, 0 };
@@ -300,21 +300,21 @@ static int default_test_proto(struct pi_adapter *pi, char *scratch)
 	return e[0] && e[1];	/* not here if both > 0 */
 }
 
-static int pi_test_proto(struct pi_adapter *pi, char *scratch)
+static int pi_test_proto(struct pi_adapter *pi)
 {
 	int res;
 
 	parport_claim_or_block(pi->pardev);
 	if (pi->proto->test_proto)
-		res = pi->proto->test_proto(pi, scratch, 1);
+		res = pi->proto->test_proto(pi);
 	else
-		res = default_test_proto(pi, scratch);
+		res = default_test_proto(pi);
 	parport_release(pi->pardev);
 
 	return res;
 }
 
-static bool pi_probe_mode(struct pi_adapter *pi, int max, char *scratch)
+static bool pi_probe_mode(struct pi_adapter *pi, int max)
 {
 	int best, range;
 
@@ -326,7 +326,7 @@ static bool pi_probe_mode(struct pi_adapter *pi, int max, char *scratch)
 			range = 8;
 		if (range == 8 && pi->port % 8)
 			return false;
-		return !pi_test_proto(pi, scratch);
+		return !pi_test_proto(pi);
 	}
 	best = -1;
 	for (pi->mode = 0; pi->mode < max; pi->mode++) {
@@ -335,14 +335,14 @@ static bool pi_probe_mode(struct pi_adapter *pi, int max, char *scratch)
 			range = 8;
 		if (range == 8 && pi->port % 8)
 			break;
-		if (!pi_test_proto(pi, scratch))
+		if (!pi_test_proto(pi))
 			best = pi->mode;
 	}
 	pi->mode = best;
 	return best > -1;
 }
 
-static bool pi_probe_unit(struct pi_adapter *pi, int unit, char *scratch)
+static bool pi_probe_unit(struct pi_adapter *pi, int unit)
 {
 	int max, s, e;
 
@@ -367,14 +367,14 @@ static bool pi_probe_unit(struct pi_adapter *pi, int unit, char *scratch)
 		for (pi->unit = s; pi->unit < e; pi->unit++) {
 			if (pi->proto->probe_unit(pi)) {
 				parport_release(pi->pardev);
-				return pi_probe_mode(pi, max, scratch);
+				return pi_probe_mode(pi, max);
 			}
 		}
 		parport_release(pi->pardev);
 		return false;
 	}
 
-	return pi_probe_mode(pi, max, scratch);
+	return pi_probe_mode(pi, max);
 }
 
 static void pata_parport_dev_release(struct device *dev)
@@ -399,7 +399,7 @@ static struct device pata_parport_bus = {
 	.release = pata_parport_bus_release,
 };
 
-static struct scsi_host_template pata_parport_sht = {
+static const struct scsi_host_template pata_parport_sht = {
 	PATA_PARPORT_SHT("pata_parport")
 };
 
@@ -420,7 +420,6 @@ static struct pi_adapter *pi_init_one(struct parport *parport,
 			struct pi_protocol *pr, int mode, int unit, int delay)
 {
 	struct pardev_cb par_cb = { };
-	char scratch[512];
 	const struct ata_port_info *ppi[] = { &pata_parport_port_info };
 	struct ata_host *host;
 	struct pi_adapter *pi;
@@ -473,12 +472,12 @@ static struct pi_adapter *pi_init_one(struct parport *parport,
 	if (!pi->pardev)
 		goto out_module_put;
 
-	if (!pi_probe_unit(pi, unit, scratch)) {
+	if (!pi_probe_unit(pi, unit)) {
 		dev_info(&pi->dev, "Adapter not found\n");
 		goto out_unreg_parport;
 	}
 
-	pi->proto->log_adapter(pi, scratch, 1);
+	pi->proto->log_adapter(pi);
 
 	host = ata_host_alloc_pinfo(&pi->pardev->dev, ppi, 1);
 	if (!host)
@@ -534,7 +533,7 @@ int pata_parport_register_driver(struct pi_protocol *pr)
 	if (probe) {
 		/* probe all parports using this protocol */
 		idr_for_each_entry(&parport_list, parport, port_num)
-			pi_init_one(parport, pr, -1, 0, -1);
+			pi_init_one(parport, pr, -1, -1, -1);
 	}
 	mutex_unlock(&pi_mutex);
 
@@ -558,8 +557,7 @@ void pata_parport_unregister_driver(struct pi_protocol *pr)
 }
 EXPORT_SYMBOL_GPL(pata_parport_unregister_driver);
 
-static ssize_t new_device_store(struct bus_type *bus, const char *buf,
-				size_t count)
+static ssize_t new_device_store(const struct bus_type *bus, const char *buf, size_t count)
 {
 	char port[12] = "auto";
 	char protocol[8] = "auto";
@@ -633,8 +631,7 @@ static void pi_remove_one(struct device *dev)
 	/* pata_parport_dev_release will do ida_free(dev->id) and kfree(pi) */
 }
 
-static ssize_t delete_device_store(struct bus_type *bus, const char *buf,
-				   size_t count)
+static ssize_t delete_device_store(const struct bus_type *bus, const char *buf, size_t count)
 {
 	struct device *dev;
 
@@ -669,7 +666,7 @@ static void pata_parport_attach(struct parport *port)
 	if (probe) {
 		/* probe this port using all protocols */
 		idr_for_each_entry(&protocols, pr, pr_num)
-			pi_init_one(port, pr, -1, 0, -1);
+			pi_init_one(port, pr, -1, -1, -1);
 	}
 	mutex_unlock(&pi_mutex);
 }
diff --git a/drivers/ata/pata_parport/pata_parport.h b/drivers/ata/pata_parport/pata_parport.h
new file mode 100644
index 000000000000..bbfa4e63ee85
--- /dev/null
+++ b/drivers/ata/pata_parport/pata_parport.h
@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ *	pata_parport.h	(c) 1997-8  Grant R. Guenther <grant@torque.net>
+ *				    Under the terms of the GPL.
+ *
+ * This file defines the interface for parallel port IDE adapter chip drivers.
+ */
+
+#ifndef LINUX_PATA_PARPORT_H
+#define LINUX_PATA_PARPORT_H
+
+#include <linux/libata.h>
+
+struct pi_adapter {
+	struct device dev;
+	struct pi_protocol *proto;	/* adapter protocol */
+	int port;			/* base address of parallel port */
+	int mode;			/* transfer mode in use */
+	int delay;			/* adapter delay setting */
+	int unit;			/* unit number for chained adapters */
+	int saved_r0;			/* saved port state */
+	int saved_r2;			/* saved port state */
+	unsigned long private;		/* for protocol module */
+	struct pardevice *pardev;	/* pointer to pardevice */
+};
+
+/* registers are addressed as (cont,regr)
+ *	cont: 0 for command register file, 1 for control register(s)
+ *	regr: 0-7 for register number.
+ */
+
+/* macros and functions exported to the protocol modules */
+#define delay_p			(pi->delay ? udelay(pi->delay) : (void)0)
+#define out_p(offs, byte)	do { outb(byte, pi->port + offs); delay_p; } while (0)
+#define in_p(offs)		(delay_p, inb(pi->port + offs))
+
+#define w0(byte)		out_p(0, byte)
+#define r0()			in_p(0)
+#define w1(byte)		out_p(1, byte)
+#define r1()			in_p(1)
+#define w2(byte)		out_p(2, byte)
+#define r2()			in_p(2)
+#define w3(byte)		out_p(3, byte)
+#define w4(byte)		out_p(4, byte)
+#define r4()			in_p(4)
+#define w4w(data)		do { outw(data, pi->port + 4); delay_p; } while (0)
+#define w4l(data)		do { outl(data, pi->port + 4); delay_p; } while (0)
+#define r4w()			(delay_p, inw(pi->port + 4))
+#define r4l()			(delay_p, inl(pi->port + 4))
+
+struct pi_protocol {
+	char name[8];
+
+	int max_mode;
+	int epp_first;		/* modes >= this use 8 ports */
+
+	int default_delay;
+	int max_units;		/* max chained units probed for */
+
+	void (*write_regr)(struct pi_adapter *pi, int cont, int regr, int val);
+	int (*read_regr)(struct pi_adapter *pi, int cont, int regr);
+	void (*write_block)(struct pi_adapter *pi, char *buf, int count);
+	void (*read_block)(struct pi_adapter *pi, char *buf, int count);
+
+	void (*connect)(struct pi_adapter *pi);
+	void (*disconnect)(struct pi_adapter *pi);
+
+	int (*test_port)(struct pi_adapter *pi);
+	int (*probe_unit)(struct pi_adapter *pi);
+	int (*test_proto)(struct pi_adapter *pi);
+	void (*log_adapter)(struct pi_adapter *pi);
+
+	int (*init_proto)(struct pi_adapter *pi);
+	void (*release_proto)(struct pi_adapter *pi);
+	struct module *owner;
+	struct device_driver driver;
+	struct scsi_host_template sht;
+};
+
+#define PATA_PARPORT_SHT ATA_PIO_SHT
+
+int pata_parport_register_driver(struct pi_protocol *pr);
+void pata_parport_unregister_driver(struct pi_protocol *pr);
+
+/**
+ * module_pata_parport_driver() - Helper macro for registering a pata_parport driver
+ * @__pi_protocol: pi_protocol struct
+ *
+ * Helper macro for pata_parport drivers which do not do anything special in module
+ * init/exit. This eliminates a lot of boilerplate. Each module may only
+ * use this macro once, and calling it replaces module_init() and module_exit()
+ */
+#define module_pata_parport_driver(__pi_protocol) \
+	module_driver(__pi_protocol, pata_parport_register_driver, pata_parport_unregister_driver)
+
+#endif /* LINUX_PATA_PARPORT_H */
diff --git a/drivers/ata/pata_parport/ppc6lnx.c b/drivers/ata/pata_parport/ppc6lnx.c
deleted file mode 100644
index 5e5521d3b1dd..000000000000
--- a/drivers/ata/pata_parport/ppc6lnx.c
+++ /dev/null
@@ -1,726 +0,0 @@
-/*
-	ppc6lnx.c (c) 2001 Micro Solutions Inc.
-		Released under the terms of the GNU General Public license
-
-	ppc6lnx.c  is a par of the protocol driver for the Micro Solutions
-		"BACKPACK" parallel port IDE adapter
-		(Works on Series 6 drives)
-
-*/
-
-//***************************************************************************
-
-// PPC 6 Code in C sanitized for LINUX
-// Original x86 ASM by Ron, Converted to C by Clive
-
-//***************************************************************************
-
-
-#define port_stb					1
-#define port_afd					2
-#define cmd_stb						port_afd
-#define port_init					4
-#define data_stb					port_init
-#define port_sel					8
-#define port_int					16
-#define port_dir					0x20
-
-#define ECR_EPP	0x80
-#define ECR_BI	0x20
-
-//***************************************************************************
-
-//  60772 Commands
-
-#define ACCESS_REG				0x00
-#define ACCESS_PORT				0x40
-
-#define ACCESS_READ				0x00
-#define ACCESS_WRITE			0x20
-
-//  60772 Command Prefix
-
-#define CMD_PREFIX_SET		0xe0		// Special command that modifies the next command's operation
-#define CMD_PREFIX_RESET	0xc0		// Resets current cmd modifier reg bits
- #define PREFIX_IO16			0x01		// perform 16-bit wide I/O
- #define PREFIX_FASTWR		0x04		// enable PPC mode fast-write
- #define PREFIX_BLK				0x08		// enable block transfer mode
-
-// 60772 Registers
-
-#define REG_STATUS				0x00		// status register
- #define STATUS_IRQA			0x01		// Peripheral IRQA line
- #define STATUS_EEPROM_DO	0x40		// Serial EEPROM data bit
-#define REG_VERSION				0x01		// PPC version register (read)
-#define REG_HWCFG					0x02		// Hardware Config register
-#define REG_RAMSIZE				0x03		// Size of RAM Buffer
- #define RAMSIZE_128K			0x02
-#define REG_EEPROM				0x06		// EEPROM control register
- #define EEPROM_SK				0x01		// eeprom SK bit
- #define EEPROM_DI				0x02		// eeprom DI bit
- #define EEPROM_CS				0x04		// eeprom CS bit
- #define EEPROM_EN				0x08		// eeprom output enable
-#define REG_BLKSIZE				0x08		// Block transfer len (24 bit)
-
-//***************************************************************************
-
-typedef struct ppc_storage {
-	u16	lpt_addr;				// LPT base address
-	u8	ppc_id;
-	u8	mode;						// operating mode
-					// 0 = PPC Uni SW
-					// 1 = PPC Uni FW
-					// 2 = PPC Bi SW
-					// 3 = PPC Bi FW
-					// 4 = EPP Byte
-					// 5 = EPP Word
-					// 6 = EPP Dword
-	u8	ppc_flags;
-	u8	org_data;				// original LPT data port contents
-	u8	org_ctrl;				// original LPT control port contents
-	u8	cur_ctrl;				// current control port contents
-} Interface;
-
-//***************************************************************************
-
-// ppc_flags
-
-#define fifo_wait					0x10
-
-//***************************************************************************
-
-// DONT CHANGE THESE LEST YOU BREAK EVERYTHING - BIT FIELD DEPENDENCIES
-
-#define PPCMODE_UNI_SW		0
-#define PPCMODE_UNI_FW		1
-#define PPCMODE_BI_SW			2
-#define PPCMODE_BI_FW			3
-#define PPCMODE_EPP_BYTE	4
-#define PPCMODE_EPP_WORD	5
-#define PPCMODE_EPP_DWORD	6
-
-//***************************************************************************
-
-static int ppc6_select(Interface *ppc);
-static void ppc6_deselect(Interface *ppc);
-static void ppc6_send_cmd(Interface *ppc, u8 cmd);
-static void ppc6_wr_data_byte(Interface *ppc, u8 data);
-static u8 ppc6_rd_data_byte(Interface *ppc);
-static u8 ppc6_rd_port(Interface *ppc, u8 port);
-static void ppc6_wr_port(Interface *ppc, u8 port, u8 data);
-static void ppc6_rd_data_blk(Interface *ppc, u8 *data, long count);
-static void ppc6_wait_for_fifo(Interface *ppc);
-static void ppc6_wr_data_blk(Interface *ppc, u8 *data, long count);
-static void ppc6_rd_port16_blk(Interface *ppc, u8 port, u8 *data, long length);
-static void ppc6_wr_port16_blk(Interface *ppc, u8 port, u8 *data, long length);
-static void ppc6_wr_extout(Interface *ppc, u8 regdata);
-static int ppc6_open(Interface *ppc);
-static void ppc6_close(Interface *ppc);
-
-//***************************************************************************
-
-static int ppc6_select(Interface *ppc)
-{
-	u8 i, j, k;
-
-	i = inb(ppc->lpt_addr + 1);
-
-	if (i & 1)
-		outb(i, ppc->lpt_addr + 1);
-
-	ppc->org_data = inb(ppc->lpt_addr);
-
-	ppc->org_ctrl = inb(ppc->lpt_addr + 2) & 0x5F; // readback ctrl
-
-	ppc->cur_ctrl = ppc->org_ctrl;
-
-	ppc->cur_ctrl |= port_sel;
-
-	outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-	if (ppc->org_data == 'b')
-		outb('x', ppc->lpt_addr);
-
-	outb('b', ppc->lpt_addr);
-	outb('p', ppc->lpt_addr);
-	outb(ppc->ppc_id, ppc->lpt_addr);
-	outb(~ppc->ppc_id,ppc->lpt_addr);
-
-	ppc->cur_ctrl &= ~port_sel;
-
-	outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-	ppc->cur_ctrl = (ppc->cur_ctrl & port_int) | port_init;
-
-	outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-	i = ppc->mode & 0x0C;
-
-	if (i == 0)
-		i = (ppc->mode & 2) | 1;
-
-	outb(i, ppc->lpt_addr);
-
-	ppc->cur_ctrl |= port_sel;
-
-	outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-	// DELAY
-
-	ppc->cur_ctrl |= port_afd;
-
-	outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-	j = ((i & 0x08) << 4) | ((i & 0x07) << 3);
-
-	k = inb(ppc->lpt_addr + 1) & 0xB8;
-
-	if (j == k)
-	{
-		ppc->cur_ctrl &= ~port_afd;
-
-		outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-		k = (inb(ppc->lpt_addr + 1) & 0xB8) ^ 0xB8;
-
-		if (j == k)
-		{
-			if (i & 4)	// EPP
-				ppc->cur_ctrl &= ~(port_sel | port_init);
-			else				// PPC/ECP
-				ppc->cur_ctrl &= ~port_sel;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			return(1);
-		}
-	}
-
-	outb(ppc->org_ctrl, ppc->lpt_addr + 2);
-
-	outb(ppc->org_data, ppc->lpt_addr);
-
-	return(0); // FAIL
-}
-
-//***************************************************************************
-
-static void ppc6_deselect(Interface *ppc)
-{
-	if (ppc->mode & 4)	// EPP
-		ppc->cur_ctrl |= port_init;
-	else								// PPC/ECP
-		ppc->cur_ctrl |= port_sel;
-
-	outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-	outb(ppc->org_data, ppc->lpt_addr);
-
-	outb((ppc->org_ctrl | port_sel), ppc->lpt_addr + 2);
-
-	outb(ppc->org_ctrl, ppc->lpt_addr + 2);
-}
-
-//***************************************************************************
-
-static void ppc6_send_cmd(Interface *ppc, u8 cmd)
-{
-	switch(ppc->mode)
-	{
-		case PPCMODE_UNI_SW :
-		case PPCMODE_UNI_FW :
-		case PPCMODE_BI_SW :
-		case PPCMODE_BI_FW :
-		{
-			outb(cmd, ppc->lpt_addr);
-
-			ppc->cur_ctrl ^= cmd_stb;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			break;
-		}
-
-		case PPCMODE_EPP_BYTE :
-		case PPCMODE_EPP_WORD :
-		case PPCMODE_EPP_DWORD :
-		{
-			outb(cmd, ppc->lpt_addr + 3);
-
-			break;
-		}
-	}
-}
-
-//***************************************************************************
-
-static void ppc6_wr_data_byte(Interface *ppc, u8 data)
-{
-	switch(ppc->mode)
-	{
-		case PPCMODE_UNI_SW :
-		case PPCMODE_UNI_FW :
-		case PPCMODE_BI_SW :
-		case PPCMODE_BI_FW :
-		{
-			outb(data, ppc->lpt_addr);
-
-			ppc->cur_ctrl ^= data_stb;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			break;
-		}
-
-		case PPCMODE_EPP_BYTE :
-		case PPCMODE_EPP_WORD :
-		case PPCMODE_EPP_DWORD :
-		{
-			outb(data, ppc->lpt_addr + 4);
-
-			break;
-		}
-	}
-}
-
-//***************************************************************************
-
-static u8 ppc6_rd_data_byte(Interface *ppc)
-{
-	u8 data = 0;
-
-	switch(ppc->mode)
-	{
-		case PPCMODE_UNI_SW :
-		case PPCMODE_UNI_FW :
-		{
-			ppc->cur_ctrl = (ppc->cur_ctrl & ~port_stb) ^ data_stb;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			// DELAY
-
-			data = inb(ppc->lpt_addr + 1);
-
-			data = ((data & 0x80) >> 1) | ((data & 0x38) >> 3);
-
-			ppc->cur_ctrl |= port_stb;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			// DELAY
-
-			data |= inb(ppc->lpt_addr + 1) & 0xB8;
-
-			break;
-		}
-
-		case PPCMODE_BI_SW :
-		case PPCMODE_BI_FW :
-		{
-			ppc->cur_ctrl |= port_dir;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			ppc->cur_ctrl = (ppc->cur_ctrl | port_stb) ^ data_stb;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			data = inb(ppc->lpt_addr);
-
-			ppc->cur_ctrl &= ~port_stb;
-
-			outb(ppc->cur_ctrl,ppc->lpt_addr + 2);
-
-			ppc->cur_ctrl &= ~port_dir;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			break;
-		}
-
-		case PPCMODE_EPP_BYTE :
-		case PPCMODE_EPP_WORD :
-		case PPCMODE_EPP_DWORD :
-		{
-			outb((ppc->cur_ctrl | port_dir),ppc->lpt_addr + 2);
-
-			data = inb(ppc->lpt_addr + 4);
-
-			outb(ppc->cur_ctrl,ppc->lpt_addr + 2);
-
-			break;
-		}
-	}
-
-	return(data);
-}
-
-//***************************************************************************
-
-static u8 ppc6_rd_port(Interface *ppc, u8 port)
-{
-	ppc6_send_cmd(ppc,(u8)(port | ACCESS_PORT | ACCESS_READ));
-
-	return(ppc6_rd_data_byte(ppc));
-}
-
-//***************************************************************************
-
-static void ppc6_wr_port(Interface *ppc, u8 port, u8 data)
-{
-	ppc6_send_cmd(ppc,(u8)(port | ACCESS_PORT | ACCESS_WRITE));
-
-	ppc6_wr_data_byte(ppc, data);
-}
-
-//***************************************************************************
-
-static void ppc6_rd_data_blk(Interface *ppc, u8 *data, long count)
-{
-	switch(ppc->mode)
-	{
-		case PPCMODE_UNI_SW :
-		case PPCMODE_UNI_FW :
-		{
-			while(count)
-			{
-				u8 d;
-
-				ppc->cur_ctrl = (ppc->cur_ctrl & ~port_stb) ^ data_stb;
-
-				outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-				// DELAY
-
-				d = inb(ppc->lpt_addr + 1);
-
-				d = ((d & 0x80) >> 1) | ((d & 0x38) >> 3);
-
-				ppc->cur_ctrl |= port_stb;
-
-				outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-				// DELAY
-
-				d |= inb(ppc->lpt_addr + 1) & 0xB8;
-
-				*data++ = d;
-				count--;
-			}
-
-			break;
-		}
-
-		case PPCMODE_BI_SW :
-		case PPCMODE_BI_FW :
-		{
-			ppc->cur_ctrl |= port_dir;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			ppc->cur_ctrl |= port_stb;
-
-			while(count)
-			{
-				ppc->cur_ctrl ^= data_stb;
-
-				outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-				*data++ = inb(ppc->lpt_addr);
-				count--;
-			}
-
-			ppc->cur_ctrl &= ~port_stb;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			ppc->cur_ctrl &= ~port_dir;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			break;
-		}
-
-		case PPCMODE_EPP_BYTE :
-		{
-			outb((ppc->cur_ctrl | port_dir), ppc->lpt_addr + 2);
-
-			// DELAY
-
-			while(count)
-			{
-				*data++ = inb(ppc->lpt_addr + 4);
-				count--;
-			}
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			break;
-		}
-
-		case PPCMODE_EPP_WORD :
-		{
-			outb((ppc->cur_ctrl | port_dir), ppc->lpt_addr + 2);
-
-			// DELAY
-
-			while(count > 1)
-			{
-				*((u16 *)data) = inw(ppc->lpt_addr + 4);
-				data  += 2;
-				count -= 2;
-			}
-
-			while(count)
-			{
-				*data++ = inb(ppc->lpt_addr + 4);
-				count--;
-			}
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			break;
-		}
-
-		case PPCMODE_EPP_DWORD :
-		{
-			outb((ppc->cur_ctrl | port_dir),ppc->lpt_addr + 2);
-
-			// DELAY
-
-			while(count > 3)
-			{
-				*((u32 *)data) = inl(ppc->lpt_addr + 4);
-				data  += 4;
-				count -= 4;
-			}
-
-			while(count)
-			{
-				*data++ = inb(ppc->lpt_addr + 4);
-				count--;
-			}
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			break;
-		}
-	}
-
-}
-
-//***************************************************************************
-
-static void ppc6_wait_for_fifo(Interface *ppc)
-{
-	int i;
-
-	if (ppc->ppc_flags & fifo_wait)
-	{
-		for(i=0; i<20; i++)
-			inb(ppc->lpt_addr + 1);
-	}
-}
-
-//***************************************************************************
-
-static void ppc6_wr_data_blk(Interface *ppc, u8 *data, long count)
-{
-	switch(ppc->mode)
-	{
-		case PPCMODE_UNI_SW :
-		case PPCMODE_BI_SW :
-		{
-			while(count--)
-			{
-				outb(*data++, ppc->lpt_addr);
-
-				ppc->cur_ctrl ^= data_stb;
-
-				outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-			}
-
-			break;
-		}
-
-		case PPCMODE_UNI_FW :
-		case PPCMODE_BI_FW :
-		{
-			u8 this, last;
-
-			ppc6_send_cmd(ppc,(CMD_PREFIX_SET | PREFIX_FASTWR));
-
-			ppc->cur_ctrl |= port_stb;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			last = *data;
-
-			outb(last, ppc->lpt_addr);
-
-			while(count)
-			{
-				this = *data++;
-				count--;
-
-				if (this == last)
-				{
-					ppc->cur_ctrl ^= data_stb;
-
-					outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-				}
-				else
-				{
-					outb(this, ppc->lpt_addr);
-
-					last = this;
-				}
-			}
-
-			ppc->cur_ctrl &= ~port_stb;
-
-			outb(ppc->cur_ctrl, ppc->lpt_addr + 2);
-
-			ppc6_send_cmd(ppc,(CMD_PREFIX_RESET | PREFIX_FASTWR));
-
-			break;
-		}
-
-		case PPCMODE_EPP_BYTE :
-		{
-			while(count)
-			{
-				outb(*data++,ppc->lpt_addr + 4);
-				count--;
-			}
-
-			ppc6_wait_for_fifo(ppc);
-
-			break;
-		}
-
-		case PPCMODE_EPP_WORD :
-		{
-			while(count > 1)
-			{
-				outw(*((u16 *)data),ppc->lpt_addr + 4);
-				data  += 2;
-				count -= 2;
-			}
-
-			while(count)
-			{
-				outb(*data++,ppc->lpt_addr + 4);
-				count--;
-			}
-
-			ppc6_wait_for_fifo(ppc);
-
-			break;
-		}
-
-		case PPCMODE_EPP_DWORD :
-		{
-			while(count > 3)
-			{
-				outl(*((u32 *)data),ppc->lpt_addr + 4);
-				data  += 4;
-				count -= 4;
-			}
-
-			while(count)
-			{
-				outb(*data++,ppc->lpt_addr + 4);
-				count--;
-			}
-
-			ppc6_wait_for_fifo(ppc);
-
-			break;
-		}
-	}
-}
-
-//***************************************************************************
-
-static void ppc6_rd_port16_blk(Interface *ppc, u8 port, u8 *data, long length)
-{
-	length = length << 1;
-
-	ppc6_send_cmd(ppc, (REG_BLKSIZE | ACCESS_REG | ACCESS_WRITE));
-	ppc6_wr_data_byte(ppc,(u8)length);
-	ppc6_wr_data_byte(ppc,(u8)(length >> 8));
-	ppc6_wr_data_byte(ppc,0);
-
-	ppc6_send_cmd(ppc, (CMD_PREFIX_SET | PREFIX_IO16 | PREFIX_BLK));
-
-	ppc6_send_cmd(ppc, (u8)(port | ACCESS_PORT | ACCESS_READ));
-
-	ppc6_rd_data_blk(ppc, data, length);
-
-	ppc6_send_cmd(ppc, (CMD_PREFIX_RESET | PREFIX_IO16 | PREFIX_BLK));
-}
-
-//***************************************************************************
-
-static void ppc6_wr_port16_blk(Interface *ppc, u8 port, u8 *data, long length)
-{
-	length = length << 1;
-
-	ppc6_send_cmd(ppc, (REG_BLKSIZE | ACCESS_REG | ACCESS_WRITE));
-	ppc6_wr_data_byte(ppc,(u8)length);
-	ppc6_wr_data_byte(ppc,(u8)(length >> 8));
-	ppc6_wr_data_byte(ppc,0);
-
-	ppc6_send_cmd(ppc, (CMD_PREFIX_SET | PREFIX_IO16 | PREFIX_BLK));
-
-	ppc6_send_cmd(ppc, (u8)(port | ACCESS_PORT | ACCESS_WRITE));
-
-	ppc6_wr_data_blk(ppc, data, length);
-
-	ppc6_send_cmd(ppc, (CMD_PREFIX_RESET | PREFIX_IO16 | PREFIX_BLK));
-}
-
-//***************************************************************************
-
-static void ppc6_wr_extout(Interface *ppc, u8 regdata)
-{
-	ppc6_send_cmd(ppc,(REG_VERSION | ACCESS_REG | ACCESS_WRITE));
-
-	ppc6_wr_data_byte(ppc, (u8)((regdata & 0x03) << 6));
-}
-
-//***************************************************************************
-
-static int ppc6_open(Interface *ppc)
-{
-	int ret;
-
-	ret = ppc6_select(ppc);
-
-	if (ret == 0)
-		return(ret);
-
-	ppc->ppc_flags &= ~fifo_wait;
-
-	ppc6_send_cmd(ppc, (ACCESS_REG | ACCESS_WRITE | REG_RAMSIZE));
-	ppc6_wr_data_byte(ppc, RAMSIZE_128K);
-
-	ppc6_send_cmd(ppc, (ACCESS_REG | ACCESS_READ | REG_VERSION));
-
-	if ((ppc6_rd_data_byte(ppc) & 0x3F) == 0x0C)
-		ppc->ppc_flags |= fifo_wait;
-
-	return(ret);
-}
-
-//***************************************************************************
-
-static void ppc6_close(Interface *ppc)
-{
-	ppc6_deselect(ppc);
-}
-
-//***************************************************************************
-
diff --git a/drivers/ata/pata_pcmcia.c b/drivers/ata/pata_pcmcia.c
index 8eb066abbd9c..5b602206c522 100644
--- a/drivers/ata/pata_pcmcia.c
+++ b/drivers/ata/pata_pcmcia.c
@@ -132,7 +132,7 @@ static void pcmcia_8bit_drain_fifo(struct ata_queued_cmd *qc)
 
 }
 
-static struct scsi_host_template pcmcia_sht = {
+static const struct scsi_host_template pcmcia_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_pdc2027x.c b/drivers/ata/pata_pdc2027x.c
index 4191aa61c8e4..6820c5597b14 100644
--- a/drivers/ata/pata_pdc2027x.c
+++ b/drivers/ata/pata_pdc2027x.c
@@ -122,7 +122,7 @@ static struct pci_driver pdc2027x_pci_driver = {
 #endif
 };
 
-static struct scsi_host_template pdc2027x_sht = {
+static const struct scsi_host_template pdc2027x_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_pdc202xx_old.c b/drivers/ata/pata_pdc202xx_old.c
index f894ff2de0a9..a32723e46357 100644
--- a/drivers/ata/pata_pdc202xx_old.c
+++ b/drivers/ata/pata_pdc202xx_old.c
@@ -289,7 +289,7 @@ static int pdc2026x_check_atapi_dma(struct ata_queued_cmd *qc)
 	return 1;
 }
 
-static struct scsi_host_template pdc202xx_sht = {
+static const struct scsi_host_template pdc202xx_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_piccolo.c b/drivers/ata/pata_piccolo.c
index 389b63b13c70..ced906bf56be 100644
--- a/drivers/ata/pata_piccolo.c
+++ b/drivers/ata/pata_piccolo.c
@@ -62,7 +62,7 @@ static void tosh_set_dmamode(struct ata_port *ap, struct ata_device *adev)
 }
 
 
-static struct scsi_host_template tosh_sht = {
+static const struct scsi_host_template tosh_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_platform.c b/drivers/ata/pata_platform.c
index 21fb059859bd..87479bc893b2 100644
--- a/drivers/ata/pata_platform.c
+++ b/drivers/ata/pata_platform.c
@@ -45,7 +45,7 @@ static int pata_platform_set_mode(struct ata_link *link, struct ata_device **unu
 	return 0;
 }
 
-static struct scsi_host_template pata_platform_sht = {
+static const struct scsi_host_template pata_platform_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
@@ -97,7 +97,7 @@ static void pata_platform_setup_port(struct ata_ioports *ioaddr,
 int __pata_platform_probe(struct device *dev, struct resource *io_res,
 			  struct resource *ctl_res, struct resource *irq_res,
 			  unsigned int ioport_shift, int __pio_mask,
-			  struct scsi_host_template *sht, bool use16bit)
+			  const struct scsi_host_template *sht, bool use16bit)
 {
 	struct ata_host *host;
 	struct ata_port *ap;
diff --git a/drivers/ata/pata_pxa.c b/drivers/ata/pata_pxa.c
index 985f42c4fd70..ea402e02c46e 100644
--- a/drivers/ata/pata_pxa.c
+++ b/drivers/ata/pata_pxa.c
@@ -136,7 +136,7 @@ static int pxa_check_atapi_dma(struct ata_queued_cmd *qc)
 	return -EOPNOTSUPP;
 }
 
-static struct scsi_host_template pxa_ata_sht = {
+static const struct scsi_host_template pxa_ata_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_radisys.c b/drivers/ata/pata_radisys.c
index 3aca8fe3fdb6..84b001097093 100644
--- a/drivers/ata/pata_radisys.c
+++ b/drivers/ata/pata_radisys.c
@@ -183,7 +183,7 @@ static unsigned int radisys_qc_issue(struct ata_queued_cmd *qc)
 }
 
 
-static struct scsi_host_template radisys_sht = {
+static const struct scsi_host_template radisys_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_rb532_cf.c b/drivers/ata/pata_rb532_cf.c
index 2e110aefe59b..3974d294a341 100644
--- a/drivers/ata/pata_rb532_cf.c
+++ b/drivers/ata/pata_rb532_cf.c
@@ -73,7 +73,7 @@ static struct ata_port_operations rb532_pata_port_ops = {
 
 /* ------------------------------------------------------------------------ */
 
-static struct scsi_host_template rb532_pata_sht = {
+static const struct scsi_host_template rb532_pata_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_rdc.c b/drivers/ata/pata_rdc.c
index ecb229c2c1a2..0a9689862f71 100644
--- a/drivers/ata/pata_rdc.c
+++ b/drivers/ata/pata_rdc.c
@@ -288,7 +288,7 @@ static const struct ata_port_info rdc_port_info = {
 	.port_ops	= &rdc_pata_ops,
 };
 
-static struct scsi_host_template rdc_sht = {
+static const struct scsi_host_template rdc_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_rz1000.c b/drivers/ata/pata_rz1000.c
index fb00c3e5fd19..8e2606793091 100644
--- a/drivers/ata/pata_rz1000.c
+++ b/drivers/ata/pata_rz1000.c
@@ -50,7 +50,7 @@ static int rz1000_set_mode(struct ata_link *link, struct ata_device **unused)
 }
 
 
-static struct scsi_host_template rz1000_sht = {
+static const struct scsi_host_template rz1000_sht = {
 	ATA_PIO_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_sc1200.c b/drivers/ata/pata_sc1200.c
index f28daf62a37d..a388dfb97ad8 100644
--- a/drivers/ata/pata_sc1200.c
+++ b/drivers/ata/pata_sc1200.c
@@ -192,7 +192,7 @@ static int sc1200_qc_defer(struct ata_queued_cmd *qc)
 	return 0;
 }
 
-static struct scsi_host_template sc1200_sht = {
+static const struct scsi_host_template sc1200_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize	= LIBATA_DUMB_MAX_PRD,
 	.dma_boundary	= ATA_DMA_BOUNDARY,
diff --git a/drivers/ata/pata_sch.c b/drivers/ata/pata_sch.c
index 4f9c2aefd807..8356f1f2a025 100644
--- a/drivers/ata/pata_sch.c
+++ b/drivers/ata/pata_sch.c
@@ -57,7 +57,7 @@ static struct pci_driver sch_pci_driver = {
 #endif
 };
 
-static struct scsi_host_template sch_sht = {
+static const struct scsi_host_template sch_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_serverworks.c b/drivers/ata/pata_serverworks.c
index c0bc4af0d196..549ff24a9823 100644
--- a/drivers/ata/pata_serverworks.c
+++ b/drivers/ata/pata_serverworks.c
@@ -252,13 +252,13 @@ static void serverworks_set_dmamode(struct ata_port *ap, struct ata_device *adev
 	pci_write_config_byte(pdev, 0x54, ultra_cfg);
 }
 
-static struct scsi_host_template serverworks_osb4_sht = {
+static const struct scsi_host_template serverworks_osb4_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize	= LIBATA_DUMB_MAX_PRD,
 	.dma_boundary	= ATA_DMA_BOUNDARY,
 };
 
-static struct scsi_host_template serverworks_csb_sht = {
+static const struct scsi_host_template serverworks_csb_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
@@ -413,7 +413,7 @@ static int serverworks_init_one(struct pci_dev *pdev, const struct pci_device_id
 		}
 	};
 	const struct ata_port_info *ppi[] = { &info[id->driver_data], NULL };
-	struct scsi_host_template *sht = &serverworks_csb_sht;
+	const struct scsi_host_template *sht = &serverworks_csb_sht;
 	int rc;
 
 	rc = pcim_enable_device(pdev);
diff --git a/drivers/ata/pata_sil680.c b/drivers/ata/pata_sil680.c
index 67ef2e26d7df..abe64b5f83cf 100644
--- a/drivers/ata/pata_sil680.c
+++ b/drivers/ata/pata_sil680.c
@@ -223,7 +223,7 @@ static bool sil680_sff_irq_check(struct ata_port *ap)
 	return val & 0x08;
 }
 
-static struct scsi_host_template sil680_sht = {
+static const struct scsi_host_template sil680_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_sis.c b/drivers/ata/pata_sis.c
index 92e4cf05de2c..31de06b66221 100644
--- a/drivers/ata/pata_sis.c
+++ b/drivers/ata/pata_sis.c
@@ -539,7 +539,7 @@ static unsigned int sis_133_mode_filter(struct ata_device *adev, unsigned int ma
 	return mask;
 }
 
-static struct scsi_host_template sis_sht = {
+static const struct scsi_host_template sis_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_sl82c105.c b/drivers/ata/pata_sl82c105.c
index 8487470e2e01..3b62ea482f1a 100644
--- a/drivers/ata/pata_sl82c105.c
+++ b/drivers/ata/pata_sl82c105.c
@@ -238,7 +238,7 @@ static bool sl82c105_sff_irq_check(struct ata_port *ap)
 	return val & mask;
 }
 
-static struct scsi_host_template sl82c105_sht = {
+static const struct scsi_host_template sl82c105_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_triflex.c b/drivers/ata/pata_triflex.c
index 782162d2f3f8..26d448a869e2 100644
--- a/drivers/ata/pata_triflex.c
+++ b/drivers/ata/pata_triflex.c
@@ -160,7 +160,7 @@ static void triflex_bmdma_stop(struct ata_queued_cmd *qc)
 	triflex_load_timing(qc->ap, qc->dev, qc->dev->pio_mode);
 }
 
-static struct scsi_host_template triflex_sht = {
+static const struct scsi_host_template triflex_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pata_via.c b/drivers/ata/pata_via.c
index 34f00f389932..696b99720dcb 100644
--- a/drivers/ata/pata_via.c
+++ b/drivers/ata/pata_via.c
@@ -443,7 +443,7 @@ static int via_port_start(struct ata_port *ap)
 	return 0;
 }
 
-static struct scsi_host_template via_sht = {
+static const struct scsi_host_template via_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/pdc_adma.c b/drivers/ata/pdc_adma.c
index 35b823ac20c9..8e6b2599f0d5 100644
--- a/drivers/ata/pdc_adma.c
+++ b/drivers/ata/pdc_adma.c
@@ -123,7 +123,7 @@ static void adma_freeze(struct ata_port *ap);
 static void adma_thaw(struct ata_port *ap);
 static int adma_prereset(struct ata_link *link, unsigned long deadline);
 
-static struct scsi_host_template adma_ata_sht = {
+static const struct scsi_host_template adma_ata_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= LIBATA_MAX_PRD,
 	.dma_boundary		= ADMA_DMA_BOUNDARY,
diff --git a/drivers/ata/sata_dwc_460ex.c b/drivers/ata/sata_dwc_460ex.c
index 21d77633a98f..fabdd1e380f9 100644
--- a/drivers/ata/sata_dwc_460ex.c
+++ b/drivers/ata/sata_dwc_460ex.c
@@ -810,7 +810,7 @@ static int sata_dwc_dma_get_channel(struct sata_dwc_device_port *hsdevp)
 	struct device *dev = hsdev->dev;
 
 #ifdef CONFIG_SATA_DWC_OLD_DMA
-	if (!of_find_property(dev->of_node, "dmas", NULL))
+	if (!of_property_present(dev->of_node, "dmas"))
 		return sata_dwc_dma_get_channel_old(hsdevp);
 #endif
 
@@ -1076,7 +1076,7 @@ static void sata_dwc_dev_select(struct ata_port *ap, unsigned int device)
 /*
  * scsi mid-layer and libata interface structures
  */
-static struct scsi_host_template sata_dwc_sht = {
+static const struct scsi_host_template sata_dwc_sht = {
 	ATA_NCQ_SHT(DRV_NAME),
 	/*
 	 * test-only: Currently this driver doesn't handle NCQ
@@ -1180,7 +1180,7 @@ static int sata_dwc_probe(struct platform_device *ofdev)
 	}
 
 #ifdef CONFIG_SATA_DWC_OLD_DMA
-	if (!of_find_property(np, "dmas", NULL)) {
+	if (!of_property_present(np, "dmas")) {
 		err = sata_dwc_dma_init_old(ofdev, hsdev);
 		if (err)
 			return err;
diff --git a/drivers/ata/sata_fsl.c b/drivers/ata/sata_fsl.c
index b052c5a65c17..ccd99b9aa9ff 100644
--- a/drivers/ata/sata_fsl.c
+++ b/drivers/ata/sata_fsl.c
@@ -1376,7 +1376,7 @@ static void sata_fsl_host_stop(struct ata_host *host)
 /*
  * scsi mid-layer and libata interface structures
  */
-static struct scsi_host_template sata_fsl_sht = {
+static const struct scsi_host_template sata_fsl_sht = {
 	ATA_NCQ_SHT_QD("sata_fsl", SATA_FSL_QUEUE_DEPTH),
 	.sg_tablesize = SATA_FSL_MAX_PRD_USABLE,
 	.dma_boundary = ATA_DMA_BOUNDARY,
diff --git a/drivers/ata/sata_highbank.c b/drivers/ata/sata_highbank.c
index dfbf9493e451..8237ece4a46f 100644
--- a/drivers/ata/sata_highbank.c
+++ b/drivers/ata/sata_highbank.c
@@ -438,7 +438,7 @@ static const struct ata_port_info ahci_highbank_port_info = {
 	.port_ops       = &ahci_highbank_ops,
 };
 
-static struct scsi_host_template ahci_highbank_platform_sht = {
+static const struct scsi_host_template ahci_highbank_platform_sht = {
 	AHCI_SHT("sata_highbank"),
 };
 
diff --git a/drivers/ata/sata_inic162x.c b/drivers/ata/sata_inic162x.c
index 2833c722118d..2c8c78ed86c1 100644
--- a/drivers/ata/sata_inic162x.c
+++ b/drivers/ata/sata_inic162x.c
@@ -242,7 +242,7 @@ struct inic_port_priv {
 	dma_addr_t	cpb_tbl_dma;
 };
 
-static struct scsi_host_template inic_sht = {
+static const struct scsi_host_template inic_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= LIBATA_MAX_PRD, /* maybe it can be larger? */
 
diff --git a/drivers/ata/sata_mv.c b/drivers/ata/sata_mv.c
index e3cff01201b8..d404e631d152 100644
--- a/drivers/ata/sata_mv.c
+++ b/drivers/ata/sata_mv.c
@@ -659,13 +659,13 @@ static u8 mv_sff_check_status(struct ata_port *ap);
  * PRDs for 64K boundaries in mv_fill_sg().
  */
 #ifdef CONFIG_PCI
-static struct scsi_host_template mv5_sht = {
+static const struct scsi_host_template mv5_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= MV_MAX_SG_CT / 2,
 	.dma_boundary		= MV_DMA_BOUNDARY,
 };
 #endif
-static struct scsi_host_template mv6_sht = {
+static const struct scsi_host_template mv6_sht = {
 	__ATA_BASE_SHT(DRV_NAME),
 	.can_queue		= MV_MAX_Q_DEPTH - 1,
 	.sg_tablesize		= MV_MAX_SG_CT / 2,
diff --git a/drivers/ata/sata_nv.c b/drivers/ata/sata_nv.c
index 9b2d289e89e1..abf5651c87ab 100644
--- a/drivers/ata/sata_nv.c
+++ b/drivers/ata/sata_nv.c
@@ -371,11 +371,11 @@ static struct pci_driver nv_pci_driver = {
 	.remove			= ata_pci_remove_one,
 };
 
-static struct scsi_host_template nv_sht = {
+static const struct scsi_host_template nv_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
-static struct scsi_host_template nv_adma_sht = {
+static const struct scsi_host_template nv_adma_sht = {
 	__ATA_BASE_SHT(DRV_NAME),
 	.can_queue		= NV_ADMA_MAX_CPBS,
 	.sg_tablesize		= NV_ADMA_SGTBL_TOTAL_LEN,
@@ -386,7 +386,7 @@ static struct scsi_host_template nv_adma_sht = {
 	.tag_alloc_policy	= BLK_TAG_ALLOC_RR,
 };
 
-static struct scsi_host_template nv_swncq_sht = {
+static const struct scsi_host_template nv_swncq_sht = {
 	__ATA_BASE_SHT(DRV_NAME),
 	.can_queue		= ATA_MAX_QUEUE - 1,
 	.sg_tablesize		= LIBATA_MAX_PRD,
@@ -520,7 +520,7 @@ static struct ata_port_operations nv_swncq_ops = {
 
 struct nv_pi_priv {
 	irq_handler_t			irq_handler;
-	struct scsi_host_template	*sht;
+	const struct scsi_host_template	*sht;
 };
 
 #define NV_PI_PRIV(_irq_handler, _sht) \
diff --git a/drivers/ata/sata_promise.c b/drivers/ata/sata_promise.c
index 4e60e6c4c35a..2df1a070b25a 100644
--- a/drivers/ata/sata_promise.c
+++ b/drivers/ata/sata_promise.c
@@ -158,7 +158,7 @@ static void pdc_error_handler(struct ata_port *ap);
 static void pdc_post_internal_cmd(struct ata_queued_cmd *qc);
 static int pdc_pata_cable_detect(struct ata_port *ap);
 
-static struct scsi_host_template pdc_ata_sht = {
+static const struct scsi_host_template pdc_ata_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= PDC_MAX_PRD,
 	.dma_boundary		= ATA_DMA_BOUNDARY,
diff --git a/drivers/ata/sata_qstor.c b/drivers/ata/sata_qstor.c
index 8ca0810aad26..8a6286159044 100644
--- a/drivers/ata/sata_qstor.c
+++ b/drivers/ata/sata_qstor.c
@@ -108,7 +108,7 @@ static void qs_thaw(struct ata_port *ap);
 static int qs_prereset(struct ata_link *link, unsigned long deadline);
 static void qs_error_handler(struct ata_port *ap);
 
-static struct scsi_host_template qs_ata_sht = {
+static const struct scsi_host_template qs_ata_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= QS_MAX_PRD,
 	.dma_boundary		= QS_DMA_BOUNDARY,
diff --git a/drivers/ata/sata_rcar.c b/drivers/ata/sata_rcar.c
index 0195eb29f6c2..34790f15c1b8 100644
--- a/drivers/ata/sata_rcar.c
+++ b/drivers/ata/sata_rcar.c
@@ -608,7 +608,7 @@ static u8 sata_rcar_bmdma_status(struct ata_port *ap)
 	return host_stat;
 }
 
-static struct scsi_host_template sata_rcar_sht = {
+static const struct scsi_host_template sata_rcar_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	/*
 	 * This controller allows transfer chunks up to 512MB which cross 64KB
diff --git a/drivers/ata/sata_sil.c b/drivers/ata/sata_sil.c
index 3b989a52879d..cc77c0248284 100644
--- a/drivers/ata/sata_sil.c
+++ b/drivers/ata/sata_sil.c
@@ -156,7 +156,7 @@ static struct pci_driver sil_pci_driver = {
 #endif
 };
 
-static struct scsi_host_template sil_sht = {
+static const struct scsi_host_template sil_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	/** These controllers support Large Block Transfer which allows
 	    transfer chunks up to 2GB and which cross 64KB boundaries,
diff --git a/drivers/ata/sata_sil24.c b/drivers/ata/sata_sil24.c
index 22cc9e9789dd..e72a0257990d 100644
--- a/drivers/ata/sata_sil24.c
+++ b/drivers/ata/sata_sil24.c
@@ -373,7 +373,7 @@ static struct pci_driver sil24_pci_driver = {
 #endif
 };
 
-static struct scsi_host_template sil24_sht = {
+static const struct scsi_host_template sil24_sht = {
 	__ATA_BASE_SHT(DRV_NAME),
 	.can_queue		= SIL24_MAX_CMDS,
 	.sg_tablesize		= SIL24_MAX_SGE,
diff --git a/drivers/ata/sata_sis.c b/drivers/ata/sata_sis.c
index 316237362aa9..ef8724986de3 100644
--- a/drivers/ata/sata_sis.c
+++ b/drivers/ata/sata_sis.c
@@ -72,7 +72,7 @@ static struct pci_driver sis_pci_driver = {
 #endif
 };
 
-static struct scsi_host_template sis_sht = {
+static const struct scsi_host_template sis_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/sata_svw.c b/drivers/ata/sata_svw.c
index 2e3418a82b44..c47c3fb434d5 100644
--- a/drivers/ata/sata_svw.c
+++ b/drivers/ata/sata_svw.c
@@ -330,7 +330,7 @@ static int k2_sata_show_info(struct seq_file *m, struct Scsi_Host *shost)
 	return 0;
 }
 
-static struct scsi_host_template k2_sata_sht = {
+static const struct scsi_host_template k2_sata_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 	.show_info		= k2_sata_show_info,
 };
diff --git a/drivers/ata/sata_sx4.c b/drivers/ata/sata_sx4.c
index a92c60455b1d..ccc016072637 100644
--- a/drivers/ata/sata_sx4.c
+++ b/drivers/ata/sata_sx4.c
@@ -226,7 +226,7 @@ static void pdc_post_internal_cmd(struct ata_queued_cmd *qc);
 static int pdc_check_atapi_dma(struct ata_queued_cmd *qc);
 
 
-static struct scsi_host_template pdc_sata_sht = {
+static const struct scsi_host_template pdc_sata_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= LIBATA_MAX_PRD,
 	.dma_boundary		= ATA_DMA_BOUNDARY,
diff --git a/drivers/ata/sata_uli.c b/drivers/ata/sata_uli.c
index 815e6af75310..60ea45926cd1 100644
--- a/drivers/ata/sata_uli.c
+++ b/drivers/ata/sata_uli.c
@@ -59,7 +59,7 @@ static struct pci_driver uli_pci_driver = {
 	.remove			= ata_pci_remove_one,
 };
 
-static struct scsi_host_template uli_sht = {
+static const struct scsi_host_template uli_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/sata_via.c b/drivers/ata/sata_via.c
index c7891cc84ea0..57cbf2cef618 100644
--- a/drivers/ata/sata_via.c
+++ b/drivers/ata/sata_via.c
@@ -107,7 +107,7 @@ static struct pci_driver svia_pci_driver = {
 	.remove			= ata_pci_remove_one,
 };
 
-static struct scsi_host_template svia_sht = {
+static const struct scsi_host_template svia_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/ata/sata_vsc.c b/drivers/ata/sata_vsc.c
index 87e4ed66b306..d39b87537168 100644
--- a/drivers/ata/sata_vsc.c
+++ b/drivers/ata/sata_vsc.c
@@ -277,7 +277,7 @@ out:
 }
 
 
-static struct scsi_host_template vsc_sata_sht = {
+static const struct scsi_host_template vsc_sata_sht = {
 	ATA_BMDMA_SHT(DRV_NAME),
 };
 
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
index 6f04b831a5c0..2b8fd6bb7da0 100644
--- a/drivers/base/Kconfig
+++ b/drivers/base/Kconfig
@@ -230,4 +230,16 @@ config GENERIC_ARCH_NUMA
 	  Enable support for generic NUMA implementation. Currently, RISC-V
 	  and ARM64 use it.
 
+config FW_DEVLINK_SYNC_STATE_TIMEOUT
+	bool "sync_state() behavior defaults to timeout instead of strict"
+	help
+	  This is build time equivalent of adding kernel command line parameter
+	  "fw_devlink.sync_state=timeout". Give up waiting on consumers and
+	  call sync_state() on any devices that haven't yet received their
+	  sync_state() calls after deferred_probe_timeout has expired or by
+	  late_initcall() if !CONFIG_MODULES. You should almost always want to
+	  select N here unless you have already successfully tested with the
+	  command line option on every system/board your kernel is expected to
+	  work on.
+
 endmenu
diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
index b1c1dd38ab01..b741b5ba82bd 100644
--- a/drivers/base/arch_topology.c
+++ b/drivers/base/arch_topology.c
@@ -835,18 +835,19 @@ void __init init_cpu_topology(void)
 	if (ret) {
 		/*
 		 * Discard anything that was parsed if we hit an error so we
-		 * don't use partial information.
+		 * don't use partial information. But do not return yet to give
+		 * arch-specific early cache level detection a chance to run.
 		 */
 		reset_cpu_topology();
-		return;
 	}
 
 	for_each_possible_cpu(cpu) {
 		ret = fetch_cache_info(cpu);
-		if (ret) {
+		if (!ret)
+			continue;
+		else if (ret != -ENOENT)
 			pr_err("Early cacheinfo failed, ret = %d\n", ret);
-			break;
-		}
+		return;
 	}
 }
 
diff --git a/drivers/base/base.h b/drivers/base/base.h
index 726a12a244c0..eb4c0ace9242 100644
--- a/drivers/base/base.h
+++ b/drivers/base/base.h
@@ -27,11 +27,13 @@
  *                 on this bus.
  * @bus - pointer back to the struct bus_type that this structure is associated
  *        with.
+ * @dev_root: Default device to use as the parent.
  *
  * @glue_dirs - "glue" directory to put in-between the parent device to
  *              avoid namespace conflicts
  * @class - pointer back to the struct class that this structure is associated
  *          with.
+ * @lock_key:	Lock class key for use by the lock validator
  *
  * This structure is the one that is the actual kobject allowing struct
  * bus_type/class to be statically allocated safely.  Nothing outside of the
@@ -48,10 +50,11 @@ struct subsys_private {
 	struct klist klist_drivers;
 	struct blocking_notifier_head bus_notifier;
 	unsigned int drivers_autoprobe:1;
-	struct bus_type *bus;
+	const struct bus_type *bus;
+	struct device *dev_root;
 
 	struct kset glue_dirs;
-	struct class *class;
+	const struct class *class;
 
 	struct lock_class_key lock_key;
 };
@@ -70,6 +73,8 @@ static inline void subsys_put(struct subsys_private *sp)
 		kset_put(&sp->subsys);
 }
 
+struct subsys_private *class_to_subsys(const struct class *class);
+
 struct driver_private {
 	struct kobject kobj;
 	struct klist klist_devices;
@@ -122,69 +127,73 @@ struct device_private {
 	container_of(obj, struct device_private, knode_class)
 
 /* initialisation functions */
-extern int devices_init(void);
-extern int buses_init(void);
-extern int classes_init(void);
-extern int firmware_init(void);
+int devices_init(void);
+int buses_init(void);
+int classes_init(void);
+int firmware_init(void);
 #ifdef CONFIG_SYS_HYPERVISOR
-extern int hypervisor_init(void);
+int hypervisor_init(void);
 #else
 static inline int hypervisor_init(void) { return 0; }
 #endif
-extern int platform_bus_init(void);
-extern void cpu_dev_init(void);
-extern void container_dev_init(void);
+int platform_bus_init(void);
+void cpu_dev_init(void);
+void container_dev_init(void);
 #ifdef CONFIG_AUXILIARY_BUS
-extern void auxiliary_bus_init(void);
+void auxiliary_bus_init(void);
 #else
 static inline void auxiliary_bus_init(void) { }
 #endif
 
 struct kobject *virtual_device_parent(struct device *dev);
 
-extern int bus_add_device(struct device *dev);
-extern void bus_probe_device(struct device *dev);
-extern void bus_remove_device(struct device *dev);
+int bus_add_device(struct device *dev);
+void bus_probe_device(struct device *dev);
+void bus_remove_device(struct device *dev);
 void bus_notify(struct device *dev, enum bus_notifier_event value);
 bool bus_is_registered(const struct bus_type *bus);
 
-extern int bus_add_driver(struct device_driver *drv);
-extern void bus_remove_driver(struct device_driver *drv);
-extern void device_release_driver_internal(struct device *dev,
-					   struct device_driver *drv,
-					   struct device *parent);
+int bus_add_driver(struct device_driver *drv);
+void bus_remove_driver(struct device_driver *drv);
+void device_release_driver_internal(struct device *dev, struct device_driver *drv,
+				    struct device *parent);
 
-extern void driver_detach(struct device_driver *drv);
-extern void driver_deferred_probe_del(struct device *dev);
-extern void device_set_deferred_probe_reason(const struct device *dev,
-					     struct va_format *vaf);
+void driver_detach(struct device_driver *drv);
+void driver_deferred_probe_del(struct device *dev);
+void device_set_deferred_probe_reason(const struct device *dev, struct va_format *vaf);
 static inline int driver_match_device(struct device_driver *drv,
 				      struct device *dev)
 {
 	return drv->bus->match ? drv->bus->match(dev, drv) : 1;
 }
 
-extern int driver_add_groups(struct device_driver *drv,
-			     const struct attribute_group **groups);
-extern void driver_remove_groups(struct device_driver *drv,
-				 const struct attribute_group **groups);
+static inline void dev_sync_state(struct device *dev)
+{
+	if (dev->bus->sync_state)
+		dev->bus->sync_state(dev);
+	else if (dev->driver && dev->driver->sync_state)
+		dev->driver->sync_state(dev);
+}
+
+int driver_add_groups(struct device_driver *drv, const struct attribute_group **groups);
+void driver_remove_groups(struct device_driver *drv, const struct attribute_group **groups);
 void device_driver_detach(struct device *dev);
 
-extern int devres_release_all(struct device *dev);
-extern void device_block_probing(void);
-extern void device_unblock_probing(void);
-extern void deferred_probe_extend_timeout(void);
-extern void driver_deferred_probe_trigger(void);
+int devres_release_all(struct device *dev);
+void device_block_probing(void);
+void device_unblock_probing(void);
+void deferred_probe_extend_timeout(void);
+void driver_deferred_probe_trigger(void);
 const char *device_get_devnode(const struct device *dev, umode_t *mode,
 			       kuid_t *uid, kgid_t *gid, const char **tmp);
 
 /* /sys/devices directory */
 extern struct kset *devices_kset;
-extern void devices_kset_move_last(struct device *dev);
+void devices_kset_move_last(struct device *dev);
 
 #if defined(CONFIG_MODULES) && defined(CONFIG_SYSFS)
-extern void module_add_driver(struct module *mod, struct device_driver *drv);
-extern void module_remove_driver(struct device_driver *drv);
+void module_add_driver(struct module *mod, struct device_driver *drv);
+void module_remove_driver(struct device_driver *drv);
 #else
 static inline void module_add_driver(struct module *mod,
 				     struct device_driver *drv) { }
@@ -192,23 +201,34 @@ static inline void module_remove_driver(struct device_driver *drv) { }
 #endif
 
 #ifdef CONFIG_DEVTMPFS
-extern int devtmpfs_init(void);
+int devtmpfs_init(void);
 #else
 static inline int devtmpfs_init(void) { return 0; }
 #endif
 
+#ifdef CONFIG_BLOCK
+extern struct class block_class;
+static inline bool is_blockdev(struct device *dev)
+{
+	return dev->class == &block_class;
+}
+#else
+static inline bool is_blockdev(struct device *dev) { return false; }
+#endif
+
 /* Device links support */
-extern int device_links_read_lock(void);
-extern void device_links_read_unlock(int idx);
-extern int device_links_read_lock_held(void);
-extern int device_links_check_suppliers(struct device *dev);
-extern void device_links_force_bind(struct device *dev);
-extern void device_links_driver_bound(struct device *dev);
-extern void device_links_driver_cleanup(struct device *dev);
-extern void device_links_no_driver(struct device *dev);
-extern bool device_links_busy(struct device *dev);
-extern void device_links_unbind_consumers(struct device *dev);
-extern void fw_devlink_drivers_done(void);
+int device_links_read_lock(void);
+void device_links_read_unlock(int idx);
+int device_links_read_lock_held(void);
+int device_links_check_suppliers(struct device *dev);
+void device_links_force_bind(struct device *dev);
+void device_links_driver_bound(struct device *dev);
+void device_links_driver_cleanup(struct device *dev);
+void device_links_no_driver(struct device *dev);
+bool device_links_busy(struct device *dev);
+void device_links_unbind_consumers(struct device *dev);
+void fw_devlink_drivers_done(void);
+void fw_devlink_probing_done(void);
 
 /* device pm support */
 void device_pm_move_to_tail(struct device *dev);
diff --git a/drivers/base/bus.c b/drivers/base/bus.c
index dd4b82d7510f..84a21084d67d 100644
--- a/drivers/base/bus.c
+++ b/drivers/base/bus.c
@@ -84,7 +84,7 @@ done:
 	return sp;
 }
 
-static struct bus_type *bus_get(struct bus_type *bus)
+static const struct bus_type *bus_get(const struct bus_type *bus)
 {
 	struct subsys_private *sp = bus_to_subsys(bus);
 
@@ -233,7 +233,7 @@ static const struct kset_uevent_ops bus_uevent_ops = {
 static ssize_t unbind_store(struct device_driver *drv, const char *buf,
 			    size_t count)
 {
-	struct bus_type *bus = bus_get(drv->bus);
+	const struct bus_type *bus = bus_get(drv->bus);
 	struct device *dev;
 	int err = -ENODEV;
 
@@ -256,7 +256,7 @@ static DRIVER_ATTR_IGNORE_LOCKDEP(unbind, 0200, NULL, unbind_store);
 static ssize_t bind_store(struct device_driver *drv, const char *buf,
 			  size_t count)
 {
-	struct bus_type *bus = bus_get(drv->bus);
+	const struct bus_type *bus = bus_get(drv->bus);
 	struct device *dev;
 	int err = -ENODEV;
 
@@ -274,7 +274,7 @@ static ssize_t bind_store(struct device_driver *drv, const char *buf,
 }
 static DRIVER_ATTR_IGNORE_LOCKDEP(bind, 0200, NULL, bind_store);
 
-static ssize_t drivers_autoprobe_show(struct bus_type *bus, char *buf)
+static ssize_t drivers_autoprobe_show(const struct bus_type *bus, char *buf)
 {
 	struct subsys_private *sp = bus_to_subsys(bus);
 	int ret;
@@ -287,7 +287,7 @@ static ssize_t drivers_autoprobe_show(struct bus_type *bus, char *buf)
 	return ret;
 }
 
-static ssize_t drivers_autoprobe_store(struct bus_type *bus,
+static ssize_t drivers_autoprobe_store(const struct bus_type *bus,
 				       const char *buf, size_t count)
 {
 	struct subsys_private *sp = bus_to_subsys(bus);
@@ -304,7 +304,7 @@ static ssize_t drivers_autoprobe_store(struct bus_type *bus,
 	return count;
 }
 
-static ssize_t drivers_probe_store(struct bus_type *bus,
+static ssize_t drivers_probe_store(const struct bus_type *bus,
 				   const char *buf, size_t count)
 {
 	struct device *dev;
@@ -769,7 +769,7 @@ static int __must_check bus_rescan_devices_helper(struct device *dev,
  * attached and rescan it against existing drivers to see if it matches
  * any by calling device_attach() for the unbound devices.
  */
-int bus_rescan_devices(struct bus_type *bus)
+int bus_rescan_devices(const struct bus_type *bus)
 {
 	return bus_for_each_dev(bus, NULL, NULL, bus_rescan_devices_helper);
 }
@@ -808,7 +808,7 @@ static void klist_devices_put(struct klist_node *n)
 	put_device(dev);
 }
 
-static ssize_t bus_uevent_store(struct bus_type *bus,
+static ssize_t bus_uevent_store(const struct bus_type *bus,
 				const char *buf, size_t count)
 {
 	struct subsys_private *sp = bus_to_subsys(bus);
@@ -841,7 +841,7 @@ static struct bus_attribute bus_attr_uevent = __ATTR(uevent, 0200, NULL,
  * infrastructure, then register the children subsystems it has:
  * the devices and drivers that belong to the subsystem.
  */
-int bus_register(struct bus_type *bus)
+int bus_register(const struct bus_type *bus)
 {
 	int retval;
 	struct subsys_private *priv;
@@ -935,8 +935,8 @@ void bus_unregister(const struct bus_type *bus)
 		return;
 
 	pr_debug("bus: '%s': unregistering\n", bus->name);
-	if (bus->dev_root)
-		device_unregister(bus->dev_root);
+	if (sp->dev_root)
+		device_unregister(sp->dev_root);
 
 	bus_kobj = &sp->subsys.kobj;
 	sysfs_remove_groups(bus_kobj, bus->bus_groups);
@@ -1198,6 +1198,7 @@ static int subsys_register(struct bus_type *subsys,
 			   const struct attribute_group **groups,
 			   struct kobject *parent_of_root)
 {
+	struct subsys_private *sp;
 	struct device *dev;
 	int err;
 
@@ -1205,6 +1206,12 @@ static int subsys_register(struct bus_type *subsys,
 	if (err < 0)
 		return err;
 
+	sp = bus_to_subsys(subsys);
+	if (!sp) {
+		err = -EINVAL;
+		goto err_sp;
+	}
+
 	dev = kzalloc(sizeof(struct device), GFP_KERNEL);
 	if (!dev) {
 		err = -ENOMEM;
@@ -1223,7 +1230,8 @@ static int subsys_register(struct bus_type *subsys,
 	if (err < 0)
 		goto err_dev_reg;
 
-	subsys->dev_root = dev;
+	sp->dev_root = dev;
+	subsys_put(sp);
 	return 0;
 
 err_dev_reg:
@@ -1232,6 +1240,8 @@ err_dev_reg:
 err_name:
 	kfree(dev);
 err_dev:
+	subsys_put(sp);
+err_sp:
 	bus_unregister(subsys);
 	return err;
 }
@@ -1297,7 +1307,7 @@ EXPORT_SYMBOL_GPL(subsys_virtual_register);
  * from being unregistered or unloaded while the caller is using it.
  * The caller is responsible for preventing this.
  */
-struct device_driver *driver_find(const char *name, struct bus_type *bus)
+struct device_driver *driver_find(const char *name, const struct bus_type *bus)
 {
 	struct subsys_private *sp = bus_to_subsys(bus);
 	struct kobject *k;
@@ -1349,9 +1359,15 @@ bool bus_is_registered(const struct bus_type *bus)
  */
 struct device *bus_get_dev_root(const struct bus_type *bus)
 {
-	if (bus)
-		return get_device(bus->dev_root);
-	return NULL;
+	struct subsys_private *sp = bus_to_subsys(bus);
+	struct device *dev_root;
+
+	if (!sp)
+		return NULL;
+
+	dev_root = get_device(sp->dev_root);
+	subsys_put(sp);
+	return dev_root;
 }
 EXPORT_SYMBOL_GPL(bus_get_dev_root);
 
diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index f3903d002819..bba3482ddeb8 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -14,7 +14,7 @@
 #include <linux/cpu.h>
 #include <linux/device.h>
 #include <linux/init.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/smp.h>
@@ -28,6 +28,9 @@ static DEFINE_PER_CPU(struct cpu_cacheinfo, ci_cpu_cacheinfo);
 #define per_cpu_cacheinfo_idx(cpu, idx)		\
 				(per_cpu_cacheinfo(cpu) + (idx))
 
+/* Set if no cache information is found in DT/ACPI. */
+static bool use_arch_info;
+
 struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu)
 {
 	return ci_cacheinfo(cpu);
@@ -38,11 +41,11 @@ static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
 {
 	/*
 	 * For non DT/ACPI systems, assume unique level 1 caches,
-	 * system-wide shared caches for all other levels. This will be used
-	 * only if arch specific code has not populated shared_cpu_map
+	 * system-wide shared caches for all other levels.
 	 */
-	if (!(IS_ENABLED(CONFIG_OF) || IS_ENABLED(CONFIG_ACPI)))
-		return !(this_leaf->level == 1);
+	if (!(IS_ENABLED(CONFIG_OF) || IS_ENABLED(CONFIG_ACPI)) ||
+	    use_arch_info)
+		return (this_leaf->level != 1) && (sib_leaf->level != 1);
 
 	if ((sib_leaf->attributes & CACHE_ID) &&
 	    (this_leaf->attributes & CACHE_ID))
@@ -79,6 +82,9 @@ bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y)
 }
 
 #ifdef CONFIG_OF
+
+static bool of_check_cache_nodes(struct device_node *np);
+
 /* OF properties to query for a given cache type */
 struct cache_type_info {
 	const char *size_prop;
@@ -206,6 +212,11 @@ static int cache_setup_of_node(unsigned int cpu)
 		return -ENOENT;
 	}
 
+	if (!of_check_cache_nodes(np)) {
+		of_node_put(np);
+		return -ENOENT;
+	}
+
 	prev = np;
 
 	while (index < cache_leaves(cpu)) {
@@ -230,6 +241,25 @@ static int cache_setup_of_node(unsigned int cpu)
 	return 0;
 }
 
+static bool of_check_cache_nodes(struct device_node *np)
+{
+	struct device_node *next;
+
+	if (of_property_present(np, "cache-size")   ||
+	    of_property_present(np, "i-cache-size") ||
+	    of_property_present(np, "d-cache-size") ||
+	    of_property_present(np, "cache-unified"))
+		return true;
+
+	next = of_find_next_cache_node(np);
+	if (next) {
+		of_node_put(next);
+		return true;
+	}
+
+	return false;
+}
+
 static int of_count_cache_leaves(struct device_node *np)
 {
 	unsigned int leaves = 0;
@@ -261,6 +291,11 @@ int init_of_cache_level(unsigned int cpu)
 	struct device_node *prev = NULL;
 	unsigned int levels = 0, leaves, level;
 
+	if (!of_check_cache_nodes(np)) {
+		of_node_put(np);
+		return -ENOENT;
+	}
+
 	leaves = of_count_cache_leaves(np);
 	if (leaves > 0)
 		levels = 1;
@@ -312,6 +347,10 @@ static int cache_setup_properties(unsigned int cpu)
 	else if (!acpi_disabled)
 		ret = cache_setup_acpi(cpu);
 
+	// Assume there is no cache information available in DT/ACPI from now.
+	if (ret && use_arch_cache_info())
+		use_arch_info = true;
+
 	return ret;
 }
 
@@ -330,7 +369,7 @@ static int cache_shared_cpu_map_setup(unsigned int cpu)
 	 * to update the shared cpu_map if the cache attributes were
 	 * populated early before all the cpus are brought online
 	 */
-	if (!last_level_cache_is_valid(cpu)) {
+	if (!last_level_cache_is_valid(cpu) && !use_arch_info) {
 		ret = cache_setup_properties(cpu);
 		if (ret)
 			return ret;
@@ -398,6 +437,11 @@ static void free_cache_attributes(unsigned int cpu)
 	cache_shared_cpu_map_remove(cpu);
 }
 
+int __weak early_cache_level(unsigned int cpu)
+{
+	return -ENOENT;
+}
+
 int __weak init_cache_level(unsigned int cpu)
 {
 	return -ENOENT;
@@ -423,56 +467,82 @@ int allocate_cache_info(int cpu)
 
 int fetch_cache_info(unsigned int cpu)
 {
-	struct cpu_cacheinfo *this_cpu_ci;
+	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
 	unsigned int levels = 0, split_levels = 0;
 	int ret;
 
 	if (acpi_disabled) {
 		ret = init_of_cache_level(cpu);
-		if (ret < 0)
-			return ret;
 	} else {
 		ret = acpi_get_cache_info(cpu, &levels, &split_levels);
-		if (ret < 0)
+		if (!ret) {
+			this_cpu_ci->num_levels = levels;
+			/*
+			 * This assumes that:
+			 * - there cannot be any split caches (data/instruction)
+			 *   above a unified cache
+			 * - data/instruction caches come by pair
+			 */
+			this_cpu_ci->num_leaves = levels + split_levels;
+		}
+	}
+
+	if (ret || !cache_leaves(cpu)) {
+		ret = early_cache_level(cpu);
+		if (ret)
 			return ret;
 
-		this_cpu_ci = get_cpu_cacheinfo(cpu);
-		this_cpu_ci->num_levels = levels;
-		/*
-		 * This assumes that:
-		 * - there cannot be any split caches (data/instruction)
-		 *   above a unified cache
-		 * - data/instruction caches come by pair
-		 */
-		this_cpu_ci->num_leaves = levels + split_levels;
+		if (!cache_leaves(cpu))
+			return -ENOENT;
+
+		this_cpu_ci->early_ci_levels = true;
 	}
-	if (!cache_leaves(cpu))
-		return -ENOENT;
 
 	return allocate_cache_info(cpu);
 }
 
-int detect_cache_attributes(unsigned int cpu)
+static inline int init_level_allocate_ci(unsigned int cpu)
 {
-	int ret;
+	unsigned int early_leaves = cache_leaves(cpu);
 
 	/* Since early initialization/allocation of the cacheinfo is allowed
 	 * via fetch_cache_info() and this also gets called as CPU hotplug
 	 * callbacks via cacheinfo_cpu_online, the init/alloc can be skipped
 	 * as it will happen only once (the cacheinfo memory is never freed).
-	 * Just populate the cacheinfo.
+	 * Just populate the cacheinfo. However, if the cacheinfo has been
+	 * allocated early through the arch-specific early_cache_level() call,
+	 * there is a chance the info is wrong (this can happen on arm64). In
+	 * that case, call init_cache_level() anyway to give the arch-specific
+	 * code a chance to make things right.
 	 */
-	if (per_cpu_cacheinfo(cpu))
-		goto populate_leaves;
+	if (per_cpu_cacheinfo(cpu) && !ci_cacheinfo(cpu)->early_ci_levels)
+		return 0;
 
 	if (init_cache_level(cpu) || !cache_leaves(cpu))
 		return -ENOENT;
 
-	ret = allocate_cache_info(cpu);
+	/*
+	 * Now that we have properly initialized the cache level info, make
+	 * sure we don't try to do that again the next time we are called
+	 * (e.g. as CPU hotplug callbacks).
+	 */
+	ci_cacheinfo(cpu)->early_ci_levels = false;
+
+	if (cache_leaves(cpu) <= early_leaves)
+		return 0;
+
+	kfree(per_cpu_cacheinfo(cpu));
+	return allocate_cache_info(cpu);
+}
+
+int detect_cache_attributes(unsigned int cpu)
+{
+	int ret;
+
+	ret = init_level_allocate_ci(cpu);
 	if (ret)
 		return ret;
 
-populate_leaves:
 	/*
 	 * If LLC is valid the cache leaves were already populated so just go to
 	 * update the cpu map.
diff --git a/drivers/base/class.c b/drivers/base/class.c
index 2373b3e210d8..ac1808d1a2e8 100644
--- a/drivers/base/class.c
+++ b/drivers/base/class.c
@@ -20,8 +20,52 @@
 #include <linux/mutex.h>
 #include "base.h"
 
+/* /sys/class */
+static struct kset *class_kset;
+
 #define to_class_attr(_attr) container_of(_attr, struct class_attribute, attr)
 
+/**
+ * class_to_subsys - Turn a struct class into a struct subsys_private
+ *
+ * @class: pointer to the struct bus_type to look up
+ *
+ * The driver core internals need to work on the subsys_private structure, not
+ * the external struct class pointer.  This function walks the list of
+ * registered classes in the system and finds the matching one and returns the
+ * internal struct subsys_private that relates to that class.
+ *
+ * Note, the reference count of the return value is INCREMENTED if it is not
+ * NULL.  A call to subsys_put() must be done when finished with the pointer in
+ * order for it to be properly freed.
+ */
+struct subsys_private *class_to_subsys(const struct class *class)
+{
+	struct subsys_private *sp = NULL;
+	struct kobject *kobj;
+
+	if (!class || !class_kset)
+		return NULL;
+
+	spin_lock(&class_kset->list_lock);
+
+	if (list_empty(&class_kset->list))
+		goto done;
+
+	list_for_each_entry(kobj, &class_kset->list, entry) {
+		struct kset *kset = container_of(kobj, struct kset, kobj);
+
+		sp = container_of_const(kset, struct subsys_private, subsys);
+		if (sp->class == class)
+			goto done;
+	}
+	sp = NULL;
+done:
+	sp = subsys_get(sp);
+	spin_unlock(&class_kset->list_lock);
+	return sp;
+}
+
 static ssize_t class_attr_show(struct kobject *kobj, struct attribute *attr,
 			       char *buf)
 {
@@ -49,25 +93,24 @@ static ssize_t class_attr_store(struct kobject *kobj, struct attribute *attr,
 static void class_release(struct kobject *kobj)
 {
 	struct subsys_private *cp = to_subsys_private(kobj);
-	struct class *class = cp->class;
+	const struct class *class = cp->class;
 
 	pr_debug("class '%s': release.\n", class->name);
 
-	class->p = NULL;
-
 	if (class->class_release)
 		class->class_release(class);
 	else
 		pr_debug("class '%s' does not have a release() function, "
 			 "be careful\n", class->name);
 
+	lockdep_unregister_key(&cp->lock_key);
 	kfree(cp);
 }
 
 static const struct kobj_ns_type_operations *class_child_ns_type(const struct kobject *kobj)
 {
 	const struct subsys_private *cp = to_subsys_private(kobj);
-	struct class *class = cp->class;
+	const struct class *class = cp->class;
 
 	return class->ns_type;
 }
@@ -83,44 +126,34 @@ static const struct kobj_type class_ktype = {
 	.child_ns_type	= class_child_ns_type,
 };
 
-/* Hotplug events for classes go to the class subsys */
-static struct kset *class_kset;
-
-
-int class_create_file_ns(struct class *cls, const struct class_attribute *attr,
+int class_create_file_ns(const struct class *cls, const struct class_attribute *attr,
 			 const void *ns)
 {
+	struct subsys_private *sp = class_to_subsys(cls);
 	int error;
 
-	if (cls)
-		error = sysfs_create_file_ns(&cls->p->subsys.kobj,
-					     &attr->attr, ns);
-	else
-		error = -EINVAL;
+	if (!sp)
+		return -EINVAL;
+
+	error = sysfs_create_file_ns(&sp->subsys.kobj, &attr->attr, ns);
+	subsys_put(sp);
+
 	return error;
 }
 EXPORT_SYMBOL_GPL(class_create_file_ns);
 
-void class_remove_file_ns(struct class *cls, const struct class_attribute *attr,
+void class_remove_file_ns(const struct class *cls, const struct class_attribute *attr,
 			  const void *ns)
 {
-	if (cls)
-		sysfs_remove_file_ns(&cls->p->subsys.kobj, &attr->attr, ns);
-}
-EXPORT_SYMBOL_GPL(class_remove_file_ns);
+	struct subsys_private *sp = class_to_subsys(cls);
 
-static struct class *class_get(struct class *cls)
-{
-	if (cls)
-		kset_get(&cls->p->subsys);
-	return cls;
-}
+	if (!sp)
+		return;
 
-static void class_put(struct class *cls)
-{
-	if (cls)
-		kset_put(&cls->p->subsys);
+	sysfs_remove_file_ns(&sp->subsys.kobj, &attr->attr, ns);
+	subsys_put(sp);
 }
+EXPORT_SYMBOL_GPL(class_remove_file_ns);
 
 static struct device *klist_class_to_dev(struct klist_node *n)
 {
@@ -142,21 +175,10 @@ static void klist_class_dev_put(struct klist_node *n)
 	put_device(dev);
 }
 
-static int class_add_groups(struct class *cls,
-			    const struct attribute_group **groups)
-{
-	return sysfs_create_groups(&cls->p->subsys.kobj, groups);
-}
-
-static void class_remove_groups(struct class *cls,
-				const struct attribute_group **groups)
-{
-	return sysfs_remove_groups(&cls->p->subsys.kobj, groups);
-}
-
-int __class_register(struct class *cls, struct lock_class_key *key)
+int class_register(const struct class *cls)
 {
 	struct subsys_private *cp;
+	struct lock_class_key *key;
 	int error;
 
 	pr_debug("device class '%s': registering\n", cls->name);
@@ -167,6 +189,8 @@ int __class_register(struct class *cls, struct lock_class_key *key)
 	klist_init(&cp->klist_devices, klist_class_dev_get, klist_class_dev_put);
 	INIT_LIST_HEAD(&cp->interfaces);
 	kset_init(&cp->glue_dirs);
+	key = &cp->lock_key;
+	lockdep_register_key(key);
 	__mutex_init(&cp->mutex, "subsys mutex", key);
 	error = kobject_set_name(&cp->subsys.kobj, "%s", cls->name);
 	if (error) {
@@ -174,27 +198,15 @@ int __class_register(struct class *cls, struct lock_class_key *key)
 		return error;
 	}
 
-	/* set the default /sys/dev directory for devices of this class */
-	if (!cls->dev_kobj)
-		cls->dev_kobj = sysfs_dev_char_kobj;
-
-#if defined(CONFIG_BLOCK)
-	/* let the block class directory show up in the root of sysfs */
-	if (!sysfs_deprecated || cls != &block_class)
-		cp->subsys.kobj.kset = class_kset;
-#else
 	cp->subsys.kobj.kset = class_kset;
-#endif
 	cp->subsys.kobj.ktype = &class_ktype;
 	cp->class = cls;
-	cls->p = cp;
 
 	error = kset_register(&cp->subsys);
 	if (error)
 		goto err_out;
 
-	error = class_add_groups(class_get(cls), cls->class_groups);
-	class_put(cls);
+	error = sysfs_create_groups(&cp->subsys.kobj, cls->class_groups);
 	if (error) {
 		kobject_del(&cp->subsys.kobj);
 		kfree_const(cp->subsys.kobj.name);
@@ -204,30 +216,34 @@ int __class_register(struct class *cls, struct lock_class_key *key)
 
 err_out:
 	kfree(cp);
-	cls->p = NULL;
 	return error;
 }
-EXPORT_SYMBOL_GPL(__class_register);
+EXPORT_SYMBOL_GPL(class_register);
 
-void class_unregister(struct class *cls)
+void class_unregister(const struct class *cls)
 {
+	struct subsys_private *sp = class_to_subsys(cls);
+
+	if (!sp)
+		return;
+
 	pr_debug("device class '%s': unregistering\n", cls->name);
-	class_remove_groups(cls, cls->class_groups);
-	kset_unregister(&cls->p->subsys);
+
+	sysfs_remove_groups(&sp->subsys.kobj, cls->class_groups);
+	kset_unregister(&sp->subsys);
+	subsys_put(sp);
 }
 EXPORT_SYMBOL_GPL(class_unregister);
 
-static void class_create_release(struct class *cls)
+static void class_create_release(const struct class *cls)
 {
 	pr_debug("%s called for %s\n", __func__, cls->name);
 	kfree(cls);
 }
 
 /**
- * __class_create - create a struct class structure
- * @owner: pointer to the module that is to "own" this struct class
+ * class_create - create a struct class structure
  * @name: pointer to a string for the name of this class.
- * @key: the lock_class_key for this class; used by mutex lock debugging
  *
  * This is used to create a struct class pointer that can then be used
  * in calls to device_create().
@@ -237,8 +253,7 @@ static void class_create_release(struct class *cls)
  * Note, the pointer created here is to be destroyed when finished by
  * making a call to class_destroy().
  */
-struct class *__class_create(struct module *owner, const char *name,
-			     struct lock_class_key *key)
+struct class *class_create(const char *name)
 {
 	struct class *cls;
 	int retval;
@@ -250,10 +265,9 @@ struct class *__class_create(struct module *owner, const char *name,
 	}
 
 	cls->name = name;
-	cls->owner = owner;
 	cls->class_release = class_create_release;
 
-	retval = __class_register(cls, key);
+	retval = class_register(cls);
 	if (retval)
 		goto error;
 
@@ -263,7 +277,7 @@ error:
 	kfree(cls);
 	return ERR_PTR(retval);
 }
-EXPORT_SYMBOL_GPL(__class_create);
+EXPORT_SYMBOL_GPL(class_create);
 
 /**
  * class_destroy - destroys a struct class structure
@@ -272,7 +286,7 @@ EXPORT_SYMBOL_GPL(__class_create);
  * Note, the pointer to be destroyed must have been created with a call
  * to class_create().
  */
-void class_destroy(struct class *cls)
+void class_destroy(const struct class *cls)
 {
 	if (IS_ERR_OR_NULL(cls))
 		return;
@@ -293,14 +307,18 @@ EXPORT_SYMBOL_GPL(class_destroy);
  * otherwise if it is NULL, the iteration starts at the beginning of
  * the list.
  */
-void class_dev_iter_init(struct class_dev_iter *iter, struct class *class,
-			 struct device *start, const struct device_type *type)
+void class_dev_iter_init(struct class_dev_iter *iter, const struct class *class,
+			 const struct device *start, const struct device_type *type)
 {
+	struct subsys_private *sp = class_to_subsys(class);
 	struct klist_node *start_knode = NULL;
 
+	if (!sp)
+		return;
+
 	if (start)
 		start_knode = &start->p->knode_class;
-	klist_iter_init_node(&class->p->klist_devices, &iter->ki, start_knode);
+	klist_iter_init_node(&sp->klist_devices, &iter->ki, start_knode);
 	iter->type = type;
 }
 EXPORT_SYMBOL_GPL(class_dev_iter_init);
@@ -364,16 +382,17 @@ EXPORT_SYMBOL_GPL(class_dev_iter_exit);
  * @fn is allowed to do anything including calling back into class
  * code.  There's no locking restriction.
  */
-int class_for_each_device(struct class *class, struct device *start,
+int class_for_each_device(const struct class *class, const struct device *start,
 			  void *data, int (*fn)(struct device *, void *))
 {
+	struct subsys_private *sp = class_to_subsys(class);
 	struct class_dev_iter iter;
 	struct device *dev;
 	int error = 0;
 
 	if (!class)
 		return -EINVAL;
-	if (!class->p) {
+	if (!sp) {
 		WARN(1, "%s called for class '%s' before it was initialized",
 		     __func__, class->name);
 		return -EINVAL;
@@ -386,6 +405,7 @@ int class_for_each_device(struct class *class, struct device *start,
 			break;
 	}
 	class_dev_iter_exit(&iter);
+	subsys_put(sp);
 
 	return error;
 }
@@ -411,16 +431,17 @@ EXPORT_SYMBOL_GPL(class_for_each_device);
  * @match is allowed to do anything including calling back into class
  * code.  There's no locking restriction.
  */
-struct device *class_find_device(struct class *class, struct device *start,
+struct device *class_find_device(const struct class *class, const struct device *start,
 				 const void *data,
 				 int (*match)(struct device *, const void *))
 {
+	struct subsys_private *sp = class_to_subsys(class);
 	struct class_dev_iter iter;
 	struct device *dev;
 
 	if (!class)
 		return NULL;
-	if (!class->p) {
+	if (!sp) {
 		WARN(1, "%s called for class '%s' before it was initialized",
 		     __func__, class->name);
 		return NULL;
@@ -434,6 +455,7 @@ struct device *class_find_device(struct class *class, struct device *start,
 		}
 	}
 	class_dev_iter_exit(&iter);
+	subsys_put(sp);
 
 	return dev;
 }
@@ -441,26 +463,33 @@ EXPORT_SYMBOL_GPL(class_find_device);
 
 int class_interface_register(struct class_interface *class_intf)
 {
-	struct class *parent;
+	struct subsys_private *sp;
+	const struct class *parent;
 	struct class_dev_iter iter;
 	struct device *dev;
 
 	if (!class_intf || !class_intf->class)
 		return -ENODEV;
 
-	parent = class_get(class_intf->class);
-	if (!parent)
+	parent = class_intf->class;
+	sp = class_to_subsys(parent);
+	if (!sp)
 		return -EINVAL;
 
-	mutex_lock(&parent->p->mutex);
-	list_add_tail(&class_intf->node, &parent->p->interfaces);
+	/*
+	 * Reference in sp is now incremented and will be dropped when
+	 * the interface is removed in the call to class_interface_unregister()
+	 */
+
+	mutex_lock(&sp->mutex);
+	list_add_tail(&class_intf->node, &sp->interfaces);
 	if (class_intf->add_dev) {
 		class_dev_iter_init(&iter, parent, NULL, NULL);
 		while ((dev = class_dev_iter_next(&iter)))
-			class_intf->add_dev(dev, class_intf);
+			class_intf->add_dev(dev);
 		class_dev_iter_exit(&iter);
 	}
-	mutex_unlock(&parent->p->mutex);
+	mutex_unlock(&sp->mutex);
 
 	return 0;
 }
@@ -468,29 +497,40 @@ EXPORT_SYMBOL_GPL(class_interface_register);
 
 void class_interface_unregister(struct class_interface *class_intf)
 {
-	struct class *parent = class_intf->class;
+	struct subsys_private *sp;
+	const struct class *parent = class_intf->class;
 	struct class_dev_iter iter;
 	struct device *dev;
 
 	if (!parent)
 		return;
 
-	mutex_lock(&parent->p->mutex);
+	sp = class_to_subsys(parent);
+	if (!sp)
+		return;
+
+	mutex_lock(&sp->mutex);
 	list_del_init(&class_intf->node);
 	if (class_intf->remove_dev) {
 		class_dev_iter_init(&iter, parent, NULL, NULL);
 		while ((dev = class_dev_iter_next(&iter)))
-			class_intf->remove_dev(dev, class_intf);
+			class_intf->remove_dev(dev);
 		class_dev_iter_exit(&iter);
 	}
-	mutex_unlock(&parent->p->mutex);
+	mutex_unlock(&sp->mutex);
 
-	class_put(parent);
+	/*
+	 * Decrement the reference count twice, once for the class_to_subsys()
+	 * call in the start of this function, and the second one from the
+	 * reference increment in class_interface_register()
+	 */
+	subsys_put(sp);
+	subsys_put(sp);
 }
 EXPORT_SYMBOL_GPL(class_interface_unregister);
 
-ssize_t show_class_attr_string(struct class *class,
-			       struct class_attribute *attr, char *buf)
+ssize_t show_class_attr_string(const struct class *class,
+			       const struct class_attribute *attr, char *buf)
 {
 	struct class_attribute_string *cs;
 
@@ -587,6 +627,31 @@ void class_compat_remove_link(struct class_compat *cls, struct device *dev,
 }
 EXPORT_SYMBOL_GPL(class_compat_remove_link);
 
+/**
+ * class_is_registered - determine if at this moment in time, a class is
+ *			 registered in the driver core or not.
+ * @class: the class to check
+ *
+ * Returns a boolean to state if the class is registered in the driver core
+ * or not.  Note that the value could switch right after this call is made,
+ * so only use this in places where you "know" it is safe to do so (usually
+ * to determine if the specific class has been registered yet or not).
+ *
+ * Be careful in using this.
+ */
+bool class_is_registered(const struct class *class)
+{
+	struct subsys_private *sp = class_to_subsys(class);
+	bool is_initialized = false;
+
+	if (sp) {
+		is_initialized = true;
+		subsys_put(sp);
+	}
+	return is_initialized;
+}
+EXPORT_SYMBOL_GPL(class_is_registered);
+
 int __init classes_init(void)
 {
 	class_kset = kset_create_and_add("class", NULL, NULL);
diff --git a/drivers/base/core.c b/drivers/base/core.c
index 6878dfcbf0d6..3dff5037943e 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -36,19 +36,6 @@
 #include "physical_location.h"
 #include "power/power.h"
 
-#ifdef CONFIG_SYSFS_DEPRECATED
-#ifdef CONFIG_SYSFS_DEPRECATED_V2
-long sysfs_deprecated = 1;
-#else
-long sysfs_deprecated = 0;
-#endif
-static int __init sysfs_deprecated_setup(char *arg)
-{
-	return kstrtol(arg, 10, &sysfs_deprecated);
-}
-early_param("sysfs.deprecated", sysfs_deprecated_setup);
-#endif
-
 /* Device links support. */
 static LIST_HEAD(deferred_sync);
 static unsigned int defer_sync_state_count = 1;
@@ -550,13 +537,11 @@ static void devlink_dev_release(struct device *dev)
 
 static struct class devlink_class = {
 	.name = "devlink",
-	.owner = THIS_MODULE,
 	.dev_groups = devlink_groups,
 	.dev_release = devlink_dev_release,
 };
 
-static int devlink_add_symlinks(struct device *dev,
-				struct class_interface *class_intf)
+static int devlink_add_symlinks(struct device *dev)
 {
 	int ret;
 	size_t len;
@@ -605,8 +590,7 @@ out:
 	return ret;
 }
 
-static void devlink_remove_symlinks(struct device *dev,
-				   struct class_interface *class_intf)
+static void devlink_remove_symlinks(struct device *dev)
 {
 	struct device_link *link = to_devlink(dev);
 	size_t len;
@@ -1173,10 +1157,7 @@ static void device_links_flush_sync_list(struct list_head *list,
 		if (dev != dont_lock_dev)
 			device_lock(dev);
 
-		if (dev->bus->sync_state)
-			dev->bus->sync_state(dev);
-		else if (dev->driver && dev->driver->sync_state)
-			dev->driver->sync_state(dev);
+		dev_sync_state(dev);
 
 		if (dev != dont_lock_dev)
 			device_unlock(dev);
@@ -1685,6 +1666,31 @@ static int __init fw_devlink_strict_setup(char *arg)
 }
 early_param("fw_devlink.strict", fw_devlink_strict_setup);
 
+#define FW_DEVLINK_SYNC_STATE_STRICT	0
+#define FW_DEVLINK_SYNC_STATE_TIMEOUT	1
+
+#ifndef CONFIG_FW_DEVLINK_SYNC_STATE_TIMEOUT
+static int fw_devlink_sync_state;
+#else
+static int fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_TIMEOUT;
+#endif
+
+static int __init fw_devlink_sync_state_setup(char *arg)
+{
+	if (!arg)
+		return -EINVAL;
+
+	if (strcmp(arg, "strict") == 0) {
+		fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_STRICT;
+		return 0;
+	} else if (strcmp(arg, "timeout") == 0) {
+		fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_TIMEOUT;
+		return 0;
+	}
+	return -EINVAL;
+}
+early_param("fw_devlink.sync_state", fw_devlink_sync_state_setup);
+
 static inline u32 fw_devlink_get_flags(u8 fwlink_flags)
 {
 	if (fwlink_flags & FWLINK_FLAG_CYCLE)
@@ -1755,6 +1761,44 @@ void fw_devlink_drivers_done(void)
 	device_links_write_unlock();
 }
 
+static int fw_devlink_dev_sync_state(struct device *dev, void *data)
+{
+	struct device_link *link = to_devlink(dev);
+	struct device *sup = link->supplier;
+
+	if (!(link->flags & DL_FLAG_MANAGED) ||
+	    link->status == DL_STATE_ACTIVE || sup->state_synced ||
+	    !dev_has_sync_state(sup))
+		return 0;
+
+	if (fw_devlink_sync_state == FW_DEVLINK_SYNC_STATE_STRICT) {
+		dev_warn(sup, "sync_state() pending due to %s\n",
+			 dev_name(link->consumer));
+		return 0;
+	}
+
+	if (!list_empty(&sup->links.defer_sync))
+		return 0;
+
+	dev_warn(sup, "Timed out. Forcing sync_state()\n");
+	sup->state_synced = true;
+	get_device(sup);
+	list_add_tail(&sup->links.defer_sync, data);
+
+	return 0;
+}
+
+void fw_devlink_probing_done(void)
+{
+	LIST_HEAD(sync_list);
+
+	device_links_write_lock();
+	class_for_each_device(&devlink_class, NULL, &sync_list,
+			      fw_devlink_dev_sync_state);
+	device_links_write_unlock();
+	device_links_flush_sync_list(&sync_list, NULL);
+}
+
 /**
  * wait_for_init_devices_probe - Try to probe any device needed for init
  *
@@ -2209,8 +2253,12 @@ static void fw_devlink_link_device(struct device *dev)
 int (*platform_notify)(struct device *dev) = NULL;
 int (*platform_notify_remove)(struct device *dev) = NULL;
 static struct kobject *dev_kobj;
-struct kobject *sysfs_dev_char_kobj;
-struct kobject *sysfs_dev_block_kobj;
+
+/* /sys/dev/char */
+static struct kobject *sysfs_dev_char_kobj;
+
+/* /sys/dev/block */
+static struct kobject *sysfs_dev_block_kobj;
 
 static DEFINE_MUTEX(device_hotplug_lock);
 
@@ -2779,7 +2827,7 @@ EXPORT_SYMBOL_GPL(devm_device_add_groups);
 
 static int device_add_attrs(struct device *dev)
 {
-	struct class *class = dev->class;
+	const struct class *class = dev->class;
 	const struct device_type *type = dev->type;
 	int error;
 
@@ -2846,7 +2894,7 @@ static int device_add_attrs(struct device *dev)
 
 static void device_remove_attrs(struct device *dev)
 {
-	struct class *class = dev->class;
+	const struct class *class = dev->class;
 	const struct device_type *type = dev->type;
 
 	if (dev->physical_location) {
@@ -3079,7 +3127,7 @@ struct kobject *virtual_device_parent(struct device *dev)
 
 struct class_dir {
 	struct kobject kobj;
-	struct class *class;
+	const struct class *class;
 };
 
 #define to_class_dir(obj) container_of(obj, struct class_dir, kobj)
@@ -3103,8 +3151,8 @@ static const struct kobj_type class_dir_ktype = {
 	.child_ns_type	= class_dir_child_ns_type
 };
 
-static struct kobject *
-class_dir_create_and_add(struct class *class, struct kobject *parent_kobj)
+static struct kobject *class_dir_create_and_add(struct subsys_private *sp,
+						struct kobject *parent_kobj)
 {
 	struct class_dir *dir;
 	int retval;
@@ -3113,12 +3161,12 @@ class_dir_create_and_add(struct class *class, struct kobject *parent_kobj)
 	if (!dir)
 		return ERR_PTR(-ENOMEM);
 
-	dir->class = class;
+	dir->class = sp->class;
 	kobject_init(&dir->kobj, &class_dir_ktype);
 
-	dir->kobj.kset = &class->p->glue_dirs;
+	dir->kobj.kset = &sp->glue_dirs;
 
-	retval = kobject_add(&dir->kobj, parent_kobj, "%s", class->name);
+	retval = kobject_add(&dir->kobj, parent_kobj, "%s", sp->class->name);
 	if (retval < 0) {
 		kobject_put(&dir->kobj);
 		return ERR_PTR(retval);
@@ -3131,21 +3179,13 @@ static DEFINE_MUTEX(gdp_mutex);
 static struct kobject *get_device_parent(struct device *dev,
 					 struct device *parent)
 {
+	struct subsys_private *sp = class_to_subsys(dev->class);
 	struct kobject *kobj = NULL;
 
-	if (dev->class) {
+	if (sp) {
 		struct kobject *parent_kobj;
 		struct kobject *k;
 
-#ifdef CONFIG_BLOCK
-		/* block disks show up in /sys/block */
-		if (sysfs_deprecated && dev->class == &block_class) {
-			if (parent && parent->class == &block_class)
-				return &parent->kobj;
-			return &block_class.p->subsys.kobj;
-		}
-#endif
-
 		/*
 		 * If we have no parent, we live in "virtual".
 		 * Class-devices with a non class-device as parent, live
@@ -3153,30 +3193,34 @@ static struct kobject *get_device_parent(struct device *dev,
 		 */
 		if (parent == NULL)
 			parent_kobj = virtual_device_parent(dev);
-		else if (parent->class && !dev->class->ns_type)
+		else if (parent->class && !dev->class->ns_type) {
+			subsys_put(sp);
 			return &parent->kobj;
-		else
+		} else {
 			parent_kobj = &parent->kobj;
+		}
 
 		mutex_lock(&gdp_mutex);
 
 		/* find our class-directory at the parent and reference it */
-		spin_lock(&dev->class->p->glue_dirs.list_lock);
-		list_for_each_entry(k, &dev->class->p->glue_dirs.list, entry)
+		spin_lock(&sp->glue_dirs.list_lock);
+		list_for_each_entry(k, &sp->glue_dirs.list, entry)
 			if (k->parent == parent_kobj) {
 				kobj = kobject_get(k);
 				break;
 			}
-		spin_unlock(&dev->class->p->glue_dirs.list_lock);
+		spin_unlock(&sp->glue_dirs.list_lock);
 		if (kobj) {
 			mutex_unlock(&gdp_mutex);
+			subsys_put(sp);
 			return kobj;
 		}
 
 		/* or create a new class-directory at the parent device */
-		k = class_dir_create_and_add(dev->class, parent_kobj);
+		k = class_dir_create_and_add(sp, parent_kobj);
 		/* do not emit an uevent for this simple "glue" directory */
 		mutex_unlock(&gdp_mutex);
+		subsys_put(sp);
 		return k;
 	}
 
@@ -3199,10 +3243,23 @@ static struct kobject *get_device_parent(struct device *dev,
 static inline bool live_in_glue_dir(struct kobject *kobj,
 				    struct device *dev)
 {
-	if (!kobj || !dev->class ||
-	    kobj->kset != &dev->class->p->glue_dirs)
+	struct subsys_private *sp;
+	bool retval;
+
+	if (!kobj || !dev->class)
 		return false;
-	return true;
+
+	sp = class_to_subsys(dev->class);
+	if (!sp)
+		return false;
+
+	if (kobj->kset == &sp->glue_dirs)
+		retval = true;
+	else
+		retval = false;
+
+	subsys_put(sp);
+	return retval;
 }
 
 static inline struct kobject *get_glue_dir(struct device *dev)
@@ -3299,6 +3356,7 @@ static void cleanup_glue_dir(struct device *dev, struct kobject *glue_dir)
 static int device_add_class_symlinks(struct device *dev)
 {
 	struct device_node *of_node = dev_of_node(dev);
+	struct subsys_private *sp;
 	int error;
 
 	if (of_node) {
@@ -3308,12 +3366,11 @@ static int device_add_class_symlinks(struct device *dev)
 		/* An error here doesn't warrant bringing down the device */
 	}
 
-	if (!dev->class)
+	sp = class_to_subsys(dev->class);
+	if (!sp)
 		return 0;
 
-	error = sysfs_create_link(&dev->kobj,
-				  &dev->class->p->subsys.kobj,
-				  "subsystem");
+	error = sysfs_create_link(&dev->kobj, &sp->subsys.kobj, "subsystem");
 	if (error)
 		goto out_devnode;
 
@@ -3324,46 +3381,38 @@ static int device_add_class_symlinks(struct device *dev)
 			goto out_subsys;
 	}
 
-#ifdef CONFIG_BLOCK
-	/* /sys/block has directories and does not need symlinks */
-	if (sysfs_deprecated && dev->class == &block_class)
-		return 0;
-#endif
-
 	/* link in the class directory pointing to the device */
-	error = sysfs_create_link(&dev->class->p->subsys.kobj,
-				  &dev->kobj, dev_name(dev));
+	error = sysfs_create_link(&sp->subsys.kobj, &dev->kobj, dev_name(dev));
 	if (error)
 		goto out_device;
-
-	return 0;
+	goto exit;
 
 out_device:
 	sysfs_remove_link(&dev->kobj, "device");
-
 out_subsys:
 	sysfs_remove_link(&dev->kobj, "subsystem");
 out_devnode:
 	sysfs_remove_link(&dev->kobj, "of_node");
+exit:
+	subsys_put(sp);
 	return error;
 }
 
 static void device_remove_class_symlinks(struct device *dev)
 {
+	struct subsys_private *sp = class_to_subsys(dev->class);
+
 	if (dev_of_node(dev))
 		sysfs_remove_link(&dev->kobj, "of_node");
 
-	if (!dev->class)
+	if (!sp)
 		return;
 
 	if (dev->parent && device_is_not_partition(dev))
 		sysfs_remove_link(&dev->kobj, "device");
 	sysfs_remove_link(&dev->kobj, "subsystem");
-#ifdef CONFIG_BLOCK
-	if (sysfs_deprecated && dev->class == &block_class)
-		return;
-#endif
-	sysfs_delete_link(&dev->class->p->subsys.kobj, &dev->kobj, dev_name(dev));
+	sysfs_delete_link(&sp->subsys.kobj, &dev->kobj, dev_name(dev));
+	subsys_put(sp);
 }
 
 /**
@@ -3383,27 +3432,13 @@ int dev_set_name(struct device *dev, const char *fmt, ...)
 }
 EXPORT_SYMBOL_GPL(dev_set_name);
 
-/**
- * device_to_dev_kobj - select a /sys/dev/ directory for the device
- * @dev: device
- *
- * By default we select char/ for new entries.  Setting class->dev_obj
- * to NULL prevents an entry from being created.  class->dev_kobj must
- * be set (or cleared) before any devices are registered to the class
- * otherwise device_create_sys_dev_entry() and
- * device_remove_sys_dev_entry() will disagree about the presence of
- * the link.
- */
+/* select a /sys/dev/ directory for the device */
 static struct kobject *device_to_dev_kobj(struct device *dev)
 {
-	struct kobject *kobj;
-
-	if (dev->class)
-		kobj = dev->class->dev_kobj;
+	if (is_blockdev(dev))
+		return sysfs_dev_block_kobj;
 	else
-		kobj = sysfs_dev_char_kobj;
-
-	return kobj;
+		return sysfs_dev_char_kobj;
 }
 
 static int device_create_sys_dev_entry(struct device *dev)
@@ -3472,6 +3507,7 @@ static int device_private_init(struct device *dev)
  */
 int device_add(struct device *dev)
 {
+	struct subsys_private *sp;
 	struct device *parent;
 	struct kobject *kobj;
 	struct class_interface *class_intf;
@@ -3600,18 +3636,18 @@ int device_add(struct device *dev)
 		klist_add_tail(&dev->p->knode_parent,
 			       &parent->p->klist_children);
 
-	if (dev->class) {
-		mutex_lock(&dev->class->p->mutex);
+	sp = class_to_subsys(dev->class);
+	if (sp) {
+		mutex_lock(&sp->mutex);
 		/* tie the class to the device */
-		klist_add_tail(&dev->p->knode_class,
-			       &dev->class->p->klist_devices);
+		klist_add_tail(&dev->p->knode_class, &sp->klist_devices);
 
 		/* notify any interfaces that the device is here */
-		list_for_each_entry(class_intf,
-				    &dev->class->p->interfaces, node)
+		list_for_each_entry(class_intf, &sp->interfaces, node)
 			if (class_intf->add_dev)
-				class_intf->add_dev(dev, class_intf);
-		mutex_unlock(&dev->class->p->mutex);
+				class_intf->add_dev(dev);
+		mutex_unlock(&sp->mutex);
+		subsys_put(sp);
 	}
 done:
 	put_device(dev);
@@ -3731,6 +3767,7 @@ EXPORT_SYMBOL_GPL(kill_device);
  */
 void device_del(struct device *dev)
 {
+	struct subsys_private *sp;
 	struct device *parent = dev->parent;
 	struct kobject *glue_dir = NULL;
 	struct class_interface *class_intf;
@@ -3757,18 +3794,20 @@ void device_del(struct device *dev)
 		device_remove_sys_dev_entry(dev);
 		device_remove_file(dev, &dev_attr_dev);
 	}
-	if (dev->class) {
+
+	sp = class_to_subsys(dev->class);
+	if (sp) {
 		device_remove_class_symlinks(dev);
 
-		mutex_lock(&dev->class->p->mutex);
+		mutex_lock(&sp->mutex);
 		/* notify any interfaces that the device is now gone */
-		list_for_each_entry(class_intf,
-				    &dev->class->p->interfaces, node)
+		list_for_each_entry(class_intf, &sp->interfaces, node)
 			if (class_intf->remove_dev)
-				class_intf->remove_dev(dev, class_intf);
+				class_intf->remove_dev(dev);
 		/* remove the device from the class list */
 		klist_del(&dev->p->knode_class);
-		mutex_unlock(&dev->class->p->mutex);
+		mutex_unlock(&sp->mutex);
+		subsys_put(sp);
 	}
 	device_remove_file(dev, &dev_attr_uevent);
 	device_remove_attrs(dev);
@@ -4231,7 +4270,7 @@ static void device_create_release(struct device *dev)
 }
 
 static __printf(6, 0) struct device *
-device_create_groups_vargs(struct class *class, struct device *parent,
+device_create_groups_vargs(const struct class *class, struct device *parent,
 			   dev_t devt, void *drvdata,
 			   const struct attribute_group **groups,
 			   const char *fmt, va_list args)
@@ -4291,11 +4330,8 @@ error:
  * pointer.
  *
  * Returns &struct device pointer on success, or ERR_PTR() on error.
- *
- * Note: the struct class passed to this function must have previously
- * been created with a call to class_create().
  */
-struct device *device_create(struct class *class, struct device *parent,
+struct device *device_create(const struct class *class, struct device *parent,
 			     dev_t devt, void *drvdata, const char *fmt, ...)
 {
 	va_list vargs;
@@ -4332,11 +4368,8 @@ EXPORT_SYMBOL_GPL(device_create);
  * pointer.
  *
  * Returns &struct device pointer on success, or ERR_PTR() on error.
- *
- * Note: the struct class passed to this function must have previously
- * been created with a call to class_create().
  */
-struct device *device_create_with_groups(struct class *class,
+struct device *device_create_with_groups(const struct class *class,
 					 struct device *parent, dev_t devt,
 					 void *drvdata,
 					 const struct attribute_group **groups,
@@ -4361,7 +4394,7 @@ EXPORT_SYMBOL_GPL(device_create_with_groups);
  * This call unregisters and cleans up a device that was created with a
  * call to device_create().
  */
-void device_destroy(struct class *class, dev_t devt)
+void device_destroy(const struct class *class, dev_t devt)
 {
 	struct device *dev;
 
@@ -4383,9 +4416,12 @@ EXPORT_SYMBOL_GPL(device_destroy);
  * on the same device to ensure that new_name is valid and
  * won't conflict with other devices.
  *
- * Note: Don't call this function.  Currently, the networking layer calls this
- * function, but that will change.  The following text from Kay Sievers offers
- * some insight:
+ * Note: given that some subsystems (networking and infiniband) use this
+ * function, with no immediate plans for this to change, we cannot assume or
+ * require that this function not be called at all.
+ *
+ * However, if you're writing new code, do not call this function. The following
+ * text from Kay Sievers offers some insight:
  *
  * Renaming devices is racy at many levels, symlinks and other stuff are not
  * replaced atomically, and you get a "move" uevent, but it's not easy to
@@ -4399,13 +4435,6 @@ EXPORT_SYMBOL_GPL(device_destroy);
  * kernel device renaming. Besides that, it's not even implemented now for
  * other things than (driver-core wise very simple) network devices.
  *
- * We are currently about to change network renaming in udev to completely
- * disallow renaming of devices in the same namespace as the kernel uses,
- * because we can't solve the problems properly, that arise with swapping names
- * of multiple interfaces without races. Means, renaming of eth[0-9]* will only
- * be allowed to some other name than eth[0-9]*, for the aforementioned
- * reasons.
- *
  * Make up a "real" name in the driver before you register anything, or add
  * some other attributes for userspace to find the device, or use udev to add
  * symlinks -- but never rename kernel devices later, it's a complete mess. We
@@ -4431,9 +4460,16 @@ int device_rename(struct device *dev, const char *new_name)
 	}
 
 	if (dev->class) {
-		error = sysfs_rename_link_ns(&dev->class->p->subsys.kobj,
-					     kobj, old_device_name,
+		struct subsys_private *sp = class_to_subsys(dev->class);
+
+		if (!sp) {
+			error = -EINVAL;
+			goto out;
+		}
+
+		error = sysfs_rename_link_ns(&sp->subsys.kobj, kobj, old_device_name,
 					     new_name, kobject_namespace(kobj));
+		subsys_put(sp);
 		if (error)
 			goto out;
 	}
@@ -4558,7 +4594,7 @@ static int device_attrs_change_owner(struct device *dev, kuid_t kuid,
 				     kgid_t kgid)
 {
 	struct kobject *kobj = &dev->kobj;
-	struct class *class = dev->class;
+	const struct class *class = dev->class;
 	const struct device_type *type = dev->type;
 	int error;
 
@@ -4616,6 +4652,7 @@ int device_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid)
 {
 	int error;
 	struct kobject *kobj = &dev->kobj;
+	struct subsys_private *sp;
 
 	dev = get_device(dev);
 	if (!dev)
@@ -4652,21 +4689,19 @@ int device_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid)
 	if (error)
 		goto out;
 
-#ifdef CONFIG_BLOCK
-	if (sysfs_deprecated && dev->class == &block_class)
-		goto out;
-#endif
-
 	/*
 	 * Change the owner of the symlink located in the class directory of
 	 * the device class associated with @dev which points to the actual
 	 * directory entry for @dev to @kuid/@kgid. This ensures that the
 	 * symlink shows the same permissions as its target.
 	 */
-	error = sysfs_link_change_owner(&dev->class->p->subsys.kobj, &dev->kobj,
-					dev_name(dev), kuid, kgid);
-	if (error)
+	sp = class_to_subsys(dev->class);
+	if (!sp) {
+		error = -EINVAL;
 		goto out;
+	}
+	error = sysfs_link_change_owner(&sp->subsys.kobj, &dev->kobj, dev_name(dev), kuid, kgid);
+	subsys_put(sp);
 
 out:
 	put_device(dev);
@@ -4965,9 +5000,13 @@ void set_primary_fwnode(struct device *dev, struct fwnode_handle *fwnode)
 	} else {
 		if (fwnode_is_primary(fn)) {
 			dev->fwnode = fn->secondary;
+
+			/* Skip nullifying fn->secondary if the primary is shared */
+			if (parent && fn == parent->fwnode)
+				return;
+
 			/* Set fn->secondary = NULL, so fn remains the primary fwnode */
-			if (!(parent && fn == parent->fwnode))
-				fn->secondary = NULL;
+			fn->secondary = NULL;
 		} else {
 			dev->fwnode = NULL;
 		}
diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index 182c6122f815..c1815b9dae68 100644
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -487,7 +487,8 @@ static const struct attribute_group *cpu_root_attr_groups[] = {
 bool cpu_is_hotpluggable(unsigned int cpu)
 {
 	struct device *dev = get_cpu_device(cpu);
-	return dev && container_of(dev, struct cpu, dev)->hotpluggable;
+	return dev && container_of(dev, struct cpu, dev)->hotpluggable
+		&& tick_nohz_cpu_hotpluggable(cpu);
 }
 EXPORT_SYMBOL_GPL(cpu_is_hotpluggable);
 
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 8def2ba08a82..9c09ca5c4ab6 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -315,6 +315,8 @@ static void deferred_probe_timeout_work_func(struct work_struct *work)
 	list_for_each_entry(p, &deferred_probe_pending_list, deferred_probe)
 		dev_info(p->device, "deferred probe pending\n");
 	mutex_unlock(&deferred_probe_mutex);
+
+	fw_devlink_probing_done();
 }
 static DECLARE_DELAYED_WORK(deferred_probe_timeout_work, deferred_probe_timeout_work_func);
 
@@ -364,6 +366,10 @@ static int deferred_probe_initcall(void)
 		schedule_delayed_work(&deferred_probe_timeout_work,
 			driver_deferred_probe_timeout * HZ);
 	}
+
+	if (!IS_ENABLED(CONFIG_MODULES))
+		fw_devlink_probing_done();
+
 	return 0;
 }
 late_initcall(deferred_probe_initcall);
@@ -504,6 +510,27 @@ EXPORT_SYMBOL_GPL(device_bind_driver);
 static atomic_t probe_count = ATOMIC_INIT(0);
 static DECLARE_WAIT_QUEUE_HEAD(probe_waitqueue);
 
+static ssize_t state_synced_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t count)
+{
+	int ret = 0;
+
+	if (strcmp("1", buf))
+		return -EINVAL;
+
+	device_lock(dev);
+	if (!dev->state_synced) {
+		dev->state_synced = true;
+		dev_sync_state(dev);
+	} else {
+		ret = -EINVAL;
+	}
+	device_unlock(dev);
+
+	return ret ? ret : count;
+}
+
 static ssize_t state_synced_show(struct device *dev,
 				 struct device_attribute *attr, char *buf)
 {
@@ -515,7 +542,7 @@ static ssize_t state_synced_show(struct device *dev,
 
 	return sysfs_emit(buf, "%u\n", val);
 }
-static DEVICE_ATTR_RO(state_synced);
+static DEVICE_ATTR_RW(state_synced);
 
 static void device_unbind_cleanup(struct device *dev)
 {
@@ -708,7 +735,12 @@ static int really_probe_debug(struct device *dev, struct device_driver *drv)
 	calltime = ktime_get();
 	ret = really_probe(dev, drv);
 	rettime = ktime_get();
-	pr_debug("probe of %s returned %d after %lld usecs\n",
+	/*
+	 * Don't change this to pr_debug() because that requires
+	 * CONFIG_DYNAMIC_DEBUG and we want a simple 'initcall_debug' on the
+	 * kernel commandline to print this all the time at the debug level.
+	 */
+	printk(KERN_DEBUG "probe of %s returned %d after %lld usecs\n",
 		 dev_name(dev), ret, ktime_us_delta(rettime, calltime));
 	return ret;
 }
diff --git a/drivers/base/devcoredump.c b/drivers/base/devcoredump.c
index 1c06781f7114..91536ee05f14 100644
--- a/drivers/base/devcoredump.c
+++ b/drivers/base/devcoredump.c
@@ -167,7 +167,7 @@ static int devcd_free(struct device *dev, void *data)
 	return 0;
 }
 
-static ssize_t disabled_show(struct class *class, struct class_attribute *attr,
+static ssize_t disabled_show(const struct class *class, const struct class_attribute *attr,
 			     char *buf)
 {
 	return sysfs_emit(buf, "%d\n", devcd_disabled);
@@ -197,7 +197,7 @@ static ssize_t disabled_show(struct class *class, struct class_attribute *attr,
  * so, above situation would not occur.
  */
 
-static ssize_t disabled_store(struct class *class, struct class_attribute *attr,
+static ssize_t disabled_store(const struct class *class, const struct class_attribute *attr,
 			      const char *buf, size_t count)
 {
 	long tmp = simple_strtol(buf, NULL, 10);
@@ -226,7 +226,6 @@ ATTRIBUTE_GROUPS(devcd_class);
 
 static struct class devcd_class = {
 	.name		= "devcoredump",
-	.owner		= THIS_MODULE,
 	.dev_release	= devcd_dev_release,
 	.dev_groups	= devcd_dev_groups,
 	.class_groups	= devcd_class_groups,
diff --git a/drivers/base/devres.c b/drivers/base/devres.c
index c0e100074aa3..5c998cfac335 100644
--- a/drivers/base/devres.c
+++ b/drivers/base/devres.c
@@ -722,20 +722,21 @@ static void devm_action_release(struct device *dev, void *res)
 }
 
 /**
- * devm_add_action() - add a custom action to list of managed resources
+ * __devm_add_action() - add a custom action to list of managed resources
  * @dev: Device that owns the action
  * @action: Function that should be called
  * @data: Pointer to data passed to @action implementation
+ * @name: Name of the resource (for debugging purposes)
  *
  * This adds a custom action to the list of managed resources so that
  * it gets executed as part of standard resource unwinding.
  */
-int devm_add_action(struct device *dev, void (*action)(void *), void *data)
+int __devm_add_action(struct device *dev, void (*action)(void *), void *data, const char *name)
 {
 	struct action_devres *devres;
 
-	devres = devres_alloc(devm_action_release,
-			      sizeof(struct action_devres), GFP_KERNEL);
+	devres = __devres_alloc_node(devm_action_release, sizeof(struct action_devres),
+				     GFP_KERNEL, NUMA_NO_NODE, name);
 	if (!devres)
 		return -ENOMEM;
 
@@ -745,7 +746,7 @@ int devm_add_action(struct device *dev, void (*action)(void *), void *data)
 	devres_add(dev, devres);
 	return 0;
 }
-EXPORT_SYMBOL_GPL(devm_add_action);
+EXPORT_SYMBOL_GPL(__devm_add_action);
 
 /**
  * devm_remove_action() - removes previously added custom action
diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
index ae72d4ba8547..b848764ef018 100644
--- a/drivers/base/devtmpfs.c
+++ b/drivers/base/devtmpfs.c
@@ -94,15 +94,6 @@ static struct file_system_type dev_fs_type = {
 	.mount = public_dev_mount,
 };
 
-#ifdef CONFIG_BLOCK
-static inline int is_blockdev(struct device *dev)
-{
-	return dev->class == &block_class;
-}
-#else
-static inline int is_blockdev(struct device *dev) { return 0; }
-#endif
-
 static int devtmpfs_submit_req(struct req *req, const char *tmp)
 {
 	init_completion(&req->done);
diff --git a/drivers/base/firmware_loader/Kconfig b/drivers/base/firmware_loader/Kconfig
index 5166b323a0f8..5ca00e02fe82 100644
--- a/drivers/base/firmware_loader/Kconfig
+++ b/drivers/base/firmware_loader/Kconfig
@@ -3,6 +3,8 @@ menu "Firmware loader"
 
 config FW_LOADER
 	tristate "Firmware loading facility" if EXPERT
+	select CRYPTO_HASH if FW_LOADER_DEBUG
+	select CRYPTO_SHA256 if FW_LOADER_DEBUG
 	default y
 	help
 	  This enables the firmware loading facility in the kernel. The kernel
@@ -24,6 +26,17 @@ config FW_LOADER
 	  You also want to be sure to enable this built-in if you are going to
 	  enable built-in firmware (CONFIG_EXTRA_FIRMWARE).
 
+config FW_LOADER_DEBUG
+	bool "Log filenames and checksums for loaded firmware"
+	depends on CRYPTO = FW_LOADER || CRYPTO=y
+	depends on DYNAMIC_DEBUG
+	depends on FW_LOADER
+	default FW_LOADER
+	help
+	  Select this option to use dynamic debug to log firmware filenames and
+	  SHA256 checksums to the kernel log for each firmware file that is
+	  loaded.
+
 if FW_LOADER
 
 config FW_LOADER_PAGED_BUF
diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c
index 017c4cdb219e..9d79d5ad9102 100644
--- a/drivers/base/firmware_loader/main.c
+++ b/drivers/base/firmware_loader/main.c
@@ -493,9 +493,9 @@ fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv,
 					     const void *in_buffer))
 {
 	size_t size;
-	int i, len;
+	int i, len, maxlen = 0;
 	int rc = -ENOENT;
-	char *path;
+	char *path, *nt = NULL;
 	size_t msize = INT_MAX;
 	void *buffer = NULL;
 
@@ -518,8 +518,17 @@ fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv,
 		if (!fw_path[i][0])
 			continue;
 
-		len = snprintf(path, PATH_MAX, "%s/%s%s",
-			       fw_path[i], fw_priv->fw_name, suffix);
+		/* strip off \n from customized path */
+		maxlen = strlen(fw_path[i]);
+		if (i == 0) {
+			nt = strchr(fw_path[i], '\n');
+			if (nt)
+				maxlen = nt - fw_path[i];
+		}
+
+		len = snprintf(path, PATH_MAX, "%.*s/%s%s",
+			       maxlen, fw_path[i],
+			       fw_priv->fw_name, suffix);
 		if (len >= PATH_MAX) {
 			rc = -ENAMETOOLONG;
 			break;
@@ -791,6 +800,50 @@ static void fw_abort_batch_reqs(struct firmware *fw)
 	mutex_unlock(&fw_lock);
 }
 
+#if defined(CONFIG_FW_LOADER_DEBUG)
+#include <crypto/hash.h>
+#include <crypto/sha2.h>
+
+static void fw_log_firmware_info(const struct firmware *fw, const char *name, struct device *device)
+{
+	struct shash_desc *shash;
+	struct crypto_shash *alg;
+	u8 *sha256buf;
+	char *outbuf;
+
+	alg = crypto_alloc_shash("sha256", 0, 0);
+	if (!alg)
+		return;
+
+	sha256buf = kmalloc(SHA256_DIGEST_SIZE, GFP_KERNEL);
+	outbuf = kmalloc(SHA256_BLOCK_SIZE + 1, GFP_KERNEL);
+	shash = kmalloc(sizeof(*shash) + crypto_shash_descsize(alg), GFP_KERNEL);
+	if (!sha256buf || !outbuf || !shash)
+		goto out_free;
+
+	shash->tfm = alg;
+
+	if (crypto_shash_digest(shash, fw->data, fw->size, sha256buf) < 0)
+		goto out_shash;
+
+	for (int i = 0; i < SHA256_DIGEST_SIZE; i++)
+		sprintf(&outbuf[i * 2], "%02x", sha256buf[i]);
+	outbuf[SHA256_BLOCK_SIZE] = 0;
+	dev_dbg(device, "Loaded FW: %s, sha256: %s\n", name, outbuf);
+
+out_shash:
+	crypto_free_shash(alg);
+out_free:
+	kfree(shash);
+	kfree(outbuf);
+	kfree(sha256buf);
+}
+#else
+static void fw_log_firmware_info(const struct firmware *fw, const char *name,
+				 struct device *device)
+{}
+#endif
+
 /* called from request_firmware() and request_firmware_work_func() */
 static int
 _request_firmware(const struct firmware **firmware_p, const char *name,
@@ -861,11 +914,13 @@ _request_firmware(const struct firmware **firmware_p, const char *name,
 	revert_creds(old_cred);
 	put_cred(kern_cred);
 
- out:
+out:
 	if (ret < 0) {
 		fw_abort_batch_reqs(fw);
 		release_firmware(fw);
 		fw = NULL;
+	} else {
+		fw_log_firmware_info(fw, name, device);
 	}
 
 	*firmware_p = fw;
diff --git a/drivers/base/firmware_loader/sysfs.c b/drivers/base/firmware_loader/sysfs.c
index 56911d75b90a..c9c93b47d9a5 100644
--- a/drivers/base/firmware_loader/sysfs.c
+++ b/drivers/base/firmware_loader/sysfs.c
@@ -25,7 +25,7 @@ void __fw_load_abort(struct fw_priv *fw_priv)
 }
 
 #ifdef CONFIG_FW_LOADER_USER_HELPER
-static ssize_t timeout_show(struct class *class, struct class_attribute *attr,
+static ssize_t timeout_show(const struct class *class, const struct class_attribute *attr,
 			    char *buf)
 {
 	return sysfs_emit(buf, "%d\n", __firmware_loading_timeout());
@@ -44,7 +44,7 @@ static ssize_t timeout_show(struct class *class, struct class_attribute *attr,
  *
  *	Note: zero means 'wait forever'.
  **/
-static ssize_t timeout_store(struct class *class, struct class_attribute *attr,
+static ssize_t timeout_store(const struct class *class, const struct class_attribute *attr,
 			     const char *buf, size_t count)
 {
 	int tmp_loading_timeout = simple_strtol(buf, NULL, 10);
diff --git a/drivers/base/physical_location.h b/drivers/base/physical_location.h
index 82cde9f1b161..3f3f61307998 100644
--- a/drivers/base/physical_location.h
+++ b/drivers/base/physical_location.h
@@ -8,7 +8,7 @@
 #include <linux/device.h>
 
 #ifdef CONFIG_ACPI
-extern bool dev_add_physical_location(struct device *dev);
+bool dev_add_physical_location(struct device *dev);
 extern const struct attribute_group dev_attr_physical_location_group;
 #else
 static inline bool dev_add_physical_location(struct device *dev) { return false; };
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index c50139207794..f85f3515c258 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -679,7 +679,7 @@ static bool dpm_async_fn(struct device *dev, async_func_t func)
 
 static void async_resume_noirq(void *data, async_cookie_t cookie)
 {
-	struct device *dev = (struct device *)data;
+	struct device *dev = data;
 	int error;
 
 	error = device_resume_noirq(dev, pm_transition, true);
@@ -816,7 +816,7 @@ Out:
 
 static void async_resume_early(void *data, async_cookie_t cookie)
 {
-	struct device *dev = (struct device *)data;
+	struct device *dev = data;
 	int error;
 
 	error = device_resume_early(dev, pm_transition, true);
@@ -980,7 +980,7 @@ static int device_resume(struct device *dev, pm_message_t state, bool async)
 
 static void async_resume(void *data, async_cookie_t cookie)
 {
-	struct device *dev = (struct device *)data;
+	struct device *dev = data;
 	int error;
 
 	error = device_resume(dev, pm_transition, true);
@@ -1269,7 +1269,7 @@ Complete:
 
 static void async_suspend_noirq(void *data, async_cookie_t cookie)
 {
-	struct device *dev = (struct device *)data;
+	struct device *dev = data;
 	int error;
 
 	error = __device_suspend_noirq(dev, pm_transition, true);
@@ -1450,7 +1450,7 @@ Complete:
 
 static void async_suspend_late(void *data, async_cookie_t cookie)
 {
-	struct device *dev = (struct device *)data;
+	struct device *dev = data;
 	int error;
 
 	error = __device_suspend_late(dev, pm_transition, true);
@@ -1727,7 +1727,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 
 static void async_suspend(void *data, async_cookie_t cookie)
 {
-	struct device *dev = (struct device *)data;
+	struct device *dev = data;
 	int error;
 
 	error = __device_suspend(dev, pm_transition, true);
diff --git a/drivers/base/power/wakeup_stats.c b/drivers/base/power/wakeup_stats.c
index 924fac493c4f..6732ed2869f9 100644
--- a/drivers/base/power/wakeup_stats.c
+++ b/drivers/base/power/wakeup_stats.c
@@ -210,7 +210,7 @@ void wakeup_source_sysfs_remove(struct wakeup_source *ws)
 
 static int __init wakeup_sources_sysfs_init(void)
 {
-	wakeup_class = class_create(THIS_MODULE, "wakeup");
+	wakeup_class = class_create("wakeup");
 
 	return PTR_ERR_OR_ZERO(wakeup_class);
 }
diff --git a/drivers/base/property.c b/drivers/base/property.c
index 083a95791d3b..f6117ec9805c 100644
--- a/drivers/base/property.c
+++ b/drivers/base/property.c
@@ -37,8 +37,10 @@ EXPORT_SYMBOL_GPL(__dev_fwnode_const);
  * @propname: Name of the property
  *
  * Check if property @propname is present in the device firmware description.
+ *
+ * Return: true if property @propname is present. Otherwise, returns false.
  */
-bool device_property_present(struct device *dev, const char *propname)
+bool device_property_present(const struct device *dev, const char *propname)
 {
 	return fwnode_property_present(dev_fwnode(dev), propname);
 }
@@ -48,6 +50,8 @@ EXPORT_SYMBOL_GPL(device_property_present);
  * fwnode_property_present - check if a property of a firmware node is present
  * @fwnode: Firmware node whose property to check
  * @propname: Name of the property
+ *
+ * Return: true if property @propname is present. Otherwise, returns false.
  */
 bool fwnode_property_present(const struct fwnode_handle *fwnode,
 			     const char *propname)
@@ -86,7 +90,7 @@ EXPORT_SYMBOL_GPL(fwnode_property_present);
  *	   %-EOVERFLOW if the size of the property is not as expected.
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int device_property_read_u8_array(struct device *dev, const char *propname,
+int device_property_read_u8_array(const struct device *dev, const char *propname,
 				  u8 *val, size_t nval)
 {
 	return fwnode_property_read_u8_array(dev_fwnode(dev), propname, val, nval);
@@ -114,7 +118,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u8_array);
  *	   %-EOVERFLOW if the size of the property is not as expected.
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int device_property_read_u16_array(struct device *dev, const char *propname,
+int device_property_read_u16_array(const struct device *dev, const char *propname,
 				   u16 *val, size_t nval)
 {
 	return fwnode_property_read_u16_array(dev_fwnode(dev), propname, val, nval);
@@ -142,7 +146,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u16_array);
  *	   %-EOVERFLOW if the size of the property is not as expected.
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int device_property_read_u32_array(struct device *dev, const char *propname,
+int device_property_read_u32_array(const struct device *dev, const char *propname,
 				   u32 *val, size_t nval)
 {
 	return fwnode_property_read_u32_array(dev_fwnode(dev), propname, val, nval);
@@ -170,7 +174,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u32_array);
  *	   %-EOVERFLOW if the size of the property is not as expected.
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int device_property_read_u64_array(struct device *dev, const char *propname,
+int device_property_read_u64_array(const struct device *dev, const char *propname,
 				   u64 *val, size_t nval)
 {
 	return fwnode_property_read_u64_array(dev_fwnode(dev), propname, val, nval);
@@ -198,7 +202,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u64_array);
  *	   %-EOVERFLOW if the size of the property is not as expected.
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int device_property_read_string_array(struct device *dev, const char *propname,
+int device_property_read_string_array(const struct device *dev, const char *propname,
 				      const char **val, size_t nval)
 {
 	return fwnode_property_read_string_array(dev_fwnode(dev), propname, val, nval);
@@ -220,7 +224,7 @@ EXPORT_SYMBOL_GPL(device_property_read_string_array);
  *	   %-EPROTO or %-EILSEQ if the property type is not a string.
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int device_property_read_string(struct device *dev, const char *propname,
+int device_property_read_string(const struct device *dev, const char *propname,
 				const char **val)
 {
 	return fwnode_property_read_string(dev_fwnode(dev), propname, val);
@@ -242,7 +246,7 @@ EXPORT_SYMBOL_GPL(device_property_read_string);
  *	   %-EPROTO if the property is not an array of strings,
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int device_property_match_string(struct device *dev, const char *propname,
+int device_property_match_string(const struct device *dev, const char *propname,
 				 const char *string)
 {
 	return fwnode_property_match_string(dev_fwnode(dev), propname, string);
@@ -508,10 +512,10 @@ EXPORT_SYMBOL_GPL(fwnode_property_match_string);
  * Obtain a reference based on a named property in an fwnode, with
  * integer arguments.
  *
- * Caller is responsible to call fwnode_handle_put() on the returned
- * args->fwnode pointer.
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * @args->fwnode pointer.
  *
- * Returns: %0 on success
+ * Return: %0 on success
  *	    %-ENOENT when the index is out of bounds, the index has an empty
  *		     reference or the property was not found
  *	    %-EINVAL on parse error
@@ -547,8 +551,11 @@ EXPORT_SYMBOL_GPL(fwnode_property_get_reference_args);
  *
  * @index can be used when the named reference holds a table of references.
  *
- * Returns pointer to the reference fwnode, or ERR_PTR. Caller is responsible to
- * call fwnode_handle_put() on the returned fwnode pointer.
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
+ *
+ * Return: a pointer to the reference fwnode, when found. Otherwise,
+ * returns an error pointer.
  */
 struct fwnode_handle *fwnode_find_reference(const struct fwnode_handle *fwnode,
 					    const char *name,
@@ -567,7 +574,7 @@ EXPORT_SYMBOL_GPL(fwnode_find_reference);
  * fwnode_get_name - Return the name of a node
  * @fwnode: The firmware node
  *
- * Returns a pointer to the node name.
+ * Return: a pointer to the node name, or %NULL.
  */
 const char *fwnode_get_name(const struct fwnode_handle *fwnode)
 {
@@ -579,7 +586,7 @@ EXPORT_SYMBOL_GPL(fwnode_get_name);
  * fwnode_get_name_prefix - Return the prefix of node for printing purposes
  * @fwnode: The firmware node
  *
- * Returns the prefix of a node, intended to be printed right before the node.
+ * Return: the prefix of a node, intended to be printed right before the node.
  * The prefix works also as a separator between the nodes.
  */
 const char *fwnode_get_name_prefix(const struct fwnode_handle *fwnode)
@@ -591,7 +598,10 @@ const char *fwnode_get_name_prefix(const struct fwnode_handle *fwnode)
  * fwnode_get_parent - Return parent firwmare node
  * @fwnode: Firmware whose parent is retrieved
  *
- * Return parent firmware node of the given node if possible or %NULL if no
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
+ *
+ * Return: parent firmware node of the given node if possible or %NULL if no
  * parent was available.
  */
 struct fwnode_handle *fwnode_get_parent(const struct fwnode_handle *fwnode)
@@ -608,8 +618,12 @@ EXPORT_SYMBOL_GPL(fwnode_get_parent);
  * on the passed node, making it suitable for iterating through a
  * node's parents.
  *
- * Returns a node pointer with refcount incremented, use
- * fwnode_handle_put() on it when done.
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer. Note that this function also puts a reference to @fwnode
+ * unconditionally.
+ *
+ * Return: parent firmware node of the given node if possible or %NULL if no
+ * parent was available.
  */
 struct fwnode_handle *fwnode_get_next_parent(struct fwnode_handle *fwnode)
 {
@@ -629,10 +643,12 @@ EXPORT_SYMBOL_GPL(fwnode_get_next_parent);
  * firmware node that has a corresponding struct device and returns that struct
  * device.
  *
- * The caller of this function is expected to call put_device() on the returned
- * device when they are done.
+ * The caller is responsible for calling put_device() on the returned device
+ * pointer.
+ *
+ * Return: a pointer to the device of the @fwnode's closest ancestor.
  */
-struct device *fwnode_get_next_parent_dev(struct fwnode_handle *fwnode)
+struct device *fwnode_get_next_parent_dev(const struct fwnode_handle *fwnode)
 {
 	struct fwnode_handle *parent;
 	struct device *dev;
@@ -651,7 +667,7 @@ struct device *fwnode_get_next_parent_dev(struct fwnode_handle *fwnode)
  * fwnode_count_parents - Return the number of parents a node has
  * @fwnode: The node the parents of which are to be counted
  *
- * Returns the number of parents a node has.
+ * Return: the number of parents a node has.
  */
 unsigned int fwnode_count_parents(const struct fwnode_handle *fwnode)
 {
@@ -670,12 +686,12 @@ EXPORT_SYMBOL_GPL(fwnode_count_parents);
  * @fwnode: The node the parent of which is requested
  * @depth: Distance of the parent from the node
  *
- * Returns the nth parent of a node. If there is no parent at the requested
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
+ *
+ * Return: the nth parent of a node. If there is no parent at the requested
  * @depth, %NULL is returned. If @depth is 0, the functionality is equivalent to
  * fwnode_handle_get(). For @depth == 1, it is fwnode_get_parent() and so on.
- *
- * The caller is responsible for calling fwnode_handle_put() for the returned
- * node.
  */
 struct fwnode_handle *fwnode_get_nth_parent(struct fwnode_handle *fwnode,
 					    unsigned int depth)
@@ -700,9 +716,9 @@ EXPORT_SYMBOL_GPL(fwnode_get_nth_parent);
  *
  * A node is considered an ancestor of itself too.
  *
- * Returns true if @ancestor is an ancestor of @child. Otherwise, returns false.
+ * Return: true if @ancestor is an ancestor of @child. Otherwise, returns false.
  */
-bool fwnode_is_ancestor_of(struct fwnode_handle *ancestor, struct fwnode_handle *child)
+bool fwnode_is_ancestor_of(const struct fwnode_handle *ancestor, const struct fwnode_handle *child)
 {
 	struct fwnode_handle *parent;
 
@@ -725,6 +741,10 @@ bool fwnode_is_ancestor_of(struct fwnode_handle *ancestor, struct fwnode_handle
  * fwnode_get_next_child_node - Return the next child node handle for a node
  * @fwnode: Firmware node to find the next child node for.
  * @child: Handle to one of the node's child nodes or a %NULL handle.
+ *
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer. Note that this function also puts a reference to @child
+ * unconditionally.
  */
 struct fwnode_handle *
 fwnode_get_next_child_node(const struct fwnode_handle *fwnode,
@@ -735,10 +755,13 @@ fwnode_get_next_child_node(const struct fwnode_handle *fwnode,
 EXPORT_SYMBOL_GPL(fwnode_get_next_child_node);
 
 /**
- * fwnode_get_next_available_child_node - Return the next
- * available child node handle for a node
+ * fwnode_get_next_available_child_node - Return the next available child node handle for a node
  * @fwnode: Firmware node to find the next child node for.
  * @child: Handle to one of the node's child nodes or a %NULL handle.
+ *
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer. Note that this function also puts a reference to @child
+ * unconditionally.
  */
 struct fwnode_handle *
 fwnode_get_next_available_child_node(const struct fwnode_handle *fwnode,
@@ -762,7 +785,11 @@ EXPORT_SYMBOL_GPL(fwnode_get_next_available_child_node);
 /**
  * device_get_next_child_node - Return the next child node handle for a device
  * @dev: Device to find the next child node for.
- * @child: Handle to one of the device's child nodes or a null handle.
+ * @child: Handle to one of the device's child nodes or a %NULL handle.
+ *
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer. Note that this function also puts a reference to @child
+ * unconditionally.
  */
 struct fwnode_handle *device_get_next_child_node(const struct device *dev,
 						 struct fwnode_handle *child)
@@ -787,6 +814,9 @@ EXPORT_SYMBOL_GPL(device_get_next_child_node);
  * fwnode_get_named_child_node - Return first matching named child node handle
  * @fwnode: Firmware node to find the named child node for.
  * @childname: String to match child node name against.
+ *
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
  */
 struct fwnode_handle *
 fwnode_get_named_child_node(const struct fwnode_handle *fwnode,
@@ -800,6 +830,9 @@ EXPORT_SYMBOL_GPL(fwnode_get_named_child_node);
  * device_get_named_child_node - Return first matching named child node handle
  * @dev: Device to find the named child node for.
  * @childname: String to match child node name against.
+ *
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
  */
 struct fwnode_handle *device_get_named_child_node(const struct device *dev,
 						  const char *childname)
@@ -812,7 +845,10 @@ EXPORT_SYMBOL_GPL(device_get_named_child_node);
  * fwnode_handle_get - Obtain a reference to a device node
  * @fwnode: Pointer to the device node to obtain the reference to.
  *
- * Returns the fwnode handle.
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
+ *
+ * Return: the fwnode handle.
  */
 struct fwnode_handle *fwnode_handle_get(struct fwnode_handle *fwnode)
 {
@@ -841,6 +877,8 @@ EXPORT_SYMBOL_GPL(fwnode_handle_put);
  * fwnode_device_is_available - check if a device is available for use
  * @fwnode: Pointer to the fwnode of the device.
  *
+ * Return: true if device is available for use. Otherwise, returns false.
+ *
  * For fwnode node types that don't implement the .device_is_available()
  * operation, this function returns true.
  */
@@ -859,6 +897,8 @@ EXPORT_SYMBOL_GPL(fwnode_device_is_available);
 /**
  * device_get_child_node_count - return the number of child nodes for device
  * @dev: Device to cound the child nodes for
+ *
+ * Return: the number of child nodes for a given device.
  */
 unsigned int device_get_child_node_count(const struct device *dev)
 {
@@ -895,7 +935,7 @@ EXPORT_SYMBOL_GPL(device_get_dma_attr);
  * 'phy-connection-type', and return its index in phy_modes table, or errno in
  * error case.
  */
-int fwnode_get_phy_mode(struct fwnode_handle *fwnode)
+int fwnode_get_phy_mode(const struct fwnode_handle *fwnode)
 {
 	const char *pm;
 	int err, i;
@@ -934,7 +974,7 @@ EXPORT_SYMBOL_GPL(device_get_phy_mode);
  * @fwnode:	Pointer to the firmware node
  * @index:	Index of the IO range
  *
- * Returns a pointer to the mapped memory.
+ * Return: a pointer to the mapped memory.
  */
 void __iomem *fwnode_iomap(struct fwnode_handle *fwnode, int index)
 {
@@ -947,8 +987,8 @@ EXPORT_SYMBOL(fwnode_iomap);
  * @fwnode:	Pointer to the firmware node
  * @index:	Zero-based index of the IRQ
  *
- * Returns Linux IRQ number on success. Other values are determined
- * accordingly to acpi_/of_ irq_get() operation.
+ * Return: Linux IRQ number on success. Other values are determined
+ * according to acpi_irq_get() or of_irq_get() operation.
  */
 int fwnode_irq_get(const struct fwnode_handle *fwnode, unsigned int index)
 {
@@ -967,8 +1007,7 @@ EXPORT_SYMBOL(fwnode_irq_get);
  * number of the IRQ resource corresponding to the index of the matched
  * string.
  *
- * Return:
- * Linux IRQ number on success, or negative errno otherwise.
+ * Return: Linux IRQ number on success, or negative errno otherwise.
  */
 int fwnode_irq_get_byname(const struct fwnode_handle *fwnode, const char *name)
 {
@@ -990,7 +1029,11 @@ EXPORT_SYMBOL(fwnode_irq_get_byname);
  * @fwnode: Pointer to the parent firmware node
  * @prev: Previous endpoint node or %NULL to get the first
  *
- * Returns an endpoint firmware node pointer or %NULL if no more endpoints
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer. Note that this function also puts a reference to @prev
+ * unconditionally.
+ *
+ * Return: an endpoint firmware node pointer or %NULL if no more endpoints
  * are available.
  */
 struct fwnode_handle *
@@ -1030,6 +1073,9 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_next_endpoint);
  * fwnode_graph_get_port_parent - Return the device fwnode of a port endpoint
  * @endpoint: Endpoint firmware node of the port
  *
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
+ *
  * Return: the firmware node of the device the @endpoint belongs to.
  */
 struct fwnode_handle *
@@ -1051,6 +1097,9 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_port_parent);
  * @fwnode: Endpoint firmware node pointing to the remote endpoint
  *
  * Extracts firmware node of a remote device the @fwnode points to.
+ *
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
  */
 struct fwnode_handle *
 fwnode_graph_get_remote_port_parent(const struct fwnode_handle *fwnode)
@@ -1071,6 +1120,9 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_port_parent);
  * @fwnode: Endpoint firmware node pointing to the remote endpoint
  *
  * Extracts firmware node of a remote port the @fwnode points to.
+ *
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
  */
 struct fwnode_handle *
 fwnode_graph_get_remote_port(const struct fwnode_handle *fwnode)
@@ -1084,6 +1136,9 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_port);
  * @fwnode: Endpoint firmware node pointing to the remote endpoint
  *
  * Extracts firmware node of a remote endpoint the @fwnode points to.
+ *
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
  */
 struct fwnode_handle *
 fwnode_graph_get_remote_endpoint(const struct fwnode_handle *fwnode)
@@ -1111,8 +1166,11 @@ static bool fwnode_graph_remote_available(struct fwnode_handle *ep)
  * @endpoint: identifier of the endpoint node under the port node
  * @flags: fwnode lookup flags
  *
- * Return the fwnode handle of the local endpoint corresponding the port and
- * endpoint IDs or NULL if not found.
+ * The caller is responsible for calling fwnode_handle_put() on the returned
+ * fwnode pointer.
+ *
+ * Return: the fwnode handle of the local endpoint corresponding the port and
+ * endpoint IDs or %NULL if not found.
  *
  * If FWNODE_GRAPH_ENDPOINT_NEXT is passed in @flags and the specified endpoint
  * has not been found, look for the closest endpoint ID greater than the
@@ -1120,9 +1178,6 @@ static bool fwnode_graph_remote_available(struct fwnode_handle *ep)
  *
  * Does not return endpoints that belong to disabled devices or endpoints that
  * are unconnected, unless FWNODE_GRAPH_DEVICE_DISABLED is passed in @flags.
- *
- * The returned endpoint needs to be released by calling fwnode_handle_put() on
- * it when it is not needed any more.
  */
 struct fwnode_handle *
 fwnode_graph_get_endpoint_by_id(const struct fwnode_handle *fwnode,
@@ -1180,7 +1235,7 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_endpoint_by_id);
  * If FWNODE_GRAPH_DEVICE_DISABLED flag is specified, also unconnected endpoints
  * and endpoints connected to disabled devices are counted.
  */
-unsigned int fwnode_graph_get_endpoint_count(struct fwnode_handle *fwnode,
+unsigned int fwnode_graph_get_endpoint_count(const struct fwnode_handle *fwnode,
 					     unsigned long flags)
 {
 	struct fwnode_handle *ep;
@@ -1328,7 +1383,8 @@ EXPORT_SYMBOL_GPL(fwnode_connection_find_match);
  * @fwnode and other device nodes. @match will be used to convert the
  * connection description to data the caller is expecting to be returned
  * through the @matches array.
- * If @matches is NULL @matches_len is ignored and the total number of resolved
+ *
+ * If @matches is %NULL @matches_len is ignored and the total number of resolved
  * matches is returned.
  *
  * Return: Number of matches resolved, or negative errno.
diff --git a/drivers/base/regmap/Kconfig b/drivers/base/regmap/Kconfig
index cd4bb642b9de..33a8366e22a5 100644
--- a/drivers/base/regmap/Kconfig
+++ b/drivers/base/regmap/Kconfig
@@ -9,10 +9,12 @@ config REGMAP
 	select MDIO_BUS if REGMAP_MDIO
 	bool
 
-config REGCACHE_COMPRESSED
-	select LZO_COMPRESS
-	select LZO_DECOMPRESS
-	bool
+config REGMAP_KUNIT
+	tristate "KUnit tests for regmap"
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	select REGMAP
+	select REGMAP_RAM
 
 config REGMAP_AC97
 	tristate
@@ -46,6 +48,9 @@ config REGMAP_MMIO
 config REGMAP_IRQ
 	bool
 
+config REGMAP_RAM
+	tristate
+
 config REGMAP_SOUNDWIRE
 	tristate
 	depends on SOUNDWIRE
diff --git a/drivers/base/regmap/Makefile b/drivers/base/regmap/Makefile
index 6990de7ca9a9..f6c6cb017200 100644
--- a/drivers/base/regmap/Makefile
+++ b/drivers/base/regmap/Makefile
@@ -3,11 +3,12 @@
 CFLAGS_regmap.o := -I$(src)
 
 obj-$(CONFIG_REGMAP) += regmap.o regcache.o
-obj-$(CONFIG_REGMAP) += regcache-rbtree.o regcache-flat.o
-obj-$(CONFIG_REGCACHE_COMPRESSED) += regcache-lzo.o
+obj-$(CONFIG_REGMAP) += regcache-rbtree.o regcache-flat.o regcache-maple.o
 obj-$(CONFIG_DEBUG_FS) += regmap-debugfs.o
+obj-$(CONFIG_REGMAP_KUNIT) += regmap-kunit.o
 obj-$(CONFIG_REGMAP_AC97) += regmap-ac97.o
 obj-$(CONFIG_REGMAP_I2C) += regmap-i2c.o
+obj-$(CONFIG_REGMAP_RAM) += regmap-ram.o
 obj-$(CONFIG_REGMAP_SLIMBUS) += regmap-slimbus.o
 obj-$(CONFIG_REGMAP_SPI) += regmap-spi.o
 obj-$(CONFIG_REGMAP_SPMI) += regmap-spmi.o
diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h
index da8996e7a1f1..9bd0dfd1e259 100644
--- a/drivers/base/regmap/internal.h
+++ b/drivers/base/regmap/internal.h
@@ -31,8 +31,8 @@ struct regmap_format {
 	size_t buf_size;
 	size_t reg_bytes;
 	size_t pad_bytes;
-	size_t reg_downshift;
 	size_t val_bytes;
+	s8 reg_shift;
 	void (*format_write)(struct regmap *map,
 			     unsigned int reg, unsigned int val);
 	void (*format_reg)(void *buf, unsigned int reg, unsigned int shift);
@@ -270,6 +270,7 @@ unsigned int regcache_get_val(struct regmap *map, const void *base,
 bool regcache_set_val(struct regmap *map, void *base, unsigned int idx,
 		      unsigned int val);
 int regcache_lookup_reg(struct regmap *map, unsigned int reg);
+int regcache_sync_val(struct regmap *map, unsigned int reg, unsigned int val);
 
 int _regmap_raw_write(struct regmap *map, unsigned int reg,
 		      const void *val, size_t val_len, bool noinc);
@@ -281,7 +282,7 @@ enum regmap_endian regmap_get_val_endian(struct device *dev,
 					 const struct regmap_config *config);
 
 extern struct regcache_ops regcache_rbtree_ops;
-extern struct regcache_ops regcache_lzo_ops;
+extern struct regcache_ops regcache_maple_ops;
 extern struct regcache_ops regcache_flat_ops;
 
 static inline const char *regmap_name(const struct regmap *map)
@@ -307,4 +308,23 @@ static inline unsigned int regcache_get_index_by_order(const struct regmap *map,
 	return reg >> map->reg_stride_order;
 }
 
+struct regmap_ram_data {
+	unsigned int *vals;  /* Allocatd by caller */
+	bool *read;
+	bool *written;
+};
+
+/*
+ * Create a test register map with data stored in RAM, not intended
+ * for practical use.
+ */
+struct regmap *__regmap_init_ram(const struct regmap_config *config,
+				 struct regmap_ram_data *data,
+				 struct lock_class_key *lock_key,
+				 const char *lock_name);
+
+#define regmap_init_ram(config, data)					\
+	__regmap_lockdep_wrapper(__regmap_init_ram, #config, config, data)
+
+
 #endif
diff --git a/drivers/base/regmap/regcache-lzo.c b/drivers/base/regmap/regcache-lzo.c
deleted file mode 100644
index 7886303eb026..000000000000
--- a/drivers/base/regmap/regcache-lzo.c
+++ /dev/null
@@ -1,368 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-//
-// Register cache access API - LZO caching support
-//
-// Copyright 2011 Wolfson Microelectronics plc
-//
-// Author: Dimitris Papastamos <dp@opensource.wolfsonmicro.com>
-
-#include <linux/device.h>
-#include <linux/lzo.h>
-#include <linux/slab.h>
-
-#include "internal.h"
-
-static int regcache_lzo_exit(struct regmap *map);
-
-struct regcache_lzo_ctx {
-	void *wmem;
-	void *dst;
-	const void *src;
-	size_t src_len;
-	size_t dst_len;
-	size_t decompressed_size;
-	unsigned long *sync_bmp;
-	int sync_bmp_nbits;
-};
-
-#define LZO_BLOCK_NUM 8
-static int regcache_lzo_block_count(struct regmap *map)
-{
-	return LZO_BLOCK_NUM;
-}
-
-static int regcache_lzo_prepare(struct regcache_lzo_ctx *lzo_ctx)
-{
-	lzo_ctx->wmem = kmalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL);
-	if (!lzo_ctx->wmem)
-		return -ENOMEM;
-	return 0;
-}
-
-static int regcache_lzo_compress(struct regcache_lzo_ctx *lzo_ctx)
-{
-	size_t compress_size;
-	int ret;
-
-	ret = lzo1x_1_compress(lzo_ctx->src, lzo_ctx->src_len,
-			       lzo_ctx->dst, &compress_size, lzo_ctx->wmem);
-	if (ret != LZO_E_OK || compress_size > lzo_ctx->dst_len)
-		return -EINVAL;
-	lzo_ctx->dst_len = compress_size;
-	return 0;
-}
-
-static int regcache_lzo_decompress(struct regcache_lzo_ctx *lzo_ctx)
-{
-	size_t dst_len;
-	int ret;
-
-	dst_len = lzo_ctx->dst_len;
-	ret = lzo1x_decompress_safe(lzo_ctx->src, lzo_ctx->src_len,
-				    lzo_ctx->dst, &dst_len);
-	if (ret != LZO_E_OK || dst_len != lzo_ctx->dst_len)
-		return -EINVAL;
-	return 0;
-}
-
-static int regcache_lzo_compress_cache_block(struct regmap *map,
-		struct regcache_lzo_ctx *lzo_ctx)
-{
-	int ret;
-
-	lzo_ctx->dst_len = lzo1x_worst_compress(PAGE_SIZE);
-	lzo_ctx->dst = kmalloc(lzo_ctx->dst_len, GFP_KERNEL);
-	if (!lzo_ctx->dst) {
-		lzo_ctx->dst_len = 0;
-		return -ENOMEM;
-	}
-
-	ret = regcache_lzo_compress(lzo_ctx);
-	if (ret < 0)
-		return ret;
-	return 0;
-}
-
-static int regcache_lzo_decompress_cache_block(struct regmap *map,
-		struct regcache_lzo_ctx *lzo_ctx)
-{
-	int ret;
-
-	lzo_ctx->dst_len = lzo_ctx->decompressed_size;
-	lzo_ctx->dst = kmalloc(lzo_ctx->dst_len, GFP_KERNEL);
-	if (!lzo_ctx->dst) {
-		lzo_ctx->dst_len = 0;
-		return -ENOMEM;
-	}
-
-	ret = regcache_lzo_decompress(lzo_ctx);
-	if (ret < 0)
-		return ret;
-	return 0;
-}
-
-static inline int regcache_lzo_get_blkindex(struct regmap *map,
-					    unsigned int reg)
-{
-	return ((reg / map->reg_stride) * map->cache_word_size) /
-		DIV_ROUND_UP(map->cache_size_raw,
-			     regcache_lzo_block_count(map));
-}
-
-static inline int regcache_lzo_get_blkpos(struct regmap *map,
-					  unsigned int reg)
-{
-	return (reg / map->reg_stride) %
-		    (DIV_ROUND_UP(map->cache_size_raw,
-				  regcache_lzo_block_count(map)) /
-		     map->cache_word_size);
-}
-
-static inline int regcache_lzo_get_blksize(struct regmap *map)
-{
-	return DIV_ROUND_UP(map->cache_size_raw,
-			    regcache_lzo_block_count(map));
-}
-
-static int regcache_lzo_init(struct regmap *map)
-{
-	struct regcache_lzo_ctx **lzo_blocks;
-	size_t bmp_size;
-	int ret, i, blksize, blkcount;
-	const char *p, *end;
-	unsigned long *sync_bmp;
-
-	ret = 0;
-
-	blkcount = regcache_lzo_block_count(map);
-	map->cache = kcalloc(blkcount, sizeof(*lzo_blocks),
-			     GFP_KERNEL);
-	if (!map->cache)
-		return -ENOMEM;
-	lzo_blocks = map->cache;
-
-	/*
-	 * allocate a bitmap to be used when syncing the cache with
-	 * the hardware.  Each time a register is modified, the corresponding
-	 * bit is set in the bitmap, so we know that we have to sync
-	 * that register.
-	 */
-	bmp_size = map->num_reg_defaults_raw;
-	sync_bmp = bitmap_zalloc(bmp_size, GFP_KERNEL);
-	if (!sync_bmp) {
-		ret = -ENOMEM;
-		goto err;
-	}
-
-	/* allocate the lzo blocks and initialize them */
-	for (i = 0; i < blkcount; i++) {
-		lzo_blocks[i] = kzalloc(sizeof **lzo_blocks,
-					GFP_KERNEL);
-		if (!lzo_blocks[i]) {
-			bitmap_free(sync_bmp);
-			ret = -ENOMEM;
-			goto err;
-		}
-		lzo_blocks[i]->sync_bmp = sync_bmp;
-		lzo_blocks[i]->sync_bmp_nbits = bmp_size;
-		/* alloc the working space for the compressed block */
-		ret = regcache_lzo_prepare(lzo_blocks[i]);
-		if (ret < 0)
-			goto err;
-	}
-
-	blksize = regcache_lzo_get_blksize(map);
-	p = map->reg_defaults_raw;
-	end = map->reg_defaults_raw + map->cache_size_raw;
-	/* compress the register map and fill the lzo blocks */
-	for (i = 0; i < blkcount; i++, p += blksize) {
-		lzo_blocks[i]->src = p;
-		if (p + blksize > end)
-			lzo_blocks[i]->src_len = end - p;
-		else
-			lzo_blocks[i]->src_len = blksize;
-		ret = regcache_lzo_compress_cache_block(map,
-						       lzo_blocks[i]);
-		if (ret < 0)
-			goto err;
-		lzo_blocks[i]->decompressed_size =
-			lzo_blocks[i]->src_len;
-	}
-
-	return 0;
-err:
-	regcache_lzo_exit(map);
-	return ret;
-}
-
-static int regcache_lzo_exit(struct regmap *map)
-{
-	struct regcache_lzo_ctx **lzo_blocks;
-	int i, blkcount;
-
-	lzo_blocks = map->cache;
-	if (!lzo_blocks)
-		return 0;
-
-	blkcount = regcache_lzo_block_count(map);
-	/*
-	 * the pointer to the bitmap used for syncing the cache
-	 * is shared amongst all lzo_blocks.  Ensure it is freed
-	 * only once.
-	 */
-	if (lzo_blocks[0])
-		bitmap_free(lzo_blocks[0]->sync_bmp);
-	for (i = 0; i < blkcount; i++) {
-		if (lzo_blocks[i]) {
-			kfree(lzo_blocks[i]->wmem);
-			kfree(lzo_blocks[i]->dst);
-		}
-		/* each lzo_block is a pointer returned by kmalloc or NULL */
-		kfree(lzo_blocks[i]);
-	}
-	kfree(lzo_blocks);
-	map->cache = NULL;
-	return 0;
-}
-
-static int regcache_lzo_read(struct regmap *map,
-			     unsigned int reg, unsigned int *value)
-{
-	struct regcache_lzo_ctx *lzo_block, **lzo_blocks;
-	int ret, blkindex, blkpos;
-	size_t tmp_dst_len;
-	void *tmp_dst;
-
-	/* index of the compressed lzo block */
-	blkindex = regcache_lzo_get_blkindex(map, reg);
-	/* register index within the decompressed block */
-	blkpos = regcache_lzo_get_blkpos(map, reg);
-	lzo_blocks = map->cache;
-	lzo_block = lzo_blocks[blkindex];
-
-	/* save the pointer and length of the compressed block */
-	tmp_dst = lzo_block->dst;
-	tmp_dst_len = lzo_block->dst_len;
-
-	/* prepare the source to be the compressed block */
-	lzo_block->src = lzo_block->dst;
-	lzo_block->src_len = lzo_block->dst_len;
-
-	/* decompress the block */
-	ret = regcache_lzo_decompress_cache_block(map, lzo_block);
-	if (ret >= 0)
-		/* fetch the value from the cache */
-		*value = regcache_get_val(map, lzo_block->dst, blkpos);
-
-	kfree(lzo_block->dst);
-	/* restore the pointer and length of the compressed block */
-	lzo_block->dst = tmp_dst;
-	lzo_block->dst_len = tmp_dst_len;
-
-	return ret;
-}
-
-static int regcache_lzo_write(struct regmap *map,
-			      unsigned int reg, unsigned int value)
-{
-	struct regcache_lzo_ctx *lzo_block, **lzo_blocks;
-	int ret, blkindex, blkpos;
-	size_t tmp_dst_len;
-	void *tmp_dst;
-
-	/* index of the compressed lzo block */
-	blkindex = regcache_lzo_get_blkindex(map, reg);
-	/* register index within the decompressed block */
-	blkpos = regcache_lzo_get_blkpos(map, reg);
-	lzo_blocks = map->cache;
-	lzo_block = lzo_blocks[blkindex];
-
-	/* save the pointer and length of the compressed block */
-	tmp_dst = lzo_block->dst;
-	tmp_dst_len = lzo_block->dst_len;
-
-	/* prepare the source to be the compressed block */
-	lzo_block->src = lzo_block->dst;
-	lzo_block->src_len = lzo_block->dst_len;
-
-	/* decompress the block */
-	ret = regcache_lzo_decompress_cache_block(map, lzo_block);
-	if (ret < 0) {
-		kfree(lzo_block->dst);
-		goto out;
-	}
-
-	/* write the new value to the cache */
-	if (regcache_set_val(map, lzo_block->dst, blkpos, value)) {
-		kfree(lzo_block->dst);
-		goto out;
-	}
-
-	/* prepare the source to be the decompressed block */
-	lzo_block->src = lzo_block->dst;
-	lzo_block->src_len = lzo_block->dst_len;
-
-	/* compress the block */
-	ret = regcache_lzo_compress_cache_block(map, lzo_block);
-	if (ret < 0) {
-		kfree(lzo_block->dst);
-		kfree(lzo_block->src);
-		goto out;
-	}
-
-	/* set the bit so we know we have to sync this register */
-	set_bit(reg / map->reg_stride, lzo_block->sync_bmp);
-	kfree(tmp_dst);
-	kfree(lzo_block->src);
-	return 0;
-out:
-	lzo_block->dst = tmp_dst;
-	lzo_block->dst_len = tmp_dst_len;
-	return ret;
-}
-
-static int regcache_lzo_sync(struct regmap *map, unsigned int min,
-			     unsigned int max)
-{
-	struct regcache_lzo_ctx **lzo_blocks;
-	unsigned int val;
-	int i;
-	int ret;
-
-	lzo_blocks = map->cache;
-	i = min;
-	for_each_set_bit_from(i, lzo_blocks[0]->sync_bmp,
-			      lzo_blocks[0]->sync_bmp_nbits) {
-		if (i > max)
-			continue;
-
-		ret = regcache_read(map, i, &val);
-		if (ret)
-			return ret;
-
-		/* Is this the hardware default?  If so skip. */
-		ret = regcache_lookup_reg(map, i);
-		if (ret > 0 && val == map->reg_defaults[ret].def)
-			continue;
-
-		map->cache_bypass = true;
-		ret = _regmap_write(map, i, val);
-		map->cache_bypass = false;
-		if (ret)
-			return ret;
-		dev_dbg(map->dev, "Synced register %#x, value %#x\n",
-			i, val);
-	}
-
-	return 0;
-}
-
-struct regcache_ops regcache_lzo_ops = {
-	.type = REGCACHE_COMPRESSED,
-	.name = "lzo",
-	.init = regcache_lzo_init,
-	.exit = regcache_lzo_exit,
-	.read = regcache_lzo_read,
-	.write = regcache_lzo_write,
-	.sync = regcache_lzo_sync
-};
diff --git a/drivers/base/regmap/regcache-maple.c b/drivers/base/regmap/regcache-maple.c
new file mode 100644
index 000000000000..9b1b559107ef
--- /dev/null
+++ b/drivers/base/regmap/regcache-maple.c
@@ -0,0 +1,279 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Register cache access API - maple tree based cache
+//
+// Copyright 2023 Arm, Ltd
+//
+// Author: Mark Brown <broonie@kernel.org>
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/maple_tree.h>
+#include <linux/slab.h>
+
+#include "internal.h"
+
+static int regcache_maple_read(struct regmap *map,
+			       unsigned int reg, unsigned int *value)
+{
+	struct maple_tree *mt = map->cache;
+	MA_STATE(mas, mt, reg, reg);
+	unsigned long *entry;
+
+	rcu_read_lock();
+
+	entry = mas_walk(&mas);
+	if (!entry) {
+		rcu_read_unlock();
+		return -ENOENT;
+	}
+
+	*value = entry[reg - mas.index];
+
+	rcu_read_unlock();
+
+	return 0;
+}
+
+static int regcache_maple_write(struct regmap *map, unsigned int reg,
+				unsigned int val)
+{
+	struct maple_tree *mt = map->cache;
+	MA_STATE(mas, mt, reg, reg);
+	unsigned long *entry, *upper, *lower;
+	unsigned long index, last;
+	size_t lower_sz, upper_sz;
+	int ret;
+
+	rcu_read_lock();
+
+	entry = mas_walk(&mas);
+	if (entry) {
+		entry[reg - mas.index] = val;
+		rcu_read_unlock();
+		return 0;
+	}
+
+	/* Any adjacent entries to extend/merge? */
+	mas_set_range(&mas, reg - 1, reg + 1);
+	index = reg;
+	last = reg;
+
+	lower = mas_find(&mas, reg - 1);
+	if (lower) {
+		index = mas.index;
+		lower_sz = (mas.last - mas.index + 1) * sizeof(unsigned long);
+	}
+
+	upper = mas_find(&mas, reg + 1);
+	if (upper) {
+		last = mas.last;
+		upper_sz = (mas.last - mas.index + 1) * sizeof(unsigned long);
+	}
+
+	rcu_read_unlock();
+
+	entry = kmalloc((last - index + 1) * sizeof(unsigned long),
+			GFP_KERNEL);
+	if (!entry)
+		return -ENOMEM;
+
+	if (lower)
+		memcpy(entry, lower, lower_sz);
+	entry[reg - index] = val;
+	if (upper)
+		memcpy(&entry[reg - index + 1], upper, upper_sz);
+
+	/*
+	 * This is safe because the regmap lock means the Maple lock
+	 * is redundant, but we need to take it due to lockdep asserts
+	 * in the maple tree code.
+	 */
+	mas_lock(&mas);
+
+	mas_set_range(&mas, index, last);
+	ret = mas_store_gfp(&mas, entry, GFP_KERNEL);
+
+	mas_unlock(&mas);
+
+	if (ret == 0) {
+		kfree(lower);
+		kfree(upper);
+	}
+	
+	return ret;
+}
+
+static int regcache_maple_drop(struct regmap *map, unsigned int min,
+			       unsigned int max)
+{
+	struct maple_tree *mt = map->cache;
+	MA_STATE(mas, mt, min, max);
+	unsigned long *entry, *lower, *upper;
+	unsigned long lower_index, lower_last;
+	unsigned long upper_index, upper_last;
+	int ret;
+
+	lower = NULL;
+	upper = NULL;
+
+	mas_lock(&mas);
+
+	mas_for_each(&mas, entry, max) {
+		/*
+		 * This is safe because the regmap lock means the
+		 * Maple lock is redundant, but we need to take it due
+		 * to lockdep asserts in the maple tree code.
+		 */
+		mas_unlock(&mas);
+
+		/* Do we need to save any of this entry? */
+		if (mas.index < min) {
+			lower_index = mas.index;
+			lower_last = min -1;
+
+			lower = kmemdup(entry, ((min - mas.index) *
+						sizeof(unsigned long)),
+					GFP_KERNEL);
+			if (!lower) {
+				ret = -ENOMEM;
+				goto out_unlocked;
+			}
+		}
+
+		if (mas.last > max) {
+			upper_index = max + 1;
+			upper_last = mas.last;
+
+			upper = kmemdup(&entry[max + 1],
+					((mas.last - max) *
+					 sizeof(unsigned long)),
+					GFP_KERNEL);
+			if (!upper) {
+				ret = -ENOMEM;
+				goto out_unlocked;
+			}
+		}
+
+		kfree(entry);
+		mas_lock(&mas);
+		mas_erase(&mas);
+
+		/* Insert new nodes with the saved data */
+		if (lower) {
+			mas_set_range(&mas, lower_index, lower_last);
+			ret = mas_store_gfp(&mas, lower, GFP_KERNEL);
+			if (ret != 0)
+				goto out;
+			lower = NULL;
+		}
+
+		if (upper) {
+			mas_set_range(&mas, upper_index, upper_last);
+			ret = mas_store_gfp(&mas, upper, GFP_KERNEL);
+			if (ret != 0)
+				goto out;
+			upper = NULL;
+		}
+	}
+
+out:
+	mas_unlock(&mas);
+out_unlocked:
+	kfree(lower);
+	kfree(upper);
+
+	return ret;
+}
+
+static int regcache_maple_sync(struct regmap *map, unsigned int min,
+			       unsigned int max)
+{
+	struct maple_tree *mt = map->cache;
+	unsigned long *entry;
+	MA_STATE(mas, mt, min, max);
+	unsigned long lmin = min;
+	unsigned long lmax = max;
+	unsigned int r;
+	int ret;
+
+	map->cache_bypass = true;
+
+	rcu_read_lock();
+
+	mas_for_each(&mas, entry, max) {
+		for (r = max(mas.index, lmin); r <= min(mas.last, lmax); r++) {
+			ret = regcache_sync_val(map, r, entry[r - mas.index]);
+			if (ret != 0)
+				goto out;
+		}
+	}
+
+out:
+	rcu_read_unlock();
+
+	map->cache_bypass = false;
+
+	return ret;
+}
+
+static int regcache_maple_exit(struct regmap *map)
+{
+	struct maple_tree *mt = map->cache;
+	MA_STATE(mas, mt, 0, UINT_MAX);
+	unsigned int *entry;;
+
+	/* if we've already been called then just return */
+	if (!mt)
+		return 0;
+
+	mas_lock(&mas);
+	mas_for_each(&mas, entry, UINT_MAX)
+		kfree(entry);
+	__mt_destroy(mt);
+	mas_unlock(&mas);
+
+	kfree(mt);
+	map->cache = NULL;
+
+	return 0;
+}
+
+static int regcache_maple_init(struct regmap *map)
+{
+	struct maple_tree *mt;
+	int i;
+	int ret;
+
+	mt = kmalloc(sizeof(*mt), GFP_KERNEL);
+	if (!mt)
+		return -ENOMEM;
+	map->cache = mt;
+
+	mt_init(mt);
+
+	for (i = 0; i < map->num_reg_defaults; i++) {
+		ret = regcache_maple_write(map,
+					   map->reg_defaults[i].reg,
+					   map->reg_defaults[i].def);
+		if (ret)
+			goto err;
+	}
+
+	return 0;
+
+err:
+	regcache_maple_exit(map);
+	return ret;
+}
+
+struct regcache_ops regcache_maple_ops = {
+	.type = REGCACHE_MAPLE,
+	.name = "maple",
+	.init = regcache_maple_init,
+	.exit = regcache_maple_exit,
+	.read = regcache_maple_read,
+	.write = regcache_maple_write,
+	.drop = regcache_maple_drop,
+	.sync = regcache_maple_sync,
+};
diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c
index 362e043e26d8..029564695dbb 100644
--- a/drivers/base/regmap/regcache.c
+++ b/drivers/base/regmap/regcache.c
@@ -17,9 +17,7 @@
 
 static const struct regcache_ops *cache_types[] = {
 	&regcache_rbtree_ops,
-#if IS_ENABLED(CONFIG_REGCACHE_COMPRESSED)
-	&regcache_lzo_ops,
-#endif
+	&regcache_maple_ops,
 	&regcache_flat_ops,
 };
 
@@ -148,7 +146,7 @@ int regcache_init(struct regmap *map, const struct regmap_config *config)
 			break;
 
 	if (i == ARRAY_SIZE(cache_types)) {
-		dev_err(map->dev, "Could not match compress type: %d\n",
+		dev_err(map->dev, "Could not match cache type: %d\n",
 			map->cache_type);
 		return -EINVAL;
 	}
@@ -242,7 +240,7 @@ int regcache_read(struct regmap *map,
 	int ret;
 
 	if (map->cache_type == REGCACHE_NONE)
-		return -ENOSYS;
+		return -EINVAL;
 
 	BUG_ON(!map->cache_ops);
 
@@ -311,6 +309,8 @@ static int regcache_default_sync(struct regmap *map, unsigned int min,
 			continue;
 
 		ret = regcache_read(map, reg, &val);
+		if (ret == -ENOENT)
+			continue;
 		if (ret)
 			return ret;
 
@@ -349,6 +349,9 @@ int regcache_sync(struct regmap *map)
 	const char *name;
 	bool bypass;
 
+	if (WARN_ON(map->cache_type == REGCACHE_NONE))
+		return -EINVAL;
+
 	BUG_ON(!map->cache_ops);
 
 	map->lock(map->lock_arg);
@@ -418,6 +421,9 @@ int regcache_sync_region(struct regmap *map, unsigned int min,
 	const char *name;
 	bool bypass;
 
+	if (WARN_ON(map->cache_type == REGCACHE_NONE))
+		return -EINVAL;
+
 	BUG_ON(!map->cache_ops);
 
 	map->lock(map->lock_arg);
@@ -672,6 +678,30 @@ static bool regcache_reg_present(unsigned long *cache_present, unsigned int idx)
 	return test_bit(idx, cache_present);
 }
 
+int regcache_sync_val(struct regmap *map, unsigned int reg, unsigned int val)
+{
+	int ret;
+
+	if (!regcache_reg_needs_sync(map, reg, val))
+		return 0;
+
+	map->cache_bypass = true;
+
+	ret = _regmap_write(map, reg, val);
+
+	map->cache_bypass = false;
+
+	if (ret != 0) {
+		dev_err(map->dev, "Unable to sync register %#x. %d\n",
+			reg, ret);
+		return ret;
+	}
+	dev_dbg(map->dev, "Synced register %#x, value %#x\n",
+		reg, val);
+
+	return 0;
+}
+
 static int regcache_sync_block_single(struct regmap *map, void *block,
 				      unsigned long *cache_present,
 				      unsigned int block_base,
@@ -688,21 +718,9 @@ static int regcache_sync_block_single(struct regmap *map, void *block,
 			continue;
 
 		val = regcache_get_val(map, block, i);
-		if (!regcache_reg_needs_sync(map, regtmp, val))
-			continue;
-
-		map->cache_bypass = true;
-
-		ret = _regmap_write(map, regtmp, val);
-
-		map->cache_bypass = false;
-		if (ret != 0) {
-			dev_err(map->dev, "Unable to sync register %#x. %d\n",
-				regtmp, ret);
+		ret = regcache_sync_val(map, regtmp, val);
+		if (ret != 0)
 			return ret;
-		}
-		dev_dbg(map->dev, "Synced register %#x, value %#x\n",
-			regtmp, val);
 	}
 
 	return 0;
diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c
index 8c903b8c9714..b99bb2369fff 100644
--- a/drivers/base/regmap/regmap-irq.c
+++ b/drivers/base/regmap/regmap-irq.c
@@ -329,8 +329,8 @@ static int regmap_irq_set_type(struct irq_data *data, unsigned int type)
 	}
 
 	if (d->chip->set_type_config) {
-		ret = d->chip->set_type_config(d->config_buf, type,
-					       irq_data, reg);
+		ret = d->chip->set_type_config(d->config_buf, type, irq_data,
+					       reg, d->chip->irq_drv_data);
 		if (ret)
 			return ret;
 	}
@@ -433,7 +433,10 @@ static irqreturn_t regmap_irq_thread(int irq, void *d)
 	 * possible in order to reduce the I/O overheads.
 	 */
 
-	if (chip->num_main_regs) {
+	if (chip->no_status) {
+		/* no status register so default to all active */
+		memset32(data->status_buf, GENMASK(31, 0), chip->num_regs);
+	} else if (chip->num_main_regs) {
 		unsigned int max_main_bits;
 		unsigned long size;
 
@@ -651,13 +654,15 @@ EXPORT_SYMBOL_GPL(regmap_irq_get_irq_reg_linear);
  * @type: The requested IRQ type.
  * @irq_data: The IRQ being configured.
  * @idx: Index of the irq's config registers within each array `buf[i]`
+ * @irq_drv_data: Driver specific IRQ data
  *
  * This is a &struct regmap_irq_chip->set_type_config callback suitable for
  * chips with one config register. Register values are updated according to
  * the &struct regmap_irq_type data associated with an IRQ.
  */
 int regmap_irq_set_type_config_simple(unsigned int **buf, unsigned int type,
-				      const struct regmap_irq *irq_data, int idx)
+				      const struct regmap_irq *irq_data,
+				      int idx, void *irq_drv_data)
 {
 	const struct regmap_irq_type *t = &irq_data->type;
 
@@ -949,12 +954,17 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode,
 			continue;
 
 		/* Ack masked but set interrupts */
-		reg = d->get_irq_reg(d, d->chip->status_base, i);
-		ret = regmap_read(map, reg, &d->status_buf[i]);
-		if (ret != 0) {
-			dev_err(map->dev, "Failed to read IRQ status: %d\n",
-				ret);
-			goto err_alloc;
+		if (d->chip->no_status) {
+			/* no status register so default to all active */
+			d->status_buf[i] = GENMASK(31, 0);
+		} else {
+			reg = d->get_irq_reg(d, d->chip->status_base, i);
+			ret = regmap_read(map, reg, &d->status_buf[i]);
+			if (ret != 0) {
+				dev_err(map->dev, "Failed to read IRQ status: %d\n",
+					ret);
+				goto err_alloc;
+			}
 		}
 
 		if (chip->status_invert)
diff --git a/drivers/base/regmap/regmap-kunit.c b/drivers/base/regmap/regmap-kunit.c
new file mode 100644
index 000000000000..f76d41688134
--- /dev/null
+++ b/drivers/base/regmap/regmap-kunit.c
@@ -0,0 +1,739 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// regmap KUnit tests
+//
+// Copyright 2023 Arm Ltd
+
+#include <kunit/test.h>
+#include "internal.h"
+
+#define BLOCK_TEST_SIZE 12
+
+static const struct regmap_config test_regmap_config = {
+	.max_register = BLOCK_TEST_SIZE,
+	.reg_stride = 1,
+	.val_bits = sizeof(unsigned int) * 8,
+};
+
+struct regcache_types {
+	enum regcache_type type;
+	const char *name;
+};
+
+static void case_to_desc(const struct regcache_types *t, char *desc)
+{
+	strcpy(desc, t->name);
+}
+
+static const struct regcache_types regcache_types_list[] = {
+	{ REGCACHE_NONE, "none" },
+	{ REGCACHE_FLAT, "flat" },
+	{ REGCACHE_RBTREE, "rbtree" },
+	{ REGCACHE_MAPLE, "maple" },
+};
+
+KUNIT_ARRAY_PARAM(regcache_types, regcache_types_list, case_to_desc);
+
+static const struct regcache_types real_cache_types_list[] = {
+	{ REGCACHE_FLAT, "flat" },
+	{ REGCACHE_RBTREE, "rbtree" },
+	{ REGCACHE_MAPLE, "maple" },
+};
+
+KUNIT_ARRAY_PARAM(real_cache_types, real_cache_types_list, case_to_desc);
+
+static const struct regcache_types sparse_cache_types_list[] = {
+	{ REGCACHE_RBTREE, "rbtree" },
+	{ REGCACHE_MAPLE, "maple" },
+};
+
+KUNIT_ARRAY_PARAM(sparse_cache_types, sparse_cache_types_list, case_to_desc);
+
+static struct regmap *gen_regmap(struct regmap_config *config,
+				 struct regmap_ram_data **data)
+{
+	unsigned int *buf;
+	struct regmap *ret;
+	size_t size = (config->max_register + 1) * sizeof(unsigned int);
+	int i;
+	struct reg_default *defaults;
+
+	buf = kmalloc(size, GFP_KERNEL);
+	if (!buf)
+		return ERR_PTR(-ENOMEM);
+
+	get_random_bytes(buf, size);
+
+	*data = kzalloc(sizeof(**data), GFP_KERNEL);
+	if (!(*data))
+		return ERR_PTR(-ENOMEM);
+	(*data)->vals = buf;
+
+	if (config->num_reg_defaults) {
+		defaults = kcalloc(config->num_reg_defaults,
+				   sizeof(struct reg_default),
+				   GFP_KERNEL);
+		if (!defaults)
+			return ERR_PTR(-ENOMEM);
+		config->reg_defaults = defaults;
+
+		for (i = 0; i < config->num_reg_defaults; i++) {
+			defaults[i].reg = i * config->reg_stride;
+			defaults[i].def = buf[i * config->reg_stride];
+		}
+	}
+
+	ret = regmap_init_ram(config, *data);
+	if (IS_ERR(ret)) {
+		kfree(buf);
+		kfree(*data);
+	}
+
+	return ret;
+}
+
+static void basic_read_write(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int val, rval;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	get_random_bytes(&val, sizeof(val));
+
+	/* If we write a value to a register we can read it back */
+	KUNIT_EXPECT_EQ(test, 0, regmap_write(map, 0, val));
+	KUNIT_EXPECT_EQ(test, 0, regmap_read(map, 0, &rval));
+	KUNIT_EXPECT_EQ(test, val, rval);
+
+	/* If using a cache the cache satisfied the read */
+	KUNIT_EXPECT_EQ(test, t->type == REGCACHE_NONE, data->read[0]);
+
+	regmap_exit(map);
+}
+
+static void bulk_write(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int val[BLOCK_TEST_SIZE], rval[BLOCK_TEST_SIZE];
+	int i;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	get_random_bytes(&val, sizeof(val));
+
+	/*
+	 * Data written via the bulk API can be read back with single
+	 * reads.
+	 */
+	KUNIT_EXPECT_EQ(test, 0, regmap_bulk_write(map, 0, val,
+						   BLOCK_TEST_SIZE));
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &rval[i]));
+
+	KUNIT_EXPECT_MEMEQ(test, val, rval, sizeof(val));
+
+	/* If using a cache the cache satisfied the read */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, t->type == REGCACHE_NONE, data->read[i]);
+
+	regmap_exit(map);
+}
+
+static void bulk_read(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int val[BLOCK_TEST_SIZE], rval[BLOCK_TEST_SIZE];
+	int i;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	get_random_bytes(&val, sizeof(val));
+
+	/* Data written as single writes can be read via the bulk API */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, val[i]));
+	KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval,
+						  BLOCK_TEST_SIZE));
+	KUNIT_EXPECT_MEMEQ(test, val, rval, sizeof(val));
+
+	/* If using a cache the cache satisfied the read */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, t->type == REGCACHE_NONE, data->read[i]);
+
+	regmap_exit(map);
+}
+
+static void reg_defaults(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int rval[BLOCK_TEST_SIZE];
+	int i;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+	config.num_reg_defaults = BLOCK_TEST_SIZE;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	/* Read back the expected default data */
+	KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval,
+						  BLOCK_TEST_SIZE));
+	KUNIT_EXPECT_MEMEQ(test, data->vals, rval, sizeof(rval));
+
+	/* The data should have been read from cache if there was one */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, t->type == REGCACHE_NONE, data->read[i]);
+}
+
+static void reg_defaults_read_dev(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int rval[BLOCK_TEST_SIZE];
+	int i;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+	config.num_reg_defaults_raw = BLOCK_TEST_SIZE;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	/* We should have read the cache defaults back from the map */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++) {
+		KUNIT_EXPECT_EQ(test, t->type != REGCACHE_NONE, data->read[i]);
+		data->read[i] = false;
+	}
+
+	/* Read back the expected default data */
+	KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval,
+						  BLOCK_TEST_SIZE));
+	KUNIT_EXPECT_MEMEQ(test, data->vals, rval, sizeof(rval));
+
+	/* The data should have been read from cache if there was one */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, t->type == REGCACHE_NONE, data->read[i]);
+}
+
+static void register_patch(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	struct reg_sequence patch[2];
+	unsigned int rval[BLOCK_TEST_SIZE];
+	int i;
+
+	/* We need defaults so readback works */
+	config = test_regmap_config;
+	config.cache_type = t->type;
+	config.num_reg_defaults = BLOCK_TEST_SIZE;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	/* Stash the original values */
+	KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval,
+						  BLOCK_TEST_SIZE));
+
+	/* Patch a couple of values */
+	patch[0].reg = 2;
+	patch[0].def = rval[2] + 1;
+	patch[0].delay_us = 0;
+	patch[1].reg = 5;
+	patch[1].def = rval[5] + 1;
+	patch[1].delay_us = 0;
+	KUNIT_EXPECT_EQ(test, 0, regmap_register_patch(map, patch,
+						       ARRAY_SIZE(patch)));
+
+	/* Only the patched registers are written */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++) {
+		switch (i) {
+		case 2:
+		case 5:
+			KUNIT_EXPECT_TRUE(test, data->written[i]);
+			KUNIT_EXPECT_EQ(test, data->vals[i], rval[i] + 1);
+			break;
+		default:
+			KUNIT_EXPECT_FALSE(test, data->written[i]);
+			KUNIT_EXPECT_EQ(test, data->vals[i], rval[i]);
+			break;
+		}
+	}
+
+	regmap_exit(map);
+}
+
+static void stride(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int rval;
+	int i;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+	config.reg_stride = 2;
+	config.num_reg_defaults = BLOCK_TEST_SIZE / 2;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	/* Only even registers can be accessed, try both read and write */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++) {
+		data->read[i] = false;
+		data->written[i] = false;
+
+		if (i % 2) {
+			KUNIT_EXPECT_NE(test, 0, regmap_read(map, i, &rval));
+			KUNIT_EXPECT_NE(test, 0, regmap_write(map, i, rval));
+			KUNIT_EXPECT_FALSE(test, data->read[i]);
+			KUNIT_EXPECT_FALSE(test, data->written[i]);
+		} else {
+			KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &rval));
+			KUNIT_EXPECT_EQ(test, data->vals[i], rval);
+			KUNIT_EXPECT_EQ(test, t->type == REGCACHE_NONE,
+					data->read[i]);
+
+			KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, rval));
+			KUNIT_EXPECT_TRUE(test, data->written[i]);
+		}
+	}
+
+	regmap_exit(map);
+}
+
+static struct regmap_range_cfg test_range = {
+	.selector_reg = 1,
+	.selector_mask = 0xff,
+
+	.window_start = 4,
+	.window_len = 10,
+
+	.range_min = 20,
+	.range_max = 40,
+};
+
+static bool test_range_volatile(struct device *dev, unsigned int reg)
+{
+	if (reg >= test_range.window_start &&
+	    reg <= test_range.selector_reg + test_range.window_len)
+		return true;
+
+	if (reg >= test_range.range_min && reg <= test_range.range_max)
+		return true;
+
+	return false;
+}
+
+static void basic_ranges(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int val;
+	int i;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+	config.volatile_reg = test_range_volatile;
+	config.ranges = &test_range;
+	config.num_ranges = 1;
+	config.max_register = test_range.range_max;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	for (i = test_range.range_min; i < test_range.range_max; i++) {
+		data->read[i] = false;
+		data->written[i] = false;
+	}
+
+	/* Reset the page to a non-zero value to trigger a change */
+	KUNIT_EXPECT_EQ(test, 0, regmap_write(map, test_range.selector_reg,
+					      test_range.range_max));
+
+	/* Check we set the page and use the window for writes */
+	data->written[test_range.selector_reg] = false;
+	data->written[test_range.window_start] = false;
+	KUNIT_EXPECT_EQ(test, 0, regmap_write(map, test_range.range_min, 0));
+	KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]);
+	KUNIT_EXPECT_TRUE(test, data->written[test_range.window_start]);
+
+	data->written[test_range.selector_reg] = false;
+	data->written[test_range.window_start] = false;
+	KUNIT_EXPECT_EQ(test, 0, regmap_write(map,
+					      test_range.range_min +
+					      test_range.window_len,
+					      0));
+	KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]);
+	KUNIT_EXPECT_TRUE(test, data->written[test_range.window_start]);
+
+	/* Same for reads */
+	data->written[test_range.selector_reg] = false;
+	data->read[test_range.window_start] = false;
+	KUNIT_EXPECT_EQ(test, 0, regmap_read(map, test_range.range_min, &val));
+	KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]);
+	KUNIT_EXPECT_TRUE(test, data->read[test_range.window_start]);
+
+	data->written[test_range.selector_reg] = false;
+	data->read[test_range.window_start] = false;
+	KUNIT_EXPECT_EQ(test, 0, regmap_read(map,
+					     test_range.range_min +
+					     test_range.window_len,
+					     &val));
+	KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]);
+	KUNIT_EXPECT_TRUE(test, data->read[test_range.window_start]);
+
+	/* No physical access triggered in the virtual range */
+	for (i = test_range.range_min; i < test_range.range_max; i++) {
+		KUNIT_EXPECT_FALSE(test, data->read[i]);
+		KUNIT_EXPECT_FALSE(test, data->written[i]);
+	}
+
+	regmap_exit(map);
+}
+
+/* Try to stress dynamic creation of cache data structures */
+static void stress_insert(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int rval, *vals;
+	size_t buf_sz;
+	int i;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+	config.max_register = 300;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	vals = kunit_kcalloc(test, sizeof(unsigned long), config.max_register,
+			     GFP_KERNEL);
+	KUNIT_ASSERT_FALSE(test, vals == NULL);
+	buf_sz = sizeof(unsigned long) * config.max_register;
+
+	get_random_bytes(vals, buf_sz);
+
+	/* Write data into the map/cache in ever decreasing strides */
+	for (i = 0; i < config.max_register; i += 100)
+		KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i]));
+	for (i = 0; i < config.max_register; i += 50)
+		KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i]));
+	for (i = 0; i < config.max_register; i += 25)
+		KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i]));
+	for (i = 0; i < config.max_register; i += 10)
+		KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i]));
+	for (i = 0; i < config.max_register; i += 5)
+		KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i]));
+	for (i = 0; i < config.max_register; i += 3)
+		KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i]));
+	for (i = 0; i < config.max_register; i += 2)
+		KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i]));
+	for (i = 0; i < config.max_register; i++)
+		KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i]));
+
+	/* Do reads from the cache (if there is one) match? */
+	for (i = 0; i < config.max_register; i ++) {
+		KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &rval));
+		KUNIT_EXPECT_EQ(test, rval, vals[i]);
+		KUNIT_EXPECT_EQ(test, t->type == REGCACHE_NONE, data->read[i]);
+	}
+
+	regmap_exit(map);
+}
+
+static void cache_bypass(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int val, rval;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	get_random_bytes(&val, sizeof(val));
+
+	/* Ensure the cache has a value in it */
+	KUNIT_EXPECT_EQ(test, 0, regmap_write(map, 0, val));
+
+	/* Bypass then write a different value */
+	regcache_cache_bypass(map, true);
+	KUNIT_EXPECT_EQ(test, 0, regmap_write(map, 0, val + 1));
+
+	/* Read the bypassed value */
+	KUNIT_EXPECT_EQ(test, 0, regmap_read(map, 0, &rval));
+	KUNIT_EXPECT_EQ(test, val + 1, rval);
+	KUNIT_EXPECT_EQ(test, data->vals[0], rval);
+
+	/* Disable bypass, the cache should still return the original value */
+	regcache_cache_bypass(map, false);
+	KUNIT_EXPECT_EQ(test, 0, regmap_read(map, 0, &rval));
+	KUNIT_EXPECT_EQ(test, val, rval);
+
+	regmap_exit(map);
+}
+
+static void cache_sync(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int val[BLOCK_TEST_SIZE];
+	int i;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	get_random_bytes(&val, sizeof(val));
+
+	/* Put some data into the cache */
+	KUNIT_EXPECT_EQ(test, 0, regmap_bulk_write(map, 0, val,
+						   BLOCK_TEST_SIZE));
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		data->written[i] = false;
+
+	/* Trash the data on the device itself then resync */
+	regcache_mark_dirty(map);
+	memset(data->vals, 0, sizeof(val));
+	KUNIT_EXPECT_EQ(test, 0, regcache_sync(map));
+
+	/* Did we just write the correct data out? */
+	KUNIT_EXPECT_MEMEQ(test, data->vals, val, sizeof(val));
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, true, data->written[i]);
+
+	regmap_exit(map);
+}
+
+static void cache_sync_defaults(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int val;
+	int i;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+	config.num_reg_defaults = BLOCK_TEST_SIZE;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	get_random_bytes(&val, sizeof(val));
+
+	/* Change the value of one register */
+	KUNIT_EXPECT_EQ(test, 0, regmap_write(map, 2, val));
+
+	/* Resync */
+	regcache_mark_dirty(map);
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		data->written[i] = false;
+	KUNIT_EXPECT_EQ(test, 0, regcache_sync(map));
+
+	/* Did we just sync the one register we touched? */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, i == 2, data->written[i]);
+
+	regmap_exit(map);
+}
+
+static void cache_sync_patch(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	struct reg_sequence patch[2];
+	unsigned int rval[BLOCK_TEST_SIZE], val;
+	int i;
+
+	/* We need defaults so readback works */
+	config = test_regmap_config;
+	config.cache_type = t->type;
+	config.num_reg_defaults = BLOCK_TEST_SIZE;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	/* Stash the original values */
+	KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval,
+						  BLOCK_TEST_SIZE));
+
+	/* Patch a couple of values */
+	patch[0].reg = 2;
+	patch[0].def = rval[2] + 1;
+	patch[0].delay_us = 0;
+	patch[1].reg = 5;
+	patch[1].def = rval[5] + 1;
+	patch[1].delay_us = 0;
+	KUNIT_EXPECT_EQ(test, 0, regmap_register_patch(map, patch,
+						       ARRAY_SIZE(patch)));
+
+	/* Sync the cache */
+	regcache_mark_dirty(map);
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		data->written[i] = false;
+	KUNIT_EXPECT_EQ(test, 0, regcache_sync(map));
+
+	/* The patch should be on the device but not in the cache */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++) {
+		KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &val));
+		KUNIT_EXPECT_EQ(test, val, rval[i]);
+
+		switch (i) {
+		case 2:
+		case 5:
+			KUNIT_EXPECT_EQ(test, true, data->written[i]);
+			KUNIT_EXPECT_EQ(test, data->vals[i], rval[i] + 1);
+			break;
+		default:
+			KUNIT_EXPECT_EQ(test, false, data->written[i]);
+			KUNIT_EXPECT_EQ(test, data->vals[i], rval[i]);
+			break;
+		}
+	}
+
+	regmap_exit(map);
+}
+
+static void cache_drop(struct kunit *test)
+{
+	struct regcache_types *t = (struct regcache_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int rval[BLOCK_TEST_SIZE];
+	int i;
+
+	config = test_regmap_config;
+	config.cache_type = t->type;
+	config.num_reg_defaults = BLOCK_TEST_SIZE;
+
+	map = gen_regmap(&config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	/* Ensure the data is read from the cache */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		data->read[i] = false;
+	KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval,
+						  BLOCK_TEST_SIZE));
+	for (i = 0; i < BLOCK_TEST_SIZE; i++) {
+		KUNIT_EXPECT_FALSE(test, data->read[i]);
+		data->read[i] = false;
+	}
+	KUNIT_EXPECT_MEMEQ(test, data->vals, rval, sizeof(rval));
+
+	/* Drop some registers */
+	KUNIT_EXPECT_EQ(test, 0, regcache_drop_region(map, 3, 5));
+
+	/* Reread and check only the dropped registers hit the device. */
+	KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval,
+						  BLOCK_TEST_SIZE));
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, data->read[i], i >= 3 && i <= 5);
+	KUNIT_EXPECT_MEMEQ(test, data->vals, rval, sizeof(rval));
+
+	regmap_exit(map);
+}
+
+static struct kunit_case regmap_test_cases[] = {
+	KUNIT_CASE_PARAM(basic_read_write, regcache_types_gen_params),
+	KUNIT_CASE_PARAM(bulk_write, regcache_types_gen_params),
+	KUNIT_CASE_PARAM(bulk_read, regcache_types_gen_params),
+	KUNIT_CASE_PARAM(reg_defaults, regcache_types_gen_params),
+	KUNIT_CASE_PARAM(reg_defaults_read_dev, regcache_types_gen_params),
+	KUNIT_CASE_PARAM(register_patch, regcache_types_gen_params),
+	KUNIT_CASE_PARAM(stride, regcache_types_gen_params),
+	KUNIT_CASE_PARAM(basic_ranges, regcache_types_gen_params),
+	KUNIT_CASE_PARAM(stress_insert, regcache_types_gen_params),
+	KUNIT_CASE_PARAM(cache_bypass, real_cache_types_gen_params),
+	KUNIT_CASE_PARAM(cache_sync, real_cache_types_gen_params),
+	KUNIT_CASE_PARAM(cache_sync_defaults, real_cache_types_gen_params),
+	KUNIT_CASE_PARAM(cache_sync_patch, real_cache_types_gen_params),
+	KUNIT_CASE_PARAM(cache_drop, sparse_cache_types_gen_params),
+	{}
+};
+
+static struct kunit_suite regmap_test_suite = {
+	.name = "regmap",
+	.test_cases = regmap_test_cases,
+};
+kunit_test_suite(regmap_test_suite);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/base/regmap/regmap-ram.c b/drivers/base/regmap/regmap-ram.c
new file mode 100644
index 000000000000..85f34a5dee04
--- /dev/null
+++ b/drivers/base/regmap/regmap-ram.c
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Register map access API - Memory region
+//
+// This is intended for testing only
+//
+// Copyright (c) 2023, Arm Ltd
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/swab.h>
+
+#include "internal.h"
+
+static int regmap_ram_write(void *context, unsigned int reg, unsigned int val)
+{
+	struct regmap_ram_data *data = context;
+
+	data->vals[reg] = val;
+	data->written[reg] = true;
+
+	return 0;
+}
+
+static int regmap_ram_read(void *context, unsigned int reg, unsigned int *val)
+{
+	struct regmap_ram_data *data = context;
+
+	*val = data->vals[reg];
+	data->read[reg] = true;
+
+	return 0;
+}
+
+static void regmap_ram_free_context(void *context)
+{
+	struct regmap_ram_data *data = context;
+
+	kfree(data->vals);
+	kfree(data->read);
+	kfree(data->written);
+	kfree(data);
+}
+
+static const struct regmap_bus regmap_ram = {
+	.fast_io = true,
+	.reg_write = regmap_ram_write,
+	.reg_read = regmap_ram_read,
+	.free_context = regmap_ram_free_context,
+};
+
+struct regmap *__regmap_init_ram(const struct regmap_config *config,
+				 struct regmap_ram_data *data,
+				 struct lock_class_key *lock_key,
+				 const char *lock_name)
+{
+	struct regmap *map;
+
+	if (!config->max_register) {
+		pr_crit("No max_register specified for RAM regmap\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	data->read = kcalloc(sizeof(bool), config->max_register + 1,
+			     GFP_KERNEL);
+	if (!data->read)
+		return ERR_PTR(-ENOMEM);
+
+	data->written = kcalloc(sizeof(bool), config->max_register + 1,
+				GFP_KERNEL);
+	if (!data->written)
+		return ERR_PTR(-ENOMEM);
+
+	map = __regmap_init(NULL, &regmap_ram, data, config,
+			    lock_key, lock_name);
+
+	return map;
+}
+EXPORT_SYMBOL_GPL(__regmap_init_ram);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/base/regmap/regmap-sdw.c b/drivers/base/regmap/regmap-sdw.c
index 81b0327f719d..09899ae99fc1 100644
--- a/drivers/base/regmap/regmap-sdw.c
+++ b/drivers/base/regmap/regmap-sdw.c
@@ -6,44 +6,53 @@
 #include <linux/module.h>
 #include <linux/regmap.h>
 #include <linux/soundwire/sdw.h>
+#include <linux/types.h>
 #include "internal.h"
 
-static int regmap_sdw_write(void *context, unsigned int reg, unsigned int val)
+static int regmap_sdw_write(void *context, const void *val_buf, size_t val_size)
 {
 	struct device *dev = context;
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
+	/* First word of buffer contains the destination address */
+	u32 addr = le32_to_cpu(*(const __le32 *)val_buf);
+	const u8 *val = val_buf;
 
-	return sdw_write_no_pm(slave, reg, val);
+	return sdw_nwrite_no_pm(slave, addr, val_size - sizeof(addr), val + sizeof(addr));
 }
 
-static int regmap_sdw_read(void *context, unsigned int reg, unsigned int *val)
+static int regmap_sdw_gather_write(void *context,
+				   const void *reg_buf, size_t reg_size,
+				   const void *val_buf, size_t val_size)
 {
 	struct device *dev = context;
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
-	int read;
+	u32 addr = le32_to_cpu(*(const __le32 *)reg_buf);
 
-	read = sdw_read_no_pm(slave, reg);
-	if (read < 0)
-		return read;
+	return sdw_nwrite_no_pm(slave, addr, val_size, val_buf);
+}
 
-	*val = read;
-	return 0;
+static int regmap_sdw_read(void *context,
+			   const void *reg_buf, size_t reg_size,
+			   void *val_buf, size_t val_size)
+{
+	struct device *dev = context;
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);
+	u32 addr = le32_to_cpu(*(const __le32 *)reg_buf);
+
+	return sdw_nread_no_pm(slave, addr, val_size, val_buf);
 }
 
 static const struct regmap_bus regmap_sdw = {
-	.reg_read = regmap_sdw_read,
-	.reg_write = regmap_sdw_write,
+	.write = regmap_sdw_write,
+	.gather_write = regmap_sdw_gather_write,
+	.read = regmap_sdw_read,
 	.reg_format_endian_default = REGMAP_ENDIAN_LITTLE,
 	.val_format_endian_default = REGMAP_ENDIAN_LITTLE,
 };
 
 static int regmap_sdw_config_check(const struct regmap_config *config)
 {
-	/* All register are 8-bits wide as per MIPI Soundwire 1.0 Spec */
-	if (config->val_bits != 8)
-		return -ENOTSUPP;
-
-	/* Registers are 32 bits wide */
+	/* Register addresses are 32 bits wide */
 	if (config->reg_bits != 32)
 		return -ENOTSUPP;
 
diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c
index d2a54eb0efd9..db7851f0e3b8 100644
--- a/drivers/base/regmap/regmap.c
+++ b/drivers/base/regmap/regmap.c
@@ -814,7 +814,7 @@ struct regmap *__regmap_init(struct device *dev,
 
 	map->format.reg_bytes = DIV_ROUND_UP(config->reg_bits, 8);
 	map->format.pad_bytes = config->pad_bits / 8;
-	map->format.reg_downshift = config->reg_downshift;
+	map->format.reg_shift = config->reg_shift;
 	map->format.val_bytes = DIV_ROUND_UP(config->val_bits, 8);
 	map->format.buf_size = DIV_ROUND_UP(config->reg_bits +
 			config->val_bits + config->pad_bits, 8);
@@ -1676,6 +1676,18 @@ static void regmap_set_work_buf_flag_mask(struct regmap *map, int max_bytes,
 		buf[i] |= (mask >> (8 * i)) & 0xff;
 }
 
+static unsigned int regmap_reg_addr(struct regmap *map, unsigned int reg)
+{
+	reg += map->reg_base;
+
+	if (map->format.reg_shift > 0)
+		reg >>= map->format.reg_shift;
+	else if (map->format.reg_shift < 0)
+		reg <<= -(map->format.reg_shift);
+
+	return reg;
+}
+
 static int _regmap_raw_write_impl(struct regmap *map, unsigned int reg,
 				  const void *val, size_t val_len, bool noinc)
 {
@@ -1753,8 +1765,7 @@ static int _regmap_raw_write_impl(struct regmap *map, unsigned int reg,
 			return ret;
 	}
 
-	reg += map->reg_base;
-	reg >>= map->format.reg_downshift;
+	reg = regmap_reg_addr(map, reg);
 	map->format.format_reg(map->work_buf, reg, map->reg_shift);
 	regmap_set_work_buf_flag_mask(map, map->format.reg_bytes,
 				      map->write_flag_mask);
@@ -1924,8 +1935,7 @@ static int _regmap_bus_formatted_write(void *context, unsigned int reg,
 			return ret;
 	}
 
-	reg += map->reg_base;
-	reg >>= map->format.reg_downshift;
+	reg = regmap_reg_addr(map, reg);
 	map->format.format_write(map, reg, val);
 
 	trace_regmap_hw_write_start(map, reg, 1);
@@ -1941,9 +1951,17 @@ static int _regmap_bus_reg_write(void *context, unsigned int reg,
 				 unsigned int val)
 {
 	struct regmap *map = context;
+	struct regmap_range_node *range;
+	int ret;
 
-	reg += map->reg_base;
-	reg >>= map->format.reg_downshift;
+	range = _regmap_range_lookup(map, reg);
+	if (range) {
+		ret = _regmap_select_page(map, &reg, range, 1);
+		if (ret != 0)
+			return ret;
+	}
+
+	reg = regmap_reg_addr(map, reg);
 	return map->bus->reg_write(map->bus_context, reg, val);
 }
 
@@ -2494,8 +2512,7 @@ static int _regmap_raw_multi_reg_write(struct regmap *map,
 		unsigned int reg = regs[i].reg;
 		unsigned int val = regs[i].def;
 		trace_regmap_hw_write_start(map, reg, 1);
-		reg += map->reg_base;
-		reg >>= map->format.reg_downshift;
+		reg = regmap_reg_addr(map, reg);
 		map->format.format_reg(u8, reg, map->reg_shift);
 		u8 += reg_bytes + pad_bytes;
 		map->format.format_val(u8, val, 0);
@@ -2821,8 +2838,7 @@ static int _regmap_raw_read(struct regmap *map, unsigned int reg, void *val,
 			return ret;
 	}
 
-	reg += map->reg_base;
-	reg >>= map->format.reg_downshift;
+	reg = regmap_reg_addr(map, reg);
 	map->format.format_reg(map->work_buf, reg, map->reg_shift);
 	regmap_set_work_buf_flag_mask(map, map->format.reg_bytes,
 				      map->read_flag_mask);
@@ -2841,9 +2857,17 @@ static int _regmap_bus_reg_read(void *context, unsigned int reg,
 				unsigned int *val)
 {
 	struct regmap *map = context;
+	struct regmap_range_node *range;
+	int ret;
 
-	reg += map->reg_base;
-	reg >>= map->format.reg_downshift;
+	range = _regmap_range_lookup(map, reg);
+	if (range) {
+		ret = _regmap_select_page(map, &reg, range, 1);
+		if (ret != 0)
+			return ret;
+	}
+
+	reg = regmap_reg_addr(map, reg);
 	return map->bus->reg_read(map->bus_context, reg, val);
 }
 
@@ -3235,8 +3259,7 @@ static int _regmap_update_bits(struct regmap *map, unsigned int reg,
 		*change = false;
 
 	if (regmap_volatile(map, reg) && map->reg_update_bits) {
-		reg += map->reg_base;
-		reg >>= map->format.reg_downshift;
+		reg = regmap_reg_addr(map, reg);
 		ret = map->reg_update_bits(map->bus_context, reg, mask, val);
 		if (ret == 0 && change)
 			*change = true;
diff --git a/drivers/base/soc.c b/drivers/base/soc.c
index 0fb1d4ab9d8a..8dec5228fde3 100644
--- a/drivers/base/soc.c
+++ b/drivers/base/soc.c
@@ -7,6 +7,7 @@
 
 #include <linux/sysfs.h>
 #include <linux/init.h>
+#include <linux/of.h>
 #include <linux/stat.h>
 #include <linux/slab.h>
 #include <linux/idr.h>
@@ -110,6 +111,18 @@ static void soc_release(struct device *dev)
 	kfree(soc_dev);
 }
 
+static void soc_device_get_machine(struct soc_device_attribute *soc_dev_attr)
+{
+	struct device_node *np;
+
+	if (soc_dev_attr->machine)
+		return;
+
+	np = of_find_node_by_path("/");
+	of_property_read_string(np, "model", &soc_dev_attr->machine);
+	of_node_put(np);
+}
+
 static struct soc_device_attribute *early_soc_dev_attr;
 
 struct soc_device *soc_device_register(struct soc_device_attribute *soc_dev_attr)
@@ -118,6 +131,8 @@ struct soc_device *soc_device_register(struct soc_device_attribute *soc_dev_attr
 	const struct attribute_group **soc_attr_groups;
 	int ret;
 
+	soc_device_get_machine(soc_dev_attr);
+
 	if (!soc_bus_registered) {
 		if (early_soc_dev_attr)
 			return ERR_PTR(-EBUSY);
diff --git a/drivers/bcma/driver_mips.c b/drivers/bcma/driver_mips.c
index 4f01e6b17bb9..9be0806eb033 100644
--- a/drivers/bcma/driver_mips.c
+++ b/drivers/bcma/driver_mips.c
@@ -46,12 +46,6 @@ static inline bool bcma_core_mips_bcm5357b0_quirk(struct bcma_device *dev)
 	       dev->id.id == BCMA_CORE_USB20_HOST;
 }
 
-static inline u32 mips_read32(struct bcma_drv_mips *mcore,
-			      u16 offset)
-{
-	return bcma_read32(mcore->core, offset);
-}
-
 static u32 bcma_core_mips_irqflag(struct bcma_device *dev)
 {
 	u32 flag;
diff --git a/drivers/bcma/main.c b/drivers/bcma/main.c
index 7b39f010bbb3..7061d3ee836a 100644
--- a/drivers/bcma/main.c
+++ b/drivers/bcma/main.c
@@ -14,6 +14,7 @@
 #include <linux/slab.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
+#include <linux/of_device.h>
 #include <linux/of_platform.h>
 
 MODULE_DESCRIPTION("Broadcom's specific AMBA driver");
@@ -140,17 +141,17 @@ static struct device_node *bcma_of_find_child_device(struct device *parent,
 						     struct bcma_device *core)
 {
 	struct device_node *node;
-	u64 size;
-	const __be32 *reg;
+	int ret;
 
 	if (!parent->of_node)
 		return NULL;
 
 	for_each_child_of_node(parent->of_node, node) {
-		reg = of_get_address(node, 0, &size, NULL);
-		if (!reg)
+		struct resource res;
+		ret = of_address_to_resource(node, 0, &res);
+		if (ret)
 			continue;
-		if (of_translate_address(node, reg) == core->addr)
+		if (res.start == core->addr)
 			return node;
 	}
 	return NULL;
diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
index f79f20430ef7..5b9d4aaebb81 100644
--- a/drivers/block/Kconfig
+++ b/drivers/block/Kconfig
@@ -385,6 +385,23 @@ config BLK_DEV_UBLK
 	  can handle batch more effectively, but task_work_add() isn't exported
 	  for module, so ublk has to be built to kernel.
 
+config BLKDEV_UBLK_LEGACY_OPCODES
+	bool "Support legacy command opcode"
+	depends on BLK_DEV_UBLK
+	default y
+	help
+	  ublk driver started to take plain command encoding, which turns out
+	  one bad way. The traditional ioctl command opcode encodes more
+	  info and basically defines each code uniquely, so opcode conflict
+	  is avoided, and driver can handle wrong command easily, meantime it
+	  may help security subsystem to audit io_uring command.
+
+	  Say Y if your application still uses legacy command opcode.
+
+	  Say N if you don't want to support legacy command opcode. It is
+	  suggested to enable N if your application(ublk server) switches to
+	  ioctl command encoding.
+
 source "drivers/block/rnbd/Kconfig"
 
 endif # BLK_DEV
diff --git a/drivers/block/aoe/aoechr.c b/drivers/block/aoe/aoechr.c
index 7a368c90467d..4c666f72203f 100644
--- a/drivers/block/aoe/aoechr.c
+++ b/drivers/block/aoe/aoechr.c
@@ -290,7 +290,7 @@ aoechr_init(void)
 	}
 	init_completion(&emsgs_comp);
 	spin_lock_init(&emsgs_lock);
-	aoe_class = class_create(THIS_MODULE, "aoe");
+	aoe_class = class_create("aoe");
 	if (IS_ERR(aoe_class)) {
 		unregister_chrdev(AOE_MAJOR, "aoechr");
 		return PTR_ERR(aoe_class);
diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c
index 429255876800..64b3a1c76f03 100644
--- a/drivers/block/drbd/drbd_actlog.c
+++ b/drivers/block/drbd/drbd_actlog.c
@@ -735,8 +735,9 @@ static bool update_rs_extent(struct drbd_device *device,
 	return false;
 }
 
-void drbd_advance_rs_marks(struct drbd_device *device, unsigned long still_to_go)
+void drbd_advance_rs_marks(struct drbd_peer_device *peer_device, unsigned long still_to_go)
 {
+	struct drbd_device *device = peer_device->device;
 	unsigned long now = jiffies;
 	unsigned long last = device->rs_mark_time[device->rs_last_mark];
 	int next = (device->rs_last_mark + 1) % DRBD_SYNC_MARKS;
@@ -819,7 +820,7 @@ static int update_sync_bits(struct drbd_device *device,
 		if (mode == SET_IN_SYNC) {
 			unsigned long still_to_go = drbd_bm_total_weight(device);
 			bool rs_is_done = (still_to_go <= device->rs_failed);
-			drbd_advance_rs_marks(device, still_to_go);
+			drbd_advance_rs_marks(first_peer_device(device), still_to_go);
 			if (cleared || rs_is_done)
 				maybe_schedule_on_disk_bitmap_update(device, rs_is_done);
 		} else if (mode == RECORD_RS_FAILED)
@@ -843,10 +844,11 @@ static bool plausible_request_size(int size)
  * called by worker on C_SYNC_TARGET and receiver on SyncSource.
  *
  */
-int __drbd_change_sync(struct drbd_device *device, sector_t sector, int size,
+int __drbd_change_sync(struct drbd_peer_device *peer_device, sector_t sector, int size,
 		enum update_sync_bits_mode mode)
 {
 	/* Is called from worker and receiver context _only_ */
+	struct drbd_device *device = peer_device->device;
 	unsigned long sbnr, ebnr, lbnr;
 	unsigned long count = 0;
 	sector_t esector, nr_sectors;
@@ -1009,14 +1011,15 @@ retry:
  * tries to set it to BME_LOCKED. Returns 0 upon success, and -EAGAIN
  * if there is still application IO going on in this area.
  */
-int drbd_try_rs_begin_io(struct drbd_device *device, sector_t sector)
+int drbd_try_rs_begin_io(struct drbd_peer_device *peer_device, sector_t sector)
 {
+	struct drbd_device *device = peer_device->device;
 	unsigned int enr = BM_SECT_TO_EXT(sector);
 	const unsigned int al_enr = enr*AL_EXT_PER_BM_SECT;
 	struct lc_element *e;
 	struct bm_extent *bm_ext;
 	int i;
-	bool throttle = drbd_rs_should_slow_down(device, sector, true);
+	bool throttle = drbd_rs_should_slow_down(peer_device, sector, true);
 
 	/* If we need to throttle, a half-locked (only marked BME_NO_WRITES,
 	 * not yet BME_LOCKED) extent needs to be kicked out explicitly if we
diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c
index 289876ffbc31..6ac8c54b44c7 100644
--- a/drivers/block/drbd/drbd_bitmap.c
+++ b/drivers/block/drbd/drbd_bitmap.c
@@ -1216,7 +1216,9 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned
  * drbd_bm_read() - Read the whole bitmap from its on disk location.
  * @device:	DRBD device.
  */
-int drbd_bm_read(struct drbd_device *device) __must_hold(local)
+int drbd_bm_read(struct drbd_device *device,
+		 struct drbd_peer_device *peer_device) __must_hold(local)
+
 {
 	return bm_rw(device, BM_AIO_READ, 0);
 }
@@ -1227,7 +1229,8 @@ int drbd_bm_read(struct drbd_device *device) __must_hold(local)
  *
  * Will only write pages that have changed since last IO.
  */
-int drbd_bm_write(struct drbd_device *device) __must_hold(local)
+int drbd_bm_write(struct drbd_device *device,
+		 struct drbd_peer_device *peer_device) __must_hold(local)
 {
 	return bm_rw(device, 0, 0);
 }
@@ -1238,7 +1241,8 @@ int drbd_bm_write(struct drbd_device *device) __must_hold(local)
  *
  * Will write all pages.
  */
-int drbd_bm_write_all(struct drbd_device *device) __must_hold(local)
+int drbd_bm_write_all(struct drbd_device *device,
+		struct drbd_peer_device *peer_device) __must_hold(local)
 {
 	return bm_rw(device, BM_AIO_WRITE_ALL_PAGES, 0);
 }
@@ -1264,7 +1268,8 @@ int drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_ho
  * verify is aborted due to a failed peer disk, while local IO continues, or
  * pending resync acks are still being processed.
  */
-int drbd_bm_write_copy_pages(struct drbd_device *device) __must_hold(local)
+int drbd_bm_write_copy_pages(struct drbd_device *device,
+		struct drbd_peer_device *peer_device) __must_hold(local)
 {
 	return bm_rw(device, BM_AIO_COPY_PAGES, 0);
 }
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index d89b7d03d4c8..a30a5ed811be 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -66,6 +66,7 @@ extern int drbd_proc_details;
 
 struct drbd_device;
 struct drbd_connection;
+struct drbd_peer_device;
 
 /* Defines to control fault insertion */
 enum {
@@ -126,8 +127,8 @@ struct bm_xfer_ctx {
 	unsigned bytes[2];
 };
 
-extern void INFO_bm_xfer_stats(struct drbd_device *device,
-		const char *direction, struct bm_xfer_ctx *c);
+extern void INFO_bm_xfer_stats(struct drbd_peer_device *peer_device,
+			       const char *direction, struct bm_xfer_ctx *c);
 
 static inline void bm_xfer_ctx_bit_to_word_offset(struct bm_xfer_ctx *c)
 {
@@ -541,9 +542,10 @@ struct drbd_md_io {
 
 struct bm_io_work {
 	struct drbd_work w;
+	struct drbd_peer_device *peer_device;
 	char *why;
 	enum bm_flag flags;
-	int (*io_fn)(struct drbd_device *device);
+	int (*io_fn)(struct drbd_device *device, struct drbd_peer_device *peer_device);
 	void (*done)(struct drbd_device *device, int rv);
 };
 
@@ -1041,7 +1043,7 @@ extern int drbd_send_drequest_csum(struct drbd_peer_device *, sector_t sector,
 				   enum drbd_packet cmd);
 extern int drbd_send_ov_request(struct drbd_peer_device *, sector_t sector, int size);
 
-extern int drbd_send_bitmap(struct drbd_device *device);
+extern int drbd_send_bitmap(struct drbd_device *device, struct drbd_peer_device *peer_device);
 extern void drbd_send_sr_reply(struct drbd_peer_device *, enum drbd_state_rv retcode);
 extern void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode);
 extern int drbd_send_rs_deallocated(struct drbd_peer_device *, struct drbd_peer_request *);
@@ -1065,17 +1067,22 @@ extern void drbd_md_clear_flag(struct drbd_device *device, int flags)__must_hold
 extern int drbd_md_test_flag(struct drbd_backing_dev *, int);
 extern void drbd_md_mark_dirty(struct drbd_device *device);
 extern void drbd_queue_bitmap_io(struct drbd_device *device,
-				 int (*io_fn)(struct drbd_device *),
+				 int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
 				 void (*done)(struct drbd_device *, int),
-				 char *why, enum bm_flag flags);
+				 char *why, enum bm_flag flags,
+				 struct drbd_peer_device *peer_device);
 extern int drbd_bitmap_io(struct drbd_device *device,
-		int (*io_fn)(struct drbd_device *),
-		char *why, enum bm_flag flags);
+		int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
+		char *why, enum bm_flag flags,
+		struct drbd_peer_device *peer_device);
 extern int drbd_bitmap_io_from_worker(struct drbd_device *device,
-		int (*io_fn)(struct drbd_device *),
-		char *why, enum bm_flag flags);
-extern int drbd_bmio_set_n_write(struct drbd_device *device) __must_hold(local);
-extern int drbd_bmio_clear_n_write(struct drbd_device *device) __must_hold(local);
+		int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
+		char *why, enum bm_flag flags,
+		struct drbd_peer_device *peer_device);
+extern int drbd_bmio_set_n_write(struct drbd_device *device,
+		struct drbd_peer_device *peer_device) __must_hold(local);
+extern int drbd_bmio_clear_n_write(struct drbd_device *device,
+		struct drbd_peer_device *peer_device) __must_hold(local);
 
 /* Meta data layout
  *
@@ -1284,14 +1291,18 @@ extern void _drbd_bm_set_bits(struct drbd_device *device,
 		const unsigned long s, const unsigned long e);
 extern int  drbd_bm_test_bit(struct drbd_device *device, unsigned long bitnr);
 extern int  drbd_bm_e_weight(struct drbd_device *device, unsigned long enr);
-extern int  drbd_bm_read(struct drbd_device *device) __must_hold(local);
+extern int  drbd_bm_read(struct drbd_device *device,
+		struct drbd_peer_device *peer_device) __must_hold(local);
 extern void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr);
-extern int  drbd_bm_write(struct drbd_device *device) __must_hold(local);
+extern int  drbd_bm_write(struct drbd_device *device,
+		struct drbd_peer_device *peer_device) __must_hold(local);
 extern void drbd_bm_reset_al_hints(struct drbd_device *device) __must_hold(local);
 extern int  drbd_bm_write_hinted(struct drbd_device *device) __must_hold(local);
 extern int  drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_hold(local);
-extern int drbd_bm_write_all(struct drbd_device *device) __must_hold(local);
-extern int  drbd_bm_write_copy_pages(struct drbd_device *device) __must_hold(local);
+extern int drbd_bm_write_all(struct drbd_device *device,
+		struct drbd_peer_device *peer_device) __must_hold(local);
+extern int  drbd_bm_write_copy_pages(struct drbd_device *device,
+		struct drbd_peer_device *peer_device) __must_hold(local);
 extern size_t	     drbd_bm_words(struct drbd_device *device);
 extern unsigned long drbd_bm_bits(struct drbd_device *device);
 extern sector_t      drbd_bm_capacity(struct drbd_device *device);
@@ -1422,21 +1433,24 @@ void drbd_resync_after_changed(struct drbd_device *device);
 extern void drbd_start_resync(struct drbd_device *device, enum drbd_conns side);
 extern void resume_next_sg(struct drbd_device *device);
 extern void suspend_other_sg(struct drbd_device *device);
-extern int drbd_resync_finished(struct drbd_device *device);
+extern int drbd_resync_finished(struct drbd_peer_device *peer_device);
 /* maybe rather drbd_main.c ? */
 extern void *drbd_md_get_buffer(struct drbd_device *device, const char *intent);
 extern void drbd_md_put_buffer(struct drbd_device *device);
 extern int drbd_md_sync_page_io(struct drbd_device *device,
 		struct drbd_backing_dev *bdev, sector_t sector, enum req_op op);
-extern void drbd_ov_out_of_sync_found(struct drbd_device *, sector_t, int);
+extern void drbd_ov_out_of_sync_found(struct drbd_peer_device *peer_device,
+		sector_t sector, int size);
 extern void wait_until_done_or_force_detached(struct drbd_device *device,
 		struct drbd_backing_dev *bdev, unsigned int *done);
-extern void drbd_rs_controller_reset(struct drbd_device *device);
+extern void drbd_rs_controller_reset(struct drbd_peer_device *peer_device);
 
-static inline void ov_out_of_sync_print(struct drbd_device *device)
+static inline void ov_out_of_sync_print(struct drbd_peer_device *peer_device)
 {
+	struct drbd_device *device = peer_device->device;
+
 	if (device->ov_last_oos_size) {
-		drbd_err(device, "Out of sync: start=%llu, size=%lu (sectors)\n",
+		drbd_err(peer_device, "Out of sync: start=%llu, size=%lu (sectors)\n",
 		     (unsigned long long)device->ov_last_oos_start,
 		     (unsigned long)device->ov_last_oos_size);
 	}
@@ -1475,7 +1489,7 @@ extern int drbd_ack_receiver(struct drbd_thread *thi);
 extern void drbd_send_ping_wf(struct work_struct *ws);
 extern void drbd_send_acks_wf(struct work_struct *ws);
 extern bool drbd_rs_c_min_rate_throttle(struct drbd_device *device);
-extern bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector,
+extern bool drbd_rs_should_slow_down(struct drbd_peer_device *peer_device, sector_t sector,
 		bool throttle_if_app_is_waiting);
 extern int drbd_submit_peer_request(struct drbd_peer_request *peer_req);
 extern int drbd_free_peer_reqs(struct drbd_device *, struct list_head *);
@@ -1531,22 +1545,22 @@ extern void drbd_al_begin_io(struct drbd_device *device, struct drbd_interval *i
 extern void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i);
 extern void drbd_rs_complete_io(struct drbd_device *device, sector_t sector);
 extern int drbd_rs_begin_io(struct drbd_device *device, sector_t sector);
-extern int drbd_try_rs_begin_io(struct drbd_device *device, sector_t sector);
+extern int drbd_try_rs_begin_io(struct drbd_peer_device *peer_device, sector_t sector);
 extern void drbd_rs_cancel_all(struct drbd_device *device);
 extern int drbd_rs_del_all(struct drbd_device *device);
-extern void drbd_rs_failed_io(struct drbd_device *device,
+extern void drbd_rs_failed_io(struct drbd_peer_device *peer_device,
 		sector_t sector, int size);
-extern void drbd_advance_rs_marks(struct drbd_device *device, unsigned long still_to_go);
+extern void drbd_advance_rs_marks(struct drbd_peer_device *peer_device, unsigned long still_to_go);
 
 enum update_sync_bits_mode { RECORD_RS_FAILED, SET_OUT_OF_SYNC, SET_IN_SYNC };
-extern int __drbd_change_sync(struct drbd_device *device, sector_t sector, int size,
+extern int __drbd_change_sync(struct drbd_peer_device *peer_device, sector_t sector, int size,
 		enum update_sync_bits_mode mode);
-#define drbd_set_in_sync(device, sector, size) \
-	__drbd_change_sync(device, sector, size, SET_IN_SYNC)
-#define drbd_set_out_of_sync(device, sector, size) \
-	__drbd_change_sync(device, sector, size, SET_OUT_OF_SYNC)
-#define drbd_rs_failed_io(device, sector, size) \
-	__drbd_change_sync(device, sector, size, RECORD_RS_FAILED)
+#define drbd_set_in_sync(peer_device, sector, size) \
+	__drbd_change_sync(peer_device, sector, size, SET_IN_SYNC)
+#define drbd_set_out_of_sync(peer_device, sector, size) \
+	__drbd_change_sync(peer_device, sector, size, SET_OUT_OF_SYNC)
+#define drbd_rs_failed_io(peer_device, sector, size) \
+	__drbd_change_sync(peer_device, sector, size, RECORD_RS_FAILED)
 extern void drbd_al_shrink(struct drbd_device *device);
 extern int drbd_al_initialize(struct drbd_device *, void *);
 
@@ -1918,18 +1932,14 @@ static inline void inc_ap_pending(struct drbd_device *device)
 	atomic_inc(&device->ap_pending_cnt);
 }
 
-#define ERR_IF_CNT_IS_NEGATIVE(which, func, line)			\
-	if (atomic_read(&device->which) < 0)				\
-		drbd_err(device, "in %s:%d: " #which " = %d < 0 !\n",	\
-			func, line,					\
-			atomic_read(&device->which))
-
-#define dec_ap_pending(device) _dec_ap_pending(device, __func__, __LINE__)
-static inline void _dec_ap_pending(struct drbd_device *device, const char *func, int line)
+#define dec_ap_pending(device) ((void)expect((device), __dec_ap_pending(device) >= 0))
+static inline int __dec_ap_pending(struct drbd_device *device)
 {
-	if (atomic_dec_and_test(&device->ap_pending_cnt))
+	int ap_pending_cnt = atomic_dec_return(&device->ap_pending_cnt);
+
+	if (ap_pending_cnt == 0)
 		wake_up(&device->misc_wait);
-	ERR_IF_CNT_IS_NEGATIVE(ap_pending_cnt, func, line);
+	return ap_pending_cnt;
 }
 
 /* counts how many resync-related answers we still expect from the peer
@@ -1938,16 +1948,16 @@ static inline void _dec_ap_pending(struct drbd_device *device, const char *func,
  * C_SYNC_SOURCE sends P_RS_DATA_REPLY   (and expects P_WRITE_ACK with ID_SYNCER)
  *					   (or P_NEG_ACK with ID_SYNCER)
  */
-static inline void inc_rs_pending(struct drbd_device *device)
+static inline void inc_rs_pending(struct drbd_peer_device *peer_device)
 {
-	atomic_inc(&device->rs_pending_cnt);
+	atomic_inc(&peer_device->device->rs_pending_cnt);
 }
 
-#define dec_rs_pending(device) _dec_rs_pending(device, __func__, __LINE__)
-static inline void _dec_rs_pending(struct drbd_device *device, const char *func, int line)
+#define dec_rs_pending(peer_device) \
+	((void)expect((peer_device), __dec_rs_pending(peer_device) >= 0))
+static inline int __dec_rs_pending(struct drbd_peer_device *peer_device)
 {
-	atomic_dec(&device->rs_pending_cnt);
-	ERR_IF_CNT_IS_NEGATIVE(rs_pending_cnt, func, line);
+	return atomic_dec_return(&peer_device->device->rs_pending_cnt);
 }
 
 /* counts how many answers we still need to send to the peer.
@@ -1964,18 +1974,16 @@ static inline void inc_unacked(struct drbd_device *device)
 	atomic_inc(&device->unacked_cnt);
 }
 
-#define dec_unacked(device) _dec_unacked(device, __func__, __LINE__)
-static inline void _dec_unacked(struct drbd_device *device, const char *func, int line)
+#define dec_unacked(device) ((void)expect(device, __dec_unacked(device) >= 0))
+static inline int __dec_unacked(struct drbd_device *device)
 {
-	atomic_dec(&device->unacked_cnt);
-	ERR_IF_CNT_IS_NEGATIVE(unacked_cnt, func, line);
+	return atomic_dec_return(&device->unacked_cnt);
 }
 
-#define sub_unacked(device, n) _sub_unacked(device, n, __func__, __LINE__)
-static inline void _sub_unacked(struct drbd_device *device, int n, const char *func, int line)
+#define sub_unacked(device, n) ((void)expect(device, __sub_unacked(device) >= 0))
+static inline int __sub_unacked(struct drbd_device *device, int n)
 {
-	atomic_sub(n, &device->unacked_cnt);
-	ERR_IF_CNT_IS_NEGATIVE(unacked_cnt, func, line);
+	return atomic_sub_return(n, &device->unacked_cnt);
 }
 
 static inline bool is_sync_target_state(enum drbd_conns connection_state)
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 2c764f7ee4a7..83987e7a5ef2 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -231,9 +231,11 @@ void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
 		}
 	req = list_prepare_entry(tmp, &connection->transfer_log, tl_requests);
 	list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
+		struct drbd_peer_device *peer_device;
 		if (req->epoch != expect_epoch)
 			break;
-		_req_mod(req, BARRIER_ACKED);
+		peer_device = conn_peer_device(connection, req->device->vnr);
+		_req_mod(req, BARRIER_ACKED, peer_device);
 	}
 	spin_unlock_irq(&connection->resource->req_lock);
 
@@ -256,10 +258,13 @@ bail:
 /* must hold resource->req_lock */
 void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
 {
+	struct drbd_peer_device *peer_device;
 	struct drbd_request *req, *r;
 
-	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests)
-		_req_mod(req, what);
+	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
+		peer_device = conn_peer_device(connection, req->device->vnr);
+		_req_mod(req, what, peer_device);
+	}
 }
 
 void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
@@ -297,7 +302,7 @@ void tl_abort_disk_io(struct drbd_device *device)
 			continue;
 		if (req->device != device)
 			continue;
-		_req_mod(req, ABORT_DISK_IO);
+		_req_mod(req, ABORT_DISK_IO, NULL);
 	}
 	spin_unlock_irq(&connection->resource->req_lock);
 }
@@ -1198,10 +1203,11 @@ static int fill_bitmap_rle_bits(struct drbd_device *device,
  * code upon failure.
  */
 static int
-send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
+send_bitmap_rle_or_plain(struct drbd_peer_device *peer_device, struct bm_xfer_ctx *c)
 {
-	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
-	unsigned int header_size = drbd_header_size(first_peer_device(device)->connection);
+	struct drbd_device *device = peer_device->device;
+	struct drbd_socket *sock = &peer_device->connection->data;
+	unsigned int header_size = drbd_header_size(peer_device->connection);
 	struct p_compressed_bm *p = sock->sbuf + header_size;
 	int len, err;
 
@@ -1212,7 +1218,7 @@ send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
 
 	if (len) {
 		dcbp_set_code(p, RLE_VLI_Bits);
-		err = __send_command(first_peer_device(device)->connection, device->vnr, sock,
+		err = __send_command(peer_device->connection, device->vnr, sock,
 				     P_COMPRESSED_BITMAP, sizeof(*p) + len,
 				     NULL, 0);
 		c->packets[0]++;
@@ -1233,7 +1239,8 @@ send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
 		len = num_words * sizeof(*p);
 		if (len)
 			drbd_bm_get_lel(device, c->word_offset, num_words, p);
-		err = __send_command(first_peer_device(device)->connection, device->vnr, sock, P_BITMAP, len, NULL, 0);
+		err = __send_command(peer_device->connection, device->vnr, sock, P_BITMAP,
+				     len, NULL, 0);
 		c->word_offset += num_words;
 		c->bit_offset = c->word_offset * BITS_PER_LONG;
 
@@ -1245,7 +1252,7 @@ send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
 	}
 	if (!err) {
 		if (len == 0) {
-			INFO_bm_xfer_stats(device, "send", c);
+			INFO_bm_xfer_stats(peer_device, "send", c);
 			return 0;
 		} else
 			return 1;
@@ -1254,7 +1261,8 @@ send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
 }
 
 /* See the comment at receive_bitmap() */
-static int _drbd_send_bitmap(struct drbd_device *device)
+static int _drbd_send_bitmap(struct drbd_device *device,
+			    struct drbd_peer_device *peer_device)
 {
 	struct bm_xfer_ctx c;
 	int err;
@@ -1266,7 +1274,7 @@ static int _drbd_send_bitmap(struct drbd_device *device)
 		if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC)) {
 			drbd_info(device, "Writing the whole bitmap, MDF_FullSync was set.\n");
 			drbd_bm_set_all(device);
-			if (drbd_bm_write(device)) {
+			if (drbd_bm_write(device, peer_device)) {
 				/* write_bm did fail! Leave full sync flag set in Meta P_DATA
 				 * but otherwise process as per normal - need to tell other
 				 * side that a full resync is required! */
@@ -1285,20 +1293,20 @@ static int _drbd_send_bitmap(struct drbd_device *device)
 	};
 
 	do {
-		err = send_bitmap_rle_or_plain(device, &c);
+		err = send_bitmap_rle_or_plain(peer_device, &c);
 	} while (err > 0);
 
 	return err == 0;
 }
 
-int drbd_send_bitmap(struct drbd_device *device)
+int drbd_send_bitmap(struct drbd_device *device, struct drbd_peer_device *peer_device)
 {
-	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
+	struct drbd_socket *sock = &peer_device->connection->data;
 	int err = -1;
 
 	mutex_lock(&sock->mutex);
 	if (sock->socket)
-		err = !_drbd_send_bitmap(device);
+		err = !_drbd_send_bitmap(device, peer_device);
 	mutex_unlock(&sock->mutex);
 	return err;
 }
@@ -3406,7 +3414,9 @@ void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
  *
  * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
  */
-int drbd_bmio_set_n_write(struct drbd_device *device) __must_hold(local)
+int drbd_bmio_set_n_write(struct drbd_device *device,
+			  struct drbd_peer_device *peer_device) __must_hold(local)
+
 {
 	int rv = -EIO;
 
@@ -3414,7 +3424,7 @@ int drbd_bmio_set_n_write(struct drbd_device *device) __must_hold(local)
 	drbd_md_sync(device);
 	drbd_bm_set_all(device);
 
-	rv = drbd_bm_write(device);
+	rv = drbd_bm_write(device, peer_device);
 
 	if (!rv) {
 		drbd_md_clear_flag(device, MDF_FULL_SYNC);
@@ -3430,11 +3440,13 @@ int drbd_bmio_set_n_write(struct drbd_device *device) __must_hold(local)
  *
  * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
  */
-int drbd_bmio_clear_n_write(struct drbd_device *device) __must_hold(local)
+int drbd_bmio_clear_n_write(struct drbd_device *device,
+			  struct drbd_peer_device *peer_device) __must_hold(local)
+
 {
 	drbd_resume_al(device);
 	drbd_bm_clear_all(device);
-	return drbd_bm_write(device);
+	return drbd_bm_write(device, peer_device);
 }
 
 static int w_bitmap_io(struct drbd_work *w, int unused)
@@ -3453,7 +3465,7 @@ static int w_bitmap_io(struct drbd_work *w, int unused)
 
 	if (get_ldev(device)) {
 		drbd_bm_lock(device, work->why, work->flags);
-		rv = work->io_fn(device);
+		rv = work->io_fn(device, work->peer_device);
 		drbd_bm_unlock(device);
 		put_ldev(device);
 	}
@@ -3488,11 +3500,12 @@ static int w_bitmap_io(struct drbd_work *w, int unused)
  * put_ldev().
  */
 void drbd_queue_bitmap_io(struct drbd_device *device,
-			  int (*io_fn)(struct drbd_device *),
+			  int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
 			  void (*done)(struct drbd_device *, int),
-			  char *why, enum bm_flag flags)
+			  char *why, enum bm_flag flags,
+			  struct drbd_peer_device *peer_device)
 {
-	D_ASSERT(device, current == first_peer_device(device)->connection->worker.task);
+	D_ASSERT(device, current == peer_device->connection->worker.task);
 
 	D_ASSERT(device, !test_bit(BITMAP_IO_QUEUED, &device->flags));
 	D_ASSERT(device, !test_bit(BITMAP_IO, &device->flags));
@@ -3501,6 +3514,7 @@ void drbd_queue_bitmap_io(struct drbd_device *device,
 		drbd_err(device, "FIXME going to queue '%s' but '%s' still pending?\n",
 			why, device->bm_io_work.why);
 
+	device->bm_io_work.peer_device = peer_device;
 	device->bm_io_work.io_fn = io_fn;
 	device->bm_io_work.done = done;
 	device->bm_io_work.why = why;
@@ -3512,7 +3526,7 @@ void drbd_queue_bitmap_io(struct drbd_device *device,
 	 * application IO does not conflict anyways. */
 	if (flags == BM_LOCKED_CHANGE_ALLOWED || atomic_read(&device->ap_bio_cnt) == 0) {
 		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
-			drbd_queue_work(&first_peer_device(device)->connection->sender_work,
+			drbd_queue_work(&peer_device->connection->sender_work,
 					&device->bm_io_work.w);
 	}
 	spin_unlock_irq(&device->resource->req_lock);
@@ -3528,8 +3542,10 @@ void drbd_queue_bitmap_io(struct drbd_device *device,
  * freezes application IO while that the actual IO operations runs. This
  * functions MAY NOT be called from worker context.
  */
-int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *),
-		char *why, enum bm_flag flags)
+int drbd_bitmap_io(struct drbd_device *device,
+		int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
+		char *why, enum bm_flag flags,
+		struct drbd_peer_device *peer_device)
 {
 	/* Only suspend io, if some operation is supposed to be locked out */
 	const bool do_suspend_io = flags & (BM_DONT_CLEAR|BM_DONT_SET|BM_DONT_TEST);
@@ -3541,7 +3557,7 @@ int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *
 		drbd_suspend_io(device);
 
 	drbd_bm_lock(device, why, flags);
-	rv = io_fn(device);
+	rv = io_fn(device, peer_device);
 	drbd_bm_unlock(device);
 
 	if (do_suspend_io)
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 60757ac31701..1a5d3d72d91d 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -1053,7 +1053,7 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 			 la_size_changed ? "size changed" : "md moved");
 		/* next line implicitly does drbd_suspend_io()+drbd_resume_io() */
 		drbd_bitmap_io(device, md_moved ? &drbd_bm_write_all : &drbd_bm_write,
-			       "size changed", BM_LOCKED_MASK);
+			       "size changed", BM_LOCKED_MASK, NULL);
 
 		/* on-disk bitmap and activity log is authoritative again
 		 * (unless there was an IO error meanwhile...) */
@@ -1615,7 +1615,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 			drbd_send_sync_param(peer_device);
 	}
 
-	kvfree_rcu(old_disk_conf);
+	kvfree_rcu_mightsleep(old_disk_conf);
 	kfree(old_plan);
 	mod_timer(&device->request_timer, jiffies + HZ);
 	goto success;
@@ -2027,13 +2027,15 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 		drbd_info(device, "Assuming that all blocks are out of sync "
 		     "(aka FullSync)\n");
 		if (drbd_bitmap_io(device, &drbd_bmio_set_n_write,
-			"set_n_write from attaching", BM_LOCKED_MASK)) {
+			"set_n_write from attaching", BM_LOCKED_MASK,
+			NULL)) {
 			retcode = ERR_IO_MD_DISK;
 			goto force_diskless_dec;
 		}
 	} else {
 		if (drbd_bitmap_io(device, &drbd_bm_read,
-			"read from attaching", BM_LOCKED_MASK)) {
+			"read from attaching", BM_LOCKED_MASK,
+			NULL)) {
 			retcode = ERR_IO_MD_DISK;
 			goto force_diskless_dec;
 		}
@@ -2446,7 +2448,7 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
 
 	mutex_unlock(&connection->resource->conf_update);
 	mutex_unlock(&connection->data.mutex);
-	kvfree_rcu(old_net_conf);
+	kvfree_rcu_mightsleep(old_net_conf);
 
 	if (connection->cstate >= C_WF_REPORT_PARAMS) {
 		struct drbd_peer_device *peer_device;
@@ -2860,7 +2862,7 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
 		new_disk_conf->disk_size = (sector_t)rs.resize_size;
 		rcu_assign_pointer(device->ldev->disk_conf, new_disk_conf);
 		mutex_unlock(&device->resource->conf_update);
-		kvfree_rcu(old_disk_conf);
+		kvfree_rcu_mightsleep(old_disk_conf);
 		new_disk_conf = NULL;
 	}
 
@@ -2972,7 +2974,7 @@ int drbd_adm_invalidate(struct sk_buff *skb, struct genl_info *info)
 		retcode = drbd_request_state(device, NS(disk, D_INCONSISTENT));
 		if (retcode >= SS_SUCCESS) {
 			if (drbd_bitmap_io(device, &drbd_bmio_set_n_write,
-				"set_n_write from invalidate", BM_LOCKED_MASK))
+				"set_n_write from invalidate", BM_LOCKED_MASK, NULL))
 				retcode = ERR_IO_MD_DISK;
 		}
 	} else
@@ -3005,11 +3007,12 @@ out:
 	return 0;
 }
 
-static int drbd_bmio_set_susp_al(struct drbd_device *device) __must_hold(local)
+static int drbd_bmio_set_susp_al(struct drbd_device *device,
+		struct drbd_peer_device *peer_device) __must_hold(local)
 {
 	int rv;
 
-	rv = drbd_bmio_set_n_write(device);
+	rv = drbd_bmio_set_n_write(device, peer_device);
 	drbd_suspend_al(device);
 	return rv;
 }
@@ -3052,7 +3055,7 @@ int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info)
 		if (retcode >= SS_SUCCESS) {
 			if (drbd_bitmap_io(device, &drbd_bmio_set_susp_al,
 				"set_n_write from invalidate_peer",
-				BM_LOCKED_SET_ALLOWED))
+				BM_LOCKED_SET_ALLOWED, NULL))
 				retcode = ERR_IO_MD_DISK;
 		}
 	} else
@@ -4148,7 +4151,7 @@ int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info)
 
 	if (args.clear_bm) {
 		err = drbd_bitmap_io(device, &drbd_bmio_clear_n_write,
-			"clear_n_write from new_c_uuid", BM_LOCKED_MASK);
+			"clear_n_write from new_c_uuid", BM_LOCKED_MASK, NULL);
 		if (err) {
 			drbd_err(device, "Writing bitmap failed with %d\n", err);
 			retcode = ERR_IO_MD_DISK;
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 757f4692b5bd..719a7260c22b 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -2044,11 +2044,11 @@ static int e_end_resync_block(struct drbd_work *w, int unused)
 	D_ASSERT(device, drbd_interval_empty(&peer_req->i));
 
 	if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
-		drbd_set_in_sync(device, sector, peer_req->i.size);
+		drbd_set_in_sync(peer_device, sector, peer_req->i.size);
 		err = drbd_send_ack(peer_device, P_RS_WRITE_ACK, peer_req);
 	} else {
 		/* Record failure to sync */
-		drbd_rs_failed_io(device, sector, peer_req->i.size);
+		drbd_rs_failed_io(peer_device, sector, peer_req->i.size);
 
 		err  = drbd_send_ack(peer_device, P_NEG_ACK, peer_req);
 	}
@@ -2067,7 +2067,7 @@ static int recv_resync_read(struct drbd_peer_device *peer_device, sector_t secto
 	if (!peer_req)
 		goto fail;
 
-	dec_rs_pending(device);
+	dec_rs_pending(peer_device);
 
 	inc_unacked(device);
 	/* corresponding dec_unacked() in e_end_resync_block()
@@ -2138,7 +2138,7 @@ static int receive_DataReply(struct drbd_connection *connection, struct packet_i
 
 	err = recv_dless_read(peer_device, req, sector, pi->size);
 	if (!err)
-		req_mod(req, DATA_RECEIVED);
+		req_mod(req, DATA_RECEIVED, peer_device);
 	/* else: nothing. handled from drbd_disconnect...
 	 * I don't think we may complete this just yet
 	 * in case we are "on-disconnect: freeze" */
@@ -2196,7 +2196,7 @@ static void restart_conflicting_writes(struct drbd_device *device,
 			continue;
 		/* as it is RQ_POSTPONED, this will cause it to
 		 * be queued on the retry workqueue. */
-		__req_mod(req, CONFLICT_RESOLVED, NULL);
+		__req_mod(req, CONFLICT_RESOLVED, NULL, NULL);
 	}
 }
 
@@ -2220,7 +2220,7 @@ static int e_end_block(struct drbd_work *w, int cancel)
 				P_RS_WRITE_ACK : P_WRITE_ACK;
 			err = drbd_send_ack(peer_device, pcmd, peer_req);
 			if (pcmd == P_RS_WRITE_ACK)
-				drbd_set_in_sync(device, sector, peer_req->i.size);
+				drbd_set_in_sync(peer_device, sector, peer_req->i.size);
 		} else {
 			err = drbd_send_ack(peer_device, P_NEG_ACK, peer_req);
 			/* we expect it to be marked out of sync anyways...
@@ -2420,6 +2420,7 @@ static blk_opf_t wire_flags_to_bio(struct drbd_connection *connection, u32 dpf)
 static void fail_postponed_requests(struct drbd_device *device, sector_t sector,
 				    unsigned int size)
 {
+	struct drbd_peer_device *peer_device = first_peer_device(device);
 	struct drbd_interval *i;
 
     repeat:
@@ -2433,7 +2434,7 @@ static void fail_postponed_requests(struct drbd_device *device, sector_t sector,
 		if (!(req->rq_state & RQ_POSTPONED))
 			continue;
 		req->rq_state &= ~RQ_POSTPONED;
-		__req_mod(req, NEG_ACKED, &m);
+		__req_mod(req, NEG_ACKED, peer_device, &m);
 		spin_unlock_irq(&device->resource->req_lock);
 		if (m.bio)
 			complete_master_bio(device, &m);
@@ -2690,7 +2691,7 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info *
 
 	if (device->state.pdsk < D_INCONSISTENT) {
 		/* In case we have the only disk of the cluster, */
-		drbd_set_out_of_sync(device, peer_req->i.sector, peer_req->i.size);
+		drbd_set_out_of_sync(peer_device, peer_req->i.sector, peer_req->i.size);
 		peer_req->flags &= ~EE_MAY_SET_IN_SYNC;
 		drbd_al_begin_io(device, &peer_req->i);
 		peer_req->flags |= EE_CALL_AL_COMPLETE_IO;
@@ -2729,9 +2730,10 @@ out_interrupted:
  * The current sync rate used here uses only the most recent two step marks,
  * to have a short time average so we can react faster.
  */
-bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector,
+bool drbd_rs_should_slow_down(struct drbd_peer_device *peer_device, sector_t sector,
 		bool throttle_if_app_is_waiting)
 {
+	struct drbd_device *device = peer_device->device;
 	struct lc_element *tmp;
 	bool throttle = drbd_rs_c_min_rate_throttle(device);
 
@@ -2843,7 +2845,7 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet
 			break;
 		case P_OV_REPLY:
 			verb = 0;
-			dec_rs_pending(device);
+			dec_rs_pending(peer_device);
 			drbd_send_ack_ex(peer_device, P_OV_RESULT, sector, size, ID_IN_SYNC);
 			break;
 		default:
@@ -2914,7 +2916,7 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet
 			/* track progress, we may need to throttle */
 			atomic_add(size >> 9, &device->rs_sect_in);
 			peer_req->w.cb = w_e_end_ov_reply;
-			dec_rs_pending(device);
+			dec_rs_pending(peer_device);
 			/* drbd_rs_begin_io done when we sent this request,
 			 * but accounting still needs to be done. */
 			goto submit_for_resync;
@@ -2977,7 +2979,7 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet
 
 	update_receiver_timing_details(connection, drbd_rs_should_slow_down);
 	if (device->state.peer != R_PRIMARY
-	&& drbd_rs_should_slow_down(device, sector, false))
+	&& drbd_rs_should_slow_down(peer_device, sector, false))
 		schedule_timeout_uninterruptible(HZ/10);
 	update_receiver_timing_details(connection, drbd_rs_begin_io);
 	if (drbd_rs_begin_io(device, sector))
@@ -3226,10 +3228,11 @@ static void drbd_uuid_dump(struct drbd_device *device, char *text, u64 *uuid,
 -1096   requires proto 96
  */
 
-static int drbd_uuid_compare(struct drbd_device *const device, enum drbd_role const peer_role, int *rule_nr) __must_hold(local)
+static int drbd_uuid_compare(struct drbd_peer_device *const peer_device,
+		enum drbd_role const peer_role, int *rule_nr) __must_hold(local)
 {
-	struct drbd_peer_device *const peer_device = first_peer_device(device);
-	struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL;
+	struct drbd_connection *const connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
 	u64 self, peer;
 	int i, j;
 
@@ -3465,7 +3468,7 @@ static enum drbd_conns drbd_sync_handshake(struct drbd_peer_device *peer_device,
 	drbd_uuid_dump(device, "peer", device->p_uuid,
 		       device->p_uuid[UI_SIZE], device->p_uuid[UI_FLAGS]);
 
-	hg = drbd_uuid_compare(device, peer_role, &rule_nr);
+	hg = drbd_uuid_compare(peer_device, peer_role, &rule_nr);
 	spin_unlock_irq(&device->ldev->md.uuid_lock);
 
 	drbd_info(device, "uuid_compare()=%d by rule %d\n", hg, rule_nr);
@@ -3591,7 +3594,7 @@ static enum drbd_conns drbd_sync_handshake(struct drbd_peer_device *peer_device,
 	if (abs(hg) >= 2) {
 		drbd_info(device, "Writing the whole bitmap, full sync required after drbd_sync_handshake.\n");
 		if (drbd_bitmap_io(device, &drbd_bmio_set_n_write, "set_n_write from sync_handshake",
-					BM_LOCKED_SET_ALLOWED))
+					BM_LOCKED_SET_ALLOWED, NULL))
 			return C_MASK;
 	}
 
@@ -3759,7 +3762,7 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in
 		drbd_info(connection, "peer data-integrity-alg: %s\n",
 			  integrity_alg[0] ? integrity_alg : "(none)");
 
-	kvfree_rcu(old_net_conf);
+	kvfree_rcu_mightsleep(old_net_conf);
 	return 0;
 
 disconnect_rcu_unlock:
@@ -4127,7 +4130,7 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info
 
 			rcu_assign_pointer(device->ldev->disk_conf, new_disk_conf);
 			mutex_unlock(&connection->resource->conf_update);
-			kvfree_rcu(old_disk_conf);
+			kvfree_rcu_mightsleep(old_disk_conf);
 
 			drbd_info(device, "Peer sets u_size to %lu sectors (old: %lu)\n",
 				 (unsigned long)p_usize, (unsigned long)my_usize);
@@ -4270,7 +4273,7 @@ static int receive_uuids(struct drbd_connection *connection, struct packet_info
 			drbd_info(device, "Accepted new current UUID, preparing to skip initial sync\n");
 			drbd_bitmap_io(device, &drbd_bmio_clear_n_write,
 					"clear_n_write from receive_uuids",
-					BM_LOCKED_TEST_ALLOWED);
+					BM_LOCKED_TEST_ALLOWED, NULL);
 			_drbd_uuid_set(device, UI_CURRENT, p_uuid[UI_CURRENT]);
 			_drbd_uuid_set(device, UI_BITMAP, 0);
 			_drbd_set_state(_NS2(device, disk, D_UP_TO_DATE, pdsk, D_UP_TO_DATE),
@@ -4448,7 +4451,7 @@ static int receive_state(struct drbd_connection *connection, struct packet_info
 		else if (os.conn >= C_SYNC_SOURCE &&
 			 peer_state.conn == C_CONNECTED) {
 			if (drbd_bm_total_weight(device) <= device->rs_failed)
-				drbd_resync_finished(device);
+				drbd_resync_finished(peer_device);
 			return 0;
 		}
 	}
@@ -4456,8 +4459,8 @@ static int receive_state(struct drbd_connection *connection, struct packet_info
 	/* explicit verify finished notification, stop sector reached. */
 	if (os.conn == C_VERIFY_T && os.disk == D_UP_TO_DATE &&
 	    peer_state.conn == C_CONNECTED && real_peer_disk == D_UP_TO_DATE) {
-		ov_out_of_sync_print(device);
-		drbd_resync_finished(device);
+		ov_out_of_sync_print(peer_device);
+		drbd_resync_finished(peer_device);
 		return 0;
 	}
 
@@ -4766,11 +4769,11 @@ decode_bitmap_c(struct drbd_peer_device *peer_device,
 	return -EIO;
 }
 
-void INFO_bm_xfer_stats(struct drbd_device *device,
+void INFO_bm_xfer_stats(struct drbd_peer_device *peer_device,
 		const char *direction, struct bm_xfer_ctx *c)
 {
 	/* what would it take to transfer it "plaintext" */
-	unsigned int header_size = drbd_header_size(first_peer_device(device)->connection);
+	unsigned int header_size = drbd_header_size(peer_device->connection);
 	unsigned int data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
 	unsigned int plain =
 		header_size * (DIV_ROUND_UP(c->bm_words, data_size) + 1) +
@@ -4794,7 +4797,7 @@ void INFO_bm_xfer_stats(struct drbd_device *device,
 		r = 1000;
 
 	r = 1000 - r;
-	drbd_info(device, "%s bitmap stats [Bytes(packets)]: plain %u(%u), RLE %u(%u), "
+	drbd_info(peer_device, "%s bitmap stats [Bytes(packets)]: plain %u(%u), RLE %u(%u), "
 	     "total %u; compression: %u.%u%%\n",
 			direction,
 			c->bytes[1], c->packets[1],
@@ -4872,12 +4875,12 @@ static int receive_bitmap(struct drbd_connection *connection, struct packet_info
 			goto out;
 	}
 
-	INFO_bm_xfer_stats(device, "receive", &c);
+	INFO_bm_xfer_stats(peer_device, "receive", &c);
 
 	if (device->state.conn == C_WF_BITMAP_T) {
 		enum drbd_state_rv rv;
 
-		err = drbd_send_bitmap(device);
+		err = drbd_send_bitmap(device, peer_device);
 		if (err)
 			goto out;
 		/* Omit CS_ORDERED with this state transition to avoid deadlocks. */
@@ -4935,7 +4938,7 @@ static int receive_out_of_sync(struct drbd_connection *connection, struct packet
 				drbd_conn_str(device->state.conn));
 	}
 
-	drbd_set_out_of_sync(device, be64_to_cpu(p->sector), be32_to_cpu(p->blksize));
+	drbd_set_out_of_sync(peer_device, be64_to_cpu(p->sector), be32_to_cpu(p->blksize));
 
 	return 0;
 }
@@ -4956,7 +4959,7 @@ static int receive_rs_deallocated(struct drbd_connection *connection, struct pac
 	sector = be64_to_cpu(p->sector);
 	size = be32_to_cpu(p->blksize);
 
-	dec_rs_pending(device);
+	dec_rs_pending(peer_device);
 
 	if (get_ldev(device)) {
 		struct drbd_peer_request *peer_req;
@@ -5214,7 +5217,7 @@ static int drbd_disconnected(struct drbd_peer_device *peer_device)
 
 	if (get_ldev(device)) {
 		drbd_bitmap_io(device, &drbd_bm_write_copy_pages,
-				"write from disconnected", BM_LOCKED_CHANGE_ALLOWED);
+				"write from disconnected", BM_LOCKED_CHANGE_ALLOWED, NULL);
 		put_ldev(device);
 	}
 
@@ -5648,22 +5651,23 @@ static int got_IsInSync(struct drbd_connection *connection, struct packet_info *
 
 	if (get_ldev(device)) {
 		drbd_rs_complete_io(device, sector);
-		drbd_set_in_sync(device, sector, blksize);
+		drbd_set_in_sync(peer_device, sector, blksize);
 		/* rs_same_csums is supposed to count in units of BM_BLOCK_SIZE */
 		device->rs_same_csum += (blksize >> BM_BLOCK_SHIFT);
 		put_ldev(device);
 	}
-	dec_rs_pending(device);
+	dec_rs_pending(peer_device);
 	atomic_add(blksize >> 9, &device->rs_sect_in);
 
 	return 0;
 }
 
 static int
-validate_req_change_req_state(struct drbd_device *device, u64 id, sector_t sector,
+validate_req_change_req_state(struct drbd_peer_device *peer_device, u64 id, sector_t sector,
 			      struct rb_root *root, const char *func,
 			      enum drbd_req_event what, bool missing_ok)
 {
+	struct drbd_device *device = peer_device->device;
 	struct drbd_request *req;
 	struct bio_and_error m;
 
@@ -5673,7 +5677,7 @@ validate_req_change_req_state(struct drbd_device *device, u64 id, sector_t secto
 		spin_unlock_irq(&device->resource->req_lock);
 		return -EIO;
 	}
-	__req_mod(req, what, &m);
+	__req_mod(req, what, peer_device, &m);
 	spin_unlock_irq(&device->resource->req_lock);
 
 	if (m.bio)
@@ -5698,8 +5702,8 @@ static int got_BlockAck(struct drbd_connection *connection, struct packet_info *
 	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
 
 	if (p->block_id == ID_SYNCER) {
-		drbd_set_in_sync(device, sector, blksize);
-		dec_rs_pending(device);
+		drbd_set_in_sync(peer_device, sector, blksize);
+		dec_rs_pending(peer_device);
 		return 0;
 	}
 	switch (pi->cmd) {
@@ -5722,7 +5726,7 @@ static int got_BlockAck(struct drbd_connection *connection, struct packet_info *
 		BUG();
 	}
 
-	return validate_req_change_req_state(device, p->block_id, sector,
+	return validate_req_change_req_state(peer_device, p->block_id, sector,
 					     &device->write_requests, __func__,
 					     what, false);
 }
@@ -5744,12 +5748,12 @@ static int got_NegAck(struct drbd_connection *connection, struct packet_info *pi
 	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
 
 	if (p->block_id == ID_SYNCER) {
-		dec_rs_pending(device);
-		drbd_rs_failed_io(device, sector, size);
+		dec_rs_pending(peer_device);
+		drbd_rs_failed_io(peer_device, sector, size);
 		return 0;
 	}
 
-	err = validate_req_change_req_state(device, p->block_id, sector,
+	err = validate_req_change_req_state(peer_device, p->block_id, sector,
 					    &device->write_requests, __func__,
 					    NEG_ACKED, true);
 	if (err) {
@@ -5758,7 +5762,7 @@ static int got_NegAck(struct drbd_connection *connection, struct packet_info *pi
 		   request is no longer in the collision hash. */
 		/* In Protocol B we might already have got a P_RECV_ACK
 		   but then get a P_NEG_ACK afterwards. */
-		drbd_set_out_of_sync(device, sector, size);
+		drbd_set_out_of_sync(peer_device, sector, size);
 	}
 	return 0;
 }
@@ -5780,7 +5784,7 @@ static int got_NegDReply(struct drbd_connection *connection, struct packet_info
 	drbd_err(device, "Got NegDReply; Sector %llus, len %u.\n",
 	    (unsigned long long)sector, be32_to_cpu(p->blksize));
 
-	return validate_req_change_req_state(device, p->block_id, sector,
+	return validate_req_change_req_state(peer_device, p->block_id, sector,
 					     &device->read_requests, __func__,
 					     NEG_ACKED, false);
 }
@@ -5803,13 +5807,13 @@ static int got_NegRSDReply(struct drbd_connection *connection, struct packet_inf
 
 	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
 
-	dec_rs_pending(device);
+	dec_rs_pending(peer_device);
 
 	if (get_ldev_if_state(device, D_FAILED)) {
 		drbd_rs_complete_io(device, sector);
 		switch (pi->cmd) {
 		case P_NEG_RS_DREPLY:
-			drbd_rs_failed_io(device, sector, size);
+			drbd_rs_failed_io(peer_device, sector, size);
 			break;
 		case P_RS_CANCEL:
 			break;
@@ -5866,21 +5870,21 @@ static int got_OVResult(struct drbd_connection *connection, struct packet_info *
 	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
 
 	if (be64_to_cpu(p->block_id) == ID_OUT_OF_SYNC)
-		drbd_ov_out_of_sync_found(device, sector, size);
+		drbd_ov_out_of_sync_found(peer_device, sector, size);
 	else
-		ov_out_of_sync_print(device);
+		ov_out_of_sync_print(peer_device);
 
 	if (!get_ldev(device))
 		return 0;
 
 	drbd_rs_complete_io(device, sector);
-	dec_rs_pending(device);
+	dec_rs_pending(peer_device);
 
 	--device->ov_left;
 
 	/* let's advance progress step marks only for every other megabyte */
 	if ((device->ov_left & 0x200) == 0x200)
-		drbd_advance_rs_marks(device, device->ov_left);
+		drbd_advance_rs_marks(peer_device, device->ov_left);
 
 	if (device->ov_left == 0) {
 		dw = kmalloc(sizeof(*dw), GFP_NOIO);
@@ -5890,8 +5894,8 @@ static int got_OVResult(struct drbd_connection *connection, struct packet_info *
 			drbd_queue_work(&peer_device->connection->sender_work, &dw->w);
 		} else {
 			drbd_err(device, "kmalloc(dw) failed.");
-			ov_out_of_sync_print(device);
-			drbd_resync_finished(device);
+			ov_out_of_sync_print(peer_device);
+			drbd_resync_finished(peer_device);
 		}
 	}
 	put_ldev(device);
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index e36216d50753..380e6584a4ee 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -122,12 +122,13 @@ void drbd_req_destroy(struct kref *kref)
 		 * before it even was submitted or sent.
 		 * In that case we do not want to touch the bitmap at all.
 		 */
+		struct drbd_peer_device *peer_device = first_peer_device(device);
 		if ((s & (RQ_POSTPONED|RQ_LOCAL_MASK|RQ_NET_MASK)) != RQ_POSTPONED) {
 			if (!(s & RQ_NET_OK) || !(s & RQ_LOCAL_OK))
-				drbd_set_out_of_sync(device, req->i.sector, req->i.size);
+				drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size);
 
 			if ((s & RQ_NET_OK) && (s & RQ_LOCAL_OK) && (s & RQ_NET_SIS))
-				drbd_set_in_sync(device, req->i.sector, req->i.size);
+				drbd_set_in_sync(peer_device, req->i.sector, req->i.size);
 		}
 
 		/* one might be tempted to move the drbd_al_complete_io
@@ -552,12 +553,15 @@ static inline bool is_pending_write_protocol_A(struct drbd_request *req)
  *  happen "atomically" within the req_lock,
  *  and it enforces that we have to think in a very structured manner
  *  about the "events" that may happen to a request during its life time ...
+ *
+ *
+ * peer_device == NULL means local disk
  */
 int __req_mod(struct drbd_request *req, enum drbd_req_event what,
+		struct drbd_peer_device *peer_device,
 		struct bio_and_error *m)
 {
 	struct drbd_device *const device = req->device;
-	struct drbd_peer_device *const peer_device = first_peer_device(device);
 	struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL;
 	struct net_conf *nc;
 	int p, rv = 0;
@@ -617,7 +621,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		break;
 
 	case READ_COMPLETED_WITH_ERROR:
-		drbd_set_out_of_sync(device, req->i.sector, req->i.size);
+		drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size);
 		drbd_report_io_error(device, req);
 		__drbd_chk_io_error(device, DRBD_READ_ERROR);
 		fallthrough;
@@ -1100,6 +1104,7 @@ static bool drbd_should_send_out_of_sync(union drbd_dev_state s)
 static int drbd_process_write_request(struct drbd_request *req)
 {
 	struct drbd_device *device = req->device;
+	struct drbd_peer_device *peer_device = first_peer_device(device);
 	int remote, send_oos;
 
 	remote = drbd_should_do_remote(device->state);
@@ -1115,7 +1120,7 @@ static int drbd_process_write_request(struct drbd_request *req)
 		/* The only size==0 bios we expect are empty flushes. */
 		D_ASSERT(device, req->master_bio->bi_opf & REQ_PREFLUSH);
 		if (remote)
-			_req_mod(req, QUEUE_AS_DRBD_BARRIER);
+			_req_mod(req, QUEUE_AS_DRBD_BARRIER, peer_device);
 		return remote;
 	}
 
@@ -1125,10 +1130,10 @@ static int drbd_process_write_request(struct drbd_request *req)
 	D_ASSERT(device, !(remote && send_oos));
 
 	if (remote) {
-		_req_mod(req, TO_BE_SENT);
-		_req_mod(req, QUEUE_FOR_NET_WRITE);
-	} else if (drbd_set_out_of_sync(device, req->i.sector, req->i.size))
-		_req_mod(req, QUEUE_FOR_SEND_OOS);
+		_req_mod(req, TO_BE_SENT, peer_device);
+		_req_mod(req, QUEUE_FOR_NET_WRITE, peer_device);
+	} else if (drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size))
+		_req_mod(req, QUEUE_FOR_SEND_OOS, peer_device);
 
 	return remote;
 }
@@ -1312,6 +1317,7 @@ static void drbd_update_plug(struct drbd_plug_cb *plug, struct drbd_request *req
 static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request *req)
 {
 	struct drbd_resource *resource = device->resource;
+	struct drbd_peer_device *peer_device = first_peer_device(device);
 	const int rw = bio_data_dir(req->master_bio);
 	struct bio_and_error m = { NULL, };
 	bool no_remote = false;
@@ -1375,8 +1381,8 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request
 		/* We either have a private_bio, or we can read from remote.
 		 * Otherwise we had done the goto nodata above. */
 		if (req->private_bio == NULL) {
-			_req_mod(req, TO_BE_SENT);
-			_req_mod(req, QUEUE_FOR_NET_READ);
+			_req_mod(req, TO_BE_SENT, peer_device);
+			_req_mod(req, QUEUE_FOR_NET_READ, peer_device);
 		} else
 			no_remote = true;
 	}
@@ -1397,7 +1403,7 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request
 		req->pre_submit_jif = jiffies;
 		list_add_tail(&req->req_pending_local,
 			&device->pending_completion[rw == WRITE]);
-		_req_mod(req, TO_BE_SUBMITTED);
+		_req_mod(req, TO_BE_SUBMITTED, NULL);
 		/* but we need to give up the spinlock to submit */
 		submit_private_bio = true;
 	} else if (no_remote) {
diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h
index b4017b5c3fbc..9ae860e7591b 100644
--- a/drivers/block/drbd/drbd_req.h
+++ b/drivers/block/drbd/drbd_req.h
@@ -267,6 +267,7 @@ struct bio_and_error {
 extern void start_new_tl_epoch(struct drbd_connection *connection);
 extern void drbd_req_destroy(struct kref *kref);
 extern int __req_mod(struct drbd_request *req, enum drbd_req_event what,
+		struct drbd_peer_device *peer_device,
 		struct bio_and_error *m);
 extern void complete_master_bio(struct drbd_device *device,
 		struct bio_and_error *m);
@@ -280,14 +281,15 @@ extern void drbd_restart_request(struct drbd_request *req);
 
 /* use this if you don't want to deal with calling complete_master_bio()
  * outside the spinlock, e.g. when walking some list on cleanup. */
-static inline int _req_mod(struct drbd_request *req, enum drbd_req_event what)
+static inline int _req_mod(struct drbd_request *req, enum drbd_req_event what,
+		struct drbd_peer_device *peer_device)
 {
 	struct drbd_device *device = req->device;
 	struct bio_and_error m;
 	int rv;
 
 	/* __req_mod possibly frees req, do not touch req after that! */
-	rv = __req_mod(req, what, &m);
+	rv = __req_mod(req, what, peer_device, &m);
 	if (m.bio)
 		complete_master_bio(device, &m);
 
@@ -299,7 +301,8 @@ static inline int _req_mod(struct drbd_request *req, enum drbd_req_event what)
  * of the lower level driver completion callback, so we need to
  * spin_lock_irqsave here. */
 static inline int req_mod(struct drbd_request *req,
-		enum drbd_req_event what)
+		enum drbd_req_event what,
+		struct drbd_peer_device *peer_device)
 {
 	unsigned long flags;
 	struct drbd_device *device = req->device;
@@ -307,7 +310,7 @@ static inline int req_mod(struct drbd_request *req,
 	int rv;
 
 	spin_lock_irqsave(&device->resource->req_lock, flags);
-	rv = __req_mod(req, what, &m);
+	rv = __req_mod(req, what, peer_device, &m);
 	spin_unlock_irqrestore(&device->resource->req_lock, flags);
 
 	if (m.bio)
diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index 75d13ea0024f..287a8d1d3f70 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -1222,9 +1222,11 @@ void drbd_resume_al(struct drbd_device *device)
 }
 
 /* helper for _drbd_set_state */
-static void set_ov_position(struct drbd_device *device, enum drbd_conns cs)
+static void set_ov_position(struct drbd_peer_device *peer_device, enum drbd_conns cs)
 {
-	if (first_peer_device(device)->connection->agreed_pro_version < 90)
+	struct drbd_device *device = peer_device->device;
+
+	if (peer_device->connection->agreed_pro_version < 90)
 		device->ov_start_sector = 0;
 	device->rs_total = drbd_bm_bits(device);
 	device->ov_position = 0;
@@ -1387,7 +1389,7 @@ _drbd_set_state(struct drbd_device *device, union drbd_state ns,
 		unsigned long now = jiffies;
 		int i;
 
-		set_ov_position(device, ns.conn);
+		set_ov_position(peer_device, ns.conn);
 		device->rs_start = now;
 		device->rs_last_sect_ev = 0;
 		device->ov_last_oos_size = 0;
@@ -1398,7 +1400,7 @@ _drbd_set_state(struct drbd_device *device, union drbd_state ns,
 			device->rs_mark_time[i] = now;
 		}
 
-		drbd_rs_controller_reset(device);
+		drbd_rs_controller_reset(peer_device);
 
 		if (ns.conn == C_VERIFY_S) {
 			drbd_info(device, "Starting Online Verify from sector %llu\n",
@@ -1518,8 +1520,9 @@ static void abw_start_sync(struct drbd_device *device, int rv)
 }
 
 int drbd_bitmap_io_from_worker(struct drbd_device *device,
-		int (*io_fn)(struct drbd_device *),
-		char *why, enum bm_flag flags)
+		int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
+		char *why, enum bm_flag flags,
+		struct drbd_peer_device *peer_device)
 {
 	int rv;
 
@@ -1529,7 +1532,7 @@ int drbd_bitmap_io_from_worker(struct drbd_device *device,
 	atomic_inc(&device->suspend_cnt);
 
 	drbd_bm_lock(device, why, flags);
-	rv = io_fn(device);
+	rv = io_fn(device, peer_device);
 	drbd_bm_unlock(device);
 
 	drbd_resume_io(device);
@@ -1809,7 +1812,7 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os,
 	    device->state.conn == C_WF_BITMAP_S)
 		drbd_queue_bitmap_io(device, &drbd_send_bitmap, NULL,
 				"send_bitmap (WFBitMapS)",
-				BM_LOCKED_TEST_ALLOWED);
+				BM_LOCKED_TEST_ALLOWED, peer_device);
 
 	/* Lost contact to peer's copy of the data */
 	if (lost_contact_to_peer_data(os.pdsk, ns.pdsk)) {
@@ -1839,7 +1842,7 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os,
 			 * No harm done if the bitmap still changes,
 			 * redirtied pages will follow later. */
 			drbd_bitmap_io_from_worker(device, &drbd_bm_write,
-				"demote diskless peer", BM_LOCKED_SET_ALLOWED);
+				"demote diskless peer", BM_LOCKED_SET_ALLOWED, peer_device);
 		put_ldev(device);
 	}
 
@@ -1851,7 +1854,7 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os,
 		/* No changes to the bitmap expected this time, so assert that,
 		 * even though no harm was done if it did change. */
 		drbd_bitmap_io_from_worker(device, &drbd_bm_write,
-				"demote", BM_LOCKED_TEST_ALLOWED);
+				"demote", BM_LOCKED_TEST_ALLOWED, peer_device);
 		put_ldev(device);
 	}
 
@@ -1888,7 +1891,8 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os,
 		/* no other bitmap changes expected during this phase */
 		drbd_queue_bitmap_io(device,
 			&drbd_bmio_set_n_write, &abw_start_sync,
-			"set_n_write from StartingSync", BM_LOCKED_TEST_ALLOWED);
+			"set_n_write from StartingSync", BM_LOCKED_TEST_ALLOWED,
+			peer_device);
 
 	/* first half of local IO error, failure to attach,
 	 * or administrative detach */
@@ -2011,7 +2015,8 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os,
 	if ((os.conn > C_CONNECTED && os.conn < C_AHEAD) &&
 	    (ns.conn == C_CONNECTED || ns.conn >= C_AHEAD) && get_ldev(device)) {
 		drbd_queue_bitmap_io(device, &drbd_bm_write_copy_pages, NULL,
-			"write from resync_finished", BM_LOCKED_CHANGE_ALLOWED);
+			"write from resync_finished", BM_LOCKED_CHANGE_ALLOWED,
+			peer_device);
 		put_ldev(device);
 	}
 
@@ -2071,7 +2076,7 @@ static int w_after_conn_state_ch(struct drbd_work *w, int unused)
 		conn_free_crypto(connection);
 		mutex_unlock(&connection->resource->conf_update);
 
-		kvfree_rcu(old_conf);
+		kvfree_rcu_mightsleep(old_conf);
 	}
 
 	if (ns_max.susp_fen) {
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
index f46738040d6b..4352a50fbb3f 100644
--- a/drivers/block/drbd/drbd_worker.c
+++ b/drivers/block/drbd/drbd_worker.c
@@ -28,8 +28,8 @@
 #include "drbd_protocol.h"
 #include "drbd_req.h"
 
-static int make_ov_request(struct drbd_device *, int);
-static int make_resync_request(struct drbd_device *, int);
+static int make_ov_request(struct drbd_peer_device *, int);
+static int make_resync_request(struct drbd_peer_device *, int);
 
 /* endio handlers:
  *   drbd_md_endio (defined here)
@@ -124,7 +124,7 @@ void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(l
 		 * In case of a write error, send the neg ack anyways. */
 		if (!__test_and_set_bit(__EE_SEND_WRITE_ACK, &peer_req->flags))
 			inc_unacked(device);
-		drbd_set_out_of_sync(device, peer_req->i.sector, peer_req->i.size);
+		drbd_set_out_of_sync(peer_device, peer_req->i.sector, peer_req->i.size);
 	}
 
 	spin_lock_irqsave(&device->resource->req_lock, flags);
@@ -276,7 +276,7 @@ void drbd_request_endio(struct bio *bio)
 
 	/* not req_mod(), we need irqsave here! */
 	spin_lock_irqsave(&device->resource->req_lock, flags);
-	__req_mod(req, what, &m);
+	__req_mod(req, what, NULL, &m);
 	spin_unlock_irqrestore(&device->resource->req_lock, flags);
 	put_ldev(device);
 
@@ -363,7 +363,7 @@ static int w_e_send_csum(struct drbd_work *w, int cancel)
 		 * drbd_alloc_pages due to pp_in_use > max_buffers. */
 		drbd_free_peer_req(device, peer_req);
 		peer_req = NULL;
-		inc_rs_pending(device);
+		inc_rs_pending(peer_device);
 		err = drbd_send_drequest_csum(peer_device, sector, size,
 					      digest, digest_size,
 					      P_CSUM_RS_REQUEST);
@@ -430,10 +430,10 @@ int w_resync_timer(struct drbd_work *w, int cancel)
 
 	switch (device->state.conn) {
 	case C_VERIFY_S:
-		make_ov_request(device, cancel);
+		make_ov_request(first_peer_device(device), cancel);
 		break;
 	case C_SYNC_TARGET:
-		make_resync_request(device, cancel);
+		make_resync_request(first_peer_device(device), cancel);
 		break;
 	}
 
@@ -493,8 +493,9 @@ struct fifo_buffer *fifo_alloc(unsigned int fifo_size)
 	return fb;
 }
 
-static int drbd_rs_controller(struct drbd_device *device, unsigned int sect_in)
+static int drbd_rs_controller(struct drbd_peer_device *peer_device, unsigned int sect_in)
 {
+	struct drbd_device *device = peer_device->device;
 	struct disk_conf *dc;
 	unsigned int want;     /* The number of sectors we want in-flight */
 	int req_sect; /* Number of sectors to request in this turn */
@@ -545,8 +546,9 @@ static int drbd_rs_controller(struct drbd_device *device, unsigned int sect_in)
 	return req_sect;
 }
 
-static int drbd_rs_number_requests(struct drbd_device *device)
+static int drbd_rs_number_requests(struct drbd_peer_device *peer_device)
 {
+	struct drbd_device *device = peer_device->device;
 	unsigned int sect_in;  /* Number of sectors that came in since the last turn */
 	int number, mxb;
 
@@ -556,7 +558,7 @@ static int drbd_rs_number_requests(struct drbd_device *device)
 	rcu_read_lock();
 	mxb = drbd_get_max_buffers(device) / 2;
 	if (rcu_dereference(device->rs_plan_s)->size) {
-		number = drbd_rs_controller(device, sect_in) >> (BM_BLOCK_SHIFT - 9);
+		number = drbd_rs_controller(peer_device, sect_in) >> (BM_BLOCK_SHIFT - 9);
 		device->c_sync_rate = number * HZ * (BM_BLOCK_SIZE / 1024) / SLEEP_TIME;
 	} else {
 		device->c_sync_rate = rcu_dereference(device->ldev->disk_conf)->resync_rate;
@@ -580,9 +582,9 @@ static int drbd_rs_number_requests(struct drbd_device *device)
 	return number;
 }
 
-static int make_resync_request(struct drbd_device *const device, int cancel)
+static int make_resync_request(struct drbd_peer_device *const peer_device, int cancel)
 {
-	struct drbd_peer_device *const peer_device = first_peer_device(device);
+	struct drbd_device *const device = peer_device->device;
 	struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL;
 	unsigned long bit;
 	sector_t sector;
@@ -598,7 +600,7 @@ static int make_resync_request(struct drbd_device *const device, int cancel)
 
 	if (device->rs_total == 0) {
 		/* empty resync? */
-		drbd_resync_finished(device);
+		drbd_resync_finished(peer_device);
 		return 0;
 	}
 
@@ -618,7 +620,7 @@ static int make_resync_request(struct drbd_device *const device, int cancel)
 	}
 
 	max_bio_size = queue_max_hw_sectors(device->rq_queue) << 9;
-	number = drbd_rs_number_requests(device);
+	number = drbd_rs_number_requests(peer_device);
 	if (number <= 0)
 		goto requeue;
 
@@ -653,7 +655,7 @@ next_sector:
 
 		sector = BM_BIT_TO_SECT(bit);
 
-		if (drbd_try_rs_begin_io(device, sector)) {
+		if (drbd_try_rs_begin_io(peer_device, sector)) {
 			device->bm_resync_fo = bit;
 			goto requeue;
 		}
@@ -729,13 +731,13 @@ next_sector:
 		} else {
 			int err;
 
-			inc_rs_pending(device);
+			inc_rs_pending(peer_device);
 			err = drbd_send_drequest(peer_device,
 						 size == discard_granularity ? P_RS_THIN_REQ : P_RS_DATA_REQUEST,
 						 sector, size, ID_SYNCER);
 			if (err) {
 				drbd_err(device, "drbd_send_drequest() failed, aborting...\n");
-				dec_rs_pending(device);
+				dec_rs_pending(peer_device);
 				put_ldev(device);
 				return err;
 			}
@@ -760,8 +762,9 @@ next_sector:
 	return 0;
 }
 
-static int make_ov_request(struct drbd_device *device, int cancel)
+static int make_ov_request(struct drbd_peer_device *peer_device, int cancel)
 {
+	struct drbd_device *device = peer_device->device;
 	int number, i, size;
 	sector_t sector;
 	const sector_t capacity = get_capacity(device->vdisk);
@@ -770,7 +773,7 @@ static int make_ov_request(struct drbd_device *device, int cancel)
 	if (unlikely(cancel))
 		return 1;
 
-	number = drbd_rs_number_requests(device);
+	number = drbd_rs_number_requests(peer_device);
 
 	sector = device->ov_position;
 	for (i = 0; i < number; i++) {
@@ -788,7 +791,7 @@ static int make_ov_request(struct drbd_device *device, int cancel)
 
 		size = BM_BLOCK_SIZE;
 
-		if (drbd_try_rs_begin_io(device, sector)) {
+		if (drbd_try_rs_begin_io(peer_device, sector)) {
 			device->ov_position = sector;
 			goto requeue;
 		}
@@ -796,9 +799,9 @@ static int make_ov_request(struct drbd_device *device, int cancel)
 		if (sector + (size>>9) > capacity)
 			size = (capacity-sector)<<9;
 
-		inc_rs_pending(device);
+		inc_rs_pending(peer_device);
 		if (drbd_send_ov_request(first_peer_device(device), sector, size)) {
-			dec_rs_pending(device);
+			dec_rs_pending(peer_device);
 			return 0;
 		}
 		sector += BM_SECT_PER_BIT;
@@ -818,8 +821,8 @@ int w_ov_finished(struct drbd_work *w, int cancel)
 		container_of(w, struct drbd_device_work, w);
 	struct drbd_device *device = dw->device;
 	kfree(dw);
-	ov_out_of_sync_print(device);
-	drbd_resync_finished(device);
+	ov_out_of_sync_print(first_peer_device(device));
+	drbd_resync_finished(first_peer_device(device));
 
 	return 0;
 }
@@ -831,7 +834,7 @@ static int w_resync_finished(struct drbd_work *w, int cancel)
 	struct drbd_device *device = dw->device;
 	kfree(dw);
 
-	drbd_resync_finished(device);
+	drbd_resync_finished(first_peer_device(device));
 
 	return 0;
 }
@@ -846,9 +849,10 @@ static void ping_peer(struct drbd_device *device)
 		   test_bit(GOT_PING_ACK, &connection->flags) || device->state.conn < C_CONNECTED);
 }
 
-int drbd_resync_finished(struct drbd_device *device)
+int drbd_resync_finished(struct drbd_peer_device *peer_device)
 {
-	struct drbd_connection *connection = first_peer_device(device)->connection;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
 	unsigned long db, dt, dbdt;
 	unsigned long n_oos;
 	union drbd_state os, ns;
@@ -1129,7 +1133,7 @@ int w_e_end_rsdata_req(struct drbd_work *w, int cancel)
 		err = drbd_send_ack(peer_device, P_RS_CANCEL, peer_req);
 	} else if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
 		if (likely(device->state.pdsk >= D_INCONSISTENT)) {
-			inc_rs_pending(device);
+			inc_rs_pending(peer_device);
 			if (peer_req->flags & EE_RS_THIN_REQ && all_zero(peer_req))
 				err = drbd_send_rs_deallocated(peer_device, peer_req);
 			else
@@ -1148,7 +1152,7 @@ int w_e_end_rsdata_req(struct drbd_work *w, int cancel)
 		err = drbd_send_ack(peer_device, P_NEG_RS_DREPLY, peer_req);
 
 		/* update resync data with failure */
-		drbd_rs_failed_io(device, peer_req->i.sector, peer_req->i.size);
+		drbd_rs_failed_io(peer_device, peer_req->i.sector, peer_req->i.size);
 	}
 
 	dec_unacked(device);
@@ -1199,12 +1203,12 @@ int w_e_end_csum_rs_req(struct drbd_work *w, int cancel)
 		}
 
 		if (eq) {
-			drbd_set_in_sync(device, peer_req->i.sector, peer_req->i.size);
+			drbd_set_in_sync(peer_device, peer_req->i.sector, peer_req->i.size);
 			/* rs_same_csums unit is BM_BLOCK_SIZE */
 			device->rs_same_csum += peer_req->i.size >> BM_BLOCK_SHIFT;
 			err = drbd_send_ack(peer_device, P_RS_IS_IN_SYNC, peer_req);
 		} else {
-			inc_rs_pending(device);
+			inc_rs_pending(peer_device);
 			peer_req->block_id = ID_SYNCER; /* By setting block_id, digest pointer becomes invalid! */
 			peer_req->flags &= ~EE_HAS_DIGEST; /* This peer request no longer has a digest pointer */
 			kfree(di);
@@ -1257,10 +1261,10 @@ int w_e_end_ov_req(struct drbd_work *w, int cancel)
 	 * drbd_alloc_pages due to pp_in_use > max_buffers. */
 	drbd_free_peer_req(device, peer_req);
 	peer_req = NULL;
-	inc_rs_pending(device);
+	inc_rs_pending(peer_device);
 	err = drbd_send_drequest_csum(peer_device, sector, size, digest, digest_size, P_OV_REPLY);
 	if (err)
-		dec_rs_pending(device);
+		dec_rs_pending(peer_device);
 	kfree(digest);
 
 out:
@@ -1270,15 +1274,16 @@ out:
 	return err;
 }
 
-void drbd_ov_out_of_sync_found(struct drbd_device *device, sector_t sector, int size)
+void drbd_ov_out_of_sync_found(struct drbd_peer_device *peer_device, sector_t sector, int size)
 {
+	struct drbd_device *device = peer_device->device;
 	if (device->ov_last_oos_start + device->ov_last_oos_size == sector) {
 		device->ov_last_oos_size += size>>9;
 	} else {
 		device->ov_last_oos_start = sector;
 		device->ov_last_oos_size = size>>9;
 	}
-	drbd_set_out_of_sync(device, sector, size);
+	drbd_set_out_of_sync(peer_device, sector, size);
 }
 
 int w_e_end_ov_reply(struct drbd_work *w, int cancel)
@@ -1328,9 +1333,9 @@ int w_e_end_ov_reply(struct drbd_work *w, int cancel)
 	 * drbd_alloc_pages due to pp_in_use > max_buffers. */
 	drbd_free_peer_req(device, peer_req);
 	if (!eq)
-		drbd_ov_out_of_sync_found(device, sector, size);
+		drbd_ov_out_of_sync_found(peer_device, sector, size);
 	else
-		ov_out_of_sync_print(device);
+		ov_out_of_sync_print(peer_device);
 
 	err = drbd_send_ack_ex(peer_device, P_OV_RESULT, sector, size,
 			       eq ? ID_IN_SYNC : ID_OUT_OF_SYNC);
@@ -1341,14 +1346,14 @@ int w_e_end_ov_reply(struct drbd_work *w, int cancel)
 
 	/* let's advance progress step marks only for every other megabyte */
 	if ((device->ov_left & 0x200) == 0x200)
-		drbd_advance_rs_marks(device, device->ov_left);
+		drbd_advance_rs_marks(peer_device, device->ov_left);
 
 	stop_sector_reached = verify_can_do_stop_sector(device) &&
 		(sector + (size>>9)) >= device->ov_stop_sector;
 
 	if (device->ov_left == 0 || stop_sector_reached) {
-		ov_out_of_sync_print(device);
-		drbd_resync_finished(device);
+		ov_out_of_sync_print(peer_device);
+		drbd_resync_finished(peer_device);
 	}
 
 	return err;
@@ -1425,7 +1430,7 @@ int w_send_out_of_sync(struct drbd_work *w, int cancel)
 	int err;
 
 	if (unlikely(cancel)) {
-		req_mod(req, SEND_CANCELED);
+		req_mod(req, SEND_CANCELED, peer_device);
 		return 0;
 	}
 	req->pre_send_jif = jiffies;
@@ -1437,7 +1442,7 @@ int w_send_out_of_sync(struct drbd_work *w, int cancel)
 	maybe_send_barrier(connection, req->epoch);
 
 	err = drbd_send_out_of_sync(peer_device, req);
-	req_mod(req, OOS_HANDED_TO_NETWORK);
+	req_mod(req, OOS_HANDED_TO_NETWORK, peer_device);
 
 	return err;
 }
@@ -1457,7 +1462,7 @@ int w_send_dblock(struct drbd_work *w, int cancel)
 	int err;
 
 	if (unlikely(cancel)) {
-		req_mod(req, SEND_CANCELED);
+		req_mod(req, SEND_CANCELED, peer_device);
 		return 0;
 	}
 	req->pre_send_jif = jiffies;
@@ -1467,7 +1472,7 @@ int w_send_dblock(struct drbd_work *w, int cancel)
 	connection->send.current_epoch_writes++;
 
 	err = drbd_send_dblock(peer_device, req);
-	req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK);
+	req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK, peer_device);
 
 	if (do_send_unplug && !err)
 		pd_send_unplug_remote(peer_device);
@@ -1490,7 +1495,7 @@ int w_send_read_req(struct drbd_work *w, int cancel)
 	int err;
 
 	if (unlikely(cancel)) {
-		req_mod(req, SEND_CANCELED);
+		req_mod(req, SEND_CANCELED, peer_device);
 		return 0;
 	}
 	req->pre_send_jif = jiffies;
@@ -1502,7 +1507,7 @@ int w_send_read_req(struct drbd_work *w, int cancel)
 	err = drbd_send_drequest(peer_device, P_DATA_REQUEST, req->i.sector, req->i.size,
 				 (unsigned long)req);
 
-	req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK);
+	req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK, peer_device);
 
 	if (do_send_unplug && !err)
 		pd_send_unplug_remote(peer_device);
@@ -1668,8 +1673,9 @@ void drbd_resync_after_changed(struct drbd_device *device)
 	} while (changed);
 }
 
-void drbd_rs_controller_reset(struct drbd_device *device)
+void drbd_rs_controller_reset(struct drbd_peer_device *peer_device)
 {
+	struct drbd_device *device = peer_device->device;
 	struct gendisk *disk = device->ldev->backing_bdev->bd_disk;
 	struct fifo_buffer *plan;
 
@@ -1891,10 +1897,10 @@ void drbd_start_resync(struct drbd_device *device, enum drbd_conns side)
 				rcu_read_unlock();
 				schedule_timeout_interruptible(timeo);
 			}
-			drbd_resync_finished(device);
+			drbd_resync_finished(peer_device);
 		}
 
-		drbd_rs_controller_reset(device);
+		drbd_rs_controller_reset(peer_device);
 		/* ns.conn may already be != device->state.conn,
 		 * we may have been paused in between, or become paused until
 		 * the timer triggers.
@@ -1909,8 +1915,9 @@ out:
 	mutex_unlock(device->state_mutex);
 }
 
-static void update_on_disk_bitmap(struct drbd_device *device, bool resync_done)
+static void update_on_disk_bitmap(struct drbd_peer_device *peer_device, bool resync_done)
 {
+	struct drbd_device *device = peer_device->device;
 	struct sib_info sib = { .sib_reason = SIB_SYNC_PROGRESS, };
 	device->rs_last_bcast = jiffies;
 
@@ -1919,7 +1926,7 @@ static void update_on_disk_bitmap(struct drbd_device *device, bool resync_done)
 
 	drbd_bm_write_lazy(device, 0);
 	if (resync_done && is_sync_state(device->state.conn))
-		drbd_resync_finished(device);
+		drbd_resync_finished(peer_device);
 
 	drbd_bcast_event(device, &sib);
 	/* update timestamp, in case it took a while to write out stuff */
@@ -1945,6 +1952,7 @@ static void drbd_ldev_destroy(struct drbd_device *device)
 
 static void go_diskless(struct drbd_device *device)
 {
+	struct drbd_peer_device *peer_device = first_peer_device(device);
 	D_ASSERT(device, device->state.disk == D_FAILED);
 	/* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
 	 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
@@ -1970,7 +1978,7 @@ static void go_diskless(struct drbd_device *device)
 		 * Any modifications would not be expected anymore, though.
 		 */
 		if (drbd_bitmap_io_from_worker(device, drbd_bm_write,
-					"detach", BM_LOCKED_TEST_ALLOWED)) {
+					"detach", BM_LOCKED_TEST_ALLOWED, peer_device)) {
 			if (test_bit(WAS_READ_ERROR, &device->flags)) {
 				drbd_md_set_flag(device, MDF_FULL_SYNC);
 				drbd_md_sync(device);
@@ -2017,7 +2025,7 @@ static void do_device_work(struct drbd_device *device, const unsigned long todo)
 		do_md_sync(device);
 	if (test_bit(RS_DONE, &todo) ||
 	    test_bit(RS_PROGRESS, &todo))
-		update_on_disk_bitmap(device, test_bit(RS_DONE, &todo));
+		update_on_disk_bitmap(first_peer_device(device), test_bit(RS_DONE, &todo));
 	if (test_bit(GO_DISKLESS, &todo))
 		go_diskless(device);
 	if (test_bit(DESTROY_DISK, &todo))
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 592cfa8b765a..d445fd0934bd 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -325,6 +325,9 @@ static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize,
 	if (blk_validate_block_size(blksize))
 		return -EINVAL;
 
+	if (bytesize < 0)
+		return -EINVAL;
+
 	nbd->config->bytesize = bytesize;
 	nbd->config->blksize_bits = __ffs(blksize);
 
@@ -1111,6 +1114,9 @@ static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg,
 	struct nbd_sock *nsock;
 	int err;
 
+	/* Arg will be cast to int, check it to avoid overflow */
+	if (arg > INT_MAX)
+		return -EINVAL;
 	sock = nbd_get_socket(nbd, arg, &err);
 	if (!sock)
 		return err;
@@ -1934,11 +1940,11 @@ static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info)
 			return -EINVAL;
 		}
 	}
-	if (!info->attrs[NBD_ATTR_SOCKETS]) {
+	if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_SOCKETS)) {
 		pr_err("must specify at least one socket\n");
 		return -EINVAL;
 	}
-	if (!info->attrs[NBD_ATTR_SIZE_BYTES]) {
+	if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_SIZE_BYTES)) {
 		pr_err("must specify a size in bytes for the device\n");
 		return -EINVAL;
 	}
@@ -2123,7 +2129,7 @@ static int nbd_genl_disconnect(struct sk_buff *skb, struct genl_info *info)
 	if (!netlink_capable(skb, CAP_SYS_ADMIN))
 		return -EPERM;
 
-	if (!info->attrs[NBD_ATTR_INDEX]) {
+	if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_INDEX)) {
 		pr_err("must specify an index to disconnect\n");
 		return -EINVAL;
 	}
@@ -2161,7 +2167,7 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 	if (!netlink_capable(skb, CAP_SYS_ADMIN))
 		return -EPERM;
 
-	if (!info->attrs[NBD_ATTR_INDEX]) {
+	if (GENL_REQ_ATTR_CHECK(info, NBD_ATTR_INDEX)) {
 		pr_err("must specify a device to reconfigure\n");
 		return -EINVAL;
 	}
@@ -2325,6 +2331,7 @@ static struct genl_family nbd_genl_family __ro_after_init = {
 	.n_small_ops	= ARRAY_SIZE(nbd_connect_genl_ops),
 	.resv_start_op	= NBD_CMD_STATUS + 1,
 	.maxattr	= NBD_ATTR_MAX,
+	.netnsok	= 1,
 	.policy = nbd_attr_policy,
 	.mcgrps		= nbd_mcast_grps,
 	.n_mcgrps	= ARRAY_SIZE(nbd_mcast_grps),
diff --git a/drivers/block/null_blk/Kconfig b/drivers/block/null_blk/Kconfig
index 6bf1f8ca20a2..ff23bb9346d0 100644
--- a/drivers/block/null_blk/Kconfig
+++ b/drivers/block/null_blk/Kconfig
@@ -9,4 +9,4 @@ config BLK_DEV_NULL_BLK
 
 config BLK_DEV_NULL_BLK_FAULT_INJECTION
 	bool "Support fault injection for Null test block driver"
-	depends on BLK_DEV_NULL_BLK && FAULT_INJECTION
+	depends on BLK_DEV_NULL_BLK && FAULT_INJECTION_CONFIGFS
diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c
index 9e6b032c8ecc..b195b8b9fe32 100644
--- a/drivers/block/null_blk/main.c
+++ b/drivers/block/null_blk/main.c
@@ -250,7 +250,7 @@ static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
 
 static inline struct nullb_device *to_nullb_device(struct config_item *item)
 {
-	return item ? container_of(item, struct nullb_device, item) : NULL;
+	return item ? container_of(to_config_group(item), struct nullb_device, group) : NULL;
 }
 
 static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
@@ -593,8 +593,29 @@ static const struct config_item_type nullb_device_type = {
 	.ct_owner	= THIS_MODULE,
 };
 
+#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
+
+static void nullb_add_fault_config(struct nullb_device *dev)
+{
+	fault_config_init(&dev->timeout_config, "timeout_inject");
+	fault_config_init(&dev->requeue_config, "requeue_inject");
+	fault_config_init(&dev->init_hctx_fault_config, "init_hctx_fault_inject");
+
+	configfs_add_default_group(&dev->timeout_config.group, &dev->group);
+	configfs_add_default_group(&dev->requeue_config.group, &dev->group);
+	configfs_add_default_group(&dev->init_hctx_fault_config.group, &dev->group);
+}
+
+#else
+
+static void nullb_add_fault_config(struct nullb_device *dev)
+{
+}
+
+#endif
+
 static struct
-config_item *nullb_group_make_item(struct config_group *group, const char *name)
+config_group *nullb_group_make_group(struct config_group *group, const char *name)
 {
 	struct nullb_device *dev;
 
@@ -605,9 +626,10 @@ config_item *nullb_group_make_item(struct config_group *group, const char *name)
 	if (!dev)
 		return ERR_PTR(-ENOMEM);
 
-	config_item_init_type_name(&dev->item, name, &nullb_device_type);
+	config_group_init_type_name(&dev->group, name, &nullb_device_type);
+	nullb_add_fault_config(dev);
 
-	return &dev->item;
+	return &dev->group;
 }
 
 static void
@@ -645,7 +667,7 @@ static struct configfs_attribute *nullb_group_attrs[] = {
 };
 
 static struct configfs_group_operations nullb_group_ops = {
-	.make_item	= nullb_group_make_item,
+	.make_group	= nullb_group_make_group,
 	.drop_item	= nullb_group_drop_item,
 };
 
@@ -676,6 +698,13 @@ static struct nullb_device *null_alloc_dev(void)
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 	if (!dev)
 		return NULL;
+
+#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
+	dev->timeout_config.attr = null_timeout_attr;
+	dev->requeue_config.attr = null_requeue_attr;
+	dev->init_hctx_fault_config.attr = null_init_hctx_attr;
+#endif
+
 	INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
 	INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
 	if (badblocks_init(&dev->badblocks, 0)) {
@@ -1030,8 +1059,8 @@ static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
 	if (!t_page)
 		return -ENOMEM;
 
-	src = kmap_atomic(c_page->page);
-	dst = kmap_atomic(t_page->page);
+	src = kmap_local_page(c_page->page);
+	dst = kmap_local_page(t_page->page);
 
 	for (i = 0; i < PAGE_SECTORS;
 			i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
@@ -1043,8 +1072,8 @@ static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
 		}
 	}
 
-	kunmap_atomic(dst);
-	kunmap_atomic(src);
+	kunmap_local(dst);
+	kunmap_local(src);
 
 	ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
 	null_free_page(ret);
@@ -1112,7 +1141,6 @@ static int copy_to_nullb(struct nullb *nullb, struct page *source,
 	size_t temp, count = 0;
 	unsigned int offset;
 	struct nullb_page *t_page;
-	void *dst, *src;
 
 	while (count < n) {
 		temp = min_t(size_t, nullb->dev->blocksize, n - count);
@@ -1126,11 +1154,7 @@ static int copy_to_nullb(struct nullb *nullb, struct page *source,
 		if (!t_page)
 			return -ENOSPC;
 
-		src = kmap_atomic(source);
-		dst = kmap_atomic(t_page->page);
-		memcpy(dst + offset, src + off + count, temp);
-		kunmap_atomic(dst);
-		kunmap_atomic(src);
+		memcpy_page(t_page->page, offset, source, off + count, temp);
 
 		__set_bit(sector & SECTOR_MASK, t_page->bitmap);
 
@@ -1149,7 +1173,6 @@ static int copy_from_nullb(struct nullb *nullb, struct page *dest,
 	size_t temp, count = 0;
 	unsigned int offset;
 	struct nullb_page *t_page;
-	void *dst, *src;
 
 	while (count < n) {
 		temp = min_t(size_t, nullb->dev->blocksize, n - count);
@@ -1158,16 +1181,11 @@ static int copy_from_nullb(struct nullb *nullb, struct page *dest,
 		t_page = null_lookup_page(nullb, sector, false,
 			!null_cache_active(nullb));
 
-		dst = kmap_atomic(dest);
-		if (!t_page) {
-			memset(dst + off + count, 0, temp);
-			goto next;
-		}
-		src = kmap_atomic(t_page->page);
-		memcpy(dst + off + count, src + offset, temp);
-		kunmap_atomic(src);
-next:
-		kunmap_atomic(dst);
+		if (t_page)
+			memcpy_page(dest, off + count, t_page->page, offset,
+				    temp);
+		else
+			zero_user(dest, off + count, temp);
 
 		count += temp;
 		sector += temp >> SECTOR_SHIFT;
@@ -1178,11 +1196,7 @@ next:
 static void nullb_fill_pattern(struct nullb *nullb, struct page *page,
 			       unsigned int len, unsigned int off)
 {
-	void *dst;
-
-	dst = kmap_atomic(page);
-	memset(dst + off, 0xFF, len);
-	kunmap_atomic(dst);
+	memset_page(page, off, 0xff, len);
 }
 
 blk_status_t null_handle_discard(struct nullb_device *dev,
@@ -1529,24 +1543,48 @@ static void null_submit_bio(struct bio *bio)
 	null_handle_cmd(alloc_cmd(nq, bio), sector, nr_sectors, bio_op(bio));
 }
 
+#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
+
+static bool should_timeout_request(struct request *rq)
+{
+	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
+	struct nullb_device *dev = cmd->nq->dev;
+
+	return should_fail(&dev->timeout_config.attr, 1);
+}
+
+static bool should_requeue_request(struct request *rq)
+{
+	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
+	struct nullb_device *dev = cmd->nq->dev;
+
+	return should_fail(&dev->requeue_config.attr, 1);
+}
+
+static bool should_init_hctx_fail(struct nullb_device *dev)
+{
+	return should_fail(&dev->init_hctx_fault_config.attr, 1);
+}
+
+#else
+
 static bool should_timeout_request(struct request *rq)
 {
-#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
-	if (g_timeout_str[0])
-		return should_fail(&null_timeout_attr, 1);
-#endif
 	return false;
 }
 
 static bool should_requeue_request(struct request *rq)
 {
-#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
-	if (g_requeue_str[0])
-		return should_fail(&null_requeue_attr, 1);
-#endif
 	return false;
 }
 
+static bool should_init_hctx_fail(struct nullb_device *dev)
+{
+	return false;
+}
+
+#endif
+
 static void null_map_queues(struct blk_mq_tag_set *set)
 {
 	struct nullb *nullb = set->driver_data;
@@ -1743,10 +1781,8 @@ static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
 	struct nullb *nullb = hctx->queue->queuedata;
 	struct nullb_queue *nq;
 
-#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
-	if (g_init_hctx_str[0] && should_fail(&null_init_hctx_attr, 1))
+	if (should_init_hctx_fail(nullb->dev))
 		return -EFAULT;
-#endif
 
 	nq = &nullb->queues[hctx_idx];
 	hctx->driver_data = nq;
@@ -1964,6 +2000,11 @@ static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
 
 static int null_validate_conf(struct nullb_device *dev)
 {
+	if (dev->queue_mode == NULL_Q_RQ) {
+		pr_err("legacy IO path is no longer available\n");
+		return -EINVAL;
+	}
+
 	dev->blocksize = round_down(dev->blocksize, 512);
 	dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
 
@@ -2066,9 +2107,6 @@ static int null_add_dev(struct nullb_device *dev)
 		if (rv)
 			goto out_cleanup_queues;
 
-		if (!null_setup_fault())
-			goto out_cleanup_tags;
-
 		nullb->tag_set->timeout = 5 * HZ;
 		nullb->disk = blk_mq_alloc_disk(nullb->tag_set, nullb);
 		if (IS_ERR(nullb->disk)) {
@@ -2130,10 +2168,10 @@ static int null_add_dev(struct nullb_device *dev)
 
 	null_config_discard(nullb);
 
-	if (config_item_name(&dev->item)) {
+	if (config_item_name(&dev->group.cg_item)) {
 		/* Use configfs dir name as the device name */
 		snprintf(nullb->disk_name, sizeof(nullb->disk_name),
-			 "%s", config_item_name(&dev->item));
+			 "%s", config_item_name(&dev->group.cg_item));
 	} else {
 		sprintf(nullb->disk_name, "nullb%d", nullb->index);
 	}
@@ -2233,6 +2271,9 @@ static int __init null_init(void)
 		g_home_node = NUMA_NO_NODE;
 	}
 
+	if (!null_setup_fault())
+		return -EINVAL;
+
 	if (g_queue_mode == NULL_Q_RQ) {
 		pr_err("legacy IO path is no longer available\n");
 		return -EINVAL;
diff --git a/drivers/block/null_blk/null_blk.h b/drivers/block/null_blk/null_blk.h
index eb5972c50be8..929f659dd255 100644
--- a/drivers/block/null_blk/null_blk.h
+++ b/drivers/block/null_blk/null_blk.h
@@ -69,7 +69,12 @@ enum {
 
 struct nullb_device {
 	struct nullb *nullb;
-	struct config_item item;
+	struct config_group group;
+#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
+	struct fault_config timeout_config;
+	struct fault_config requeue_config;
+	struct fault_config init_hctx_fault_config;
+#endif
 	struct radix_tree_root data; /* data stored in the disk */
 	struct radix_tree_root cache; /* disk cache data */
 	unsigned long flags; /* device flags */
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index 2f1a92509271..d5d7884cedd4 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -100,7 +100,8 @@ static struct mutex ctl_mutex;	/* Serialize open/close/setup/teardown */
 static mempool_t psd_pool;
 static struct bio_set pkt_bio_set;
 
-static struct class	*class_pktcdvd = NULL;    /* /sys/class/pktcdvd */
+/* /sys/class/pktcdvd */
+static struct class	class_pktcdvd;
 static struct dentry	*pkt_debugfs_root = NULL; /* /sys/kernel/debug/pktcdvd */
 
 /* forward declaration */
@@ -315,8 +316,8 @@ static const struct attribute_group *pkt_groups[] = {
 
 static void pkt_sysfs_dev_new(struct pktcdvd_device *pd)
 {
-	if (class_pktcdvd) {
-		pd->dev = device_create_with_groups(class_pktcdvd, NULL,
+	if (class_is_registered(&class_pktcdvd)) {
+		pd->dev = device_create_with_groups(&class_pktcdvd, NULL,
 						    MKDEV(0, 0), pd, pkt_groups,
 						    "%s", pd->name);
 		if (IS_ERR(pd->dev))
@@ -326,7 +327,7 @@ static void pkt_sysfs_dev_new(struct pktcdvd_device *pd)
 
 static void pkt_sysfs_dev_remove(struct pktcdvd_device *pd)
 {
-	if (class_pktcdvd)
+	if (class_is_registered(&class_pktcdvd))
 		device_unregister(pd->dev);
 }
 
@@ -338,12 +339,7 @@ static void pkt_sysfs_dev_remove(struct pktcdvd_device *pd)
                      device_map     show mappings
  *******************************************************************/
 
-static void class_pktcdvd_release(struct class *cls)
-{
-	kfree(cls);
-}
-
-static ssize_t device_map_show(struct class *c, struct class_attribute *attr,
+static ssize_t device_map_show(const struct class *c, const struct class_attribute *attr,
 			       char *data)
 {
 	int n = 0;
@@ -364,7 +360,7 @@ static ssize_t device_map_show(struct class *c, struct class_attribute *attr,
 }
 static CLASS_ATTR_RO(device_map);
 
-static ssize_t add_store(struct class *c, struct class_attribute *attr,
+static ssize_t add_store(const struct class *c, const struct class_attribute *attr,
 			 const char *buf, size_t count)
 {
 	unsigned int major, minor;
@@ -385,7 +381,7 @@ static ssize_t add_store(struct class *c, struct class_attribute *attr,
 }
 static CLASS_ATTR_WO(add);
 
-static ssize_t remove_store(struct class *c, struct class_attribute *attr,
+static ssize_t remove_store(const struct class *c, const struct class_attribute *attr,
 			    const char *buf, size_t count)
 {
 	unsigned int major, minor;
@@ -405,36 +401,23 @@ static struct attribute *class_pktcdvd_attrs[] = {
 };
 ATTRIBUTE_GROUPS(class_pktcdvd);
 
+static struct class class_pktcdvd = {
+	.name		= DRIVER_NAME,
+	.class_groups	= class_pktcdvd_groups,
+};
+
 static int pkt_sysfs_init(void)
 {
-	int ret = 0;
-
 	/*
 	 * create control files in sysfs
 	 * /sys/class/pktcdvd/...
 	 */
-	class_pktcdvd = kzalloc(sizeof(*class_pktcdvd), GFP_KERNEL);
-	if (!class_pktcdvd)
-		return -ENOMEM;
-	class_pktcdvd->name = DRIVER_NAME;
-	class_pktcdvd->owner = THIS_MODULE;
-	class_pktcdvd->class_release = class_pktcdvd_release;
-	class_pktcdvd->class_groups = class_pktcdvd_groups;
-	ret = class_register(class_pktcdvd);
-	if (ret) {
-		kfree(class_pktcdvd);
-		class_pktcdvd = NULL;
-		pr_err("failed to create class pktcdvd\n");
-		return ret;
-	}
-	return 0;
+	return class_register(&class_pktcdvd);
 }
 
 static void pkt_sysfs_cleanup(void)
 {
-	if (class_pktcdvd)
-		class_destroy(class_pktcdvd);
-	class_pktcdvd = NULL;
+	class_unregister(&class_pktcdvd);
 }
 
 /********************************************************************
@@ -1869,12 +1852,12 @@ static noinline_for_stack int pkt_probe_settings(struct pktcdvd_device *pd)
 /*
  * enable/disable write caching on drive
  */
-static noinline_for_stack int pkt_write_caching(struct pktcdvd_device *pd,
-						int set)
+static noinline_for_stack int pkt_write_caching(struct pktcdvd_device *pd)
 {
 	struct packet_command cgc;
 	struct scsi_sense_hdr sshdr;
 	unsigned char buf[64];
+	bool set = IS_ENABLED(CONFIG_CDROM_PKTCDVD_WCACHE);
 	int ret;
 
 	init_cdrom_command(&cgc, buf, sizeof(buf), CGC_DATA_READ);
@@ -1890,7 +1873,12 @@ static noinline_for_stack int pkt_write_caching(struct pktcdvd_device *pd,
 	if (ret)
 		return ret;
 
-	buf[pd->mode_offset + 10] |= (!!set << 2);
+	/*
+	 * use drive write caching -- we need deferred error handling to be
+	 * able to successfully recover with this option (drive will return good
+	 * status as soon as the cdb is validated).
+	 */
+	buf[pd->mode_offset + 10] |= (set << 2);
 
 	cgc.buflen = cgc.cmd[8] = 2 + ((buf[0] << 8) | (buf[1] & 0xff));
 	ret = pkt_mode_select(pd, &cgc);
@@ -2085,7 +2073,7 @@ static int pkt_open_write(struct pktcdvd_device *pd)
 		return -EIO;
 	}
 
-	pkt_write_caching(pd, USE_WCACHING);
+	pkt_write_caching(pd);
 
 	ret = pkt_get_max_speed(pd, &write_speed);
 	if (ret)
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index 5cb008b9700a..84ad3b17956f 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -491,12 +491,12 @@ static bool single_major = true;
 module_param(single_major, bool, 0444);
 MODULE_PARM_DESC(single_major, "Use a single major number for all rbd devices (default: true)");
 
-static ssize_t add_store(struct bus_type *bus, const char *buf, size_t count);
-static ssize_t remove_store(struct bus_type *bus, const char *buf,
+static ssize_t add_store(const struct bus_type *bus, const char *buf, size_t count);
+static ssize_t remove_store(const struct bus_type *bus, const char *buf,
 			    size_t count);
-static ssize_t add_single_major_store(struct bus_type *bus, const char *buf,
+static ssize_t add_single_major_store(const struct bus_type *bus, const char *buf,
 				      size_t count);
-static ssize_t remove_single_major_store(struct bus_type *bus, const char *buf,
+static ssize_t remove_single_major_store(const struct bus_type *bus, const char *buf,
 					 size_t count);
 static int rbd_dev_image_probe(struct rbd_device *rbd_dev, int depth);
 
@@ -538,7 +538,7 @@ static bool rbd_is_lock_owner(struct rbd_device *rbd_dev)
 	return is_lock_owner;
 }
 
-static ssize_t supported_features_show(struct bus_type *bus, char *buf)
+static ssize_t supported_features_show(const struct bus_type *bus, char *buf)
 {
 	return sprintf(buf, "0x%llx\n", RBD_FEATURES_SUPPORTED);
 }
@@ -6967,9 +6967,7 @@ err_out_format:
 	return ret;
 }
 
-static ssize_t do_rbd_add(struct bus_type *bus,
-			  const char *buf,
-			  size_t count)
+static ssize_t do_rbd_add(const char *buf, size_t count)
 {
 	struct rbd_device *rbd_dev = NULL;
 	struct ceph_options *ceph_opts = NULL;
@@ -7081,18 +7079,18 @@ err_out_args:
 	goto out;
 }
 
-static ssize_t add_store(struct bus_type *bus, const char *buf, size_t count)
+static ssize_t add_store(const struct bus_type *bus, const char *buf, size_t count)
 {
 	if (single_major)
 		return -EINVAL;
 
-	return do_rbd_add(bus, buf, count);
+	return do_rbd_add(buf, count);
 }
 
-static ssize_t add_single_major_store(struct bus_type *bus, const char *buf,
+static ssize_t add_single_major_store(const struct bus_type *bus, const char *buf,
 				      size_t count)
 {
-	return do_rbd_add(bus, buf, count);
+	return do_rbd_add(buf, count);
 }
 
 static void rbd_dev_remove_parent(struct rbd_device *rbd_dev)
@@ -7122,9 +7120,7 @@ static void rbd_dev_remove_parent(struct rbd_device *rbd_dev)
 	}
 }
 
-static ssize_t do_rbd_remove(struct bus_type *bus,
-			     const char *buf,
-			     size_t count)
+static ssize_t do_rbd_remove(const char *buf, size_t count)
 {
 	struct rbd_device *rbd_dev = NULL;
 	struct list_head *tmp;
@@ -7196,18 +7192,18 @@ static ssize_t do_rbd_remove(struct bus_type *bus,
 	return count;
 }
 
-static ssize_t remove_store(struct bus_type *bus, const char *buf, size_t count)
+static ssize_t remove_store(const struct bus_type *bus, const char *buf, size_t count)
 {
 	if (single_major)
 		return -EINVAL;
 
-	return do_rbd_remove(bus, buf, count);
+	return do_rbd_remove(buf, count);
 }
 
-static ssize_t remove_single_major_store(struct bus_type *bus, const char *buf,
+static ssize_t remove_single_major_store(const struct bus_type *bus, const char *buf,
 					 size_t count)
 {
-	return do_rbd_remove(bus, buf, count);
+	return do_rbd_remove(buf, count);
 }
 
 /*
diff --git a/drivers/block/rnbd/rnbd-clt-sysfs.c b/drivers/block/rnbd/rnbd-clt-sysfs.c
index e7c7d9a68168..8c6087949794 100644
--- a/drivers/block/rnbd/rnbd-clt-sysfs.c
+++ b/drivers/block/rnbd/rnbd-clt-sysfs.c
@@ -646,7 +646,7 @@ int rnbd_clt_create_sysfs_files(void)
 {
 	int err;
 
-	rnbd_dev_class = class_create(THIS_MODULE, "rnbd-client");
+	rnbd_dev_class = class_create("rnbd-client");
 	if (IS_ERR(rnbd_dev_class))
 		return PTR_ERR(rnbd_dev_class);
 
diff --git a/drivers/block/rnbd/rnbd-srv-sysfs.c b/drivers/block/rnbd/rnbd-srv-sysfs.c
index 297a6924ff4e..d5d9267e1fa5 100644
--- a/drivers/block/rnbd/rnbd-srv-sysfs.c
+++ b/drivers/block/rnbd/rnbd-srv-sysfs.c
@@ -215,7 +215,7 @@ int rnbd_srv_create_sysfs_files(void)
 {
 	int err;
 
-	rnbd_dev_class = class_create(THIS_MODULE, "rnbd-server");
+	rnbd_dev_class = class_create("rnbd-server");
 	if (IS_ERR(rnbd_dev_class))
 		return PTR_ERR(rnbd_dev_class);
 
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 604c1a13c76e..6b8aa0d030f7 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -53,7 +53,8 @@
 		| UBLK_F_NEED_GET_DATA \
 		| UBLK_F_USER_RECOVERY \
 		| UBLK_F_USER_RECOVERY_REISSUE \
-		| UBLK_F_UNPRIVILEGED_DEV)
+		| UBLK_F_UNPRIVILEGED_DEV \
+		| UBLK_F_CMD_IOCTL_ENCODE)
 
 /* All UBLK_PARAM_TYPE_* should be included here */
 #define UBLK_PARAM_TYPE_ALL (UBLK_PARAM_TYPE_BASIC | \
@@ -298,9 +299,7 @@ static inline bool ublk_can_use_task_work(const struct ublk_queue *ubq)
 
 static inline bool ublk_need_get_data(const struct ublk_queue *ubq)
 {
-	if (ubq->flags & UBLK_F_NEED_GET_DATA)
-		return true;
-	return false;
+	return ubq->flags & UBLK_F_NEED_GET_DATA;
 }
 
 static struct ublk_device *ublk_get_device(struct ublk_device *ub)
@@ -349,25 +348,19 @@ static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub, int q_id)
 static inline bool ublk_queue_can_use_recovery_reissue(
 		struct ublk_queue *ubq)
 {
-	if ((ubq->flags & UBLK_F_USER_RECOVERY) &&
-			(ubq->flags & UBLK_F_USER_RECOVERY_REISSUE))
-		return true;
-	return false;
+	return (ubq->flags & UBLK_F_USER_RECOVERY) &&
+			(ubq->flags & UBLK_F_USER_RECOVERY_REISSUE);
 }
 
 static inline bool ublk_queue_can_use_recovery(
 		struct ublk_queue *ubq)
 {
-	if (ubq->flags & UBLK_F_USER_RECOVERY)
-		return true;
-	return false;
+	return ubq->flags & UBLK_F_USER_RECOVERY;
 }
 
 static inline bool ublk_can_use_recovery(struct ublk_device *ub)
 {
-	if (ub->dev_info.flags & UBLK_F_USER_RECOVERY)
-		return true;
-	return false;
+	return ub->dev_info.flags & UBLK_F_USER_RECOVERY;
 }
 
 static void ublk_free_disk(struct gendisk *disk)
@@ -428,10 +421,9 @@ static const struct block_device_operations ub_fops = {
 #define UBLK_MAX_PIN_PAGES	32
 
 struct ublk_map_data {
-	const struct ublk_queue *ubq;
 	const struct request *rq;
-	const struct ublk_io *io;
-	unsigned max_bytes;
+	unsigned long	ubuf;
+	unsigned int	len;
 };
 
 struct ublk_io_iter {
@@ -488,18 +480,17 @@ static inline unsigned ublk_copy_io_pages(struct ublk_io_iter *data,
 	return done;
 }
 
-static inline int ublk_copy_user_pages(struct ublk_map_data *data,
-		bool to_vm)
+static int ublk_copy_user_pages(struct ublk_map_data *data, bool to_vm)
 {
 	const unsigned int gup_flags = to_vm ? FOLL_WRITE : 0;
-	const unsigned long start_vm = data->io->addr;
+	const unsigned long start_vm = data->ubuf;
 	unsigned int done = 0;
 	struct ublk_io_iter iter = {
 		.pg_off	= start_vm & (PAGE_SIZE - 1),
 		.bio	= data->rq->bio,
 		.iter	= data->rq->bio->bi_iter,
 	};
-	const unsigned int nr_pages = round_up(data->max_bytes +
+	const unsigned int nr_pages = round_up(data->len +
 			(start_vm & (PAGE_SIZE - 1)), PAGE_SIZE) >> PAGE_SHIFT;
 
 	while (done < nr_pages) {
@@ -512,42 +503,49 @@ static inline int ublk_copy_user_pages(struct ublk_map_data *data,
 				iter.pages);
 		if (iter.nr_pages <= 0)
 			return done == 0 ? iter.nr_pages : done;
-		len = ublk_copy_io_pages(&iter, data->max_bytes, to_vm);
+		len = ublk_copy_io_pages(&iter, data->len, to_vm);
 		for (i = 0; i < iter.nr_pages; i++) {
 			if (to_vm)
 				set_page_dirty(iter.pages[i]);
 			put_page(iter.pages[i]);
 		}
-		data->max_bytes -= len;
+		data->len -= len;
 		done += iter.nr_pages;
 	}
 
 	return done;
 }
 
+static inline bool ublk_need_map_req(const struct request *req)
+{
+	return ublk_rq_has_data(req) && req_op(req) == REQ_OP_WRITE;
+}
+
+static inline bool ublk_need_unmap_req(const struct request *req)
+{
+	return ublk_rq_has_data(req) && req_op(req) == REQ_OP_READ;
+}
+
 static int ublk_map_io(const struct ublk_queue *ubq, const struct request *req,
 		struct ublk_io *io)
 {
 	const unsigned int rq_bytes = blk_rq_bytes(req);
+
 	/*
 	 * no zero copy, we delay copy WRITE request data into ublksrv
 	 * context and the big benefit is that pinning pages in current
 	 * context is pretty fast, see ublk_pin_user_pages
 	 */
-	if (req_op(req) != REQ_OP_WRITE && req_op(req) != REQ_OP_FLUSH)
-		return rq_bytes;
-
-	if (ublk_rq_has_data(req)) {
+	if (ublk_need_map_req(req)) {
 		struct ublk_map_data data = {
-			.ubq	=	ubq,
 			.rq	=	req,
-			.io	=	io,
-			.max_bytes =	rq_bytes,
+			.ubuf	=	io->addr,
+			.len	=	rq_bytes,
 		};
 
 		ublk_copy_user_pages(&data, true);
 
-		return rq_bytes - data.max_bytes;
+		return rq_bytes - data.len;
 	}
 	return rq_bytes;
 }
@@ -558,19 +556,18 @@ static int ublk_unmap_io(const struct ublk_queue *ubq,
 {
 	const unsigned int rq_bytes = blk_rq_bytes(req);
 
-	if (req_op(req) == REQ_OP_READ && ublk_rq_has_data(req)) {
+	if (ublk_need_unmap_req(req)) {
 		struct ublk_map_data data = {
-			.ubq	=	ubq,
 			.rq	=	req,
-			.io	=	io,
-			.max_bytes =	io->res,
+			.ubuf	=	io->addr,
+			.len	=	io->res,
 		};
 
 		WARN_ON_ONCE(io->res > rq_bytes);
 
 		ublk_copy_user_pages(&data, false);
 
-		return io->res - data.max_bytes;
+		return io->res - data.len;
 	}
 	return rq_bytes;
 }
@@ -655,14 +652,15 @@ static void ublk_complete_rq(struct request *req)
 	struct ublk_queue *ubq = req->mq_hctx->driver_data;
 	struct ublk_io *io = &ubq->ios[req->tag];
 	unsigned int unmapped_bytes;
+	blk_status_t res = BLK_STS_OK;
 
 	/* failed read IO if nothing is read */
 	if (!io->res && req_op(req) == REQ_OP_READ)
 		io->res = -EIO;
 
 	if (io->res < 0) {
-		blk_mq_end_request(req, errno_to_blk_status(io->res));
-		return;
+		res = errno_to_blk_status(io->res);
+		goto exit;
 	}
 
 	/*
@@ -671,10 +669,8 @@ static void ublk_complete_rq(struct request *req)
 	 *
 	 * Both the two needn't unmap.
 	 */
-	if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE) {
-		blk_mq_end_request(req, BLK_STS_OK);
-		return;
-	}
+	if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE)
+		goto exit;
 
 	/* for READ request, writing data in iod->addr to rq buffers */
 	unmapped_bytes = ublk_unmap_io(ubq, req, io);
@@ -691,6 +687,10 @@ static void ublk_complete_rq(struct request *req)
 		blk_mq_requeue_request(req, true);
 	else
 		__blk_mq_end_request(req, BLK_STS_OK);
+
+	return;
+exit:
+	blk_mq_end_request(req, res);
 }
 
 /*
@@ -771,9 +771,7 @@ static inline void __ublk_rq_task_work(struct request *req,
 		return;
 	}
 
-	if (ublk_need_get_data(ubq) &&
-			(req_op(req) == REQ_OP_WRITE ||
-			req_op(req) == REQ_OP_FLUSH)) {
+	if (ublk_need_get_data(ubq) && ublk_need_map_req(req)) {
 		/*
 		 * We have not handled UBLK_IO_NEED_GET_DATA command yet,
 		 * so immepdately pass UBLK_IO_RES_NEED_GET_DATA to ublksrv
@@ -1261,6 +1259,19 @@ static void ublk_handle_need_get_data(struct ublk_device *ub, int q_id,
 	ublk_queue_cmd(ubq, req);
 }
 
+static inline int ublk_check_cmd_op(u32 cmd_op)
+{
+	u32 ioc_type = _IOC_TYPE(cmd_op);
+
+	if (IS_ENABLED(CONFIG_BLKDEV_UBLK_LEGACY_OPCODES) && ioc_type != 'u')
+		return -EOPNOTSUPP;
+
+	if (ioc_type != 'u' && ioc_type != 0)
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
 static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
 			       unsigned int issue_flags,
 			       struct ublksrv_io_cmd *ub_cmd)
@@ -1303,10 +1314,15 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
 	 * iff the driver have set the UBLK_IO_FLAG_NEED_GET_DATA.
 	 */
 	if ((!!(io->flags & UBLK_IO_FLAG_NEED_GET_DATA))
-			^ (cmd_op == UBLK_IO_NEED_GET_DATA))
+			^ (_IOC_NR(cmd_op) == UBLK_IO_NEED_GET_DATA))
+		goto out;
+
+	ret = ublk_check_cmd_op(cmd_op);
+	if (ret)
 		goto out;
 
-	switch (cmd_op) {
+	ret = -EINVAL;
+	switch (_IOC_NR(cmd_op)) {
 	case UBLK_IO_FETCH_REQ:
 		/* UBLK_IO_FETCH_REQ is only allowed before queue is setup */
 		if (ublk_queue_ready(ubq)) {
@@ -1770,6 +1786,8 @@ static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)
 	if (!IS_BUILTIN(CONFIG_BLK_DEV_UBLK))
 		ub->dev_info.flags |= UBLK_F_URING_CMD_COMP_IN_TASK;
 
+	ub->dev_info.flags |= UBLK_F_CMD_IOCTL_ENCODE;
+
 	/* We are not ready to support zero copy */
 	ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY;
 
@@ -2128,7 +2146,7 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,
 		 * know if the specified device is created as unprivileged
 		 * mode.
 		 */
-		if (cmd->cmd_op != UBLK_CMD_GET_DEV_INFO2)
+		if (_IOC_NR(cmd->cmd_op) != UBLK_CMD_GET_DEV_INFO2)
 			return 0;
 	}
 
@@ -2154,7 +2172,7 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,
 	dev_path[header->dev_path_len] = 0;
 
 	ret = -EINVAL;
-	switch (cmd->cmd_op) {
+	switch (_IOC_NR(cmd->cmd_op)) {
 	case UBLK_CMD_GET_DEV_INFO:
 	case UBLK_CMD_GET_DEV_INFO2:
 	case UBLK_CMD_GET_QUEUE_AFFINITY:
@@ -2193,6 +2211,7 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
 {
 	struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
 	struct ublk_device *ub = NULL;
+	u32 cmd_op = cmd->cmd_op;
 	int ret = -EINVAL;
 
 	if (issue_flags & IO_URING_F_NONBLOCK)
@@ -2203,22 +2222,22 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
 	if (!(issue_flags & IO_URING_F_SQE128))
 		goto out;
 
-	if (cmd->cmd_op != UBLK_CMD_ADD_DEV) {
+	ret = ublk_check_cmd_op(cmd_op);
+	if (ret)
+		goto out;
+
+	if (_IOC_NR(cmd_op) != UBLK_CMD_ADD_DEV) {
 		ret = -ENODEV;
 		ub = ublk_get_device_from_id(header->dev_id);
 		if (!ub)
 			goto out;
 
 		ret = ublk_ctrl_uring_cmd_permission(ub, cmd);
-	} else {
-		/* ADD_DEV permission check is done in command handler */
-		ret = 0;
+		if (ret)
+			goto put_dev;
 	}
 
-	if (ret)
-		goto put_dev;
-
-	switch (cmd->cmd_op) {
+	switch (_IOC_NR(cmd_op)) {
 	case UBLK_CMD_START_DEV:
 		ret = ublk_ctrl_start_dev(ub, cmd);
 		break;
@@ -2292,7 +2311,7 @@ static int __init ublk_init(void)
 	if (ret)
 		goto unregister_mis;
 
-	ublk_chr_class = class_create(THIS_MODULE, "ublk-char");
+	ublk_chr_class = class_create("ublk-char");
 	if (IS_ERR(ublk_chr_class)) {
 		ret = PTR_ERR(ublk_chr_class);
 		goto free_chrdev_region;
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index aa490da3cef2..d8217529be6f 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -2424,8 +2424,8 @@ static int zram_remove(struct zram *zram)
  * creates a new un-initialized zram device and returns back this device's
  * device_id (or an error code if it fails to create a new device).
  */
-static ssize_t hot_add_show(struct class *class,
-			struct class_attribute *attr,
+static ssize_t hot_add_show(const struct class *class,
+			const struct class_attribute *attr,
 			char *buf)
 {
 	int ret;
@@ -2438,11 +2438,12 @@ static ssize_t hot_add_show(struct class *class,
 		return ret;
 	return scnprintf(buf, PAGE_SIZE, "%d\n", ret);
 }
+/* This attribute must be set to 0400, so CLASS_ATTR_RO() can not be used */
 static struct class_attribute class_attr_hot_add =
 	__ATTR(hot_add, 0400, hot_add_show, NULL);
 
-static ssize_t hot_remove_store(struct class *class,
-			struct class_attribute *attr,
+static ssize_t hot_remove_store(const struct class *class,
+			const struct class_attribute *attr,
 			const char *buf,
 			size_t count)
 {
@@ -2481,7 +2482,6 @@ ATTRIBUTE_GROUPS(zram_control_class);
 
 static struct class zram_control_class = {
 	.name		= "zram-control",
-	.owner		= THIS_MODULE,
 	.class_groups	= zram_control_class_groups,
 };
 
diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig
index 5a1a7bec3c42..bc211c324206 100644
--- a/drivers/bluetooth/Kconfig
+++ b/drivers/bluetooth/Kconfig
@@ -363,6 +363,7 @@ config BT_HCIBLUECARD
 
 config BT_HCIVHCI
 	tristate "HCI VHCI (Virtual HCI device) driver"
+	select WANT_DEV_COREDUMP
 	help
 	  Bluetooth Virtual HCI device driver.
 	  This driver is required if you want to use HCI Emulation software.
@@ -465,4 +466,17 @@ config BT_VIRTIO
 	  Say Y here to compile support for HCI over Virtio into the
 	  kernel or say M to compile as a module.
 
+config BT_NXPUART
+	tristate "NXP protocol support"
+	depends on SERIAL_DEV_BUS
+	select CRC32
+	select CRC8
+	help
+	  NXP is serial driver required for NXP Bluetooth
+	  devices with UART interface.
+
+	  Say Y here to compile support for NXP Bluetooth UART device into
+	  the kernel, or say M here to compile as a module (btnxpuart).
+
+
 endmenu
diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile
index e0b261f24fc9..7a5967e9ac48 100644
--- a/drivers/bluetooth/Makefile
+++ b/drivers/bluetooth/Makefile
@@ -29,6 +29,7 @@ obj-$(CONFIG_BT_QCA)		+= btqca.o
 obj-$(CONFIG_BT_MTK)		+= btmtk.o
 
 obj-$(CONFIG_BT_VIRTIO)		+= virtio_bt.o
+obj-$(CONFIG_BT_NXPUART)	+= btnxpuart.o
 
 obj-$(CONFIG_BT_HCIUART_NOKIA)	+= hci_nokia.o
 
diff --git a/drivers/bluetooth/btbcm.c b/drivers/bluetooth/btbcm.c
index 43e98a598bd9..de2ea589aa49 100644
--- a/drivers/bluetooth/btbcm.c
+++ b/drivers/bluetooth/btbcm.c
@@ -6,6 +6,7 @@
  *  Copyright (C) 2015  Intel Corporation
  */
 
+#include <linux/efi.h>
 #include <linux/module.h>
 #include <linux/firmware.h>
 #include <linux/dmi.h>
@@ -34,6 +35,43 @@
 /* For kmalloc-ing the fw-name array instead of putting it on the stack */
 typedef char bcm_fw_name[BCM_FW_NAME_LEN];
 
+#ifdef CONFIG_EFI
+static int btbcm_set_bdaddr_from_efi(struct hci_dev *hdev)
+{
+	efi_guid_t guid = EFI_GUID(0x74b00bd9, 0x805a, 0x4d61, 0xb5, 0x1f,
+				   0x43, 0x26, 0x81, 0x23, 0xd1, 0x13);
+	bdaddr_t efi_bdaddr, bdaddr;
+	efi_status_t status;
+	unsigned long len;
+	int ret;
+
+	if (!efi_rt_services_supported(EFI_RT_SUPPORTED_GET_VARIABLE))
+		return -EOPNOTSUPP;
+
+	len = sizeof(efi_bdaddr);
+	status = efi.get_variable(L"BDADDR", &guid, NULL, &len, &efi_bdaddr);
+	if (status != EFI_SUCCESS)
+		return -ENXIO;
+
+	if (len != sizeof(efi_bdaddr))
+		return -EIO;
+
+	baswap(&bdaddr, &efi_bdaddr);
+
+	ret = btbcm_set_bdaddr(hdev, &bdaddr);
+	if (ret)
+		return ret;
+
+	bt_dev_info(hdev, "BCM: Using EFI device address (%pMR)", &bdaddr);
+	return 0;
+}
+#else
+static int btbcm_set_bdaddr_from_efi(struct hci_dev *hdev)
+{
+	return -EOPNOTSUPP;
+}
+#endif
+
 int btbcm_check_bdaddr(struct hci_dev *hdev)
 {
 	struct hci_rp_read_bd_addr *bda;
@@ -87,9 +125,12 @@ int btbcm_check_bdaddr(struct hci_dev *hdev)
 	    !bacmp(&bda->bdaddr, BDADDR_BCM4345C5) ||
 	    !bacmp(&bda->bdaddr, BDADDR_BCM43430A0) ||
 	    !bacmp(&bda->bdaddr, BDADDR_BCM43341B)) {
-		bt_dev_info(hdev, "BCM: Using default device address (%pMR)",
-			    &bda->bdaddr);
-		set_bit(HCI_QUIRK_INVALID_BDADDR, &hdev->quirks);
+		/* Try falling back to BDADDR EFI variable */
+		if (btbcm_set_bdaddr_from_efi(hdev) != 0) {
+			bt_dev_info(hdev, "BCM: Using default device address (%pMR)",
+				    &bda->bdaddr);
+			set_bit(HCI_QUIRK_INVALID_BDADDR, &hdev->quirks);
+		}
 	}
 
 	kfree_skb(skb);
diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c
index af774688f1c0..d9349ba48281 100644
--- a/drivers/bluetooth/btintel.c
+++ b/drivers/bluetooth/btintel.c
@@ -43,6 +43,12 @@ struct cmd_write_boot_params {
 	u8  fw_build_yy;
 } __packed;
 
+static struct {
+	const char *driver_name;
+	u8         hw_variant;
+	u32        fw_build_num;
+} coredump_info;
+
 int btintel_check_bdaddr(struct hci_dev *hdev)
 {
 	struct hci_rp_read_bd_addr *bda;
@@ -315,6 +321,9 @@ int btintel_version_info(struct hci_dev *hdev, struct intel_version *ver)
 		return -EINVAL;
 	}
 
+	coredump_info.hw_variant = ver->hw_variant;
+	coredump_info.fw_build_num = ver->fw_build_num;
+
 	bt_dev_info(hdev, "%s revision %u.%u build %u week %u %u",
 		    variant, ver->fw_revision >> 4, ver->fw_revision & 0x0f,
 		    ver->fw_build_num, ver->fw_build_ww,
@@ -509,6 +518,9 @@ static int btintel_version_info_tlv(struct hci_dev *hdev,
 		return -EINVAL;
 	}
 
+	coredump_info.hw_variant = INTEL_HW_VARIANT(version->cnvi_bt);
+	coredump_info.fw_build_num = version->build_num;
+
 	bt_dev_info(hdev, "%s timestamp %u.%u buildtype %u build %u", variant,
 		    2000 + (version->timestamp >> 8), version->timestamp & 0xff,
 		    version->build_type, version->build_num);
@@ -1462,6 +1474,59 @@ int btintel_set_quality_report(struct hci_dev *hdev, bool enable)
 }
 EXPORT_SYMBOL_GPL(btintel_set_quality_report);
 
+static void btintel_coredump(struct hci_dev *hdev)
+{
+	struct sk_buff *skb;
+
+	skb = __hci_cmd_sync(hdev, 0xfc4e, 0, NULL, HCI_CMD_TIMEOUT);
+	if (IS_ERR(skb)) {
+		bt_dev_err(hdev, "Coredump failed (%ld)", PTR_ERR(skb));
+		return;
+	}
+
+	kfree_skb(skb);
+}
+
+static void btintel_dmp_hdr(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	char buf[80];
+
+	snprintf(buf, sizeof(buf), "Controller Name: 0x%X\n",
+		 coredump_info.hw_variant);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Firmware Version: 0x%X\n",
+		 coredump_info.fw_build_num);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Driver: %s\n", coredump_info.driver_name);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Vendor: Intel\n");
+	skb_put_data(skb, buf, strlen(buf));
+}
+
+static int btintel_register_devcoredump_support(struct hci_dev *hdev)
+{
+	struct intel_debug_features features;
+	int err;
+
+	err = btintel_read_debug_features(hdev, &features);
+	if (err) {
+		bt_dev_info(hdev, "Error reading debug features");
+		return err;
+	}
+
+	if (!(features.page1[0] & 0x3f)) {
+		bt_dev_dbg(hdev, "Telemetry exception format not supported");
+		return -EOPNOTSUPP;
+	}
+
+	hci_devcd_register(hdev, btintel_coredump, btintel_dmp_hdr, NULL);
+
+	return err;
+}
+
 static const struct firmware *btintel_legacy_rom_get_fw(struct hci_dev *hdev,
 					       struct intel_version *ver)
 {
@@ -2597,6 +2662,7 @@ static int btintel_setup_combined(struct hci_dev *hdev)
 			btintel_set_msft_opcode(hdev, ver.hw_variant);
 
 			err = btintel_bootloader_setup(hdev, &ver);
+			btintel_register_devcoredump_support(hdev);
 			break;
 		default:
 			bt_dev_err(hdev, "Unsupported Intel hw variant (%u)",
@@ -2670,6 +2736,7 @@ static int btintel_setup_combined(struct hci_dev *hdev)
 		btintel_set_msft_opcode(hdev, ver.hw_variant);
 
 		err = btintel_bootloader_setup(hdev, &ver);
+		btintel_register_devcoredump_support(hdev);
 		break;
 	case 0x17:
 	case 0x18:
@@ -2684,15 +2751,15 @@ static int btintel_setup_combined(struct hci_dev *hdev)
 		 */
 		set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED, &hdev->quirks);
 
-		/* Valid LE States quirk for GfP */
-		if (INTEL_HW_VARIANT(ver_tlv.cnvi_bt) == 0x18)
-			set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks);
+		/* Apply LE States quirk from solar onwards */
+		set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks);
 
 		/* Setup MSFT Extension support */
 		btintel_set_msft_opcode(hdev,
 					INTEL_HW_VARIANT(ver_tlv.cnvi_bt));
 
 		err = btintel_bootloader_setup_tlv(hdev, &ver_tlv);
+		btintel_register_devcoredump_support(hdev);
 		break;
 	default:
 		bt_dev_err(hdev, "Unsupported Intel hw variant (%u)",
@@ -2742,7 +2809,7 @@ static int btintel_shutdown_combined(struct hci_dev *hdev)
 	return 0;
 }
 
-int btintel_configure_setup(struct hci_dev *hdev)
+int btintel_configure_setup(struct hci_dev *hdev, const char *driver_name)
 {
 	hdev->manufacturer = 2;
 	hdev->setup = btintel_setup_combined;
@@ -2751,6 +2818,8 @@ int btintel_configure_setup(struct hci_dev *hdev)
 	hdev->set_diag = btintel_set_diag_combined;
 	hdev->set_bdaddr = btintel_set_bdaddr;
 
+	coredump_info.driver_name = driver_name;
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(btintel_configure_setup);
diff --git a/drivers/bluetooth/btintel.h b/drivers/bluetooth/btintel.h
index 8fdb65b66315..d6a1dc8d8a82 100644
--- a/drivers/bluetooth/btintel.h
+++ b/drivers/bluetooth/btintel.h
@@ -143,6 +143,13 @@ struct btintel_loc_aware_reg {
 	__le32 delta;
 } __packed;
 
+#define INTEL_TLV_TYPE_ID		0x01
+
+#define INTEL_TLV_SYSTEM_EXCEPTION	0x00
+#define INTEL_TLV_FATAL_EXCEPTION	0x01
+#define INTEL_TLV_DEBUG_EXCEPTION	0x02
+#define INTEL_TLV_TEST_EXCEPTION	0xDE
+
 #define INTEL_HW_PLATFORM(cnvx_bt)	((u8)(((cnvx_bt) & 0x0000ff00) >> 8))
 #define INTEL_HW_VARIANT(cnvx_bt)	((u8)(((cnvx_bt) & 0x003f0000) >> 16))
 #define INTEL_CNVX_TOP_TYPE(cnvx_top)	((cnvx_top) & 0x00000fff)
@@ -212,7 +219,7 @@ int btintel_read_boot_params(struct hci_dev *hdev,
 			     struct intel_boot_params *params);
 int btintel_download_firmware(struct hci_dev *dev, struct intel_version *ver,
 			      const struct firmware *fw, u32 *boot_param);
-int btintel_configure_setup(struct hci_dev *hdev);
+int btintel_configure_setup(struct hci_dev *hdev, const char *driver_name);
 void btintel_bootup(struct hci_dev *hdev, const void *ptr, unsigned int len);
 void btintel_secure_send_result(struct hci_dev *hdev,
 				const void *ptr, unsigned int len);
@@ -293,7 +300,8 @@ static inline int btintel_download_firmware(struct hci_dev *dev,
 	return -EOPNOTSUPP;
 }
 
-static inline int btintel_configure_setup(struct hci_dev *hdev)
+static inline int btintel_configure_setup(struct hci_dev *hdev,
+					  const char *driver_name)
 {
 	return -ENODEV;
 }
diff --git a/drivers/bluetooth/btmrvl_sdio.c b/drivers/bluetooth/btmrvl_sdio.c
index ba057ebfda5c..d76c799553aa 100644
--- a/drivers/bluetooth/btmrvl_sdio.c
+++ b/drivers/bluetooth/btmrvl_sdio.c
@@ -40,7 +40,7 @@ static struct memory_type_mapping mem_type_mapping_tbl[] = {
 	{"EXTLAST", NULL, 0, 0xFE},
 };
 
-static const struct of_device_id btmrvl_sdio_of_match_table[] = {
+static const struct of_device_id btmrvl_sdio_of_match_table[] __maybe_unused = {
 	{ .compatible = "marvell,sd8897-bt" },
 	{ .compatible = "marvell,sd8997-bt" },
 	{ }
diff --git a/drivers/bluetooth/btmtkuart.c b/drivers/bluetooth/btmtkuart.c
index c98691cdbbd5..7680c67cdb35 100644
--- a/drivers/bluetooth/btmtkuart.c
+++ b/drivers/bluetooth/btmtkuart.c
@@ -959,16 +959,16 @@ static void btmtkuart_remove(struct serdev_device *serdev)
 	hci_free_dev(hdev);
 }
 
-static const struct btmtkuart_data mt7622_data = {
+static const struct btmtkuart_data mt7622_data __maybe_unused = {
 	.fwname = FIRMWARE_MT7622,
 };
 
-static const struct btmtkuart_data mt7663_data = {
+static const struct btmtkuart_data mt7663_data __maybe_unused = {
 	.flags = BTMTKUART_FLAG_STANDALONE_HW,
 	.fwname = FIRMWARE_MT7663,
 };
 
-static const struct btmtkuart_data mt7668_data = {
+static const struct btmtkuart_data mt7668_data __maybe_unused = {
 	.flags = BTMTKUART_FLAG_STANDALONE_HW,
 	.fwname = FIRMWARE_MT7668,
 };
diff --git a/drivers/bluetooth/btnxpuart.c b/drivers/bluetooth/btnxpuart.c
new file mode 100644
index 000000000000..3a34d7c1475b
--- /dev/null
+++ b/drivers/bluetooth/btnxpuart.c
@@ -0,0 +1,1352 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *  NXP Bluetooth driver
+ *  Copyright 2023 NXP
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+
+#include <linux/serdev.h>
+#include <linux/of.h>
+#include <linux/skbuff.h>
+#include <asm/unaligned.h>
+#include <linux/firmware.h>
+#include <linux/string.h>
+#include <linux/crc8.h>
+#include <linux/crc32.h>
+#include <linux/string_helpers.h>
+
+#include <net/bluetooth/bluetooth.h>
+#include <net/bluetooth/hci_core.h>
+
+#include "h4_recv.h"
+
+#define MANUFACTURER_NXP		37
+
+#define BTNXPUART_TX_STATE_ACTIVE	1
+#define BTNXPUART_FW_DOWNLOADING	2
+#define BTNXPUART_CHECK_BOOT_SIGNATURE	3
+#define BTNXPUART_SERDEV_OPEN		4
+
+#define FIRMWARE_W8987	"nxp/uartuart8987_bt.bin"
+#define FIRMWARE_W8997	"nxp/uartuart8997_bt_v4.bin"
+#define FIRMWARE_W9098	"nxp/uartuart9098_bt_v1.bin"
+#define FIRMWARE_IW416	"nxp/uartiw416_bt_v0.bin"
+#define FIRMWARE_IW612	"nxp/uartspi_n61x_v1.bin.se"
+#define FIRMWARE_HELPER	"nxp/helper_uart_3000000.bin"
+
+#define CHIP_ID_W9098		0x5c03
+#define CHIP_ID_IW416		0x7201
+#define CHIP_ID_IW612		0x7601
+
+#define HCI_NXP_PRI_BAUDRATE	115200
+#define HCI_NXP_SEC_BAUDRATE	3000000
+
+#define MAX_FW_FILE_NAME_LEN    50
+
+/* Default ps timeout period in milliseconds */
+#define PS_DEFAULT_TIMEOUT_PERIOD_MS     2000
+
+/* wakeup methods */
+#define WAKEUP_METHOD_DTR       0
+#define WAKEUP_METHOD_BREAK     1
+#define WAKEUP_METHOD_EXT_BREAK 2
+#define WAKEUP_METHOD_RTS       3
+#define WAKEUP_METHOD_INVALID   0xff
+
+/* power save mode status */
+#define PS_MODE_DISABLE         0
+#define PS_MODE_ENABLE          1
+
+/* Power Save Commands to ps_work_func  */
+#define PS_CMD_EXIT_PS          1
+#define PS_CMD_ENTER_PS         2
+
+/* power save state */
+#define PS_STATE_AWAKE          0
+#define PS_STATE_SLEEP          1
+
+/* Bluetooth vendor command : Sleep mode */
+#define HCI_NXP_AUTO_SLEEP_MODE	0xfc23
+/* Bluetooth vendor command : Wakeup method */
+#define HCI_NXP_WAKEUP_METHOD	0xfc53
+/* Bluetooth vendor command : Set operational baudrate */
+#define HCI_NXP_SET_OPER_SPEED	0xfc09
+/* Bluetooth vendor command: Independent Reset */
+#define HCI_NXP_IND_RESET	0xfcfc
+
+/* Bluetooth Power State : Vendor cmd params */
+#define BT_PS_ENABLE			0x02
+#define BT_PS_DISABLE			0x03
+
+/* Bluetooth Host Wakeup Methods */
+#define BT_HOST_WAKEUP_METHOD_NONE      0x00
+#define BT_HOST_WAKEUP_METHOD_DTR       0x01
+#define BT_HOST_WAKEUP_METHOD_BREAK     0x02
+#define BT_HOST_WAKEUP_METHOD_GPIO      0x03
+
+/* Bluetooth Chip Wakeup Methods */
+#define BT_CTRL_WAKEUP_METHOD_DSR       0x00
+#define BT_CTRL_WAKEUP_METHOD_BREAK     0x01
+#define BT_CTRL_WAKEUP_METHOD_GPIO      0x02
+#define BT_CTRL_WAKEUP_METHOD_EXT_BREAK 0x04
+#define BT_CTRL_WAKEUP_METHOD_RTS       0x05
+
+struct ps_data {
+	u8    target_ps_mode;	/* ps mode to be set */
+	u8    cur_psmode;	/* current ps_mode */
+	u8    ps_state;		/* controller's power save state */
+	u8    ps_cmd;
+	u8    h2c_wakeupmode;
+	u8    cur_h2c_wakeupmode;
+	u8    c2h_wakeupmode;
+	u8    c2h_wakeup_gpio;
+	u8    h2c_wakeup_gpio;
+	bool  driver_sent_cmd;
+	u16   h2c_ps_interval;
+	u16   c2h_ps_interval;
+	struct hci_dev *hdev;
+	struct work_struct work;
+	struct timer_list ps_timer;
+};
+
+struct wakeup_cmd_payload {
+	u8 c2h_wakeupmode;
+	u8 c2h_wakeup_gpio;
+	u8 h2c_wakeupmode;
+	u8 h2c_wakeup_gpio;
+} __packed;
+
+struct psmode_cmd_payload {
+	u8 ps_cmd;
+	__le16 c2h_ps_interval;
+} __packed;
+
+struct btnxpuart_data {
+	const char *helper_fw_name;
+	const char *fw_name;
+};
+
+struct btnxpuart_dev {
+	struct hci_dev *hdev;
+	struct serdev_device *serdev;
+
+	struct work_struct tx_work;
+	unsigned long tx_state;
+	struct sk_buff_head txq;
+	struct sk_buff *rx_skb;
+
+	const struct firmware *fw;
+	u8 fw_name[MAX_FW_FILE_NAME_LEN];
+	u32 fw_dnld_v1_offset;
+	u32 fw_v1_sent_bytes;
+	u32 fw_v3_offset_correction;
+	u32 fw_v1_expected_len;
+	wait_queue_head_t fw_dnld_done_wait_q;
+	wait_queue_head_t check_boot_sign_wait_q;
+
+	u32 new_baudrate;
+	u32 current_baudrate;
+	u32 fw_init_baudrate;
+	bool timeout_changed;
+	bool baudrate_changed;
+	bool helper_downloaded;
+
+	struct ps_data psdata;
+	struct btnxpuart_data *nxp_data;
+};
+
+#define NXP_V1_FW_REQ_PKT	0xa5
+#define NXP_V1_CHIP_VER_PKT	0xaa
+#define NXP_V3_FW_REQ_PKT	0xa7
+#define NXP_V3_CHIP_VER_PKT	0xab
+
+#define NXP_ACK_V1		0x5a
+#define NXP_NAK_V1		0xbf
+#define NXP_ACK_V3		0x7a
+#define NXP_NAK_V3		0x7b
+#define NXP_CRC_ERROR_V3	0x7c
+
+#define HDR_LEN			16
+
+#define NXP_RECV_CHIP_VER_V1 \
+	.type = NXP_V1_CHIP_VER_PKT, \
+	.hlen = 4, \
+	.loff = 0, \
+	.lsize = 0, \
+	.maxlen = 4
+
+#define NXP_RECV_FW_REQ_V1 \
+	.type = NXP_V1_FW_REQ_PKT, \
+	.hlen = 4, \
+	.loff = 0, \
+	.lsize = 0, \
+	.maxlen = 4
+
+#define NXP_RECV_CHIP_VER_V3 \
+	.type = NXP_V3_CHIP_VER_PKT, \
+	.hlen = 4, \
+	.loff = 0, \
+	.lsize = 0, \
+	.maxlen = 4
+
+#define NXP_RECV_FW_REQ_V3 \
+	.type = NXP_V3_FW_REQ_PKT, \
+	.hlen = 9, \
+	.loff = 0, \
+	.lsize = 0, \
+	.maxlen = 9
+
+struct v1_data_req {
+	__le16 len;
+	__le16 len_comp;
+} __packed;
+
+struct v1_start_ind {
+	__le16 chip_id;
+	__le16 chip_id_comp;
+} __packed;
+
+struct v3_data_req {
+	__le16 len;
+	__le32 offset;
+	__le16 error;
+	u8 crc;
+} __packed;
+
+struct v3_start_ind {
+	__le16 chip_id;
+	u8 loader_ver;
+	u8 crc;
+} __packed;
+
+/* UART register addresses of BT chip */
+#define CLKDIVADDR	0x7f00008f
+#define UARTDIVADDR	0x7f000090
+#define UARTMCRADDR	0x7f000091
+#define UARTREINITADDR	0x7f000092
+#define UARTICRADDR	0x7f000093
+#define UARTFCRADDR	0x7f000094
+
+#define MCR		0x00000022
+#define INIT		0x00000001
+#define ICR		0x000000c7
+#define FCR		0x000000c7
+
+#define POLYNOMIAL8	0x07
+
+struct uart_reg {
+	__le32 address;
+	__le32 value;
+} __packed;
+
+struct uart_config {
+	struct uart_reg clkdiv;
+	struct uart_reg uartdiv;
+	struct uart_reg mcr;
+	struct uart_reg re_init;
+	struct uart_reg icr;
+	struct uart_reg fcr;
+	__be32 crc;
+} __packed;
+
+struct nxp_bootloader_cmd {
+	__le32 header;
+	__le32 arg;
+	__le32 payload_len;
+	__be32 crc;
+} __packed;
+
+static u8 crc8_table[CRC8_TABLE_SIZE];
+
+/* Default configurations */
+#define DEFAULT_H2C_WAKEUP_MODE	WAKEUP_METHOD_BREAK
+#define DEFAULT_PS_MODE		PS_MODE_DISABLE
+#define FW_INIT_BAUDRATE	HCI_NXP_PRI_BAUDRATE
+
+static struct sk_buff *nxp_drv_send_cmd(struct hci_dev *hdev, u16 opcode,
+					u32 plen,
+					void *param)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct ps_data *psdata = &nxpdev->psdata;
+	struct sk_buff *skb;
+
+	/* set flag to prevent nxp_enqueue from parsing values from this command and
+	 * calling hci_cmd_sync_queue() again.
+	 */
+	psdata->driver_sent_cmd = true;
+	skb = __hci_cmd_sync(hdev, opcode, plen, param, HCI_CMD_TIMEOUT);
+	psdata->driver_sent_cmd = false;
+
+	return skb;
+}
+
+static void btnxpuart_tx_wakeup(struct btnxpuart_dev *nxpdev)
+{
+	if (schedule_work(&nxpdev->tx_work))
+		set_bit(BTNXPUART_TX_STATE_ACTIVE, &nxpdev->tx_state);
+}
+
+/* NXP Power Save Feature */
+static void ps_start_timer(struct btnxpuart_dev *nxpdev)
+{
+	struct ps_data *psdata = &nxpdev->psdata;
+
+	if (!psdata)
+		return;
+
+	if (psdata->cur_psmode == PS_MODE_ENABLE)
+		mod_timer(&psdata->ps_timer, jiffies + msecs_to_jiffies(psdata->h2c_ps_interval));
+}
+
+static void ps_cancel_timer(struct btnxpuart_dev *nxpdev)
+{
+	struct ps_data *psdata = &nxpdev->psdata;
+
+	flush_work(&psdata->work);
+	del_timer_sync(&psdata->ps_timer);
+}
+
+static void ps_control(struct hci_dev *hdev, u8 ps_state)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct ps_data *psdata = &nxpdev->psdata;
+	int status;
+
+	if (psdata->ps_state == ps_state ||
+	    !test_bit(BTNXPUART_SERDEV_OPEN, &nxpdev->tx_state))
+		return;
+
+	switch (psdata->cur_h2c_wakeupmode) {
+	case WAKEUP_METHOD_DTR:
+		if (ps_state == PS_STATE_AWAKE)
+			status = serdev_device_set_tiocm(nxpdev->serdev, TIOCM_DTR, 0);
+		else
+			status = serdev_device_set_tiocm(nxpdev->serdev, 0, TIOCM_DTR);
+		break;
+	case WAKEUP_METHOD_BREAK:
+	default:
+		if (ps_state == PS_STATE_AWAKE)
+			status = serdev_device_break_ctl(nxpdev->serdev, 0);
+		else
+			status = serdev_device_break_ctl(nxpdev->serdev, -1);
+		bt_dev_dbg(hdev, "Set UART break: %s, status=%d",
+			   str_on_off(ps_state == PS_STATE_SLEEP), status);
+		break;
+	}
+	if (!status)
+		psdata->ps_state = ps_state;
+	if (ps_state == PS_STATE_AWAKE)
+		btnxpuart_tx_wakeup(nxpdev);
+}
+
+static void ps_work_func(struct work_struct *work)
+{
+	struct ps_data *data = container_of(work, struct ps_data, work);
+
+	if (data->ps_cmd == PS_CMD_ENTER_PS && data->cur_psmode == PS_MODE_ENABLE)
+		ps_control(data->hdev, PS_STATE_SLEEP);
+	else if (data->ps_cmd == PS_CMD_EXIT_PS)
+		ps_control(data->hdev, PS_STATE_AWAKE);
+}
+
+static void ps_timeout_func(struct timer_list *t)
+{
+	struct ps_data *data = from_timer(data, t, ps_timer);
+	struct hci_dev *hdev = data->hdev;
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+
+	if (test_bit(BTNXPUART_TX_STATE_ACTIVE, &nxpdev->tx_state)) {
+		ps_start_timer(nxpdev);
+	} else {
+		data->ps_cmd = PS_CMD_ENTER_PS;
+		schedule_work(&data->work);
+	}
+}
+
+static int ps_init_work(struct hci_dev *hdev)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct ps_data *psdata = &nxpdev->psdata;
+
+	psdata->h2c_ps_interval = PS_DEFAULT_TIMEOUT_PERIOD_MS;
+	psdata->ps_state = PS_STATE_AWAKE;
+	psdata->target_ps_mode = DEFAULT_PS_MODE;
+	psdata->hdev = hdev;
+	psdata->c2h_wakeupmode = BT_HOST_WAKEUP_METHOD_NONE;
+	psdata->c2h_wakeup_gpio = 0xff;
+
+	switch (DEFAULT_H2C_WAKEUP_MODE) {
+	case WAKEUP_METHOD_DTR:
+		psdata->h2c_wakeupmode = WAKEUP_METHOD_DTR;
+		break;
+	case WAKEUP_METHOD_BREAK:
+	default:
+		psdata->h2c_wakeupmode = WAKEUP_METHOD_BREAK;
+		break;
+	}
+	psdata->cur_psmode = PS_MODE_DISABLE;
+	psdata->cur_h2c_wakeupmode = WAKEUP_METHOD_INVALID;
+	INIT_WORK(&psdata->work, ps_work_func);
+
+	return 0;
+}
+
+static void ps_init_timer(struct hci_dev *hdev)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct ps_data *psdata = &nxpdev->psdata;
+
+	timer_setup(&psdata->ps_timer, ps_timeout_func, 0);
+}
+
+static void ps_wakeup(struct btnxpuart_dev *nxpdev)
+{
+	struct ps_data *psdata = &nxpdev->psdata;
+
+	if (psdata->ps_state != PS_STATE_AWAKE) {
+		psdata->ps_cmd = PS_CMD_EXIT_PS;
+		schedule_work(&psdata->work);
+	}
+}
+
+static int send_ps_cmd(struct hci_dev *hdev, void *data)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct ps_data *psdata = &nxpdev->psdata;
+	struct psmode_cmd_payload pcmd;
+	struct sk_buff *skb;
+	u8 *status;
+
+	if (psdata->target_ps_mode == PS_MODE_ENABLE)
+		pcmd.ps_cmd = BT_PS_ENABLE;
+	else
+		pcmd.ps_cmd = BT_PS_DISABLE;
+	pcmd.c2h_ps_interval = __cpu_to_le16(psdata->c2h_ps_interval);
+
+	skb = nxp_drv_send_cmd(hdev, HCI_NXP_AUTO_SLEEP_MODE, sizeof(pcmd), &pcmd);
+	if (IS_ERR(skb)) {
+		bt_dev_err(hdev, "Setting Power Save mode failed (%ld)", PTR_ERR(skb));
+		return PTR_ERR(skb);
+	}
+
+	status = skb_pull_data(skb, 1);
+	if (status) {
+		if (!*status)
+			psdata->cur_psmode = psdata->target_ps_mode;
+		else
+			psdata->target_ps_mode = psdata->cur_psmode;
+		if (psdata->cur_psmode == PS_MODE_ENABLE)
+			ps_start_timer(nxpdev);
+		else
+			ps_wakeup(nxpdev);
+		bt_dev_dbg(hdev, "Power Save mode response: status=%d, ps_mode=%d",
+			   *status, psdata->cur_psmode);
+	}
+	kfree_skb(skb);
+
+	return 0;
+}
+
+static int send_wakeup_method_cmd(struct hci_dev *hdev, void *data)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct ps_data *psdata = &nxpdev->psdata;
+	struct wakeup_cmd_payload pcmd;
+	struct sk_buff *skb;
+	u8 *status;
+
+	pcmd.c2h_wakeupmode = psdata->c2h_wakeupmode;
+	pcmd.c2h_wakeup_gpio = psdata->c2h_wakeup_gpio;
+	switch (psdata->h2c_wakeupmode) {
+	case WAKEUP_METHOD_DTR:
+		pcmd.h2c_wakeupmode = BT_CTRL_WAKEUP_METHOD_DSR;
+		break;
+	case WAKEUP_METHOD_BREAK:
+	default:
+		pcmd.h2c_wakeupmode = BT_CTRL_WAKEUP_METHOD_BREAK;
+		break;
+	}
+	pcmd.h2c_wakeup_gpio = 0xff;
+
+	skb = nxp_drv_send_cmd(hdev, HCI_NXP_WAKEUP_METHOD, sizeof(pcmd), &pcmd);
+	if (IS_ERR(skb)) {
+		bt_dev_err(hdev, "Setting wake-up method failed (%ld)", PTR_ERR(skb));
+		return PTR_ERR(skb);
+	}
+
+	status = skb_pull_data(skb, 1);
+	if (status) {
+		if (*status == 0)
+			psdata->cur_h2c_wakeupmode = psdata->h2c_wakeupmode;
+		else
+			psdata->h2c_wakeupmode = psdata->cur_h2c_wakeupmode;
+		bt_dev_dbg(hdev, "Set Wakeup Method response: status=%d, h2c_wakeupmode=%d",
+			   *status, psdata->cur_h2c_wakeupmode);
+	}
+	kfree_skb(skb);
+
+	return 0;
+}
+
+static void ps_init(struct hci_dev *hdev)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct ps_data *psdata = &nxpdev->psdata;
+
+	serdev_device_set_tiocm(nxpdev->serdev, 0, TIOCM_RTS);
+	usleep_range(5000, 10000);
+	serdev_device_set_tiocm(nxpdev->serdev, TIOCM_RTS, 0);
+	usleep_range(5000, 10000);
+
+	switch (psdata->h2c_wakeupmode) {
+	case WAKEUP_METHOD_DTR:
+		serdev_device_set_tiocm(nxpdev->serdev, 0, TIOCM_DTR);
+		serdev_device_set_tiocm(nxpdev->serdev, TIOCM_DTR, 0);
+		break;
+	case WAKEUP_METHOD_BREAK:
+	default:
+		serdev_device_break_ctl(nxpdev->serdev, -1);
+		usleep_range(5000, 10000);
+		serdev_device_break_ctl(nxpdev->serdev, 0);
+		usleep_range(5000, 10000);
+		break;
+	}
+	if (psdata->cur_h2c_wakeupmode != psdata->h2c_wakeupmode)
+		hci_cmd_sync_queue(hdev, send_wakeup_method_cmd, NULL, NULL);
+	if (psdata->cur_psmode != psdata->target_ps_mode)
+		hci_cmd_sync_queue(hdev, send_ps_cmd, NULL, NULL);
+}
+
+/* NXP Firmware Download Feature */
+static int nxp_download_firmware(struct hci_dev *hdev)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	int err = 0;
+
+	nxpdev->fw_dnld_v1_offset = 0;
+	nxpdev->fw_v1_sent_bytes = 0;
+	nxpdev->fw_v1_expected_len = HDR_LEN;
+	nxpdev->fw_v3_offset_correction = 0;
+	nxpdev->baudrate_changed = false;
+	nxpdev->timeout_changed = false;
+	nxpdev->helper_downloaded = false;
+
+	serdev_device_set_baudrate(nxpdev->serdev, HCI_NXP_PRI_BAUDRATE);
+	serdev_device_set_flow_control(nxpdev->serdev, false);
+	nxpdev->current_baudrate = HCI_NXP_PRI_BAUDRATE;
+
+	/* Wait till FW is downloaded and CTS becomes low */
+	err = wait_event_interruptible_timeout(nxpdev->fw_dnld_done_wait_q,
+					       !test_bit(BTNXPUART_FW_DOWNLOADING,
+							 &nxpdev->tx_state),
+					       msecs_to_jiffies(60000));
+	if (err == 0) {
+		bt_dev_err(hdev, "FW Download Timeout.");
+		return -ETIMEDOUT;
+	}
+
+	serdev_device_set_flow_control(nxpdev->serdev, true);
+	err = serdev_device_wait_for_cts(nxpdev->serdev, 1, 60000);
+	if (err < 0) {
+		bt_dev_err(hdev, "CTS is still high. FW Download failed.");
+		return err;
+	}
+	release_firmware(nxpdev->fw);
+	memset(nxpdev->fw_name, 0, sizeof(nxpdev->fw_name));
+
+	/* Allow the downloaded FW to initialize */
+	usleep_range(800 * USEC_PER_MSEC, 1 * USEC_PER_SEC);
+
+	return 0;
+}
+
+static void nxp_send_ack(u8 ack, struct hci_dev *hdev)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	u8 ack_nak[2];
+	int len = 1;
+
+	ack_nak[0] = ack;
+	if (ack == NXP_ACK_V3) {
+		ack_nak[1] = crc8(crc8_table, ack_nak, 1, 0xff);
+		len = 2;
+	}
+	serdev_device_write_buf(nxpdev->serdev, ack_nak, len);
+}
+
+static bool nxp_fw_change_baudrate(struct hci_dev *hdev, u16 req_len)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct nxp_bootloader_cmd nxp_cmd5;
+	struct uart_config uart_config;
+
+	if (req_len == sizeof(nxp_cmd5)) {
+		nxp_cmd5.header = __cpu_to_le32(5);
+		nxp_cmd5.arg = 0;
+		nxp_cmd5.payload_len = __cpu_to_le32(sizeof(uart_config));
+		/* FW expects swapped CRC bytes */
+		nxp_cmd5.crc = __cpu_to_be32(crc32_be(0UL, (char *)&nxp_cmd5,
+						      sizeof(nxp_cmd5) - 4));
+
+		serdev_device_write_buf(nxpdev->serdev, (u8 *)&nxp_cmd5, sizeof(nxp_cmd5));
+		nxpdev->fw_v3_offset_correction += req_len;
+	} else if (req_len == sizeof(uart_config)) {
+		uart_config.clkdiv.address = __cpu_to_le32(CLKDIVADDR);
+		uart_config.clkdiv.value = __cpu_to_le32(0x00c00000);
+		uart_config.uartdiv.address = __cpu_to_le32(UARTDIVADDR);
+		uart_config.uartdiv.value = __cpu_to_le32(1);
+		uart_config.mcr.address = __cpu_to_le32(UARTMCRADDR);
+		uart_config.mcr.value = __cpu_to_le32(MCR);
+		uart_config.re_init.address = __cpu_to_le32(UARTREINITADDR);
+		uart_config.re_init.value = __cpu_to_le32(INIT);
+		uart_config.icr.address = __cpu_to_le32(UARTICRADDR);
+		uart_config.icr.value = __cpu_to_le32(ICR);
+		uart_config.fcr.address = __cpu_to_le32(UARTFCRADDR);
+		uart_config.fcr.value = __cpu_to_le32(FCR);
+		/* FW expects swapped CRC bytes */
+		uart_config.crc = __cpu_to_be32(crc32_be(0UL, (char *)&uart_config,
+							 sizeof(uart_config) - 4));
+
+		serdev_device_write_buf(nxpdev->serdev, (u8 *)&uart_config, sizeof(uart_config));
+		serdev_device_wait_until_sent(nxpdev->serdev, 0);
+		nxpdev->fw_v3_offset_correction += req_len;
+		return true;
+	}
+	return false;
+}
+
+static bool nxp_fw_change_timeout(struct hci_dev *hdev, u16 req_len)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct nxp_bootloader_cmd nxp_cmd7;
+
+	if (req_len != sizeof(nxp_cmd7))
+		return false;
+
+	nxp_cmd7.header = __cpu_to_le32(7);
+	nxp_cmd7.arg = __cpu_to_le32(0x70);
+	nxp_cmd7.payload_len = 0;
+	/* FW expects swapped CRC bytes */
+	nxp_cmd7.crc = __cpu_to_be32(crc32_be(0UL, (char *)&nxp_cmd7,
+					      sizeof(nxp_cmd7) - 4));
+	serdev_device_write_buf(nxpdev->serdev, (u8 *)&nxp_cmd7, sizeof(nxp_cmd7));
+	serdev_device_wait_until_sent(nxpdev->serdev, 0);
+	nxpdev->fw_v3_offset_correction += req_len;
+	return true;
+}
+
+static u32 nxp_get_data_len(const u8 *buf)
+{
+	struct nxp_bootloader_cmd *hdr = (struct nxp_bootloader_cmd *)buf;
+
+	return __le32_to_cpu(hdr->payload_len);
+}
+
+static bool is_fw_downloading(struct btnxpuart_dev *nxpdev)
+{
+	return test_bit(BTNXPUART_FW_DOWNLOADING, &nxpdev->tx_state);
+}
+
+static bool process_boot_signature(struct btnxpuart_dev *nxpdev)
+{
+	if (test_bit(BTNXPUART_CHECK_BOOT_SIGNATURE, &nxpdev->tx_state)) {
+		clear_bit(BTNXPUART_CHECK_BOOT_SIGNATURE, &nxpdev->tx_state);
+		wake_up_interruptible(&nxpdev->check_boot_sign_wait_q);
+		return false;
+	}
+	return is_fw_downloading(nxpdev);
+}
+
+static int nxp_request_firmware(struct hci_dev *hdev, const char *fw_name)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	int err = 0;
+
+	if (!strlen(nxpdev->fw_name)) {
+		snprintf(nxpdev->fw_name, MAX_FW_FILE_NAME_LEN, "%s", fw_name);
+
+		bt_dev_dbg(hdev, "Request Firmware: %s", nxpdev->fw_name);
+		err = request_firmware(&nxpdev->fw, nxpdev->fw_name, &hdev->dev);
+		if (err < 0) {
+			bt_dev_err(hdev, "Firmware file %s not found", nxpdev->fw_name);
+			clear_bit(BTNXPUART_FW_DOWNLOADING, &nxpdev->tx_state);
+		}
+	}
+	return err;
+}
+
+/* for legacy chipsets with V1 bootloader */
+static int nxp_recv_chip_ver_v1(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct v1_start_ind *req;
+	__u16 chip_id;
+
+	req = skb_pull_data(skb, sizeof(*req));
+	if (!req)
+		goto free_skb;
+
+	chip_id = le16_to_cpu(req->chip_id ^ req->chip_id_comp);
+	if (chip_id == 0xffff) {
+		nxpdev->fw_dnld_v1_offset = 0;
+		nxpdev->fw_v1_sent_bytes = 0;
+		nxpdev->fw_v1_expected_len = HDR_LEN;
+		release_firmware(nxpdev->fw);
+		memset(nxpdev->fw_name, 0, sizeof(nxpdev->fw_name));
+		nxp_send_ack(NXP_ACK_V1, hdev);
+	}
+
+free_skb:
+	kfree_skb(skb);
+	return 0;
+}
+
+static int nxp_recv_fw_req_v1(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct btnxpuart_data *nxp_data = nxpdev->nxp_data;
+	struct v1_data_req *req;
+	__u16 len;
+
+	if (!process_boot_signature(nxpdev))
+		goto free_skb;
+
+	req = skb_pull_data(skb, sizeof(*req));
+	if (!req)
+		goto free_skb;
+
+	len = __le16_to_cpu(req->len ^ req->len_comp);
+	if (len != 0xffff) {
+		bt_dev_dbg(hdev, "ERR: Send NAK");
+		nxp_send_ack(NXP_NAK_V1, hdev);
+		goto free_skb;
+	}
+	nxp_send_ack(NXP_ACK_V1, hdev);
+
+	len = __le16_to_cpu(req->len);
+
+	if (!nxp_data->helper_fw_name) {
+		if (!nxpdev->timeout_changed) {
+			nxpdev->timeout_changed = nxp_fw_change_timeout(hdev,
+									len);
+			goto free_skb;
+		}
+		if (!nxpdev->baudrate_changed) {
+			nxpdev->baudrate_changed = nxp_fw_change_baudrate(hdev,
+									  len);
+			if (nxpdev->baudrate_changed) {
+				serdev_device_set_baudrate(nxpdev->serdev,
+							   HCI_NXP_SEC_BAUDRATE);
+				serdev_device_set_flow_control(nxpdev->serdev, true);
+				nxpdev->current_baudrate = HCI_NXP_SEC_BAUDRATE;
+			}
+			goto free_skb;
+		}
+	}
+
+	if (!nxp_data->helper_fw_name || nxpdev->helper_downloaded) {
+		if (nxp_request_firmware(hdev, nxp_data->fw_name))
+			goto free_skb;
+	} else if (nxp_data->helper_fw_name && !nxpdev->helper_downloaded) {
+		if (nxp_request_firmware(hdev, nxp_data->helper_fw_name))
+			goto free_skb;
+	}
+
+	if (!len) {
+		bt_dev_dbg(hdev, "FW Downloaded Successfully: %zu bytes",
+			   nxpdev->fw->size);
+		if (nxp_data->helper_fw_name && !nxpdev->helper_downloaded) {
+			nxpdev->helper_downloaded = true;
+			serdev_device_wait_until_sent(nxpdev->serdev, 0);
+			serdev_device_set_baudrate(nxpdev->serdev,
+						   HCI_NXP_SEC_BAUDRATE);
+			serdev_device_set_flow_control(nxpdev->serdev, true);
+		} else {
+			clear_bit(BTNXPUART_FW_DOWNLOADING, &nxpdev->tx_state);
+			wake_up_interruptible(&nxpdev->fw_dnld_done_wait_q);
+		}
+		goto free_skb;
+	}
+	if (len & 0x01) {
+		/* The CRC did not match at the other end.
+		 * Simply send the same bytes again.
+		 */
+		len = nxpdev->fw_v1_sent_bytes;
+		bt_dev_dbg(hdev, "CRC error. Resend %d bytes of FW.", len);
+	} else {
+		nxpdev->fw_dnld_v1_offset += nxpdev->fw_v1_sent_bytes;
+
+		/* The FW bin file is made up of many blocks of
+		 * 16 byte header and payload data chunks. If the
+		 * FW has requested a header, read the payload length
+		 * info from the header, before sending the header.
+		 * In the next iteration, the FW should request the
+		 * payload data chunk, which should be equal to the
+		 * payload length read from header. If there is a
+		 * mismatch, clearly the driver and FW are out of sync,
+		 * and we need to re-send the previous header again.
+		 */
+		if (len == nxpdev->fw_v1_expected_len) {
+			if (len == HDR_LEN)
+				nxpdev->fw_v1_expected_len = nxp_get_data_len(nxpdev->fw->data +
+									nxpdev->fw_dnld_v1_offset);
+			else
+				nxpdev->fw_v1_expected_len = HDR_LEN;
+		} else if (len == HDR_LEN) {
+			/* FW download out of sync. Send previous chunk again */
+			nxpdev->fw_dnld_v1_offset -= nxpdev->fw_v1_sent_bytes;
+			nxpdev->fw_v1_expected_len = HDR_LEN;
+		}
+	}
+
+	if (nxpdev->fw_dnld_v1_offset + len <= nxpdev->fw->size)
+		serdev_device_write_buf(nxpdev->serdev, nxpdev->fw->data +
+					nxpdev->fw_dnld_v1_offset, len);
+	nxpdev->fw_v1_sent_bytes = len;
+
+free_skb:
+	kfree_skb(skb);
+	return 0;
+}
+
+static char *nxp_get_fw_name_from_chipid(struct hci_dev *hdev, u16 chipid)
+{
+	char *fw_name = NULL;
+
+	switch (chipid) {
+	case CHIP_ID_W9098:
+		fw_name = FIRMWARE_W9098;
+		break;
+	case CHIP_ID_IW416:
+		fw_name = FIRMWARE_IW416;
+		break;
+	case CHIP_ID_IW612:
+		fw_name = FIRMWARE_IW612;
+		break;
+	default:
+		bt_dev_err(hdev, "Unknown chip signature %04x", chipid);
+		break;
+	}
+	return fw_name;
+}
+
+static int nxp_recv_chip_ver_v3(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct v3_start_ind *req = skb_pull_data(skb, sizeof(*req));
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	u16 chip_id;
+
+	if (!process_boot_signature(nxpdev))
+		goto free_skb;
+
+	chip_id = le16_to_cpu(req->chip_id);
+	if (!nxp_request_firmware(hdev, nxp_get_fw_name_from_chipid(hdev,
+								    chip_id)))
+		nxp_send_ack(NXP_ACK_V3, hdev);
+
+free_skb:
+	kfree_skb(skb);
+	return 0;
+}
+
+static int nxp_recv_fw_req_v3(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct v3_data_req *req;
+	__u16 len;
+	__u32 offset;
+
+	if (!process_boot_signature(nxpdev))
+		goto free_skb;
+
+	req = skb_pull_data(skb, sizeof(*req));
+	if (!req || !nxpdev->fw)
+		goto free_skb;
+
+	nxp_send_ack(NXP_ACK_V3, hdev);
+
+	len = __le16_to_cpu(req->len);
+
+	if (!nxpdev->timeout_changed) {
+		nxpdev->timeout_changed = nxp_fw_change_timeout(hdev, len);
+		goto free_skb;
+	}
+
+	if (!nxpdev->baudrate_changed) {
+		nxpdev->baudrate_changed = nxp_fw_change_baudrate(hdev, len);
+		if (nxpdev->baudrate_changed) {
+			serdev_device_set_baudrate(nxpdev->serdev,
+						   HCI_NXP_SEC_BAUDRATE);
+			serdev_device_set_flow_control(nxpdev->serdev, true);
+			nxpdev->current_baudrate = HCI_NXP_SEC_BAUDRATE;
+		}
+		goto free_skb;
+	}
+
+	if (req->len == 0) {
+		bt_dev_dbg(hdev, "FW Downloaded Successfully: %zu bytes",
+			   nxpdev->fw->size);
+		clear_bit(BTNXPUART_FW_DOWNLOADING, &nxpdev->tx_state);
+		wake_up_interruptible(&nxpdev->fw_dnld_done_wait_q);
+		goto free_skb;
+	}
+	if (req->error)
+		bt_dev_dbg(hdev, "FW Download received err 0x%02x from chip",
+			   req->error);
+
+	offset = __le32_to_cpu(req->offset);
+	if (offset < nxpdev->fw_v3_offset_correction) {
+		/* This scenario should ideally never occur. But if it ever does,
+		 * FW is out of sync and needs a power cycle.
+		 */
+		bt_dev_err(hdev, "Something went wrong during FW download");
+		bt_dev_err(hdev, "Please power cycle and try again");
+		goto free_skb;
+	}
+
+	serdev_device_write_buf(nxpdev->serdev, nxpdev->fw->data + offset -
+				nxpdev->fw_v3_offset_correction, len);
+
+free_skb:
+	kfree_skb(skb);
+	return 0;
+}
+
+static int nxp_set_baudrate_cmd(struct hci_dev *hdev, void *data)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	__le32 new_baudrate = __cpu_to_le32(nxpdev->new_baudrate);
+	struct ps_data *psdata = &nxpdev->psdata;
+	struct sk_buff *skb;
+	u8 *status;
+
+	if (!psdata)
+		return 0;
+
+	skb = nxp_drv_send_cmd(hdev, HCI_NXP_SET_OPER_SPEED, 4, (u8 *)&new_baudrate);
+	if (IS_ERR(skb)) {
+		bt_dev_err(hdev, "Setting baudrate failed (%ld)", PTR_ERR(skb));
+		return PTR_ERR(skb);
+	}
+
+	status = (u8 *)skb_pull_data(skb, 1);
+	if (status) {
+		if (*status == 0) {
+			serdev_device_set_baudrate(nxpdev->serdev, nxpdev->new_baudrate);
+			nxpdev->current_baudrate = nxpdev->new_baudrate;
+		}
+		bt_dev_dbg(hdev, "Set baudrate response: status=%d, baudrate=%d",
+			   *status, nxpdev->new_baudrate);
+	}
+	kfree_skb(skb);
+
+	return 0;
+}
+
+static int nxp_set_ind_reset(struct hci_dev *hdev, void *data)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct sk_buff *skb;
+	u8 *status;
+	u8 pcmd = 0;
+	int err = 0;
+
+	skb = nxp_drv_send_cmd(hdev, HCI_NXP_IND_RESET, 1, &pcmd);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	status = skb_pull_data(skb, 1);
+	if (!status || *status)
+		goto free_skb;
+
+	set_bit(BTNXPUART_FW_DOWNLOADING, &nxpdev->tx_state);
+	err = nxp_download_firmware(hdev);
+	if (err < 0)
+		goto free_skb;
+	serdev_device_set_baudrate(nxpdev->serdev, nxpdev->fw_init_baudrate);
+	nxpdev->current_baudrate = nxpdev->fw_init_baudrate;
+	if (nxpdev->current_baudrate != HCI_NXP_SEC_BAUDRATE) {
+		nxpdev->new_baudrate = HCI_NXP_SEC_BAUDRATE;
+		nxp_set_baudrate_cmd(hdev, NULL);
+	}
+	hci_cmd_sync_queue(hdev, send_wakeup_method_cmd, NULL, NULL);
+	hci_cmd_sync_queue(hdev, send_ps_cmd, NULL, NULL);
+
+free_skb:
+	kfree_skb(skb);
+	return err;
+}
+
+/* NXP protocol */
+static int nxp_check_boot_sign(struct btnxpuart_dev *nxpdev)
+{
+	serdev_device_set_baudrate(nxpdev->serdev, HCI_NXP_PRI_BAUDRATE);
+	serdev_device_set_flow_control(nxpdev->serdev, true);
+	set_bit(BTNXPUART_CHECK_BOOT_SIGNATURE, &nxpdev->tx_state);
+
+	return wait_event_interruptible_timeout(nxpdev->check_boot_sign_wait_q,
+					       !test_bit(BTNXPUART_CHECK_BOOT_SIGNATURE,
+							 &nxpdev->tx_state),
+					       msecs_to_jiffies(1000));
+}
+
+static int nxp_setup(struct hci_dev *hdev)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	int err = 0;
+
+	set_bit(BTNXPUART_FW_DOWNLOADING, &nxpdev->tx_state);
+	init_waitqueue_head(&nxpdev->fw_dnld_done_wait_q);
+	init_waitqueue_head(&nxpdev->check_boot_sign_wait_q);
+
+	if (nxp_check_boot_sign(nxpdev)) {
+		bt_dev_dbg(hdev, "Need FW Download.");
+		err = nxp_download_firmware(hdev);
+		if (err < 0)
+			return err;
+	} else {
+		bt_dev_dbg(hdev, "FW already running.");
+		clear_bit(BTNXPUART_FW_DOWNLOADING, &nxpdev->tx_state);
+	}
+
+	device_property_read_u32(&nxpdev->serdev->dev, "fw-init-baudrate",
+				 &nxpdev->fw_init_baudrate);
+	if (!nxpdev->fw_init_baudrate)
+		nxpdev->fw_init_baudrate = FW_INIT_BAUDRATE;
+	serdev_device_set_baudrate(nxpdev->serdev, nxpdev->fw_init_baudrate);
+	nxpdev->current_baudrate = nxpdev->fw_init_baudrate;
+
+	if (nxpdev->current_baudrate != HCI_NXP_SEC_BAUDRATE) {
+		nxpdev->new_baudrate = HCI_NXP_SEC_BAUDRATE;
+		hci_cmd_sync_queue(hdev, nxp_set_baudrate_cmd, NULL, NULL);
+	}
+
+	ps_init(hdev);
+
+	return 0;
+}
+
+static int btnxpuart_queue_skb(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+
+	/* Prepend skb with frame type */
+	memcpy(skb_push(skb, 1), &hci_skb_pkt_type(skb), 1);
+	skb_queue_tail(&nxpdev->txq, skb);
+	btnxpuart_tx_wakeup(nxpdev);
+	return 0;
+}
+
+static int nxp_enqueue(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	struct ps_data *psdata = &nxpdev->psdata;
+	struct hci_command_hdr *hdr;
+	struct psmode_cmd_payload ps_parm;
+	struct wakeup_cmd_payload wakeup_parm;
+	__le32 baudrate_parm;
+
+	/* if vendor commands are received from user space (e.g. hcitool), update
+	 * driver flags accordingly and ask driver to re-send the command to FW.
+	 * In case the payload for any command does not match expected payload
+	 * length, let the firmware and user space program handle it, or throw
+	 * an error.
+	 */
+	if (bt_cb(skb)->pkt_type == HCI_COMMAND_PKT && !psdata->driver_sent_cmd) {
+		hdr = (struct hci_command_hdr *)skb->data;
+		if (hdr->plen != (skb->len - HCI_COMMAND_HDR_SIZE))
+			return btnxpuart_queue_skb(hdev, skb);
+
+		switch (__le16_to_cpu(hdr->opcode)) {
+		case HCI_NXP_AUTO_SLEEP_MODE:
+			if (hdr->plen == sizeof(ps_parm)) {
+				memcpy(&ps_parm, skb->data + HCI_COMMAND_HDR_SIZE, hdr->plen);
+				if (ps_parm.ps_cmd == BT_PS_ENABLE)
+					psdata->target_ps_mode = PS_MODE_ENABLE;
+				else if (ps_parm.ps_cmd == BT_PS_DISABLE)
+					psdata->target_ps_mode = PS_MODE_DISABLE;
+				psdata->c2h_ps_interval = __le16_to_cpu(ps_parm.c2h_ps_interval);
+				hci_cmd_sync_queue(hdev, send_ps_cmd, NULL, NULL);
+				goto free_skb;
+			}
+			break;
+		case HCI_NXP_WAKEUP_METHOD:
+			if (hdr->plen == sizeof(wakeup_parm)) {
+				memcpy(&wakeup_parm, skb->data + HCI_COMMAND_HDR_SIZE, hdr->plen);
+				psdata->c2h_wakeupmode = wakeup_parm.c2h_wakeupmode;
+				psdata->c2h_wakeup_gpio = wakeup_parm.c2h_wakeup_gpio;
+				psdata->h2c_wakeup_gpio = wakeup_parm.h2c_wakeup_gpio;
+				switch (wakeup_parm.h2c_wakeupmode) {
+				case BT_CTRL_WAKEUP_METHOD_DSR:
+					psdata->h2c_wakeupmode = WAKEUP_METHOD_DTR;
+					break;
+				case BT_CTRL_WAKEUP_METHOD_BREAK:
+				default:
+					psdata->h2c_wakeupmode = WAKEUP_METHOD_BREAK;
+					break;
+				}
+				hci_cmd_sync_queue(hdev, send_wakeup_method_cmd, NULL, NULL);
+				goto free_skb;
+			}
+			break;
+		case HCI_NXP_SET_OPER_SPEED:
+			if (hdr->plen == sizeof(baudrate_parm)) {
+				memcpy(&baudrate_parm, skb->data + HCI_COMMAND_HDR_SIZE, hdr->plen);
+				nxpdev->new_baudrate = __le32_to_cpu(baudrate_parm);
+				hci_cmd_sync_queue(hdev, nxp_set_baudrate_cmd, NULL, NULL);
+				goto free_skb;
+			}
+			break;
+		case HCI_NXP_IND_RESET:
+			if (hdr->plen == 1) {
+				hci_cmd_sync_queue(hdev, nxp_set_ind_reset, NULL, NULL);
+				goto free_skb;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+	return btnxpuart_queue_skb(hdev, skb);
+
+free_skb:
+	kfree_skb(skb);
+	return 0;
+}
+
+static struct sk_buff *nxp_dequeue(void *data)
+{
+	struct btnxpuart_dev *nxpdev = (struct btnxpuart_dev *)data;
+
+	ps_wakeup(nxpdev);
+	ps_start_timer(nxpdev);
+	return skb_dequeue(&nxpdev->txq);
+}
+
+/* btnxpuart based on serdev */
+static void btnxpuart_tx_work(struct work_struct *work)
+{
+	struct btnxpuart_dev *nxpdev = container_of(work, struct btnxpuart_dev,
+						   tx_work);
+	struct serdev_device *serdev = nxpdev->serdev;
+	struct hci_dev *hdev = nxpdev->hdev;
+	struct sk_buff *skb;
+	int len;
+
+	while ((skb = nxp_dequeue(nxpdev))) {
+		len = serdev_device_write_buf(serdev, skb->data, skb->len);
+		hdev->stat.byte_tx += len;
+
+		skb_pull(skb, len);
+		if (skb->len > 0) {
+			skb_queue_head(&nxpdev->txq, skb);
+			break;
+		}
+
+		switch (hci_skb_pkt_type(skb)) {
+		case HCI_COMMAND_PKT:
+			hdev->stat.cmd_tx++;
+			break;
+		case HCI_ACLDATA_PKT:
+			hdev->stat.acl_tx++;
+			break;
+		case HCI_SCODATA_PKT:
+			hdev->stat.sco_tx++;
+			break;
+		}
+
+		kfree_skb(skb);
+	}
+	clear_bit(BTNXPUART_TX_STATE_ACTIVE, &nxpdev->tx_state);
+}
+
+static int btnxpuart_open(struct hci_dev *hdev)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+	int err = 0;
+
+	err = serdev_device_open(nxpdev->serdev);
+	if (err) {
+		bt_dev_err(hdev, "Unable to open UART device %s",
+			   dev_name(&nxpdev->serdev->dev));
+	} else {
+		set_bit(BTNXPUART_SERDEV_OPEN, &nxpdev->tx_state);
+	}
+	return err;
+}
+
+static int btnxpuart_close(struct hci_dev *hdev)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+
+	ps_wakeup(nxpdev);
+	serdev_device_close(nxpdev->serdev);
+	clear_bit(BTNXPUART_SERDEV_OPEN, &nxpdev->tx_state);
+	return 0;
+}
+
+static int btnxpuart_flush(struct hci_dev *hdev)
+{
+	struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev);
+
+	/* Flush any pending characters */
+	serdev_device_write_flush(nxpdev->serdev);
+	skb_queue_purge(&nxpdev->txq);
+
+	cancel_work_sync(&nxpdev->tx_work);
+
+	kfree_skb(nxpdev->rx_skb);
+	nxpdev->rx_skb = NULL;
+
+	return 0;
+}
+
+static const struct h4_recv_pkt nxp_recv_pkts[] = {
+	{ H4_RECV_ACL,          .recv = hci_recv_frame },
+	{ H4_RECV_SCO,          .recv = hci_recv_frame },
+	{ H4_RECV_EVENT,        .recv = hci_recv_frame },
+	{ NXP_RECV_CHIP_VER_V1, .recv = nxp_recv_chip_ver_v1 },
+	{ NXP_RECV_FW_REQ_V1,   .recv = nxp_recv_fw_req_v1 },
+	{ NXP_RECV_CHIP_VER_V3, .recv = nxp_recv_chip_ver_v3 },
+	{ NXP_RECV_FW_REQ_V3,   .recv = nxp_recv_fw_req_v3 },
+};
+
+static int btnxpuart_receive_buf(struct serdev_device *serdev, const u8 *data,
+				 size_t count)
+{
+	struct btnxpuart_dev *nxpdev = serdev_device_get_drvdata(serdev);
+
+	ps_start_timer(nxpdev);
+
+	nxpdev->rx_skb = h4_recv_buf(nxpdev->hdev, nxpdev->rx_skb, data, count,
+				     nxp_recv_pkts, ARRAY_SIZE(nxp_recv_pkts));
+	if (IS_ERR(nxpdev->rx_skb)) {
+		int err = PTR_ERR(nxpdev->rx_skb);
+		/* Safe to ignore out-of-sync bootloader signatures */
+		if (is_fw_downloading(nxpdev))
+			return count;
+		bt_dev_err(nxpdev->hdev, "Frame reassembly failed (%d)", err);
+		nxpdev->rx_skb = NULL;
+		return err;
+	}
+	nxpdev->hdev->stat.byte_rx += count;
+	return count;
+}
+
+static void btnxpuart_write_wakeup(struct serdev_device *serdev)
+{
+	serdev_device_write_wakeup(serdev);
+}
+
+static const struct serdev_device_ops btnxpuart_client_ops = {
+	.receive_buf = btnxpuart_receive_buf,
+	.write_wakeup = btnxpuart_write_wakeup,
+};
+
+static int nxp_serdev_probe(struct serdev_device *serdev)
+{
+	struct hci_dev *hdev;
+	struct btnxpuart_dev *nxpdev;
+
+	nxpdev = devm_kzalloc(&serdev->dev, sizeof(*nxpdev), GFP_KERNEL);
+	if (!nxpdev)
+		return -ENOMEM;
+
+	nxpdev->nxp_data = (struct btnxpuart_data *)device_get_match_data(&serdev->dev);
+
+	nxpdev->serdev = serdev;
+	serdev_device_set_drvdata(serdev, nxpdev);
+
+	serdev_device_set_client_ops(serdev, &btnxpuart_client_ops);
+
+	INIT_WORK(&nxpdev->tx_work, btnxpuart_tx_work);
+	skb_queue_head_init(&nxpdev->txq);
+
+	crc8_populate_msb(crc8_table, POLYNOMIAL8);
+
+	/* Initialize and register HCI device */
+	hdev = hci_alloc_dev();
+	if (!hdev) {
+		dev_err(&serdev->dev, "Can't allocate HCI device\n");
+		return -ENOMEM;
+	}
+
+	nxpdev->hdev = hdev;
+
+	hdev->bus = HCI_UART;
+	hci_set_drvdata(hdev, nxpdev);
+
+	hdev->manufacturer = MANUFACTURER_NXP;
+	hdev->open  = btnxpuart_open;
+	hdev->close = btnxpuart_close;
+	hdev->flush = btnxpuart_flush;
+	hdev->setup = nxp_setup;
+	hdev->send  = nxp_enqueue;
+	SET_HCIDEV_DEV(hdev, &serdev->dev);
+
+	if (hci_register_dev(hdev) < 0) {
+		dev_err(&serdev->dev, "Can't register HCI device\n");
+		hci_free_dev(hdev);
+		return -ENODEV;
+	}
+
+	ps_init_work(hdev);
+	ps_init_timer(hdev);
+
+	return 0;
+}
+
+static void nxp_serdev_remove(struct serdev_device *serdev)
+{
+	struct btnxpuart_dev *nxpdev = serdev_device_get_drvdata(serdev);
+	struct hci_dev *hdev = nxpdev->hdev;
+
+	/* Restore FW baudrate to fw_init_baudrate if changed.
+	 * This will ensure FW baudrate is in sync with
+	 * driver baudrate in case this driver is re-inserted.
+	 */
+	if (nxpdev->current_baudrate != nxpdev->fw_init_baudrate) {
+		nxpdev->new_baudrate = nxpdev->fw_init_baudrate;
+		nxp_set_baudrate_cmd(hdev, NULL);
+	}
+
+	ps_cancel_timer(nxpdev);
+	hci_unregister_dev(hdev);
+	hci_free_dev(hdev);
+}
+
+static struct btnxpuart_data w8987_data = {
+	.helper_fw_name = NULL,
+	.fw_name = FIRMWARE_W8987,
+};
+
+static struct btnxpuart_data w8997_data = {
+	.helper_fw_name = FIRMWARE_HELPER,
+	.fw_name = FIRMWARE_W8997,
+};
+
+static const struct of_device_id nxpuart_of_match_table[] = {
+	{ .compatible = "nxp,88w8987-bt", .data = &w8987_data },
+	{ .compatible = "nxp,88w8997-bt", .data = &w8997_data },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, nxpuart_of_match_table);
+
+static struct serdev_device_driver nxp_serdev_driver = {
+	.probe = nxp_serdev_probe,
+	.remove = nxp_serdev_remove,
+	.driver = {
+		.name = "btnxpuart",
+		.of_match_table = of_match_ptr(nxpuart_of_match_table),
+	},
+};
+
+module_serdev_device_driver(nxp_serdev_driver);
+
+MODULE_AUTHOR("Neeraj Sanjay Kale <neeraj.sanjaykale@nxp.com>");
+MODULE_DESCRIPTION("NXP Bluetooth Serial driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c
index c9064d34d830..fd0941fe8608 100644
--- a/drivers/bluetooth/btqca.c
+++ b/drivers/bluetooth/btqca.c
@@ -614,6 +614,9 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
 		config.type = ELF_TYPE_PATCH;
 		snprintf(config.fwname, sizeof(config.fwname),
 			 "qca/msbtfw%02x.mbn", rom_ver);
+	} else if (soc_type == QCA_WCN6855) {
+		snprintf(config.fwname, sizeof(config.fwname),
+			 "qca/hpbtfw%02x.tlv", rom_ver);
 	} else {
 		snprintf(config.fwname, sizeof(config.fwname),
 			 "qca/rampatch_%08x.bin", soc_ver);
@@ -648,6 +651,9 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
 	else if (soc_type == QCA_WCN6750)
 		snprintf(config.fwname, sizeof(config.fwname),
 			 "qca/msnv%02x.bin", rom_ver);
+	else if (soc_type == QCA_WCN6855)
+		snprintf(config.fwname, sizeof(config.fwname),
+			 "qca/hpnv%02x.bin", rom_ver);
 	else
 		snprintf(config.fwname, sizeof(config.fwname),
 			 "qca/nvm_%08x.bin", soc_ver);
@@ -685,11 +691,17 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
 		return err;
 	}
 
-	if (soc_type == QCA_WCN3991 || soc_type == QCA_WCN6750) {
+	switch (soc_type) {
+	case QCA_WCN3991:
+	case QCA_WCN6750:
+	case QCA_WCN6855:
 		/* get fw build info */
 		err = qca_read_fw_build_info(hdev);
 		if (err < 0)
 			return err;
+		break;
+	default:
+		break;
 	}
 
 	bt_dev_info(hdev, "QCA setup on UART is completed");
diff --git a/drivers/bluetooth/btqca.h b/drivers/bluetooth/btqca.h
index 61e9a50e66ae..b884095bcd9d 100644
--- a/drivers/bluetooth/btqca.h
+++ b/drivers/bluetooth/btqca.h
@@ -147,6 +147,7 @@ enum qca_btsoc_type {
 	QCA_WCN3991,
 	QCA_QCA6390,
 	QCA_WCN6750,
+	QCA_WCN6855,
 };
 
 #if IS_ENABLED(CONFIG_BT_QCA)
@@ -168,6 +169,10 @@ static inline bool qca_is_wcn6750(enum qca_btsoc_type soc_type)
 {
 	return soc_type == QCA_WCN6750;
 }
+static inline bool qca_is_wcn6855(enum qca_btsoc_type soc_type)
+{
+	return soc_type == QCA_WCN6855;
+}
 
 #else
 
@@ -206,6 +211,11 @@ static inline bool qca_is_wcn6750(enum qca_btsoc_type soc_type)
 	return false;
 }
 
+static inline bool qca_is_wcn6855(enum qca_btsoc_type soc_type)
+{
+	return false;
+}
+
 static inline int qca_send_pre_shutdown_cmd(struct hci_dev *hdev)
 {
 	return -EOPNOTSUPP;
diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c
index 69c3fe649ca7..2915c82d719d 100644
--- a/drivers/bluetooth/btrtl.c
+++ b/drivers/bluetooth/btrtl.c
@@ -17,19 +17,26 @@
 
 #define VERSION "0.1"
 
+#define RTL_CHIP_8723CS_CG	3
+#define RTL_CHIP_8723CS_VF	4
+#define RTL_CHIP_8723CS_XX	5
 #define RTL_EPATCH_SIGNATURE	"Realtech"
+#define RTL_EPATCH_SIGNATURE_V2	"RTBTCore"
+#define RTL_ROM_LMP_8703B	0x8703
 #define RTL_ROM_LMP_8723A	0x1200
 #define RTL_ROM_LMP_8723B	0x8723
 #define RTL_ROM_LMP_8821A	0x8821
 #define RTL_ROM_LMP_8761A	0x8761
 #define RTL_ROM_LMP_8822B	0x8822
 #define RTL_ROM_LMP_8852A	0x8852
+#define RTL_ROM_LMP_8851B	0x8851
 #define RTL_CONFIG_MAGIC	0x8723ab55
 
 #define IC_MATCH_FL_LMPSUBV	(1 << 0)
 #define IC_MATCH_FL_HCIREV	(1 << 1)
 #define IC_MATCH_FL_HCIVER	(1 << 2)
 #define IC_MATCH_FL_HCIBUS	(1 << 3)
+#define IC_MATCH_FL_CHIP_TYPE	(1 << 4)
 #define IC_INFO(lmps, hcir, hciv, bus) \
 	.match_flags = IC_MATCH_FL_LMPSUBV | IC_MATCH_FL_HCIREV | \
 		       IC_MATCH_FL_HCIVER | IC_MATCH_FL_HCIBUS, \
@@ -38,6 +45,14 @@
 	.hci_ver = (hciv), \
 	.hci_bus = (bus)
 
+#define	RTL_CHIP_SUBVER (&(struct rtl_vendor_cmd) {{0x10, 0x38, 0x04, 0x28, 0x80}})
+#define	RTL_CHIP_REV    (&(struct rtl_vendor_cmd) {{0x10, 0x3A, 0x04, 0x28, 0x80}})
+#define	RTL_SEC_PROJ    (&(struct rtl_vendor_cmd) {{0x10, 0xA4, 0x0D, 0x00, 0xb0}})
+
+#define RTL_PATCH_SNIPPETS		0x01
+#define RTL_PATCH_DUMMY_HEADER		0x02
+#define RTL_PATCH_SECURITY_HEADER	0x03
+
 enum btrtl_chip_id {
 	CHIP_ID_8723A,
 	CHIP_ID_8723B,
@@ -51,6 +66,7 @@ enum btrtl_chip_id {
 	CHIP_ID_8852A = 18,
 	CHIP_ID_8852B = 20,
 	CHIP_ID_8852C = 25,
+	CHIP_ID_8851B = 36,
 };
 
 struct id_table {
@@ -59,6 +75,7 @@ struct id_table {
 	__u16 hci_rev;
 	__u8 hci_ver;
 	__u8 hci_bus;
+	__u8 chip_type;
 	bool config_needed;
 	bool has_rom_version;
 	bool has_msft_ext;
@@ -75,6 +92,8 @@ struct btrtl_device_info {
 	int cfg_len;
 	bool drop_fw;
 	int project_id;
+	u8 key_id;
+	struct list_head patch_subsecs;
 };
 
 static const struct id_table ic_id_table[] = {
@@ -99,6 +118,39 @@ static const struct id_table ic_id_table[] = {
 	  .fw_name  = "rtl_bt/rtl8723b_fw.bin",
 	  .cfg_name = "rtl_bt/rtl8723b_config" },
 
+	/* 8723CS-CG */
+	{ .match_flags = IC_MATCH_FL_LMPSUBV | IC_MATCH_FL_CHIP_TYPE |
+			 IC_MATCH_FL_HCIBUS,
+	  .lmp_subver = RTL_ROM_LMP_8703B,
+	  .chip_type = RTL_CHIP_8723CS_CG,
+	  .hci_bus = HCI_UART,
+	  .config_needed = true,
+	  .has_rom_version = true,
+	  .fw_name  = "rtl_bt/rtl8723cs_cg_fw.bin",
+	  .cfg_name = "rtl_bt/rtl8723cs_cg_config" },
+
+	/* 8723CS-VF */
+	{ .match_flags = IC_MATCH_FL_LMPSUBV | IC_MATCH_FL_CHIP_TYPE |
+			 IC_MATCH_FL_HCIBUS,
+	  .lmp_subver = RTL_ROM_LMP_8703B,
+	  .chip_type = RTL_CHIP_8723CS_VF,
+	  .hci_bus = HCI_UART,
+	  .config_needed = true,
+	  .has_rom_version = true,
+	  .fw_name  = "rtl_bt/rtl8723cs_vf_fw.bin",
+	  .cfg_name = "rtl_bt/rtl8723cs_vf_config" },
+
+	/* 8723CS-XX */
+	{ .match_flags = IC_MATCH_FL_LMPSUBV | IC_MATCH_FL_CHIP_TYPE |
+			 IC_MATCH_FL_HCIBUS,
+	  .lmp_subver = RTL_ROM_LMP_8703B,
+	  .chip_type = RTL_CHIP_8723CS_XX,
+	  .hci_bus = HCI_UART,
+	  .config_needed = true,
+	  .has_rom_version = true,
+	  .fw_name  = "rtl_bt/rtl8723cs_xx_fw.bin",
+	  .cfg_name = "rtl_bt/rtl8723cs_xx_config" },
+
 	/* 8723D */
 	{ IC_INFO(RTL_ROM_LMP_8723B, 0xd, 0x8, HCI_USB),
 	  .config_needed = true,
@@ -128,6 +180,14 @@ static const struct id_table ic_id_table[] = {
 	  .fw_name  = "rtl_bt/rtl8821c_fw.bin",
 	  .cfg_name = "rtl_bt/rtl8821c_config" },
 
+	/* 8821CS */
+	{ IC_INFO(RTL_ROM_LMP_8821A, 0xc, 0x8, HCI_UART),
+	  .config_needed = true,
+	  .has_rom_version = true,
+	  .has_msft_ext = true,
+	  .fw_name  = "rtl_bt/rtl8821cs_fw.bin",
+	  .cfg_name = "rtl_bt/rtl8821cs_config" },
+
 	/* 8761A */
 	{ IC_INFO(RTL_ROM_LMP_8761A, 0xa, 0x6, HCI_USB),
 	  .config_needed = false,
@@ -190,6 +250,14 @@ static const struct id_table ic_id_table[] = {
 	  .fw_name  = "rtl_bt/rtl8852au_fw.bin",
 	  .cfg_name = "rtl_bt/rtl8852au_config" },
 
+	/* 8852B with UART interface */
+	{ IC_INFO(RTL_ROM_LMP_8852A, 0xb, 0xb, HCI_UART),
+	  .config_needed = true,
+	  .has_rom_version = true,
+	  .has_msft_ext = true,
+	  .fw_name  = "rtl_bt/rtl8852bs_fw.bin",
+	  .cfg_name = "rtl_bt/rtl8852bs_config" },
+
 	/* 8852B */
 	{ IC_INFO(RTL_ROM_LMP_8852A, 0xb, 0xb, HCI_USB),
 	  .config_needed = false,
@@ -205,10 +273,19 @@ static const struct id_table ic_id_table[] = {
 	  .has_msft_ext = true,
 	  .fw_name  = "rtl_bt/rtl8852cu_fw.bin",
 	  .cfg_name = "rtl_bt/rtl8852cu_config" },
+
+	/* 8851B */
+	{ IC_INFO(RTL_ROM_LMP_8851B, 0xb, 0xc, HCI_USB),
+	  .config_needed = false,
+	  .has_rom_version = true,
+	  .has_msft_ext = false,
+	  .fw_name  = "rtl_bt/rtl8851bu_fw.bin",
+	  .cfg_name = "rtl_bt/rtl8851bu_config" },
 	};
 
 static const struct id_table *btrtl_match_ic(u16 lmp_subver, u16 hci_rev,
-					     u8 hci_ver, u8 hci_bus)
+					     u8 hci_ver, u8 hci_bus,
+					     u8 chip_type)
 {
 	int i;
 
@@ -225,6 +302,9 @@ static const struct id_table *btrtl_match_ic(u16 lmp_subver, u16 hci_rev,
 		if ((ic_id_table[i].match_flags & IC_MATCH_FL_HCIBUS) &&
 		    (ic_id_table[i].hci_bus != hci_bus))
 			continue;
+		if ((ic_id_table[i].match_flags & IC_MATCH_FL_CHIP_TYPE) &&
+		    (ic_id_table[i].chip_type != chip_type))
+			continue;
 
 		break;
 	}
@@ -284,6 +364,227 @@ static int rtl_read_rom_version(struct hci_dev *hdev, u8 *version)
 	return 0;
 }
 
+static int btrtl_vendor_read_reg16(struct hci_dev *hdev,
+				   struct rtl_vendor_cmd *cmd, u8 *rp)
+{
+	struct sk_buff *skb;
+	int err = 0;
+
+	skb = __hci_cmd_sync(hdev, 0xfc61, sizeof(*cmd), cmd,
+			     HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		err = PTR_ERR(skb);
+		rtl_dev_err(hdev, "RTL: Read reg16 failed (%d)", err);
+		return err;
+	}
+
+	if (skb->len != 3 || skb->data[0]) {
+		bt_dev_err(hdev, "RTL: Read reg16 length mismatch");
+		kfree_skb(skb);
+		return -EIO;
+	}
+
+	if (rp)
+		memcpy(rp, skb->data + 1, 2);
+
+	kfree_skb(skb);
+
+	return 0;
+}
+
+static void *rtl_iov_pull_data(struct rtl_iovec *iov, u32 len)
+{
+	void *data = iov->data;
+
+	if (iov->len < len)
+		return NULL;
+
+	iov->data += len;
+	iov->len  -= len;
+
+	return data;
+}
+
+static void btrtl_insert_ordered_subsec(struct rtl_subsection *node,
+					struct btrtl_device_info *btrtl_dev)
+{
+	struct list_head *pos;
+	struct list_head *next;
+	struct rtl_subsection *subsec;
+
+	list_for_each_safe(pos, next, &btrtl_dev->patch_subsecs) {
+		subsec = list_entry(pos, struct rtl_subsection, list);
+		if (subsec->prio >= node->prio)
+			break;
+	}
+	__list_add(&node->list, pos->prev, pos);
+}
+
+static int btrtl_parse_section(struct hci_dev *hdev,
+			       struct btrtl_device_info *btrtl_dev, u32 opcode,
+			       u8 *data, u32 len)
+{
+	struct rtl_section_hdr *hdr;
+	struct rtl_subsection *subsec;
+	struct rtl_common_subsec *common_subsec;
+	struct rtl_sec_hdr *sec_hdr;
+	int i;
+	u8 *ptr;
+	u16 num_subsecs;
+	u32 subsec_len;
+	int rc = 0;
+	struct rtl_iovec iov = {
+		.data = data,
+		.len  = len,
+	};
+
+	hdr = rtl_iov_pull_data(&iov, sizeof(*hdr));
+	if (!hdr)
+		return -EINVAL;
+	num_subsecs = le16_to_cpu(hdr->num);
+
+	for (i = 0; i < num_subsecs; i++) {
+		common_subsec = rtl_iov_pull_data(&iov, sizeof(*common_subsec));
+		if (!common_subsec)
+			break;
+		subsec_len = le32_to_cpu(common_subsec->len);
+
+		rtl_dev_dbg(hdev, "subsec, eco 0x%02x, len %08x",
+			    common_subsec->eco, subsec_len);
+
+		ptr = rtl_iov_pull_data(&iov, subsec_len);
+		if (!ptr)
+			break;
+
+		if (common_subsec->eco != btrtl_dev->rom_version + 1)
+			continue;
+
+		switch (opcode) {
+		case RTL_PATCH_SECURITY_HEADER:
+			sec_hdr = (void *)common_subsec;
+			if (sec_hdr->key_id != btrtl_dev->key_id)
+				continue;
+			break;
+		}
+
+		subsec = kzalloc(sizeof(*subsec), GFP_KERNEL);
+		if (!subsec)
+			return -ENOMEM;
+		subsec->opcode = opcode;
+		subsec->prio = common_subsec->prio;
+		subsec->len  = subsec_len;
+		subsec->data = ptr;
+		btrtl_insert_ordered_subsec(subsec, btrtl_dev);
+		rc  += subsec_len;
+	}
+
+	return rc;
+}
+
+static int rtlbt_parse_firmware_v2(struct hci_dev *hdev,
+				   struct btrtl_device_info *btrtl_dev,
+				   unsigned char **_buf)
+{
+	struct rtl_epatch_header_v2 *hdr;
+	int rc;
+	u8 reg_val[2];
+	u8 key_id;
+	u32 num_sections;
+	struct rtl_section *section;
+	struct rtl_subsection *entry, *tmp;
+	u32 section_len;
+	u32 opcode;
+	int len = 0;
+	int i;
+	u8 *ptr;
+	struct rtl_iovec iov = {
+		.data = btrtl_dev->fw_data,
+		.len  = btrtl_dev->fw_len - 7, /* Cut the tail */
+	};
+
+	rc = btrtl_vendor_read_reg16(hdev, RTL_SEC_PROJ, reg_val);
+	if (rc < 0)
+		return -EIO;
+	key_id = reg_val[0];
+
+	rtl_dev_dbg(hdev, "%s: key id %u", __func__, key_id);
+
+	btrtl_dev->key_id = key_id;
+
+	hdr = rtl_iov_pull_data(&iov, sizeof(*hdr));
+	if (!hdr)
+		return -EINVAL;
+	num_sections = le32_to_cpu(hdr->num_sections);
+
+	rtl_dev_dbg(hdev, "FW version %08x-%08x", *((u32 *)hdr->fw_version),
+		    *((u32 *)(hdr->fw_version + 4)));
+
+	for (i = 0; i < num_sections; i++) {
+		section = rtl_iov_pull_data(&iov, sizeof(*section));
+		if (!section)
+			break;
+		section_len = le32_to_cpu(section->len);
+		opcode      = le32_to_cpu(section->opcode);
+
+		rtl_dev_dbg(hdev, "opcode 0x%04x", section->opcode);
+
+		ptr = rtl_iov_pull_data(&iov, section_len);
+		if (!ptr)
+			break;
+
+		switch (opcode) {
+		case RTL_PATCH_SNIPPETS:
+			rc = btrtl_parse_section(hdev, btrtl_dev, opcode,
+						 ptr, section_len);
+			break;
+		case RTL_PATCH_SECURITY_HEADER:
+			/* If key_id from chip is zero, ignore all security
+			 * headers.
+			 */
+			if (!key_id)
+				break;
+			rc = btrtl_parse_section(hdev, btrtl_dev, opcode,
+						 ptr, section_len);
+			break;
+		case RTL_PATCH_DUMMY_HEADER:
+			rc = btrtl_parse_section(hdev, btrtl_dev, opcode,
+						 ptr, section_len);
+			break;
+		default:
+			rc = 0;
+			break;
+		}
+		if (rc < 0) {
+			rtl_dev_err(hdev, "RTL: Parse section (%u) err %d",
+				    opcode, rc);
+			return rc;
+		}
+		len += rc;
+	}
+
+	if (!len)
+		return -ENODATA;
+
+	/* Allocate mem and copy all found subsecs. */
+	ptr = kvmalloc(len, GFP_KERNEL);
+	if (!ptr)
+		return -ENOMEM;
+
+	len = 0;
+	list_for_each_entry_safe(entry, tmp, &btrtl_dev->patch_subsecs, list) {
+		rtl_dev_dbg(hdev, "RTL: opcode %08x, addr %p, len 0x%x",
+			    entry->opcode, entry->data, entry->len);
+		memcpy(ptr + len, entry->data, entry->len);
+		len += entry->len;
+	}
+
+	if (!len)
+		return -EPERM;
+
+	*_buf = ptr;
+	return len;
+}
+
 static int rtlbt_parse_firmware(struct hci_dev *hdev,
 				struct btrtl_device_info *btrtl_dev,
 				unsigned char **_buf)
@@ -307,6 +608,7 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev,
 		{ RTL_ROM_LMP_8723B, 1 },
 		{ RTL_ROM_LMP_8821A, 2 },
 		{ RTL_ROM_LMP_8761A, 3 },
+		{ RTL_ROM_LMP_8703B, 7 },
 		{ RTL_ROM_LMP_8822B, 8 },
 		{ RTL_ROM_LMP_8723B, 9 },	/* 8723D */
 		{ RTL_ROM_LMP_8821A, 10 },	/* 8821C */
@@ -315,9 +617,21 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev,
 		{ RTL_ROM_LMP_8852A, 18 },	/* 8852A */
 		{ RTL_ROM_LMP_8852A, 20 },	/* 8852B */
 		{ RTL_ROM_LMP_8852A, 25 },	/* 8852C */
+		{ RTL_ROM_LMP_8851B, 36 },	/* 8851B */
 	};
 
-	min_size = sizeof(struct rtl_epatch_header) + sizeof(extension_sig) + 3;
+	if (btrtl_dev->fw_len <= 8)
+		return -EINVAL;
+
+	if (!memcmp(btrtl_dev->fw_data, RTL_EPATCH_SIGNATURE, 8))
+		min_size = sizeof(struct rtl_epatch_header) +
+				sizeof(extension_sig) + 3;
+	else if (!memcmp(btrtl_dev->fw_data, RTL_EPATCH_SIGNATURE_V2, 8))
+		min_size = sizeof(struct rtl_epatch_header_v2) +
+				sizeof(extension_sig) + 3;
+	else
+		return -EINVAL;
+
 	if (btrtl_dev->fw_len < min_size)
 		return -EINVAL;
 
@@ -382,12 +696,14 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev,
 		return -EINVAL;
 	}
 
-	epatch_info = (struct rtl_epatch_header *)btrtl_dev->fw_data;
-	if (memcmp(epatch_info->signature, RTL_EPATCH_SIGNATURE, 8) != 0) {
+	if (memcmp(btrtl_dev->fw_data, RTL_EPATCH_SIGNATURE, 8) != 0) {
+		if (!memcmp(btrtl_dev->fw_data, RTL_EPATCH_SIGNATURE_V2, 8))
+			return rtlbt_parse_firmware_v2(hdev, btrtl_dev, _buf);
 		rtl_dev_err(hdev, "bad EPATCH signature");
 		return -EINVAL;
 	}
 
+	epatch_info = (struct rtl_epatch_header *)btrtl_dev->fw_data;
 	num_patches = le16_to_cpu(epatch_info->num_patches);
 	BT_DBG("fw_version=%x, num_patches=%d",
 	       le32_to_cpu(epatch_info->fw_version), num_patches);
@@ -451,6 +767,7 @@ static int rtl_download_firmware(struct hci_dev *hdev,
 	int frag_len = RTL_FRAG_LEN;
 	int ret = 0;
 	int i;
+	int j = 0;
 	struct sk_buff *skb;
 	struct hci_rp_read_local_version *rp;
 
@@ -461,17 +778,16 @@ static int rtl_download_firmware(struct hci_dev *hdev,
 	for (i = 0; i < frag_num; i++) {
 		struct sk_buff *skb;
 
-		BT_DBG("download fw (%d/%d)", i, frag_num);
-
-		if (i > 0x7f)
-			dl_cmd->index = (i & 0x7f) + 1;
-		else
-			dl_cmd->index = i;
+		dl_cmd->index = j++;
+		if (dl_cmd->index == 0x7f)
+			j = 1;
 
 		if (i == (frag_num - 1)) {
 			dl_cmd->index |= 0x80; /* data end */
 			frag_len = fw_len % RTL_FRAG_LEN;
 		}
+		rtl_dev_dbg(hdev, "download fw (%d/%d). index = %d", i,
+				frag_num, dl_cmd->index);
 		memcpy(dl_cmd->data, data, frag_len);
 
 		/* Send download command */
@@ -587,10 +903,60 @@ out:
 	return ret;
 }
 
+static bool rtl_has_chip_type(u16 lmp_subver)
+{
+	switch (lmp_subver) {
+	case RTL_ROM_LMP_8703B:
+		return true;
+	default:
+		break;
+	}
+
+	return  false;
+}
+
+static int rtl_read_chip_type(struct hci_dev *hdev, u8 *type)
+{
+	struct rtl_chip_type_evt *chip_type;
+	struct sk_buff *skb;
+	const unsigned char cmd_buf[] = {0x00, 0x94, 0xa0, 0x00, 0xb0};
+
+	/* Read RTL chip type command */
+	skb = __hci_cmd_sync(hdev, 0xfc61, 5, cmd_buf, HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		rtl_dev_err(hdev, "Read chip type failed (%ld)",
+			    PTR_ERR(skb));
+		return PTR_ERR(skb);
+	}
+
+	chip_type = skb_pull_data(skb, sizeof(*chip_type));
+	if (!chip_type) {
+		rtl_dev_err(hdev, "RTL chip type event length mismatch");
+		kfree_skb(skb);
+		return -EIO;
+	}
+
+	rtl_dev_info(hdev, "chip_type status=%x type=%x",
+		     chip_type->status, chip_type->type);
+
+	*type = chip_type->type & 0x0f;
+
+	kfree_skb(skb);
+	return 0;
+}
+
 void btrtl_free(struct btrtl_device_info *btrtl_dev)
 {
+	struct rtl_subsection *entry, *tmp;
+
 	kvfree(btrtl_dev->fw_data);
 	kvfree(btrtl_dev->cfg_data);
+
+	list_for_each_entry_safe(entry, tmp, &btrtl_dev->patch_subsecs, list) {
+		list_del(&entry->list);
+		kfree(entry);
+	}
+
 	kfree(btrtl_dev);
 }
 EXPORT_SYMBOL_GPL(btrtl_free);
@@ -603,10 +969,11 @@ struct btrtl_device_info *btrtl_initialize(struct hci_dev *hdev,
 	struct hci_rp_read_local_version *resp;
 	char cfg_name[40];
 	u16 hci_rev, lmp_subver;
-	u8 hci_ver;
+	u8 hci_ver, lmp_ver, chip_type = 0;
 	int ret;
 	u16 opcode;
 	u8 cmd[2];
+	u8 reg_val[2];
 
 	btrtl_dev = kzalloc(sizeof(*btrtl_dev), GFP_KERNEL);
 	if (!btrtl_dev) {
@@ -614,6 +981,31 @@ struct btrtl_device_info *btrtl_initialize(struct hci_dev *hdev,
 		goto err_alloc;
 	}
 
+	INIT_LIST_HEAD(&btrtl_dev->patch_subsecs);
+
+check_version:
+	ret = btrtl_vendor_read_reg16(hdev, RTL_CHIP_SUBVER, reg_val);
+	if (ret < 0)
+		goto err_free;
+	lmp_subver = get_unaligned_le16(reg_val);
+
+	if (lmp_subver == RTL_ROM_LMP_8822B) {
+		ret = btrtl_vendor_read_reg16(hdev, RTL_CHIP_REV, reg_val);
+		if (ret < 0)
+			goto err_free;
+		hci_rev = get_unaligned_le16(reg_val);
+
+		/* 8822E */
+		if (hci_rev == 0x000e) {
+			hci_ver = 0x0c;
+			lmp_ver = 0x0c;
+			btrtl_dev->ic_info = btrtl_match_ic(lmp_subver, hci_rev,
+							    hci_ver, hdev->bus,
+							    chip_type);
+			goto next;
+		}
+	}
+
 	skb = btrtl_read_local_version(hdev);
 	if (IS_ERR(skb)) {
 		ret = PTR_ERR(skb);
@@ -621,19 +1013,32 @@ struct btrtl_device_info *btrtl_initialize(struct hci_dev *hdev,
 	}
 
 	resp = (struct hci_rp_read_local_version *)skb->data;
-	rtl_dev_info(hdev, "examining hci_ver=%02x hci_rev=%04x lmp_ver=%02x lmp_subver=%04x",
-		     resp->hci_ver, resp->hci_rev,
-		     resp->lmp_ver, resp->lmp_subver);
 
-	hci_ver = resp->hci_ver;
-	hci_rev = le16_to_cpu(resp->hci_rev);
+	hci_ver    = resp->hci_ver;
+	hci_rev    = le16_to_cpu(resp->hci_rev);
+	lmp_ver    = resp->lmp_ver;
 	lmp_subver = le16_to_cpu(resp->lmp_subver);
 
+	kfree_skb(skb);
+
+	if (rtl_has_chip_type(lmp_subver)) {
+		ret = rtl_read_chip_type(hdev, &chip_type);
+		if (ret)
+			goto err_free;
+	}
+
 	btrtl_dev->ic_info = btrtl_match_ic(lmp_subver, hci_rev, hci_ver,
-					    hdev->bus);
+					    hdev->bus, chip_type);
 
-	if (!btrtl_dev->ic_info)
+next:
+	rtl_dev_info(hdev, "examining hci_ver=%02x hci_rev=%04x lmp_ver=%02x lmp_subver=%04x",
+		     hci_ver, hci_rev,
+		     lmp_ver, lmp_subver);
+
+	if (!btrtl_dev->ic_info && !btrtl_dev->drop_fw)
 		btrtl_dev->drop_fw = true;
+	else
+		btrtl_dev->drop_fw = false;
 
 	if (btrtl_dev->drop_fw) {
 		opcode = hci_opcode_pack(0x3f, 0x66);
@@ -642,41 +1047,25 @@ struct btrtl_device_info *btrtl_initialize(struct hci_dev *hdev,
 
 		skb = bt_skb_alloc(sizeof(cmd), GFP_KERNEL);
 		if (!skb)
-			goto out_free;
+			goto err_free;
 
 		skb_put_data(skb, cmd, sizeof(cmd));
 		hci_skb_pkt_type(skb) = HCI_COMMAND_PKT;
 
-		hdev->send(hdev, skb);
+		ret = hdev->send(hdev, skb);
+		if (ret < 0) {
+			bt_dev_err(hdev, "sending frame failed (%d)", ret);
+			kfree_skb(skb);
+			goto err_free;
+		}
 
 		/* Ensure the above vendor command is sent to controller and
 		 * process has done.
 		 */
 		msleep(200);
 
-		/* Read the local version again. Expect to have the vanilla
-		 * version as cold boot.
-		 */
-		skb = btrtl_read_local_version(hdev);
-		if (IS_ERR(skb)) {
-			ret = PTR_ERR(skb);
-			goto err_free;
-		}
-
-		resp = (struct hci_rp_read_local_version *)skb->data;
-		rtl_dev_info(hdev, "examining hci_ver=%02x hci_rev=%04x lmp_ver=%02x lmp_subver=%04x",
-			     resp->hci_ver, resp->hci_rev,
-			     resp->lmp_ver, resp->lmp_subver);
-
-		hci_ver = resp->hci_ver;
-		hci_rev = le16_to_cpu(resp->hci_rev);
-		lmp_subver = le16_to_cpu(resp->lmp_subver);
-
-		btrtl_dev->ic_info = btrtl_match_ic(lmp_subver, hci_rev, hci_ver,
-						    hdev->bus);
+		goto check_version;
 	}
-out_free:
-	kfree_skb(skb);
 
 	if (!btrtl_dev->ic_info) {
 		rtl_dev_info(hdev, "unknown IC info, lmp subver %04x, hci rev %04x, hci ver %04x",
@@ -755,6 +1144,8 @@ int btrtl_download_firmware(struct hci_dev *hdev,
 	case RTL_ROM_LMP_8761A:
 	case RTL_ROM_LMP_8822B:
 	case RTL_ROM_LMP_8852A:
+	case RTL_ROM_LMP_8703B:
+	case RTL_ROM_LMP_8851B:
 		return btrtl_setup_rtl8723b(hdev, btrtl_dev);
 	default:
 		rtl_dev_info(hdev, "assuming no firmware upload needed");
@@ -779,6 +1170,7 @@ void btrtl_set_quirks(struct hci_dev *hdev, struct btrtl_device_info *btrtl_dev)
 	case CHIP_ID_8852A:
 	case CHIP_ID_8852B:
 	case CHIP_ID_8852C:
+	case CHIP_ID_8851B:
 		set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks);
 		set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED, &hdev->quirks);
 
@@ -795,6 +1187,22 @@ void btrtl_set_quirks(struct hci_dev *hdev, struct btrtl_device_info *btrtl_dev)
 		rtl_dev_dbg(hdev, "WBS supported not enabled.");
 		break;
 	}
+
+	if (!btrtl_dev->ic_info)
+		return;
+
+	switch (btrtl_dev->ic_info->lmp_subver) {
+	case RTL_ROM_LMP_8703B:
+		/* 8723CS reports two pages for local ext features,
+		 * but it doesn't support any features from page 2 -
+		 * it either responds with garbage or with error status
+		 */
+		set_bit(HCI_QUIRK_BROKEN_LOCAL_EXT_FEATURES_PAGE_2,
+			&hdev->quirks);
+		break;
+	default:
+		break;
+	}
 }
 EXPORT_SYMBOL_GPL(btrtl_set_quirks);
 
@@ -953,6 +1361,12 @@ MODULE_FIRMWARE("rtl_bt/rtl8723b_fw.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8723b_config.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8723bs_fw.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8723bs_config.bin");
+MODULE_FIRMWARE("rtl_bt/rtl8723cs_cg_fw.bin");
+MODULE_FIRMWARE("rtl_bt/rtl8723cs_cg_config.bin");
+MODULE_FIRMWARE("rtl_bt/rtl8723cs_vf_fw.bin");
+MODULE_FIRMWARE("rtl_bt/rtl8723cs_vf_config.bin");
+MODULE_FIRMWARE("rtl_bt/rtl8723cs_xx_fw.bin");
+MODULE_FIRMWARE("rtl_bt/rtl8723cs_xx_config.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8723ds_fw.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8723ds_config.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8761a_fw.bin");
@@ -963,7 +1377,11 @@ MODULE_FIRMWARE("rtl_bt/rtl8822b_fw.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8822b_config.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8852au_fw.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8852au_config.bin");
+MODULE_FIRMWARE("rtl_bt/rtl8852bs_fw.bin");
+MODULE_FIRMWARE("rtl_bt/rtl8852bs_config.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8852bu_fw.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8852bu_config.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8852cu_fw.bin");
 MODULE_FIRMWARE("rtl_bt/rtl8852cu_config.bin");
+MODULE_FIRMWARE("rtl_bt/rtl8851bu_fw.bin");
+MODULE_FIRMWARE("rtl_bt/rtl8851bu_config.bin");
diff --git a/drivers/bluetooth/btrtl.h b/drivers/bluetooth/btrtl.h
index ebf0101c959b..adb4c2c9abc5 100644
--- a/drivers/bluetooth/btrtl.h
+++ b/drivers/bluetooth/btrtl.h
@@ -14,6 +14,11 @@
 
 struct btrtl_device_info;
 
+struct rtl_chip_type_evt {
+	__u8 status;
+	__u8 type;
+} __packed;
+
 struct rtl_download_cmd {
 	__u8 index;
 	__u8 data[RTL_FRAG_LEN];
@@ -44,7 +49,58 @@ struct rtl_vendor_config_entry {
 struct rtl_vendor_config {
 	__le32 signature;
 	__le16 total_len;
-	struct rtl_vendor_config_entry entry[];
+	__u8 entry[];
+} __packed;
+
+struct rtl_epatch_header_v2 {
+	__u8   signature[8];
+	__u8   fw_version[8];
+	__le32 num_sections;
+} __packed;
+
+struct rtl_section {
+	__le32 opcode;
+	__le32 len;
+	u8     data[];
+} __packed;
+
+struct rtl_section_hdr {
+	__le16 num;
+	__le16 reserved;
+} __packed;
+
+struct rtl_common_subsec {
+	__u8   eco;
+	__u8   prio;
+	__u8   cb[2];
+	__le32 len;
+	__u8   data[];
+};
+
+struct rtl_sec_hdr {
+	__u8   eco;
+	__u8   prio;
+	__u8   key_id;
+	__u8   reserved;
+	__le32 len;
+	__u8   data[];
+} __packed;
+
+struct rtl_subsection {
+	struct list_head list;
+	u32 opcode;
+	u32 len;
+	u8 prio;
+	u8 *data;
+};
+
+struct rtl_iovec {
+	u8  *data;
+	u32 len;
+};
+
+struct rtl_vendor_cmd {
+	__u8 param[5];
 } __packed;
 
 enum {
diff --git a/drivers/bluetooth/btsdio.c b/drivers/bluetooth/btsdio.c
index 51000320e1ea..f19d31ee37ea 100644
--- a/drivers/bluetooth/btsdio.c
+++ b/drivers/bluetooth/btsdio.c
@@ -354,7 +354,6 @@ static void btsdio_remove(struct sdio_func *func)
 
 	BT_DBG("func %p", func);
 
-	cancel_work_sync(&data->work);
 	if (!data)
 		return;
 
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index 5c536151ef83..2a8e2bb038f5 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -540,6 +540,10 @@ static const struct usb_device_id blacklist_table[] = {
 	/* Realtek 8852BE Bluetooth devices */
 	{ USB_DEVICE(0x0cb8, 0xc559), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0x887b), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3571), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
 
 	/* Realtek Bluetooth devices */
 	{ USB_VENDOR_AND_INTERFACE_INFO(0x0bda, 0xe0, 0x01, 0x01),
@@ -558,6 +562,9 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x043e, 0x310c), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH |
 						     BTUSB_VALID_LE_STATES },
+	{ USB_DEVICE(0x04ca, 0x3801), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH |
+						     BTUSB_VALID_LE_STATES },
 
 	/* Additional MediaTek MT7668 Bluetooth devices */
 	{ USB_DEVICE(0x043e, 0x3109), .driver_info = BTUSB_MEDIATEK |
@@ -612,6 +619,9 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x0489, 0xe0e2), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH |
 						     BTUSB_VALID_LE_STATES },
+	{ USB_DEVICE(0x0489, 0xe0e4), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH |
+						     BTUSB_VALID_LE_STATES },
 	{ USB_DEVICE(0x0489, 0xe0f2), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH |
 						     BTUSB_VALID_LE_STATES },
@@ -723,6 +733,16 @@ static const struct dmi_system_id btusb_needs_reset_resume_table[] = {
 	{}
 };
 
+struct qca_dump_info {
+	/* fields for dump collection */
+	u16 id_vendor;
+	u16 id_product;
+	u32 fw_version;
+	u32 controller_id;
+	u32 ram_dump_size;
+	u16 ram_dump_seqno;
+};
+
 #define BTUSB_MAX_ISOC_FRAMES	10
 
 #define BTUSB_INTR_RUNNING	0
@@ -742,6 +762,7 @@ static const struct dmi_system_id btusb_needs_reset_resume_table[] = {
 #define BTUSB_WAKEUP_AUTOSUSPEND	14
 #define BTUSB_USE_ALT3_FOR_WBS	15
 #define BTUSB_ALT6_CONTINUOUS_TX	16
+#define BTUSB_HW_SSR_ACTIVE	17
 
 struct btusb_data {
 	struct hci_dev       *hdev;
@@ -804,6 +825,8 @@ struct btusb_data {
 
 	int oob_wake_irq;   /* irq for out-of-band wake-on-bt */
 	unsigned cmd_timeout_cnt;
+
+	struct qca_dump_info qca_dump;
 };
 
 static void btusb_reset(struct hci_dev *hdev)
@@ -894,6 +917,11 @@ static void btusb_qca_cmd_timeout(struct hci_dev *hdev)
 	struct btusb_data *data = hci_get_drvdata(hdev);
 	struct gpio_desc *reset_gpio = data->reset_gpio;
 
+	if (test_bit(BTUSB_HW_SSR_ACTIVE, &data->flags)) {
+		bt_dev_info(hdev, "Ramdump in progress, defer cmd_timeout");
+		return;
+	}
+
 	if (++data->cmd_timeout_cnt < 5)
 		return;
 
@@ -2376,16 +2404,47 @@ static int btusb_recv_bulk_intel(struct btusb_data *data, void *buffer,
 	return btusb_recv_bulk(data, buffer, count);
 }
 
+static int btusb_intel_diagnostics(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct intel_tlv *tlv = (void *)&skb->data[5];
+
+	/* The first event is always an event type TLV */
+	if (tlv->type != INTEL_TLV_TYPE_ID)
+		goto recv_frame;
+
+	switch (tlv->val[0]) {
+	case INTEL_TLV_SYSTEM_EXCEPTION:
+	case INTEL_TLV_FATAL_EXCEPTION:
+	case INTEL_TLV_DEBUG_EXCEPTION:
+	case INTEL_TLV_TEST_EXCEPTION:
+		/* Generate devcoredump from exception */
+		if (!hci_devcd_init(hdev, skb->len)) {
+			hci_devcd_append(hdev, skb);
+			hci_devcd_complete(hdev);
+		} else {
+			bt_dev_err(hdev, "Failed to generate devcoredump");
+			kfree_skb(skb);
+		}
+		return 0;
+	default:
+		bt_dev_err(hdev, "Invalid exception type %02X", tlv->val[0]);
+	}
+
+recv_frame:
+	return hci_recv_frame(hdev, skb);
+}
+
 static int btusb_recv_event_intel(struct hci_dev *hdev, struct sk_buff *skb)
 {
-	if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) {
-		struct hci_event_hdr *hdr = (void *)skb->data;
+	struct hci_event_hdr *hdr = (void *)skb->data;
+	const char diagnostics_hdr[] = { 0x87, 0x80, 0x03 };
 
-		if (skb->len > HCI_EVENT_HDR_SIZE && hdr->evt == 0xff &&
-		    hdr->plen > 0) {
-			const void *ptr = skb->data + HCI_EVENT_HDR_SIZE + 1;
-			unsigned int len = skb->len - HCI_EVENT_HDR_SIZE - 1;
+	if (skb->len > HCI_EVENT_HDR_SIZE && hdr->evt == 0xff &&
+	    hdr->plen > 0) {
+		const void *ptr = skb->data + HCI_EVENT_HDR_SIZE + 1;
+		unsigned int len = skb->len - HCI_EVENT_HDR_SIZE - 1;
 
+		if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) {
 			switch (skb->data[2]) {
 			case 0x02:
 				/* When switching to the operational firmware
@@ -2404,6 +2463,15 @@ static int btusb_recv_event_intel(struct hci_dev *hdev, struct sk_buff *skb)
 				break;
 			}
 		}
+
+		/* Handle all diagnostics events separately. May still call
+		 * hci_recv_frame.
+		 */
+		if (len >= sizeof(diagnostics_hdr) &&
+		    memcmp(&skb->data[2], diagnostics_hdr,
+			   sizeof(diagnostics_hdr)) == 0) {
+			return btusb_intel_diagnostics(hdev, skb);
+		}
 	}
 
 	return hci_recv_frame(hdev, skb);
@@ -3244,6 +3312,202 @@ static int btusb_set_bdaddr_wcn6855(struct hci_dev *hdev,
 	return 0;
 }
 
+#define QCA_MEMDUMP_ACL_HANDLE 0x2EDD
+#define QCA_MEMDUMP_SIZE_MAX  0x100000
+#define QCA_MEMDUMP_VSE_CLASS 0x01
+#define QCA_MEMDUMP_MSG_TYPE 0x08
+#define QCA_MEMDUMP_PKT_SIZE 248
+#define QCA_LAST_SEQUENCE_NUM 0xffff
+
+struct qca_dump_hdr {
+	u8 vse_class;
+	u8 msg_type;
+	__le16 seqno;
+	u8 reserved;
+	union {
+		u8 data[0];
+		struct {
+			__le32 ram_dump_size;
+			u8 data0[0];
+		} __packed;
+	};
+} __packed;
+
+
+static void btusb_dump_hdr_qca(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	char buf[128];
+	struct btusb_data *btdata = hci_get_drvdata(hdev);
+
+	snprintf(buf, sizeof(buf), "Controller Name: 0x%x\n",
+			btdata->qca_dump.controller_id);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Firmware Version: 0x%x\n",
+			btdata->qca_dump.fw_version);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Driver: %s\nVendor: qca\n",
+			btusb_driver.name);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "VID: 0x%x\nPID:0x%x\n",
+			btdata->qca_dump.id_vendor, btdata->qca_dump.id_product);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Lmp Subversion: 0x%x\n",
+			hdev->lmp_subver);
+	skb_put_data(skb, buf, strlen(buf));
+}
+
+static void btusb_coredump_qca(struct hci_dev *hdev)
+{
+	static const u8 param[] = { 0x26 };
+	struct sk_buff *skb;
+
+	skb = __hci_cmd_sync(hdev, 0xfc0c, 1, param, HCI_CMD_TIMEOUT);
+	if (IS_ERR(skb))
+		bt_dev_err(hdev, "%s: triggle crash failed (%ld)", __func__, PTR_ERR(skb));
+	kfree_skb(skb);
+}
+
+/*
+ * ==0: not a dump pkt.
+ * < 0: fails to handle a dump pkt
+ * > 0: otherwise.
+ */
+static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	int ret = 1;
+	u8 pkt_type;
+	u8 *sk_ptr;
+	unsigned int sk_len;
+	u16 seqno;
+	u32 dump_size;
+
+	struct hci_event_hdr *event_hdr;
+	struct hci_acl_hdr *acl_hdr;
+	struct qca_dump_hdr *dump_hdr;
+	struct btusb_data *btdata = hci_get_drvdata(hdev);
+	struct usb_device *udev = btdata->udev;
+
+	pkt_type = hci_skb_pkt_type(skb);
+	sk_ptr = skb->data;
+	sk_len = skb->len;
+
+	if (pkt_type == HCI_ACLDATA_PKT) {
+		acl_hdr = hci_acl_hdr(skb);
+		if (le16_to_cpu(acl_hdr->handle) != QCA_MEMDUMP_ACL_HANDLE)
+			return 0;
+		sk_ptr += HCI_ACL_HDR_SIZE;
+		sk_len -= HCI_ACL_HDR_SIZE;
+		event_hdr = (struct hci_event_hdr *)sk_ptr;
+	} else {
+		event_hdr = hci_event_hdr(skb);
+	}
+
+	if ((event_hdr->evt != HCI_VENDOR_PKT)
+		|| (event_hdr->plen != (sk_len - HCI_EVENT_HDR_SIZE)))
+		return 0;
+
+	sk_ptr += HCI_EVENT_HDR_SIZE;
+	sk_len -= HCI_EVENT_HDR_SIZE;
+
+	dump_hdr = (struct qca_dump_hdr *)sk_ptr;
+	if ((sk_len < offsetof(struct qca_dump_hdr, data))
+		|| (dump_hdr->vse_class != QCA_MEMDUMP_VSE_CLASS)
+	    || (dump_hdr->msg_type != QCA_MEMDUMP_MSG_TYPE))
+		return 0;
+
+	/*it is dump pkt now*/
+	seqno = le16_to_cpu(dump_hdr->seqno);
+	if (seqno == 0) {
+		set_bit(BTUSB_HW_SSR_ACTIVE, &btdata->flags);
+		dump_size = le32_to_cpu(dump_hdr->ram_dump_size);
+		if (!dump_size || (dump_size > QCA_MEMDUMP_SIZE_MAX)) {
+			ret = -EILSEQ;
+			bt_dev_err(hdev, "Invalid memdump size(%u)",
+				   dump_size);
+			goto out;
+		}
+
+		ret = hci_devcd_init(hdev, dump_size);
+		if (ret < 0) {
+			bt_dev_err(hdev, "memdump init error(%d)", ret);
+			goto out;
+		}
+
+		btdata->qca_dump.ram_dump_size = dump_size;
+		btdata->qca_dump.ram_dump_seqno = 0;
+		sk_ptr += offsetof(struct qca_dump_hdr, data0);
+		sk_len -= offsetof(struct qca_dump_hdr, data0);
+
+		usb_disable_autosuspend(udev);
+		bt_dev_info(hdev, "%s memdump size(%u)\n",
+			    (pkt_type == HCI_ACLDATA_PKT) ? "ACL" : "event",
+			    dump_size);
+	} else {
+		sk_ptr += offsetof(struct qca_dump_hdr, data);
+		sk_len -= offsetof(struct qca_dump_hdr, data);
+	}
+
+	if (!btdata->qca_dump.ram_dump_size) {
+		ret = -EINVAL;
+		bt_dev_err(hdev, "memdump is not active");
+		goto out;
+	}
+
+	if ((seqno > btdata->qca_dump.ram_dump_seqno + 1) && (seqno != QCA_LAST_SEQUENCE_NUM)) {
+		dump_size = QCA_MEMDUMP_PKT_SIZE * (seqno - btdata->qca_dump.ram_dump_seqno - 1);
+		hci_devcd_append_pattern(hdev, 0x0, dump_size);
+		bt_dev_err(hdev,
+			   "expected memdump seqno(%u) is not received(%u)\n",
+			   btdata->qca_dump.ram_dump_seqno, seqno);
+		btdata->qca_dump.ram_dump_seqno = seqno;
+		kfree_skb(skb);
+		return ret;
+	}
+
+	skb_pull(skb, skb->len - sk_len);
+	hci_devcd_append(hdev, skb);
+	btdata->qca_dump.ram_dump_seqno++;
+	if (seqno == QCA_LAST_SEQUENCE_NUM) {
+		bt_dev_info(hdev,
+				"memdump done: pkts(%u), total(%u)\n",
+				btdata->qca_dump.ram_dump_seqno, btdata->qca_dump.ram_dump_size);
+
+		hci_devcd_complete(hdev);
+		goto out;
+	}
+	return ret;
+
+out:
+	if (btdata->qca_dump.ram_dump_size)
+		usb_enable_autosuspend(udev);
+	btdata->qca_dump.ram_dump_size = 0;
+	btdata->qca_dump.ram_dump_seqno = 0;
+	clear_bit(BTUSB_HW_SSR_ACTIVE, &btdata->flags);
+
+	if (ret < 0)
+		kfree_skb(skb);
+	return ret;
+}
+
+static int btusb_recv_acl_qca(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	if (handle_dump_pkt_qca(hdev, skb))
+		return 0;
+	return hci_recv_frame(hdev, skb);
+}
+
+static int btusb_recv_evt_qca(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	if (handle_dump_pkt_qca(hdev, skb))
+		return 0;
+	return hci_recv_frame(hdev, skb);
+}
+
+
 #define QCA_DFU_PACKET_LEN	4096
 
 #define QCA_GET_TARGET_VERSION	0x09
@@ -3578,6 +3842,9 @@ static int btusb_setup_qca(struct hci_dev *hdev)
 	if (err < 0)
 		return err;
 
+	btdata->qca_dump.fw_version = le32_to_cpu(ver.patch_version);
+	btdata->qca_dump.controller_id = le32_to_cpu(ver.rom_version);
+
 	if (!(status & QCA_SYSCFG_UPDATED)) {
 		err = btusb_setup_qca_load_nvm(hdev, &ver, info);
 		if (err < 0)
@@ -3831,13 +4098,9 @@ static int btusb_probe(struct usb_interface *intf,
 
 	BT_DBG("intf %p id %p", intf, id);
 
-	/* interface numbers are hardcoded in the spec */
-	if (intf->cur_altsetting->desc.bInterfaceNumber != 0) {
-		if (!(id->driver_info & BTUSB_IFNUM_2))
-			return -ENODEV;
-		if (intf->cur_altsetting->desc.bInterfaceNumber != 2)
-			return -ENODEV;
-	}
+	if ((id->driver_info & BTUSB_IFNUM_2) &&
+	    (intf->cur_altsetting->desc.bInterfaceNumber != 2))
+		return -ENODEV;
 
 	ifnum_base = intf->cur_altsetting->desc.bInterfaceNumber;
 
@@ -4012,7 +4275,7 @@ static int btusb_probe(struct usb_interface *intf,
 
 	/* Combined Intel Device setup to support multiple setup routine */
 	if (id->driver_info & BTUSB_INTEL_COMBINED) {
-		err = btintel_configure_setup(hdev);
+		err = btintel_configure_setup(hdev, btusb_driver.name);
 		if (err)
 			goto out_free_dev;
 
@@ -4071,6 +4334,11 @@ static int btusb_probe(struct usb_interface *intf,
 	}
 
 	if (id->driver_info & BTUSB_QCA_WCN6855) {
+		data->qca_dump.id_vendor = id->idVendor;
+		data->qca_dump.id_product = id->idProduct;
+		data->recv_event = btusb_recv_evt_qca;
+		data->recv_acl = btusb_recv_acl_qca;
+		hci_devcd_register(hdev, btusb_coredump_qca, btusb_dump_hdr_qca, NULL);
 		data->setup_on_usb = btusb_setup_qca;
 		hdev->shutdown = btusb_shutdown_qca;
 		hdev->set_bdaddr = btusb_set_bdaddr_wcn6855;
@@ -4102,6 +4370,9 @@ static int btusb_probe(struct usb_interface *intf,
 	if (id->driver_info & BTUSB_ACTIONS_SEMI) {
 		/* Support is advertised, but not implemented */
 		set_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks);
+		set_bit(HCI_QUIRK_BROKEN_READ_TRANSMIT_POWER, &hdev->quirks);
+		set_bit(HCI_QUIRK_BROKEN_SET_RPA_TIMEOUT, &hdev->quirks);
+		set_bit(HCI_QUIRK_BROKEN_EXT_SCAN, &hdev->quirks);
 	}
 
 	if (!reset)
@@ -4389,6 +4660,17 @@ done:
 }
 #endif
 
+#ifdef CONFIG_DEV_COREDUMP
+static void btusb_coredump(struct device *dev)
+{
+	struct btusb_data *data = dev_get_drvdata(dev);
+	struct hci_dev *hdev = data->hdev;
+
+	if (hdev->dump.coredump)
+		hdev->dump.coredump(hdev);
+}
+#endif
+
 static struct usb_driver btusb_driver = {
 	.name		= "btusb",
 	.probe		= btusb_probe,
@@ -4400,6 +4682,14 @@ static struct usb_driver btusb_driver = {
 	.id_table	= btusb_table,
 	.supports_autosuspend = 1,
 	.disable_hub_initiated_lpm = 1,
+
+#ifdef CONFIG_DEV_COREDUMP
+	.drvwrap = {
+		.driver = {
+			.coredump = btusb_coredump,
+		},
+	},
+#endif
 };
 
 module_usb_driver(btusb_driver);
diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c
index 2b6c0e1922cb..83bf5d4330c4 100644
--- a/drivers/bluetooth/hci_bcm.c
+++ b/drivers/bluetooth/hci_bcm.c
@@ -55,12 +55,14 @@
  * @drive_rts_on_open: drive RTS signal on ->open() when platform requires it
  * @no_uart_clock_set: UART clock set command for >3Mbps mode is unavailable
  * @max_autobaud_speed: max baudrate supported by device in autobaud mode
+ * @max_speed: max baudrate supported
  */
 struct bcm_device_data {
 	bool	no_early_set_baudrate;
 	bool	drive_rts_on_open;
 	bool	no_uart_clock_set;
 	u32	max_autobaud_speed;
+	u32	max_speed;
 };
 
 /**
@@ -888,7 +890,7 @@ unlock:
 #endif
 
 /* Some firmware reports an IRQ which does not work (wrong pin in fw table?) */
-static struct gpiod_lookup_table asus_tf103c_irq_gpios = {
+static struct gpiod_lookup_table irq_on_int33fc02_pin17_gpios = {
 	.dev_id = "serial0-0",
 	.table = {
 		GPIO_LOOKUP("INT33FC:02", 17, "host-wakeup-alt", GPIO_ACTIVE_HIGH),
@@ -898,12 +900,31 @@ static struct gpiod_lookup_table asus_tf103c_irq_gpios = {
 
 static const struct dmi_system_id bcm_broken_irq_dmi_table[] = {
 	{
+		.ident = "Acer Iconia One 7 B1-750",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Insyde"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "VESPA2"),
+		},
+		.driver_data = &irq_on_int33fc02_pin17_gpios,
+	},
+	{
 		.ident = "Asus TF103C",
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
 			DMI_MATCH(DMI_PRODUCT_NAME, "TF103C"),
 		},
-		.driver_data = &asus_tf103c_irq_gpios,
+		.driver_data = &irq_on_int33fc02_pin17_gpios,
+	},
+	{
+		.ident = "Lenovo Yoga Tablet 2 830F/L / 1050F/L",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Intel Corp."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "VALLEYVIEW C0 PLATFORM"),
+			DMI_MATCH(DMI_BOARD_NAME, "BYT-T FFD8"),
+			/* Partial match on beginning of BIOS version */
+			DMI_MATCH(DMI_BIOS_VERSION, "BLADE_21"),
+		},
+		.driver_data = &irq_on_int33fc02_pin17_gpios,
 	},
 	{
 		.ident = "Meegopad T08",
@@ -1300,6 +1321,12 @@ static const struct hci_uart_proto bcm_proto = {
 };
 
 #ifdef CONFIG_ACPI
+
+/* bcm43430a0/a1 BT does not support 48MHz UART clock, limit to 2000000 baud */
+static struct bcm_device_data bcm43430_device_data = {
+	.max_speed = 2000000,
+};
+
 static const struct acpi_device_id bcm_acpi_match[] = {
 	{ "BCM2E00" },
 	{ "BCM2E01" },
@@ -1414,19 +1441,19 @@ static const struct acpi_device_id bcm_acpi_match[] = {
 	{ "BCM2E71" },
 	{ "BCM2E72" },
 	{ "BCM2E73" },
-	{ "BCM2E74" },
-	{ "BCM2E75" },
+	{ "BCM2E74", (long)&bcm43430_device_data },
+	{ "BCM2E75", (long)&bcm43430_device_data },
 	{ "BCM2E76" },
 	{ "BCM2E77" },
 	{ "BCM2E78" },
 	{ "BCM2E79" },
 	{ "BCM2E7A" },
-	{ "BCM2E7B" },
+	{ "BCM2E7B", (long)&bcm43430_device_data },
 	{ "BCM2E7C" },
 	{ "BCM2E7D" },
 	{ "BCM2E7E" },
 	{ "BCM2E7F" },
-	{ "BCM2E80" },
+	{ "BCM2E80", (long)&bcm43430_device_data },
 	{ "BCM2E81" },
 	{ "BCM2E82" },
 	{ "BCM2E83" },
@@ -1435,7 +1462,7 @@ static const struct acpi_device_id bcm_acpi_match[] = {
 	{ "BCM2E86" },
 	{ "BCM2E87" },
 	{ "BCM2E88" },
-	{ "BCM2E89" },
+	{ "BCM2E89", (long)&bcm43430_device_data },
 	{ "BCM2E8A" },
 	{ "BCM2E8B" },
 	{ "BCM2E8C" },
@@ -1444,29 +1471,30 @@ static const struct acpi_device_id bcm_acpi_match[] = {
 	{ "BCM2E90" },
 	{ "BCM2E92" },
 	{ "BCM2E93" },
-	{ "BCM2E94" },
+	{ "BCM2E94", (long)&bcm43430_device_data },
 	{ "BCM2E95" },
 	{ "BCM2E96" },
 	{ "BCM2E97" },
 	{ "BCM2E98" },
-	{ "BCM2E99" },
+	{ "BCM2E99", (long)&bcm43430_device_data },
 	{ "BCM2E9A" },
-	{ "BCM2E9B" },
+	{ "BCM2E9B", (long)&bcm43430_device_data },
 	{ "BCM2E9C" },
 	{ "BCM2E9D" },
+	{ "BCM2E9F", (long)&bcm43430_device_data },
 	{ "BCM2EA0" },
 	{ "BCM2EA1" },
-	{ "BCM2EA2" },
-	{ "BCM2EA3" },
+	{ "BCM2EA2", (long)&bcm43430_device_data },
+	{ "BCM2EA3", (long)&bcm43430_device_data },
 	{ "BCM2EA4" },
 	{ "BCM2EA5" },
 	{ "BCM2EA6" },
 	{ "BCM2EA7" },
 	{ "BCM2EA8" },
 	{ "BCM2EA9" },
-	{ "BCM2EAA" },
-	{ "BCM2EAB" },
-	{ "BCM2EAC" },
+	{ "BCM2EAA", (long)&bcm43430_device_data },
+	{ "BCM2EAB", (long)&bcm43430_device_data },
+	{ "BCM2EAC", (long)&bcm43430_device_data },
 	{ },
 };
 MODULE_DEVICE_TABLE(acpi, bcm_acpi_match);
@@ -1535,6 +1563,8 @@ static int bcm_serdev_probe(struct serdev_device *serdev)
 		bcmdev->no_early_set_baudrate = data->no_early_set_baudrate;
 		bcmdev->drive_rts_on_open = data->drive_rts_on_open;
 		bcmdev->no_uart_clock_set = data->no_uart_clock_set;
+		if (data->max_speed && bcmdev->oper_speed > data->max_speed)
+			bcmdev->oper_speed = data->max_speed;
 	}
 
 	return hci_uart_register_device(&bcmdev->serdev_hu, &bcm_proto);
diff --git a/drivers/bluetooth/hci_h5.c b/drivers/bluetooth/hci_h5.c
index 6455bc4fb5bb..fefc37b98b4a 100644
--- a/drivers/bluetooth/hci_h5.c
+++ b/drivers/bluetooth/hci_h5.c
@@ -463,6 +463,8 @@ static int h5_rx_3wire_hdr(struct hci_uart *hu, unsigned char c)
 	if (H5_HDR_RELIABLE(hdr) && H5_HDR_SEQ(hdr) != h5->tx_ack) {
 		bt_dev_err(hu->hdev, "Out-of-order packet arrived (%u != %u)",
 			   H5_HDR_SEQ(hdr), h5->tx_ack);
+		set_bit(H5_TX_ACK_REQ, &h5->flags);
+		hci_uart_tx_wakeup(hu);
 		h5_reset_rx(h5);
 		return 0;
 	}
@@ -936,6 +938,8 @@ static int h5_btrtl_setup(struct h5 *h5)
 	err = btrtl_download_firmware(h5->hu->hdev, btrtl_dev);
 	/* Give the device some time before the hci-core sends it a reset */
 	usleep_range(10000, 20000);
+	if (err)
+		goto out_free;
 
 	btrtl_set_quirks(h5->hu->hdev, btrtl_dev);
 
@@ -1100,6 +1104,8 @@ static const struct of_device_id rtl_bluetooth_of_match[] = {
 	  .data = (const void *)&h5_data_rtl8822cs },
 	{ .compatible = "realtek,rtl8723bs-bt",
 	  .data = (const void *)&h5_data_rtl8723bs },
+	{ .compatible = "realtek,rtl8723cs-bt",
+	  .data = (const void *)&h5_data_rtl8723bs },
 	{ .compatible = "realtek,rtl8723ds-bt",
 	  .data = (const void *)&h5_data_rtl8723bs },
 #endif
diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c
index 865112e96ff9..efdda2c3fce8 100644
--- a/drivers/bluetooth/hci_ldisc.c
+++ b/drivers/bluetooth/hci_ldisc.c
@@ -323,9 +323,9 @@ void hci_uart_set_flow_control(struct hci_uart *hu, bool enable)
 		/* Disable hardware flow control */
 		ktermios = tty->termios;
 		ktermios.c_cflag &= ~CRTSCTS;
-		status = tty_set_termios(tty, &ktermios);
+		tty_set_termios(tty, &ktermios);
 		BT_DBG("Disabling hardware flow control: %s",
-		       status ? "failed" : "success");
+		       (tty->termios.c_cflag & CRTSCTS) ? "failed" : "success");
 
 		/* Clear RTS to prevent the device from sending */
 		/* Most UARTs need OUT2 to enable interrupts */
@@ -357,9 +357,9 @@ void hci_uart_set_flow_control(struct hci_uart *hu, bool enable)
 		/* Re-enable hardware flow control */
 		ktermios = tty->termios;
 		ktermios.c_cflag |= CRTSCTS;
-		status = tty_set_termios(tty, &ktermios);
+		tty_set_termios(tty, &ktermios);
 		BT_DBG("Enabling hardware flow control: %s",
-		       status ? "failed" : "success");
+		       !(tty->termios.c_cflag & CRTSCTS) ? "failed" : "success");
 	}
 }
 
diff --git a/drivers/bluetooth/hci_ll.c b/drivers/bluetooth/hci_ll.c
index 5abc01a2acf7..4a0b5c3160c2 100644
--- a/drivers/bluetooth/hci_ll.c
+++ b/drivers/bluetooth/hci_ll.c
@@ -786,7 +786,7 @@ MODULE_DEVICE_TABLE(of, hci_ti_of_match);
 static struct serdev_device_driver hci_ti_drv = {
 	.driver		= {
 		.name	= "hci-ti",
-		.of_match_table = of_match_ptr(hci_ti_of_match),
+		.of_match_table = hci_ti_of_match,
 	},
 	.probe	= hci_ti_probe,
 	.remove	= hci_ti_remove,
diff --git a/drivers/bluetooth/hci_mrvl.c b/drivers/bluetooth/hci_mrvl.c
index fbc3f7c3a5c7..e08222395772 100644
--- a/drivers/bluetooth/hci_mrvl.c
+++ b/drivers/bluetooth/hci_mrvl.c
@@ -27,10 +27,12 @@
 #define MRVL_ACK 0x5A
 #define MRVL_NAK 0xBF
 #define MRVL_RAW_DATA 0x1F
+#define MRVL_SET_BAUDRATE 0xFC09
 
 enum {
 	STATE_CHIP_VER_PENDING,
 	STATE_FW_REQ_PENDING,
+	STATE_FW_LOADED,
 };
 
 struct mrvl_data {
@@ -254,6 +256,14 @@ static int mrvl_recv(struct hci_uart *hu, const void *data, int count)
 	if (!test_bit(HCI_UART_REGISTERED, &hu->flags))
 		return -EUNATCH;
 
+	/* We might receive some noise when there is no firmware loaded. Therefore,
+	 * we drop data if the firmware is not loaded yet and if there is no fw load
+	 * request pending.
+	 */
+	if (!test_bit(STATE_FW_REQ_PENDING, &mrvl->flags) &&
+				!test_bit(STATE_FW_LOADED, &mrvl->flags))
+		return count;
+
 	mrvl->rx_skb = h4_recv_buf(hu->hdev, mrvl->rx_skb, data, count,
 				    mrvl_recv_pkts,
 				    ARRAY_SIZE(mrvl_recv_pkts));
@@ -354,6 +364,7 @@ static int mrvl_load_firmware(struct hci_dev *hdev, const char *name)
 static int mrvl_setup(struct hci_uart *hu)
 {
 	int err;
+	struct mrvl_data *mrvl = hu->priv;
 
 	hci_uart_set_flow_control(hu, true);
 
@@ -367,9 +378,9 @@ static int mrvl_setup(struct hci_uart *hu)
 	hci_uart_wait_until_sent(hu);
 
 	if (hu->serdev)
-		serdev_device_set_baudrate(hu->serdev, 3000000);
+		serdev_device_set_baudrate(hu->serdev, hu->oper_speed);
 	else
-		hci_uart_set_baudrate(hu, 3000000);
+		hci_uart_set_baudrate(hu, hu->oper_speed);
 
 	hci_uart_set_flow_control(hu, false);
 
@@ -377,13 +388,54 @@ static int mrvl_setup(struct hci_uart *hu)
 	if (err)
 		return err;
 
+	set_bit(STATE_FW_LOADED, &mrvl->flags);
+
+	return 0;
+}
+
+static int mrvl_set_baudrate(struct hci_uart *hu, unsigned int speed)
+{
+	int err;
+	struct mrvl_data *mrvl = hu->priv;
+	__le32 speed_le = cpu_to_le32(speed);
+
+	/* The firmware might be loaded by the Wifi driver over SDIO. We wait
+	 * up to 10s for the CTS to go up. Afterward, we know that the firmware
+	 * is ready.
+	 */
+	err = serdev_device_wait_for_cts(hu->serdev, true, 10000);
+	if (err) {
+		bt_dev_err(hu->hdev, "Wait for CTS failed with %d\n", err);
+		return err;
+	}
+
+	set_bit(STATE_FW_LOADED, &mrvl->flags);
+
+	err = __hci_cmd_sync_status(hu->hdev, MRVL_SET_BAUDRATE,
+				    sizeof(speed_le), &speed_le,
+				    HCI_INIT_TIMEOUT);
+	if (err) {
+		bt_dev_err(hu->hdev, "send command failed: %d", err);
+		return err;
+	}
+
+	serdev_device_set_baudrate(hu->serdev, speed);
+
+	/* We forcefully have to send a command to the bluetooth module so that
+	 * the driver detects it after a baudrate change. This is foreseen by
+	 * hci_serdev by setting HCI_UART_VND_DETECT which then causes a dummy
+	 * local version read.
+	 */
+	set_bit(HCI_UART_VND_DETECT, &hu->hdev_flags);
+
 	return 0;
 }
 
-static const struct hci_uart_proto mrvl_proto = {
+static const struct hci_uart_proto mrvl_proto_8897 = {
 	.id		= HCI_UART_MRVL,
 	.name		= "Marvell",
 	.init_speed	= 115200,
+	.oper_speed	= 3000000,
 	.open		= mrvl_open,
 	.close		= mrvl_close,
 	.flush		= mrvl_flush,
@@ -393,18 +445,37 @@ static const struct hci_uart_proto mrvl_proto = {
 	.dequeue	= mrvl_dequeue,
 };
 
+static const struct hci_uart_proto mrvl_proto_8997 = {
+	.id		= HCI_UART_MRVL,
+	.name		= "Marvell 8997",
+	.init_speed	= 115200,
+	.oper_speed	= 3000000,
+	.open		= mrvl_open,
+	.close		= mrvl_close,
+	.flush		= mrvl_flush,
+	.set_baudrate   = mrvl_set_baudrate,
+	.recv		= mrvl_recv,
+	.enqueue	= mrvl_enqueue,
+	.dequeue	= mrvl_dequeue,
+};
+
 static int mrvl_serdev_probe(struct serdev_device *serdev)
 {
 	struct mrvl_serdev *mrvldev;
+	const struct hci_uart_proto *mrvl_proto = device_get_match_data(&serdev->dev);
 
 	mrvldev = devm_kzalloc(&serdev->dev, sizeof(*mrvldev), GFP_KERNEL);
 	if (!mrvldev)
 		return -ENOMEM;
 
+	mrvldev->hu.oper_speed = mrvl_proto->oper_speed;
+	if (mrvl_proto->set_baudrate)
+		of_property_read_u32(serdev->dev.of_node, "max-speed", &mrvldev->hu.oper_speed);
+
 	mrvldev->hu.serdev = serdev;
 	serdev_device_set_drvdata(serdev, mrvldev);
 
-	return hci_uart_register_device(&mrvldev->hu, &mrvl_proto);
+	return hci_uart_register_device(&mrvldev->hu, mrvl_proto);
 }
 
 static void mrvl_serdev_remove(struct serdev_device *serdev)
@@ -414,13 +485,12 @@ static void mrvl_serdev_remove(struct serdev_device *serdev)
 	hci_uart_unregister_device(&mrvldev->hu);
 }
 
-#ifdef CONFIG_OF
-static const struct of_device_id mrvl_bluetooth_of_match[] = {
-	{ .compatible = "mrvl,88w8897" },
+static const struct of_device_id __maybe_unused mrvl_bluetooth_of_match[] = {
+	{ .compatible = "mrvl,88w8897", .data = &mrvl_proto_8897},
+	{ .compatible = "mrvl,88w8997", .data = &mrvl_proto_8997},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, mrvl_bluetooth_of_match);
-#endif
 
 static struct serdev_device_driver mrvl_serdev_driver = {
 	.probe = mrvl_serdev_probe,
@@ -435,12 +505,12 @@ int __init mrvl_init(void)
 {
 	serdev_device_driver_register(&mrvl_serdev_driver);
 
-	return hci_uart_register_proto(&mrvl_proto);
+	return hci_uart_register_proto(&mrvl_proto_8897);
 }
 
 int __exit mrvl_deinit(void)
 {
 	serdev_device_driver_unregister(&mrvl_serdev_driver);
 
-	return hci_uart_unregister_proto(&mrvl_proto);
+	return hci_uart_unregister_proto(&mrvl_proto_8897);
 }
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index 3df8c3606e93..1b064504b388 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -1317,7 +1317,8 @@ static int qca_set_baudrate(struct hci_dev *hdev, uint8_t baudrate)
 
 	/* Give the controller time to process the request */
 	if (qca_is_wcn399x(qca_soc_type(hu)) ||
-	    qca_is_wcn6750(qca_soc_type(hu)))
+	    qca_is_wcn6750(qca_soc_type(hu)) ||
+	    qca_is_wcn6855(qca_soc_type(hu)))
 		usleep_range(1000, 10000);
 	else
 		msleep(300);
@@ -1394,7 +1395,8 @@ static unsigned int qca_get_speed(struct hci_uart *hu,
 static int qca_check_speeds(struct hci_uart *hu)
 {
 	if (qca_is_wcn399x(qca_soc_type(hu)) ||
-	    qca_is_wcn6750(qca_soc_type(hu))) {
+	    qca_is_wcn6750(qca_soc_type(hu)) ||
+	    qca_is_wcn6855(qca_soc_type(hu))) {
 		if (!qca_get_speed(hu, QCA_INIT_SPEED) &&
 		    !qca_get_speed(hu, QCA_OPER_SPEED))
 			return -EINVAL;
@@ -1428,7 +1430,8 @@ static int qca_set_speed(struct hci_uart *hu, enum qca_speed_type speed_type)
 		 * changing the baudrate of chip and host.
 		 */
 		if (qca_is_wcn399x(soc_type) ||
-		    qca_is_wcn6750(soc_type))
+		    qca_is_wcn6750(soc_type) ||
+		    qca_is_wcn6855(soc_type))
 			hci_uart_set_flow_control(hu, true);
 
 		if (soc_type == QCA_WCN3990) {
@@ -1446,7 +1449,8 @@ static int qca_set_speed(struct hci_uart *hu, enum qca_speed_type speed_type)
 
 error:
 		if (qca_is_wcn399x(soc_type) ||
-		    qca_is_wcn6750(soc_type))
+		    qca_is_wcn6750(soc_type) ||
+		    qca_is_wcn6855(soc_type))
 			hci_uart_set_flow_control(hu, false);
 
 		if (soc_type == QCA_WCN3990) {
@@ -1682,7 +1686,8 @@ static int qca_power_on(struct hci_dev *hdev)
 		return 0;
 
 	if (qca_is_wcn399x(soc_type) ||
-	    qca_is_wcn6750(soc_type)) {
+	    qca_is_wcn6750(soc_type) ||
+	    qca_is_wcn6855(soc_type)) {
 		ret = qca_regulator_init(hu);
 	} else {
 		qcadev = serdev_device_get_drvdata(hu->serdev);
@@ -1723,7 +1728,8 @@ static int qca_setup(struct hci_uart *hu)
 
 	bt_dev_info(hdev, "setting up %s",
 		qca_is_wcn399x(soc_type) ? "wcn399x" :
-		(soc_type == QCA_WCN6750) ? "wcn6750" : "ROME/QCA6390");
+		(soc_type == QCA_WCN6750) ? "wcn6750" :
+		(soc_type == QCA_WCN6855) ? "wcn6855" : "ROME/QCA6390");
 
 	qca->memdump_state = QCA_MEMDUMP_IDLE;
 
@@ -1735,7 +1741,8 @@ retry:
 	clear_bit(QCA_SSR_TRIGGERED, &qca->flags);
 
 	if (qca_is_wcn399x(soc_type) ||
-	    qca_is_wcn6750(soc_type)) {
+	    qca_is_wcn6750(soc_type) ||
+	    qca_is_wcn6855(soc_type)) {
 		set_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &hdev->quirks);
 		hci_set_aosp_capable(hdev);
 
@@ -1757,7 +1764,8 @@ retry:
 	}
 
 	if (!(qca_is_wcn399x(soc_type) ||
-	     qca_is_wcn6750(soc_type))) {
+	      qca_is_wcn6750(soc_type) ||
+	      qca_is_wcn6855(soc_type))) {
 		/* Get QCA version information */
 		ret = qca_read_soc_version(hdev, &ver, soc_type);
 		if (ret)
@@ -1827,7 +1835,7 @@ static const struct hci_uart_proto qca_proto = {
 	.dequeue	= qca_dequeue,
 };
 
-static const struct qca_device_data qca_soc_data_wcn3990 = {
+static const struct qca_device_data qca_soc_data_wcn3990 __maybe_unused = {
 	.soc_type = QCA_WCN3990,
 	.vregs = (struct qca_vreg []) {
 		{ "vddio", 15000  },
@@ -1838,7 +1846,7 @@ static const struct qca_device_data qca_soc_data_wcn3990 = {
 	.num_vregs = 4,
 };
 
-static const struct qca_device_data qca_soc_data_wcn3991 = {
+static const struct qca_device_data qca_soc_data_wcn3991 __maybe_unused = {
 	.soc_type = QCA_WCN3991,
 	.vregs = (struct qca_vreg []) {
 		{ "vddio", 15000  },
@@ -1850,7 +1858,7 @@ static const struct qca_device_data qca_soc_data_wcn3991 = {
 	.capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES,
 };
 
-static const struct qca_device_data qca_soc_data_wcn3998 = {
+static const struct qca_device_data qca_soc_data_wcn3998 __maybe_unused = {
 	.soc_type = QCA_WCN3998,
 	.vregs = (struct qca_vreg []) {
 		{ "vddio", 10000  },
@@ -1861,12 +1869,12 @@ static const struct qca_device_data qca_soc_data_wcn3998 = {
 	.num_vregs = 4,
 };
 
-static const struct qca_device_data qca_soc_data_qca6390 = {
+static const struct qca_device_data qca_soc_data_qca6390 __maybe_unused = {
 	.soc_type = QCA_QCA6390,
 	.num_vregs = 0,
 };
 
-static const struct qca_device_data qca_soc_data_wcn6750 = {
+static const struct qca_device_data qca_soc_data_wcn6750 __maybe_unused = {
 	.soc_type = QCA_WCN6750,
 	.vregs = (struct qca_vreg []) {
 		{ "vddio", 5000 },
@@ -1883,6 +1891,20 @@ static const struct qca_device_data qca_soc_data_wcn6750 = {
 	.capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES,
 };
 
+static const struct qca_device_data qca_soc_data_wcn6855 __maybe_unused = {
+	.soc_type = QCA_WCN6855,
+	.vregs = (struct qca_vreg []) {
+		{ "vddio", 5000 },
+		{ "vddbtcxmx", 126000 },
+		{ "vddrfacmn", 12500 },
+		{ "vddrfa0p8", 102000 },
+		{ "vddrfa1p7", 302000 },
+		{ "vddrfa1p2", 257000 },
+	},
+	.num_vregs = 6,
+	.capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES,
+};
+
 static void qca_power_shutdown(struct hci_uart *hu)
 {
 	struct qca_serdev *qcadev;
@@ -1912,7 +1934,7 @@ static void qca_power_shutdown(struct hci_uart *hu)
 		host_set_baudrate(hu, 2400);
 		qca_send_power_pulse(hu, false);
 		qca_regulator_disable(qcadev);
-	} else if (soc_type == QCA_WCN6750) {
+	} else if (soc_type == QCA_WCN6750 || soc_type == QCA_WCN6855) {
 		gpiod_set_value_cansleep(qcadev->bt_en, 0);
 		msleep(100);
 		qca_regulator_disable(qcadev);
@@ -2047,7 +2069,8 @@ static int qca_serdev_probe(struct serdev_device *serdev)
 
 	if (data &&
 	    (qca_is_wcn399x(data->soc_type) ||
-	    qca_is_wcn6750(data->soc_type))) {
+	     qca_is_wcn6750(data->soc_type) ||
+	     qca_is_wcn6855(data->soc_type))) {
 		qcadev->btsoc_type = data->soc_type;
 		qcadev->bt_power = devm_kzalloc(&serdev->dev,
 						sizeof(struct qca_power),
@@ -2067,14 +2090,18 @@ static int qca_serdev_probe(struct serdev_device *serdev)
 
 		qcadev->bt_en = devm_gpiod_get_optional(&serdev->dev, "enable",
 					       GPIOD_OUT_LOW);
-		if (IS_ERR_OR_NULL(qcadev->bt_en) && data->soc_type == QCA_WCN6750) {
+		if (IS_ERR_OR_NULL(qcadev->bt_en) &&
+		    (data->soc_type == QCA_WCN6750 ||
+		     data->soc_type == QCA_WCN6855)) {
 			dev_err(&serdev->dev, "failed to acquire BT_EN gpio\n");
 			power_ctrl_enabled = false;
 		}
 
 		qcadev->sw_ctrl = devm_gpiod_get_optional(&serdev->dev, "swctrl",
 					       GPIOD_IN);
-		if (IS_ERR_OR_NULL(qcadev->sw_ctrl) && data->soc_type == QCA_WCN6750)
+		if (IS_ERR_OR_NULL(qcadev->sw_ctrl) &&
+		    (data->soc_type == QCA_WCN6750 ||
+		     data->soc_type == QCA_WCN6855))
 			dev_warn(&serdev->dev, "failed to acquire SW_CTRL gpio\n");
 
 		qcadev->susclk = devm_clk_get_optional(&serdev->dev, NULL);
@@ -2150,8 +2177,9 @@ static void qca_serdev_remove(struct serdev_device *serdev)
 	struct qca_power *power = qcadev->bt_power;
 
 	if ((qca_is_wcn399x(qcadev->btsoc_type) ||
-	     qca_is_wcn6750(qcadev->btsoc_type)) &&
-	     power->vregs_on)
+	     qca_is_wcn6750(qcadev->btsoc_type) ||
+	     qca_is_wcn6855(qcadev->btsoc_type)) &&
+	    power->vregs_on)
 		qca_power_shutdown(&qcadev->serdev_hu);
 	else if (qcadev->susclk)
 		clk_disable_unprepare(qcadev->susclk);
@@ -2335,6 +2363,7 @@ static const struct of_device_id qca_bluetooth_of_match[] = {
 	{ .compatible = "qcom,wcn3991-bt", .data = &qca_soc_data_wcn3991},
 	{ .compatible = "qcom,wcn3998-bt", .data = &qca_soc_data_wcn3998},
 	{ .compatible = "qcom,wcn6750-bt", .data = &qca_soc_data_wcn6750},
+	{ .compatible = "qcom,wcn6855-bt", .data = &qca_soc_data_wcn6855},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, qca_bluetooth_of_match);
diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c
index c443c3b0a4da..40e2b9fa11a2 100644
--- a/drivers/bluetooth/hci_vhci.c
+++ b/drivers/bluetooth/hci_vhci.c
@@ -278,6 +278,104 @@ static int vhci_setup(struct hci_dev *hdev)
 	return 0;
 }
 
+static void vhci_coredump(struct hci_dev *hdev)
+{
+	/* No need to do anything */
+}
+
+static void vhci_coredump_hdr(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	char buf[80];
+
+	snprintf(buf, sizeof(buf), "Controller Name: vhci_ctrl\n");
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Firmware Version: vhci_fw\n");
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Driver: vhci_drv\n");
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Vendor: vhci\n");
+	skb_put_data(skb, buf, strlen(buf));
+}
+
+#define MAX_COREDUMP_LINE_LEN	40
+
+struct devcoredump_test_data {
+	enum devcoredump_state state;
+	unsigned int timeout;
+	char data[MAX_COREDUMP_LINE_LEN];
+};
+
+static inline void force_devcd_timeout(struct hci_dev *hdev,
+				       unsigned int timeout)
+{
+#ifdef CONFIG_DEV_COREDUMP
+	hdev->dump.timeout = msecs_to_jiffies(timeout * 1000);
+#endif
+}
+
+static ssize_t force_devcd_write(struct file *file, const char __user *user_buf,
+				 size_t count, loff_t *ppos)
+{
+	struct vhci_data *data = file->private_data;
+	struct hci_dev *hdev = data->hdev;
+	struct sk_buff *skb = NULL;
+	struct devcoredump_test_data dump_data;
+	size_t data_size;
+	int ret;
+
+	if (count < offsetof(struct devcoredump_test_data, data) ||
+	    count > sizeof(dump_data))
+		return -EINVAL;
+
+	if (copy_from_user(&dump_data, user_buf, count))
+		return -EFAULT;
+
+	data_size = count - offsetof(struct devcoredump_test_data, data);
+	skb = alloc_skb(data_size, GFP_ATOMIC);
+	if (!skb)
+		return -ENOMEM;
+	skb_put_data(skb, &dump_data.data, data_size);
+
+	hci_devcd_register(hdev, vhci_coredump, vhci_coredump_hdr, NULL);
+
+	/* Force the devcoredump timeout */
+	if (dump_data.timeout)
+		force_devcd_timeout(hdev, dump_data.timeout);
+
+	ret = hci_devcd_init(hdev, skb->len);
+	if (ret) {
+		BT_ERR("Failed to generate devcoredump");
+		kfree_skb(skb);
+		return ret;
+	}
+
+	hci_devcd_append(hdev, skb);
+
+	switch (dump_data.state) {
+	case HCI_DEVCOREDUMP_DONE:
+		hci_devcd_complete(hdev);
+		break;
+	case HCI_DEVCOREDUMP_ABORT:
+		hci_devcd_abort(hdev);
+		break;
+	case HCI_DEVCOREDUMP_TIMEOUT:
+		/* Do nothing */
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+static const struct file_operations force_devcoredump_fops = {
+	.open		= simple_open,
+	.write		= force_devcd_write,
+};
+
 static int __vhci_create_device(struct vhci_data *data, __u8 opcode)
 {
 	struct hci_dev *hdev;
@@ -355,6 +453,9 @@ static int __vhci_create_device(struct vhci_data *data, __u8 opcode)
 		debugfs_create_file("aosp_capable", 0644, hdev->debugfs, data,
 				    &aosp_capable_fops);
 
+	debugfs_create_file("force_devcoredump", 0644, hdev->debugfs, data,
+			    &force_devcoredump_fops);
+
 	hci_skb_pkt_type(skb) = HCI_VENDOR_PKT;
 
 	skb_put_u8(skb, 0xff);
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 7bfe998f3514..fcfa280df98a 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -81,7 +81,7 @@ config MOXTET
 config HISILICON_LPC
 	bool "Support for ISA I/O space on HiSilicon Hip06/7"
 	depends on (ARM64 && ARCH_HISI) || (COMPILE_TEST && !ALPHA && !HEXAGON && !PARISC)
-	depends on HAS_IOMEM
+	depends on HAS_IOPORT
 	select INDIRECT_PIO if ARM64
 	help
 	  Driver to enable I/O access to devices attached to the Low Pin
diff --git a/drivers/bus/arm-integrator-lm.c b/drivers/bus/arm-integrator-lm.c
index 2344d560b144..b715c8ab36e8 100644
--- a/drivers/bus/arm-integrator-lm.c
+++ b/drivers/bus/arm-integrator-lm.c
@@ -126,4 +126,3 @@ static struct platform_driver integrator_ap_lm_driver = {
 module_platform_driver(integrator_ap_lm_driver);
 MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
 MODULE_DESCRIPTION("Integrator AP Logical Module driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/brcmstb_gisb.c b/drivers/bus/brcmstb_gisb.c
index b0c3704777e9..b6dfe4340da2 100644
--- a/drivers/bus/brcmstb_gisb.c
+++ b/drivers/bus/brcmstb_gisb.c
@@ -401,12 +401,10 @@ static int __init brcmstb_gisb_arb_probe(struct platform_device *pdev)
 	struct device_node *dn = pdev->dev.of_node;
 	struct brcmstb_gisb_arb_device *gdev;
 	const struct of_device_id *of_id;
-	struct resource *r;
 	int err, timeout_irq, tea_irq, bp_irq;
 	unsigned int num_masters, j = 0;
 	int i, first, last;
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	timeout_irq = platform_get_irq(pdev, 0);
 	tea_irq = platform_get_irq(pdev, 1);
 	bp_irq = platform_get_irq(pdev, 2);
@@ -418,7 +416,7 @@ static int __init brcmstb_gisb_arb_probe(struct platform_device *pdev)
 	mutex_init(&gdev->lock);
 	INIT_LIST_HEAD(&gdev->next);
 
-	gdev->base = devm_ioremap_resource(&pdev->dev, r);
+	gdev->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
 	if (IS_ERR(gdev->base))
 		return PTR_ERR(gdev->base);
 
diff --git a/drivers/bus/bt1-apb.c b/drivers/bus/bt1-apb.c
index 63b1b4a76671..e97c1d1c7578 100644
--- a/drivers/bus/bt1-apb.c
+++ b/drivers/bus/bt1-apb.c
@@ -416,4 +416,3 @@ module_platform_driver(bt1_apb_driver);
 
 MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
 MODULE_DESCRIPTION("Baikal-T1 APB-bus driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/bt1-axi.c b/drivers/bus/bt1-axi.c
index 70e49a6e5374..4007e7322cf2 100644
--- a/drivers/bus/bt1-axi.c
+++ b/drivers/bus/bt1-axi.c
@@ -309,4 +309,3 @@ module_platform_driver(bt1_axi_driver);
 
 MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
 MODULE_DESCRIPTION("Baikal-T1 AXI-bus driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c
index 36cb091a33b4..653e2d4c116f 100644
--- a/drivers/bus/fsl-mc/fsl-mc-bus.c
+++ b/drivers/bus/fsl-mc/fsl-mc-bus.c
@@ -231,7 +231,7 @@ exit:
 	return 0;
 }
 
-static ssize_t rescan_store(struct bus_type *bus,
+static ssize_t rescan_store(const struct bus_type *bus,
 			    const char *buf, size_t count)
 {
 	unsigned long val;
@@ -284,7 +284,7 @@ exit:
 	return 0;
 }
 
-static ssize_t autorescan_store(struct bus_type *bus,
+static ssize_t autorescan_store(const struct bus_type *bus,
 				const char *buf, size_t count)
 {
 	bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_set_autorescan);
@@ -292,7 +292,7 @@ static ssize_t autorescan_store(struct bus_type *bus,
 	return count;
 }
 
-static ssize_t autorescan_show(struct bus_type *bus, char *buf)
+static ssize_t autorescan_show(const struct bus_type *bus, char *buf)
 {
 	bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_get_autorescan);
 	return strlen(buf);
diff --git a/drivers/bus/imx-weim.c b/drivers/bus/imx-weim.c
index cf463c1d2102..52a5d0447390 100644
--- a/drivers/bus/imx-weim.c
+++ b/drivers/bus/imx-weim.c
@@ -10,6 +10,7 @@
 #include <linux/module.h>
 #include <linux/clk.h>
 #include <linux/io.h>
+#include <linux/of_address.h>
 #include <linux/of_device.h>
 #include <linux/mfd/syscon.h>
 #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
@@ -86,8 +87,8 @@ MODULE_DEVICE_TABLE(of, weim_id_table);
 static int imx_weim_gpr_setup(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
-	struct property *prop;
-	const __be32 *p;
+	struct of_range_parser parser;
+	struct of_range range;
 	struct regmap *gpr;
 	u32 gprvals[4] = {
 		05,	/* CS0(128M) CS1(0M)  CS2(0M)  CS3(0M)  */
@@ -106,13 +107,13 @@ static int imx_weim_gpr_setup(struct platform_device *pdev)
 		return 0;
 	}
 
-	of_property_for_each_u32(np, "ranges", prop, p, val) {
-		if (i % 4 == 0) {
-			cs = val;
-		} else if (i % 4 == 3 && val) {
-			val = (val / SZ_32M) | 1;
-			gprval |= val << cs * 3;
-		}
+	if (of_range_parser_init(&parser, np))
+		goto err;
+
+	for_each_of_range(&parser, &range) {
+		cs = range.bus_addr >> 32;
+		val = (range.size / SZ_32M) | 1;
+		gprval |= val << cs * 3;
 		i++;
 	}
 
diff --git a/drivers/bus/intel-ixp4xx-eb.c b/drivers/bus/intel-ixp4xx-eb.c
index 91db001eb69a..f5ba6bee6fd8 100644
--- a/drivers/bus/intel-ixp4xx-eb.c
+++ b/drivers/bus/intel-ixp4xx-eb.c
@@ -423,4 +423,3 @@ static struct platform_driver ixp4xx_exp_driver = {
 module_platform_driver(ixp4xx_exp_driver);
 MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
 MODULE_DESCRIPTION("Intel IXP4xx external bus driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c
index a6a48e515478..600881808982 100644
--- a/drivers/bus/mhi/ep/main.c
+++ b/drivers/bus/mhi/ep/main.c
@@ -126,7 +126,7 @@ static int mhi_ep_process_cmd_ring(struct mhi_ep_ring *ring, struct mhi_ring_ele
 
 	/* Check if the channel is supported by the controller */
 	if ((ch_id >= mhi_cntrl->max_chan) || !mhi_cntrl->mhi_chan[ch_id].name) {
-		dev_err(dev, "Channel (%u) not supported!\n", ch_id);
+		dev_dbg(dev, "Channel (%u) not supported!\n", ch_id);
 		return -ENODEV;
 	}
 
@@ -702,7 +702,7 @@ static void mhi_ep_cmd_ring_worker(struct work_struct *work)
 		el = &ring->ring_cache[ring->rd_offset];
 
 		ret = mhi_ep_process_cmd_ring(ring, el);
-		if (ret)
+		if (ret && ret != -ENODEV)
 			dev_err(dev, "Error processing cmd ring element: %zu\n", ring->rd_offset);
 
 		mhi_ep_ring_inc_index(ring);
diff --git a/drivers/bus/mhi/host/boot.c b/drivers/bus/mhi/host/boot.c
index 1c69feee1703..d2a19b07ccb8 100644
--- a/drivers/bus/mhi/host/boot.c
+++ b/drivers/bus/mhi/host/boot.c
@@ -391,6 +391,7 @@ void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl)
 {
 	const struct firmware *firmware = NULL;
 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
+	enum mhi_pm_state new_state;
 	const char *fw_name;
 	void *buf;
 	dma_addr_t dma_addr;
@@ -508,14 +509,18 @@ error_ready_state:
 	}
 
 error_fw_load:
-	mhi_cntrl->pm_state = MHI_PM_FW_DL_ERR;
-	wake_up_all(&mhi_cntrl->state_event);
+	write_lock_irq(&mhi_cntrl->pm_lock);
+	new_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_FW_DL_ERR);
+	write_unlock_irq(&mhi_cntrl->pm_lock);
+	if (new_state == MHI_PM_FW_DL_ERR)
+		wake_up_all(&mhi_cntrl->state_event);
 }
 
 int mhi_download_amss_image(struct mhi_controller *mhi_cntrl)
 {
 	struct image_info *image_info = mhi_cntrl->fbc_image;
 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
+	enum mhi_pm_state new_state;
 	int ret;
 
 	if (!image_info)
@@ -526,8 +531,11 @@ int mhi_download_amss_image(struct mhi_controller *mhi_cntrl)
 			       &image_info->mhi_buf[image_info->entries - 1]);
 	if (ret) {
 		dev_err(dev, "MHI did not load AMSS, ret:%d\n", ret);
-		mhi_cntrl->pm_state = MHI_PM_FW_DL_ERR;
-		wake_up_all(&mhi_cntrl->state_event);
+		write_lock_irq(&mhi_cntrl->pm_lock);
+		new_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_FW_DL_ERR);
+		write_unlock_irq(&mhi_cntrl->pm_lock);
+		if (new_state == MHI_PM_FW_DL_ERR)
+			wake_up_all(&mhi_cntrl->state_event);
 	}
 
 	return ret;
diff --git a/drivers/bus/mhi/host/init.c b/drivers/bus/mhi/host/init.c
index 3d779ee6396d..f72fcb66f408 100644
--- a/drivers/bus/mhi/host/init.c
+++ b/drivers/bus/mhi/host/init.c
@@ -516,6 +516,12 @@ int mhi_init_mmio(struct mhi_controller *mhi_cntrl)
 		return -EIO;
 	}
 
+	if (val >= mhi_cntrl->reg_len - (8 * MHI_DEV_WAKE_DB)) {
+		dev_err(dev, "CHDB offset: 0x%x is out of range: 0x%zx\n",
+			val, mhi_cntrl->reg_len - (8 * MHI_DEV_WAKE_DB));
+		return -ERANGE;
+	}
+
 	/* Setup wake db */
 	mhi_cntrl->wake_db = base + val + (8 * MHI_DEV_WAKE_DB);
 	mhi_cntrl->wake_set = false;
@@ -532,6 +538,12 @@ int mhi_init_mmio(struct mhi_controller *mhi_cntrl)
 		return -EIO;
 	}
 
+	if (val >= mhi_cntrl->reg_len - (8 * mhi_cntrl->total_ev_rings)) {
+		dev_err(dev, "ERDB offset: 0x%x is out of range: 0x%zx\n",
+			val, mhi_cntrl->reg_len - (8 * mhi_cntrl->total_ev_rings));
+		return -ERANGE;
+	}
+
 	/* Setup event db address for each ev_ring */
 	mhi_event = mhi_cntrl->mhi_event;
 	for (i = 0; i < mhi_cntrl->total_ev_rings; i++, val += 8, mhi_event++) {
@@ -1100,7 +1112,7 @@ int mhi_prepare_for_power_up(struct mhi_controller *mhi_cntrl)
 	if (bhi_off >= mhi_cntrl->reg_len) {
 		dev_err(dev, "BHI offset: 0x%x is out of range: 0x%zx\n",
 			bhi_off, mhi_cntrl->reg_len);
-		ret = -EINVAL;
+		ret = -ERANGE;
 		goto error_reg_offset;
 	}
 	mhi_cntrl->bhi = mhi_cntrl->regs + bhi_off;
@@ -1117,7 +1129,7 @@ int mhi_prepare_for_power_up(struct mhi_controller *mhi_cntrl)
 			dev_err(dev,
 				"BHIe offset: 0x%x is out of range: 0x%zx\n",
 				bhie_off, mhi_cntrl->reg_len);
-			ret = -EINVAL;
+			ret = -ERANGE;
 			goto error_reg_offset;
 		}
 		mhi_cntrl->bhie = mhi_cntrl->regs + bhie_off;
diff --git a/drivers/bus/mhi/host/main.c b/drivers/bus/mhi/host/main.c
index df0fbfee7b78..74a75439c713 100644
--- a/drivers/bus/mhi/host/main.c
+++ b/drivers/bus/mhi/host/main.c
@@ -503,7 +503,7 @@ irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *priv)
 	}
 	write_unlock_irq(&mhi_cntrl->pm_lock);
 
-	if (pm_state != MHI_PM_SYS_ERR_DETECT || ee == mhi_cntrl->ee)
+	if (pm_state != MHI_PM_SYS_ERR_DETECT)
 		goto exit_intvec;
 
 	switch (ee) {
@@ -961,7 +961,9 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
 	}
 
 	read_lock_bh(&mhi_cntrl->pm_lock);
-	if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)))
+
+	/* Ring EV DB only if there is any pending element to process */
+	if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)) && count)
 		mhi_ring_er_db(mhi_event);
 	read_unlock_bh(&mhi_cntrl->pm_lock);
 
@@ -1031,7 +1033,9 @@ int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl,
 		count++;
 	}
 	read_lock_bh(&mhi_cntrl->pm_lock);
-	if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)))
+
+	/* Ring EV DB only if there is any pending element to process */
+	if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)) && count)
 		mhi_ring_er_db(mhi_event);
 	read_unlock_bh(&mhi_cntrl->pm_lock);
 
@@ -1679,18 +1683,3 @@ void mhi_unprepare_from_transfer(struct mhi_device *mhi_dev)
 	}
 }
 EXPORT_SYMBOL_GPL(mhi_unprepare_from_transfer);
-
-int mhi_poll(struct mhi_device *mhi_dev, u32 budget)
-{
-	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
-	struct mhi_chan *mhi_chan = mhi_dev->dl_chan;
-	struct mhi_event *mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index];
-	int ret;
-
-	spin_lock_bh(&mhi_event->lock);
-	ret = mhi_event->process_event(mhi_cntrl, mhi_event, budget);
-	spin_unlock_bh(&mhi_event->lock);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(mhi_poll);
diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c
index f39657f71483..db0a0b062d8e 100644
--- a/drivers/bus/mhi/host/pci_generic.c
+++ b/drivers/bus/mhi/host/pci_generic.c
@@ -8,7 +8,6 @@
  * Copyright (C) 2020 Linaro Ltd <loic.poulain@linaro.org>
  */
 
-#include <linux/aer.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/mhi.h>
@@ -344,8 +343,6 @@ static const struct mhi_channel_config mhi_foxconn_sdx55_channels[] = {
 	MHI_CHANNEL_CONFIG_DL(13, "MBIM", 32, 0),
 	MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
 	MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
-	MHI_CHANNEL_CONFIG_UL(92, "DUN2", 32, 1),
-	MHI_CHANNEL_CONFIG_DL(93, "DUN2", 32, 1),
 	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0_MBIM", 128, 2),
 	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0_MBIM", 128, 3),
 };
@@ -366,6 +363,15 @@ static const struct mhi_controller_config modem_foxconn_sdx55_config = {
 	.event_cfg = mhi_foxconn_sdx55_events,
 };
 
+static const struct mhi_pci_dev_info mhi_foxconn_sdx24_info = {
+	.name = "foxconn-sdx24",
+	.config = &modem_foxconn_sdx55_config,
+	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
+	.dma_data_width = 32,
+	.mru_default = 32768,
+	.sideband_wake = false,
+};
+
 static const struct mhi_pci_dev_info mhi_foxconn_sdx55_info = {
 	.name = "foxconn-sdx55",
 	.fw = "qcom/sdx55m/sbl1.mbn",
@@ -590,6 +596,15 @@ static const struct pci_device_id mhi_pci_id_table[] = {
 	/* T99W373 (sdx62) */
 	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0d9),
 		.driver_data = (kernel_ulong_t) &mhi_foxconn_sdx65_info },
+	/* T99W510 (sdx24), variant 1 */
+	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0f0),
+		.driver_data = (kernel_ulong_t) &mhi_foxconn_sdx24_info },
+	/* T99W510 (sdx24), variant 2 */
+	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0f1),
+		.driver_data = (kernel_ulong_t) &mhi_foxconn_sdx24_info },
+	/* T99W510 (sdx24), variant 3 */
+	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0f2),
+		.driver_data = (kernel_ulong_t) &mhi_foxconn_sdx24_info },
 	/* MV31-W (Cinterion) */
 	{ PCI_DEVICE(PCI_VENDOR_ID_THALES, 0x00b3),
 		.driver_data = (kernel_ulong_t) &mhi_mv31_info },
@@ -903,11 +918,9 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	mhi_pdev->pci_state = pci_store_saved_state(pdev);
 	pci_load_saved_state(pdev, NULL);
 
-	pci_enable_pcie_error_reporting(pdev);
-
 	err = mhi_register_controller(mhi_cntrl, mhi_cntrl_config);
 	if (err)
-		goto err_disable_reporting;
+		return err;
 
 	/* MHI bus does not power up the controller by default */
 	err = mhi_prepare_for_power_up(mhi_cntrl);
@@ -941,8 +954,6 @@ err_unprepare:
 	mhi_unprepare_after_power_down(mhi_cntrl);
 err_unregister:
 	mhi_unregister_controller(mhi_cntrl);
-err_disable_reporting:
-	pci_disable_pcie_error_reporting(pdev);
 
 	return err;
 }
@@ -965,7 +976,6 @@ static void mhi_pci_remove(struct pci_dev *pdev)
 		pm_runtime_get_noresume(&pdev->dev);
 
 	mhi_unregister_controller(mhi_cntrl);
-	pci_disable_pcie_error_reporting(pdev);
 }
 
 static void mhi_pci_shutdown(struct pci_dev *pdev)
diff --git a/drivers/bus/mvebu-mbus.c b/drivers/bus/mvebu-mbus.c
index d51573ac525e..00cb792bda18 100644
--- a/drivers/bus/mvebu-mbus.c
+++ b/drivers/bus/mvebu-mbus.c
@@ -1179,74 +1179,32 @@ static int __init mbus_dt_setup_win(struct mvebu_mbus_state *mbus,
 	return 0;
 }
 
-static int __init
-mbus_parse_ranges(struct device_node *node,
-		  int *addr_cells, int *c_addr_cells, int *c_size_cells,
-		  int *cell_count, const __be32 **ranges_start,
-		  const __be32 **ranges_end)
-{
-	const __be32 *prop;
-	int ranges_len, tuple_len;
-
-	/* Allow a node with no 'ranges' property */
-	*ranges_start = of_get_property(node, "ranges", &ranges_len);
-	if (*ranges_start == NULL) {
-		*addr_cells = *c_addr_cells = *c_size_cells = *cell_count = 0;
-		*ranges_start = *ranges_end = NULL;
-		return 0;
-	}
-	*ranges_end = *ranges_start + ranges_len / sizeof(__be32);
-
-	*addr_cells = of_n_addr_cells(node);
-
-	prop = of_get_property(node, "#address-cells", NULL);
-	*c_addr_cells = be32_to_cpup(prop);
-
-	prop = of_get_property(node, "#size-cells", NULL);
-	*c_size_cells = be32_to_cpup(prop);
-
-	*cell_count = *addr_cells + *c_addr_cells + *c_size_cells;
-	tuple_len = (*cell_count) * sizeof(__be32);
-
-	if (ranges_len % tuple_len) {
-		pr_warn("malformed ranges entry '%pOFn'\n", node);
-		return -EINVAL;
-	}
-	return 0;
-}
-
 static int __init mbus_dt_setup(struct mvebu_mbus_state *mbus,
 				struct device_node *np)
 {
-	int addr_cells, c_addr_cells, c_size_cells;
-	int i, ret, cell_count;
-	const __be32 *r, *ranges_start, *ranges_end;
+	int ret;
+	struct of_range_parser parser;
+	struct of_range range;
 
-	ret = mbus_parse_ranges(np, &addr_cells, &c_addr_cells,
-				&c_size_cells, &cell_count,
-				&ranges_start, &ranges_end);
+	ret = of_range_parser_init(&parser, np);
 	if (ret < 0)
-		return ret;
+		return 0;
 
-	for (i = 0, r = ranges_start; r < ranges_end; r += cell_count, i++) {
-		u32 windowid, base, size;
+	for_each_of_range(&parser, &range) {
+		u32 windowid = upper_32_bits(range.bus_addr);
 		u8 target, attr;
 
 		/*
 		 * An entry with a non-zero custom field do not
 		 * correspond to a static window, so skip it.
 		 */
-		windowid = of_read_number(r, 1);
 		if (CUSTOM(windowid))
 			continue;
 
 		target = TARGET(windowid);
 		attr = ATTR(windowid);
 
-		base = of_read_number(r + c_addr_cells, addr_cells);
-		size = of_read_number(r + c_addr_cells + addr_cells,
-				      c_size_cells);
-		ret = mbus_dt_setup_win(mbus, base, size, target, attr);
+		ret = mbus_dt_setup_win(mbus, range.cpu_addr, range.size, target, attr);
 		if (ret < 0)
 			return ret;
 	}
diff --git a/drivers/bus/qcom-ebi2.c b/drivers/bus/qcom-ebi2.c
index 663c82749222..c1fef1b4bd89 100644
--- a/drivers/bus/qcom-ebi2.c
+++ b/drivers/bus/qcom-ebi2.c
@@ -403,4 +403,3 @@ static struct platform_driver qcom_ebi2_driver = {
 module_platform_driver(qcom_ebi2_driver);
 MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
 MODULE_DESCRIPTION("Qualcomm EBI2 driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/bus/qcom-ssc-block-bus.c b/drivers/bus/qcom-ssc-block-bus.c
index eedeb29a5ff3..3fef18a43c01 100644
--- a/drivers/bus/qcom-ssc-block-bus.c
+++ b/drivers/bus/qcom-ssc-block-bus.c
@@ -386,4 +386,3 @@ module_platform_driver(qcom_ssc_block_bus_driver);
 
 MODULE_DESCRIPTION("A driver for handling the init sequence needed for accessing the SSC block on (some) qcom SoCs over AHB");
 MODULE_AUTHOR("Michael Srba <Michael.Srba@seznam.cz>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/simple-pm-bus.c b/drivers/bus/simple-pm-bus.c
index 7afe1947e1c0..4da77ca7b75a 100644
--- a/drivers/bus/simple-pm-bus.c
+++ b/drivers/bus/simple-pm-bus.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Simple Power-Managed Bus Driver
  *
@@ -138,4 +139,3 @@ module_platform_driver(simple_pm_bus_driver);
 
 MODULE_DESCRIPTION("Simple Power-Managed Bus Driver");
 MODULE_AUTHOR("Geert Uytterhoeven <geert+renesas@glider.be>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/tegra-gmi.c b/drivers/bus/tegra-gmi.c
index 662266719682..e3506ef37051 100644
--- a/drivers/bus/tegra-gmi.c
+++ b/drivers/bus/tegra-gmi.c
@@ -9,7 +9,9 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/module.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/reset.h>
 
diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c
index 6afae9897843..6c49de37d5e9 100644
--- a/drivers/bus/ti-sysc.c
+++ b/drivers/bus/ti-sysc.c
@@ -648,43 +648,20 @@ static int sysc_init_resets(struct sysc *ddata)
 static int sysc_parse_and_check_child_range(struct sysc *ddata)
 {
 	struct device_node *np = ddata->dev->of_node;
-	const __be32 *ranges;
-	u32 nr_addr, nr_size;
-	int len, error;
-
-	ranges = of_get_property(np, "ranges", &len);
-	if (!ranges) {
-		dev_err(ddata->dev, "missing ranges for %pOF\n", np);
-
-		return -ENOENT;
-	}
-
-	len /= sizeof(*ranges);
-
-	if (len < 3) {
-		dev_err(ddata->dev, "incomplete ranges for %pOF\n", np);
-
-		return -EINVAL;
-	}
-
-	error = of_property_read_u32(np, "#address-cells", &nr_addr);
-	if (error)
-		return -ENOENT;
+	struct of_range_parser parser;
+	struct of_range range;
+	int error;
 
-	error = of_property_read_u32(np, "#size-cells", &nr_size);
+	error = of_range_parser_init(&parser, np);
 	if (error)
-		return -ENOENT;
-
-	if (nr_addr != 1 || nr_size != 1) {
-		dev_err(ddata->dev, "invalid ranges for %pOF\n", np);
+		return error;
 
-		return -EINVAL;
+	for_each_of_range(&parser, &range) {
+		ddata->module_pa = range.cpu_addr;
+		ddata->module_size = range.size;
+		break;
 	}
 
-	ranges++;
-	ddata->module_pa = of_translate_address(np, ranges++);
-	ddata->module_size = be32_to_cpup(ranges);
-
 	return 0;
 }
 
@@ -913,7 +890,7 @@ static int sysc_check_registers(struct sysc *ddata)
  * within the interconnect target module range. For example, SGX has
  * them at offset 0x1fc00 in the 32MB module address space. And cpsw
  * has them at offset 0x1200 in the CPSW_WR child. Usually the
- * the interconnect target module registers are at the beginning of
+ * interconnect target module registers are at the beginning of
  * the module range though.
  */
 static int sysc_ioremap(struct sysc *ddata)
@@ -964,7 +941,7 @@ static int sysc_map_and_check_registers(struct sysc *ddata)
 
 	sysc_check_children(ddata);
 
-	if (!of_get_property(np, "reg", NULL))
+	if (!of_property_present(np, "reg"))
 		return 0;
 
 	error = sysc_parse_registers(ddata);
@@ -2530,11 +2507,9 @@ static struct dev_pm_domain sysc_child_pm_domain = {
 static void sysc_reinit_modules(struct sysc_soc_info *soc)
 {
 	struct sysc_module *module;
-	struct list_head *pos;
 	struct sysc *ddata;
 
-	list_for_each(pos, &sysc_soc->restored_modules) {
-		module = list_entry(pos, struct sysc_module, node);
+	list_for_each_entry(module, &sysc_soc->restored_modules, node) {
 		ddata = module->ddata;
 		sysc_reinit_module(ddata, ddata->enabled);
 	}
@@ -3214,12 +3189,10 @@ static void sysc_cleanup_static_data(void)
 static int sysc_check_disabled_devices(struct sysc *ddata)
 {
 	struct sysc_address *disabled_module;
-	struct list_head *pos;
 	int error = 0;
 
 	mutex_lock(&sysc_soc->list_lock);
-	list_for_each(pos, &sysc_soc->disabled_modules) {
-		disabled_module = list_entry(pos, struct sysc_address, node);
+	list_for_each_entry(disabled_module, &sysc_soc->disabled_modules, node) {
 		if (ddata->module_pa == disabled_module->base) {
 			dev_dbg(ddata->dev, "module disabled for this SoC\n");
 			error = -ENODEV;
diff --git a/drivers/bus/uniphier-system-bus.c b/drivers/bus/uniphier-system-bus.c
index f70dedace20b..cb5c89ce7b86 100644
--- a/drivers/bus/uniphier-system-bus.c
+++ b/drivers/bus/uniphier-system-bus.c
@@ -176,10 +176,9 @@ static int uniphier_system_bus_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct uniphier_system_bus_priv *priv;
-	const __be32 *ranges;
-	u32 cells, addr, size;
-	u64 paddr;
-	int pna, bank, rlen, rone, ret;
+	struct of_range_parser parser;
+	struct of_range range;
+	int ret;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -191,48 +190,17 @@ static int uniphier_system_bus_probe(struct platform_device *pdev)
 
 	priv->dev = dev;
 
-	pna = of_n_addr_cells(dev->of_node);
-
-	ret = of_property_read_u32(dev->of_node, "#address-cells", &cells);
-	if (ret) {
-		dev_err(dev, "failed to get #address-cells\n");
-		return ret;
-	}
-	if (cells != 2) {
-		dev_err(dev, "#address-cells must be 2\n");
-		return -EINVAL;
-	}
-
-	ret = of_property_read_u32(dev->of_node, "#size-cells", &cells);
-	if (ret) {
-		dev_err(dev, "failed to get #size-cells\n");
+	ret = of_range_parser_init(&parser, dev->of_node);
+	if (ret)
 		return ret;
-	}
-	if (cells != 1) {
-		dev_err(dev, "#size-cells must be 1\n");
-		return -EINVAL;
-	}
 
-	ranges = of_get_property(dev->of_node, "ranges", &rlen);
-	if (!ranges) {
-		dev_err(dev, "failed to get ranges property\n");
-		return -ENOENT;
-	}
-
-	rlen /= sizeof(*ranges);
-	rone = pna + 2;
-
-	for (; rlen >= rone; rlen -= rone) {
-		bank = be32_to_cpup(ranges++);
-		addr = be32_to_cpup(ranges++);
-		paddr = of_translate_address(dev->of_node, ranges);
-		if (paddr == OF_BAD_ADDR)
+	for_each_of_range(&parser, &range) {
+		if (range.cpu_addr == OF_BAD_ADDR)
 			return -EINVAL;
-		ranges += pna;
-		size = be32_to_cpup(ranges++);
-
-		ret = uniphier_system_bus_add_bank(priv, bank, addr,
-						   paddr, size);
+		ret = uniphier_system_bus_add_bank(priv,
+						   upper_32_bits(range.bus_addr),
+						   lower_32_bits(range.bus_addr),
+						   range.cpu_addr, range.size);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/bus/vexpress-config.c b/drivers/bus/vexpress-config.c
index a58ac0c8e282..472a570bd53a 100644
--- a/drivers/bus/vexpress-config.c
+++ b/drivers/bus/vexpress-config.c
@@ -10,7 +10,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
-#include <linux/of_device.h>
+#include <linux/of_platform.h>
 #include <linux/sched/signal.h>
 #include <linux/slab.h>
 #include <linux/vexpress.h>
diff --git a/drivers/cdx/Kconfig b/drivers/cdx/Kconfig
new file mode 100644
index 000000000000..a08958485e31
--- /dev/null
+++ b/drivers/cdx/Kconfig
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# CDX bus configuration
+#
+# Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+#
+
+config CDX_BUS
+	bool "CDX Bus driver"
+	depends on OF && ARM64
+	help
+	  Driver to enable Composable DMA Transfer(CDX) Bus. CDX bus
+	  exposes Fabric devices which uses composable DMA IP to the
+	  APU. CDX bus provides a mechanism for scanning and probing
+	  of CDX devices. CDX devices are memory mapped on system bus
+	  for embedded CPUs. CDX bus uses CDX controller and firmware
+	  to scan these CDX devices.
+
+source "drivers/cdx/controller/Kconfig"
diff --git a/drivers/cdx/Makefile b/drivers/cdx/Makefile
new file mode 100644
index 000000000000..0324e4914f6e
--- /dev/null
+++ b/drivers/cdx/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for CDX
+#
+# Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+#
+
+obj-$(CONFIG_CDX_BUS) += cdx.o controller/
diff --git a/drivers/cdx/cdx.c b/drivers/cdx/cdx.c
new file mode 100644
index 000000000000..38511fd36325
--- /dev/null
+++ b/drivers/cdx/cdx.c
@@ -0,0 +1,535 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * CDX bus driver.
+ *
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+/*
+ * Architecture Overview
+ * =====================
+ * CDX is a Hardware Architecture designed for AMD FPGA devices. It
+ * consists of sophisticated mechanism for interaction between FPGA,
+ * Firmware and the APUs (Application CPUs).
+ *
+ * Firmware resides on RPU (Realtime CPUs) which interacts with
+ * the FPGA program manager and the APUs. The RPU provides memory-mapped
+ * interface (RPU if) which is used to communicate with APUs.
+ *
+ * The diagram below shows an overview of the CDX architecture:
+ *
+ *          +--------------------------------------+
+ *          |    Application CPUs (APU)            |
+ *          |                                      |
+ *          |                    CDX device drivers|
+ *          |     Linux OS                |        |
+ *          |                        CDX bus       |
+ *          |                             |        |
+ *          |                     CDX controller   |
+ *          |                             |        |
+ *          +-----------------------------|--------+
+ *                                        | (discover, config,
+ *                                        |  reset, rescan)
+ *                                        |
+ *          +------------------------| RPU if |----+
+ *          |                             |        |
+ *          |                             V        |
+ *          |          Realtime CPUs (RPU)         |
+ *          |                                      |
+ *          +--------------------------------------+
+ *                                |
+ *          +---------------------|----------------+
+ *          |  FPGA               |                |
+ *          |      +-----------------------+       |
+ *          |      |           |           |       |
+ *          | +-------+    +-------+   +-------+   |
+ *          | | dev 1 |    | dev 2 |   | dev 3 |   |
+ *          | +-------+    +-------+   +-------+   |
+ *          +--------------------------------------+
+ *
+ * The RPU firmware extracts the device information from the loaded FPGA
+ * image and implements a mechanism that allows the APU drivers to
+ * enumerate such devices (device personality and resource details) via
+ * a dedicated communication channel. RPU mediates operations such as
+ * discover, reset and rescan of the FPGA devices for the APU. This is
+ * done using memory mapped interface provided by the RPU to APU.
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/of_device.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+#include <linux/xarray.h>
+#include <linux/cdx/cdx_bus.h>
+#include "cdx.h"
+
+/* Default DMA mask for devices on a CDX bus */
+#define CDX_DEFAULT_DMA_MASK	(~0ULL)
+#define MAX_CDX_CONTROLLERS 16
+
+/* CDX controllers registered with the CDX bus */
+static DEFINE_XARRAY_ALLOC(cdx_controllers);
+
+/**
+ * cdx_dev_reset - Reset a CDX device
+ * @dev: CDX device
+ *
+ * Return: -errno on failure, 0 on success.
+ */
+int cdx_dev_reset(struct device *dev)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	struct cdx_controller *cdx = cdx_dev->cdx;
+	struct cdx_device_config dev_config = {0};
+	struct cdx_driver *cdx_drv;
+	int ret;
+
+	cdx_drv = to_cdx_driver(dev->driver);
+	/* Notify driver that device is being reset */
+	if (cdx_drv && cdx_drv->reset_prepare)
+		cdx_drv->reset_prepare(cdx_dev);
+
+	dev_config.type = CDX_DEV_RESET_CONF;
+	ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num,
+				      cdx_dev->dev_num, &dev_config);
+	if (ret)
+		dev_err(dev, "cdx device reset failed\n");
+
+	/* Notify driver that device reset is complete */
+	if (cdx_drv && cdx_drv->reset_done)
+		cdx_drv->reset_done(cdx_dev);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(cdx_dev_reset);
+
+/**
+ * cdx_unregister_device - Unregister a CDX device
+ * @dev: CDX device
+ * @data: This is always passed as NULL, and is not used in this API,
+ *	  but is required here as the bus_for_each_dev() API expects
+ *	  the passed function (cdx_unregister_device) to have this
+ *	  as an argument.
+ *
+ * Return: 0 on success.
+ */
+static int cdx_unregister_device(struct device *dev,
+				 void *data)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	kfree(cdx_dev->driver_override);
+	cdx_dev->driver_override = NULL;
+	/*
+	 * Do not free cdx_dev here as it would be freed in
+	 * cdx_device_release() called from within put_device().
+	 */
+	device_del(&cdx_dev->dev);
+	put_device(&cdx_dev->dev);
+
+	return 0;
+}
+
+static void cdx_unregister_devices(struct bus_type *bus)
+{
+	/* Reset all the devices attached to cdx bus */
+	bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device);
+}
+
+/**
+ * cdx_match_one_device - Tell if a CDX device structure has a matching
+ *			  CDX device id structure
+ * @id: single CDX device id structure to match
+ * @dev: the CDX device structure to match against
+ *
+ * Return: matching cdx_device_id structure or NULL if there is no match.
+ */
+static inline const struct cdx_device_id *
+cdx_match_one_device(const struct cdx_device_id *id,
+		     const struct cdx_device *dev)
+{
+	/* Use vendor ID and device ID for matching */
+	if ((id->vendor == CDX_ANY_ID || id->vendor == dev->vendor) &&
+	    (id->device == CDX_ANY_ID || id->device == dev->device))
+		return id;
+	return NULL;
+}
+
+/**
+ * cdx_match_id - See if a CDX device matches a given cdx_id table
+ * @ids: array of CDX device ID structures to search in
+ * @dev: the CDX device structure to match against.
+ *
+ * Used by a driver to check whether a CDX device is in its list of
+ * supported devices. Returns the matching cdx_device_id structure or
+ * NULL if there is no match.
+ *
+ * Return: matching cdx_device_id structure or NULL if there is no match.
+ */
+static inline const struct cdx_device_id *
+cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev)
+{
+	if (ids) {
+		while (ids->vendor || ids->device) {
+			if (cdx_match_one_device(ids, dev))
+				return ids;
+			ids++;
+		}
+	}
+	return NULL;
+}
+
+/**
+ * cdx_bus_match - device to driver matching callback
+ * @dev: the cdx device to match against
+ * @drv: the device driver to search for matching cdx device
+ * structures
+ *
+ * Return: true on success, false otherwise.
+ */
+static int cdx_bus_match(struct device *dev, struct device_driver *drv)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	struct cdx_driver *cdx_drv = to_cdx_driver(drv);
+	const struct cdx_device_id *found_id = NULL;
+	const struct cdx_device_id *ids;
+
+	ids = cdx_drv->match_id_table;
+
+	/* When driver_override is set, only bind to the matching driver */
+	if (cdx_dev->driver_override && strcmp(cdx_dev->driver_override, drv->name))
+		return false;
+
+	found_id = cdx_match_id(ids, cdx_dev);
+	if (!found_id)
+		return false;
+
+	do {
+		/*
+		 * In case override_only was set, enforce driver_override
+		 * matching.
+		 */
+		if (!found_id->override_only)
+			return true;
+		if (cdx_dev->driver_override)
+			return true;
+
+		ids = found_id + 1;
+		found_id = cdx_match_id(ids, cdx_dev);
+	} while (found_id);
+
+	return false;
+}
+
+static int cdx_probe(struct device *dev)
+{
+	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	int error;
+
+	error = cdx_drv->probe(cdx_dev);
+	if (error) {
+		dev_err_probe(dev, error, "%s failed\n", __func__);
+		return error;
+	}
+
+	return 0;
+}
+
+static void cdx_remove(struct device *dev)
+{
+	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	if (cdx_drv && cdx_drv->remove)
+		cdx_drv->remove(cdx_dev);
+}
+
+static void cdx_shutdown(struct device *dev)
+{
+	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	if (cdx_drv && cdx_drv->shutdown)
+		cdx_drv->shutdown(cdx_dev);
+}
+
+static int cdx_dma_configure(struct device *dev)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	u32 input_id = cdx_dev->req_id;
+	int ret;
+
+	ret = of_dma_configure_id(dev, dev->parent->of_node, 0, &input_id);
+	if (ret && ret != -EPROBE_DEFER) {
+		dev_err(dev, "of_dma_configure_id() failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+/* show configuration fields */
+#define cdx_config_attr(field, format_string)	\
+static ssize_t	\
+field##_show(struct device *dev, struct device_attribute *attr, char *buf)	\
+{	\
+	struct cdx_device *cdx_dev = to_cdx_device(dev);	\
+	return sysfs_emit(buf, format_string, cdx_dev->field);	\
+}	\
+static DEVICE_ATTR_RO(field)
+
+cdx_config_attr(vendor, "0x%04x\n");
+cdx_config_attr(device, "0x%04x\n");
+
+static ssize_t remove_store(struct device *dev,
+			    struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	bool val;
+
+	if (kstrtobool(buf, &val) < 0)
+		return -EINVAL;
+
+	if (!val)
+		return -EINVAL;
+
+	if (device_remove_file_self(dev, attr)) {
+		int ret;
+
+		ret = cdx_unregister_device(dev, NULL);
+		if (ret)
+			return ret;
+	}
+
+	return count;
+}
+static DEVICE_ATTR_WO(remove);
+
+static ssize_t reset_store(struct device *dev, struct device_attribute *attr,
+			   const char *buf, size_t count)
+{
+	bool val;
+	int ret;
+
+	if (kstrtobool(buf, &val) < 0)
+		return -EINVAL;
+
+	if (!val)
+		return -EINVAL;
+
+	ret = cdx_dev_reset(dev);
+	if (ret)
+		return ret;
+
+	return count;
+}
+static DEVICE_ATTR_WO(reset);
+
+static ssize_t driver_override_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	int ret;
+
+	if (WARN_ON(dev->bus != &cdx_bus_type))
+		return -EINVAL;
+
+	ret = driver_set_override(dev, &cdx_dev->driver_override, buf, count);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static ssize_t driver_override_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	return sysfs_emit(buf, "%s\n", cdx_dev->driver_override);
+}
+static DEVICE_ATTR_RW(driver_override);
+
+static struct attribute *cdx_dev_attrs[] = {
+	&dev_attr_remove.attr,
+	&dev_attr_reset.attr,
+	&dev_attr_vendor.attr,
+	&dev_attr_device.attr,
+	&dev_attr_driver_override.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(cdx_dev);
+
+static ssize_t rescan_store(const struct bus_type *bus,
+			    const char *buf, size_t count)
+{
+	struct cdx_controller *cdx;
+	unsigned long index;
+	bool val;
+
+	if (kstrtobool(buf, &val) < 0)
+		return -EINVAL;
+
+	if (!val)
+		return -EINVAL;
+
+	/* Unregister all the devices on the bus */
+	cdx_unregister_devices(&cdx_bus_type);
+
+	/* Rescan all the devices */
+	xa_for_each(&cdx_controllers, index, cdx) {
+		int ret;
+
+		ret = cdx->ops->scan(cdx);
+		if (ret)
+			dev_err(cdx->dev, "cdx bus scanning failed\n");
+	}
+
+	return count;
+}
+static BUS_ATTR_WO(rescan);
+
+static struct attribute *cdx_bus_attrs[] = {
+	&bus_attr_rescan.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(cdx_bus);
+
+struct bus_type cdx_bus_type = {
+	.name		= "cdx",
+	.match		= cdx_bus_match,
+	.probe		= cdx_probe,
+	.remove		= cdx_remove,
+	.shutdown	= cdx_shutdown,
+	.dma_configure	= cdx_dma_configure,
+	.bus_groups	= cdx_bus_groups,
+	.dev_groups	= cdx_dev_groups,
+};
+EXPORT_SYMBOL_GPL(cdx_bus_type);
+
+int __cdx_driver_register(struct cdx_driver *cdx_driver,
+			  struct module *owner)
+{
+	int error;
+
+	cdx_driver->driver.owner = owner;
+	cdx_driver->driver.bus = &cdx_bus_type;
+
+	error = driver_register(&cdx_driver->driver);
+	if (error) {
+		pr_err("driver_register() failed for %s: %d\n",
+		       cdx_driver->driver.name, error);
+		return error;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(__cdx_driver_register);
+
+void cdx_driver_unregister(struct cdx_driver *cdx_driver)
+{
+	driver_unregister(&cdx_driver->driver);
+}
+EXPORT_SYMBOL_GPL(cdx_driver_unregister);
+
+static void cdx_device_release(struct device *dev)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	kfree(cdx_dev);
+}
+
+int cdx_device_add(struct cdx_dev_params *dev_params)
+{
+	struct cdx_controller *cdx = dev_params->cdx;
+	struct device *parent = cdx->dev;
+	struct cdx_device *cdx_dev;
+	int ret;
+
+	cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
+	if (!cdx_dev)
+		return -ENOMEM;
+
+	/* Populate resource */
+	memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) *
+		dev_params->res_count);
+	cdx_dev->res_count = dev_params->res_count;
+
+	/* Populate CDX dev params */
+	cdx_dev->req_id = dev_params->req_id;
+	cdx_dev->vendor = dev_params->vendor;
+	cdx_dev->device = dev_params->device;
+	cdx_dev->bus_num = dev_params->bus_num;
+	cdx_dev->dev_num = dev_params->dev_num;
+	cdx_dev->cdx = dev_params->cdx;
+	cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK;
+
+	/* Initialize generic device */
+	device_initialize(&cdx_dev->dev);
+	cdx_dev->dev.parent = parent;
+	cdx_dev->dev.bus = &cdx_bus_type;
+	cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
+	cdx_dev->dev.release = cdx_device_release;
+
+	/* Set Name */
+	dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x",
+		     ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)),
+		     cdx_dev->dev_num);
+
+	ret = device_add(&cdx_dev->dev);
+	if (ret) {
+		dev_err(&cdx_dev->dev,
+			"cdx device add failed: %d", ret);
+		goto fail;
+	}
+
+	return 0;
+fail:
+	/*
+	 * Do not free cdx_dev here as it would be freed in
+	 * cdx_device_release() called from put_device().
+	 */
+	put_device(&cdx_dev->dev);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(cdx_device_add);
+
+int cdx_register_controller(struct cdx_controller *cdx)
+{
+	int ret;
+
+	ret = xa_alloc(&cdx_controllers, &cdx->id, cdx,
+		       XA_LIMIT(0, MAX_CDX_CONTROLLERS - 1), GFP_KERNEL);
+	if (ret) {
+		dev_err(cdx->dev,
+			"No free index available. Maximum controllers already registered\n");
+		cdx->id = (u8)MAX_CDX_CONTROLLERS;
+		return ret;
+	}
+
+	/* Scan all the devices */
+	cdx->ops->scan(cdx);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cdx_register_controller);
+
+void cdx_unregister_controller(struct cdx_controller *cdx)
+{
+	if (cdx->id >= MAX_CDX_CONTROLLERS)
+		return;
+
+	device_for_each_child(cdx->dev, NULL, cdx_unregister_device);
+	xa_erase(&cdx_controllers, cdx->id);
+}
+EXPORT_SYMBOL_GPL(cdx_unregister_controller);
+
+static int __init cdx_bus_init(void)
+{
+	return bus_register(&cdx_bus_type);
+}
+postcore_initcall(cdx_bus_init);
diff --git a/drivers/cdx/cdx.h b/drivers/cdx/cdx.h
new file mode 100644
index 000000000000..c436ac7ac86f
--- /dev/null
+++ b/drivers/cdx/cdx.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Header file for the CDX Bus
+ *
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _CDX_H_
+#define _CDX_H_
+
+#include <linux/cdx/cdx_bus.h>
+
+/**
+ * struct cdx_dev_params - CDX device parameters
+ * @cdx: CDX controller associated with the device
+ * @parent: Associated CDX controller
+ * @vendor: Vendor ID for CDX device
+ * @device: Device ID for CDX device
+ * @bus_num: Bus number for this CDX device
+ * @dev_num: Device number for this device
+ * @res: array of MMIO region entries
+ * @res_count: number of valid MMIO regions
+ * @req_id: Requestor ID associated with CDX device
+ */
+struct cdx_dev_params {
+	struct cdx_controller *cdx;
+	u16 vendor;
+	u16 device;
+	u8 bus_num;
+	u8 dev_num;
+	struct resource res[MAX_CDX_DEV_RESOURCES];
+	u8 res_count;
+	u32 req_id;
+};
+
+/**
+ * cdx_register_controller - Register a CDX controller and its ports
+ *		on the CDX bus.
+ * @cdx: The CDX controller to register
+ *
+ * Return: -errno on failure, 0 on success.
+ */
+int cdx_register_controller(struct cdx_controller *cdx);
+
+/**
+ * cdx_unregister_controller - Unregister a CDX controller
+ * @cdx: The CDX controller to unregister
+ */
+void cdx_unregister_controller(struct cdx_controller *cdx);
+
+/**
+ * cdx_device_add - Add a CDX device. This function adds a CDX device
+ *		on the CDX bus as per the device parameters provided
+ *		by caller. It also creates and registers an associated
+ *		Linux generic device.
+ * @dev_params: device parameters associated with the device to be created.
+ *
+ * Return: -errno on failure, 0 on success.
+ */
+int cdx_device_add(struct cdx_dev_params *dev_params);
+
+#endif /* _CDX_H_ */
diff --git a/drivers/cdx/controller/Kconfig b/drivers/cdx/controller/Kconfig
new file mode 100644
index 000000000000..c3e3b9ff8dfe
--- /dev/null
+++ b/drivers/cdx/controller/Kconfig
@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# CDX controller configuration
+#
+# Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+#
+
+if CDX_BUS
+
+config CDX_CONTROLLER
+	tristate "CDX bus controller"
+	select REMOTEPROC
+	select RPMSG
+	help
+	  CDX controller drives the CDX bus. It interacts with
+	  firmware to get the hardware devices and registers with
+	  the CDX bus. Say Y to enable the CDX hardware driver.
+
+	  If unsure, say N.
+
+config MCDI_LOGGING
+	bool "MCDI Logging for the CDX controller"
+	depends on CDX_CONTROLLER
+	help
+	  Enable MCDI Logging for
+	  the CDX Controller for debug
+	  purpose.
+
+	  If unsure, say N.
+
+endif
diff --git a/drivers/cdx/controller/Makefile b/drivers/cdx/controller/Makefile
new file mode 100644
index 000000000000..f071be411d96
--- /dev/null
+++ b/drivers/cdx/controller/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for CDX controller drivers
+#
+# Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+#
+
+obj-$(CONFIG_CDX_CONTROLLER) += cdx-controller.o
+cdx-controller-objs := cdx_controller.o cdx_rpmsg.o mcdi.o mcdi_functions.o
diff --git a/drivers/cdx/controller/bitfield.h b/drivers/cdx/controller/bitfield.h
new file mode 100644
index 000000000000..567f8ec47582
--- /dev/null
+++ b/drivers/cdx/controller/bitfield.h
@@ -0,0 +1,90 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2005-2006 Fen Systems Ltd.
+ * Copyright 2006-2013 Solarflare Communications Inc.
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#ifndef CDX_BITFIELD_H
+#define CDX_BITFIELD_H
+
+#include <linux/bitfield.h>
+
+/* Lowest bit numbers and widths */
+#define CDX_DWORD_LBN 0
+#define CDX_DWORD_WIDTH 32
+
+/* Specified attribute (e.g. LBN) of the specified field */
+#define CDX_VAL(field, attribute) field ## _ ## attribute
+/* Low bit number of the specified field */
+#define CDX_LOW_BIT(field) CDX_VAL(field, LBN)
+/* Bit width of the specified field */
+#define CDX_WIDTH(field) CDX_VAL(field, WIDTH)
+/* High bit number of the specified field */
+#define CDX_HIGH_BIT(field) (CDX_LOW_BIT(field) + CDX_WIDTH(field) - 1)
+
+/* A doubleword (i.e. 4 byte) datatype - little-endian in HW */
+struct cdx_dword {
+	__le32 cdx_u32;
+};
+
+/* Value expanders for printk */
+#define CDX_DWORD_VAL(dword)				\
+	((unsigned int)le32_to_cpu((dword).cdx_u32))
+
+/*
+ * Extract bit field portion [low,high) from the 32-bit little-endian
+ * element which contains bits [min,max)
+ */
+#define CDX_DWORD_FIELD(dword, field)					\
+	(FIELD_GET(GENMASK(CDX_HIGH_BIT(field), CDX_LOW_BIT(field)),	\
+		   le32_to_cpu((dword).cdx_u32)))
+
+/*
+ * Creates the portion of the named bit field that lies within the
+ * range [min,max).
+ */
+#define CDX_INSERT_FIELD(field, value)				\
+	(FIELD_PREP(GENMASK(CDX_HIGH_BIT(field),		\
+			    CDX_LOW_BIT(field)), value))
+
+/*
+ * Creates the portion of the named bit fields that lie within the
+ * range [min,max).
+ */
+#define CDX_INSERT_FIELDS(field1, value1,		\
+			  field2, value2,		\
+			  field3, value3,		\
+			  field4, value4,		\
+			  field5, value5,		\
+			  field6, value6,		\
+			  field7, value7)		\
+	(CDX_INSERT_FIELD(field1, (value1)) |		\
+	 CDX_INSERT_FIELD(field2, (value2)) |		\
+	 CDX_INSERT_FIELD(field3, (value3)) |		\
+	 CDX_INSERT_FIELD(field4, (value4)) |		\
+	 CDX_INSERT_FIELD(field5, (value5)) |		\
+	 CDX_INSERT_FIELD(field6, (value6)) |		\
+	 CDX_INSERT_FIELD(field7, (value7)))
+
+#define CDX_POPULATE_DWORD(dword, ...)					\
+	(dword).cdx_u32 = cpu_to_le32(CDX_INSERT_FIELDS(__VA_ARGS__))
+
+/* Populate a dword field with various numbers of arguments */
+#define CDX_POPULATE_DWORD_7 CDX_POPULATE_DWORD
+#define CDX_POPULATE_DWORD_6(dword, ...) \
+	CDX_POPULATE_DWORD_7(dword, CDX_DWORD, 0, __VA_ARGS__)
+#define CDX_POPULATE_DWORD_5(dword, ...) \
+	CDX_POPULATE_DWORD_6(dword, CDX_DWORD, 0, __VA_ARGS__)
+#define CDX_POPULATE_DWORD_4(dword, ...) \
+	CDX_POPULATE_DWORD_5(dword, CDX_DWORD, 0, __VA_ARGS__)
+#define CDX_POPULATE_DWORD_3(dword, ...) \
+	CDX_POPULATE_DWORD_4(dword, CDX_DWORD, 0, __VA_ARGS__)
+#define CDX_POPULATE_DWORD_2(dword, ...) \
+	CDX_POPULATE_DWORD_3(dword, CDX_DWORD, 0, __VA_ARGS__)
+#define CDX_POPULATE_DWORD_1(dword, ...) \
+	CDX_POPULATE_DWORD_2(dword, CDX_DWORD, 0, __VA_ARGS__)
+#define CDX_SET_DWORD(dword) \
+	CDX_POPULATE_DWORD_1(dword, CDX_DWORD, 0xffffffff)
+
+#endif /* CDX_BITFIELD_H */
diff --git a/drivers/cdx/controller/cdx_controller.c b/drivers/cdx/controller/cdx_controller.c
new file mode 100644
index 000000000000..dc52f95f8978
--- /dev/null
+++ b/drivers/cdx/controller/cdx_controller.c
@@ -0,0 +1,230 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * CDX host controller driver for AMD versal-net platform.
+ *
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/of_platform.h>
+#include <linux/slab.h>
+#include <linux/cdx/cdx_bus.h>
+
+#include "cdx_controller.h"
+#include "../cdx.h"
+#include "mcdi_functions.h"
+#include "mcdi.h"
+
+static unsigned int cdx_mcdi_rpc_timeout(struct cdx_mcdi *cdx, unsigned int cmd)
+{
+	return MCDI_RPC_TIMEOUT;
+}
+
+static void cdx_mcdi_request(struct cdx_mcdi *cdx,
+			     const struct cdx_dword *hdr, size_t hdr_len,
+			     const struct cdx_dword *sdu, size_t sdu_len)
+{
+	if (cdx_rpmsg_send(cdx, hdr, hdr_len, sdu, sdu_len))
+		dev_err(&cdx->rpdev->dev, "Failed to send rpmsg data\n");
+}
+
+static const struct cdx_mcdi_ops mcdi_ops = {
+	.mcdi_rpc_timeout = cdx_mcdi_rpc_timeout,
+	.mcdi_request = cdx_mcdi_request,
+};
+
+void cdx_rpmsg_post_probe(struct cdx_controller *cdx)
+{
+	/* Register CDX controller with CDX bus driver */
+	if (cdx_register_controller(cdx))
+		dev_err(cdx->dev, "Failed to register CDX controller\n");
+}
+
+void cdx_rpmsg_pre_remove(struct cdx_controller *cdx)
+{
+	cdx_unregister_controller(cdx);
+	cdx_mcdi_wait_for_quiescence(cdx->priv, MCDI_RPC_TIMEOUT);
+}
+
+static int cdx_configure_device(struct cdx_controller *cdx,
+				u8 bus_num, u8 dev_num,
+				struct cdx_device_config *dev_config)
+{
+	int ret = 0;
+
+	switch (dev_config->type) {
+	case CDX_DEV_RESET_CONF:
+		ret = cdx_mcdi_reset_device(cdx->priv, bus_num, dev_num);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int cdx_scan_devices(struct cdx_controller *cdx)
+{
+	struct cdx_mcdi *cdx_mcdi = cdx->priv;
+	u8 bus_num, dev_num, num_cdx_bus;
+	int ret;
+
+	/* MCDI FW Read: Fetch the number of CDX buses on this controller */
+	ret = cdx_mcdi_get_num_buses(cdx_mcdi);
+	if (ret < 0) {
+		dev_err(cdx->dev,
+			"Get number of CDX buses failed: %d\n", ret);
+		return ret;
+	}
+	num_cdx_bus = (u8)ret;
+
+	for (bus_num = 0; bus_num < num_cdx_bus; bus_num++) {
+		u8 num_cdx_dev;
+
+		/* MCDI FW Read: Fetch the number of devices present */
+		ret = cdx_mcdi_get_num_devs(cdx_mcdi, bus_num);
+		if (ret < 0) {
+			dev_err(cdx->dev,
+				"Get devices on CDX bus %d failed: %d\n", bus_num, ret);
+			continue;
+		}
+		num_cdx_dev = (u8)ret;
+
+		for (dev_num = 0; dev_num < num_cdx_dev; dev_num++) {
+			struct cdx_dev_params dev_params;
+
+			/* MCDI FW: Get the device config */
+			ret = cdx_mcdi_get_dev_config(cdx_mcdi, bus_num,
+						      dev_num, &dev_params);
+			if (ret) {
+				dev_err(cdx->dev,
+					"CDX device config get failed for %d(bus):%d(dev), %d\n",
+					bus_num, dev_num, ret);
+				continue;
+			}
+			dev_params.cdx = cdx;
+
+			/* Add the device to the cdx bus */
+			ret = cdx_device_add(&dev_params);
+			if (ret) {
+				dev_err(cdx->dev, "registering cdx dev: %d failed: %d\n",
+					dev_num, ret);
+				continue;
+			}
+
+			dev_dbg(cdx->dev, "CDX dev: %d on cdx bus: %d created\n",
+				dev_num, bus_num);
+		}
+	}
+
+	return 0;
+}
+
+static struct cdx_ops cdx_ops = {
+	.scan		= cdx_scan_devices,
+	.dev_configure	= cdx_configure_device,
+};
+
+static int xlnx_cdx_probe(struct platform_device *pdev)
+{
+	struct cdx_controller *cdx;
+	struct cdx_mcdi *cdx_mcdi;
+	int ret;
+
+	cdx_mcdi = kzalloc(sizeof(*cdx_mcdi), GFP_KERNEL);
+	if (!cdx_mcdi)
+		return -ENOMEM;
+
+	/* Store the MCDI ops */
+	cdx_mcdi->mcdi_ops = &mcdi_ops;
+	/* MCDI FW: Initialize the FW path */
+	ret = cdx_mcdi_init(cdx_mcdi);
+	if (ret) {
+		dev_err_probe(&pdev->dev, ret, "MCDI Initialization failed\n");
+		goto mcdi_init_fail;
+	}
+
+	cdx = kzalloc(sizeof(*cdx), GFP_KERNEL);
+	if (!cdx) {
+		ret = -ENOMEM;
+		goto cdx_alloc_fail;
+	}
+	platform_set_drvdata(pdev, cdx);
+
+	cdx->dev = &pdev->dev;
+	cdx->priv = cdx_mcdi;
+	cdx->ops = &cdx_ops;
+
+	ret = cdx_setup_rpmsg(pdev);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Failed to register CDX RPMsg transport\n");
+		goto cdx_rpmsg_fail;
+	}
+
+	dev_info(&pdev->dev, "Successfully registered CDX controller with RPMsg as transport\n");
+	return 0;
+
+cdx_rpmsg_fail:
+	kfree(cdx);
+cdx_alloc_fail:
+	cdx_mcdi_finish(cdx_mcdi);
+mcdi_init_fail:
+	kfree(cdx_mcdi);
+
+	return ret;
+}
+
+static int xlnx_cdx_remove(struct platform_device *pdev)
+{
+	struct cdx_controller *cdx = platform_get_drvdata(pdev);
+	struct cdx_mcdi *cdx_mcdi = cdx->priv;
+
+	cdx_destroy_rpmsg(pdev);
+
+	kfree(cdx);
+
+	cdx_mcdi_finish(cdx_mcdi);
+	kfree(cdx_mcdi);
+
+	return 0;
+}
+
+static const struct of_device_id cdx_match_table[] = {
+	{.compatible = "xlnx,versal-net-cdx",},
+	{ },
+};
+
+MODULE_DEVICE_TABLE(of, cdx_match_table);
+
+static struct platform_driver cdx_pdriver = {
+	.driver = {
+		   .name = "cdx-controller",
+		   .pm = NULL,
+		   .of_match_table = cdx_match_table,
+		   },
+	.probe = xlnx_cdx_probe,
+	.remove = xlnx_cdx_remove,
+};
+
+static int __init cdx_controller_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&cdx_pdriver);
+	if (ret)
+		pr_err("platform_driver_register() failed: %d\n", ret);
+
+	return ret;
+}
+
+static void __exit cdx_controller_exit(void)
+{
+	platform_driver_unregister(&cdx_pdriver);
+}
+
+module_init(cdx_controller_init);
+module_exit(cdx_controller_exit);
+
+MODULE_AUTHOR("AMD Inc.");
+MODULE_DESCRIPTION("CDX controller for AMD devices");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cdx/controller/cdx_controller.h b/drivers/cdx/controller/cdx_controller.h
new file mode 100644
index 000000000000..43b7c742df87
--- /dev/null
+++ b/drivers/cdx/controller/cdx_controller.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Header file for the CDX Controller
+ *
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _CDX_CONTROLLER_H_
+#define _CDX_CONTROLLER_H_
+
+#include <linux/cdx/cdx_bus.h>
+#include "mcdi_functions.h"
+
+void cdx_rpmsg_post_probe(struct cdx_controller *cdx);
+
+void cdx_rpmsg_pre_remove(struct cdx_controller *cdx);
+
+int cdx_rpmsg_send(struct cdx_mcdi *cdx_mcdi,
+		   const struct cdx_dword *hdr, size_t hdr_len,
+		   const struct cdx_dword *sdu, size_t sdu_len);
+
+void cdx_rpmsg_read_resp(struct cdx_mcdi *cdx_mcdi,
+			 struct cdx_dword *outbuf, size_t offset,
+			 size_t outlen);
+
+int cdx_setup_rpmsg(struct platform_device *pdev);
+
+void cdx_destroy_rpmsg(struct platform_device *pdev);
+
+#endif /* _CDX_CONT_PRIV_H_ */
diff --git a/drivers/cdx/controller/cdx_rpmsg.c b/drivers/cdx/controller/cdx_rpmsg.c
new file mode 100644
index 000000000000..f37e639d6ce3
--- /dev/null
+++ b/drivers/cdx/controller/cdx_rpmsg.c
@@ -0,0 +1,202 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Platform driver for CDX bus.
+ *
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/rpmsg.h>
+#include <linux/remoteproc.h>
+#include <linux/of_platform.h>
+#include <linux/cdx/cdx_bus.h>
+#include <linux/module.h>
+
+#include "../cdx.h"
+#include "cdx_controller.h"
+#include "mcdi_functions.h"
+#include "mcdi.h"
+
+static struct rpmsg_device_id cdx_rpmsg_id_table[] = {
+	{ .name = "mcdi_ipc" },
+	{ },
+};
+MODULE_DEVICE_TABLE(rpmsg, cdx_rpmsg_id_table);
+
+int cdx_rpmsg_send(struct cdx_mcdi *cdx_mcdi,
+		   const struct cdx_dword *hdr, size_t hdr_len,
+		   const struct cdx_dword *sdu, size_t sdu_len)
+{
+	unsigned char *send_buf;
+	int ret;
+
+	send_buf = kzalloc(hdr_len + sdu_len, GFP_KERNEL);
+	if (!send_buf)
+		return -ENOMEM;
+
+	memcpy(send_buf, hdr, hdr_len);
+	memcpy(send_buf + hdr_len, sdu, sdu_len);
+
+	ret = rpmsg_send(cdx_mcdi->ept, send_buf, hdr_len + sdu_len);
+	kfree(send_buf);
+
+	return ret;
+}
+
+static int cdx_attach_to_rproc(struct platform_device *pdev)
+{
+	struct device_node *r5_core_node;
+	struct cdx_controller *cdx_c;
+	struct cdx_mcdi *cdx_mcdi;
+	struct device *dev;
+	struct rproc *rp;
+	int ret;
+
+	dev = &pdev->dev;
+	cdx_c = platform_get_drvdata(pdev);
+	cdx_mcdi = cdx_c->priv;
+
+	r5_core_node = of_parse_phandle(dev->of_node, "xlnx,rproc", 0);
+	if (!r5_core_node) {
+		dev_err(&pdev->dev, "xlnx,rproc: invalid phandle\n");
+		return -EINVAL;
+	}
+
+	rp = rproc_get_by_phandle(r5_core_node->phandle);
+	if (!rp) {
+		ret = -EPROBE_DEFER;
+		goto pdev_err;
+	}
+
+	/* Attach to remote processor */
+	ret = rproc_boot(rp);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to attach to remote processor\n");
+		rproc_put(rp);
+		goto pdev_err;
+	}
+
+	cdx_mcdi->r5_rproc = rp;
+pdev_err:
+	of_node_put(r5_core_node);
+	return ret;
+}
+
+static void cdx_detach_to_r5(struct platform_device *pdev)
+{
+	struct cdx_controller *cdx_c;
+	struct cdx_mcdi *cdx_mcdi;
+
+	cdx_c = platform_get_drvdata(pdev);
+	cdx_mcdi = cdx_c->priv;
+
+	rproc_detach(cdx_mcdi->r5_rproc);
+	rproc_put(cdx_mcdi->r5_rproc);
+}
+
+static int cdx_rpmsg_cb(struct rpmsg_device *rpdev, void *data,
+			int len, void *priv, u32 src)
+{
+	struct cdx_controller *cdx_c = dev_get_drvdata(&rpdev->dev);
+	struct cdx_mcdi *cdx_mcdi = cdx_c->priv;
+
+	if (len > MCDI_BUF_LEN)
+		return -EINVAL;
+
+	cdx_mcdi_process_cmd(cdx_mcdi, (struct cdx_dword *)data, len);
+
+	return 0;
+}
+
+static void cdx_rpmsg_post_probe_work(struct work_struct *work)
+{
+	struct cdx_controller *cdx_c;
+	struct cdx_mcdi *cdx_mcdi;
+
+	cdx_mcdi = container_of(work, struct cdx_mcdi, work);
+	cdx_c = dev_get_drvdata(&cdx_mcdi->rpdev->dev);
+	cdx_rpmsg_post_probe(cdx_c);
+}
+
+static int cdx_rpmsg_probe(struct rpmsg_device *rpdev)
+{
+	struct rpmsg_channel_info chinfo = {0};
+	struct cdx_controller *cdx_c;
+	struct cdx_mcdi *cdx_mcdi;
+
+	cdx_c = (struct cdx_controller *)cdx_rpmsg_id_table[0].driver_data;
+	cdx_mcdi = cdx_c->priv;
+
+	chinfo.src = RPMSG_ADDR_ANY;
+	chinfo.dst = rpdev->dst;
+	strscpy(chinfo.name, cdx_rpmsg_id_table[0].name,
+		strlen(cdx_rpmsg_id_table[0].name));
+
+	cdx_mcdi->ept = rpmsg_create_ept(rpdev, cdx_rpmsg_cb, NULL, chinfo);
+	if (!cdx_mcdi->ept) {
+		dev_err_probe(&rpdev->dev, -ENXIO,
+			      "Failed to create ept for channel %s\n",
+			      chinfo.name);
+		return -EINVAL;
+	}
+
+	cdx_mcdi->rpdev = rpdev;
+	dev_set_drvdata(&rpdev->dev, cdx_c);
+
+	schedule_work(&cdx_mcdi->work);
+	return 0;
+}
+
+static void cdx_rpmsg_remove(struct rpmsg_device *rpdev)
+{
+	struct cdx_controller *cdx_c = dev_get_drvdata(&rpdev->dev);
+	struct cdx_mcdi *cdx_mcdi = cdx_c->priv;
+
+	flush_work(&cdx_mcdi->work);
+	cdx_rpmsg_pre_remove(cdx_c);
+
+	rpmsg_destroy_ept(cdx_mcdi->ept);
+	dev_set_drvdata(&rpdev->dev, NULL);
+}
+
+static struct rpmsg_driver cdx_rpmsg_driver = {
+	.drv.name = KBUILD_MODNAME,
+	.id_table = cdx_rpmsg_id_table,
+	.probe = cdx_rpmsg_probe,
+	.remove = cdx_rpmsg_remove,
+	.callback = cdx_rpmsg_cb,
+};
+
+int cdx_setup_rpmsg(struct platform_device *pdev)
+{
+	struct cdx_controller *cdx_c;
+	struct cdx_mcdi *cdx_mcdi;
+	int ret;
+
+	/* Attach to remote processor */
+	ret = cdx_attach_to_rproc(pdev);
+	if (ret)
+		return ret;
+
+	cdx_c = platform_get_drvdata(pdev);
+	cdx_mcdi = cdx_c->priv;
+
+	/* Register RPMsg driver */
+	cdx_rpmsg_id_table[0].driver_data = (kernel_ulong_t)cdx_c;
+
+	INIT_WORK(&cdx_mcdi->work, cdx_rpmsg_post_probe_work);
+	ret = register_rpmsg_driver(&cdx_rpmsg_driver);
+	if (ret) {
+		dev_err(&pdev->dev,
+			"Failed to register cdx RPMsg driver: %d\n", ret);
+		cdx_detach_to_r5(pdev);
+	}
+
+	return ret;
+}
+
+void cdx_destroy_rpmsg(struct platform_device *pdev)
+{
+	unregister_rpmsg_driver(&cdx_rpmsg_driver);
+
+	cdx_detach_to_r5(pdev);
+}
diff --git a/drivers/cdx/controller/mc_cdx_pcol.h b/drivers/cdx/controller/mc_cdx_pcol.h
new file mode 100644
index 000000000000..4ccb7b52951b
--- /dev/null
+++ b/drivers/cdx/controller/mc_cdx_pcol.h
@@ -0,0 +1,590 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Driver for AMD network controllers and boards
+ *
+ * Copyright (C) 2021, Xilinx, Inc.
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#ifndef MC_CDX_PCOL_H
+#define MC_CDX_PCOL_H
+
+/* The current version of the MCDI protocol. */
+#define MCDI_PCOL_VERSION		2
+
+/*
+ * Each MCDI request starts with an MCDI_HEADER, which is a 32bit
+ * structure, filled in by the client.
+ *
+ *       0       7  8     16    20     22  23  24    31
+ *      | CODE | R | LEN | SEQ | Rsvd | E | R | XFLAGS |
+ *               |                      |   |
+ *               |                      |   \--- Response
+ *               |                      \------- Error
+ *               \------------------------------ Resync (always set)
+ *
+ * The client writes its request into MC shared memory, and rings the
+ * doorbell. Each request is completed either by the MC writing
+ * back into shared memory, or by writing out an event.
+ *
+ * All MCDI commands support completion by shared memory response. Each
+ * request may also contain additional data (accounted for by HEADER.LEN),
+ * and some responses may also contain additional data (again, accounted
+ * for by HEADER.LEN).
+ *
+ * Some MCDI commands support completion by event, in which any associated
+ * response data is included in the event.
+ *
+ * The protocol requires one response to be delivered for every request; a
+ * request should not be sent unless the response for the previous request
+ * has been received (either by polling shared memory, or by receiving
+ * an event).
+ */
+
+/** Request/Response structure */
+#define MCDI_HEADER_OFST		0
+#define MCDI_HEADER_CODE_LBN		0
+#define MCDI_HEADER_CODE_WIDTH		7
+#define MCDI_HEADER_RESYNC_LBN		7
+#define MCDI_HEADER_RESYNC_WIDTH	1
+#define MCDI_HEADER_DATALEN_LBN		8
+#define MCDI_HEADER_DATALEN_WIDTH	8
+#define MCDI_HEADER_SEQ_LBN		16
+#define MCDI_HEADER_SEQ_WIDTH		4
+#define MCDI_HEADER_RSVD_LBN		20
+#define MCDI_HEADER_RSVD_WIDTH		1
+#define MCDI_HEADER_NOT_EPOCH_LBN	21
+#define MCDI_HEADER_NOT_EPOCH_WIDTH	1
+#define MCDI_HEADER_ERROR_LBN		22
+#define MCDI_HEADER_ERROR_WIDTH		1
+#define MCDI_HEADER_RESPONSE_LBN	23
+#define MCDI_HEADER_RESPONSE_WIDTH	1
+#define MCDI_HEADER_XFLAGS_LBN		24
+#define MCDI_HEADER_XFLAGS_WIDTH	8
+/* Request response using event */
+#define MCDI_HEADER_XFLAGS_EVREQ	0x01
+/* Request (and signal) early doorbell return */
+#define MCDI_HEADER_XFLAGS_DBRET	0x02
+
+/* Maximum number of payload bytes */
+#define MCDI_CTL_SDU_LEN_MAX_V2		0x400
+
+#define MCDI_CTL_SDU_LEN_MAX MCDI_CTL_SDU_LEN_MAX_V2
+
+/*
+ * The MC can generate events for two reasons:
+ *   - To advance a shared memory request if XFLAGS_EVREQ was set
+ *   - As a notification (link state, i2c event), controlled
+ *     via MC_CMD_LOG_CTRL
+ *
+ * Both events share a common structure:
+ *
+ *  0      32     33      36    44     52     60
+ * | Data | Cont | Level | Src | Code | Rsvd |
+ *           |
+ *           \ There is another event pending in this notification
+ *
+ * If Code==CMDDONE, then the fields are further interpreted as:
+ *
+ *   - LEVEL==INFO    Command succeeded
+ *   - LEVEL==ERR     Command failed
+ *
+ *    0     8         16      24     32
+ *   | Seq | Datalen | Errno | Rsvd |
+ *
+ *   These fields are taken directly out of the standard MCDI header, i.e.,
+ *   LEVEL==ERR, Datalen == 0 => Reboot
+ *
+ * Events can be squirted out of the UART (using LOG_CTRL) without a
+ * MCDI header.  An event can be distinguished from a MCDI response by
+ * examining the first byte which is 0xc0.  This corresponds to the
+ * non-existent MCDI command MC_CMD_DEBUG_LOG.
+ *
+ *      0         7        8
+ *     | command | Resync |     = 0xc0
+ *
+ * Since the event is written in big-endian byte order, this works
+ * providing bits 56-63 of the event are 0xc0.
+ *
+ *      56     60  63
+ *     | Rsvd | Code |    = 0xc0
+ *
+ * Which means for convenience the event code is 0xc for all MC
+ * generated events.
+ */
+
+/*
+ * the errno value may be followed by the (0-based) number of the
+ * first argument that could not be processed.
+ */
+#define MC_CMD_ERR_ARG_OFST		4
+
+/* MC_CMD_ERR MCDI error codes. */
+/* Operation not permitted. */
+#define MC_CMD_ERR_EPERM		0x1
+/* Non-existent command target */
+#define MC_CMD_ERR_ENOENT		0x2
+/* assert() has killed the MC */
+#define MC_CMD_ERR_EINTR		0x4
+/* I/O failure */
+#define MC_CMD_ERR_EIO			0x5
+/* Already exists */
+#define MC_CMD_ERR_EEXIST		0x6
+/* Try again */
+#define MC_CMD_ERR_EAGAIN		0xb
+/* Out of memory */
+#define MC_CMD_ERR_ENOMEM		0xc
+/* Caller does not hold required locks */
+#define MC_CMD_ERR_EACCES		0xd
+/* Resource is currently unavailable (e.g. lock contention) */
+#define MC_CMD_ERR_EBUSY		0x10
+/* No such device */
+#define MC_CMD_ERR_ENODEV		0x13
+/* Invalid argument to target */
+#define MC_CMD_ERR_EINVAL		0x16
+/* No space */
+#define MC_CMD_ERR_ENOSPC		0x1c
+/* Read-only */
+#define MC_CMD_ERR_EROFS		0x1e
+/* Broken pipe */
+#define MC_CMD_ERR_EPIPE		0x20
+/* Out of range */
+#define MC_CMD_ERR_ERANGE		0x22
+/* Non-recursive resource is already acquired */
+#define MC_CMD_ERR_EDEADLK		0x23
+/* Operation not implemented */
+#define MC_CMD_ERR_ENOSYS		0x26
+/* Operation timed out */
+#define MC_CMD_ERR_ETIME		0x3e
+/* Link has been severed */
+#define MC_CMD_ERR_ENOLINK		0x43
+/* Protocol error */
+#define MC_CMD_ERR_EPROTO		0x47
+/* Bad message */
+#define MC_CMD_ERR_EBADMSG		0x4a
+/* Operation not supported */
+#define MC_CMD_ERR_ENOTSUP		0x5f
+/* Address not available */
+#define MC_CMD_ERR_EADDRNOTAVAIL	0x63
+/* Not connected */
+#define MC_CMD_ERR_ENOTCONN		0x6b
+/* Operation already in progress */
+#define MC_CMD_ERR_EALREADY		0x72
+/* Stale handle. The handle references resource that no longer exists */
+#define MC_CMD_ERR_ESTALE		0x74
+/* Resource allocation failed. */
+#define MC_CMD_ERR_ALLOC_FAIL		0x1000
+/* V-adaptor not found. */
+#define MC_CMD_ERR_NO_VADAPTOR		0x1001
+/* EVB port not found. */
+#define MC_CMD_ERR_NO_EVB_PORT		0x1002
+/* V-switch not found. */
+#define MC_CMD_ERR_NO_VSWITCH		0x1003
+/* Too many VLAN tags. */
+#define MC_CMD_ERR_VLAN_LIMIT		0x1004
+/* Bad PCI function number. */
+#define MC_CMD_ERR_BAD_PCI_FUNC		0x1005
+/* Invalid VLAN mode. */
+#define MC_CMD_ERR_BAD_VLAN_MODE	0x1006
+/* Invalid v-switch type. */
+#define MC_CMD_ERR_BAD_VSWITCH_TYPE	0x1007
+/* Invalid v-port type. */
+#define MC_CMD_ERR_BAD_VPORT_TYPE	0x1008
+/* MAC address exists. */
+#define MC_CMD_ERR_MAC_EXIST		0x1009
+/* Slave core not present */
+#define MC_CMD_ERR_SLAVE_NOT_PRESENT	0x100a
+/* The datapath is disabled. */
+#define MC_CMD_ERR_DATAPATH_DISABLED	0x100b
+/* The requesting client is not a function */
+#define MC_CMD_ERR_CLIENT_NOT_FN	0x100c
+/*
+ * The requested operation might require the command to be passed between
+ * MCs, and the transport doesn't support that. Should only ever been seen over
+ * the UART.
+ */
+#define MC_CMD_ERR_NO_PRIVILEGE		0x1013
+/*
+ * Workaround 26807 could not be turned on/off because some functions
+ * have already installed filters. See the comment at
+ * MC_CMD_WORKAROUND_BUG26807. May also returned for other operations such as
+ * sub-variant switching.
+ */
+#define MC_CMD_ERR_FILTERS_PRESENT	0x1014
+/* The clock whose frequency you've attempted to set doesn't exist */
+#define MC_CMD_ERR_NO_CLOCK		0x1015
+/*
+ * Returned by MC_CMD_TESTASSERT if the action that should have caused an
+ * assertion failed to do so.
+ */
+#define MC_CMD_ERR_UNREACHABLE		0x1016
+/*
+ * This command needs to be processed in the background but there were no
+ * resources to do so. Send it again after a command has completed.
+ */
+#define MC_CMD_ERR_QUEUE_FULL		0x1017
+/*
+ * The operation could not be completed because the PCIe link has gone
+ * away. This error code is never expected to be returned over the TLP
+ * transport.
+ */
+#define MC_CMD_ERR_NO_PCIE		0x1018
+/*
+ * The operation could not be completed because the datapath has gone
+ * away. This is distinct from MC_CMD_ERR_DATAPATH_DISABLED in that the
+ * datapath absence may be temporary
+ */
+#define MC_CMD_ERR_NO_DATAPATH		0x1019
+/* The operation could not complete because some VIs are allocated */
+#define MC_CMD_ERR_VIS_PRESENT		0x101a
+/*
+ * The operation could not complete because some PIO buffers are
+ * allocated
+ */
+#define MC_CMD_ERR_PIOBUFS_PRESENT	0x101b
+
+/***********************************/
+/*
+ * MC_CMD_CDX_BUS_ENUM_BUSES
+ * CDX bus hosts devices (functions) that are implemented using the Composable
+ * DMA subsystem and directly mapped into the memory space of the FGPA PSX
+ * Application Processors (APUs). As such, they only apply to the PSX APU side,
+ * not the host (PCIe). Unlike PCIe, these devices have no native configuration
+ * space or enumeration mechanism, so this message set provides a minimal
+ * interface for discovery and management (bus reset, FLR, BME) of such
+ * devices. This command returns the number of CDX buses present in the system.
+ */
+#define MC_CMD_CDX_BUS_ENUM_BUSES				0x1
+#define MC_CMD_CDX_BUS_ENUM_BUSES_MSGSET			0x1
+#undef MC_CMD_0x1_PRIVILEGE_CTG
+
+#define MC_CMD_0x1_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+
+/* MC_CMD_CDX_BUS_ENUM_BUSES_IN msgrequest */
+#define MC_CMD_CDX_BUS_ENUM_BUSES_IN_LEN			0
+
+/* MC_CMD_CDX_BUS_ENUM_BUSES_OUT msgresponse */
+#define MC_CMD_CDX_BUS_ENUM_BUSES_OUT_LEN			4
+/*
+ * Number of CDX buses present in the system. Buses are numbered 0 to
+ * BUS_COUNT-1
+ */
+#define MC_CMD_CDX_BUS_ENUM_BUSES_OUT_BUS_COUNT_OFST		0
+#define MC_CMD_CDX_BUS_ENUM_BUSES_OUT_BUS_COUNT_LEN		4
+
+/***********************************/
+/*
+ * MC_CMD_CDX_BUS_ENUM_DEVICES
+ * Enumerate CDX bus devices on a given bus
+ */
+#define MC_CMD_CDX_BUS_ENUM_DEVICES				0x2
+#define MC_CMD_CDX_BUS_ENUM_DEVICES_MSGSET			0x2
+#undef MC_CMD_0x2_PRIVILEGE_CTG
+
+#define MC_CMD_0x2_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+
+/* MC_CMD_CDX_BUS_ENUM_DEVICES_IN msgrequest */
+#define MC_CMD_CDX_BUS_ENUM_DEVICES_IN_LEN			4
+/*
+ * Bus number to enumerate, in range 0 to BUS_COUNT-1, as returned by
+ * MC_CMD_CDX_BUS_ENUM_BUSES_OUT
+ */
+#define MC_CMD_CDX_BUS_ENUM_DEVICES_IN_BUS_OFST			0
+#define MC_CMD_CDX_BUS_ENUM_DEVICES_IN_BUS_LEN			4
+
+/* MC_CMD_CDX_BUS_ENUM_DEVICES_OUT msgresponse */
+#define MC_CMD_CDX_BUS_ENUM_DEVICES_OUT_LEN			4
+/*
+ * Number of devices present on the bus. Devices on the bus are numbered 0 to
+ * DEVICE_COUNT-1. Returns EAGAIN if number of devices unknown or if the target
+ * devices are not ready (e.g. undergoing a bus reset)
+ */
+#define MC_CMD_CDX_BUS_ENUM_DEVICES_OUT_DEVICE_COUNT_OFST	0
+#define MC_CMD_CDX_BUS_ENUM_DEVICES_OUT_DEVICE_COUNT_LEN	4
+
+/***********************************/
+/*
+ * MC_CMD_CDX_BUS_GET_DEVICE_CONFIG
+ * Returns device identification and MMIO/MSI resource data for a CDX device.
+ * The expected usage is for the caller to first retrieve the number of devices
+ * on the bus using MC_CMD_BUS_ENUM_DEVICES, then loop through the range (0,
+ * DEVICE_COUNT - 1), retrieving device resource data. May return EAGAIN if the
+ * number of exposed devices or device resources change during enumeration (due
+ * to e.g. a PL reload / bus reset), in which case the caller is expected to
+ * restart the enumeration loop. MMIO addresses are specified in terms of bus
+ * addresses (prior to any potential IOMMU translation). For versal-net, these
+ * are equivalent to APU physical addresses. Implementation note - for this to
+ * work, the implementation needs to keep state (generation count) per client.
+ */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG					0x3
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_MSGSET					0x3
+#undef MC_CMD_0x3_PRIVILEGE_CTG
+
+#define MC_CMD_0x3_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+
+/* MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_IN msgrequest */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_IN_LEN					8
+/* Device bus number, in range 0 to BUS_COUNT-1 */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_IN_BUS_OFST				0
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_IN_BUS_LEN				4
+/* Device number relative to the bus, in range 0 to DEVICE_COUNT-1 for that bus */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_IN_DEVICE_OFST				4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_IN_DEVICE_LEN				4
+
+/* MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT msgresponse */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_LEN				88
+/* 16-bit Vendor identifier, compliant with PCI-SIG VendorID assignment. */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_VENDOR_ID_OFST			0
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_VENDOR_ID_LEN			2
+/* 16-bit Device ID assigned by the vendor */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_DEVICE_ID_OFST			2
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_DEVICE_ID_LEN			2
+/*
+ * 16-bit Subsystem Vendor ID, , compliant with PCI-SIG VendorID assignment.
+ * For further device differentiation, as required. 0 if unused.
+ */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_SUBSYS_VENDOR_ID_OFST		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_SUBSYS_VENDOR_ID_LEN		2
+/*
+ * 16-bit Subsystem Device ID assigned by the vendor. For further device
+ * differentiation, as required. 0 if unused.
+ */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_SUBSYS_DEVICE_ID_OFST		6
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_SUBSYS_DEVICE_ID_LEN		2
+/* 24-bit Device Class code, compliant with PCI-SIG Device Class codes */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_DEVICE_CLASS_OFST			8
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_DEVICE_CLASS_LEN			3
+/* 8-bit vendor-assigned revision */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_DEVICE_REVISION_OFST		11
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_DEVICE_REVISION_LEN		1
+/* Reserved (alignment) */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_RESERVED_OFST			12
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_RESERVED_LEN			4
+/* MMIO region 0 base address (bus address), 0 if unused */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE_OFST		16
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE_LEN		8
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE_LO_OFST		16
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE_LO_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE_LO_LBN		128
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE_LO_WIDTH		32
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE_HI_OFST		20
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE_HI_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE_HI_LBN		160
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE_HI_WIDTH		32
+/* MMIO region 0 size, 0 if unused */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE_OFST		24
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE_LEN		8
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE_LO_OFST		24
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE_LO_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE_LO_LBN		192
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE_LO_WIDTH		32
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE_HI_OFST		28
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE_HI_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE_HI_LBN		224
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE_HI_WIDTH		32
+/* MMIO region 1 base address (bus address), 0 if unused */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE_OFST		32
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE_LEN		8
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE_LO_OFST		32
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE_LO_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE_LO_LBN		256
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE_LO_WIDTH		32
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE_HI_OFST		36
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE_HI_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE_HI_LBN		288
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE_HI_WIDTH		32
+/* MMIO region 1 size, 0 if unused */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE_OFST		40
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE_LEN		8
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE_LO_OFST		40
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE_LO_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE_LO_LBN		320
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE_LO_WIDTH		32
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE_HI_OFST		44
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE_HI_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE_HI_LBN		352
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE_HI_WIDTH		32
+/* MMIO region 2 base address (bus address), 0 if unused */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE_OFST		48
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE_LEN		8
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE_LO_OFST		48
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE_LO_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE_LO_LBN		384
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE_LO_WIDTH		32
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE_HI_OFST		52
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE_HI_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE_HI_LBN		416
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE_HI_WIDTH		32
+/* MMIO region 2 size, 0 if unused */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE_OFST		56
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE_LEN		8
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE_LO_OFST		56
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE_LO_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE_LO_LBN		448
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE_LO_WIDTH		32
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE_HI_OFST		60
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE_HI_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE_HI_LBN		480
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE_HI_WIDTH		32
+/* MMIO region 3 base address (bus address), 0 if unused */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE_OFST		64
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE_LEN		8
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE_LO_OFST		64
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE_LO_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE_LO_LBN		512
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE_LO_WIDTH		32
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE_HI_OFST		68
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE_HI_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE_HI_LBN		544
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE_HI_WIDTH		32
+/* MMIO region 3 size, 0 if unused */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE_OFST		72
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE_LEN		8
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE_LO_OFST		72
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE_LO_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE_LO_LBN		576
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE_LO_WIDTH		32
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE_HI_OFST		76
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE_HI_LEN		4
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE_HI_LBN		608
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE_HI_WIDTH		32
+/* MSI vector count */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MSI_COUNT_OFST			80
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_MSI_COUNT_LEN			4
+/* Requester ID used by device (SMMU StreamID, GIC ITS DeviceID) */
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_REQUESTER_ID_OFST			84
+#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_REQUESTER_ID_LEN			4
+
+/***********************************/
+/*
+ * MC_CMD_CDX_DEVICE_RESET
+ * After this call completes, device DMA and interrupts are quiesced, devices
+ * logic is reset in a hardware-specific way and DMA bus mastering is disabled.
+ */
+#define MC_CMD_CDX_DEVICE_RESET				0x6
+#define MC_CMD_CDX_DEVICE_RESET_MSGSET			0x6
+#undef MC_CMD_0x6_PRIVILEGE_CTG
+
+#define MC_CMD_0x6_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+
+/* MC_CMD_CDX_DEVICE_RESET_IN msgrequest */
+#define MC_CMD_CDX_DEVICE_RESET_IN_LEN			8
+/* Device bus number, in range 0 to BUS_COUNT-1 */
+#define MC_CMD_CDX_DEVICE_RESET_IN_BUS_OFST		0
+#define MC_CMD_CDX_DEVICE_RESET_IN_BUS_LEN		4
+/* Device number relative to the bus, in range 0 to DEVICE_COUNT-1 for that bus */
+#define MC_CMD_CDX_DEVICE_RESET_IN_DEVICE_OFST		4
+#define MC_CMD_CDX_DEVICE_RESET_IN_DEVICE_LEN		4
+
+/*
+ * MC_CMD_CDX_DEVICE_RESET_OUT msgresponse: The device is quiesced and all
+ * pending device initiated DMA has completed.
+ */
+#define MC_CMD_CDX_DEVICE_RESET_OUT_LEN 0
+
+/***********************************/
+/*
+ * MC_CMD_CDX_DEVICE_CONTROL_SET
+ * If BUS_MASTER is set to disabled, device DMA and interrupts are quiesced.
+ * Pending DMA requests and MSI interrupts are flushed and no further DMA or
+ * interrupts are issued after this command returns. If BUS_MASTER is set to
+ * enabled, device is allowed to initiate DMA. Whether interrupts are enabled
+ * also depends on the value of MSI_ENABLE bit. Note that, in this case, the
+ * device may start DMA before the host receives and processes the MCDI
+ * response. MSI_ENABLE masks or unmasks device interrupts only. Note that for
+ * interrupts to be delivered to the host, both BUS_MASTER and MSI_ENABLE needs
+ * to be set. MMIO_REGIONS_ENABLE enables or disables host accesses to device
+ * MMIO regions. Note that an implementation is allowed to permanently set this
+ * bit to 1, in which case MC_CMD_CDX_DEVICE_CONTROL_GET will always return 1
+ * for this bit, regardless of the value set here.
+ */
+#define MC_CMD_CDX_DEVICE_CONTROL_SET					0x7
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_MSGSET				0x7
+#undef MC_CMD_0x7_PRIVILEGE_CTG
+
+#define MC_CMD_0x7_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+
+/* MC_CMD_CDX_DEVICE_CONTROL_SET_IN msgrequest */
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_LEN				12
+/* Device bus number, in range 0 to BUS_COUNT-1 */
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_BUS_OFST			0
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_BUS_LEN			4
+/* Device number relative to the bus, in range 0 to DEVICE_COUNT-1 for that bus */
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_DEVICE_OFST			4
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_DEVICE_LEN			4
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_FLAGS_OFST			8
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_FLAGS_LEN			4
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_BUS_MASTER_ENABLE_OFST		8
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_BUS_MASTER_ENABLE_LBN		0
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_BUS_MASTER_ENABLE_WIDTH	1
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_MSI_ENABLE_OFST		8
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_MSI_ENABLE_LBN			1
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_MSI_ENABLE_WIDTH		1
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_MMIO_REGIONS_ENABLE_OFST	8
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_MMIO_REGIONS_ENABLE_LBN	2
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_IN_MMIO_REGIONS_ENABLE_WIDTH	1
+
+/* MC_CMD_CDX_DEVICE_CONTROL_SET_OUT msgresponse */
+#define MC_CMD_CDX_DEVICE_CONTROL_SET_OUT_LEN				0
+
+/***********************************/
+/*
+ * MC_CMD_CDX_DEVICE_CONTROL_GET
+ * Returns device DMA, interrupt and MMIO region access control bits. See
+ * MC_CMD_CDX_DEVICE_CONTROL_SET for definition of the available control bits.
+ */
+#define MC_CMD_CDX_DEVICE_CONTROL_GET					0x8
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_MSGSET				0x8
+#undef MC_CMD_0x8_PRIVILEGE_CTG
+
+#define MC_CMD_0x8_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+
+/* MC_CMD_CDX_DEVICE_CONTROL_GET_IN msgrequest */
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_IN_LEN				8
+/* Device bus number, in range 0 to BUS_COUNT-1 */
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_IN_BUS_OFST			0
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_IN_BUS_LEN			4
+/* Device number relative to the bus, in range 0 to DEVICE_COUNT-1 for that bus */
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_IN_DEVICE_OFST			4
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_IN_DEVICE_LEN			4
+
+/* MC_CMD_CDX_DEVICE_CONTROL_GET_OUT msgresponse */
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_LEN				4
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_FLAGS_OFST			0
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_FLAGS_LEN			4
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_BUS_MASTER_ENABLE_OFST	0
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_BUS_MASTER_ENABLE_LBN		0
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_BUS_MASTER_ENABLE_WIDTH	1
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_MSI_ENABLE_OFST		0
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_MSI_ENABLE_LBN		1
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_MSI_ENABLE_WIDTH		1
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_MMIO_REGIONS_ENABLE_OFST	0
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_MMIO_REGIONS_ENABLE_LBN	2
+#define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_MMIO_REGIONS_ENABLE_WIDTH	1
+
+/***********************************/
+/* MC_CMD_V2_EXTN - Encapsulation for a v2 extended command */
+#define MC_CMD_V2_EXTN					0x7f
+
+/* MC_CMD_V2_EXTN_IN msgrequest */
+#define MC_CMD_V2_EXTN_IN_LEN				4
+/* the extended command number */
+#define MC_CMD_V2_EXTN_IN_EXTENDED_CMD_LBN		0
+#define MC_CMD_V2_EXTN_IN_EXTENDED_CMD_WIDTH		15
+#define MC_CMD_V2_EXTN_IN_UNUSED_LBN			15
+#define MC_CMD_V2_EXTN_IN_UNUSED_WIDTH			1
+/* the actual length of the encapsulated command */
+#define MC_CMD_V2_EXTN_IN_ACTUAL_LEN_LBN		16
+#define MC_CMD_V2_EXTN_IN_ACTUAL_LEN_WIDTH		10
+#define MC_CMD_V2_EXTN_IN_UNUSED2_LBN			26
+#define MC_CMD_V2_EXTN_IN_UNUSED2_WIDTH			2
+/* Type of command/response */
+#define MC_CMD_V2_EXTN_IN_MESSAGE_TYPE_LBN		28
+#define MC_CMD_V2_EXTN_IN_MESSAGE_TYPE_WIDTH		4
+/*
+ * enum: MCDI command directed to versal-net. MCDI responses of this type
+ * are not defined.
+ */
+#define MC_CMD_V2_EXTN_IN_MCDI_MESSAGE_TYPE_PLATFORM	0x2
+
+#endif /* MC_CDX_PCOL_H */
diff --git a/drivers/cdx/controller/mcdi.c b/drivers/cdx/controller/mcdi.c
new file mode 100644
index 000000000000..a211a2ca762e
--- /dev/null
+++ b/drivers/cdx/controller/mcdi.c
@@ -0,0 +1,903 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Management-Controller-to-Driver Interface
+ *
+ * Copyright 2008-2013 Solarflare Communications Inc.
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/if_vlan.h>
+#include <linux/timer.h>
+#include <linux/list.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/rwsem.h>
+#include <linux/vmalloc.h>
+#include <net/netevent.h>
+#include <linux/log2.h>
+#include <linux/net_tstamp.h>
+#include <linux/wait.h>
+
+#include "bitfield.h"
+#include "mcdi.h"
+
+struct cdx_mcdi_copy_buffer {
+	struct cdx_dword buffer[DIV_ROUND_UP(MCDI_CTL_SDU_LEN_MAX, 4)];
+};
+
+#ifdef CONFIG_MCDI_LOGGING
+#define LOG_LINE_MAX		(1024 - 32)
+#endif
+
+static void cdx_mcdi_cancel_cmd(struct cdx_mcdi *cdx, struct cdx_mcdi_cmd *cmd);
+static void cdx_mcdi_wait_for_cleanup(struct cdx_mcdi *cdx);
+static int cdx_mcdi_rpc_async_internal(struct cdx_mcdi *cdx,
+				       struct cdx_mcdi_cmd *cmd,
+				       unsigned int *handle);
+static void cdx_mcdi_start_or_queue(struct cdx_mcdi_iface *mcdi,
+				    bool allow_retry);
+static void cdx_mcdi_cmd_start_or_queue(struct cdx_mcdi_iface *mcdi,
+					struct cdx_mcdi_cmd *cmd);
+static bool cdx_mcdi_complete_cmd(struct cdx_mcdi_iface *mcdi,
+				  struct cdx_mcdi_cmd *cmd,
+				  struct cdx_dword *outbuf,
+				  int len,
+				  struct list_head *cleanup_list);
+static void cdx_mcdi_timeout_cmd(struct cdx_mcdi_iface *mcdi,
+				 struct cdx_mcdi_cmd *cmd,
+				 struct list_head *cleanup_list);
+static void cdx_mcdi_cmd_work(struct work_struct *context);
+static void cdx_mcdi_mode_fail(struct cdx_mcdi *cdx, struct list_head *cleanup_list);
+static void _cdx_mcdi_display_error(struct cdx_mcdi *cdx, unsigned int cmd,
+				    size_t inlen, int raw, int arg, int err_no);
+
+static bool cdx_cmd_cancelled(struct cdx_mcdi_cmd *cmd)
+{
+	return cmd->state == MCDI_STATE_RUNNING_CANCELLED;
+}
+
+static void cdx_mcdi_cmd_release(struct kref *ref)
+{
+	kfree(container_of(ref, struct cdx_mcdi_cmd, ref));
+}
+
+static unsigned int cdx_mcdi_cmd_handle(struct cdx_mcdi_cmd *cmd)
+{
+	return cmd->handle;
+}
+
+static void _cdx_mcdi_remove_cmd(struct cdx_mcdi_iface *mcdi,
+				 struct cdx_mcdi_cmd *cmd,
+				 struct list_head *cleanup_list)
+{
+	/* if cancelled, the completers have already been called */
+	if (cdx_cmd_cancelled(cmd))
+		return;
+
+	if (cmd->completer) {
+		list_add_tail(&cmd->cleanup_list, cleanup_list);
+		++mcdi->outstanding_cleanups;
+		kref_get(&cmd->ref);
+	}
+}
+
+static void cdx_mcdi_remove_cmd(struct cdx_mcdi_iface *mcdi,
+				struct cdx_mcdi_cmd *cmd,
+				struct list_head *cleanup_list)
+{
+	list_del(&cmd->list);
+	_cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
+	cmd->state = MCDI_STATE_FINISHED;
+	kref_put(&cmd->ref, cdx_mcdi_cmd_release);
+	if (list_empty(&mcdi->cmd_list))
+		wake_up(&mcdi->cmd_complete_wq);
+}
+
+static unsigned long cdx_mcdi_rpc_timeout(struct cdx_mcdi *cdx, unsigned int cmd)
+{
+	if (!cdx->mcdi_ops->mcdi_rpc_timeout)
+		return MCDI_RPC_TIMEOUT;
+	else
+		return cdx->mcdi_ops->mcdi_rpc_timeout(cdx, cmd);
+}
+
+int cdx_mcdi_init(struct cdx_mcdi *cdx)
+{
+	struct cdx_mcdi_iface *mcdi;
+	int rc = -ENOMEM;
+
+	cdx->mcdi = kzalloc(sizeof(*cdx->mcdi), GFP_KERNEL);
+	if (!cdx->mcdi)
+		goto fail;
+
+	mcdi = cdx_mcdi_if(cdx);
+	mcdi->cdx = cdx;
+
+#ifdef CONFIG_MCDI_LOGGING
+	mcdi->logging_buffer = kmalloc(LOG_LINE_MAX, GFP_KERNEL);
+	if (!mcdi->logging_buffer)
+		goto fail2;
+#endif
+	mcdi->workqueue = alloc_ordered_workqueue("mcdi_wq", 0);
+	if (!mcdi->workqueue)
+		goto fail3;
+	mutex_init(&mcdi->iface_lock);
+	mcdi->mode = MCDI_MODE_EVENTS;
+	INIT_LIST_HEAD(&mcdi->cmd_list);
+	init_waitqueue_head(&mcdi->cmd_complete_wq);
+
+	mcdi->new_epoch = true;
+
+	return 0;
+fail3:
+#ifdef CONFIG_MCDI_LOGGING
+	kfree(mcdi->logging_buffer);
+fail2:
+#endif
+	kfree(cdx->mcdi);
+	cdx->mcdi = NULL;
+fail:
+	return rc;
+}
+
+void cdx_mcdi_finish(struct cdx_mcdi *cdx)
+{
+	struct cdx_mcdi_iface *mcdi;
+
+	mcdi = cdx_mcdi_if(cdx);
+	if (!mcdi)
+		return;
+
+	cdx_mcdi_wait_for_cleanup(cdx);
+
+#ifdef CONFIG_MCDI_LOGGING
+	kfree(mcdi->logging_buffer);
+#endif
+
+	destroy_workqueue(mcdi->workqueue);
+	kfree(cdx->mcdi);
+	cdx->mcdi = NULL;
+}
+
+static bool cdx_mcdi_flushed(struct cdx_mcdi_iface *mcdi, bool ignore_cleanups)
+{
+	bool flushed;
+
+	mutex_lock(&mcdi->iface_lock);
+	flushed = list_empty(&mcdi->cmd_list) &&
+		  (ignore_cleanups || !mcdi->outstanding_cleanups);
+	mutex_unlock(&mcdi->iface_lock);
+	return flushed;
+}
+
+/* Wait for outstanding MCDI commands to complete. */
+static void cdx_mcdi_wait_for_cleanup(struct cdx_mcdi *cdx)
+{
+	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
+
+	if (!mcdi)
+		return;
+
+	wait_event(mcdi->cmd_complete_wq,
+		   cdx_mcdi_flushed(mcdi, false));
+}
+
+int cdx_mcdi_wait_for_quiescence(struct cdx_mcdi *cdx,
+				 unsigned int timeout_jiffies)
+{
+	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
+	DEFINE_WAIT_FUNC(wait, woken_wake_function);
+	int rc = 0;
+
+	if (!mcdi)
+		return -EINVAL;
+
+	flush_workqueue(mcdi->workqueue);
+
+	add_wait_queue(&mcdi->cmd_complete_wq, &wait);
+
+	while (!cdx_mcdi_flushed(mcdi, true)) {
+		rc = wait_woken(&wait, TASK_IDLE, timeout_jiffies);
+		if (rc)
+			continue;
+		break;
+	}
+
+	remove_wait_queue(&mcdi->cmd_complete_wq, &wait);
+
+	if (rc > 0)
+		rc = 0;
+	else if (rc == 0)
+		rc = -ETIMEDOUT;
+
+	return rc;
+}
+
+static u8 cdx_mcdi_payload_csum(const struct cdx_dword *hdr, size_t hdr_len,
+				const struct cdx_dword *sdu, size_t sdu_len)
+{
+	u8 *p = (u8 *)hdr;
+	u8 csum = 0;
+	int i;
+
+	for (i = 0; i < hdr_len; i++)
+		csum += p[i];
+
+	p = (u8 *)sdu;
+	for (i = 0; i < sdu_len; i++)
+		csum += p[i];
+
+	return ~csum & 0xff;
+}
+
+static void cdx_mcdi_send_request(struct cdx_mcdi *cdx,
+				  struct cdx_mcdi_cmd *cmd)
+{
+	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
+	const struct cdx_dword *inbuf = cmd->inbuf;
+	size_t inlen = cmd->inlen;
+	struct cdx_dword hdr[2];
+	size_t hdr_len;
+	bool not_epoch;
+	u32 xflags;
+#ifdef CONFIG_MCDI_LOGGING
+	char *buf;
+#endif
+
+	if (!mcdi)
+		return;
+#ifdef CONFIG_MCDI_LOGGING
+	buf = mcdi->logging_buffer; /* page-sized */
+#endif
+
+	mcdi->prev_seq = cmd->seq;
+	mcdi->seq_held_by[cmd->seq] = cmd;
+	mcdi->db_held_by = cmd;
+	cmd->started = jiffies;
+
+	not_epoch = !mcdi->new_epoch;
+	xflags = 0;
+
+	/* MCDI v2 */
+	WARN_ON(inlen > MCDI_CTL_SDU_LEN_MAX_V2);
+	CDX_POPULATE_DWORD_7(hdr[0],
+			     MCDI_HEADER_RESPONSE, 0,
+			     MCDI_HEADER_RESYNC, 1,
+			     MCDI_HEADER_CODE, MC_CMD_V2_EXTN,
+			     MCDI_HEADER_DATALEN, 0,
+			     MCDI_HEADER_SEQ, cmd->seq,
+			     MCDI_HEADER_XFLAGS, xflags,
+			     MCDI_HEADER_NOT_EPOCH, not_epoch);
+	CDX_POPULATE_DWORD_3(hdr[1],
+			     MC_CMD_V2_EXTN_IN_EXTENDED_CMD, cmd->cmd,
+			     MC_CMD_V2_EXTN_IN_ACTUAL_LEN, inlen,
+			     MC_CMD_V2_EXTN_IN_MESSAGE_TYPE,
+			     MC_CMD_V2_EXTN_IN_MCDI_MESSAGE_TYPE_PLATFORM);
+	hdr_len = 8;
+
+#ifdef CONFIG_MCDI_LOGGING
+	if (!WARN_ON_ONCE(!buf)) {
+		const struct cdx_dword *frags[] = { hdr, inbuf };
+		const size_t frag_len[] = { hdr_len, round_up(inlen, 4) };
+		int bytes = 0;
+		int i, j;
+
+		for (j = 0; j < ARRAY_SIZE(frags); j++) {
+			const struct cdx_dword *frag;
+
+			frag = frags[j];
+			for (i = 0;
+			     i < frag_len[j] / 4;
+			     i++) {
+				/*
+				 * Do not exceed the internal printk limit.
+				 * The string before that is just over 70 bytes.
+				 */
+				if ((bytes + 75) > LOG_LINE_MAX) {
+					pr_info("MCDI RPC REQ:%s \\\n", buf);
+					bytes = 0;
+				}
+				bytes += snprintf(buf + bytes,
+						  LOG_LINE_MAX - bytes, " %08x",
+						  le32_to_cpu(frag[i].cdx_u32));
+			}
+		}
+
+		pr_info("MCDI RPC REQ:%s\n", buf);
+	}
+#endif
+	hdr[0].cdx_u32 |= (__force __le32)(cdx_mcdi_payload_csum(hdr, hdr_len, inbuf, inlen) <<
+			 MCDI_HEADER_XFLAGS_LBN);
+	cdx->mcdi_ops->mcdi_request(cdx, hdr, hdr_len, inbuf, inlen);
+
+	mcdi->new_epoch = false;
+}
+
+static int cdx_mcdi_errno(struct cdx_mcdi *cdx, unsigned int mcdi_err)
+{
+	switch (mcdi_err) {
+	case 0:
+	case MC_CMD_ERR_QUEUE_FULL:
+		return mcdi_err;
+	case MC_CMD_ERR_EPERM:
+		return -EPERM;
+	case MC_CMD_ERR_ENOENT:
+		return -ENOENT;
+	case MC_CMD_ERR_EINTR:
+		return -EINTR;
+	case MC_CMD_ERR_EAGAIN:
+		return -EAGAIN;
+	case MC_CMD_ERR_EACCES:
+		return -EACCES;
+	case MC_CMD_ERR_EBUSY:
+		return -EBUSY;
+	case MC_CMD_ERR_EINVAL:
+		return -EINVAL;
+	case MC_CMD_ERR_ERANGE:
+		return -ERANGE;
+	case MC_CMD_ERR_EDEADLK:
+		return -EDEADLK;
+	case MC_CMD_ERR_ENOSYS:
+		return -EOPNOTSUPP;
+	case MC_CMD_ERR_ETIME:
+		return -ETIME;
+	case MC_CMD_ERR_EALREADY:
+		return -EALREADY;
+	case MC_CMD_ERR_ENOSPC:
+		return -ENOSPC;
+	case MC_CMD_ERR_ENOMEM:
+		return -ENOMEM;
+	case MC_CMD_ERR_ENOTSUP:
+		return -EOPNOTSUPP;
+	case MC_CMD_ERR_ALLOC_FAIL:
+		return -ENOBUFS;
+	case MC_CMD_ERR_MAC_EXIST:
+		return -EADDRINUSE;
+	case MC_CMD_ERR_NO_EVB_PORT:
+		return -EAGAIN;
+	default:
+		return -EPROTO;
+	}
+}
+
+static void cdx_mcdi_process_cleanup_list(struct cdx_mcdi *cdx,
+					  struct list_head *cleanup_list)
+{
+	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
+	unsigned int cleanups = 0;
+
+	if (!mcdi)
+		return;
+
+	while (!list_empty(cleanup_list)) {
+		struct cdx_mcdi_cmd *cmd =
+			list_first_entry(cleanup_list,
+					 struct cdx_mcdi_cmd, cleanup_list);
+		cmd->completer(cdx, cmd->cookie, cmd->rc,
+			       cmd->outbuf, cmd->outlen);
+		list_del(&cmd->cleanup_list);
+		kref_put(&cmd->ref, cdx_mcdi_cmd_release);
+		++cleanups;
+	}
+
+	if (cleanups) {
+		bool all_done;
+
+		mutex_lock(&mcdi->iface_lock);
+		CDX_WARN_ON_PARANOID(cleanups > mcdi->outstanding_cleanups);
+		all_done = (mcdi->outstanding_cleanups -= cleanups) == 0;
+		mutex_unlock(&mcdi->iface_lock);
+		if (all_done)
+			wake_up(&mcdi->cmd_complete_wq);
+	}
+}
+
+static void _cdx_mcdi_cancel_cmd(struct cdx_mcdi_iface *mcdi,
+				 unsigned int handle,
+				 struct list_head *cleanup_list)
+{
+	struct cdx_mcdi_cmd *cmd;
+
+	list_for_each_entry(cmd, &mcdi->cmd_list, list)
+		if (cdx_mcdi_cmd_handle(cmd) == handle) {
+			switch (cmd->state) {
+			case MCDI_STATE_QUEUED:
+			case MCDI_STATE_RETRY:
+				pr_debug("command %#x inlen %zu cancelled in queue\n",
+					 cmd->cmd, cmd->inlen);
+				/* if not yet running, properly cancel it */
+				cmd->rc = -EPIPE;
+				cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
+				break;
+			case MCDI_STATE_RUNNING:
+			case MCDI_STATE_RUNNING_CANCELLED:
+			case MCDI_STATE_FINISHED:
+			default:
+				/* invalid state? */
+				WARN_ON(1);
+			}
+			break;
+		}
+}
+
+static void cdx_mcdi_cancel_cmd(struct cdx_mcdi *cdx, struct cdx_mcdi_cmd *cmd)
+{
+	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
+	LIST_HEAD(cleanup_list);
+
+	if (!mcdi)
+		return;
+
+	mutex_lock(&mcdi->iface_lock);
+	cdx_mcdi_timeout_cmd(mcdi, cmd, &cleanup_list);
+	mutex_unlock(&mcdi->iface_lock);
+	cdx_mcdi_process_cleanup_list(cdx, &cleanup_list);
+}
+
+struct cdx_mcdi_blocking_data {
+	struct kref ref;
+	bool done;
+	wait_queue_head_t wq;
+	int rc;
+	struct cdx_dword *outbuf;
+	size_t outlen;
+	size_t outlen_actual;
+};
+
+static void cdx_mcdi_blocking_data_release(struct kref *ref)
+{
+	kfree(container_of(ref, struct cdx_mcdi_blocking_data, ref));
+}
+
+static void cdx_mcdi_rpc_completer(struct cdx_mcdi *cdx, unsigned long cookie,
+				   int rc, struct cdx_dword *outbuf,
+				   size_t outlen_actual)
+{
+	struct cdx_mcdi_blocking_data *wait_data =
+		(struct cdx_mcdi_blocking_data *)cookie;
+
+	wait_data->rc = rc;
+	memcpy(wait_data->outbuf, outbuf,
+	       min(outlen_actual, wait_data->outlen));
+	wait_data->outlen_actual = outlen_actual;
+	/* memory barrier */
+	smp_wmb();
+	wait_data->done = true;
+	wake_up(&wait_data->wq);
+	kref_put(&wait_data->ref, cdx_mcdi_blocking_data_release);
+}
+
+static int cdx_mcdi_rpc_sync(struct cdx_mcdi *cdx, unsigned int cmd,
+			     const struct cdx_dword *inbuf, size_t inlen,
+			     struct cdx_dword *outbuf, size_t outlen,
+			     size_t *outlen_actual, bool quiet)
+{
+	struct cdx_mcdi_blocking_data *wait_data;
+	struct cdx_mcdi_cmd *cmd_item;
+	unsigned int handle;
+	int rc;
+
+	if (outlen_actual)
+		*outlen_actual = 0;
+
+	wait_data = kmalloc(sizeof(*wait_data), GFP_KERNEL);
+	if (!wait_data)
+		return -ENOMEM;
+
+	cmd_item = kmalloc(sizeof(*cmd_item), GFP_KERNEL);
+	if (!cmd_item) {
+		kfree(wait_data);
+		return -ENOMEM;
+	}
+
+	kref_init(&wait_data->ref);
+	wait_data->done = false;
+	init_waitqueue_head(&wait_data->wq);
+	wait_data->outbuf = outbuf;
+	wait_data->outlen = outlen;
+
+	kref_init(&cmd_item->ref);
+	cmd_item->quiet = quiet;
+	cmd_item->cookie = (unsigned long)wait_data;
+	cmd_item->completer = &cdx_mcdi_rpc_completer;
+	cmd_item->cmd = cmd;
+	cmd_item->inlen = inlen;
+	cmd_item->inbuf = inbuf;
+
+	/* Claim an extra reference for the completer to put. */
+	kref_get(&wait_data->ref);
+	rc = cdx_mcdi_rpc_async_internal(cdx, cmd_item, &handle);
+	if (rc) {
+		kref_put(&wait_data->ref, cdx_mcdi_blocking_data_release);
+		goto out;
+	}
+
+	if (!wait_event_timeout(wait_data->wq, wait_data->done,
+				cdx_mcdi_rpc_timeout(cdx, cmd)) &&
+	    !wait_data->done) {
+		pr_err("MC command 0x%x inlen %zu timed out (sync)\n",
+		       cmd, inlen);
+
+		cdx_mcdi_cancel_cmd(cdx, cmd_item);
+
+		wait_data->rc = -ETIMEDOUT;
+		wait_data->outlen_actual = 0;
+	}
+
+	if (outlen_actual)
+		*outlen_actual = wait_data->outlen_actual;
+	rc = wait_data->rc;
+
+out:
+	kref_put(&wait_data->ref, cdx_mcdi_blocking_data_release);
+
+	return rc;
+}
+
+static bool cdx_mcdi_get_seq(struct cdx_mcdi_iface *mcdi, unsigned char *seq)
+{
+	*seq = mcdi->prev_seq;
+	do {
+		*seq = (*seq + 1) % ARRAY_SIZE(mcdi->seq_held_by);
+	} while (mcdi->seq_held_by[*seq] && *seq != mcdi->prev_seq);
+	return !mcdi->seq_held_by[*seq];
+}
+
+static int cdx_mcdi_rpc_async_internal(struct cdx_mcdi *cdx,
+				       struct cdx_mcdi_cmd *cmd,
+				       unsigned int *handle)
+{
+	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
+	LIST_HEAD(cleanup_list);
+
+	if (!mcdi) {
+		kref_put(&cmd->ref, cdx_mcdi_cmd_release);
+		return -ENETDOWN;
+	}
+
+	if (mcdi->mode == MCDI_MODE_FAIL) {
+		kref_put(&cmd->ref, cdx_mcdi_cmd_release);
+		return -ENETDOWN;
+	}
+
+	cmd->mcdi = mcdi;
+	INIT_WORK(&cmd->work, cdx_mcdi_cmd_work);
+	INIT_LIST_HEAD(&cmd->list);
+	INIT_LIST_HEAD(&cmd->cleanup_list);
+	cmd->rc = 0;
+	cmd->outbuf = NULL;
+	cmd->outlen = 0;
+
+	queue_work(mcdi->workqueue, &cmd->work);
+	return 0;
+}
+
+static void cdx_mcdi_cmd_start_or_queue(struct cdx_mcdi_iface *mcdi,
+					struct cdx_mcdi_cmd *cmd)
+{
+	struct cdx_mcdi *cdx = mcdi->cdx;
+	u8 seq;
+
+	if (!mcdi->db_held_by &&
+	    cdx_mcdi_get_seq(mcdi, &seq)) {
+		cmd->seq = seq;
+		cmd->reboot_seen = false;
+		cdx_mcdi_send_request(cdx, cmd);
+		cmd->state = MCDI_STATE_RUNNING;
+	} else {
+		cmd->state = MCDI_STATE_QUEUED;
+	}
+}
+
+/* try to advance other commands */
+static void cdx_mcdi_start_or_queue(struct cdx_mcdi_iface *mcdi,
+				    bool allow_retry)
+{
+	struct cdx_mcdi_cmd *cmd, *tmp;
+
+	list_for_each_entry_safe(cmd, tmp, &mcdi->cmd_list, list)
+		if (cmd->state == MCDI_STATE_QUEUED ||
+		    (cmd->state == MCDI_STATE_RETRY && allow_retry))
+			cdx_mcdi_cmd_start_or_queue(mcdi, cmd);
+}
+
+void cdx_mcdi_process_cmd(struct cdx_mcdi *cdx, struct cdx_dword *outbuf, int len)
+{
+	struct cdx_mcdi_iface *mcdi;
+	struct cdx_mcdi_cmd *cmd;
+	LIST_HEAD(cleanup_list);
+	unsigned int respseq;
+
+	if (!len || !outbuf) {
+		pr_err("Got empty MC response\n");
+		return;
+	}
+
+	mcdi = cdx_mcdi_if(cdx);
+	if (!mcdi)
+		return;
+
+	respseq = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_SEQ);
+
+	mutex_lock(&mcdi->iface_lock);
+	cmd = mcdi->seq_held_by[respseq];
+
+	if (cmd) {
+		if (cmd->state == MCDI_STATE_FINISHED) {
+			mutex_unlock(&mcdi->iface_lock);
+			kref_put(&cmd->ref, cdx_mcdi_cmd_release);
+			return;
+		}
+
+		cdx_mcdi_complete_cmd(mcdi, cmd, outbuf, len, &cleanup_list);
+	} else {
+		pr_err("MC response unexpected for seq : %0X\n", respseq);
+	}
+
+	mutex_unlock(&mcdi->iface_lock);
+
+	cdx_mcdi_process_cleanup_list(mcdi->cdx, &cleanup_list);
+}
+
+static void cdx_mcdi_cmd_work(struct work_struct *context)
+{
+	struct cdx_mcdi_cmd *cmd =
+		container_of(context, struct cdx_mcdi_cmd, work);
+	struct cdx_mcdi_iface *mcdi = cmd->mcdi;
+
+	mutex_lock(&mcdi->iface_lock);
+
+	cmd->handle = mcdi->prev_handle++;
+	list_add_tail(&cmd->list, &mcdi->cmd_list);
+	cdx_mcdi_cmd_start_or_queue(mcdi, cmd);
+
+	mutex_unlock(&mcdi->iface_lock);
+}
+
+/*
+ * Returns true if the MCDI module is finished with the command.
+ * (examples of false would be if the command was proxied, or it was
+ * rejected by the MC due to lack of resources and requeued).
+ */
+static bool cdx_mcdi_complete_cmd(struct cdx_mcdi_iface *mcdi,
+				  struct cdx_mcdi_cmd *cmd,
+				  struct cdx_dword *outbuf,
+				  int len,
+				  struct list_head *cleanup_list)
+{
+	size_t resp_hdr_len, resp_data_len;
+	struct cdx_mcdi *cdx = mcdi->cdx;
+	unsigned int respcmd, error;
+	bool completed = false;
+	int rc;
+
+	/* ensure the command can't go away before this function returns */
+	kref_get(&cmd->ref);
+
+	respcmd = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_CODE);
+	error = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_ERROR);
+
+	if (respcmd != MC_CMD_V2_EXTN) {
+		resp_hdr_len = 4;
+		resp_data_len = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_DATALEN);
+	} else {
+		resp_data_len = 0;
+		resp_hdr_len = 8;
+		if (len >= 8)
+			resp_data_len =
+				CDX_DWORD_FIELD(outbuf[1], MC_CMD_V2_EXTN_IN_ACTUAL_LEN);
+	}
+
+	if ((resp_hdr_len + resp_data_len) > len) {
+		pr_warn("Incomplete MCDI response received %d. Expected %zu\n",
+			len, (resp_hdr_len + resp_data_len));
+		resp_data_len = 0;
+	}
+
+#ifdef CONFIG_MCDI_LOGGING
+	if (!WARN_ON_ONCE(!mcdi->logging_buffer)) {
+		char *log = mcdi->logging_buffer;
+		int i, bytes = 0;
+		size_t rlen;
+
+		WARN_ON_ONCE(resp_hdr_len % 4);
+
+		rlen = resp_hdr_len / 4 + DIV_ROUND_UP(resp_data_len, 4);
+
+		for (i = 0; i < rlen; i++) {
+			if ((bytes + 75) > LOG_LINE_MAX) {
+				pr_info("MCDI RPC RESP:%s \\\n", log);
+				bytes = 0;
+			}
+			bytes += snprintf(log + bytes, LOG_LINE_MAX - bytes,
+					  " %08x", le32_to_cpu(outbuf[i].cdx_u32));
+		}
+
+		pr_info("MCDI RPC RESP:%s\n", log);
+	}
+#endif
+
+	if (error && resp_data_len == 0) {
+		/* MC rebooted during command */
+		rc = -EIO;
+	} else {
+		if (WARN_ON_ONCE(error && resp_data_len < 4))
+			resp_data_len = 4;
+		if (error) {
+			rc = CDX_DWORD_FIELD(outbuf[resp_hdr_len / 4], CDX_DWORD);
+			if (!cmd->quiet) {
+				int err_arg = 0;
+
+				if (resp_data_len >= MC_CMD_ERR_ARG_OFST + 4) {
+					int offset = (resp_hdr_len + MC_CMD_ERR_ARG_OFST) / 4;
+
+					err_arg = CDX_DWORD_VAL(outbuf[offset]);
+				}
+
+				_cdx_mcdi_display_error(cdx, cmd->cmd,
+							cmd->inlen, rc, err_arg,
+							cdx_mcdi_errno(cdx, rc));
+			}
+			rc = cdx_mcdi_errno(cdx, rc);
+		} else {
+			rc = 0;
+		}
+	}
+
+	/* free doorbell */
+	if (mcdi->db_held_by == cmd)
+		mcdi->db_held_by = NULL;
+
+	if (cdx_cmd_cancelled(cmd)) {
+		list_del(&cmd->list);
+		kref_put(&cmd->ref, cdx_mcdi_cmd_release);
+		completed = true;
+	} else if (rc == MC_CMD_ERR_QUEUE_FULL) {
+		cmd->state = MCDI_STATE_RETRY;
+	} else {
+		cmd->rc = rc;
+		cmd->outbuf = outbuf + DIV_ROUND_UP(resp_hdr_len, 4);
+		cmd->outlen = resp_data_len;
+		cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
+		completed = true;
+	}
+
+	/* free sequence number and buffer */
+	mcdi->seq_held_by[cmd->seq] = NULL;
+
+	cdx_mcdi_start_or_queue(mcdi, rc != MC_CMD_ERR_QUEUE_FULL);
+
+	/* wake up anyone waiting for flush */
+	wake_up(&mcdi->cmd_complete_wq);
+
+	kref_put(&cmd->ref, cdx_mcdi_cmd_release);
+
+	return completed;
+}
+
+static void cdx_mcdi_timeout_cmd(struct cdx_mcdi_iface *mcdi,
+				 struct cdx_mcdi_cmd *cmd,
+				 struct list_head *cleanup_list)
+{
+	struct cdx_mcdi *cdx = mcdi->cdx;
+
+	pr_err("MC command 0x%x inlen %zu state %d timed out after %u ms\n",
+	       cmd->cmd, cmd->inlen, cmd->state,
+	       jiffies_to_msecs(jiffies - cmd->started));
+
+	cmd->rc = -ETIMEDOUT;
+	cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
+
+	cdx_mcdi_mode_fail(cdx, cleanup_list);
+}
+
+/**
+ * cdx_mcdi_rpc - Issue an MCDI command and wait for completion
+ * @cdx: NIC through which to issue the command
+ * @cmd: Command type number
+ * @inbuf: Command parameters
+ * @inlen: Length of command parameters, in bytes. Must be a multiple
+ *	of 4 and no greater than %MCDI_CTL_SDU_LEN_MAX_V1.
+ * @outbuf: Response buffer. May be %NULL if @outlen is 0.
+ * @outlen: Length of response buffer, in bytes. If the actual
+ *	response is longer than @outlen & ~3, it will be truncated
+ *	to that length.
+ * @outlen_actual: Pointer through which to return the actual response
+ *	length. May be %NULL if this is not needed.
+ *
+ * This function may sleep and therefore must be called in process
+ * context.
+ *
+ * Return: A negative error code, or zero if successful. The error
+ *	code may come from the MCDI response or may indicate a failure
+ *	to communicate with the MC. In the former case, the response
+ *	will still be copied to @outbuf and *@outlen_actual will be
+ *	set accordingly. In the latter case, *@outlen_actual will be
+ *	set to zero.
+ */
+int cdx_mcdi_rpc(struct cdx_mcdi *cdx, unsigned int cmd,
+		 const struct cdx_dword *inbuf, size_t inlen,
+		 struct cdx_dword *outbuf, size_t outlen,
+		 size_t *outlen_actual)
+{
+	return cdx_mcdi_rpc_sync(cdx, cmd, inbuf, inlen, outbuf, outlen,
+				 outlen_actual, false);
+}
+
+/**
+ * cdx_mcdi_rpc_async - Schedule an MCDI command to run asynchronously
+ * @cdx: NIC through which to issue the command
+ * @cmd: Command type number
+ * @inbuf: Command parameters
+ * @inlen: Length of command parameters, in bytes
+ * @complete: Function to be called on completion or cancellation.
+ * @cookie: Arbitrary value to be passed to @complete.
+ *
+ * This function does not sleep and therefore may be called in atomic
+ * context.  It will fail if event queues are disabled or if MCDI
+ * event completions have been disabled due to an error.
+ *
+ * If it succeeds, the @complete function will be called exactly once
+ * in process context, when one of the following occurs:
+ * (a) the completion event is received (in process context)
+ * (b) event queues are disabled (in the process that disables them)
+ */
+int
+cdx_mcdi_rpc_async(struct cdx_mcdi *cdx, unsigned int cmd,
+		   const struct cdx_dword *inbuf, size_t inlen,
+		   cdx_mcdi_async_completer *complete, unsigned long cookie)
+{
+	struct cdx_mcdi_cmd *cmd_item =
+		kmalloc(sizeof(struct cdx_mcdi_cmd) + inlen, GFP_ATOMIC);
+
+	if (!cmd_item)
+		return -ENOMEM;
+
+	kref_init(&cmd_item->ref);
+	cmd_item->quiet = true;
+	cmd_item->cookie = cookie;
+	cmd_item->completer = complete;
+	cmd_item->cmd = cmd;
+	cmd_item->inlen = inlen;
+	/* inbuf is probably not valid after return, so take a copy */
+	cmd_item->inbuf = (struct cdx_dword *)(cmd_item + 1);
+	memcpy(cmd_item + 1, inbuf, inlen);
+
+	return cdx_mcdi_rpc_async_internal(cdx, cmd_item, NULL);
+}
+
+static void _cdx_mcdi_display_error(struct cdx_mcdi *cdx, unsigned int cmd,
+				    size_t inlen, int raw, int arg, int err_no)
+{
+	pr_err("MC command 0x%x inlen %d failed err_no=%d (raw=%d) arg=%d\n",
+	       cmd, (int)inlen, err_no, raw, arg);
+}
+
+/*
+ * Set MCDI mode to fail to prevent any new commands, then cancel any
+ * outstanding commands.
+ * Caller must hold the mcdi iface_lock.
+ */
+static void cdx_mcdi_mode_fail(struct cdx_mcdi *cdx, struct list_head *cleanup_list)
+{
+	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
+
+	if (!mcdi)
+		return;
+
+	mcdi->mode = MCDI_MODE_FAIL;
+
+	while (!list_empty(&mcdi->cmd_list)) {
+		struct cdx_mcdi_cmd *cmd;
+
+		cmd = list_first_entry(&mcdi->cmd_list, struct cdx_mcdi_cmd,
+				       list);
+		_cdx_mcdi_cancel_cmd(mcdi, cdx_mcdi_cmd_handle(cmd), cleanup_list);
+	}
+}
diff --git a/drivers/cdx/controller/mcdi.h b/drivers/cdx/controller/mcdi.h
new file mode 100644
index 000000000000..0bfbeab04e43
--- /dev/null
+++ b/drivers/cdx/controller/mcdi.h
@@ -0,0 +1,248 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2008-2013 Solarflare Communications Inc.
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#ifndef CDX_MCDI_H
+#define CDX_MCDI_H
+
+#include <linux/mutex.h>
+#include <linux/kref.h>
+#include <linux/rpmsg.h>
+
+#include "bitfield.h"
+#include "mc_cdx_pcol.h"
+
+#ifdef DEBUG
+#define CDX_WARN_ON_ONCE_PARANOID(x) WARN_ON_ONCE(x)
+#define CDX_WARN_ON_PARANOID(x) WARN_ON(x)
+#else
+#define CDX_WARN_ON_ONCE_PARANOID(x) do {} while (0)
+#define CDX_WARN_ON_PARANOID(x) do {} while (0)
+#endif
+
+/**
+ * enum cdx_mcdi_mode - MCDI transaction mode
+ * @MCDI_MODE_EVENTS: wait for an mcdi response callback.
+ * @MCDI_MODE_FAIL: we think MCDI is dead, so fail-fast all calls
+ */
+enum cdx_mcdi_mode {
+	MCDI_MODE_EVENTS,
+	MCDI_MODE_FAIL,
+};
+
+#define MCDI_RPC_TIMEOUT	(10 * HZ)
+#define MCDI_RPC_LONG_TIMEOU	(60 * HZ)
+#define MCDI_RPC_POST_RST_TIME	(10 * HZ)
+
+#define MCDI_BUF_LEN (8 + MCDI_CTL_SDU_LEN_MAX)
+
+/**
+ * enum cdx_mcdi_cmd_state - State for an individual MCDI command
+ * @MCDI_STATE_QUEUED: Command not started and is waiting to run.
+ * @MCDI_STATE_RETRY: Command was submitted and MC rejected with no resources,
+ *	as MC have too many outstanding commands. Command will be retried once
+ *	another command returns.
+ * @MCDI_STATE_RUNNING: Command was accepted and is running.
+ * @MCDI_STATE_RUNNING_CANCELLED: Command is running but the issuer cancelled
+ *	the command.
+ * @MCDI_STATE_FINISHED: Processing of this command has completed.
+ */
+
+enum cdx_mcdi_cmd_state {
+	MCDI_STATE_QUEUED,
+	MCDI_STATE_RETRY,
+	MCDI_STATE_RUNNING,
+	MCDI_STATE_RUNNING_CANCELLED,
+	MCDI_STATE_FINISHED,
+};
+
+/**
+ * struct cdx_mcdi - CDX MCDI Firmware interface, to interact
+ *	with CDX controller.
+ * @mcdi: MCDI interface
+ * @mcdi_ops: MCDI operations
+ * @r5_rproc : R5 Remoteproc device handle
+ * @rpdev: RPMsg device
+ * @ept: RPMsg endpoint
+ * @work: Post probe work
+ */
+struct cdx_mcdi {
+	/* MCDI interface */
+	struct cdx_mcdi_data *mcdi;
+	const struct cdx_mcdi_ops *mcdi_ops;
+
+	struct rproc *r5_rproc;
+	struct rpmsg_device *rpdev;
+	struct rpmsg_endpoint *ept;
+	struct work_struct work;
+};
+
+struct cdx_mcdi_ops {
+	void (*mcdi_request)(struct cdx_mcdi *cdx,
+			     const struct cdx_dword *hdr, size_t hdr_len,
+			     const struct cdx_dword *sdu, size_t sdu_len);
+	unsigned int (*mcdi_rpc_timeout)(struct cdx_mcdi *cdx, unsigned int cmd);
+};
+
+typedef void cdx_mcdi_async_completer(struct cdx_mcdi *cdx,
+				      unsigned long cookie, int rc,
+				      struct cdx_dword *outbuf,
+				      size_t outlen_actual);
+
+/**
+ * struct cdx_mcdi_cmd - An outstanding MCDI command
+ * @ref: Reference count. There will be one reference if the command is
+ *	in the mcdi_iface cmd_list, another if it's on a cleanup list,
+ *	and a third if it's queued in the work queue.
+ * @list: The data for this entry in mcdi->cmd_list
+ * @cleanup_list: The data for this entry in a cleanup list
+ * @work: The work item for this command, queued in mcdi->workqueue
+ * @mcdi: The mcdi_iface for this command
+ * @state: The state of this command
+ * @inlen: inbuf length
+ * @inbuf: Input buffer
+ * @quiet: Whether to silence errors
+ * @reboot_seen: Whether a reboot has been seen during this command,
+ *	to prevent duplicates
+ * @seq: Sequence number
+ * @started: Jiffies this command was started at
+ * @cookie: Context for completion function
+ * @completer: Completion function
+ * @handle: Command handle
+ * @cmd: Command number
+ * @rc: Return code
+ * @outlen: Length of output buffer
+ * @outbuf: Output buffer
+ */
+struct cdx_mcdi_cmd {
+	struct kref ref;
+	struct list_head list;
+	struct list_head cleanup_list;
+	struct work_struct work;
+	struct cdx_mcdi_iface *mcdi;
+	enum cdx_mcdi_cmd_state state;
+	size_t inlen;
+	const struct cdx_dword *inbuf;
+	bool quiet;
+	bool reboot_seen;
+	u8 seq;
+	unsigned long started;
+	unsigned long cookie;
+	cdx_mcdi_async_completer *completer;
+	unsigned int handle;
+	unsigned int cmd;
+	int rc;
+	size_t outlen;
+	struct cdx_dword *outbuf;
+	/* followed by inbuf data if necessary */
+};
+
+/**
+ * struct cdx_mcdi_iface - MCDI protocol context
+ * @cdx: The associated NIC
+ * @iface_lock: Serialise access to this structure
+ * @outstanding_cleanups: Count of cleanups
+ * @cmd_list: List of outstanding and running commands
+ * @workqueue: Workqueue used for delayed processing
+ * @cmd_complete_wq: Waitqueue for command completion
+ * @db_held_by: Command the MC doorbell is in use by
+ * @seq_held_by: Command each sequence number is in use by
+ * @prev_handle: The last used command handle
+ * @mode: Poll for mcdi completion, or wait for an mcdi_event
+ * @prev_seq: The last used sequence number
+ * @new_epoch: Indicates start of day or start of MC reboot recovery
+ * @logging_buffer: Buffer that may be used to build MCDI tracing messages
+ * @logging_enabled: Whether to trace MCDI
+ */
+struct cdx_mcdi_iface {
+	struct cdx_mcdi *cdx;
+	/* Serialise access */
+	struct mutex iface_lock;
+	unsigned int outstanding_cleanups;
+	struct list_head cmd_list;
+	struct workqueue_struct *workqueue;
+	wait_queue_head_t cmd_complete_wq;
+	struct cdx_mcdi_cmd *db_held_by;
+	struct cdx_mcdi_cmd *seq_held_by[16];
+	unsigned int prev_handle;
+	enum cdx_mcdi_mode mode;
+	u8 prev_seq;
+	bool new_epoch;
+#ifdef CONFIG_MCDI_LOGGING
+	bool logging_enabled;
+	char *logging_buffer;
+#endif
+};
+
+/**
+ * struct cdx_mcdi_data - extra state for NICs that implement MCDI
+ * @iface: Interface/protocol state
+ * @fn_flags: Flags for this function, as returned by %MC_CMD_DRV_ATTACH.
+ */
+struct cdx_mcdi_data {
+	struct cdx_mcdi_iface iface;
+	u32 fn_flags;
+};
+
+static inline struct cdx_mcdi_iface *cdx_mcdi_if(struct cdx_mcdi *cdx)
+{
+	return cdx->mcdi ? &cdx->mcdi->iface : NULL;
+}
+
+int cdx_mcdi_init(struct cdx_mcdi *cdx);
+void cdx_mcdi_finish(struct cdx_mcdi *cdx);
+
+void cdx_mcdi_process_cmd(struct cdx_mcdi *cdx, struct cdx_dword *outbuf, int len);
+int cdx_mcdi_rpc(struct cdx_mcdi *cdx, unsigned int cmd,
+		 const struct cdx_dword *inbuf, size_t inlen,
+		 struct cdx_dword *outbuf, size_t outlen, size_t *outlen_actual);
+int cdx_mcdi_rpc_async(struct cdx_mcdi *cdx, unsigned int cmd,
+		       const struct cdx_dword *inbuf, size_t inlen,
+		       cdx_mcdi_async_completer *complete,
+		       unsigned long cookie);
+int cdx_mcdi_wait_for_quiescence(struct cdx_mcdi *cdx,
+				 unsigned int timeout_jiffies);
+
+/*
+ * We expect that 16- and 32-bit fields in MCDI requests and responses
+ * are appropriately aligned, but 64-bit fields are only
+ * 32-bit-aligned.
+ */
+#define MCDI_DECLARE_BUF(_name, _len) struct cdx_dword _name[DIV_ROUND_UP(_len, 4)] = {{0}}
+#define _MCDI_PTR(_buf, _offset)					\
+	((u8 *)(_buf) + (_offset))
+#define MCDI_PTR(_buf, _field)						\
+	_MCDI_PTR(_buf, MC_CMD_ ## _field ## _OFST)
+#define _MCDI_CHECK_ALIGN(_ofst, _align)				\
+	((void)BUILD_BUG_ON_ZERO((_ofst) & ((_align) - 1)),		\
+	 (_ofst))
+#define _MCDI_DWORD(_buf, _field)					\
+	((_buf) + (_MCDI_CHECK_ALIGN(MC_CMD_ ## _field ## _OFST, 4) >> 2))
+
+#define MCDI_BYTE(_buf, _field)						\
+	((void)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 1),	\
+	 *MCDI_PTR(_buf, _field))
+#define MCDI_WORD(_buf, _field)						\
+	((void)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 2),	\
+	 le16_to_cpu(*(__force const __le16 *)MCDI_PTR(_buf, _field)))
+#define MCDI_SET_DWORD(_buf, _field, _value)				\
+	CDX_POPULATE_DWORD_1(*_MCDI_DWORD(_buf, _field), CDX_DWORD, _value)
+#define MCDI_DWORD(_buf, _field)					\
+	CDX_DWORD_FIELD(*_MCDI_DWORD(_buf, _field), CDX_DWORD)
+#define MCDI_POPULATE_DWORD_1(_buf, _field, _name1, _value1)		\
+	CDX_POPULATE_DWORD_1(*_MCDI_DWORD(_buf, _field),		\
+			     MC_CMD_ ## _name1, _value1)
+#define MCDI_SET_QWORD(_buf, _field, _value)				\
+	do {								\
+		CDX_POPULATE_DWORD_1(_MCDI_DWORD(_buf, _field)[0],	\
+				     CDX_DWORD, (u32)(_value));	\
+		CDX_POPULATE_DWORD_1(_MCDI_DWORD(_buf, _field)[1],	\
+				     CDX_DWORD, (u64)(_value) >> 32);	\
+	} while (0)
+#define MCDI_QWORD(_buf, _field)					\
+	(CDX_DWORD_FIELD(_MCDI_DWORD(_buf, _field)[0], CDX_DWORD) |	\
+	(u64)CDX_DWORD_FIELD(_MCDI_DWORD(_buf, _field)[1], CDX_DWORD) << 32)
+
+#endif /* CDX_MCDI_H */
diff --git a/drivers/cdx/controller/mcdi_functions.c b/drivers/cdx/controller/mcdi_functions.c
new file mode 100644
index 000000000000..0158f26533dd
--- /dev/null
+++ b/drivers/cdx/controller/mcdi_functions.c
@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/module.h>
+
+#include "mcdi.h"
+#include "mcdi_functions.h"
+
+int cdx_mcdi_get_num_buses(struct cdx_mcdi *cdx)
+{
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_CDX_BUS_ENUM_BUSES_OUT_LEN);
+	size_t outlen;
+	int ret;
+
+	ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_BUS_ENUM_BUSES, NULL, 0,
+			   outbuf, sizeof(outbuf), &outlen);
+	if (ret)
+		return ret;
+
+	if (outlen != MC_CMD_CDX_BUS_ENUM_BUSES_OUT_LEN)
+		return -EIO;
+
+	return MCDI_DWORD(outbuf, CDX_BUS_ENUM_BUSES_OUT_BUS_COUNT);
+}
+
+int cdx_mcdi_get_num_devs(struct cdx_mcdi *cdx, int bus_num)
+{
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_CDX_BUS_ENUM_DEVICES_OUT_LEN);
+	MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_BUS_ENUM_DEVICES_IN_LEN);
+	size_t outlen;
+	int ret;
+
+	MCDI_SET_DWORD(inbuf, CDX_BUS_ENUM_DEVICES_IN_BUS, bus_num);
+
+	ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_BUS_ENUM_DEVICES, inbuf, sizeof(inbuf),
+			   outbuf, sizeof(outbuf), &outlen);
+	if (ret)
+		return ret;
+
+	if (outlen != MC_CMD_CDX_BUS_ENUM_DEVICES_OUT_LEN)
+		return -EIO;
+
+	return MCDI_DWORD(outbuf, CDX_BUS_ENUM_DEVICES_OUT_DEVICE_COUNT);
+}
+
+int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx,
+			    u8 bus_num, u8 dev_num,
+			    struct cdx_dev_params *dev_params)
+{
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_LEN);
+	MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_IN_LEN);
+	struct resource *res = &dev_params->res[0];
+	size_t outlen;
+	u32 req_id;
+	int ret;
+
+	MCDI_SET_DWORD(inbuf, CDX_BUS_GET_DEVICE_CONFIG_IN_BUS, bus_num);
+	MCDI_SET_DWORD(inbuf, CDX_BUS_GET_DEVICE_CONFIG_IN_DEVICE, dev_num);
+
+	ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_BUS_GET_DEVICE_CONFIG, inbuf, sizeof(inbuf),
+			   outbuf, sizeof(outbuf), &outlen);
+	if (ret)
+		return ret;
+
+	if (outlen != MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_LEN)
+		return -EIO;
+
+	dev_params->bus_num = bus_num;
+	dev_params->dev_num = dev_num;
+
+	req_id = MCDI_DWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_REQUESTER_ID);
+	dev_params->req_id = req_id;
+
+	dev_params->res_count = 0;
+	if (MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE) != 0) {
+		res[dev_params->res_count].start =
+			MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE);
+		res[dev_params->res_count].end =
+			MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_BASE) +
+				   MCDI_QWORD(outbuf,
+					      CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE) - 1;
+		res[dev_params->res_count].flags = IORESOURCE_MEM;
+		dev_params->res_count++;
+	}
+
+	if (MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE) != 0) {
+		res[dev_params->res_count].start =
+			MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE);
+		res[dev_params->res_count].end =
+			MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_BASE) +
+				   MCDI_QWORD(outbuf,
+					      CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION1_SIZE) - 1;
+		res[dev_params->res_count].flags = IORESOURCE_MEM;
+		dev_params->res_count++;
+	}
+
+	if (MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE) != 0) {
+		res[dev_params->res_count].start =
+			MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE);
+		res[dev_params->res_count].end =
+			MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_BASE) +
+				   MCDI_QWORD(outbuf,
+					      CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION2_SIZE) - 1;
+		res[dev_params->res_count].flags = IORESOURCE_MEM;
+		dev_params->res_count++;
+	}
+
+	if (MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE) != 0) {
+		res[dev_params->res_count].start =
+			MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE);
+		res[dev_params->res_count].end =
+			MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_BASE) +
+				   MCDI_QWORD(outbuf,
+					      CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION3_SIZE) - 1;
+		res[dev_params->res_count].flags = IORESOURCE_MEM;
+		dev_params->res_count++;
+	}
+
+	dev_params->vendor = MCDI_WORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_VENDOR_ID);
+	dev_params->device = MCDI_WORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_DEVICE_ID);
+
+	return 0;
+}
+
+int cdx_mcdi_reset_device(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num)
+{
+	MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_DEVICE_RESET_IN_LEN);
+	int ret;
+
+	MCDI_SET_DWORD(inbuf, CDX_DEVICE_RESET_IN_BUS, bus_num);
+	MCDI_SET_DWORD(inbuf, CDX_DEVICE_RESET_IN_DEVICE, dev_num);
+
+	ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_DEVICE_RESET, inbuf, sizeof(inbuf),
+			   NULL, 0, NULL);
+
+	return ret;
+}
diff --git a/drivers/cdx/controller/mcdi_functions.h b/drivers/cdx/controller/mcdi_functions.h
new file mode 100644
index 000000000000..7440ace5539a
--- /dev/null
+++ b/drivers/cdx/controller/mcdi_functions.h
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Header file for MCDI FW interaction for CDX bus.
+ *
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#ifndef CDX_MCDI_FUNCTIONS_H
+#define CDX_MCDI_FUNCTIONS_H
+
+#include "mcdi.h"
+#include "../cdx.h"
+
+/**
+ * cdx_mcdi_get_num_buses - Get the total number of buses on
+ *	the controller.
+ * @cdx: pointer to MCDI interface.
+ *
+ * Return: total number of buses available on the controller,
+ *	<0 on failure
+ */
+int cdx_mcdi_get_num_buses(struct cdx_mcdi *cdx);
+
+/**
+ * cdx_mcdi_get_num_devs - Get the total number of devices on
+ *	a particular bus of the controller.
+ * @cdx: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ *
+ * Return: total number of devices available on the bus, <0 on failure
+ */
+int cdx_mcdi_get_num_devs(struct cdx_mcdi *cdx, int bus_num);
+
+/**
+ * cdx_mcdi_get_dev_config - Get configuration for a particular
+ *	bus_num:dev_num
+ * @cdx: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @dev_num: Device number.
+ * @dev_params: Pointer to cdx_dev_params, this is populated by this
+ *	device with the configuration corresponding to the provided
+ *	bus_num:dev_num.
+ *
+ * Return: 0 total number of devices available on the bus, <0 on failure
+ */
+int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx,
+			    u8 bus_num, u8 dev_num,
+			    struct cdx_dev_params *dev_params);
+
+/**
+ * cdx_mcdi_reset_device - Reset cdx device represented by bus_num:dev_num
+ * @cdx: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @dev_num: Device number.
+ *
+ * Return: 0 on success, <0 on failure
+ */
+int cdx_mcdi_reset_device(struct cdx_mcdi *cdx,
+			  u8 bus_num, u8 dev_num);
+
+#endif /* CDX_MCDI_FUNCTIONS_H */
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index 30fe9848dac1..801d6c83f896 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -247,8 +247,6 @@ config SONYPI
 	  To compile this driver as a module, choose M here: the
 	  module will be called sonypi.
 
-source "drivers/char/pcmcia/Kconfig"
-
 config MWAVE
 	tristate "ACP Modem (Mwave) support"
 	depends on X86 && TTY
diff --git a/drivers/char/Makefile b/drivers/char/Makefile
index 1b35d1724565..c5f532e412f1 100644
--- a/drivers/char/Makefile
+++ b/drivers/char/Makefile
@@ -35,7 +35,6 @@ obj-$(CONFIG_TELCLOCK)		+= tlclk.o
 
 obj-$(CONFIG_MWAVE)		+= mwave/
 obj-y				+= agp/
-obj-$(CONFIG_PCMCIA)		+= pcmcia/
 
 obj-$(CONFIG_HANGCHECK_TIMER)	+= hangcheck-timer.o
 obj-$(CONFIG_TCG_TPM)		+= tpm/
diff --git a/drivers/char/bsr.c b/drivers/char/bsr.c
index d5f943938427..ff429ba02fa4 100644
--- a/drivers/char/bsr.c
+++ b/drivers/char/bsr.c
@@ -293,7 +293,7 @@ static int __init bsr_init(void)
 	if (!np)
 		goto out_err;
 
-	bsr_class = class_create(THIS_MODULE, "bsr");
+	bsr_class = class_create("bsr");
 	if (IS_ERR(bsr_class)) {
 		printk(KERN_ERR "class_create() failed for bsr_class\n");
 		ret = PTR_ERR(bsr_class);
diff --git a/drivers/char/dsp56k.c b/drivers/char/dsp56k.c
index 06749e295ada..b3eaf3e5ef2e 100644
--- a/drivers/char/dsp56k.c
+++ b/drivers/char/dsp56k.c
@@ -504,7 +504,7 @@ static int __init dsp56k_init_driver(void)
 		printk("DSP56k driver: Unable to register driver\n");
 		return -ENODEV;
 	}
-	dsp56k_class = class_create(THIS_MODULE, "dsp56k");
+	dsp56k_class = class_create("dsp56k");
 	if (IS_ERR(dsp56k_class)) {
 		err = PTR_ERR(dsp56k_class);
 		goto out_chrdev;
diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c
index 8bb30282ca46..a4eb8e35f13d 100644
--- a/drivers/char/hw_random/meson-rng.c
+++ b/drivers/char/hw_random/meson-rng.c
@@ -18,9 +18,7 @@
 
 struct meson_rng_data {
 	void __iomem *base;
-	struct platform_device *pdev;
 	struct hwrng rng;
-	struct clk *core_clk;
 };
 
 static int meson_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait)
@@ -33,47 +31,28 @@ static int meson_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait)
 	return sizeof(u32);
 }
 
-static void meson_rng_clk_disable(void *data)
-{
-	clk_disable_unprepare(data);
-}
-
 static int meson_rng_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct meson_rng_data *data;
-	int ret;
+	struct clk *core_clk;
 
 	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
-	data->pdev = pdev;
-
 	data->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(data->base))
 		return PTR_ERR(data->base);
 
-	data->core_clk = devm_clk_get_optional(dev, "core");
-	if (IS_ERR(data->core_clk))
-		return dev_err_probe(dev, PTR_ERR(data->core_clk),
+	core_clk = devm_clk_get_optional_enabled(dev, "core");
+	if (IS_ERR(core_clk))
+		return dev_err_probe(dev, PTR_ERR(core_clk),
 				     "Failed to get core clock\n");
 
-	if (data->core_clk) {
-		ret = clk_prepare_enable(data->core_clk);
-		if (ret)
-			return ret;
-		ret = devm_add_action_or_reset(dev, meson_rng_clk_disable,
-					       data->core_clk);
-		if (ret)
-			return ret;
-	}
-
 	data->rng.name = pdev->name;
 	data->rng.read = meson_rng_read;
 
-	platform_set_drvdata(pdev, data);
-
 	return devm_hwrng_register(dev, &data->rng);
 }
 
diff --git a/drivers/char/hw_random/xgene-rng.c b/drivers/char/hw_random/xgene-rng.c
index 008e6db9ce01..7c8f3cb7c6af 100644
--- a/drivers/char/hw_random/xgene-rng.c
+++ b/drivers/char/hw_random/xgene-rng.c
@@ -84,7 +84,6 @@ struct xgene_rng_dev {
 	unsigned long failure_ts;/* First failure timestamp */
 	struct timer_list failure_timer;
 	struct device *dev;
-	struct clk *clk;
 };
 
 static void xgene_rng_expired_timer(struct timer_list *t)
@@ -200,7 +199,7 @@ static void xgene_rng_chk_overflow(struct xgene_rng_dev *ctx)
 
 static irqreturn_t xgene_rng_irq_handler(int irq, void *id)
 {
-	struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) id;
+	struct xgene_rng_dev *ctx = id;
 
 	/* RNG Alarm Counter overflow */
 	xgene_rng_chk_overflow(ctx);
@@ -314,6 +313,7 @@ static struct hwrng xgene_rng_func = {
 static int xgene_rng_probe(struct platform_device *pdev)
 {
 	struct xgene_rng_dev *ctx;
+	struct clk *clk;
 	int rc = 0;
 
 	ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL);
@@ -337,58 +337,36 @@ static int xgene_rng_probe(struct platform_device *pdev)
 
 	rc = devm_request_irq(&pdev->dev, ctx->irq, xgene_rng_irq_handler, 0,
 				dev_name(&pdev->dev), ctx);
-	if (rc) {
-		dev_err(&pdev->dev, "Could not request RNG alarm IRQ\n");
-		return rc;
-	}
+	if (rc)
+		return dev_err_probe(&pdev->dev, rc, "Could not request RNG alarm IRQ\n");
 
 	/* Enable IP clock */
-	ctx->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(ctx->clk)) {
-		dev_warn(&pdev->dev, "Couldn't get the clock for RNG\n");
-	} else {
-		rc = clk_prepare_enable(ctx->clk);
-		if (rc) {
-			dev_warn(&pdev->dev,
-				 "clock prepare enable failed for RNG");
-			return rc;
-		}
-	}
+	clk = devm_clk_get_optional_enabled(&pdev->dev, NULL);
+	if (IS_ERR(clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(clk), "Couldn't get the clock for RNG\n");
 
 	xgene_rng_func.priv = (unsigned long) ctx;
 
 	rc = devm_hwrng_register(&pdev->dev, &xgene_rng_func);
-	if (rc) {
-		dev_err(&pdev->dev, "RNG registering failed error %d\n", rc);
-		if (!IS_ERR(ctx->clk))
-			clk_disable_unprepare(ctx->clk);
-		return rc;
-	}
+	if (rc)
+		return dev_err_probe(&pdev->dev, rc, "RNG registering failed\n");
 
 	rc = device_init_wakeup(&pdev->dev, 1);
-	if (rc) {
-		dev_err(&pdev->dev, "RNG device_init_wakeup failed error %d\n",
-			rc);
-		if (!IS_ERR(ctx->clk))
-			clk_disable_unprepare(ctx->clk);
-		return rc;
-	}
+	if (rc)
+		return dev_err_probe(&pdev->dev, rc, "RNG device_init_wakeup failed\n");
 
 	return 0;
 }
 
 static int xgene_rng_remove(struct platform_device *pdev)
 {
-	struct xgene_rng_dev *ctx = platform_get_drvdata(pdev);
 	int rc;
 
 	rc = device_init_wakeup(&pdev->dev, 0);
 	if (rc)
 		dev_err(&pdev->dev, "RNG init wakeup failed error %d\n", rc);
-	if (!IS_ERR(ctx->clk))
-		clk_disable_unprepare(ctx->clk);
 
-	return rc;
+	return 0;
 }
 
 static const struct of_device_id xgene_rng_of_match[] = {
diff --git a/drivers/char/ipmi/Kconfig b/drivers/char/ipmi/Kconfig
index b6c0d35fc1a5..f4adc6feb3b2 100644
--- a/drivers/char/ipmi/Kconfig
+++ b/drivers/char/ipmi/Kconfig
@@ -162,7 +162,8 @@ config IPMI_KCS_BMC_SERIO
 
 config ASPEED_BT_IPMI_BMC
 	depends on ARCH_ASPEED || COMPILE_TEST
-	depends on REGMAP && REGMAP_MMIO && MFD_SYSCON
+	depends on MFD_SYSCON
+	select REGMAP_MMIO
 	tristate "BT IPMI bmc driver"
 	help
 	  Provides a driver for the BT (Block Transfer) IPMI interface
diff --git a/drivers/char/ipmi/ipmi_devintf.c b/drivers/char/ipmi/ipmi_devintf.c
index d160fa4c73fe..73e5a9e28f85 100644
--- a/drivers/char/ipmi/ipmi_devintf.c
+++ b/drivers/char/ipmi/ipmi_devintf.c
@@ -860,7 +860,7 @@ static int __init init_ipmi_devintf(void)
 
 	pr_info("ipmi device interface\n");
 
-	ipmi_class = class_create(THIS_MODULE, "ipmi");
+	ipmi_class = class_create("ipmi");
 	if (IS_ERR(ipmi_class)) {
 		pr_err("ipmi: can't register device class\n");
 		return PTR_ERR(ipmi_class);
diff --git a/drivers/char/ipmi/ipmi_poweroff.c b/drivers/char/ipmi/ipmi_poweroff.c
index 163ec9749e55..870659d91db2 100644
--- a/drivers/char/ipmi/ipmi_poweroff.c
+++ b/drivers/char/ipmi/ipmi_poweroff.c
@@ -659,20 +659,6 @@ static struct ctl_table ipmi_table[] = {
 	{ }
 };
 
-static struct ctl_table ipmi_dir_table[] = {
-	{ .procname	= "ipmi",
-	  .mode		= 0555,
-	  .child	= ipmi_table },
-	{ }
-};
-
-static struct ctl_table ipmi_root_table[] = {
-	{ .procname	= "dev",
-	  .mode		= 0555,
-	  .child	= ipmi_dir_table },
-	{ }
-};
-
 static struct ctl_table_header *ipmi_table_header;
 #endif /* CONFIG_PROC_FS */
 
@@ -689,7 +675,7 @@ static int __init ipmi_poweroff_init(void)
 		pr_info("Power cycle is enabled\n");
 
 #ifdef CONFIG_PROC_FS
-	ipmi_table_header = register_sysctl_table(ipmi_root_table);
+	ipmi_table_header = register_sysctl("dev/ipmi", ipmi_table);
 	if (!ipmi_table_header) {
 		pr_err("Unable to register powercycle sysctl\n");
 		rv = -ENOMEM;
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
index a5ddebb1edea..3b921c78ba08 100644
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -557,8 +557,10 @@ static void retry_timeout(struct timer_list *t)
 
 	if (waiting)
 		start_get(ssif_info);
-	if (resend)
+	if (resend) {
 		start_resend(ssif_info);
+		ssif_inc_stat(ssif_info, send_retries);
+	}
 }
 
 static void watch_timeout(struct timer_list *t)
@@ -784,9 +786,9 @@ static void msg_done_handler(struct ssif_info *ssif_info, int result,
 		} else if (data[0] != (IPMI_NETFN_APP_REQUEST | 1) << 2
 			   || data[1] != IPMI_GET_MSG_FLAGS_CMD) {
 			/*
-			 * Don't abort here, maybe it was a queued
-			 * response to a previous command.
+			 * Recv error response, give up.
 			 */
+			ssif_info->ssif_state = SSIF_IDLE;
 			ipmi_ssif_unlock_cond(ssif_info, flags);
 			dev_warn(&ssif_info->client->dev,
 				 "Invalid response getting flags: %x %x\n",
@@ -1279,11 +1281,8 @@ static void ssif_remove(struct i2c_client *client)
 	struct ssif_info *ssif_info = i2c_get_clientdata(client);
 	struct ssif_addr_info *addr_info;
 
-	if (!ssif_info)
-		return;
-
 	/*
-	 * After this point, we won't deliver anything asychronously
+	 * After this point, we won't deliver anything asynchronously
 	 * to the message handler.  We can unregister ourself.
 	 */
 	ipmi_unregister_smi(ssif_info->intf);
@@ -2071,9 +2070,6 @@ static int ssif_platform_remove(struct platform_device *dev)
 {
 	struct ssif_addr_info *addr_info = dev_get_drvdata(&dev->dev);
 
-	if (!addr_info)
-		return 0;
-
 	mutex_lock(&ssif_infos_mutex);
 	list_del(&addr_info->link);
 	kfree(addr_info);
diff --git a/drivers/char/lp.c b/drivers/char/lp.c
index 38aad99ebb61..70cfc5140c2c 100644
--- a/drivers/char/lp.c
+++ b/drivers/char/lp.c
@@ -1049,7 +1049,7 @@ static int __init lp_init(void)
 		return -EIO;
 	}
 
-	lp_class = class_create(THIS_MODULE, "printer");
+	lp_class = class_create("printer");
 	if (IS_ERR(lp_class)) {
 		err = PTR_ERR(lp_class);
 		goto out_reg;
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index ffb101d349f0..f494d31f2b98 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -762,7 +762,7 @@ static int __init chr_dev_init(void)
 	if (register_chrdev(MEM_MAJOR, "mem", &memory_fops))
 		printk("unable to get major %d for memory devs\n", MEM_MAJOR);
 
-	mem_class = class_create(THIS_MODULE, "mem");
+	mem_class = class_create("mem");
 	if (IS_ERR(mem_class))
 		return PTR_ERR(mem_class);
 
diff --git a/drivers/char/misc.c b/drivers/char/misc.c
index 7a1388b0572b..1c44c29a666e 100644
--- a/drivers/char/misc.c
+++ b/drivers/char/misc.c
@@ -286,7 +286,7 @@ static int __init misc_init(void)
 	struct proc_dir_entry *ret;
 
 	ret = proc_create_seq("misc", 0, NULL, &misc_seq_ops);
-	misc_class = class_create(THIS_MODULE, "misc");
+	misc_class = class_create("misc");
 	err = PTR_ERR(misc_class);
 	if (IS_ERR(misc_class))
 		goto fail_remove;
diff --git a/drivers/char/pcmcia/Kconfig b/drivers/char/pcmcia/Kconfig
deleted file mode 100644
index f5d589b2be44..000000000000
--- a/drivers/char/pcmcia/Kconfig
+++ /dev/null
@@ -1,68 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# PCMCIA character device configuration
-#
-
-menu "PCMCIA character devices"
-	depends on PCMCIA!=n
-
-config SYNCLINK_CS
-	tristate "SyncLink PC Card support"
-	depends on PCMCIA && TTY
-	help
-	  Enable support for the SyncLink PC Card serial adapter, running
-	  asynchronous and HDLC communications up to 512Kbps. The port is
-	  selectable for RS-232, V.35, RS-449, RS-530, and X.21
-
-	  This driver may be built as a module ( = code which can be
-	  inserted in and removed from the running kernel whenever you want).
-	  The module will be called synclink_cs.  If you want to do that, say M
-	  here.
-
-config CARDMAN_4000
-	tristate "Omnikey Cardman 4000 support"
-	depends on PCMCIA
-	select BITREVERSE
-	help
-	  Enable support for the Omnikey Cardman 4000 PCMCIA Smartcard
-	  reader.
-
-	  This kernel driver requires additional userspace support, either
-	  by the vendor-provided PC/SC ifd_handler (http://www.omnikey.com/),
-	  or via the cm4000 backend of OpenCT (http://www.opensc-project.org/opensc).
-
-config CARDMAN_4040
-	tristate "Omnikey CardMan 4040 support"
-	depends on PCMCIA
-	help
-	  Enable support for the Omnikey CardMan 4040 PCMCIA Smartcard
-	  reader.
-
-	  This card is basically a USB CCID device connected to a FIFO
-	  in I/O space.  To use the kernel driver, you will need either the
-	  PC/SC ifdhandler provided from the Omnikey homepage
-	  (http://www.omnikey.com/), or a current development version of OpenCT
-	  (http://www.opensc-project.org/opensc).
-
-config SCR24X
-	tristate "SCR24x Chip Card Interface support"
-	depends on PCMCIA
-	help
-	  Enable support for the SCR24x PCMCIA Chip Card Interface.
-
-	  To compile this driver as a module, choose M here.
-	  The module will be called scr24x_cs..
-
-	  If unsure say N.
-
-config IPWIRELESS
-	tristate "IPWireless 3G UMTS PCMCIA card support"
-	depends on PCMCIA && NETDEVICES && TTY
-	select PPP
-	help
-	  This is a driver for 3G UMTS PCMCIA card from IPWireless company. In
-	  some countries (for example Czech Republic, T-Mobile ISP) this card
-	  is shipped for service called UMTS 4G.
-
-endmenu
-
diff --git a/drivers/char/pcmcia/Makefile b/drivers/char/pcmcia/Makefile
deleted file mode 100644
index 024eed1c4ca5..000000000000
--- a/drivers/char/pcmcia/Makefile
+++ /dev/null
@@ -1,11 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# drivers/char/pcmcia/Makefile
-#
-# Makefile for the Linux PCMCIA char device drivers.
-#
-
-obj-$(CONFIG_SYNCLINK_CS) += synclink_cs.o
-obj-$(CONFIG_CARDMAN_4000) += cm4000_cs.o
-obj-$(CONFIG_CARDMAN_4040) += cm4040_cs.o
-obj-$(CONFIG_SCR24X) += scr24x_cs.o
diff --git a/drivers/char/pcmcia/cm4000_cs.c b/drivers/char/pcmcia/cm4000_cs.c
deleted file mode 100644
index e656f42a28ac..000000000000
--- a/drivers/char/pcmcia/cm4000_cs.c
+++ /dev/null
@@ -1,1912 +0,0 @@
- /*
-  * A driver for the PCMCIA Smartcard Reader "Omnikey CardMan Mobile 4000"
-  *
-  * cm4000_cs.c support.linux@omnikey.com
-  *
-  * Tue Oct 23 11:32:43 GMT 2001 herp - cleaned up header files
-  * Sun Jan 20 10:11:15 MET 2002 herp - added modversion header files
-  * Thu Nov 14 16:34:11 GMT 2002 mh   - added PPS functionality
-  * Tue Nov 19 16:36:27 GMT 2002 mh   - added SUSPEND/RESUME functionailty
-  * Wed Jul 28 12:55:01 CEST 2004 mh  - kernel 2.6 adjustments
-  *
-  * current version: 2.4.0gm4
-  *
-  * (C) 2000,2001,2002,2003,2004 Omnikey AG
-  *
-  * (C) 2005-2006 Harald Welte <laforge@gnumonks.org>
-  * 	- Adhere to Kernel process/coding-style.rst
-  * 	- Port to 2.6.13 "new" style PCMCIA
-  * 	- Check for copy_{from,to}_user return values
-  * 	- Use nonseekable_open()
-  * 	- add class interface for udev device creation
-  *
-  * All rights reserved. Licensed under dual BSD/GPL license.
-  */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/init.h>
-#include <linux/fs.h>
-#include <linux/delay.h>
-#include <linux/bitrev.h>
-#include <linux/mutex.h>
-#include <linux/uaccess.h>
-#include <linux/io.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/cisreg.h>
-#include <pcmcia/ciscode.h>
-#include <pcmcia/ds.h>
-
-#include <linux/cm4000_cs.h>
-
-/* #define ATR_CSUM */
-
-#define reader_to_dev(x)	(&x->p_dev->dev)
-
-/* n (debug level) is ignored */
-/* additional debug output may be enabled by re-compiling with
- * CM4000_DEBUG set */
-/* #define CM4000_DEBUG */
-#define DEBUGP(n, rdr, x, args...) do { 		\
-		dev_dbg(reader_to_dev(rdr), "%s:" x, 	\
-			   __func__ , ## args);		\
-	} while (0)
-
-static DEFINE_MUTEX(cmm_mutex);
-
-#define	T_1SEC		(HZ)
-#define	T_10MSEC	msecs_to_jiffies(10)
-#define	T_20MSEC	msecs_to_jiffies(20)
-#define	T_40MSEC	msecs_to_jiffies(40)
-#define	T_50MSEC	msecs_to_jiffies(50)
-#define	T_100MSEC	msecs_to_jiffies(100)
-#define	T_500MSEC	msecs_to_jiffies(500)
-
-static void cm4000_release(struct pcmcia_device *link);
-
-static int major;		/* major number we get from the kernel */
-
-/* note: the first state has to have number 0 always */
-
-#define	M_FETCH_ATR	0
-#define	M_TIMEOUT_WAIT	1
-#define	M_READ_ATR_LEN	2
-#define	M_READ_ATR	3
-#define	M_ATR_PRESENT	4
-#define	M_BAD_CARD	5
-#define M_CARDOFF	6
-
-#define	LOCK_IO			0
-#define	LOCK_MONITOR		1
-
-#define IS_AUTOPPS_ACT		 6
-#define	IS_PROCBYTE_PRESENT	 7
-#define	IS_INVREV		 8
-#define IS_ANY_T0		 9
-#define	IS_ANY_T1		10
-#define	IS_ATR_PRESENT		11
-#define	IS_ATR_VALID		12
-#define	IS_CMM_ABSENT		13
-#define	IS_BAD_LENGTH		14
-#define	IS_BAD_CSUM		15
-#define	IS_BAD_CARD		16
-
-#define REG_FLAGS0(x)		(x + 0)
-#define REG_FLAGS1(x)		(x + 1)
-#define REG_NUM_BYTES(x)	(x + 2)
-#define REG_BUF_ADDR(x)		(x + 3)
-#define REG_BUF_DATA(x)		(x + 4)
-#define REG_NUM_SEND(x)		(x + 5)
-#define REG_BAUDRATE(x)		(x + 6)
-#define REG_STOPBITS(x)		(x + 7)
-
-struct cm4000_dev {
-	struct pcmcia_device *p_dev;
-
-	unsigned char atr[MAX_ATR];
-	unsigned char rbuf[512];
-	unsigned char sbuf[512];
-
-	wait_queue_head_t devq;		/* when removing cardman must not be
-					   zeroed! */
-
-	wait_queue_head_t ioq;		/* if IO is locked, wait on this Q */
-	wait_queue_head_t atrq;		/* wait for ATR valid */
-	wait_queue_head_t readq;	/* used by write to wake blk.read */
-
-	/* warning: do not move this struct group.
-	 * initialising to zero depends on it - see ZERO_DEV below.  */
-	struct_group(init,
-	unsigned char atr_csum;
-	unsigned char atr_len_retry;
-	unsigned short atr_len;
-	unsigned short rlen;	/* bytes avail. after write */
-	unsigned short rpos;	/* latest read pos. write zeroes */
-	unsigned char procbyte;	/* T=0 procedure byte */
-	unsigned char mstate;	/* state of card monitor */
-	unsigned char cwarn;	/* slow down warning */
-	unsigned char flags0;	/* cardman IO-flags 0 */
-	unsigned char flags1;	/* cardman IO-flags 1 */
-	unsigned int mdelay;	/* variable monitor speeds, in jiffies */
-
-	unsigned int baudv;	/* baud value for speed */
-	unsigned char ta1;
-	unsigned char proto;	/* T=0, T=1, ... */
-	unsigned long flags;	/* lock+flags (MONITOR,IO,ATR) * for concurrent
-				   access */
-
-	unsigned char pts[4];
-
-	struct timer_list timer;	/* used to keep monitor running */
-	int monitor_running;
-	);
-};
-
-#define	ZERO_DEV(dev)	memset(&((dev)->init), 0, sizeof((dev)->init))
-
-static struct pcmcia_device *dev_table[CM4000_MAX_DEV];
-static struct class *cmm_class;
-
-/* This table doesn't use spaces after the comma between fields and thus
- * violates process/coding-style.rst.  However, I don't really think wrapping it around will
- * make it any clearer to read -HW */
-static unsigned char fi_di_table[10][14] = {
-/*FI     00   01   02   03   04   05   06   07   08   09   10   11   12   13 */
-/*DI */
-/* 0 */ {0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11},
-/* 1 */ {0x01,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x91,0x11,0x11,0x11,0x11},
-/* 2 */ {0x02,0x12,0x22,0x32,0x11,0x11,0x11,0x11,0x11,0x92,0xA2,0xB2,0x11,0x11},
-/* 3 */ {0x03,0x13,0x23,0x33,0x43,0x53,0x63,0x11,0x11,0x93,0xA3,0xB3,0xC3,0xD3},
-/* 4 */ {0x04,0x14,0x24,0x34,0x44,0x54,0x64,0x11,0x11,0x94,0xA4,0xB4,0xC4,0xD4},
-/* 5 */ {0x00,0x15,0x25,0x35,0x45,0x55,0x65,0x11,0x11,0x95,0xA5,0xB5,0xC5,0xD5},
-/* 6 */ {0x06,0x16,0x26,0x36,0x46,0x56,0x66,0x11,0x11,0x96,0xA6,0xB6,0xC6,0xD6},
-/* 7 */ {0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11},
-/* 8 */ {0x08,0x11,0x28,0x38,0x48,0x58,0x68,0x11,0x11,0x98,0xA8,0xB8,0xC8,0xD8},
-/* 9 */ {0x09,0x19,0x29,0x39,0x49,0x59,0x69,0x11,0x11,0x99,0xA9,0xB9,0xC9,0xD9}
-};
-
-#ifndef CM4000_DEBUG
-#define	xoutb	outb
-#define	xinb	inb
-#else
-static inline void xoutb(unsigned char val, unsigned short port)
-{
-	pr_debug("outb(val=%.2x,port=%.4x)\n", val, port);
-	outb(val, port);
-}
-static inline unsigned char xinb(unsigned short port)
-{
-	unsigned char val;
-
-	val = inb(port);
-	pr_debug("%.2x=inb(%.4x)\n", val, port);
-
-	return val;
-}
-#endif
-
-static inline unsigned char invert_revert(unsigned char ch)
-{
-	return bitrev8(~ch);
-}
-
-static void str_invert_revert(unsigned char *b, int len)
-{
-	int i;
-
-	for (i = 0; i < len; i++)
-		b[i] = invert_revert(b[i]);
-}
-
-#define	ATRLENCK(dev,pos) \
-	if (pos>=dev->atr_len || pos>=MAX_ATR) \
-		goto return_0;
-
-static unsigned int calc_baudv(unsigned char fidi)
-{
-	unsigned int wcrcf, wbrcf, fi_rfu, di_rfu;
-
-	fi_rfu = 372;
-	di_rfu = 1;
-
-	/* FI */
-	switch ((fidi >> 4) & 0x0F) {
-	case 0x00:
-		wcrcf = 372;
-		break;
-	case 0x01:
-		wcrcf = 372;
-		break;
-	case 0x02:
-		wcrcf = 558;
-		break;
-	case 0x03:
-		wcrcf = 744;
-		break;
-	case 0x04:
-		wcrcf = 1116;
-		break;
-	case 0x05:
-		wcrcf = 1488;
-		break;
-	case 0x06:
-		wcrcf = 1860;
-		break;
-	case 0x07:
-		wcrcf = fi_rfu;
-		break;
-	case 0x08:
-		wcrcf = fi_rfu;
-		break;
-	case 0x09:
-		wcrcf = 512;
-		break;
-	case 0x0A:
-		wcrcf = 768;
-		break;
-	case 0x0B:
-		wcrcf = 1024;
-		break;
-	case 0x0C:
-		wcrcf = 1536;
-		break;
-	case 0x0D:
-		wcrcf = 2048;
-		break;
-	default:
-		wcrcf = fi_rfu;
-		break;
-	}
-
-	/* DI */
-	switch (fidi & 0x0F) {
-	case 0x00:
-		wbrcf = di_rfu;
-		break;
-	case 0x01:
-		wbrcf = 1;
-		break;
-	case 0x02:
-		wbrcf = 2;
-		break;
-	case 0x03:
-		wbrcf = 4;
-		break;
-	case 0x04:
-		wbrcf = 8;
-		break;
-	case 0x05:
-		wbrcf = 16;
-		break;
-	case 0x06:
-		wbrcf = 32;
-		break;
-	case 0x07:
-		wbrcf = di_rfu;
-		break;
-	case 0x08:
-		wbrcf = 12;
-		break;
-	case 0x09:
-		wbrcf = 20;
-		break;
-	default:
-		wbrcf = di_rfu;
-		break;
-	}
-
-	return (wcrcf / wbrcf);
-}
-
-static unsigned short io_read_num_rec_bytes(unsigned int iobase,
-					    unsigned short *s)
-{
-	unsigned short tmp;
-
-	tmp = *s = 0;
-	do {
-		*s = tmp;
-		tmp = inb(REG_NUM_BYTES(iobase)) |
-				(inb(REG_FLAGS0(iobase)) & 4 ? 0x100 : 0);
-	} while (tmp != *s);
-
-	return *s;
-}
-
-static int parse_atr(struct cm4000_dev *dev)
-{
-	unsigned char any_t1, any_t0;
-	unsigned char ch, ifno;
-	int ix, done;
-
-	DEBUGP(3, dev, "-> parse_atr: dev->atr_len = %i\n", dev->atr_len);
-
-	if (dev->atr_len < 3) {
-		DEBUGP(5, dev, "parse_atr: atr_len < 3\n");
-		return 0;
-	}
-
-	if (dev->atr[0] == 0x3f)
-		set_bit(IS_INVREV, &dev->flags);
-	else
-		clear_bit(IS_INVREV, &dev->flags);
-	ix = 1;
-	ifno = 1;
-	ch = dev->atr[1];
-	dev->proto = 0;		/* XXX PROTO */
-	any_t1 = any_t0 = done = 0;
-	dev->ta1 = 0x11;	/* defaults to 9600 baud */
-	do {
-		if (ifno == 1 && (ch & 0x10)) {
-			/* read first interface byte and TA1 is present */
-			dev->ta1 = dev->atr[2];
-			DEBUGP(5, dev, "Card says FiDi is 0x%.2x\n", dev->ta1);
-			ifno++;
-		} else if ((ifno == 2) && (ch & 0x10)) { /* TA(2) */
-			dev->ta1 = 0x11;
-			ifno++;
-		}
-
-		DEBUGP(5, dev, "Yi=%.2x\n", ch & 0xf0);
-		ix += ((ch & 0x10) >> 4)	/* no of int.face chars */
-		    +((ch & 0x20) >> 5)
-		    + ((ch & 0x40) >> 6)
-		    + ((ch & 0x80) >> 7);
-		/* ATRLENCK(dev,ix); */
-		if (ch & 0x80) {	/* TDi */
-			ch = dev->atr[ix];
-			if ((ch & 0x0f)) {
-				any_t1 = 1;
-				DEBUGP(5, dev, "card is capable of T=1\n");
-			} else {
-				any_t0 = 1;
-				DEBUGP(5, dev, "card is capable of T=0\n");
-			}
-		} else
-			done = 1;
-	} while (!done);
-
-	DEBUGP(5, dev, "ix=%d noHist=%d any_t1=%d\n",
-	      ix, dev->atr[1] & 15, any_t1);
-	if (ix + 1 + (dev->atr[1] & 0x0f) + any_t1 != dev->atr_len) {
-		DEBUGP(5, dev, "length error\n");
-		return 0;
-	}
-	if (any_t0)
-		set_bit(IS_ANY_T0, &dev->flags);
-
-	if (any_t1) {		/* compute csum */
-		dev->atr_csum = 0;
-#ifdef ATR_CSUM
-		for (i = 1; i < dev->atr_len; i++)
-			dev->atr_csum ^= dev->atr[i];
-		if (dev->atr_csum) {
-			set_bit(IS_BAD_CSUM, &dev->flags);
-			DEBUGP(5, dev, "bad checksum\n");
-			goto return_0;
-		}
-#endif
-		if (any_t0 == 0)
-			dev->proto = 1;	/* XXX PROTO */
-		set_bit(IS_ANY_T1, &dev->flags);
-	}
-
-	return 1;
-}
-
-struct card_fixup {
-	char atr[12];
-	u_int8_t atr_len;
-	u_int8_t stopbits;
-};
-
-static struct card_fixup card_fixups[] = {
-	{	/* ACOS */
-		.atr = { 0x3b, 0xb3, 0x11, 0x00, 0x00, 0x41, 0x01 },
-		.atr_len = 7,
-		.stopbits = 0x03,
-	},
-	{	/* Motorola */
-		.atr = {0x3b, 0x76, 0x13, 0x00, 0x00, 0x80, 0x62, 0x07,
-			0x41, 0x81, 0x81 },
-		.atr_len = 11,
-		.stopbits = 0x04,
-	},
-};
-
-static void set_cardparameter(struct cm4000_dev *dev)
-{
-	int i;
-	unsigned int iobase = dev->p_dev->resource[0]->start;
-	u_int8_t stopbits = 0x02; /* ISO default */
-
-	DEBUGP(3, dev, "-> set_cardparameter\n");
-
-	dev->flags1 = dev->flags1 | (((dev->baudv - 1) & 0x0100) >> 8);
-	xoutb(dev->flags1, REG_FLAGS1(iobase));
-	DEBUGP(5, dev, "flags1 = 0x%02x\n", dev->flags1);
-
-	/* set baudrate */
-	xoutb((unsigned char)((dev->baudv - 1) & 0xFF), REG_BAUDRATE(iobase));
-
-	DEBUGP(5, dev, "baudv = %i -> write 0x%02x\n", dev->baudv,
-	      ((dev->baudv - 1) & 0xFF));
-
-	/* set stopbits */
-	for (i = 0; i < ARRAY_SIZE(card_fixups); i++) {
-		if (!memcmp(dev->atr, card_fixups[i].atr,
-			    card_fixups[i].atr_len))
-			stopbits = card_fixups[i].stopbits;
-	}
-	xoutb(stopbits, REG_STOPBITS(iobase));
-
-	DEBUGP(3, dev, "<- set_cardparameter\n");
-}
-
-static int set_protocol(struct cm4000_dev *dev, struct ptsreq *ptsreq)
-{
-
-	unsigned long tmp, i;
-	unsigned short num_bytes_read;
-	unsigned char pts_reply[4];
-	ssize_t rc;
-	unsigned int iobase = dev->p_dev->resource[0]->start;
-
-	rc = 0;
-
-	DEBUGP(3, dev, "-> set_protocol\n");
-	DEBUGP(5, dev, "ptsreq->Protocol = 0x%.8x, ptsreq->Flags=0x%.8x, "
-		 "ptsreq->pts1=0x%.2x, ptsreq->pts2=0x%.2x, "
-		 "ptsreq->pts3=0x%.2x\n", (unsigned int)ptsreq->protocol,
-		 (unsigned int)ptsreq->flags, ptsreq->pts1, ptsreq->pts2,
-		 ptsreq->pts3);
-
-	/* Fill PTS structure */
-	dev->pts[0] = 0xff;
-	dev->pts[1] = 0x00;
-	tmp = ptsreq->protocol;
-	while ((tmp = (tmp >> 1)) > 0)
-		dev->pts[1]++;
-	dev->proto = dev->pts[1];	/* Set new protocol */
-	dev->pts[1] = (0x01 << 4) | (dev->pts[1]);
-
-	/* Correct Fi/Di according to CM4000 Fi/Di table */
-	DEBUGP(5, dev, "Ta(1) from ATR is 0x%.2x\n", dev->ta1);
-	/* set Fi/Di according to ATR TA(1) */
-	dev->pts[2] = fi_di_table[dev->ta1 & 0x0F][(dev->ta1 >> 4) & 0x0F];
-
-	/* Calculate PCK character */
-	dev->pts[3] = dev->pts[0] ^ dev->pts[1] ^ dev->pts[2];
-
-	DEBUGP(5, dev, "pts0=%.2x, pts1=%.2x, pts2=%.2x, pts3=%.2x\n",
-	       dev->pts[0], dev->pts[1], dev->pts[2], dev->pts[3]);
-
-	/* check card convention */
-	if (test_bit(IS_INVREV, &dev->flags))
-		str_invert_revert(dev->pts, 4);
-
-	/* reset SM */
-	xoutb(0x80, REG_FLAGS0(iobase));
-
-	/* Enable access to the message buffer */
-	DEBUGP(5, dev, "Enable access to the messages buffer\n");
-	dev->flags1 = 0x20	/* T_Active */
-	    | (test_bit(IS_INVREV, &dev->flags) ? 0x02 : 0x00) /* inv parity */
-	    | ((dev->baudv >> 8) & 0x01);	/* MSB-baud */
-	xoutb(dev->flags1, REG_FLAGS1(iobase));
-
-	DEBUGP(5, dev, "Enable message buffer -> flags1 = 0x%.2x\n",
-	       dev->flags1);
-
-	/* write challenge to the buffer */
-	DEBUGP(5, dev, "Write challenge to buffer: ");
-	for (i = 0; i < 4; i++) {
-		xoutb(i, REG_BUF_ADDR(iobase));
-		xoutb(dev->pts[i], REG_BUF_DATA(iobase));	/* buf data */
-#ifdef CM4000_DEBUG
-		pr_debug("0x%.2x ", dev->pts[i]);
-	}
-	pr_debug("\n");
-#else
-	}
-#endif
-
-	/* set number of bytes to write */
-	DEBUGP(5, dev, "Set number of bytes to write\n");
-	xoutb(0x04, REG_NUM_SEND(iobase));
-
-	/* Trigger CARDMAN CONTROLLER */
-	xoutb(0x50, REG_FLAGS0(iobase));
-
-	/* Monitor progress */
-	/* wait for xmit done */
-	DEBUGP(5, dev, "Waiting for NumRecBytes getting valid\n");
-
-	for (i = 0; i < 100; i++) {
-		if (inb(REG_FLAGS0(iobase)) & 0x08) {
-			DEBUGP(5, dev, "NumRecBytes is valid\n");
-			break;
-		}
-		/* can not sleep as this is in atomic context */
-		mdelay(10);
-	}
-	if (i == 100) {
-		DEBUGP(5, dev, "Timeout waiting for NumRecBytes getting "
-		       "valid\n");
-		rc = -EIO;
-		goto exit_setprotocol;
-	}
-
-	DEBUGP(5, dev, "Reading NumRecBytes\n");
-	for (i = 0; i < 100; i++) {
-		io_read_num_rec_bytes(iobase, &num_bytes_read);
-		if (num_bytes_read >= 4) {
-			DEBUGP(2, dev, "NumRecBytes = %i\n", num_bytes_read);
-			if (num_bytes_read > 4) {
-				rc = -EIO;
-				goto exit_setprotocol;
-			}
-			break;
-		}
-		/* can not sleep as this is in atomic context */
-		mdelay(10);
-	}
-
-	/* check whether it is a short PTS reply? */
-	if (num_bytes_read == 3)
-		i = 0;
-
-	if (i == 100) {
-		DEBUGP(5, dev, "Timeout reading num_bytes_read\n");
-		rc = -EIO;
-		goto exit_setprotocol;
-	}
-
-	DEBUGP(5, dev, "Reset the CARDMAN CONTROLLER\n");
-	xoutb(0x80, REG_FLAGS0(iobase));
-
-	/* Read PPS reply */
-	DEBUGP(5, dev, "Read PPS reply\n");
-	for (i = 0; i < num_bytes_read; i++) {
-		xoutb(i, REG_BUF_ADDR(iobase));
-		pts_reply[i] = inb(REG_BUF_DATA(iobase));
-	}
-
-#ifdef CM4000_DEBUG
-	DEBUGP(2, dev, "PTSreply: ");
-	for (i = 0; i < num_bytes_read; i++) {
-		pr_debug("0x%.2x ", pts_reply[i]);
-	}
-	pr_debug("\n");
-#endif	/* CM4000_DEBUG */
-
-	DEBUGP(5, dev, "Clear Tactive in Flags1\n");
-	xoutb(0x20, REG_FLAGS1(iobase));
-
-	/* Compare ptsreq and ptsreply */
-	if ((dev->pts[0] == pts_reply[0]) &&
-	    (dev->pts[1] == pts_reply[1]) &&
-	    (dev->pts[2] == pts_reply[2]) && (dev->pts[3] == pts_reply[3])) {
-		/* setcardparameter according to PPS */
-		dev->baudv = calc_baudv(dev->pts[2]);
-		set_cardparameter(dev);
-	} else if ((dev->pts[0] == pts_reply[0]) &&
-		   ((dev->pts[1] & 0xef) == pts_reply[1]) &&
-		   ((pts_reply[0] ^ pts_reply[1]) == pts_reply[2])) {
-		/* short PTS reply, set card parameter to default values */
-		dev->baudv = calc_baudv(0x11);
-		set_cardparameter(dev);
-	} else
-		rc = -EIO;
-
-exit_setprotocol:
-	DEBUGP(3, dev, "<- set_protocol\n");
-	return rc;
-}
-
-static int io_detect_cm4000(unsigned int iobase, struct cm4000_dev *dev)
-{
-
-	/* note: statemachine is assumed to be reset */
-	if (inb(REG_FLAGS0(iobase)) & 8) {
-		clear_bit(IS_ATR_VALID, &dev->flags);
-		set_bit(IS_CMM_ABSENT, &dev->flags);
-		return 0;	/* detect CMM = 1 -> failure */
-	}
-	/* xoutb(0x40, REG_FLAGS1(iobase)); detectCMM */
-	xoutb(dev->flags1 | 0x40, REG_FLAGS1(iobase));
-	if ((inb(REG_FLAGS0(iobase)) & 8) == 0) {
-		clear_bit(IS_ATR_VALID, &dev->flags);
-		set_bit(IS_CMM_ABSENT, &dev->flags);
-		return 0;	/* detect CMM=0 -> failure */
-	}
-	/* clear detectCMM again by restoring original flags1 */
-	xoutb(dev->flags1, REG_FLAGS1(iobase));
-	return 1;
-}
-
-static void terminate_monitor(struct cm4000_dev *dev)
-{
-
-	/* tell the monitor to stop and wait until
-	 * it terminates.
-	 */
-	DEBUGP(3, dev, "-> terminate_monitor\n");
-	wait_event_interruptible(dev->devq,
-				 test_and_set_bit(LOCK_MONITOR,
-						  (void *)&dev->flags));
-
-	/* now, LOCK_MONITOR has been set.
-	 * allow a last cycle in the monitor.
-	 * the monitor will indicate that it has
-	 * finished by clearing this bit.
-	 */
-	DEBUGP(5, dev, "Now allow last cycle of monitor!\n");
-	while (test_bit(LOCK_MONITOR, (void *)&dev->flags))
-		msleep(25);
-
-	DEBUGP(5, dev, "Delete timer\n");
-	del_timer_sync(&dev->timer);
-#ifdef CM4000_DEBUG
-	dev->monitor_running = 0;
-#endif
-
-	DEBUGP(3, dev, "<- terminate_monitor\n");
-}
-
-/*
- * monitor the card every 50msec. as a side-effect, retrieve the
- * atr once a card is inserted. another side-effect of retrieving the
- * atr is that the card will be powered on, so there is no need to
- * power on the card explicitly from the application: the driver
- * is already doing that for you.
- */
-
-static void monitor_card(struct timer_list *t)
-{
-	struct cm4000_dev *dev = from_timer(dev, t, timer);
-	unsigned int iobase = dev->p_dev->resource[0]->start;
-	unsigned short s;
-	struct ptsreq ptsreq;
-	int i, atrc;
-
-	DEBUGP(7, dev, "->  monitor_card\n");
-
-	/* if someone has set the lock for us: we're done! */
-	if (test_and_set_bit(LOCK_MONITOR, &dev->flags)) {
-		DEBUGP(4, dev, "About to stop monitor\n");
-		/* no */
-		dev->rlen =
-		    dev->rpos =
-		    dev->atr_csum = dev->atr_len_retry = dev->cwarn = 0;
-		dev->mstate = M_FETCH_ATR;
-		clear_bit(LOCK_MONITOR, &dev->flags);
-		/* close et al. are sleeping on devq, so wake it */
-		wake_up_interruptible(&dev->devq);
-		DEBUGP(2, dev, "<- monitor_card (we are done now)\n");
-		return;
-	}
-
-	/* try to lock io: if it is already locked, just add another timer */
-	if (test_and_set_bit(LOCK_IO, (void *)&dev->flags)) {
-		DEBUGP(4, dev, "Couldn't get IO lock\n");
-		goto return_with_timer;
-	}
-
-	/* is a card/a reader inserted at all ? */
-	dev->flags0 = xinb(REG_FLAGS0(iobase));
-	DEBUGP(7, dev, "dev->flags0 = 0x%2x\n", dev->flags0);
-	DEBUGP(7, dev, "smartcard present: %s\n",
-	       dev->flags0 & 1 ? "yes" : "no");
-	DEBUGP(7, dev, "cardman present: %s\n",
-	       dev->flags0 == 0xff ? "no" : "yes");
-
-	if ((dev->flags0 & 1) == 0	/* no smartcard inserted */
-	    || dev->flags0 == 0xff) {	/* no cardman inserted */
-		/* no */
-		dev->rlen =
-		    dev->rpos =
-		    dev->atr_csum = dev->atr_len_retry = dev->cwarn = 0;
-		dev->mstate = M_FETCH_ATR;
-
-		dev->flags &= 0x000000ff; /* only keep IO and MONITOR locks */
-
-		if (dev->flags0 == 0xff) {
-			DEBUGP(4, dev, "set IS_CMM_ABSENT bit\n");
-			set_bit(IS_CMM_ABSENT, &dev->flags);
-		} else if (test_bit(IS_CMM_ABSENT, &dev->flags)) {
-			DEBUGP(4, dev, "clear IS_CMM_ABSENT bit "
-			       "(card is removed)\n");
-			clear_bit(IS_CMM_ABSENT, &dev->flags);
-		}
-
-		goto release_io;
-	} else if ((dev->flags0 & 1) && test_bit(IS_CMM_ABSENT, &dev->flags)) {
-		/* cardman and card present but cardman was absent before
-		 * (after suspend with inserted card) */
-		DEBUGP(4, dev, "clear IS_CMM_ABSENT bit (card is inserted)\n");
-		clear_bit(IS_CMM_ABSENT, &dev->flags);
-	}
-
-	if (test_bit(IS_ATR_VALID, &dev->flags) == 1) {
-		DEBUGP(7, dev, "believe ATR is already valid (do nothing)\n");
-		goto release_io;
-	}
-
-	switch (dev->mstate) {
-	case M_CARDOFF: {
-		unsigned char flags0;
-
-		DEBUGP(4, dev, "M_CARDOFF\n");
-		flags0 = inb(REG_FLAGS0(iobase));
-		if (flags0 & 0x02) {
-			/* wait until Flags0 indicate power is off */
-			dev->mdelay = T_10MSEC;
-		} else {
-			/* Flags0 indicate power off and no card inserted now;
-			 * Reset CARDMAN CONTROLLER */
-			xoutb(0x80, REG_FLAGS0(iobase));
-
-			/* prepare for fetching ATR again: after card off ATR
-			 * is read again automatically */
-			dev->rlen =
-			    dev->rpos =
-			    dev->atr_csum =
-			    dev->atr_len_retry = dev->cwarn = 0;
-			dev->mstate = M_FETCH_ATR;
-
-			/* minimal gap between CARDOFF and read ATR is 50msec */
-			dev->mdelay = T_50MSEC;
-		}
-		break;
-	}
-	case M_FETCH_ATR:
-		DEBUGP(4, dev, "M_FETCH_ATR\n");
-		xoutb(0x80, REG_FLAGS0(iobase));
-		DEBUGP(4, dev, "Reset BAUDV to 9600\n");
-		dev->baudv = 0x173;	/* 9600 */
-		xoutb(0x02, REG_STOPBITS(iobase));	/* stopbits=2 */
-		xoutb(0x73, REG_BAUDRATE(iobase));	/* baud value */
-		xoutb(0x21, REG_FLAGS1(iobase));	/* T_Active=1, baud
-							   value */
-		/* warm start vs. power on: */
-		xoutb(dev->flags0 & 2 ? 0x46 : 0x44, REG_FLAGS0(iobase));
-		dev->mdelay = T_40MSEC;
-		dev->mstate = M_TIMEOUT_WAIT;
-		break;
-	case M_TIMEOUT_WAIT:
-		DEBUGP(4, dev, "M_TIMEOUT_WAIT\n");
-		/* numRecBytes */
-		io_read_num_rec_bytes(iobase, &dev->atr_len);
-		dev->mdelay = T_10MSEC;
-		dev->mstate = M_READ_ATR_LEN;
-		break;
-	case M_READ_ATR_LEN:
-		DEBUGP(4, dev, "M_READ_ATR_LEN\n");
-		/* infinite loop possible, since there is no timeout */
-
-#define	MAX_ATR_LEN_RETRY	100
-
-		if (dev->atr_len == io_read_num_rec_bytes(iobase, &s)) {
-			if (dev->atr_len_retry++ >= MAX_ATR_LEN_RETRY) {					/* + XX msec */
-				dev->mdelay = T_10MSEC;
-				dev->mstate = M_READ_ATR;
-			}
-		} else {
-			dev->atr_len = s;
-			dev->atr_len_retry = 0;	/* set new timeout */
-		}
-
-		DEBUGP(4, dev, "Current ATR_LEN = %i\n", dev->atr_len);
-		break;
-	case M_READ_ATR:
-		DEBUGP(4, dev, "M_READ_ATR\n");
-		xoutb(0x80, REG_FLAGS0(iobase));	/* reset SM */
-		for (i = 0; i < dev->atr_len; i++) {
-			xoutb(i, REG_BUF_ADDR(iobase));
-			dev->atr[i] = inb(REG_BUF_DATA(iobase));
-		}
-		/* Deactivate T_Active flags */
-		DEBUGP(4, dev, "Deactivate T_Active flags\n");
-		dev->flags1 = 0x01;
-		xoutb(dev->flags1, REG_FLAGS1(iobase));
-
-		/* atr is present (which doesn't mean it's valid) */
-		set_bit(IS_ATR_PRESENT, &dev->flags);
-		if (dev->atr[0] == 0x03)
-			str_invert_revert(dev->atr, dev->atr_len);
-		atrc = parse_atr(dev);
-		if (atrc == 0) {	/* atr invalid */
-			dev->mdelay = 0;
-			dev->mstate = M_BAD_CARD;
-		} else {
-			dev->mdelay = T_50MSEC;
-			dev->mstate = M_ATR_PRESENT;
-			set_bit(IS_ATR_VALID, &dev->flags);
-		}
-
-		if (test_bit(IS_ATR_VALID, &dev->flags) == 1) {
-			DEBUGP(4, dev, "monitor_card: ATR valid\n");
- 			/* if ta1 == 0x11, no PPS necessary (default values) */
-			/* do not do PPS with multi protocol cards */
-			if ((test_bit(IS_AUTOPPS_ACT, &dev->flags) == 0) &&
-			    (dev->ta1 != 0x11) &&
-			    !(test_bit(IS_ANY_T0, &dev->flags) &&
-			    test_bit(IS_ANY_T1, &dev->flags))) {
-				DEBUGP(4, dev, "Perform AUTOPPS\n");
-				set_bit(IS_AUTOPPS_ACT, &dev->flags);
-				ptsreq.protocol = (0x01 << dev->proto);
-				ptsreq.flags = 0x01;
-				ptsreq.pts1 = 0x00;
-				ptsreq.pts2 = 0x00;
-				ptsreq.pts3 = 0x00;
-				if (set_protocol(dev, &ptsreq) == 0) {
-					DEBUGP(4, dev, "AUTOPPS ret SUCC\n");
-					clear_bit(IS_AUTOPPS_ACT, &dev->flags);
-					wake_up_interruptible(&dev->atrq);
-				} else {
-					DEBUGP(4, dev, "AUTOPPS failed: "
-					       "repower using defaults\n");
-					/* prepare for repowering  */
-					clear_bit(IS_ATR_PRESENT, &dev->flags);
-					clear_bit(IS_ATR_VALID, &dev->flags);
-					dev->rlen =
-					    dev->rpos =
-					    dev->atr_csum =
-					    dev->atr_len_retry = dev->cwarn = 0;
-					dev->mstate = M_FETCH_ATR;
-
-					dev->mdelay = T_50MSEC;
-				}
-			} else {
-				/* for cards which use slightly different
-				 * params (extra guard time) */
-				set_cardparameter(dev);
-				if (test_bit(IS_AUTOPPS_ACT, &dev->flags) == 1)
-					DEBUGP(4, dev, "AUTOPPS already active "
-					       "2nd try:use default values\n");
-				if (dev->ta1 == 0x11)
-					DEBUGP(4, dev, "No AUTOPPS necessary "
-					       "TA(1)==0x11\n");
-				if (test_bit(IS_ANY_T0, &dev->flags)
-				    && test_bit(IS_ANY_T1, &dev->flags))
-					DEBUGP(4, dev, "Do NOT perform AUTOPPS "
-					       "with multiprotocol cards\n");
-				clear_bit(IS_AUTOPPS_ACT, &dev->flags);
-				wake_up_interruptible(&dev->atrq);
-			}
-		} else {
-			DEBUGP(4, dev, "ATR invalid\n");
-			wake_up_interruptible(&dev->atrq);
-		}
-		break;
-	case M_BAD_CARD:
-		DEBUGP(4, dev, "M_BAD_CARD\n");
-		/* slow down warning, but prompt immediately after insertion */
-		if (dev->cwarn == 0 || dev->cwarn == 10) {
-			set_bit(IS_BAD_CARD, &dev->flags);
-			dev_warn(&dev->p_dev->dev, MODULE_NAME ": ");
-			if (test_bit(IS_BAD_CSUM, &dev->flags)) {
-				DEBUGP(4, dev, "ATR checksum (0x%.2x, should "
-				       "be zero) failed\n", dev->atr_csum);
-			}
-#ifdef CM4000_DEBUG
-			else if (test_bit(IS_BAD_LENGTH, &dev->flags)) {
-				DEBUGP(4, dev, "ATR length error\n");
-			} else {
-				DEBUGP(4, dev, "card damaged or wrong way "
-					"inserted\n");
-			}
-#endif
-			dev->cwarn = 0;
-			wake_up_interruptible(&dev->atrq);	/* wake open */
-		}
-		dev->cwarn++;
-		dev->mdelay = T_100MSEC;
-		dev->mstate = M_FETCH_ATR;
-		break;
-	default:
-		DEBUGP(7, dev, "Unknown action\n");
-		break;		/* nothing */
-	}
-
-release_io:
-	DEBUGP(7, dev, "release_io\n");
-	clear_bit(LOCK_IO, &dev->flags);
-	wake_up_interruptible(&dev->ioq);	/* whoever needs IO */
-
-return_with_timer:
-	DEBUGP(7, dev, "<- monitor_card (returns with timer)\n");
-	mod_timer(&dev->timer, jiffies + dev->mdelay);
-	clear_bit(LOCK_MONITOR, &dev->flags);
-}
-
-/* Interface to userland (file_operations) */
-
-static ssize_t cmm_read(struct file *filp, __user char *buf, size_t count,
-			loff_t *ppos)
-{
-	struct cm4000_dev *dev = filp->private_data;
-	unsigned int iobase = dev->p_dev->resource[0]->start;
-	ssize_t rc;
-	int i, j, k;
-
-	DEBUGP(2, dev, "-> cmm_read(%s,%d)\n", current->comm, current->pid);
-
-	if (count == 0)		/* according to manpage */
-		return 0;
-
-	if (!pcmcia_dev_present(dev->p_dev) || /* device removed */
-	    test_bit(IS_CMM_ABSENT, &dev->flags))
-		return -ENODEV;
-
-	if (test_bit(IS_BAD_CSUM, &dev->flags))
-		return -EIO;
-
-	/* also see the note about this in cmm_write */
-	if (wait_event_interruptible
-	    (dev->atrq,
-	     ((filp->f_flags & O_NONBLOCK)
-	      || (test_bit(IS_ATR_PRESENT, (void *)&dev->flags) != 0)))) {
-		if (filp->f_flags & O_NONBLOCK)
-			return -EAGAIN;
-		return -ERESTARTSYS;
-	}
-
-	if (test_bit(IS_ATR_VALID, &dev->flags) == 0)
-		return -EIO;
-
-	/* this one implements blocking IO */
-	if (wait_event_interruptible
-	    (dev->readq,
-	     ((filp->f_flags & O_NONBLOCK) || (dev->rpos < dev->rlen)))) {
-		if (filp->f_flags & O_NONBLOCK)
-			return -EAGAIN;
-		return -ERESTARTSYS;
-	}
-
-	/* lock io */
-	if (wait_event_interruptible
-	    (dev->ioq,
-	     ((filp->f_flags & O_NONBLOCK)
-	      || (test_and_set_bit(LOCK_IO, (void *)&dev->flags) == 0)))) {
-		if (filp->f_flags & O_NONBLOCK)
-			return -EAGAIN;
-		return -ERESTARTSYS;
-	}
-
-	rc = 0;
-	dev->flags0 = inb(REG_FLAGS0(iobase));
-	if ((dev->flags0 & 1) == 0	/* no smartcard inserted */
-	    || dev->flags0 == 0xff) {	/* no cardman inserted */
-		clear_bit(IS_ATR_VALID, &dev->flags);
-		if (dev->flags0 & 1) {
-			set_bit(IS_CMM_ABSENT, &dev->flags);
-			rc = -ENODEV;
-		} else {
-			rc = -EIO;
-		}
-		goto release_io;
-	}
-
-	DEBUGP(4, dev, "begin read answer\n");
-	j = min(count, (size_t)(dev->rlen - dev->rpos));
-	k = dev->rpos;
-	if (k + j > 255)
-		j = 256 - k;
-	DEBUGP(4, dev, "read1 j=%d\n", j);
-	for (i = 0; i < j; i++) {
-		xoutb(k++, REG_BUF_ADDR(iobase));
-		dev->rbuf[i] = xinb(REG_BUF_DATA(iobase));
-	}
-	j = min(count, (size_t)(dev->rlen - dev->rpos));
-	if (k + j > 255) {
-		DEBUGP(4, dev, "read2 j=%d\n", j);
-		dev->flags1 |= 0x10;	/* MSB buf addr set */
-		xoutb(dev->flags1, REG_FLAGS1(iobase));
-		for (; i < j; i++) {
-			xoutb(k++, REG_BUF_ADDR(iobase));
-			dev->rbuf[i] = xinb(REG_BUF_DATA(iobase));
-		}
-	}
-
-	if (dev->proto == 0 && count > dev->rlen - dev->rpos && i) {
-		DEBUGP(4, dev, "T=0 and count > buffer\n");
-		dev->rbuf[i] = dev->rbuf[i - 1];
-		dev->rbuf[i - 1] = dev->procbyte;
-		j++;
-	}
-	count = j;
-
-	dev->rpos = dev->rlen + 1;
-
-	/* Clear T1Active */
-	DEBUGP(4, dev, "Clear T1Active\n");
-	dev->flags1 &= 0xdf;
-	xoutb(dev->flags1, REG_FLAGS1(iobase));
-
-	xoutb(0, REG_FLAGS1(iobase));	/* clear detectCMM */
-	/* last check before exit */
-	if (!io_detect_cm4000(iobase, dev)) {
-		rc = -ENODEV;
-		goto release_io;
-	}
-
-	if (test_bit(IS_INVREV, &dev->flags) && count > 0)
-		str_invert_revert(dev->rbuf, count);
-
-	if (copy_to_user(buf, dev->rbuf, count))
-		rc = -EFAULT;
-
-release_io:
-	clear_bit(LOCK_IO, &dev->flags);
-	wake_up_interruptible(&dev->ioq);
-
-	DEBUGP(2, dev, "<- cmm_read returns: rc = %zi\n",
-	       (rc < 0 ? rc : count));
-	return rc < 0 ? rc : count;
-}
-
-static ssize_t cmm_write(struct file *filp, const char __user *buf,
-			 size_t count, loff_t *ppos)
-{
-	struct cm4000_dev *dev = filp->private_data;
-	unsigned int iobase = dev->p_dev->resource[0]->start;
-	unsigned short s;
-	unsigned char infolen;
-	unsigned char sendT0;
-	unsigned short nsend;
-	unsigned short nr;
-	ssize_t rc;
-	int i;
-
-	DEBUGP(2, dev, "-> cmm_write(%s,%d)\n", current->comm, current->pid);
-
-	if (count == 0)		/* according to manpage */
-		return 0;
-
-	if (dev->proto == 0 && count < 4) {
-		/* T0 must have at least 4 bytes */
-		DEBUGP(4, dev, "T0 short write\n");
-		return -EIO;
-	}
-
-	nr = count & 0x1ff;	/* max bytes to write */
-
-	sendT0 = dev->proto ? 0 : nr > 5 ? 0x08 : 0;
-
-	if (!pcmcia_dev_present(dev->p_dev) || /* device removed */
-	    test_bit(IS_CMM_ABSENT, &dev->flags))
-		return -ENODEV;
-
-	if (test_bit(IS_BAD_CSUM, &dev->flags)) {
-		DEBUGP(4, dev, "bad csum\n");
-		return -EIO;
-	}
-
-	/*
-	 * wait for atr to become valid.
-	 * note: it is important to lock this code. if we dont, the monitor
-	 * could be run between test_bit and the call to sleep on the
-	 * atr-queue.  if *then* the monitor detects atr valid, it will wake up
-	 * any process on the atr-queue, *but* since we have been interrupted,
-	 * we do not yet sleep on this queue. this would result in a missed
-	 * wake_up and the calling process would sleep forever (until
-	 * interrupted).  also, do *not* restore_flags before sleep_on, because
-	 * this could result in the same situation!
-	 */
-	if (wait_event_interruptible
-	    (dev->atrq,
-	     ((filp->f_flags & O_NONBLOCK)
-	      || (test_bit(IS_ATR_PRESENT, (void *)&dev->flags) != 0)))) {
-		if (filp->f_flags & O_NONBLOCK)
-			return -EAGAIN;
-		return -ERESTARTSYS;
-	}
-
-	if (test_bit(IS_ATR_VALID, &dev->flags) == 0) {	/* invalid atr */
-		DEBUGP(4, dev, "invalid ATR\n");
-		return -EIO;
-	}
-
-	/* lock io */
-	if (wait_event_interruptible
-	    (dev->ioq,
-	     ((filp->f_flags & O_NONBLOCK)
-	      || (test_and_set_bit(LOCK_IO, (void *)&dev->flags) == 0)))) {
-		if (filp->f_flags & O_NONBLOCK)
-			return -EAGAIN;
-		return -ERESTARTSYS;
-	}
-
-	if (copy_from_user(dev->sbuf, buf, ((count > 512) ? 512 : count)))
-		return -EFAULT;
-
-	rc = 0;
-	dev->flags0 = inb(REG_FLAGS0(iobase));
-	if ((dev->flags0 & 1) == 0	/* no smartcard inserted */
-	    || dev->flags0 == 0xff) {	/* no cardman inserted */
-		clear_bit(IS_ATR_VALID, &dev->flags);
-		if (dev->flags0 & 1) {
-			set_bit(IS_CMM_ABSENT, &dev->flags);
-			rc = -ENODEV;
-		} else {
-			DEBUGP(4, dev, "IO error\n");
-			rc = -EIO;
-		}
-		goto release_io;
-	}
-
-	xoutb(0x80, REG_FLAGS0(iobase));	/* reset SM  */
-
-	if (!io_detect_cm4000(iobase, dev)) {
-		rc = -ENODEV;
-		goto release_io;
-	}
-
-	/* reflect T=0 send/read mode in flags1 */
-	dev->flags1 |= (sendT0);
-
-	set_cardparameter(dev);
-
-	/* dummy read, reset flag procedure received */
-	inb(REG_FLAGS1(iobase));
-
-	dev->flags1 = 0x20	/* T_Active */
-	    | (sendT0)
-	    | (test_bit(IS_INVREV, &dev->flags) ? 2 : 0)/* inverse parity  */
-	    | (((dev->baudv - 1) & 0x0100) >> 8);	/* MSB-Baud */
-	DEBUGP(1, dev, "set dev->flags1 = 0x%.2x\n", dev->flags1);
-	xoutb(dev->flags1, REG_FLAGS1(iobase));
-
-	/* xmit data */
-	DEBUGP(4, dev, "Xmit data\n");
-	for (i = 0; i < nr; i++) {
-		if (i >= 256) {
-			dev->flags1 = 0x20	/* T_Active */
-			    | (sendT0)	/* SendT0 */
-				/* inverse parity: */
-			    | (test_bit(IS_INVREV, &dev->flags) ? 2 : 0)
-			    | (((dev->baudv - 1) & 0x0100) >> 8) /* MSB-Baud */
-			    | 0x10;	/* set address high */
-			DEBUGP(4, dev, "dev->flags = 0x%.2x - set address "
-			       "high\n", dev->flags1);
-			xoutb(dev->flags1, REG_FLAGS1(iobase));
-		}
-		if (test_bit(IS_INVREV, &dev->flags)) {
-			DEBUGP(4, dev, "Apply inverse convention for 0x%.2x "
-				"-> 0x%.2x\n", (unsigned char)dev->sbuf[i],
-			      invert_revert(dev->sbuf[i]));
-			xoutb(i, REG_BUF_ADDR(iobase));
-			xoutb(invert_revert(dev->sbuf[i]),
-			      REG_BUF_DATA(iobase));
-		} else {
-			xoutb(i, REG_BUF_ADDR(iobase));
-			xoutb(dev->sbuf[i], REG_BUF_DATA(iobase));
-		}
-	}
-	DEBUGP(4, dev, "Xmit done\n");
-
-	if (dev->proto == 0) {
-		/* T=0 proto: 0 byte reply  */
-		if (nr == 4) {
-			DEBUGP(4, dev, "T=0 assumes 0 byte reply\n");
-			xoutb(i, REG_BUF_ADDR(iobase));
-			if (test_bit(IS_INVREV, &dev->flags))
-				xoutb(0xff, REG_BUF_DATA(iobase));
-			else
-				xoutb(0x00, REG_BUF_DATA(iobase));
-		}
-
-		/* numSendBytes */
-		if (sendT0)
-			nsend = nr;
-		else {
-			if (nr == 4)
-				nsend = 5;
-			else {
-				nsend = 5 + (unsigned char)dev->sbuf[4];
-				if (dev->sbuf[4] == 0)
-					nsend += 0x100;
-			}
-		}
-	} else
-		nsend = nr;
-
-	/* T0: output procedure byte */
-	if (test_bit(IS_INVREV, &dev->flags)) {
-		DEBUGP(4, dev, "T=0 set Procedure byte (inverse-reverse) "
-		       "0x%.2x\n", invert_revert(dev->sbuf[1]));
-		xoutb(invert_revert(dev->sbuf[1]), REG_NUM_BYTES(iobase));
-	} else {
-		DEBUGP(4, dev, "T=0 set Procedure byte 0x%.2x\n", dev->sbuf[1]);
-		xoutb(dev->sbuf[1], REG_NUM_BYTES(iobase));
-	}
-
-	DEBUGP(1, dev, "set NumSendBytes = 0x%.2x\n",
-	       (unsigned char)(nsend & 0xff));
-	xoutb((unsigned char)(nsend & 0xff), REG_NUM_SEND(iobase));
-
-	DEBUGP(1, dev, "Trigger CARDMAN CONTROLLER (0x%.2x)\n",
-	       0x40	/* SM_Active */
-	      | (dev->flags0 & 2 ? 0 : 4)	/* power on if needed */
-	      |(dev->proto ? 0x10 : 0x08)	/* T=1/T=0 */
-	      |(nsend & 0x100) >> 8 /* MSB numSendBytes */ );
-	xoutb(0x40		/* SM_Active */
-	      | (dev->flags0 & 2 ? 0 : 4)	/* power on if needed */
-	      |(dev->proto ? 0x10 : 0x08)	/* T=1/T=0 */
-	      |(nsend & 0x100) >> 8,	/* MSB numSendBytes */
-	      REG_FLAGS0(iobase));
-
-	/* wait for xmit done */
-	if (dev->proto == 1) {
-		DEBUGP(4, dev, "Wait for xmit done\n");
-		for (i = 0; i < 1000; i++) {
-			if (inb(REG_FLAGS0(iobase)) & 0x08)
-				break;
-			msleep_interruptible(10);
-		}
-		if (i == 1000) {
-			DEBUGP(4, dev, "timeout waiting for xmit done\n");
-			rc = -EIO;
-			goto release_io;
-		}
-	}
-
-	/* T=1: wait for infoLen */
-
-	infolen = 0;
-	if (dev->proto) {
-		/* wait until infoLen is valid */
-		for (i = 0; i < 6000; i++) {	/* max waiting time of 1 min */
-			io_read_num_rec_bytes(iobase, &s);
-			if (s >= 3) {
-				infolen = inb(REG_FLAGS1(iobase));
-				DEBUGP(4, dev, "infolen=%d\n", infolen);
-				break;
-			}
-			msleep_interruptible(10);
-		}
-		if (i == 6000) {
-			DEBUGP(4, dev, "timeout waiting for infoLen\n");
-			rc = -EIO;
-			goto release_io;
-		}
-	} else
-		clear_bit(IS_PROCBYTE_PRESENT, &dev->flags);
-
-	/* numRecBytes | bit9 of numRecytes */
-	io_read_num_rec_bytes(iobase, &dev->rlen);
-	for (i = 0; i < 600; i++) {	/* max waiting time of 2 sec */
-		if (dev->proto) {
-			if (dev->rlen >= infolen + 4)
-				break;
-		}
-		msleep_interruptible(10);
-		/* numRecBytes | bit9 of numRecytes */
-		io_read_num_rec_bytes(iobase, &s);
-		if (s > dev->rlen) {
-			DEBUGP(1, dev, "NumRecBytes inc (reset timeout)\n");
-			i = 0;	/* reset timeout */
-			dev->rlen = s;
-		}
-		/* T=0: we are done when numRecBytes doesn't
-		 *      increment any more and NoProcedureByte
-		 *      is set and numRecBytes == bytes sent + 6
-		 *      (header bytes + data + 1 for sw2)
-		 *      except when the card replies an error
-		 *      which means, no data will be sent back.
-		 */
-		else if (dev->proto == 0) {
-			if ((inb(REG_BUF_ADDR(iobase)) & 0x80)) {
-				/* no procedure byte received since last read */
-				DEBUGP(1, dev, "NoProcedure byte set\n");
-				/* i=0; */
-			} else {
-				/* procedure byte received since last read */
-				DEBUGP(1, dev, "NoProcedure byte unset "
-					"(reset timeout)\n");
-				dev->procbyte = inb(REG_FLAGS1(iobase));
-				DEBUGP(1, dev, "Read procedure byte 0x%.2x\n",
-				      dev->procbyte);
-				i = 0;	/* resettimeout */
-			}
-			if (inb(REG_FLAGS0(iobase)) & 0x08) {
-				DEBUGP(1, dev, "T0Done flag (read reply)\n");
-				break;
-			}
-		}
-		if (dev->proto)
-			infolen = inb(REG_FLAGS1(iobase));
-	}
-	if (i == 600) {
-		DEBUGP(1, dev, "timeout waiting for numRecBytes\n");
-		rc = -EIO;
-		goto release_io;
-	} else {
-		if (dev->proto == 0) {
-			DEBUGP(1, dev, "Wait for T0Done bit to be  set\n");
-			for (i = 0; i < 1000; i++) {
-				if (inb(REG_FLAGS0(iobase)) & 0x08)
-					break;
-				msleep_interruptible(10);
-			}
-			if (i == 1000) {
-				DEBUGP(1, dev, "timeout waiting for T0Done\n");
-				rc = -EIO;
-				goto release_io;
-			}
-
-			dev->procbyte = inb(REG_FLAGS1(iobase));
-			DEBUGP(4, dev, "Read procedure byte 0x%.2x\n",
-			      dev->procbyte);
-
-			io_read_num_rec_bytes(iobase, &dev->rlen);
-			DEBUGP(4, dev, "Read NumRecBytes = %i\n", dev->rlen);
-
-		}
-	}
-	/* T=1: read offset=zero, T=0: read offset=after challenge */
-	dev->rpos = dev->proto ? 0 : nr == 4 ? 5 : nr > dev->rlen ? 5 : nr;
-	DEBUGP(4, dev, "dev->rlen = %i,  dev->rpos = %i, nr = %i\n",
-	      dev->rlen, dev->rpos, nr);
-
-release_io:
-	DEBUGP(4, dev, "Reset SM\n");
-	xoutb(0x80, REG_FLAGS0(iobase));	/* reset SM */
-
-	if (rc < 0) {
-		DEBUGP(4, dev, "Write failed but clear T_Active\n");
-		dev->flags1 &= 0xdf;
-		xoutb(dev->flags1, REG_FLAGS1(iobase));
-	}
-
-	clear_bit(LOCK_IO, &dev->flags);
-	wake_up_interruptible(&dev->ioq);
-	wake_up_interruptible(&dev->readq);	/* tell read we have data */
-
-	/* ITSEC E2: clear write buffer */
-	memset((char *)dev->sbuf, 0, 512);
-
-	/* return error or actually written bytes */
-	DEBUGP(2, dev, "<- cmm_write\n");
-	return rc < 0 ? rc : nr;
-}
-
-static void start_monitor(struct cm4000_dev *dev)
-{
-	DEBUGP(3, dev, "-> start_monitor\n");
-	if (!dev->monitor_running) {
-		DEBUGP(5, dev, "create, init and add timer\n");
-		timer_setup(&dev->timer, monitor_card, 0);
-		dev->monitor_running = 1;
-		mod_timer(&dev->timer, jiffies);
-	} else
-		DEBUGP(5, dev, "monitor already running\n");
-	DEBUGP(3, dev, "<- start_monitor\n");
-}
-
-static void stop_monitor(struct cm4000_dev *dev)
-{
-	DEBUGP(3, dev, "-> stop_monitor\n");
-	if (dev->monitor_running) {
-		DEBUGP(5, dev, "stopping monitor\n");
-		terminate_monitor(dev);
-		/* reset monitor SM */
-		clear_bit(IS_ATR_VALID, &dev->flags);
-		clear_bit(IS_ATR_PRESENT, &dev->flags);
-	} else
-		DEBUGP(5, dev, "monitor already stopped\n");
-	DEBUGP(3, dev, "<- stop_monitor\n");
-}
-
-static long cmm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
-{
-	struct cm4000_dev *dev = filp->private_data;
-	unsigned int iobase = dev->p_dev->resource[0]->start;
-	struct inode *inode = file_inode(filp);
-	struct pcmcia_device *link;
-	int rc;
-	void __user *argp = (void __user *)arg;
-#ifdef CM4000_DEBUG
-	char *ioctl_names[CM_IOC_MAXNR + 1] = {
-		[_IOC_NR(CM_IOCGSTATUS)] "CM_IOCGSTATUS",
-		[_IOC_NR(CM_IOCGATR)] "CM_IOCGATR",
-		[_IOC_NR(CM_IOCARDOFF)] "CM_IOCARDOFF",
-		[_IOC_NR(CM_IOCSPTS)] "CM_IOCSPTS",
-		[_IOC_NR(CM_IOSDBGLVL)] "CM4000_DBGLVL",
-	};
-	DEBUGP(3, dev, "cmm_ioctl(device=%d.%d) %s\n", imajor(inode),
-	       iminor(inode), ioctl_names[_IOC_NR(cmd)]);
-#endif
-
-	mutex_lock(&cmm_mutex);
-	rc = -ENODEV;
-	link = dev_table[iminor(inode)];
-	if (!pcmcia_dev_present(link)) {
-		DEBUGP(4, dev, "DEV_OK false\n");
-		goto out;
-	}
-
-	if (test_bit(IS_CMM_ABSENT, &dev->flags)) {
-		DEBUGP(4, dev, "CMM_ABSENT flag set\n");
-		goto out;
-	}
-	rc = -EINVAL;
-
-	if (_IOC_TYPE(cmd) != CM_IOC_MAGIC) {
-		DEBUGP(4, dev, "ioctype mismatch\n");
-		goto out;
-	}
-	if (_IOC_NR(cmd) > CM_IOC_MAXNR) {
-		DEBUGP(4, dev, "iocnr mismatch\n");
-		goto out;
-	}
-	rc = 0;
-
-	switch (cmd) {
-	case CM_IOCGSTATUS:
-		DEBUGP(4, dev, " ... in CM_IOCGSTATUS\n");
-		{
-			int status;
-
-			/* clear other bits, but leave inserted & powered as
-			 * they are */
-			status = dev->flags0 & 3;
-			if (test_bit(IS_ATR_PRESENT, &dev->flags))
-				status |= CM_ATR_PRESENT;
-			if (test_bit(IS_ATR_VALID, &dev->flags))
-				status |= CM_ATR_VALID;
-			if (test_bit(IS_CMM_ABSENT, &dev->flags))
-				status |= CM_NO_READER;
-			if (test_bit(IS_BAD_CARD, &dev->flags))
-				status |= CM_BAD_CARD;
-			if (copy_to_user(argp, &status, sizeof(int)))
-				rc = -EFAULT;
-		}
-		break;
-	case CM_IOCGATR:
-		DEBUGP(4, dev, "... in CM_IOCGATR\n");
-		{
-			struct atreq __user *atreq = argp;
-			int tmp;
-			/* allow nonblocking io and being interrupted */
-			if (wait_event_interruptible
-			    (dev->atrq,
-			     ((filp->f_flags & O_NONBLOCK)
-			      || (test_bit(IS_ATR_PRESENT, (void *)&dev->flags)
-				  != 0)))) {
-				if (filp->f_flags & O_NONBLOCK)
-					rc = -EAGAIN;
-				else
-					rc = -ERESTARTSYS;
-				break;
-			}
-
-			rc = -EFAULT;
-			if (test_bit(IS_ATR_VALID, &dev->flags) == 0) {
-				tmp = -1;
-				if (copy_to_user(&(atreq->atr_len), &tmp,
-						 sizeof(int)))
-					break;
-			} else {
-				if (copy_to_user(atreq->atr, dev->atr,
-						 dev->atr_len))
-					break;
-
-				tmp = dev->atr_len;
-				if (copy_to_user(&(atreq->atr_len), &tmp, sizeof(int)))
-					break;
-			}
-			rc = 0;
-			break;
-		}
-	case CM_IOCARDOFF:
-
-#ifdef CM4000_DEBUG
-		DEBUGP(4, dev, "... in CM_IOCARDOFF\n");
-		if (dev->flags0 & 0x01) {
-			DEBUGP(4, dev, "    Card inserted\n");
-		} else {
-			DEBUGP(2, dev, "    No card inserted\n");
-		}
-		if (dev->flags0 & 0x02) {
-			DEBUGP(4, dev, "    Card powered\n");
-		} else {
-			DEBUGP(2, dev, "    Card not powered\n");
-		}
-#endif
-
-		/* is a card inserted and powered? */
-		if ((dev->flags0 & 0x01) && (dev->flags0 & 0x02)) {
-
-			/* get IO lock */
-			if (wait_event_interruptible
-			    (dev->ioq,
-			     ((filp->f_flags & O_NONBLOCK)
-			      || (test_and_set_bit(LOCK_IO, (void *)&dev->flags)
-				  == 0)))) {
-				if (filp->f_flags & O_NONBLOCK)
-					rc = -EAGAIN;
-				else
-					rc = -ERESTARTSYS;
-				break;
-			}
-			/* Set Flags0 = 0x42 */
-			DEBUGP(4, dev, "Set Flags0=0x42 \n");
-			xoutb(0x42, REG_FLAGS0(iobase));
-			clear_bit(IS_ATR_PRESENT, &dev->flags);
-			clear_bit(IS_ATR_VALID, &dev->flags);
-			dev->mstate = M_CARDOFF;
-			clear_bit(LOCK_IO, &dev->flags);
-			if (wait_event_interruptible
-			    (dev->atrq,
-			     ((filp->f_flags & O_NONBLOCK)
-			      || (test_bit(IS_ATR_VALID, (void *)&dev->flags) !=
-				  0)))) {
-				if (filp->f_flags & O_NONBLOCK)
-					rc = -EAGAIN;
-				else
-					rc = -ERESTARTSYS;
-				break;
-			}
-		}
-		/* release lock */
-		clear_bit(LOCK_IO, &dev->flags);
-		wake_up_interruptible(&dev->ioq);
-
-		rc = 0;
-		break;
-	case CM_IOCSPTS:
-		{
-			struct ptsreq krnptsreq;
-
-			if (copy_from_user(&krnptsreq, argp,
-					   sizeof(struct ptsreq))) {
-				rc = -EFAULT;
-				break;
-			}
-
-			rc = 0;
-			DEBUGP(4, dev, "... in CM_IOCSPTS\n");
-			/* wait for ATR to get valid */
-			if (wait_event_interruptible
-			    (dev->atrq,
-			     ((filp->f_flags & O_NONBLOCK)
-			      || (test_bit(IS_ATR_PRESENT, (void *)&dev->flags)
-				  != 0)))) {
-				if (filp->f_flags & O_NONBLOCK)
-					rc = -EAGAIN;
-				else
-					rc = -ERESTARTSYS;
-				break;
-			}
-			/* get IO lock */
-			if (wait_event_interruptible
-			    (dev->ioq,
-			     ((filp->f_flags & O_NONBLOCK)
-			      || (test_and_set_bit(LOCK_IO, (void *)&dev->flags)
-				  == 0)))) {
-				if (filp->f_flags & O_NONBLOCK)
-					rc = -EAGAIN;
-				else
-					rc = -ERESTARTSYS;
-				break;
-			}
-
-			if ((rc = set_protocol(dev, &krnptsreq)) != 0) {
-				/* auto power_on again */
-				dev->mstate = M_FETCH_ATR;
-				clear_bit(IS_ATR_VALID, &dev->flags);
-			}
-			/* release lock */
-			clear_bit(LOCK_IO, &dev->flags);
-			wake_up_interruptible(&dev->ioq);
-
-		}
-		break;
-#ifdef CM4000_DEBUG
-	case CM_IOSDBGLVL:
-		rc = -ENOTTY;
-		break;
-#endif
-	default:
-		DEBUGP(4, dev, "... in default (unknown IOCTL code)\n");
-		rc = -ENOTTY;
-	}
-out:
-	mutex_unlock(&cmm_mutex);
-	return rc;
-}
-
-static int cmm_open(struct inode *inode, struct file *filp)
-{
-	struct cm4000_dev *dev;
-	struct pcmcia_device *link;
-	int minor = iminor(inode);
-	int ret;
-
-	if (minor >= CM4000_MAX_DEV)
-		return -ENODEV;
-
-	mutex_lock(&cmm_mutex);
-	link = dev_table[minor];
-	if (link == NULL || !pcmcia_dev_present(link)) {
-		ret = -ENODEV;
-		goto out;
-	}
-
-	if (link->open) {
-		ret = -EBUSY;
-		goto out;
-	}
-
-	dev = link->priv;
-	filp->private_data = dev;
-
-	DEBUGP(2, dev, "-> cmm_open(device=%d.%d process=%s,%d)\n",
-	      imajor(inode), minor, current->comm, current->pid);
-
-	/* init device variables, they may be "polluted" after close
-	 * or, the device may never have been closed (i.e. open failed)
-	 */
-
-	ZERO_DEV(dev);
-
-	/* opening will always block since the
-	 * monitor will be started by open, which
-	 * means we have to wait for ATR becoming
-	 * valid = block until valid (or card
-	 * inserted)
-	 */
-	if (filp->f_flags & O_NONBLOCK) {
-		ret = -EAGAIN;
-		goto out;
-	}
-
-	dev->mdelay = T_50MSEC;
-
-	/* start monitoring the cardstatus */
-	start_monitor(dev);
-
-	link->open = 1;		/* only one open per device */
-
-	DEBUGP(2, dev, "<- cmm_open\n");
-	ret = stream_open(inode, filp);
-out:
-	mutex_unlock(&cmm_mutex);
-	return ret;
-}
-
-static int cmm_close(struct inode *inode, struct file *filp)
-{
-	struct cm4000_dev *dev;
-	struct pcmcia_device *link;
-	int minor = iminor(inode);
-
-	if (minor >= CM4000_MAX_DEV)
-		return -ENODEV;
-
-	link = dev_table[minor];
-	if (link == NULL)
-		return -ENODEV;
-
-	dev = link->priv;
-
-	DEBUGP(2, dev, "-> cmm_close(maj/min=%d.%d)\n",
-	       imajor(inode), minor);
-
-	stop_monitor(dev);
-
-	ZERO_DEV(dev);
-
-	link->open = 0;		/* only one open per device */
-	wake_up(&dev->devq);	/* socket removed? */
-
-	DEBUGP(2, dev, "cmm_close\n");
-	return 0;
-}
-
-static void cmm_cm4000_release(struct pcmcia_device * link)
-{
-	struct cm4000_dev *dev = link->priv;
-
-	/* dont terminate the monitor, rather rely on
-	 * close doing that for us.
-	 */
-	DEBUGP(3, dev, "-> cmm_cm4000_release\n");
-	while (link->open) {
-		printk(KERN_INFO MODULE_NAME ": delaying release until "
-		       "process has terminated\n");
-		/* note: don't interrupt us:
-		 * close the applications which own
-		 * the devices _first_ !
-		 */
-		wait_event(dev->devq, (link->open == 0));
-	}
-	/* dev->devq=NULL;	this cannot be zeroed earlier */
-	DEBUGP(3, dev, "<- cmm_cm4000_release\n");
-	return;
-}
-
-/*==== Interface to PCMCIA Layer =======================================*/
-
-static int cm4000_config_check(struct pcmcia_device *p_dev, void *priv_data)
-{
-	return pcmcia_request_io(p_dev);
-}
-
-static int cm4000_config(struct pcmcia_device * link, int devno)
-{
-	link->config_flags |= CONF_AUTO_SET_IO;
-
-	/* read the config-tuples */
-	if (pcmcia_loop_config(link, cm4000_config_check, NULL))
-		goto cs_release;
-
-	if (pcmcia_enable_device(link))
-		goto cs_release;
-
-	return 0;
-
-cs_release:
-	cm4000_release(link);
-	return -ENODEV;
-}
-
-static int cm4000_suspend(struct pcmcia_device *link)
-{
-	struct cm4000_dev *dev;
-
-	dev = link->priv;
-	stop_monitor(dev);
-
-	return 0;
-}
-
-static int cm4000_resume(struct pcmcia_device *link)
-{
-	struct cm4000_dev *dev;
-
-	dev = link->priv;
-	if (link->open)
-		start_monitor(dev);
-
-	return 0;
-}
-
-static void cm4000_release(struct pcmcia_device *link)
-{
-	cmm_cm4000_release(link);	/* delay release until device closed */
-	pcmcia_disable_device(link);
-}
-
-static int cm4000_probe(struct pcmcia_device *link)
-{
-	struct cm4000_dev *dev;
-	int i, ret;
-
-	for (i = 0; i < CM4000_MAX_DEV; i++)
-		if (dev_table[i] == NULL)
-			break;
-
-	if (i == CM4000_MAX_DEV) {
-		printk(KERN_NOTICE MODULE_NAME ": all devices in use\n");
-		return -ENODEV;
-	}
-
-	/* create a new cm4000_cs device */
-	dev = kzalloc(sizeof(struct cm4000_dev), GFP_KERNEL);
-	if (dev == NULL)
-		return -ENOMEM;
-
-	dev->p_dev = link;
-	link->priv = dev;
-	dev_table[i] = link;
-
-	init_waitqueue_head(&dev->devq);
-	init_waitqueue_head(&dev->ioq);
-	init_waitqueue_head(&dev->atrq);
-	init_waitqueue_head(&dev->readq);
-
-	ret = cm4000_config(link, i);
-	if (ret) {
-		dev_table[i] = NULL;
-		kfree(dev);
-		return ret;
-	}
-
-	device_create(cmm_class, NULL, MKDEV(major, i), NULL, "cmm%d", i);
-
-	return 0;
-}
-
-static void cm4000_detach(struct pcmcia_device *link)
-{
-	struct cm4000_dev *dev = link->priv;
-	int devno;
-
-	/* find device */
-	for (devno = 0; devno < CM4000_MAX_DEV; devno++)
-		if (dev_table[devno] == link)
-			break;
-	if (devno == CM4000_MAX_DEV)
-		return;
-
-	stop_monitor(dev);
-
-	cm4000_release(link);
-
-	dev_table[devno] = NULL;
-	kfree(dev);
-
-	device_destroy(cmm_class, MKDEV(major, devno));
-
-	return;
-}
-
-static const struct file_operations cm4000_fops = {
-	.owner	= THIS_MODULE,
-	.read	= cmm_read,
-	.write	= cmm_write,
-	.unlocked_ioctl	= cmm_ioctl,
-	.open	= cmm_open,
-	.release= cmm_close,
-	.llseek = no_llseek,
-};
-
-static const struct pcmcia_device_id cm4000_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD(0x0223, 0x0002),
-	PCMCIA_DEVICE_PROD_ID12("CardMan", "4000", 0x2FB368CA, 0xA2BD8C39),
-	PCMCIA_DEVICE_NULL,
-};
-MODULE_DEVICE_TABLE(pcmcia, cm4000_ids);
-
-static struct pcmcia_driver cm4000_driver = {
-	.owner	  = THIS_MODULE,
-	.name	  = "cm4000_cs",
-	.probe    = cm4000_probe,
-	.remove   = cm4000_detach,
-	.suspend  = cm4000_suspend,
-	.resume   = cm4000_resume,
-	.id_table = cm4000_ids,
-};
-
-static int __init cmm_init(void)
-{
-	int rc;
-
-	cmm_class = class_create(THIS_MODULE, "cardman_4000");
-	if (IS_ERR(cmm_class))
-		return PTR_ERR(cmm_class);
-
-	major = register_chrdev(0, DEVICE_NAME, &cm4000_fops);
-	if (major < 0) {
-		printk(KERN_WARNING MODULE_NAME
-			": could not get major number\n");
-		class_destroy(cmm_class);
-		return major;
-	}
-
-	rc = pcmcia_register_driver(&cm4000_driver);
-	if (rc < 0) {
-		unregister_chrdev(major, DEVICE_NAME);
-		class_destroy(cmm_class);
-		return rc;
-	}
-
-	return 0;
-}
-
-static void __exit cmm_exit(void)
-{
-	pcmcia_unregister_driver(&cm4000_driver);
-	unregister_chrdev(major, DEVICE_NAME);
-	class_destroy(cmm_class);
-};
-
-module_init(cmm_init);
-module_exit(cmm_exit);
-MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/char/pcmcia/cm4040_cs.c b/drivers/char/pcmcia/cm4040_cs.c
deleted file mode 100644
index 827711911da4..000000000000
--- a/drivers/char/pcmcia/cm4040_cs.c
+++ /dev/null
@@ -1,684 +0,0 @@
-/*
- * A driver for the Omnikey PCMCIA smartcard reader CardMan 4040
- *
- * (c) 2000-2004 Omnikey AG (http://www.omnikey.com/)
- *
- * (C) 2005-2006 Harald Welte <laforge@gnumonks.org>
- * 	- add support for poll()
- * 	- driver cleanup
- * 	- add waitqueues
- * 	- adhere to linux kernel coding style and policies
- * 	- support 2.6.13 "new style" pcmcia interface
- * 	- add class interface for udev device creation
- *
- * The device basically is a USB CCID compliant device that has been
- * attached to an I/O-Mapped FIFO.
- *
- * All rights reserved, Dual BSD/GPL Licensed.
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/init.h>
-#include <linux/fs.h>
-#include <linux/delay.h>
-#include <linux/poll.h>
-#include <linux/mutex.h>
-#include <linux/wait.h>
-#include <linux/uaccess.h>
-#include <asm/io.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/cisreg.h>
-#include <pcmcia/ciscode.h>
-#include <pcmcia/ds.h>
-
-#include "cm4040_cs.h"
-
-
-#define reader_to_dev(x)	(&x->p_dev->dev)
-
-/* n (debug level) is ignored */
-/* additional debug output may be enabled by re-compiling with
- * CM4040_DEBUG set */
-/* #define CM4040_DEBUG */
-#define DEBUGP(n, rdr, x, args...) do { 		\
-		dev_dbg(reader_to_dev(rdr), "%s:" x, 	\
-			   __func__ , ## args);		\
-	} while (0)
-
-static DEFINE_MUTEX(cm4040_mutex);
-
-#define	CCID_DRIVER_BULK_DEFAULT_TIMEOUT  	(150*HZ)
-#define	CCID_DRIVER_ASYNC_POWERUP_TIMEOUT 	(35*HZ)
-#define	CCID_DRIVER_MINIMUM_TIMEOUT 		(3*HZ)
-#define READ_WRITE_BUFFER_SIZE 512
-#define POLL_LOOP_COUNT				1000
-
-/* how often to poll for fifo status change */
-#define POLL_PERIOD 				msecs_to_jiffies(10)
-
-static void reader_release(struct pcmcia_device *link);
-
-static int major;
-static struct class *cmx_class;
-
-#define		BS_READABLE	0x01
-#define		BS_WRITABLE	0x02
-
-struct reader_dev {
-	struct pcmcia_device	*p_dev;
-	wait_queue_head_t	devq;
-	wait_queue_head_t	poll_wait;
-	wait_queue_head_t	read_wait;
-	wait_queue_head_t	write_wait;
-	unsigned long 	  	buffer_status;
-	unsigned long     	timeout;
-	unsigned char     	s_buf[READ_WRITE_BUFFER_SIZE];
-	unsigned char     	r_buf[READ_WRITE_BUFFER_SIZE];
-	struct timer_list 	poll_timer;
-};
-
-static struct pcmcia_device *dev_table[CM_MAX_DEV];
-
-#ifndef CM4040_DEBUG
-#define	xoutb	outb
-#define	xinb	inb
-#else
-static inline void xoutb(unsigned char val, unsigned short port)
-{
-	pr_debug("outb(val=%.2x,port=%.4x)\n", val, port);
-	outb(val, port);
-}
-
-static inline unsigned char xinb(unsigned short port)
-{
-	unsigned char val;
-
-	val = inb(port);
-	pr_debug("%.2x=inb(%.4x)\n", val, port);
-	return val;
-}
-#endif
-
-/* poll the device fifo status register.  not to be confused with
- * the poll syscall. */
-static void cm4040_do_poll(struct timer_list *t)
-{
-	struct reader_dev *dev = from_timer(dev, t, poll_timer);
-	unsigned int obs = xinb(dev->p_dev->resource[0]->start
-				+ REG_OFFSET_BUFFER_STATUS);
-
-	if ((obs & BSR_BULK_IN_FULL)) {
-		set_bit(BS_READABLE, &dev->buffer_status);
-		DEBUGP(4, dev, "waking up read_wait\n");
-		wake_up_interruptible(&dev->read_wait);
-	} else
-		clear_bit(BS_READABLE, &dev->buffer_status);
-
-	if (!(obs & BSR_BULK_OUT_FULL)) {
-		set_bit(BS_WRITABLE, &dev->buffer_status);
-		DEBUGP(4, dev, "waking up write_wait\n");
-		wake_up_interruptible(&dev->write_wait);
-	} else
-		clear_bit(BS_WRITABLE, &dev->buffer_status);
-
-	if (dev->buffer_status)
-		wake_up_interruptible(&dev->poll_wait);
-
-	mod_timer(&dev->poll_timer, jiffies + POLL_PERIOD);
-}
-
-static void cm4040_stop_poll(struct reader_dev *dev)
-{
-	del_timer_sync(&dev->poll_timer);
-}
-
-static int wait_for_bulk_out_ready(struct reader_dev *dev)
-{
-	int i, rc;
-	int iobase = dev->p_dev->resource[0]->start;
-
-	for (i = 0; i < POLL_LOOP_COUNT; i++) {
-		if ((xinb(iobase + REG_OFFSET_BUFFER_STATUS)
-		    & BSR_BULK_OUT_FULL) == 0) {
-			DEBUGP(4, dev, "BulkOut empty (i=%d)\n", i);
-			return 1;
-		}
-	}
-
-	DEBUGP(4, dev, "wait_event_interruptible_timeout(timeout=%ld\n",
-		dev->timeout);
-	rc = wait_event_interruptible_timeout(dev->write_wait,
-					      test_and_clear_bit(BS_WRITABLE,
-						       &dev->buffer_status),
-					      dev->timeout);
-
-	if (rc > 0)
-		DEBUGP(4, dev, "woke up: BulkOut empty\n");
-	else if (rc == 0)
-		DEBUGP(4, dev, "woke up: BulkOut full, returning 0 :(\n");
-	else if (rc < 0)
-		DEBUGP(4, dev, "woke up: signal arrived\n");
-
-	return rc;
-}
-
-/* Write to Sync Control Register */
-static int write_sync_reg(unsigned char val, struct reader_dev *dev)
-{
-	int iobase = dev->p_dev->resource[0]->start;
-	int rc;
-
-	rc = wait_for_bulk_out_ready(dev);
-	if (rc <= 0)
-		return rc;
-
-	xoutb(val, iobase + REG_OFFSET_SYNC_CONTROL);
-	rc = wait_for_bulk_out_ready(dev);
-	if (rc <= 0)
-		return rc;
-
-	return 1;
-}
-
-static int wait_for_bulk_in_ready(struct reader_dev *dev)
-{
-	int i, rc;
-	int iobase = dev->p_dev->resource[0]->start;
-
-	for (i = 0; i < POLL_LOOP_COUNT; i++) {
-		if ((xinb(iobase + REG_OFFSET_BUFFER_STATUS)
-		    & BSR_BULK_IN_FULL) == BSR_BULK_IN_FULL) {
-			DEBUGP(3, dev, "BulkIn full (i=%d)\n", i);
-			return 1;
-		}
-	}
-
-	DEBUGP(4, dev, "wait_event_interruptible_timeout(timeout=%ld\n",
-		dev->timeout);
-	rc = wait_event_interruptible_timeout(dev->read_wait,
-					      test_and_clear_bit(BS_READABLE,
-						 	&dev->buffer_status),
-					      dev->timeout);
-	if (rc > 0)
-		DEBUGP(4, dev, "woke up: BulkIn full\n");
-	else if (rc == 0)
-		DEBUGP(4, dev, "woke up: BulkIn not full, returning 0 :(\n");
-	else if (rc < 0)
-		DEBUGP(4, dev, "woke up: signal arrived\n");
-
-	return rc;
-}
-
-static ssize_t cm4040_read(struct file *filp, char __user *buf,
-			size_t count, loff_t *ppos)
-{
-	struct reader_dev *dev = filp->private_data;
-	int iobase = dev->p_dev->resource[0]->start;
-	size_t bytes_to_read;
-	unsigned long i;
-	size_t min_bytes_to_read;
-	int rc;
-
-	DEBUGP(2, dev, "-> cm4040_read(%s,%d)\n", current->comm, current->pid);
-
-	if (count == 0)
-		return 0;
-
-	if (count < 10)
-		return -EFAULT;
-
-	if (filp->f_flags & O_NONBLOCK) {
-		DEBUGP(4, dev, "filep->f_flags O_NONBLOCK set\n");
-		DEBUGP(2, dev, "<- cm4040_read (failure)\n");
-		return -EAGAIN;
-	}
-
-	if (!pcmcia_dev_present(dev->p_dev))
-		return -ENODEV;
-
-	for (i = 0; i < 5; i++) {
-		rc = wait_for_bulk_in_ready(dev);
-		if (rc <= 0) {
-			DEBUGP(5, dev, "wait_for_bulk_in_ready rc=%.2x\n", rc);
-			DEBUGP(2, dev, "<- cm4040_read (failed)\n");
-			if (rc == -ERESTARTSYS)
-				return rc;
-			return -EIO;
-		}
-	  	dev->r_buf[i] = xinb(iobase + REG_OFFSET_BULK_IN);
-#ifdef CM4040_DEBUG
-		pr_debug("%lu:%2x ", i, dev->r_buf[i]);
-	}
-	pr_debug("\n");
-#else
-	}
-#endif
-
-	bytes_to_read = 5 + le32_to_cpu(*(__le32 *)&dev->r_buf[1]);
-
-	DEBUGP(6, dev, "BytesToRead=%zu\n", bytes_to_read);
-
-	min_bytes_to_read = min(count, bytes_to_read + 5);
-	min_bytes_to_read = min_t(size_t, min_bytes_to_read, READ_WRITE_BUFFER_SIZE);
-
-	DEBUGP(6, dev, "Min=%zu\n", min_bytes_to_read);
-
-	for (i = 0; i < (min_bytes_to_read-5); i++) {
-		rc = wait_for_bulk_in_ready(dev);
-		if (rc <= 0) {
-			DEBUGP(5, dev, "wait_for_bulk_in_ready rc=%.2x\n", rc);
-			DEBUGP(2, dev, "<- cm4040_read (failed)\n");
-			if (rc == -ERESTARTSYS)
-				return rc;
-			return -EIO;
-		}
-		dev->r_buf[i+5] = xinb(iobase + REG_OFFSET_BULK_IN);
-#ifdef CM4040_DEBUG
-		pr_debug("%lu:%2x ", i, dev->r_buf[i]);
-	}
-	pr_debug("\n");
-#else
-	}
-#endif
-
-	*ppos = min_bytes_to_read;
-	if (copy_to_user(buf, dev->r_buf, min_bytes_to_read))
-		return -EFAULT;
-
-	rc = wait_for_bulk_in_ready(dev);
-	if (rc <= 0) {
-		DEBUGP(5, dev, "wait_for_bulk_in_ready rc=%.2x\n", rc);
-		DEBUGP(2, dev, "<- cm4040_read (failed)\n");
-		if (rc == -ERESTARTSYS)
-			return rc;
-		return -EIO;
-	}
-
-	rc = write_sync_reg(SCR_READER_TO_HOST_DONE, dev);
-	if (rc <= 0) {
-		DEBUGP(5, dev, "write_sync_reg c=%.2x\n", rc);
-		DEBUGP(2, dev, "<- cm4040_read (failed)\n");
-		if (rc == -ERESTARTSYS)
-			return rc;
-		else
-			return -EIO;
-	}
-
-	xinb(iobase + REG_OFFSET_BULK_IN);
-
-	DEBUGP(2, dev, "<- cm4040_read (successfully)\n");
-	return min_bytes_to_read;
-}
-
-static ssize_t cm4040_write(struct file *filp, const char __user *buf,
-			 size_t count, loff_t *ppos)
-{
-	struct reader_dev *dev = filp->private_data;
-	int iobase = dev->p_dev->resource[0]->start;
-	ssize_t rc;
-	int i;
-	unsigned int bytes_to_write;
-
-	DEBUGP(2, dev, "-> cm4040_write(%s,%d)\n", current->comm, current->pid);
-
-	if (count == 0) {
-		DEBUGP(2, dev, "<- cm4040_write empty read (successfully)\n");
-		return 0;
-	}
-
-	if ((count < 5) || (count > READ_WRITE_BUFFER_SIZE)) {
-		DEBUGP(2, dev, "<- cm4040_write buffersize=%zd < 5\n", count);
-		return -EIO;
-	}
-
-	if (filp->f_flags & O_NONBLOCK) {
-		DEBUGP(4, dev, "filep->f_flags O_NONBLOCK set\n");
-		DEBUGP(4, dev, "<- cm4040_write (failure)\n");
-		return -EAGAIN;
-	}
-
-	if (!pcmcia_dev_present(dev->p_dev))
-		return -ENODEV;
-
-	bytes_to_write = count;
-	if (copy_from_user(dev->s_buf, buf, bytes_to_write))
-		return -EFAULT;
-
-	switch (dev->s_buf[0]) {
-		case CMD_PC_TO_RDR_XFRBLOCK:
-		case CMD_PC_TO_RDR_SECURE:
-		case CMD_PC_TO_RDR_TEST_SECURE:
-		case CMD_PC_TO_RDR_OK_SECURE:
-			dev->timeout = CCID_DRIVER_BULK_DEFAULT_TIMEOUT;
-			break;
-
-		case CMD_PC_TO_RDR_ICCPOWERON:
-			dev->timeout = CCID_DRIVER_ASYNC_POWERUP_TIMEOUT;
-			break;
-
-		case CMD_PC_TO_RDR_GETSLOTSTATUS:
-		case CMD_PC_TO_RDR_ICCPOWEROFF:
-		case CMD_PC_TO_RDR_GETPARAMETERS:
-		case CMD_PC_TO_RDR_RESETPARAMETERS:
-		case CMD_PC_TO_RDR_SETPARAMETERS:
-		case CMD_PC_TO_RDR_ESCAPE:
-		case CMD_PC_TO_RDR_ICCCLOCK:
-		default:
-			dev->timeout = CCID_DRIVER_MINIMUM_TIMEOUT;
-			break;
-	}
-
-	rc = write_sync_reg(SCR_HOST_TO_READER_START, dev);
-	if (rc <= 0) {
-		DEBUGP(5, dev, "write_sync_reg c=%.2zx\n", rc);
-		DEBUGP(2, dev, "<- cm4040_write (failed)\n");
-		if (rc == -ERESTARTSYS)
-			return rc;
-		else
-			return -EIO;
-	}
-
-	DEBUGP(4, dev, "start \n");
-
-	for (i = 0; i < bytes_to_write; i++) {
-		rc = wait_for_bulk_out_ready(dev);
-		if (rc <= 0) {
-			DEBUGP(5, dev, "wait_for_bulk_out_ready rc=%.2zx\n",
-			       rc);
-			DEBUGP(2, dev, "<- cm4040_write (failed)\n");
-			if (rc == -ERESTARTSYS)
-				return rc;
-			else
-				return -EIO;
-		}
-
-		xoutb(dev->s_buf[i],iobase + REG_OFFSET_BULK_OUT);
-	}
-	DEBUGP(4, dev, "end\n");
-
-	rc = write_sync_reg(SCR_HOST_TO_READER_DONE, dev);
-
-	if (rc <= 0) {
-		DEBUGP(5, dev, "write_sync_reg c=%.2zx\n", rc);
-		DEBUGP(2, dev, "<- cm4040_write (failed)\n");
-		if (rc == -ERESTARTSYS)
-			return rc;
-		else
-			return -EIO;
-	}
-
-	DEBUGP(2, dev, "<- cm4040_write (successfully)\n");
-	return count;
-}
-
-static __poll_t cm4040_poll(struct file *filp, poll_table *wait)
-{
-	struct reader_dev *dev = filp->private_data;
-	__poll_t mask = 0;
-
-	poll_wait(filp, &dev->poll_wait, wait);
-
-	if (test_and_clear_bit(BS_READABLE, &dev->buffer_status))
-		mask |= EPOLLIN | EPOLLRDNORM;
-	if (test_and_clear_bit(BS_WRITABLE, &dev->buffer_status))
-		mask |= EPOLLOUT | EPOLLWRNORM;
-
-	DEBUGP(2, dev, "<- cm4040_poll(%u)\n", mask);
-
-	return mask;
-}
-
-static int cm4040_open(struct inode *inode, struct file *filp)
-{
-	struct reader_dev *dev;
-	struct pcmcia_device *link;
-	int minor = iminor(inode);
-	int ret;
-
-	if (minor >= CM_MAX_DEV)
-		return -ENODEV;
-
-	mutex_lock(&cm4040_mutex);
-	link = dev_table[minor];
-	if (link == NULL || !pcmcia_dev_present(link)) {
-		ret = -ENODEV;
-		goto out;
-	}
-
-	if (link->open) {
-		ret = -EBUSY;
-		goto out;
-	}
-
-	dev = link->priv;
-	filp->private_data = dev;
-
-	if (filp->f_flags & O_NONBLOCK) {
-		DEBUGP(4, dev, "filep->f_flags O_NONBLOCK set\n");
-		ret = -EAGAIN;
-		goto out;
-	}
-
-	link->open = 1;
-
-	mod_timer(&dev->poll_timer, jiffies + POLL_PERIOD);
-
-	DEBUGP(2, dev, "<- cm4040_open (successfully)\n");
-	ret = nonseekable_open(inode, filp);
-out:
-	mutex_unlock(&cm4040_mutex);
-	return ret;
-}
-
-static int cm4040_close(struct inode *inode, struct file *filp)
-{
-	struct reader_dev *dev = filp->private_data;
-	struct pcmcia_device *link;
-	int minor = iminor(inode);
-
-	DEBUGP(2, dev, "-> cm4040_close(maj/min=%d.%d)\n", imajor(inode),
-	      iminor(inode));
-
-	if (minor >= CM_MAX_DEV)
-		return -ENODEV;
-
-	link = dev_table[minor];
-	if (link == NULL)
-		return -ENODEV;
-
-	cm4040_stop_poll(dev);
-
-	link->open = 0;
-	wake_up(&dev->devq);
-
-	DEBUGP(2, dev, "<- cm4040_close\n");
-	return 0;
-}
-
-static void cm4040_reader_release(struct pcmcia_device *link)
-{
-	struct reader_dev *dev = link->priv;
-
-	DEBUGP(3, dev, "-> cm4040_reader_release\n");
-	while (link->open) {
-		DEBUGP(3, dev, MODULE_NAME ": delaying release "
-		       "until process has terminated\n");
- 		wait_event(dev->devq, (link->open == 0));
-	}
-	DEBUGP(3, dev, "<- cm4040_reader_release\n");
-	return;
-}
-
-static int cm4040_config_check(struct pcmcia_device *p_dev, void *priv_data)
-{
-	return pcmcia_request_io(p_dev);
-}
-
-
-static int reader_config(struct pcmcia_device *link, int devno)
-{
-	struct reader_dev *dev;
-	int fail_rc;
-
-	link->config_flags |= CONF_AUTO_SET_IO;
-
-	if (pcmcia_loop_config(link, cm4040_config_check, NULL))
-		goto cs_release;
-
-	fail_rc = pcmcia_enable_device(link);
-	if (fail_rc != 0) {
-		dev_info(&link->dev, "pcmcia_enable_device failed 0x%x\n",
-			 fail_rc);
-		goto cs_release;
-	}
-
-	dev = link->priv;
-
-	DEBUGP(2, dev, "device " DEVICE_NAME "%d at %pR\n", devno,
-	      link->resource[0]);
-	DEBUGP(2, dev, "<- reader_config (succ)\n");
-
-	return 0;
-
-cs_release:
-	reader_release(link);
-	return -ENODEV;
-}
-
-static void reader_release(struct pcmcia_device *link)
-{
-	cm4040_reader_release(link);
-	pcmcia_disable_device(link);
-}
-
-static int reader_probe(struct pcmcia_device *link)
-{
-	struct reader_dev *dev;
-	int i, ret;
-
-	for (i = 0; i < CM_MAX_DEV; i++) {
-		if (dev_table[i] == NULL)
-			break;
-	}
-
-	if (i == CM_MAX_DEV)
-		return -ENODEV;
-
-	dev = kzalloc(sizeof(struct reader_dev), GFP_KERNEL);
-	if (dev == NULL)
-		return -ENOMEM;
-
-	dev->timeout = CCID_DRIVER_MINIMUM_TIMEOUT;
-	dev->buffer_status = 0;
-
-	link->priv = dev;
-	dev->p_dev = link;
-
-	dev_table[i] = link;
-
-	init_waitqueue_head(&dev->devq);
-	init_waitqueue_head(&dev->poll_wait);
-	init_waitqueue_head(&dev->read_wait);
-	init_waitqueue_head(&dev->write_wait);
-	timer_setup(&dev->poll_timer, cm4040_do_poll, 0);
-
-	ret = reader_config(link, i);
-	if (ret) {
-		dev_table[i] = NULL;
-		kfree(dev);
-		return ret;
-	}
-
-	device_create(cmx_class, NULL, MKDEV(major, i), NULL, "cmx%d", i);
-
-	return 0;
-}
-
-static void reader_detach(struct pcmcia_device *link)
-{
-	struct reader_dev *dev = link->priv;
-	int devno;
-
-	/* find device */
-	for (devno = 0; devno < CM_MAX_DEV; devno++) {
-		if (dev_table[devno] == link)
-			break;
-	}
-	if (devno == CM_MAX_DEV)
-		return;
-
-	reader_release(link);
-
-	dev_table[devno] = NULL;
-	kfree(dev);
-
-	device_destroy(cmx_class, MKDEV(major, devno));
-
-	return;
-}
-
-static const struct file_operations reader_fops = {
-	.owner		= THIS_MODULE,
-	.read		= cm4040_read,
-	.write		= cm4040_write,
-	.open		= cm4040_open,
-	.release	= cm4040_close,
-	.poll		= cm4040_poll,
-	.llseek		= no_llseek,
-};
-
-static const struct pcmcia_device_id cm4040_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD(0x0223, 0x0200),
-	PCMCIA_DEVICE_PROD_ID12("OMNIKEY", "CardMan 4040",
-				0xE32CDD8C, 0x8F23318B),
-	PCMCIA_DEVICE_NULL,
-};
-MODULE_DEVICE_TABLE(pcmcia, cm4040_ids);
-
-static struct pcmcia_driver reader_driver = {
-  	.owner		= THIS_MODULE,
-	.name		= "cm4040_cs",
-	.probe		= reader_probe,
-	.remove		= reader_detach,
-	.id_table	= cm4040_ids,
-};
-
-static int __init cm4040_init(void)
-{
-	int rc;
-
-	cmx_class = class_create(THIS_MODULE, "cardman_4040");
-	if (IS_ERR(cmx_class))
-		return PTR_ERR(cmx_class);
-
-	major = register_chrdev(0, DEVICE_NAME, &reader_fops);
-	if (major < 0) {
-		printk(KERN_WARNING MODULE_NAME
-			": could not get major number\n");
-		class_destroy(cmx_class);
-		return major;
-	}
-
-	rc = pcmcia_register_driver(&reader_driver);
-	if (rc < 0) {
-		unregister_chrdev(major, DEVICE_NAME);
-		class_destroy(cmx_class);
-		return rc;
-	}
-
-	return 0;
-}
-
-static void __exit cm4040_exit(void)
-{
-	pcmcia_unregister_driver(&reader_driver);
-	unregister_chrdev(major, DEVICE_NAME);
-	class_destroy(cmx_class);
-}
-
-module_init(cm4040_init);
-module_exit(cm4040_exit);
-MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/char/pcmcia/cm4040_cs.h b/drivers/char/pcmcia/cm4040_cs.h
deleted file mode 100644
index e2ffff995d51..000000000000
--- a/drivers/char/pcmcia/cm4040_cs.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef	_CM4040_H_
-#define	_CM4040_H_
-
-#define	CM_MAX_DEV		4
-
-#define	DEVICE_NAME		"cmx"
-#define	MODULE_NAME		"cm4040_cs"
-
-#define REG_OFFSET_BULK_OUT      0
-#define REG_OFFSET_BULK_IN       0
-#define REG_OFFSET_BUFFER_STATUS 1
-#define REG_OFFSET_SYNC_CONTROL  2
-
-#define BSR_BULK_IN_FULL  0x02
-#define BSR_BULK_OUT_FULL 0x01
-
-#define SCR_HOST_TO_READER_START 0x80
-#define SCR_ABORT                0x40
-#define SCR_EN_NOTIFY            0x20
-#define SCR_ACK_NOTIFY           0x10
-#define SCR_READER_TO_HOST_DONE  0x08
-#define SCR_HOST_TO_READER_DONE  0x04
-#define SCR_PULSE_INTERRUPT      0x02
-#define SCR_POWER_DOWN           0x01
-
-
-#define  CMD_PC_TO_RDR_ICCPOWERON       0x62
-#define  CMD_PC_TO_RDR_GETSLOTSTATUS    0x65
-#define  CMD_PC_TO_RDR_ICCPOWEROFF      0x63
-#define  CMD_PC_TO_RDR_SECURE           0x69
-#define  CMD_PC_TO_RDR_GETPARAMETERS    0x6C
-#define  CMD_PC_TO_RDR_RESETPARAMETERS  0x6D
-#define  CMD_PC_TO_RDR_SETPARAMETERS    0x61
-#define  CMD_PC_TO_RDR_XFRBLOCK         0x6F
-#define  CMD_PC_TO_RDR_ESCAPE           0x6B
-#define  CMD_PC_TO_RDR_ICCCLOCK         0x6E
-#define  CMD_PC_TO_RDR_TEST_SECURE      0x74
-#define  CMD_PC_TO_RDR_OK_SECURE        0x89
-
-
-#define  CMD_RDR_TO_PC_SLOTSTATUS         0x81
-#define  CMD_RDR_TO_PC_DATABLOCK          0x80
-#define  CMD_RDR_TO_PC_PARAMETERS         0x82
-#define  CMD_RDR_TO_PC_ESCAPE             0x83
-#define  CMD_RDR_TO_PC_OK_SECURE          0x89
-
-#endif	/* _CM4040_H_ */
diff --git a/drivers/char/pcmcia/scr24x_cs.c b/drivers/char/pcmcia/scr24x_cs.c
deleted file mode 100644
index 1bdce08fae3d..000000000000
--- a/drivers/char/pcmcia/scr24x_cs.c
+++ /dev/null
@@ -1,359 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * SCR24x PCMCIA Smart Card Reader Driver
- *
- * Copyright (C) 2005-2006 TL Sudheendran
- * Copyright (C) 2016 Lubomir Rintel
- *
- * Derived from "scr24x_v4.2.6_Release.tar.gz" driver by TL Sudheendran.
- */
-
-#include <linux/device.h>
-#include <linux/module.h>
-#include <linux/delay.h>
-#include <linux/cdev.h>
-#include <linux/slab.h>
-#include <linux/fs.h>
-#include <linux/io.h>
-#include <linux/uaccess.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/ds.h>
-
-#define CCID_HEADER_SIZE	10
-#define CCID_LENGTH_OFFSET	1
-#define CCID_MAX_LEN		271
-
-#define SCR24X_DATA(n)		(1 + n)
-#define SCR24X_CMD_STATUS	7
-#define CMD_START		0x40
-#define CMD_WRITE_BYTE		0x41
-#define CMD_READ_BYTE		0x42
-#define STATUS_BUSY		0x80
-
-struct scr24x_dev {
-	struct device *dev;
-	struct cdev c_dev;
-	unsigned char buf[CCID_MAX_LEN];
-	int devno;
-	struct mutex lock;
-	struct kref refcnt;
-	u8 __iomem *regs;
-};
-
-#define SCR24X_DEVS 8
-static DECLARE_BITMAP(scr24x_minors, SCR24X_DEVS);
-
-static struct class *scr24x_class;
-static dev_t scr24x_devt;
-
-static void scr24x_delete(struct kref *kref)
-{
-	struct scr24x_dev *dev = container_of(kref, struct scr24x_dev,
-								refcnt);
-
-	kfree(dev);
-}
-
-static int scr24x_wait_ready(struct scr24x_dev *dev)
-{
-	u_char status;
-	int timeout = 100;
-
-	do {
-		status = ioread8(dev->regs + SCR24X_CMD_STATUS);
-		if (!(status & STATUS_BUSY))
-			return 0;
-
-		msleep(20);
-	} while (--timeout);
-
-	return -EIO;
-}
-
-static int scr24x_open(struct inode *inode, struct file *filp)
-{
-	struct scr24x_dev *dev = container_of(inode->i_cdev,
-				struct scr24x_dev, c_dev);
-
-	kref_get(&dev->refcnt);
-	filp->private_data = dev;
-
-	return stream_open(inode, filp);
-}
-
-static int scr24x_release(struct inode *inode, struct file *filp)
-{
-	struct scr24x_dev *dev = filp->private_data;
-
-	/* We must not take the dev->lock here as scr24x_delete()
-	 * might be called to remove the dev structure altogether.
-	 * We don't need the lock anyway, since after the reference
-	 * acquired in probe() is released in remove() the chrdev
-	 * is already unregistered and noone can possibly acquire
-	 * a reference via open() anymore. */
-	kref_put(&dev->refcnt, scr24x_delete);
-	return 0;
-}
-
-static int read_chunk(struct scr24x_dev *dev, size_t offset, size_t limit)
-{
-	size_t i, y;
-	int ret;
-
-	for (i = offset; i < limit; i += 5) {
-		iowrite8(CMD_READ_BYTE, dev->regs + SCR24X_CMD_STATUS);
-		ret = scr24x_wait_ready(dev);
-		if (ret < 0)
-			return ret;
-
-		for (y = 0; y < 5 && i + y < limit; y++)
-			dev->buf[i + y] = ioread8(dev->regs + SCR24X_DATA(y));
-	}
-
-	return 0;
-}
-
-static ssize_t scr24x_read(struct file *filp, char __user *buf, size_t count,
-								loff_t *ppos)
-{
-	struct scr24x_dev *dev = filp->private_data;
-	int ret;
-	int len;
-
-	if (count < CCID_HEADER_SIZE)
-		return -EINVAL;
-
-	if (mutex_lock_interruptible(&dev->lock))
-		return -ERESTARTSYS;
-
-	if (!dev->dev) {
-		ret = -ENODEV;
-		goto out;
-	}
-
-	ret = scr24x_wait_ready(dev);
-	if (ret < 0)
-		goto out;
-	len = CCID_HEADER_SIZE;
-	ret = read_chunk(dev, 0, len);
-	if (ret < 0)
-		goto out;
-
-	len += le32_to_cpu(*(__le32 *)(&dev->buf[CCID_LENGTH_OFFSET]));
-	if (len > sizeof(dev->buf)) {
-		ret = -EIO;
-		goto out;
-	}
-	ret = read_chunk(dev, CCID_HEADER_SIZE, len);
-	if (ret < 0)
-		goto out;
-
-	if (len < count)
-		count = len;
-
-	if (copy_to_user(buf, dev->buf, count)) {
-		ret = -EFAULT;
-		goto out;
-	}
-
-	ret = count;
-out:
-	mutex_unlock(&dev->lock);
-	return ret;
-}
-
-static ssize_t scr24x_write(struct file *filp, const char __user *buf,
-					size_t count, loff_t *ppos)
-{
-	struct scr24x_dev *dev = filp->private_data;
-	size_t i, y;
-	int ret;
-
-	if (mutex_lock_interruptible(&dev->lock))
-		return -ERESTARTSYS;
-
-	if (!dev->dev) {
-		ret = -ENODEV;
-		goto out;
-	}
-
-	if (count > sizeof(dev->buf)) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	if (copy_from_user(dev->buf, buf, count)) {
-		ret = -EFAULT;
-		goto out;
-	}
-
-	ret = scr24x_wait_ready(dev);
-	if (ret < 0)
-		goto out;
-
-	iowrite8(CMD_START, dev->regs + SCR24X_CMD_STATUS);
-	ret = scr24x_wait_ready(dev);
-	if (ret < 0)
-		goto out;
-
-	for (i = 0; i < count; i += 5) {
-		for (y = 0; y < 5 && i + y < count; y++)
-			iowrite8(dev->buf[i + y], dev->regs + SCR24X_DATA(y));
-
-		iowrite8(CMD_WRITE_BYTE, dev->regs + SCR24X_CMD_STATUS);
-		ret = scr24x_wait_ready(dev);
-		if (ret < 0)
-			goto out;
-	}
-
-	ret = count;
-out:
-	mutex_unlock(&dev->lock);
-	return ret;
-}
-
-static const struct file_operations scr24x_fops = {
-	.owner		= THIS_MODULE,
-	.read		= scr24x_read,
-	.write		= scr24x_write,
-	.open		= scr24x_open,
-	.release	= scr24x_release,
-	.llseek		= no_llseek,
-};
-
-static int scr24x_config_check(struct pcmcia_device *link, void *priv_data)
-{
-	if (resource_size(link->resource[PCMCIA_IOPORT_0]) != 0x11)
-		return -ENODEV;
-	return pcmcia_request_io(link);
-}
-
-static int scr24x_probe(struct pcmcia_device *link)
-{
-	struct scr24x_dev *dev;
-	int ret;
-
-	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
-
-	dev->devno = find_first_zero_bit(scr24x_minors, SCR24X_DEVS);
-	if (dev->devno >= SCR24X_DEVS) {
-		ret = -EBUSY;
-		goto err;
-	}
-
-	mutex_init(&dev->lock);
-	kref_init(&dev->refcnt);
-
-	link->priv = dev;
-	link->config_flags |= CONF_ENABLE_IRQ | CONF_AUTO_SET_IO;
-
-	ret = pcmcia_loop_config(link, scr24x_config_check, NULL);
-	if (ret < 0)
-		goto err;
-
-	dev->dev = &link->dev;
-	dev->regs = devm_ioport_map(&link->dev,
-				link->resource[PCMCIA_IOPORT_0]->start,
-				resource_size(link->resource[PCMCIA_IOPORT_0]));
-	if (!dev->regs) {
-		ret = -EIO;
-		goto err;
-	}
-
-	cdev_init(&dev->c_dev, &scr24x_fops);
-	dev->c_dev.owner = THIS_MODULE;
-	ret = cdev_add(&dev->c_dev, MKDEV(MAJOR(scr24x_devt), dev->devno), 1);
-	if (ret < 0)
-		goto err;
-
-	ret = pcmcia_enable_device(link);
-	if (ret < 0) {
-		pcmcia_disable_device(link);
-		goto err;
-	}
-
-	device_create(scr24x_class, NULL, MKDEV(MAJOR(scr24x_devt), dev->devno),
-		      NULL, "scr24x%d", dev->devno);
-
-	dev_info(&link->dev, "SCR24x Chip Card Interface\n");
-	return 0;
-
-err:
-	if (dev->devno < SCR24X_DEVS)
-		clear_bit(dev->devno, scr24x_minors);
-	kfree (dev);
-	return ret;
-}
-
-static void scr24x_remove(struct pcmcia_device *link)
-{
-	struct scr24x_dev *dev = (struct scr24x_dev *)link->priv;
-
-	device_destroy(scr24x_class, MKDEV(MAJOR(scr24x_devt), dev->devno));
-	mutex_lock(&dev->lock);
-	pcmcia_disable_device(link);
-	cdev_del(&dev->c_dev);
-	clear_bit(dev->devno, scr24x_minors);
-	dev->dev = NULL;
-	mutex_unlock(&dev->lock);
-
-	kref_put(&dev->refcnt, scr24x_delete);
-}
-
-static const struct pcmcia_device_id scr24x_ids[] = {
-	PCMCIA_DEVICE_PROD_ID12("HP", "PC Card Smart Card Reader",
-					0x53cb94f9, 0xbfdf89a5),
-	PCMCIA_DEVICE_PROD_ID1("SCR241 PCMCIA", 0x6271efa3),
-	PCMCIA_DEVICE_PROD_ID1("SCR243 PCMCIA", 0x2054e8de),
-	PCMCIA_DEVICE_PROD_ID1("SCR24x PCMCIA", 0x54a33665),
-	PCMCIA_DEVICE_NULL
-};
-MODULE_DEVICE_TABLE(pcmcia, scr24x_ids);
-
-static struct pcmcia_driver scr24x_driver = {
-	.owner		= THIS_MODULE,
-	.name		= "scr24x_cs",
-	.probe		= scr24x_probe,
-	.remove		= scr24x_remove,
-	.id_table	= scr24x_ids,
-};
-
-static int __init scr24x_init(void)
-{
-	int ret;
-
-	scr24x_class = class_create(THIS_MODULE, "scr24x");
-	if (IS_ERR(scr24x_class))
-		return PTR_ERR(scr24x_class);
-
-	ret = alloc_chrdev_region(&scr24x_devt, 0, SCR24X_DEVS, "scr24x");
-	if (ret < 0)  {
-		class_destroy(scr24x_class);
-		return ret;
-	}
-
-	ret = pcmcia_register_driver(&scr24x_driver);
-	if (ret < 0) {
-		unregister_chrdev_region(scr24x_devt, SCR24X_DEVS);
-		class_destroy(scr24x_class);
-	}
-
-	return ret;
-}
-
-static void __exit scr24x_exit(void)
-{
-	pcmcia_unregister_driver(&scr24x_driver);
-	unregister_chrdev_region(scr24x_devt, SCR24X_DEVS);
-	class_destroy(scr24x_class);
-}
-
-module_init(scr24x_init);
-module_exit(scr24x_exit);
-
-MODULE_AUTHOR("Lubomir Rintel");
-MODULE_DESCRIPTION("SCR24x PCMCIA Smart Card Reader Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/char/pcmcia/synclink_cs.c b/drivers/char/pcmcia/synclink_cs.c
deleted file mode 100644
index 6ddfeb2fe98f..000000000000
--- a/drivers/char/pcmcia/synclink_cs.c
+++ /dev/null
@@ -1,4290 +0,0 @@
-/*
- * linux/drivers/char/pcmcia/synclink_cs.c
- *
- * $Id: synclink_cs.c,v 4.34 2005/09/08 13:20:54 paulkf Exp $
- *
- * Device driver for Microgate SyncLink PC Card
- * multiprotocol serial adapter.
- *
- * written by Paul Fulghum for Microgate Corporation
- * paulkf@microgate.com
- *
- * Microgate and SyncLink are trademarks of Microgate Corporation
- *
- * This code is released under the GNU General Public License (GPL)
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
- * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
- * OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#define VERSION(ver,rel,seq) (((ver)<<16) | ((rel)<<8) | (seq))
-#if defined(__i386__)
-#  define BREAKPOINT() asm("   int $3");
-#else
-#  define BREAKPOINT() { }
-#endif
-
-#define MAX_DEVICE_COUNT 4
-
-#include <linux/module.h>
-#include <linux/errno.h>
-#include <linux/signal.h>
-#include <linux/sched.h>
-#include <linux/timer.h>
-#include <linux/time.h>
-#include <linux/interrupt.h>
-#include <linux/tty.h>
-#include <linux/tty_flip.h>
-#include <linux/serial.h>
-#include <linux/major.h>
-#include <linux/string.h>
-#include <linux/fcntl.h>
-#include <linux/ptrace.h>
-#include <linux/ioport.h>
-#include <linux/mm.h>
-#include <linux/seq_file.h>
-#include <linux/slab.h>
-#include <linux/netdevice.h>
-#include <linux/vmalloc.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/ioctl.h>
-#include <linux/synclink.h>
-
-#include <asm/io.h>
-#include <asm/irq.h>
-#include <asm/dma.h>
-#include <linux/bitops.h>
-#include <asm/types.h>
-#include <linux/termios.h>
-#include <linux/workqueue.h>
-#include <linux/hdlc.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/cisreg.h>
-#include <pcmcia/ds.h>
-
-#if defined(CONFIG_HDLC) || (defined(CONFIG_HDLC_MODULE) && defined(CONFIG_SYNCLINK_CS_MODULE))
-#define SYNCLINK_GENERIC_HDLC 1
-#else
-#define SYNCLINK_GENERIC_HDLC 0
-#endif
-
-#define GET_USER(error,value,addr) error = get_user(value,addr)
-#define COPY_FROM_USER(error,dest,src,size) error = copy_from_user(dest,src,size) ? -EFAULT : 0
-#define PUT_USER(error,value,addr) error = put_user(value,addr)
-#define COPY_TO_USER(error,dest,src,size) error = copy_to_user(dest,src,size) ? -EFAULT : 0
-
-#include <linux/uaccess.h>
-
-static MGSL_PARAMS default_params = {
-	MGSL_MODE_HDLC,			/* unsigned long mode */
-	0,				/* unsigned char loopback; */
-	HDLC_FLAG_UNDERRUN_ABORT15,	/* unsigned short flags; */
-	HDLC_ENCODING_NRZI_SPACE,	/* unsigned char encoding; */
-	0,				/* unsigned long clock_speed; */
-	0xff,				/* unsigned char addr_filter; */
-	HDLC_CRC_16_CCITT,		/* unsigned short crc_type; */
-	HDLC_PREAMBLE_LENGTH_8BITS,	/* unsigned char preamble_length; */
-	HDLC_PREAMBLE_PATTERN_NONE,	/* unsigned char preamble; */
-	9600,				/* unsigned long data_rate; */
-	8,				/* unsigned char data_bits; */
-	1,				/* unsigned char stop_bits; */
-	ASYNC_PARITY_NONE		/* unsigned char parity; */
-};
-
-typedef struct {
-	int count;
-	unsigned char status;
-	char data[1];
-} RXBUF;
-
-/* The queue of BH actions to be performed */
-
-#define BH_RECEIVE  1
-#define BH_TRANSMIT 2
-#define BH_STATUS   4
-
-#define IO_PIN_SHUTDOWN_LIMIT 100
-
-#define RELEVANT_IFLAG(iflag) (iflag & (IGNBRK|BRKINT|IGNPAR|PARMRK|INPCK))
-
-struct _input_signal_events {
-	int	ri_up;
-	int	ri_down;
-	int	dsr_up;
-	int	dsr_down;
-	int	dcd_up;
-	int	dcd_down;
-	int	cts_up;
-	int	cts_down;
-};
-
-
-/*
- * Device instance data structure
- */
-
-typedef struct _mgslpc_info {
-	struct tty_port		port;
-	void *if_ptr;	/* General purpose pointer (used by SPPP) */
-	int			magic;
-	int			line;
-
-	struct mgsl_icount	icount;
-
-	int			timeout;
-	int			x_char;		/* xon/xoff character */
-	unsigned char		read_status_mask;
-	unsigned char		ignore_status_mask;
-
-	unsigned char *tx_buf;
-	int            tx_put;
-	int            tx_get;
-	int            tx_count;
-
-	/* circular list of fixed length rx buffers */
-
-	unsigned char  *rx_buf;        /* memory allocated for all rx buffers */
-	int            rx_buf_total_size; /* size of memory allocated for rx buffers */
-	int            rx_put;         /* index of next empty rx buffer */
-	int            rx_get;         /* index of next full rx buffer */
-	int            rx_buf_size;    /* size in bytes of single rx buffer */
-	int            rx_buf_count;   /* total number of rx buffers */
-	int            rx_frame_count; /* number of full rx buffers */
-
-	wait_queue_head_t	status_event_wait_q;
-	wait_queue_head_t	event_wait_q;
-	struct timer_list	tx_timer;	/* HDLC transmit timeout timer */
-	struct _mgslpc_info	*next_device;	/* device list link */
-
-	unsigned short imra_value;
-	unsigned short imrb_value;
-	unsigned char  pim_value;
-
-	spinlock_t lock;
-	struct work_struct task;		/* task structure for scheduling bh */
-
-	u32 max_frame_size;
-
-	u32 pending_bh;
-
-	bool bh_running;
-	bool bh_requested;
-
-	int dcd_chkcount; /* check counts to prevent */
-	int cts_chkcount; /* too many IRQs if a signal */
-	int dsr_chkcount; /* is floating */
-	int ri_chkcount;
-
-	bool rx_enabled;
-	bool rx_overflow;
-
-	bool tx_enabled;
-	bool tx_active;
-	bool tx_aborting;
-	u32 idle_mode;
-
-	int if_mode; /* serial interface selection (RS-232, v.35 etc) */
-
-	char device_name[25];		/* device instance name */
-
-	unsigned int io_base;	/* base I/O address of adapter */
-	unsigned int irq_level;
-
-	MGSL_PARAMS params;		/* communications parameters */
-
-	unsigned char serial_signals;	/* current serial signal states */
-
-	bool irq_occurred;		/* for diagnostics use */
-	char testing_irq;
-	unsigned int init_error;	/* startup error (DIAGS)	*/
-
-	char *flag_buf;
-	bool drop_rts_on_tx_done;
-
-	struct	_input_signal_events	input_signal_events;
-
-	/* PCMCIA support */
-	struct pcmcia_device	*p_dev;
-	int		      stop;
-
-	/* SPPP/Cisco HDLC device parts */
-	int netcount;
-	spinlock_t netlock;
-
-#if SYNCLINK_GENERIC_HDLC
-	struct net_device *netdev;
-#endif
-
-} MGSLPC_INFO;
-
-#define MGSLPC_MAGIC 0x5402
-
-/*
- * The size of the serial xmit buffer is 1 page, or 4096 bytes
- */
-#define TXBUFSIZE 4096
-
-
-#define CHA     0x00   /* channel A offset */
-#define CHB     0x40   /* channel B offset */
-
-/*
- *  FIXME: PPC has PVR defined in asm/reg.h.  For now we just undef it.
- */
-#undef PVR
-
-#define RXFIFO  0
-#define TXFIFO  0
-#define STAR    0x20
-#define CMDR    0x20
-#define RSTA    0x21
-#define PRE     0x21
-#define MODE    0x22
-#define TIMR    0x23
-#define XAD1    0x24
-#define XAD2    0x25
-#define RAH1    0x26
-#define RAH2    0x27
-#define DAFO    0x27
-#define RAL1    0x28
-#define RFC     0x28
-#define RHCR    0x29
-#define RAL2    0x29
-#define RBCL    0x2a
-#define XBCL    0x2a
-#define RBCH    0x2b
-#define XBCH    0x2b
-#define CCR0    0x2c
-#define CCR1    0x2d
-#define CCR2    0x2e
-#define CCR3    0x2f
-#define VSTR    0x34
-#define BGR     0x34
-#define RLCR    0x35
-#define AML     0x36
-#define AMH     0x37
-#define GIS     0x38
-#define IVA     0x38
-#define IPC     0x39
-#define ISR     0x3a
-#define IMR     0x3a
-#define PVR     0x3c
-#define PIS     0x3d
-#define PIM     0x3d
-#define PCR     0x3e
-#define CCR4    0x3f
-
-// IMR/ISR
-
-#define IRQ_BREAK_ON    BIT15   // rx break detected
-#define IRQ_DATAOVERRUN BIT14	// receive data overflow
-#define IRQ_ALLSENT     BIT13	// all sent
-#define IRQ_UNDERRUN    BIT12	// transmit data underrun
-#define IRQ_TIMER       BIT11	// timer interrupt
-#define IRQ_CTS         BIT10	// CTS status change
-#define IRQ_TXREPEAT    BIT9	// tx message repeat
-#define IRQ_TXFIFO      BIT8	// transmit pool ready
-#define IRQ_RXEOM       BIT7	// receive message end
-#define IRQ_EXITHUNT    BIT6	// receive frame start
-#define IRQ_RXTIME      BIT6    // rx char timeout
-#define IRQ_DCD         BIT2	// carrier detect status change
-#define IRQ_OVERRUN     BIT1	// receive frame overflow
-#define IRQ_RXFIFO      BIT0	// receive pool full
-
-// STAR
-
-#define XFW   BIT6		// transmit FIFO write enable
-#define CEC   BIT2		// command executing
-#define CTS   BIT1		// CTS state
-
-#define PVR_DTR      BIT0
-#define PVR_DSR      BIT1
-#define PVR_RI       BIT2
-#define PVR_AUTOCTS  BIT3
-#define PVR_RS232    0x20   /* 0010b */
-#define PVR_V35      0xe0   /* 1110b */
-#define PVR_RS422    0x40   /* 0100b */
-
-/* Register access functions */
-
-#define write_reg(info, reg, val) outb((val),(info)->io_base + (reg))
-#define read_reg(info, reg) inb((info)->io_base + (reg))
-
-#define read_reg16(info, reg) inw((info)->io_base + (reg))
-#define write_reg16(info, reg, val) outw((val), (info)->io_base + (reg))
-
-#define set_reg_bits(info, reg, mask) \
-	write_reg(info, (reg), \
-		 (unsigned char) (read_reg(info, (reg)) | (mask)))
-#define clear_reg_bits(info, reg, mask) \
-	write_reg(info, (reg), \
-		 (unsigned char) (read_reg(info, (reg)) & ~(mask)))
-/*
- * interrupt enable/disable routines
- */
-static void irq_disable(MGSLPC_INFO *info, unsigned char channel, unsigned short mask)
-{
-	if (channel == CHA) {
-		info->imra_value |= mask;
-		write_reg16(info, CHA + IMR, info->imra_value);
-	} else {
-		info->imrb_value |= mask;
-		write_reg16(info, CHB + IMR, info->imrb_value);
-	}
-}
-static void irq_enable(MGSLPC_INFO *info, unsigned char channel, unsigned short mask)
-{
-	if (channel == CHA) {
-		info->imra_value &= ~mask;
-		write_reg16(info, CHA + IMR, info->imra_value);
-	} else {
-		info->imrb_value &= ~mask;
-		write_reg16(info, CHB + IMR, info->imrb_value);
-	}
-}
-
-#define port_irq_disable(info, mask) \
-	{ info->pim_value |= (mask); write_reg(info, PIM, info->pim_value); }
-
-#define port_irq_enable(info, mask) \
-	{ info->pim_value &= ~(mask); write_reg(info, PIM, info->pim_value); }
-
-static void rx_start(MGSLPC_INFO *info);
-static void rx_stop(MGSLPC_INFO *info);
-
-static void tx_start(MGSLPC_INFO *info, struct tty_struct *tty);
-static void tx_stop(MGSLPC_INFO *info);
-static void tx_set_idle(MGSLPC_INFO *info);
-
-static void get_signals(MGSLPC_INFO *info);
-static void set_signals(MGSLPC_INFO *info);
-
-static void reset_device(MGSLPC_INFO *info);
-
-static void hdlc_mode(MGSLPC_INFO *info);
-static void async_mode(MGSLPC_INFO *info);
-
-static void tx_timeout(struct timer_list *t);
-
-static bool carrier_raised(struct tty_port *port);
-static void dtr_rts(struct tty_port *port, bool active);
-
-#if SYNCLINK_GENERIC_HDLC
-#define dev_to_port(D) (dev_to_hdlc(D)->priv)
-static void hdlcdev_tx_done(MGSLPC_INFO *info);
-static void hdlcdev_rx(MGSLPC_INFO *info, char *buf, int size);
-static int  hdlcdev_init(MGSLPC_INFO *info);
-static void hdlcdev_exit(MGSLPC_INFO *info);
-#endif
-
-static void trace_block(MGSLPC_INFO *info,const char* data, int count, int xmit);
-
-static bool register_test(MGSLPC_INFO *info);
-static bool irq_test(MGSLPC_INFO *info);
-static int adapter_test(MGSLPC_INFO *info);
-
-static int claim_resources(MGSLPC_INFO *info);
-static void release_resources(MGSLPC_INFO *info);
-static int mgslpc_add_device(MGSLPC_INFO *info);
-static void mgslpc_remove_device(MGSLPC_INFO *info);
-
-static bool rx_get_frame(MGSLPC_INFO *info, struct tty_struct *tty);
-static void rx_reset_buffers(MGSLPC_INFO *info);
-static int  rx_alloc_buffers(MGSLPC_INFO *info);
-static void rx_free_buffers(MGSLPC_INFO *info);
-
-static irqreturn_t mgslpc_isr(int irq, void *dev_id);
-
-/*
- * Bottom half interrupt handlers
- */
-static void bh_handler(struct work_struct *work);
-static void bh_transmit(MGSLPC_INFO *info, struct tty_struct *tty);
-static void bh_status(MGSLPC_INFO *info);
-
-/*
- * ioctl handlers
- */
-static int tiocmget(struct tty_struct *tty);
-static int tiocmset(struct tty_struct *tty,
-					unsigned int set, unsigned int clear);
-static int get_stats(MGSLPC_INFO *info, struct mgsl_icount __user *user_icount);
-static int get_params(MGSLPC_INFO *info, MGSL_PARAMS __user *user_params);
-static int set_params(MGSLPC_INFO *info, MGSL_PARAMS __user *new_params, struct tty_struct *tty);
-static int get_txidle(MGSLPC_INFO *info, int __user *idle_mode);
-static int set_txidle(MGSLPC_INFO *info, int idle_mode);
-static int set_txenable(MGSLPC_INFO *info, int enable, struct tty_struct *tty);
-static int tx_abort(MGSLPC_INFO *info);
-static int set_rxenable(MGSLPC_INFO *info, int enable);
-static int wait_events(MGSLPC_INFO *info, int __user *mask);
-
-static MGSLPC_INFO *mgslpc_device_list = NULL;
-static int mgslpc_device_count = 0;
-
-/*
- * Set this param to non-zero to load eax with the
- * .text section address and breakpoint on module load.
- * This is useful for use with gdb and add-symbol-file command.
- */
-static bool break_on_load;
-
-/*
- * Driver major number, defaults to zero to get auto
- * assigned major number. May be forced as module parameter.
- */
-static int ttymajor=0;
-
-static int debug_level = 0;
-static int maxframe[MAX_DEVICE_COUNT] = {0,};
-
-module_param(break_on_load, bool, 0);
-module_param(ttymajor, int, 0);
-module_param(debug_level, int, 0);
-module_param_array(maxframe, int, NULL, 0);
-
-MODULE_LICENSE("GPL");
-
-static char *driver_name = "SyncLink PC Card driver";
-static char *driver_version = "$Revision: 4.34 $";
-
-static struct tty_driver *serial_driver;
-
-/* number of characters left in xmit buffer before we ask for more */
-#define WAKEUP_CHARS 256
-
-static void mgslpc_change_params(MGSLPC_INFO *info, struct tty_struct *tty);
-static void mgslpc_wait_until_sent(struct tty_struct *tty, int timeout);
-
-/* PCMCIA prototypes */
-
-static int mgslpc_config(struct pcmcia_device *link);
-static void mgslpc_release(u_long arg);
-static void mgslpc_detach(struct pcmcia_device *p_dev);
-
-/*
- * 1st function defined in .text section. Calling this function in
- * init_module() followed by a breakpoint allows a remote debugger
- * (gdb) to get the .text address for the add-symbol-file command.
- * This allows remote debugging of dynamically loadable modules.
- */
-static void* mgslpc_get_text_ptr(void)
-{
-	return mgslpc_get_text_ptr;
-}
-
-/*
- * line discipline callback wrappers
- *
- * The wrappers maintain line discipline references
- * while calling into the line discipline.
- *
- * ldisc_receive_buf  - pass receive data to line discipline
- */
-
-static void ldisc_receive_buf(struct tty_struct *tty,
-			      const __u8 *data, char *flags, int count)
-{
-	struct tty_ldisc *ld;
-	if (!tty)
-		return;
-	ld = tty_ldisc_ref(tty);
-	if (ld) {
-		if (ld->ops->receive_buf)
-			ld->ops->receive_buf(tty, data, flags, count);
-		tty_ldisc_deref(ld);
-	}
-}
-
-static const struct tty_port_operations mgslpc_port_ops = {
-	.carrier_raised = carrier_raised,
-	.dtr_rts = dtr_rts
-};
-
-static int mgslpc_probe(struct pcmcia_device *link)
-{
-	MGSLPC_INFO *info;
-	int ret;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("mgslpc_attach\n");
-
-	info = kzalloc(sizeof(MGSLPC_INFO), GFP_KERNEL);
-	if (!info) {
-		printk("Error can't allocate device instance data\n");
-		return -ENOMEM;
-	}
-
-	info->magic = MGSLPC_MAGIC;
-	tty_port_init(&info->port);
-	info->port.ops = &mgslpc_port_ops;
-	INIT_WORK(&info->task, bh_handler);
-	info->max_frame_size = 4096;
-	init_waitqueue_head(&info->status_event_wait_q);
-	init_waitqueue_head(&info->event_wait_q);
-	spin_lock_init(&info->lock);
-	spin_lock_init(&info->netlock);
-	memcpy(&info->params,&default_params,sizeof(MGSL_PARAMS));
-	info->idle_mode = HDLC_TXIDLE_FLAGS;
-	info->imra_value = 0xffff;
-	info->imrb_value = 0xffff;
-	info->pim_value = 0xff;
-
-	info->p_dev = link;
-	link->priv = info;
-
-	/* Initialize the struct pcmcia_device structure */
-
-	ret = mgslpc_config(link);
-	if (ret != 0)
-		goto failed;
-
-	ret = mgslpc_add_device(info);
-	if (ret != 0)
-		goto failed_release;
-
-	return 0;
-
-failed_release:
-	mgslpc_release((u_long)link);
-failed:
-	tty_port_destroy(&info->port);
-	kfree(info);
-	return ret;
-}
-
-/* Card has been inserted.
- */
-
-static int mgslpc_ioprobe(struct pcmcia_device *p_dev, void *priv_data)
-{
-	return pcmcia_request_io(p_dev);
-}
-
-static int mgslpc_config(struct pcmcia_device *link)
-{
-	MGSLPC_INFO *info = link->priv;
-	int ret;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("mgslpc_config(0x%p)\n", link);
-
-	link->config_flags |= CONF_ENABLE_IRQ | CONF_AUTO_SET_IO;
-
-	ret = pcmcia_loop_config(link, mgslpc_ioprobe, NULL);
-	if (ret != 0)
-		goto failed;
-
-	link->config_index = 8;
-	link->config_regs = PRESENT_OPTION;
-
-	ret = pcmcia_request_irq(link, mgslpc_isr);
-	if (ret)
-		goto failed;
-	ret = pcmcia_enable_device(link);
-	if (ret)
-		goto failed;
-
-	info->io_base = link->resource[0]->start;
-	info->irq_level = link->irq;
-	return 0;
-
-failed:
-	mgslpc_release((u_long)link);
-	return -ENODEV;
-}
-
-/* Card has been removed.
- * Unregister device and release PCMCIA configuration.
- * If device is open, postpone until it is closed.
- */
-static void mgslpc_release(u_long arg)
-{
-	struct pcmcia_device *link = (struct pcmcia_device *)arg;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("mgslpc_release(0x%p)\n", link);
-
-	pcmcia_disable_device(link);
-}
-
-static void mgslpc_detach(struct pcmcia_device *link)
-{
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("mgslpc_detach(0x%p)\n", link);
-
-	((MGSLPC_INFO *)link->priv)->stop = 1;
-	mgslpc_release((u_long)link);
-
-	mgslpc_remove_device((MGSLPC_INFO *)link->priv);
-}
-
-static int mgslpc_suspend(struct pcmcia_device *link)
-{
-	MGSLPC_INFO *info = link->priv;
-
-	info->stop = 1;
-
-	return 0;
-}
-
-static int mgslpc_resume(struct pcmcia_device *link)
-{
-	MGSLPC_INFO *info = link->priv;
-
-	info->stop = 0;
-
-	return 0;
-}
-
-
-static inline bool mgslpc_paranoia_check(MGSLPC_INFO *info,
-					char *name, const char *routine)
-{
-#ifdef MGSLPC_PARANOIA_CHECK
-	static const char *badmagic =
-		"Warning: bad magic number for mgsl struct (%s) in %s\n";
-	static const char *badinfo =
-		"Warning: null mgslpc_info for (%s) in %s\n";
-
-	if (!info) {
-		printk(badinfo, name, routine);
-		return true;
-	}
-	if (info->magic != MGSLPC_MAGIC) {
-		printk(badmagic, name, routine);
-		return true;
-	}
-#else
-	if (!info)
-		return true;
-#endif
-	return false;
-}
-
-
-#define CMD_RXFIFO      BIT7	// release current rx FIFO
-#define CMD_RXRESET     BIT6	// receiver reset
-#define CMD_RXFIFO_READ BIT5
-#define CMD_START_TIMER BIT4
-#define CMD_TXFIFO      BIT3	// release current tx FIFO
-#define CMD_TXEOM       BIT1	// transmit end message
-#define CMD_TXRESET     BIT0	// transmit reset
-
-static bool wait_command_complete(MGSLPC_INFO *info, unsigned char channel)
-{
-	int i = 0;
-	/* wait for command completion */
-	while (read_reg(info, (unsigned char)(channel+STAR)) & BIT2) {
-		udelay(1);
-		if (i++ == 1000)
-			return false;
-	}
-	return true;
-}
-
-static void issue_command(MGSLPC_INFO *info, unsigned char channel, unsigned char cmd)
-{
-	wait_command_complete(info, channel);
-	write_reg(info, (unsigned char) (channel + CMDR), cmd);
-}
-
-static void tx_pause(struct tty_struct *tty)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (mgslpc_paranoia_check(info, tty->name, "tx_pause"))
-		return;
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("tx_pause(%s)\n", info->device_name);
-
-	spin_lock_irqsave(&info->lock, flags);
-	if (info->tx_enabled)
-		tx_stop(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-}
-
-static void tx_release(struct tty_struct *tty)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (mgslpc_paranoia_check(info, tty->name, "tx_release"))
-		return;
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("tx_release(%s)\n", info->device_name);
-
-	spin_lock_irqsave(&info->lock, flags);
-	if (!info->tx_enabled)
-		tx_start(info, tty);
-	spin_unlock_irqrestore(&info->lock, flags);
-}
-
-/* Return next bottom half action to perform.
- * or 0 if nothing to do.
- */
-static int bh_action(MGSLPC_INFO *info)
-{
-	unsigned long flags;
-	int rc = 0;
-
-	spin_lock_irqsave(&info->lock, flags);
-
-	if (info->pending_bh & BH_RECEIVE) {
-		info->pending_bh &= ~BH_RECEIVE;
-		rc = BH_RECEIVE;
-	} else if (info->pending_bh & BH_TRANSMIT) {
-		info->pending_bh &= ~BH_TRANSMIT;
-		rc = BH_TRANSMIT;
-	} else if (info->pending_bh & BH_STATUS) {
-		info->pending_bh &= ~BH_STATUS;
-		rc = BH_STATUS;
-	}
-
-	if (!rc) {
-		/* Mark BH routine as complete */
-		info->bh_running = false;
-		info->bh_requested = false;
-	}
-
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	return rc;
-}
-
-static void bh_handler(struct work_struct *work)
-{
-	MGSLPC_INFO *info = container_of(work, MGSLPC_INFO, task);
-	struct tty_struct *tty;
-	int action;
-
-	if (debug_level >= DEBUG_LEVEL_BH)
-		printk("%s(%d):bh_handler(%s) entry\n",
-			__FILE__,__LINE__,info->device_name);
-
-	info->bh_running = true;
-	tty = tty_port_tty_get(&info->port);
-
-	while((action = bh_action(info)) != 0) {
-
-		/* Process work item */
-		if (debug_level >= DEBUG_LEVEL_BH)
-			printk("%s(%d):bh_handler() work item action=%d\n",
-				__FILE__,__LINE__,action);
-
-		switch (action) {
-
-		case BH_RECEIVE:
-			while(rx_get_frame(info, tty));
-			break;
-		case BH_TRANSMIT:
-			bh_transmit(info, tty);
-			break;
-		case BH_STATUS:
-			bh_status(info);
-			break;
-		default:
-			/* unknown work item ID */
-			printk("Unknown work item ID=%08X!\n", action);
-			break;
-		}
-	}
-
-	tty_kref_put(tty);
-	if (debug_level >= DEBUG_LEVEL_BH)
-		printk("%s(%d):bh_handler(%s) exit\n",
-			__FILE__,__LINE__,info->device_name);
-}
-
-static void bh_transmit(MGSLPC_INFO *info, struct tty_struct *tty)
-{
-	if (debug_level >= DEBUG_LEVEL_BH)
-		printk("bh_transmit() entry on %s\n", info->device_name);
-
-	if (tty)
-		tty_wakeup(tty);
-}
-
-static void bh_status(MGSLPC_INFO *info)
-{
-	info->ri_chkcount = 0;
-	info->dsr_chkcount = 0;
-	info->dcd_chkcount = 0;
-	info->cts_chkcount = 0;
-}
-
-/* eom: non-zero = end of frame */
-static void rx_ready_hdlc(MGSLPC_INFO *info, int eom)
-{
-	unsigned char data[2];
-	unsigned char fifo_count, read_count, i;
-	RXBUF *buf = (RXBUF*)(info->rx_buf + (info->rx_put * info->rx_buf_size));
-
-	if (debug_level >= DEBUG_LEVEL_ISR)
-		printk("%s(%d):rx_ready_hdlc(eom=%d)\n", __FILE__, __LINE__, eom);
-
-	if (!info->rx_enabled)
-		return;
-
-	if (info->rx_frame_count >= info->rx_buf_count) {
-		/* no more free buffers */
-		issue_command(info, CHA, CMD_RXRESET);
-		info->pending_bh |= BH_RECEIVE;
-		info->rx_overflow = true;
-		info->icount.buf_overrun++;
-		return;
-	}
-
-	if (eom) {
-		/* end of frame, get FIFO count from RBCL register */
-		fifo_count = (unsigned char)(read_reg(info, CHA+RBCL) & 0x1f);
-		if (fifo_count == 0)
-			fifo_count = 32;
-	} else
-		fifo_count = 32;
-
-	do {
-		if (fifo_count == 1) {
-			read_count = 1;
-			data[0] = read_reg(info, CHA + RXFIFO);
-		} else {
-			read_count = 2;
-			*((unsigned short *) data) = read_reg16(info, CHA + RXFIFO);
-		}
-		fifo_count -= read_count;
-		if (!fifo_count && eom)
-			buf->status = data[--read_count];
-
-		for (i = 0; i < read_count; i++) {
-			if (buf->count >= info->max_frame_size) {
-				/* frame too large, reset receiver and reset current buffer */
-				issue_command(info, CHA, CMD_RXRESET);
-				buf->count = 0;
-				return;
-			}
-			*(buf->data + buf->count) = data[i];
-			buf->count++;
-		}
-	} while (fifo_count);
-
-	if (eom) {
-		info->pending_bh |= BH_RECEIVE;
-		info->rx_frame_count++;
-		info->rx_put++;
-		if (info->rx_put >= info->rx_buf_count)
-			info->rx_put = 0;
-	}
-	issue_command(info, CHA, CMD_RXFIFO);
-}
-
-static void rx_ready_async(MGSLPC_INFO *info, int tcd)
-{
-	struct tty_port *port = &info->port;
-	unsigned char data, status, flag;
-	int fifo_count;
-	int work = 0;
-	struct mgsl_icount *icount = &info->icount;
-
-	if (tcd) {
-		/* early termination, get FIFO count from RBCL register */
-		fifo_count = (unsigned char)(read_reg(info, CHA+RBCL) & 0x1f);
-
-		/* Zero fifo count could mean 0 or 32 bytes available.
-		 * If BIT5 of STAR is set then at least 1 byte is available.
-		 */
-		if (!fifo_count && (read_reg(info,CHA+STAR) & BIT5))
-			fifo_count = 32;
-	} else
-		fifo_count = 32;
-
-	tty_buffer_request_room(port, fifo_count);
-	/* Flush received async data to receive data buffer. */
-	while (fifo_count) {
-		data   = read_reg(info, CHA + RXFIFO);
-		status = read_reg(info, CHA + RXFIFO);
-		fifo_count -= 2;
-
-		icount->rx++;
-		flag = TTY_NORMAL;
-
-		// if no frameing/crc error then save data
-		// BIT7:parity error
-		// BIT6:framing error
-
-		if (status & (BIT7 | BIT6)) {
-			if (status & BIT7)
-				icount->parity++;
-			else
-				icount->frame++;
-
-			/* discard char if tty control flags say so */
-			if (status & info->ignore_status_mask)
-				continue;
-
-			status &= info->read_status_mask;
-
-			if (status & BIT7)
-				flag = TTY_PARITY;
-			else if (status & BIT6)
-				flag = TTY_FRAME;
-		}
-		work += tty_insert_flip_char(port, data, flag);
-	}
-	issue_command(info, CHA, CMD_RXFIFO);
-
-	if (debug_level >= DEBUG_LEVEL_ISR) {
-		printk("%s(%d):rx_ready_async",
-			__FILE__,__LINE__);
-		printk("%s(%d):rx=%d brk=%d parity=%d frame=%d overrun=%d\n",
-			__FILE__,__LINE__,icount->rx,icount->brk,
-			icount->parity,icount->frame,icount->overrun);
-	}
-
-	if (work)
-		tty_flip_buffer_push(port);
-}
-
-
-static void tx_done(MGSLPC_INFO *info, struct tty_struct *tty)
-{
-	if (!info->tx_active)
-		return;
-
-	info->tx_active = false;
-	info->tx_aborting = false;
-
-	if (info->params.mode == MGSL_MODE_ASYNC)
-		return;
-
-	info->tx_count = info->tx_put = info->tx_get = 0;
-	del_timer(&info->tx_timer);
-
-	if (info->drop_rts_on_tx_done) {
-		get_signals(info);
-		if (info->serial_signals & SerialSignal_RTS) {
-			info->serial_signals &= ~SerialSignal_RTS;
-			set_signals(info);
-		}
-		info->drop_rts_on_tx_done = false;
-	}
-
-#if SYNCLINK_GENERIC_HDLC
-	if (info->netcount)
-		hdlcdev_tx_done(info);
-	else
-#endif
-	{
-		if (tty && (tty->flow.stopped || tty->hw_stopped)) {
-			tx_stop(info);
-			return;
-		}
-		info->pending_bh |= BH_TRANSMIT;
-	}
-}
-
-static void tx_ready(MGSLPC_INFO *info, struct tty_struct *tty)
-{
-	unsigned char fifo_count = 32;
-	int c;
-
-	if (debug_level >= DEBUG_LEVEL_ISR)
-		printk("%s(%d):tx_ready(%s)\n", __FILE__, __LINE__, info->device_name);
-
-	if (info->params.mode == MGSL_MODE_HDLC) {
-		if (!info->tx_active)
-			return;
-	} else {
-		if (tty && (tty->flow.stopped || tty->hw_stopped)) {
-			tx_stop(info);
-			return;
-		}
-		if (!info->tx_count)
-			info->tx_active = false;
-	}
-
-	if (!info->tx_count)
-		return;
-
-	while (info->tx_count && fifo_count) {
-		c = min(2, min_t(int, fifo_count, min(info->tx_count, TXBUFSIZE - info->tx_get)));
-
-		if (c == 1) {
-			write_reg(info, CHA + TXFIFO, *(info->tx_buf + info->tx_get));
-		} else {
-			write_reg16(info, CHA + TXFIFO,
-					  *((unsigned short*)(info->tx_buf + info->tx_get)));
-		}
-		info->tx_count -= c;
-		info->tx_get = (info->tx_get + c) & (TXBUFSIZE - 1);
-		fifo_count -= c;
-	}
-
-	if (info->params.mode == MGSL_MODE_ASYNC) {
-		if (info->tx_count < WAKEUP_CHARS)
-			info->pending_bh |= BH_TRANSMIT;
-		issue_command(info, CHA, CMD_TXFIFO);
-	} else {
-		if (info->tx_count)
-			issue_command(info, CHA, CMD_TXFIFO);
-		else
-			issue_command(info, CHA, CMD_TXFIFO + CMD_TXEOM);
-	}
-}
-
-static void cts_change(MGSLPC_INFO *info, struct tty_struct *tty)
-{
-	get_signals(info);
-	if ((info->cts_chkcount)++ >= IO_PIN_SHUTDOWN_LIMIT)
-		irq_disable(info, CHB, IRQ_CTS);
-	info->icount.cts++;
-	if (info->serial_signals & SerialSignal_CTS)
-		info->input_signal_events.cts_up++;
-	else
-		info->input_signal_events.cts_down++;
-	wake_up_interruptible(&info->status_event_wait_q);
-	wake_up_interruptible(&info->event_wait_q);
-
-	if (tty && tty_port_cts_enabled(&info->port)) {
-		if (tty->hw_stopped) {
-			if (info->serial_signals & SerialSignal_CTS) {
-				if (debug_level >= DEBUG_LEVEL_ISR)
-					printk("CTS tx start...");
-				tty->hw_stopped = 0;
-				tx_start(info, tty);
-				info->pending_bh |= BH_TRANSMIT;
-				return;
-			}
-		} else {
-			if (!(info->serial_signals & SerialSignal_CTS)) {
-				if (debug_level >= DEBUG_LEVEL_ISR)
-					printk("CTS tx stop...");
-				tty->hw_stopped = 1;
-				tx_stop(info);
-			}
-		}
-	}
-	info->pending_bh |= BH_STATUS;
-}
-
-static void dcd_change(MGSLPC_INFO *info, struct tty_struct *tty)
-{
-	get_signals(info);
-	if ((info->dcd_chkcount)++ >= IO_PIN_SHUTDOWN_LIMIT)
-		irq_disable(info, CHB, IRQ_DCD);
-	info->icount.dcd++;
-	if (info->serial_signals & SerialSignal_DCD) {
-		info->input_signal_events.dcd_up++;
-	}
-	else
-		info->input_signal_events.dcd_down++;
-#if SYNCLINK_GENERIC_HDLC
-	if (info->netcount) {
-		if (info->serial_signals & SerialSignal_DCD)
-			netif_carrier_on(info->netdev);
-		else
-			netif_carrier_off(info->netdev);
-	}
-#endif
-	wake_up_interruptible(&info->status_event_wait_q);
-	wake_up_interruptible(&info->event_wait_q);
-
-	if (tty_port_check_carrier(&info->port)) {
-		if (debug_level >= DEBUG_LEVEL_ISR)
-			printk("%s CD now %s...", info->device_name,
-			       (info->serial_signals & SerialSignal_DCD) ? "on" : "off");
-		if (info->serial_signals & SerialSignal_DCD)
-			wake_up_interruptible(&info->port.open_wait);
-		else {
-			if (debug_level >= DEBUG_LEVEL_ISR)
-				printk("doing serial hangup...");
-			if (tty)
-				tty_hangup(tty);
-		}
-	}
-	info->pending_bh |= BH_STATUS;
-}
-
-static void dsr_change(MGSLPC_INFO *info)
-{
-	get_signals(info);
-	if ((info->dsr_chkcount)++ >= IO_PIN_SHUTDOWN_LIMIT)
-		port_irq_disable(info, PVR_DSR);
-	info->icount.dsr++;
-	if (info->serial_signals & SerialSignal_DSR)
-		info->input_signal_events.dsr_up++;
-	else
-		info->input_signal_events.dsr_down++;
-	wake_up_interruptible(&info->status_event_wait_q);
-	wake_up_interruptible(&info->event_wait_q);
-	info->pending_bh |= BH_STATUS;
-}
-
-static void ri_change(MGSLPC_INFO *info)
-{
-	get_signals(info);
-	if ((info->ri_chkcount)++ >= IO_PIN_SHUTDOWN_LIMIT)
-		port_irq_disable(info, PVR_RI);
-	info->icount.rng++;
-	if (info->serial_signals & SerialSignal_RI)
-		info->input_signal_events.ri_up++;
-	else
-		info->input_signal_events.ri_down++;
-	wake_up_interruptible(&info->status_event_wait_q);
-	wake_up_interruptible(&info->event_wait_q);
-	info->pending_bh |= BH_STATUS;
-}
-
-/* Interrupt service routine entry point.
- *
- * Arguments:
- *
- * irq     interrupt number that caused interrupt
- * dev_id  device ID supplied during interrupt registration
- */
-static irqreturn_t mgslpc_isr(int dummy, void *dev_id)
-{
-	MGSLPC_INFO *info = dev_id;
-	struct tty_struct *tty;
-	unsigned short isr;
-	unsigned char gis, pis;
-	int count=0;
-
-	if (debug_level >= DEBUG_LEVEL_ISR)
-		printk("mgslpc_isr(%d) entry.\n", info->irq_level);
-
-	if (!(info->p_dev->_locked))
-		return IRQ_HANDLED;
-
-	tty = tty_port_tty_get(&info->port);
-
-	spin_lock(&info->lock);
-
-	while ((gis = read_reg(info, CHA + GIS))) {
-		if (debug_level >= DEBUG_LEVEL_ISR)
-			printk("mgslpc_isr %s gis=%04X\n", info->device_name,gis);
-
-		if ((gis & 0x70) || count > 1000) {
-			printk("synclink_cs:hardware failed or ejected\n");
-			break;
-		}
-		count++;
-
-		if (gis & (BIT1 | BIT0)) {
-			isr = read_reg16(info, CHB + ISR);
-			if (isr & IRQ_DCD)
-				dcd_change(info, tty);
-			if (isr & IRQ_CTS)
-				cts_change(info, tty);
-		}
-		if (gis & (BIT3 | BIT2))
-		{
-			isr = read_reg16(info, CHA + ISR);
-			if (isr & IRQ_TIMER) {
-				info->irq_occurred = true;
-				irq_disable(info, CHA, IRQ_TIMER);
-			}
-
-			/* receive IRQs */
-			if (isr & IRQ_EXITHUNT) {
-				info->icount.exithunt++;
-				wake_up_interruptible(&info->event_wait_q);
-			}
-			if (isr & IRQ_BREAK_ON) {
-				info->icount.brk++;
-				if (info->port.flags & ASYNC_SAK)
-					do_SAK(tty);
-			}
-			if (isr & IRQ_RXTIME) {
-				issue_command(info, CHA, CMD_RXFIFO_READ);
-			}
-			if (isr & (IRQ_RXEOM | IRQ_RXFIFO)) {
-				if (info->params.mode == MGSL_MODE_HDLC)
-					rx_ready_hdlc(info, isr & IRQ_RXEOM);
-				else
-					rx_ready_async(info, isr & IRQ_RXEOM);
-			}
-
-			/* transmit IRQs */
-			if (isr & IRQ_UNDERRUN) {
-				if (info->tx_aborting)
-					info->icount.txabort++;
-				else
-					info->icount.txunder++;
-				tx_done(info, tty);
-			}
-			else if (isr & IRQ_ALLSENT) {
-				info->icount.txok++;
-				tx_done(info, tty);
-			}
-			else if (isr & IRQ_TXFIFO)
-				tx_ready(info, tty);
-		}
-		if (gis & BIT7) {
-			pis = read_reg(info, CHA + PIS);
-			if (pis & BIT1)
-				dsr_change(info);
-			if (pis & BIT2)
-				ri_change(info);
-		}
-	}
-
-	/* Request bottom half processing if there's something
-	 * for it to do and the bh is not already running
-	 */
-
-	if (info->pending_bh && !info->bh_running && !info->bh_requested) {
-		if (debug_level >= DEBUG_LEVEL_ISR)
-			printk("%s(%d):%s queueing bh task.\n",
-				__FILE__,__LINE__,info->device_name);
-		schedule_work(&info->task);
-		info->bh_requested = true;
-	}
-
-	spin_unlock(&info->lock);
-	tty_kref_put(tty);
-
-	if (debug_level >= DEBUG_LEVEL_ISR)
-		printk("%s(%d):mgslpc_isr(%d)exit.\n",
-		       __FILE__, __LINE__, info->irq_level);
-
-	return IRQ_HANDLED;
-}
-
-/* Initialize and start device.
- */
-static int startup(MGSLPC_INFO * info, struct tty_struct *tty)
-{
-	int retval = 0;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):startup(%s)\n", __FILE__, __LINE__, info->device_name);
-
-	if (tty_port_initialized(&info->port))
-		return 0;
-
-	if (!info->tx_buf) {
-		/* allocate a page of memory for a transmit buffer */
-		info->tx_buf = (unsigned char *)get_zeroed_page(GFP_KERNEL);
-		if (!info->tx_buf) {
-			printk(KERN_ERR"%s(%d):%s can't allocate transmit buffer\n",
-				__FILE__, __LINE__, info->device_name);
-			return -ENOMEM;
-		}
-	}
-
-	info->pending_bh = 0;
-
-	memset(&info->icount, 0, sizeof(info->icount));
-
-	timer_setup(&info->tx_timer, tx_timeout, 0);
-
-	/* Allocate and claim adapter resources */
-	retval = claim_resources(info);
-
-	/* perform existence check and diagnostics */
-	if (!retval)
-		retval = adapter_test(info);
-
-	if (retval) {
-		if (capable(CAP_SYS_ADMIN) && tty)
-			set_bit(TTY_IO_ERROR, &tty->flags);
-		release_resources(info);
-		return retval;
-	}
-
-	/* program hardware for current parameters */
-	mgslpc_change_params(info, tty);
-
-	if (tty)
-		clear_bit(TTY_IO_ERROR, &tty->flags);
-
-	tty_port_set_initialized(&info->port, true);
-
-	return 0;
-}
-
-/* Called by mgslpc_close() and mgslpc_hangup() to shutdown hardware
- */
-static void shutdown(MGSLPC_INFO * info, struct tty_struct *tty)
-{
-	unsigned long flags;
-
-	if (!tty_port_initialized(&info->port))
-		return;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_shutdown(%s)\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	/* clear status wait queue because status changes */
-	/* can't happen after shutting down the hardware */
-	wake_up_interruptible(&info->status_event_wait_q);
-	wake_up_interruptible(&info->event_wait_q);
-
-	del_timer_sync(&info->tx_timer);
-
-	if (info->tx_buf) {
-		free_page((unsigned long) info->tx_buf);
-		info->tx_buf = NULL;
-	}
-
-	spin_lock_irqsave(&info->lock, flags);
-
-	rx_stop(info);
-	tx_stop(info);
-
-	/* TODO:disable interrupts instead of reset to preserve signal states */
-	reset_device(info);
-
-	if (!tty || C_HUPCL(tty)) {
-		info->serial_signals &= ~(SerialSignal_RTS | SerialSignal_DTR);
-		set_signals(info);
-	}
-
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	release_resources(info);
-
-	if (tty)
-		set_bit(TTY_IO_ERROR, &tty->flags);
-
-	tty_port_set_initialized(&info->port, false);
-}
-
-static void mgslpc_program_hw(MGSLPC_INFO *info, struct tty_struct *tty)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&info->lock, flags);
-
-	rx_stop(info);
-	tx_stop(info);
-	info->tx_count = info->tx_put = info->tx_get = 0;
-
-	if (info->params.mode == MGSL_MODE_HDLC || info->netcount)
-		hdlc_mode(info);
-	else
-		async_mode(info);
-
-	set_signals(info);
-
-	info->dcd_chkcount = 0;
-	info->cts_chkcount = 0;
-	info->ri_chkcount = 0;
-	info->dsr_chkcount = 0;
-
-	irq_enable(info, CHB, IRQ_DCD | IRQ_CTS);
-	port_irq_enable(info, (unsigned char) PVR_DSR | PVR_RI);
-	get_signals(info);
-
-	if (info->netcount || (tty && C_CREAD(tty)))
-		rx_start(info);
-
-	spin_unlock_irqrestore(&info->lock, flags);
-}
-
-/* Reconfigure adapter based on new parameters
- */
-static void mgslpc_change_params(MGSLPC_INFO *info, struct tty_struct *tty)
-{
-	unsigned cflag;
-	int bits_per_char;
-
-	if (!tty)
-		return;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_change_params(%s)\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	cflag = tty->termios.c_cflag;
-
-	/* if B0 rate (hangup) specified then negate RTS and DTR */
-	/* otherwise assert RTS and DTR */
-	if (cflag & CBAUD)
-		info->serial_signals |= SerialSignal_RTS | SerialSignal_DTR;
-	else
-		info->serial_signals &= ~(SerialSignal_RTS | SerialSignal_DTR);
-
-	/* byte size and parity */
-	if ((cflag & CSIZE) != CS8) {
-		cflag &= ~CSIZE;
-		cflag |= CS7;
-		tty->termios.c_cflag = cflag;
-	}
-	info->params.data_bits = tty_get_char_size(cflag);
-
-	if (cflag & CSTOPB)
-		info->params.stop_bits = 2;
-	else
-		info->params.stop_bits = 1;
-
-	info->params.parity = ASYNC_PARITY_NONE;
-	if (cflag & PARENB) {
-		if (cflag & PARODD)
-			info->params.parity = ASYNC_PARITY_ODD;
-		else
-			info->params.parity = ASYNC_PARITY_EVEN;
-		if (cflag & CMSPAR)
-			info->params.parity = ASYNC_PARITY_SPACE;
-	}
-
-	/* calculate number of jiffies to transmit a full
-	 * FIFO (32 bytes) at specified data rate
-	 */
-	bits_per_char = info->params.data_bits +
-			info->params.stop_bits + 1;
-
-	/* if port data rate is set to 460800 or less then
-	 * allow tty settings to override, otherwise keep the
-	 * current data rate.
-	 */
-	if (info->params.data_rate <= 460800) {
-		info->params.data_rate = tty_get_baud_rate(tty);
-	}
-
-	if (info->params.data_rate) {
-		info->timeout = (32*HZ*bits_per_char) /
-				info->params.data_rate;
-	}
-	info->timeout += HZ/50;		/* Add .02 seconds of slop */
-
-	tty_port_set_cts_flow(&info->port, cflag & CRTSCTS);
-	tty_port_set_check_carrier(&info->port, ~cflag & CLOCAL);
-
-	/* process tty input control flags */
-
-	info->read_status_mask = 0;
-	if (I_INPCK(tty))
-		info->read_status_mask |= BIT7 | BIT6;
-	if (I_IGNPAR(tty))
-		info->ignore_status_mask |= BIT7 | BIT6;
-
-	mgslpc_program_hw(info, tty);
-}
-
-/* Add a character to the transmit buffer
- */
-static int mgslpc_put_char(struct tty_struct *tty, unsigned char ch)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO) {
-		printk("%s(%d):mgslpc_put_char(%d) on %s\n",
-			__FILE__, __LINE__, ch, info->device_name);
-	}
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_put_char"))
-		return 0;
-
-	if (!info->tx_buf)
-		return 0;
-
-	spin_lock_irqsave(&info->lock, flags);
-
-	if (info->params.mode == MGSL_MODE_ASYNC || !info->tx_active) {
-		if (info->tx_count < TXBUFSIZE - 1) {
-			info->tx_buf[info->tx_put++] = ch;
-			info->tx_put &= TXBUFSIZE-1;
-			info->tx_count++;
-		}
-	}
-
-	spin_unlock_irqrestore(&info->lock, flags);
-	return 1;
-}
-
-/* Enable transmitter so remaining characters in the
- * transmit buffer are sent.
- */
-static void mgslpc_flush_chars(struct tty_struct *tty)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_flush_chars() entry on %s tx_count=%d\n",
-			__FILE__, __LINE__, info->device_name, info->tx_count);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_flush_chars"))
-		return;
-
-	if (info->tx_count <= 0 || tty->flow.stopped ||
-	    tty->hw_stopped || !info->tx_buf)
-		return;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_flush_chars() entry on %s starting transmitter\n",
-			__FILE__, __LINE__, info->device_name);
-
-	spin_lock_irqsave(&info->lock, flags);
-	if (!info->tx_active)
-		tx_start(info, tty);
-	spin_unlock_irqrestore(&info->lock, flags);
-}
-
-/* Send a block of data
- *
- * Arguments:
- *
- * tty        pointer to tty information structure
- * buf	      pointer to buffer containing send data
- * count      size of send data in bytes
- *
- * Returns: number of characters written
- */
-static int mgslpc_write(struct tty_struct * tty,
-			const unsigned char *buf, int count)
-{
-	int c, ret = 0;
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_write(%s) count=%d\n",
-			__FILE__, __LINE__, info->device_name, count);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_write") ||
-		!info->tx_buf)
-		goto cleanup;
-
-	if (info->params.mode == MGSL_MODE_HDLC) {
-		if (count > TXBUFSIZE) {
-			ret = -EIO;
-			goto cleanup;
-		}
-		if (info->tx_active)
-			goto cleanup;
-		else if (info->tx_count)
-			goto start;
-	}
-
-	for (;;) {
-		c = min(count,
-			min(TXBUFSIZE - info->tx_count - 1,
-			    TXBUFSIZE - info->tx_put));
-		if (c <= 0)
-			break;
-
-		memcpy(info->tx_buf + info->tx_put, buf, c);
-
-		spin_lock_irqsave(&info->lock, flags);
-		info->tx_put = (info->tx_put + c) & (TXBUFSIZE-1);
-		info->tx_count += c;
-		spin_unlock_irqrestore(&info->lock, flags);
-
-		buf += c;
-		count -= c;
-		ret += c;
-	}
-start:
-	if (info->tx_count && !tty->flow.stopped && !tty->hw_stopped) {
-		spin_lock_irqsave(&info->lock, flags);
-		if (!info->tx_active)
-			tx_start(info, tty);
-		spin_unlock_irqrestore(&info->lock, flags);
-	}
-cleanup:
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_write(%s) returning=%d\n",
-			__FILE__, __LINE__, info->device_name, ret);
-	return ret;
-}
-
-/* Return the count of free bytes in transmit buffer
- */
-static unsigned int mgslpc_write_room(struct tty_struct *tty)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	int ret;
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_write_room"))
-		return 0;
-
-	if (info->params.mode == MGSL_MODE_HDLC) {
-		/* HDLC (frame oriented) mode */
-		if (info->tx_active)
-			return 0;
-		else
-			return HDLC_MAX_FRAME_SIZE;
-	} else {
-		ret = TXBUFSIZE - info->tx_count - 1;
-		if (ret < 0)
-			ret = 0;
-	}
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_write_room(%s)=%d\n",
-			 __FILE__, __LINE__, info->device_name, ret);
-	return ret;
-}
-
-/* Return the count of bytes in transmit buffer
- */
-static unsigned int mgslpc_chars_in_buffer(struct tty_struct *tty)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned int rc;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_chars_in_buffer(%s)\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_chars_in_buffer"))
-		return 0;
-
-	if (info->params.mode == MGSL_MODE_HDLC)
-		rc = info->tx_active ? info->max_frame_size : 0;
-	else
-		rc = info->tx_count;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_chars_in_buffer(%s)=%u\n",
-			 __FILE__, __LINE__, info->device_name, rc);
-
-	return rc;
-}
-
-/* Discard all data in the send buffer
- */
-static void mgslpc_flush_buffer(struct tty_struct *tty)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_flush_buffer(%s) entry\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_flush_buffer"))
-		return;
-
-	spin_lock_irqsave(&info->lock, flags);
-	info->tx_count = info->tx_put = info->tx_get = 0;
-	del_timer(&info->tx_timer);
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	wake_up_interruptible(&tty->write_wait);
-	tty_wakeup(tty);
-}
-
-/* Send a high-priority XON/XOFF character
- */
-static void mgslpc_send_xchar(struct tty_struct *tty, char ch)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_send_xchar(%s,%d)\n",
-			 __FILE__, __LINE__, info->device_name, ch);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_send_xchar"))
-		return;
-
-	info->x_char = ch;
-	if (ch) {
-		spin_lock_irqsave(&info->lock, flags);
-		if (!info->tx_enabled)
-			tx_start(info, tty);
-		spin_unlock_irqrestore(&info->lock, flags);
-	}
-}
-
-/* Signal remote device to throttle send data (our receive data)
- */
-static void mgslpc_throttle(struct tty_struct * tty)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_throttle(%s) entry\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_throttle"))
-		return;
-
-	if (I_IXOFF(tty))
-		mgslpc_send_xchar(tty, STOP_CHAR(tty));
-
-	if (C_CRTSCTS(tty)) {
-		spin_lock_irqsave(&info->lock, flags);
-		info->serial_signals &= ~SerialSignal_RTS;
-		set_signals(info);
-		spin_unlock_irqrestore(&info->lock, flags);
-	}
-}
-
-/* Signal remote device to stop throttling send data (our receive data)
- */
-static void mgslpc_unthrottle(struct tty_struct * tty)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_unthrottle(%s) entry\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_unthrottle"))
-		return;
-
-	if (I_IXOFF(tty)) {
-		if (info->x_char)
-			info->x_char = 0;
-		else
-			mgslpc_send_xchar(tty, START_CHAR(tty));
-	}
-
-	if (C_CRTSCTS(tty)) {
-		spin_lock_irqsave(&info->lock, flags);
-		info->serial_signals |= SerialSignal_RTS;
-		set_signals(info);
-		spin_unlock_irqrestore(&info->lock, flags);
-	}
-}
-
-/* get the current serial statistics
- */
-static int get_stats(MGSLPC_INFO * info, struct mgsl_icount __user *user_icount)
-{
-	int err;
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("get_params(%s)\n", info->device_name);
-	if (!user_icount) {
-		memset(&info->icount, 0, sizeof(info->icount));
-	} else {
-		COPY_TO_USER(err, user_icount, &info->icount, sizeof(struct mgsl_icount));
-		if (err)
-			return -EFAULT;
-	}
-	return 0;
-}
-
-/* get the current serial parameters
- */
-static int get_params(MGSLPC_INFO * info, MGSL_PARAMS __user *user_params)
-{
-	int err;
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("get_params(%s)\n", info->device_name);
-	COPY_TO_USER(err,user_params, &info->params, sizeof(MGSL_PARAMS));
-	if (err)
-		return -EFAULT;
-	return 0;
-}
-
-/* set the serial parameters
- *
- * Arguments:
- *
- *	info		pointer to device instance data
- *	new_params	user buffer containing new serial params
- *
- * Returns:	0 if success, otherwise error code
- */
-static int set_params(MGSLPC_INFO * info, MGSL_PARAMS __user *new_params, struct tty_struct *tty)
-{
-	unsigned long flags;
-	MGSL_PARAMS tmp_params;
-	int err;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):set_params %s\n", __FILE__,__LINE__,
-			info->device_name);
-	COPY_FROM_USER(err,&tmp_params, new_params, sizeof(MGSL_PARAMS));
-	if (err) {
-		if (debug_level >= DEBUG_LEVEL_INFO)
-			printk("%s(%d):set_params(%s) user buffer copy failed\n",
-				__FILE__, __LINE__, info->device_name);
-		return -EFAULT;
-	}
-
-	spin_lock_irqsave(&info->lock, flags);
-	memcpy(&info->params,&tmp_params,sizeof(MGSL_PARAMS));
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	mgslpc_change_params(info, tty);
-
-	return 0;
-}
-
-static int get_txidle(MGSLPC_INFO * info, int __user *idle_mode)
-{
-	int err;
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("get_txidle(%s)=%d\n", info->device_name, info->idle_mode);
-	COPY_TO_USER(err,idle_mode, &info->idle_mode, sizeof(int));
-	if (err)
-		return -EFAULT;
-	return 0;
-}
-
-static int set_txidle(MGSLPC_INFO * info, int idle_mode)
-{
-	unsigned long flags;
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("set_txidle(%s,%d)\n", info->device_name, idle_mode);
-	spin_lock_irqsave(&info->lock, flags);
-	info->idle_mode = idle_mode;
-	tx_set_idle(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-	return 0;
-}
-
-static int get_interface(MGSLPC_INFO * info, int __user *if_mode)
-{
-	int err;
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("get_interface(%s)=%d\n", info->device_name, info->if_mode);
-	COPY_TO_USER(err,if_mode, &info->if_mode, sizeof(int));
-	if (err)
-		return -EFAULT;
-	return 0;
-}
-
-static int set_interface(MGSLPC_INFO * info, int if_mode)
-{
-	unsigned long flags;
-	unsigned char val;
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("set_interface(%s,%d)\n", info->device_name, if_mode);
-	spin_lock_irqsave(&info->lock, flags);
-	info->if_mode = if_mode;
-
-	val = read_reg(info, PVR) & 0x0f;
-	switch (info->if_mode)
-	{
-	case MGSL_INTERFACE_RS232: val |= PVR_RS232; break;
-	case MGSL_INTERFACE_V35:   val |= PVR_V35;   break;
-	case MGSL_INTERFACE_RS422: val |= PVR_RS422; break;
-	}
-	write_reg(info, PVR, val);
-
-	spin_unlock_irqrestore(&info->lock, flags);
-	return 0;
-}
-
-static int set_txenable(MGSLPC_INFO * info, int enable, struct tty_struct *tty)
-{
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("set_txenable(%s,%d)\n", info->device_name, enable);
-
-	spin_lock_irqsave(&info->lock, flags);
-	if (enable) {
-		if (!info->tx_enabled)
-			tx_start(info, tty);
-	} else {
-		if (info->tx_enabled)
-			tx_stop(info);
-	}
-	spin_unlock_irqrestore(&info->lock, flags);
-	return 0;
-}
-
-static int tx_abort(MGSLPC_INFO * info)
-{
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("tx_abort(%s)\n", info->device_name);
-
-	spin_lock_irqsave(&info->lock, flags);
-	if (info->tx_active && info->tx_count &&
-	    info->params.mode == MGSL_MODE_HDLC) {
-		/* clear data count so FIFO is not filled on next IRQ.
-		 * This results in underrun and abort transmission.
-		 */
-		info->tx_count = info->tx_put = info->tx_get = 0;
-		info->tx_aborting = true;
-	}
-	spin_unlock_irqrestore(&info->lock, flags);
-	return 0;
-}
-
-static int set_rxenable(MGSLPC_INFO * info, int enable)
-{
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("set_rxenable(%s,%d)\n", info->device_name, enable);
-
-	spin_lock_irqsave(&info->lock, flags);
-	if (enable) {
-		if (!info->rx_enabled)
-			rx_start(info);
-	} else {
-		if (info->rx_enabled)
-			rx_stop(info);
-	}
-	spin_unlock_irqrestore(&info->lock, flags);
-	return 0;
-}
-
-/* wait for specified event to occur
- *
- * Arguments:		info	pointer to device instance data
- *			mask	pointer to bitmask of events to wait for
- * Return Value:	0	if successful and bit mask updated with
- *				of events triggerred,
- *			otherwise error code
- */
-static int wait_events(MGSLPC_INFO * info, int __user *mask_ptr)
-{
-	unsigned long flags;
-	int s;
-	int rc=0;
-	struct mgsl_icount cprev, cnow;
-	int events;
-	int mask;
-	struct	_input_signal_events oldsigs, newsigs;
-	DECLARE_WAITQUEUE(wait, current);
-
-	COPY_FROM_USER(rc,&mask, mask_ptr, sizeof(int));
-	if (rc)
-		return  -EFAULT;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("wait_events(%s,%d)\n", info->device_name, mask);
-
-	spin_lock_irqsave(&info->lock, flags);
-
-	/* return immediately if state matches requested events */
-	get_signals(info);
-	s = info->serial_signals;
-	events = mask &
-		( ((s & SerialSignal_DSR) ? MgslEvent_DsrActive:MgslEvent_DsrInactive) +
-		  ((s & SerialSignal_DCD) ? MgslEvent_DcdActive:MgslEvent_DcdInactive) +
-		  ((s & SerialSignal_CTS) ? MgslEvent_CtsActive:MgslEvent_CtsInactive) +
-		  ((s & SerialSignal_RI)  ? MgslEvent_RiActive :MgslEvent_RiInactive) );
-	if (events) {
-		spin_unlock_irqrestore(&info->lock, flags);
-		goto exit;
-	}
-
-	/* save current irq counts */
-	cprev = info->icount;
-	oldsigs = info->input_signal_events;
-
-	if ((info->params.mode == MGSL_MODE_HDLC) &&
-	    (mask & MgslEvent_ExitHuntMode))
-		irq_enable(info, CHA, IRQ_EXITHUNT);
-
-	set_current_state(TASK_INTERRUPTIBLE);
-	add_wait_queue(&info->event_wait_q, &wait);
-
-	spin_unlock_irqrestore(&info->lock, flags);
-
-
-	for(;;) {
-		schedule();
-		if (signal_pending(current)) {
-			rc = -ERESTARTSYS;
-			break;
-		}
-
-		/* get current irq counts */
-		spin_lock_irqsave(&info->lock, flags);
-		cnow = info->icount;
-		newsigs = info->input_signal_events;
-		set_current_state(TASK_INTERRUPTIBLE);
-		spin_unlock_irqrestore(&info->lock, flags);
-
-		/* if no change, wait aborted for some reason */
-		if (newsigs.dsr_up   == oldsigs.dsr_up   &&
-		    newsigs.dsr_down == oldsigs.dsr_down &&
-		    newsigs.dcd_up   == oldsigs.dcd_up   &&
-		    newsigs.dcd_down == oldsigs.dcd_down &&
-		    newsigs.cts_up   == oldsigs.cts_up   &&
-		    newsigs.cts_down == oldsigs.cts_down &&
-		    newsigs.ri_up    == oldsigs.ri_up    &&
-		    newsigs.ri_down  == oldsigs.ri_down  &&
-		    cnow.exithunt    == cprev.exithunt   &&
-		    cnow.rxidle      == cprev.rxidle) {
-			rc = -EIO;
-			break;
-		}
-
-		events = mask &
-			( (newsigs.dsr_up   != oldsigs.dsr_up   ? MgslEvent_DsrActive:0)   +
-			  (newsigs.dsr_down != oldsigs.dsr_down ? MgslEvent_DsrInactive:0) +
-			  (newsigs.dcd_up   != oldsigs.dcd_up   ? MgslEvent_DcdActive:0)   +
-			  (newsigs.dcd_down != oldsigs.dcd_down ? MgslEvent_DcdInactive:0) +
-			  (newsigs.cts_up   != oldsigs.cts_up   ? MgslEvent_CtsActive:0)   +
-			  (newsigs.cts_down != oldsigs.cts_down ? MgslEvent_CtsInactive:0) +
-			  (newsigs.ri_up    != oldsigs.ri_up    ? MgslEvent_RiActive:0)    +
-			  (newsigs.ri_down  != oldsigs.ri_down  ? MgslEvent_RiInactive:0)  +
-			  (cnow.exithunt    != cprev.exithunt   ? MgslEvent_ExitHuntMode:0) +
-			  (cnow.rxidle      != cprev.rxidle     ? MgslEvent_IdleReceived:0) );
-		if (events)
-			break;
-
-		cprev = cnow;
-		oldsigs = newsigs;
-	}
-
-	remove_wait_queue(&info->event_wait_q, &wait);
-	set_current_state(TASK_RUNNING);
-
-	if (mask & MgslEvent_ExitHuntMode) {
-		spin_lock_irqsave(&info->lock, flags);
-		if (!waitqueue_active(&info->event_wait_q))
-			irq_disable(info, CHA, IRQ_EXITHUNT);
-		spin_unlock_irqrestore(&info->lock, flags);
-	}
-exit:
-	if (rc == 0)
-		PUT_USER(rc, events, mask_ptr);
-	return rc;
-}
-
-static int modem_input_wait(MGSLPC_INFO *info,int arg)
-{
-	unsigned long flags;
-	int rc;
-	struct mgsl_icount cprev, cnow;
-	DECLARE_WAITQUEUE(wait, current);
-
-	/* save current irq counts */
-	spin_lock_irqsave(&info->lock, flags);
-	cprev = info->icount;
-	add_wait_queue(&info->status_event_wait_q, &wait);
-	set_current_state(TASK_INTERRUPTIBLE);
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	for(;;) {
-		schedule();
-		if (signal_pending(current)) {
-			rc = -ERESTARTSYS;
-			break;
-		}
-
-		/* get new irq counts */
-		spin_lock_irqsave(&info->lock, flags);
-		cnow = info->icount;
-		set_current_state(TASK_INTERRUPTIBLE);
-		spin_unlock_irqrestore(&info->lock, flags);
-
-		/* if no change, wait aborted for some reason */
-		if (cnow.rng == cprev.rng && cnow.dsr == cprev.dsr &&
-		    cnow.dcd == cprev.dcd && cnow.cts == cprev.cts) {
-			rc = -EIO;
-			break;
-		}
-
-		/* check for change in caller specified modem input */
-		if ((arg & TIOCM_RNG && cnow.rng != cprev.rng) ||
-		    (arg & TIOCM_DSR && cnow.dsr != cprev.dsr) ||
-		    (arg & TIOCM_CD  && cnow.dcd != cprev.dcd) ||
-		    (arg & TIOCM_CTS && cnow.cts != cprev.cts)) {
-			rc = 0;
-			break;
-		}
-
-		cprev = cnow;
-	}
-	remove_wait_queue(&info->status_event_wait_q, &wait);
-	set_current_state(TASK_RUNNING);
-	return rc;
-}
-
-/* return the state of the serial control and status signals
- */
-static int tiocmget(struct tty_struct *tty)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned int result;
-	unsigned long flags;
-
-	spin_lock_irqsave(&info->lock, flags);
-	get_signals(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	result = ((info->serial_signals & SerialSignal_RTS) ? TIOCM_RTS:0) +
-		((info->serial_signals & SerialSignal_DTR) ? TIOCM_DTR:0) +
-		((info->serial_signals & SerialSignal_DCD) ? TIOCM_CAR:0) +
-		((info->serial_signals & SerialSignal_RI)  ? TIOCM_RNG:0) +
-		((info->serial_signals & SerialSignal_DSR) ? TIOCM_DSR:0) +
-		((info->serial_signals & SerialSignal_CTS) ? TIOCM_CTS:0);
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):%s tiocmget() value=%08X\n",
-			 __FILE__, __LINE__, info->device_name, result);
-	return result;
-}
-
-/* set modem control signals (DTR/RTS)
- */
-static int tiocmset(struct tty_struct *tty,
-		    unsigned int set, unsigned int clear)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):%s tiocmset(%x,%x)\n",
-			__FILE__, __LINE__, info->device_name, set, clear);
-
-	if (set & TIOCM_RTS)
-		info->serial_signals |= SerialSignal_RTS;
-	if (set & TIOCM_DTR)
-		info->serial_signals |= SerialSignal_DTR;
-	if (clear & TIOCM_RTS)
-		info->serial_signals &= ~SerialSignal_RTS;
-	if (clear & TIOCM_DTR)
-		info->serial_signals &= ~SerialSignal_DTR;
-
-	spin_lock_irqsave(&info->lock, flags);
-	set_signals(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	return 0;
-}
-
-/* Set or clear transmit break condition
- *
- * Arguments:		tty		pointer to tty instance data
- *			break_state	-1=set break condition, 0=clear
- */
-static int mgslpc_break(struct tty_struct *tty, int break_state)
-{
-	MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_break(%s,%d)\n",
-			 __FILE__, __LINE__, info->device_name, break_state);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_break"))
-		return -EINVAL;
-
-	spin_lock_irqsave(&info->lock, flags);
-	if (break_state == -1)
-		set_reg_bits(info, CHA+DAFO, BIT6);
-	else
-		clear_reg_bits(info, CHA+DAFO, BIT6);
-	spin_unlock_irqrestore(&info->lock, flags);
-	return 0;
-}
-
-static int mgslpc_get_icount(struct tty_struct *tty,
-				struct serial_icounter_struct *icount)
-{
-	MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data;
-	struct mgsl_icount cnow;	/* kernel counter temps */
-	unsigned long flags;
-
-	spin_lock_irqsave(&info->lock, flags);
-	cnow = info->icount;
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	icount->cts = cnow.cts;
-	icount->dsr = cnow.dsr;
-	icount->rng = cnow.rng;
-	icount->dcd = cnow.dcd;
-	icount->rx = cnow.rx;
-	icount->tx = cnow.tx;
-	icount->frame = cnow.frame;
-	icount->overrun = cnow.overrun;
-	icount->parity = cnow.parity;
-	icount->brk = cnow.brk;
-	icount->buf_overrun = cnow.buf_overrun;
-
-	return 0;
-}
-
-/* Service an IOCTL request
- *
- * Arguments:
- *
- *	tty	pointer to tty instance data
- *	cmd	IOCTL command code
- *	arg	command argument/context
- *
- * Return Value:	0 if success, otherwise error code
- */
-static int mgslpc_ioctl(struct tty_struct *tty,
-			unsigned int cmd, unsigned long arg)
-{
-	MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data;
-	void __user *argp = (void __user *)arg;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_ioctl %s cmd=%08X\n", __FILE__, __LINE__,
-			info->device_name, cmd);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_ioctl"))
-		return -ENODEV;
-
-	if (cmd != TIOCMIWAIT) {
-		if (tty_io_error(tty))
-		    return -EIO;
-	}
-
-	switch (cmd) {
-	case MGSL_IOCGPARAMS:
-		return get_params(info, argp);
-	case MGSL_IOCSPARAMS:
-		return set_params(info, argp, tty);
-	case MGSL_IOCGTXIDLE:
-		return get_txidle(info, argp);
-	case MGSL_IOCSTXIDLE:
-		return set_txidle(info, (int)arg);
-	case MGSL_IOCGIF:
-		return get_interface(info, argp);
-	case MGSL_IOCSIF:
-		return set_interface(info,(int)arg);
-	case MGSL_IOCTXENABLE:
-		return set_txenable(info,(int)arg, tty);
-	case MGSL_IOCRXENABLE:
-		return set_rxenable(info,(int)arg);
-	case MGSL_IOCTXABORT:
-		return tx_abort(info);
-	case MGSL_IOCGSTATS:
-		return get_stats(info, argp);
-	case MGSL_IOCWAITEVENT:
-		return wait_events(info, argp);
-	case TIOCMIWAIT:
-		return modem_input_wait(info,(int)arg);
-	default:
-		return -ENOIOCTLCMD;
-	}
-	return 0;
-}
-
-/* Set new termios settings
- *
- * Arguments:
- *
- *	tty		pointer to tty structure
- *	termios		pointer to buffer to hold returned old termios
- */
-static void mgslpc_set_termios(struct tty_struct *tty,
-			       const struct ktermios *old_termios)
-{
-	MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_set_termios %s\n", __FILE__, __LINE__,
-			tty->driver->name);
-
-	/* just return if nothing has changed */
-	if ((tty->termios.c_cflag == old_termios->c_cflag)
-	    && (RELEVANT_IFLAG(tty->termios.c_iflag)
-		== RELEVANT_IFLAG(old_termios->c_iflag)))
-	  return;
-
-	mgslpc_change_params(info, tty);
-
-	/* Handle transition to B0 status */
-	if ((old_termios->c_cflag & CBAUD) && !C_BAUD(tty)) {
-		info->serial_signals &= ~(SerialSignal_RTS | SerialSignal_DTR);
-		spin_lock_irqsave(&info->lock, flags);
-		set_signals(info);
-		spin_unlock_irqrestore(&info->lock, flags);
-	}
-
-	/* Handle transition away from B0 status */
-	if (!(old_termios->c_cflag & CBAUD) && C_BAUD(tty)) {
-		info->serial_signals |= SerialSignal_DTR;
-		if (!C_CRTSCTS(tty) || !tty_throttled(tty))
-			info->serial_signals |= SerialSignal_RTS;
-		spin_lock_irqsave(&info->lock, flags);
-		set_signals(info);
-		spin_unlock_irqrestore(&info->lock, flags);
-	}
-
-	/* Handle turning off CRTSCTS */
-	if (old_termios->c_cflag & CRTSCTS && !C_CRTSCTS(tty)) {
-		tty->hw_stopped = 0;
-		tx_release(tty);
-	}
-}
-
-static void mgslpc_close(struct tty_struct *tty, struct file * filp)
-{
-	MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data;
-	struct tty_port *port = &info->port;
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_close"))
-		return;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_close(%s) entry, count=%d\n",
-			 __FILE__, __LINE__, info->device_name, port->count);
-
-	if (tty_port_close_start(port, tty, filp) == 0)
-		goto cleanup;
-
-	if (tty_port_initialized(port))
-		mgslpc_wait_until_sent(tty, info->timeout);
-
-	mgslpc_flush_buffer(tty);
-
-	tty_ldisc_flush(tty);
-	shutdown(info, tty);
-	
-	tty_port_close_end(port, tty);
-	tty_port_tty_set(port, NULL);
-cleanup:
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_close(%s) exit, count=%d\n", __FILE__, __LINE__,
-			tty->driver->name, port->count);
-}
-
-/* Wait until the transmitter is empty.
- */
-static void mgslpc_wait_until_sent(struct tty_struct *tty, int timeout)
-{
-	MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data;
-	unsigned long orig_jiffies, char_time;
-
-	if (!info)
-		return;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_wait_until_sent(%s) entry\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_wait_until_sent"))
-		return;
-
-	if (!tty_port_initialized(&info->port))
-		goto exit;
-
-	orig_jiffies = jiffies;
-
-	/* Set check interval to 1/5 of estimated time to
-	 * send a character, and make it at least 1. The check
-	 * interval should also be less than the timeout.
-	 * Note: use tight timings here to satisfy the NIST-PCTS.
-	 */
-
-	if (info->params.data_rate) {
-	     	char_time = info->timeout/(32 * 5);
-		if (!char_time)
-			char_time++;
-	} else
-		char_time = 1;
-
-	if (timeout)
-		char_time = min_t(unsigned long, char_time, timeout);
-
-	if (info->params.mode == MGSL_MODE_HDLC) {
-		while (info->tx_active) {
-			msleep_interruptible(jiffies_to_msecs(char_time));
-			if (signal_pending(current))
-				break;
-			if (timeout && time_after(jiffies, orig_jiffies + timeout))
-				break;
-		}
-	} else {
-		while ((info->tx_count || info->tx_active) &&
-			info->tx_enabled) {
-			msleep_interruptible(jiffies_to_msecs(char_time));
-			if (signal_pending(current))
-				break;
-			if (timeout && time_after(jiffies, orig_jiffies + timeout))
-				break;
-		}
-	}
-
-exit:
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_wait_until_sent(%s) exit\n",
-			 __FILE__, __LINE__, info->device_name);
-}
-
-/* Called by tty_hangup() when a hangup is signaled.
- * This is the same as closing all open files for the port.
- */
-static void mgslpc_hangup(struct tty_struct *tty)
-{
-	MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_hangup(%s)\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_hangup"))
-		return;
-
-	mgslpc_flush_buffer(tty);
-	shutdown(info, tty);
-	tty_port_hangup(&info->port);
-}
-
-static bool carrier_raised(struct tty_port *port)
-{
-	MGSLPC_INFO *info = container_of(port, MGSLPC_INFO, port);
-	unsigned long flags;
-
-	spin_lock_irqsave(&info->lock, flags);
-	get_signals(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	return info->serial_signals & SerialSignal_DCD;
-}
-
-static void dtr_rts(struct tty_port *port, bool active)
-{
-	MGSLPC_INFO *info = container_of(port, MGSLPC_INFO, port);
-	unsigned long flags;
-
-	spin_lock_irqsave(&info->lock, flags);
-	if (active)
-		info->serial_signals |= SerialSignal_RTS | SerialSignal_DTR;
-	else
-		info->serial_signals &= ~(SerialSignal_RTS | SerialSignal_DTR);
-	set_signals(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-}
-
-
-static int mgslpc_open(struct tty_struct *tty, struct file * filp)
-{
-	MGSLPC_INFO	*info;
-	struct tty_port *port;
-	int		retval, line;
-	unsigned long	flags;
-
-	/* verify range of specified line number */
-	line = tty->index;
-	if (line >= mgslpc_device_count) {
-		printk("%s(%d):mgslpc_open with invalid line #%d.\n",
-			__FILE__, __LINE__, line);
-		return -ENODEV;
-	}
-
-	/* find the info structure for the specified line */
-	info = mgslpc_device_list;
-	while(info && info->line != line)
-		info = info->next_device;
-	if (mgslpc_paranoia_check(info, tty->name, "mgslpc_open"))
-		return -ENODEV;
-
-	port = &info->port;
-	tty->driver_data = info;
-	tty_port_tty_set(port, tty);
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_open(%s), old ref count = %d\n",
-			 __FILE__, __LINE__, tty->driver->name, port->count);
-
-	spin_lock_irqsave(&info->netlock, flags);
-	if (info->netcount) {
-		retval = -EBUSY;
-		spin_unlock_irqrestore(&info->netlock, flags);
-		goto cleanup;
-	}
-	spin_lock(&port->lock);
-	port->count++;
-	spin_unlock(&port->lock);
-	spin_unlock_irqrestore(&info->netlock, flags);
-
-	if (port->count == 1) {
-		/* 1st open on this device, init hardware */
-		retval = startup(info, tty);
-		if (retval < 0)
-			goto cleanup;
-	}
-
-	retval = tty_port_block_til_ready(&info->port, tty, filp);
-	if (retval) {
-		if (debug_level >= DEBUG_LEVEL_INFO)
-			printk("%s(%d):block_til_ready(%s) returned %d\n",
-				 __FILE__, __LINE__, info->device_name, retval);
-		goto cleanup;
-	}
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):mgslpc_open(%s) success\n",
-			 __FILE__, __LINE__, info->device_name);
-	retval = 0;
-
-cleanup:
-	return retval;
-}
-
-/*
- * /proc fs routines....
- */
-
-static inline void line_info(struct seq_file *m, MGSLPC_INFO *info)
-{
-	char	stat_buf[30];
-	unsigned long flags;
-
-	seq_printf(m, "%s:io:%04X irq:%d",
-		      info->device_name, info->io_base, info->irq_level);
-
-	/* output current serial signal states */
-	spin_lock_irqsave(&info->lock, flags);
-	get_signals(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	stat_buf[0] = 0;
-	stat_buf[1] = 0;
-	if (info->serial_signals & SerialSignal_RTS)
-		strcat(stat_buf, "|RTS");
-	if (info->serial_signals & SerialSignal_CTS)
-		strcat(stat_buf, "|CTS");
-	if (info->serial_signals & SerialSignal_DTR)
-		strcat(stat_buf, "|DTR");
-	if (info->serial_signals & SerialSignal_DSR)
-		strcat(stat_buf, "|DSR");
-	if (info->serial_signals & SerialSignal_DCD)
-		strcat(stat_buf, "|CD");
-	if (info->serial_signals & SerialSignal_RI)
-		strcat(stat_buf, "|RI");
-
-	if (info->params.mode == MGSL_MODE_HDLC) {
-		seq_printf(m, " HDLC txok:%d rxok:%d",
-			      info->icount.txok, info->icount.rxok);
-		if (info->icount.txunder)
-			seq_printf(m, " txunder:%d", info->icount.txunder);
-		if (info->icount.txabort)
-			seq_printf(m, " txabort:%d", info->icount.txabort);
-		if (info->icount.rxshort)
-			seq_printf(m, " rxshort:%d", info->icount.rxshort);
-		if (info->icount.rxlong)
-			seq_printf(m, " rxlong:%d", info->icount.rxlong);
-		if (info->icount.rxover)
-			seq_printf(m, " rxover:%d", info->icount.rxover);
-		if (info->icount.rxcrc)
-			seq_printf(m, " rxcrc:%d", info->icount.rxcrc);
-	} else {
-		seq_printf(m, " ASYNC tx:%d rx:%d",
-			      info->icount.tx, info->icount.rx);
-		if (info->icount.frame)
-			seq_printf(m, " fe:%d", info->icount.frame);
-		if (info->icount.parity)
-			seq_printf(m, " pe:%d", info->icount.parity);
-		if (info->icount.brk)
-			seq_printf(m, " brk:%d", info->icount.brk);
-		if (info->icount.overrun)
-			seq_printf(m, " oe:%d", info->icount.overrun);
-	}
-
-	/* Append serial signal status to end */
-	seq_printf(m, " %s\n", stat_buf+1);
-
-	seq_printf(m, "txactive=%d bh_req=%d bh_run=%d pending_bh=%x\n",
-		       info->tx_active,info->bh_requested,info->bh_running,
-		       info->pending_bh);
-}
-
-/* Called to print information about devices
- */
-static int mgslpc_proc_show(struct seq_file *m, void *v)
-{
-	MGSLPC_INFO *info;
-
-	seq_printf(m, "synclink driver:%s\n", driver_version);
-
-	info = mgslpc_device_list;
-	while (info) {
-		line_info(m, info);
-		info = info->next_device;
-	}
-	return 0;
-}
-
-static int rx_alloc_buffers(MGSLPC_INFO *info)
-{
-	/* each buffer has header and data */
-	info->rx_buf_size = sizeof(RXBUF) + info->max_frame_size;
-
-	/* calculate total allocation size for 8 buffers */
-	info->rx_buf_total_size = info->rx_buf_size * 8;
-
-	/* limit total allocated memory */
-	if (info->rx_buf_total_size > 0x10000)
-		info->rx_buf_total_size = 0x10000;
-
-	/* calculate number of buffers */
-	info->rx_buf_count = info->rx_buf_total_size / info->rx_buf_size;
-
-	info->rx_buf = kmalloc(info->rx_buf_total_size, GFP_KERNEL);
-	if (info->rx_buf == NULL)
-		return -ENOMEM;
-
-	/* unused flag buffer to satisfy receive_buf calling interface */
-	info->flag_buf = kzalloc(info->max_frame_size, GFP_KERNEL);
-	if (!info->flag_buf) {
-		kfree(info->rx_buf);
-		info->rx_buf = NULL;
-		return -ENOMEM;
-	}
-	
-	rx_reset_buffers(info);
-	return 0;
-}
-
-static void rx_free_buffers(MGSLPC_INFO *info)
-{
-	kfree(info->rx_buf);
-	info->rx_buf = NULL;
-	kfree(info->flag_buf);
-	info->flag_buf = NULL;
-}
-
-static int claim_resources(MGSLPC_INFO *info)
-{
-	if (rx_alloc_buffers(info) < 0) {
-		printk("Can't allocate rx buffer %s\n", info->device_name);
-		release_resources(info);
-		return -ENODEV;
-	}
-	return 0;
-}
-
-static void release_resources(MGSLPC_INFO *info)
-{
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("release_resources(%s)\n", info->device_name);
-	rx_free_buffers(info);
-}
-
-/* Add the specified device instance data structure to the
- * global linked list of devices and increment the device count.
- *
- * Arguments:		info	pointer to device instance data
- */
-static int mgslpc_add_device(MGSLPC_INFO *info)
-{
-	MGSLPC_INFO *current_dev = NULL;
-	struct device *tty_dev;
-	int ret;
-
-	info->next_device = NULL;
-	info->line = mgslpc_device_count;
-	sprintf(info->device_name,"ttySLP%d",info->line);
-
-	if (info->line < MAX_DEVICE_COUNT) {
-		if (maxframe[info->line])
-			info->max_frame_size = maxframe[info->line];
-	}
-
-	mgslpc_device_count++;
-
-	if (!mgslpc_device_list)
-		mgslpc_device_list = info;
-	else {
-		current_dev = mgslpc_device_list;
-		while (current_dev->next_device)
-			current_dev = current_dev->next_device;
-		current_dev->next_device = info;
-	}
-
-	if (info->max_frame_size < 4096)
-		info->max_frame_size = 4096;
-	else if (info->max_frame_size > 65535)
-		info->max_frame_size = 65535;
-
-	printk("SyncLink PC Card %s:IO=%04X IRQ=%d\n",
-		info->device_name, info->io_base, info->irq_level);
-
-#if SYNCLINK_GENERIC_HDLC
-	ret = hdlcdev_init(info);
-	if (ret != 0)
-		goto failed;
-#endif
-
-	tty_dev = tty_port_register_device(&info->port, serial_driver, info->line,
-			&info->p_dev->dev);
-	if (IS_ERR(tty_dev)) {
-		ret = PTR_ERR(tty_dev);
-#if SYNCLINK_GENERIC_HDLC
-		hdlcdev_exit(info);
-#endif
-		goto failed;
-	}
-
-	return 0;
-
-failed:
-	if (current_dev)
-		current_dev->next_device = NULL;
-	else
-		mgslpc_device_list = NULL;
-	mgslpc_device_count--;
-	return ret;
-}
-
-static void mgslpc_remove_device(MGSLPC_INFO *remove_info)
-{
-	MGSLPC_INFO *info = mgslpc_device_list;
-	MGSLPC_INFO *last = NULL;
-
-	while(info) {
-		if (info == remove_info) {
-			if (last)
-				last->next_device = info->next_device;
-			else
-				mgslpc_device_list = info->next_device;
-			tty_unregister_device(serial_driver, info->line);
-#if SYNCLINK_GENERIC_HDLC
-			hdlcdev_exit(info);
-#endif
-			release_resources(info);
-			tty_port_destroy(&info->port);
-			kfree(info);
-			mgslpc_device_count--;
-			return;
-		}
-		last = info;
-		info = info->next_device;
-	}
-}
-
-static const struct pcmcia_device_id mgslpc_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD(0x02c5, 0x0050),
-	PCMCIA_DEVICE_NULL
-};
-MODULE_DEVICE_TABLE(pcmcia, mgslpc_ids);
-
-static struct pcmcia_driver mgslpc_driver = {
-	.owner		= THIS_MODULE,
-	.name		= "synclink_cs",
-	.probe		= mgslpc_probe,
-	.remove		= mgslpc_detach,
-	.id_table	= mgslpc_ids,
-	.suspend	= mgslpc_suspend,
-	.resume		= mgslpc_resume,
-};
-
-static const struct tty_operations mgslpc_ops = {
-	.open = mgslpc_open,
-	.close = mgslpc_close,
-	.write = mgslpc_write,
-	.put_char = mgslpc_put_char,
-	.flush_chars = mgslpc_flush_chars,
-	.write_room = mgslpc_write_room,
-	.chars_in_buffer = mgslpc_chars_in_buffer,
-	.flush_buffer = mgslpc_flush_buffer,
-	.ioctl = mgslpc_ioctl,
-	.throttle = mgslpc_throttle,
-	.unthrottle = mgslpc_unthrottle,
-	.send_xchar = mgslpc_send_xchar,
-	.break_ctl = mgslpc_break,
-	.wait_until_sent = mgslpc_wait_until_sent,
-	.set_termios = mgslpc_set_termios,
-	.stop = tx_pause,
-	.start = tx_release,
-	.hangup = mgslpc_hangup,
-	.tiocmget = tiocmget,
-	.tiocmset = tiocmset,
-	.get_icount = mgslpc_get_icount,
-	.proc_show = mgslpc_proc_show,
-};
-
-static int __init synclink_cs_init(void)
-{
-	int rc;
-
-	if (break_on_load) {
-		mgslpc_get_text_ptr();
-		BREAKPOINT();
-	}
-
-	serial_driver = tty_alloc_driver(MAX_DEVICE_COUNT,
-			TTY_DRIVER_REAL_RAW |
-			TTY_DRIVER_DYNAMIC_DEV);
-	if (IS_ERR(serial_driver)) {
-		rc = PTR_ERR(serial_driver);
-		goto err;
-	}
-
-	/* Initialize the tty_driver structure */
-	serial_driver->driver_name = "synclink_cs";
-	serial_driver->name = "ttySLP";
-	serial_driver->major = ttymajor;
-	serial_driver->minor_start = 64;
-	serial_driver->type = TTY_DRIVER_TYPE_SERIAL;
-	serial_driver->subtype = SERIAL_TYPE_NORMAL;
-	serial_driver->init_termios = tty_std_termios;
-	serial_driver->init_termios.c_cflag =
-	B9600 | CS8 | CREAD | HUPCL | CLOCAL;
-	tty_set_operations(serial_driver, &mgslpc_ops);
-
-	rc = tty_register_driver(serial_driver);
-	if (rc < 0) {
-		printk(KERN_ERR "%s(%d):Couldn't register serial driver\n",
-				__FILE__, __LINE__);
-		goto err_put_tty;
-	}
-
-	rc = pcmcia_register_driver(&mgslpc_driver);
-	if (rc < 0)
-		goto err_unreg_tty;
-
-	printk(KERN_INFO "%s %s, tty major#%d\n", driver_name, driver_version,
-			serial_driver->major);
-
-	return 0;
-err_unreg_tty:
-	tty_unregister_driver(serial_driver);
-err_put_tty:
-	tty_driver_kref_put(serial_driver);
-err:
-	return rc;
-}
-
-static void __exit synclink_cs_exit(void)
-{
-	pcmcia_unregister_driver(&mgslpc_driver);
-	tty_unregister_driver(serial_driver);
-	tty_driver_kref_put(serial_driver);
-}
-
-module_init(synclink_cs_init);
-module_exit(synclink_cs_exit);
-
-static void mgslpc_set_rate(MGSLPC_INFO *info, unsigned char channel, unsigned int rate)
-{
-	unsigned int M, N;
-	unsigned char val;
-
-	/* note:standard BRG mode is broken in V3.2 chip
-	 * so enhanced mode is always used
-	 */
-
-	if (rate) {
-		N = 3686400 / rate;
-		if (!N)
-			N = 1;
-		N >>= 1;
-		for (M = 1; N > 64 && M < 16; M++)
-			N >>= 1;
-		N--;
-
-		/* BGR[5..0] = N
-		 * BGR[9..6] = M
-		 * BGR[7..0] contained in BGR register
-		 * BGR[9..8] contained in CCR2[7..6]
-		 * divisor = (N+1)*2^M
-		 *
-		 * Note: M *must* not be zero (causes asymetric duty cycle)
-		 */
-		write_reg(info, (unsigned char) (channel + BGR),
-				  (unsigned char) ((M << 6) + N));
-		val = read_reg(info, (unsigned char) (channel + CCR2)) & 0x3f;
-		val |= ((M << 4) & 0xc0);
-		write_reg(info, (unsigned char) (channel + CCR2), val);
-	}
-}
-
-/* Enabled the AUX clock output at the specified frequency.
- */
-static void enable_auxclk(MGSLPC_INFO *info)
-{
-	unsigned char val;
-
-	/* MODE
-	 *
-	 * 07..06  MDS[1..0] 10 = transparent HDLC mode
-	 * 05      ADM Address Mode, 0 = no addr recognition
-	 * 04      TMD Timer Mode, 0 = external
-	 * 03      RAC Receiver Active, 0 = inactive
-	 * 02      RTS 0=RTS active during xmit, 1=RTS always active
-	 * 01      TRS Timer Resolution, 1=512
-	 * 00      TLP Test Loop, 0 = no loop
-	 *
-	 * 1000 0010
-	 */
-	val = 0x82;
-
-	/* channel B RTS is used to enable AUXCLK driver on SP505 */
-	if (info->params.mode == MGSL_MODE_HDLC && info->params.clock_speed)
-		val |= BIT2;
-	write_reg(info, CHB + MODE, val);
-
-	/* CCR0
-	 *
-	 * 07      PU Power Up, 1=active, 0=power down
-	 * 06      MCE Master Clock Enable, 1=enabled
-	 * 05      Reserved, 0
-	 * 04..02  SC[2..0] Encoding
-	 * 01..00  SM[1..0] Serial Mode, 00=HDLC
-	 *
-	 * 11000000
-	 */
-	write_reg(info, CHB + CCR0, 0xc0);
-
-	/* CCR1
-	 *
-	 * 07      SFLG Shared Flag, 0 = disable shared flags
-	 * 06      GALP Go Active On Loop, 0 = not used
-	 * 05      GLP Go On Loop, 0 = not used
-	 * 04      ODS Output Driver Select, 1=TxD is push-pull output
-	 * 03      ITF Interframe Time Fill, 0=mark, 1=flag
-	 * 02..00  CM[2..0] Clock Mode
-	 *
-	 * 0001 0111
-	 */
-	write_reg(info, CHB + CCR1, 0x17);
-
-	/* CCR2 (Channel B)
-	 *
-	 * 07..06  BGR[9..8] Baud rate bits 9..8
-	 * 05      BDF Baud rate divisor factor, 0=1, 1=BGR value
-	 * 04      SSEL Clock source select, 1=submode b
-	 * 03      TOE 0=TxCLK is input, 1=TxCLK is output
-	 * 02      RWX Read/Write Exchange 0=disabled
-	 * 01      C32, CRC select, 0=CRC-16, 1=CRC-32
-	 * 00      DIV, data inversion 0=disabled, 1=enabled
-	 *
-	 * 0011 1000
-	 */
-	if (info->params.mode == MGSL_MODE_HDLC && info->params.clock_speed)
-		write_reg(info, CHB + CCR2, 0x38);
-	else
-		write_reg(info, CHB + CCR2, 0x30);
-
-	/* CCR4
-	 *
-	 * 07      MCK4 Master Clock Divide by 4, 1=enabled
-	 * 06      EBRG Enhanced Baud Rate Generator Mode, 1=enabled
-	 * 05      TST1 Test Pin, 0=normal operation
-	 * 04      ICD Ivert Carrier Detect, 1=enabled (active low)
-	 * 03..02  Reserved, must be 0
-	 * 01..00  RFT[1..0] RxFIFO Threshold 00=32 bytes
-	 *
-	 * 0101 0000
-	 */
-	write_reg(info, CHB + CCR4, 0x50);
-
-	/* if auxclk not enabled, set internal BRG so
-	 * CTS transitions can be detected (requires TxC)
-	 */
-	if (info->params.mode == MGSL_MODE_HDLC && info->params.clock_speed)
-		mgslpc_set_rate(info, CHB, info->params.clock_speed);
-	else
-		mgslpc_set_rate(info, CHB, 921600);
-}
-
-static void loopback_enable(MGSLPC_INFO *info)
-{
-	unsigned char val;
-
-	/* CCR1:02..00  CM[2..0] Clock Mode = 111 (clock mode 7) */
-	val = read_reg(info, CHA + CCR1) | (BIT2 | BIT1 | BIT0);
-	write_reg(info, CHA + CCR1, val);
-
-	/* CCR2:04 SSEL Clock source select, 1=submode b */
-	val = read_reg(info, CHA + CCR2) | (BIT4 | BIT5);
-	write_reg(info, CHA + CCR2, val);
-
-	/* set LinkSpeed if available, otherwise default to 2Mbps */
-	if (info->params.clock_speed)
-		mgslpc_set_rate(info, CHA, info->params.clock_speed);
-	else
-		mgslpc_set_rate(info, CHA, 1843200);
-
-	/* MODE:00 TLP Test Loop, 1=loopback enabled */
-	val = read_reg(info, CHA + MODE) | BIT0;
-	write_reg(info, CHA + MODE, val);
-}
-
-static void hdlc_mode(MGSLPC_INFO *info)
-{
-	unsigned char val;
-	unsigned char clkmode, clksubmode;
-
-	/* disable all interrupts */
-	irq_disable(info, CHA, 0xffff);
-	irq_disable(info, CHB, 0xffff);
-	port_irq_disable(info, 0xff);
-
-	/* assume clock mode 0a, rcv=RxC xmt=TxC */
-	clkmode = clksubmode = 0;
-	if (info->params.flags & HDLC_FLAG_RXC_DPLL
-	    && info->params.flags & HDLC_FLAG_TXC_DPLL) {
-		/* clock mode 7a, rcv = DPLL, xmt = DPLL */
-		clkmode = 7;
-	} else if (info->params.flags & HDLC_FLAG_RXC_BRG
-		 && info->params.flags & HDLC_FLAG_TXC_BRG) {
-		/* clock mode 7b, rcv = BRG, xmt = BRG */
-		clkmode = 7;
-		clksubmode = 1;
-	} else if (info->params.flags & HDLC_FLAG_RXC_DPLL) {
-		if (info->params.flags & HDLC_FLAG_TXC_BRG) {
-			/* clock mode 6b, rcv = DPLL, xmt = BRG/16 */
-			clkmode = 6;
-			clksubmode = 1;
-		} else {
-			/* clock mode 6a, rcv = DPLL, xmt = TxC */
-			clkmode = 6;
-		}
-	} else if (info->params.flags & HDLC_FLAG_TXC_BRG) {
-		/* clock mode 0b, rcv = RxC, xmt = BRG */
-		clksubmode = 1;
-	}
-
-	/* MODE
-	 *
-	 * 07..06  MDS[1..0] 10 = transparent HDLC mode
-	 * 05      ADM Address Mode, 0 = no addr recognition
-	 * 04      TMD Timer Mode, 0 = external
-	 * 03      RAC Receiver Active, 0 = inactive
-	 * 02      RTS 0=RTS active during xmit, 1=RTS always active
-	 * 01      TRS Timer Resolution, 1=512
-	 * 00      TLP Test Loop, 0 = no loop
-	 *
-	 * 1000 0010
-	 */
-	val = 0x82;
-	if (info->params.loopback)
-		val |= BIT0;
-
-	/* preserve RTS state */
-	if (info->serial_signals & SerialSignal_RTS)
-		val |= BIT2;
-	write_reg(info, CHA + MODE, val);
-
-	/* CCR0
-	 *
-	 * 07      PU Power Up, 1=active, 0=power down
-	 * 06      MCE Master Clock Enable, 1=enabled
-	 * 05      Reserved, 0
-	 * 04..02  SC[2..0] Encoding
-	 * 01..00  SM[1..0] Serial Mode, 00=HDLC
-	 *
-	 * 11000000
-	 */
-	val = 0xc0;
-	switch (info->params.encoding)
-	{
-	case HDLC_ENCODING_NRZI:
-		val |= BIT3;
-		break;
-	case HDLC_ENCODING_BIPHASE_SPACE:
-		val |= BIT4;
-		break;		// FM0
-	case HDLC_ENCODING_BIPHASE_MARK:
-		val |= BIT4 | BIT2;
-		break;		// FM1
-	case HDLC_ENCODING_BIPHASE_LEVEL:
-		val |= BIT4 | BIT3;
-		break;		// Manchester
-	}
-	write_reg(info, CHA + CCR0, val);
-
-	/* CCR1
-	 *
-	 * 07      SFLG Shared Flag, 0 = disable shared flags
-	 * 06      GALP Go Active On Loop, 0 = not used
-	 * 05      GLP Go On Loop, 0 = not used
-	 * 04      ODS Output Driver Select, 1=TxD is push-pull output
-	 * 03      ITF Interframe Time Fill, 0=mark, 1=flag
-	 * 02..00  CM[2..0] Clock Mode
-	 *
-	 * 0001 0000
-	 */
-	val = 0x10 + clkmode;
-	write_reg(info, CHA + CCR1, val);
-
-	/* CCR2
-	 *
-	 * 07..06  BGR[9..8] Baud rate bits 9..8
-	 * 05      BDF Baud rate divisor factor, 0=1, 1=BGR value
-	 * 04      SSEL Clock source select, 1=submode b
-	 * 03      TOE 0=TxCLK is input, 0=TxCLK is input
-	 * 02      RWX Read/Write Exchange 0=disabled
-	 * 01      C32, CRC select, 0=CRC-16, 1=CRC-32
-	 * 00      DIV, data inversion 0=disabled, 1=enabled
-	 *
-	 * 0000 0000
-	 */
-	val = 0x00;
-	if (clkmode == 2 || clkmode == 3 || clkmode == 6
-	    || clkmode == 7 || (clkmode == 0 && clksubmode == 1))
-		val |= BIT5;
-	if (clksubmode)
-		val |= BIT4;
-	if (info->params.crc_type == HDLC_CRC_32_CCITT)
-		val |= BIT1;
-	if (info->params.encoding == HDLC_ENCODING_NRZB)
-		val |= BIT0;
-	write_reg(info, CHA + CCR2, val);
-
-	/* CCR3
-	 *
-	 * 07..06  PRE[1..0] Preamble count 00=1, 01=2, 10=4, 11=8
-	 * 05      EPT Enable preamble transmission, 1=enabled
-	 * 04      RADD Receive address pushed to FIFO, 0=disabled
-	 * 03      CRL CRC Reset Level, 0=FFFF
-	 * 02      RCRC Rx CRC 0=On 1=Off
-	 * 01      TCRC Tx CRC 0=On 1=Off
-	 * 00      PSD DPLL Phase Shift Disable
-	 *
-	 * 0000 0000
-	 */
-	val = 0x00;
-	if (info->params.crc_type == HDLC_CRC_NONE)
-		val |= BIT2 | BIT1;
-	if (info->params.preamble != HDLC_PREAMBLE_PATTERN_NONE)
-		val |= BIT5;
-	switch (info->params.preamble_length)
-	{
-	case HDLC_PREAMBLE_LENGTH_16BITS:
-		val |= BIT6;
-		break;
-	case HDLC_PREAMBLE_LENGTH_32BITS:
-		val |= BIT6;
-		break;
-	case HDLC_PREAMBLE_LENGTH_64BITS:
-		val |= BIT7 | BIT6;
-		break;
-	}
-	write_reg(info, CHA + CCR3, val);
-
-	/* PRE - Preamble pattern */
-	val = 0;
-	switch (info->params.preamble)
-	{
-	case HDLC_PREAMBLE_PATTERN_FLAGS: val = 0x7e; break;
-	case HDLC_PREAMBLE_PATTERN_10:    val = 0xaa; break;
-	case HDLC_PREAMBLE_PATTERN_01:    val = 0x55; break;
-	case HDLC_PREAMBLE_PATTERN_ONES:  val = 0xff; break;
-	}
-	write_reg(info, CHA + PRE, val);
-
-	/* CCR4
-	 *
-	 * 07      MCK4 Master Clock Divide by 4, 1=enabled
-	 * 06      EBRG Enhanced Baud Rate Generator Mode, 1=enabled
-	 * 05      TST1 Test Pin, 0=normal operation
-	 * 04      ICD Ivert Carrier Detect, 1=enabled (active low)
-	 * 03..02  Reserved, must be 0
-	 * 01..00  RFT[1..0] RxFIFO Threshold 00=32 bytes
-	 *
-	 * 0101 0000
-	 */
-	val = 0x50;
-	write_reg(info, CHA + CCR4, val);
-	if (info->params.flags & HDLC_FLAG_RXC_DPLL)
-		mgslpc_set_rate(info, CHA, info->params.clock_speed * 16);
-	else
-		mgslpc_set_rate(info, CHA, info->params.clock_speed);
-
-	/* RLCR Receive length check register
-	 *
-	 * 7     1=enable receive length check
-	 * 6..0  Max frame length = (RL + 1) * 32
-	 */
-	write_reg(info, CHA + RLCR, 0);
-
-	/* XBCH Transmit Byte Count High
-	 *
-	 * 07      DMA mode, 0 = interrupt driven
-	 * 06      NRM, 0=ABM (ignored)
-	 * 05      CAS Carrier Auto Start
-	 * 04      XC Transmit Continuously (ignored)
-	 * 03..00  XBC[10..8] Transmit byte count bits 10..8
-	 *
-	 * 0000 0000
-	 */
-	val = 0x00;
-	if (info->params.flags & HDLC_FLAG_AUTO_DCD)
-		val |= BIT5;
-	write_reg(info, CHA + XBCH, val);
-	enable_auxclk(info);
-	if (info->params.loopback || info->testing_irq)
-		loopback_enable(info);
-	if (info->params.flags & HDLC_FLAG_AUTO_CTS)
-	{
-		irq_enable(info, CHB, IRQ_CTS);
-		/* PVR[3] 1=AUTO CTS active */
-		set_reg_bits(info, CHA + PVR, BIT3);
-	} else
-		clear_reg_bits(info, CHA + PVR, BIT3);
-
-	irq_enable(info, CHA,
-			 IRQ_RXEOM | IRQ_RXFIFO | IRQ_ALLSENT |
-			 IRQ_UNDERRUN | IRQ_TXFIFO);
-	issue_command(info, CHA, CMD_TXRESET + CMD_RXRESET);
-	wait_command_complete(info, CHA);
-	read_reg16(info, CHA + ISR);	/* clear pending IRQs */
-
-	/* Master clock mode enabled above to allow reset commands
-	 * to complete even if no data clocks are present.
-	 *
-	 * Disable master clock mode for normal communications because
-	 * V3.2 of the ESCC2 has a bug that prevents the transmit all sent
-	 * IRQ when in master clock mode.
-	 *
-	 * Leave master clock mode enabled for IRQ test because the
-	 * timer IRQ used by the test can only happen in master clock mode.
-	 */
-	if (!info->testing_irq)
-		clear_reg_bits(info, CHA + CCR0, BIT6);
-
-	tx_set_idle(info);
-
-	tx_stop(info);
-	rx_stop(info);
-}
-
-static void rx_stop(MGSLPC_INFO *info)
-{
-	if (debug_level >= DEBUG_LEVEL_ISR)
-		printk("%s(%d):rx_stop(%s)\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	/* MODE:03 RAC Receiver Active, 0=inactive */
-	clear_reg_bits(info, CHA + MODE, BIT3);
-
-	info->rx_enabled = false;
-	info->rx_overflow = false;
-}
-
-static void rx_start(MGSLPC_INFO *info)
-{
-	if (debug_level >= DEBUG_LEVEL_ISR)
-		printk("%s(%d):rx_start(%s)\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	rx_reset_buffers(info);
-	info->rx_enabled = false;
-	info->rx_overflow = false;
-
-	/* MODE:03 RAC Receiver Active, 1=active */
-	set_reg_bits(info, CHA + MODE, BIT3);
-
-	info->rx_enabled = true;
-}
-
-static void tx_start(MGSLPC_INFO *info, struct tty_struct *tty)
-{
-	if (debug_level >= DEBUG_LEVEL_ISR)
-		printk("%s(%d):tx_start(%s)\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	if (info->tx_count) {
-		/* If auto RTS enabled and RTS is inactive, then assert */
-		/* RTS and set a flag indicating that the driver should */
-		/* negate RTS when the transmission completes. */
-		info->drop_rts_on_tx_done = false;
-
-		if (info->params.flags & HDLC_FLAG_AUTO_RTS) {
-			get_signals(info);
-			if (!(info->serial_signals & SerialSignal_RTS)) {
-				info->serial_signals |= SerialSignal_RTS;
-				set_signals(info);
-				info->drop_rts_on_tx_done = true;
-			}
-		}
-
-		if (info->params.mode == MGSL_MODE_ASYNC) {
-			if (!info->tx_active) {
-				info->tx_active = true;
-				tx_ready(info, tty);
-			}
-		} else {
-			info->tx_active = true;
-			tx_ready(info, tty);
-			mod_timer(&info->tx_timer, jiffies +
-					msecs_to_jiffies(5000));
-		}
-	}
-
-	if (!info->tx_enabled)
-		info->tx_enabled = true;
-}
-
-static void tx_stop(MGSLPC_INFO *info)
-{
-	if (debug_level >= DEBUG_LEVEL_ISR)
-		printk("%s(%d):tx_stop(%s)\n",
-			 __FILE__, __LINE__, info->device_name);
-
-	del_timer(&info->tx_timer);
-
-	info->tx_enabled = false;
-	info->tx_active = false;
-}
-
-/* Reset the adapter to a known state and prepare it for further use.
- */
-static void reset_device(MGSLPC_INFO *info)
-{
-	/* power up both channels (set BIT7) */
-	write_reg(info, CHA + CCR0, 0x80);
-	write_reg(info, CHB + CCR0, 0x80);
-	write_reg(info, CHA + MODE, 0);
-	write_reg(info, CHB + MODE, 0);
-
-	/* disable all interrupts */
-	irq_disable(info, CHA, 0xffff);
-	irq_disable(info, CHB, 0xffff);
-	port_irq_disable(info, 0xff);
-
-	/* PCR Port Configuration Register
-	 *
-	 * 07..04  DEC[3..0] Serial I/F select outputs
-	 * 03      output, 1=AUTO CTS control enabled
-	 * 02      RI Ring Indicator input 0=active
-	 * 01      DSR input 0=active
-	 * 00      DTR output 0=active
-	 *
-	 * 0000 0110
-	 */
-	write_reg(info, PCR, 0x06);
-
-	/* PVR Port Value Register
-	 *
-	 * 07..04  DEC[3..0] Serial I/F select (0000=disabled)
-	 * 03      AUTO CTS output 1=enabled
-	 * 02      RI Ring Indicator input
-	 * 01      DSR input
-	 * 00      DTR output (1=inactive)
-	 *
-	 * 0000 0001
-	 */
-//	write_reg(info, PVR, PVR_DTR);
-
-	/* IPC Interrupt Port Configuration
-	 *
-	 * 07      VIS 1=Masked interrupts visible
-	 * 06..05  Reserved, 0
-	 * 04..03  SLA Slave address, 00 ignored
-	 * 02      CASM Cascading Mode, 1=daisy chain
-	 * 01..00  IC[1..0] Interrupt Config, 01=push-pull output, active low
-	 *
-	 * 0000 0101
-	 */
-	write_reg(info, IPC, 0x05);
-}
-
-static void async_mode(MGSLPC_INFO *info)
-{
-	unsigned char val;
-
-	/* disable all interrupts */
-	irq_disable(info, CHA, 0xffff);
-	irq_disable(info, CHB, 0xffff);
-	port_irq_disable(info, 0xff);
-
-	/* MODE
-	 *
-	 * 07      Reserved, 0
-	 * 06      FRTS RTS State, 0=active
-	 * 05      FCTS Flow Control on CTS
-	 * 04      FLON Flow Control Enable
-	 * 03      RAC Receiver Active, 0 = inactive
-	 * 02      RTS 0=Auto RTS, 1=manual RTS
-	 * 01      TRS Timer Resolution, 1=512
-	 * 00      TLP Test Loop, 0 = no loop
-	 *
-	 * 0000 0110
-	 */
-	val = 0x06;
-	if (info->params.loopback)
-		val |= BIT0;
-
-	/* preserve RTS state */
-	if (!(info->serial_signals & SerialSignal_RTS))
-		val |= BIT6;
-	write_reg(info, CHA + MODE, val);
-
-	/* CCR0
-	 *
-	 * 07      PU Power Up, 1=active, 0=power down
-	 * 06      MCE Master Clock Enable, 1=enabled
-	 * 05      Reserved, 0
-	 * 04..02  SC[2..0] Encoding, 000=NRZ
-	 * 01..00  SM[1..0] Serial Mode, 11=Async
-	 *
-	 * 1000 0011
-	 */
-	write_reg(info, CHA + CCR0, 0x83);
-
-	/* CCR1
-	 *
-	 * 07..05  Reserved, 0
-	 * 04      ODS Output Driver Select, 1=TxD is push-pull output
-	 * 03      BCR Bit Clock Rate, 1=16x
-	 * 02..00  CM[2..0] Clock Mode, 111=BRG
-	 *
-	 * 0001 1111
-	 */
-	write_reg(info, CHA + CCR1, 0x1f);
-
-	/* CCR2 (channel A)
-	 *
-	 * 07..06  BGR[9..8] Baud rate bits 9..8
-	 * 05      BDF Baud rate divisor factor, 0=1, 1=BGR value
-	 * 04      SSEL Clock source select, 1=submode b
-	 * 03      TOE 0=TxCLK is input, 0=TxCLK is input
-	 * 02      RWX Read/Write Exchange 0=disabled
-	 * 01      Reserved, 0
-	 * 00      DIV, data inversion 0=disabled, 1=enabled
-	 *
-	 * 0001 0000
-	 */
-	write_reg(info, CHA + CCR2, 0x10);
-
-	/* CCR3
-	 *
-	 * 07..01  Reserved, 0
-	 * 00      PSD DPLL Phase Shift Disable
-	 *
-	 * 0000 0000
-	 */
-	write_reg(info, CHA + CCR3, 0);
-
-	/* CCR4
-	 *
-	 * 07      MCK4 Master Clock Divide by 4, 1=enabled
-	 * 06      EBRG Enhanced Baud Rate Generator Mode, 1=enabled
-	 * 05      TST1 Test Pin, 0=normal operation
-	 * 04      ICD Ivert Carrier Detect, 1=enabled (active low)
-	 * 03..00  Reserved, must be 0
-	 *
-	 * 0101 0000
-	 */
-	write_reg(info, CHA + CCR4, 0x50);
-	mgslpc_set_rate(info, CHA, info->params.data_rate * 16);
-
-	/* DAFO Data Format
-	 *
-	 * 07      Reserved, 0
-	 * 06      XBRK transmit break, 0=normal operation
-	 * 05      Stop bits (0=1, 1=2)
-	 * 04..03  PAR[1..0] Parity (01=odd, 10=even)
-	 * 02      PAREN Parity Enable
-	 * 01..00  CHL[1..0] Character Length (00=8, 01=7)
-	 *
-	 */
-	val = 0x00;
-	if (info->params.data_bits != 8)
-		val |= BIT0;	/* 7 bits */
-	if (info->params.stop_bits != 1)
-		val |= BIT5;
-	if (info->params.parity != ASYNC_PARITY_NONE)
-	{
-		val |= BIT2;	/* Parity enable */
-		if (info->params.parity == ASYNC_PARITY_ODD)
-			val |= BIT3;
-		else
-			val |= BIT4;
-	}
-	write_reg(info, CHA + DAFO, val);
-
-	/* RFC Rx FIFO Control
-	 *
-	 * 07      Reserved, 0
-	 * 06      DPS, 1=parity bit not stored in data byte
-	 * 05      DXS, 0=all data stored in FIFO (including XON/XOFF)
-	 * 04      RFDF Rx FIFO Data Format, 1=status byte stored in FIFO
-	 * 03..02  RFTH[1..0], rx threshold, 11=16 status + 16 data byte
-	 * 01      Reserved, 0
-	 * 00      TCDE Terminate Char Detect Enable, 0=disabled
-	 *
-	 * 0101 1100
-	 */
-	write_reg(info, CHA + RFC, 0x5c);
-
-	/* RLCR Receive length check register
-	 *
-	 * Max frame length = (RL + 1) * 32
-	 */
-	write_reg(info, CHA + RLCR, 0);
-
-	/* XBCH Transmit Byte Count High
-	 *
-	 * 07      DMA mode, 0 = interrupt driven
-	 * 06      NRM, 0=ABM (ignored)
-	 * 05      CAS Carrier Auto Start
-	 * 04      XC Transmit Continuously (ignored)
-	 * 03..00  XBC[10..8] Transmit byte count bits 10..8
-	 *
-	 * 0000 0000
-	 */
-	val = 0x00;
-	if (info->params.flags & HDLC_FLAG_AUTO_DCD)
-		val |= BIT5;
-	write_reg(info, CHA + XBCH, val);
-	if (info->params.flags & HDLC_FLAG_AUTO_CTS)
-		irq_enable(info, CHA, IRQ_CTS);
-
-	/* MODE:03 RAC Receiver Active, 1=active */
-	set_reg_bits(info, CHA + MODE, BIT3);
-	enable_auxclk(info);
-	if (info->params.flags & HDLC_FLAG_AUTO_CTS) {
-		irq_enable(info, CHB, IRQ_CTS);
-		/* PVR[3] 1=AUTO CTS active */
-		set_reg_bits(info, CHA + PVR, BIT3);
-	} else
-		clear_reg_bits(info, CHA + PVR, BIT3);
-	irq_enable(info, CHA,
-			  IRQ_RXEOM | IRQ_RXFIFO | IRQ_BREAK_ON | IRQ_RXTIME |
-			  IRQ_ALLSENT | IRQ_TXFIFO);
-	issue_command(info, CHA, CMD_TXRESET + CMD_RXRESET);
-	wait_command_complete(info, CHA);
-	read_reg16(info, CHA + ISR);	/* clear pending IRQs */
-}
-
-/* Set the HDLC idle mode for the transmitter.
- */
-static void tx_set_idle(MGSLPC_INFO *info)
-{
-	/* Note: ESCC2 only supports flags and one idle modes */
-	if (info->idle_mode == HDLC_TXIDLE_FLAGS)
-		set_reg_bits(info, CHA + CCR1, BIT3);
-	else
-		clear_reg_bits(info, CHA + CCR1, BIT3);
-}
-
-/* get state of the V24 status (input) signals.
- */
-static void get_signals(MGSLPC_INFO *info)
-{
-	unsigned char status = 0;
-
-	/* preserve RTS and DTR */
-	info->serial_signals &= SerialSignal_RTS | SerialSignal_DTR;
-
-	if (read_reg(info, CHB + VSTR) & BIT7)
-		info->serial_signals |= SerialSignal_DCD;
-	if (read_reg(info, CHB + STAR) & BIT1)
-		info->serial_signals |= SerialSignal_CTS;
-
-	status = read_reg(info, CHA + PVR);
-	if (!(status & PVR_RI))
-		info->serial_signals |= SerialSignal_RI;
-	if (!(status & PVR_DSR))
-		info->serial_signals |= SerialSignal_DSR;
-}
-
-/* Set the state of RTS and DTR based on contents of
- * serial_signals member of device extension.
- */
-static void set_signals(MGSLPC_INFO *info)
-{
-	unsigned char val;
-
-	val = read_reg(info, CHA + MODE);
-	if (info->params.mode == MGSL_MODE_ASYNC) {
-		if (info->serial_signals & SerialSignal_RTS)
-			val &= ~BIT6;
-		else
-			val |= BIT6;
-	} else {
-		if (info->serial_signals & SerialSignal_RTS)
-			val |= BIT2;
-		else
-			val &= ~BIT2;
-	}
-	write_reg(info, CHA + MODE, val);
-
-	if (info->serial_signals & SerialSignal_DTR)
-		clear_reg_bits(info, CHA + PVR, PVR_DTR);
-	else
-		set_reg_bits(info, CHA + PVR, PVR_DTR);
-}
-
-static void rx_reset_buffers(MGSLPC_INFO *info)
-{
-	RXBUF *buf;
-	int i;
-
-	info->rx_put = 0;
-	info->rx_get = 0;
-	info->rx_frame_count = 0;
-	for (i=0 ; i < info->rx_buf_count ; i++) {
-		buf = (RXBUF*)(info->rx_buf + (i * info->rx_buf_size));
-		buf->status = buf->count = 0;
-	}
-}
-
-/* Attempt to return a received HDLC frame
- * Only frames received without errors are returned.
- *
- * Returns true if frame returned, otherwise false
- */
-static bool rx_get_frame(MGSLPC_INFO *info, struct tty_struct *tty)
-{
-	unsigned short status;
-	RXBUF *buf;
-	unsigned int framesize = 0;
-	unsigned long flags;
-	bool return_frame = false;
-
-	if (info->rx_frame_count == 0)
-		return false;
-
-	buf = (RXBUF*)(info->rx_buf + (info->rx_get * info->rx_buf_size));
-
-	status = buf->status;
-
-	/* 07  VFR  1=valid frame
-	 * 06  RDO  1=data overrun
-	 * 05  CRC  1=OK, 0=error
-	 * 04  RAB  1=frame aborted
-	 */
-	if ((status & 0xf0) != 0xA0) {
-		if (!(status & BIT7) || (status & BIT4))
-			info->icount.rxabort++;
-		else if (status & BIT6)
-			info->icount.rxover++;
-		else if (!(status & BIT5)) {
-			info->icount.rxcrc++;
-			if (info->params.crc_type & HDLC_CRC_RETURN_EX)
-				return_frame = true;
-		}
-		framesize = 0;
-#if SYNCLINK_GENERIC_HDLC
-		{
-			info->netdev->stats.rx_errors++;
-			info->netdev->stats.rx_frame_errors++;
-		}
-#endif
-	} else
-		return_frame = true;
-
-	if (return_frame)
-		framesize = buf->count;
-
-	if (debug_level >= DEBUG_LEVEL_BH)
-		printk("%s(%d):rx_get_frame(%s) status=%04X size=%d\n",
-			__FILE__, __LINE__, info->device_name, status, framesize);
-
-	if (debug_level >= DEBUG_LEVEL_DATA)
-		trace_block(info, buf->data, framesize, 0);
-
-	if (framesize) {
-		if ((info->params.crc_type & HDLC_CRC_RETURN_EX &&
-		      framesize+1 > info->max_frame_size) ||
-		    framesize > info->max_frame_size)
-			info->icount.rxlong++;
-		else {
-			if (status & BIT5)
-				info->icount.rxok++;
-
-			if (info->params.crc_type & HDLC_CRC_RETURN_EX) {
-				*(buf->data + framesize) = status & BIT5 ? RX_OK:RX_CRC_ERROR;
-				++framesize;
-			}
-
-#if SYNCLINK_GENERIC_HDLC
-			if (info->netcount)
-				hdlcdev_rx(info, buf->data, framesize);
-			else
-#endif
-				ldisc_receive_buf(tty, buf->data, info->flag_buf, framesize);
-		}
-	}
-
-	spin_lock_irqsave(&info->lock, flags);
-	buf->status = buf->count = 0;
-	info->rx_frame_count--;
-	info->rx_get++;
-	if (info->rx_get >= info->rx_buf_count)
-		info->rx_get = 0;
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	return true;
-}
-
-static bool register_test(MGSLPC_INFO *info)
-{
-	static unsigned char patterns[] =
-	    { 0x00, 0xff, 0xaa, 0x55, 0x69, 0x96, 0x0f };
-	static unsigned int count = ARRAY_SIZE(patterns);
-	unsigned int i;
-	bool rc = true;
-	unsigned long flags;
-
-	spin_lock_irqsave(&info->lock, flags);
-	reset_device(info);
-
-	for (i = 0; i < count; i++) {
-		write_reg(info, XAD1, patterns[i]);
-		write_reg(info, XAD2, patterns[(i + 1) % count]);
-		if ((read_reg(info, XAD1) != patterns[i]) ||
-		    (read_reg(info, XAD2) != patterns[(i + 1) % count])) {
-			rc = false;
-			break;
-		}
-	}
-
-	spin_unlock_irqrestore(&info->lock, flags);
-	return rc;
-}
-
-static bool irq_test(MGSLPC_INFO *info)
-{
-	unsigned long end_time;
-	unsigned long flags;
-
-	spin_lock_irqsave(&info->lock, flags);
-	reset_device(info);
-
-	info->testing_irq = true;
-	hdlc_mode(info);
-
-	info->irq_occurred = false;
-
-	/* init hdlc mode */
-
-	irq_enable(info, CHA, IRQ_TIMER);
-	write_reg(info, CHA + TIMR, 0);	/* 512 cycles */
-	issue_command(info, CHA, CMD_START_TIMER);
-
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	end_time=100;
-	while(end_time-- && !info->irq_occurred) {
-		msleep_interruptible(10);
-	}
-
-	info->testing_irq = false;
-
-	spin_lock_irqsave(&info->lock, flags);
-	reset_device(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	return info->irq_occurred;
-}
-
-static int adapter_test(MGSLPC_INFO *info)
-{
-	if (!register_test(info)) {
-		info->init_error = DiagStatus_AddressFailure;
-		printk("%s(%d):Register test failure for device %s Addr=%04X\n",
-			__FILE__, __LINE__, info->device_name, (unsigned short)(info->io_base));
-		return -ENODEV;
-	}
-
-	if (!irq_test(info)) {
-		info->init_error = DiagStatus_IrqFailure;
-		printk("%s(%d):Interrupt test failure for device %s IRQ=%d\n",
-			__FILE__, __LINE__, info->device_name, (unsigned short)(info->irq_level));
-		return -ENODEV;
-	}
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):device %s passed diagnostics\n",
-			__FILE__, __LINE__, info->device_name);
-	return 0;
-}
-
-static void trace_block(MGSLPC_INFO *info,const char* data, int count, int xmit)
-{
-	int i;
-	int linecount;
-	if (xmit)
-		printk("%s tx data:\n", info->device_name);
-	else
-		printk("%s rx data:\n", info->device_name);
-
-	while(count) {
-		if (count > 16)
-			linecount = 16;
-		else
-			linecount = count;
-
-		for(i=0;i<linecount;i++)
-			printk("%02X ", (unsigned char)data[i]);
-		for(;i<17;i++)
-			printk("   ");
-		for(i=0;i<linecount;i++) {
-			if (data[i]>=040 && data[i]<=0176)
-				printk("%c", data[i]);
-			else
-				printk(".");
-		}
-		printk("\n");
-
-		data  += linecount;
-		count -= linecount;
-	}
-}
-
-/* HDLC frame time out
- * update stats and do tx completion processing
- */
-static void tx_timeout(struct timer_list *t)
-{
-	MGSLPC_INFO *info = from_timer(info, t, tx_timer);
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s(%d):tx_timeout(%s)\n",
-			__FILE__, __LINE__, info->device_name);
-	if (info->tx_active &&
-	    info->params.mode == MGSL_MODE_HDLC) {
-		info->icount.txtimeout++;
-	}
-	spin_lock_irqsave(&info->lock, flags);
-	info->tx_active = false;
-	info->tx_count = info->tx_put = info->tx_get = 0;
-
-	spin_unlock_irqrestore(&info->lock, flags);
-
-#if SYNCLINK_GENERIC_HDLC
-	if (info->netcount)
-		hdlcdev_tx_done(info);
-	else
-#endif
-	{
-		struct tty_struct *tty = tty_port_tty_get(&info->port);
-		bh_transmit(info, tty);
-		tty_kref_put(tty);
-	}
-}
-
-#if SYNCLINK_GENERIC_HDLC
-
-/*
- * called by generic HDLC layer when protocol selected (PPP, frame relay, etc.)
- * set encoding and frame check sequence (FCS) options
- *
- * dev       pointer to network device structure
- * encoding  serial encoding setting
- * parity    FCS setting
- *
- * returns 0 if success, otherwise error code
- */
-static int hdlcdev_attach(struct net_device *dev, unsigned short encoding,
-			  unsigned short parity)
-{
-	MGSLPC_INFO *info = dev_to_port(dev);
-	struct tty_struct *tty;
-	unsigned char  new_encoding;
-	unsigned short new_crctype;
-
-	/* return error if TTY interface open */
-	if (info->port.count)
-		return -EBUSY;
-
-	switch (encoding)
-	{
-	case ENCODING_NRZ:        new_encoding = HDLC_ENCODING_NRZ; break;
-	case ENCODING_NRZI:       new_encoding = HDLC_ENCODING_NRZI_SPACE; break;
-	case ENCODING_FM_MARK:    new_encoding = HDLC_ENCODING_BIPHASE_MARK; break;
-	case ENCODING_FM_SPACE:   new_encoding = HDLC_ENCODING_BIPHASE_SPACE; break;
-	case ENCODING_MANCHESTER: new_encoding = HDLC_ENCODING_BIPHASE_LEVEL; break;
-	default: return -EINVAL;
-	}
-
-	switch (parity)
-	{
-	case PARITY_NONE:            new_crctype = HDLC_CRC_NONE; break;
-	case PARITY_CRC16_PR1_CCITT: new_crctype = HDLC_CRC_16_CCITT; break;
-	case PARITY_CRC32_PR1_CCITT: new_crctype = HDLC_CRC_32_CCITT; break;
-	default: return -EINVAL;
-	}
-
-	info->params.encoding = new_encoding;
-	info->params.crc_type = new_crctype;
-
-	/* if network interface up, reprogram hardware */
-	if (info->netcount) {
-		tty = tty_port_tty_get(&info->port);
-		mgslpc_program_hw(info, tty);
-		tty_kref_put(tty);
-	}
-
-	return 0;
-}
-
-/*
- * called by generic HDLC layer to send frame
- *
- * skb  socket buffer containing HDLC frame
- * dev  pointer to network device structure
- */
-static netdev_tx_t hdlcdev_xmit(struct sk_buff *skb,
-				      struct net_device *dev)
-{
-	MGSLPC_INFO *info = dev_to_port(dev);
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk(KERN_INFO "%s:hdlc_xmit(%s)\n", __FILE__, dev->name);
-
-	/* stop sending until this frame completes */
-	netif_stop_queue(dev);
-
-	/* copy data to device buffers */
-	skb_copy_from_linear_data(skb, info->tx_buf, skb->len);
-	info->tx_get = 0;
-	info->tx_put = info->tx_count = skb->len;
-
-	/* update network statistics */
-	dev->stats.tx_packets++;
-	dev->stats.tx_bytes += skb->len;
-
-	/* done with socket buffer, so free it */
-	dev_kfree_skb(skb);
-
-	/* save start time for transmit timeout detection */
-	netif_trans_update(dev);
-
-	/* start hardware transmitter if necessary */
-	spin_lock_irqsave(&info->lock, flags);
-	if (!info->tx_active) {
-		struct tty_struct *tty = tty_port_tty_get(&info->port);
-		tx_start(info, tty);
-		tty_kref_put(tty);
-	}
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	return NETDEV_TX_OK;
-}
-
-/*
- * called by network layer when interface enabled
- * claim resources and initialize hardware
- *
- * dev  pointer to network device structure
- *
- * returns 0 if success, otherwise error code
- */
-static int hdlcdev_open(struct net_device *dev)
-{
-	MGSLPC_INFO *info = dev_to_port(dev);
-	struct tty_struct *tty;
-	int rc;
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s:hdlcdev_open(%s)\n", __FILE__, dev->name);
-
-	/* generic HDLC layer open processing */
-	rc = hdlc_open(dev);
-	if (rc != 0)
-		return rc;
-
-	/* arbitrate between network and tty opens */
-	spin_lock_irqsave(&info->netlock, flags);
-	if (info->port.count != 0 || info->netcount != 0) {
-		printk(KERN_WARNING "%s: hdlc_open returning busy\n", dev->name);
-		spin_unlock_irqrestore(&info->netlock, flags);
-		return -EBUSY;
-	}
-	info->netcount=1;
-	spin_unlock_irqrestore(&info->netlock, flags);
-
-	tty = tty_port_tty_get(&info->port);
-	/* claim resources and init adapter */
-	rc = startup(info, tty);
-	if (rc != 0) {
-		tty_kref_put(tty);
-		spin_lock_irqsave(&info->netlock, flags);
-		info->netcount=0;
-		spin_unlock_irqrestore(&info->netlock, flags);
-		return rc;
-	}
-	/* assert RTS and DTR, apply hardware settings */
-	info->serial_signals |= SerialSignal_RTS | SerialSignal_DTR;
-	mgslpc_program_hw(info, tty);
-	tty_kref_put(tty);
-
-	/* enable network layer transmit */
-	netif_trans_update(dev);
-	netif_start_queue(dev);
-
-	/* inform generic HDLC layer of current DCD status */
-	spin_lock_irqsave(&info->lock, flags);
-	get_signals(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-	if (info->serial_signals & SerialSignal_DCD)
-		netif_carrier_on(dev);
-	else
-		netif_carrier_off(dev);
-	return 0;
-}
-
-/*
- * called by network layer when interface is disabled
- * shutdown hardware and release resources
- *
- * dev  pointer to network device structure
- *
- * returns 0 if success, otherwise error code
- */
-static int hdlcdev_close(struct net_device *dev)
-{
-	MGSLPC_INFO *info = dev_to_port(dev);
-	struct tty_struct *tty = tty_port_tty_get(&info->port);
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s:hdlcdev_close(%s)\n", __FILE__, dev->name);
-
-	netif_stop_queue(dev);
-
-	/* shutdown adapter and release resources */
-	shutdown(info, tty);
-	tty_kref_put(tty);
-	hdlc_close(dev);
-
-	spin_lock_irqsave(&info->netlock, flags);
-	info->netcount=0;
-	spin_unlock_irqrestore(&info->netlock, flags);
-
-	return 0;
-}
-
-/*
- * called by network layer to process IOCTL call to network device
- *
- * dev  pointer to network device structure
- * ifs  pointer to network interface settings structure
- *
- * returns 0 if success, otherwise error code
- */
-static int hdlcdev_wan_ioctl(struct net_device *dev, struct if_settings *ifs)
-{
-	const size_t size = sizeof(sync_serial_settings);
-	sync_serial_settings new_line;
-	sync_serial_settings __user *line = ifs->ifs_ifsu.sync;
-	MGSLPC_INFO *info = dev_to_port(dev);
-	unsigned int flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("%s:hdlcdev_ioctl(%s)\n", __FILE__, dev->name);
-
-	/* return error if TTY interface open */
-	if (info->port.count)
-		return -EBUSY;
-
-	memset(&new_line, 0, size);
-
-	switch (ifs->type) {
-	case IF_GET_IFACE: /* return current sync_serial_settings */
-
-		ifs->type = IF_IFACE_SYNC_SERIAL;
-		if (ifs->size < size) {
-			ifs->size = size; /* data size wanted */
-			return -ENOBUFS;
-		}
-
-		flags = info->params.flags & (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_RXC_DPLL |
-					      HDLC_FLAG_RXC_BRG    | HDLC_FLAG_RXC_TXCPIN |
-					      HDLC_FLAG_TXC_TXCPIN | HDLC_FLAG_TXC_DPLL |
-					      HDLC_FLAG_TXC_BRG    | HDLC_FLAG_TXC_RXCPIN);
-
-		switch (flags){
-		case (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_TXCPIN): new_line.clock_type = CLOCK_EXT; break;
-		case (HDLC_FLAG_RXC_BRG    | HDLC_FLAG_TXC_BRG):    new_line.clock_type = CLOCK_INT; break;
-		case (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_BRG):    new_line.clock_type = CLOCK_TXINT; break;
-		case (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_RXCPIN): new_line.clock_type = CLOCK_TXFROMRX; break;
-		default: new_line.clock_type = CLOCK_DEFAULT;
-		}
-
-		new_line.clock_rate = info->params.clock_speed;
-		new_line.loopback   = info->params.loopback ? 1:0;
-
-		if (copy_to_user(line, &new_line, size))
-			return -EFAULT;
-		return 0;
-
-	case IF_IFACE_SYNC_SERIAL: /* set sync_serial_settings */
-
-		if(!capable(CAP_NET_ADMIN))
-			return -EPERM;
-		if (copy_from_user(&new_line, line, size))
-			return -EFAULT;
-
-		switch (new_line.clock_type)
-		{
-		case CLOCK_EXT:      flags = HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_TXCPIN; break;
-		case CLOCK_TXFROMRX: flags = HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_RXCPIN; break;
-		case CLOCK_INT:      flags = HDLC_FLAG_RXC_BRG    | HDLC_FLAG_TXC_BRG;    break;
-		case CLOCK_TXINT:    flags = HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_BRG;    break;
-		case CLOCK_DEFAULT:  flags = info->params.flags &
-					     (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_RXC_DPLL |
-					      HDLC_FLAG_RXC_BRG    | HDLC_FLAG_RXC_TXCPIN |
-					      HDLC_FLAG_TXC_TXCPIN | HDLC_FLAG_TXC_DPLL |
-					      HDLC_FLAG_TXC_BRG    | HDLC_FLAG_TXC_RXCPIN); break;
-		default: return -EINVAL;
-		}
-
-		if (new_line.loopback != 0 && new_line.loopback != 1)
-			return -EINVAL;
-
-		info->params.flags &= ~(HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_RXC_DPLL |
-					HDLC_FLAG_RXC_BRG    | HDLC_FLAG_RXC_TXCPIN |
-					HDLC_FLAG_TXC_TXCPIN | HDLC_FLAG_TXC_DPLL |
-					HDLC_FLAG_TXC_BRG    | HDLC_FLAG_TXC_RXCPIN);
-		info->params.flags |= flags;
-
-		info->params.loopback = new_line.loopback;
-
-		if (flags & (HDLC_FLAG_RXC_BRG | HDLC_FLAG_TXC_BRG))
-			info->params.clock_speed = new_line.clock_rate;
-		else
-			info->params.clock_speed = 0;
-
-		/* if network interface up, reprogram hardware */
-		if (info->netcount) {
-			struct tty_struct *tty = tty_port_tty_get(&info->port);
-			mgslpc_program_hw(info, tty);
-			tty_kref_put(tty);
-		}
-		return 0;
-	default:
-		return hdlc_ioctl(dev, ifs);
-	}
-}
-
-/*
- * called by network layer when transmit timeout is detected
- *
- * dev  pointer to network device structure
- */
-static void hdlcdev_tx_timeout(struct net_device *dev, unsigned int txqueue)
-{
-	MGSLPC_INFO *info = dev_to_port(dev);
-	unsigned long flags;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("hdlcdev_tx_timeout(%s)\n", dev->name);
-
-	dev->stats.tx_errors++;
-	dev->stats.tx_aborted_errors++;
-
-	spin_lock_irqsave(&info->lock, flags);
-	tx_stop(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	netif_wake_queue(dev);
-}
-
-/*
- * called by device driver when transmit completes
- * reenable network layer transmit if stopped
- *
- * info  pointer to device instance information
- */
-static void hdlcdev_tx_done(MGSLPC_INFO *info)
-{
-	if (netif_queue_stopped(info->netdev))
-		netif_wake_queue(info->netdev);
-}
-
-/*
- * called by device driver when frame received
- * pass frame to network layer
- *
- * info  pointer to device instance information
- * buf   pointer to buffer contianing frame data
- * size  count of data bytes in buf
- */
-static void hdlcdev_rx(MGSLPC_INFO *info, char *buf, int size)
-{
-	struct sk_buff *skb = dev_alloc_skb(size);
-	struct net_device *dev = info->netdev;
-
-	if (debug_level >= DEBUG_LEVEL_INFO)
-		printk("hdlcdev_rx(%s)\n", dev->name);
-
-	if (skb == NULL) {
-		printk(KERN_NOTICE "%s: can't alloc skb, dropping packet\n", dev->name);
-		dev->stats.rx_dropped++;
-		return;
-	}
-
-	skb_put_data(skb, buf, size);
-
-	skb->protocol = hdlc_type_trans(skb, dev);
-
-	dev->stats.rx_packets++;
-	dev->stats.rx_bytes += size;
-
-	netif_rx(skb);
-}
-
-static const struct net_device_ops hdlcdev_ops = {
-	.ndo_open       = hdlcdev_open,
-	.ndo_stop       = hdlcdev_close,
-	.ndo_start_xmit = hdlc_start_xmit,
-	.ndo_siocwandev = hdlcdev_wan_ioctl,
-	.ndo_tx_timeout = hdlcdev_tx_timeout,
-};
-
-/*
- * called by device driver when adding device instance
- * do generic HDLC initialization
- *
- * info  pointer to device instance information
- *
- * returns 0 if success, otherwise error code
- */
-static int hdlcdev_init(MGSLPC_INFO *info)
-{
-	int rc;
-	struct net_device *dev;
-	hdlc_device *hdlc;
-
-	/* allocate and initialize network and HDLC layer objects */
-
-	dev = alloc_hdlcdev(info);
-	if (dev == NULL) {
-		printk(KERN_ERR "%s:hdlc device allocation failure\n", __FILE__);
-		return -ENOMEM;
-	}
-
-	/* for network layer reporting purposes only */
-	dev->base_addr = info->io_base;
-	dev->irq       = info->irq_level;
-
-	/* network layer callbacks and settings */
-	dev->netdev_ops	    = &hdlcdev_ops;
-	dev->watchdog_timeo = 10 * HZ;
-	dev->tx_queue_len   = 50;
-
-	/* generic HDLC layer callbacks and settings */
-	hdlc         = dev_to_hdlc(dev);
-	hdlc->attach = hdlcdev_attach;
-	hdlc->xmit   = hdlcdev_xmit;
-
-	/* register objects with HDLC layer */
-	rc = register_hdlc_device(dev);
-	if (rc) {
-		printk(KERN_WARNING "%s:unable to register hdlc device\n", __FILE__);
-		free_netdev(dev);
-		return rc;
-	}
-
-	info->netdev = dev;
-	return 0;
-}
-
-/*
- * called by device driver when removing device instance
- * do generic HDLC cleanup
- *
- * info  pointer to device instance information
- */
-static void hdlcdev_exit(MGSLPC_INFO *info)
-{
-	unregister_hdlc_device(info->netdev);
-	free_netdev(info->netdev);
-	info->netdev = NULL;
-}
-
-#endif /* CONFIG_HDLC */
-
diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c
index 38b46c7d1737..81ed58157b15 100644
--- a/drivers/char/ppdev.c
+++ b/drivers/char/ppdev.c
@@ -841,7 +841,7 @@ static int __init ppdev_init(void)
 		pr_warn(CHRDEV ": unable to get major %d\n", PP_MAJOR);
 		return -EIO;
 	}
-	ppdev_class = class_create(THIS_MODULE, CHRDEV);
+	ppdev_class = class_create(CHRDEV);
 	if (IS_ERR(ppdev_class)) {
 		err = PTR_ERR(ppdev_class);
 		goto out_chrdev;
diff --git a/drivers/char/tpm/eventlog/common.c b/drivers/char/tpm/eventlog/common.c
index 8512ec76d526..639c3f395a5a 100644
--- a/drivers/char/tpm/eventlog/common.c
+++ b/drivers/char/tpm/eventlog/common.c
@@ -36,7 +36,7 @@ static int tpm_bios_measurements_open(struct inode *inode,
 		inode_unlock(inode);
 		return -ENODEV;
 	}
-	chip_seqops = (struct tpm_chip_seqops *)inode->i_private;
+	chip_seqops = inode->i_private;
 	seqops = chip_seqops->seqops;
 	chip = chip_seqops->chip;
 	get_device(&chip->dev);
@@ -55,8 +55,8 @@ static int tpm_bios_measurements_open(struct inode *inode,
 static int tpm_bios_measurements_release(struct inode *inode,
 					 struct file *file)
 {
-	struct seq_file *seq = (struct seq_file *)file->private_data;
-	struct tpm_chip *chip = (struct tpm_chip *)seq->private;
+	struct seq_file *seq = file->private_data;
+	struct tpm_chip *chip = seq->private;
 
 	put_device(&chip->dev);
 
diff --git a/drivers/char/tpm/st33zp24/i2c.c b/drivers/char/tpm/st33zp24/i2c.c
index c4d0b744e3cc..2d28f55ef490 100644
--- a/drivers/char/tpm/st33zp24/i2c.c
+++ b/drivers/char/tpm/st33zp24/i2c.c
@@ -138,13 +138,13 @@ static const struct i2c_device_id st33zp24_i2c_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, st33zp24_i2c_id);
 
-static const struct of_device_id of_st33zp24_i2c_match[] = {
+static const struct of_device_id of_st33zp24_i2c_match[] __maybe_unused = {
 	{ .compatible = "st,st33zp24-i2c", },
 	{}
 };
 MODULE_DEVICE_TABLE(of, of_st33zp24_i2c_match);
 
-static const struct acpi_device_id st33zp24_i2c_acpi_match[] = {
+static const struct acpi_device_id st33zp24_i2c_acpi_match[] __maybe_unused = {
 	{"SMO3324"},
 	{}
 };
diff --git a/drivers/char/tpm/st33zp24/spi.c b/drivers/char/tpm/st33zp24/spi.c
index 2154059f0235..f5811b301d3b 100644
--- a/drivers/char/tpm/st33zp24/spi.c
+++ b/drivers/char/tpm/st33zp24/spi.c
@@ -255,13 +255,13 @@ static const struct spi_device_id st33zp24_spi_id[] = {
 };
 MODULE_DEVICE_TABLE(spi, st33zp24_spi_id);
 
-static const struct of_device_id of_st33zp24_spi_match[] = {
+static const struct of_device_id of_st33zp24_spi_match[] __maybe_unused = {
 	{ .compatible = "st,st33zp24-spi", },
 	{}
 };
 MODULE_DEVICE_TABLE(of, of_st33zp24_spi_match);
 
-static const struct acpi_device_id st33zp24_spi_acpi_match[] = {
+static const struct acpi_device_id st33zp24_spi_acpi_match[] __maybe_unused = {
 	{"SMO3324"},
 	{}
 };
diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c
index 0601e6e5e326..80aaa10f436e 100644
--- a/drivers/char/tpm/tpm-chip.c
+++ b/drivers/char/tpm/tpm-chip.c
@@ -282,7 +282,7 @@ static void tpm_dev_release(struct device *dev)
  *
  * Return: always 0 (i.e. success)
  */
-static int tpm_class_shutdown(struct device *dev)
+int tpm_class_shutdown(struct device *dev)
 {
 	struct tpm_chip *chip = container_of(dev, struct tpm_chip, dev);
 
@@ -337,7 +337,6 @@ struct tpm_chip *tpm_chip_alloc(struct device *pdev,
 	device_initialize(&chip->dev);
 
 	chip->dev.class = tpm_class;
-	chip->dev.class->shutdown_pre = tpm_class_shutdown;
 	chip->dev.release = tpm_dev_release;
 	chip->dev.parent = pdev;
 	chip->dev.groups = chip->groups;
@@ -606,6 +605,30 @@ static int tpm_get_pcr_allocation(struct tpm_chip *chip)
 }
 
 /*
+ * tpm_chip_startup() - performs auto startup and allocates the PCRs
+ * @chip: TPM chip to use.
+ */
+int tpm_chip_startup(struct tpm_chip *chip)
+{
+	int rc;
+
+	rc = tpm_chip_start(chip);
+	if (rc)
+		return rc;
+
+	rc = tpm_auto_startup(chip);
+	if (rc)
+		goto stop;
+
+	rc = tpm_get_pcr_allocation(chip);
+stop:
+	tpm_chip_stop(chip);
+
+	return rc;
+}
+EXPORT_SYMBOL_GPL(tpm_chip_startup);
+
+/*
  * tpm_chip_register() - create a character device for the TPM chip
  * @chip: TPM chip to use.
  *
@@ -620,20 +643,6 @@ int tpm_chip_register(struct tpm_chip *chip)
 {
 	int rc;
 
-	rc = tpm_chip_start(chip);
-	if (rc)
-		return rc;
-	rc = tpm_auto_startup(chip);
-	if (rc) {
-		tpm_chip_stop(chip);
-		return rc;
-	}
-
-	rc = tpm_get_pcr_allocation(chip);
-	tpm_chip_stop(chip);
-	if (rc)
-		return rc;
-
 	tpm_sysfs_add_device(chip);
 
 	tpm_bios_log_setup(chip);
@@ -682,7 +691,8 @@ EXPORT_SYMBOL_GPL(tpm_chip_register);
 void tpm_chip_unregister(struct tpm_chip *chip)
 {
 	tpm_del_legacy_sysfs(chip);
-	if (IS_ENABLED(CONFIG_HW_RANDOM_TPM) && !tpm_is_firmware_upgrade(chip))
+	if (IS_ENABLED(CONFIG_HW_RANDOM_TPM) && !tpm_is_firmware_upgrade(chip) &&
+	    !tpm_amd_is_rng_defective(chip))
 		hwrng_unregister(&chip->hwrng);
 	tpm_bios_log_teardown(chip);
 	if (chip->flags & TPM_CHIP_FLAG_TPM2 && !tpm_is_firmware_upgrade(chip))
diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c
index 7e513b771832..4463d0018290 100644
--- a/drivers/char/tpm/tpm-interface.c
+++ b/drivers/char/tpm/tpm-interface.c
@@ -466,13 +466,15 @@ static int __init tpm_init(void)
 {
 	int rc;
 
-	tpm_class = class_create(THIS_MODULE, "tpm");
+	tpm_class = class_create("tpm");
 	if (IS_ERR(tpm_class)) {
 		pr_err("couldn't create tpm class\n");
 		return PTR_ERR(tpm_class);
 	}
 
-	tpmrm_class = class_create(THIS_MODULE, "tpmrm");
+	tpm_class->shutdown_pre = tpm_class_shutdown;
+
+	tpmrm_class = class_create("tpmrm");
 	if (IS_ERR(tpmrm_class)) {
 		pr_err("couldn't create tpmrm class\n");
 		rc = PTR_ERR(tpmrm_class);
diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h
index 830014a26609..eaedf569219e 100644
--- a/drivers/char/tpm/tpm.h
+++ b/drivers/char/tpm/tpm.h
@@ -256,6 +256,7 @@ int tpm1_get_pcr_allocation(struct tpm_chip *chip);
 unsigned long tpm_calc_ordinal_duration(struct tpm_chip *chip, u32 ordinal);
 int tpm_pm_suspend(struct device *dev);
 int tpm_pm_resume(struct device *dev);
+int tpm_class_shutdown(struct device *dev);
 
 static inline void tpm_msleep(unsigned int delay_msec)
 {
@@ -263,6 +264,7 @@ static inline void tpm_msleep(unsigned int delay_msec)
 		     delay_msec * 1000);
 };
 
+int tpm_chip_startup(struct tpm_chip *chip);
 int tpm_chip_start(struct tpm_chip *chip);
 void tpm_chip_stop(struct tpm_chip *chip);
 struct tpm_chip *tpm_find_get_ops(struct tpm_chip *chip);
diff --git a/drivers/char/tpm/tpm_atmel.h b/drivers/char/tpm/tpm_atmel.h
index ba37e77e8af3..7ac3f69dcf0f 100644
--- a/drivers/char/tpm/tpm_atmel.h
+++ b/drivers/char/tpm/tpm_atmel.h
@@ -26,7 +26,7 @@ struct tpm_atmel_priv {
 
 #ifdef CONFIG_PPC64
 
-#include <asm/prom.h>
+#include <linux/of.h>
 
 #define atmel_getb(priv, offset) readb(priv->iobase + offset)
 #define atmel_putb(val, priv, offset) writeb(val, priv->iobase + offset)
diff --git a/drivers/char/tpm/tpm_ftpm_tee.c b/drivers/char/tpm/tpm_ftpm_tee.c
index deff23bb54bf..528f35b14fb6 100644
--- a/drivers/char/tpm/tpm_ftpm_tee.c
+++ b/drivers/char/tpm/tpm_ftpm_tee.c
@@ -334,11 +334,11 @@ static int ftpm_tee_remove(struct device *dev)
 	return 0;
 }
 
-static int ftpm_plat_tee_remove(struct platform_device *pdev)
+static void ftpm_plat_tee_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 
-	return ftpm_tee_remove(dev);
+	ftpm_tee_remove(dev);
 }
 
 /**
@@ -367,7 +367,7 @@ static struct platform_driver ftpm_tee_plat_driver = {
 	},
 	.shutdown = ftpm_plat_tee_shutdown,
 	.probe = ftpm_plat_tee_probe,
-	.remove = ftpm_plat_tee_remove,
+	.remove_new = ftpm_plat_tee_remove,
 };
 
 /* UUID of the fTPM TA */
diff --git a/drivers/char/tpm/tpm_tis.c b/drivers/char/tpm/tpm_tis.c
index ed5dabd3c72d..7af389806643 100644
--- a/drivers/char/tpm/tpm_tis.c
+++ b/drivers/char/tpm/tpm_tis.c
@@ -50,6 +50,45 @@ static inline struct tpm_tis_tcg_phy *to_tpm_tis_tcg_phy(struct tpm_tis_data *da
 	return container_of(data, struct tpm_tis_tcg_phy, priv);
 }
 
+#ifdef CONFIG_PREEMPT_RT
+/*
+ * Flush previous write operations with a dummy read operation to the
+ * TPM MMIO base address.
+ */
+static inline void tpm_tis_flush(void __iomem *iobase)
+{
+	ioread8(iobase + TPM_ACCESS(0));
+}
+#else
+#define tpm_tis_flush(iobase) do { } while (0)
+#endif
+
+/*
+ * Write a byte word to the TPM MMIO address, and flush the write queue.
+ * The flush ensures that the data is sent immediately over the bus and not
+ * aggregated with further requests and transferred later in a batch. The large
+ * write requests can lead to unwanted latency spikes by blocking the CPU until
+ * the complete batch has been transferred.
+ */
+static inline void tpm_tis_iowrite8(u8 b, void __iomem *iobase, u32 addr)
+{
+	iowrite8(b, iobase + addr);
+	tpm_tis_flush(iobase);
+}
+
+/*
+ * Write a 32-bit word to the TPM MMIO address, and flush the write queue.
+ * The flush ensures that the data is sent immediately over the bus and not
+ * aggregated with further requests and transferred later in a batch. The large
+ * write requests can lead to unwanted latency spikes by blocking the CPU until
+ * the complete batch has been transferred.
+ */
+static inline void tpm_tis_iowrite32(u32 b, void __iomem *iobase, u32 addr)
+{
+	iowrite32(b, iobase + addr);
+	tpm_tis_flush(iobase);
+}
+
 static int interrupts = -1;
 module_param(interrupts, int, 0444);
 MODULE_PARM_DESC(interrupts, "Enable interrupts");
@@ -186,12 +225,12 @@ static int tpm_tcg_write_bytes(struct tpm_tis_data *data, u32 addr, u16 len,
 	switch (io_mode) {
 	case TPM_TIS_PHYS_8:
 		while (len--)
-			iowrite8(*value++, phy->iobase + addr);
+			tpm_tis_iowrite8(*value++, phy->iobase, addr);
 		break;
 	case TPM_TIS_PHYS_16:
 		return -EINVAL;
 	case TPM_TIS_PHYS_32:
-		iowrite32(le32_to_cpu(*((__le32 *)value)), phy->iobase + addr);
+		tpm_tis_iowrite32(le32_to_cpu(*((__le32 *)value)), phy->iobase, addr);
 		break;
 	}
 
@@ -227,7 +266,7 @@ static int tpm_tis_init(struct device *dev, struct tpm_info *tpm_info)
 		irq = tpm_info->irq;
 
 	if (itpm || is_itpm(ACPI_COMPANION(dev)))
-		phy->priv.flags |= TPM_TIS_ITPM_WORKAROUND;
+		set_bit(TPM_TIS_ITPM_WORKAROUND, &phy->priv.flags);
 
 	return tpm_tis_core_init(dev, &phy->priv, irq, &tpm_tcg,
 				 ACPI_HANDLE(dev));
@@ -324,14 +363,12 @@ static int tpm_tis_plat_probe(struct platform_device *pdev)
 	return tpm_tis_init(&pdev->dev, &tpm_info);
 }
 
-static int tpm_tis_plat_remove(struct platform_device *pdev)
+static void tpm_tis_plat_remove(struct platform_device *pdev)
 {
 	struct tpm_chip *chip = dev_get_drvdata(&pdev->dev);
 
 	tpm_chip_unregister(chip);
 	tpm_tis_remove(chip);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
@@ -344,7 +381,7 @@ MODULE_DEVICE_TABLE(of, tis_of_platform_match);
 
 static struct platform_driver tis_drv = {
 	.probe = tpm_tis_plat_probe,
-	.remove = tpm_tis_plat_remove,
+	.remove_new = tpm_tis_plat_remove,
 	.driver = {
 		.name		= "tpm_tis",
 		.pm		= &tpm_tis_pm,
diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c
index 3f98e587b3e8..c2421162cf34 100644
--- a/drivers/char/tpm/tpm_tis_core.c
+++ b/drivers/char/tpm/tpm_tis_core.c
@@ -44,6 +44,20 @@ static bool wait_for_tpm_stat_cond(struct tpm_chip *chip, u8 mask,
 	return false;
 }
 
+static u8 tpm_tis_filter_sts_mask(u8 int_mask, u8 sts_mask)
+{
+	if (!(int_mask & TPM_INTF_STS_VALID_INT))
+		sts_mask &= ~TPM_STS_VALID;
+
+	if (!(int_mask & TPM_INTF_DATA_AVAIL_INT))
+		sts_mask &= ~TPM_STS_DATA_AVAIL;
+
+	if (!(int_mask & TPM_INTF_CMD_READY_INT))
+		sts_mask &= ~TPM_STS_COMMAND_READY;
+
+	return sts_mask;
+}
+
 static int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask,
 		unsigned long timeout, wait_queue_head_t *queue,
 		bool check_cancel)
@@ -53,41 +67,56 @@ static int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask,
 	long rc;
 	u8 status;
 	bool canceled = false;
+	u8 sts_mask;
+	int ret = 0;
 
 	/* check current status */
 	status = chip->ops->status(chip);
 	if ((status & mask) == mask)
 		return 0;
 
-	stop = jiffies + timeout;
+	sts_mask = mask & (TPM_STS_VALID | TPM_STS_DATA_AVAIL |
+			   TPM_STS_COMMAND_READY);
+	/* check what status changes can be handled by irqs */
+	sts_mask = tpm_tis_filter_sts_mask(priv->int_mask, sts_mask);
 
-	if (chip->flags & TPM_CHIP_FLAG_IRQ) {
+	stop = jiffies + timeout;
+	/* process status changes with irq support */
+	if (sts_mask) {
+		ret = -ETIME;
 again:
 		timeout = stop - jiffies;
 		if ((long)timeout <= 0)
 			return -ETIME;
 		rc = wait_event_interruptible_timeout(*queue,
-			wait_for_tpm_stat_cond(chip, mask, check_cancel,
+			wait_for_tpm_stat_cond(chip, sts_mask, check_cancel,
 					       &canceled),
 			timeout);
 		if (rc > 0) {
 			if (canceled)
 				return -ECANCELED;
-			return 0;
+			ret = 0;
 		}
 		if (rc == -ERESTARTSYS && freezing(current)) {
 			clear_thread_flag(TIF_SIGPENDING);
 			goto again;
 		}
-	} else {
-		do {
-			usleep_range(priv->timeout_min,
-				     priv->timeout_max);
-			status = chip->ops->status(chip);
-			if ((status & mask) == mask)
-				return 0;
-		} while (time_before(jiffies, stop));
 	}
+
+	if (ret)
+		return ret;
+
+	mask &= ~sts_mask;
+	if (!mask) /* all done */
+		return 0;
+	/* process status changes without irq support */
+	do {
+		status = chip->ops->status(chip);
+		if ((status & mask) == mask)
+			return 0;
+		usleep_range(priv->timeout_min,
+			     priv->timeout_max);
+	} while (time_before(jiffies, stop));
 	return -ETIME;
 }
 
@@ -136,16 +165,27 @@ static bool check_locality(struct tpm_chip *chip, int l)
 	return false;
 }
 
-static int release_locality(struct tpm_chip *chip, int l)
+static int __tpm_tis_relinquish_locality(struct tpm_tis_data *priv, int l)
+{
+	tpm_tis_write8(priv, TPM_ACCESS(l), TPM_ACCESS_ACTIVE_LOCALITY);
+
+	return 0;
+}
+
+static int tpm_tis_relinquish_locality(struct tpm_chip *chip, int l)
 {
 	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
 
-	tpm_tis_write8(priv, TPM_ACCESS(l), TPM_ACCESS_ACTIVE_LOCALITY);
+	mutex_lock(&priv->locality_count_mutex);
+	priv->locality_count--;
+	if (priv->locality_count == 0)
+		__tpm_tis_relinquish_locality(priv, l);
+	mutex_unlock(&priv->locality_count_mutex);
 
 	return 0;
 }
 
-static int request_locality(struct tpm_chip *chip, int l)
+static int __tpm_tis_request_locality(struct tpm_chip *chip, int l)
 {
 	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
 	unsigned long stop, timeout;
@@ -186,6 +226,20 @@ again:
 	return -1;
 }
 
+static int tpm_tis_request_locality(struct tpm_chip *chip, int l)
+{
+	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
+	int ret = 0;
+
+	mutex_lock(&priv->locality_count_mutex);
+	if (priv->locality_count == 0)
+		ret = __tpm_tis_request_locality(chip, l);
+	if (!ret)
+		priv->locality_count++;
+	mutex_unlock(&priv->locality_count_mutex);
+	return ret;
+}
+
 static u8 tpm_tis_status(struct tpm_chip *chip)
 {
 	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
@@ -351,7 +405,7 @@ static int tpm_tis_send_data(struct tpm_chip *chip, const u8 *buf, size_t len)
 	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
 	int rc, status, burstcnt;
 	size_t count = 0;
-	bool itpm = priv->flags & TPM_TIS_ITPM_WORKAROUND;
+	bool itpm = test_bit(TPM_TIS_ITPM_WORKAROUND, &priv->flags);
 
 	status = tpm_tis_status(chip);
 	if ((status & TPM_STS_COMMAND_READY) == 0) {
@@ -484,7 +538,8 @@ static int tpm_tis_send(struct tpm_chip *chip, u8 *buf, size_t len)
 	int rc, irq;
 	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
 
-	if (!(chip->flags & TPM_CHIP_FLAG_IRQ) || priv->irq_tested)
+	if (!(chip->flags & TPM_CHIP_FLAG_IRQ) ||
+	     test_bit(TPM_TIS_IRQ_TESTED, &priv->flags))
 		return tpm_tis_send_main(chip, buf, len);
 
 	/* Verify receipt of the expected IRQ */
@@ -494,11 +549,11 @@ static int tpm_tis_send(struct tpm_chip *chip, u8 *buf, size_t len)
 	rc = tpm_tis_send_main(chip, buf, len);
 	priv->irq = irq;
 	chip->flags |= TPM_CHIP_FLAG_IRQ;
-	if (!priv->irq_tested)
+	if (!test_bit(TPM_TIS_IRQ_TESTED, &priv->flags))
 		tpm_msleep(1);
-	if (!priv->irq_tested)
+	if (!test_bit(TPM_TIS_IRQ_TESTED, &priv->flags))
 		disable_interrupts(chip);
-	priv->irq_tested = true;
+	set_bit(TPM_TIS_IRQ_TESTED, &priv->flags);
 	return rc;
 }
 
@@ -641,7 +696,7 @@ static int probe_itpm(struct tpm_chip *chip)
 	size_t len = sizeof(cmd_getticks);
 	u16 vendor;
 
-	if (priv->flags & TPM_TIS_ITPM_WORKAROUND)
+	if (test_bit(TPM_TIS_ITPM_WORKAROUND, &priv->flags))
 		return 0;
 
 	rc = tpm_tis_read16(priv, TPM_DID_VID(0), &vendor);
@@ -652,7 +707,7 @@ static int probe_itpm(struct tpm_chip *chip)
 	if (vendor != TPM_VID_INTEL)
 		return 0;
 
-	if (request_locality(chip, 0) != 0)
+	if (tpm_tis_request_locality(chip, 0) != 0)
 		return -EBUSY;
 
 	rc = tpm_tis_send_data(chip, cmd_getticks, len);
@@ -661,19 +716,19 @@ static int probe_itpm(struct tpm_chip *chip)
 
 	tpm_tis_ready(chip);
 
-	priv->flags |= TPM_TIS_ITPM_WORKAROUND;
+	set_bit(TPM_TIS_ITPM_WORKAROUND, &priv->flags);
 
 	rc = tpm_tis_send_data(chip, cmd_getticks, len);
 	if (rc == 0)
 		dev_info(&chip->dev, "Detected an iTPM.\n");
 	else {
-		priv->flags &= ~TPM_TIS_ITPM_WORKAROUND;
+		clear_bit(TPM_TIS_ITPM_WORKAROUND, &priv->flags);
 		rc = -EFAULT;
 	}
 
 out:
 	tpm_tis_ready(chip);
-	release_locality(chip, priv->locality);
+	tpm_tis_relinquish_locality(chip, priv->locality);
 
 	return rc;
 }
@@ -702,7 +757,7 @@ static irqreturn_t tis_int_handler(int dummy, void *dev_id)
 	struct tpm_chip *chip = dev_id;
 	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
 	u32 interrupt;
-	int i, rc;
+	int rc;
 
 	rc = tpm_tis_read32(priv, TPM_INT_STATUS(priv->locality), &interrupt);
 	if (rc < 0)
@@ -711,20 +766,19 @@ static irqreturn_t tis_int_handler(int dummy, void *dev_id)
 	if (interrupt == 0)
 		return IRQ_NONE;
 
-	priv->irq_tested = true;
+	set_bit(TPM_TIS_IRQ_TESTED, &priv->flags);
 	if (interrupt & TPM_INTF_DATA_AVAIL_INT)
 		wake_up_interruptible(&priv->read_queue);
-	if (interrupt & TPM_INTF_LOCALITY_CHANGE_INT)
-		for (i = 0; i < 5; i++)
-			if (check_locality(chip, i))
-				break;
+
 	if (interrupt &
 	    (TPM_INTF_LOCALITY_CHANGE_INT | TPM_INTF_STS_VALID_INT |
 	     TPM_INTF_CMD_READY_INT))
 		wake_up_interruptible(&priv->int_queue);
 
 	/* Clear interrupts handled with TPM_EOI */
+	tpm_tis_request_locality(chip, 0);
 	rc = tpm_tis_write32(priv, TPM_INT_STATUS(priv->locality), interrupt);
+	tpm_tis_relinquish_locality(chip, 0);
 	if (rc < 0)
 		return IRQ_NONE;
 
@@ -732,25 +786,22 @@ static irqreturn_t tis_int_handler(int dummy, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static int tpm_tis_gen_interrupt(struct tpm_chip *chip)
+static void tpm_tis_gen_interrupt(struct tpm_chip *chip)
 {
 	const char *desc = "attempting to generate an interrupt";
 	u32 cap2;
 	cap_t cap;
 	int ret;
 
-	ret = request_locality(chip, 0);
-	if (ret < 0)
-		return ret;
+	chip->flags |= TPM_CHIP_FLAG_IRQ;
 
 	if (chip->flags & TPM_CHIP_FLAG_TPM2)
 		ret = tpm2_get_tpm_pt(chip, 0x100, &cap2, desc);
 	else
 		ret = tpm1_getcap(chip, TPM_CAP_PROP_TIS_TIMEOUT, &cap, desc, 0);
 
-	release_locality(chip, 0);
-
-	return ret;
+	if (ret)
+		chip->flags &= ~TPM_CHIP_FLAG_IRQ;
 }
 
 /* Register the IRQ and issue a command that will cause an interrupt. If an
@@ -765,60 +816,66 @@ static int tpm_tis_probe_irq_single(struct tpm_chip *chip, u32 intmask,
 	int rc;
 	u32 int_status;
 
-	if (devm_request_irq(chip->dev.parent, irq, tis_int_handler, flags,
-			     dev_name(&chip->dev), chip) != 0) {
+
+	rc = devm_request_threaded_irq(chip->dev.parent, irq, NULL,
+				       tis_int_handler, IRQF_ONESHOT | flags,
+				       dev_name(&chip->dev), chip);
+	if (rc) {
 		dev_info(&chip->dev, "Unable to request irq: %d for probe\n",
 			 irq);
 		return -1;
 	}
 	priv->irq = irq;
 
+	rc = tpm_tis_request_locality(chip, 0);
+	if (rc < 0)
+		return rc;
+
 	rc = tpm_tis_read8(priv, TPM_INT_VECTOR(priv->locality),
 			   &original_int_vec);
-	if (rc < 0)
+	if (rc < 0) {
+		tpm_tis_relinquish_locality(chip, priv->locality);
 		return rc;
+	}
 
 	rc = tpm_tis_write8(priv, TPM_INT_VECTOR(priv->locality), irq);
 	if (rc < 0)
-		return rc;
+		goto restore_irqs;
 
 	rc = tpm_tis_read32(priv, TPM_INT_STATUS(priv->locality), &int_status);
 	if (rc < 0)
-		return rc;
+		goto restore_irqs;
 
 	/* Clear all existing */
 	rc = tpm_tis_write32(priv, TPM_INT_STATUS(priv->locality), int_status);
 	if (rc < 0)
-		return rc;
-
+		goto restore_irqs;
 	/* Turn on */
 	rc = tpm_tis_write32(priv, TPM_INT_ENABLE(priv->locality),
 			     intmask | TPM_GLOBAL_INT_ENABLE);
 	if (rc < 0)
-		return rc;
+		goto restore_irqs;
 
-	priv->irq_tested = false;
+	clear_bit(TPM_TIS_IRQ_TESTED, &priv->flags);
 
 	/* Generate an interrupt by having the core call through to
 	 * tpm_tis_send
 	 */
-	rc = tpm_tis_gen_interrupt(chip);
-	if (rc < 0)
-		return rc;
+	tpm_tis_gen_interrupt(chip);
 
+restore_irqs:
 	/* tpm_tis_send will either confirm the interrupt is working or it
 	 * will call disable_irq which undoes all of the above.
 	 */
 	if (!(chip->flags & TPM_CHIP_FLAG_IRQ)) {
-		rc = tpm_tis_write8(priv, original_int_vec,
-				TPM_INT_VECTOR(priv->locality));
-		if (rc < 0)
-			return rc;
-
-		return 1;
+		tpm_tis_write8(priv, original_int_vec,
+			       TPM_INT_VECTOR(priv->locality));
+		rc = -1;
 	}
 
-	return 0;
+	tpm_tis_relinquish_locality(chip, priv->locality);
+
+	return rc;
 }
 
 /* Try to find the IRQ the TPM is using. This is for legacy x86 systems that
@@ -932,8 +989,8 @@ static const struct tpm_class_ops tpm_tis = {
 	.req_complete_mask = TPM_STS_DATA_AVAIL | TPM_STS_VALID,
 	.req_complete_val = TPM_STS_DATA_AVAIL | TPM_STS_VALID,
 	.req_canceled = tpm_tis_req_canceled,
-	.request_locality = request_locality,
-	.relinquish_locality = release_locality,
+	.request_locality = tpm_tis_request_locality,
+	.relinquish_locality = tpm_tis_relinquish_locality,
 	.clk_enable = tpm_tis_clkrun_enable,
 };
 
@@ -967,6 +1024,8 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
 	priv->timeout_min = TPM_TIMEOUT_USECS_MIN;
 	priv->timeout_max = TPM_TIMEOUT_USECS_MAX;
 	priv->phy_ops = phy_ops;
+	priv->locality_count = 0;
+	mutex_init(&priv->locality_count_mutex);
 
 	dev_set_drvdata(&chip->dev, priv);
 
@@ -1009,18 +1068,50 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
 	if (rc < 0)
 		goto out_err;
 
-	intmask |= TPM_INTF_CMD_READY_INT | TPM_INTF_LOCALITY_CHANGE_INT |
-		   TPM_INTF_DATA_AVAIL_INT | TPM_INTF_STS_VALID_INT;
+	/* Figure out the capabilities */
+	rc = tpm_tis_read32(priv, TPM_INTF_CAPS(priv->locality), &intfcaps);
+	if (rc < 0)
+		goto out_err;
+
+	dev_dbg(dev, "TPM interface capabilities (0x%x):\n",
+		intfcaps);
+	if (intfcaps & TPM_INTF_BURST_COUNT_STATIC)
+		dev_dbg(dev, "\tBurst Count Static\n");
+	if (intfcaps & TPM_INTF_CMD_READY_INT) {
+		intmask |= TPM_INTF_CMD_READY_INT;
+		dev_dbg(dev, "\tCommand Ready Int Support\n");
+	}
+	if (intfcaps & TPM_INTF_INT_EDGE_FALLING)
+		dev_dbg(dev, "\tInterrupt Edge Falling\n");
+	if (intfcaps & TPM_INTF_INT_EDGE_RISING)
+		dev_dbg(dev, "\tInterrupt Edge Rising\n");
+	if (intfcaps & TPM_INTF_INT_LEVEL_LOW)
+		dev_dbg(dev, "\tInterrupt Level Low\n");
+	if (intfcaps & TPM_INTF_INT_LEVEL_HIGH)
+		dev_dbg(dev, "\tInterrupt Level High\n");
+	if (intfcaps & TPM_INTF_LOCALITY_CHANGE_INT) {
+		intmask |= TPM_INTF_LOCALITY_CHANGE_INT;
+		dev_dbg(dev, "\tLocality Change Int Support\n");
+	}
+	if (intfcaps & TPM_INTF_STS_VALID_INT) {
+		intmask |= TPM_INTF_STS_VALID_INT;
+		dev_dbg(dev, "\tSts Valid Int Support\n");
+	}
+	if (intfcaps & TPM_INTF_DATA_AVAIL_INT) {
+		intmask |= TPM_INTF_DATA_AVAIL_INT;
+		dev_dbg(dev, "\tData Avail Int Support\n");
+	}
+
 	intmask &= ~TPM_GLOBAL_INT_ENABLE;
 
-	rc = request_locality(chip, 0);
+	rc = tpm_tis_request_locality(chip, 0);
 	if (rc < 0) {
 		rc = -ENODEV;
 		goto out_err;
 	}
 
 	tpm_tis_write32(priv, TPM_INT_ENABLE(priv->locality), intmask);
-	release_locality(chip, 0);
+	tpm_tis_relinquish_locality(chip, 0);
 
 	rc = tpm_chip_start(chip);
 	if (rc)
@@ -1044,35 +1135,14 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
 		goto out_err;
 	}
 
-	/* Figure out the capabilities */
-	rc = tpm_tis_read32(priv, TPM_INTF_CAPS(priv->locality), &intfcaps);
-	if (rc < 0)
-		goto out_err;
-
-	dev_dbg(dev, "TPM interface capabilities (0x%x):\n",
-		intfcaps);
-	if (intfcaps & TPM_INTF_BURST_COUNT_STATIC)
-		dev_dbg(dev, "\tBurst Count Static\n");
-	if (intfcaps & TPM_INTF_CMD_READY_INT)
-		dev_dbg(dev, "\tCommand Ready Int Support\n");
-	if (intfcaps & TPM_INTF_INT_EDGE_FALLING)
-		dev_dbg(dev, "\tInterrupt Edge Falling\n");
-	if (intfcaps & TPM_INTF_INT_EDGE_RISING)
-		dev_dbg(dev, "\tInterrupt Edge Rising\n");
-	if (intfcaps & TPM_INTF_INT_LEVEL_LOW)
-		dev_dbg(dev, "\tInterrupt Level Low\n");
-	if (intfcaps & TPM_INTF_INT_LEVEL_HIGH)
-		dev_dbg(dev, "\tInterrupt Level High\n");
-	if (intfcaps & TPM_INTF_LOCALITY_CHANGE_INT)
-		dev_dbg(dev, "\tLocality Change Int Support\n");
-	if (intfcaps & TPM_INTF_STS_VALID_INT)
-		dev_dbg(dev, "\tSts Valid Int Support\n");
-	if (intfcaps & TPM_INTF_DATA_AVAIL_INT)
-		dev_dbg(dev, "\tData Avail Int Support\n");
-
 	/* INTERRUPT Setup */
 	init_waitqueue_head(&priv->read_queue);
 	init_waitqueue_head(&priv->int_queue);
+
+	rc = tpm_chip_startup(chip);
+	if (rc)
+		goto out_err;
+
 	if (irq != -1) {
 		/*
 		 * Before doing irq testing issue a command to the TPM in polling mode
@@ -1080,13 +1150,13 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
 		 * proper timeouts for the driver.
 		 */
 
-		rc = request_locality(chip, 0);
+		rc = tpm_tis_request_locality(chip, 0);
 		if (rc < 0)
 			goto out_err;
 
 		rc = tpm_get_timeouts(chip);
 
-		release_locality(chip, 0);
+		tpm_tis_relinquish_locality(chip, 0);
 
 		if (rc) {
 			dev_err(dev, "Could not get TPM timeouts and durations\n");
@@ -1094,17 +1164,23 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
 			goto out_err;
 		}
 
-		if (irq) {
+		if (irq)
 			tpm_tis_probe_irq_single(chip, intmask, IRQF_SHARED,
 						 irq);
-			if (!(chip->flags & TPM_CHIP_FLAG_IRQ)) {
-				dev_err(&chip->dev, FW_BUG
-					"TPM interrupt not working, polling instead\n");
+		else
+			tpm_tis_probe_irq(chip, intmask);
 
-				disable_interrupts(chip);
-			}
+		if (chip->flags & TPM_CHIP_FLAG_IRQ) {
+			priv->int_mask = intmask;
 		} else {
-			tpm_tis_probe_irq(chip, intmask);
+			dev_err(&chip->dev, FW_BUG
+					"TPM interrupt not working, polling instead\n");
+
+			rc = tpm_tis_request_locality(chip, 0);
+			if (rc < 0)
+				goto out_err;
+			disable_interrupts(chip);
+			tpm_tis_relinquish_locality(chip, 0);
 		}
 	}
 
@@ -1143,13 +1219,7 @@ static void tpm_tis_reenable_interrupts(struct tpm_chip *chip)
 	if (rc < 0)
 		goto out;
 
-	rc = tpm_tis_read32(priv, TPM_INT_ENABLE(priv->locality), &intmask);
-	if (rc < 0)
-		goto out;
-
-	intmask |= TPM_INTF_CMD_READY_INT
-	    | TPM_INTF_LOCALITY_CHANGE_INT | TPM_INTF_DATA_AVAIL_INT
-	    | TPM_INTF_STS_VALID_INT | TPM_GLOBAL_INT_ENABLE;
+	intmask = priv->int_mask | TPM_GLOBAL_INT_ENABLE;
 
 	tpm_tis_write32(priv, TPM_INT_ENABLE(priv->locality), intmask);
 
@@ -1165,28 +1235,27 @@ int tpm_tis_resume(struct device *dev)
 	struct tpm_chip *chip = dev_get_drvdata(dev);
 	int ret;
 
+	ret = tpm_tis_request_locality(chip, 0);
+	if (ret < 0)
+		return ret;
+
 	if (chip->flags & TPM_CHIP_FLAG_IRQ)
 		tpm_tis_reenable_interrupts(chip);
 
 	ret = tpm_pm_resume(dev);
 	if (ret)
-		return ret;
+		goto out;
 
 	/*
 	 * TPM 1.2 requires self-test on resume. This function actually returns
 	 * an error code but for unknown reason it isn't handled.
 	 */
-	if (!(chip->flags & TPM_CHIP_FLAG_TPM2)) {
-		ret = request_locality(chip, 0);
-		if (ret < 0)
-			return ret;
-
+	if (!(chip->flags & TPM_CHIP_FLAG_TPM2))
 		tpm1_do_selftest(chip);
+out:
+	tpm_tis_relinquish_locality(chip, 0);
 
-		release_locality(chip, 0);
-	}
-
-	return 0;
+	return ret;
 }
 EXPORT_SYMBOL_GPL(tpm_tis_resume);
 #endif
diff --git a/drivers/char/tpm/tpm_tis_core.h b/drivers/char/tpm/tpm_tis_core.h
index b68479e0de10..e978f457fd4d 100644
--- a/drivers/char/tpm/tpm_tis_core.h
+++ b/drivers/char/tpm/tpm_tis_core.h
@@ -87,13 +87,16 @@ enum tpm_tis_flags {
 	TPM_TIS_ITPM_WORKAROUND		= BIT(0),
 	TPM_TIS_INVALID_STATUS		= BIT(1),
 	TPM_TIS_DEFAULT_CANCELLATION	= BIT(2),
+	TPM_TIS_IRQ_TESTED		= BIT(3),
 };
 
 struct tpm_tis_data {
 	u16 manufacturer_id;
+	struct mutex locality_count_mutex;
+	unsigned int locality_count;
 	int locality;
 	int irq;
-	bool irq_tested;
+	unsigned int int_mask;
 	unsigned long flags;
 	void __iomem *ilb_base_addr;
 	u16 clkrun_enabled;
diff --git a/drivers/char/tpm/tpm_tis_i2c_cr50.c b/drivers/char/tpm/tpm_tis_i2c_cr50.c
index 77cea5b31c6e..376ae18a04eb 100644
--- a/drivers/char/tpm/tpm_tis_i2c_cr50.c
+++ b/drivers/char/tpm/tpm_tis_i2c_cr50.c
@@ -100,8 +100,7 @@ static int tpm_cr50_i2c_wait_tpm_ready(struct tpm_chip *chip)
 	}
 
 	/* Wait for interrupt to indicate TPM is ready to respond */
-	if (!wait_for_completion_timeout(&priv->tpm_ready,
-					 msecs_to_jiffies(chip->timeout_a))) {
+	if (!wait_for_completion_timeout(&priv->tpm_ready, chip->timeout_a)) {
 		dev_warn(&chip->dev, "Timeout waiting for TPM ready\n");
 		return -ETIMEDOUT;
 	}
diff --git a/drivers/char/tpm/tpm_tis_spi_main.c b/drivers/char/tpm/tpm_tis_spi_main.c
index a0963a3e92bd..1f5207974a17 100644
--- a/drivers/char/tpm/tpm_tis_spi_main.c
+++ b/drivers/char/tpm/tpm_tis_spi_main.c
@@ -231,7 +231,7 @@ static const struct spi_device_id tpm_tis_spi_id[] = {
 };
 MODULE_DEVICE_TABLE(spi, tpm_tis_spi_id);
 
-static const struct of_device_id of_tis_spi_match[] = {
+static const struct of_device_id of_tis_spi_match[] __maybe_unused = {
 	{ .compatible = "st,st33htpm-spi", .data = tpm_tis_spi_probe },
 	{ .compatible = "infineon,slb9670", .data = tpm_tis_spi_probe },
 	{ .compatible = "tcg,tpm_tis-spi", .data = tpm_tis_spi_probe },
@@ -240,7 +240,7 @@ static const struct of_device_id of_tis_spi_match[] = {
 };
 MODULE_DEVICE_TABLE(of, of_tis_spi_match);
 
-static const struct acpi_device_id acpi_tis_spi_match[] = {
+static const struct acpi_device_id acpi_tis_spi_match[] __maybe_unused = {
 	{"SMO0768", 0},
 	{}
 };
diff --git a/drivers/char/tpm/tpm_tis_synquacer.c b/drivers/char/tpm/tpm_tis_synquacer.c
index 679196c61401..49278746b0e2 100644
--- a/drivers/char/tpm/tpm_tis_synquacer.c
+++ b/drivers/char/tpm/tpm_tis_synquacer.c
@@ -127,14 +127,12 @@ static int tpm_tis_synquacer_probe(struct platform_device *pdev)
 	return tpm_tis_synquacer_init(&pdev->dev, &tpm_info);
 }
 
-static int tpm_tis_synquacer_remove(struct platform_device *pdev)
+static void tpm_tis_synquacer_remove(struct platform_device *pdev)
 {
 	struct tpm_chip *chip = dev_get_drvdata(&pdev->dev);
 
 	tpm_chip_unregister(chip);
 	tpm_tis_remove(chip);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
@@ -155,7 +153,7 @@ MODULE_DEVICE_TABLE(acpi, tpm_synquacer_acpi_tbl);
 
 static struct platform_driver tis_synquacer_drv = {
 	.probe = tpm_tis_synquacer_probe,
-	.remove = tpm_tis_synquacer_remove,
+	.remove_new = tpm_tis_synquacer_remove,
 	.driver = {
 		.name		= "tpm_tis_synquacer",
 		.pm		= &tpm_tis_synquacer_pm,
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index d5ac4d955bc8..b65c809a4e97 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -2244,7 +2244,7 @@ static int __init virtio_console_init(void)
 {
 	int err;
 
-	pdrvdata.class = class_create(THIS_MODULE, "virtio-ports");
+	pdrvdata.class = class_create("virtio-ports");
 	if (IS_ERR(pdrvdata.class)) {
 		err = PTR_ERR(pdrvdata.class);
 		pr_err("Error %d creating virtio-ports class\n", err);
diff --git a/drivers/char/xilinx_hwicap/xilinx_hwicap.c b/drivers/char/xilinx_hwicap/xilinx_hwicap.c
index 74a4928aea1d..a46f637da959 100644
--- a/drivers/char/xilinx_hwicap/xilinx_hwicap.c
+++ b/drivers/char/xilinx_hwicap/xilinx_hwicap.c
@@ -856,7 +856,7 @@ static int __init hwicap_module_init(void)
 	dev_t devt;
 	int retval;
 
-	icap_class = class_create(THIS_MODULE, "xilinx_config");
+	icap_class = class_create("xilinx_config");
 	mutex_init(&icap_sem);
 
 	devt = MKDEV(XHWICAP_MAJOR, XHWICAP_MINOR);
diff --git a/drivers/char/xillybus/xillybus_class.c b/drivers/char/xillybus/xillybus_class.c
index e9a288e61c15..89926fe9d813 100644
--- a/drivers/char/xillybus/xillybus_class.c
+++ b/drivers/char/xillybus/xillybus_class.c
@@ -242,7 +242,7 @@ EXPORT_SYMBOL(xillybus_find_inode);
 
 static int __init xillybus_class_init(void)
 {
-	xillybus_class = class_create(THIS_MODULE, "xillybus");
+	xillybus_class = class_create("xillybus");
 
 	if (IS_ERR(xillybus_class)) {
 		pr_warn("Failed to register xillybus class\n");
diff --git a/drivers/clk/clk-bm1880.c b/drivers/clk/clk-bm1880.c
index fad78a22218e..2a19e50fff68 100644
--- a/drivers/clk/clk-bm1880.c
+++ b/drivers/clk/clk-bm1880.c
@@ -949,4 +949,3 @@ module_platform_driver(bm1880_clk_driver);
 
 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
 MODULE_DESCRIPTION("Clock driver for Bitmain BM1880 SoC");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/microchip/clk-mpfs.c b/drivers/clk/microchip/clk-mpfs.c
index 4f0a19db7ed7..d85b345f4c08 100644
--- a/drivers/clk/microchip/clk-mpfs.c
+++ b/drivers/clk/microchip/clk-mpfs.c
@@ -513,4 +513,3 @@ MODULE_DESCRIPTION("Microchip PolarFire SoC Clock Driver");
 MODULE_AUTHOR("Padmarao Begari <padmarao.begari@microchip.com>");
 MODULE_AUTHOR("Daire McNamara <daire.mcnamara@microchip.com>");
 MODULE_AUTHOR("Conor Dooley <conor.dooley@microchip.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c
index b9f210437ddf..38c06f82b91d 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.c
+++ b/drivers/clk/renesas/renesas-cpg-mssr.c
@@ -1127,4 +1127,3 @@ void __init mssr_mod_nullify(struct mssr_mod_clk *mod_clks,
 }
 
 MODULE_DESCRIPTION("Renesas CPG/MSSR Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c
index 4bf40f6ccd1d..93b02cdc98c2 100644
--- a/drivers/clk/renesas/rzg2l-cpg.c
+++ b/drivers/clk/renesas/rzg2l-cpg.c
@@ -1440,4 +1440,3 @@ static int __init rzg2l_cpg_init(void)
 subsys_initcall(rzg2l_cpg_init);
 
 MODULE_DESCRIPTION("Renesas RZ/G2L CPG Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/clocksource/em_sti.c b/drivers/clocksource/em_sti.c
index c04b47bd4868..ca8d29ab70da 100644
--- a/drivers/clocksource/em_sti.c
+++ b/drivers/clocksource/em_sti.c
@@ -363,4 +363,3 @@ module_exit(em_sti_exit);
 
 MODULE_AUTHOR("Magnus Damm");
 MODULE_DESCRIPTION("Renesas Emma Mobile STI Timer Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/clocksource/ingenic-timer.c b/drivers/clocksource/ingenic-timer.c
index 24ed0f1f089b..089ce64b1c3f 100644
--- a/drivers/clocksource/ingenic-timer.c
+++ b/drivers/clocksource/ingenic-timer.c
@@ -9,13 +9,12 @@
 #include <linux/clk.h>
 #include <linux/clockchips.h>
 #include <linux/clocksource.h>
+#include <linux/cpuhotplug.h>
 #include <linux/interrupt.h>
 #include <linux/mfd/ingenic-tcu.h>
 #include <linux/mfd/syscon.h>
 #include <linux/of.h>
-#include <linux/of_address.h>
 #include <linux/of_irq.h>
-#include <linux/of_platform.h>
 #include <linux/overflow.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c
index 8b2e079d9df2..e81c588d9afe 100644
--- a/drivers/clocksource/sh_cmt.c
+++ b/drivers/clocksource/sh_cmt.c
@@ -1174,4 +1174,3 @@ module_exit(sh_cmt_exit);
 
 MODULE_AUTHOR("Magnus Damm");
 MODULE_DESCRIPTION("SuperH CMT Timer Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/clocksource/sh_mtu2.c b/drivers/clocksource/sh_mtu2.c
index 169a1fccc497..dd19553ef0b9 100644
--- a/drivers/clocksource/sh_mtu2.c
+++ b/drivers/clocksource/sh_mtu2.c
@@ -530,4 +530,3 @@ module_exit(sh_mtu2_exit);
 
 MODULE_AUTHOR("Magnus Damm");
 MODULE_DESCRIPTION("SuperH MTU2 Timer Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c
index 932f31a7c5be..beffff81c00f 100644
--- a/drivers/clocksource/sh_tmu.c
+++ b/drivers/clocksource/sh_tmu.c
@@ -674,4 +674,3 @@ module_exit(sh_tmu_exit);
 
 MODULE_AUTHOR("Magnus Damm");
 MODULE_DESCRIPTION("SuperH TMU Timer Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/clocksource/timer-clint.c b/drivers/clocksource/timer-clint.c
index 6cfe2ab73eb0..9a55e733ae99 100644
--- a/drivers/clocksource/timer-clint.c
+++ b/drivers/clocksource/timer-clint.c
@@ -17,6 +17,9 @@
 #include <linux/sched_clock.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
 #include <linux/of_irq.h>
 #include <linux/smp.h>
 #include <linux/timex.h>
@@ -31,6 +34,7 @@
 
 /* CLINT manages IPI and Timer for RISC-V M-mode  */
 static u32 __iomem *clint_ipi_base;
+static unsigned int clint_ipi_irq;
 static u64 __iomem *clint_timer_cmp;
 static u64 __iomem *clint_timer_val;
 static unsigned long clint_timer_freq;
@@ -41,12 +45,10 @@ u64 __iomem *clint_time_val;
 EXPORT_SYMBOL(clint_time_val);
 #endif
 
-static void clint_send_ipi(const struct cpumask *target)
+#ifdef CONFIG_SMP
+static void clint_send_ipi(unsigned int cpu)
 {
-	unsigned int cpu;
-
-	for_each_cpu(cpu, target)
-		writel(1, clint_ipi_base + cpuid_to_hartid_map(cpu));
+	writel(1, clint_ipi_base + cpuid_to_hartid_map(cpu));
 }
 
 static void clint_clear_ipi(void)
@@ -54,10 +56,18 @@ static void clint_clear_ipi(void)
 	writel(0, clint_ipi_base + cpuid_to_hartid_map(smp_processor_id()));
 }
 
-static struct riscv_ipi_ops clint_ipi_ops = {
-	.ipi_inject = clint_send_ipi,
-	.ipi_clear = clint_clear_ipi,
-};
+static void clint_ipi_interrupt(struct irq_desc *desc)
+{
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+
+	chained_irq_enter(chip, desc);
+
+	clint_clear_ipi();
+	ipi_mux_process();
+
+	chained_irq_exit(chip, desc);
+}
+#endif
 
 #ifdef CONFIG_64BIT
 #define clint_get_cycles()	readq_relaxed(clint_timer_val)
@@ -125,12 +135,19 @@ static int clint_timer_starting_cpu(unsigned int cpu)
 
 	enable_percpu_irq(clint_timer_irq,
 			  irq_get_trigger_type(clint_timer_irq));
+	enable_percpu_irq(clint_ipi_irq,
+			  irq_get_trigger_type(clint_ipi_irq));
 	return 0;
 }
 
 static int clint_timer_dying_cpu(unsigned int cpu)
 {
 	disable_percpu_irq(clint_timer_irq);
+	/*
+	 * Don't disable IPI when CPU goes offline because
+	 * the masking/unmasking of virtual IPIs is done
+	 * via generic IPI-Mux
+	 */
 	return 0;
 }
 
@@ -170,6 +187,12 @@ static int __init clint_timer_init_dt(struct device_node *np)
 			return -ENODEV;
 		}
 
+		/* Find parent irq domain and map ipi irq */
+		if (!clint_ipi_irq &&
+		    oirq.args[0] == RV_IRQ_SOFT &&
+		    irq_find_host(oirq.np))
+			clint_ipi_irq = irq_of_parse_and_map(np, i);
+
 		/* Find parent irq domain and map timer irq */
 		if (!clint_timer_irq &&
 		    oirq.args[0] == RV_IRQ_TIMER &&
@@ -177,9 +200,9 @@ static int __init clint_timer_init_dt(struct device_node *np)
 			clint_timer_irq = irq_of_parse_and_map(np, i);
 	}
 
-	/* If CLINT timer irq not found then fail */
-	if (!clint_timer_irq) {
-		pr_err("%pOFP: timer irq not found\n", np);
+	/* If CLINT ipi or timer irq not found then fail */
+	if (!clint_ipi_irq || !clint_timer_irq) {
+		pr_err("%pOFP: ipi/timer irq not found\n", np);
 		return -ENODEV;
 	}
 
@@ -219,6 +242,19 @@ static int __init clint_timer_init_dt(struct device_node *np)
 		goto fail_iounmap;
 	}
 
+#ifdef CONFIG_SMP
+	rc = ipi_mux_create(BITS_PER_BYTE, clint_send_ipi);
+	if (rc <= 0) {
+		pr_err("unable to create muxed IPIs\n");
+		rc = (rc < 0) ? rc : -ENODEV;
+		goto fail_free_irq;
+	}
+
+	irq_set_chained_handler(clint_ipi_irq, clint_ipi_interrupt);
+	riscv_ipi_set_virq_range(rc, BITS_PER_BYTE, true);
+	clint_clear_ipi();
+#endif
+
 	rc = cpuhp_setup_state(CPUHP_AP_CLINT_TIMER_STARTING,
 				"clockevents/clint/timer:starting",
 				clint_timer_starting_cpu,
@@ -228,13 +264,10 @@ static int __init clint_timer_init_dt(struct device_node *np)
 		goto fail_free_irq;
 	}
 
-	riscv_set_ipi_ops(&clint_ipi_ops);
-	clint_clear_ipi();
-
 	return 0;
 
 fail_free_irq:
-	free_irq(clint_timer_irq, &clint_clock_event);
+	free_percpu_irq(clint_timer_irq, &clint_clock_event);
 fail_iounmap:
 	iounmap(base);
 	return rc;
diff --git a/drivers/clocksource/timer-stm32-lp.c b/drivers/clocksource/timer-stm32-lp.c
index db2841d0beb8..4a10fb940de5 100644
--- a/drivers/clocksource/timer-stm32-lp.c
+++ b/drivers/clocksource/timer-stm32-lp.c
@@ -218,4 +218,3 @@ module_platform_driver(stm32_clkevent_lp_driver);
 
 MODULE_ALIAS("platform:stm32-lptimer-timer");
 MODULE_DESCRIPTION("STMicroelectronics STM32 clockevent low power driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/clocksource/timer-tegra186.c b/drivers/clocksource/timer-tegra186.c
index ea742889ee06..0c52c0a21830 100644
--- a/drivers/clocksource/timer-tegra186.c
+++ b/drivers/clocksource/timer-tegra186.c
@@ -511,4 +511,3 @@ module_platform_driver(tegra186_wdt_driver);
 
 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
 MODULE_DESCRIPTION("NVIDIA Tegra186 timers driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/clocksource/timer-ti-dm.c b/drivers/clocksource/timer-ti-dm.c
index b24b903a8822..1eb39834bad4 100644
--- a/drivers/clocksource/timer-ti-dm.c
+++ b/drivers/clocksource/timer-ti-dm.c
@@ -1283,5 +1283,4 @@ static struct platform_driver omap_dm_timer_driver = {
 module_platform_driver(omap_dm_timer_driver);
 
 MODULE_DESCRIPTION("OMAP Dual-Mode Timer Driver");
-MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Texas Instruments Inc");
diff --git a/drivers/comedi/comedi_fops.c b/drivers/comedi/comedi_fops.c
index b982903aaa46..8e43918d38c4 100644
--- a/drivers/comedi/comedi_fops.c
+++ b/drivers/comedi/comedi_fops.c
@@ -3383,7 +3383,7 @@ static int __init comedi_init(void)
 	if (retval)
 		goto out_unregister_chrdev_region;
 
-	comedi_class = class_create(THIS_MODULE, "comedi");
+	comedi_class = class_create("comedi");
 	if (IS_ERR(comedi_class)) {
 		retval = PTR_ERR(comedi_class);
 		pr_err("failed to create class\n");
diff --git a/drivers/comedi/drivers/comedi_test.c b/drivers/comedi/drivers/comedi_test.c
index 0b5c0af1cebf..c02dc19a679b 100644
--- a/drivers/comedi/drivers/comedi_test.c
+++ b/drivers/comedi/drivers/comedi_test.c
@@ -795,7 +795,7 @@ static int __init comedi_test_init(void)
 	}
 
 	if (!config_mode) {
-		ctcls = class_create(THIS_MODULE, CLASS_NAME);
+		ctcls = class_create(CLASS_NAME);
 		if (IS_ERR(ctcls)) {
 			pr_warn("comedi_test: unable to create class\n");
 			goto clean3;
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index 97acaa2136fd..123b4bbfcfee 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -95,7 +95,7 @@ config ARM_BRCMSTB_AVS_CPUFREQ
 	help
 	  Some Broadcom STB SoCs use a co-processor running proprietary firmware
 	  ("AVS") to handle voltage and frequency scaling. This driver provides
-	  a standard CPUfreq interface to to the firmware.
+	  a standard CPUfreq interface to the firmware.
 
 	  Say Y, if you have a Broadcom SoC with AVS support for DFS or DVFS.
 
diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c
index 78adfb2ffff6..29904395e95f 100644
--- a/drivers/cpufreq/acpi-cpufreq.c
+++ b/drivers/cpufreq/acpi-cpufreq.c
@@ -20,6 +20,7 @@
 #include <linux/dmi.h>
 #include <linux/slab.h>
 #include <linux/string_helpers.h>
+#include <linux/platform_device.h>
 
 #include <linux/acpi.h>
 #include <linux/io.h>
@@ -965,7 +966,7 @@ static void __init acpi_cpufreq_boost_init(void)
 	acpi_cpufreq_driver.boost_enabled = boost_state(0);
 }
 
-static int __init acpi_cpufreq_init(void)
+static int __init acpi_cpufreq_probe(struct platform_device *pdev)
 {
 	int ret;
 
@@ -1010,13 +1011,32 @@ static int __init acpi_cpufreq_init(void)
 	return ret;
 }
 
-static void __exit acpi_cpufreq_exit(void)
+static int acpi_cpufreq_remove(struct platform_device *pdev)
 {
 	pr_debug("%s\n", __func__);
 
 	cpufreq_unregister_driver(&acpi_cpufreq_driver);
 
 	free_acpi_perf_data();
+
+	return 0;
+}
+
+static struct platform_driver acpi_cpufreq_platdrv = {
+	.driver = {
+		.name	= "acpi-cpufreq",
+	},
+	.remove		= acpi_cpufreq_remove,
+};
+
+static int __init acpi_cpufreq_init(void)
+{
+	return platform_driver_probe(&acpi_cpufreq_platdrv, acpi_cpufreq_probe);
+}
+
+static void __exit acpi_cpufreq_exit(void)
+{
+	platform_driver_unregister(&acpi_cpufreq_platdrv);
 }
 
 module_param(acpi_pstate_strict, uint, 0644);
@@ -1027,18 +1047,4 @@ MODULE_PARM_DESC(acpi_pstate_strict,
 late_initcall(acpi_cpufreq_init);
 module_exit(acpi_cpufreq_exit);
 
-static const struct x86_cpu_id __maybe_unused acpi_cpufreq_ids[] = {
-	X86_MATCH_FEATURE(X86_FEATURE_ACPI, NULL),
-	X86_MATCH_FEATURE(X86_FEATURE_HW_PSTATE, NULL),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, acpi_cpufreq_ids);
-
-static const struct acpi_device_id __maybe_unused processor_device_ids[] = {
-	{ACPI_PROCESSOR_OBJECT_HID, },
-	{ACPI_PROCESSOR_DEVICE_HID, },
-	{},
-};
-MODULE_DEVICE_TABLE(acpi, processor_device_ids);
-
-MODULE_ALIAS("acpi");
+MODULE_ALIAS("platform:acpi-cpufreq");
diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c
index 8dd46fad151e..5a3d4aa0f45a 100644
--- a/drivers/cpufreq/amd-pstate.c
+++ b/drivers/cpufreq/amd-pstate.c
@@ -63,7 +63,6 @@ static struct cpufreq_driver *current_pstate_driver;
 static struct cpufreq_driver amd_pstate_driver;
 static struct cpufreq_driver amd_pstate_epp_driver;
 static int cppc_state = AMD_PSTATE_DISABLE;
-struct kobject *amd_pstate_kobj;
 
 /*
  * AMD Energy Preference Performance (EPP)
@@ -106,6 +105,8 @@ static unsigned int epp_values[] = {
 	[EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE,
  };
 
+typedef int (*cppc_mode_transition_fn)(int);
+
 static inline int get_mode_idx_from_str(const char *str, size_t size)
 {
 	int i;
@@ -308,7 +309,22 @@ static int cppc_init_perf(struct amd_cpudata *cpudata)
 		   cppc_perf.lowest_nonlinear_perf);
 	WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
 
-	return 0;
+	if (cppc_state == AMD_PSTATE_ACTIVE)
+		return 0;
+
+	ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf);
+	if (ret) {
+		pr_warn("failed to get auto_sel, ret: %d\n", ret);
+		return 0;
+	}
+
+	ret = cppc_set_auto_sel(cpudata->cpu,
+			(cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
+
+	if (ret)
+		pr_warn("failed to set auto_sel, ret: %d\n", ret);
+
+	return ret;
 }
 
 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
@@ -385,12 +401,18 @@ static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
 }
 
 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
-			      u32 des_perf, u32 max_perf, bool fast_switch)
+			      u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags)
 {
 	u64 prev = READ_ONCE(cpudata->cppc_req_cached);
 	u64 value = prev;
 
 	des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
+
+	if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) {
+		min_perf = des_perf;
+		des_perf = 0;
+	}
+
 	value &= ~AMD_CPPC_MIN_PERF(~0L);
 	value |= AMD_CPPC_MIN_PERF(min_perf);
 
@@ -445,7 +467,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy,
 
 	cpufreq_freq_transition_begin(policy, &freqs);
 	amd_pstate_update(cpudata, min_perf, des_perf,
-			  max_perf, false);
+			  max_perf, false, policy->governor->flags);
 	cpufreq_freq_transition_end(policy, &freqs, false);
 
 	return 0;
@@ -479,7 +501,8 @@ static void amd_pstate_adjust_perf(unsigned int cpu,
 	if (max_perf < min_perf)
 		max_perf = min_perf;
 
-	amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true);
+	amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true,
+			policy->governor->flags);
 	cpufreq_cpu_put(policy);
 }
 
@@ -816,6 +839,98 @@ static ssize_t show_energy_performance_preference(
 	return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]);
 }
 
+static void amd_pstate_driver_cleanup(void)
+{
+	amd_pstate_enable(false);
+	cppc_state = AMD_PSTATE_DISABLE;
+	current_pstate_driver = NULL;
+}
+
+static int amd_pstate_register_driver(int mode)
+{
+	int ret;
+
+	if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED)
+		current_pstate_driver = &amd_pstate_driver;
+	else if (mode == AMD_PSTATE_ACTIVE)
+		current_pstate_driver = &amd_pstate_epp_driver;
+	else
+		return -EINVAL;
+
+	cppc_state = mode;
+	ret = cpufreq_register_driver(current_pstate_driver);
+	if (ret) {
+		amd_pstate_driver_cleanup();
+		return ret;
+	}
+	return 0;
+}
+
+static int amd_pstate_unregister_driver(int dummy)
+{
+	cpufreq_unregister_driver(current_pstate_driver);
+	amd_pstate_driver_cleanup();
+	return 0;
+}
+
+static int amd_pstate_change_mode_without_dvr_change(int mode)
+{
+	int cpu = 0;
+
+	cppc_state = mode;
+
+	if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE)
+		return 0;
+
+	for_each_present_cpu(cpu) {
+		cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
+	}
+
+	return 0;
+}
+
+static int amd_pstate_change_driver_mode(int mode)
+{
+	int ret;
+
+	ret = amd_pstate_unregister_driver(0);
+	if (ret)
+		return ret;
+
+	ret = amd_pstate_register_driver(mode);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = {
+	[AMD_PSTATE_DISABLE]         = {
+		[AMD_PSTATE_DISABLE]     = NULL,
+		[AMD_PSTATE_PASSIVE]     = amd_pstate_register_driver,
+		[AMD_PSTATE_ACTIVE]      = amd_pstate_register_driver,
+		[AMD_PSTATE_GUIDED]      = amd_pstate_register_driver,
+	},
+	[AMD_PSTATE_PASSIVE]         = {
+		[AMD_PSTATE_DISABLE]     = amd_pstate_unregister_driver,
+		[AMD_PSTATE_PASSIVE]     = NULL,
+		[AMD_PSTATE_ACTIVE]      = amd_pstate_change_driver_mode,
+		[AMD_PSTATE_GUIDED]      = amd_pstate_change_mode_without_dvr_change,
+	},
+	[AMD_PSTATE_ACTIVE]          = {
+		[AMD_PSTATE_DISABLE]     = amd_pstate_unregister_driver,
+		[AMD_PSTATE_PASSIVE]     = amd_pstate_change_driver_mode,
+		[AMD_PSTATE_ACTIVE]      = NULL,
+		[AMD_PSTATE_GUIDED]      = amd_pstate_change_driver_mode,
+	},
+	[AMD_PSTATE_GUIDED]          = {
+		[AMD_PSTATE_DISABLE]     = amd_pstate_unregister_driver,
+		[AMD_PSTATE_PASSIVE]     = amd_pstate_change_mode_without_dvr_change,
+		[AMD_PSTATE_ACTIVE]      = amd_pstate_change_driver_mode,
+		[AMD_PSTATE_GUIDED]      = NULL,
+	},
+};
+
 static ssize_t amd_pstate_show_status(char *buf)
 {
 	if (!current_pstate_driver)
@@ -824,55 +939,22 @@ static ssize_t amd_pstate_show_status(char *buf)
 	return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]);
 }
 
-static void amd_pstate_driver_cleanup(void)
-{
-	current_pstate_driver = NULL;
-}
-
 static int amd_pstate_update_status(const char *buf, size_t size)
 {
-	int ret = 0;
 	int mode_idx;
 
-	if (size > 7 || size < 6)
+	if (size > strlen("passive") || size < strlen("active"))
 		return -EINVAL;
+
 	mode_idx = get_mode_idx_from_str(buf, size);
 
-	switch(mode_idx) {
-	case AMD_PSTATE_DISABLE:
-		if (current_pstate_driver) {
-			cpufreq_unregister_driver(current_pstate_driver);
-			amd_pstate_driver_cleanup();
-		}
-		break;
-	case AMD_PSTATE_PASSIVE:
-		if (current_pstate_driver) {
-			if (current_pstate_driver == &amd_pstate_driver)
-				return 0;
-			cpufreq_unregister_driver(current_pstate_driver);
-		}
+	if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX)
+		return -EINVAL;
 
-		current_pstate_driver = &amd_pstate_driver;
-		cppc_state = AMD_PSTATE_PASSIVE;
-		ret = cpufreq_register_driver(current_pstate_driver);
-		break;
-	case AMD_PSTATE_ACTIVE:
-		if (current_pstate_driver) {
-			if (current_pstate_driver == &amd_pstate_epp_driver)
-				return 0;
-			cpufreq_unregister_driver(current_pstate_driver);
-		}
+	if (mode_state_machine[cppc_state][mode_idx])
+		return mode_state_machine[cppc_state][mode_idx](mode_idx);
 
-		current_pstate_driver = &amd_pstate_epp_driver;
-		cppc_state = AMD_PSTATE_ACTIVE;
-		ret = cpufreq_register_driver(current_pstate_driver);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
+	return 0;
 }
 
 static ssize_t show_status(struct kobject *kobj,
@@ -930,6 +1012,7 @@ static struct attribute *pstate_global_attributes[] = {
 };
 
 static const struct attribute_group amd_pstate_global_attr_group = {
+	.name = "amd_pstate",
 	.attrs = pstate_global_attributes,
 };
 
@@ -1251,6 +1334,7 @@ static struct cpufreq_driver amd_pstate_epp_driver = {
 
 static int __init amd_pstate_init(void)
 {
+	struct device *dev_root;
 	int ret;
 
 	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
@@ -1277,7 +1361,7 @@ static int __init amd_pstate_init(void)
 	/* capability check */
 	if (boot_cpu_has(X86_FEATURE_CPPC)) {
 		pr_debug("AMD CPPC MSR based functionality is supported\n");
-		if (cppc_state == AMD_PSTATE_PASSIVE)
+		if (cppc_state != AMD_PSTATE_ACTIVE)
 			current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
 	} else {
 		pr_debug("AMD CPPC shared memory based functionality is supported\n");
@@ -1297,24 +1381,19 @@ static int __init amd_pstate_init(void)
 	if (ret)
 		pr_err("failed to register with return %d\n", ret);
 
-	amd_pstate_kobj = kobject_create_and_add("amd_pstate", &cpu_subsys.dev_root->kobj);
-	if (!amd_pstate_kobj) {
-		ret = -EINVAL;
-		pr_err("global sysfs registration failed.\n");
-		goto kobject_free;
-	}
-
-	ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group);
-	if (ret) {
-		pr_err("sysfs attribute export failed with error %d.\n", ret);
-		goto global_attr_free;
+	dev_root = bus_get_dev_root(&cpu_subsys);
+	if (dev_root) {
+		ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group);
+		put_device(dev_root);
+		if (ret) {
+			pr_err("sysfs attribute export failed with error %d.\n", ret);
+			goto global_attr_free;
+		}
 	}
 
 	return ret;
 
 global_attr_free:
-	kobject_put(amd_pstate_kobj);
-kobject_free:
 	cpufreq_unregister_driver(current_pstate_driver);
 	return ret;
 }
@@ -1339,7 +1418,7 @@ static int __init amd_pstate_param(char *str)
 		if (cppc_state == AMD_PSTATE_ACTIVE)
 			current_pstate_driver = &amd_pstate_epp_driver;
 
-		if (cppc_state == AMD_PSTATE_PASSIVE)
+		if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED)
 			current_pstate_driver = &amd_pstate_driver;
 
 		return 0;
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index e85703651098..338cf6cc6596 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -6,7 +6,6 @@
 
 #include <linux/err.h>
 #include <linux/of.h>
-#include <linux/of_device.h>
 #include <linux/platform_device.h>
 
 #include "cpufreq-dt.h"
@@ -152,6 +151,7 @@ static const struct of_device_id blocklist[] __initconst = {
 	{ .compatible = "qcom,sm6115", },
 	{ .compatible = "qcom,sm6350", },
 	{ .compatible = "qcom,sm6375", },
+	{ .compatible = "qcom,sm7225", },
 	{ .compatible = "qcom,sm8150", },
 	{ .compatible = "qcom,sm8250", },
 	{ .compatible = "qcom,sm8350", },
@@ -179,7 +179,7 @@ static bool __init cpu0_node_has_opp_v2_prop(void)
 	struct device_node *np = of_cpu_device_node_get(0);
 	bool ret = false;
 
-	if (of_get_property(np, "operating-points-v2", NULL))
+	if (of_property_present(np, "operating-points-v2"))
 		ret = true;
 
 	of_node_put(np);
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 6d8fd3b8dcb5..6b52ebe5a890 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -73,6 +73,11 @@ static inline bool has_target(void)
 	return cpufreq_driver->target_index || cpufreq_driver->target;
 }
 
+bool has_target_index(void)
+{
+	return !!cpufreq_driver->target_index;
+}
+
 /* internal prototypes */
 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
 static int cpufreq_init_governor(struct cpufreq_policy *policy);
@@ -725,9 +730,9 @@ static ssize_t store_##file_name					\
 	unsigned long val;						\
 	int ret;							\
 									\
-	ret = sscanf(buf, "%lu", &val);					\
-	if (ret != 1)							\
-		return -EINVAL;						\
+	ret = kstrtoul(buf, 0, &val);					\
+	if (ret)							\
+		return ret;						\
 									\
 	ret = freq_qos_update_request(policy->object##_freq_req, val);\
 	return ret >= 0 ? count : ret;					\
@@ -1727,7 +1732,7 @@ static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, b
 		 * MHz. In such cases it is better to avoid getting into
 		 * unnecessary frequency updates.
 		 */
-		if (abs(policy->cur - new_freq) < HZ_PER_MHZ)
+		if (abs(policy->cur - new_freq) < KHZ_PER_MHZ)
 			return policy->cur;
 
 		cpufreq_out_of_sync(policy, new_freq);
@@ -2932,11 +2937,16 @@ EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
 static int __init cpufreq_core_init(void)
 {
 	struct cpufreq_governor *gov = cpufreq_default_governor();
+	struct device *dev_root;
 
 	if (cpufreq_disabled())
 		return -ENODEV;
 
-	cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
+	dev_root = bus_get_dev_root(&cpu_subsys);
+	if (dev_root) {
+		cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj);
+		put_device(dev_root);
+	}
 	BUG_ON(!cpufreq_global_kobject);
 
 	if (!strlen(default_governor))
diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c
index 67e56cf638ef..c4d4643b6ca6 100644
--- a/drivers/cpufreq/freq_table.c
+++ b/drivers/cpufreq/freq_table.c
@@ -355,8 +355,13 @@ int cpufreq_table_validate_and_sort(struct cpufreq_policy *policy)
 {
 	int ret;
 
-	if (!policy->freq_table)
+	if (!policy->freq_table) {
+		/* Freq table must be passed by drivers with target_index() */
+		if (has_target_index())
+			return -EINVAL;
+
 		return 0;
+	}
 
 	ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table);
 	if (ret)
@@ -367,4 +372,3 @@ int cpufreq_table_validate_and_sort(struct cpufreq_policy *policy)
 
 MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>");
 MODULE_DESCRIPTION("CPUfreq frequency table helpers");
-MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/imx-cpufreq-dt.c b/drivers/cpufreq/imx-cpufreq-dt.c
index 76e553af2071..535867a7dfdd 100644
--- a/drivers/cpufreq/imx-cpufreq-dt.c
+++ b/drivers/cpufreq/imx-cpufreq-dt.c
@@ -89,7 +89,7 @@ static int imx_cpufreq_dt_probe(struct platform_device *pdev)
 
 	cpu_dev = get_cpu_device(0);
 
-	if (!of_find_property(cpu_dev->of_node, "cpu-supply", NULL))
+	if (!of_property_present(cpu_dev->of_node, "cpu-supply"))
 		return -ENODEV;
 
 	if (of_machine_is_compatible("fsl,imx7ulp")) {
diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c
index ad4ce8493144..48e1772e98fd 100644
--- a/drivers/cpufreq/imx6q-cpufreq.c
+++ b/drivers/cpufreq/imx6q-cpufreq.c
@@ -222,7 +222,7 @@ static int imx6q_opp_check_speed_grading(struct device *dev)
 	u32 val;
 	int ret;
 
-	if (of_find_property(dev->of_node, "nvmem-cells", NULL)) {
+	if (of_property_present(dev->of_node, "nvmem-cells")) {
 		ret = nvmem_cell_read_u32(dev, "speed_grade", &val);
 		if (ret)
 			return ret;
@@ -279,7 +279,7 @@ static int imx6ul_opp_check_speed_grading(struct device *dev)
 	u32 val;
 	int ret = 0;
 
-	if (of_find_property(dev->of_node, "nvmem-cells", NULL)) {
+	if (of_property_present(dev->of_node, "nvmem-cells")) {
 		ret = nvmem_cell_read_u32(dev, "speed_grade", &val);
 		if (ret)
 			return ret;
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 48a4613cef1e..2548ec92faa2 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -1473,10 +1473,13 @@ static struct kobject *intel_pstate_kobject;
 
 static void __init intel_pstate_sysfs_expose_params(void)
 {
+	struct device *dev_root = bus_get_dev_root(&cpu_subsys);
 	int rc;
 
-	intel_pstate_kobject = kobject_create_and_add("intel_pstate",
-						&cpu_subsys.dev_root->kobj);
+	if (dev_root) {
+		intel_pstate_kobject = kobject_create_and_add("intel_pstate", &dev_root->kobj);
+		put_device(dev_root);
+	}
 	if (WARN_ON(!intel_pstate_kobject))
 		return;
 
@@ -2384,12 +2387,6 @@ static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
 	{}
 };
 
-static const struct x86_cpu_id intel_pstate_hwp_boost_ids[] = {
-	X86_MATCH(SKYLAKE_X,		core_funcs),
-	X86_MATCH(SKYLAKE,		core_funcs),
-	{}
-};
-
 static int intel_pstate_init_cpu(unsigned int cpunum)
 {
 	struct cpudata *cpu;
@@ -2408,12 +2405,9 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
 		cpu->epp_default = -EINVAL;
 
 		if (hwp_active) {
-			const struct x86_cpu_id *id;
-
 			intel_pstate_hwp_enable(cpu);
 
-			id = x86_match_cpu(intel_pstate_hwp_boost_ids);
-			if (id && intel_pstate_acpi_pm_profile_server())
+			if (intel_pstate_acpi_pm_profile_server())
 				hwp_boost = true;
 		}
 	} else if (hwp_active) {
diff --git a/drivers/cpufreq/kirkwood-cpufreq.c b/drivers/cpufreq/kirkwood-cpufreq.c
index 70ad8fe1d78b..95588101efbd 100644
--- a/drivers/cpufreq/kirkwood-cpufreq.c
+++ b/drivers/cpufreq/kirkwood-cpufreq.c
@@ -9,7 +9,7 @@
 #include <linux/module.h>
 #include <linux/clk.h>
 #include <linux/cpufreq.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/io.h>
 #include <asm/proc-fns.h>
diff --git a/drivers/cpufreq/maple-cpufreq.c b/drivers/cpufreq/maple-cpufreq.c
index 28d346062166..f9306410a07f 100644
--- a/drivers/cpufreq/maple-cpufreq.c
+++ b/drivers/cpufreq/maple-cpufreq.c
@@ -23,7 +23,7 @@
 #include <linux/completion.h>
 #include <linux/mutex.h>
 #include <linux/time.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 
 #define DBG(fmt...) pr_debug(fmt)
 
diff --git a/drivers/cpufreq/mediatek-cpufreq.c b/drivers/cpufreq/mediatek-cpufreq.c
index 7f2680bc9a0f..9a39a7ccfae9 100644
--- a/drivers/cpufreq/mediatek-cpufreq.c
+++ b/drivers/cpufreq/mediatek-cpufreq.c
@@ -373,13 +373,13 @@ static struct device *of_get_cci(struct device *cpu_dev)
 	struct platform_device *pdev;
 
 	np = of_parse_phandle(cpu_dev->of_node, "mediatek,cci", 0);
-	if (IS_ERR_OR_NULL(np))
-		return NULL;
+	if (!np)
+		return ERR_PTR(-ENODEV);
 
 	pdev = of_find_device_by_node(np);
 	of_node_put(np);
-	if (IS_ERR_OR_NULL(pdev))
-		return NULL;
+	if (!pdev)
+		return ERR_PTR(-ENODEV);
 
 	return &pdev->dev;
 }
@@ -401,7 +401,7 @@ static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu)
 	info->ccifreq_bound = false;
 	if (info->soc_data->ccifreq_supported) {
 		info->cci_dev = of_get_cci(info->cpu_dev);
-		if (IS_ERR_OR_NULL(info->cci_dev)) {
+		if (IS_ERR(info->cci_dev)) {
 			ret = PTR_ERR(info->cci_dev);
 			dev_err(cpu_dev, "cpu%d: failed to get cci device\n", cpu);
 			return -ENODEV;
@@ -420,7 +420,7 @@ static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu)
 		ret = PTR_ERR(info->inter_clk);
 		dev_err_probe(cpu_dev, ret,
 			      "cpu%d: failed to get intermediate clk\n", cpu);
-		goto out_free_resources;
+		goto out_free_mux_clock;
 	}
 
 	info->proc_reg = regulator_get_optional(cpu_dev, "proc");
@@ -428,13 +428,13 @@ static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu)
 		ret = PTR_ERR(info->proc_reg);
 		dev_err_probe(cpu_dev, ret,
 			      "cpu%d: failed to get proc regulator\n", cpu);
-		goto out_free_resources;
+		goto out_free_inter_clock;
 	}
 
 	ret = regulator_enable(info->proc_reg);
 	if (ret) {
 		dev_warn(cpu_dev, "cpu%d: failed to enable vproc\n", cpu);
-		goto out_free_resources;
+		goto out_free_proc_reg;
 	}
 
 	/* Both presence and absence of sram regulator are valid cases. */
@@ -442,14 +442,14 @@ static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu)
 	if (IS_ERR(info->sram_reg)) {
 		ret = PTR_ERR(info->sram_reg);
 		if (ret == -EPROBE_DEFER)
-			goto out_free_resources;
+			goto out_disable_proc_reg;
 
 		info->sram_reg = NULL;
 	} else {
 		ret = regulator_enable(info->sram_reg);
 		if (ret) {
 			dev_warn(cpu_dev, "cpu%d: failed to enable vsram\n", cpu);
-			goto out_free_resources;
+			goto out_free_sram_reg;
 		}
 	}
 
@@ -458,13 +458,13 @@ static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu)
 	if (ret) {
 		dev_err(cpu_dev,
 			"cpu%d: failed to get OPP-sharing information\n", cpu);
-		goto out_free_resources;
+		goto out_disable_sram_reg;
 	}
 
 	ret = dev_pm_opp_of_cpumask_add_table(&info->cpus);
 	if (ret) {
 		dev_warn(cpu_dev, "cpu%d: no OPP table\n", cpu);
-		goto out_free_resources;
+		goto out_disable_sram_reg;
 	}
 
 	ret = clk_prepare_enable(info->cpu_clk);
@@ -533,43 +533,41 @@ out_disable_mux_clock:
 out_free_opp_table:
 	dev_pm_opp_of_cpumask_remove_table(&info->cpus);
 
-out_free_resources:
-	if (regulator_is_enabled(info->proc_reg))
-		regulator_disable(info->proc_reg);
-	if (info->sram_reg && regulator_is_enabled(info->sram_reg))
+out_disable_sram_reg:
+	if (info->sram_reg)
 		regulator_disable(info->sram_reg);
 
-	if (!IS_ERR(info->proc_reg))
-		regulator_put(info->proc_reg);
-	if (!IS_ERR(info->sram_reg))
+out_free_sram_reg:
+	if (info->sram_reg)
 		regulator_put(info->sram_reg);
-	if (!IS_ERR(info->cpu_clk))
-		clk_put(info->cpu_clk);
-	if (!IS_ERR(info->inter_clk))
-		clk_put(info->inter_clk);
+
+out_disable_proc_reg:
+	regulator_disable(info->proc_reg);
+
+out_free_proc_reg:
+	regulator_put(info->proc_reg);
+
+out_free_inter_clock:
+	clk_put(info->inter_clk);
+
+out_free_mux_clock:
+	clk_put(info->cpu_clk);
 
 	return ret;
 }
 
 static void mtk_cpu_dvfs_info_release(struct mtk_cpu_dvfs_info *info)
 {
-	if (!IS_ERR(info->proc_reg)) {
-		regulator_disable(info->proc_reg);
-		regulator_put(info->proc_reg);
-	}
-	if (!IS_ERR(info->sram_reg)) {
+	regulator_disable(info->proc_reg);
+	regulator_put(info->proc_reg);
+	if (info->sram_reg) {
 		regulator_disable(info->sram_reg);
 		regulator_put(info->sram_reg);
 	}
-	if (!IS_ERR(info->cpu_clk)) {
-		clk_disable_unprepare(info->cpu_clk);
-		clk_put(info->cpu_clk);
-	}
-	if (!IS_ERR(info->inter_clk)) {
-		clk_disable_unprepare(info->inter_clk);
-		clk_put(info->inter_clk);
-	}
-
+	clk_disable_unprepare(info->cpu_clk);
+	clk_put(info->cpu_clk);
+	clk_disable_unprepare(info->inter_clk);
+	clk_put(info->inter_clk);
 	dev_pm_opp_of_cpumask_remove_table(&info->cpus);
 	dev_pm_opp_unregister_notifier(info->cpu_dev, &info->opp_nb);
 }
@@ -695,6 +693,15 @@ static const struct mtk_cpufreq_platform_data mt2701_platform_data = {
 	.ccifreq_supported = false,
 };
 
+static const struct mtk_cpufreq_platform_data mt7622_platform_data = {
+	.min_volt_shift = 100000,
+	.max_volt_shift = 200000,
+	.proc_max_volt = 1360000,
+	.sram_min_volt = 0,
+	.sram_max_volt = 1360000,
+	.ccifreq_supported = false,
+};
+
 static const struct mtk_cpufreq_platform_data mt8183_platform_data = {
 	.min_volt_shift = 100000,
 	.max_volt_shift = 200000,
@@ -713,20 +720,29 @@ static const struct mtk_cpufreq_platform_data mt8186_platform_data = {
 	.ccifreq_supported = true,
 };
 
+static const struct mtk_cpufreq_platform_data mt8516_platform_data = {
+	.min_volt_shift = 100000,
+	.max_volt_shift = 200000,
+	.proc_max_volt = 1310000,
+	.sram_min_volt = 0,
+	.sram_max_volt = 1310000,
+	.ccifreq_supported = false,
+};
+
 /* List of machines supported by this driver */
 static const struct of_device_id mtk_cpufreq_machines[] __initconst = {
 	{ .compatible = "mediatek,mt2701", .data = &mt2701_platform_data },
 	{ .compatible = "mediatek,mt2712", .data = &mt2701_platform_data },
-	{ .compatible = "mediatek,mt7622", .data = &mt2701_platform_data },
-	{ .compatible = "mediatek,mt7623", .data = &mt2701_platform_data },
-	{ .compatible = "mediatek,mt8167", .data = &mt2701_platform_data },
+	{ .compatible = "mediatek,mt7622", .data = &mt7622_platform_data },
+	{ .compatible = "mediatek,mt7623", .data = &mt7622_platform_data },
+	{ .compatible = "mediatek,mt8167", .data = &mt8516_platform_data },
 	{ .compatible = "mediatek,mt817x", .data = &mt2701_platform_data },
 	{ .compatible = "mediatek,mt8173", .data = &mt2701_platform_data },
 	{ .compatible = "mediatek,mt8176", .data = &mt2701_platform_data },
 	{ .compatible = "mediatek,mt8183", .data = &mt8183_platform_data },
 	{ .compatible = "mediatek,mt8186", .data = &mt8186_platform_data },
 	{ .compatible = "mediatek,mt8365", .data = &mt2701_platform_data },
-	{ .compatible = "mediatek,mt8516", .data = &mt2701_platform_data },
+	{ .compatible = "mediatek,mt8516", .data = &mt8516_platform_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, mtk_cpufreq_machines);
diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c
index 9f3fc7a073d0..1d2cfea9858a 100644
--- a/drivers/cpufreq/pcc-cpufreq.c
+++ b/drivers/cpufreq/pcc-cpufreq.c
@@ -31,6 +31,7 @@
 #include <linux/cpufreq.h>
 #include <linux/compiler.h>
 #include <linux/slab.h>
+#include <linux/platform_device.h>
 
 #include <linux/acpi.h>
 #include <linux/io.h>
@@ -384,7 +385,7 @@ out_free:
 	return ret;
 }
 
-static int __init pcc_cpufreq_probe(void)
+static int __init pcc_cpufreq_evaluate(void)
 {
 	acpi_status status;
 	struct acpi_buffer output = {ACPI_ALLOCATE_BUFFER, NULL};
@@ -576,7 +577,7 @@ static struct cpufreq_driver pcc_cpufreq_driver = {
 	.name = "pcc-cpufreq",
 };
 
-static int __init pcc_cpufreq_init(void)
+static int __init pcc_cpufreq_probe(struct platform_device *pdev)
 {
 	int ret;
 
@@ -587,9 +588,9 @@ static int __init pcc_cpufreq_init(void)
 	if (acpi_disabled)
 		return -ENODEV;
 
-	ret = pcc_cpufreq_probe();
+	ret = pcc_cpufreq_evaluate();
 	if (ret) {
-		pr_debug("pcc_cpufreq_init: PCCH evaluation failed\n");
+		pr_debug("pcc_cpufreq_probe: PCCH evaluation failed\n");
 		return ret;
 	}
 
@@ -607,21 +608,35 @@ static int __init pcc_cpufreq_init(void)
 	return ret;
 }
 
-static void __exit pcc_cpufreq_exit(void)
+static int pcc_cpufreq_remove(struct platform_device *pdev)
 {
 	cpufreq_unregister_driver(&pcc_cpufreq_driver);
 
 	pcc_clear_mapping();
 
 	free_percpu(pcc_cpu_info);
+
+	return 0;
 }
 
-static const struct acpi_device_id __maybe_unused processor_device_ids[] = {
-	{ACPI_PROCESSOR_OBJECT_HID, },
-	{ACPI_PROCESSOR_DEVICE_HID, },
-	{},
+static struct platform_driver pcc_cpufreq_platdrv = {
+	.driver = {
+		.name	= "pcc-cpufreq",
+	},
+	.remove		= pcc_cpufreq_remove,
 };
-MODULE_DEVICE_TABLE(acpi, processor_device_ids);
+
+static int __init pcc_cpufreq_init(void)
+{
+	return platform_driver_probe(&pcc_cpufreq_platdrv, pcc_cpufreq_probe);
+}
+
+static void __exit pcc_cpufreq_exit(void)
+{
+	platform_driver_unregister(&pcc_cpufreq_platdrv);
+}
+
+MODULE_ALIAS("platform:pcc-cpufreq");
 
 MODULE_AUTHOR("Matthew Garrett, Naga Chumbalkar");
 MODULE_VERSION(PCC_VERSION);
diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c
index 4b8ee2014da6..ec75e79659ac 100644
--- a/drivers/cpufreq/pmac32-cpufreq.c
+++ b/drivers/cpufreq/pmac32-cpufreq.c
@@ -23,7 +23,7 @@
 #include <linux/init.h>
 #include <linux/device.h>
 #include <linux/hardirq.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 
 #include <asm/machdep.h>
 #include <asm/irq.h>
@@ -546,7 +546,7 @@ static int pmac_cpufreq_init_7447A(struct device_node *cpunode)
 {
 	struct device_node *volt_gpio_np;
 
-	if (of_get_property(cpunode, "dynamic-power-step", NULL) == NULL)
+	if (!of_property_read_bool(cpunode, "dynamic-power-step"))
 		return 1;
 
 	volt_gpio_np = of_find_node_by_name(NULL, "cpu-vcore-select");
@@ -576,7 +576,7 @@ static int pmac_cpufreq_init_750FX(struct device_node *cpunode)
 	u32 pvr;
 	const u32 *value;
 
-	if (of_get_property(cpunode, "dynamic-power-step", NULL) == NULL)
+	if (!of_property_read_bool(cpunode, "dynamic-power-step"))
 		return 1;
 
 	hi_freq = cur_freq;
@@ -632,7 +632,7 @@ static int __init pmac_cpufreq_setup(void)
 
 	/*  Check for 7447A based MacRISC3 */
 	if (of_machine_is_compatible("MacRISC3") &&
-	    of_get_property(cpunode, "dynamic-power-step", NULL) &&
+	    of_property_read_bool(cpunode, "dynamic-power-step") &&
 	    PVR_VER(mfspr(SPRN_PVR)) == 0x8003) {
 		pmac_cpufreq_init_7447A(cpunode);
 
diff --git a/drivers/cpufreq/pmac64-cpufreq.c b/drivers/cpufreq/pmac64-cpufreq.c
index ba9c31d98bd6..2cd2b06849a2 100644
--- a/drivers/cpufreq/pmac64-cpufreq.c
+++ b/drivers/cpufreq/pmac64-cpufreq.c
@@ -21,7 +21,7 @@
 #include <linux/init.h>
 #include <linux/completion.h>
 #include <linux/mutex.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 
 #include <asm/machdep.h>
 #include <asm/irq.h>
diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c
index 2f581d2d617d..a78d7a27b4b5 100644
--- a/drivers/cpufreq/qcom-cpufreq-hw.c
+++ b/drivers/cpufreq/qcom-cpufreq-hw.c
@@ -11,10 +11,9 @@
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/of_address.h>
-#include <linux/of_platform.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
 #include <linux/pm_opp.h>
-#include <linux/pm_qos.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/units.h>
@@ -29,6 +28,8 @@
 
 #define GT_IRQ_STATUS			BIT(2)
 
+#define MAX_FREQ_DOMAINS		3
+
 struct qcom_cpufreq_soc_data {
 	u32 reg_enable;
 	u32 reg_domain_state;
@@ -43,7 +44,6 @@ struct qcom_cpufreq_soc_data {
 
 struct qcom_cpufreq_data {
 	void __iomem *base;
-	struct resource *res;
 
 	/*
 	 * Mutex to synchronize between de-init sequence and re-starting LMh
@@ -58,8 +58,6 @@ struct qcom_cpufreq_data {
 	struct clk_hw cpu_clk;
 
 	bool per_core_dcvs;
-
-	struct freq_qos_request throttle_freq_req;
 };
 
 static struct {
@@ -349,8 +347,6 @@ static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data)
 
 	throttled_freq = freq_hz / HZ_PER_KHZ;
 
-	freq_qos_update_request(&data->throttle_freq_req, throttled_freq);
-
 	/* Update thermal pressure (the boost frequencies are accepted) */
 	arch_update_thermal_pressure(policy->related_cpus, throttled_freq);
 
@@ -443,14 +439,6 @@ static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index)
 	if (data->throttle_irq < 0)
 		return data->throttle_irq;
 
-	ret = freq_qos_add_request(&policy->constraints,
-				   &data->throttle_freq_req, FREQ_QOS_MAX,
-				   FREQ_QOS_MAX_DEFAULT_VALUE);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "Failed to add freq constraint (%d)\n", ret);
-		return ret;
-	}
-
 	data->cancel_throttle = false;
 	data->policy = policy;
 
@@ -517,7 +505,6 @@ static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data)
 	if (data->throttle_irq <= 0)
 		return;
 
-	freq_qos_remove_request(&data->throttle_freq_req);
 	free_irq(data->throttle_irq, data);
 }
 
@@ -590,16 +577,12 @@ static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy)
 {
 	struct device *cpu_dev = get_cpu_device(policy->cpu);
 	struct qcom_cpufreq_data *data = policy->driver_data;
-	struct resource *res = data->res;
-	void __iomem *base = data->base;
 
 	dev_pm_opp_remove_all_dynamic(cpu_dev);
 	dev_pm_opp_of_cpumask_remove_table(policy->related_cpus);
 	qcom_cpufreq_hw_lmh_exit(data);
 	kfree(policy->freq_table);
 	kfree(data);
-	iounmap(base);
-	release_mem_region(res->start, resource_size(res));
 
 	return 0;
 }
@@ -651,10 +634,9 @@ static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev)
 {
 	struct clk_hw_onecell_data *clk_data;
 	struct device *dev = &pdev->dev;
-	struct device_node *soc_node;
 	struct device *cpu_dev;
 	struct clk *clk;
-	int ret, i, num_domains, reg_sz;
+	int ret, i, num_domains;
 
 	clk = clk_get(dev, "xo");
 	if (IS_ERR(clk))
@@ -681,24 +663,9 @@ static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	/* Allocate qcom_cpufreq_data based on the available frequency domains in DT */
-	soc_node = of_get_parent(dev->of_node);
-	if (!soc_node)
-		return -EINVAL;
-
-	ret = of_property_read_u32(soc_node, "#address-cells", &reg_sz);
-	if (ret)
-		goto of_exit;
-
-	ret = of_property_read_u32(soc_node, "#size-cells", &i);
-	if (ret)
-		goto of_exit;
-
-	reg_sz += i;
-
-	num_domains = of_property_count_elems_of_size(dev->of_node, "reg", sizeof(u32) * reg_sz);
-	if (num_domains <= 0)
-		return num_domains;
+	for (num_domains = 0; num_domains < MAX_FREQ_DOMAINS; num_domains++)
+		if (!platform_get_resource(pdev, IORESOURCE_MEM, num_domains))
+			break;
 
 	qcom_cpufreq.data = devm_kzalloc(dev, sizeof(struct qcom_cpufreq_data) * num_domains,
 					 GFP_KERNEL);
@@ -718,17 +685,15 @@ static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev)
 	for (i = 0; i < num_domains; i++) {
 		struct qcom_cpufreq_data *data = &qcom_cpufreq.data[i];
 		struct clk_init_data clk_init = {};
-		struct resource *res;
 		void __iomem *base;
 
-		base = devm_platform_get_and_ioremap_resource(pdev, i, &res);
+		base = devm_platform_ioremap_resource(pdev, i);
 		if (IS_ERR(base)) {
-			dev_err(dev, "Failed to map resource %pR\n", res);
+			dev_err(dev, "Failed to map resource index %d\n", i);
 			return PTR_ERR(base);
 		}
 
 		data->base = base;
-		data->res = res;
 
 		/* Register CPU clock for each frequency domain */
 		clk_init.name = kasprintf(GFP_KERNEL, "qcom_cpufreq%d", i);
@@ -762,9 +727,6 @@ static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev)
 	else
 		dev_dbg(dev, "QCOM CPUFreq HW driver initialized\n");
 
-of_exit:
-	of_node_put(soc_node);
-
 	return ret;
 }
 
diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c
index 513a071845c2..f34e6382a4c5 100644
--- a/drivers/cpufreq/scmi-cpufreq.c
+++ b/drivers/cpufreq/scmi-cpufreq.c
@@ -310,7 +310,7 @@ static int scmi_cpufreq_probe(struct scmi_device *sdev)
 
 #ifdef CONFIG_COMMON_CLK
 	/* dummy clock provider as needed by OPP if clocks property is used */
-	if (of_find_property(dev->of_node, "#clock-cells", NULL))
+	if (of_property_present(dev->of_node, "#clock-cells"))
 		devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, NULL);
 #endif
 
diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c
index c6fdf019dbde..78b875db6b66 100644
--- a/drivers/cpufreq/spear-cpufreq.c
+++ b/drivers/cpufreq/spear-cpufreq.c
@@ -18,7 +18,7 @@
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/module.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/types.h>
diff --git a/drivers/cpufreq/sun50i-cpufreq-nvmem.c b/drivers/cpufreq/sun50i-cpufreq-nvmem.c
index 1583a370da39..4321d7bbe769 100644
--- a/drivers/cpufreq/sun50i-cpufreq-nvmem.c
+++ b/drivers/cpufreq/sun50i-cpufreq-nvmem.c
@@ -10,9 +10,10 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/cpu.h>
 #include <linux/module.h>
 #include <linux/nvmem-consumer.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm_opp.h>
 #include <linux/slab.h>
diff --git a/drivers/cpufreq/tegra124-cpufreq.c b/drivers/cpufreq/tegra124-cpufreq.c
index 7a1ea6fdcab6..aae951d4e77c 100644
--- a/drivers/cpufreq/tegra124-cpufreq.c
+++ b/drivers/cpufreq/tegra124-cpufreq.c
@@ -11,7 +11,6 @@
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/of_device.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm_opp.h>
@@ -221,4 +220,3 @@ module_init(tegra_cpufreq_init);
 
 MODULE_AUTHOR("Tuomas Tynkkynen <ttynkkynen@nvidia.com>");
 MODULE_DESCRIPTION("cpufreq driver for NVIDIA Tegra124");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c
index 5890e25d7f77..c8d03346068a 100644
--- a/drivers/cpufreq/tegra194-cpufreq.c
+++ b/drivers/cpufreq/tegra194-cpufreq.c
@@ -12,6 +12,7 @@
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
+#include <linux/units.h>
 
 #include <asm/smp_plat.h>
 
@@ -65,12 +66,36 @@ struct tegra_cpufreq_soc {
 
 struct tegra194_cpufreq_data {
 	void __iomem *regs;
-	struct cpufreq_frequency_table **tables;
+	struct cpufreq_frequency_table **bpmp_luts;
 	const struct tegra_cpufreq_soc *soc;
+	bool icc_dram_bw_scaling;
 };
 
 static struct workqueue_struct *read_counters_wq;
 
+static int tegra_cpufreq_set_bw(struct cpufreq_policy *policy, unsigned long freq_khz)
+{
+	struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+	struct dev_pm_opp *opp;
+	struct device *dev;
+	int ret;
+
+	dev = get_cpu_device(policy->cpu);
+	if (!dev)
+		return -ENODEV;
+
+	opp = dev_pm_opp_find_freq_exact(dev, freq_khz * KHZ, true);
+	if (IS_ERR(opp))
+		return PTR_ERR(opp);
+
+	ret = dev_pm_opp_set_opp(dev, opp);
+	if (ret)
+		data->icc_dram_bw_scaling = false;
+
+	dev_pm_opp_put(opp);
+	return ret;
+}
+
 static void tegra_get_cpu_mpidr(void *mpidr)
 {
 	*((u64 *)mpidr) = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
@@ -354,7 +379,7 @@ static unsigned int tegra194_get_speed(u32 cpu)
 	 * to the last written ndiv value from freq_table. This is
 	 * done to return consistent value.
 	 */
-	cpufreq_for_each_valid_entry(pos, data->tables[clusterid]) {
+	cpufreq_for_each_valid_entry(pos, data->bpmp_luts[clusterid]) {
 		if (pos->driver_data != ndiv)
 			continue;
 
@@ -369,16 +394,93 @@ static unsigned int tegra194_get_speed(u32 cpu)
 	return rate;
 }
 
+static int tegra_cpufreq_init_cpufreq_table(struct cpufreq_policy *policy,
+					    struct cpufreq_frequency_table *bpmp_lut,
+					    struct cpufreq_frequency_table **opp_table)
+{
+	struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+	struct cpufreq_frequency_table *freq_table = NULL;
+	struct cpufreq_frequency_table *pos;
+	struct device *cpu_dev;
+	struct dev_pm_opp *opp;
+	unsigned long rate;
+	int ret, max_opps;
+	int j = 0;
+
+	cpu_dev = get_cpu_device(policy->cpu);
+	if (!cpu_dev) {
+		pr_err("%s: failed to get cpu%d device\n", __func__, policy->cpu);
+		return -ENODEV;
+	}
+
+	/* Initialize OPP table mentioned in operating-points-v2 property in DT */
+	ret = dev_pm_opp_of_add_table_indexed(cpu_dev, 0);
+	if (!ret) {
+		max_opps = dev_pm_opp_get_opp_count(cpu_dev);
+		if (max_opps <= 0) {
+			dev_err(cpu_dev, "Failed to add OPPs\n");
+			return max_opps;
+		}
+
+		/* Disable all opps and cross-validate against LUT later */
+		for (rate = 0; ; rate++) {
+			opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate);
+			if (IS_ERR(opp))
+				break;
+
+			dev_pm_opp_put(opp);
+			dev_pm_opp_disable(cpu_dev, rate);
+		}
+	} else {
+		dev_err(cpu_dev, "Invalid or empty opp table in device tree\n");
+		data->icc_dram_bw_scaling = false;
+		return ret;
+	}
+
+	freq_table = kcalloc((max_opps + 1), sizeof(*freq_table), GFP_KERNEL);
+	if (!freq_table)
+		return -ENOMEM;
+
+	/*
+	 * Cross check the frequencies from BPMP-FW LUT against the OPP's present in DT.
+	 * Enable only those DT OPP's which are present in LUT also.
+	 */
+	cpufreq_for_each_valid_entry(pos, bpmp_lut) {
+		opp = dev_pm_opp_find_freq_exact(cpu_dev, pos->frequency * KHZ, false);
+		if (IS_ERR(opp))
+			continue;
+
+		ret = dev_pm_opp_enable(cpu_dev, pos->frequency * KHZ);
+		if (ret < 0)
+			return ret;
+
+		freq_table[j].driver_data = pos->driver_data;
+		freq_table[j].frequency = pos->frequency;
+		j++;
+	}
+
+	freq_table[j].driver_data = pos->driver_data;
+	freq_table[j].frequency = CPUFREQ_TABLE_END;
+
+	*opp_table = &freq_table[0];
+
+	dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
+
+	return ret;
+}
+
 static int tegra194_cpufreq_init(struct cpufreq_policy *policy)
 {
 	struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
 	int maxcpus_per_cluster = data->soc->maxcpus_per_cluster;
+	struct cpufreq_frequency_table *freq_table;
+	struct cpufreq_frequency_table *bpmp_lut;
 	u32 start_cpu, cpu;
 	u32 clusterid;
+	int ret;
 
 	data->soc->ops->get_cpu_cluster_id(policy->cpu, NULL, &clusterid);
-
-	if (clusterid >= data->soc->num_clusters || !data->tables[clusterid])
+	if (clusterid >= data->soc->num_clusters || !data->bpmp_luts[clusterid])
 		return -EINVAL;
 
 	start_cpu = rounddown(policy->cpu, maxcpus_per_cluster);
@@ -387,9 +489,22 @@ static int tegra194_cpufreq_init(struct cpufreq_policy *policy)
 		if (cpu_possible(cpu))
 			cpumask_set_cpu(cpu, policy->cpus);
 	}
-	policy->freq_table = data->tables[clusterid];
 	policy->cpuinfo.transition_latency = TEGRA_CPUFREQ_TRANSITION_LATENCY;
 
+	bpmp_lut = data->bpmp_luts[clusterid];
+
+	if (data->icc_dram_bw_scaling) {
+		ret = tegra_cpufreq_init_cpufreq_table(policy, bpmp_lut, &freq_table);
+		if (!ret) {
+			policy->freq_table = freq_table;
+			return 0;
+		}
+	}
+
+	data->icc_dram_bw_scaling = false;
+	policy->freq_table = bpmp_lut;
+	pr_info("OPP tables missing from DT, EMC frequency scaling disabled\n");
+
 	return 0;
 }
 
@@ -406,6 +521,9 @@ static int tegra194_cpufreq_set_target(struct cpufreq_policy *policy,
 	 */
 	data->soc->ops->set_cpu_ndiv(policy, (u64)tbl->driver_data);
 
+	if (data->icc_dram_bw_scaling)
+		tegra_cpufreq_set_bw(policy, tbl->frequency);
+
 	return 0;
 }
 
@@ -439,8 +557,8 @@ static void tegra194_cpufreq_free_resources(void)
 }
 
 static struct cpufreq_frequency_table *
-init_freq_table(struct platform_device *pdev, struct tegra_bpmp *bpmp,
-		unsigned int cluster_id)
+tegra_cpufreq_bpmp_read_lut(struct platform_device *pdev, struct tegra_bpmp *bpmp,
+			    unsigned int cluster_id)
 {
 	struct cpufreq_frequency_table *freq_table;
 	struct mrq_cpu_ndiv_limits_response resp;
@@ -515,6 +633,7 @@ static int tegra194_cpufreq_probe(struct platform_device *pdev)
 	const struct tegra_cpufreq_soc *soc;
 	struct tegra194_cpufreq_data *data;
 	struct tegra_bpmp *bpmp;
+	struct device *cpu_dev;
 	int err, i;
 
 	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
@@ -530,9 +649,9 @@ static int tegra194_cpufreq_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	data->tables = devm_kcalloc(&pdev->dev, data->soc->num_clusters,
-				    sizeof(*data->tables), GFP_KERNEL);
-	if (!data->tables)
+	data->bpmp_luts = devm_kcalloc(&pdev->dev, data->soc->num_clusters,
+				       sizeof(*data->bpmp_luts), GFP_KERNEL);
+	if (!data->bpmp_luts)
 		return -ENOMEM;
 
 	if (soc->actmon_cntr_base) {
@@ -556,15 +675,26 @@ static int tegra194_cpufreq_probe(struct platform_device *pdev)
 	}
 
 	for (i = 0; i < data->soc->num_clusters; i++) {
-		data->tables[i] = init_freq_table(pdev, bpmp, i);
-		if (IS_ERR(data->tables[i])) {
-			err = PTR_ERR(data->tables[i]);
+		data->bpmp_luts[i] = tegra_cpufreq_bpmp_read_lut(pdev, bpmp, i);
+		if (IS_ERR(data->bpmp_luts[i])) {
+			err = PTR_ERR(data->bpmp_luts[i]);
 			goto err_free_res;
 		}
 	}
 
 	tegra194_cpufreq_driver.driver_data = data;
 
+	/* Check for optional OPPv2 and interconnect paths on CPU0 to enable ICC scaling */
+	cpu_dev = get_cpu_device(0);
+	if (!cpu_dev)
+		return -EPROBE_DEFER;
+
+	if (dev_pm_opp_of_get_opp_desc_node(cpu_dev)) {
+		err = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL);
+		if (!err)
+			data->icc_dram_bw_scaling = true;
+	}
+
 	err = cpufreq_register_driver(&tegra194_cpufreq_driver);
 	if (!err)
 		goto put_bpmp;
diff --git a/drivers/cpufreq/tegra20-cpufreq.c b/drivers/cpufreq/tegra20-cpufreq.c
index ab7ac7df9e62..a573186704a5 100644
--- a/drivers/cpufreq/tegra20-cpufreq.c
+++ b/drivers/cpufreq/tegra20-cpufreq.c
@@ -12,7 +12,7 @@
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/module.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm_opp.h>
 #include <linux/types.h>
@@ -25,7 +25,7 @@ static bool cpu0_node_has_opp_v2_prop(void)
 	struct device_node *np = of_cpu_device_node_get(0);
 	bool ret = false;
 
-	if (of_get_property(np, "operating-points-v2", NULL))
+	if (of_property_present(np, "operating-points-v2"))
 		ret = true;
 
 	of_node_put(np);
diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c
index 11316c3b14ca..c2d6d9c3c930 100644
--- a/drivers/cpuidle/cpuidle-psci-domain.c
+++ b/drivers/cpuidle/cpuidle-psci-domain.c
@@ -166,7 +166,7 @@ static int psci_cpuidle_domain_probe(struct platform_device *pdev)
 	 * initialize a genpd/genpd-of-provider pair when it's found.
 	 */
 	for_each_child_of_node(np, node) {
-		if (!of_find_property(node, "#power-domain-cells", NULL))
+		if (!of_property_present(node, "#power-domain-cells"))
 			continue;
 
 		ret = psci_pd_init(node, use_osi);
diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c
index 6de027f9f6f5..bf68920d038a 100644
--- a/drivers/cpuidle/cpuidle-psci.c
+++ b/drivers/cpuidle/cpuidle-psci.c
@@ -16,7 +16,6 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/psci.h>
 #include <linux/pm_domain.h>
diff --git a/drivers/cpuidle/cpuidle-qcom-spm.c b/drivers/cpuidle/cpuidle-qcom-spm.c
index c6e2e91bb4c3..1fc9968eae19 100644
--- a/drivers/cpuidle/cpuidle-qcom-spm.c
+++ b/drivers/cpuidle/cpuidle-qcom-spm.c
@@ -11,8 +11,7 @@
 #include <linux/io.h>
 #include <linux/slab.h>
 #include <linux/of.h>
-#include <linux/of_address.h>
-#include <linux/of_device.h>
+#include <linux/of_platform.h>
 #include <linux/err.h>
 #include <linux/platform_device.h>
 #include <linux/cpuidle.h>
diff --git a/drivers/cpuidle/cpuidle-riscv-sbi.c b/drivers/cpuidle/cpuidle-riscv-sbi.c
index be383f4b6855..f2ccda2c3871 100644
--- a/drivers/cpuidle/cpuidle-riscv-sbi.c
+++ b/drivers/cpuidle/cpuidle-riscv-sbi.c
@@ -8,6 +8,7 @@
 
 #define pr_fmt(fmt) "cpuidle-riscv-sbi: " fmt
 
+#include <linux/cpuhotplug.h>
 #include <linux/cpuidle.h>
 #include <linux/cpumask.h>
 #include <linux/cpu_pm.h>
@@ -15,7 +16,6 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/of_device.h>
 #include <linux/slab.h>
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
@@ -497,7 +497,7 @@ static int sbi_genpd_probe(struct device_node *np)
 	 * initialize a genpd/genpd-of-provider pair when it's found.
 	 */
 	for_each_child_of_node(np, node) {
-		if (!of_find_property(node, "#power-domain-cells", NULL))
+		if (!of_property_present(node, "#power-domain-cells"))
 			continue;
 
 		ret = sbi_pd_init(node);
@@ -548,8 +548,8 @@ static int sbi_cpuidle_probe(struct platform_device *pdev)
 	for_each_possible_cpu(cpu) {
 		np = of_cpu_device_node_get(cpu);
 		if (np &&
-		    of_find_property(np, "power-domains", NULL) &&
-		    of_find_property(np, "power-domain-names", NULL)) {
+		    of_property_present(np, "power-domains") &&
+		    of_property_present(np, "power-domain-names")) {
 			continue;
 		} else {
 			sbi_cpuidle_use_osi = false;
diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index 0b00f21cefe3..8e929f6602ce 100644
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -808,7 +808,7 @@ static int __init cpuidle_init(void)
 	if (cpuidle_disabled())
 		return -ENODEV;
 
-	return cpuidle_add_interface(cpu_subsys.dev_root);
+	return cpuidle_add_interface();
 }
 
 module_param(off, int, 0444);
diff --git a/drivers/cpuidle/cpuidle.h b/drivers/cpuidle/cpuidle.h
index 9f336af17fa6..52701d9588f1 100644
--- a/drivers/cpuidle/cpuidle.h
+++ b/drivers/cpuidle/cpuidle.h
@@ -30,7 +30,7 @@ extern int cpuidle_switch_governor(struct cpuidle_governor *gov);
 
 struct device;
 
-extern int cpuidle_add_interface(struct device *dev);
+extern int cpuidle_add_interface(void);
 extern void cpuidle_remove_interface(struct device *dev);
 extern int cpuidle_add_device_sysfs(struct cpuidle_device *device);
 extern void cpuidle_remove_device_sysfs(struct cpuidle_device *device);
diff --git a/drivers/cpuidle/dt_idle_states.c b/drivers/cpuidle/dt_idle_states.c
index 02aa0b39af9d..12fec92a85fd 100644
--- a/drivers/cpuidle/dt_idle_states.c
+++ b/drivers/cpuidle/dt_idle_states.c
@@ -14,7 +14,6 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/of_device.h>
 
 #include "dt_idle_states.h"
 
diff --git a/drivers/cpuidle/sysfs.c b/drivers/cpuidle/sysfs.c
index 48948b171749..d6f5da61cb7d 100644
--- a/drivers/cpuidle/sysfs.c
+++ b/drivers/cpuidle/sysfs.c
@@ -119,11 +119,18 @@ static struct attribute_group cpuidle_attr_group = {
 
 /**
  * cpuidle_add_interface - add CPU global sysfs attributes
- * @dev: the target device
  */
-int cpuidle_add_interface(struct device *dev)
+int cpuidle_add_interface(void)
 {
-	return sysfs_create_group(&dev->kobj, &cpuidle_attr_group);
+	struct device *dev_root = bus_get_dev_root(&cpu_subsys);
+	int retval;
+
+	if (!dev_root)
+		return -EINVAL;
+
+	retval = sysfs_create_group(&dev_root->kobj, &cpuidle_attr_group);
+	put_device(dev_root);
+	return retval;
 }
 
 /**
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 3b2516d1433f..9c440cd0fed0 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -240,21 +240,6 @@ config CRYPTO_DEV_TALITOS2
 	  Say 'Y' here to use the Freescale Security Engine (SEC)
 	  version 2 and following as found on MPC83xx, MPC85xx, etc ...
 
-config CRYPTO_DEV_IXP4XX
-	tristate "Driver for IXP4xx crypto hardware acceleration"
-	depends on ARCH_IXP4XX && IXP4XX_QMGR && IXP4XX_NPE
-	select CRYPTO_AES
-	select CRYPTO_DES
-	select CRYPTO_ECB
-	select CRYPTO_CBC
-	select CRYPTO_CTR
-	select CRYPTO_LIB_DES
-	select CRYPTO_AEAD
-	select CRYPTO_AUTHENC
-	select CRYPTO_SKCIPHER
-	help
-	  Driver for the IXP4xx NPE crypto engine.
-
 config CRYPTO_DEV_PPC4XX
 	tristate "Driver AMCC PPC4xx crypto accelerator"
 	depends on PPC && 4xx
@@ -502,10 +487,10 @@ config CRYPTO_DEV_MXS_DCP
 	  To compile this driver as a module, choose M here: the module
 	  will be called mxs-dcp.
 
-source "drivers/crypto/qat/Kconfig"
 source "drivers/crypto/cavium/cpt/Kconfig"
 source "drivers/crypto/cavium/nitrox/Kconfig"
 source "drivers/crypto/marvell/Kconfig"
+source "drivers/crypto/intel/Kconfig"
 
 config CRYPTO_DEV_CAVIUM_ZIP
 	tristate "Cavium ZIP driver"
@@ -774,7 +759,7 @@ config CRYPTO_DEV_ARTPEC6
 config CRYPTO_DEV_CCREE
 	tristate "Support for ARM TrustZone CryptoCell family of security processors"
 	depends on CRYPTO && CRYPTO_HW && OF && HAS_DMA
-	default n
+	depends on HAS_IOMEM
 	select CRYPTO_HASH
 	select CRYPTO_SKCIPHER
 	select CRYPTO_LIB_DES
@@ -810,6 +795,7 @@ config CRYPTO_DEV_SA2UL
 	select CRYPTO_AES
 	select CRYPTO_ALGAPI
 	select CRYPTO_AUTHENC
+	select CRYPTO_DES
 	select CRYPTO_SHA1
 	select CRYPTO_SHA256
 	select CRYPTO_SHA512
@@ -820,7 +806,6 @@ config CRYPTO_DEV_SA2UL
 	  used for crypto offload.  Select this if you want to use hardware
 	  acceleration for cryptographic algorithms on these devices.
 
-source "drivers/crypto/keembay/Kconfig"
 source "drivers/crypto/aspeed/Kconfig"
 
 endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 476f1a25ca32..51d36701e785 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -19,7 +19,6 @@ obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += caam/
 obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
 obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
 obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o
-obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
 obj-$(CONFIG_CRYPTO_DEV_MARVELL) += marvell/
 obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o
 obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o
@@ -33,7 +32,6 @@ obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
 obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/
-obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/
 obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
 obj-$(CONFIG_CRYPTO_DEV_QCOM_RNG) += qcom-rng.o
 obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
@@ -51,4 +49,4 @@ obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/
 obj-y += xilinx/
 obj-y += hisilicon/
 obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/
-obj-y += keembay/
+obj-y += intel/
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index 50dc783821b6..d553f3f1efbe 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -1101,7 +1101,7 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data)
 static inline irqreturn_t crypto4xx_interrupt_handler(int irq, void *data,
 						      u32 clr_val)
 {
-	struct device *dev = (struct device *)data;
+	struct device *dev = data;
 	struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);
 
 	writel(clr_val, core_dev->dev->ce_base + CRYPTO4XX_INT_CLR);
diff --git a/drivers/crypto/aspeed/aspeed-acry.c b/drivers/crypto/aspeed/aspeed-acry.c
index 1f77ebd73489..470122c87fea 100644
--- a/drivers/crypto/aspeed/aspeed-acry.c
+++ b/drivers/crypto/aspeed/aspeed-acry.c
@@ -289,7 +289,7 @@ static int aspeed_acry_rsa_ctx_copy(struct aspeed_acry_dev *acry_dev, void *buf,
 
 			if (mode == ASPEED_RSA_EXP_MODE)
 				idx = acry_dev->exp_dw_mapping[j - 1];
-			else if (mode == ASPEED_RSA_MOD_MODE)
+			else /* mode == ASPEED_RSA_MOD_MODE */
 				idx = acry_dev->mod_dw_mapping[j - 1];
 
 			dw_buf[idx] = cpu_to_le32(data);
@@ -712,7 +712,6 @@ static int aspeed_acry_probe(struct platform_device *pdev)
 {
 	struct aspeed_acry_dev *acry_dev;
 	struct device *dev = &pdev->dev;
-	struct resource *res;
 	int rc;
 
 	acry_dev = devm_kzalloc(dev, sizeof(struct aspeed_acry_dev),
@@ -724,13 +723,11 @@ static int aspeed_acry_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, acry_dev);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	acry_dev->regs = devm_ioremap_resource(dev, res);
+	acry_dev->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(acry_dev->regs))
 		return PTR_ERR(acry_dev->regs);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-	acry_dev->acry_sram = devm_ioremap_resource(dev, res);
+	acry_dev->acry_sram = devm_platform_ioremap_resource(pdev, 1);
 	if (IS_ERR(acry_dev->acry_sram))
 		return PTR_ERR(acry_dev->acry_sram);
 
@@ -782,7 +779,10 @@ static int aspeed_acry_probe(struct platform_device *pdev)
 	acry_dev->buf_addr = dmam_alloc_coherent(dev, ASPEED_ACRY_BUFF_SIZE,
 						 &acry_dev->buf_dma_addr,
 						 GFP_KERNEL);
-	memzero_explicit(acry_dev->buf_addr, ASPEED_ACRY_BUFF_SIZE);
+	if (!acry_dev->buf_addr) {
+		rc = -ENOMEM;
+		goto err_engine_rsa_start;
+	}
 
 	aspeed_acry_register(acry_dev);
 
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index ed10f2ae4523..143d33fbb316 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -493,17 +493,11 @@ static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd)
 	if (req->cryptlen < ivsize)
 		return;
 
-	if (rctx->mode & AES_FLAGS_ENCRYPT) {
+	if (rctx->mode & AES_FLAGS_ENCRYPT)
 		scatterwalk_map_and_copy(req->iv, req->dst,
 					 req->cryptlen - ivsize, ivsize, 0);
-	} else {
-		if (req->src == req->dst)
-			memcpy(req->iv, rctx->lastc, ivsize);
-		else
-			scatterwalk_map_and_copy(req->iv, req->src,
-						 req->cryptlen - ivsize,
-						 ivsize, 0);
-	}
+	else
+		memcpy(req->iv, rctx->lastc, ivsize);
 }
 
 static inline struct atmel_aes_ctr_ctx *
@@ -1146,7 +1140,7 @@ static int atmel_aes_crypt(struct skcipher_request *req, unsigned long mode)
 	rctx->mode = mode;
 
 	if (opmode != AES_FLAGS_ECB &&
-	    !(mode & AES_FLAGS_ENCRYPT) && req->src == req->dst) {
+	    !(mode & AES_FLAGS_ENCRYPT)) {
 		unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 
 		if (req->cryptlen >= ivsize)
@@ -1341,7 +1335,7 @@ static struct skcipher_alg aes_algs[] = {
 {
 	.base.cra_name		= "cfb(aes)",
 	.base.cra_driver_name	= "atmel-cfb-aes",
-	.base.cra_blocksize	= AES_BLOCK_SIZE,
+	.base.cra_blocksize	= 1,
 	.base.cra_ctxsize	= sizeof(struct atmel_aes_ctx),
 
 	.init			= atmel_aes_init_tfm,
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index e7c1db2739ec..6bef634d3c86 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -1948,14 +1948,32 @@ static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd)
 	struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);
+	struct scatterlist *sgbuf;
 	size_t hs = ctx->hash_size;
 	size_t i, num_words = hs / sizeof(u32);
 	bool use_dma = false;
 	u32 mr;
 
 	/* Special case for empty message. */
-	if (!req->nbytes)
-		return atmel_sha_complete(dd, -EINVAL); // TODO:
+	if (!req->nbytes) {
+		req->nbytes = 0;
+		ctx->bufcnt = 0;
+		ctx->digcnt[0] = 0;
+		ctx->digcnt[1] = 0;
+		switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
+		case SHA_FLAGS_SHA1:
+		case SHA_FLAGS_SHA224:
+		case SHA_FLAGS_SHA256:
+			atmel_sha_fill_padding(ctx, 64);
+			break;
+
+		case SHA_FLAGS_SHA384:
+		case SHA_FLAGS_SHA512:
+			atmel_sha_fill_padding(ctx, 128);
+			break;
+		}
+		sg_init_one(&dd->tmp, ctx->buffer, ctx->bufcnt);
+	}
 
 	/* Check DMA threshold and alignment. */
 	if (req->nbytes > ATMEL_SHA_DMA_THRESHOLD &&
@@ -1985,12 +2003,20 @@ static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd)
 
 	atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
 
+	/* Special case for empty message. */
+	if (!req->nbytes) {
+		sgbuf = &dd->tmp;
+		req->nbytes = ctx->bufcnt;
+	} else {
+		sgbuf = req->src;
+	}
+
 	/* Process data. */
 	if (use_dma)
-		return atmel_sha_dma_start(dd, req->src, req->nbytes,
+		return atmel_sha_dma_start(dd, sgbuf, req->nbytes,
 					   atmel_sha_hmac_final_done);
 
-	return atmel_sha_cpu_start(dd, req->src, req->nbytes, false, true,
+	return atmel_sha_cpu_start(dd, sgbuf, req->nbytes, false, true,
 				   atmel_sha_hmac_final_done);
 }
 
diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c
index 4403dbb0f0b1..44a185a84760 100644
--- a/drivers/crypto/atmel-sha204a.c
+++ b/drivers/crypto/atmel-sha204a.c
@@ -126,7 +126,7 @@ static void atmel_sha204a_remove(struct i2c_client *client)
 	kfree((void *)i2c_priv->hwrng.priv);
 }
 
-static const struct of_device_id atmel_sha204a_dt_ids[] = {
+static const struct of_device_id atmel_sha204a_dt_ids[] __maybe_unused = {
 	{ .compatible = "atmel,atsha204", },
 	{ .compatible = "atmel,atsha204a", },
 	{ /* sentinel */ }
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index b2d48c1649b9..c9ded8be9c39 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -565,17 +565,12 @@ atmel_tdes_set_iv_as_last_ciphertext_block(struct atmel_tdes_dev *dd)
 	if (req->cryptlen < ivsize)
 		return;
 
-	if (rctx->mode & TDES_FLAGS_ENCRYPT) {
+	if (rctx->mode & TDES_FLAGS_ENCRYPT)
 		scatterwalk_map_and_copy(req->iv, req->dst,
 					 req->cryptlen - ivsize, ivsize, 0);
-	} else {
-		if (req->src == req->dst)
-			memcpy(req->iv, rctx->lastc, ivsize);
-		else
-			scatterwalk_map_and_copy(req->iv, req->src,
-						 req->cryptlen - ivsize,
-						 ivsize, 0);
-	}
+	else
+		memcpy(req->iv, rctx->lastc, ivsize);
+
 }
 
 static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err)
@@ -722,7 +717,7 @@ static int atmel_tdes_crypt(struct skcipher_request *req, unsigned long mode)
 	rctx->mode = mode;
 
 	if ((mode & TDES_FLAGS_OPMODE_MASK) != TDES_FLAGS_ECB &&
-	    !(mode & TDES_FLAGS_ENCRYPT) && req->src == req->dst) {
+	    !(mode & TDES_FLAGS_ENCRYPT)) {
 		unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 
 		if (req->cryptlen >= ivsize)
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 12b1c8346243..feb86013dbf6 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -3,7 +3,7 @@
  * caam - Freescale FSL CAAM support for crypto API
  *
  * Copyright 2008-2011 Freescale Semiconductor, Inc.
- * Copyright 2016-2019 NXP
+ * Copyright 2016-2019, 2023 NXP
  *
  * Based on talitos crypto API driver.
  *
@@ -3542,13 +3542,14 @@ int caam_algapi_init(struct device *ctrldev)
 	 * First, detect presence and attributes of DES, AES, and MD blocks.
 	 */
 	if (priv->era < 10) {
+		struct caam_perfmon __iomem *perfmon = &priv->jr[0]->perfmon;
 		u32 cha_vid, cha_inst, aes_rn;
 
-		cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
+		cha_vid = rd_reg32(&perfmon->cha_id_ls);
 		aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
 		md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
 
-		cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
+		cha_inst = rd_reg32(&perfmon->cha_num_ls);
 		des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
 			   CHA_ID_LS_DES_SHIFT;
 		aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
@@ -3556,23 +3557,23 @@ int caam_algapi_init(struct device *ctrldev)
 		ccha_inst = 0;
 		ptha_inst = 0;
 
-		aes_rn = rd_reg32(&priv->ctrl->perfmon.cha_rev_ls) &
-			 CHA_ID_LS_AES_MASK;
+		aes_rn = rd_reg32(&perfmon->cha_rev_ls) & CHA_ID_LS_AES_MASK;
 		gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8);
 	} else {
+		struct version_regs __iomem *vreg = &priv->jr[0]->vreg;
 		u32 aesa, mdha;
 
-		aesa = rd_reg32(&priv->ctrl->vreg.aesa);
-		mdha = rd_reg32(&priv->ctrl->vreg.mdha);
+		aesa = rd_reg32(&vreg->aesa);
+		mdha = rd_reg32(&vreg->mdha);
 
 		aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
 		md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
 
-		des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
+		des_inst = rd_reg32(&vreg->desa) & CHA_VER_NUM_MASK;
 		aes_inst = aesa & CHA_VER_NUM_MASK;
 		md_inst = mdha & CHA_VER_NUM_MASK;
-		ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK;
-		ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK;
+		ccha_inst = rd_reg32(&vreg->ccha) & CHA_VER_NUM_MASK;
+		ptha_inst = rd_reg32(&vreg->ptha) & CHA_VER_NUM_MASK;
 
 		gcm_support = aesa & CHA_VER_MISC_AES_GCM;
 	}
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 82d3c730a502..80deb003f0a5 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -3,7 +3,7 @@
  * caam - Freescale FSL CAAM support for ahash functions of crypto API
  *
  * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2018-2019 NXP
+ * Copyright 2018-2019, 2023 NXP
  *
  * Based on caamalg.c crypto API driver.
  *
@@ -1956,12 +1956,14 @@ int caam_algapi_hash_init(struct device *ctrldev)
 	 * presence and attributes of MD block.
 	 */
 	if (priv->era < 10) {
-		md_vid = (rd_reg32(&priv->ctrl->perfmon.cha_id_ls) &
+		struct caam_perfmon __iomem *perfmon = &priv->jr[0]->perfmon;
+
+		md_vid = (rd_reg32(&perfmon->cha_id_ls) &
 			  CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
-		md_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
+		md_inst = (rd_reg32(&perfmon->cha_num_ls) &
 			   CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
 	} else {
-		u32 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
+		u32 mdha = rd_reg32(&priv->jr[0]->vreg.mdha);
 
 		md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
 		md_inst = mdha & CHA_VER_NUM_MASK;
diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c
index e40614fef39d..72afc249d42f 100644
--- a/drivers/crypto/caam/caampkc.c
+++ b/drivers/crypto/caam/caampkc.c
@@ -3,7 +3,7 @@
  * caam - Freescale FSL CAAM support for Public Key Cryptography
  *
  * Copyright 2016 Freescale Semiconductor, Inc.
- * Copyright 2018-2019 NXP
+ * Copyright 2018-2019, 2023 NXP
  *
  * There is no Shared Descriptor for PKC so that the Job Descriptor must carry
  * all the desired key parameters, input and output pointers.
@@ -1168,10 +1168,10 @@ int caam_pkc_init(struct device *ctrldev)
 
 	/* Determine public key hardware accelerator presence. */
 	if (priv->era < 10) {
-		pk_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
+		pk_inst = (rd_reg32(&priv->jr[0]->perfmon.cha_num_ls) &
 			   CHA_ID_LS_PK_MASK) >> CHA_ID_LS_PK_SHIFT;
 	} else {
-		pkha = rd_reg32(&priv->ctrl->vreg.pkha);
+		pkha = rd_reg32(&priv->jr[0]->vreg.pkha);
 		pk_inst = pkha & CHA_VER_NUM_MASK;
 
 		/*
diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c
index 1fd8ff965006..50eb55da45c2 100644
--- a/drivers/crypto/caam/caamrng.c
+++ b/drivers/crypto/caam/caamrng.c
@@ -3,7 +3,7 @@
  * caam - Freescale FSL CAAM support for hw_random
  *
  * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2018-2019 NXP
+ * Copyright 2018-2019, 2023 NXP
  *
  * Based on caamalg.c crypto API driver.
  *
@@ -227,10 +227,10 @@ int caam_rng_init(struct device *ctrldev)
 
 	/* Check for an instantiated RNG before registration */
 	if (priv->era < 10)
-		rng_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
+		rng_inst = (rd_reg32(&priv->jr[0]->perfmon.cha_num_ls) &
 			    CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT;
 	else
-		rng_inst = rd_reg32(&priv->ctrl->vreg.rng) & CHA_VER_NUM_MASK;
+		rng_inst = rd_reg32(&priv->jr[0]->vreg.rng) & CHA_VER_NUM_MASK;
 
 	if (!rng_inst)
 		return 0;
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 6278afb951c3..bedcc2ab3a00 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -3,7 +3,7 @@
  * Controller-level driver, kernel property detection, initialization
  *
  * Copyright 2008-2012 Freescale Semiconductor, Inc.
- * Copyright 2018-2019 NXP
+ * Copyright 2018-2019, 2023 NXP
  */
 
 #include <linux/device.h>
@@ -284,6 +284,10 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask,
 		const u32 rdsta_if = RDSTA_IF0 << sh_idx;
 		const u32 rdsta_pr = RDSTA_PR0 << sh_idx;
 		const u32 rdsta_mask = rdsta_if | rdsta_pr;
+
+		/* Clear the contents before using the descriptor */
+		memset(desc, 0x00, CAAM_CMD_SZ * 7);
+
 		/*
 		 * If the corresponding bit is set, this state handle
 		 * was initialized by somebody else, so it's left alone.
@@ -327,8 +331,6 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask,
 		}
 
 		dev_info(ctrldev, "Instantiated RNG4 SH%d\n", sh_idx);
-		/* Clear the contents before recreating the descriptor */
-		memset(desc, 0x00, CAAM_CMD_SZ * 7);
 	}
 
 	kfree(desc);
@@ -395,7 +397,7 @@ start_rng:
 		      RTMCTL_SAMP_MODE_RAW_ES_SC);
 }
 
-static int caam_get_era_from_hw(struct caam_ctrl __iomem *ctrl)
+static int caam_get_era_from_hw(struct caam_perfmon __iomem *perfmon)
 {
 	static const struct {
 		u16 ip_id;
@@ -421,12 +423,12 @@ static int caam_get_era_from_hw(struct caam_ctrl __iomem *ctrl)
 	u16 ip_id;
 	int i;
 
-	ccbvid = rd_reg32(&ctrl->perfmon.ccb_id);
+	ccbvid = rd_reg32(&perfmon->ccb_id);
 	era = (ccbvid & CCBVID_ERA_MASK) >> CCBVID_ERA_SHIFT;
 	if (era)	/* This is '0' prior to CAAM ERA-6 */
 		return era;
 
-	id_ms = rd_reg32(&ctrl->perfmon.caam_id_ms);
+	id_ms = rd_reg32(&perfmon->caam_id_ms);
 	ip_id = (id_ms & SECVID_MS_IPID_MASK) >> SECVID_MS_IPID_SHIFT;
 	maj_rev = (id_ms & SECVID_MS_MAJ_REV_MASK) >> SECVID_MS_MAJ_REV_SHIFT;
 
@@ -444,9 +446,9 @@ static int caam_get_era_from_hw(struct caam_ctrl __iomem *ctrl)
  * In case this property is not passed an attempt to retrieve the CAAM
  * era via register reads will be made.
  *
- * @ctrl:	controller region
+ * @perfmon:	Performance Monitor Registers
  */
-static int caam_get_era(struct caam_ctrl __iomem *ctrl)
+static int caam_get_era(struct caam_perfmon __iomem *perfmon)
 {
 	struct device_node *caam_node;
 	int ret;
@@ -459,7 +461,7 @@ static int caam_get_era(struct caam_ctrl __iomem *ctrl)
 	if (!ret)
 		return prop;
 	else
-		return caam_get_era_from_hw(ctrl);
+		return caam_get_era_from_hw(perfmon);
 }
 
 /*
@@ -626,12 +628,14 @@ static int caam_probe(struct platform_device *pdev)
 	struct device_node *nprop, *np;
 	struct caam_ctrl __iomem *ctrl;
 	struct caam_drv_private *ctrlpriv;
+	struct caam_perfmon __iomem *perfmon;
 	struct dentry *dfs_root;
 	u32 scfgr, comp_params;
 	u8 rng_vid;
 	int pg_size;
 	int BLOCK_OFFSET = 0;
 	bool pr_support = false;
+	bool reg_access = true;
 
 	ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(*ctrlpriv), GFP_KERNEL);
 	if (!ctrlpriv)
@@ -645,6 +649,17 @@ static int caam_probe(struct platform_device *pdev)
 	caam_imx = (bool)imx_soc_match;
 
 	if (imx_soc_match) {
+		/*
+		 * Until Layerscape and i.MX OP-TEE get in sync,
+		 * only i.MX OP-TEE use cases disallow access to
+		 * caam page 0 (controller) registers.
+		 */
+		np = of_find_compatible_node(NULL, NULL, "linaro,optee-tz");
+		ctrlpriv->optee_en = !!np;
+		of_node_put(np);
+
+		reg_access = !ctrlpriv->optee_en;
+
 		if (!imx_soc_match->data) {
 			dev_err(dev, "No clock data provided for i.MX SoC");
 			return -EINVAL;
@@ -665,10 +680,38 @@ static int caam_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	caam_little_end = !(bool)(rd_reg32(&ctrl->perfmon.status) &
+	ring = 0;
+	for_each_available_child_of_node(nprop, np)
+		if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
+		    of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
+			u32 reg;
+
+			if (of_property_read_u32_index(np, "reg", 0, &reg)) {
+				dev_err(dev, "%s read reg property error\n",
+					np->full_name);
+				continue;
+			}
+
+			ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
+					     ((__force uint8_t *)ctrl + reg);
+
+			ctrlpriv->total_jobrs++;
+			ring++;
+		}
+
+	/*
+	 * Wherever possible, instead of accessing registers from the global page,
+	 * use the alias registers in the first (cf. DT nodes order)
+	 * job ring's page.
+	 */
+	perfmon = ring ? (struct caam_perfmon __iomem *)&ctrlpriv->jr[0]->perfmon :
+			 (struct caam_perfmon __iomem *)&ctrl->perfmon;
+
+	caam_little_end = !(bool)(rd_reg32(&perfmon->status) &
 				  (CSTA_PLEND | CSTA_ALT_PLEND));
-	comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms);
-	if (comp_params & CTPR_MS_PS && rd_reg32(&ctrl->mcr) & MCFGR_LONG_PTR)
+	comp_params = rd_reg32(&perfmon->comp_parms_ms);
+	if (reg_access && comp_params & CTPR_MS_PS &&
+	    rd_reg32(&ctrl->mcr) & MCFGR_LONG_PTR)
 		caam_ptr_sz = sizeof(u64);
 	else
 		caam_ptr_sz = sizeof(u32);
@@ -733,6 +776,9 @@ static int caam_probe(struct platform_device *pdev)
 	}
 #endif
 
+	if (!reg_access)
+		goto set_dma_mask;
+
 	/*
 	 * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel,
 	 * long pointers in master configuration register.
@@ -772,13 +818,14 @@ static int caam_probe(struct platform_device *pdev)
 			      JRSTART_JR1_START | JRSTART_JR2_START |
 			      JRSTART_JR3_START);
 
+set_dma_mask:
 	ret = dma_set_mask_and_coherent(dev, caam_get_dma_mask(dev));
 	if (ret) {
 		dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret);
 		return ret;
 	}
 
-	ctrlpriv->era = caam_get_era(ctrl);
+	ctrlpriv->era = caam_get_era(perfmon);
 	ctrlpriv->domain = iommu_get_domain_for_dev(dev);
 
 	dfs_root = debugfs_create_dir(dev_name(dev), NULL);
@@ -789,7 +836,7 @@ static int caam_probe(struct platform_device *pdev)
 			return ret;
 	}
 
-	caam_debugfs_init(ctrlpriv, dfs_root);
+	caam_debugfs_init(ctrlpriv, perfmon, dfs_root);
 
 	/* Check to see if (DPAA 1.x) QI present. If so, enable */
 	if (ctrlpriv->qi_present && !caam_dpaa2) {
@@ -808,26 +855,16 @@ static int caam_probe(struct platform_device *pdev)
 #endif
 	}
 
-	ring = 0;
-	for_each_available_child_of_node(nprop, np)
-		if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
-		    of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
-			ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
-					     ((__force uint8_t *)ctrl +
-					     (ring + JR_BLOCK_NUMBER) *
-					      BLOCK_OFFSET
-					     );
-			ctrlpriv->total_jobrs++;
-			ring++;
-		}
-
 	/* If no QI and no rings specified, quit and go home */
 	if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) {
 		dev_err(dev, "no queues configured, terminating\n");
 		return -ENOMEM;
 	}
 
-	comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ls);
+	if (!reg_access)
+		goto report_live;
+
+	comp_params = rd_reg32(&perfmon->comp_parms_ls);
 	ctrlpriv->blob_present = !!(comp_params & CTPR_LS_BLOB);
 
 	/*
@@ -836,15 +873,21 @@ static int caam_probe(struct platform_device *pdev)
 	 * check both here.
 	 */
 	if (ctrlpriv->era < 10) {
-		rng_vid = (rd_reg32(&ctrl->perfmon.cha_id_ls) &
+		rng_vid = (rd_reg32(&perfmon->cha_id_ls) &
 			   CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT;
 		ctrlpriv->blob_present = ctrlpriv->blob_present &&
-			(rd_reg32(&ctrl->perfmon.cha_num_ls) & CHA_ID_LS_AES_MASK);
+			(rd_reg32(&perfmon->cha_num_ls) & CHA_ID_LS_AES_MASK);
 	} else {
-		rng_vid = (rd_reg32(&ctrl->vreg.rng) & CHA_VER_VID_MASK) >>
+		struct version_regs __iomem *vreg;
+
+		vreg =  ctrlpriv->total_jobrs ?
+			(struct version_regs __iomem *)&ctrlpriv->jr[0]->vreg :
+			(struct version_regs __iomem *)&ctrl->vreg;
+
+		rng_vid = (rd_reg32(&vreg->rng) & CHA_VER_VID_MASK) >>
 			   CHA_VER_VID_SHIFT;
 		ctrlpriv->blob_present = ctrlpriv->blob_present &&
-			(rd_reg32(&ctrl->vreg.aesa) & CHA_VER_MISC_AES_NUM_MASK);
+			(rd_reg32(&vreg->aesa) & CHA_VER_MISC_AES_NUM_MASK);
 	}
 
 	/*
@@ -923,10 +966,11 @@ static int caam_probe(struct platform_device *pdev)
 		clrsetbits_32(&ctrl->scfgr, 0, SCFGR_RDBENABLE);
 	}
 
+report_live:
 	/* NOTE: RTIC detection ought to go here, around Si time */
 
-	caam_id = (u64)rd_reg32(&ctrl->perfmon.caam_id_ms) << 32 |
-		  (u64)rd_reg32(&ctrl->perfmon.caam_id_ls);
+	caam_id = (u64)rd_reg32(&perfmon->caam_id_ms) << 32 |
+		  (u64)rd_reg32(&perfmon->caam_id_ls);
 
 	/* Report "alive" for developer to see */
 	dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id,
diff --git a/drivers/crypto/caam/debugfs.c b/drivers/crypto/caam/debugfs.c
index 806bb20d2aa1..6358d3cabf57 100644
--- a/drivers/crypto/caam/debugfs.c
+++ b/drivers/crypto/caam/debugfs.c
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
-/* Copyright 2019 NXP */
+/* Copyright 2019, 2023 NXP */
 
 #include <linux/debugfs.h>
 #include "compat.h"
@@ -42,16 +42,15 @@ void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv)
 }
 #endif
 
-void caam_debugfs_init(struct caam_drv_private *ctrlpriv, struct dentry *root)
+void caam_debugfs_init(struct caam_drv_private *ctrlpriv,
+		       struct caam_perfmon __force *perfmon,
+		       struct dentry *root)
 {
-	struct caam_perfmon *perfmon;
-
 	/*
 	 * FIXME: needs better naming distinction, as some amalgamation of
 	 * "caam" and nprop->full_name. The OF name isn't distinctive,
 	 * but does separate instances
 	 */
-	perfmon = (struct caam_perfmon __force *)&ctrlpriv->ctrl->perfmon;
 
 	ctrlpriv->ctl = debugfs_create_dir("ctl", root);
 
@@ -78,6 +77,9 @@ void caam_debugfs_init(struct caam_drv_private *ctrlpriv, struct dentry *root)
 	debugfs_create_file("fault_status", 0444, ctrlpriv->ctl,
 			    &perfmon->status, &caam_fops_u32_ro);
 
+	if (ctrlpriv->optee_en)
+		return;
+
 	/* Internal covering keys (useful in non-secure mode only) */
 	ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0];
 	ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
diff --git a/drivers/crypto/caam/debugfs.h b/drivers/crypto/caam/debugfs.h
index 661d768acdbf..8b5d1acd21a7 100644
--- a/drivers/crypto/caam/debugfs.h
+++ b/drivers/crypto/caam/debugfs.h
@@ -1,16 +1,19 @@
 /* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
-/* Copyright 2019 NXP */
+/* Copyright 2019, 2023 NXP */
 
 #ifndef CAAM_DEBUGFS_H
 #define CAAM_DEBUGFS_H
 
 struct dentry;
 struct caam_drv_private;
+struct caam_perfmon;
 
 #ifdef CONFIG_DEBUG_FS
-void caam_debugfs_init(struct caam_drv_private *ctrlpriv, struct dentry *root);
+void caam_debugfs_init(struct caam_drv_private *ctrlpriv,
+		       struct caam_perfmon __force *perfmon, struct dentry *root);
 #else
 static inline void caam_debugfs_init(struct caam_drv_private *ctrlpriv,
+				     struct caam_perfmon __force *perfmon,
 				     struct dentry *root)
 {}
 #endif
diff --git a/drivers/crypto/caam/dpseci-debugfs.c b/drivers/crypto/caam/dpseci-debugfs.c
index 0eca8c2fd916..020a9d8a8a07 100644
--- a/drivers/crypto/caam/dpseci-debugfs.c
+++ b/drivers/crypto/caam/dpseci-debugfs.c
@@ -8,7 +8,7 @@
 
 static int dpseci_dbg_fqs_show(struct seq_file *file, void *offset)
 {
-	struct dpaa2_caam_priv *priv = (struct dpaa2_caam_priv *)file->private;
+	struct dpaa2_caam_priv *priv = file->private;
 	u32 fqid, fcnt, bcnt;
 	int i, err;
 
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index 572cf66c887a..86ed1b91c22d 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -94,6 +94,7 @@ struct caam_drv_private {
 	u8 qi_present;		/* Nonzero if QI present in device */
 	u8 blob_present;	/* Nonzero if BLOB support present in device */
 	u8 mc_en;		/* Nonzero if MC f/w is active */
+	u8 optee_en;		/* Nonzero if OP-TEE f/w is active */
 	int secvio_irq;		/* Security violation interrupt number */
 	int virt_en;		/* Virtualization enabled in CAAM */
 	int era;		/* CAAM Era (internal HW revision) */
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index 724fdec18bf9..96dea5304d22 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -4,7 +4,7 @@
  * JobR backend functionality
  *
  * Copyright 2008-2012 Freescale Semiconductor, Inc.
- * Copyright 2019 NXP
+ * Copyright 2019, 2023 NXP
  */
 
 #include <linux/of_irq.h>
@@ -72,19 +72,27 @@ static void caam_jr_crypto_engine_exit(void *data)
 	crypto_engine_exit(jrpriv->engine);
 }
 
-static int caam_reset_hw_jr(struct device *dev)
+/*
+ * Put the CAAM in quiesce, ie stop
+ *
+ * Must be called with itr disabled
+ */
+static int caam_jr_stop_processing(struct device *dev, u32 jrcr_bits)
 {
 	struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 	unsigned int timeout = 100000;
 
-	/*
-	 * mask interrupts since we are going to poll
-	 * for reset completion status
-	 */
-	clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK);
+	/* Check the current status */
+	if (rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_INPROGRESS)
+		goto wait_quiesce_completion;
 
-	/* initiate flush (required prior to reset) */
-	wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
+	/* Reset the field */
+	clrsetbits_32(&jrp->rregs->jrintstatus, JRINT_ERR_HALT_MASK, 0);
+
+	/* initiate flush / park (required prior to reset) */
+	wr_reg32(&jrp->rregs->jrcommand, jrcr_bits);
+
+wait_quiesce_completion:
 	while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
 		JRINT_ERR_HALT_INPROGRESS) && --timeout)
 		cpu_relax();
@@ -95,8 +103,35 @@ static int caam_reset_hw_jr(struct device *dev)
 		return -EIO;
 	}
 
+	return 0;
+}
+
+/*
+ * Flush the job ring, so the jobs running will be stopped, jobs queued will be
+ * invalidated and the CAAM will no longer fetch fron input ring.
+ *
+ * Must be called with itr disabled
+ */
+static int caam_jr_flush(struct device *dev)
+{
+	return caam_jr_stop_processing(dev, JRCR_RESET);
+}
+
+static int caam_reset_hw_jr(struct device *dev)
+{
+	struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
+	unsigned int timeout = 100000;
+	int err;
+	/*
+	 * mask interrupts since we are going to poll
+	 * for reset completion status
+	 */
+	clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK);
+	err = caam_jr_flush(dev);
+	if (err)
+		return err;
+
 	/* initiate reset */
-	timeout = 100000;
 	wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
 	while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
 		cpu_relax();
@@ -163,6 +198,11 @@ static int caam_jr_remove(struct platform_device *pdev)
 	return ret;
 }
 
+static void caam_jr_platform_shutdown(struct platform_device *pdev)
+{
+	caam_jr_remove(pdev);
+}
+
 /* Main per-ring interrupt handler */
 static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
 {
@@ -618,6 +658,7 @@ static struct platform_driver caam_jr_driver = {
 	},
 	.probe       = caam_jr_probe,
 	.remove      = caam_jr_remove,
+	.shutdown    = caam_jr_platform_shutdown,
 };
 
 static int __init jr_driver_init(void)
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c
index 432a61aca0c5..65114f766e7d 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_main.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_main.c
@@ -1,5 +1,4 @@
 // SPDX-License-Identifier: GPL-2.0-only
-#include <linux/aer.h>
 #include <linux/delay.h>
 #include <linux/firmware.h>
 #include <linux/list.h>
diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile
index db362fe472ea..f6196495e862 100644
--- a/drivers/crypto/ccp/Makefile
+++ b/drivers/crypto/ccp/Makefile
@@ -10,7 +10,8 @@ ccp-$(CONFIG_CRYPTO_DEV_CCP_DEBUGFS) += ccp-debugfs.o
 ccp-$(CONFIG_PCI) += sp-pci.o
 ccp-$(CONFIG_CRYPTO_DEV_SP_PSP) += psp-dev.o \
                                    sev-dev.o \
-                                   tee-dev.o
+                                   tee-dev.o \
+                                   platform-access.o
 
 obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o
 ccp-crypto-objs := ccp-crypto-main.o \
diff --git a/drivers/crypto/ccp/platform-access.c b/drivers/crypto/ccp/platform-access.c
new file mode 100644
index 000000000000..939c924fc383
--- /dev/null
+++ b/drivers/crypto/ccp/platform-access.c
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD Platform Security Processor (PSP) Platform Access interface
+ *
+ * Copyright (C) 2023 Advanced Micro Devices, Inc.
+ *
+ * Author: Mario Limonciello <mario.limonciello@amd.com>
+ *
+ * Some of this code is adapted from drivers/i2c/busses/i2c-designware-amdpsp.c
+ * developed by Jan Dabros <jsd@semihalf.com> and Copyright (C) 2022 Google Inc.
+ *
+ */
+
+#include <linux/bitfield.h>
+#include <linux/errno.h>
+#include <linux/iopoll.h>
+#include <linux/mutex.h>
+
+#include "platform-access.h"
+
+#define PSP_CMD_TIMEOUT_US	(500 * USEC_PER_MSEC)
+#define DOORBELL_CMDRESP_STS	GENMASK(7, 0)
+
+/* Recovery field should be equal 0 to start sending commands */
+static int check_recovery(u32 __iomem *cmd)
+{
+	return FIELD_GET(PSP_CMDRESP_RECOVERY, ioread32(cmd));
+}
+
+static int wait_cmd(u32 __iomem *cmd)
+{
+	u32 tmp, expected;
+
+	/* Expect mbox_cmd to be cleared and ready bit to be set by PSP */
+	expected = FIELD_PREP(PSP_CMDRESP_RESP, 1);
+
+	/*
+	 * Check for readiness of PSP mailbox in a tight loop in order to
+	 * process further as soon as command was consumed.
+	 */
+	return readl_poll_timeout(cmd, tmp, (tmp & expected), 0,
+				  PSP_CMD_TIMEOUT_US);
+}
+
+int psp_check_platform_access_status(void)
+{
+	struct psp_device *psp = psp_get_master_device();
+
+	if (!psp || !psp->platform_access_data)
+		return -ENODEV;
+
+	return 0;
+}
+EXPORT_SYMBOL(psp_check_platform_access_status);
+
+int psp_send_platform_access_msg(enum psp_platform_access_msg msg,
+				 struct psp_request *req)
+{
+	struct psp_device *psp = psp_get_master_device();
+	u32 __iomem *cmd, *lo, *hi;
+	struct psp_platform_access_device *pa_dev;
+	phys_addr_t req_addr;
+	u32 cmd_reg;
+	int ret;
+
+	if (!psp || !psp->platform_access_data)
+		return -ENODEV;
+
+	pa_dev = psp->platform_access_data;
+	cmd = psp->io_regs + pa_dev->vdata->cmdresp_reg;
+	lo = psp->io_regs + pa_dev->vdata->cmdbuff_addr_lo_reg;
+	hi = psp->io_regs + pa_dev->vdata->cmdbuff_addr_hi_reg;
+
+	mutex_lock(&pa_dev->mailbox_mutex);
+
+	if (check_recovery(cmd)) {
+		dev_dbg(psp->dev, "platform mailbox is in recovery\n");
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	if (wait_cmd(cmd)) {
+		dev_dbg(psp->dev, "platform mailbox is not done processing command\n");
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	/*
+	 * Fill mailbox with address of command-response buffer, which will be
+	 * used for sending i2c requests as well as reading status returned by
+	 * PSP. Use physical address of buffer, since PSP will map this region.
+	 */
+	req_addr = __psp_pa(req);
+	iowrite32(lower_32_bits(req_addr), lo);
+	iowrite32(upper_32_bits(req_addr), hi);
+
+	print_hex_dump_debug("->psp ", DUMP_PREFIX_OFFSET, 16, 2, req,
+			     req->header.payload_size, false);
+
+	/* Write command register to trigger processing */
+	cmd_reg = FIELD_PREP(PSP_CMDRESP_CMD, msg);
+	iowrite32(cmd_reg, cmd);
+
+	if (wait_cmd(cmd)) {
+		ret = -ETIMEDOUT;
+		goto unlock;
+	}
+
+	/* Ensure it was triggered by this driver */
+	if (ioread32(lo) != lower_32_bits(req_addr) ||
+	    ioread32(hi) != upper_32_bits(req_addr)) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	/* Store the status in request header for caller to investigate */
+	cmd_reg = ioread32(cmd);
+	req->header.status = FIELD_GET(PSP_CMDRESP_STS, cmd_reg);
+	if (req->header.status) {
+		ret = -EIO;
+		goto unlock;
+	}
+
+	print_hex_dump_debug("<-psp ", DUMP_PREFIX_OFFSET, 16, 2, req,
+			     req->header.payload_size, false);
+
+	ret = 0;
+
+unlock:
+	mutex_unlock(&pa_dev->mailbox_mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(psp_send_platform_access_msg);
+
+int psp_ring_platform_doorbell(int msg, u32 *result)
+{
+	struct psp_device *psp = psp_get_master_device();
+	struct psp_platform_access_device *pa_dev;
+	u32 __iomem *button, *cmd;
+	int ret, val;
+
+	if (!psp || !psp->platform_access_data)
+		return -ENODEV;
+
+	pa_dev = psp->platform_access_data;
+	button = psp->io_regs + pa_dev->vdata->doorbell_button_reg;
+	cmd = psp->io_regs + pa_dev->vdata->doorbell_cmd_reg;
+
+	mutex_lock(&pa_dev->doorbell_mutex);
+
+	if (wait_cmd(cmd)) {
+		dev_err(psp->dev, "doorbell command not done processing\n");
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	iowrite32(FIELD_PREP(DOORBELL_CMDRESP_STS, msg), cmd);
+	iowrite32(PSP_DRBL_RING, button);
+
+	if (wait_cmd(cmd)) {
+		ret = -ETIMEDOUT;
+		goto unlock;
+	}
+
+	val = FIELD_GET(DOORBELL_CMDRESP_STS, ioread32(cmd));
+	if (val) {
+		if (result)
+			*result = val;
+		ret = -EIO;
+		goto unlock;
+	}
+
+	ret = 0;
+unlock:
+	mutex_unlock(&pa_dev->doorbell_mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(psp_ring_platform_doorbell);
+
+void platform_access_dev_destroy(struct psp_device *psp)
+{
+	struct psp_platform_access_device *pa_dev = psp->platform_access_data;
+
+	if (!pa_dev)
+		return;
+
+	mutex_destroy(&pa_dev->mailbox_mutex);
+	mutex_destroy(&pa_dev->doorbell_mutex);
+	psp->platform_access_data = NULL;
+}
+
+int platform_access_dev_init(struct psp_device *psp)
+{
+	struct device *dev = psp->dev;
+	struct psp_platform_access_device *pa_dev;
+
+	pa_dev = devm_kzalloc(dev, sizeof(*pa_dev), GFP_KERNEL);
+	if (!pa_dev)
+		return -ENOMEM;
+
+	psp->platform_access_data = pa_dev;
+	pa_dev->psp = psp;
+	pa_dev->dev = dev;
+
+	pa_dev->vdata = (struct platform_access_vdata *)psp->vdata->platform_access;
+
+	mutex_init(&pa_dev->mailbox_mutex);
+	mutex_init(&pa_dev->doorbell_mutex);
+
+	dev_dbg(dev, "platform access enabled\n");
+
+	return 0;
+}
diff --git a/drivers/crypto/ccp/platform-access.h b/drivers/crypto/ccp/platform-access.h
new file mode 100644
index 000000000000..a83f03beb869
--- /dev/null
+++ b/drivers/crypto/ccp/platform-access.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * AMD Platform Security Processor (PSP) Platform Access interface
+ *
+ * Copyright (C) 2023 Advanced Micro Devices, Inc.
+ *
+ * Author: Mario Limonciello <mario.limonciello@amd.com>
+ */
+
+#ifndef __PSP_PLATFORM_ACCESS_H__
+#define __PSP_PLATFORM_ACCESS_H__
+
+#include <linux/device.h>
+#include <linux/miscdevice.h>
+#include <linux/mutex.h>
+#include <linux/psp-platform-access.h>
+
+#include "psp-dev.h"
+
+struct psp_platform_access_device {
+	struct device *dev;
+	struct psp_device *psp;
+
+	struct platform_access_vdata *vdata;
+
+	struct mutex mailbox_mutex;
+	struct mutex doorbell_mutex;
+
+	void *platform_access_data;
+};
+
+void platform_access_dev_destroy(struct psp_device *psp);
+int platform_access_dev_init(struct psp_device *psp);
+
+#endif /* __PSP_PLATFORM_ACCESS_H__ */
diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c
index c9c741ac8442..e3d6955d3265 100644
--- a/drivers/crypto/ccp/psp-dev.c
+++ b/drivers/crypto/ccp/psp-dev.c
@@ -14,6 +14,7 @@
 #include "psp-dev.h"
 #include "sev-dev.h"
 #include "tee-dev.h"
+#include "platform-access.h"
 
 struct psp_device *psp_master;
 
@@ -42,18 +43,15 @@ static irqreturn_t psp_irq_handler(int irq, void *data)
 	/* Read the interrupt status: */
 	status = ioread32(psp->io_regs + psp->vdata->intsts_reg);
 
+	/* Clear the interrupt status by writing the same value we read. */
+	iowrite32(status, psp->io_regs + psp->vdata->intsts_reg);
+
 	/* invoke subdevice interrupt handlers */
 	if (status) {
 		if (psp->sev_irq_handler)
 			psp->sev_irq_handler(irq, psp->sev_irq_data, status);
-
-		if (psp->tee_irq_handler)
-			psp->tee_irq_handler(irq, psp->tee_irq_data, status);
 	}
 
-	/* Clear the interrupt status by writing the same value we read. */
-	iowrite32(status, psp->io_regs + psp->vdata->intsts_reg);
-
 	return IRQ_HANDLED;
 }
 
@@ -105,6 +103,17 @@ static int psp_check_tee_support(struct psp_device *psp)
 	return 0;
 }
 
+static void psp_init_platform_access(struct psp_device *psp)
+{
+	int ret;
+
+	ret = platform_access_dev_init(psp);
+	if (ret) {
+		dev_warn(psp->dev, "platform access init failed: %d\n", ret);
+		return;
+	}
+}
+
 static int psp_init(struct psp_device *psp)
 {
 	int ret;
@@ -121,6 +130,9 @@ static int psp_init(struct psp_device *psp)
 			return ret;
 	}
 
+	if (psp->vdata->platform_access)
+		psp_init_platform_access(psp);
+
 	return 0;
 }
 
@@ -201,6 +213,8 @@ void psp_dev_destroy(struct sp_device *sp)
 
 	tee_dev_destroy(psp);
 
+	platform_access_dev_destroy(psp);
+
 	sp_free_psp_irq(sp, psp);
 
 	if (sp->clear_psp_master_device)
@@ -219,18 +233,6 @@ void psp_clear_sev_irq_handler(struct psp_device *psp)
 	psp_set_sev_irq_handler(psp, NULL, NULL);
 }
 
-void psp_set_tee_irq_handler(struct psp_device *psp, psp_irq_handler_t handler,
-			     void *data)
-{
-	psp->tee_irq_data = data;
-	psp->tee_irq_handler = handler;
-}
-
-void psp_clear_tee_irq_handler(struct psp_device *psp)
-{
-	psp_set_tee_irq_handler(psp, NULL, NULL);
-}
-
 struct psp_device *psp_get_master_device(void)
 {
 	struct sp_device *sp = sp_get_psp_master_device();
diff --git a/drivers/crypto/ccp/psp-dev.h b/drivers/crypto/ccp/psp-dev.h
index d528eb04c3ef..505e4bdeaca8 100644
--- a/drivers/crypto/ccp/psp-dev.h
+++ b/drivers/crypto/ccp/psp-dev.h
@@ -17,9 +17,6 @@
 
 #include "sp-dev.h"
 
-#define PSP_CMDRESP_RESP		BIT(31)
-#define PSP_CMDRESP_ERR_MASK		0xffff
-
 #define MAX_PSP_NAME_LEN		16
 
 extern struct psp_device *psp_master;
@@ -40,11 +37,9 @@ struct psp_device {
 	psp_irq_handler_t sev_irq_handler;
 	void *sev_irq_data;
 
-	psp_irq_handler_t tee_irq_handler;
-	void *tee_irq_data;
-
 	void *sev_data;
 	void *tee_data;
+	void *platform_access_data;
 
 	unsigned int capability;
 };
@@ -53,10 +48,6 @@ void psp_set_sev_irq_handler(struct psp_device *psp, psp_irq_handler_t handler,
 			     void *data);
 void psp_clear_sev_irq_handler(struct psp_device *psp);
 
-void psp_set_tee_irq_handler(struct psp_device *psp, psp_irq_handler_t handler,
-			     void *data);
-void psp_clear_tee_irq_handler(struct psp_device *psp);
-
 struct psp_device *psp_get_master_device(void);
 
 #define PSP_CAPABILITY_SEV			BIT(0)
diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
index e346c00b132a..b1f3327f65e0 100644
--- a/drivers/crypto/ccp/sev-dev.c
+++ b/drivers/crypto/ccp/sev-dev.c
@@ -7,6 +7,7 @@
  * Author: Brijesh Singh <brijesh.singh@amd.com>
  */
 
+#include <linux/bitfield.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/kthread.h>
@@ -24,6 +25,7 @@
 #include <linux/cpufeature.h>
 #include <linux/fs.h>
 #include <linux/fs_struct.h>
+#include <linux/psp.h>
 
 #include <asm/smp.h>
 #include <asm/cacheflush.h>
@@ -102,7 +104,7 @@ static void sev_irq_handler(int irq, void *data, unsigned int status)
 
 	/* Check if it is SEV command completion: */
 	reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
-	if (reg & PSP_CMDRESP_RESP) {
+	if (FIELD_GET(PSP_CMDRESP_RESP, reg)) {
 		sev->int_rcvd = 1;
 		wake_up(&sev->int_queue);
 	}
@@ -346,9 +348,7 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
 
 	sev->int_rcvd = 0;
 
-	reg = cmd;
-	reg <<= SEV_CMDRESP_CMD_SHIFT;
-	reg |= SEV_CMDRESP_IOC;
+	reg = FIELD_PREP(SEV_CMDRESP_CMD, cmd) | SEV_CMDRESP_IOC;
 	iowrite32(reg, sev->io_regs + sev->vdata->cmdresp_reg);
 
 	/* wait for command completion */
@@ -366,11 +366,11 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
 	psp_timeout = psp_cmd_timeout;
 
 	if (psp_ret)
-		*psp_ret = reg & PSP_CMDRESP_ERR_MASK;
+		*psp_ret = FIELD_GET(PSP_CMDRESP_STS, reg);
 
-	if (reg & PSP_CMDRESP_ERR_MASK) {
-		dev_dbg(sev->dev, "sev command %#x failed (%#010x)\n",
-			cmd, reg & PSP_CMDRESP_ERR_MASK);
+	if (FIELD_GET(PSP_CMDRESP_STS, reg)) {
+		dev_dbg(sev->dev, "sev command %#x failed (%#010lx)\n",
+			cmd, FIELD_GET(PSP_CMDRESP_STS, reg));
 		ret = -EIO;
 	} else {
 		ret = sev_write_init_ex_file_if_required(cmd);
diff --git a/drivers/crypto/ccp/sev-dev.h b/drivers/crypto/ccp/sev-dev.h
index 666c21eb81ab..778c95155e74 100644
--- a/drivers/crypto/ccp/sev-dev.h
+++ b/drivers/crypto/ccp/sev-dev.h
@@ -25,8 +25,8 @@
 #include <linux/miscdevice.h>
 #include <linux/capability.h>
 
+#define SEV_CMDRESP_CMD			GENMASK(26, 16)
 #define SEV_CMD_COMPLETE		BIT(1)
-#define SEV_CMDRESP_CMD_SHIFT		16
 #define SEV_CMDRESP_IOC			BIT(0)
 
 struct sev_misc_dev {
diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h
index 20377e67f65d..1253a0217985 100644
--- a/drivers/crypto/ccp/sp-dev.h
+++ b/drivers/crypto/ccp/sp-dev.h
@@ -53,9 +53,19 @@ struct tee_vdata {
 	const unsigned int ring_rptr_reg;
 };
 
+struct platform_access_vdata {
+	const unsigned int cmdresp_reg;
+	const unsigned int cmdbuff_addr_lo_reg;
+	const unsigned int cmdbuff_addr_hi_reg;
+	const unsigned int doorbell_button_reg;
+	const unsigned int doorbell_cmd_reg;
+
+};
+
 struct psp_vdata {
 	const struct sev_vdata *sev;
 	const struct tee_vdata *tee;
+	const struct platform_access_vdata *platform_access;
 	const unsigned int feature_reg;
 	const unsigned int inten_reg;
 	const unsigned int intsts_reg;
diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c
index cde33b2ac71b..aa15bc4cac2b 100644
--- a/drivers/crypto/ccp/sp-pci.c
+++ b/drivers/crypto/ccp/sp-pci.c
@@ -361,6 +361,14 @@ static const struct tee_vdata teev1 = {
 	.ring_rptr_reg          = 0x10554,	/* C2PMSG_21 */
 };
 
+static const struct platform_access_vdata pa_v1 = {
+	.cmdresp_reg		= 0x10570,	/* C2PMSG_28 */
+	.cmdbuff_addr_lo_reg	= 0x10574,	/* C2PMSG_29 */
+	.cmdbuff_addr_hi_reg	= 0x10578,	/* C2PMSG_30 */
+	.doorbell_button_reg	= 0x10a24,	/* C2PMSG_73 */
+	.doorbell_cmd_reg	= 0x10a40,	/* C2PMSG_80 */
+};
+
 static const struct psp_vdata pspv1 = {
 	.sev			= &sevv1,
 	.feature_reg		= 0x105fc,	/* C2PMSG_63 */
@@ -377,6 +385,7 @@ static const struct psp_vdata pspv2 = {
 
 static const struct psp_vdata pspv3 = {
 	.tee			= &teev1,
+	.platform_access	= &pa_v1,
 	.feature_reg		= 0x109fc,	/* C2PMSG_63 */
 	.inten_reg		= 0x10690,	/* P2CMSG_INTEN */
 	.intsts_reg		= 0x10694,	/* P2CMSG_INTSTS */
@@ -451,9 +460,9 @@ static const struct pci_device_id sp_pci_table[] = {
 	{ PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&dev_vdata[2] },
 	{ PCI_VDEVICE(AMD, 0x1486), (kernel_ulong_t)&dev_vdata[3] },
 	{ PCI_VDEVICE(AMD, 0x15DF), (kernel_ulong_t)&dev_vdata[4] },
-	{ PCI_VDEVICE(AMD, 0x1649), (kernel_ulong_t)&dev_vdata[4] },
 	{ PCI_VDEVICE(AMD, 0x14CA), (kernel_ulong_t)&dev_vdata[5] },
 	{ PCI_VDEVICE(AMD, 0x15C7), (kernel_ulong_t)&dev_vdata[6] },
+	{ PCI_VDEVICE(AMD, 0x1649), (kernel_ulong_t)&dev_vdata[6] },
 	/* Last entry must be zero */
 	{ 0, }
 };
diff --git a/drivers/crypto/ccp/tee-dev.c b/drivers/crypto/ccp/tee-dev.c
index 5c9d47f3be37..5560bf8329a1 100644
--- a/drivers/crypto/ccp/tee-dev.c
+++ b/drivers/crypto/ccp/tee-dev.c
@@ -8,12 +8,13 @@
  * Copyright (C) 2019,2021 Advanced Micro Devices, Inc.
  */
 
+#include <linux/bitfield.h>
 #include <linux/types.h>
 #include <linux/mutex.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/gfp.h>
-#include <linux/psp-sev.h>
+#include <linux/psp.h>
 #include <linux/psp-tee.h>
 
 #include "psp-dev.h"
@@ -69,7 +70,7 @@ static int tee_wait_cmd_poll(struct psp_tee_device *tee, unsigned int timeout,
 
 	while (--nloop) {
 		*reg = ioread32(tee->io_regs + tee->vdata->cmdresp_reg);
-		if (*reg & PSP_CMDRESP_RESP)
+		if (FIELD_GET(PSP_CMDRESP_RESP, *reg))
 			return 0;
 
 		usleep_range(10000, 10100);
@@ -149,9 +150,9 @@ static int tee_init_ring(struct psp_tee_device *tee)
 		goto free_buf;
 	}
 
-	if (reg & PSP_CMDRESP_ERR_MASK) {
-		dev_err(tee->dev, "tee: ring init command failed (%#010x)\n",
-			reg & PSP_CMDRESP_ERR_MASK);
+	if (FIELD_GET(PSP_CMDRESP_STS, reg)) {
+		dev_err(tee->dev, "tee: ring init command failed (%#010lx)\n",
+			FIELD_GET(PSP_CMDRESP_STS, reg));
 		tee_free_ring(tee);
 		ret = -EIO;
 	}
@@ -179,9 +180,9 @@ static void tee_destroy_ring(struct psp_tee_device *tee)
 	ret = tee_wait_cmd_poll(tee, TEE_DEFAULT_TIMEOUT, &reg);
 	if (ret) {
 		dev_err(tee->dev, "tee: ring destroy command timed out\n");
-	} else if (reg & PSP_CMDRESP_ERR_MASK) {
-		dev_err(tee->dev, "tee: ring destroy command failed (%#010x)\n",
-			reg & PSP_CMDRESP_ERR_MASK);
+	} else if (FIELD_GET(PSP_CMDRESP_STS, reg)) {
+		dev_err(tee->dev, "tee: ring destroy command failed (%#010lx)\n",
+			FIELD_GET(PSP_CMDRESP_STS, reg));
 	}
 
 free_ring:
diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c
index d489c6f80892..c57f929805d5 100644
--- a/drivers/crypto/ccree/cc_driver.c
+++ b/drivers/crypto/ccree/cc_driver.c
@@ -350,9 +350,9 @@ static int init_cc_resources(struct platform_device *plat_dev)
 
 	/* Get device resources */
 	/* First CC registers space */
-	req_mem_cc_regs = platform_get_resource(plat_dev, IORESOURCE_MEM, 0);
 	/* Map registers space */
-	new_drvdata->cc_base = devm_ioremap_resource(dev, req_mem_cc_regs);
+	new_drvdata->cc_base = devm_platform_get_and_ioremap_resource(plat_dev,
+								      0, &req_mem_cc_regs);
 	if (IS_ERR(new_drvdata->cc_base))
 		return PTR_ERR(new_drvdata->cc_base);
 
diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c
index 5a7f6611803c..8e4a49b7ab4f 100644
--- a/drivers/crypto/hifn_795x.c
+++ b/drivers/crypto/hifn_795x.c
@@ -879,7 +879,7 @@ static int hifn_enable_crypto(struct hifn_device *dev)
 
 static void hifn_init_dma(struct hifn_device *dev)
 {
-	struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+	struct hifn_dma *dma = dev->desc_virt;
 	u32 dptr = dev->desc_dma;
 	int i;
 
@@ -1072,7 +1072,7 @@ static int hifn_setup_crypto_command(struct hifn_device *dev,
 		u8 *buf, unsigned dlen, unsigned slen,
 		u8 *key, int keylen, u8 *iv, int ivsize, u16 mode)
 {
-	struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+	struct hifn_dma *dma = dev->desc_virt;
 	struct hifn_crypt_command *cry_cmd;
 	u8 *buf_pos = buf;
 	u16 cmd_len;
@@ -1113,7 +1113,7 @@ static int hifn_setup_cmd_desc(struct hifn_device *dev,
 		struct hifn_context *ctx, struct hifn_request_context *rctx,
 		void *priv, unsigned int nbytes)
 {
-	struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+	struct hifn_dma *dma = dev->desc_virt;
 	int cmd_len, sa_idx;
 	u8 *buf, *buf_pos;
 	u16 mask;
@@ -1231,7 +1231,7 @@ err_out:
 static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
 		unsigned int offset, unsigned int size, int last)
 {
-	struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+	struct hifn_dma *dma = dev->desc_virt;
 	int idx;
 	dma_addr_t addr;
 
@@ -1264,7 +1264,7 @@ static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
 
 static void hifn_setup_res_desc(struct hifn_device *dev)
 {
-	struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+	struct hifn_dma *dma = dev->desc_virt;
 
 	dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
 			HIFN_D_VALID | HIFN_D_LAST);
@@ -1290,7 +1290,7 @@ static void hifn_setup_res_desc(struct hifn_device *dev)
 static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
 		unsigned offset, unsigned size, int last)
 {
-	struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+	struct hifn_dma *dma = dev->desc_virt;
 	int idx;
 	dma_addr_t addr;
 
@@ -1710,7 +1710,7 @@ static void hifn_process_ready(struct skcipher_request *req, int error)
 
 static void hifn_clear_rings(struct hifn_device *dev, int error)
 {
-	struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+	struct hifn_dma *dma = dev->desc_virt;
 	int i, u;
 
 	dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
@@ -1784,7 +1784,7 @@ static void hifn_work(struct work_struct *work)
 
 	spin_lock_irqsave(&dev->lock, flags);
 	if (dev->active == 0) {
-		struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+		struct hifn_dma *dma = dev->desc_virt;
 
 		if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) {
 			dev->flags &= ~HIFN_FLAG_CMD_BUSY;
@@ -1815,7 +1815,7 @@ static void hifn_work(struct work_struct *work)
 	if (reset) {
 		if (++dev->reset >= 5) {
 			int i;
-			struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+			struct hifn_dma *dma = dev->desc_virt;
 
 			dev_info(&dev->pdev->dev,
 				 "r: %08x, active: %d, started: %d, "
@@ -1848,8 +1848,8 @@ static void hifn_work(struct work_struct *work)
 
 static irqreturn_t hifn_interrupt(int irq, void *data)
 {
-	struct hifn_device *dev = (struct hifn_device *)data;
-	struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+	struct hifn_device *dev = data;
+	struct hifn_dma *dma = dev->desc_virt;
 	u32 dmacsr, restart;
 
 	dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR);
@@ -1914,7 +1914,7 @@ static void hifn_flush(struct hifn_device *dev)
 	unsigned long flags;
 	struct crypto_async_request *async_req;
 	struct skcipher_request *req;
-	struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+	struct hifn_dma *dma = dev->desc_virt;
 	int i;
 
 	for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {
diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig
index 4137a8bf131f..e8690c223584 100644
--- a/drivers/crypto/hisilicon/Kconfig
+++ b/drivers/crypto/hisilicon/Kconfig
@@ -82,3 +82,10 @@ config CRYPTO_DEV_HISI_TRNG
 	select CRYPTO_RNG
 	help
 	  Support for HiSilicon TRNG Driver.
+
+config CRYPTO_DEV_HISTB_TRNG
+	tristate "Support for HiSTB TRNG Driver"
+	depends on ARCH_HISI || COMPILE_TEST
+	select HW_RANDOM
+	help
+	  Support for HiSTB TRNG Driver.
diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile
index 8595a5a5d228..fc51e0edec69 100644
--- a/drivers/crypto/hisilicon/Makefile
+++ b/drivers/crypto/hisilicon/Makefile
@@ -5,4 +5,4 @@ obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += sec2/
 obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += hisi_qm.o
 hisi_qm-objs = qm.o sgl.o debugfs.o
 obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/
-obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += trng/
+obj-y += trng/
diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c
index 923f9c279265..5d0adfb54a34 100644
--- a/drivers/crypto/hisilicon/hpre/hpre_main.c
+++ b/drivers/crypto/hisilicon/hpre/hpre_main.c
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright (c) 2018-2019 HiSilicon Limited. */
 #include <linux/acpi.h>
-#include <linux/aer.h>
 #include <linux/bitops.h>
 #include <linux/debugfs.h>
 #include <linux/init.h>
diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
index e4c84433a88a..ad0c042b5e66 100644
--- a/drivers/crypto/hisilicon/qm.c
+++ b/drivers/crypto/hisilicon/qm.c
@@ -2,7 +2,6 @@
 /* Copyright (c) 2019 HiSilicon Limited. */
 #include <asm/page.h>
 #include <linux/acpi.h>
-#include <linux/aer.h>
 #include <linux/bitmap.h>
 #include <linux/dma-mapping.h>
 #include <linux/idr.h>
@@ -3691,7 +3690,7 @@ static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf,
 			       unsigned long *val,
 			       unsigned int *fun_index)
 {
-	struct bus_type *bus_type = qm->pdev->dev.bus;
+	const struct bus_type *bus_type = qm->pdev->dev.bus;
 	char tbuf_bdf[QM_DBG_READ_LEN] = {0};
 	char val_buf[QM_DBG_READ_LEN] = {0};
 	struct pci_dev *pdev;
diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c
index 93572c0d4faa..77f9f131b850 100644
--- a/drivers/crypto/hisilicon/sec2/sec_main.c
+++ b/drivers/crypto/hisilicon/sec2/sec_main.c
@@ -2,7 +2,6 @@
 /* Copyright (c) 2019 HiSilicon Limited. */
 
 #include <linux/acpi.h>
-#include <linux/aer.h>
 #include <linux/bitops.h>
 #include <linux/debugfs.h>
 #include <linux/init.h>
diff --git a/drivers/crypto/hisilicon/trng/Makefile b/drivers/crypto/hisilicon/trng/Makefile
index d909079f351c..cf20b057c66b 100644
--- a/drivers/crypto/hisilicon/trng/Makefile
+++ b/drivers/crypto/hisilicon/trng/Makefile
@@ -1,2 +1,5 @@
 obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += hisi-trng-v2.o
 hisi-trng-v2-objs = trng.o
+
+obj-$(CONFIG_CRYPTO_DEV_HISTB_TRNG) += histb-trng.o
+histb-trng-objs += trng-stb.o
diff --git a/drivers/crypto/hisilicon/trng/trng-stb.c b/drivers/crypto/hisilicon/trng/trng-stb.c
new file mode 100644
index 000000000000..29200a7d3d81
--- /dev/null
+++ b/drivers/crypto/hisilicon/trng/trng-stb.c
@@ -0,0 +1,176 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Device driver for True RNG in HiSTB SoCs
+ *
+ * Copyright (c) 2023 David Yang
+ */
+
+#include <crypto/internal/rng.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/hw_random.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_device.h>
+
+#define HISTB_TRNG_CTRL		0x0
+#define  RNG_SOURCE			GENMASK(1, 0)
+#define  DROP_ENABLE			BIT(5)
+#define  POST_PROCESS_ENABLE		BIT(7)
+#define  POST_PROCESS_DEPTH		GENMASK(15, 8)
+#define HISTB_TRNG_NUMBER	0x4
+#define HISTB_TRNG_STAT		0x8
+#define  DATA_COUNT			GENMASK(2, 0)	/* max 4 */
+
+struct histb_trng_priv {
+	struct hwrng rng;
+	void __iomem *base;
+};
+
+/*
+ * Observed:
+ * depth = 1 -> ~1ms
+ * depth = 255 -> ~16ms
+ */
+static int histb_trng_wait(void __iomem *base)
+{
+	u32 val;
+
+	return readl_relaxed_poll_timeout(base + HISTB_TRNG_STAT, val,
+					  val & DATA_COUNT, 1000, 30 * 1000);
+}
+
+static void histb_trng_init(void __iomem *base, unsigned int depth)
+{
+	u32 val;
+
+	val = readl_relaxed(base + HISTB_TRNG_CTRL);
+
+	val &= ~RNG_SOURCE;
+	val |= 2;
+
+	val &= ~POST_PROCESS_DEPTH;
+	val |= min(depth, 0xffu) << 8;
+
+	val |= POST_PROCESS_ENABLE;
+	val |= DROP_ENABLE;
+
+	writel_relaxed(val, base + HISTB_TRNG_CTRL);
+}
+
+static int histb_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)
+{
+	struct histb_trng_priv *priv = container_of(rng, typeof(*priv), rng);
+	void __iomem *base = priv->base;
+
+	for (int i = 0; i < max; i += sizeof(u32)) {
+		if (!(readl_relaxed(base + HISTB_TRNG_STAT) & DATA_COUNT)) {
+			if (!wait)
+				return i;
+			if (histb_trng_wait(base)) {
+				pr_err("failed to generate random number, generated %d\n",
+				       i);
+				return i ? i : -ETIMEDOUT;
+			}
+		}
+		*(u32 *) (data + i) = readl_relaxed(base + HISTB_TRNG_NUMBER);
+	}
+
+	return max;
+}
+
+static unsigned int histb_trng_get_depth(void __iomem *base)
+{
+	return (readl_relaxed(base + HISTB_TRNG_CTRL) & POST_PROCESS_DEPTH) >> 8;
+}
+
+static ssize_t
+depth_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct histb_trng_priv *priv = dev_get_drvdata(dev);
+	void __iomem *base = priv->base;
+
+	return sprintf(buf, "%d\n", histb_trng_get_depth(base));
+}
+
+static ssize_t
+depth_store(struct device *dev, struct device_attribute *attr,
+	    const char *buf, size_t count)
+{
+	struct histb_trng_priv *priv = dev_get_drvdata(dev);
+	void __iomem *base = priv->base;
+	unsigned int depth;
+
+	if (kstrtouint(buf, 0, &depth))
+		return -ERANGE;
+
+	histb_trng_init(base, depth);
+	return count;
+}
+
+static DEVICE_ATTR_RW(depth);
+
+static struct attribute *histb_trng_attrs[] = {
+	&dev_attr_depth.attr,
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(histb_trng);
+
+static int histb_trng_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct histb_trng_priv *priv;
+	void __iomem *base;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return -ENOMEM;
+
+	histb_trng_init(base, 144);
+	if (histb_trng_wait(base)) {
+		dev_err(dev, "cannot bring up device\n");
+		return -ENODEV;
+	}
+
+	priv->base = base;
+	priv->rng.name = pdev->name;
+	priv->rng.read = histb_trng_read;
+	ret = devm_hwrng_register(dev, &priv->rng);
+	if (ret) {
+		dev_err(dev, "failed to register hwrng: %d\n", ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, priv);
+	dev_set_drvdata(dev, priv);
+	return 0;
+}
+
+static const struct of_device_id histb_trng_of_match[] = {
+	{ .compatible = "hisilicon,histb-trng", },
+	{ }
+};
+
+static struct platform_driver histb_trng_driver = {
+	.probe = histb_trng_probe,
+	.driver = {
+		.name = "histb-trng",
+		.of_match_table = histb_trng_of_match,
+		.dev_groups = histb_trng_groups,
+	},
+};
+
+module_platform_driver(histb_trng_driver);
+
+MODULE_DESCRIPTION("HiSTB True RNG");
+MODULE_LICENSE("Dual MIT/GPL");
+MODULE_AUTHOR("David Yang <mmyangfl@gmail.com>");
diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c
index 1549bec3aea5..f3ce34198775 100644
--- a/drivers/crypto/hisilicon/zip/zip_main.c
+++ b/drivers/crypto/hisilicon/zip/zip_main.c
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright (c) 2019 HiSilicon Limited. */
 #include <linux/acpi.h>
-#include <linux/aer.h>
 #include <linux/bitops.h>
 #include <linux/debugfs.h>
 #include <linux/init.h>
diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c
index fe93d19e3044..359aa2b41016 100644
--- a/drivers/crypto/img-hash.c
+++ b/drivers/crypto/img-hash.c
@@ -209,7 +209,7 @@ static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf,
 
 static void img_hash_dma_callback(void *data)
 {
-	struct img_hash_dev *hdev = (struct img_hash_dev *)data;
+	struct img_hash_dev *hdev = data;
 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 
 	if (ctx->bufcnt) {
@@ -927,7 +927,7 @@ finish:
 	img_hash_finish_req(hdev->req, err);
 }
 
-static const struct of_device_id img_hash_match[] = {
+static const struct of_device_id img_hash_match[] __maybe_unused = {
 	{ .compatible = "img,hash-accelerator" },
 	{}
 };
@@ -966,8 +966,7 @@ static int img_hash_probe(struct platform_device *pdev)
 	}
 
 	/* Write port (DMA or CPU) */
-	hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-	hdev->cpu_addr = devm_ioremap_resource(dev, hash_res);
+	hdev->cpu_addr = devm_platform_get_and_ioremap_resource(pdev, 1, &hash_res);
 	if (IS_ERR(hdev->cpu_addr)) {
 		err = PTR_ERR(hdev->cpu_addr);
 		goto res_err;
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
index 6858753af6b3..9ff02b5abc4a 100644
--- a/drivers/crypto/inside-secure/safexcel.c
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -474,7 +474,7 @@ release_fw:
 		goto retry_fw;
 	}
 
-	dev_dbg(priv->dev, "Firmware load failed.\n");
+	dev_err(priv->dev, "Firmware load failed.\n");
 
 	return ret;
 }
@@ -1628,19 +1628,23 @@ static int safexcel_probe_generic(void *pdev,
 						     &priv->ring[i].rdr);
 		if (ret) {
 			dev_err(dev, "Failed to initialize rings\n");
-			return ret;
+			goto err_cleanup_rings;
 		}
 
 		priv->ring[i].rdr_req = devm_kcalloc(dev,
 			EIP197_DEFAULT_RING_SIZE,
 			sizeof(*priv->ring[i].rdr_req),
 			GFP_KERNEL);
-		if (!priv->ring[i].rdr_req)
-			return -ENOMEM;
+		if (!priv->ring[i].rdr_req) {
+			ret = -ENOMEM;
+			goto err_cleanup_rings;
+		}
 
 		ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
-		if (!ring_irq)
-			return -ENOMEM;
+		if (!ring_irq) {
+			ret = -ENOMEM;
+			goto err_cleanup_rings;
+		}
 
 		ring_irq->priv = priv;
 		ring_irq->ring = i;
@@ -1654,7 +1658,8 @@ static int safexcel_probe_generic(void *pdev,
 						ring_irq);
 		if (irq < 0) {
 			dev_err(dev, "Failed to get IRQ ID for ring %d\n", i);
-			return irq;
+			ret = irq;
+			goto err_cleanup_rings;
 		}
 
 		priv->ring[i].irq = irq;
@@ -1666,8 +1671,10 @@ static int safexcel_probe_generic(void *pdev,
 		snprintf(wq_name, 9, "wq_ring%d", i);
 		priv->ring[i].workqueue =
 			create_singlethread_workqueue(wq_name);
-		if (!priv->ring[i].workqueue)
-			return -ENOMEM;
+		if (!priv->ring[i].workqueue) {
+			ret = -ENOMEM;
+			goto err_cleanup_rings;
+		}
 
 		priv->ring[i].requests = 0;
 		priv->ring[i].busy = false;
@@ -1684,16 +1691,26 @@ static int safexcel_probe_generic(void *pdev,
 	ret = safexcel_hw_init(priv);
 	if (ret) {
 		dev_err(dev, "HW init failed (%d)\n", ret);
-		return ret;
+		goto err_cleanup_rings;
 	}
 
 	ret = safexcel_register_algorithms(priv);
 	if (ret) {
 		dev_err(dev, "Failed to register algorithms (%d)\n", ret);
-		return ret;
+		goto err_cleanup_rings;
 	}
 
 	return 0;
+
+err_cleanup_rings:
+	for (i = 0; i < priv->config.rings; i++) {
+		if (priv->ring[i].irq)
+			irq_set_affinity_hint(priv->ring[i].irq, NULL);
+		if (priv->ring[i].workqueue)
+			destroy_workqueue(priv->ring[i].workqueue);
+	}
+
+	return ret;
 }
 
 static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
diff --git a/drivers/crypto/intel/Kconfig b/drivers/crypto/intel/Kconfig
new file mode 100644
index 000000000000..3d90c87d4094
--- /dev/null
+++ b/drivers/crypto/intel/Kconfig
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+
+source "drivers/crypto/intel/keembay/Kconfig"
+source "drivers/crypto/intel/ixp4xx/Kconfig"
+source "drivers/crypto/intel/qat/Kconfig"
diff --git a/drivers/crypto/intel/Makefile b/drivers/crypto/intel/Makefile
new file mode 100644
index 000000000000..b3d0352ae188
--- /dev/null
+++ b/drivers/crypto/intel/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-y += keembay/
+obj-y += ixp4xx/
+obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/
diff --git a/drivers/crypto/intel/ixp4xx/Kconfig b/drivers/crypto/intel/ixp4xx/Kconfig
new file mode 100644
index 000000000000..af3cc5688328
--- /dev/null
+++ b/drivers/crypto/intel/ixp4xx/Kconfig
@@ -0,0 +1,14 @@
+config CRYPTO_DEV_IXP4XX
+	tristate "Driver for IXP4xx crypto hardware acceleration"
+	depends on (ARCH_IXP4XX || COMPILE_TEST) && IXP4XX_QMGR && IXP4XX_NPE
+	select CRYPTO_AES
+	select CRYPTO_DES
+	select CRYPTO_ECB
+	select CRYPTO_CBC
+	select CRYPTO_CTR
+	select CRYPTO_LIB_DES
+	select CRYPTO_AEAD
+	select CRYPTO_AUTHENC
+	select CRYPTO_SKCIPHER
+	help
+	  Driver for the IXP4xx NPE crypto engine.
diff --git a/drivers/crypto/intel/ixp4xx/Makefile b/drivers/crypto/intel/ixp4xx/Makefile
new file mode 100644
index 000000000000..74ebefd93046
--- /dev/null
+++ b/drivers/crypto/intel/ixp4xx/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c
index b63e2359a133..ed15379a9818 100644
--- a/drivers/crypto/ixp4xx_crypto.c
+++ b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c
@@ -118,9 +118,9 @@ struct crypt_ctl {
 	u8 mode;		/* NPE_OP_*  operation mode */
 #endif
 	u8 iv[MAX_IVLEN];	/* IV for CBC mode or CTR IV for CTR mode */
-	dma_addr_t icv_rev_aes;	/* icv or rev aes */
-	dma_addr_t src_buf;
-	dma_addr_t dst_buf;
+	u32 icv_rev_aes;	/* icv or rev aes */
+	u32 src_buf;
+	u32 dst_buf;
 #ifdef __ARMEB__
 	u16 auth_offs;		/* Authentication start offset */
 	u16 auth_len;		/* Authentication data length */
@@ -263,7 +263,9 @@ static int setup_crypt_desc(void)
 {
 	struct device *dev = &pdev->dev;
 
-	BUILD_BUG_ON(sizeof(struct crypt_ctl) != 64);
+	BUILD_BUG_ON(!(IS_ENABLED(CONFIG_COMPILE_TEST) &&
+		       IS_ENABLED(CONFIG_64BIT)) &&
+		     sizeof(struct crypt_ctl) != 64);
 	crypt_virt = dma_alloc_coherent(dev,
 					NPE_QLEN * sizeof(struct crypt_ctl),
 					&crypt_phys, GFP_ATOMIC);
@@ -1169,10 +1171,11 @@ static int aead_perform(struct aead_request *req, int encrypt,
 	}
 
 	if (unlikely(lastlen < authsize)) {
+		dma_addr_t dma;
 		/* The 12 hmac bytes are scattered,
 		 * we need to copy them into a safe buffer */
-		req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags,
-						    &crypt->icv_rev_aes);
+		req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags, &dma);
+		crypt->icv_rev_aes = dma;
 		if (unlikely(!req_ctx->hmac_virt))
 			goto free_buf_dst;
 		if (!encrypt) {
diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/intel/keembay/Kconfig
index 1cd62f9c3e3a..1cd62f9c3e3a 100644
--- a/drivers/crypto/keembay/Kconfig
+++ b/drivers/crypto/intel/keembay/Kconfig
diff --git a/drivers/crypto/keembay/Makefile b/drivers/crypto/intel/keembay/Makefile
index 7c12c3c138bd..7c12c3c138bd 100644
--- a/drivers/crypto/keembay/Makefile
+++ b/drivers/crypto/intel/keembay/Makefile
diff --git a/drivers/crypto/keembay/keembay-ocs-aes-core.c b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c
index 9953f5590ac4..ae31be00357a 100644
--- a/drivers/crypto/keembay/keembay-ocs-aes-core.c
+++ b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c
@@ -1580,8 +1580,6 @@ static int kmb_ocs_aes_remove(struct platform_device *pdev)
 	struct ocs_aes_dev *aes_dev;
 
 	aes_dev = platform_get_drvdata(pdev);
-	if (!aes_dev)
-		return -ENODEV;
 
 	unregister_aes_algs(aes_dev);
 
diff --git a/drivers/crypto/keembay/keembay-ocs-ecc.c b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c
index 2269df17514c..2269df17514c 100644
--- a/drivers/crypto/keembay/keembay-ocs-ecc.c
+++ b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c
diff --git a/drivers/crypto/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c
index d4bcbed1f546..d4bcbed1f546 100644
--- a/drivers/crypto/keembay/keembay-ocs-hcu-core.c
+++ b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c
diff --git a/drivers/crypto/keembay/ocs-aes.c b/drivers/crypto/intel/keembay/ocs-aes.c
index be9f32fc8f42..be9f32fc8f42 100644
--- a/drivers/crypto/keembay/ocs-aes.c
+++ b/drivers/crypto/intel/keembay/ocs-aes.c
diff --git a/drivers/crypto/keembay/ocs-aes.h b/drivers/crypto/intel/keembay/ocs-aes.h
index c035fc48b7ed..c035fc48b7ed 100644
--- a/drivers/crypto/keembay/ocs-aes.h
+++ b/drivers/crypto/intel/keembay/ocs-aes.h
diff --git a/drivers/crypto/keembay/ocs-hcu.c b/drivers/crypto/intel/keembay/ocs-hcu.c
index deb9bd460ee6..deb9bd460ee6 100644
--- a/drivers/crypto/keembay/ocs-hcu.c
+++ b/drivers/crypto/intel/keembay/ocs-hcu.c
diff --git a/drivers/crypto/keembay/ocs-hcu.h b/drivers/crypto/intel/keembay/ocs-hcu.h
index fbbbb92a0592..fbbbb92a0592 100644
--- a/drivers/crypto/keembay/ocs-hcu.h
+++ b/drivers/crypto/intel/keembay/ocs-hcu.h
diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/intel/qat/Kconfig
index 1220cc86f910..1220cc86f910 100644
--- a/drivers/crypto/qat/Kconfig
+++ b/drivers/crypto/intel/qat/Kconfig
diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/intel/qat/Makefile
index 258c8a626ce0..258c8a626ce0 100644
--- a/drivers/crypto/qat/Makefile
+++ b/drivers/crypto/intel/qat/Makefile
diff --git a/drivers/crypto/qat/qat_4xxx/Makefile b/drivers/crypto/intel/qat/qat_4xxx/Makefile
index ff9c8b5897ea..ff9c8b5897ea 100644
--- a/drivers/crypto/qat/qat_4xxx/Makefile
+++ b/drivers/crypto/intel/qat/qat_4xxx/Makefile
diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c
index 834a705180c0..7324b86a4f40 100644
--- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c
+++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c
@@ -28,13 +28,31 @@ static struct adf_fw_config adf_4xxx_fw_dc_config[] = {
 	{0x100, ADF_4XXX_ADMIN_OBJ},
 };
 
+static struct adf_fw_config adf_402xx_fw_cy_config[] = {
+	{0xF0, ADF_402XX_SYM_OBJ},
+	{0xF, ADF_402XX_ASYM_OBJ},
+	{0x100, ADF_402XX_ADMIN_OBJ},
+};
+
+static struct adf_fw_config adf_402xx_fw_dc_config[] = {
+	{0xF0, ADF_402XX_DC_OBJ},
+	{0xF, ADF_402XX_DC_OBJ},
+	{0x100, ADF_402XX_ADMIN_OBJ},
+};
+
 /* Worker thread to service arbiter mappings */
-static const u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = {
+static const u32 thrd_to_arb_map_cy[ADF_4XXX_MAX_ACCELENGINES] = {
 	0x5555555, 0x5555555, 0x5555555, 0x5555555,
 	0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA,
 	0x0
 };
 
+static const u32 thrd_to_arb_map_dc[ADF_4XXX_MAX_ACCELENGINES] = {
+	0x000000FF, 0x000000FF, 0x000000FF, 0x000000FF,
+	0x000000FF, 0x000000FF, 0x000000FF, 0x000000FF,
+	0x0
+};
+
 static struct adf_hw_device_class adf_4xxx_class = {
 	.name = ADF_4XXX_DEVICE_NAME,
 	.type = DEV_4XXX,
@@ -206,9 +224,16 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
 	return DEV_SKU_1;
 }
 
-static const u32 *adf_get_arbiter_mapping(void)
+static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev)
 {
-	return thrd_to_arb_map;
+	switch (get_service_enabled(accel_dev)) {
+	case SVC_CY:
+		return thrd_to_arb_map_cy;
+	case SVC_DC:
+		return thrd_to_arb_map_dc;
+	}
+
+	return NULL;
 }
 
 static void get_arb_info(struct arb_info *arb_info)
@@ -286,7 +311,7 @@ static u32 uof_get_num_objs(void)
 	return ARRAY_SIZE(adf_4xxx_fw_cy_config);
 }
 
-static char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num)
+static char *uof_get_name_4xxx(struct adf_accel_dev *accel_dev, u32 obj_num)
 {
 	switch (get_service_enabled(accel_dev)) {
 	case SVC_CY:
@@ -298,6 +323,18 @@ static char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num)
 	return NULL;
 }
 
+static char *uof_get_name_402xx(struct adf_accel_dev *accel_dev, u32 obj_num)
+{
+	switch (get_service_enabled(accel_dev)) {
+	case SVC_CY:
+		return adf_402xx_fw_cy_config[obj_num].obj_name;
+	case SVC_DC:
+		return adf_402xx_fw_dc_config[obj_num].obj_name;
+	}
+
+	return NULL;
+}
+
 static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num)
 {
 	switch (get_service_enabled(accel_dev)) {
@@ -310,7 +347,7 @@ static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num)
 	return 0;
 }
 
-void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
+void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id)
 {
 	hw_data->dev_class = &adf_4xxx_class;
 	hw_data->instance_id = adf_4xxx_class.instances++;
@@ -337,8 +374,6 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
 	hw_data->get_admin_info = get_admin_info;
 	hw_data->get_accel_cap = get_accel_cap;
 	hw_data->get_sku = get_sku;
-	hw_data->fw_name = ADF_4XXX_FW;
-	hw_data->fw_mmp_name = ADF_4XXX_MMP;
 	hw_data->init_admin_comms = adf_init_admin_comms;
 	hw_data->exit_admin_comms = adf_exit_admin_comms;
 	hw_data->send_admin_init = adf_send_admin_init;
@@ -349,8 +384,19 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
 	hw_data->init_device = adf_init_device;
 	hw_data->reset_device = adf_reset_flr;
 	hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK;
+	switch (dev_id) {
+	case ADF_402XX_PCI_DEVICE_ID:
+		hw_data->fw_name = ADF_402XX_FW;
+		hw_data->fw_mmp_name = ADF_402XX_MMP;
+		hw_data->uof_get_name = uof_get_name_402xx;
+		break;
+
+	default:
+		hw_data->fw_name = ADF_4XXX_FW;
+		hw_data->fw_mmp_name = ADF_4XXX_MMP;
+		hw_data->uof_get_name = uof_get_name_4xxx;
+	}
 	hw_data->uof_get_num_objs = uof_get_num_objs;
-	hw_data->uof_get_name = uof_get_name;
 	hw_data->uof_get_ae_mask = uof_get_ae_mask;
 	hw_data->set_msix_rttable = set_msix_default_rttable;
 	hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer;
diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h
index e98428ba78e2..085e259c245a 100644
--- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h
+++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h
@@ -56,6 +56,13 @@
 #define ADF_4XXX_DC_OBJ		"qat_4xxx_dc.bin"
 #define ADF_4XXX_ASYM_OBJ	"qat_4xxx_asym.bin"
 #define ADF_4XXX_ADMIN_OBJ	"qat_4xxx_admin.bin"
+/* Firmware for 402XXX */
+#define ADF_402XX_FW		"qat_402xx.bin"
+#define ADF_402XX_MMP		"qat_402xx_mmp.bin"
+#define ADF_402XX_SYM_OBJ	"qat_402xx_sym.bin"
+#define ADF_402XX_DC_OBJ	"qat_402xx_dc.bin"
+#define ADF_402XX_ASYM_OBJ	"qat_402xx_asym.bin"
+#define ADF_402XX_ADMIN_OBJ	"qat_402xx_admin.bin"
 
 /* qat_4xxx fuse bits are different from old GENs, redefine them */
 enum icp_qat_4xxx_slice_mask {
@@ -68,7 +75,7 @@ enum icp_qat_4xxx_slice_mask {
 	ICP_ACCEL_4XXX_MASK_SMX_SLICE = BIT(6),
 };
 
-void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data);
+void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id);
 void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data);
 int adf_gen4_dev_config(struct adf_accel_dev *accel_dev);
 
diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c
index b3a4c7b23864..ceb87327a5fe 100644
--- a/drivers/crypto/qat/qat_4xxx/adf_drv.c
+++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c
@@ -16,6 +16,7 @@
 static const struct pci_device_id adf_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), },
 	{ PCI_VDEVICE(INTEL, ADF_401XX_PCI_DEVICE_ID), },
+	{ PCI_VDEVICE(INTEL, ADF_402XX_PCI_DEVICE_ID), },
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
@@ -330,7 +331,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	accel_dev->hw_device = hw_data;
-	adf_init_hw_data_4xxx(accel_dev->hw_device);
+	adf_init_hw_data_4xxx(accel_dev->hw_device, ent->device);
 
 	pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
 	pci_read_config_dword(pdev, ADF_4XXX_FUSECTL4_OFFSET, &hw_data->fuses);
@@ -403,38 +404,24 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pci_set_master(pdev);
 
-	adf_enable_aer(accel_dev);
-
 	if (pci_save_state(pdev)) {
 		dev_err(&pdev->dev, "Failed to save pci state.\n");
 		ret = -ENOMEM;
-		goto out_err_disable_aer;
+		goto out_err;
 	}
 
-	ret = adf_sysfs_init(accel_dev);
-	if (ret)
-		goto out_err_disable_aer;
-
-	ret = hw_data->dev_config(accel_dev);
+	ret = adf_dev_up(accel_dev, true);
 	if (ret)
-		goto out_err_disable_aer;
-
-	ret = adf_dev_init(accel_dev);
-	if (ret)
-		goto out_err_dev_shutdown;
+		goto out_err_dev_stop;
 
-	ret = adf_dev_start(accel_dev);
+	ret = adf_sysfs_init(accel_dev);
 	if (ret)
 		goto out_err_dev_stop;
 
 	return ret;
 
 out_err_dev_stop:
-	adf_dev_stop(accel_dev);
-out_err_dev_shutdown:
-	adf_dev_shutdown(accel_dev);
-out_err_disable_aer:
-	adf_disable_aer(accel_dev);
+	adf_dev_down(accel_dev, false);
 out_err:
 	adf_cleanup_accel(accel_dev);
 	return ret;
@@ -448,9 +435,7 @@ static void adf_remove(struct pci_dev *pdev)
 		pr_err("QAT: Driver removal failed\n");
 		return;
 	}
-	adf_dev_stop(accel_dev);
-	adf_dev_shutdown(accel_dev);
-	adf_disable_aer(accel_dev);
+	adf_dev_down(accel_dev, false);
 	adf_cleanup_accel(accel_dev);
 }
 
diff --git a/drivers/crypto/qat/qat_c3xxx/Makefile b/drivers/crypto/intel/qat/qat_c3xxx/Makefile
index 92ef416ccc78..92ef416ccc78 100644
--- a/drivers/crypto/qat/qat_c3xxx/Makefile
+++ b/drivers/crypto/intel/qat/qat_c3xxx/Makefile
diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c
index c55c51a07677..475643654e64 100644
--- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c
+++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c
@@ -75,7 +75,7 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
 	return DEV_SKU_UNKNOWN;
 }
 
-static const u32 *adf_get_arbiter_mapping(void)
+static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev)
 {
 	return thrd_to_arb_map;
 }
diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h
index 336a06f11dbd..336a06f11dbd 100644
--- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h
+++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h
diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c
index 1f4fbf4562b2..bb4dca735ab5 100644
--- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c
+++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c
@@ -193,34 +193,20 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 	pci_set_master(pdev);
 
-	adf_enable_aer(accel_dev);
-
 	if (pci_save_state(pdev)) {
 		dev_err(&pdev->dev, "Failed to save pci state\n");
 		ret = -ENOMEM;
-		goto out_err_disable_aer;
+		goto out_err_free_reg;
 	}
 
-	ret = hw_data->dev_config(accel_dev);
-	if (ret)
-		goto out_err_disable_aer;
-
-	ret = adf_dev_init(accel_dev);
-	if (ret)
-		goto out_err_dev_shutdown;
-
-	ret = adf_dev_start(accel_dev);
+	ret = adf_dev_up(accel_dev, true);
 	if (ret)
 		goto out_err_dev_stop;
 
 	return ret;
 
 out_err_dev_stop:
-	adf_dev_stop(accel_dev);
-out_err_dev_shutdown:
-	adf_dev_shutdown(accel_dev);
-out_err_disable_aer:
-	adf_disable_aer(accel_dev);
+	adf_dev_down(accel_dev, false);
 out_err_free_reg:
 	pci_release_regions(accel_pci_dev->pci_dev);
 out_err_disable:
@@ -239,9 +225,7 @@ static void adf_remove(struct pci_dev *pdev)
 		pr_err("QAT: Driver removal failed\n");
 		return;
 	}
-	adf_dev_stop(accel_dev);
-	adf_dev_shutdown(accel_dev);
-	adf_disable_aer(accel_dev);
+	adf_dev_down(accel_dev, false);
 	adf_cleanup_accel(accel_dev);
 	adf_cleanup_pci_dev(accel_dev);
 	kfree(accel_dev);
diff --git a/drivers/crypto/qat/qat_c3xxxvf/Makefile b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile
index b6d76825a92c..b6d76825a92c 100644
--- a/drivers/crypto/qat/qat_c3xxxvf/Makefile
+++ b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile
diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c
index 84d9486e04de..84d9486e04de 100644
--- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c
+++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c
diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h
index 6b4bf181d15b..6b4bf181d15b 100644
--- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h
+++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h
diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c
index cf4ef83e186f..e8cc10f64134 100644
--- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c
+++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c
@@ -173,20 +173,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* Completion for VF2PF request/response message exchange */
 	init_completion(&accel_dev->vf.msg_received);
 
-	ret = adf_dev_init(accel_dev);
-	if (ret)
-		goto out_err_dev_shutdown;
-
-	ret = adf_dev_start(accel_dev);
+	ret = adf_dev_up(accel_dev, false);
 	if (ret)
 		goto out_err_dev_stop;
 
 	return ret;
 
 out_err_dev_stop:
-	adf_dev_stop(accel_dev);
-out_err_dev_shutdown:
-	adf_dev_shutdown(accel_dev);
+	adf_dev_down(accel_dev, false);
 out_err_free_reg:
 	pci_release_regions(accel_pci_dev->pci_dev);
 out_err_disable:
@@ -206,8 +200,7 @@ static void adf_remove(struct pci_dev *pdev)
 		return;
 	}
 	adf_flush_vf_wq(accel_dev);
-	adf_dev_stop(accel_dev);
-	adf_dev_shutdown(accel_dev);
+	adf_dev_down(accel_dev, false);
 	adf_cleanup_accel(accel_dev);
 	adf_cleanup_pci_dev(accel_dev);
 	kfree(accel_dev);
diff --git a/drivers/crypto/qat/qat_c62x/Makefile b/drivers/crypto/intel/qat/qat_c62x/Makefile
index d581f7c87d6c..d581f7c87d6c 100644
--- a/drivers/crypto/qat/qat_c62x/Makefile
+++ b/drivers/crypto/intel/qat/qat_c62x/Makefile
diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c
index b7aa19d2fa80..e14270703670 100644
--- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c
+++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c
@@ -77,7 +77,7 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
 	return DEV_SKU_UNKNOWN;
 }
 
-static const u32 *adf_get_arbiter_mapping(void)
+static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev)
 {
 	return thrd_to_arb_map;
 }
diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h
index 008c0a3a9769..008c0a3a9769 100644
--- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h
+++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h
diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c
index 4ccaf298250c..ca18ae14c099 100644
--- a/drivers/crypto/qat/qat_c62x/adf_drv.c
+++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c
@@ -193,34 +193,20 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 	pci_set_master(pdev);
 
-	adf_enable_aer(accel_dev);
-
 	if (pci_save_state(pdev)) {
 		dev_err(&pdev->dev, "Failed to save pci state\n");
 		ret = -ENOMEM;
-		goto out_err_disable_aer;
+		goto out_err_free_reg;
 	}
 
-	ret = hw_data->dev_config(accel_dev);
-	if (ret)
-		goto out_err_disable_aer;
-
-	ret = adf_dev_init(accel_dev);
-	if (ret)
-		goto out_err_dev_shutdown;
-
-	ret = adf_dev_start(accel_dev);
+	ret = adf_dev_up(accel_dev, true);
 	if (ret)
 		goto out_err_dev_stop;
 
 	return ret;
 
 out_err_dev_stop:
-	adf_dev_stop(accel_dev);
-out_err_dev_shutdown:
-	adf_dev_shutdown(accel_dev);
-out_err_disable_aer:
-	adf_disable_aer(accel_dev);
+	adf_dev_down(accel_dev, false);
 out_err_free_reg:
 	pci_release_regions(accel_pci_dev->pci_dev);
 out_err_disable:
@@ -239,9 +225,7 @@ static void adf_remove(struct pci_dev *pdev)
 		pr_err("QAT: Driver removal failed\n");
 		return;
 	}
-	adf_dev_stop(accel_dev);
-	adf_dev_shutdown(accel_dev);
-	adf_disable_aer(accel_dev);
+	adf_dev_down(accel_dev, false);
 	adf_cleanup_accel(accel_dev);
 	adf_cleanup_pci_dev(accel_dev);
 	kfree(accel_dev);
diff --git a/drivers/crypto/qat/qat_c62xvf/Makefile b/drivers/crypto/intel/qat/qat_c62xvf/Makefile
index 446c3d638605..446c3d638605 100644
--- a/drivers/crypto/qat/qat_c62xvf/Makefile
+++ b/drivers/crypto/intel/qat/qat_c62xvf/Makefile
diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c
index 751d7aa57fc7..751d7aa57fc7 100644
--- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c
+++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c
diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.h
index a1a62c003ebf..a1a62c003ebf 100644
--- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h
+++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.h
diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c
index 0e642c94b929..37566309df94 100644
--- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c
+++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c
@@ -173,20 +173,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* Completion for VF2PF request/response message exchange */
 	init_completion(&accel_dev->vf.msg_received);
 
-	ret = adf_dev_init(accel_dev);
-	if (ret)
-		goto out_err_dev_shutdown;
-
-	ret = adf_dev_start(accel_dev);
+	ret = adf_dev_up(accel_dev, false);
 	if (ret)
 		goto out_err_dev_stop;
 
 	return ret;
 
 out_err_dev_stop:
-	adf_dev_stop(accel_dev);
-out_err_dev_shutdown:
-	adf_dev_shutdown(accel_dev);
+	adf_dev_down(accel_dev, false);
 out_err_free_reg:
 	pci_release_regions(accel_pci_dev->pci_dev);
 out_err_disable:
@@ -206,8 +200,7 @@ static void adf_remove(struct pci_dev *pdev)
 		return;
 	}
 	adf_flush_vf_wq(accel_dev);
-	adf_dev_stop(accel_dev);
-	adf_dev_shutdown(accel_dev);
+	adf_dev_down(accel_dev, false);
 	adf_cleanup_accel(accel_dev);
 	adf_cleanup_pci_dev(accel_dev);
 	kfree(accel_dev);
diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile
index 1fb8d50f509f..1fb8d50f509f 100644
--- a/drivers/crypto/qat/qat_common/Makefile
+++ b/drivers/crypto/intel/qat/qat_common/Makefile
diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h
index 284f5aad3ee0..bd19e6460899 100644
--- a/drivers/crypto/qat/qat_common/adf_accel_devices.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h
@@ -21,6 +21,8 @@
 #define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941
 #define ADF_401XX_PCI_DEVICE_ID 0x4942
 #define ADF_401XXIOV_PCI_DEVICE_ID 0x4943
+#define ADF_402XX_PCI_DEVICE_ID 0x4944
+#define ADF_402XXIOV_PCI_DEVICE_ID 0x4945
 #define ADF_DEVICE_FUSECTL_OFFSET 0x40
 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C
 #define ADF_DEVICE_FUSECTL_MASK 0x80000000
@@ -188,7 +190,7 @@ struct adf_hw_device_data {
 	int (*send_admin_init)(struct adf_accel_dev *accel_dev);
 	int (*init_arb)(struct adf_accel_dev *accel_dev);
 	void (*exit_arb)(struct adf_accel_dev *accel_dev);
-	const u32 *(*get_arb_mapping)(void);
+	const u32 *(*get_arb_mapping)(struct adf_accel_dev *accel_dev);
 	int (*init_device)(struct adf_accel_dev *accel_dev);
 	int (*enable_pm)(struct adf_accel_dev *accel_dev);
 	bool (*handle_pm_interrupt)(struct adf_accel_dev *accel_dev);
@@ -310,6 +312,7 @@ struct adf_accel_dev {
 			u8 pf_compat_ver;
 		} vf;
 	};
+	struct mutex state_lock; /* protect state of the device */
 	bool is_vf;
 	u32 accel_id;
 };
diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c
index 4ce2b666929e..4ce2b666929e 100644
--- a/drivers/crypto/qat/qat_common/adf_accel_engine.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c
diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/intel/qat/qat_common/adf_admin.c
index 3b6184c35081..3b6184c35081 100644
--- a/drivers/crypto/qat/qat_common/adf_admin.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_admin.c
diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c
index fe9bb2f3536a..04af32a2811c 100644
--- a/drivers/crypto/qat/qat_common/adf_aer.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c
@@ -2,7 +2,6 @@
 /* Copyright(c) 2014 - 2020 Intel Corporation */
 #include <linux/kernel.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/completion.h>
 #include <linux/workqueue.h>
 #include <linux/delay.h>
@@ -90,9 +89,7 @@ static void adf_device_reset_worker(struct work_struct *work)
 	struct adf_accel_dev *accel_dev = reset_data->accel_dev;
 
 	adf_dev_restarting_notify(accel_dev);
-	adf_dev_stop(accel_dev);
-	adf_dev_shutdown(accel_dev);
-	if (adf_dev_init(accel_dev) || adf_dev_start(accel_dev)) {
+	if (adf_dev_restart(accel_dev)) {
 		/* The device hanged and we can't restart it so stop here */
 		dev_err(&GET_DEV(accel_dev), "Restart device failed\n");
 		kfree(reset_data);
@@ -173,40 +170,6 @@ const struct pci_error_handlers adf_err_handler = {
 };
 EXPORT_SYMBOL_GPL(adf_err_handler);
 
-/**
- * adf_enable_aer() - Enable Advance Error Reporting for acceleration device
- * @accel_dev:  Pointer to acceleration device.
- *
- * Function enables PCI Advance Error Reporting for the
- * QAT acceleration device accel_dev.
- * To be used by QAT device specific drivers.
- */
-void adf_enable_aer(struct adf_accel_dev *accel_dev)
-{
-	struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
-
-	pci_enable_pcie_error_reporting(pdev);
-}
-EXPORT_SYMBOL_GPL(adf_enable_aer);
-
-/**
- * adf_disable_aer() - Disable Advance Error Reporting for acceleration device
- * @accel_dev:  Pointer to acceleration device.
- *
- * Function disables PCI Advance Error Reporting for the
- * QAT acceleration device accel_dev.
- * To be used by QAT device specific drivers.
- *
- * Return: void
- */
-void adf_disable_aer(struct adf_accel_dev *accel_dev)
-{
-	struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
-
-	pci_disable_pcie_error_reporting(pdev);
-}
-EXPORT_SYMBOL_GPL(adf_disable_aer);
-
 int adf_init_aer(void)
 {
 	device_reset_wq = alloc_workqueue("qat_device_reset_wq",
diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/intel/qat/qat_common/adf_cfg.c
index 1931e5b37f2b..1931e5b37f2b 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.c
diff --git a/drivers/crypto/qat/qat_common/adf_cfg.h b/drivers/crypto/intel/qat/qat_common/adf_cfg.h
index 376cde61a60e..376cde61a60e 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.h
diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h
index 6e5de1dab97b..6e5de1dab97b 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg_common.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h
diff --git a/drivers/crypto/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h
index 5d8c3bdb258c..5d8c3bdb258c 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg_strings.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h
diff --git a/drivers/crypto/qat/qat_common/adf_cfg_user.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_user.h
index 421f4fb8b4dd..421f4fb8b4dd 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg_user.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_user.h
diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h
index 7189265573c0..db79759bee61 100644
--- a/drivers/crypto/qat/qat_common/adf_common_drv.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h
@@ -52,11 +52,9 @@ struct service_hndl {
 int adf_service_register(struct service_hndl *service);
 int adf_service_unregister(struct service_hndl *service);
 
-int adf_dev_init(struct adf_accel_dev *accel_dev);
-int adf_dev_start(struct adf_accel_dev *accel_dev);
-void adf_dev_stop(struct adf_accel_dev *accel_dev);
-void adf_dev_shutdown(struct adf_accel_dev *accel_dev);
-int adf_dev_shutdown_cache_cfg(struct adf_accel_dev *accel_dev);
+int adf_dev_up(struct adf_accel_dev *accel_dev, bool init_config);
+int adf_dev_down(struct adf_accel_dev *accel_dev, bool cache_config);
+int adf_dev_restart(struct adf_accel_dev *accel_dev);
 
 void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data);
 void adf_clean_vf_map(bool);
@@ -88,8 +86,6 @@ int adf_ae_start(struct adf_accel_dev *accel_dev);
 int adf_ae_stop(struct adf_accel_dev *accel_dev);
 
 extern const struct pci_error_handlers adf_err_handler;
-void adf_enable_aer(struct adf_accel_dev *accel_dev);
-void adf_disable_aer(struct adf_accel_dev *accel_dev);
 void adf_reset_sbr(struct adf_accel_dev *accel_dev);
 void adf_reset_flr(struct adf_accel_dev *accel_dev);
 void adf_dev_restore(struct adf_accel_dev *accel_dev);
diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c
index 9190532b27eb..29c4422f243c 100644
--- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_ctl_drv.c
@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
 /* Copyright(c) 2014 - 2020 Intel Corporation */
+
+#include <crypto/algapi.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
@@ -8,7 +10,6 @@
 #include <linux/pci.h>
 #include <linux/cdev.h>
 #include <linux/uaccess.h>
-#include <linux/crypto.h>
 
 #include "adf_accel_devices.h"
 #include "adf_common_drv.h"
@@ -56,7 +57,7 @@ static int adf_chr_drv_create(void)
 		return -EFAULT;
 	}
 
-	adf_ctl_drv.drv_class = class_create(THIS_MODULE, DEVICE_NAME);
+	adf_ctl_drv.drv_class = class_create(DEVICE_NAME);
 	if (IS_ERR(adf_ctl_drv.drv_class)) {
 		pr_err("QAT: class_create failed for adf_ctl\n");
 		goto err_chrdev_unreg;
@@ -243,8 +244,7 @@ static void adf_ctl_stop_devices(u32 id)
 			if (!accel_dev->is_vf)
 				continue;
 
-			adf_dev_stop(accel_dev);
-			adf_dev_shutdown(accel_dev);
+			adf_dev_down(accel_dev, false);
 		}
 	}
 
@@ -253,8 +253,7 @@ static void adf_ctl_stop_devices(u32 id)
 			if (!adf_dev_started(accel_dev))
 				continue;
 
-			adf_dev_stop(accel_dev);
-			adf_dev_shutdown(accel_dev);
+			adf_dev_down(accel_dev, false);
 		}
 	}
 }
@@ -308,23 +307,16 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd,
 	if (!accel_dev)
 		goto out;
 
-	if (!adf_dev_started(accel_dev)) {
-		dev_info(&GET_DEV(accel_dev),
-			 "Starting acceleration device qat_dev%d.\n",
-			 ctl_data->device_id);
-		ret = adf_dev_init(accel_dev);
-		if (!ret)
-			ret = adf_dev_start(accel_dev);
-	} else {
-		dev_info(&GET_DEV(accel_dev),
-			 "Acceleration device qat_dev%d already started.\n",
-			 ctl_data->device_id);
-	}
+	dev_info(&GET_DEV(accel_dev),
+		 "Starting acceleration device qat_dev%d.\n",
+		 ctl_data->device_id);
+
+	ret = adf_dev_up(accel_dev, false);
+
 	if (ret) {
 		dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n",
 			ctl_data->device_id);
-		adf_dev_stop(accel_dev);
-		adf_dev_shutdown(accel_dev);
+		adf_dev_down(accel_dev, false);
 	}
 out:
 	kfree(ctl_data);
diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c
index 4c752eed10fe..86ee36feefad 100644
--- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c
@@ -223,6 +223,7 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev,
 		map->attached = true;
 		list_add_tail(&map->list, &vfs_table);
 	}
+	mutex_init(&accel_dev->state_lock);
 unlock:
 	mutex_unlock(&table_lock);
 	return ret;
@@ -269,6 +270,7 @@ void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev,
 		}
 	}
 unlock:
+	mutex_destroy(&accel_dev->state_lock);
 	list_del(&accel_dev->list);
 	mutex_unlock(&table_lock);
 }
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_config.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c
index eeb30da7587a..eeb30da7587a 100644
--- a/drivers/crypto/qat/qat_common/adf_gen2_config.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_config.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.h
index 4bf9da2de68a..4bf9da2de68a 100644
--- a/drivers/crypto/qat/qat_common/adf_gen2_config.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.h
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_dc.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.c
index 47261b1c1da6..47261b1c1da6 100644
--- a/drivers/crypto/qat/qat_common/adf_gen2_dc.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.c
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_dc.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h
index 6eae023354d7..6eae023354d7 100644
--- a/drivers/crypto/qat/qat_common/adf_gen2_dc.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c
index d1884547b5a1..d1884547b5a1 100644
--- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h
index e4bc07529be4..e4bc07529be4 100644
--- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.c
index 70ef11963938..70ef11963938 100644
--- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.c
diff --git a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h
index a716545a764c..a716545a764c 100644
--- a/drivers/crypto/qat/qat_common/adf_gen2_pfvf.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_dc.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c
index 5859238e37de..5859238e37de 100644
--- a/drivers/crypto/qat/qat_common/adf_gen4_dc.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_dc.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h
index 0b1a6774412e..0b1a6774412e 100644
--- a/drivers/crypto/qat/qat_common/adf_gen4_dc.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c
index 3148a62938fd..3148a62938fd 100644
--- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h
index 4fb4b3df5a18..4fb4b3df5a18 100644
--- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.c
index 8e8efe93f3ee..8e8efe93f3ee 100644
--- a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.c
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h
index 17d1b774d4a8..17d1b774d4a8 100644
--- a/drivers/crypto/qat/qat_common/adf_gen4_pfvf.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pm.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.c
index 7037c0892a8a..7037c0892a8a 100644
--- a/drivers/crypto/qat/qat_common/adf_gen4_pm.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.c
diff --git a/drivers/crypto/qat/qat_common/adf_gen4_pm.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h
index f8f8a9ee29e5..f8f8a9ee29e5 100644
--- a/drivers/crypto/qat/qat_common/adf_gen4_pm.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h
diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c
index 64e4596a24f4..da6956699246 100644
--- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c
@@ -36,7 +36,7 @@ int adf_init_arb(struct adf_accel_dev *accel_dev)
 		WRITE_CSR_ARB_SARCONFIG(csr, arb_off, arb, arb_cfg);
 
 	/* Map worker threads to service arbiters */
-	thd_2_arb_cfg = hw_data->get_arb_mapping();
+	thd_2_arb_cfg = hw_data->get_arb_mapping(accel_dev);
 
 	for_each_set_bit(i, &ae_mask, hw_data->num_engines)
 		WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, thd_2_arb_cfg[i]);
diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/intel/qat/qat_common/adf_init.c
index cef7bb8ec007..0985f64ab11a 100644
--- a/drivers/crypto/qat/qat_common/adf_init.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_init.c
@@ -56,7 +56,7 @@ int adf_service_unregister(struct service_hndl *service)
  *
  * Return: 0 on success, error code otherwise.
  */
-int adf_dev_init(struct adf_accel_dev *accel_dev)
+static int adf_dev_init(struct adf_accel_dev *accel_dev)
 {
 	struct service_hndl *service;
 	struct list_head *list_itr;
@@ -146,7 +146,6 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(adf_dev_init);
 
 /**
  * adf_dev_start() - Start acceleration service for the given accel device
@@ -158,7 +157,7 @@ EXPORT_SYMBOL_GPL(adf_dev_init);
  *
  * Return: 0 on success, error code otherwise.
  */
-int adf_dev_start(struct adf_accel_dev *accel_dev)
+static int adf_dev_start(struct adf_accel_dev *accel_dev)
 {
 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 	struct service_hndl *service;
@@ -219,7 +218,6 @@ int adf_dev_start(struct adf_accel_dev *accel_dev)
 	}
 	return 0;
 }
-EXPORT_SYMBOL_GPL(adf_dev_start);
 
 /**
  * adf_dev_stop() - Stop acceleration service for the given accel device
@@ -231,7 +229,7 @@ EXPORT_SYMBOL_GPL(adf_dev_start);
  *
  * Return: void
  */
-void adf_dev_stop(struct adf_accel_dev *accel_dev)
+static void adf_dev_stop(struct adf_accel_dev *accel_dev)
 {
 	struct service_hndl *service;
 	struct list_head *list_itr;
@@ -276,7 +274,6 @@ void adf_dev_stop(struct adf_accel_dev *accel_dev)
 			clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
 	}
 }
-EXPORT_SYMBOL_GPL(adf_dev_stop);
 
 /**
  * adf_dev_shutdown() - shutdown acceleration services and data strucutures
@@ -285,7 +282,7 @@ EXPORT_SYMBOL_GPL(adf_dev_stop);
  * Cleanup the ring data structures and the admin comms and arbitration
  * services.
  */
-void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
+static void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
 {
 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 	struct service_hndl *service;
@@ -343,7 +340,6 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
 	adf_cleanup_etr_data(accel_dev);
 	adf_dev_restore(accel_dev);
 }
-EXPORT_SYMBOL_GPL(adf_dev_shutdown);
 
 int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev)
 {
@@ -375,7 +371,7 @@ int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev)
 	return 0;
 }
 
-int adf_dev_shutdown_cache_cfg(struct adf_accel_dev *accel_dev)
+static int adf_dev_shutdown_cache_cfg(struct adf_accel_dev *accel_dev)
 {
 	char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0};
 	int ret;
@@ -400,3 +396,85 @@ int adf_dev_shutdown_cache_cfg(struct adf_accel_dev *accel_dev)
 
 	return 0;
 }
+
+int adf_dev_down(struct adf_accel_dev *accel_dev, bool reconfig)
+{
+	int ret = 0;
+
+	if (!accel_dev)
+		return -EINVAL;
+
+	mutex_lock(&accel_dev->state_lock);
+
+	if (!adf_dev_started(accel_dev)) {
+		dev_info(&GET_DEV(accel_dev), "Device qat_dev%d already down\n",
+			 accel_dev->accel_id);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (reconfig) {
+		ret = adf_dev_shutdown_cache_cfg(accel_dev);
+		goto out;
+	}
+
+	adf_dev_stop(accel_dev);
+	adf_dev_shutdown(accel_dev);
+
+out:
+	mutex_unlock(&accel_dev->state_lock);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(adf_dev_down);
+
+int adf_dev_up(struct adf_accel_dev *accel_dev, bool config)
+{
+	int ret = 0;
+
+	if (!accel_dev)
+		return -EINVAL;
+
+	mutex_lock(&accel_dev->state_lock);
+
+	if (adf_dev_started(accel_dev)) {
+		dev_info(&GET_DEV(accel_dev), "Device qat_dev%d already up\n",
+			 accel_dev->accel_id);
+		ret = -EALREADY;
+		goto out;
+	}
+
+	if (config && GET_HW_DATA(accel_dev)->dev_config) {
+		ret = GET_HW_DATA(accel_dev)->dev_config(accel_dev);
+		if (unlikely(ret))
+			goto out;
+	}
+
+	ret = adf_dev_init(accel_dev);
+	if (unlikely(ret))
+		goto out;
+
+	ret = adf_dev_start(accel_dev);
+
+out:
+	mutex_unlock(&accel_dev->state_lock);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(adf_dev_up);
+
+int adf_dev_restart(struct adf_accel_dev *accel_dev)
+{
+	int ret = 0;
+
+	if (!accel_dev)
+		return -EFAULT;
+
+	adf_dev_down(accel_dev, false);
+
+	ret = adf_dev_up(accel_dev, false);
+	/* if device is already up return success*/
+	if (ret == -EALREADY)
+		return 0;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(adf_dev_restart);
diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_isr.c
index ad9e135b8560..ad9e135b8560 100644
--- a/drivers/crypto/qat/qat_common/adf_isr.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_isr.c
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_msg.h
index 204a42438992..204a42438992 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_msg.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_msg.h
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.c
index 14c069f0d71a..14c069f0d71a 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.c
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.h
index e8982d1ac896..e8982d1ac896 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.h
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c
index 388e58bcbcaf..388e58bcbcaf 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.h
index 165d266d023d..165d266d023d 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.h
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.c
index c5f6d77d4bb8..c5f6d77d4bb8 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_utils.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.c
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_utils.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.h
index 2be048e2287b..2be048e2287b 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_utils.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.h
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.c
index 1141258db4b6..1141258db4b6 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.c
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.h
index 71bc0e3f1d93..71bc0e3f1d93 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_msg.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.h
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.c
index 1015155b6374..1015155b6374 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.c
diff --git a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.h
index f6ee9b38c0e1..f6ee9b38c0e1 100644
--- a/drivers/crypto/qat/qat_common/adf_pfvf_vf_proto.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.h
diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/intel/qat/qat_common/adf_sriov.c
index d85a90cc387b..f44025bb6f99 100644
--- a/drivers/crypto/qat/qat_common/adf_sriov.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_sriov.c
@@ -159,7 +159,7 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs)
 			return -EBUSY;
 		}
 
-		ret = adf_dev_shutdown_cache_cfg(accel_dev);
+		ret = adf_dev_down(accel_dev, true);
 		if (ret)
 			return ret;
 	}
@@ -184,13 +184,7 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs)
 	if (!accel_dev->pf.vf_info)
 		return -ENOMEM;
 
-	if (adf_dev_init(accel_dev)) {
-		dev_err(&GET_DEV(accel_dev), "Failed to init qat_dev%d\n",
-			accel_dev->accel_id);
-		return -EFAULT;
-	}
-
-	if (adf_dev_start(accel_dev)) {
+	if (adf_dev_up(accel_dev, false)) {
 		dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n",
 			accel_dev->accel_id);
 		return -EFAULT;
diff --git a/drivers/crypto/qat/qat_common/adf_sysfs.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c
index e8b078e719c2..3eb6611ab1b1 100644
--- a/drivers/crypto/qat/qat_common/adf_sysfs.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c
@@ -50,38 +50,21 @@ static ssize_t state_store(struct device *dev, struct device_attribute *attr,
 
 	switch (ret) {
 	case DEV_DOWN:
-		if (!adf_dev_started(accel_dev)) {
-			dev_info(dev, "Device qat_dev%d already down\n",
-				 accel_id);
-			return -EINVAL;
-		}
-
 		dev_info(dev, "Stopping device qat_dev%d\n", accel_id);
 
-		ret = adf_dev_shutdown_cache_cfg(accel_dev);
+		ret = adf_dev_down(accel_dev, true);
 		if (ret < 0)
 			return -EINVAL;
 
 		break;
 	case DEV_UP:
-		if (adf_dev_started(accel_dev)) {
-			dev_info(dev, "Device qat_dev%d already up\n",
-				 accel_id);
-			return -EINVAL;
-		}
-
 		dev_info(dev, "Starting device qat_dev%d\n", accel_id);
 
-		ret = GET_HW_DATA(accel_dev)->dev_config(accel_dev);
-		if (!ret)
-			ret = adf_dev_init(accel_dev);
-		if (!ret)
-			ret = adf_dev_start(accel_dev);
-
+		ret = adf_dev_up(accel_dev, true);
 		if (ret < 0) {
 			dev_err(dev, "Failed to start device qat_dev%d\n",
 				accel_id);
-			adf_dev_shutdown_cache_cfg(accel_dev);
+			adf_dev_down(accel_dev, true);
 			return ret;
 		}
 		break;
diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/intel/qat/qat_common/adf_transport.c
index 630d0483c4e0..630d0483c4e0 100644
--- a/drivers/crypto/qat/qat_common/adf_transport.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_transport.c
diff --git a/drivers/crypto/qat/qat_common/adf_transport.h b/drivers/crypto/intel/qat/qat_common/adf_transport.h
index e6ef6f9b7691..e6ef6f9b7691 100644
--- a/drivers/crypto/qat/qat_common/adf_transport.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_transport.h
diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/intel/qat/qat_common/adf_transport_access_macros.h
index d3667dbd9826..d3667dbd9826 100644
--- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_transport_access_macros.h
diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c
index 08bca1c506c0..08bca1c506c0 100644
--- a/drivers/crypto/qat/qat_common/adf_transport_debug.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c
diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/intel/qat/qat_common/adf_transport_internal.h
index 8b2c92ba7ca1..8b2c92ba7ca1 100644
--- a/drivers/crypto/qat/qat_common/adf_transport_internal.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_transport_internal.h
diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c
index 8c95fcd8e64b..b05c3957a160 100644
--- a/drivers/crypto/qat/qat_common/adf_vf_isr.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c
@@ -71,8 +71,7 @@ static void adf_dev_stop_async(struct work_struct *work)
 	struct adf_accel_dev *accel_dev = stop_data->accel_dev;
 
 	adf_dev_restarting_notify(accel_dev);
-	adf_dev_stop(accel_dev);
-	adf_dev_shutdown(accel_dev);
+	adf_dev_down(accel_dev, false);
 
 	/* Re-enable PF2VF interrupts */
 	adf_enable_pf2vf_interrupts(accel_dev);
diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw.h
index c141160421e1..c141160421e1 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_fw.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw.h
diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_comp.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h
index a03d43fef2b3..a03d43fef2b3 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_fw_comp.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h
diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h
index 56cb827f93ea..56cb827f93ea 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h
diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_la.h
index 28fa17f14be4..28fa17f14be4 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_la.h
diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h
index 7eb5daef4f88..7eb5daef4f88 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h
diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_pke.h
index 9dddae0009fc..9dddae0009fc 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_pke.h
diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hal.h
index 20b2ee1fc65a..20b2ee1fc65a 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_hal.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hal.h
diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h
index 4042739bb6fa..4042739bb6fa 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_hw.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h
diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw_20_comp.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp.h
index 7ea8962272f2..7ea8962272f2 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_hw_20_comp.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp.h
diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw_20_comp_defs.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp_defs.h
index 208d4554283b..208d4554283b 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_hw_20_comp_defs.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp_defs.h
diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h
index 69482abdb8b9..69482abdb8b9 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/intel/qat/qat_common/qat_algs.c
index 538dcbfbcd26..538dcbfbcd26 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_algs.c
diff --git a/drivers/crypto/qat/qat_common/qat_algs_send.c b/drivers/crypto/intel/qat/qat_common/qat_algs_send.c
index bb80455b3e81..bb80455b3e81 100644
--- a/drivers/crypto/qat/qat_common/qat_algs_send.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_algs_send.c
diff --git a/drivers/crypto/qat/qat_common/qat_algs_send.h b/drivers/crypto/intel/qat/qat_common/qat_algs_send.h
index 0baca16e1eff..0baca16e1eff 100644
--- a/drivers/crypto/qat/qat_common/qat_algs_send.h
+++ b/drivers/crypto/intel/qat/qat_common/qat_algs_send.h
diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c
index 935a7e012946..935a7e012946 100644
--- a/drivers/crypto/qat/qat_common/qat_asym_algs.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c
diff --git a/drivers/crypto/qat/qat_common/qat_bl.c b/drivers/crypto/intel/qat/qat_common/qat_bl.c
index 76baed0a76c0..76baed0a76c0 100644
--- a/drivers/crypto/qat/qat_common/qat_bl.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_bl.c
diff --git a/drivers/crypto/qat/qat_common/qat_bl.h b/drivers/crypto/intel/qat/qat_common/qat_bl.h
index d87e4f35ac39..d87e4f35ac39 100644
--- a/drivers/crypto/qat/qat_common/qat_bl.h
+++ b/drivers/crypto/intel/qat/qat_common/qat_bl.h
diff --git a/drivers/crypto/qat/qat_common/qat_comp_algs.c b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c
index b533984906ec..b533984906ec 100644
--- a/drivers/crypto/qat/qat_common/qat_comp_algs.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c
diff --git a/drivers/crypto/qat/qat_common/qat_comp_req.h b/drivers/crypto/intel/qat/qat_common/qat_comp_req.h
index 404e32c5e778..404e32c5e778 100644
--- a/drivers/crypto/qat/qat_common/qat_comp_req.h
+++ b/drivers/crypto/intel/qat/qat_common/qat_comp_req.h
diff --git a/drivers/crypto/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c
index 3f1f35283266..3f1f35283266 100644
--- a/drivers/crypto/qat/qat_common/qat_compression.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c
diff --git a/drivers/crypto/qat/qat_common/qat_compression.h b/drivers/crypto/intel/qat/qat_common/qat_compression.h
index aebac2302dcf..aebac2302dcf 100644
--- a/drivers/crypto/qat/qat_common/qat_compression.h
+++ b/drivers/crypto/intel/qat/qat_common/qat_compression.h
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/intel/qat/qat_common/qat_crypto.c
index 40c8e74d1cf9..40c8e74d1cf9 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_crypto.c
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/intel/qat/qat_common/qat_crypto.h
index 6a0e961bb9dc..6a0e961bb9dc 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.h
+++ b/drivers/crypto/intel/qat/qat_common/qat_crypto.h
diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c
index 7bba35280dac..cbb946a80076 100644
--- a/drivers/crypto/qat/qat_common/qat_hal.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c
@@ -696,6 +696,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
 	switch (handle->pci_dev->device) {
 	case ADF_4XXX_PCI_DEVICE_ID:
 	case ADF_401XX_PCI_DEVICE_ID:
+	case ADF_402XX_PCI_DEVICE_ID:
 		handle->chip_info->mmp_sram_size = 0;
 		handle->chip_info->nn = false;
 		handle->chip_info->lm2lm3 = true;
diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c
index b7f7869ef8b2..3ba8ca20b3d7 100644
--- a/drivers/crypto/qat/qat_common/qat_uclo.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c
@@ -732,6 +732,7 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle)
 		return ICP_QAT_AC_C3XXX_DEV_TYPE;
 	case ADF_4XXX_PCI_DEVICE_ID:
 	case ADF_401XX_PCI_DEVICE_ID:
+	case ADF_402XX_PCI_DEVICE_ID:
 		return ICP_QAT_AC_4XXX_A_DEV_TYPE;
 	default:
 		pr_err("QAT: unsupported device 0x%x\n",
diff --git a/drivers/crypto/qat/qat_dh895xcc/Makefile b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile
index 38d6f8e1624a..38d6f8e1624a 100644
--- a/drivers/crypto/qat/qat_dh895xcc/Makefile
+++ b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
index bc80bb475118..1ebe0b351fae 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
+++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
@@ -106,7 +106,7 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
 	return DEV_SKU_UNKNOWN;
 }
 
-static const u32 *adf_get_arbiter_mapping(void)
+static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev)
 {
 	return thrd_to_arb_map;
 }
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h
index 7b674bbe4192..7b674bbe4192 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h
+++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c
index ebeb17b67fcd..e18860ab5c8e 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
+++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c
@@ -193,34 +193,20 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 	pci_set_master(pdev);
 
-	adf_enable_aer(accel_dev);
-
 	if (pci_save_state(pdev)) {
 		dev_err(&pdev->dev, "Failed to save pci state\n");
 		ret = -ENOMEM;
-		goto out_err_disable_aer;
+		goto out_err_free_reg;
 	}
 
-	ret = hw_data->dev_config(accel_dev);
-	if (ret)
-		goto out_err_disable_aer;
-
-	ret = adf_dev_init(accel_dev);
-	if (ret)
-		goto out_err_dev_shutdown;
-
-	ret = adf_dev_start(accel_dev);
+	ret = adf_dev_up(accel_dev, true);
 	if (ret)
 		goto out_err_dev_stop;
 
 	return ret;
 
 out_err_dev_stop:
-	adf_dev_stop(accel_dev);
-out_err_dev_shutdown:
-	adf_dev_shutdown(accel_dev);
-out_err_disable_aer:
-	adf_disable_aer(accel_dev);
+	adf_dev_down(accel_dev, false);
 out_err_free_reg:
 	pci_release_regions(accel_pci_dev->pci_dev);
 out_err_disable:
@@ -239,9 +225,7 @@ static void adf_remove(struct pci_dev *pdev)
 		pr_err("QAT: Driver removal failed\n");
 		return;
 	}
-	adf_dev_stop(accel_dev);
-	adf_dev_shutdown(accel_dev);
-	adf_disable_aer(accel_dev);
+	adf_dev_down(accel_dev, false);
 	adf_cleanup_accel(accel_dev);
 	adf_cleanup_pci_dev(accel_dev);
 	kfree(accel_dev);
diff --git a/drivers/crypto/qat/qat_dh895xccvf/Makefile b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile
index 0153c85ce743..0153c85ce743 100644
--- a/drivers/crypto/qat/qat_dh895xccvf/Makefile
+++ b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile
diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c
index 70e56cc16ece..70e56cc16ece 100644
--- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c
+++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c
diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h
index 6973fa967bc8..6973fa967bc8 100644
--- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h
+++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h
diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c
index c1485e702b3e..96854a1cd87e 100644
--- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c
+++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c
@@ -173,20 +173,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* Completion for VF2PF request/response message exchange */
 	init_completion(&accel_dev->vf.msg_received);
 
-	ret = adf_dev_init(accel_dev);
-	if (ret)
-		goto out_err_dev_shutdown;
-
-	ret = adf_dev_start(accel_dev);
+	ret = adf_dev_up(accel_dev, false);
 	if (ret)
 		goto out_err_dev_stop;
 
 	return ret;
 
 out_err_dev_stop:
-	adf_dev_stop(accel_dev);
-out_err_dev_shutdown:
-	adf_dev_shutdown(accel_dev);
+	adf_dev_down(accel_dev, false);
 out_err_free_reg:
 	pci_release_regions(accel_pci_dev->pci_dev);
 out_err_disable:
@@ -206,8 +200,7 @@ static void adf_remove(struct pci_dev *pdev)
 		return;
 	}
 	adf_flush_vf_wq(accel_dev);
-	adf_dev_stop(accel_dev);
-	adf_dev_shutdown(accel_dev);
+	adf_dev_down(accel_dev, false);
 	adf_cleanup_accel(accel_dev);
 	adf_cleanup_pci_dev(accel_dev);
 	kfree(accel_dev);
diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c
index 1c11946a4f0b..f6b7bce0e656 100644
--- a/drivers/crypto/mxs-dcp.c
+++ b/drivers/crypto/mxs-dcp.c
@@ -1022,21 +1022,15 @@ static int mxs_dcp_probe(struct platform_device *pdev)
 	sdcp->coh = PTR_ALIGN(sdcp->coh, DCP_ALIGNMENT);
 
 	/* DCP clock is optional, only used on some SOCs */
-	sdcp->dcp_clk = devm_clk_get(dev, "dcp");
-	if (IS_ERR(sdcp->dcp_clk)) {
-		if (sdcp->dcp_clk != ERR_PTR(-ENOENT))
-			return PTR_ERR(sdcp->dcp_clk);
-		sdcp->dcp_clk = NULL;
-	}
-	ret = clk_prepare_enable(sdcp->dcp_clk);
-	if (ret)
-		return ret;
+	sdcp->dcp_clk = devm_clk_get_optional_enabled(dev, "dcp");
+	if (IS_ERR(sdcp->dcp_clk))
+		return PTR_ERR(sdcp->dcp_clk);
 
 	/* Restart the DCP block. */
 	ret = stmp_reset_block(sdcp->base);
 	if (ret) {
 		dev_err(dev, "Failed reset\n");
-		goto err_disable_unprepare_clk;
+		return ret;
 	}
 
 	/* Initialize control register. */
@@ -1076,7 +1070,7 @@ static int mxs_dcp_probe(struct platform_device *pdev)
 	if (IS_ERR(sdcp->thread[DCP_CHAN_HASH_SHA])) {
 		dev_err(dev, "Error starting SHA thread!\n");
 		ret = PTR_ERR(sdcp->thread[DCP_CHAN_HASH_SHA]);
-		goto err_disable_unprepare_clk;
+		return ret;
 	}
 
 	sdcp->thread[DCP_CHAN_CRYPTO] = kthread_run(dcp_chan_thread_aes,
@@ -1134,9 +1128,6 @@ err_destroy_aes_thread:
 err_destroy_sha_thread:
 	kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]);
 
-err_disable_unprepare_clk:
-	clk_disable_unprepare(sdcp->dcp_clk);
-
 	return ret;
 }
 
@@ -1156,8 +1147,6 @@ static int mxs_dcp_remove(struct platform_device *pdev)
 	kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]);
 	kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]);
 
-	clk_disable_unprepare(sdcp->dcp_clk);
-
 	platform_set_drvdata(pdev, NULL);
 
 	global_sdcp = NULL;
diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c
index 74deca4f96e0..fce49c0dee3e 100644
--- a/drivers/crypto/qce/core.c
+++ b/drivers/crypto/qce/core.c
@@ -5,6 +5,7 @@
 
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
+#include <linux/interconnect.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
@@ -22,6 +23,8 @@
 #define QCE_MAJOR_VERSION5	0x05
 #define QCE_QUEUE_LENGTH	1
 
+#define QCE_DEFAULT_MEM_BANDWIDTH	393600
+
 static const struct qce_algo_ops *qce_ops[] = {
 #ifdef CONFIG_CRYPTO_DEV_QCE_SKCIPHER
 	&skcipher_ops,
@@ -206,22 +209,30 @@ static int qce_crypto_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
-	qce->core = devm_clk_get(qce->dev, "core");
+	qce->core = devm_clk_get_optional(qce->dev, "core");
 	if (IS_ERR(qce->core))
 		return PTR_ERR(qce->core);
 
-	qce->iface = devm_clk_get(qce->dev, "iface");
+	qce->iface = devm_clk_get_optional(qce->dev, "iface");
 	if (IS_ERR(qce->iface))
 		return PTR_ERR(qce->iface);
 
-	qce->bus = devm_clk_get(qce->dev, "bus");
+	qce->bus = devm_clk_get_optional(qce->dev, "bus");
 	if (IS_ERR(qce->bus))
 		return PTR_ERR(qce->bus);
 
-	ret = clk_prepare_enable(qce->core);
+	qce->mem_path = devm_of_icc_get(qce->dev, "memory");
+	if (IS_ERR(qce->mem_path))
+		return PTR_ERR(qce->mem_path);
+
+	ret = icc_set_bw(qce->mem_path, QCE_DEFAULT_MEM_BANDWIDTH, QCE_DEFAULT_MEM_BANDWIDTH);
 	if (ret)
 		return ret;
 
+	ret = clk_prepare_enable(qce->core);
+	if (ret)
+		goto err_mem_path_disable;
+
 	ret = clk_prepare_enable(qce->iface);
 	if (ret)
 		goto err_clks_core;
@@ -260,6 +271,9 @@ err_clks_iface:
 	clk_disable_unprepare(qce->iface);
 err_clks_core:
 	clk_disable_unprepare(qce->core);
+err_mem_path_disable:
+	icc_set_bw(qce->mem_path, 0, 0);
+
 	return ret;
 }
 
@@ -279,6 +293,7 @@ static int qce_crypto_remove(struct platform_device *pdev)
 static const struct of_device_id qce_crypto_of_match[] = {
 	{ .compatible = "qcom,crypto-v5.1", },
 	{ .compatible = "qcom,crypto-v5.4", },
+	{ .compatible = "qcom,qce", },
 	{}
 };
 MODULE_DEVICE_TABLE(of, qce_crypto_of_match);
diff --git a/drivers/crypto/qce/core.h b/drivers/crypto/qce/core.h
index 085774cdf641..228fcd69ec51 100644
--- a/drivers/crypto/qce/core.h
+++ b/drivers/crypto/qce/core.h
@@ -35,6 +35,7 @@ struct qce_device {
 	void __iomem *base;
 	struct device *dev;
 	struct clk *core, *iface, *bus;
+	struct icc_path *mem_path;
 	struct qce_dma_data dma;
 	int burst_size;
 	unsigned int pipe_pair_id;
diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c
index f4bc06c24ad8..df5f9d675c57 100644
--- a/drivers/crypto/sa2ul.c
+++ b/drivers/crypto/sa2ul.c
@@ -1037,7 +1037,7 @@ static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
 
 static void sa_aes_dma_in_callback(void *data)
 {
-	struct sa_rx_data *rxd = (struct sa_rx_data *)data;
+	struct sa_rx_data *rxd = data;
 	struct skcipher_request *req;
 	u32 *result;
 	__be32 *mdptr;
@@ -1351,7 +1351,7 @@ static int sa_decrypt(struct skcipher_request *req)
 
 static void sa_sha_dma_in_callback(void *data)
 {
-	struct sa_rx_data *rxd = (struct sa_rx_data *)data;
+	struct sa_rx_data *rxd = data;
 	struct ahash_request *req;
 	struct crypto_ahash *tfm;
 	unsigned int authsize;
@@ -1689,7 +1689,7 @@ static void sa_sha_cra_exit(struct crypto_tfm *tfm)
 
 static void sa_aead_dma_in_callback(void *data)
 {
-	struct sa_rx_data *rxd = (struct sa_rx_data *)data;
+	struct sa_rx_data *rxd = data;
 	struct aead_request *req;
 	struct crypto_aead *tfm;
 	unsigned int start;
diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c
index dd4c703cd855..4c799df3e883 100644
--- a/drivers/crypto/sahara.c
+++ b/drivers/crypto/sahara.c
@@ -1035,7 +1035,7 @@ static int sahara_sha_process(struct ahash_request *req)
 
 static int sahara_queue_manage(void *data)
 {
-	struct sahara_dev *dev = (struct sahara_dev *)data;
+	struct sahara_dev *dev = data;
 	struct crypto_async_request *async_req;
 	struct crypto_async_request *backlog;
 	int ret = 0;
@@ -1270,7 +1270,7 @@ static struct ahash_alg sha_v4_algs[] = {
 
 static irqreturn_t sahara_irq_handler(int irq, void *data)
 {
-	struct sahara_dev *dev = (struct sahara_dev *)data;
+	struct sahara_dev *dev = data;
 	unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
 	unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
 
diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c
index 7bf805563ac2..f0df32382719 100644
--- a/drivers/crypto/stm32/stm32-hash.c
+++ b/drivers/crypto/stm32/stm32-hash.c
@@ -7,7 +7,6 @@
  */
 
 #include <linux/clk.h>
-#include <linux/crypto.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmaengine.h>
@@ -68,7 +67,7 @@
 #define HASH_MASK_DATA_INPUT		BIT(1)
 
 /* Context swap register */
-#define HASH_CSR_REGISTER_NUMBER	53
+#define HASH_CSR_REGISTER_NUMBER	54
 
 /* Status Flags */
 #define HASH_SR_DATA_INPUT_READY	BIT(0)
@@ -96,7 +95,7 @@
 #define HASH_FLAGS_SHA1			BIT(19)
 #define HASH_FLAGS_SHA224		BIT(20)
 #define HASH_FLAGS_SHA256		BIT(21)
-#define HASH_FLAGS_ERRORS		BIT(22)
+#define HASH_FLAGS_EMPTY		BIT(22)
 #define HASH_FLAGS_HMAC			BIT(23)
 
 #define HASH_OP_UPDATE			1
@@ -127,15 +126,24 @@ struct stm32_hash_ctx {
 	int			keylen;
 };
 
+struct stm32_hash_state {
+	u32			flags;
+
+	u16			bufcnt;
+	u16			buflen;
+
+	u8 buffer[HASH_BUFLEN] __aligned(4);
+
+	/* hash state */
+	u32			hw_context[3 + HASH_CSR_REGISTER_NUMBER];
+};
+
 struct stm32_hash_request_ctx {
 	struct stm32_hash_dev	*hdev;
-	unsigned long		flags;
 	unsigned long		op;
 
 	u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));
 	size_t			digcnt;
-	size_t			bufcnt;
-	size_t			buflen;
 
 	/* DMA */
 	struct scatterlist	*sg;
@@ -149,10 +157,7 @@ struct stm32_hash_request_ctx {
 
 	u8			data_type;
 
-	u8 buffer[HASH_BUFLEN] __aligned(sizeof(u32));
-
-	/* Export Context */
-	u32			*hw_context;
+	struct stm32_hash_state state;
 };
 
 struct stm32_hash_algs_info {
@@ -183,7 +188,6 @@ struct stm32_hash_dev {
 	struct ahash_request	*req;
 	struct crypto_engine	*engine;
 
-	int			err;
 	unsigned long		flags;
 
 	struct dma_chan		*dma_lch;
@@ -270,11 +274,12 @@ static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev, int bufcnt)
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct stm32_hash_state *state = &rctx->state;
 
 	u32 reg = HASH_CR_INIT;
 
 	if (!(hdev->flags & HASH_FLAGS_INIT)) {
-		switch (rctx->flags & HASH_FLAGS_ALGO_MASK) {
+		switch (state->flags & HASH_FLAGS_ALGO_MASK) {
 		case HASH_FLAGS_MD5:
 			reg |= HASH_CR_ALGO_MD5;
 			break;
@@ -299,20 +304,13 @@ static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev, int bufcnt)
 
 		reg |= (rctx->data_type << HASH_CR_DATATYPE_POS);
 
-		if (rctx->flags & HASH_FLAGS_HMAC) {
+		if (state->flags & HASH_FLAGS_HMAC) {
 			hdev->flags |= HASH_FLAGS_HMAC;
 			reg |= HASH_CR_MODE;
 			if (ctx->keylen > HASH_LONG_KEY)
 				reg |= HASH_CR_LKEY;
 		}
 
-		/*
-		 * On the Ux500 we need to set a special flag to indicate that
-		 * the message is zero length.
-		 */
-		if (hdev->pdata->ux500 && bufcnt == 0)
-			reg |= HASH_CR_UX500_EMPTYMSG;
-
 		if (!hdev->polled)
 			stm32_hash_write(hdev, HASH_IMR, HASH_DCIE);
 
@@ -326,11 +324,12 @@ static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev, int bufcnt)
 
 static void stm32_hash_append_sg(struct stm32_hash_request_ctx *rctx)
 {
+	struct stm32_hash_state *state = &rctx->state;
 	size_t count;
 
-	while ((rctx->bufcnt < rctx->buflen) && rctx->total) {
+	while ((state->bufcnt < state->buflen) && rctx->total) {
 		count = min(rctx->sg->length - rctx->offset, rctx->total);
-		count = min(count, rctx->buflen - rctx->bufcnt);
+		count = min_t(size_t, count, state->buflen - state->bufcnt);
 
 		if (count <= 0) {
 			if ((rctx->sg->length == 0) && !sg_is_last(rctx->sg)) {
@@ -341,10 +340,10 @@ static void stm32_hash_append_sg(struct stm32_hash_request_ctx *rctx)
 			}
 		}
 
-		scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, rctx->sg,
-					 rctx->offset, count, 0);
+		scatterwalk_map_and_copy(state->buffer + state->bufcnt,
+					 rctx->sg, rctx->offset, count, 0);
 
-		rctx->bufcnt += count;
+		state->bufcnt += count;
 		rctx->offset += count;
 		rctx->total -= count;
 
@@ -361,13 +360,23 @@ static void stm32_hash_append_sg(struct stm32_hash_request_ctx *rctx)
 static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev,
 			       const u8 *buf, size_t length, int final)
 {
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
+	struct stm32_hash_state *state = &rctx->state;
 	unsigned int count, len32;
 	const u32 *buffer = (const u32 *)buf;
 	u32 reg;
 
-	if (final)
+	if (final) {
 		hdev->flags |= HASH_FLAGS_FINAL;
 
+		/* Do not process empty messages if hw is buggy. */
+		if (!(hdev->flags & HASH_FLAGS_INIT) && !length &&
+		    hdev->pdata->broken_emptymsg) {
+			state->flags |= HASH_FLAGS_EMPTY;
+			return 0;
+		}
+	}
+
 	len32 = DIV_ROUND_UP(length, sizeof(u32));
 
 	dev_dbg(hdev->dev, "%s: length: %zd, final: %x len32 %i\n",
@@ -413,36 +422,48 @@ static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev,
 static int stm32_hash_update_cpu(struct stm32_hash_dev *hdev)
 {
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
+	struct stm32_hash_state *state = &rctx->state;
+	u32 *preg = state->hw_context;
 	int bufcnt, err = 0, final;
+	int i;
 
-	dev_dbg(hdev->dev, "%s flags %lx\n", __func__, rctx->flags);
+	dev_dbg(hdev->dev, "%s flags %x\n", __func__, state->flags);
 
-	final = (rctx->flags & HASH_FLAGS_FINUP);
+	final = state->flags & HASH_FLAGS_FINAL;
 
-	while ((rctx->total >= rctx->buflen) ||
-	       (rctx->bufcnt + rctx->total >= rctx->buflen)) {
+	while ((rctx->total >= state->buflen) ||
+	       (state->bufcnt + rctx->total >= state->buflen)) {
 		stm32_hash_append_sg(rctx);
-		bufcnt = rctx->bufcnt;
-		rctx->bufcnt = 0;
-		err = stm32_hash_xmit_cpu(hdev, rctx->buffer, bufcnt, 0);
+		bufcnt = state->bufcnt;
+		state->bufcnt = 0;
+		err = stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 0);
+		if (err)
+			return err;
 	}
 
 	stm32_hash_append_sg(rctx);
 
 	if (final) {
-		bufcnt = rctx->bufcnt;
-		rctx->bufcnt = 0;
-		err = stm32_hash_xmit_cpu(hdev, rctx->buffer, bufcnt, 1);
-
-		/* If we have an IRQ, wait for that, else poll for completion */
-		if (hdev->polled) {
-			if (stm32_hash_wait_busy(hdev))
-				return -ETIMEDOUT;
-			hdev->flags |= HASH_FLAGS_OUTPUT_READY;
-			err = 0;
-		}
+		bufcnt = state->bufcnt;
+		state->bufcnt = 0;
+		return stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 1);
 	}
 
+	if (!(hdev->flags & HASH_FLAGS_INIT))
+		return 0;
+
+	if (stm32_hash_wait_busy(hdev))
+		return -ETIMEDOUT;
+
+	if (!hdev->pdata->ux500)
+		*preg++ = stm32_hash_read(hdev, HASH_IMR);
+	*preg++ = stm32_hash_read(hdev, HASH_STR);
+	*preg++ = stm32_hash_read(hdev, HASH_CR);
+	for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++)
+		*preg++ = stm32_hash_read(hdev, HASH_CSR(i));
+
+	state->flags |= HASH_FLAGS_INIT;
+
 	return err;
 }
 
@@ -584,10 +605,10 @@ static int stm32_hash_dma_init(struct stm32_hash_dev *hdev)
 static int stm32_hash_dma_send(struct stm32_hash_dev *hdev)
 {
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
+	u32 *buffer = (void *)rctx->state.buffer;
 	struct scatterlist sg[1], *tsg;
 	int err = 0, len = 0, reg, ncp = 0;
 	unsigned int i;
-	u32 *buffer = (void *)rctx->buffer;
 
 	rctx->sg = hdev->req->src;
 	rctx->total = hdev->req->nbytes;
@@ -615,7 +636,7 @@ static int stm32_hash_dma_send(struct stm32_hash_dev *hdev)
 
 				ncp = sg_pcopy_to_buffer(
 					rctx->sg, rctx->nents,
-					rctx->buffer, sg->length - len,
+					rctx->state.buffer, sg->length - len,
 					rctx->total - sg->length + len);
 
 				sg->length = len;
@@ -726,47 +747,52 @@ static int stm32_hash_init(struct ahash_request *req)
 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
+	struct stm32_hash_state *state = &rctx->state;
 
 	rctx->hdev = hdev;
 
-	rctx->flags = HASH_FLAGS_CPU;
+	state->flags = HASH_FLAGS_CPU;
 
 	rctx->digcnt = crypto_ahash_digestsize(tfm);
 	switch (rctx->digcnt) {
 	case MD5_DIGEST_SIZE:
-		rctx->flags |= HASH_FLAGS_MD5;
+		state->flags |= HASH_FLAGS_MD5;
 		break;
 	case SHA1_DIGEST_SIZE:
-		rctx->flags |= HASH_FLAGS_SHA1;
+		state->flags |= HASH_FLAGS_SHA1;
 		break;
 	case SHA224_DIGEST_SIZE:
-		rctx->flags |= HASH_FLAGS_SHA224;
+		state->flags |= HASH_FLAGS_SHA224;
 		break;
 	case SHA256_DIGEST_SIZE:
-		rctx->flags |= HASH_FLAGS_SHA256;
+		state->flags |= HASH_FLAGS_SHA256;
 		break;
 	default:
 		return -EINVAL;
 	}
 
-	rctx->bufcnt = 0;
-	rctx->buflen = HASH_BUFLEN;
+	rctx->state.bufcnt = 0;
+	rctx->state.buflen = HASH_BUFLEN;
 	rctx->total = 0;
 	rctx->offset = 0;
 	rctx->data_type = HASH_DATA_8_BITS;
 
-	memset(rctx->buffer, 0, HASH_BUFLEN);
-
 	if (ctx->flags & HASH_FLAGS_HMAC)
-		rctx->flags |= HASH_FLAGS_HMAC;
+		state->flags |= HASH_FLAGS_HMAC;
 
-	dev_dbg(hdev->dev, "%s Flags %lx\n", __func__, rctx->flags);
+	dev_dbg(hdev->dev, "%s Flags %x\n", __func__, state->flags);
 
 	return 0;
 }
 
 static int stm32_hash_update_req(struct stm32_hash_dev *hdev)
 {
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
+	struct stm32_hash_state *state = &rctx->state;
+
+	if (!(state->flags & HASH_FLAGS_CPU))
+		return stm32_hash_dma_send(hdev);
+
 	return stm32_hash_update_cpu(hdev);
 }
 
@@ -774,26 +800,15 @@ static int stm32_hash_final_req(struct stm32_hash_dev *hdev)
 {
 	struct ahash_request *req = hdev->req;
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
-	int err;
-	int buflen = rctx->bufcnt;
+	struct stm32_hash_state *state = &rctx->state;
+	int buflen = state->bufcnt;
 
-	rctx->bufcnt = 0;
+	if (state->flags & HASH_FLAGS_FINUP)
+		return stm32_hash_update_req(hdev);
 
-	if (!(rctx->flags & HASH_FLAGS_CPU))
-		err = stm32_hash_dma_send(hdev);
-	else
-		err = stm32_hash_xmit_cpu(hdev, rctx->buffer, buflen, 1);
+	state->bufcnt = 0;
 
-	/* If we have an IRQ, wait for that, else poll for completion */
-	if (hdev->polled) {
-		if (stm32_hash_wait_busy(hdev))
-			return -ETIMEDOUT;
-		hdev->flags |= HASH_FLAGS_OUTPUT_READY;
-		/* Caller will call stm32_hash_finish_req() */
-		err = 0;
-	}
-
-	return err;
+	return stm32_hash_xmit_cpu(hdev, state->buffer, buflen, 1);
 }
 
 static void stm32_hash_emptymsg_fallback(struct ahash_request *req)
@@ -828,14 +843,15 @@ static void stm32_hash_emptymsg_fallback(struct ahash_request *req)
 static void stm32_hash_copy_hash(struct ahash_request *req)
 {
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct stm32_hash_state *state = &rctx->state;
 	struct stm32_hash_dev *hdev = rctx->hdev;
 	__be32 *hash = (void *)rctx->digest;
 	unsigned int i, hashsize;
 
-	if (hdev->pdata->broken_emptymsg && !req->nbytes)
+	if (hdev->pdata->broken_emptymsg && (state->flags & HASH_FLAGS_EMPTY))
 		return stm32_hash_emptymsg_fallback(req);
 
-	switch (rctx->flags & HASH_FLAGS_ALGO_MASK) {
+	switch (state->flags & HASH_FLAGS_ALGO_MASK) {
 	case HASH_FLAGS_MD5:
 		hashsize = MD5_DIGEST_SIZE;
 		break;
@@ -882,13 +898,6 @@ static void stm32_hash_finish_req(struct ahash_request *req, int err)
 	if (!err && (HASH_FLAGS_FINAL & hdev->flags)) {
 		stm32_hash_copy_hash(req);
 		err = stm32_hash_finish(req);
-		hdev->flags &= ~(HASH_FLAGS_FINAL | HASH_FLAGS_CPU |
-				 HASH_FLAGS_INIT | HASH_FLAGS_DMA_READY |
-				 HASH_FLAGS_OUTPUT_READY | HASH_FLAGS_HMAC |
-				 HASH_FLAGS_HMAC_INIT | HASH_FLAGS_HMAC_FINAL |
-				 HASH_FLAGS_HMAC_KEY);
-	} else {
-		rctx->flags |= HASH_FLAGS_ERRORS;
 	}
 
 	pm_runtime_mark_last_busy(hdev->dev);
@@ -897,73 +906,70 @@ static void stm32_hash_finish_req(struct ahash_request *req, int err)
 	crypto_finalize_hash_request(hdev->engine, req, err);
 }
 
-static int stm32_hash_hw_init(struct stm32_hash_dev *hdev,
-			      struct stm32_hash_request_ctx *rctx)
-{
-	pm_runtime_get_sync(hdev->dev);
-
-	if (!(HASH_FLAGS_INIT & hdev->flags)) {
-		stm32_hash_write(hdev, HASH_CR, HASH_CR_INIT);
-		stm32_hash_write(hdev, HASH_STR, 0);
-		stm32_hash_write(hdev, HASH_DIN, 0);
-		stm32_hash_write(hdev, HASH_IMR, 0);
-		hdev->err = 0;
-	}
-
-	return 0;
-}
-
-static int stm32_hash_one_request(struct crypto_engine *engine, void *areq);
-static int stm32_hash_prepare_req(struct crypto_engine *engine, void *areq);
-
 static int stm32_hash_handle_queue(struct stm32_hash_dev *hdev,
 				   struct ahash_request *req)
 {
 	return crypto_transfer_hash_request_to_engine(hdev->engine, req);
 }
 
-static int stm32_hash_prepare_req(struct crypto_engine *engine, void *areq)
+static int stm32_hash_one_request(struct crypto_engine *engine, void *areq)
 {
 	struct ahash_request *req = container_of(areq, struct ahash_request,
 						 base);
 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
-	struct stm32_hash_request_ctx *rctx;
+	struct stm32_hash_state *state = &rctx->state;
+	int err = 0;
 
 	if (!hdev)
 		return -ENODEV;
 
-	hdev->req = req;
-
-	rctx = ahash_request_ctx(req);
-
 	dev_dbg(hdev->dev, "processing new req, op: %lu, nbytes %d\n",
 		rctx->op, req->nbytes);
 
-	return stm32_hash_hw_init(hdev, rctx);
-}
+	pm_runtime_get_sync(hdev->dev);
 
-static int stm32_hash_one_request(struct crypto_engine *engine, void *areq)
-{
-	struct ahash_request *req = container_of(areq, struct ahash_request,
-						 base);
-	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
-	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
-	struct stm32_hash_request_ctx *rctx;
-	int err = 0;
+	hdev->req = req;
+	hdev->flags = 0;
+
+	if (state->flags & HASH_FLAGS_INIT) {
+		u32 *preg = rctx->state.hw_context;
+		u32 reg;
+		int i;
+
+		if (!hdev->pdata->ux500)
+			stm32_hash_write(hdev, HASH_IMR, *preg++);
+		stm32_hash_write(hdev, HASH_STR, *preg++);
+		stm32_hash_write(hdev, HASH_CR, *preg);
+		reg = *preg++ | HASH_CR_INIT;
+		stm32_hash_write(hdev, HASH_CR, reg);
 
-	if (!hdev)
-		return -ENODEV;
+		for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++)
+			stm32_hash_write(hdev, HASH_CSR(i), *preg++);
 
-	hdev->req = req;
+		hdev->flags |= HASH_FLAGS_INIT;
 
-	rctx = ahash_request_ctx(req);
+		if (state->flags & HASH_FLAGS_HMAC)
+			hdev->flags |= HASH_FLAGS_HMAC |
+				       HASH_FLAGS_HMAC_KEY;
+	}
 
 	if (rctx->op == HASH_OP_UPDATE)
 		err = stm32_hash_update_req(hdev);
 	else if (rctx->op == HASH_OP_FINAL)
 		err = stm32_hash_final_req(hdev);
 
+	/* If we have an IRQ, wait for that, else poll for completion */
+	if (err == -EINPROGRESS && hdev->polled) {
+		if (stm32_hash_wait_busy(hdev))
+			err = -ETIMEDOUT;
+		else {
+			hdev->flags |= HASH_FLAGS_OUTPUT_READY;
+			err = 0;
+		}
+	}
+
 	if (err != -EINPROGRESS)
 	/* done task will not finish it, so do it here */
 		stm32_hash_finish_req(req, err);
@@ -985,15 +991,16 @@ static int stm32_hash_enqueue(struct ahash_request *req, unsigned int op)
 static int stm32_hash_update(struct ahash_request *req)
 {
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct stm32_hash_state *state = &rctx->state;
 
-	if (!req->nbytes || !(rctx->flags & HASH_FLAGS_CPU))
+	if (!req->nbytes || !(state->flags & HASH_FLAGS_CPU))
 		return 0;
 
 	rctx->total = req->nbytes;
 	rctx->sg = req->src;
 	rctx->offset = 0;
 
-	if ((rctx->bufcnt + rctx->total < rctx->buflen)) {
+	if ((state->bufcnt + rctx->total < state->buflen)) {
 		stm32_hash_append_sg(rctx);
 		return 0;
 	}
@@ -1004,8 +1011,9 @@ static int stm32_hash_update(struct ahash_request *req)
 static int stm32_hash_final(struct ahash_request *req)
 {
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct stm32_hash_state *state = &rctx->state;
 
-	rctx->flags |= HASH_FLAGS_FINUP;
+	state->flags |= HASH_FLAGS_FINAL;
 
 	return stm32_hash_enqueue(req, HASH_OP_FINAL);
 }
@@ -1015,25 +1023,21 @@ static int stm32_hash_finup(struct ahash_request *req)
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
 	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
-	int err1, err2;
+	struct stm32_hash_state *state = &rctx->state;
 
-	rctx->flags |= HASH_FLAGS_FINUP;
+	if (!req->nbytes)
+		goto out;
 
-	if (hdev->dma_lch && stm32_hash_dma_aligned_data(req))
-		rctx->flags &= ~HASH_FLAGS_CPU;
-
-	err1 = stm32_hash_update(req);
-
-	if (err1 == -EINPROGRESS || err1 == -EBUSY)
-		return err1;
+	state->flags |= HASH_FLAGS_FINUP;
+	rctx->total = req->nbytes;
+	rctx->sg = req->src;
+	rctx->offset = 0;
 
-	/*
-	 * final() has to be always called to cleanup resources
-	 * even if update() failed, except EINPROGRESS
-	 */
-	err2 = stm32_hash_final(req);
+	if (hdev->dma_lch && stm32_hash_dma_aligned_data(req))
+		state->flags &= ~HASH_FLAGS_CPU;
 
-	return err1 ?: err2;
+out:
+	return stm32_hash_final(req);
 }
 
 static int stm32_hash_digest(struct ahash_request *req)
@@ -1044,35 +1048,8 @@ static int stm32_hash_digest(struct ahash_request *req)
 static int stm32_hash_export(struct ahash_request *req, void *out)
 {
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
-	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
-	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
-	u32 *preg;
-	unsigned int i;
-	int ret;
 
-	pm_runtime_get_sync(hdev->dev);
-
-	ret = stm32_hash_wait_busy(hdev);
-	if (ret)
-		return ret;
-
-	rctx->hw_context = kmalloc_array(3 + HASH_CSR_REGISTER_NUMBER,
-					 sizeof(u32),
-					 GFP_KERNEL);
-
-	preg = rctx->hw_context;
-
-	if (!hdev->pdata->ux500)
-		*preg++ = stm32_hash_read(hdev, HASH_IMR);
-	*preg++ = stm32_hash_read(hdev, HASH_STR);
-	*preg++ = stm32_hash_read(hdev, HASH_CR);
-	for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++)
-		*preg++ = stm32_hash_read(hdev, HASH_CSR(i));
-
-	pm_runtime_mark_last_busy(hdev->dev);
-	pm_runtime_put_autosuspend(hdev->dev);
-
-	memcpy(out, rctx, sizeof(*rctx));
+	memcpy(out, &rctx->state, sizeof(rctx->state));
 
 	return 0;
 }
@@ -1080,32 +1057,9 @@ static int stm32_hash_export(struct ahash_request *req, void *out)
 static int stm32_hash_import(struct ahash_request *req, const void *in)
 {
 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
-	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
-	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
-	const u32 *preg = in;
-	u32 reg;
-	unsigned int i;
-
-	memcpy(rctx, in, sizeof(*rctx));
-
-	preg = rctx->hw_context;
-
-	pm_runtime_get_sync(hdev->dev);
-
-	if (!hdev->pdata->ux500)
-		stm32_hash_write(hdev, HASH_IMR, *preg++);
-	stm32_hash_write(hdev, HASH_STR, *preg++);
-	stm32_hash_write(hdev, HASH_CR, *preg);
-	reg = *preg++ | HASH_CR_INIT;
-	stm32_hash_write(hdev, HASH_CR, reg);
-
-	for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++)
-		stm32_hash_write(hdev, HASH_CSR(i), *preg++);
-
-	pm_runtime_mark_last_busy(hdev->dev);
-	pm_runtime_put_autosuspend(hdev->dev);
 
-	kfree(rctx->hw_context);
+	stm32_hash_init(req);
+	memcpy(&rctx->state, in, sizeof(rctx->state));
 
 	return 0;
 }
@@ -1162,8 +1116,6 @@ static int stm32_hash_cra_init_algs(struct crypto_tfm *tfm,
 		ctx->flags |= HASH_FLAGS_HMAC;
 
 	ctx->enginectx.op.do_one_request = stm32_hash_one_request;
-	ctx->enginectx.op.prepare_request = stm32_hash_prepare_req;
-	ctx->enginectx.op.unprepare_request = NULL;
 
 	return stm32_hash_init_fallback(tfm);
 }
@@ -1255,7 +1207,7 @@ static struct ahash_alg algs_md5[] = {
 		.import = stm32_hash_import,
 		.halg = {
 			.digestsize = MD5_DIGEST_SIZE,
-			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.statesize = sizeof(struct stm32_hash_state),
 			.base = {
 				.cra_name = "md5",
 				.cra_driver_name = "stm32-md5",
@@ -1282,7 +1234,7 @@ static struct ahash_alg algs_md5[] = {
 		.setkey = stm32_hash_setkey,
 		.halg = {
 			.digestsize = MD5_DIGEST_SIZE,
-			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.statesize = sizeof(struct stm32_hash_state),
 			.base = {
 				.cra_name = "hmac(md5)",
 				.cra_driver_name = "stm32-hmac-md5",
@@ -1311,7 +1263,7 @@ static struct ahash_alg algs_sha1[] = {
 		.import = stm32_hash_import,
 		.halg = {
 			.digestsize = SHA1_DIGEST_SIZE,
-			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.statesize = sizeof(struct stm32_hash_state),
 			.base = {
 				.cra_name = "sha1",
 				.cra_driver_name = "stm32-sha1",
@@ -1338,7 +1290,7 @@ static struct ahash_alg algs_sha1[] = {
 		.setkey = stm32_hash_setkey,
 		.halg = {
 			.digestsize = SHA1_DIGEST_SIZE,
-			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.statesize = sizeof(struct stm32_hash_state),
 			.base = {
 				.cra_name = "hmac(sha1)",
 				.cra_driver_name = "stm32-hmac-sha1",
@@ -1367,7 +1319,7 @@ static struct ahash_alg algs_sha224[] = {
 		.import = stm32_hash_import,
 		.halg = {
 			.digestsize = SHA224_DIGEST_SIZE,
-			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.statesize = sizeof(struct stm32_hash_state),
 			.base = {
 				.cra_name = "sha224",
 				.cra_driver_name = "stm32-sha224",
@@ -1394,7 +1346,7 @@ static struct ahash_alg algs_sha224[] = {
 		.import = stm32_hash_import,
 		.halg = {
 			.digestsize = SHA224_DIGEST_SIZE,
-			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.statesize = sizeof(struct stm32_hash_state),
 			.base = {
 				.cra_name = "hmac(sha224)",
 				.cra_driver_name = "stm32-hmac-sha224",
@@ -1423,7 +1375,7 @@ static struct ahash_alg algs_sha256[] = {
 		.import = stm32_hash_import,
 		.halg = {
 			.digestsize = SHA256_DIGEST_SIZE,
-			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.statesize = sizeof(struct stm32_hash_state),
 			.base = {
 				.cra_name = "sha256",
 				.cra_driver_name = "stm32-sha256",
@@ -1450,7 +1402,7 @@ static struct ahash_alg algs_sha256[] = {
 		.setkey = stm32_hash_setkey,
 		.halg = {
 			.digestsize = SHA256_DIGEST_SIZE,
-			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.statesize = sizeof(struct stm32_hash_state),
 			.base = {
 				.cra_name = "hmac(sha256)",
 				.cra_driver_name = "stm32-hmac-sha256",
@@ -1616,8 +1568,7 @@ static int stm32_hash_probe(struct platform_device *pdev)
 	if (!hdev)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	hdev->io_base = devm_ioremap_resource(dev, res);
+	hdev->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(hdev->io_base))
 		return PTR_ERR(hdev->io_base);
 
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 4d1f9c5b5029..72b889af9917 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -1903,7 +1903,7 @@ bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
 EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL);
 
 /* for user tooling to ensure port disable work has completed */
-static ssize_t flush_store(struct bus_type *bus, const char *buf, size_t count)
+static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
 {
 	if (sysfs_streq(buf, "1")) {
 		flush_workqueue(cxl_bus_wq);
diff --git a/drivers/dca/dca-sysfs.c b/drivers/dca/dca-sysfs.c
index 21ebd0af268b..fcc83ede0909 100644
--- a/drivers/dca/dca-sysfs.c
+++ b/drivers/dca/dca-sysfs.c
@@ -74,7 +74,7 @@ int __init dca_sysfs_init(void)
 	idr_init(&dca_idr);
 	spin_lock_init(&dca_idr_lock);
 
-	dca_class = class_create(THIS_MODULE, "dca");
+	dca_class = class_create("dca");
 	if (IS_ERR(dca_class)) {
 		idr_destroy(&dca_idr);
 		return PTR_ERR(dca_class);
diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig
index 9754d8b31621..3c4862a752b5 100644
--- a/drivers/devfreq/Kconfig
+++ b/drivers/devfreq/Kconfig
@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menuconfig PM_DEVFREQ
 	bool "Generic Dynamic Voltage and Frequency Scaling (DVFS) support"
-	select SRCU
 	select PM_OPP
 	help
 	  A device may have a list of frequencies and voltages available.
diff --git a/drivers/devfreq/devfreq-event.c b/drivers/devfreq/devfreq-event.c
index f041edccd107..3ebac2496679 100644
--- a/drivers/devfreq/devfreq-event.c
+++ b/drivers/devfreq/devfreq-event.c
@@ -469,7 +469,7 @@ ATTRIBUTE_GROUPS(devfreq_event);
 
 static int __init devfreq_event_init(void)
 {
-	devfreq_event_class = class_create(THIS_MODULE, "devfreq-event");
+	devfreq_event_class = class_create("devfreq-event");
 	if (IS_ERR(devfreq_event_class)) {
 		pr_err("%s: couldn't create class\n", __FILE__);
 		return PTR_ERR(devfreq_event_class);
diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
index 817c71da391a..e36cbb920ec8 100644
--- a/drivers/devfreq/devfreq.c
+++ b/drivers/devfreq/devfreq.c
@@ -1988,7 +1988,7 @@ DEFINE_SHOW_ATTRIBUTE(devfreq_summary);
 
 static int __init devfreq_init(void)
 {
-	devfreq_class = class_create(THIS_MODULE, "devfreq");
+	devfreq_class = class_create("devfreq");
 	if (IS_ERR(devfreq_class)) {
 		pr_err("%s: couldn't create class\n", __FILE__);
 		return PTR_ERR(devfreq_class);
diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c
index a443e7c42daf..896a6cc93b00 100644
--- a/drivers/devfreq/event/exynos-ppmu.c
+++ b/drivers/devfreq/event/exynos-ppmu.c
@@ -621,8 +621,7 @@ static int exynos_ppmu_parse_dt(struct platform_device *pdev,
 	}
 
 	/* Maps the memory mapped IO to control PPMU register */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	base = devm_ioremap_resource(dev, res);
+	base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c
index 027e8f336acc..88414445adf3 100644
--- a/drivers/devfreq/exynos-bus.c
+++ b/drivers/devfreq/exynos-bus.c
@@ -432,7 +432,7 @@ static int exynos_bus_probe(struct platform_device *pdev)
 		goto err;
 
 	/* Create child platform device for the interconnect provider */
-	if (of_get_property(dev->of_node, "#interconnect-cells", NULL)) {
+	if (of_property_present(dev->of_node, "#interconnect-cells")) {
 		bus->icc_pdev = platform_device_register_data(
 						dev, "exynos-generic-icc",
 						PLATFORM_DEVID_AUTO, NULL, 0);
@@ -513,7 +513,7 @@ static struct platform_driver exynos_bus_platdrv = {
 	.driver = {
 		.name	= "exynos-bus",
 		.pm	= &exynos_bus_pm,
-		.of_match_table = of_match_ptr(exynos_bus_of_match),
+		.of_match_table = exynos_bus_of_match,
 	},
 };
 module_platform_driver(exynos_bus_platdrv);
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index 757c0fb77a6c..aa4ea8530cb3 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -828,7 +828,7 @@ static struct sg_table * __map_dma_buf(struct dma_buf_attachment *attach,
  *     - dma_buf_attach()
  *     - dma_buf_dynamic_attach()
  *     - dma_buf_detach()
- *     - dma_buf_export(
+ *     - dma_buf_export()
  *     - dma_buf_fd()
  *     - dma_buf_get()
  *     - dma_buf_put()
diff --git a/drivers/dma-buf/dma-fence-array.c b/drivers/dma-buf/dma-fence-array.c
index 5c8a7084577b..9b3ce8948351 100644
--- a/drivers/dma-buf/dma-fence-array.c
+++ b/drivers/dma-buf/dma-fence-array.c
@@ -123,12 +123,23 @@ static void dma_fence_array_release(struct dma_fence *fence)
 	dma_fence_free(fence);
 }
 
+static void dma_fence_array_set_deadline(struct dma_fence *fence,
+					 ktime_t deadline)
+{
+	struct dma_fence_array *array = to_dma_fence_array(fence);
+	unsigned i;
+
+	for (i = 0; i < array->num_fences; ++i)
+		dma_fence_set_deadline(array->fences[i], deadline);
+}
+
 const struct dma_fence_ops dma_fence_array_ops = {
 	.get_driver_name = dma_fence_array_get_driver_name,
 	.get_timeline_name = dma_fence_array_get_timeline_name,
 	.enable_signaling = dma_fence_array_enable_signaling,
 	.signaled = dma_fence_array_signaled,
 	.release = dma_fence_array_release,
+	.set_deadline = dma_fence_array_set_deadline,
 };
 EXPORT_SYMBOL(dma_fence_array_ops);
 
diff --git a/drivers/dma-buf/dma-fence-chain.c b/drivers/dma-buf/dma-fence-chain.c
index a0d920576ba6..9663ba1bb6ac 100644
--- a/drivers/dma-buf/dma-fence-chain.c
+++ b/drivers/dma-buf/dma-fence-chain.c
@@ -206,6 +206,17 @@ static void dma_fence_chain_release(struct dma_fence *fence)
 	dma_fence_free(fence);
 }
 
+
+static void dma_fence_chain_set_deadline(struct dma_fence *fence,
+					 ktime_t deadline)
+{
+	dma_fence_chain_for_each(fence, fence) {
+		struct dma_fence *f = dma_fence_chain_contained(fence);
+
+		dma_fence_set_deadline(f, deadline);
+	}
+}
+
 const struct dma_fence_ops dma_fence_chain_ops = {
 	.use_64bit_seqno = true,
 	.get_driver_name = dma_fence_chain_get_driver_name,
@@ -213,6 +224,7 @@ const struct dma_fence_ops dma_fence_chain_ops = {
 	.enable_signaling = dma_fence_chain_enable_signaling,
 	.signaled = dma_fence_chain_signaled,
 	.release = dma_fence_chain_release,
+	.set_deadline = dma_fence_chain_set_deadline,
 };
 EXPORT_SYMBOL(dma_fence_chain_ops);
 
diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c
index 0de0482cd36e..f177c56269bb 100644
--- a/drivers/dma-buf/dma-fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -913,6 +913,65 @@ err_free_cb:
 EXPORT_SYMBOL(dma_fence_wait_any_timeout);
 
 /**
+ * DOC: deadline hints
+ *
+ * In an ideal world, it would be possible to pipeline a workload sufficiently
+ * that a utilization based device frequency governor could arrive at a minimum
+ * frequency that meets the requirements of the use-case, in order to minimize
+ * power consumption.  But in the real world there are many workloads which
+ * defy this ideal.  For example, but not limited to:
+ *
+ * * Workloads that ping-pong between device and CPU, with alternating periods
+ *   of CPU waiting for device, and device waiting on CPU.  This can result in
+ *   devfreq and cpufreq seeing idle time in their respective domains and in
+ *   result reduce frequency.
+ *
+ * * Workloads that interact with a periodic time based deadline, such as double
+ *   buffered GPU rendering vs vblank sync'd page flipping.  In this scenario,
+ *   missing a vblank deadline results in an *increase* in idle time on the GPU
+ *   (since it has to wait an additional vblank period), sending a signal to
+ *   the GPU's devfreq to reduce frequency, when in fact the opposite is what is
+ *   needed.
+ *
+ * To this end, deadline hint(s) can be set on a &dma_fence via &dma_fence_set_deadline.
+ * The deadline hint provides a way for the waiting driver, or userspace, to
+ * convey an appropriate sense of urgency to the signaling driver.
+ *
+ * A deadline hint is given in absolute ktime (CLOCK_MONOTONIC for userspace
+ * facing APIs).  The time could either be some point in the future (such as
+ * the vblank based deadline for page-flipping, or the start of a compositor's
+ * composition cycle), or the current time to indicate an immediate deadline
+ * hint (Ie. forward progress cannot be made until this fence is signaled).
+ *
+ * Multiple deadlines may be set on a given fence, even in parallel.  See the
+ * documentation for &dma_fence_ops.set_deadline.
+ *
+ * The deadline hint is just that, a hint.  The driver that created the fence
+ * may react by increasing frequency, making different scheduling choices, etc.
+ * Or doing nothing at all.
+ */
+
+/**
+ * dma_fence_set_deadline - set desired fence-wait deadline hint
+ * @fence:    the fence that is to be waited on
+ * @deadline: the time by which the waiter hopes for the fence to be
+ *            signaled
+ *
+ * Give the fence signaler a hint about an upcoming deadline, such as
+ * vblank, by which point the waiter would prefer the fence to be
+ * signaled by.  This is intended to give feedback to the fence signaler
+ * to aid in power management decisions, such as boosting GPU frequency
+ * if a periodic vblank deadline is approaching but the fence is not
+ * yet signaled..
+ */
+void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
+{
+	if (fence->ops->set_deadline && !dma_fence_is_signaled(fence))
+		fence->ops->set_deadline(fence, deadline);
+}
+EXPORT_SYMBOL(dma_fence_set_deadline);
+
+/**
  * dma_fence_describe - Dump fence describtion into seq_file
  * @fence: the 6fence to describe
  * @seq: the seq_file to put the textual description into
diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c
index c9e41e8a9e27..84ae708fafe7 100644
--- a/drivers/dma-buf/dma-heap.c
+++ b/drivers/dma-buf/dma-heap.c
@@ -314,7 +314,7 @@ static int dma_heap_init(void)
 	if (ret)
 		return ret;
 
-	dma_heap_class = class_create(THIS_MODULE, DEVNAME);
+	dma_heap_class = class_create(DEVNAME);
 	if (IS_ERR(dma_heap_class)) {
 		unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS);
 		return PTR_ERR(dma_heap_class);
diff --git a/drivers/dma-buf/dma-resv.c b/drivers/dma-buf/dma-resv.c
index 1c76aed8e262..2a594b754af1 100644
--- a/drivers/dma-buf/dma-resv.c
+++ b/drivers/dma-buf/dma-resv.c
@@ -684,6 +684,28 @@ long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage,
 }
 EXPORT_SYMBOL_GPL(dma_resv_wait_timeout);
 
+/**
+ * dma_resv_set_deadline - Set a deadline on reservation's objects fences
+ * @obj: the reservation object
+ * @usage: controls which fences to include, see enum dma_resv_usage.
+ * @deadline: the requested deadline (MONOTONIC)
+ *
+ * May be called without holding the dma_resv lock.  Sets @deadline on
+ * all fences filtered by @usage.
+ */
+void dma_resv_set_deadline(struct dma_resv *obj, enum dma_resv_usage usage,
+			   ktime_t deadline)
+{
+	struct dma_resv_iter cursor;
+	struct dma_fence *fence;
+
+	dma_resv_iter_begin(&cursor, obj, usage);
+	dma_resv_for_each_fence_unlocked(&cursor, fence) {
+		dma_fence_set_deadline(fence, deadline);
+	}
+	dma_resv_iter_end(&cursor);
+}
+EXPORT_SYMBOL_GPL(dma_resv_set_deadline);
 
 /**
  * dma_resv_test_signaled - Test if a reservation object's fences have been
diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c
index 1131fb943992..a7f048048864 100644
--- a/drivers/dma-buf/heaps/cma_heap.c
+++ b/drivers/dma-buf/heaps/cma_heap.c
@@ -407,4 +407,3 @@ static int add_default_cma_heap(void)
 }
 module_init(add_default_cma_heap);
 MODULE_DESCRIPTION("DMA-BUF CMA Heap");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c
index e8bd10e60998..79c03f5b4e28 100644
--- a/drivers/dma-buf/heaps/system_heap.c
+++ b/drivers/dma-buf/heaps/system_heap.c
@@ -440,4 +440,3 @@ static int system_heap_create(void)
 	return 0;
 }
 module_init(system_heap_create);
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c
index 740d6e426ee9..01f2e86f3f7c 100644
--- a/drivers/dma-buf/udmabuf.c
+++ b/drivers/dma-buf/udmabuf.c
@@ -430,4 +430,3 @@ module_init(udmabuf_dev_init)
 module_exit(udmabuf_dev_exit)
 
 MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c
index d19ea885c63e..5338a94f1a69 100644
--- a/drivers/dma/ep93xx_dma.c
+++ b/drivers/dma/ep93xx_dma.c
@@ -1431,4 +1431,3 @@ subsys_initcall(ep93xx_dma_module_init);
 
 MODULE_AUTHOR("Mika Westerberg <mika.westerberg@iki.fi>");
 MODULE_DESCRIPTION("EP93xx DMA driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/dma/idxd/compat.c b/drivers/dma/idxd/compat.c
index 3df21615f888..5fd38d1b9d28 100644
--- a/drivers/dma/idxd/compat.c
+++ b/drivers/dma/idxd/compat.c
@@ -16,7 +16,7 @@ extern void device_driver_detach(struct device *dev);
 
 static ssize_t unbind_store(struct device_driver *drv, const char *buf, size_t count)
 {
-	struct bus_type *bus = drv->bus;
+	const struct bus_type *bus = drv->bus;
 	struct device *dev;
 	int rc = -ENODEV;
 
@@ -32,7 +32,7 @@ static DRIVER_ATTR_IGNORE_LOCKDEP(unbind, 0200, NULL, unbind_store);
 
 static ssize_t bind_store(struct device_driver *drv, const char *buf, size_t count)
 {
-	struct bus_type *bus = drv->bus;
+	const struct bus_type *bus = drv->bus;
 	struct device *dev;
 	struct device_driver *alt_drv = NULL;
 	int rc = -ENODEV;
diff --git a/drivers/dma/ipu/ipu_idmac.c b/drivers/dma/ipu/ipu_idmac.c
index baab1ca9f621..d799b99c18bd 100644
--- a/drivers/dma/ipu/ipu_idmac.c
+++ b/drivers/dma/ipu/ipu_idmac.c
@@ -1797,6 +1797,5 @@ static int __init ipu_init(void)
 subsys_initcall(ipu_init);
 
 MODULE_DESCRIPTION("IPU core driver");
-MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Guennadi Liakhovetski <lg@denx.de>");
 MODULE_ALIAS("platform:ipu-core");
diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c
index 89790beba305..b4de7d4a3105 100644
--- a/drivers/dma/mv_xor_v2.c
+++ b/drivers/dma/mv_xor_v2.c
@@ -917,4 +917,3 @@ static struct platform_driver mv_xor_v2_driver = {
 module_platform_driver(mv_xor_v2_driver);
 
 MODULE_DESCRIPTION("DMA engine driver for Marvell's Version 2 of XOR engine");
-MODULE_LICENSE("GPL");
diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c
index 158e5e7defae..588c5f409a80 100644
--- a/drivers/dma/sh/shdma-base.c
+++ b/drivers/dma/sh/shdma-base.c
@@ -1047,6 +1047,5 @@ static void __exit shdma_exit(void)
 }
 module_exit(shdma_exit);
 
-MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("SH-DMA driver base library");
 MODULE_AUTHOR("Guennadi Liakhovetski <g.liakhovetski@gmx.de>");
diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c
index 46b884d46188..e415bd9f4f2b 100644
--- a/drivers/dma/stm32-dmamux.c
+++ b/drivers/dma/stm32-dmamux.c
@@ -398,4 +398,3 @@ arch_initcall(stm32_dmamux_init);
 MODULE_DESCRIPTION("DMA Router driver for STM32 DMA MUX");
 MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>");
 MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c
index 84e7f4f4a800..1d0e9dd72ab3 100644
--- a/drivers/dma/stm32-mdma.c
+++ b/drivers/dma/stm32-mdma.c
@@ -1814,4 +1814,3 @@ subsys_initcall(stm32_mdma_init);
 MODULE_DESCRIPTION("Driver for STM32 MDMA controller");
 MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>");
 MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c
index e7e8e624a436..8b31cd54bdb6 100644
--- a/drivers/edac/altera_edac.c
+++ b/drivers/edac/altera_edac.c
@@ -2149,10 +2149,8 @@ static int altr_edac_a10_probe(struct platform_device *pdev)
 	}
 
 	edac->sb_irq = platform_get_irq(pdev, 0);
-	if (edac->sb_irq < 0) {
-		dev_err(&pdev->dev, "No SBERR IRQ resource\n");
+	if (edac->sb_irq < 0)
 		return edac->sb_irq;
-	}
 
 	irq_set_chained_handler_and_data(edac->sb_irq,
 					 altr_edac_a10_irq_handler,
@@ -2184,10 +2182,9 @@ static int altr_edac_a10_probe(struct platform_device *pdev)
 	}
 #else
 	edac->db_irq = platform_get_irq(pdev, 1);
-	if (edac->db_irq < 0) {
-		dev_err(&pdev->dev, "No DBERR IRQ resource\n");
+	if (edac->db_irq < 0)
 		return edac->db_irq;
-	}
+
 	irq_set_chained_handler_and_data(edac->db_irq,
 					 altr_edac_a10_irq_handler, edac);
 #endif
@@ -2226,6 +2223,5 @@ static struct platform_driver altr_edac_a10_driver = {
 };
 module_platform_driver(altr_edac_a10_driver);
 
-MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Thor Thayer");
 MODULE_DESCRIPTION("EDAC Driver for Altera Memories");
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
index 5b42533f306a..5c4292e65b96 100644
--- a/drivers/edac/amd64_edac.c
+++ b/drivers/edac/amd64_edac.c
@@ -13,11 +13,9 @@ module_param(ecc_enable_override, int, 0644);
 
 static struct msr __percpu *msrs;
 
-static struct amd64_family_type *fam_type;
-
-static inline u32 get_umc_reg(u32 reg)
+static inline u32 get_umc_reg(struct amd64_pvt *pvt, u32 reg)
 {
-	if (!fam_type->flags.zn_regs_v2)
+	if (!pvt->flags.zn_regs_v2)
 		return reg;
 
 	switch (reg) {
@@ -437,7 +435,7 @@ static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
 	for (i = 0; i < pvt->csels[dct].m_cnt; i++)
 
 #define for_each_umc(i) \
-	for (i = 0; i < fam_type->max_mcs; i++)
+	for (i = 0; i < pvt->max_mcs; i++)
 
 /*
  * @input_addr is an InputAddr associated with the node given by mci. Return the
@@ -1258,40 +1256,102 @@ static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
  * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
  * are ECC capable.
  */
-static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
+static unsigned long dct_determine_edac_cap(struct amd64_pvt *pvt)
 {
 	unsigned long edac_cap = EDAC_FLAG_NONE;
 	u8 bit;
 
-	if (pvt->umc) {
-		u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
+	bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
+		? 19
+		: 17;
 
-		for_each_umc(i) {
-			if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
-				continue;
+	if (pvt->dclr0 & BIT(bit))
+		edac_cap = EDAC_FLAG_SECDED;
 
-			umc_en_mask |= BIT(i);
+	return edac_cap;
+}
 
-			/* UMC Configuration bit 12 (DimmEccEn) */
-			if (pvt->umc[i].umc_cfg & BIT(12))
-				dimm_ecc_en_mask |= BIT(i);
-		}
+static unsigned long umc_determine_edac_cap(struct amd64_pvt *pvt)
+{
+	u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
+	unsigned long edac_cap = EDAC_FLAG_NONE;
 
-		if (umc_en_mask == dimm_ecc_en_mask)
-			edac_cap = EDAC_FLAG_SECDED;
-	} else {
-		bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
-			? 19
-			: 17;
+	for_each_umc(i) {
+		if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
+			continue;
 
-		if (pvt->dclr0 & BIT(bit))
-			edac_cap = EDAC_FLAG_SECDED;
+		umc_en_mask |= BIT(i);
+
+		/* UMC Configuration bit 12 (DimmEccEn) */
+		if (pvt->umc[i].umc_cfg & BIT(12))
+			dimm_ecc_en_mask |= BIT(i);
 	}
 
+	if (umc_en_mask == dimm_ecc_en_mask)
+		edac_cap = EDAC_FLAG_SECDED;
+
 	return edac_cap;
 }
 
-static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
+/*
+ * debug routine to display the memory sizes of all logical DIMMs and its
+ * CSROWs
+ */
+static void dct_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
+{
+	u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
+	u32 dbam  = ctrl ? pvt->dbam1 : pvt->dbam0;
+	int dimm, size0, size1;
+
+	if (pvt->fam == 0xf) {
+		/* K8 families < revF not supported yet */
+		if (pvt->ext_model < K8_REV_F)
+			return;
+
+		WARN_ON(ctrl != 0);
+	}
+
+	if (pvt->fam == 0x10) {
+		dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
+							   : pvt->dbam0;
+		dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
+				 pvt->csels[1].csbases :
+				 pvt->csels[0].csbases;
+	} else if (ctrl) {
+		dbam = pvt->dbam0;
+		dcsb = pvt->csels[1].csbases;
+	}
+	edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
+		 ctrl, dbam);
+
+	edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
+
+	/* Dump memory sizes for DIMM and its CSROWs */
+	for (dimm = 0; dimm < 4; dimm++) {
+		size0 = 0;
+		if (dcsb[dimm * 2] & DCSB_CS_ENABLE)
+			/*
+			 * For F15m60h, we need multiplier for LRDIMM cs_size
+			 * calculation. We pass dimm value to the dbam_to_cs
+			 * mapper so we can find the multiplier from the
+			 * corresponding DCSM.
+			 */
+			size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
+						     DBAM_DIMM(dimm, dbam),
+						     dimm);
+
+		size1 = 0;
+		if (dcsb[dimm * 2 + 1] & DCSB_CS_ENABLE)
+			size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
+						     DBAM_DIMM(dimm, dbam),
+						     dimm);
+
+		amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
+			   dimm * 2,     size0,
+			   dimm * 2 + 1, size1);
+	}
+}
+
 
 static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
 {
@@ -1334,7 +1394,7 @@ static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
 #define CS_EVEN			(CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
 #define CS_ODD			(CS_ODD_PRIMARY | CS_ODD_SECONDARY)
 
-static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
+static int umc_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
 {
 	u8 base, count = 0;
 	int cs_mode = 0;
@@ -1366,7 +1426,85 @@ static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
 	return cs_mode;
 }
 
-static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
+static int umc_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
+				    unsigned int cs_mode, int csrow_nr)
+{
+	u32 addr_mask_orig, addr_mask_deinterleaved;
+	u32 msb, weight, num_zero_bits;
+	int cs_mask_nr = csrow_nr;
+	int dimm, size = 0;
+
+	/* No Chip Selects are enabled. */
+	if (!cs_mode)
+		return size;
+
+	/* Requested size of an even CS but none are enabled. */
+	if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
+		return size;
+
+	/* Requested size of an odd CS but none are enabled. */
+	if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
+		return size;
+
+	/*
+	 * Family 17h introduced systems with one mask per DIMM,
+	 * and two Chip Selects per DIMM.
+	 *
+	 *	CS0 and CS1 -> MASK0 / DIMM0
+	 *	CS2 and CS3 -> MASK1 / DIMM1
+	 *
+	 * Family 19h Model 10h introduced systems with one mask per Chip Select,
+	 * and two Chip Selects per DIMM.
+	 *
+	 *	CS0 -> MASK0 -> DIMM0
+	 *	CS1 -> MASK1 -> DIMM0
+	 *	CS2 -> MASK2 -> DIMM1
+	 *	CS3 -> MASK3 -> DIMM1
+	 *
+	 * Keep the mask number equal to the Chip Select number for newer systems,
+	 * and shift the mask number for older systems.
+	 */
+	dimm = csrow_nr >> 1;
+
+	if (!pvt->flags.zn_regs_v2)
+		cs_mask_nr >>= 1;
+
+	/* Asymmetric dual-rank DIMM support. */
+	if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
+		addr_mask_orig = pvt->csels[umc].csmasks_sec[cs_mask_nr];
+	else
+		addr_mask_orig = pvt->csels[umc].csmasks[cs_mask_nr];
+
+	/*
+	 * The number of zero bits in the mask is equal to the number of bits
+	 * in a full mask minus the number of bits in the current mask.
+	 *
+	 * The MSB is the number of bits in the full mask because BIT[0] is
+	 * always 0.
+	 *
+	 * In the special 3 Rank interleaving case, a single bit is flipped
+	 * without swapping with the most significant bit. This can be handled
+	 * by keeping the MSB where it is and ignoring the single zero bit.
+	 */
+	msb = fls(addr_mask_orig) - 1;
+	weight = hweight_long(addr_mask_orig);
+	num_zero_bits = msb - weight - !!(cs_mode & CS_3R_INTERLEAVE);
+
+	/* Take the number of zero bits off from the top of the mask. */
+	addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
+
+	edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm);
+	edac_dbg(1, "  Original AddrMask: 0x%x\n", addr_mask_orig);
+	edac_dbg(1, "  Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved);
+
+	/* Register [31:1] = Address [39:9]. Size is in kBs here. */
+	size = (addr_mask_deinterleaved >> 2) + 1;
+
+	/* Return size in MBs. */
+	return size >> 10;
+}
+
+static void umc_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
 {
 	int dimm, size0, size1, cs0, cs1, cs_mode;
 
@@ -1376,10 +1514,10 @@ static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
 		cs0 = dimm * 2;
 		cs1 = dimm * 2 + 1;
 
-		cs_mode = f17_get_cs_mode(dimm, ctrl, pvt);
+		cs_mode = umc_get_cs_mode(dimm, ctrl, pvt);
 
-		size0 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs0);
-		size1 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs1);
+		size0 = umc_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs0);
+		size1 = umc_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs1);
 
 		amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
 				cs0,	size0,
@@ -1387,7 +1525,7 @@ static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
 	}
 }
 
-static void __dump_misc_regs_df(struct amd64_pvt *pvt)
+static void umc_dump_misc_regs(struct amd64_pvt *pvt)
 {
 	struct amd64_umc *umc;
 	u32 i, tmp, umc_base;
@@ -1420,18 +1558,17 @@ static void __dump_misc_regs_df(struct amd64_pvt *pvt)
 
 		if (umc->dram_type == MEM_LRDDR4 || umc->dram_type == MEM_LRDDR5) {
 			amd_smn_read(pvt->mc_node_id,
-				     umc_base + get_umc_reg(UMCCH_ADDR_CFG),
+				     umc_base + get_umc_reg(pvt, UMCCH_ADDR_CFG),
 				     &tmp);
 			edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
 					i, 1 << ((tmp >> 4) & 0x3));
 		}
 
-		debug_display_dimm_sizes_df(pvt, i);
+		umc_debug_display_dimm_sizes(pvt, i);
 	}
 }
 
-/* Display and decode various NB registers for debug purposes. */
-static void __dump_misc_regs(struct amd64_pvt *pvt)
+static void dct_dump_misc_regs(struct amd64_pvt *pvt)
 {
 	edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
 
@@ -1451,28 +1588,19 @@ static void __dump_misc_regs(struct amd64_pvt *pvt)
 		 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
 				   : f10_dhar_offset(pvt));
 
-	debug_display_dimm_sizes(pvt, 0);
+	dct_debug_display_dimm_sizes(pvt, 0);
 
 	/* everything below this point is Fam10h and above */
 	if (pvt->fam == 0xf)
 		return;
 
-	debug_display_dimm_sizes(pvt, 1);
+	dct_debug_display_dimm_sizes(pvt, 1);
 
 	/* Only if NOT ganged does dclr1 have valid info */
 	if (!dct_ganging_enabled(pvt))
 		debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
 
 	edac_dbg(1, "  DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
-}
-
-/* Display and decode various NB registers for debug purposes. */
-static void dump_misc_regs(struct amd64_pvt *pvt)
-{
-	if (pvt->umc)
-		__dump_misc_regs_df(pvt);
-	else
-		__dump_misc_regs(pvt);
 
 	amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz);
 }
@@ -1480,7 +1608,7 @@ static void dump_misc_regs(struct amd64_pvt *pvt)
 /*
  * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
  */
-static void prep_chip_selects(struct amd64_pvt *pvt)
+static void dct_prep_chip_selects(struct amd64_pvt *pvt)
 {
 	if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
 		pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
@@ -1488,21 +1616,23 @@ static void prep_chip_selects(struct amd64_pvt *pvt)
 	} else if (pvt->fam == 0x15 && pvt->model == 0x30) {
 		pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
 		pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
-	} else if (pvt->fam >= 0x17) {
-		int umc;
-
-		for_each_umc(umc) {
-			pvt->csels[umc].b_cnt = 4;
-			pvt->csels[umc].m_cnt = fam_type->flags.zn_regs_v2 ? 4 : 2;
-		}
-
 	} else {
 		pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
 		pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
 	}
 }
 
-static void read_umc_base_mask(struct amd64_pvt *pvt)
+static void umc_prep_chip_selects(struct amd64_pvt *pvt)
+{
+	int umc;
+
+	for_each_umc(umc) {
+		pvt->csels[umc].b_cnt = 4;
+		pvt->csels[umc].m_cnt = pvt->flags.zn_regs_v2 ? 4 : 2;
+	}
+}
+
+static void umc_read_base_mask(struct amd64_pvt *pvt)
 {
 	u32 umc_base_reg, umc_base_reg_sec;
 	u32 umc_mask_reg, umc_mask_reg_sec;
@@ -1533,7 +1663,7 @@ static void read_umc_base_mask(struct amd64_pvt *pvt)
 		}
 
 		umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK;
-		umc_mask_reg_sec = get_umc_base(umc) + get_umc_reg(UMCCH_ADDR_MASK_SEC);
+		umc_mask_reg_sec = get_umc_base(umc) + get_umc_reg(pvt, UMCCH_ADDR_MASK_SEC);
 
 		for_each_chip_select_mask(cs, umc, pvt) {
 			mask = &pvt->csels[umc].csmasks[cs];
@@ -1556,15 +1686,10 @@ static void read_umc_base_mask(struct amd64_pvt *pvt)
 /*
  * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
  */
-static void read_dct_base_mask(struct amd64_pvt *pvt)
+static void dct_read_base_mask(struct amd64_pvt *pvt)
 {
 	int cs;
 
-	prep_chip_selects(pvt);
-
-	if (pvt->umc)
-		return read_umc_base_mask(pvt);
-
 	for_each_chip_select(cs, 0, pvt) {
 		int reg0   = DCSB0 + (cs * 4);
 		int reg1   = DCSB1 + (cs * 4);
@@ -1604,7 +1729,7 @@ static void read_dct_base_mask(struct amd64_pvt *pvt)
 	}
 }
 
-static void determine_memory_type_df(struct amd64_pvt *pvt)
+static void umc_determine_memory_type(struct amd64_pvt *pvt)
 {
 	struct amd64_umc *umc;
 	u32 i;
@@ -1621,7 +1746,7 @@ static void determine_memory_type_df(struct amd64_pvt *pvt)
 		 * Check if the system supports the "DDR Type" field in UMC Config
 		 * and has DDR5 DIMMs in use.
 		 */
-		if (fam_type->flags.zn_regs_v2 && ((umc->umc_cfg & GENMASK(2, 0)) == 0x1)) {
+		if (pvt->flags.zn_regs_v2 && ((umc->umc_cfg & GENMASK(2, 0)) == 0x1)) {
 			if (umc->dimm_cfg & BIT(5))
 				umc->dram_type = MEM_LRDDR5;
 			else if (umc->dimm_cfg & BIT(4))
@@ -1641,13 +1766,10 @@ static void determine_memory_type_df(struct amd64_pvt *pvt)
 	}
 }
 
-static void determine_memory_type(struct amd64_pvt *pvt)
+static void dct_determine_memory_type(struct amd64_pvt *pvt)
 {
 	u32 dram_ctrl, dcsm;
 
-	if (pvt->umc)
-		return determine_memory_type_df(pvt);
-
 	switch (pvt->fam) {
 	case 0xf:
 		if (pvt->ext_model >= K8_REV_F)
@@ -1697,6 +1819,8 @@ static void determine_memory_type(struct amd64_pvt *pvt)
 		WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
 		pvt->dram_type = MEM_EMPTY;
 	}
+
+	edac_dbg(1, "  DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
 	return;
 
 ddr3:
@@ -2081,84 +2205,6 @@ static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
 		return ddr3_cs_size(cs_mode, false);
 }
 
-static int f17_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
-				    unsigned int cs_mode, int csrow_nr)
-{
-	u32 addr_mask_orig, addr_mask_deinterleaved;
-	u32 msb, weight, num_zero_bits;
-	int cs_mask_nr = csrow_nr;
-	int dimm, size = 0;
-
-	/* No Chip Selects are enabled. */
-	if (!cs_mode)
-		return size;
-
-	/* Requested size of an even CS but none are enabled. */
-	if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
-		return size;
-
-	/* Requested size of an odd CS but none are enabled. */
-	if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
-		return size;
-
-	/*
-	 * Family 17h introduced systems with one mask per DIMM,
-	 * and two Chip Selects per DIMM.
-	 *
-	 *	CS0 and CS1 -> MASK0 / DIMM0
-	 *	CS2 and CS3 -> MASK1 / DIMM1
-	 *
-	 * Family 19h Model 10h introduced systems with one mask per Chip Select,
-	 * and two Chip Selects per DIMM.
-	 *
-	 *	CS0 -> MASK0 -> DIMM0
-	 *	CS1 -> MASK1 -> DIMM0
-	 *	CS2 -> MASK2 -> DIMM1
-	 *	CS3 -> MASK3 -> DIMM1
-	 *
-	 * Keep the mask number equal to the Chip Select number for newer systems,
-	 * and shift the mask number for older systems.
-	 */
-	dimm = csrow_nr >> 1;
-
-	if (!fam_type->flags.zn_regs_v2)
-		cs_mask_nr >>= 1;
-
-	/* Asymmetric dual-rank DIMM support. */
-	if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
-		addr_mask_orig = pvt->csels[umc].csmasks_sec[cs_mask_nr];
-	else
-		addr_mask_orig = pvt->csels[umc].csmasks[cs_mask_nr];
-
-	/*
-	 * The number of zero bits in the mask is equal to the number of bits
-	 * in a full mask minus the number of bits in the current mask.
-	 *
-	 * The MSB is the number of bits in the full mask because BIT[0] is
-	 * always 0.
-	 *
-	 * In the special 3 Rank interleaving case, a single bit is flipped
-	 * without swapping with the most significant bit. This can be handled
-	 * by keeping the MSB where it is and ignoring the single zero bit.
-	 */
-	msb = fls(addr_mask_orig) - 1;
-	weight = hweight_long(addr_mask_orig);
-	num_zero_bits = msb - weight - !!(cs_mode & CS_3R_INTERLEAVE);
-
-	/* Take the number of zero bits off from the top of the mask. */
-	addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
-
-	edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm);
-	edac_dbg(1, "  Original AddrMask: 0x%x\n", addr_mask_orig);
-	edac_dbg(1, "  Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved);
-
-	/* Register [31:1] = Address [39:9]. Size is in kBs here. */
-	size = (addr_mask_deinterleaved >> 2) + 1;
-
-	/* Return size in MBs. */
-	return size >> 10;
-}
-
 static void read_dram_ctl_register(struct amd64_pvt *pvt)
 {
 
@@ -2682,196 +2728,6 @@ static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
 }
 
 /*
- * debug routine to display the memory sizes of all logical DIMMs and its
- * CSROWs
- */
-static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
-{
-	int dimm, size0, size1;
-	u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
-	u32 dbam  = ctrl ? pvt->dbam1 : pvt->dbam0;
-
-	if (pvt->fam == 0xf) {
-		/* K8 families < revF not supported yet */
-	       if (pvt->ext_model < K8_REV_F)
-			return;
-	       else
-		       WARN_ON(ctrl != 0);
-	}
-
-	if (pvt->fam == 0x10) {
-		dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
-							   : pvt->dbam0;
-		dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
-				 pvt->csels[1].csbases :
-				 pvt->csels[0].csbases;
-	} else if (ctrl) {
-		dbam = pvt->dbam0;
-		dcsb = pvt->csels[1].csbases;
-	}
-	edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
-		 ctrl, dbam);
-
-	edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
-
-	/* Dump memory sizes for DIMM and its CSROWs */
-	for (dimm = 0; dimm < 4; dimm++) {
-
-		size0 = 0;
-		if (dcsb[dimm*2] & DCSB_CS_ENABLE)
-			/*
-			 * For F15m60h, we need multiplier for LRDIMM cs_size
-			 * calculation. We pass dimm value to the dbam_to_cs
-			 * mapper so we can find the multiplier from the
-			 * corresponding DCSM.
-			 */
-			size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
-						     DBAM_DIMM(dimm, dbam),
-						     dimm);
-
-		size1 = 0;
-		if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
-			size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
-						     DBAM_DIMM(dimm, dbam),
-						     dimm);
-
-		amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
-				dimm * 2,     size0,
-				dimm * 2 + 1, size1);
-	}
-}
-
-static struct amd64_family_type family_types[] = {
-	[K8_CPUS] = {
-		.ctl_name = "K8",
-		.f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
-		.f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
-		.max_mcs = 2,
-		.ops = {
-			.map_sysaddr_to_csrow	= k8_map_sysaddr_to_csrow,
-			.dbam_to_cs		= k8_dbam_to_chip_select,
-		}
-	},
-	[F10_CPUS] = {
-		.ctl_name = "F10h",
-		.f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
-		.f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
-		.max_mcs = 2,
-		.ops = {
-			.map_sysaddr_to_csrow	= f1x_map_sysaddr_to_csrow,
-			.dbam_to_cs		= f10_dbam_to_chip_select,
-		}
-	},
-	[F15_CPUS] = {
-		.ctl_name = "F15h",
-		.f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
-		.f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
-		.max_mcs = 2,
-		.ops = {
-			.map_sysaddr_to_csrow	= f1x_map_sysaddr_to_csrow,
-			.dbam_to_cs		= f15_dbam_to_chip_select,
-		}
-	},
-	[F15_M30H_CPUS] = {
-		.ctl_name = "F15h_M30h",
-		.f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
-		.f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
-		.max_mcs = 2,
-		.ops = {
-			.map_sysaddr_to_csrow	= f1x_map_sysaddr_to_csrow,
-			.dbam_to_cs		= f16_dbam_to_chip_select,
-		}
-	},
-	[F15_M60H_CPUS] = {
-		.ctl_name = "F15h_M60h",
-		.f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
-		.f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
-		.max_mcs = 2,
-		.ops = {
-			.map_sysaddr_to_csrow	= f1x_map_sysaddr_to_csrow,
-			.dbam_to_cs		= f15_m60h_dbam_to_chip_select,
-		}
-	},
-	[F16_CPUS] = {
-		.ctl_name = "F16h",
-		.f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
-		.f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
-		.max_mcs = 2,
-		.ops = {
-			.map_sysaddr_to_csrow	= f1x_map_sysaddr_to_csrow,
-			.dbam_to_cs		= f16_dbam_to_chip_select,
-		}
-	},
-	[F16_M30H_CPUS] = {
-		.ctl_name = "F16h_M30h",
-		.f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
-		.f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
-		.max_mcs = 2,
-		.ops = {
-			.map_sysaddr_to_csrow	= f1x_map_sysaddr_to_csrow,
-			.dbam_to_cs		= f16_dbam_to_chip_select,
-		}
-	},
-	[F17_CPUS] = {
-		.ctl_name = "F17h",
-		.max_mcs = 2,
-		.ops = {
-			.dbam_to_cs		= f17_addr_mask_to_cs_size,
-		}
-	},
-	[F17_M10H_CPUS] = {
-		.ctl_name = "F17h_M10h",
-		.max_mcs = 2,
-		.ops = {
-			.dbam_to_cs		= f17_addr_mask_to_cs_size,
-		}
-	},
-	[F17_M30H_CPUS] = {
-		.ctl_name = "F17h_M30h",
-		.max_mcs = 8,
-		.ops = {
-			.dbam_to_cs		= f17_addr_mask_to_cs_size,
-		}
-	},
-	[F17_M60H_CPUS] = {
-		.ctl_name = "F17h_M60h",
-		.max_mcs = 2,
-		.ops = {
-			.dbam_to_cs		= f17_addr_mask_to_cs_size,
-		}
-	},
-	[F17_M70H_CPUS] = {
-		.ctl_name = "F17h_M70h",
-		.max_mcs = 2,
-		.ops = {
-			.dbam_to_cs		= f17_addr_mask_to_cs_size,
-		}
-	},
-	[F19_CPUS] = {
-		.ctl_name = "F19h",
-		.max_mcs = 8,
-		.ops = {
-			.dbam_to_cs		= f17_addr_mask_to_cs_size,
-		}
-	},
-	[F19_M10H_CPUS] = {
-		.ctl_name = "F19h_M10h",
-		.max_mcs = 12,
-		.flags.zn_regs_v2 = 1,
-		.ops = {
-			.dbam_to_cs		= f17_addr_mask_to_cs_size,
-		}
-	},
-	[F19_M50H_CPUS] = {
-		.ctl_name = "F19h_M50h",
-		.max_mcs = 2,
-		.ops = {
-			.dbam_to_cs		= f17_addr_mask_to_cs_size,
-		}
-	},
-};
-
-/*
  * These are tables of eigenvectors (one per line) which can be used for the
  * construction of the syndrome tables. The modified syndrome search algorithm
  * uses those to find the symbol in error and thus the DIMM.
@@ -3118,10 +2974,14 @@ static inline void decode_bus_error(int node_id, struct mce *m)
  * Currently, we can derive the channel number by looking at the 6th nibble in
  * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel
  * number.
+ *
+ * For DRAM ECC errors, the Chip Select number is given in bits [2:0] of
+ * the MCA_SYND[ErrorInformation] field.
  */
-static int find_umc_channel(struct mce *m)
+static void umc_get_err_info(struct mce *m, struct err_info *err)
 {
-	return (m->ipid & GENMASK(31, 0)) >> 20;
+	err->channel = (m->ipid & GENMASK(31, 0)) >> 20;
+	err->csrow = m->synd & 0x7;
 }
 
 static void decode_umc_error(int node_id, struct mce *m)
@@ -3143,8 +3003,6 @@ static void decode_umc_error(int node_id, struct mce *m)
 	if (m->status & MCI_STATUS_DEFERRED)
 		ecc_type = 3;
 
-	err.channel = find_umc_channel(m);
-
 	if (!(m->status & MCI_STATUS_SYNDV)) {
 		err.err_code = ERR_SYND;
 		goto log_error;
@@ -3159,7 +3017,7 @@ static void decode_umc_error(int node_id, struct mce *m)
 			err.err_code = ERR_CHANNEL;
 	}
 
-	err.csrow = m->synd & 0x7;
+	pvt->ops->get_err_info(m, &err);
 
 	if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
 		err.err_code = ERR_NORM_ADDR;
@@ -3179,9 +3037,6 @@ log_error:
 static int
 reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
 {
-	if (pvt->umc)
-		return 0;
-
 	/* Reserve the ADDRESS MAP Device */
 	pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
 	if (!pvt->F1) {
@@ -3209,36 +3064,11 @@ reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
 	return 0;
 }
 
-static void free_mc_sibling_devs(struct amd64_pvt *pvt)
-{
-	if (pvt->umc) {
-		return;
-	} else {
-		pci_dev_put(pvt->F1);
-		pci_dev_put(pvt->F2);
-	}
-}
-
 static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
 {
 	pvt->ecc_sym_sz = 4;
 
-	if (pvt->umc) {
-		u8 i;
-
-		for_each_umc(i) {
-			/* Check enabled channels only: */
-			if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
-				if (pvt->umc[i].ecc_ctrl & BIT(9)) {
-					pvt->ecc_sym_sz = 16;
-					return;
-				} else if (pvt->umc[i].ecc_ctrl & BIT(7)) {
-					pvt->ecc_sym_sz = 8;
-					return;
-				}
-			}
-		}
-	} else if (pvt->fam >= 0x10) {
+	if (pvt->fam >= 0x10) {
 		u32 tmp;
 
 		amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
@@ -3255,7 +3085,7 @@ static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
 /*
  * Retrieve the hardware registers of the memory controller.
  */
-static void __read_mc_regs_df(struct amd64_pvt *pvt)
+static void umc_read_mc_regs(struct amd64_pvt *pvt)
 {
 	u8 nid = pvt->mc_node_id;
 	struct amd64_umc *umc;
@@ -3267,7 +3097,7 @@ static void __read_mc_regs_df(struct amd64_pvt *pvt)
 		umc_base = get_umc_base(i);
 		umc = &pvt->umc[i];
 
-		amd_smn_read(nid, umc_base + get_umc_reg(UMCCH_DIMM_CFG), &umc->dimm_cfg);
+		amd_smn_read(nid, umc_base + get_umc_reg(pvt, UMCCH_DIMM_CFG), &umc->dimm_cfg);
 		amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
 		amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
 		amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
@@ -3279,7 +3109,7 @@ static void __read_mc_regs_df(struct amd64_pvt *pvt)
  * Retrieve the hardware registers of the memory controller (this includes the
  * 'Address Map' and 'Misc' device regs)
  */
-static void read_mc_regs(struct amd64_pvt *pvt)
+static void dct_read_mc_regs(struct amd64_pvt *pvt)
 {
 	unsigned int range;
 	u64 msr_val;
@@ -3300,12 +3130,6 @@ static void read_mc_regs(struct amd64_pvt *pvt)
 		edac_dbg(0, "  TOP_MEM2 disabled\n");
 	}
 
-	if (pvt->umc) {
-		__read_mc_regs_df(pvt);
-
-		goto skip;
-	}
-
 	amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
 
 	read_dram_ctl_register(pvt);
@@ -3346,14 +3170,6 @@ static void read_mc_regs(struct amd64_pvt *pvt)
 		amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
 	}
 
-skip:
-	read_dct_base_mask(pvt);
-
-	determine_memory_type(pvt);
-
-	if (!pvt->umc)
-		edac_dbg(1, "  DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
-
 	determine_ecc_sym_sz(pvt);
 }
 
@@ -3391,36 +3207,47 @@ skip:
  *	encompasses
  *
  */
-static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
+static u32 dct_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
 {
 	u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
-	int csrow_nr = csrow_nr_orig;
 	u32 cs_mode, nr_pages;
 
-	if (!pvt->umc) {
-		csrow_nr >>= 1;
-		cs_mode = DBAM_DIMM(csrow_nr, dbam);
-	} else {
-		cs_mode = f17_get_cs_mode(csrow_nr >> 1, dct, pvt);
-	}
+	csrow_nr >>= 1;
+	cs_mode = DBAM_DIMM(csrow_nr, dbam);
 
 	nr_pages   = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
 	nr_pages <<= 20 - PAGE_SHIFT;
 
 	edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
-		    csrow_nr_orig, dct,  cs_mode);
+		    csrow_nr, dct,  cs_mode);
 	edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
 
 	return nr_pages;
 }
 
-static int init_csrows_df(struct mem_ctl_info *mci)
+static u32 umc_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
+{
+	int csrow_nr = csrow_nr_orig;
+	u32 cs_mode, nr_pages;
+
+	cs_mode = umc_get_cs_mode(csrow_nr >> 1, dct, pvt);
+
+	nr_pages   = umc_addr_mask_to_cs_size(pvt, dct, cs_mode, csrow_nr);
+	nr_pages <<= 20 - PAGE_SHIFT;
+
+	edac_dbg(0, "csrow: %d, channel: %d, cs_mode %d\n",
+		 csrow_nr_orig, dct,  cs_mode);
+	edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
+
+	return nr_pages;
+}
+
+static void umc_init_csrows(struct mem_ctl_info *mci)
 {
 	struct amd64_pvt *pvt = mci->pvt_info;
 	enum edac_type edac_mode = EDAC_NONE;
 	enum dev_type dev_type = DEV_UNKNOWN;
 	struct dimm_info *dimm;
-	int empty = 1;
 	u8 umc, cs;
 
 	if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) {
@@ -3441,40 +3268,34 @@ static int init_csrows_df(struct mem_ctl_info *mci)
 			if (!csrow_enabled(cs, umc, pvt))
 				continue;
 
-			empty = 0;
 			dimm = mci->csrows[cs]->channels[umc]->dimm;
 
 			edac_dbg(1, "MC node: %d, csrow: %d\n",
 					pvt->mc_node_id, cs);
 
-			dimm->nr_pages = get_csrow_nr_pages(pvt, umc, cs);
+			dimm->nr_pages = umc_get_csrow_nr_pages(pvt, umc, cs);
 			dimm->mtype = pvt->umc[umc].dram_type;
 			dimm->edac_mode = edac_mode;
 			dimm->dtype = dev_type;
 			dimm->grain = 64;
 		}
 	}
-
-	return empty;
 }
 
 /*
  * Initialize the array of csrow attribute instances, based on the values
  * from pci config hardware registers.
  */
-static int init_csrows(struct mem_ctl_info *mci)
+static void dct_init_csrows(struct mem_ctl_info *mci)
 {
 	struct amd64_pvt *pvt = mci->pvt_info;
 	enum edac_type edac_mode = EDAC_NONE;
 	struct csrow_info *csrow;
 	struct dimm_info *dimm;
-	int i, j, empty = 1;
 	int nr_pages = 0;
+	int i, j;
 	u32 val;
 
-	if (pvt->umc)
-		return init_csrows_df(mci);
-
 	amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
 
 	pvt->nbcfg = val;
@@ -3497,19 +3318,18 @@ static int init_csrows(struct mem_ctl_info *mci)
 			continue;
 
 		csrow = mci->csrows[i];
-		empty = 0;
 
 		edac_dbg(1, "MC node: %d, csrow: %d\n",
 			    pvt->mc_node_id, i);
 
 		if (row_dct0) {
-			nr_pages = get_csrow_nr_pages(pvt, 0, i);
+			nr_pages = dct_get_csrow_nr_pages(pvt, 0, i);
 			csrow->channels[0]->dimm->nr_pages = nr_pages;
 		}
 
 		/* K8 has only one DCT */
 		if (pvt->fam != 0xf && row_dct1) {
-			int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
+			int row_dct1_pages = dct_get_csrow_nr_pages(pvt, 1, i);
 
 			csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
 			nr_pages += row_dct1_pages;
@@ -3524,15 +3344,13 @@ static int init_csrows(struct mem_ctl_info *mci)
 					: EDAC_SECDED;
 		}
 
-		for (j = 0; j < fam_type->max_mcs; j++) {
+		for (j = 0; j < pvt->max_mcs; j++) {
 			dimm = csrow->channels[j]->dimm;
 			dimm->mtype = pvt->dram_type;
 			dimm->edac_mode = edac_mode;
 			dimm->grain = 64;
 		}
 	}
-
-	return empty;
 }
 
 /* get all cores on this DCT */
@@ -3695,59 +3513,66 @@ static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
 		amd64_warn("Error restoring NB MCGCTL settings!\n");
 }
 
-static bool ecc_enabled(struct amd64_pvt *pvt)
+static bool dct_ecc_enabled(struct amd64_pvt *pvt)
 {
 	u16 nid = pvt->mc_node_id;
 	bool nb_mce_en = false;
-	u8 ecc_en = 0, i;
+	u8 ecc_en = 0;
 	u32 value;
 
-	if (boot_cpu_data.x86 >= 0x17) {
-		u8 umc_en_mask = 0, ecc_en_mask = 0;
-		struct amd64_umc *umc;
+	amd64_read_pci_cfg(pvt->F3, NBCFG, &value);
 
-		for_each_umc(i) {
-			umc = &pvt->umc[i];
+	ecc_en = !!(value & NBCFG_ECC_ENABLE);
 
-			/* Only check enabled UMCs. */
-			if (!(umc->sdp_ctrl & UMC_SDP_INIT))
-				continue;
+	nb_mce_en = nb_mce_bank_enabled_on_node(nid);
+	if (!nb_mce_en)
+		edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
+			 MSR_IA32_MCG_CTL, nid);
 
-			umc_en_mask |= BIT(i);
+	edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
 
-			if (umc->umc_cap_hi & UMC_ECC_ENABLED)
-				ecc_en_mask |= BIT(i);
-		}
+	if (!ecc_en || !nb_mce_en)
+		return false;
+	else
+		return true;
+}
 
-		/* Check whether at least one UMC is enabled: */
-		if (umc_en_mask)
-			ecc_en = umc_en_mask == ecc_en_mask;
-		else
-			edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
+static bool umc_ecc_enabled(struct amd64_pvt *pvt)
+{
+	u8 umc_en_mask = 0, ecc_en_mask = 0;
+	u16 nid = pvt->mc_node_id;
+	struct amd64_umc *umc;
+	u8 ecc_en = 0, i;
 
-		/* Assume UMC MCA banks are enabled. */
-		nb_mce_en = true;
-	} else {
-		amd64_read_pci_cfg(pvt->F3, NBCFG, &value);
+	for_each_umc(i) {
+		umc = &pvt->umc[i];
 
-		ecc_en = !!(value & NBCFG_ECC_ENABLE);
+		/* Only check enabled UMCs. */
+		if (!(umc->sdp_ctrl & UMC_SDP_INIT))
+			continue;
+
+		umc_en_mask |= BIT(i);
 
-		nb_mce_en = nb_mce_bank_enabled_on_node(nid);
-		if (!nb_mce_en)
-			edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
-				     MSR_IA32_MCG_CTL, nid);
+		if (umc->umc_cap_hi & UMC_ECC_ENABLED)
+			ecc_en_mask |= BIT(i);
 	}
 
+	/* Check whether at least one UMC is enabled: */
+	if (umc_en_mask)
+		ecc_en = umc_en_mask == ecc_en_mask;
+	else
+		edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
+
 	edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
 
-	if (!ecc_en || !nb_mce_en)
+	if (!ecc_en)
 		return false;
 	else
 		return true;
 }
 
 static inline void
-f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
+umc_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
 {
 	u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1;
 
@@ -3777,145 +3602,234 @@ f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
 	}
 }
 
-static void setup_mci_misc_attrs(struct mem_ctl_info *mci)
+static void dct_setup_mci_misc_attrs(struct mem_ctl_info *mci)
 {
 	struct amd64_pvt *pvt = mci->pvt_info;
 
 	mci->mtype_cap		= MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
 	mci->edac_ctl_cap	= EDAC_FLAG_NONE;
 
-	if (pvt->umc) {
-		f17h_determine_edac_ctl_cap(mci, pvt);
-	} else {
-		if (pvt->nbcap & NBCAP_SECDED)
-			mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
+	if (pvt->nbcap & NBCAP_SECDED)
+		mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
 
-		if (pvt->nbcap & NBCAP_CHIPKILL)
-			mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
-	}
+	if (pvt->nbcap & NBCAP_CHIPKILL)
+		mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
 
-	mci->edac_cap		= determine_edac_cap(pvt);
+	mci->edac_cap		= dct_determine_edac_cap(pvt);
 	mci->mod_name		= EDAC_MOD_STR;
-	mci->ctl_name		= fam_type->ctl_name;
+	mci->ctl_name		= pvt->ctl_name;
 	mci->dev_name		= pci_name(pvt->F3);
 	mci->ctl_page_to_phys	= NULL;
 
-	if (pvt->fam >= 0x17)
-		return;
-
 	/* memory scrubber interface */
 	mci->set_sdram_scrub_rate = set_scrub_rate;
 	mci->get_sdram_scrub_rate = get_scrub_rate;
+
+	dct_init_csrows(mci);
 }
 
-/*
- * returns a pointer to the family descriptor on success, NULL otherwise.
- */
-static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
+static void umc_setup_mci_misc_attrs(struct mem_ctl_info *mci)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	mci->mtype_cap		= MEM_FLAG_DDR4 | MEM_FLAG_RDDR4;
+	mci->edac_ctl_cap	= EDAC_FLAG_NONE;
+
+	umc_determine_edac_ctl_cap(mci, pvt);
+
+	mci->edac_cap		= umc_determine_edac_cap(pvt);
+	mci->mod_name		= EDAC_MOD_STR;
+	mci->ctl_name		= pvt->ctl_name;
+	mci->dev_name		= pci_name(pvt->F3);
+	mci->ctl_page_to_phys	= NULL;
+
+	umc_init_csrows(mci);
+}
+
+static int dct_hw_info_get(struct amd64_pvt *pvt)
+{
+	int ret = reserve_mc_sibling_devs(pvt, pvt->f1_id, pvt->f2_id);
+
+	if (ret)
+		return ret;
+
+	dct_prep_chip_selects(pvt);
+	dct_read_base_mask(pvt);
+	dct_read_mc_regs(pvt);
+	dct_determine_memory_type(pvt);
+
+	return 0;
+}
+
+static int umc_hw_info_get(struct amd64_pvt *pvt)
+{
+	pvt->umc = kcalloc(pvt->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
+	if (!pvt->umc)
+		return -ENOMEM;
+
+	umc_prep_chip_selects(pvt);
+	umc_read_base_mask(pvt);
+	umc_read_mc_regs(pvt);
+	umc_determine_memory_type(pvt);
+
+	return 0;
+}
+
+static void hw_info_put(struct amd64_pvt *pvt)
+{
+	pci_dev_put(pvt->F1);
+	pci_dev_put(pvt->F2);
+	kfree(pvt->umc);
+}
+
+static struct low_ops umc_ops = {
+	.hw_info_get			= umc_hw_info_get,
+	.ecc_enabled			= umc_ecc_enabled,
+	.setup_mci_misc_attrs		= umc_setup_mci_misc_attrs,
+	.dump_misc_regs			= umc_dump_misc_regs,
+	.get_err_info			= umc_get_err_info,
+};
+
+/* Use Family 16h versions for defaults and adjust as needed below. */
+static struct low_ops dct_ops = {
+	.map_sysaddr_to_csrow		= f1x_map_sysaddr_to_csrow,
+	.dbam_to_cs			= f16_dbam_to_chip_select,
+	.hw_info_get			= dct_hw_info_get,
+	.ecc_enabled			= dct_ecc_enabled,
+	.setup_mci_misc_attrs		= dct_setup_mci_misc_attrs,
+	.dump_misc_regs			= dct_dump_misc_regs,
+};
+
+static int per_family_init(struct amd64_pvt *pvt)
 {
 	pvt->ext_model  = boot_cpu_data.x86_model >> 4;
 	pvt->stepping	= boot_cpu_data.x86_stepping;
 	pvt->model	= boot_cpu_data.x86_model;
 	pvt->fam	= boot_cpu_data.x86;
+	pvt->max_mcs	= 2;
+
+	/*
+	 * Decide on which ops group to use here and do any family/model
+	 * overrides below.
+	 */
+	if (pvt->fam >= 0x17)
+		pvt->ops = &umc_ops;
+	else
+		pvt->ops = &dct_ops;
 
 	switch (pvt->fam) {
 	case 0xf:
-		fam_type	= &family_types[K8_CPUS];
-		pvt->ops	= &family_types[K8_CPUS].ops;
+		pvt->ctl_name				= (pvt->ext_model >= K8_REV_F) ?
+							  "K8 revF or later" : "K8 revE or earlier";
+		pvt->f1_id				= PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP;
+		pvt->f2_id				= PCI_DEVICE_ID_AMD_K8_NB_MEMCTL;
+		pvt->ops->map_sysaddr_to_csrow		= k8_map_sysaddr_to_csrow;
+		pvt->ops->dbam_to_cs			= k8_dbam_to_chip_select;
 		break;
 
 	case 0x10:
-		fam_type	= &family_types[F10_CPUS];
-		pvt->ops	= &family_types[F10_CPUS].ops;
+		pvt->ctl_name				= "F10h";
+		pvt->f1_id				= PCI_DEVICE_ID_AMD_10H_NB_MAP;
+		pvt->f2_id				= PCI_DEVICE_ID_AMD_10H_NB_DRAM;
+		pvt->ops->dbam_to_cs			= f10_dbam_to_chip_select;
 		break;
 
 	case 0x15:
-		if (pvt->model == 0x30) {
-			fam_type = &family_types[F15_M30H_CPUS];
-			pvt->ops = &family_types[F15_M30H_CPUS].ops;
+		switch (pvt->model) {
+		case 0x30:
+			pvt->ctl_name			= "F15h_M30h";
+			pvt->f1_id			= PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
+			pvt->f2_id			= PCI_DEVICE_ID_AMD_15H_M30H_NB_F2;
 			break;
-		} else if (pvt->model == 0x60) {
-			fam_type = &family_types[F15_M60H_CPUS];
-			pvt->ops = &family_types[F15_M60H_CPUS].ops;
+		case 0x60:
+			pvt->ctl_name			= "F15h_M60h";
+			pvt->f1_id			= PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
+			pvt->f2_id			= PCI_DEVICE_ID_AMD_15H_M60H_NB_F2;
+			pvt->ops->dbam_to_cs		= f15_m60h_dbam_to_chip_select;
+			break;
+		case 0x13:
+			/* Richland is only client */
+			return -ENODEV;
+		default:
+			pvt->ctl_name			= "F15h";
+			pvt->f1_id			= PCI_DEVICE_ID_AMD_15H_NB_F1;
+			pvt->f2_id			= PCI_DEVICE_ID_AMD_15H_NB_F2;
+			pvt->ops->dbam_to_cs		= f15_dbam_to_chip_select;
 			break;
-		/* Richland is only client */
-		} else if (pvt->model == 0x13) {
-			return NULL;
-		} else {
-			fam_type	= &family_types[F15_CPUS];
-			pvt->ops	= &family_types[F15_CPUS].ops;
 		}
 		break;
 
 	case 0x16:
-		if (pvt->model == 0x30) {
-			fam_type = &family_types[F16_M30H_CPUS];
-			pvt->ops = &family_types[F16_M30H_CPUS].ops;
+		switch (pvt->model) {
+		case 0x30:
+			pvt->ctl_name			= "F16h_M30h";
+			pvt->f1_id			= PCI_DEVICE_ID_AMD_16H_M30H_NB_F1;
+			pvt->f2_id			= PCI_DEVICE_ID_AMD_16H_M30H_NB_F2;
+			break;
+		default:
+			pvt->ctl_name			= "F16h";
+			pvt->f1_id			= PCI_DEVICE_ID_AMD_16H_NB_F1;
+			pvt->f2_id			= PCI_DEVICE_ID_AMD_16H_NB_F2;
 			break;
 		}
-		fam_type	= &family_types[F16_CPUS];
-		pvt->ops	= &family_types[F16_CPUS].ops;
 		break;
 
 	case 0x17:
-		if (pvt->model >= 0x10 && pvt->model <= 0x2f) {
-			fam_type = &family_types[F17_M10H_CPUS];
-			pvt->ops = &family_types[F17_M10H_CPUS].ops;
+		switch (pvt->model) {
+		case 0x10 ... 0x2f:
+			pvt->ctl_name			= "F17h_M10h";
 			break;
-		} else if (pvt->model >= 0x30 && pvt->model <= 0x3f) {
-			fam_type = &family_types[F17_M30H_CPUS];
-			pvt->ops = &family_types[F17_M30H_CPUS].ops;
+		case 0x30 ... 0x3f:
+			pvt->ctl_name			= "F17h_M30h";
+			pvt->max_mcs			= 8;
 			break;
-		} else if (pvt->model >= 0x60 && pvt->model <= 0x6f) {
-			fam_type = &family_types[F17_M60H_CPUS];
-			pvt->ops = &family_types[F17_M60H_CPUS].ops;
+		case 0x60 ... 0x6f:
+			pvt->ctl_name			= "F17h_M60h";
 			break;
-		} else if (pvt->model >= 0x70 && pvt->model <= 0x7f) {
-			fam_type = &family_types[F17_M70H_CPUS];
-			pvt->ops = &family_types[F17_M70H_CPUS].ops;
+		case 0x70 ... 0x7f:
+			pvt->ctl_name			= "F17h_M70h";
+			break;
+		default:
+			pvt->ctl_name			= "F17h";
 			break;
 		}
-		fallthrough;
-	case 0x18:
-		fam_type	= &family_types[F17_CPUS];
-		pvt->ops	= &family_types[F17_CPUS].ops;
+		break;
 
-		if (pvt->fam == 0x18)
-			family_types[F17_CPUS].ctl_name = "F18h";
+	case 0x18:
+		pvt->ctl_name				= "F18h";
 		break;
 
 	case 0x19:
-		if (pvt->model >= 0x10 && pvt->model <= 0x1f) {
-			fam_type = &family_types[F19_M10H_CPUS];
-			pvt->ops = &family_types[F19_M10H_CPUS].ops;
+		switch (pvt->model) {
+		case 0x00 ... 0x0f:
+			pvt->ctl_name			= "F19h";
+			pvt->max_mcs			= 8;
 			break;
-		} else if (pvt->model >= 0x20 && pvt->model <= 0x2f) {
-			fam_type = &family_types[F17_M70H_CPUS];
-			pvt->ops = &family_types[F17_M70H_CPUS].ops;
-			fam_type->ctl_name = "F19h_M20h";
+		case 0x10 ... 0x1f:
+			pvt->ctl_name			= "F19h_M10h";
+			pvt->max_mcs			= 12;
+			pvt->flags.zn_regs_v2		= 1;
 			break;
-		} else if (pvt->model >= 0x50 && pvt->model <= 0x5f) {
-			fam_type = &family_types[F19_M50H_CPUS];
-			pvt->ops = &family_types[F19_M50H_CPUS].ops;
-			fam_type->ctl_name = "F19h_M50h";
+		case 0x20 ... 0x2f:
+			pvt->ctl_name			= "F19h_M20h";
 			break;
-		} else if (pvt->model >= 0xa0 && pvt->model <= 0xaf) {
-			fam_type = &family_types[F19_M10H_CPUS];
-			pvt->ops = &family_types[F19_M10H_CPUS].ops;
-			fam_type->ctl_name = "F19h_MA0h";
+		case 0x50 ... 0x5f:
+			pvt->ctl_name			= "F19h_M50h";
+			break;
+		case 0xa0 ... 0xaf:
+			pvt->ctl_name			= "F19h_MA0h";
+			pvt->max_mcs			= 12;
+			pvt->flags.zn_regs_v2		= 1;
 			break;
 		}
-		fam_type	= &family_types[F19_CPUS];
-		pvt->ops	= &family_types[F19_CPUS].ops;
-		family_types[F19_CPUS].ctl_name = "F19h";
 		break;
 
 	default:
 		amd64_err("Unsupported family!\n");
-		return NULL;
+		return -ENODEV;
 	}
 
-	return fam_type;
+	return 0;
 }
 
 static const struct attribute_group *amd64_edac_attr_groups[] = {
@@ -3926,37 +3840,6 @@ static const struct attribute_group *amd64_edac_attr_groups[] = {
 	NULL
 };
 
-static int hw_info_get(struct amd64_pvt *pvt)
-{
-	u16 pci_id1 = 0, pci_id2 = 0;
-	int ret;
-
-	if (pvt->fam >= 0x17) {
-		pvt->umc = kcalloc(fam_type->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
-		if (!pvt->umc)
-			return -ENOMEM;
-	} else {
-		pci_id1 = fam_type->f1_id;
-		pci_id2 = fam_type->f2_id;
-	}
-
-	ret = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
-	if (ret)
-		return ret;
-
-	read_mc_regs(pvt);
-
-	return 0;
-}
-
-static void hw_info_put(struct amd64_pvt *pvt)
-{
-	if (pvt->F1)
-		free_mc_sibling_devs(pvt);
-
-	kfree(pvt->umc);
-}
-
 static int init_one_instance(struct amd64_pvt *pvt)
 {
 	struct mem_ctl_info *mci = NULL;
@@ -3967,7 +3850,7 @@ static int init_one_instance(struct amd64_pvt *pvt)
 	layers[0].size = pvt->csels[0].b_cnt;
 	layers[0].is_virt_csrow = true;
 	layers[1].type = EDAC_MC_LAYER_CHANNEL;
-	layers[1].size = fam_type->max_mcs;
+	layers[1].size = pvt->max_mcs;
 	layers[1].is_virt_csrow = false;
 
 	mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0);
@@ -3977,10 +3860,7 @@ static int init_one_instance(struct amd64_pvt *pvt)
 	mci->pvt_info = pvt;
 	mci->pdev = &pvt->F3->dev;
 
-	setup_mci_misc_attrs(mci);
-
-	if (init_csrows(mci))
-		mci->edac_cap = EDAC_FLAG_NONE;
+	pvt->ops->setup_mci_misc_attrs(mci);
 
 	ret = -ENODEV;
 	if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
@@ -3997,7 +3877,7 @@ static bool instance_has_memory(struct amd64_pvt *pvt)
 	bool cs_enabled = false;
 	int cs = 0, dct = 0;
 
-	for (dct = 0; dct < fam_type->max_mcs; dct++) {
+	for (dct = 0; dct < pvt->max_mcs; dct++) {
 		for_each_chip_select(cs, dct, pvt)
 			cs_enabled |= csrow_enabled(cs, dct, pvt);
 	}
@@ -4026,12 +3906,11 @@ static int probe_one_instance(unsigned int nid)
 	pvt->mc_node_id	= nid;
 	pvt->F3 = F3;
 
-	ret = -ENODEV;
-	fam_type = per_family_init(pvt);
-	if (!fam_type)
+	ret = per_family_init(pvt);
+	if (ret < 0)
 		goto err_enable;
 
-	ret = hw_info_get(pvt);
+	ret = pvt->ops->hw_info_get(pvt);
 	if (ret < 0)
 		goto err_enable;
 
@@ -4041,7 +3920,7 @@ static int probe_one_instance(unsigned int nid)
 		goto err_enable;
 	}
 
-	if (!ecc_enabled(pvt)) {
+	if (!pvt->ops->ecc_enabled(pvt)) {
 		ret = -ENODEV;
 
 		if (!ecc_enable_override)
@@ -4067,13 +3946,10 @@ static int probe_one_instance(unsigned int nid)
 		goto err_enable;
 	}
 
-	amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
-		     (pvt->fam == 0xf ?
-				(pvt->ext_model >= K8_REV_F  ? "revF or later "
-							     : "revE or earlier ")
-				 : ""), pvt->mc_node_id);
+	amd64_info("%s detected (node %d).\n", pvt->ctl_name, pvt->mc_node_id);
 
-	dump_misc_regs(pvt);
+	/* Display and decode various registers for debug purposes. */
+	pvt->ops->dump_misc_regs(pvt);
 
 	return ret;
 
@@ -4244,10 +4120,8 @@ module_init(amd64_edac_init);
 module_exit(amd64_edac_exit);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
-		"Dave Peterson, Thayne Harbaugh");
-MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
-		EDAC_AMD64_VERSION);
+MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, Dave Peterson, Thayne Harbaugh; AMD");
+MODULE_DESCRIPTION("MC support for AMD64 memory controllers - " EDAC_AMD64_VERSION);
 
 module_param(edac_op_state, int, 0444);
 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h
index e4329dff8cf2..e84fe0d4120a 100644
--- a/drivers/edac/amd64_edac.h
+++ b/drivers/edac/amd64_edac.h
@@ -273,25 +273,6 @@
 
 #define UMC_SDP_INIT			BIT(31)
 
-enum amd_families {
-	K8_CPUS = 0,
-	F10_CPUS,
-	F15_CPUS,
-	F15_M30H_CPUS,
-	F15_M60H_CPUS,
-	F16_CPUS,
-	F16_M30H_CPUS,
-	F17_CPUS,
-	F17_M10H_CPUS,
-	F17_M30H_CPUS,
-	F17_M60H_CPUS,
-	F17_M70H_CPUS,
-	F19_CPUS,
-	F19_M10H_CPUS,
-	F19_M50H_CPUS,
-	NUM_FAMILIES,
-};
-
 /* Error injection control structure */
 struct error_injection {
 	u32	 section;
@@ -334,6 +315,16 @@ struct amd64_umc {
 	enum mem_type dram_type;
 };
 
+struct amd64_family_flags {
+	/*
+	 * Indicates that the system supports the new register offsets, etc.
+	 * first introduced with Family 19h Model 10h.
+	 */
+	__u64 zn_regs_v2	: 1,
+
+	      __reserved	: 63;
+};
+
 struct amd64_pvt {
 	struct low_ops *ops;
 
@@ -375,6 +366,12 @@ struct amd64_pvt {
 	/* x4, x8, or x16 syndromes in use */
 	u8 ecc_sym_sz;
 
+	const char *ctl_name;
+	u16 f1_id, f2_id;
+	/* Maximum number of memory controllers per die/node. */
+	u8 max_mcs;
+
+	struct amd64_family_flags flags;
 	/* place to store error injection parameters prior to issue */
 	struct error_injection injection;
 
@@ -465,29 +462,15 @@ struct ecc_settings {
  * functions and per device encoding/decoding logic.
  */
 struct low_ops {
-	void (*map_sysaddr_to_csrow)	(struct mem_ctl_info *mci, u64 sys_addr,
-					 struct err_info *);
-	int (*dbam_to_cs)		(struct amd64_pvt *pvt, u8 dct,
-					 unsigned cs_mode, int cs_mask_nr);
-};
-
-struct amd64_family_flags {
-	/*
-	 * Indicates that the system supports the new register offsets, etc.
-	 * first introduced with Family 19h Model 10h.
-	 */
-	__u64 zn_regs_v2	: 1,
-
-	      __reserved	: 63;
-};
-
-struct amd64_family_type {
-	const char *ctl_name;
-	u16 f1_id, f2_id;
-	/* Maximum number of memory controllers per die/node. */
-	u8 max_mcs;
-	struct amd64_family_flags flags;
-	struct low_ops ops;
+	void (*map_sysaddr_to_csrow)(struct mem_ctl_info *mci, u64 sys_addr,
+				     struct err_info *err);
+	int  (*dbam_to_cs)(struct amd64_pvt *pvt, u8 dct,
+			   unsigned int cs_mode, int cs_mask_nr);
+	int (*hw_info_get)(struct amd64_pvt *pvt);
+	bool (*ecc_enabled)(struct amd64_pvt *pvt);
+	void (*setup_mci_misc_attrs)(struct mem_ctl_info *mci);
+	void (*dump_misc_regs)(struct amd64_pvt *pvt);
+	void (*get_err_info)(struct mce *m, struct err_info *err);
 };
 
 int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
diff --git a/drivers/edac/amd8111_edac.c b/drivers/edac/amd8111_edac.c
index 7508aa416ddb..ca718f63fcbc 100644
--- a/drivers/edac/amd8111_edac.c
+++ b/drivers/edac/amd8111_edac.c
@@ -593,5 +593,5 @@ module_init(amd8111_edac_init);
 module_exit(amd8111_edac_exit);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Cao Qingtao <qingtao.cao@windriver.com>\n");
+MODULE_AUTHOR("Cao Qingtao <qingtao.cao@windriver.com>");
 MODULE_DESCRIPTION("AMD8111 HyperTransport I/O Hub EDAC kernel module");
diff --git a/drivers/edac/amd8131_edac.c b/drivers/edac/amd8131_edac.c
index 169353710982..28610ba514f4 100644
--- a/drivers/edac/amd8131_edac.c
+++ b/drivers/edac/amd8131_edac.c
@@ -354,5 +354,5 @@ module_init(amd8131_edac_init);
 module_exit(amd8131_edac_exit);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Cao Qingtao <qingtao.cao@windriver.com>\n");
+MODULE_AUTHOR("Cao Qingtao <qingtao.cao@windriver.com>");
 MODULE_DESCRIPTION("AMD8131 HyperTransport PCI-X Tunnel EDAC kernel module");
diff --git a/drivers/edac/e752x_edac.c b/drivers/edac/e752x_edac.c
index ac7c9b42d4c7..7221b4bb6df2 100644
--- a/drivers/edac/e752x_edac.c
+++ b/drivers/edac/e752x_edac.c
@@ -1462,7 +1462,7 @@ module_init(e752x_init);
 module_exit(e752x_exit);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Linux Networx (http://lnxi.com) Tom Zimmerman\n");
+MODULE_AUTHOR("Linux Networx (http://lnxi.com) Tom Zimmerman");
 MODULE_DESCRIPTION("MC support for Intel e752x/3100 memory controllers");
 
 module_param(force_function_unhide, int, 0444);
diff --git a/drivers/edac/e7xxx_edac.c b/drivers/edac/e7xxx_edac.c
index 497e710fca3d..5852b95fa470 100644
--- a/drivers/edac/e7xxx_edac.c
+++ b/drivers/edac/e7xxx_edac.c
@@ -596,8 +596,7 @@ module_init(e7xxx_init);
 module_exit(e7xxx_exit);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Linux Networx (http://lnxi.com) Thayne Harbaugh et al\n"
-		"Based on.work by Dan Hollis et al");
+MODULE_AUTHOR("Linux Networx (http://lnxi.com) Thayne Harbaugh et al");
 MODULE_DESCRIPTION("MC support for Intel e7xxx memory controllers");
 module_param(edac_op_state, int, 0444);
 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
diff --git a/drivers/edac/edac_device_sysfs.c b/drivers/edac/edac_device_sysfs.c
index ac678b4a21fc..010c26be5846 100644
--- a/drivers/edac/edac_device_sysfs.c
+++ b/drivers/edac/edac_device_sysfs.c
@@ -228,8 +228,9 @@ static struct kobj_type ktype_device_ctrl = {
  */
 int edac_device_register_sysfs_main_kobj(struct edac_device_ctl_info *edac_dev)
 {
+	struct device *dev_root;
 	struct bus_type *edac_subsys;
-	int err;
+	int err = -ENODEV;
 
 	edac_dbg(1, "\n");
 
@@ -247,15 +248,16 @@ int edac_device_register_sysfs_main_kobj(struct edac_device_ctl_info *edac_dev)
 	 */
 	edac_dev->owner = THIS_MODULE;
 
-	if (!try_module_get(edac_dev->owner)) {
-		err = -ENODEV;
+	if (!try_module_get(edac_dev->owner))
 		goto err_out;
-	}
 
 	/* register */
-	err = kobject_init_and_add(&edac_dev->kobj, &ktype_device_ctrl,
-				   &edac_subsys->dev_root->kobj,
-				   "%s", edac_dev->name);
+	dev_root = bus_get_dev_root(edac_subsys);
+	if (dev_root) {
+		err = kobject_init_and_add(&edac_dev->kobj, &ktype_device_ctrl,
+					   &dev_root->kobj, "%s", edac_dev->name);
+		put_device(dev_root);
+	}
 	if (err) {
 		edac_dbg(1, "Failed to register '.../edac/%s'\n",
 			 edac_dev->name);
diff --git a/drivers/edac/edac_pci_sysfs.c b/drivers/edac/edac_pci_sysfs.c
index 888d5728ecef..287cc51dbc86 100644
--- a/drivers/edac/edac_pci_sysfs.c
+++ b/drivers/edac/edac_pci_sysfs.c
@@ -337,8 +337,9 @@ static struct kobj_type ktype_edac_pci_main_kobj = {
  */
 static int edac_pci_main_kobj_setup(void)
 {
-	int err;
+	int err = -ENODEV;
 	struct bus_type *edac_subsys;
+	struct device *dev_root;
 
 	edac_dbg(0, "\n");
 
@@ -357,7 +358,6 @@ static int edac_pci_main_kobj_setup(void)
 	 */
 	if (!try_module_get(THIS_MODULE)) {
 		edac_dbg(1, "try_module_get() failed\n");
-		err = -ENODEV;
 		goto decrement_count_fail;
 	}
 
@@ -369,9 +369,13 @@ static int edac_pci_main_kobj_setup(void)
 	}
 
 	/* Instanstiate the pci object */
-	err = kobject_init_and_add(edac_pci_top_main_kobj,
-				   &ktype_edac_pci_main_kobj,
-				   &edac_subsys->dev_root->kobj, "pci");
+	dev_root = bus_get_dev_root(edac_subsys);
+	if (dev_root) {
+		err = kobject_init_and_add(edac_pci_top_main_kobj,
+					   &ktype_edac_pci_main_kobj,
+					   &dev_root->kobj, "pci");
+		put_device(dev_root);
+	}
 	if (err) {
 		edac_dbg(1, "Failed to register '.../edac/pci'\n");
 		goto kobject_init_and_add_fail;
diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c
index 0a4691792801..a897b6aff368 100644
--- a/drivers/edac/i10nm_base.c
+++ b/drivers/edac/i10nm_base.c
@@ -906,6 +906,7 @@ static const struct x86_cpu_id i10nm_cpuids[] = {
 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SAPPHIRERAPIDS_X,	X86_STEPPINGS(0x0, 0xf), &spr_cfg),
 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(EMERALDRAPIDS_X,	X86_STEPPINGS(0x0, 0xf), &spr_cfg),
 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(GRANITERAPIDS_X,	X86_STEPPINGS(0x0, 0xf), &gnr_cfg),
+	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SIERRAFOREST_X,	X86_STEPPINGS(0x0, 0xf), &gnr_cfg),
 	{}
 };
 MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids);
diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c
index ba46057d4220..4b5a71f8739d 100644
--- a/drivers/edac/i5000_edac.c
+++ b/drivers/edac/i5000_edac.c
@@ -1573,13 +1573,10 @@ module_init(i5000_init);
 module_exit(i5000_exit);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR
-    ("Linux Networx (http://lnxi.com) Doug Thompson <norsk5@xmission.com>");
-MODULE_DESCRIPTION("MC Driver for Intel I5000 memory controllers - "
-		I5000_REVISION);
+MODULE_AUTHOR("Linux Networx (http://lnxi.com) Doug Thompson <norsk5@xmission.com>");
+MODULE_DESCRIPTION("MC Driver for Intel I5000 memory controllers - " I5000_REVISION);
 
 module_param(edac_op_state, int, 0444);
 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
 module_param(misc_messages, int, 0444);
 MODULE_PARM_DESC(misc_messages, "Log miscellaneous non fatal messages");
-
diff --git a/drivers/edac/i5100_edac.c b/drivers/edac/i5100_edac.c
index f5d82518c15e..d470afe65001 100644
--- a/drivers/edac/i5100_edac.c
+++ b/drivers/edac/i5100_edac.c
@@ -909,7 +909,7 @@ static void i5100_do_inject(struct mem_ctl_info *mci)
 	 *
 	 * The injection code don't work without setting this register.
 	 * The register needs to be flipped off then on else the hardware
-	 * will only preform the first injection.
+	 * will only perform the first injection.
 	 *
 	 * Stop condition bits 7:4
 	 * 1010 - Stop after one injection
@@ -1220,6 +1220,5 @@ module_init(i5100_init);
 module_exit(i5100_exit);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR
-    ("Arthur Jones <ajones@riverbed.com>");
+MODULE_AUTHOR("Arthur Jones <ajones@riverbed.com>");
 MODULE_DESCRIPTION("MC Driver for Intel I5100 memory controllers");
diff --git a/drivers/edac/i82860_edac.c b/drivers/edac/i82860_edac.c
index fbec90d00f1e..b8a497f0de28 100644
--- a/drivers/edac/i82860_edac.c
+++ b/drivers/edac/i82860_edac.c
@@ -355,8 +355,7 @@ module_init(i82860_init);
 module_exit(i82860_exit);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com) "
-		"Ben Woodard <woodard@redhat.com>");
+MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com) Ben Woodard <woodard@redhat.com>");
 MODULE_DESCRIPTION("ECC support for Intel 82860 memory hub controllers");
 
 module_param(edac_op_state, int, 0444);
diff --git a/drivers/edac/layerscape_edac.c b/drivers/edac/layerscape_edac.c
index 35ceaca578e1..7c5e2b3c0daa 100644
--- a/drivers/edac/layerscape_edac.c
+++ b/drivers/edac/layerscape_edac.c
@@ -72,5 +72,4 @@ module_exit(fsl_ddr_mc_exit);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("NXP Semiconductor");
 module_param(edac_op_state, int, 0444);
-MODULE_PARM_DESC(edac_op_state,
-		 "EDAC Error Reporting state: 0=Poll, 2=Interrupt");
+MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll, 2=Interrupt");
diff --git a/drivers/edac/mpc85xx_edac.c b/drivers/edac/mpc85xx_edac.c
index e50d7928bf8f..55320546c174 100644
--- a/drivers/edac/mpc85xx_edac.c
+++ b/drivers/edac/mpc85xx_edac.c
@@ -711,5 +711,4 @@ module_exit(mpc85xx_mc_exit);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Montavista Software, Inc.");
 module_param(edac_op_state, int, 0444);
-MODULE_PARM_DESC(edac_op_state,
-		 "EDAC Error Reporting state: 0=Poll, 2=Interrupt");
+MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll, 2=Interrupt");
diff --git a/drivers/edac/qcom_edac.c b/drivers/edac/qcom_edac.c
index 3256254c3722..265e0fb39bc7 100644
--- a/drivers/edac/qcom_edac.c
+++ b/drivers/edac/qcom_edac.c
@@ -76,6 +76,8 @@
 #define DRP0_INTERRUPT_ENABLE           BIT(6)
 #define SB_DB_DRP_INTERRUPT_ENABLE      0x3
 
+#define ECC_POLL_MSEC			5000
+
 enum {
 	LLCC_DRAM_CE = 0,
 	LLCC_DRAM_UE,
@@ -213,7 +215,7 @@ dump_syn_reg_values(struct llcc_drv_data *drv, u32 bank, int err_type)
 
 	for (i = 0; i < reg_data.reg_cnt; i++) {
 		synd_reg = reg_data.synd_reg + (i * 4);
-		ret = regmap_read(drv->regmap, drv->offsets[bank] + synd_reg,
+		ret = regmap_read(drv->regmaps[bank], synd_reg,
 				  &synd_val);
 		if (ret)
 			goto clear;
@@ -222,8 +224,7 @@ dump_syn_reg_values(struct llcc_drv_data *drv, u32 bank, int err_type)
 			    reg_data.name, i, synd_val);
 	}
 
-	ret = regmap_read(drv->regmap,
-			  drv->offsets[bank] + reg_data.count_status_reg,
+	ret = regmap_read(drv->regmaps[bank], reg_data.count_status_reg,
 			  &err_cnt);
 	if (ret)
 		goto clear;
@@ -233,8 +234,7 @@ dump_syn_reg_values(struct llcc_drv_data *drv, u32 bank, int err_type)
 	edac_printk(KERN_CRIT, EDAC_LLCC, "%s: Error count: 0x%4x\n",
 		    reg_data.name, err_cnt);
 
-	ret = regmap_read(drv->regmap,
-			  drv->offsets[bank] + reg_data.ways_status_reg,
+	ret = regmap_read(drv->regmaps[bank], reg_data.ways_status_reg,
 			  &err_ways);
 	if (ret)
 		goto clear;
@@ -285,8 +285,7 @@ dump_syn_reg(struct edac_device_ctl_info *edev_ctl, int err_type, u32 bank)
 	return ret;
 }
 
-static irqreturn_t
-llcc_ecc_irq_handler(int irq, void *edev_ctl)
+static irqreturn_t llcc_ecc_irq_handler(int irq, void *edev_ctl)
 {
 	struct edac_device_ctl_info *edac_dev_ctl = edev_ctl;
 	struct llcc_drv_data *drv = edac_dev_ctl->dev->platform_data;
@@ -296,8 +295,7 @@ llcc_ecc_irq_handler(int irq, void *edev_ctl)
 
 	/* Iterate over the banks and look for Tag RAM or Data RAM errors */
 	for (i = 0; i < drv->num_banks; i++) {
-		ret = regmap_read(drv->regmap,
-				  drv->offsets[i] + DRP_INTERRUPT_STATUS,
+		ret = regmap_read(drv->regmaps[i], DRP_INTERRUPT_STATUS,
 				  &drp_error);
 
 		if (!ret && (drp_error & SB_ECC_ERROR)) {
@@ -312,8 +310,7 @@ llcc_ecc_irq_handler(int irq, void *edev_ctl)
 		if (!ret)
 			irq_rc = IRQ_HANDLED;
 
-		ret = regmap_read(drv->regmap,
-				  drv->offsets[i] + TRP_INTERRUPT_0_STATUS,
+		ret = regmap_read(drv->regmaps[i], TRP_INTERRUPT_0_STATUS,
 				  &trp_error);
 
 		if (!ret && (trp_error & SB_ECC_ERROR)) {
@@ -332,6 +329,11 @@ llcc_ecc_irq_handler(int irq, void *edev_ctl)
 	return irq_rc;
 }
 
+static void llcc_ecc_check(struct edac_device_ctl_info *edev_ctl)
+{
+	llcc_ecc_irq_handler(0, edev_ctl);
+}
+
 static int qcom_llcc_edac_probe(struct platform_device *pdev)
 {
 	struct llcc_drv_data *llcc_driv_data = pdev->dev.platform_data;
@@ -359,29 +361,31 @@ static int qcom_llcc_edac_probe(struct platform_device *pdev)
 	edev_ctl->ctl_name = "llcc";
 	edev_ctl->panic_on_ue = LLCC_ERP_PANIC_ON_UE;
 
-	rc = edac_device_add_device(edev_ctl);
-	if (rc)
-		goto out_mem;
-
-	platform_set_drvdata(pdev, edev_ctl);
-
-	/* Request for ecc irq */
+	/* Check if LLCC driver has passed ECC IRQ */
 	ecc_irq = llcc_driv_data->ecc_irq;
-	if (ecc_irq < 0) {
-		rc = -ENODEV;
-		goto out_dev;
-	}
-	rc = devm_request_irq(dev, ecc_irq, llcc_ecc_irq_handler,
+	if (ecc_irq > 0) {
+		/* Use interrupt mode if IRQ is available */
+		rc = devm_request_irq(dev, ecc_irq, llcc_ecc_irq_handler,
 			      IRQF_TRIGGER_HIGH, "llcc_ecc", edev_ctl);
-	if (rc)
-		goto out_dev;
+		if (!rc) {
+			edac_op_state = EDAC_OPSTATE_INT;
+			goto irq_done;
+		}
+	}
 
-	return rc;
+	/* Fall back to polling mode otherwise */
+	edev_ctl->poll_msec = ECC_POLL_MSEC;
+	edev_ctl->edac_check = llcc_ecc_check;
+	edac_op_state = EDAC_OPSTATE_POLL;
 
-out_dev:
-	edac_device_del_device(edev_ctl->dev);
-out_mem:
-	edac_device_free_ctl_info(edev_ctl);
+irq_done:
+	rc = edac_device_add_device(edev_ctl);
+	if (rc) {
+		edac_device_free_ctl_info(edev_ctl);
+		return rc;
+	}
+
+	platform_set_drvdata(pdev, edev_ctl);
 
 	return rc;
 }
diff --git a/drivers/edac/r82600_edac.c b/drivers/edac/r82600_edac.c
index d0aef83dca2a..61e979d5437a 100644
--- a/drivers/edac/r82600_edac.c
+++ b/drivers/edac/r82600_edac.c
@@ -415,8 +415,7 @@ module_init(r82600_init);
 module_exit(r82600_exit);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Tim Small <tim@buttersideup.com> - WPAD Ltd. "
-		"on behalf of EADS Astrium");
+MODULE_AUTHOR("Tim Small <tim@buttersideup.com> - WPAD Ltd. on behalf of EADS Astrium");
 MODULE_DESCRIPTION("MC support for Radisys 82600 memory controllers");
 
 module_param(disable_hardware_scrub, bool, 0644);
diff --git a/drivers/edac/skx_base.c b/drivers/edac/skx_base.c
index 9397abb42c49..0a862336a7ce 100644
--- a/drivers/edac/skx_base.c
+++ b/drivers/edac/skx_base.c
@@ -510,7 +510,7 @@ rir_found:
 }
 
 static u8 skx_close_row[] = {
-	15, 16, 17, 18, 20, 21, 22, 28, 10, 11, 12, 13, 29, 30, 31, 32, 33
+	15, 16, 17, 18, 20, 21, 22, 28, 10, 11, 12, 13, 29, 30, 31, 32, 33, 34
 };
 
 static u8 skx_close_column[] = {
@@ -518,7 +518,7 @@ static u8 skx_close_column[] = {
 };
 
 static u8 skx_open_row[] = {
-	14, 15, 16, 20, 28, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31, 32, 33
+	14, 15, 16, 20, 28, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31, 32, 33, 34
 };
 
 static u8 skx_open_column[] = {
diff --git a/drivers/eisa/pci_eisa.c b/drivers/eisa/pci_eisa.c
index 930c2332c3c4..8173e60bb808 100644
--- a/drivers/eisa/pci_eisa.c
+++ b/drivers/eisa/pci_eisa.c
@@ -20,8 +20,8 @@ static struct eisa_root_device pci_eisa_root;
 
 static int __init pci_eisa_init(struct pci_dev *pdev)
 {
-	int rc, i;
 	struct resource *res, *bus_res = NULL;
+	int rc;
 
 	if ((rc = pci_enable_device (pdev))) {
 		dev_err(&pdev->dev, "Could not enable device\n");
@@ -38,7 +38,7 @@ static int __init pci_eisa_init(struct pci_dev *pdev)
 	 * eisa_root_register() can only deal with a single io port resource,
 	*  so we use the first valid io port resource.
 	 */
-	pci_bus_for_each_resource(pdev->bus, res, i)
+	pci_bus_for_each_resource(pdev->bus, res)
 		if (res && (res->flags & IORESOURCE_IO)) {
 			bus_res = res;
 			break;
diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c
index e1c71359b605..d43ba8e7260d 100644
--- a/drivers/extcon/extcon.c
+++ b/drivers/extcon/extcon.c
@@ -1013,7 +1013,7 @@ ATTRIBUTE_GROUPS(extcon);
 static int create_extcon_class(void)
 {
 	if (!extcon_class) {
-		extcon_class = class_create(THIS_MODULE, "extcon");
+		extcon_class = class_create("extcon");
 		if (IS_ERR(extcon_class))
 			return PTR_ERR(extcon_class);
 		extcon_class->dev_groups = extcon_groups;
diff --git a/drivers/firewire/init_ohci1394_dma.c b/drivers/firewire/init_ohci1394_dma.c
index 659256927b42..48b879e9e831 100644
--- a/drivers/firewire/init_ohci1394_dma.c
+++ b/drivers/firewire/init_ohci1394_dma.c
@@ -251,7 +251,7 @@ static inline void __init init_ohci1394_controller(int num, int slot, int func)
 }
 
 /**
- * debug_init_ohci1394_dma - scan for OHCI1394 controllers and init DMA on them
+ * init_ohci1394_dma_on_all_controllers - scan for OHCI1394 controllers and init DMA on them
  * Scans the whole PCI space for OHCI1394 controllers and inits DMA on them
  */
 void __init init_ohci1394_dma_on_all_controllers(void)
@@ -283,7 +283,7 @@ void __init init_ohci1394_dma_on_all_controllers(void)
 }
 
 /**
- * setup_init_ohci1394_early - enables early OHCI1394 DMA initialization
+ * setup_ohci1394_dma - enables early OHCI1394 DMA initialization
  */
 static int __init setup_ohci1394_dma(char *opt)
 {
diff --git a/drivers/firewire/sbp2.c b/drivers/firewire/sbp2.c
index 60051c0cabea..26db5b8dfc1e 100644
--- a/drivers/firewire/sbp2.c
+++ b/drivers/firewire/sbp2.c
@@ -1117,7 +1117,7 @@ static void sbp2_init_workarounds(struct sbp2_target *tgt, u32 model,
 	tgt->workarounds = w;
 }
 
-static struct scsi_host_template scsi_driver_template;
+static const struct scsi_host_template scsi_driver_template;
 static void sbp2_remove(struct fw_unit *unit);
 
 static int sbp2_probe(struct fw_unit *unit, const struct ieee1394_device_id *id)
@@ -1586,7 +1586,7 @@ static struct attribute *sbp2_scsi_sysfs_attrs[] = {
 
 ATTRIBUTE_GROUPS(sbp2_scsi_sysfs);
 
-static struct scsi_host_template scsi_driver_template = {
+static const struct scsi_host_template scsi_driver_template = {
 	.module			= THIS_MODULE,
 	.name			= "SBP-2 IEEE-1394",
 	.proc_name		= "sbp2",
diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c
index dbc474ff62b7..e7d97b59963b 100644
--- a/drivers/firmware/arm_scmi/driver.c
+++ b/drivers/firmware/arm_scmi/driver.c
@@ -2289,7 +2289,7 @@ static int scmi_xfer_info_init(struct scmi_info *sinfo)
 		return ret;
 
 	ret = __scmi_xfer_info_init(sinfo, &sinfo->tx_minfo);
-	if (!ret && idr_find(&sinfo->rx_idr, SCMI_PROTOCOL_BASE))
+	if (!ret && !idr_is_empty(&sinfo->rx_idr))
 		ret = __scmi_xfer_info_init(sinfo, &sinfo->rx_minfo);
 
 	return ret;
diff --git a/drivers/firmware/arm_scmi/mailbox.c b/drivers/firmware/arm_scmi/mailbox.c
index 112c285deb97..1efa5e9392c4 100644
--- a/drivers/firmware/arm_scmi/mailbox.c
+++ b/drivers/firmware/arm_scmi/mailbox.c
@@ -19,13 +19,15 @@
  * struct scmi_mailbox - Structure representing a SCMI mailbox transport
  *
  * @cl: Mailbox Client
- * @chan: Transmit/Receive mailbox channel
+ * @chan: Transmit/Receive mailbox uni/bi-directional channel
+ * @chan_receiver: Optional Receiver mailbox unidirectional channel
  * @cinfo: SCMI channel info
  * @shmem: Transmit/Receive shared memory area
  */
 struct scmi_mailbox {
 	struct mbox_client cl;
 	struct mbox_chan *chan;
+	struct mbox_chan *chan_receiver;
 	struct scmi_chan_info *cinfo;
 	struct scmi_shared_mem __iomem *shmem;
 };
@@ -48,30 +50,62 @@ static void rx_callback(struct mbox_client *cl, void *m)
 
 static bool mailbox_chan_available(struct device_node *of_node, int idx)
 {
+	int num_mb;
+
+	/*
+	 * Just check if bidirrectional channels are involved, and check the
+	 * index accordingly; proper full validation will be made later
+	 * in mailbox_chan_setup().
+	 */
+	num_mb = of_count_phandle_with_args(of_node, "mboxes", "#mbox-cells");
+	if (num_mb == 3 && idx == 1)
+		idx = 2;
+
 	return !of_parse_phandle_with_args(of_node, "mboxes",
 					   "#mbox-cells", idx, NULL);
 }
 
-static int mailbox_chan_validate(struct device *cdev)
+/**
+ * mailbox_chan_validate  - Validate transport configuration and map channels
+ *
+ * @cdev: Reference to the underlying transport device carrying the
+ *	  of_node descriptor to analyze.
+ * @a2p_rx_chan: A reference to an optional unidirectional channel to use
+ *		 for replies on the a2p channel. Set as zero if not present.
+ * @p2a_chan: A reference to the optional p2a channel.
+ *	      Set as zero if not present.
+ *
+ * At first, validate the transport configuration as described in terms of
+ * 'mboxes' and 'shmem', then determin which mailbox channel indexes are
+ * appropriate to be use in the current configuration.
+ *
+ * Return: 0 on Success or error
+ */
+static int mailbox_chan_validate(struct device *cdev,
+				 int *a2p_rx_chan, int *p2a_chan)
 {
 	int num_mb, num_sh, ret = 0;
 	struct device_node *np = cdev->of_node;
 
 	num_mb = of_count_phandle_with_args(np, "mboxes", "#mbox-cells");
 	num_sh = of_count_phandle_with_args(np, "shmem", NULL);
+	dev_dbg(cdev, "Found %d mboxes and %d shmems !\n", num_mb, num_sh);
+
 	/* Bail out if mboxes and shmem descriptors are inconsistent */
-	if (num_mb <= 0 || num_sh > 2 || num_mb != num_sh) {
-		dev_warn(cdev, "Invalid channel descriptor for '%s'\n",
-			 of_node_full_name(np));
+	if (num_mb <= 0 || num_sh <= 0 || num_sh > 2 || num_mb > 3 ||
+	    (num_mb == 1 && num_sh != 1) || (num_mb == 3 && num_sh != 2)) {
+		dev_warn(cdev,
+			 "Invalid channel descriptor for '%s' - mbs:%d  shm:%d\n",
+			 of_node_full_name(np), num_mb, num_sh);
 		return -EINVAL;
 	}
 
+	/* Bail out if provided shmem descriptors do not refer distinct areas  */
 	if (num_sh > 1) {
 		struct device_node *np_tx, *np_rx;
 
 		np_tx = of_parse_phandle(np, "shmem", 0);
 		np_rx = of_parse_phandle(np, "shmem", 1);
-		/* SCMI Tx and Rx shared mem areas have to be distinct */
 		if (!np_tx || !np_rx || np_tx == np_rx) {
 			dev_warn(cdev, "Invalid shmem descriptor for '%s'\n",
 				 of_node_full_name(np));
@@ -82,6 +116,29 @@ static int mailbox_chan_validate(struct device *cdev)
 		of_node_put(np_rx);
 	}
 
+	/* Calculate channels IDs to use depending on mboxes/shmem layout */
+	if (!ret) {
+		switch (num_mb) {
+		case 1:
+			*a2p_rx_chan = 0;
+			*p2a_chan = 0;
+			break;
+		case 2:
+			if (num_sh == 2) {
+				*a2p_rx_chan = 0;
+				*p2a_chan = 1;
+			} else {
+				*a2p_rx_chan = 1;
+				*p2a_chan = 0;
+			}
+			break;
+		case 3:
+			*a2p_rx_chan = 1;
+			*p2a_chan = 2;
+			break;
+		}
+	}
+
 	return ret;
 }
 
@@ -92,15 +149,18 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
 	struct device *cdev = cinfo->dev;
 	struct scmi_mailbox *smbox;
 	struct device_node *shmem;
-	int ret, idx = tx ? 0 : 1;
+	int ret, a2p_rx_chan, p2a_chan, idx = tx ? 0 : 1;
 	struct mbox_client *cl;
 	resource_size_t size;
 	struct resource res;
 
-	ret = mailbox_chan_validate(cdev);
+	ret = mailbox_chan_validate(cdev, &a2p_rx_chan, &p2a_chan);
 	if (ret)
 		return ret;
 
+	if (!tx && !p2a_chan)
+		return -ENODEV;
+
 	smbox = devm_kzalloc(dev, sizeof(*smbox), GFP_KERNEL);
 	if (!smbox)
 		return -ENOMEM;
@@ -130,15 +190,26 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
 	cl->tx_block = false;
 	cl->knows_txdone = tx;
 
-	smbox->chan = mbox_request_channel(cl, tx ? 0 : 1);
+	smbox->chan = mbox_request_channel(cl, tx ? 0 : p2a_chan);
 	if (IS_ERR(smbox->chan)) {
 		ret = PTR_ERR(smbox->chan);
 		if (ret != -EPROBE_DEFER)
-			dev_err(cdev, "failed to request SCMI %s mailbox\n",
-				tx ? "Tx" : "Rx");
+			dev_err(cdev,
+				"failed to request SCMI %s mailbox\n", desc);
 		return ret;
 	}
 
+	/* Additional unidirectional channel for TX if needed */
+	if (tx && a2p_rx_chan) {
+		smbox->chan_receiver = mbox_request_channel(cl, a2p_rx_chan);
+		if (IS_ERR(smbox->chan_receiver)) {
+			ret = PTR_ERR(smbox->chan_receiver);
+			if (ret != -EPROBE_DEFER)
+				dev_err(cdev, "failed to request SCMI Tx Receiver mailbox\n");
+			return ret;
+		}
+	}
+
 	cinfo->transport_info = smbox;
 	smbox->cinfo = cinfo;
 
@@ -152,8 +223,10 @@ static int mailbox_chan_free(int id, void *p, void *data)
 
 	if (smbox && !IS_ERR(smbox->chan)) {
 		mbox_free_channel(smbox->chan);
+		mbox_free_channel(smbox->chan_receiver);
 		cinfo->transport_info = NULL;
 		smbox->chan = NULL;
+		smbox->chan_receiver = NULL;
 		smbox->cinfo = NULL;
 	}
 
diff --git a/drivers/firmware/arm_scmi/optee.c b/drivers/firmware/arm_scmi/optee.c
index 929720387102..e123de6e8c67 100644
--- a/drivers/firmware/arm_scmi/optee.c
+++ b/drivers/firmware/arm_scmi/optee.c
@@ -403,7 +403,7 @@ out:
 static int setup_shmem(struct device *dev, struct scmi_chan_info *cinfo,
 		       struct scmi_optee_channel *channel)
 {
-	if (of_find_property(cinfo->dev->of_node, "shmem", NULL))
+	if (of_property_present(cinfo->dev->of_node, "shmem"))
 		return setup_static_shmem(dev, cinfo, channel);
 	else
 		return setup_dynamic_shmem(dev, channel);
diff --git a/drivers/firmware/arm_sdei.c b/drivers/firmware/arm_sdei.c
index 1e1a51510e83..f9040bd61081 100644
--- a/drivers/firmware/arm_sdei.c
+++ b/drivers/firmware/arm_sdei.c
@@ -43,6 +43,8 @@ static asmlinkage void (*sdei_firmware_call)(unsigned long function_id,
 /* entry point from firmware to arch asm code */
 static unsigned long sdei_entry_point;
 
+static int sdei_hp_state;
+
 struct sdei_event {
 	/* These three are protected by the sdei_list_lock */
 	struct list_head	list;
@@ -301,8 +303,6 @@ int sdei_mask_local_cpu(void)
 {
 	int err;
 
-	WARN_ON_ONCE(preemptible());
-
 	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_MASK, 0, 0, 0, 0, 0, NULL);
 	if (err && err != -EIO) {
 		pr_warn_once("failed to mask CPU[%u]: %d\n",
@@ -315,6 +315,7 @@ int sdei_mask_local_cpu(void)
 
 static void _ipi_mask_cpu(void *ignored)
 {
+	WARN_ON_ONCE(preemptible());
 	sdei_mask_local_cpu();
 }
 
@@ -322,8 +323,6 @@ int sdei_unmask_local_cpu(void)
 {
 	int err;
 
-	WARN_ON_ONCE(preemptible());
-
 	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_UNMASK, 0, 0, 0, 0, 0, NULL);
 	if (err && err != -EIO) {
 		pr_warn_once("failed to unmask CPU[%u]: %d\n",
@@ -336,6 +335,7 @@ int sdei_unmask_local_cpu(void)
 
 static void _ipi_unmask_cpu(void *ignored)
 {
+	WARN_ON_ONCE(preemptible());
 	sdei_unmask_local_cpu();
 }
 
@@ -343,6 +343,8 @@ static void _ipi_private_reset(void *ignored)
 {
 	int err;
 
+	WARN_ON_ONCE(preemptible());
+
 	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PRIVATE_RESET, 0, 0, 0, 0, 0,
 			     NULL);
 	if (err && err != -EIO)
@@ -389,8 +391,6 @@ static void _local_event_enable(void *data)
 	int err;
 	struct sdei_crosscall_args *arg = data;
 
-	WARN_ON_ONCE(preemptible());
-
 	err = sdei_api_event_enable(arg->event->event_num);
 
 	sdei_cross_call_return(arg, err);
@@ -479,8 +479,6 @@ static void _local_event_unregister(void *data)
 	int err;
 	struct sdei_crosscall_args *arg = data;
 
-	WARN_ON_ONCE(preemptible());
-
 	err = sdei_api_event_unregister(arg->event->event_num);
 
 	sdei_cross_call_return(arg, err);
@@ -561,8 +559,6 @@ static void _local_event_register(void *data)
 	struct sdei_registered_event *reg;
 	struct sdei_crosscall_args *arg = data;
 
-	WARN_ON(preemptible());
-
 	reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
 	err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
 				      reg, 0, 0);
@@ -717,6 +713,8 @@ static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
 {
 	int rv;
 
+	WARN_ON_ONCE(preemptible());
+
 	switch (action) {
 	case CPU_PM_ENTER:
 		rv = sdei_mask_local_cpu();
@@ -765,7 +763,7 @@ static int sdei_device_freeze(struct device *dev)
 	int err;
 
 	/* unregister private events */
-	cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
+	cpuhp_remove_state(sdei_entry_point);
 
 	err = sdei_unregister_shared();
 	if (err)
@@ -786,12 +784,15 @@ static int sdei_device_thaw(struct device *dev)
 		return err;
 	}
 
-	err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "SDEI",
 				&sdei_cpuhp_up, &sdei_cpuhp_down);
-	if (err)
+	if (err < 0) {
 		pr_warn("Failed to re-register CPU hotplug notifier...\n");
+		return err;
+	}
 
-	return err;
+	sdei_hp_state = err;
+	return 0;
 }
 
 static int sdei_device_restore(struct device *dev)
@@ -823,7 +824,7 @@ static int sdei_reboot_notifier(struct notifier_block *nb, unsigned long action,
 	 * We are going to reset the interface, after this there is no point
 	 * doing work when we take CPUs offline.
 	 */
-	cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
+	cpuhp_remove_state(sdei_hp_state);
 
 	sdei_platform_reset();
 
@@ -1003,13 +1004,15 @@ static int sdei_probe(struct platform_device *pdev)
 		goto remove_cpupm;
 	}
 
-	err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "SDEI",
 				&sdei_cpuhp_up, &sdei_cpuhp_down);
-	if (err) {
+	if (err < 0) {
 		pr_warn("Failed to register CPU hotplug notifier...\n");
 		goto remove_reboot;
 	}
 
+	sdei_hp_state = err;
+
 	return 0;
 
 remove_reboot:
diff --git a/drivers/firmware/broadcom/bcm47xx_nvram.c b/drivers/firmware/broadcom/bcm47xx_nvram.c
index 5f47dbf4889a..0ea5206be4c9 100644
--- a/drivers/firmware/broadcom/bcm47xx_nvram.c
+++ b/drivers/firmware/broadcom/bcm47xx_nvram.c
@@ -255,4 +255,3 @@ char *bcm47xx_nvram_get_contents(size_t *nvram_size)
 }
 EXPORT_SYMBOL(bcm47xx_nvram_get_contents);
 
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c
index f558b390fbfe..e4ccfb6a8fa5 100644
--- a/drivers/firmware/cirrus/cs_dsp.c
+++ b/drivers/firmware/cirrus/cs_dsp.c
@@ -14,6 +14,7 @@
 #include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
+#include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 
@@ -301,6 +302,7 @@ struct cs_dsp_ops {
 static const struct cs_dsp_ops cs_dsp_adsp1_ops;
 static const struct cs_dsp_ops cs_dsp_adsp2_ops[];
 static const struct cs_dsp_ops cs_dsp_halo_ops;
+static const struct cs_dsp_ops cs_dsp_halo_ao_ops;
 
 struct cs_dsp_buf {
 	struct list_head list;
@@ -456,6 +458,33 @@ static const struct {
 	},
 };
 
+static int cs_dsp_coeff_base_reg(struct cs_dsp_coeff_ctl *ctl, unsigned int *reg,
+				 unsigned int off);
+
+static int cs_dsp_debugfs_read_controls_show(struct seq_file *s, void *ignored)
+{
+	struct cs_dsp *dsp = s->private;
+	struct cs_dsp_coeff_ctl *ctl;
+	unsigned int reg;
+
+	list_for_each_entry(ctl, &dsp->ctl_list, list) {
+		cs_dsp_coeff_base_reg(ctl, &reg, 0);
+		seq_printf(s, "%22.*s: %#8zx %s:%08x %#8x %s %#8x %#4x %c%c%c%c %s %s\n",
+			   ctl->subname_len, ctl->subname, ctl->len,
+			   cs_dsp_mem_region_name(ctl->alg_region.type),
+			   ctl->offset, reg, ctl->fw_name, ctl->alg_region.alg, ctl->type,
+			   ctl->flags & WMFW_CTL_FLAG_VOLATILE ? 'V' : '-',
+			   ctl->flags & WMFW_CTL_FLAG_SYS ? 'S' : '-',
+			   ctl->flags & WMFW_CTL_FLAG_READABLE ? 'R' : '-',
+			   ctl->flags & WMFW_CTL_FLAG_WRITEABLE ? 'W' : '-',
+			   ctl->enabled ? "enabled" : "disabled",
+			   ctl->set ? "dirty" : "clean");
+	}
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(cs_dsp_debugfs_read_controls);
+
 /**
  * cs_dsp_init_debugfs() - Create and populate DSP representation in debugfs
  * @dsp: pointer to DSP structure
@@ -478,6 +507,9 @@ void cs_dsp_init_debugfs(struct cs_dsp *dsp, struct dentry *debugfs_root)
 		debugfs_create_file(cs_dsp_debugfs_fops[i].name, 0444, root,
 				    dsp, &cs_dsp_debugfs_fops[i].fops);
 
+	debugfs_create_file("controls", 0444, root, dsp,
+			    &cs_dsp_debugfs_read_controls_fops);
+
 	dsp->debugfs_root = root;
 }
 EXPORT_SYMBOL_NS_GPL(cs_dsp_init_debugfs, FW_CS_DSP);
@@ -1300,6 +1332,9 @@ static int cs_dsp_load(struct cs_dsp *dsp, const struct firmware *firmware,
 	int regions = 0;
 	int ret, offset, type;
 
+	if (!firmware)
+		return 0;
+
 	ret = -EINVAL;
 
 	pos = sizeof(*header) + sizeof(*adsp1_sizes) + sizeof(*footer);
@@ -2821,7 +2856,10 @@ EXPORT_SYMBOL_NS_GPL(cs_dsp_adsp2_init, FW_CS_DSP);
  */
 int cs_dsp_halo_init(struct cs_dsp *dsp)
 {
-	dsp->ops = &cs_dsp_halo_ops;
+	if (dsp->no_core_startstop)
+		dsp->ops = &cs_dsp_halo_ao_ops;
+	else
+		dsp->ops = &cs_dsp_halo_ops;
 
 	return cs_dsp_common_init(dsp);
 }
@@ -3187,6 +3225,14 @@ static const struct cs_dsp_ops cs_dsp_halo_ops = {
 	.stop_core = cs_dsp_halo_stop_core,
 };
 
+static const struct cs_dsp_ops cs_dsp_halo_ao_ops = {
+	.parse_sizes = cs_dsp_adsp2_parse_sizes,
+	.validate_version = cs_dsp_halo_validate_version,
+	.setup_algs = cs_dsp_halo_setup_algs,
+	.region_to_reg = cs_dsp_halo_region_to_reg,
+	.show_fw_status = cs_dsp_halo_show_fw_status,
+};
+
 /**
  * cs_dsp_chunk_write() - Format data to a DSP memory chunk
  * @ch: Pointer to the chunk structure
diff --git a/drivers/firmware/dmi-sysfs.c b/drivers/firmware/dmi-sysfs.c
index ed5aff0a4204..03708ab64e56 100644
--- a/drivers/firmware/dmi-sysfs.c
+++ b/drivers/firmware/dmi-sysfs.c
@@ -304,7 +304,7 @@ static struct attribute *dmi_sysfs_sel_attrs[] = {
 };
 ATTRIBUTE_GROUPS(dmi_sysfs_sel);
 
-static struct kobj_type dmi_system_event_log_ktype = {
+static const struct kobj_type dmi_system_event_log_ktype = {
 	.release = dmi_entry_free,
 	.sysfs_ops = &dmi_sysfs_specialize_attr_ops,
 	.default_groups = dmi_sysfs_sel_groups,
@@ -563,7 +563,7 @@ static void dmi_sysfs_entry_release(struct kobject *kobj)
 	kfree(entry);
 }
 
-static struct kobj_type dmi_sysfs_entry_ktype = {
+static const struct kobj_type dmi_sysfs_entry_ktype = {
 	.release = dmi_sysfs_entry_release,
 	.sysfs_ops = &dmi_sysfs_attr_ops,
 	.default_groups = dmi_sysfs_entry_groups,
diff --git a/drivers/firmware/edd.c b/drivers/firmware/edd.c
index 5cc238916551..55dec4eb2c00 100644
--- a/drivers/firmware/edd.c
+++ b/drivers/firmware/edd.c
@@ -608,7 +608,7 @@ static void edd_release(struct kobject * kobj)
 	kfree(dev);
 }
 
-static struct kobj_type edd_ktype = {
+static const struct kobj_type edd_ktype = {
 	.release	= edd_release,
 	.sysfs_ops	= &edd_attr_ops,
 };
diff --git a/drivers/firmware/efi/cper-arm.c b/drivers/firmware/efi/cper-arm.c
index 36d3b8b9da47..fa9c1c3bf168 100644
--- a/drivers/firmware/efi/cper-arm.c
+++ b/drivers/firmware/efi/cper-arm.c
@@ -12,7 +12,6 @@
 #include <linux/dmi.h>
 #include <linux/acpi.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/printk.h>
 #include <linux/bcd.h>
 #include <acpi/ghes.h>
diff --git a/drivers/firmware/efi/runtime-wrappers.c b/drivers/firmware/efi/runtime-wrappers.c
index 1fba4e09cdcf..a400c4312c82 100644
--- a/drivers/firmware/efi/runtime-wrappers.c
+++ b/drivers/firmware/efi/runtime-wrappers.c
@@ -158,7 +158,7 @@ void efi_call_virt_check_flags(unsigned long flags, const char *call)
  * none of the remaining functions are actually ever called at runtime.
  * So let's just use a single lock to serialize all Runtime Services calls.
  */
-static DEFINE_SEMAPHORE(efi_runtime_lock);
+static DEFINE_SEMAPHORE(efi_runtime_lock, 1);
 
 /*
  * Expose the EFI runtime lock to the UV platform
diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c
index bd75b87f5fc1..bfc5fa6aa47b 100644
--- a/drivers/firmware/efi/vars.c
+++ b/drivers/firmware/efi/vars.c
@@ -21,7 +21,7 @@
 /* Private pointer to registered efivars */
 static struct efivars *__efivars;
 
-static DEFINE_SEMAPHORE(efivars_lock);
+static DEFINE_SEMAPHORE(efivars_lock, 1);
 
 static efi_status_t check_var_size(bool nonblocking, u32 attributes,
 				   unsigned long size)
diff --git a/drivers/firmware/imx/imx-scu.c b/drivers/firmware/imx/imx-scu.c
index dca79caccd01..47db49911e7b 100644
--- a/drivers/firmware/imx/imx-scu.c
+++ b/drivers/firmware/imx/imx-scu.c
@@ -310,9 +310,8 @@ static int imx_scu_probe(struct platform_device *pdev)
 		sc_chan->ch = mbox_request_channel_byname(cl, chan_name);
 		if (IS_ERR(sc_chan->ch)) {
 			ret = PTR_ERR(sc_chan->ch);
-			if (ret != -EPROBE_DEFER)
-				dev_err(dev, "Failed to request mbox chan %s ret %d\n",
-					chan_name, ret);
+			dev_err_probe(dev, ret, "Failed to request mbox chan %s\n",
+				      chan_name);
 			kfree(chan_name);
 			return ret;
 		}
diff --git a/drivers/firmware/imx/scu-pd.c b/drivers/firmware/imx/scu-pd.c
index 2a4f07423365..84b673427073 100644
--- a/drivers/firmware/imx/scu-pd.c
+++ b/drivers/firmware/imx/scu-pd.c
@@ -180,7 +180,11 @@ static const struct imx_sc_pd_range imx8qxp_scu_pd_ranges[] = {
 
 	/* LVDS SS */
 	{ "lvds0", IMX_SC_R_LVDS_0, 1, false, 0 },
+	{ "lvds0-pwm", IMX_SC_R_LVDS_0_PWM_0, 1, false, 0 },
+	{ "lvds0-lpi2c", IMX_SC_R_LVDS_0_I2C_0, 2, true, 0 },
 	{ "lvds1", IMX_SC_R_LVDS_1, 1, false, 0 },
+	{ "lvds1-pwm", IMX_SC_R_LVDS_1_PWM_0, 1, false, 0 },
+	{ "lvds1-lpi2c", IMX_SC_R_LVDS_1_I2C_0, 2, true, 0 },
 
 	/* DC SS */
 	{ "dc0", IMX_SC_R_DC_0, 1, false, 0 },
diff --git a/drivers/firmware/meson/meson_sm.c b/drivers/firmware/meson/meson_sm.c
index 3f5ff9ed668e..798bcdb05d84 100644
--- a/drivers/firmware/meson/meson_sm.c
+++ b/drivers/firmware/meson/meson_sm.c
@@ -311,11 +311,14 @@ static int __init meson_sm_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, fw);
 
-	pr_info("secure-monitor enabled\n");
+	if (devm_of_platform_populate(dev))
+		goto out_in_base;
 
 	if (sysfs_create_group(&pdev->dev.kobj, &meson_sm_sysfs_attr_group))
 		goto out_in_base;
 
+	pr_info("secure-monitor enabled\n");
+
 	return 0;
 
 out_in_base:
diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
index 29619f49873a..d9629ff87861 100644
--- a/drivers/firmware/psci/psci.c
+++ b/drivers/firmware/psci/psci.c
@@ -167,7 +167,8 @@ int psci_set_osi_mode(bool enable)
 
 	err = invoke_psci_fn(PSCI_1_0_FN_SET_SUSPEND_MODE, suspend_mode, 0, 0);
 	if (err < 0)
-		pr_warn("failed to set %s mode: %d\n", enable ? "OSI" : "PC", err);
+		pr_info(FW_BUG "failed to set %s mode: %d\n",
+				enable ? "OSI" : "PC", err);
 	return psci_to_linux_errno(err);
 }
 
diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c
index b1e11f85b805..fde33acd46b7 100644
--- a/drivers/firmware/qcom_scm.c
+++ b/drivers/firmware/qcom_scm.c
@@ -905,7 +905,7 @@ static int __qcom_scm_assign_mem(struct device *dev, phys_addr_t mem_region,
  * Return negative errno on failure or 0 on success with @srcvm updated.
  */
 int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz,
-			unsigned int *srcvm,
+			u64 *srcvm,
 			const struct qcom_scm_vmperm *newvm,
 			unsigned int dest_cnt)
 {
@@ -922,9 +922,9 @@ int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz,
 	__le32 *src;
 	void *ptr;
 	int ret, i, b;
-	unsigned long srcvm_bits = *srcvm;
+	u64 srcvm_bits = *srcvm;
 
-	src_sz = hweight_long(srcvm_bits) * sizeof(*src);
+	src_sz = hweight64(srcvm_bits) * sizeof(*src);
 	mem_to_map_sz = sizeof(*mem_to_map);
 	dest_sz = dest_cnt * sizeof(*destvm);
 	ptr_sz = ALIGN(src_sz, SZ_64) + ALIGN(mem_to_map_sz, SZ_64) +
@@ -937,8 +937,10 @@ int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz,
 	/* Fill source vmid detail */
 	src = ptr;
 	i = 0;
-	for_each_set_bit(b, &srcvm_bits, BITS_PER_LONG)
-		src[i++] = cpu_to_le32(b);
+	for (b = 0; b < BITS_PER_TYPE(u64); b++) {
+		if (srcvm_bits & BIT(b))
+			src[i++] = cpu_to_le32(b);
+	}
 
 	/* Fill details of mem buff to map */
 	mem_to_map = ptr + ALIGN(src_sz, SZ_64);
@@ -1506,8 +1508,7 @@ static int qcom_scm_probe(struct platform_device *pdev)
 static void qcom_scm_shutdown(struct platform_device *pdev)
 {
 	/* Clean shutdown, disable download mode to allow normal restart */
-	if (download_mode)
-		qcom_scm_set_download_mode(false);
+	qcom_scm_set_download_mode(false);
 }
 
 static const struct of_device_id qcom_scm_dt_match[] = {
@@ -1542,6 +1543,7 @@ static const struct of_device_id qcom_scm_dt_match[] = {
 	},
 	{ .compatible = "qcom,scm-msm8994" },
 	{ .compatible = "qcom,scm-msm8996" },
+	{ .compatible = "qcom,scm-sm6375", .data = (void *)SCM_HAS_CORE_CLK },
 	{ .compatible = "qcom,scm" },
 	{}
 };
diff --git a/drivers/firmware/smccc/smccc.c b/drivers/firmware/smccc/smccc.c
index 60ccf3e90d7d..db818f9dcb8e 100644
--- a/drivers/firmware/smccc/smccc.c
+++ b/drivers/firmware/smccc/smccc.c
@@ -17,9 +17,13 @@ static enum arm_smccc_conduit smccc_conduit = SMCCC_CONDUIT_NONE;
 
 bool __ro_after_init smccc_trng_available = false;
 u64 __ro_after_init smccc_has_sve_hint = false;
+s32 __ro_after_init smccc_soc_id_version = SMCCC_RET_NOT_SUPPORTED;
+s32 __ro_after_init smccc_soc_id_revision = SMCCC_RET_NOT_SUPPORTED;
 
 void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit)
 {
+	struct arm_smccc_res res;
+
 	smccc_version = version;
 	smccc_conduit = conduit;
 
@@ -27,6 +31,18 @@ void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit)
 	if (IS_ENABLED(CONFIG_ARM64_SVE) &&
 	    smccc_version >= ARM_SMCCC_VERSION_1_3)
 		smccc_has_sve_hint = true;
+
+	if ((smccc_version >= ARM_SMCCC_VERSION_1_2) &&
+	    (smccc_conduit != SMCCC_CONDUIT_NONE)) {
+		arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+				     ARM_SMCCC_ARCH_SOC_ID, &res);
+		if ((s32)res.a0 >= 0) {
+			arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_SOC_ID, 0, &res);
+			smccc_soc_id_version = (s32)res.a0;
+			arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_SOC_ID, 1, &res);
+			smccc_soc_id_revision = (s32)res.a0;
+		}
+	}
 }
 
 enum arm_smccc_conduit arm_smccc_1_1_get_conduit(void)
@@ -44,6 +60,16 @@ u32 arm_smccc_get_version(void)
 }
 EXPORT_SYMBOL_GPL(arm_smccc_get_version);
 
+s32 arm_smccc_get_soc_id_version(void)
+{
+	return smccc_soc_id_version;
+}
+
+s32 arm_smccc_get_soc_id_revision(void)
+{
+	return smccc_soc_id_revision;
+}
+
 static int __init smccc_devices_init(void)
 {
 	struct platform_device *pdev;
diff --git a/drivers/firmware/smccc/soc_id.c b/drivers/firmware/smccc/soc_id.c
index dd7c3d5e8b0b..890eb454599a 100644
--- a/drivers/firmware/smccc/soc_id.c
+++ b/drivers/firmware/smccc/soc_id.c
@@ -42,41 +42,23 @@ static int __init smccc_soc_init(void)
 	if (arm_smccc_get_version() < ARM_SMCCC_VERSION_1_2)
 		return 0;
 
-	if (arm_smccc_1_1_get_conduit() == SMCCC_CONDUIT_NONE) {
-		pr_err("%s: invalid SMCCC conduit\n", __func__);
-		return -EOPNOTSUPP;
-	}
-
-	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
-			     ARM_SMCCC_ARCH_SOC_ID, &res);
-
-	if ((int)res.a0 == SMCCC_RET_NOT_SUPPORTED) {
+	soc_id_version = arm_smccc_get_soc_id_version();
+	if (soc_id_version == SMCCC_RET_NOT_SUPPORTED) {
 		pr_info("ARCH_SOC_ID not implemented, skipping ....\n");
 		return 0;
 	}
 
-	if ((int)res.a0 < 0) {
-		pr_info("ARCH_FEATURES(ARCH_SOC_ID) returned error: %lx\n",
-			res.a0);
-		return -EINVAL;
-	}
-
-	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_SOC_ID, 0, &res);
-	if ((int)res.a0 < 0) {
+	if (soc_id_version < 0) {
 		pr_err("ARCH_SOC_ID(0) returned error: %lx\n", res.a0);
 		return -EINVAL;
 	}
 
-	soc_id_version = res.a0;
-
-	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_SOC_ID, 1, &res);
-	if ((int)res.a0 < 0) {
+	soc_id_rev = arm_smccc_get_soc_id_revision();
+	if (soc_id_rev < 0) {
 		pr_err("ARCH_SOC_ID(1) returned error: %lx\n", res.a0);
 		return -EINVAL;
 	}
 
-	soc_id_rev = res.a0;
-
 	soc_dev_attr = kzalloc(sizeof(*soc_dev_attr), GFP_KERNEL);
 	if (!soc_dev_attr)
 		return -ENOMEM;
diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c
index bde1f543f529..80f4e2d14e04 100644
--- a/drivers/firmware/stratix10-svc.c
+++ b/drivers/firmware/stratix10-svc.c
@@ -1133,8 +1133,8 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev)
 		return ret;
 
 	genpool = svc_create_memory_pool(pdev, sh_memory);
-	if (!genpool)
-		return -ENOMEM;
+	if (IS_ERR(genpool))
+		return PTR_ERR(genpool);
 
 	/* allocate service controller and supporting channel */
 	controller = devm_kzalloc(dev, sizeof(*controller), GFP_KERNEL);
diff --git a/drivers/firmware/tegra/bpmp-debugfs.c b/drivers/firmware/tegra/bpmp-debugfs.c
index 3ca2b5d9e66f..6dfe3d34109e 100644
--- a/drivers/firmware/tegra/bpmp-debugfs.c
+++ b/drivers/firmware/tegra/bpmp-debugfs.c
@@ -193,7 +193,7 @@ static int mrq_debug_read(struct tegra_bpmp *bpmp, const char *name,
 		},
 	};
 	u32 fd = 0, len = 0;
-	int remaining, err;
+	int remaining, err, close_err;
 
 	mutex_lock(&bpmp_debug_lock);
 	err = mrq_debug_open(bpmp, name, &fd, &len, 0);
@@ -231,7 +231,9 @@ static int mrq_debug_read(struct tegra_bpmp *bpmp, const char *name,
 	*nbytes = len;
 
 close:
-	err = mrq_debug_close(bpmp, fd);
+	close_err = mrq_debug_close(bpmp, fd);
+	if (!err)
+		err = close_err;
 out:
 	mutex_unlock(&bpmp_debug_lock);
 	return err;
@@ -319,7 +321,7 @@ static int bpmp_debug_show(struct seq_file *m, void *p)
 		},
 	};
 	u32 fd = 0, len = 0;
-	int remaining, err;
+	int remaining, err, close_err;
 
 	filename = get_filename(bpmp, file, fnamebuf, sizeof(fnamebuf));
 	if (!filename)
@@ -353,7 +355,9 @@ static int bpmp_debug_show(struct seq_file *m, void *p)
 	}
 
 close:
-	err = mrq_debug_close(bpmp, fd);
+	close_err = mrq_debug_close(bpmp, fd);
+	if (!err)
+		err = close_err;
 out:
 	mutex_unlock(&bpmp_debug_lock);
 	return err;
diff --git a/drivers/firmware/tegra/bpmp.c b/drivers/firmware/tegra/bpmp.c
index 042c2043929d..8b5e5daa9fae 100644
--- a/drivers/firmware/tegra/bpmp.c
+++ b/drivers/firmware/tegra/bpmp.c
@@ -764,19 +764,19 @@ static int tegra_bpmp_probe(struct platform_device *pdev)
 	if (err < 0)
 		goto free_mrq;
 
-	if (of_find_property(pdev->dev.of_node, "#clock-cells", NULL)) {
+	if (of_property_present(pdev->dev.of_node, "#clock-cells")) {
 		err = tegra_bpmp_init_clocks(bpmp);
 		if (err < 0)
 			goto free_mrq;
 	}
 
-	if (of_find_property(pdev->dev.of_node, "#reset-cells", NULL)) {
+	if (of_property_present(pdev->dev.of_node, "#reset-cells")) {
 		err = tegra_bpmp_init_resets(bpmp);
 		if (err < 0)
 			goto free_mrq;
 	}
 
-	if (of_find_property(pdev->dev.of_node, "#power-domain-cells", NULL)) {
+	if (of_property_present(pdev->dev.of_node, "#power-domain-cells")) {
 		err = tegra_bpmp_init_powergates(bpmp);
 		if (err < 0)
 			goto free_mrq;
diff --git a/drivers/firmware/turris-mox-rwtm.c b/drivers/firmware/turris-mox-rwtm.c
index 6ea5789a89e2..2de0fb139ce1 100644
--- a/drivers/firmware/turris-mox-rwtm.c
+++ b/drivers/firmware/turris-mox-rwtm.c
@@ -104,7 +104,7 @@ static void mox_kobj_release(struct kobject *kobj)
 	kfree(to_rwtm(kobj)->kobj);
 }
 
-static struct kobj_type mox_kobj_ktype = {
+static const struct kobj_type mox_kobj_ktype = {
 	.release	= mox_kobj_release,
 	.sysfs_ops	= &kobj_sysfs_ops,
 };
diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c
index ce86a1850305..a736db4a5825 100644
--- a/drivers/firmware/xilinx/zynqmp.c
+++ b/drivers/firmware/xilinx/zynqmp.c
@@ -972,6 +972,39 @@ int zynqmp_pm_fpga_get_status(u32 *value)
 EXPORT_SYMBOL_GPL(zynqmp_pm_fpga_get_status);
 
 /**
+ * zynqmp_pm_fpga_get_config_status - Get the FPGA configuration status.
+ * @value: Buffer to store FPGA configuration status.
+ *
+ * This function provides access to the pmufw to get the FPGA configuration
+ * status
+ *
+ * Return: 0 on success, a negative value on error
+ */
+int zynqmp_pm_fpga_get_config_status(u32 *value)
+{
+	u32 ret_payload[PAYLOAD_ARG_CNT];
+	u32 buf, lower_addr, upper_addr;
+	int ret;
+
+	if (!value)
+		return -EINVAL;
+
+	lower_addr = lower_32_bits((u64)&buf);
+	upper_addr = upper_32_bits((u64)&buf);
+
+	ret = zynqmp_pm_invoke_fn(PM_FPGA_READ,
+				  XILINX_ZYNQMP_PM_FPGA_CONFIG_STAT_OFFSET,
+				  lower_addr, upper_addr,
+				  XILINX_ZYNQMP_PM_FPGA_READ_CONFIG_REG,
+				  ret_payload);
+
+	*value = ret_payload[1];
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_fpga_get_config_status);
+
+/**
  * zynqmp_pm_pinctrl_request - Request Pin from firmware
  * @pin: Pin number to request
  *
diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c
index 0914e7328b1a..1bc04378118c 100644
--- a/drivers/fpga/dfl-pci.c
+++ b/drivers/fpga/dfl-pci.c
@@ -21,7 +21,6 @@
 #include <linux/module.h>
 #include <linux/stddef.h>
 #include <linux/errno.h>
-#include <linux/aer.h>
 
 #include "dfl.h"
 
@@ -376,10 +375,6 @@ int cci_pci_probe(struct pci_dev *pcidev, const struct pci_device_id *pcidevid)
 		return ret;
 	}
 
-	ret = pci_enable_pcie_error_reporting(pcidev);
-	if (ret && ret != -EINVAL)
-		dev_info(&pcidev->dev, "PCIE AER unavailable %d.\n", ret);
-
 	pci_set_master(pcidev);
 
 	ret = dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(64));
@@ -387,24 +382,22 @@ int cci_pci_probe(struct pci_dev *pcidev, const struct pci_device_id *pcidevid)
 		ret = dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(32));
 	if (ret) {
 		dev_err(&pcidev->dev, "No suitable DMA support available.\n");
-		goto disable_error_report_exit;
+		return ret;
 	}
 
 	ret = cci_init_drvdata(pcidev);
 	if (ret) {
 		dev_err(&pcidev->dev, "Fail to init drvdata %d.\n", ret);
-		goto disable_error_report_exit;
+		return ret;
 	}
 
 	ret = cci_enumerate_feature_devs(pcidev);
-	if (!ret)
+	if (ret) {
+		dev_err(&pcidev->dev, "enumeration failure %d.\n", ret);
 		return ret;
+	}
 
-	dev_err(&pcidev->dev, "enumeration failure %d.\n", ret);
-
-disable_error_report_exit:
-	pci_disable_pcie_error_reporting(pcidev);
-	return ret;
+	return 0;
 }
 
 static int cci_pci_sriov_configure(struct pci_dev *pcidev, int num_vfs)
@@ -448,7 +441,6 @@ static void cci_pci_remove(struct pci_dev *pcidev)
 		cci_pci_sriov_configure(pcidev, 0);
 
 	cci_remove_feature_devs(pcidev);
-	pci_disable_pcie_error_reporting(pcidev);
 }
 
 static struct pci_driver cci_pci_driver = {
diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c
index 5cd40acab5bf..a6c25dee9cc1 100644
--- a/drivers/fpga/fpga-bridge.c
+++ b/drivers/fpga/fpga-bridge.c
@@ -115,7 +115,7 @@ static int fpga_bridge_dev_match(struct device *dev, const void *data)
 /**
  * fpga_bridge_get - get an exclusive reference to an fpga bridge
  * @dev:	parent device that fpga bridge was registered with
- * @info:	fpga manager info
+ * @info:	fpga image specific information
  *
  * Given a device, get an exclusive reference to an fpga bridge.
  *
@@ -363,7 +363,6 @@ fpga_bridge_register(struct device *parent, const char *name,
 	bridge->dev.parent = parent;
 	bridge->dev.of_node = parent->of_node;
 	bridge->dev.id = id;
-	of_platform_populate(bridge->dev.of_node, NULL, NULL, &bridge->dev);
 
 	ret = dev_set_name(&bridge->dev, "br%d", id);
 	if (ret)
@@ -375,6 +374,8 @@ fpga_bridge_register(struct device *parent, const char *name,
 		return ERR_PTR(ret);
 	}
 
+	of_platform_populate(bridge->dev.of_node, NULL, NULL, &bridge->dev);
+
 	return bridge;
 
 error_device:
@@ -416,7 +417,7 @@ static void fpga_bridge_dev_release(struct device *dev)
 
 static int __init fpga_bridge_dev_init(void)
 {
-	fpga_bridge_class = class_create(THIS_MODULE, "fpga_bridge");
+	fpga_bridge_class = class_create("fpga_bridge");
 	if (IS_ERR(fpga_bridge_class))
 		return PTR_ERR(fpga_bridge_class);
 
diff --git a/drivers/fpga/fpga-mgr.c b/drivers/fpga/fpga-mgr.c
index 8efa67620e21..eb583f86a0b9 100644
--- a/drivers/fpga/fpga-mgr.c
+++ b/drivers/fpga/fpga-mgr.c
@@ -971,7 +971,7 @@ static int __init fpga_mgr_class_init(void)
 {
 	pr_info("FPGA manager framework\n");
 
-	fpga_mgr_class = class_create(THIS_MODULE, "fpga_manager");
+	fpga_mgr_class = class_create("fpga_manager");
 	if (IS_ERR(fpga_mgr_class))
 		return PTR_ERR(fpga_mgr_class);
 
diff --git a/drivers/fpga/fpga-region.c b/drivers/fpga/fpga-region.c
index 27ff9dea04ae..ccf6fdab1360 100644
--- a/drivers/fpga/fpga-region.c
+++ b/drivers/fpga/fpga-region.c
@@ -293,7 +293,7 @@ static void fpga_region_dev_release(struct device *dev)
  */
 static int __init fpga_region_init(void)
 {
-	fpga_region_class = class_create(THIS_MODULE, "fpga_region");
+	fpga_region_class = class_create("fpga_region");
 	if (IS_ERR(fpga_region_class))
 		return PTR_ERR(fpga_region_class);
 
diff --git a/drivers/fpga/intel-m10-bmc-sec-update.c b/drivers/fpga/intel-m10-bmc-sec-update.c
index f0acedc80182..d7e2f9f461bc 100644
--- a/drivers/fpga/intel-m10-bmc-sec-update.c
+++ b/drivers/fpga/intel-m10-bmc-sec-update.c
@@ -474,7 +474,7 @@ static enum fw_upload_err rsu_send_data(struct m10bmc_sec *sec)
 
 	ret = sec->ops->rsu_status(sec);
 	if (ret < 0)
-		return ret;
+		return FW_UPLOAD_ERR_HW_ERROR;
 	status = ret;
 
 	if (!rsu_status_ok(status)) {
diff --git a/drivers/fpga/lattice-sysconfig-spi.c b/drivers/fpga/lattice-sysconfig-spi.c
index 2702b26b7f55..44691cfcf50a 100644
--- a/drivers/fpga/lattice-sysconfig-spi.c
+++ b/drivers/fpga/lattice-sysconfig-spi.c
@@ -3,6 +3,7 @@
  * Lattice FPGA programming over slave SPI sysCONFIG interface.
  */
 
+#include <linux/of.h>
 #include <linux/spi/spi.h>
 
 #include "lattice-sysconfig.h"
diff --git a/drivers/fpga/xilinx-pr-decoupler.c b/drivers/fpga/xilinx-pr-decoupler.c
index 2d9c491f7be9..b76d85449b8f 100644
--- a/drivers/fpga/xilinx-pr-decoupler.c
+++ b/drivers/fpga/xilinx-pr-decoupler.c
@@ -69,7 +69,7 @@ static int xlnx_pr_decoupler_enable_show(struct fpga_bridge *bridge)
 	if (err)
 		return err;
 
-	status = readl(priv->io_base);
+	status = xlnx_pr_decouple_read(priv, CTRL_OFFSET);
 
 	clk_disable(priv->clk);
 
diff --git a/drivers/fpga/zynqmp-fpga.c b/drivers/fpga/zynqmp-fpga.c
index c60f20949c47..f3434e2c487b 100644
--- a/drivers/fpga/zynqmp-fpga.c
+++ b/drivers/fpga/zynqmp-fpga.c
@@ -77,6 +77,26 @@ static enum fpga_mgr_states zynqmp_fpga_ops_state(struct fpga_manager *mgr)
 	return FPGA_MGR_STATE_UNKNOWN;
 }
 
+static ssize_t status_show(struct device *dev,
+			   struct device_attribute *attr, char *buf)
+{
+	u32 status;
+	int ret;
+
+	ret = zynqmp_pm_fpga_get_config_status(&status);
+	if (ret)
+		return ret;
+
+	return sysfs_emit(buf, "0x%x\n", status);
+}
+static DEVICE_ATTR_RO(status);
+
+static struct attribute *zynqmp_fpga_attrs[] = {
+	&dev_attr_status.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(zynqmp_fpga);
+
 static const struct fpga_manager_ops zynqmp_fpga_ops = {
 	.state = zynqmp_fpga_ops_state,
 	.write_init = zynqmp_fpga_ops_write_init,
@@ -113,6 +133,7 @@ static struct platform_driver zynqmp_fpga_driver = {
 	.driver = {
 		.name = "zynqmp_fpga_manager",
 		.of_match_table = of_match_ptr(zynqmp_fpga_of_match),
+		.dev_groups = zynqmp_fpga_groups,
 	},
 };
 
diff --git a/drivers/gnss/core.c b/drivers/gnss/core.c
index 77a4b280c552..48f2ee0f78c4 100644
--- a/drivers/gnss/core.c
+++ b/drivers/gnss/core.c
@@ -387,7 +387,7 @@ static int __init gnss_module_init(void)
 		return ret;
 	}
 
-	gnss_class = class_create(THIS_MODULE, "gnss");
+	gnss_class = class_create("gnss");
 	if (IS_ERR(gnss_class)) {
 		ret = PTR_ERR(gnss_class);
 		pr_err("failed to create class: %d\n", ret);
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index badbe0582318..b81393072715 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -3,14 +3,6 @@
 # GPIO infrastructure and drivers
 #
 
-config ARCH_HAVE_CUSTOM_GPIO_H
-	bool
-	help
-	  Selecting this config option from the architecture Kconfig allows
-	  the architecture to provide a custom asm/gpio.h implementation
-	  overriding the default implementations.  New uses of this are
-	  strongly discouraged.
-
 menuconfig GPIOLIB
 	bool "GPIO Support"
 	help
@@ -47,6 +39,14 @@ config GPIOLIB_IRQCHIP
 	select IRQ_DOMAIN
 	bool
 
+config OF_GPIO_MM_GPIOCHIP
+	bool
+	help
+	  This adds support for the legacy 'struct of_mm_gpio_chip' interface
+	  from PowerPC. Existing drivers using this interface need to select
+	  this symbol, but new drivers should use the generic gpio-regmap
+	  infrastructure instead.
+
 config DEBUG_GPIO
 	bool "Debug GPIO calls"
 	depends on DEBUG_KERNEL
@@ -139,6 +139,7 @@ config GPIO_ALTERA
 	tristate "Altera GPIO"
 	depends on OF_GPIO
 	select GPIOLIB_IRQCHIP
+	select OF_GPIO_MM_GPIOCHIP
 	help
 	  Say Y or M here to build support for the Altera PIO device.
 
@@ -380,6 +381,18 @@ config GPIO_LOONGSON
 	help
 	  Driver for GPIO functionality on Loongson-2F/3A/3B processors.
 
+config GPIO_LOONGSON_64BIT
+	tristate "Loongson 64 bit GPIO support"
+	depends on LOONGARCH || COMPILE_TEST
+	depends on OF_GPIO
+	select GPIO_GENERIC
+	help
+	  Say yes here to support the GPIO functionality of a number of
+	  Loongson series of chips. The Loongson GPIO controller supports
+	  up to 60 GPIOS in total, 4 of which are dedicated GPIO pins, and
+	  the remaining 56 are reused with other functions, with edge or
+	  level triggered interrupts.
+
 config GPIO_LPC18XX
 	tristate "NXP LPC18XX/43XX GPIO support"
 	default y if ARCH_LPC18XX
@@ -411,6 +424,7 @@ config GPIO_MENZ127
 config GPIO_MM_LANTIQ
 	bool "Lantiq Memory mapped GPIOs"
 	depends on LANTIQ && SOC_XWAY
+	select OF_GPIO_MM_GPIOCHIP
 	help
 	  This enables support for memory mapped GPIOs on the External Bus Unit
 	  (EBU) found on Lantiq SoCs. The GPIOs are output only as they are
@@ -419,6 +433,7 @@ config GPIO_MM_LANTIQ
 config GPIO_MPC5200
 	def_bool y
 	depends on PPC_MPC52xx
+	select OF_GPIO_MM_GPIOCHIP
 
 config GPIO_MPC8XXX
 	bool "MPC512x/MPC8xxx/QorIQ GPIO support"
@@ -485,14 +500,6 @@ config GPIO_PL061
 	help
 	  Say yes here to support the PrimeCell PL061 GPIO device.
 
-config GPIO_PMIC_EIC_SPRD
-	tristate "Spreadtrum PMIC EIC support"
-	depends on MFD_SC27XX_PMIC || COMPILE_TEST
-	depends on OF_GPIO
-	select GPIOLIB_IRQCHIP
-	help
-	  Say yes here to support Spreadtrum PMIC EIC device.
-
 config GPIO_PXA
 	bool "PXA GPIO support"
 	depends on ARCH_PXA || ARCH_MMP || COMPILE_TEST
@@ -616,6 +623,17 @@ config GPIO_SYSCON
 	help
 	  Say yes here to support GPIO functionality though SYSCON driver.
 
+config GPIO_TANGIER
+	tristate
+	select GPIOLIB_IRQCHIP
+	help
+	  GPIO support for Intel Tangier and compatible platforms.
+	  Currently supported:
+	   - Elkhart Lake
+	   - Merrifield
+
+	  If built as a module its name will be gpio-tangier.
+
 config GPIO_TB10X
 	bool
 	select GPIO_GENERIC
@@ -1000,6 +1018,16 @@ config GPIO_ADNP
 	  enough to represent all pins, but the driver will assume a
 	  register layout for 64 pins (8 registers).
 
+config GPIO_FXL6408
+	tristate "FXL6408 I2C GPIO expander"
+	select GPIO_REGMAP
+	select REGMAP_I2C
+	help
+	  GPIO driver for Fairchild Semiconductor FXL6408 GPIO expander.
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called gpio-fxl6408.
+
 config GPIO_GW_PLD
 	tristate "Gateworks PLD GPIO Expander"
 	depends on OF_GPIO
@@ -1235,6 +1263,17 @@ config HTC_EGPIO
 	  several HTC phones.  It provides basic support for input
 	  pins, output pins, and IRQs.
 
+config GPIO_ELKHARTLAKE
+	tristate "Intel Elkhart Lake PSE GPIO support"
+	depends on X86 || COMPILE_TEST
+	select GPIO_TANGIER
+	help
+	  Select this option to enable GPIO support for Intel Elkhart Lake
+	  PSE GPIO IP.
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called gpio-elkhartlake.
+
 config GPIO_JANZ_TTL
 	tristate "Janz VMOD-TTL Digital IO Module"
 	depends on MFD_JANZ_CMODIO
@@ -1311,6 +1350,14 @@ config GPIO_PALMAS
 	  Select this option to enable GPIO driver for the TI PALMAS
 	  series chip family.
 
+config GPIO_PMIC_EIC_SPRD
+	tristate "Spreadtrum PMIC EIC support"
+	depends on MFD_SC27XX_PMIC || COMPILE_TEST
+	depends on OF_GPIO
+	select GPIOLIB_IRQCHIP
+	help
+	  Say yes here to support Spreadtrum PMIC EIC device.
+
 config GPIO_RC5T583
 	bool "RICOH RC5T583 GPIO"
 	depends on MFD_RC5T583
@@ -1505,7 +1552,7 @@ config GPIO_BT8XX
 config GPIO_MERRIFIELD
 	tristate "Intel Merrifield GPIO support"
 	depends on X86_INTEL_MID
-	select GPIOLIB_IRQCHIP
+	select GPIO_TANGIER
 	help
 	  Say Y here to support Intel Merrifield GPIO.
 
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index c048ba003367..8309b4ff128b 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -54,12 +54,14 @@ obj-$(CONFIG_GPIO_DAVINCI)		+= gpio-davinci.o
 obj-$(CONFIG_GPIO_DLN2)			+= gpio-dln2.o
 obj-$(CONFIG_GPIO_DWAPB)		+= gpio-dwapb.o
 obj-$(CONFIG_GPIO_EIC_SPRD)		+= gpio-eic-sprd.o
+obj-$(CONFIG_GPIO_ELKHARTLAKE)		+= gpio-elkhartlake.o
 obj-$(CONFIG_GPIO_EM)			+= gpio-em.o
 obj-$(CONFIG_GPIO_EN7523)		+= gpio-en7523.o
 obj-$(CONFIG_GPIO_EP93XX)		+= gpio-ep93xx.o
 obj-$(CONFIG_GPIO_EXAR)			+= gpio-exar.o
 obj-$(CONFIG_GPIO_F7188X)		+= gpio-f7188x.o
 obj-$(CONFIG_GPIO_FTGPIO010)		+= gpio-ftgpio010.o
+obj-$(CONFIG_GPIO_FXL6408)		+= gpio-fxl6408.o
 obj-$(CONFIG_GPIO_GE_FPGA)		+= gpio-ge.o
 obj-$(CONFIG_GPIO_GPIO_MM)		+= gpio-gpio-mm.o
 obj-$(CONFIG_GPIO_GRGPIO)		+= gpio-grgpio.o
@@ -80,6 +82,7 @@ obj-$(CONFIG_GPIO_LATCH)		+= gpio-latch.o
 obj-$(CONFIG_GPIO_LOGICVC)		+= gpio-logicvc.o
 obj-$(CONFIG_GPIO_LOONGSON1)		+= gpio-loongson1.o
 obj-$(CONFIG_GPIO_LOONGSON)		+= gpio-loongson.o
+obj-$(CONFIG_GPIO_LOONGSON_64BIT)	+= gpio-loongson-64bit.o
 obj-$(CONFIG_GPIO_LP3943)		+= gpio-lp3943.o
 obj-$(CONFIG_GPIO_LP873X)		+= gpio-lp873x.o
 obj-$(CONFIG_GPIO_LP87565)		+= gpio-lp87565.o
@@ -145,6 +148,7 @@ obj-$(CONFIG_GPIO_SPRD)			+= gpio-sprd.o
 obj-$(CONFIG_GPIO_STMPE)		+= gpio-stmpe.o
 obj-$(CONFIG_GPIO_STP_XWAY)		+= gpio-stp-xway.o
 obj-$(CONFIG_GPIO_SYSCON)		+= gpio-syscon.o
+obj-$(CONFIG_GPIO_TANGIER)		+= gpio-tangier.o
 obj-$(CONFIG_GPIO_TB10X)		+= gpio-tb10x.o
 obj-$(CONFIG_GPIO_TC3589X)		+= gpio-tc3589x.o
 obj-$(CONFIG_GPIO_TEGRA186)		+= gpio-tegra186.o
diff --git a/drivers/gpio/TODO b/drivers/gpio/TODO
index 68ada1066941..189c3abe7e79 100644
--- a/drivers/gpio/TODO
+++ b/drivers/gpio/TODO
@@ -59,11 +59,6 @@ the device tree back-end. It is legacy and should not be used in new code.
 
 Work items:
 
-- Get rid of struct of_mm_gpio_chip altogether: use the generic  MMIO
-  GPIO for all current users (see below). Delete struct of_mm_gpio_chip,
-  to_of_mm_gpio_chip(), of_mm_gpiochip_add_data(), of_mm_gpiochip_remove()
-  from the kernel.
-
 - Change all consumer drivers that #include <linux/of_gpio.h> to
   #include <linux/gpio/consumer.h> and stop doing custom parsing of the
   GPIO lines from the device tree. This can be tricky and often ivolves
@@ -81,6 +76,16 @@ Work items:
   uses <linux/gpio/consumer.h> or <linux/gpio/driver.h> instead.
 
 
+Get rid of <linux/gpio/legacy-of-mm-gpiochip.h>
+
+Work items:
+
+- Get rid of struct of_mm_gpio_chip altogether: use the generic  MMIO
+  GPIO for all current users (see below). Delete struct of_mm_gpio_chip,
+  to_of_mm_gpio_chip(), of_mm_gpiochip_add_data(), of_mm_gpiochip_remove(),
+  CONFIG_OF_GPIO_MM_GPIOCHIP from the kernel.
+
+
 Get rid of <linux/gpio.h>
 
 This legacy header is a one stop shop for anything GPIO is closely tied
diff --git a/drivers/gpio/gpio-104-dio-48e.c b/drivers/gpio/gpio-104-dio-48e.c
index a3846faf3780..f2253fd5ab4b 100644
--- a/drivers/gpio/gpio-104-dio-48e.c
+++ b/drivers/gpio/gpio-104-dio-48e.c
@@ -86,6 +86,7 @@ static const struct regmap_config dio48e_regmap_config = {
 	.volatile_table = &dio48e_volatile_table,
 	.precious_table = &dio48e_precious_table,
 	.cache_type = REGCACHE_FLAT,
+	.use_raw_spinlock = true,
 };
 
 /* only bit 3 on each respective Port C supports interrupts */
@@ -106,7 +107,6 @@ static int dio48e_handle_mask_sync(struct regmap *const map, const int index,
 {
 	unsigned int *const irq_mask = irq_drv_data;
 	const unsigned int prev_mask = *irq_mask;
-	const unsigned int all_masked = GENMASK(1, 0);
 	int err;
 	unsigned int val;
 
@@ -118,7 +118,7 @@ static int dio48e_handle_mask_sync(struct regmap *const map, const int index,
 	*irq_mask = mask_buf;
 
 	/* if all previously masked, enable interrupts when unmasking */
-	if (prev_mask == all_masked) {
+	if (prev_mask == mask_buf_def) {
 		err = regmap_write(map, DIO48E_CLEAR_INTERRUPT, 0x00);
 		if (err)
 			return err;
@@ -126,7 +126,7 @@ static int dio48e_handle_mask_sync(struct regmap *const map, const int index,
 	}
 
 	/* if all are currently masked, disable interrupts */
-	if (mask_buf == all_masked)
+	if (mask_buf == mask_buf_def)
 		return regmap_read(map, DIO48E_DISABLE_INTERRUPT, &val);
 
 	return 0;
@@ -195,13 +195,9 @@ static int dio48e_probe(struct device *dev, unsigned int id)
 		return -ENOMEM;
 
 	chip->name = name;
-	/* No IRQ status register so use CLEAR_INTERRUPT register instead */
-	chip->status_base = DIO48E_CLEAR_INTERRUPT;
 	chip->mask_base = DIO48E_ENABLE_INTERRUPT;
 	chip->ack_base = DIO48E_CLEAR_INTERRUPT;
-	/* CLEAR_INTERRUPT doubles as status register so we need it cleared */
-	chip->clear_ack = true;
-	chip->status_invert = true;
+	chip->no_status = true;
 	chip->num_regs = 1;
 	chip->irqs = dio48e_regmap_irqs;
 	chip->num_irqs = ARRAY_SIZE(dio48e_regmap_irqs);
diff --git a/drivers/gpio/gpio-104-idi-48.c b/drivers/gpio/gpio-104-idi-48.c
index ca2175b84e24..ba73ee9c0c29 100644
--- a/drivers/gpio/gpio-104-idi-48.c
+++ b/drivers/gpio/gpio-104-idi-48.c
@@ -81,6 +81,7 @@ static const struct regmap_config idi48_regmap_config = {
 	.wr_table = &idi_48_wr_table,
 	.rd_table = &idi_48_rd_table,
 	.precious_table = &idi_48_precious_table,
+	.use_raw_spinlock = true,
 };
 
 #define IDI48_NGPIO 48
diff --git a/drivers/gpio/gpio-adnp.c b/drivers/gpio/gpio-adnp.c
index a6439e3daff0..9b01c391efce 100644
--- a/drivers/gpio/gpio-adnp.c
+++ b/drivers/gpio/gpio-adnp.c
@@ -307,6 +307,7 @@ static void adnp_irq_mask(struct irq_data *d)
 	unsigned int pos = d->hwirq & 7;
 
 	adnp->irq_enable[reg] &= ~BIT(pos);
+	gpiochip_disable_irq(gc, irqd_to_hwirq(d));
 }
 
 static void adnp_irq_unmask(struct irq_data *d)
@@ -316,6 +317,7 @@ static void adnp_irq_unmask(struct irq_data *d)
 	unsigned int reg = d->hwirq >> adnp->reg_shift;
 	unsigned int pos = d->hwirq & 7;
 
+	gpiochip_enable_irq(gc, irqd_to_hwirq(d));
 	adnp->irq_enable[reg] |= BIT(pos);
 }
 
@@ -372,13 +374,15 @@ static void adnp_irq_bus_unlock(struct irq_data *d)
 	mutex_unlock(&adnp->irq_lock);
 }
 
-static struct irq_chip adnp_irq_chip = {
+static const struct irq_chip adnp_irq_chip = {
 	.name = "gpio-adnp",
 	.irq_mask = adnp_irq_mask,
 	.irq_unmask = adnp_irq_unmask,
 	.irq_set_type = adnp_irq_set_type,
 	.irq_bus_lock = adnp_irq_bus_lock,
 	.irq_bus_sync_unlock = adnp_irq_bus_unlock,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static int adnp_irq_setup(struct adnp *adnp)
@@ -469,7 +473,8 @@ static int adnp_gpio_setup(struct adnp *adnp, unsigned int num_gpios,
 			return err;
 
 		girq = &chip->irq;
-		girq->chip = &adnp_irq_chip;
+		gpio_irq_chip_set_chip(girq, &adnp_irq_chip);
+
 		/* This will let us handle the parent IRQ in the driver */
 		girq->parent_handler = NULL;
 		girq->num_parents = 0;
diff --git a/drivers/gpio/gpio-aggregator.c b/drivers/gpio/gpio-aggregator.c
index 6d17d262ad91..20a686f12df7 100644
--- a/drivers/gpio/gpio-aggregator.c
+++ b/drivers/gpio/gpio-aggregator.c
@@ -10,19 +10,20 @@
 #include <linux/bitmap.h>
 #include <linux/bitops.h>
 #include <linux/ctype.h>
-#include <linux/gpio.h>
-#include <linux/gpio/consumer.h>
-#include <linux/gpio/driver.h>
-#include <linux/gpio/machine.h>
 #include <linux/idr.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/overflow.h>
 #include <linux/platform_device.h>
+#include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
 
+#include <linux/gpio/consumer.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/machine.h>
+
 #define AGGREGATOR_MAX_GPIOS 512
 
 /*
diff --git a/drivers/gpio/gpio-altera.c b/drivers/gpio/gpio-altera.c
index b59fae993626..54d7c450c596 100644
--- a/drivers/gpio/gpio-altera.c
+++ b/drivers/gpio/gpio-altera.c
@@ -7,7 +7,7 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/gpio/driver.h>
-#include <linux/of_gpio.h> /* For of_mm_gpio_chip */
+#include <linux/gpio/legacy-of-mm-gpiochip.h>
 #include <linux/platform_device.h>
 
 #define ALTERA_GPIO_MAX_NGPIO		32
@@ -24,14 +24,12 @@
 * @interrupt_trigger	: specifies the hardware configured IRQ trigger type
 *			  (rising, falling, both, high)
 * @mapped_irq		: kernel mapped irq number.
-* @irq_chip		: IRQ chip configuration
 */
 struct altera_gpio_chip {
 	struct of_mm_gpio_chip mmchip;
 	raw_spinlock_t gpio_lock;
 	int interrupt_trigger;
 	int mapped_irq;
-	struct irq_chip irq_chip;
 };
 
 static void altera_gpio_irq_unmask(struct irq_data *d)
@@ -43,6 +41,7 @@ static void altera_gpio_irq_unmask(struct irq_data *d)
 
 	altera_gc = gpiochip_get_data(irq_data_get_irq_chip_data(d));
 	mm_gc = &altera_gc->mmchip;
+	gpiochip_enable_irq(&mm_gc->gc, irqd_to_hwirq(d));
 
 	raw_spin_lock_irqsave(&altera_gc->gpio_lock, flags);
 	intmask = readl(mm_gc->regs + ALTERA_GPIO_IRQ_MASK);
@@ -68,6 +67,7 @@ static void altera_gpio_irq_mask(struct irq_data *d)
 	intmask &= ~BIT(irqd_to_hwirq(d));
 	writel(intmask, mm_gc->regs + ALTERA_GPIO_IRQ_MASK);
 	raw_spin_unlock_irqrestore(&altera_gc->gpio_lock, flags);
+	gpiochip_disable_irq(&mm_gc->gc, irqd_to_hwirq(d));
 }
 
 /*
@@ -233,6 +233,17 @@ static void altera_gpio_irq_leveL_high_handler(struct irq_desc *desc)
 	chained_irq_exit(chip, desc);
 }
 
+static const struct irq_chip altera_gpio_irq_chip = {
+	.name = "altera-gpio",
+	.irq_mask = altera_gpio_irq_mask,
+	.irq_unmask = altera_gpio_irq_unmask,
+	.irq_set_type = altera_gpio_irq_set_type,
+	.irq_startup  = altera_gpio_irq_startup,
+	.irq_shutdown = altera_gpio_irq_mask,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 static int altera_gpio_probe(struct platform_device *pdev)
 {
 	struct device_node *node = pdev->dev.of_node;
@@ -278,15 +289,9 @@ static int altera_gpio_probe(struct platform_device *pdev)
 	}
 	altera_gc->interrupt_trigger = reg;
 
-	altera_gc->irq_chip.name = "altera-gpio";
-	altera_gc->irq_chip.irq_mask     = altera_gpio_irq_mask;
-	altera_gc->irq_chip.irq_unmask   = altera_gpio_irq_unmask;
-	altera_gc->irq_chip.irq_set_type = altera_gpio_irq_set_type;
-	altera_gc->irq_chip.irq_startup  = altera_gpio_irq_startup;
-	altera_gc->irq_chip.irq_shutdown = altera_gpio_irq_mask;
-
 	girq = &altera_gc->mmchip.gc.irq;
-	girq->chip = &altera_gc->irq_chip;
+	gpio_irq_chip_set_chip(girq, &altera_gpio_irq_chip);
+
 	if (altera_gc->interrupt_trigger == IRQ_TYPE_LEVEL_HIGH)
 		girq->parent_handler = altera_gpio_irq_leveL_high_handler;
 	else
@@ -330,7 +335,7 @@ MODULE_DEVICE_TABLE(of, altera_gpio_of_match);
 static struct platform_driver altera_gpio_driver = {
 	.driver = {
 		.name	= "altera_gpio",
-		.of_match_table = of_match_ptr(altera_gpio_of_match),
+		.of_match_table = altera_gpio_of_match,
 	},
 	.probe		= altera_gpio_probe,
 	.remove		= altera_gpio_remove,
diff --git a/drivers/gpio/gpio-aspeed-sgpio.c b/drivers/gpio/gpio-aspeed-sgpio.c
index 454cefbeecf0..72755fee6478 100644
--- a/drivers/gpio/gpio-aspeed-sgpio.c
+++ b/drivers/gpio/gpio-aspeed-sgpio.c
@@ -14,6 +14,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/seq_file.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
 
@@ -29,7 +30,7 @@ struct aspeed_sgpio_pdata {
 
 struct aspeed_sgpio {
 	struct gpio_chip chip;
-	struct irq_chip intc;
+	struct device *dev;
 	struct clk *pclk;
 	raw_spinlock_t lock;
 	void __iomem *base;
@@ -296,6 +297,10 @@ static void aspeed_sgpio_irq_set_mask(struct irq_data *d, bool set)
 	irqd_to_aspeed_sgpio_data(d, &gpio, &bank, &bit, &offset);
 	addr = bank_reg(gpio, bank, reg_irq_enable);
 
+	/* Unmasking the IRQ */
+	if (set)
+		gpiochip_enable_irq(&gpio->chip, irqd_to_hwirq(d));
+
 	raw_spin_lock_irqsave(&gpio->lock, flags);
 
 	reg = ioread32(addr);
@@ -307,6 +312,12 @@ static void aspeed_sgpio_irq_set_mask(struct irq_data *d, bool set)
 	iowrite32(reg, addr);
 
 	raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+	/* Masking the IRQ */
+	if (!set)
+		gpiochip_disable_irq(&gpio->chip, irqd_to_hwirq(d));
+
+
 }
 
 static void aspeed_sgpio_irq_mask(struct irq_data *d)
@@ -401,6 +412,27 @@ static void aspeed_sgpio_irq_handler(struct irq_desc *desc)
 	chained_irq_exit(ic, desc);
 }
 
+static void aspeed_sgpio_irq_print_chip(struct irq_data *d, struct seq_file *p)
+{
+	const struct aspeed_sgpio_bank *bank;
+	struct aspeed_sgpio *gpio;
+	u32 bit;
+	int offset;
+
+	irqd_to_aspeed_sgpio_data(d, &gpio, &bank, &bit, &offset);
+	seq_printf(p, dev_name(gpio->dev));
+}
+
+static const struct irq_chip aspeed_sgpio_irq_chip = {
+	.irq_ack = aspeed_sgpio_irq_ack,
+	.irq_mask = aspeed_sgpio_irq_mask,
+	.irq_unmask = aspeed_sgpio_irq_unmask,
+	.irq_set_type = aspeed_sgpio_set_type,
+	.irq_print_chip = aspeed_sgpio_irq_print_chip,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 static int aspeed_sgpio_setup_irqs(struct aspeed_sgpio *gpio,
 				   struct platform_device *pdev)
 {
@@ -423,14 +455,8 @@ static int aspeed_sgpio_setup_irqs(struct aspeed_sgpio *gpio,
 		iowrite32(0xffffffff, bank_reg(gpio, bank, reg_irq_status));
 	}
 
-	gpio->intc.name = dev_name(&pdev->dev);
-	gpio->intc.irq_ack = aspeed_sgpio_irq_ack;
-	gpio->intc.irq_mask = aspeed_sgpio_irq_mask;
-	gpio->intc.irq_unmask = aspeed_sgpio_irq_unmask;
-	gpio->intc.irq_set_type = aspeed_sgpio_set_type;
-
 	irq = &gpio->chip.irq;
-	irq->chip = &gpio->intc;
+	gpio_irq_chip_set_chip(irq, &aspeed_sgpio_irq_chip);
 	irq->init_valid_mask = aspeed_sgpio_irq_init_valid_mask;
 	irq->handler = handle_bad_irq;
 	irq->default_type = IRQ_TYPE_NONE;
@@ -524,6 +550,8 @@ static int __init aspeed_sgpio_probe(struct platform_device *pdev)
 	if (IS_ERR(gpio->base))
 		return PTR_ERR(gpio->base);
 
+	gpio->dev = &pdev->dev;
+
 	pdata = device_get_match_data(&pdev->dev);
 	if (!pdata)
 		return -EINVAL;
@@ -609,4 +637,3 @@ static struct platform_driver aspeed_sgpio_driver = {
 
 module_platform_driver_probe(aspeed_sgpio_driver, aspeed_sgpio_probe);
 MODULE_DESCRIPTION("Aspeed Serial GPIO Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-aspeed.c b/drivers/gpio/gpio-aspeed.c
index a94da80d3a95..da33bbbdacb9 100644
--- a/drivers/gpio/gpio-aspeed.c
+++ b/drivers/gpio/gpio-aspeed.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
+#include <linux/seq_file.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
 
@@ -53,7 +54,7 @@ struct aspeed_gpio_config {
  */
 struct aspeed_gpio {
 	struct gpio_chip chip;
-	struct irq_chip irqc;
+	struct device *dev;
 	raw_spinlock_t lock;
 	void __iomem *base;
 	int irq;
@@ -566,6 +567,10 @@ static void aspeed_gpio_irq_set_mask(struct irq_data *d, bool set)
 
 	addr = bank_reg(gpio, bank, reg_irq_enable);
 
+	/* Unmasking the IRQ */
+	if (set)
+		gpiochip_enable_irq(&gpio->chip, irqd_to_hwirq(d));
+
 	raw_spin_lock_irqsave(&gpio->lock, flags);
 	copro = aspeed_gpio_copro_request(gpio, offset);
 
@@ -579,6 +584,10 @@ static void aspeed_gpio_irq_set_mask(struct irq_data *d, bool set)
 	if (copro)
 		aspeed_gpio_copro_release(gpio, offset);
 	raw_spin_unlock_irqrestore(&gpio->lock, flags);
+
+	/* Masking the IRQ */
+	if (!set)
+		gpiochip_disable_irq(&gpio->chip, irqd_to_hwirq(d));
 }
 
 static void aspeed_gpio_irq_mask(struct irq_data *d)
@@ -1080,6 +1089,30 @@ int aspeed_gpio_copro_release_gpio(struct gpio_desc *desc)
 }
 EXPORT_SYMBOL_GPL(aspeed_gpio_copro_release_gpio);
 
+static void aspeed_gpio_irq_print_chip(struct irq_data *d, struct seq_file *p)
+{
+	const struct aspeed_gpio_bank *bank;
+	struct aspeed_gpio *gpio;
+	u32 bit;
+	int rc, offset;
+
+	rc = irqd_to_aspeed_gpio_data(d, &gpio, &bank, &bit, &offset);
+	if (rc)
+		return;
+
+	seq_printf(p, dev_name(gpio->dev));
+}
+
+static const struct irq_chip aspeed_gpio_irq_chip = {
+	.irq_ack = aspeed_gpio_irq_ack,
+	.irq_mask = aspeed_gpio_irq_mask,
+	.irq_unmask = aspeed_gpio_irq_unmask,
+	.irq_set_type = aspeed_gpio_set_type,
+	.irq_print_chip = aspeed_gpio_irq_print_chip,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 /*
  * Any banks not specified in a struct aspeed_bank_props array are assumed to
  * have the properties:
@@ -1137,8 +1170,9 @@ MODULE_DEVICE_TABLE(of, aspeed_gpio_of_table);
 static int __init aspeed_gpio_probe(struct platform_device *pdev)
 {
 	const struct of_device_id *gpio_id;
+	struct gpio_irq_chip *girq;
 	struct aspeed_gpio *gpio;
-	int rc, i, banks, err;
+	int rc, irq, i, banks, err;
 	u32 ngpio;
 
 	gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
@@ -1149,6 +1183,8 @@ static int __init aspeed_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(gpio->base))
 		return PTR_ERR(gpio->base);
 
+	gpio->dev = &pdev->dev;
+
 	raw_spin_lock_init(&gpio->lock);
 
 	gpio_id = of_match_node(aspeed_gpio_of_table, pdev->dev.of_node);
@@ -1201,31 +1237,23 @@ static int __init aspeed_gpio_probe(struct platform_device *pdev)
 		aspeed_gpio_change_cmd_source(gpio, bank, 3, GPIO_CMDSRC_ARM);
 	}
 
-	/* Optionally set up an irqchip if there is an IRQ */
-	rc = platform_get_irq(pdev, 0);
-	if (rc > 0) {
-		struct gpio_irq_chip *girq;
-
-		gpio->irq = rc;
-		girq = &gpio->chip.irq;
-		girq->chip = &gpio->irqc;
-		girq->chip->name = dev_name(&pdev->dev);
-		girq->chip->irq_ack = aspeed_gpio_irq_ack;
-		girq->chip->irq_mask = aspeed_gpio_irq_mask;
-		girq->chip->irq_unmask = aspeed_gpio_irq_unmask;
-		girq->chip->irq_set_type = aspeed_gpio_set_type;
-		girq->parent_handler = aspeed_gpio_irq_handler;
-		girq->num_parents = 1;
-		girq->parents = devm_kcalloc(&pdev->dev, 1,
-					     sizeof(*girq->parents),
-					     GFP_KERNEL);
-		if (!girq->parents)
-			return -ENOMEM;
-		girq->parents[0] = gpio->irq;
-		girq->default_type = IRQ_TYPE_NONE;
-		girq->handler = handle_bad_irq;
-		girq->init_valid_mask = aspeed_init_irq_valid_mask;
-	}
+	/* Set up an irqchip */
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+	gpio->irq = irq;
+	girq = &gpio->chip.irq;
+	gpio_irq_chip_set_chip(girq, &aspeed_gpio_irq_chip);
+
+	girq->parent_handler = aspeed_gpio_irq_handler;
+	girq->num_parents = 1;
+	girq->parents = devm_kcalloc(&pdev->dev, 1, sizeof(*girq->parents), GFP_KERNEL);
+	if (!girq->parents)
+		return -ENOMEM;
+	girq->parents[0] = gpio->irq;
+	girq->default_type = IRQ_TYPE_NONE;
+	girq->handler = handle_bad_irq;
+	girq->init_valid_mask = aspeed_init_irq_valid_mask;
 
 	gpio->offset_timer =
 		devm_kzalloc(&pdev->dev, gpio->chip.ngpio, GFP_KERNEL);
diff --git a/drivers/gpio/gpio-ath79.c b/drivers/gpio/gpio-ath79.c
index 3958c6d97639..aa0a954b8392 100644
--- a/drivers/gpio/gpio-ath79.c
+++ b/drivers/gpio/gpio-ath79.c
@@ -71,6 +71,7 @@ static void ath79_gpio_irq_unmask(struct irq_data *data)
 	u32 mask = BIT(irqd_to_hwirq(data));
 	unsigned long flags;
 
+	gpiochip_enable_irq(&ctrl->gc, irqd_to_hwirq(data));
 	raw_spin_lock_irqsave(&ctrl->lock, flags);
 	ath79_gpio_update_bits(ctrl, AR71XX_GPIO_REG_INT_MASK, mask, mask);
 	raw_spin_unlock_irqrestore(&ctrl->lock, flags);
@@ -85,6 +86,7 @@ static void ath79_gpio_irq_mask(struct irq_data *data)
 	raw_spin_lock_irqsave(&ctrl->lock, flags);
 	ath79_gpio_update_bits(ctrl, AR71XX_GPIO_REG_INT_MASK, mask, 0);
 	raw_spin_unlock_irqrestore(&ctrl->lock, flags);
+	gpiochip_disable_irq(&ctrl->gc, irqd_to_hwirq(data));
 }
 
 static void ath79_gpio_irq_enable(struct irq_data *data)
@@ -169,13 +171,15 @@ static int ath79_gpio_irq_set_type(struct irq_data *data,
 	return 0;
 }
 
-static struct irq_chip ath79_gpio_irqchip = {
+static const struct irq_chip ath79_gpio_irqchip = {
 	.name = "gpio-ath79",
 	.irq_enable = ath79_gpio_irq_enable,
 	.irq_disable = ath79_gpio_irq_disable,
 	.irq_mask = ath79_gpio_irq_mask,
 	.irq_unmask = ath79_gpio_irq_unmask,
 	.irq_set_type = ath79_gpio_irq_set_type,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static void ath79_gpio_irq_handler(struct irq_desc *desc)
@@ -274,7 +278,7 @@ static int ath79_gpio_probe(struct platform_device *pdev)
 	/* Optional interrupt setup */
 	if (!np || of_property_read_bool(np, "interrupt-controller")) {
 		girq = &ctrl->gc.irq;
-		girq->chip = &ath79_gpio_irqchip;
+		gpio_irq_chip_set_chip(girq, &ath79_gpio_irqchip);
 		girq->parent_handler = ath79_gpio_irq_handler;
 		girq->num_parents = 1;
 		girq->parents = devm_kcalloc(dev, 1, sizeof(*girq->parents),
diff --git a/drivers/gpio/gpio-cadence.c b/drivers/gpio/gpio-cadence.c
index 137aea49ba02..3720b90cad10 100644
--- a/drivers/gpio/gpio-cadence.c
+++ b/drivers/gpio/gpio-cadence.c
@@ -70,6 +70,7 @@ static void cdns_gpio_irq_mask(struct irq_data *d)
 	struct cdns_gpio_chip *cgpio = gpiochip_get_data(chip);
 
 	iowrite32(BIT(d->hwirq), cgpio->regs + CDNS_GPIO_IRQ_DIS);
+	gpiochip_disable_irq(chip, irqd_to_hwirq(d));
 }
 
 static void cdns_gpio_irq_unmask(struct irq_data *d)
@@ -77,6 +78,7 @@ static void cdns_gpio_irq_unmask(struct irq_data *d)
 	struct gpio_chip *chip = irq_data_get_irq_chip_data(d);
 	struct cdns_gpio_chip *cgpio = gpiochip_get_data(chip);
 
+	gpiochip_enable_irq(chip, irqd_to_hwirq(d));
 	iowrite32(BIT(d->hwirq), cgpio->regs + CDNS_GPIO_IRQ_EN);
 }
 
@@ -138,11 +140,13 @@ static void cdns_gpio_irq_handler(struct irq_desc *desc)
 	chained_irq_exit(irqchip, desc);
 }
 
-static struct irq_chip cdns_gpio_irqchip = {
+static const struct irq_chip cdns_gpio_irqchip = {
 	.name		= "cdns-gpio",
 	.irq_mask	= cdns_gpio_irq_mask,
 	.irq_unmask	= cdns_gpio_irq_unmask,
-	.irq_set_type	= cdns_gpio_irq_set_type
+	.irq_set_type	= cdns_gpio_irq_set_type,
+	.flags		= IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static int cdns_gpio_probe(struct platform_device *pdev)
@@ -222,7 +226,7 @@ static int cdns_gpio_probe(struct platform_device *pdev)
 		struct gpio_irq_chip *girq;
 
 		girq = &cgpio->gc.irq;
-		girq->chip = &cdns_gpio_irqchip;
+		gpio_irq_chip_set_chip(girq, &cdns_gpio_irqchip);
 		girq->parent_handler = cdns_gpio_irq_handler;
 		girq->num_parents = 1;
 		girq->parents = devm_kcalloc(&pdev->dev, 1,
diff --git a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c
index 43b2dc8821e6..aaaf61dc2632 100644
--- a/drivers/gpio/gpio-davinci.c
+++ b/drivers/gpio/gpio-davinci.c
@@ -24,8 +24,6 @@
 #include <linux/spinlock.h>
 #include <linux/pm_runtime.h>
 
-#include <asm-generic/gpio.h>
-
 #define MAX_REGS_BANKS 5
 #define MAX_INT_PER_BANK 32
 
diff --git a/drivers/gpio/gpio-elkhartlake.c b/drivers/gpio/gpio-elkhartlake.c
new file mode 100644
index 000000000000..a9c8b16215be
--- /dev/null
+++ b/drivers/gpio/gpio-elkhartlake.c
@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Intel Elkhart Lake PSE GPIO driver
+ *
+ * Copyright (c) 2023 Intel Corporation.
+ *
+ * Authors: Pandith N <pandith.n@intel.com>
+ *          Raag Jadav <raag.jadav@intel.com>
+ */
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+
+#include "gpio-tangier.h"
+
+/* Each Intel EHL PSE GPIO Controller has 30 GPIO pins */
+#define EHL_PSE_NGPIO		30
+
+static int ehl_gpio_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct tng_gpio *priv;
+	int irq, ret;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(priv->reg_base))
+		return PTR_ERR(priv->reg_base);
+
+	priv->dev = dev;
+	priv->irq = irq;
+
+	priv->info.base = -1;
+	priv->info.ngpio = EHL_PSE_NGPIO;
+
+	priv->wake_regs.gwmr = GWMR_EHL;
+	priv->wake_regs.gwsr = GWSR_EHL;
+	priv->wake_regs.gsir = GSIR_EHL;
+
+	ret = devm_tng_gpio_probe(dev, priv);
+	if (ret)
+		return dev_err_probe(dev, ret, "tng_gpio_probe error\n");
+
+	platform_set_drvdata(pdev, priv);
+	return 0;
+}
+
+static int ehl_gpio_suspend(struct device *dev)
+{
+	return tng_gpio_suspend(dev);
+}
+
+static int ehl_gpio_resume(struct device *dev)
+{
+	return tng_gpio_resume(dev);
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(ehl_gpio_pm_ops, ehl_gpio_suspend, ehl_gpio_resume);
+
+static const struct platform_device_id ehl_gpio_ids[] = {
+	{ "gpio-elkhartlake" },
+	{ }
+};
+MODULE_DEVICE_TABLE(platform, ehl_gpio_ids);
+
+static struct platform_driver ehl_gpio_driver = {
+	.driver	= {
+		.name	= "gpio-elkhartlake",
+		.pm	= pm_sleep_ptr(&ehl_gpio_pm_ops),
+	},
+	.probe		= ehl_gpio_probe,
+	.id_table	= ehl_gpio_ids,
+};
+module_platform_driver(ehl_gpio_driver);
+
+MODULE_AUTHOR("Pandith N <pandith.n@intel.com>");
+MODULE_AUTHOR("Raag Jadav <raag.jadav@intel.com>");
+MODULE_DESCRIPTION("Intel Elkhart Lake PSE GPIO driver");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS(GPIO_TANGIER);
diff --git a/drivers/gpio/gpio-ftgpio010.c b/drivers/gpio/gpio-ftgpio010.c
index 2728672ef9f8..31e26072f6ae 100644
--- a/drivers/gpio/gpio-ftgpio010.c
+++ b/drivers/gpio/gpio-ftgpio010.c
@@ -349,7 +349,7 @@ static const struct of_device_id ftgpio_gpio_of_match[] = {
 static struct platform_driver ftgpio_gpio_driver = {
 	.driver = {
 		.name		= "ftgpio010-gpio",
-		.of_match_table = of_match_ptr(ftgpio_gpio_of_match),
+		.of_match_table = ftgpio_gpio_of_match,
 	},
 	.probe = ftgpio_gpio_probe,
 	.remove = ftgpio_gpio_remove,
diff --git a/drivers/gpio/gpio-fxl6408.c b/drivers/gpio/gpio-fxl6408.c
new file mode 100644
index 000000000000..208fa851e82a
--- /dev/null
+++ b/drivers/gpio/gpio-fxl6408.c
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * FXL6408 GPIO driver
+ *
+ * Copyright 2023 Toradex
+ *
+ * Author: Emanuele Ghidoli <emanuele.ghidoli@toradex.com>
+ */
+
+#include <linux/err.h>
+#include <linux/gpio/regmap.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+
+#define FXL6408_REG_DEVICE_ID		0x01
+#define FXL6408_MF_FAIRCHILD		0b101
+#define FXL6408_MF_SHIFT		5
+
+/* Bits set here indicate that the GPIO is an output. */
+#define FXL6408_REG_IO_DIR		0x03
+
+/*
+ * Bits set here, when the corresponding bit of IO_DIR is set, drive
+ * the output high instead of low.
+ */
+#define FXL6408_REG_OUTPUT		0x05
+
+/* Bits here make the output High-Z, instead of the OUTPUT value. */
+#define FXL6408_REG_OUTPUT_HIGH_Z	0x07
+
+/* Returns the current status (1 = HIGH) of the input pins. */
+#define FXL6408_REG_INPUT_STATUS	0x0f
+
+/*
+ * Return the current interrupt status
+ * This bit is HIGH if input GPIO != default state (register 09h).
+ * The flag is cleared after being read (bit returns to 0).
+ * The input must go back to default state and change again before this flag is raised again.
+ */
+#define FXL6408_REG_INT_STS		0x13
+
+#define FXL6408_NGPIO			8
+
+static const struct regmap_range rd_range[] = {
+	{ FXL6408_REG_DEVICE_ID, FXL6408_REG_DEVICE_ID },
+	{ FXL6408_REG_IO_DIR, FXL6408_REG_OUTPUT },
+	{ FXL6408_REG_INPUT_STATUS, FXL6408_REG_INPUT_STATUS },
+};
+
+static const struct regmap_range wr_range[] = {
+	{ FXL6408_REG_DEVICE_ID, FXL6408_REG_DEVICE_ID },
+	{ FXL6408_REG_IO_DIR, FXL6408_REG_OUTPUT },
+	{ FXL6408_REG_OUTPUT_HIGH_Z, FXL6408_REG_OUTPUT_HIGH_Z },
+};
+
+static const struct regmap_range volatile_range[] = {
+	{ FXL6408_REG_DEVICE_ID, FXL6408_REG_DEVICE_ID },
+	{ FXL6408_REG_INPUT_STATUS, FXL6408_REG_INPUT_STATUS },
+};
+
+static const struct regmap_access_table rd_table = {
+	.yes_ranges = rd_range,
+	.n_yes_ranges = ARRAY_SIZE(rd_range),
+};
+
+static const struct regmap_access_table wr_table = {
+	.yes_ranges = wr_range,
+	.n_yes_ranges = ARRAY_SIZE(wr_range),
+};
+
+static const struct regmap_access_table volatile_table = {
+	.yes_ranges = volatile_range,
+	.n_yes_ranges = ARRAY_SIZE(volatile_range),
+};
+
+static const struct regmap_config regmap = {
+	.reg_bits = 8,
+	.val_bits = 8,
+
+	.max_register = FXL6408_REG_INT_STS,
+	.wr_table = &wr_table,
+	.rd_table = &rd_table,
+	.volatile_table = &volatile_table,
+
+	.cache_type = REGCACHE_RBTREE,
+	.num_reg_defaults_raw = FXL6408_REG_INT_STS + 1,
+};
+
+static int fxl6408_identify(struct device *dev, struct regmap *regmap)
+{
+	int val, ret;
+
+	ret = regmap_read(regmap, FXL6408_REG_DEVICE_ID, &val);
+	if (ret)
+		return dev_err_probe(dev, ret, "error reading DEVICE_ID\n");
+	if (val >> FXL6408_MF_SHIFT != FXL6408_MF_FAIRCHILD)
+		return dev_err_probe(dev, -ENODEV, "invalid device id 0x%02x\n", val);
+
+	return 0;
+}
+
+static int fxl6408_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	int ret;
+	struct gpio_regmap_config gpio_config = {
+		.parent = dev,
+		.ngpio = FXL6408_NGPIO,
+		.reg_dat_base = GPIO_REGMAP_ADDR(FXL6408_REG_INPUT_STATUS),
+		.reg_set_base = GPIO_REGMAP_ADDR(FXL6408_REG_OUTPUT),
+		.reg_dir_out_base = GPIO_REGMAP_ADDR(FXL6408_REG_IO_DIR),
+		.ngpio_per_reg = FXL6408_NGPIO,
+	};
+
+	gpio_config.regmap = devm_regmap_init_i2c(client, &regmap);
+	if (IS_ERR(gpio_config.regmap))
+		return dev_err_probe(dev, PTR_ERR(gpio_config.regmap),
+				     "failed to allocate register map\n");
+
+	ret = fxl6408_identify(dev, gpio_config.regmap);
+	if (ret)
+		return ret;
+
+	/* Disable High-Z of outputs, so that our OUTPUT updates actually take effect. */
+	ret = regmap_write(gpio_config.regmap, FXL6408_REG_OUTPUT_HIGH_Z, 0);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to write 'output high Z' register\n");
+
+	return PTR_ERR_OR_ZERO(devm_gpio_regmap_register(dev, &gpio_config));
+}
+
+static const __maybe_unused struct of_device_id fxl6408_dt_ids[] = {
+	{ .compatible = "fcs,fxl6408" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, fxl6408_dt_ids);
+
+static const struct i2c_device_id fxl6408_id[] = {
+	{ "fxl6408", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, fxl6408_id);
+
+static struct i2c_driver fxl6408_driver = {
+	.driver = {
+		.name	= "fxl6408",
+		.of_match_table = fxl6408_dt_ids,
+	},
+	.probe_new	= fxl6408_probe,
+	.id_table	= fxl6408_id,
+};
+module_i2c_driver(fxl6408_driver);
+
+MODULE_AUTHOR("Emanuele Ghidoli <emanuele.ghidoli@toradex.com>");
+MODULE_DESCRIPTION("FXL6408 GPIO driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-hisi.c b/drivers/gpio/gpio-hisi.c
index 55bd69043bf4..29a03de37fd8 100644
--- a/drivers/gpio/gpio-hisi.c
+++ b/drivers/gpio/gpio-hisi.c
@@ -37,7 +37,6 @@ struct hisi_gpio {
 	struct device		*dev;
 	void __iomem		*reg_base;
 	unsigned int		line_num;
-	struct irq_chip		irq_chip;
 	int			irq;
 };
 
@@ -100,12 +99,14 @@ static void hisi_gpio_irq_set_mask(struct irq_data *d)
 	struct gpio_chip *chip = irq_data_get_irq_chip_data(d);
 
 	hisi_gpio_write_reg(chip, HISI_GPIO_INTMASK_SET_WX, BIT(irqd_to_hwirq(d)));
+	gpiochip_disable_irq(chip, irqd_to_hwirq(d));
 }
 
 static void hisi_gpio_irq_clr_mask(struct irq_data *d)
 {
 	struct gpio_chip *chip = irq_data_get_irq_chip_data(d);
 
+	gpiochip_enable_irq(chip, irqd_to_hwirq(d));
 	hisi_gpio_write_reg(chip, HISI_GPIO_INTMASK_CLR_WX, BIT(irqd_to_hwirq(d)));
 }
 
@@ -191,20 +192,24 @@ static void hisi_gpio_irq_handler(struct irq_desc *desc)
 	chained_irq_exit(irq_c, desc);
 }
 
+static const struct irq_chip hisi_gpio_irq_chip = {
+	.name = "HISI-GPIO",
+	.irq_ack = hisi_gpio_set_ack,
+	.irq_mask = hisi_gpio_irq_set_mask,
+	.irq_unmask = hisi_gpio_irq_clr_mask,
+	.irq_set_type = hisi_gpio_irq_set_type,
+	.irq_enable = hisi_gpio_irq_enable,
+	.irq_disable = hisi_gpio_irq_disable,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 static void hisi_gpio_init_irq(struct hisi_gpio *hisi_gpio)
 {
 	struct gpio_chip *chip = &hisi_gpio->chip;
 	struct gpio_irq_chip *girq_chip = &chip->irq;
 
-	/* Set hooks for irq_chip */
-	hisi_gpio->irq_chip.irq_ack = hisi_gpio_set_ack;
-	hisi_gpio->irq_chip.irq_mask = hisi_gpio_irq_set_mask;
-	hisi_gpio->irq_chip.irq_unmask = hisi_gpio_irq_clr_mask;
-	hisi_gpio->irq_chip.irq_set_type = hisi_gpio_irq_set_type;
-	hisi_gpio->irq_chip.irq_enable = hisi_gpio_irq_enable;
-	hisi_gpio->irq_chip.irq_disable = hisi_gpio_irq_disable;
-
-	girq_chip->chip = &hisi_gpio->irq_chip;
+	gpio_irq_chip_set_chip(girq_chip, &hisi_gpio_irq_chip);
 	girq_chip->default_type = IRQ_TYPE_NONE;
 	girq_chip->num_parents = 1;
 	girq_chip->parents = &hisi_gpio->irq;
diff --git a/drivers/gpio/gpio-hlwd.c b/drivers/gpio/gpio-hlwd.c
index 4e13e937f832..c208ac1c54a6 100644
--- a/drivers/gpio/gpio-hlwd.c
+++ b/drivers/gpio/gpio-hlwd.c
@@ -11,6 +11,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
+#include <linux/seq_file.h>
 #include <linux/slab.h>
 
 /*
@@ -48,7 +49,7 @@
 
 struct hlwd_gpio {
 	struct gpio_chip gpioc;
-	struct irq_chip irqc;
+	struct device *dev;
 	void __iomem *regs;
 	int irq;
 	u32 edge_emulation;
@@ -123,6 +124,7 @@ static void hlwd_gpio_irq_mask(struct irq_data *data)
 	mask &= ~BIT(data->hwirq);
 	iowrite32be(mask, hlwd->regs + HW_GPIOB_INTMASK);
 	raw_spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags);
+	gpiochip_disable_irq(&hlwd->gpioc, irqd_to_hwirq(data));
 }
 
 static void hlwd_gpio_irq_unmask(struct irq_data *data)
@@ -132,6 +134,7 @@ static void hlwd_gpio_irq_unmask(struct irq_data *data)
 	unsigned long flags;
 	u32 mask;
 
+	gpiochip_enable_irq(&hlwd->gpioc, irqd_to_hwirq(data));
 	raw_spin_lock_irqsave(&hlwd->gpioc.bgpio_lock, flags);
 	mask = ioread32be(hlwd->regs + HW_GPIOB_INTMASK);
 	mask |= BIT(data->hwirq);
@@ -202,6 +205,24 @@ static int hlwd_gpio_irq_set_type(struct irq_data *data, unsigned int flow_type)
 	return 0;
 }
 
+static void hlwd_gpio_irq_print_chip(struct irq_data *data, struct seq_file *p)
+{
+	struct hlwd_gpio *hlwd =
+		gpiochip_get_data(irq_data_get_irq_chip_data(data));
+
+	seq_printf(p, dev_name(hlwd->dev));
+}
+
+static const struct irq_chip hlwd_gpio_irq_chip = {
+	.irq_mask = hlwd_gpio_irq_mask,
+	.irq_unmask = hlwd_gpio_irq_unmask,
+	.irq_enable = hlwd_gpio_irq_enable,
+	.irq_set_type = hlwd_gpio_irq_set_type,
+	.irq_print_chip = hlwd_gpio_irq_print_chip,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 static int hlwd_gpio_probe(struct platform_device *pdev)
 {
 	struct hlwd_gpio *hlwd;
@@ -216,6 +237,8 @@ static int hlwd_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(hlwd->regs))
 		return PTR_ERR(hlwd->regs);
 
+	hlwd->dev = &pdev->dev;
+
 	/*
 	 * Claim all GPIOs using the OWNER register. This will not work on
 	 * systems where the AHBPROT memory firewall hasn't been configured to
@@ -259,14 +282,8 @@ static int hlwd_gpio_probe(struct platform_device *pdev)
 			return hlwd->irq;
 		}
 
-		hlwd->irqc.name = dev_name(&pdev->dev);
-		hlwd->irqc.irq_mask = hlwd_gpio_irq_mask;
-		hlwd->irqc.irq_unmask = hlwd_gpio_irq_unmask;
-		hlwd->irqc.irq_enable = hlwd_gpio_irq_enable;
-		hlwd->irqc.irq_set_type = hlwd_gpio_irq_set_type;
-
 		girq = &hlwd->gpioc.irq;
-		girq->chip = &hlwd->irqc;
+		gpio_irq_chip_set_chip(girq, &hlwd_gpio_irq_chip);
 		girq->parent_handler = hlwd_gpio_irqhandler;
 		girq->num_parents = 1;
 		girq->parents = devm_kcalloc(&pdev->dev, 1,
diff --git a/drivers/gpio/gpio-ich.c b/drivers/gpio/gpio-ich.c
index 3b31f5e9bf40..0be9285efebc 100644
--- a/drivers/gpio/gpio-ich.c
+++ b/drivers/gpio/gpio-ich.c
@@ -457,7 +457,7 @@ static int ichx_gpio_probe(struct platform_device *pdev)
 
 init:
 	ichx_gpiolib_setup(&ichx_priv.chip);
-	err = gpiochip_add_data(&ichx_priv.chip, NULL);
+	err = devm_gpiochip_add_data(dev, &ichx_priv.chip, NULL);
 	if (err) {
 		dev_err(dev, "Failed to register GPIOs\n");
 		return err;
@@ -469,19 +469,11 @@ init:
 	return 0;
 }
 
-static int ichx_gpio_remove(struct platform_device *pdev)
-{
-	gpiochip_remove(&ichx_priv.chip);
-
-	return 0;
-}
-
 static struct platform_driver ichx_gpio_driver = {
 	.driver		= {
 		.name	= DRV_NAME,
 	},
 	.probe		= ichx_gpio_probe,
-	.remove		= ichx_gpio_remove,
 };
 
 module_platform_driver(ichx_gpio_driver);
diff --git a/drivers/gpio/gpio-idt3243x.c b/drivers/gpio/gpio-idt3243x.c
index 1cafdf46f875..00f547d26254 100644
--- a/drivers/gpio/gpio-idt3243x.c
+++ b/drivers/gpio/gpio-idt3243x.c
@@ -92,6 +92,8 @@ static void idt_gpio_mask(struct irq_data *d)
 	writel(ctrl->mask_cache, ctrl->pic + IDT_PIC_IRQ_MASK);
 
 	raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags);
+
+	gpiochip_disable_irq(gc, irqd_to_hwirq(d));
 }
 
 static void idt_gpio_unmask(struct irq_data *d)
@@ -100,6 +102,7 @@ static void idt_gpio_unmask(struct irq_data *d)
 	struct idt_gpio_ctrl *ctrl = gpiochip_get_data(gc);
 	unsigned long flags;
 
+	gpiochip_enable_irq(gc, irqd_to_hwirq(d));
 	raw_spin_lock_irqsave(&gc->bgpio_lock, flags);
 
 	ctrl->mask_cache &= ~BIT(d->hwirq);
@@ -119,12 +122,14 @@ static int idt_gpio_irq_init_hw(struct gpio_chip *gc)
 	return 0;
 }
 
-static struct irq_chip idt_gpio_irqchip = {
+static const struct irq_chip idt_gpio_irqchip = {
 	.name = "IDTGPIO",
 	.irq_mask = idt_gpio_mask,
 	.irq_ack = idt_gpio_ack,
 	.irq_unmask = idt_gpio_unmask,
-	.irq_set_type = idt_gpio_irq_set_type
+	.irq_set_type = idt_gpio_irq_set_type,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static int idt_gpio_probe(struct platform_device *pdev)
@@ -168,7 +173,7 @@ static int idt_gpio_probe(struct platform_device *pdev)
 			return parent_irq;
 
 		girq = &ctrl->gc.irq;
-		girq->chip = &idt_gpio_irqchip;
+		gpio_irq_chip_set_chip(girq, &idt_gpio_irqchip);
 		girq->init_hw = idt_gpio_irq_init_hw;
 		girq->parent_handler = idt_gpio_dispatch;
 		girq->num_parents = 1;
diff --git a/drivers/gpio/gpio-imx-scu.c b/drivers/gpio/gpio-imx-scu.c
index 17be21b8f3b7..e190bde5397d 100644
--- a/drivers/gpio/gpio-imx-scu.c
+++ b/drivers/gpio/gpio-imx-scu.c
@@ -136,4 +136,3 @@ subsys_initcall_sync(_imx_scu_gpio_init);
 
 MODULE_AUTHOR("Shenwei Wang <shenwei.wang@nxp.com>");
 MODULE_DESCRIPTION("NXP GPIO over IMX SCU API");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-loongson-64bit.c b/drivers/gpio/gpio-loongson-64bit.c
new file mode 100644
index 000000000000..06213bbfabdd
--- /dev/null
+++ b/drivers/gpio/gpio-loongson-64bit.c
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Loongson GPIO Support
+ *
+ * Copyright (C) 2022-2023 Loongson Technology Corporation Limited
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/err.h>
+#include <linux/gpio/driver.h>
+#include <linux/platform_device.h>
+#include <linux/bitops.h>
+#include <asm/types.h>
+
+enum loongson_gpio_mode {
+	BIT_CTRL_MODE,
+	BYTE_CTRL_MODE,
+};
+
+struct loongson_gpio_chip_data {
+	const char		*label;
+	enum loongson_gpio_mode	mode;
+	unsigned int		conf_offset;
+	unsigned int		out_offset;
+	unsigned int		in_offset;
+};
+
+struct loongson_gpio_chip {
+	struct gpio_chip	chip;
+	struct fwnode_handle	*fwnode;
+	spinlock_t		lock;
+	void __iomem		*reg_base;
+	const struct loongson_gpio_chip_data *chip_data;
+};
+
+static inline struct loongson_gpio_chip *to_loongson_gpio_chip(struct gpio_chip *chip)
+{
+	return container_of(chip, struct loongson_gpio_chip, chip);
+}
+
+static inline void loongson_commit_direction(struct loongson_gpio_chip *lgpio, unsigned int pin,
+					     int input)
+{
+	u8 bval = input ? 1 : 0;
+
+	writeb(bval, lgpio->reg_base + lgpio->chip_data->conf_offset + pin);
+}
+
+static void loongson_commit_level(struct loongson_gpio_chip *lgpio, unsigned int pin, int high)
+{
+	u8 bval = high ? 1 : 0;
+
+	writeb(bval, lgpio->reg_base + lgpio->chip_data->out_offset + pin);
+}
+
+static int loongson_gpio_direction_input(struct gpio_chip *chip, unsigned int pin)
+{
+	unsigned long flags;
+	struct loongson_gpio_chip *lgpio = to_loongson_gpio_chip(chip);
+
+	spin_lock_irqsave(&lgpio->lock, flags);
+	loongson_commit_direction(lgpio, pin, 1);
+	spin_unlock_irqrestore(&lgpio->lock, flags);
+
+	return 0;
+}
+
+static int loongson_gpio_direction_output(struct gpio_chip *chip, unsigned int pin, int value)
+{
+	unsigned long flags;
+	struct loongson_gpio_chip *lgpio = to_loongson_gpio_chip(chip);
+
+	spin_lock_irqsave(&lgpio->lock, flags);
+	loongson_commit_level(lgpio, pin, value);
+	loongson_commit_direction(lgpio, pin, 0);
+	spin_unlock_irqrestore(&lgpio->lock, flags);
+
+	return 0;
+}
+
+static int loongson_gpio_get(struct gpio_chip *chip, unsigned int pin)
+{
+	u8  bval;
+	int val;
+	struct loongson_gpio_chip *lgpio = to_loongson_gpio_chip(chip);
+
+	bval = readb(lgpio->reg_base + lgpio->chip_data->in_offset + pin);
+	val = bval & 1;
+
+	return val;
+}
+
+static int loongson_gpio_get_direction(struct gpio_chip *chip, unsigned int pin)
+{
+	u8  bval;
+	struct loongson_gpio_chip *lgpio = to_loongson_gpio_chip(chip);
+
+	bval = readb(lgpio->reg_base + lgpio->chip_data->conf_offset + pin);
+	if (bval & 1)
+		return GPIO_LINE_DIRECTION_IN;
+
+	return GPIO_LINE_DIRECTION_OUT;
+}
+
+static void loongson_gpio_set(struct gpio_chip *chip, unsigned int pin, int value)
+{
+	unsigned long flags;
+	struct loongson_gpio_chip *lgpio = to_loongson_gpio_chip(chip);
+
+	spin_lock_irqsave(&lgpio->lock, flags);
+	loongson_commit_level(lgpio, pin, value);
+	spin_unlock_irqrestore(&lgpio->lock, flags);
+}
+
+static int loongson_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
+{
+	struct platform_device *pdev = to_platform_device(chip->parent);
+
+	return platform_get_irq(pdev, offset);
+}
+
+static int loongson_gpio_init(struct device *dev, struct loongson_gpio_chip *lgpio,
+			      struct device_node *np, void __iomem *reg_base)
+{
+	int ret;
+	u32 ngpios;
+
+	lgpio->reg_base = reg_base;
+
+	if (lgpio->chip_data->mode == BIT_CTRL_MODE) {
+		ret = bgpio_init(&lgpio->chip, dev, 8,
+				lgpio->reg_base + lgpio->chip_data->in_offset,
+				lgpio->reg_base + lgpio->chip_data->out_offset,
+				NULL, NULL,
+				lgpio->reg_base + lgpio->chip_data->conf_offset,
+				0);
+		if (ret) {
+			dev_err(dev, "unable to init generic GPIO\n");
+			return ret;
+		}
+	} else {
+		lgpio->chip.direction_input = loongson_gpio_direction_input;
+		lgpio->chip.get = loongson_gpio_get;
+		lgpio->chip.get_direction = loongson_gpio_get_direction;
+		lgpio->chip.direction_output = loongson_gpio_direction_output;
+		lgpio->chip.set = loongson_gpio_set;
+		lgpio->chip.parent = dev;
+		spin_lock_init(&lgpio->lock);
+	}
+
+	device_property_read_u32(dev, "ngpios", &ngpios);
+
+	lgpio->chip.can_sleep = 0;
+	lgpio->chip.ngpio = ngpios;
+	lgpio->chip.label = lgpio->chip_data->label;
+	lgpio->chip.to_irq = loongson_gpio_to_irq;
+
+	return devm_gpiochip_add_data(dev, &lgpio->chip, lgpio);
+}
+
+static int loongson_gpio_probe(struct platform_device *pdev)
+{
+	void __iomem *reg_base;
+	struct loongson_gpio_chip *lgpio;
+	struct device_node *np = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+
+	lgpio = devm_kzalloc(dev, sizeof(*lgpio), GFP_KERNEL);
+	if (!lgpio)
+		return -ENOMEM;
+
+	lgpio->chip_data = device_get_match_data(dev);
+
+	reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(reg_base))
+		return PTR_ERR(reg_base);
+
+	return loongson_gpio_init(dev, lgpio, np, reg_base);
+}
+
+static const struct loongson_gpio_chip_data loongson_gpio_ls2k_data = {
+	.label = "ls2k_gpio",
+	.mode = BIT_CTRL_MODE,
+	.conf_offset = 0x0,
+	.in_offset = 0x20,
+	.out_offset = 0x10,
+};
+
+static const struct loongson_gpio_chip_data loongson_gpio_ls7a_data = {
+	.label = "ls7a_gpio",
+	.mode = BYTE_CTRL_MODE,
+	.conf_offset = 0x800,
+	.in_offset = 0xa00,
+	.out_offset = 0x900,
+};
+
+static const struct of_device_id loongson_gpio_of_match[] = {
+	{
+		.compatible = "loongson,ls2k-gpio",
+		.data = &loongson_gpio_ls2k_data,
+	},
+	{
+		.compatible = "loongson,ls7a-gpio",
+		.data = &loongson_gpio_ls7a_data,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, loongson_gpio_of_match);
+
+static const struct acpi_device_id loongson_gpio_acpi_match[] = {
+	{
+		.id = "LOON0002",
+		.driver_data = (kernel_ulong_t)&loongson_gpio_ls7a_data,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(acpi, loongson_gpio_acpi_match);
+
+static struct platform_driver loongson_gpio_driver = {
+	.driver = {
+		.name = "loongson-gpio",
+		.of_match_table = loongson_gpio_of_match,
+		.acpi_match_table = loongson_gpio_acpi_match,
+	},
+	.probe = loongson_gpio_probe,
+};
+
+static int __init loongson_gpio_setup(void)
+{
+	return platform_driver_register(&loongson_gpio_driver);
+}
+postcore_initcall(loongson_gpio_setup);
+
+MODULE_DESCRIPTION("Loongson gpio driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-loongson1.c b/drivers/gpio/gpio-loongson1.c
index 5d90b3bc5a25..6ca3b969db4d 100644
--- a/drivers/gpio/gpio-loongson1.c
+++ b/drivers/gpio/gpio-loongson1.c
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * GPIO Driver for Loongson 1 SoC
  *
- * Copyright (C) 2015-2016 Zhang, Keguang <keguang.zhang@gmail.com>
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2. This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
+ * Copyright (C) 2015-2023 Keguang Zhang <keguang.zhang@gmail.com>
  */
 
 #include <linux/module.h>
@@ -19,15 +16,19 @@
 #define GPIO_DATA		0x20
 #define GPIO_OUTPUT		0x30
 
-static void __iomem *gpio_reg_base;
+struct ls1x_gpio_chip {
+	struct gpio_chip gc;
+	void __iomem *reg_base;
+};
 
 static int ls1x_gpio_request(struct gpio_chip *gc, unsigned int offset)
 {
+	struct ls1x_gpio_chip *ls1x_gc = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&gc->bgpio_lock, flags);
-	__raw_writel(__raw_readl(gpio_reg_base + GPIO_CFG) | BIT(offset),
-		     gpio_reg_base + GPIO_CFG);
+	__raw_writel(__raw_readl(ls1x_gc->reg_base + GPIO_CFG) | BIT(offset),
+		     ls1x_gc->reg_base + GPIO_CFG);
 	raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
 	return 0;
@@ -35,61 +36,75 @@ static int ls1x_gpio_request(struct gpio_chip *gc, unsigned int offset)
 
 static void ls1x_gpio_free(struct gpio_chip *gc, unsigned int offset)
 {
+	struct ls1x_gpio_chip *ls1x_gc = gpiochip_get_data(gc);
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&gc->bgpio_lock, flags);
-	__raw_writel(__raw_readl(gpio_reg_base + GPIO_CFG) & ~BIT(offset),
-		     gpio_reg_base + GPIO_CFG);
+	__raw_writel(__raw_readl(ls1x_gc->reg_base + GPIO_CFG) & ~BIT(offset),
+		     ls1x_gc->reg_base + GPIO_CFG);
 	raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 }
 
 static int ls1x_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct gpio_chip *gc;
+	struct ls1x_gpio_chip *ls1x_gc;
 	int ret;
 
-	gc = devm_kzalloc(dev, sizeof(*gc), GFP_KERNEL);
-	if (!gc)
+	ls1x_gc = devm_kzalloc(dev, sizeof(*ls1x_gc), GFP_KERNEL);
+	if (!ls1x_gc)
 		return -ENOMEM;
 
-	gpio_reg_base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(gpio_reg_base))
-		return PTR_ERR(gpio_reg_base);
+	ls1x_gc->reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(ls1x_gc->reg_base))
+		return PTR_ERR(ls1x_gc->reg_base);
 
-	ret = bgpio_init(gc, dev, 4, gpio_reg_base + GPIO_DATA,
-			 gpio_reg_base + GPIO_OUTPUT, NULL,
-			 NULL, gpio_reg_base + GPIO_DIR, 0);
+	ret = bgpio_init(&ls1x_gc->gc, dev, 4, ls1x_gc->reg_base + GPIO_DATA,
+			 ls1x_gc->reg_base + GPIO_OUTPUT, NULL,
+			 NULL, ls1x_gc->reg_base + GPIO_DIR, 0);
 	if (ret)
 		goto err;
 
-	gc->owner = THIS_MODULE;
-	gc->request = ls1x_gpio_request;
-	gc->free = ls1x_gpio_free;
-	gc->base = pdev->id * 32;
+	ls1x_gc->gc.owner = THIS_MODULE;
+	ls1x_gc->gc.request = ls1x_gpio_request;
+	ls1x_gc->gc.free = ls1x_gpio_free;
+	/*
+	 * Clear ngpio to let gpiolib get the correct number
+	 * by reading ngpios property
+	 */
+	ls1x_gc->gc.ngpio = 0;
 
-	ret = devm_gpiochip_add_data(dev, gc, NULL);
+	ret = devm_gpiochip_add_data(dev, &ls1x_gc->gc, ls1x_gc);
 	if (ret)
 		goto err;
 
-	platform_set_drvdata(pdev, gc);
-	dev_info(dev, "Loongson1 GPIO driver registered\n");
+	platform_set_drvdata(pdev, ls1x_gc);
+
+	dev_info(dev, "GPIO controller registered with %d pins\n",
+		 ls1x_gc->gc.ngpio);
 
 	return 0;
 err:
-	dev_err(dev, "failed to register GPIO device\n");
+	dev_err(dev, "failed to register GPIO controller\n");
 	return ret;
 }
 
+static const struct of_device_id ls1x_gpio_dt_ids[] = {
+	{ .compatible = "loongson,ls1x-gpio" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, ls1x_gpio_dt_ids);
+
 static struct platform_driver ls1x_gpio_driver = {
 	.probe	= ls1x_gpio_probe,
 	.driver	= {
 		.name	= "ls1x-gpio",
+		.of_match_table = ls1x_gpio_dt_ids,
 	},
 };
 
 module_platform_driver(ls1x_gpio_driver);
 
-MODULE_AUTHOR("Kelvin Cheung <keguang.zhang@gmail.com>");
+MODULE_AUTHOR("Keguang Zhang <keguang.zhang@gmail.com>");
 MODULE_DESCRIPTION("Loongson1 GPIO driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-max732x.c b/drivers/gpio/gpio-max732x.c
index 68e982cdee73..7f2fde191755 100644
--- a/drivers/gpio/gpio-max732x.c
+++ b/drivers/gpio/gpio-max732x.c
@@ -351,6 +351,7 @@ static void max732x_irq_mask(struct irq_data *d)
 	struct max732x_chip *chip = gpiochip_get_data(gc);
 
 	chip->irq_mask_cur &= ~(1 << d->hwirq);
+	gpiochip_disable_irq(gc, irqd_to_hwirq(d));
 }
 
 static void max732x_irq_unmask(struct irq_data *d)
@@ -358,6 +359,7 @@ static void max732x_irq_unmask(struct irq_data *d)
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct max732x_chip *chip = gpiochip_get_data(gc);
 
+	gpiochip_enable_irq(gc, irqd_to_hwirq(d));
 	chip->irq_mask_cur |= 1 << d->hwirq;
 }
 
@@ -429,7 +431,7 @@ static int max732x_irq_set_wake(struct irq_data *data, unsigned int on)
 	return 0;
 }
 
-static struct irq_chip max732x_irq_chip = {
+static const struct irq_chip max732x_irq_chip = {
 	.name			= "max732x",
 	.irq_mask		= max732x_irq_mask,
 	.irq_unmask		= max732x_irq_unmask,
@@ -437,6 +439,8 @@ static struct irq_chip max732x_irq_chip = {
 	.irq_bus_sync_unlock	= max732x_irq_bus_sync_unlock,
 	.irq_set_type		= max732x_irq_set_type,
 	.irq_set_wake		= max732x_irq_set_wake,
+	.flags			= IRQCHIP_IMMUTABLE,
+	 GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static uint8_t max732x_irq_pending(struct max732x_chip *chip)
@@ -517,7 +521,7 @@ static int max732x_irq_setup(struct max732x_chip *chip,
 		}
 
 		girq = &chip->gpio_chip.irq;
-		girq->chip = &max732x_irq_chip;
+		gpio_irq_chip_set_chip(girq, &max732x_irq_chip);
 		/* This will let us handle the parent IRQ in the driver */
 		girq->parent_handler = NULL;
 		girq->num_parents = 0;
diff --git a/drivers/gpio/gpio-merrifield.c b/drivers/gpio/gpio-merrifield.c
index 92ea8411050d..421d7e3a6c66 100644
--- a/drivers/gpio/gpio-merrifield.c
+++ b/drivers/gpio/gpio-merrifield.c
@@ -2,60 +2,25 @@
 /*
  * Intel Merrifield SoC GPIO driver
  *
- * Copyright (c) 2016 Intel Corporation.
+ * Copyright (c) 2016, 2023 Intel Corporation.
  * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
  */
 
 #include <linux/acpi.h>
 #include <linux/bitops.h>
-#include <linux/gpio/driver.h>
-#include <linux/interrupt.h>
+#include <linux/device.h>
+#include <linux/err.h>
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/pci.h>
-#include <linux/pinctrl/consumer.h>
-#include <linux/string_helpers.h>
+#include <linux/types.h>
 
-#define GCCR		0x000	/* controller configuration */
-#define GPLR		0x004	/* pin level r/o */
-#define GPDR		0x01c	/* pin direction */
-#define GPSR		0x034	/* pin set w/o */
-#define GPCR		0x04c	/* pin clear w/o */
-#define GRER		0x064	/* rising edge detect */
-#define GFER		0x07c	/* falling edge detect */
-#define GFBR		0x094	/* glitch filter bypass */
-#define GIMR		0x0ac	/* interrupt mask */
-#define GISR		0x0c4	/* interrupt source */
-#define GITR		0x300	/* input type */
-#define GLPR		0x318	/* level input polarity */
-#define GWMR		0x400	/* wake mask */
-#define GWSR		0x418	/* wake source */
-#define GSIR		0xc00	/* secure input */
+#include "gpio-tangier.h"
 
 /* Intel Merrifield has 192 GPIO pins */
 #define MRFLD_NGPIO	192
 
-struct mrfld_gpio_pinrange {
-	unsigned int gpio_base;
-	unsigned int pin_base;
-	unsigned int npins;
-};
-
-#define GPIO_PINRANGE(gstart, gend, pstart)		\
-	{						\
-		.gpio_base = (gstart),			\
-		.pin_base = (pstart),			\
-		.npins = (gend) - (gstart) + 1,		\
-	}
-
-struct mrfld_gpio {
-	struct gpio_chip	chip;
-	void __iomem		*reg_base;
-	raw_spinlock_t		lock;
-	struct device		*dev;
-};
-
-static const struct mrfld_gpio_pinrange mrfld_gpio_ranges[] = {
+static const struct tng_gpio_pinrange mrfld_gpio_ranges[] = {
 	GPIO_PINRANGE(0, 11, 146),
 	GPIO_PINRANGE(12, 13, 144),
 	GPIO_PINRANGE(14, 15, 35),
@@ -84,323 +49,15 @@ static const struct mrfld_gpio_pinrange mrfld_gpio_ranges[] = {
 	GPIO_PINRANGE(190, 191, 178),
 };
 
-static void __iomem *gpio_reg(struct gpio_chip *chip, unsigned int offset,
-			      unsigned int reg_type_offset)
-{
-	struct mrfld_gpio *priv = gpiochip_get_data(chip);
-	u8 reg = offset / 32;
-
-	return priv->reg_base + reg_type_offset + reg * 4;
-}
-
-static int mrfld_gpio_get(struct gpio_chip *chip, unsigned int offset)
-{
-	void __iomem *gplr = gpio_reg(chip, offset, GPLR);
-
-	return !!(readl(gplr) & BIT(offset % 32));
-}
-
-static void mrfld_gpio_set(struct gpio_chip *chip, unsigned int offset,
-			   int value)
-{
-	struct mrfld_gpio *priv = gpiochip_get_data(chip);
-	void __iomem *gpsr, *gpcr;
-	unsigned long flags;
-
-	raw_spin_lock_irqsave(&priv->lock, flags);
-
-	if (value) {
-		gpsr = gpio_reg(chip, offset, GPSR);
-		writel(BIT(offset % 32), gpsr);
-	} else {
-		gpcr = gpio_reg(chip, offset, GPCR);
-		writel(BIT(offset % 32), gpcr);
-	}
-
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-}
-
-static int mrfld_gpio_direction_input(struct gpio_chip *chip,
-				      unsigned int offset)
-{
-	struct mrfld_gpio *priv = gpiochip_get_data(chip);
-	void __iomem *gpdr = gpio_reg(chip, offset, GPDR);
-	unsigned long flags;
-	u32 value;
-
-	raw_spin_lock_irqsave(&priv->lock, flags);
-
-	value = readl(gpdr);
-	value &= ~BIT(offset % 32);
-	writel(value, gpdr);
-
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
-	return 0;
-}
-
-static int mrfld_gpio_direction_output(struct gpio_chip *chip,
-				       unsigned int offset, int value)
-{
-	struct mrfld_gpio *priv = gpiochip_get_data(chip);
-	void __iomem *gpdr = gpio_reg(chip, offset, GPDR);
-	unsigned long flags;
-
-	mrfld_gpio_set(chip, offset, value);
-
-	raw_spin_lock_irqsave(&priv->lock, flags);
-
-	value = readl(gpdr);
-	value |= BIT(offset % 32);
-	writel(value, gpdr);
-
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
-	return 0;
-}
-
-static int mrfld_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
-{
-	void __iomem *gpdr = gpio_reg(chip, offset, GPDR);
-
-	if (readl(gpdr) & BIT(offset % 32))
-		return GPIO_LINE_DIRECTION_OUT;
-
-	return GPIO_LINE_DIRECTION_IN;
-}
-
-static int mrfld_gpio_set_debounce(struct gpio_chip *chip, unsigned int offset,
-				   unsigned int debounce)
-{
-	struct mrfld_gpio *priv = gpiochip_get_data(chip);
-	void __iomem *gfbr = gpio_reg(chip, offset, GFBR);
-	unsigned long flags;
-	u32 value;
-
-	raw_spin_lock_irqsave(&priv->lock, flags);
-
-	if (debounce)
-		value = readl(gfbr) & ~BIT(offset % 32);
-	else
-		value = readl(gfbr) | BIT(offset % 32);
-	writel(value, gfbr);
-
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
-	return 0;
-}
-
-static int mrfld_gpio_set_config(struct gpio_chip *chip, unsigned int offset,
-				 unsigned long config)
-{
-	u32 debounce;
-
-	if ((pinconf_to_config_param(config) == PIN_CONFIG_BIAS_DISABLE) ||
-	    (pinconf_to_config_param(config) == PIN_CONFIG_BIAS_PULL_UP) ||
-	    (pinconf_to_config_param(config) == PIN_CONFIG_BIAS_PULL_DOWN))
-		return gpiochip_generic_config(chip, offset, config);
-
-	if (pinconf_to_config_param(config) != PIN_CONFIG_INPUT_DEBOUNCE)
-		return -ENOTSUPP;
-
-	debounce = pinconf_to_config_argument(config);
-	return mrfld_gpio_set_debounce(chip, offset, debounce);
-}
-
-static void mrfld_irq_ack(struct irq_data *d)
-{
-	struct mrfld_gpio *priv = irq_data_get_irq_chip_data(d);
-	u32 gpio = irqd_to_hwirq(d);
-	void __iomem *gisr = gpio_reg(&priv->chip, gpio, GISR);
-	unsigned long flags;
-
-	raw_spin_lock_irqsave(&priv->lock, flags);
-
-	writel(BIT(gpio % 32), gisr);
-
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-}
-
-static void mrfld_irq_unmask_mask(struct mrfld_gpio *priv, u32 gpio, bool unmask)
-{
-	void __iomem *gimr = gpio_reg(&priv->chip, gpio, GIMR);
-	unsigned long flags;
-	u32 value;
-
-	raw_spin_lock_irqsave(&priv->lock, flags);
-
-	if (unmask)
-		value = readl(gimr) | BIT(gpio % 32);
-	else
-		value = readl(gimr) & ~BIT(gpio % 32);
-	writel(value, gimr);
-
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-}
-
-static void mrfld_irq_mask(struct irq_data *d)
-{
-	struct mrfld_gpio *priv = irq_data_get_irq_chip_data(d);
-	u32 gpio = irqd_to_hwirq(d);
-
-	mrfld_irq_unmask_mask(priv, gpio, false);
-	gpiochip_disable_irq(&priv->chip, gpio);
-}
-
-static void mrfld_irq_unmask(struct irq_data *d)
-{
-	struct mrfld_gpio *priv = irq_data_get_irq_chip_data(d);
-	u32 gpio = irqd_to_hwirq(d);
-
-	gpiochip_enable_irq(&priv->chip, gpio);
-	mrfld_irq_unmask_mask(priv, gpio, true);
-}
-
-static int mrfld_irq_set_type(struct irq_data *d, unsigned int type)
-{
-	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct mrfld_gpio *priv = gpiochip_get_data(gc);
-	u32 gpio = irqd_to_hwirq(d);
-	void __iomem *grer = gpio_reg(&priv->chip, gpio, GRER);
-	void __iomem *gfer = gpio_reg(&priv->chip, gpio, GFER);
-	void __iomem *gitr = gpio_reg(&priv->chip, gpio, GITR);
-	void __iomem *glpr = gpio_reg(&priv->chip, gpio, GLPR);
-	unsigned long flags;
-	u32 value;
-
-	raw_spin_lock_irqsave(&priv->lock, flags);
-
-	if (type & IRQ_TYPE_EDGE_RISING)
-		value = readl(grer) | BIT(gpio % 32);
-	else
-		value = readl(grer) & ~BIT(gpio % 32);
-	writel(value, grer);
-
-	if (type & IRQ_TYPE_EDGE_FALLING)
-		value = readl(gfer) | BIT(gpio % 32);
-	else
-		value = readl(gfer) & ~BIT(gpio % 32);
-	writel(value, gfer);
-
-	/*
-	 * To prevent glitches from triggering an unintended level interrupt,
-	 * configure GLPR register first and then configure GITR.
-	 */
-	if (type & IRQ_TYPE_LEVEL_LOW)
-		value = readl(glpr) | BIT(gpio % 32);
-	else
-		value = readl(glpr) & ~BIT(gpio % 32);
-	writel(value, glpr);
-
-	if (type & IRQ_TYPE_LEVEL_MASK) {
-		value = readl(gitr) | BIT(gpio % 32);
-		writel(value, gitr);
-
-		irq_set_handler_locked(d, handle_level_irq);
-	} else if (type & IRQ_TYPE_EDGE_BOTH) {
-		value = readl(gitr) & ~BIT(gpio % 32);
-		writel(value, gitr);
-
-		irq_set_handler_locked(d, handle_edge_irq);
-	}
-
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
-	return 0;
-}
-
-static int mrfld_irq_set_wake(struct irq_data *d, unsigned int on)
-{
-	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct mrfld_gpio *priv = gpiochip_get_data(gc);
-	u32 gpio = irqd_to_hwirq(d);
-	void __iomem *gwmr = gpio_reg(&priv->chip, gpio, GWMR);
-	void __iomem *gwsr = gpio_reg(&priv->chip, gpio, GWSR);
-	unsigned long flags;
-	u32 value;
-
-	raw_spin_lock_irqsave(&priv->lock, flags);
-
-	/* Clear the existing wake status */
-	writel(BIT(gpio % 32), gwsr);
-
-	if (on)
-		value = readl(gwmr) | BIT(gpio % 32);
-	else
-		value = readl(gwmr) & ~BIT(gpio % 32);
-	writel(value, gwmr);
-
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
-	dev_dbg(priv->dev, "%s wake for gpio %u\n", str_enable_disable(on), gpio);
-	return 0;
-}
-
-static const struct irq_chip mrfld_irqchip = {
-	.name		= "gpio-merrifield",
-	.irq_ack	= mrfld_irq_ack,
-	.irq_mask	= mrfld_irq_mask,
-	.irq_unmask	= mrfld_irq_unmask,
-	.irq_set_type	= mrfld_irq_set_type,
-	.irq_set_wake	= mrfld_irq_set_wake,
-	.flags		= IRQCHIP_IMMUTABLE,
-	GPIOCHIP_IRQ_RESOURCE_HELPERS,
-};
-
-static void mrfld_irq_handler(struct irq_desc *desc)
-{
-	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct mrfld_gpio *priv = gpiochip_get_data(gc);
-	struct irq_chip *irqchip = irq_desc_get_chip(desc);
-	unsigned long base, gpio;
-
-	chained_irq_enter(irqchip, desc);
-
-	/* Check GPIO controller to check which pin triggered the interrupt */
-	for (base = 0; base < priv->chip.ngpio; base += 32) {
-		void __iomem *gisr = gpio_reg(&priv->chip, base, GISR);
-		void __iomem *gimr = gpio_reg(&priv->chip, base, GIMR);
-		unsigned long pending, enabled;
-
-		pending = readl(gisr);
-		enabled = readl(gimr);
-
-		/* Only interrupts that are enabled */
-		pending &= enabled;
-
-		for_each_set_bit(gpio, &pending, 32)
-			generic_handle_domain_irq(gc->irq.domain, base + gpio);
-	}
-
-	chained_irq_exit(irqchip, desc);
-}
-
-static int mrfld_irq_init_hw(struct gpio_chip *chip)
-{
-	struct mrfld_gpio *priv = gpiochip_get_data(chip);
-	void __iomem *reg;
-	unsigned int base;
-
-	for (base = 0; base < priv->chip.ngpio; base += 32) {
-		/* Clear the rising-edge detect register */
-		reg = gpio_reg(&priv->chip, base, GRER);
-		writel(0, reg);
-		/* Clear the falling-edge detect register */
-		reg = gpio_reg(&priv->chip, base, GFER);
-		writel(0, reg);
-	}
-
-	return 0;
-}
-
-static const char *mrfld_gpio_get_pinctrl_dev_name(struct mrfld_gpio *priv)
+static const char *mrfld_gpio_get_pinctrl_dev_name(struct tng_gpio *priv)
 {
+	struct device *dev = priv->dev;
 	struct acpi_device *adev;
 	const char *name;
 
 	adev = acpi_dev_get_first_match_dev("INTC1002", NULL, -1);
 	if (adev) {
-		name = devm_kstrdup(priv->dev, acpi_dev_name(adev), GFP_KERNEL);
+		name = devm_kstrdup(dev, acpi_dev_name(adev), GFP_KERNEL);
 		acpi_dev_put(adev);
 	} else {
 		name = "pinctrl-merrifield";
@@ -409,37 +66,10 @@ static const char *mrfld_gpio_get_pinctrl_dev_name(struct mrfld_gpio *priv)
 	return name;
 }
 
-static int mrfld_gpio_add_pin_ranges(struct gpio_chip *chip)
-{
-	struct mrfld_gpio *priv = gpiochip_get_data(chip);
-	const struct mrfld_gpio_pinrange *range;
-	const char *pinctrl_dev_name;
-	unsigned int i;
-	int retval;
-
-	pinctrl_dev_name = mrfld_gpio_get_pinctrl_dev_name(priv);
-	if (!pinctrl_dev_name)
-		return -ENOMEM;
-
-	for (i = 0; i < ARRAY_SIZE(mrfld_gpio_ranges); i++) {
-		range = &mrfld_gpio_ranges[i];
-		retval = gpiochip_add_pin_range(&priv->chip, pinctrl_dev_name,
-						range->gpio_base,
-						range->pin_base,
-						range->npins);
-		if (retval) {
-			dev_err(priv->dev, "failed to add GPIO pin range\n");
-			return retval;
-		}
-	}
-
-	return 0;
-}
-
 static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
-	struct gpio_irq_chip *girq;
-	struct mrfld_gpio *priv;
+	struct device *dev = &pdev->dev;
+	struct tng_gpio *priv;
 	u32 gpio_base, irq_base;
 	void __iomem *base;
 	int retval;
@@ -449,10 +79,8 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
 		return retval;
 
 	retval = pcim_iomap_regions(pdev, BIT(1) | BIT(0), pci_name(pdev));
-	if (retval) {
-		dev_err(&pdev->dev, "I/O memory mapping error\n");
-		return retval;
-	}
+	if (retval)
+		return dev_err_probe(dev, retval, "I/O memory mapping error\n");
 
 	base = pcim_iomap_table(pdev)[1];
 
@@ -462,53 +90,36 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
 	/* Release the IO mapping, since we already get the info from BAR1 */
 	pcim_iounmap_regions(pdev, BIT(1));
 
-	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	priv->dev = &pdev->dev;
+	priv->dev = dev;
 	priv->reg_base = pcim_iomap_table(pdev)[0];
 
-	priv->chip.label = dev_name(&pdev->dev);
-	priv->chip.parent = &pdev->dev;
-	priv->chip.request = gpiochip_generic_request;
-	priv->chip.free = gpiochip_generic_free;
-	priv->chip.direction_input = mrfld_gpio_direction_input;
-	priv->chip.direction_output = mrfld_gpio_direction_output;
-	priv->chip.get = mrfld_gpio_get;
-	priv->chip.set = mrfld_gpio_set;
-	priv->chip.get_direction = mrfld_gpio_get_direction;
-	priv->chip.set_config = mrfld_gpio_set_config;
-	priv->chip.base = gpio_base;
-	priv->chip.ngpio = MRFLD_NGPIO;
-	priv->chip.can_sleep = false;
-	priv->chip.add_pin_ranges = mrfld_gpio_add_pin_ranges;
+	priv->pin_info.pin_ranges = mrfld_gpio_ranges;
+	priv->pin_info.nranges = ARRAY_SIZE(mrfld_gpio_ranges);
+	priv->pin_info.name = mrfld_gpio_get_pinctrl_dev_name(priv);
+	if (!priv->pin_info.name)
+		return -ENOMEM;
 
-	raw_spin_lock_init(&priv->lock);
+	priv->info.base = gpio_base;
+	priv->info.ngpio = MRFLD_NGPIO;
+	priv->info.first = irq_base;
 
 	retval = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
 	if (retval < 0)
 		return retval;
 
-	girq = &priv->chip.irq;
-	gpio_irq_chip_set_chip(girq, &mrfld_irqchip);
-	girq->init_hw = mrfld_irq_init_hw;
-	girq->parent_handler = mrfld_irq_handler;
-	girq->num_parents = 1;
-	girq->parents = devm_kcalloc(&pdev->dev, girq->num_parents,
-				     sizeof(*girq->parents), GFP_KERNEL);
-	if (!girq->parents)
-		return -ENOMEM;
-	girq->parents[0] = pci_irq_vector(pdev, 0);
-	girq->first = irq_base;
-	girq->default_type = IRQ_TYPE_NONE;
-	girq->handler = handle_bad_irq;
+	priv->irq = pci_irq_vector(pdev, 0);
 
-	retval = devm_gpiochip_add_data(&pdev->dev, &priv->chip, priv);
-	if (retval) {
-		dev_err(&pdev->dev, "gpiochip_add error %d\n", retval);
-		return retval;
-	}
+	priv->wake_regs.gwmr = GWMR_MRFLD;
+	priv->wake_regs.gwsr = GWSR_MRFLD;
+	priv->wake_regs.gsir = GSIR_MRFLD;
+
+	retval = devm_tng_gpio_probe(dev, priv);
+	if (retval)
+		return dev_err_probe(dev, retval, "tng_gpio_probe error\n");
 
 	pci_set_drvdata(pdev, priv);
 	return 0;
@@ -525,9 +136,9 @@ static struct pci_driver mrfld_gpio_driver = {
 	.id_table	= mrfld_gpio_ids,
 	.probe		= mrfld_gpio_probe,
 };
-
 module_pci_driver(mrfld_gpio_driver);
 
 MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
 MODULE_DESCRIPTION("Intel Merrifield SoC GPIO driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS(GPIO_TANGIER);
diff --git a/drivers/gpio/gpio-mlxbf2.c b/drivers/gpio/gpio-mlxbf2.c
index 77a41151c921..6abe01bc39c3 100644
--- a/drivers/gpio/gpio-mlxbf2.c
+++ b/drivers/gpio/gpio-mlxbf2.c
@@ -17,6 +17,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm.h>
 #include <linux/resource.h>
+#include <linux/seq_file.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
 
@@ -65,10 +66,10 @@ struct mlxbf2_gpio_context_save_regs {
 /* BlueField-2 gpio block context structure. */
 struct mlxbf2_gpio_context {
 	struct gpio_chip gc;
-	struct irq_chip irq_chip;
 
 	/* YU GPIO blocks address */
 	void __iomem *gpio_io;
+	struct device *dev;
 
 	struct mlxbf2_gpio_context_save_regs *csave_regs;
 };
@@ -237,6 +238,7 @@ static void mlxbf2_gpio_irq_enable(struct irq_data *irqd)
 	unsigned long flags;
 	u32 val;
 
+	gpiochip_enable_irq(gc, irqd_to_hwirq(irqd));
 	raw_spin_lock_irqsave(&gs->gc.bgpio_lock, flags);
 	val = readl(gs->gpio_io + YU_GPIO_CAUSE_OR_CLRCAUSE);
 	val |= BIT(offset);
@@ -261,6 +263,7 @@ static void mlxbf2_gpio_irq_disable(struct irq_data *irqd)
 	val &= ~BIT(offset);
 	writel(val, gs->gpio_io + YU_GPIO_CAUSE_OR_EVTEN0);
 	raw_spin_unlock_irqrestore(&gs->gc.bgpio_lock, flags);
+	gpiochip_disable_irq(gc, irqd_to_hwirq(irqd));
 }
 
 static irqreturn_t mlxbf2_gpio_irq_handler(int irq, void *ptr)
@@ -322,6 +325,24 @@ mlxbf2_gpio_irq_set_type(struct irq_data *irqd, unsigned int type)
 	return 0;
 }
 
+static void mlxbf2_gpio_irq_print_chip(struct irq_data *irqd,
+				       struct seq_file *p)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(irqd);
+	struct mlxbf2_gpio_context *gs = gpiochip_get_data(gc);
+
+	seq_printf(p, dev_name(gs->dev));
+}
+
+static const struct irq_chip mlxbf2_gpio_irq_chip = {
+	.irq_set_type = mlxbf2_gpio_irq_set_type,
+	.irq_enable = mlxbf2_gpio_irq_enable,
+	.irq_disable = mlxbf2_gpio_irq_disable,
+	.irq_print_chip = mlxbf2_gpio_irq_print_chip,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 /* BlueField-2 GPIO driver initialization routine. */
 static int
 mlxbf2_gpio_probe(struct platform_device *pdev)
@@ -340,6 +361,8 @@ mlxbf2_gpio_probe(struct platform_device *pdev)
 	if (!gs)
 		return -ENOMEM;
 
+	gs->dev = dev;
+
 	/* YU GPIO block address */
 	gs->gpio_io = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(gs->gpio_io))
@@ -376,13 +399,8 @@ mlxbf2_gpio_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq >= 0) {
-		gs->irq_chip.name = name;
-		gs->irq_chip.irq_set_type = mlxbf2_gpio_irq_set_type;
-		gs->irq_chip.irq_enable = mlxbf2_gpio_irq_enable;
-		gs->irq_chip.irq_disable = mlxbf2_gpio_irq_disable;
-
 		girq = &gs->gc.irq;
-		girq->chip = &gs->irq_chip;
+		gpio_irq_chip_set_chip(girq, &mlxbf2_gpio_irq_chip);
 		girq->handler = handle_simple_irq;
 		girq->default_type = IRQ_TYPE_NONE;
 		/* This will let us handle the parent IRQ in the driver */
diff --git a/drivers/gpio/gpio-mm-lantiq.c b/drivers/gpio/gpio-mm-lantiq.c
index 538e31fe8903..f3c158259636 100644
--- a/drivers/gpio/gpio-mm-lantiq.c
+++ b/drivers/gpio/gpio-mm-lantiq.c
@@ -10,8 +10,8 @@
 #include <linux/platform_device.h>
 #include <linux/mutex.h>
 #include <linux/gpio/driver.h>
+#include <linux/gpio/legacy-of-mm-gpiochip.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/io.h>
 #include <linux/slab.h>
 
diff --git a/drivers/gpio/gpio-mpc5200.c b/drivers/gpio/gpio-mpc5200.c
index 000494e0c533..3b0bfff8c778 100644
--- a/drivers/gpio/gpio-mpc5200.c
+++ b/drivers/gpio/gpio-mpc5200.c
@@ -8,7 +8,7 @@
 #include <linux/of.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/legacy-of-mm-gpiochip.h>
 #include <linux/io.h>
 #include <linux/of_platform.h>
 #include <linux/module.h>
diff --git a/drivers/gpio/gpio-msc313.c b/drivers/gpio/gpio-msc313.c
index b0773e5652fa..036ad2324892 100644
--- a/drivers/gpio/gpio-msc313.c
+++ b/drivers/gpio/gpio-msc313.c
@@ -532,17 +532,35 @@ static int msc313_gpio_direction_output(struct gpio_chip *chip, unsigned int off
 	return 0;
 }
 
+static void msc313_gpio_irq_mask(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+
+	irq_chip_mask_parent(d);
+	gpiochip_disable_irq(gc, d->hwirq);
+}
+
+static void msc313_gpio_irq_unmask(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+
+	gpiochip_enable_irq(gc, d->hwirq);
+	irq_chip_unmask_parent(d);
+}
+
 /*
  * The interrupt handling happens in the parent interrupt controller,
  * we don't do anything here.
  */
-static struct irq_chip msc313_gpio_irqchip = {
+static const struct irq_chip msc313_gpio_irqchip = {
 	.name = "GPIO",
 	.irq_eoi = irq_chip_eoi_parent,
-	.irq_mask = irq_chip_mask_parent,
-	.irq_unmask = irq_chip_unmask_parent,
+	.irq_mask = msc313_gpio_irq_mask,
+	.irq_unmask = msc313_gpio_irq_unmask,
 	.irq_set_type = irq_chip_set_type_parent,
 	.irq_set_affinity = irq_chip_set_affinity_parent,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 /*
@@ -644,7 +662,7 @@ static int msc313_gpio_probe(struct platform_device *pdev)
 	gpiochip->names = gpio->gpio_data->names;
 
 	gpioirqchip = &gpiochip->irq;
-	gpioirqchip->chip = &msc313_gpio_irqchip;
+	gpio_irq_chip_set_chip(gpioirqchip, &msc313_gpio_irqchip);
 	gpioirqchip->fwnode = of_node_to_fwnode(dev->of_node);
 	gpioirqchip->parent_domain = parent_domain;
 	gpioirqchip->child_to_parent_hwirq = msc313e_gpio_child_to_parent_hwirq;
diff --git a/drivers/gpio/gpio-mxs.c b/drivers/gpio/gpio-mxs.c
index 7f59e5d936c2..390e619a2831 100644
--- a/drivers/gpio/gpio-mxs.c
+++ b/drivers/gpio/gpio-mxs.c
@@ -364,4 +364,3 @@ MODULE_AUTHOR("Freescale Semiconductor, "
 	      "Daniel Mack <danielncaiaq.de>, "
 	      "Juergen Beisert <kernel@pengutronix.de>");
 MODULE_DESCRIPTION("Freescale MXS GPIO");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c
index f5f3d4b22452..a08be5bf6808 100644
--- a/drivers/gpio/gpio-omap.c
+++ b/drivers/gpio/gpio-omap.c
@@ -12,6 +12,7 @@
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/interrupt.h>
+#include <linux/seq_file.h>
 #include <linux/syscore_ops.h>
 #include <linux/err.h>
 #include <linux/clk.h>
@@ -47,6 +48,7 @@ struct gpio_regs {
 struct gpio_bank {
 	void __iomem *base;
 	const struct omap_gpio_reg_offs *regs;
+	struct device *dev;
 
 	int irq;
 	u32 non_wakeup_gpios;
@@ -681,6 +683,7 @@ static void omap_gpio_mask_irq(struct irq_data *d)
 	omap_set_gpio_triggering(bank, offset, IRQ_TYPE_NONE);
 	omap_set_gpio_irqenable(bank, offset, 0);
 	raw_spin_unlock_irqrestore(&bank->lock, flags);
+	gpiochip_disable_irq(&bank->chip, offset);
 }
 
 static void omap_gpio_unmask_irq(struct irq_data *d)
@@ -690,6 +693,7 @@ static void omap_gpio_unmask_irq(struct irq_data *d)
 	u32 trigger = irqd_get_trigger_type(d);
 	unsigned long flags;
 
+	gpiochip_enable_irq(&bank->chip, offset);
 	raw_spin_lock_irqsave(&bank->lock, flags);
 	omap_set_gpio_irqenable(bank, offset, 1);
 
@@ -708,6 +712,40 @@ static void omap_gpio_unmask_irq(struct irq_data *d)
 	raw_spin_unlock_irqrestore(&bank->lock, flags);
 }
 
+static void omap_gpio_irq_print_chip(struct irq_data *d, struct seq_file *p)
+{
+	struct gpio_bank *bank = omap_irq_data_get_bank(d);
+
+	seq_printf(p, dev_name(bank->dev));
+}
+
+static const struct irq_chip omap_gpio_irq_chip = {
+	.irq_startup = omap_gpio_irq_startup,
+	.irq_shutdown = omap_gpio_irq_shutdown,
+	.irq_mask = omap_gpio_mask_irq,
+	.irq_unmask = omap_gpio_unmask_irq,
+	.irq_set_type = omap_gpio_irq_type,
+	.irq_set_wake = omap_gpio_wake_enable,
+	.irq_bus_lock = omap_gpio_irq_bus_lock,
+	.irq_bus_sync_unlock = gpio_irq_bus_sync_unlock,
+	.irq_print_chip = omap_gpio_irq_print_chip,
+	.flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_IMMUTABLE,
+	 GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
+static const struct irq_chip omap_gpio_irq_chip_nowake = {
+	.irq_startup = omap_gpio_irq_startup,
+	.irq_shutdown = omap_gpio_irq_shutdown,
+	.irq_mask = omap_gpio_mask_irq,
+	.irq_unmask = omap_gpio_unmask_irq,
+	.irq_set_type = omap_gpio_irq_type,
+	.irq_bus_lock = omap_gpio_irq_bus_lock,
+	.irq_bus_sync_unlock = gpio_irq_bus_sync_unlock,
+	.irq_print_chip = omap_gpio_irq_print_chip,
+	.flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_IMMUTABLE,
+	 GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 /*---------------------------------------------------------------------*/
 
 static int omap_mpuio_suspend_noirq(struct device *dev)
@@ -986,13 +1024,11 @@ static void omap_gpio_mod_init(struct gpio_bank *bank)
 		writel_relaxed(0, base + bank->regs->ctrl);
 }
 
-static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc,
-			       struct device *pm_dev)
+static int omap_gpio_chip_init(struct gpio_bank *bank, struct device *pm_dev)
 {
 	struct gpio_irq_chip *irq;
 	static int gpio;
 	const char *label;
-	int irq_base = 0;
 	int ret;
 
 	/*
@@ -1024,30 +1060,16 @@ static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc,
 	}
 	bank->chip.ngpio = bank->width;
 
-#ifdef CONFIG_ARCH_OMAP1
-	/*
-	 * REVISIT: Once we have OMAP1 supporting SPARSE_IRQ, we can drop
-	 * irq_alloc_descs() since a base IRQ offset will no longer be needed.
-	 */
-	irq_base = devm_irq_alloc_descs(bank->chip.parent,
-					-1, 0, bank->width, 0);
-	if (irq_base < 0) {
-		dev_err(bank->chip.parent, "Couldn't allocate IRQ numbers\n");
-		return -ENODEV;
-	}
-#endif
-
+	irq = &bank->chip.irq;
 	/* MPUIO is a bit different, reading IRQ status clears it */
 	if (bank->is_mpuio && !bank->regs->wkup_en)
-		irqc->irq_set_wake = NULL;
-
-	irq = &bank->chip.irq;
-	irq->chip = irqc;
+		gpio_irq_chip_set_chip(irq, &omap_gpio_irq_chip_nowake);
+	else
+		gpio_irq_chip_set_chip(irq, &omap_gpio_irq_chip);
 	irq->handler = handle_bad_irq;
 	irq->default_type = IRQ_TYPE_NONE;
 	irq->num_parents = 1;
 	irq->parents = &bank->irq;
-	irq->first = irq_base;
 
 	ret = gpiochip_add_data(&bank->chip, bank);
 	if (ret)
@@ -1376,7 +1398,6 @@ static int omap_gpio_probe(struct platform_device *pdev)
 	struct device_node *node = dev->of_node;
 	const struct omap_gpio_platform_data *pdata;
 	struct gpio_bank *bank;
-	struct irq_chip *irqc;
 	int ret;
 
 	pdata = device_get_match_data(dev);
@@ -1389,21 +1410,7 @@ static int omap_gpio_probe(struct platform_device *pdev)
 	if (!bank)
 		return -ENOMEM;
 
-	irqc = devm_kzalloc(dev, sizeof(*irqc), GFP_KERNEL);
-	if (!irqc)
-		return -ENOMEM;
-
-	irqc->irq_startup = omap_gpio_irq_startup,
-	irqc->irq_shutdown = omap_gpio_irq_shutdown,
-	irqc->irq_ack = dummy_irq_chip.irq_ack,
-	irqc->irq_mask = omap_gpio_mask_irq,
-	irqc->irq_unmask = omap_gpio_unmask_irq,
-	irqc->irq_set_type = omap_gpio_irq_type,
-	irqc->irq_set_wake = omap_gpio_wake_enable,
-	irqc->irq_bus_lock = omap_gpio_irq_bus_lock,
-	irqc->irq_bus_sync_unlock = gpio_irq_bus_sync_unlock,
-	irqc->name = dev_name(&pdev->dev);
-	irqc->flags = IRQCHIP_MASK_ON_SUSPEND;
+	bank->dev = dev;
 
 	bank->irq = platform_get_irq(pdev, 0);
 	if (bank->irq <= 0) {
@@ -1467,7 +1474,7 @@ static int omap_gpio_probe(struct platform_device *pdev)
 
 	omap_gpio_mod_init(bank);
 
-	ret = omap_gpio_chip_init(bank, irqc, dev);
+	ret = omap_gpio_chip_init(bank, dev);
 	if (ret) {
 		pm_runtime_put_sync(dev);
 		pm_runtime_disable(dev);
diff --git a/drivers/gpio/gpio-pci-idio-16.c b/drivers/gpio/gpio-pci-idio-16.c
index a86ce748384b..6726c32e31e6 100644
--- a/drivers/gpio/gpio-pci-idio-16.c
+++ b/drivers/gpio/gpio-pci-idio-16.c
@@ -107,6 +107,8 @@ static void idio_16_irq_mask(struct irq_data *data)
 
 		raw_spin_unlock_irqrestore(&idio16gpio->lock, flags);
 	}
+
+	gpiochip_disable_irq(chip, irqd_to_hwirq(data));
 }
 
 static void idio_16_irq_unmask(struct irq_data *data)
@@ -117,6 +119,8 @@ static void idio_16_irq_unmask(struct irq_data *data)
 	const unsigned long prev_irq_mask = idio16gpio->irq_mask;
 	unsigned long flags;
 
+	gpiochip_enable_irq(chip, irqd_to_hwirq(data));
+
 	idio16gpio->irq_mask |= mask;
 
 	if (!prev_irq_mask) {
@@ -138,12 +142,14 @@ static int idio_16_irq_set_type(struct irq_data *data, unsigned int flow_type)
 	return 0;
 }
 
-static struct irq_chip idio_16_irqchip = {
+static const struct irq_chip idio_16_irqchip = {
 	.name = "pci-idio-16",
 	.irq_ack = idio_16_irq_ack,
 	.irq_mask = idio_16_irq_mask,
 	.irq_unmask = idio_16_irq_unmask,
-	.irq_set_type = idio_16_irq_set_type
+	.irq_set_type = idio_16_irq_set_type,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static irqreturn_t idio_16_irq_handler(int irq, void *dev_id)
@@ -242,7 +248,7 @@ static int idio_16_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	idio_16_state_init(&idio16gpio->state);
 
 	girq = &idio16gpio->chip.irq;
-	girq->chip = &idio_16_irqchip;
+	gpio_irq_chip_set_chip(girq, &idio_16_irqchip);
 	/* This will let us handle the parent IRQ in the driver */
 	girq->parent_handler = NULL;
 	girq->num_parents = 0;
diff --git a/drivers/gpio/gpio-pcie-idio-24.c b/drivers/gpio/gpio-pcie-idio-24.c
index 8a9b98fa418f..463c0613abb9 100644
--- a/drivers/gpio/gpio-pcie-idio-24.c
+++ b/drivers/gpio/gpio-pcie-idio-24.c
@@ -3,15 +3,6 @@
  * GPIO driver for the ACCES PCIe-IDIO-24 family
  * Copyright (C) 2018 William Breathitt Gray
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2, as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
  * This driver supports the following ACCES devices: PCIe-IDIO-24,
  * PCIe-IDI-24, PCIe-IDO-24, and PCIe-IDIO-12.
  */
@@ -396,6 +387,8 @@ static void idio_24_irq_mask(struct irq_data *data)
 	}
 
 	raw_spin_unlock_irqrestore(&idio24gpio->lock, flags);
+
+	gpiochip_disable_irq(chip, irqd_to_hwirq(data));
 }
 
 static void idio_24_irq_unmask(struct irq_data *data)
@@ -408,6 +401,8 @@ static void idio_24_irq_unmask(struct irq_data *data)
 	const unsigned long bank_offset = bit_offset / 8;
 	unsigned char cos_enable_state;
 
+	gpiochip_enable_irq(chip, irqd_to_hwirq(data));
+
 	raw_spin_lock_irqsave(&idio24gpio->lock, flags);
 
 	prev_irq_mask = idio24gpio->irq_mask >> bank_offset * 8;
@@ -437,12 +432,14 @@ static int idio_24_irq_set_type(struct irq_data *data, unsigned int flow_type)
 	return 0;
 }
 
-static struct irq_chip idio_24_irqchip = {
+static const struct irq_chip idio_24_irqchip = {
 	.name = "pcie-idio-24",
 	.irq_ack = idio_24_irq_ack,
 	.irq_mask = idio_24_irq_mask,
 	.irq_unmask = idio_24_irq_unmask,
-	.irq_set_type = idio_24_irq_set_type
+	.irq_set_type = idio_24_irq_set_type,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static irqreturn_t idio_24_irq_handler(int irq, void *dev_id)
@@ -535,7 +532,7 @@ static int idio_24_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	idio24gpio->chip.set_multiple = idio_24_gpio_set_multiple;
 
 	girq = &idio24gpio->chip.irq;
-	girq->chip = &idio_24_irqchip;
+	gpio_irq_chip_set_chip(girq, &idio_24_irqchip);
 	/* This will let us handle the parent IRQ in the driver */
 	girq->parent_handler = NULL;
 	girq->num_parents = 0;
diff --git a/drivers/gpio/gpio-pxa.c b/drivers/gpio/gpio-pxa.c
index 1198ab0305d0..a1630ed4b741 100644
--- a/drivers/gpio/gpio-pxa.c
+++ b/drivers/gpio/gpio-pxa.c
@@ -171,11 +171,6 @@ static inline struct pxa_gpio_bank *gpio_to_pxabank(struct gpio_chip *c,
 	return chip_to_pxachip(c)->banks + gpio / 32;
 }
 
-static inline int gpio_is_pxa_type(int type)
-{
-	return (type & MMP_GPIO) == 0;
-}
-
 static inline int gpio_is_mmp_type(int type)
 {
 	return (type & MMP_GPIO) != 0;
diff --git a/drivers/gpio/gpio-raspberrypi-exp.c b/drivers/gpio/gpio-raspberrypi-exp.c
index 3c414e0005fc..ecb0d3800dfe 100644
--- a/drivers/gpio/gpio-raspberrypi-exp.c
+++ b/drivers/gpio/gpio-raspberrypi-exp.c
@@ -234,7 +234,7 @@ static int rpi_exp_gpio_probe(struct platform_device *pdev)
 	return devm_gpiochip_add_data(dev, &rpi_gpio->gc, rpi_gpio);
 }
 
-static const struct of_device_id rpi_exp_gpio_ids[] = {
+static const struct of_device_id rpi_exp_gpio_ids[] __maybe_unused = {
 	{ .compatible = "raspberrypi,firmware-gpio" },
 	{ }
 };
diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c
index 5b117f3bd322..2525adb52f4f 100644
--- a/drivers/gpio/gpio-rcar.c
+++ b/drivers/gpio/gpio-rcar.c
@@ -663,7 +663,7 @@ static struct platform_driver gpio_rcar_device_driver = {
 	.driver		= {
 		.name	= "gpio_rcar",
 		.pm     = &gpio_rcar_pm_ops,
-		.of_match_table = of_match_ptr(gpio_rcar_of_table),
+		.of_match_table = gpio_rcar_of_table,
 	}
 };
 
diff --git a/drivers/gpio/gpio-rda.c b/drivers/gpio/gpio-rda.c
index 62ba18b3a602..cb2f63eee2aa 100644
--- a/drivers/gpio/gpio-rda.c
+++ b/drivers/gpio/gpio-rda.c
@@ -38,7 +38,6 @@ struct rda_gpio {
 	struct gpio_chip chip;
 	void __iomem *base;
 	spinlock_t lock;
-	struct irq_chip irq_chip;
 	int irq;
 };
 
@@ -74,6 +73,7 @@ static void rda_gpio_irq_mask(struct irq_data *data)
 	value |= BIT(offset) << RDA_GPIO_IRQ_FALL_SHIFT;
 
 	writel_relaxed(value, base + RDA_GPIO_INT_CTRL_CLR);
+	gpiochip_disable_irq(chip, offset);
 }
 
 static void rda_gpio_irq_ack(struct irq_data *data)
@@ -154,6 +154,7 @@ static void rda_gpio_irq_unmask(struct irq_data *data)
 	u32 offset = irqd_to_hwirq(data);
 	u32 trigger = irqd_get_trigger_type(data);
 
+	gpiochip_enable_irq(chip, offset);
 	rda_gpio_set_irq(chip, offset, trigger);
 }
 
@@ -195,6 +196,16 @@ static void rda_gpio_irq_handler(struct irq_desc *desc)
 	chained_irq_exit(ic, desc);
 }
 
+static const struct irq_chip rda_gpio_irq_chip = {
+	.name = "rda-gpio",
+	.irq_ack = rda_gpio_irq_ack,
+	.irq_mask = rda_gpio_irq_mask,
+	.irq_unmask = rda_gpio_irq_unmask,
+	.irq_set_type = rda_gpio_irq_set_type,
+	.flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 static int rda_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -241,15 +252,8 @@ static int rda_gpio_probe(struct platform_device *pdev)
 	rda_gpio->chip.base = -1;
 
 	if (rda_gpio->irq >= 0) {
-		rda_gpio->irq_chip.name = "rda-gpio",
-		rda_gpio->irq_chip.irq_ack = rda_gpio_irq_ack,
-		rda_gpio->irq_chip.irq_mask = rda_gpio_irq_mask,
-		rda_gpio->irq_chip.irq_unmask = rda_gpio_irq_unmask,
-		rda_gpio->irq_chip.irq_set_type = rda_gpio_irq_set_type,
-		rda_gpio->irq_chip.flags = IRQCHIP_SKIP_SET_WAKE,
-
 		girq = &rda_gpio->chip.irq;
-		girq->chip = &rda_gpio->irq_chip;
+		gpio_irq_chip_set_chip(girq, &rda_gpio_irq_chip);
 		girq->handler = handle_bad_irq;
 		girq->default_type = IRQ_TYPE_NONE;
 		girq->parent_handler = rda_gpio_irq_handler;
@@ -286,4 +290,3 @@ module_platform_driver_probe(rda_gpio_driver, rda_gpio_probe);
 
 MODULE_DESCRIPTION("RDA Micro GPIO driver");
 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-reg.c b/drivers/gpio/gpio-reg.c
index d35169bde25a..73c7260d89c0 100644
--- a/drivers/gpio/gpio-reg.c
+++ b/drivers/gpio/gpio-reg.c
@@ -4,11 +4,19 @@
  *
  * Copyright (C) 2016 Russell King
  */
-#include <linux/gpio/driver.h>
-#include <linux/gpio/gpio-reg.h>
+#include <linux/bits.h>
+#include <linux/container_of.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
 #include <linux/io.h>
+#include <linux/irqdomain.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/types.h>
+
+#include <linux/gpio/driver.h>
+#include <linux/gpio/gpio-reg.h>
 
 struct gpio_reg {
 	struct gpio_chip gc;
diff --git a/drivers/gpio/gpio-regmap.c b/drivers/gpio/gpio-regmap.c
index fca17d478984..c08c8e528867 100644
--- a/drivers/gpio/gpio-regmap.c
+++ b/drivers/gpio/gpio-regmap.c
@@ -5,11 +5,17 @@
  * Copyright 2020 Michael Walle <michael@walle.cc>
  */
 
-#include <linux/gpio/driver.h>
-#include <linux/gpio/regmap.h>
-#include <linux/kernel.h>
+#include <linux/bits.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/io.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include <linux/gpio/driver.h>
+#include <linux/gpio/regmap.h>
 
 struct gpio_regmap {
 	struct device *parent;
diff --git a/drivers/gpio/gpio-sama5d2-piobu.c b/drivers/gpio/gpio-sama5d2-piobu.c
index 3e95da717fc9..767c33ae3213 100644
--- a/drivers/gpio/gpio-sama5d2-piobu.c
+++ b/drivers/gpio/gpio-sama5d2-piobu.c
@@ -236,7 +236,7 @@ MODULE_DEVICE_TABLE(of, sama5d2_piobu_ids);
 static struct platform_driver sama5d2_piobu_driver = {
 	.driver = {
 		.name		= "sama5d2-piobu",
-		.of_match_table	= of_match_ptr(sama5d2_piobu_ids)
+		.of_match_table	= sama5d2_piobu_ids,
 	},
 	.probe = sama5d2_piobu_probe,
 };
diff --git a/drivers/gpio/gpio-sifive.c b/drivers/gpio/gpio-sifive.c
index bc5660f61c57..98939cd4a71e 100644
--- a/drivers/gpio/gpio-sifive.c
+++ b/drivers/gpio/gpio-sifive.c
@@ -270,7 +270,7 @@ static struct platform_driver sifive_gpio_driver = {
 	.probe		= sifive_gpio_probe,
 	.driver = {
 		.name	= "sifive_gpio",
-		.of_match_table = of_match_ptr(sifive_gpio_match),
+		.of_match_table = sifive_gpio_match,
 	},
 };
 builtin_platform_driver(sifive_gpio_driver)
diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c
index e5dfd636c63c..a1c8702f362c 100644
--- a/drivers/gpio/gpio-sim.c
+++ b/drivers/gpio/gpio-sim.c
@@ -953,9 +953,9 @@ static void gpio_sim_device_deactivate_unlocked(struct gpio_sim_device *dev)
 
 	swnode = dev_fwnode(&dev->pdev->dev);
 	platform_device_unregister(dev->pdev);
+	gpio_sim_remove_hogs(dev);
 	gpio_sim_remove_swnode_recursive(swnode);
 	dev->pdev = NULL;
-	gpio_sim_remove_hogs(dev);
 }
 
 static ssize_t
diff --git a/drivers/gpio/gpio-siox.c b/drivers/gpio/gpio-siox.c
index f8c5e9fc4bac..051bc99bdfb2 100644
--- a/drivers/gpio/gpio-siox.c
+++ b/drivers/gpio/gpio-siox.c
@@ -10,7 +10,6 @@
 
 struct gpio_siox_ddata {
 	struct gpio_chip gchip;
-	struct irq_chip ichip;
 	struct mutex lock;
 	u8 setdata[1];
 	u8 getdata[3];
@@ -97,9 +96,8 @@ static int gpio_siox_get_data(struct siox_device *sdevice, const u8 buf[])
 
 static void gpio_siox_irq_ack(struct irq_data *d)
 {
-	struct irq_chip *ic = irq_data_get_irq_chip(d);
-	struct gpio_siox_ddata *ddata =
-		container_of(ic, struct gpio_siox_ddata, ichip);
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct gpio_siox_ddata *ddata = gpiochip_get_data(gc);
 
 	raw_spin_lock(&ddata->irqlock);
 	ddata->irq_status &= ~(1 << d->hwirq);
@@ -108,21 +106,21 @@ static void gpio_siox_irq_ack(struct irq_data *d)
 
 static void gpio_siox_irq_mask(struct irq_data *d)
 {
-	struct irq_chip *ic = irq_data_get_irq_chip(d);
-	struct gpio_siox_ddata *ddata =
-		container_of(ic, struct gpio_siox_ddata, ichip);
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct gpio_siox_ddata *ddata = gpiochip_get_data(gc);
 
 	raw_spin_lock(&ddata->irqlock);
 	ddata->irq_enable &= ~(1 << d->hwirq);
 	raw_spin_unlock(&ddata->irqlock);
+	gpiochip_disable_irq(gc, irqd_to_hwirq(d));
 }
 
 static void gpio_siox_irq_unmask(struct irq_data *d)
 {
-	struct irq_chip *ic = irq_data_get_irq_chip(d);
-	struct gpio_siox_ddata *ddata =
-		container_of(ic, struct gpio_siox_ddata, ichip);
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct gpio_siox_ddata *ddata = gpiochip_get_data(gc);
 
+	gpiochip_enable_irq(gc, irqd_to_hwirq(d));
 	raw_spin_lock(&ddata->irqlock);
 	ddata->irq_enable |= 1 << d->hwirq;
 	raw_spin_unlock(&ddata->irqlock);
@@ -130,9 +128,8 @@ static void gpio_siox_irq_unmask(struct irq_data *d)
 
 static int gpio_siox_irq_set_type(struct irq_data *d, u32 type)
 {
-	struct irq_chip *ic = irq_data_get_irq_chip(d);
-	struct gpio_siox_ddata *ddata =
-		container_of(ic, struct gpio_siox_ddata, ichip);
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct gpio_siox_ddata *ddata = gpiochip_get_data(gc);
 
 	raw_spin_lock(&ddata->irqlock);
 	ddata->irq_type[d->hwirq] = type;
@@ -143,8 +140,7 @@ static int gpio_siox_irq_set_type(struct irq_data *d, u32 type)
 
 static int gpio_siox_get(struct gpio_chip *chip, unsigned int offset)
 {
-	struct gpio_siox_ddata *ddata =
-		container_of(chip, struct gpio_siox_ddata, gchip);
+	struct gpio_siox_ddata *ddata = gpiochip_get_data(chip);
 	int ret;
 
 	mutex_lock(&ddata->lock);
@@ -167,8 +163,7 @@ static int gpio_siox_get(struct gpio_chip *chip, unsigned int offset)
 static void gpio_siox_set(struct gpio_chip *chip,
 			  unsigned int offset, int value)
 {
-	struct gpio_siox_ddata *ddata =
-		container_of(chip, struct gpio_siox_ddata, gchip);
+	struct gpio_siox_ddata *ddata = gpiochip_get_data(chip);
 	u8 mask = 1 << (19 - offset);
 
 	mutex_lock(&ddata->lock);
@@ -208,11 +203,22 @@ static int gpio_siox_get_direction(struct gpio_chip *chip, unsigned int offset)
 		return GPIO_LINE_DIRECTION_OUT;
 }
 
+static const struct irq_chip gpio_siox_irq_chip = {
+	.name = "siox-gpio",
+	.irq_ack = gpio_siox_irq_ack,
+	.irq_mask = gpio_siox_irq_mask,
+	.irq_unmask = gpio_siox_irq_unmask,
+	.irq_set_type = gpio_siox_irq_set_type,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 static int gpio_siox_probe(struct siox_device *sdevice)
 {
 	struct gpio_siox_ddata *ddata;
 	struct gpio_irq_chip *girq;
 	struct device *dev = &sdevice->dev;
+	struct gpio_chip *gc;
 	int ret;
 
 	ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL);
@@ -224,30 +230,25 @@ static int gpio_siox_probe(struct siox_device *sdevice)
 	mutex_init(&ddata->lock);
 	raw_spin_lock_init(&ddata->irqlock);
 
-	ddata->gchip.base = -1;
-	ddata->gchip.can_sleep = 1;
-	ddata->gchip.parent = dev;
-	ddata->gchip.owner = THIS_MODULE;
-	ddata->gchip.get = gpio_siox_get;
-	ddata->gchip.set = gpio_siox_set;
-	ddata->gchip.direction_input = gpio_siox_direction_input;
-	ddata->gchip.direction_output = gpio_siox_direction_output;
-	ddata->gchip.get_direction = gpio_siox_get_direction;
-	ddata->gchip.ngpio = 20;
-
-	ddata->ichip.name = "siox-gpio";
-	ddata->ichip.irq_ack = gpio_siox_irq_ack;
-	ddata->ichip.irq_mask = gpio_siox_irq_mask;
-	ddata->ichip.irq_unmask = gpio_siox_irq_unmask;
-	ddata->ichip.irq_set_type = gpio_siox_irq_set_type;
-
-	girq = &ddata->gchip.irq;
-	girq->chip = &ddata->ichip;
+	gc = &ddata->gchip;
+	gc->base = -1;
+	gc->can_sleep = 1;
+	gc->parent = dev;
+	gc->owner = THIS_MODULE;
+	gc->get = gpio_siox_get;
+	gc->set = gpio_siox_set;
+	gc->direction_input = gpio_siox_direction_input;
+	gc->direction_output = gpio_siox_direction_output;
+	gc->get_direction = gpio_siox_get_direction;
+	gc->ngpio = 20;
+
+	girq = &gc->irq;
+	gpio_irq_chip_set_chip(girq, &gpio_siox_irq_chip);
 	girq->default_type = IRQ_TYPE_NONE;
 	girq->handler = handle_level_irq;
 	girq->threaded = true;
 
-	ret = devm_gpiochip_add_data(dev, &ddata->gchip, NULL);
+	ret = devm_gpiochip_add_data(dev, gc, ddata);
 	if (ret)
 		dev_err(dev, "Failed to register gpio chip (%d)\n", ret);
 
diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c
index 0fa4f0a93378..27cc4da53565 100644
--- a/drivers/gpio/gpio-stmpe.c
+++ b/drivers/gpio/gpio-stmpe.c
@@ -234,6 +234,7 @@ static void stmpe_gpio_irq_mask(struct irq_data *d)
 	int mask = BIT(offset % 8);
 
 	stmpe_gpio->regs[REG_IE][regoffset] &= ~mask;
+	gpiochip_disable_irq(gc, offset);
 }
 
 static void stmpe_gpio_irq_unmask(struct irq_data *d)
@@ -244,6 +245,7 @@ static void stmpe_gpio_irq_unmask(struct irq_data *d)
 	int regoffset = offset / 8;
 	int mask = BIT(offset % 8);
 
+	gpiochip_enable_irq(gc, offset);
 	stmpe_gpio->regs[REG_IE][regoffset] |= mask;
 }
 
@@ -357,13 +359,15 @@ static void stmpe_dbg_show(struct seq_file *s, struct gpio_chip *gc)
 	}
 }
 
-static struct irq_chip stmpe_gpio_irq_chip = {
+static const struct irq_chip stmpe_gpio_irq_chip = {
 	.name			= "stmpe-gpio",
 	.irq_bus_lock		= stmpe_gpio_irq_lock,
 	.irq_bus_sync_unlock	= stmpe_gpio_irq_sync_unlock,
 	.irq_mask		= stmpe_gpio_irq_mask,
 	.irq_unmask		= stmpe_gpio_irq_unmask,
 	.irq_set_type		= stmpe_gpio_irq_set_type,
+	.flags			= IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 #define MAX_GPIOS 24
@@ -511,7 +515,7 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
 		}
 
 		girq = &stmpe_gpio->chip.irq;
-		girq->chip = &stmpe_gpio_irq_chip;
+		gpio_irq_chip_set_chip(girq, &stmpe_gpio_irq_chip);
 		/* This will let us handle the parent IRQ in the driver */
 		girq->parent_handler = NULL;
 		girq->num_parents = 0;
diff --git a/drivers/gpio/gpio-stp-xway.c b/drivers/gpio/gpio-stp-xway.c
index 0ce1543426a4..4750ea34204c 100644
--- a/drivers/gpio/gpio-stp-xway.c
+++ b/drivers/gpio/gpio-stp-xway.c
@@ -292,7 +292,7 @@ static int xway_stp_probe(struct platform_device *pdev)
 	}
 
 	/* check which edge trigger we should use, default to a falling edge */
-	if (!of_find_property(pdev->dev.of_node, "lantiq,rising", NULL))
+	if (!of_property_read_bool(pdev->dev.of_node, "lantiq,rising"))
 		chip->edge = XWAY_STP_FALLING;
 
 	clk = devm_clk_get(&pdev->dev, NULL);
diff --git a/drivers/gpio/gpio-tangier.c b/drivers/gpio/gpio-tangier.c
new file mode 100644
index 000000000000..e990781935ba
--- /dev/null
+++ b/drivers/gpio/gpio-tangier.c
@@ -0,0 +1,536 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Intel Tangier GPIO driver
+ *
+ * Copyright (c) 2016, 2021, 2023 Intel Corporation.
+ *
+ * Authors: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
+ *          Pandith N <pandith.n@intel.com>
+ *          Raag Jadav <raag.jadav@intel.com>
+ */
+
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/export.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/spinlock.h>
+#include <linux/string_helpers.h>
+#include <linux/types.h>
+
+#include <linux/gpio/driver.h>
+
+#include "gpio-tangier.h"
+
+#define GCCR		0x000	/* Controller configuration */
+#define GPLR		0x004	/* Pin level r/o */
+#define GPDR		0x01c	/* Pin direction */
+#define GPSR		0x034	/* Pin set w/o */
+#define GPCR		0x04c	/* Pin clear w/o */
+#define GRER		0x064	/* Rising edge detect */
+#define GFER		0x07c	/* Falling edge detect */
+#define GFBR		0x094	/* Glitch filter bypass */
+#define GIMR		0x0ac	/* Interrupt mask */
+#define GISR		0x0c4	/* Interrupt source */
+#define GITR		0x300	/* Input type */
+#define GLPR		0x318	/* Level input polarity */
+
+/**
+ * struct tng_gpio_context - Context to be saved during suspend-resume
+ * @level: Pin level
+ * @gpdr: Pin direction
+ * @grer: Rising edge detect enable
+ * @gfer: Falling edge detect enable
+ * @gimr: Interrupt mask
+ * @gwmr: Wake mask
+ */
+struct tng_gpio_context {
+	u32 level;
+	u32 gpdr;
+	u32 grer;
+	u32 gfer;
+	u32 gimr;
+	u32 gwmr;
+};
+
+static void __iomem *gpio_reg(struct gpio_chip *chip, unsigned int offset,
+			      unsigned int reg)
+{
+	struct tng_gpio *priv = gpiochip_get_data(chip);
+	u8 reg_offset = offset / 32;
+
+	return priv->reg_base + reg + reg_offset * 4;
+}
+
+static void __iomem *gpio_reg_and_bit(struct gpio_chip *chip, unsigned int offset,
+				      unsigned int reg, u8 *bit)
+{
+	struct tng_gpio *priv = gpiochip_get_data(chip);
+	u8 reg_offset = offset / 32;
+	u8 shift = offset % 32;
+
+	*bit = shift;
+	return priv->reg_base + reg + reg_offset * 4;
+}
+
+static int tng_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	void __iomem *gplr;
+	u8 shift;
+
+	gplr = gpio_reg_and_bit(chip, offset, GPLR, &shift);
+
+	return !!(readl(gplr) & BIT(shift));
+}
+
+static void tng_gpio_set(struct gpio_chip *chip, unsigned int offset, int value)
+{
+	struct tng_gpio *priv = gpiochip_get_data(chip);
+	unsigned long flags;
+	void __iomem *reg;
+	u8 shift;
+
+	reg = gpio_reg_and_bit(chip, offset, value ? GPSR : GPCR, &shift);
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+
+	writel(BIT(shift), reg);
+
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+static int tng_gpio_direction_input(struct gpio_chip *chip, unsigned int offset)
+{
+	struct tng_gpio *priv = gpiochip_get_data(chip);
+	unsigned long flags;
+	void __iomem *gpdr;
+	u32 value;
+	u8 shift;
+
+	gpdr = gpio_reg_and_bit(chip, offset, GPDR, &shift);
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+
+	value = readl(gpdr);
+	value &= ~BIT(shift);
+	writel(value, gpdr);
+
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+
+	return 0;
+}
+
+static int tng_gpio_direction_output(struct gpio_chip *chip, unsigned int offset,
+				     int value)
+{
+	struct tng_gpio *priv = gpiochip_get_data(chip);
+	unsigned long flags;
+	void __iomem *gpdr;
+	u8 shift;
+
+	gpdr = gpio_reg_and_bit(chip, offset, GPDR, &shift);
+	tng_gpio_set(chip, offset, value);
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+
+	value = readl(gpdr);
+	value |= BIT(shift);
+	writel(value, gpdr);
+
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+
+	return 0;
+}
+
+static int tng_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+	void __iomem *gpdr;
+	u8 shift;
+
+	gpdr = gpio_reg_and_bit(chip, offset, GPDR, &shift);
+
+	if (readl(gpdr) & BIT(shift))
+		return GPIO_LINE_DIRECTION_OUT;
+
+	return GPIO_LINE_DIRECTION_IN;
+}
+
+static int tng_gpio_set_debounce(struct gpio_chip *chip, unsigned int offset,
+				 unsigned int debounce)
+{
+	struct tng_gpio *priv = gpiochip_get_data(chip);
+	unsigned long flags;
+	void __iomem *gfbr;
+	u32 value;
+	u8 shift;
+
+	gfbr = gpio_reg_and_bit(chip, offset, GFBR, &shift);
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+
+	value = readl(gfbr);
+	if (debounce)
+		value &= ~BIT(shift);
+	else
+		value |= BIT(shift);
+	writel(value, gfbr);
+
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+
+	return 0;
+}
+
+static int tng_gpio_set_config(struct gpio_chip *chip, unsigned int offset,
+			       unsigned long config)
+{
+	u32 debounce;
+
+	switch (pinconf_to_config_param(config)) {
+	case PIN_CONFIG_BIAS_DISABLE:
+	case PIN_CONFIG_BIAS_PULL_UP:
+	case PIN_CONFIG_BIAS_PULL_DOWN:
+		return gpiochip_generic_config(chip, offset, config);
+	case PIN_CONFIG_INPUT_DEBOUNCE:
+		debounce = pinconf_to_config_argument(config);
+		return tng_gpio_set_debounce(chip, offset, debounce);
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static void tng_irq_ack(struct irq_data *d)
+{
+	struct tng_gpio *priv = irq_data_get_irq_chip_data(d);
+	irq_hw_number_t gpio = irqd_to_hwirq(d);
+	unsigned long flags;
+	void __iomem *gisr;
+	u8 shift;
+
+	gisr = gpio_reg_and_bit(&priv->chip, gpio, GISR, &shift);
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+	writel(BIT(shift), gisr);
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+static void tng_irq_unmask_mask(struct tng_gpio *priv, u32 gpio, bool unmask)
+{
+	unsigned long flags;
+	void __iomem *gimr;
+	u32 value;
+	u8 shift;
+
+	gimr = gpio_reg_and_bit(&priv->chip, gpio, GIMR, &shift);
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+
+	value = readl(gimr);
+	if (unmask)
+		value |= BIT(shift);
+	else
+		value &= ~BIT(shift);
+	writel(value, gimr);
+
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+static void tng_irq_mask(struct irq_data *d)
+{
+	struct tng_gpio *priv = irq_data_get_irq_chip_data(d);
+	irq_hw_number_t gpio = irqd_to_hwirq(d);
+
+	tng_irq_unmask_mask(priv, gpio, false);
+	gpiochip_disable_irq(&priv->chip, gpio);
+}
+
+static void tng_irq_unmask(struct irq_data *d)
+{
+	struct tng_gpio *priv = irq_data_get_irq_chip_data(d);
+	irq_hw_number_t gpio = irqd_to_hwirq(d);
+
+	gpiochip_enable_irq(&priv->chip, gpio);
+	tng_irq_unmask_mask(priv, gpio, true);
+}
+
+static int tng_irq_set_type(struct irq_data *d, unsigned int type)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct tng_gpio *priv = gpiochip_get_data(gc);
+	irq_hw_number_t gpio = irqd_to_hwirq(d);
+	void __iomem *grer = gpio_reg(&priv->chip, gpio, GRER);
+	void __iomem *gfer = gpio_reg(&priv->chip, gpio, GFER);
+	void __iomem *gitr = gpio_reg(&priv->chip, gpio, GITR);
+	void __iomem *glpr = gpio_reg(&priv->chip, gpio, GLPR);
+	u8 shift = gpio % 32;
+	unsigned long flags;
+	u32 value;
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+
+	value = readl(grer);
+	if (type & IRQ_TYPE_EDGE_RISING)
+		value |= BIT(shift);
+	else
+		value &= ~BIT(shift);
+	writel(value, grer);
+
+	value = readl(gfer);
+	if (type & IRQ_TYPE_EDGE_FALLING)
+		value |= BIT(shift);
+	else
+		value &= ~BIT(shift);
+	writel(value, gfer);
+
+	/*
+	 * To prevent glitches from triggering an unintended level interrupt,
+	 * configure GLPR register first and then configure GITR.
+	 */
+	value = readl(glpr);
+	if (type & IRQ_TYPE_LEVEL_LOW)
+		value |= BIT(shift);
+	else
+		value &= ~BIT(shift);
+	writel(value, glpr);
+
+	if (type & IRQ_TYPE_LEVEL_MASK) {
+		value = readl(gitr);
+		value |= BIT(shift);
+		writel(value, gitr);
+
+		irq_set_handler_locked(d, handle_level_irq);
+	} else if (type & IRQ_TYPE_EDGE_BOTH) {
+		value = readl(gitr);
+		value &= ~BIT(shift);
+		writel(value, gitr);
+
+		irq_set_handler_locked(d, handle_edge_irq);
+	}
+
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+
+	return 0;
+}
+
+static int tng_irq_set_wake(struct irq_data *d, unsigned int on)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct tng_gpio *priv = gpiochip_get_data(gc);
+	irq_hw_number_t gpio = irqd_to_hwirq(d);
+	void __iomem *gwmr = gpio_reg(&priv->chip, gpio, priv->wake_regs.gwmr);
+	void __iomem *gwsr = gpio_reg(&priv->chip, gpio, priv->wake_regs.gwsr);
+	u8 shift = gpio % 32;
+	unsigned long flags;
+	u32 value;
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+
+	/* Clear the existing wake status */
+	writel(BIT(shift), gwsr);
+
+	value = readl(gwmr);
+	if (on)
+		value |= BIT(shift);
+	else
+		value &= ~BIT(shift);
+	writel(value, gwmr);
+
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+
+	dev_dbg(priv->dev, "%s wake for gpio %lu\n", str_enable_disable(on), gpio);
+	return 0;
+}
+
+static const struct irq_chip tng_irqchip = {
+	.name		= "gpio-tangier",
+	.irq_ack	= tng_irq_ack,
+	.irq_mask	= tng_irq_mask,
+	.irq_unmask	= tng_irq_unmask,
+	.irq_set_type	= tng_irq_set_type,
+	.irq_set_wake	= tng_irq_set_wake,
+	.flags          = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
+static void tng_irq_handler(struct irq_desc *desc)
+{
+	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
+	struct tng_gpio *priv = gpiochip_get_data(gc);
+	struct irq_chip *irqchip = irq_desc_get_chip(desc);
+	unsigned long base, gpio;
+
+	chained_irq_enter(irqchip, desc);
+
+	/* Check GPIO controller to check which pin triggered the interrupt */
+	for (base = 0; base < priv->chip.ngpio; base += 32) {
+		void __iomem *gisr = gpio_reg(&priv->chip, base, GISR);
+		void __iomem *gimr = gpio_reg(&priv->chip, base, GIMR);
+		unsigned long pending, enabled;
+
+		pending = readl(gisr);
+		enabled = readl(gimr);
+
+		/* Only interrupts that are enabled */
+		pending &= enabled;
+
+		for_each_set_bit(gpio, &pending, 32)
+			generic_handle_domain_irq(gc->irq.domain, base + gpio);
+	}
+
+	chained_irq_exit(irqchip, desc);
+}
+
+static int tng_irq_init_hw(struct gpio_chip *chip)
+{
+	struct tng_gpio *priv = gpiochip_get_data(chip);
+	void __iomem *reg;
+	unsigned int base;
+
+	for (base = 0; base < priv->chip.ngpio; base += 32) {
+		/* Clear the rising-edge detect register */
+		reg = gpio_reg(&priv->chip, base, GRER);
+		writel(0, reg);
+
+		/* Clear the falling-edge detect register */
+		reg = gpio_reg(&priv->chip, base, GFER);
+		writel(0, reg);
+	}
+
+	return 0;
+}
+
+static int tng_gpio_add_pin_ranges(struct gpio_chip *chip)
+{
+	struct tng_gpio *priv = gpiochip_get_data(chip);
+	const struct tng_gpio_pinrange *range;
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < priv->pin_info.nranges; i++) {
+		range = &priv->pin_info.pin_ranges[i];
+		ret = gpiochip_add_pin_range(&priv->chip,
+					     priv->pin_info.name,
+					     range->gpio_base,
+					     range->pin_base,
+					     range->npins);
+		if (ret) {
+			dev_err(priv->dev, "failed to add GPIO pin range\n");
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+int devm_tng_gpio_probe(struct device *dev, struct tng_gpio *gpio)
+{
+	const struct tng_gpio_info *info = &gpio->info;
+	struct gpio_irq_chip *girq;
+	int ret;
+
+	gpio->ctx = devm_kcalloc(dev, DIV_ROUND_UP(info->ngpio, 32), sizeof(*gpio->ctx), GFP_KERNEL);
+	if (!gpio->ctx)
+		return -ENOMEM;
+
+	gpio->chip.label = dev_name(dev);
+	gpio->chip.parent = dev;
+	gpio->chip.request = gpiochip_generic_request;
+	gpio->chip.free = gpiochip_generic_free;
+	gpio->chip.direction_input = tng_gpio_direction_input;
+	gpio->chip.direction_output = tng_gpio_direction_output;
+	gpio->chip.get = tng_gpio_get;
+	gpio->chip.set = tng_gpio_set;
+	gpio->chip.get_direction = tng_gpio_get_direction;
+	gpio->chip.set_config = tng_gpio_set_config;
+	gpio->chip.base = info->base;
+	gpio->chip.ngpio = info->ngpio;
+	gpio->chip.can_sleep = false;
+	gpio->chip.add_pin_ranges = tng_gpio_add_pin_ranges;
+
+	raw_spin_lock_init(&gpio->lock);
+
+	girq = &gpio->chip.irq;
+	gpio_irq_chip_set_chip(girq, &tng_irqchip);
+	girq->init_hw = tng_irq_init_hw;
+	girq->parent_handler = tng_irq_handler;
+	girq->num_parents = 1;
+	girq->parents = devm_kcalloc(dev, girq->num_parents,
+				     sizeof(*girq->parents), GFP_KERNEL);
+	if (!girq->parents)
+		return -ENOMEM;
+
+	girq->parents[0] = gpio->irq;
+	girq->first = info->first;
+	girq->default_type = IRQ_TYPE_NONE;
+	girq->handler = handle_bad_irq;
+
+	ret = devm_gpiochip_add_data(dev, &gpio->chip, gpio);
+	if (ret)
+		return dev_err_probe(dev, ret, "gpiochip_add error\n");
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(devm_tng_gpio_probe, GPIO_TANGIER);
+
+int tng_gpio_suspend(struct device *dev)
+{
+	struct tng_gpio *priv = dev_get_drvdata(dev);
+	struct tng_gpio_context *ctx = priv->ctx;
+	unsigned long flags;
+	unsigned int base;
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+
+	for (base = 0; base < priv->chip.ngpio; base += 32, ctx++) {
+		/* GPLR is RO, values read will be restored using GPSR */
+		ctx->level = readl(gpio_reg(&priv->chip, base, GPLR));
+
+		ctx->gpdr = readl(gpio_reg(&priv->chip, base, GPDR));
+		ctx->grer = readl(gpio_reg(&priv->chip, base, GRER));
+		ctx->gfer = readl(gpio_reg(&priv->chip, base, GFER));
+		ctx->gimr = readl(gpio_reg(&priv->chip, base, GIMR));
+
+		ctx->gwmr = readl(gpio_reg(&priv->chip, base, priv->wake_regs.gwmr));
+	}
+
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(tng_gpio_suspend, GPIO_TANGIER);
+
+int tng_gpio_resume(struct device *dev)
+{
+	struct tng_gpio *priv = dev_get_drvdata(dev);
+	struct tng_gpio_context *ctx = priv->ctx;
+	unsigned long flags;
+	unsigned int base;
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+
+	for (base = 0; base < priv->chip.ngpio; base += 32, ctx++) {
+		/* GPLR is RO, values read will be restored using GPSR */
+		writel(ctx->level, gpio_reg(&priv->chip, base, GPSR));
+
+		writel(ctx->gpdr, gpio_reg(&priv->chip, base, GPDR));
+		writel(ctx->grer, gpio_reg(&priv->chip, base, GRER));
+		writel(ctx->gfer, gpio_reg(&priv->chip, base, GFER));
+		writel(ctx->gimr, gpio_reg(&priv->chip, base, GIMR));
+
+		writel(ctx->gwmr, gpio_reg(&priv->chip, base, priv->wake_regs.gwmr));
+	}
+
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(tng_gpio_resume, GPIO_TANGIER);
+
+MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
+MODULE_AUTHOR("Pandith N <pandith.n@intel.com>");
+MODULE_AUTHOR("Raag Jadav <raag.jadav@intel.com>");
+MODULE_DESCRIPTION("Intel Tangier GPIO driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-tangier.h b/drivers/gpio/gpio-tangier.h
new file mode 100644
index 000000000000..16c4f22908fb
--- /dev/null
+++ b/drivers/gpio/gpio-tangier.h
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Intel Tangier GPIO functions
+ *
+ * Copyright (c) 2016, 2021, 2023 Intel Corporation.
+ *
+ * Authors: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
+ *          Pandith N <pandith.n@intel.com>
+ *          Raag Jadav <raag.jadav@intel.com>
+ */
+
+#ifndef _GPIO_TANGIER_H_
+#define _GPIO_TANGIER_H_
+
+#include <linux/gpio/driver.h>
+#include <linux/spinlock_types.h>
+#include <linux/types.h>
+
+struct device;
+
+struct tng_gpio_context;
+
+/* Elkhart Lake specific wake registers */
+#define GWMR_EHL	0x100	/* Wake mask */
+#define GWSR_EHL	0x118	/* Wake source */
+#define GSIR_EHL	0x130	/* Secure input */
+
+/* Merrifield specific wake registers */
+#define GWMR_MRFLD	0x400	/* Wake mask */
+#define GWSR_MRFLD	0x418	/* Wake source */
+#define GSIR_MRFLD	0xc00	/* Secure input */
+
+/**
+ * struct tng_wake_regs - Platform specific wake registers
+ * @gwmr: Wake mask
+ * @gwsr: Wake source
+ * @gsir: Secure input
+ */
+struct tng_wake_regs {
+	u32 gwmr;
+	u32 gwsr;
+	u32 gsir;
+};
+
+/**
+ * struct tng_gpio_pinrange - Map pin numbers to gpio numbers
+ * @gpio_base: Starting GPIO number of this range
+ * @pin_base: Starting pin number of this range
+ * @npins: Number of pins in this range
+ */
+struct tng_gpio_pinrange {
+	unsigned int gpio_base;
+	unsigned int pin_base;
+	unsigned int npins;
+};
+
+#define GPIO_PINRANGE(gstart, gend, pstart)		\
+(struct tng_gpio_pinrange) {				\
+		.gpio_base = (gstart),			\
+		.pin_base = (pstart),			\
+		.npins = (gend) - (gstart) + 1,		\
+	}
+
+/**
+ * struct tng_gpio_pin_info - Platform specific pinout information
+ * @pin_ranges: Pin to GPIO mapping
+ * @nranges: Number of pin ranges
+ * @name: Respective pinctrl device name
+ */
+struct tng_gpio_pin_info {
+	const struct tng_gpio_pinrange *pin_ranges;
+	unsigned int nranges;
+	const char *name;
+};
+
+/**
+ * struct tng_gpio_info - Platform specific GPIO and IRQ information
+ * @base: GPIO base to start numbering with
+ * @ngpio: Amount of GPIOs supported by the controller
+ * @first: First IRQ to start numbering with
+ */
+struct tng_gpio_info {
+	int base;
+	u16 ngpio;
+	unsigned int first;
+};
+
+/**
+ * struct tng_gpio - Platform specific private data
+ * @chip: Instance of the struct gpio_chip
+ * @reg_base: Base address of MMIO registers
+ * @irq: Interrupt for the GPIO device
+ * @lock: Synchronization lock to prevent I/O race conditions
+ * @dev: The GPIO device
+ * @ctx: Context to be saved during suspend-resume
+ * @wake_regs: Platform specific wake registers
+ * @pin_info: Platform specific pinout information
+ * @info: Platform specific GPIO and IRQ information
+ */
+struct tng_gpio {
+	struct gpio_chip chip;
+	void __iomem *reg_base;
+	int irq;
+	raw_spinlock_t lock;
+	struct device *dev;
+	struct tng_gpio_context *ctx;
+	struct tng_wake_regs wake_regs;
+	struct tng_gpio_pin_info pin_info;
+	struct tng_gpio_info info;
+};
+
+int devm_tng_gpio_probe(struct device *dev, struct tng_gpio *gpio);
+
+int tng_gpio_suspend(struct device *dev);
+int tng_gpio_resume(struct device *dev);
+
+#endif /* _GPIO_TANGIER_H_ */
diff --git a/drivers/gpio/gpio-tb10x.c b/drivers/gpio/gpio-tb10x.c
index de6afa3f9716..78f8790168ae 100644
--- a/drivers/gpio/gpio-tb10x.c
+++ b/drivers/gpio/gpio-tb10x.c
@@ -167,7 +167,7 @@ static int tb10x_gpio_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, tb10x_gpio);
 
-	if (of_find_property(np, "interrupt-controller", NULL)) {
+	if (of_property_read_bool(np, "interrupt-controller")) {
 		struct irq_chip_generic *gc;
 
 		ret = platform_get_irq(pdev, 0);
diff --git a/drivers/gpio/gpio-thunderx.c b/drivers/gpio/gpio-thunderx.c
index cc62c6e64103..8521c6aacace 100644
--- a/drivers/gpio/gpio-thunderx.c
+++ b/drivers/gpio/gpio-thunderx.c
@@ -354,16 +354,22 @@ static int thunderx_gpio_irq_set_type(struct irq_data *d,
 	return IRQ_SET_MASK_OK;
 }
 
-static void thunderx_gpio_irq_enable(struct irq_data *data)
+static void thunderx_gpio_irq_enable(struct irq_data *d)
 {
-	irq_chip_enable_parent(data);
-	thunderx_gpio_irq_unmask(data);
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+
+	gpiochip_enable_irq(gc, irqd_to_hwirq(d));
+	irq_chip_enable_parent(d);
+	thunderx_gpio_irq_unmask(d);
 }
 
-static void thunderx_gpio_irq_disable(struct irq_data *data)
+static void thunderx_gpio_irq_disable(struct irq_data *d)
 {
-	thunderx_gpio_irq_mask(data);
-	irq_chip_disable_parent(data);
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+
+	thunderx_gpio_irq_mask(d);
+	irq_chip_disable_parent(d);
+	gpiochip_disable_irq(gc, irqd_to_hwirq(d));
 }
 
 /*
@@ -372,7 +378,7 @@ static void thunderx_gpio_irq_disable(struct irq_data *data)
  * semantics and other acknowledgment tasks associated with the GPIO
  * mechanism.
  */
-static struct irq_chip thunderx_gpio_irq_chip = {
+static const struct irq_chip thunderx_gpio_irq_chip = {
 	.name			= "GPIO",
 	.irq_enable		= thunderx_gpio_irq_enable,
 	.irq_disable		= thunderx_gpio_irq_disable,
@@ -383,8 +389,8 @@ static struct irq_chip thunderx_gpio_irq_chip = {
 	.irq_eoi		= irq_chip_eoi_parent,
 	.irq_set_affinity	= irq_chip_set_affinity_parent,
 	.irq_set_type		= thunderx_gpio_irq_set_type,
-
-	.flags			= IRQCHIP_SET_TYPE_MASKED
+	.flags			= IRQCHIP_SET_TYPE_MASKED | IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static int thunderx_gpio_child_to_parent_hwirq(struct gpio_chip *gc,
@@ -526,7 +532,7 @@ static int thunderx_gpio_probe(struct pci_dev *pdev,
 	chip->set_multiple = thunderx_gpio_set_multiple;
 	chip->set_config = thunderx_gpio_set_config;
 	girq = &chip->irq;
-	girq->chip = &thunderx_gpio_irq_chip;
+	gpio_irq_chip_set_chip(girq, &thunderx_gpio_irq_chip);
 	girq->fwnode = of_node_to_fwnode(dev->of_node);
 	girq->parent_domain =
 		irq_get_irq_data(txgpio->msix_entries[0].vector)->domain;
diff --git a/drivers/gpio/gpio-tqmx86.c b/drivers/gpio/gpio-tqmx86.c
index e739dcea61b2..6f8bd1155db7 100644
--- a/drivers/gpio/gpio-tqmx86.c
+++ b/drivers/gpio/gpio-tqmx86.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/seq_file.h>
 #include <linux/slab.h>
 
 #define TQMX86_NGPIO	8
@@ -34,7 +35,6 @@
 
 struct tqmx86_gpio_data {
 	struct gpio_chip	chip;
-	struct irq_chip		irq_chip;
 	void __iomem		*io_base;
 	int			irq;
 	raw_spinlock_t		spinlock;
@@ -122,6 +122,7 @@ static void tqmx86_gpio_irq_mask(struct irq_data *data)
 	gpiic &= ~mask;
 	tqmx86_gpio_write(gpio, gpiic, TQMX86_GPIIC);
 	raw_spin_unlock_irqrestore(&gpio->spinlock, flags);
+	gpiochip_disable_irq(&gpio->chip, irqd_to_hwirq(data));
 }
 
 static void tqmx86_gpio_irq_unmask(struct irq_data *data)
@@ -134,6 +135,7 @@ static void tqmx86_gpio_irq_unmask(struct irq_data *data)
 
 	mask = TQMX86_GPII_MASK << (offset * TQMX86_GPII_BITS);
 
+	gpiochip_enable_irq(&gpio->chip, irqd_to_hwirq(data));
 	raw_spin_lock_irqsave(&gpio->spinlock, flags);
 	gpiic = tqmx86_gpio_read(gpio, TQMX86_GPIIC);
 	gpiic &= ~mask;
@@ -226,6 +228,22 @@ static void tqmx86_init_irq_valid_mask(struct gpio_chip *chip,
 	clear_bit(3, valid_mask);
 }
 
+static void tqmx86_gpio_irq_print_chip(struct irq_data *d, struct seq_file *p)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+
+	seq_printf(p, gc->label);
+}
+
+static const struct irq_chip tqmx86_gpio_irq_chip = {
+	.irq_mask = tqmx86_gpio_irq_mask,
+	.irq_unmask = tqmx86_gpio_irq_unmask,
+	.irq_set_type = tqmx86_gpio_irq_set_type,
+	.irq_print_chip = tqmx86_gpio_irq_print_chip,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 static int tqmx86_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -277,14 +295,8 @@ static int tqmx86_gpio_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 
 	if (irq > 0) {
-		struct irq_chip *irq_chip = &gpio->irq_chip;
 		u8 irq_status;
 
-		irq_chip->name = chip->label;
-		irq_chip->irq_mask = tqmx86_gpio_irq_mask;
-		irq_chip->irq_unmask = tqmx86_gpio_irq_unmask;
-		irq_chip->irq_set_type = tqmx86_gpio_irq_set_type;
-
 		/* Mask all interrupts */
 		tqmx86_gpio_write(gpio, 0, TQMX86_GPIIC);
 
@@ -293,7 +305,7 @@ static int tqmx86_gpio_probe(struct platform_device *pdev)
 		tqmx86_gpio_write(gpio, irq_status, TQMX86_GPIIS);
 
 		girq = &chip->irq;
-		girq->chip = irq_chip;
+		gpio_irq_chip_set_chip(girq, &tqmx86_gpio_irq_chip);
 		girq->parent_handler = tqmx86_gpio_irq_handler;
 		girq->num_parents = 1;
 		girq->parents = devm_kcalloc(&pdev->dev, 1,
diff --git a/drivers/gpio/gpio-visconti.c b/drivers/gpio/gpio-visconti.c
index 5e108ba9956a..6734e7e1e2a4 100644
--- a/drivers/gpio/gpio-visconti.c
+++ b/drivers/gpio/gpio-visconti.c
@@ -15,6 +15,7 @@
 #include <linux/io.h>
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
+#include <linux/seq_file.h>
 #include <linux/bitops.h>
 
 /* register offset */
@@ -31,7 +32,7 @@ struct visconti_gpio {
 	void __iomem *base;
 	spinlock_t lock; /* protect gpio register */
 	struct gpio_chip gpio_chip;
-	struct irq_chip irq_chip;
+	struct device *dev;
 };
 
 static int visconti_gpio_irq_set_type(struct irq_data *d, unsigned int type)
@@ -119,11 +120,45 @@ static int visconti_gpio_populate_parent_fwspec(struct gpio_chip *chip,
 	return 0;
 }
 
+static void visconti_gpio_mask_irq(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+
+	irq_chip_mask_parent(d);
+	gpiochip_disable_irq(gc, irqd_to_hwirq(d));
+}
+
+static void visconti_gpio_unmask_irq(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+
+	gpiochip_enable_irq(gc, irqd_to_hwirq(d));
+	irq_chip_unmask_parent(d);
+}
+
+static void visconti_gpio_irq_print_chip(struct irq_data *d, struct seq_file *p)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct visconti_gpio *priv = gpiochip_get_data(gc);
+
+	seq_printf(p, dev_name(priv->dev));
+}
+
+static const struct irq_chip visconti_gpio_irq_chip = {
+	.irq_mask = visconti_gpio_mask_irq,
+	.irq_unmask = visconti_gpio_unmask_irq,
+	.irq_eoi = irq_chip_eoi_parent,
+	.irq_set_type = visconti_gpio_irq_set_type,
+	.irq_print_chip = visconti_gpio_irq_print_chip,
+	.flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND |
+		 IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 static int visconti_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct visconti_gpio *priv;
-	struct irq_chip *irq_chip;
 	struct gpio_irq_chip *girq;
 	struct irq_domain *parent;
 	struct device_node *irq_parent;
@@ -134,6 +169,7 @@ static int visconti_gpio_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	spin_lock_init(&priv->lock);
+	priv->dev = dev;
 
 	priv->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(priv->base))
@@ -164,16 +200,8 @@ static int visconti_gpio_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	irq_chip = &priv->irq_chip;
-	irq_chip->name = dev_name(dev);
-	irq_chip->irq_mask = irq_chip_mask_parent;
-	irq_chip->irq_unmask = irq_chip_unmask_parent;
-	irq_chip->irq_eoi = irq_chip_eoi_parent;
-	irq_chip->irq_set_type = visconti_gpio_irq_set_type;
-	irq_chip->flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND;
-
 	girq = &priv->gpio_chip.irq;
-	girq->chip = irq_chip;
+	gpio_irq_chip_set_chip(girq, &visconti_gpio_irq_chip);
 	girq->fwnode = of_node_to_fwnode(dev->of_node);
 	girq->parent_domain = parent;
 	girq->child_to_parent_hwirq = visconti_gpio_child_to_parent_hwirq;
@@ -194,7 +222,7 @@ static struct platform_driver visconti_gpio_driver = {
 	.probe		= visconti_gpio_probe,
 	.driver		= {
 		.name	= "visconti_gpio",
-		.of_match_table = of_match_ptr(visconti_gpio_of_match),
+		.of_match_table = visconti_gpio_of_match,
 	}
 };
 module_platform_driver(visconti_gpio_driver);
diff --git a/drivers/gpio/gpio-xgs-iproc.c b/drivers/gpio/gpio-xgs-iproc.c
index fd88500399c6..2d23b27d55af 100644
--- a/drivers/gpio/gpio-xgs-iproc.c
+++ b/drivers/gpio/gpio-xgs-iproc.c
@@ -11,6 +11,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/seq_file.h>
 #include <linux/spinlock.h>
 
 #define IPROC_CCA_INT_F_GPIOINT		BIT(0)
@@ -27,7 +28,6 @@
 #define IPROC_GPIO_CCA_INT_EDGE		0x24
 
 struct iproc_gpio_chip {
-	struct irq_chip irqchip;
 	struct gpio_chip gc;
 	spinlock_t lock;
 	struct device *dev;
@@ -69,6 +69,7 @@ static void iproc_gpio_irq_unmask(struct irq_data *d)
 	u32 irq = d->irq;
 	u32 int_mask, irq_type, event_mask;
 
+	gpiochip_enable_irq(gc, pin);
 	spin_lock_irqsave(&chip->lock, flags);
 	irq_type = irq_get_trigger_type(irq);
 	event_mask = readl_relaxed(chip->base + IPROC_GPIO_CCA_INT_EVENT_MASK);
@@ -110,6 +111,7 @@ static void iproc_gpio_irq_mask(struct irq_data *d)
 			       chip->base + IPROC_GPIO_CCA_INT_LEVEL_MASK);
 	}
 	spin_unlock_irqrestore(&chip->lock, flags);
+	gpiochip_disable_irq(gc, pin);
 }
 
 static int iproc_gpio_irq_set_type(struct irq_data *d, u32 type)
@@ -191,6 +193,24 @@ static irqreturn_t iproc_gpio_irq_handler(int irq, void *data)
 	return int_bits ? IRQ_HANDLED : IRQ_NONE;
 }
 
+static void iproc_gpio_irq_print_chip(struct irq_data *d, struct seq_file *p)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct iproc_gpio_chip *chip = to_iproc_gpio(gc);
+
+	seq_printf(p, dev_name(chip->dev));
+}
+
+static const struct irq_chip iproc_gpio_irq_chip = {
+	.irq_ack = iproc_gpio_irq_ack,
+	.irq_mask = iproc_gpio_irq_mask,
+	.irq_unmask = iproc_gpio_irq_unmask,
+	.irq_set_type = iproc_gpio_irq_set_type,
+	.irq_print_chip = iproc_gpio_irq_print_chip,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 static int iproc_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -230,16 +250,8 @@ static int iproc_gpio_probe(struct platform_device *pdev)
 	irq = platform_get_irq(pdev, 0);
 	if (irq > 0) {
 		struct gpio_irq_chip *girq;
-		struct irq_chip *irqc;
 		u32 val;
 
-		irqc = &chip->irqchip;
-		irqc->name = dev_name(dev);
-		irqc->irq_ack = iproc_gpio_irq_ack;
-		irqc->irq_mask = iproc_gpio_irq_mask;
-		irqc->irq_unmask = iproc_gpio_irq_unmask;
-		irqc->irq_set_type = iproc_gpio_irq_set_type;
-
 		chip->intr = devm_platform_ioremap_resource(pdev, 1);
 		if (IS_ERR(chip->intr))
 			return PTR_ERR(chip->intr);
@@ -261,7 +273,7 @@ static int iproc_gpio_probe(struct platform_device *pdev)
 		}
 
 		girq = &chip->gc.irq;
-		girq->chip = irqc;
+		gpio_irq_chip_set_chip(girq, &iproc_gpio_irq_chip);
 		/* This will let us handle the parent IRQ in the driver */
 		girq->parent_handler = NULL;
 		girq->num_parents = 0;
diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c
index e248809965ca..1fa66f2a667f 100644
--- a/drivers/gpio/gpio-xilinx.c
+++ b/drivers/gpio/gpio-xilinx.c
@@ -68,7 +68,6 @@ struct xgpio_instance {
 	DECLARE_BITMAP(dir, 64);
 	spinlock_t gpio_lock;	/* For serializing operations */
 	int irq;
-	struct irq_chip irqchip;
 	DECLARE_BITMAP(enable, 64);
 	DECLARE_BITMAP(rising_edge, 64);
 	DECLARE_BITMAP(falling_edge, 64);
@@ -416,6 +415,8 @@ static void xgpio_irq_mask(struct irq_data *irq_data)
 		xgpio_writereg(chip->regs + XGPIO_IPIER_OFFSET, temp);
 	}
 	spin_unlock_irqrestore(&chip->gpio_lock, flags);
+
+	gpiochip_disable_irq(&chip->gc, irq_offset);
 }
 
 /**
@@ -431,6 +432,8 @@ static void xgpio_irq_unmask(struct irq_data *irq_data)
 	u32 old_enable = xgpio_get_value32(chip->enable, bit);
 	u32 mask = BIT(bit / 32), val;
 
+	gpiochip_enable_irq(&chip->gc, irq_offset);
+
 	spin_lock_irqsave(&chip->gpio_lock, flags);
 
 	__set_bit(bit, chip->enable);
@@ -544,6 +547,16 @@ static void xgpio_irqhandler(struct irq_desc *desc)
 	chained_irq_exit(irqchip, desc);
 }
 
+static const struct irq_chip xgpio_irq_chip = {
+	.name = "gpio-xilinx",
+	.irq_ack = xgpio_irq_ack,
+	.irq_mask = xgpio_irq_mask,
+	.irq_unmask = xgpio_irq_unmask,
+	.irq_set_type = xgpio_set_irq_type,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
 /**
  * xgpio_probe - Probe method for the GPIO device.
  * @pdev: pointer to the platform device
@@ -653,12 +666,6 @@ static int xgpio_probe(struct platform_device *pdev)
 	if (chip->irq <= 0)
 		goto skip_irq;
 
-	chip->irqchip.name = "gpio-xilinx";
-	chip->irqchip.irq_ack = xgpio_irq_ack;
-	chip->irqchip.irq_mask = xgpio_irq_mask;
-	chip->irqchip.irq_unmask = xgpio_irq_unmask;
-	chip->irqchip.irq_set_type = xgpio_set_irq_type;
-
 	/* Disable per-channel interrupts */
 	xgpio_writereg(chip->regs + XGPIO_IPIER_OFFSET, 0);
 	/* Clear any existing per-channel interrupts */
@@ -668,7 +675,7 @@ static int xgpio_probe(struct platform_device *pdev)
 	xgpio_writereg(chip->regs + XGPIO_GIER_OFFSET, XGPIO_GIER_IE);
 
 	girq = &chip->gc.irq;
-	girq->chip = &chip->irqchip;
+	gpio_irq_chip_set_chip(girq, &xgpio_irq_chip);
 	girq->parent_handler = xgpio_irqhandler;
 	girq->num_parents = 1;
 	girq->parents = devm_kcalloc(&pdev->dev, 1,
diff --git a/drivers/gpio/gpio-xlp.c b/drivers/gpio/gpio-xlp.c
index 0199f545335f..b4b52213bcd9 100644
--- a/drivers/gpio/gpio-xlp.c
+++ b/drivers/gpio/gpio-xlp.c
@@ -90,6 +90,13 @@ static void xlp_gpio_set_reg(void __iomem *addr, unsigned gpio, int state)
 	writel(value, addr + regset);
 }
 
+static void xlp_gpio_irq_enable(struct irq_data *d)
+{
+	struct gpio_chip *gc  = irq_data_get_irq_chip_data(d);
+
+	gpiochip_enable_irq(gc, irqd_to_hwirq(d));
+}
+
 static void xlp_gpio_irq_disable(struct irq_data *d)
 {
 	struct gpio_chip *gc  = irq_data_get_irq_chip_data(d);
@@ -100,6 +107,7 @@ static void xlp_gpio_irq_disable(struct irq_data *d)
 	xlp_gpio_set_reg(priv->gpio_intr_en, d->hwirq, 0x0);
 	__clear_bit(d->hwirq, priv->gpio_enabled_mask);
 	spin_unlock_irqrestore(&priv->lock, flags);
+	gpiochip_disable_irq(gc, irqd_to_hwirq(d));
 }
 
 static void xlp_gpio_irq_mask_ack(struct irq_data *d)
@@ -163,10 +171,12 @@ static int xlp_gpio_set_irq_type(struct irq_data *d, unsigned int type)
 static struct irq_chip xlp_gpio_irq_chip = {
 	.name		= "XLP-GPIO",
 	.irq_mask_ack	= xlp_gpio_irq_mask_ack,
+	.irq_enable	= xlp_gpio_irq_enable,
 	.irq_disable	= xlp_gpio_irq_disable,
 	.irq_set_type	= xlp_gpio_set_irq_type,
 	.irq_unmask	= xlp_gpio_irq_unmask,
-	.flags		= IRQCHIP_ONESHOT_SAFE,
+	.flags		= IRQCHIP_ONESHOT_SAFE | IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static void xlp_gpio_generic_handler(struct irq_desc *desc)
@@ -272,7 +282,7 @@ static int xlp_gpio_probe(struct platform_device *pdev)
 	spin_lock_init(&priv->lock);
 
 	girq = &gc->irq;
-	girq->chip = &xlp_gpio_irq_chip;
+	gpio_irq_chip_set_chip(girq, &xlp_gpio_irq_chip);
 	girq->parent_handler = xlp_gpio_generic_handler;
 	girq->num_parents = 1;
 	girq->parents = devm_kcalloc(&pdev->dev, 1,
diff --git a/drivers/gpio/gpio-xra1403.c b/drivers/gpio/gpio-xra1403.c
index 49c878cfd5c6..51d6119e1bb4 100644
--- a/drivers/gpio/gpio-xra1403.c
+++ b/drivers/gpio/gpio-xra1403.c
@@ -195,7 +195,7 @@ static const struct spi_device_id xra1403_ids[] = {
 };
 MODULE_DEVICE_TABLE(spi, xra1403_ids);
 
-static const struct of_device_id xra1403_spi_of_match[] = {
+static const struct of_device_id xra1403_spi_of_match[] __maybe_unused = {
 	{ .compatible = "exar,xra1403" },
 	{},
 };
diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c
index 31ae0adbb295..97496c0f9133 100644
--- a/drivers/gpio/gpiolib-acpi.c
+++ b/drivers/gpio/gpiolib-acpi.c
@@ -7,17 +7,19 @@
  *          Mika Westerberg <mika.westerberg@linux.intel.com>
  */
 
+#include <linux/acpi.h>
 #include <linux/dmi.h>
 #include <linux/errno.h>
-#include <linux/gpio/consumer.h>
-#include <linux/gpio/driver.h>
-#include <linux/gpio/machine.h>
 #include <linux/export.h>
-#include <linux/acpi.h>
 #include <linux/interrupt.h>
+#include <linux/irq.h>
 #include <linux/mutex.h>
 #include <linux/pinctrl/pinctrl.h>
 
+#include <linux/gpio/consumer.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/machine.h>
+
 #include "gpiolib.h"
 #include "gpiolib-acpi.h"
 
@@ -126,7 +128,7 @@ static bool acpi_gpio_deferred_req_irqs_done;
 
 static int acpi_gpiochip_find(struct gpio_chip *gc, void *data)
 {
-	return gc->parent && device_match_acpi_handle(gc->parent, data);
+	return ACPI_HANDLE_FWNODE(gc->fwnode) == data;
 }
 
 /**
@@ -646,7 +648,7 @@ static bool acpi_get_driver_gpio_data(struct acpi_device *adev,
 {
 	const struct acpi_gpio_mapping *gm;
 
-	if (!adev->driver_gpios)
+	if (!adev || !adev->driver_gpios)
 		return false;
 
 	for (gm = adev->driver_gpios; gm->name; gm++)
@@ -840,13 +842,10 @@ static int acpi_gpio_property_lookup(struct fwnode_handle *fwnode,
 	ret = __acpi_node_get_property_reference(fwnode, propname, index, 3,
 						 &args);
 	if (ret) {
-		struct acpi_device *adev = to_acpi_device_node(fwnode);
+		struct acpi_device *adev;
 
-		if (!adev)
-			return ret;
-
-		if (!acpi_get_driver_gpio_data(adev, propname, index, &args,
-					       &quirks))
+		adev = to_acpi_device_node(fwnode);
+		if (!acpi_get_driver_gpio_data(adev, propname, index, &args, &quirks))
 			return ret;
 	}
 	/*
@@ -1624,6 +1623,19 @@ static const struct dmi_system_id gpiolib_acpi_quirks[] __initconst = {
 		 * https://gitlab.freedesktop.org/drm/amd/-/issues/1722#note_1720627
 		 */
 		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "NL5xNU"),
+		},
+		.driver_data = &(struct acpi_gpiolib_dmi_quirk) {
+			.ignore_wake = "ELAN0415:00@9",
+		},
+	},
+	{
+		/*
+		 * Spurious wakeups from TP_ATTN# pin
+		 * Found in BIOS 1.7.8
+		 * https://gitlab.freedesktop.org/drm/amd/-/issues/1722#note_1720627
+		 */
+		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "NL5xRU"),
 		},
 		.driver_data = &(struct acpi_gpiolib_dmi_quirk) {
diff --git a/drivers/gpio/gpiolib-acpi.h b/drivers/gpio/gpiolib-acpi.h
index 90fd6b04f24d..0fcd7e14d7f9 100644
--- a/drivers/gpio/gpiolib-acpi.h
+++ b/drivers/gpio/gpiolib-acpi.h
@@ -9,7 +9,6 @@
 #define GPIOLIB_ACPI_H
 
 #include <linux/err.h>
-#include <linux/errno.h>
 #include <linux/types.h>
 
 #include <linux/gpio/consumer.h>
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 266352b1a966..1436cdb5fa26 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -10,14 +10,16 @@
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/errno.h>
-#include <linux/module.h>
 #include <linux/io.h>
-#include <linux/gpio/consumer.h>
+#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_gpio.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/slab.h>
+#include <linux/string.h>
+
+#include <linux/gpio/consumer.h>
 #include <linux/gpio/machine.h>
 
 #include "gpiolib.h"
@@ -892,6 +894,8 @@ static int of_gpio_simple_xlate(struct gpio_chip *gc,
 	return gpiospec->args[0];
 }
 
+#if IS_ENABLED(CONFIG_OF_GPIO_MM_GPIOCHIP)
+#include <linux/gpio/legacy-of-mm-gpiochip.h>
 /**
  * of_mm_gpiochip_add_data - Add memory mapped GPIO chip (bank)
  * @np:		device node of the GPIO chip
@@ -964,6 +968,7 @@ void of_mm_gpiochip_remove(struct of_mm_gpio_chip *mm_gc)
 	kfree(gc->label);
 }
 EXPORT_SYMBOL_GPL(of_mm_gpiochip_remove);
+#endif
 
 #ifdef CONFIG_PINCTRL
 static int of_gpiochip_add_pin_range(struct gpio_chip *chip)
diff --git a/drivers/gpio/gpiolib-of.h b/drivers/gpio/gpiolib-of.h
index e5bb065d82ef..6b3a5347c5d9 100644
--- a/drivers/gpio/gpiolib-of.h
+++ b/drivers/gpio/gpiolib-of.h
@@ -4,7 +4,6 @@
 #define GPIOLIB_OF_H
 
 #include <linux/err.h>
-#include <linux/errno.h>
 #include <linux/types.h>
 
 #include <linux/notifier.h>
diff --git a/drivers/gpio/gpiolib-swnode.c b/drivers/gpio/gpiolib-swnode.c
index dd9ccac214d1..b5a6eaf3729b 100644
--- a/drivers/gpio/gpiolib-swnode.c
+++ b/drivers/gpio/gpiolib-swnode.c
@@ -6,13 +6,14 @@
  */
 #include <linux/err.h>
 #include <linux/errno.h>
-#include <linux/gpio/consumer.h>
-#include <linux/gpio/driver.h>
 #include <linux/kernel.h>
 #include <linux/printk.h>
 #include <linux/property.h>
 #include <linux/string.h>
 
+#include <linux/gpio/consumer.h>
+#include <linux/gpio/driver.h>
+
 #include "gpiolib.h"
 #include "gpiolib-swnode.h"
 
diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c
index cd27bf173dec..530dfd19d7b5 100644
--- a/drivers/gpio/gpiolib-sysfs.c
+++ b/drivers/gpio/gpiolib-sysfs.c
@@ -1,18 +1,29 @@
 // SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bitops.h>
+#include <linux/device.h>
 #include <linux/idr.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/kdev_t.h>
+#include <linux/kstrtox.h>
+#include <linux/list.h>
 #include <linux/mutex.h>
-#include <linux/device.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
 #include <linux/sysfs.h>
+#include <linux/types.h>
+
 #include <linux/gpio/consumer.h>
 #include <linux/gpio/driver.h>
-#include <linux/interrupt.h>
-#include <linux/kdev_t.h>
-#include <linux/slab.h>
-#include <linux/ctype.h>
 
 #include "gpiolib.h"
 #include "gpiolib-sysfs.h"
 
+struct kernfs_node;
+
 #define GPIO_IRQF_TRIGGER_NONE		0
 #define GPIO_IRQF_TRIGGER_FALLING	BIT(0)
 #define GPIO_IRQF_TRIGGER_RISING	BIT(1)
@@ -426,8 +437,8 @@ ATTRIBUTE_GROUPS(gpiochip);
  * /sys/class/gpio/unexport ... write-only
  *	integer N ... number of GPIO to unexport
  */
-static ssize_t export_store(struct class *class,
-				struct class_attribute *attr,
+static ssize_t export_store(const struct class *class,
+				const struct class_attribute *attr,
 				const char *buf, size_t len)
 {
 	long			gpio;
@@ -478,8 +489,8 @@ done:
 }
 static CLASS_ATTR_WO(export);
 
-static ssize_t unexport_store(struct class *class,
-				struct class_attribute *attr,
+static ssize_t unexport_store(const struct class *class,
+				const struct class_attribute *attr,
 				const char *buf, size_t len)
 {
 	long			gpio;
@@ -491,7 +502,7 @@ static ssize_t unexport_store(struct class *class,
 		goto done;
 
 	desc = gpio_to_desc(gpio);
-	/* reject bogus commands (gpio_unexport ignores them) */
+	/* reject bogus commands (gpiod_unexport() ignores them) */
 	if (!desc) {
 		pr_warn("%s: invalid GPIO %ld\n", __func__, gpio);
 		return -EINVAL;
@@ -523,8 +534,6 @@ ATTRIBUTE_GROUPS(gpio_class);
 
 static struct class gpio_class = {
 	.name =		"gpio",
-	.owner =	THIS_MODULE,
-
 	.class_groups = gpio_class_groups,
 };
 
@@ -556,7 +565,7 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change)
 	int			offset;
 
 	/* can't export until sysfs is available ... */
-	if (!gpio_class.p) {
+	if (!class_is_registered(&gpio_class)) {
 		pr_debug("%s: called too early!\n", __func__);
 		return -ENOENT;
 	}
@@ -730,7 +739,7 @@ int gpiochip_sysfs_register(struct gpio_device *gdev)
 	 * register later, in gpiolib_sysfs_init() ... here we just
 	 * verify that _some_ field of gpio_class got initialized.
 	 */
-	if (!gpio_class.p)
+	if (!class_is_registered(&gpio_class))
 		return 0;
 
 	/*
@@ -790,7 +799,7 @@ static int __init gpiolib_sysfs_init(void)
 	 * early (e.g. before the class_register above was called).
 	 *
 	 * We run before arch_initcall() so chip->dev nodes can have
-	 * registered, and so arch_initcall() can always gpio_export().
+	 * registered, and so arch_initcall() can always gpiod_export().
 	 */
 	spin_lock_irqsave(&gpio_lock, flags);
 	list_for_each_entry(gdev, &gpio_devices, list) {
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index 19bd23044b01..04fb05df805b 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -6,22 +6,25 @@
 #include <linux/debugfs.h>
 #include <linux/device.h>
 #include <linux/err.h>
+#include <linux/errno.h>
 #include <linux/file.h>
 #include <linux/fs.h>
-#include <linux/gpio.h>
-#include <linux/gpio/driver.h>
-#include <linux/gpio/machine.h>
 #include <linux/idr.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 
+#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/machine.h>
+
 #include <uapi/linux/gpio.h>
 
 #include "gpiolib-acpi.h"
@@ -58,7 +61,20 @@
 static DEFINE_IDA(gpio_ida);
 static dev_t gpio_devt;
 #define GPIO_DEV_MAX 256 /* 256 GPIO chip devices supported */
-static int gpio_bus_match(struct device *dev, struct device_driver *drv);
+
+static int gpio_bus_match(struct device *dev, struct device_driver *drv)
+{
+	struct fwnode_handle *fwnode = dev_fwnode(dev);
+
+	/*
+	 * Only match if the fwnode doesn't already have a proper struct device
+	 * created for it.
+	 */
+	if (fwnode && fwnode->dev != dev)
+		return 0;
+	return 1;
+}
+
 static struct bus_type gpio_bus_type = {
 	.name = "gpio",
 	.match = gpio_bus_match,
@@ -357,7 +373,7 @@ static int gpiochip_set_desc_names(struct gpio_chip *gc)
 }
 
 /*
- * devprop_gpiochip_set_names - Set GPIO line names using device properties
+ * gpiochip_set_names - Set GPIO line names using device properties
  * @chip: GPIO chip whose lines should be named, if possible
  *
  * Looks for device property "gpio-line-names" and if it exists assigns
@@ -365,7 +381,7 @@ static int gpiochip_set_desc_names(struct gpio_chip *gc)
  * names belong to the underlying firmware node and should not be released
  * by the caller.
  */
-static int devprop_gpiochip_set_names(struct gpio_chip *chip)
+static int gpiochip_set_names(struct gpio_chip *chip)
 {
 	struct gpio_device *gdev = chip->gpiodev;
 	struct device *dev = &gdev->dev;
@@ -556,7 +572,7 @@ bool gpiochip_line_is_valid(const struct gpio_chip *gc,
 }
 EXPORT_SYMBOL_GPL(gpiochip_line_is_valid);
 
-static void gpiodevice_release(struct device *dev)
+static void gpiodev_release(struct device *dev)
 {
 	struct gpio_device *gdev = to_gpio_device(dev);
 	unsigned long flags;
@@ -585,21 +601,22 @@ static void gpiodevice_release(struct device *dev)
 
 static int gpiochip_setup_dev(struct gpio_device *gdev)
 {
+	struct fwnode_handle *fwnode = dev_fwnode(&gdev->dev);
 	int ret;
 
 	/*
 	 * If fwnode doesn't belong to another device, it's safe to clear its
 	 * initialized flag.
 	 */
-	if (gdev->dev.fwnode && !gdev->dev.fwnode->dev)
-		fwnode_dev_initialized(gdev->dev.fwnode, false);
+	if (fwnode && !fwnode->dev)
+		fwnode_dev_initialized(fwnode, false);
 
 	ret = gcdev_register(gdev, gpio_devt);
 	if (ret)
 		return ret;
 
 	/* From this point, the .release() function cleans up gpio_device */
-	gdev->dev.release = gpiodevice_release;
+	gdev->dev.release = gpiodev_release;
 
 	ret = gpiochip_sysfs_register(gdev);
 	if (ret)
@@ -663,11 +680,28 @@ static void gpiochip_setup_devs(void)
 	}
 }
 
+static void gpiochip_set_data(struct gpio_chip *gc, void *data)
+{
+	gc->gpiodev->data = data;
+}
+
+/**
+ * gpiochip_get_data() - get per-subdriver data for the chip
+ * @gc: GPIO chip
+ *
+ * Returns:
+ * The per-subdriver data for the chip.
+ */
+void *gpiochip_get_data(struct gpio_chip *gc)
+{
+	return gc->gpiodev->data;
+}
+EXPORT_SYMBOL_GPL(gpiochip_get_data);
+
 int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 			       struct lock_class_key *lock_key,
 			       struct lock_class_key *request_key)
 {
-	struct fwnode_handle *fwnode = NULL;
 	struct gpio_device *gdev;
 	unsigned long flags;
 	unsigned int i;
@@ -675,12 +709,12 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 	int base = 0;
 	int ret = 0;
 
-	/* If the calling driver did not initialize firmware node, do it here */
-	if (gc->fwnode)
-		fwnode = gc->fwnode;
-	else if (gc->parent)
-		fwnode = dev_fwnode(gc->parent);
-	gc->fwnode = fwnode;
+	/*
+	 * If the calling driver did not initialize firmware node, do it here
+	 * using the parent device, if any.
+	 */
+	if (!gc->fwnode && gc->parent)
+		gc->fwnode = dev_fwnode(gc->parent);
 
 	/*
 	 * First: allocate and populate the internal stat container, and
@@ -692,7 +726,9 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 	gdev->dev.bus = &gpio_bus_type;
 	gdev->dev.parent = gc->parent;
 	gdev->chip = gc;
+
 	gc->gpiodev = gdev;
+	gpiochip_set_data(gc, data);
 
 	device_set_node(&gdev->dev, gc->fwnode);
 
@@ -759,7 +795,6 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 	}
 
 	gdev->ngpio = gc->ngpio;
-	gdev->data = data;
 
 	spin_lock_irqsave(&gpio_lock, flags);
 
@@ -816,7 +851,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 		if (ret)
 			goto err_remove_from_list;
 	}
-	ret = devprop_gpiochip_set_names(gc);
+	ret = gpiochip_set_names(gc);
 	if (ret)
 		goto err_remove_from_list;
 
@@ -922,19 +957,6 @@ err_print_message:
 EXPORT_SYMBOL_GPL(gpiochip_add_data_with_key);
 
 /**
- * gpiochip_get_data() - get per-subdriver data for the chip
- * @gc: GPIO chip
- *
- * Returns:
- * The per-subdriver data for the chip.
- */
-void *gpiochip_get_data(struct gpio_chip *gc)
-{
-	return gc->gpiodev->data;
-}
-EXPORT_SYMBOL_GPL(gpiochip_get_data);
-
-/**
  * gpiochip_remove() - unregister a gpio_chip
  * @gc: the chip to unregister
  *
@@ -960,9 +982,9 @@ void gpiochip_remove(struct gpio_chip *gc)
 	gpiochip_free_valid_mask(gc);
 	/*
 	 * We accept no more calls into the driver from this point, so
-	 * NULL the driver data pointer
+	 * NULL the driver data pointer.
 	 */
-	gdev->data = NULL;
+	gpiochip_set_data(gc, NULL);
 
 	spin_lock_irqsave(&gpio_lock, flags);
 	for (i = 0; i < gdev->ngpio; i++) {
@@ -1201,7 +1223,7 @@ static int gpiochip_hierarchy_irq_domain_alloc(struct irq_domain *d,
 	if (ret)
 		return ret;
 
-	chip_dbg(gc, "allocate IRQ %d, hwirq %lu\n", irq,  hwirq);
+	chip_dbg(gc, "allocate IRQ %d, hwirq %lu\n", irq, hwirq);
 
 	ret = girq->child_to_parent_hwirq(gc, hwirq, type,
 					  &parent_hwirq, &parent_type);
@@ -1369,8 +1391,7 @@ static bool gpiochip_hierarchy_is_hierarchical(struct gpio_chip *gc)
  * gpiochip by assigning the gpiochip as chip data, and using the irqchip
  * stored inside the gpiochip.
  */
-int gpiochip_irq_map(struct irq_domain *d, unsigned int irq,
-		     irq_hw_number_t hwirq)
+int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwirq)
 {
 	struct gpio_chip *gc = d->host_data;
 	int ret = 0;
@@ -1446,8 +1467,9 @@ int gpiochip_irq_domain_activate(struct irq_domain *domain,
 				 struct irq_data *data, bool reserve)
 {
 	struct gpio_chip *gc = domain->host_data;
+	unsigned int hwirq = irqd_to_hwirq(data);
 
-	return gpiochip_lock_as_irq(gc, data->hwirq);
+	return gpiochip_lock_as_irq(gc, hwirq);
 }
 EXPORT_SYMBOL_GPL(gpiochip_irq_domain_activate);
 
@@ -1464,8 +1486,9 @@ void gpiochip_irq_domain_deactivate(struct irq_domain *domain,
 				    struct irq_data *data)
 {
 	struct gpio_chip *gc = domain->host_data;
+	unsigned int hwirq = irqd_to_hwirq(data);
 
-	return gpiochip_unlock_as_irq(gc, data->hwirq);
+	return gpiochip_unlock_as_irq(gc, hwirq);
 }
 EXPORT_SYMBOL_GPL(gpiochip_irq_domain_deactivate);
 
@@ -1505,33 +1528,37 @@ static int gpiochip_to_irq(struct gpio_chip *gc, unsigned int offset)
 int gpiochip_irq_reqres(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	unsigned int hwirq = irqd_to_hwirq(d);
 
-	return gpiochip_reqres_irq(gc, d->hwirq);
+	return gpiochip_reqres_irq(gc, hwirq);
 }
 EXPORT_SYMBOL(gpiochip_irq_reqres);
 
 void gpiochip_irq_relres(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	unsigned int hwirq = irqd_to_hwirq(d);
 
-	gpiochip_relres_irq(gc, d->hwirq);
+	gpiochip_relres_irq(gc, hwirq);
 }
 EXPORT_SYMBOL(gpiochip_irq_relres);
 
 static void gpiochip_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	unsigned int hwirq = irqd_to_hwirq(d);
 
 	if (gc->irq.irq_mask)
 		gc->irq.irq_mask(d);
-	gpiochip_disable_irq(gc, d->hwirq);
+	gpiochip_disable_irq(gc, hwirq);
 }
 
 static void gpiochip_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	unsigned int hwirq = irqd_to_hwirq(d);
 
-	gpiochip_enable_irq(gc, d->hwirq);
+	gpiochip_enable_irq(gc, hwirq);
 	if (gc->irq.irq_unmask)
 		gc->irq.irq_unmask(d);
 }
@@ -1539,17 +1566,19 @@ static void gpiochip_irq_unmask(struct irq_data *d)
 static void gpiochip_irq_enable(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	unsigned int hwirq = irqd_to_hwirq(d);
 
-	gpiochip_enable_irq(gc, d->hwirq);
+	gpiochip_enable_irq(gc, hwirq);
 	gc->irq.irq_enable(d);
 }
 
 static void gpiochip_irq_disable(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	unsigned int hwirq = irqd_to_hwirq(d);
 
 	gc->irq.irq_disable(d);
-	gpiochip_disable_irq(gc, d->hwirq);
+	gpiochip_disable_irq(gc, hwirq);
 }
 
 static void gpiochip_set_irq_hooks(struct gpio_chip *gc)
@@ -3909,13 +3938,10 @@ static struct gpio_desc *gpiod_find_and_request(struct device *consumer,
 						bool platform_lookup_allowed)
 {
 	unsigned long lookupflags = GPIO_LOOKUP_FLAGS_DEFAULT;
-	struct gpio_desc *desc = ERR_PTR(-ENOENT);
+	struct gpio_desc *desc;
 	int ret;
 
-	if (!IS_ERR_OR_NULL(fwnode))
-		desc = gpiod_find_by_fwnode(fwnode, consumer, con_id, idx,
-					    &flags, &lookupflags);
-
+	desc = gpiod_find_by_fwnode(fwnode, consumer, con_id, idx, &flags, &lookupflags);
 	if (gpiod_not_found(desc) && platform_lookup_allowed) {
 		/*
 		 * Either we are not using DT or ACPI, or their lookup did not
@@ -4256,16 +4282,18 @@ struct gpio_descs *__must_check gpiod_get_array(struct device *dev,
 	struct gpio_array *array_info = NULL;
 	struct gpio_chip *gc;
 	int count, bitmap_size;
+	size_t descs_size;
 
 	count = gpiod_count(dev, con_id);
 	if (count < 0)
 		return ERR_PTR(count);
 
-	descs = kzalloc(struct_size(descs, desc, count), GFP_KERNEL);
+	descs_size = struct_size(descs, desc, count);
+	descs = kzalloc(descs_size, GFP_KERNEL);
 	if (!descs)
 		return ERR_PTR(-ENOMEM);
 
-	for (descs->ndescs = 0; descs->ndescs < count; ) {
+	for (descs->ndescs = 0; descs->ndescs < count; descs->ndescs++) {
 		desc = gpiod_get_index(dev, con_id, descs->ndescs, flags);
 		if (IS_ERR(desc)) {
 			gpiod_put_array(descs);
@@ -4285,20 +4313,17 @@ struct gpio_descs *__must_check gpiod_get_array(struct device *dev,
 			bitmap_size = BITS_TO_LONGS(gc->ngpio > count ?
 						    gc->ngpio : count);
 
-			array = kzalloc(struct_size(descs, desc, count) +
-					struct_size(array_info, invert_mask,
-					3 * bitmap_size), GFP_KERNEL);
+			array = krealloc(descs, descs_size +
+					 struct_size(array_info, invert_mask, 3 * bitmap_size),
+					 GFP_KERNEL | __GFP_ZERO);
 			if (!array) {
 				gpiod_put_array(descs);
 				return ERR_PTR(-ENOMEM);
 			}
 
-			memcpy(array, descs,
-			       struct_size(descs, desc, descs->ndescs + 1));
-			kfree(descs);
-
 			descs = array;
-			array_info = (void *)(descs->desc + count);
+
+			array_info = (void *)descs + descs_size;
 			array_info->get_mask = array_info->invert_mask +
 						  bitmap_size;
 			array_info->set_mask = array_info->get_mask +
@@ -4313,8 +4338,13 @@ struct gpio_descs *__must_check gpiod_get_array(struct device *dev,
 				   count - descs->ndescs);
 			descs->info = array_info;
 		}
+
+		/* If there is no cache for fast bitmap processing path, continue */
+		if (!array_info)
+			continue;
+
 		/* Unmark array members which don't belong to the 'fast' chip */
-		if (array_info && array_info->chip != gc) {
+		if (array_info->chip != gc) {
 			__clear_bit(descs->ndescs, array_info->get_mask);
 			__clear_bit(descs->ndescs, array_info->set_mask);
 		}
@@ -4322,8 +4352,7 @@ struct gpio_descs *__must_check gpiod_get_array(struct device *dev,
 		 * Detect array members which belong to the 'fast' chip
 		 * but their pins are not in hardware order.
 		 */
-		else if (array_info &&
-			   gpio_chip_hwgpio(desc) != descs->ndescs) {
+		else if (gpio_chip_hwgpio(desc) != descs->ndescs) {
 			/*
 			 * Don't use fast path if all array members processed so
 			 * far belong to the same chip as this one but its pin
@@ -4337,7 +4366,7 @@ struct gpio_descs *__must_check gpiod_get_array(struct device *dev,
 				__clear_bit(descs->ndescs,
 					    array_info->set_mask);
 			}
-		} else if (array_info) {
+		} else {
 			/* Exclude open drain or open source from fast output */
 			if (gpiochip_line_is_open_drain(gc, descs->ndescs) ||
 			    gpiochip_line_is_open_source(gc, descs->ndescs))
@@ -4348,8 +4377,6 @@ struct gpio_descs *__must_check gpiod_get_array(struct device *dev,
 				__set_bit(descs->ndescs,
 					  array_info->invert_mask);
 		}
-
-		descs->ndescs++;
 	}
 	if (array_info)
 		dev_dbg(dev,
@@ -4413,20 +4440,6 @@ void gpiod_put_array(struct gpio_descs *descs)
 }
 EXPORT_SYMBOL_GPL(gpiod_put_array);
 
-
-static int gpio_bus_match(struct device *dev, struct device_driver *drv)
-{
-	struct fwnode_handle *fwnode = dev_fwnode(dev);
-
-	/*
-	 * Only match if the fwnode doesn't already have a proper struct device
-	 * created for it.
-	 */
-	if (fwnode && fwnode->dev != dev)
-		return 0;
-	return 1;
-}
-
 static int gpio_stub_drv_probe(struct device *dev)
 {
 	/*
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index dc0f94f02a82..ba3fb04bb691 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -10,13 +10,13 @@ menuconfig DRM
 	depends on (AGP || AGP=n) && !EMULATED_CMPXCHG && HAS_DMA
 	select DRM_PANEL_ORIENTATION_QUIRKS
 	select HDMI
-	select FB_CMDLINE
 	select I2C
 	select DMA_SHARED_BUFFER
 	select SYNC_FILE
 # gallium uses SYS_kcmp for os_same_file_description() to de-duplicate
 # device and dmabuf fd. Let's make sure that is available for our userspace.
 	select KCMP
+	select VIDEO_CMDLINE
 	select VIDEO_NOMODESET
 	help
 	  Kernel-level support for the Direct Rendering Infrastructure (DRI)
@@ -232,6 +232,10 @@ config DRM_GEM_SHMEM_HELPER
 	help
 	  Choose this if you need the GEM shmem helper functions
 
+config DRM_SUBALLOC_HELPER
+	tristate
+	depends on DRM
+
 config DRM_SCHED
 	tristate
 	depends on DRM
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index ab4460fcd63f..a33257d2bc7f 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -82,12 +82,16 @@ obj-$(CONFIG_DRM_PANEL_ORIENTATION_QUIRKS) += drm_panel_orientation_quirks.o
 obj-$(CONFIG_DRM_BUDDY) += drm_buddy.o
 
 drm_dma_helper-y := drm_gem_dma_helper.o
+drm_dma_helper-$(CONFIG_DRM_FBDEV_EMULATION) += drm_fbdev_dma.o
 drm_dma_helper-$(CONFIG_DRM_KMS_HELPER) += drm_fb_dma_helper.o
 obj-$(CONFIG_DRM_GEM_DMA_HELPER) += drm_dma_helper.o
 
 drm_shmem_helper-y := drm_gem_shmem_helper.o
 obj-$(CONFIG_DRM_GEM_SHMEM_HELPER) += drm_shmem_helper.o
 
+drm_suballoc_helper-y := drm_suballoc.o
+obj-$(CONFIG_DRM_SUBALLOC_HELPER) += drm_suballoc_helper.o
+
 drm_vram_helper-y := drm_gem_vram_helper.o
 obj-$(CONFIG_DRM_VRAM_HELPER) += drm_vram_helper.o
 
diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig
index a82d36ea88e2..12adca8c7819 100644
--- a/drivers/gpu/drm/amd/amdgpu/Kconfig
+++ b/drivers/gpu/drm/amd/amdgpu/Kconfig
@@ -3,6 +3,7 @@
 config DRM_AMDGPU
 	tristate "AMD GPU"
 	depends on DRM && PCI && MMU
+	depends on !UML
 	select FW_LOADER
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HDMI_HELPER
@@ -19,6 +20,7 @@ config DRM_AMDGPU
 	select BACKLIGHT_CLASS_DEVICE
 	select INTERVAL_TREE
 	select DRM_BUDDY
+	select DRM_SUBALLOC_HELPER
 	# amdgpu depends on ACPI_VIDEO when ACPI is enabled, for select to work
 	# ACPI_VIDEO's dependencies must also be selected.
 	select INPUT if ACPI
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 1d72cbc85348..415a7fa395c4 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -54,7 +54,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
 	amdgpu_gtt_mgr.o amdgpu_preempt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o \
 	amdgpu_atomfirmware.o amdgpu_vf_error.o amdgpu_sched.o \
-	amdgpu_debugfs.o amdgpu_ids.o amdgpu_gmc.o \
+	amdgpu_debugfs.o amdgpu_ids.o amdgpu_gmc.o amdgpu_mmhub.o amdgpu_hdp.o \
 	amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \
 	amdgpu_vm_sdma.o amdgpu_discovery.o amdgpu_ras_eeprom.o amdgpu_nbio.o \
 	amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \
@@ -77,7 +77,8 @@ amdgpu-y += \
 	vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o \
 	vega20_reg_init.o nbio_v7_4.o nbio_v2_3.o nv.o arct_reg_init.o mxgpu_nv.o \
 	nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o \
-	sienna_cichlid.o smu_v13_0_10.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o
+	sienna_cichlid.o smu_v13_0_10.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o \
+	nbio_v7_9.o
 
 # add DF block
 amdgpu-y += \
@@ -92,7 +93,7 @@ amdgpu-y += \
 	gfxhub_v1_0.o mmhub_v1_0.o gmc_v9_0.o gfxhub_v1_1.o mmhub_v9_4.o \
 	gfxhub_v2_0.o mmhub_v2_0.o gmc_v10_0.o gfxhub_v2_1.o mmhub_v2_3.o \
 	mmhub_v1_7.o gfxhub_v3_0.o mmhub_v3_0.o mmhub_v3_0_2.o gmc_v11_0.o \
-	mmhub_v3_0_1.o gfxhub_v3_0_3.o
+	mmhub_v3_0_1.o gfxhub_v3_0_3.o gfxhub_v1_2.o mmhub_v1_8.o
 
 # add UMC block
 amdgpu-y += \
@@ -135,6 +136,7 @@ amdgpu-y += \
 	gfx_v9_0.o \
 	gfx_v9_4.o \
 	gfx_v9_4_2.o \
+	gfx_v9_4_3.o \
 	gfx_v10_0.o \
 	imu_v11_0.o \
 	gfx_v11_0.o \
@@ -148,6 +150,7 @@ amdgpu-y += \
 	sdma_v3_0.o \
 	sdma_v4_0.o \
 	sdma_v4_4.o \
+	sdma_v4_4_2.o \
 	sdma_v5_0.o \
 	sdma_v5_2.o \
 	sdma_v6_0.o
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 39018f784f9c..02b827785e39 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -50,7 +50,6 @@
 #include <linux/hashtable.h>
 #include <linux/dma-fence.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 
 #include <drm/ttm/ttm_bo.h>
 #include <drm/ttm/ttm_placement.h>
@@ -186,7 +185,6 @@ extern char *amdgpu_disable_cu;
 extern char *amdgpu_virtual_display;
 extern uint amdgpu_pp_feature_mask;
 extern uint amdgpu_force_long_training;
-extern int amdgpu_job_hang_limit;
 extern int amdgpu_lbpw;
 extern int amdgpu_compute_multipipe;
 extern int amdgpu_gpu_recovery;
@@ -424,29 +422,11 @@ struct amdgpu_clock {
  * alignment).
  */
 
-#define AMDGPU_SA_NUM_FENCE_LISTS	32
-
 struct amdgpu_sa_manager {
-	wait_queue_head_t	wq;
-	struct amdgpu_bo	*bo;
-	struct list_head	*hole;
-	struct list_head	flist[AMDGPU_SA_NUM_FENCE_LISTS];
-	struct list_head	olist;
-	unsigned		size;
-	uint64_t		gpu_addr;
-	void			*cpu_ptr;
-	uint32_t		domain;
-	uint32_t		align;
-};
-
-/* sub-allocation buffer */
-struct amdgpu_sa_bo {
-	struct list_head		olist;
-	struct list_head		flist;
-	struct amdgpu_sa_manager	*manager;
-	unsigned			soffset;
-	unsigned			eoffset;
-	struct dma_fence	        *fence;
+	struct drm_suballoc_manager	base;
+	struct amdgpu_bo		*bo;
+	uint64_t			gpu_addr;
+	void				*cpu_ptr;
 };
 
 int amdgpu_fence_slab_init(void);
@@ -490,7 +470,7 @@ int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv);
 /*
  * Writeback
  */
-#define AMDGPU_MAX_WB 256	/* Reserve at most 256 WB slots for amdgpu-owned rings. */
+#define AMDGPU_MAX_WB 1024	/* Reserve at most 1024 WB slots for amdgpu-owned rings. */
 
 struct amdgpu_wb {
 	struct amdgpu_bo	*wb_obj;
@@ -1023,7 +1003,6 @@ struct amdgpu_device {
 	bool                            in_runpm;
 	bool                            has_pr3;
 
-	bool                            pm_sysfs_en;
 	bool                            ucode_sysfs_en;
 	bool                            psp_sysfs_en;
 
@@ -1113,18 +1092,14 @@ void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset);
 
 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
-				u32 pcie_index, u32 pcie_data,
 				u32 reg_addr);
 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
-				  u32 pcie_index, u32 pcie_data,
 				  u32 reg_addr);
 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
-				 u32 pcie_index, u32 pcie_data,
 				 u32 reg_addr, u32 reg_data);
 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
-				   u32 pcie_index, u32 pcie_data,
 				   u32 reg_addr, u64 reg_data);
-
+u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev);
 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type);
 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev);
 
@@ -1246,7 +1221,7 @@ int emu_soc_asic_init(struct amdgpu_device *adev);
 	((adev)->asic_funcs->flush_hdp ? (adev)->asic_funcs->flush_hdp((adev), (r)) : (adev)->hdp.funcs->flush_hdp((adev), (r)))
 #define amdgpu_asic_invalidate_hdp(adev, r) \
 	((adev)->asic_funcs->invalidate_hdp ? (adev)->asic_funcs->invalidate_hdp((adev), (r)) : \
-	 ((adev)->hdp.funcs->invalidate_hdp ? (adev)->hdp.funcs->invalidate_hdp((adev), (r)) : 0))
+	 ((adev)->hdp.funcs->invalidate_hdp ? (adev)->hdp.funcs->invalidate_hdp((adev), (r)) : (void)0))
 #define amdgpu_asic_need_full_reset(adev) (adev)->asic_funcs->need_full_reset((adev))
 #define amdgpu_asic_init_doorbell_index(adev) (adev)->asic_funcs->init_doorbell_index((adev))
 #define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1)))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index f99d4873bf22..0385f7f69278 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -96,7 +96,7 @@ static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 					 size_t *start_offset)
 {
 	/*
-	 * The first num_doorbells are used by amdgpu.
+	 * The first num_kernel_doorbells are used by amdgpu.
 	 * amdkfd takes whatever's left in the aperture.
 	 */
 	if (adev->enable_mes) {
@@ -109,11 +109,11 @@ static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 		*aperture_base = adev->doorbell.base;
 		*aperture_size = 0;
 		*start_offset = 0;
-	} else if (adev->doorbell.size > adev->doorbell.num_doorbells *
+	} else if (adev->doorbell.size > adev->doorbell.num_kernel_doorbells *
 						sizeof(u32)) {
 		*aperture_base = adev->doorbell.base;
 		*aperture_size = adev->doorbell.size;
-		*start_offset = adev->doorbell.num_doorbells * sizeof(u32);
+		*start_offset = adev->doorbell.num_kernel_doorbells * sizeof(u32);
 	} else {
 		*aperture_base = 0;
 		*aperture_size = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 333780491867..01ba3589b60a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -308,6 +308,8 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
 				      uint64_t va, void *drm_priv,
 				      struct kgd_mem **mem, uint64_t *size,
 				      uint64_t *mmap_offset);
+int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem,
+				      struct dma_buf **dmabuf);
 int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
 				struct tile_config *config);
 void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index d6320c836251..83a83ced2439 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -82,6 +82,25 @@ static bool kfd_mem_is_attached(struct amdgpu_vm *avm,
 	return false;
 }
 
+/**
+ * reuse_dmamap() - Check whether adev can share the original
+ * userptr BO
+ *
+ * If both adev and bo_adev are in direct mapping or
+ * in the same iommu group, they can share the original BO.
+ *
+ * @adev: Device to which can or cannot share the original BO
+ * @bo_adev: Device to which allocated BO belongs to
+ *
+ * Return: returns true if adev can share original userptr BO,
+ * false otherwise.
+ */
+static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev)
+{
+	return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) ||
+			(adev->dev->iommu_group == bo_adev->dev->iommu_group);
+}
+
 /* Set memory usage limits. Current, limits are
  *  System (TTM + userptr) memory - 15/16th System RAM
  *  TTM memory - 3/8th System RAM
@@ -253,15 +272,19 @@ create_dmamap_sg_bo(struct amdgpu_device *adev,
 		 struct kgd_mem *mem, struct amdgpu_bo **bo_out)
 {
 	struct drm_gem_object *gem_obj;
-	int ret, align;
+	int ret;
+	uint64_t flags = 0;
 
 	ret = amdgpu_bo_reserve(mem->bo, false);
 	if (ret)
 		return ret;
 
-	align = 1;
-	ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
-			AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)
+		flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
+					AMDGPU_GEM_CREATE_UNCACHED);
+
+	ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1,
+			AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags,
 			ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
 
 	amdgpu_bo_unreserve(mem->bo);
@@ -481,9 +504,6 @@ kfd_mem_dmamap_userptr(struct kgd_mem *mem,
 	if (unlikely(ret))
 		goto release_sg;
 
-	drm_prime_sg_to_dma_addr_array(ttm->sg, ttm->dma_address,
-				       ttm->num_pages);
-
 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (ret)
@@ -711,6 +731,21 @@ kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
 	}
 }
 
+static int kfd_mem_export_dmabuf(struct kgd_mem *mem)
+{
+	if (!mem->dmabuf) {
+		struct dma_buf *ret = amdgpu_gem_prime_export(
+			&mem->bo->tbo.base,
+			mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+				DRM_RDWR : 0);
+		if (IS_ERR(ret))
+			return PTR_ERR(ret);
+		mem->dmabuf = ret;
+	}
+
+	return 0;
+}
+
 static int
 kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
 		      struct amdgpu_bo **bo)
@@ -718,16 +753,9 @@ kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
 	struct drm_gem_object *gobj;
 	int ret;
 
-	if (!mem->dmabuf) {
-		mem->dmabuf = amdgpu_gem_prime_export(&mem->bo->tbo.base,
-			mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
-				DRM_RDWR : 0);
-		if (IS_ERR(mem->dmabuf)) {
-			ret = PTR_ERR(mem->dmabuf);
-			mem->dmabuf = NULL;
-			return ret;
-		}
-	}
+	ret = kfd_mem_export_dmabuf(mem);
+	if (ret)
+		return ret;
 
 	gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf);
 	if (IS_ERR(gobj))
@@ -797,11 +825,11 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
 			 va + bo_size, vm);
 
 		if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
-		    (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
-		    same_hive) {
+		    (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) ||
+			same_hive) {
 			/* Mappings on the local GPU, or VRAM mappings in the
-			 * local hive, or userptr mapping IOMMU direct map mode
-			 * share the original BO
+			 * local hive, or userptr mapping can reuse dma map
+			 * address space share the original BO
 			 */
 			attachment[i]->type = KFD_MEM_ATT_SHARED;
 			bo[i] = mem->bo;
@@ -1575,7 +1603,7 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
 {
 	uint64_t reserved_for_pt =
 		ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
-	size_t available;
+	ssize_t available;
 
 	spin_lock(&kfd_mem_limit.mem_limit_lock);
 	available = adev->gmc.real_vram_size
@@ -1584,6 +1612,9 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
 		- reserved_for_pt;
 	spin_unlock(&kfd_mem_limit.mem_limit_lock);
 
+	if (available < 0)
+		available = 0;
+
 	return ALIGN_DOWN(available, VRAM_AVAILABLITY_ALIGN);
 }
 
@@ -2210,30 +2241,27 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
 	struct amdgpu_bo *bo;
 	int ret;
 
-	if (dma_buf->ops != &amdgpu_dmabuf_ops)
-		/* Can't handle non-graphics buffers */
-		return -EINVAL;
-
-	obj = dma_buf->priv;
-	if (drm_to_adev(obj->dev) != adev)
-		/* Can't handle buffers from other devices */
-		return -EINVAL;
+	obj = amdgpu_gem_prime_import(adev_to_drm(adev), dma_buf);
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
 
 	bo = gem_to_amdgpu_bo(obj);
 	if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
-				    AMDGPU_GEM_DOMAIN_GTT)))
+				    AMDGPU_GEM_DOMAIN_GTT))) {
 		/* Only VRAM and GTT BOs are supported */
-		return -EINVAL;
+		ret = -EINVAL;
+		goto err_put_obj;
+	}
 
 	*mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
-	if (!*mem)
-		return -ENOMEM;
+	if (!*mem) {
+		ret = -ENOMEM;
+		goto err_put_obj;
+	}
 
 	ret = drm_vma_node_allow(&obj->vma_node, drm_priv);
-	if (ret) {
-		kfree(*mem);
-		return ret;
-	}
+	if (ret)
+		goto err_free_mem;
 
 	if (size)
 		*size = amdgpu_bo_size(bo);
@@ -2250,7 +2278,8 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
 		| KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE
 		| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
 
-	drm_gem_object_get(&bo->tbo.base);
+	get_dma_buf(dma_buf);
+	(*mem)->dmabuf = dma_buf;
 	(*mem)->bo = bo;
 	(*mem)->va = va;
 	(*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
@@ -2262,6 +2291,29 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
 	(*mem)->is_imported = true;
 
 	return 0;
+
+err_free_mem:
+	kfree(*mem);
+err_put_obj:
+	drm_gem_object_put(obj);
+	return ret;
+}
+
+int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem,
+				      struct dma_buf **dma_buf)
+{
+	int ret;
+
+	mutex_lock(&mem->lock);
+	ret = kfd_mem_export_dmabuf(mem);
+	if (ret)
+		goto out;
+
+	get_dma_buf(mem->dmabuf);
+	*dma_buf = mem->dmabuf;
+out:
+	mutex_unlock(&mem->lock);
+	return ret;
 }
 
 /* Evict a userptr BO by stopping the queues if necessary
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 3d98fc2ad36b..a2292acf06d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -35,6 +35,7 @@
 #include <linux/devcoredump.h>
 #include <generated/utsrelease.h>
 #include <linux/pci-p2pdma.h>
+#include <linux/apple-gmux.h>
 
 #include <drm/drm_aperture.h>
 #include <drm/drm_atomic_helper.h>
@@ -601,7 +602,7 @@ u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 	if (amdgpu_device_skip_hw_access(adev))
 		return 0;
 
-	if (index < adev->doorbell.num_doorbells) {
+	if (index < adev->doorbell.num_kernel_doorbells) {
 		return readl(adev->doorbell.ptr + index);
 	} else {
 		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
@@ -624,7 +625,7 @@ void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 	if (amdgpu_device_skip_hw_access(adev))
 		return;
 
-	if (index < adev->doorbell.num_doorbells) {
+	if (index < adev->doorbell.num_kernel_doorbells) {
 		writel(v, adev->doorbell.ptr + index);
 	} else {
 		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
@@ -645,7 +646,7 @@ u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 	if (amdgpu_device_skip_hw_access(adev))
 		return 0;
 
-	if (index < adev->doorbell.num_doorbells) {
+	if (index < adev->doorbell.num_kernel_doorbells) {
 		return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 	} else {
 		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
@@ -668,7 +669,7 @@ void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 	if (amdgpu_device_skip_hw_access(adev))
 		return;
 
-	if (index < adev->doorbell.num_doorbells) {
+	if (index < adev->doorbell.num_kernel_doorbells) {
 		atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 	} else {
 		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
@@ -679,20 +680,20 @@ void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
  * amdgpu_device_indirect_rreg - read an indirect register
  *
  * @adev: amdgpu_device pointer
- * @pcie_index: mmio register offset
- * @pcie_data: mmio register offset
  * @reg_addr: indirect register address to read from
  *
  * Returns the value of indirect register @reg_addr
  */
 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
-				u32 pcie_index, u32 pcie_data,
 				u32 reg_addr)
 {
-	unsigned long flags;
-	u32 r;
+	unsigned long flags, pcie_index, pcie_data;
 	void __iomem *pcie_index_offset;
 	void __iomem *pcie_data_offset;
+	u32 r;
+
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 
 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
@@ -710,20 +711,20 @@ u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
  *
  * @adev: amdgpu_device pointer
- * @pcie_index: mmio register offset
- * @pcie_data: mmio register offset
  * @reg_addr: indirect register address to read from
  *
  * Returns the value of indirect register @reg_addr
  */
 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
-				  u32 pcie_index, u32 pcie_data,
 				  u32 reg_addr)
 {
-	unsigned long flags;
-	u64 r;
+	unsigned long flags, pcie_index, pcie_data;
 	void __iomem *pcie_index_offset;
 	void __iomem *pcie_data_offset;
+	u64 r;
+
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 
 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
@@ -753,13 +754,15 @@ u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
  *
  */
 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
-				 u32 pcie_index, u32 pcie_data,
 				 u32 reg_addr, u32 reg_data)
 {
-	unsigned long flags;
+	unsigned long flags, pcie_index, pcie_data;
 	void __iomem *pcie_index_offset;
 	void __iomem *pcie_data_offset;
 
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
+
 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
@@ -782,13 +785,15 @@ void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
  *
  */
 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
-				   u32 pcie_index, u32 pcie_data,
 				   u32 reg_addr, u64 reg_data)
 {
-	unsigned long flags;
+	unsigned long flags, pcie_index, pcie_data;
 	void __iomem *pcie_index_offset;
 	void __iomem *pcie_data_offset;
 
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
+
 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
@@ -807,6 +812,18 @@ void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 }
 
 /**
+ * amdgpu_device_get_rev_id - query device rev_id
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Return device rev_id
+ */
+u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
+{
+	return adev->nbio.funcs->get_rev_id(adev);
+}
+
+/**
  * amdgpu_invalid_rreg - dummy reg read function
  *
  * @adev: amdgpu_device pointer
@@ -1043,7 +1060,7 @@ static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
 	if (adev->asic_type < CHIP_BONAIRE) {
 		adev->doorbell.base = 0;
 		adev->doorbell.size = 0;
-		adev->doorbell.num_doorbells = 0;
+		adev->doorbell.num_kernel_doorbells = 0;
 		adev->doorbell.ptr = NULL;
 		return 0;
 	}
@@ -1058,27 +1075,27 @@ static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
 	adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 
 	if (adev->enable_mes) {
-		adev->doorbell.num_doorbells =
+		adev->doorbell.num_kernel_doorbells =
 			adev->doorbell.size / sizeof(u32);
 	} else {
-		adev->doorbell.num_doorbells =
+		adev->doorbell.num_kernel_doorbells =
 			min_t(u32, adev->doorbell.size / sizeof(u32),
 			      adev->doorbell_index.max_assignment+1);
-		if (adev->doorbell.num_doorbells == 0)
+		if (adev->doorbell.num_kernel_doorbells == 0)
 			return -EINVAL;
 
 		/* For Vega, reserve and map two pages on doorbell BAR since SDMA
 		 * paging queue doorbell use the second page. The
 		 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
 		 * doorbells are in the first page. So with paging queue enabled,
-		 * the max num_doorbells should + 1 page (0x400 in dword)
+		 * the max num_kernel_doorbells should + 1 page (0x400 in dword)
 		 */
 		if (adev->asic_type >= CHIP_VEGA10)
-			adev->doorbell.num_doorbells += 0x400;
+			adev->doorbell.num_kernel_doorbells += 0x400;
 	}
 
 	adev->doorbell.ptr = ioremap(adev->doorbell.base,
-				     adev->doorbell.num_doorbells *
+				     adev->doorbell.num_kernel_doorbells *
 				     sizeof(u32));
 	if (adev->doorbell.ptr == NULL)
 		return -ENOMEM;
@@ -2167,7 +2184,6 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
 		adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
 	}
 
-	amdgpu_amdkfd_device_probe(adev);
 
 	adev->pm.pp_feature = amdgpu_pp_feature_mask;
 	if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
@@ -2223,6 +2239,7 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
 	if (!total)
 		return -ENODEV;
 
+	amdgpu_amdkfd_device_probe(adev);
 	adev->cg_flags &= amdgpu_cg_mask;
 	adev->pg_flags &= amdgpu_pg_mask;
 
@@ -2348,7 +2365,7 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
 		}
 
 		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
-				   ring->num_hw_submission, amdgpu_job_hang_limit,
+				   ring->num_hw_submission, 0,
 				   timeout, adev->reset_domain->wq,
 				   ring->sched_score, ring->name,
 				   adev->dev);
@@ -3164,9 +3181,11 @@ static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
 		AMD_IP_BLOCK_TYPE_DCE,
 		AMD_IP_BLOCK_TYPE_GFX,
 		AMD_IP_BLOCK_TYPE_SDMA,
+		AMD_IP_BLOCK_TYPE_MES,
 		AMD_IP_BLOCK_TYPE_UVD,
 		AMD_IP_BLOCK_TYPE_VCE,
-		AMD_IP_BLOCK_TYPE_VCN
+		AMD_IP_BLOCK_TYPE_VCN,
+		AMD_IP_BLOCK_TYPE_JPEG
 	};
 
 	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
@@ -3286,9 +3305,11 @@ static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
 {
 	int r;
 
-	r = amdgpu_amdkfd_resume_iommu(adev);
-	if (r)
-		return r;
+	if (!adev->in_s0ix) {
+		r = amdgpu_amdkfd_resume_iommu(adev);
+		if (r)
+			return r;
+	}
 
 	r = amdgpu_device_ip_resume_phase1(adev);
 	if (r)
@@ -3788,8 +3809,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		}
 	}
 
-	pci_enable_pcie_error_reporting(adev->pdev);
-
 	/* Post card if necessary */
 	if (amdgpu_device_need_post(adev)) {
 		if (!adev->bios) {
@@ -3879,11 +3898,8 @@ fence_driver_init:
 	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
 
 	r = amdgpu_pm_sysfs_init(adev);
-	if (r) {
-		adev->pm_sysfs_en = false;
-		DRM_ERROR("registering pm debugfs failed (%d).\n", r);
-	} else
-		adev->pm_sysfs_en = true;
+	if (r)
+		DRM_ERROR("registering pm sysfs failed (%d).\n", r);
 
 	r = amdgpu_ucode_sysfs_init(adev);
 	if (r) {
@@ -3945,12 +3961,15 @@ fence_driver_init:
 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
 		vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
 
-	if (amdgpu_device_supports_px(ddev)) {
-		px = true;
+	px = amdgpu_device_supports_px(ddev);
+
+	if (px || (!pci_is_thunderbolt_attached(adev->pdev) &&
+				apple_gmux_detect(NULL, NULL)))
 		vga_switcheroo_register_client(adev->pdev,
 					       &amdgpu_switcheroo_ops, px);
+
+	if (px)
 		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
-	}
 
 	if (adev->gmc.xgmi.pending_reset)
 		queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
@@ -4026,7 +4045,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
 	if (adev->mman.initialized)
 		drain_workqueue(adev->mman.bdev.wq);
 
-	if (adev->pm_sysfs_en)
+	if (adev->pm.sysfs_initialized)
 		amdgpu_pm_sysfs_fini(adev);
 	if (adev->ucode_sysfs_en)
 		amdgpu_ucode_sysfs_fini(adev);
@@ -4054,6 +4073,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
 {
 	int idx;
+	bool px;
 
 	amdgpu_fence_driver_sw_fini(adev);
 	amdgpu_device_ip_fini(adev);
@@ -4072,10 +4092,16 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
 
 	kfree(adev->bios);
 	adev->bios = NULL;
-	if (amdgpu_device_supports_px(adev_to_drm(adev))) {
+
+	px = amdgpu_device_supports_px(adev_to_drm(adev));
+
+	if (px || (!pci_is_thunderbolt_attached(adev->pdev) &&
+				apple_gmux_detect(NULL, NULL)))
 		vga_switcheroo_unregister_client(adev->pdev);
+
+	if (px)
 		vga_switcheroo_fini_domain_pm_ops(adev->dev);
-	}
+
 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
 		vga_client_unregister(adev->pdev);
 
@@ -5154,6 +5180,7 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
  *
  * @adev: amdgpu_device pointer
  * @job: which job trigger hang
+ * @reset_context: amdgpu reset context pointer
  *
  * Attempt to reset the GPU if it has hung (all asics).
  * Attempt to do soft-reset or full-reset and reinitialize Asic
@@ -5323,8 +5350,9 @@ retry:	/* Rest of adevs pre asic reset from XGMI hive. */
 		if (r)
 			adev->asic_reset_res = r;
 
-		/* Aldebaran supports ras in SRIOV, so need resume ras during reset */
-		if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
+		/* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
+		if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
+		    adev->ip_versions[GC_HWIP][0] == IP_VERSION(11, 0, 3))
 			amdgpu_ras_resume(adev);
 	} else {
 		r = amdgpu_do_asic_reset(device_list_handle, reset_context);
@@ -5593,7 +5621,7 @@ int amdgpu_device_baco_enter(struct drm_device *dev)
 	struct amdgpu_device *adev = drm_to_adev(dev);
 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 
-	if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
+	if (!amdgpu_device_supports_baco(dev))
 		return -ENOTSUPP;
 
 	if (ras && adev->ras_enabled &&
@@ -5609,7 +5637,7 @@ int amdgpu_device_baco_exit(struct drm_device *dev)
 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 	int ret = 0;
 
-	if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
+	if (!amdgpu_device_supports_baco(dev))
 		return -ENOTSUPP;
 
 	ret = amdgpu_dpm_baco_exit(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index 1a3cb53d2e0d..0ecce0b92b82 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -37,10 +37,12 @@
 #include "nbio_v6_1.h"
 #include "nbio_v7_0.h"
 #include "nbio_v7_4.h"
+#include "nbio_v7_9.h"
 #include "hdp_v4_0.h"
 #include "vega10_ih.h"
 #include "vega20_ih.h"
 #include "sdma_v4_0.h"
+#include "sdma_v4_4_2.h"
 #include "uvd_v7_0.h"
 #include "vce_v4_0.h"
 #include "vcn_v1_0.h"
@@ -711,7 +713,7 @@ static void ip_hw_instance_release(struct kobject *kobj)
 	kfree(ip_hw_instance);
 }
 
-static struct kobj_type ip_hw_instance_ktype = {
+static const struct kobj_type ip_hw_instance_ktype = {
 	.release = ip_hw_instance_release,
 	.sysfs_ops = &ip_hw_instance_sysfs_ops,
 	.default_groups = ip_hw_instance_groups,
@@ -730,7 +732,7 @@ static void ip_hw_id_release(struct kobject *kobj)
 	kfree(ip_hw_id);
 }
 
-static struct kobj_type ip_hw_id_ktype = {
+static const struct kobj_type ip_hw_id_ktype = {
 	.release = ip_hw_id_release,
 	.sysfs_ops = &kobj_sysfs_ops,
 };
@@ -793,18 +795,18 @@ static const struct sysfs_ops ip_die_entry_sysfs_ops = {
 	.show = ip_die_entry_attr_show,
 };
 
-static struct kobj_type ip_die_entry_ktype = {
+static const struct kobj_type ip_die_entry_ktype = {
 	.release = ip_die_entry_release,
 	.sysfs_ops = &ip_die_entry_sysfs_ops,
 	.default_groups = ip_die_entry_groups,
 };
 
-static struct kobj_type die_kobj_ktype = {
+static const struct kobj_type die_kobj_ktype = {
 	.release = die_kobj_release,
 	.sysfs_ops = &kobj_sysfs_ops,
 };
 
-static struct kobj_type ip_discovery_ktype = {
+static const struct kobj_type ip_discovery_ktype = {
 	.release = ip_disc_release,
 	.sysfs_ops = &kobj_sysfs_ops,
 };
@@ -1500,6 +1502,7 @@ static int amdgpu_discovery_set_common_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(9, 4, 0):
 	case IP_VERSION(9, 4, 1):
 	case IP_VERSION(9, 4, 2):
+	case IP_VERSION(9, 4, 3):
 		amdgpu_device_ip_block_add(adev, &vega10_common_ip_block);
 		break;
 	case IP_VERSION(10, 1, 10):
@@ -1545,6 +1548,7 @@ static int amdgpu_discovery_set_gmc_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(9, 4, 0):
 	case IP_VERSION(9, 4, 1):
 	case IP_VERSION(9, 4, 2):
+	case IP_VERSION(9, 4, 3):
 		amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block);
 		break;
 	case IP_VERSION(10, 1, 10):
@@ -1591,6 +1595,7 @@ static int amdgpu_discovery_set_ih_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(4, 2, 0):
 	case IP_VERSION(4, 2, 1):
 	case IP_VERSION(4, 4, 0):
+	case IP_VERSION(4, 4, 2):
 		amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block);
 		break;
 	case IP_VERSION(5, 0, 0):
@@ -1649,6 +1654,7 @@ static int amdgpu_discovery_set_psp_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(13, 0, 2):
 	case IP_VERSION(13, 0, 3):
 	case IP_VERSION(13, 0, 5):
+	case IP_VERSION(13, 0, 6):
 	case IP_VERSION(13, 0, 7):
 	case IP_VERSION(13, 0, 8):
 	case IP_VERSION(13, 0, 10):
@@ -1842,6 +1848,9 @@ static int amdgpu_discovery_set_sdma_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(4, 4, 0):
 		amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block);
 		break;
+	case IP_VERSION(4, 4, 2):
+		amdgpu_device_ip_block_add(adev, &sdma_v4_4_2_ip_block);
+		break;
 	case IP_VERSION(5, 0, 0):
 	case IP_VERSION(5, 0, 1):
 	case IP_VERSION(5, 0, 2):
@@ -1942,9 +1951,8 @@ static int amdgpu_discovery_set_mm_ip_blocks(struct amdgpu_device *adev)
 		case IP_VERSION(4, 0, 2):
 		case IP_VERSION(4, 0, 4):
 			amdgpu_device_ip_block_add(adev, &vcn_v4_0_ip_block);
-			if (!amdgpu_sriov_vf(adev))
-				amdgpu_device_ip_block_add(adev, &jpeg_v4_0_ip_block);
-			break;
+			amdgpu_device_ip_block_add(adev, &jpeg_v4_0_ip_block);
+			return 0;
 		default:
 			dev_err(adev->dev,
 				"Failed to add vcn/jpeg ip block(UVD_HWIP:0x%x)\n",
@@ -2175,6 +2183,7 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(9, 4, 0):
 	case IP_VERSION(9, 4, 1):
 	case IP_VERSION(9, 4, 2):
+	case IP_VERSION(9, 4, 3):
 		adev->family = AMDGPU_FAMILY_AI;
 		break;
 	case IP_VERSION(9, 1, 0):
@@ -2259,6 +2268,10 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 		adev->nbio.funcs = &nbio_v7_4_funcs;
 		adev->nbio.hdp_flush_reg = &nbio_v7_4_hdp_flush_reg;
 		break;
+	case IP_VERSION(7, 9, 0):
+		adev->nbio.funcs = &nbio_v7_9_funcs;
+		adev->nbio.hdp_flush_reg = &nbio_v7_9_hdp_flush_reg;
+		break;
 	case IP_VERSION(7, 2, 0):
 	case IP_VERSION(7, 2, 1):
 	case IP_VERSION(7, 3, 0):
@@ -2304,6 +2317,7 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(4, 2, 0):
 	case IP_VERSION(4, 2, 1):
 	case IP_VERSION(4, 4, 0):
+	case IP_VERSION(4, 4, 2):
 		adev->hdp.funcs = &hdp_v4_0_funcs;
 		break;
 	case IP_VERSION(5, 0, 0):
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
index 7199b6b0be81..8fd11497faba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
@@ -21,6 +21,9 @@
  *
  */
 
+#ifndef AMDGPU_DOORBELL_H
+#define AMDGPU_DOORBELL_H
+
 /*
  * GPU doorbell structures, functions & helpers
  */
@@ -29,7 +32,9 @@ struct amdgpu_doorbell {
 	resource_size_t		base;
 	resource_size_t		size;
 	u32 __iomem		*ptr;
-	u32			num_doorbells;	/* Number of doorbells actually reserved for amdgpu. */
+
+	/* Number of doorbells reserved for amdgpu kernel driver */
+	u32 num_kernel_doorbells;
 };
 
 /* Reserved doorbells for amdgpu (including multimedia).
@@ -306,3 +311,4 @@ void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
 #define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
 #define WDOORBELL64(index, v) amdgpu_mm_wdoorbell64(adev, (index), (v))
 
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index ba5def374368..b1ca1ab6d6ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -148,8 +148,8 @@ uint amdgpu_pcie_lane_cap;
 u64 amdgpu_cg_mask = 0xffffffffffffffff;
 uint amdgpu_pg_mask = 0xffffffff;
 uint amdgpu_sdma_phase_quantum = 32;
-char *amdgpu_disable_cu = NULL;
-char *amdgpu_virtual_display = NULL;
+char *amdgpu_disable_cu;
+char *amdgpu_virtual_display;
 
 /*
  * OverDrive(bit 14) disabled by default
@@ -157,7 +157,6 @@ char *amdgpu_virtual_display = NULL;
  */
 uint amdgpu_pp_feature_mask = 0xfff7bfff;
 uint amdgpu_force_long_training;
-int amdgpu_job_hang_limit;
 int amdgpu_lbpw = -1;
 int amdgpu_compute_multipipe = -1;
 int amdgpu_gpu_recovery = -1; /* auto */
@@ -521,13 +520,6 @@ MODULE_PARM_DESC(virtual_display,
 module_param_named(virtual_display, amdgpu_virtual_display, charp, 0444);
 
 /**
- * DOC: job_hang_limit (int)
- * Set how much time allow a job hang and not drop it. The default is 0.
- */
-MODULE_PARM_DESC(job_hang_limit, "how much time allow a job hang and not drop it (default 0)");
-module_param_named(job_hang_limit, amdgpu_job_hang_limit, int ,0444);
-
-/**
  * DOC: lbpw (int)
  * Override Load Balancing Per Watt (LBPW) support (1 = enable, 0 = disable). The default is -1 (auto, enabled).
  */
@@ -822,7 +814,7 @@ MODULE_PARM_DESC(no_system_mem_limit, "disable system memory limit (false = defa
  * DOC: no_queue_eviction_on_vm_fault (int)
  * If set, process queues will not be evicted on gpuvm fault. This is to keep the wavefront context for debugging (0 = queue eviction, 1 = no queue eviction). The default is 0 (queue eviction).
  */
-int amdgpu_no_queue_eviction_on_vm_fault = 0;
+int amdgpu_no_queue_eviction_on_vm_fault;
 MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (0 = queue eviction, 1 = no queue eviction)");
 module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fdinfo.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fdinfo.c
index 99a7855ab1bc..c57252f004e8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fdinfo.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fdinfo.c
@@ -60,12 +60,13 @@ void amdgpu_show_fdinfo(struct seq_file *m, struct file *f)
 	struct amdgpu_fpriv *fpriv = file->driver_priv;
 	struct amdgpu_vm *vm = &fpriv->vm;
 
-	uint64_t vram_mem = 0, gtt_mem = 0, cpu_mem = 0;
+	struct amdgpu_mem_stats stats;
 	ktime_t usage[AMDGPU_HW_IP_NUM];
 	uint32_t bus, dev, fn, domain;
 	unsigned int hw_ip;
 	int ret;
 
+	memset(&stats, 0, sizeof(stats));
 	bus = adev->pdev->bus->number;
 	domain = pci_domain_nr(adev->pdev->bus);
 	dev = PCI_SLOT(adev->pdev->devfn);
@@ -75,7 +76,7 @@ void amdgpu_show_fdinfo(struct seq_file *m, struct file *f)
 	if (ret)
 		return;
 
-	amdgpu_vm_get_memory(vm, &vram_mem, &gtt_mem, &cpu_mem);
+	amdgpu_vm_get_memory(vm, &stats);
 	amdgpu_bo_unreserve(vm->root.bo);
 
 	amdgpu_ctx_mgr_usage(&fpriv->ctx_mgr, usage);
@@ -90,9 +91,22 @@ void amdgpu_show_fdinfo(struct seq_file *m, struct file *f)
 	seq_printf(m, "drm-driver:\t%s\n", file->minor->dev->driver->name);
 	seq_printf(m, "drm-pdev:\t%04x:%02x:%02x.%d\n", domain, bus, dev, fn);
 	seq_printf(m, "drm-client-id:\t%Lu\n", vm->immediate.fence_context);
-	seq_printf(m, "drm-memory-vram:\t%llu KiB\n", vram_mem/1024UL);
-	seq_printf(m, "drm-memory-gtt: \t%llu KiB\n", gtt_mem/1024UL);
-	seq_printf(m, "drm-memory-cpu: \t%llu KiB\n", cpu_mem/1024UL);
+	seq_printf(m, "drm-memory-vram:\t%llu KiB\n", stats.vram/1024UL);
+	seq_printf(m, "drm-memory-gtt: \t%llu KiB\n", stats.gtt/1024UL);
+	seq_printf(m, "drm-memory-cpu: \t%llu KiB\n", stats.cpu/1024UL);
+	seq_printf(m, "amd-memory-visible-vram:\t%llu KiB\n",
+		   stats.visible_vram/1024UL);
+	seq_printf(m, "amd-evicted-vram:\t%llu KiB\n",
+		   stats.evicted_vram/1024UL);
+	seq_printf(m, "amd-evicted-visible-vram:\t%llu KiB\n",
+		   stats.evicted_visible_vram/1024UL);
+	seq_printf(m, "amd-requested-vram:\t%llu KiB\n",
+		   stats.requested_vram/1024UL);
+	seq_printf(m, "amd-requested-visible-vram:\t%llu KiB\n",
+		   stats.requested_visible_vram/1024UL);
+	seq_printf(m, "amd-requested-gtt:\t%llu KiB\n",
+		   stats.requested_gtt/1024UL);
+
 	for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
 		if (!usage[hw_ip])
 			continue;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index d8e683688daa..863cb668e000 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -969,7 +969,7 @@ static int amdgpu_debugfs_gem_info_show(struct seq_file *m, void *unused)
 		 * Therefore, we need to protect this ->comm access using RCU.
 		 */
 		rcu_read_lock();
-		task = pid_task(file->pid, PIDTYPE_PID);
+		task = pid_task(file->pid, PIDTYPE_TGID);
 		seq_printf(m, "pid %8d command %s:\n", pid_nr(file->pid),
 			   task ? task->comm : "<unknown>");
 		rcu_read_unlock();
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 35ed46b9249c..9d3a0542c996 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -305,6 +305,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,
 	ring->ring_obj = NULL;
 	ring->use_doorbell = true;
 	ring->doorbell_index = adev->doorbell_index.kiq;
+	ring->vm_hub = AMDGPU_GFXHUB_0;
 
 	r = amdgpu_gfx_kiq_acquire(adev, ring);
 	if (r)
@@ -725,7 +726,7 @@ int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev)
 
 	/* If not define special ras_late_init function, use gfx default ras_late_init */
 	if (!ras->ras_block.ras_late_init)
-		ras->ras_block.ras_late_init = amdgpu_ras_block_late_init;
+		ras->ras_block.ras_late_init = amdgpu_gfx_ras_late_init;
 
 	/* If not defined special ras_cb function, use default ras_cb */
 	if (!ras->ras_block.ras_cb)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index de9e7a00bb15..bfabea76d166 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -42,6 +42,8 @@
 #define AMDGPU_GFX_CG_DISABLED_MODE		0x00000004L
 #define AMDGPU_GFX_LBPW_DISABLED_MODE		0x00000008L
 
+#define AMDGPU_MAX_GC_INSTANCES		8
+
 #define AMDGPU_MAX_GFX_QUEUES KGD_MAX_QUEUES
 #define AMDGPU_MAX_COMPUTE_QUEUES KGD_MAX_QUEUES
 
@@ -53,6 +55,15 @@ enum amdgpu_gfx_pipe_priority {
 #define AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM  0
 #define AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM  15
 
+enum amdgpu_gfx_partition {
+	AMDGPU_SPX_PARTITION_MODE = 0,
+	AMDGPU_DPX_PARTITION_MODE = 1,
+	AMDGPU_TPX_PARTITION_MODE = 2,
+	AMDGPU_QPX_PARTITION_MODE = 3,
+	AMDGPU_CPX_PARTITION_MODE = 4,
+	AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE,
+};
+
 struct amdgpu_mec {
 	struct amdgpu_bo	*hpd_eop_obj;
 	u64			hpd_eop_gpu_addr;
@@ -323,7 +334,7 @@ struct amdgpu_gfx {
 	bool				cp_fw_write_wait;
 	struct amdgpu_ring		gfx_ring[AMDGPU_MAX_GFX_RINGS];
 	unsigned			num_gfx_rings;
-	struct amdgpu_ring		compute_ring[AMDGPU_MAX_COMPUTE_RINGS];
+	struct amdgpu_ring		compute_ring[AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES];
 	unsigned			num_compute_rings;
 	struct amdgpu_irq_src		eop_irq;
 	struct amdgpu_irq_src		priv_reg_irq;
@@ -364,6 +375,10 @@ struct amdgpu_gfx {
 
 	struct amdgpu_ring		sw_gfx_ring[AMDGPU_MAX_SW_GFX_RINGS];
 	struct amdgpu_ring_mux          muxer;
+
+	enum amdgpu_gfx_partition	partition_mode;
+	uint32_t			num_xcd;
+	uint32_t			num_xcc_per_xcp;
 };
 
 #define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 12a6826caef4..4e2531758866 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -395,8 +395,21 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev,
 	while (fault->timestamp >= stamp) {
 		uint64_t tmp;
 
-		if (atomic64_read(&fault->key) == key)
-			return true;
+		if (atomic64_read(&fault->key) == key) {
+			/*
+			 * if we get a fault which is already present in
+			 * the fault_ring and the timestamp of
+			 * the fault is after the expired timestamp,
+			 * then this is a new fault that needs to be added
+			 * into the fault ring.
+			 */
+			if (fault->timestamp_expiry != 0 &&
+			    amdgpu_ih_ts_after(fault->timestamp_expiry,
+					       timestamp))
+				break;
+			else
+				return true;
+		}
 
 		tmp = fault->timestamp;
 		fault = &gmc->fault_ring[fault->next];
@@ -432,28 +445,74 @@ void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
 {
 	struct amdgpu_gmc *gmc = &adev->gmc;
 	uint64_t key = amdgpu_gmc_fault_key(addr, pasid);
+	struct amdgpu_ih_ring *ih;
 	struct amdgpu_gmc_fault *fault;
+	uint32_t last_wptr;
+	uint64_t last_ts;
 	uint32_t hash;
 	uint64_t tmp;
 
+	ih = adev->irq.retry_cam_enabled ? &adev->irq.ih_soft : &adev->irq.ih1;
+	/* Get the WPTR of the last entry in IH ring */
+	last_wptr = amdgpu_ih_get_wptr(adev, ih);
+	/* Order wptr with ring data. */
+	rmb();
+	/* Get the timetamp of the last entry in IH ring */
+	last_ts = amdgpu_ih_decode_iv_ts(adev, ih, last_wptr, -1);
+
 	hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER);
 	fault = &gmc->fault_ring[gmc->fault_hash[hash].idx];
 	do {
-		if (atomic64_cmpxchg(&fault->key, key, 0) == key)
+		if (atomic64_read(&fault->key) == key) {
+			/*
+			 * Update the timestamp when this fault
+			 * expired.
+			 */
+			fault->timestamp_expiry = last_ts;
 			break;
+		}
 
 		tmp = fault->timestamp;
 		fault = &gmc->fault_ring[fault->next];
 	} while (fault->timestamp < tmp);
 }
 
-int amdgpu_gmc_ras_early_init(struct amdgpu_device *adev)
+int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev)
 {
-	if (!adev->gmc.xgmi.connected_to_cpu) {
-		adev->gmc.xgmi.ras = &xgmi_ras;
-		amdgpu_ras_register_ras_block(adev, &adev->gmc.xgmi.ras->ras_block);
-		adev->gmc.xgmi.ras_if = &adev->gmc.xgmi.ras->ras_block.ras_comm;
-	}
+	int r;
+
+	/* umc ras block */
+	r = amdgpu_umc_ras_sw_init(adev);
+	if (r)
+		return r;
+
+	/* mmhub ras block */
+	r = amdgpu_mmhub_ras_sw_init(adev);
+	if (r)
+		return r;
+
+	/* hdp ras block */
+	r = amdgpu_hdp_ras_sw_init(adev);
+	if (r)
+		return r;
+
+	/* mca.x ras block */
+	r = amdgpu_mca_mp0_ras_sw_init(adev);
+	if (r)
+		return r;
+
+	r = amdgpu_mca_mp1_ras_sw_init(adev);
+	if (r)
+		return r;
+
+	r = amdgpu_mca_mpio_ras_sw_init(adev);
+	if (r)
+		return r;
+
+	/* xgmi ras block */
+	r = amdgpu_xgmi_ras_sw_init(adev);
+	if (r)
+		return r;
 
 	return 0;
 }
@@ -495,7 +554,7 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev)
 
 	for (i = 0; i < adev->num_rings; ++i) {
 		ring = adev->rings[i];
-		vmhub = ring->funcs->vmhub;
+		vmhub = ring->vm_hub;
 
 		if (ring == &adev->mes.ring)
 			continue;
@@ -511,7 +570,7 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev)
 		vm_inv_engs[vmhub] &= ~(1 << ring->vm_inv_eng);
 
 		dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n",
-			 ring->name, ring->vm_inv_eng, ring->funcs->vmhub);
+			 ring->name, ring->vm_inv_eng, ring->vm_hub);
 	}
 
 	return 0;
@@ -595,6 +654,7 @@ void amdgpu_gmc_noretry_set(struct amdgpu_device *adev)
 				gc_ver == IP_VERSION(9, 4, 0) ||
 				gc_ver == IP_VERSION(9, 4, 1) ||
 				gc_ver == IP_VERSION(9, 4, 2) ||
+				gc_ver == IP_VERSION(9, 4, 3) ||
 				gc_ver >= IP_VERSION(10, 3, 0));
 
 	gmc->noretry = (amdgpu_noretry == -1) ? noretry_default : amdgpu_noretry;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 0305b660cd17..6d105d7fb98b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -70,6 +70,7 @@ struct amdgpu_gmc_fault {
 	uint64_t	timestamp:48;
 	uint64_t	next:AMDGPU_GMC_FAULT_RING_ORDER;
 	atomic64_t	key;
+	uint64_t	timestamp_expiry:48;
 };
 
 /*
@@ -104,6 +105,8 @@ struct amdgpu_vmhub {
 	uint32_t	vm_cntx_cntl_vm_fault;
 	uint32_t	vm_l2_bank_select_reserved_cid2;
 
+	uint32_t	vm_contexts_disable;
+
 	const struct amdgpu_vmhub_funcs *vmhub_funcs;
 };
 
@@ -351,7 +354,7 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev,
 			      uint16_t pasid, uint64_t timestamp);
 void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
 				     uint16_t pasid);
-int amdgpu_gmc_ras_early_init(struct amdgpu_device *adev);
+int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev);
 int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev);
 void amdgpu_gmc_ras_fini(struct amdgpu_device *adev);
 int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.c
new file mode 100644
index 000000000000..b6cf801939aa
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "amdgpu_ras.h"
+
+int amdgpu_hdp_ras_sw_init(struct amdgpu_device *adev)
+{
+	int err;
+	struct amdgpu_hdp_ras *ras;
+
+	if (!adev->hdp.ras)
+		return 0;
+
+	ras = adev->hdp.ras;
+	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
+	if (err) {
+		dev_err(adev->dev, "Failed to register hdp ras block!\n");
+		return err;
+	}
+
+	strcpy(ras->ras_block.ras_comm.name, "hdp");
+	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__HDP;
+	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
+	adev->hdp.ras_if = &ras->ras_block.ras_comm;
+
+	/* hdp ras follows amdgpu_ras_block_late_init_default for late init */
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h
index ac5c61d3de2b..7b8a6152dc8d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h
@@ -43,5 +43,5 @@ struct amdgpu_hdp {
 	struct amdgpu_hdp_ras	*ras;
 };
 
-int amdgpu_hdp_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block);
+int amdgpu_hdp_ras_sw_init(struct amdgpu_device *adev);
 #endif /* __AMDGPU_HDP_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index bcccc348dbe2..4ff348e10e4d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -69,7 +69,7 @@ int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 	if (size) {
 		r = amdgpu_sa_bo_new(&adev->ib_pools[pool_type],
-				      &ib->sa_bo, size, 256);
+				     &ib->sa_bo, size);
 		if (r) {
 			dev_err(adev->dev, "failed to get a new IB (%d)\n", r);
 			return r;
@@ -267,7 +267,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	if (r) {
 		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
 		if (job && job->vmid)
-			amdgpu_vmid_reset(adev, ring->funcs->vmhub, job->vmid);
+			amdgpu_vmid_reset(adev, ring->vm_hub, job->vmid);
 		amdgpu_ring_undo(ring);
 		return r;
 	}
@@ -309,8 +309,7 @@ int amdgpu_ib_pool_init(struct amdgpu_device *adev)
 
 	for (i = 0; i < AMDGPU_IB_POOL_MAX; i++) {
 		r = amdgpu_sa_bo_manager_init(adev, &adev->ib_pools[i],
-					      AMDGPU_IB_POOL_SIZE,
-					      AMDGPU_GPU_PAGE_SIZE,
+					      AMDGPU_IB_POOL_SIZE, 256,
 					      AMDGPU_GEM_DOMAIN_GTT);
 		if (r)
 			goto error;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
index 3f07b1a2ce47..c991ca0b7a1c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
@@ -202,7 +202,7 @@ static int amdgpu_vmid_grab_idle(struct amdgpu_vm *vm,
 				 struct dma_fence **fence)
 {
 	struct amdgpu_device *adev = ring->adev;
-	unsigned vmhub = ring->funcs->vmhub;
+	unsigned vmhub = ring->vm_hub;
 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 	struct dma_fence **fences;
 	unsigned i;
@@ -277,7 +277,7 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm,
 				     struct dma_fence **fence)
 {
 	struct amdgpu_device *adev = ring->adev;
-	unsigned vmhub = ring->funcs->vmhub;
+	unsigned vmhub = ring->vm_hub;
 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 	uint64_t fence_context = adev->fence_context + ring->idx;
 	bool needs_flush = vm->use_cpu_for_update;
@@ -338,7 +338,7 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm,
 				 struct dma_fence **fence)
 {
 	struct amdgpu_device *adev = ring->adev;
-	unsigned vmhub = ring->funcs->vmhub;
+	unsigned vmhub = ring->vm_hub;
 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 	uint64_t fence_context = adev->fence_context + ring->idx;
 	uint64_t updates = amdgpu_vm_tlb_seq(vm);
@@ -398,7 +398,7 @@ int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 		     struct amdgpu_job *job, struct dma_fence **fence)
 {
 	struct amdgpu_device *adev = ring->adev;
-	unsigned vmhub = ring->funcs->vmhub;
+	unsigned vmhub = ring->vm_hub;
 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 	struct amdgpu_vmid *idle = NULL;
 	struct amdgpu_vmid *id = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index d0a1cc88832c..fafebec5b7b6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -596,6 +596,9 @@ int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
 	if (!src->enabled_types || !src->funcs->set)
 		return -EINVAL;
 
+	if (WARN_ON(!amdgpu_irq_enabled(adev, src, type)))
+		return -EINVAL;
+
 	if (atomic_dec_and_test(&src->enabled_types[type]))
 		return amdgpu_irq_update(adev, src, type);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
index e9f2c11ea416..be243adf3e65 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
@@ -98,6 +98,8 @@ struct amdgpu_irq {
 	struct irq_domain		*domain; /* GPU irq controller domain */
 	unsigned			virq[AMDGPU_MAX_IRQ_SRC_ID];
 	uint32_t                        srbm_soft_reset;
+	u32                             retry_cam_doorbell_index;
+	bool                            retry_cam_enabled;
 };
 
 void amdgpu_irq_disable_all(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
index 6f81ed4fb0d9..b07c000fc8ba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
@@ -118,6 +118,10 @@ int amdgpu_jpeg_dec_ring_test_ring(struct amdgpu_ring *ring)
 	unsigned i;
 	int r;
 
+	/* JPEG in SRIOV does not support direct register read/write */
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
 	WREG32(adev->jpeg.inst[ring->me].external.jpeg_pitch, 0xCAFEDEAD);
 	r = amdgpu_ring_alloc(ring, 3);
 	if (r)
@@ -202,17 +206,18 @@ int amdgpu_jpeg_dec_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	} else {
 		r = 0;
 	}
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		tmp = RREG32(adev->jpeg.inst[ring->me].external.jpeg_pitch);
-		if (tmp == 0xDEADBEEF)
-			break;
-		udelay(1);
+	if (!amdgpu_sriov_vf(adev)) {
+		for (i = 0; i < adev->usec_timeout; i++) {
+			tmp = RREG32(adev->jpeg.inst[ring->me].external.jpeg_pitch);
+			if (tmp == 0xDEADBEEF)
+				break;
+			udelay(1);
+		}
+
+		if (i >= adev->usec_timeout)
+			r = -ETIMEDOUT;
 	}
 
-	if (i >= adev->usec_timeout)
-		r = -ETIMEDOUT;
-
 	dma_fence_put(fence);
 error:
 	return r;
@@ -236,19 +241,28 @@ int amdgpu_jpeg_process_poison_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
-void jpeg_set_ras_funcs(struct amdgpu_device *adev)
+int amdgpu_jpeg_ras_sw_init(struct amdgpu_device *adev)
 {
+	int err;
+	struct amdgpu_jpeg_ras *ras;
+
 	if (!adev->jpeg.ras)
-		return;
+		return 0;
+
+	ras = adev->jpeg.ras;
+	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
+	if (err) {
+		dev_err(adev->dev, "Failed to register jpeg ras block!\n");
+		return err;
+	}
 
-	amdgpu_ras_register_ras_block(adev, &adev->jpeg.ras->ras_block);
+	strcpy(ras->ras_block.ras_comm.name, "jpeg");
+	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__JPEG;
+	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__POISON;
+	adev->jpeg.ras_if = &ras->ras_block.ras_comm;
 
-	strcpy(adev->jpeg.ras->ras_block.ras_comm.name, "jpeg");
-	adev->jpeg.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__JPEG;
-	adev->jpeg.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__POISON;
-	adev->jpeg.ras_if = &adev->jpeg.ras->ras_block.ras_comm;
+	if (!ras->ras_block.ras_late_init)
+		ras->ras_block.ras_late_init = amdgpu_ras_block_late_init;
 
-	/* If don't define special ras_late_init function, use default ras_late_init */
-	if (!adev->jpeg.ras->ras_block.ras_late_init)
-		adev->jpeg.ras->ras_block.ras_late_init = amdgpu_ras_block_late_init;
+	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h
index e8ca3e32ad52..0ca76f0f23e9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h
@@ -72,6 +72,6 @@ int amdgpu_jpeg_dec_ring_test_ib(struct amdgpu_ring *ring, long timeout);
 int amdgpu_jpeg_process_poison_irq(struct amdgpu_device *adev,
 				struct amdgpu_irq_src *source,
 				struct amdgpu_iv_entry *entry);
-void jpeg_set_ras_funcs(struct amdgpu_device *adev);
+int amdgpu_jpeg_ras_sw_init(struct amdgpu_device *adev);
 
 #endif /*__AMDGPU_JPEG_H__*/
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
index 51c2a82e2fa4..8d9ff9e151de 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
@@ -70,3 +70,75 @@ void amdgpu_mca_query_ras_error_count(struct amdgpu_device *adev,
 
 	amdgpu_mca_reset_error_count(adev, mc_status_addr);
 }
+
+int amdgpu_mca_mp0_ras_sw_init(struct amdgpu_device *adev)
+{
+	int err;
+	struct amdgpu_mca_ras_block *ras;
+
+	if (!adev->mca.mp0.ras)
+		return 0;
+
+	ras = adev->mca.mp0.ras;
+
+	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
+	if (err) {
+		dev_err(adev->dev, "Failed to register mca.mp0 ras block!\n");
+		return err;
+	}
+
+	strcpy(ras->ras_block.ras_comm.name, "mca.mp0");
+	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA;
+	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
+	adev->mca.mp0.ras_if = &ras->ras_block.ras_comm;
+
+	return 0;
+}
+
+int amdgpu_mca_mp1_ras_sw_init(struct amdgpu_device *adev)
+{
+	int err;
+	struct amdgpu_mca_ras_block *ras;
+
+	if (!adev->mca.mp1.ras)
+		return 0;
+
+	ras = adev->mca.mp1.ras;
+
+	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
+	if (err) {
+		dev_err(adev->dev, "Failed to register mca.mp1 ras block!\n");
+		return err;
+	}
+
+	strcpy(ras->ras_block.ras_comm.name, "mca.mp1");
+	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA;
+	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
+	adev->mca.mp1.ras_if = &ras->ras_block.ras_comm;
+
+	return 0;
+}
+
+int amdgpu_mca_mpio_ras_sw_init(struct amdgpu_device *adev)
+{
+	int err;
+	struct amdgpu_mca_ras_block *ras;
+
+	if (!adev->mca.mpio.ras)
+		return 0;
+
+	ras = adev->mca.mpio.ras;
+
+	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
+	if (err) {
+		dev_err(adev->dev, "Failed to register mca.mpio ras block!\n");
+		return err;
+	}
+
+	strcpy(ras->ras_block.ras_comm.name, "mca.mpio");
+	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA;
+	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
+	adev->mca.mpio.ras_if = &ras->ras_block.ras_comm;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h
index 7ce16d16e34b..997a073e2409 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h
@@ -30,12 +30,7 @@ struct amdgpu_mca_ras {
 	struct amdgpu_mca_ras_block *ras;
 };
 
-struct amdgpu_mca_funcs {
-	void (*init)(struct amdgpu_device *adev);
-};
-
 struct amdgpu_mca {
-	const struct amdgpu_mca_funcs *funcs;
 	struct amdgpu_mca_ras mp0;
 	struct amdgpu_mca_ras mp1;
 	struct amdgpu_mca_ras mpio;
@@ -55,5 +50,7 @@ void amdgpu_mca_reset_error_count(struct amdgpu_device *adev,
 void amdgpu_mca_query_ras_error_count(struct amdgpu_device *adev,
 				      uint64_t mc_status_addr,
 				      void *ras_error_status);
-
+int amdgpu_mca_mp0_ras_sw_init(struct amdgpu_device *adev);
+int amdgpu_mca_mp1_ras_sw_init(struct amdgpu_device *adev);
+int amdgpu_mca_mpio_ras_sw_init(struct amdgpu_device *adev);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 82e27bd4f038..f0f00466b59f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -1104,6 +1104,11 @@ int amdgpu_mes_ctx_alloc_meta_data(struct amdgpu_device *adev,
 			    &ctx_data->meta_data_obj,
 			    &ctx_data->meta_data_mc_addr,
 			    &ctx_data->meta_data_ptr);
+	if (r) {
+		dev_warn(adev->dev, "(%d) create CTX bo failed\n", r);
+		return r;
+	}
+
 	if (!ctx_data->meta_data_obj)
 		return -ENOMEM;
 
@@ -1328,12 +1333,9 @@ int amdgpu_mes_self_test(struct amdgpu_device *adev)
 	struct amdgpu_mes_ctx_data ctx_data = {0};
 	struct amdgpu_ring *added_rings[AMDGPU_MES_CTX_MAX_RINGS] = { NULL };
 	int gang_ids[3] = {0};
-	int queue_types[][2] = { { AMDGPU_RING_TYPE_GFX,
-				   AMDGPU_MES_CTX_MAX_GFX_RINGS},
-				 { AMDGPU_RING_TYPE_COMPUTE,
-				   AMDGPU_MES_CTX_MAX_COMPUTE_RINGS},
-				 { AMDGPU_RING_TYPE_SDMA,
-				   AMDGPU_MES_CTX_MAX_SDMA_RINGS } };
+	int queue_types[][2] = { { AMDGPU_RING_TYPE_GFX, 1 },
+				 { AMDGPU_RING_TYPE_COMPUTE, 1 },
+				 { AMDGPU_RING_TYPE_SDMA, 1} };
 	int i, r, pasid, k = 0;
 
 	pasid = amdgpu_pasid_alloc(16);
@@ -1432,13 +1434,31 @@ int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe)
 	struct amdgpu_firmware_info *info;
 	char ucode_prefix[30];
 	char fw_name[40];
+	bool need_retry = false;
 	int r;
 
-	amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
-	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin",
-		ucode_prefix,
-		pipe == AMDGPU_MES_SCHED_PIPE ? "" : "1");
+	amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix,
+				       sizeof(ucode_prefix));
+	if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0)) {
+		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin",
+			 ucode_prefix,
+			 pipe == AMDGPU_MES_SCHED_PIPE ? "_2" : "1");
+		need_retry = true;
+	} else {
+		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin",
+			 ucode_prefix,
+			 pipe == AMDGPU_MES_SCHED_PIPE ? "" : "1");
+	}
+
 	r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], fw_name);
+	if (r && need_retry && pipe == AMDGPU_MES_SCHED_PIPE) {
+		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes.bin",
+			 ucode_prefix);
+		DRM_INFO("try to fall back to %s\n", fw_name);
+		r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe],
+					 fw_name);
+	}
+
 	if (r)
 		goto out;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.c
new file mode 100644
index 000000000000..0f6b1021fef3
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.c
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2023  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "amdgpu.h"
+#include "amdgpu_ras.h"
+
+int amdgpu_mmhub_ras_sw_init(struct amdgpu_device *adev)
+{
+	int err;
+	struct amdgpu_mmhub_ras *ras;
+
+	if (!adev->mmhub.ras)
+		return 0;
+
+	ras = adev->mmhub.ras;
+	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
+	if (err) {
+		dev_err(adev->dev, "Failed to register mmhub ras block!\n");
+		return err;
+	}
+
+	strcpy(ras->ras_block.ras_comm.name, "mmhub");
+	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MMHUB;
+	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
+	adev->mmhub.ras_if = &ras->ras_block.ras_comm;
+
+	/* mmhub ras follows amdgpu_ras_block_late_init_default for late init */
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
index 93430d3823c9..d21bb6dae56e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
@@ -48,5 +48,7 @@ struct amdgpu_mmhub {
 	struct amdgpu_mmhub_ras  *ras;
 };
 
+int amdgpu_mmhub_ras_sw_init(struct amdgpu_device *adev);
+
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c
index 37d779b8e4a6..a3bc00577a7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c
@@ -22,6 +22,29 @@
 #include "amdgpu.h"
 #include "amdgpu_ras.h"
 
+int amdgpu_nbio_ras_sw_init(struct amdgpu_device *adev)
+{
+	int err;
+	struct amdgpu_nbio_ras *ras;
+
+	if (!adev->nbio.ras)
+		return 0;
+
+	ras = adev->nbio.ras;
+	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
+	if (err) {
+		dev_err(adev->dev, "Failed to register pcie_bif ras block!\n");
+		return err;
+	}
+
+	strcpy(ras->ras_block.ras_comm.name, "pcie_bif");
+	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__PCIE_BIF;
+	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
+	adev->nbio.ras_if = &ras->ras_block.ras_comm;
+
+	return 0;
+}
+
 int amdgpu_nbio_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
 {
 	int r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
index a240336bbc6b..c686ff4bcc39 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
@@ -106,5 +106,6 @@ struct amdgpu_nbio {
 	struct amdgpu_nbio_ras  *ras;
 };
 
+int amdgpu_nbio_ras_sw_init(struct amdgpu_device *adev);
 int amdgpu_nbio_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 6c7d672412b2..2bd1a54ee866 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -600,7 +600,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 
 	if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
 	    bo->tbo.resource->mem_type == TTM_PL_VRAM &&
-	    bo->tbo.resource->start < adev->gmc.visible_vram_size >> PAGE_SHIFT)
+	    amdgpu_bo_in_cpu_visible_vram(bo))
 		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
 					     ctx.bytes_moved);
 	else
@@ -1265,24 +1265,41 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 	trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
 }
 
-void amdgpu_bo_get_memory(struct amdgpu_bo *bo, uint64_t *vram_mem,
-				uint64_t *gtt_mem, uint64_t *cpu_mem)
+void amdgpu_bo_get_memory(struct amdgpu_bo *bo,
+			  struct amdgpu_mem_stats *stats)
 {
 	unsigned int domain;
+	uint64_t size = amdgpu_bo_size(bo);
 
 	domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
 	switch (domain) {
 	case AMDGPU_GEM_DOMAIN_VRAM:
-		*vram_mem += amdgpu_bo_size(bo);
+		stats->vram += size;
+		if (amdgpu_bo_in_cpu_visible_vram(bo))
+			stats->visible_vram += size;
 		break;
 	case AMDGPU_GEM_DOMAIN_GTT:
-		*gtt_mem += amdgpu_bo_size(bo);
+		stats->gtt += size;
 		break;
 	case AMDGPU_GEM_DOMAIN_CPU:
 	default:
-		*cpu_mem += amdgpu_bo_size(bo);
+		stats->cpu += size;
 		break;
 	}
+
+	if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) {
+		stats->requested_vram += size;
+		if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
+			stats->requested_visible_vram += size;
+
+		if (domain != AMDGPU_GEM_DOMAIN_VRAM) {
+			stats->evicted_vram += size;
+			if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
+				stats->evicted_visible_vram += size;
+		}
+	} else if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_GTT) {
+		stats->requested_gtt += size;
+	}
 }
 
 /**
@@ -1346,7 +1363,6 @@ vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 	struct ttm_operation_ctx ctx = { false, false };
 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
-	unsigned long offset;
 	int r;
 
 	/* Remember that this BO was accessed by the CPU */
@@ -1355,8 +1371,7 @@ vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 	if (bo->resource->mem_type != TTM_PL_VRAM)
 		return 0;
 
-	offset = bo->resource->start << PAGE_SHIFT;
-	if ((offset + bo->base.size) <= adev->gmc.visible_vram_size)
+	if (amdgpu_bo_in_cpu_visible_vram(abo))
 		return 0;
 
 	/* Can't move a pinned BO to visible VRAM */
@@ -1378,10 +1393,9 @@ vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 	else if (unlikely(r))
 		return VM_FAULT_SIGBUS;
 
-	offset = bo->resource->start << PAGE_SHIFT;
 	/* this should never happen */
 	if (bo->resource->mem_type == TTM_PL_VRAM &&
-	    (offset + bo->base.size) > adev->gmc.visible_vram_size)
+	    !amdgpu_bo_in_cpu_visible_vram(abo))
 		return VM_FAULT_SIGBUS;
 
 	ttm_bo_move_to_lru_tail_unlocked(bo);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 93207badf83f..35b8106816a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -126,6 +126,27 @@ struct amdgpu_bo_vm {
 	struct amdgpu_vm_bo_base        entries[];
 };
 
+struct amdgpu_mem_stats {
+	/* current VRAM usage, includes visible VRAM */
+	uint64_t vram;
+	/* current visible VRAM usage */
+	uint64_t visible_vram;
+	/* current GTT usage */
+	uint64_t gtt;
+	/* current system memory usage */
+	uint64_t cpu;
+	/* sum of evicted buffers, includes visible VRAM */
+	uint64_t evicted_vram;
+	/* sum of evicted buffers due to CPU access */
+	uint64_t evicted_visible_vram;
+	/* how much userspace asked for, includes vis.VRAM */
+	uint64_t requested_vram;
+	/* how much userspace asked for */
+	uint64_t requested_visible_vram;
+	/* how much userspace asked for */
+	uint64_t requested_gtt;
+};
+
 static inline struct amdgpu_bo *ttm_to_amdgpu_bo(struct ttm_buffer_object *tbo)
 {
 	return container_of(tbo, struct amdgpu_bo, tbo);
@@ -325,8 +346,8 @@ int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr);
 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo);
 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo);
-void amdgpu_bo_get_memory(struct amdgpu_bo *bo, uint64_t *vram_mem,
-				uint64_t *gtt_mem, uint64_t *cpu_mem);
+void amdgpu_bo_get_memory(struct amdgpu_bo *bo,
+			  struct amdgpu_mem_stats *stats);
 void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm *vmbo);
 int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow,
 			     struct dma_fence **fence);
@@ -336,15 +357,22 @@ uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
 /*
  * sub allocation
  */
+static inline struct amdgpu_sa_manager *
+to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
+{
+	return container_of(manager, struct amdgpu_sa_manager, base);
+}
 
-static inline uint64_t amdgpu_sa_bo_gpu_addr(struct amdgpu_sa_bo *sa_bo)
+static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
 {
-	return sa_bo->manager->gpu_addr + sa_bo->soffset;
+	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
+		drm_suballoc_soffset(sa_bo);
 }
 
-static inline void * amdgpu_sa_bo_cpu_addr(struct amdgpu_sa_bo *sa_bo)
+static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
 {
-	return sa_bo->manager->cpu_ptr + sa_bo->soffset;
+	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
+		drm_suballoc_soffset(sa_bo);
 }
 
 int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
@@ -355,11 +383,11 @@ void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
 int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
 				      struct amdgpu_sa_manager *sa_manager);
 int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
-		     struct amdgpu_sa_bo **sa_bo,
-		     unsigned size, unsigned align);
+		     struct drm_suballoc **sa_bo,
+		     unsigned int size);
 void amdgpu_sa_bo_free(struct amdgpu_device *adev,
-			      struct amdgpu_sa_bo **sa_bo,
-			      struct dma_fence *fence);
+		       struct drm_suballoc **sa_bo,
+		       struct dma_fence *fence);
 #if defined(CONFIG_DEBUG_FS)
 void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
 					 struct seq_file *m);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 3f5d13035aff..9d7e6e0e73ed 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -148,6 +148,7 @@ static int psp_init_sriov_microcode(struct psp_context *psp)
 		break;
 	case IP_VERSION(13, 0, 10):
 		adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MES1_DATA;
+		ret = psp_init_cap_microcode(psp, ucode_prefix);
 		break;
 	default:
 		return -EINVAL;
@@ -191,6 +192,7 @@ static int psp_early_init(void *handle)
 		psp_v12_0_set_psp_funcs(psp);
 		break;
 	case IP_VERSION(13, 0, 2):
+	case IP_VERSION(13, 0, 6):
 		psp_v13_0_set_psp_funcs(psp);
 		break;
 	case IP_VERSION(13, 0, 1):
@@ -837,7 +839,15 @@ static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp,
 static int psp_tmr_unload(struct psp_context *psp)
 {
 	int ret;
-	struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
+	struct psp_gfx_cmd_resp *cmd;
+
+	/* skip TMR unload for Navi12 and CHIP_SIENNA_CICHLID SRIOV,
+	 * as TMR is not loaded at all
+	 */
+	if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
+		return 0;
+
+	cmd = acquire_psp_cmd_buf(psp);
 
 	psp_prep_tmr_unload_cmd_buf(psp, cmd);
 	dev_dbg(psp->adev->dev, "free PSP TMR buffer\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 63dfcc98152d..3ab8a88789c8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -34,6 +34,7 @@
 #include "amdgpu_atomfirmware.h"
 #include "amdgpu_xgmi.h"
 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
+#include "nbio_v4_3.h"
 #include "atom.h"
 #include "amdgpu_reset.h"
 
@@ -2428,6 +2429,13 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
 			else
 				adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
 							1 << AMDGPU_RAS_BLOCK__JPEG);
+
+			/*
+			 * XGMI RAS is not supported if xgmi num physical nodes
+			 * is zero
+			 */
+			if (!adev->gmc.xgmi.num_physical_nodes)
+				adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL);
 		} else {
 			dev_info(adev->dev, "SRAM ECC is not presented.\n");
 		}
@@ -2554,21 +2562,34 @@ int amdgpu_ras_init(struct amdgpu_device *adev)
 	/* initialize nbio ras function ahead of any other
 	 * ras functions so hardware fatal error interrupt
 	 * can be enabled as early as possible */
-	switch (adev->asic_type) {
-	case CHIP_VEGA20:
-	case CHIP_ARCTURUS:
-	case CHIP_ALDEBARAN:
-		if (!adev->gmc.xgmi.connected_to_cpu) {
+	switch (adev->ip_versions[NBIO_HWIP][0]) {
+	case IP_VERSION(7, 4, 0):
+	case IP_VERSION(7, 4, 1):
+	case IP_VERSION(7, 4, 4):
+		if (!adev->gmc.xgmi.connected_to_cpu)
 			adev->nbio.ras = &nbio_v7_4_ras;
-			amdgpu_ras_register_ras_block(adev, &adev->nbio.ras->ras_block);
-			adev->nbio.ras_if = &adev->nbio.ras->ras_block.ras_comm;
-		}
+		break;
+	case IP_VERSION(4, 3, 0):
+		if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
+			/* unlike other generation of nbio ras,
+			 * nbio v4_3 only support fatal error interrupt
+			 * to inform software that DF is freezed due to
+			 * system fatal error event. driver should not
+			 * enable nbio ras in such case. Instead,
+			 * check DF RAS */
+			adev->nbio.ras = &nbio_v4_3_ras;
 		break;
 	default:
 		/* nbio ras is not available */
 		break;
 	}
 
+	/* nbio ras block needs to be enabled ahead of other ras blocks
+	 * to handle fatal error */
+	r = amdgpu_nbio_ras_sw_init(adev);
+	if (r)
+		return r;
+
 	if (adev->nbio.ras &&
 	    adev->nbio.ras->init_ras_controller_interrupt) {
 		r = adev->nbio.ras->init_ras_controller_interrupt(adev);
@@ -3073,9 +3094,6 @@ int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
 	if (!adev || !ras_block_obj)
 		return -EINVAL;
 
-	if (!amdgpu_ras_asic_supported(adev))
-		return 0;
-
 	ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
 	if (!ras_node)
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
index ef38f4c93df0..17b3d1992e80 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
@@ -583,6 +583,10 @@ amdgpu_ras_block_to_ta(enum amdgpu_ras_block block) {
 		return TA_RAS_BLOCK__FUSE;
 	case AMDGPU_RAS_BLOCK__MCA:
 		return TA_RAS_BLOCK__MCA;
+	case AMDGPU_RAS_BLOCK__VCN:
+		return TA_RAS_BLOCK__VCN;
+	case AMDGPU_RAS_BLOCK__JPEG:
+		return TA_RAS_BLOCK__JPEG;
 	default:
 		WARN_ONCE(1, "RAS ERROR: unexpected block id %d\n", block);
 		return TA_RAS_BLOCK__UMC;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
index 2e08fce87521..c2c2a7718613 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
@@ -107,47 +107,12 @@
 
 static bool __is_ras_eeprom_supported(struct amdgpu_device *adev)
 {
-	if (adev->asic_type == CHIP_IP_DISCOVERY) {
-		switch (adev->ip_versions[MP1_HWIP][0]) {
-		case IP_VERSION(13, 0, 0):
-		case IP_VERSION(13, 0, 10):
-			return true;
-		default:
-			return false;
-		}
-	}
-
-	return  adev->asic_type == CHIP_VEGA20 ||
-		adev->asic_type == CHIP_ARCTURUS ||
-		adev->asic_type == CHIP_SIENNA_CICHLID ||
-		adev->asic_type == CHIP_ALDEBARAN;
-}
-
-static bool __get_eeprom_i2c_addr_arct(struct amdgpu_device *adev,
-				       struct amdgpu_ras_eeprom_control *control)
-{
-	struct atom_context *atom_ctx = adev->mode_info.atom_context;
-
-	if (!control || !atom_ctx)
-		return false;
-
-	if (strnstr(atom_ctx->vbios_version,
-	            "D342",
-		    sizeof(atom_ctx->vbios_version)))
-		control->i2c_address = EEPROM_I2C_MADDR_0;
-	else
-		control->i2c_address = EEPROM_I2C_MADDR_4;
-
-	return true;
-}
-
-static bool __get_eeprom_i2c_addr_ip_discovery(struct amdgpu_device *adev,
-				       struct amdgpu_ras_eeprom_control *control)
-{
 	switch (adev->ip_versions[MP1_HWIP][0]) {
+	case IP_VERSION(11, 0, 2): /* VEGA20 and ARCTURUS */
+	case IP_VERSION(11, 0, 7): /* Sienna cichlid */
 	case IP_VERSION(13, 0, 0):
+	case IP_VERSION(13, 0, 2): /* Aldebaran */
 	case IP_VERSION(13, 0, 10):
-		control->i2c_address = EEPROM_I2C_MADDR_4;
 		return true;
 	default:
 		return false;
@@ -178,43 +143,35 @@ static bool __get_eeprom_i2c_addr(struct amdgpu_device *adev,
 		return true;
 	}
 
-	switch (adev->asic_type) {
-	case CHIP_VEGA20:
-		control->i2c_address = EEPROM_I2C_MADDR_0;
-		break;
-
-	case CHIP_ARCTURUS:
-		return __get_eeprom_i2c_addr_arct(adev, control);
-
-	case CHIP_SIENNA_CICHLID:
+	switch (adev->ip_versions[MP1_HWIP][0]) {
+	case IP_VERSION(11, 0, 2):
+		/* VEGA20 and ARCTURUS */
+		if (adev->asic_type == CHIP_VEGA20)
+			control->i2c_address = EEPROM_I2C_MADDR_0;
+		else if (strnstr(atom_ctx->vbios_version,
+				 "D342",
+				 sizeof(atom_ctx->vbios_version)))
+			control->i2c_address = EEPROM_I2C_MADDR_0;
+		else
+			control->i2c_address = EEPROM_I2C_MADDR_4;
+		return true;
+	case IP_VERSION(11, 0, 7):
 		control->i2c_address = EEPROM_I2C_MADDR_0;
-		break;
-
-	case CHIP_ALDEBARAN:
+		return true;
+	case IP_VERSION(13, 0, 2):
 		if (strnstr(atom_ctx->vbios_version, "D673",
 			    sizeof(atom_ctx->vbios_version)))
 			control->i2c_address = EEPROM_I2C_MADDR_4;
 		else
 			control->i2c_address = EEPROM_I2C_MADDR_0;
-		break;
-
-	case CHIP_IP_DISCOVERY:
-		return __get_eeprom_i2c_addr_ip_discovery(adev, control);
-
-	default:
-		return false;
-	}
-
-	switch (adev->ip_versions[MP1_HWIP][0]) {
+		return true;
 	case IP_VERSION(13, 0, 0):
+	case IP_VERSION(13, 0, 10):
 		control->i2c_address = EEPROM_I2C_MADDR_4;
-		break;
-
+		return true;
 	default:
-		break;
+		return false;
 	}
-
-	return true;
 }
 
 static void
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 3989e755a5b4..d8749444b689 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -27,6 +27,7 @@
 #include <drm/amdgpu_drm.h>
 #include <drm/gpu_scheduler.h>
 #include <drm/drm_print.h>
+#include <drm/drm_suballoc.h>
 
 struct amdgpu_device;
 struct amdgpu_ring;
@@ -92,7 +93,7 @@ enum amdgpu_ib_pool_type {
 };
 
 struct amdgpu_ib {
-	struct amdgpu_sa_bo		*sa_bo;
+	struct drm_suballoc		*sa_bo;
 	uint32_t			length_dw;
 	uint64_t			gpu_addr;
 	uint32_t			*ptr;
@@ -164,7 +165,6 @@ struct amdgpu_ring_funcs {
 	bool			support_64bit_ptrs;
 	bool			no_user_fence;
 	bool			secure_submission_supported;
-	unsigned		vmhub;
 	unsigned		extra_dw;
 
 	/* ring read/write ptr handling */
@@ -249,6 +249,7 @@ struct amdgpu_ring {
 	uint64_t		ptr_mask;
 	uint32_t		buf_mask;
 	u32			idx;
+	u32			xcc_id;
 	u32			me;
 	u32			pipe;
 	u32			queue;
@@ -274,6 +275,7 @@ struct amdgpu_ring {
 	unsigned		cond_exe_offs;
 	u64			cond_exe_gpu_addr;
 	volatile u32		*cond_exe_cpu_addr;
+	unsigned		vm_hub;
 	unsigned		vm_inv_eng;
 	struct dma_fence	*vmid_wait;
 	bool			has_compute_vm_bug;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
index 524d10b21041..c6b4337eb20c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
@@ -44,327 +44,63 @@
 
 #include "amdgpu.h"
 
-static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo);
-static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager);
-
 int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
 			      struct amdgpu_sa_manager *sa_manager,
-			      unsigned size, u32 align, u32 domain)
+			      unsigned int size, u32 suballoc_align, u32 domain)
 {
-	int i, r;
-
-	init_waitqueue_head(&sa_manager->wq);
-	sa_manager->bo = NULL;
-	sa_manager->size = size;
-	sa_manager->domain = domain;
-	sa_manager->align = align;
-	sa_manager->hole = &sa_manager->olist;
-	INIT_LIST_HEAD(&sa_manager->olist);
-	for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
-		INIT_LIST_HEAD(&sa_manager->flist[i]);
+	int r;
 
-	r = amdgpu_bo_create_kernel(adev, size, align, domain, &sa_manager->bo,
-				&sa_manager->gpu_addr, &sa_manager->cpu_ptr);
+	r = amdgpu_bo_create_kernel(adev, size, AMDGPU_GPU_PAGE_SIZE, domain,
+				    &sa_manager->bo, &sa_manager->gpu_addr,
+				    &sa_manager->cpu_ptr);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate bo for manager\n", r);
 		return r;
 	}
 
-	memset(sa_manager->cpu_ptr, 0, sa_manager->size);
+	memset(sa_manager->cpu_ptr, 0, size);
+	drm_suballoc_manager_init(&sa_manager->base, size, suballoc_align);
 	return r;
 }
 
 void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
 			       struct amdgpu_sa_manager *sa_manager)
 {
-	struct amdgpu_sa_bo *sa_bo, *tmp;
-
 	if (sa_manager->bo == NULL) {
 		dev_err(adev->dev, "no bo for sa manager\n");
 		return;
 	}
 
-	if (!list_empty(&sa_manager->olist)) {
-		sa_manager->hole = &sa_manager->olist,
-		amdgpu_sa_bo_try_free(sa_manager);
-		if (!list_empty(&sa_manager->olist)) {
-			dev_err(adev->dev, "sa_manager is not empty, clearing anyway\n");
-		}
-	}
-	list_for_each_entry_safe(sa_bo, tmp, &sa_manager->olist, olist) {
-		amdgpu_sa_bo_remove_locked(sa_bo);
-	}
+	drm_suballoc_manager_fini(&sa_manager->base);
 
 	amdgpu_bo_free_kernel(&sa_manager->bo, &sa_manager->gpu_addr, &sa_manager->cpu_ptr);
-	sa_manager->size = 0;
 }
 
-static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo)
-{
-	struct amdgpu_sa_manager *sa_manager = sa_bo->manager;
-	if (sa_manager->hole == &sa_bo->olist) {
-		sa_manager->hole = sa_bo->olist.prev;
-	}
-	list_del_init(&sa_bo->olist);
-	list_del_init(&sa_bo->flist);
-	dma_fence_put(sa_bo->fence);
-	kfree(sa_bo);
-}
-
-static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager)
+int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
+		     struct drm_suballoc **sa_bo,
+		     unsigned int size)
 {
-	struct amdgpu_sa_bo *sa_bo, *tmp;
+	struct drm_suballoc *sa = drm_suballoc_new(&sa_manager->base, size,
+						   GFP_KERNEL, true, 0);
 
-	if (sa_manager->hole->next == &sa_manager->olist)
-		return;
+	if (IS_ERR(sa)) {
+		*sa_bo = NULL;
 
-	sa_bo = list_entry(sa_manager->hole->next, struct amdgpu_sa_bo, olist);
-	list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) {
-		if (sa_bo->fence == NULL ||
-		    !dma_fence_is_signaled(sa_bo->fence)) {
-			return;
-		}
-		amdgpu_sa_bo_remove_locked(sa_bo);
+		return PTR_ERR(sa);
 	}
-}
 
-static inline unsigned amdgpu_sa_bo_hole_soffset(struct amdgpu_sa_manager *sa_manager)
-{
-	struct list_head *hole = sa_manager->hole;
-
-	if (hole != &sa_manager->olist) {
-		return list_entry(hole, struct amdgpu_sa_bo, olist)->eoffset;
-	}
+	*sa_bo = sa;
 	return 0;
 }
 
-static inline unsigned amdgpu_sa_bo_hole_eoffset(struct amdgpu_sa_manager *sa_manager)
-{
-	struct list_head *hole = sa_manager->hole;
-
-	if (hole->next != &sa_manager->olist) {
-		return list_entry(hole->next, struct amdgpu_sa_bo, olist)->soffset;
-	}
-	return sa_manager->size;
-}
-
-static bool amdgpu_sa_bo_try_alloc(struct amdgpu_sa_manager *sa_manager,
-				   struct amdgpu_sa_bo *sa_bo,
-				   unsigned size, unsigned align)
-{
-	unsigned soffset, eoffset, wasted;
-
-	soffset = amdgpu_sa_bo_hole_soffset(sa_manager);
-	eoffset = amdgpu_sa_bo_hole_eoffset(sa_manager);
-	wasted = (align - (soffset % align)) % align;
-
-	if ((eoffset - soffset) >= (size + wasted)) {
-		soffset += wasted;
-
-		sa_bo->manager = sa_manager;
-		sa_bo->soffset = soffset;
-		sa_bo->eoffset = soffset + size;
-		list_add(&sa_bo->olist, sa_manager->hole);
-		INIT_LIST_HEAD(&sa_bo->flist);
-		sa_manager->hole = &sa_bo->olist;
-		return true;
-	}
-	return false;
-}
-
-/**
- * amdgpu_sa_event - Check if we can stop waiting
- *
- * @sa_manager: pointer to the sa_manager
- * @size: number of bytes we want to allocate
- * @align: alignment we need to match
- *
- * Check if either there is a fence we can wait for or
- * enough free memory to satisfy the allocation directly
- */
-static bool amdgpu_sa_event(struct amdgpu_sa_manager *sa_manager,
-			    unsigned size, unsigned align)
-{
-	unsigned soffset, eoffset, wasted;
-	int i;
-
-	for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
-		if (!list_empty(&sa_manager->flist[i]))
-			return true;
-
-	soffset = amdgpu_sa_bo_hole_soffset(sa_manager);
-	eoffset = amdgpu_sa_bo_hole_eoffset(sa_manager);
-	wasted = (align - (soffset % align)) % align;
-
-	if ((eoffset - soffset) >= (size + wasted)) {
-		return true;
-	}
-
-	return false;
-}
-
-static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager,
-				   struct dma_fence **fences,
-				   unsigned *tries)
-{
-	struct amdgpu_sa_bo *best_bo = NULL;
-	unsigned i, soffset, best, tmp;
-
-	/* if hole points to the end of the buffer */
-	if (sa_manager->hole->next == &sa_manager->olist) {
-		/* try again with its beginning */
-		sa_manager->hole = &sa_manager->olist;
-		return true;
-	}
-
-	soffset = amdgpu_sa_bo_hole_soffset(sa_manager);
-	/* to handle wrap around we add sa_manager->size */
-	best = sa_manager->size * 2;
-	/* go over all fence list and try to find the closest sa_bo
-	 * of the current last
-	 */
-	for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i) {
-		struct amdgpu_sa_bo *sa_bo;
-
-		fences[i] = NULL;
-
-		if (list_empty(&sa_manager->flist[i]))
-			continue;
-
-		sa_bo = list_first_entry(&sa_manager->flist[i],
-					 struct amdgpu_sa_bo, flist);
-
-		if (!dma_fence_is_signaled(sa_bo->fence)) {
-			fences[i] = sa_bo->fence;
-			continue;
-		}
-
-		/* limit the number of tries each ring gets */
-		if (tries[i] > 2) {
-			continue;
-		}
-
-		tmp = sa_bo->soffset;
-		if (tmp < soffset) {
-			/* wrap around, pretend it's after */
-			tmp += sa_manager->size;
-		}
-		tmp -= soffset;
-		if (tmp < best) {
-			/* this sa bo is the closest one */
-			best = tmp;
-			best_bo = sa_bo;
-		}
-	}
-
-	if (best_bo) {
-		uint32_t idx = best_bo->fence->context;
-
-		idx %= AMDGPU_SA_NUM_FENCE_LISTS;
-		++tries[idx];
-		sa_manager->hole = best_bo->olist.prev;
-
-		/* we knew that this one is signaled,
-		   so it's save to remote it */
-		amdgpu_sa_bo_remove_locked(best_bo);
-		return true;
-	}
-	return false;
-}
-
-int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
-		     struct amdgpu_sa_bo **sa_bo,
-		     unsigned size, unsigned align)
-{
-	struct dma_fence *fences[AMDGPU_SA_NUM_FENCE_LISTS];
-	unsigned tries[AMDGPU_SA_NUM_FENCE_LISTS];
-	unsigned count;
-	int i, r;
-	signed long t;
-
-	if (WARN_ON_ONCE(align > sa_manager->align))
-		return -EINVAL;
-
-	if (WARN_ON_ONCE(size > sa_manager->size))
-		return -EINVAL;
-
-	*sa_bo = kmalloc(sizeof(struct amdgpu_sa_bo), GFP_KERNEL);
-	if (!(*sa_bo))
-		return -ENOMEM;
-	(*sa_bo)->manager = sa_manager;
-	(*sa_bo)->fence = NULL;
-	INIT_LIST_HEAD(&(*sa_bo)->olist);
-	INIT_LIST_HEAD(&(*sa_bo)->flist);
-
-	spin_lock(&sa_manager->wq.lock);
-	do {
-		for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
-			tries[i] = 0;
-
-		do {
-			amdgpu_sa_bo_try_free(sa_manager);
-
-			if (amdgpu_sa_bo_try_alloc(sa_manager, *sa_bo,
-						   size, align)) {
-				spin_unlock(&sa_manager->wq.lock);
-				return 0;
-			}
-
-			/* see if we can skip over some allocations */
-		} while (amdgpu_sa_bo_next_hole(sa_manager, fences, tries));
-
-		for (i = 0, count = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
-			if (fences[i])
-				fences[count++] = dma_fence_get(fences[i]);
-
-		if (count) {
-			spin_unlock(&sa_manager->wq.lock);
-			t = dma_fence_wait_any_timeout(fences, count, false,
-						       MAX_SCHEDULE_TIMEOUT,
-						       NULL);
-			for (i = 0; i < count; ++i)
-				dma_fence_put(fences[i]);
-
-			r = (t > 0) ? 0 : t;
-			spin_lock(&sa_manager->wq.lock);
-		} else {
-			/* if we have nothing to wait for block */
-			r = wait_event_interruptible_locked(
-				sa_manager->wq,
-				amdgpu_sa_event(sa_manager, size, align)
-			);
-		}
-
-	} while (!r);
-
-	spin_unlock(&sa_manager->wq.lock);
-	kfree(*sa_bo);
-	*sa_bo = NULL;
-	return r;
-}
-
-void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo,
+void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct drm_suballoc **sa_bo,
 		       struct dma_fence *fence)
 {
-	struct amdgpu_sa_manager *sa_manager;
-
 	if (sa_bo == NULL || *sa_bo == NULL) {
 		return;
 	}
 
-	sa_manager = (*sa_bo)->manager;
-	spin_lock(&sa_manager->wq.lock);
-	if (fence && !dma_fence_is_signaled(fence)) {
-		uint32_t idx;
-
-		(*sa_bo)->fence = dma_fence_get(fence);
-		idx = fence->context % AMDGPU_SA_NUM_FENCE_LISTS;
-		list_add_tail(&(*sa_bo)->flist, &sa_manager->flist[idx]);
-	} else {
-		amdgpu_sa_bo_remove_locked(*sa_bo);
-	}
-	wake_up_all_locked(&sa_manager->wq);
-	spin_unlock(&sa_manager->wq.lock);
+	drm_suballoc_free(*sa_bo, fence);
 	*sa_bo = NULL;
 }
 
@@ -373,26 +109,8 @@ void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo,
 void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
 				  struct seq_file *m)
 {
-	struct amdgpu_sa_bo *i;
-
-	spin_lock(&sa_manager->wq.lock);
-	list_for_each_entry(i, &sa_manager->olist, olist) {
-		uint64_t soffset = i->soffset + sa_manager->gpu_addr;
-		uint64_t eoffset = i->eoffset + sa_manager->gpu_addr;
-		if (&i->olist == sa_manager->hole) {
-			seq_printf(m, ">");
-		} else {
-			seq_printf(m, " ");
-		}
-		seq_printf(m, "[0x%010llx 0x%010llx] size %8lld",
-			   soffset, eoffset, eoffset - soffset);
+	struct drm_printer p = drm_seq_file_printer(m);
 
-		if (i->fence)
-			seq_printf(m, " protected by 0x%016llx on context %llu",
-				   i->fence->seqno, i->fence->context);
-
-		seq_printf(m, "\n");
-	}
-	spin_unlock(&sa_manager->wq.lock);
+	drm_suballoc_dump_debug_info(&sa_manager->base, &p, sa_manager->gpu_addr);
 }
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 98d91ebf5c26..525dffbe046a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -233,7 +233,7 @@ TRACE_EVENT(amdgpu_vm_grab_id,
 			   __entry->pasid = vm->pasid;
 			   __assign_str(ring, ring->name);
 			   __entry->vmid = job->vmid;
-			   __entry->vm_hub = ring->funcs->vmhub,
+			   __entry->vm_hub = ring->vm_hub,
 			   __entry->pd_addr = job->vm_pd_addr;
 			   __entry->needs_flush = job->vm_needs_flush;
 			   ),
@@ -427,7 +427,7 @@ TRACE_EVENT(amdgpu_vm_flush,
 	    TP_fast_assign(
 			   __assign_str(ring, ring->name);
 			   __entry->vmid = vmid;
-			   __entry->vm_hub = ring->funcs->vmhub;
+			   __entry->vm_hub = ring->vm_hub;
 			   __entry->pd_addr = pd_addr;
 			   ),
 	    TP_printk("ring=%s, id=%u, hub=%u, pd_addr=%010Lx",
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index c5ef7f7bdc15..2cd081cbf706 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -466,11 +466,7 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 			return r;
 	}
 
-	/* Can't move a pinned BO */
 	abo = ttm_to_amdgpu_bo(bo);
-	if (WARN_ON_ONCE(abo->tbo.pin_count > 0))
-		return -EINVAL;
-
 	adev = amdgpu_ttm_adev(bo->bdev);
 
 	if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM &&
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 380b89114341..f76b1cb8baf8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -126,19 +126,6 @@ void amdgpu_ucode_print_gfx_hdr(const struct common_firmware_header *hdr)
 	}
 }
 
-void amdgpu_ucode_print_imu_hdr(const struct common_firmware_header *hdr)
-{
-	uint16_t version_major = le16_to_cpu(hdr->header_version_major);
-	uint16_t version_minor = le16_to_cpu(hdr->header_version_minor);
-
-	DRM_DEBUG("IMU\n");
-	amdgpu_ucode_print_common_hdr(hdr);
-
-	if (version_major != 1) {
-		DRM_ERROR("Unknown GFX ucode version: %u.%u\n", version_major, version_minor);
-	}
-}
-
 void amdgpu_ucode_print_rlc_hdr(const struct common_firmware_header *hdr)
 {
 	uint16_t version_major = le16_to_cpu(hdr->header_version_major);
@@ -472,6 +459,12 @@ void amdgpu_ucode_print_psp_hdr(const struct common_firmware_header *hdr)
 				DRM_DEBUG("psp_dbg_drv_size_bytes: %u\n",
 					  le32_to_cpu(desc->size_bytes));
 				break;
+			case PSP_FW_TYPE_PSP_RAS_DRV:
+				DRM_DEBUG("psp_ras_drv_version: %u\n",
+					  le32_to_cpu(desc->fw_version));
+				DRM_DEBUG("psp_ras_drv_size_bytes: %u\n",
+					  le32_to_cpu(desc->size_bytes));
+				break;
 			default:
 				DRM_DEBUG("Unsupported PSP fw type: %d\n", desc->fw_type);
 				break;
@@ -669,6 +662,8 @@ const char *amdgpu_ucode_name(enum AMDGPU_UCODE_ID ucode_id)
 		return "VCN1_RAM";
 	case AMDGPU_UCODE_ID_DMCUB:
 		return "DMCUB";
+	case AMDGPU_UCODE_ID_CAP:
+		return "CAP";
 	default:
 		return "UNKNOWN UCODE";
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
index 1b8574bc4463..1edf8e6aeb16 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
@@ -208,6 +208,36 @@ int amdgpu_umc_process_ras_data_cb(struct amdgpu_device *adev,
 	return amdgpu_umc_do_page_retirement(adev, ras_error_status, entry, true);
 }
 
+int amdgpu_umc_ras_sw_init(struct amdgpu_device *adev)
+{
+	int err;
+	struct amdgpu_umc_ras *ras;
+
+	if (!adev->umc.ras)
+		return 0;
+
+	ras = adev->umc.ras;
+
+	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
+	if (err) {
+		dev_err(adev->dev, "Failed to register umc ras block!\n");
+		return err;
+	}
+
+	strcpy(adev->umc.ras->ras_block.ras_comm.name, "umc");
+	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__UMC;
+	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
+	adev->umc.ras_if = &ras->ras_block.ras_comm;
+
+	if (!ras->ras_block.ras_late_init)
+		ras->ras_block.ras_late_init = amdgpu_umc_ras_late_init;
+
+	if (!ras->ras_block.ras_cb)
+		ras->ras_block.ras_cb = amdgpu_umc_process_ras_data_cb;
+
+	return 0;
+}
+
 int amdgpu_umc_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
 {
 	int r;
@@ -272,3 +302,34 @@ void amdgpu_umc_fill_error_record(struct ras_err_data *err_data,
 
 	err_data->err_addr_cnt++;
 }
+
+int amdgpu_umc_loop_channels(struct amdgpu_device *adev,
+			umc_func func, void *data)
+{
+	uint32_t node_inst       = 0;
+	uint32_t umc_inst        = 0;
+	uint32_t ch_inst         = 0;
+	int ret = 0;
+
+	if (adev->umc.node_inst_num) {
+		LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) {
+			ret = func(adev, node_inst, umc_inst, ch_inst, data);
+			if (ret) {
+				dev_err(adev->dev, "Node %d umc %d ch %d func returns %d\n",
+					node_inst, umc_inst, ch_inst, ret);
+				return ret;
+			}
+		}
+	} else {
+		LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) {
+			ret = func(adev, 0, umc_inst, ch_inst, data);
+			if (ret) {
+				dev_err(adev->dev, "Umc %d ch %d func returns %d\n",
+					umc_inst, ch_inst, ret);
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
index 36e19336f3b3..86133f77a9a4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
@@ -47,6 +47,10 @@
 #define LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) \
 		LOOP_UMC_NODE_INST((node_inst)) LOOP_UMC_INST_AND_CH((umc_inst), (ch_inst))
 
+
+typedef int (*umc_func)(struct amdgpu_device *adev, uint32_t node_inst,
+			uint32_t umc_inst, uint32_t ch_inst, void *data);
+
 struct amdgpu_umc_ras {
 	struct amdgpu_ras_block_object ras_block;
 	void (*err_cnt_init)(struct amdgpu_device *adev);
@@ -87,6 +91,7 @@ struct amdgpu_umc {
 	unsigned long active_mask;
 };
 
+int amdgpu_umc_ras_sw_init(struct amdgpu_device *adev);
 int amdgpu_umc_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block);
 int amdgpu_umc_poison_handler(struct amdgpu_device *adev, bool reset);
 int amdgpu_umc_process_ecc_irq(struct amdgpu_device *adev,
@@ -103,4 +108,7 @@ int amdgpu_umc_process_ras_data_cb(struct amdgpu_device *adev,
 		struct amdgpu_iv_entry *entry);
 int amdgpu_umc_page_retirement_mca(struct amdgpu_device *adev,
 			uint64_t err_addr, uint32_t ch_inst, uint32_t umc_inst);
+
+int amdgpu_umc_loop_channels(struct amdgpu_device *adev,
+			umc_func func, void *data);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index 632a6ded5735..6887109abb13 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -1118,14 +1118,11 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct dma_fence *f = NULL;
+	uint32_t offset, data[4];
 	struct amdgpu_job *job;
 	struct amdgpu_ib *ib;
-	uint32_t data[4];
 	uint64_t addr;
-	long r;
-	int i;
-	unsigned offset_idx = 0;
-	unsigned offset[3] = { UVD_BASE_SI, 0, 0 };
+	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->uvd.entity,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
@@ -1134,16 +1131,15 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 	if (r)
 		return r;
 
-	if (adev->asic_type >= CHIP_VEGA10) {
-		offset_idx = 1 + ring->me;
-		offset[1] = adev->reg_offset[UVD_HWIP][0][1];
-		offset[2] = adev->reg_offset[UVD_HWIP][1][1];
-	}
+	if (adev->asic_type >= CHIP_VEGA10)
+		offset = adev->reg_offset[UVD_HWIP][ring->me][1];
+	else
+		offset = UVD_BASE_SI;
 
-	data[0] = PACKET0(offset[offset_idx] + UVD_GPCOM_VCPU_DATA0, 0);
-	data[1] = PACKET0(offset[offset_idx] + UVD_GPCOM_VCPU_DATA1, 0);
-	data[2] = PACKET0(offset[offset_idx] + UVD_GPCOM_VCPU_CMD, 0);
-	data[3] = PACKET0(offset[offset_idx] + UVD_NO_OP, 0);
+	data[0] = PACKET0(offset + UVD_GPCOM_VCPU_DATA0, 0);
+	data[1] = PACKET0(offset + UVD_GPCOM_VCPU_DATA1, 0);
+	data[2] = PACKET0(offset + UVD_GPCOM_VCPU_CMD, 0);
+	data[3] = PACKET0(offset + UVD_NO_OP, 0);
 
 	ib = &job->ibs[0];
 	addr = amdgpu_bo_gpu_offset(bo);
@@ -1160,14 +1156,6 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 	ib->length_dw = 16;
 
 	if (direct) {
-		r = dma_resv_wait_timeout(bo->tbo.base.resv,
-					  DMA_RESV_USAGE_KERNEL, false,
-					  msecs_to_jiffies(10));
-		if (r == 0)
-			r = -ETIMEDOUT;
-		if (r < 0)
-			goto err_free;
-
 		r = amdgpu_job_submit_direct(job, ring, &f);
 		if (r)
 			goto err_free;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 2fb61410b1c0..e2b7324a70cb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -585,6 +585,7 @@ err:
 /**
  * amdgpu_vce_validate_bo - make sure not to cross 4GB boundary
  *
+ * @p: cs parser
  * @ib: indirect buffer to use
  * @lo: address of lower dword
  * @hi: address of higher dword
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index e7974de8b035..e63fcc58e8e0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -1181,19 +1181,28 @@ int amdgpu_vcn_process_poison_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
-void amdgpu_vcn_set_ras_funcs(struct amdgpu_device *adev)
+int amdgpu_vcn_ras_sw_init(struct amdgpu_device *adev)
 {
+	int err;
+	struct amdgpu_vcn_ras *ras;
+
 	if (!adev->vcn.ras)
-		return;
+		return 0;
 
-	amdgpu_ras_register_ras_block(adev, &adev->vcn.ras->ras_block);
+	ras = adev->vcn.ras;
+	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
+	if (err) {
+		dev_err(adev->dev, "Failed to register vcn ras block!\n");
+		return err;
+	}
 
-	strcpy(adev->vcn.ras->ras_block.ras_comm.name, "vcn");
-	adev->vcn.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__VCN;
-	adev->vcn.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__POISON;
-	adev->vcn.ras_if = &adev->vcn.ras->ras_block.ras_comm;
+	strcpy(ras->ras_block.ras_comm.name, "vcn");
+	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__VCN;
+	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__POISON;
+	adev->vcn.ras_if = &ras->ras_block.ras_comm;
 
-	/* If don't define special ras_late_init function, use default ras_late_init */
-	if (!adev->vcn.ras->ras_block.ras_late_init)
-		adev->vcn.ras->ras_block.ras_late_init = amdgpu_ras_block_late_init;
+	if (!ras->ras_block.ras_late_init)
+		ras->ras_block.ras_late_init = amdgpu_ras_block_late_init;
+
+	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
index d3e2af902907..c730949ece7d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
@@ -400,6 +400,6 @@ void amdgpu_debugfs_vcn_fwlog_init(struct amdgpu_device *adev,
 int amdgpu_vcn_process_poison_irq(struct amdgpu_device *adev,
 			struct amdgpu_irq_src *source,
 			struct amdgpu_iv_entry *entry);
-void amdgpu_vcn_set_ras_funcs(struct amdgpu_device *adev);
+int amdgpu_vcn_ras_sw_init(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index b9441ab457ea..3c0310576b3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -483,7 +483,7 @@ bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 				  struct amdgpu_job *job)
 {
 	struct amdgpu_device *adev = ring->adev;
-	unsigned vmhub = ring->funcs->vmhub;
+	unsigned vmhub = ring->vm_hub;
 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 
 	if (job->vmid == 0)
@@ -517,7 +517,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 		    bool need_pipe_sync)
 {
 	struct amdgpu_device *adev = ring->adev;
-	unsigned vmhub = ring->funcs->vmhub;
+	unsigned vmhub = ring->vm_hub;
 	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
 	bool spm_update_needed = job->spm_update_needed;
@@ -867,6 +867,8 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 					    pages_addr[idx - 1] + PAGE_SIZE))
 						break;
 				}
+				if (!contiguous)
+					count--;
 				num_entries = count *
 					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 			}
@@ -918,8 +920,8 @@ error_unlock:
 	return r;
 }
 
-void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
-				uint64_t *gtt_mem, uint64_t *cpu_mem)
+void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
+			  struct amdgpu_mem_stats *stats)
 {
 	struct amdgpu_bo_va *bo_va, *tmp;
 
@@ -927,41 +929,36 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
-		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
-				gtt_mem, cpu_mem);
+		amdgpu_bo_get_memory(bo_va->base.bo, stats);
 	}
 	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
-		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
-				gtt_mem, cpu_mem);
+		amdgpu_bo_get_memory(bo_va->base.bo, stats);
 	}
 	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
-		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
-				gtt_mem, cpu_mem);
+		amdgpu_bo_get_memory(bo_va->base.bo, stats);
 	}
 	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
-		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
-				gtt_mem, cpu_mem);
+		amdgpu_bo_get_memory(bo_va->base.bo, stats);
 	}
 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
-		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
-				gtt_mem, cpu_mem);
+		amdgpu_bo_get_memory(bo_va->base.bo, stats);
 	}
 	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
-		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
-				gtt_mem, cpu_mem);
+		amdgpu_bo_get_memory(bo_va->base.bo, stats);
 	}
 	spin_unlock(&vm->status_lock);
 }
+
 /**
  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 856a64bc7a89..6f085f0b4ef3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -40,6 +40,7 @@ struct amdgpu_bo_va;
 struct amdgpu_job;
 struct amdgpu_bo_list_entry;
 struct amdgpu_bo_vm;
+struct amdgpu_mem_stats;
 
 /*
  * GPUVM handling
@@ -457,8 +458,8 @@ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm);
 
 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
 				struct amdgpu_vm *vm);
-void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
-				uint64_t *gtt_mem, uint64_t *cpu_mem);
+void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
+			  struct amdgpu_mem_stats *stats);
 
 int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		       struct amdgpu_bo_vm *vmbo, bool immediate);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 01e42bdd8e4e..df63dc3bca18 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -673,6 +673,7 @@ void amdgpu_vm_pt_free_work(struct work_struct *work)
  * @adev: amdgpu device structure
  * @vm: amdgpu vm structure
  * @start: optional cursor where to start freeing PDs/PTs
+ * @unlocked: vm resv unlock status
  *
  * Free the page directory or page table level and all sub levels.
  */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
index 535cd6569bcc..349416e176a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -171,7 +171,7 @@ static void amdgpu_vm_sdma_copy_ptes(struct amdgpu_vm_update_params *p,
 
 	src += p->num_dw_left * 4;
 
-	pe += amdgpu_gmc_sign_extend(amdgpu_bo_gpu_offset_no_check(bo));
+	pe += amdgpu_bo_gpu_offset_no_check(bo);
 	trace_amdgpu_vm_copy_ptes(pe, src, count, p->immediate);
 
 	amdgpu_vm_copy_pte(p->adev, ib, pe, src, count);
@@ -198,7 +198,7 @@ static void amdgpu_vm_sdma_set_ptes(struct amdgpu_vm_update_params *p,
 {
 	struct amdgpu_ib *ib = p->job->ibs;
 
-	pe += amdgpu_gmc_sign_extend(amdgpu_bo_gpu_offset_no_check(bo));
+	pe += amdgpu_bo_gpu_offset_no_check(bo);
 	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags, p->immediate);
 	if (count < 3) {
 		amdgpu_vm_write_pte(p->adev, ib, pe, addr | flags,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 4340d08f7607..439925477fb8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -308,7 +308,7 @@ static const struct sysfs_ops amdgpu_xgmi_hive_ops = {
 	.show = amdgpu_xgmi_show_attrs,
 };
 
-struct kobj_type amdgpu_xgmi_hive_type = {
+static const struct kobj_type amdgpu_xgmi_hive_type = {
 	.release = amdgpu_xgmi_hive_release,
 	.sysfs_ops = &amdgpu_xgmi_hive_ops,
 	.default_groups = amdgpu_xgmi_hive_groups,
@@ -1048,12 +1048,30 @@ struct amdgpu_ras_block_hw_ops  xgmi_ras_hw_ops = {
 
 struct amdgpu_xgmi_ras xgmi_ras = {
 	.ras_block = {
-		.ras_comm = {
-			.name = "xgmi_wafl",
-			.block = AMDGPU_RAS_BLOCK__XGMI_WAFL,
-			.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
-		},
 		.hw_ops = &xgmi_ras_hw_ops,
 		.ras_late_init = amdgpu_xgmi_ras_late_init,
 	},
 };
+
+int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev)
+{
+	int err;
+	struct amdgpu_xgmi_ras *ras;
+
+	if (!adev->gmc.xgmi.ras)
+		return 0;
+
+	ras = adev->gmc.xgmi.ras;
+	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
+	if (err) {
+		dev_err(adev->dev, "Failed to register xgmi_wafl_pcs ras block!\n");
+		return err;
+	}
+
+	strcpy(ras->ras_block.ras_comm.name, "xgmi_wafl");
+	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__XGMI_WAFL;
+	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
+	adev->gmc.xgmi.ras_if = &ras->ras_block.ras_comm;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 30dcc1681b4e..86fbf56938f4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -73,5 +73,6 @@ static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
 		adev->gmc.xgmi.hive_id &&
 		adev->gmc.xgmi.hive_id == bo_adev->gmc.xgmi.hive_id);
 }
+int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index 6983acc456b2..f5b5ce1051a2 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -4461,6 +4461,7 @@ static int gfx_v10_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
 		ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
 	else
 		ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
+	ring->vm_hub = AMDGPU_GFXHUB_0;
 	sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 
 	irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
@@ -4489,6 +4490,7 @@ static int gfx_v10_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
 	ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
 	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
 				+ (ring_id * GFX10_MEC_HPD_SIZE);
+	ring->vm_hub = AMDGPU_GFXHUB_0;
 	sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 
 	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
@@ -7266,7 +7268,6 @@ static int gfx_v10_0_hw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int r;
-	uint32_t tmp;
 
 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
@@ -7285,17 +7286,9 @@ static int gfx_v10_0_hw_fini(void *handle)
 
 	if (amdgpu_sriov_vf(adev)) {
 		gfx_v10_0_cp_gfx_enable(adev, false);
-		/* Program KIQ position of RLC_CP_SCHEDULERS during destroy */
-		if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(10, 3, 0)) {
-			tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS_Sienna_Cichlid);
-			tmp &= 0xffffff00;
-			WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS_Sienna_Cichlid, tmp);
-		} else {
-			tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
-			tmp &= 0xffffff00;
-			WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
-		}
-
+		/* Remove the steps of clearing KIQ position.
+		 * It causes GFX hang when another Win guest is rendering.
+		 */
 		return 0;
 	}
 	gfx_v10_0_cp_enable(adev, false);
@@ -9258,7 +9251,6 @@ static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_gfx = {
 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.support_64bit_ptrs = true,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = gfx_v10_0_ring_get_rptr_gfx,
 	.get_wptr = gfx_v10_0_ring_get_wptr_gfx,
 	.set_wptr = gfx_v10_0_ring_set_wptr_gfx,
@@ -9313,7 +9305,6 @@ static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_compute = {
 	.align_mask = 0xff,
 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.support_64bit_ptrs = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = gfx_v10_0_ring_get_rptr_compute,
 	.get_wptr = gfx_v10_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v10_0_ring_set_wptr_compute,
@@ -9349,7 +9340,6 @@ static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_kiq = {
 	.align_mask = 0xff,
 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.support_64bit_ptrs = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = gfx_v10_0_ring_get_rptr_compute,
 	.get_wptr = gfx_v10_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v10_0_ring_set_wptr_compute,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index ecf8ceb53311..a9da0486467a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -866,6 +866,7 @@ static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
 		ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
 	else
 		ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
+	ring->vm_hub = AMDGPU_GFXHUB_0;
 	sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 
 	irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
@@ -896,6 +897,7 @@ static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
 	ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
 	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
 				+ (ring_id * GFX11_MEC_HPD_SIZE);
+	ring->vm_hub = AMDGPU_GFXHUB_0;
 	sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 
 	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
@@ -4671,11 +4673,24 @@ static int gfx_v11_0_post_soft_reset(void *handle)
 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
 {
 	uint64_t clock;
+	uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after;
 
 	amdgpu_gfx_off_ctrl(adev, false);
 	mutex_lock(&adev->gfx.gpu_clock_mutex);
-	clock = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER) |
-		((uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER) << 32ULL);
+	if (amdgpu_sriov_vf(adev)) {
+		clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
+		clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
+		clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
+		if (clock_counter_hi_pre != clock_counter_hi_after)
+			clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
+	} else {
+		clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
+		clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
+		clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
+		if (clock_counter_hi_pre != clock_counter_hi_after)
+			clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
+	}
+	clock = clock_counter_lo | (clock_counter_hi_after << 32ULL);
 	mutex_unlock(&adev->gfx.gpu_clock_mutex);
 	amdgpu_gfx_off_ctrl(adev, true);
 	return clock;
@@ -6191,7 +6206,6 @@ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.support_64bit_ptrs = true,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = gfx_v11_0_ring_get_rptr_gfx,
 	.get_wptr = gfx_v11_0_ring_get_wptr_gfx,
 	.set_wptr = gfx_v11_0_ring_set_wptr_gfx,
@@ -6239,7 +6253,6 @@ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {
 	.align_mask = 0xff,
 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.support_64bit_ptrs = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = gfx_v11_0_ring_get_rptr_compute,
 	.get_wptr = gfx_v11_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v11_0_ring_set_wptr_compute,
@@ -6275,7 +6288,6 @@ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
 	.align_mask = 0xff,
 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.support_64bit_ptrs = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = gfx_v11_0_ring_get_rptr_compute,
 	.get_wptr = gfx_v11_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v11_0_ring_set_wptr_compute,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0_3.c
index b07a72ca25d9..068b9586a223 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0_3.c
@@ -62,10 +62,18 @@ static int gfx_v11_0_3_rlc_gc_fed_irq(struct amdgpu_device *adev,
 		return -EINVAL;
 	}
 
-	ih_data.head = *ras_if;
-
 	dev_warn(adev->dev, "RLC %s FED IRQ\n", ras_if->name);
-	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
+
+	if (!amdgpu_sriov_vf(adev)) {
+		ih_data.head = *ras_if;
+		amdgpu_ras_interrupt_dispatch(adev, &ih_data);
+	} else {
+		if (adev->virt.ops && adev->virt.ops->ras_poison_handler)
+			adev->virt.ops->ras_poison_handler(adev);
+		else
+			dev_warn(adev->dev,
+				"No ras_poison_handler interface in SRIOV for %s!\n", ras_if->name);
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index ae09fc1cfe6b..adbcd8127c82 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -149,6 +149,16 @@ MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir                0x0026
 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX       1
 
+#define mmGOLDEN_TSC_COUNT_UPPER_Raven   0x007a
+#define mmGOLDEN_TSC_COUNT_UPPER_Raven_BASE_IDX 0
+#define mmGOLDEN_TSC_COUNT_LOWER_Raven   0x007b
+#define mmGOLDEN_TSC_COUNT_LOWER_Raven_BASE_IDX 0
+
+#define mmGOLDEN_TSC_COUNT_UPPER_Raven2   0x0068
+#define mmGOLDEN_TSC_COUNT_UPPER_Raven2_BASE_IDX 0
+#define mmGOLDEN_TSC_COUNT_LOWER_Raven2   0x0069
+#define mmGOLDEN_TSC_COUNT_LOWER_Raven2_BASE_IDX 0
+
 enum ta_ras_gfx_subblock {
 	/*CPC*/
 	TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
@@ -1995,6 +2005,7 @@ static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
 	ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
 	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
 				+ (ring_id * GFX9_MEC_HPD_SIZE);
+	ring->vm_hub = AMDGPU_GFXHUB_0;
 	sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 
 	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
@@ -2094,6 +2105,7 @@ static int gfx_v9_0_sw_init(void *handle)
 
 		/* disable scheduler on the real ring */
 		ring->no_scheduler = true;
+		ring->vm_hub = AMDGPU_GFXHUB_0;
 		r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
 				     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
 				     AMDGPU_RING_PRIO_DEFAULT, NULL);
@@ -2111,6 +2123,7 @@ static int gfx_v9_0_sw_init(void *handle)
 			ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
 			ring->is_sw_ring = true;
 			hw_prio = amdgpu_sw_ring_priority(i);
+			ring->vm_hub = AMDGPU_GFXHUB_0;
 			r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
 					     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,
 					     NULL);
@@ -3988,6 +4001,36 @@ static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
 		preempt_enable();
 		clock = clock_lo | (clock_hi << 32ULL);
 		break;
+	case IP_VERSION(9, 1, 0):
+		preempt_disable();
+		clock_hi = RREG32_SOC15_NO_KIQ(PWR, 0, mmGOLDEN_TSC_COUNT_UPPER_Raven);
+		clock_lo = RREG32_SOC15_NO_KIQ(PWR, 0, mmGOLDEN_TSC_COUNT_LOWER_Raven);
+		hi_check = RREG32_SOC15_NO_KIQ(PWR, 0, mmGOLDEN_TSC_COUNT_UPPER_Raven);
+		/* The PWR TSC clock frequency is 100MHz, which sets 32-bit carry over
+		 * roughly every 42 seconds.
+		 */
+		if (hi_check != clock_hi) {
+			clock_lo = RREG32_SOC15_NO_KIQ(PWR, 0, mmGOLDEN_TSC_COUNT_LOWER_Raven);
+			clock_hi = hi_check;
+		}
+		preempt_enable();
+		clock = clock_lo | (clock_hi << 32ULL);
+		break;
+	case IP_VERSION(9, 2, 2):
+		preempt_disable();
+		clock_hi = RREG32_SOC15_NO_KIQ(PWR, 0, mmGOLDEN_TSC_COUNT_UPPER_Raven2);
+		clock_lo = RREG32_SOC15_NO_KIQ(PWR, 0, mmGOLDEN_TSC_COUNT_LOWER_Raven2);
+		hi_check = RREG32_SOC15_NO_KIQ(PWR, 0, mmGOLDEN_TSC_COUNT_UPPER_Raven2);
+		/* The PWR TSC clock frequency is 100MHz, which sets 32-bit carry over
+		 * roughly every 42 seconds.
+		 */
+		if (hi_check != clock_hi) {
+			clock_lo = RREG32_SOC15_NO_KIQ(PWR, 0, mmGOLDEN_TSC_COUNT_LOWER_Raven2);
+			clock_hi = hi_check;
+		}
+		preempt_enable();
+		clock = clock_lo | (clock_hi << 32ULL);
+		break;
 	default:
 		amdgpu_gfx_off_ctrl(adev, false);
 		mutex_lock(&adev->gfx.gpu_clock_mutex);
@@ -6750,7 +6793,6 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.support_64bit_ptrs = true,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = gfx_v9_0_ring_get_rptr_gfx,
 	.get_wptr = gfx_v9_0_ring_get_wptr_gfx,
 	.set_wptr = gfx_v9_0_ring_set_wptr_gfx,
@@ -6804,7 +6846,6 @@ static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {
 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.support_64bit_ptrs = true,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = amdgpu_sw_ring_get_rptr_gfx,
 	.get_wptr = amdgpu_sw_ring_get_wptr_gfx,
 	.set_wptr = amdgpu_sw_ring_set_wptr_gfx,
@@ -6858,7 +6899,6 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
 	.align_mask = 0xff,
 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.support_64bit_ptrs = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = gfx_v9_0_ring_get_rptr_compute,
 	.get_wptr = gfx_v9_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v9_0_ring_set_wptr_compute,
@@ -6897,7 +6937,6 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
 	.align_mask = 0xff,
 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.support_64bit_ptrs = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = gfx_v9_0_ring_get_rptr_compute,
 	.get_wptr = gfx_v9_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v9_0_ring_set_wptr_compute,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
new file mode 100644
index 000000000000..5f8500577c02
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
@@ -0,0 +1,430 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/firmware.h>
+
+#include "amdgpu.h"
+#include "amdgpu_gfx.h"
+#include "soc15.h"
+#include "soc15_common.h"
+#include "vega10_enum.h"
+
+#include "gc/gc_9_4_3_offset.h"
+#include "gc/gc_9_4_3_sh_mask.h"
+
+#include "gfx_v9_4_3.h"
+
+#define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
+
+static uint64_t gfx_v9_4_3_get_gpu_clock_counter(struct amdgpu_device *adev)
+{
+	uint64_t clock;
+
+	amdgpu_gfx_off_ctrl(adev, false);
+	mutex_lock(&adev->gfx.gpu_clock_mutex);
+	WREG32_SOC15(GC, 0, regRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
+	clock = (uint64_t)RREG32_SOC15(GC, 0, regRLC_GPU_CLOCK_COUNT_LSB) |
+		((uint64_t)RREG32_SOC15(GC, 0, regRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
+	mutex_unlock(&adev->gfx.gpu_clock_mutex);
+	amdgpu_gfx_off_ctrl(adev, true);
+
+	return clock;
+}
+
+static void gfx_v9_4_3_select_se_sh(struct amdgpu_device *adev,
+				    u32 se_num,
+				    u32 sh_num,
+				    u32 instance)
+{
+	u32 data;
+
+	if (instance == 0xffffffff)
+		data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
+				     INSTANCE_BROADCAST_WRITES, 1);
+	else
+		data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
+				     INSTANCE_INDEX, instance);
+
+	if (se_num == 0xffffffff)
+		data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+				     SE_BROADCAST_WRITES, 1);
+	else
+		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
+
+	if (sh_num == 0xffffffff)
+		data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+				     SH_BROADCAST_WRITES, 1);
+	else
+		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
+
+	WREG32_SOC15_RLC_SHADOW_EX(reg, GC, 0, regGRBM_GFX_INDEX, data);
+}
+
+static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
+{
+	WREG32_SOC15_RLC(GC, 0, regSQ_IND_INDEX,
+		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
+		(address << SQ_IND_INDEX__INDEX__SHIFT) |
+		(SQ_IND_INDEX__FORCE_READ_MASK));
+	return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
+}
+
+static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
+			   uint32_t wave, uint32_t thread,
+			   uint32_t regno, uint32_t num, uint32_t *out)
+{
+	WREG32_SOC15_RLC(GC, 0, regSQ_IND_INDEX,
+		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
+		(regno << SQ_IND_INDEX__INDEX__SHIFT) |
+		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
+		(SQ_IND_INDEX__FORCE_READ_MASK) |
+		(SQ_IND_INDEX__AUTO_INCR_MASK));
+	while (num--)
+		*(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
+}
+
+static void gfx_v9_4_3_read_wave_data(struct amdgpu_device *adev,
+				      uint32_t simd, uint32_t wave,
+				      uint32_t *dst, int *no_fields)
+{
+	/* type 1 wave data */
+	dst[(*no_fields)++] = 1;
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
+}
+
+static void gfx_v9_4_3_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
+				       uint32_t wave, uint32_t start,
+				       uint32_t size, uint32_t *dst)
+{
+	wave_read_regs(adev, simd, wave, 0,
+		       start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
+}
+
+static void gfx_v9_4_3_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
+				       uint32_t wave, uint32_t thread,
+				       uint32_t start, uint32_t size,
+				       uint32_t *dst)
+{
+	wave_read_regs(adev, simd, wave, thread,
+		       start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
+}
+
+static void gfx_v9_4_3_select_me_pipe_q(struct amdgpu_device *adev,
+					u32 me, u32 pipe, u32 q, u32 vm)
+{
+	soc15_grbm_select(adev, me, pipe, q, vm);
+}
+
+static bool gfx_v9_4_3_is_rlc_enabled(struct amdgpu_device *adev)
+{
+	uint32_t rlc_setting;
+
+	/* if RLC is not enabled, do nothing */
+	rlc_setting = RREG32_SOC15(GC, 0, regRLC_CNTL);
+	if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
+		return false;
+
+	return true;
+}
+
+static void gfx_v9_4_3_set_safe_mode(struct amdgpu_device *adev)
+{
+	uint32_t data;
+	unsigned i;
+
+	data = RLC_SAFE_MODE__CMD_MASK;
+	data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
+	WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
+
+	/* wait for RLC_SAFE_MODE */
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
+			break;
+		udelay(1);
+	}
+}
+
+static void gfx_v9_4_3_unset_safe_mode(struct amdgpu_device *adev)
+{
+	uint32_t data;
+
+	data = RLC_SAFE_MODE__CMD_MASK;
+	WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
+}
+
+static int gfx_v9_4_3_rlc_init(struct amdgpu_device *adev)
+{
+	/* init spm vmid with 0xf */
+	if (adev->gfx.rlc.funcs->update_spm_vmid)
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
+
+	return 0;
+}
+
+static void gfx_v9_4_3_wait_for_rlc_serdes(struct amdgpu_device *adev)
+{
+	u32 i, j, k;
+	u32 mask;
+
+	mutex_lock(&adev->grbm_idx_mutex);
+	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
+		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
+			gfx_v9_4_3_select_se_sh(adev, i, j, 0xffffffff);
+			for (k = 0; k < adev->usec_timeout; k++) {
+				if (RREG32_SOC15(GC, 0, regRLC_SERDES_CU_MASTER_BUSY) == 0)
+					break;
+				udelay(1);
+			}
+			if (k == adev->usec_timeout) {
+				gfx_v9_4_3_select_se_sh(adev, 0xffffffff,
+						      0xffffffff, 0xffffffff);
+				mutex_unlock(&adev->grbm_idx_mutex);
+				DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
+					 i, j);
+				return;
+			}
+		}
+	}
+	gfx_v9_4_3_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+	mutex_unlock(&adev->grbm_idx_mutex);
+
+	mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
+		RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
+		RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
+		RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
+	for (k = 0; k < adev->usec_timeout; k++) {
+		if ((RREG32_SOC15(GC, 0, regRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
+			break;
+		udelay(1);
+	}
+}
+
+static void gfx_v9_4_3_enable_gui_idle_interrupt(struct amdgpu_device *adev,
+					       bool enable)
+{
+	u32 tmp;
+
+	/* These interrupts should be enabled to drive DS clock */
+
+	tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0);
+
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
+	if (adev->gfx.num_gfx_rings)
+		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
+
+	WREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0, tmp);
+}
+
+static void gfx_v9_4_3_rlc_stop(struct amdgpu_device *adev)
+{
+	WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
+	gfx_v9_4_3_enable_gui_idle_interrupt(adev, false);
+	gfx_v9_4_3_wait_for_rlc_serdes(adev);
+}
+
+static void gfx_v9_4_3_rlc_reset(struct amdgpu_device *adev)
+{
+	WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
+	udelay(50);
+	WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
+	udelay(50);
+}
+
+static void gfx_v9_4_3_rlc_start(struct amdgpu_device *adev)
+{
+#ifdef AMDGPU_RLC_DEBUG_RETRY
+	u32 rlc_ucode_ver;
+#endif
+
+	WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
+	udelay(50);
+
+	/* carrizo do enable cp interrupt after cp inited */
+	if (!(adev->flags & AMD_IS_APU)) {
+		gfx_v9_4_3_enable_gui_idle_interrupt(adev, true);
+		udelay(50);
+	}
+
+#ifdef AMDGPU_RLC_DEBUG_RETRY
+	/* RLC_GPM_GENERAL_6 : RLC Ucode version */
+	rlc_ucode_ver = RREG32_SOC15(GC, 0, regRLC_GPM_GENERAL_6);
+	if (rlc_ucode_ver == 0x108) {
+		dev_info(adev->dev,
+			 "Using rlc debug ucode. regRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
+			 rlc_ucode_ver, adev->gfx.rlc_fw_version);
+		/* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
+		 * default is 0x9C4 to create a 100us interval */
+		WREG32_SOC15(GC, 0, regRLC_GPM_TIMER_INT_3, 0x9C4);
+		/* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
+		 * to disable the page fault retry interrupts, default is
+		 * 0x100 (256) */
+		WREG32_SOC15(GC, 0, regRLC_GPM_GENERAL_12, 0x100);
+	}
+#endif
+}
+
+static int gfx_v9_4_3_rlc_load_microcode(struct amdgpu_device *adev)
+{
+	const struct rlc_firmware_header_v2_0 *hdr;
+	const __le32 *fw_data;
+	unsigned i, fw_size;
+
+	if (!adev->gfx.rlc_fw)
+		return -EINVAL;
+
+	hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
+	amdgpu_ucode_print_rlc_hdr(&hdr->header);
+
+	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+			   le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
+
+	WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
+			RLCG_UCODE_LOADING_START_ADDRESS);
+	for (i = 0; i < fw_size; i++) {
+		if (amdgpu_emu_mode == 1 && i % 100 == 0) {
+			dev_info(adev->dev, "Write RLC ucode data %u DWs\n", i);
+			msleep(1);
+		}
+		WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
+	}
+	WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
+
+	return 0;
+}
+
+static int gfx_v9_4_3_rlc_resume(struct amdgpu_device *adev)
+{
+	int r;
+
+	adev->gfx.rlc.funcs->stop(adev);
+
+	/* disable CG */
+	WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
+
+	/* TODO: revisit pg function */
+	/* gfx_v9_4_3_init_pg(adev);*/
+
+	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
+		/* legacy rlc firmware loading */
+		r = gfx_v9_4_3_rlc_load_microcode(adev);
+		if (r)
+			return r;
+	}
+
+	adev->gfx.rlc.funcs->start(adev);
+
+	return 0;
+}
+
+static void gfx_v9_4_3_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
+{
+	u32 reg, data;
+
+	reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
+	if (amdgpu_sriov_is_pp_one_vf(adev))
+		data = RREG32_NO_KIQ(reg);
+	else
+		data = RREG32(reg);
+
+	data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
+	data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
+
+	if (amdgpu_sriov_is_pp_one_vf(adev))
+		WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
+	else
+		WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
+}
+
+static const struct soc15_reg_rlcg rlcg_access_gc_9_4_3[] = {
+	{SOC15_REG_ENTRY(GC, 0, regGRBM_GFX_INDEX)},
+	{SOC15_REG_ENTRY(GC, 0, regSQ_IND_INDEX)},
+};
+
+static bool gfx_v9_4_3_check_rlcg_range(struct amdgpu_device *adev,
+					uint32_t offset,
+					struct soc15_reg_rlcg *entries, int arr_size)
+{
+	int i;
+	uint32_t reg;
+
+	if (!entries)
+		return false;
+
+	for (i = 0; i < arr_size; i++) {
+		const struct soc15_reg_rlcg *entry;
+
+		entry = &entries[i];
+		reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
+		if (offset == reg)
+			return true;
+	}
+
+	return false;
+}
+
+static bool gfx_v9_4_3_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
+{
+	return gfx_v9_4_3_check_rlcg_range(adev, offset,
+					(void *)rlcg_access_gc_9_4_3,
+					ARRAY_SIZE(rlcg_access_gc_9_4_3));
+}
+
+const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = {
+	.get_gpu_clock_counter = &gfx_v9_4_3_get_gpu_clock_counter,
+	.select_se_sh = &gfx_v9_4_3_select_se_sh,
+	.read_wave_data = &gfx_v9_4_3_read_wave_data,
+	.read_wave_sgprs = &gfx_v9_4_3_read_wave_sgprs,
+	.read_wave_vgprs = &gfx_v9_4_3_read_wave_vgprs,
+	.select_me_pipe_q = &gfx_v9_4_3_select_me_pipe_q,
+};
+
+const struct amdgpu_rlc_funcs gfx_v9_4_3_rlc_funcs = {
+	.is_rlc_enabled = gfx_v9_4_3_is_rlc_enabled,
+	.set_safe_mode = gfx_v9_4_3_set_safe_mode,
+	.unset_safe_mode = gfx_v9_4_3_unset_safe_mode,
+	.init = gfx_v9_4_3_rlc_init,
+	.resume = gfx_v9_4_3_rlc_resume,
+	.stop = gfx_v9_4_3_rlc_stop,
+	.reset = gfx_v9_4_3_rlc_reset,
+	.start = gfx_v9_4_3_rlc_start,
+	.update_spm_vmid = gfx_v9_4_3_update_spm_vmid,
+	.is_rlcg_access_range = gfx_v9_4_3_is_rlcg_access_range,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.h b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.h
new file mode 100644
index 000000000000..84e69701b81a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __GFX_V9_4_3_H__
+#define __GFX_V9_4_3_H__
+
+extern const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs;
+extern const struct amdgpu_rlc_funcs gfx_v9_4_3_rlc_funcs;
+
+#endif /* __GFX_V9_4_3_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
new file mode 100644
index 000000000000..c59c6c85fbff
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
@@ -0,0 +1,471 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "gfxhub_v1_2.h"
+#include "gfxhub_v1_1.h"
+
+#include "gc/gc_9_4_3_offset.h"
+#include "gc/gc_9_4_3_sh_mask.h"
+#include "vega10_enum.h"
+
+#include "soc15_common.h"
+
+#define regVM_L2_CNTL3_DEFAULT	0x80100007
+#define regVM_L2_CNTL4_DEFAULT	0x000000c1
+
+static u64 gfxhub_v1_2_get_mc_fb_offset(struct amdgpu_device *adev)
+{
+	return (u64)RREG32_SOC15(GC, 0, regMC_VM_FB_OFFSET) << 24;
+}
+
+static void gfxhub_v1_2_setup_vm_pt_regs(struct amdgpu_device *adev,
+					 uint32_t vmid,
+					 uint64_t page_table_base)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+
+	WREG32_SOC15_OFFSET(GC, 0, regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+			    hub->ctx_addr_distance * vmid,
+			    lower_32_bits(page_table_base));
+
+	WREG32_SOC15_OFFSET(GC, 0, regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+			    hub->ctx_addr_distance * vmid,
+			    upper_32_bits(page_table_base));
+}
+
+static void gfxhub_v1_2_init_gart_aperture_regs(struct amdgpu_device *adev)
+{
+	uint64_t pt_base;
+
+	if (adev->gmc.pdb0_bo)
+		pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo);
+	else
+		pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+
+	gfxhub_v1_2_setup_vm_pt_regs(adev, 0, pt_base);
+
+	/* If use GART for FB translation, vmid0 page table covers both
+	 * vram and system memory (gart)
+	 */
+	if (adev->gmc.pdb0_bo) {
+		WREG32_SOC15(GC, 0, regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+				(u32)(adev->gmc.fb_start >> 12));
+		WREG32_SOC15(GC, 0, regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+				(u32)(adev->gmc.fb_start >> 44));
+
+		WREG32_SOC15(GC, 0, regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+				(u32)(adev->gmc.gart_end >> 12));
+		WREG32_SOC15(GC, 0, regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+				(u32)(adev->gmc.gart_end >> 44));
+	} else {
+		WREG32_SOC15(GC, 0, regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+				(u32)(adev->gmc.gart_start >> 12));
+		WREG32_SOC15(GC, 0, regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+				(u32)(adev->gmc.gart_start >> 44));
+
+		WREG32_SOC15(GC, 0, regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+				(u32)(adev->gmc.gart_end >> 12));
+		WREG32_SOC15(GC, 0, regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+				(u32)(adev->gmc.gart_end >> 44));
+	}
+}
+
+static void gfxhub_v1_2_init_system_aperture_regs(struct amdgpu_device *adev)
+{
+	uint64_t value;
+	uint32_t tmp;
+
+	/* Program the AGP BAR */
+	WREG32_SOC15_RLC(GC, 0, regMC_VM_AGP_BASE, 0);
+	WREG32_SOC15_RLC(GC, 0, regMC_VM_AGP_BOT, adev->gmc.agp_start >> 24);
+	WREG32_SOC15_RLC(GC, 0, regMC_VM_AGP_TOP, adev->gmc.agp_end >> 24);
+
+	if (!amdgpu_sriov_vf(adev) || adev->asic_type <= CHIP_VEGA10) {
+		/* Program the system aperture low logical page number. */
+		WREG32_SOC15_RLC(GC, 0, regMC_VM_SYSTEM_APERTURE_LOW_ADDR,
+			min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18);
+
+		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
+			/*
+			* Raven2 has a HW issue that it is unable to use the
+			* vram which is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR.
+			* So here is the workaround that increase system
+			* aperture high address (add 1) to get rid of the VM
+			* fault and hardware hang.
+			*/
+			WREG32_SOC15_RLC(GC, 0,
+					 regMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+					 max((adev->gmc.fb_end >> 18) + 0x1,
+					     adev->gmc.agp_end >> 18));
+		else
+			WREG32_SOC15_RLC(GC, 0,
+				regMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+				max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
+
+		/* Set default page address. */
+		value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
+		WREG32_SOC15(GC, 0, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
+			     (u32)(value >> 12));
+		WREG32_SOC15(GC, 0, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
+			     (u32)(value >> 44));
+
+		/* Program "protection fault". */
+		WREG32_SOC15(GC, 0, regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
+			     (u32)(adev->dummy_page_addr >> 12));
+		WREG32_SOC15(GC, 0, regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
+			     (u32)((u64)adev->dummy_page_addr >> 44));
+
+		tmp = RREG32_SOC15(GC, 0, regVM_L2_PROTECTION_FAULT_CNTL2);
+		tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2,
+				    ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+		WREG32_SOC15(GC, 0, regVM_L2_PROTECTION_FAULT_CNTL2, tmp);
+	}
+
+	/* In the case squeezing vram into GART aperture, we don't use
+	 * FB aperture and AGP aperture. Disable them.
+	 */
+	if (adev->gmc.pdb0_bo) {
+		WREG32_SOC15(GC, 0, regMC_VM_FB_LOCATION_TOP, 0);
+		WREG32_SOC15(GC, 0, regMC_VM_FB_LOCATION_BASE, 0x00FFFFFF);
+		WREG32_SOC15(GC, 0, regMC_VM_AGP_TOP, 0);
+		WREG32_SOC15(GC, 0, regMC_VM_AGP_BOT, 0xFFFFFF);
+		WREG32_SOC15(GC, 0, regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 0x3FFFFFFF);
+		WREG32_SOC15(GC, 0, regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0);
+	}
+}
+
+static void gfxhub_v1_2_init_tlb_regs(struct amdgpu_device *adev)
+{
+	uint32_t tmp;
+
+	/* Setup TLB control */
+	tmp = RREG32_SOC15(GC, 0, regMC_VM_MX_L1_TLB_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+			    ENABLE_ADVANCED_DRIVER_MODEL, 1);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+			    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+			    MTYPE, MTYPE_UC);/* XXX for emulation. */
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
+
+	WREG32_SOC15_RLC(GC, 0, regMC_VM_MX_L1_TLB_CNTL, tmp);
+}
+
+static void gfxhub_v1_2_init_cache_regs(struct amdgpu_device *adev)
+{
+	uint32_t tmp;
+
+	/* Setup L2 cache */
+	tmp = RREG32_SOC15(GC, 0, regVM_L2_CNTL);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
+	/* XXX for emulation, Refer to closed source code.*/
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
+			    0);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
+	WREG32_SOC15_RLC(GC, 0, regVM_L2_CNTL, tmp);
+
+	tmp = RREG32_SOC15(GC, 0, regVM_L2_CNTL2);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
+	WREG32_SOC15_RLC(GC, 0, regVM_L2_CNTL2, tmp);
+
+	tmp = regVM_L2_CNTL3_DEFAULT;
+	if (adev->gmc.translate_further) {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
+				    L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
+	} else {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
+				    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
+	}
+	WREG32_SOC15_RLC(GC, 0, regVM_L2_CNTL3, tmp);
+
+	tmp = regVM_L2_CNTL4_DEFAULT;
+	if (adev->gmc.xgmi.connected_to_cpu) {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 1);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 1);
+	} else {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
+	}
+	WREG32_SOC15_RLC(GC, 0, regVM_L2_CNTL4, tmp);
+}
+
+static void gfxhub_v1_2_enable_system_domain(struct amdgpu_device *adev)
+{
+	uint32_t tmp;
+
+	tmp = RREG32_SOC15(GC, 0, regVM_CONTEXT0_CNTL);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH,
+			adev->gmc.vmid0_page_table_depth);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_BLOCK_SIZE,
+			adev->gmc.vmid0_page_table_block_size);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL,
+			    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
+	WREG32_SOC15(GC, 0, regVM_CONTEXT0_CNTL, tmp);
+}
+
+static void gfxhub_v1_2_disable_identity_aperture(struct amdgpu_device *adev)
+{
+	WREG32_SOC15(GC, 0, regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
+		     0XFFFFFFFF);
+	WREG32_SOC15(GC, 0, regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
+		     0x0000000F);
+
+	WREG32_SOC15(GC, 0, regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
+		     0);
+	WREG32_SOC15(GC, 0, regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
+		     0);
+
+	WREG32_SOC15(GC, 0, regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
+	WREG32_SOC15(GC, 0, regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);
+
+}
+
+static void gfxhub_v1_2_setup_vmid_config(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+	unsigned num_level, block_size;
+	uint32_t tmp;
+	int i;
+
+	num_level = adev->vm_manager.num_level;
+	block_size = adev->vm_manager.block_size;
+	if (adev->gmc.translate_further)
+		num_level -= 1;
+	else
+		block_size -= 9;
+
+	for (i = 0; i <= 14; i++) {
+		tmp = RREG32_SOC15_OFFSET(GC, 0, regVM_CONTEXT1_CNTL, i);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
+				    num_level);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
+				    1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    PAGE_TABLE_BLOCK_SIZE,
+				    block_size);
+		/* Send no-retry XNACK on fault to suppress VM fault storm.
+		 * On Aldebaran, XNACK can be enabled in the SQ per-process.
+		 * Retry faults need to be enabled for that to work.
+		 */
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
+				    !adev->gmc.noretry ||
+				    adev->asic_type == CHIP_ALDEBARAN);
+		WREG32_SOC15_OFFSET(GC, 0, regVM_CONTEXT1_CNTL,
+				    i * hub->ctx_distance, tmp);
+		WREG32_SOC15_OFFSET(GC, 0,
+				    regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
+				    i * hub->ctx_addr_distance, 0);
+		WREG32_SOC15_OFFSET(GC, 0,
+				    regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
+				    i * hub->ctx_addr_distance, 0);
+		WREG32_SOC15_OFFSET(GC, 0,
+				    regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
+				    i * hub->ctx_addr_distance,
+				    lower_32_bits(adev->vm_manager.max_pfn - 1));
+		WREG32_SOC15_OFFSET(GC, 0,
+				    regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
+				    i * hub->ctx_addr_distance,
+				    upper_32_bits(adev->vm_manager.max_pfn - 1));
+	}
+}
+
+static void gfxhub_v1_2_program_invalidation(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+	unsigned i;
+
+	for (i = 0 ; i < 18; ++i) {
+		WREG32_SOC15_OFFSET(GC, 0, regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
+				    i * hub->eng_addr_distance, 0xffffffff);
+		WREG32_SOC15_OFFSET(GC, 0, regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
+				    i * hub->eng_addr_distance, 0x1f);
+	}
+}
+
+static int gfxhub_v1_2_gart_enable(struct amdgpu_device *adev)
+{
+	if (amdgpu_sriov_vf(adev) && adev->asic_type != CHIP_ARCTURUS) {
+		/*
+		 * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
+		 * VF copy registers so vbios post doesn't program them, for
+		 * SRIOV driver need to program them
+		 */
+		WREG32_SOC15_RLC(GC, 0, regMC_VM_FB_LOCATION_BASE,
+			     adev->gmc.vram_start >> 24);
+		WREG32_SOC15_RLC(GC, 0, regMC_VM_FB_LOCATION_TOP,
+			     adev->gmc.vram_end >> 24);
+	}
+
+	/* GART Enable. */
+	gfxhub_v1_2_init_gart_aperture_regs(adev);
+	gfxhub_v1_2_init_system_aperture_regs(adev);
+	gfxhub_v1_2_init_tlb_regs(adev);
+	if (!amdgpu_sriov_vf(adev))
+		gfxhub_v1_2_init_cache_regs(adev);
+
+	gfxhub_v1_2_enable_system_domain(adev);
+	if (!amdgpu_sriov_vf(adev))
+		gfxhub_v1_2_disable_identity_aperture(adev);
+	gfxhub_v1_2_setup_vmid_config(adev);
+	gfxhub_v1_2_program_invalidation(adev);
+
+	return 0;
+}
+
+static void gfxhub_v1_2_gart_disable(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+	u32 tmp;
+	u32 i;
+
+	/* Disable all tables */
+	for (i = 0; i < 16; i++)
+		WREG32_SOC15_OFFSET(GC, 0, regVM_CONTEXT0_CNTL,
+				    i * hub->ctx_distance, 0);
+
+	/* Setup TLB control */
+	tmp = RREG32_SOC15(GC, 0, regMC_VM_MX_L1_TLB_CNTL);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
+	tmp = REG_SET_FIELD(tmp,
+				MC_VM_MX_L1_TLB_CNTL,
+				ENABLE_ADVANCED_DRIVER_MODEL,
+				0);
+	WREG32_SOC15_RLC(GC, 0, regMC_VM_MX_L1_TLB_CNTL, tmp);
+
+	/* Setup L2 cache */
+	tmp = RREG32_SOC15(GC, 0, regVM_L2_CNTL);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
+	WREG32_SOC15(GC, 0, regVM_L2_CNTL, tmp);
+	WREG32_SOC15(GC, 0, regVM_L2_CNTL3, 0);
+}
+
+/**
+ * gfxhub_v1_2_set_fault_enable_default - update GART/VM fault handling
+ *
+ * @adev: amdgpu_device pointer
+ * @value: true redirects VM faults to the default page
+ */
+static void gfxhub_v1_2_set_fault_enable_default(struct amdgpu_device *adev,
+						 bool value)
+{
+	u32 tmp;
+	tmp = RREG32_SOC15(GC, 0, regVM_L2_PROTECTION_FAULT_CNTL);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp,
+			VM_L2_PROTECTION_FAULT_CNTL,
+			TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
+			value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	if (!value) {
+		tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+				CRASH_ON_NO_RETRY_FAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+				CRASH_ON_RETRY_FAULT, 1);
+	}
+	WREG32_SOC15(GC, 0, regVM_L2_PROTECTION_FAULT_CNTL, tmp);
+}
+
+static void gfxhub_v1_2_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+
+	hub->ctx0_ptb_addr_lo32 =
+		SOC15_REG_OFFSET(GC, 0,
+				regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
+	hub->ctx0_ptb_addr_hi32 =
+		SOC15_REG_OFFSET(GC, 0,
+				regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
+	hub->vm_inv_eng0_sem =
+		SOC15_REG_OFFSET(GC, 0, regVM_INVALIDATE_ENG0_SEM);
+	hub->vm_inv_eng0_req =
+		SOC15_REG_OFFSET(GC, 0, regVM_INVALIDATE_ENG0_REQ);
+	hub->vm_inv_eng0_ack =
+		SOC15_REG_OFFSET(GC, 0, regVM_INVALIDATE_ENG0_ACK);
+	hub->vm_context0_cntl =
+		SOC15_REG_OFFSET(GC, 0, regVM_CONTEXT0_CNTL);
+	hub->vm_l2_pro_fault_status =
+		SOC15_REG_OFFSET(GC, 0, regVM_L2_PROTECTION_FAULT_STATUS);
+	hub->vm_l2_pro_fault_cntl =
+		SOC15_REG_OFFSET(GC, 0, regVM_L2_PROTECTION_FAULT_CNTL);
+
+	hub->ctx_distance = regVM_CONTEXT1_CNTL - regVM_CONTEXT0_CNTL;
+	hub->ctx_addr_distance = regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 -
+		regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
+	hub->eng_distance = regVM_INVALIDATE_ENG1_REQ - regVM_INVALIDATE_ENG0_REQ;
+	hub->eng_addr_distance = regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 -
+		regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
+}
+
+
+const struct amdgpu_gfxhub_funcs gfxhub_v1_2_funcs = {
+	.get_mc_fb_offset = gfxhub_v1_2_get_mc_fb_offset,
+	.setup_vm_pt_regs = gfxhub_v1_2_setup_vm_pt_regs,
+	.gart_enable = gfxhub_v1_2_gart_enable,
+	.gart_disable = gfxhub_v1_2_gart_disable,
+	.set_fault_enable_default = gfxhub_v1_2_set_fault_enable_default,
+	.init = gfxhub_v1_2_init,
+	.get_xgmi_info = gfxhub_v1_1_get_xgmi_info,
+};
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.h
index c26e7258a91c..e2d508f5a7ee 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
+ * Copyright 2022 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -19,9 +19,11 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
  *
- * Authors: AMD
- *
  */
 
-// TODO - remove this file after external build dependencies is resolved.
-/* NOTE: This file is pending to be removed, do not add new code to this file */
\ No newline at end of file
+#ifndef __GFXHUB_V1_2_H__
+#define __GFXHUB_V1_2_H__
+
+extern const struct amdgpu_gfxhub_funcs gfxhub_v1_2_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
index be0d0f47415e..13712640fa46 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
@@ -417,34 +417,12 @@ static void gfxhub_v3_0_set_fault_enable_default(struct amdgpu_device *adev,
 	tmp = REG_SET_FIELD(tmp, CP_DEBUG, CPG_UTCL1_ERROR_HALT_DISABLE, 1);
 	WREG32_SOC15(GC, 0, regCP_DEBUG, tmp);
 
-	/**
-	 * Set GRBM_GFX_INDEX in broad cast mode
-	 * before programming GL1C_UTCL0_CNTL1 and SQG_CONFIG
-	 */
-	WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, regGRBM_GFX_INDEX_DEFAULT);
-
-	/**
-	 * Retry respond mode: RETRY
-	 * Error (no retry) respond mode: SUCCESS
-	 */
-	tmp = RREG32_SOC15(GC, 0, regGL1C_UTCL0_CNTL1);
-	tmp = REG_SET_FIELD(tmp, GL1C_UTCL0_CNTL1, RESP_MODE, 0);
-	tmp = REG_SET_FIELD(tmp, GL1C_UTCL0_CNTL1, RESP_FAULT_MODE, 0x2);
-	WREG32_SOC15(GC, 0, regGL1C_UTCL0_CNTL1, tmp);
-
 	/* These registers are not accessible to VF-SRIOV.
 	 * The PF will program them instead.
 	 */
 	if (amdgpu_sriov_vf(adev))
 		return;
 
-	/* Disable SQ XNACK interrupt for all VMIDs */
-	tmp = RREG32_SOC15(GC, 0, regSQG_CONFIG);
-	tmp = REG_SET_FIELD(tmp, SQG_CONFIG, XNACK_INTR_MASK,
-			    SQG_CONFIG__XNACK_INTR_MASK_MASK >>
-			    SQG_CONFIG__XNACK_INTR_MASK__SHIFT);
-	WREG32_SOC15(GC, 0, regSQG_CONFIG, tmp);
-
 	tmp = RREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL);
 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
 			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index ab2556ca984e..7d6f4a68f416 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -479,8 +479,8 @@ static int gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
 static uint64_t gmc_v10_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
 					     unsigned vmid, uint64_t pd_addr)
 {
-	bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 	uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
 	unsigned eng = ring->vm_inv_eng;
 
@@ -534,7 +534,7 @@ static void gmc_v10_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid
 	if (ring->is_mes_queue)
 		return;
 
-	if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
+	if (ring->vm_hub == AMDGPU_GFXHUB_0)
 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
 	else
 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
@@ -699,25 +699,8 @@ static void gmc_v10_0_set_umc_funcs(struct amdgpu_device *adev)
 	default:
 		break;
 	}
-	if (adev->umc.ras) {
-		amdgpu_ras_register_ras_block(adev, &adev->umc.ras->ras_block);
-
-		strcpy(adev->umc.ras->ras_block.ras_comm.name, "umc");
-		adev->umc.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__UMC;
-		adev->umc.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
-		adev->umc.ras_if = &adev->umc.ras->ras_block.ras_comm;
-
-		/* If don't define special ras_late_init function, use default ras_late_init */
-		if (!adev->umc.ras->ras_block.ras_late_init)
-				adev->umc.ras->ras_block.ras_late_init = amdgpu_umc_ras_late_init;
-
-		/* If not defined special ras_cb function, use default ras_cb */
-		if (!adev->umc.ras->ras_block.ras_cb)
-			adev->umc.ras->ras_block.ras_cb = amdgpu_umc_process_ras_data_cb;
-	}
 }
 
-
 static void gmc_v10_0_set_mmhub_funcs(struct amdgpu_device *adev)
 {
 	switch (adev->ip_versions[MMHUB_HWIP][0]) {
@@ -754,7 +737,6 @@ static void gmc_v10_0_set_gfxhub_funcs(struct amdgpu_device *adev)
 
 static int gmc_v10_0_early_init(void *handle)
 {
-	int r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	gmc_v10_0_set_mmhub_funcs(adev);
@@ -770,10 +752,6 @@ static int gmc_v10_0_early_init(void *handle)
 	adev->gmc.private_aperture_end =
 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
 
-	r = amdgpu_gmc_ras_early_init(adev);
-	if (r)
-		return r;
-
 	return 0;
 }
 
@@ -1024,6 +1002,10 @@ static int gmc_v10_0_sw_init(void *handle)
 
 	amdgpu_vm_manager_init(adev);
 
+	r = amdgpu_gmc_ras_sw_init(adev);
+	if (r)
+		return r;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
index af7b3ba1ca00..d809f2ed5600 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
@@ -274,6 +274,8 @@ static void gmc_v11_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
  *
  * @adev: amdgpu_device pointer
  * @vmid: vm instance to flush
+ * @vmhub: which hub to flush
+ * @flush_type: the flush type
  *
  * Flush the TLB for the requested page table.
  */
@@ -313,6 +315,8 @@ static void gmc_v11_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
  *
  * @adev: amdgpu_device pointer
  * @pasid: pasid to be flush
+ * @flush_type: the flush type
+ * @all_hub: flush all hubs
  *
  * Flush the TLB for the requested pasid.
  */
@@ -374,8 +378,8 @@ static int gmc_v11_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
 static uint64_t gmc_v11_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
 					     unsigned vmid, uint64_t pd_addr)
 {
-	bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 	uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
 	unsigned eng = ring->vm_inv_eng;
 
@@ -429,7 +433,7 @@ static void gmc_v11_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid
 	if (ring->is_mes_queue)
 		return;
 
-	if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
+	if (ring->vm_hub == AMDGPU_GFXHUB_0)
 		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid;
 	else
 		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid;
@@ -581,23 +585,6 @@ static void gmc_v11_0_set_umc_funcs(struct amdgpu_device *adev)
 	default:
 		break;
 	}
-
-	if (adev->umc.ras) {
-		amdgpu_ras_register_ras_block(adev, &adev->umc.ras->ras_block);
-
-		strcpy(adev->umc.ras->ras_block.ras_comm.name, "umc");
-		adev->umc.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__UMC;
-		adev->umc.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
-		adev->umc.ras_if = &adev->umc.ras->ras_block.ras_comm;
-
-		/* If don't define special ras_late_init function, use default ras_late_init */
-		if (!adev->umc.ras->ras_block.ras_late_init)
-			adev->umc.ras->ras_block.ras_late_init = amdgpu_umc_ras_late_init;
-
-		/* If not define special ras_cb function, use default ras_cb */
-		if (!adev->umc.ras->ras_block.ras_cb)
-			adev->umc.ras->ras_block.ras_cb = amdgpu_umc_process_ras_data_cb;
-	}
 }
 
 
@@ -846,6 +833,10 @@ static int gmc_v11_0_sw_init(void *handle)
 
 	amdgpu_vm_manager_init(adev);
 
+	r = amdgpu_gmc_ras_sw_init(adev);
+	if (r)
+		return r;
+
 	return 0;
 }
 
@@ -875,6 +866,12 @@ static int gmc_v11_0_sw_fini(void *handle)
 
 static void gmc_v11_0_init_golden_registers(struct amdgpu_device *adev)
 {
+	if (amdgpu_sriov_vf(adev)) {
+		struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0];
+
+		WREG32(hub->vm_contexts_disable, 0);
+		return;
+	}
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index b06170c00dfc..64ab1a306dfe 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -49,8 +49,10 @@
 #include "mmhub_v1_0.h"
 #include "athub_v1_0.h"
 #include "gfxhub_v1_1.h"
+#include "gfxhub_v1_2.h"
 #include "mmhub_v9_4.h"
 #include "mmhub_v1_7.h"
+#include "mmhub_v1_8.h"
 #include "umc_v6_1.h"
 #include "umc_v6_0.h"
 #include "umc_v6_7.h"
@@ -553,32 +555,49 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 	const char *mmhub_cid;
 	const char *hub_name;
 	u64 addr;
+	uint32_t cam_index = 0;
+	int ret;
 
 	addr = (u64)entry->src_data[0] << 12;
 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
 
 	if (retry_fault) {
-		/* Returning 1 here also prevents sending the IV to the KFD */
+		if (adev->irq.retry_cam_enabled) {
+			/* Delegate it to a different ring if the hardware hasn't
+			 * already done it.
+			 */
+			if (entry->ih == &adev->irq.ih) {
+				amdgpu_irq_delegate(adev, entry, 8);
+				return 1;
+			}
+
+			cam_index = entry->src_data[2] & 0x3ff;
 
-		/* Process it onyl if it's the first fault for this address */
-		if (entry->ih != &adev->irq.ih_soft &&
-		    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
+			ret = amdgpu_vm_handle_fault(adev, entry->pasid, addr, write_fault);
+			WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
+			if (ret)
+				return 1;
+		} else {
+			/* Process it onyl if it's the first fault for this address */
+			if (entry->ih != &adev->irq.ih_soft &&
+			    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
 					     entry->timestamp))
-			return 1;
+				return 1;
 
-		/* Delegate it to a different ring if the hardware hasn't
-		 * already done it.
-		 */
-		if (entry->ih == &adev->irq.ih) {
-			amdgpu_irq_delegate(adev, entry, 8);
-			return 1;
-		}
+			/* Delegate it to a different ring if the hardware hasn't
+			 * already done it.
+			 */
+			if (entry->ih == &adev->irq.ih) {
+				amdgpu_irq_delegate(adev, entry, 8);
+				return 1;
+			}
 
-		/* Try to handle the recoverable page faults by filling page
-		 * tables
-		 */
-		if (amdgpu_vm_handle_fault(adev, entry->pasid, addr, write_fault))
-			return 1;
+			/* Try to handle the recoverable page faults by filling page
+			 * tables
+			 */
+			if (amdgpu_vm_handle_fault(adev, entry->pasid, addr, write_fault))
+				return 1;
+		}
 	}
 
 	if (!printk_ratelimit())
@@ -657,6 +676,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 		case IP_VERSION(2, 4, 0):
 			mmhub_cid = mmhub_client_ids_renoir[cid][rw];
 			break;
+		case IP_VERSION(1, 8, 0):
 		case IP_VERSION(9, 4, 2):
 			mmhub_cid = mmhub_client_ids_aldebaran[cid][rw];
 			break;
@@ -735,7 +755,8 @@ static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
 static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
 				       uint32_t vmhub)
 {
-	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
+	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
+	    adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
 		return false;
 
 	return ((vmhub == AMDGPU_MMHUB_0 ||
@@ -986,9 +1007,9 @@ static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
 					    unsigned vmid, uint64_t pd_addr)
 {
-	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
+	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &adev->vmhub[ring->vm_hub];
 	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
 	unsigned eng = ring->vm_inv_eng;
 
@@ -1039,10 +1060,10 @@ static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
 	uint32_t reg;
 
 	/* Do nothing because there's no lut register for mmhub1. */
-	if (ring->funcs->vmhub == AMDGPU_MMHUB_1)
+	if (ring->vm_hub == AMDGPU_MMHUB_1)
 		return;
 
-	if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
+	if (ring->vm_hub == AMDGPU_GFXHUB_0)
 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
 	else
 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
@@ -1144,6 +1165,7 @@ static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
 	switch (adev->ip_versions[GC_HWIP][0]) {
 	case IP_VERSION(9, 4, 1):
 	case IP_VERSION(9, 4, 2):
+	case IP_VERSION(9, 4, 3):
 		if (is_vram) {
 			if (bo_adev == adev) {
 				if (uncached)
@@ -1155,8 +1177,8 @@ static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
 				/* FIXME: is this still needed? Or does
 				 * amdgpu_ttm_tt_pde_flags already handle this?
 				 */
-				if (adev->ip_versions[GC_HWIP][0] ==
-					IP_VERSION(9, 4, 2) &&
+				if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
+				     adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) &&
 				    adev->gmc.xgmi.connected_to_cpu)
 					snoop = true;
 			} else {
@@ -1318,23 +1340,6 @@ static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
 	default:
 		break;
 	}
-
-	if (adev->umc.ras) {
-		amdgpu_ras_register_ras_block(adev, &adev->umc.ras->ras_block);
-
-		strcpy(adev->umc.ras->ras_block.ras_comm.name, "umc");
-		adev->umc.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__UMC;
-		adev->umc.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
-		adev->umc.ras_if = &adev->umc.ras->ras_block.ras_comm;
-
-		/* If don't define special ras_late_init function, use default ras_late_init */
-		if (!adev->umc.ras->ras_block.ras_late_init)
-				adev->umc.ras->ras_block.ras_late_init = amdgpu_umc_ras_late_init;
-
-		/* If not defined special ras_cb function, use default ras_cb */
-		if (!adev->umc.ras->ras_block.ras_cb)
-			adev->umc.ras->ras_block.ras_cb = amdgpu_umc_process_ras_data_cb;
-	}
 }
 
 static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
@@ -1346,6 +1351,9 @@ static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
 	case IP_VERSION(9, 4, 2):
 		adev->mmhub.funcs = &mmhub_v1_7_funcs;
 		break;
+	case IP_VERSION(1, 8, 0):
+		adev->mmhub.funcs = &mmhub_v1_8_funcs;
+		break;
 	default:
 		adev->mmhub.funcs = &mmhub_v1_0_funcs;
 		break;
@@ -1368,45 +1376,47 @@ static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev)
 		/* mmhub ras is not available */
 		break;
 	}
-
-	if (adev->mmhub.ras) {
-		amdgpu_ras_register_ras_block(adev, &adev->mmhub.ras->ras_block);
-
-		strcpy(adev->mmhub.ras->ras_block.ras_comm.name, "mmhub");
-		adev->mmhub.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MMHUB;
-		adev->mmhub.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
-		adev->mmhub.ras_if = &adev->mmhub.ras->ras_block.ras_comm;
-	}
 }
 
 static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
 {
-	adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
+	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
+		adev->gfxhub.funcs = &gfxhub_v1_2_funcs;
+	else
+		adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
 }
 
 static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
 {
 	adev->hdp.ras = &hdp_v4_0_ras;
-	amdgpu_ras_register_ras_block(adev, &adev->hdp.ras->ras_block);
-	adev->hdp.ras_if = &adev->hdp.ras->ras_block.ras_comm;
 }
 
-static void gmc_v9_0_set_mca_funcs(struct amdgpu_device *adev)
+static void gmc_v9_0_set_mca_ras_funcs(struct amdgpu_device *adev)
 {
+	struct amdgpu_mca *mca = &adev->mca;
+
 	/* is UMC the right IP to check for MCA?  Maybe DF? */
 	switch (adev->ip_versions[UMC_HWIP][0]) {
 	case IP_VERSION(6, 7, 0):
-		if (!adev->gmc.xgmi.connected_to_cpu)
-			adev->mca.funcs = &mca_v3_0_funcs;
+		if (!adev->gmc.xgmi.connected_to_cpu) {
+			mca->mp0.ras = &mca_v3_0_mp0_ras;
+			mca->mp1.ras = &mca_v3_0_mp1_ras;
+			mca->mpio.ras = &mca_v3_0_mpio_ras;
+		}
 		break;
 	default:
 		break;
 	}
 }
 
+static void gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device *adev)
+{
+	if (!adev->gmc.xgmi.connected_to_cpu)
+		adev->gmc.xgmi.ras = &xgmi_ras;
+}
+
 static int gmc_v9_0_early_init(void *handle)
 {
-	int r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	/* ARCT and VEGA20 don't have XGMI defined in their IP discovery tables */
@@ -1427,7 +1437,8 @@ static int gmc_v9_0_early_init(void *handle)
 	gmc_v9_0_set_mmhub_ras_funcs(adev);
 	gmc_v9_0_set_gfxhub_funcs(adev);
 	gmc_v9_0_set_hdp_ras_funcs(adev);
-	gmc_v9_0_set_mca_funcs(adev);
+	gmc_v9_0_set_mca_ras_funcs(adev);
+	gmc_v9_0_set_xgmi_ras_funcs(adev);
 
 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
 	adev->gmc.shared_aperture_end =
@@ -1436,10 +1447,6 @@ static int gmc_v9_0_early_init(void *handle)
 	adev->gmc.private_aperture_end =
 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
 
-	r = amdgpu_gmc_ras_early_init(adev);
-	if (r)
-		return r;
-
 	return 0;
 }
 
@@ -1565,6 +1572,7 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
 		case IP_VERSION(9, 4, 0):
 		case IP_VERSION(9, 4, 1):
 		case IP_VERSION(9, 4, 2):
+		case IP_VERSION(9, 4, 3):
 		default:
 			adev->gmc.gart_size = 512ULL << 20;
 			break;
@@ -1644,8 +1652,6 @@ static int gmc_v9_0_sw_init(void *handle)
 	adev->gfxhub.funcs->init(adev);
 
 	adev->mmhub.funcs->init(adev);
-	if (adev->mca.funcs)
-		adev->mca.funcs->init(adev);
 
 	spin_lock_init(&adev->gmc.invalidate_lock);
 
@@ -1696,6 +1702,7 @@ static int gmc_v9_0_sw_init(void *handle)
 	case IP_VERSION(9, 4, 0):
 	case IP_VERSION(9, 3, 0):
 	case IP_VERSION(9, 4, 2):
+	case IP_VERSION(9, 4, 3):
 		adev->num_vmhubs = 2;
 
 
@@ -1792,12 +1799,17 @@ static int gmc_v9_0_sw_init(void *handle)
 	 */
 	adev->vm_manager.first_kfd_vmid =
 		(adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) ||
-		 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) ? 3 : 8;
+		 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
+		 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) ? 3 : 8;
 
 	amdgpu_vm_manager_init(adev);
 
 	gmc_v9_0_save_registers(adev);
 
+	r = amdgpu_gmc_ras_sw_init(adev);
+	if (r)
+		return r;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c
index adf89680f53e..71d1a2e3bac9 100644
--- a/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c
@@ -49,7 +49,8 @@ static void hdp_v4_0_flush_hdp(struct amdgpu_device *adev,
 static void hdp_v4_0_invalidate_hdp(struct amdgpu_device *adev,
 				    struct amdgpu_ring *ring)
 {
-	if (adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 4, 0))
+	if (adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 4, 0) ||
+	    adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 4, 2))
 		return;
 
 	if (!ring || !ring->funcs->emit_wreg)
@@ -160,11 +161,6 @@ struct amdgpu_ras_block_hw_ops hdp_v4_0_ras_hw_ops = {
 
 struct amdgpu_hdp_ras hdp_v4_0_ras = {
 	.ras_block = {
-		.ras_comm = {
-			.name = "hdp",
-			.block = AMDGPU_RAS_BLOCK__HDP,
-			.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
-		},
 		.hw_ops = &hdp_v4_0_ras_hw_ops,
 	},
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c b/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
index 7cd79a3844b2..b02e1cef78a7 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
@@ -119,7 +119,7 @@ force_update_wptr_for_self_int(struct amdgpu_device *adev,
  * ih_v6_0_toggle_ring_interrupts - toggle the interrupt ring buffer
  *
  * @adev: amdgpu_device pointer
- * @ih: amdgpu_ih_ring pointet
+ * @ih: amdgpu_ih_ring pointer
  * @enable: true - enable the interrupts, false - disable the interrupts
  *
  * Toggle the interrupt ring buffer (IH_V6_0)
@@ -381,6 +381,7 @@ static void ih_v6_0_irq_disable(struct amdgpu_device *adev)
  * ih_v6_0_get_wptr - get the IH ring buffer wptr
  *
  * @adev: amdgpu_device pointer
+ * @ih: amdgpu_ih_ring pointer
  *
  * Get the IH ring buffer wptr from either the register
  * or the writeback memory buffer.  Also check for
@@ -425,6 +426,7 @@ out:
  * ih_v6_0_irq_rearm - rearm IRQ if lost
  *
  * @adev: amdgpu_device pointer
+ * @ih: amdgpu_ih_ring pointer
  *
  */
 static void ih_v6_0_irq_rearm(struct amdgpu_device *adev,
@@ -450,6 +452,7 @@ static void ih_v6_0_irq_rearm(struct amdgpu_device *adev,
  * ih_v6_0_set_rptr - set the IH ring buffer rptr
  *
  * @adev: amdgpu_device pointer
+ * @ih: amdgpu_ih_ring pointer
  *
  * Set the IH ring buffer rptr.
  */
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v1_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v1_0.c
index 9360204da7fb..a3076eb8af6a 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v1_0.c
@@ -376,7 +376,7 @@ static void jpeg_v1_0_decode_ring_emit_reg_wait(struct amdgpu_ring *ring,
 static void jpeg_v1_0_decode_ring_emit_vm_flush(struct amdgpu_ring *ring,
 		unsigned vmid, uint64_t pd_addr)
 {
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 	uint32_t data0, data1, mask;
 
 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
@@ -485,6 +485,7 @@ int jpeg_v1_0_sw_init(void *handle)
 		return r;
 
 	ring = &adev->jpeg.inst->ring_dec;
+	ring->vm_hub = AMDGPU_MMHUB_0;
 	sprintf(ring->name, "jpeg_dec");
 	r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq,
 			     0, AMDGPU_RING_PRIO_DEFAULT, NULL);
@@ -548,7 +549,6 @@ static const struct amdgpu_ring_funcs jpeg_v1_0_decode_ring_vm_funcs = {
 	.nop = PACKET0(0x81ff, 0),
 	.support_64bit_ptrs = false,
 	.no_user_fence = true,
-	.vmhub = AMDGPU_MMHUB_0,
 	.extra_dw = 64,
 	.get_rptr = jpeg_v1_0_decode_ring_get_rptr,
 	.get_wptr = jpeg_v1_0_decode_ring_get_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
index f3c1af5130ab..0eddf7c824a7 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
@@ -86,6 +86,7 @@ static int jpeg_v2_0_sw_init(void *handle)
 	ring = &adev->jpeg.inst->ring_dec;
 	ring->use_doorbell = true;
 	ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1;
+	ring->vm_hub = AMDGPU_MMHUB_0;
 	sprintf(ring->name, "jpeg_dec");
 	r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq,
 			     0, AMDGPU_RING_PRIO_DEFAULT, NULL);
@@ -613,7 +614,7 @@ void jpeg_v2_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 void jpeg_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
 				unsigned vmid, uint64_t pd_addr)
 {
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 	uint32_t data0, data1, mask;
 
 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
@@ -762,7 +763,6 @@ static const struct amd_ip_funcs jpeg_v2_0_ip_funcs = {
 static const struct amdgpu_ring_funcs jpeg_v2_0_dec_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_JPEG,
 	.align_mask = 0xf,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = jpeg_v2_0_dec_ring_get_rptr,
 	.get_wptr = jpeg_v2_0_dec_ring_get_wptr,
 	.set_wptr = jpeg_v2_0_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
index f2b743a93915..b040f51d9aa9 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
@@ -127,6 +127,10 @@ static int jpeg_v2_5_sw_init(void *handle)
 
 		ring = &adev->jpeg.inst[i].ring_dec;
 		ring->use_doorbell = true;
+		if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(2, 5, 0))
+			ring->vm_hub = AMDGPU_MMHUB_1;
+		else
+			ring->vm_hub = AMDGPU_MMHUB_0;
 		ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1 + 8 * i;
 		sprintf(ring->name, "jpeg_dec_%d", i);
 		r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst[i].irq,
@@ -138,6 +142,10 @@ static int jpeg_v2_5_sw_init(void *handle)
 		adev->jpeg.inst[i].external.jpeg_pitch = SOC15_REG_OFFSET(JPEG, i, mmUVD_JPEG_PITCH);
 	}
 
+	r = amdgpu_jpeg_ras_sw_init(adev);
+	if (r)
+		return r;
+
 	return 0;
 }
 
@@ -641,7 +649,6 @@ static const struct amd_ip_funcs jpeg_v2_6_ip_funcs = {
 static const struct amdgpu_ring_funcs jpeg_v2_5_dec_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_JPEG,
 	.align_mask = 0xf,
-	.vmhub = AMDGPU_MMHUB_1,
 	.get_rptr = jpeg_v2_5_dec_ring_get_rptr,
 	.get_wptr = jpeg_v2_5_dec_ring_get_wptr,
 	.set_wptr = jpeg_v2_5_dec_ring_set_wptr,
@@ -671,7 +678,6 @@ static const struct amdgpu_ring_funcs jpeg_v2_5_dec_ring_vm_funcs = {
 static const struct amdgpu_ring_funcs jpeg_v2_6_dec_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_JPEG,
 	.align_mask = 0xf,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = jpeg_v2_5_dec_ring_get_rptr,
 	.get_wptr = jpeg_v2_5_dec_ring_get_wptr,
 	.set_wptr = jpeg_v2_5_dec_ring_set_wptr,
@@ -806,6 +812,4 @@ static void jpeg_v2_5_set_ras_funcs(struct amdgpu_device *adev)
 	default:
 		break;
 	}
-
-	jpeg_set_ras_funcs(adev);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
index a1b751d9ac06..c55e09432e26 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
@@ -100,6 +100,7 @@ static int jpeg_v3_0_sw_init(void *handle)
 	ring = &adev->jpeg.inst->ring_dec;
 	ring->use_doorbell = true;
 	ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1;
+	ring->vm_hub = AMDGPU_MMHUB_0;
 	sprintf(ring->name, "jpeg_dec");
 	r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq, 0,
 			     AMDGPU_RING_PRIO_DEFAULT, NULL);
@@ -559,7 +560,6 @@ static const struct amd_ip_funcs jpeg_v3_0_ip_funcs = {
 static const struct amdgpu_ring_funcs jpeg_v3_0_dec_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_JPEG,
 	.align_mask = 0xf,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = jpeg_v3_0_dec_ring_get_rptr,
 	.get_wptr = jpeg_v3_0_dec_ring_get_wptr,
 	.set_wptr = jpeg_v3_0_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
index 3beb731b2ce5..a6ad678fd507 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
@@ -28,6 +28,7 @@
 #include "soc15d.h"
 #include "jpeg_v2_0.h"
 #include "jpeg_v4_0.h"
+#include "mmsch_v4_0.h"
 
 #include "vcn/vcn_4_0_0_offset.h"
 #include "vcn/vcn_4_0_0_sh_mask.h"
@@ -35,12 +36,15 @@
 
 #define regUVD_JPEG_PITCH_INTERNAL_OFFSET                  0x401f
 
+static int jpeg_v4_0_start_sriov(struct amdgpu_device *adev);
 static void jpeg_v4_0_set_dec_ring_funcs(struct amdgpu_device *adev);
 static void jpeg_v4_0_set_irq_funcs(struct amdgpu_device *adev);
 static int jpeg_v4_0_set_powergating_state(void *handle,
 				enum amd_powergating_state state);
 static void jpeg_v4_0_set_ras_funcs(struct amdgpu_device *adev);
 
+static void jpeg_v4_0_dec_ring_set_wptr(struct amdgpu_ring *ring);
+
 /**
  * jpeg_v4_0_early_init - set function pointers
  *
@@ -103,7 +107,9 @@ static int jpeg_v4_0_sw_init(void *handle)
 
 	ring = &adev->jpeg.inst->ring_dec;
 	ring->use_doorbell = true;
-	ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1;
+	ring->doorbell_index = amdgpu_sriov_vf(adev) ? (((adev->doorbell_index.vcn.vcn_ring0_1) << 1) + 4) : ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1);
+	ring->vm_hub = AMDGPU_MMHUB_0;
+
 	sprintf(ring->name, "jpeg_dec");
 	r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq, 0,
 			     AMDGPU_RING_PRIO_DEFAULT, NULL);
@@ -113,6 +119,10 @@ static int jpeg_v4_0_sw_init(void *handle)
 	adev->jpeg.internal.jpeg_pitch = regUVD_JPEG_PITCH_INTERNAL_OFFSET;
 	adev->jpeg.inst->external.jpeg_pitch = SOC15_REG_OFFSET(JPEG, 0, regUVD_JPEG_PITCH);
 
+	r = amdgpu_jpeg_ras_sw_init(adev);
+	if (r)
+		return r;
+
 	return 0;
 }
 
@@ -149,16 +159,26 @@ static int jpeg_v4_0_hw_init(void *handle)
 	struct amdgpu_ring *ring = &adev->jpeg.inst->ring_dec;
 	int r;
 
-	adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
-					(adev->doorbell_index.vcn.vcn_ring0_1 << 1), 0);
+	if (amdgpu_sriov_vf(adev)) {
+		r = jpeg_v4_0_start_sriov(adev);
+		if (r)
+			return r;
+		ring->wptr = 0;
+		ring->wptr_old = 0;
+		jpeg_v4_0_dec_ring_set_wptr(ring);
+		ring->sched.ready = true;
+	} else {
+		adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
+						(adev->doorbell_index.vcn.vcn_ring0_1 << 1), 0);
 
-	WREG32_SOC15(VCN, 0, regVCN_JPEG_DB_CTRL,
-		ring->doorbell_index << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
-		VCN_JPEG_DB_CTRL__EN_MASK);
+		WREG32_SOC15(VCN, 0, regVCN_JPEG_DB_CTRL,
+			ring->doorbell_index << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
+			VCN_JPEG_DB_CTRL__EN_MASK);
 
-	r = amdgpu_ring_test_helper(ring);
-	if (r)
-		return r;
+		r = amdgpu_ring_test_helper(ring);
+		if (r)
+			return r;
+	}
 
 	DRM_DEV_INFO(adev->dev, "JPEG decode initialized successfully.\n");
 
@@ -177,11 +197,11 @@ static int jpeg_v4_0_hw_fini(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	cancel_delayed_work_sync(&adev->vcn.idle_work);
-
-	if (adev->jpeg.cur_state != AMD_PG_STATE_GATE &&
-	      RREG32_SOC15(JPEG, 0, regUVD_JRBC_STATUS))
-		jpeg_v4_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
-
+	if (!amdgpu_sriov_vf(adev)) {
+		if (adev->jpeg.cur_state != AMD_PG_STATE_GATE &&
+			RREG32_SOC15(JPEG, 0, regUVD_JRBC_STATUS))
+			jpeg_v4_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
+	}
 	amdgpu_irq_put(adev, &adev->jpeg.inst->irq, 0);
 
 	return 0;
@@ -386,6 +406,120 @@ static int jpeg_v4_0_start(struct amdgpu_device *adev)
 	return 0;
 }
 
+static int jpeg_v4_0_start_sriov(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	uint64_t ctx_addr;
+	uint32_t param, resp, expected;
+	uint32_t tmp, timeout;
+
+	struct amdgpu_mm_table *table = &adev->virt.mm_table;
+	uint32_t *table_loc;
+	uint32_t table_size;
+	uint32_t size, size_dw;
+	uint32_t init_status;
+
+	struct mmsch_v4_0_cmd_direct_write
+		direct_wt = { {0} };
+	struct mmsch_v4_0_cmd_end end = { {0} };
+	struct mmsch_v4_0_init_header header;
+
+	direct_wt.cmd_header.command_type =
+		MMSCH_COMMAND__DIRECT_REG_WRITE;
+	end.cmd_header.command_type =
+		MMSCH_COMMAND__END;
+
+	header.version = MMSCH_VERSION;
+	header.total_size = sizeof(struct mmsch_v4_0_init_header) >> 2;
+
+	header.jpegdec.init_status = 0;
+	header.jpegdec.table_offset = 0;
+	header.jpegdec.table_size = 0;
+
+	table_loc = (uint32_t *)table->cpu_addr;
+	table_loc += header.total_size;
+
+	table_size = 0;
+
+	ring = &adev->jpeg.inst->ring_dec;
+
+	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(JPEG, 0,
+		regUVD_LMI_JRBC_RB_64BIT_BAR_LOW),
+		lower_32_bits(ring->gpu_addr));
+	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(JPEG, 0,
+		regUVD_LMI_JRBC_RB_64BIT_BAR_HIGH),
+		upper_32_bits(ring->gpu_addr));
+	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(JPEG, 0,
+		regUVD_JRBC_RB_SIZE), ring->ring_size / 4);
+
+	/* add end packet */
+	MMSCH_V4_0_INSERT_END();
+
+	/* refine header */
+	header.jpegdec.init_status = 0;
+	header.jpegdec.table_offset = header.total_size;
+	header.jpegdec.table_size = table_size;
+	header.total_size += table_size;
+
+	/* Update init table header in memory */
+	size = sizeof(struct mmsch_v4_0_init_header);
+	table_loc = (uint32_t *)table->cpu_addr;
+	memcpy((void *)table_loc, &header, size);
+
+	/* message MMSCH (in VCN[0]) to initialize this client
+	 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
+	 * of memory descriptor location
+	 */
+	ctx_addr = table->gpu_addr;
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
+
+	/* 2, update vmid of descriptor */
+	tmp = RREG32_SOC15(VCN, 0, regMMSCH_VF_VMID);
+	tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
+	/* use domain0 for MM scheduler */
+	tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_VMID, tmp);
+
+	/* 3, notify mmsch about the size of this descriptor */
+	size = header.total_size;
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_SIZE, size);
+
+	/* 4, set resp to zero */
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP, 0);
+
+	/* 5, kick off the initialization and wait until
+	 * MMSCH_VF_MAILBOX_RESP becomes non-zero
+	 */
+	param = 0x00000001;
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_HOST, param);
+	tmp = 0;
+	timeout = 1000;
+	resp = 0;
+	expected = MMSCH_VF_MAILBOX_RESP__OK;
+	init_status = ((struct mmsch_v4_0_init_header *)(table_loc))->jpegdec.init_status;
+	while (resp != expected) {
+		resp = RREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP);
+
+		if (resp != 0)
+			break;
+		udelay(10);
+		tmp = tmp + 10;
+		if (tmp >= timeout) {
+			DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
+				" waiting for regMMSCH_VF_MAILBOX_RESP "\
+				"(expected=0x%08x, readback=0x%08x)\n",
+				tmp, expected, resp);
+			return -EBUSY;
+		}
+	}
+	if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE && init_status != MMSCH_VF_ENGINE_STATUS__PASS)
+		DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init status for jpeg: %x\n", resp, init_status);
+
+	return 0;
+
+}
+
 /**
  * jpeg_v4_0_stop - stop JPEG block
  *
@@ -509,6 +643,11 @@ static int jpeg_v4_0_set_powergating_state(void *handle,
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int ret;
 
+	if (amdgpu_sriov_vf(adev)) {
+		adev->jpeg.cur_state = AMD_PG_STATE_UNGATE;
+		return 0;
+	}
+
 	if (state == adev->jpeg.cur_state)
 		return 0;
 
@@ -577,7 +716,6 @@ static const struct amd_ip_funcs jpeg_v4_0_ip_funcs = {
 static const struct amdgpu_ring_funcs jpeg_v4_0_dec_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_JPEG,
 	.align_mask = 0xf,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = jpeg_v4_0_dec_ring_get_rptr,
 	.get_wptr = jpeg_v4_0_dec_ring_get_wptr,
 	.set_wptr = jpeg_v4_0_dec_ring_set_wptr,
@@ -685,6 +823,4 @@ static void jpeg_v4_0_set_ras_funcs(struct amdgpu_device *adev)
 	default:
 		break;
 	}
-
-	jpeg_set_ras_funcs(adev);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/mca_v3_0.c b/drivers/gpu/drm/amd/amdgpu/mca_v3_0.c
index d4bd7d1d2649..6dae4a2e2767 100644
--- a/drivers/gpu/drm/amd/amdgpu/mca_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mca_v3_0.c
@@ -51,19 +51,13 @@ static int mca_v3_0_ras_block_match(struct amdgpu_ras_block_object *block_obj,
 	return -EINVAL;
 }
 
-const struct amdgpu_ras_block_hw_ops mca_v3_0_mp0_hw_ops = {
+static const struct amdgpu_ras_block_hw_ops mca_v3_0_mp0_hw_ops = {
 	.query_ras_error_count = mca_v3_0_mp0_query_ras_error_count,
 	.query_ras_error_address = NULL,
 };
 
 struct amdgpu_mca_ras_block mca_v3_0_mp0_ras = {
 	.ras_block = {
-		.ras_comm = {
-			.block = AMDGPU_RAS_BLOCK__MCA,
-			.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MP0,
-			.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
-			.name = "mp0",
-		},
 		.hw_ops = &mca_v3_0_mp0_hw_ops,
 		.ras_block_match = mca_v3_0_ras_block_match,
 	},
@@ -77,19 +71,13 @@ static void mca_v3_0_mp1_query_ras_error_count(struct amdgpu_device *adev,
 				         ras_error_status);
 }
 
-const struct amdgpu_ras_block_hw_ops mca_v3_0_mp1_hw_ops = {
+static const struct amdgpu_ras_block_hw_ops mca_v3_0_mp1_hw_ops = {
 	.query_ras_error_count = mca_v3_0_mp1_query_ras_error_count,
 	.query_ras_error_address = NULL,
 };
 
 struct amdgpu_mca_ras_block mca_v3_0_mp1_ras = {
 	.ras_block = {
-		.ras_comm = {
-			.block = AMDGPU_RAS_BLOCK__MCA,
-			.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MP1,
-			.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
-			.name = "mp1",
-		},
 		.hw_ops = &mca_v3_0_mp1_hw_ops,
 		.ras_block_match = mca_v3_0_ras_block_match,
 	},
@@ -103,40 +91,14 @@ static void mca_v3_0_mpio_query_ras_error_count(struct amdgpu_device *adev,
 				         ras_error_status);
 }
 
-const struct amdgpu_ras_block_hw_ops mca_v3_0_mpio_hw_ops = {
+static const struct amdgpu_ras_block_hw_ops mca_v3_0_mpio_hw_ops = {
 	.query_ras_error_count = mca_v3_0_mpio_query_ras_error_count,
 	.query_ras_error_address = NULL,
 };
 
 struct amdgpu_mca_ras_block mca_v3_0_mpio_ras = {
 	.ras_block = {
-		.ras_comm = {
-			.block = AMDGPU_RAS_BLOCK__MCA,
-			.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MPIO,
-			.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
-			.name = "mpio",
-		},
 		.hw_ops = &mca_v3_0_mpio_hw_ops,
 		.ras_block_match = mca_v3_0_ras_block_match,
 	},
 };
-
-
-static void mca_v3_0_init(struct amdgpu_device *adev)
-{
-	struct amdgpu_mca *mca = &adev->mca;
-
-	mca->mp0.ras = &mca_v3_0_mp0_ras;
-	mca->mp1.ras = &mca_v3_0_mp1_ras;
-	mca->mpio.ras = &mca_v3_0_mpio_ras;
-	amdgpu_ras_register_ras_block(adev, &mca->mp0.ras->ras_block);
-	amdgpu_ras_register_ras_block(adev, &mca->mp1.ras->ras_block);
-	amdgpu_ras_register_ras_block(adev, &mca->mpio.ras->ras_block);
-	mca->mp0.ras_if = &mca->mp0.ras->ras_block.ras_comm;
-	mca->mp1.ras_if = &mca->mp1.ras->ras_block.ras_comm;
-	mca->mpio.ras_if = &mca->mpio.ras->ras_block.ras_comm;
-}
-
-const struct amdgpu_mca_funcs mca_v3_0_funcs = {
-	.init = mca_v3_0_init,
-};
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/amdgpu/mca_v3_0.h b/drivers/gpu/drm/amd/amdgpu/mca_v3_0.h
index b899b86194c2..d3eaef0d7f2d 100644
--- a/drivers/gpu/drm/amd/amdgpu/mca_v3_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/mca_v3_0.h
@@ -21,6 +21,8 @@
 #ifndef __MCA_V3_0_H__
 #define __MCA_V3_0_H__
 
-extern const struct amdgpu_mca_funcs mca_v3_0_funcs;
+extern struct amdgpu_mca_ras_block mca_v3_0_mp0_ras;
+extern struct amdgpu_mca_ras_block mca_v3_0_mp1_ras;
+extern struct amdgpu_mca_ras_block mca_v3_0_mpio_ras;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index 5826eac270d7..45280f047180 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -33,14 +33,19 @@
 #include "mes_v11_api_def.h"
 
 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes_2.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes1.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes_2.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes1.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes_2.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes1.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes_2.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes1.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes_2.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes1.bin");
 
 static int mes_v11_0_hw_fini(void *handle);
@@ -1089,13 +1094,14 @@ static int mes_v11_0_sw_fini(void *handle)
 	return 0;
 }
 
-static void mes_v11_0_kiq_dequeue_sched(struct amdgpu_device *adev)
+static void mes_v11_0_kiq_dequeue(struct amdgpu_ring *ring)
 {
 	uint32_t data;
 	int i;
+	struct amdgpu_device *adev = ring->adev;
 
 	mutex_lock(&adev->srbm_mutex);
-	soc21_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0);
+	soc21_grbm_select(adev, 3, ring->pipe, 0, 0);
 
 	/* disable the queue if it's active */
 	if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
@@ -1121,8 +1127,6 @@ static void mes_v11_0_kiq_dequeue_sched(struct amdgpu_device *adev)
 
 	soc21_grbm_select(adev, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);
-
-	adev->mes.ring.sched.ready = false;
 }
 
 static void mes_v11_0_kiq_setting(struct amdgpu_ring *ring)
@@ -1139,6 +1143,16 @@ static void mes_v11_0_kiq_setting(struct amdgpu_ring *ring)
 	WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
 }
 
+static void mes_v11_0_kiq_clear(struct amdgpu_device *adev)
+{
+	uint32_t tmp;
+
+	/* tell RLC which is KIQ dequeue */
+	tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
+	tmp &= ~RLC_CP_SCHEDULERS__scheduler0_MASK;
+	WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
+}
+
 static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev)
 {
 	int r = 0;
@@ -1176,11 +1190,17 @@ failure:
 
 static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev)
 {
-	if (adev->mes.ring.sched.ready)
-		mes_v11_0_kiq_dequeue_sched(adev);
+	if (adev->mes.ring.sched.ready) {
+		mes_v11_0_kiq_dequeue(&adev->mes.ring);
+		adev->mes.ring.sched.ready = false;
+	}
 
-	if (!amdgpu_sriov_vf(adev))
-		mes_v11_0_enable(adev, false);
+	if (amdgpu_sriov_vf(adev)) {
+		mes_v11_0_kiq_dequeue(&adev->gfx.kiq.ring);
+		mes_v11_0_kiq_clear(adev);
+	}
+
+	mes_v11_0_enable(adev, false);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c
new file mode 100644
index 000000000000..342d1702104c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c
@@ -0,0 +1,477 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "mmhub_v1_8.h"
+
+#include "mmhub/mmhub_1_8_0_offset.h"
+#include "mmhub/mmhub_1_8_0_sh_mask.h"
+#include "vega10_enum.h"
+
+#include "soc15_common.h"
+#include "soc15.h"
+
+#define regVM_L2_CNTL3_DEFAULT	0x80100007
+#define regVM_L2_CNTL4_DEFAULT	0x000000c1
+
+static u64 mmhub_v1_8_get_fb_location(struct amdgpu_device *adev)
+{
+	u64 base = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_BASE);
+	u64 top = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_TOP);
+
+	base &= MC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
+	base <<= 24;
+
+	top &= MC_VM_FB_LOCATION_TOP__FB_TOP_MASK;
+	top <<= 24;
+
+	adev->gmc.fb_start = base;
+	adev->gmc.fb_end = top;
+
+	return base;
+}
+
+static void mmhub_v1_8_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid,
+				uint64_t page_table_base)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0];
+
+	WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+			    hub->ctx_addr_distance * vmid, lower_32_bits(page_table_base));
+
+	WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+			    hub->ctx_addr_distance * vmid, upper_32_bits(page_table_base));
+}
+
+static void mmhub_v1_8_init_gart_aperture_regs(struct amdgpu_device *adev)
+{
+	uint64_t pt_base;
+
+	if (adev->gmc.pdb0_bo)
+		pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo);
+	else
+		pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+
+	mmhub_v1_8_setup_vm_pt_regs(adev, 0, pt_base);
+
+	/* If use GART for FB translation, vmid0 page table covers both
+	 * vram and system memory (gart)
+	 */
+	if (adev->gmc.pdb0_bo) {
+		WREG32_SOC15(MMHUB, 0, regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+			     (u32)(adev->gmc.fb_start >> 12));
+		WREG32_SOC15(MMHUB, 0, regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+			     (u32)(adev->gmc.fb_start >> 44));
+
+		WREG32_SOC15(MMHUB, 0, regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+			     (u32)(adev->gmc.gart_end >> 12));
+		WREG32_SOC15(MMHUB, 0, regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+			     (u32)(adev->gmc.gart_end >> 44));
+
+	} else {
+		WREG32_SOC15(MMHUB, 0, regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+			     (u32)(adev->gmc.gart_start >> 12));
+		WREG32_SOC15(MMHUB, 0, regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+			     (u32)(adev->gmc.gart_start >> 44));
+
+		WREG32_SOC15(MMHUB, 0, regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+			     (u32)(adev->gmc.gart_end >> 12));
+		WREG32_SOC15(MMHUB, 0, regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+			     (u32)(adev->gmc.gart_end >> 44));
+	}
+}
+
+static void mmhub_v1_8_init_system_aperture_regs(struct amdgpu_device *adev)
+{
+	uint64_t value;
+	uint32_t tmp;
+
+	/* Program the AGP BAR */
+	WREG32_SOC15(MMHUB, 0, regMC_VM_AGP_BASE, 0);
+	WREG32_SOC15(MMHUB, 0, regMC_VM_AGP_BOT, adev->gmc.agp_start >> 24);
+	WREG32_SOC15(MMHUB, 0, regMC_VM_AGP_TOP, adev->gmc.agp_end >> 24);
+
+	/* Program the system aperture low logical page number. */
+	WREG32_SOC15(MMHUB, 0, regMC_VM_SYSTEM_APERTURE_LOW_ADDR,
+		     min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18);
+
+	WREG32_SOC15(MMHUB, 0, regMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+		     max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
+
+	/* In the case squeezing vram into GART aperture, we don't use
+	 * FB aperture and AGP aperture. Disable them.
+	 */
+	if (adev->gmc.pdb0_bo) {
+		WREG32_SOC15(MMHUB, 0, regMC_VM_AGP_BOT, 0xFFFFFF);
+		WREG32_SOC15(MMHUB, 0, regMC_VM_AGP_TOP, 0);
+		WREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_TOP, 0);
+		WREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_BASE, 0x00FFFFFF);
+		WREG32_SOC15(MMHUB, 0, regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 0x3FFFFFFF);
+		WREG32_SOC15(MMHUB, 0, regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0);
+	}
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	/* Set default page address. */
+	value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
+	WREG32_SOC15(MMHUB, 0, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
+		     (u32)(value >> 12));
+	WREG32_SOC15(MMHUB, 0, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
+		     (u32)(value >> 44));
+
+	/* Program "protection fault". */
+	WREG32_SOC15(MMHUB, 0, regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
+		     (u32)(adev->dummy_page_addr >> 12));
+	WREG32_SOC15(MMHUB, 0, regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
+		     (u32)((u64)adev->dummy_page_addr >> 44));
+
+	tmp = RREG32_SOC15(MMHUB, 0, regVM_L2_PROTECTION_FAULT_CNTL2);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2,
+			    ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	WREG32_SOC15(MMHUB, 0, regVM_L2_PROTECTION_FAULT_CNTL2, tmp);
+}
+
+static void mmhub_v1_8_init_tlb_regs(struct amdgpu_device *adev)
+{
+	uint32_t tmp;
+
+	/* Setup TLB control */
+	tmp = RREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+			    ENABLE_ADVANCED_DRIVER_MODEL, 1);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+			    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+			    MTYPE, MTYPE_UC);/* XXX for emulation. */
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
+
+	WREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL, tmp);
+}
+
+static void mmhub_v1_8_init_cache_regs(struct amdgpu_device *adev)
+{
+	uint32_t tmp;
+
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	/* Setup L2 cache */
+	tmp = RREG32_SOC15(MMHUB, 0, regVM_L2_CNTL);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
+	/* XXX for emulation, Refer to closed source code.*/
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
+			    0);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
+	WREG32_SOC15(MMHUB, 0, regVM_L2_CNTL, tmp);
+
+	tmp = RREG32_SOC15(MMHUB, 0, regVM_L2_CNTL2);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
+	WREG32_SOC15(MMHUB, 0, regVM_L2_CNTL2, tmp);
+
+	tmp = regVM_L2_CNTL3_DEFAULT;
+	if (adev->gmc.translate_further) {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
+				    L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
+	} else {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
+				    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
+	}
+	WREG32_SOC15(MMHUB, 0, regVM_L2_CNTL3, tmp);
+
+	tmp = regVM_L2_CNTL4_DEFAULT;
+	if (adev->gmc.xgmi.connected_to_cpu) {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4,
+				    VMC_TAP_PDE_REQUEST_PHYSICAL, 1);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4,
+				    VMC_TAP_PTE_REQUEST_PHYSICAL, 1);
+	} else {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4,
+				    VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4,
+				    VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
+	}
+	WREG32_SOC15(MMHUB, 0, regVM_L2_CNTL4, tmp);
+}
+
+static void mmhub_v1_8_enable_system_domain(struct amdgpu_device *adev)
+{
+	uint32_t tmp;
+
+	tmp = RREG32_SOC15(MMHUB, 0, regVM_CONTEXT0_CNTL);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH,
+			adev->gmc.vmid0_page_table_depth);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_BLOCK_SIZE,
+			adev->gmc.vmid0_page_table_block_size);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL,
+			    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
+	WREG32_SOC15(MMHUB, 0, regVM_CONTEXT0_CNTL, tmp);
+}
+
+static void mmhub_v1_8_disable_identity_aperture(struct amdgpu_device *adev)
+{
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	WREG32_SOC15(MMHUB, 0, regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 0xFFFFFFFF);
+	WREG32_SOC15(MMHUB, 0, regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 0x0000000F);
+
+	WREG32_SOC15(MMHUB, 0, regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 0);
+	WREG32_SOC15(MMHUB, 0, regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 0);
+
+	WREG32_SOC15(MMHUB, 0, regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
+	WREG32_SOC15(MMHUB, 0, regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);
+}
+
+static void mmhub_v1_8_setup_vmid_config(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0];
+	unsigned num_level, block_size;
+	uint32_t tmp;
+	int i;
+
+	num_level = adev->vm_manager.num_level;
+	block_size = adev->vm_manager.block_size;
+	if (adev->gmc.translate_further)
+		num_level -= 1;
+	else
+		block_size -= 9;
+
+	for (i = 0; i <= 14; i++) {
+		tmp = RREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT1_CNTL, i);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
+				    num_level);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
+				    1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    PAGE_TABLE_BLOCK_SIZE,
+				    block_size);
+		/* On Aldebaran, XNACK can be enabled in the SQ per-process.
+		 * Retry faults need to be enabled for that to work.
+		 */
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
+				    1);
+		WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT1_CNTL,
+				    i * hub->ctx_distance, tmp);
+		WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
+				    i * hub->ctx_addr_distance, 0);
+		WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
+				    i * hub->ctx_addr_distance, 0);
+		WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
+				    i * hub->ctx_addr_distance,
+				    lower_32_bits(adev->vm_manager.max_pfn - 1));
+		WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
+				    i * hub->ctx_addr_distance,
+				    upper_32_bits(adev->vm_manager.max_pfn - 1));
+	}
+}
+
+static void mmhub_v1_8_program_invalidation(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0];
+	unsigned i;
+
+	for (i = 0; i < 18; ++i) {
+		WREG32_SOC15_OFFSET(MMHUB, 0, regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
+				    i * hub->eng_addr_distance, 0xffffffff);
+		WREG32_SOC15_OFFSET(MMHUB, 0, regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
+				    i * hub->eng_addr_distance, 0x1f);
+	}
+}
+
+static int mmhub_v1_8_gart_enable(struct amdgpu_device *adev)
+{
+	if (amdgpu_sriov_vf(adev)) {
+		/*
+		 * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
+		 * VF copy registers so vbios post doesn't program them, for
+		 * SRIOV driver need to program them
+		 */
+		WREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_BASE,
+			     adev->gmc.vram_start >> 24);
+		WREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_TOP,
+			     adev->gmc.vram_end >> 24);
+	}
+
+	/* GART Enable. */
+	mmhub_v1_8_init_gart_aperture_regs(adev);
+	mmhub_v1_8_init_system_aperture_regs(adev);
+	mmhub_v1_8_init_tlb_regs(adev);
+	mmhub_v1_8_init_cache_regs(adev);
+
+	mmhub_v1_8_enable_system_domain(adev);
+	mmhub_v1_8_disable_identity_aperture(adev);
+	mmhub_v1_8_setup_vmid_config(adev);
+	mmhub_v1_8_program_invalidation(adev);
+
+	return 0;
+}
+
+static void mmhub_v1_8_gart_disable(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0];
+	u32 tmp;
+	u32 i;
+
+	/* Disable all tables */
+	for (i = 0; i < 16; i++)
+		WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT0_CNTL,
+				    i * hub->ctx_distance, 0);
+
+	/* Setup TLB control */
+	tmp = RREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
+	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
+			    ENABLE_ADVANCED_DRIVER_MODEL, 0);
+	WREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL, tmp);
+
+	if (!amdgpu_sriov_vf(adev)) {
+		/* Setup L2 cache */
+		tmp = RREG32_SOC15(MMHUB, 0, regVM_L2_CNTL);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
+		WREG32_SOC15(MMHUB, 0, regVM_L2_CNTL, tmp);
+		WREG32_SOC15(MMHUB, 0, regVM_L2_CNTL3, 0);
+	}
+}
+
+/**
+ * mmhub_v1_8_set_fault_enable_default - update GART/VM fault handling
+ *
+ * @adev: amdgpu_device pointer
+ * @value: true redirects VM faults to the default page
+ */
+static void mmhub_v1_8_set_fault_enable_default(struct amdgpu_device *adev, bool value)
+{
+	u32 tmp;
+
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	tmp = RREG32_SOC15(MMHUB, 0, regVM_L2_PROTECTION_FAULT_CNTL);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
+			    value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+			    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	if (!value) {
+		tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+				    CRASH_ON_NO_RETRY_FAULT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+				    CRASH_ON_RETRY_FAULT, 1);
+	}
+
+	WREG32_SOC15(MMHUB, 0, regVM_L2_PROTECTION_FAULT_CNTL, tmp);
+}
+
+static void mmhub_v1_8_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0];
+
+	hub->ctx0_ptb_addr_lo32 =
+		SOC15_REG_OFFSET(MMHUB, 0,
+				 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
+	hub->ctx0_ptb_addr_hi32 =
+		SOC15_REG_OFFSET(MMHUB, 0,
+				 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
+	hub->vm_inv_eng0_req =
+		SOC15_REG_OFFSET(MMHUB, 0, regVM_INVALIDATE_ENG0_REQ);
+	hub->vm_inv_eng0_ack =
+		SOC15_REG_OFFSET(MMHUB, 0, regVM_INVALIDATE_ENG0_ACK);
+	hub->vm_context0_cntl =
+		SOC15_REG_OFFSET(MMHUB, 0, regVM_CONTEXT0_CNTL);
+	hub->vm_l2_pro_fault_status =
+		SOC15_REG_OFFSET(MMHUB, 0, regVM_L2_PROTECTION_FAULT_STATUS);
+	hub->vm_l2_pro_fault_cntl =
+		SOC15_REG_OFFSET(MMHUB, 0, regVM_L2_PROTECTION_FAULT_CNTL);
+
+	hub->ctx_distance = regVM_CONTEXT1_CNTL - regVM_CONTEXT0_CNTL;
+	hub->ctx_addr_distance = regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 -
+		regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
+	hub->eng_distance = regVM_INVALIDATE_ENG1_REQ - regVM_INVALIDATE_ENG0_REQ;
+	hub->eng_addr_distance = regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 -
+		regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
+
+}
+
+static int mmhub_v1_8_set_clockgating(struct amdgpu_device *adev,
+				      enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static void mmhub_v1_8_get_clockgating(struct amdgpu_device *adev, u64 *flags)
+{
+
+}
+
+const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = {
+	.get_fb_location = mmhub_v1_8_get_fb_location,
+	.init = mmhub_v1_8_init,
+	.gart_enable = mmhub_v1_8_gart_enable,
+	.set_fault_enable_default = mmhub_v1_8_set_fault_enable_default,
+	.gart_disable = mmhub_v1_8_gart_disable,
+	.setup_vm_pt_regs = mmhub_v1_8_setup_vm_pt_regs,
+	.set_clockgating = mmhub_v1_8_set_clockgating,
+	.get_clockgating = mmhub_v1_8_get_clockgating,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.h b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.h
new file mode 100644
index 000000000000..0bb36200e4e5
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __MMHUB_V1_8_H__
+#define __MMHUB_V1_8_H__
+
+extern const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c
index 164948c50ac3..17a792616979 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c
@@ -517,6 +517,9 @@ static void mmhub_v3_0_init(struct amdgpu_device *adev)
 	hub->vm_l2_bank_select_reserved_cid2 =
 		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_L2_BANK_SELECT_RESERVED_CID2);
 
+	hub->vm_contexts_disable =
+		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_CONTEXTS_DISABLE);
+
 	hub->vmhub_funcs = &mmhub_v3_0_vmhub_funcs;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mmsch_v4_0.h b/drivers/gpu/drm/amd/amdgpu/mmsch_v4_0.h
index 0312c71c3af9..83653a50a1a2 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmsch_v4_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/mmsch_v4_0.h
@@ -38,6 +38,11 @@
 #define MMSCH_VF_MAILBOX_RESP__OK 0x1
 #define MMSCH_VF_MAILBOX_RESP__INCOMPLETE 0x2
 
+#define MMSCH_VF_ENGINE_STATUS__PASS 0x1
+
+#define MMSCH_VF_MAILBOX_RESP__OK 0x1
+#define MMSCH_VF_MAILBOX_RESP__INCOMPLETE 0x2
+
 enum mmsch_v4_0_command_type {
 	MMSCH_COMMAND__DIRECT_REG_WRITE = 0,
 	MMSCH_COMMAND__DIRECT_REG_POLLING = 2,
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
index 09fdcd20cb91..d5ed9e0e1a5f 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
@@ -26,6 +26,7 @@
 
 #include "nbio/nbio_4_3_0_offset.h"
 #include "nbio/nbio_4_3_0_sh_mask.h"
+#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
 #include <uapi/linux/kfd_ioctl.h>
 
 static void nbio_v4_3_remap_hdp_registers(struct amdgpu_device *adev)
@@ -538,3 +539,81 @@ const struct amdgpu_nbio_funcs nbio_v4_3_sriov_funcs = {
 	.remap_hdp_registers = nbio_v4_3_remap_hdp_registers,
 	.get_rom_offset = nbio_v4_3_get_rom_offset,
 };
+
+static int nbio_v4_3_set_ras_err_event_athub_irq_state(struct amdgpu_device *adev,
+						       struct amdgpu_irq_src *src,
+						       unsigned type,
+						       enum amdgpu_interrupt_state state)
+{
+	/* The ras_controller_irq enablement should be done in psp bl when it
+	 * tries to enable ras feature. Driver only need to set the correct interrupt
+	 * vector for bare-metal and sriov use case respectively
+	 */
+	uint32_t bif_doorbell_int_cntl;
+
+	bif_doorbell_int_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_DOORBELL_INT_CNTL);
+	bif_doorbell_int_cntl = REG_SET_FIELD(bif_doorbell_int_cntl,
+					      BIF_BX0_BIF_DOORBELL_INT_CNTL,
+					      RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE,
+					      (state == AMDGPU_IRQ_STATE_ENABLE) ? 0 : 1);
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_DOORBELL_INT_CNTL, bif_doorbell_int_cntl);
+
+	return 0;
+}
+
+static int nbio_v4_3_process_err_event_athub_irq(struct amdgpu_device *adev,
+						 struct amdgpu_irq_src *source,
+						 struct amdgpu_iv_entry *entry)
+{
+	/* By design, the ih cookie for err_event_athub_irq should be written
+	 * to bif ring. since bif ring is not enabled, just leave process callback
+	 * as a dummy one.
+	 */
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs nbio_v4_3_ras_err_event_athub_irq_funcs = {
+	.set = nbio_v4_3_set_ras_err_event_athub_irq_state,
+	.process = nbio_v4_3_process_err_event_athub_irq,
+};
+
+static void nbio_v4_3_handle_ras_err_event_athub_intr_no_bifring(struct amdgpu_device *adev)
+{
+	uint32_t bif_doorbell_int_cntl;
+
+	bif_doorbell_int_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_DOORBELL_INT_CNTL);
+	if (REG_GET_FIELD(bif_doorbell_int_cntl,
+			  BIF_DOORBELL_INT_CNTL,
+			  RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS)) {
+		/* driver has to clear the interrupt status when bif ring is disabled */
+		bif_doorbell_int_cntl = REG_SET_FIELD(bif_doorbell_int_cntl,
+						BIF_DOORBELL_INT_CNTL,
+						RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR, 1);
+		WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_DOORBELL_INT_CNTL, bif_doorbell_int_cntl);
+		amdgpu_ras_global_ras_isr(adev);
+	}
+}
+
+static int nbio_v4_3_init_ras_err_event_athub_interrupt(struct amdgpu_device *adev)
+{
+
+	int r;
+
+	/* init the irq funcs */
+	adev->nbio.ras_err_event_athub_irq.funcs =
+		&nbio_v4_3_ras_err_event_athub_irq_funcs;
+	adev->nbio.ras_err_event_athub_irq.num_types = 1;
+
+	/* register ras err event athub interrupt
+	 * nbio v4_3 uses the same irq source as nbio v7_4 */
+	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_BIF,
+			      NBIF_7_4__SRCID__ERREVENT_ATHUB_INTERRUPT,
+			      &adev->nbio.ras_err_event_athub_irq);
+
+	return r;
+}
+
+struct amdgpu_nbio_ras nbio_v4_3_ras = {
+	.handle_ras_err_event_athub_intr_no_bifring = nbio_v4_3_handle_ras_err_event_athub_intr_no_bifring,
+	.init_ras_err_event_athub_interrupt = nbio_v4_3_init_ras_err_event_athub_interrupt,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.h b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.h
index 711999ceedf4..399037cdf4fb 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.h
@@ -29,5 +29,6 @@
 extern const struct nbio_hdp_flush_reg nbio_v4_3_hdp_flush_reg;
 extern const struct amdgpu_nbio_funcs nbio_v4_3_funcs;
 extern const struct amdgpu_nbio_funcs nbio_v4_3_sriov_funcs;
+extern struct amdgpu_nbio_ras nbio_v4_3_ras;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c
index 19455a725939..685abf57ffdd 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c
@@ -238,7 +238,7 @@ static void nbio_v7_4_ih_doorbell_range(struct amdgpu_device *adev,
 
 	if (use_doorbell) {
 		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, BIF_IH_DOORBELL_RANGE, OFFSET, doorbell_index);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, BIF_IH_DOORBELL_RANGE, SIZE, 4);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, BIF_IH_DOORBELL_RANGE, SIZE, 8);
 	} else
 		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, BIF_IH_DOORBELL_RANGE, SIZE, 0);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
new file mode 100644
index 000000000000..24d12075ca3a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
@@ -0,0 +1,369 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "amdgpu_atombios.h"
+#include "nbio_v7_9.h"
+#include "amdgpu_ras.h"
+
+#include "nbio/nbio_7_9_0_offset.h"
+#include "nbio/nbio_7_9_0_sh_mask.h"
+#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
+#include <uapi/linux/kfd_ioctl.h>
+
+static void nbio_v7_9_remap_hdp_registers(struct amdgpu_device *adev)
+{
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL,
+		adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL);
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL,
+		adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL);
+}
+
+static u32 nbio_v7_9_get_rev_id(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0);
+	tmp = REG_GET_FIELD(tmp, RCC_STRAP0_RCC_DEV0_EPF0_STRAP0, STRAP_ATI_REV_ID_DEV0_F0);
+
+	return tmp;
+}
+
+static void nbio_v7_9_mc_access_enable(struct amdgpu_device *adev, bool enable)
+{
+	if (enable)
+		WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN,
+			BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK | BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK);
+	else
+		WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0);
+}
+
+static u32 nbio_v7_9_get_memsize(struct amdgpu_device *adev)
+{
+	return RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE);
+}
+
+static void nbio_v7_9_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
+			bool use_doorbell, int doorbell_index, int doorbell_size)
+{
+	u32 doorbell_range = 0, doorbell_ctrl = 0;
+
+	doorbell_range =
+		REG_SET_FIELD(doorbell_range, DOORBELL0_CTRL_ENTRY_0,
+			BIF_DOORBELL0_RANGE_OFFSET_ENTRY, doorbell_index);
+	doorbell_range =
+		REG_SET_FIELD(doorbell_range, DOORBELL0_CTRL_ENTRY_0,
+			BIF_DOORBELL0_RANGE_SIZE_ENTRY, doorbell_size);
+	doorbell_ctrl =
+		REG_SET_FIELD(doorbell_ctrl, S2A_DOORBELL_ENTRY_1_CTRL,
+			S2A_DOORBELL_PORT1_ENABLE, 1);
+	doorbell_ctrl =
+		REG_SET_FIELD(doorbell_ctrl, S2A_DOORBELL_ENTRY_1_CTRL,
+			S2A_DOORBELL_PORT1_RANGE_SIZE, doorbell_size);
+
+	switch (instance) {
+	case 0:
+		WREG32_SOC15(NBIO, 0, regDOORBELL0_CTRL_ENTRY_1, doorbell_range);
+
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_AWID, 0xe);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_RANGE_OFFSET, 0xe);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
+					0x1);
+		WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_1_CTRL, doorbell_ctrl);
+		break;
+	case 1:
+		WREG32_SOC15(NBIO, 0, regDOORBELL0_CTRL_ENTRY_2, doorbell_range);
+
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_AWID, 0x8);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_RANGE_OFFSET, 0x8);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
+					0x2);
+		WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_2_CTRL, doorbell_ctrl);
+		break;
+	case 2:
+		WREG32_SOC15(NBIO, 0, regDOORBELL0_CTRL_ENTRY_3, doorbell_range);
+
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_AWID, 0x9);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_RANGE_OFFSET, 0x9);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
+					0x8);
+		WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_5_CTRL, doorbell_ctrl);
+		break;
+	case 3:
+		WREG32_SOC15(NBIO, 0, regDOORBELL0_CTRL_ENTRY_4, doorbell_range);
+
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_AWID, 0xa);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_RANGE_OFFSET, 0xa);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+					S2A_DOORBELL_ENTRY_1_CTRL,
+					S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
+					0x9);
+		WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_6_CTRL, doorbell_ctrl);
+		break;
+	default:
+		break;
+	};
+
+	return;
+}
+
+static void nbio_v7_9_vcn_doorbell_range(struct amdgpu_device *adev, bool use_doorbell,
+					 int doorbell_index, int instance)
+{
+	u32 doorbell_range = 0, doorbell_ctrl = 0;
+
+	if (use_doorbell) {
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+				DOORBELL0_CTRL_ENTRY_0,
+				BIF_DOORBELL0_RANGE_OFFSET_ENTRY,
+				doorbell_index);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+				DOORBELL0_CTRL_ENTRY_0,
+				BIF_DOORBELL0_RANGE_SIZE_ENTRY,
+				0x8);
+
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_ENABLE, 1);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_AWID, 0x4);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_RANGE_OFFSET, 0x4);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_RANGE_SIZE, 0x8);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, 0x4);
+	} else {
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+				DOORBELL0_CTRL_ENTRY_0,
+				BIF_DOORBELL0_RANGE_SIZE_ENTRY, 0);
+		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_RANGE_SIZE, 0);
+	}
+
+	WREG32_SOC15(NBIO, 0, regDOORBELL0_CTRL_ENTRY_17, doorbell_range);
+	WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_4_CTRL, doorbell_ctrl);
+}
+
+static void nbio_v7_9_enable_doorbell_aperture(struct amdgpu_device *adev,
+					       bool enable)
+{
+	/* Enable to allow doorbell pass thru on pre-silicon bare-metal */
+	WREG32_SOC15(NBIO, 0, regBIFC_DOORBELL_ACCESS_EN_PF, 0xfffff);
+	WREG32_FIELD15_PREREG(NBIO, 0, RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN,
+			BIF_DOORBELL_APER_EN, enable ? 1 : 0);
+}
+
+static void nbio_v7_9_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
+							bool enable)
+{
+	u32 tmp = 0;
+
+	if (enable) {
+		tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_EN, 1) |
+		      REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_MODE, 1) |
+		      REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_SIZE, 0);
+
+		WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW,
+			     lower_32_bits(adev->doorbell.base));
+		WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH,
+			     upper_32_bits(adev->doorbell.base));
+	}
+
+	WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, tmp);
+}
+
+static void nbio_v7_9_ih_doorbell_range(struct amdgpu_device *adev,
+					bool use_doorbell, int doorbell_index)
+{
+	u32 ih_doorbell_range = 0, ih_doorbell_ctrl = 0;
+
+	if (use_doorbell) {
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+				DOORBELL0_CTRL_ENTRY_0,
+				BIF_DOORBELL0_RANGE_OFFSET_ENTRY,
+				doorbell_index);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+				DOORBELL0_CTRL_ENTRY_0,
+				BIF_DOORBELL0_RANGE_SIZE_ENTRY,
+				0x4);
+
+		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_ENABLE, 1);
+		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_AWID, 0);
+		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_RANGE_OFFSET, 0);
+		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_RANGE_SIZE, 0x4);
+		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, 0);
+	} else {
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+				DOORBELL0_CTRL_ENTRY_0,
+				BIF_DOORBELL0_RANGE_SIZE_ENTRY, 0);
+		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
+				S2A_DOORBELL_ENTRY_1_CTRL,
+				S2A_DOORBELL_PORT1_RANGE_SIZE, 0);
+	}
+
+	WREG32_SOC15(NBIO, 0, regDOORBELL0_CTRL_ENTRY_0, ih_doorbell_range);
+	WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_3_CTRL, ih_doorbell_ctrl);
+}
+
+
+static void nbio_v7_9_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+						       bool enable)
+{
+}
+
+static void nbio_v7_9_update_medium_grain_light_sleep(struct amdgpu_device *adev,
+						      bool enable)
+{
+}
+
+static void nbio_v7_9_get_clockgating_state(struct amdgpu_device *adev,
+					    u64 *flags)
+{
+}
+
+static void nbio_v7_9_ih_control(struct amdgpu_device *adev)
+{
+	u32 interrupt_cntl;
+
+	/* setup interrupt control */
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
+	interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL);
+	/* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
+	 * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
+	 */
+	interrupt_cntl =
+		REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL, IH_DUMMY_RD_OVERRIDE, 0);
+	/* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
+	interrupt_cntl =
+		REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0);
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL, interrupt_cntl);
+}
+
+static u32 nbio_v7_9_get_hdp_flush_req_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_REQ);
+}
+
+static u32 nbio_v7_9_get_hdp_flush_done_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_DONE);
+}
+
+static u32 nbio_v7_9_get_pcie_index_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2);
+}
+
+static u32 nbio_v7_9_get_pcie_data_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_DATA2);
+}
+
+const struct nbio_hdp_flush_reg nbio_v7_9_hdp_flush_reg = {
+	.ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK,
+	.ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK,
+	.ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK,
+	.ref_and_mask_cp3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK,
+	.ref_and_mask_cp4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK,
+	.ref_and_mask_cp5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK,
+	.ref_and_mask_cp6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK,
+	.ref_and_mask_cp7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK,
+	.ref_and_mask_cp8 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK,
+	.ref_and_mask_cp9 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK,
+	.ref_and_mask_sdma0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK,
+	.ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK,
+	.ref_and_mask_sdma2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK,
+	.ref_and_mask_sdma3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK,
+	.ref_and_mask_sdma4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK,
+	.ref_and_mask_sdma5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK,
+	.ref_and_mask_sdma6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK,
+	.ref_and_mask_sdma7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK,
+};
+
+static void nbio_v7_9_enable_doorbell_interrupt(struct amdgpu_device *adev,
+						bool enable)
+{
+	WREG32_FIELD15_PREREG(NBIO, 0, BIF_BX0_BIF_DOORBELL_INT_CNTL,
+			      DOORBELL_INTERRUPT_DISABLE, enable ? 0 : 1);
+}
+
+const struct amdgpu_nbio_funcs nbio_v7_9_funcs = {
+	.get_hdp_flush_req_offset = nbio_v7_9_get_hdp_flush_req_offset,
+	.get_hdp_flush_done_offset = nbio_v7_9_get_hdp_flush_done_offset,
+	.get_pcie_index_offset = nbio_v7_9_get_pcie_index_offset,
+	.get_pcie_data_offset = nbio_v7_9_get_pcie_data_offset,
+	.get_rev_id = nbio_v7_9_get_rev_id,
+	.mc_access_enable = nbio_v7_9_mc_access_enable,
+	.get_memsize = nbio_v7_9_get_memsize,
+	.sdma_doorbell_range = nbio_v7_9_sdma_doorbell_range,
+	.vcn_doorbell_range = nbio_v7_9_vcn_doorbell_range,
+	.enable_doorbell_aperture = nbio_v7_9_enable_doorbell_aperture,
+	.enable_doorbell_selfring_aperture = nbio_v7_9_enable_doorbell_selfring_aperture,
+	.ih_doorbell_range = nbio_v7_9_ih_doorbell_range,
+	.enable_doorbell_interrupt = nbio_v7_9_enable_doorbell_interrupt,
+	.update_medium_grain_clock_gating = nbio_v7_9_update_medium_grain_clock_gating,
+	.update_medium_grain_light_sleep = nbio_v7_9_update_medium_grain_light_sleep,
+	.get_clockgating_state = nbio_v7_9_get_clockgating_state,
+	.ih_control = nbio_v7_9_ih_control,
+	.remap_hdp_registers = nbio_v7_9_remap_hdp_registers,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.h b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.h
new file mode 100644
index 000000000000..8e04eb484328
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __NBIO_V7_9_H__
+#define __NBIO_V7_9_H__
+
+#include "soc15_common.h"
+
+extern const struct nbio_hdp_flush_reg nbio_v7_9_hdp_flush_reg;
+extern const struct amdgpu_nbio_funcs nbio_v7_9_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c
index ebe0e2d7dbd1..47420b403871 100644
--- a/drivers/gpu/drm/amd/amdgpu/nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/nv.c
@@ -280,47 +280,6 @@ static int nv_query_video_codecs(struct amdgpu_device *adev, bool encode,
 	}
 }
 
-/*
- * Indirect registers accessor
- */
-static u32 nv_pcie_rreg(struct amdgpu_device *adev, u32 reg)
-{
-	unsigned long address, data;
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	return amdgpu_device_indirect_rreg(adev, address, data, reg);
-}
-
-static void nv_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
-{
-	unsigned long address, data;
-
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	amdgpu_device_indirect_wreg(adev, address, data, reg, v);
-}
-
-static u64 nv_pcie_rreg64(struct amdgpu_device *adev, u32 reg)
-{
-	unsigned long address, data;
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	return amdgpu_device_indirect_rreg64(adev, address, data, reg);
-}
-
-static void nv_pcie_wreg64(struct amdgpu_device *adev, u32 reg, u64 v)
-{
-	unsigned long address, data;
-
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	amdgpu_device_indirect_wreg64(adev, address, data, reg, v);
-}
-
 static u32 nv_didt_rreg(struct amdgpu_device *adev, u32 reg)
 {
 	unsigned long flags, address, data;
@@ -561,21 +520,6 @@ static int nv_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
 	return 0;
 }
 
-static void nv_pcie_gen3_enable(struct amdgpu_device *adev)
-{
-	if (pci_is_root_bus(adev->pdev->bus))
-		return;
-
-	if (amdgpu_pcie_gen2 == 0)
-		return;
-
-	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
-					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
-		return;
-
-	/* todo */
-}
-
 static void nv_program_aspm(struct amdgpu_device *adev)
 {
 	if (!amdgpu_device_should_use_aspm(adev) || !amdgpu_device_aspm_support_quirk())
@@ -608,11 +552,6 @@ void nv_set_virt_ops(struct amdgpu_device *adev)
 	adev->virt.ops = &xgpu_nv_virt_ops;
 }
 
-static uint32_t nv_get_rev_id(struct amdgpu_device *adev)
-{
-	return adev->nbio.funcs->get_rev_id(adev);
-}
-
 static bool nv_need_full_reset(struct amdgpu_device *adev)
 {
 	return true;
@@ -738,10 +677,10 @@ static int nv_common_early_init(void *handle)
 	}
 	adev->smc_rreg = NULL;
 	adev->smc_wreg = NULL;
-	adev->pcie_rreg = &nv_pcie_rreg;
-	adev->pcie_wreg = &nv_pcie_wreg;
-	adev->pcie_rreg64 = &nv_pcie_rreg64;
-	adev->pcie_wreg64 = &nv_pcie_wreg64;
+	adev->pcie_rreg = &amdgpu_device_indirect_rreg;
+	adev->pcie_wreg = &amdgpu_device_indirect_wreg;
+	adev->pcie_rreg64 = &amdgpu_device_indirect_rreg64;
+	adev->pcie_wreg64 = &amdgpu_device_indirect_wreg64;
 	adev->pciep_rreg = amdgpu_device_pcie_port_rreg;
 	adev->pciep_wreg = amdgpu_device_pcie_port_wreg;
 
@@ -754,7 +693,7 @@ static int nv_common_early_init(void *handle)
 
 	adev->asic_funcs = &nv_asic_funcs;
 
-	adev->rev_id = nv_get_rev_id(adev);
+	adev->rev_id = amdgpu_device_get_rev_id(adev);
 	adev->external_rev_id = 0xff;
 	/* TODO: split the GC and PG flags based on the relevant IP version for which
 	 * they are relevant.
@@ -1088,8 +1027,6 @@ static int nv_common_hw_init(void *handle)
 	if (adev->nbio.funcs->apply_l1_link_width_reconfig_wa)
 		adev->nbio.funcs->apply_l1_link_width_reconfig_wa(adev);
 
-	/* enable pcie gen2/3 link */
-	nv_pcie_gen3_enable(adev);
 	/* enable aspm */
 	nv_program_aspm(adev);
 	/* setup nbio registers */
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
index d62fcc77af95..caee76ab7110 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
@@ -48,6 +48,7 @@ MODULE_FIRMWARE("amdgpu/psp_13_0_10_sos.bin");
 MODULE_FIRMWARE("amdgpu/psp_13_0_10_ta.bin");
 MODULE_FIRMWARE("amdgpu/psp_13_0_11_toc.bin");
 MODULE_FIRMWARE("amdgpu/psp_13_0_11_ta.bin");
+MODULE_FIRMWARE("amdgpu/psp_13_0_6_sos.bin");
 
 /* For large FW files the time to complete can be very long */
 #define USBC_PD_POLLING_LIMIT_S 240
@@ -100,6 +101,7 @@ static int psp_v13_0_init_microcode(struct psp_context *psp)
 			return err;
 		break;
 	case IP_VERSION(13, 0, 0):
+	case IP_VERSION(13, 0, 6):
 	case IP_VERSION(13, 0, 7):
 	case IP_VERSION(13, 0, 10):
 		err = psp_init_sos_microcode(psp, ucode_prefix);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index b5affba22156..b3cc04dd8653 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -1823,6 +1823,15 @@ static int sdma_v4_0_sw_init(void *handle)
 		/* doorbell size is 2 dwords, get DWORD offset */
 		ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
 
+		/*
+		 * On Arcturus, SDMA instance 5~7 has a different vmhub
+		 * type(AMDGPU_MMHUB_1).
+		 */
+		if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 2) && i >= 5)
+			ring->vm_hub = AMDGPU_MMHUB_1;
+		else
+			ring->vm_hub = AMDGPU_MMHUB_0;
+
 		sprintf(ring->name, "sdma%d", i);
 		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
 				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
@@ -1841,6 +1850,11 @@ static int sdma_v4_0_sw_init(void *handle)
 			ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
 			ring->doorbell_index += 0x400;
 
+			if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 2) && i >= 5)
+				ring->vm_hub = AMDGPU_MMHUB_1;
+			else
+				ring->vm_hub = AMDGPU_MMHUB_0;
+
 			sprintf(ring->name, "page%d", i);
 			r = amdgpu_ring_init(adev, ring, 1024,
 					     &adev->sdma.trap_irq,
@@ -1870,7 +1884,7 @@ static int sdma_v4_0_sw_fini(void *handle)
 			amdgpu_ring_fini(&adev->sdma.instance[i].page);
 	}
 
-	if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 0) ||
+	if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 2) ||
             adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 0))
 		amdgpu_sdma_destroy_inst_ctx(adev, true);
 	else
@@ -2294,44 +2308,6 @@ static const struct amdgpu_ring_funcs sdma_v4_0_ring_funcs = {
 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
 	.support_64bit_ptrs = true,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_MMHUB_0,
-	.get_rptr = sdma_v4_0_ring_get_rptr,
-	.get_wptr = sdma_v4_0_ring_get_wptr,
-	.set_wptr = sdma_v4_0_ring_set_wptr,
-	.emit_frame_size =
-		6 + /* sdma_v4_0_ring_emit_hdp_flush */
-		3 + /* hdp invalidate */
-		6 + /* sdma_v4_0_ring_emit_pipeline_sync */
-		/* sdma_v4_0_ring_emit_vm_flush */
-		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
-		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
-		10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */
-	.emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */
-	.emit_ib = sdma_v4_0_ring_emit_ib,
-	.emit_fence = sdma_v4_0_ring_emit_fence,
-	.emit_pipeline_sync = sdma_v4_0_ring_emit_pipeline_sync,
-	.emit_vm_flush = sdma_v4_0_ring_emit_vm_flush,
-	.emit_hdp_flush = sdma_v4_0_ring_emit_hdp_flush,
-	.test_ring = sdma_v4_0_ring_test_ring,
-	.test_ib = sdma_v4_0_ring_test_ib,
-	.insert_nop = sdma_v4_0_ring_insert_nop,
-	.pad_ib = sdma_v4_0_ring_pad_ib,
-	.emit_wreg = sdma_v4_0_ring_emit_wreg,
-	.emit_reg_wait = sdma_v4_0_ring_emit_reg_wait,
-	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
-};
-
-/*
- * On Arcturus, SDMA instance 5~7 has a different vmhub type(AMDGPU_MMHUB_1).
- * So create a individual constant ring_funcs for those instances.
- */
-static const struct amdgpu_ring_funcs sdma_v4_0_ring_funcs_2nd_mmhub = {
-	.type = AMDGPU_RING_TYPE_SDMA,
-	.align_mask = 0xf,
-	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
-	.support_64bit_ptrs = true,
-	.secure_submission_supported = true,
-	.vmhub = AMDGPU_MMHUB_1,
 	.get_rptr = sdma_v4_0_ring_get_rptr,
 	.get_wptr = sdma_v4_0_ring_get_wptr,
 	.set_wptr = sdma_v4_0_ring_set_wptr,
@@ -2364,40 +2340,6 @@ static const struct amdgpu_ring_funcs sdma_v4_0_page_ring_funcs = {
 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
 	.support_64bit_ptrs = true,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_MMHUB_0,
-	.get_rptr = sdma_v4_0_ring_get_rptr,
-	.get_wptr = sdma_v4_0_page_ring_get_wptr,
-	.set_wptr = sdma_v4_0_page_ring_set_wptr,
-	.emit_frame_size =
-		6 + /* sdma_v4_0_ring_emit_hdp_flush */
-		3 + /* hdp invalidate */
-		6 + /* sdma_v4_0_ring_emit_pipeline_sync */
-		/* sdma_v4_0_ring_emit_vm_flush */
-		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
-		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
-		10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */
-	.emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */
-	.emit_ib = sdma_v4_0_ring_emit_ib,
-	.emit_fence = sdma_v4_0_ring_emit_fence,
-	.emit_pipeline_sync = sdma_v4_0_ring_emit_pipeline_sync,
-	.emit_vm_flush = sdma_v4_0_ring_emit_vm_flush,
-	.emit_hdp_flush = sdma_v4_0_ring_emit_hdp_flush,
-	.test_ring = sdma_v4_0_ring_test_ring,
-	.test_ib = sdma_v4_0_ring_test_ib,
-	.insert_nop = sdma_v4_0_ring_insert_nop,
-	.pad_ib = sdma_v4_0_ring_pad_ib,
-	.emit_wreg = sdma_v4_0_ring_emit_wreg,
-	.emit_reg_wait = sdma_v4_0_ring_emit_reg_wait,
-	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
-};
-
-static const struct amdgpu_ring_funcs sdma_v4_0_page_ring_funcs_2nd_mmhub = {
-	.type = AMDGPU_RING_TYPE_SDMA,
-	.align_mask = 0xf,
-	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
-	.support_64bit_ptrs = true,
-	.secure_submission_supported = true,
-	.vmhub = AMDGPU_MMHUB_1,
 	.get_rptr = sdma_v4_0_ring_get_rptr,
 	.get_wptr = sdma_v4_0_page_ring_get_wptr,
 	.set_wptr = sdma_v4_0_page_ring_set_wptr,
@@ -2429,19 +2371,10 @@ static void sdma_v4_0_set_ring_funcs(struct amdgpu_device *adev)
 	int i;
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
-		if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 2) && i >= 5)
-			adev->sdma.instance[i].ring.funcs =
-					&sdma_v4_0_ring_funcs_2nd_mmhub;
-		else
-			adev->sdma.instance[i].ring.funcs =
-					&sdma_v4_0_ring_funcs;
+		adev->sdma.instance[i].ring.funcs = &sdma_v4_0_ring_funcs;
 		adev->sdma.instance[i].ring.me = i;
 		if (adev->sdma.has_page_queue) {
-			if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 2) && i >= 5)
-				adev->sdma.instance[i].page.funcs =
-					&sdma_v4_0_page_ring_funcs_2nd_mmhub;
-			else
-				adev->sdma.instance[i].page.funcs =
+			adev->sdma.instance[i].page.funcs =
 					&sdma_v4_0_page_ring_funcs;
 			adev->sdma.instance[i].page.me = i;
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
new file mode 100644
index 000000000000..64dcaa2670dd
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
@@ -0,0 +1,1967 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include "amdgpu.h"
+#include "amdgpu_ucode.h"
+#include "amdgpu_trace.h"
+
+#include "sdma/sdma_4_4_2_offset.h"
+#include "sdma/sdma_4_4_2_sh_mask.h"
+
+#include "soc15_common.h"
+#include "soc15.h"
+#include "vega10_sdma_pkt_open.h"
+
+#include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h"
+#include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h"
+
+#include "amdgpu_ras.h"
+
+MODULE_FIRMWARE("amdgpu/sdma_4_4_2.bin");
+
+#define WREG32_SDMA(instance, offset, value) \
+	WREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)), value)
+#define RREG32_SDMA(instance, offset) \
+	RREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)))
+
+static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev);
+static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev);
+static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev);
+static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev);
+
+static u32 sdma_v4_4_2_get_reg_offset(struct amdgpu_device *adev,
+		u32 instance, u32 offset)
+{
+	return (adev->reg_offset[SDMA0_HWIP][instance][0] + offset);
+}
+
+static unsigned sdma_v4_4_2_seq_to_irq_id(int seq_num)
+{
+	switch (seq_num) {
+	case 0:
+		return SOC15_IH_CLIENTID_SDMA0;
+	case 1:
+		return SOC15_IH_CLIENTID_SDMA1;
+	case 2:
+		return SOC15_IH_CLIENTID_SDMA2;
+	case 3:
+		return SOC15_IH_CLIENTID_SDMA3;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sdma_v4_4_2_irq_id_to_seq(unsigned client_id)
+{
+	switch (client_id) {
+	case SOC15_IH_CLIENTID_SDMA0:
+		return 0;
+	case SOC15_IH_CLIENTID_SDMA1:
+		return 1;
+	case SOC15_IH_CLIENTID_SDMA2:
+		return 2;
+	case SOC15_IH_CLIENTID_SDMA3:
+		return 3;
+	default:
+		return -EINVAL;
+	}
+}
+
+static void sdma_v4_4_2_init_golden_registers(struct amdgpu_device *adev)
+{
+	switch (adev->ip_versions[SDMA0_HWIP][0]) {
+	case IP_VERSION(4, 4, 2):
+		break;
+	default:
+		break;
+	}
+}
+
+/**
+ * sdma_v4_4_2_init_microcode - load ucode images from disk
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Use the firmware interface to load the ucode images into
+ * the driver (not loaded into hw).
+ * Returns 0 on success, error on failure.
+ */
+static int sdma_v4_4_2_init_microcode(struct amdgpu_device *adev)
+{
+	int ret, i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 2)) {
+			ret = amdgpu_sdma_init_microcode(adev, 0, true);
+			break;
+		} else {
+			ret = amdgpu_sdma_init_microcode(adev, i, false);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return ret;
+}
+
+/**
+ * sdma_v4_4_2_ring_get_rptr - get the current read pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Get the current rptr from the hardware.
+ */
+static uint64_t sdma_v4_4_2_ring_get_rptr(struct amdgpu_ring *ring)
+{
+	u64 *rptr;
+
+	/* XXX check if swapping is necessary on BE */
+	rptr = ((u64 *)&ring->adev->wb.wb[ring->rptr_offs]);
+
+	DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
+	return ((*rptr) >> 2);
+}
+
+/**
+ * sdma_v4_4_2_ring_get_wptr - get the current write pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Get the current wptr from the hardware.
+ */
+static uint64_t sdma_v4_4_2_ring_get_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	u64 wptr;
+
+	if (ring->use_doorbell) {
+		/* XXX check if swapping is necessary on BE */
+		wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
+		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
+	} else {
+		wptr = RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI);
+		wptr = wptr << 32;
+		wptr |= RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR);
+		DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n",
+				ring->me, wptr);
+	}
+
+	return wptr >> 2;
+}
+
+/**
+ * sdma_v4_4_2_ring_set_wptr - commit the write pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Write the wptr back to the hardware.
+ */
+static void sdma_v4_4_2_ring_set_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	DRM_DEBUG("Setting write pointer\n");
+	if (ring->use_doorbell) {
+		u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
+
+		DRM_DEBUG("Using doorbell -- "
+				"wptr_offs == 0x%08x "
+				"lower_32_bits(ring->wptr) << 2 == 0x%08x "
+				"upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
+				ring->wptr_offs,
+				lower_32_bits(ring->wptr << 2),
+				upper_32_bits(ring->wptr << 2));
+		/* XXX check if swapping is necessary on BE */
+		WRITE_ONCE(*wb, (ring->wptr << 2));
+		DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
+				ring->doorbell_index, ring->wptr << 2);
+		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
+	} else {
+		DRM_DEBUG("Not using doorbell -- "
+				"regSDMA%i_GFX_RB_WPTR == 0x%08x "
+				"regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
+				ring->me,
+				lower_32_bits(ring->wptr << 2),
+				ring->me,
+				upper_32_bits(ring->wptr << 2));
+		WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR,
+			    lower_32_bits(ring->wptr << 2));
+		WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI,
+			    upper_32_bits(ring->wptr << 2));
+	}
+}
+
+/**
+ * sdma_v4_4_2_page_ring_get_wptr - get the current write pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Get the current wptr from the hardware.
+ */
+static uint64_t sdma_v4_4_2_page_ring_get_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	u64 wptr;
+
+	if (ring->use_doorbell) {
+		/* XXX check if swapping is necessary on BE */
+		wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
+	} else {
+		wptr = RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI);
+		wptr = wptr << 32;
+		wptr |= RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR);
+	}
+
+	return wptr >> 2;
+}
+
+/**
+ * sdma_v4_4_2_page_ring_set_wptr - commit the write pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Write the wptr back to the hardware.
+ */
+static void sdma_v4_4_2_page_ring_set_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	if (ring->use_doorbell) {
+		u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
+
+		/* XXX check if swapping is necessary on BE */
+		WRITE_ONCE(*wb, (ring->wptr << 2));
+		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
+	} else {
+		uint64_t wptr = ring->wptr << 2;
+
+		WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR,
+			    lower_32_bits(wptr));
+		WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI,
+			    upper_32_bits(wptr));
+	}
+}
+
+static void sdma_v4_4_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	int i;
+
+	for (i = 0; i < count; i++)
+		if (sdma && sdma->burst_nop && (i == 0))
+			amdgpu_ring_write(ring, ring->funcs->nop |
+				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
+		else
+			amdgpu_ring_write(ring, ring->funcs->nop);
+}
+
+/**
+ * sdma_v4_4_2_ring_emit_ib - Schedule an IB on the DMA engine
+ *
+ * @ring: amdgpu ring pointer
+ * @job: job to retrieve vmid from
+ * @ib: IB object to schedule
+ * @flags: unused
+ *
+ * Schedule an IB in the DMA ring.
+ */
+static void sdma_v4_4_2_ring_emit_ib(struct amdgpu_ring *ring,
+				   struct amdgpu_job *job,
+				   struct amdgpu_ib *ib,
+				   uint32_t flags)
+{
+	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
+
+	/* IB packet must end on a 8 DW boundary */
+	sdma_v4_4_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
+
+	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
+			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
+	/* base must be 32 byte aligned */
+	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
+	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+	amdgpu_ring_write(ring, ib->length_dw);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, 0);
+
+}
+
+static void sdma_v4_4_2_wait_reg_mem(struct amdgpu_ring *ring,
+				   int mem_space, int hdp,
+				   uint32_t addr0, uint32_t addr1,
+				   uint32_t ref, uint32_t mask,
+				   uint32_t inv)
+{
+	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(hdp) |
+			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(mem_space) |
+			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
+	if (mem_space) {
+		/* memory */
+		amdgpu_ring_write(ring, addr0);
+		amdgpu_ring_write(ring, addr1);
+	} else {
+		/* registers */
+		amdgpu_ring_write(ring, addr0 << 2);
+		amdgpu_ring_write(ring, addr1 << 2);
+	}
+	amdgpu_ring_write(ring, ref); /* reference */
+	amdgpu_ring_write(ring, mask); /* mask */
+	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
+			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(inv)); /* retry count, poll interval */
+}
+
+/**
+ * sdma_v4_4_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Emit an hdp flush packet on the requested DMA ring.
+ */
+static void sdma_v4_4_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	u32 ref_and_mask = 0;
+	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
+
+	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
+
+	sdma_v4_4_2_wait_reg_mem(ring, 0, 1,
+			       adev->nbio.funcs->get_hdp_flush_done_offset(adev),
+			       adev->nbio.funcs->get_hdp_flush_req_offset(adev),
+			       ref_and_mask, ref_and_mask, 10);
+}
+
+/**
+ * sdma_v4_4_2_ring_emit_fence - emit a fence on the DMA ring
+ *
+ * @ring: amdgpu ring pointer
+ * @addr: address
+ * @seq: sequence number
+ * @flags: fence related flags
+ *
+ * Add a DMA fence packet to the ring to write
+ * the fence seq number and DMA trap packet to generate
+ * an interrupt if needed.
+ */
+static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
+				      unsigned flags)
+{
+	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
+	/* write the fence */
+	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
+	/* zero in first two bits */
+	BUG_ON(addr & 0x3);
+	amdgpu_ring_write(ring, lower_32_bits(addr));
+	amdgpu_ring_write(ring, upper_32_bits(addr));
+	amdgpu_ring_write(ring, lower_32_bits(seq));
+
+	/* optionally write high bits as well */
+	if (write64bit) {
+		addr += 4;
+		amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
+		/* zero in first two bits */
+		BUG_ON(addr & 0x3);
+		amdgpu_ring_write(ring, lower_32_bits(addr));
+		amdgpu_ring_write(ring, upper_32_bits(addr));
+		amdgpu_ring_write(ring, upper_32_bits(seq));
+	}
+
+	/* generate an interrupt */
+	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
+	amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
+}
+
+
+/**
+ * sdma_v4_4_2_gfx_stop - stop the gfx async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Stop the gfx async dma ring buffers.
+ */
+static void sdma_v4_4_2_gfx_stop(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
+	u32 rb_cntl, ib_cntl;
+	int i, unset = 0;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		sdma[i] = &adev->sdma.instance[i].ring;
+
+		if ((adev->mman.buffer_funcs_ring == sdma[i]) && unset != 1) {
+			amdgpu_ttm_set_buffer_funcs_status(adev, false);
+			unset = 1;
+		}
+
+		rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
+		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 0);
+		WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
+		ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
+		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 0);
+		WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
+	}
+}
+
+/**
+ * sdma_v4_4_2_rlc_stop - stop the compute async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Stop the compute async dma queues.
+ */
+static void sdma_v4_4_2_rlc_stop(struct amdgpu_device *adev)
+{
+	/* XXX todo */
+}
+
+/**
+ * sdma_v4_4_2_page_stop - stop the page async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Stop the page async dma ring buffers.
+ */
+static void sdma_v4_4_2_page_stop(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
+	u32 rb_cntl, ib_cntl;
+	int i;
+	bool unset = false;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		sdma[i] = &adev->sdma.instance[i].page;
+
+		if ((adev->mman.buffer_funcs_ring == sdma[i]) &&
+			(!unset)) {
+			amdgpu_ttm_set_buffer_funcs_status(adev, false);
+			unset = true;
+		}
+
+		rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
+		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
+					RB_ENABLE, 0);
+		WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
+		ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
+		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL,
+					IB_ENABLE, 0);
+		WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
+	}
+}
+
+/**
+ * sdma_v4_4_2_ctx_switch_enable - stop the async dma engines context switch
+ *
+ * @adev: amdgpu_device pointer
+ * @enable: enable/disable the DMA MEs context switch.
+ *
+ * Halt or unhalt the async dma engines context switch.
+ */
+static void sdma_v4_4_2_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
+{
+	u32 f32_cntl, phase_quantum = 0;
+	int i;
+
+	if (amdgpu_sdma_phase_quantum) {
+		unsigned value = amdgpu_sdma_phase_quantum;
+		unsigned unit = 0;
+
+		while (value > (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
+				SDMA_PHASE0_QUANTUM__VALUE__SHIFT)) {
+			value = (value + 1) >> 1;
+			unit++;
+		}
+		if (unit > (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
+			    SDMA_PHASE0_QUANTUM__UNIT__SHIFT)) {
+			value = (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
+				 SDMA_PHASE0_QUANTUM__VALUE__SHIFT);
+			unit = (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
+				SDMA_PHASE0_QUANTUM__UNIT__SHIFT);
+			WARN_ONCE(1,
+			"clamping sdma_phase_quantum to %uK clock cycles\n",
+				  value << unit);
+		}
+		phase_quantum =
+			value << SDMA_PHASE0_QUANTUM__VALUE__SHIFT |
+			unit  << SDMA_PHASE0_QUANTUM__UNIT__SHIFT;
+	}
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		f32_cntl = RREG32_SDMA(i, regSDMA_CNTL);
+		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_CNTL,
+				AUTO_CTXSW_ENABLE, enable ? 1 : 0);
+		if (enable && amdgpu_sdma_phase_quantum) {
+			WREG32_SDMA(i, regSDMA_PHASE0_QUANTUM, phase_quantum);
+			WREG32_SDMA(i, regSDMA_PHASE1_QUANTUM, phase_quantum);
+			WREG32_SDMA(i, regSDMA_PHASE2_QUANTUM, phase_quantum);
+		}
+		WREG32_SDMA(i, regSDMA_CNTL, f32_cntl);
+
+		/* Extend page fault timeout to avoid interrupt storm */
+		WREG32_SDMA(i, regSDMA_UTCL1_TIMEOUT, 0x00800080);
+	}
+
+}
+
+/**
+ * sdma_v4_4_2_enable - stop the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ * @enable: enable/disable the DMA MEs.
+ *
+ * Halt or unhalt the async dma engines.
+ */
+static void sdma_v4_4_2_enable(struct amdgpu_device *adev, bool enable)
+{
+	u32 f32_cntl;
+	int i;
+
+	if (!enable) {
+		sdma_v4_4_2_gfx_stop(adev);
+		sdma_v4_4_2_rlc_stop(adev);
+		if (adev->sdma.has_page_queue)
+			sdma_v4_4_2_page_stop(adev);
+	}
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		f32_cntl = RREG32_SDMA(i, regSDMA_F32_CNTL);
+		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_F32_CNTL, HALT, enable ? 0 : 1);
+		WREG32_SDMA(i, regSDMA_F32_CNTL, f32_cntl);
+	}
+}
+
+/*
+ * sdma_v4_4_2_rb_cntl - get parameters for rb_cntl
+ */
+static uint32_t sdma_v4_4_2_rb_cntl(struct amdgpu_ring *ring, uint32_t rb_cntl)
+{
+	/* Set ring buffer size in dwords */
+	uint32_t rb_bufsz = order_base_2(ring->ring_size / 4);
+
+	barrier(); /* work around https://bugs.llvm.org/show_bug.cgi?id=42576 */
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
+#ifdef __BIG_ENDIAN
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
+				RPTR_WRITEBACK_SWAP_ENABLE, 1);
+#endif
+	return rb_cntl;
+}
+
+/**
+ * sdma_v4_4_2_gfx_resume - setup and start the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ * @i: instance to resume
+ *
+ * Set up the gfx DMA ring buffers and enable them.
+ * Returns 0 for success, error for failure.
+ */
+static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i)
+{
+	struct amdgpu_ring *ring = &adev->sdma.instance[i].ring;
+	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
+	u32 wb_offset;
+	u32 doorbell;
+	u32 doorbell_offset;
+	u64 wptr_gpu_addr;
+
+	wb_offset = (ring->rptr_offs * 4);
+
+	rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
+	rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
+	WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
+
+	/* Initialize the ring buffer's read and write pointers */
+	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, 0);
+	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, 0);
+	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, 0);
+	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, 0);
+
+	/* set the wb address whether it's enabled or not */
+	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_HI,
+	       upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
+	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_LO,
+	       lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
+
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
+				RPTR_WRITEBACK_ENABLE, 1);
+
+	WREG32_SDMA(i, regSDMA_GFX_RB_BASE, ring->gpu_addr >> 8);
+	WREG32_SDMA(i, regSDMA_GFX_RB_BASE_HI, ring->gpu_addr >> 40);
+
+	ring->wptr = 0;
+
+	/* before programing wptr to a less value, need set minor_ptr_update first */
+	WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 1);
+
+	doorbell = RREG32_SDMA(i, regSDMA_GFX_DOORBELL);
+	doorbell_offset = RREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET);
+
+	doorbell = REG_SET_FIELD(doorbell, SDMA_GFX_DOORBELL, ENABLE,
+				 ring->use_doorbell);
+	doorbell_offset = REG_SET_FIELD(doorbell_offset,
+					SDMA_GFX_DOORBELL_OFFSET,
+					OFFSET, ring->doorbell_index);
+	WREG32_SDMA(i, regSDMA_GFX_DOORBELL, doorbell);
+	WREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET, doorbell_offset);
+
+	sdma_v4_4_2_ring_set_wptr(ring);
+
+	/* set minor_ptr_update to 0 after wptr programed */
+	WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 0);
+
+	/* setup the wptr shadow polling */
+	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_LO,
+		    lower_32_bits(wptr_gpu_addr));
+	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_HI,
+		    upper_32_bits(wptr_gpu_addr));
+	wptr_poll_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL);
+	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
+				       SDMA_GFX_RB_WPTR_POLL_CNTL,
+				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
+	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
+
+	/* enable DMA RB */
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 1);
+	WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
+
+	ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
+	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 1);
+#ifdef __BIG_ENDIAN
+	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
+#endif
+	/* enable DMA IBs */
+	WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
+
+	ring->sched.ready = true;
+}
+
+/**
+ * sdma_v4_4_2_page_resume - setup and start the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ * @i: instance to resume
+ *
+ * Set up the page DMA ring buffers and enable them.
+ * Returns 0 for success, error for failure.
+ */
+static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i)
+{
+	struct amdgpu_ring *ring = &adev->sdma.instance[i].page;
+	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
+	u32 wb_offset;
+	u32 doorbell;
+	u32 doorbell_offset;
+	u64 wptr_gpu_addr;
+
+	wb_offset = (ring->rptr_offs * 4);
+
+	rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
+	rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
+
+	/* Initialize the ring buffer's read and write pointers */
+	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR, 0);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_HI, 0);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR, 0);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_HI, 0);
+
+	/* set the wb address whether it's enabled or not */
+	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_HI,
+	       upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_LO,
+	       lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
+
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
+				RPTR_WRITEBACK_ENABLE, 1);
+
+	WREG32_SDMA(i, regSDMA_PAGE_RB_BASE, ring->gpu_addr >> 8);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_BASE_HI, ring->gpu_addr >> 40);
+
+	ring->wptr = 0;
+
+	/* before programing wptr to a less value, need set minor_ptr_update first */
+	WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 1);
+
+	doorbell = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL);
+	doorbell_offset = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET);
+
+	doorbell = REG_SET_FIELD(doorbell, SDMA_PAGE_DOORBELL, ENABLE,
+				 ring->use_doorbell);
+	doorbell_offset = REG_SET_FIELD(doorbell_offset,
+					SDMA_PAGE_DOORBELL_OFFSET,
+					OFFSET, ring->doorbell_index);
+	WREG32_SDMA(i, regSDMA_PAGE_DOORBELL, doorbell);
+	WREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET, doorbell_offset);
+
+	/* paging queue doorbell range is setup at sdma_v4_4_2_gfx_resume */
+	sdma_v4_4_2_page_ring_set_wptr(ring);
+
+	/* set minor_ptr_update to 0 after wptr programed */
+	WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 0);
+
+	/* setup the wptr shadow polling */
+	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_LO,
+		    lower_32_bits(wptr_gpu_addr));
+	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_HI,
+		    upper_32_bits(wptr_gpu_addr));
+	wptr_poll_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL);
+	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
+				       SDMA_PAGE_RB_WPTR_POLL_CNTL,
+				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
+
+	/* enable DMA RB */
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL, RB_ENABLE, 1);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
+
+	ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
+	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_ENABLE, 1);
+#ifdef __BIG_ENDIAN
+	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_SWAP_ENABLE, 1);
+#endif
+	/* enable DMA IBs */
+	WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
+
+	ring->sched.ready = true;
+}
+
+static void sdma_v4_4_2_init_pg(struct amdgpu_device *adev)
+{
+
+}
+
+/**
+ * sdma_v4_4_2_rlc_resume - setup and start the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Set up the compute DMA queues and enable them.
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v4_4_2_rlc_resume(struct amdgpu_device *adev)
+{
+	sdma_v4_4_2_init_pg(adev);
+
+	return 0;
+}
+
+/**
+ * sdma_v4_4_2_load_microcode - load the sDMA ME ucode
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Loads the sDMA0/1 ucode.
+ * Returns 0 for success, -EINVAL if the ucode is not available.
+ */
+static int sdma_v4_4_2_load_microcode(struct amdgpu_device *adev)
+{
+	const struct sdma_firmware_header_v1_0 *hdr;
+	const __le32 *fw_data;
+	u32 fw_size;
+	int i, j;
+
+	/* halt the MEs */
+	sdma_v4_4_2_enable(adev, false);
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		if (!adev->sdma.instance[i].fw)
+			return -EINVAL;
+
+		hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
+		amdgpu_ucode_print_sdma_hdr(&hdr->header);
+		fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
+
+		fw_data = (const __le32 *)
+			(adev->sdma.instance[i].fw->data +
+				le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+
+		WREG32_SDMA(i, regSDMA_UCODE_ADDR, 0);
+
+		for (j = 0; j < fw_size; j++)
+			WREG32_SDMA(i, regSDMA_UCODE_DATA,
+				    le32_to_cpup(fw_data++));
+
+		WREG32_SDMA(i, regSDMA_UCODE_ADDR,
+			    adev->sdma.instance[i].fw_version);
+	}
+
+	return 0;
+}
+
+/**
+ * sdma_v4_4_2_start - setup and start the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Set up the DMA engines and enable them.
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v4_4_2_start(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	int i, r = 0;
+
+	if (amdgpu_sriov_vf(adev)) {
+		sdma_v4_4_2_ctx_switch_enable(adev, false);
+		sdma_v4_4_2_enable(adev, false);
+	} else {
+		/* bypass sdma microcode loading on Gopher */
+		if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP &&
+		    !(adev->pdev->device == 0x49) && !(adev->pdev->device == 0x50)) {
+			r = sdma_v4_4_2_load_microcode(adev);
+			if (r)
+				return r;
+		}
+
+		/* unhalt the MEs */
+		sdma_v4_4_2_enable(adev, true);
+		/* enable sdma ring preemption */
+		sdma_v4_4_2_ctx_switch_enable(adev, true);
+	}
+
+	/* start the gfx rings and rlc compute queues */
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		uint32_t temp;
+
+		WREG32_SDMA(i, regSDMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
+		sdma_v4_4_2_gfx_resume(adev, i);
+		if (adev->sdma.has_page_queue)
+			sdma_v4_4_2_page_resume(adev, i);
+
+		/* set utc l1 enable flag always to 1 */
+		temp = RREG32_SDMA(i, regSDMA_CNTL);
+		temp = REG_SET_FIELD(temp, SDMA_CNTL, UTC_L1_ENABLE, 1);
+		WREG32_SDMA(i, regSDMA_CNTL, temp);
+
+		if (!amdgpu_sriov_vf(adev)) {
+			ring = &adev->sdma.instance[i].ring;
+			adev->nbio.funcs->sdma_doorbell_range(adev, i,
+				ring->use_doorbell, ring->doorbell_index,
+				adev->doorbell_index.sdma_doorbell_range);
+
+			/* unhalt engine */
+			temp = RREG32_SDMA(i, regSDMA_F32_CNTL);
+			temp = REG_SET_FIELD(temp, SDMA_F32_CNTL, HALT, 0);
+			WREG32_SDMA(i, regSDMA_F32_CNTL, temp);
+		}
+	}
+
+	if (amdgpu_sriov_vf(adev)) {
+		sdma_v4_4_2_ctx_switch_enable(adev, true);
+		sdma_v4_4_2_enable(adev, true);
+	} else {
+		r = sdma_v4_4_2_rlc_resume(adev);
+		if (r)
+			return r;
+	}
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		ring = &adev->sdma.instance[i].ring;
+
+		r = amdgpu_ring_test_helper(ring);
+		if (r)
+			return r;
+
+		if (adev->sdma.has_page_queue) {
+			struct amdgpu_ring *page = &adev->sdma.instance[i].page;
+
+			r = amdgpu_ring_test_helper(page);
+			if (r)
+				return r;
+
+			if (adev->mman.buffer_funcs_ring == page)
+				amdgpu_ttm_set_buffer_funcs_status(adev, true);
+		}
+
+		if (adev->mman.buffer_funcs_ring == ring)
+			amdgpu_ttm_set_buffer_funcs_status(adev, true);
+	}
+
+	return r;
+}
+
+/**
+ * sdma_v4_4_2_ring_test_ring - simple async dma engine test
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ *
+ * Test the DMA engine by writing using it to write an
+ * value to memory.
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v4_4_2_ring_test_ring(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	unsigned i;
+	unsigned index;
+	int r;
+	u32 tmp;
+	u64 gpu_addr;
+
+	r = amdgpu_device_wb_get(adev, &index);
+	if (r)
+		return r;
+
+	gpu_addr = adev->wb.gpu_addr + (index * 4);
+	tmp = 0xCAFEDEAD;
+	adev->wb.wb[index] = cpu_to_le32(tmp);
+
+	r = amdgpu_ring_alloc(ring, 5);
+	if (r)
+		goto error_free_wb;
+
+	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+			  SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
+	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
+	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
+	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
+	amdgpu_ring_write(ring, 0xDEADBEEF);
+	amdgpu_ring_commit(ring);
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		tmp = le32_to_cpu(adev->wb.wb[index]);
+		if (tmp == 0xDEADBEEF)
+			break;
+		udelay(1);
+	}
+
+	if (i >= adev->usec_timeout)
+		r = -ETIMEDOUT;
+
+error_free_wb:
+	amdgpu_device_wb_free(adev, index);
+	return r;
+}
+
+/**
+ * sdma_v4_4_2_ring_test_ib - test an IB on the DMA engine
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
+ *
+ * Test a simple IB in the DMA ring.
+ * Returns 0 on success, error on failure.
+ */
+static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
+{
+	struct amdgpu_device *adev = ring->adev;
+	struct amdgpu_ib ib;
+	struct dma_fence *f = NULL;
+	unsigned index;
+	long r;
+	u32 tmp = 0;
+	u64 gpu_addr;
+
+	r = amdgpu_device_wb_get(adev, &index);
+	if (r)
+		return r;
+
+	gpu_addr = adev->wb.gpu_addr + (index * 4);
+	tmp = 0xCAFEDEAD;
+	adev->wb.wb[index] = cpu_to_le32(tmp);
+	memset(&ib, 0, sizeof(ib));
+	r = amdgpu_ib_get(adev, NULL, 256,
+					AMDGPU_IB_POOL_DIRECT, &ib);
+	if (r)
+		goto err0;
+
+	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
+	ib.ptr[1] = lower_32_bits(gpu_addr);
+	ib.ptr[2] = upper_32_bits(gpu_addr);
+	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+	ib.ptr[4] = 0xDEADBEEF;
+	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib.length_dw = 8;
+
+	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
+	if (r)
+		goto err1;
+
+	r = dma_fence_wait_timeout(f, false, timeout);
+	if (r == 0) {
+		r = -ETIMEDOUT;
+		goto err1;
+	} else if (r < 0) {
+		goto err1;
+	}
+	tmp = le32_to_cpu(adev->wb.wb[index]);
+	if (tmp == 0xDEADBEEF)
+		r = 0;
+	else
+		r = -EINVAL;
+
+err1:
+	amdgpu_ib_free(adev, &ib, NULL);
+	dma_fence_put(f);
+err0:
+	amdgpu_device_wb_free(adev, index);
+	return r;
+}
+
+
+/**
+ * sdma_v4_4_2_vm_copy_pte - update PTEs by copying them from the GART
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @src: src addr to copy from
+ * @count: number of page entries to update
+ *
+ * Update PTEs by copying them from the GART using sDMA.
+ */
+static void sdma_v4_4_2_vm_copy_pte(struct amdgpu_ib *ib,
+				  uint64_t pe, uint64_t src,
+				  unsigned count)
+{
+	unsigned bytes = count * 8;
+
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
+		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+	ib->ptr[ib->length_dw++] = bytes - 1;
+	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+	ib->ptr[ib->length_dw++] = lower_32_bits(src);
+	ib->ptr[ib->length_dw++] = upper_32_bits(src);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+
+}
+
+/**
+ * sdma_v4_4_2_vm_write_pte - update PTEs by writing them manually
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @value: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ *
+ * Update PTEs by writing them manually using sDMA.
+ */
+static void sdma_v4_4_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
+				   uint64_t value, unsigned count,
+				   uint32_t incr)
+{
+	unsigned ndw = count * 2;
+
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	ib->ptr[ib->length_dw++] = ndw - 1;
+	for (; ndw > 0; ndw -= 2) {
+		ib->ptr[ib->length_dw++] = lower_32_bits(value);
+		ib->ptr[ib->length_dw++] = upper_32_bits(value);
+		value += incr;
+	}
+}
+
+/**
+ * sdma_v4_4_2_vm_set_pte_pde - update the page tables using sDMA
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: access flags
+ *
+ * Update the page tables using sDMA.
+ */
+static void sdma_v4_4_2_vm_set_pte_pde(struct amdgpu_ib *ib,
+				     uint64_t pe,
+				     uint64_t addr, unsigned count,
+				     uint32_t incr, uint64_t flags)
+{
+	/* for physically contiguous pages (vram) */
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
+	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
+	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
+	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
+	ib->ptr[ib->length_dw++] = incr; /* increment size */
+	ib->ptr[ib->length_dw++] = 0;
+	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
+}
+
+/**
+ * sdma_v4_4_2_ring_pad_ib - pad the IB to the required number of dw
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ * @ib: indirect buffer to fill with padding
+ */
+static void sdma_v4_4_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
+{
+	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	u32 pad_count;
+	int i;
+
+	pad_count = (-ib->length_dw) & 7;
+	for (i = 0; i < pad_count; i++)
+		if (sdma && sdma->burst_nop && (i == 0))
+			ib->ptr[ib->length_dw++] =
+				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
+				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
+		else
+			ib->ptr[ib->length_dw++] =
+				SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
+}
+
+
+/**
+ * sdma_v4_4_2_ring_emit_pipeline_sync - sync the pipeline
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Make sure all previous operations are completed (CIK).
+ */
+static void sdma_v4_4_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+	uint32_t seq = ring->fence_drv.sync_seq;
+	uint64_t addr = ring->fence_drv.gpu_addr;
+
+	/* wait for idle */
+	sdma_v4_4_2_wait_reg_mem(ring, 1, 0,
+			       addr & 0xfffffffc,
+			       upper_32_bits(addr) & 0xffffffff,
+			       seq, 0xffffffff, 4);
+}
+
+
+/**
+ * sdma_v4_4_2_ring_emit_vm_flush - vm flush using sDMA
+ *
+ * @ring: amdgpu_ring pointer
+ * @vmid: vmid number to use
+ * @pd_addr: address
+ *
+ * Update the page table base and flush the VM TLB
+ * using sDMA.
+ */
+static void sdma_v4_4_2_ring_emit_vm_flush(struct amdgpu_ring *ring,
+					 unsigned vmid, uint64_t pd_addr)
+{
+	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
+}
+
+static void sdma_v4_4_2_ring_emit_wreg(struct amdgpu_ring *ring,
+				     uint32_t reg, uint32_t val)
+{
+	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
+			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
+	amdgpu_ring_write(ring, reg);
+	amdgpu_ring_write(ring, val);
+}
+
+static void sdma_v4_4_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
+					 uint32_t val, uint32_t mask)
+{
+	sdma_v4_4_2_wait_reg_mem(ring, 0, 0, reg, 0, val, mask, 10);
+}
+
+static bool sdma_v4_4_2_fw_support_paging_queue(struct amdgpu_device *adev)
+{
+	switch (adev->ip_versions[SDMA0_HWIP][0]) {
+	case IP_VERSION(4, 4, 2):
+		return false;
+	default:
+		return false;
+	}
+}
+
+static int sdma_v4_4_2_early_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int r;
+
+	r = sdma_v4_4_2_init_microcode(adev);
+	if (r) {
+		DRM_ERROR("Failed to load sdma firmware!\n");
+		return r;
+	}
+
+	/* TODO: Page queue breaks driver reload under SRIOV */
+	if (sdma_v4_4_2_fw_support_paging_queue(adev))
+		adev->sdma.has_page_queue = true;
+
+	sdma_v4_4_2_set_ring_funcs(adev);
+	sdma_v4_4_2_set_buffer_funcs(adev);
+	sdma_v4_4_2_set_vm_pte_funcs(adev);
+	sdma_v4_4_2_set_irq_funcs(adev);
+
+	return 0;
+}
+
+#if 0
+static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
+		void *err_data,
+		struct amdgpu_iv_entry *entry);
+#endif
+
+static int sdma_v4_4_2_late_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+#if 0
+	struct ras_ih_if ih_info = {
+		.cb = sdma_v4_4_2_process_ras_data_cb,
+	};
+#endif
+	if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
+		if (adev->sdma.ras && adev->sdma.ras->ras_block.hw_ops &&
+		    adev->sdma.ras->ras_block.hw_ops->reset_ras_error_count)
+			adev->sdma.ras->ras_block.hw_ops->reset_ras_error_count(adev);
+	}
+
+	return 0;
+}
+
+static int sdma_v4_4_2_sw_init(void *handle)
+{
+	struct amdgpu_ring *ring;
+	int r, i;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	/* SDMA trap event */
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
+				      SDMA0_4_0__SRCID__SDMA_TRAP,
+				      &adev->sdma.trap_irq);
+		if (r)
+			return r;
+	}
+
+	/* SDMA SRAM ECC event */
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
+				      SDMA0_4_0__SRCID__SDMA_SRAM_ECC,
+				      &adev->sdma.ecc_irq);
+		if (r)
+			return r;
+	}
+
+	/* SDMA VM_HOLE/DOORBELL_INV/POLL_TIMEOUT/SRBM_WRITE_PROTECTION event*/
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
+				      SDMA0_4_0__SRCID__SDMA_VM_HOLE,
+				      &adev->sdma.vm_hole_irq);
+		if (r)
+			return r;
+
+		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
+				      SDMA0_4_0__SRCID__SDMA_DOORBELL_INVALID,
+				      &adev->sdma.doorbell_invalid_irq);
+		if (r)
+			return r;
+
+		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
+				      SDMA0_4_0__SRCID__SDMA_POLL_TIMEOUT,
+				      &adev->sdma.pool_timeout_irq);
+		if (r)
+			return r;
+
+		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
+				      SDMA0_4_0__SRCID__SDMA_SRBMWRITE,
+				      &adev->sdma.srbm_write_irq);
+		if (r)
+			return r;
+	}
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		ring = &adev->sdma.instance[i].ring;
+		ring->ring_obj = NULL;
+		ring->use_doorbell = true;
+
+		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
+				ring->use_doorbell?"true":"false");
+
+		/* doorbell size is 2 dwords, get DWORD offset */
+		ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
+		ring->vm_hub = AMDGPU_MMHUB_0;
+
+		sprintf(ring->name, "sdma%d", i);
+		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
+				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
+				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+		if (r)
+			return r;
+
+		if (adev->sdma.has_page_queue) {
+			ring = &adev->sdma.instance[i].page;
+			ring->ring_obj = NULL;
+			ring->use_doorbell = true;
+
+			/* paging queue use same doorbell index/routing as gfx queue
+			 * with 0x400 (4096 dwords) offset on second doorbell page
+			 */
+			ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
+			ring->doorbell_index += 0x400;
+			ring->vm_hub = AMDGPU_MMHUB_0;
+
+			sprintf(ring->name, "page%d", i);
+			r = amdgpu_ring_init(adev, ring, 1024,
+					     &adev->sdma.trap_irq,
+					     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
+					     AMDGPU_RING_PRIO_DEFAULT, NULL);
+			if (r)
+				return r;
+		}
+	}
+
+	return r;
+}
+
+static int sdma_v4_4_2_sw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
+		if (adev->sdma.has_page_queue)
+			amdgpu_ring_fini(&adev->sdma.instance[i].page);
+	}
+
+	if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 2))
+		amdgpu_sdma_destroy_inst_ctx(adev, true);
+	else
+		amdgpu_sdma_destroy_inst_ctx(adev, false);
+
+	return 0;
+}
+
+static int sdma_v4_4_2_hw_init(void *handle)
+{
+	int r;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->flags & AMD_IS_APU)
+		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_SDMA, false);
+
+	if (!amdgpu_sriov_vf(adev))
+		sdma_v4_4_2_init_golden_registers(adev);
+
+	r = sdma_v4_4_2_start(adev);
+
+	return r;
+}
+
+static int sdma_v4_4_2_hw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int i;
+
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
+			       AMDGPU_SDMA_IRQ_INSTANCE0 + i);
+	}
+
+	sdma_v4_4_2_ctx_switch_enable(adev, false);
+	sdma_v4_4_2_enable(adev, false);
+
+	return 0;
+}
+
+static int sdma_v4_4_2_suspend(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return sdma_v4_4_2_hw_fini(adev);
+}
+
+static int sdma_v4_4_2_resume(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return sdma_v4_4_2_hw_init(adev);
+}
+
+static bool sdma_v4_4_2_is_idle(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		u32 tmp = RREG32_SDMA(i, regSDMA_STATUS_REG);
+
+		if (!(tmp & SDMA_STATUS_REG__IDLE_MASK))
+			return false;
+	}
+
+	return true;
+}
+
+static int sdma_v4_4_2_wait_for_idle(void *handle)
+{
+	unsigned i, j;
+	u32 sdma[AMDGPU_MAX_SDMA_INSTANCES];
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		for (j = 0; j < adev->sdma.num_instances; j++) {
+			sdma[j] = RREG32_SDMA(j, regSDMA_STATUS_REG);
+			if (!(sdma[j] & SDMA_STATUS_REG__IDLE_MASK))
+				break;
+		}
+		if (j == adev->sdma.num_instances)
+			return 0;
+		udelay(1);
+	}
+	return -ETIMEDOUT;
+}
+
+static int sdma_v4_4_2_soft_reset(void *handle)
+{
+	/* todo */
+
+	return 0;
+}
+
+static int sdma_v4_4_2_set_trap_irq_state(struct amdgpu_device *adev,
+					struct amdgpu_irq_src *source,
+					unsigned type,
+					enum amdgpu_interrupt_state state)
+{
+	u32 sdma_cntl;
+
+	sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
+	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL, TRAP_ENABLE,
+		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
+	WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
+
+	return 0;
+}
+
+static int sdma_v4_4_2_process_trap_irq(struct amdgpu_device *adev,
+				      struct amdgpu_irq_src *source,
+				      struct amdgpu_iv_entry *entry)
+{
+	uint32_t instance;
+
+	DRM_DEBUG("IH: SDMA trap\n");
+	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
+	switch (entry->ring_id) {
+	case 0:
+		amdgpu_fence_process(&adev->sdma.instance[instance].ring);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+#if 0
+static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
+		void *err_data,
+		struct amdgpu_iv_entry *entry)
+{
+	int instance;
+
+	/* When “Full RAS” is enabled, the per-IP interrupt sources should
+	 * be disabled and the driver should only look for the aggregated
+	 * interrupt via sync flood
+	 */
+	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
+		goto out;
+
+	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
+	if (instance < 0)
+		goto out;
+
+	amdgpu_sdma_process_ras_data_cb(adev, err_data, entry);
+
+out:
+	return AMDGPU_RAS_SUCCESS;
+}
+#endif
+
+static int sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *source,
+					      struct amdgpu_iv_entry *entry)
+{
+	int instance;
+
+	DRM_ERROR("Illegal instruction in SDMA command stream\n");
+
+	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
+	if (instance < 0)
+		return 0;
+
+	switch (entry->ring_id) {
+	case 0:
+		drm_sched_fault(&adev->sdma.instance[instance].ring.sched);
+		break;
+	}
+	return 0;
+}
+
+static int sdma_v4_4_2_set_ecc_irq_state(struct amdgpu_device *adev,
+					struct amdgpu_irq_src *source,
+					unsigned type,
+					enum amdgpu_interrupt_state state)
+{
+	u32 sdma_edc_config;
+
+	sdma_edc_config = RREG32_SDMA(type, regCC_SDMA_EDC_CONFIG);
+	/*
+	 * FIXME: This was inherited from Aldebaran, but no this field
+	 * definition in the regspec of both Aldebaran and SDMA 4.4.2
+	 */
+	sdma_edc_config |= (state == AMDGPU_IRQ_STATE_ENABLE) ? (1 << 2) : 0;
+	WREG32_SDMA(type, regCC_SDMA_EDC_CONFIG, sdma_edc_config);
+
+	return 0;
+}
+
+static int sdma_v4_4_2_print_iv_entry(struct amdgpu_device *adev,
+					      struct amdgpu_iv_entry *entry)
+{
+	int instance;
+	struct amdgpu_task_info task_info;
+	u64 addr;
+
+	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
+	if (instance < 0 || instance >= adev->sdma.num_instances) {
+		dev_err(adev->dev, "sdma instance invalid %d\n", instance);
+		return -EINVAL;
+	}
+
+	addr = (u64)entry->src_data[0] << 12;
+	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
+
+	memset(&task_info, 0, sizeof(struct amdgpu_task_info));
+	amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
+
+	dev_dbg_ratelimited(adev->dev,
+		   "[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u "
+		   "pasid:%u, for process %s pid %d thread %s pid %d\n",
+		   instance, addr, entry->src_id, entry->ring_id, entry->vmid,
+		   entry->pasid, task_info.process_name, task_info.tgid,
+		   task_info.task_name, task_info.pid);
+	return 0;
+}
+
+static int sdma_v4_4_2_process_vm_hole_irq(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *source,
+					      struct amdgpu_iv_entry *entry)
+{
+	dev_dbg_ratelimited(adev->dev, "MC or SEM address in VM hole\n");
+	sdma_v4_4_2_print_iv_entry(adev, entry);
+	return 0;
+}
+
+static int sdma_v4_4_2_process_doorbell_invalid_irq(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *source,
+					      struct amdgpu_iv_entry *entry)
+{
+
+	dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
+	sdma_v4_4_2_print_iv_entry(adev, entry);
+	return 0;
+}
+
+static int sdma_v4_4_2_process_pool_timeout_irq(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *source,
+					      struct amdgpu_iv_entry *entry)
+{
+	dev_dbg_ratelimited(adev->dev,
+		"Polling register/memory timeout executing POLL_REG/MEM with finite timer\n");
+	sdma_v4_4_2_print_iv_entry(adev, entry);
+	return 0;
+}
+
+static int sdma_v4_4_2_process_srbm_write_irq(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *source,
+					      struct amdgpu_iv_entry *entry)
+{
+	dev_dbg_ratelimited(adev->dev,
+		"SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n");
+	sdma_v4_4_2_print_iv_entry(adev, entry);
+	return 0;
+}
+
+static void sdma_v4_4_2_update_medium_grain_clock_gating(
+		struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t data, def;
+	int i;
+
+	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
+		for (i = 0; i < adev->sdma.num_instances; i++) {
+			def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
+			data &= ~(SDMA_CLK_CTRL__SOFT_OVERRIDE7_MASK |
+				  SDMA_CLK_CTRL__SOFT_OVERRIDE6_MASK |
+				  SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
+				  SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
+				  SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
+				  SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
+				  SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
+				  SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
+			if (def != data)
+				WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
+		}
+	} else {
+		for (i = 0; i < adev->sdma.num_instances; i++) {
+			def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
+			data |= (SDMA_CLK_CTRL__SOFT_OVERRIDE7_MASK |
+				 SDMA_CLK_CTRL__SOFT_OVERRIDE6_MASK |
+				 SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
+				 SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
+				 SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
+				 SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
+				 SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
+				 SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
+			if (def != data)
+				WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
+		}
+	}
+}
+
+
+static void sdma_v4_4_2_update_medium_grain_light_sleep(
+		struct amdgpu_device *adev,
+		bool enable)
+{
+	uint32_t data, def;
+	int i;
+
+	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {
+		for (i = 0; i < adev->sdma.num_instances; i++) {
+			/* 1-not override: enable sdma mem light sleep */
+			def = data = RREG32_SDMA(0, regSDMA_POWER_CNTL);
+			data |= SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
+			if (def != data)
+				WREG32_SDMA(0, regSDMA_POWER_CNTL, data);
+		}
+	} else {
+		for (i = 0; i < adev->sdma.num_instances; i++) {
+		/* 0-override:disable sdma mem light sleep */
+			def = data = RREG32_SDMA(0, regSDMA_POWER_CNTL);
+			data &= ~SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
+			if (def != data)
+				WREG32_SDMA(0, regSDMA_POWER_CNTL, data);
+		}
+	}
+}
+
+static int sdma_v4_4_2_set_clockgating_state(void *handle,
+					  enum amd_clockgating_state state)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	sdma_v4_4_2_update_medium_grain_clock_gating(adev,
+			state == AMD_CG_STATE_GATE);
+	sdma_v4_4_2_update_medium_grain_light_sleep(adev,
+			state == AMD_CG_STATE_GATE);
+	return 0;
+}
+
+static int sdma_v4_4_2_set_powergating_state(void *handle,
+					  enum amd_powergating_state state)
+{
+	return 0;
+}
+
+static void sdma_v4_4_2_get_clockgating_state(void *handle, u64 *flags)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int data;
+
+	if (amdgpu_sriov_vf(adev))
+		*flags = 0;
+
+	/* AMD_CG_SUPPORT_SDMA_MGCG */
+	data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, regSDMA_CLK_CTRL));
+	if (!(data & SDMA_CLK_CTRL__SOFT_OVERRIDE7_MASK))
+		*flags |= AMD_CG_SUPPORT_SDMA_MGCG;
+
+	/* AMD_CG_SUPPORT_SDMA_LS */
+	data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, regSDMA_POWER_CNTL));
+	if (data & SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
+		*flags |= AMD_CG_SUPPORT_SDMA_LS;
+}
+
+const struct amd_ip_funcs sdma_v4_4_2_ip_funcs = {
+	.name = "sdma_v4_4_2",
+	.early_init = sdma_v4_4_2_early_init,
+	.late_init = sdma_v4_4_2_late_init,
+	.sw_init = sdma_v4_4_2_sw_init,
+	.sw_fini = sdma_v4_4_2_sw_fini,
+	.hw_init = sdma_v4_4_2_hw_init,
+	.hw_fini = sdma_v4_4_2_hw_fini,
+	.suspend = sdma_v4_4_2_suspend,
+	.resume = sdma_v4_4_2_resume,
+	.is_idle = sdma_v4_4_2_is_idle,
+	.wait_for_idle = sdma_v4_4_2_wait_for_idle,
+	.soft_reset = sdma_v4_4_2_soft_reset,
+	.set_clockgating_state = sdma_v4_4_2_set_clockgating_state,
+	.set_powergating_state = sdma_v4_4_2_set_powergating_state,
+	.get_clockgating_state = sdma_v4_4_2_get_clockgating_state,
+};
+
+static const struct amdgpu_ring_funcs sdma_v4_4_2_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_SDMA,
+	.align_mask = 0xf,
+	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
+	.support_64bit_ptrs = true,
+	.get_rptr = sdma_v4_4_2_ring_get_rptr,
+	.get_wptr = sdma_v4_4_2_ring_get_wptr,
+	.set_wptr = sdma_v4_4_2_ring_set_wptr,
+	.emit_frame_size =
+		6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
+		3 + /* hdp invalidate */
+		6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
+		/* sdma_v4_4_2_ring_emit_vm_flush */
+		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
+		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
+		10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
+	.emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
+	.emit_ib = sdma_v4_4_2_ring_emit_ib,
+	.emit_fence = sdma_v4_4_2_ring_emit_fence,
+	.emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
+	.emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
+	.emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
+	.test_ring = sdma_v4_4_2_ring_test_ring,
+	.test_ib = sdma_v4_4_2_ring_test_ib,
+	.insert_nop = sdma_v4_4_2_ring_insert_nop,
+	.pad_ib = sdma_v4_4_2_ring_pad_ib,
+	.emit_wreg = sdma_v4_4_2_ring_emit_wreg,
+	.emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
+};
+
+static const struct amdgpu_ring_funcs sdma_v4_4_2_page_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_SDMA,
+	.align_mask = 0xf,
+	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
+	.support_64bit_ptrs = true,
+	.get_rptr = sdma_v4_4_2_ring_get_rptr,
+	.get_wptr = sdma_v4_4_2_page_ring_get_wptr,
+	.set_wptr = sdma_v4_4_2_page_ring_set_wptr,
+	.emit_frame_size =
+		6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
+		3 + /* hdp invalidate */
+		6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
+		/* sdma_v4_4_2_ring_emit_vm_flush */
+		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
+		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
+		10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
+	.emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
+	.emit_ib = sdma_v4_4_2_ring_emit_ib,
+	.emit_fence = sdma_v4_4_2_ring_emit_fence,
+	.emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
+	.emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
+	.emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
+	.test_ring = sdma_v4_4_2_ring_test_ring,
+	.test_ib = sdma_v4_4_2_ring_test_ib,
+	.insert_nop = sdma_v4_4_2_ring_insert_nop,
+	.pad_ib = sdma_v4_4_2_ring_pad_ib,
+	.emit_wreg = sdma_v4_4_2_ring_emit_wreg,
+	.emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
+};
+
+static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev)
+{
+	int i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		adev->sdma.instance[i].ring.funcs = &sdma_v4_4_2_ring_funcs;
+		adev->sdma.instance[i].ring.me = i;
+		if (adev->sdma.has_page_queue) {
+			adev->sdma.instance[i].page.funcs =
+				&sdma_v4_4_2_page_ring_funcs;
+			adev->sdma.instance[i].page.me = i;
+		}
+	}
+}
+
+static const struct amdgpu_irq_src_funcs sdma_v4_4_2_trap_irq_funcs = {
+	.set = sdma_v4_4_2_set_trap_irq_state,
+	.process = sdma_v4_4_2_process_trap_irq,
+};
+
+static const struct amdgpu_irq_src_funcs sdma_v4_4_2_illegal_inst_irq_funcs = {
+	.process = sdma_v4_4_2_process_illegal_inst_irq,
+};
+
+static const struct amdgpu_irq_src_funcs sdma_v4_4_2_ecc_irq_funcs = {
+	.set = sdma_v4_4_2_set_ecc_irq_state,
+	.process = amdgpu_sdma_process_ecc_irq,
+};
+
+static const struct amdgpu_irq_src_funcs sdma_v4_4_2_vm_hole_irq_funcs = {
+	.process = sdma_v4_4_2_process_vm_hole_irq,
+};
+
+static const struct amdgpu_irq_src_funcs sdma_v4_4_2_doorbell_invalid_irq_funcs = {
+	.process = sdma_v4_4_2_process_doorbell_invalid_irq,
+};
+
+static const struct amdgpu_irq_src_funcs sdma_v4_4_2_pool_timeout_irq_funcs = {
+	.process = sdma_v4_4_2_process_pool_timeout_irq,
+};
+
+static const struct amdgpu_irq_src_funcs sdma_v4_4_2_srbm_write_irq_funcs = {
+	.process = sdma_v4_4_2_process_srbm_write_irq,
+};
+
+static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->sdma.trap_irq.num_types = adev->sdma.num_instances;
+	adev->sdma.ecc_irq.num_types = adev->sdma.num_instances;
+	adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances;
+	adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances;
+	adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances;
+	adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances;
+
+	adev->sdma.trap_irq.funcs = &sdma_v4_4_2_trap_irq_funcs;
+	adev->sdma.illegal_inst_irq.funcs = &sdma_v4_4_2_illegal_inst_irq_funcs;
+	adev->sdma.ecc_irq.funcs = &sdma_v4_4_2_ecc_irq_funcs;
+	adev->sdma.vm_hole_irq.funcs = &sdma_v4_4_2_vm_hole_irq_funcs;
+	adev->sdma.doorbell_invalid_irq.funcs = &sdma_v4_4_2_doorbell_invalid_irq_funcs;
+	adev->sdma.pool_timeout_irq.funcs = &sdma_v4_4_2_pool_timeout_irq_funcs;
+	adev->sdma.srbm_write_irq.funcs = &sdma_v4_4_2_srbm_write_irq_funcs;
+}
+
+/**
+ * sdma_v4_4_2_emit_copy_buffer - copy buffer using the sDMA engine
+ *
+ * @ib: indirect buffer to copy to
+ * @src_offset: src GPU address
+ * @dst_offset: dst GPU address
+ * @byte_count: number of bytes to xfer
+ * @tmz: if a secure copy should be used
+ *
+ * Copy GPU buffers using the DMA engine.
+ * Used by the amdgpu ttm implementation to move pages if
+ * registered as the asic copy callback.
+ */
+static void sdma_v4_4_2_emit_copy_buffer(struct amdgpu_ib *ib,
+				       uint64_t src_offset,
+				       uint64_t dst_offset,
+				       uint32_t byte_count,
+				       bool tmz)
+{
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
+		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
+		SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
+	ib->ptr[ib->length_dw++] = byte_count - 1;
+	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
+	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
+	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
+}
+
+/**
+ * sdma_v4_4_2_emit_fill_buffer - fill buffer using the sDMA engine
+ *
+ * @ib: indirect buffer to copy to
+ * @src_data: value to write to buffer
+ * @dst_offset: dst GPU address
+ * @byte_count: number of bytes to xfer
+ *
+ * Fill GPU buffers using the DMA engine.
+ */
+static void sdma_v4_4_2_emit_fill_buffer(struct amdgpu_ib *ib,
+				       uint32_t src_data,
+				       uint64_t dst_offset,
+				       uint32_t byte_count)
+{
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
+	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
+	ib->ptr[ib->length_dw++] = src_data;
+	ib->ptr[ib->length_dw++] = byte_count - 1;
+}
+
+static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = {
+	.copy_max_bytes = 0x400000,
+	.copy_num_dw = 7,
+	.emit_copy_buffer = sdma_v4_4_2_emit_copy_buffer,
+
+	.fill_max_bytes = 0x400000,
+	.fill_num_dw = 5,
+	.emit_fill_buffer = sdma_v4_4_2_emit_fill_buffer,
+};
+
+static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev)
+{
+	adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs;
+	if (adev->sdma.has_page_queue)
+		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
+	else
+		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+}
+
+static const struct amdgpu_vm_pte_funcs sdma_v4_4_2_vm_pte_funcs = {
+	.copy_pte_num_dw = 7,
+	.copy_pte = sdma_v4_4_2_vm_copy_pte,
+
+	.write_pte = sdma_v4_4_2_vm_write_pte,
+	.set_pte_pde = sdma_v4_4_2_vm_set_pte_pde,
+};
+
+static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev)
+{
+	struct drm_gpu_scheduler *sched;
+	unsigned i;
+
+	adev->vm_manager.vm_pte_funcs = &sdma_v4_4_2_vm_pte_funcs;
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		if (adev->sdma.has_page_queue)
+			sched = &adev->sdma.instance[i].page.sched;
+		else
+			sched = &adev->sdma.instance[i].ring.sched;
+		adev->vm_manager.vm_pte_scheds[i] = sched;
+	}
+	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
+}
+
+const struct amdgpu_ip_block_version sdma_v4_4_2_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_SDMA,
+	.major = 4,
+	.minor = 4,
+	.rev = 0,
+	.funcs = &sdma_v4_4_2_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.h b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.h
new file mode 100644
index 000000000000..4814e8a074d6
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __SDMA_V4_4_2_H__
+#define __SDMA_V4_4_2_H__
+
+extern const struct amd_ip_funcs sdma_v4_4_2_ip_funcs;
+extern const struct amdgpu_ip_block_version sdma_v4_4_2_ip_block;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
index 1941b3b7c5d9..92e1299be021 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
@@ -1389,6 +1389,7 @@ static int sdma_v5_0_sw_init(void *handle)
 			(adev->doorbell_index.sdma_engine[0] << 1) //get DWORD offset
 			: (adev->doorbell_index.sdma_engine[1] << 1); // get DWORD offset
 
+		ring->vm_hub = AMDGPU_GFXHUB_0;
 		sprintf(ring->name, "sdma%d", i);
 		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
 				     (i == 0) ? AMDGPU_SDMA_IRQ_INSTANCE0 :
@@ -1765,7 +1766,6 @@ static const struct amdgpu_ring_funcs sdma_v5_0_ring_funcs = {
 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
 	.support_64bit_ptrs = true,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = sdma_v5_0_ring_get_rptr,
 	.get_wptr = sdma_v5_0_ring_get_wptr,
 	.set_wptr = sdma_v5_0_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
index 8e445eb9dd49..ca7e8757d78e 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
@@ -1253,6 +1253,7 @@ static int sdma_v5_2_sw_init(void *handle)
 		ring->doorbell_index =
 			(adev->doorbell_index.sdma_engine[i] << 1); //get DWORD offset
 
+		ring->vm_hub = AMDGPU_GFXHUB_0;
 		sprintf(ring->name, "sdma%d", i);
 		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
 				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
@@ -1653,7 +1654,6 @@ static const struct amdgpu_ring_funcs sdma_v5_2_ring_funcs = {
 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
 	.support_64bit_ptrs = true,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = sdma_v5_2_ring_get_rptr,
 	.get_wptr = sdma_v5_2_ring_get_wptr,
 	.set_wptr = sdma_v5_2_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index 40e6b22daa22..eb722830531f 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -403,15 +403,26 @@ static void sdma_v6_0_rlc_stop(struct amdgpu_device *adev)
 }
 
 /**
- * sdma_v6_0_ctx_switch_enable - stop the async dma engines context switch
+ * sdma_v6_0_ctxempty_int_enable - enable or disable context empty interrupts
  *
  * @adev: amdgpu_device pointer
- * @enable: enable/disable the DMA MEs context switch.
+ * @enable: enable/disable context switching due to queue empty conditions
  *
- * Halt or unhalt the async dma engines context switch.
+ * Enable or disable the async dma engines queue empty context switch.
  */
-static void sdma_v6_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
+static void sdma_v6_0_ctxempty_int_enable(struct amdgpu_device *adev, bool enable)
 {
+	u32 f32_cntl;
+	int i;
+
+	if (!amdgpu_sriov_vf(adev)) {
+		for (i = 0; i < adev->sdma.num_instances; i++) {
+			f32_cntl = RREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_CNTL));
+			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
+					CTXEMPTY_INT_ENABLE, enable ? 1 : 0);
+			WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_CNTL), f32_cntl);
+		}
+	}
 }
 
 /**
@@ -579,10 +590,8 @@ static int sdma_v6_0_gfx_resume(struct amdgpu_device *adev)
 
 		ring->sched.ready = true;
 
-		if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */
-			sdma_v6_0_ctx_switch_enable(adev, true);
+		if (amdgpu_sriov_vf(adev))
 			sdma_v6_0_enable(adev, true);
-		}
 
 		r = amdgpu_ring_test_helper(ring);
 		if (r) {
@@ -778,7 +787,6 @@ static int sdma_v6_0_start(struct amdgpu_device *adev)
 	int r = 0;
 
 	if (amdgpu_sriov_vf(adev)) {
-		sdma_v6_0_ctx_switch_enable(adev, false);
 		sdma_v6_0_enable(adev, false);
 
 		/* set RB registers */
@@ -799,7 +807,7 @@ static int sdma_v6_0_start(struct amdgpu_device *adev)
 	/* unhalt the MEs */
 	sdma_v6_0_enable(adev, true);
 	/* enable sdma ring preemption */
-	sdma_v6_0_ctx_switch_enable(adev, true);
+	sdma_v6_0_ctxempty_int_enable(adev, true);
 
 	/* start the gfx rings and rlc compute queues */
 	r = sdma_v6_0_gfx_resume(adev);
@@ -1173,7 +1181,28 @@ static void sdma_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 static void sdma_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 					 unsigned vmid, uint64_t pd_addr)
 {
-	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
+	uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
+
+	/* Update the PD address for this VMID. */
+	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
+			      (hub->ctx_addr_distance * vmid),
+			      lower_32_bits(pd_addr));
+	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
+			      (hub->ctx_addr_distance * vmid),
+			      upper_32_bits(pd_addr));
+
+	/* Trigger invalidation. */
+	amdgpu_ring_write(ring,
+			  SDMA_PKT_VM_INVALIDATION_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+			  SDMA_PKT_VM_INVALIDATION_HEADER_SUB_OP(SDMA_SUBOP_VM_INVALIDATION) |
+			  SDMA_PKT_VM_INVALIDATION_HEADER_GFX_ENG_ID(ring->vm_inv_eng) |
+			  SDMA_PKT_VM_INVALIDATION_HEADER_MM_ENG_ID(0x1f));
+	amdgpu_ring_write(ring, req);
+	amdgpu_ring_write(ring, 0xFFFFFFFF);
+	amdgpu_ring_write(ring,
+			  SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_INVALIDATEACK(1 << vmid) |
+			  SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_ADDRESSRANGEHI(0x1F));
 }
 
 static void sdma_v6_0_ring_emit_wreg(struct amdgpu_ring *ring,
@@ -1272,6 +1301,7 @@ static int sdma_v6_0_sw_init(void *handle)
 		ring->doorbell_index =
 			(adev->doorbell_index.sdma_engine[i] << 1); // get DWORD offset
 
+		ring->vm_hub = AMDGPU_GFXHUB_0;
 		sprintf(ring->name, "sdma%d", i);
 		r = amdgpu_ring_init(adev, ring, 1024,
 				     &adev->sdma.trap_irq,
@@ -1319,7 +1349,7 @@ static int sdma_v6_0_hw_fini(void *handle)
 		return 0;
 	}
 
-	sdma_v6_0_ctx_switch_enable(adev, false);
+	sdma_v6_0_ctxempty_int_enable(adev, false);
 	sdma_v6_0_enable(adev, false);
 
 	return 0;
@@ -1528,7 +1558,6 @@ static const struct amdgpu_ring_funcs sdma_v6_0_ring_funcs = {
 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
 	.support_64bit_ptrs = true,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_GFXHUB_0,
 	.get_rptr = sdma_v6_0_ring_get_rptr,
 	.get_wptr = sdma_v6_0_ring_get_wptr,
 	.set_wptr = sdma_v6_0_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 2eddd7f6cd41..bc5dd80f10c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -191,47 +191,6 @@ static int soc15_query_video_codecs(struct amdgpu_device *adev, bool encode,
 	}
 }
 
-/*
- * Indirect registers accessor
- */
-static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
-{
-	unsigned long address, data;
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	return amdgpu_device_indirect_rreg(adev, address, data, reg);
-}
-
-static void soc15_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
-{
-	unsigned long address, data;
-
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	amdgpu_device_indirect_wreg(adev, address, data, reg, v);
-}
-
-static u64 soc15_pcie_rreg64(struct amdgpu_device *adev, u32 reg)
-{
-	unsigned long address, data;
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	return amdgpu_device_indirect_rreg64(adev, address, data, reg);
-}
-
-static void soc15_pcie_wreg64(struct amdgpu_device *adev, u32 reg, u64 v)
-{
-	unsigned long address, data;
-
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	amdgpu_device_indirect_wreg64(adev, address, data, reg, v);
-}
-
 static u32 soc15_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 {
 	unsigned long flags, address, data;
@@ -342,11 +301,10 @@ static u32 soc15_get_xclk(struct amdgpu_device *adev)
 	u32 reference_clock = adev->clock.spll.reference_freq;
 
 	if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(12, 0, 0) ||
-	    adev->ip_versions[MP1_HWIP][0] == IP_VERSION(12, 0, 1))
-		return 10000;
-	if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(10, 0, 0) ||
+	    adev->ip_versions[MP1_HWIP][0] == IP_VERSION(12, 0, 1) ||
+	    adev->ip_versions[MP1_HWIP][0] == IP_VERSION(10, 0, 0) ||
 	    adev->ip_versions[MP1_HWIP][0] == IP_VERSION(10, 0, 1))
-		return reference_clock / 4;
+		return 10000;
 
 	return reference_clock;
 }
@@ -651,24 +609,6 @@ static int soc15_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk
 	return 0;
 }
 
-static void soc15_pcie_gen3_enable(struct amdgpu_device *adev)
-{
-	if (pci_is_root_bus(adev->pdev->bus))
-		return;
-
-	if (amdgpu_pcie_gen2 == 0)
-		return;
-
-	if (adev->flags & AMD_IS_APU)
-		return;
-
-	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
-					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
-		return;
-
-	/* todo */
-}
-
 static void soc15_program_aspm(struct amdgpu_device *adev)
 {
 	if (!amdgpu_device_should_use_aspm(adev))
@@ -695,11 +635,6 @@ const struct amdgpu_ip_block_version vega10_common_ip_block =
 	.funcs = &soc15_common_ip_funcs,
 };
 
-static uint32_t soc15_get_rev_id(struct amdgpu_device *adev)
-{
-	return adev->nbio.funcs->get_rev_id(adev);
-}
-
 static void soc15_reg_base_init(struct amdgpu_device *adev)
 {
 	/* Set IP register base before any HW register access */
@@ -936,10 +871,10 @@ static int soc15_common_early_init(void *handle)
 	}
 	adev->smc_rreg = NULL;
 	adev->smc_wreg = NULL;
-	adev->pcie_rreg = &soc15_pcie_rreg;
-	adev->pcie_wreg = &soc15_pcie_wreg;
-	adev->pcie_rreg64 = &soc15_pcie_rreg64;
-	adev->pcie_wreg64 = &soc15_pcie_wreg64;
+	adev->pcie_rreg = &amdgpu_device_indirect_rreg;
+	adev->pcie_wreg = &amdgpu_device_indirect_wreg;
+	adev->pcie_rreg64 = &amdgpu_device_indirect_rreg64;
+	adev->pcie_wreg64 = &amdgpu_device_indirect_wreg64;
 	adev->uvd_ctx_rreg = &soc15_uvd_ctx_rreg;
 	adev->uvd_ctx_wreg = &soc15_uvd_ctx_wreg;
 	adev->didt_rreg = &soc15_didt_rreg;
@@ -949,7 +884,7 @@ static int soc15_common_early_init(void *handle)
 	adev->se_cac_rreg = &soc15_se_cac_rreg;
 	adev->se_cac_wreg = &soc15_se_cac_wreg;
 
-	adev->rev_id = soc15_get_rev_id(adev);
+	adev->rev_id = amdgpu_device_get_rev_id(adev);
 	adev->external_rev_id = 0xFF;
 	/* TODO: split the GC and PG flags based on the relevant IP version for which
 	 * they are relevant.
@@ -1165,6 +1100,11 @@ static int soc15_common_early_init(void *handle)
 		adev->pg_flags = AMD_PG_SUPPORT_VCN_DPG;
 		adev->external_rev_id = adev->rev_id + 0x3c;
 		break;
+	case IP_VERSION(9, 4, 3):
+		adev->asic_funcs = &vega20_asic_funcs;
+		adev->cg_flags = 0;
+		adev->pg_flags = 0;
+		break;
 	default:
 		/* FIXME: not supported yet */
 		return -EINVAL;
@@ -1230,8 +1170,6 @@ static int soc15_common_hw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	/* enable pcie gen2/3 link */
-	soc15_pcie_gen3_enable(adev);
 	/* enable aspm */
 	soc15_program_aspm(adev);
 	/* setup nbio registers */
diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c
index c82b3a7ea5f0..514bfc705d5a 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc21.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc21.c
@@ -196,46 +196,6 @@ static int soc21_query_video_codecs(struct amdgpu_device *adev, bool encode,
 		return -EINVAL;
 	}
 }
-/*
- * Indirect registers accessor
- */
-static u32 soc21_pcie_rreg(struct amdgpu_device *adev, u32 reg)
-{
-	unsigned long address, data;
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	return amdgpu_device_indirect_rreg(adev, address, data, reg);
-}
-
-static void soc21_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
-{
-	unsigned long address, data;
-
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	amdgpu_device_indirect_wreg(adev, address, data, reg, v);
-}
-
-static u64 soc21_pcie_rreg64(struct amdgpu_device *adev, u32 reg)
-{
-	unsigned long address, data;
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	return amdgpu_device_indirect_rreg64(adev, address, data, reg);
-}
-
-static void soc21_pcie_wreg64(struct amdgpu_device *adev, u32 reg, u64 v)
-{
-	unsigned long address, data;
-
-	address = adev->nbio.funcs->get_pcie_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	amdgpu_device_indirect_wreg64(adev, address, data, reg, v);
-}
 
 static u32 soc21_didt_rreg(struct amdgpu_device *adev, u32 reg)
 {
@@ -480,21 +440,6 @@ static int soc21_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk
 	return 0;
 }
 
-static void soc21_pcie_gen3_enable(struct amdgpu_device *adev)
-{
-	if (pci_is_root_bus(adev->pdev->bus))
-		return;
-
-	if (amdgpu_pcie_gen2 == 0)
-		return;
-
-	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
-					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
-		return;
-
-	/* todo */
-}
-
 static void soc21_program_aspm(struct amdgpu_device *adev)
 {
 	if (!amdgpu_device_should_use_aspm(adev))
@@ -521,11 +466,6 @@ const struct amdgpu_ip_block_version soc21_common_ip_block =
 	.funcs = &soc21_common_ip_funcs,
 };
 
-static uint32_t soc21_get_rev_id(struct amdgpu_device *adev)
-{
-	return adev->nbio.funcs->get_rev_id(adev);
-}
-
 static bool soc21_need_full_reset(struct amdgpu_device *adev)
 {
 	switch (adev->ip_versions[GC_HWIP][0]) {
@@ -650,10 +590,10 @@ static int soc21_common_early_init(void *handle)
 	adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
 	adev->smc_rreg = NULL;
 	adev->smc_wreg = NULL;
-	adev->pcie_rreg = &soc21_pcie_rreg;
-	adev->pcie_wreg = &soc21_pcie_wreg;
-	adev->pcie_rreg64 = &soc21_pcie_rreg64;
-	adev->pcie_wreg64 = &soc21_pcie_wreg64;
+	adev->pcie_rreg = &amdgpu_device_indirect_rreg;
+	adev->pcie_wreg = &amdgpu_device_indirect_wreg;
+	adev->pcie_rreg64 = &amdgpu_device_indirect_rreg64;
+	adev->pcie_wreg64 = &amdgpu_device_indirect_wreg64;
 	adev->pciep_rreg = amdgpu_device_pcie_port_rreg;
 	adev->pciep_wreg = amdgpu_device_pcie_port_wreg;
 
@@ -666,7 +606,7 @@ static int soc21_common_early_init(void *handle)
 
 	adev->asic_funcs = &soc21_asic_funcs;
 
-	adev->rev_id = soc21_get_rev_id(adev);
+	adev->rev_id = amdgpu_device_get_rev_id(adev);
 	adev->external_rev_id = 0xff;
 	switch (adev->ip_versions[GC_HWIP][0]) {
 	case IP_VERSION(11, 0, 0):
@@ -814,6 +754,14 @@ static int soc21_common_late_init(void *handle)
 							     sriov_vcn_4_0_0_video_codecs_decode_array_vcn0,
 							     ARRAY_SIZE(sriov_vcn_4_0_0_video_codecs_decode_array_vcn0));
 		}
+	} else {
+		if (adev->nbio.ras &&
+		    adev->nbio.ras_err_event_athub_irq.funcs)
+			/* don't need to fail gpu late init
+			 * if enabling athub_err_event interrupt failed
+			 * nbio v4_3 only support fatal error hanlding
+			 * just enable the interrupt directly */
+			amdgpu_irq_get(adev, &adev->nbio.ras_err_event_athub_irq, 0);
 	}
 
 	return 0;
@@ -838,8 +786,6 @@ static int soc21_common_hw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	/* enable pcie gen2/3 link */
-	soc21_pcie_gen3_enable(adev);
 	/* enable aspm */
 	soc21_program_aspm(adev);
 	/* setup nbio registers */
@@ -863,8 +809,13 @@ static int soc21_common_hw_fini(void *handle)
 	/* disable the doorbell aperture */
 	soc21_enable_doorbell_aperture(adev, false);
 
-	if (amdgpu_sriov_vf(adev))
+	if (amdgpu_sriov_vf(adev)) {
 		xgpu_nv_mailbox_put_irq(adev);
+	} else {
+		if (adev->nbio.ras &&
+		    adev->nbio.ras_err_event_athub_irq.funcs)
+			amdgpu_irq_put(adev, &adev->nbio.ras_err_event_athub_irq, 0);
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
index 509d8a1945eb..30d0482ac466 100644
--- a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
@@ -84,6 +84,8 @@ enum ta_ras_block {
 	TA_RAS_BLOCK__MP1,
 	TA_RAS_BLOCK__FUSE,
 	TA_RAS_BLOCK__MCA,
+	TA_RAS_BLOCK__VCN,
+	TA_RAS_BLOCK__JPEG,
 	TA_NUM_BLOCK_MAX
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c
index e08e25a3a1a9..530549314ce4 100644
--- a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c
+++ b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c
@@ -160,24 +160,28 @@ static void umc_v6_7_ecc_info_querry_uncorrectable_error_count(struct amdgpu_dev
 	}
 }
 
+static int umc_v6_7_ecc_info_querry_ecc_error_count(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
+{
+	struct ras_err_data *err_data = (struct ras_err_data *)data;
+
+	umc_v6_7_ecc_info_query_correctable_error_count(adev,
+		umc_inst, ch_inst,
+		&(err_data->ce_count));
+
+	umc_v6_7_ecc_info_querry_uncorrectable_error_count(adev,
+		umc_inst, ch_inst,
+		&(err_data->ue_count));
+
+	return 0;
+}
+
 static void umc_v6_7_ecc_info_query_ras_error_count(struct amdgpu_device *adev,
 					   void *ras_error_status)
 {
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-
-	/*TODO: driver needs to toggle DF Cstate to ensure
-	 * safe access of UMC registers. Will add the protection */
-	LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) {
-		umc_v6_7_ecc_info_query_correctable_error_count(adev,
-						      umc_inst, ch_inst,
-						      &(err_data->ce_count));
-		umc_v6_7_ecc_info_querry_uncorrectable_error_count(adev,
-						      umc_inst, ch_inst,
-							  &(err_data->ue_count));
-	}
+	amdgpu_umc_loop_channels(adev,
+		umc_v6_7_ecc_info_querry_ecc_error_count, ras_error_status);
 }
 
 void umc_v6_7_convert_error_address(struct amdgpu_device *adev,
@@ -215,23 +219,23 @@ void umc_v6_7_convert_error_address(struct amdgpu_device *adev,
 	}
 }
 
-static void umc_v6_7_ecc_info_query_error_address(struct amdgpu_device *adev,
-					 struct ras_err_data *err_data,
-					 uint32_t ch_inst,
-					 uint32_t umc_inst)
+static int umc_v6_7_ecc_info_query_error_address(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
 {
 	uint64_t mc_umc_status, err_addr;
 	uint32_t eccinfo_table_idx;
 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
+	struct ras_err_data *err_data = (struct ras_err_data *)data;
 
 	eccinfo_table_idx = umc_inst * adev->umc.channel_inst_num + ch_inst;
 	mc_umc_status = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_status;
 
 	if (mc_umc_status == 0)
-		return;
+		return 0;
 
 	if (!err_data->err_addr)
-		return;
+		return 0;
 
 	/* calculate error address if ue error is detected */
 	if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
@@ -243,25 +247,15 @@ static void umc_v6_7_ecc_info_query_error_address(struct amdgpu_device *adev,
 		umc_v6_7_convert_error_address(adev, err_data, err_addr,
 					ch_inst, umc_inst);
 	}
+
+	return 0;
 }
 
 static void umc_v6_7_ecc_info_query_ras_error_address(struct amdgpu_device *adev,
 					     void *ras_error_status)
 {
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-
-	/*TODO: driver needs to toggle DF Cstate to ensure
-	 * safe access of UMC resgisters. Will add the protection
-	 * when firmware interface is ready */
-	LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) {
-		umc_v6_7_ecc_info_query_error_address(adev,
-					     err_data,
-					     ch_inst,
-					     umc_inst);
-	}
+	amdgpu_umc_loop_channels(adev,
+	    umc_v6_7_ecc_info_query_error_address, ras_error_status);
 }
 
 static void umc_v6_7_query_correctable_error_count(struct amdgpu_device *adev,
@@ -364,11 +358,14 @@ static void umc_v6_7_querry_uncorrectable_error_count(struct amdgpu_device *adev
 	}
 }
 
-static void umc_v6_7_reset_error_count_per_channel(struct amdgpu_device *adev,
-						   uint32_t umc_reg_offset)
+static int umc_v6_7_reset_error_count_per_channel(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
 {
 	uint32_t ecc_err_cnt_addr;
 	uint32_t ecc_err_cnt_sel, ecc_err_cnt_sel_addr;
+	uint32_t umc_reg_offset =
+		get_umc_v6_7_reg_offset(adev, umc_inst, ch_inst);
 
 	ecc_err_cnt_sel_addr =
 		SOC15_REG_OFFSET(UMC, 0,
@@ -402,58 +399,54 @@ static void umc_v6_7_reset_error_count_per_channel(struct amdgpu_device *adev,
 	/* clear higher chip error count */
 	WREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4,
 			UMC_V6_7_CE_CNT_INIT);
+
+	return 0;
 }
 
 static void umc_v6_7_reset_error_count(struct amdgpu_device *adev)
 {
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-	uint32_t umc_reg_offset  = 0;
+	amdgpu_umc_loop_channels(adev,
+		umc_v6_7_reset_error_count_per_channel, NULL);
+}
+
+static int umc_v6_7_query_ecc_error_count(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
+{
+	struct ras_err_data *err_data = (struct ras_err_data *)data;
+	uint32_t umc_reg_offset =
+		get_umc_v6_7_reg_offset(adev, umc_inst, ch_inst);
+
+	umc_v6_7_query_correctable_error_count(adev,
+					umc_reg_offset,
+					&(err_data->ce_count),
+					ch_inst, umc_inst);
 
-	LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) {
-		umc_reg_offset = get_umc_v6_7_reg_offset(adev,
-							 umc_inst,
-							 ch_inst);
+	umc_v6_7_querry_uncorrectable_error_count(adev,
+					umc_reg_offset,
+					&(err_data->ue_count));
 
-		umc_v6_7_reset_error_count_per_channel(adev,
-						       umc_reg_offset);
-	}
+	return 0;
 }
 
 static void umc_v6_7_query_ras_error_count(struct amdgpu_device *adev,
 					   void *ras_error_status)
 {
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-	uint32_t umc_reg_offset  = 0;
-
-	/*TODO: driver needs to toggle DF Cstate to ensure
-	 * safe access of UMC registers. Will add the protection */
-	LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) {
-		umc_reg_offset = get_umc_v6_7_reg_offset(adev,
-							 umc_inst,
-							 ch_inst);
-		umc_v6_7_query_correctable_error_count(adev,
-						       umc_reg_offset,
-						       &(err_data->ce_count),
-						       ch_inst, umc_inst);
-		umc_v6_7_querry_uncorrectable_error_count(adev,
-							  umc_reg_offset,
-							  &(err_data->ue_count));
-	}
+	amdgpu_umc_loop_channels(adev,
+		umc_v6_7_query_ecc_error_count, ras_error_status);
 
 	umc_v6_7_reset_error_count(adev);
 }
 
-static void umc_v6_7_query_error_address(struct amdgpu_device *adev,
-					 struct ras_err_data *err_data,
-					 uint32_t umc_reg_offset, uint32_t ch_inst,
-					 uint32_t umc_inst)
+static int umc_v6_7_query_error_address(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
 {
 	uint32_t mc_umc_status_addr;
 	uint64_t mc_umc_status = 0, mc_umc_addrt0, err_addr;
+	struct ras_err_data *err_data = (struct ras_err_data *)data;
+	uint32_t umc_reg_offset =
+		get_umc_v6_7_reg_offset(adev, umc_inst, ch_inst);
 
 	mc_umc_status_addr =
 		SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
@@ -463,12 +456,12 @@ static void umc_v6_7_query_error_address(struct amdgpu_device *adev,
 	mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4);
 
 	if (mc_umc_status == 0)
-		return;
+		return 0;
 
 	if (!err_data->err_addr) {
 		/* clear umc status */
 		WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
-		return;
+		return 0;
 	}
 
 	/* calculate error address if ue error is detected */
@@ -484,29 +477,15 @@ static void umc_v6_7_query_error_address(struct amdgpu_device *adev,
 
 	/* clear umc status */
 	WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
+
+	return 0;
 }
 
 static void umc_v6_7_query_ras_error_address(struct amdgpu_device *adev,
 					     void *ras_error_status)
 {
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-	uint32_t umc_reg_offset  = 0;
-
-	/*TODO: driver needs to toggle DF Cstate to ensure
-	 * safe access of UMC resgisters. Will add the protection
-	 * when firmware interface is ready */
-	LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) {
-		umc_reg_offset = get_umc_v6_7_reg_offset(adev,
-							 umc_inst,
-							 ch_inst);
-		umc_v6_7_query_error_address(adev,
-					     err_data,
-					     umc_reg_offset, ch_inst,
-					     umc_inst);
-	}
+	amdgpu_umc_loop_channels(adev,
+		umc_v6_7_query_error_address, ras_error_status);
 }
 
 static uint32_t umc_v6_7_query_ras_poison_mode_per_channel(
diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c b/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c
index fb55e8cb9967..d51ae0bc36f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c
+++ b/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c
@@ -76,10 +76,13 @@ static inline uint32_t get_umc_v8_10_reg_offset(struct amdgpu_device *adev,
 		UMC_8_NODE_DIST * node_inst;
 }
 
-static void umc_v8_10_clear_error_count_per_channel(struct amdgpu_device *adev,
-					uint32_t umc_reg_offset)
+static int umc_v8_10_clear_error_count_per_channel(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
 {
 	uint32_t ecc_err_cnt_addr;
+	uint32_t umc_reg_offset =
+		get_umc_v8_10_reg_offset(adev, node_inst, umc_inst, ch_inst);
 
 	ecc_err_cnt_addr =
 		SOC15_REG_OFFSET(UMC, 0, regUMCCH0_0_GeccErrCnt);
@@ -87,24 +90,14 @@ static void umc_v8_10_clear_error_count_per_channel(struct amdgpu_device *adev,
 	/* clear error count */
 	WREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4,
 			UMC_V8_10_CE_CNT_INIT);
+
+	return 0;
 }
 
 static void umc_v8_10_clear_error_count(struct amdgpu_device *adev)
 {
-	uint32_t node_inst       = 0;
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-	uint32_t umc_reg_offset  = 0;
-
-	LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) {
-		umc_reg_offset = get_umc_v8_10_reg_offset(adev,
-						node_inst,
-						umc_inst,
-						ch_inst);
-
-		umc_v8_10_clear_error_count_per_channel(adev,
-						umc_reg_offset);
-	}
+	amdgpu_umc_loop_channels(adev,
+		umc_v8_10_clear_error_count_per_channel, NULL);
 }
 
 static void umc_v8_10_query_correctable_error_count(struct amdgpu_device *adev,
@@ -147,29 +140,29 @@ static void umc_v8_10_query_uncorrectable_error_count(struct amdgpu_device *adev
 		*error_count += 1;
 }
 
+static int umc_v8_10_query_ecc_error_count(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
+{
+	struct ras_err_data *err_data = (struct ras_err_data *)data;
+	uint32_t umc_reg_offset =
+		get_umc_v8_10_reg_offset(adev, node_inst, umc_inst, ch_inst);
+
+	umc_v8_10_query_correctable_error_count(adev,
+					umc_reg_offset,
+					&(err_data->ce_count));
+	umc_v8_10_query_uncorrectable_error_count(adev,
+					umc_reg_offset,
+					&(err_data->ue_count));
+
+	return 0;
+}
+
 static void umc_v8_10_query_ras_error_count(struct amdgpu_device *adev,
 					   void *ras_error_status)
 {
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-
-	uint32_t node_inst       = 0;
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-	uint32_t umc_reg_offset  = 0;
-
-	LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) {
-		umc_reg_offset = get_umc_v8_10_reg_offset(adev,
-						node_inst,
-						umc_inst,
-						ch_inst);
-
-		umc_v8_10_query_correctable_error_count(adev,
-						umc_reg_offset,
-						&(err_data->ce_count));
-		umc_v8_10_query_uncorrectable_error_count(adev,
-						umc_reg_offset,
-						&(err_data->ue_count));
-	}
+	amdgpu_umc_loop_channels(adev,
+		umc_v8_10_query_ecc_error_count, ras_error_status);
 
 	umc_v8_10_clear_error_count(adev);
 }
@@ -248,28 +241,28 @@ static void umc_v8_10_convert_error_address(struct amdgpu_device *adev,
 	}
 }
 
-static void umc_v8_10_query_error_address(struct amdgpu_device *adev,
-					 struct ras_err_data *err_data,
-					 uint32_t umc_reg_offset,
-					 uint32_t node_inst,
-					 uint32_t ch_inst,
-					 uint32_t umc_inst)
+static int umc_v8_10_query_error_address(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
 {
 	uint64_t mc_umc_status_addr;
 	uint64_t mc_umc_status, err_addr;
 	uint64_t mc_umc_addrt0;
+	struct ras_err_data *err_data = (struct ras_err_data *)data;
+	uint32_t umc_reg_offset =
+		get_umc_v8_10_reg_offset(adev, node_inst, umc_inst, ch_inst);
 
 	mc_umc_status_addr =
 		SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
 	mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4);
 
 	if (mc_umc_status == 0)
-		return;
+		return 0;
 
 	if (!err_data->err_addr) {
 		/* clear umc status */
 		WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
-		return;
+		return 0;
 	}
 
 	/* calculate error address if ue error is detected */
@@ -287,37 +280,25 @@ static void umc_v8_10_query_error_address(struct amdgpu_device *adev,
 
 	/* clear umc status */
 	WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
+
+	return 0;
 }
 
 static void umc_v8_10_query_ras_error_address(struct amdgpu_device *adev,
 					     void *ras_error_status)
 {
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-	uint32_t node_inst       = 0;
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-	uint32_t umc_reg_offset  = 0;
-
-	LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) {
-		umc_reg_offset = get_umc_v8_10_reg_offset(adev,
-						node_inst,
-						umc_inst,
-						ch_inst);
-
-		umc_v8_10_query_error_address(adev,
-					err_data,
-					umc_reg_offset,
-					node_inst,
-					ch_inst,
-					umc_inst);
-	}
+	amdgpu_umc_loop_channels(adev,
+		umc_v8_10_query_error_address, ras_error_status);
 }
 
-static void umc_v8_10_err_cnt_init_per_channel(struct amdgpu_device *adev,
-					      uint32_t umc_reg_offset)
+static int umc_v8_10_err_cnt_init_per_channel(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
 {
 	uint32_t ecc_err_cnt_sel, ecc_err_cnt_sel_addr;
 	uint32_t ecc_err_cnt_addr;
+	uint32_t umc_reg_offset =
+		get_umc_v8_10_reg_offset(adev, node_inst, umc_inst, ch_inst);
 
 	ecc_err_cnt_sel_addr =
 		SOC15_REG_OFFSET(UMC, 0, regUMCCH0_0_GeccErrCntSel);
@@ -332,23 +313,14 @@ static void umc_v8_10_err_cnt_init_per_channel(struct amdgpu_device *adev,
 	WREG32_PCIE((ecc_err_cnt_sel_addr + umc_reg_offset) * 4, ecc_err_cnt_sel);
 	/* set error count to initial value */
 	WREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4, UMC_V8_10_CE_CNT_INIT);
+
+	return 0;
 }
 
 static void umc_v8_10_err_cnt_init(struct amdgpu_device *adev)
 {
-	uint32_t node_inst       = 0;
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-	uint32_t umc_reg_offset  = 0;
-
-	LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) {
-		umc_reg_offset = get_umc_v8_10_reg_offset(adev,
-						node_inst,
-						umc_inst,
-						ch_inst);
-
-		umc_v8_10_err_cnt_init_per_channel(adev, umc_reg_offset);
-	}
+	amdgpu_umc_loop_channels(adev,
+		umc_v8_10_err_cnt_init_per_channel, NULL);
 }
 
 static bool umc_v8_10_query_ras_poison_mode(struct amdgpu_device *adev)
@@ -406,37 +378,35 @@ static void umc_v8_10_ecc_info_query_uncorrectable_error_count(struct amdgpu_dev
 	}
 }
 
+static int umc_v8_10_ecc_info_query_ecc_error_count(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
+{
+	struct ras_err_data *err_data = (struct ras_err_data *)data;
+
+	umc_v8_10_ecc_info_query_correctable_error_count(adev,
+					node_inst, umc_inst, ch_inst,
+					&(err_data->ce_count));
+	umc_v8_10_ecc_info_query_uncorrectable_error_count(adev,
+					node_inst, umc_inst, ch_inst,
+					&(err_data->ue_count));
+	return 0;
+}
+
 static void umc_v8_10_ecc_info_query_ras_error_count(struct amdgpu_device *adev,
 					void *ras_error_status)
 {
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-
-	uint32_t node_inst       = 0;
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-
-	/* TODO: driver needs to toggle DF Cstate to ensure
-	 * safe access of UMC registers. Will add the protection
-	 */
-	LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) {
-		umc_v8_10_ecc_info_query_correctable_error_count(adev,
-							node_inst, umc_inst, ch_inst,
-							&(err_data->ce_count));
-		umc_v8_10_ecc_info_query_uncorrectable_error_count(adev,
-							node_inst, umc_inst, ch_inst,
-							&(err_data->ue_count));
-	}
+	amdgpu_umc_loop_channels(adev,
+		umc_v8_10_ecc_info_query_ecc_error_count, ras_error_status);
 }
 
-static void umc_v8_10_ecc_info_query_error_address(struct amdgpu_device *adev,
-					struct ras_err_data *err_data,
-					uint32_t ch_inst,
-					uint32_t umc_inst,
-					uint32_t node_inst)
+static int umc_v8_10_ecc_info_query_error_address(struct amdgpu_device *adev,
+					uint32_t node_inst, uint32_t umc_inst,
+					uint32_t ch_inst, void *data)
 {
 	uint32_t eccinfo_table_idx;
 	uint64_t mc_umc_status, err_addr;
-
+	struct ras_err_data *err_data = (struct ras_err_data *)data;
 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 
 	eccinfo_table_idx = node_inst * adev->umc.umc_inst_num *
@@ -447,10 +417,10 @@ static void umc_v8_10_ecc_info_query_error_address(struct amdgpu_device *adev,
 	mc_umc_status = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_status;
 
 	if (mc_umc_status == 0)
-		return;
+		return 0;
 
 	if (!err_data->err_addr)
-		return;
+		return 0;
 
 	/* calculate error address if ue error is detected */
 	if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
@@ -463,28 +433,15 @@ static void umc_v8_10_ecc_info_query_error_address(struct amdgpu_device *adev,
 		umc_v8_10_convert_error_address(adev, err_data, err_addr,
 					ch_inst, umc_inst, node_inst, mc_umc_status);
 	}
+
+	return 0;
 }
 
 static void umc_v8_10_ecc_info_query_ras_error_address(struct amdgpu_device *adev,
 					void *ras_error_status)
 {
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-
-	uint32_t node_inst       = 0;
-	uint32_t umc_inst        = 0;
-	uint32_t ch_inst         = 0;
-
-	/* TODO: driver needs to toggle DF Cstate to ensure
-	 * safe access of UMC resgisters. Will add the protection
-	 * when firmware interface is ready
-	 */
-	LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) {
-		umc_v8_10_ecc_info_query_error_address(adev,
-						err_data,
-						ch_inst,
-						umc_inst,
-						node_inst);
-	}
+	amdgpu_umc_loop_channels(adev,
+		umc_v8_10_ecc_info_query_error_address, ras_error_status);
 }
 
 const struct amdgpu_ras_block_hw_ops umc_v8_10_ras_hw_ops = {
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index e407be6cb63c..e32b656b3dab 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -444,6 +444,7 @@ static int uvd_v7_0_sw_init(void *handle)
 			continue;
 		if (!amdgpu_sriov_vf(adev)) {
 			ring = &adev->uvd.inst[j].ring;
+			ring->vm_hub = AMDGPU_MMHUB_0;
 			sprintf(ring->name, "uvd_%d", ring->me);
 			r = amdgpu_ring_init(adev, ring, 512,
 					     &adev->uvd.inst[j].irq, 0,
@@ -454,6 +455,7 @@ static int uvd_v7_0_sw_init(void *handle)
 
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
 			ring = &adev->uvd.inst[j].ring_enc[i];
+			ring->vm_hub = AMDGPU_MMHUB_0;
 			sprintf(ring->name, "uvd_enc_%d.%d", ring->me, i);
 			if (amdgpu_sriov_vf(adev)) {
 				ring->use_doorbell = true;
@@ -1397,7 +1399,7 @@ static void uvd_v7_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 					unsigned vmid, uint64_t pd_addr)
 {
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 	uint32_t data0, data1, mask;
 
 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
@@ -1440,7 +1442,7 @@ static void uvd_v7_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring,
 static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
 					    unsigned int vmid, uint64_t pd_addr)
 {
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 
 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
 
@@ -1802,7 +1804,6 @@ static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {
 	.align_mask = 0xf,
 	.support_64bit_ptrs = false,
 	.no_user_fence = true,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = uvd_v7_0_ring_get_rptr,
 	.get_wptr = uvd_v7_0_ring_get_wptr,
 	.set_wptr = uvd_v7_0_ring_set_wptr,
@@ -1835,7 +1836,6 @@ static const struct amdgpu_ring_funcs uvd_v7_0_enc_ring_vm_funcs = {
 	.nop = HEVC_ENC_CMD_NO_OP,
 	.support_64bit_ptrs = false,
 	.no_user_fence = true,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = uvd_v7_0_enc_ring_get_rptr,
 	.get_wptr = uvd_v7_0_enc_ring_get_wptr,
 	.set_wptr = uvd_v7_0_enc_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index 66cd3d11aa4b..57b85bb6a1e4 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -466,6 +466,7 @@ static int vce_v4_0_sw_init(void *handle)
 		enum amdgpu_ring_priority_level hw_prio = amdgpu_vce_get_ring_prio(i);
 
 		ring = &adev->vce.ring[i];
+		ring->vm_hub = AMDGPU_MMHUB_0;
 		sprintf(ring->name, "vce%d", i);
 		if (amdgpu_sriov_vf(adev)) {
 			/* DOORBELL only works under SRIOV */
@@ -1021,7 +1022,7 @@ static void vce_v4_0_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,
 				   unsigned int vmid, uint64_t pd_addr)
 {
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 
 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
 
@@ -1103,7 +1104,6 @@ static const struct amdgpu_ring_funcs vce_v4_0_ring_vm_funcs = {
 	.nop = VCE_CMD_NO_OP,
 	.support_64bit_ptrs = false,
 	.no_user_fence = true,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = vce_v4_0_ring_get_rptr,
 	.get_wptr = vce_v4_0_ring_get_wptr,
 	.set_wptr = vce_v4_0_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_sw_ring.c b/drivers/gpu/drm/amd/amdgpu/vcn_sw_ring.c
index 1ceda3d0cd5b..2b9ddb3d2fe1 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_sw_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_sw_ring.c
@@ -65,7 +65,7 @@ void vcn_dec_sw_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 void vcn_dec_sw_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	uint32_t vmid, uint64_t pd_addr)
 {
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 	uint32_t data0, data1, mask;
 
 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index c305b2cb8490..761c28fa6ec1 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -120,6 +120,7 @@ static int vcn_v1_0_sw_init(void *handle)
 		return r;
 
 	ring = &adev->vcn.inst->ring_dec;
+	ring->vm_hub = AMDGPU_MMHUB_0;
 	sprintf(ring->name, "vcn_dec");
 	r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0,
 			     AMDGPU_RING_PRIO_DEFAULT, NULL);
@@ -141,6 +142,7 @@ static int vcn_v1_0_sw_init(void *handle)
 		enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(i);
 
 		ring = &adev->vcn.inst->ring_enc[i];
+		ring->vm_hub = AMDGPU_MMHUB_0;
 		sprintf(ring->name, "vcn_enc%d", i);
 		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0,
 				     hw_prio, NULL);
@@ -1548,7 +1550,7 @@ static void vcn_v1_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring,
 static void vcn_v1_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
 					    unsigned vmid, uint64_t pd_addr)
 {
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 	uint32_t data0, data1, mask;
 
 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
@@ -1693,7 +1695,7 @@ static void vcn_v1_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring,
 static void vcn_v1_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
 					    unsigned int vmid, uint64_t pd_addr)
 {
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 
 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
 
@@ -1977,7 +1979,6 @@ static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
 	.support_64bit_ptrs = false,
 	.no_user_fence = true,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = vcn_v1_0_dec_ring_get_rptr,
 	.get_wptr = vcn_v1_0_dec_ring_get_wptr,
 	.set_wptr = vcn_v1_0_dec_ring_set_wptr,
@@ -2012,7 +2013,6 @@ static const struct amdgpu_ring_funcs vcn_v1_0_enc_ring_vm_funcs = {
 	.nop = VCN_ENC_CMD_NO_OP,
 	.support_64bit_ptrs = false,
 	.no_user_fence = true,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = vcn_v1_0_enc_ring_get_rptr,
 	.get_wptr = vcn_v1_0_enc_ring_get_wptr,
 	.set_wptr = vcn_v1_0_enc_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
index 4b4cd88414e0..7c2b3aa48083 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
@@ -129,6 +129,7 @@ static int vcn_v2_0_sw_init(void *handle)
 
 	ring->use_doorbell = true;
 	ring->doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
+	ring->vm_hub = AMDGPU_MMHUB_0;
 
 	sprintf(ring->name, "vcn_dec");
 	r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0,
@@ -159,6 +160,7 @@ static int vcn_v2_0_sw_init(void *handle)
 
 		ring = &adev->vcn.inst->ring_enc[i];
 		ring->use_doorbell = true;
+		ring->vm_hub = AMDGPU_MMHUB_0;
 		if (!amdgpu_sriov_vf(adev))
 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + i;
 		else
@@ -1511,7 +1513,7 @@ void vcn_v2_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 void vcn_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
 				unsigned vmid, uint64_t pd_addr)
 {
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 	uint32_t data0, data1, mask;
 
 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
@@ -1671,7 +1673,7 @@ void vcn_v2_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 void vcn_v2_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
 				unsigned int vmid, uint64_t pd_addr)
 {
-	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 
 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
 
@@ -2014,7 +2016,6 @@ static const struct amdgpu_ring_funcs vcn_v2_0_dec_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_DEC,
 	.align_mask = 0xf,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = vcn_v2_0_dec_ring_get_rptr,
 	.get_wptr = vcn_v2_0_dec_ring_get_wptr,
 	.set_wptr = vcn_v2_0_dec_ring_set_wptr,
@@ -2045,7 +2046,6 @@ static const struct amdgpu_ring_funcs vcn_v2_0_enc_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_ENC,
 	.align_mask = 0x3f,
 	.nop = VCN_ENC_CMD_NO_OP,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = vcn_v2_0_enc_ring_get_rptr,
 	.get_wptr = vcn_v2_0_enc_ring_get_wptr,
 	.set_wptr = vcn_v2_0_enc_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
index b0b0e69c6a94..ab0b45d0ead1 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
@@ -186,6 +186,12 @@ static int vcn_v2_5_sw_init(void *handle)
 
 		ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 				(amdgpu_sriov_vf(adev) ? 2*j : 8*j);
+
+		if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(2, 5, 0))
+			ring->vm_hub = AMDGPU_MMHUB_1;
+		else
+			ring->vm_hub = AMDGPU_MMHUB_0;
+
 		sprintf(ring->name, "vcn_dec_%d", j);
 		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[j].irq,
 				     0, AMDGPU_RING_PRIO_DEFAULT, NULL);
@@ -201,6 +207,11 @@ static int vcn_v2_5_sw_init(void *handle)
 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 					(amdgpu_sriov_vf(adev) ? (1 + i + 2*j) : (2 + i + 8*j));
 
+			if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(2, 5, 0))
+				ring->vm_hub = AMDGPU_MMHUB_1;
+			else
+				ring->vm_hub = AMDGPU_MMHUB_0;
+
 			sprintf(ring->name, "vcn_enc_%d.%d", j, i);
 			r = amdgpu_ring_init(adev, ring, 512,
 					     &adev->vcn.inst[j].irq, 0,
@@ -225,6 +236,10 @@ static int vcn_v2_5_sw_init(void *handle)
 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
 		adev->vcn.pause_dpg_mode = vcn_v2_5_pause_dpg_mode;
 
+	r = amdgpu_vcn_ras_sw_init(adev);
+	if (r)
+		return r;
+
 	return 0;
 }
 
@@ -1558,38 +1573,6 @@ static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_DEC,
 	.align_mask = 0xf,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_MMHUB_1,
-	.get_rptr = vcn_v2_5_dec_ring_get_rptr,
-	.get_wptr = vcn_v2_5_dec_ring_get_wptr,
-	.set_wptr = vcn_v2_5_dec_ring_set_wptr,
-	.emit_frame_size =
-		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
-		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
-		8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
-		14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
-		6,
-	.emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
-	.emit_ib = vcn_v2_0_dec_ring_emit_ib,
-	.emit_fence = vcn_v2_0_dec_ring_emit_fence,
-	.emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
-	.test_ring = vcn_v2_0_dec_ring_test_ring,
-	.test_ib = amdgpu_vcn_dec_ring_test_ib,
-	.insert_nop = vcn_v2_0_dec_ring_insert_nop,
-	.insert_start = vcn_v2_0_dec_ring_insert_start,
-	.insert_end = vcn_v2_0_dec_ring_insert_end,
-	.pad_ib = amdgpu_ring_generic_pad_ib,
-	.begin_use = amdgpu_vcn_ring_begin_use,
-	.end_use = amdgpu_vcn_ring_end_use,
-	.emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
-	.emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
-	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
-};
-
-static const struct amdgpu_ring_funcs vcn_v2_6_dec_ring_vm_funcs = {
-	.type = AMDGPU_RING_TYPE_VCN_DEC,
-	.align_mask = 0xf,
-	.secure_submission_supported = true,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = vcn_v2_5_dec_ring_get_rptr,
 	.get_wptr = vcn_v2_5_dec_ring_get_wptr,
 	.set_wptr = vcn_v2_5_dec_ring_set_wptr,
@@ -1689,7 +1672,6 @@ static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_ENC,
 	.align_mask = 0x3f,
 	.nop = VCN_ENC_CMD_NO_OP,
-	.vmhub = AMDGPU_MMHUB_1,
 	.get_rptr = vcn_v2_5_enc_ring_get_rptr,
 	.get_wptr = vcn_v2_5_enc_ring_get_wptr,
 	.set_wptr = vcn_v2_5_enc_ring_set_wptr,
@@ -1715,36 +1697,6 @@ static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = {
 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
 };
 
-static const struct amdgpu_ring_funcs vcn_v2_6_enc_ring_vm_funcs = {
-        .type = AMDGPU_RING_TYPE_VCN_ENC,
-        .align_mask = 0x3f,
-        .nop = VCN_ENC_CMD_NO_OP,
-        .vmhub = AMDGPU_MMHUB_0,
-        .get_rptr = vcn_v2_5_enc_ring_get_rptr,
-        .get_wptr = vcn_v2_5_enc_ring_get_wptr,
-        .set_wptr = vcn_v2_5_enc_ring_set_wptr,
-        .emit_frame_size =
-                SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
-                SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
-                4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
-                5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
-                1, /* vcn_v2_0_enc_ring_insert_end */
-        .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
-        .emit_ib = vcn_v2_0_enc_ring_emit_ib,
-        .emit_fence = vcn_v2_0_enc_ring_emit_fence,
-        .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
-        .test_ring = amdgpu_vcn_enc_ring_test_ring,
-        .test_ib = amdgpu_vcn_enc_ring_test_ib,
-        .insert_nop = amdgpu_ring_insert_nop,
-        .insert_end = vcn_v2_0_enc_ring_insert_end,
-        .pad_ib = amdgpu_ring_generic_pad_ib,
-        .begin_use = amdgpu_vcn_ring_begin_use,
-        .end_use = amdgpu_vcn_ring_end_use,
-        .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
-        .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
-        .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
-};
-
 static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev)
 {
 	int i;
@@ -1752,10 +1704,7 @@ static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev)
 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 		if (adev->vcn.harvest_config & (1 << i))
 			continue;
-		if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(2, 5, 0))
-			adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_5_dec_ring_vm_funcs;
-		else /* CHIP_ALDEBARAN */
-			adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_6_dec_ring_vm_funcs;
+		adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_5_dec_ring_vm_funcs;
 		adev->vcn.inst[i].ring_dec.me = i;
 		DRM_INFO("VCN(%d) decode is enabled in VM mode\n", i);
 	}
@@ -1769,10 +1718,7 @@ static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev)
 		if (adev->vcn.harvest_config & (1 << j))
 			continue;
 		for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
-			if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(2, 5, 0))
-				adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_5_enc_ring_vm_funcs;
-			else /* CHIP_ALDEBARAN */
-				adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_6_enc_ring_vm_funcs;
+			adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_5_enc_ring_vm_funcs;
 			adev->vcn.inst[j].ring_enc[i].me = j;
 		}
 		DRM_INFO("VCN(%d) encode is enabled in VM mode\n", j);
@@ -2031,6 +1977,4 @@ static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev)
 	default:
 		break;
 	}
-
-	amdgpu_vcn_set_ras_funcs(adev);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
index 66439388faee..3eab186261aa 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
@@ -189,6 +189,7 @@ static int vcn_v3_0_sw_init(void *handle)
 		} else {
 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i;
 		}
+		ring->vm_hub = AMDGPU_MMHUB_0;
 		sprintf(ring->name, "vcn_dec_%d", i);
 		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
 				     AMDGPU_RING_PRIO_DEFAULT,
@@ -212,6 +213,7 @@ static int vcn_v3_0_sw_init(void *handle)
 			} else {
 				ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;
 			}
+			ring->vm_hub = AMDGPU_MMHUB_0;
 			sprintf(ring->name, "vcn_enc_%d.%d", i, j);
 			r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
 					     hw_prio, &adev->vcn.inst[i].sched_score);
@@ -1738,7 +1740,6 @@ static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
 	.align_mask = 0x3f,
 	.nop = VCN_DEC_SW_CMD_NO_OP,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = vcn_v3_0_dec_ring_get_rptr,
 	.get_wptr = vcn_v3_0_dec_ring_get_wptr,
 	.set_wptr = vcn_v3_0_dec_ring_set_wptr,
@@ -1899,7 +1900,6 @@ static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_DEC,
 	.align_mask = 0xf,
 	.secure_submission_supported = true,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = vcn_v3_0_dec_ring_get_rptr,
 	.get_wptr = vcn_v3_0_dec_ring_get_wptr,
 	.set_wptr = vcn_v3_0_dec_ring_set_wptr,
@@ -2000,7 +2000,6 @@ static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_ENC,
 	.align_mask = 0x3f,
 	.nop = VCN_ENC_CMD_NO_OP,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = vcn_v3_0_enc_ring_get_rptr,
 	.get_wptr = vcn_v3_0_enc_ring_get_wptr,
 	.set_wptr = vcn_v3_0_enc_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
index 43d587404c3e..bf0674039598 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
@@ -149,7 +149,7 @@ static int vcn_v4_0_sw_init(void *handle)
 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + i * (adev->vcn.num_enc_rings + 1) + 1;
 		else
 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;
-
+		ring->vm_hub = AMDGPU_MMHUB_0;
 		sprintf(ring->name, "vcn_unified_%d", i);
 
 		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
@@ -181,6 +181,10 @@ static int vcn_v4_0_sw_init(void *handle)
 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
 		adev->vcn.pause_dpg_mode = vcn_v4_0_pause_dpg_mode;
 
+	r = amdgpu_vcn_ras_sw_init(adev);
+	if (r)
+		return r;
+
 	return 0;
 }
 
@@ -1794,7 +1798,6 @@ static const struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_ENC,
 	.align_mask = 0x3f,
 	.nop = VCN_ENC_CMD_NO_OP,
-	.vmhub = AMDGPU_MMHUB_0,
 	.get_rptr = vcn_v4_0_unified_ring_get_rptr,
 	.get_wptr = vcn_v4_0_unified_ring_get_wptr,
 	.set_wptr = vcn_v4_0_unified_ring_set_wptr,
@@ -2123,6 +2126,4 @@ static void vcn_v4_0_set_ras_funcs(struct amdgpu_device *adev)
 	default:
 		break;
 	}
-
-	amdgpu_vcn_set_ras_funcs(adev);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c
index 1706081d054d..536128447b71 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c
@@ -38,6 +38,11 @@
 #define mmIH_CHICKEN_ALDEBARAN			0x18d
 #define mmIH_CHICKEN_ALDEBARAN_BASE_IDX		0
 
+#define mmIH_RETRY_INT_CAM_CNTL_ALDEBARAN		0x00ea
+#define mmIH_RETRY_INT_CAM_CNTL_ALDEBARAN_BASE_IDX	0
+#define IH_RETRY_INT_CAM_CNTL_ALDEBARAN__ENABLE__SHIFT	0x10
+#define IH_RETRY_INT_CAM_CNTL_ALDEBARAN__ENABLE_MASK	0x00010000L
+
 static void vega20_ih_set_interrupt_funcs(struct amdgpu_device *adev);
 
 /**
@@ -251,36 +256,14 @@ static int vega20_ih_enable_ring(struct amdgpu_device *adev,
 	return 0;
 }
 
-/**
- * vega20_ih_reroute_ih - reroute VMC/UTCL2 ih to an ih ring
- *
- * @adev: amdgpu_device pointer
- *
- * Reroute VMC and UMC interrupts on primary ih ring to
- * ih ring 1 so they won't lose when bunches of page faults
- * interrupts overwhelms the interrupt handler(VEGA20)
- */
-static void vega20_ih_reroute_ih(struct amdgpu_device *adev)
+static uint32_t vega20_setup_retry_doorbell(u32 doorbell_index)
 {
-	uint32_t tmp;
+	u32 val = 0;
 
-	/* vega20 ih reroute will go through psp this
-	 * function is used for newer asics starting arcturus
-	 */
-	if (adev->ip_versions[OSSSYS_HWIP][0] >= IP_VERSION(4, 2, 1)) {
-		/* Reroute to IH ring 1 for VMC */
-		WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x12);
-		tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA);
-		tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1);
-		tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1);
-		WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp);
-
-		/* Reroute IH ring 1 for UTCL2 */
-		WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x1B);
-		tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA);
-		tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1);
-		WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp);
-	}
+	val = REG_SET_FIELD(val, IH_DOORBELL_RPTR, OFFSET, doorbell_index);
+	val = REG_SET_FIELD(val, IH_DOORBELL_RPTR, ENABLE, 1);
+
+	return val;
 }
 
 /**
@@ -321,7 +304,8 @@ static int vega20_ih_irq_init(struct amdgpu_device *adev)
 	/* psp firmware won't program IH_CHICKEN for aldebaran
 	 * driver needs to program it properly according to
 	 * MC_SPACE type in IH_RB_CNTL */
-	if (adev->ip_versions[OSSSYS_HWIP][0] == IP_VERSION(4, 4, 0)) {
+	if ((adev->ip_versions[OSSSYS_HWIP][0] == IP_VERSION(4, 4, 0)) ||
+	    (adev->ip_versions[OSSSYS_HWIP][0] == IP_VERSION(4, 4, 2))) {
 		ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN_ALDEBARAN);
 		if (adev->irq.ih.use_bus_addr) {
 			ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN,
@@ -332,8 +316,6 @@ static int vega20_ih_irq_init(struct amdgpu_device *adev)
 
 	for (i = 0; i < ARRAY_SIZE(ih); i++) {
 		if (ih[i]->ring_size) {
-			if (i == 1)
-				vega20_ih_reroute_ih(adev);
 			ret = vega20_ih_enable_ring(adev, ih[i]);
 			if (ret)
 				return ret;
@@ -346,6 +328,20 @@ static int vega20_ih_irq_init(struct amdgpu_device *adev)
 
 	pci_set_master(adev->pdev);
 
+	/* Allocate the doorbell for IH Retry CAM */
+	adev->irq.retry_cam_doorbell_index = (adev->doorbell_index.ih + 3) << 1;
+	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RETRY_CAM,
+		vega20_setup_retry_doorbell(adev->irq.retry_cam_doorbell_index));
+
+	/* Enable IH Retry CAM */
+	if (adev->ip_versions[OSSSYS_HWIP][0] == IP_VERSION(4, 4, 0))
+		WREG32_FIELD15(OSSSYS, 0, IH_RETRY_INT_CAM_CNTL_ALDEBARAN,
+			       ENABLE, 1);
+	else
+		WREG32_FIELD15(OSSSYS, 0, IH_RETRY_INT_CAM_CNTL, ENABLE, 1);
+
+	adev->irq.retry_cam_enabled = true;
+
 	/* enable interrupts */
 	ret = vega20_ih_toggle_interrupts(adev, true);
 	if (ret)
@@ -551,12 +547,14 @@ static int vega20_ih_sw_init(void *handle)
 	adev->irq.ih1.use_doorbell = true;
 	adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1;
 
-	r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true);
-	if (r)
-		return r;
+	if (adev->ip_versions[OSSSYS_HWIP][0] != IP_VERSION(4, 4, 2)) {
+		r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true);
+		if (r)
+			return r;
 
-	adev->irq.ih2.use_doorbell = true;
-	adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1;
+		adev->irq.ih2.use_doorbell = true;
+		adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1;
+	}
 
 	/* initialize ih control registers offset */
 	vega20_ih_init_register_offset(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index ceab8783575c..531f173ade2d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -1101,24 +1101,6 @@ static int vi_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
 	return 0;
 }
 
-static void vi_pcie_gen3_enable(struct amdgpu_device *adev)
-{
-	if (pci_is_root_bus(adev->pdev->bus))
-		return;
-
-	if (amdgpu_pcie_gen2 == 0)
-		return;
-
-	if (adev->flags & AMD_IS_APU)
-		return;
-
-	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
-					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
-		return;
-
-	/* todo */
-}
-
 static void vi_enable_aspm(struct amdgpu_device *adev)
 {
 	u32 data, orig;
@@ -1728,8 +1710,6 @@ static int vi_common_hw_init(void *handle)
 
 	/* move the golden regs per IP block */
 	vi_init_golden_registers(adev);
-	/* enable pcie gen2/3 link */
-	vi_pcie_gen3_enable(adev);
 	/* enable aspm */
 	vi_program_aspm(adev);
 	/* enable the doorbell aperture */
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
index 0c4c5499bb5c..73ca9aebf086 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
@@ -2936,3 +2936,490 @@ static const uint32_t cwsr_trap_gfx11_hex[] = {
 	0xbf9f0000, 0xbf9f0000,
 	0xbf9f0000, 0x00000000,
 };
+
+static const uint32_t cwsr_trap_gfx9_4_3_hex[] = {
+	0xbf820001, 0xbf8202d6,
+	0xb8f8f802, 0x89788678,
+	0xb8fbf803, 0x866eff78,
+	0x00002000, 0xbf840009,
+	0x866eff6d, 0x00ff0000,
+	0xbf85001a, 0x866eff7b,
+	0x00000400, 0xbf85004d,
+	0xbf8e0010, 0xb8fbf803,
+	0xbf82fffa, 0x866eff7b,
+	0x03c00900, 0xbf850011,
+	0x866eff7b, 0x000071ff,
+	0xbf840008, 0x866fff7b,
+	0x00007080, 0xbf840001,
+	0xbeee1a87, 0xb8eff801,
+	0x8e6e8c6e, 0x866e6f6e,
+	0xbf850006, 0x866eff6d,
+	0x00ff0000, 0xbf850003,
+	0x866eff7b, 0x00000400,
+	0xbf850036, 0xb8faf807,
+	0x867aff7a, 0x001f8000,
+	0x8e7a8b7a, 0x8979ff79,
+	0xfc000000, 0x87797a79,
+	0xba7ff807, 0x00000000,
+	0xb8faf812, 0xb8fbf813,
+	0x8efa887a, 0xc0031bbd,
+	0x00000010, 0xbf8cc07f,
+	0x8e6e976e, 0x8979ff79,
+	0x00800000, 0x87796e79,
+	0xc0071bbd, 0x00000000,
+	0xbf8cc07f, 0xc0071ebd,
+	0x00000008, 0xbf8cc07f,
+	0x86ee6e6e, 0xbf840001,
+	0xbe801d6e, 0x866eff6d,
+	0x01ff0000, 0xbf850005,
+	0x8778ff78, 0x00002000,
+	0x80ec886c, 0x82ed806d,
+	0xbf820005, 0x866eff6d,
+	0x01000000, 0xbf850002,
+	0x806c846c, 0x826d806d,
+	0x866dff6d, 0x0000ffff,
+	0x8f7a8b79, 0x867aff7a,
+	0x001f8000, 0xb97af807,
+	0x86fe7e7e, 0x86ea6a6a,
+	0x8f6e8378, 0xb96ee0c2,
+	0xbf800002, 0xb9780002,
+	0xbe801f6c, 0x866dff6d,
+	0x0000ffff, 0xbefa0080,
+	0xb97a0283, 0xb8faf807,
+	0x867aff7a, 0x001f8000,
+	0x8e7a8b7a, 0x8979ff79,
+	0xfc000000, 0x87797a79,
+	0xba7ff807, 0x00000000,
+	0xbeee007e, 0xbeef007f,
+	0xbefe0180, 0xbf900004,
+	0x877a8478, 0xb97af802,
+	0xbf8e0002, 0xbf88fffe,
+	0xb8fa2985, 0x807a817a,
+	0x8e7a8a7a, 0x8e7a817a,
+	0xb8fb1605, 0x807b817b,
+	0x8e7b867b, 0x807a7b7a,
+	0x807a7e7a, 0x827b807f,
+	0x867bff7b, 0x0000ffff,
+	0xc04b1c3d, 0x00000050,
+	0xbf8cc07f, 0xc04b1d3d,
+	0x00000060, 0xbf8cc07f,
+	0xc0431e7d, 0x00000074,
+	0xbf8cc07f, 0xbef4007e,
+	0x8675ff7f, 0x0000ffff,
+	0x8775ff75, 0x00040000,
+	0xbef60080, 0xbef700ff,
+	0x00807fac, 0xbef1007c,
+	0xbef00080, 0xb8f02985,
+	0x80708170, 0x8e708a70,
+	0x8e708170, 0xb8fa1605,
+	0x807a817a, 0x8e7a867a,
+	0x80707a70, 0xbef60084,
+	0xbef600ff, 0x01000000,
+	0xbefe007c, 0xbefc0070,
+	0xc0611c7a, 0x0000007c,
+	0xbf8cc07f, 0x80708470,
+	0xbefc007e, 0xbefe007c,
+	0xbefc0070, 0xc0611b3a,
+	0x0000007c, 0xbf8cc07f,
+	0x80708470, 0xbefc007e,
+	0xbefe007c, 0xbefc0070,
+	0xc0611b7a, 0x0000007c,
+	0xbf8cc07f, 0x80708470,
+	0xbefc007e, 0xbefe007c,
+	0xbefc0070, 0xc0611bba,
+	0x0000007c, 0xbf8cc07f,
+	0x80708470, 0xbefc007e,
+	0xbefe007c, 0xbefc0070,
+	0xc0611bfa, 0x0000007c,
+	0xbf8cc07f, 0x80708470,
+	0xbefc007e, 0xbefe007c,
+	0xbefc0070, 0xc0611e3a,
+	0x0000007c, 0xbf8cc07f,
+	0x80708470, 0xbefc007e,
+	0xb8fbf803, 0xbefe007c,
+	0xbefc0070, 0xc0611efa,
+	0x0000007c, 0xbf8cc07f,
+	0x80708470, 0xbefc007e,
+	0xbefe007c, 0xbefc0070,
+	0xc0611a3a, 0x0000007c,
+	0xbf8cc07f, 0x80708470,
+	0xbefc007e, 0xbefe007c,
+	0xbefc0070, 0xc0611a7a,
+	0x0000007c, 0xbf8cc07f,
+	0x80708470, 0xbefc007e,
+	0xb8f1f801, 0xbefe007c,
+	0xbefc0070, 0xc0611c7a,
+	0x0000007c, 0xbf8cc07f,
+	0x80708470, 0xbefc007e,
+	0x867aff7f, 0x04000000,
+	0xbeef0080, 0x876f6f7a,
+	0xb8f02985, 0x80708170,
+	0x8e708a70, 0x8e708170,
+	0xb8fb1605, 0x807b817b,
+	0x8e7b847b, 0x8e76827b,
+	0xbef600ff, 0x01000000,
+	0xbef20174, 0x80747074,
+	0x82758075, 0xbefc0080,
+	0xbf800000, 0xbe802b00,
+	0xbe822b02, 0xbe842b04,
+	0xbe862b06, 0xbe882b08,
+	0xbe8a2b0a, 0xbe8c2b0c,
+	0xbe8e2b0e, 0xc06b003a,
+	0x00000000, 0xbf8cc07f,
+	0xc06b013a, 0x00000010,
+	0xbf8cc07f, 0xc06b023a,
+	0x00000020, 0xbf8cc07f,
+	0xc06b033a, 0x00000030,
+	0xbf8cc07f, 0x8074c074,
+	0x82758075, 0x807c907c,
+	0xbf0a7b7c, 0xbf85ffe7,
+	0xbef40172, 0xbef00080,
+	0xbefe00c1, 0xbeff00c1,
+	0xbee80080, 0xbee90080,
+	0xbef600ff, 0x01000000,
+	0x867aff78, 0x00400000,
+	0xbf850003, 0xb8faf803,
+	0x897a7aff, 0x10000000,
+	0xbf85004d, 0xbe840080,
+	0xd2890000, 0x00000900,
+	0x80048104, 0xd2890001,
+	0x00000900, 0x80048104,
+	0xd2890002, 0x00000900,
+	0x80048104, 0xd2890003,
+	0x00000900, 0x80048104,
+	0xc069003a, 0x00000070,
+	0xbf8cc07f, 0x80709070,
+	0xbf06c004, 0xbf84ffee,
+	0xbe840080, 0xd2890000,
+	0x00000901, 0x80048104,
+	0xd2890001, 0x00000901,
+	0x80048104, 0xd2890002,
+	0x00000901, 0x80048104,
+	0xd2890003, 0x00000901,
+	0x80048104, 0xc069003a,
+	0x00000070, 0xbf8cc07f,
+	0x80709070, 0xbf06c004,
+	0xbf84ffee, 0xbe840080,
+	0xd2890000, 0x00000902,
+	0x80048104, 0xd2890001,
+	0x00000902, 0x80048104,
+	0xd2890002, 0x00000902,
+	0x80048104, 0xd2890003,
+	0x00000902, 0x80048104,
+	0xc069003a, 0x00000070,
+	0xbf8cc07f, 0x80709070,
+	0xbf06c004, 0xbf84ffee,
+	0xbe840080, 0xd2890000,
+	0x00000903, 0x80048104,
+	0xd2890001, 0x00000903,
+	0x80048104, 0xd2890002,
+	0x00000903, 0x80048104,
+	0xd2890003, 0x00000903,
+	0x80048104, 0xc069003a,
+	0x00000070, 0xbf8cc07f,
+	0x80709070, 0xbf06c004,
+	0xbf84ffee, 0xbf820008,
+	0xe0724000, 0x701d0000,
+	0xe0724100, 0x701d0100,
+	0xe0724200, 0x701d0200,
+	0xe0724300, 0x701d0300,
+	0xbefe00c1, 0xbeff00c1,
+	0xb8fb4306, 0x867bc17b,
+	0xbf840064, 0xbf8a0000,
+	0x867aff6f, 0x04000000,
+	0xbf840060, 0x8e7b867b,
+	0x8e7b827b, 0xbef6007b,
+	0xb8f02985, 0x80708170,
+	0x8e708a70, 0x8e708170,
+	0xb8fa1605, 0x807a817a,
+	0x8e7a867a, 0x80707a70,
+	0x8070ff70, 0x00000080,
+	0xbef600ff, 0x01000000,
+	0xbefc0080, 0xd28c0002,
+	0x000100c1, 0xd28d0003,
+	0x000204c1, 0x867aff78,
+	0x00400000, 0xbf850003,
+	0xb8faf803, 0x897a7aff,
+	0x10000000, 0xbf850030,
+	0x24040682, 0xd86e4000,
+	0x00000002, 0xbf8cc07f,
+	0xbe840080, 0xd2890000,
+	0x00000900, 0x80048104,
+	0xd2890001, 0x00000900,
+	0x80048104, 0xd2890002,
+	0x00000900, 0x80048104,
+	0xd2890003, 0x00000900,
+	0x80048104, 0xc069003a,
+	0x00000070, 0xbf8cc07f,
+	0x80709070, 0xbf06c004,
+	0xbf84ffee, 0xbe840080,
+	0xd2890000, 0x00000901,
+	0x80048104, 0xd2890001,
+	0x00000901, 0x80048104,
+	0xd2890002, 0x00000901,
+	0x80048104, 0xd2890003,
+	0x00000901, 0x80048104,
+	0xc069003a, 0x00000070,
+	0xbf8cc07f, 0x80709070,
+	0xbf06c004, 0xbf84ffee,
+	0x680404ff, 0x00000200,
+	0xd0c9006a, 0x0000f702,
+	0xbf87ffd2, 0xbf820015,
+	0xd1060002, 0x00011103,
+	0x7e0602ff, 0x00000200,
+	0xbefc00ff, 0x00010000,
+	0xbe800077, 0x8677ff77,
+	0xff7fffff, 0x8777ff77,
+	0x00058000, 0xd8ec0000,
+	0x00000002, 0xbf8cc07f,
+	0xe0765000, 0x701d0002,
+	0x68040702, 0xd0c9006a,
+	0x0000f702, 0xbf87fff7,
+	0xbef70000, 0xbef000ff,
+	0x00000400, 0xbefe00c1,
+	0xbeff00c1, 0xb8fb2b05,
+	0x807b817b, 0x8e7b827b,
+	0xbef600ff, 0x01000000,
+	0xbefc0084, 0xbf0a7b7c,
+	0xbf84006d, 0xbf11017c,
+	0x807bff7b, 0x00001000,
+	0x867aff78, 0x00400000,
+	0xbf850003, 0xb8faf803,
+	0x897a7aff, 0x10000000,
+	0xbf850051, 0xbe840080,
+	0xd2890000, 0x00000900,
+	0x80048104, 0xd2890001,
+	0x00000900, 0x80048104,
+	0xd2890002, 0x00000900,
+	0x80048104, 0xd2890003,
+	0x00000900, 0x80048104,
+	0xc069003a, 0x00000070,
+	0xbf8cc07f, 0x80709070,
+	0xbf06c004, 0xbf84ffee,
+	0xbe840080, 0xd2890000,
+	0x00000901, 0x80048104,
+	0xd2890001, 0x00000901,
+	0x80048104, 0xd2890002,
+	0x00000901, 0x80048104,
+	0xd2890003, 0x00000901,
+	0x80048104, 0xc069003a,
+	0x00000070, 0xbf8cc07f,
+	0x80709070, 0xbf06c004,
+	0xbf84ffee, 0xbe840080,
+	0xd2890000, 0x00000902,
+	0x80048104, 0xd2890001,
+	0x00000902, 0x80048104,
+	0xd2890002, 0x00000902,
+	0x80048104, 0xd2890003,
+	0x00000902, 0x80048104,
+	0xc069003a, 0x00000070,
+	0xbf8cc07f, 0x80709070,
+	0xbf06c004, 0xbf84ffee,
+	0xbe840080, 0xd2890000,
+	0x00000903, 0x80048104,
+	0xd2890001, 0x00000903,
+	0x80048104, 0xd2890002,
+	0x00000903, 0x80048104,
+	0xd2890003, 0x00000903,
+	0x80048104, 0xc069003a,
+	0x00000070, 0xbf8cc07f,
+	0x80709070, 0xbf06c004,
+	0xbf84ffee, 0x807c847c,
+	0xbf0a7b7c, 0xbf85ffb1,
+	0xbf9c0000, 0xbf820012,
+	0x7e000300, 0x7e020301,
+	0x7e040302, 0x7e060303,
+	0xe0724000, 0x701d0000,
+	0xe0724100, 0x701d0100,
+	0xe0724200, 0x701d0200,
+	0xe0724300, 0x701d0300,
+	0x807c847c, 0x8070ff70,
+	0x00000400, 0xbf0a7b7c,
+	0xbf85ffef, 0xbf9c0000,
+	0xb8fb2985, 0x807b817b,
+	0x8e7b837b, 0xb8fa2b05,
+	0x807a817a, 0x8e7a827a,
+	0x80fb7a7b, 0x867b7b7b,
+	0xbf84007a, 0x807bff7b,
+	0x00001000, 0xbefc0080,
+	0xbf11017c, 0x867aff78,
+	0x00400000, 0xbf850003,
+	0xb8faf803, 0x897a7aff,
+	0x10000000, 0xbf850059,
+	0xd3d84000, 0x18000100,
+	0xd3d84001, 0x18000101,
+	0xd3d84002, 0x18000102,
+	0xd3d84003, 0x18000103,
+	0xbe840080, 0xd2890000,
+	0x00000900, 0x80048104,
+	0xd2890001, 0x00000900,
+	0x80048104, 0xd2890002,
+	0x00000900, 0x80048104,
+	0xd2890003, 0x00000900,
+	0x80048104, 0xc069003a,
+	0x00000070, 0xbf8cc07f,
+	0x80709070, 0xbf06c004,
+	0xbf84ffee, 0xbe840080,
+	0xd2890000, 0x00000901,
+	0x80048104, 0xd2890001,
+	0x00000901, 0x80048104,
+	0xd2890002, 0x00000901,
+	0x80048104, 0xd2890003,
+	0x00000901, 0x80048104,
+	0xc069003a, 0x00000070,
+	0xbf8cc07f, 0x80709070,
+	0xbf06c004, 0xbf84ffee,
+	0xbe840080, 0xd2890000,
+	0x00000902, 0x80048104,
+	0xd2890001, 0x00000902,
+	0x80048104, 0xd2890002,
+	0x00000902, 0x80048104,
+	0xd2890003, 0x00000902,
+	0x80048104, 0xc069003a,
+	0x00000070, 0xbf8cc07f,
+	0x80709070, 0xbf06c004,
+	0xbf84ffee, 0xbe840080,
+	0xd2890000, 0x00000903,
+	0x80048104, 0xd2890001,
+	0x00000903, 0x80048104,
+	0xd2890002, 0x00000903,
+	0x80048104, 0xd2890003,
+	0x00000903, 0x80048104,
+	0xc069003a, 0x00000070,
+	0xbf8cc07f, 0x80709070,
+	0xbf06c004, 0xbf84ffee,
+	0x807c847c, 0xbf0a7b7c,
+	0xbf85ffa9, 0xbf9c0000,
+	0xbf820016, 0xd3d84000,
+	0x18000100, 0xd3d84001,
+	0x18000101, 0xd3d84002,
+	0x18000102, 0xd3d84003,
+	0x18000103, 0xe0724000,
+	0x701d0000, 0xe0724100,
+	0x701d0100, 0xe0724200,
+	0x701d0200, 0xe0724300,
+	0x701d0300, 0x807c847c,
+	0x8070ff70, 0x00000400,
+	0xbf0a7b7c, 0xbf85ffeb,
+	0xbf9c0000, 0xbf8200ee,
+	0xbef4007e, 0x8675ff7f,
+	0x0000ffff, 0x8775ff75,
+	0x00040000, 0xbef60080,
+	0xbef700ff, 0x00807fac,
+	0x866eff7f, 0x04000000,
+	0xbf84001f, 0xbefe00c1,
+	0xbeff00c1, 0xb8ef4306,
+	0x866fc16f, 0xbf84001a,
+	0x8e6f866f, 0x8e6f826f,
+	0xbef6006f, 0xb8f82985,
+	0x80788178, 0x8e788a78,
+	0x8e788178, 0xb8ee1605,
+	0x806e816e, 0x8e6e866e,
+	0x80786e78, 0x8078ff78,
+	0x00000080, 0xbef600ff,
+	0x01000000, 0xbefc0080,
+	0xe0510000, 0x781d0000,
+	0xe0510100, 0x781d0000,
+	0x807cff7c, 0x00000200,
+	0x8078ff78, 0x00000200,
+	0xbf0a6f7c, 0xbf85fff6,
+	0xbefe00c1, 0xbeff00c1,
+	0xbef600ff, 0x01000000,
+	0xb8ef2b05, 0x806f816f,
+	0x8e6f826f, 0x806fff6f,
+	0x00008000, 0xbef80080,
+	0xbeee0078, 0x8078ff78,
+	0x00000400, 0xbefc0084,
+	0xbf11087c, 0xe0524000,
+	0x781d0000, 0xe0524100,
+	0x781d0100, 0xe0524200,
+	0x781d0200, 0xe0524300,
+	0x781d0300, 0xbf8c0f70,
+	0x7e000300, 0x7e020301,
+	0x7e040302, 0x7e060303,
+	0x807c847c, 0x8078ff78,
+	0x00000400, 0xbf0a6f7c,
+	0xbf85ffee, 0xb8ef2985,
+	0x806f816f, 0x8e6f836f,
+	0xb8f92b05, 0x80798179,
+	0x8e798279, 0x80ef796f,
+	0x866f6f6f, 0xbf84001a,
+	0x806fff6f, 0x00008000,
+	0xbefc0080, 0xbf11087c,
+	0xe0524000, 0x781d0000,
+	0xe0524100, 0x781d0100,
+	0xe0524200, 0x781d0200,
+	0xe0524300, 0x781d0300,
+	0xbf8c0f70, 0xd3d94000,
+	0x18000100, 0xd3d94001,
+	0x18000101, 0xd3d94002,
+	0x18000102, 0xd3d94003,
+	0x18000103, 0x807c847c,
+	0x8078ff78, 0x00000400,
+	0xbf0a6f7c, 0xbf85ffea,
+	0xbf9c0000, 0xe0524000,
+	0x6e1d0000, 0xe0524100,
+	0x6e1d0100, 0xe0524200,
+	0x6e1d0200, 0xe0524300,
+	0x6e1d0300, 0xbf8c0f70,
+	0xb8f82985, 0x80788178,
+	0x8e788a78, 0x8e788178,
+	0xb8ee1605, 0x806e816e,
+	0x8e6e866e, 0x80786e78,
+	0x80f8c078, 0xb8ef1605,
+	0x806f816f, 0x8e6f846f,
+	0x8e76826f, 0xbef600ff,
+	0x01000000, 0xbefc006f,
+	0xc031003a, 0x00000078,
+	0x80f8c078, 0xbf8cc07f,
+	0x80fc907c, 0xbf800000,
+	0xbe802d00, 0xbe822d02,
+	0xbe842d04, 0xbe862d06,
+	0xbe882d08, 0xbe8a2d0a,
+	0xbe8c2d0c, 0xbe8e2d0e,
+	0xbf06807c, 0xbf84fff0,
+	0xb8f82985, 0x80788178,
+	0x8e788a78, 0x8e788178,
+	0xb8ee1605, 0x806e816e,
+	0x8e6e866e, 0x80786e78,
+	0xbef60084, 0xbef600ff,
+	0x01000000, 0xc0211bfa,
+	0x00000078, 0x80788478,
+	0xc0211b3a, 0x00000078,
+	0x80788478, 0xc0211b7a,
+	0x00000078, 0x80788478,
+	0xc0211c3a, 0x00000078,
+	0x80788478, 0xc0211c7a,
+	0x00000078, 0x80788478,
+	0xc0211eba, 0x00000078,
+	0x80788478, 0xc0211efa,
+	0x00000078, 0x80788478,
+	0xc0211a3a, 0x00000078,
+	0x80788478, 0xc0211a7a,
+	0x00000078, 0x80788478,
+	0xc0211cfa, 0x00000078,
+	0x80788478, 0xbf8cc07f,
+	0xbefc006f, 0xbefe0070,
+	0xbeff0071, 0x866f7bff,
+	0x000003ff, 0xb96f4803,
+	0x866f7bff, 0xfffff800,
+	0x8f6f8b6f, 0xb96fa2c3,
+	0xb973f801, 0xb8ee2985,
+	0x806e816e, 0x8e6e8a6e,
+	0x8e6e816e, 0xb8ef1605,
+	0x806f816f, 0x8e6f866f,
+	0x806e6f6e, 0x806e746e,
+	0x826f8075, 0x866fff6f,
+	0x0000ffff, 0xc00b1c37,
+	0x00000050, 0xc00b1d37,
+	0x00000060, 0xc0031e77,
+	0x00000074, 0xbf8cc07f,
+	0x8f6e8b79, 0x866eff6e,
+	0x001f8000, 0xb96ef807,
+	0x866dff6d, 0x0000ffff,
+	0x86fe7e7e, 0x86ea6a6a,
+	0x8f6e837a, 0xb96ee0c2,
+	0xbf800002, 0xb97a0002,
+	0xbf8a0000, 0xbe801f6c,
+	0xbf810000, 0x00000000,
+};
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
index 6770cbe3250a..f2087cc2e89d 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
@@ -33,15 +33,20 @@
  * aldebaran:
  *   cpp -DASIC_FAMILY=CHIP_ALDEBARAN cwsr_trap_handler_gfx9.asm -P -o aldebaran.sp3
  *   sp3 aldebaran.sp3 -hex aldebaran.hex
+ *
+ * gc_9_4_3:
+ *   cpp -DASIC_FAMILY=GC_9_4_3 cwsr_trap_handler_gfx9.asm -P -o gc_9_4_3.sp3
+ *   sp3 gc_9_4_3.sp3 -hex gc_9_4_3.hex
  */
 
 #define CHIP_VEGAM 18
 #define CHIP_ARCTURUS 23
 #define CHIP_ALDEBARAN 25
+#define CHIP_GC_9_4_3 26
 
 var ACK_SQC_STORE		    =	1		    //workaround for suspected SQC store bug causing incorrect stores under concurrency
 var SAVE_AFTER_XNACK_ERROR	    =	1		    //workaround for TCP store failure after XNACK error when ALLOW_REPLAY=0, for debugger
-var SINGLE_STEP_MISSED_WORKAROUND   =	1		    //workaround for lost MODE.DEBUG_EN exception when SAVECTX raised
+var SINGLE_STEP_MISSED_WORKAROUND   =	(ASIC_FAMILY <= CHIP_ALDEBARAN)	//workaround for lost MODE.DEBUG_EN exception when SAVECTX raised
 
 /**************************************************************************/
 /*			variables					  */
@@ -77,6 +82,10 @@ var SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK =	0x80
 var SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT =	7
 var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK   =	0x100
 var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT  =	8
+var SQ_WAVE_TRAPSTS_HOST_TRAP_MASK  =	0x400000
+var SQ_WAVE_TRAPSTS_WAVE_BEGIN_MASK =	0x800000
+var SQ_WAVE_TRAPSTS_WAVE_END_MASK   =	0x1000000
+var SQ_WAVE_TRAPSTS_TRAP_AFTER_INST_MASK =  0x2000000
 var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK	=   0x3FF
 var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT	=   0x0
 var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE	=   10
@@ -95,10 +104,10 @@ var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK	= 0x1F8000
 
 var SQ_WAVE_MODE_DEBUG_EN_MASK		=   0x800
 
-var TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT	=   26			// bits [31:26] unused by SPI debug data
-var TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK	=   0xFC000000
-var TTMP11_DEBUG_TRAP_ENABLED_SHIFT	=   23
-var TTMP11_DEBUG_TRAP_ENABLED_MASK	=   0x800000
+var TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT	=   26			// bits [31:26] unused by SPI debug data
+var TTMP_SAVE_RCNT_FIRST_REPLAY_MASK	=   0xFC000000
+var TTMP_DEBUG_TRAP_ENABLED_SHIFT	=   23
+var TTMP_DEBUG_TRAP_ENABLED_MASK	=   0x800000
 
 /*	Save	    */
 var S_SAVE_BUF_RSRC_WORD1_STRIDE	=   0x00040000		//stride is 4 bytes
@@ -129,6 +138,11 @@ var s_save_alloc_size	    =	s_save_trapsts		//conflict
 var s_save_m0		    =	ttmp5
 var s_save_ttmps_lo	    =	s_save_tmp		//no conflict
 var s_save_ttmps_hi	    =	s_save_trapsts		//no conflict
+#if ASIC_FAMILY >= CHIP_GC_9_4_3
+var s_save_ib_sts       =	ttmp13
+#else
+var s_save_ib_sts       =	ttmp11
+#endif
 
 /*	Restore	    */
 var S_RESTORE_BUF_RSRC_WORD1_STRIDE	    =	S_SAVE_BUF_RSRC_WORD1_STRIDE
@@ -215,9 +229,15 @@ L_NOT_HALTED:
     // Any concurrent SAVECTX will be handled upon re-entry once halted.
 
     // Check non-maskable exceptions. memory_violation, illegal_instruction
-    // and xnack_error exceptions always cause the wave to enter the trap
-    // handler.
-    s_and_b32       ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK|SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK
+    // and debugger (host trap, wave start/end, trap after instruction)
+    // exceptions always cause the wave to enter the trap handler.
+    s_and_b32       ttmp2, s_save_trapsts,      \
+        SQ_WAVE_TRAPSTS_MEM_VIOL_MASK         | \
+        SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK     | \
+        SQ_WAVE_TRAPSTS_HOST_TRAP_MASK        | \
+        SQ_WAVE_TRAPSTS_WAVE_BEGIN_MASK       | \
+        SQ_WAVE_TRAPSTS_WAVE_END_MASK         | \
+        SQ_WAVE_TRAPSTS_TRAP_AFTER_INST_MASK
     s_cbranch_scc1  L_FETCH_2ND_TRAP
 
     // Check for maskable exceptions in trapsts.excp and trapsts.excp_hi.
@@ -265,9 +285,9 @@ L_FETCH_2ND_TRAP:
 
     s_load_dword    ttmp2, [ttmp14, ttmp15], 0x10 glc:1 // debug trap enabled flag
     s_waitcnt       lgkmcnt(0)
-    s_lshl_b32      ttmp2, ttmp2, TTMP11_DEBUG_TRAP_ENABLED_SHIFT
-    s_andn2_b32     ttmp11, ttmp11, TTMP11_DEBUG_TRAP_ENABLED_MASK
-    s_or_b32        ttmp11, ttmp11, ttmp2
+    s_lshl_b32      ttmp2, ttmp2, TTMP_DEBUG_TRAP_ENABLED_SHIFT
+    s_andn2_b32     s_save_ib_sts, s_save_ib_sts, TTMP_DEBUG_TRAP_ENABLED_MASK
+    s_or_b32        s_save_ib_sts, s_save_ib_sts, ttmp2
 
     s_load_dwordx2  [ttmp2, ttmp3], [ttmp14, ttmp15], 0x0 glc:1 // second-level TBA
     s_waitcnt       lgkmcnt(0)
@@ -1058,17 +1078,17 @@ function set_status_without_spi_prio(status, tmp)
 end
 
 function save_and_clear_ib_sts(tmp)
-    // Save IB_STS.FIRST_REPLAY[15] and IB_STS.RCNT[20:16] into unused space ttmp11[31:26].
+    // Save IB_STS.FIRST_REPLAY[15] and IB_STS.RCNT[20:16] into unused space s_save_ib_sts[31:26].
     s_getreg_b32    tmp, hwreg(HW_REG_IB_STS)
     s_and_b32       tmp, tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
-    s_lshl_b32      tmp, tmp, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
-    s_andn2_b32     ttmp11, ttmp11, TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK
-    s_or_b32        ttmp11, ttmp11, tmp
+    s_lshl_b32      tmp, tmp, (TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
+    s_andn2_b32     s_save_ib_sts, s_save_ib_sts, TTMP_SAVE_RCNT_FIRST_REPLAY_MASK
+    s_or_b32        s_save_ib_sts, s_save_ib_sts, tmp
     s_setreg_imm32_b32 hwreg(HW_REG_IB_STS), 0x0
 end
 
 function restore_ib_sts(tmp)
-    s_lshr_b32      tmp, ttmp11, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
+    s_lshr_b32      tmp, s_save_ib_sts, (TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
     s_and_b32       tmp, tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
     s_setreg_b32    hwreg(HW_REG_IB_STS), tmp
 end
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index de310ed367ca..1b54a9aaae70 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -93,7 +93,7 @@ int kfd_chardev_init(void)
 	if (err < 0)
 		goto err_register_chrdev;
 
-	kfd_class = class_create(THIS_MODULE, kfd_dev_name);
+	kfd_class = class_create(kfd_dev_name);
 	err = PTR_ERR(kfd_class);
 	if (IS_ERR(kfd_class))
 		goto err_class_create;
@@ -1590,6 +1590,58 @@ err_unlock:
 	return r;
 }
 
+static int kfd_ioctl_export_dmabuf(struct file *filep,
+				   struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_export_dmabuf_args *args = data;
+	struct kfd_process_device *pdd;
+	struct dma_buf *dmabuf;
+	struct kfd_dev *dev;
+	void *mem;
+	int ret = 0;
+
+	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
+	if (!dev)
+		return -EINVAL;
+
+	mutex_lock(&p->mutex);
+
+	pdd = kfd_get_process_device_data(dev, p);
+	if (!pdd) {
+		ret = -EINVAL;
+		goto err_unlock;
+	}
+
+	mem = kfd_process_device_translate_handle(pdd,
+						GET_IDR_HANDLE(args->handle));
+	if (!mem) {
+		ret = -EINVAL;
+		goto err_unlock;
+	}
+
+	ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, &dmabuf);
+	mutex_unlock(&p->mutex);
+	if (ret)
+		goto err_out;
+
+	ret = dma_buf_fd(dmabuf, args->flags);
+	if (ret < 0) {
+		dma_buf_put(dmabuf);
+		goto err_out;
+	}
+	/* dma_buf_fd assigns the reference count to the fd, no need to
+	 * put the reference here.
+	 */
+	args->dmabuf_fd = ret;
+
+	return 0;
+
+err_unlock:
+	mutex_unlock(&p->mutex);
+err_out:
+	return ret;
+}
+
 /* Handle requests for watching SMI events */
 static int kfd_ioctl_smi_events(struct file *filep,
 				struct kfd_process *p, void *data)
@@ -2772,6 +2824,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
 
 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
 			kfd_ioctl_get_available_memory, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_EXPORT_DMABUF,
+				kfd_ioctl_export_dmabuf, 0),
 };
 
 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index 3251f4783ba1..475e47027354 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -1462,6 +1462,7 @@ int kfd_get_gpu_cache_info(struct kfd_dev *kdev, struct kfd_gpu_cache_info **pca
 			num_of_cache_types = ARRAY_SIZE(vega20_cache_info);
 			break;
 		case IP_VERSION(9, 4, 2):
+		case IP_VERSION(9, 4, 3):
 			*pcache_info = aldebaran_cache_info;
 			num_of_cache_types = ARRAY_SIZE(aldebaran_cache_info);
 			break;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index ec70a1658dc3..00f528eb9812 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -204,6 +204,14 @@ static void kfd_device_info_init(struct kfd_dev *kfd,
 			/* Navi1x+ */
 			if (gc_version >= IP_VERSION(10, 1, 1))
 				kfd->device_info.needs_pci_atomics = true;
+		} else if (gc_version < IP_VERSION(12, 0, 0)) {
+			/*
+			 * PCIe atomics support acknowledgment in GFX11 RS64 CPFW requires
+			 * MEC version >= 509. Prior RS64 CPFW versions (and all F32) require
+			 * PCIe atomics support.
+			 */
+			kfd->device_info.needs_pci_atomics = true;
+			kfd->device_info.no_atomic_fw_version = kfd->adev->gfx.rs64_enable ? 509 : 0;
 		}
 	} else {
 		kfd->device_info.doorbell_size = 4;
@@ -318,6 +326,10 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
 			gfx_target_version = 90010;
 			f2g = &aldebaran_kfd2kgd;
 			break;
+		case IP_VERSION(9, 4, 3):
+			gfx_target_version = 90400;
+			f2g = &aldebaran_kfd2kgd;
+			break;
 		/* Navi10 */
 		case IP_VERSION(10, 1, 10):
 			gfx_target_version = 100100;
@@ -450,6 +462,10 @@ static void kfd_cwsr_init(struct kfd_dev *kfd)
 			BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) > PAGE_SIZE);
 			kfd->cwsr_isa = cwsr_trap_aldebaran_hex;
 			kfd->cwsr_isa_size = sizeof(cwsr_trap_aldebaran_hex);
+		} else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3)) {
+			BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex) > PAGE_SIZE);
+			kfd->cwsr_isa = cwsr_trap_gfx9_4_3_hex;
+			kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_4_3_hex);
 		} else if (KFD_GC_VERSION(kfd) < IP_VERSION(10, 1, 1)) {
 			BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE);
 			kfd->cwsr_isa = cwsr_trap_gfx9_hex;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
index 4a9af800b1f1..5aa75f72caa1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
@@ -143,6 +143,13 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 			1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
 			1 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
 
+	/*
+	 * GFX11 RS64 CPFW version >= 509 supports PCIe atomics support
+	 * acknowledgment.
+	 */
+	if (amdgpu_amdkfd_have_atomics_support(mm->dev->adev))
+		m->cp_hqd_hq_status0 |= 1 << 29;
+
 	if (q->format == KFD_QUEUE_FORMAT_AQL) {
 		m->cp_hqd_aql_control =
 			1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
@@ -350,6 +357,10 @@ static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
 	m->sdmax_rlcx_doorbell_offset =
 		q->doorbell_off << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
 
+	m->sdmax_rlcx_sched_cntl = (amdgpu_sdma_phase_quantum
+		<< SDMA0_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT)
+		 & SDMA0_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK;
+
 	m->sdma_engine_id = q->sdma_engine_id;
 	m->sdma_queue_id = q->sdma_queue_id;
 	m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 0778e587a2d6..fdbfd725841f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -135,6 +135,7 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 {
 	uint64_t addr;
 	struct v9_mqd *m;
+	struct amdgpu_device *adev = (struct amdgpu_device *)mm->dev->adev;
 
 	m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
 	addr = mqd_mem_obj->gpu_addr;
@@ -167,6 +168,20 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 	if (q->format == KFD_QUEUE_FORMAT_AQL) {
 		m->cp_hqd_aql_control =
 			1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
+		if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
+			/* On GC 9.4.3, DW 41 is re-purposed as
+			 * compute_tg_chunk_size.
+			 * TODO: review this setting when active CUs in the
+			 * partition play a role
+			 */
+			m->compute_static_thread_mgmt_se6 = 1;
+		}
+	} else {
+		/* PM4 queue */
+		if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
+			m->compute_static_thread_mgmt_se6 = 0;
+			/* TODO: program pm4_target_xcc */
+		}
 	}
 
 	if (q->tba_addr) {
@@ -209,6 +224,7 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
 			struct queue_properties *q,
 			struct mqd_update_info *minfo)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)mm->dev->adev;
 	struct v9_mqd *m;
 
 	m = get_mqd(mqd);
@@ -254,10 +270,13 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
 	m->cp_hqd_vmid = q->vmid;
 
 	if (q->format == KFD_QUEUE_FORMAT_AQL) {
-		m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
+		m->cp_hqd_pq_control |=
 				2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT |
 				1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT |
 				1 << CP_HQD_PQ_CONTROL__WPP_CLAMP_EN__SHIFT;
+		if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 3))
+			m->cp_hqd_pq_control |=
+				 CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK;
 		m->cp_hqd_pq_doorbell_control |= 1 <<
 			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;
 	}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index ed02b6d8bf63..f612325241aa 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -238,7 +238,8 @@ int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
 		pm->pmf = &kfd_vi_pm_funcs;
 		break;
 	default:
-		if (KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 2))
+		if (KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 2) ||
+		    KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 3))
 			pm->pmf = &kfd_aldebaran_pm_funcs;
 		else if (KFD_GC_VERSION(dqm->dev) >= IP_VERSION(9, 0, 1))
 			pm->pmf = &kfd_v9_pm_funcs;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 7e4d992e48b3..94a438956868 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -207,7 +207,8 @@ enum cache_policy {
 #define KFD_GC_VERSION(dev) ((dev)->adev->ip_versions[GC_HWIP][0])
 #define KFD_IS_SOC15(dev)   ((KFD_GC_VERSION(dev)) >= (IP_VERSION(9, 0, 1)))
 #define KFD_SUPPORT_XNACK_PER_PROCESS(dev)\
-		(KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2))
+	((KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) ||	\
+	 (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)))
 
 struct kfd_event_interrupt_class {
 	bool (*interrupt_isr)(struct kfd_dev *dev,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 4208e0f01064..07a9eaf9b7d8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -344,7 +344,7 @@ static const struct sysfs_ops kfd_procfs_ops = {
 	.show = kfd_procfs_show,
 };
 
-static struct kobj_type procfs_type = {
+static const struct kobj_type procfs_type = {
 	.release = kfd_procfs_kobj_release,
 	.sysfs_ops = &kfd_procfs_ops,
 };
@@ -469,7 +469,7 @@ static const struct sysfs_ops procfs_queue_ops = {
 	.show = kfd_procfs_queue_show,
 };
 
-static struct kobj_type procfs_queue_type = {
+static const struct kobj_type procfs_queue_type = {
 	.sysfs_ops = &procfs_queue_ops,
 	.default_groups = procfs_queue_groups,
 };
@@ -478,7 +478,7 @@ static const struct sysfs_ops procfs_stats_ops = {
 	.show = kfd_procfs_stats_show,
 };
 
-static struct kobj_type procfs_stats_type = {
+static const struct kobj_type procfs_stats_type = {
 	.sysfs_ops = &procfs_stats_ops,
 	.release = kfd_procfs_kobj_release,
 };
@@ -487,7 +487,7 @@ static const struct sysfs_ops sysfs_counters_ops = {
 	.show = kfd_sysfs_counters_show,
 };
 
-static struct kobj_type sysfs_counters_type = {
+static const struct kobj_type sysfs_counters_type = {
 	.sysfs_ops = &sysfs_counters_ops,
 	.release = kfd_procfs_kobj_release,
 };
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index dc6fd6967050..96a138a39515 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -2172,7 +2172,15 @@ restart:
 		pr_debug("drain retry fault gpu %d svms %p\n", i, svms);
 
 		amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev,
-						     &pdd->dev->adev->irq.ih1);
+				pdd->dev->adev->irq.retry_cam_enabled ?
+				&pdd->dev->adev->irq.ih :
+				&pdd->dev->adev->irq.ih1);
+
+		if (pdd->dev->adev->irq.retry_cam_enabled)
+			amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev,
+				&pdd->dev->adev->irq.ih_soft);
+
+
 		pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms);
 	}
 	if (atomic_cmpxchg(&svms->drain_pagefaults, drain, 0) != drain)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 3fdaba56be6f..8e4124dcb6e4 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -278,7 +278,7 @@ static const struct sysfs_ops sysprops_ops = {
 	.show = sysprops_show,
 };
 
-static struct kobj_type sysprops_type = {
+static const struct kobj_type sysprops_type = {
 	.release = kfd_topology_kobj_release,
 	.sysfs_ops = &sysprops_ops,
 };
@@ -318,7 +318,7 @@ static const struct sysfs_ops iolink_ops = {
 	.show = iolink_show,
 };
 
-static struct kobj_type iolink_type = {
+static const struct kobj_type iolink_type = {
 	.release = kfd_topology_kobj_release,
 	.sysfs_ops = &iolink_ops,
 };
@@ -350,7 +350,7 @@ static const struct sysfs_ops mem_ops = {
 	.show = mem_show,
 };
 
-static struct kobj_type mem_type = {
+static const struct kobj_type mem_type = {
 	.release = kfd_topology_kobj_release,
 	.sysfs_ops = &mem_ops,
 };
@@ -395,7 +395,7 @@ static const struct sysfs_ops cache_ops = {
 	.show = kfd_cache_show,
 };
 
-static struct kobj_type cache_type = {
+static const struct kobj_type cache_type = {
 	.release = kfd_topology_kobj_release,
 	.sysfs_ops = &cache_ops,
 };
@@ -566,7 +566,7 @@ static const struct sysfs_ops node_ops = {
 	.show = node_show,
 };
 
-static struct kobj_type node_type = {
+static const struct kobj_type node_type = {
 	.release = kfd_topology_kobj_release,
 	.sysfs_ops = &node_ops,
 };
diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig
index 0c9bd0a53e60..06b438217c61 100644
--- a/drivers/gpu/drm/amd/display/Kconfig
+++ b/drivers/gpu/drm/amd/display/Kconfig
@@ -8,7 +8,7 @@ config DRM_AMD_DC
 	depends on BROKEN || !CC_IS_CLANG || X86_64 || SPARC64 || ARM64
 	select SND_HDA_COMPONENT if SND_HDA_CORE
 	# !CC_IS_CLANG: https://github.com/ClangBuiltLinux/linux/issues/1752
-	select DRM_AMD_DC_DCN if (X86 || PPC_LONG_DOUBLE_128 || (ARM64 && KERNEL_MODE_NEON && !CC_IS_CLANG))
+	select DRM_AMD_DC_FP if (X86 || PPC64 || (ARM64 && KERNEL_MODE_NEON && !CC_IS_CLANG))
 	help
 	  Choose this option if you want to use the new display engine
 	  support for AMDGPU. This adds required support for Vega and
@@ -20,16 +20,10 @@ config DRM_AMD_DC
 	  panic on most architectures.  We'll revert this when the following bug report
 	  has been resolved: https://github.com/llvm/llvm-project/issues/41896.
 
-config DRM_AMD_DC_DCN
+config DRM_AMD_DC_FP
 	def_bool n
 	help
-	  Raven, Navi, and newer family support for display engine
-
-config DRM_AMD_DC_HDCP
-	bool "Enable HDCP support in DC"
-	depends on DRM_AMD_DC
-	help
-	  Choose this option if you want to support HDCP authentication.
+	  Floating point support, required for DCN-based SoCs
 
 config DRM_AMD_DC_SI
 	bool "AMD DC support for Southern Islands ASICs"
@@ -50,7 +44,7 @@ config DEBUG_KERNEL_DC
 config DRM_AMD_SECURE_DISPLAY
         bool "Enable secure display support"
         depends on DEBUG_FS
-        depends on DRM_AMD_DC_DCN
+        depends on DRM_AMD_DC_FP
         help
             Choose this option if you want to
             support secure display
diff --git a/drivers/gpu/drm/amd/display/Makefile b/drivers/gpu/drm/amd/display/Makefile
index 2633de77de5e..0d610cb376bb 100644
--- a/drivers/gpu/drm/amd/display/Makefile
+++ b/drivers/gpu/drm/amd/display/Makefile
@@ -36,18 +36,14 @@ subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/info_packet
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/power
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dmub/inc
 
-ifdef CONFIG_DRM_AMD_DC_HDCP
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/hdcp
-endif
 
 #TODO: remove when Timing Sync feature is complete
 subdir-ccflags-y += -DBUILD_FEATURE_TIMING_SYNC=0
 
 DAL_LIBS = amdgpu_dm dc	modules/freesync modules/color modules/info_packet modules/power dmub/src
 
-ifdef CONFIG_DRM_AMD_DC_HDCP
 DAL_LIBS += modules/hdcp
-endif
 
 AMD_DAL = $(addsuffix /Makefile, $(addprefix $(FULL_AMD_DISPLAY_PATH)/,$(DAL_LIBS)))
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile b/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile
index 90fb0f3cdb6f..249b073f6a23 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile
@@ -33,7 +33,7 @@ AMDGPUDM = \
 	amdgpu_dm_mst_types.o \
 	amdgpu_dm_color.o
 
-ifdef CONFIG_DRM_AMD_DC_DCN
+ifdef CONFIG_DRM_AMD_DC_FP
 AMDGPUDM += dc_fpu.o
 endif
 
@@ -41,9 +41,7 @@ ifneq ($(CONFIG_DRM_AMD_DC),)
 AMDGPUDM += amdgpu_dm_services.o amdgpu_dm_helpers.o amdgpu_dm_pp_smu.o amdgpu_dm_psr.o
 endif
 
-ifdef CONFIG_DRM_AMD_DC_HDCP
 AMDGPUDM += amdgpu_dm_hdcp.o
-endif
 
 ifneq ($(CONFIG_DEBUG_FS),)
 AMDGPUDM += amdgpu_dm_crc.o amdgpu_dm_debugfs.o
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index a01fd41643fc..c4aa87086a4e 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -52,10 +52,8 @@
 #include "amdgpu_dm.h"
 #include "amdgpu_dm_plane.h"
 #include "amdgpu_dm_crtc.h"
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 #include "amdgpu_dm_hdcp.h"
 #include <drm/display/drm_hdcp_helper.h>
-#endif
 #include "amdgpu_pm.h"
 #include "amdgpu_atombios.h"
 
@@ -344,12 +342,52 @@ static inline bool is_dc_timing_adjust_needed(struct dm_crtc_state *old_state,
 {
 	if (new_state->freesync_config.state ==  VRR_STATE_ACTIVE_FIXED)
 		return true;
-	else if (amdgpu_dm_vrr_active(old_state) != amdgpu_dm_vrr_active(new_state))
+	else if (amdgpu_dm_crtc_vrr_active(old_state) != amdgpu_dm_crtc_vrr_active(new_state))
 		return true;
 	else
 		return false;
 }
 
+static inline void reverse_planes_order(struct dc_surface_update *array_of_surface_update,
+					int planes_count)
+{
+	int i, j;
+
+	for (i = 0, j = planes_count - 1; i < j; i++, j--)
+		swap(array_of_surface_update[i], array_of_surface_update[j]);
+}
+
+/**
+ * update_planes_and_stream_adapter() - Send planes to be updated in DC
+ *
+ * DC has a generic way to update planes and stream via
+ * dc_update_planes_and_stream function; however, DM might need some
+ * adjustments and preparation before calling it. This function is a wrapper
+ * for the dc_update_planes_and_stream that does any required configuration
+ * before passing control to DC.
+ */
+static inline bool update_planes_and_stream_adapter(struct dc *dc,
+						    int update_type,
+						    int planes_count,
+						    struct dc_stream_state *stream,
+						    struct dc_stream_update *stream_update,
+						    struct dc_surface_update *array_of_surface_update)
+{
+	reverse_planes_order(array_of_surface_update, planes_count);
+
+	/*
+	 * Previous frame finished and HW is ready for optimization.
+	 */
+	if (update_type == UPDATE_TYPE_FAST)
+		dc_post_update_surfaces_to_stream(dc);
+
+	return dc_update_planes_and_stream(dc,
+					   array_of_surface_update,
+					   planes_count,
+					   stream,
+					   stream_update);
+}
+
 /**
  * dm_pflip_high_irq() - Handle pageflip interrupt
  * @interrupt_params: ignored
@@ -394,7 +432,7 @@ static void dm_pflip_high_irq(void *interrupt_params)
 
 	WARN_ON(!e);
 
-	vrr_active = amdgpu_dm_vrr_active_irq(amdgpu_crtc);
+	vrr_active = amdgpu_dm_crtc_vrr_active_irq(amdgpu_crtc);
 
 	/* Fixed refresh rate, or VRR scanout position outside front-porch? */
 	if (!vrr_active ||
@@ -468,7 +506,7 @@ static void dm_vupdate_high_irq(void *interrupt_params)
 	acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE);
 
 	if (acrtc) {
-		vrr_active = amdgpu_dm_vrr_active_irq(acrtc);
+		vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
 		drm_dev = acrtc->base.dev;
 		vblank = &drm_dev->vblank[acrtc->base.index];
 		previous_timestamp = atomic64_read(&irq_params->previous_timestamp);
@@ -492,7 +530,7 @@ static void dm_vupdate_high_irq(void *interrupt_params)
 		 * if a pageflip happened inside front-porch.
 		 */
 		if (vrr_active) {
-			dm_crtc_handle_vblank(acrtc);
+			amdgpu_dm_crtc_handle_vblank(acrtc);
 
 			/* BTR processing for pre-DCE12 ASICs */
 			if (acrtc->dm_irq_params.stream &&
@@ -532,7 +570,7 @@ static void dm_crtc_high_irq(void *interrupt_params)
 	if (!acrtc)
 		return;
 
-	vrr_active = amdgpu_dm_vrr_active_irq(acrtc);
+	vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
 
 	DC_LOG_VBLANK("crtc:%d, vupdate-vrr:%d, planes:%d\n", acrtc->crtc_id,
 		      vrr_active, acrtc->dm_irq_params.active_planes);
@@ -544,7 +582,7 @@ static void dm_crtc_high_irq(void *interrupt_params)
 	 * to dm_vupdate_high_irq after end of front-porch.
 	 */
 	if (!vrr_active)
-		dm_crtc_handle_vblank(acrtc);
+		amdgpu_dm_crtc_handle_vblank(acrtc);
 
 	/**
 	 * Following stuff must happen at start of vblank, for crc
@@ -675,7 +713,14 @@ static void dmub_hpd_callback(struct amdgpu_device *adev,
 	drm_for_each_connector_iter(connector, &iter) {
 		aconnector = to_amdgpu_dm_connector(connector);
 		if (link && aconnector->dc_link == link) {
-			DRM_INFO("DMUB HPD callback: link_index=%u\n", link_index);
+			if (notify->type == DMUB_NOTIFICATION_HPD)
+				DRM_INFO("DMUB HPD callback: link_index=%u\n", link_index);
+			else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ)
+				DRM_INFO("DMUB HPD IRQ callback: link_index=%u\n", link_index);
+			else
+				DRM_WARN("DMUB Unknown HPD callback type %d, link_index=%u\n",
+						notify->type, link_index);
+
 			hpd_aconnector = aconnector;
 			break;
 		}
@@ -775,15 +820,14 @@ static void dm_dmub_outbox1_low_irq(void *interrupt_params)
 					DRM_ERROR("Failed to allocate dmub_hpd_wrk");
 					return;
 				}
-				dmub_hpd_wrk->dmub_notify = kzalloc(sizeof(struct dmub_notification), GFP_ATOMIC);
+				dmub_hpd_wrk->dmub_notify = kmemdup(&notify, sizeof(struct dmub_notification),
+								    GFP_ATOMIC);
 				if (!dmub_hpd_wrk->dmub_notify) {
 					kfree(dmub_hpd_wrk);
 					DRM_ERROR("Failed to allocate dmub_hpd_wrk->dmub_notify");
 					return;
 				}
 				INIT_WORK(&dmub_hpd_wrk->handle_hpd_work, dm_handle_hpd_work);
-				if (dmub_hpd_wrk->dmub_notify)
-					memcpy(dmub_hpd_wrk->dmub_notify, &notify, sizeof(struct dmub_notification));
 				dmub_hpd_wrk->adev = adev;
 				if (notify.type == DMUB_NOTIFICATION_HPD) {
 					plink = adev->dm.dc->links[notify.link_index];
@@ -1488,9 +1532,7 @@ static void retrieve_dmi_info(struct amdgpu_display_manager *dm)
 static int amdgpu_dm_init(struct amdgpu_device *adev)
 {
 	struct dc_init_data init_data;
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	struct dc_callback_init init_params;
-#endif
 	int r;
 
 	adev->dm.ddev = adev_to_drm(adev);
@@ -1498,9 +1540,7 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 
 	/* Zero all the fields */
 	memset(&init_data, 0, sizeof(init_data));
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	memset(&init_params, 0, sizeof(init_params));
-#endif
 
 	mutex_init(&adev->dm.dpia_aux_lock);
 	mutex_init(&adev->dm.dc_lock);
@@ -1726,7 +1766,6 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 			DRM_ERROR("amdgpu: failed to initialize vblank_workqueue.\n");
 	}
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	if (adev->dm.dc->caps.max_links > 0 && adev->family >= AMDGPU_FAMILY_RV) {
 		adev->dm.hdcp_workqueue = hdcp_create_workqueue(adev, &init_params.cp_psp, adev->dm.dc);
 
@@ -1737,7 +1776,6 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 
 		dc_init_callbacks(adev->dm.dc, &init_params);
 	}
-#endif
 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
 	adev->dm.secure_display_ctxs = amdgpu_dm_crtc_secure_display_create_contexts(adev);
 	if (!adev->dm.secure_display_ctxs) {
@@ -1844,7 +1882,6 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev)
 		adev->dm.secure_display_ctxs = NULL;
 	}
 #endif
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	if (adev->dm.hdcp_workqueue) {
 		hdcp_destroy(&adev->dev->kobj, adev->dm.hdcp_workqueue);
 		adev->dm.hdcp_workqueue = NULL;
@@ -1852,9 +1889,9 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev)
 
 	if (adev->dm.dc)
 		dc_deinit_callbacks(adev->dm.dc);
-#endif
 
-	dc_dmub_srv_destroy(&adev->dm.dc->ctx->dmub_srv);
+	if (adev->dm.dc)
+		dc_dmub_srv_destroy(&adev->dm.dc->ctx->dmub_srv);
 
 	if (dc_enable_dmub_notifications(adev->dm.dc)) {
 		kfree(adev->dm.dmub_notify);
@@ -2263,9 +2300,9 @@ static int dm_late_init(void *handle)
 	 */
 	params.min_abm_backlight = 0x28F;
 	/* In the case where abm is implemented on dmcub,
-	* dmcu object will be null.
-	* ABM 2.4 and up are implemented on dmcub.
-	*/
+	 * dmcu object will be null.
+	 * ABM 2.4 and up are implemented on dmcub.
+	 */
 	if (dmcu) {
 		if (!dmcu_load_iram(dmcu, params))
 			return -EINVAL;
@@ -2273,7 +2310,7 @@ static int dm_late_init(void *handle)
 		struct dc_link *edp_links[MAX_NUM_EDP];
 		int edp_num;
 
-		get_edp_links(adev->dm.dc, edp_links, &edp_num);
+		dc_get_edp_links(adev->dm.dc, edp_links, &edp_num);
 		for (i = 0; i < edp_num; i++) {
 			if (!dmub_init_abm_config(adev->dm.dc->res_pool, params, i))
 				return -EINVAL;
@@ -2449,11 +2486,11 @@ static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev,
 					 enable ? "enable" : "disable");
 
 			if (enable) {
-				rc = dm_enable_vblank(&acrtc->base);
+				rc = amdgpu_dm_crtc_enable_vblank(&acrtc->base);
 				if (rc)
 					DRM_WARN("Failed to enable vblank interrupts\n");
 			} else {
-				dm_disable_vblank(&acrtc->base);
+				amdgpu_dm_crtc_disable_vblank(&acrtc->base);
 			}
 
 		}
@@ -2496,7 +2533,7 @@ static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc)
 			goto fail;
 	}
 
-	res = dc_commit_state(dc, context);
+	res = dc_commit_streams(dc, context->streams, context->stream_count);
 
 fail:
 	dc_release_state(context);
@@ -2682,10 +2719,13 @@ static void dm_gpureset_commit_state(struct dc_state *dc_state,
 			bundle->surface_updates[m].surface->force_full_update =
 				true;
 		}
-		dc_commit_updates_for_stream(
-			dm->dc, bundle->surface_updates,
-			dc_state->stream_status->plane_count,
-			dc_state->streams[k], &bundle->stream_update, dc_state);
+
+		update_planes_and_stream_adapter(dm->dc,
+					 UPDATE_TYPE_FULL,
+					 dc_state->stream_status->plane_count,
+					 dc_state->streams[k],
+					 &bundle->stream_update,
+					 bundle->surface_updates);
 	}
 
 cleanup:
@@ -2755,7 +2795,7 @@ static int dm_resume(void *handle)
 			dc_enable_dmub_outbox(adev->dm.dc);
 		}
 
-		WARN_ON(!dc_commit_state(dm->dc, dc_state));
+		WARN_ON(!dc_commit_streams(dm->dc, dc_state->streams, dc_state->stream_count));
 
 		dm_gpureset_commit_state(dm->cached_dc_state, dm);
 
@@ -2923,7 +2963,7 @@ const struct amdgpu_ip_block_version dm_ip_block =
 
 static const struct drm_mode_config_funcs amdgpu_dm_mode_funcs = {
 	.fb_create = amdgpu_display_user_framebuffer_create,
-	.get_format_info = amd_get_format_info,
+	.get_format_info = amdgpu_dm_plane_get_format_info,
 	.atomic_check = amdgpu_dm_atomic_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -2936,30 +2976,18 @@ static struct drm_mode_config_helper_funcs amdgpu_dm_mode_config_helperfuncs = {
 static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector)
 {
 	struct amdgpu_dm_backlight_caps *caps;
-	struct amdgpu_display_manager *dm;
 	struct drm_connector *conn_base;
 	struct amdgpu_device *adev;
-	struct dc_link *link = NULL;
 	struct drm_luminance_range_info *luminance_range;
-	int i;
 
-	if (!aconnector || !aconnector->dc_link)
-		return;
-
-	link = aconnector->dc_link;
-	if (link->connector_signal != SIGNAL_TYPE_EDP)
+	if (aconnector->bl_idx == -1 ||
+	    aconnector->dc_link->connector_signal != SIGNAL_TYPE_EDP)
 		return;
 
 	conn_base = &aconnector->base;
 	adev = drm_to_adev(conn_base->dev);
-	dm = &adev->dm;
-	for (i = 0; i < dm->num_of_edps; i++) {
-		if (link == dm->backlight_link[i])
-			break;
-	}
-	if (i >= dm->num_of_edps)
-		return;
-	caps = &dm->backlight_caps[i];
+
+	caps = &adev->dm.backlight_caps[aconnector->bl_idx];
 	caps->ext_caps = &aconnector->dc_link->dpcd_sink_ext_caps;
 	caps->aux_support = false;
 
@@ -2974,8 +3002,14 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector)
 		caps->aux_support = true;
 
 	luminance_range = &conn_base->display_info.luminance_range;
-	caps->aux_min_input_signal = luminance_range->min_luminance;
-	caps->aux_max_input_signal = luminance_range->max_luminance;
+
+	if (luminance_range->max_luminance) {
+		caps->aux_min_input_signal = luminance_range->min_luminance;
+		caps->aux_max_input_signal = luminance_range->max_luminance;
+	} else {
+		caps->aux_min_input_signal = 0;
+		caps->aux_max_input_signal = 512;
+	}
 }
 
 void amdgpu_dm_update_connector_after_detect(
@@ -3111,11 +3145,9 @@ void amdgpu_dm_update_connector_after_detect(
 		aconnector->edid = NULL;
 		kfree(aconnector->timing_requested);
 		aconnector->timing_requested = NULL;
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 		/* Set CP to DESIRED if it was ENABLED, so we can re-enable it again on hotplug */
 		if (connector->state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED)
 			connector->state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED;
-#endif
 	}
 
 	mutex_unlock(&dev->mode_config.mutex);
@@ -3132,9 +3164,7 @@ static void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector)
 	struct drm_device *dev = connector->dev;
 	enum dc_connection_type new_connection_type = dc_connection_none;
 	struct amdgpu_device *adev = drm_to_adev(dev);
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state);
-#endif
 	bool ret = false;
 
 	if (adev->dm.disable_hpd_irq)
@@ -3146,12 +3176,10 @@ static void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector)
 	 */
 	mutex_lock(&aconnector->hpd_lock);
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	if (adev->dm.hdcp_workqueue) {
 		hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index);
 		dm_con_state->update_hdcp = true;
 	}
-#endif
 	if (aconnector->fake_enable)
 		aconnector->fake_enable = false;
 
@@ -3398,12 +3426,10 @@ out:
 			}
 		}
 	}
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	if (hpd_irq_data.bytes.device_service_irq.bits.CP_IRQ) {
 		if (adev->dm.hdcp_workqueue)
 			hdcp_handle_cpirq(adev->dm.hdcp_workqueue,  aconnector->base.index);
 	}
-#endif
 
 	if (dc_link->type != dc_connection_mst_branch)
 		drm_dp_cec_irq(&aconnector->dm_dp_aux.aux);
@@ -4152,16 +4178,18 @@ static const struct backlight_ops amdgpu_dm_backlight_ops = {
 };
 
 static void
-amdgpu_dm_register_backlight_device(struct amdgpu_display_manager *dm)
+amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector)
 {
-	char bl_name[16];
+	struct drm_device *drm = aconnector->base.dev;
+	struct amdgpu_display_manager *dm = &drm_to_adev(drm)->dm;
 	struct backlight_properties props = { 0 };
+	char bl_name[16];
 
-	amdgpu_dm_update_backlight_caps(dm, dm->num_of_edps);
-	dm->brightness[dm->num_of_edps] = AMDGPU_MAX_BL_LEVEL;
+	if (aconnector->bl_idx == -1)
+		return;
 
 	if (!acpi_video_backlight_use_native()) {
-		drm_info(adev_to_drm(dm->adev), "Skipping amdgpu DM backlight registration\n");
+		drm_info(drm, "Skipping amdgpu DM backlight registration\n");
 		/* Try registering an ACPI video backlight device instead. */
 		acpi_video_register_backlight();
 		return;
@@ -4172,17 +4200,16 @@ amdgpu_dm_register_backlight_device(struct amdgpu_display_manager *dm)
 	props.type = BACKLIGHT_RAW;
 
 	snprintf(bl_name, sizeof(bl_name), "amdgpu_bl%d",
-		 adev_to_drm(dm->adev)->primary->index + dm->num_of_edps);
+		 drm->primary->index + aconnector->bl_idx);
 
-	dm->backlight_dev[dm->num_of_edps] = backlight_device_register(bl_name,
-								       adev_to_drm(dm->adev)->dev,
-								       dm,
-								       &amdgpu_dm_backlight_ops,
-								       &props);
+	dm->backlight_dev[aconnector->bl_idx] =
+		backlight_device_register(bl_name, aconnector->base.kdev, dm,
+					  &amdgpu_dm_backlight_ops, &props);
 
-	if (IS_ERR(dm->backlight_dev[dm->num_of_edps]))
+	if (IS_ERR(dm->backlight_dev[aconnector->bl_idx])) {
 		DRM_ERROR("DM: Backlight registration failed!\n");
-	else
+		dm->backlight_dev[aconnector->bl_idx] = NULL;
+	} else
 		DRM_DEBUG_DRIVER("DM: Registered Backlight device: %s\n", bl_name);
 }
 
@@ -4227,24 +4254,29 @@ static int initialize_plane(struct amdgpu_display_manager *dm,
 }
 
 
-static void register_backlight_device(struct amdgpu_display_manager *dm,
-				      struct dc_link *link)
+static void setup_backlight_device(struct amdgpu_display_manager *dm,
+				   struct amdgpu_dm_connector *aconnector)
 {
-	if ((link->connector_signal & (SIGNAL_TYPE_EDP | SIGNAL_TYPE_LVDS)) &&
-	    link->type != dc_connection_none) {
-		/*
-		 * Event if registration failed, we should continue with
-		 * DM initialization because not having a backlight control
-		 * is better then a black screen.
-		 */
-		if (!dm->backlight_dev[dm->num_of_edps])
-			amdgpu_dm_register_backlight_device(dm);
+	struct dc_link *link = aconnector->dc_link;
+	int bl_idx = dm->num_of_edps;
 
-		if (dm->backlight_dev[dm->num_of_edps]) {
-			dm->backlight_link[dm->num_of_edps] = link;
-			dm->num_of_edps++;
-		}
+	if (!(link->connector_signal & (SIGNAL_TYPE_EDP | SIGNAL_TYPE_LVDS)) ||
+	    link->type == dc_connection_none)
+		return;
+
+	if (dm->num_of_edps >= AMDGPU_DM_MAX_NUM_EDP) {
+		drm_warn(adev_to_drm(dm->adev), "Too much eDP connections, skipping backlight setup for additional eDPs\n");
+		return;
 	}
+
+	aconnector->bl_idx = bl_idx;
+
+	amdgpu_dm_update_backlight_caps(dm, bl_idx);
+	dm->brightness[bl_idx] = AMDGPU_MAX_BL_LEVEL;
+	dm->backlight_link[bl_idx] = link;
+	dm->num_of_edps++;
+
+	update_connector_ext_caps(aconnector);
 }
 
 static void amdgpu_set_panel_orientation(struct drm_connector *connector);
@@ -4320,9 +4352,6 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 		if (plane->type != DC_PLANE_TYPE_DCN_UNIVERSAL)
 			continue;
 
-		if (!plane->blends_with_above || !plane->blends_with_below)
-			continue;
-
 		if (!plane->pixel_format_support.argb8888)
 			continue;
 
@@ -4427,10 +4456,7 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 
 			if (ret) {
 				amdgpu_dm_update_connector_after_detect(aconnector);
-				register_backlight_device(dm, link);
-
-				if (dm->num_of_edps)
-					update_connector_ext_caps(aconnector);
+				setup_backlight_device(dm, aconnector);
 
 				if (psr_feature_enabled)
 					amdgpu_dm_set_psr_caps(link);
@@ -4445,10 +4471,6 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 		amdgpu_set_panel_orientation(&aconnector->base);
 	}
 
-	/* If we didn't find a panel, notify the acpi video detection */
-	if (dm->adev->flags & AMD_IS_APU && dm->num_of_edps == 0)
-		acpi_video_report_nolcd();
-
 	/* Software is initialized. Now we can register interrupt handlers. */
 	switch (adev->asic_type) {
 #if defined(CONFIG_DRM_AMD_DC_SI)
@@ -4947,7 +4969,7 @@ fill_dc_plane_info_and_addr(struct amdgpu_device *adev,
 	if (ret)
 		return ret;
 
-	ret = fill_plane_buffer_attributes(adev, afb, plane_info->format,
+	ret = amdgpu_dm_plane_fill_plane_buffer_attributes(adev, afb, plane_info->format,
 					   plane_info->rotation, tiling_flags,
 					   &plane_info->tiling_info,
 					   &plane_info->plane_size,
@@ -4956,7 +4978,7 @@ fill_dc_plane_info_and_addr(struct amdgpu_device *adev,
 	if (ret)
 		return ret;
 
-	fill_blending_from_plane_state(
+	amdgpu_dm_plane_fill_blending_from_plane_state(
 		plane_state, &plane_info->per_pixel_alpha, &plane_info->pre_multiplied_alpha,
 		&plane_info->global_alpha, &plane_info->global_alpha_value);
 
@@ -4975,7 +4997,7 @@ static int fill_dc_plane_attributes(struct amdgpu_device *adev,
 	int ret;
 	bool force_disable_dcc = false;
 
-	ret = fill_dc_scaling_info(adev, plane_state, &scaling_info);
+	ret = amdgpu_dm_plane_fill_dc_scaling_info(adev, plane_state, &scaling_info);
 	if (ret)
 		return ret;
 
@@ -5753,7 +5775,6 @@ static bool is_freesync_video_mode(const struct drm_display_mode *mode,
 		return true;
 }
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 static void update_dsc_caps(struct amdgpu_dm_connector *aconnector,
 			    struct dc_sink *sink, struct dc_stream_state *stream,
 			    struct dsc_dec_dpcd_caps *dsc_caps)
@@ -5784,6 +5805,10 @@ static void apply_dsc_policy_for_edp(struct amdgpu_dm_connector *aconnector,
 	struct dc *dc = sink->ctx->dc;
 	struct dc_dsc_bw_range bw_range = {0};
 	struct dc_dsc_config dsc_cfg = {0};
+	struct dc_dsc_config_options dsc_options = {0};
+
+	dc_dsc_get_default_config_option(dc, &dsc_options);
+	dsc_options.max_target_bpp_limit_override_x16 = max_dsc_target_bpp_limit_override * 16;
 
 	verified_link_cap = dc_link_get_link_cap(stream->link);
 	link_bw_in_kbps = dc_link_bandwidth_kbps(stream->link, verified_link_cap);
@@ -5806,8 +5831,7 @@ static void apply_dsc_policy_for_edp(struct amdgpu_dm_connector *aconnector,
 		if (bw_range.max_kbps < link_bw_in_kbps) {
 			if (dc_dsc_compute_config(dc->res_pool->dscs[0],
 					dsc_caps,
-					dc->debug.dsc_min_slice_height_override,
-					max_dsc_target_bpp_limit_override,
+					&dsc_options,
 					0,
 					&stream->timing,
 					&dsc_cfg)) {
@@ -5821,8 +5845,7 @@ static void apply_dsc_policy_for_edp(struct amdgpu_dm_connector *aconnector,
 
 	if (dc_dsc_compute_config(dc->res_pool->dscs[0],
 				dsc_caps,
-				dc->debug.dsc_min_slice_height_override,
-				max_dsc_target_bpp_limit_override,
+				&dsc_options,
 				link_bw_in_kbps,
 				&stream->timing,
 				&dsc_cfg)) {
@@ -5843,6 +5866,10 @@ static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
 	u32 dsc_max_supported_bw_in_kbps;
 	u32 max_dsc_target_bpp_limit_override =
 		drm_connector->display_info.max_dsc_bpp;
+	struct dc_dsc_config_options dsc_options = {0};
+
+	dc_dsc_get_default_config_option(dc, &dsc_options);
+	dsc_options.max_target_bpp_limit_override_x16 = max_dsc_target_bpp_limit_override * 16;
 
 	link_bandwidth_kbps = dc_link_bandwidth_kbps(aconnector->dc_link,
 							dc_link_get_link_cap(aconnector->dc_link));
@@ -5861,8 +5888,7 @@ static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
 		if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_NONE) {
 			if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0],
 						dsc_caps,
-						aconnector->dc_link->ctx->dc->debug.dsc_min_slice_height_override,
-						max_dsc_target_bpp_limit_override,
+						&dsc_options,
 						link_bandwidth_kbps,
 						&stream->timing,
 						&stream->timing.dsc_cfg)) {
@@ -5879,8 +5905,7 @@ static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
 					dsc_max_supported_bw_in_kbps > 0)
 				if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0],
 						dsc_caps,
-						aconnector->dc_link->ctx->dc->debug.dsc_min_slice_height_override,
-						max_dsc_target_bpp_limit_override,
+						&dsc_options,
 						dsc_max_supported_bw_in_kbps,
 						&stream->timing,
 						&stream->timing.dsc_cfg)) {
@@ -5904,7 +5929,6 @@ static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
 	if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_bits_per_pixel)
 		stream->timing.dsc_cfg.bits_per_pixel = aconnector->dsc_settings.dsc_bits_per_pixel;
 }
-#endif /* CONFIG_DRM_AMD_DC_DCN */
 
 static struct dc_stream_state *
 create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
@@ -5927,9 +5951,7 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
 	int mode_refresh;
 	int preferred_refresh = 0;
 	enum color_transfer_func tf = TRANSFER_FUNC_UNKNOWN;
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	struct dsc_dec_dpcd_caps dsc_caps;
-#endif
 
 	struct dc_sink *sink = NULL;
 
@@ -6028,12 +6050,10 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
 		stream->timing = *aconnector->timing_requested;
 	}
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	/* SST DSC determination policy */
 	update_dsc_caps(aconnector, sink, stream, &dsc_caps);
 	if (aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE && dsc_caps.is_dsc_supported)
 		apply_dsc_policy_for_stream(aconnector, sink, stream, &dsc_caps);
-#endif
 
 	update_stream_scaling_settings(&mode, dm_state, stream);
 
@@ -6208,10 +6228,8 @@ static void amdgpu_dm_connector_unregister(struct drm_connector *connector)
 static void amdgpu_dm_connector_destroy(struct drm_connector *connector)
 {
 	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
-	const struct dc_link *link = aconnector->dc_link;
 	struct amdgpu_device *adev = drm_to_adev(connector->dev);
 	struct amdgpu_display_manager *dm = &adev->dm;
-	int i;
 
 	/*
 	 * Call only if mst_mgr was initialized before since it's not done
@@ -6220,11 +6238,9 @@ static void amdgpu_dm_connector_destroy(struct drm_connector *connector)
 	if (aconnector->mst_mgr.dev)
 		drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr);
 
-	for (i = 0; i < dm->num_of_edps; i++) {
-		if ((link == dm->backlight_link[i]) && dm->backlight_dev[i]) {
-			backlight_device_unregister(dm->backlight_dev[i]);
-			dm->backlight_dev[i] = NULL;
-		}
+	if (aconnector->bl_idx != -1) {
+		backlight_device_unregister(dm->backlight_dev[aconnector->bl_idx]);
+		dm->backlight_dev[aconnector->bl_idx] = NULL;
 	}
 
 	if (aconnector->dc_em_sink)
@@ -6306,6 +6322,8 @@ amdgpu_dm_connector_late_register(struct drm_connector *connector)
 		to_amdgpu_dm_connector(connector);
 	int r;
 
+	amdgpu_dm_register_backlight_device(amdgpu_dm_connector);
+
 	if ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) ||
 	    (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) {
 		amdgpu_dm_connector->dm_dp_aux.aux.dev = connector->kdev;
@@ -6759,7 +6777,6 @@ const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs = {
 	.atomic_check = dm_encoder_helper_atomic_check
 };
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_state *state,
 					    struct dc_state *dc_state,
 					    struct dsc_mst_fairness_vars *vars)
@@ -6833,7 +6850,6 @@ static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_state *state,
 	}
 	return 0;
 }
-#endif
 
 static int to_drm_connector_type(enum signal_type st)
 {
@@ -7072,13 +7088,13 @@ static uint add_fs_modes(struct amdgpu_dm_connector *aconnector)
 	/* Standard FPS values
 	 *
 	 * 23.976       - TV/NTSC
-	 * 24 	        - Cinema
-	 * 25 	        - TV/PAL
+	 * 24           - Cinema
+	 * 25           - TV/PAL
 	 * 29.97        - TV/NTSC
-	 * 30 	        - TV/NTSC
-	 * 48 	        - Cinema HFR
-	 * 50 	        - TV/PAL
-	 * 60 	        - Commonly used
+	 * 30           - TV/NTSC
+	 * 48           - Cinema HFR
+	 * 50           - TV/PAL
+	 * 60           - Commonly used
 	 * 48,72,96,120 - Multiples of 24
 	 */
 	static const u32 common_rates[] = {
@@ -7158,12 +7174,18 @@ static int amdgpu_dm_connector_get_modes(struct drm_connector *connector)
 			to_amdgpu_dm_connector(connector);
 	struct drm_encoder *encoder;
 	struct edid *edid = amdgpu_dm_connector->edid;
+	struct dc_link_settings *verified_link_cap =
+			&amdgpu_dm_connector->dc_link->verified_link_cap;
+	const struct dc *dc = amdgpu_dm_connector->dc_link->dc;
 
 	encoder = amdgpu_dm_connector_to_encoder(connector);
 
 	if (!drm_edid_is_valid(edid)) {
 		amdgpu_dm_connector->num_modes =
 				drm_add_modes_noedid(connector, 640, 480);
+		if (dc->link_srv->dp_get_encoding_format(verified_link_cap) == DP_128b_132b_ENCODING)
+			amdgpu_dm_connector->num_modes +=
+				drm_add_modes_noedid(connector, 1920, 1080);
 	} else {
 		amdgpu_dm_connector_ddc_get_modes(connector, edid);
 		amdgpu_dm_connector_add_common_modes(encoder, connector);
@@ -7190,6 +7212,7 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
 		aconnector->base.funcs->reset(&aconnector->base);
 
 	aconnector->connector_id = link_index;
+	aconnector->bl_idx = -1;
 	aconnector->dc_link = link;
 	aconnector->base.interlace_allowed = false;
 	aconnector->base.doublescan_allowed = false;
@@ -7261,10 +7284,8 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
 		if (!aconnector->mst_root)
 			drm_connector_attach_vrr_capable_property(&aconnector->base);
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 		if (adev->dm.hdcp_workqueue)
 			drm_connector_attach_content_protection_property(&aconnector->base, true);
-#endif
 	}
 }
 
@@ -7526,7 +7547,6 @@ is_scaling_state_different(const struct dm_connector_state *dm_state,
 	return false;
 }
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 static bool is_content_protection_different(struct drm_crtc_state *new_crtc_state,
 					    struct drm_crtc_state *old_crtc_state,
 					    struct drm_connector_state *new_conn_state,
@@ -7646,7 +7666,6 @@ static bool is_content_protection_different(struct drm_crtc_state *new_crtc_stat
 	pr_debug("[HDCP_DM] DESIRED->ENABLED %s :false\n", __func__);
 	return false;
 }
-#endif
 
 static void remove_stream(struct amdgpu_device *adev,
 			  struct amdgpu_crtc *acrtc,
@@ -7704,7 +7723,7 @@ static void update_freesync_state_on_stream(
 		return;
 
 	spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
-        vrr_params = acrtc->dm_irq_params.vrr_params;
+	vrr_params = acrtc->dm_irq_params.vrr_params;
 
 	if (surface) {
 		mod_freesync_handle_preflip(
@@ -7715,7 +7734,7 @@ static void update_freesync_state_on_stream(
 			&vrr_params);
 
 		if (adev->family < AMDGPU_FAMILY_AI &&
-		    amdgpu_dm_vrr_active(new_crtc_state)) {
+		    amdgpu_dm_crtc_vrr_active(new_crtc_state)) {
 			mod_freesync_handle_v_update(dm->freesync_module,
 						     new_stream, &vrr_params);
 
@@ -7833,8 +7852,8 @@ static void update_stream_irq_parameters(
 static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state,
 					    struct dm_crtc_state *new_state)
 {
-	bool old_vrr_active = amdgpu_dm_vrr_active(old_state);
-	bool new_vrr_active = amdgpu_dm_vrr_active(new_state);
+	bool old_vrr_active = amdgpu_dm_crtc_vrr_active(old_state);
+	bool new_vrr_active = amdgpu_dm_crtc_vrr_active(new_state);
 
 	if (!old_vrr_active && new_vrr_active) {
 		/* Transition VRR inactive -> active:
@@ -7845,7 +7864,7 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state,
 		 * We also need vupdate irq for the actual core vblank handling
 		 * at end of vblank.
 		 */
-		WARN_ON(dm_set_vupdate_irq(new_state->base.crtc, true) != 0);
+		WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0);
 		WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0);
 		DRM_DEBUG_DRIVER("%s: crtc=%u VRR off->on: Get vblank ref\n",
 				 __func__, new_state->base.crtc->base.id);
@@ -7853,7 +7872,7 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state,
 		/* Transition VRR active -> inactive:
 		 * Allow vblank irq disable again for fixed refresh rate.
 		 */
-		WARN_ON(dm_set_vupdate_irq(new_state->base.crtc, false) != 0);
+		WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0);
 		drm_crtc_vblank_put(new_state->base.crtc);
 		DRM_DEBUG_DRIVER("%s: crtc=%u VRR on->off: Drop vblank ref\n",
 				 __func__, new_state->base.crtc->base.id);
@@ -7872,7 +7891,7 @@ static void amdgpu_dm_commit_cursors(struct drm_atomic_state *state)
 	 */
 	for_each_old_plane_in_state(state, plane, old_plane_state, i)
 		if (plane->type == DRM_PLANE_TYPE_CURSOR)
-			handle_cursor_update(plane, old_plane_state);
+			amdgpu_dm_plane_handle_cursor_update(plane, old_plane_state);
 }
 
 static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
@@ -7895,7 +7914,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 	int planes_count = 0, vpos, hpos;
 	unsigned long flags;
 	u32 target_vblank, last_flip_vblank;
-	bool vrr_active = amdgpu_dm_vrr_active(acrtc_state);
+	bool vrr_active = amdgpu_dm_crtc_vrr_active(acrtc_state);
 	bool cursor_update = false;
 	bool pflip_present = false;
 	bool dirty_rects_changed = false;
@@ -7957,7 +7976,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 			bundle->surface_updates[planes_count].gamut_remap_matrix = &dc_plane->gamut_remap_matrix;
 		}
 
-		fill_dc_scaling_info(dm->adev, new_plane_state,
+		amdgpu_dm_plane_fill_dc_scaling_info(dm->adev, new_plane_state,
 				     &bundle->scaling_infos[planes_count]);
 
 		bundle->surface_updates[planes_count].scaling_info =
@@ -8177,12 +8196,12 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 				acrtc_state->stream->link->psr_settings.psr_allow_active)
 			amdgpu_dm_psr_disable(acrtc_state->stream);
 
-		dc_commit_updates_for_stream(dm->dc,
-						     bundle->surface_updates,
-						     planes_count,
-						     acrtc_state->stream,
-						     &bundle->stream_update,
-						     dc_state);
+		update_planes_and_stream_adapter(dm->dc,
+					 acrtc_state->update_type,
+					 planes_count,
+					 acrtc_state->stream,
+					 &bundle->stream_update,
+					 bundle->surface_updates);
 
 		/**
 		 * Enable or disable the interrupts on the backend.
@@ -8285,7 +8304,7 @@ static void amdgpu_dm_commit_audio(struct drm_device *dev,
 		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
 			continue;
 
-	notify:
+notify:
 		aconnector = to_amdgpu_dm_connector(connector);
 
 		mutex_lock(&adev->dm.audio_lock);
@@ -8445,7 +8464,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 		 * aconnector as needed
 		 */
 
-		if (modeset_required(new_crtc_state, dm_new_crtc_state->stream, dm_old_crtc_state->stream)) {
+		if (amdgpu_dm_crtc_modeset_required(new_crtc_state, dm_new_crtc_state->stream, dm_old_crtc_state->stream)) {
 
 			DRM_DEBUG_ATOMIC("Atomic commit: SET crtc id %d: [%p]\n", acrtc->crtc_id, acrtc);
 
@@ -8500,7 +8519,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 
 		dm_enable_per_frame_crtc_master_sync(dc_state);
 		mutex_lock(&dm->dc_lock);
-		WARN_ON(!dc_commit_state(dm->dc, dc_state));
+		WARN_ON(!dc_commit_streams(dm->dc, dc_state->streams, dc_state->stream_count));
 
 		/* Allow idle optimization when vblank count is 0 for display off */
 		if (dm->active_vblank_irq_count == 0)
@@ -8526,7 +8545,6 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 				acrtc->otg_inst = status->primary_otg_inst;
 		}
 	}
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) {
 		struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state);
 		struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc);
@@ -8637,7 +8655,6 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 				new_con_state->hdcp_content_type, enable_encryption);
 		}
 	}
-#endif
 
 	/* Handle connector state changes */
 	for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) {
@@ -8714,12 +8731,11 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 
 
 		mutex_lock(&dm->dc_lock);
-		dc_commit_updates_for_stream(dm->dc,
-						     dummy_updates,
-						     status->plane_count,
-						     dm_new_crtc_state->stream,
-						     &stream_update,
-						     dc_state);
+		dc_update_planes_and_stream(dm->dc,
+					    dummy_updates,
+					    status->plane_count,
+					    dm_new_crtc_state->stream,
+					    &stream_update);
 		mutex_unlock(&dm->dc_lock);
 	}
 
@@ -9273,7 +9289,7 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 		if (modereset_required(new_crtc_state))
 			goto skip_modeset;
 
-		if (modeset_required(new_crtc_state, new_stream,
+		if (amdgpu_dm_crtc_modeset_required(new_crtc_state, new_stream,
 				     dm_old_crtc_state->stream)) {
 
 			WARN_ON(dm_new_crtc_state->stream);
@@ -9304,7 +9320,7 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 skip_modeset:
 	/* Release extra reference */
 	if (new_stream)
-		 dc_stream_release(new_stream);
+		dc_stream_release(new_stream);
 
 	/*
 	 * We want to do dc stream updates that do not require a
@@ -9624,7 +9640,7 @@ static int dm_update_plane_state(struct dc *dc,
 		if (!needs_reset)
 			return 0;
 
-		ret = dm_plane_helper_check_state(new_plane_state, new_crtc_state);
+		ret = amdgpu_dm_plane_helper_check_state(new_plane_state, new_crtc_state);
 		if (ret)
 			return ret;
 
@@ -9770,7 +9786,6 @@ static int dm_check_crtc_cursor(struct drm_atomic_state *state,
 	return 0;
 }
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm_crtc *crtc)
 {
 	struct drm_connector *connector;
@@ -9796,7 +9811,6 @@ static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm
 
 	return drm_dp_mst_add_affected_dsc_crtcs(state, &aconnector->mst_root->mst_mgr);
 }
-#endif
 
 /**
  * amdgpu_dm_atomic_check() - Atomic check implementation for AMDgpu DM.
@@ -9840,11 +9854,9 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	bool lock_and_validation_needed = false;
 	bool is_top_most_overlay = true;
 	struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state;
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	struct drm_dp_mst_topology_mgr *mgr;
 	struct drm_dp_mst_topology_state *mst_state;
 	struct dsc_mst_fairness_vars vars[MAX_PIPES];
-#endif
 
 	trace_amdgpu_dm_atomic_check_begin(state);
 
@@ -9875,7 +9887,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 			new_crtc_state->connectors_changed = true;
 	}
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	if (dc_resource_is_dsc_encoding_supported(dc)) {
 		for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 			if (drm_atomic_crtc_needs_modeset(new_crtc_state)) {
@@ -9887,7 +9898,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 			}
 		}
 	}
-#endif
 	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 		dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
 
@@ -10025,13 +10035,11 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		}
 	}
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	if (dc_resource_is_dsc_encoding_supported(dc)) {
 		ret = pre_validate_dsc(state, &dm_state, vars);
 		if (ret != 0)
 			goto fail;
 	}
-#endif
 
 	/* Run this here since we want to validate the streams we created */
 	ret = drm_atomic_helper_check_planes(dev, state);
@@ -10097,7 +10105,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		lock_and_validation_needed = true;
 	}
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	/* set the slot info for each mst_state based on the link encoding format */
 	for_each_new_mst_mgr_in_state(state, mgr, mst_state, i) {
 		struct amdgpu_dm_connector *aconnector;
@@ -10117,7 +10124,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		}
 		drm_connector_list_iter_end(&iter);
 	}
-#endif
 
 	/**
 	 * Streams and planes are reset when there are changes that affect
@@ -10145,7 +10151,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 			goto fail;
 		}
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 		ret = compute_mst_dsc_configs_for_state(state, dm_state->context, vars);
 		if (ret) {
 			DRM_DEBUG_DRIVER("compute_mst_dsc_configs_for_state() failed\n");
@@ -10157,7 +10162,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 			DRM_DEBUG_DRIVER("dm_update_mst_vcpi_slots_for_dsc() failed\n");
 			goto fail;
 		}
-#endif
 
 		/*
 		 * Perform validation of MST topology in the state:
@@ -10650,7 +10654,7 @@ int amdgpu_dm_process_dmub_aux_transfer_sync(
 	if (!dc_process_dmub_aux_transfer_async(ctx->dc, link_index, payload)) {
 		*operation_result = AUX_RET_ERROR_ENGINE_ACQUIRE;
 		goto out;
- 	}
+	}
 
 	if (!wait_for_completion_timeout(&adev->dm.dmub_aux_transfer_done, 10 * HZ)) {
 		DRM_ERROR("wait_for_completion_timeout timeout!");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index ed5cbe9da40c..2e2413fd73a4 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -461,9 +461,7 @@ struct amdgpu_display_manager {
 	struct amdgpu_dm_backlight_caps backlight_caps[AMDGPU_DM_MAX_NUM_EDP];
 
 	struct mod_freesync *freesync_module;
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	struct hdcp_workqueue *hdcp_workqueue;
-#endif
 
 	/**
 	 * @vblank_control_workqueue:
@@ -612,6 +610,7 @@ struct amdgpu_dm_connector {
 
 	struct drm_connector base;
 	uint32_t connector_id;
+	int bl_idx;
 
 	/* we need to mind the EDID between detect
 	   and get modes due to analog/digital/tvencoder */
@@ -747,9 +746,7 @@ struct dm_connector_state {
 	uint8_t underscan_hborder;
 	bool underscan_enable;
 	bool freesync_capable;
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	bool update_hdcp;
-#endif
 	uint8_t abm_level;
 	int vcpi_slots;
 	uint64_t pbn;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
index dc4f37240beb..e3762e806617 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
@@ -34,7 +34,7 @@
 #include "amdgpu_dm_trace.h"
 #include "amdgpu_dm_debugfs.h"
 
-void dm_crtc_handle_vblank(struct amdgpu_crtc *acrtc)
+void amdgpu_dm_crtc_handle_vblank(struct amdgpu_crtc *acrtc)
 {
 	struct drm_crtc *crtc = &acrtc->base;
 	struct drm_device *dev = crtc->dev;
@@ -54,14 +54,14 @@ void dm_crtc_handle_vblank(struct amdgpu_crtc *acrtc)
 	spin_unlock_irqrestore(&dev->event_lock, flags);
 }
 
-bool modeset_required(struct drm_crtc_state *crtc_state,
+bool amdgpu_dm_crtc_modeset_required(struct drm_crtc_state *crtc_state,
 			     struct dc_stream_state *new_stream,
 			     struct dc_stream_state *old_stream)
 {
 	return crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state);
 }
 
-bool amdgpu_dm_vrr_active_irq(struct amdgpu_crtc *acrtc)
+bool amdgpu_dm_crtc_vrr_active_irq(struct amdgpu_crtc *acrtc)
 
 {
 	return acrtc->dm_irq_params.freesync_config.state ==
@@ -70,7 +70,7 @@ bool amdgpu_dm_vrr_active_irq(struct amdgpu_crtc *acrtc)
 		       VRR_STATE_ACTIVE_FIXED;
 }
 
-int dm_set_vupdate_irq(struct drm_crtc *crtc, bool enable)
+int amdgpu_dm_crtc_set_vupdate_irq(struct drm_crtc *crtc, bool enable)
 {
 	enum dc_irq_source irq_source;
 	struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
@@ -89,7 +89,7 @@ int dm_set_vupdate_irq(struct drm_crtc *crtc, bool enable)
 	return rc;
 }
 
-bool amdgpu_dm_vrr_active(struct dm_crtc_state *dm_state)
+bool amdgpu_dm_crtc_vrr_active(struct dm_crtc_state *dm_state)
 {
 	return dm_state->freesync_config.state == VRR_STATE_ACTIVE_VARIABLE ||
 	       dm_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED;
@@ -159,20 +159,31 @@ static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable)
 
 	if (enable) {
 		/* vblank irq on -> Only need vupdate irq in vrr mode */
-		if (amdgpu_dm_vrr_active(acrtc_state))
-			rc = dm_set_vupdate_irq(crtc, true);
+		if (amdgpu_dm_crtc_vrr_active(acrtc_state))
+			rc = amdgpu_dm_crtc_set_vupdate_irq(crtc, true);
 	} else {
 		/* vblank irq off -> vupdate irq off */
-		rc = dm_set_vupdate_irq(crtc, false);
+		rc = amdgpu_dm_crtc_set_vupdate_irq(crtc, false);
 	}
 
 	if (rc)
 		return rc;
 
-	irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst;
+	if (amdgpu_in_reset(adev)) {
+		irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst;
+		/* During gpu-reset we disable and then enable vblank irq, so
+		 * don't use amdgpu_irq_get/put() to avoid refcount change.
+		 */
+		if (!dc_interrupt_set(adev->dm.dc, irq_source, enable))
+			rc = -EBUSY;
+	} else {
+		rc = (enable)
+			? amdgpu_irq_get(adev, &adev->crtc_irq, acrtc->crtc_id)
+			: amdgpu_irq_put(adev, &adev->crtc_irq, acrtc->crtc_id);
+	}
 
-	if (!dc_interrupt_set(adev->dm.dc, irq_source, enable))
-		return -EBUSY;
+	if (rc)
+		return rc;
 
 skip:
 	if (amdgpu_in_reset(adev))
@@ -199,12 +210,12 @@ skip:
 	return 0;
 }
 
-int dm_enable_vblank(struct drm_crtc *crtc)
+int amdgpu_dm_crtc_enable_vblank(struct drm_crtc *crtc)
 {
 	return dm_set_vblank(crtc, true);
 }
 
-void dm_disable_vblank(struct drm_crtc *crtc)
+void amdgpu_dm_crtc_disable_vblank(struct drm_crtc *crtc)
 {
 	dm_set_vblank(crtc, false);
 }
@@ -300,8 +311,8 @@ static const struct drm_crtc_funcs amdgpu_dm_crtc_funcs = {
 	.verify_crc_source = amdgpu_dm_crtc_verify_crc_source,
 	.get_crc_sources = amdgpu_dm_crtc_get_crc_sources,
 	.get_vblank_counter = amdgpu_get_vblank_counter_kms,
-	.enable_vblank = dm_enable_vblank,
-	.disable_vblank = dm_disable_vblank,
+	.enable_vblank = amdgpu_dm_crtc_enable_vblank,
+	.disable_vblank = amdgpu_dm_crtc_disable_vblank,
 	.get_vblank_timestamp = drm_crtc_vblank_helper_get_vblank_timestamp,
 #if defined(CONFIG_DEBUG_FS)
 	.late_register = amdgpu_dm_crtc_late_register,
@@ -381,7 +392,7 @@ static int dm_crtc_helper_atomic_check(struct drm_crtc *crtc,
 	dm_update_crtc_active_planes(crtc, crtc_state);
 
 	if (WARN_ON(unlikely(!dm_crtc_state->stream &&
-			modeset_required(crtc_state, NULL, dm_crtc_state->stream)))) {
+			amdgpu_dm_crtc_modeset_required(crtc_state, NULL, dm_crtc_state->stream)))) {
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
index 1ac8692354cf..17e948753f59 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
@@ -27,21 +27,21 @@
 #ifndef __AMDGPU_DM_CRTC_H__
 #define __AMDGPU_DM_CRTC_H__
 
-void dm_crtc_handle_vblank(struct amdgpu_crtc *acrtc);
+void amdgpu_dm_crtc_handle_vblank(struct amdgpu_crtc *acrtc);
 
-bool modeset_required(struct drm_crtc_state *crtc_state,
+bool amdgpu_dm_crtc_modeset_required(struct drm_crtc_state *crtc_state,
 		      struct dc_stream_state *new_stream,
 		      struct dc_stream_state *old_stream);
 
-int dm_set_vupdate_irq(struct drm_crtc *crtc, bool enable);
+int amdgpu_dm_crtc_set_vupdate_irq(struct drm_crtc *crtc, bool enable);
 
-bool amdgpu_dm_vrr_active_irq(struct amdgpu_crtc *acrtc);
+bool amdgpu_dm_crtc_vrr_active_irq(struct amdgpu_crtc *acrtc);
 
-bool amdgpu_dm_vrr_active(struct dm_crtc_state *dm_state);
+bool amdgpu_dm_crtc_vrr_active(struct dm_crtc_state *dm_state);
 
-int dm_enable_vblank(struct drm_crtc *crtc);
+int amdgpu_dm_crtc_enable_vblank(struct drm_crtc *crtc);
 
-void dm_disable_vblank(struct drm_crtc *crtc);
+void amdgpu_dm_crtc_disable_vblank(struct drm_crtc *crtc);
 
 int amdgpu_dm_crtc_init(struct amdgpu_display_manager *dm,
 			struct drm_plane *plane,
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
index 09a3efa517da..827fcb4fb3b3 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -724,7 +724,7 @@ static ssize_t dp_phy_test_pattern_debugfs_write(struct file *f, const char __us
 	for (i = 0; i < (unsigned int)(link_training_settings.link_settings.lane_count); i++)
 		link_training_settings.hw_lane_settings[i] = link->cur_lane_setting[i];
 
-	dc_link_set_test_pattern(
+	dc_link_dp_set_test_pattern(
 		link,
 		test_pattern,
 		DP_TEST_PATTERN_COLOR_SPACE_RGB,
@@ -947,7 +947,6 @@ static ssize_t dp_dsc_passthrough_set(struct file *f, const char __user *buf,
 	return 0;
 }
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 /*
  * Returns the HDCP capability of the Display (1.4 for now).
  *
@@ -984,7 +983,6 @@ static int hdcp_sink_capability_show(struct seq_file *m, void *data)
 
 	return 0;
 }
-#endif
 
 /*
  * Returns whether the connected display is internal and not hotpluggable.
@@ -2593,9 +2591,7 @@ DEFINE_SHOW_ATTRIBUTE(dp_dsc_fec_support);
 DEFINE_SHOW_ATTRIBUTE(dmub_fw_state);
 DEFINE_SHOW_ATTRIBUTE(dmub_tracebuffer);
 DEFINE_SHOW_ATTRIBUTE(dp_lttpr_status);
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 DEFINE_SHOW_ATTRIBUTE(hdcp_sink_capability);
-#endif
 DEFINE_SHOW_ATTRIBUTE(internal_display);
 DEFINE_SHOW_ATTRIBUTE(psr_capability);
 DEFINE_SHOW_ATTRIBUTE(dp_is_mst_connector);
@@ -2726,9 +2722,7 @@ static const struct {
 		{"phy_settings", &dp_phy_settings_debugfs_fop},
 		{"lttpr_status", &dp_lttpr_status_fops},
 		{"test_pattern", &dp_phy_test_pattern_fops},
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 		{"hdcp_sink_capability", &hdcp_sink_capability_fops},
-#endif
 		{"sdp_message", &sdp_message_fops},
 		{"aux_dpcd_address", &dp_dpcd_address_debugfs_fops},
 		{"aux_dpcd_size", &dp_dpcd_size_debugfs_fops},
@@ -2749,14 +2743,13 @@ static const struct {
 		{"is_dpia_link", &is_dpia_link_fops}
 };
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 static const struct {
 	char *name;
 	const struct file_operations *fops;
 } hdmi_debugfs_entries[] = {
 		{"hdcp_sink_capability", &hdcp_sink_capability_fops}
 };
-#endif
+
 /*
  * Force YUV420 output if available from the given mode
  */
@@ -2801,6 +2794,22 @@ static int psr_get(void *data, u64 *val)
 }
 
 /*
+ *  Read PSR state residency
+ */
+static int psr_read_residency(void *data, u64 *val)
+{
+	struct amdgpu_dm_connector *connector = data;
+	struct dc_link *link = connector->dc_link;
+	u32 residency;
+
+	link->dc->link_srv->edp_get_psr_residency(link, &residency);
+
+	*val = (u64)residency;
+
+	return 0;
+}
+
+/*
  * Set dmcub trace event IRQ enable or disable.
  * Usage to enable dmcub trace event IRQ: echo 1 > /sys/kernel/debug/dri/0/amdgpu_dm_dmcub_trace_event_en
  * Usage to disable dmcub trace event IRQ: echo 0 > /sys/kernel/debug/dri/0/amdgpu_dm_dmcub_trace_event_en
@@ -2835,6 +2844,8 @@ DEFINE_DEBUGFS_ATTRIBUTE(dmcub_trace_event_state_fops, dmcub_trace_event_state_g
 			 dmcub_trace_event_state_set, "%llu\n");
 
 DEFINE_DEBUGFS_ATTRIBUTE(psr_fops, psr_get, NULL, "%llu\n");
+DEFINE_DEBUGFS_ATTRIBUTE(psr_residency_fops, psr_read_residency, NULL,
+			 "%llu\n");
 
 DEFINE_SHOW_ATTRIBUTE(current_backlight);
 DEFINE_SHOW_ATTRIBUTE(target_backlight);
@@ -2998,6 +3009,8 @@ void connector_debugfs_init(struct amdgpu_dm_connector *connector)
 	if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP) {
 		debugfs_create_file_unsafe("psr_capability", 0444, dir, connector, &psr_capability_fops);
 		debugfs_create_file_unsafe("psr_state", 0444, dir, connector, &psr_fops);
+		debugfs_create_file_unsafe("psr_residency", 0444, dir,
+					   connector, &psr_residency_fops);
 		debugfs_create_file("amdgpu_current_backlight_pwm", 0444, dir, connector,
 				    &current_backlight_fops);
 		debugfs_create_file("amdgpu_target_backlight_pwm", 0444, dir, connector,
@@ -3015,7 +3028,6 @@ void connector_debugfs_init(struct amdgpu_dm_connector *connector)
 	connector->debugfs_dpcd_address = 0;
 	connector->debugfs_dpcd_size = 0;
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	if (connector->base.connector_type == DRM_MODE_CONNECTOR_HDMIA) {
 		for (i = 0; i < ARRAY_SIZE(hdmi_debugfs_entries); i++) {
 			debugfs_create_file(hdmi_debugfs_entries[i].name,
@@ -3023,7 +3035,6 @@ void connector_debugfs_init(struct amdgpu_dm_connector *connector)
 					    hdmi_debugfs_entries[i].fops);
 		}
 	}
-#endif
 }
 
 #ifdef CONFIG_DRM_AMD_SECURE_DISPLAY
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c
index 4abfd2c9679f..5536d17306d0 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c
@@ -559,6 +559,7 @@ static void update_config(void *handle, struct cp_psp_stream_config *config)
 	link->dp.rev = aconnector->dc_link->dpcd_caps.dpcd_rev.raw;
 	link->dp.assr_enabled = config->assr_enabled;
 	link->dp.mst_enabled = config->mst_enabled;
+	link->dp.dp2_enabled = config->dp2_enabled;
 	link->dp.usb4_enabled = config->usb4_enabled;
 	display->adjust.disable = MOD_HDCP_DISPLAY_DISABLE_AUTHENTICATION;
 	link->adjust.auth_delay = 2;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index efd025d8961e..330ab036c830 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -44,6 +44,9 @@
 #include "dm_helpers.h"
 #include "ddc_service_types.h"
 
+/* MST Dock */
+static const uint8_t SYNAPTICS_DEVICE_ID[] = "SYNA";
+
 /* dm_helpers_parse_edid_caps
  *
  * Parse edid caps
@@ -511,8 +514,8 @@ bool dm_helpers_dp_read_dpcd(
 		return false;
 	}
 
-	return drm_dp_dpcd_read(&aconnector->dm_dp_aux.aux, address,
-			data, size) > 0;
+	return drm_dp_dpcd_read(&aconnector->dm_dp_aux.aux, address, data,
+				size) == size;
 }
 
 bool dm_helpers_dp_write_dpcd(
@@ -568,7 +571,6 @@ bool dm_helpers_submit_i2c(
 	return result;
 }
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 static bool execute_synaptics_rc_command(struct drm_dp_aux *aux,
 		bool is_write_cmd,
 		unsigned char cmd,
@@ -736,7 +738,6 @@ static uint8_t write_dsc_enable_synaptics_non_virtual_dpcd_mst(
 
 	return ret;
 }
-#endif
 
 bool dm_helpers_dp_write_dsc_enable(
 		struct dc_context *ctx,
@@ -762,13 +763,11 @@ bool dm_helpers_dp_write_dsc_enable(
 		if (!aconnector->dsc_aux)
 			return false;
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 		// apply w/a to synaptics
 		if (needs_dsc_aux_workaround(aconnector->dc_link) &&
 		    (aconnector->mst_downstream_port_present.byte & 0x7) != 0x3)
 			return write_dsc_enable_synaptics_non_virtual_dpcd_mst(
 				aconnector->dsc_aux, stream, enable_dsc);
-#endif
 
 		port = aconnector->mst_output_port;
 
@@ -806,17 +805,13 @@ bool dm_helpers_dp_write_dsc_enable(
 	}
 
 	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT || stream->signal == SIGNAL_TYPE_EDP) {
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 		if (stream->sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_NONE) {
-#endif
 			ret = dm_helpers_dp_write_dpcd(ctx, stream->link, DP_DSC_ENABLE, &enable_dsc, 1);
 			DC_LOG_DC("Send DSC %s to SST RX\n", enable_dsc ? "enable" : "disable");
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 		} else if (stream->sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER) {
 			ret = dm_helpers_dp_write_dpcd(ctx, stream->link, DP_DSC_ENABLE, &enable_dsc, 1);
 			DC_LOG_DC("Send DSC %s to DP-HDMI PCON\n", enable_dsc ? "enable" : "disable");
 		}
-#endif
 	}
 
 	return ret;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
index 8dc442f90eaf..994ba426ca66 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
@@ -31,10 +31,7 @@
 #include "amdgpu.h"
 #include "amdgpu_dm.h"
 #include "amdgpu_dm_mst_types.h"
-
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 #include "amdgpu_dm_hdcp.h"
-#endif
 
 #include "dc.h"
 #include "dm_helpers.h"
@@ -201,7 +198,6 @@ static const struct drm_connector_funcs dm_dp_mst_connector_funcs = {
 	.early_unregister = amdgpu_dm_mst_connector_early_unregister,
 };
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 bool needs_dsc_aux_workaround(struct dc_link *link)
 {
 	if (link->dpcd_caps.branch_dev_id == DP_BRANCH_DEVICE_ID_90CC24 &&
@@ -212,7 +208,7 @@ bool needs_dsc_aux_workaround(struct dc_link *link)
 	return false;
 }
 
-bool is_synaptics_cascaded_panamera(struct dc_link *link, struct drm_dp_mst_port *port)
+static bool is_synaptics_cascaded_panamera(struct dc_link *link, struct drm_dp_mst_port *port)
 {
 	u8 branch_vendor_data[4] = { 0 }; // Vendor data 0x50C ~ 0x50F
 
@@ -290,7 +286,6 @@ static bool retrieve_downstream_port_device(struct amdgpu_dm_connector *aconnect
 
 	return true;
 }
-#endif
 
 static int dm_dp_mst_get_modes(struct drm_connector *connector)
 {
@@ -381,7 +376,6 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector)
 		 * plugged back with same display index, its hdcp properties
 		 * will be retrieved from hdcp_work within dm_dp_mst_get_modes
 		 */
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 		if (aconnector->dc_sink && connector->state) {
 			struct drm_device *dev = connector->dev;
 			struct amdgpu_device *adev = drm_to_adev(dev);
@@ -393,13 +387,11 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector)
 			connector->state->content_protection =
 			hdcp_w->content_protection[connector->index];
 		}
-#endif
 
 		if (aconnector->dc_sink) {
 			amdgpu_dm_update_freesync_caps(
 					connector, aconnector->edid);
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 			if (!validate_dsc_caps_on_connector(aconnector))
 				memset(&aconnector->dc_sink->dsc_caps,
 				       0, sizeof(aconnector->dc_sink->dsc_caps));
@@ -407,7 +399,6 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector)
 			if (!retrieve_downstream_port_device(aconnector))
 				memset(&aconnector->mst_downstream_port_present,
 					0, sizeof(aconnector->mst_downstream_port_present));
-#endif
 		}
 	}
 
@@ -666,8 +657,6 @@ int dm_mst_get_pbn_divider(struct dc_link *link)
 			dc_link_get_link_cap(link)) / (8 * 1000 * 54);
 }
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
-
 struct dsc_mst_fairness_params {
 	struct dc_crtc_timing *timing;
 	struct dc_sink *sink;
@@ -710,16 +699,19 @@ static void set_dsc_configs_from_fairness_vars(struct dsc_mst_fairness_params *p
 {
 	struct drm_connector *drm_connector;
 	int i;
+	struct dc_dsc_config_options dsc_options = {0};
 
 	for (i = 0; i < count; i++) {
 		drm_connector = &params[i].aconnector->base;
 
+		dc_dsc_get_default_config_option(params[i].sink->ctx->dc, &dsc_options);
+		dsc_options.max_target_bpp_limit_override_x16 = drm_connector->display_info.max_dsc_bpp * 16;
+
 		memset(&params[i].timing->dsc_cfg, 0, sizeof(params[i].timing->dsc_cfg));
 		if (vars[i + k].dsc_enabled && dc_dsc_compute_config(
 					params[i].sink->ctx->dc->res_pool->dscs[0],
 					&params[i].sink->dsc_caps.dsc_dec_caps,
-					params[i].sink->ctx->dc->debug.dsc_min_slice_height_override,
-					drm_connector->display_info.max_dsc_bpp,
+					&dsc_options,
 					0,
 					params[i].timing,
 					&params[i].timing->dsc_cfg)) {
@@ -762,15 +754,16 @@ static int bpp_x16_from_pbn(struct dsc_mst_fairness_params param, int pbn)
 	u64 kbps;
 
 	struct drm_connector *drm_connector = &param.aconnector->base;
-	uint32_t max_dsc_target_bpp_limit_override =
-		drm_connector->display_info.max_dsc_bpp;
+	struct dc_dsc_config_options dsc_options = {0};
+
+	dc_dsc_get_default_config_option(param.sink->ctx->dc, &dsc_options);
+	dsc_options.max_target_bpp_limit_override_x16 = drm_connector->display_info.max_dsc_bpp * 16;
 
 	kbps = div_u64((u64)pbn * 994 * 8 * 54, 64);
 	dc_dsc_compute_config(
 			param.sink->ctx->dc->res_pool->dscs[0],
 			&param.sink->dsc_caps.dsc_dec_caps,
-			param.sink->ctx->dc->debug.dsc_min_slice_height_override,
-			max_dsc_target_bpp_limit_override,
+			&dsc_options,
 			(int) kbps, param.timing, &dsc_config);
 
 	return dsc_config.bits_per_pixel;
@@ -1189,6 +1182,7 @@ int compute_mst_dsc_configs_for_state(struct drm_atomic_state *state,
 	bool computed_streams[MAX_PIPES];
 	struct amdgpu_dm_connector *aconnector;
 	struct drm_dp_mst_topology_mgr *mst_mgr;
+	struct resource_pool *res_pool;
 	int link_vars_start_index = 0;
 	int ret = 0;
 
@@ -1197,6 +1191,7 @@ int compute_mst_dsc_configs_for_state(struct drm_atomic_state *state,
 
 	for (i = 0; i < dc_state->stream_count; i++) {
 		stream = dc_state->streams[i];
+		res_pool = stream->ctx->dc->res_pool;
 
 		if (stream->signal != SIGNAL_TYPE_DISPLAY_PORT_MST)
 			continue;
@@ -1212,7 +1207,8 @@ int compute_mst_dsc_configs_for_state(struct drm_atomic_state *state,
 		if (computed_streams[i])
 			continue;
 
-		if (dcn20_remove_stream_from_ctx(stream->ctx->dc, dc_state, stream) != DC_OK)
+		if (!res_pool->funcs->remove_stream_from_ctx ||
+		    res_pool->funcs->remove_stream_from_ctx(stream->ctx->dc, dc_state, stream) != DC_OK)
 			return -EINVAL;
 
 		if (!is_dsc_need_re_compute(state, dc_state, stream->link))
@@ -1470,14 +1466,12 @@ static bool is_dsc_common_config_possible(struct dc_stream_state *stream,
 
 	return bw_range->max_target_bpp_x16 && bw_range->min_target_bpp_x16;
 }
-#endif /* CONFIG_DRM_AMD_DC_DCN */
 
 enum dc_status dm_dp_mst_is_port_support_mode(
 	struct amdgpu_dm_connector *aconnector,
 	struct dc_stream_state *stream)
 {
 	int bpp, pbn, branch_max_throughput_mps = 0;
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	struct dc_link_settings cur_link_settings;
 	unsigned int end_to_end_bw_in_kbps = 0;
 	unsigned int upper_link_bw_in_kbps = 0, down_link_bw_in_kbps = 0;
@@ -1519,16 +1513,13 @@ enum dc_status dm_dp_mst_is_port_support_mode(
 			return DC_FAIL_BANDWIDTH_VALIDATE;
 		}
 	} else {
-#endif
 		/* check if mode could be supported within full_pbn */
 		bpp = convert_dc_color_depth_into_bpc(stream->timing.display_color_depth) * 3;
 		pbn = drm_dp_calc_pbn_mode(stream->timing.pix_clk_100hz / 10, bpp, false);
 
 		if (pbn > aconnector->mst_output_port->full_pbn)
 			return DC_FAIL_BANDWIDTH_VALIDATE;
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	}
-#endif
 
 	/* check is mst dsc output bandwidth branch_overall_throughput_0_mps */
 	switch (stream->timing.pixel_encoding) {
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
index 28fb1f02591a..322668973747 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
@@ -90,12 +90,12 @@ enum dm_micro_swizzle {
 	MICRO_SWIZZLE_R = 3
 };
 
-const struct drm_format_info *amd_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
+const struct drm_format_info *amdgpu_dm_plane_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
 {
 	return amdgpu_lookup_format_info(cmd->pixel_format, cmd->modifier[0]);
 }
 
-void fill_blending_from_plane_state(const struct drm_plane_state *plane_state,
+void amdgpu_dm_plane_fill_blending_from_plane_state(const struct drm_plane_state *plane_state,
 			       bool *per_pixel_alpha, bool *pre_multiplied_alpha,
 			       bool *global_alpha, int *global_alpha_value)
 {
@@ -741,25 +741,7 @@ static int get_plane_formats(const struct drm_plane *plane,
 	return num_formats;
 }
 
-#ifdef CONFIG_DRM_AMD_DC_HDR
-static int attach_color_mgmt_properties(struct amdgpu_display_manager *dm, struct drm_plane *plane)
-{
-	drm_object_attach_property(&plane->base,
-				   dm->degamma_lut_property,
-				   0);
-	drm_object_attach_property(&plane->base,
-				   dm->degamma_lut_size_property,
-				   MAX_COLOR_LUT_ENTRIES);
-	drm_object_attach_property(&plane->base, dm->ctm_property,
-				   0);
-	drm_object_attach_property(&plane->base, dm->sdr_boost_property,
-				   DEFAULT_SDR_BOOST);
-
-	return 0;
-}
-#endif
-
-int fill_plane_buffer_attributes(struct amdgpu_device *adev,
+int amdgpu_dm_plane_fill_plane_buffer_attributes(struct amdgpu_device *adev,
 			     const struct amdgpu_framebuffer *afb,
 			     const enum surface_pixel_format format,
 			     const enum dc_rotation_angle rotation,
@@ -918,7 +900,7 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane,
 			dm_plane_state_new->dc_state;
 		bool force_disable_dcc = !plane_state->dcc.enable;
 
-		fill_plane_buffer_attributes(
+		amdgpu_dm_plane_fill_plane_buffer_attributes(
 			adev, afb, plane_state->format, plane_state->rotation,
 			afb->tiling_flags,
 			&plane_state->tiling_info, &plane_state->plane_size,
@@ -999,7 +981,7 @@ static void get_min_max_dc_plane_scaling(struct drm_device *dev,
 		*min_downscale = 1000;
 }
 
-int dm_plane_helper_check_state(struct drm_plane_state *state,
+int amdgpu_dm_plane_helper_check_state(struct drm_plane_state *state,
 				       struct drm_crtc_state *new_crtc_state)
 {
 	struct drm_framebuffer *fb = state->fb;
@@ -1053,7 +1035,7 @@ int dm_plane_helper_check_state(struct drm_plane_state *state,
 		state, new_crtc_state, min_scale, max_scale, true, true);
 }
 
-int fill_dc_scaling_info(struct amdgpu_device *adev,
+int amdgpu_dm_plane_fill_dc_scaling_info(struct amdgpu_device *adev,
 				const struct drm_plane_state *state,
 				struct dc_scaling_info *scaling_info)
 {
@@ -1161,11 +1143,11 @@ static int dm_plane_atomic_check(struct drm_plane *plane,
 	if (!new_crtc_state)
 		return -EINVAL;
 
-	ret = dm_plane_helper_check_state(new_plane_state, new_crtc_state);
+	ret = amdgpu_dm_plane_helper_check_state(new_plane_state, new_crtc_state);
 	if (ret)
 		return ret;
 
-	ret = fill_dc_scaling_info(adev, new_plane_state, &scaling_info);
+	ret = amdgpu_dm_plane_fill_dc_scaling_info(adev, new_plane_state, &scaling_info);
 	if (ret)
 		return ret;
 
@@ -1229,7 +1211,7 @@ static int get_cursor_position(struct drm_plane *plane, struct drm_crtc *crtc,
 	return 0;
 }
 
-void handle_cursor_update(struct drm_plane *plane,
+void amdgpu_dm_plane_handle_cursor_update(struct drm_plane *plane,
 				 struct drm_plane_state *old_plane_state)
 {
 	struct amdgpu_device *adev = drm_to_adev(plane->dev);
@@ -1314,7 +1296,7 @@ static void dm_plane_atomic_async_update(struct drm_plane *plane,
 	plane->state->crtc_w = new_state->crtc_w;
 	plane->state->crtc_h = new_state->crtc_h;
 
-	handle_cursor_update(plane, old_state);
+	amdgpu_dm_plane_handle_cursor_update(plane, old_state);
 }
 
 static const struct drm_plane_helper_funcs dm_plane_helper_funcs = {
@@ -1337,10 +1319,6 @@ static void dm_drm_plane_reset(struct drm_plane *plane)
 
 	if (amdgpu_state)
 		__drm_atomic_helper_plane_reset(plane, &amdgpu_state->base);
-#ifdef CONFIG_DRM_AMD_DC_HDR
-	if (amdgpu_state)
-		amdgpu_state->sdr_boost = DEFAULT_SDR_BOOST;
-#endif
 }
 
 static struct drm_plane_state *
@@ -1360,15 +1338,6 @@ dm_drm_plane_duplicate_state(struct drm_plane *plane)
 		dc_plane_state_retain(dm_plane_state->dc_state);
 	}
 
-#ifdef CONFIG_DRM_AMD_DC_HDR
-	if (dm_plane_state->degamma_lut)
-		drm_property_blob_get(dm_plane_state->degamma_lut);
-	if (dm_plane_state->ctm)
-		drm_property_blob_get(dm_plane_state->ctm);
-
-	dm_plane_state->sdr_boost = old_dm_plane_state->sdr_boost;
-#endif
-
 	return &dm_plane_state->base;
 }
 
@@ -1436,103 +1405,12 @@ static void dm_drm_plane_destroy_state(struct drm_plane *plane,
 {
 	struct dm_plane_state *dm_plane_state = to_dm_plane_state(state);
 
-#ifdef CONFIG_DRM_AMD_DC_HDR
-	drm_property_blob_put(dm_plane_state->degamma_lut);
-	drm_property_blob_put(dm_plane_state->ctm);
-#endif
 	if (dm_plane_state->dc_state)
 		dc_plane_state_release(dm_plane_state->dc_state);
 
 	drm_atomic_helper_plane_destroy_state(plane, state);
 }
 
-#ifdef CONFIG_DRM_AMD_DC_HDR
-/* copied from drm_atomic_uapi.c */
-static int atomic_replace_property_blob_from_id(struct drm_device *dev,
-					 struct drm_property_blob **blob,
-					 uint64_t blob_id,
-					 ssize_t expected_size,
-					 ssize_t expected_elem_size,
-					 bool *replaced)
-{
-	struct drm_property_blob *new_blob = NULL;
-
-	if (blob_id != 0) {
-		new_blob = drm_property_lookup_blob(dev, blob_id);
-		if (new_blob == NULL)
-			return -EINVAL;
-
-		if (expected_size > 0 &&
-		    new_blob->length != expected_size) {
-			drm_property_blob_put(new_blob);
-			return -EINVAL;
-		}
-		if (expected_elem_size > 0 &&
-		    new_blob->length % expected_elem_size != 0) {
-			drm_property_blob_put(new_blob);
-			return -EINVAL;
-		}
-	}
-
-	*replaced |= drm_property_replace_blob(blob, new_blob);
-	drm_property_blob_put(new_blob);
-
-	return 0;
-}
-
-int dm_drm_plane_set_property(struct drm_plane *plane,
-			      struct drm_plane_state *state,
-			      struct drm_property *property,
-			      uint64_t val)
-{
-	struct amdgpu_device *adev = drm_to_adev(plane->dev);
-	struct dm_plane_state *dm_plane_state = to_dm_plane_state(state);
-	int ret = 0;
-	bool replaced;
-
-	if (property == adev->dm.degamma_lut_property) {
-		ret = atomic_replace_property_blob_from_id(adev_to_drm(adev),
-				&dm_plane_state->degamma_lut,
-				val, -1, sizeof(struct drm_color_lut),
-				&replaced);
-	} else if (property == adev->dm.ctm_property) {
-		ret = atomic_replace_property_blob_from_id(adev_to_drm(adev),
-				&dm_plane_state->ctm,
-				val,
-				sizeof(struct drm_color_ctm), -1,
-				&replaced);
-	} else if (property == adev->dm.sdr_boost_property) {
-		dm_plane_state->sdr_boost = val;
-	} else {
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-int dm_drm_plane_get_property(struct drm_plane *plane,
-			      const struct drm_plane_state *state,
-			      struct drm_property *property,
-			      uint64_t *val)
-{
-	struct dm_plane_state *dm_plane_state = to_dm_plane_state(state);
-	struct amdgpu_device *adev = drm_to_adev(plane->dev);
-
-	if (property == adev->dm.degamma_lut_property) {
-		*val = (dm_plane_state->degamma_lut) ?
-			dm_plane_state->degamma_lut->base.id : 0;
-	} else if (property == adev->dm.ctm_property) {
-		*val = (dm_plane_state->ctm) ? dm_plane_state->ctm->base.id : 0;
-	} else if (property == adev->dm.sdr_boost_property) {
-		*val = dm_plane_state->sdr_boost;
-	} else {
-		return -EINVAL;
-	}
-
-	return 0;
-}
-#endif
-
 static const struct drm_plane_funcs dm_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
@@ -1541,10 +1419,6 @@ static const struct drm_plane_funcs dm_plane_funcs = {
 	.atomic_duplicate_state = dm_drm_plane_duplicate_state,
 	.atomic_destroy_state = dm_drm_plane_destroy_state,
 	.format_mod_supported = dm_plane_format_mod_supported,
-#ifdef CONFIG_DRM_AMD_DC_HDR
-	.atomic_set_property = dm_drm_plane_set_property,
-	.atomic_get_property = dm_drm_plane_get_property,
-#endif
 };
 
 int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm,
@@ -1615,9 +1489,6 @@ int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm,
 
 	drm_plane_helper_add(plane, &dm_plane_helper_funcs);
 
-#ifdef CONFIG_DRM_AMD_DC_HDR
-	attach_color_mgmt_properties(dm, plane);
-#endif
 	/* Create (reset) the plane state */
 	if (plane->funcs->reset)
 		plane->funcs->reset(plane);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h
index a4bee8528a51..930f1572f898 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h
@@ -29,17 +29,17 @@
 
 #include "dc.h"
 
-void handle_cursor_update(struct drm_plane *plane,
+void amdgpu_dm_plane_handle_cursor_update(struct drm_plane *plane,
 			  struct drm_plane_state *old_plane_state);
 
-int fill_dc_scaling_info(struct amdgpu_device *adev,
+int amdgpu_dm_plane_fill_dc_scaling_info(struct amdgpu_device *adev,
 			 const struct drm_plane_state *state,
 			 struct dc_scaling_info *scaling_info);
 
-int dm_plane_helper_check_state(struct drm_plane_state *state,
+int amdgpu_dm_plane_helper_check_state(struct drm_plane_state *state,
 				struct drm_crtc_state *new_crtc_state);
 
-int fill_plane_buffer_attributes(struct amdgpu_device *adev,
+int amdgpu_dm_plane_fill_plane_buffer_attributes(struct amdgpu_device *adev,
 				 const struct amdgpu_framebuffer *afb,
 				 const enum surface_pixel_format format,
 				 const enum dc_rotation_angle rotation,
@@ -56,9 +56,9 @@ int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm,
 			 unsigned long possible_crtcs,
 			 const struct dc_plane_cap *plane_cap);
 
-const struct drm_format_info *amd_get_format_info(const struct drm_mode_fb_cmd2 *cmd);
+const struct drm_format_info *amdgpu_dm_plane_get_format_info(const struct drm_mode_fb_cmd2 *cmd);
 
-void fill_blending_from_plane_state(const struct drm_plane_state *plane_state,
+void amdgpu_dm_plane_fill_blending_from_plane_state(const struct drm_plane_state *plane_state,
 				    bool *per_pixel_alpha, bool *pre_multiplied_alpha,
 				    bool *global_alpha, int *global_alpha_value);
 
diff --git a/drivers/gpu/drm/amd/display/dc/Makefile b/drivers/gpu/drm/amd/display/dc/Makefile
index 94f156d57220..69ffd4424dc7 100644
--- a/drivers/gpu/drm/amd/display/dc/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/Makefile
@@ -22,14 +22,13 @@
 #
 # Makefile for Display Core (dc) component.
 
-DC_LIBS = basics bios dml clk_mgr dce gpio irq link virtual
+DC_LIBS = basics bios dml clk_mgr dce gpio irq link virtual dsc
 
-ifdef CONFIG_DRM_AMD_DC_DCN
+ifdef CONFIG_DRM_AMD_DC_FP
 
 KCOV_INSTRUMENT := n
 
 DC_LIBS += dcn20
-DC_LIBS += dsc
 DC_LIBS += dcn10
 DC_LIBS += dcn21
 DC_LIBS += dcn201
@@ -56,9 +55,7 @@ ifdef CONFIG_DRM_AMD_DC_SI
 DC_LIBS += dce60
 endif
 
-ifdef CONFIG_DRM_AMD_DC_HDCP
 DC_LIBS += hdcp
-endif
 
 AMD_DC = $(addsuffix /Makefile, $(addprefix $(FULL_AMD_DISPLAY_PATH)/dc/,$(DC_LIBS)))
 
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
index e381de2429fa..cce47d3f1a13 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
@@ -515,11 +515,8 @@ static enum bp_result get_gpio_i2c_info(
 	info->i2c_slave_address = record->i2c_slave_addr;
 
 	/* TODO: check how to get register offset for en, Y, etc. */
-	info->gpio_info.clk_a_register_index =
-			le16_to_cpu(
-			header->gpio_pin[table_index].data_a_reg_index);
-	info->gpio_info.clk_a_shift =
-			header->gpio_pin[table_index].gpio_bitshift;
+	info->gpio_info.clk_a_register_index = le16_to_cpu(pin->data_a_reg_index);
+	info->gpio_info.clk_a_shift = pin->gpio_bitshift;
 
 	return BP_RESULT_OK;
 }
@@ -2064,7 +2061,7 @@ static enum bp_result bios_parser_get_encoder_cap_info(
 	if (!info)
 		return BP_RESULT_BADINPUT;
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 	/* encoder cap record not available in v1_5 */
 	if (bp->object_info_tbl.revision.minor == 5)
 		return BP_RESULT_NORECORD;
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
index 271d8e573181..ad390e4cd0a9 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
@@ -74,7 +74,7 @@ CLK_MGR_DCE120 = dce120_clk_mgr.o
 AMD_DAL_CLK_MGR_DCE120 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dce120/,$(CLK_MGR_DCE120))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCE120)
-ifdef CONFIG_DRM_AMD_DC_DCN
+ifdef CONFIG_DRM_AMD_DC_FP
 ###############################################################################
 # DCN10
 ###############################################################################
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
index 69691daf4dbb..6127d6045336 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
@@ -104,7 +104,7 @@ void clk_mgr_exit_optimized_pwr_state(const struct dc *dc, struct clk_mgr *clk_m
 	int edp_num;
 	unsigned int panel_inst;
 
-	get_edp_links(dc, edp_links, &edp_num);
+	dc_get_edp_links(dc, edp_links, &edp_num);
 	if (dc->hwss.exit_optimized_pwr_state)
 		dc->hwss.exit_optimized_pwr_state(dc, dc->current_state);
 
@@ -116,7 +116,7 @@ void clk_mgr_exit_optimized_pwr_state(const struct dc *dc, struct clk_mgr *clk_m
 			if (!edp_link->psr_settings.psr_feature_enabled)
 				continue;
 			clk_mgr->psr_allow_active_cache = edp_link->psr_settings.psr_allow_active;
-			dc_link_set_psr_allow_active(edp_link, &allow_active, false, false, NULL);
+			dc->link_srv->edp_set_psr_allow_active(edp_link, &allow_active, false, false, NULL);
 		}
 	}
 
@@ -129,13 +129,13 @@ void clk_mgr_optimize_pwr_state(const struct dc *dc, struct clk_mgr *clk_mgr)
 	int edp_num;
 	unsigned int panel_inst;
 
-	get_edp_links(dc, edp_links, &edp_num);
+	dc_get_edp_links(dc, edp_links, &edp_num);
 	if (edp_num) {
 		for (panel_inst = 0; panel_inst < edp_num; panel_inst++) {
 			edp_link = edp_links[panel_inst];
 			if (!edp_link->psr_settings.psr_feature_enabled)
 				continue;
-			dc_link_set_psr_allow_active(edp_link,
+			dc->link_srv->edp_set_psr_allow_active(edp_link,
 					&clk_mgr->psr_allow_active_cache, false, false, NULL);
 		}
 	}
@@ -221,7 +221,7 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p
 			dce120_clk_mgr_construct(ctx, clk_mgr);
 		return &clk_mgr->base;
 	}
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 	case FAMILY_RV: {
 		struct clk_mgr_internal *clk_mgr = kzalloc(sizeof(*clk_mgr), GFP_KERNEL);
 
@@ -351,7 +351,7 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p
 	}
 	break;
 
-#endif
+#endif /* CONFIG_DRM_AMD_DC_FP - Family RV */
 	default:
 		ASSERT(0); /* Unknown Asic */
 		break;
@@ -364,7 +364,7 @@ void dc_destroy_clk_mgr(struct clk_mgr *clk_mgr_base)
 {
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 
-#ifdef CONFIG_DRM_AMD_DC_DCN
+#ifdef CONFIG_DRM_AMD_DC_FP
 	switch (clk_mgr_base->ctx->asic_id.chip_family) {
 	case FAMILY_NV:
 		if (ASICREV_IS_SIENNA_CICHLID_P(clk_mgr_base->ctx->asic_id.hw_internal_rev)) {
@@ -405,7 +405,7 @@ void dc_destroy_clk_mgr(struct clk_mgr *clk_mgr_base)
 	default:
 		break;
 	}
-#endif
+#endif /* CONFIG_DRM_AMD_DC_FP */
 
 	kfree(clk_mgr);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c
index f0577dcd1af6..811720749faf 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c
@@ -162,7 +162,7 @@ static void dcn201_update_clocks(struct clk_mgr *clk_mgr_base,
 	}
 }
 
-struct clk_mgr_funcs dcn201_funcs = {
+static struct clk_mgr_funcs dcn201_funcs = {
 	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
 	.update_clocks = dcn201_update_clocks,
 	.init_clocks = dcn201_init_clocks,
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
index ca6dfd2d7561..bd9fd0b54f46 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
@@ -706,7 +706,7 @@ void rn_clk_mgr_construct(
 	enum pp_smu_status status = 0;
 	int is_green_sardine = 0;
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 	is_green_sardine = ASICREV_IS_GREEN_SARDINE(ctx->asic_id.hw_internal_rev);
 #endif
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
index 89df7244b272..5cb44f838bde 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
@@ -108,6 +108,11 @@ static int dcn314_get_active_display_cnt_wa(
 				stream->signal == SIGNAL_TYPE_DVI_SINGLE_LINK ||
 				stream->signal == SIGNAL_TYPE_DVI_DUAL_LINK)
 			tmds_present = true;
+
+		/* Checking stream / link detection ensuring that PHY is active*/
+		if (dc_is_dp_signal(stream->signal) && !stream->dpms_off)
+			display_count++;
+
 	}
 
 	for (i = 0; i < dc->link_count; i++) {
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
index a737782b2840..b737cbc468f5 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
@@ -522,6 +522,11 @@ static void dcn315_clk_mgr_helper_populate_bw_params(
 		bw_params->clk_table.entries[i].dcfclk_mhz = clock_table->DcfClocks[0];
 		bw_params->clk_table.entries[i].wck_ratio = 1;
 		i++;
+	} else if (clock_table->NumDcfClkLevelsEnabled != clock_table->NumSocClkLevelsEnabled) {
+		bw_params->clk_table.entries[i-1].voltage = clock_table->SocVoltage[clock_table->NumSocClkLevelsEnabled - 1];
+		bw_params->clk_table.entries[i-1].socclk_mhz = clock_table->SocClocks[clock_table->NumSocClkLevelsEnabled - 1];
+		bw_params->clk_table.entries[i-1].dispclk_mhz = clock_table->DispClocks[clock_table->NumDispClkLevelsEnabled - 1];
+		bw_params->clk_table.entries[i-1].dppclk_mhz = clock_table->DppClocks[clock_table->NumDispClkLevelsEnabled - 1];
 	}
 	bw_params->clk_table.num_entries = i;
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
index 61768bf726f8..ea753f8fa175 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
@@ -255,27 +255,60 @@ static void dcn32_update_dppclk_dispclk_freq(struct clk_mgr_internal *clk_mgr, s
 	}
 }
 
+void dcn32_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
+		struct dc_state *context, bool safe_to_lower)
+{
+	int i;
+
+	clk_mgr->dccg->ref_dppclk = clk_mgr->base.clks.dppclk_khz;
+	for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
+		int dpp_inst, dppclk_khz, prev_dppclk_khz;
+
+		dppclk_khz = context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz;
+
+		if (context->res_ctx.pipe_ctx[i].plane_res.dpp)
+			dpp_inst = context->res_ctx.pipe_ctx[i].plane_res.dpp->inst;
+		else if (!context->res_ctx.pipe_ctx[i].plane_res.dpp && dppclk_khz == 0) {
+			/* dpp == NULL && dppclk_khz == 0 is valid because of pipe harvesting.
+			 * In this case just continue in loop
+			 */
+			continue;
+		} else if (!context->res_ctx.pipe_ctx[i].plane_res.dpp && dppclk_khz > 0) {
+			/* The software state is not valid if dpp resource is NULL and
+			 * dppclk_khz > 0.
+			 */
+			ASSERT(false);
+			continue;
+		}
+
+		prev_dppclk_khz = clk_mgr->dccg->pipe_dppclk_khz[i];
+
+		if (safe_to_lower || prev_dppclk_khz < dppclk_khz)
+			clk_mgr->dccg->funcs->update_dpp_dto(
+							clk_mgr->dccg, dpp_inst, dppclk_khz);
+	}
+}
+
 static void dcn32_update_clocks_update_dentist(
 		struct clk_mgr_internal *clk_mgr,
-		struct dc_state *context,
-		uint32_t old_dispclk_khz)
+		struct dc_state *context)
 {
 	uint32_t new_disp_divider = 0;
-	uint32_t old_disp_divider = 0;
 	uint32_t new_dispclk_wdivider = 0;
 	uint32_t old_dispclk_wdivider = 0;
 	uint32_t i;
+	uint32_t dentist_dispclk_wdivider_readback = 0;
+	struct dc *dc = clk_mgr->base.ctx->dc;
 
-	if (old_dispclk_khz == 0 || clk_mgr->base.clks.dispclk_khz == 0)
+	if (clk_mgr->base.clks.dispclk_khz == 0)
 		return;
 
 	new_disp_divider = DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 			* clk_mgr->base.dentist_vco_freq_khz / clk_mgr->base.clks.dispclk_khz;
-	old_disp_divider = DENTIST_DIVIDER_RANGE_SCALE_FACTOR
-			* clk_mgr->base.dentist_vco_freq_khz / old_dispclk_khz;
 
 	new_dispclk_wdivider = dentist_get_did_from_divider(new_disp_divider);
-	old_dispclk_wdivider = dentist_get_did_from_divider(old_disp_divider);
+	REG_GET(DENTIST_DISPCLK_CNTL,
+			DENTIST_DISPCLK_WDIVIDER, &old_dispclk_wdivider);
 
 	/* When changing divider to or from 127, some extra programming is required to prevent corruption */
 	if (old_dispclk_wdivider == 127 && new_dispclk_wdivider != 127) {
@@ -314,6 +347,17 @@ static void dcn32_update_clocks_update_dentist(
 		if (clk_mgr->smu_present)
 			dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DISPCLK, khz_to_mhz_ceil(temp_dispclk_khz));
 
+		if (dc->debug.override_dispclk_programming) {
+			REG_GET(DENTIST_DISPCLK_CNTL,
+					DENTIST_DISPCLK_WDIVIDER, &dentist_dispclk_wdivider_readback);
+
+			if (dentist_dispclk_wdivider_readback != 126) {
+				REG_UPDATE(DENTIST_DISPCLK_CNTL,
+						DENTIST_DISPCLK_WDIVIDER, 126);
+				REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 2000);
+			}
+		}
+
 		for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
 			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 			struct dccg *dccg = clk_mgr->base.ctx->dc->res_pool->dccg;
@@ -341,8 +385,37 @@ static void dcn32_update_clocks_update_dentist(
 	/* do requested DISPCLK updates*/
 	if (clk_mgr->smu_present)
 		dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DISPCLK, khz_to_mhz_ceil(clk_mgr->base.clks.dispclk_khz));
+
+	if (dc->debug.override_dispclk_programming) {
+		REG_GET(DENTIST_DISPCLK_CNTL,
+				DENTIST_DISPCLK_WDIVIDER, &dentist_dispclk_wdivider_readback);
+
+		if (dentist_dispclk_wdivider_readback > new_dispclk_wdivider) {
+			REG_UPDATE(DENTIST_DISPCLK_CNTL,
+					DENTIST_DISPCLK_WDIVIDER, new_dispclk_wdivider);
+			REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 2000);
+		}
+	}
+
+}
+
+static int dcn32_get_dispclk_from_dentist(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	uint32_t dispclk_wdivider;
+	int disp_divider;
+
+	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, &dispclk_wdivider);
+	disp_divider = dentist_get_divider_from_did(dispclk_wdivider);
+
+	/* Return DISPCLK freq in Khz */
+	if (disp_divider)
+		return (DENTIST_DIVIDER_RANGE_SCALE_FACTOR * clk_mgr->base.dentist_vco_freq_khz) / disp_divider;
+
+	return 0;
 }
 
+
 static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 			struct dc_state *context,
 			bool safe_to_lower)
@@ -361,7 +434,6 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 	bool p_state_change_support;
 	bool fclk_p_state_change_support;
 	int total_plane_count;
-	int old_dispclk_khz = clk_mgr_base->clks.dispclk_khz;
 
 	if (dc->work_arounds.skip_clock_update)
 		return;
@@ -503,19 +575,19 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 	if (dc->config.forced_clocks == false || (force_reset && safe_to_lower)) {
 		if (dpp_clock_lowered) {
 			/* if clock is being lowered, increase DTO before lowering refclk */
-			dcn20_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower);
-			dcn32_update_clocks_update_dentist(clk_mgr, context, old_dispclk_khz);
+			dcn32_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower);
+			dcn32_update_clocks_update_dentist(clk_mgr, context);
 			if (clk_mgr->smu_present)
 				dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DPPCLK, khz_to_mhz_ceil(clk_mgr_base->clks.dppclk_khz));
 		} else {
 			/* if clock is being raised, increase refclk before lowering DTO */
 			if (update_dppclk || update_dispclk)
-				dcn32_update_clocks_update_dentist(clk_mgr, context, old_dispclk_khz);
+				dcn32_update_clocks_update_dentist(clk_mgr, context);
 			/* There is a check inside dcn20_update_clocks_update_dpp_dto which ensures
 			 * that we do not lower dto when it is not safe to lower. We do not need to
 			 * compare the current and new dppclk before calling this function.
 			 */
-			dcn20_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower);
+			dcn32_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower);
 		}
 	}
 
@@ -797,6 +869,7 @@ static struct clk_mgr_funcs dcn32_funcs = {
 		.are_clock_states_equal = dcn32_are_clock_states_equal,
 		.enable_pme_wa = dcn32_enable_pme_wa,
 		.is_smu_present = dcn32_is_smu_present,
+		.get_dispclk_from_dentist = dcn32_get_dispclk_from_dentist,
 };
 
 void dcn32_clk_mgr_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.h
index 57e09c7c95f5..186daada7b03 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.h
@@ -32,6 +32,9 @@ void dcn32_clk_mgr_construct(struct dc_context *ctx,
 		struct pp_smu_funcs *pp_smu,
 		struct dccg *dccg);
 
+void dcn32_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
+		struct dc_state *context, bool safe_to_lower);
+
 void dcn32_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr);
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 1c218c526650..52564b93f7eb 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -53,7 +53,6 @@
 #include "link_encoder.h"
 #include "link_enc_cfg.h"
 
-#include "dc_link.h"
 #include "link.h"
 #include "dm_helpers.h"
 #include "mem_input.h"
@@ -74,6 +73,8 @@
 
 #include "dc_trace.h"
 
+#include "hw_sequencer_private.h"
+
 #include "dce/dmub_outbox.h"
 
 #define CTX \
@@ -147,7 +148,7 @@ static void destroy_links(struct dc *dc)
 
 	for (i = 0; i < dc->link_count; i++) {
 		if (NULL != dc->links[i])
-			link_destroy(&dc->links[i]);
+			dc->link_srv->destroy_link(&dc->links[i]);
 	}
 }
 
@@ -216,7 +217,7 @@ static bool create_links(
 		link_init_params.connector_index = i;
 		link_init_params.link_index = dc->link_count;
 		link_init_params.dc = dc;
-		link = link_create(&link_init_params);
+		link = dc->link_srv->create_link(&link_init_params);
 
 		if (link) {
 			dc->links[dc->link_count] = link;
@@ -238,7 +239,7 @@ static bool create_links(
 		link_init_params.dc = dc;
 		link_init_params.is_dpia_link = true;
 
-		link = link_create(&link_init_params);
+		link = dc->link_srv->create_link(&link_init_params);
 		if (link) {
 			dc->links[dc->link_count] = link;
 			link->dc = dc;
@@ -399,6 +400,14 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 {
 	int i;
 
+	/*
+	 * Don't adjust DRR while there's bandwidth optimizations pending to
+	 * avoid conflicting with firmware updates.
+	 */
+	if (dc->ctx->dce_version > DCE_VERSION_MAX)
+		if (dc->optimized_required || dc->wm_optimized_required)
+			return false;
+
 	stream->adjust.v_total_max = adjust->v_total_max;
 	stream->adjust.v_total_mid = adjust->v_total_mid;
 	stream->adjust.v_total_mid_frame_num = adjust->v_total_mid_frame_num;
@@ -814,6 +823,9 @@ static void dc_destruct(struct dc *dc)
 
 	dc_destroy_resource_pool(dc);
 
+	if (dc->link_srv)
+		link_destroy_link_service(&dc->link_srv);
+
 	if (dc->ctx->gpio_service)
 		dal_gpio_service_destroy(&dc->ctx->gpio_service);
 
@@ -876,6 +888,10 @@ static bool dc_construct_ctx(struct dc *dc,
 
 	dc->ctx = dc_ctx;
 
+	dc->link_srv = link_create_link_service();
+	if (!dc->link_srv)
+		return false;
+
 	return true;
 }
 
@@ -984,7 +1000,7 @@ static bool dc_construct(struct dc *dc,
 	dc->clk_mgr = dc_clk_mgr_create(dc->ctx, dc->res_pool->pp_smu, dc->res_pool->dccg);
 	if (!dc->clk_mgr)
 		goto fail;
-#ifdef CONFIG_DRM_AMD_DC_DCN
+#ifdef CONFIG_DRM_AMD_DC_FP
 	dc->clk_mgr->force_smu_not_present = init_params->force_smu_not_present;
 
 	if (dc->res_pool->funcs->update_bw_bounding_box) {
@@ -1057,6 +1073,53 @@ static void apply_ctx_interdependent_lock(struct dc *dc, struct dc_state *contex
 	}
 }
 
+static void phantom_pipe_blank(
+		struct dc *dc,
+		struct timing_generator *tg,
+		int width,
+		int height)
+{
+	struct dce_hwseq *hws = dc->hwseq;
+	enum dc_color_space color_space;
+	struct tg_color black_color = {0};
+	struct output_pixel_processor *opp = NULL;
+	uint32_t num_opps, opp_id_src0, opp_id_src1;
+	uint32_t otg_active_width, otg_active_height;
+	uint32_t i;
+
+	/* program opp dpg blank color */
+	color_space = COLOR_SPACE_SRGB;
+	color_space_to_black_color(dc, color_space, &black_color);
+
+	otg_active_width = width;
+	otg_active_height = height;
+
+	/* get the OPTC source */
+	tg->funcs->get_optc_source(tg, &num_opps, &opp_id_src0, &opp_id_src1);
+	ASSERT(opp_id_src0 < dc->res_pool->res_cap->num_opp);
+
+	for (i = 0; i < dc->res_pool->res_cap->num_opp; i++) {
+		if (dc->res_pool->opps[i] != NULL && dc->res_pool->opps[i]->inst == opp_id_src0) {
+			opp = dc->res_pool->opps[i];
+			break;
+		}
+	}
+
+	if (opp && opp->funcs->opp_set_disp_pattern_generator)
+		opp->funcs->opp_set_disp_pattern_generator(
+				opp,
+				CONTROLLER_DP_TEST_PATTERN_SOLID_COLOR,
+				CONTROLLER_DP_COLOR_SPACE_UDEFINED,
+				COLOR_DEPTH_UNDEFINED,
+				&black_color,
+				otg_active_width,
+				otg_active_height,
+				0);
+
+	if (tg->funcs->is_tg_enabled(tg))
+		hws->funcs.wait_for_blank_complete(opp);
+}
+
 static void disable_dangling_plane(struct dc *dc, struct dc_state *context)
 {
 	int i, j;
@@ -1115,8 +1178,14 @@ static void disable_dangling_plane(struct dc *dc, struct dc_state *context)
 			 * again for different use.
 			 */
 			if (old_stream->mall_stream_config.type == SUBVP_PHANTOM) {
-				if (tg->funcs->enable_crtc)
+				if (tg->funcs->enable_crtc) {
+					int main_pipe_width, main_pipe_height;
+
+					main_pipe_width = old_stream->mall_stream_config.paired_stream->dst.width;
+					main_pipe_height = old_stream->mall_stream_config.paired_stream->dst.height;
+					phantom_pipe_blank(dc, tg, main_pipe_width, main_pipe_height);
 					tg->funcs->enable_crtc(tg);
+				}
 			}
 			dc_rem_all_planes_for_stream(dc, old_stream, dangling_context);
 			disable_all_writeback_pipes_for_stream(dc, old_stream, dangling_context);
@@ -1199,7 +1268,7 @@ static void disable_vbios_mode_if_required(
 						pipe->stream_res.pix_clk_params.requested_pix_clk_100hz;
 
 					if (pix_clk_100hz != requested_pix_clk_100hz) {
-						link_set_dpms_off(pipe);
+						dc->link_srv->set_dpms_off(pipe);
 						pipe->stream->dpms_off = false;
 					}
 				}
@@ -1298,7 +1367,7 @@ static void detect_edp_presence(struct dc *dc)
 	int i;
 	int edp_num;
 
-	get_edp_links(dc, edp_links, &edp_num);
+	dc_get_edp_links(dc, edp_links, &edp_num);
 	if (!edp_num)
 		return;
 
@@ -1324,16 +1393,12 @@ void dc_hardware_init(struct dc *dc)
 void dc_init_callbacks(struct dc *dc,
 		const struct dc_callback_init *init_params)
 {
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	dc->ctx->cp_psp = init_params->cp_psp;
-#endif
 }
 
 void dc_deinit_callbacks(struct dc *dc)
 {
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	memset(&dc->ctx->cp_psp, 0, sizeof(dc->ctx->cp_psp));
-#endif
 }
 
 void dc_destroy(struct dc **dc)
@@ -1658,7 +1723,7 @@ bool dc_validate_boot_timing(const struct dc *dc,
 		return false;
 	}
 
-	if (link_is_edp_ilr_optimization_required(link, crtc_timing)) {
+	if (dc->link_srv->edp_is_ilr_optimization_required(link, crtc_timing)) {
 		DC_LOG_EVENT_LINK_TRAINING("Seamless boot disabled to optimize eDP link rate\n");
 		return false;
 	}
@@ -2001,53 +2066,6 @@ context_alloc_fail:
 	return res;
 }
 
-/* TODO: When the transition to the new commit sequence is done, remove this
- * function in favor of dc_commit_streams. */
-bool dc_commit_state(struct dc *dc, struct dc_state *context)
-{
-	enum dc_status result = DC_ERROR_UNEXPECTED;
-	int i;
-
-	/* TODO: Since change commit sequence can have a huge impact,
-	 * we decided to only enable it for DCN3x. However, as soon as
-	 * we get more confident about this change we'll need to enable
-	 * the new sequence for all ASICs. */
-	if (dc->ctx->dce_version >= DCN_VERSION_3_2) {
-		result = dc_commit_streams(dc, context->streams, context->stream_count);
-		return result == DC_OK;
-	}
-
-	if (!streams_changed(dc, context->streams, context->stream_count)) {
-		return DC_OK;
-	}
-
-	DC_LOG_DC("%s: %d streams\n",
-				__func__, context->stream_count);
-
-	for (i = 0; i < context->stream_count; i++) {
-		struct dc_stream_state *stream = context->streams[i];
-
-		dc_stream_log(dc, stream);
-	}
-
-	/*
-	 * Previous validation was perfomred with fast_validation = true and
-	 * the full DML state required for hardware programming was skipped.
-	 *
-	 * Re-validate here to calculate these parameters / watermarks.
-	 */
-	result = dc_validate_global_state(dc, context, false);
-	if (result != DC_OK) {
-		DC_LOG_ERROR("DC commit global validation failure: %s (%d)",
-			     dc_status_to_str(result), result);
-		return result;
-	}
-
-	result = dc_commit_state_no_check(dc, context);
-
-	return (result == DC_OK);
-}
-
 bool dc_acquire_release_mpc_3dlut(
 		struct dc *dc, bool acquire,
 		struct dc_stream_state *stream,
@@ -2134,27 +2152,33 @@ void dc_post_update_surfaces_to_stream(struct dc *dc)
 
 	post_surface_trace(dc);
 
-	if (dc->ctx->dce_version >= DCE_VERSION_MAX)
-		TRACE_DCN_CLOCK_STATE(&context->bw_ctx.bw.dcn.clk);
-	else
+	/*
+	 * Only relevant for DCN behavior where we can guarantee the optimization
+	 * is safe to apply - retain the legacy behavior for DCE.
+	 */
+
+	if (dc->ctx->dce_version < DCE_VERSION_MAX)
 		TRACE_DCE_CLOCK_STATE(&context->bw_ctx.bw.dce);
+	else {
+		TRACE_DCN_CLOCK_STATE(&context->bw_ctx.bw.dcn.clk);
 
-	if (is_flip_pending_in_pipes(dc, context))
-		return;
+		if (is_flip_pending_in_pipes(dc, context))
+			return;
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++)
-		if (context->res_ctx.pipe_ctx[i].stream == NULL ||
-		    context->res_ctx.pipe_ctx[i].plane_state == NULL) {
-			context->res_ctx.pipe_ctx[i].pipe_idx = i;
-			dc->hwss.disable_plane(dc, &context->res_ctx.pipe_ctx[i]);
-		}
+		for (i = 0; i < dc->res_pool->pipe_count; i++)
+			if (context->res_ctx.pipe_ctx[i].stream == NULL ||
+					context->res_ctx.pipe_ctx[i].plane_state == NULL) {
+				context->res_ctx.pipe_ctx[i].pipe_idx = i;
+				dc->hwss.disable_plane(dc, &context->res_ctx.pipe_ctx[i]);
+			}
 
-	process_deferred_updates(dc);
+		process_deferred_updates(dc);
 
-	dc->hwss.optimize_bandwidth(dc, context);
+		dc->hwss.optimize_bandwidth(dc, context);
 
-	if (dc->debug.enable_double_buffered_dsc_pg_support)
-		dc->hwss.update_dsc_pg(dc, context, true);
+		if (dc->debug.enable_double_buffered_dsc_pg_support)
+			dc->hwss.update_dsc_pg(dc, context, true);
+	}
 
 	dc->optimized_required = false;
 	dc->wm_optimized_required = false;
@@ -3173,7 +3197,9 @@ static void commit_planes_do_stream_update(struct dc *dc,
 				dc->hwss.update_info_frame(pipe_ctx);
 
 				if (dc_is_dp_signal(pipe_ctx->stream->signal))
-					link_dp_source_sequence_trace(pipe_ctx->stream->link, DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
+					dc->link_srv->dp_trace_source_sequence(
+							pipe_ctx->stream->link,
+							DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
 			}
 
 			if (stream_update->hdr_static_metadata &&
@@ -3209,13 +3235,15 @@ static void commit_planes_do_stream_update(struct dc *dc,
 				continue;
 
 			if (stream_update->dsc_config)
-				link_update_dsc_config(pipe_ctx);
+				dc->link_srv->update_dsc_config(pipe_ctx);
 
 			if (stream_update->mst_bw_update) {
 				if (stream_update->mst_bw_update->is_increase)
-					link_increase_mst_payload(pipe_ctx, stream_update->mst_bw_update->mst_stream_bw);
+					dc->link_srv->increase_mst_payload(pipe_ctx,
+							stream_update->mst_bw_update->mst_stream_bw);
  				else
-					link_reduce_mst_payload(pipe_ctx, stream_update->mst_bw_update->mst_stream_bw);
+					dc->link_srv->reduce_mst_payload(pipe_ctx,
+							stream_update->mst_bw_update->mst_stream_bw);
  			}
 
 			if (stream_update->pending_test_pattern) {
@@ -3229,7 +3257,7 @@ static void commit_planes_do_stream_update(struct dc *dc,
 
 			if (stream_update->dpms_off) {
 				if (*stream_update->dpms_off) {
-					link_set_dpms_off(pipe_ctx);
+					dc->link_srv->set_dpms_off(pipe_ctx);
 					/* for dpms, keep acquired resources*/
 					if (pipe_ctx->stream_res.audio && !dc->debug.az_endpoint_mute_only)
 						pipe_ctx->stream_res.audio->funcs->az_disable(pipe_ctx->stream_res.audio);
@@ -3239,7 +3267,7 @@ static void commit_planes_do_stream_update(struct dc *dc,
 				} else {
 					if (get_seamless_boot_stream_count(context) == 0)
 						dc->hwss.prepare_bandwidth(dc, dc->current_state);
-					link_set_dpms_on(dc->current_state, pipe_ctx);
+					dc->link_srv->set_dpms_on(dc->current_state, pipe_ctx);
 				}
 			}
 
@@ -3466,22 +3494,6 @@ static void commit_planes_for_stream(struct dc *dc,
 
 	dc_dmub_update_dirty_rect(dc, surface_count, stream, srf_updates, context);
 
-	if (update_type != UPDATE_TYPE_FAST) {
-		for (i = 0; i < dc->res_pool->pipe_count; i++) {
-			struct pipe_ctx *new_pipe = &context->res_ctx.pipe_ctx[i];
-
-			if ((new_pipe->stream && new_pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) ||
-					subvp_prev_use) {
-				// If old context or new context has phantom pipes, apply
-				// the phantom timings now. We can't change the phantom
-				// pipe configuration safely without driver acquiring
-				// the DMCUB lock first.
-				dc->hwss.apply_ctx_to_hw(dc, context);
-				break;
-			}
-		}
-	}
-
 	// Stream updates
 	if (stream_update)
 		commit_planes_do_stream_update(dc, stream, stream_update, update_type, context);
@@ -3510,14 +3522,9 @@ static void commit_planes_for_stream(struct dc *dc,
 		/* Since phantom pipe programming is moved to post_unlock_program_front_end,
 		 * move the SubVP lock to after the phantom pipes have been setup
 		 */
-		if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
-			if (dc->hwss.subvp_pipe_control_lock)
-				dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use);
-		} else {
-			if (dc->hwss.subvp_pipe_control_lock)
-				dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use);
-		}
-
+		if (dc->hwss.subvp_pipe_control_lock)
+			dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes,
+							 NULL, subvp_prev_use);
 		return;
 	}
 
@@ -3702,6 +3709,9 @@ static void commit_planes_for_stream(struct dc *dc,
 		}
 	}
 
+	if (update_type != UPDATE_TYPE_FAST)
+		dc->hwss.post_unlock_program_front_end(dc, context);
+
 	if (subvp_prev_use && !subvp_curr_use) {
 		/* If disabling subvp, disable phantom streams after front end
 		 * programming has completed (we turn on phantom OTG in order
@@ -3711,15 +3721,8 @@ static void commit_planes_for_stream(struct dc *dc,
 	}
 
 	if (update_type != UPDATE_TYPE_FAST)
-		dc->hwss.post_unlock_program_front_end(dc, context);
-	if (update_type != UPDATE_TYPE_FAST)
 		if (dc->hwss.commit_subvp_config)
 			dc->hwss.commit_subvp_config(dc, context);
-
-	if (update_type != UPDATE_TYPE_FAST)
-		if (dc->hwss.commit_subvp_config)
-			dc->hwss.commit_subvp_config(dc, context);
-
 	/* Since phantom pipe programming is moved to post_unlock_program_front_end,
 	 * move the SubVP lock to after the phantom pipes have been setup
 	 */
@@ -4083,24 +4086,30 @@ void dc_commit_updates_for_stream(struct dc *dc,
 	struct dc_context *dc_ctx = dc->ctx;
 	int i, j;
 
+	stream_status = dc_stream_get_status(stream);
+	context = dc->current_state;
+
+	update_type = dc_check_update_surfaces_for_stream(
+				dc, srf_updates, surface_count, stream_update, stream_status);
+
 	/* TODO: Since change commit sequence can have a huge impact,
 	 * we decided to only enable it for DCN3x. However, as soon as
 	 * we get more confident about this change we'll need to enable
 	 * the new sequence for all ASICs.
 	 */
 	if (dc->ctx->dce_version >= DCN_VERSION_3_2) {
+		/*
+		 * Previous frame finished and HW is ready for optimization.
+		 */
+		if (update_type == UPDATE_TYPE_FAST)
+			dc_post_update_surfaces_to_stream(dc);
+
 		dc_update_planes_and_stream(dc, srf_updates,
 					    surface_count, stream,
 					    stream_update);
 		return;
 	}
 
-	stream_status = dc_stream_get_status(stream);
-	context = dc->current_state;
-
-	update_type = dc_check_update_surfaces_for_stream(
-				dc, srf_updates, surface_count, stream_update, stream_status);
-
 	if (update_type >= update_surface_trace_level)
 		update_surface_trace(dc, srf_updates, surface_count);
 
@@ -4123,12 +4132,9 @@ void dc_commit_updates_for_stream(struct dc *dc,
 			if (new_pipe->plane_state && new_pipe->plane_state != old_pipe->plane_state)
 				new_pipe->plane_state->force_full_update = true;
 		}
-	} else if (update_type == UPDATE_TYPE_FAST && dc_ctx->dce_version >= DCE_VERSION_MAX) {
+	} else if (update_type == UPDATE_TYPE_FAST) {
 		/*
 		 * Previous frame finished and HW is ready for optimization.
-		 *
-		 * Only relevant for DCN behavior where we can guarantee the optimization
-		 * is safe to apply - retain the legacy behavior for DCE.
 		 */
 		dc_post_update_surfaces_to_stream(dc);
 	}
@@ -4305,7 +4311,7 @@ void dc_resume(struct dc *dc)
 	uint32_t i;
 
 	for (i = 0; i < dc->link_count; i++)
-		link_resume(dc->links[i]);
+		dc->link_srv->resume(dc->links[i]);
 }
 
 bool dc_is_dmcu_initialized(struct dc *dc)
@@ -4317,157 +4323,6 @@ bool dc_is_dmcu_initialized(struct dc *dc)
 	return false;
 }
 
-bool dc_is_oem_i2c_device_present(
-	struct dc *dc,
-	size_t slave_address)
-{
-	if (dc->res_pool->oem_device)
-		return dce_i2c_oem_device_present(
-			dc->res_pool,
-			dc->res_pool->oem_device,
-			slave_address);
-
-	return false;
-}
-
-bool dc_submit_i2c(
-		struct dc *dc,
-		uint32_t link_index,
-		struct i2c_command *cmd)
-{
-
-	struct dc_link *link = dc->links[link_index];
-	struct ddc_service *ddc = link->ddc;
-	return dce_i2c_submit_command(
-		dc->res_pool,
-		ddc->ddc_pin,
-		cmd);
-}
-
-bool dc_submit_i2c_oem(
-		struct dc *dc,
-		struct i2c_command *cmd)
-{
-	struct ddc_service *ddc = dc->res_pool->oem_device;
-	if (ddc)
-		return dce_i2c_submit_command(
-			dc->res_pool,
-			ddc->ddc_pin,
-			cmd);
-
-	return false;
-}
-
-static bool link_add_remote_sink_helper(struct dc_link *dc_link, struct dc_sink *sink)
-{
-	if (dc_link->sink_count >= MAX_SINKS_PER_LINK) {
-		BREAK_TO_DEBUGGER();
-		return false;
-	}
-
-	dc_sink_retain(sink);
-
-	dc_link->remote_sinks[dc_link->sink_count] = sink;
-	dc_link->sink_count++;
-
-	return true;
-}
-
-/*
- * dc_link_add_remote_sink() - Create a sink and attach it to an existing link
- *
- * EDID length is in bytes
- */
-struct dc_sink *dc_link_add_remote_sink(
-		struct dc_link *link,
-		const uint8_t *edid,
-		int len,
-		struct dc_sink_init_data *init_data)
-{
-	struct dc_sink *dc_sink;
-	enum dc_edid_status edid_status;
-
-	if (len > DC_MAX_EDID_BUFFER_SIZE) {
-		dm_error("Max EDID buffer size breached!\n");
-		return NULL;
-	}
-
-	if (!init_data) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	if (!init_data->link) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dc_sink = dc_sink_create(init_data);
-
-	if (!dc_sink)
-		return NULL;
-
-	memmove(dc_sink->dc_edid.raw_edid, edid, len);
-	dc_sink->dc_edid.length = len;
-
-	if (!link_add_remote_sink_helper(
-			link,
-			dc_sink))
-		goto fail_add_sink;
-
-	edid_status = dm_helpers_parse_edid_caps(
-			link,
-			&dc_sink->dc_edid,
-			&dc_sink->edid_caps);
-
-	/*
-	 * Treat device as no EDID device if EDID
-	 * parsing fails
-	 */
-	if (edid_status != EDID_OK && edid_status != EDID_PARTIAL_VALID) {
-		dc_sink->dc_edid.length = 0;
-		dm_error("Bad EDID, status%d!\n", edid_status);
-	}
-
-	return dc_sink;
-
-fail_add_sink:
-	dc_sink_release(dc_sink);
-	return NULL;
-}
-
-/*
- * dc_link_remove_remote_sink() - Remove a remote sink from a dc_link
- *
- * Note that this just removes the struct dc_sink - it doesn't
- * program hardware or alter other members of dc_link
- */
-void dc_link_remove_remote_sink(struct dc_link *link, struct dc_sink *sink)
-{
-	int i;
-
-	if (!link->sink_count) {
-		BREAK_TO_DEBUGGER();
-		return;
-	}
-
-	for (i = 0; i < link->sink_count; i++) {
-		if (link->remote_sinks[i] == sink) {
-			dc_sink_release(sink);
-			link->remote_sinks[i] = NULL;
-
-			/* shrink array to remove empty place */
-			while (i < link->sink_count - 1) {
-				link->remote_sinks[i] = link->remote_sinks[i+1];
-				i++;
-			}
-			link->remote_sinks[i] = NULL;
-			link->sink_count--;
-			return;
-		}
-	}
-}
-
 void get_clock_requirements_for_state(struct dc_state *state, struct AsicStateEx *info)
 {
 	info->displayClock				= (unsigned int)state->bw_ctx.bw.dcn.clk.dispclk_khz;
@@ -4990,7 +4845,7 @@ void dc_notify_vsync_int_state(struct dc *dc, struct dc_stream_state *stream, bo
 		return;
 	}
 
-	get_edp_links(dc, edp_links, &edp_num);
+	dc_get_edp_links(dc, edp_links, &edp_num);
 
 	/* Determine panel inst */
 	for (i = 0; i < edp_num; i++) {
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index 652270a0b498..2acbf692193f 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -73,28 +73,38 @@ struct out_csc_color_matrix_type {
 
 static const struct out_csc_color_matrix_type output_csc_matrix[] = {
 	{ COLOR_SPACE_RGB_TYPE,
-		{ 0x2000, 0, 0, 0, 0, 0x2000, 0, 0, 0, 0, 0x2000, 0} },
+		{ 0x2000, 0,      0,      0,
+		  0,      0x2000, 0,      0,
+		  0,      0,      0x2000, 0} },
 	{ COLOR_SPACE_RGB_LIMITED_TYPE,
-		{ 0x1B67, 0, 0, 0x201, 0, 0x1B67, 0, 0x201, 0, 0, 0x1B67, 0x201} },
+		{ 0x1B67, 0,      0,      0x201,
+		  0,      0x1B67, 0,      0x201,
+		  0,      0,      0x1B67, 0x201} },
 	{ COLOR_SPACE_YCBCR601_TYPE,
-		{ 0xE04, 0xF444, 0xFDB9, 0x1004, 0x831, 0x1016, 0x320, 0x201, 0xFB45,
-				0xF6B7, 0xE04, 0x1004} },
+		{ 0xE04,  0xF444, 0xFDB9, 0x1004,
+		  0x831,  0x1016, 0x320,  0x201,
+		  0xFB45, 0xF6B7, 0xE04,  0x1004} },
 	{ COLOR_SPACE_YCBCR709_TYPE,
-		{ 0xE04, 0xF345, 0xFEB7, 0x1004, 0x5D3, 0x1399, 0x1FA,
-				0x201, 0xFCCA, 0xF533, 0xE04, 0x1004} },
+		{ 0xE04,  0xF345, 0xFEB7, 0x1004,
+		  0x5D3,  0x1399, 0x1FA,  0x201,
+		  0xFCCA, 0xF533, 0xE04,  0x1004} },
 	/* TODO: correct values below */
 	{ COLOR_SPACE_YCBCR601_LIMITED_TYPE,
-		{ 0xE00, 0xF447, 0xFDB9, 0x1000, 0x991,
-				0x12C9, 0x3A6, 0x200, 0xFB47, 0xF6B9, 0xE00, 0x1000} },
+		{ 0xE00,  0xF447, 0xFDB9, 0x1000,
+		  0x991,  0x12C9, 0x3A6,  0x200,
+		  0xFB47, 0xF6B9, 0xE00,  0x1000} },
 	{ COLOR_SPACE_YCBCR709_LIMITED_TYPE,
-		{ 0xE00, 0xF349, 0xFEB7, 0x1000, 0x6CE, 0x16E3,
-				0x24F, 0x200, 0xFCCB, 0xF535, 0xE00, 0x1000} },
+		{ 0xE00, 0xF349, 0xFEB7, 0x1000,
+		  0x6CE, 0x16E3, 0x24F,  0x200,
+		  0xFCCB, 0xF535, 0xE00, 0x1000} },
 	{ COLOR_SPACE_YCBCR2020_TYPE,
-		{ 0x1000, 0xF149, 0xFEB7, 0x1004, 0x0868, 0x15B2,
-				0x01E6, 0x201, 0xFB88, 0xF478, 0x1000, 0x1004} },
+		{ 0x1000, 0xF149, 0xFEB7, 0x1004,
+		  0x0868, 0x15B2, 0x01E6, 0x201,
+		  0xFB88, 0xF478, 0x1000, 0x1004} },
 	{ COLOR_SPACE_YCBCR709_BLACK_TYPE,
-		{ 0x0000, 0x0000, 0x0000, 0x1000, 0x0000, 0x0000,
-				0x0000, 0x0200, 0x0000, 0x0000, 0x0000, 0x1000} },
+		{ 0x0000, 0x0000, 0x0000, 0x1000,
+		  0x0000, 0x0000, 0x0000, 0x0200,
+		  0x0000, 0x0000, 0x0000, 0x1000} },
 };
 
 static bool is_rgb_type(
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c
index 74e465ba158d..30c0644d4418 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c
@@ -48,7 +48,7 @@ static bool is_dig_link_enc_stream(struct dc_stream_state *stream)
 					/* DIGs do not support DP2.0 streams with 128b/132b encoding. */
 					struct dc_link_settings link_settings = {0};
 
-					link_decide_link_settings(stream, &link_settings);
+					stream->ctx->dc->link_srv->dp_decide_link_settings(stream, &link_settings);
 					if ((link_settings.link_rate >= LINK_RATE_LOW) &&
 							link_settings.link_rate <= LINK_RATE_HIGH3) {
 						is_dig_stream = true;
@@ -622,7 +622,6 @@ bool link_enc_cfg_validate(struct dc *dc, struct dc_state *state)
 	int i, j;
 	uint8_t valid_count = 0;
 	uint8_t dig_stream_count = 0;
-	int matching_stream_ptrs = 0;
 	int eng_ids_per_ep_id[MAX_PIPES] = {0};
 	int ep_ids_per_eng_id[MAX_PIPES] = {0};
 	int valid_bitmap = 0;
@@ -645,9 +644,7 @@ bool link_enc_cfg_validate(struct dc *dc, struct dc_state *state)
 		struct link_enc_assignment assignment = state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i];
 
 		if (assignment.valid) {
-			if (assignment.stream == state->streams[i])
-				matching_stream_ptrs++;
-			else
+			if (assignment.stream != state->streams[i])
 				valid_stream_ptrs = false;
 		}
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
index a951e10416ee..18e098568cb4 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
@@ -34,70 +34,447 @@
  * in this file which calls link functions.
  */
 #include "link.h"
+#include "dce/dce_i2c.h"
+struct dc_link *dc_get_link_at_index(struct dc *dc, uint32_t link_index)
+{
+	return dc->links[link_index];
+}
+
+void dc_get_edp_links(const struct dc *dc,
+		struct dc_link **edp_links,
+		int *edp_num)
+{
+	int i;
+
+	*edp_num = 0;
+	for (i = 0; i < dc->link_count; i++) {
+		// report any eDP links, even unconnected DDI's
+		if (!dc->links[i])
+			continue;
+		if (dc->links[i]->connector_signal == SIGNAL_TYPE_EDP) {
+			edp_links[*edp_num] = dc->links[i];
+			if (++(*edp_num) == MAX_NUM_EDP)
+				return;
+		}
+	}
+}
+
+bool dc_get_edp_link_panel_inst(const struct dc *dc,
+		const struct dc_link *link,
+		unsigned int *inst_out)
+{
+	struct dc_link *edp_links[MAX_NUM_EDP];
+	int edp_num, i;
+
+	*inst_out = 0;
+	if (link->connector_signal != SIGNAL_TYPE_EDP)
+		return false;
+	dc_get_edp_links(dc, edp_links, &edp_num);
+	for (i = 0; i < edp_num; i++) {
+		if (link == edp_links[i])
+			break;
+		(*inst_out)++;
+	}
+	return true;
+}
 
 bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 {
-	return link_detect(link, reason);
+	return link->dc->link_srv->detect_link(link, reason);
 }
 
 bool dc_link_detect_connection_type(struct dc_link *link,
 		enum dc_connection_type *type)
 {
-	return link_detect_connection_type(link, type);
+	return link->dc->link_srv->detect_connection_type(link, type);
 }
 
 const struct dc_link_status *dc_link_get_status(const struct dc_link *link)
 {
-	return link_get_status(link);
+	return link->dc->link_srv->get_status(link);
 }
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 
 /* return true if the connected receiver supports the hdcp version */
 bool dc_link_is_hdcp14(struct dc_link *link, enum signal_type signal)
 {
-	return link_is_hdcp14(link, signal);
+	return link->dc->link_srv->is_hdcp1x_supported(link, signal);
 }
 
 bool dc_link_is_hdcp22(struct dc_link *link, enum signal_type signal)
 {
-	return link_is_hdcp22(link, signal);
+	return link->dc->link_srv->is_hdcp2x_supported(link, signal);
 }
-#endif
 
 void dc_link_clear_dprx_states(struct dc_link *link)
 {
-	link_clear_dprx_states(link);
+	link->dc->link_srv->clear_dprx_states(link);
 }
 
 bool dc_link_reset_cur_dp_mst_topology(struct dc_link *link)
 {
-	return link_reset_cur_dp_mst_topology(link);
+	return link->dc->link_srv->reset_cur_dp_mst_topology(link);
 }
 
 uint32_t dc_link_bandwidth_kbps(
 	const struct dc_link *link,
 	const struct dc_link_settings *link_settings)
 {
-	return dp_link_bandwidth_kbps(link, link_settings);
-}
-
-uint32_t dc_bandwidth_in_kbps_from_timing(
-	const struct dc_crtc_timing *timing)
-{
-	return link_timing_bandwidth_kbps(timing);
+	return link->dc->link_srv->dp_link_bandwidth_kbps(link, link_settings);
 }
 
 void dc_get_cur_link_res_map(const struct dc *dc, uint32_t *map)
 {
-	link_get_cur_res_map(dc, map);
+	dc->link_srv->get_cur_res_map(dc, map);
 }
 
 void dc_restore_link_res_map(const struct dc *dc, uint32_t *map)
 {
-	link_restore_res_map(dc, map);
+	dc->link_srv->restore_res_map(dc, map);
 }
 
 bool dc_link_update_dsc_config(struct pipe_ctx *pipe_ctx)
 {
-	return link_update_dsc_config(pipe_ctx);
+	struct dc_link *link = pipe_ctx->stream->link;
+
+	return link->dc->link_srv->update_dsc_config(pipe_ctx);
+}
+
+bool dc_is_oem_i2c_device_present(
+	struct dc *dc,
+	size_t slave_address)
+{
+	if (dc->res_pool->oem_device)
+		return dce_i2c_oem_device_present(
+			dc->res_pool,
+			dc->res_pool->oem_device,
+			slave_address);
+
+	return false;
+}
+
+bool dc_submit_i2c(
+		struct dc *dc,
+		uint32_t link_index,
+		struct i2c_command *cmd)
+{
+
+	struct dc_link *link = dc->links[link_index];
+	struct ddc_service *ddc = link->ddc;
+
+	return dce_i2c_submit_command(
+		dc->res_pool,
+		ddc->ddc_pin,
+		cmd);
+}
+
+bool dc_submit_i2c_oem(
+		struct dc *dc,
+		struct i2c_command *cmd)
+{
+	struct ddc_service *ddc = dc->res_pool->oem_device;
+
+	if (ddc)
+		return dce_i2c_submit_command(
+			dc->res_pool,
+			ddc->ddc_pin,
+			cmd);
+
+	return false;
+}
+
+void dc_link_dp_handle_automated_test(struct dc_link *link)
+{
+	link->dc->link_srv->dp_handle_automated_test(link);
+}
+
+bool dc_link_dp_set_test_pattern(
+	struct dc_link *link,
+	enum dp_test_pattern test_pattern,
+	enum dp_test_pattern_color_space test_pattern_color_space,
+	const struct link_training_settings *p_link_settings,
+	const unsigned char *p_custom_pattern,
+	unsigned int cust_pattern_size)
+{
+	return link->dc->link_srv->dp_set_test_pattern(link, test_pattern,
+			test_pattern_color_space, p_link_settings,
+			p_custom_pattern, cust_pattern_size);
+}
+
+void dc_link_set_drive_settings(struct dc *dc,
+				struct link_training_settings *lt_settings,
+				struct dc_link *link)
+{
+	struct link_resource link_res;
+
+	dc->link_srv->get_cur_link_res(link, &link_res);
+	dc->link_srv->dp_set_drive_settings(link, &link_res, lt_settings);
+}
+
+void dc_link_set_preferred_link_settings(struct dc *dc,
+					 struct dc_link_settings *link_setting,
+					 struct dc_link *link)
+{
+	dc->link_srv->dp_set_preferred_link_settings(dc, link_setting, link);
+}
+
+void dc_link_set_preferred_training_settings(struct dc *dc,
+		struct dc_link_settings *link_setting,
+		struct dc_link_training_overrides *lt_overrides,
+		struct dc_link *link,
+		bool skip_immediate_retrain)
+{
+	dc->link_srv->dp_set_preferred_training_settings(dc, link_setting,
+			lt_overrides, link, skip_immediate_retrain);
+}
+
+bool dc_dp_trace_is_initialized(struct dc_link *link)
+{
+	return link->dc->link_srv->dp_trace_is_initialized(link);
+}
+
+void dc_dp_trace_set_is_logged_flag(struct dc_link *link,
+		bool in_detection,
+		bool is_logged)
+{
+	link->dc->link_srv->dp_trace_set_is_logged_flag(link, in_detection, is_logged);
+}
+
+bool dc_dp_trace_is_logged(struct dc_link *link, bool in_detection)
+{
+	return link->dc->link_srv->dp_trace_is_logged(link, in_detection);
+}
+
+unsigned long long dc_dp_trace_get_lt_end_timestamp(struct dc_link *link,
+		bool in_detection)
+{
+	return link->dc->link_srv->dp_trace_get_lt_end_timestamp(link, in_detection);
+}
+
+const struct dp_trace_lt_counts *dc_dp_trace_get_lt_counts(struct dc_link *link,
+		bool in_detection)
+{
+	return link->dc->link_srv->dp_trace_get_lt_counts(link, in_detection);
+}
+
+unsigned int dc_dp_trace_get_link_loss_count(struct dc_link *link)
+{
+	return link->dc->link_srv->dp_trace_get_link_loss_count(link);
+}
+
+struct dc_sink *dc_link_add_remote_sink(
+		struct dc_link *link,
+		const uint8_t *edid,
+		int len,
+		struct dc_sink_init_data *init_data)
+{
+	return link->dc->link_srv->add_remote_sink(link, edid, len, init_data);
+}
+
+void dc_link_remove_remote_sink(struct dc_link *link, struct dc_sink *sink)
+{
+	link->dc->link_srv->remove_remote_sink(link, sink);
+}
+
+int dc_link_aux_transfer_raw(struct ddc_service *ddc,
+		struct aux_payload *payload,
+		enum aux_return_code_type *operation_result)
+{
+	const struct dc *dc = ddc->link->dc;
+
+	return dc->link_srv->aux_transfer_raw(
+			ddc, payload, operation_result);
+}
+
+uint32_t dc_link_bw_kbps_from_raw_frl_link_rate_data(const struct dc *dc, uint8_t bw)
+{
+	return dc->link_srv->bw_kbps_from_raw_frl_link_rate_data(bw);
+}
+
+bool dc_link_decide_edp_link_settings(struct dc_link *link,
+		struct dc_link_settings *link_setting, uint32_t req_bw)
+{
+	return link->dc->link_srv->edp_decide_link_settings(link, link_setting, req_bw);
+}
+
+
+bool dc_link_dp_get_max_link_enc_cap(const struct dc_link *link,
+		struct dc_link_settings *max_link_enc_cap)
+{
+	return link->dc->link_srv->dp_get_max_link_enc_cap(link, max_link_enc_cap);
+}
+
+enum dp_link_encoding dc_link_dp_mst_decide_link_encoding_format(
+		const struct dc_link *link)
+{
+	return link->dc->link_srv->mst_decide_link_encoding_format(link);
+}
+
+const struct dc_link_settings *dc_link_get_link_cap(const struct dc_link *link)
+{
+	return link->dc->link_srv->dp_get_verified_link_cap(link);
+}
+
+bool dc_link_is_dp_sink_present(struct dc_link *link)
+{
+	return link->dc->link_srv->dp_is_sink_present(link);
+}
+
+bool dc_link_is_fec_supported(const struct dc_link *link)
+{
+	return link->dc->link_srv->dp_is_fec_supported(link);
+}
+
+void dc_link_overwrite_extended_receiver_cap(
+		struct dc_link *link)
+{
+	link->dc->link_srv->dp_overwrite_extended_receiver_cap(link);
+}
+
+bool dc_link_should_enable_fec(const struct dc_link *link)
+{
+	return link->dc->link_srv->dp_should_enable_fec(link);
+}
+
+int dc_link_dp_dpia_handle_usb4_bandwidth_allocation_for_link(
+		struct dc_link *link, int peak_bw)
+{
+	return link->dc->link_srv->dpia_handle_usb4_bandwidth_allocation_for_link(link, peak_bw);
+}
+
+void dc_link_handle_usb4_bw_alloc_response(struct dc_link *link, uint8_t bw, uint8_t result)
+{
+	link->dc->link_srv->dpia_handle_bw_alloc_response(link, bw, result);
+}
+
+bool dc_link_check_link_loss_status(
+	struct dc_link *link,
+	union hpd_irq_data *hpd_irq_dpcd_data)
+{
+	return link->dc->link_srv->dp_parse_link_loss_status(link, hpd_irq_dpcd_data);
+}
+
+bool dc_link_dp_allow_hpd_rx_irq(const struct dc_link *link)
+{
+	return link->dc->link_srv->dp_should_allow_hpd_rx_irq(link);
+}
+
+void dc_link_dp_handle_link_loss(struct dc_link *link)
+{
+	link->dc->link_srv->dp_handle_link_loss(link);
+}
+
+enum dc_status dc_link_dp_read_hpd_rx_irq_data(
+	struct dc_link *link,
+	union hpd_irq_data *irq_data)
+{
+	return link->dc->link_srv->dp_read_hpd_rx_irq_data(link, irq_data);
+}
+
+bool dc_link_handle_hpd_rx_irq(struct dc_link *link,
+		union hpd_irq_data *out_hpd_irq_dpcd_data, bool *out_link_loss,
+		bool defer_handling, bool *has_left_work)
+{
+	return link->dc->link_srv->dp_handle_hpd_rx_irq(link, out_hpd_irq_dpcd_data,
+			out_link_loss, defer_handling, has_left_work);
+}
+
+void dc_link_dp_receiver_power_ctrl(struct dc_link *link, bool on)
+{
+	link->dc->link_srv->dpcd_write_rx_power_ctrl(link, on);
+}
+
+enum lttpr_mode dc_link_decide_lttpr_mode(struct dc_link *link,
+		struct dc_link_settings *link_setting)
+{
+	return link->dc->link_srv->dp_decide_lttpr_mode(link, link_setting);
+}
+
+void dc_link_edp_panel_backlight_power_on(struct dc_link *link, bool wait_for_hpd)
+{
+	link->dc->link_srv->edp_panel_backlight_power_on(link, wait_for_hpd);
+}
+
+int dc_link_get_backlight_level(const struct dc_link *link)
+{
+	return link->dc->link_srv->edp_get_backlight_level(link);
+}
+
+bool dc_link_get_backlight_level_nits(struct dc_link *link,
+		uint32_t *backlight_millinits_avg,
+		uint32_t *backlight_millinits_peak)
+{
+	return link->dc->link_srv->edp_get_backlight_level_nits(link,
+			backlight_millinits_avg,
+			backlight_millinits_peak);
+}
+
+bool dc_link_set_backlight_level(const struct dc_link *link,
+		uint32_t backlight_pwm_u16_16,
+		uint32_t frame_ramp)
+{
+	return link->dc->link_srv->edp_set_backlight_level(link,
+			backlight_pwm_u16_16, frame_ramp);
+}
+
+bool dc_link_set_backlight_level_nits(struct dc_link *link,
+		bool isHDR,
+		uint32_t backlight_millinits,
+		uint32_t transition_time_in_ms)
+{
+	return link->dc->link_srv->edp_set_backlight_level_nits(link, isHDR,
+			backlight_millinits, transition_time_in_ms);
+}
+
+int dc_link_get_target_backlight_pwm(const struct dc_link *link)
+{
+	return link->dc->link_srv->edp_get_target_backlight_pwm(link);
+}
+
+bool dc_link_get_psr_state(const struct dc_link *link, enum dc_psr_state *state)
+{
+	return link->dc->link_srv->edp_get_psr_state(link, state);
+}
+
+bool dc_link_set_psr_allow_active(struct dc_link *link, const bool *allow_active,
+		bool wait, bool force_static, const unsigned int *power_opts)
+{
+	return link->dc->link_srv->edp_set_psr_allow_active(link, allow_active, wait,
+			force_static, power_opts);
+}
+
+bool dc_link_setup_psr(struct dc_link *link,
+		const struct dc_stream_state *stream, struct psr_config *psr_config,
+		struct psr_context *psr_context)
+{
+	return link->dc->link_srv->edp_setup_psr(link, stream, psr_config, psr_context);
+}
+
+bool dc_link_wait_for_t12(struct dc_link *link)
+{
+	return link->dc->link_srv->edp_wait_for_t12(link);
+}
+
+bool dc_link_get_hpd_state(struct dc_link *link)
+{
+	return link->dc->link_srv->get_hpd_state(link);
+}
+
+void dc_link_enable_hpd(const struct dc_link *link)
+{
+	link->dc->link_srv->enable_hpd(link);
+}
+
+void dc_link_disable_hpd(const struct dc_link *link)
+{
+	link->dc->link_srv->disable_hpd(link);
+}
+
+void dc_link_enable_hpd_filter(struct dc_link *link, bool enable)
+{
+	link->dc->link_srv->enable_hpd_filter(link, enable);
+}
+
+bool dc_link_validate(struct dc *dc, const struct dc_stream_state *streams, const unsigned int count)
+{
+	return dc->link_srv->validate_dpia_bandwidth(streams, count);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index d9f2ef242b0f..85d54bfb595c 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -232,7 +232,7 @@ struct resource_pool *dc_create_resource_pool(struct dc  *dc,
 				init_data->num_virtual_links, dc);
 		break;
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 	case DCN_VERSION_1_0:
 	case DCN_VERSION_1_01:
 		res_pool = dcn10_create_resource_pool(init_data, dc);
@@ -276,7 +276,7 @@ struct resource_pool *dc_create_resource_pool(struct dc  *dc,
 	case DCN_VERSION_3_21:
 		res_pool = dcn321_create_resource_pool(init_data, dc);
 		break;
-#endif
+#endif /* CONFIG_DRM_AMD_DC_FP */
 	default:
 		break;
 	}
@@ -2213,7 +2213,7 @@ enum dc_status dc_remove_stream_from_ctx(
 			del_pipe->stream_res.stream_enc,
 			false);
 
-	if (link_is_dp_128b_132b_signal(del_pipe)) {
+	if (dc->link_srv->dp_is_128b_132b_signal(del_pipe)) {
 		update_hpo_dp_stream_engine_usage(
 			&new_ctx->res_ctx, dc->res_pool,
 			del_pipe->stream_res.hpo_dp_stream_enc,
@@ -2513,9 +2513,10 @@ enum dc_status resource_map_pool_resources(
 	 * and link settings
 	 */
 	if (dc_is_dp_signal(stream->signal)) {
-		if (!link_decide_link_settings(stream, &pipe_ctx->link_config.dp_link_settings))
+		if (!dc->link_srv->dp_decide_link_settings(stream, &pipe_ctx->link_config.dp_link_settings))
 			return DC_FAIL_DP_LINK_BANDWIDTH;
-		if (link_dp_get_encoding_format(&pipe_ctx->link_config.dp_link_settings) == DP_128b_132b_ENCODING) {
+		if (dc->link_srv->dp_get_encoding_format(
+				&pipe_ctx->link_config.dp_link_settings) == DP_128b_132b_ENCODING) {
 			pipe_ctx->stream_res.hpo_dp_stream_enc =
 					find_first_free_match_hpo_dp_stream_enc_for_link(
 							&context->res_ctx, pool, stream);
@@ -3685,7 +3686,7 @@ enum dc_status dc_validate_stream(struct dc *dc, struct dc_stream_state *stream)
 	/* TODO: validate audio ASIC caps, encoder */
 
 	if (res == DC_OK)
-		res = link_validate_mode_timing(stream,
+		res = dc->link_srv->validate_mode_timing(stream,
 		      link,
 		      &stream->timing);
 
@@ -3812,7 +3813,7 @@ bool get_temp_dp_link_res(struct dc_link *link,
 
 	memset(link_res, 0, sizeof(*link_res));
 
-	if (link_dp_get_encoding_format(link_settings) == DP_128b_132b_ENCODING) {
+	if (dc->link_srv->dp_get_encoding_format(link_settings) == DP_128b_132b_ENCODING) {
 		link_res->hpo_dp_link_enc = get_temp_hpo_dp_link_enc(res_ctx,
 				dc->res_pool, link);
 		if (!link_res->hpo_dp_link_enc)
@@ -4027,14 +4028,14 @@ bool dc_resource_acquire_secondary_pipe_for_mpc_odm(
 		else
 			sec_pipe->stream_res.opp = sec_pipe->top_pipe->stream_res.opp;
 		if (sec_pipe->stream->timing.flags.DSC == 1) {
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 			dcn20_acquire_dsc(dc, &state->res_ctx, &sec_pipe->stream_res.dsc, pipe_idx);
 #endif
 			ASSERT(sec_pipe->stream_res.dsc);
 			if (sec_pipe->stream_res.dsc == NULL)
 				return false;
 		}
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 		dcn20_build_mapped_resource(dc, state, sec_pipe->stream);
 #endif
 	}
@@ -4046,7 +4047,7 @@ enum dc_status update_dp_encoder_resources_for_test_harness(const struct dc *dc,
 		struct dc_state *context,
 		struct pipe_ctx *pipe_ctx)
 {
-	if (link_dp_get_encoding_format(&pipe_ctx->link_config.dp_link_settings) == DP_128b_132b_ENCODING) {
+	if (dc->link_srv->dp_get_encoding_format(&pipe_ctx->link_config.dp_link_settings) == DP_128b_132b_ENCODING) {
 		if (pipe_ctx->stream_res.hpo_dp_stream_enc == NULL) {
 			pipe_ctx->stream_res.hpo_dp_stream_enc =
 					find_first_free_match_hpo_dp_stream_enc_for_link(
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stat.c b/drivers/gpu/drm/amd/display/dc/core/dc_stat.c
index 6c06587dd88c..5f6392ae31a6 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stat.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stat.c
@@ -35,19 +35,15 @@
  */
 
 /**
- *****************************************************************************
- *  Function: dc_stat_get_dmub_notification
+ *  dc_stat_get_dmub_notification
  *
- *  @brief
- *		Calls dmub layer to retrieve dmub notification
+ * Calls dmub layer to retrieve dmub notification
  *
- *  @param
- *		[in] dc: dc structure
- *		[in] notify: dmub notification structure
+ * @dc: dc structure
+ * @notify: dmub notification structure
  *
- *  @return
+ * Returns
  *     None
- *****************************************************************************
  */
 void dc_stat_get_dmub_notification(const struct dc *dc, struct dmub_notification *notify)
 {
@@ -73,19 +69,15 @@ void dc_stat_get_dmub_notification(const struct dc *dc, struct dmub_notification
 }
 
 /**
- *****************************************************************************
- *  Function: dc_stat_get_dmub_dataout
+ * dc_stat_get_dmub_dataout
  *
- *  @brief
- *		Calls dmub layer to retrieve dmub gpint dataout
+ * Calls dmub layer to retrieve dmub gpint dataout
  *
- *  @param
- *		[in] dc: dc structure
- *		[in] dataout: dmub gpint dataout
+ * @dc: dc structure
+ * @dataout: dmub gpint dataout
  *
- *  @return
+ * Returns
  *     None
- *****************************************************************************
  */
 void dc_stat_get_dmub_dataout(const struct dc *dc, uint32_t *dataout)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c b/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c
index cde8ed2560b3..eda2152dcd1f 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c
@@ -47,9 +47,7 @@ int dc_setup_system_context(struct dc *dc, struct dc_phy_addr_space_config *pa_c
 		 */
 		memcpy(&dc->vm_pa_config, pa_config, sizeof(struct dc_phy_addr_space_config));
 		dc->vm_pa_config.valid = true;
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 		dc_z10_save_init(dc);
-#endif
 	}
 
 	return num_vmids;
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 1fde43378689..23ee63b98dcd 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -29,9 +29,7 @@
 #include "dc_types.h"
 #include "grph_object_defs.h"
 #include "logger_types.h"
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
-#include "hdcp_types.h"
-#endif
+#include "hdcp_msg_types.h"
 #include "gpio_types.h"
 #include "link_service_types.h"
 #include "grph_object_ctrl_defs.h"
@@ -47,7 +45,7 @@ struct aux_payload;
 struct set_config_cmd_payload;
 struct dmub_notification;
 
-#define DC_VER "3.2.223"
+#define DC_VER "3.2.230"
 
 #define MAX_SURFACES 3
 #define MAX_PLANES 6
@@ -84,8 +82,6 @@ enum det_size {
 
 struct dc_plane_cap {
 	enum dc_plane_type type;
-	uint32_t blends_with_above : 1;
-	uint32_t blends_with_below : 1;
 	uint32_t per_pixel_alpha : 1;
 	struct {
 		uint32_t argb8888 : 1;
@@ -409,6 +405,7 @@ struct dc_config {
 	bool force_bios_enable_lttpr;
 	uint8_t force_bios_fixed_vs;
 	int sdpif_request_limit_words_per_umc;
+	bool use_old_fixed_vs_sequence;
 	bool disable_subvp_drr;
 };
 
@@ -716,6 +713,7 @@ struct dc_bounding_box_overrides {
 struct dc_state;
 struct resource_pool;
 struct dce_hwseq;
+struct link_service;
 
 /**
  * struct dc_debug_options - DC debug struct
@@ -795,6 +793,7 @@ struct dc_debug_options {
 	unsigned int force_odm_combine; //bit vector based on otg inst
 	unsigned int seamless_boot_odm_combine;
 	unsigned int force_odm_combine_4to1; //bit vector based on otg inst
+	int minimum_z8_residency_time;
 	bool disable_z9_mpc;
 	unsigned int force_fclk_khz;
 	bool enable_tri_buf;
@@ -874,6 +873,12 @@ struct dc_debug_options {
 	bool disable_unbounded_requesting;
 	bool dig_fifo_off_in_blank;
 	bool temp_mst_deallocation_sequence;
+	bool override_dispclk_programming;
+	bool disable_fpo_optimizations;
+	bool support_eDP1_5;
+	uint32_t fpo_vactive_margin_us;
+	bool disable_fpo_vactive;
+	bool disable_boot_optimizations;
 };
 
 struct gpu_info_soc_bounding_box_v1_0;
@@ -890,6 +895,7 @@ struct dc {
 
 	uint8_t link_count;
 	struct dc_link *links[MAX_PIPES * 2];
+	struct link_service *link_srv;
 
 	struct dc_state *current_state;
 	struct resource_pool *res_pool;
@@ -991,11 +997,7 @@ struct dc_init_data {
 };
 
 struct dc_callback_init {
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	struct cp_psp cp_psp;
-#else
-	uint8_t reserved;
-#endif
 };
 
 struct dc *dc_create(const struct dc_init_data *init_params);
@@ -1362,10 +1364,6 @@ enum dc_status dc_commit_streams(struct dc *dc,
 				 struct dc_stream_state *streams[],
 				 uint8_t stream_count);
 
-/* TODO: When the transition to the new commit sequence is done, remove this
- * function in favor of dc_commit_streams. */
-bool dc_commit_state(struct dc *dc, struct dc_state *context);
-
 struct dc_state *dc_create_state(struct dc *dc);
 struct dc_state *dc_copy_state(struct dc_state *src_ctx);
 void dc_retain_state(struct dc_state *context);
@@ -1378,9 +1376,163 @@ struct dc_plane_state *dc_get_surface_for_mpcc(struct dc *dc,
 
 uint32_t dc_get_opp_for_plane(struct dc *dc, struct dc_plane_state *plane);
 
+/* The function returns minimum bandwidth required to drive a given timing
+ * return - minimum required timing bandwidth in kbps.
+ */
+uint32_t dc_bandwidth_in_kbps_from_timing(const struct dc_crtc_timing *timing);
+
 /* Link Interfaces */
-/* TODO: remove this after resolving external dependencies */
-#include "dc_link.h"
+/*
+ * A link contains one or more sinks and their connected status.
+ * The currently active signal type (HDMI, DP-SST, DP-MST) is also reported.
+ */
+struct dc_link {
+	struct dc_sink *remote_sinks[MAX_SINKS_PER_LINK];
+	unsigned int sink_count;
+	struct dc_sink *local_sink;
+	unsigned int link_index;
+	enum dc_connection_type type;
+	enum signal_type connector_signal;
+	enum dc_irq_source irq_source_hpd;
+	enum dc_irq_source irq_source_hpd_rx;/* aka DP Short Pulse  */
+
+	bool is_hpd_filter_disabled;
+	bool dp_ss_off;
+
+	/**
+	 * @link_state_valid:
+	 *
+	 * If there is no link and local sink, this variable should be set to
+	 * false. Otherwise, it should be set to true; usually, the function
+	 * core_link_enable_stream sets this field to true.
+	 */
+	bool link_state_valid;
+	bool aux_access_disabled;
+	bool sync_lt_in_progress;
+	bool skip_stream_reenable;
+	bool is_internal_display;
+	/** @todo Rename. Flag an endpoint as having a programmable mapping to a DIG encoder. */
+	bool is_dig_mapping_flexible;
+	bool hpd_status; /* HPD status of link without physical HPD pin. */
+	bool is_hpd_pending; /* Indicates a new received hpd */
+	bool is_automated; /* Indicates automated testing */
+
+	bool edp_sink_present;
+
+	struct dp_trace dp_trace;
+
+	/* caps is the same as reported_link_cap. link_traing use
+	 * reported_link_cap. Will clean up.  TODO
+	 */
+	struct dc_link_settings reported_link_cap;
+	struct dc_link_settings verified_link_cap;
+	struct dc_link_settings cur_link_settings;
+	struct dc_lane_settings cur_lane_setting[LANE_COUNT_DP_MAX];
+	struct dc_link_settings preferred_link_setting;
+	/* preferred_training_settings are override values that
+	 * come from DM. DM is responsible for the memory
+	 * management of the override pointers.
+	 */
+	struct dc_link_training_overrides preferred_training_settings;
+	struct dp_audio_test_data audio_test_data;
+
+	uint8_t ddc_hw_inst;
+
+	uint8_t hpd_src;
+
+	uint8_t link_enc_hw_inst;
+	/* DIG link encoder ID. Used as index in link encoder resource pool.
+	 * For links with fixed mapping to DIG, this is not changed after dc_link
+	 * object creation.
+	 */
+	enum engine_id eng_id;
+
+	bool test_pattern_enabled;
+	union compliance_test_state compliance_test_state;
+
+	void *priv;
+
+	struct ddc_service *ddc;
+
+	bool aux_mode;
+
+	/* Private to DC core */
+
+	const struct dc *dc;
+
+	struct dc_context *ctx;
+
+	struct panel_cntl *panel_cntl;
+	struct link_encoder *link_enc;
+	struct graphics_object_id link_id;
+	/* Endpoint type distinguishes display endpoints which do not have entries
+	 * in the BIOS connector table from those that do. Helps when tracking link
+	 * encoder to display endpoint assignments.
+	 */
+	enum display_endpoint_type ep_type;
+	union ddi_channel_mapping ddi_channel_mapping;
+	struct connector_device_tag_info device_tag;
+	struct dpcd_caps dpcd_caps;
+	uint32_t dongle_max_pix_clk;
+	unsigned short chip_caps;
+	unsigned int dpcd_sink_count;
+	struct hdcp_caps hdcp_caps;
+	enum edp_revision edp_revision;
+	union dpcd_sink_ext_caps dpcd_sink_ext_caps;
+
+	struct psr_settings psr_settings;
+
+	/* Drive settings read from integrated info table */
+	struct dc_lane_settings bios_forced_drive_settings;
+
+	/* Vendor specific LTTPR workaround variables */
+	uint8_t vendor_specific_lttpr_link_rate_wa;
+	bool apply_vendor_specific_lttpr_link_rate_wa;
+
+	/* MST record stream using this link */
+	struct link_flags {
+		bool dp_keep_receiver_powered;
+		bool dp_skip_DID2;
+		bool dp_skip_reset_segment;
+		bool dp_skip_fs_144hz;
+		bool dp_mot_reset_segment;
+		/* Some USB4 docks do not handle turning off MST DSC once it has been enabled. */
+		bool dpia_mst_dsc_always_on;
+		/* Forced DPIA into TBT3 compatibility mode. */
+		bool dpia_forced_tbt3_mode;
+		bool dongle_mode_timing_override;
+	} wa_flags;
+	struct link_mst_stream_allocation_table mst_stream_alloc_table;
+
+	struct dc_link_status link_status;
+	struct dprx_states dprx_states;
+
+	struct gpio *hpd_gpio;
+	enum dc_link_fec_state fec_state;
+	bool link_powered_externally;	// Used to bypass hardware sequencing delays when panel is powered down forcibly
+
+	struct dc_panel_config panel_config;
+	struct phy_state phy_state;
+	// BW ALLOCATON USB4 ONLY
+	struct dc_dpia_bw_alloc dpia_bw_alloc_config;
+};
+
+/* Return an enumerated dc_link.
+ * dc_link order is constant and determined at
+ * boot time.  They cannot be created or destroyed.
+ * Use dc_get_caps() to get number of links.
+ */
+struct dc_link *dc_get_link_at_index(struct dc *dc, uint32_t link_index);
+
+/* Return instance id of the edp link. Inst 0 is primary edp link. */
+bool dc_get_edp_link_panel_inst(const struct dc *dc,
+		const struct dc_link *link,
+		unsigned int *inst_out);
+
+/* Return an array of link pointers to edp links. */
+void dc_get_edp_links(const struct dc *dc,
+		struct dc_link **edp_links,
+		int *edp_num);
 
 /* The function initiates detection handshake over the given link. It first
  * determines if there are display connections over the link. If so it initiates
@@ -1404,6 +1556,38 @@ uint32_t dc_get_opp_for_plane(struct dc *dc, struct dc_plane_state *plane);
  */
 bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason);
 
+struct dc_sink_init_data;
+
+/* When link connection type is dc_connection_mst_branch, remote sink can be
+ * added to the link. The interface creates a remote sink and associates it with
+ * current link. The sink will be retained by link until remove remote sink is
+ * called.
+ *
+ * @dc_link - link the remote sink will be added to.
+ * @edid - byte array of EDID raw data.
+ * @len - size of the edid in byte
+ * @init_data -
+ */
+struct dc_sink *dc_link_add_remote_sink(
+		struct dc_link *dc_link,
+		const uint8_t *edid,
+		int len,
+		struct dc_sink_init_data *init_data);
+
+/* Remove remote sink from a link with dc_connection_mst_branch connection type.
+ * @link - link the sink should be removed from
+ * @sink - sink to be removed.
+ */
+void dc_link_remove_remote_sink(
+	struct dc_link *link,
+	struct dc_sink *sink);
+
+/* Enable HPD interrupt handler for a given link */
+void dc_link_enable_hpd(const struct dc_link *link);
+
+/* Disable HPD interrupt handler for a given link */
+void dc_link_disable_hpd(const struct dc_link *link);
+
 /* determine if there is a sink connected to the link
  *
  * @type - dc_connection_single if connected, dc_connection_none otherwise.
@@ -1417,14 +1601,115 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason);
 bool dc_link_detect_connection_type(struct dc_link *link,
 		enum dc_connection_type *type);
 
+/* query current hpd pin value
+ * return - true HPD is asserted (HPD high), false otherwise (HPD low)
+ *
+ */
+bool dc_link_get_hpd_state(struct dc_link *link);
+
 /* Getter for cached link status from given link */
 const struct dc_link_status *dc_link_get_status(const struct dc_link *link);
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
+/* enable/disable hardware HPD filter.
+ *
+ * @link - The link the HPD pin is associated with.
+ * @enable = true - enable hardware HPD filter. HPD event will only queued to irq
+ * handler once after no HPD change has been detected within dc default HPD
+ * filtering interval since last HPD event. i.e if display keeps toggling hpd
+ * pulses within default HPD interval, no HPD event will be received until HPD
+ * toggles have stopped. Then HPD event will be queued to irq handler once after
+ * dc default HPD filtering interval since last HPD event.
+ *
+ * @enable = false - disable hardware HPD filter. HPD event will be queued
+ * immediately to irq handler after no HPD change has been detected within
+ * IRQ_HPD (aka HPD short pulse) interval (i.e 2ms).
+ */
+void dc_link_enable_hpd_filter(struct dc_link *link, bool enable);
+
+/* submit i2c read/write payloads through ddc channel
+ * @link_index - index to a link with ddc in i2c mode
+ * @cmd - i2c command structure
+ * return - true if success, false otherwise.
+ */
+bool dc_submit_i2c(
+		struct dc *dc,
+		uint32_t link_index,
+		struct i2c_command *cmd);
+
+/* submit i2c read/write payloads through oem channel
+ * @link_index - index to a link with ddc in i2c mode
+ * @cmd - i2c command structure
+ * return - true if success, false otherwise.
+ */
+bool dc_submit_i2c_oem(
+		struct dc *dc,
+		struct i2c_command *cmd);
+
+enum aux_return_code_type;
+/* Attempt to transfer the given aux payload. This function does not perform
+ * retries or handle error states. The reply is returned in the payload->reply
+ * and the result through operation_result. Returns the number of bytes
+ * transferred,or -1 on a failure.
+ */
+int dc_link_aux_transfer_raw(struct ddc_service *ddc,
+		struct aux_payload *payload,
+		enum aux_return_code_type *operation_result);
+
+bool dc_is_oem_i2c_device_present(
+	struct dc *dc,
+	size_t slave_address
+);
+
 /* return true if the connected receiver supports the hdcp version */
 bool dc_link_is_hdcp14(struct dc_link *link, enum signal_type signal);
 bool dc_link_is_hdcp22(struct dc_link *link, enum signal_type signal);
-#endif
+
+/* Notify DC about DP RX Interrupt (aka DP IRQ_HPD).
+ *
+ * TODO - When defer_handling is true the function will have a different purpose.
+ * It no longer does complete hpd rx irq handling. We should create a separate
+ * interface specifically for this case.
+ *
+ * Return:
+ * true - Downstream port status changed. DM should call DC to do the
+ * detection.
+ * false - no change in Downstream port status. No further action required
+ * from DM.
+ */
+bool dc_link_handle_hpd_rx_irq(struct dc_link *dc_link,
+		union hpd_irq_data *hpd_irq_dpcd_data, bool *out_link_loss,
+		bool defer_handling, bool *has_left_work);
+/* handle DP specs define test automation sequence*/
+void dc_link_dp_handle_automated_test(struct dc_link *link);
+
+/* handle DP Link loss sequence and try to recover RX link loss with best
+ * effort
+ */
+void dc_link_dp_handle_link_loss(struct dc_link *link);
+
+/* Determine if hpd rx irq should be handled or ignored
+ * return true - hpd rx irq should be handled.
+ * return false - it is safe to ignore hpd rx irq event
+ */
+bool dc_link_dp_allow_hpd_rx_irq(const struct dc_link *link);
+
+/* Determine if link loss is indicated with a given hpd_irq_dpcd_data.
+ * @link - link the hpd irq data associated with
+ * @hpd_irq_dpcd_data - input hpd irq data
+ * return - true if hpd irq data indicates a link lost
+ */
+bool dc_link_check_link_loss_status(struct dc_link *link,
+		union hpd_irq_data *hpd_irq_dpcd_data);
+
+/* Read hpd rx irq data from a given link
+ * @link - link where the hpd irq data should be read from
+ * @irq_data - output hpd irq data
+ * return - DC_OK if hpd irq data is read successfully, otherwise hpd irq data
+ * read has failed.
+ */
+enum dc_status dc_link_dp_read_hpd_rx_irq_data(
+	struct dc_link *link,
+	union hpd_irq_data *irq_data);
 
 /* The function clears recorded DP RX states in the link. DM should call this
  * function when it is resuming from S3 power state to previously connected links.
@@ -1450,12 +1735,6 @@ uint32_t dc_link_bandwidth_kbps(
 	const struct dc_link *link,
 	const struct dc_link_settings *link_setting);
 
-/* The function returns minimum bandwidth required to drive a given timing
- * return - minimum required timing bandwidth in kbps.
- */
-uint32_t dc_bandwidth_in_kbps_from_timing(
-	const struct dc_crtc_timing *timing);
-
 /* The function takes a snapshot of current link resource allocation state
  * @dc: pointer to dc of the dm calling this
  * @map: a dc link resource snapshot defined internally to dc.
@@ -1493,6 +1772,281 @@ void dc_restore_link_res_map(const struct dc *dc, uint32_t *map);
  * interface i.e stream_update->dsc_config
  */
 bool dc_link_update_dsc_config(struct pipe_ctx *pipe_ctx);
+
+/* translate a raw link rate data to bandwidth in kbps */
+uint32_t dc_link_bw_kbps_from_raw_frl_link_rate_data(const struct dc *dc, uint8_t bw);
+
+/* determine the optimal bandwidth given link and required bw.
+ * @link - current detected link
+ * @req_bw - requested bandwidth in kbps
+ * @link_settings - returned most optimal link settings that can fit the
+ * requested bandwidth
+ * return - false if link can't support requested bandwidth, true if link
+ * settings is found.
+ */
+bool dc_link_decide_edp_link_settings(struct dc_link *link,
+		struct dc_link_settings *link_settings,
+		uint32_t req_bw);
+
+/* return the max dp link settings can be driven by the link without considering
+ * connected RX device and its capability
+ */
+bool dc_link_dp_get_max_link_enc_cap(const struct dc_link *link,
+		struct dc_link_settings *max_link_enc_cap);
+
+/* determine when the link is driving MST mode, what DP link channel coding
+ * format will be used. The decision will remain unchanged until next HPD event.
+ *
+ * @link -  a link with DP RX connection
+ * return - if stream is committed to this link with MST signal type, type of
+ * channel coding format dc will choose.
+ */
+enum dp_link_encoding dc_link_dp_mst_decide_link_encoding_format(
+		const struct dc_link *link);
+
+/* get max dp link settings the link can enable with all things considered. (i.e
+ * TX/RX/Cable capabilities and dp override policies.
+ *
+ * @link - a link with DP RX connection
+ * return - max dp link settings the link can enable.
+ *
+ */
+const struct dc_link_settings *dc_link_get_link_cap(const struct dc_link *link);
+
+/* Check if a RX (ex. DP sink, MST hub, passive or active dongle) is connected
+ * to a link with dp connector signal type.
+ * @link - a link with dp connector signal type
+ * return - true if connected, false otherwise
+ */
+bool dc_link_is_dp_sink_present(struct dc_link *link);
+
+/* Force DP lane settings update to main-link video signal and notify the change
+ * to DP RX via DPCD. This is a debug interface used for video signal integrity
+ * tuning purpose. The interface assumes link has already been enabled with DP
+ * signal.
+ *
+ * @lt_settings - a container structure with desired hw_lane_settings
+ */
+void dc_link_set_drive_settings(struct dc *dc,
+				struct link_training_settings *lt_settings,
+				struct dc_link *link);
+
+/* Enable a test pattern in Link or PHY layer in an active link for compliance
+ * test or debugging purpose. The test pattern will remain until next un-plug.
+ *
+ * @link - active link with DP signal output enabled.
+ * @test_pattern - desired test pattern to output.
+ * NOTE: set to DP_TEST_PATTERN_VIDEO_MODE to disable previous test pattern.
+ * @test_pattern_color_space - for video test pattern choose a desired color
+ * space.
+ * @p_link_settings - For PHY pattern choose a desired link settings
+ * @p_custom_pattern - some test pattern will require a custom input to
+ * customize some pattern details. Otherwise keep it to NULL.
+ * @cust_pattern_size - size of the custom pattern input.
+ *
+ */
+bool dc_link_dp_set_test_pattern(
+	struct dc_link *link,
+	enum dp_test_pattern test_pattern,
+	enum dp_test_pattern_color_space test_pattern_color_space,
+	const struct link_training_settings *p_link_settings,
+	const unsigned char *p_custom_pattern,
+	unsigned int cust_pattern_size);
+
+/* Force DP link settings to always use a specific value until reboot to a
+ * specific link. If link has already been enabled, the interface will also
+ * switch to desired link settings immediately. This is a debug interface to
+ * generic dp issue trouble shooting.
+ */
+void dc_link_set_preferred_link_settings(struct dc *dc,
+		struct dc_link_settings *link_setting,
+		struct dc_link *link);
+
+/* Force DP link to customize a specific link training behavior by overriding to
+ * standard DP specs defined protocol. This is a debug interface to trouble shoot
+ * display specific link training issues or apply some display specific
+ * workaround in link training.
+ *
+ * @link_settings - if not NULL, force preferred link settings to the link.
+ * @lt_override - a set of override pointers. If any pointer is none NULL, dc
+ * will apply this particular override in future link training. If NULL is
+ * passed in, dc resets previous overrides.
+ * NOTE: DM must keep the memory from override pointers until DM resets preferred
+ * training settings.
+ */
+void dc_link_set_preferred_training_settings(struct dc *dc,
+		struct dc_link_settings *link_setting,
+		struct dc_link_training_overrides *lt_overrides,
+		struct dc_link *link,
+		bool skip_immediate_retrain);
+
+/* return - true if FEC is supported with connected DP RX, false otherwise */
+bool dc_link_is_fec_supported(const struct dc_link *link);
+
+/* query FEC enablement policy to determine if FEC will be enabled by dc during
+ * link enablement.
+ * return - true if FEC should be enabled, false otherwise.
+ */
+bool dc_link_should_enable_fec(const struct dc_link *link);
+
+/* determine lttpr mode the current link should be enabled with a specific link
+ * settings.
+ */
+enum lttpr_mode dc_link_decide_lttpr_mode(struct dc_link *link,
+		struct dc_link_settings *link_setting);
+
+/* Force DP RX to update its power state.
+ * NOTE: this interface doesn't update dp main-link. Calling this function will
+ * cause DP TX main-link and DP RX power states out of sync. DM has to restore
+ * RX power state back upon finish DM specific execution requiring DP RX in a
+ * specific power state.
+ * @on - true to set DP RX in D0 power state, false to set DP RX in D3 power
+ * state.
+ */
+void dc_link_dp_receiver_power_ctrl(struct dc_link *link, bool on);
+
+/* Force link to read base dp receiver caps from dpcd 000h - 00Fh and overwrite
+ * current value read from extended receiver cap from 02200h - 0220Fh.
+ * Some DP RX has problems of providing accurate DP receiver caps from extended
+ * field, this interface is a workaround to revert link back to use base caps.
+ */
+void dc_link_overwrite_extended_receiver_cap(
+		struct dc_link *link);
+
+void dc_link_edp_panel_backlight_power_on(struct dc_link *link,
+		bool wait_for_hpd);
+
+/* Set backlight level of an embedded panel (eDP, LVDS).
+ * backlight_pwm_u16_16 is unsigned 32 bit with 16 bit integer
+ * and 16 bit fractional, where 1.0 is max backlight value.
+ */
+bool dc_link_set_backlight_level(const struct dc_link *dc_link,
+		uint32_t backlight_pwm_u16_16,
+		uint32_t frame_ramp);
+
+/* Set/get nits-based backlight level of an embedded panel (eDP, LVDS). */
+bool dc_link_set_backlight_level_nits(struct dc_link *link,
+		bool isHDR,
+		uint32_t backlight_millinits,
+		uint32_t transition_time_in_ms);
+
+bool dc_link_get_backlight_level_nits(struct dc_link *link,
+		uint32_t *backlight_millinits,
+		uint32_t *backlight_millinits_peak);
+
+int dc_link_get_backlight_level(const struct dc_link *dc_link);
+
+int dc_link_get_target_backlight_pwm(const struct dc_link *link);
+
+bool dc_link_set_psr_allow_active(struct dc_link *dc_link, const bool *enable,
+		bool wait, bool force_static, const unsigned int *power_opts);
+
+bool dc_link_get_psr_state(const struct dc_link *dc_link, enum dc_psr_state *state);
+
+bool dc_link_setup_psr(struct dc_link *dc_link,
+		const struct dc_stream_state *stream, struct psr_config *psr_config,
+		struct psr_context *psr_context);
+
+/* On eDP links this function call will stall until T12 has elapsed.
+ * If the panel is not in power off state, this function will return
+ * immediately.
+ */
+bool dc_link_wait_for_t12(struct dc_link *link);
+
+/* Determine if dp trace has been initialized to reflect upto date result *
+ * return - true if trace is initialized and has valid data. False dp trace
+ * doesn't have valid result.
+ */
+bool dc_dp_trace_is_initialized(struct dc_link *link);
+
+/* Query a dp trace flag to indicate if the current dp trace data has been
+ * logged before
+ */
+bool dc_dp_trace_is_logged(struct dc_link *link,
+		bool in_detection);
+
+/* Set dp trace flag to indicate whether DM has already logged the current dp
+ * trace data. DM can set is_logged to true upon logging and check
+ * dc_dp_trace_is_logged before logging to avoid logging the same result twice.
+ */
+void dc_dp_trace_set_is_logged_flag(struct dc_link *link,
+		bool in_detection,
+		bool is_logged);
+
+/* Obtain driver time stamp for last dp link training end. The time stamp is
+ * formatted based on dm_get_timestamp DM function.
+ * @in_detection - true to get link training end time stamp of last link
+ * training in detection sequence. false to get link training end time stamp
+ * of last link training in commit (dpms) sequence
+ */
+unsigned long long dc_dp_trace_get_lt_end_timestamp(struct dc_link *link,
+		bool in_detection);
+
+/* Get how many link training attempts dc has done with latest sequence.
+ * @in_detection - true to get link training count of last link
+ * training in detection sequence. false to get link training count of last link
+ * training in commit (dpms) sequence
+ */
+const struct dp_trace_lt_counts *dc_dp_trace_get_lt_counts(struct dc_link *link,
+		bool in_detection);
+
+/* Get how many link loss has happened since last link training attempts */
+unsigned int dc_dp_trace_get_link_loss_count(struct dc_link *link);
+
+/*
+ *  USB4 DPIA BW ALLOCATION PUBLIC FUNCTIONS
+ */
+/*
+ * Send a request from DP-Tx requesting to allocate BW remotely after
+ * allocating it locally. This will get processed by CM and a CB function
+ * will be called.
+ *
+ * @link: pointer to the dc_link struct instance
+ * @req_bw: The requested bw in Kbyte to allocated
+ *
+ * return: none
+ */
+void dc_link_set_usb4_req_bw_req(struct dc_link *link, int req_bw);
+
+/*
+ * Handle function for when the status of the Request above is complete.
+ * We will find out the result of allocating on CM and update structs.
+ *
+ * @link: pointer to the dc_link struct instance
+ * @bw: Allocated or Estimated BW depending on the result
+ * @result: Response type
+ *
+ * return: none
+ */
+void dc_link_handle_usb4_bw_alloc_response(struct dc_link *link,
+		uint8_t bw, uint8_t result);
+
+/*
+ * Handle the USB4 BW Allocation related functionality here:
+ * Plug => Try to allocate max bw from timing parameters supported by the sink
+ * Unplug => de-allocate bw
+ *
+ * @link: pointer to the dc_link struct instance
+ * @peak_bw: Peak bw used by the link/sink
+ *
+ * return: allocated bw else return 0
+ */
+int dc_link_dp_dpia_handle_usb4_bandwidth_allocation_for_link(
+		struct dc_link *link, int peak_bw);
+
+/*
+ * Validate the BW of all the valid DPIA links to make sure it doesn't exceed
+ * available BW for each host router
+ *
+ * @dc: pointer to dc struct
+ * @stream: pointer to all possible streams
+ * @num_streams: number of valid DPIA streams
+ *
+ * return: TRUE if bw used by DPIAs doesn't exceed available BW else return FALSE
+ */
+bool dc_link_validate(struct dc *dc, const struct dc_stream_state *streams,
+		const unsigned int count);
+
 /* Sink Interfaces - A sink corresponds to a display output device */
 
 struct dc_container_id {
@@ -1511,7 +2065,7 @@ struct dc_sink_dsc_caps {
 	// 'true' if these are virtual DPCD's DSC caps (immediately upstream of sink in MST topology),
 	// 'false' if they are sink's DSC caps
 	bool is_virtual_dpcd_dsc;
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 	// 'true' if MST topology supports DSC passthrough for sink
 	// 'false' if MST topology does not support DSC passthrough
 	bool is_dsc_passthrough_supported;
@@ -1603,7 +2157,6 @@ void dc_resume(struct dc *dc);
 
 void dc_power_down_on_boot(struct dc *dc);
 
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 /*
  * HDCP Interfaces
  */
@@ -1611,7 +2164,6 @@ enum hdcp_message_status dc_process_hdcp_msg(
 		enum signal_type signal,
 		struct dc_link *link,
 		struct hdcp_protection_message *message_info);
-#endif
 bool dc_is_dmcu_initialized(struct dc *dc);
 
 enum dc_status dc_set_clock(struct dc *dc, enum dc_clock_type clock_type, uint32_t clk_khz, uint32_t stepping);
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
index c2092775ca88..a9b9490a532c 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
@@ -302,29 +302,32 @@ static uint8_t dc_dmub_srv_get_pipes_for_stream(struct dc *dc, struct dc_stream_
 	return pipes;
 }
 
-static int dc_dmub_srv_get_timing_generator_offset(struct dc *dc, struct dc_stream_state *stream)
+static void dc_dmub_srv_populate_fams_pipe_info(struct dc *dc, struct dc_state *context,
+		struct pipe_ctx *head_pipe,
+		struct dmub_cmd_fw_assisted_mclk_switch_pipe_data *fams_pipe_data)
 {
-	int  tg_inst = 0;
-	int i = 0;
+	int j;
+	int pipe_idx = 0;
 
-	for (i = 0; i < MAX_PIPES; i++) {
-		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+	fams_pipe_data->pipe_index[pipe_idx++] = head_pipe->plane_res.hubp->inst;
+	for (j = 0; j < dc->res_pool->pipe_count; j++) {
+		struct pipe_ctx *split_pipe = &context->res_ctx.pipe_ctx[j];
 
-		if (pipe->stream == stream && pipe->stream_res.tg) {
-			tg_inst = pipe->stream_res.tg->inst;
-			break;
+		if (split_pipe->stream == head_pipe->stream && (split_pipe->top_pipe || split_pipe->prev_odm_pipe)) {
+			fams_pipe_data->pipe_index[pipe_idx++] = split_pipe->plane_res.hubp->inst;
 		}
 	}
-	return tg_inst;
+	fams_pipe_data->pipe_count = pipe_idx;
 }
 
 bool dc_dmub_srv_p_state_delegate(struct dc *dc, bool should_manage_pstate, struct dc_state *context)
 {
 	union dmub_rb_cmd cmd = { 0 };
 	struct dmub_cmd_fw_assisted_mclk_switch_config *config_data = &cmd.fw_assisted_mclk_switch.config_data;
-	int i = 0;
+	int i = 0, k = 0;
 	int ramp_up_num_steps = 1; // TODO: Ramp is currently disabled. Reenable it.
 	uint8_t visual_confirm_enabled;
+	int pipe_idx = 0;
 
 	if (dc == NULL)
 		return false;
@@ -337,17 +340,40 @@ bool dc_dmub_srv_p_state_delegate(struct dc *dc, bool should_manage_pstate, stru
 	cmd.fw_assisted_mclk_switch.config_data.fams_enabled = should_manage_pstate;
 	cmd.fw_assisted_mclk_switch.config_data.visual_confirm_enabled = visual_confirm_enabled;
 
-	for (i = 0; context && i < context->stream_count; i++) {
-		struct dc_stream_state *stream = context->streams[i];
-		uint8_t min_refresh_in_hz = (stream->timing.min_refresh_in_uhz + 999999) / 1000000;
-		int  tg_inst = dc_dmub_srv_get_timing_generator_offset(dc, stream);
+	if (should_manage_pstate) {
+		for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+			struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+			if (!pipe->stream)
+				continue;
 
-		config_data->pipe_data[tg_inst].pix_clk_100hz = stream->timing.pix_clk_100hz;
-		config_data->pipe_data[tg_inst].min_refresh_in_hz = min_refresh_in_hz;
-		config_data->pipe_data[tg_inst].max_ramp_step = ramp_up_num_steps;
-		config_data->pipe_data[tg_inst].pipes = dc_dmub_srv_get_pipes_for_stream(dc, stream);
+			/* If FAMS is being used to support P-State and there is a stream
+			 * that does not use FAMS, we are in an FPO + VActive scenario.
+			 * Assign vactive stretch margin in this case.
+			 */
+			if (!pipe->stream->fpo_in_use) {
+				cmd.fw_assisted_mclk_switch.config_data.vactive_stretch_margin_us = dc->debug.fpo_vactive_margin_us;
+				break;
+			}
+			pipe_idx++;
+		}
 	}
 
+	for (i = 0, k = 0; context && i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (!pipe->top_pipe && !pipe->prev_odm_pipe && pipe->stream && pipe->stream->fpo_in_use) {
+			struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+			uint8_t min_refresh_in_hz = (pipe->stream->timing.min_refresh_in_uhz + 999999) / 1000000;
+
+			config_data->pipe_data[k].pix_clk_100hz = pipe->stream->timing.pix_clk_100hz;
+			config_data->pipe_data[k].min_refresh_in_hz = min_refresh_in_hz;
+			config_data->pipe_data[k].max_ramp_step = ramp_up_num_steps;
+			config_data->pipe_data[k].pipes = dc_dmub_srv_get_pipes_for_stream(dc, pipe->stream);
+			dc_dmub_srv_populate_fams_pipe_info(dc, context, pipe, &config_data->pipe_data[k]);
+			k++;
+		}
+	}
 	cmd.fw_assisted_mclk_switch.header.payload_bytes =
 		sizeof(cmd.fw_assisted_mclk_switch) - sizeof(cmd.fw_assisted_mclk_switch.header);
 
@@ -421,7 +447,6 @@ void dc_dmub_srv_get_visual_confirm_color_cmd(struct dc *dc, struct pipe_ctx *pi
 	}
 }
 
-#ifdef CONFIG_DRM_AMD_DC_DCN
 /**
  * populate_subvp_cmd_drr_info - Helper to populate DRR pipe info for the DMCUB subvp command
  *
@@ -638,7 +663,7 @@ static void populate_subvp_cmd_pipe_info(struct dc *dc,
 	pipe_data->pipe_config.subvp_data.main_vblank_end =
 			main_timing->v_total - main_timing->v_front_porch - main_timing->v_addressable;
 	pipe_data->pipe_config.subvp_data.mall_region_lines = phantom_timing->v_addressable;
-	pipe_data->pipe_config.subvp_data.main_pipe_index = subvp_pipe->pipe_idx;
+	pipe_data->pipe_config.subvp_data.main_pipe_index = subvp_pipe->stream_res.tg->inst;
 	pipe_data->pipe_config.subvp_data.is_drr = subvp_pipe->stream->ignore_msa_timing_param;
 
 	/* Calculate the scaling factor from the src and dst height.
@@ -680,11 +705,11 @@ static void populate_subvp_cmd_pipe_info(struct dc *dc,
 		struct pipe_ctx *phantom_pipe = &context->res_ctx.pipe_ctx[j];
 
 		if (phantom_pipe->stream == subvp_pipe->stream->mall_stream_config.paired_stream) {
-			pipe_data->pipe_config.subvp_data.phantom_pipe_index = phantom_pipe->pipe_idx;
+			pipe_data->pipe_config.subvp_data.phantom_pipe_index = phantom_pipe->stream_res.tg->inst;
 			if (phantom_pipe->bottom_pipe) {
-				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = phantom_pipe->bottom_pipe->pipe_idx;
+				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = phantom_pipe->bottom_pipe->plane_res.hubp->inst;
 			} else if (phantom_pipe->next_odm_pipe) {
-				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = phantom_pipe->next_odm_pipe->pipe_idx;
+				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = phantom_pipe->next_odm_pipe->plane_res.hubp->inst;
 			} else {
 				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = 0;
 			}
@@ -750,7 +775,8 @@ void dc_dmub_setup_subvp_dmub_command(struct dc *dc,
 					!pipe->top_pipe && !pipe->prev_odm_pipe &&
 					pipe->stream->mall_stream_config.type == SUBVP_MAIN) {
 				populate_subvp_cmd_pipe_info(dc, context, &cmd, pipe, cmd_pipe_index++);
-			} else if (pipe->plane_state && pipe->stream->mall_stream_config.type == SUBVP_NONE) {
+			} else if (pipe->plane_state && pipe->stream->mall_stream_config.type == SUBVP_NONE &&
+				    !pipe->top_pipe && !pipe->prev_odm_pipe) {
 				// Don't need to check for ActiveDRAMClockChangeMargin < 0, not valid in cases where
 				// we run through DML without calculating "natural" P-state support
 				populate_subvp_cmd_vblank_pipe_info(dc, context, &cmd, pipe, cmd_pipe_index++);
@@ -775,7 +801,6 @@ void dc_dmub_setup_subvp_dmub_command(struct dc *dc,
 	dc_dmub_srv_cmd_execute(dc->ctx->dmub_srv);
 	dc_dmub_srv_wait_idle(dc->ctx->dmub_srv);
 }
-#endif
 
 bool dc_dmub_srv_get_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv, struct dmub_diagnostic_data *diag_data)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
index 809a1851f196..49aab1924665 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
@@ -47,14 +47,15 @@ enum dc_lane_count {
  */
 enum dc_link_rate {
 	LINK_RATE_UNKNOWN = 0,
-	LINK_RATE_LOW = 0x06,		// Rate_1 (RBR)	- 1.62 Gbps/Lane
-	LINK_RATE_RATE_2 = 0x08,	// Rate_2		- 2.16 Gbps/Lane
-	LINK_RATE_RATE_3 = 0x09,	// Rate_3		- 2.43 Gbps/Lane
-	LINK_RATE_HIGH = 0x0A,		// Rate_4 (HBR)	- 2.70 Gbps/Lane
-	LINK_RATE_RBR2 = 0x0C,		// Rate_5 (RBR2)- 3.24 Gbps/Lane
-	LINK_RATE_RATE_6 = 0x10,	// Rate_6		- 4.32 Gbps/Lane
-	LINK_RATE_HIGH2 = 0x14,		// Rate_7 (HBR2)- 5.40 Gbps/Lane
-	LINK_RATE_HIGH3 = 0x1E,		// Rate_8 (HBR3)- 8.10 Gbps/Lane
+	LINK_RATE_LOW = 0x06,		// Rate_1 (RBR)  - 1.62 Gbps/Lane
+	LINK_RATE_RATE_2 = 0x08,	// Rate_2        - 2.16 Gbps/Lane
+	LINK_RATE_RATE_3 = 0x09,	// Rate_3        - 2.43 Gbps/Lane
+	LINK_RATE_HIGH = 0x0A,		// Rate_4 (HBR)  - 2.70 Gbps/Lane
+	LINK_RATE_RBR2 = 0x0C,		// Rate_5 (RBR2) - 3.24 Gbps/Lane
+	LINK_RATE_RATE_6 = 0x10,	// Rate_6        - 4.32 Gbps/Lane
+	LINK_RATE_HIGH2 = 0x14,		// Rate_7 (HBR2) - 5.40 Gbps/Lane
+	LINK_RATE_RATE_8 = 0x19,	// Rate_8        - 6.75 Gbps/Lane
+	LINK_RATE_HIGH3 = 0x1E,		// Rate_9 (HBR3) - 8.10 Gbps/Lane
 	/* Starting from DP2.0 link rate enum directly represents actual
 	 * link rate value in unit of 10 mbps
 	 */
@@ -921,12 +922,6 @@ struct dpcd_usb4_dp_tunneling_info {
 #ifndef DP_DFP_CAPABILITY_EXTENSION_SUPPORT
 #define DP_DFP_CAPABILITY_EXTENSION_SUPPORT		0x0A3
 #endif
-#ifndef DP_LINK_SQUARE_PATTERN
-#define DP_LINK_SQUARE_PATTERN				0x10F
-#endif
-#ifndef DP_CABLE_ATTRIBUTES_UPDATED_BY_DPTX
-#define DP_CABLE_ATTRIBUTES_UPDATED_BY_DPTX		0x110
-#endif
 #ifndef DP_DSC_CONFIGURATION
 #define DP_DSC_CONFIGURATION				0x161
 #endif
@@ -939,12 +934,6 @@ struct dpcd_usb4_dp_tunneling_info {
 #ifndef DP_128b_132b_TRAINING_AUX_RD_INTERVAL
 #define DP_128b_132b_TRAINING_AUX_RD_INTERVAL		0x2216
 #endif
-#ifndef DP_LINK_SQUARE_PATTERN
-#define DP_LINK_SQUARE_PATTERN				0x10F
-#endif
-#ifndef DP_CABLE_ATTRIBUTES_UPDATED_BY_DPRX
-#define DP_CABLE_ATTRIBUTES_UPDATED_BY_DPRX		0x2217
-#endif
 #ifndef DP_TEST_264BIT_CUSTOM_PATTERN_7_0
 #define DP_TEST_264BIT_CUSTOM_PATTERN_7_0		0X2230
 #endif
@@ -988,10 +977,6 @@ struct dpcd_usb4_dp_tunneling_info {
 #define DP_INTRA_HOP_AUX_REPLY_INDICATION		(1 << 3)
 /* TODO - Use DRM header to replace above once available */
 #endif // DP_INTRA_HOP_AUX_REPLY_INDICATION
-
-#ifndef DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE
-#define DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE	0x50
-#endif
 union dp_main_line_channel_coding_cap {
 	struct {
 		uint8_t DP_8b_10b_SUPPORTED	:1;
@@ -1261,4 +1246,161 @@ union dpcd_sink_ext_caps {
 	} bits;
 	uint8_t raw;
 };
+
+enum dc_link_fec_state {
+	dc_link_fec_not_ready,
+	dc_link_fec_ready,
+	dc_link_fec_enabled
+};
+
+union dpcd_psr_configuration {
+	struct {
+		unsigned char ENABLE                    : 1;
+		unsigned char TRANSMITTER_ACTIVE_IN_PSR : 1;
+		unsigned char CRC_VERIFICATION          : 1;
+		unsigned char FRAME_CAPTURE_INDICATION  : 1;
+		/* For eDP 1.4, PSR v2*/
+		unsigned char LINE_CAPTURE_INDICATION   : 1;
+		/* For eDP 1.4, PSR v2*/
+		unsigned char IRQ_HPD_WITH_CRC_ERROR    : 1;
+		unsigned char ENABLE_PSR2               : 1;
+		unsigned char EARLY_TRANSPORT_ENABLE    : 1;
+	} bits;
+	unsigned char raw;
+};
+
+union dpcd_alpm_configuration {
+	struct {
+		unsigned char ENABLE                    : 1;
+		unsigned char IRQ_HPD_ENABLE            : 1;
+		unsigned char RESERVED                  : 6;
+	} bits;
+	unsigned char raw;
+};
+
+union dpcd_sink_active_vtotal_control_mode {
+	struct {
+		unsigned char ENABLE                    : 1;
+		unsigned char RESERVED                  : 7;
+	} bits;
+	unsigned char raw;
+};
+
+union psr_error_status {
+	struct {
+		unsigned char LINK_CRC_ERROR        :1;
+		unsigned char RFB_STORAGE_ERROR     :1;
+		unsigned char VSC_SDP_ERROR         :1;
+		unsigned char RESERVED              :5;
+	} bits;
+	unsigned char raw;
+};
+
+union psr_sink_psr_status {
+	struct {
+	unsigned char SINK_SELF_REFRESH_STATUS  :3;
+	unsigned char RESERVED                  :5;
+	} bits;
+	unsigned char raw;
+};
+
+struct edp_trace_power_timestamps {
+	uint64_t poweroff;
+	uint64_t poweron;
+};
+
+struct dp_trace_lt_counts {
+	unsigned int total;
+	unsigned int fail;
+};
+
+enum link_training_result {
+	LINK_TRAINING_SUCCESS,
+	LINK_TRAINING_CR_FAIL_LANE0,
+	LINK_TRAINING_CR_FAIL_LANE1,
+	LINK_TRAINING_CR_FAIL_LANE23,
+	/* CR DONE bit is cleared during EQ step */
+	LINK_TRAINING_EQ_FAIL_CR,
+	/* CR DONE bit is cleared but LANE0_CR_DONE is set during EQ step */
+	LINK_TRAINING_EQ_FAIL_CR_PARTIAL,
+	/* other failure during EQ step */
+	LINK_TRAINING_EQ_FAIL_EQ,
+	LINK_TRAINING_LQA_FAIL,
+	/* one of the CR,EQ or symbol lock is dropped */
+	LINK_TRAINING_LINK_LOSS,
+	/* Abort link training (because sink unplugged) */
+	LINK_TRAINING_ABORT,
+	DP_128b_132b_LT_FAILED,
+	DP_128b_132b_MAX_LOOP_COUNT_REACHED,
+	DP_128b_132b_CHANNEL_EQ_DONE_TIMEOUT,
+	DP_128b_132b_CDS_DONE_TIMEOUT,
+};
+
+struct dp_trace_lt {
+	struct dp_trace_lt_counts counts;
+	struct dp_trace_timestamps {
+		unsigned long long start;
+		unsigned long long end;
+	} timestamps;
+	enum link_training_result result;
+	bool is_logged;
+};
+
+struct dp_trace {
+	struct dp_trace_lt detect_lt_trace;
+	struct dp_trace_lt commit_lt_trace;
+	unsigned int link_loss_count;
+	bool is_initialized;
+	struct edp_trace_power_timestamps edp_trace_power_timestamps;
+};
+
+/* TODO - This is a temporary location for any new DPCD definitions.
+ * We should move these to drm_dp header.
+ */
+#ifndef DP_LINK_SQUARE_PATTERN
+#define DP_LINK_SQUARE_PATTERN				0x10F
+#endif
+#ifndef DP_CABLE_ATTRIBUTES_UPDATED_BY_DPRX
+#define DP_CABLE_ATTRIBUTES_UPDATED_BY_DPRX		0x2217
+#endif
+#ifndef DP_CABLE_ATTRIBUTES_UPDATED_BY_DPTX
+#define DP_CABLE_ATTRIBUTES_UPDATED_BY_DPTX		0x110
+#endif
+#ifndef DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE
+#define DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE	0x50
+#endif
+#ifndef DP_TUNNELING_IRQ
+#define DP_TUNNELING_IRQ				(1 << 5)
+#endif
+/** USB4 DPCD BW Allocation Registers Chapter 10.7 **/
+#ifndef DP_TUNNELING_CAPABILITIES
+#define DP_TUNNELING_CAPABILITIES			0xE000D /* 1.4a */
+#endif
+#ifndef USB4_DRIVER_ID
+#define USB4_DRIVER_ID					0xE000F /* 1.4a */
+#endif
+#ifndef USB4_DRIVER_BW_CAPABILITY
+#define USB4_DRIVER_BW_CAPABILITY			0xE0020 /* 1.4a */
+#endif
+#ifndef DP_IN_ADAPTER_TUNNEL_INFO
+#define DP_IN_ADAPTER_TUNNEL_INFO			0xE0021 /* 1.4a */
+#endif
+#ifndef DP_BW_GRANULALITY
+#define DP_BW_GRANULALITY				0xE0022 /* 1.4a */
+#endif
+#ifndef ESTIMATED_BW
+#define ESTIMATED_BW					0xE0023 /* 1.4a */
+#endif
+#ifndef ALLOCATED_BW
+#define ALLOCATED_BW					0xE0024 /* 1.4a */
+#endif
+#ifndef DP_TUNNELING_STATUS
+#define DP_TUNNELING_STATUS				0xE0025 /* 1.4a */
+#endif
+#ifndef DPTX_BW_ALLOCATION_MODE_CONTROL
+#define DPTX_BW_ALLOCATION_MODE_CONTROL			0xE0030 /* 1.4a */
+#endif
+#ifndef REQUESTED_BW
+#define REQUESTED_BW					0xE0031 /* 1.4a */
+#endif
 #endif /* DC_DP_TYPES_H */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dsc.h b/drivers/gpu/drm/amd/display/dc/dc_dsc.h
index 684713b2cff7..0e92a322c2ed 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dsc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dsc.h
@@ -54,6 +54,12 @@ struct dc_dsc_policy {
 	bool enable_dsc_when_not_needed;
 };
 
+struct dc_dsc_config_options {
+	uint32_t dsc_min_slice_height_override;
+	uint32_t max_target_bpp_limit_override_x16;
+	uint32_t slice_height_granularity;
+};
+
 bool dc_dsc_parse_dsc_dpcd(const struct dc *dc,
 		const uint8_t *dpcd_dsc_basic_data,
 		const uint8_t *dpcd_dsc_ext_data,
@@ -71,8 +77,7 @@ bool dc_dsc_compute_bandwidth_range(
 bool dc_dsc_compute_config(
 		const struct display_stream_compressor *dsc,
 		const struct dsc_dec_dpcd_caps *dsc_sink_caps,
-		uint32_t dsc_min_slice_height_override,
-		uint32_t max_target_bpp_limit_override,
+		const struct dc_dsc_config_options *options,
 		uint32_t target_bandwidth_kbps,
 		const struct dc_crtc_timing *timing,
 		struct dc_dsc_config *dsc_cfg);
@@ -100,4 +105,6 @@ void dc_dsc_policy_set_enable_dsc_when_not_needed(bool enable);
 
 void dc_dsc_policy_set_disable_dsc_stream_overhead(bool disable);
 
+void dc_dsc_get_default_config_option(const struct dc *dc, struct dc_dsc_config_options *options);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h b/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h
index c364744b4c83..b015e80672ec 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h
@@ -50,7 +50,6 @@ struct dp_hdmi_dongle_signature_data {
 
 /* DP-HDMI dongle slave address for retrieving dongle signature*/
 #define DP_HDMI_DONGLE_ADDRESS 0x40
-static const uint8_t dp_hdmi_dongle_signature_str[] = "DP-HDMI ADAPTOR";
 #define DP_HDMI_DONGLE_SIGNATURE_EOT 0x04
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index cc3d6fb39364..100d62162b71 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -829,7 +829,7 @@ struct dc_dsc_config {
 	uint32_t version_minor; /* DSC minor version. Full version is formed as 1.version_minor. */
 	bool ycbcr422_simple; /* Tell DSC engine to convert YCbCr 4:2:2 to 'YCbCr 4:2:2 simple'. */
 	int32_t rc_buffer_size; /* DSC RC buffer block size in bytes */
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 	bool is_frl; /* indicate if DSC is applied based on HDMI FRL sink's capability */
 #endif
 	bool is_dp; /* indicate if DSC is applied based on DP's capability */
@@ -1085,5 +1085,19 @@ struct tg_color {
 	uint16_t color_b_cb;
 };
 
+enum symclk_state {
+	SYMCLK_OFF_TX_OFF,
+	SYMCLK_ON_TX_ON,
+	SYMCLK_ON_TX_OFF,
+};
+
+struct phy_state {
+	struct {
+		uint8_t otg		: 1;
+		uint8_t reserved	: 7;
+	} symclk_ref_cnts;
+	enum symclk_state symclk_state;
+};
+
 #endif /* DC_HW_TYPES_H */
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h
deleted file mode 100644
index cecd807f5ed8..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dc_link.h
+++ /dev/null
@@ -1,577 +0,0 @@
-/*
- * Copyright 2012-14 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef DC_LINK_H_
-#define DC_LINK_H_
-
-#include "dc.h"
-#include "dc_types.h"
-#include "grph_object_defs.h"
-
-struct link_resource;
-enum aux_return_code_type;
-
-enum dc_link_fec_state {
-	dc_link_fec_not_ready,
-	dc_link_fec_ready,
-	dc_link_fec_enabled
-};
-
-/* DP MST stream allocation (payload bandwidth number) */
-struct link_mst_stream_allocation {
-	/* DIG front */
-	const struct stream_encoder *stream_enc;
-	/* HPO DP Stream Encoder */
-	const struct hpo_dp_stream_encoder *hpo_dp_stream_enc;
-	/* associate DRM payload table with DC stream encoder */
-	uint8_t vcp_id;
-	/* number of slots required for the DP stream in transport packet */
-	uint8_t slot_count;
-};
-
-/* DP MST stream allocation table */
-struct link_mst_stream_allocation_table {
-	/* number of DP video streams */
-	int stream_count;
-	/* array of stream allocations */
-	struct link_mst_stream_allocation stream_allocations[MAX_CONTROLLER_NUM];
-};
-
-struct edp_trace_power_timestamps {
-	uint64_t poweroff;
-	uint64_t poweron;
-};
-
-struct dp_trace_lt_counts {
-	unsigned int total;
-	unsigned int fail;
-};
-
-struct dp_trace_lt {
-	struct dp_trace_lt_counts counts;
-	struct dp_trace_timestamps {
-		unsigned long long start;
-		unsigned long long end;
-	} timestamps;
-	enum link_training_result result;
-	bool is_logged;
-};
-
-struct dp_trace {
-	struct dp_trace_lt detect_lt_trace;
-	struct dp_trace_lt commit_lt_trace;
-	unsigned int link_loss_count;
-	bool is_initialized;
-	struct edp_trace_power_timestamps edp_trace_power_timestamps;
-};
-
-/* PSR feature flags */
-struct psr_settings {
-	bool psr_feature_enabled;		// PSR is supported by sink
-	bool psr_allow_active;			// PSR is currently active
-	enum dc_psr_version psr_version;		// Internal PSR version, determined based on DPCD
-	bool psr_vtotal_control_support;	// Vtotal control is supported by sink
-	unsigned long long psr_dirty_rects_change_timestamp_ns;	// for delay of enabling PSR-SU
-
-	/* These parameters are calculated in Driver,
-	 * based on display timing and Sink capabilities.
-	 * If VBLANK region is too small and Sink takes a long time
-	 * to set up RFB, it may take an extra frame to enter PSR state.
-	 */
-	bool psr_frame_capture_indication_req;
-	unsigned int psr_sdp_transmit_line_num_deadline;
-	uint8_t force_ffu_mode;
-	unsigned int psr_power_opt;
-};
-
-/* To split out "global" and "per-panel" config settings.
- * Add a struct dc_panel_config under dc_link
- */
-struct dc_panel_config {
-	/* extra panel power sequence parameters */
-	struct pps {
-		unsigned int extra_t3_ms;
-		unsigned int extra_t7_ms;
-		unsigned int extra_delay_backlight_off;
-		unsigned int extra_post_t7_ms;
-		unsigned int extra_pre_t11_ms;
-		unsigned int extra_t12_ms;
-		unsigned int extra_post_OUI_ms;
-	} pps;
-	/* PSR */
-	struct psr {
-		bool disable_psr;
-		bool disallow_psrsu;
-		bool rc_disable;
-		bool rc_allow_static_screen;
-		bool rc_allow_fullscreen_VPB;
-	} psr;
-	/* ABM */
-	struct varib {
-		unsigned int varibright_feature_enable;
-		unsigned int def_varibright_level;
-		unsigned int abm_config_setting;
-	} varib;
-	/* edp DSC */
-	struct dsc {
-		bool disable_dsc_edp;
-		unsigned int force_dsc_edp_policy;
-	} dsc;
-	/* eDP ILR */
-	struct ilr {
-		bool optimize_edp_link_rate; /* eDP ILR */
-	} ilr;
-};
-
-/*
- *  USB4 DPIA BW ALLOCATION STRUCTS
- */
-struct dc_dpia_bw_alloc {
-	int sink_verified_bw;  // The Verified BW that sink can allocated and use that has been verified already
-	int sink_allocated_bw; // The Actual Allocated BW that sink currently allocated
-	int sink_max_bw;       // The Max BW that sink can require/support
-	int estimated_bw;      // The estimated available BW for this DPIA
-	int bw_granularity;    // BW Granularity
-	bool bw_alloc_enabled; // The BW Alloc Mode Support is turned ON for all 3:  DP-Tx & Dpia & CM
-	bool response_ready;   // Response ready from the CM side
-};
-
-#define MAX_SINKS_PER_LINK 4
-
-/*
- * A link contains one or more sinks and their connected status.
- * The currently active signal type (HDMI, DP-SST, DP-MST) is also reported.
- */
-struct dc_link {
-	struct dc_sink *remote_sinks[MAX_SINKS_PER_LINK];
-	unsigned int sink_count;
-	struct dc_sink *local_sink;
-	unsigned int link_index;
-	enum dc_connection_type type;
-	enum signal_type connector_signal;
-	enum dc_irq_source irq_source_hpd;
-	enum dc_irq_source irq_source_hpd_rx;/* aka DP Short Pulse  */
-	bool is_hpd_filter_disabled;
-	bool dp_ss_off;
-
-	/**
-	 * @link_state_valid:
-	 *
-	 * If there is no link and local sink, this variable should be set to
-	 * false. Otherwise, it should be set to true; usually, the function
-	 * core_link_enable_stream sets this field to true.
-	 */
-	bool link_state_valid;
-	bool aux_access_disabled;
-	bool sync_lt_in_progress;
-	bool is_internal_display;
-
-	/* TODO: Rename. Flag an endpoint as having a programmable mapping to a
-	 * DIG encoder. */
-	bool is_dig_mapping_flexible;
-	bool hpd_status; /* HPD status of link without physical HPD pin. */
-	bool is_hpd_pending; /* Indicates a new received hpd */
-	bool is_automated; /* Indicates automated testing */
-
-	bool edp_sink_present;
-
-	struct dp_trace dp_trace;
-
-	/* caps is the same as reported_link_cap. link_traing use
-	 * reported_link_cap. Will clean up.  TODO
-	 */
-	struct dc_link_settings reported_link_cap;
-	struct dc_link_settings verified_link_cap;
-	struct dc_link_settings cur_link_settings;
-	struct dc_lane_settings cur_lane_setting[LANE_COUNT_DP_MAX];
-	struct dc_link_settings preferred_link_setting;
-	/* preferred_training_settings are override values that
-	 * come from DM. DM is responsible for the memory
-	 * management of the override pointers.
-	 */
-	struct dc_link_training_overrides preferred_training_settings;
-	struct dp_audio_test_data audio_test_data;
-
-	uint8_t ddc_hw_inst;
-
-	uint8_t hpd_src;
-
-	uint8_t link_enc_hw_inst;
-	/* DIG link encoder ID. Used as index in link encoder resource pool.
-	 * For links with fixed mapping to DIG, this is not changed after dc_link
-	 * object creation.
-	 */
-	enum engine_id eng_id;
-
-	bool test_pattern_enabled;
-	union compliance_test_state compliance_test_state;
-
-	void *priv;
-
-	struct ddc_service *ddc;
-
-	bool aux_mode;
-
-	/* Private to DC core */
-
-	const struct dc *dc;
-
-	struct dc_context *ctx;
-
-	struct panel_cntl *panel_cntl;
-	struct link_encoder *link_enc;
-	struct graphics_object_id link_id;
-	/* Endpoint type distinguishes display endpoints which do not have entries
-	 * in the BIOS connector table from those that do. Helps when tracking link
-	 * encoder to display endpoint assignments.
-	 */
-	enum display_endpoint_type ep_type;
-	union ddi_channel_mapping ddi_channel_mapping;
-	struct connector_device_tag_info device_tag;
-	struct dpcd_caps dpcd_caps;
-	uint32_t dongle_max_pix_clk;
-	unsigned short chip_caps;
-	unsigned int dpcd_sink_count;
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
-	struct hdcp_caps hdcp_caps;
-#endif
-	enum edp_revision edp_revision;
-	union dpcd_sink_ext_caps dpcd_sink_ext_caps;
-
-	struct psr_settings psr_settings;
-
-	/* Drive settings read from integrated info table */
-	struct dc_lane_settings bios_forced_drive_settings;
-
-	/* Vendor specific LTTPR workaround variables */
-	uint8_t vendor_specific_lttpr_link_rate_wa;
-	bool apply_vendor_specific_lttpr_link_rate_wa;
-
-	/* MST record stream using this link */
-	struct link_flags {
-		bool dp_keep_receiver_powered;
-		bool dp_skip_DID2;
-		bool dp_skip_reset_segment;
-		bool dp_skip_fs_144hz;
-		bool dp_mot_reset_segment;
-		/* Some USB4 docks do not handle turning off MST DSC once it has been enabled. */
-		bool dpia_mst_dsc_always_on;
-		/* Forced DPIA into TBT3 compatibility mode. */
-		bool dpia_forced_tbt3_mode;
-		bool dongle_mode_timing_override;
-	} wa_flags;
-	struct link_mst_stream_allocation_table mst_stream_alloc_table;
-
-	struct dc_link_status link_status;
-	struct dprx_states dprx_states;
-
-	struct gpio *hpd_gpio;
-	enum dc_link_fec_state fec_state;
-	bool link_powered_externally;	// Used to bypass hardware sequencing delays when panel is powered down forcibly
-
-	struct dc_panel_config panel_config;
-	struct phy_state phy_state;
-};
-
-
-/**
- * dc_get_link_at_index() - Return an enumerated dc_link.
- *
- * dc_link order is constant and determined at
- * boot time.  They cannot be created or destroyed.
- * Use dc_get_caps() to get number of links.
- */
-static inline struct dc_link *dc_get_link_at_index(struct dc *dc, uint32_t link_index)
-{
-	return dc->links[link_index];
-}
-
-static inline void get_edp_links(const struct dc *dc,
-		struct dc_link **edp_links,
-		int *edp_num)
-{
-	int i;
-
-	*edp_num = 0;
-	for (i = 0; i < dc->link_count; i++) {
-		// report any eDP links, even unconnected DDI's
-		if (!dc->links[i])
-			continue;
-		if (dc->links[i]->connector_signal == SIGNAL_TYPE_EDP) {
-			edp_links[*edp_num] = dc->links[i];
-			if (++(*edp_num) == MAX_NUM_EDP)
-				return;
-		}
-	}
-}
-
-static inline bool dc_get_edp_link_panel_inst(const struct dc *dc,
-		const struct dc_link *link,
-		unsigned int *inst_out)
-{
-	struct dc_link *edp_links[MAX_NUM_EDP];
-	int edp_num, i;
-
-	*inst_out = 0;
-	if (link->connector_signal != SIGNAL_TYPE_EDP)
-		return false;
-	get_edp_links(dc, edp_links, &edp_num);
-	for (i = 0; i < edp_num; i++) {
-		if (link == edp_links[i])
-			break;
-		(*inst_out)++;
-	}
-	return true;
-}
-
-/* Set backlight level of an embedded panel (eDP, LVDS).
- * backlight_pwm_u16_16 is unsigned 32 bit with 16 bit integer
- * and 16 bit fractional, where 1.0 is max backlight value.
- */
-bool dc_link_set_backlight_level(const struct dc_link *dc_link,
-		uint32_t backlight_pwm_u16_16,
-		uint32_t frame_ramp);
-
-/* Set/get nits-based backlight level of an embedded panel (eDP, LVDS). */
-bool dc_link_set_backlight_level_nits(struct dc_link *link,
-		bool isHDR,
-		uint32_t backlight_millinits,
-		uint32_t transition_time_in_ms);
-
-bool dc_link_get_backlight_level_nits(struct dc_link *link,
-		uint32_t *backlight_millinits,
-		uint32_t *backlight_millinits_peak);
-
-int dc_link_get_backlight_level(const struct dc_link *dc_link);
-
-int dc_link_get_target_backlight_pwm(const struct dc_link *link);
-
-bool dc_link_set_psr_allow_active(struct dc_link *dc_link, const bool *enable,
-		bool wait, bool force_static, const unsigned int *power_opts);
-
-bool dc_link_get_psr_state(const struct dc_link *dc_link, enum dc_psr_state *state);
-
-bool dc_link_setup_psr(struct dc_link *dc_link,
-		const struct dc_stream_state *stream, struct psr_config *psr_config,
-		struct psr_context *psr_context);
-
-bool dc_link_get_hpd_state(struct dc_link *dc_link);
-
-/* Notify DC about DP RX Interrupt (aka Short Pulse Interrupt).
- * Return:
- * true - Downstream port status changed. DM should call DC to do the
- * detection.
- * false - no change in Downstream port status. No further action required
- * from DM. */
-bool dc_link_handle_hpd_rx_irq(struct dc_link *dc_link,
-		union hpd_irq_data *hpd_irq_dpcd_data, bool *out_link_loss,
-		bool defer_handling, bool *has_left_work);
-
-/*
- * On eDP links this function call will stall until T12 has elapsed.
- * If the panel is not in power off state, this function will return
- * immediately.
- */
-bool dc_link_wait_for_t12(struct dc_link *link);
-
-void dc_link_dp_handle_automated_test(struct dc_link *link);
-void dc_link_dp_handle_link_loss(struct dc_link *link);
-bool dc_link_dp_allow_hpd_rx_irq(const struct dc_link *link);
-bool dc_link_check_link_loss_status(struct dc_link *link,
-		union hpd_irq_data *hpd_irq_dpcd_data);
-enum dc_status dc_link_dp_read_hpd_rx_irq_data(
-	struct dc_link *link,
-	union hpd_irq_data *irq_data);
-struct dc_sink_init_data;
-
-struct dc_sink *dc_link_add_remote_sink(
-		struct dc_link *dc_link,
-		const uint8_t *edid,
-		int len,
-		struct dc_sink_init_data *init_data);
-
-void dc_link_remove_remote_sink(
-	struct dc_link *link,
-	struct dc_sink *sink);
-
-/* Used by diagnostics for virtual link at the moment */
-
-bool dc_link_dp_set_test_pattern(
-	struct dc_link *link,
-	enum dp_test_pattern test_pattern,
-	enum dp_test_pattern_color_space test_pattern_color_space,
-	const struct link_training_settings *p_link_settings,
-	const unsigned char *p_custom_pattern,
-	unsigned int cust_pattern_size);
-
-bool dc_link_dp_get_max_link_enc_cap(const struct dc_link *link, struct dc_link_settings *max_link_enc_cap);
-
-/**
- *****************************************************************************
- *  Function: dc_link_enable_hpd_filter
- *
- *  @brief
- *     If enable is true, programs HPD filter on associated HPD line to default
- *     values dependent on link->connector_signal
- *
- *     If enable is false, programs HPD filter on associated HPD line with no
- *     delays on connect or disconnect
- *
- *  @param [in] link: pointer to the dc link
- *  @param [in] enable: boolean specifying whether to enable hbd
- *****************************************************************************
- */
-void dc_link_enable_hpd_filter(struct dc_link *link, bool enable);
-
-bool dc_link_is_dp_sink_present(struct dc_link *link);
-/*
- * DPCD access interfaces
- */
-
-void dc_link_set_drive_settings(struct dc *dc,
-				struct link_training_settings *lt_settings,
-				const struct dc_link *link);
-void dc_link_set_preferred_link_settings(struct dc *dc,
-					 struct dc_link_settings *link_setting,
-					 struct dc_link *link);
-void dc_link_set_preferred_training_settings(struct dc *dc,
-					struct dc_link_settings *link_setting,
-					struct dc_link_training_overrides *lt_overrides,
-					struct dc_link *link,
-					bool skip_immediate_retrain);
-void dc_link_enable_hpd(const struct dc_link *link);
-void dc_link_disable_hpd(const struct dc_link *link);
-void dc_link_set_test_pattern(struct dc_link *link,
-			enum dp_test_pattern test_pattern,
-			enum dp_test_pattern_color_space test_pattern_color_space,
-			const struct link_training_settings *p_link_settings,
-			const unsigned char *p_custom_pattern,
-			unsigned int cust_pattern_size);
-
-const struct dc_link_settings *dc_link_get_link_cap(
-		const struct dc_link *link);
-
-void dc_link_overwrite_extended_receiver_cap(
-		struct dc_link *link);
-
-bool dc_is_oem_i2c_device_present(
-	struct dc *dc,
-	size_t slave_address
-);
-
-bool dc_submit_i2c(
-		struct dc *dc,
-		uint32_t link_index,
-		struct i2c_command *cmd);
-
-bool dc_submit_i2c_oem(
-		struct dc *dc,
-		struct i2c_command *cmd);
-
-bool dc_link_is_fec_supported(const struct dc_link *link);
-bool dc_link_should_enable_fec(const struct dc_link *link);
-
-uint32_t dc_link_bw_kbps_from_raw_frl_link_rate_data(uint8_t bw);
-enum dp_link_encoding dc_link_dp_mst_decide_link_encoding_format(const struct dc_link *link);
-
-/* take a snapshot of current link resource allocation state */
-void dc_get_cur_link_res_map(const struct dc *dc, uint32_t *map);
-/* restore link resource allocation state from a snapshot */
-void dc_restore_link_res_map(const struct dc *dc, uint32_t *map);
-void dp_trace_reset(struct dc_link *link);
-bool dc_dp_trace_is_initialized(struct dc_link *link);
-unsigned long long dc_dp_trace_get_lt_end_timestamp(struct dc_link *link,
-		bool in_detection);
-void dc_dp_trace_set_is_logged_flag(struct dc_link *link,
-		bool in_detection,
-		bool is_logged);
-bool dc_dp_trace_is_logged(struct dc_link *link,
-		bool in_detection);
-struct dp_trace_lt_counts *dc_dp_trace_get_lt_counts(struct dc_link *link,
-		bool in_detection);
-unsigned int dc_dp_trace_get_link_loss_count(struct dc_link *link);
-
-/* Attempt to transfer the given aux payload. This function does not perform
- * retries or handle error states. The reply is returned in the payload->reply
- * and the result through operation_result. Returns the number of bytes
- * transferred,or -1 on a failure.
- */
-int dc_link_aux_transfer_raw(struct ddc_service *ddc,
-		struct aux_payload *payload,
-		enum aux_return_code_type *operation_result);
-
-enum lttpr_mode dc_link_decide_lttpr_mode(struct dc_link *link,
-		struct dc_link_settings *link_setting);
-void dc_link_dp_receiver_power_ctrl(struct dc_link *link, bool on);
-bool dc_link_decide_edp_link_settings(struct dc_link *link,
-		struct dc_link_settings *link_setting,
-		uint32_t req_bw);
-void dc_link_edp_panel_backlight_power_on(struct dc_link *link,
-		bool wait_for_hpd);
-
-/*
- *  USB4 DPIA BW ALLOCATION PUBLIC FUNCTIONS
- */
-/*
- * Send a request from DP-Tx requesting to allocate BW remotely after
- * allocating it locally. This will get processed by CM and a CB function
- * will be called.
- *
- * @link: pointer to the dc_link struct instance
- * @req_bw: The requested bw in Kbyte to allocated
- *
- * return: none
- */
-void dc_link_set_usb4_req_bw_req(struct dc_link *link, int req_bw);
-
-/*
- * CB function for when the status of the Req above is complete. We will
- * find out the result of allocating on CM and update structs accordingly
- *
- * @link: pointer to the dc_link struct instance
- * @bw: Allocated or Estimated BW depending on the result
- * @result: Response type
- *
- * return: none
- */
-void dc_link_get_usb4_req_bw_resp(struct dc_link *link, uint8_t bw, uint8_t result);
-
-/*
- * Handle the USB4 BW Allocation related functionality here:
- * Plug => Try to allocate max bw from timing parameters supported by the sink
- * Unplug => de-allocate bw
- *
- * @link: pointer to the dc_link struct instance
- * @peak_bw: Peak bw used by the link/sink
- *
- * return: allocated bw else return 0
- */
-int dc_link_dp_dpia_handle_usb4_bandwidth_allocation_for_link(struct dc_link *link, int peak_bw);
-
-/* TODO: this is not meant to be exposed to DM. Should switch to stream update
- * interface i.e stream_update->dsc_config
- */
-bool dc_link_update_dsc_config(struct pipe_ctx *pipe_ctx);
-#endif /* DC_LINK_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index 567452599659..181a3408cc61 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -293,6 +293,7 @@ struct dc_stream_state {
 
 	bool has_non_synchronizable_pclk;
 	bool vblank_synchronized;
+	bool fpo_in_use;
 	struct mall_stream_config mall_stream_config;
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index 27d0242d6cbd..45ab48fe5d00 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -38,10 +38,9 @@
 #include "dc_hw_types.h"
 #include "dal_types.h"
 #include "grph_object_defs.h"
+#include "grph_object_ctrl_defs.h"
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 #include "dm_cp_psp.h"
-#endif
 
 /* forward declarations */
 struct dc_plane_state;
@@ -812,9 +811,7 @@ struct dc_context {
 	uint32_t dc_edp_id_count;
 	uint64_t fbc_gpu_addr;
 	struct dc_dmub_srv *dmub_srv;
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 	struct cp_psp cp_psp;
-#endif
 	uint32_t *dcn_reg_offsets;
 	uint32_t *nbio_reg_offsets;
 };
@@ -954,7 +951,6 @@ struct dc_link_status {
 	struct dpcd_caps *dpcd_caps;
 };
 
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 union hdcp_rx_caps {
 	struct {
 		uint8_t version;
@@ -981,5 +977,114 @@ struct hdcp_caps {
 	union hdcp_rx_caps rx_caps;
 	union hdcp_bcaps bcaps;
 };
-#endif
+
+/* DP MST stream allocation (payload bandwidth number) */
+struct link_mst_stream_allocation {
+	/* DIG front */
+	const struct stream_encoder *stream_enc;
+	/* HPO DP Stream Encoder */
+	const struct hpo_dp_stream_encoder *hpo_dp_stream_enc;
+	/* associate DRM payload table with DC stream encoder */
+	uint8_t vcp_id;
+	/* number of slots required for the DP stream in transport packet */
+	uint8_t slot_count;
+};
+
+#define MAX_CONTROLLER_NUM 6
+
+/* DP MST stream allocation table */
+struct link_mst_stream_allocation_table {
+	/* number of DP video streams */
+	int stream_count;
+	/* array of stream allocations */
+	struct link_mst_stream_allocation stream_allocations[MAX_CONTROLLER_NUM];
+};
+
+/* PSR feature flags */
+struct psr_settings {
+	bool psr_feature_enabled;		// PSR is supported by sink
+	bool psr_allow_active;			// PSR is currently active
+	enum dc_psr_version psr_version;		// Internal PSR version, determined based on DPCD
+	bool psr_vtotal_control_support;	// Vtotal control is supported by sink
+	unsigned long long psr_dirty_rects_change_timestamp_ns;	// for delay of enabling PSR-SU
+
+	/* These parameters are calculated in Driver,
+	 * based on display timing and Sink capabilities.
+	 * If VBLANK region is too small and Sink takes a long time
+	 * to set up RFB, it may take an extra frame to enter PSR state.
+	 */
+	bool psr_frame_capture_indication_req;
+	unsigned int psr_sdp_transmit_line_num_deadline;
+	uint8_t force_ffu_mode;
+	unsigned int psr_power_opt;
+};
+
+/* To split out "global" and "per-panel" config settings.
+ * Add a struct dc_panel_config under dc_link
+ */
+struct dc_panel_config {
+	/* extra panel power sequence parameters */
+	struct pps {
+		unsigned int extra_t3_ms;
+		unsigned int extra_t7_ms;
+		unsigned int extra_delay_backlight_off;
+		unsigned int extra_post_t7_ms;
+		unsigned int extra_pre_t11_ms;
+		unsigned int extra_t12_ms;
+		unsigned int extra_post_OUI_ms;
+	} pps;
+	/* nit brightness */
+	struct nits_brightness {
+		unsigned int peak; /* nits */
+		unsigned int max_avg; /* nits */
+		unsigned int min; /* 1/10000 nits */
+		unsigned int max_nonboost_brightness_millinits;
+		unsigned int min_brightness_millinits;
+	} nits_brightness;
+	/* PSR */
+	struct psr {
+		bool disable_psr;
+		bool disallow_psrsu;
+		bool rc_disable;
+		bool rc_allow_static_screen;
+		bool rc_allow_fullscreen_VPB;
+	} psr;
+	/* ABM */
+	struct varib {
+		unsigned int varibright_feature_enable;
+		unsigned int def_varibright_level;
+		unsigned int abm_config_setting;
+	} varib;
+	/* edp DSC */
+	struct dsc {
+		bool disable_dsc_edp;
+		unsigned int force_dsc_edp_policy;
+	} dsc;
+	/* eDP ILR */
+	struct ilr {
+		bool optimize_edp_link_rate; /* eDP ILR */
+	} ilr;
+};
+
+/*
+ *  USB4 DPIA BW ALLOCATION STRUCTS
+ */
+struct dc_dpia_bw_alloc {
+	int sink_verified_bw;  // The Verified BW that sink can allocated and use that has been verified already
+	int sink_allocated_bw; // The Actual Allocated BW that sink currently allocated
+	int sink_max_bw;       // The Max BW that sink can require/support
+	int estimated_bw;      // The estimated available BW for this DPIA
+	int bw_granularity;    // BW Granularity
+	bool bw_alloc_enabled; // The BW Alloc Mode Support is turned ON for all 3:  DP-Tx & Dpia & CM
+	bool response_ready;   // Response ready from the CM side
+};
+
+#define MAX_SINKS_PER_LINK 4
+
+enum dc_hpd_enable_select {
+	HPD_EN_FOR_ALL_EDP = 0,
+	HPD_EN_FOR_PRIMARY_EDP_ONLY,
+	HPD_EN_FOR_SECONDARY_EDP_ONLY,
+};
+
 #endif /* DC_TYPES_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
index 140297c8ff55..739298d2dff3 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
@@ -832,13 +832,8 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
 									LOG_FLAG_I2cAux_DceAux,
 									"dce_aux_transfer_with_retries: payload->defer_delay=%u",
 									payload->defer_delay);
-						if (payload->defer_delay > 1) {
-							msleep(payload->defer_delay);
-							defer_time_in_ms += payload->defer_delay;
-						} else if (payload->defer_delay <= 1) {
-							udelay(payload->defer_delay * 1000);
-							defer_time_in_ms += payload->defer_delay;
-						}
+						fsleep(payload->defer_delay * 1000);
+						defer_time_in_ms += payload->defer_delay;
 					}
 				}
 				break;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
index 165392380842..462c7a3ec3cc 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
@@ -930,7 +930,13 @@ static bool dce112_program_pix_clk(
 		REG_WRITE(MODULO[inst], dp_dto_ref_100hz);
 
 		/* Enable DTO */
-		REG_UPDATE(PIXEL_RATE_CNTL[inst], DP_DTO0_ENABLE, 1);
+		if (clk_src->cs_mask->PIPE0_DTO_SRC_SEL)
+			REG_UPDATE_2(PIXEL_RATE_CNTL[inst],
+					DP_DTO0_ENABLE, 1,
+					PIPE0_DTO_SRC_SEL, 1);
+		else
+			REG_UPDATE(PIXEL_RATE_CNTL[inst],
+					DP_DTO0_ENABLE, 1);
 		return true;
 	}
 	/* First disable SS
@@ -995,7 +1001,6 @@ static bool dcn31_program_pix_clk(
 			REG_WRITE(PHASE[inst], pll_settings->actual_pix_clk_100hz * 100);
 			REG_WRITE(MODULO[inst], dp_dto_ref_khz * 1000);
 		}
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 		/* Enable DTO */
 		if (clk_src->cs_mask->PIPE0_DTO_SRC_SEL)
 			if (encoding == DP_128b_132b_ENCODING)
@@ -1009,9 +1014,6 @@ static bool dcn31_program_pix_clk(
 		else
 			REG_UPDATE(PIXEL_RATE_CNTL[inst],
 					DP_DTO0_ENABLE, 1);
-#else
-		REG_UPDATE(PIXEL_RATE_CNTL[inst], DP_DTO0_ENABLE, 1);
-#endif
 	} else {
 		if (IS_FPGA_MAXIMUS_DC(clock_source->ctx->dce_environment)) {
 			unsigned int inst = pix_clk_params->controller_id - CONTROLLER_ID_D0;
@@ -1023,7 +1025,6 @@ static bool dcn31_program_pix_clk(
 			REG_WRITE(MODULO[inst], dp_dto_ref_100hz);
 
 			/* Enable DTO */
-	#if defined(CONFIG_DRM_AMD_DC_DCN)
 			if (clk_src->cs_mask->PIPE0_DTO_SRC_SEL)
 				REG_UPDATE_2(PIXEL_RATE_CNTL[inst],
 						DP_DTO0_ENABLE, 1,
@@ -1031,17 +1032,12 @@ static bool dcn31_program_pix_clk(
 			else
 				REG_UPDATE(PIXEL_RATE_CNTL[inst],
 						DP_DTO0_ENABLE, 1);
-	#else
-			REG_UPDATE(PIXEL_RATE_CNTL[inst], DP_DTO0_ENABLE, 1);
-	#endif
 			return true;
 		}
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 		if (clk_src->cs_mask->PIPE0_DTO_SRC_SEL)
 			REG_UPDATE(PIXEL_RATE_CNTL[inst],
 					PIPE0_DTO_SRC_SEL, 0);
-#endif
 
 		/*ATOMBIOS expects pixel rate adjusted by deep color ratio)*/
 		bp_pc_params.controller_id = pix_clk_params->controller_id;
@@ -1161,6 +1157,7 @@ const struct pixel_rate_range_table_entry video_optimized_pixel_rates[] = {
 	{25170, 25180, 25200, 1000, 1001},	//25.2MHz   ->   25.17
 	{59340, 59350, 59400, 1000, 1001},	//59.4Mhz   ->   59.340
 	{74170, 74180, 74250, 1000, 1001},	//74.25Mhz  ->   74.1758
+	{89910, 90000, 90000, 1000, 1001},	//90Mhz     ->   89.91
 	{125870, 125880, 126000, 1000, 1001},	//126Mhz    ->  125.87
 	{148350, 148360, 148500, 1000, 1001},	//148.5Mhz  ->  148.3516
 	{167830, 167840, 168000, 1000, 1001},	//168Mhz    ->  167.83
@@ -1274,7 +1271,14 @@ static bool dcn3_program_pix_clk(
 			REG_WRITE(PHASE[inst], pll_settings->actual_pix_clk_100hz * 100);
 			REG_WRITE(MODULO[inst], dp_dto_ref_khz * 1000);
 		}
-		REG_UPDATE(PIXEL_RATE_CNTL[inst], DP_DTO0_ENABLE, 1);
+		/* Enable DTO */
+		if (clk_src->cs_mask->PIPE0_DTO_SRC_SEL)
+			REG_UPDATE_2(PIXEL_RATE_CNTL[inst],
+					DP_DTO0_ENABLE, 1,
+					PIPE0_DTO_SRC_SEL, 1);
+		else
+			REG_UPDATE(PIXEL_RATE_CNTL[inst],
+					DP_DTO0_ENABLE, 1);
 	} else
 		// For other signal types(HDMI_TYPE_A, DVI) Driver still to call VBIOS Command table
 		dce112_program_pix_clk(clock_source, pix_clk_params, encoding, pll_settings);
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.h b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.h
index aaf33c79b09b..f600b7431e23 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.h
@@ -204,23 +204,17 @@
 	type DP_DTO0_MODULO; \
 	type DP_DTO0_ENABLE;
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 #define CS_REG_FIELD_LIST_DCN32(type) \
 	type PIPE0_DTO_SRC_SEL;
-#endif
 
 struct dce110_clk_src_shift {
 	CS_REG_FIELD_LIST(uint8_t)
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	CS_REG_FIELD_LIST_DCN32(uint8_t)
-#endif
 };
 
 struct dce110_clk_src_mask{
 	CS_REG_FIELD_LIST(uint32_t)
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	CS_REG_FIELD_LIST_DCN32(uint32_t)
-#endif
 };
 
 struct dce110_clk_src_regs {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
index d3cc5ec46956..e74266cc0098 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
@@ -586,7 +586,7 @@ static void dcn10_dmcu_set_psr_enable(struct dmcu *dmcu, bool enable, bool wait)
 				if (state == PSR_STATE0)
 					break;
 			}
-			udelay(500);
+			fsleep(500);
 		}
 
 		/* assert if max retry hit */
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
index d9fd4ec60588..670d5ab9d998 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
@@ -1009,7 +1009,7 @@ static void dce_transform_set_pixel_storage_depth(
 		color_depth = COLOR_DEPTH_101010;
 		pixel_depth = 0;
 		expan_mode  = 1;
-		BREAK_TO_DEBUGGER();
+		DC_LOG_DC("The pixel depth %d is not valid, set COLOR_DEPTH_101010 instead.", depth);
 		break;
 	}
 
@@ -1023,8 +1023,7 @@ static void dce_transform_set_pixel_storage_depth(
 	if (!(xfm_dce->lb_pixel_depth_supported & depth)) {
 		/*we should use unsupported capabilities
 		 *  unless it is required by w/a*/
-		DC_LOG_WARNING("%s: Capability not supported",
-			__func__);
+		DC_LOG_DC("%s: Capability not supported", __func__);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c
index fb0dec4ed3a6..9fc48208c2e4 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c
@@ -148,7 +148,7 @@ static bool dmub_abm_set_level(struct abm *abm, uint32_t level)
 	int edp_num;
 	uint8_t panel_mask = 0;
 
-	get_edp_links(dc->dc, edp_links, &edp_num);
+	dc_get_edp_links(dc->dc, edp_links, &edp_num);
 
 	for (i = 0; i < edp_num; i++) {
 		if (edp_links[i]->link_status.link_active)
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
index 1e2d2cbe2c37..9705d8f88382 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
@@ -33,6 +33,9 @@
 
 #define MAX_PIPES 6
 
+static const uint8_t DP_SINK_DEVICE_STR_ID_1[] = {7, 1, 8, 7, 3};
+static const uint8_t DP_SINK_DEVICE_STR_ID_2[] = {7, 1, 8, 7, 5};
+
 /*
  * Convert dmcub psr state to dmcu psr state.
  */
@@ -215,7 +218,7 @@ static void dmub_psr_enable(struct dmub_psr *dmub, bool enable, bool wait, uint8
 					break;
 			}
 
-			udelay(500);
+			fsleep(500);
 		}
 
 		/* assert if max retry hit */
@@ -250,7 +253,7 @@ static void dmub_psr_set_level(struct dmub_psr *dmub, uint16_t psr_level, uint8_
 	dc_dmub_srv_wait_idle(dc->dmub_srv);
 }
 
-/**
+/*
  * Set PSR vtotal requirement for FreeSync PSR.
  */
 static void dmub_psr_set_sink_vtotal_in_psr_active(struct dmub_psr *dmub,
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.h b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.h
index 74005b9d352a..289e42070ece 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.h
@@ -26,8 +26,9 @@
 #ifndef _DMUB_PSR_H_
 #define _DMUB_PSR_H_
 
-#include "os_types.h"
-#include "dc_link.h"
+#include "dc_types.h"
+struct dc_link;
+struct dmub_psr_funcs;
 
 struct dmub_psr {
 	struct dc_context *ctx;
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
index 0d4d3d586166..9fe0ce91db00 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
@@ -741,7 +741,7 @@ void dce110_edp_wait_for_hpd_ready(
 
 	/* obtain HPD */
 	/* TODO what to do with this? */
-	hpd = link_get_hpd_gpio(ctx->dc_bios, connector, ctx->gpio_service);
+	hpd = ctx->dc->link_srv->get_hpd_gpio(ctx->dc_bios, connector, ctx->gpio_service);
 
 	if (!hpd) {
 		BREAK_TO_DEBUGGER();
@@ -809,19 +809,19 @@ void dce110_edp_power_control(
 				div64_u64(dm_get_elapse_time_in_ns(
 						ctx,
 						current_ts,
-						link_dp_trace_get_edp_poweroff_timestamp(link)), 1000000);
+						ctx->dc->link_srv->dp_trace_get_edp_poweroff_timestamp(link)), 1000000);
 		unsigned long long time_since_edp_poweron_ms =
 				div64_u64(dm_get_elapse_time_in_ns(
 						ctx,
 						current_ts,
-						link_dp_trace_get_edp_poweron_timestamp(link)), 1000000);
+						ctx->dc->link_srv->dp_trace_get_edp_poweron_timestamp(link)), 1000000);
 		DC_LOG_HW_RESUME_S3(
 				"%s: transition: power_up=%d current_ts=%llu edp_poweroff=%llu edp_poweron=%llu time_since_edp_poweroff_ms=%llu time_since_edp_poweron_ms=%llu",
 				__func__,
 				power_up,
 				current_ts,
-				link_dp_trace_get_edp_poweroff_timestamp(link),
-				link_dp_trace_get_edp_poweron_timestamp(link),
+				ctx->dc->link_srv->dp_trace_get_edp_poweroff_timestamp(link),
+				ctx->dc->link_srv->dp_trace_get_edp_poweron_timestamp(link),
 				time_since_edp_poweroff_ms,
 				time_since_edp_poweron_ms);
 
@@ -836,7 +836,7 @@ void dce110_edp_power_control(
 					link->panel_config.pps.extra_t12_ms;
 
 			/* Adjust remaining_min_edp_poweroff_time_ms if this is not the first time. */
-			if (link_dp_trace_get_edp_poweroff_timestamp(link) != 0) {
+			if (ctx->dc->link_srv->dp_trace_get_edp_poweroff_timestamp(link) != 0) {
 				if (time_since_edp_poweroff_ms < remaining_min_edp_poweroff_time_ms)
 					remaining_min_edp_poweroff_time_ms =
 						remaining_min_edp_poweroff_time_ms - time_since_edp_poweroff_ms;
@@ -896,13 +896,13 @@ void dce110_edp_power_control(
 				__func__, (power_up ? "On":"Off"),
 				bp_result);
 
-		link_dp_trace_set_edp_power_timestamp(link, power_up);
+		ctx->dc->link_srv->dp_trace_set_edp_power_timestamp(link, power_up);
 
 		DC_LOG_HW_RESUME_S3(
 				"%s: updated values: edp_poweroff=%llu edp_poweron=%llu\n",
 				__func__,
-				link_dp_trace_get_edp_poweroff_timestamp(link),
-				link_dp_trace_get_edp_poweron_timestamp(link));
+				ctx->dc->link_srv->dp_trace_get_edp_poweroff_timestamp(link),
+				ctx->dc->link_srv->dp_trace_get_edp_poweron_timestamp(link));
 
 		if (bp_result != BP_RESULT_OK)
 			DC_LOG_ERROR(
@@ -930,14 +930,14 @@ void dce110_edp_wait_for_T12(
 		return;
 
 	if (!link->panel_cntl->funcs->is_panel_powered_on(link->panel_cntl) &&
-			link_dp_trace_get_edp_poweroff_timestamp(link) != 0) {
+			ctx->dc->link_srv->dp_trace_get_edp_poweroff_timestamp(link) != 0) {
 		unsigned int t12_duration = 500; // Default T12 as per spec
 		unsigned long long current_ts = dm_get_timestamp(ctx);
 		unsigned long long time_since_edp_poweroff_ms =
 				div64_u64(dm_get_elapse_time_in_ns(
 						ctx,
 						current_ts,
-						link_dp_trace_get_edp_poweroff_timestamp(link)), 1000000);
+						ctx->dc->link_srv->dp_trace_get_edp_poweroff_timestamp(link)), 1000000);
 
 		t12_duration += link->panel_config.pps.extra_t12_ms; // Add extra T12
 
@@ -1018,7 +1018,7 @@ void dce110_edp_backlight_control(
 		 * we shouldn't be doing power-sequencing, hence we can skip
 		 * waiting for T7-ready.
 		 */
-			link_edp_receiver_ready_T7(link);
+			ctx->dc->link_srv->edp_receiver_ready_T7(link);
 		else
 			DC_LOG_DC("edp_receiver_ready_T7 skipped\n");
 	}
@@ -1049,7 +1049,7 @@ void dce110_edp_backlight_control(
 	if (link->dpcd_sink_ext_caps.bits.oled ||
 		link->dpcd_sink_ext_caps.bits.hdr_aux_backlight_control == 1 ||
 		link->dpcd_sink_ext_caps.bits.sdr_aux_backlight_control == 1)
-		link_backlight_enable_aux(link, enable);
+		ctx->dc->link_srv->edp_backlight_enable_aux(link, enable);
 
 	/*edp 1.2*/
 	if (cntl.action == TRANSMITTER_CONTROL_BACKLIGHT_OFF) {
@@ -1061,7 +1061,7 @@ void dce110_edp_backlight_control(
 		 * we shouldn't be doing power-sequencing, hence we can skip
 		 * waiting for T9-ready.
 		 */
-			link_edp_add_delay_for_T9(link);
+			ctx->dc->link_srv->edp_add_delay_for_T9(link);
 		else
 			DC_LOG_DC("edp_receiver_ready_T9 skipped\n");
 	}
@@ -1161,7 +1161,7 @@ void dce110_disable_stream(struct pipe_ctx *pipe_ctx)
 			pipe_ctx->stream_res.stream_enc);
 	}
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->stop_dp_info_packets(
 					pipe_ctx->stream_res.hpo_dp_stream_enc);
 	} else if (dc_is_dp_signal(pipe_ctx->stream->signal))
@@ -1172,7 +1172,7 @@ void dce110_disable_stream(struct pipe_ctx *pipe_ctx)
 
 	link_hwss->reset_stream_encoder(pipe_ctx);
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		dto_params.otg_inst = tg->inst;
 		dto_params.timing = &pipe_ctx->stream->timing;
 		dp_hpo_inst = pipe_ctx->stream_res.hpo_dp_stream_enc->inst;
@@ -1181,7 +1181,7 @@ void dce110_disable_stream(struct pipe_ctx *pipe_ctx)
 		dccg->funcs->set_dpstreamclk(dccg, REFCLK, tg->inst, dp_hpo_inst);
 	}
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		/* TODO: This looks like a bug to me as we are disabling HPO IO when
 		 * we are just disabling a single HPO stream. Shouldn't we disable HPO
 		 * HW control only when HPOs for all streams are disabled?
@@ -1223,7 +1223,7 @@ void dce110_blank_stream(struct pipe_ctx *pipe_ctx)
 		link->dc->hwss.set_abm_immediate_disable(pipe_ctx);
 	}
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (link->dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		/* TODO - DP2.0 HW: Set ODM mode in dp hpo encoder here */
 		pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_blank(
 				pipe_ctx->stream_res.hpo_dp_stream_enc);
@@ -1245,7 +1245,7 @@ void dce110_blank_stream(struct pipe_ctx *pipe_ctx)
 				 * we shouldn't be doing power-sequencing, hence we can skip
 				 * waiting for T9-ready.
 				 */
-				link_edp_receiver_ready_T9(link);
+				link->dc->link_srv->edp_receiver_ready_T9(link);
 			}
 		}
 	}
@@ -1428,7 +1428,7 @@ static enum dc_status dce110_enable_stream_timing(
 		if (false == pipe_ctx->clock_source->funcs->program_pix_clk(
 				pipe_ctx->clock_source,
 				&pipe_ctx->stream_res.pix_clk_params,
-				link_dp_get_encoding_format(&pipe_ctx->link_config.dp_link_settings),
+				dc->link_srv->dp_get_encoding_format(&pipe_ctx->link_config.dp_link_settings),
 				&pipe_ctx->pll_settings)) {
 			BREAK_TO_DEBUGGER();
 			return DC_ERROR_UNEXPECTED;
@@ -1532,7 +1532,7 @@ static enum dc_status apply_single_controller_ctx_to_hw(
 	 * To do so, move calling function enable_stream_timing to only be done AFTER calling
 	 * function core_link_enable_stream
 	 */
-	if (!(hws->wa.dp_hpo_and_otg_sequence && link_is_dp_128b_132b_signal(pipe_ctx)))
+	if (!(hws->wa.dp_hpo_and_otg_sequence && dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)))
 		/*  */
 		/* Do not touch stream timing on seamless boot optimization. */
 		if (!pipe_ctx->stream->apply_seamless_boot_optimization)
@@ -1564,17 +1564,17 @@ static enum dc_status apply_single_controller_ctx_to_hw(
 			pipe_ctx->stream_res.tg->inst);
 
 	if (dc_is_dp_signal(pipe_ctx->stream->signal))
-		link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_CONNECT_DIG_FE_OTG);
+		dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_CONNECT_DIG_FE_OTG);
 
 	if (!stream->dpms_off)
-		link_set_dpms_on(context, pipe_ctx);
+		dc->link_srv->set_dpms_on(context, pipe_ctx);
 
 	/* DCN3.1 FPGA Workaround
 	 * Need to enable HPO DP Stream Encoder before setting OTG master enable.
 	 * To do so, move calling function enable_stream_timing to only be done AFTER calling
 	 * function core_link_enable_stream
 	 */
-	if (hws->wa.dp_hpo_and_otg_sequence && link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (hws->wa.dp_hpo_and_otg_sequence && dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		if (!pipe_ctx->stream->apply_seamless_boot_optimization)
 			hws->funcs.enable_stream_timing(pipe_ctx, context, dc);
 	}
@@ -1600,7 +1600,7 @@ static void power_down_encoders(struct dc *dc)
 	for (i = 0; i < dc->link_count; i++) {
 		enum signal_type signal = dc->links[i]->connector_signal;
 
-		link_blank_dp_stream(dc->links[i], false);
+		dc->link_srv->blank_dp_stream(dc->links[i], false);
 
 		if (signal != SIGNAL_TYPE_EDP)
 			signal = SIGNAL_TYPE_NONE;
@@ -1739,7 +1739,7 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context)
 
 
 	get_edp_links_with_sink(dc, edp_links_with_sink, &edp_with_sink_num);
-	get_edp_links(dc, edp_links, &edp_num);
+	dc_get_edp_links(dc, edp_links, &edp_num);
 
 	if (hws->funcs.init_pipes)
 		hws->funcs.init_pipes(dc, context);
@@ -2083,7 +2083,7 @@ static void dce110_reset_hw_ctx_wrap(
 			 * disabled already, no need to disable again.
 			 */
 			if (!pipe_ctx->stream || !pipe_ctx->stream->dpms_off) {
-				link_set_dpms_off(pipe_ctx_old);
+				dc->link_srv->set_dpms_off(pipe_ctx_old);
 
 				/* free acquired resources*/
 				if (pipe_ctx_old->stream_res.audio) {
@@ -3054,13 +3054,13 @@ void dce110_enable_dp_link_output(
 				pipes[i].clock_source->funcs->program_pix_clk(
 						pipes[i].clock_source,
 						&pipes[i].stream_res.pix_clk_params,
-						link_dp_get_encoding_format(link_settings),
+						dc->link_srv->dp_get_encoding_format(link_settings),
 						&pipes[i].pll_settings);
 			}
 		}
 	}
 
-	if (link_dp_get_encoding_format(link_settings) == DP_8b_10b_ENCODING) {
+	if (dc->link_srv->dp_get_encoding_format(link_settings) == DP_8b_10b_ENCODING) {
 		if (dc->clk_mgr->funcs->notify_link_rate_change)
 			dc->clk_mgr->funcs->notify_link_rate_change(dc->clk_mgr, link);
 	}
@@ -3077,7 +3077,7 @@ void dce110_enable_dp_link_output(
 	if (dmcu != NULL && dmcu->funcs->unlock_phy)
 		dmcu->funcs->unlock_phy(dmcu);
 
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_LINK_PHY);
+	dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_LINK_PHY);
 }
 
 void dce110_disable_link_output(struct dc_link *link,
@@ -3102,7 +3102,7 @@ void dce110_disable_link_output(struct dc_link *link,
 		link->dc->hwss.edp_power_control(link, false);
 	else if (dmcu != NULL && dmcu->funcs->lock_phy)
 		dmcu->funcs->unlock_phy(dmcu);
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
+	dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
 }
 
 static const struct hw_sequencer_funcs dce110_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
index 394d83a97f33..08028a1779ae 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
@@ -71,8 +71,6 @@ void dce110_optimize_bandwidth(
 		struct dc *dc,
 		struct dc_state *context);
 
-void dc_link_dp_receiver_power_ctrl(struct dc_link *link, bool on);
-
 void dce110_edp_power_control(
 		struct dc_link *link,
 		bool power_up);
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
index f808315b2835..a4a45a6ce61e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
@@ -401,8 +401,6 @@ static const struct resource_caps stoney_resource_cap = {
 
 static const struct dc_plane_cap plane_cap = {
 		.type = DC_PLANE_TYPE_DCE_RGB,
-		.blends_with_below = true,
-		.blends_with_above = true,
 		.per_pixel_alpha = 1,
 
 		.pixel_format_support = {
@@ -428,7 +426,6 @@ static const struct dc_plane_cap plane_cap = {
 
 static const struct dc_plane_cap underlay_plane_cap = {
 		.type = DC_PLANE_TYPE_DCE_UNDERLAY,
-		.blends_with_above = true,
 		.per_pixel_alpha = 1,
 
 		.pixel_format_support = {
diff --git a/drivers/gpu/drm/amd/display/dc/dce60/Makefile b/drivers/gpu/drm/amd/display/dc/dce60/Makefile
index dda596fa1cd7..fee331accc0e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce60/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dce60/Makefile
@@ -23,7 +23,7 @@
 # Makefile for the 'controller' sub-component of DAL.
 # It provides the control and status of HW CRTC block.
 
-CFLAGS_AMDDALPATH)/dc/dce60/dce60_resource.o = $(call cc-disable-warning, override-init)
+CFLAGS_$(AMDDALPATH)/dc/dce60/dce60_resource.o = $(call cc-disable-warning, override-init)
 
 DCE60 = dce60_timing_generator.o dce60_hw_sequencer.o \
 	dce60_resource.o
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
index 71b3a6949001..c9e045666dcc 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
@@ -59,6 +59,7 @@
 	SRI(LB_DATA_FORMAT, DSCL, id), \
 	SRI(LB_MEMORY_CTRL, DSCL, id), \
 	SRI(DSCL_AUTOCAL, DSCL, id), \
+	SRI(DSCL_CONTROL, DSCL, id), \
 	SRI(SCL_BLACK_OFFSET, DSCL, id), \
 	SRI(SCL_TAP_CONTROL, DSCL, id), \
 	SRI(SCL_COEF_RAM_TAP_SELECT, DSCL, id), \
@@ -209,6 +210,7 @@
 	TF_SF(DSCL0_DSCL_AUTOCAL, AUTOCAL_MODE, mask_sh),\
 	TF_SF(DSCL0_DSCL_AUTOCAL, AUTOCAL_NUM_PIPE, mask_sh),\
 	TF_SF(DSCL0_DSCL_AUTOCAL, AUTOCAL_PIPE_ID, mask_sh),\
+	TF_SF(DSCL0_DSCL_CONTROL, SCL_BOUNDARY_MODE, mask_sh),\
 	TF_SF(DSCL0_SCL_BLACK_OFFSET, SCL_BLACK_OFFSET_RGB_Y, mask_sh),\
 	TF_SF(DSCL0_SCL_BLACK_OFFSET, SCL_BLACK_OFFSET_CBCR, mask_sh),\
 	TF_SF(DSCL0_SCL_TAP_CONTROL, SCL_V_NUM_TAPS, mask_sh),\
@@ -495,6 +497,7 @@
 	type AUTOCAL_MODE; \
 	type AUTOCAL_NUM_PIPE; \
 	type AUTOCAL_PIPE_ID; \
+	type SCL_BOUNDARY_MODE; \
 	type SCL_BLACK_OFFSET_RGB_Y; \
 	type SCL_BLACK_OFFSET_CBCR; \
 	type SCL_V_NUM_TAPS; \
@@ -1108,6 +1111,7 @@ struct dcn_dpp_mask {
 	uint32_t LB_DATA_FORMAT; \
 	uint32_t LB_MEMORY_CTRL; \
 	uint32_t DSCL_AUTOCAL; \
+	uint32_t DSCL_CONTROL; \
 	uint32_t SCL_BLACK_OFFSET; \
 	uint32_t SCL_TAP_CONTROL; \
 	uint32_t SCL_COEF_RAM_TAP_SELECT; \
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
index f62368da875d..b33955928bd0 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
@@ -655,6 +655,10 @@ void dpp1_dscl_set_scaler_manual_scale(struct dpp *dpp_base,
 		AUTOCAL_NUM_PIPE, 0,
 		AUTOCAL_PIPE_ID, 0);
 
+	/*clean scaler boundary mode when Autocal off*/
+	REG_SET(DSCL_CONTROL, 0,
+		SCL_BOUNDARY_MODE, 0);
+
 	/* Recout */
 	dpp1_dscl_set_recout(dpp, &scl_data->recout);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.c
index b6391a5ead78..365a3215f6d5 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.c
@@ -23,8 +23,6 @@
  *
  */
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
-
 #include "reg_helper.h"
 #include "resource.h"
 #include "dwb.h"
@@ -129,6 +127,3 @@ void dcn10_dwbc_construct(struct dcn10_dwbc *dwbc10,
 	dwbc10->dwbc_shift = dwbc_shift;
 	dwbc10->dwbc_mask = dwbc_mask;
 }
-
-
-#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.h
index d56ea7c8171e..5268c46ae907 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.h
@@ -24,8 +24,6 @@
 #ifndef __DC_DWBC_DCN10_H__
 #define __DC_DWBC_DCN10_H__
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
-
 /* DCN */
 #define BASE_INNER(seg) \
 	DCE_BASE__INST0_SEG ## seg
@@ -267,5 +265,3 @@ void dcn10_dwbc_construct(struct dcn10_dwbc *dwbc10,
 		int inst);
 
 #endif
-
-#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
index a142a00bc432..bf399819ca80 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
@@ -755,8 +755,8 @@ bool hubp1_is_flip_pending(struct hubp *hubp)
 	return false;
 }
 
-uint32_t aperture_default_system = 1;
-uint32_t context0_default_system; /* = 0;*/
+static uint32_t aperture_default_system = 1;
+static uint32_t context0_default_system; /* = 0;*/
 
 static void hubp1_set_vm_system_aperture_settings(struct hubp *hubp,
 		struct vm_system_aperture_param *apt)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
index a1a29c508394..1c3b6f25a782 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
@@ -726,11 +726,15 @@ void dcn10_hubp_pg_control(
 	}
 }
 
-static void power_on_plane(
+static void power_on_plane_resources(
 	struct dce_hwseq *hws,
 	int plane_id)
 {
 	DC_LOGGER_INIT(hws->ctx->logger);
+
+	if (hws->funcs.dpp_root_clock_control)
+		hws->funcs.dpp_root_clock_control(hws, plane_id, true);
+
 	if (REG(DC_IP_REQUEST_CNTL)) {
 		REG_SET(DC_IP_REQUEST_CNTL, 0,
 				IP_REQUEST_EN, 1);
@@ -919,7 +923,7 @@ enum dc_status dcn10_enable_stream_timing(
 	if (false == pipe_ctx->clock_source->funcs->program_pix_clk(
 			pipe_ctx->clock_source,
 			&pipe_ctx->stream_res.pix_clk_params,
-			link_dp_get_encoding_format(&pipe_ctx->link_config.dp_link_settings),
+			dc->link_srv->dp_get_encoding_format(&pipe_ctx->link_config.dp_link_settings),
 			&pipe_ctx->pll_settings)) {
 		BREAK_TO_DEBUGGER();
 		return DC_ERROR_UNEXPECTED;
@@ -1017,7 +1021,7 @@ static void dcn10_reset_back_end_for_pipe(
 		 * VBIOS lit up eDP, so check link status too.
 		 */
 		if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
-			link_set_dpms_off(pipe_ctx);
+			dc->link_srv->set_dpms_off(pipe_ctx);
 		else if (pipe_ctx->stream_res.audio)
 			dc->hwss.disable_audio_stream(pipe_ctx);
 
@@ -1237,11 +1241,15 @@ void dcn10_plane_atomic_power_down(struct dc *dc,
 			hws->funcs.hubp_pg_control(hws, hubp->inst, false);
 
 		dpp->funcs->dpp_reset(dpp);
+
 		REG_SET(DC_IP_REQUEST_CNTL, 0,
 				IP_REQUEST_EN, 0);
 		DC_LOG_DEBUG(
 				"Power gated front end %d\n", hubp->inst);
 	}
+
+	if (hws->funcs.dpp_root_clock_control)
+		hws->funcs.dpp_root_clock_control(hws, dpp->inst, false);
 }
 
 /* disable HW used by plane.
@@ -1564,7 +1572,7 @@ void dcn10_init_hw(struct dc *dc)
 	}
 
 	/* we want to turn off all dp displays before doing detection */
-	link_blank_all_dp_displays(dc);
+	dc->link_srv->blank_all_dp_displays(dc);
 
 	if (hws->funcs.enable_power_gating_plane)
 		hws->funcs.enable_power_gating_plane(dc->hwseq, true);
@@ -1638,7 +1646,7 @@ void dcn10_power_down_on_boot(struct dc *dc)
 	int edp_num;
 	int i = 0;
 
-	get_edp_links(dc, edp_links, &edp_num);
+	dc_get_edp_links(dc, edp_links, &edp_num);
 	if (edp_num)
 		edp_link = edp_links[0];
 
@@ -2462,7 +2470,7 @@ static void dcn10_enable_plane(
 
 	undo_DEGVIDCN10_253_wa(dc);
 
-	power_on_plane(dc->hwseq,
+	power_on_plane_resources(dc->hwseq,
 		pipe_ctx->plane_res.hubp->inst);
 
 	/* enable DCFCLK current DCHUB */
@@ -3385,7 +3393,9 @@ static bool dcn10_can_pipe_disable_cursor(struct pipe_ctx *pipe_ctx)
 	for (test_pipe = pipe_ctx->top_pipe; test_pipe;
 	     test_pipe = test_pipe->top_pipe) {
 		// Skip invisible layer and pipe-split plane on same layer
-		if (!test_pipe->plane_state->visible || test_pipe->plane_state->layer_index == cur_layer)
+		if (!test_pipe->plane_state ||
+		    !test_pipe->plane_state->visible ||
+		    test_pipe->plane_state->layer_index == cur_layer)
 			continue;
 
 		r2 = test_pipe->plane_res.scl_data.recout;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
index c4287147b853..ee08b545aaea 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
@@ -1219,7 +1219,6 @@ void dcn10_link_encoder_update_mst_stream_allocation_table(
 	const struct link_mst_stream_allocation_table *table)
 {
 	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
-	uint32_t value0 = 0;
 	uint32_t value1 = 0;
 	uint32_t value2 = 0;
 	uint32_t slots = 0;
@@ -1321,7 +1320,7 @@ void dcn10_link_encoder_update_mst_stream_allocation_table(
 	do {
 		udelay(10);
 
-		value0 = REG_READ(DP_MSE_SAT_UPDATE);
+		REG_READ(DP_MSE_SAT_UPDATE);
 
 		REG_GET(DP_MSE_SAT_UPDATE,
 				DP_MSE_SAT_UPDATE, &value1);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
index 0b37bb0e184b..db766689af58 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
@@ -161,10 +161,20 @@ struct dcn_optc_registers {
 	uint32_t OTG_CRC_CNTL2;
 	uint32_t OTG_CRC0_DATA_RG;
 	uint32_t OTG_CRC0_DATA_B;
+	uint32_t OTG_CRC1_DATA_B;
+	uint32_t OTG_CRC2_DATA_B;
+	uint32_t OTG_CRC3_DATA_B;
+	uint32_t OTG_CRC1_DATA_RG;
+	uint32_t OTG_CRC2_DATA_RG;
+	uint32_t OTG_CRC3_DATA_RG;
 	uint32_t OTG_CRC0_WINDOWA_X_CONTROL;
 	uint32_t OTG_CRC0_WINDOWA_Y_CONTROL;
 	uint32_t OTG_CRC0_WINDOWB_X_CONTROL;
 	uint32_t OTG_CRC0_WINDOWB_Y_CONTROL;
+	uint32_t OTG_CRC1_WINDOWA_X_CONTROL;
+	uint32_t OTG_CRC1_WINDOWA_Y_CONTROL;
+	uint32_t OTG_CRC1_WINDOWB_X_CONTROL;
+	uint32_t OTG_CRC1_WINDOWB_Y_CONTROL;
 	uint32_t GSL_SOURCE_SELECT;
 	uint32_t DWB_SOURCE_SELECT;
 	uint32_t OTG_DSC_START_POSITION;
@@ -464,6 +474,15 @@ struct dcn_optc_registers {
 	type CRC0_R_CR;\
 	type CRC0_G_Y;\
 	type CRC0_B_CB;\
+	type CRC1_R_CR;\
+	type CRC1_G_Y;\
+	type CRC1_B_CB;\
+	type CRC2_R_CR;\
+	type CRC2_G_Y;\
+	type CRC2_B_CB;\
+	type CRC3_R_CR;\
+	type CRC3_G_Y;\
+	type CRC3_B_CB;\
 	type OTG_CRC0_WINDOWA_X_START;\
 	type OTG_CRC0_WINDOWA_X_END;\
 	type OTG_CRC0_WINDOWA_Y_START;\
@@ -472,6 +491,15 @@ struct dcn_optc_registers {
 	type OTG_CRC0_WINDOWB_X_END;\
 	type OTG_CRC0_WINDOWB_Y_START;\
 	type OTG_CRC0_WINDOWB_Y_END;\
+	type OTG_CRC_WINDOW_DB_EN;\
+	type OTG_CRC1_WINDOWA_X_START;\
+	type OTG_CRC1_WINDOWA_X_END;\
+	type OTG_CRC1_WINDOWA_Y_START;\
+	type OTG_CRC1_WINDOWA_Y_END;\
+	type OTG_CRC1_WINDOWB_X_START;\
+	type OTG_CRC1_WINDOWB_X_END;\
+	type OTG_CRC1_WINDOWB_Y_START;\
+	type OTG_CRC1_WINDOWB_Y_END;\
 	type GSL0_READY_SOURCE_SEL;\
 	type GSL1_READY_SOURCE_SEL;\
 	type GSL2_READY_SOURCE_SEL;\
@@ -525,6 +553,7 @@ struct dcn_optc_registers {
 #define TG_REG_FIELD_LIST_DCN3_2(type) \
 	type OTG_H_TIMING_DIV_MODE_MANUAL;
 
+
 struct dcn_optc_shift {
 	TG_REG_FIELD_LIST(uint8_t)
 	TG_REG_FIELD_LIST_DCN3_2(uint8_t)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
index 6bfac8088ab0..21ec1ba5ed75 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
@@ -504,8 +504,6 @@ static const struct resource_caps rv2_res_cap = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
@@ -544,8 +542,8 @@ static const struct dc_debug_options debug_defaults_drv = {
 		.disable_pplib_clock_request = false,
 		.disable_pplib_wm_range = false,
 		.pplib_wm_report_mode = WM_REPORT_DEFAULT,
-		.pipe_split_policy = MPC_SPLIT_AVOID,
-		.force_single_disp_pipe_split = false,
+		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+		.force_single_disp_pipe_split = true,
 		.disable_dcc = DCC_ENABLE,
 		.voltage_align_fclk = true,
 		.disable_stereo_support = true,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
index 3c451ab5d3ca..f496e952ceec 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
@@ -933,7 +933,7 @@ void enc1_stream_encoder_dp_blank(
 	/* disable DP stream */
 	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, 0);
 
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_DP_VID_STREAM);
+	link->dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_DP_VID_STREAM);
 
 	/* the encoder stops sending the video stream
 	 * at the start of the vertical blanking.
@@ -952,7 +952,7 @@ void enc1_stream_encoder_dp_blank(
 
 	REG_UPDATE(DP_STEER_FIFO, DP_STEER_FIFO_RESET, true);
 
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_FIFO_STEER_RESET);
+	link->dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_FIFO_STEER_RESET);
 }
 
 /* output video stream to link encoder */
@@ -1025,7 +1025,8 @@ void enc1_stream_encoder_dp_unblank(
 
 	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, true);
 
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_DP_VID_STREAM);
+	link->dc->link_srv->dp_trace_source_sequence(link,
+			DPCD_SOURCE_SEQ_AFTER_ENABLE_DP_VID_STREAM);
 }
 
 void enc1_stream_encoder_set_avmute(
@@ -1470,10 +1471,9 @@ void enc1_se_hdmi_audio_setup(
 void enc1_se_hdmi_audio_disable(
 	struct stream_encoder *enc)
 {
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	if (enc->afmt && enc->afmt->funcs->afmt_powerdown)
 		enc->afmt->funcs->afmt_powerdown(enc->afmt);
-#endif
+
 	enc1_se_enable_audio_clock(enc, false);
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
index 915a20461c77..7bdc146f7cb5 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
@@ -205,6 +205,11 @@
 	type PHYDSYMCLK_GATE_DISABLE; \
 	type PHYESYMCLK_GATE_DISABLE;
 
+#define DCCG314_REG_FIELD_LIST(type) \
+	type DSCCLK3_DTO_PHASE;\
+	type DSCCLK3_DTO_MODULO;\
+	type DSCCLK3_DTO_ENABLE;
+
 #define DCCG32_REG_FIELD_LIST(type) \
 	type DPSTREAMCLK0_EN;\
 	type DPSTREAMCLK1_EN;\
@@ -230,12 +235,14 @@
 	type DTBCLK_P2_SRC_SEL;\
 	type DTBCLK_P2_EN;\
 	type DTBCLK_P3_SRC_SEL;\
-	type DTBCLK_P3_EN;
+	type DTBCLK_P3_EN;\
+	type DENTIST_DISPCLK_CHG_DONE;
 
 struct dccg_shift {
 	DCCG_REG_FIELD_LIST(uint8_t)
 	DCCG3_REG_FIELD_LIST(uint8_t)
 	DCCG31_REG_FIELD_LIST(uint8_t)
+	DCCG314_REG_FIELD_LIST(uint8_t)
 	DCCG32_REG_FIELD_LIST(uint8_t)
 };
 
@@ -243,6 +250,7 @@ struct dccg_mask {
 	DCCG_REG_FIELD_LIST(uint32_t)
 	DCCG3_REG_FIELD_LIST(uint32_t)
 	DCCG31_REG_FIELD_LIST(uint32_t)
+	DCCG314_REG_FIELD_LIST(uint32_t)
 	DCCG32_REG_FIELD_LIST(uint32_t)
 };
 
@@ -272,6 +280,7 @@ struct dccg_registers {
 	uint32_t DSCCLK0_DTO_PARAM;
 	uint32_t DSCCLK1_DTO_PARAM;
 	uint32_t DSCCLK2_DTO_PARAM;
+	uint32_t DSCCLK3_DTO_PARAM;
 	uint32_t DPSTREAMCLK_ROOT_GATE_DISABLE;
 	uint32_t DPSTREAMCLK_GATE_DISABLE;
 	uint32_t DCCG_GATE_DISABLE_CNTL;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.c
index 42344aec60d6..5bd698cd6d20 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.c
@@ -50,7 +50,7 @@ static void dsc2_enable(struct display_stream_compressor *dsc, int opp_pipe);
 static void dsc2_disable(struct display_stream_compressor *dsc);
 static void dsc2_disconnect(struct display_stream_compressor *dsc);
 
-const struct dsc_funcs dcn20_dsc_funcs = {
+static const struct dsc_funcs dcn20_dsc_funcs = {
 	.dsc_get_enc_caps = dsc2_get_enc_caps,
 	.dsc_read_state = dsc2_read_state,
 	.dsc_validate_stream = dsc2_validate_stream,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c
index f1490e97b6ce..f8667be57046 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c
@@ -301,7 +301,7 @@ void dwb2_set_scaler(struct dwbc *dwbc, struct dc_dwb_params *params)
 
 }
 
-const struct dwbc_funcs dcn20_dwbc_funcs = {
+static const struct dwbc_funcs dcn20_dwbc_funcs = {
 	.get_caps		= dwb2_get_caps,
 	.enable			= dwb2_enable,
 	.disable		= dwb2_disable,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
index b83873a3a534..5403e9399a46 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
@@ -190,10 +190,15 @@ void dcn20_enable_power_gating_plane(
 	bool enable)
 {
 	bool force_on = true; /* disable power gating */
+	uint32_t org_ip_request_cntl = 0;
 
 	if (enable)
 		force_on = false;
 
+	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+
 	/* DCHUBP0/1/2/3/4/5 */
 	REG_UPDATE(DOMAIN0_PG_CONFIG, DOMAIN0_POWER_FORCEON, force_on);
 	REG_UPDATE(DOMAIN2_PG_CONFIG, DOMAIN2_POWER_FORCEON, force_on);
@@ -224,6 +229,10 @@ void dcn20_enable_power_gating_plane(
 		REG_UPDATE(DOMAIN20_PG_CONFIG, DOMAIN20_POWER_FORCEON, force_on);
 	if (REG(DOMAIN21_PG_CONFIG))
 		REG_UPDATE(DOMAIN21_PG_CONFIG, DOMAIN21_POWER_FORCEON, force_on);
+
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 0);
+
 }
 
 void dcn20_dccg_init(struct dce_hwseq *hws)
@@ -711,7 +720,7 @@ enum dc_status dcn20_enable_stream_timing(
 	if (false == pipe_ctx->clock_source->funcs->program_pix_clk(
 			pipe_ctx->clock_source,
 			&pipe_ctx->stream_res.pix_clk_params,
-			link_dp_get_encoding_format(&pipe_ctx->link_config.dp_link_settings),
+			dc->link_srv->dp_get_encoding_format(&pipe_ctx->link_config.dp_link_settings),
 			&pipe_ctx->pll_settings)) {
 		BREAK_TO_DEBUGGER();
 		return DC_ERROR_UNEXPECTED;
@@ -1121,11 +1130,15 @@ void dcn20_blank_pixel_data(
 }
 
 
-static void dcn20_power_on_plane(
+static void dcn20_power_on_plane_resources(
 	struct dce_hwseq *hws,
 	struct pipe_ctx *pipe_ctx)
 {
 	DC_LOGGER_INIT(hws->ctx->logger);
+
+	if (hws->funcs.dpp_root_clock_control)
+		hws->funcs.dpp_root_clock_control(hws, pipe_ctx->plane_res.dpp->inst, true);
+
 	if (REG(DC_IP_REQUEST_CNTL)) {
 		REG_SET(DC_IP_REQUEST_CNTL, 0,
 				IP_REQUEST_EN, 1);
@@ -1149,7 +1162,7 @@ static void dcn20_enable_plane(struct dc *dc, struct pipe_ctx *pipe_ctx,
 	//if (dc->debug.sanity_checks) {
 	//	dcn10_verify_allow_pstate_change_high(dc);
 	//}
-	dcn20_power_on_plane(dc->hwseq, pipe_ctx);
+	dcn20_power_on_plane_resources(dc->hwseq, pipe_ctx);
 
 	/* enable DCFCLK current DCHUB */
 	pipe_ctx->plane_res.hubp->funcs->hubp_clk_cntl(pipe_ctx->plane_res.hubp, true);
@@ -1711,10 +1724,8 @@ static void dcn20_program_pipe(
 				pipe_ctx->pipe_dlg_param.vupdate_offset,
 				pipe_ctx->pipe_dlg_param.vupdate_width);
 
-		if (pipe_ctx->stream->mall_stream_config.type != SUBVP_PHANTOM) {
-			pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK);
+		if (pipe_ctx->stream->mall_stream_config.type != SUBVP_PHANTOM)
 			pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE);
-		}
 
 		pipe_ctx->stream_res.tg->funcs->set_vtg_params(
 				pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing, true);
@@ -1982,6 +1993,16 @@ void dcn20_post_unlock_program_front_end(
 		}
 	}
 
+	/* P-State support transitions:
+	 * Natural -> FPO: 		P-State disabled in prepare, force disallow anytime is safe
+	 * FPO -> Natural: 		Unforce anytime after FW disable is safe (P-State will assert naturally)
+	 * Unsupported -> FPO:	P-State enabled in optimize, force disallow anytime is safe
+	 * FPO -> Unsupported:	P-State disabled in prepare, unforce disallow anytime is safe
+	 * FPO <-> SubVP:		Force disallow is maintained on the FPO / SubVP pipes
+	 */
+	if (hwseq && hwseq->funcs.update_force_pstate)
+		dc->hwseq->funcs.update_force_pstate(dc, context);
+
 	/* Only program the MALL registers after all the main and phantom pipes
 	 * are done programming.
 	 */
@@ -2396,7 +2417,7 @@ void dcn20_unblank_stream(struct pipe_ctx *pipe_ctx,
 
 	params.link_settings.link_rate = link_settings->link_rate;
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (link->dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		/* TODO - DP2.0 HW: Set ODM mode in dp hpo encoder here */
 		pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_unblank(
 				pipe_ctx->stream_res.hpo_dp_stream_enc,
@@ -2449,7 +2470,7 @@ static void dcn20_reset_back_end_for_pipe(
 		 * VBIOS lit up eDP, so check link status too.
 		 */
 		if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
-			link_set_dpms_off(pipe_ctx);
+			dc->link_srv->set_dpms_off(pipe_ctx);
 		else if (pipe_ctx->stream_res.audio)
 			dc->hwss.disable_audio_stream(pipe_ctx);
 
@@ -2469,7 +2490,7 @@ static void dcn20_reset_back_end_for_pipe(
 		}
 	}
 	else if (pipe_ctx->stream_res.dsc) {
-		link_set_dsc_enable(pipe_ctx, false);
+		dc->link_srv->set_dsc_enable(pipe_ctx, false);
 	}
 
 	/* by upper caller loop, parent pipe: pipe0, will be reset last.
@@ -2704,12 +2725,12 @@ void dcn20_enable_stream(struct pipe_ctx *pipe_ctx)
 	unsigned int k1_div = PIXEL_RATE_DIV_NA;
 	unsigned int k2_div = PIXEL_RATE_DIV_NA;
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		if (dc->hwseq->funcs.setup_hpo_hw_control)
 			dc->hwseq->funcs.setup_hpo_hw_control(dc->hwseq, true);
 	}
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		dp_hpo_inst = pipe_ctx->stream_res.hpo_dp_stream_enc->inst;
 		dccg->funcs->set_dpstreamclk(dccg, DTBCLK0, tg->inst, dp_hpo_inst);
 
@@ -2743,7 +2764,7 @@ void dcn20_enable_stream(struct pipe_ctx *pipe_ctx)
 	dc->hwss.update_info_frame(pipe_ctx);
 
 	if (dc_is_dp_signal(pipe_ctx->stream->signal))
-		link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
+		dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
 
 	/* enable early control to avoid corruption on DP monitor*/
 	active_total_with_borders =
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c
index ccd91792991b..259a98e4ee2c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c
@@ -297,7 +297,7 @@ void mcifwb2_dump_frame(struct mcif_wb *mcif_wb,
 	dump_info->size		= dest_height * (mcif_params->luma_pitch + mcif_params->chroma_pitch);
 }
 
-const struct mcif_wb_funcs dcn20_mmhubbub_funcs = {
+static const struct mcif_wb_funcs dcn20_mmhubbub_funcs = {
 	.enable_mcif		= mmhubbub2_enable_mcif,
 	.disable_mcif		= mmhubbub2_disable_mcif,
 	.config_mcif_buf	= mmhubbub2_config_mcif_buf,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c
index 116f67a0b989..5da6e44f284a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c
@@ -542,7 +542,7 @@ static struct mpcc *mpc2_get_mpcc_for_dpp(struct mpc_tree *tree, int dpp_id)
 	return NULL;
 }
 
-const struct mpc_funcs dcn20_mpc_funcs = {
+static const struct mpc_funcs dcn20_mpc_funcs = {
 	.read_mpcc_state = mpc1_read_mpcc_state,
 	.insert_plane = mpc1_insert_plane,
 	.remove_mpcc = mpc1_remove_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c
index 3af24ef9cb2d..1d8c5805ef20 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c
@@ -670,8 +670,6 @@ static const struct resource_caps res_cap_nv10 = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
@@ -714,7 +712,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 		.timing_trace = false,
 		.clock_trace = true,
 		.disable_pplib_clock_request = true,
-		.pipe_split_policy = MPC_SPLIT_AVOID_MULT_DISP,
+		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
 		.force_single_disp_pipe_split = false,
 		.disable_dcc = DCC_ENABLE,
 		.vsr_support = true,
@@ -1213,8 +1211,11 @@ static void dcn20_resource_destruct(struct dcn20_resource_pool *pool)
 	if (pool->base.pp_smu != NULL)
 		dcn20_pp_smu_destroy(&pool->base.pp_smu);
 
-	if (pool->base.oem_device != NULL)
-		link_destroy_ddc_service(&pool->base.oem_device);
+	if (pool->base.oem_device != NULL) {
+		struct dc *dc = pool->base.oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
+	}
 }
 
 struct hubp *dcn20_hubp_create(
@@ -2765,7 +2766,7 @@ static bool dcn20_resource_construct(
 		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
 		ddc_init_data.id.enum_id = 0;
 		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->base.oem_device = link_create_ddc_service(&ddc_init_data);
+		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
 	} else {
 		pool->base.oem_device = NULL;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.c
index 42865d6c0cdd..0b47aeb60e79 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.c
@@ -546,7 +546,8 @@ void enc2_stream_encoder_dp_unblank(
 
 	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, true);
 
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_DP_VID_STREAM);
+	link->dc->link_srv->dp_trace_source_sequence(link,
+			DPCD_SOURCE_SEQ_AFTER_ENABLE_DP_VID_STREAM);
 }
 
 static void enc2_dp_set_odm_combine(
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_dpp.c
index f50ab961bc17..a7268027a472 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_dpp.c
@@ -185,13 +185,6 @@ static bool dpp201_get_optimal_number_of_taps(
 		struct scaler_data *scl_data,
 		const struct scaling_taps *in_taps)
 {
-	uint32_t pixel_width;
-
-	if (scl_data->viewport.width > scl_data->recout.width)
-		pixel_width = scl_data->recout.width;
-	else
-		pixel_width = scl_data->viewport.width;
-
 	if (scl_data->viewport.width  != scl_data->h_active &&
 		scl_data->viewport.height != scl_data->v_active &&
 		dpp->caps->dscl_data_proc_format == DSCL_DATA_PRCESSING_FIXED_FORMAT &&
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c
index 61bcfa03c4e7..1aeb04fbd89d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c
@@ -541,8 +541,6 @@ void dcn201_pipe_control_lock(
 	bool lock)
 {
 	struct dce_hwseq *hws = dc->hwseq;
-	struct hubp *hubp = NULL;
-	hubp = dc->res_pool->hubps[pipe->pipe_idx];
 	/* use TG master update lock to lock everything on the TG
 	 * therefore only top pipe need to lock
 	 */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_mpc.c
index 95c4c55f067c..1af03a86ec9b 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_mpc.c
@@ -76,7 +76,7 @@ static void mpc201_init_mpcc(struct mpcc *mpcc, int mpcc_inst)
 	mpcc->shared_bottom = false;
 }
 
-const struct mpc_funcs dcn201_mpc_funcs = {
+static const struct mpc_funcs dcn201_mpc_funcs = {
 	.read_mpcc_state = mpc1_read_mpcc_state,
 	.insert_plane = mpc1_insert_plane,
 	.remove_mpcc = mpc1_remove_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_resource.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_resource.c
index 407d995bfa99..6ea70da28aaa 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_resource.c
@@ -74,7 +74,7 @@
 #define MIN_DISP_CLK_KHZ 100000
 #define MIN_DPP_CLK_KHZ 100000
 
-struct _vcs_dpi_ip_params_st dcn201_ip = {
+static struct _vcs_dpi_ip_params_st dcn201_ip = {
 	.gpuvm_enable = 0,
 	.hostvm_enable = 0,
 	.gpuvm_max_page_table_levels = 4,
@@ -136,7 +136,7 @@ struct _vcs_dpi_ip_params_st dcn201_ip = {
 	.number_of_cursors = 1,
 };
 
-struct _vcs_dpi_soc_bounding_box_st dcn201_soc = {
+static struct _vcs_dpi_soc_bounding_box_st dcn201_soc = {
 	.clock_limits = {
 			{
 				.state = 0,
@@ -571,8 +571,6 @@ static const struct resource_caps res_cap_dnc201 = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c
index 15475c7e2cf9..2a182c2f57d6 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c
@@ -132,8 +132,8 @@ void dcn21_PLAT_58856_wa(struct dc_state *context, struct pipe_ctx *pipe_ctx)
 		return;
 
 	pipe_ctx->stream->dpms_off = false;
-	link_set_dpms_on(context, pipe_ctx);
-	link_set_dpms_off(pipe_ctx);
+	pipe_ctx->stream->ctx->dc->link_srv->set_dpms_on(context, pipe_ctx);
+	pipe_ctx->stream->ctx->dc->link_srv->set_dpms_off(pipe_ctx);
 	pipe_ctx->stream->dpms_off = true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c
index 8f9244fe5c86..19aaa557b2db 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c
@@ -609,8 +609,6 @@ static const struct resource_caps res_cap_rn_FPGA_2pipe_dsc = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
@@ -642,7 +640,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 		.clock_trace = true,
 		.disable_pplib_clock_request = true,
 		.min_disp_clk_khz = 100000,
-		.pipe_split_policy = MPC_SPLIT_AVOID_MULT_DISP,
+		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
 		.force_single_disp_pipe_split = false,
 		.disable_dcc = DCC_ENABLE,
 		.vsr_support = true,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_afmt.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_afmt.c
index 95528e5ef89e..55e388c4c98b 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_afmt.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_afmt.c
@@ -123,7 +123,6 @@ void afmt3_se_audio_setup(
 {
 	struct dcn30_afmt *afmt3 = DCN30_AFMT_FROM_AFMT(afmt);
 
-	uint32_t speakers = 0;
 	uint32_t channels = 0;
 
 	ASSERT(audio_info);
@@ -131,7 +130,6 @@ void afmt3_se_audio_setup(
 	if (audio_info == NULL)
 		return;
 
-	speakers = audio_info->flags.info.ALLSPEAKERS;
 	channels = speakers_to_channels(audio_info->flags.speaker_flags).all;
 
 	/* setup the audio stream source select (audio -> dig mapping) */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.c
index 5f9079d3943a..9d08127d209b 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.c
@@ -28,6 +28,7 @@
 #include "dcn30_dio_stream_encoder.h"
 #include "reg_helper.h"
 #include "hw_shared.h"
+#include "dc.h"
 #include "core_types.h"
 #include <linux/delay.h>
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.h
index 6263408d71fc..2082372d69ee 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.h
@@ -102,6 +102,7 @@
 	SRI(LB_DATA_FORMAT, DSCL, id), \
 	SRI(LB_MEMORY_CTRL, DSCL, id), \
 	SRI(DSCL_AUTOCAL, DSCL, id), \
+	SRI(DSCL_CONTROL, DSCL, id), \
 	SRI(SCL_TAP_CONTROL, DSCL, id), \
 	SRI(SCL_COEF_RAM_TAP_SELECT, DSCL, id), \
 	SRI(SCL_COEF_RAM_TAP_DATA, DSCL, id), \
@@ -237,6 +238,7 @@
 	TF_SF(DSCL0_LB_MEMORY_CTRL, LB_MAX_PARTITIONS, mask_sh),\
 	TF_SF(DSCL0_DSCL_AUTOCAL, AUTOCAL_MODE, mask_sh),\
 	TF_SF(DSCL0_DSCL_AUTOCAL, AUTOCAL_NUM_PIPE, mask_sh),\
+	TF_SF(DSCL0_DSCL_CONTROL, SCL_BOUNDARY_MODE, mask_sh),\
 	TF_SF(DSCL0_DSCL_AUTOCAL, AUTOCAL_PIPE_ID, mask_sh),\
 	TF_SF(DSCL0_SCL_TAP_CONTROL, SCL_V_NUM_TAPS, mask_sh),\
 	TF_SF(DSCL0_SCL_TAP_CONTROL, SCL_H_NUM_TAPS, mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.c
index f14f69616692..0d98918bf0fc 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.c
@@ -220,7 +220,7 @@ void dwb3_set_denorm(struct dwbc *dwbc, struct dc_dwb_params *params)
 }
 
 
-const struct dwbc_funcs dcn30_dwbc_funcs = {
+static const struct dwbc_funcs dcn30_dwbc_funcs = {
 	.get_caps		= dwb3_get_caps,
 	.enable			= dwb3_enable,
 	.disable		= dwb3_disable,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.c
index dc3e8df706b3..e46bbe7ddcc9 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.c
@@ -47,13 +47,9 @@ void hubp3_set_vm_system_aperture_settings(struct hubp *hubp,
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	PHYSICAL_ADDRESS_LOC mc_vm_apt_default;
 	PHYSICAL_ADDRESS_LOC mc_vm_apt_low;
 	PHYSICAL_ADDRESS_LOC mc_vm_apt_high;
 
-	// The format of default addr is 48:12 of the 48 bit addr
-	mc_vm_apt_default.quad_part = apt->sys_default.quad_part >> 12;
-
 	// The format of high/low are 48:18 of the 48 bit addr
 	mc_vm_apt_low.quad_part = apt->sys_low.quad_part >> 18;
 	mc_vm_apt_high.quad_part = apt->sys_high.quad_part >> 18;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c
index df787fcf8e86..0e071fbc9154 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c
@@ -323,13 +323,10 @@ void dcn30_enable_writeback(
 {
 	struct dwbc *dwb;
 	struct mcif_wb *mcif_wb;
-	struct timing_generator *optc;
 
 	dwb = dc->res_pool->dwbc[wb_info->dwb_pipe_inst];
 	mcif_wb = dc->res_pool->mcif_wb[wb_info->dwb_pipe_inst];
 
-	/* set the OPTC source mux */
-	optc = dc->res_pool->timing_generators[dwb->otg_inst];
 	DC_LOG_DWB("%s dwb_pipe_inst = %d, mpcc_inst = %d",\
 		__func__, wb_info->dwb_pipe_inst,\
 		wb_info->mpcc_inst);
@@ -534,13 +531,8 @@ void dcn30_init_hw(struct dc *dc)
 		}
 	}
 
-	/* Power gate DSCs */
-	for (i = 0; i < res_pool->res_cap->num_dsc; i++)
-		if (hws->funcs.dsc_pg_control != NULL)
-			hws->funcs.dsc_pg_control(hws, res_pool->dscs[i]->inst, false);
-
 	/* we want to turn off all dp displays before doing detection */
-	link_blank_all_dp_displays(dc);
+	dc->link_srv->blank_all_dp_displays(dc);
 
 	if (hws->funcs.enable_power_gating_plane)
 		hws->funcs.enable_power_gating_plane(dc->hwseq, true);
@@ -567,7 +559,7 @@ void dcn30_init_hw(struct dc *dc)
 		struct dc_link *edp_links[MAX_NUM_EDP];
 		struct dc_link *edp_link = NULL;
 
-		get_edp_links(dc, edp_links, &edp_num);
+		dc_get_edp_links(dc, edp_links, &edp_num);
 		if (edp_num)
 			edp_link = edp_links[0];
 		if (edp_link && edp_link->link_enc->funcs->is_dig_enabled &&
@@ -629,7 +621,8 @@ void dcn30_init_hw(struct dc *dc)
 	if (dc->clk_mgr->funcs->notify_wm_ranges)
 		dc->clk_mgr->funcs->notify_wm_ranges(dc->clk_mgr);
 
-	if (dc->clk_mgr->funcs->set_hard_max_memclk)
+	//if softmax is enabled then hardmax will be set by a different call
+	if (dc->clk_mgr->funcs->set_hard_max_memclk && !dc->clk_mgr->dc_mode_softmax_enabled)
 		dc->clk_mgr->funcs->set_hard_max_memclk(dc->clk_mgr);
 
 	if (dc->res_pool->hubbub->funcs->force_pstate_change_control)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.c
index 7a93eff183d9..6f2a0d5d963b 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.c
@@ -211,7 +211,7 @@ static void mmhubbub3_config_mcif_arb(struct mcif_wb *mcif_wb,
 	REG_UPDATE(MCIF_WB_ARBITRATION_CONTROL, MCIF_WB_CLIENT_ARBITRATION_SLICE,  params->arbitration_slice);
 }
 
-const struct mcif_wb_funcs dcn30_mmhubbub_funcs = {
+static const struct mcif_wb_funcs dcn30_mmhubbub_funcs = {
 	.warmup_mcif		= mmhubbub3_warmup_mcif,
 	.enable_mcif		= mmhubbub2_enable_mcif,
 	.disable_mcif		= mmhubbub2_disable_mcif,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c
index ad1c1b703874..6cf40c1332bc 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c
@@ -1399,7 +1399,7 @@ static void mpc3_set_mpc_mem_lp_mode(struct mpc *mpc)
 	}
 }
 
-const struct mpc_funcs dcn30_mpc_funcs = {
+static const struct mpc_funcs dcn30_mpc_funcs = {
 	.read_mpcc_state = mpc1_read_mpcc_state,
 	.insert_plane = mpc1_insert_plane,
 	.remove_mpcc = mpc1_remove_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c
index 08b92715e2e6..c95f000b63b2 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c
@@ -291,7 +291,7 @@ static void optc3_set_timing_double_buffer(struct timing_generator *optc, bool e
 		   OTG_DRR_TIMING_DBUF_UPDATE_MODE, mode);
 }
 
-void optc3_wait_drr_doublebuffer_pending_clear(struct timing_generator *optc)
+static void optc3_wait_drr_doublebuffer_pending_clear(struct timing_generator *optc)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
index b5b5320c7bef..965f5ceb33f7 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
@@ -680,8 +680,6 @@ static const struct resource_caps res_cap_dcn3 = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
@@ -1207,8 +1205,11 @@ static void dcn30_resource_destruct(struct dcn30_resource_pool *pool)
 	if (pool->base.dccg != NULL)
 		dcn_dccg_destroy(&pool->base.dccg);
 
-	if (pool->base.oem_device != NULL)
-		link_destroy_ddc_service(&pool->base.oem_device);
+	if (pool->base.oem_device != NULL) {
+		struct dc *dc = pool->base.oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
+	}
 }
 
 static struct hubp *dcn30_hubp_create(
@@ -2015,6 +2016,8 @@ bool dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc,
 	if (context->streams[0]->vrr_active_variable)
 		return false;
 
+	context->streams[0]->fpo_in_use = true;
+
 	return true;
 }
 
@@ -2592,7 +2595,7 @@ static bool dcn30_resource_construct(
 		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
 		ddc_init_data.id.enum_id = 0;
 		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->base.oem_device = link_create_ddc_service(&ddc_init_data);
+		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
 	} else {
 		pool->base.oem_device = NULL;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
index ee62ae3eb98f..5ac2a272c380 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
@@ -651,8 +651,6 @@ static struct resource_caps res_cap_dcn301 = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
@@ -689,7 +687,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.disable_clock_gate = true,
 	.disable_pplib_clock_request = true,
 	.disable_pplib_wm_range = true,
-	.pipe_split_policy = MPC_SPLIT_AVOID,
+	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
 	.force_single_disp_pipe_split = false,
 	.disable_dcc = DCC_ENABLE,
 	.vsr_support = true,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
index 03ddf4f5f065..9f93c43115ba 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
@@ -147,8 +147,6 @@ static const struct resource_caps res_cap_dcn302 = {
 
 static const struct dc_plane_cap plane_cap = {
 		.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-		.blends_with_above = true,
-		.blends_with_below = true,
 		.per_pixel_alpha = true,
 		.pixel_format_support = {
 				.argb8888 = true,
@@ -1127,8 +1125,11 @@ static void dcn302_resource_destruct(struct resource_pool *pool)
 	if (pool->dccg != NULL)
 		dcn_dccg_destroy(&pool->dccg);
 
-	if (pool->oem_device != NULL)
-		link_destroy_ddc_service(&pool->oem_device);
+	if (pool->oem_device != NULL) {
+		struct dc *dc = pool->oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->oem_device);
+	}
 }
 
 static void dcn302_destroy_resource_pool(struct resource_pool **pool)
@@ -1508,7 +1509,7 @@ static bool dcn302_resource_construct(
 		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
 		ddc_init_data.id.enum_id = 0;
 		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->oem_device = link_create_ddc_service(&ddc_init_data);
+		pool->oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
 	} else {
 		pool->oem_device = NULL;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
index 31e212064168..7f72ef882ca4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
@@ -126,8 +126,6 @@ static const struct resource_caps res_cap_dcn303 = {
 
 static const struct dc_plane_cap plane_cap = {
 		.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-		.blends_with_above = true,
-		.blends_with_below = true,
 		.per_pixel_alpha = true,
 		.pixel_format_support = {
 				.argb8888 = true,
@@ -1053,8 +1051,11 @@ static void dcn303_resource_destruct(struct resource_pool *pool)
 	if (pool->dccg != NULL)
 		dcn_dccg_destroy(&pool->dccg);
 
-	if (pool->oem_device != NULL)
-		link_destroy_ddc_service(&pool->oem_device);
+	if (pool->oem_device != NULL) {
+		struct dc *dc = pool->oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->oem_device);
+	}
 }
 
 static void dcn303_destroy_resource_pool(struct resource_pool **pool)
@@ -1163,7 +1164,6 @@ static bool dcn303_resource_construct(
 	dc->caps.max_cursor_size = 256;
 	dc->caps.min_horizontal_blanking_period = 80;
 	dc->caps.dmdata_alloc_size = 2048;
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	dc->caps.mall_size_per_mem_channel = 4;
 	/* total size = mall per channel * num channels * 1024 * 1024 */
 	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel *
@@ -1171,7 +1171,6 @@ static bool dcn303_resource_construct(
 				   1024 * 1024;
 	dc->caps.cursor_cache_size =
 		dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
-#endif
 	dc->caps.max_slave_planes = 1;
 	dc->caps.post_blend_color_processing = true;
 	dc->caps.force_dp_tps4_for_cp2520 = true;
@@ -1421,7 +1420,7 @@ static bool dcn303_resource_construct(
 		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
 		ddc_init_data.id.enum_id = 0;
 		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->oem_device = link_create_ddc_service(&ddc_init_data);
+		pool->oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
 	} else {
 		pool->oem_device = NULL;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c
index 24e9ff65434d..05aac3e444b4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c
@@ -72,40 +72,6 @@ static void apg31_disable(
 	REG_UPDATE(APG_CONTROL2, APG_ENABLE, 0);
 }
 
-static union audio_cea_channels speakers_to_channels(
-	struct audio_speaker_flags speaker_flags)
-{
-	union audio_cea_channels cea_channels = {0};
-
-	/* these are one to one */
-	cea_channels.channels.FL = speaker_flags.FL_FR;
-	cea_channels.channels.FR = speaker_flags.FL_FR;
-	cea_channels.channels.LFE = speaker_flags.LFE;
-	cea_channels.channels.FC = speaker_flags.FC;
-
-	/* if Rear Left and Right exist move RC speaker to channel 7
-	 * otherwise to channel 5
-	 */
-	if (speaker_flags.RL_RR) {
-		cea_channels.channels.RL_RC = speaker_flags.RL_RR;
-		cea_channels.channels.RR = speaker_flags.RL_RR;
-		cea_channels.channels.RC_RLC_FLC = speaker_flags.RC;
-	} else {
-		cea_channels.channels.RL_RC = speaker_flags.RC;
-	}
-
-	/* FRONT Left Right Center and REAR Left Right Center are exclusive */
-	if (speaker_flags.FLC_FRC) {
-		cea_channels.channels.RC_RLC_FLC = speaker_flags.FLC_FRC;
-		cea_channels.channels.RRC_FRC = speaker_flags.FLC_FRC;
-	} else {
-		cea_channels.channels.RC_RLC_FLC = speaker_flags.RLC_RRC;
-		cea_channels.channels.RRC_FRC = speaker_flags.RLC_RRC;
-	}
-
-	return cea_channels;
-}
-
 static void apg31_se_audio_setup(
 	struct apg *apg,
 	unsigned int az_inst,
@@ -113,24 +79,17 @@ static void apg31_se_audio_setup(
 {
 	struct dcn31_apg *apg31 = DCN31_APG_FROM_APG(apg);
 
-	uint32_t speakers = 0;
-	uint32_t channels = 0;
-
 	ASSERT(audio_info);
 	/* This should not happen.it does so we don't get BSOD*/
 	if (audio_info == NULL)
 		return;
 
-	speakers = audio_info->flags.info.ALLSPEAKERS;
-	channels = speakers_to_channels(audio_info->flags.speaker_flags).all;
-
 	/* DisplayPort only allows for one audio stream with stream ID 0 */
 	REG_UPDATE(APG_CONTROL2, APG_DP_AUDIO_STREAM_ID, 0);
 
 	/* When running in "pair mode", pairs of audio channels have their own enable
 	 * this is for really old audio drivers */
 	REG_UPDATE(APG_DBG_GEN_CONTROL, APG_DBG_AUDIO_CHANNEL_ENABLE, 0xFF);
-	// REG_UPDATE(APG_DBG_GEN_CONTROL, APG_DBG_AUDIO_CHANNEL_ENABLE, channels);
 
 	/* Disable forced mem power off */
 	REG_UPDATE(APG_MEM_PWR, APG_MEM_PWR_FORCE, 0);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dccg.c
index 7f34418e6308..4c2fdfea162f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dccg.c
@@ -66,17 +66,8 @@ void dccg31_update_dpp_dto(struct dccg *dccg, int dpp_inst, int req_dppclk)
 		REG_UPDATE(DPPCLK_DTO_CTRL,
 				DPPCLK_DTO_ENABLE[dpp_inst], 1);
 	} else {
-		//DTO must be enabled to generate a 0Hz clock output
-		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpp) {
-			REG_UPDATE(DPPCLK_DTO_CTRL,
-					DPPCLK_DTO_ENABLE[dpp_inst], 1);
-			REG_SET_2(DPPCLK_DTO_PARAM[dpp_inst], 0,
-					DPPCLK0_DTO_PHASE, 0,
-					DPPCLK0_DTO_MODULO, 1);
-		} else {
-			REG_UPDATE(DPPCLK_DTO_CTRL,
-					DPPCLK_DTO_ENABLE[dpp_inst], 0);
-		}
+		REG_UPDATE(DPPCLK_DTO_CTRL,
+				DPPCLK_DTO_ENABLE[dpp_inst], 0);
 	}
 	dccg->pipe_dppclk_khz[dpp_inst] = req_dppclk;
 }
@@ -369,6 +360,15 @@ void dccg31_disable_dscclk(struct dccg *dccg, int inst)
 				DSCCLK2_DTO_PHASE, 0,
 				DSCCLK2_DTO_MODULO, 1);
 		break;
+	case 3:
+		if (REG(DSCCLK3_DTO_PARAM)) {
+			REG_UPDATE(DSCCLK_DTO_CTRL,
+					DSCCLK3_DTO_ENABLE, 1);
+			REG_UPDATE_2(DSCCLK3_DTO_PARAM,
+					DSCCLK3_DTO_PHASE, 0,
+					DSCCLK3_DTO_MODULO, 1);
+		}
+		break;
 	default:
 		BREAK_TO_DEBUGGER();
 		return;
@@ -404,6 +404,15 @@ void dccg31_enable_dscclk(struct dccg *dccg, int inst)
 		REG_UPDATE(DSCCLK_DTO_CTRL,
 				DSCCLK2_DTO_ENABLE, 0);
 		break;
+	case 3:
+		if (REG(DSCCLK3_DTO_PARAM)) {
+			REG_UPDATE(DSCCLK_DTO_CTRL,
+					DSCCLK3_DTO_ENABLE, 0);
+			REG_UPDATE_2(DSCCLK3_DTO_PARAM,
+					DSCCLK3_DTO_PHASE, 0,
+					DSCCLK3_DTO_MODULO, 0);
+		}
+		break;
 	default:
 		BREAK_TO_DEBUGGER();
 		return;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.c
index 275e78c06dee..745a5d187a98 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.c
@@ -37,6 +37,7 @@
 #include "link_enc_cfg.h"
 #include "dc_dmub_srv.h"
 #include "dal_asic_id.h"
+#include "link.h"
 
 #define CTX \
 	enc10->base.ctx
@@ -485,7 +486,7 @@ void dcn31_link_encoder_enable_dp_output(
 
 		if (link) {
 			dpia_control.dpia_id = link->ddc_hw_inst;
-			dpia_control.fec_rdy = dc_link_should_enable_fec(link);
+			dpia_control.fec_rdy = link->dc->link_srv->dp_should_enable_fec(link);
 		} else {
 			DC_LOG_ERROR("%s: Failed to execute DPIA enable DMUB command.\n", __func__);
 			BREAK_TO_DEBUGGER();
@@ -532,7 +533,7 @@ void dcn31_link_encoder_enable_dp_mst_output(
 
 		if (link) {
 			dpia_control.dpia_id = link->ddc_hw_inst;
-			dpia_control.fec_rdy = dc_link_should_enable_fec(link);
+			dpia_control.fec_rdy = link->dc->link_srv->dp_should_enable_fec(link);
 		} else {
 			DC_LOG_ERROR("%s: Failed to execute DPIA enable DMUB command.\n", __func__);
 			BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_link_encoder.c
index 0b317ed31f91..5b7ad38f85e0 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_link_encoder.c
@@ -26,7 +26,6 @@
 #include "dc_bios_types.h"
 #include "dcn31_hpo_dp_link_encoder.h"
 #include "reg_helper.h"
-#include "dc_link.h"
 #include "stream_encoder.h"
 
 #define DC_LOGGER \
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c
index d76f55a12eb4..0278bae50a9d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c
@@ -26,7 +26,7 @@
 #include "dc_bios_types.h"
 #include "dcn31_hpo_dp_stream_encoder.h"
 #include "reg_helper.h"
-#include "dc_link.h"
+#include "dc.h"
 
 #define DC_LOGGER \
 		enc3->base.ctx->logger
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c
index d13e46eeee3c..7ac6e69cff37 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c
@@ -97,7 +97,7 @@ static void enable_memory_low_power(struct dc *dc)
 		// Power down VPGs
 		for (i = 0; i < dc->res_pool->stream_enc_count; i++)
 			dc->res_pool->stream_enc[i]->vpg->funcs->vpg_powerdown(dc->res_pool->stream_enc[i]->vpg);
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 		for (i = 0; i < dc->res_pool->hpo_dp_stream_enc_count; i++)
 			dc->res_pool->hpo_dp_stream_enc[i]->vpg->funcs->vpg_powerdown(dc->res_pool->hpo_dp_stream_enc[i]->vpg);
 #endif
@@ -202,7 +202,7 @@ void dcn31_init_hw(struct dc *dc)
 		dmub_enable_outbox_notification(dc->ctx->dmub_srv);
 
 	/* we want to turn off all dp displays before doing detection */
-	link_blank_all_dp_displays(dc);
+	dc->link_srv->blank_all_dp_displays(dc);
 
 	if (hws->funcs.enable_power_gating_plane)
 		hws->funcs.enable_power_gating_plane(dc->hwseq, true);
@@ -230,7 +230,7 @@ void dcn31_init_hw(struct dc *dc)
 				}
 
 				if (num_opps > 1) {
-					link_blank_all_edp_displays(dc);
+					dc->link_srv->blank_all_edp_displays(dc);
 					break;
 				}
 			}
@@ -285,13 +285,13 @@ void dcn31_init_hw(struct dc *dc)
 	if (dc->clk_mgr->funcs->notify_wm_ranges)
 		dc->clk_mgr->funcs->notify_wm_ranges(dc->clk_mgr);
 
-	if (dc->clk_mgr->funcs->set_hard_max_memclk)
+	if (dc->clk_mgr->funcs->set_hard_max_memclk && !dc->clk_mgr->dc_mode_softmax_enabled)
 		dc->clk_mgr->funcs->set_hard_max_memclk(dc->clk_mgr);
 
 	if (dc->res_pool->hubbub->funcs->force_pstate_change_control)
 		dc->res_pool->hubbub->funcs->force_pstate_change_control(
 				dc->res_pool->hubbub, false, false);
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 	if (dc->res_pool->hubbub->funcs->init_crb)
 		dc->res_pool->hubbub->funcs->init_crb(dc->res_pool->hubbub);
 #endif
@@ -414,7 +414,7 @@ void dcn31_update_info_frame(struct pipe_ctx *pipe_ctx)
 		pipe_ctx->stream_res.stream_enc->funcs->update_hdmi_info_packets(
 			pipe_ctx->stream_res.stream_enc,
 			&pipe_ctx->stream_res.encoder_info_frame);
-	else if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	else if (pipe_ctx->stream->ctx->dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->update_dp_info_packets(
 				pipe_ctx->stream_res.hpo_dp_stream_enc,
 				&pipe_ctx->stream_res.encoder_info_frame);
@@ -565,7 +565,7 @@ static void dcn31_reset_back_end_for_pipe(
 		 * VBIOS lit up eDP, so check link status too.
 		 */
 		if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
-			link_set_dpms_off(pipe_ctx);
+			dc->link_srv->set_dpms_off(pipe_ctx);
 		else if (pipe_ctx->stream_res.audio)
 			dc->hwss.disable_audio_stream(pipe_ctx);
 
@@ -584,7 +584,7 @@ static void dcn31_reset_back_end_for_pipe(
 			}
 		}
 	} else if (pipe_ctx->stream_res.dsc) {
-			link_set_dsc_enable(pipe_ctx, false);
+		dc->link_srv->set_dsc_enable(pipe_ctx, false);
 	}
 
 	pipe_ctx->stream = NULL;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
index d3918a10773a..ff8cd5076434 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
@@ -827,8 +827,6 @@ static const struct resource_caps res_cap_dcn31 = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
@@ -1967,6 +1965,8 @@ static bool dcn31_resource_construct(
 	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
 	dc->caps.color.mpc.ocsc = 1;
 
+	dc->config.use_old_fixed_vs_sequence = true;
+
 	/* Use pipe context based otg sync logic */
 	dc->config.use_pipe_ctx_sync_logic = true;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c
index 0b769ee71405..6f879265ad9c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c
@@ -274,6 +274,32 @@ static void dccg314_set_dpstreamclk(
 	}
 }
 
+void dccg314_init(struct dccg *dccg)
+{
+	int otg_inst;
+
+	/* Set HPO stream encoder to use refclk to avoid case where PHY is
+	 * disabled and SYMCLK32 for HPO SE is sourced from PHYD32CLK which
+	 * will cause DCN to hang.
+	 */
+	for (otg_inst = 0; otg_inst < 4; otg_inst++)
+		dccg31_disable_symclk32_se(dccg, otg_inst);
+
+	if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_le)
+		for (otg_inst = 0; otg_inst < 2; otg_inst++)
+			dccg31_disable_symclk32_le(dccg, otg_inst);
+
+	if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream)
+		for (otg_inst = 0; otg_inst < 4; otg_inst++)
+			dccg314_set_dpstreamclk(dccg, REFCLK, otg_inst,
+						otg_inst);
+
+	if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+		for (otg_inst = 0; otg_inst < 5; otg_inst++)
+			dccg31_set_physymclk(dccg, otg_inst,
+					     PHYSYMCLK_FORCE_SRC_SYMCLK, false);
+}
+
 static void dccg314_set_valid_pixel_rate(
 		struct dccg *dccg,
 		int ref_dtbclk_khz,
@@ -289,10 +315,33 @@ static void dccg314_set_valid_pixel_rate(
 	dccg314_set_dtbclk_dto(dccg, &dto_params);
 }
 
+static void dccg314_dpp_root_clock_control(
+		struct dccg *dccg,
+		unsigned int dpp_inst,
+		bool clock_on)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	if (clock_on) {
+		/* turn off the DTO and leave phase/modulo at max */
+		REG_UPDATE(DPPCLK_DTO_CTRL, DPPCLK_DTO_ENABLE[dpp_inst], 0);
+		REG_SET_2(DPPCLK_DTO_PARAM[dpp_inst], 0,
+			  DPPCLK0_DTO_PHASE, 0xFF,
+			  DPPCLK0_DTO_MODULO, 0xFF);
+	} else {
+		/* turn on the DTO to generate a 0hz clock */
+		REG_UPDATE(DPPCLK_DTO_CTRL, DPPCLK_DTO_ENABLE[dpp_inst], 1);
+		REG_SET_2(DPPCLK_DTO_PARAM[dpp_inst], 0,
+			  DPPCLK0_DTO_PHASE, 0,
+			  DPPCLK0_DTO_MODULO, 1);
+	}
+}
+
 static const struct dccg_funcs dccg314_funcs = {
 	.update_dpp_dto = dccg31_update_dpp_dto,
+	.dpp_root_clock_control = dccg314_dpp_root_clock_control,
 	.get_dccg_ref_freq = dccg31_get_dccg_ref_freq,
-	.dccg_init = dccg31_init,
+	.dccg_init = dccg314_init,
 	.set_dpstreamclk = dccg314_set_dpstreamclk,
 	.enable_symclk32_se = dccg31_enable_symclk32_se,
 	.disable_symclk32_se = dccg31_disable_symclk32_se,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h
index 6a35986307af..90687a9e8fdd 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h
@@ -68,6 +68,7 @@
 	SR(DSCCLK0_DTO_PARAM),\
 	SR(DSCCLK1_DTO_PARAM),\
 	SR(DSCCLK2_DTO_PARAM),\
+	SR(DSCCLK3_DTO_PARAM),\
 	SR(DSCCLK_DTO_CTRL),\
 	SR(DCCG_GATE_DISABLE_CNTL2),\
 	SR(DCCG_GATE_DISABLE_CNTL3),\
@@ -149,12 +150,20 @@
 	DCCG_SF(DSCCLK1_DTO_PARAM, DSCCLK1_DTO_MODULO, mask_sh),\
 	DCCG_SF(DSCCLK2_DTO_PARAM, DSCCLK2_DTO_PHASE, mask_sh),\
 	DCCG_SF(DSCCLK2_DTO_PARAM, DSCCLK2_DTO_MODULO, mask_sh),\
+	DCCG_SF(DSCCLK3_DTO_PARAM, DSCCLK3_DTO_PHASE, mask_sh),\
+	DCCG_SF(DSCCLK3_DTO_PARAM, DSCCLK3_DTO_MODULO, mask_sh),\
 	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE0_GATE_DISABLE, mask_sh),\
 	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE1_GATE_DISABLE, mask_sh),\
 	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE2_GATE_DISABLE, mask_sh),\
 	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE3_GATE_DISABLE, mask_sh),\
 	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE0_GATE_DISABLE, mask_sh),\
 	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE2_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE3_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE1_GATE_DISABLE, mask_sh),\
 	DCCG_SF(HDMISTREAMCLK0_DTO_PARAM, HDMISTREAMCLK0_DTO_PHASE, mask_sh),\
 	DCCG_SF(HDMISTREAMCLK0_DTO_PARAM, HDMISTREAMCLK0_DTO_MODULO, mask_sh)
 
@@ -178,6 +187,7 @@
 	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK0_DTO_ENABLE, mask_sh),\
 	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK1_DTO_ENABLE, mask_sh),\
 	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK2_DTO_ENABLE, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK3_DTO_ENABLE, mask_sh),\
 	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_GATE_DISABLE, mask_sh),\
 	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_GATE_DISABLE, mask_sh),\
 	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_GATE_DISABLE, mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c
index 962a2c02b422..467509a65fa7 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c
@@ -296,12 +296,14 @@ static void enc314_stream_encoder_dp_unblank(
 		uint32_t n_vid = 0x8000;
 		uint32_t m_vid;
 		uint32_t n_multiply = 0;
+		uint32_t pix_per_cycle = 0;
 		uint64_t m_vid_l = n_vid;
 
 		/* YCbCr 4:2:0 : Computed VID_M will be 2X the input rate */
 		if (is_two_pixels_per_containter(&param->timing) || param->opp_cnt > 1) {
 			/*this logic should be the same in get_pixel_clock_parameters() */
 			n_multiply = 1;
+			pix_per_cycle = 1;
 		}
 		/* M / N = Fstream / Flink
 		 * m_vid / n_vid = pixel rate / link rate
@@ -329,6 +331,10 @@ static void enc314_stream_encoder_dp_unblank(
 		REG_UPDATE_2(DP_VID_TIMING,
 				DP_VID_M_N_GEN_EN, 1,
 				DP_VID_N_MUL, n_multiply);
+
+		REG_UPDATE(DP_PIXEL_FORMAT,
+				DP_PIXEL_PER_CYCLE_PROCESSING_MODE,
+				pix_per_cycle);
 	}
 
 	/* make sure stream is disabled before resetting steer fifo */
@@ -366,7 +372,7 @@ static void enc314_stream_encoder_dp_unblank(
 	 */
 	enc314_enable_fifo(enc);
 
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_DP_VID_STREAM);
+	link->dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_DP_VID_STREAM);
 }
 
 /* Set DSC-related configuration.
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c
index 575d3501c848..40c488b26901 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c
@@ -346,7 +346,7 @@ unsigned int dcn314_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsig
 	two_pix_per_container = optc2_is_two_pixels_per_containter(&stream->timing);
 	odm_combine_factor = get_odm_config(pipe_ctx, NULL);
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (stream->ctx->dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		*k1_div = PIXEL_RATE_DIV_BY_1;
 		*k2_div = PIXEL_RATE_DIV_BY_1;
 	} else if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal) || dc_is_dvi_signal(pipe_ctx->stream->signal)) {
@@ -390,6 +390,16 @@ void dcn314_set_pixels_per_cycle(struct pipe_ctx *pipe_ctx)
 				pix_per_cycle);
 }
 
+void dcn314_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool clock_on)
+{
+	if (!hws->ctx->dc->debug.root_clock_optimization.bits.dpp)
+		return;
+
+	if (hws->ctx->dc->res_pool->dccg->funcs->dpp_root_clock_control)
+		hws->ctx->dc->res_pool->dccg->funcs->dpp_root_clock_control(
+			hws->ctx->dc->res_pool->dccg, dpp_inst, clock_on);
+}
+
 void dcn314_hubp_pg_control(struct dce_hwseq *hws, unsigned int hubp_inst, bool power_on)
 {
 	struct dc_context *ctx = hws->ctx;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.h b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.h
index c419d3dbdfee..c786d5e6a428 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.h
@@ -43,4 +43,6 @@ void dcn314_set_pixels_per_cycle(struct pipe_ctx *pipe_ctx);
 
 void dcn314_hubp_pg_control(struct dce_hwseq *hws, unsigned int hubp_inst, bool power_on);
 
+void dcn314_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool clock_on);
+
 #endif /* __DC_HWSS_DCN314_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c
index 343f4d9dd5e3..5267e901a35c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c
@@ -137,6 +137,7 @@ static const struct hwseq_private_funcs dcn314_private_funcs = {
 	.plane_atomic_disable = dcn20_plane_atomic_disable,
 	.plane_atomic_power_down = dcn10_plane_atomic_power_down,
 	.enable_power_gating_plane = dcn314_enable_power_gating_plane,
+	.dpp_root_clock_control = dcn314_dpp_root_clock_control,
 	.hubp_pg_control = dcn314_hubp_pg_control,
 	.program_all_writeback_pipes_in_tree = dcn30_program_all_writeback_pipes_in_tree,
 	.update_odm = dcn314_update_odm,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c
index 54ed3de869d3..24806acc8438 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c
@@ -855,8 +855,6 @@ static const struct resource_caps res_cap_dcn314 = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
@@ -887,6 +885,7 @@ static const struct dc_plane_cap plane_cap = {
 static const struct dc_debug_options debug_defaults_drv = {
 	.disable_z10 = false,
 	.enable_z9_disable_interface = true,
+	.minimum_z8_residency_time = 3080,
 	.psr_skip_crtc_disable = true,
 	.disable_dmcu = true,
 	.force_abm_enable = false,
@@ -1697,6 +1696,23 @@ static void dcn314_get_panel_config_defaults(struct dc_panel_config *panel_confi
 	*panel_config = panel_config_defaults;
 }
 
+static bool filter_modes_for_single_channel_workaround(struct dc *dc,
+		struct dc_state *context)
+{
+	// Filter 2K@240Hz+8K@24fps above combination timing if memory only has single dimm LPDDR
+	if (dc->clk_mgr->bw_params->vram_type == 34 && dc->clk_mgr->bw_params->num_channels < 2) {
+		int total_phy_pix_clk = 0;
+
+		for (int i = 0; i < context->stream_count; i++)
+			if (context->res_ctx.pipe_ctx[i].stream)
+				total_phy_pix_clk += context->res_ctx.pipe_ctx[i].stream->phy_pix_clk;
+
+		if (total_phy_pix_clk >= (1148928+826260)) //2K@240Hz+8K@24fps
+			return true;
+	}
+	return false;
+}
+
 bool dcn314_validate_bandwidth(struct dc *dc,
 		struct dc_state *context,
 		bool fast_validate)
@@ -1712,6 +1728,9 @@ bool dcn314_validate_bandwidth(struct dc *dc,
 
 	BW_VAL_TRACE_COUNT();
 
+	if (filter_modes_for_single_channel_workaround(dc, context))
+		goto validate_fail;
+
 	DC_FP_START();
 	// do not support self refresh only
 	out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate, false);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c
index 7887078c5f64..41c972c8eb19 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c
@@ -824,8 +824,6 @@ static const struct resource_caps res_cap_dcn31 = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c
index dc0b49506275..9ead347a33e9 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c
@@ -824,8 +824,6 @@ static const struct resource_caps res_cap_dcn31 = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c
index 3fb4bcc34353..ffbb739d85b6 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c
@@ -42,6 +42,20 @@
 #define DC_LOGGER \
 	dccg->ctx->logger
 
+/* This function is a workaround for writing to OTG_PIXEL_RATE_DIV
+ * without the probability of causing a DIG FIFO error.
+ */
+static void dccg32_wait_for_dentist_change_done(
+	struct dccg *dccg)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	uint32_t dentist_dispclk_value = REG_READ(DENTIST_DISPCLK_CNTL);
+
+	REG_WRITE(DENTIST_DISPCLK_CNTL, dentist_dispclk_value);
+	REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 2000);
+}
+
 static void dccg32_get_pixel_rate_div(
 		struct dccg *dccg,
 		uint32_t otg_inst,
@@ -110,21 +124,29 @@ static void dccg32_set_pixel_rate_div(
 		REG_UPDATE_2(OTG_PIXEL_RATE_DIV,
 				OTG0_PIXEL_RATE_DIVK1, k1,
 				OTG0_PIXEL_RATE_DIVK2, k2);
+
+		dccg32_wait_for_dentist_change_done(dccg);
 		break;
 	case 1:
 		REG_UPDATE_2(OTG_PIXEL_RATE_DIV,
 				OTG1_PIXEL_RATE_DIVK1, k1,
 				OTG1_PIXEL_RATE_DIVK2, k2);
+
+		dccg32_wait_for_dentist_change_done(dccg);
 		break;
 	case 2:
 		REG_UPDATE_2(OTG_PIXEL_RATE_DIV,
 				OTG2_PIXEL_RATE_DIVK1, k1,
 				OTG2_PIXEL_RATE_DIVK2, k2);
+
+		dccg32_wait_for_dentist_change_done(dccg);
 		break;
 	case 3:
 		REG_UPDATE_2(OTG_PIXEL_RATE_DIV,
 				OTG3_PIXEL_RATE_DIVK1, k1,
 				OTG3_PIXEL_RATE_DIVK2, k2);
+
+		dccg32_wait_for_dentist_change_done(dccg);
 		break;
 	default:
 		BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.h
index 1c46fad0977b..8071ab98d708 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.h
@@ -31,42 +31,6 @@
 #define DCCG_SFII(block, reg_name, field_prefix, field_name, inst, post_fix)\
 	.field_prefix ## _ ## field_name[inst] = block ## inst ## _ ## reg_name ## __ ## field_prefix ## inst ## _ ## field_name ## post_fix
 
-
-#define DCCG_REG_LIST_DCN32() \
-	SR(DPPCLK_DTO_CTRL),\
-	DCCG_SRII(DTO_PARAM, DPPCLK, 0),\
-	DCCG_SRII(DTO_PARAM, DPPCLK, 1),\
-	DCCG_SRII(DTO_PARAM, DPPCLK, 2),\
-	DCCG_SRII(DTO_PARAM, DPPCLK, 3),\
-	DCCG_SRII(CLOCK_CNTL, HDMICHARCLK, 0),\
-	SR(PHYASYMCLK_CLOCK_CNTL),\
-	SR(PHYBSYMCLK_CLOCK_CNTL),\
-	SR(PHYCSYMCLK_CLOCK_CNTL),\
-	SR(PHYDSYMCLK_CLOCK_CNTL),\
-	SR(PHYESYMCLK_CLOCK_CNTL),\
-	SR(DPSTREAMCLK_CNTL),\
-	SR(HDMISTREAMCLK_CNTL),\
-	SR(SYMCLK32_SE_CNTL),\
-	SR(SYMCLK32_LE_CNTL),\
-	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 0),\
-	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 1),\
-	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 2),\
-	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 3),\
-	DCCG_SRII(MODULO, DTBCLK_DTO, 0),\
-	DCCG_SRII(MODULO, DTBCLK_DTO, 1),\
-	DCCG_SRII(MODULO, DTBCLK_DTO, 2),\
-	DCCG_SRII(MODULO, DTBCLK_DTO, 3),\
-	DCCG_SRII(PHASE, DTBCLK_DTO, 0),\
-	DCCG_SRII(PHASE, DTBCLK_DTO, 1),\
-	DCCG_SRII(PHASE, DTBCLK_DTO, 2),\
-	DCCG_SRII(PHASE, DTBCLK_DTO, 3),\
-	SR(DCCG_AUDIO_DTBCLK_DTO_MODULO),\
-	SR(DCCG_AUDIO_DTBCLK_DTO_PHASE),\
-	SR(OTG_PIXEL_RATE_DIV),\
-	SR(DTBCLK_P_CNTL),\
-	SR(DCCG_AUDIO_DTO_SOURCE)
-
-
 #define DCCG_MASK_SH_LIST_DCN32(mask_sh) \
 	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 0, mask_sh),\
 	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 0, mask_sh),\
@@ -147,7 +111,8 @@
 	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P3_SRC_SEL, mask_sh),\
 	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P3_EN, mask_sh),\
 	DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO_SEL, mask_sh),\
-	DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_SOURCE_SEL, mask_sh)
+	DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_SOURCE_SEL, mask_sh),\
+	DCCG_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, mask_sh)
 
 
 struct dccg *dccg32_create(
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c
index 36e6f5657942..2fef1419ae91 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c
@@ -211,10 +211,8 @@ static void enc32_stream_encoder_hdmi_set_stream_attribute(
 		HDMI_GC_SEND, 1,
 		HDMI_NULL_SEND, 1);
 
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 	/* Disable Audio Content Protection packet transmission */
 	REG_UPDATE(HDMI_VBI_PACKET_CONTROL, HDMI_ACP_SEND, 0);
-#endif
 
 	/* following belongs to audio */
 	/* Enable Audio InfoFrame packet transmission. */
@@ -276,10 +274,10 @@ static bool is_dp_dig_pixel_rate_div_policy(struct dc *dc, const struct dc_crtc_
 		dc->debug.enable_dp_dig_pixel_rate_div_policy;
 }
 
-static void enc32_stream_encoder_dp_unblank(
-        struct dc_link *link,
-		struct stream_encoder *enc,
-		const struct encoder_unblank_param *param)
+void enc32_stream_encoder_dp_unblank(
+	struct dc_link *link,
+	struct stream_encoder *enc,
+	const struct encoder_unblank_param *param)
 {
 	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
 	struct dc *dc = enc->ctx->dc;
@@ -288,6 +286,7 @@ static void enc32_stream_encoder_dp_unblank(
 		uint32_t n_vid = 0x8000;
 		uint32_t m_vid;
 		uint32_t n_multiply = 0;
+		uint32_t pix_per_cycle = 0;
 		uint64_t m_vid_l = n_vid;
 
 		/* YCbCr 4:2:0 : Computed VID_M will be 2X the input rate */
@@ -295,6 +294,7 @@ static void enc32_stream_encoder_dp_unblank(
 			|| is_dp_dig_pixel_rate_div_policy(dc, &param->timing)) {
 			/*this logic should be the same in get_pixel_clock_parameters() */
 			n_multiply = 1;
+			pix_per_cycle = 1;
 		}
 		/* M / N = Fstream / Flink
 		 * m_vid / n_vid = pixel rate / link rate
@@ -322,6 +322,10 @@ static void enc32_stream_encoder_dp_unblank(
 		REG_UPDATE_2(DP_VID_TIMING,
 				DP_VID_M_N_GEN_EN, 1,
 				DP_VID_N_MUL, n_multiply);
+
+		REG_UPDATE(DP_PIXEL_FORMAT,
+				DP_PIXEL_PER_CYCLE_PROCESSING_MODE,
+				pix_per_cycle);
 	}
 
 	/* make sure stream is disabled before resetting steer fifo */
@@ -373,7 +377,7 @@ static void enc32_stream_encoder_dp_unblank(
 
 	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, true);
 
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_DP_VID_STREAM);
+	link->dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_DP_VID_STREAM);
 }
 
 /* Set DSC-related configuration.
@@ -436,7 +440,7 @@ static void enc32_reset_fifo(struct stream_encoder *enc, bool reset)
 		udelay(10);
 }
 
-static void enc32_enable_fifo(struct stream_encoder *enc)
+void enc32_enable_fifo(struct stream_encoder *enc)
 {
 	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h
index ecd041a446d2..1be5410cce97 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h
@@ -31,70 +31,6 @@
 #include "stream_encoder.h"
 #include "dcn20/dcn20_stream_encoder.h"
 
-#define SE_DCN32_REG_LIST(id)\
-	SRI(AFMT_CNTL, DIG, id), \
-	SRI(DIG_FE_CNTL, DIG, id), \
-	SRI(HDMI_CONTROL, DIG, id), \
-	SRI(HDMI_DB_CONTROL, DIG, id), \
-	SRI(HDMI_GC, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL0, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL1, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL2, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL3, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL4, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL5, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL6, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL7, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL8, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL9, DIG, id), \
-	SRI(HDMI_GENERIC_PACKET_CONTROL10, DIG, id), \
-	SRI(HDMI_INFOFRAME_CONTROL0, DIG, id), \
-	SRI(HDMI_INFOFRAME_CONTROL1, DIG, id), \
-	SRI(HDMI_VBI_PACKET_CONTROL, DIG, id), \
-	SRI(HDMI_AUDIO_PACKET_CONTROL, DIG, id),\
-	SRI(HDMI_ACR_PACKET_CONTROL, DIG, id),\
-	SRI(HDMI_ACR_32_0, DIG, id),\
-	SRI(HDMI_ACR_32_1, DIG, id),\
-	SRI(HDMI_ACR_44_0, DIG, id),\
-	SRI(HDMI_ACR_44_1, DIG, id),\
-	SRI(HDMI_ACR_48_0, DIG, id),\
-	SRI(HDMI_ACR_48_1, DIG, id),\
-	SRI(DP_DB_CNTL, DP, id), \
-	SRI(DP_MSA_MISC, DP, id), \
-	SRI(DP_MSA_VBID_MISC, DP, id), \
-	SRI(DP_MSA_COLORIMETRY, DP, id), \
-	SRI(DP_MSA_TIMING_PARAM1, DP, id), \
-	SRI(DP_MSA_TIMING_PARAM2, DP, id), \
-	SRI(DP_MSA_TIMING_PARAM3, DP, id), \
-	SRI(DP_MSA_TIMING_PARAM4, DP, id), \
-	SRI(DP_MSE_RATE_CNTL, DP, id), \
-	SRI(DP_MSE_RATE_UPDATE, DP, id), \
-	SRI(DP_PIXEL_FORMAT, DP, id), \
-	SRI(DP_SEC_CNTL, DP, id), \
-	SRI(DP_SEC_CNTL1, DP, id), \
-	SRI(DP_SEC_CNTL2, DP, id), \
-	SRI(DP_SEC_CNTL5, DP, id), \
-	SRI(DP_SEC_CNTL6, DP, id), \
-	SRI(DP_STEER_FIFO, DP, id), \
-	SRI(DP_VID_M, DP, id), \
-	SRI(DP_VID_N, DP, id), \
-	SRI(DP_VID_STREAM_CNTL, DP, id), \
-	SRI(DP_VID_TIMING, DP, id), \
-	SRI(DP_SEC_AUD_N, DP, id), \
-	SRI(DP_SEC_TIMESTAMP, DP, id), \
-	SRI(DP_DSC_CNTL, DP, id), \
-	SRI(DP_SEC_METADATA_TRANSMISSION, DP, id), \
-	SRI(HDMI_METADATA_PACKET_CONTROL, DIG, id), \
-	SRI(DP_SEC_FRAMING4, DP, id), \
-	SRI(DP_GSP11_CNTL, DP, id), \
-	SRI(DME_CONTROL, DME, id),\
-	SRI(DP_SEC_METADATA_TRANSMISSION, DP, id), \
-	SRI(HDMI_METADATA_PACKET_CONTROL, DIG, id), \
-	SRI(DIG_FE_CNTL, DIG, id), \
-	SRI(DIG_CLOCK_PATTERN, DIG, id), \
-	SRI(DIG_FIFO_CTRL0, DIG, id)
-
-
 #define SE_COMMON_MASK_SH_LIST_DCN32(mask_sh)\
 	SE_SF(DP0_DP_PIXEL_FORMAT, DP_PIXEL_ENCODING, mask_sh),\
 	SE_SF(DP0_DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH, mask_sh),\
@@ -258,4 +194,12 @@ void dcn32_dio_stream_encoder_construct(
 	const struct dcn10_stream_encoder_shift *se_shift,
 	const struct dcn10_stream_encoder_mask *se_mask);
 
+
+void enc32_enable_fifo(struct stream_encoder *enc);
+
+void enc32_stream_encoder_dp_unblank(
+		struct dc_link *link,
+		struct stream_encoder *enc,
+		const struct encoder_unblank_param *param);
+
 #endif /* __DC_DIO_STREAM_ENCODER_DCN32_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hpo_dp_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hpo_dp_link_encoder.c
index 4dbad8d4b4fc..8af01f579690 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hpo_dp_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hpo_dp_link_encoder.c
@@ -26,7 +26,6 @@
 #include "dcn31/dcn31_hpo_dp_link_encoder.h"
 #include "dcn32_hpo_dp_link_encoder.h"
 #include "reg_helper.h"
-#include "dc_link.h"
 #include "stream_encoder.h"
 
 #define DC_LOGGER \
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h
index b20eb04724bb..ad33427192c6 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h
@@ -28,68 +28,6 @@
 
 #include "dcn21/dcn21_hubbub.h"
 
-#define HUBBUB_REG_LIST_DCN32(id)\
-	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A),\
-	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B),\
-	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C),\
-	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D),\
-	SR(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL),\
-	SR(DCHUBBUB_ARB_DRAM_STATE_CNTL),\
-	SR(DCHUBBUB_ARB_SAT_LEVEL),\
-	SR(DCHUBBUB_ARB_DF_REQ_OUTSTAND),\
-	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
-	SR(DCHUBBUB_SOFT_RESET),\
-	SR(DCHUBBUB_CRC_CTRL), \
-	SR(DCN_VM_FB_LOCATION_BASE),\
-	SR(DCN_VM_FB_LOCATION_TOP),\
-	SR(DCN_VM_FB_OFFSET),\
-	SR(DCN_VM_AGP_BOT),\
-	SR(DCN_VM_AGP_TOP),\
-	SR(DCN_VM_AGP_BASE),\
-	HUBBUB_SR_WATERMARK_REG_LIST(), \
-	SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_A),\
-	SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_B),\
-	SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_C),\
-	SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_D),\
-	SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A),\
-	SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B),\
-	SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C),\
-	SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D),\
-	SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A),\
-	SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B),\
-	SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C),\
-	SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D),\
-	SR(DCHUBBUB_DET0_CTRL),\
-	SR(DCHUBBUB_DET1_CTRL),\
-	SR(DCHUBBUB_DET2_CTRL),\
-	SR(DCHUBBUB_DET3_CTRL),\
-	SR(DCHUBBUB_COMPBUF_CTRL),\
-	SR(COMPBUF_RESERVED_SPACE),\
-	SR(DCHUBBUB_DEBUG_CTRL_0),\
-	SR(DCHUBBUB_ARB_USR_RETRAINING_CNTL),\
-	SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A),\
-	SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B),\
-	SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C),\
-	SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D),\
-	SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A),\
-	SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B),\
-	SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C),\
-	SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D),\
-	SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A),\
-	SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B),\
-	SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C),\
-	SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D),\
-	SR(DCN_VM_FAULT_ADDR_MSB),\
-	SR(DCN_VM_FAULT_ADDR_LSB),\
-	SR(DCN_VM_FAULT_CNTL),\
-	SR(DCN_VM_FAULT_STATUS),\
-	SR(SDPIF_REQUEST_RATE_LIMIT),\
-	SR(DCHUBBUB_CLOCK_CNTL),\
-	SR(DCHUBBUB_SDPIF_CFG0),\
-	SR(DCHUBBUB_SDPIF_CFG1),\
-	SR(DCHUBBUB_MEM_PWR_MODE_CTRL)
-
-
 #define HUBBUB_MASK_SH_LIST_DCN32(mask_sh)\
 	HUBBUB_SF(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, mask_sh), \
 	HUBBUB_SF(DCHUBBUB_SOFT_RESET, DCHUBBUB_GLOBAL_SOFT_RESET, mask_sh), \
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.c
index fe0cd177744c..2d604f7ee782 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.c
@@ -47,6 +47,15 @@ void hubp32_update_force_pstate_disallow(struct hubp *hubp, bool pstate_disallow
 			DATA_UCLK_PSTATE_FORCE_VALUE, 0);
 }
 
+void hubp32_update_force_cursor_pstate_disallow(struct hubp *hubp, bool pstate_disallow)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+	REG_UPDATE_2(UCLK_PSTATE_FORCE,
+			CURSOR_UCLK_PSTATE_FORCE_EN, pstate_disallow,
+			CURSOR_UCLK_PSTATE_FORCE_VALUE, 0);
+}
+
 void hubp32_update_mall_sel(struct hubp *hubp, uint32_t mall_sel, bool c_cursor)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
@@ -188,6 +197,7 @@ static struct hubp_funcs dcn32_hubp_funcs = {
 	.hubp_set_flip_int = hubp1_set_flip_int,
 	.hubp_in_blank = hubp1_in_blank,
 	.hubp_update_force_pstate_disallow = hubp32_update_force_pstate_disallow,
+	.hubp_update_force_cursor_pstate_disallow = hubp32_update_force_cursor_pstate_disallow,
 	.phantom_hubp_post_enable = hubp32_phantom_hubp_post_enable,
 	.hubp_update_mall_sel = hubp32_update_mall_sel,
 	.hubp_prepare_subvp_buffering = hubp32_prepare_subvp_buffering
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.h
index 4cdbf63c952b..d2acbc129609 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.h
@@ -31,12 +31,6 @@
 #include "dcn30/dcn30_hubp.h"
 #include "dcn31/dcn31_hubp.h"
 
-#define HUBP_REG_LIST_DCN32(id)\
-	HUBP_REG_LIST_DCN30(id),\
-	SRI(DCHUBP_MALL_CONFIG, HUBP, id),\
-	SRI(DCHUBP_VMPG_CONFIG, HUBP, id),\
-	SRI(UCLK_PSTATE_FORCE, HUBPREQ, id)
-
 #define HUBP_MASK_SH_LIST_DCN32(mask_sh)\
 	HUBP_MASK_SH_LIST_DCN31(mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_MALL_CONFIG, USE_MALL_SEL, mask_sh),\
@@ -52,6 +46,8 @@
 
 void hubp32_update_force_pstate_disallow(struct hubp *hubp, bool pstate_disallow);
 
+void hubp32_update_force_cursor_pstate_disallow(struct hubp *hubp, bool pstate_disallow);
+
 void hubp32_update_mall_sel(struct hubp *hubp, uint32_t mall_sel, bool c_cursor);
 
 void hubp32_prepare_subvp_buffering(struct hubp *hubp, bool enable);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c
index 9d14045cccd6..db0974fe58ab 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c
@@ -131,10 +131,15 @@ void dcn32_enable_power_gating_plane(
 	bool enable)
 {
 	bool force_on = true; /* disable power gating */
+	uint32_t org_ip_request_cntl = 0;
 
 	if (enable)
 		force_on = false;
 
+	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+
 	/* DCHUBP0/1/2/3 */
 	REG_UPDATE(DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, force_on);
 	REG_UPDATE(DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, force_on);
@@ -146,6 +151,9 @@ void dcn32_enable_power_gating_plane(
 	REG_UPDATE(DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, force_on);
 	REG_UPDATE(DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, force_on);
 	REG_UPDATE(DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, force_on);
+
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 0);
 }
 
 void dcn32_hubp_pg_control(struct dce_hwseq *hws, unsigned int hubp_inst, bool power_on)
@@ -406,7 +414,7 @@ void dcn32_subvp_pipe_control_lock(struct dc *dc,
 }
 
 
-static bool dcn32_set_mpc_shaper_3dlut(
+bool dcn32_set_mpc_shaper_3dlut(
 	struct pipe_ctx *pipe_ctx, const struct dc_stream_state *stream)
 {
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
@@ -563,39 +571,56 @@ bool dcn32_set_output_transfer_func(struct dc *dc,
 	return ret;
 }
 
-/* Program P-State force value according to if pipe is using SubVP or not:
+/* Program P-State force value according to if pipe is using SubVP / FPO or not:
  * 1. Reset P-State force on all pipes first
  * 2. For each main pipe, force P-State disallow (P-State allow moderated by DMUB)
  */
-void dcn32_subvp_update_force_pstate(struct dc *dc, struct dc_state *context)
+void dcn32_update_force_pstate(struct dc *dc, struct dc_state *context)
 {
 	int i;
-	int num_subvp = 0;
-	/* Unforce p-state for each pipe
+
+	/* Unforce p-state for each pipe if it is not FPO or SubVP.
+	 * For FPO and SubVP, if it's already forced disallow, leave
+	 * it as disallow.
 	 */
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
 		struct hubp *hubp = pipe->plane_res.hubp;
 
-		if (hubp && hubp->funcs->hubp_update_force_pstate_disallow)
-			hubp->funcs->hubp_update_force_pstate_disallow(hubp, false);
-		if (pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_MAIN)
-			num_subvp++;
-	}
+		if (!pipe->stream || (pipe->stream && !(pipe->stream->mall_stream_config.type == SUBVP_MAIN ||
+						pipe->stream->fpo_in_use))) {
+			if (hubp && hubp->funcs->hubp_update_force_pstate_disallow)
+				hubp->funcs->hubp_update_force_pstate_disallow(hubp, false);
+		}
 
-	if (num_subvp == 0)
-		return;
+		/* Today only FPO uses cursor P-State force. Only clear cursor P-State force
+		 * if it's not FPO.
+		 */
+		if (!pipe->stream || (pipe->stream && !pipe->stream->fpo_in_use)) {
+			if (hubp && hubp->funcs->hubp_update_force_cursor_pstate_disallow)
+				hubp->funcs->hubp_update_force_cursor_pstate_disallow(hubp, false);
+		}
+	}
 
 	/* Loop through each pipe -- for each subvp main pipe force p-state allow equal to false.
 	 */
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct hubp *hubp = pipe->plane_res.hubp;
 
-		if (pipe->stream && pipe->plane_state && (pipe->stream->mall_stream_config.type == SUBVP_MAIN)) {
-			struct hubp *hubp = pipe->plane_res.hubp;
+		if (pipe->stream && pipe->plane_state && pipe->stream->mall_stream_config.type == SUBVP_MAIN) {
+			if (hubp && hubp->funcs->hubp_update_force_pstate_disallow)
+				hubp->funcs->hubp_update_force_pstate_disallow(hubp, true);
+		}
 
+		if (pipe->stream && pipe->stream->fpo_in_use) {
 			if (hubp && hubp->funcs->hubp_update_force_pstate_disallow)
 				hubp->funcs->hubp_update_force_pstate_disallow(hubp, true);
+			/* For now only force cursor p-state disallow for FPO
+			 * Needs to be added for subvp once FW side gets updated
+			 */
+			if (hubp && hubp->funcs->hubp_update_force_cursor_pstate_disallow)
+				hubp->funcs->hubp_update_force_cursor_pstate_disallow(hubp, true);
 		}
 	}
 }
@@ -669,10 +694,6 @@ void dcn32_program_mall_pipe_config(struct dc *dc, struct dc_state *context)
 	if (hws && hws->funcs.update_mall_sel)
 		hws->funcs.update_mall_sel(dc, context);
 
-	//update subvp force pstate
-	if (hws && hws->funcs.subvp_update_force_pstate)
-		dc->hwseq->funcs.subvp_update_force_pstate(dc, context);
-
 	// Program FORCE_ONE_ROW_FOR_FRAME and CURSOR_REQ_MODE for main subvp pipes
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
@@ -700,10 +721,19 @@ static void dcn32_initialize_min_clocks(struct dc *dc)
 	clocks->socclk_khz = dc->clk_mgr->bw_params->clk_table.entries[0].socclk_mhz * 1000;
 	clocks->dramclk_khz = dc->clk_mgr->bw_params->clk_table.entries[0].memclk_mhz * 1000;
 	clocks->dppclk_khz = dc->clk_mgr->bw_params->clk_table.entries[0].dppclk_mhz * 1000;
-	clocks->dispclk_khz = dc->clk_mgr->bw_params->clk_table.entries[0].dispclk_mhz * 1000;
-	clocks->ref_dtbclk_khz = dc->clk_mgr->bw_params->clk_table.entries[0].dtbclk_mhz * 1000;
-	clocks->fclk_p_state_change_support = true;
-	clocks->p_state_change_support = true;
+	if (dc->debug.disable_boot_optimizations) {
+		clocks->dispclk_khz = dc->clk_mgr->bw_params->clk_table.entries[0].dispclk_mhz * 1000;
+	} else {
+		/* Even though DPG_EN = 1 for the connected display, it still requires the
+		 * correct timing so we cannot set DISPCLK to min freq or it could cause
+		 * audio corruption. Read current DISPCLK from DENTIST and request the same
+		 * freq to ensure that the timing is valid and unchanged.
+		 */
+		clocks->dispclk_khz = dc->clk_mgr->funcs->get_dispclk_from_dentist(dc->clk_mgr);
+		clocks->ref_dtbclk_khz = dc->clk_mgr->bw_params->clk_table.entries[0].dtbclk_mhz * 1000;
+		clocks->fclk_p_state_change_support = true;
+		clocks->p_state_change_support = true;
+	}
 
 	dc->clk_mgr->funcs->update_clocks(
 			dc->clk_mgr,
@@ -786,13 +816,14 @@ void dcn32_init_hw(struct dc *dc)
 		}
 	}
 
-	/* Power gate DSCs */
-	for (i = 0; i < res_pool->res_cap->num_dsc; i++)
-		if (hws->funcs.dsc_pg_control != NULL)
-			hws->funcs.dsc_pg_control(hws, res_pool->dscs[i]->inst, false);
+	/* enable_power_gating_plane before dsc_pg_control because
+	 * FORCEON = 1 with hw default value on bootup, resume from s3
+	 */
+	if (hws->funcs.enable_power_gating_plane)
+		hws->funcs.enable_power_gating_plane(dc->hwseq, true);
 
 	/* we want to turn off all dp displays before doing detection */
-	link_blank_all_dp_displays(dc);
+	dc->link_srv->blank_all_dp_displays(dc);
 
 	/* If taking control over from VBIOS, we may want to optimize our first
 	 * mode set, so we need to skip powering down pipes until we know which
@@ -801,7 +832,14 @@ void dcn32_init_hw(struct dc *dc)
 	 * everything down.
 	 */
 	if (dcb->funcs->is_accelerated_mode(dcb) || !dc->config.seamless_boot_edp_requested) {
-		hws->funcs.init_pipes(dc, dc->current_state);
+		/* Disable boot optimizations means power down everything including PHY, DIG,
+		 * and OTG (i.e. the boot is not optimized because we do a full power down).
+		 */
+		if (dc->hwss.enable_accelerated_mode && dc->debug.disable_boot_optimizations)
+			dc->hwss.enable_accelerated_mode(dc, dc->current_state);
+		else
+			hws->funcs.init_pipes(dc, dc->current_state);
+
 		if (dc->res_pool->hubbub->funcs->allow_self_refresh_control)
 			dc->res_pool->hubbub->funcs->allow_self_refresh_control(dc->res_pool->hubbub,
 					!dc->res_pool->hubbub->ctx->dc->debug.disable_stutter);
@@ -828,7 +866,7 @@ void dcn32_init_hw(struct dc *dc)
 		struct dc_link *edp_links[MAX_NUM_EDP];
 		struct dc_link *edp_link;
 
-		get_edp_links(dc, edp_links, &edp_num);
+		dc_get_edp_links(dc, edp_links, &edp_num);
 		if (edp_num) {
 			for (i = 0; i < edp_num; i++) {
 				edp_link = edp_links[i];
@@ -886,8 +924,6 @@ void dcn32_init_hw(struct dc *dc)
 
 		REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
 	}
-	if (hws->funcs.enable_power_gating_plane)
-		hws->funcs.enable_power_gating_plane(dc->hwseq, true);
 
 	if (!dcb->funcs->is_accelerated_mode(dcb) && dc->res_pool->hubbub->funcs->init_watermarks)
 		dc->res_pool->hubbub->funcs->init_watermarks(dc->res_pool->hubbub);
@@ -895,7 +931,7 @@ void dcn32_init_hw(struct dc *dc)
 	if (dc->clk_mgr->funcs->notify_wm_ranges)
 		dc->clk_mgr->funcs->notify_wm_ranges(dc->clk_mgr);
 
-	if (dc->clk_mgr->funcs->set_hard_max_memclk)
+	if (dc->clk_mgr->funcs->set_hard_max_memclk && !dc->clk_mgr->dc_mode_softmax_enabled)
 		dc->clk_mgr->funcs->set_hard_max_memclk(dc->clk_mgr);
 
 	if (dc->res_pool->hubbub->funcs->force_pstate_change_control)
@@ -1095,7 +1131,7 @@ unsigned int dcn32_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsign
 	two_pix_per_container = optc2_is_two_pixels_per_containter(&stream->timing);
 	odm_combine_factor = get_odm_config(pipe_ctx, NULL);
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (stream->ctx->dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		*k1_div = PIXEL_RATE_DIV_BY_1;
 		*k2_div = PIXEL_RATE_DIV_BY_1;
 	} else if (dc_is_hdmi_tmds_signal(stream->signal) || dc_is_dvi_signal(stream->signal)) {
@@ -1104,7 +1140,7 @@ unsigned int dcn32_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsign
 			*k2_div = PIXEL_RATE_DIV_BY_2;
 		else
 			*k2_div = PIXEL_RATE_DIV_BY_4;
-	} else if (dc_is_dp_signal(stream->signal)) {
+	} else if (dc_is_dp_signal(stream->signal) || dc_is_virtual_signal(stream->signal)) {
 		if (two_pix_per_container) {
 			*k1_div = PIXEL_RATE_DIV_BY_1;
 			*k2_div = PIXEL_RATE_DIV_BY_2;
@@ -1159,7 +1195,7 @@ void dcn32_unblank_stream(struct pipe_ctx *pipe_ctx,
 
 	params.link_settings.link_rate = link_settings->link_rate;
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (link->dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		/* TODO - DP2.0 HW: Set ODM mode in dp hpo encoder here */
 		pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_unblank(
 				pipe_ctx->stream_res.hpo_dp_stream_enc,
@@ -1186,7 +1222,7 @@ bool dcn32_is_dp_dig_pixel_rate_div_policy(struct pipe_ctx *pipe_ctx)
 	if (!is_h_timing_divisible_by_2(pipe_ctx->stream))
 		return false;
 
-	if (dc_is_dp_signal(pipe_ctx->stream->signal) && !link_is_dp_128b_132b_signal(pipe_ctx) &&
+	if (dc_is_dp_signal(pipe_ctx->stream->signal) && !dc->link_srv->dp_is_128b_132b_signal(pipe_ctx) &&
 		dc->debug.enable_dp_dig_pixel_rate_div_policy)
 		return true;
 	return false;
@@ -1220,7 +1256,8 @@ static void apply_symclk_on_tx_off_wa(struct dc_link *link)
 				pipe_ctx->clock_source->funcs->program_pix_clk(
 						pipe_ctx->clock_source,
 						&pipe_ctx->stream_res.pix_clk_params,
-						link_dp_get_encoding_format(&pipe_ctx->link_config.dp_link_settings),
+						dc->link_srv->dp_get_encoding_format(
+								&pipe_ctx->link_config.dp_link_settings),
 						&pipe_ctx->pll_settings);
 				link->phy_state.symclk_state = SYMCLK_ON_TX_OFF;
 				break;
@@ -1252,7 +1289,7 @@ void dcn32_disable_link_output(struct dc_link *link,
 	else if (dmcu != NULL && dmcu->funcs->lock_phy)
 		dmcu->funcs->unlock_phy(dmcu);
 
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
+	dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
 
 	apply_symclk_on_tx_off_wa(link);
 }
@@ -1406,3 +1443,86 @@ void dcn32_enable_phantom_streams(struct dc *dc, struct dc_state *context)
 		}
 	}
 }
+
+/* Blank pixel data during initialization */
+void dcn32_init_blank(
+		struct dc *dc,
+		struct timing_generator *tg)
+{
+	struct dce_hwseq *hws = dc->hwseq;
+	enum dc_color_space color_space;
+	struct tg_color black_color = {0};
+	struct output_pixel_processor *opp = NULL;
+	struct output_pixel_processor *bottom_opp = NULL;
+	uint32_t num_opps, opp_id_src0, opp_id_src1;
+	uint32_t otg_active_width, otg_active_height;
+	uint32_t i;
+
+	/* program opp dpg blank color */
+	color_space = COLOR_SPACE_SRGB;
+	color_space_to_black_color(dc, color_space, &black_color);
+
+	/* get the OTG active size */
+	tg->funcs->get_otg_active_size(tg,
+			&otg_active_width,
+			&otg_active_height);
+
+	/* get the OPTC source */
+	tg->funcs->get_optc_source(tg, &num_opps, &opp_id_src0, &opp_id_src1);
+
+	if (opp_id_src0 >= dc->res_pool->res_cap->num_opp) {
+		ASSERT(false);
+		return;
+	}
+
+	for (i = 0; i < dc->res_pool->res_cap->num_opp; i++) {
+		if (dc->res_pool->opps[i] != NULL && dc->res_pool->opps[i]->inst == opp_id_src0) {
+			opp = dc->res_pool->opps[i];
+			break;
+		}
+	}
+
+	if (num_opps == 2) {
+		otg_active_width = otg_active_width / 2;
+
+		if (opp_id_src1 >= dc->res_pool->res_cap->num_opp) {
+			ASSERT(false);
+			return;
+		}
+		for (i = 0; i < dc->res_pool->res_cap->num_opp; i++) {
+			if (dc->res_pool->opps[i] != NULL && dc->res_pool->opps[i]->inst == opp_id_src1) {
+				bottom_opp = dc->res_pool->opps[i];
+				break;
+			}
+		}
+	}
+
+	if (opp && opp->funcs->opp_set_disp_pattern_generator)
+		opp->funcs->opp_set_disp_pattern_generator(
+				opp,
+				CONTROLLER_DP_TEST_PATTERN_SOLID_COLOR,
+				CONTROLLER_DP_COLOR_SPACE_UDEFINED,
+				COLOR_DEPTH_UNDEFINED,
+				&black_color,
+				otg_active_width,
+				otg_active_height,
+				0);
+
+	if (num_opps == 2) {
+		if (bottom_opp && bottom_opp->funcs->opp_set_disp_pattern_generator) {
+			bottom_opp->funcs->opp_set_disp_pattern_generator(
+					bottom_opp,
+					CONTROLLER_DP_TEST_PATTERN_SOLID_COLOR,
+					CONTROLLER_DP_COLOR_SPACE_UDEFINED,
+					COLOR_DEPTH_UNDEFINED,
+					&black_color,
+					otg_active_width,
+					otg_active_height,
+					0);
+			hws->funcs.wait_for_blank_complete(bottom_opp);
+		}
+	}
+
+	if (opp)
+		hws->funcs.wait_for_blank_complete(opp);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h
index e9e9534f3668..6694c1d14aa3 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h
@@ -54,6 +54,9 @@ bool dcn32_set_input_transfer_func(struct dc *dc,
 				struct pipe_ctx *pipe_ctx,
 				const struct dc_plane_state *plane_state);
 
+bool dcn32_set_mpc_shaper_3dlut(
+	struct pipe_ctx *pipe_ctx, const struct dc_stream_state *stream);
+
 bool dcn32_set_output_transfer_func(struct dc *dc,
 				struct pipe_ctx *pipe_ctx,
 				const struct dc_stream_state *stream);
@@ -64,7 +67,7 @@ void dcn32_program_mall_pipe_config(struct dc *dc, struct dc_state *context);
 
 void dcn32_update_mall_sel(struct dc *dc, struct dc_state *context);
 
-void dcn32_subvp_update_force_pstate(struct dc *dc, struct dc_state *context);
+void dcn32_update_force_pstate(struct dc *dc, struct dc_state *context);
 
 void dcn32_update_odm(struct dc *dc, struct dc_state *context, struct pipe_ctx *pipe_ctx);
 
@@ -104,4 +107,8 @@ void dcn32_update_dsc_pg(struct dc *dc,
 
 void dcn32_enable_phantom_streams(struct dc *dc, struct dc_state *context);
 
+void dcn32_init_blank(
+		struct dc *dc,
+		struct timing_generator *tg);
+
 #endif /* __DC_HWSS_DCN32_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c
index 0694fa3a3680..8085f2acb1a9 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c
@@ -132,7 +132,7 @@ static const struct hwseq_private_funcs dcn32_private_funcs = {
 	.enable_stream_gating = dcn20_enable_stream_gating,
 	.setup_vupdate_interrupt = dcn20_setup_vupdate_interrupt,
 	.did_underflow_occur = dcn10_did_underflow_occur,
-	.init_blank = dcn20_init_blank,
+	.init_blank = dcn32_init_blank,
 	.disable_vga = dcn20_disable_vga,
 	.bios_golden_init = dcn10_bios_golden_init,
 	.plane_atomic_disable = dcn20_plane_atomic_disable,
@@ -149,7 +149,7 @@ static const struct hwseq_private_funcs dcn32_private_funcs = {
 	.dccg_init = dcn20_dccg_init,
 	.set_mcm_luts = dcn32_set_mcm_luts,
 	.program_mall_pipe_config = dcn32_program_mall_pipe_config,
-	.subvp_update_force_pstate = dcn32_subvp_update_force_pstate,
+	.update_force_pstate = dcn32_update_force_pstate,
 	.update_mall_sel = dcn32_update_mall_sel,
 	.calculate_dccg_k1_k2_values = dcn32_calculate_dccg_k1_k2_values,
 	.set_pixels_per_cycle = dcn32_set_pixels_per_cycle,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c
index 206a5ddbaf6d..c8041cfd594d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c
@@ -42,7 +42,7 @@
 	mpc30->mpc_shift->field_name, mpc30->mpc_mask->field_name
 
 
-static void mpc32_mpc_init(struct mpc *mpc)
+void mpc32_mpc_init(struct mpc *mpc)
 {
 	struct dcn30_mpc *mpc30 = TO_DCN30_MPC(mpc);
 	int mpcc_id;
@@ -254,7 +254,7 @@ static void mpc32_program_post1dlut_pwl(
 	}
 }
 
-static bool mpc32_program_post1dlut(
+bool mpc32_program_post1dlut(
 		struct mpc *mpc,
 		const struct pwl_params *params,
 		uint32_t mpcc_id)
@@ -701,7 +701,7 @@ static void mpc32_power_on_shaper_3dlut(
 }
 
 
-static bool mpc32_program_shaper(
+bool mpc32_program_shaper(
 		struct mpc *mpc,
 		const struct pwl_params *params,
 		uint32_t mpcc_id)
@@ -897,7 +897,7 @@ static void mpc32_set_3dlut_mode(
 }
 
 
-static bool mpc32_program_3dlut(
+bool mpc32_program_3dlut(
 		struct mpc *mpc,
 		const struct tetrahedral_params *params,
 		int mpcc_id)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.h
index 61f33c0d8e59..2c2ecd053806 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.h
@@ -310,6 +310,19 @@ struct dcn32_mpc_registers {
 	MPC_REG_VARIABLE_LIST_DCN3_0;
 	MPC_REG_VARIABLE_LIST_DCN32;
 };
+void mpc32_mpc_init(struct mpc *mpc);
+bool mpc32_program_3dlut(
+		struct mpc *mpc,
+		const struct tetrahedral_params *params,
+		int mpcc_id);
+bool mpc32_program_post1dlut(
+		struct mpc *mpc,
+		const struct pwl_params *params,
+		uint32_t mpcc_id);
+bool mpc32_program_shaper(
+		struct mpc *mpc,
+		const struct pwl_params *params,
+		uint32_t mpcc_id);
 
 void dcn32_mpc_construct(struct dcn30_mpc *mpc30,
 	struct dc_context *ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.h
index 5e57c39235fa..b92ba8c75694 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.h
@@ -28,77 +28,6 @@
 
 #include "dcn10/dcn10_optc.h"
 
-#define OPTC_COMMON_REG_LIST_DCN3_2(inst) \
-	SRI(OTG_VSTARTUP_PARAM, OTG, inst),\
-	SRI(OTG_VUPDATE_PARAM, OTG, inst),\
-	SRI(OTG_VREADY_PARAM, OTG, inst),\
-	SRI(OTG_MASTER_UPDATE_LOCK, OTG, inst),\
-	SRI(OTG_GLOBAL_CONTROL0, OTG, inst),\
-	SRI(OTG_GLOBAL_CONTROL1, OTG, inst),\
-	SRI(OTG_GLOBAL_CONTROL2, OTG, inst),\
-	SRI(OTG_GLOBAL_CONTROL4, OTG, inst),\
-	SRI(OTG_DOUBLE_BUFFER_CONTROL, OTG, inst),\
-	SRI(OTG_H_TOTAL, OTG, inst),\
-	SRI(OTG_H_BLANK_START_END, OTG, inst),\
-	SRI(OTG_H_SYNC_A, OTG, inst),\
-	SRI(OTG_H_SYNC_A_CNTL, OTG, inst),\
-	SRI(OTG_H_TIMING_CNTL, OTG, inst),\
-	SRI(OTG_V_TOTAL, OTG, inst),\
-	SRI(OTG_V_BLANK_START_END, OTG, inst),\
-	SRI(OTG_V_SYNC_A, OTG, inst),\
-	SRI(OTG_V_SYNC_A_CNTL, OTG, inst),\
-	SRI(OTG_CONTROL, OTG, inst),\
-	SRI(OTG_STEREO_CONTROL, OTG, inst),\
-	SRI(OTG_3D_STRUCTURE_CONTROL, OTG, inst),\
-	SRI(OTG_STEREO_STATUS, OTG, inst),\
-	SRI(OTG_V_TOTAL_MAX, OTG, inst),\
-	SRI(OTG_V_TOTAL_MIN, OTG, inst),\
-	SRI(OTG_V_TOTAL_CONTROL, OTG, inst),\
-	SRI(OTG_TRIGA_CNTL, OTG, inst),\
-	SRI(OTG_FORCE_COUNT_NOW_CNTL, OTG, inst),\
-	SRI(OTG_STATIC_SCREEN_CONTROL, OTG, inst),\
-	SRI(OTG_STATUS_FRAME_COUNT, OTG, inst),\
-	SRI(OTG_STATUS, OTG, inst),\
-	SRI(OTG_STATUS_POSITION, OTG, inst),\
-	SRI(OTG_NOM_VERT_POSITION, OTG, inst),\
-	SRI(OTG_M_CONST_DTO0, OTG, inst),\
-	SRI(OTG_M_CONST_DTO1, OTG, inst),\
-	SRI(OTG_CLOCK_CONTROL, OTG, inst),\
-	SRI(OTG_VERTICAL_INTERRUPT0_CONTROL, OTG, inst),\
-	SRI(OTG_VERTICAL_INTERRUPT0_POSITION, OTG, inst),\
-	SRI(OTG_VERTICAL_INTERRUPT1_CONTROL, OTG, inst),\
-	SRI(OTG_VERTICAL_INTERRUPT1_POSITION, OTG, inst),\
-	SRI(OTG_VERTICAL_INTERRUPT2_CONTROL, OTG, inst),\
-	SRI(OTG_VERTICAL_INTERRUPT2_POSITION, OTG, inst),\
-	SRI(OPTC_INPUT_CLOCK_CONTROL, ODM, inst),\
-	SRI(OPTC_DATA_SOURCE_SELECT, ODM, inst),\
-	SRI(OPTC_INPUT_GLOBAL_CONTROL, ODM, inst),\
-	SRI(CONTROL, VTG, inst),\
-	SRI(OTG_VERT_SYNC_CONTROL, OTG, inst),\
-	SRI(OTG_GSL_CONTROL, OTG, inst),\
-	SRI(OTG_CRC_CNTL, OTG, inst),\
-	SRI(OTG_CRC0_DATA_RG, OTG, inst),\
-	SRI(OTG_CRC0_DATA_B, OTG, inst),\
-	SRI(OTG_CRC0_WINDOWA_X_CONTROL, OTG, inst),\
-	SRI(OTG_CRC0_WINDOWA_Y_CONTROL, OTG, inst),\
-	SRI(OTG_CRC0_WINDOWB_X_CONTROL, OTG, inst),\
-	SRI(OTG_CRC0_WINDOWB_Y_CONTROL, OTG, inst),\
-	SR(GSL_SOURCE_SELECT),\
-	SRI(OTG_TRIGA_MANUAL_TRIG, OTG, inst),\
-	SRI(OTG_GLOBAL_CONTROL1, OTG, inst),\
-	SRI(OTG_GLOBAL_CONTROL2, OTG, inst),\
-	SRI(OTG_GSL_WINDOW_X, OTG, inst),\
-	SRI(OTG_GSL_WINDOW_Y, OTG, inst),\
-	SRI(OTG_VUPDATE_KEEPOUT, OTG, inst),\
-	SRI(OTG_DSC_START_POSITION, OTG, inst),\
-	SRI(OTG_DRR_TRIGGER_WINDOW, OTG, inst),\
-	SRI(OTG_DRR_V_TOTAL_CHANGE, OTG, inst),\
-	SRI(OPTC_DATA_FORMAT_CONTROL, ODM, inst),\
-	SRI(OPTC_BYTES_PER_PIXEL, ODM, inst),\
-	SRI(OPTC_WIDTH_CONTROL, ODM, inst),\
-	SRI(OPTC_MEMORY_CONFIG, ODM, inst),\
-	SRI(OTG_DRR_CONTROL, OTG, inst)
-
 #define OPTC_COMMON_MASK_SH_LIST_DCN3_2(mask_sh)\
 	SF(OTG0_OTG_VSTARTUP_PARAM, VSTARTUP_START, mask_sh),\
 	SF(OTG0_OTG_VUPDATE_PARAM, VUPDATE_OFFSET, mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c
index 4b7abb4af623..e30d1f60695d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c
@@ -657,8 +657,6 @@ static const struct resource_caps res_cap_dcn32 = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
@@ -726,6 +724,11 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.alloc_extra_way_for_cursor = true,
 	.min_prefetch_in_strobe_ns = 60000, // 60us
 	.disable_unbounded_requesting = false,
+	.override_dispclk_programming = true,
+	.disable_fpo_optimizations = false,
+	.fpo_vactive_margin_us = 2000, // 2000us
+	.disable_fpo_vactive = true,
+	.disable_boot_optimizations = false,
 };
 
 static const struct dc_debug_options debug_defaults_diags = {
@@ -1506,8 +1509,11 @@ static void dcn32_resource_destruct(struct dcn32_resource_pool *pool)
 	if (pool->base.dccg != NULL)
 		dcn_dccg_destroy(&pool->base.dccg);
 
-	if (pool->base.oem_device != NULL)
-		link_destroy_ddc_service(&pool->base.oem_device);
+	if (pool->base.oem_device != NULL) {
+		struct dc *dc = pool->base.oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
+	}
 }
 
 
@@ -1611,7 +1617,6 @@ bool dcn32_acquire_post_bldn_3dlut(
 		struct dc_transfer_func **shaper)
 {
 	bool ret = false;
-	union dc_3dlut_state *state;
 
 	ASSERT(*lut == NULL && *shaper == NULL);
 	*lut = NULL;
@@ -1620,7 +1625,6 @@ bool dcn32_acquire_post_bldn_3dlut(
 	if (!res_ctx->is_mpc_3dlut_acquired[mpcc_id]) {
 		*lut = pool->mpc_lut[mpcc_id];
 		*shaper = pool->mpc_shaper[mpcc_id];
-		state = &pool->mpc_lut[mpcc_id]->state;
 		res_ctx->is_mpc_3dlut_acquired[mpcc_id] = true;
 		ret = true;
 	}
@@ -1913,7 +1917,6 @@ int dcn32_populate_dml_pipes_from_context(
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe;
 	bool subvp_in_use = false;
-	uint8_t is_pipe_split_expected[MAX_PIPES] = {0};
 	struct dc_crtc_timing *timing;
 	bool vsr_odm_support = false;
 
@@ -2006,7 +2009,7 @@ int dcn32_populate_dml_pipes_from_context(
 		}
 
 		DC_FP_START();
-		is_pipe_split_expected[i] = dcn32_predict_pipe_split(context, &pipes[pipe_cnt]);
+		dcn32_predict_pipe_split(context, &pipes[pipe_cnt]);
 		DC_FP_END();
 
 		pipe_cnt++;
@@ -2460,7 +2463,7 @@ static bool dcn32_resource_construct(
 		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
 		ddc_init_data.id.enum_id = 0;
 		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->base.oem_device = link_create_ddc_service(&ddc_init_data);
+		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
 	} else {
 		pool->base.oem_device = NULL;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h
index aca928edc4e3..3937dbc1e552 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h
@@ -39,6 +39,7 @@
 #define DCN3_2_MBLK_HEIGHT_8BPE 64
 #define DCN3_2_VMIN_DISPCLK_HZ 717000000
 #define DCN3_2_DCFCLK_DS_INIT_KHZ 10000 // Choose 10Mhz for init DCFCLK DS freq
+#define DCN3_2_MIN_ACTIVE_SWITCH_MARGIN_FPO_US 100 // Only allow FPO + Vactive if active margin >= 100
 
 #define TO_DCN32_RES_POOL(pool)\
 	container_of(pool, struct dcn32_resource_pool, base)
@@ -146,6 +147,8 @@ void dcn32_restore_mall_state(struct dc *dc,
 		struct dc_state *context,
 		struct mall_temp_config *temp_config);
 
+struct dc_stream_state *dcn32_can_support_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, const struct dc_state *context);
+
 bool dcn32_allow_subvp_with_active_margin(struct pipe_ctx *pipe);
 
 unsigned int dcn32_calc_num_avail_chans_for_mall(struct dc *dc, int num_chans);
@@ -472,6 +475,7 @@ double dcn32_determine_max_vratio_prefetch(struct dc *dc, struct dc_state *conte
       SRI_ARR(OTG_H_BLANK, DSCL, id), SRI_ARR(OTG_V_BLANK, DSCL, id),          \
       SRI_ARR(SCL_MODE, DSCL, id), SRI_ARR(LB_DATA_FORMAT, DSCL, id),          \
       SRI_ARR(LB_MEMORY_CTRL, DSCL, id), SRI_ARR(DSCL_AUTOCAL, DSCL, id),      \
+      SRI_ARR(DSCL_CONTROL, DSCL, id),                                         \
       SRI_ARR(SCL_TAP_CONTROL, DSCL, id),                                      \
       SRI_ARR(SCL_COEF_RAM_TAP_SELECT, DSCL, id),                              \
       SRI_ARR(SCL_COEF_RAM_TAP_DATA, DSCL, id),                                \
@@ -1276,7 +1280,8 @@ double dcn32_determine_max_vratio_prefetch(struct dc *dc, struct dc_state *conte
       DCCG_SRII(PHASE, DTBCLK_DTO, 0), DCCG_SRII(PHASE, DTBCLK_DTO, 1),        \
       DCCG_SRII(PHASE, DTBCLK_DTO, 2), DCCG_SRII(PHASE, DTBCLK_DTO, 3),        \
       SR(DCCG_AUDIO_DTBCLK_DTO_MODULO), SR(DCCG_AUDIO_DTBCLK_DTO_PHASE),       \
-      SR(OTG_PIXEL_RATE_DIV), SR(DTBCLK_P_CNTL), SR(DCCG_AUDIO_DTO_SOURCE)     \
+      SR(OTG_PIXEL_RATE_DIV), SR(DTBCLK_P_CNTL),                               \
+      SR(DCCG_AUDIO_DTO_SOURCE), SR(DENTIST_DISPCLK_CNTL)                      \
   )
 
 /* VMID */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
index 3a2d7bcc4b6d..eeca16faf31a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
@@ -27,6 +27,7 @@
 #include "dcn32_resource.h"
 #include "dcn20/dcn20_resource.h"
 #include "dml/dcn32/display_mode_vba_util_32.h"
+#include "dml/dcn32/dcn32_fpu.h"
 
 static bool is_dual_plane(enum surface_pixel_format format)
 {
@@ -59,25 +60,21 @@ uint32_t dcn32_helper_calculate_mall_bytes_for_cursor(
 {
 	struct hubp *hubp = pipe_ctx->plane_res.hubp;
 	uint32_t cursor_size = hubp->curs_attr.pitch * hubp->curs_attr.height;
-	uint32_t cursor_bpp = 4;
 	uint32_t cursor_mall_size_bytes = 0;
 
 	switch (pipe_ctx->stream->cursor_attributes.color_format) {
 	case CURSOR_MODE_MONO:
 		cursor_size /= 2;
-		cursor_bpp = 4;
 		break;
 	case CURSOR_MODE_COLOR_1BIT_AND:
 	case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA:
 	case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA:
 		cursor_size *= 4;
-		cursor_bpp = 4;
 		break;
 
 	case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED:
 	case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED:
 		cursor_size *= 8;
-		cursor_bpp = 8;
 		break;
 	}
 
@@ -261,6 +258,8 @@ bool dcn32_is_psr_capable(struct pipe_ctx *pipe)
 	return psr_capable;
 }
 
+#define DCN3_2_NEW_DET_OVERRIDE_MIN_MULTIPLIER 7
+
 /**
  * *******************************************************************************************
  * dcn32_determine_det_override: Determine DET allocation for each pipe
@@ -272,7 +271,6 @@ bool dcn32_is_psr_capable(struct pipe_ctx *pipe)
  * If there is a plane that's driven by more than 1 pipe (i.e. pipe split), then the
  * number of DET for that given plane will be split among the pipes driving that plane.
  *
- *
  * High level algorithm:
  * 1. Split total DET among number of streams
  * 2. For each stream, split DET among the planes
@@ -280,6 +278,18 @@ bool dcn32_is_psr_capable(struct pipe_ctx *pipe)
  *    among those pipes.
  * 4. Assign the DET override to the DML pipes.
  *
+ * Special cases:
+ *
+ * For two displays that have a large difference in pixel rate, we may experience
+ *  underflow on the larger display when we divide the DET equally. For this, we
+ *  will implement a modified algorithm to assign more DET to larger display.
+ *
+ * 1. Calculate difference in pixel rates ( multiplier ) between two displays
+ * 2. If the multiplier exceeds DCN3_2_NEW_DET_OVERRIDE_MIN_MULTIPLIER, then
+ *    implement the modified DET override algorithm.
+ * 3. Assign smaller DET size for lower pixel display and higher DET size for
+ *    higher pixel display
+ *
  * @param [in]: dc: Current DC state
  * @param [in]: context: New DC state to be programmed
  * @param [in]: pipes: Array of DML pipes
@@ -299,18 +309,46 @@ void dcn32_determine_det_override(struct dc *dc,
 	struct dc_plane_state *current_plane = NULL;
 	uint8_t stream_count = 0;
 
+	int phy_pix_clk_mult, lower_mode_stream_index;
+	int phy_pix_clk[MAX_PIPES] = {0};
+	bool use_new_det_override_algorithm = false;
+
 	for (i = 0; i < context->stream_count; i++) {
 		/* Don't count SubVP streams for DET allocation */
 		if (context->streams[i]->mall_stream_config.type != SUBVP_PHANTOM) {
+			phy_pix_clk[i] = context->streams[i]->phy_pix_clk;
 			stream_count++;
 		}
 	}
 
+	/* Check for special case with two displays, one with much higher pixel rate */
+	if (stream_count == 2) {
+		ASSERT((phy_pix_clk[0] > 0) && (phy_pix_clk[1] > 0));
+		if (phy_pix_clk[0] < phy_pix_clk[1]) {
+			lower_mode_stream_index = 0;
+			phy_pix_clk_mult = phy_pix_clk[1] / phy_pix_clk[0];
+		} else {
+			lower_mode_stream_index = 1;
+			phy_pix_clk_mult = phy_pix_clk[0] / phy_pix_clk[1];
+		}
+
+		if (phy_pix_clk_mult >= DCN3_2_NEW_DET_OVERRIDE_MIN_MULTIPLIER)
+			use_new_det_override_algorithm = true;
+	}
+
 	if (stream_count > 0) {
 		stream_segments = 18 / stream_count;
 		for (i = 0; i < context->stream_count; i++) {
 			if (context->streams[i]->mall_stream_config.type == SUBVP_PHANTOM)
 				continue;
+
+			if (use_new_det_override_algorithm) {
+				if (i == lower_mode_stream_index)
+					stream_segments = 4;
+				else
+					stream_segments = 14;
+			}
+
 			if (context->stream_status[i].plane_count > 0)
 				plane_segments = stream_segments / context->stream_status[i].plane_count;
 			else
@@ -463,3 +501,158 @@ void dcn32_restore_mall_state(struct dc *dc,
 			pipe->plane_state->is_phantom = temp_config->is_phantom_plane[i];
 	}
 }
+
+#define MAX_STRETCHED_V_BLANK 1000 // in micro-seconds (must ensure to match value in FW)
+/*
+ * Scaling factor for v_blank stretch calculations considering timing in
+ * micro-seconds and pixel clock in 100hz.
+ * Note: the parenthesis are necessary to ensure the correct order of
+ * operation where V_SCALE is used.
+ */
+#define V_SCALE (10000 / MAX_STRETCHED_V_BLANK)
+
+static int get_frame_rate_at_max_stretch_100hz(
+		struct dc_stream_state *fpo_candidate_stream,
+		uint32_t fpo_vactive_margin_us)
+{
+	struct dc_crtc_timing *timing = NULL;
+	uint32_t sec_per_100_lines;
+	uint32_t max_v_blank;
+	uint32_t curr_v_blank;
+	uint32_t v_stretch_max;
+	uint32_t stretched_frame_pix_cnt;
+	uint32_t scaled_stretched_frame_pix_cnt;
+	uint32_t scaled_refresh_rate;
+	uint32_t v_scale;
+
+	if (fpo_candidate_stream == NULL)
+		return 0;
+
+	/* check if refresh rate at least 120hz */
+	timing = &fpo_candidate_stream->timing;
+	if (timing == NULL)
+		return 0;
+
+	v_scale = 10000 / (MAX_STRETCHED_V_BLANK + fpo_vactive_margin_us);
+
+	sec_per_100_lines = timing->pix_clk_100hz / timing->h_total + 1;
+	max_v_blank = sec_per_100_lines / v_scale + 1;
+	curr_v_blank = timing->v_total - timing->v_addressable;
+	v_stretch_max = (max_v_blank > curr_v_blank) ? (max_v_blank - curr_v_blank) : (0);
+	stretched_frame_pix_cnt = (v_stretch_max + timing->v_total) * timing->h_total;
+	scaled_stretched_frame_pix_cnt = stretched_frame_pix_cnt / 10000;
+	scaled_refresh_rate = (timing->pix_clk_100hz) / scaled_stretched_frame_pix_cnt + 1;
+
+	return scaled_refresh_rate;
+
+}
+
+static bool is_refresh_rate_support_mclk_switch_using_fw_based_vblank_stretch(
+		struct dc_stream_state *fpo_candidate_stream, uint32_t fpo_vactive_margin_us)
+{
+	int refresh_rate_max_stretch_100hz;
+	int min_refresh_100hz;
+
+	if (fpo_candidate_stream == NULL)
+		return false;
+
+	refresh_rate_max_stretch_100hz = get_frame_rate_at_max_stretch_100hz(fpo_candidate_stream, fpo_vactive_margin_us);
+	min_refresh_100hz = fpo_candidate_stream->timing.min_refresh_in_uhz / 10000;
+
+	if (refresh_rate_max_stretch_100hz < min_refresh_100hz)
+		return false;
+
+	return true;
+}
+
+static int get_refresh_rate(struct dc_stream_state *fpo_candidate_stream)
+{
+	int refresh_rate = 0;
+	int h_v_total = 0;
+	struct dc_crtc_timing *timing = NULL;
+
+	if (fpo_candidate_stream == NULL)
+		return 0;
+
+	/* check if refresh rate at least 120hz */
+	timing = &fpo_candidate_stream->timing;
+	if (timing == NULL)
+		return 0;
+
+	h_v_total = timing->h_total * timing->v_total;
+	if (h_v_total == 0)
+		return 0;
+
+	refresh_rate = ((timing->pix_clk_100hz * 100) / (h_v_total)) + 1;
+	return refresh_rate;
+}
+
+/**
+ * dcn32_can_support_mclk_switch_using_fw_based_vblank_stretch - Determines if config can support FPO
+ *
+ * @param [in]: dc - current dc state
+ * @param [in]: context - new dc state
+ *
+ * Return: Pointer to FPO stream candidate if config can support FPO, otherwise NULL
+ */
+struct dc_stream_state *dcn32_can_support_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, const struct dc_state *context)
+{
+	int refresh_rate = 0;
+	const int minimum_refreshrate_supported = 120;
+	struct dc_stream_state *fpo_candidate_stream = NULL;
+	bool is_fpo_vactive = false;
+	uint32_t fpo_vactive_margin_us = 0;
+
+	if (context == NULL)
+		return NULL;
+
+	if (dc->debug.disable_fams)
+		return NULL;
+
+	if (!dc->caps.dmub_caps.mclk_sw)
+		return NULL;
+
+	if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching_shut_down)
+		return NULL;
+
+	/* For FPO we can support up to 2 display configs if:
+	 * - first display uses FPO
+	 * - Second display switches in VACTIVE */
+	if (context->stream_count > 2)
+		return NULL;
+	else if (context->stream_count == 2) {
+		DC_FP_START();
+		dcn32_assign_fpo_vactive_candidate(dc, context, &fpo_candidate_stream);
+		DC_FP_END();
+
+		DC_FP_START();
+		is_fpo_vactive = dcn32_find_vactive_pipe(dc, context, DCN3_2_MIN_ACTIVE_SWITCH_MARGIN_FPO_US);
+		DC_FP_END();
+		if (!is_fpo_vactive || dc->debug.disable_fpo_vactive)
+			return NULL;
+	} else
+		fpo_candidate_stream = context->streams[0];
+
+	if (!fpo_candidate_stream)
+		return NULL;
+
+	if (fpo_candidate_stream->sink->edid_caps.panel_patch.disable_fams)
+		return NULL;
+
+	refresh_rate = get_refresh_rate(fpo_candidate_stream);
+	if (refresh_rate < minimum_refreshrate_supported)
+		return NULL;
+
+	fpo_vactive_margin_us = is_fpo_vactive ? dc->debug.fpo_vactive_margin_us : 0; // For now hardcode the FPO + Vactive stretch margin to be 2000us
+	if (!is_refresh_rate_support_mclk_switch_using_fw_based_vblank_stretch(fpo_candidate_stream, fpo_vactive_margin_us))
+		return NULL;
+
+	// check if freesync enabled
+	if (!fpo_candidate_stream->allow_freesync)
+		return NULL;
+
+	if (fpo_candidate_stream->vrr_active_variable)
+		return NULL;
+
+	return fpo_candidate_stream;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c
index 55f918b44077..e5ab7f3077c4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c
@@ -655,8 +655,6 @@ static const struct resource_caps res_cap_dcn321 = {
 
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.blends_with_above = true,
-	.blends_with_below = true,
 	.per_pixel_alpha = true,
 
 	.pixel_format_support = {
@@ -724,6 +722,11 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.alloc_extra_way_for_cursor = true,
 	.min_prefetch_in_strobe_ns = 60000, // 60us
 	.disable_unbounded_requesting = false,
+	.override_dispclk_programming = true,
+	.disable_fpo_optimizations = false,
+	.fpo_vactive_margin_us = 2000, // 2000us
+	.disable_fpo_vactive = true,
+	.disable_boot_optimizations = false,
 };
 
 static const struct dc_debug_options debug_defaults_diags = {
@@ -1491,8 +1494,11 @@ static void dcn321_resource_destruct(struct dcn321_resource_pool *pool)
 	if (pool->base.dccg != NULL)
 		dcn_dccg_destroy(&pool->base.dccg);
 
-	if (pool->base.oem_device != NULL)
-		link_destroy_ddc_service(&pool->base.oem_device);
+	if (pool->base.oem_device != NULL) {
+		struct dc *dc = pool->base.oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
+	}
 }
 
 
@@ -1996,7 +2002,7 @@ static bool dcn321_resource_construct(
 		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
 		ddc_init_data.id.enum_id = 0;
 		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->base.oem_device = link_create_ddc_service(&ddc_init_data);
+		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
 	} else {
 		pool->base.oem_device = NULL;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile
index 9d0f79dff2e3..01db035589c5 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile
@@ -129,7 +129,7 @@ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/calcs/dcn_calc_math.o := $(dml_rcflags)
 
 DML = calcs/dce_calcs.o calcs/custom_float.o calcs/bw_fixed.o
 
-ifdef CONFIG_DRM_AMD_DC_DCN
+ifdef CONFIG_DRM_AMD_DC_FP
 DML += display_mode_lib.o display_rq_dlg_helpers.o dml1_display_rq_dlg_calc.o
 DML += dcn10/dcn10_fpu.o
 DML += dcn20/dcn20_fpu.o
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
index d3ba65efe1d2..38d1f2be8cf3 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
@@ -938,7 +938,7 @@ static bool is_dtbclk_required(struct dc *dc, struct dc_state *context)
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		if (!context->res_ctx.pipe_ctx[i].stream)
 			continue;
-		if (link_is_dp_128b_132b_signal(&context->res_ctx.pipe_ctx[i]))
+		if (dc->link_srv->dp_is_128b_132b_signal(&context->res_ctx.pipe_ctx[i]))
 			return true;
 	}
 	return false;
@@ -973,7 +973,8 @@ static enum dcn_zstate_support_state  decide_zstate_support(struct dc *dc, struc
 	else if (context->stream_count == 1 &&  context->streams[0]->signal == SIGNAL_TYPE_EDP) {
 		struct dc_link *link = context->streams[0]->sink->link;
 		struct dc_stream_status *stream_status = &context->stream_status[0];
-		bool allow_z8 = context->bw_ctx.dml.vba.StutterPeriod > 1000.0;
+		int minmum_z8_residency = dc->debug.minimum_z8_residency_time > 0 ? dc->debug.minimum_z8_residency_time : 1000;
+		bool allow_z8 = context->bw_ctx.dml.vba.StutterPeriod > (double)minmum_z8_residency;
 		bool is_pwrseq0 = link->link_index == 0;
 
 		if (dc_extended_blank_supported(dc)) {
@@ -1340,7 +1341,7 @@ int dcn20_populate_dml_pipes_from_context(
 		case SIGNAL_TYPE_DISPLAY_PORT_MST:
 		case SIGNAL_TYPE_DISPLAY_PORT:
 			pipes[pipe_cnt].dout.output_type = dm_dp;
-			if (link_is_dp_128b_132b_signal(&res_ctx->pipe_ctx[i]))
+			if (dc->link_srv->dp_is_128b_132b_signal(&res_ctx->pipe_ctx[i]))
 				pipes[pipe_cnt].dout.output_type = dm_dp2p0;
 			break;
 		case SIGNAL_TYPE_EDP:
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
index 4fa636364793..80972ee5e55b 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
@@ -384,9 +384,38 @@ void dcn30_fpu_calculate_wm_and_dlg(
 	int i, pipe_idx;
 	double dcfclk = context->bw_ctx.dml.vba.DCFCLKState[vlevel][maxMpcComb];
 	bool pstate_en = context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][maxMpcComb] != dm_dram_clock_change_unsupported;
+	unsigned int dummy_latency_index = 0;
 
 	dc_assert_fp_enabled();
 
+	context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching = false;
+    for (i = 0; i < context->stream_count; i++) {
+		if (context->streams[i])
+			context->streams[i]->fpo_in_use = false;
+	}
+
+	if (!pstate_en) {
+		/* only when the mclk switch can not be natural, is the fw based vblank stretch attempted */
+		context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching =
+			dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch(dc, context);
+
+		if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching) {
+			dummy_latency_index = dcn30_find_dummy_latency_index_for_fw_based_mclk_switch(dc,
+				context, pipes, pipe_cnt, vlevel);
+
+			/* After calling dcn30_find_dummy_latency_index_for_fw_based_mclk_switch
+			 * we reinstate the original dram_clock_change_latency_us on the context
+			 * and all variables that may have changed up to this point, except the
+			 * newly found dummy_latency_index
+			 */
+			context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.pstate_latency_us;
+			dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, false, true);
+			maxMpcComb = context->bw_ctx.dml.vba.maxMpcComb;
+			dcfclk = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
+			pstate_en = context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][maxMpcComb] != dm_dram_clock_change_unsupported;
+		}
+	}
+
 	if (context->bw_ctx.dml.soc.min_dcfclk > dcfclk)
 		dcfclk = context->bw_ctx.dml.soc.min_dcfclk;
 
@@ -449,15 +478,29 @@ void dcn30_fpu_calculate_wm_and_dlg(
 		unsigned int min_dram_speed_mts = context->bw_ctx.dml.vba.DRAMSpeed;
 		unsigned int min_dram_speed_mts_margin = 160;
 
-		if (context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb] == dm_dram_clock_change_unsupported)
-			min_dram_speed_mts = dc->clk_mgr->bw_params->clk_table.entries[dc->clk_mgr->bw_params->clk_table.num_entries - 1].memclk_mhz * 16;
+		context->bw_ctx.dml.soc.dram_clock_change_latency_us =
+			dc->clk_mgr->bw_params->dummy_pstate_table[0].dummy_pstate_latency_us;
+
+		if (context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][maxMpcComb] ==
+			dm_dram_clock_change_unsupported) {
+			int min_dram_speed_mts_offset = dc->clk_mgr->bw_params->clk_table.num_entries - 1;
+
+			min_dram_speed_mts =
+				dc->clk_mgr->bw_params->clk_table.entries[min_dram_speed_mts_offset].memclk_mhz * 16;
+		}
 
-		/* find largest table entry that is lower than dram speed, but lower than DPM0 still uses DPM0 */
-		for (i = 3; i > 0; i--)
-			if (min_dram_speed_mts + min_dram_speed_mts_margin > dc->clk_mgr->bw_params->dummy_pstate_table[i].dram_speed_mts)
-				break;
+		if (!context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching) {
+			/* find largest table entry that is lower than dram speed,
+			 * but lower than DPM0 still uses DPM0
+			 */
+			for (dummy_latency_index = 3; dummy_latency_index > 0; dummy_latency_index--)
+				if (min_dram_speed_mts + min_dram_speed_mts_margin >
+					dc->clk_mgr->bw_params->dummy_pstate_table[dummy_latency_index].dram_speed_mts)
+					break;
+		}
 
-		context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->dummy_pstate_table[i].dummy_pstate_latency_us;
+		context->bw_ctx.dml.soc.dram_clock_change_latency_us =
+			dc->clk_mgr->bw_params->dummy_pstate_table[dummy_latency_index].dummy_pstate_latency_us;
 
 		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_enter_plus_exit_time_us;
 		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_exit_time_us;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
index c3d75e56410c..7d0626e42ea6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
@@ -23,9 +23,7 @@
  *
  */
 
-#ifdef CONFIG_DRM_AMD_DC_DCN
 #include "dc.h"
-#include "dc_link.h"
 #include "../display_mode_lib.h"
 #include "display_mode_vba_30.h"
 #include "../dml_inline_defs.h"
@@ -4866,7 +4864,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							v->DETBufferSizeCThisState[k],
 							&v->UrgentBurstFactorCursorPre[k],
 							&v->UrgentBurstFactorLumaPre[k],
-							&v->UrgentBurstFactorChroma[k],
+							&v->UrgentBurstFactorChromaPre[k],
 							&v->NoUrgentLatencyHidingPre[k]);
 				}
 
@@ -6635,4 +6633,3 @@ static noinline_for_stack void UseMinimumDCFCLK(
 	}
 }
 
-#endif /* CONFIG_DRM_AMD_DC_DCN */
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c
index 8179be1f34bb..cd3cfcb2a2b0 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c
@@ -23,8 +23,6 @@
  *
  */
 
-#ifdef CONFIG_DRM_AMD_DC_DCN
-
 #include "../display_mode_lib.h"
 #include "../display_mode_vba.h"
 #include "../dml_inline_defs.h"
@@ -1792,4 +1790,3 @@ void dml30_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	dml_print("DML_DLG: Calculation for pipe[%d] end\n", pipe_idx);
 }
 
-#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
index b37d14369a62..59836570603a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
@@ -222,7 +222,7 @@ struct _vcs_dpi_ip_params_st dcn3_15_ip = {
 	.maximum_dsc_bits_per_component = 10,
 	.dsc422_native_support = false,
 	.is_line_buffer_bpp_fixed = true,
-	.line_buffer_fixed_bpp = 49,
+	.line_buffer_fixed_bpp = 48,
 	.line_buffer_size_bits = 789504,
 	.max_line_buffer_lines = 12,
 	.writeback_interface_buffer_size_kbytes = 90,
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
index 27f488405335..bd674dc30df3 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
@@ -24,7 +24,6 @@
  */
 
 #include "dc.h"
-#include "dc_link.h"
 #include "../display_mode_lib.h"
 #include "../dcn30/display_mode_vba_30.h"
 #include "display_mode_vba_31.h"
@@ -4308,11 +4307,11 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							v->AudioSampleRate[k],
 							v->AudioSampleLayout[k],
 							v->ODMCombineEnablePerState[i][k]);
-				} else if (v->Output[k] == dm_dp || v->Output[k] == dm_edp) {
+				} else if (v->Output[k] == dm_dp || v->Output[k] == dm_edp || v->Output[k] == dm_dp2p0) {
 					if (v->DSCEnable[k] == true) {
 						v->RequiresDSC[i][k] = true;
 						v->LinkDSCEnable = true;
-						if (v->Output[k] == dm_dp) {
+						if (v->Output[k] == dm_dp || v->Output[k] == dm_dp2p0) {
 							v->RequiresFEC[i][k] = true;
 						} else {
 							v->RequiresFEC[i][k] = false;
@@ -4320,107 +4319,201 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					} else {
 						v->RequiresDSC[i][k] = false;
 						v->LinkDSCEnable = false;
-						v->RequiresFEC[i][k] = false;
-					}
-
-					v->Outbpp = BPP_INVALID;
-					if (v->PHYCLKPerState[i] >= 270.0) {
-						v->Outbpp = TruncToValidBPP(
-								(1.0 - v->Downspreading / 100.0) * 2700,
-								v->OutputLinkDPLanes[k],
-								v->HTotal[k],
-								v->HActive[k],
-								v->PixelClockBackEnd[k],
-								v->ForcedOutputLinkBPP[k],
-								v->LinkDSCEnable,
-								v->Output[k],
-								v->OutputFormat[k],
-								v->DSCInputBitPerComponent[k],
-								v->NumberOfDSCSlices[k],
-								v->AudioSampleRate[k],
-								v->AudioSampleLayout[k],
-								v->ODMCombineEnablePerState[i][k]);
-						v->OutputBppPerState[i][k] = v->Outbpp;
-						// TODO: Need some other way to handle this nonsense
-						// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR"
-					}
-					if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 540.0) {
-						v->Outbpp = TruncToValidBPP(
-								(1.0 - v->Downspreading / 100.0) * 5400,
-								v->OutputLinkDPLanes[k],
-								v->HTotal[k],
-								v->HActive[k],
-								v->PixelClockBackEnd[k],
-								v->ForcedOutputLinkBPP[k],
-								v->LinkDSCEnable,
-								v->Output[k],
-								v->OutputFormat[k],
-								v->DSCInputBitPerComponent[k],
-								v->NumberOfDSCSlices[k],
-								v->AudioSampleRate[k],
-								v->AudioSampleLayout[k],
-								v->ODMCombineEnablePerState[i][k]);
-						v->OutputBppPerState[i][k] = v->Outbpp;
-						// TODO: Need some other way to handle this nonsense
-						// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR2"
-					}
-					if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 810.0) {
-						v->Outbpp = TruncToValidBPP(
-								(1.0 - v->Downspreading / 100.0) * 8100,
-								v->OutputLinkDPLanes[k],
-								v->HTotal[k],
-								v->HActive[k],
-								v->PixelClockBackEnd[k],
-								v->ForcedOutputLinkBPP[k],
-								v->LinkDSCEnable,
-								v->Output[k],
-								v->OutputFormat[k],
-								v->DSCInputBitPerComponent[k],
-								v->NumberOfDSCSlices[k],
-								v->AudioSampleRate[k],
-								v->AudioSampleLayout[k],
-								v->ODMCombineEnablePerState[i][k]);
-						v->OutputBppPerState[i][k] = v->Outbpp;
-						// TODO: Need some other way to handle this nonsense
-						// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR3"
-					}
-					if (v->Outbpp == BPP_INVALID && v->PHYCLKD18PerState[i] >= 10000.0 / 18) {
-						v->Outbpp = TruncToValidBPP(
-								(1.0 - v->Downspreading / 100.0) * 10000,
-								4,
-								v->HTotal[k],
-								v->HActive[k],
-								v->PixelClockBackEnd[k],
-								v->ForcedOutputLinkBPP[k],
-								v->LinkDSCEnable,
-								v->Output[k],
-								v->OutputFormat[k],
-								v->DSCInputBitPerComponent[k],
-								v->NumberOfDSCSlices[k],
-								v->AudioSampleRate[k],
-								v->AudioSampleLayout[k],
-								v->ODMCombineEnablePerState[i][k]);
-						v->OutputBppPerState[i][k] = v->Outbpp;
-						//v->OutputTypeAndRatePerState[i][k] = v->Output[k] & "10x4";
+						if (v->Output[k] == dm_dp2p0) {
+							v->RequiresFEC[i][k] = true;
+						} else {
+							v->RequiresFEC[i][k] = false;
+						}
 					}
-					if (v->Outbpp == BPP_INVALID && v->PHYCLKD18PerState[i] >= 12000.0 / 18) {
-						v->Outbpp = TruncToValidBPP(
-								12000,
-								4,
-								v->HTotal[k],
-								v->HActive[k],
-								v->PixelClockBackEnd[k],
-								v->ForcedOutputLinkBPP[k],
-								v->LinkDSCEnable,
-								v->Output[k],
-								v->OutputFormat[k],
-								v->DSCInputBitPerComponent[k],
-								v->NumberOfDSCSlices[k],
-								v->AudioSampleRate[k],
-								v->AudioSampleLayout[k],
-								v->ODMCombineEnablePerState[i][k]);
-						v->OutputBppPerState[i][k] = v->Outbpp;
-						//v->OutputTypeAndRatePerState[i][k] = v->Output[k] & "12x4";
+					if (v->Output[k] == dm_dp2p0) {
+						v->Outbpp = BPP_INVALID;
+						if ((v->OutputLinkDPRate[k] == dm_dp_rate_na || v->OutputLinkDPRate[k] == dm_dp_rate_uhbr10) &&
+							v->PHYCLKD18PerState[k] >= 10000.0 / 18.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 10000,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							if (v->Outbpp == BPP_INVALID && v->PHYCLKD18PerState[k] < 13500.0 / 18.0 &&
+								v->DSCEnable[k] == true && v->ForcedOutputLinkBPP[k] == 0) {
+								v->RequiresDSC[i][k] = true;
+								v->LinkDSCEnable = true;
+								v->Outbpp = TruncToValidBPP(
+										(1.0 - v->Downspreading / 100.0) * 10000,
+										v->OutputLinkDPLanes[k],
+										v->HTotal[k],
+										v->HActive[k],
+										v->PixelClockBackEnd[k],
+										v->ForcedOutputLinkBPP[k],
+										v->LinkDSCEnable,
+										v->Output[k],
+										v->OutputFormat[k],
+										v->DSCInputBitPerComponent[k],
+										v->NumberOfDSCSlices[k],
+										v->AudioSampleRate[k],
+										v->AudioSampleLayout[k],
+										v->ODMCombineEnablePerState[i][k]);
+							}
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " UHBR10"
+						}
+						if (v->Outbpp == BPP_INVALID &&
+							(v->OutputLinkDPRate[k] == dm_dp_rate_na || v->OutputLinkDPRate[k] == dm_dp_rate_uhbr13p5) &&
+							v->PHYCLKD18PerState[k] >= 13500.0 / 18.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 13500,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							if (v->Outbpp == BPP_INVALID && v->PHYCLKD18PerState[k] < 20000.0 / 18.0 &&
+								v->DSCEnable[k] == true && v->ForcedOutputLinkBPP[k] == 0) {
+								v->RequiresDSC[i][k] = true;
+								v->LinkDSCEnable = true;
+								v->Outbpp = TruncToValidBPP(
+										(1.0 - v->Downspreading / 100.0) * 13500,
+										v->OutputLinkDPLanes[k],
+										v->HTotal[k],
+										v->HActive[k],
+										v->PixelClockBackEnd[k],
+										v->ForcedOutputLinkBPP[k],
+										v->LinkDSCEnable,
+										v->Output[k],
+										v->OutputFormat[k],
+										v->DSCInputBitPerComponent[k],
+										v->NumberOfDSCSlices[k],
+										v->AudioSampleRate[k],
+										v->AudioSampleLayout[k],
+										v->ODMCombineEnablePerState[i][k]);
+							}
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " UHBR13p5"
+						}
+						if (v->Outbpp == BPP_INVALID &&
+							(v->OutputLinkDPRate[k] == dm_dp_rate_na || v->OutputLinkDPRate[k] == dm_dp_rate_uhbr20) &&
+							v->PHYCLKD18PerState[k] >= 20000.0 / 18.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 20000,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							if (v->Outbpp == BPP_INVALID && v->DSCEnable[k] == true &&
+								v->ForcedOutputLinkBPP[k] == 0) {
+								v->RequiresDSC[i][k] = true;
+								v->LinkDSCEnable = true;
+								v->Outbpp = TruncToValidBPP(
+										(1.0 - v->Downspreading / 100.0) * 20000,
+										v->OutputLinkDPLanes[k],
+										v->HTotal[k],
+										v->HActive[k],
+										v->PixelClockBackEnd[k],
+										v->ForcedOutputLinkBPP[k],
+										v->LinkDSCEnable,
+										v->Output[k],
+										v->OutputFormat[k],
+										v->DSCInputBitPerComponent[k],
+										v->NumberOfDSCSlices[k],
+										v->AudioSampleRate[k],
+										v->AudioSampleLayout[k],
+										v->ODMCombineEnablePerState[i][k]);
+							}
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " UHBR20"
+						}
+					} else {
+						v->Outbpp = BPP_INVALID;
+						if (v->PHYCLKPerState[i] >= 270.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 2700,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR"
+						}
+						if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 540.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 5400,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR2"
+						}
+						if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 810.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 8100,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR3"
+						}
 					}
 				}
 			} else {
@@ -5098,7 +5191,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							v->DETBufferSizeCThisState[k],
 							&v->UrgentBurstFactorCursorPre[k],
 							&v->UrgentBurstFactorLumaPre[k],
-							&v->UrgentBurstFactorChroma[k],
+							&v->UrgentBurstFactorChromaPre[k],
 							&v->NotUrgentLatencyHidingPre[k]);
 				}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c
index 35d10b4d018b..2244e4fb8c96 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c
@@ -902,7 +902,6 @@ static void dml_rq_dlg_get_dlg_params(
 	double hratio_c;
 	double vratio_l;
 	double vratio_c;
-	bool scl_enable;
 
 	unsigned int swath_width_ub_l;
 	unsigned int dpte_groups_per_row_ub_l;
@@ -1020,7 +1019,6 @@ static void dml_rq_dlg_get_dlg_params(
 	hratio_c = scl->hscl_ratio_c;
 	vratio_l = scl->vscl_ratio;
 	vratio_c = scl->vscl_ratio_c;
-	scl_enable = scl->scl_enable;
 
 	swath_width_ub_l = rq_dlg_param->rq_l.swath_width_ub;
 	dpte_groups_per_row_ub_l = rq_dlg_param->rq_l.dpte_groups_per_row_ub;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
index acda3e1babd4..44082f65de1f 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
@@ -97,7 +97,7 @@ struct _vcs_dpi_ip_params_st dcn3_14_ip = {
 	.dcc_supported = true,
 };
 
-struct _vcs_dpi_soc_bounding_box_st dcn3_14_soc = {
+static struct _vcs_dpi_soc_bounding_box_st dcn3_14_soc = {
 		/*TODO: correct dispclk/dppclk voltage level determination*/
 	.clock_limits = {
 		{
@@ -308,6 +308,10 @@ int dcn314_populate_dml_pipes_from_context_fpu(struct dc *dc, struct dc_state *c
 				pipe->plane_state->src_rect.width < pipe->plane_state->dst_rect.width))
 			upscaled = true;
 
+		/* Apply HostVM policy - either based on hypervisor globally enabled, or rIOMMU active */
+		if (dc->debug.dml_hostvm_override == DML_HOSTVM_NO_OVERRIDE)
+			pipes[i].pipe.src.hostvm = dc->vm_pa_config.is_hvm_enabled || dc->res_pool->hubbub->riommu_active;
+
 		/*
 		 * Immediate flip can be set dynamically after enabling the plane.
 		 * We need to require support for immediate flip or underflow can be
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
index c843b394aeb4..7eb2173b7691 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
@@ -27,7 +27,6 @@
 #define UNIT_TEST 0
 #if !UNIT_TEST
 #include "dc.h"
-#include "dc_link.h"
 #endif
 #include "../display_mode_lib.h"
 #include "display_mode_vba_314.h"
@@ -4406,11 +4405,11 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 							v->AudioSampleRate[k],
 							v->AudioSampleLayout[k],
 							v->ODMCombineEnablePerState[i][k]);
-				} else if (v->Output[k] == dm_dp || v->Output[k] == dm_edp) {
+				} else if (v->Output[k] == dm_dp || v->Output[k] == dm_edp || v->Output[k] == dm_dp2p0) {
 					if (v->DSCEnable[k] == true) {
 						v->RequiresDSC[i][k] = true;
 						v->LinkDSCEnable = true;
-						if (v->Output[k] == dm_dp) {
+						if (v->Output[k] == dm_dp || v->Output[k] == dm_dp2p0) {
 							v->RequiresFEC[i][k] = true;
 						} else {
 							v->RequiresFEC[i][k] = false;
@@ -4418,107 +4417,201 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 					} else {
 						v->RequiresDSC[i][k] = false;
 						v->LinkDSCEnable = false;
-						v->RequiresFEC[i][k] = false;
-					}
-
-					v->Outbpp = BPP_INVALID;
-					if (v->PHYCLKPerState[i] >= 270.0) {
-						v->Outbpp = TruncToValidBPP(
-								(1.0 - v->Downspreading / 100.0) * 2700,
-								v->OutputLinkDPLanes[k],
-								v->HTotal[k],
-								v->HActive[k],
-								v->PixelClockBackEnd[k],
-								v->ForcedOutputLinkBPP[k],
-								v->LinkDSCEnable,
-								v->Output[k],
-								v->OutputFormat[k],
-								v->DSCInputBitPerComponent[k],
-								v->NumberOfDSCSlices[k],
-								v->AudioSampleRate[k],
-								v->AudioSampleLayout[k],
-								v->ODMCombineEnablePerState[i][k]);
-						v->OutputBppPerState[i][k] = v->Outbpp;
-						// TODO: Need some other way to handle this nonsense
-						// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR"
-					}
-					if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 540.0) {
-						v->Outbpp = TruncToValidBPP(
-								(1.0 - v->Downspreading / 100.0) * 5400,
-								v->OutputLinkDPLanes[k],
-								v->HTotal[k],
-								v->HActive[k],
-								v->PixelClockBackEnd[k],
-								v->ForcedOutputLinkBPP[k],
-								v->LinkDSCEnable,
-								v->Output[k],
-								v->OutputFormat[k],
-								v->DSCInputBitPerComponent[k],
-								v->NumberOfDSCSlices[k],
-								v->AudioSampleRate[k],
-								v->AudioSampleLayout[k],
-								v->ODMCombineEnablePerState[i][k]);
-						v->OutputBppPerState[i][k] = v->Outbpp;
-						// TODO: Need some other way to handle this nonsense
-						// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR2"
-					}
-					if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 810.0) {
-						v->Outbpp = TruncToValidBPP(
-								(1.0 - v->Downspreading / 100.0) * 8100,
-								v->OutputLinkDPLanes[k],
-								v->HTotal[k],
-								v->HActive[k],
-								v->PixelClockBackEnd[k],
-								v->ForcedOutputLinkBPP[k],
-								v->LinkDSCEnable,
-								v->Output[k],
-								v->OutputFormat[k],
-								v->DSCInputBitPerComponent[k],
-								v->NumberOfDSCSlices[k],
-								v->AudioSampleRate[k],
-								v->AudioSampleLayout[k],
-								v->ODMCombineEnablePerState[i][k]);
-						v->OutputBppPerState[i][k] = v->Outbpp;
-						// TODO: Need some other way to handle this nonsense
-						// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR3"
-					}
-					if (v->Outbpp == BPP_INVALID && v->PHYCLKD18PerState[i] >= 10000.0 / 18) {
-						v->Outbpp = TruncToValidBPP(
-								(1.0 - v->Downspreading / 100.0) * 10000,
-								4,
-								v->HTotal[k],
-								v->HActive[k],
-								v->PixelClockBackEnd[k],
-								v->ForcedOutputLinkBPP[k],
-								v->LinkDSCEnable,
-								v->Output[k],
-								v->OutputFormat[k],
-								v->DSCInputBitPerComponent[k],
-								v->NumberOfDSCSlices[k],
-								v->AudioSampleRate[k],
-								v->AudioSampleLayout[k],
-								v->ODMCombineEnablePerState[i][k]);
-						v->OutputBppPerState[i][k] = v->Outbpp;
-						//v->OutputTypeAndRatePerState[i][k] = v->Output[k] & "10x4";
+						if (v->Output[k] == dm_dp2p0) {
+							v->RequiresFEC[i][k] = true;
+						} else {
+							v->RequiresFEC[i][k] = false;
+						}
 					}
-					if (v->Outbpp == BPP_INVALID && v->PHYCLKD18PerState[i] >= 12000.0 / 18) {
-						v->Outbpp = TruncToValidBPP(
-								12000,
-								4,
-								v->HTotal[k],
-								v->HActive[k],
-								v->PixelClockBackEnd[k],
-								v->ForcedOutputLinkBPP[k],
-								v->LinkDSCEnable,
-								v->Output[k],
-								v->OutputFormat[k],
-								v->DSCInputBitPerComponent[k],
-								v->NumberOfDSCSlices[k],
-								v->AudioSampleRate[k],
-								v->AudioSampleLayout[k],
-								v->ODMCombineEnablePerState[i][k]);
-						v->OutputBppPerState[i][k] = v->Outbpp;
-						//v->OutputTypeAndRatePerState[i][k] = v->Output[k] & "12x4";
+					if (v->Output[k] == dm_dp2p0) {
+						v->Outbpp = BPP_INVALID;
+						if ((v->OutputLinkDPRate[k] == dm_dp_rate_na || v->OutputLinkDPRate[k] == dm_dp_rate_uhbr10) &&
+							v->PHYCLKD18PerState[k] >= 10000.0 / 18.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 10000,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							if (v->Outbpp == BPP_INVALID && v->PHYCLKD18PerState[k] < 13500.0 / 18.0 &&
+								v->DSCEnable[k] == true && v->ForcedOutputLinkBPP[k] == 0) {
+								v->RequiresDSC[i][k] = true;
+								v->LinkDSCEnable = true;
+								v->Outbpp = TruncToValidBPP(
+										(1.0 - v->Downspreading / 100.0) * 10000,
+										v->OutputLinkDPLanes[k],
+										v->HTotal[k],
+										v->HActive[k],
+										v->PixelClockBackEnd[k],
+										v->ForcedOutputLinkBPP[k],
+										v->LinkDSCEnable,
+										v->Output[k],
+										v->OutputFormat[k],
+										v->DSCInputBitPerComponent[k],
+										v->NumberOfDSCSlices[k],
+										v->AudioSampleRate[k],
+										v->AudioSampleLayout[k],
+										v->ODMCombineEnablePerState[i][k]);
+							}
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " UHBR10"
+						}
+						if (v->Outbpp == BPP_INVALID &&
+							(v->OutputLinkDPRate[k] == dm_dp_rate_na || v->OutputLinkDPRate[k] == dm_dp_rate_uhbr13p5) &&
+							v->PHYCLKD18PerState[k] >= 13500.0 / 18.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 13500,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							if (v->Outbpp == BPP_INVALID && v->PHYCLKD18PerState[k] < 20000.0 / 18.0 &&
+								v->DSCEnable[k] == true && v->ForcedOutputLinkBPP[k] == 0) {
+								v->RequiresDSC[i][k] = true;
+								v->LinkDSCEnable = true;
+								v->Outbpp = TruncToValidBPP(
+										(1.0 - v->Downspreading / 100.0) * 13500,
+										v->OutputLinkDPLanes[k],
+										v->HTotal[k],
+										v->HActive[k],
+										v->PixelClockBackEnd[k],
+										v->ForcedOutputLinkBPP[k],
+										v->LinkDSCEnable,
+										v->Output[k],
+										v->OutputFormat[k],
+										v->DSCInputBitPerComponent[k],
+										v->NumberOfDSCSlices[k],
+										v->AudioSampleRate[k],
+										v->AudioSampleLayout[k],
+										v->ODMCombineEnablePerState[i][k]);
+							}
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " UHBR13p5"
+						}
+						if (v->Outbpp == BPP_INVALID &&
+							(v->OutputLinkDPRate[k] == dm_dp_rate_na || v->OutputLinkDPRate[k] == dm_dp_rate_uhbr20) &&
+							v->PHYCLKD18PerState[k] >= 20000.0 / 18.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 20000,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							if (v->Outbpp == BPP_INVALID && v->DSCEnable[k] == true &&
+								v->ForcedOutputLinkBPP[k] == 0) {
+								v->RequiresDSC[i][k] = true;
+								v->LinkDSCEnable = true;
+								v->Outbpp = TruncToValidBPP(
+										(1.0 - v->Downspreading / 100.0) * 20000,
+										v->OutputLinkDPLanes[k],
+										v->HTotal[k],
+										v->HActive[k],
+										v->PixelClockBackEnd[k],
+										v->ForcedOutputLinkBPP[k],
+										v->LinkDSCEnable,
+										v->Output[k],
+										v->OutputFormat[k],
+										v->DSCInputBitPerComponent[k],
+										v->NumberOfDSCSlices[k],
+										v->AudioSampleRate[k],
+										v->AudioSampleLayout[k],
+										v->ODMCombineEnablePerState[i][k]);
+							}
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " UHBR20"
+						}
+					} else {
+						v->Outbpp = BPP_INVALID;
+						if (v->PHYCLKPerState[i] >= 270.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 2700,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR"
+						}
+						if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 540.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 5400,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR2"
+						}
+						if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 810.0) {
+							v->Outbpp = TruncToValidBPP(
+									(1.0 - v->Downspreading / 100.0) * 8100,
+									v->OutputLinkDPLanes[k],
+									v->HTotal[k],
+									v->HActive[k],
+									v->PixelClockBackEnd[k],
+									v->ForcedOutputLinkBPP[k],
+									v->LinkDSCEnable,
+									v->Output[k],
+									v->OutputFormat[k],
+									v->DSCInputBitPerComponent[k],
+									v->NumberOfDSCSlices[k],
+									v->AudioSampleRate[k],
+									v->AudioSampleLayout[k],
+									v->ODMCombineEnablePerState[i][k]);
+							v->OutputBppPerState[i][k] = v->Outbpp;
+							// TODO: Need some other way to handle this nonsense
+							// v->OutputTypeAndRatePerState[i][k] = v->Output[k] & " HBR3"
+						}
 					}
 				}
 			} else {
@@ -5195,7 +5288,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 							v->DETBufferSizeCThisState[k],
 							&v->UrgentBurstFactorCursorPre[k],
 							&v->UrgentBurstFactorLumaPre[k],
-							&v->UrgentBurstFactorChroma[k],
+							&v->UrgentBurstFactorChromaPre[k],
 							&v->NotUrgentLatencyHidingPre[k]);
 				}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_rq_dlg_calc_314.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_rq_dlg_calc_314.c
index 6576b897a512..ea4eb66066c4 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_rq_dlg_calc_314.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_rq_dlg_calc_314.c
@@ -988,7 +988,6 @@ static void dml_rq_dlg_get_dlg_params(
 	double hratio_c;
 	double vratio_l;
 	double vratio_c;
-	bool scl_enable;
 
 	unsigned int swath_width_ub_l;
 	unsigned int dpte_groups_per_row_ub_l;
@@ -1001,7 +1000,6 @@ static void dml_rq_dlg_get_dlg_params(
 	unsigned int vupdate_width;
 	unsigned int vready_offset;
 
-	unsigned int dppclk_delay_subtotal;
 	unsigned int dispclk_delay_subtotal;
 
 	unsigned int vstartup_start;
@@ -1118,7 +1116,6 @@ static void dml_rq_dlg_get_dlg_params(
 	hratio_c = scl->hscl_ratio_c;
 	vratio_l = scl->vscl_ratio;
 	vratio_c = scl->vscl_ratio_c;
-	scl_enable = scl->scl_enable;
 
 	swath_width_ub_l = rq_dlg_param->rq_l.swath_width_ub;
 	dpte_groups_per_row_ub_l = rq_dlg_param->rq_l.dpte_groups_per_row_ub;
@@ -1130,17 +1127,8 @@ static void dml_rq_dlg_get_dlg_params(
 	vupdate_offset = dst->vupdate_offset;
 	vupdate_width = dst->vupdate_width;
 	vready_offset = dst->vready_offset;
-
-	dppclk_delay_subtotal = mode_lib->ip.dppclk_delay_subtotal;
 	dispclk_delay_subtotal = mode_lib->ip.dispclk_delay_subtotal;
 
-	if (scl_enable)
-		dppclk_delay_subtotal += mode_lib->ip.dppclk_delay_scl;
-	else
-		dppclk_delay_subtotal += mode_lib->ip.dppclk_delay_scl_lb_only;
-
-	dppclk_delay_subtotal += mode_lib->ip.dppclk_delay_cnvc_formatter + src->num_cursors * mode_lib->ip.dppclk_delay_cnvc_cursor;
-
 	if (dout->dsc_enable) {
 		double dsc_delay = get_dsc_delay(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // FROM VBA
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
index e47828e3b6d5..4548320217fc 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
@@ -131,7 +131,7 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_2_soc = {
 	.urgent_latency_pixel_data_only_us = 4.0,
 	.urgent_latency_pixel_mixed_with_vm_data_us = 4.0,
 	.urgent_latency_vm_data_only_us = 4.0,
-	.fclk_change_latency_us = 20,
+	.fclk_change_latency_us = 25,
 	.usr_retraining_latency_us = 2,
 	.smn_latency_us = 2,
 	.mall_allocated_for_dcn_mbytes = 64,
@@ -1270,7 +1270,7 @@ static bool is_dtbclk_required(struct dc *dc, struct dc_state *context)
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		if (!context->res_ctx.pipe_ctx[i].stream)
 			continue;
-		if (link_is_dp_128b_132b_signal(&context->res_ctx.pipe_ctx[i]))
+		if (dc->link_srv->dp_is_128b_132b_signal(&context->res_ctx.pipe_ctx[i]))
 			return true;
 	}
 	return false;
@@ -1927,6 +1927,7 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 	unsigned int min_dram_speed_mts_margin;
 	bool need_fclk_lat_as_dummy = false;
 	bool is_subvp_p_drr = false;
+	struct dc_stream_state *fpo_candidate_stream = NULL;
 
 	dc_assert_fp_enabled();
 
@@ -1960,11 +1961,19 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 	}
 
 	context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching = false;
+	for (i = 0; i < context->stream_count; i++) {
+		if (context->streams[i])
+			context->streams[i]->fpo_in_use = false;
+	}
 
-	if (!pstate_en) {
+	if (!pstate_en || (!dc->debug.disable_fpo_optimizations &&
+			pstate_en && vlevel != 0)) {
 		/* only when the mclk switch can not be natural, is the fw based vblank stretch attempted */
-		context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching =
-			dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch(dc, context);
+		fpo_candidate_stream = dcn32_can_support_mclk_switch_using_fw_based_vblank_stretch(dc, context);
+		if (fpo_candidate_stream) {
+			fpo_candidate_stream->fpo_in_use = true;
+			context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching = true;
+		}
 
 		if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching) {
 			dummy_latency_index = dcn32_find_dummy_latency_index_for_fw_based_mclk_switch(dc,
@@ -1985,11 +1994,25 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 				context->bw_ctx.dml.soc.fclk_change_latency_us =
 						dc->clk_mgr->bw_params->dummy_pstate_table[dummy_latency_index].dummy_pstate_latency_us;
 			}
-			dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, false);
-			maxMpcComb = context->bw_ctx.dml.vba.maxMpcComb;
-			dcfclk_from_fw_based_mclk_switching = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
-			pstate_en = context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][maxMpcComb] !=
-					dm_dram_clock_change_unsupported;
+			dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel_temp, false);
+			if (vlevel_temp < vlevel) {
+				vlevel = vlevel_temp;
+				maxMpcComb = context->bw_ctx.dml.vba.maxMpcComb;
+				dcfclk_from_fw_based_mclk_switching = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
+				pstate_en = true;
+			} else {
+				/* Restore FCLK latency and re-run validation to go back to original validation
+				 * output if we find that enabling FPO does not give us any benefit (i.e. lower
+				 * voltage level)
+				 */
+				context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching = false;
+				for (i = 0; i < context->stream_count; i++) {
+					if (context->streams[i])
+						context->streams[i]->fpo_in_use = false;
+				}
+				context->bw_ctx.dml.soc.fclk_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.fclk_change_latency_us;
+				dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, false);
+			}
 		}
 	}
 
@@ -2142,7 +2165,13 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 		 * DCFCLK: Min, as reported by PM FW, when available
 		 * UCLK: Min, as reported by PM FW, when available
 		 */
-		dc->res_pool->funcs->update_soc_for_wm_a(dc, context);
+
+		/* For set A set the correct latency values (i.e. non-dummy values) unconditionally
+		 */
+		context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.pstate_latency_us;
+		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.sr_enter_plus_exit_time_us;
+		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.sr_exit_time_us;
+
 		context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
 		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
 		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
@@ -2188,10 +2217,6 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 		context->bw_ctx.dml.soc.dram_clock_change_latency_us =
 				dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.pstate_latency_us;
 
-	if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching) {
-		dcn30_setup_mclk_switch_using_fw_based_vblank_stretch(dc, context);
-	}
-
 	/* revert fclk lat changes if required */
 	if (need_fclk_lat_as_dummy)
 		context->bw_ctx.dml.soc.fclk_change_latency_us =
@@ -2315,6 +2340,9 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params,
 			num_dcfclk_dpms++;
 	}
 
+	if (num_dcfclk_dpms > 0 && bw_params->clk_table.entries[0].fclk_mhz > min_fclk_mhz)
+		min_fclk_mhz = bw_params->clk_table.entries[0].fclk_mhz;
+
 	if (!max_dcfclk_mhz || !max_dispclk_mhz || !max_dtbclk_mhz)
 		return -1;
 
@@ -2423,7 +2451,6 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params,
 		for (i = *num_entries - 1; i >= 0 ; i--) {
 			if (table[i].fabricclk_mhz < min_fclk_mhz) {
 				table[i].fabricclk_mhz = min_fclk_mhz;
-				break;
 			}
 		}
 	}
@@ -2432,7 +2459,6 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params,
 	for (i = *num_entries - 1; i >= 0 ; i--) {
 		if (table[i].dcfclk_mhz < min_dcfclk_mhz) {
 			table[i].dcfclk_mhz = min_dcfclk_mhz;
-			break;
 		}
 	}
 
@@ -2780,3 +2806,68 @@ double dcn32_determine_max_vratio_prefetch(struct dc *dc, struct dc_state *conte
 	}
 	return max_vratio_pre;
 }
+
+/**
+ * dcn32_assign_fpo_vactive_candidate - Assign the FPO stream candidate for FPO + VActive case
+ *
+ * This function chooses the FPO candidate stream for FPO + VActive cases (2 stream config).
+ * For FPO + VAtive cases, the assumption is that one display has ActiveMargin > 0, and the
+ * other display has ActiveMargin <= 0. This function will choose the pipe/stream that has
+ * ActiveMargin <= 0 to be the FPO stream candidate if found.
+ *
+ *
+ * @param [in]: dc - current dc state
+ * @param [in]: context - new dc state
+ * @param [out]: fpo_candidate_stream - pointer to FPO stream candidate if one is found
+ *
+ * Return: void
+ */
+void dcn32_assign_fpo_vactive_candidate(struct dc *dc, const struct dc_state *context, struct dc_stream_state **fpo_candidate_stream)
+{
+	unsigned int i, pipe_idx;
+	const struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
+
+	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+		const struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (!pipe->stream)
+			continue;
+
+		if (vba->ActiveDRAMClockChangeLatencyMarginPerState[vba->VoltageLevel][vba->maxMpcComb][vba->pipe_plane[pipe_idx]] <= 0) {
+			*fpo_candidate_stream = pipe->stream;
+			break;
+		}
+		pipe_idx++;
+	}
+}
+
+/**
+ * dcn32_find_vactive_pipe - Determines if the config has a pipe that can switch in VACTIVE
+ *
+ * @param [in]: dc - current dc state
+ * @param [in]: context - new dc state
+ * @param [in]: vactive_margin_req_us - The vactive marign required for a vactive pipe to be
+ *                                      considered "found"
+ *
+ * Return: True if VACTIVE display is found, false otherwise
+ */
+bool dcn32_find_vactive_pipe(struct dc *dc, const struct dc_state *context, uint32_t vactive_margin_req_us)
+{
+	unsigned int i, pipe_idx;
+	const struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
+	bool vactive_found = false;
+
+	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+		const struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (!pipe->stream)
+			continue;
+
+		if (vba->ActiveDRAMClockChangeLatencyMarginPerState[vba->VoltageLevel][vba->maxMpcComb][vba->pipe_plane[pipe_idx]] >= vactive_margin_req_us) {
+			vactive_found = true;
+			break;
+		}
+		pipe_idx++;
+	}
+	return vactive_found;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h
index ab010e7e840b..9a0806a0e2ef 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h
@@ -76,4 +76,8 @@ void dcn32_patch_dpm_table(struct clk_bw_params *bw_params);
 void dcn32_zero_pipe_dcc_fraction(display_e2e_pipe_params_st *pipes,
 				  int pipe_cnt);
 
+void dcn32_assign_fpo_vactive_candidate(struct dc *dc, const struct dc_state *context, struct dc_stream_state **fpo_candidate_stream);
+
+bool dcn32_find_vactive_pipe(struct dc *dc, const struct dc_state *context, uint32_t vactive_margin_req);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c
index 3b2a014ccf8f..13c7e7394b1c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c
@@ -24,7 +24,6 @@
  */
 
 #include "dc.h"
-#include "dc_link.h"
 #include "../display_mode_lib.h"
 #include "display_mode_vba_32.h"
 #include "../dml_inline_defs.h"
@@ -690,7 +689,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 						mode_lib->vba.PixelClock,
 						mode_lib->vba.VRatio,
 						mode_lib->vba.VRatioChroma,
-						mode_lib->vba.UsesMALLForPStateChange);
+						mode_lib->vba.UsesMALLForPStateChange,
+						mode_lib->vba.UseUnboundedRequesting);
 
 	for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) {
 		v->MaxVStartupLines[k] = ((mode_lib->vba.Interlace[k] &&
@@ -3216,7 +3216,8 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					mode_lib->vba.PixelClock,
 					mode_lib->vba.VRatio,
 					mode_lib->vba.VRatioChroma,
-					mode_lib->vba.UsesMALLForPStateChange);
+					mode_lib->vba.UsesMALLForPStateChange,
+					mode_lib->vba.UseUnboundedRequesting);
 
 			v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.VMDataOnlyReturnBWPerState = dml32_get_return_bw_mbps_vm_only(&mode_lib->vba.soc, i,
 					mode_lib->vba.DCFCLKState[i][j], mode_lib->vba.FabricClockPerState[i],
@@ -3353,7 +3354,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							/* Output */
 							&mode_lib->vba.UrgentBurstFactorCursorPre[k],
 							&mode_lib->vba.UrgentBurstFactorLumaPre[k],
-							&mode_lib->vba.UrgentBurstFactorChroma[k],
+							&mode_lib->vba.UrgentBurstFactorChromaPre[k],
 							&mode_lib->vba.NotUrgentLatencyHidingPre[k]);
 				}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
index d1000aa4c481..61cc4904ade4 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
@@ -6271,7 +6271,8 @@ bool dml32_CalculateDETSwathFillLatencyHiding(unsigned int NumberOfActiveSurface
 		double	PixelClock[],
 		double	VRatioY[],
 		double	VRatioC[],
-		enum dm_use_mall_for_pstate_change_mode UsesMALLForPStateChange[])
+		enum dm_use_mall_for_pstate_change_mode UsesMALLForPStateChange[],
+		enum unbounded_requesting_policy UseUnboundedRequesting)
 {
 	int k;
 	double SwathSizeAllSurfaces = 0;
@@ -6283,6 +6284,9 @@ bool dml32_CalculateDETSwathFillLatencyHiding(unsigned int NumberOfActiveSurface
 	double SwathSizePerSurfaceC[DC__NUM_DPP__MAX];
 	bool NotEnoughDETSwathFillLatencyHiding = false;
 
+	if (UseUnboundedRequesting == dm_unbounded_requesting)
+		return false;
+
 	/* calculate sum of single swath size for all pipes in bytes */
 	for (k = 0; k < NumberOfActiveSurfaces; k++) {
 		SwathSizePerSurfaceY[k] = SwathHeightY[k] * SwathWidthY[k] * BytePerPixelInDETY[k] * NumOfDPP[k];
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h
index 9ba792c633a5..592d174df6c6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h
@@ -1163,6 +1163,7 @@ bool dml32_CalculateDETSwathFillLatencyHiding(unsigned int NumberOfActiveSurface
 		double	PixelClock[],
 		double	VRatioY[],
 		double	VRatioC[],
-		enum dm_use_mall_for_pstate_change_mode UsesMALLForPStateChange[]);
+		enum dm_use_mall_for_pstate_change_mode UsesMALLForPStateChange[],
+		enum unbounded_requesting_policy UseUnboundedRequesting);
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c
index b80cef70fa60..57b9bd896678 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c
@@ -294,6 +294,9 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params,
 			num_dcfclk_dpms++;
 	}
 
+	if (num_dcfclk_dpms > 0 && bw_params->clk_table.entries[0].fclk_mhz > min_fclk_mhz)
+		min_fclk_mhz = bw_params->clk_table.entries[0].fclk_mhz;
+
 	if (!max_dcfclk_mhz || !max_dispclk_mhz || !max_dtbclk_mhz)
 		return -1;
 
@@ -402,7 +405,6 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params,
 		for (i = *num_entries - 1; i >= 0 ; i--) {
 			if (table[i].fabricclk_mhz < min_fclk_mhz) {
 				table[i].fabricclk_mhz = min_fclk_mhz;
-				break;
 			}
 		}
 	}
@@ -411,7 +413,6 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params,
 	for (i = *num_entries - 1; i >= 0 ; i--) {
 		if (table[i].dcfclk_mhz < min_dcfclk_mhz) {
 			table[i].dcfclk_mhz = min_dcfclk_mhz;
-			break;
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
index 4125d3d111d1..bdf3ac6cadd5 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
@@ -41,51 +41,51 @@
 #include "dcn32/display_rq_dlg_calc_32.h"
 #include "dml_logger.h"
 
-const struct dml_funcs dml20_funcs = {
+static const struct dml_funcs dml20_funcs = {
 	.validate = dml20_ModeSupportAndSystemConfigurationFull,
 	.recalculate = dml20_recalculate,
 	.rq_dlg_get_dlg_reg = dml20_rq_dlg_get_dlg_reg,
 	.rq_dlg_get_rq_reg = dml20_rq_dlg_get_rq_reg
 };
 
-const struct dml_funcs dml20v2_funcs = {
+static const struct dml_funcs dml20v2_funcs = {
 	.validate = dml20v2_ModeSupportAndSystemConfigurationFull,
 	.recalculate = dml20v2_recalculate,
 	.rq_dlg_get_dlg_reg = dml20v2_rq_dlg_get_dlg_reg,
 	.rq_dlg_get_rq_reg = dml20v2_rq_dlg_get_rq_reg
 };
 
-const struct dml_funcs dml21_funcs = {
-        .validate = dml21_ModeSupportAndSystemConfigurationFull,
-        .recalculate = dml21_recalculate,
-        .rq_dlg_get_dlg_reg = dml21_rq_dlg_get_dlg_reg,
-        .rq_dlg_get_rq_reg = dml21_rq_dlg_get_rq_reg
+static const struct dml_funcs dml21_funcs = {
+	.validate = dml21_ModeSupportAndSystemConfigurationFull,
+	.recalculate = dml21_recalculate,
+	.rq_dlg_get_dlg_reg = dml21_rq_dlg_get_dlg_reg,
+	.rq_dlg_get_rq_reg = dml21_rq_dlg_get_rq_reg
 };
 
-const struct dml_funcs dml30_funcs = {
+static const struct dml_funcs dml30_funcs = {
 	.validate = dml30_ModeSupportAndSystemConfigurationFull,
 	.recalculate = dml30_recalculate,
 	.rq_dlg_get_dlg_reg = dml30_rq_dlg_get_dlg_reg,
 	.rq_dlg_get_rq_reg = dml30_rq_dlg_get_rq_reg
 };
 
-const struct dml_funcs dml31_funcs = {
+static const struct dml_funcs dml31_funcs = {
 	.validate = dml31_ModeSupportAndSystemConfigurationFull,
 	.recalculate = dml31_recalculate,
 	.rq_dlg_get_dlg_reg = dml31_rq_dlg_get_dlg_reg,
 	.rq_dlg_get_rq_reg = dml31_rq_dlg_get_rq_reg
 };
 
-const struct dml_funcs dml314_funcs = {
+static const struct dml_funcs dml314_funcs = {
 	.validate = dml314_ModeSupportAndSystemConfigurationFull,
 	.recalculate = dml314_recalculate,
 	.rq_dlg_get_dlg_reg = dml314_rq_dlg_get_dlg_reg,
 	.rq_dlg_get_rq_reg = dml314_rq_dlg_get_rq_reg
 };
 
-const struct dml_funcs dml32_funcs = {
+static const struct dml_funcs dml32_funcs = {
 	.validate = dml32_ModeSupportAndSystemConfigurationFull,
-    .recalculate = dml32_recalculate,
+	.recalculate = dml32_recalculate,
 	.rq_dlg_get_dlg_reg_v2 = dml32_rq_dlg_get_dlg_reg,
 	.rq_dlg_get_rq_reg_v2 = dml32_rq_dlg_get_rq_reg
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
index d52cbc0e9b67..2bdc47615543 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
@@ -47,6 +47,59 @@ static bool dsc_policy_disable_dsc_stream_overhead;
 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
 #endif
 
+uint32_t dc_bandwidth_in_kbps_from_timing(
+	const struct dc_crtc_timing *timing)
+{
+	uint32_t bits_per_channel = 0;
+	uint32_t kbps;
+
+	if (timing->flags.DSC)
+		return dc_dsc_stream_bandwidth_in_kbps(timing,
+				timing->dsc_cfg.bits_per_pixel,
+				timing->dsc_cfg.num_slices_h,
+				timing->dsc_cfg.is_dp);
+
+	switch (timing->display_color_depth) {
+	case COLOR_DEPTH_666:
+		bits_per_channel = 6;
+		break;
+	case COLOR_DEPTH_888:
+		bits_per_channel = 8;
+		break;
+	case COLOR_DEPTH_101010:
+		bits_per_channel = 10;
+		break;
+	case COLOR_DEPTH_121212:
+		bits_per_channel = 12;
+		break;
+	case COLOR_DEPTH_141414:
+		bits_per_channel = 14;
+		break;
+	case COLOR_DEPTH_161616:
+		bits_per_channel = 16;
+		break;
+	default:
+		ASSERT(bits_per_channel != 0);
+		bits_per_channel = 8;
+		break;
+	}
+
+	kbps = timing->pix_clk_100hz / 10;
+	kbps *= bits_per_channel;
+
+	if (timing->flags.Y_ONLY != 1) {
+		/*Only YOnly make reduce bandwidth by 1/3 compares to RGB*/
+		kbps *= 3;
+		if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
+			kbps /= 2;
+		else if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
+			kbps = kbps * 2 / 3;
+	}
+
+	return kbps;
+}
+
+
 /* Forward Declerations */
 static bool decide_dsc_bandwidth_range(
 		const uint32_t min_bpp_x16,
@@ -79,8 +132,7 @@ static bool setup_dsc_config(
 		const struct dsc_enc_caps *dsc_enc_caps,
 		int target_bandwidth_kbps,
 		const struct dc_crtc_timing *timing,
-		int min_slice_height_override,
-		int max_dsc_target_bpp_limit_override_x16,
+		const struct dc_dsc_config_options *options,
 		struct dc_dsc_config *dsc_cfg);
 
 static bool dsc_buff_block_size_from_dpcd(int dpcd_buff_block_size, int *buff_block_size)
@@ -352,6 +404,11 @@ bool dc_dsc_compute_bandwidth_range(
 	struct dsc_enc_caps dsc_enc_caps;
 	struct dsc_enc_caps dsc_common_caps;
 	struct dc_dsc_config config;
+	struct dc_dsc_config_options options = {0};
+
+	options.dsc_min_slice_height_override = dsc_min_slice_height_override;
+	options.max_target_bpp_limit_override_x16 = max_bpp_x16;
+	options.slice_height_granularity = 1;
 
 	get_dsc_enc_caps(dsc, &dsc_enc_caps, timing->pix_clk_100hz);
 
@@ -360,7 +417,7 @@ bool dc_dsc_compute_bandwidth_range(
 
 	if (is_dsc_possible)
 		is_dsc_possible = setup_dsc_config(dsc_sink_caps, &dsc_enc_caps, 0, timing,
-				dsc_min_slice_height_override, max_bpp_x16, &config);
+				&options, &config);
 
 	if (is_dsc_possible)
 		is_dsc_possible = decide_dsc_bandwidth_range(min_bpp_x16, max_bpp_x16,
@@ -740,8 +797,7 @@ static bool setup_dsc_config(
 		const struct dsc_enc_caps *dsc_enc_caps,
 		int target_bandwidth_kbps,
 		const struct dc_crtc_timing *timing,
-		int min_slice_height_override,
-		int max_dsc_target_bpp_limit_override_x16,
+		const struct dc_dsc_config_options *options,
 		struct dc_dsc_config *dsc_cfg)
 {
 	struct dsc_enc_caps dsc_common_caps;
@@ -760,7 +816,7 @@ static bool setup_dsc_config(
 
 	memset(dsc_cfg, 0, sizeof(struct dc_dsc_config));
 
-	dc_dsc_get_policy_for_timing(timing, max_dsc_target_bpp_limit_override_x16, &policy);
+	dc_dsc_get_policy_for_timing(timing, options->max_target_bpp_limit_override_x16, &policy);
 	pic_width = timing->h_addressable + timing->h_border_left + timing->h_border_right;
 	pic_height = timing->v_addressable + timing->v_border_top + timing->v_border_bottom;
 
@@ -909,12 +965,13 @@ static bool setup_dsc_config(
 
 	// Slice height (i.e. number of slices per column): start with policy and pick the first one that height is divisible by.
 	// For 4:2:0 make sure the slice height is divisible by 2 as well.
-	if (min_slice_height_override == 0)
+	if (options->dsc_min_slice_height_override == 0)
 		slice_height = min(policy.min_slice_height, pic_height);
 	else
-		slice_height = min(min_slice_height_override, pic_height);
+		slice_height = min((int)(options->dsc_min_slice_height_override), pic_height);
 
 	while (slice_height < pic_height && (pic_height % slice_height != 0 ||
+		slice_height % options->slice_height_granularity != 0 ||
 		(timing->pixel_encoding == PIXEL_ENCODING_YCBCR420 && slice_height % 2 != 0)))
 		slice_height++;
 
@@ -958,8 +1015,7 @@ done:
 bool dc_dsc_compute_config(
 		const struct display_stream_compressor *dsc,
 		const struct dsc_dec_dpcd_caps *dsc_sink_caps,
-		uint32_t dsc_min_slice_height_override,
-		uint32_t max_target_bpp_limit_override,
+		const struct dc_dsc_config_options *options,
 		uint32_t target_bandwidth_kbps,
 		const struct dc_crtc_timing *timing,
 		struct dc_dsc_config *dsc_cfg)
@@ -971,8 +1027,7 @@ bool dc_dsc_compute_config(
 	is_dsc_possible = setup_dsc_config(dsc_sink_caps,
 		&dsc_enc_caps,
 		target_bandwidth_kbps,
-		timing, dsc_min_slice_height_override,
-		max_target_bpp_limit_override * 16, dsc_cfg);
+		timing, options, dsc_cfg);
 	return is_dsc_possible;
 }
 
@@ -1104,3 +1159,10 @@ void dc_dsc_policy_set_disable_dsc_stream_overhead(bool disable)
 {
 	dsc_policy_disable_dsc_stream_overhead = disable;
 }
+
+void dc_dsc_get_default_config_option(const struct dc *dc, struct dc_dsc_config_options *options)
+{
+	options->dsc_min_slice_height_override = dc->debug.dsc_min_slice_height_override;
+	options->max_target_bpp_limit_override_x16 = 0;
+	options->slice_height_granularity = 1;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
index e97cf09be9d5..64cee8c80110 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
@@ -39,6 +39,7 @@
  */
 void calc_rc_params(struct rc_params *rc, const struct drm_dsc_config *pps)
 {
+#if defined(CONFIG_DRM_AMD_DC_FP)
 	enum colour_mode mode;
 	enum bits_per_comp bpc;
 	bool is_navite_422_or_420;
@@ -59,4 +60,5 @@ void calc_rc_params(struct rc_params *rc, const struct drm_dsc_config *pps)
 			   slice_width, slice_height,
 			   pps->dsc_version_minor);
 	DC_FP_END();
+#endif
 }
diff --git a/drivers/gpu/drm/amd/display/dc/hdcp/hdcp_msg.c b/drivers/gpu/drm/amd/display/dc/hdcp/hdcp_msg.c
index e1422e5e86c9..25ffc052d53b 100644
--- a/drivers/gpu/drm/amd/display/dc/hdcp/hdcp_msg.c
+++ b/drivers/gpu/drm/amd/display/dc/hdcp/hdcp_msg.c
@@ -27,7 +27,7 @@
 
 #include "dm_services.h"
 #include "dm_helpers.h"
-#include "include/hdcp_types.h"
+#include "include/hdcp_msg_types.h"
 #include "include/signal_types.h"
 #include "core_types.h"
 #include "link.h"
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index ed3c03108da6..2eb597a24425 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -51,9 +51,7 @@ void enable_surface_flip_reporting(struct dc_plane_state *plane_state,
 #include "clock_source.h"
 #include "audio.h"
 #include "dm_pp_smu.h"
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 #include "dm_cp_psp.h"
-#endif
 #include "link_hwss.h"
 
 /********** DAL Core*********************/
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
index 591ab1389e3b..bef843cc32a1 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
@@ -293,6 +293,9 @@ struct clk_mgr_funcs {
 
 	/* Get SMU present */
 	bool (*is_smu_present)(struct clk_mgr *clk_mgr);
+
+	int (*get_dispclk_from_dentist)(struct clk_mgr *clk_mgr_base);
+
 };
 
 struct clk_mgr {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
index ce006762f257..ad6acd1b34e1 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
@@ -148,18 +148,21 @@ struct dccg_funcs {
 		struct dccg *dccg,
 		int inst);
 
-void (*set_pixel_rate_div)(
-        struct dccg *dccg,
-        uint32_t otg_inst,
-        enum pixel_rate_div k1,
-        enum pixel_rate_div k2);
-
-void (*set_valid_pixel_rate)(
-        struct dccg *dccg,
-	int ref_dtbclk_khz,
-        int otg_inst,
-        int pixclk_khz);
+	void (*set_pixel_rate_div)(struct dccg *dccg,
+			uint32_t otg_inst,
+			enum pixel_rate_div k1,
+			enum pixel_rate_div k2);
 
+	void (*set_valid_pixel_rate)(
+			struct dccg *dccg,
+			int ref_dtbclk_khz,
+			int otg_inst,
+			int pixclk_khz);
+
+	void (*dpp_root_clock_control)(
+			struct dccg *dccg,
+			unsigned int dpp_inst,
+			bool clock_on);
 };
 
 #endif //__DAL_DCCG_H__
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
index 131fcfa28bca..f4aa76e02518 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
@@ -70,28 +70,38 @@ struct dpp_input_csc_matrix {
 };
 
 static const struct dpp_input_csc_matrix __maybe_unused dpp_input_csc_matrix[] = {
-	{COLOR_SPACE_SRGB,
-		{0x2000, 0, 0, 0, 0, 0x2000, 0, 0, 0, 0, 0x2000, 0} },
-	{COLOR_SPACE_SRGB_LIMITED,
-		{0x2000, 0, 0, 0, 0, 0x2000, 0, 0, 0, 0, 0x2000, 0} },
-	{COLOR_SPACE_YCBCR601,
-		{0x2cdd, 0x2000, 0, 0xe991, 0xe926, 0x2000, 0xf4fd, 0x10ef,
-						0, 0x2000, 0x38b4, 0xe3a6} },
-	{COLOR_SPACE_YCBCR601_LIMITED,
-		{0x3353, 0x2568, 0, 0xe400, 0xe5dc, 0x2568, 0xf367, 0x1108,
-						0, 0x2568, 0x40de, 0xdd3a} },
-	{COLOR_SPACE_YCBCR709,
-		{0x3265, 0x2000, 0, 0xe6ce, 0xf105, 0x2000, 0xfa01, 0xa7d, 0,
-						0x2000, 0x3b61, 0xe24f} },
-	{COLOR_SPACE_YCBCR709_LIMITED,
-		{0x39a6, 0x2568, 0, 0xe0d6, 0xeedd, 0x2568, 0xf925, 0x9a8, 0,
-						0x2568, 0x43ee, 0xdbb2} },
-	{COLOR_SPACE_2020_YCBCR,
-		{0x2F30, 0x2000, 0, 0xE869, 0xEDB7, 0x2000, 0xFABC, 0xBC6, 0,
-						0x2000, 0x3C34, 0xE1E6} },
-	{COLOR_SPACE_2020_RGB_LIMITEDRANGE,
-		{0x35E0, 0x255F, 0, 0xE2B3, 0xEB20, 0x255F, 0xF9FD, 0xB1E, 0,
-						0x255F, 0x44BD, 0xDB43} }
+	{ COLOR_SPACE_SRGB,
+		{ 0x2000, 0,      0,      0,
+		  0,      0x2000, 0,      0,
+		  0,      0,      0x2000, 0 } },
+	{ COLOR_SPACE_SRGB_LIMITED,
+		{ 0x2000, 0,      0,      0,
+		  0,      0x2000, 0,      0,
+		  0,      0,      0x2000, 0 } },
+	{ COLOR_SPACE_YCBCR601,
+		{ 0x2cdd, 0x2000, 0,      0xe991,
+		  0xe926, 0x2000, 0xf4fd, 0x10ef,
+		  0,      0x2000, 0x38b4, 0xe3a6 } },
+	{ COLOR_SPACE_YCBCR601_LIMITED,
+		{ 0x3353, 0x2568, 0,      0xe400,
+		  0xe5dc, 0x2568, 0xf367, 0x1108,
+		  0,      0x2568, 0x40de, 0xdd3a } },
+	{ COLOR_SPACE_YCBCR709,
+		{ 0x3265, 0x2000, 0,      0xe6ce,
+		  0xf105, 0x2000, 0xfa01, 0xa7d,
+		  0,      0x2000, 0x3b61, 0xe24f } },
+	{ COLOR_SPACE_YCBCR709_LIMITED,
+		{ 0x39a6, 0x2568, 0,      0xe0d6,
+		  0xeedd, 0x2568, 0xf925, 0x9a8,
+		  0,      0x2568, 0x43ee, 0xdbb2 } },
+	{ COLOR_SPACE_2020_YCBCR,
+		{ 0x2F30, 0x2000, 0,      0xE869,
+		  0xEDB7, 0x2000, 0xFABC, 0xBC6,
+		  0,      0x2000, 0x3C34, 0xE1E6 } },
+	{ COLOR_SPACE_2020_RGB_LIMITEDRANGE,
+		{ 0x35E0, 0x255F, 0,      0xE2B3,
+		  0xEB20, 0x255F, 0xF9FD, 0xB1E,
+		  0,      0x255F, 0x44BD, 0xDB43 } }
 };
 
 struct dpp_grph_csc_adjustment {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h
index b982be64c792..86b711dcc785 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h
@@ -53,9 +53,7 @@ enum dwb_source {
 /* DCN1.x, DCN2.x support 2 pipes */
 enum dwb_pipe {
 	dwb_pipe0 = 0,
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	dwb_pipe1,
-#endif
 	dwb_pipe_max_num,
 };
 
@@ -72,14 +70,11 @@ enum wbscl_coef_filter_type_sel {
 };
 
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 enum dwb_boundary_mode {
 	DWBSCL_BOUNDARY_MODE_EDGE  = 0,
 	DWBSCL_BOUNDARY_MODE_BLACK = 1
 };
-#endif
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 enum dwb_output_csc_mode {
 	DWB_OUTPUT_CSC_DISABLE = 0,
 	DWB_OUTPUT_CSC_COEF_A = 1,
@@ -132,7 +127,6 @@ struct dwb_efc_display_settings {
 	unsigned int	dwbOutputBlack;	// 0 - Normal, 1 - Output Black
 };
 
-#endif
 struct dwb_warmup_params {
 	bool	warmup_en;	/* false: normal mode, true: enable pattern generator */
 	bool	warmup_mode;	/* false: 420, true: 444 */
@@ -208,7 +202,7 @@ struct dwbc_funcs {
 		struct dwb_warmup_params *warmup_params);
 
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 
 	void (*dwb_program_output_csc)(
 		struct dwbc *dwbc,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
index d5ea7545583e..7f3f9b69e903 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
@@ -146,7 +146,7 @@ struct hubp_funcs {
 
 	void (*set_blank)(struct hubp *hubp, bool blank);
 	void (*set_blank_regs)(struct hubp *hubp, bool blank);
-#ifdef CONFIG_DRM_AMD_DC_DCN
+#ifdef CONFIG_DRM_AMD_DC_FP
 	void (*phantom_hubp_post_enable)(struct hubp *hubp);
 #endif
 	void (*set_hubp_blank_en)(struct hubp *hubp, bool blank);
@@ -203,6 +203,7 @@ struct hubp_funcs {
 	void (*hubp_soft_reset)(struct hubp *hubp, bool reset);
 
 	void (*hubp_update_force_pstate_disallow)(struct hubp *hubp, bool allow);
+	void (*hubp_update_force_cursor_pstate_disallow)(struct hubp *hubp, bool allow);
 	void (*hubp_update_mall_sel)(struct hubp *hubp, uint32_t mall_sel, bool c_cursor);
 	void (*hubp_prepare_subvp_buffering)(struct hubp *hubp, bool enable);
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
index a819f0f97c5f..b95ae9596c3b 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
@@ -275,20 +275,6 @@ enum dc_lut_mode {
 	LUT_RAM_B
 };
 
-enum symclk_state {
-	SYMCLK_OFF_TX_OFF,
-	SYMCLK_ON_TX_ON,
-	SYMCLK_ON_TX_OFF,
-};
-
-struct phy_state {
-	struct {
-		uint8_t otg		: 1;
-		uint8_t reserved	: 7;
-	} symclk_ref_cnts;
-	enum symclk_state symclk_state;
-};
-
 /**
  * speakersToChannels
  *
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
index ec572a9e4054..dbe7afa9d3a2 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
@@ -75,58 +75,6 @@ struct encoder_feature_support {
 	bool fec_supported;
 };
 
-union dpcd_psr_configuration {
-	struct {
-		unsigned char ENABLE                    : 1;
-		unsigned char TRANSMITTER_ACTIVE_IN_PSR : 1;
-		unsigned char CRC_VERIFICATION          : 1;
-		unsigned char FRAME_CAPTURE_INDICATION  : 1;
-		/* For eDP 1.4, PSR v2*/
-		unsigned char LINE_CAPTURE_INDICATION   : 1;
-		/* For eDP 1.4, PSR v2*/
-		unsigned char IRQ_HPD_WITH_CRC_ERROR    : 1;
-		unsigned char ENABLE_PSR2               : 1;
-		/* For eDP 1.5, PSR v2 w/ early transport */
-		unsigned char EARLY_TRANSPORT_ENABLE    : 1;
-	} bits;
-	unsigned char raw;
-};
-
-union dpcd_alpm_configuration {
-	struct {
-		unsigned char ENABLE                    : 1;
-		unsigned char IRQ_HPD_ENABLE            : 1;
-		unsigned char RESERVED                  : 6;
-	} bits;
-	unsigned char raw;
-};
-
-union dpcd_sink_active_vtotal_control_mode {
-	struct {
-		unsigned char ENABLE                    : 1;
-		unsigned char RESERVED                  : 7;
-	} bits;
-	unsigned char raw;
-};
-
-union psr_error_status {
-	struct {
-		unsigned char LINK_CRC_ERROR        :1;
-		unsigned char RFB_STORAGE_ERROR     :1;
-		unsigned char VSC_SDP_ERROR         :1;
-		unsigned char RESERVED              :5;
-	} bits;
-	unsigned char raw;
-};
-
-union psr_sink_psr_status {
-	struct {
-	unsigned char SINK_SELF_REFRESH_STATUS  :3;
-	unsigned char RESERVED                  :5;
-	} bits;
-	unsigned char raw;
-};
-
 struct link_encoder {
 	const struct link_encoder_funcs *funcs;
 	int32_t aux_channel_offset;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
index bb5ad70d4266..c4fbbf08ef86 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
@@ -30,7 +30,6 @@
 
 #include "audio_types.h"
 #include "hw_shared.h"
-#include "dc_link.h"
 
 struct dc_bios;
 struct dc_context;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
index 1d9f9c53d2bd..c21e7ffd5bd0 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
@@ -182,7 +182,7 @@ struct timing_generator_funcs {
 
 	bool (*enable_crtc)(struct timing_generator *tg);
 	bool (*disable_crtc)(struct timing_generator *tg);
-#ifdef CONFIG_DRM_AMD_DC_DCN
+#ifdef CONFIG_DRM_AMD_DC_FP
 	void (*phantom_crtc_post_enable)(struct timing_generator *tg);
 #endif
 	void (*disable_phantom_crtc)(struct timing_generator *tg);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h
index a4d61bb724b6..4513544559be 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h
@@ -115,6 +115,10 @@ struct hwseq_private_funcs {
 	void (*plane_atomic_disable)(struct dc *dc, struct pipe_ctx *pipe_ctx);
 	void (*enable_power_gating_plane)(struct dce_hwseq *hws,
 		bool enable);
+	void (*dpp_root_clock_control)(
+			struct dce_hwseq *hws,
+			unsigned int dpp_inst,
+			bool clock_on);
 	void (*dpp_pg_control)(struct dce_hwseq *hws,
 			unsigned int dpp_inst,
 			bool power_on);
@@ -148,9 +152,9 @@ struct hwseq_private_funcs {
 	void (*PLAT_58856_wa)(struct dc_state *context,
 			struct pipe_ctx *pipe_ctx);
 	void (*setup_hpo_hw_control)(const struct dce_hwseq *hws, bool enable);
-#ifdef CONFIG_DRM_AMD_DC_DCN
+#ifdef CONFIG_DRM_AMD_DC_FP
 	void (*program_mall_pipe_config)(struct dc *dc, struct dc_state *context);
-	void (*subvp_update_force_pstate)(struct dc *dc, struct dc_state *context);
+	void (*update_force_pstate)(struct dc *dc, struct dc_state *context);
 	void (*update_mall_sel)(struct dc *dc, struct dc_state *context);
 	unsigned int (*calculate_dccg_k1_k2_values)(struct pipe_ctx *pipe_ctx,
 			unsigned int *k1_div,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/link.h b/drivers/gpu/drm/amd/display/dc/inc/link.h
index e70fa0059223..f839494d59d8 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/link.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/link.h
@@ -28,17 +28,57 @@
 
 /* FILE POLICY AND INTENDED USAGE:
  *
- * This header declares link functions exposed to dc. All functions must have
- * "link_" as prefix. For example link_run_my_function. This header is strictly
- * private in dc and should never be included in other header files. dc
- * components should include this header in their .c files in order to access
- * functions in link folder. This file should never include any header files in
- * link folder. If there is a need to expose a function declared in one of
- * header files in side link folder, you need to move the function declaration
- * into this file and prefix it with "link_".
+ * This header defines link component function interfaces aka link_service.
+ * link_service provides the only entry point to link functions with function
+ * pointer style. This header is strictly private in dc and should never be
+ * included by DM because it exposes too much dc detail including all dc
+ * private types defined in core_types.h. Otherwise it will break DM - DC
+ * encapsulation and turn DM into a maintenance nightmare.
+ *
+ * The following shows a link component relation map.
+ *
+ * DM to DC:
+ * DM includes dc.h
+ * dc_link_exports.c or other dc files implement dc.h
+ *
+ * DC to Link:
+ * dc_link_exports.c or other dc files include link.h
+ * link_factory.c implements link.h
+ *
+ * Link sub-component to Link sub-component:
+ * link_factory.c includes --> link_xxx.h
+ * link_xxx.c implements link_xxx.h
+
+ * As you can see if you ever need to add a new dc link function and call it on
+ * DM/dc side, it is very difficult because you will need layers of translation.
+ * The most appropriate approach to implement new requirements on DM/dc side is
+ * to extend or generalize the functionality of existing link function
+ * interfaces so minimal modification is needed outside link component to
+ * achieve your new requirements. This approach reduces or even eliminates the
+ * effort needed outside link component to support a new link feature. This also
+ * reduces code discrepancy among DMs to support the same link feature. If we
+ * test full code path on one version of DM, and there is no feature specific
+ * modification required on other DMs, then we can have higher confidence that
+ * the feature will run on other DMs and produce the same result. The following
+ * are some good examples to start with:
+ *
+ * - detect_link --> to add new link detection or capability retrieval routines
+ *
+ * - validate_mode_timing --> to add new timing validation conditions
+ *
+ * - set_dpms_on/set_dpms_off --> to include new link enablement sequences
+ *
+ * If you must add new link functions, you will need to:
+ * 1. declare the function pointer here under the suitable commented category.
+ * 2. Implement your function in the suitable link_xxx.c file.
+ * 3. Assign the function to link_service in link_factory.c
+ * 4. NEVER include link_xxx.h headers outside link component.
+ * 5. NEVER include link.h on DM side.
  */
 #include "core_types.h"
-#include "dc_link.h"
+
+struct link_service *link_create_link_service(void);
+void link_destroy_link_service(struct link_service **link_srv);
 
 struct link_init_data {
 	const struct dc *dc;
@@ -49,14 +89,6 @@ struct link_init_data {
 	bool is_dpia_link;
 };
 
-struct dc_link *link_create(const struct link_init_data *init_params);
-void link_destroy(struct dc_link **link);
-
-// TODO - convert any function declarations below to function pointers
-struct gpio *link_get_hpd_gpio(struct dc_bios *dcb,
-		struct graphics_object_id link_id,
-		struct gpio_service *gpio_service);
-
 struct ddc_service_init_data {
 	struct graphics_object_id id;
 	struct dc_context *ctx;
@@ -64,94 +96,224 @@ struct ddc_service_init_data {
 	bool is_dpia_link;
 };
 
-struct ddc_service *link_create_ddc_service(
-		struct ddc_service_init_data *ddc_init_data);
+struct link_service {
+	/************************** Factory ***********************************/
+	struct dc_link *(*create_link)(
+			const struct link_init_data *init_params);
+	void (*destroy_link)(struct dc_link **link);
 
-void link_destroy_ddc_service(struct ddc_service **ddc);
 
-bool link_is_in_aux_transaction_mode(struct ddc_service *ddc);
+	/************************** Detection *********************************/
+	bool (*detect_link)(struct dc_link *link, enum dc_detect_reason reason);
+	bool (*detect_connection_type)(struct dc_link *link,
+			enum dc_connection_type *type);
+	struct dc_sink *(*add_remote_sink)(
+			struct dc_link *link,
+			const uint8_t *edid,
+			int len,
+			struct dc_sink_init_data *init_data);
+	void (*remove_remote_sink)(struct dc_link *link, struct dc_sink *sink);
+	bool (*get_hpd_state)(struct dc_link *link);
+	struct gpio *(*get_hpd_gpio)(struct dc_bios *dcb,
+			struct graphics_object_id link_id,
+			struct gpio_service *gpio_service);
+	void (*enable_hpd)(const struct dc_link *link);
+	void (*disable_hpd)(const struct dc_link *link);
+	void (*enable_hpd_filter)(struct dc_link *link, bool enable);
+	bool (*reset_cur_dp_mst_topology)(struct dc_link *link);
+	const struct dc_link_status *(*get_status)(const struct dc_link *link);
+	bool (*is_hdcp1x_supported)(struct dc_link *link,
+			enum signal_type signal);
+	bool (*is_hdcp2x_supported)(struct dc_link *link,
+			enum signal_type signal);
+	void (*clear_dprx_states)(struct dc_link *link);
 
-bool link_query_ddc_data(
-		struct ddc_service *ddc,
-		uint32_t address,
-		uint8_t *write_buf,
-		uint32_t write_size,
-		uint8_t *read_buf,
-		uint32_t read_size);
 
+	/*************************** Resource *********************************/
+	void (*get_cur_res_map)(const struct dc *dc, uint32_t *map);
+	void (*restore_res_map)(const struct dc *dc, uint32_t *map);
+	void (*get_cur_link_res)(const struct dc_link *link,
+			struct link_resource *link_res);
 
-/* Attempt to submit an aux payload, retrying on timeouts, defers, and busy
- * states as outlined in the DP spec.  Returns true if the request was
- * successful.
- *
- * NOTE: The function requires explicit mutex on DM side in order to prevent
- * potential race condition. DC components should call the dpcd read/write
- * function in dm_helpers in order to access dpcd safely
- */
-bool link_aux_transfer_with_retries_no_mutex(struct ddc_service *ddc,
-		struct aux_payload *payload);
 
-uint32_t link_get_aux_defer_delay(struct ddc_service *ddc);
+	/*************************** Validation *******************************/
+	enum dc_status (*validate_mode_timing)(
+			const struct dc_stream_state *stream,
+			struct dc_link *link,
+			const struct dc_crtc_timing *timing);
+	uint32_t (*dp_link_bandwidth_kbps)(
+		const struct dc_link *link,
+		const struct dc_link_settings *link_settings);
+	bool (*validate_dpia_bandwidth)(
+			const struct dc_stream_state *stream,
+			const unsigned int num_streams);
 
-bool link_is_dp_128b_132b_signal(struct pipe_ctx *pipe_ctx);
 
-enum dp_link_encoding link_dp_get_encoding_format(
-		const struct dc_link_settings *link_settings);
+	/*************************** DPMS *************************************/
+	void (*set_dpms_on)(struct dc_state *state, struct pipe_ctx *pipe_ctx);
+	void (*set_dpms_off)(struct pipe_ctx *pipe_ctx);
+	void (*resume)(struct dc_link *link);
+	void (*blank_all_dp_displays)(struct dc *dc);
+	void (*blank_all_edp_displays)(struct dc *dc);
+	void (*blank_dp_stream)(struct dc_link *link, bool hw_init);
+	enum dc_status (*increase_mst_payload)(
+			struct pipe_ctx *pipe_ctx, uint32_t req_pbn);
+	enum dc_status (*reduce_mst_payload)(
+			struct pipe_ctx *pipe_ctx, uint32_t req_pbn);
+	void (*set_dsc_on_stream)(struct pipe_ctx *pipe_ctx, bool enable);
+	bool (*set_dsc_enable)(struct pipe_ctx *pipe_ctx, bool enable);
+	bool (*update_dsc_config)(struct pipe_ctx *pipe_ctx);
 
-bool link_decide_link_settings(
-	struct dc_stream_state *stream,
-	struct dc_link_settings *link_setting);
-
-void link_dp_trace_set_edp_power_timestamp(struct dc_link *link,
-		bool power_up);
-uint64_t link_dp_trace_get_edp_poweron_timestamp(struct dc_link *link);
-uint64_t link_dp_trace_get_edp_poweroff_timestamp(struct dc_link *link);
-
-bool link_is_edp_ilr_optimization_required(struct dc_link *link,
-		struct dc_crtc_timing *crtc_timing);
-
-bool link_backlight_enable_aux(struct dc_link *link, bool enable);
-void link_edp_add_delay_for_T9(struct dc_link *link);
-bool link_edp_receiver_ready_T9(struct dc_link *link);
-bool link_edp_receiver_ready_T7(struct dc_link *link);
-bool link_power_alpm_dpcd_enable(struct dc_link *link, bool enable);
-bool link_set_sink_vtotal_in_psr_active(const struct dc_link *link,
-		uint16_t psr_vtotal_idle, uint16_t psr_vtotal_su);
-void link_get_psr_residency(const struct dc_link *link, uint32_t *residency);
-enum dc_status link_increase_mst_payload(struct pipe_ctx *pipe_ctx, uint32_t req_pbn);
-enum dc_status link_reduce_mst_payload(struct pipe_ctx *pipe_ctx, uint32_t req_pbn);
-void link_blank_all_dp_displays(struct dc *dc);
-void link_blank_all_edp_displays(struct dc *dc);
-void link_blank_dp_stream(struct dc_link *link, bool hw_init);
-void link_resume(struct dc_link *link);
-void link_set_dpms_on(
-		struct dc_state *state,
-		struct pipe_ctx *pipe_ctx);
-void link_set_dpms_off(struct pipe_ctx *pipe_ctx);
-void link_dp_source_sequence_trace(struct dc_link *link, uint8_t dp_test_mode);
-void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable);
-bool link_set_dsc_enable(struct pipe_ctx *pipe_ctx, bool enable);
-bool link_update_dsc_config(struct pipe_ctx *pipe_ctx);
-enum dc_status link_validate_mode_timing(
-		const struct dc_stream_state *stream,
+
+	/*************************** DDC **************************************/
+	struct ddc_service *(*create_ddc_service)(
+			struct ddc_service_init_data *ddc_init_data);
+	void (*destroy_ddc_service)(struct ddc_service **ddc);
+	bool (*query_ddc_data)(
+			struct ddc_service *ddc,
+			uint32_t address,
+			uint8_t *write_buf,
+			uint32_t write_size,
+			uint8_t *read_buf,
+			uint32_t read_size);
+	int (*aux_transfer_raw)(struct ddc_service *ddc,
+			struct aux_payload *payload,
+			enum aux_return_code_type *operation_result);
+	bool (*aux_transfer_with_retries_no_mutex)(struct ddc_service *ddc,
+			struct aux_payload *payload);
+	bool (*is_in_aux_transaction_mode)(struct ddc_service *ddc);
+	uint32_t (*get_aux_defer_delay)(struct ddc_service *ddc);
+
+
+	/*************************** DP Capability ****************************/
+	bool (*dp_is_sink_present)(struct dc_link *link);
+	bool (*dp_is_fec_supported)(const struct dc_link *link);
+	bool (*dp_is_128b_132b_signal)(struct pipe_ctx *pipe_ctx);
+	bool (*dp_get_max_link_enc_cap)(const struct dc_link *link,
+			struct dc_link_settings *max_link_enc_cap);
+	const struct dc_link_settings *(*dp_get_verified_link_cap)(
+			const struct dc_link *link);
+	enum dp_link_encoding (*dp_get_encoding_format)(
+			const struct dc_link_settings *link_settings);
+	bool (*dp_should_enable_fec)(const struct dc_link *link);
+	bool (*dp_decide_link_settings)(
+		struct dc_stream_state *stream,
+		struct dc_link_settings *link_setting);
+	enum dp_link_encoding (*mst_decide_link_encoding_format)(
+			const struct dc_link *link);
+	bool (*edp_decide_link_settings)(struct dc_link *link,
+			struct dc_link_settings *link_setting, uint32_t req_bw);
+	uint32_t (*bw_kbps_from_raw_frl_link_rate_data)(uint8_t bw);
+	bool (*dp_overwrite_extended_receiver_cap)(struct dc_link *link);
+	enum lttpr_mode (*dp_decide_lttpr_mode)(struct dc_link *link,
+			struct dc_link_settings *link_setting);
+
+
+	/*************************** DP DPIA/PHY ******************************/
+	int (*dpia_handle_usb4_bandwidth_allocation_for_link)(
+			struct dc_link *link, int peak_bw);
+	void (*dpia_handle_bw_alloc_response)(
+			struct dc_link *link, uint8_t bw, uint8_t result);
+	void (*dp_set_drive_settings)(
 		struct dc_link *link,
-		const struct dc_crtc_timing *timing);
-bool link_detect(struct dc_link *link, enum dc_detect_reason reason);
-bool link_detect_connection_type(struct dc_link *link,
-		enum dc_connection_type *type);
-const struct dc_link_status *link_get_status(const struct dc_link *link);
-#ifdef CONFIG_DRM_AMD_DC_HDCP
-/* return true if the connected receiver supports the hdcp version */
-bool link_is_hdcp14(struct dc_link *link, enum signal_type signal);
-bool link_is_hdcp22(struct dc_link *link, enum signal_type signal);
-#endif
-void link_clear_dprx_states(struct dc_link *link);
-bool link_reset_cur_dp_mst_topology(struct dc_link *link);
-uint32_t dp_link_bandwidth_kbps(
-	const struct dc_link *link,
-	const struct dc_link_settings *link_settings);
-uint32_t link_timing_bandwidth_kbps(const struct dc_crtc_timing *timing);
-void link_get_cur_res_map(const struct dc *dc, uint32_t *map);
-void link_restore_res_map(const struct dc *dc, uint32_t *map);
+		const struct link_resource *link_res,
+		struct link_training_settings *lt_settings);
+	void (*dpcd_write_rx_power_ctrl)(struct dc_link *link, bool on);
+
 
+	/*************************** DP IRQ Handler ***************************/
+	bool (*dp_parse_link_loss_status)(
+		struct dc_link *link,
+		union hpd_irq_data *hpd_irq_dpcd_data);
+	bool (*dp_should_allow_hpd_rx_irq)(const struct dc_link *link);
+	void (*dp_handle_link_loss)(struct dc_link *link);
+	enum dc_status (*dp_read_hpd_rx_irq_data)(
+		struct dc_link *link,
+		union hpd_irq_data *irq_data);
+	bool (*dp_handle_hpd_rx_irq)(struct dc_link *link,
+			union hpd_irq_data *out_hpd_irq_dpcd_data,
+			bool *out_link_loss,
+			bool defer_handling, bool *has_left_work);
+
+
+	/*************************** eDP Panel Control ************************/
+	void (*edp_panel_backlight_power_on)(
+			struct dc_link *link, bool wait_for_hpd);
+	int (*edp_get_backlight_level)(const struct dc_link *link);
+	bool (*edp_get_backlight_level_nits)(struct dc_link *link,
+			uint32_t *backlight_millinits_avg,
+			uint32_t *backlight_millinits_peak);
+	bool (*edp_set_backlight_level)(const struct dc_link *link,
+			uint32_t backlight_pwm_u16_16,
+			uint32_t frame_ramp);
+	bool (*edp_set_backlight_level_nits)(struct dc_link *link,
+			bool isHDR,
+			uint32_t backlight_millinits,
+			uint32_t transition_time_in_ms);
+	int (*edp_get_target_backlight_pwm)(const struct dc_link *link);
+	bool (*edp_get_psr_state)(
+			const struct dc_link *link, enum dc_psr_state *state);
+	bool (*edp_set_psr_allow_active)(
+			struct dc_link *link,
+			const bool *allow_active,
+			bool wait,
+			bool force_static,
+			const unsigned int *power_opts);
+	bool (*edp_setup_psr)(struct dc_link *link,
+			const struct dc_stream_state *stream,
+			struct psr_config *psr_config,
+			struct psr_context *psr_context);
+	bool (*edp_set_sink_vtotal_in_psr_active)(
+			const struct dc_link *link,
+			uint16_t psr_vtotal_idle,
+			uint16_t psr_vtotal_su);
+	void (*edp_get_psr_residency)(
+			const struct dc_link *link, uint32_t *residency);
+	bool (*edp_wait_for_t12)(struct dc_link *link);
+	bool (*edp_is_ilr_optimization_required)(struct dc_link *link,
+			struct dc_crtc_timing *crtc_timing);
+	bool (*edp_backlight_enable_aux)(struct dc_link *link, bool enable);
+	void (*edp_add_delay_for_T9)(struct dc_link *link);
+	bool (*edp_receiver_ready_T9)(struct dc_link *link);
+	bool (*edp_receiver_ready_T7)(struct dc_link *link);
+	bool (*edp_power_alpm_dpcd_enable)(struct dc_link *link, bool enable);
+
+
+	/*************************** DP CTS ************************************/
+	void (*dp_handle_automated_test)(struct dc_link *link);
+	bool (*dp_set_test_pattern)(
+			struct dc_link *link,
+			enum dp_test_pattern test_pattern,
+			enum dp_test_pattern_color_space test_pattern_color_space,
+			const struct link_training_settings *p_link_settings,
+			const unsigned char *p_custom_pattern,
+			unsigned int cust_pattern_size);
+	void (*dp_set_preferred_link_settings)(struct dc *dc,
+			struct dc_link_settings *link_setting,
+			struct dc_link *link);
+	void (*dp_set_preferred_training_settings)(struct dc *dc,
+			struct dc_link_settings *link_setting,
+			struct dc_link_training_overrides *lt_overrides,
+			struct dc_link *link,
+			bool skip_immediate_retrain);
+
+
+	/*************************** DP Trace *********************************/
+	bool (*dp_trace_is_initialized)(struct dc_link *link);
+	void (*dp_trace_set_is_logged_flag)(struct dc_link *link,
+			bool in_detection,
+			bool is_logged);
+	bool (*dp_trace_is_logged)(struct dc_link *link, bool in_detection);
+	unsigned long long (*dp_trace_get_lt_end_timestamp)(
+			struct dc_link *link, bool in_detection);
+	const struct dp_trace_lt_counts *(*dp_trace_get_lt_counts)(
+			struct dc_link *link, bool in_detection);
+	unsigned int (*dp_trace_get_link_loss_count)(struct dc_link *link);
+	void (*dp_trace_set_edp_power_timestamp)(struct dc_link *link,
+			bool power_up);
+	uint64_t (*dp_trace_get_edp_poweron_timestamp)(struct dc_link *link);
+	uint64_t (*dp_trace_get_edp_poweroff_timestamp)(struct dc_link *link);
+	void (*dp_trace_source_sequence)(
+			struct dc_link *link, uint8_t dp_test_mode);
+};
 #endif /* __DC_LINK_HPD_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h
index fa6da93caa88..eaeb684c8a48 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/resource.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h
@@ -201,7 +201,7 @@ bool get_temp_dp_link_res(struct dc_link *link,
 		struct link_resource *link_res,
 		struct dc_link_settings *link_settings);
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 struct hpo_dp_link_encoder *resource_get_hpo_dp_link_enc_for_det_lt(
 		const struct resource_context *res_ctx,
 		const struct resource_pool *pool,
diff --git a/drivers/gpu/drm/amd/display/dc/link/Makefile b/drivers/gpu/drm/amd/display/dc/link/Makefile
index 40352d8d7648..a52b56e2859e 100644
--- a/drivers/gpu/drm/amd/display/dc/link/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/link/Makefile
@@ -55,7 +55,7 @@ LINK_PROTOCOLS = link_hpd.o link_ddc.o link_dpcd.o link_dp_dpia.o \
 link_dp_training.o link_dp_training_8b_10b.o link_dp_training_128b_132b.o \
 link_dp_training_dpia.o link_dp_training_auxless.o \
 link_dp_training_fixed_vs_pe_retimer.o link_dp_phy.o link_dp_capability.o \
-link_edp_panel_control.o link_dp_irq_handler.o
+link_edp_panel_control.o link_dp_irq_handler.o link_dp_dpia_bw.o
 
 AMD_DAL_LINK_PROTOCOLS = $(addprefix $(AMDDALPATH)/dc/link/protocols/, \
 $(LINK_PROTOCOLS))
diff --git a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c
index 942300e0bd92..db9f1baa27e5 100644
--- a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c
+++ b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c
@@ -28,6 +28,7 @@
 #include "link/protocols/link_dp_training.h"
 #include "link/protocols/link_dp_phy.h"
 #include "link/protocols/link_dp_training_fixed_vs_pe_retimer.h"
+#include "link/protocols/link_dp_capability.h"
 #include "link/link_dpms.h"
 #include "resource.h"
 #include "dm_helpers.h"
@@ -75,7 +76,7 @@ static bool is_dp_phy_pattern(enum dp_test_pattern test_pattern)
 		return false;
 }
 
-void dp_retrain_link_dp_test(struct dc_link *link,
+static void dp_retrain_link_dp_test(struct dc_link *link,
 			struct dc_link_settings *link_setting,
 			bool skip_video_pattern)
 {
@@ -250,7 +251,7 @@ static void dp_test_send_phy_test_pattern(struct dc_link *link)
 	/* prepare link training settings */
 	link_training_settings.link_settings = link->cur_link_settings;
 
-	link_training_settings.lttpr_mode = dc_link_decide_lttpr_mode(link, &link->cur_link_settings);
+	link_training_settings.lttpr_mode = dp_decide_lttpr_mode(link, &link->cur_link_settings);
 
 	if ((link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) &&
 			link_training_settings.lttpr_mode == LTTPR_MODE_TRANSPARENT)
@@ -408,7 +409,7 @@ static void dp_test_send_phy_test_pattern(struct dc_link *link)
 	 * all the time. Do not touch it.
 	 * forward request to DS
 	 */
-	dc_link_dp_set_test_pattern(
+	dp_set_test_pattern(
 		link,
 		test_pattern,
 		DP_TEST_PATTERN_COLOR_SPACE_UNDEFINED,
@@ -585,7 +586,7 @@ static void set_crtc_test_pattern(struct dc_link *link,
 	}
 }
 
-void dc_link_dp_handle_automated_test(struct dc_link *link)
+void dp_handle_automated_test(struct dc_link *link)
 {
 	union test_request test_request;
 	union test_response test_response;
@@ -651,7 +652,7 @@ void dc_link_dp_handle_automated_test(struct dc_link *link)
 			sizeof(test_response));
 }
 
-bool dc_link_dp_set_test_pattern(
+bool dp_set_test_pattern(
 	struct dc_link *link,
 	enum dp_test_pattern test_pattern,
 	enum dp_test_pattern_color_space test_pattern_color_space,
@@ -941,28 +942,9 @@ bool dc_link_dp_set_test_pattern(
 	return true;
 }
 
-void dc_link_set_drive_settings(struct dc *dc,
-				struct link_training_settings *lt_settings,
-				const struct dc_link *link)
-{
-
-	int i;
-	struct link_resource link_res;
-
-	for (i = 0; i < dc->link_count; i++)
-		if (dc->links[i] == link)
-			break;
-
-	if (i >= dc->link_count)
-		ASSERT_CRITICAL(false);
-
-	link_get_cur_link_res(link, &link_res);
-	dp_set_drive_settings(dc->links[i], &link_res, lt_settings);
-}
-
-void dc_link_set_preferred_link_settings(struct dc *dc,
-					 struct dc_link_settings *link_setting,
-					 struct dc_link *link)
+void dp_set_preferred_link_settings(struct dc *dc,
+		struct dc_link_settings *link_setting,
+		struct dc_link *link)
 {
 	int i;
 	struct pipe_ctx *pipe;
@@ -1001,11 +983,11 @@ void dc_link_set_preferred_link_settings(struct dc *dc,
 		dp_retrain_link_dp_test(link, &store_settings, false);
 }
 
-void dc_link_set_preferred_training_settings(struct dc *dc,
-						 struct dc_link_settings *link_setting,
-						 struct dc_link_training_overrides *lt_overrides,
-						 struct dc_link *link,
-						 bool skip_immediate_retrain)
+void dp_set_preferred_training_settings(struct dc *dc,
+		struct dc_link_settings *link_setting,
+		struct dc_link_training_overrides *lt_overrides,
+		struct dc_link *link,
+		bool skip_immediate_retrain)
 {
 	if (lt_overrides != NULL)
 		link->preferred_training_settings = *lt_overrides;
@@ -1025,22 +1007,5 @@ void dc_link_set_preferred_training_settings(struct dc *dc,
 
 	/* Retrain now, or wait until next stream update to apply */
 	if (skip_immediate_retrain == false)
-		dc_link_set_preferred_link_settings(dc, &link->preferred_link_setting, link);
-}
-
-void dc_link_set_test_pattern(struct dc_link *link,
-		enum dp_test_pattern test_pattern,
-		enum dp_test_pattern_color_space test_pattern_color_space,
-		const struct link_training_settings *p_link_settings,
-		const unsigned char *p_custom_pattern,
-		unsigned int cust_pattern_size)
-{
-	if (link != NULL)
-		dc_link_dp_set_test_pattern(
-			link,
-			test_pattern,
-			test_pattern_color_space,
-			p_link_settings,
-			p_custom_pattern,
-			cust_pattern_size);
+		dp_set_preferred_link_settings(dc, &link->preferred_link_setting, link);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.h b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.h
index 7f17838b653b..eae23ea7f6ec 100644
--- a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.h
+++ b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.h
@@ -25,9 +25,20 @@
 #ifndef __LINK_DP_CTS_H__
 #define __LINK_DP_CTS_H__
 #include "link.h"
-
-void dp_retrain_link_dp_test(struct dc_link *link,
+void dp_handle_automated_test(struct dc_link *link);
+bool dp_set_test_pattern(
+		struct dc_link *link,
+		enum dp_test_pattern test_pattern,
+		enum dp_test_pattern_color_space test_pattern_color_space,
+		const struct link_training_settings *p_link_settings,
+		const unsigned char *p_custom_pattern,
+		unsigned int cust_pattern_size);
+void dp_set_preferred_link_settings(struct dc *dc,
 		struct dc_link_settings *link_setting,
-		bool skip_video_pattern);
-
+		struct dc_link *link);
+void dp_set_preferred_training_settings(struct dc *dc,
+		struct dc_link_settings *link_setting,
+		struct dc_link_training_overrides *lt_overrides,
+		struct dc_link *link,
+		bool skip_immediate_retrain);
 #endif /* __LINK_DP_CTS_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.c b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.c
index 459b362ed374..fbcd8fb58ea8 100644
--- a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.c
+++ b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.c
@@ -37,7 +37,7 @@ void dp_trace_reset(struct dc_link *link)
 	memset(&link->dp_trace, 0, sizeof(link->dp_trace));
 }
 
-bool dc_dp_trace_is_initialized(struct dc_link *link)
+bool dp_trace_is_initialized(struct dc_link *link)
 {
 	return link->dp_trace.is_initialized;
 }
@@ -76,7 +76,7 @@ void dp_trace_lt_total_count_increment(struct dc_link *link,
 		link->dp_trace.commit_lt_trace.counts.total++;
 }
 
-void dc_dp_trace_set_is_logged_flag(struct dc_link *link,
+void dp_trace_set_is_logged_flag(struct dc_link *link,
 		bool in_detection,
 		bool is_logged)
 {
@@ -86,8 +86,7 @@ void dc_dp_trace_set_is_logged_flag(struct dc_link *link,
 		link->dp_trace.commit_lt_trace.is_logged = is_logged;
 }
 
-bool dc_dp_trace_is_logged(struct dc_link *link,
-		bool in_detection)
+bool dp_trace_is_logged(struct dc_link *link, bool in_detection)
 {
 	if (in_detection)
 		return link->dp_trace.detect_lt_trace.is_logged;
@@ -123,7 +122,7 @@ void dp_trace_set_lt_end_timestamp(struct dc_link *link,
 		link->dp_trace.commit_lt_trace.timestamps.end = dm_get_timestamp(link->dc->ctx);
 }
 
-unsigned long long dc_dp_trace_get_lt_end_timestamp(struct dc_link *link,
+unsigned long long dp_trace_get_lt_end_timestamp(struct dc_link *link,
 		bool in_detection)
 {
 	if (in_detection)
@@ -132,7 +131,7 @@ unsigned long long dc_dp_trace_get_lt_end_timestamp(struct dc_link *link,
 		return link->dp_trace.commit_lt_trace.timestamps.end;
 }
 
-struct dp_trace_lt_counts *dc_dp_trace_get_lt_counts(struct dc_link *link,
+const struct dp_trace_lt_counts *dp_trace_get_lt_counts(struct dc_link *link,
 		bool in_detection)
 {
 	if (in_detection)
@@ -141,12 +140,12 @@ struct dp_trace_lt_counts *dc_dp_trace_get_lt_counts(struct dc_link *link,
 		return &link->dp_trace.commit_lt_trace.counts;
 }
 
-unsigned int dc_dp_trace_get_link_loss_count(struct dc_link *link)
+unsigned int dp_trace_get_link_loss_count(struct dc_link *link)
 {
 	return link->dp_trace.link_loss_count;
 }
 
-void link_dp_trace_set_edp_power_timestamp(struct dc_link *link,
+void dp_trace_set_edp_power_timestamp(struct dc_link *link,
 		bool power_up)
 {
 	if (!power_up)
@@ -156,17 +155,17 @@ void link_dp_trace_set_edp_power_timestamp(struct dc_link *link,
 		link->dp_trace.edp_trace_power_timestamps.poweron = dm_get_timestamp(link->dc->ctx);
 }
 
-uint64_t link_dp_trace_get_edp_poweron_timestamp(struct dc_link *link)
+uint64_t dp_trace_get_edp_poweron_timestamp(struct dc_link *link)
 {
 	return link->dp_trace.edp_trace_power_timestamps.poweron;
 }
 
-uint64_t link_dp_trace_get_edp_poweroff_timestamp(struct dc_link *link)
+uint64_t dp_trace_get_edp_poweroff_timestamp(struct dc_link *link)
 {
 	return link->dp_trace.edp_trace_power_timestamps.poweroff;
 }
 
-void link_dp_source_sequence_trace(struct dc_link *link, uint8_t dp_test_mode)
+void dp_trace_source_sequence(struct dc_link *link, uint8_t dp_test_mode)
 {
 	if (link != NULL && link->dc->debug.enable_driver_sequence_debug)
 		core_link_write_dpcd(link, DP_SOURCE_SEQUENCE,
diff --git a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.h b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.h
index 89feea1b2692..ab437a0c9101 100644
--- a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.h
+++ b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.h
@@ -28,7 +28,7 @@
 
 void dp_trace_init(struct dc_link *link);
 void dp_trace_reset(struct dc_link *link);
-bool dc_dp_trace_is_initialized(struct dc_link *link);
+bool dp_trace_is_initialized(struct dc_link *link);
 void dp_trace_detect_lt_init(struct dc_link *link);
 void dp_trace_commit_lt_init(struct dc_link *link);
 void dp_trace_link_loss_increment(struct dc_link *link);
@@ -37,10 +37,10 @@ void dp_trace_lt_fail_count_update(struct dc_link *link,
 		bool in_detection);
 void dp_trace_lt_total_count_increment(struct dc_link *link,
 		bool in_detection);
-void dc_dp_trace_set_is_logged_flag(struct dc_link *link,
+void dp_trace_set_is_logged_flag(struct dc_link *link,
 		bool in_detection,
 		bool is_logged);
-bool dc_dp_trace_is_logged(struct dc_link *link,
+bool dp_trace_is_logged(struct dc_link *link,
 		bool in_detection);
 void dp_trace_lt_result_update(struct dc_link *link,
 		enum link_training_result result,
@@ -49,10 +49,15 @@ void dp_trace_set_lt_start_timestamp(struct dc_link *link,
 		bool in_detection);
 void dp_trace_set_lt_end_timestamp(struct dc_link *link,
 		bool in_detection);
-unsigned long long dc_dp_trace_get_lt_end_timestamp(struct dc_link *link,
+unsigned long long dp_trace_get_lt_end_timestamp(struct dc_link *link,
 		bool in_detection);
-struct dp_trace_lt_counts *dc_dp_trace_get_lt_counts(struct dc_link *link,
+const struct dp_trace_lt_counts *dp_trace_get_lt_counts(struct dc_link *link,
 		bool in_detection);
-unsigned int dc_dp_trace_get_link_loss_count(struct dc_link *link);
+unsigned int dp_trace_get_link_loss_count(struct dc_link *link);
+void dp_trace_set_edp_power_timestamp(struct dc_link *link,
+		bool power_up);
+uint64_t dp_trace_get_edp_poweron_timestamp(struct dc_link *link);
+uint64_t dp_trace_get_edp_poweroff_timestamp(struct dc_link *link);
+void dp_trace_source_sequence(struct dc_link *link, uint8_t dp_test_mode);
 
 #endif /* __LINK_DP_TRACE_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_dio.c b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_dio.c
index b092b00b3599..bebf9c4c8702 100644
--- a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_dio.c
+++ b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_dio.c
@@ -44,7 +44,7 @@ void setup_dio_stream_encoder(struct pipe_ctx *pipe_ctx)
 	link_enc->funcs->connect_dig_be_to_fe(link_enc,
 			pipe_ctx->stream_res.stream_enc->id, true);
 	if (dc_is_dp_signal(pipe_ctx->stream->signal))
-		link_dp_source_sequence_trace(pipe_ctx->stream->link,
+		pipe_ctx->stream->ctx->dc->link_srv->dp_trace_source_sequence(pipe_ctx->stream->link,
 				DPCD_SOURCE_SEQ_AFTER_CONNECT_DIG_FE_BE);
 	if (stream_enc->funcs->enable_fifo)
 		stream_enc->funcs->enable_fifo(stream_enc);
@@ -63,7 +63,8 @@ void reset_dio_stream_encoder(struct pipe_ctx *pipe_ctx)
 			pipe_ctx->stream_res.stream_enc->id,
 			false);
 	if (dc_is_dp_signal(pipe_ctx->stream->signal))
-		link_dp_source_sequence_trace(pipe_ctx->stream->link,
+		pipe_ctx->stream->ctx->dc->link_srv->dp_trace_source_sequence(
+				pipe_ctx->stream->link,
 				DPCD_SOURCE_SEQ_AFTER_DISCONNECT_DIG_FE_BE);
 
 }
@@ -105,7 +106,8 @@ void setup_dio_stream_attribute(struct pipe_ctx *pipe_ctx)
 				&stream->timing);
 
 	if (dc_is_dp_signal(stream->signal))
-		link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DP_STREAM_ATTR);
+		link->dc->link_srv->dp_trace_source_sequence(link,
+				DPCD_SOURCE_SEQ_AFTER_DP_STREAM_ATTR);
 }
 
 void enable_dio_dp_link_output(struct dc_link *link,
@@ -126,7 +128,8 @@ void enable_dio_dp_link_output(struct dc_link *link,
 				link_enc,
 				link_settings,
 				clock_source);
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_LINK_PHY);
+	link->dc->link_srv->dp_trace_source_sequence(link,
+			DPCD_SOURCE_SEQ_AFTER_ENABLE_LINK_PHY);
 }
 
 void disable_dio_link_output(struct dc_link *link,
@@ -136,7 +139,8 @@ void disable_dio_link_output(struct dc_link *link,
 	struct link_encoder *link_enc = link_enc_cfg_get_link_enc(link);
 
 	link_enc->funcs->disable_output(link_enc, signal);
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
+	link->dc->link_srv->dp_trace_source_sequence(link,
+			DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
 }
 
 void set_dio_dp_link_test_pattern(struct dc_link *link,
@@ -146,7 +150,7 @@ void set_dio_dp_link_test_pattern(struct dc_link *link,
 	struct link_encoder *link_enc = link_enc_cfg_get_link_enc(link);
 
 	link_enc->funcs->dp_set_phy_pattern(link_enc, tp_params);
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_SET_SOURCE_PATTERN);
+	link->dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_SET_SOURCE_PATTERN);
 }
 
 void set_dio_dp_lane_settings(struct dc_link *link,
@@ -195,7 +199,8 @@ void enable_dio_audio_packet(struct pipe_ctx *pipe_ctx)
 			pipe_ctx->stream_res.stream_enc, false);
 
 	if (dc_is_dp_signal(pipe_ctx->stream->signal))
-		link_dp_source_sequence_trace(pipe_ctx->stream->link,
+		pipe_ctx->stream->ctx->dc->link_srv->dp_trace_source_sequence(
+				pipe_ctx->stream->link,
 				DPCD_SOURCE_SEQ_AFTER_ENABLE_AUDIO_STREAM);
 }
 
@@ -214,7 +219,8 @@ void disable_dio_audio_packet(struct pipe_ctx *pipe_ctx)
 	}
 
 	if (dc_is_dp_signal(pipe_ctx->stream->signal))
-		link_dp_source_sequence_trace(pipe_ctx->stream->link,
+		pipe_ctx->stream->ctx->dc->link_srv->dp_trace_source_sequence(
+				pipe_ctx->stream->link,
 				DPCD_SOURCE_SEQ_AFTER_DISABLE_AUDIO_STREAM);
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c
index aa1c5e253b43..edd7d026a762 100644
--- a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c
+++ b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c
@@ -68,7 +68,8 @@ static void set_hpo_dp_hblank_min_symbol_width(struct pipe_ctx *pipe_ctx,
 	struct dc_crtc_timing *timing = &pipe_ctx->stream->timing;
 	struct fixed31_32 h_blank_in_ms, time_slot_in_ms, mtp_cnt_per_h_blank;
 	uint32_t link_bw_in_kbps =
-			dc_link_bandwidth_kbps(pipe_ctx->stream->link, link_settings);
+			hpo_dp_stream_encoder->ctx->dc->link_srv->dp_link_bandwidth_kbps(
+					pipe_ctx->stream->link, link_settings);
 	uint16_t hblank_min_symbol_width = 0;
 
 	if (link_bw_in_kbps > 0) {
@@ -115,7 +116,8 @@ static void setup_hpo_dp_stream_attribute(struct pipe_ctx *pipe_ctx)
 			stream->use_vsc_sdp_for_colorimetry,
 			stream->timing.flags.DSC,
 			false);
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DP_STREAM_ATTR);
+	link->dc->link_srv->dp_trace_source_sequence(link,
+			DPCD_SOURCE_SEQ_AFTER_DP_STREAM_ATTR);
 }
 
 static void enable_hpo_dp_fpga_link_output(struct dc_link *link,
@@ -201,7 +203,7 @@ static void set_hpo_dp_link_test_pattern(struct dc_link *link,
 {
 	link_res->hpo_dp_link_enc->funcs->set_link_test_pattern(
 			link_res->hpo_dp_link_enc, tp_params);
-	link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_SET_SOURCE_PATTERN);
+	link->dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_SET_SOURCE_PATTERN);
 }
 
 static void set_hpo_dp_lane_settings(struct dc_link *link,
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
index f70025ef7b69..a131e30fd7d6 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
@@ -60,6 +60,10 @@
  */
 #define LINK_TRAINING_MAX_VERIFY_RETRY 2
 
+static const u8 DP_SINK_BRANCH_DEV_NAME_7580[] = "7580\x80u";
+
+static const uint8_t dp_hdmi_dongle_signature_str[] = "DP-HDMI ADAPTOR";
+
 static enum ddc_transaction_type get_ddc_transaction_type(enum signal_type sink_signal)
 {
 	enum ddc_transaction_type transaction_type = DDC_TRANSACTION_TYPE_NONE;
@@ -466,7 +470,6 @@ static void link_disconnect_remap(struct dc_sink *prev_sink, struct dc_link *lin
 	link->local_sink = prev_sink;
 }
 
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 static void query_hdcp_capability(enum signal_type signal, struct dc_link *link)
 {
 	struct hdcp_protection_message msg22;
@@ -495,8 +498,6 @@ static void query_hdcp_capability(enum signal_type signal, struct dc_link *link)
 	dc_process_hdcp_msg(signal, link, &msg22);
 
 	if (signal == SIGNAL_TYPE_DISPLAY_PORT || signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
-		enum hdcp_message_status status = HDCP_MESSAGE_UNSUPPORTED;
-
 		msg14.data = &link->hdcp_caps.bcaps.raw;
 		msg14.length = sizeof(link->hdcp_caps.bcaps.raw);
 		msg14.msg_id = HDCP_MESSAGE_ID_READ_BCAPS;
@@ -504,11 +505,10 @@ static void query_hdcp_capability(enum signal_type signal, struct dc_link *link)
 		msg14.link = HDCP_LINK_PRIMARY;
 		msg14.max_retries = 5;
 
-		status = dc_process_hdcp_msg(signal, link, &msg14);
+		dc_process_hdcp_msg(signal, link, &msg14);
 	}
 
 }
-#endif // CONFIG_DRM_AMD_DC_HDCP
 static void read_current_link_settings_on_detect(struct dc_link *link)
 {
 	union lane_count_set lane_count_set = {0};
@@ -832,7 +832,7 @@ static void verify_link_capability(struct dc_link *link, struct dc_sink *sink,
 		verify_link_capability_non_destructive(link);
 }
 
-/**
+/*
  * detect_link_and_local_sink() - Detect if a sink is attached to a given link
  *
  * link->local_sink is created or destroyed as needed.
@@ -879,7 +879,7 @@ static bool detect_link_and_local_sink(struct dc_link *link,
 		return true;
 	}
 
-	if (!dc_link_detect_connection_type(link, &new_connection_type)) {
+	if (!link_detect_connection_type(link, &new_connection_type)) {
 		BREAK_TO_DEBUGGER();
 		return false;
 	}
@@ -1087,9 +1087,7 @@ static bool detect_link_and_local_sink(struct dc_link *link,
 			 * TODO debug why certain monitors don't like
 			 *  two link trainings
 			 */
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 			query_hdcp_capability(sink->sink_signal, link);
-#endif
 		} else {
 			// If edid is the same, then discard new sink and revert back to original sink
 			if (same_edid) {
@@ -1097,9 +1095,7 @@ static bool detect_link_and_local_sink(struct dc_link *link,
 				sink = prev_sink;
 				prev_sink = NULL;
 			}
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 			query_hdcp_capability(sink->sink_signal, link);
-#endif
 		}
 
 		/* HDMI-DVI Dongle */
@@ -1165,9 +1161,7 @@ static bool detect_link_and_local_sink(struct dc_link *link,
 		/* From Connected-to-Disconnected. */
 		link->type = dc_connection_none;
 		sink_caps.signal = SIGNAL_TYPE_NONE;
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 		memset(&link->hdcp_caps, 0, sizeof(struct hdcp_caps));
-#endif
 		/* When we unplug a passive DP-HDMI dongle connection, dongle_max_pix_clk
 		 *  is not cleared. If we emulate a DP signal on this connection, it thinks
 		 *  the dongle is still there and limits the number of modes we can emulate.
@@ -1191,8 +1185,8 @@ static bool detect_link_and_local_sink(struct dc_link *link,
 	return true;
 }
 
-/**
- * dc_link_detect_connection_type() - Determine if there is a sink connected
+/*
+ * link_detect_connection_type() - Determine if there is a sink connected
  *
  * @type: Returned connection type
  * Does not detect downstream devices, such as MST sinks
@@ -1216,7 +1210,7 @@ bool link_detect_connection_type(struct dc_link *link, enum dc_connection_type *
 
 	/* Link may not have physical HPD pin. */
 	if (link->ep_type != DISPLAY_ENDPOINT_PHY) {
-		if (link->is_hpd_pending || !dc_link_dpia_query_hpd_status(link))
+		if (link->is_hpd_pending || !dpia_query_hpd_status(link))
 			*type = dc_connection_none;
 		else
 			*type = dc_connection_single;
@@ -1233,6 +1227,11 @@ bool link_detect_connection_type(struct dc_link *link, enum dc_connection_type *
 		/* TODO: need to do the actual detection */
 	} else {
 		*type = dc_connection_none;
+		if (link->connector_signal == SIGNAL_TYPE_EDP) {
+			/* eDP is not connected, power down it */
+			if (!link->dc->config.edp_no_power_sequencing)
+				link->dc->hwss.edp_power_control(link, false);
+		}
 	}
 
 	return true;
@@ -1274,7 +1273,6 @@ void link_clear_dprx_states(struct dc_link *link)
 {
 	memset(&link->dprx_states, 0, sizeof(link->dprx_states));
 }
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 
 bool link_is_hdcp14(struct dc_link *link, enum signal_type signal)
 {
@@ -1322,10 +1320,108 @@ bool link_is_hdcp22(struct dc_link *link, enum signal_type signal)
 
 	return ret;
 }
-#endif // CONFIG_DRM_AMD_DC_HDCP
 
 const struct dc_link_status *link_get_status(const struct dc_link *link)
 {
 	return &link->link_status;
 }
 
+
+static bool link_add_remote_sink_helper(struct dc_link *dc_link, struct dc_sink *sink)
+{
+	if (dc_link->sink_count >= MAX_SINKS_PER_LINK) {
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	dc_sink_retain(sink);
+
+	dc_link->remote_sinks[dc_link->sink_count] = sink;
+	dc_link->sink_count++;
+
+	return true;
+}
+
+struct dc_sink *link_add_remote_sink(
+		struct dc_link *link,
+		const uint8_t *edid,
+		int len,
+		struct dc_sink_init_data *init_data)
+{
+	struct dc_sink *dc_sink;
+	enum dc_edid_status edid_status;
+
+	if (len > DC_MAX_EDID_BUFFER_SIZE) {
+		dm_error("Max EDID buffer size breached!\n");
+		return NULL;
+	}
+
+	if (!init_data) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	if (!init_data->link) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dc_sink = dc_sink_create(init_data);
+
+	if (!dc_sink)
+		return NULL;
+
+	memmove(dc_sink->dc_edid.raw_edid, edid, len);
+	dc_sink->dc_edid.length = len;
+
+	if (!link_add_remote_sink_helper(
+			link,
+			dc_sink))
+		goto fail_add_sink;
+
+	edid_status = dm_helpers_parse_edid_caps(
+			link,
+			&dc_sink->dc_edid,
+			&dc_sink->edid_caps);
+
+	/*
+	 * Treat device as no EDID device if EDID
+	 * parsing fails
+	 */
+	if (edid_status != EDID_OK && edid_status != EDID_PARTIAL_VALID) {
+		dc_sink->dc_edid.length = 0;
+		dm_error("Bad EDID, status%d!\n", edid_status);
+	}
+
+	return dc_sink;
+
+fail_add_sink:
+	dc_sink_release(dc_sink);
+	return NULL;
+}
+
+void link_remove_remote_sink(struct dc_link *link, struct dc_sink *sink)
+{
+	int i;
+
+	if (!link->sink_count) {
+		BREAK_TO_DEBUGGER();
+		return;
+	}
+
+	for (i = 0; i < link->sink_count; i++) {
+		if (link->remote_sinks[i] == sink) {
+			dc_sink_release(sink);
+			link->remote_sinks[i] = NULL;
+
+			/* shrink array to remove empty place */
+			while (i < link->sink_count - 1) {
+				link->remote_sinks[i] = link->remote_sinks[i+1];
+				i++;
+			}
+			link->remote_sinks[i] = NULL;
+			link->sink_count--;
+			return;
+		}
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.h b/drivers/gpu/drm/amd/display/dc/link/link_detection.h
index 1831636516fb..7da05078721e 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_detection.h
+++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.h
@@ -26,5 +26,18 @@
 #ifndef __DC_LINK_DETECTION_H__
 #define __DC_LINK_DETECTION_H__
 #include "link.h"
-
+bool link_detect(struct dc_link *link, enum dc_detect_reason reason);
+bool link_detect_connection_type(struct dc_link *link,
+		enum dc_connection_type *type);
+struct dc_sink *link_add_remote_sink(
+		struct dc_link *link,
+		const uint8_t *edid,
+		int len,
+		struct dc_sink_init_data *init_data);
+void link_remove_remote_sink(struct dc_link *link, struct dc_sink *sink);
+bool link_reset_cur_dp_mst_topology(struct dc_link *link);
+const struct dc_link_status *link_get_status(const struct dc_link *link);
+bool link_is_hdcp14(struct dc_link *link, enum signal_type signal);
+bool link_is_hdcp22(struct dc_link *link, enum signal_type signal);
+void link_clear_dprx_states(struct dc_link *link);
 #endif /* __DC_LINK_DETECTION_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
index 257e1c3ba00a..027ad1f0144d 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
@@ -37,6 +37,7 @@
 
 #include "link_dpms.h"
 #include "link_hwss.h"
+#include "link_validation.h"
 #include "accessories/link_fpga.h"
 #include "accessories/link_dp_trace.h"
 #include "protocols/link_dpcd.h"
@@ -46,6 +47,7 @@
 #include "protocols/link_dp_capability.h"
 #include "protocols/link_dp_training.h"
 #include "protocols/link_edp_panel_control.h"
+#include "protocols/link_dp_dpia_bw.h"
 
 #include "dm_helpers.h"
 #include "link_enc_cfg.h"
@@ -136,7 +138,7 @@ void link_blank_dp_stream(struct dc_link *link, bool hw_init)
 		}
 
 		if ((!link->wa_flags.dp_keep_receiver_powered) || hw_init)
-			dc_link_dp_receiver_power_ctrl(link, false);
+			dpcd_write_rx_power_ctrl(link, false);
 	}
 }
 
@@ -646,7 +648,6 @@ static void write_i2c_redriver_setting(
 	if (!i2c_success)
 		DC_LOG_DEBUG("Set redriver failed");
 }
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 
 static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off)
 {
@@ -672,7 +673,7 @@ static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off)
 
 	/* stream encoder index */
 	config.stream_enc_idx = pipe_ctx->stream_res.stream_enc->id - ENGINE_ID_DIGA;
-	if (link_is_dp_128b_132b_signal(pipe_ctx))
+	if (dp_is_128b_132b_signal(pipe_ctx))
 		config.stream_enc_idx =
 				pipe_ctx->stream_res.hpo_dp_stream_enc->id - ENGINE_ID_HPO_DP_0;
 
@@ -681,7 +682,7 @@ static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off)
 
 	/* link encoder index */
 	config.link_enc_idx = link_enc->transmitter - TRANSMITTER_UNIPHY_A;
-	if (link_is_dp_128b_132b_signal(pipe_ctx))
+	if (dp_is_128b_132b_signal(pipe_ctx))
 		config.link_enc_idx = pipe_ctx->link_res.hpo_dp_link_enc->inst;
 
 	/* dio output index is dpia index for DPIA endpoint & dcio index by default */
@@ -702,7 +703,7 @@ static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off)
 	config.assr_enabled = (panel_mode == DP_PANEL_MODE_EDP) ? 1 : 0;
 	config.mst_enabled = (pipe_ctx->stream->signal ==
 			SIGNAL_TYPE_DISPLAY_PORT_MST) ? 1 : 0;
-	config.dp2_enabled = link_is_dp_128b_132b_signal(pipe_ctx) ? 1 : 0;
+	config.dp2_enabled = dp_is_128b_132b_signal(pipe_ctx) ? 1 : 0;
 	config.usb4_enabled = (pipe_ctx->stream->link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA) ?
 			1 : 0;
 	config.dpms_off = dpms_off;
@@ -712,7 +713,6 @@ static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off)
 
 	cp_psp->funcs.update_stream_config(cp_psp->handle, &config);
 }
-#endif
 
 static void set_avmute(struct pipe_ctx *pipe_ctx, bool enable)
 {
@@ -817,7 +817,7 @@ void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 
 		/* Enable DSC in encoder */
 		if (dc_is_dp_signal(stream->signal) && !IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)
-				&& !link_is_dp_128b_132b_signal(pipe_ctx)) {
+				&& !dp_is_128b_132b_signal(pipe_ctx)) {
 			DC_LOG_DSC("Setting stream encoder DSC config for engine %d:", (int)pipe_ctx->stream_res.stream_enc->id);
 			dsc_optc_config_log(dsc, &dsc_optc_cfg);
 			pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config(pipe_ctx->stream_res.stream_enc,
@@ -843,7 +843,7 @@ void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 
 		/* disable DSC in stream encoder */
 		if (dc_is_dp_signal(stream->signal)) {
-			if (link_is_dp_128b_132b_signal(pipe_ctx))
+			if (dp_is_128b_132b_signal(pipe_ctx))
 				pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_set_dsc_pps_info_packet(
 										pipe_ctx->stream_res.hpo_dp_stream_enc,
 										false,
@@ -902,7 +902,7 @@ bool link_set_dsc_pps_packet(struct pipe_ctx *pipe_ctx, bool enable, bool immedi
 		memcpy(&stream->dsc_packed_pps[0], &dsc_packed_pps[0], sizeof(stream->dsc_packed_pps));
 		if (dc_is_dp_signal(stream->signal)) {
 			DC_LOG_DSC("Setting stream encoder DSC PPS SDP for engine %d\n", (int)pipe_ctx->stream_res.stream_enc->id);
-			if (link_is_dp_128b_132b_signal(pipe_ctx))
+			if (dp_is_128b_132b_signal(pipe_ctx))
 				pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_set_dsc_pps_info_packet(
 										pipe_ctx->stream_res.hpo_dp_stream_enc,
 										true,
@@ -919,7 +919,7 @@ bool link_set_dsc_pps_packet(struct pipe_ctx *pipe_ctx, bool enable, bool immedi
 		/* disable DSC PPS in stream encoder */
 		memset(&stream->dsc_packed_pps[0], 0, sizeof(stream->dsc_packed_pps));
 		if (dc_is_dp_signal(stream->signal)) {
-			if (link_is_dp_128b_132b_signal(pipe_ctx))
+			if (dp_is_128b_132b_signal(pipe_ctx))
 				pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_set_dsc_pps_info_packet(
 										pipe_ctx->stream_res.hpo_dp_stream_enc,
 										false,
@@ -1001,7 +1001,7 @@ static void enable_stream_features(struct pipe_ctx *pipe_ctx)
 	}
 }
 
-static void dc_log_vcp_x_y(const struct dc_link *link, struct fixed31_32 avg_time_slots_per_mtp)
+static void log_vcp_x_y(const struct dc_link *link, struct fixed31_32 avg_time_slots_per_mtp)
 {
 	const uint32_t VCP_Y_PRECISION = 1000;
 	uint64_t vcp_x, vcp_y;
@@ -1044,7 +1044,7 @@ static void dc_log_vcp_x_y(const struct dc_link *link, struct fixed31_32 avg_tim
 static struct fixed31_32 get_pbn_per_slot(struct dc_stream_state *stream)
 {
 	struct fixed31_32 mbytes_per_sec;
-	uint32_t link_rate_in_mbytes_per_sec = dc_link_bandwidth_kbps(stream->link,
+	uint32_t link_rate_in_mbytes_per_sec = dp_link_bandwidth_kbps(stream->link,
 			&stream->link->cur_link_settings);
 	link_rate_in_mbytes_per_sec /= 8000; /* Kbits to MBytes */
 
@@ -1153,7 +1153,7 @@ static bool poll_for_allocation_change_trigger(struct dc_link *link)
 			break;
 		}
 
-		msleep(5);
+		fsleep(5000);
 	}
 
 	if (result == ACT_FAILED) {
@@ -1517,7 +1517,7 @@ static enum dc_status allocate_mst_payload(struct pipe_ctx *pipe_ctx)
 	pbn = get_pbn_from_timing(pipe_ctx);
 	avg_time_slots_per_mtp = dc_fixpt_div(pbn, pbn_per_slot);
 
-	dc_log_vcp_x_y(link, avg_time_slots_per_mtp);
+	log_vcp_x_y(link, avg_time_slots_per_mtp);
 
 	if (link_hwss->ext.set_throttled_vcp_size)
 		link_hwss->ext.set_throttled_vcp_size(pipe_ctx, avg_time_slots_per_mtp);
@@ -1535,7 +1535,7 @@ struct fixed31_32 link_calculate_sst_avg_time_slots_per_mtp(
 {
 	struct fixed31_32 link_bw_effective =
 			dc_fixpt_from_int(
-					dc_link_bandwidth_kbps(link, &link->cur_link_settings));
+					dp_link_bandwidth_kbps(link, &link->cur_link_settings));
 	struct fixed31_32 timeslot_bw_effective =
 			dc_fixpt_div_int(link_bw_effective, MAX_MTP_SLOT_COUNT);
 	struct fixed31_32 timing_bw =
@@ -1640,7 +1640,7 @@ static bool write_128b_132b_sst_payload_allocation_table(
 			}
 		}
 		retries++;
-		msleep(5);
+		fsleep(5000);
 	}
 
 	if (!result && retries == max_retries) {
@@ -1670,7 +1670,7 @@ static enum dc_status update_sst_payload(struct pipe_ctx *pipe_ctx,
 	if (!allocate) {
 		avg_time_slots_per_mtp = dc_fixpt_from_int(0);
 
-		dc_log_vcp_x_y(link, avg_time_slots_per_mtp);
+		log_vcp_x_y(link, avg_time_slots_per_mtp);
 
 		if (link_hwss->ext.set_throttled_vcp_size)
 			link_hwss->ext.set_throttled_vcp_size(pipe_ctx,
@@ -1721,7 +1721,7 @@ static enum dc_status update_sst_payload(struct pipe_ctx *pipe_ctx,
 			DP_128b_132b_ENCODING) {
 		avg_time_slots_per_mtp = link_calculate_sst_avg_time_slots_per_mtp(stream, link);
 
-		dc_log_vcp_x_y(link, avg_time_slots_per_mtp);
+		log_vcp_x_y(link, avg_time_slots_per_mtp);
 
 		if (link_hwss->ext.set_throttled_vcp_size)
 			link_hwss->ext.set_throttled_vcp_size(pipe_ctx,
@@ -2035,6 +2035,12 @@ static enum dc_status enable_link_dp(struct dc_state *state,
 	uint32_t post_oui_delay = 30; // 30ms
 	/* Reduce link bandwidth between failed link training attempts. */
 	bool do_fallback = false;
+	int lt_attempts = LINK_TRAINING_ATTEMPTS;
+
+	// Increase retry count if attempting DP1.x on FIXED_VS link
+	if ((link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) &&
+			link_dp_get_encoding_format(link_settings) == DP_8b_10b_ENCODING)
+		lt_attempts = 10;
 
 	// check for seamless boot
 	for (i = 0; i < state->stream_count; i++) {
@@ -2044,11 +2050,17 @@ static enum dc_status enable_link_dp(struct dc_state *state,
 		}
 	}
 
-	/* Train with fallback when enabling DPIA link. Conventional links are
+	/*
+	 * If the link is DP-over-USB4 do the following:
+	 * - Train with fallback when enabling DPIA link. Conventional links are
 	 * trained with fallback during sink detection.
+	 * - Allocate only what the stream needs for bw in Gbps. Inform the CM
+	 * in case stream needs more or less bw from what has been allocated
+	 * earlier at plug time.
 	 */
-	if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA)
+	if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA) {
 		do_fallback = true;
+	}
 
 	/*
 	 * Temporary w/a to get DP2.0 link rates to work with SST.
@@ -2093,7 +2105,7 @@ static enum dc_status enable_link_dp(struct dc_state *state,
 
 	if (perform_link_training_with_retries(link_settings,
 					       skip_video_pattern,
-					       LINK_TRAINING_ATTEMPTS,
+					       lt_attempts,
 					       pipe_ctx,
 					       pipe_ctx->stream->signal,
 					       do_fallback)) {
@@ -2117,7 +2129,7 @@ static enum dc_status enable_link_dp(struct dc_state *state,
 		set_default_brightness_aux(link); // TODO: use cached if known
 		if (link->dpcd_sink_ext_caps.bits.oled == 1)
 			msleep(bl_oled_enable_delay);
-		link_backlight_enable_aux(link, true);
+		edp_backlight_enable_aux(link, true);
 	}
 
 	return status;
@@ -2154,6 +2166,7 @@ static enum dc_status enable_link_dp_mst(
 		struct pipe_ctx *pipe_ctx)
 {
 	struct dc_link *link = pipe_ctx->stream->link;
+	unsigned char mstm_cntl;
 
 	/* sink signal type after MST branch is MST. Multiple MST sinks
 	 * share one link. Link DP PHY is enable or training only once.
@@ -2162,7 +2175,9 @@ static enum dc_status enable_link_dp_mst(
 		return DC_OK;
 
 	/* clear payload table */
-	dm_helpers_dp_mst_clear_payload_allocation_table(link->ctx, link);
+	core_link_read_dpcd(link, DP_MSTM_CTRL, &mstm_cntl, 1);
+	if (mstm_cntl & DP_MST_EN)
+		dm_helpers_dp_mst_clear_payload_allocation_table(link->ctx, link);
 
 	/* to make sure the pending down rep can be processed
 	 * before enabling the link
@@ -2237,7 +2252,7 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 
 	ASSERT(is_master_pipe_for_link(link, pipe_ctx));
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx))
+	if (dp_is_128b_132b_signal(pipe_ctx))
 		vpg = pipe_ctx->stream_res.hpo_dp_stream_enc->vpg;
 
 	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
@@ -2262,15 +2277,13 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 
 	dc->hwss.disable_audio_stream(pipe_ctx);
 
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 	update_psp_stream_config(pipe_ctx, true);
-#endif
 	dc->hwss.blank_stream(pipe_ctx);
 
 	if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
 		deallocate_mst_payload(pipe_ctx);
 	else if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT &&
-			link_is_dp_128b_132b_signal(pipe_ctx))
+			dp_is_128b_132b_signal(pipe_ctx))
 		update_sst_payload(pipe_ctx, false);
 
 	if (dc_is_hdmi_signal(pipe_ctx->stream->signal)) {
@@ -2299,7 +2312,7 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 	}
 
 	if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT &&
-			!link_is_dp_128b_132b_signal(pipe_ctx)) {
+			!dp_is_128b_132b_signal(pipe_ctx)) {
 
 		/* In DP1.x SST mode, our encoder will go to TPS1
 		 * when link is on but stream is off.
@@ -2319,7 +2332,7 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 		if (dc_is_dp_signal(pipe_ctx->stream->signal))
 			link_set_dsc_enable(pipe_ctx, false);
 	}
-	if (link_is_dp_128b_132b_signal(pipe_ctx)) {
+	if (dp_is_128b_132b_signal(pipe_ctx)) {
 		if (pipe_ctx->stream_res.tg->funcs->set_out_mux)
 			pipe_ctx->stream_res.tg->funcs->set_out_mux(pipe_ctx->stream_res.tg, OUT_MUX_DIO);
 	}
@@ -2343,7 +2356,7 @@ void link_set_dpms_on(
 
 	ASSERT(is_master_pipe_for_link(link, pipe_ctx));
 
-	if (link_is_dp_128b_132b_signal(pipe_ctx))
+	if (dp_is_128b_132b_signal(pipe_ctx))
 		vpg = pipe_ctx->stream_res.hpo_dp_stream_enc->vpg;
 
 	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
@@ -2365,7 +2378,7 @@ void link_set_dpms_on(
 	ASSERT(link_enc);
 
 	if (!dc_is_virtual_signal(pipe_ctx->stream->signal)
-			&& !link_is_dp_128b_132b_signal(pipe_ctx)) {
+			&& !dp_is_128b_132b_signal(pipe_ctx)) {
 		if (link_enc)
 			link_enc->funcs->setup(
 				link_enc,
@@ -2375,7 +2388,7 @@ void link_set_dpms_on(
 	pipe_ctx->stream->link->link_state_valid = true;
 
 	if (pipe_ctx->stream_res.tg->funcs->set_out_mux) {
-		if (link_is_dp_128b_132b_signal(pipe_ctx))
+		if (dp_is_128b_132b_signal(pipe_ctx))
 			otg_out_dest = OUT_MUX_HPO_DP;
 		else
 			otg_out_dest = OUT_MUX_DIO;
@@ -2398,7 +2411,7 @@ void link_set_dpms_on(
 		dc->hwss.update_info_frame(pipe_ctx);
 
 		if (dc_is_dp_signal(pipe_ctx->stream->signal))
-			link_dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
+			dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
 
 		/* Do not touch link on seamless boot optimization. */
 		if (pipe_ctx->stream->apply_seamless_boot_optimization) {
@@ -2410,9 +2423,7 @@ void link_set_dpms_on(
 				dc->hwss.enable_audio_stream(pipe_ctx);
 			}
 
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 			update_psp_stream_config(pipe_ctx, false);
-#endif
 			return;
 		}
 
@@ -2422,9 +2433,7 @@ void link_set_dpms_on(
 					!pipe_ctx->stream->timing.flags.DSC &&
 					!pipe_ctx->next_odm_pipe) {
 			pipe_ctx->stream->dpms_off = false;
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 			update_psp_stream_config(pipe_ctx, false);
-#endif
 			return;
 		}
 
@@ -2477,11 +2486,12 @@ void link_set_dpms_on(
 		 * from transmitter control.
 		 */
 		if (!(dc_is_virtual_signal(pipe_ctx->stream->signal) ||
-				link_is_dp_128b_132b_signal(pipe_ctx)))
-			if (link_enc)
-				link_enc->funcs->setup(
-					link_enc,
-					pipe_ctx->stream->signal);
+				dp_is_128b_132b_signal(pipe_ctx))) {
+				if (link_enc)
+					link_enc->funcs->setup(
+						link_enc,
+						pipe_ctx->stream->signal);
+			}
 
 		dc->hwss.enable_stream(pipe_ctx);
 
@@ -2497,7 +2507,7 @@ void link_set_dpms_on(
 		if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
 			allocate_mst_payload(pipe_ctx);
 		else if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT &&
-				link_is_dp_128b_132b_signal(pipe_ctx))
+				dp_is_128b_132b_signal(pipe_ctx))
 			update_sst_payload(pipe_ctx, true);
 
 		dc->hwss.unblank_stream(pipe_ctx,
@@ -2508,14 +2518,12 @@ void link_set_dpms_on(
 
 		if (dc_is_dp_signal(pipe_ctx->stream->signal))
 			enable_stream_features(pipe_ctx);
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
 		update_psp_stream_config(pipe_ctx, false);
-#endif
 
 		dc->hwss.enable_audio_stream(pipe_ctx);
 
 	} else { // if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
-		if (link_is_dp_128b_132b_signal(pipe_ctx))
+		if (dp_is_128b_132b_signal(pipe_ctx))
 			dp_fpga_hpo_enable_link_and_stream(state, pipe_ctx);
 		if (dc_is_dp_signal(pipe_ctx->stream->signal) ||
 				dc_is_virtual_signal(pipe_ctx->stream->signal))
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.h b/drivers/gpu/drm/amd/display/dc/link/link_dpms.h
index 33d312dabdb8..9398f9c1666a 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.h
+++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.h
@@ -27,14 +27,27 @@
 #define __DC_LINK_DPMS_H__
 
 #include "link.h"
-bool link_set_dsc_pps_packet(struct pipe_ctx *pipe_ctx,
-		bool enable, bool immediate_update);
-struct fixed31_32 link_calculate_sst_avg_time_slots_per_mtp(
-		const struct dc_stream_state *stream,
-		const struct dc_link *link);
+void link_set_dpms_on(
+		struct dc_state *state,
+		struct pipe_ctx *pipe_ctx);
+void link_set_dpms_off(struct pipe_ctx *pipe_ctx);
+void link_resume(struct dc_link *link);
+void link_blank_all_dp_displays(struct dc *dc);
+void link_blank_all_edp_displays(struct dc *dc);
+void link_blank_dp_stream(struct dc_link *link, bool hw_init);
 void link_set_all_streams_dpms_off_for_link(struct dc_link *link);
 void link_get_master_pipes_with_dpms_on(const struct dc_link *link,
 		struct dc_state *state,
 		uint8_t *count,
 		struct pipe_ctx *pipes[MAX_PIPES]);
+enum dc_status link_increase_mst_payload(struct pipe_ctx *pipe_ctx, uint32_t req_pbn);
+enum dc_status link_reduce_mst_payload(struct pipe_ctx *pipe_ctx, uint32_t req_pbn);
+bool link_set_dsc_pps_packet(struct pipe_ctx *pipe_ctx,
+		bool enable, bool immediate_update);
+struct fixed31_32 link_calculate_sst_avg_time_slots_per_mtp(
+		const struct dc_stream_state *stream,
+		const struct dc_link *link);
+void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable);
+bool link_set_dsc_enable(struct pipe_ctx *pipe_ctx, bool enable);
+bool link_update_dsc_config(struct pipe_ctx *pipe_ctx);
 #endif /* __DC_LINK_DPMS_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
index aeb26a4d539e..1515c817f03b 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
@@ -27,7 +27,20 @@
  * This file owns the creation/destruction of link structure.
  */
 #include "link_factory.h"
+#include "link_detection.h"
+#include "link_resource.h"
+#include "link_validation.h"
+#include "link_dpms.h"
+#include "accessories/link_dp_cts.h"
+#include "accessories/link_dp_trace.h"
+#include "accessories/link_fpga.h"
 #include "protocols/link_ddc.h"
+#include "protocols/link_dp_capability.h"
+#include "protocols/link_dp_dpia_bw.h"
+#include "protocols/link_dp_dpia.h"
+#include "protocols/link_dp_irq_handler.h"
+#include "protocols/link_dp_phy.h"
+#include "protocols/link_dp_training.h"
 #include "protocols/link_edp_panel_control.h"
 #include "protocols/link_hpd.h"
 #include "gpio_service_interface.h"
@@ -39,7 +52,249 @@
 	DC_LOG_HW_HOTPLUG(  \
 		__VA_ARGS__)
 
-static enum transmitter translate_encoder_to_transmitter(struct graphics_object_id encoder)
+/* link factory owns the creation/destruction of link structures. */
+static void construct_link_service_factory(struct link_service *link_srv)
+{
+
+	link_srv->create_link = link_create;
+	link_srv->destroy_link = link_destroy;
+}
+
+/* link_detection manages link detection states and receiver states by using
+ * various link protocols. It also provides helper functions to interpret
+ * certain capabilities or status based on the states it manages or retrieve
+ * them directly from connected receivers.
+ */
+static void construct_link_service_detection(struct link_service *link_srv)
+{
+	link_srv->detect_link = link_detect;
+	link_srv->detect_connection_type = link_detect_connection_type;
+	link_srv->add_remote_sink = link_add_remote_sink;
+	link_srv->remove_remote_sink = link_remove_remote_sink;
+	link_srv->get_hpd_state = link_get_hpd_state;
+	link_srv->get_hpd_gpio = link_get_hpd_gpio;
+	link_srv->enable_hpd = link_enable_hpd;
+	link_srv->disable_hpd = link_disable_hpd;
+	link_srv->enable_hpd_filter = link_enable_hpd_filter;
+	link_srv->reset_cur_dp_mst_topology = link_reset_cur_dp_mst_topology;
+	link_srv->get_status = link_get_status;
+	link_srv->is_hdcp1x_supported = link_is_hdcp14;
+	link_srv->is_hdcp2x_supported = link_is_hdcp22;
+	link_srv->clear_dprx_states = link_clear_dprx_states;
+}
+
+/* link resource implements accessors to link resource. */
+static void construct_link_service_resource(struct link_service *link_srv)
+{
+	link_srv->get_cur_res_map = link_get_cur_res_map;
+	link_srv->restore_res_map = link_restore_res_map;
+	link_srv->get_cur_link_res = link_get_cur_link_res;
+}
+
+/* link validation owns timing validation against various link limitations. (ex.
+ * link bandwidth, receiver capability or our hardware capability) It also
+ * provides helper functions exposing bandwidth formulas used in validation.
+ */
+static void construct_link_service_validation(struct link_service *link_srv)
+{
+	link_srv->validate_mode_timing = link_validate_mode_timing;
+	link_srv->dp_link_bandwidth_kbps = dp_link_bandwidth_kbps;
+	link_srv->validate_dpia_bandwidth = link_validate_dpia_bandwidth;
+}
+
+/* link dpms owns the programming sequence of stream's dpms state associated
+ * with the link and link's enable/disable sequences as result of the stream's
+ * dpms state change.
+ */
+static void construct_link_service_dpms(struct link_service *link_srv)
+{
+	link_srv->set_dpms_on = link_set_dpms_on;
+	link_srv->set_dpms_off = link_set_dpms_off;
+	link_srv->resume = link_resume;
+	link_srv->blank_all_dp_displays = link_blank_all_dp_displays;
+	link_srv->blank_all_edp_displays = link_blank_all_edp_displays;
+	link_srv->blank_dp_stream = link_blank_dp_stream;
+	link_srv->increase_mst_payload = link_increase_mst_payload;
+	link_srv->reduce_mst_payload = link_reduce_mst_payload;
+	link_srv->set_dsc_on_stream = link_set_dsc_on_stream;
+	link_srv->set_dsc_enable = link_set_dsc_enable;
+	link_srv->update_dsc_config = link_update_dsc_config;
+}
+
+/* link ddc implements generic display communication protocols such as i2c, aux
+ * and scdc. It should not contain any specific applications of these
+ * protocols such as display capability query, detection, or handshaking such as
+ * link training.
+ */
+static void construct_link_service_ddc(struct link_service *link_srv)
+{
+	link_srv->create_ddc_service = link_create_ddc_service;
+	link_srv->destroy_ddc_service = link_destroy_ddc_service;
+	link_srv->query_ddc_data = link_query_ddc_data;
+	link_srv->aux_transfer_raw = link_aux_transfer_raw;
+	link_srv->aux_transfer_with_retries_no_mutex =
+			link_aux_transfer_with_retries_no_mutex;
+	link_srv->is_in_aux_transaction_mode = link_is_in_aux_transaction_mode;
+	link_srv->get_aux_defer_delay = link_get_aux_defer_delay;
+}
+
+/* link dp capability implements dp specific link capability retrieval sequence.
+ * It is responsible for retrieving, parsing, overriding, deciding capability
+ * obtained from dp link. Link capability consists of encoders, DPRXs, cables,
+ * retimers, usb and all other possible backend capabilities.
+ */
+static void construct_link_service_dp_capability(struct link_service *link_srv)
+{
+	link_srv->dp_is_sink_present = dp_is_sink_present;
+	link_srv->dp_is_fec_supported = dp_is_fec_supported;
+	link_srv->dp_is_128b_132b_signal = dp_is_128b_132b_signal;
+	link_srv->dp_get_max_link_enc_cap = dp_get_max_link_enc_cap;
+	link_srv->dp_get_verified_link_cap = dp_get_verified_link_cap;
+	link_srv->dp_get_encoding_format = link_dp_get_encoding_format;
+	link_srv->dp_should_enable_fec = dp_should_enable_fec;
+	link_srv->dp_decide_link_settings = link_decide_link_settings;
+	link_srv->mst_decide_link_encoding_format =
+			mst_decide_link_encoding_format;
+	link_srv->edp_decide_link_settings = edp_decide_link_settings;
+	link_srv->bw_kbps_from_raw_frl_link_rate_data =
+			link_bw_kbps_from_raw_frl_link_rate_data;
+	link_srv->dp_overwrite_extended_receiver_cap =
+			dp_overwrite_extended_receiver_cap;
+	link_srv->dp_decide_lttpr_mode = dp_decide_lttpr_mode;
+}
+
+/* link dp phy/dpia implements basic dp phy/dpia functionality such as
+ * enable/disable output and set lane/drive settings. It is responsible for
+ * maintaining and update software state representing current phy/dpia status
+ * such as current link settings.
+ */
+static void construct_link_service_dp_phy_or_dpia(struct link_service *link_srv)
+{
+	link_srv->dpia_handle_usb4_bandwidth_allocation_for_link =
+			dpia_handle_usb4_bandwidth_allocation_for_link;
+	link_srv->dpia_handle_bw_alloc_response = dpia_handle_bw_alloc_response;
+	link_srv->dp_set_drive_settings = dp_set_drive_settings;
+	link_srv->dpcd_write_rx_power_ctrl = dpcd_write_rx_power_ctrl;
+}
+
+/* link dp irq handler implements DP HPD short pulse handling sequence according
+ * to DP specifications
+ */
+static void construct_link_service_dp_irq_handler(struct link_service *link_srv)
+{
+	link_srv->dp_parse_link_loss_status = dp_parse_link_loss_status;
+	link_srv->dp_should_allow_hpd_rx_irq = dp_should_allow_hpd_rx_irq;
+	link_srv->dp_handle_link_loss = dp_handle_link_loss;
+	link_srv->dp_read_hpd_rx_irq_data = dp_read_hpd_rx_irq_data;
+	link_srv->dp_handle_hpd_rx_irq = dp_handle_hpd_rx_irq;
+}
+
+/* link edp panel control implements retrieval and configuration of eDP panel
+ * features such as PSR and ABM and it also manages specs defined eDP panel
+ * power sequences.
+ */
+static void construct_link_service_edp_panel_control(struct link_service *link_srv)
+{
+	link_srv->edp_panel_backlight_power_on = edp_panel_backlight_power_on;
+	link_srv->edp_get_backlight_level = edp_get_backlight_level;
+	link_srv->edp_get_backlight_level_nits = edp_get_backlight_level_nits;
+	link_srv->edp_set_backlight_level = edp_set_backlight_level;
+	link_srv->edp_set_backlight_level_nits = edp_set_backlight_level_nits;
+	link_srv->edp_get_target_backlight_pwm = edp_get_target_backlight_pwm;
+	link_srv->edp_get_psr_state = edp_get_psr_state;
+	link_srv->edp_set_psr_allow_active = edp_set_psr_allow_active;
+	link_srv->edp_setup_psr = edp_setup_psr;
+	link_srv->edp_set_sink_vtotal_in_psr_active =
+			edp_set_sink_vtotal_in_psr_active;
+	link_srv->edp_get_psr_residency = edp_get_psr_residency;
+	link_srv->edp_wait_for_t12 = edp_wait_for_t12;
+	link_srv->edp_is_ilr_optimization_required =
+			edp_is_ilr_optimization_required;
+	link_srv->edp_backlight_enable_aux = edp_backlight_enable_aux;
+	link_srv->edp_add_delay_for_T9 = edp_add_delay_for_T9;
+	link_srv->edp_receiver_ready_T9 = edp_receiver_ready_T9;
+	link_srv->edp_receiver_ready_T7 = edp_receiver_ready_T7;
+	link_srv->edp_power_alpm_dpcd_enable = edp_power_alpm_dpcd_enable;
+}
+
+/* link dp cts implements dp compliance test automation protocols and manual
+ * testing interfaces for debugging and certification purpose.
+ */
+static void construct_link_service_dp_cts(struct link_service *link_srv)
+{
+	link_srv->dp_handle_automated_test = dp_handle_automated_test;
+	link_srv->dp_set_test_pattern = dp_set_test_pattern;
+	link_srv->dp_set_preferred_link_settings =
+			dp_set_preferred_link_settings;
+	link_srv->dp_set_preferred_training_settings =
+			dp_set_preferred_training_settings;
+}
+
+/* link dp trace implements tracing interfaces for tracking major dp sequences
+ * including execution status and timestamps
+ */
+static void construct_link_service_dp_trace(struct link_service *link_srv)
+{
+	link_srv->dp_trace_is_initialized = dp_trace_is_initialized;
+	link_srv->dp_trace_set_is_logged_flag = dp_trace_set_is_logged_flag;
+	link_srv->dp_trace_is_logged = dp_trace_is_logged;
+	link_srv->dp_trace_get_lt_end_timestamp = dp_trace_get_lt_end_timestamp;
+	link_srv->dp_trace_get_lt_counts = dp_trace_get_lt_counts;
+	link_srv->dp_trace_get_link_loss_count = dp_trace_get_link_loss_count;
+	link_srv->dp_trace_set_edp_power_timestamp =
+			dp_trace_set_edp_power_timestamp;
+	link_srv->dp_trace_get_edp_poweron_timestamp =
+			dp_trace_get_edp_poweron_timestamp;
+	link_srv->dp_trace_get_edp_poweroff_timestamp =
+			dp_trace_get_edp_poweroff_timestamp;
+	link_srv->dp_trace_source_sequence = dp_trace_source_sequence;
+}
+
+static void construct_link_service(struct link_service *link_srv)
+{
+	/* All link service functions should fall under some sub categories.
+	 * If a new function doesn't perfectly fall under an existing sub
+	 * category, it must be that you are either adding a whole new aspect of
+	 * responsibility to link service or something doesn't belong to link
+	 * service. In that case please contact the arch owner to arrange a
+	 * design review meeting.
+	 */
+	construct_link_service_factory(link_srv);
+	construct_link_service_detection(link_srv);
+	construct_link_service_resource(link_srv);
+	construct_link_service_validation(link_srv);
+	construct_link_service_dpms(link_srv);
+	construct_link_service_ddc(link_srv);
+	construct_link_service_dp_capability(link_srv);
+	construct_link_service_dp_phy_or_dpia(link_srv);
+	construct_link_service_dp_irq_handler(link_srv);
+	construct_link_service_edp_panel_control(link_srv);
+	construct_link_service_dp_cts(link_srv);
+	construct_link_service_dp_trace(link_srv);
+}
+
+struct link_service *link_create_link_service(void)
+{
+	struct link_service *link_srv = kzalloc(sizeof(*link_srv), GFP_KERNEL);
+
+	if (link_srv == NULL)
+		goto fail;
+
+	construct_link_service(link_srv);
+
+	return link_srv;
+fail:
+	return NULL;
+}
+
+void link_destroy_link_service(struct link_service **link_srv)
+{
+	kfree(*link_srv);
+	*link_srv = NULL;
+}
+
+static enum transmitter translate_encoder_to_transmitter(
+		struct graphics_object_id encoder)
 {
 	switch (encoder.id) {
 	case ENCODER_ID_INTERNAL_UNIPHY:
@@ -181,7 +436,7 @@ static enum channel_id get_ddc_line(struct dc_link *link)
 	return channel;
 }
 
-static bool dc_link_construct_phy(struct dc_link *link,
+static bool construct_phy(struct dc_link *link,
 			      const struct link_init_data *init_params)
 {
 	uint8_t i;
@@ -274,14 +529,18 @@ static bool dc_link_construct_phy(struct dc_link *link,
 				link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
 
 			switch (link->dc->config.allow_edp_hotplug_detection) {
-			case 1: // only the 1st eDP handles hotplug
+			case HPD_EN_FOR_ALL_EDP:
+				link->irq_source_hpd_rx =
+						dal_irq_get_rx_source(link->hpd_gpio);
+				break;
+			case HPD_EN_FOR_PRIMARY_EDP_ONLY:
 				if (link->link_index == 0)
 					link->irq_source_hpd_rx =
 						dal_irq_get_rx_source(link->hpd_gpio);
 				else
 					link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
 				break;
-			case 2: // only the 2nd eDP handles hotplug
+			case HPD_EN_FOR_SECONDARY_EDP_ONLY:
 				if (link->link_index == 1)
 					link->irq_source_hpd_rx =
 						dal_irq_get_rx_source(link->hpd_gpio);
@@ -289,6 +548,7 @@ static bool dc_link_construct_phy(struct dc_link *link,
 					link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
 				break;
 			default:
+				link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
 				break;
 			}
 		}
@@ -473,7 +733,7 @@ create_fail:
 	return false;
 }
 
-static bool dc_link_construct_dpia(struct dc_link *link,
+static bool construct_dpia(struct dc_link *link,
 			      const struct link_init_data *init_params)
 {
 	struct ddc_service_init_data ddc_service_init_data = { 0 };
@@ -543,9 +803,9 @@ static bool link_construct(struct dc_link *link,
 {
 	/* Handle dpia case */
 	if (init_params->is_dpia_link == true)
-		return dc_link_construct_dpia(link, init_params);
+		return construct_dpia(link, init_params);
 	else
-		return dc_link_construct_phy(link, init_params);
+		return construct_phy(link, init_params);
 }
 
 struct dc_link *link_create(const struct link_init_data *init_params)
@@ -574,4 +834,3 @@ void link_destroy(struct dc_link **link)
 	kfree(*link);
 	*link = NULL;
 }
-
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.h b/drivers/gpu/drm/amd/display/dc/link/link_factory.h
index 5b846147c4a6..e96220d48d03 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_factory.h
+++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.h
@@ -25,5 +25,7 @@
 #ifndef __LINK_FACTORY_H__
 #define __LINK_FACTORY_H__
 #include "link.h"
+struct dc_link *link_create(const struct link_init_data *init_params);
+void link_destroy(struct dc_link **link);
 
 #endif /* __LINK_FACTORY_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_resource.h b/drivers/gpu/drm/amd/display/dc/link/link_resource.h
index 45554d30adf0..1907bda3cb6e 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/link/link_resource.h
@@ -25,7 +25,8 @@
 #ifndef __LINK_RESOURCE_H__
 #define __LINK_RESOURCE_H__
 #include "link.h"
+void link_get_cur_res_map(const struct dc *dc, uint32_t *map);
+void link_restore_res_map(const struct dc *dc, uint32_t *map);
 void link_get_cur_link_res(const struct dc_link *link,
 		struct link_resource *link_res);
-
 #endif /* __LINK_RESOURCE_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_validation.c b/drivers/gpu/drm/amd/display/dc/link/link_validation.c
index d4f6ee6ca948..d4b7da526f0a 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_validation.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_validation.c
@@ -29,6 +29,8 @@
  * provides helper functions exposing bandwidth formulas used in validation.
  */
 #include "link_validation.h"
+#include "protocols/link_dp_capability.h"
+#include "protocols/link_dp_dpia_bw.h"
 #include "resource.h"
 
 #define DC_LOGGER_INIT(logger)
@@ -123,7 +125,7 @@ static bool dp_active_dongle_validate_timing(
 		if (dongle_caps->dp_hdmi_frl_max_link_bw_in_kbps > 0) { // DP to HDMI FRL converter
 			struct dc_crtc_timing outputTiming = *timing;
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 			if (timing->flags.DSC && !timing->dsc_cfg.is_frl)
 				/* DP input has DSC, HDMI FRL output doesn't have DSC, remove DSC from output timing */
 				outputTiming.flags.DSC = 0;
@@ -233,7 +235,7 @@ uint32_t dp_link_bandwidth_kbps(
 		 */
 		link_rate_per_lane_kbps = link_settings->link_rate * LINK_RATE_REF_FREQ_IN_KHZ * BITS_PER_DP_BYTE;
 		total_data_bw_efficiency_x10000 = DATA_EFFICIENCY_8b_10b_x10000;
-		if (dc_link_should_enable_fec(link)) {
+		if (dp_should_enable_fec(link)) {
 			total_data_bw_efficiency_x10000 /= 100;
 			total_data_bw_efficiency_x10000 *= DATA_EFFICIENCY_8b_10b_FEC_EFFICIENCY_x100;
 		}
@@ -254,60 +256,6 @@ uint32_t dp_link_bandwidth_kbps(
 	return link_rate_per_lane_kbps * link_settings->lane_count / 10000 * total_data_bw_efficiency_x10000;
 }
 
-uint32_t link_timing_bandwidth_kbps(
-	const struct dc_crtc_timing *timing)
-{
-	uint32_t bits_per_channel = 0;
-	uint32_t kbps;
-
-#if defined(CONFIG_DRM_AMD_DC_DCN)
-	if (timing->flags.DSC)
-		return dc_dsc_stream_bandwidth_in_kbps(timing,
-				timing->dsc_cfg.bits_per_pixel,
-				timing->dsc_cfg.num_slices_h,
-				timing->dsc_cfg.is_dp);
-#endif /* CONFIG_DRM_AMD_DC_DCN */
-
-	switch (timing->display_color_depth) {
-	case COLOR_DEPTH_666:
-		bits_per_channel = 6;
-		break;
-	case COLOR_DEPTH_888:
-		bits_per_channel = 8;
-		break;
-	case COLOR_DEPTH_101010:
-		bits_per_channel = 10;
-		break;
-	case COLOR_DEPTH_121212:
-		bits_per_channel = 12;
-		break;
-	case COLOR_DEPTH_141414:
-		bits_per_channel = 14;
-		break;
-	case COLOR_DEPTH_161616:
-		bits_per_channel = 16;
-		break;
-	default:
-		ASSERT(bits_per_channel != 0);
-		bits_per_channel = 8;
-		break;
-	}
-
-	kbps = timing->pix_clk_100hz / 10;
-	kbps *= bits_per_channel;
-
-	if (timing->flags.Y_ONLY != 1) {
-		/*Only YOnly make reduce bandwidth by 1/3 compares to RGB*/
-		kbps *= 3;
-		if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
-			kbps /= 2;
-		else if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
-			kbps = kbps * 2 / 3;
-	}
-
-	return kbps;
-}
-
 static bool dp_validate_mode_timing(
 	struct dc_link *link,
 	const struct dc_crtc_timing *timing)
@@ -329,7 +277,7 @@ static bool dp_validate_mode_timing(
 		timing->v_addressable == (uint32_t) 480)
 		return true;
 
-	link_setting = dc_link_get_link_cap(link);
+	link_setting = dp_get_verified_link_cap(link);
 
 	/* TODO: DYNAMIC_VALIDATION needs to be implemented */
 	/*if (flags.DYNAMIC_VALIDATION == 1 &&
@@ -338,7 +286,7 @@ static bool dp_validate_mode_timing(
 	*/
 
 	req_bw = dc_bandwidth_in_kbps_from_timing(timing);
-	max_bw = dc_link_bandwidth_kbps(link, link_setting);
+	max_bw = dp_link_bandwidth_kbps(link, link_setting);
 
 	if (req_bw <= max_bw) {
 		/* remember the biggest mode here, during
@@ -396,3 +344,20 @@ enum dc_status link_validate_mode_timing(
 
 	return DC_OK;
 }
+
+bool link_validate_dpia_bandwidth(const struct dc_stream_state *stream, const unsigned int num_streams)
+{
+	bool ret = true;
+	int bw_needed[MAX_DPIA_NUM];
+	struct dc_link *link[MAX_DPIA_NUM];
+
+	if (!num_streams || num_streams > MAX_DPIA_NUM)
+		return ret;
+
+	for (uint8_t i = 0; i < num_streams; ++i) {
+
+		link[i] = stream[i].link;
+		bw_needed[i] = dc_bandwidth_in_kbps_from_timing(&stream[i].timing);
+	}
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_validation.h b/drivers/gpu/drm/amd/display/dc/link/link_validation.h
index ab6a44f50032..4a954317d0da 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_validation.h
+++ b/drivers/gpu/drm/amd/display/dc/link/link_validation.h
@@ -25,4 +25,15 @@
 #ifndef __LINK_VALIDATION_H__
 #define __LINK_VALIDATION_H__
 #include "link.h"
+enum dc_status link_validate_mode_timing(
+		const struct dc_stream_state *stream,
+		struct dc_link *link,
+		const struct dc_crtc_timing *timing);
+bool link_validate_dpia_bandwidth(
+		const struct dc_stream_state *stream,
+		const unsigned int num_streams);
+uint32_t dp_link_bandwidth_kbps(
+	const struct dc_link *link,
+	const struct dc_link_settings *link_settings);
+
 #endif /* __LINK_VALIDATION_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c
index 5269125bc2a4..0fa1228bc178 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c
@@ -53,7 +53,7 @@ struct aux_payloads {
 	struct vector payloads;
 };
 
-static bool dal_ddc_i2c_payloads_create(
+static bool i2c_payloads_create(
 		struct dc_context *ctx,
 		struct i2c_payloads *payloads,
 		uint32_t count)
@@ -65,16 +65,24 @@ static bool dal_ddc_i2c_payloads_create(
 	return false;
 }
 
-static struct i2c_payload *dal_ddc_i2c_payloads_get(struct i2c_payloads *p)
+static struct i2c_payload *i2c_payloads_get(struct i2c_payloads *p)
 {
 	return (struct i2c_payload *)p->payloads.container;
 }
 
-static uint32_t dal_ddc_i2c_payloads_get_count(struct i2c_payloads *p)
+static uint32_t i2c_payloads_get_count(struct i2c_payloads *p)
 {
 	return p->payloads.count;
 }
 
+static void i2c_payloads_destroy(struct i2c_payloads *p)
+{
+	if (!p)
+		return;
+
+	dal_vector_destruct(&p->payloads);
+}
+
 #define DDC_MIN(a, b) (((a) < (b)) ? (a) : (b))
 
 static void i2c_payloads_add(
@@ -364,10 +372,10 @@ bool link_query_ddc_data(
 		struct i2c_command command = {0};
 		struct i2c_payloads payloads;
 
-		if (!dal_ddc_i2c_payloads_create(ddc->ctx, &payloads, payloads_num))
+		if (!i2c_payloads_create(ddc->ctx, &payloads, payloads_num))
 			return false;
 
-		command.payloads = dal_ddc_i2c_payloads_get(&payloads);
+		command.payloads = i2c_payloads_get(&payloads);
 		command.number_of_payloads = 0;
 		command.engine = DDC_I2C_COMMAND_ENGINE;
 		command.speed = ddc->ctx->dc->caps.i2c_speed_in_khz;
@@ -379,20 +387,20 @@ bool link_query_ddc_data(
 			&payloads, address, read_size, read_buf, false);
 
 		command.number_of_payloads =
-			dal_ddc_i2c_payloads_get_count(&payloads);
+			i2c_payloads_get_count(&payloads);
 
 		success = dm_helpers_submit_i2c(
 				ddc->ctx,
 				ddc->link,
 				&command);
 
-		dal_vector_destruct(&payloads.payloads);
+		i2c_payloads_destroy(&payloads);
 	}
 
 	return success;
 }
 
-int dc_link_aux_transfer_raw(struct ddc_service *ddc,
+int link_aux_transfer_raw(struct ddc_service *ddc,
 		struct aux_payload *payload,
 		enum aux_return_code_type *operation_result)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.h
index aaa5064408ba..860ef15d7f1b 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.h
@@ -37,13 +37,41 @@
 
 #define EDID_SEGMENT_SIZE 256
 
+struct ddc_service *link_create_ddc_service(
+		struct ddc_service_init_data *ddc_init_data);
+
+void link_destroy_ddc_service(struct ddc_service **ddc);
+
 void set_ddc_transaction_type(
 		struct ddc_service *ddc,
 		enum ddc_transaction_type type);
 
+uint32_t link_get_aux_defer_delay(struct ddc_service *ddc);
+
+bool link_is_in_aux_transaction_mode(struct ddc_service *ddc);
+
 bool try_to_configure_aux_timeout(struct ddc_service *ddc,
 		uint32_t timeout);
 
+bool link_query_ddc_data(
+		struct ddc_service *ddc,
+		uint32_t address,
+		uint8_t *write_buf,
+		uint32_t write_size,
+		uint8_t *read_buf,
+		uint32_t read_size);
+
+/* Attempt to submit an aux payload, retrying on timeouts, defers, and busy
+ * states as outlined in the DP spec.  Returns true if the request was
+ * successful.
+ *
+ * NOTE: The function requires explicit mutex on DM side in order to prevent
+ * potential race condition. DC components should call the dpcd read/write
+ * function in dm_helpers in order to access dpcd safely
+ */
+bool link_aux_transfer_with_retries_no_mutex(struct ddc_service *ddc,
+		struct aux_payload *payload);
+
 void write_scdc_data(
 		struct ddc_service *ddc_service,
 		uint32_t pix_clk,
@@ -57,5 +85,8 @@ void set_dongle_type(struct ddc_service *ddc,
 
 struct ddc *get_ddc_pin(struct ddc_service *ddc_service);
 
+int link_aux_transfer_raw(struct ddc_service *ddc,
+		struct aux_payload *payload,
+		enum aux_return_code_type *operation_result);
 #endif /* __DAL_DDC_SERVICE_H__ */
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
index d4370856f164..ba98013fecd0 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
@@ -42,6 +42,8 @@
 #include "link_edp_panel_control.h"
 #include "link_dp_irq_handler.h"
 #include "link/accessories/link_dp_trace.h"
+#include "link/link_detection.h"
+#include "link/link_validation.h"
 #include "link_dp_training.h"
 #include "atomfirmware.h"
 #include "resource.h"
@@ -155,7 +157,7 @@ uint8_t dp_parse_lttpr_repeater_count(uint8_t lttpr_repeater_count)
 	return 0; // invalid value
 }
 
-uint32_t dc_link_bw_kbps_from_raw_frl_link_rate_data(uint8_t bw)
+uint32_t link_bw_kbps_from_raw_frl_link_rate_data(uint8_t bw)
 {
 	switch (bw) {
 	case 0b001:
@@ -201,8 +203,11 @@ static enum dc_link_rate linkRateInKHzToLinkRateMultiplier(uint32_t link_rate_in
 	case 5400000:
 		link_rate = LINK_RATE_HIGH2;	// Rate_7 (HBR2)- 5.40 Gbps/Lane
 		break;
+	case 6750000:
+		link_rate = LINK_RATE_RATE_8;	// Rate_8	- 6.75 Gbps/Lane
+		break;
 	case 8100000:
-		link_rate = LINK_RATE_HIGH3;	// Rate_8 (HBR3)- 8.10 Gbps/Lane
+		link_rate = LINK_RATE_HIGH3;	// Rate_9 (HBR3)- 8.10 Gbps/Lane
 		break;
 	default:
 		link_rate = LINK_RATE_UNKNOWN;
@@ -278,7 +283,7 @@ static void dp_wa_power_up_0010FA(struct dc_link *link, uint8_t *dpcd_data,
 
 	if (!link->dpcd_caps.dpcd_rev.raw) {
 		do {
-			dc_link_dp_receiver_power_ctrl(link, true);
+			dpcd_write_rx_power_ctrl(link, true);
 			core_link_read_dpcd(link, DP_DPCD_REV,
 							dpcd_data, length);
 			link->dpcd_caps.dpcd_rev.raw = dpcd_data[
@@ -309,7 +314,7 @@ static void dp_wa_power_up_0010FA(struct dc_link *link, uint8_t *dpcd_data,
 		link->wa_flags.dp_keep_receiver_powered = false;
 }
 
-bool dc_link_is_fec_supported(const struct dc_link *link)
+bool dp_is_fec_supported(const struct dc_link *link)
 {
 	/* TODO - use asic cap instead of link_enc->features
 	 * we no longer know which link enc to use for this link before commit
@@ -325,7 +330,7 @@ bool dc_link_is_fec_supported(const struct dc_link *link)
 			!IS_FPGA_MAXIMUS_DC(link->ctx->dce_environment));
 }
 
-bool dc_link_should_enable_fec(const struct dc_link *link)
+bool dp_should_enable_fec(const struct dc_link *link)
 {
 	bool force_disable = false;
 
@@ -342,10 +347,10 @@ bool dc_link_should_enable_fec(const struct dc_link *link)
 				|| !link->dc->caps.edp_dsc_support))
 		force_disable = true;
 
-	return !force_disable && dc_link_is_fec_supported(link);
+	return !force_disable && dp_is_fec_supported(link);
 }
 
-bool link_is_dp_128b_132b_signal(struct pipe_ctx *pipe_ctx)
+bool dp_is_128b_132b_signal(struct pipe_ctx *pipe_ctx)
 {
 	/* If this assert is hit then we have a link encoder dynamic management issue */
 	ASSERT(pipe_ctx->stream_res.hpo_dp_stream_enc ? pipe_ctx->link_res.hpo_dp_link_enc != NULL : true);
@@ -442,8 +447,12 @@ static enum dc_lane_count reduce_lane_count(enum dc_lane_count lane_count)
 	}
 }
 
-static enum dc_link_rate reduce_link_rate(enum dc_link_rate link_rate)
+static enum dc_link_rate reduce_link_rate(const struct dc_link *link, enum dc_link_rate link_rate)
 {
+	// NEEDSWORK: provide some details about why this function never returns some of the
+	// obscure link rates such as 4.32 Gbps or 3.24 Gbps and if such behavior is intended.
+	//
+
 	switch (link_rate) {
 	case LINK_RATE_UHBR20:
 		return LINK_RATE_UHBR13_5;
@@ -452,13 +461,22 @@ static enum dc_link_rate reduce_link_rate(enum dc_link_rate link_rate)
 	case LINK_RATE_UHBR10:
 		return LINK_RATE_HIGH3;
 	case LINK_RATE_HIGH3:
+		if (link->connector_signal == SIGNAL_TYPE_EDP && link->dc->debug.support_eDP1_5)
+			return LINK_RATE_RATE_8;
+		return LINK_RATE_HIGH2;
+	case LINK_RATE_RATE_8:
 		return LINK_RATE_HIGH2;
 	case LINK_RATE_HIGH2:
 		return LINK_RATE_HIGH;
+	case LINK_RATE_RATE_6:
+	case LINK_RATE_RBR2:
+		return LINK_RATE_HIGH;
 	case LINK_RATE_HIGH:
 		return LINK_RATE_LOW;
+	case LINK_RATE_RATE_3:
+	case LINK_RATE_RATE_2:
+		return LINK_RATE_LOW;
 	case LINK_RATE_LOW:
-		return LINK_RATE_UNKNOWN;
 	default:
 		return LINK_RATE_UNKNOWN;
 	}
@@ -581,7 +599,7 @@ bool decide_fallback_link_setting(
 	case LINK_TRAINING_LQA_FAIL:
 	{
 		if (!reached_minimum_link_rate(cur->link_rate)) {
-			cur->link_rate = reduce_link_rate(cur->link_rate);
+			cur->link_rate = reduce_link_rate(link, cur->link_rate);
 		} else if (!reached_minimum_lane_count(cur->lane_count)) {
 			cur->link_rate = max->link_rate;
 			if (training_result == LINK_TRAINING_CR_FAIL_LANE0)
@@ -603,7 +621,7 @@ bool decide_fallback_link_setting(
 		if (!reached_minimum_lane_count(cur->lane_count)) {
 			cur->lane_count = reduce_lane_count(cur->lane_count);
 		} else if (!reached_minimum_link_rate(cur->link_rate)) {
-			cur->link_rate = reduce_link_rate(cur->link_rate);
+			cur->link_rate = reduce_link_rate(link, cur->link_rate);
 			/* Reduce max link rate to avoid potential infinite loop.
 			 * Needed so that any subsequent CR_FAIL fallback can't
 			 * re-set the link rate higher than the link rate from
@@ -619,7 +637,7 @@ bool decide_fallback_link_setting(
 	case LINK_TRAINING_EQ_FAIL_CR:
 	{
 		if (!reached_minimum_link_rate(cur->link_rate)) {
-			cur->link_rate = reduce_link_rate(cur->link_rate);
+			cur->link_rate = reduce_link_rate(link, cur->link_rate);
 			/* Reduce max link rate to avoid potential infinite loop.
 			 * Needed so that any subsequent CR_FAIL fallback can't
 			 * re-set the link rate higher than the link rate from
@@ -645,7 +663,7 @@ static bool decide_dp_link_settings(struct dc_link *link, struct dc_link_setting
 			initial_link_setting;
 	uint32_t link_bw;
 
-	if (req_bw > dc_link_bandwidth_kbps(link, &link->verified_link_cap))
+	if (req_bw > dp_link_bandwidth_kbps(link, &link->verified_link_cap))
 		return false;
 
 	/* search for the minimum link setting that:
@@ -654,7 +672,7 @@ static bool decide_dp_link_settings(struct dc_link *link, struct dc_link_setting
 	 */
 	while (current_link_setting.link_rate <=
 			link->verified_link_cap.link_rate) {
-		link_bw = dc_link_bandwidth_kbps(
+		link_bw = dp_link_bandwidth_kbps(
 				link,
 				&current_link_setting);
 		if (req_bw <= link_bw) {
@@ -679,7 +697,8 @@ static bool decide_dp_link_settings(struct dc_link *link, struct dc_link_setting
 	return false;
 }
 
-bool dc_link_decide_edp_link_settings(struct dc_link *link, struct dc_link_settings *link_setting, uint32_t req_bw)
+bool edp_decide_link_settings(struct dc_link *link,
+		struct dc_link_settings *link_setting, uint32_t req_bw)
 {
 	struct dc_link_settings initial_link_setting;
 	struct dc_link_settings current_link_setting;
@@ -709,7 +728,7 @@ bool dc_link_decide_edp_link_settings(struct dc_link *link, struct dc_link_setti
 	 */
 	while (current_link_setting.link_rate <=
 			link->verified_link_cap.link_rate) {
-		link_bw = dc_link_bandwidth_kbps(
+		link_bw = dp_link_bandwidth_kbps(
 				link,
 				&current_link_setting);
 		if (req_bw <= link_bw) {
@@ -764,7 +783,7 @@ bool decide_edp_link_settings_with_dsc(struct dc_link *link,
 		initial_link_setting.use_link_rate_set = false;
 		initial_link_setting.link_rate_set = 0;
 		current_link_setting = initial_link_setting;
-		if (req_bw > dc_link_bandwidth_kbps(link, &link->verified_link_cap))
+		if (req_bw > dp_link_bandwidth_kbps(link, &link->verified_link_cap))
 			return false;
 
 		/* search for the minimum link setting that:
@@ -773,7 +792,7 @@ bool decide_edp_link_settings_with_dsc(struct dc_link *link,
 		 */
 		while (current_link_setting.link_rate <=
 				max_link_rate) {
-			link_bw = dc_link_bandwidth_kbps(
+			link_bw = dp_link_bandwidth_kbps(
 					link,
 					&current_link_setting);
 			if (req_bw <= link_bw) {
@@ -830,7 +849,7 @@ bool decide_edp_link_settings_with_dsc(struct dc_link *link,
 	 */
 	while (current_link_setting.link_rate <=
 			max_link_rate) {
-		link_bw = dc_link_bandwidth_kbps(
+		link_bw = dp_link_bandwidth_kbps(
 				link,
 				&current_link_setting);
 		if (req_bw <= link_bw) {
@@ -922,12 +941,12 @@ bool link_decide_link_settings(struct dc_stream_state *stream,
 				tmp_link_setting.link_rate = LINK_RATE_UNKNOWN;
 				tmp_timing.flags.DSC = 0;
 				orig_req_bw = dc_bandwidth_in_kbps_from_timing(&tmp_timing);
-				dc_link_decide_edp_link_settings(link, &tmp_link_setting, orig_req_bw);
+				edp_decide_link_settings(link, &tmp_link_setting, orig_req_bw);
 				max_link_rate = tmp_link_setting.link_rate;
 			}
 			decide_edp_link_settings_with_dsc(link, link_setting, req_bw, max_link_rate);
 		} else {
-			dc_link_decide_edp_link_settings(link, link_setting, req_bw);
+			edp_decide_link_settings(link, link_setting, req_bw);
 		}
 	} else {
 		decide_dp_link_settings(link, link_setting, req_bw);
@@ -948,7 +967,7 @@ enum dp_link_encoding link_dp_get_encoding_format(const struct dc_link_settings
 	return DP_UNKNOWN_ENCODING;
 }
 
-enum dp_link_encoding dc_link_dp_mst_decide_link_encoding_format(const struct dc_link *link)
+enum dp_link_encoding mst_decide_link_encoding_format(const struct dc_link *link)
 {
 	struct dc_link_settings link_settings = {0};
 
@@ -1005,7 +1024,7 @@ static enum dc_status wake_up_aux_channel(struct dc_link *link)
 		 * signal and may need up to 1 ms before being able to reply.
 		 */
 		if (status != DC_OK || dpcd_power_state == DP_SET_POWER_D3) {
-			udelay(1000);
+			fsleep(1000);
 			aux_channel_retry_cnt++;
 		}
 	}
@@ -1024,6 +1043,9 @@ static enum dc_status wake_up_aux_channel(struct dc_link *link)
 				DP_SET_POWER,
 				&dpcd_power_state,
 				sizeof(dpcd_power_state));
+		if (status < 0)
+			DC_LOG_DC("%s: Failed to power up sink: %s\n", __func__,
+				  dpcd_power_state == DP_SET_POWER_D0 ? "D0" : "D3");
 		return DC_ERROR_UNEXPECTED;
 	}
 
@@ -1121,7 +1143,7 @@ static void get_active_converter_info(
 						union hdmi_encoded_link_bw hdmi_encoded_link_bw;
 
 						link->dpcd_caps.dongle_caps.dp_hdmi_frl_max_link_bw_in_kbps =
-								dc_link_bw_kbps_from_raw_frl_link_rate_data(
+								link_bw_kbps_from_raw_frl_link_rate_data(
 										hdmi_color_caps.bits.MAX_ENCODED_LINK_BW_SUPPORT);
 
 						// Intersect reported max link bw support with the supported link rate post FRL link training
@@ -1216,7 +1238,7 @@ static void apply_usbc_combo_phy_reset_wa(struct dc_link *link,
 	dp_disable_link_phy(link, &link_res, link->connector_signal);
 }
 
-static bool dp_overwrite_extended_receiver_cap(struct dc_link *link)
+bool dp_overwrite_extended_receiver_cap(struct dc_link *link)
 {
 	uint8_t dpcd_data[16];
 	uint32_t read_dpcd_retry_cnt = 3;
@@ -1278,16 +1300,10 @@ static bool dp_overwrite_extended_receiver_cap(struct dc_link *link)
 	return true;
 }
 
-void dc_link_overwrite_extended_receiver_cap(
-		struct dc_link *link)
-{
-	dp_overwrite_extended_receiver_cap(link);
-}
-
 void dpcd_set_source_specific_data(struct dc_link *link)
 {
 	if (!link->dc->vendor_signature.is_valid) {
-		enum dc_status result_write_min_hblank = DC_NOT_SUPPORTED;
+		enum dc_status __maybe_unused result_write_min_hblank = DC_NOT_SUPPORTED;
 		struct dpcd_amd_signature amd_signature = {0};
 		struct dpcd_amd_device_id amd_device_id = {0};
 
@@ -1972,7 +1988,7 @@ void detect_edp_sink_caps(struct dc_link *link)
 			sizeof(link->dpcd_caps.alpm_caps.raw));
 }
 
-bool dc_link_dp_get_max_link_enc_cap(const struct dc_link *link, struct dc_link_settings *max_link_enc_cap)
+bool dp_get_max_link_enc_cap(const struct dc_link *link, struct dc_link_settings *max_link_enc_cap)
 {
 	struct link_encoder *link_enc = NULL;
 
@@ -1995,7 +2011,7 @@ bool dc_link_dp_get_max_link_enc_cap(const struct dc_link *link, struct dc_link_
 	return false;
 }
 
-const struct dc_link_settings *dc_link_get_link_cap(
+const struct dc_link_settings *dp_get_verified_link_cap(
 		const struct dc_link *link)
 {
 	if (link->preferred_link_setting.lane_count != LANE_COUNT_UNKNOWN &&
@@ -2121,9 +2137,9 @@ static bool dp_verify_link_cap(
 
 		if (status == LINK_TRAINING_SUCCESS) {
 			success = true;
-			udelay(1000);
-			if (dc_link_dp_read_hpd_rx_irq_data(link, &irq_data) == DC_OK &&
-					dc_link_check_link_loss_status(
+			fsleep(1000);
+			if (dp_read_hpd_rx_irq_data(link, &irq_data) == DC_OK &&
+					dp_parse_link_loss_status(
 							link,
 							&irq_data))
 				(*fail_count)++;
@@ -2163,7 +2179,7 @@ bool dp_verify_link_cap_with_retries(
 
 		memset(&link->verified_link_cap, 0,
 				sizeof(struct dc_link_settings));
-		if (!dc_link_detect_connection_type(link, &type) || type == dc_connection_none) {
+		if (!link_detect_connection_type(link, &type) || type == dc_connection_none) {
 			link->verified_link_cap = fail_safe_link_settings;
 			break;
 		} else if (dp_verify_link_cap(link, known_limit_link_setting,
@@ -2171,7 +2187,7 @@ bool dp_verify_link_cap_with_retries(
 			success = true;
 			break;
 		}
-		msleep(10);
+		fsleep(10 * 1000);
 	}
 
 	dp_trace_lt_fail_count_update(link, fail_count, true);
@@ -2180,11 +2196,10 @@ bool dp_verify_link_cap_with_retries(
 	return success;
 }
 
-/**
- * dc_link_is_dp_sink_present() - Check if there is a native DP
- * or passive DP-HDMI dongle connected
+/*
+ * Check if there is a native DP or passive DP-HDMI dongle connected
  */
-bool dc_link_is_dp_sink_present(struct dc_link *link)
+bool dp_is_sink_present(struct dc_link *link)
 {
 	enum gpio_result gpio_result;
 	uint32_t clock_pin = 0;
@@ -2231,7 +2246,7 @@ bool dc_link_is_dp_sink_present(struct dc_link *link)
 		gpio_result = dal_gpio_get_value(ddc->pin_clock, &clock_pin);
 		ASSERT(gpio_result == GPIO_RESULT_OK);
 		if (clock_pin)
-			udelay(1000);
+			fsleep(1000);
 		else
 			break;
 	} while (retry++ < 3);
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.h
index f79e4a4a9db6..8f0ce97f2362 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.h
@@ -34,32 +34,56 @@ void detect_edp_sink_caps(struct dc_link *link);
 
 struct dc_link_settings dp_get_max_link_cap(struct dc_link *link);
 
+bool dp_get_max_link_enc_cap(const struct dc_link *link,
+		struct dc_link_settings *max_link_enc_cap);
+
+const struct dc_link_settings *dp_get_verified_link_cap(
+		const struct dc_link *link);
+
+enum dp_link_encoding link_dp_get_encoding_format(
+		const struct dc_link_settings *link_settings);
 
 enum dc_status dp_retrieve_lttpr_cap(struct dc_link *link);
 
 /* Convert PHY repeater count read from DPCD uint8_t. */
 uint8_t dp_parse_lttpr_repeater_count(uint8_t lttpr_repeater_count);
 
+bool dp_is_sink_present(struct dc_link *link);
+
 bool dp_is_lttpr_present(struct dc_link *link);
 
+bool dp_is_fec_supported(const struct dc_link *link);
+
 bool is_dp_active_dongle(const struct dc_link *link);
 
 bool is_dp_branch_device(const struct dc_link *link);
 
 void dpcd_write_cable_id_to_dprx(struct dc_link *link);
 
+bool dp_should_enable_fec(const struct dc_link *link);
+
+bool dp_is_128b_132b_signal(struct pipe_ctx *pipe_ctx);
+
 /* Initialize output parameter lt_settings. */
 void dp_decide_training_settings(
 	struct dc_link *link,
 	const struct dc_link_settings *link_setting,
 	struct link_training_settings *lt_settings);
 
+bool link_decide_link_settings(
+	struct dc_stream_state *stream,
+	struct dc_link_settings *link_setting);
+
+bool edp_decide_link_settings(struct dc_link *link,
+		struct dc_link_settings *link_setting, uint32_t req_bw);
 
 bool decide_edp_link_settings_with_dsc(struct dc_link *link,
 		struct dc_link_settings *link_setting,
 		uint32_t req_bw,
 		enum dc_link_rate max_link_rate);
 
+enum dp_link_encoding mst_decide_link_encoding_format(const struct dc_link *link);
+
 void dpcd_set_source_specific_data(struct dc_link *link);
 
 /*query dpcd for version and mst cap addresses*/
@@ -76,4 +100,8 @@ bool dp_verify_link_cap_with_retries(
 	struct dc_link_settings *known_limit_link_setting,
 	int attempts);
 
+uint32_t link_bw_kbps_from_raw_frl_link_rate_data(uint8_t bw);
+
+bool dp_overwrite_extended_receiver_cap(struct dc_link *link);
+
 #endif /* __DC_LINK_DP_CAPABILITY_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c
index 32f48a48e9dd..4626fabc0a96 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c
@@ -26,7 +26,6 @@
 
 #include "dc.h"
 #include "inc/core_status.h"
-#include "dc_link.h"
 #include "dpcd_defs.h"
 
 #include "link_dp_dpia.h"
@@ -79,7 +78,7 @@ enum dc_status dpcd_get_tunneling_device_data(struct dc_link *link)
 	return status;
 }
 
-bool dc_link_dpia_query_hpd_status(struct dc_link *link)
+bool dpia_query_hpd_status(struct dc_link *link)
 {
 	union dmub_rb_cmd cmd = {0};
 	struct dc_dmub_srv *dmub_srv = link->ctx->dmub_srv;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.h
index 98935cc10bb7..363f45a1a964 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.h
@@ -37,7 +37,5 @@ enum dc_status dpcd_get_tunneling_device_data(struct dc_link *link);
 /* Query hot plug status of USB4 DP tunnel.
  * Returns true if HPD high.
  */
-bool dc_link_dpia_query_hpd_status(struct dc_link *link);
-
-
+bool dpia_query_hpd_status(struct dc_link *link);
 #endif /* __DC_LINK_DPIA_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c
index f69e681b3b5b..7581023daa47 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c
@@ -26,11 +26,12 @@
 /*********************************************************************/
 //				USB4 DPIA BANDWIDTH ALLOCATION LOGIC
 /*********************************************************************/
-#include "dc.h"
-#include "dc_link.h"
 #include "link_dp_dpia_bw.h"
-#include "drm_dp_helper_dc.h"
 #include "link_dpcd.h"
+#include "dc_dmub_srv.h"
+
+#define DC_LOGGER \
+	link->ctx->logger
 
 #define Kbps_TO_Gbps (1000 * 1000)
 
@@ -84,12 +85,11 @@ static int get_estimated_bw(struct dc_link *link)
 {
 	uint8_t bw_estimated_bw = 0;
 
-	if (core_link_read_dpcd(
-		link,
-		ESTIMATED_BW,
-		&bw_estimated_bw,
-		sizeof(uint8_t)) != DC_OK)
-		dm_output_to_console("%s: AUX W/R ERROR @ 0x%x\n", __func__, ESTIMATED_BW);
+	core_link_read_dpcd(
+			link,
+			ESTIMATED_BW,
+			&bw_estimated_bw,
+			sizeof(uint8_t));
 
 	return bw_estimated_bw * (Kbps_TO_Gbps / link->dpia_bw_alloc_config.bw_granularity);
 }
@@ -133,8 +133,9 @@ static uint8_t get_lowest_dpia_index(struct dc_link *link)
 {
 	const struct dc *dc_struct = link->dc;
 	uint8_t idx = 0xFF;
+	int i;
 
-	for (int i = 0; i < MAX_PIPES * 2; ++i) {
+	for (i = 0; i < MAX_PIPES * 2; ++i) {
 
 		if (!dc_struct->links[i] ||
 				dc_struct->links[i]->ep_type != DISPLAY_ENDPOINT_USB4_DPIA)
@@ -161,8 +162,9 @@ static int get_host_router_total_bw(struct dc_link *link, uint8_t type)
 	uint8_t idx = (link->link_index - lowest_dpia_index) / 2, idx_temp = 0;
 	struct dc_link *link_temp;
 	int total_bw = 0;
+	int i;
 
-	for (int i = 0; i < MAX_PIPES * 2; ++i) {
+	for (i = 0; i < MAX_PIPES * 2; ++i) {
 
 		if (!dc_struct->links[i] || dc_struct->links[i]->ep_type != DISPLAY_ENDPOINT_USB4_DPIA)
 			continue;
@@ -194,15 +196,13 @@ static int get_host_router_total_bw(struct dc_link *link, uint8_t type)
  */
 static bool dpia_bw_alloc_unplug(struct dc_link *link)
 {
-	bool ret = false;
-
 	if (!link)
 		return true;
 
 	return deallocate_usb4_bw(&link->dpia_bw_alloc_config.sink_allocated_bw,
 			link->dpia_bw_alloc_config.sink_allocated_bw, link);
 }
-static void dc_link_set_usb4_req_bw_req(struct dc_link *link, int req_bw)
+static void set_usb4_req_bw_req(struct dc_link *link, int req_bw)
 {
 	uint8_t requested_bw;
 	uint32_t temp;
@@ -227,9 +227,7 @@ static void dc_link_set_usb4_req_bw_req(struct dc_link *link, int req_bw)
 		link,
 		REQUESTED_BW,
 		&requested_bw,
-		sizeof(uint8_t)) != DC_OK)
-		dm_output_to_console("%s: AUX W/R ERROR @ 0x%x\n", __func__, REQUESTED_BW);
-	else
+		sizeof(uint8_t)) == DC_OK)
 		link->dpia_bw_alloc_config.response_ready = false; // Reset flag
 }
 /*
@@ -246,7 +244,7 @@ static bool get_cm_response_ready_flag(struct dc_link *link)
 // ------------------------------------------------------------------
 //					PUBLIC FUNCTIONS
 // ------------------------------------------------------------------
-bool set_dptx_usb4_bw_alloc_support(struct dc_link *link)
+bool link_dp_dpia_set_dptx_usb4_bw_alloc_support(struct dc_link *link)
 {
 	bool ret = false;
 	uint8_t response = 0,
@@ -257,22 +255,18 @@ bool set_dptx_usb4_bw_alloc_support(struct dc_link *link)
 		goto out;
 
 	if (core_link_read_dpcd(
-		link,
-		DP_TUNNELING_CAPABILITIES,
-		&response,
-		sizeof(uint8_t)) != DC_OK)
-		dm_output_to_console("%s: AUX W/R ERROR @ 0x%x\n", __func__, DP_TUNNELING_CAPABILITIES);
-
-	bw_support_dpia = (response >> 7) & 1;
+			link,
+			DP_TUNNELING_CAPABILITIES,
+			&response,
+			sizeof(uint8_t)) == DC_OK)
+		bw_support_dpia = (response >> 7) & 1;
 
 	if (core_link_read_dpcd(
 		link,
 		USB4_DRIVER_BW_CAPABILITY,
 		&response,
-		sizeof(uint8_t)) != DC_OK)
-		dm_output_to_console("%s: AUX W/R ERROR @ 0x%x\n", __func__, DP_TUNNELING_CAPABILITIES);
-
-	bw_support_cm = (response >> 7) & 1;
+		sizeof(uint8_t)) == DC_OK)
+		bw_support_cm = (response >> 7) & 1;
 
 	/* Send request acknowledgment to Turn ON DPTX support */
 	if (bw_support_cm && bw_support_dpia) {
@@ -282,15 +276,14 @@ bool set_dptx_usb4_bw_alloc_support(struct dc_link *link)
 				link,
 				DPTX_BW_ALLOCATION_MODE_CONTROL,
 				&response,
-				sizeof(uint8_t)) != DC_OK)
-			dm_output_to_console("%s: AUX W/R ERROR @ 0x%x\n",
-					"**** FAILURE Enabling DPtx BW Allocation Mode Support ***\n",
-					__func__, DP_TUNNELING_CAPABILITIES);
-		else {
-
+				sizeof(uint8_t)) != DC_OK) {
+			DC_LOG_DEBUG("%s: **** FAILURE Enabling DPtx BW Allocation Mode Support ***\n",
+					__func__);
+		} else {
 			// SUCCESS Enabled DPtx BW Allocation Mode Support
 			link->dpia_bw_alloc_config.bw_alloc_enabled = true;
-			dm_output_to_console("**** SUCCESS Enabling DPtx BW Allocation Mode Support ***\n");
+			DC_LOG_DEBUG("%s: **** SUCCESS Enabling DPtx BW Allocation Mode Support ***\n",
+					__func__);
 
 			ret = true;
 			init_usb4_bw_struct(link);
@@ -300,8 +293,12 @@ bool set_dptx_usb4_bw_alloc_support(struct dc_link *link)
 out:
 	return ret;
 }
-void dc_link_get_usb4_req_bw_resp(struct dc_link *link, uint8_t bw, uint8_t result)
+void dpia_handle_bw_alloc_response(struct dc_link *link, uint8_t bw, uint8_t result)
 {
+	int bw_needed = 0;
+	int estimated = 0;
+	int host_router_total_estimated_bw = 0;
+
 	if (!get_bw_alloc_proceed_flag((link)))
 		return;
 
@@ -309,13 +306,13 @@ void dc_link_get_usb4_req_bw_resp(struct dc_link *link, uint8_t bw, uint8_t resu
 
 	case DPIA_BW_REQ_FAILED:
 
-		dm_output_to_console("%s: *** *** BW REQ FAILURE for DP-TX Request *** ***\n", __func__);
+		DC_LOG_DEBUG("%s: *** *** BW REQ FAILURE for DP-TX Request *** ***\n", __func__);
 
 		// Update the new Estimated BW value updated by CM
 		link->dpia_bw_alloc_config.estimated_bw =
 				bw * (Kbps_TO_Gbps / link->dpia_bw_alloc_config.bw_granularity);
 
-		dc_link_set_usb4_req_bw_req(link, link->dpia_bw_alloc_config.estimated_bw);
+		set_usb4_req_bw_req(link, link->dpia_bw_alloc_config.estimated_bw);
 		link->dpia_bw_alloc_config.response_ready = false;
 
 		/*
@@ -329,18 +326,18 @@ void dc_link_get_usb4_req_bw_resp(struct dc_link *link, uint8_t bw, uint8_t resu
 
 	case DPIA_BW_REQ_SUCCESS:
 
-		dm_output_to_console("%s: *** BW REQ SUCCESS for DP-TX Request ***\n", __func__);
+		DC_LOG_DEBUG("%s: *** BW REQ SUCCESS for DP-TX Request ***\n", __func__);
 
 		// 1. SUCCESS 1st time before any Pruning is done
 		// 2. SUCCESS after prev. FAIL before any Pruning is done
 		// 3. SUCCESS after Pruning is done but before enabling link
 
-		int needed = bw * (Kbps_TO_Gbps / link->dpia_bw_alloc_config.bw_granularity);
+		bw_needed = bw * (Kbps_TO_Gbps / link->dpia_bw_alloc_config.bw_granularity);
 
 		// 1.
 		if (!link->dpia_bw_alloc_config.sink_allocated_bw) {
 
-			allocate_usb4_bw(&link->dpia_bw_alloc_config.sink_allocated_bw, needed, link);
+			allocate_usb4_bw(&link->dpia_bw_alloc_config.sink_allocated_bw, bw_needed, link);
 			link->dpia_bw_alloc_config.sink_verified_bw =
 					link->dpia_bw_alloc_config.sink_allocated_bw;
 
@@ -354,12 +351,12 @@ void dc_link_get_usb4_req_bw_resp(struct dc_link *link, uint8_t bw, uint8_t resu
 		else if (link->dpia_bw_alloc_config.sink_allocated_bw) {
 
 			// Find out how much do we need to de-alloc
-			if (link->dpia_bw_alloc_config.sink_allocated_bw > needed)
+			if (link->dpia_bw_alloc_config.sink_allocated_bw > bw_needed)
 				deallocate_usb4_bw(&link->dpia_bw_alloc_config.sink_allocated_bw,
-						link->dpia_bw_alloc_config.sink_allocated_bw - needed, link);
+						link->dpia_bw_alloc_config.sink_allocated_bw - bw_needed, link);
 			else
 				allocate_usb4_bw(&link->dpia_bw_alloc_config.sink_allocated_bw,
-						needed - link->dpia_bw_alloc_config.sink_allocated_bw, link);
+						bw_needed - link->dpia_bw_alloc_config.sink_allocated_bw, link);
 		}
 
 		// 4. If this is the 2nd sink then any unused bw will be reallocated to master DPIA
@@ -370,27 +367,19 @@ void dc_link_get_usb4_req_bw_resp(struct dc_link *link, uint8_t bw, uint8_t resu
 
 	case DPIA_EST_BW_CHANGED:
 
-		dm_output_to_console("%s: *** ESTIMATED BW CHANGED for DP-TX Request ***\n", __func__);
+		DC_LOG_DEBUG("%s: *** ESTIMATED BW CHANGED for DP-TX Request ***\n", __func__);
 
-		int available = 0, estimated = bw * (Kbps_TO_Gbps / link->dpia_bw_alloc_config.bw_granularity);
-		int host_router_total_estimated_bw = get_host_router_total_bw(link, HOST_ROUTER_BW_ESTIMATED);
+		estimated = bw * (Kbps_TO_Gbps / link->dpia_bw_alloc_config.bw_granularity);
+		host_router_total_estimated_bw = get_host_router_total_bw(link, HOST_ROUTER_BW_ESTIMATED);
 
 		// 1. If due to unplug of other sink
 		if (estimated == host_router_total_estimated_bw) {
-
 			// First update the estimated & max_bw fields
-			if (link->dpia_bw_alloc_config.estimated_bw < estimated) {
-				available = estimated - link->dpia_bw_alloc_config.estimated_bw;
+			if (link->dpia_bw_alloc_config.estimated_bw < estimated)
 				link->dpia_bw_alloc_config.estimated_bw = estimated;
-			}
 		}
 		// 2. If due to realloc bw btw 2 dpia due to plug OR realloc unused Bw
 		else {
-
-			// We took from another unplugged/problematic sink to give to us
-			if (link->dpia_bw_alloc_config.estimated_bw < estimated)
-				available = estimated - link->dpia_bw_alloc_config.estimated_bw;
-
 			// We lost estimated bw usually due to plug event of other dpia
 			link->dpia_bw_alloc_config.estimated_bw = estimated;
 		}
@@ -398,12 +387,12 @@ void dc_link_get_usb4_req_bw_resp(struct dc_link *link, uint8_t bw, uint8_t resu
 
 	case DPIA_BW_ALLOC_CAPS_CHANGED:
 
-		dm_output_to_console("%s: *** BW ALLOC CAPABILITY CHANGED for DP-TX Request ***\n", __func__);
+		DC_LOG_DEBUG("%s: *** BW ALLOC CAPABILITY CHANGED for DP-TX Request ***\n", __func__);
 		link->dpia_bw_alloc_config.bw_alloc_enabled = false;
 		break;
 	}
 }
-int dc_link_dp_dpia_handle_usb4_bandwidth_allocation_for_link(struct dc_link *link, int peak_bw)
+int dpia_handle_usb4_bandwidth_allocation_for_link(struct dc_link *link, int peak_bw)
 {
 	int ret = 0;
 	uint8_t timeout = 10;
@@ -417,14 +406,14 @@ int dc_link_dp_dpia_handle_usb4_bandwidth_allocation_for_link(struct dc_link *li
 
 		// If DP over USB4 then we need to check BW allocation
 		link->dpia_bw_alloc_config.sink_max_bw = peak_bw;
-		dc_link_set_usb4_req_bw_req(link, link->dpia_bw_alloc_config.sink_max_bw);
+		set_usb4_req_bw_req(link, link->dpia_bw_alloc_config.sink_max_bw);
 
 		do {
-			if (!timeout > 0)
+			if (!(timeout > 0))
 				timeout--;
 			else
 				break;
-			udelay(10 * 1000);
+			fsleep(10 * 1000);
 		} while (!get_cm_response_ready_flag(link));
 
 		if (!timeout)
@@ -439,3 +428,65 @@ int dc_link_dp_dpia_handle_usb4_bandwidth_allocation_for_link(struct dc_link *li
 out:
 	return ret;
 }
+int link_dp_dpia_allocate_usb4_bandwidth_for_stream(struct dc_link *link, int req_bw)
+{
+	int ret = 0;
+	uint8_t timeout = 10;
+
+	if (!get_bw_alloc_proceed_flag(link))
+		goto out;
+
+	/*
+	 * Sometimes stream uses same timing parameters as the already
+	 * allocated max sink bw so no need to re-alloc
+	 */
+	if (req_bw != link->dpia_bw_alloc_config.sink_allocated_bw) {
+		set_usb4_req_bw_req(link, req_bw);
+		do {
+			if (!(timeout > 0))
+				timeout--;
+			else
+				break;
+			udelay(10 * 1000);
+		} while (!get_cm_response_ready_flag(link));
+
+		if (!timeout)
+			ret = 0;// ERROR TIMEOUT waiting for response for allocating bw
+		else if (link->dpia_bw_alloc_config.sink_allocated_bw > 0)
+			ret = get_host_router_total_bw(link, HOST_ROUTER_BW_ALLOCATED);
+	}
+
+out:
+	return ret;
+}
+bool dpia_validate_usb4_bw(struct dc_link **link, int *bw_needed_per_dpia, const unsigned int num_dpias)
+{
+	bool ret = true;
+	int bw_needed_per_hr[MAX_HR_NUM] = { 0, 0 };
+	uint8_t lowest_dpia_index = 0, dpia_index = 0;
+	uint8_t i;
+
+	if (!num_dpias || num_dpias > MAX_DPIA_NUM)
+		return ret;
+
+	//Get total Host Router BW & Validate against each Host Router max BW
+	for (i = 0; i < num_dpias; ++i) {
+
+		if (!link[i]->dpia_bw_alloc_config.bw_alloc_enabled)
+			continue;
+
+		lowest_dpia_index = get_lowest_dpia_index(link[i]);
+		if (link[i]->link_index < lowest_dpia_index)
+			continue;
+
+		dpia_index = (link[i]->link_index - lowest_dpia_index) / 2;
+		bw_needed_per_hr[dpia_index] += bw_needed_per_dpia[i];
+		if (bw_needed_per_hr[dpia_index] > get_host_router_total_bw(link[i], HOST_ROUTER_BW_ALLOCATED)) {
+
+			ret = false;
+			break;
+		}
+	}
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.h
index c2c3049adcd1..7292690383ae 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.h
@@ -26,6 +26,13 @@
 #ifndef DC_INC_LINK_DP_DPIA_BW_H_
 #define DC_INC_LINK_DP_DPIA_BW_H_
 
+#include "link.h"
+
+/* Number of Host Routers per motherboard is 2 */
+#define MAX_HR_NUM			2
+/* Number of DPIA per host router is 2 */
+#define MAX_DPIA_NUM		(MAX_HR_NUM * 2)
+
 /*
  * Host Router BW type
  */
@@ -42,6 +49,54 @@ enum bw_type {
  *
  * return: SUCCESS or FAILURE
  */
-bool set_dptx_usb4_bw_alloc_support(struct dc_link *link);
+bool link_dp_dpia_set_dptx_usb4_bw_alloc_support(struct dc_link *link);
+
+/*
+ * Allocates only what the stream needs for bw, so if:
+ * If (stream_req_bw < or > already_allocated_bw_at_HPD)
+ * => Deallocate Max Bw & then allocate only what the stream needs
+ *
+ * @link: pointer to the dc_link struct instance
+ * @req_bw: Bw requested by the stream
+ *
+ * return: allocated bw else return 0
+ */
+int link_dp_dpia_allocate_usb4_bandwidth_for_stream(struct dc_link *link, int req_bw);
+
+/*
+ * Handle the USB4 BW Allocation related functionality here:
+ * Plug => Try to allocate max bw from timing parameters supported by the sink
+ * Unplug => de-allocate bw
+ *
+ * @link: pointer to the dc_link struct instance
+ * @peak_bw: Peak bw used by the link/sink
+ *
+ * return: allocated bw else return 0
+ */
+int dpia_handle_usb4_bandwidth_allocation_for_link(struct dc_link *link, int peak_bw);
+
+/*
+ * Handle function for when the status of the Request above is complete.
+ * We will find out the result of allocating on CM and update structs.
+ *
+ * @link: pointer to the dc_link struct instance
+ * @bw: Allocated or Estimated BW depending on the result
+ * @result: Response type
+ *
+ * return: none
+ */
+void dpia_handle_bw_alloc_response(struct dc_link *link, uint8_t bw, uint8_t result);
+
+/*
+ * Handle the validation of total BW here and confirm that the bw used by each
+ * DPIA doesn't exceed available BW for each host router (HR)
+ *
+ * @link[]: array of link pointer to all possible DPIA links
+ * @bw_needed[]: bw needed for each DPIA link based on timing
+ * @num_dpias: Number of DPIAs for the above 2 arrays. Should always be <= MAX_DPIA_NUM
+ *
+ * return: TRUE if bw used by DPIAs doesn't exceed available BW else return FALSE
+ */
+bool dpia_validate_usb4_bw(struct dc_link **link, int *bw_needed, const unsigned int num_dpias);
 
 #endif /* DC_INC_LINK_DP_DPIA_BW_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
index 9d80427520cf..ba95facc4ee8 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
@@ -33,13 +33,14 @@
 #include "link_dpcd.h"
 #include "link_dp_training.h"
 #include "link_dp_capability.h"
+#include "link_edp_panel_control.h"
 #include "link/accessories/link_dp_trace.h"
 #include "link/link_dpms.h"
 #include "dm_helpers.h"
 
 #define DC_LOGGER_INIT(logger)
 
-bool dc_link_check_link_loss_status(
+bool dp_parse_link_loss_status(
 	struct dc_link *link,
 	union hpd_irq_data *hpd_irq_dpcd_data)
 {
@@ -155,9 +156,9 @@ static bool handle_hpd_irq_psr_sink(struct dc_link *link)
 			/* PSR error, disable and re-enable PSR */
 			if (link->psr_settings.psr_allow_active) {
 				allow_active = false;
-				dc_link_set_psr_allow_active(link, &allow_active, true, false, NULL);
+				edp_set_psr_allow_active(link, &allow_active, true, false, NULL);
 				allow_active = true;
-				dc_link_set_psr_allow_active(link, &allow_active, true, false, NULL);
+				edp_set_psr_allow_active(link, &allow_active, true, false, NULL);
 			}
 
 			return true;
@@ -174,7 +175,7 @@ static bool handle_hpd_irq_psr_sink(struct dc_link *link)
 	return false;
 }
 
-void dc_link_dp_handle_link_loss(struct dc_link *link)
+void dp_handle_link_loss(struct dc_link *link)
 {
 	struct pipe_ctx *pipes[MAX_PIPES];
 	struct dc_state *state = link->dc->current_state;
@@ -200,7 +201,7 @@ void dc_link_dp_handle_link_loss(struct dc_link *link)
 	}
 }
 
-enum dc_status dc_link_dp_read_hpd_rx_irq_data(
+enum dc_status dp_read_hpd_rx_irq_data(
 	struct dc_link *link,
 	union hpd_irq_data *irq_data)
 {
@@ -247,7 +248,7 @@ enum dc_status dc_link_dp_read_hpd_rx_irq_data(
 }
 
 /*************************Short Pulse IRQ***************************/
-bool dc_link_dp_allow_hpd_rx_irq(const struct dc_link *link)
+bool dp_should_allow_hpd_rx_irq(const struct dc_link *link)
 {
 	/*
 	 * Don't handle RX IRQ unless one of following is met:
@@ -262,8 +263,9 @@ bool dc_link_dp_allow_hpd_rx_irq(const struct dc_link *link)
 	return false;
 }
 
-bool dc_link_handle_hpd_rx_irq(struct dc_link *link, union hpd_irq_data *out_hpd_irq_dpcd_data, bool *out_link_loss,
-							bool defer_handling, bool *has_left_work)
+bool dp_handle_hpd_rx_irq(struct dc_link *link,
+		union hpd_irq_data *out_hpd_irq_dpcd_data, bool *out_link_loss,
+		bool defer_handling, bool *has_left_work)
 {
 	union hpd_irq_data hpd_irq_dpcd_data = {0};
 	union device_service_irq device_service_clear = {0};
@@ -288,7 +290,7 @@ bool dc_link_handle_hpd_rx_irq(struct dc_link *link, union hpd_irq_data *out_hpd
 		 * dal_dpsst_ls_read_hpd_irq_data
 		 * Order of calls is important too
 		 */
-	result = dc_link_dp_read_hpd_rx_irq_data(link, &hpd_irq_dpcd_data);
+	result = dp_read_hpd_rx_irq_data(link, &hpd_irq_dpcd_data);
 	if (out_hpd_irq_dpcd_data)
 		*out_hpd_irq_dpcd_data = hpd_irq_dpcd_data;
 
@@ -315,7 +317,7 @@ bool dc_link_handle_hpd_rx_irq(struct dc_link *link, union hpd_irq_data *out_hpd
 		return false;
 	}
 
-	if (!dc_link_dp_allow_hpd_rx_irq(link)) {
+	if (!dp_should_allow_hpd_rx_irq(link)) {
 		DC_LOG_HW_HPD_IRQ("%s: skipping HPD handling on %d\n",
 			__func__, link->link_index);
 		return false;
@@ -348,9 +350,9 @@ bool dc_link_handle_hpd_rx_irq(struct dc_link *link, union hpd_irq_data *out_hpd
 	 * then DM should call DC to do the detection.
 	 * NOTE: Do not handle link loss on eDP since it is internal link*/
 	if ((link->connector_signal != SIGNAL_TYPE_EDP) &&
-		dc_link_check_link_loss_status(
-			link,
-			&hpd_irq_dpcd_data)) {
+			dp_parse_link_loss_status(
+					link,
+					&hpd_irq_dpcd_data)) {
 		/* Connectivity log: link loss */
 		CONN_DATA_LINK_LOSS(link,
 					hpd_irq_dpcd_data.raw,
@@ -360,7 +362,7 @@ bool dc_link_handle_hpd_rx_irq(struct dc_link *link, union hpd_irq_data *out_hpd
 		if (defer_handling && has_left_work)
 			*has_left_work = true;
 		else
-			dc_link_dp_handle_link_loss(link);
+			dp_handle_link_loss(link);
 
 		status = false;
 		if (out_link_loss)
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.h
index 39b2e51ea79d..ac33730fedd4 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.h
@@ -27,5 +27,15 @@
 #define __DC_LINK_DP_IRQ_HANDLER_H__
 
 #include "link.h"
-
+bool dp_parse_link_loss_status(
+	struct dc_link *link,
+	union hpd_irq_data *hpd_irq_dpcd_data);
+bool dp_should_allow_hpd_rx_irq(const struct dc_link *link);
+void dp_handle_link_loss(struct dc_link *link);
+enum dc_status dp_read_hpd_rx_irq_data(
+	struct dc_link *link,
+	union hpd_irq_data *irq_data);
+bool dp_handle_hpd_rx_irq(struct dc_link *link,
+		union hpd_irq_data *out_hpd_irq_dpcd_data, bool *out_link_loss,
+		bool defer_handling, bool *has_left_work);
 #endif /* __DC_LINK_DP_IRQ_HANDLER_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.c
index cd9fb8126bcf..b7abba55bc2f 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.c
@@ -40,7 +40,7 @@
 #define DC_LOGGER \
 	link->ctx->logger
 
-void dc_link_dp_receiver_power_ctrl(struct dc_link *link, bool on)
+void dpcd_write_rx_power_ctrl(struct dc_link *link, bool on)
 {
 	uint8_t state;
 
@@ -64,7 +64,7 @@ void dp_enable_link_phy(
 	link->cur_link_settings = *link_settings;
 	link->dc->hwss.enable_dp_link_output(link, link_res, signal,
 			clock_source, link_settings);
-	dc_link_dp_receiver_power_ctrl(link, true);
+	dpcd_write_rx_power_ctrl(link, true);
 }
 
 void dp_disable_link_phy(struct dc_link *link,
@@ -74,7 +74,7 @@ void dp_disable_link_phy(struct dc_link *link,
 	struct dc  *dc = link->ctx->dc;
 
 	if (!link->wa_flags.dp_keep_receiver_powered)
-		dc_link_dp_receiver_power_ctrl(link, false);
+		dpcd_write_rx_power_ctrl(link, false);
 
 	dc->hwss.disable_link_output(link, link_res, signal);
 	/* Clear current link setting.*/
@@ -143,7 +143,7 @@ enum dc_status dp_set_fec_ready(struct dc_link *link, const struct link_resource
 	link_enc = link_enc_cfg_get_link_enc(link);
 	ASSERT(link_enc);
 
-	if (!dc_link_should_enable_fec(link))
+	if (!dp_should_enable_fec(link))
 		return status;
 
 	if (link_enc->funcs->fec_set_ready &&
@@ -183,7 +183,7 @@ void dp_set_fec_enable(struct dc_link *link, bool enable)
 	link_enc = link_enc_cfg_get_link_enc(link);
 	ASSERT(link_enc);
 
-	if (!dc_link_should_enable_fec(link))
+	if (!dp_should_enable_fec(link))
 		return;
 
 	if (link_enc->funcs->fec_set_enable &&
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.h
index dba1f29df319..1eb0619d6710 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_phy.h
@@ -51,6 +51,9 @@ void dp_set_drive_settings(
 
 enum dc_status dp_set_fec_ready(struct dc_link *link,
 		const struct link_resource *link_res, bool ready);
+
 void dp_set_fec_enable(struct dc_link *link, bool enable);
 
+void dpcd_write_rx_power_ctrl(struct dc_link *link, bool on);
+
 #endif /* __DC_LINK_DP_PHY_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c
index b48d4d822991..170f33835930 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c
@@ -41,6 +41,8 @@
 #include "link_dp_phy.h"
 #include "link_dp_capability.h"
 #include "link_edp_panel_control.h"
+#include "link/link_detection.h"
+#include "link/link_validation.h"
 #include "atomfirmware.h"
 #include "link_enc_cfg.h"
 #include "resource.h"
@@ -84,6 +86,9 @@ void dp_log_training_result(
 	case LINK_RATE_HIGH2:
 		link_rate = "HBR2";
 		break;
+	case LINK_RATE_RATE_8:
+		link_rate = "R8";
+		break;
 	case LINK_RATE_HIGH3:
 		link_rate = "HBR3";
 		break;
@@ -207,27 +212,36 @@ enum dpcd_training_patterns
 
 	switch (pattern) {
 	case DP_TRAINING_PATTERN_SEQUENCE_1:
+		DC_LOG_HW_LINK_TRAINING("%s: Using DP training pattern TPS1\n", __func__);
 		dpcd_tr_pattern = DPCD_TRAINING_PATTERN_1;
 		break;
 	case DP_TRAINING_PATTERN_SEQUENCE_2:
+		DC_LOG_HW_LINK_TRAINING("%s: Using DP training pattern TPS2\n", __func__);
 		dpcd_tr_pattern = DPCD_TRAINING_PATTERN_2;
 		break;
 	case DP_TRAINING_PATTERN_SEQUENCE_3:
+		DC_LOG_HW_LINK_TRAINING("%s: Using DP training pattern TPS3\n", __func__);
 		dpcd_tr_pattern = DPCD_TRAINING_PATTERN_3;
 		break;
 	case DP_TRAINING_PATTERN_SEQUENCE_4:
+		DC_LOG_HW_LINK_TRAINING("%s: Using DP training pattern TPS4\n", __func__);
 		dpcd_tr_pattern = DPCD_TRAINING_PATTERN_4;
 		break;
 	case DP_128b_132b_TPS1:
+		DC_LOG_HW_LINK_TRAINING("%s: Using DP 128b/132b training pattern TPS1\n", __func__);
 		dpcd_tr_pattern = DPCD_128b_132b_TPS1;
 		break;
 	case DP_128b_132b_TPS2:
+		DC_LOG_HW_LINK_TRAINING("%s: Using DP 128b/132b training pattern TPS2\n", __func__);
 		dpcd_tr_pattern = DPCD_128b_132b_TPS2;
 		break;
 	case DP_128b_132b_TPS2_CDS:
+		DC_LOG_HW_LINK_TRAINING("%s: Using DP 128b/132b training pattern TPS2 CDS\n",
+					__func__);
 		dpcd_tr_pattern = DPCD_128b_132b_TPS2_CDS;
 		break;
 	case DP_TRAINING_PATTERN_VIDEOIDLE:
+		DC_LOG_HW_LINK_TRAINING("%s: Using DP training pattern videoidle\n", __func__);
 		dpcd_tr_pattern = DPCD_TRAINING_PATTERN_VIDEOIDLE;
 		break;
 	default:
@@ -258,10 +272,7 @@ void dp_wait_for_training_aux_rd_interval(
 	struct dc_link *link,
 	uint32_t wait_in_micro_secs)
 {
-	if (wait_in_micro_secs > 1000)
-		msleep(wait_in_micro_secs/1000);
-	else
-		udelay(wait_in_micro_secs);
+	fsleep(wait_in_micro_secs);
 
 	DC_LOG_HW_LINK_TRAINING("%s:\n wait = %d\n",
 		__func__,
@@ -725,12 +736,10 @@ void override_training_settings(
 	if (link->preferred_training_settings.fec_enable != NULL)
 		lt_settings->should_set_fec_ready = *link->preferred_training_settings.fec_enable;
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	/* Check DP tunnel LTTPR mode debug option. */
 	if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA && link->dc->debug.dpia_debug.bits.force_non_lttpr)
 		lt_settings->lttpr_mode = LTTPR_MODE_NON_LTTPR;
 
-#endif
 	dp_get_lttpr_mode_override(link, &lt_settings->lttpr_mode);
 
 }
@@ -780,7 +789,7 @@ enum dc_dp_training_pattern decide_eq_training_pattern(struct dc_link *link,
 	return pattern;
 }
 
-enum lttpr_mode dc_link_decide_lttpr_mode(struct dc_link *link,
+enum lttpr_mode dp_decide_lttpr_mode(struct dc_link *link,
 		struct dc_link_settings *link_setting)
 {
 	enum dp_link_encoding encoding = link_dp_get_encoding_format(link_setting);
@@ -865,8 +874,9 @@ static enum dc_status configure_lttpr_mode_non_transparent(
 	uint8_t repeater_id;
 	enum dc_status result = DC_ERROR_UNEXPECTED;
 	uint8_t repeater_mode = DP_PHY_REPEATER_MODE_TRANSPARENT;
+	const struct dc *dc = link->dc;
 
-	enum dp_link_encoding encoding = link_dp_get_encoding_format(&lt_settings->link_settings);
+	enum dp_link_encoding encoding = dc->link_srv->dp_get_encoding_format(&lt_settings->link_settings);
 
 	if (encoding == DP_8b_10b_ENCODING) {
 		DC_LOG_HW_LINK_TRAINING("%s\n Set LTTPR to Transparent Mode\n", __func__);
@@ -970,7 +980,7 @@ static void dpcd_exit_training_mode(struct dc_link *link, enum dp_link_encoding
 			if ((core_link_read_dpcd(link, DP_SINK_STATUS, &sink_status, 1) == DC_OK) &&
 					(sink_status & DP_INTRA_HOP_AUX_REPLY_INDICATION) == 0)
 				break;
-			udelay(1000);
+			fsleep(1000);
 		}
 	}
 }
@@ -1495,7 +1505,10 @@ enum link_training_result dp_perform_link_training(
 	 * Non-LT AUX transactions inside training mode.
 	 */
 	if ((link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) && encoding == DP_8b_10b_ENCODING)
-		status = dp_perform_fixed_vs_pe_training_sequence(link, link_res, &lt_settings);
+		if (link->dc->config.use_old_fixed_vs_sequence)
+			status = dp_perform_fixed_vs_pe_training_sequence_legacy(link, link_res, &lt_settings);
+		else
+			status = dp_perform_fixed_vs_pe_training_sequence(link, link_res, &lt_settings);
 	else if (encoding == DP_8b_10b_ENCODING)
 		status = dp_perform_8b_10b_link_training(link, link_res, &lt_settings);
 	else if (encoding == DP_128b_132b_ENCODING)
@@ -1556,9 +1569,10 @@ bool perform_link_training_with_retries(
 	j = 0;
 	while (j < attempts && fail_count < (attempts * 10)) {
 
-		DC_LOG_HW_LINK_TRAINING("%s: Beginning link(%d) training attempt %u of %d @ rate(%d) x lane(%d)\n",
-			__func__, link->link_index, (unsigned int)j + 1, attempts, cur_link_settings.link_rate,
-			cur_link_settings.lane_count);
+		DC_LOG_HW_LINK_TRAINING("%s: Beginning link(%d) training attempt %u of %d @ rate(%d) x lane(%d) @ spread = %x\n",
+					__func__, link->link_index, (unsigned int)j + 1, attempts,
+				       cur_link_settings.link_rate, cur_link_settings.lane_count,
+				       cur_link_settings.link_spread);
 
 		dp_enable_link_phy(
 			link,
@@ -1573,7 +1587,6 @@ bool perform_link_training_with_retries(
 			msleep(delay_dp_power_up_in_ms);
 		}
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 		if (panel_mode == DP_PANEL_MODE_EDP) {
 			struct cp_psp *cp_psp = &stream->ctx->cp_psp;
 
@@ -1583,21 +1596,19 @@ bool perform_link_training_with_retries(
 				 * Report and continue with eDP panel mode to
 				 * perform eDP link training with right settings
 				 */
-				bool result;
-				result = cp_psp->funcs.enable_assr(cp_psp->handle, link);
+				cp_psp->funcs.enable_assr(cp_psp->handle, link);
 			}
 		}
-#endif
 
 		dp_set_panel_mode(link, panel_mode);
 
 		if (link->aux_access_disabled) {
-			dc_link_dp_perform_link_training_skip_aux(link, &pipe_ctx->link_res, &cur_link_settings);
+			dp_perform_link_training_skip_aux(link, &pipe_ctx->link_res, &cur_link_settings);
 			return true;
 		} else {
 			/** @todo Consolidate USB4 DP and DPx.x training. */
 			if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA) {
-				status = dc_link_dpia_perform_link_training(
+				status = dpia_perform_link_training(
 						link,
 						&pipe_ctx->link_res,
 						&cur_link_settings,
@@ -1639,9 +1650,10 @@ bool perform_link_training_with_retries(
 				break;
 		}
 
-		DC_LOG_WARNING("%s: Link(%d) training attempt %u of %d failed @ rate(%d) x lane(%d) : fail reason:(%d)\n",
-			__func__, link->link_index, (unsigned int)j + 1, attempts, cur_link_settings.link_rate,
-			cur_link_settings.lane_count, status);
+		DC_LOG_WARNING("%s: Link(%d) training attempt %u of %d failed @ rate(%d) x lane(%d) @ spread = %x : fail reason:(%d)\n",
+			       __func__, link->link_index, (unsigned int)j + 1, attempts,
+			      cur_link_settings.link_rate, cur_link_settings.lane_count,
+			      cur_link_settings.link_spread, status);
 
 		dp_disable_link_phy(link, &pipe_ctx->link_res, signal);
 
@@ -1649,7 +1661,7 @@ bool perform_link_training_with_retries(
 		if (status == LINK_TRAINING_ABORT) {
 			enum dc_connection_type type = dc_connection_none;
 
-			dc_link_detect_connection_type(link, &type);
+			link_detect_connection_type(link, &type);
 			if (type == dc_connection_none) {
 				DC_LOG_HW_LINK_TRAINING("%s: Aborting training because sink unplugged\n", __func__);
 				break;
@@ -1682,7 +1694,7 @@ bool perform_link_training_with_retries(
 			 * minimum link bandwidth.
 			 */
 			req_bw = dc_bandwidth_in_kbps_from_timing(&stream->timing);
-			link_bw = dc_link_bandwidth_kbps(link, &cur_link_settings);
+			link_bw = dp_link_bandwidth_kbps(link, &cur_link_settings);
 			is_link_bw_low = (req_bw > link_bw);
 			is_link_bw_min = ((cur_link_settings.link_rate <= LINK_RATE_LOW) &&
 				(cur_link_settings.lane_count <= LANE_COUNT_ONE));
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.h
index a04948635369..7d027bac8255 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.h
@@ -119,6 +119,9 @@ enum dc_dp_training_pattern decide_cr_training_pattern(
 enum dc_dp_training_pattern decide_eq_training_pattern(struct dc_link *link,
 		const struct dc_link_settings *link_settings);
 
+enum lttpr_mode dp_decide_lttpr_mode(struct dc_link *link,
+		struct dc_link_settings *link_setting);
+
 void dp_get_lttpr_mode_override(struct dc_link *link,
 		enum lttpr_mode *override);
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c
index 14b98e096d39..3889ebb2256b 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c
@@ -225,8 +225,10 @@ enum link_training_result perform_8b_10b_clock_recovery_sequence(
 				offset);
 
 		/* 5. check CR done*/
-		if (dp_is_cr_done(lane_count, dpcd_lane_status))
+		if (dp_is_cr_done(lane_count, dpcd_lane_status)) {
+			DC_LOG_HW_LINK_TRAINING("%s: Clock recovery OK\n", __func__);
 			return LINK_TRAINING_SUCCESS;
+		}
 
 		/* 6. max VS reached*/
 		if ((link_dp_get_encoding_format(&lt_settings->link_settings) ==
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_auxless.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_auxless.c
index e50ec5012559..4c6b886a9da8 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_auxless.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_auxless.c
@@ -30,7 +30,7 @@
 #include "link_dp_phy.h"
 #define DC_LOGGER \
 	link->ctx->logger
-bool dc_link_dp_perform_link_training_skip_aux(
+bool dp_perform_link_training_skip_aux(
 	struct dc_link *link,
 	const struct link_resource *link_res,
 	const struct dc_link_settings *link_setting)
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_auxless.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_auxless.h
index 413999cd03c4..546387a5f32d 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_auxless.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_auxless.h
@@ -28,7 +28,7 @@
 #define __DC_LINK_DP_TRAINING_AUXLESS_H__
 #include "link_dp_training.h"
 
-bool dc_link_dp_perform_link_training_skip_aux(
+bool dp_perform_link_training_skip_aux(
 	struct dc_link *link,
 	const struct link_resource *link_res,
 	const struct dc_link_settings *link_setting);
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.c
index e60da0532c53..4f4e899e5c46 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.c
@@ -29,7 +29,6 @@
 #include "link_dp_training_dpia.h"
 #include "dc.h"
 #include "inc/core_status.h"
-#include "dc_link.h"
 #include "dpcd_defs.h"
 
 #include "link_dp_dpia.h"
@@ -402,6 +401,7 @@ static enum link_training_result dpia_training_cr_non_transparent(
 
 		/* Check if clock recovery successful. */
 		if (dp_is_cr_done(lane_count, dpcd_lane_status)) {
+			DC_LOG_HW_LINK_TRAINING("%s: Clock recovery OK\n", __func__);
 			result = LINK_TRAINING_SUCCESS;
 			break;
 		}
@@ -509,6 +509,7 @@ static enum link_training_result dpia_training_cr_transparent(
 
 		/* Check if clock recovery successful. */
 		if (dp_is_cr_done(lane_count, dpcd_lane_status)) {
+			DC_LOG_HW_LINK_TRAINING("%s: Clock recovery OK\n", __func__);
 			result = LINK_TRAINING_SUCCESS;
 			break;
 		}
@@ -986,7 +987,7 @@ static void dpia_training_abort(
 	core_link_send_set_config(link, DPIA_SET_CFG_SET_LINK, data);
 }
 
-enum link_training_result dc_link_dpia_perform_link_training(
+enum link_training_result dpia_perform_link_training(
 	struct dc_link *link,
 	const struct link_resource *link_res,
 	const struct dc_link_settings *link_setting,
@@ -999,7 +1000,7 @@ enum link_training_result dc_link_dpia_perform_link_training(
 
 	struct dc_link_settings link_settings = *link_setting; // non-const copy to pass in
 
-	lt_settings.lttpr_mode = dc_link_decide_lttpr_mode(link, &link_settings);
+	lt_settings.lttpr_mode = dp_decide_lttpr_mode(link, &link_settings);
 
 	/* Configure link as prescribed in link_setting and set LTTPR mode. */
 	result = dpia_configure_link(link, link_res, link_setting, &lt_settings);
@@ -1035,7 +1036,7 @@ enum link_training_result dc_link_dpia_perform_link_training(
 	 * falling back to lower bandwidth settings possible.
 	 */
 	if (result == LINK_TRAINING_SUCCESS) {
-		msleep(5);
+		fsleep(5000);
 		if (!link->is_automated)
 			result = dp_check_link_loss_status(link, &lt_settings);
 	} else if (result == LINK_TRAINING_ABORT)
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.h
index 0150f2916421..b39fb9faf1c2 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.h
@@ -32,7 +32,7 @@
  * DPIA equivalent of dc_link_dp_perfrorm_link_training.
  * Aborts link training upon detection of sink unplug.
  */
-enum link_training_result dc_link_dpia_perform_link_training(
+enum link_training_result dpia_perform_link_training(
 	struct dc_link *link,
 	const struct link_resource *link_res,
 	const struct dc_link_settings *link_setting,
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c
index a4071d2959a0..5731c4b61f9f 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c
@@ -223,7 +223,7 @@ static enum link_training_result perform_fixed_vs_pe_nontransparent_training_seq
 }
 
 
-enum link_training_result dp_perform_fixed_vs_pe_training_sequence(
+enum link_training_result dp_perform_fixed_vs_pe_training_sequence_legacy(
 	struct dc_link *link,
 	const struct link_resource *link_res,
 	struct link_training_settings *lt_settings)
@@ -577,3 +577,379 @@ enum link_training_result dp_perform_fixed_vs_pe_training_sequence(
 
 	return status;
 }
+
+enum link_training_result dp_perform_fixed_vs_pe_training_sequence(
+	struct dc_link *link,
+	const struct link_resource *link_res,
+	struct link_training_settings *lt_settings)
+{
+	const uint8_t vendor_lttpr_write_data_reset[4] = {0x1, 0x50, 0x63, 0xFF};
+	const uint8_t offset = dp_parse_lttpr_repeater_count(
+			link->dpcd_caps.lttpr_caps.phy_repeater_cnt);
+	const uint8_t vendor_lttpr_write_data_intercept_en[4] = {0x1, 0x55, 0x63, 0x0};
+	const uint8_t vendor_lttpr_write_data_intercept_dis[4] = {0x1, 0x55, 0x63, 0x6E};
+	const uint8_t vendor_lttpr_write_data_adicora_eq1[4] = {0x1, 0x55, 0x63, 0x2E};
+	const uint8_t vendor_lttpr_write_data_adicora_eq2[4] = {0x1, 0x55, 0x63, 0x01};
+	const uint8_t vendor_lttpr_write_data_adicora_eq3[4] = {0x1, 0x55, 0x63, 0x68};
+	uint32_t pre_disable_intercept_delay_ms = link->dc->debug.fixed_vs_aux_delay_config_wa;
+	uint8_t vendor_lttpr_write_data_vs[4] = {0x1, 0x51, 0x63, 0x0};
+	uint8_t vendor_lttpr_write_data_pe[4] = {0x1, 0x52, 0x63, 0x0};
+
+	uint32_t vendor_lttpr_write_address = 0xF004F;
+	enum link_training_result status = LINK_TRAINING_SUCCESS;
+	uint8_t lane = 0;
+	union down_spread_ctrl downspread = {0};
+	union lane_count_set lane_count_set = {0};
+	uint8_t toggle_rate;
+	uint8_t rate;
+
+	/* Only 8b/10b is supported */
+	ASSERT(link_dp_get_encoding_format(&lt_settings->link_settings) ==
+			DP_8b_10b_ENCODING);
+
+	if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
+		status = perform_fixed_vs_pe_nontransparent_training_sequence(link, link_res, lt_settings);
+		return status;
+	}
+
+	if (offset != 0xFF) {
+		vendor_lttpr_write_address +=
+				((DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE) * (offset - 1));
+
+		/* Certain display and cable configuration require extra delay */
+		if (offset > 2)
+			pre_disable_intercept_delay_ms = link->dc->debug.fixed_vs_aux_delay_config_wa * 2;
+	}
+
+	/* Vendor specific: Reset lane settings */
+	core_link_write_dpcd(
+			link,
+			vendor_lttpr_write_address,
+			&vendor_lttpr_write_data_reset[0],
+			sizeof(vendor_lttpr_write_data_reset));
+	core_link_write_dpcd(
+			link,
+			vendor_lttpr_write_address,
+			&vendor_lttpr_write_data_vs[0],
+			sizeof(vendor_lttpr_write_data_vs));
+	core_link_write_dpcd(
+			link,
+			vendor_lttpr_write_address,
+			&vendor_lttpr_write_data_pe[0],
+			sizeof(vendor_lttpr_write_data_pe));
+
+	/* Vendor specific: Enable intercept */
+	core_link_write_dpcd(
+			link,
+			vendor_lttpr_write_address,
+			&vendor_lttpr_write_data_intercept_en[0],
+			sizeof(vendor_lttpr_write_data_intercept_en));
+
+	/* 1. set link rate, lane count and spread. */
+
+	downspread.raw = (uint8_t)(lt_settings->link_settings.link_spread);
+
+	lane_count_set.bits.LANE_COUNT_SET =
+	lt_settings->link_settings.lane_count;
+
+	lane_count_set.bits.ENHANCED_FRAMING = lt_settings->enhanced_framing;
+	lane_count_set.bits.POST_LT_ADJ_REQ_GRANTED = 0;
+
+
+	if (lt_settings->pattern_for_eq < DP_TRAINING_PATTERN_SEQUENCE_4) {
+		lane_count_set.bits.POST_LT_ADJ_REQ_GRANTED =
+				link->dpcd_caps.max_ln_count.bits.POST_LT_ADJ_REQ_SUPPORTED;
+	}
+
+	core_link_write_dpcd(link, DP_DOWNSPREAD_CTRL,
+		&downspread.raw, sizeof(downspread));
+
+	core_link_write_dpcd(link, DP_LANE_COUNT_SET,
+		&lane_count_set.raw, 1);
+
+	rate = get_dpcd_link_rate(&lt_settings->link_settings);
+
+	/* Vendor specific: Toggle link rate */
+	toggle_rate = (rate == 0x6) ? 0xA : 0x6;
+
+	if (link->vendor_specific_lttpr_link_rate_wa == rate) {
+		core_link_write_dpcd(
+				link,
+				DP_LINK_BW_SET,
+				&toggle_rate,
+				1);
+	}
+
+	link->vendor_specific_lttpr_link_rate_wa = rate;
+
+	core_link_write_dpcd(link, DP_LINK_BW_SET, &rate, 1);
+
+	DC_LOG_HW_LINK_TRAINING("%s\n %x rate = %x\n %x lane = %x framing = %x\n %x spread = %x\n",
+		__func__,
+		DP_LINK_BW_SET,
+		lt_settings->link_settings.link_rate,
+		DP_LANE_COUNT_SET,
+		lt_settings->link_settings.lane_count,
+		lt_settings->enhanced_framing,
+		DP_DOWNSPREAD_CTRL,
+		lt_settings->link_settings.link_spread);
+
+	/* 2. Perform link training */
+
+	/* Perform Clock Recovery Sequence */
+	if (status == LINK_TRAINING_SUCCESS) {
+		const uint8_t max_vendor_dpcd_retries = 10;
+		uint32_t retries_cr;
+		uint32_t retry_count;
+		uint32_t wait_time_microsec;
+		enum dc_lane_count lane_count = lt_settings->link_settings.lane_count;
+		union lane_status dpcd_lane_status[LANE_COUNT_DP_MAX];
+		union lane_align_status_updated dpcd_lane_status_updated;
+		union lane_adjust dpcd_lane_adjust[LANE_COUNT_DP_MAX] = {0};
+		enum dc_status dpcd_status = DC_OK;
+		uint8_t i = 0;
+
+		retries_cr = 0;
+		retry_count = 0;
+
+		memset(&dpcd_lane_status, '\0', sizeof(dpcd_lane_status));
+		memset(&dpcd_lane_status_updated, '\0',
+		sizeof(dpcd_lane_status_updated));
+
+		while ((retries_cr < LINK_TRAINING_MAX_RETRY_COUNT) &&
+			(retry_count < LINK_TRAINING_MAX_CR_RETRY)) {
+
+
+			/* 1. call HWSS to set lane settings */
+			dp_set_hw_lane_settings(
+					link,
+					link_res,
+					lt_settings,
+					0);
+
+			/* 2. update DPCD of the receiver */
+			if (!retry_count) {
+				/* EPR #361076 - write as a 5-byte burst,
+				 * but only for the 1-st iteration.
+				 */
+				dpcd_set_lt_pattern_and_lane_settings(
+						link,
+						lt_settings,
+						lt_settings->pattern_for_cr,
+						0);
+				/* Vendor specific: Disable intercept */
+				for (i = 0; i < max_vendor_dpcd_retries; i++) {
+					msleep(pre_disable_intercept_delay_ms);
+					dpcd_status = core_link_write_dpcd(
+							link,
+							vendor_lttpr_write_address,
+							&vendor_lttpr_write_data_intercept_dis[0],
+							sizeof(vendor_lttpr_write_data_intercept_dis));
+
+					if (dpcd_status == DC_OK)
+						break;
+
+					core_link_write_dpcd(
+							link,
+							vendor_lttpr_write_address,
+							&vendor_lttpr_write_data_intercept_en[0],
+							sizeof(vendor_lttpr_write_data_intercept_en));
+				}
+			} else {
+				vendor_lttpr_write_data_vs[3] = 0;
+				vendor_lttpr_write_data_pe[3] = 0;
+
+				for (lane = 0; lane < lane_count; lane++) {
+					vendor_lttpr_write_data_vs[3] |=
+							lt_settings->dpcd_lane_settings[lane].bits.VOLTAGE_SWING_SET << (2 * lane);
+					vendor_lttpr_write_data_pe[3] |=
+							lt_settings->dpcd_lane_settings[lane].bits.PRE_EMPHASIS_SET << (2 * lane);
+				}
+
+				/* Vendor specific: Update VS and PE to DPRX requested value */
+				core_link_write_dpcd(
+						link,
+						vendor_lttpr_write_address,
+						&vendor_lttpr_write_data_vs[0],
+						sizeof(vendor_lttpr_write_data_vs));
+				core_link_write_dpcd(
+						link,
+						vendor_lttpr_write_address,
+						&vendor_lttpr_write_data_pe[0],
+						sizeof(vendor_lttpr_write_data_pe));
+
+				dpcd_set_lane_settings(
+						link,
+						lt_settings,
+						0);
+			}
+
+			/* 3. wait receiver to lock-on*/
+			wait_time_microsec = lt_settings->cr_pattern_time;
+
+			dp_wait_for_training_aux_rd_interval(
+					link,
+					wait_time_microsec);
+
+			/* 4. Read lane status and requested drive
+			 * settings as set by the sink
+			 */
+			dp_get_lane_status_and_lane_adjust(
+					link,
+					lt_settings,
+					dpcd_lane_status,
+					&dpcd_lane_status_updated,
+					dpcd_lane_adjust,
+					0);
+
+			/* 5. check CR done*/
+			if (dp_is_cr_done(lane_count, dpcd_lane_status)) {
+				status = LINK_TRAINING_SUCCESS;
+				break;
+			}
+
+			/* 6. max VS reached*/
+			if (dp_is_max_vs_reached(lt_settings))
+				break;
+
+			/* 7. same lane settings */
+			/* Note: settings are the same for all lanes,
+			 * so comparing first lane is sufficient
+			 */
+			if (lt_settings->dpcd_lane_settings[0].bits.VOLTAGE_SWING_SET ==
+					dpcd_lane_adjust[0].bits.VOLTAGE_SWING_LANE)
+				retries_cr++;
+			else
+				retries_cr = 0;
+
+			/* 8. update VS/PE/PC2 in lt_settings*/
+			dp_decide_lane_settings(lt_settings, dpcd_lane_adjust,
+					lt_settings->hw_lane_settings, lt_settings->dpcd_lane_settings);
+			retry_count++;
+		}
+
+		if (retry_count >= LINK_TRAINING_MAX_CR_RETRY) {
+			ASSERT(0);
+			DC_LOG_ERROR("%s: Link Training Error, could not get CR after %d tries. Possibly voltage swing issue",
+				__func__,
+				LINK_TRAINING_MAX_CR_RETRY);
+
+		}
+
+		status = dp_get_cr_failure(lane_count, dpcd_lane_status);
+	}
+
+	/* Perform Channel EQ Sequence */
+	if (status == LINK_TRAINING_SUCCESS) {
+		enum dc_dp_training_pattern tr_pattern;
+		uint32_t retries_ch_eq;
+		uint32_t wait_time_microsec;
+		enum dc_lane_count lane_count = lt_settings->link_settings.lane_count;
+		union lane_align_status_updated dpcd_lane_status_updated = {0};
+		union lane_status dpcd_lane_status[LANE_COUNT_DP_MAX] = {0};
+		union lane_adjust dpcd_lane_adjust[LANE_COUNT_DP_MAX] = {0};
+
+		core_link_write_dpcd(
+				link,
+				vendor_lttpr_write_address,
+				&vendor_lttpr_write_data_adicora_eq1[0],
+				sizeof(vendor_lttpr_write_data_adicora_eq1));
+		core_link_write_dpcd(
+				link,
+				vendor_lttpr_write_address,
+				&vendor_lttpr_write_data_adicora_eq2[0],
+				sizeof(vendor_lttpr_write_data_adicora_eq2));
+
+		/* Note: also check that TPS4 is a supported feature*/
+		tr_pattern = lt_settings->pattern_for_eq;
+
+		dp_set_hw_training_pattern(link, link_res, tr_pattern, 0);
+
+		status = LINK_TRAINING_EQ_FAIL_EQ;
+
+		for (retries_ch_eq = 0; retries_ch_eq <= LINK_TRAINING_MAX_RETRY_COUNT;
+			retries_ch_eq++) {
+
+			dp_set_hw_lane_settings(link, link_res, lt_settings, 0);
+
+			vendor_lttpr_write_data_vs[3] = 0;
+			vendor_lttpr_write_data_pe[3] = 0;
+
+			for (lane = 0; lane < lane_count; lane++) {
+				vendor_lttpr_write_data_vs[3] |=
+						lt_settings->dpcd_lane_settings[lane].bits.VOLTAGE_SWING_SET << (2 * lane);
+				vendor_lttpr_write_data_pe[3] |=
+						lt_settings->dpcd_lane_settings[lane].bits.PRE_EMPHASIS_SET << (2 * lane);
+			}
+
+			/* Vendor specific: Update VS and PE to DPRX requested value */
+			core_link_write_dpcd(
+					link,
+					vendor_lttpr_write_address,
+					&vendor_lttpr_write_data_vs[0],
+					sizeof(vendor_lttpr_write_data_vs));
+			core_link_write_dpcd(
+					link,
+					vendor_lttpr_write_address,
+					&vendor_lttpr_write_data_pe[0],
+					sizeof(vendor_lttpr_write_data_pe));
+
+			/* 2. update DPCD*/
+			if (!retries_ch_eq) {
+				/* EPR #361076 - write as a 5-byte burst,
+				 * but only for the 1-st iteration
+				 */
+
+				dpcd_set_lt_pattern_and_lane_settings(
+					link,
+					lt_settings,
+					tr_pattern, 0);
+
+				core_link_write_dpcd(
+					link,
+					vendor_lttpr_write_address,
+					&vendor_lttpr_write_data_adicora_eq3[0],
+					sizeof(vendor_lttpr_write_data_adicora_eq3));
+			} else
+				dpcd_set_lane_settings(link, lt_settings, 0);
+
+			/* 3. wait for receiver to lock-on*/
+			wait_time_microsec = lt_settings->eq_pattern_time;
+
+			dp_wait_for_training_aux_rd_interval(
+					link,
+					wait_time_microsec);
+
+			/* 4. Read lane status and requested
+			 * drive settings as set by the sink
+			 */
+			dp_get_lane_status_and_lane_adjust(
+				link,
+				lt_settings,
+				dpcd_lane_status,
+				&dpcd_lane_status_updated,
+				dpcd_lane_adjust,
+				0);
+
+			/* 5. check CR done*/
+			if (!dp_is_cr_done(lane_count, dpcd_lane_status)) {
+				status = LINK_TRAINING_EQ_FAIL_CR;
+				break;
+			}
+
+			/* 6. check CHEQ done*/
+			if (dp_is_ch_eq_done(lane_count, dpcd_lane_status) &&
+					dp_is_symbol_locked(lane_count, dpcd_lane_status) &&
+					dp_is_interlane_aligned(dpcd_lane_status_updated)) {
+				status = LINK_TRAINING_SUCCESS;
+				break;
+			}
+
+			/* 7. update VS/PE/PC2 in lt_settings*/
+			dp_decide_lane_settings(lt_settings, dpcd_lane_adjust,
+					lt_settings->hw_lane_settings, lt_settings->dpcd_lane_settings);
+		}
+	}
+
+	return status;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.h
index e61970e27661..c0d6ea329504 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.h
@@ -28,6 +28,11 @@
 #define __DC_LINK_DP_FIXED_VS_PE_RETIMER_H__
 #include "link_dp_training.h"
 
+enum link_training_result dp_perform_fixed_vs_pe_training_sequence_legacy(
+	struct dc_link *link,
+	const struct link_resource *link_res,
+	struct link_training_settings *lt_settings);
+
 enum link_training_result dp_perform_fixed_vs_pe_training_sequence(
 	struct dc_link *link,
 	const struct link_resource *link_res,
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
index 97e02b5b21ae..d895046787bc 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
@@ -37,6 +37,11 @@
 #include "abm.h"
 #define DC_LOGGER_INIT(logger)
 
+/* Travis */
+static const uint8_t DP_VGA_LVDS_CONVERTER_ID_2[] = "sivarT";
+/* Nutmeg */
+static const uint8_t DP_VGA_LVDS_CONVERTER_ID_3[] = "dnomlA";
+
 void dp_set_panel_mode(struct dc_link *link, enum dp_panel_mode panel_mode)
 {
 	union dpcd_edp_config edp_config_set;
@@ -139,7 +144,7 @@ enum dp_panel_mode dp_get_panel_mode(struct dc_link *link)
 	return DP_PANEL_MODE_DEFAULT;
 }
 
-bool dc_link_set_backlight_level_nits(struct dc_link *link,
+bool edp_set_backlight_level_nits(struct dc_link *link,
 		bool isHDR,
 		uint32_t backlight_millinits,
 		uint32_t transition_time_in_ms)
@@ -171,7 +176,7 @@ bool dc_link_set_backlight_level_nits(struct dc_link *link,
 	return true;
 }
 
-bool dc_link_get_backlight_level_nits(struct dc_link *link,
+bool edp_get_backlight_level_nits(struct dc_link *link,
 		uint32_t *backlight_millinits_avg,
 		uint32_t *backlight_millinits_peak)
 {
@@ -201,7 +206,7 @@ bool dc_link_get_backlight_level_nits(struct dc_link *link,
 	return true;
 }
 
-bool link_backlight_enable_aux(struct dc_link *link, bool enable)
+bool edp_backlight_enable_aux(struct dc_link *link, bool enable)
 {
 	uint8_t backlight_enable = enable ? 1 : 0;
 
@@ -243,13 +248,13 @@ bool set_default_brightness_aux(struct dc_link *link)
 		if (default_backlight < 5000 || default_backlight > 5000000)
 			default_backlight = 150000; //
 
-		return dc_link_set_backlight_level_nits(link, true,
+		return edp_set_backlight_level_nits(link, true,
 				default_backlight, 0);
 	}
 	return false;
 }
 
-bool link_is_edp_ilr_optimization_required(struct dc_link *link,
+bool edp_is_ilr_optimization_required(struct dc_link *link,
 		struct dc_crtc_timing *crtc_timing)
 {
 	struct dc_link_settings link_setting;
@@ -285,7 +290,7 @@ bool link_is_edp_ilr_optimization_required(struct dc_link *link,
 	req_bw = dc_bandwidth_in_kbps_from_timing(crtc_timing);
 
 	if (!crtc_timing->flags.DSC)
-		dc_link_decide_edp_link_settings(link, &link_setting, req_bw);
+		edp_decide_link_settings(link, &link_setting, req_bw);
 	else
 		decide_edp_link_settings_with_dsc(link, &link_setting, req_bw, LINK_RATE_UNKNOWN);
 
@@ -299,7 +304,7 @@ bool link_is_edp_ilr_optimization_required(struct dc_link *link,
 	return false;
 }
 
-void dc_link_edp_panel_backlight_power_on(struct dc_link *link, bool wait_for_hpd)
+void edp_panel_backlight_power_on(struct dc_link *link, bool wait_for_hpd)
 {
 	if (link->connector_signal != SIGNAL_TYPE_EDP)
 		return;
@@ -311,7 +316,7 @@ void dc_link_edp_panel_backlight_power_on(struct dc_link *link, bool wait_for_hp
 		link->dc->hwss.edp_backlight_control(link, true);
 }
 
-bool dc_link_wait_for_t12(struct dc_link *link)
+bool edp_wait_for_t12(struct dc_link *link)
 {
 	if (link->connector_signal == SIGNAL_TYPE_EDP && link->dc->hwss.edp_wait_for_T12) {
 		link->dc->hwss.edp_wait_for_T12(link);
@@ -322,13 +327,13 @@ bool dc_link_wait_for_t12(struct dc_link *link)
 	return false;
 }
 
-void link_edp_add_delay_for_T9(struct dc_link *link)
+void edp_add_delay_for_T9(struct dc_link *link)
 {
 	if (link && link->panel_config.pps.extra_delay_backlight_off > 0)
-		udelay(link->panel_config.pps.extra_delay_backlight_off * 1000);
+		fsleep(link->panel_config.pps.extra_delay_backlight_off * 1000);
 }
 
-bool link_edp_receiver_ready_T9(struct dc_link *link)
+bool edp_receiver_ready_T9(struct dc_link *link)
 {
 	unsigned int tries = 0;
 	unsigned char sinkstatus = 0;
@@ -353,7 +358,7 @@ bool link_edp_receiver_ready_T9(struct dc_link *link)
 	return result;
 }
 
-bool link_edp_receiver_ready_T7(struct dc_link *link)
+bool edp_receiver_ready_T7(struct dc_link *link)
 {
 	unsigned char sinkstatus = 0;
 	unsigned char edpRev = 0;
@@ -383,12 +388,12 @@ bool link_edp_receiver_ready_T7(struct dc_link *link)
 	}
 
 	if (link && link->panel_config.pps.extra_t7_ms > 0)
-		udelay(link->panel_config.pps.extra_t7_ms * 1000);
+		fsleep(link->panel_config.pps.extra_t7_ms * 1000);
 
 	return result;
 }
 
-bool link_power_alpm_dpcd_enable(struct dc_link *link, bool enable)
+bool edp_power_alpm_dpcd_enable(struct dc_link *link, bool enable)
 {
 	bool ret = false;
 	union dpcd_alpm_configuration alpm_config;
@@ -422,7 +427,7 @@ static struct pipe_ctx *get_pipe_from_link(const struct dc_link *link)
 	return pipe_ctx;
 }
 
-bool dc_link_set_backlight_level(const struct dc_link *link,
+bool edp_set_backlight_level(const struct dc_link *link,
 		uint32_t backlight_pwm_u16_16,
 		uint32_t frame_ramp)
 {
@@ -453,7 +458,7 @@ bool dc_link_set_backlight_level(const struct dc_link *link,
 	return true;
 }
 
-bool dc_link_set_psr_allow_active(struct dc_link *link, const bool *allow_active,
+bool edp_set_psr_allow_active(struct dc_link *link, const bool *allow_active,
 		bool wait, bool force_static, const unsigned int *power_opts)
 {
 	struct dc  *dc = link->ctx->dc;
@@ -502,7 +507,7 @@ bool dc_link_set_psr_allow_active(struct dc_link *link, const bool *allow_active
 	return true;
 }
 
-bool dc_link_get_psr_state(const struct dc_link *link, enum dc_psr_state *state)
+bool edp_get_psr_state(const struct dc_link *link, enum dc_psr_state *state)
 {
 	struct dc  *dc = link->ctx->dc;
 	struct dmcu *dmcu = dc->res_pool->dmcu;
@@ -557,7 +562,7 @@ transmitter_to_phy_id(struct dc_link *link)
 	}
 }
 
-bool dc_link_setup_psr(struct dc_link *link,
+bool edp_setup_psr(struct dc_link *link,
 		const struct dc_stream_state *stream, struct psr_config *psr_config,
 		struct psr_context *psr_context)
 {
@@ -623,7 +628,7 @@ bool dc_link_setup_psr(struct dc_link *link,
 		sizeof(psr_configuration.raw));
 
 	if (link->psr_settings.psr_version == DC_PSR_VERSION_SU_1) {
-		link_power_alpm_dpcd_enable(link, true);
+		edp_power_alpm_dpcd_enable(link, true);
 		psr_context->su_granularity_required =
 			psr_config->su_granularity_required;
 		psr_context->su_y_granularity =
@@ -695,7 +700,6 @@ bool dc_link_setup_psr(struct dc_link *link,
 	psr_context->psr_level.u32all = 0;
 
 	/*skip power down the single pipe since it blocks the cstate*/
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	if (link->ctx->asic_id.chip_family >= FAMILY_RV) {
 		switch (link->ctx->asic_id.chip_family) {
 		case FAMILY_YELLOW_CARP:
@@ -709,10 +713,6 @@ bool dc_link_setup_psr(struct dc_link *link,
 			break;
 		}
 	}
-#else
-	if (link->ctx->asic_id.chip_family >= FAMILY_RV)
-		psr_context->psr_level.bits.SKIP_CRTC_DISABLE = true;
-#endif
 
 	/* SMU will perform additional powerdown sequence.
 	 * For unsupported ASICs, set psr_level flag to skip PSR
@@ -757,7 +757,7 @@ bool dc_link_setup_psr(struct dc_link *link,
 
 }
 
-void link_get_psr_residency(const struct dc_link *link, uint32_t *residency)
+void edp_get_psr_residency(const struct dc_link *link, uint32_t *residency)
 {
 	struct dc  *dc = link->ctx->dc;
 	struct dmub_psr *psr = dc->res_pool->psr;
@@ -772,7 +772,7 @@ void link_get_psr_residency(const struct dc_link *link, uint32_t *residency)
 	else
 		*residency = 0;
 }
-bool link_set_sink_vtotal_in_psr_active(const struct dc_link *link, uint16_t psr_vtotal_idle, uint16_t psr_vtotal_su)
+bool edp_set_sink_vtotal_in_psr_active(const struct dc_link *link, uint16_t psr_vtotal_idle, uint16_t psr_vtotal_su)
 {
 	struct dc *dc = link->ctx->dc;
 	struct dmub_psr *psr = dc->res_pool->psr;
@@ -803,7 +803,7 @@ static struct abm *get_abm_from_stream_res(const struct dc_link *link)
 	return abm;
 }
 
-int dc_link_get_backlight_level(const struct dc_link *link)
+int edp_get_backlight_level(const struct dc_link *link)
 {
 	struct abm *abm = get_abm_from_stream_res(link);
 	struct panel_cntl *panel_cntl = link->panel_cntl;
@@ -822,7 +822,7 @@ int dc_link_get_backlight_level(const struct dc_link *link)
 		return DC_ERROR_UNEXPECTED;
 }
 
-int dc_link_get_target_backlight_pwm(const struct dc_link *link)
+int edp_get_target_backlight_pwm(const struct dc_link *link)
 {
 	struct abm *abm = get_abm_from_stream_res(link);
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
index 7f91a564b089..28f552080558 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
@@ -30,4 +30,34 @@
 enum dp_panel_mode dp_get_panel_mode(struct dc_link *link);
 void dp_set_panel_mode(struct dc_link *link, enum dp_panel_mode panel_mode);
 bool set_default_brightness_aux(struct dc_link *link);
+void edp_panel_backlight_power_on(struct dc_link *link, bool wait_for_hpd);
+int edp_get_backlight_level(const struct dc_link *link);
+bool edp_get_backlight_level_nits(struct dc_link *link,
+		uint32_t *backlight_millinits_avg,
+		uint32_t *backlight_millinits_peak);
+bool edp_set_backlight_level(const struct dc_link *link,
+		uint32_t backlight_pwm_u16_16,
+		uint32_t frame_ramp);
+bool edp_set_backlight_level_nits(struct dc_link *link,
+		bool isHDR,
+		uint32_t backlight_millinits,
+		uint32_t transition_time_in_ms);
+int edp_get_target_backlight_pwm(const struct dc_link *link);
+bool edp_get_psr_state(const struct dc_link *link, enum dc_psr_state *state);
+bool edp_set_psr_allow_active(struct dc_link *link, const bool *allow_active,
+		bool wait, bool force_static, const unsigned int *power_opts);
+bool edp_setup_psr(struct dc_link *link,
+		const struct dc_stream_state *stream, struct psr_config *psr_config,
+		struct psr_context *psr_context);
+bool edp_set_sink_vtotal_in_psr_active(const struct dc_link *link,
+       uint16_t psr_vtotal_idle, uint16_t psr_vtotal_su);
+void edp_get_psr_residency(const struct dc_link *link, uint32_t *residency);
+bool edp_wait_for_t12(struct dc_link *link);
+bool edp_is_ilr_optimization_required(struct dc_link *link,
+       struct dc_crtc_timing *crtc_timing);
+bool edp_backlight_enable_aux(struct dc_link *link, bool enable);
+void edp_add_delay_for_T9(struct dc_link *link);
+bool edp_receiver_ready_T9(struct dc_link *link);
+bool edp_receiver_ready_T7(struct dc_link *link);
+bool edp_power_alpm_dpcd_enable(struct dc_link *link, bool enable);
 #endif /* __DC_LINK_EDP_POWER_CONTROL_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c
index 5f39dfe06e9a..e3d729ab5b9f 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c
@@ -33,18 +33,18 @@
 #include "link_hpd.h"
 #include "gpio_service_interface.h"
 
-bool dc_link_get_hpd_state(struct dc_link *dc_link)
+bool link_get_hpd_state(struct dc_link *link)
 {
 	uint32_t state;
 
-	dal_gpio_lock_pin(dc_link->hpd_gpio);
-	dal_gpio_get_value(dc_link->hpd_gpio, &state);
-	dal_gpio_unlock_pin(dc_link->hpd_gpio);
+	dal_gpio_lock_pin(link->hpd_gpio);
+	dal_gpio_get_value(link->hpd_gpio, &state);
+	dal_gpio_unlock_pin(link->hpd_gpio);
 
 	return state;
 }
 
-void dc_link_enable_hpd(const struct dc_link *link)
+void link_enable_hpd(const struct dc_link *link)
 {
 	struct link_encoder *encoder = link->link_enc;
 
@@ -52,7 +52,7 @@ void dc_link_enable_hpd(const struct dc_link *link)
 		encoder->funcs->enable_hpd(encoder);
 }
 
-void dc_link_disable_hpd(const struct dc_link *link)
+void link_disable_hpd(const struct dc_link *link)
 {
 	struct link_encoder *encoder = link->link_enc;
 
@@ -60,7 +60,7 @@ void dc_link_disable_hpd(const struct dc_link *link)
 		encoder->funcs->disable_hpd(encoder);
 }
 
-void dc_link_enable_hpd_filter(struct dc_link *link, bool enable)
+void link_enable_hpd_filter(struct dc_link *link, bool enable)
 {
 	struct gpio *hpd;
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.h
index 3d122def0c88..4fb526b264f9 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.h
@@ -44,4 +44,11 @@ bool program_hpd_filter(const struct dc_link *link);
  */
 bool dpia_query_hpd_status(struct dc_link *link);
 bool query_hpd_status(struct dc_link *link, uint32_t *is_hpd_high);
+bool link_get_hpd_state(struct dc_link *link);
+struct gpio *link_get_hpd_gpio(struct dc_bios *dcb,
+		struct graphics_object_id link_id,
+		struct gpio_service *gpio_service);
+void link_enable_hpd(const struct dc_link *link);
+void link_disable_hpd(const struct dc_link *link);
+void link_enable_hpd_filter(struct dc_link *link, bool enable);
 #endif /* __DC_LINK_HPD_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/os_types.h b/drivers/gpu/drm/amd/display/dc/os_types.h
index 6b88ae14f1f9..aad8095660c9 100644
--- a/drivers/gpu/drm/amd/display/dc/os_types.h
+++ b/drivers/gpu/drm/amd/display/dc/os_types.h
@@ -53,11 +53,11 @@
 
 #define dm_error(fmt, ...) DRM_ERROR(fmt, ##__VA_ARGS__)
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
+#if defined(CONFIG_DRM_AMD_DC_FP)
 #include "amdgpu_dm/dc_fpu.h"
 #define DC_FP_START() dc_fpu_begin(__func__, __LINE__)
 #define DC_FP_END() dc_fpu_end(__func__, __LINE__)
-#endif
+#endif /* CONFIG_DRM_AMD_DC_FP */
 
 /*
  *
diff --git a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
index a391b939d709..ba1715e2d25a 100644
--- a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
@@ -261,6 +261,8 @@ struct dmub_srv_hw_params {
 	bool usb4_cm_version;
 	bool fw_in_system_memory;
 	bool dpia_hpd_int_enable_supported;
+	bool disable_clock_gate;
+	bool disallow_dispclk_dppclk_ds;
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
index 007d6bdc3e39..598fa1de54ce 100644
--- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
+++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
@@ -95,6 +95,13 @@
 /* Maximum number of SubVP streams */
 #define DMUB_MAX_SUBVP_STREAMS 2
 
+/* Define max FPO streams as 4 for now. Current implementation today
+ * only supports 1, but could be more in the future. Reduce array
+ * size to ensure the command size remains less than 64 bytes if
+ * adding new fields.
+ */
+#define DMUB_MAX_FPO_STREAMS 4
+
 /* Maximum number of streams on any ASIC. */
 #define DMUB_MAX_STREAMS 6
 
@@ -355,7 +362,7 @@ union dmub_fw_boot_status {
 		uint32_t defer_load : 1; /**< 1 if VBIOS data is deferred programmed */
 		uint32_t reserved : 1;
 		uint32_t detection_required: 1; /**<  if detection need to be triggered by driver */
-
+		uint32_t hw_power_init_done: 1; /**< 1 if hw power init is completed */
 	} bits; /**< status bits */
 	uint32_t all; /**< 32-bit access to status bits */
 };
@@ -370,6 +377,7 @@ enum dmub_fw_boot_status_bit {
 	DMUB_FW_BOOT_STATUS_BIT_RESTORE_REQUIRED = (1 << 3), /**< 1 if driver should call restore */
 	DMUB_FW_BOOT_STATUS_BIT_DEFERRED_LOADED = (1 << 4), /**< 1 if VBIOS data is deferred programmed */
 	DMUB_FW_BOOT_STATUS_BIT_DETECTION_REQUIRED = (1 << 6), /**< 1 if detection need to be triggered by driver*/
+	DMUB_FW_BOOT_STATUS_BIT_HW_POWER_INIT_DONE = (1 << 7), /**< 1 if hw power init is completed */
 };
 
 /* Register bit definition for SCRATCH5 */
@@ -410,8 +418,8 @@ union dmub_fw_boot_options {
 		uint32_t usb4_cm_version: 1; /**< 1 CM support */
 		uint32_t dpia_hpd_int_enable_supported: 1; /* 1 if dpia hpd int enable supported */
 		uint32_t usb4_dpia_bw_alloc_supported: 1; /* 1 if USB4 dpia BW allocation supported */
-
-		uint32_t reserved : 15; /**< reserved */
+		uint32_t disable_clk_ds: 1; /* 1 if disallow dispclk_ds and dppclk_ds*/
+		uint32_t reserved : 14; /**< reserved */
 	} bits; /**< boot bits */
 	uint32_t all; /**< 32-bit access to bits */
 };
@@ -1097,7 +1105,12 @@ enum dmub_cmd_idle_opt_type {
 	/**
 	 * DCN hardware save.
 	 */
-	DMUB_CMD__IDLE_OPT_DCN_SAVE_INIT = 1
+	DMUB_CMD__IDLE_OPT_DCN_SAVE_INIT = 1,
+
+	/**
+	 * DCN hardware notify idle.
+	 */
+	DMUB_CMD__IDLE_OPT_DCN_NOTIFY_IDLE = 2
 };
 
 /**
@@ -1108,6 +1121,24 @@ struct dmub_rb_cmd_idle_opt_dcn_restore {
 };
 
 /**
+ * struct dmub_dcn_notify_idle_cntl_data - Data passed to FW in a DMUB_CMD__IDLE_OPT_DCN_NOTIFY_IDLE command.
+ */
+struct dmub_dcn_notify_idle_cntl_data {
+	uint8_t driver_idle;
+	uint8_t d3_entry;
+	uint8_t trigger;
+	uint8_t pad[1];
+};
+
+/**
+ * struct dmub_rb_cmd_idle_opt_dcn_notify_idle - Data passed to FW in a DMUB_CMD__IDLE_OPT_DCN_NOTIFY_IDLE command.
+ */
+struct dmub_rb_cmd_idle_opt_dcn_notify_idle {
+	struct dmub_cmd_header header; /**< header */
+	struct dmub_dcn_notify_idle_cntl_data cntl_data;
+};
+
+/**
  * struct dmub_clocks - Clock update notification.
  */
 struct dmub_clocks {
@@ -1971,7 +2002,7 @@ struct dmub_cmd_psr_copy_settings_data {
 	uint8_t cmd_version;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -2029,7 +2060,7 @@ struct dmub_cmd_psr_set_level_data {
 	uint8_t cmd_version;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -2056,7 +2087,7 @@ struct dmub_rb_cmd_psr_enable_data {
 	uint8_t cmd_version;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -2100,7 +2131,7 @@ struct dmub_cmd_psr_set_version_data {
 	uint8_t cmd_version;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -2131,7 +2162,7 @@ struct dmub_cmd_psr_force_static_data {
 	uint8_t cmd_version;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -2206,7 +2237,7 @@ struct dmub_cmd_update_dirty_rect_data {
 	uint8_t cmd_version;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -2344,7 +2375,7 @@ struct dmub_cmd_update_cursor_payload0 {
 	uint8_t cmd_version;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -2391,7 +2422,7 @@ struct dmub_cmd_psr_set_vtotal_data {
 	uint8_t cmd_version;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -2429,7 +2460,7 @@ struct dmub_cmd_psr_set_power_opt_data {
 	uint8_t cmd_version;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -3084,14 +3115,15 @@ struct dmub_cmd_fw_assisted_mclk_switch_pipe_data {
 	uint8_t max_ramp_step;
 	uint8_t pipes;
 	uint8_t min_refresh_in_hz;
-	uint8_t padding[1];
+	uint8_t pipe_count;
+	uint8_t pipe_index[4];
 };
 
 struct dmub_cmd_fw_assisted_mclk_switch_config {
 	uint8_t fams_enabled;
 	uint8_t visual_confirm_enabled;
-	uint8_t padding[2];
-	struct dmub_cmd_fw_assisted_mclk_switch_pipe_data pipe_data[DMUB_MAX_STREAMS];
+	uint16_t vactive_stretch_margin_us; // Extra vblank stretch required when doing FPO + Vactive
+	struct dmub_cmd_fw_assisted_mclk_switch_pipe_data pipe_data[DMUB_MAX_FPO_STREAMS];
 };
 
 struct dmub_rb_cmd_fw_assisted_mclk_switch {
diff --git a/drivers/gpu/drm/amd/display/include/dal_asic_id.h b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
index c3089c673975..e317089cf6ee 100644
--- a/drivers/gpu/drm/amd/display/include/dal_asic_id.h
+++ b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
@@ -246,6 +246,7 @@ enum {
 
 #define AMDGPU_FAMILY_GC_11_0_0 145
 #define AMDGPU_FAMILY_GC_11_0_1 148
+#define AMDGPU_FAMILY_GC_11_5_0 150
 #define GC_11_0_0_A0 0x1
 #define GC_11_0_2_A0 0x10
 #define GC_11_0_3_A0 0x20
diff --git a/drivers/gpu/drm/amd/display/include/ddc_service_types.h b/drivers/gpu/drm/amd/display/include/ddc_service_types.h
index 31a12ce79a8e..f843fc497855 100644
--- a/drivers/gpu/drm/amd/display/include/ddc_service_types.h
+++ b/drivers/gpu/drm/amd/display/include/ddc_service_types.h
@@ -129,17 +129,4 @@ struct av_sync_data {
 	uint8_t aud_del_ins3;/* DPCD 0002Dh */
 };
 
-static const uint8_t DP_SINK_DEVICE_STR_ID_1[] = {7, 1, 8, 7, 3};
-static const uint8_t DP_SINK_DEVICE_STR_ID_2[] = {7, 1, 8, 7, 5};
-
-static const u8 DP_SINK_BRANCH_DEV_NAME_7580[] = "7580\x80u";
-
-/*Travis*/
-static const uint8_t DP_VGA_LVDS_CONVERTER_ID_2[] = "sivarT";
-/*Nutmeg*/
-static const uint8_t DP_VGA_LVDS_CONVERTER_ID_3[] = "dnomlA";
-
-/*MST Dock*/
-static const uint8_t SYNAPTICS_DEVICE_ID[] = "SYNA";
-
 #endif /* __DAL_DDC_SERVICE_TYPES_H__ */
diff --git a/drivers/gpu/drm/amd/display/include/hdcp_types.h b/drivers/gpu/drm/amd/display/include/hdcp_msg_types.h
index 42229b4effdc..42229b4effdc 100644
--- a/drivers/gpu/drm/amd/display/include/hdcp_types.h
+++ b/drivers/gpu/drm/amd/display/include/hdcp_msg_types.h
diff --git a/drivers/gpu/drm/amd/display/include/link_service_types.h b/drivers/gpu/drm/amd/display/include/link_service_types.h
index 18b9173d5a96..cd870af5fd25 100644
--- a/drivers/gpu/drm/amd/display/include/link_service_types.h
+++ b/drivers/gpu/drm/amd/display/include/link_service_types.h
@@ -34,10 +34,6 @@
 struct ddc;
 struct irq_manager;
 
-enum {
-	MAX_CONTROLLER_NUM = 6
-};
-
 enum dp_power_state {
 	DP_POWER_STATE_D0 = 1,
 	DP_POWER_STATE_D3
@@ -60,28 +56,6 @@ enum {
 	DATA_EFFICIENCY_128b_132b_x10000 = 9646, /* 96.71% data efficiency x 99.75% downspread factor */
 };
 
-enum link_training_result {
-	LINK_TRAINING_SUCCESS,
-	LINK_TRAINING_CR_FAIL_LANE0,
-	LINK_TRAINING_CR_FAIL_LANE1,
-	LINK_TRAINING_CR_FAIL_LANE23,
-	/* CR DONE bit is cleared during EQ step */
-	LINK_TRAINING_EQ_FAIL_CR,
-	/* CR DONE bit is cleared but LANE0_CR_DONE is set during EQ step */
-	LINK_TRAINING_EQ_FAIL_CR_PARTIAL,
-	/* other failure during EQ step */
-	LINK_TRAINING_EQ_FAIL_EQ,
-	LINK_TRAINING_LQA_FAIL,
-	/* one of the CR,EQ or symbol lock is dropped */
-	LINK_TRAINING_LINK_LOSS,
-	/* Abort link training (because sink unplugged) */
-	LINK_TRAINING_ABORT,
-	DP_128b_132b_LT_FAILED,
-	DP_128b_132b_MAX_LOOP_COUNT_REACHED,
-	DP_128b_132b_CHANNEL_EQ_DONE_TIMEOUT,
-	DP_128b_132b_CDS_DONE_TIMEOUT,
-};
-
 enum lttpr_mode {
 	LTTPR_MODE_UNKNOWN,
 	LTTPR_MODE_NON_LTTPR,
diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
index 2be45b314922..5c41a4751db4 100644
--- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
+++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
@@ -955,26 +955,20 @@ void mod_freesync_build_vrr_infopacket(struct mod_freesync *mod_freesync,
 	 * Check if Freesync is supported. Return if false. If true,
 	 * set the corresponding bit in the info packet
 	 */
-	bool freesync_on_desktop;
-	bool fams_enable;
-
-	fams_enable = stream->ctx->dc->current_state->bw_ctx.bw.dcn.clk.fw_based_mclk_switching;
-	freesync_on_desktop = stream->freesync_on_desktop && fams_enable;
-
 	if (!vrr->send_info_frame)
 		return;
 
 	switch (packet_type) {
 	case PACKET_TYPE_FS_V3:
-		build_vrr_infopacket_v3(stream->signal, vrr, app_tf, infopacket, freesync_on_desktop);
+		build_vrr_infopacket_v3(stream->signal, vrr, app_tf, infopacket, stream->freesync_on_desktop);
 		break;
 	case PACKET_TYPE_FS_V2:
-		build_vrr_infopacket_v2(stream->signal, vrr, app_tf, infopacket, freesync_on_desktop);
+		build_vrr_infopacket_v2(stream->signal, vrr, app_tf, infopacket, stream->freesync_on_desktop);
 		break;
 	case PACKET_TYPE_VRR:
 	case PACKET_TYPE_FS_V1:
 	default:
-		build_vrr_infopacket_v1(stream->signal, vrr, infopacket, freesync_on_desktop);
+		build_vrr_infopacket_v1(stream->signal, vrr, infopacket, stream->freesync_on_desktop);
 	}
 
 	if (true == pack_sdp_v1_3 &&
@@ -1135,7 +1129,6 @@ void mod_freesync_handle_preflip(struct mod_freesync *mod_freesync,
 {
 	struct core_freesync *core_freesync = NULL;
 	unsigned int last_render_time_in_us = 0;
-	unsigned int average_render_time_in_us = 0;
 
 	if (mod_freesync == NULL)
 		return;
@@ -1144,7 +1137,6 @@ void mod_freesync_handle_preflip(struct mod_freesync *mod_freesync,
 
 	if (in_out_vrr->supported &&
 			in_out_vrr->state == VRR_STATE_ACTIVE_VARIABLE) {
-		unsigned int i = 0;
 		unsigned int oldest_index = plane->time.index + 1;
 
 		if (oldest_index >= DC_PLANE_UPDATE_TIMES_MAX)
@@ -1153,18 +1145,6 @@ void mod_freesync_handle_preflip(struct mod_freesync *mod_freesync,
 		last_render_time_in_us = curr_time_stamp_in_us -
 				plane->time.prev_update_time_in_us;
 
-		/* Sum off all entries except oldest one */
-		for (i = 0; i < DC_PLANE_UPDATE_TIMES_MAX; i++) {
-			average_render_time_in_us +=
-					plane->time.time_elapsed_in_us[i];
-		}
-		average_render_time_in_us -=
-				plane->time.time_elapsed_in_us[oldest_index];
-
-		/* Add render time for current flip */
-		average_render_time_in_us += last_render_time_in_us;
-		average_render_time_in_us /= DC_PLANE_UPDATE_TIMES_MAX;
-
 		if (in_out_vrr->btr.btr_enabled) {
 			apply_below_the_range(core_freesync,
 					stream,
diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.h b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.h
index eb6f9b9c504a..c62df3bcc7cb 100644
--- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.h
+++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.h
@@ -26,13 +26,11 @@
 #ifndef MOD_HDCP_LOG_H_
 #define MOD_HDCP_LOG_H_
 
-#ifdef CONFIG_DRM_AMD_DC_HDCP
 #define HDCP_LOG_ERR(hdcp, ...) DRM_DEBUG_KMS(__VA_ARGS__)
 #define HDCP_LOG_VER(hdcp, ...) DRM_DEBUG_KMS(__VA_ARGS__)
 #define HDCP_LOG_FSM(hdcp, ...) DRM_DEBUG_KMS(__VA_ARGS__)
 #define HDCP_LOG_TOP(hdcp, ...) pr_debug("[HDCP_TOP]:"__VA_ARGS__)
 #define HDCP_LOG_DDC(hdcp, ...) pr_debug("[HDCP_DDC]:"__VA_ARGS__)
-#endif
 
 /* default logs */
 #define HDCP_ERROR_TRACE(hdcp, status) \
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h b/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h
index 3348bb97ef81..a4d344a4db9e 100644
--- a/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h
@@ -104,6 +104,7 @@ struct mod_hdcp_displayport {
 	uint8_t rev;
 	uint8_t assr_enabled;
 	uint8_t mst_enabled;
+	uint8_t dp2_enabled;
 	uint8_t usb4_enabled;
 };
 
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
index e39b133d05af..51e76bce92ea 100644
--- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
+++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
@@ -678,13 +678,8 @@ bool dmub_init_abm_config(struct resource_pool *res_pool,
 	bool result = false;
 	uint32_t i, j = 0;
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	if (res_pool->abm == NULL && res_pool->multiple_abms[inst] == NULL)
 		return false;
-#else
-	if (res_pool->abm == NULL)
-		return false;
-#endif
 
 	memset(&ram_table, 0, sizeof(ram_table));
 	memset(&config, 0, sizeof(config));
@@ -737,12 +732,10 @@ bool dmub_init_abm_config(struct resource_pool *res_pool,
 
 	config.min_abm_backlight = ram_table.min_abm_backlight;
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
 	if (res_pool->multiple_abms[inst]) {
 		result = res_pool->multiple_abms[inst]->funcs->init_abm_config(
 			res_pool->multiple_abms[inst], (char *)(&config), sizeof(struct abm_config_table), inst);
 	} else
-#endif
 		result = res_pool->abm->funcs->init_abm_config(
 			res_pool->abm, (char *)(&config), sizeof(struct abm_config_table), 0);
 
@@ -765,8 +758,8 @@ bool dmcu_load_iram(struct dmcu *dmcu,
 
 	if (dmcu->dmcu_version.abm_version == 0x24) {
 		fill_iram_v_2_3((struct iram_table_v_2_2 *)ram_table, params, true);
-			result = dmcu->funcs->load_iram(
-					dmcu, 0, (char *)(&ram_table), IRAM_RESERVE_AREA_START_V2_2);
+		result = dmcu->funcs->load_iram(dmcu, 0, (char *)(&ram_table),
+						IRAM_RESERVE_AREA_START_V2_2);
 	} else if (dmcu->dmcu_version.abm_version == 0x23) {
 		fill_iram_v_2_3((struct iram_table_v_2_2 *)ram_table, params, true);
 
@@ -934,6 +927,10 @@ bool psr_su_set_dsc_slice_height(struct dc *dc, struct dc_link *link,
 
 	pic_height = stream->timing.v_addressable +
 		stream->timing.v_border_top + stream->timing.v_border_bottom;
+
+	if (stream->timing.dsc_cfg.num_slices_v == 0)
+		return false;
+
 	slice_height = pic_height / stream->timing.dsc_cfg.num_slices_v;
 	config->dsc_slice_height = slice_height;
 
diff --git a/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_8_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_8_0_offset.h
new file mode 100644
index 000000000000..ff74a661f477
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_8_0_offset.h
@@ -0,0 +1,411 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _athub_1_8_0_OFFSET_HEADER
+#define _athub_1_8_0_OFFSET_HEADER
+
+
+
+// addressBlock: aid_athub_atsdec
+// base address: 0x3080
+#define regATC_ATS_CNTL                                                                                 0x0000
+#define regATC_ATS_CNTL_BASE_IDX                                                                        0
+#define regATC_ATS_CNTL2                                                                                0x0001
+#define regATC_ATS_CNTL2_BASE_IDX                                                                       0
+#define regATC_ATS_CNTL3                                                                                0x0002
+#define regATC_ATS_CNTL3_BASE_IDX                                                                       0
+#define regATC_ATS_CNTL4                                                                                0x0003
+#define regATC_ATS_CNTL4_BASE_IDX                                                                       0
+#define regATC_ATS_MISC_CNTL                                                                            0x0005
+#define regATC_ATS_MISC_CNTL_BASE_IDX                                                                   0
+#define regATC_ATS_STATUS                                                                               0x0009
+#define regATC_ATS_STATUS_BASE_IDX                                                                      0
+#define regATC_PERFCOUNTER0_CFG                                                                         0x000a
+#define regATC_PERFCOUNTER0_CFG_BASE_IDX                                                                0
+#define regATC_PERFCOUNTER1_CFG                                                                         0x000b
+#define regATC_PERFCOUNTER1_CFG_BASE_IDX                                                                0
+#define regATC_PERFCOUNTER2_CFG                                                                         0x000c
+#define regATC_PERFCOUNTER2_CFG_BASE_IDX                                                                0
+#define regATC_PERFCOUNTER3_CFG                                                                         0x000d
+#define regATC_PERFCOUNTER3_CFG_BASE_IDX                                                                0
+#define regATC_PERFCOUNTER_RSLT_CNTL                                                                    0x000e
+#define regATC_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                           0
+#define regATC_PERFCOUNTER_LO                                                                           0x000f
+#define regATC_PERFCOUNTER_LO_BASE_IDX                                                                  0
+#define regATC_PERFCOUNTER_HI                                                                           0x0010
+#define regATC_PERFCOUNTER_HI_BASE_IDX                                                                  0
+#define regATC_ATS_FAULT_CNTL                                                                           0x0012
+#define regATC_ATS_FAULT_CNTL_BASE_IDX                                                                  0
+#define regATC_ATS_FAULT_STATUS_INFO                                                                    0x0013
+#define regATC_ATS_FAULT_STATUS_INFO_BASE_IDX                                                           0
+#define regATC_ATS_FAULT_STATUS_INFO2                                                                   0x0014
+#define regATC_ATS_FAULT_STATUS_INFO2_BASE_IDX                                                          0
+#define regATC_ATS_FAULT_STATUS_INFO3                                                                   0x0015
+#define regATC_ATS_FAULT_STATUS_INFO3_BASE_IDX                                                          0
+#define regATC_ATS_FAULT_STATUS_INFO4                                                                   0x0016
+#define regATC_ATS_FAULT_STATUS_INFO4_BASE_IDX                                                          0
+#define regATC_ATS_FAULT_STATUS_ADDR                                                                    0x0017
+#define regATC_ATS_FAULT_STATUS_ADDR_BASE_IDX                                                           0
+#define regATC_ATS_DEFAULT_PAGE_LOW                                                                     0x0018
+#define regATC_ATS_DEFAULT_PAGE_LOW_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL                                                                          0x001f
+#define regATHUB_PCIE_ATS_CNTL_BASE_IDX                                                                 0
+#define regATHUB_PCIE_PASID_CNTL                                                                        0x0020
+#define regATHUB_PCIE_PASID_CNTL_BASE_IDX                                                               0
+#define regATHUB_PCIE_PAGE_REQ_CNTL                                                                     0x0021
+#define regATHUB_PCIE_PAGE_REQ_CNTL_BASE_IDX                                                            0
+#define regATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC                                                           0x0022
+#define regATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC_BASE_IDX                                                  0
+#define regATHUB_COMMAND                                                                                0x0023
+#define regATHUB_COMMAND_BASE_IDX                                                                       0
+#define regATHUB_PCIE_ATS_CNTL_VF_0                                                                     0x0024
+#define regATHUB_PCIE_ATS_CNTL_VF_0_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL_VF_1                                                                     0x0025
+#define regATHUB_PCIE_ATS_CNTL_VF_1_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL_VF_2                                                                     0x0026
+#define regATHUB_PCIE_ATS_CNTL_VF_2_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL_VF_3                                                                     0x0027
+#define regATHUB_PCIE_ATS_CNTL_VF_3_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL_VF_4                                                                     0x0028
+#define regATHUB_PCIE_ATS_CNTL_VF_4_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL_VF_5                                                                     0x0029
+#define regATHUB_PCIE_ATS_CNTL_VF_5_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL_VF_6                                                                     0x002a
+#define regATHUB_PCIE_ATS_CNTL_VF_6_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL_VF_7                                                                     0x002b
+#define regATHUB_PCIE_ATS_CNTL_VF_7_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL_VF_8                                                                     0x002c
+#define regATHUB_PCIE_ATS_CNTL_VF_8_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL_VF_9                                                                     0x002d
+#define regATHUB_PCIE_ATS_CNTL_VF_9_BASE_IDX                                                            0
+#define regATHUB_PCIE_ATS_CNTL_VF_10                                                                    0x002e
+#define regATHUB_PCIE_ATS_CNTL_VF_10_BASE_IDX                                                           0
+#define regATHUB_PCIE_ATS_CNTL_VF_11                                                                    0x002f
+#define regATHUB_PCIE_ATS_CNTL_VF_11_BASE_IDX                                                           0
+#define regATHUB_PCIE_ATS_CNTL_VF_12                                                                    0x0030
+#define regATHUB_PCIE_ATS_CNTL_VF_12_BASE_IDX                                                           0
+#define regATHUB_PCIE_ATS_CNTL_VF_13                                                                    0x0031
+#define regATHUB_PCIE_ATS_CNTL_VF_13_BASE_IDX                                                           0
+#define regATHUB_PCIE_ATS_CNTL_VF_14                                                                    0x0032
+#define regATHUB_PCIE_ATS_CNTL_VF_14_BASE_IDX                                                           0
+#define regATHUB_PCIE_ATS_CNTL_VF_15                                                                    0x0033
+#define regATHUB_PCIE_ATS_CNTL_VF_15_BASE_IDX                                                           0
+#define regATHUB_SHARED_VIRT_RESET_REQ                                                                  0x0034
+#define regATHUB_SHARED_VIRT_RESET_REQ_BASE_IDX                                                         0
+#define regATHUB_SHARED_ACTIVE_FCN_ID                                                                   0x0036
+#define regATHUB_SHARED_ACTIVE_FCN_ID_BASE_IDX                                                          0
+#define regATC_ATS_SDPPORT_CNTL                                                                         0x0038
+#define regATC_ATS_SDPPORT_CNTL_BASE_IDX                                                                0
+#define regATC_VMID_PASID_MAPPING_UPDATE_STATUS                                                         0x003e
+#define regATC_VMID_PASID_MAPPING_UPDATE_STATUS_BASE_IDX                                                0
+#define regATC_VMID0_PASID_MAPPING                                                                      0x003f
+#define regATC_VMID0_PASID_MAPPING_BASE_IDX                                                             0
+#define regATC_VMID1_PASID_MAPPING                                                                      0x0040
+#define regATC_VMID1_PASID_MAPPING_BASE_IDX                                                             0
+#define regATC_VMID2_PASID_MAPPING                                                                      0x0041
+#define regATC_VMID2_PASID_MAPPING_BASE_IDX                                                             0
+#define regATC_VMID3_PASID_MAPPING                                                                      0x0042
+#define regATC_VMID3_PASID_MAPPING_BASE_IDX                                                             0
+#define regATC_VMID4_PASID_MAPPING                                                                      0x0043
+#define regATC_VMID4_PASID_MAPPING_BASE_IDX                                                             0
+#define regATC_VMID5_PASID_MAPPING                                                                      0x0044
+#define regATC_VMID5_PASID_MAPPING_BASE_IDX                                                             0
+#define regATC_VMID6_PASID_MAPPING                                                                      0x0045
+#define regATC_VMID6_PASID_MAPPING_BASE_IDX                                                             0
+#define regATC_VMID7_PASID_MAPPING                                                                      0x0046
+#define regATC_VMID7_PASID_MAPPING_BASE_IDX                                                             0
+#define regATC_VMID8_PASID_MAPPING                                                                      0x0047
+#define regATC_VMID8_PASID_MAPPING_BASE_IDX                                                             0
+#define regATC_VMID9_PASID_MAPPING                                                                      0x0048
+#define regATC_VMID9_PASID_MAPPING_BASE_IDX                                                             0
+#define regATC_VMID10_PASID_MAPPING                                                                     0x0049
+#define regATC_VMID10_PASID_MAPPING_BASE_IDX                                                            0
+#define regATC_VMID11_PASID_MAPPING                                                                     0x004a
+#define regATC_VMID11_PASID_MAPPING_BASE_IDX                                                            0
+#define regATC_VMID12_PASID_MAPPING                                                                     0x004b
+#define regATC_VMID12_PASID_MAPPING_BASE_IDX                                                            0
+#define regATC_VMID13_PASID_MAPPING                                                                     0x004c
+#define regATC_VMID13_PASID_MAPPING_BASE_IDX                                                            0
+#define regATC_VMID14_PASID_MAPPING                                                                     0x004d
+#define regATC_VMID14_PASID_MAPPING_BASE_IDX                                                            0
+#define regATC_VMID15_PASID_MAPPING                                                                     0x004e
+#define regATC_VMID15_PASID_MAPPING_BASE_IDX                                                            0
+#define regATC_TRANS_FAULT_RSPCNTRL                                                                     0x0050
+#define regATC_TRANS_FAULT_RSPCNTRL_BASE_IDX                                                            0
+#define regATC_ATS_VMID_STATUS                                                                          0x0051
+#define regATC_ATS_VMID_STATUS_BASE_IDX                                                                 0
+#define regATHUB_MISC_CNTL                                                                              0x0055
+#define regATHUB_MISC_CNTL_BASE_IDX                                                                     0
+#define regATHUB_MEM_POWER_LS                                                                           0x0056
+#define regATHUB_MEM_POWER_LS_BASE_IDX                                                                  0
+#define regATHUB_IH_CREDIT                                                                              0x0057
+#define regATHUB_IH_CREDIT_BASE_IDX                                                                     0
+
+
+// addressBlock: aid_athub_xpbdec
+// base address: 0x46000
+#define regXPB_RTR_SRC_APRTR0                                                                           0x0000
+#define regXPB_RTR_SRC_APRTR0_BASE_IDX                                                                  1
+#define regXPB_RTR_SRC_APRTR1                                                                           0x0001
+#define regXPB_RTR_SRC_APRTR1_BASE_IDX                                                                  1
+#define regXPB_RTR_SRC_APRTR2                                                                           0x0002
+#define regXPB_RTR_SRC_APRTR2_BASE_IDX                                                                  1
+#define regXPB_RTR_SRC_APRTR3                                                                           0x0003
+#define regXPB_RTR_SRC_APRTR3_BASE_IDX                                                                  1
+#define regXPB_RTR_SRC_APRTR4                                                                           0x0004
+#define regXPB_RTR_SRC_APRTR4_BASE_IDX                                                                  1
+#define regXPB_RTR_SRC_APRTR5                                                                           0x0005
+#define regXPB_RTR_SRC_APRTR5_BASE_IDX                                                                  1
+#define regXPB_RTR_SRC_APRTR6                                                                           0x0006
+#define regXPB_RTR_SRC_APRTR6_BASE_IDX                                                                  1
+#define regXPB_RTR_SRC_APRTR7                                                                           0x0007
+#define regXPB_RTR_SRC_APRTR7_BASE_IDX                                                                  1
+#define regXPB_RTR_SRC_APRTR8                                                                           0x0008
+#define regXPB_RTR_SRC_APRTR8_BASE_IDX                                                                  1
+#define regXPB_RTR_SRC_APRTR9                                                                           0x0009
+#define regXPB_RTR_SRC_APRTR9_BASE_IDX                                                                  1
+#define regXPB_XDMA_RTR_SRC_APRTR0                                                                      0x000a
+#define regXPB_XDMA_RTR_SRC_APRTR0_BASE_IDX                                                             1
+#define regXPB_XDMA_RTR_SRC_APRTR1                                                                      0x000b
+#define regXPB_XDMA_RTR_SRC_APRTR1_BASE_IDX                                                             1
+#define regXPB_XDMA_RTR_SRC_APRTR2                                                                      0x000c
+#define regXPB_XDMA_RTR_SRC_APRTR2_BASE_IDX                                                             1
+#define regXPB_XDMA_RTR_SRC_APRTR3                                                                      0x000d
+#define regXPB_XDMA_RTR_SRC_APRTR3_BASE_IDX                                                             1
+#define regXPB_RTR_DEST_MAP0                                                                            0x000e
+#define regXPB_RTR_DEST_MAP0_BASE_IDX                                                                   1
+#define regXPB_RTR_DEST_MAP1                                                                            0x000f
+#define regXPB_RTR_DEST_MAP1_BASE_IDX                                                                   1
+#define regXPB_RTR_DEST_MAP2                                                                            0x0010
+#define regXPB_RTR_DEST_MAP2_BASE_IDX                                                                   1
+#define regXPB_RTR_DEST_MAP3                                                                            0x0011
+#define regXPB_RTR_DEST_MAP3_BASE_IDX                                                                   1
+#define regXPB_RTR_DEST_MAP4                                                                            0x0012
+#define regXPB_RTR_DEST_MAP4_BASE_IDX                                                                   1
+#define regXPB_RTR_DEST_MAP5                                                                            0x0013
+#define regXPB_RTR_DEST_MAP5_BASE_IDX                                                                   1
+#define regXPB_RTR_DEST_MAP6                                                                            0x0014
+#define regXPB_RTR_DEST_MAP6_BASE_IDX                                                                   1
+#define regXPB_RTR_DEST_MAP7                                                                            0x0015
+#define regXPB_RTR_DEST_MAP7_BASE_IDX                                                                   1
+#define regXPB_RTR_DEST_MAP8                                                                            0x0016
+#define regXPB_RTR_DEST_MAP8_BASE_IDX                                                                   1
+#define regXPB_RTR_DEST_MAP9                                                                            0x0017
+#define regXPB_RTR_DEST_MAP9_BASE_IDX                                                                   1
+#define regXPB_XDMA_RTR_DEST_MAP0                                                                       0x0018
+#define regXPB_XDMA_RTR_DEST_MAP0_BASE_IDX                                                              1
+#define regXPB_XDMA_RTR_DEST_MAP1                                                                       0x0019
+#define regXPB_XDMA_RTR_DEST_MAP1_BASE_IDX                                                              1
+#define regXPB_XDMA_RTR_DEST_MAP2                                                                       0x001a
+#define regXPB_XDMA_RTR_DEST_MAP2_BASE_IDX                                                              1
+#define regXPB_XDMA_RTR_DEST_MAP3                                                                       0x001b
+#define regXPB_XDMA_RTR_DEST_MAP3_BASE_IDX                                                              1
+#define regXPB_CLG_CFG0                                                                                 0x001c
+#define regXPB_CLG_CFG0_BASE_IDX                                                                        1
+#define regXPB_CLG_CFG1                                                                                 0x001d
+#define regXPB_CLG_CFG1_BASE_IDX                                                                        1
+#define regXPB_CLG_CFG2                                                                                 0x001e
+#define regXPB_CLG_CFG2_BASE_IDX                                                                        1
+#define regXPB_CLG_CFG3                                                                                 0x001f
+#define regXPB_CLG_CFG3_BASE_IDX                                                                        1
+#define regXPB_CLG_CFG4                                                                                 0x0020
+#define regXPB_CLG_CFG4_BASE_IDX                                                                        1
+#define regXPB_CLG_CFG5                                                                                 0x0021
+#define regXPB_CLG_CFG5_BASE_IDX                                                                        1
+#define regXPB_CLG_CFG6                                                                                 0x0022
+#define regXPB_CLG_CFG6_BASE_IDX                                                                        1
+#define regXPB_CLG_CFG7                                                                                 0x0023
+#define regXPB_CLG_CFG7_BASE_IDX                                                                        1
+#define regXPB_CLG_EXTRA0                                                                               0x0024
+#define regXPB_CLG_EXTRA0_BASE_IDX                                                                      1
+#define regXPB_CLG_EXTRA1                                                                               0x0025
+#define regXPB_CLG_EXTRA1_BASE_IDX                                                                      1
+#define regXPB_CLG_EXTRA_MSK                                                                            0x0026
+#define regXPB_CLG_EXTRA_MSK_BASE_IDX                                                                   1
+#define regXPB_LB_ADDR                                                                                  0x0027
+#define regXPB_LB_ADDR_BASE_IDX                                                                         1
+#define regXPB_WCB_STS                                                                                  0x0028
+#define regXPB_WCB_STS_BASE_IDX                                                                         1
+#define regXPB_HST_CFG                                                                                  0x0029
+#define regXPB_HST_CFG_BASE_IDX                                                                         1
+#define regXPB_P2P_BAR_CFG                                                                              0x002a
+#define regXPB_P2P_BAR_CFG_BASE_IDX                                                                     1
+#define regXPB_P2P_BAR0                                                                                 0x002b
+#define regXPB_P2P_BAR0_BASE_IDX                                                                        1
+#define regXPB_P2P_BAR1                                                                                 0x002c
+#define regXPB_P2P_BAR1_BASE_IDX                                                                        1
+#define regXPB_P2P_BAR2                                                                                 0x002d
+#define regXPB_P2P_BAR2_BASE_IDX                                                                        1
+#define regXPB_P2P_BAR3                                                                                 0x002e
+#define regXPB_P2P_BAR3_BASE_IDX                                                                        1
+#define regXPB_P2P_BAR4                                                                                 0x002f
+#define regXPB_P2P_BAR4_BASE_IDX                                                                        1
+#define regXPB_P2P_BAR5                                                                                 0x0030
+#define regXPB_P2P_BAR5_BASE_IDX                                                                        1
+#define regXPB_P2P_BAR6                                                                                 0x0031
+#define regXPB_P2P_BAR6_BASE_IDX                                                                        1
+#define regXPB_P2P_BAR7                                                                                 0x0032
+#define regXPB_P2P_BAR7_BASE_IDX                                                                        1
+#define regXPB_P2P_BAR_SETUP                                                                            0x0033
+#define regXPB_P2P_BAR_SETUP_BASE_IDX                                                                   1
+#define regXPB_P2P_BAR_DELTA_ABOVE                                                                      0x0035
+#define regXPB_P2P_BAR_DELTA_ABOVE_BASE_IDX                                                             1
+#define regXPB_P2P_BAR_DELTA_BELOW                                                                      0x0036
+#define regXPB_P2P_BAR_DELTA_BELOW_BASE_IDX                                                             1
+#define regXPB_PEER_SYS_BAR0                                                                            0x0037
+#define regXPB_PEER_SYS_BAR0_BASE_IDX                                                                   1
+#define regXPB_PEER_SYS_BAR1                                                                            0x0038
+#define regXPB_PEER_SYS_BAR1_BASE_IDX                                                                   1
+#define regXPB_PEER_SYS_BAR2                                                                            0x0039
+#define regXPB_PEER_SYS_BAR2_BASE_IDX                                                                   1
+#define regXPB_PEER_SYS_BAR3                                                                            0x003a
+#define regXPB_PEER_SYS_BAR3_BASE_IDX                                                                   1
+#define regXPB_PEER_SYS_BAR4                                                                            0x003b
+#define regXPB_PEER_SYS_BAR4_BASE_IDX                                                                   1
+#define regXPB_PEER_SYS_BAR5                                                                            0x003c
+#define regXPB_PEER_SYS_BAR5_BASE_IDX                                                                   1
+#define regXPB_PEER_SYS_BAR6                                                                            0x003d
+#define regXPB_PEER_SYS_BAR6_BASE_IDX                                                                   1
+#define regXPB_PEER_SYS_BAR7                                                                            0x003e
+#define regXPB_PEER_SYS_BAR7_BASE_IDX                                                                   1
+#define regXPB_PEER_SYS_BAR8                                                                            0x003f
+#define regXPB_PEER_SYS_BAR8_BASE_IDX                                                                   1
+#define regXPB_PEER_SYS_BAR9                                                                            0x0040
+#define regXPB_PEER_SYS_BAR9_BASE_IDX                                                                   1
+#define regXPB_XDMA_PEER_SYS_BAR0                                                                       0x0041
+#define regXPB_XDMA_PEER_SYS_BAR0_BASE_IDX                                                              1
+#define regXPB_XDMA_PEER_SYS_BAR1                                                                       0x0042
+#define regXPB_XDMA_PEER_SYS_BAR1_BASE_IDX                                                              1
+#define regXPB_XDMA_PEER_SYS_BAR2                                                                       0x0043
+#define regXPB_XDMA_PEER_SYS_BAR2_BASE_IDX                                                              1
+#define regXPB_XDMA_PEER_SYS_BAR3                                                                       0x0044
+#define regXPB_XDMA_PEER_SYS_BAR3_BASE_IDX                                                              1
+#define regXPB_CLK_GAT                                                                                  0x0045
+#define regXPB_CLK_GAT_BASE_IDX                                                                         1
+#define regXPB_INTF_CFG                                                                                 0x0046
+#define regXPB_INTF_CFG_BASE_IDX                                                                        1
+#define regXPB_INTF_STS                                                                                 0x0047
+#define regXPB_INTF_STS_BASE_IDX                                                                        1
+#define regXPB_PIPE_STS                                                                                 0x0048
+#define regXPB_PIPE_STS_BASE_IDX                                                                        1
+#define regXPB_SUB_CTRL                                                                                 0x0049
+#define regXPB_SUB_CTRL_BASE_IDX                                                                        1
+#define regXPB_MAP_INVERT_FLUSH_NUM_LSB                                                                 0x004a
+#define regXPB_MAP_INVERT_FLUSH_NUM_LSB_BASE_IDX                                                        1
+#define regXPB_PERF_KNOBS                                                                               0x004b
+#define regXPB_PERF_KNOBS_BASE_IDX                                                                      1
+#define regXPB_STICKY                                                                                   0x004c
+#define regXPB_STICKY_BASE_IDX                                                                          1
+#define regXPB_STICKY_W1C                                                                               0x004d
+#define regXPB_STICKY_W1C_BASE_IDX                                                                      1
+#define regXPB_MISC_CFG                                                                                 0x004e
+#define regXPB_MISC_CFG_BASE_IDX                                                                        1
+#define regXPB_INTF_CFG2                                                                                0x004f
+#define regXPB_INTF_CFG2_BASE_IDX                                                                       1
+#define regXPB_CLG_EXTRA_RD                                                                             0x0050
+#define regXPB_CLG_EXTRA_RD_BASE_IDX                                                                    1
+#define regXPB_CLG_EXTRA_MSK_RD                                                                         0x0051
+#define regXPB_CLG_EXTRA_MSK_RD_BASE_IDX                                                                1
+#define regXPB_CLG_GFX_MATCH                                                                            0x0052
+#define regXPB_CLG_GFX_MATCH_BASE_IDX                                                                   1
+#define regXPB_CLG_GFX_MATCH_VLD                                                                        0x0053
+#define regXPB_CLG_GFX_MATCH_VLD_BASE_IDX                                                               1
+#define regXPB_CLG_GFX_MATCH_MSK                                                                        0x0054
+#define regXPB_CLG_GFX_MATCH_MSK_BASE_IDX                                                               1
+#define regXPB_CLG_MM_MATCH                                                                             0x0055
+#define regXPB_CLG_MM_MATCH_BASE_IDX                                                                    1
+#define regXPB_CLG_MM_MATCH_VLD                                                                         0x0056
+#define regXPB_CLG_MM_MATCH_VLD_BASE_IDX                                                                1
+#define regXPB_CLG_MM_MATCH_MSK                                                                         0x0057
+#define regXPB_CLG_MM_MATCH_MSK_BASE_IDX                                                                1
+#define regXPB_CLG_GFX_UNITID_MAPPING0                                                                  0x0058
+#define regXPB_CLG_GFX_UNITID_MAPPING0_BASE_IDX                                                         1
+#define regXPB_CLG_GFX_UNITID_MAPPING1                                                                  0x0059
+#define regXPB_CLG_GFX_UNITID_MAPPING1_BASE_IDX                                                         1
+#define regXPB_CLG_GFX_UNITID_MAPPING2                                                                  0x005a
+#define regXPB_CLG_GFX_UNITID_MAPPING2_BASE_IDX                                                         1
+#define regXPB_CLG_GFX_UNITID_MAPPING3                                                                  0x005b
+#define regXPB_CLG_GFX_UNITID_MAPPING3_BASE_IDX                                                         1
+#define regXPB_CLG_GFX_UNITID_MAPPING4                                                                  0x005c
+#define regXPB_CLG_GFX_UNITID_MAPPING4_BASE_IDX                                                         1
+#define regXPB_CLG_GFX_UNITID_MAPPING5                                                                  0x005d
+#define regXPB_CLG_GFX_UNITID_MAPPING5_BASE_IDX                                                         1
+#define regXPB_CLG_GFX_UNITID_MAPPING6                                                                  0x005e
+#define regXPB_CLG_GFX_UNITID_MAPPING6_BASE_IDX                                                         1
+#define regXPB_CLG_GFX_UNITID_MAPPING7                                                                  0x005f
+#define regXPB_CLG_GFX_UNITID_MAPPING7_BASE_IDX                                                         1
+#define regXPB_CLG_MM_UNITID_MAPPING0                                                                   0x0060
+#define regXPB_CLG_MM_UNITID_MAPPING0_BASE_IDX                                                          1
+#define regXPB_CLG_MM_UNITID_MAPPING1                                                                   0x0061
+#define regXPB_CLG_MM_UNITID_MAPPING1_BASE_IDX                                                          1
+#define regXPB_CLG_MM_UNITID_MAPPING2                                                                   0x0062
+#define regXPB_CLG_MM_UNITID_MAPPING2_BASE_IDX                                                          1
+#define regXPB_CLG_MM_UNITID_MAPPING3                                                                   0x0063
+#define regXPB_CLG_MM_UNITID_MAPPING3_BASE_IDX                                                          1
+
+
+// addressBlock: aid_athub_rpbdec
+// base address: 0x46200
+#define regRPB_PASSPW_CONF                                                                              0x0080
+#define regRPB_PASSPW_CONF_BASE_IDX                                                                     1
+#define regRPB_BLOCKLEVEL_CONF                                                                          0x0081
+#define regRPB_BLOCKLEVEL_CONF_BASE_IDX                                                                 1
+#define regRPB_TAG_CONF                                                                                 0x0082
+#define regRPB_TAG_CONF_BASE_IDX                                                                        1
+#define regRPB_TAG_CONF2                                                                                0x0083
+#define regRPB_TAG_CONF2_BASE_IDX                                                                       1
+#define regRPB_ARB_CNTL                                                                                 0x0085
+#define regRPB_ARB_CNTL_BASE_IDX                                                                        1
+#define regRPB_ARB_CNTL2                                                                                0x0086
+#define regRPB_ARB_CNTL2_BASE_IDX                                                                       1
+#define regRPB_BIF_CNTL                                                                                 0x0087
+#define regRPB_BIF_CNTL_BASE_IDX                                                                        1
+#define regRPB_BIF_CNTL2                                                                                0x0088
+#define regRPB_BIF_CNTL2_BASE_IDX                                                                       1
+#define regRPB_PERF_COUNTER_CNTL                                                                        0x008a
+#define regRPB_PERF_COUNTER_CNTL_BASE_IDX                                                               1
+#define regRPB_DEINTRLV_COMBINE_CNTL                                                                    0x008c
+#define regRPB_DEINTRLV_COMBINE_CNTL_BASE_IDX                                                           1
+#define regRPB_VC_SWITCH_RDWR                                                                           0x008d
+#define regRPB_VC_SWITCH_RDWR_BASE_IDX                                                                  1
+#define regRPB_PERFCOUNTER_LO                                                                           0x008e
+#define regRPB_PERFCOUNTER_LO_BASE_IDX                                                                  1
+#define regRPB_PERFCOUNTER_HI                                                                           0x008f
+#define regRPB_PERFCOUNTER_HI_BASE_IDX                                                                  1
+#define regRPB_PERFCOUNTER0_CFG                                                                         0x0090
+#define regRPB_PERFCOUNTER0_CFG_BASE_IDX                                                                1
+#define regRPB_PERFCOUNTER1_CFG                                                                         0x0091
+#define regRPB_PERFCOUNTER1_CFG_BASE_IDX                                                                1
+#define regRPB_PERFCOUNTER2_CFG                                                                         0x0092
+#define regRPB_PERFCOUNTER2_CFG_BASE_IDX                                                                1
+#define regRPB_PERFCOUNTER3_CFG                                                                         0x0093
+#define regRPB_PERFCOUNTER3_CFG_BASE_IDX                                                                1
+#define regRPB_PERFCOUNTER_RSLT_CNTL                                                                    0x0094
+#define regRPB_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                           1
+#define regRPB_ATS_CNTL                                                                                 0x0096
+#define regRPB_ATS_CNTL_BASE_IDX                                                                        1
+#define regRPB_ATS_CNTL2                                                                                0x0097
+#define regRPB_ATS_CNTL2_BASE_IDX                                                                       1
+#define regRPB_SDPPORT_CNTL                                                                             0x0098
+#define regRPB_SDPPORT_CNTL_BASE_IDX                                                                    1
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_8_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_8_0_sh_mask.h
new file mode 100644
index 000000000000..91454c528ad7
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_8_0_sh_mask.h
@@ -0,0 +1,1807 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _athub_1_8_0_SH_MASK_HEADER
+#define _athub_1_8_0_SH_MASK_HEADER
+
+
+// addressBlock: aid_athub_atsdec
+//ATC_ATS_CNTL
+#define ATC_ATS_CNTL__DISABLE_ATC__SHIFT                                                                      0x0
+#define ATC_ATS_CNTL__DISABLE_PRI__SHIFT                                                                      0x1
+#define ATC_ATS_CNTL__DISABLE_PASID__SHIFT                                                                    0x2
+#define ATC_ATS_CNTL__CREDITS_ATS_RPB__SHIFT                                                                  0x8
+#define ATC_ATS_CNTL__DEBUG_ECO__SHIFT                                                                        0x10
+#define ATC_ATS_CNTL__TRANS_EXE_RETURN__SHIFT                                                                 0x16
+#define ATC_ATS_CNTL__DISABLE_ATC_MASK                                                                        0x00000001L
+#define ATC_ATS_CNTL__DISABLE_PRI_MASK                                                                        0x00000002L
+#define ATC_ATS_CNTL__DISABLE_PASID_MASK                                                                      0x00000004L
+#define ATC_ATS_CNTL__CREDITS_ATS_RPB_MASK                                                                    0x00003F00L
+#define ATC_ATS_CNTL__DEBUG_ECO_MASK                                                                          0x000F0000L
+#define ATC_ATS_CNTL__TRANS_EXE_RETURN_MASK                                                                   0x00C00000L
+//ATC_ATS_CNTL2
+#define ATC_ATS_CNTL2__CREDITS_ATS_RPB_VC5TR__SHIFT                                                           0x0
+#define ATC_ATS_CNTL2__CREDITS_ATS_RPB_VC0TR__SHIFT                                                           0x8
+#define ATC_ATS_CNTL2__CREDITS_ATS_RPB_PRINV__SHIFT                                                           0x10
+#define ATC_ATS_CNTL2__TRANSLATION_STALL__SHIFT                                                               0x18
+#define ATC_ATS_CNTL2__GC_TRANS_VC5_ENABLE__SHIFT                                                             0x19
+#define ATC_ATS_CNTL2__MM_TRANS_VC5_ENABLE__SHIFT                                                             0x1a
+#define ATC_ATS_CNTL2__RESERVED__SHIFT                                                                        0x1b
+#define ATC_ATS_CNTL2__CREDITS_ATS_RPB_VC5TR_MASK                                                             0x000000FFL
+#define ATC_ATS_CNTL2__CREDITS_ATS_RPB_VC0TR_MASK                                                             0x0000FF00L
+#define ATC_ATS_CNTL2__CREDITS_ATS_RPB_PRINV_MASK                                                             0x00FF0000L
+#define ATC_ATS_CNTL2__TRANSLATION_STALL_MASK                                                                 0x01000000L
+#define ATC_ATS_CNTL2__GC_TRANS_VC5_ENABLE_MASK                                                               0x02000000L
+#define ATC_ATS_CNTL2__MM_TRANS_VC5_ENABLE_MASK                                                               0x04000000L
+#define ATC_ATS_CNTL2__RESERVED_MASK                                                                          0xF8000000L
+//ATC_ATS_CNTL3
+#define ATC_ATS_CNTL3__RESERVED__SHIFT                                                                        0x0
+#define ATC_ATS_CNTL3__RESERVED_MASK                                                                          0xFFFFFFFFL
+//ATC_ATS_CNTL4
+#define ATC_ATS_CNTL4__RESERVED__SHIFT                                                                        0x0
+#define ATC_ATS_CNTL4__RESERVED_MASK                                                                          0xFFFFFFFFL
+//ATC_ATS_MISC_CNTL
+#define ATC_ATS_MISC_CNTL__TRANS_RESP_IN_INV_COLLISION_HOST__SHIFT                                            0x10
+#define ATC_ATS_MISC_CNTL__TRANS_RESP_IN_INV_COLLISION_GUEST__SHIFT                                           0x11
+#define ATC_ATS_MISC_CNTL__DEBUG_COLLISION__SHIFT                                                             0x12
+#define ATC_ATS_MISC_CNTL__EFFECTIVE_TRANS_WORK_QUEUE__SHIFT                                                  0x13
+#define ATC_ATS_MISC_CNTL__TRANS_RESP_NULL_PASID_SEL__SHIFT                                                   0x1d
+#define ATC_ATS_MISC_CNTL__RESERVED__SHIFT                                                                    0x1e
+#define ATC_ATS_MISC_CNTL__TRANS_RESP_IN_INV_COLLISION_HOST_MASK                                              0x00010000L
+#define ATC_ATS_MISC_CNTL__TRANS_RESP_IN_INV_COLLISION_GUEST_MASK                                             0x00020000L
+#define ATC_ATS_MISC_CNTL__DEBUG_COLLISION_MASK                                                               0x00040000L
+#define ATC_ATS_MISC_CNTL__EFFECTIVE_TRANS_WORK_QUEUE_MASK                                                    0x1FF80000L
+#define ATC_ATS_MISC_CNTL__TRANS_RESP_NULL_PASID_SEL_MASK                                                     0x20000000L
+#define ATC_ATS_MISC_CNTL__RESERVED_MASK                                                                      0xC0000000L
+//ATC_ATS_STATUS
+#define ATC_ATS_STATUS__BUSY__SHIFT                                                                           0x0
+#define ATC_ATS_STATUS__CRASHED__SHIFT                                                                        0x1
+#define ATC_ATS_STATUS__DEADLOCK_DETECTION__SHIFT                                                             0x2
+#define ATC_ATS_STATUS__FED_IND__SHIFT                                                                        0x3
+#define ATC_ATS_STATUS__BUSY_MASK                                                                             0x00000001L
+#define ATC_ATS_STATUS__CRASHED_MASK                                                                          0x00000002L
+#define ATC_ATS_STATUS__DEADLOCK_DETECTION_MASK                                                               0x00000004L
+#define ATC_ATS_STATUS__FED_IND_MASK                                                                          0x00000008L
+//ATC_PERFCOUNTER0_CFG
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                                 0x0
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                             0x8
+#define ATC_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                                0x18
+#define ATC_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                   0x1c
+#define ATC_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                    0x1d
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                   0x000000FFL
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
+#define ATC_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                  0x0F000000L
+#define ATC_PERFCOUNTER0_CFG__ENABLE_MASK                                                                     0x10000000L
+#define ATC_PERFCOUNTER0_CFG__CLEAR_MASK                                                                      0x20000000L
+//ATC_PERFCOUNTER1_CFG
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                                 0x0
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                             0x8
+#define ATC_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                                0x18
+#define ATC_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                   0x1c
+#define ATC_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                    0x1d
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                   0x000000FFL
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
+#define ATC_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                  0x0F000000L
+#define ATC_PERFCOUNTER1_CFG__ENABLE_MASK                                                                     0x10000000L
+#define ATC_PERFCOUNTER1_CFG__CLEAR_MASK                                                                      0x20000000L
+//ATC_PERFCOUNTER2_CFG
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                                 0x0
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                             0x8
+#define ATC_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                                0x18
+#define ATC_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                   0x1c
+#define ATC_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                    0x1d
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                   0x000000FFL
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
+#define ATC_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                  0x0F000000L
+#define ATC_PERFCOUNTER2_CFG__ENABLE_MASK                                                                     0x10000000L
+#define ATC_PERFCOUNTER2_CFG__CLEAR_MASK                                                                      0x20000000L
+//ATC_PERFCOUNTER3_CFG
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                                 0x0
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                             0x8
+#define ATC_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                                0x18
+#define ATC_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                                   0x1c
+#define ATC_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                                    0x1d
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                                   0x000000FFL
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
+#define ATC_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                                  0x0F000000L
+#define ATC_PERFCOUNTER3_CFG__ENABLE_MASK                                                                     0x10000000L
+#define ATC_PERFCOUNTER3_CFG__CLEAR_MASK                                                                      0x20000000L
+//ATC_PERFCOUNTER_RSLT_CNTL
+#define ATC_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                                 0x0
+#define ATC_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                       0x8
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                        0x10
+#define ATC_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                          0x18
+#define ATC_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                           0x19
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                                0x1a
+#define ATC_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                   0x0000000FL
+#define ATC_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                         0x0000FF00L
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                          0x00FF0000L
+#define ATC_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                            0x01000000L
+#define ATC_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                             0x02000000L
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                  0x04000000L
+//ATC_PERFCOUNTER_LO
+#define ATC_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                                 0x0
+#define ATC_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                   0xFFFFFFFFL
+//ATC_PERFCOUNTER_HI
+#define ATC_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                                 0x0
+#define ATC_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                              0x10
+#define ATC_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                   0x0000FFFFL
+#define ATC_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                                0xFFFF0000L
+//ATC_ATS_FAULT_CNTL
+#define ATC_ATS_FAULT_CNTL__FAULT_REGISTER_LOG__SHIFT                                                         0x0
+#define ATC_ATS_FAULT_CNTL__FAULT_INTERRUPT_TABLE__SHIFT                                                      0xa
+#define ATC_ATS_FAULT_CNTL__FAULT_CRASH_TABLE__SHIFT                                                          0x14
+#define ATC_ATS_FAULT_CNTL__FAULT_REGISTER_LOG_MASK                                                           0x000001FFL
+#define ATC_ATS_FAULT_CNTL__FAULT_INTERRUPT_TABLE_MASK                                                        0x0007FC00L
+#define ATC_ATS_FAULT_CNTL__FAULT_CRASH_TABLE_MASK                                                            0x1FF00000L
+//ATC_ATS_FAULT_STATUS_INFO
+#define ATC_ATS_FAULT_STATUS_INFO__FAULT_TYPE__SHIFT                                                          0x0
+#define ATC_ATS_FAULT_STATUS_INFO__VMID_ALL__SHIFT                                                            0x9
+#define ATC_ATS_FAULT_STATUS_INFO__VMID__SHIFT                                                                0xa
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO__SHIFT                                                          0xf
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO2__SHIFT                                                         0x10
+#define ATC_ATS_FAULT_STATUS_INFO__INVALIDATION__SHIFT                                                        0x11
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_REQUEST__SHIFT                                                        0x12
+#define ATC_ATS_FAULT_STATUS_INFO__STATUS__SHIFT                                                              0x13
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_ADDR_HIGH__SHIFT                                                      0x18
+#define ATC_ATS_FAULT_STATUS_INFO__PHYSICAL_ADDR_HIGH__SHIFT                                                  0x1c
+#define ATC_ATS_FAULT_STATUS_INFO__FAULT_TYPE_MASK                                                            0x000001FFL
+#define ATC_ATS_FAULT_STATUS_INFO__VMID_ALL_MASK                                                              0x00000200L
+#define ATC_ATS_FAULT_STATUS_INFO__VMID_MASK                                                                  0x00007C00L
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO_MASK                                                            0x00008000L
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO2_MASK                                                           0x00010000L
+#define ATC_ATS_FAULT_STATUS_INFO__INVALIDATION_MASK                                                          0x00020000L
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_REQUEST_MASK                                                          0x00040000L
+#define ATC_ATS_FAULT_STATUS_INFO__STATUS_MASK                                                                0x00F80000L
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_ADDR_HIGH_MASK                                                        0x0F000000L
+#define ATC_ATS_FAULT_STATUS_INFO__PHYSICAL_ADDR_HIGH_MASK                                                    0xF0000000L
+//ATC_ATS_FAULT_STATUS_INFO2
+#define ATC_ATS_FAULT_STATUS_INFO2__VF__SHIFT                                                                 0x0
+#define ATC_ATS_FAULT_STATUS_INFO2__VFID__SHIFT                                                               0x1
+#define ATC_ATS_FAULT_STATUS_INFO2__L2NUM__SHIFT                                                              0x9
+#define ATC_ATS_FAULT_STATUS_INFO2__INV_VMID_GFX1__SHIFT                                                      0xd
+#define ATC_ATS_FAULT_STATUS_INFO2__INV_VMID_GFX2__SHIFT                                                      0x13
+#define ATC_ATS_FAULT_STATUS_INFO2__INV_VMID_GFX3__SHIFT                                                      0x19
+#define ATC_ATS_FAULT_STATUS_INFO2__VF_MASK                                                                   0x00000001L
+#define ATC_ATS_FAULT_STATUS_INFO2__VFID_MASK                                                                 0x0000001EL
+#define ATC_ATS_FAULT_STATUS_INFO2__L2NUM_MASK                                                                0x00001E00L
+#define ATC_ATS_FAULT_STATUS_INFO2__INV_VMID_GFX1_MASK                                                        0x0007E000L
+#define ATC_ATS_FAULT_STATUS_INFO2__INV_VMID_GFX2_MASK                                                        0x01F80000L
+#define ATC_ATS_FAULT_STATUS_INFO2__INV_VMID_GFX3_MASK                                                        0x7E000000L
+//ATC_ATS_FAULT_STATUS_INFO3
+#define ATC_ATS_FAULT_STATUS_INFO3__INV_VMID_GFX4__SHIFT                                                      0x0
+#define ATC_ATS_FAULT_STATUS_INFO3__INV_VMID_GFX5__SHIFT                                                      0x6
+#define ATC_ATS_FAULT_STATUS_INFO3__INV_VMID_GFX6__SHIFT                                                      0xc
+#define ATC_ATS_FAULT_STATUS_INFO3__INV_VMID_GFX7__SHIFT                                                      0x12
+#define ATC_ATS_FAULT_STATUS_INFO3__INV_VMID_MM0__SHIFT                                                       0x18
+#define ATC_ATS_FAULT_STATUS_INFO3__INV_VMID_GFX4_MASK                                                        0x0000003FL
+#define ATC_ATS_FAULT_STATUS_INFO3__INV_VMID_GFX5_MASK                                                        0x00000FC0L
+#define ATC_ATS_FAULT_STATUS_INFO3__INV_VMID_GFX6_MASK                                                        0x0003F000L
+#define ATC_ATS_FAULT_STATUS_INFO3__INV_VMID_GFX7_MASK                                                        0x00FC0000L
+#define ATC_ATS_FAULT_STATUS_INFO3__INV_VMID_MM0_MASK                                                         0x3F000000L
+//ATC_ATS_FAULT_STATUS_INFO4
+#define ATC_ATS_FAULT_STATUS_INFO4__INV_VMID_MM1__SHIFT                                                       0x0
+#define ATC_ATS_FAULT_STATUS_INFO4__INV_VMID_MM2__SHIFT                                                       0x6
+#define ATC_ATS_FAULT_STATUS_INFO4__INV_VMID_MM3__SHIFT                                                       0xc
+#define ATC_ATS_FAULT_STATUS_INFO4__INV_VMID_MM1_MASK                                                         0x0000003FL
+#define ATC_ATS_FAULT_STATUS_INFO4__INV_VMID_MM2_MASK                                                         0x00000FC0L
+#define ATC_ATS_FAULT_STATUS_INFO4__INV_VMID_MM3_MASK                                                         0x0003F000L
+//ATC_ATS_FAULT_STATUS_ADDR
+#define ATC_ATS_FAULT_STATUS_ADDR__PAGE_ADDR__SHIFT                                                           0x0
+#define ATC_ATS_FAULT_STATUS_ADDR__PAGE_ADDR_MASK                                                             0xFFFFFFFFL
+//ATC_ATS_DEFAULT_PAGE_LOW
+#define ATC_ATS_DEFAULT_PAGE_LOW__DEFAULT_PAGE__SHIFT                                                         0x0
+#define ATC_ATS_DEFAULT_PAGE_LOW__DEFAULT_PAGE_MASK                                                           0xFFFFFFFFL
+//ATHUB_PCIE_ATS_CNTL
+#define ATHUB_PCIE_ATS_CNTL__STU__SHIFT                                                                       0x10
+#define ATHUB_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                                0x1f
+#define ATHUB_PCIE_ATS_CNTL__STU_MASK                                                                         0x001F0000L
+#define ATHUB_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                                  0x80000000L
+//ATHUB_PCIE_PASID_CNTL
+#define ATHUB_PCIE_PASID_CNTL__PASID_EN__SHIFT                                                                0x10
+#define ATHUB_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                             0x11
+#define ATHUB_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                                        0x12
+#define ATHUB_PCIE_PASID_CNTL__PASID_EN_MASK                                                                  0x00010000L
+#define ATHUB_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                               0x00020000L
+#define ATHUB_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                                          0x00040000L
+//ATHUB_PCIE_PAGE_REQ_CNTL
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_ENABLE__SHIFT                                                           0x0
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_RESET__SHIFT                                                            0x1
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_ENABLE_MASK                                                             0x00000001L
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_RESET_MASK                                                              0x00000002L
+//ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC
+#define ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC__SHIFT                                    0x0
+#define ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC_MASK                                      0xFFFFFFFFL
+//ATHUB_COMMAND
+#define ATHUB_COMMAND__BUS_MASTER_EN__SHIFT                                                                   0x2
+#define ATHUB_COMMAND__BUS_MASTER_EN_MASK                                                                     0x00000004L
+//ATHUB_PCIE_ATS_CNTL_VF_0
+#define ATHUB_PCIE_ATS_CNTL_VF_0__ATC_ENABLE__SHIFT                                                           0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_0__ATC_ENABLE_MASK                                                             0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_1
+#define ATHUB_PCIE_ATS_CNTL_VF_1__ATC_ENABLE__SHIFT                                                           0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_1__ATC_ENABLE_MASK                                                             0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_2
+#define ATHUB_PCIE_ATS_CNTL_VF_2__ATC_ENABLE__SHIFT                                                           0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_2__ATC_ENABLE_MASK                                                             0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_3
+#define ATHUB_PCIE_ATS_CNTL_VF_3__ATC_ENABLE__SHIFT                                                           0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_3__ATC_ENABLE_MASK                                                             0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_4
+#define ATHUB_PCIE_ATS_CNTL_VF_4__ATC_ENABLE__SHIFT                                                           0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_4__ATC_ENABLE_MASK                                                             0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_5
+#define ATHUB_PCIE_ATS_CNTL_VF_5__ATC_ENABLE__SHIFT                                                           0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_5__ATC_ENABLE_MASK                                                             0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_6
+#define ATHUB_PCIE_ATS_CNTL_VF_6__ATC_ENABLE__SHIFT                                                           0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_6__ATC_ENABLE_MASK                                                             0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_7
+#define ATHUB_PCIE_ATS_CNTL_VF_7__ATC_ENABLE__SHIFT                                                           0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_7__ATC_ENABLE_MASK                                                             0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_8
+#define ATHUB_PCIE_ATS_CNTL_VF_8__ATC_ENABLE__SHIFT                                                           0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_8__ATC_ENABLE_MASK                                                             0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_9
+#define ATHUB_PCIE_ATS_CNTL_VF_9__ATC_ENABLE__SHIFT                                                           0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_9__ATC_ENABLE_MASK                                                             0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_10
+#define ATHUB_PCIE_ATS_CNTL_VF_10__ATC_ENABLE__SHIFT                                                          0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_10__ATC_ENABLE_MASK                                                            0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_11
+#define ATHUB_PCIE_ATS_CNTL_VF_11__ATC_ENABLE__SHIFT                                                          0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_11__ATC_ENABLE_MASK                                                            0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_12
+#define ATHUB_PCIE_ATS_CNTL_VF_12__ATC_ENABLE__SHIFT                                                          0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_12__ATC_ENABLE_MASK                                                            0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_13
+#define ATHUB_PCIE_ATS_CNTL_VF_13__ATC_ENABLE__SHIFT                                                          0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_13__ATC_ENABLE_MASK                                                            0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_14
+#define ATHUB_PCIE_ATS_CNTL_VF_14__ATC_ENABLE__SHIFT                                                          0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_14__ATC_ENABLE_MASK                                                            0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_15
+#define ATHUB_PCIE_ATS_CNTL_VF_15__ATC_ENABLE__SHIFT                                                          0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_15__ATC_ENABLE_MASK                                                            0x80000000L
+//ATHUB_SHARED_VIRT_RESET_REQ
+#define ATHUB_SHARED_VIRT_RESET_REQ__VF__SHIFT                                                                0x0
+#define ATHUB_SHARED_VIRT_RESET_REQ__PF__SHIFT                                                                0x1f
+#define ATHUB_SHARED_VIRT_RESET_REQ__VF_MASK                                                                  0x0000FFFFL
+#define ATHUB_SHARED_VIRT_RESET_REQ__PF_MASK                                                                  0x80000000L
+//ATHUB_SHARED_ACTIVE_FCN_ID
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VFID__SHIFT                                                               0x0
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VF__SHIFT                                                                 0x1f
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VFID_MASK                                                                 0x0000000FL
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VF_MASK                                                                   0x80000000L
+//ATC_ATS_SDPPORT_CNTL
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_SELF_ACTIVATE__SHIFT                                                    0x0
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_CFG_MODE__SHIFT                                                         0x1
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_HALT_THRESHOLD__SHIFT                                                   0x3
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_REISSUE_CREDIT__SHIFT                                                   0x7
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_ENABLE_DISRUPT_FULLDIS__SHIFT                                           0x8
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_SELF_ACTIVATE__SHIFT                                                0x9
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_QUICK_COMACK__SHIFT                                                 0xa
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_HALT_THRESHOLD__SHIFT                                               0xb
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_PASSIVE_MODE__SHIFT                                                 0xf
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_RDY_MODE__SHIFT                                                           0x10
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_REISSUE_CREDIT__SHIFT                                               0x11
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_ENABLE_DISRUPT_FULLDIS__SHIFT                                       0x12
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_INVALREQ_RDRSPSTATUS_CNTL__SHIFT                                    0x13
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKEN__SHIFT                                              0x16
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKENRCV__SHIFT                                           0x17
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKEN__SHIFT                                          0x18
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKENRCV__SHIFT                                       0x19
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKEN__SHIFT                                              0x1a
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKENRCV__SHIFT                                           0x1b
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKEN__SHIFT                                                0x1c
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKENRCV__SHIFT                                             0x1d
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKEN__SHIFT                                           0x1e
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKENRCV__SHIFT                                        0x1f
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_SELF_ACTIVATE_MASK                                                      0x00000001L
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_CFG_MODE_MASK                                                           0x00000006L
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_HALT_THRESHOLD_MASK                                                     0x00000078L
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_REISSUE_CREDIT_MASK                                                     0x00000080L
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_ENABLE_DISRUPT_FULLDIS_MASK                                             0x00000100L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_SELF_ACTIVATE_MASK                                                  0x00000200L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_QUICK_COMACK_MASK                                                   0x00000400L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_HALT_THRESHOLD_MASK                                                 0x00007800L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_PASSIVE_MODE_MASK                                                   0x00008000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_RDY_MODE_MASK                                                             0x00010000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_REISSUE_CREDIT_MASK                                                 0x00020000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_ENABLE_DISRUPT_FULLDIS_MASK                                         0x00040000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_INVALREQ_RDRSPSTATUS_CNTL_MASK                                      0x00380000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKEN_MASK                                                0x00400000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKENRCV_MASK                                             0x00800000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKEN_MASK                                            0x01000000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKENRCV_MASK                                         0x02000000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKEN_MASK                                                0x04000000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKENRCV_MASK                                             0x08000000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKEN_MASK                                                  0x10000000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKENRCV_MASK                                               0x20000000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKEN_MASK                                             0x40000000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKENRCV_MASK                                          0x80000000L
+//ATC_VMID_PASID_MAPPING_UPDATE_STATUS
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID0_REMAPPING_FINISHED__SHIFT                                 0x0
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID1_REMAPPING_FINISHED__SHIFT                                 0x1
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID2_REMAPPING_FINISHED__SHIFT                                 0x2
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID3_REMAPPING_FINISHED__SHIFT                                 0x3
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID4_REMAPPING_FINISHED__SHIFT                                 0x4
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID5_REMAPPING_FINISHED__SHIFT                                 0x5
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID6_REMAPPING_FINISHED__SHIFT                                 0x6
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID7_REMAPPING_FINISHED__SHIFT                                 0x7
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID8_REMAPPING_FINISHED__SHIFT                                 0x8
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID9_REMAPPING_FINISHED__SHIFT                                 0x9
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID10_REMAPPING_FINISHED__SHIFT                                0xa
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID11_REMAPPING_FINISHED__SHIFT                                0xb
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID12_REMAPPING_FINISHED__SHIFT                                0xc
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID13_REMAPPING_FINISHED__SHIFT                                0xd
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID14_REMAPPING_FINISHED__SHIFT                                0xe
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID15_REMAPPING_FINISHED__SHIFT                                0xf
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID0_REMAPPING_FINISHED_MASK                                   0x00000001L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID1_REMAPPING_FINISHED_MASK                                   0x00000002L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID2_REMAPPING_FINISHED_MASK                                   0x00000004L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID3_REMAPPING_FINISHED_MASK                                   0x00000008L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID4_REMAPPING_FINISHED_MASK                                   0x00000010L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID5_REMAPPING_FINISHED_MASK                                   0x00000020L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID6_REMAPPING_FINISHED_MASK                                   0x00000040L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID7_REMAPPING_FINISHED_MASK                                   0x00000080L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID8_REMAPPING_FINISHED_MASK                                   0x00000100L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID9_REMAPPING_FINISHED_MASK                                   0x00000200L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID10_REMAPPING_FINISHED_MASK                                  0x00000400L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID11_REMAPPING_FINISHED_MASK                                  0x00000800L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID12_REMAPPING_FINISHED_MASK                                  0x00001000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID13_REMAPPING_FINISHED_MASK                                  0x00002000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID14_REMAPPING_FINISHED_MASK                                  0x00004000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID15_REMAPPING_FINISHED_MASK                                  0x00008000L
+//ATC_VMID0_PASID_MAPPING
+#define ATC_VMID0_PASID_MAPPING__PASID__SHIFT                                                                 0x0
+#define ATC_VMID0_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
+#define ATC_VMID0_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
+#define ATC_VMID0_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
+#define ATC_VMID0_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
+#define ATC_VMID0_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
+//ATC_VMID1_PASID_MAPPING
+#define ATC_VMID1_PASID_MAPPING__PASID__SHIFT                                                                 0x0
+#define ATC_VMID1_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
+#define ATC_VMID1_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
+#define ATC_VMID1_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
+#define ATC_VMID1_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
+#define ATC_VMID1_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
+//ATC_VMID2_PASID_MAPPING
+#define ATC_VMID2_PASID_MAPPING__PASID__SHIFT                                                                 0x0
+#define ATC_VMID2_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
+#define ATC_VMID2_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
+#define ATC_VMID2_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
+#define ATC_VMID2_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
+#define ATC_VMID2_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
+//ATC_VMID3_PASID_MAPPING
+#define ATC_VMID3_PASID_MAPPING__PASID__SHIFT                                                                 0x0
+#define ATC_VMID3_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
+#define ATC_VMID3_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
+#define ATC_VMID3_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
+#define ATC_VMID3_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
+#define ATC_VMID3_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
+//ATC_VMID4_PASID_MAPPING
+#define ATC_VMID4_PASID_MAPPING__PASID__SHIFT                                                                 0x0
+#define ATC_VMID4_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
+#define ATC_VMID4_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
+#define ATC_VMID4_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
+#define ATC_VMID4_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
+#define ATC_VMID4_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
+//ATC_VMID5_PASID_MAPPING
+#define ATC_VMID5_PASID_MAPPING__PASID__SHIFT                                                                 0x0
+#define ATC_VMID5_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
+#define ATC_VMID5_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
+#define ATC_VMID5_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
+#define ATC_VMID5_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
+#define ATC_VMID5_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
+//ATC_VMID6_PASID_MAPPING
+#define ATC_VMID6_PASID_MAPPING__PASID__SHIFT                                                                 0x0
+#define ATC_VMID6_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
+#define ATC_VMID6_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
+#define ATC_VMID6_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
+#define ATC_VMID6_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
+#define ATC_VMID6_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
+//ATC_VMID7_PASID_MAPPING
+#define ATC_VMID7_PASID_MAPPING__PASID__SHIFT                                                                 0x0
+#define ATC_VMID7_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
+#define ATC_VMID7_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
+#define ATC_VMID7_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
+#define ATC_VMID7_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
+#define ATC_VMID7_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
+//ATC_VMID8_PASID_MAPPING
+#define ATC_VMID8_PASID_MAPPING__PASID__SHIFT                                                                 0x0
+#define ATC_VMID8_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
+#define ATC_VMID8_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
+#define ATC_VMID8_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
+#define ATC_VMID8_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
+#define ATC_VMID8_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
+//ATC_VMID9_PASID_MAPPING
+#define ATC_VMID9_PASID_MAPPING__PASID__SHIFT                                                                 0x0
+#define ATC_VMID9_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
+#define ATC_VMID9_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
+#define ATC_VMID9_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
+#define ATC_VMID9_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
+#define ATC_VMID9_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
+//ATC_VMID10_PASID_MAPPING
+#define ATC_VMID10_PASID_MAPPING__PASID__SHIFT                                                                0x0
+#define ATC_VMID10_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
+#define ATC_VMID10_PASID_MAPPING__VALID__SHIFT                                                                0x1f
+#define ATC_VMID10_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
+#define ATC_VMID10_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
+#define ATC_VMID10_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
+//ATC_VMID11_PASID_MAPPING
+#define ATC_VMID11_PASID_MAPPING__PASID__SHIFT                                                                0x0
+#define ATC_VMID11_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
+#define ATC_VMID11_PASID_MAPPING__VALID__SHIFT                                                                0x1f
+#define ATC_VMID11_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
+#define ATC_VMID11_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
+#define ATC_VMID11_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
+//ATC_VMID12_PASID_MAPPING
+#define ATC_VMID12_PASID_MAPPING__PASID__SHIFT                                                                0x0
+#define ATC_VMID12_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
+#define ATC_VMID12_PASID_MAPPING__VALID__SHIFT                                                                0x1f
+#define ATC_VMID12_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
+#define ATC_VMID12_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
+#define ATC_VMID12_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
+//ATC_VMID13_PASID_MAPPING
+#define ATC_VMID13_PASID_MAPPING__PASID__SHIFT                                                                0x0
+#define ATC_VMID13_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
+#define ATC_VMID13_PASID_MAPPING__VALID__SHIFT                                                                0x1f
+#define ATC_VMID13_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
+#define ATC_VMID13_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
+#define ATC_VMID13_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
+//ATC_VMID14_PASID_MAPPING
+#define ATC_VMID14_PASID_MAPPING__PASID__SHIFT                                                                0x0
+#define ATC_VMID14_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
+#define ATC_VMID14_PASID_MAPPING__VALID__SHIFT                                                                0x1f
+#define ATC_VMID14_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
+#define ATC_VMID14_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
+#define ATC_VMID14_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
+//ATC_VMID15_PASID_MAPPING
+#define ATC_VMID15_PASID_MAPPING__PASID__SHIFT                                                                0x0
+#define ATC_VMID15_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
+#define ATC_VMID15_PASID_MAPPING__VALID__SHIFT                                                                0x1f
+#define ATC_VMID15_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
+#define ATC_VMID15_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
+#define ATC_VMID15_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
+//ATC_TRANS_FAULT_RSPCNTRL
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID0__SHIFT                                                                0x0
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID1__SHIFT                                                                0x1
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID2__SHIFT                                                                0x2
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID3__SHIFT                                                                0x3
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID4__SHIFT                                                                0x4
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID5__SHIFT                                                                0x5
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID6__SHIFT                                                                0x6
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID7__SHIFT                                                                0x7
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID8__SHIFT                                                                0x8
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID9__SHIFT                                                                0x9
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID10__SHIFT                                                               0xa
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID11__SHIFT                                                               0xb
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID12__SHIFT                                                               0xc
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID13__SHIFT                                                               0xd
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID14__SHIFT                                                               0xe
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID15__SHIFT                                                               0xf
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID0_MASK                                                                  0x00000001L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID1_MASK                                                                  0x00000002L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID2_MASK                                                                  0x00000004L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID3_MASK                                                                  0x00000008L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID4_MASK                                                                  0x00000010L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID5_MASK                                                                  0x00000020L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID6_MASK                                                                  0x00000040L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID7_MASK                                                                  0x00000080L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID8_MASK                                                                  0x00000100L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID9_MASK                                                                  0x00000200L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID10_MASK                                                                 0x00000400L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID11_MASK                                                                 0x00000800L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID12_MASK                                                                 0x00001000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID13_MASK                                                                 0x00002000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID14_MASK                                                                 0x00004000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID15_MASK                                                                 0x00008000L
+//ATC_ATS_VMID_STATUS
+#define ATC_ATS_VMID_STATUS__VMID0_OUTSTANDING__SHIFT                                                         0x0
+#define ATC_ATS_VMID_STATUS__VMID1_OUTSTANDING__SHIFT                                                         0x1
+#define ATC_ATS_VMID_STATUS__VMID2_OUTSTANDING__SHIFT                                                         0x2
+#define ATC_ATS_VMID_STATUS__VMID3_OUTSTANDING__SHIFT                                                         0x3
+#define ATC_ATS_VMID_STATUS__VMID4_OUTSTANDING__SHIFT                                                         0x4
+#define ATC_ATS_VMID_STATUS__VMID5_OUTSTANDING__SHIFT                                                         0x5
+#define ATC_ATS_VMID_STATUS__VMID6_OUTSTANDING__SHIFT                                                         0x6
+#define ATC_ATS_VMID_STATUS__VMID7_OUTSTANDING__SHIFT                                                         0x7
+#define ATC_ATS_VMID_STATUS__VMID8_OUTSTANDING__SHIFT                                                         0x8
+#define ATC_ATS_VMID_STATUS__VMID9_OUTSTANDING__SHIFT                                                         0x9
+#define ATC_ATS_VMID_STATUS__VMID10_OUTSTANDING__SHIFT                                                        0xa
+#define ATC_ATS_VMID_STATUS__VMID11_OUTSTANDING__SHIFT                                                        0xb
+#define ATC_ATS_VMID_STATUS__VMID12_OUTSTANDING__SHIFT                                                        0xc
+#define ATC_ATS_VMID_STATUS__VMID13_OUTSTANDING__SHIFT                                                        0xd
+#define ATC_ATS_VMID_STATUS__VMID14_OUTSTANDING__SHIFT                                                        0xe
+#define ATC_ATS_VMID_STATUS__VMID15_OUTSTANDING__SHIFT                                                        0xf
+#define ATC_ATS_VMID_STATUS__VMID0_OUTSTANDING_MASK                                                           0x00000001L
+#define ATC_ATS_VMID_STATUS__VMID1_OUTSTANDING_MASK                                                           0x00000002L
+#define ATC_ATS_VMID_STATUS__VMID2_OUTSTANDING_MASK                                                           0x00000004L
+#define ATC_ATS_VMID_STATUS__VMID3_OUTSTANDING_MASK                                                           0x00000008L
+#define ATC_ATS_VMID_STATUS__VMID4_OUTSTANDING_MASK                                                           0x00000010L
+#define ATC_ATS_VMID_STATUS__VMID5_OUTSTANDING_MASK                                                           0x00000020L
+#define ATC_ATS_VMID_STATUS__VMID6_OUTSTANDING_MASK                                                           0x00000040L
+#define ATC_ATS_VMID_STATUS__VMID7_OUTSTANDING_MASK                                                           0x00000080L
+#define ATC_ATS_VMID_STATUS__VMID8_OUTSTANDING_MASK                                                           0x00000100L
+#define ATC_ATS_VMID_STATUS__VMID9_OUTSTANDING_MASK                                                           0x00000200L
+#define ATC_ATS_VMID_STATUS__VMID10_OUTSTANDING_MASK                                                          0x00000400L
+#define ATC_ATS_VMID_STATUS__VMID11_OUTSTANDING_MASK                                                          0x00000800L
+#define ATC_ATS_VMID_STATUS__VMID12_OUTSTANDING_MASK                                                          0x00001000L
+#define ATC_ATS_VMID_STATUS__VMID13_OUTSTANDING_MASK                                                          0x00002000L
+#define ATC_ATS_VMID_STATUS__VMID14_OUTSTANDING_MASK                                                          0x00004000L
+#define ATC_ATS_VMID_STATUS__VMID15_OUTSTANDING_MASK                                                          0x00008000L
+//ATHUB_MISC_CNTL
+#define ATHUB_MISC_CNTL__CG_OFFDLY__SHIFT                                                                     0x6
+#define ATHUB_MISC_CNTL__CG_ENABLE__SHIFT                                                                     0x12
+#define ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE__SHIFT                                                              0x13
+#define ATHUB_MISC_CNTL__PG_ENABLE__SHIFT                                                                     0x14
+#define ATHUB_MISC_CNTL__PG_OFFDLY__SHIFT                                                                     0x15
+#define ATHUB_MISC_CNTL__CG_STATUS__SHIFT                                                                     0x1b
+#define ATHUB_MISC_CNTL__PG_STATUS__SHIFT                                                                     0x1c
+#define ATHUB_MISC_CNTL__SRAM_FGCG_ENABLE__SHIFT                                                              0x1d
+#define ATHUB_MISC_CNTL__CG_OFFDLY_MASK                                                                       0x00000FC0L
+#define ATHUB_MISC_CNTL__CG_ENABLE_MASK                                                                       0x00040000L
+#define ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK                                                                0x00080000L
+#define ATHUB_MISC_CNTL__PG_ENABLE_MASK                                                                       0x00100000L
+#define ATHUB_MISC_CNTL__PG_OFFDLY_MASK                                                                       0x07E00000L
+#define ATHUB_MISC_CNTL__CG_STATUS_MASK                                                                       0x08000000L
+#define ATHUB_MISC_CNTL__PG_STATUS_MASK                                                                       0x10000000L
+#define ATHUB_MISC_CNTL__SRAM_FGCG_ENABLE_MASK                                                                0x20000000L
+//ATHUB_MEM_POWER_LS
+#define ATHUB_MEM_POWER_LS__LS_SETUP__SHIFT                                                                   0x0
+#define ATHUB_MEM_POWER_LS__LS_HOLD__SHIFT                                                                    0x6
+#define ATHUB_MEM_POWER_LS__LS_DELAY_ENABLE__SHIFT                                                            0x13
+#define ATHUB_MEM_POWER_LS__LS_DELAY_TIME__SHIFT                                                              0x14
+#define ATHUB_MEM_POWER_LS__LS_SETUP_MASK                                                                     0x0000003FL
+#define ATHUB_MEM_POWER_LS__LS_HOLD_MASK                                                                      0x0007FFC0L
+#define ATHUB_MEM_POWER_LS__LS_DELAY_ENABLE_MASK                                                              0x00080000L
+#define ATHUB_MEM_POWER_LS__LS_DELAY_TIME_MASK                                                                0x03F00000L
+//ATHUB_IH_CREDIT
+#define ATHUB_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                  0x0
+#define ATHUB_IH_CREDIT__IH_CLIENT_ID__SHIFT                                                                  0x10
+#define ATHUB_IH_CREDIT__CREDIT_VALUE_MASK                                                                    0x00000003L
+#define ATHUB_IH_CREDIT__IH_CLIENT_ID_MASK                                                                    0x00FF0000L
+
+
+// addressBlock: aid_athub_xpbdec
+//XPB_RTR_SRC_APRTR0
+#define XPB_RTR_SRC_APRTR0__BASE_ADDR__SHIFT                                                                  0x0
+#define XPB_RTR_SRC_APRTR0__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR1
+#define XPB_RTR_SRC_APRTR1__BASE_ADDR__SHIFT                                                                  0x0
+#define XPB_RTR_SRC_APRTR1__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR2
+#define XPB_RTR_SRC_APRTR2__BASE_ADDR__SHIFT                                                                  0x0
+#define XPB_RTR_SRC_APRTR2__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR3
+#define XPB_RTR_SRC_APRTR3__BASE_ADDR__SHIFT                                                                  0x0
+#define XPB_RTR_SRC_APRTR3__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR4
+#define XPB_RTR_SRC_APRTR4__BASE_ADDR__SHIFT                                                                  0x0
+#define XPB_RTR_SRC_APRTR4__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR5
+#define XPB_RTR_SRC_APRTR5__BASE_ADDR__SHIFT                                                                  0x0
+#define XPB_RTR_SRC_APRTR5__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR6
+#define XPB_RTR_SRC_APRTR6__BASE_ADDR__SHIFT                                                                  0x0
+#define XPB_RTR_SRC_APRTR6__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR7
+#define XPB_RTR_SRC_APRTR7__BASE_ADDR__SHIFT                                                                  0x0
+#define XPB_RTR_SRC_APRTR7__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR8
+#define XPB_RTR_SRC_APRTR8__BASE_ADDR__SHIFT                                                                  0x0
+#define XPB_RTR_SRC_APRTR8__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR9
+#define XPB_RTR_SRC_APRTR9__BASE_ADDR__SHIFT                                                                  0x0
+#define XPB_RTR_SRC_APRTR9__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR0
+#define XPB_XDMA_RTR_SRC_APRTR0__BASE_ADDR__SHIFT                                                             0x0
+#define XPB_XDMA_RTR_SRC_APRTR0__BASE_ADDR_MASK                                                               0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR1
+#define XPB_XDMA_RTR_SRC_APRTR1__BASE_ADDR__SHIFT                                                             0x0
+#define XPB_XDMA_RTR_SRC_APRTR1__BASE_ADDR_MASK                                                               0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR2
+#define XPB_XDMA_RTR_SRC_APRTR2__BASE_ADDR__SHIFT                                                             0x0
+#define XPB_XDMA_RTR_SRC_APRTR2__BASE_ADDR_MASK                                                               0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR3
+#define XPB_XDMA_RTR_SRC_APRTR3__BASE_ADDR__SHIFT                                                             0x0
+#define XPB_XDMA_RTR_SRC_APRTR3__BASE_ADDR_MASK                                                               0x7FFFFFFFL
+//XPB_RTR_DEST_MAP0
+#define XPB_RTR_DEST_MAP0__NMR__SHIFT                                                                         0x0
+#define XPB_RTR_DEST_MAP0__DEST_OFFSET__SHIFT                                                                 0x1
+#define XPB_RTR_DEST_MAP0__DEST_SEL__SHIFT                                                                    0x14
+#define XPB_RTR_DEST_MAP0__DEST_SEL_RPB__SHIFT                                                                0x18
+#define XPB_RTR_DEST_MAP0__SIDE_OK__SHIFT                                                                     0x19
+#define XPB_RTR_DEST_MAP0__APRTR_SIZE__SHIFT                                                                  0x1a
+#define XPB_RTR_DEST_MAP0__NMR_MASK                                                                           0x00000001L
+#define XPB_RTR_DEST_MAP0__DEST_OFFSET_MASK                                                                   0x000FFFFEL
+#define XPB_RTR_DEST_MAP0__DEST_SEL_MASK                                                                      0x00F00000L
+#define XPB_RTR_DEST_MAP0__DEST_SEL_RPB_MASK                                                                  0x01000000L
+#define XPB_RTR_DEST_MAP0__SIDE_OK_MASK                                                                       0x02000000L
+#define XPB_RTR_DEST_MAP0__APRTR_SIZE_MASK                                                                    0x7C000000L
+//XPB_RTR_DEST_MAP1
+#define XPB_RTR_DEST_MAP1__NMR__SHIFT                                                                         0x0
+#define XPB_RTR_DEST_MAP1__DEST_OFFSET__SHIFT                                                                 0x1
+#define XPB_RTR_DEST_MAP1__DEST_SEL__SHIFT                                                                    0x14
+#define XPB_RTR_DEST_MAP1__DEST_SEL_RPB__SHIFT                                                                0x18
+#define XPB_RTR_DEST_MAP1__SIDE_OK__SHIFT                                                                     0x19
+#define XPB_RTR_DEST_MAP1__APRTR_SIZE__SHIFT                                                                  0x1a
+#define XPB_RTR_DEST_MAP1__NMR_MASK                                                                           0x00000001L
+#define XPB_RTR_DEST_MAP1__DEST_OFFSET_MASK                                                                   0x000FFFFEL
+#define XPB_RTR_DEST_MAP1__DEST_SEL_MASK                                                                      0x00F00000L
+#define XPB_RTR_DEST_MAP1__DEST_SEL_RPB_MASK                                                                  0x01000000L
+#define XPB_RTR_DEST_MAP1__SIDE_OK_MASK                                                                       0x02000000L
+#define XPB_RTR_DEST_MAP1__APRTR_SIZE_MASK                                                                    0x7C000000L
+//XPB_RTR_DEST_MAP2
+#define XPB_RTR_DEST_MAP2__NMR__SHIFT                                                                         0x0
+#define XPB_RTR_DEST_MAP2__DEST_OFFSET__SHIFT                                                                 0x1
+#define XPB_RTR_DEST_MAP2__DEST_SEL__SHIFT                                                                    0x14
+#define XPB_RTR_DEST_MAP2__DEST_SEL_RPB__SHIFT                                                                0x18
+#define XPB_RTR_DEST_MAP2__SIDE_OK__SHIFT                                                                     0x19
+#define XPB_RTR_DEST_MAP2__APRTR_SIZE__SHIFT                                                                  0x1a
+#define XPB_RTR_DEST_MAP2__NMR_MASK                                                                           0x00000001L
+#define XPB_RTR_DEST_MAP2__DEST_OFFSET_MASK                                                                   0x000FFFFEL
+#define XPB_RTR_DEST_MAP2__DEST_SEL_MASK                                                                      0x00F00000L
+#define XPB_RTR_DEST_MAP2__DEST_SEL_RPB_MASK                                                                  0x01000000L
+#define XPB_RTR_DEST_MAP2__SIDE_OK_MASK                                                                       0x02000000L
+#define XPB_RTR_DEST_MAP2__APRTR_SIZE_MASK                                                                    0x7C000000L
+//XPB_RTR_DEST_MAP3
+#define XPB_RTR_DEST_MAP3__NMR__SHIFT                                                                         0x0
+#define XPB_RTR_DEST_MAP3__DEST_OFFSET__SHIFT                                                                 0x1
+#define XPB_RTR_DEST_MAP3__DEST_SEL__SHIFT                                                                    0x14
+#define XPB_RTR_DEST_MAP3__DEST_SEL_RPB__SHIFT                                                                0x18
+#define XPB_RTR_DEST_MAP3__SIDE_OK__SHIFT                                                                     0x19
+#define XPB_RTR_DEST_MAP3__APRTR_SIZE__SHIFT                                                                  0x1a
+#define XPB_RTR_DEST_MAP3__NMR_MASK                                                                           0x00000001L
+#define XPB_RTR_DEST_MAP3__DEST_OFFSET_MASK                                                                   0x000FFFFEL
+#define XPB_RTR_DEST_MAP3__DEST_SEL_MASK                                                                      0x00F00000L
+#define XPB_RTR_DEST_MAP3__DEST_SEL_RPB_MASK                                                                  0x01000000L
+#define XPB_RTR_DEST_MAP3__SIDE_OK_MASK                                                                       0x02000000L
+#define XPB_RTR_DEST_MAP3__APRTR_SIZE_MASK                                                                    0x7C000000L
+//XPB_RTR_DEST_MAP4
+#define XPB_RTR_DEST_MAP4__NMR__SHIFT                                                                         0x0
+#define XPB_RTR_DEST_MAP4__DEST_OFFSET__SHIFT                                                                 0x1
+#define XPB_RTR_DEST_MAP4__DEST_SEL__SHIFT                                                                    0x14
+#define XPB_RTR_DEST_MAP4__DEST_SEL_RPB__SHIFT                                                                0x18
+#define XPB_RTR_DEST_MAP4__SIDE_OK__SHIFT                                                                     0x19
+#define XPB_RTR_DEST_MAP4__APRTR_SIZE__SHIFT                                                                  0x1a
+#define XPB_RTR_DEST_MAP4__NMR_MASK                                                                           0x00000001L
+#define XPB_RTR_DEST_MAP4__DEST_OFFSET_MASK                                                                   0x000FFFFEL
+#define XPB_RTR_DEST_MAP4__DEST_SEL_MASK                                                                      0x00F00000L
+#define XPB_RTR_DEST_MAP4__DEST_SEL_RPB_MASK                                                                  0x01000000L
+#define XPB_RTR_DEST_MAP4__SIDE_OK_MASK                                                                       0x02000000L
+#define XPB_RTR_DEST_MAP4__APRTR_SIZE_MASK                                                                    0x7C000000L
+//XPB_RTR_DEST_MAP5
+#define XPB_RTR_DEST_MAP5__NMR__SHIFT                                                                         0x0
+#define XPB_RTR_DEST_MAP5__DEST_OFFSET__SHIFT                                                                 0x1
+#define XPB_RTR_DEST_MAP5__DEST_SEL__SHIFT                                                                    0x14
+#define XPB_RTR_DEST_MAP5__DEST_SEL_RPB__SHIFT                                                                0x18
+#define XPB_RTR_DEST_MAP5__SIDE_OK__SHIFT                                                                     0x19
+#define XPB_RTR_DEST_MAP5__APRTR_SIZE__SHIFT                                                                  0x1a
+#define XPB_RTR_DEST_MAP5__NMR_MASK                                                                           0x00000001L
+#define XPB_RTR_DEST_MAP5__DEST_OFFSET_MASK                                                                   0x000FFFFEL
+#define XPB_RTR_DEST_MAP5__DEST_SEL_MASK                                                                      0x00F00000L
+#define XPB_RTR_DEST_MAP5__DEST_SEL_RPB_MASK                                                                  0x01000000L
+#define XPB_RTR_DEST_MAP5__SIDE_OK_MASK                                                                       0x02000000L
+#define XPB_RTR_DEST_MAP5__APRTR_SIZE_MASK                                                                    0x7C000000L
+//XPB_RTR_DEST_MAP6
+#define XPB_RTR_DEST_MAP6__NMR__SHIFT                                                                         0x0
+#define XPB_RTR_DEST_MAP6__DEST_OFFSET__SHIFT                                                                 0x1
+#define XPB_RTR_DEST_MAP6__DEST_SEL__SHIFT                                                                    0x14
+#define XPB_RTR_DEST_MAP6__DEST_SEL_RPB__SHIFT                                                                0x18
+#define XPB_RTR_DEST_MAP6__SIDE_OK__SHIFT                                                                     0x19
+#define XPB_RTR_DEST_MAP6__APRTR_SIZE__SHIFT                                                                  0x1a
+#define XPB_RTR_DEST_MAP6__NMR_MASK                                                                           0x00000001L
+#define XPB_RTR_DEST_MAP6__DEST_OFFSET_MASK                                                                   0x000FFFFEL
+#define XPB_RTR_DEST_MAP6__DEST_SEL_MASK                                                                      0x00F00000L
+#define XPB_RTR_DEST_MAP6__DEST_SEL_RPB_MASK                                                                  0x01000000L
+#define XPB_RTR_DEST_MAP6__SIDE_OK_MASK                                                                       0x02000000L
+#define XPB_RTR_DEST_MAP6__APRTR_SIZE_MASK                                                                    0x7C000000L
+//XPB_RTR_DEST_MAP7
+#define XPB_RTR_DEST_MAP7__NMR__SHIFT                                                                         0x0
+#define XPB_RTR_DEST_MAP7__DEST_OFFSET__SHIFT                                                                 0x1
+#define XPB_RTR_DEST_MAP7__DEST_SEL__SHIFT                                                                    0x14
+#define XPB_RTR_DEST_MAP7__DEST_SEL_RPB__SHIFT                                                                0x18
+#define XPB_RTR_DEST_MAP7__SIDE_OK__SHIFT                                                                     0x19
+#define XPB_RTR_DEST_MAP7__APRTR_SIZE__SHIFT                                                                  0x1a
+#define XPB_RTR_DEST_MAP7__NMR_MASK                                                                           0x00000001L
+#define XPB_RTR_DEST_MAP7__DEST_OFFSET_MASK                                                                   0x000FFFFEL
+#define XPB_RTR_DEST_MAP7__DEST_SEL_MASK                                                                      0x00F00000L
+#define XPB_RTR_DEST_MAP7__DEST_SEL_RPB_MASK                                                                  0x01000000L
+#define XPB_RTR_DEST_MAP7__SIDE_OK_MASK                                                                       0x02000000L
+#define XPB_RTR_DEST_MAP7__APRTR_SIZE_MASK                                                                    0x7C000000L
+//XPB_RTR_DEST_MAP8
+#define XPB_RTR_DEST_MAP8__NMR__SHIFT                                                                         0x0
+#define XPB_RTR_DEST_MAP8__DEST_OFFSET__SHIFT                                                                 0x1
+#define XPB_RTR_DEST_MAP8__DEST_SEL__SHIFT                                                                    0x14
+#define XPB_RTR_DEST_MAP8__DEST_SEL_RPB__SHIFT                                                                0x18
+#define XPB_RTR_DEST_MAP8__SIDE_OK__SHIFT                                                                     0x19
+#define XPB_RTR_DEST_MAP8__APRTR_SIZE__SHIFT                                                                  0x1a
+#define XPB_RTR_DEST_MAP8__NMR_MASK                                                                           0x00000001L
+#define XPB_RTR_DEST_MAP8__DEST_OFFSET_MASK                                                                   0x000FFFFEL
+#define XPB_RTR_DEST_MAP8__DEST_SEL_MASK                                                                      0x00F00000L
+#define XPB_RTR_DEST_MAP8__DEST_SEL_RPB_MASK                                                                  0x01000000L
+#define XPB_RTR_DEST_MAP8__SIDE_OK_MASK                                                                       0x02000000L
+#define XPB_RTR_DEST_MAP8__APRTR_SIZE_MASK                                                                    0x7C000000L
+//XPB_RTR_DEST_MAP9
+#define XPB_RTR_DEST_MAP9__NMR__SHIFT                                                                         0x0
+#define XPB_RTR_DEST_MAP9__DEST_OFFSET__SHIFT                                                                 0x1
+#define XPB_RTR_DEST_MAP9__DEST_SEL__SHIFT                                                                    0x14
+#define XPB_RTR_DEST_MAP9__DEST_SEL_RPB__SHIFT                                                                0x18
+#define XPB_RTR_DEST_MAP9__SIDE_OK__SHIFT                                                                     0x19
+#define XPB_RTR_DEST_MAP9__APRTR_SIZE__SHIFT                                                                  0x1a
+#define XPB_RTR_DEST_MAP9__NMR_MASK                                                                           0x00000001L
+#define XPB_RTR_DEST_MAP9__DEST_OFFSET_MASK                                                                   0x000FFFFEL
+#define XPB_RTR_DEST_MAP9__DEST_SEL_MASK                                                                      0x00F00000L
+#define XPB_RTR_DEST_MAP9__DEST_SEL_RPB_MASK                                                                  0x01000000L
+#define XPB_RTR_DEST_MAP9__SIDE_OK_MASK                                                                       0x02000000L
+#define XPB_RTR_DEST_MAP9__APRTR_SIZE_MASK                                                                    0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP0
+#define XPB_XDMA_RTR_DEST_MAP0__NMR__SHIFT                                                                    0x0
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_OFFSET__SHIFT                                                            0x1
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL__SHIFT                                                               0x14
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_RPB__SHIFT                                                           0x18
+#define XPB_XDMA_RTR_DEST_MAP0__SIDE_OK__SHIFT                                                                0x19
+#define XPB_XDMA_RTR_DEST_MAP0__APRTR_SIZE__SHIFT                                                             0x1a
+#define XPB_XDMA_RTR_DEST_MAP0__NMR_MASK                                                                      0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_OFFSET_MASK                                                              0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_MASK                                                                 0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_RPB_MASK                                                             0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP0__SIDE_OK_MASK                                                                  0x02000000L
+#define XPB_XDMA_RTR_DEST_MAP0__APRTR_SIZE_MASK                                                               0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP1
+#define XPB_XDMA_RTR_DEST_MAP1__NMR__SHIFT                                                                    0x0
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_OFFSET__SHIFT                                                            0x1
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL__SHIFT                                                               0x14
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_RPB__SHIFT                                                           0x18
+#define XPB_XDMA_RTR_DEST_MAP1__SIDE_OK__SHIFT                                                                0x19
+#define XPB_XDMA_RTR_DEST_MAP1__APRTR_SIZE__SHIFT                                                             0x1a
+#define XPB_XDMA_RTR_DEST_MAP1__NMR_MASK                                                                      0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_OFFSET_MASK                                                              0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_MASK                                                                 0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_RPB_MASK                                                             0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP1__SIDE_OK_MASK                                                                  0x02000000L
+#define XPB_XDMA_RTR_DEST_MAP1__APRTR_SIZE_MASK                                                               0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP2
+#define XPB_XDMA_RTR_DEST_MAP2__NMR__SHIFT                                                                    0x0
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_OFFSET__SHIFT                                                            0x1
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL__SHIFT                                                               0x14
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_RPB__SHIFT                                                           0x18
+#define XPB_XDMA_RTR_DEST_MAP2__SIDE_OK__SHIFT                                                                0x19
+#define XPB_XDMA_RTR_DEST_MAP2__APRTR_SIZE__SHIFT                                                             0x1a
+#define XPB_XDMA_RTR_DEST_MAP2__NMR_MASK                                                                      0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_OFFSET_MASK                                                              0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_MASK                                                                 0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_RPB_MASK                                                             0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP2__SIDE_OK_MASK                                                                  0x02000000L
+#define XPB_XDMA_RTR_DEST_MAP2__APRTR_SIZE_MASK                                                               0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP3
+#define XPB_XDMA_RTR_DEST_MAP3__NMR__SHIFT                                                                    0x0
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_OFFSET__SHIFT                                                            0x1
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL__SHIFT                                                               0x14
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_RPB__SHIFT                                                           0x18
+#define XPB_XDMA_RTR_DEST_MAP3__SIDE_OK__SHIFT                                                                0x19
+#define XPB_XDMA_RTR_DEST_MAP3__APRTR_SIZE__SHIFT                                                             0x1a
+#define XPB_XDMA_RTR_DEST_MAP3__NMR_MASK                                                                      0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_OFFSET_MASK                                                              0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_MASK                                                                 0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_RPB_MASK                                                             0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP3__SIDE_OK_MASK                                                                  0x02000000L
+#define XPB_XDMA_RTR_DEST_MAP3__APRTR_SIZE_MASK                                                               0x7C000000L
+//XPB_CLG_CFG0
+#define XPB_CLG_CFG0__WCB_NUM__SHIFT                                                                          0x0
+#define XPB_CLG_CFG0__LB_TYPE__SHIFT                                                                          0x4
+#define XPB_CLG_CFG0__P2P_BAR__SHIFT                                                                          0x7
+#define XPB_CLG_CFG0__HOST_FLUSH__SHIFT                                                                       0xa
+#define XPB_CLG_CFG0__SIDE_FLUSH__SHIFT                                                                       0xe
+#define XPB_CLG_CFG0__WCB_NUM_MASK                                                                            0x0000000FL
+#define XPB_CLG_CFG0__LB_TYPE_MASK                                                                            0x00000070L
+#define XPB_CLG_CFG0__P2P_BAR_MASK                                                                            0x00000380L
+#define XPB_CLG_CFG0__HOST_FLUSH_MASK                                                                         0x00003C00L
+#define XPB_CLG_CFG0__SIDE_FLUSH_MASK                                                                         0x0003C000L
+//XPB_CLG_CFG1
+#define XPB_CLG_CFG1__WCB_NUM__SHIFT                                                                          0x0
+#define XPB_CLG_CFG1__LB_TYPE__SHIFT                                                                          0x4
+#define XPB_CLG_CFG1__P2P_BAR__SHIFT                                                                          0x7
+#define XPB_CLG_CFG1__HOST_FLUSH__SHIFT                                                                       0xa
+#define XPB_CLG_CFG1__SIDE_FLUSH__SHIFT                                                                       0xe
+#define XPB_CLG_CFG1__WCB_NUM_MASK                                                                            0x0000000FL
+#define XPB_CLG_CFG1__LB_TYPE_MASK                                                                            0x00000070L
+#define XPB_CLG_CFG1__P2P_BAR_MASK                                                                            0x00000380L
+#define XPB_CLG_CFG1__HOST_FLUSH_MASK                                                                         0x00003C00L
+#define XPB_CLG_CFG1__SIDE_FLUSH_MASK                                                                         0x0003C000L
+//XPB_CLG_CFG2
+#define XPB_CLG_CFG2__WCB_NUM__SHIFT                                                                          0x0
+#define XPB_CLG_CFG2__LB_TYPE__SHIFT                                                                          0x4
+#define XPB_CLG_CFG2__P2P_BAR__SHIFT                                                                          0x7
+#define XPB_CLG_CFG2__HOST_FLUSH__SHIFT                                                                       0xa
+#define XPB_CLG_CFG2__SIDE_FLUSH__SHIFT                                                                       0xe
+#define XPB_CLG_CFG2__WCB_NUM_MASK                                                                            0x0000000FL
+#define XPB_CLG_CFG2__LB_TYPE_MASK                                                                            0x00000070L
+#define XPB_CLG_CFG2__P2P_BAR_MASK                                                                            0x00000380L
+#define XPB_CLG_CFG2__HOST_FLUSH_MASK                                                                         0x00003C00L
+#define XPB_CLG_CFG2__SIDE_FLUSH_MASK                                                                         0x0003C000L
+//XPB_CLG_CFG3
+#define XPB_CLG_CFG3__WCB_NUM__SHIFT                                                                          0x0
+#define XPB_CLG_CFG3__LB_TYPE__SHIFT                                                                          0x4
+#define XPB_CLG_CFG3__P2P_BAR__SHIFT                                                                          0x7
+#define XPB_CLG_CFG3__HOST_FLUSH__SHIFT                                                                       0xa
+#define XPB_CLG_CFG3__SIDE_FLUSH__SHIFT                                                                       0xe
+#define XPB_CLG_CFG3__WCB_NUM_MASK                                                                            0x0000000FL
+#define XPB_CLG_CFG3__LB_TYPE_MASK                                                                            0x00000070L
+#define XPB_CLG_CFG3__P2P_BAR_MASK                                                                            0x00000380L
+#define XPB_CLG_CFG3__HOST_FLUSH_MASK                                                                         0x00003C00L
+#define XPB_CLG_CFG3__SIDE_FLUSH_MASK                                                                         0x0003C000L
+//XPB_CLG_CFG4
+#define XPB_CLG_CFG4__WCB_NUM__SHIFT                                                                          0x0
+#define XPB_CLG_CFG4__LB_TYPE__SHIFT                                                                          0x4
+#define XPB_CLG_CFG4__P2P_BAR__SHIFT                                                                          0x7
+#define XPB_CLG_CFG4__HOST_FLUSH__SHIFT                                                                       0xa
+#define XPB_CLG_CFG4__SIDE_FLUSH__SHIFT                                                                       0xe
+#define XPB_CLG_CFG4__WCB_NUM_MASK                                                                            0x0000000FL
+#define XPB_CLG_CFG4__LB_TYPE_MASK                                                                            0x00000070L
+#define XPB_CLG_CFG4__P2P_BAR_MASK                                                                            0x00000380L
+#define XPB_CLG_CFG4__HOST_FLUSH_MASK                                                                         0x00003C00L
+#define XPB_CLG_CFG4__SIDE_FLUSH_MASK                                                                         0x0003C000L
+//XPB_CLG_CFG5
+#define XPB_CLG_CFG5__WCB_NUM__SHIFT                                                                          0x0
+#define XPB_CLG_CFG5__LB_TYPE__SHIFT                                                                          0x4
+#define XPB_CLG_CFG5__P2P_BAR__SHIFT                                                                          0x7
+#define XPB_CLG_CFG5__HOST_FLUSH__SHIFT                                                                       0xa
+#define XPB_CLG_CFG5__SIDE_FLUSH__SHIFT                                                                       0xe
+#define XPB_CLG_CFG5__WCB_NUM_MASK                                                                            0x0000000FL
+#define XPB_CLG_CFG5__LB_TYPE_MASK                                                                            0x00000070L
+#define XPB_CLG_CFG5__P2P_BAR_MASK                                                                            0x00000380L
+#define XPB_CLG_CFG5__HOST_FLUSH_MASK                                                                         0x00003C00L
+#define XPB_CLG_CFG5__SIDE_FLUSH_MASK                                                                         0x0003C000L
+//XPB_CLG_CFG6
+#define XPB_CLG_CFG6__WCB_NUM__SHIFT                                                                          0x0
+#define XPB_CLG_CFG6__LB_TYPE__SHIFT                                                                          0x4
+#define XPB_CLG_CFG6__P2P_BAR__SHIFT                                                                          0x7
+#define XPB_CLG_CFG6__HOST_FLUSH__SHIFT                                                                       0xa
+#define XPB_CLG_CFG6__SIDE_FLUSH__SHIFT                                                                       0xe
+#define XPB_CLG_CFG6__WCB_NUM_MASK                                                                            0x0000000FL
+#define XPB_CLG_CFG6__LB_TYPE_MASK                                                                            0x00000070L
+#define XPB_CLG_CFG6__P2P_BAR_MASK                                                                            0x00000380L
+#define XPB_CLG_CFG6__HOST_FLUSH_MASK                                                                         0x00003C00L
+#define XPB_CLG_CFG6__SIDE_FLUSH_MASK                                                                         0x0003C000L
+//XPB_CLG_CFG7
+#define XPB_CLG_CFG7__WCB_NUM__SHIFT                                                                          0x0
+#define XPB_CLG_CFG7__LB_TYPE__SHIFT                                                                          0x4
+#define XPB_CLG_CFG7__P2P_BAR__SHIFT                                                                          0x7
+#define XPB_CLG_CFG7__HOST_FLUSH__SHIFT                                                                       0xa
+#define XPB_CLG_CFG7__SIDE_FLUSH__SHIFT                                                                       0xe
+#define XPB_CLG_CFG7__WCB_NUM_MASK                                                                            0x0000000FL
+#define XPB_CLG_CFG7__LB_TYPE_MASK                                                                            0x00000070L
+#define XPB_CLG_CFG7__P2P_BAR_MASK                                                                            0x00000380L
+#define XPB_CLG_CFG7__HOST_FLUSH_MASK                                                                         0x00003C00L
+#define XPB_CLG_CFG7__SIDE_FLUSH_MASK                                                                         0x0003C000L
+//XPB_CLG_EXTRA0
+#define XPB_CLG_EXTRA0__CMP0_HIGH__SHIFT                                                                      0x0
+#define XPB_CLG_EXTRA0__CMP0_LOW__SHIFT                                                                       0x8
+#define XPB_CLG_EXTRA0__VLD0__SHIFT                                                                           0xd
+#define XPB_CLG_EXTRA0__CLG0_NUM__SHIFT                                                                       0xe
+#define XPB_CLG_EXTRA0__CMP0_HIGH_MASK                                                                        0x000000FFL
+#define XPB_CLG_EXTRA0__CMP0_LOW_MASK                                                                         0x00001F00L
+#define XPB_CLG_EXTRA0__VLD0_MASK                                                                             0x00002000L
+#define XPB_CLG_EXTRA0__CLG0_NUM_MASK                                                                         0x0001C000L
+//XPB_CLG_EXTRA1
+#define XPB_CLG_EXTRA1__CMP1_HIGH__SHIFT                                                                      0x0
+#define XPB_CLG_EXTRA1__CMP1_LOW__SHIFT                                                                       0x8
+#define XPB_CLG_EXTRA1__VLD1__SHIFT                                                                           0xd
+#define XPB_CLG_EXTRA1__CLG1_NUM__SHIFT                                                                       0xe
+#define XPB_CLG_EXTRA1__CMP1_HIGH_MASK                                                                        0x000000FFL
+#define XPB_CLG_EXTRA1__CMP1_LOW_MASK                                                                         0x00001F00L
+#define XPB_CLG_EXTRA1__VLD1_MASK                                                                             0x00002000L
+#define XPB_CLG_EXTRA1__CLG1_NUM_MASK                                                                         0x0001C000L
+//XPB_CLG_EXTRA_MSK
+#define XPB_CLG_EXTRA_MSK__MSK0_HIGH__SHIFT                                                                   0x0
+#define XPB_CLG_EXTRA_MSK__MSK0_LOW__SHIFT                                                                    0x8
+#define XPB_CLG_EXTRA_MSK__MSK1_HIGH__SHIFT                                                                   0xd
+#define XPB_CLG_EXTRA_MSK__MSK1_LOW__SHIFT                                                                    0x15
+#define XPB_CLG_EXTRA_MSK__MSK0_HIGH_MASK                                                                     0x000000FFL
+#define XPB_CLG_EXTRA_MSK__MSK0_LOW_MASK                                                                      0x00001F00L
+#define XPB_CLG_EXTRA_MSK__MSK1_HIGH_MASK                                                                     0x001FE000L
+#define XPB_CLG_EXTRA_MSK__MSK1_LOW_MASK                                                                      0x03E00000L
+//XPB_LB_ADDR
+#define XPB_LB_ADDR__CMP0__SHIFT                                                                              0x0
+#define XPB_LB_ADDR__MASK0__SHIFT                                                                             0xa
+#define XPB_LB_ADDR__CMP1__SHIFT                                                                              0x14
+#define XPB_LB_ADDR__MASK1__SHIFT                                                                             0x1a
+#define XPB_LB_ADDR__CMP0_MASK                                                                                0x000003FFL
+#define XPB_LB_ADDR__MASK0_MASK                                                                               0x000FFC00L
+#define XPB_LB_ADDR__CMP1_MASK                                                                                0x03F00000L
+#define XPB_LB_ADDR__MASK1_MASK                                                                               0xFC000000L
+//XPB_WCB_STS
+#define XPB_WCB_STS__PBUF_VLD__SHIFT                                                                          0x0
+#define XPB_WCB_STS__WCB_HST_DATA_BUF_CNT__SHIFT                                                              0x10
+#define XPB_WCB_STS__WCB_SID_DATA_BUF_CNT__SHIFT                                                              0x17
+#define XPB_WCB_STS__PBUF_VLD_MASK                                                                            0x0000FFFFL
+#define XPB_WCB_STS__WCB_HST_DATA_BUF_CNT_MASK                                                                0x007F0000L
+#define XPB_WCB_STS__WCB_SID_DATA_BUF_CNT_MASK                                                                0x3F800000L
+//XPB_HST_CFG
+#define XPB_HST_CFG__BAR_UP_WR_CMD__SHIFT                                                                     0x0
+#define XPB_HST_CFG__BAR_UP_WR_CMD_MASK                                                                       0x00000001L
+//XPB_P2P_BAR_CFG
+#define XPB_P2P_BAR_CFG__ADDR_SIZE__SHIFT                                                                     0x0
+#define XPB_P2P_BAR_CFG__SEND_BAR__SHIFT                                                                      0x4
+#define XPB_P2P_BAR_CFG__SNOOP__SHIFT                                                                         0x6
+#define XPB_P2P_BAR_CFG__SEND_DIS__SHIFT                                                                      0x7
+#define XPB_P2P_BAR_CFG__COMPRESS_DIS__SHIFT                                                                  0x8
+#define XPB_P2P_BAR_CFG__UPDATE_DIS__SHIFT                                                                    0x9
+#define XPB_P2P_BAR_CFG__REGBAR_FROM_SYSBAR__SHIFT                                                            0xa
+#define XPB_P2P_BAR_CFG__RD_EN__SHIFT                                                                         0xb
+#define XPB_P2P_BAR_CFG__ATC_TRANSLATED__SHIFT                                                                0xc
+#define XPB_P2P_BAR_CFG__ADDR_SIZE_MASK                                                                       0x0000000FL
+#define XPB_P2P_BAR_CFG__SEND_BAR_MASK                                                                        0x00000030L
+#define XPB_P2P_BAR_CFG__SNOOP_MASK                                                                           0x00000040L
+#define XPB_P2P_BAR_CFG__SEND_DIS_MASK                                                                        0x00000080L
+#define XPB_P2P_BAR_CFG__COMPRESS_DIS_MASK                                                                    0x00000100L
+#define XPB_P2P_BAR_CFG__UPDATE_DIS_MASK                                                                      0x00000200L
+#define XPB_P2P_BAR_CFG__REGBAR_FROM_SYSBAR_MASK                                                              0x00000400L
+#define XPB_P2P_BAR_CFG__RD_EN_MASK                                                                           0x00000800L
+#define XPB_P2P_BAR_CFG__ATC_TRANSLATED_MASK                                                                  0x00001000L
+//XPB_P2P_BAR0
+#define XPB_P2P_BAR0__HOST_FLUSH__SHIFT                                                                       0x0
+#define XPB_P2P_BAR0__REG_SYS_BAR__SHIFT                                                                      0x4
+#define XPB_P2P_BAR0__MEM_SYS_BAR__SHIFT                                                                      0x8
+#define XPB_P2P_BAR0__VALID__SHIFT                                                                            0xc
+#define XPB_P2P_BAR0__SEND_DIS__SHIFT                                                                         0xd
+#define XPB_P2P_BAR0__COMPRESS_DIS__SHIFT                                                                     0xe
+#define XPB_P2P_BAR0__RESERVED__SHIFT                                                                         0xf
+#define XPB_P2P_BAR0__ADDRESS__SHIFT                                                                          0x10
+#define XPB_P2P_BAR0__HOST_FLUSH_MASK                                                                         0x0000000FL
+#define XPB_P2P_BAR0__REG_SYS_BAR_MASK                                                                        0x000000F0L
+#define XPB_P2P_BAR0__MEM_SYS_BAR_MASK                                                                        0x00000F00L
+#define XPB_P2P_BAR0__VALID_MASK                                                                              0x00001000L
+#define XPB_P2P_BAR0__SEND_DIS_MASK                                                                           0x00002000L
+#define XPB_P2P_BAR0__COMPRESS_DIS_MASK                                                                       0x00004000L
+#define XPB_P2P_BAR0__RESERVED_MASK                                                                           0x00008000L
+#define XPB_P2P_BAR0__ADDRESS_MASK                                                                            0xFFFF0000L
+//XPB_P2P_BAR1
+#define XPB_P2P_BAR1__HOST_FLUSH__SHIFT                                                                       0x0
+#define XPB_P2P_BAR1__REG_SYS_BAR__SHIFT                                                                      0x4
+#define XPB_P2P_BAR1__MEM_SYS_BAR__SHIFT                                                                      0x8
+#define XPB_P2P_BAR1__VALID__SHIFT                                                                            0xc
+#define XPB_P2P_BAR1__SEND_DIS__SHIFT                                                                         0xd
+#define XPB_P2P_BAR1__COMPRESS_DIS__SHIFT                                                                     0xe
+#define XPB_P2P_BAR1__RESERVED__SHIFT                                                                         0xf
+#define XPB_P2P_BAR1__ADDRESS__SHIFT                                                                          0x10
+#define XPB_P2P_BAR1__HOST_FLUSH_MASK                                                                         0x0000000FL
+#define XPB_P2P_BAR1__REG_SYS_BAR_MASK                                                                        0x000000F0L
+#define XPB_P2P_BAR1__MEM_SYS_BAR_MASK                                                                        0x00000F00L
+#define XPB_P2P_BAR1__VALID_MASK                                                                              0x00001000L
+#define XPB_P2P_BAR1__SEND_DIS_MASK                                                                           0x00002000L
+#define XPB_P2P_BAR1__COMPRESS_DIS_MASK                                                                       0x00004000L
+#define XPB_P2P_BAR1__RESERVED_MASK                                                                           0x00008000L
+#define XPB_P2P_BAR1__ADDRESS_MASK                                                                            0xFFFF0000L
+//XPB_P2P_BAR2
+#define XPB_P2P_BAR2__HOST_FLUSH__SHIFT                                                                       0x0
+#define XPB_P2P_BAR2__REG_SYS_BAR__SHIFT                                                                      0x4
+#define XPB_P2P_BAR2__MEM_SYS_BAR__SHIFT                                                                      0x8
+#define XPB_P2P_BAR2__VALID__SHIFT                                                                            0xc
+#define XPB_P2P_BAR2__SEND_DIS__SHIFT                                                                         0xd
+#define XPB_P2P_BAR2__COMPRESS_DIS__SHIFT                                                                     0xe
+#define XPB_P2P_BAR2__RESERVED__SHIFT                                                                         0xf
+#define XPB_P2P_BAR2__ADDRESS__SHIFT                                                                          0x10
+#define XPB_P2P_BAR2__HOST_FLUSH_MASK                                                                         0x0000000FL
+#define XPB_P2P_BAR2__REG_SYS_BAR_MASK                                                                        0x000000F0L
+#define XPB_P2P_BAR2__MEM_SYS_BAR_MASK                                                                        0x00000F00L
+#define XPB_P2P_BAR2__VALID_MASK                                                                              0x00001000L
+#define XPB_P2P_BAR2__SEND_DIS_MASK                                                                           0x00002000L
+#define XPB_P2P_BAR2__COMPRESS_DIS_MASK                                                                       0x00004000L
+#define XPB_P2P_BAR2__RESERVED_MASK                                                                           0x00008000L
+#define XPB_P2P_BAR2__ADDRESS_MASK                                                                            0xFFFF0000L
+//XPB_P2P_BAR3
+#define XPB_P2P_BAR3__HOST_FLUSH__SHIFT                                                                       0x0
+#define XPB_P2P_BAR3__REG_SYS_BAR__SHIFT                                                                      0x4
+#define XPB_P2P_BAR3__MEM_SYS_BAR__SHIFT                                                                      0x8
+#define XPB_P2P_BAR3__VALID__SHIFT                                                                            0xc
+#define XPB_P2P_BAR3__SEND_DIS__SHIFT                                                                         0xd
+#define XPB_P2P_BAR3__COMPRESS_DIS__SHIFT                                                                     0xe
+#define XPB_P2P_BAR3__RESERVED__SHIFT                                                                         0xf
+#define XPB_P2P_BAR3__ADDRESS__SHIFT                                                                          0x10
+#define XPB_P2P_BAR3__HOST_FLUSH_MASK                                                                         0x0000000FL
+#define XPB_P2P_BAR3__REG_SYS_BAR_MASK                                                                        0x000000F0L
+#define XPB_P2P_BAR3__MEM_SYS_BAR_MASK                                                                        0x00000F00L
+#define XPB_P2P_BAR3__VALID_MASK                                                                              0x00001000L
+#define XPB_P2P_BAR3__SEND_DIS_MASK                                                                           0x00002000L
+#define XPB_P2P_BAR3__COMPRESS_DIS_MASK                                                                       0x00004000L
+#define XPB_P2P_BAR3__RESERVED_MASK                                                                           0x00008000L
+#define XPB_P2P_BAR3__ADDRESS_MASK                                                                            0xFFFF0000L
+//XPB_P2P_BAR4
+#define XPB_P2P_BAR4__HOST_FLUSH__SHIFT                                                                       0x0
+#define XPB_P2P_BAR4__REG_SYS_BAR__SHIFT                                                                      0x4
+#define XPB_P2P_BAR4__MEM_SYS_BAR__SHIFT                                                                      0x8
+#define XPB_P2P_BAR4__VALID__SHIFT                                                                            0xc
+#define XPB_P2P_BAR4__SEND_DIS__SHIFT                                                                         0xd
+#define XPB_P2P_BAR4__COMPRESS_DIS__SHIFT                                                                     0xe
+#define XPB_P2P_BAR4__RESERVED__SHIFT                                                                         0xf
+#define XPB_P2P_BAR4__ADDRESS__SHIFT                                                                          0x10
+#define XPB_P2P_BAR4__HOST_FLUSH_MASK                                                                         0x0000000FL
+#define XPB_P2P_BAR4__REG_SYS_BAR_MASK                                                                        0x000000F0L
+#define XPB_P2P_BAR4__MEM_SYS_BAR_MASK                                                                        0x00000F00L
+#define XPB_P2P_BAR4__VALID_MASK                                                                              0x00001000L
+#define XPB_P2P_BAR4__SEND_DIS_MASK                                                                           0x00002000L
+#define XPB_P2P_BAR4__COMPRESS_DIS_MASK                                                                       0x00004000L
+#define XPB_P2P_BAR4__RESERVED_MASK                                                                           0x00008000L
+#define XPB_P2P_BAR4__ADDRESS_MASK                                                                            0xFFFF0000L
+//XPB_P2P_BAR5
+#define XPB_P2P_BAR5__HOST_FLUSH__SHIFT                                                                       0x0
+#define XPB_P2P_BAR5__REG_SYS_BAR__SHIFT                                                                      0x4
+#define XPB_P2P_BAR5__MEM_SYS_BAR__SHIFT                                                                      0x8
+#define XPB_P2P_BAR5__VALID__SHIFT                                                                            0xc
+#define XPB_P2P_BAR5__SEND_DIS__SHIFT                                                                         0xd
+#define XPB_P2P_BAR5__COMPRESS_DIS__SHIFT                                                                     0xe
+#define XPB_P2P_BAR5__RESERVED__SHIFT                                                                         0xf
+#define XPB_P2P_BAR5__ADDRESS__SHIFT                                                                          0x10
+#define XPB_P2P_BAR5__HOST_FLUSH_MASK                                                                         0x0000000FL
+#define XPB_P2P_BAR5__REG_SYS_BAR_MASK                                                                        0x000000F0L
+#define XPB_P2P_BAR5__MEM_SYS_BAR_MASK                                                                        0x00000F00L
+#define XPB_P2P_BAR5__VALID_MASK                                                                              0x00001000L
+#define XPB_P2P_BAR5__SEND_DIS_MASK                                                                           0x00002000L
+#define XPB_P2P_BAR5__COMPRESS_DIS_MASK                                                                       0x00004000L
+#define XPB_P2P_BAR5__RESERVED_MASK                                                                           0x00008000L
+#define XPB_P2P_BAR5__ADDRESS_MASK                                                                            0xFFFF0000L
+//XPB_P2P_BAR6
+#define XPB_P2P_BAR6__HOST_FLUSH__SHIFT                                                                       0x0
+#define XPB_P2P_BAR6__REG_SYS_BAR__SHIFT                                                                      0x4
+#define XPB_P2P_BAR6__MEM_SYS_BAR__SHIFT                                                                      0x8
+#define XPB_P2P_BAR6__VALID__SHIFT                                                                            0xc
+#define XPB_P2P_BAR6__SEND_DIS__SHIFT                                                                         0xd
+#define XPB_P2P_BAR6__COMPRESS_DIS__SHIFT                                                                     0xe
+#define XPB_P2P_BAR6__RESERVED__SHIFT                                                                         0xf
+#define XPB_P2P_BAR6__ADDRESS__SHIFT                                                                          0x10
+#define XPB_P2P_BAR6__HOST_FLUSH_MASK                                                                         0x0000000FL
+#define XPB_P2P_BAR6__REG_SYS_BAR_MASK                                                                        0x000000F0L
+#define XPB_P2P_BAR6__MEM_SYS_BAR_MASK                                                                        0x00000F00L
+#define XPB_P2P_BAR6__VALID_MASK                                                                              0x00001000L
+#define XPB_P2P_BAR6__SEND_DIS_MASK                                                                           0x00002000L
+#define XPB_P2P_BAR6__COMPRESS_DIS_MASK                                                                       0x00004000L
+#define XPB_P2P_BAR6__RESERVED_MASK                                                                           0x00008000L
+#define XPB_P2P_BAR6__ADDRESS_MASK                                                                            0xFFFF0000L
+//XPB_P2P_BAR7
+#define XPB_P2P_BAR7__HOST_FLUSH__SHIFT                                                                       0x0
+#define XPB_P2P_BAR7__REG_SYS_BAR__SHIFT                                                                      0x4
+#define XPB_P2P_BAR7__MEM_SYS_BAR__SHIFT                                                                      0x8
+#define XPB_P2P_BAR7__VALID__SHIFT                                                                            0xc
+#define XPB_P2P_BAR7__SEND_DIS__SHIFT                                                                         0xd
+#define XPB_P2P_BAR7__COMPRESS_DIS__SHIFT                                                                     0xe
+#define XPB_P2P_BAR7__RESERVED__SHIFT                                                                         0xf
+#define XPB_P2P_BAR7__ADDRESS__SHIFT                                                                          0x10
+#define XPB_P2P_BAR7__HOST_FLUSH_MASK                                                                         0x0000000FL
+#define XPB_P2P_BAR7__REG_SYS_BAR_MASK                                                                        0x000000F0L
+#define XPB_P2P_BAR7__MEM_SYS_BAR_MASK                                                                        0x00000F00L
+#define XPB_P2P_BAR7__VALID_MASK                                                                              0x00001000L
+#define XPB_P2P_BAR7__SEND_DIS_MASK                                                                           0x00002000L
+#define XPB_P2P_BAR7__COMPRESS_DIS_MASK                                                                       0x00004000L
+#define XPB_P2P_BAR7__RESERVED_MASK                                                                           0x00008000L
+#define XPB_P2P_BAR7__ADDRESS_MASK                                                                            0xFFFF0000L
+//XPB_P2P_BAR_SETUP
+#define XPB_P2P_BAR_SETUP__SEL__SHIFT                                                                         0x0
+#define XPB_P2P_BAR_SETUP__REG_SYS_BAR__SHIFT                                                                 0x8
+#define XPB_P2P_BAR_SETUP__VALID__SHIFT                                                                       0xc
+#define XPB_P2P_BAR_SETUP__SEND_DIS__SHIFT                                                                    0xd
+#define XPB_P2P_BAR_SETUP__COMPRESS_DIS__SHIFT                                                                0xe
+#define XPB_P2P_BAR_SETUP__RESERVED__SHIFT                                                                    0xf
+#define XPB_P2P_BAR_SETUP__ADDRESS__SHIFT                                                                     0x10
+#define XPB_P2P_BAR_SETUP__SEL_MASK                                                                           0x000000FFL
+#define XPB_P2P_BAR_SETUP__REG_SYS_BAR_MASK                                                                   0x00000F00L
+#define XPB_P2P_BAR_SETUP__VALID_MASK                                                                         0x00001000L
+#define XPB_P2P_BAR_SETUP__SEND_DIS_MASK                                                                      0x00002000L
+#define XPB_P2P_BAR_SETUP__COMPRESS_DIS_MASK                                                                  0x00004000L
+#define XPB_P2P_BAR_SETUP__RESERVED_MASK                                                                      0x00008000L
+#define XPB_P2P_BAR_SETUP__ADDRESS_MASK                                                                       0xFFFF0000L
+//XPB_P2P_BAR_DELTA_ABOVE
+#define XPB_P2P_BAR_DELTA_ABOVE__EN__SHIFT                                                                    0x0
+#define XPB_P2P_BAR_DELTA_ABOVE__DELTA__SHIFT                                                                 0x8
+#define XPB_P2P_BAR_DELTA_ABOVE__EN_MASK                                                                      0x000000FFL
+#define XPB_P2P_BAR_DELTA_ABOVE__DELTA_MASK                                                                   0x0FFFFF00L
+//XPB_P2P_BAR_DELTA_BELOW
+#define XPB_P2P_BAR_DELTA_BELOW__EN__SHIFT                                                                    0x0
+#define XPB_P2P_BAR_DELTA_BELOW__DELTA__SHIFT                                                                 0x8
+#define XPB_P2P_BAR_DELTA_BELOW__EN_MASK                                                                      0x000000FFL
+#define XPB_P2P_BAR_DELTA_BELOW__DELTA_MASK                                                                   0x0FFFFF00L
+//XPB_PEER_SYS_BAR0
+#define XPB_PEER_SYS_BAR0__VALID__SHIFT                                                                       0x0
+#define XPB_PEER_SYS_BAR0__ADDR__SHIFT                                                                        0x1
+#define XPB_PEER_SYS_BAR0__VALID_MASK                                                                         0x00000001L
+#define XPB_PEER_SYS_BAR0__ADDR_MASK                                                                          0xFFFFFFFEL
+//XPB_PEER_SYS_BAR1
+#define XPB_PEER_SYS_BAR1__VALID__SHIFT                                                                       0x0
+#define XPB_PEER_SYS_BAR1__ADDR__SHIFT                                                                        0x1
+#define XPB_PEER_SYS_BAR1__VALID_MASK                                                                         0x00000001L
+#define XPB_PEER_SYS_BAR1__ADDR_MASK                                                                          0xFFFFFFFEL
+//XPB_PEER_SYS_BAR2
+#define XPB_PEER_SYS_BAR2__VALID__SHIFT                                                                       0x0
+#define XPB_PEER_SYS_BAR2__ADDR__SHIFT                                                                        0x1
+#define XPB_PEER_SYS_BAR2__VALID_MASK                                                                         0x00000001L
+#define XPB_PEER_SYS_BAR2__ADDR_MASK                                                                          0xFFFFFFFEL
+//XPB_PEER_SYS_BAR3
+#define XPB_PEER_SYS_BAR3__VALID__SHIFT                                                                       0x0
+#define XPB_PEER_SYS_BAR3__ADDR__SHIFT                                                                        0x1
+#define XPB_PEER_SYS_BAR3__VALID_MASK                                                                         0x00000001L
+#define XPB_PEER_SYS_BAR3__ADDR_MASK                                                                          0xFFFFFFFEL
+//XPB_PEER_SYS_BAR4
+#define XPB_PEER_SYS_BAR4__VALID__SHIFT                                                                       0x0
+#define XPB_PEER_SYS_BAR4__ADDR__SHIFT                                                                        0x1
+#define XPB_PEER_SYS_BAR4__VALID_MASK                                                                         0x00000001L
+#define XPB_PEER_SYS_BAR4__ADDR_MASK                                                                          0xFFFFFFFEL
+//XPB_PEER_SYS_BAR5
+#define XPB_PEER_SYS_BAR5__VALID__SHIFT                                                                       0x0
+#define XPB_PEER_SYS_BAR5__ADDR__SHIFT                                                                        0x1
+#define XPB_PEER_SYS_BAR5__VALID_MASK                                                                         0x00000001L
+#define XPB_PEER_SYS_BAR5__ADDR_MASK                                                                          0xFFFFFFFEL
+//XPB_PEER_SYS_BAR6
+#define XPB_PEER_SYS_BAR6__VALID__SHIFT                                                                       0x0
+#define XPB_PEER_SYS_BAR6__ADDR__SHIFT                                                                        0x1
+#define XPB_PEER_SYS_BAR6__VALID_MASK                                                                         0x00000001L
+#define XPB_PEER_SYS_BAR6__ADDR_MASK                                                                          0xFFFFFFFEL
+//XPB_PEER_SYS_BAR7
+#define XPB_PEER_SYS_BAR7__VALID__SHIFT                                                                       0x0
+#define XPB_PEER_SYS_BAR7__ADDR__SHIFT                                                                        0x1
+#define XPB_PEER_SYS_BAR7__VALID_MASK                                                                         0x00000001L
+#define XPB_PEER_SYS_BAR7__ADDR_MASK                                                                          0xFFFFFFFEL
+//XPB_PEER_SYS_BAR8
+#define XPB_PEER_SYS_BAR8__VALID__SHIFT                                                                       0x0
+#define XPB_PEER_SYS_BAR8__ADDR__SHIFT                                                                        0x1
+#define XPB_PEER_SYS_BAR8__VALID_MASK                                                                         0x00000001L
+#define XPB_PEER_SYS_BAR8__ADDR_MASK                                                                          0xFFFFFFFEL
+//XPB_PEER_SYS_BAR9
+#define XPB_PEER_SYS_BAR9__VALID__SHIFT                                                                       0x0
+#define XPB_PEER_SYS_BAR9__ADDR__SHIFT                                                                        0x1
+#define XPB_PEER_SYS_BAR9__VALID_MASK                                                                         0x00000001L
+#define XPB_PEER_SYS_BAR9__ADDR_MASK                                                                          0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR0
+#define XPB_XDMA_PEER_SYS_BAR0__VALID__SHIFT                                                                  0x0
+#define XPB_XDMA_PEER_SYS_BAR0__ADDR__SHIFT                                                                   0x1
+#define XPB_XDMA_PEER_SYS_BAR0__VALID_MASK                                                                    0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR0__ADDR_MASK                                                                     0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR1
+#define XPB_XDMA_PEER_SYS_BAR1__VALID__SHIFT                                                                  0x0
+#define XPB_XDMA_PEER_SYS_BAR1__ADDR__SHIFT                                                                   0x1
+#define XPB_XDMA_PEER_SYS_BAR1__VALID_MASK                                                                    0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR1__ADDR_MASK                                                                     0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR2
+#define XPB_XDMA_PEER_SYS_BAR2__VALID__SHIFT                                                                  0x0
+#define XPB_XDMA_PEER_SYS_BAR2__ADDR__SHIFT                                                                   0x1
+#define XPB_XDMA_PEER_SYS_BAR2__VALID_MASK                                                                    0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR2__ADDR_MASK                                                                     0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR3
+#define XPB_XDMA_PEER_SYS_BAR3__VALID__SHIFT                                                                  0x0
+#define XPB_XDMA_PEER_SYS_BAR3__ADDR__SHIFT                                                                   0x1
+#define XPB_XDMA_PEER_SYS_BAR3__VALID_MASK                                                                    0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR3__ADDR_MASK                                                                     0xFFFFFFFEL
+//XPB_CLK_GAT
+#define XPB_CLK_GAT__ONDLY__SHIFT                                                                             0x0
+#define XPB_CLK_GAT__OFFDLY__SHIFT                                                                            0x6
+#define XPB_CLK_GAT__RDYDLY__SHIFT                                                                            0xc
+#define XPB_CLK_GAT__ENABLE__SHIFT                                                                            0x12
+#define XPB_CLK_GAT__MEM_LS_ENABLE__SHIFT                                                                     0x13
+#define XPB_CLK_GAT__ONDLY_MASK                                                                               0x0000003FL
+#define XPB_CLK_GAT__OFFDLY_MASK                                                                              0x00000FC0L
+#define XPB_CLK_GAT__RDYDLY_MASK                                                                              0x0003F000L
+#define XPB_CLK_GAT__ENABLE_MASK                                                                              0x00040000L
+#define XPB_CLK_GAT__MEM_LS_ENABLE_MASK                                                                       0x00080000L
+//XPB_INTF_CFG
+#define XPB_INTF_CFG__RPB_WRREQ_CRD__SHIFT                                                                    0x0
+#define XPB_INTF_CFG__MC_WRRET_ASK__SHIFT                                                                     0x8
+#define XPB_INTF_CFG__XSP_REQ_CRD__SHIFT                                                                      0x10
+#define XPB_INTF_CFG__BIF_REG_SNOOP_SEL__SHIFT                                                                0x17
+#define XPB_INTF_CFG__BIF_REG_SNOOP_VAL__SHIFT                                                                0x18
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_SEL__SHIFT                                                                0x19
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_VAL__SHIFT                                                                0x1a
+#define XPB_INTF_CFG__XSP_SNOOP_SEL__SHIFT                                                                    0x1b
+#define XPB_INTF_CFG__XSP_SNOOP_VAL__SHIFT                                                                    0x1d
+#define XPB_INTF_CFG__XSP_ORDERING_SEL__SHIFT                                                                 0x1e
+#define XPB_INTF_CFG__QUALIFY_P2P_FOR_GPA__SHIFT                                                              0x1f
+#define XPB_INTF_CFG__RPB_WRREQ_CRD_MASK                                                                      0x000000FFL
+#define XPB_INTF_CFG__MC_WRRET_ASK_MASK                                                                       0x0000FF00L
+#define XPB_INTF_CFG__XSP_REQ_CRD_MASK                                                                        0x007F0000L
+#define XPB_INTF_CFG__BIF_REG_SNOOP_SEL_MASK                                                                  0x00800000L
+#define XPB_INTF_CFG__BIF_REG_SNOOP_VAL_MASK                                                                  0x01000000L
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_SEL_MASK                                                                  0x02000000L
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_VAL_MASK                                                                  0x04000000L
+#define XPB_INTF_CFG__XSP_SNOOP_SEL_MASK                                                                      0x18000000L
+#define XPB_INTF_CFG__XSP_SNOOP_VAL_MASK                                                                      0x20000000L
+#define XPB_INTF_CFG__XSP_ORDERING_SEL_MASK                                                                   0x40000000L
+#define XPB_INTF_CFG__QUALIFY_P2P_FOR_GPA_MASK                                                                0x80000000L
+//XPB_INTF_STS
+#define XPB_INTF_STS__RPB_WRREQ_CRD__SHIFT                                                                    0x0
+#define XPB_INTF_STS__XSP_REQ_CRD__SHIFT                                                                      0x8
+#define XPB_INTF_STS__HOP_DATA_BUF_FULL__SHIFT                                                                0xf
+#define XPB_INTF_STS__HOP_ATTR_BUF_FULL__SHIFT                                                                0x10
+#define XPB_INTF_STS__CNS_BUF_FULL__SHIFT                                                                     0x11
+#define XPB_INTF_STS__CNS_BUF_BUSY__SHIFT                                                                     0x12
+#define XPB_INTF_STS__RPB_RDREQ_CRD__SHIFT                                                                    0x13
+#define XPB_INTF_STS__RPB_WRREQ_CRD_MASK                                                                      0x000000FFL
+#define XPB_INTF_STS__XSP_REQ_CRD_MASK                                                                        0x00007F00L
+#define XPB_INTF_STS__HOP_DATA_BUF_FULL_MASK                                                                  0x00008000L
+#define XPB_INTF_STS__HOP_ATTR_BUF_FULL_MASK                                                                  0x00010000L
+#define XPB_INTF_STS__CNS_BUF_FULL_MASK                                                                       0x00020000L
+#define XPB_INTF_STS__CNS_BUF_BUSY_MASK                                                                       0x00040000L
+#define XPB_INTF_STS__RPB_RDREQ_CRD_MASK                                                                      0x07F80000L
+//XPB_PIPE_STS
+#define XPB_PIPE_STS__WCB_ANY_PBUF__SHIFT                                                                     0x0
+#define XPB_PIPE_STS__WCB_HST_DATA_BUF_CNT__SHIFT                                                             0x1
+#define XPB_PIPE_STS__WCB_SID_DATA_BUF_CNT__SHIFT                                                             0x8
+#define XPB_PIPE_STS__WCB_HST_RD_PTR_BUF_FULL__SHIFT                                                          0xf
+#define XPB_PIPE_STS__WCB_SID_RD_PTR_BUF_FULL__SHIFT                                                          0x10
+#define XPB_PIPE_STS__WCB_HST_REQ_FIFO_FULL__SHIFT                                                            0x11
+#define XPB_PIPE_STS__WCB_SID_REQ_FIFO_FULL__SHIFT                                                            0x12
+#define XPB_PIPE_STS__WCB_HST_REQ_OBUF_FULL__SHIFT                                                            0x13
+#define XPB_PIPE_STS__WCB_SID_REQ_OBUF_FULL__SHIFT                                                            0x14
+#define XPB_PIPE_STS__WCB_HST_DATA_OBUF_FULL__SHIFT                                                           0x15
+#define XPB_PIPE_STS__WCB_SID_DATA_OBUF_FULL__SHIFT                                                           0x16
+#define XPB_PIPE_STS__RET_BUF_FULL__SHIFT                                                                     0x17
+#define XPB_PIPE_STS__XPB_CLK_BUSY_BITS__SHIFT                                                                0x18
+#define XPB_PIPE_STS__WCB_ANY_PBUF_MASK                                                                       0x00000001L
+#define XPB_PIPE_STS__WCB_HST_DATA_BUF_CNT_MASK                                                               0x000000FEL
+#define XPB_PIPE_STS__WCB_SID_DATA_BUF_CNT_MASK                                                               0x00007F00L
+#define XPB_PIPE_STS__WCB_HST_RD_PTR_BUF_FULL_MASK                                                            0x00008000L
+#define XPB_PIPE_STS__WCB_SID_RD_PTR_BUF_FULL_MASK                                                            0x00010000L
+#define XPB_PIPE_STS__WCB_HST_REQ_FIFO_FULL_MASK                                                              0x00020000L
+#define XPB_PIPE_STS__WCB_SID_REQ_FIFO_FULL_MASK                                                              0x00040000L
+#define XPB_PIPE_STS__WCB_HST_REQ_OBUF_FULL_MASK                                                              0x00080000L
+#define XPB_PIPE_STS__WCB_SID_REQ_OBUF_FULL_MASK                                                              0x00100000L
+#define XPB_PIPE_STS__WCB_HST_DATA_OBUF_FULL_MASK                                                             0x00200000L
+#define XPB_PIPE_STS__WCB_SID_DATA_OBUF_FULL_MASK                                                             0x00400000L
+#define XPB_PIPE_STS__RET_BUF_FULL_MASK                                                                       0x00800000L
+#define XPB_PIPE_STS__XPB_CLK_BUSY_BITS_MASK                                                                  0xFF000000L
+//XPB_SUB_CTRL
+#define XPB_SUB_CTRL__WRREQ_BYPASS_XPB__SHIFT                                                                 0x0
+#define XPB_SUB_CTRL__STALL_CNS_RTR_REQ__SHIFT                                                                0x1
+#define XPB_SUB_CTRL__STALL_RTR_RPB_WRREQ__SHIFT                                                              0x2
+#define XPB_SUB_CTRL__STALL_RTR_MAP_REQ__SHIFT                                                                0x3
+#define XPB_SUB_CTRL__STALL_MAP_WCB_REQ__SHIFT                                                                0x4
+#define XPB_SUB_CTRL__STALL_WCB_SID_REQ__SHIFT                                                                0x5
+#define XPB_SUB_CTRL__STALL_MC_XSP_REQ_SEND__SHIFT                                                            0x6
+#define XPB_SUB_CTRL__STALL_WCB_HST_REQ__SHIFT                                                                0x7
+#define XPB_SUB_CTRL__STALL_HST_HOP_REQ__SHIFT                                                                0x8
+#define XPB_SUB_CTRL__STALL_XPB_RPB_REQ_ATTR__SHIFT                                                           0x9
+#define XPB_SUB_CTRL__RESET_CNS__SHIFT                                                                        0xa
+#define XPB_SUB_CTRL__RESET_RTR__SHIFT                                                                        0xb
+#define XPB_SUB_CTRL__RESET_RET__SHIFT                                                                        0xc
+#define XPB_SUB_CTRL__RESET_MAP__SHIFT                                                                        0xd
+#define XPB_SUB_CTRL__RESET_WCB__SHIFT                                                                        0xe
+#define XPB_SUB_CTRL__RESET_HST__SHIFT                                                                        0xf
+#define XPB_SUB_CTRL__RESET_HOP__SHIFT                                                                        0x10
+#define XPB_SUB_CTRL__RESET_SID__SHIFT                                                                        0x11
+#define XPB_SUB_CTRL__RESET_SRB__SHIFT                                                                        0x12
+#define XPB_SUB_CTRL__RESET_CGR__SHIFT                                                                        0x13
+#define XPB_SUB_CTRL__WRREQ_BYPASS_XPB_MASK                                                                   0x00000001L
+#define XPB_SUB_CTRL__STALL_CNS_RTR_REQ_MASK                                                                  0x00000002L
+#define XPB_SUB_CTRL__STALL_RTR_RPB_WRREQ_MASK                                                                0x00000004L
+#define XPB_SUB_CTRL__STALL_RTR_MAP_REQ_MASK                                                                  0x00000008L
+#define XPB_SUB_CTRL__STALL_MAP_WCB_REQ_MASK                                                                  0x00000010L
+#define XPB_SUB_CTRL__STALL_WCB_SID_REQ_MASK                                                                  0x00000020L
+#define XPB_SUB_CTRL__STALL_MC_XSP_REQ_SEND_MASK                                                              0x00000040L
+#define XPB_SUB_CTRL__STALL_WCB_HST_REQ_MASK                                                                  0x00000080L
+#define XPB_SUB_CTRL__STALL_HST_HOP_REQ_MASK                                                                  0x00000100L
+#define XPB_SUB_CTRL__STALL_XPB_RPB_REQ_ATTR_MASK                                                             0x00000200L
+#define XPB_SUB_CTRL__RESET_CNS_MASK                                                                          0x00000400L
+#define XPB_SUB_CTRL__RESET_RTR_MASK                                                                          0x00000800L
+#define XPB_SUB_CTRL__RESET_RET_MASK                                                                          0x00001000L
+#define XPB_SUB_CTRL__RESET_MAP_MASK                                                                          0x00002000L
+#define XPB_SUB_CTRL__RESET_WCB_MASK                                                                          0x00004000L
+#define XPB_SUB_CTRL__RESET_HST_MASK                                                                          0x00008000L
+#define XPB_SUB_CTRL__RESET_HOP_MASK                                                                          0x00010000L
+#define XPB_SUB_CTRL__RESET_SID_MASK                                                                          0x00020000L
+#define XPB_SUB_CTRL__RESET_SRB_MASK                                                                          0x00040000L
+#define XPB_SUB_CTRL__RESET_CGR_MASK                                                                          0x00080000L
+//XPB_MAP_INVERT_FLUSH_NUM_LSB
+#define XPB_MAP_INVERT_FLUSH_NUM_LSB__ALTER_FLUSH_NUM__SHIFT                                                  0x0
+#define XPB_MAP_INVERT_FLUSH_NUM_LSB__ALTER_FLUSH_NUM_MASK                                                    0x0000FFFFL
+//XPB_PERF_KNOBS
+#define XPB_PERF_KNOBS__CNS_FIFO_DEPTH__SHIFT                                                                 0x0
+#define XPB_PERF_KNOBS__WCB_HST_FIFO_DEPTH__SHIFT                                                             0x6
+#define XPB_PERF_KNOBS__WCB_SID_FIFO_DEPTH__SHIFT                                                             0xc
+#define XPB_PERF_KNOBS__CNS_FIFO_DEPTH_MASK                                                                   0x0000003FL
+#define XPB_PERF_KNOBS__WCB_HST_FIFO_DEPTH_MASK                                                               0x00000FC0L
+#define XPB_PERF_KNOBS__WCB_SID_FIFO_DEPTH_MASK                                                               0x0003F000L
+//XPB_STICKY
+#define XPB_STICKY__BITS__SHIFT                                                                               0x0
+#define XPB_STICKY__BITS_MASK                                                                                 0xFFFFFFFFL
+//XPB_STICKY_W1C
+#define XPB_STICKY_W1C__BITS__SHIFT                                                                           0x0
+#define XPB_STICKY_W1C__BITS_MASK                                                                             0xFFFFFFFFL
+//XPB_MISC_CFG
+#define XPB_MISC_CFG__FIELDNAME0__SHIFT                                                                       0x0
+#define XPB_MISC_CFG__FIELDNAME1__SHIFT                                                                       0x8
+#define XPB_MISC_CFG__FIELDNAME2__SHIFT                                                                       0x10
+#define XPB_MISC_CFG__FIELDNAME3__SHIFT                                                                       0x18
+#define XPB_MISC_CFG__TRIGGERNAME__SHIFT                                                                      0x1f
+#define XPB_MISC_CFG__FIELDNAME0_MASK                                                                         0x000000FFL
+#define XPB_MISC_CFG__FIELDNAME1_MASK                                                                         0x0000FF00L
+#define XPB_MISC_CFG__FIELDNAME2_MASK                                                                         0x00FF0000L
+#define XPB_MISC_CFG__FIELDNAME3_MASK                                                                         0x7F000000L
+#define XPB_MISC_CFG__TRIGGERNAME_MASK                                                                        0x80000000L
+//XPB_INTF_CFG2
+#define XPB_INTF_CFG2__RPB_RDREQ_CRD__SHIFT                                                                   0x0
+#define XPB_INTF_CFG2__RPB_RDREQ_CRD_MASK                                                                     0x000000FFL
+//XPB_CLG_EXTRA_RD
+#define XPB_CLG_EXTRA_RD__CMP0_HIGH__SHIFT                                                                    0x0
+#define XPB_CLG_EXTRA_RD__CMP0_LOW__SHIFT                                                                     0x6
+#define XPB_CLG_EXTRA_RD__VLD0__SHIFT                                                                         0xb
+#define XPB_CLG_EXTRA_RD__CLG0_NUM__SHIFT                                                                     0xc
+#define XPB_CLG_EXTRA_RD__CMP1_HIGH__SHIFT                                                                    0xf
+#define XPB_CLG_EXTRA_RD__CMP1_LOW__SHIFT                                                                     0x15
+#define XPB_CLG_EXTRA_RD__VLD1__SHIFT                                                                         0x1a
+#define XPB_CLG_EXTRA_RD__CLG1_NUM__SHIFT                                                                     0x1b
+#define XPB_CLG_EXTRA_RD__CMP0_HIGH_MASK                                                                      0x0000003FL
+#define XPB_CLG_EXTRA_RD__CMP0_LOW_MASK                                                                       0x000007C0L
+#define XPB_CLG_EXTRA_RD__VLD0_MASK                                                                           0x00000800L
+#define XPB_CLG_EXTRA_RD__CLG0_NUM_MASK                                                                       0x00007000L
+#define XPB_CLG_EXTRA_RD__CMP1_HIGH_MASK                                                                      0x001F8000L
+#define XPB_CLG_EXTRA_RD__CMP1_LOW_MASK                                                                       0x03E00000L
+#define XPB_CLG_EXTRA_RD__VLD1_MASK                                                                           0x04000000L
+#define XPB_CLG_EXTRA_RD__CLG1_NUM_MASK                                                                       0x38000000L
+//XPB_CLG_EXTRA_MSK_RD
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_HIGH__SHIFT                                                                0x0
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_LOW__SHIFT                                                                 0x6
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_HIGH__SHIFT                                                                0xb
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_LOW__SHIFT                                                                 0x11
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_HIGH_MASK                                                                  0x0000003FL
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_LOW_MASK                                                                   0x000007C0L
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_HIGH_MASK                                                                  0x0001F800L
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_LOW_MASK                                                                   0x003E0000L
+//XPB_CLG_GFX_MATCH
+#define XPB_CLG_GFX_MATCH__FARBIRC0_ID__SHIFT                                                                 0x0
+#define XPB_CLG_GFX_MATCH__FARBIRC1_ID__SHIFT                                                                 0x8
+#define XPB_CLG_GFX_MATCH__FARBIRC2_ID__SHIFT                                                                 0x10
+#define XPB_CLG_GFX_MATCH__FARBIRC3_ID__SHIFT                                                                 0x18
+#define XPB_CLG_GFX_MATCH__FARBIRC0_ID_MASK                                                                   0x000000FFL
+#define XPB_CLG_GFX_MATCH__FARBIRC1_ID_MASK                                                                   0x0000FF00L
+#define XPB_CLG_GFX_MATCH__FARBIRC2_ID_MASK                                                                   0x00FF0000L
+#define XPB_CLG_GFX_MATCH__FARBIRC3_ID_MASK                                                                   0xFF000000L
+//XPB_CLG_GFX_MATCH_VLD
+#define XPB_CLG_GFX_MATCH_VLD__FARBIRC0_VLD__SHIFT                                                            0x0
+#define XPB_CLG_GFX_MATCH_VLD__FARBIRC1_VLD__SHIFT                                                            0x1
+#define XPB_CLG_GFX_MATCH_VLD__FARBIRC2_VLD__SHIFT                                                            0x2
+#define XPB_CLG_GFX_MATCH_VLD__FARBIRC3_VLD__SHIFT                                                            0x3
+#define XPB_CLG_GFX_MATCH_VLD__FARBIRC0_VLD_MASK                                                              0x00000001L
+#define XPB_CLG_GFX_MATCH_VLD__FARBIRC1_VLD_MASK                                                              0x00000002L
+#define XPB_CLG_GFX_MATCH_VLD__FARBIRC2_VLD_MASK                                                              0x00000004L
+#define XPB_CLG_GFX_MATCH_VLD__FARBIRC3_VLD_MASK                                                              0x00000008L
+//XPB_CLG_GFX_MATCH_MSK
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC0_ID_MSK__SHIFT                                                         0x0
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC1_ID_MSK__SHIFT                                                         0x8
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC2_ID_MSK__SHIFT                                                         0x10
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC3_ID_MSK__SHIFT                                                         0x18
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC0_ID_MSK_MASK                                                           0x000000FFL
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC1_ID_MSK_MASK                                                           0x0000FF00L
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC2_ID_MSK_MASK                                                           0x00FF0000L
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC3_ID_MSK_MASK                                                           0xFF000000L
+//XPB_CLG_MM_MATCH
+#define XPB_CLG_MM_MATCH__FARBIRC0_ID__SHIFT                                                                  0x0
+#define XPB_CLG_MM_MATCH__FARBIRC1_ID__SHIFT                                                                  0x8
+#define XPB_CLG_MM_MATCH__FARBIRC2_ID__SHIFT                                                                  0x10
+#define XPB_CLG_MM_MATCH__FARBIRC3_ID__SHIFT                                                                  0x18
+#define XPB_CLG_MM_MATCH__FARBIRC0_ID_MASK                                                                    0x000000FFL
+#define XPB_CLG_MM_MATCH__FARBIRC1_ID_MASK                                                                    0x0000FF00L
+#define XPB_CLG_MM_MATCH__FARBIRC2_ID_MASK                                                                    0x00FF0000L
+#define XPB_CLG_MM_MATCH__FARBIRC3_ID_MASK                                                                    0xFF000000L
+//XPB_CLG_MM_MATCH_VLD
+#define XPB_CLG_MM_MATCH_VLD__FARBIRC0_VLD__SHIFT                                                             0x0
+#define XPB_CLG_MM_MATCH_VLD__FARBIRC1_VLD__SHIFT                                                             0x1
+#define XPB_CLG_MM_MATCH_VLD__FARBIRC2_VLD__SHIFT                                                             0x2
+#define XPB_CLG_MM_MATCH_VLD__FARBIRC3_VLD__SHIFT                                                             0x3
+#define XPB_CLG_MM_MATCH_VLD__FARBIRC0_VLD_MASK                                                               0x00000001L
+#define XPB_CLG_MM_MATCH_VLD__FARBIRC1_VLD_MASK                                                               0x00000002L
+#define XPB_CLG_MM_MATCH_VLD__FARBIRC2_VLD_MASK                                                               0x00000004L
+#define XPB_CLG_MM_MATCH_VLD__FARBIRC3_VLD_MASK                                                               0x00000008L
+//XPB_CLG_MM_MATCH_MSK
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC0_ID_MSK__SHIFT                                                          0x0
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC1_ID_MSK__SHIFT                                                          0x8
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC2_ID_MSK__SHIFT                                                          0x10
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC3_ID_MSK__SHIFT                                                          0x18
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC0_ID_MSK_MASK                                                            0x000000FFL
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC1_ID_MSK_MASK                                                            0x0000FF00L
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC2_ID_MSK_MASK                                                            0x00FF0000L
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC3_ID_MSK_MASK                                                            0xFF000000L
+//XPB_CLG_GFX_UNITID_MAPPING0
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_LOW__SHIFT                                                        0x0
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_VLD__SHIFT                                                        0x5
+#define XPB_CLG_GFX_UNITID_MAPPING0__DEST_CLG_NUM__SHIFT                                                      0x6
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_LOW_MASK                                                          0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_VLD_MASK                                                          0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING0__DEST_CLG_NUM_MASK                                                        0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING1
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_LOW__SHIFT                                                        0x0
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_VLD__SHIFT                                                        0x5
+#define XPB_CLG_GFX_UNITID_MAPPING1__DEST_CLG_NUM__SHIFT                                                      0x6
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_LOW_MASK                                                          0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_VLD_MASK                                                          0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING1__DEST_CLG_NUM_MASK                                                        0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING2
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_LOW__SHIFT                                                        0x0
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_VLD__SHIFT                                                        0x5
+#define XPB_CLG_GFX_UNITID_MAPPING2__DEST_CLG_NUM__SHIFT                                                      0x6
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_LOW_MASK                                                          0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_VLD_MASK                                                          0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING2__DEST_CLG_NUM_MASK                                                        0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING3
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_LOW__SHIFT                                                        0x0
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_VLD__SHIFT                                                        0x5
+#define XPB_CLG_GFX_UNITID_MAPPING3__DEST_CLG_NUM__SHIFT                                                      0x6
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_LOW_MASK                                                          0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_VLD_MASK                                                          0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING3__DEST_CLG_NUM_MASK                                                        0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING4
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_LOW__SHIFT                                                        0x0
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_VLD__SHIFT                                                        0x5
+#define XPB_CLG_GFX_UNITID_MAPPING4__DEST_CLG_NUM__SHIFT                                                      0x6
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_LOW_MASK                                                          0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_VLD_MASK                                                          0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING4__DEST_CLG_NUM_MASK                                                        0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING5
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_LOW__SHIFT                                                        0x0
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_VLD__SHIFT                                                        0x5
+#define XPB_CLG_GFX_UNITID_MAPPING5__DEST_CLG_NUM__SHIFT                                                      0x6
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_LOW_MASK                                                          0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_VLD_MASK                                                          0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING5__DEST_CLG_NUM_MASK                                                        0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING6
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_LOW__SHIFT                                                        0x0
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_VLD__SHIFT                                                        0x5
+#define XPB_CLG_GFX_UNITID_MAPPING6__DEST_CLG_NUM__SHIFT                                                      0x6
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_LOW_MASK                                                          0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_VLD_MASK                                                          0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING6__DEST_CLG_NUM_MASK                                                        0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING7
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_LOW__SHIFT                                                        0x0
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_VLD__SHIFT                                                        0x5
+#define XPB_CLG_GFX_UNITID_MAPPING7__DEST_CLG_NUM__SHIFT                                                      0x6
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_LOW_MASK                                                          0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_VLD_MASK                                                          0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING7__DEST_CLG_NUM_MASK                                                        0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING0
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_LOW__SHIFT                                                         0x0
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_VLD__SHIFT                                                         0x5
+#define XPB_CLG_MM_UNITID_MAPPING0__DEST_CLG_NUM__SHIFT                                                       0x6
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_LOW_MASK                                                           0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_VLD_MASK                                                           0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING0__DEST_CLG_NUM_MASK                                                         0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING1
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_LOW__SHIFT                                                         0x0
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_VLD__SHIFT                                                         0x5
+#define XPB_CLG_MM_UNITID_MAPPING1__DEST_CLG_NUM__SHIFT                                                       0x6
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_LOW_MASK                                                           0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_VLD_MASK                                                           0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING1__DEST_CLG_NUM_MASK                                                         0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING2
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_LOW__SHIFT                                                         0x0
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_VLD__SHIFT                                                         0x5
+#define XPB_CLG_MM_UNITID_MAPPING2__DEST_CLG_NUM__SHIFT                                                       0x6
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_LOW_MASK                                                           0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_VLD_MASK                                                           0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING2__DEST_CLG_NUM_MASK                                                         0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING3
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_LOW__SHIFT                                                         0x0
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_VLD__SHIFT                                                         0x5
+#define XPB_CLG_MM_UNITID_MAPPING3__DEST_CLG_NUM__SHIFT                                                       0x6
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_LOW_MASK                                                           0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_VLD_MASK                                                           0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING3__DEST_CLG_NUM_MASK                                                         0x000001C0L
+
+
+// addressBlock: aid_athub_rpbdec
+//RPB_PASSPW_CONF
+#define RPB_PASSPW_CONF__XPB_PASSPW_OVERRIDE__SHIFT                                                           0x0
+#define RPB_PASSPW_CONF__XPB_RSPPASSPW_OVERRIDE__SHIFT                                                        0x1
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE__SHIFT                                                        0x2
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE__SHIFT                                                      0x3
+#define RPB_PASSPW_CONF__WR_PASSPW_OVERRIDE__SHIFT                                                            0x4
+#define RPB_PASSPW_CONF__RD_PASSPW_OVERRIDE__SHIFT                                                            0x5
+#define RPB_PASSPW_CONF__WR_RSPPASSPW_OVERRIDE__SHIFT                                                         0x6
+#define RPB_PASSPW_CONF__RD_RSPPASSPW_OVERRIDE__SHIFT                                                         0x7
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE__SHIFT                                                        0x8
+#define RPB_PASSPW_CONF__ATOMIC_PASSPW_OVERRIDE__SHIFT                                                        0x9
+#define RPB_PASSPW_CONF__ATOMIC_RSPPASSPW_OVERRIDE__SHIFT                                                     0xa
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_EN__SHIFT                                                     0xb
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_EN__SHIFT                                                   0xc
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_EN__SHIFT                                                     0xd
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE__SHIFT                                                         0xe
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_EN__SHIFT                                                      0xf
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE__SHIFT                                                         0x10
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_EN__SHIFT                                                      0x11
+#define RPB_PASSPW_CONF__XPB_PASSPW_OVERRIDE_MASK                                                             0x00000001L
+#define RPB_PASSPW_CONF__XPB_RSPPASSPW_OVERRIDE_MASK                                                          0x00000002L
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_MASK                                                          0x00000004L
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_MASK                                                        0x00000008L
+#define RPB_PASSPW_CONF__WR_PASSPW_OVERRIDE_MASK                                                              0x00000010L
+#define RPB_PASSPW_CONF__RD_PASSPW_OVERRIDE_MASK                                                              0x00000020L
+#define RPB_PASSPW_CONF__WR_RSPPASSPW_OVERRIDE_MASK                                                           0x00000040L
+#define RPB_PASSPW_CONF__RD_RSPPASSPW_OVERRIDE_MASK                                                           0x00000080L
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_MASK                                                          0x00000100L
+#define RPB_PASSPW_CONF__ATOMIC_PASSPW_OVERRIDE_MASK                                                          0x00000200L
+#define RPB_PASSPW_CONF__ATOMIC_RSPPASSPW_OVERRIDE_MASK                                                       0x00000400L
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_EN_MASK                                                       0x00000800L
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_EN_MASK                                                     0x00001000L
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_EN_MASK                                                       0x00002000L
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_MASK                                                           0x00004000L
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_EN_MASK                                                        0x00008000L
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_MASK                                                           0x00010000L
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_EN_MASK                                                        0x00020000L
+//RPB_BLOCKLEVEL_CONF
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE__SHIFT                                                   0x0
+#define RPB_BLOCKLEVEL_CONF__ATC_VC0_TR_BLOCKLEVEL__SHIFT                                                     0x2
+#define RPB_BLOCKLEVEL_CONF__ATC_VC5_TR_BLOCKLEVEL__SHIFT                                                     0x4
+#define RPB_BLOCKLEVEL_CONF__ATC_PAGE_BLOCKLEVEL__SHIFT                                                       0x6
+#define RPB_BLOCKLEVEL_CONF__ATC_INV_BLOCKLEVEL__SHIFT                                                        0x8
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE__SHIFT                                                 0xa
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE__SHIFT                                                 0xc
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE__SHIFT                                                0xe
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_EN__SHIFT                                                0x10
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_EN__SHIFT                                              0x11
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_EN__SHIFT                                              0x12
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_EN__SHIFT                                             0x13
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_MASK                                                     0x00000003L
+#define RPB_BLOCKLEVEL_CONF__ATC_VC0_TR_BLOCKLEVEL_MASK                                                       0x0000000CL
+#define RPB_BLOCKLEVEL_CONF__ATC_VC5_TR_BLOCKLEVEL_MASK                                                       0x00000030L
+#define RPB_BLOCKLEVEL_CONF__ATC_PAGE_BLOCKLEVEL_MASK                                                         0x000000C0L
+#define RPB_BLOCKLEVEL_CONF__ATC_INV_BLOCKLEVEL_MASK                                                          0x00000300L
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_MASK                                                   0x00000C00L
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_MASK                                                   0x00003000L
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_MASK                                                  0x0000C000L
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_EN_MASK                                                  0x00010000L
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_EN_MASK                                                0x00020000L
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_EN_MASK                                                0x00040000L
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_EN_MASK                                               0x00080000L
+//RPB_TAG_CONF
+#define RPB_TAG_CONF__RPB_ATS_VC0_TR__SHIFT                                                                   0x0
+#define RPB_TAG_CONF__RPB_ATS_VC5_TR__SHIFT                                                                   0xa
+#define RPB_TAG_CONF__RPB_ATS_PR__SHIFT                                                                       0x14
+#define RPB_TAG_CONF__RPB_ATS_VC0_TR_MASK                                                                     0x000003FFL
+#define RPB_TAG_CONF__RPB_ATS_VC5_TR_MASK                                                                     0x000FFC00L
+#define RPB_TAG_CONF__RPB_ATS_PR_MASK                                                                         0x3FF00000L
+//RPB_TAG_CONF2
+#define RPB_TAG_CONF2__RPB_IO_WR__SHIFT                                                                       0x0
+#define RPB_TAG_CONF2__RPB_IO_MAX_LIMIT__SHIFT                                                                0xa
+#define RPB_TAG_CONF2__RPB_IO_RD_MARGIN__SHIFT                                                                0x15
+#define RPB_TAG_CONF2__RPB_IO_WR_MASK                                                                         0x000003FFL
+#define RPB_TAG_CONF2__RPB_IO_MAX_LIMIT_MASK                                                                  0x001FFC00L
+#define RPB_TAG_CONF2__RPB_IO_RD_MARGIN_MASK                                                                  0xFFE00000L
+//RPB_ARB_CNTL
+#define RPB_ARB_CNTL__RD_SWITCH_NUM__SHIFT                                                                    0x0
+#define RPB_ARB_CNTL__WR_SWITCH_NUM__SHIFT                                                                    0x8
+#define RPB_ARB_CNTL__ATC_TR_SWITCH_NUM__SHIFT                                                                0x10
+#define RPB_ARB_CNTL__ARB_MODE__SHIFT                                                                         0x18
+#define RPB_ARB_CNTL__SWITCH_NUM_MODE__SHIFT                                                                  0x19
+#define RPB_ARB_CNTL__RD_SWITCH_NUM_MASK                                                                      0x000000FFL
+#define RPB_ARB_CNTL__WR_SWITCH_NUM_MASK                                                                      0x0000FF00L
+#define RPB_ARB_CNTL__ATC_TR_SWITCH_NUM_MASK                                                                  0x00FF0000L
+#define RPB_ARB_CNTL__ARB_MODE_MASK                                                                           0x01000000L
+#define RPB_ARB_CNTL__SWITCH_NUM_MODE_MASK                                                                    0x02000000L
+//RPB_ARB_CNTL2
+#define RPB_ARB_CNTL2__P2P_SWITCH_NUM__SHIFT                                                                  0x0
+#define RPB_ARB_CNTL2__ATOMIC_SWITCH_NUM__SHIFT                                                               0x8
+#define RPB_ARB_CNTL2__ATC_PAGE_SWITCH_NUM__SHIFT                                                             0x10
+#define RPB_ARB_CNTL2__P2P_SWITCH_NUM_MASK                                                                    0x000000FFL
+#define RPB_ARB_CNTL2__ATOMIC_SWITCH_NUM_MASK                                                                 0x0000FF00L
+#define RPB_ARB_CNTL2__ATC_PAGE_SWITCH_NUM_MASK                                                               0x00FF0000L
+//RPB_BIF_CNTL
+#define RPB_BIF_CNTL__ARB_MODE__SHIFT                                                                         0x0
+#define RPB_BIF_CNTL__DRAIN_VC_NUM__SHIFT                                                                     0x1
+#define RPB_BIF_CNTL__SWITCH_ENABLE__SHIFT                                                                    0x3
+#define RPB_BIF_CNTL__SWITCH_THRESHOLD__SHIFT                                                                 0x4
+#define RPB_BIF_CNTL__PAGE_PRI_EN__SHIFT                                                                      0xc
+#define RPB_BIF_CNTL__VC0TR_PRI_EN__SHIFT                                                                     0xd
+#define RPB_BIF_CNTL__VC5TR_PRI_EN__SHIFT                                                                     0xe
+#define RPB_BIF_CNTL__VC0_CHAINED_OVERRIDE__SHIFT                                                             0xf
+#define RPB_BIF_CNTL__ARB_MODE_MASK                                                                           0x00000001L
+#define RPB_BIF_CNTL__DRAIN_VC_NUM_MASK                                                                       0x00000006L
+#define RPB_BIF_CNTL__SWITCH_ENABLE_MASK                                                                      0x00000008L
+#define RPB_BIF_CNTL__SWITCH_THRESHOLD_MASK                                                                   0x00000FF0L
+#define RPB_BIF_CNTL__PAGE_PRI_EN_MASK                                                                        0x00001000L
+#define RPB_BIF_CNTL__VC0TR_PRI_EN_MASK                                                                       0x00002000L
+#define RPB_BIF_CNTL__VC5TR_PRI_EN_MASK                                                                       0x00004000L
+#define RPB_BIF_CNTL__VC0_CHAINED_OVERRIDE_MASK                                                               0x00008000L
+//RPB_BIF_CNTL2
+#define RPB_BIF_CNTL2__VC0_SWITCH_NUM__SHIFT                                                                  0x0
+#define RPB_BIF_CNTL2__VC1_SWITCH_NUM__SHIFT                                                                  0x8
+#define RPB_BIF_CNTL2__VC5_SWITCH_NUM__SHIFT                                                                  0x10
+#define RPB_BIF_CNTL2__VC0_SWITCH_NUM_MASK                                                                    0x000000FFL
+#define RPB_BIF_CNTL2__VC1_SWITCH_NUM_MASK                                                                    0x0000FF00L
+#define RPB_BIF_CNTL2__VC5_SWITCH_NUM_MASK                                                                    0x00FF0000L
+//RPB_PERF_COUNTER_CNTL
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_SELECT__SHIFT                                                     0x0
+#define RPB_PERF_COUNTER_CNTL__CLEAR_SELECTED_PERF_COUNTER__SHIFT                                             0x2
+#define RPB_PERF_COUNTER_CNTL__CLEAR_ALL_PERF_COUNTERS__SHIFT                                                 0x3
+#define RPB_PERF_COUNTER_CNTL__STOP_ON_COUNTER_SATURATION__SHIFT                                              0x4
+#define RPB_PERF_COUNTER_CNTL__ENABLE_PERF_COUNTERS__SHIFT                                                    0x5
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_0__SHIFT                                                   0x9
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_1__SHIFT                                                   0xe
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_2__SHIFT                                                   0x13
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_3__SHIFT                                                   0x18
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_SELECT_MASK                                                       0x00000003L
+#define RPB_PERF_COUNTER_CNTL__CLEAR_SELECTED_PERF_COUNTER_MASK                                               0x00000004L
+#define RPB_PERF_COUNTER_CNTL__CLEAR_ALL_PERF_COUNTERS_MASK                                                   0x00000008L
+#define RPB_PERF_COUNTER_CNTL__STOP_ON_COUNTER_SATURATION_MASK                                                0x00000010L
+#define RPB_PERF_COUNTER_CNTL__ENABLE_PERF_COUNTERS_MASK                                                      0x000001E0L
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_0_MASK                                                     0x00003E00L
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_1_MASK                                                     0x0007C000L
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_2_MASK                                                     0x00F80000L
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_3_MASK                                                     0x1F000000L
+//RPB_DEINTRLV_COMBINE_CNTL
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_FLUSH_TIMER__SHIFT                                              0x0
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_BREAK_EN__SHIFT                                                 0x4
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_HANDLE_CHECK_DISABLE__SHIFT                                             0x5
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CLI_INTLV_EN__SHIFT                                                     0x6
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_FLUSH_TIMER_MASK                                                0x0000000FL
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_BREAK_EN_MASK                                                   0x00000010L
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_HANDLE_CHECK_DISABLE_MASK                                               0x00000020L
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CLI_INTLV_EN_MASK                                                       0x00000040L
+//RPB_VC_SWITCH_RDWR
+#define RPB_VC_SWITCH_RDWR__MODE__SHIFT                                                                       0x0
+#define RPB_VC_SWITCH_RDWR__NUM_RD__SHIFT                                                                     0x2
+#define RPB_VC_SWITCH_RDWR__NUM_WR__SHIFT                                                                     0xa
+#define RPB_VC_SWITCH_RDWR__XPB_RDREQ_CRD__SHIFT                                                              0x12
+#define RPB_VC_SWITCH_RDWR__MODE_MASK                                                                         0x00000003L
+#define RPB_VC_SWITCH_RDWR__NUM_RD_MASK                                                                       0x000003FCL
+#define RPB_VC_SWITCH_RDWR__NUM_WR_MASK                                                                       0x0003FC00L
+#define RPB_VC_SWITCH_RDWR__XPB_RDREQ_CRD_MASK                                                                0x03FC0000L
+//RPB_PERFCOUNTER_LO
+#define RPB_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                                 0x0
+#define RPB_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                   0xFFFFFFFFL
+//RPB_PERFCOUNTER_HI
+#define RPB_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                                 0x0
+#define RPB_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                              0x10
+#define RPB_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                   0x0000FFFFL
+#define RPB_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                                0xFFFF0000L
+//RPB_PERFCOUNTER0_CFG
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                                 0x0
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                             0x8
+#define RPB_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                                0x18
+#define RPB_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                   0x1c
+#define RPB_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                    0x1d
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                   0x000000FFL
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
+#define RPB_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                  0x0F000000L
+#define RPB_PERFCOUNTER0_CFG__ENABLE_MASK                                                                     0x10000000L
+#define RPB_PERFCOUNTER0_CFG__CLEAR_MASK                                                                      0x20000000L
+//RPB_PERFCOUNTER1_CFG
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                                 0x0
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                             0x8
+#define RPB_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                                0x18
+#define RPB_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                   0x1c
+#define RPB_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                    0x1d
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                   0x000000FFL
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
+#define RPB_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                  0x0F000000L
+#define RPB_PERFCOUNTER1_CFG__ENABLE_MASK                                                                     0x10000000L
+#define RPB_PERFCOUNTER1_CFG__CLEAR_MASK                                                                      0x20000000L
+//RPB_PERFCOUNTER2_CFG
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                                 0x0
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                             0x8
+#define RPB_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                                0x18
+#define RPB_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                   0x1c
+#define RPB_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                    0x1d
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                   0x000000FFL
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
+#define RPB_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                  0x0F000000L
+#define RPB_PERFCOUNTER2_CFG__ENABLE_MASK                                                                     0x10000000L
+#define RPB_PERFCOUNTER2_CFG__CLEAR_MASK                                                                      0x20000000L
+//RPB_PERFCOUNTER3_CFG
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                                 0x0
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                             0x8
+#define RPB_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                                0x18
+#define RPB_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                                   0x1c
+#define RPB_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                                    0x1d
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                                   0x000000FFL
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
+#define RPB_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                                  0x0F000000L
+#define RPB_PERFCOUNTER3_CFG__ENABLE_MASK                                                                     0x10000000L
+#define RPB_PERFCOUNTER3_CFG__CLEAR_MASK                                                                      0x20000000L
+//RPB_PERFCOUNTER_RSLT_CNTL
+#define RPB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                                 0x0
+#define RPB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                       0x8
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                        0x10
+#define RPB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                          0x18
+#define RPB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                           0x19
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                                0x1a
+#define RPB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                   0x0000000FL
+#define RPB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                         0x0000FF00L
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                          0x00FF0000L
+#define RPB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                            0x01000000L
+#define RPB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                             0x02000000L
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                  0x04000000L
+//RPB_ATS_CNTL
+#define RPB_ATS_CNTL__PAGE_MIN_LATENCY_ENABLE__SHIFT                                                          0x0
+#define RPB_ATS_CNTL__TR_MIN_LATENCY_ENABLE__SHIFT                                                            0x1
+#define RPB_ATS_CNTL__SWITCH_THRESHOLD__SHIFT                                                                 0x2
+#define RPB_ATS_CNTL__TIME_SLICE__SHIFT                                                                       0x7
+#define RPB_ATS_CNTL__ATCTR_SWITCH_NUM__SHIFT                                                                 0xf
+#define RPB_ATS_CNTL__ATCPAGE_SWITCH_NUM__SHIFT                                                               0x13
+#define RPB_ATS_CNTL__WR_AT__SHIFT                                                                            0x17
+#define RPB_ATS_CNTL__INVAL_COM_CMD__SHIFT                                                                    0x19
+#define RPB_ATS_CNTL__PAGE_MIN_LATENCY_ENABLE_MASK                                                            0x00000001L
+#define RPB_ATS_CNTL__TR_MIN_LATENCY_ENABLE_MASK                                                              0x00000002L
+#define RPB_ATS_CNTL__SWITCH_THRESHOLD_MASK                                                                   0x0000007CL
+#define RPB_ATS_CNTL__TIME_SLICE_MASK                                                                         0x00007F80L
+#define RPB_ATS_CNTL__ATCTR_SWITCH_NUM_MASK                                                                   0x00078000L
+#define RPB_ATS_CNTL__ATCPAGE_SWITCH_NUM_MASK                                                                 0x00780000L
+#define RPB_ATS_CNTL__WR_AT_MASK                                                                              0x01800000L
+#define RPB_ATS_CNTL__INVAL_COM_CMD_MASK                                                                      0x7E000000L
+//RPB_ATS_CNTL2
+#define RPB_ATS_CNTL2__TRANS_CMD__SHIFT                                                                       0x0
+#define RPB_ATS_CNTL2__PAGE_REQ_CMD__SHIFT                                                                    0x6
+#define RPB_ATS_CNTL2__PAGE_ROUTING_CODE__SHIFT                                                               0xc
+#define RPB_ATS_CNTL2__INVAL_COM_ROUTING_CODE__SHIFT                                                          0xf
+#define RPB_ATS_CNTL2__VENDOR_ID__SHIFT                                                                       0x12
+#define RPB_ATS_CNTL2__MM_TRANS_VC5_ENABLE__SHIFT                                                             0x14
+#define RPB_ATS_CNTL2__GC_TRANS_VC5_ENABLE__SHIFT                                                             0x15
+#define RPB_ATS_CNTL2__RPB_VC5_CRD__SHIFT                                                                     0x16
+#define RPB_ATS_CNTL2__TRANS_CMD_MASK                                                                         0x0000003FL
+#define RPB_ATS_CNTL2__PAGE_REQ_CMD_MASK                                                                      0x00000FC0L
+#define RPB_ATS_CNTL2__PAGE_ROUTING_CODE_MASK                                                                 0x00007000L
+#define RPB_ATS_CNTL2__INVAL_COM_ROUTING_CODE_MASK                                                            0x00038000L
+#define RPB_ATS_CNTL2__VENDOR_ID_MASK                                                                         0x000C0000L
+#define RPB_ATS_CNTL2__MM_TRANS_VC5_ENABLE_MASK                                                               0x00100000L
+#define RPB_ATS_CNTL2__GC_TRANS_VC5_ENABLE_MASK                                                               0x00200000L
+#define RPB_ATS_CNTL2__RPB_VC5_CRD_MASK                                                                       0x07C00000L
+//RPB_SDPPORT_CNTL
+#define RPB_SDPPORT_CNTL__NBIF_DMA_SELF_ACTIVATE__SHIFT                                                       0x0
+#define RPB_SDPPORT_CNTL__NBIF_DMA_CFG_MODE__SHIFT                                                            0x1
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_REISSUE_CREDIT__SHIFT                                               0x3
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_SATURATE_COUNTER__SHIFT                                             0x4
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_DISRUPT_FULLDIS__SHIFT                                              0x5
+#define RPB_SDPPORT_CNTL__NBIF_DMA_HALT_THRESHOLD__SHIFT                                                      0x6
+#define RPB_SDPPORT_CNTL__NBIF_HST_SELF_ACTIVATE__SHIFT                                                       0xa
+#define RPB_SDPPORT_CNTL__NBIF_HST_CFG_MODE__SHIFT                                                            0xb
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_REISSUE_CREDIT__SHIFT                                               0xd
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_SATURATE_COUNTER__SHIFT                                             0xe
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_DISRUPT_FULLDIS__SHIFT                                              0xf
+#define RPB_SDPPORT_CNTL__NBIF_HST_HALT_THRESHOLD__SHIFT                                                      0x10
+#define RPB_SDPPORT_CNTL__NBIF_HST_PASSIVE_MODE__SHIFT                                                        0x14
+#define RPB_SDPPORT_CNTL__NBIF_HST_QUICK_COMACK__SHIFT                                                        0x15
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKEN__SHIFT                                                         0x16
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKENRCV__SHIFT                                                      0x17
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKEN__SHIFT                                                     0x18
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKENRCV__SHIFT                                                  0x19
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKEN__SHIFT                                                         0x1a
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKENRCV__SHIFT                                                      0x1b
+#define RPB_SDPPORT_CNTL__DF_HALT_THRESHOLD__SHIFT                                                            0x1c
+#define RPB_SDPPORT_CNTL__NBIF_DMA_SELF_ACTIVATE_MASK                                                         0x00000001L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_CFG_MODE_MASK                                                              0x00000006L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_REISSUE_CREDIT_MASK                                                 0x00000008L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_SATURATE_COUNTER_MASK                                               0x00000010L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_DISRUPT_FULLDIS_MASK                                                0x00000020L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_HALT_THRESHOLD_MASK                                                        0x000003C0L
+#define RPB_SDPPORT_CNTL__NBIF_HST_SELF_ACTIVATE_MASK                                                         0x00000400L
+#define RPB_SDPPORT_CNTL__NBIF_HST_CFG_MODE_MASK                                                              0x00001800L
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_REISSUE_CREDIT_MASK                                                 0x00002000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_SATURATE_COUNTER_MASK                                               0x00004000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_DISRUPT_FULLDIS_MASK                                                0x00008000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_HALT_THRESHOLD_MASK                                                        0x000F0000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_PASSIVE_MODE_MASK                                                          0x00100000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_QUICK_COMACK_MASK                                                          0x00200000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKEN_MASK                                                           0x00400000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKENRCV_MASK                                                        0x00800000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKEN_MASK                                                       0x01000000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKENRCV_MASK                                                    0x02000000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKEN_MASK                                                           0x04000000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKENRCV_MASK                                                        0x08000000L
+#define RPB_SDPPORT_CNTL__DF_HALT_THRESHOLD_MASK                                                              0xF0000000L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_offset.h
index 18d34bbceebe..79c41004c0b6 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_offset.h
@@ -4868,6 +4868,10 @@
 #define mmCP_ME2_PIPE2_INT_STATUS_BASE_IDX                                                             0
 #define mmCP_ME2_PIPE3_INT_STATUS                                                                      0x1e34
 #define mmCP_ME2_PIPE3_INT_STATUS_BASE_IDX                                                             0
+#define mmCP_ME1_INT_STAT_DEBUG                                                                        0x1e35
+#define mmCP_ME1_INT_STAT_DEBUG_BASE_IDX                                                               0
+#define mmCP_ME2_INT_STAT_DEBUG                                                                        0x1e36
+#define mmCP_ME2_INT_STAT_DEBUG_BASE_IDX                                                               0
 #define mmCP_GFX_QUEUE_INDEX                                                                           0x1e37
 #define mmCP_GFX_QUEUE_INDEX_BASE_IDX                                                                  0
 #define mmCC_GC_EDC_CONFIG                                                                             0x1e38
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_sh_mask.h
index 4127896ffcdf..52043e143067 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_sh_mask.h
@@ -18680,6 +18680,60 @@
 //CC_GC_EDC_CONFIG
 #define CC_GC_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
 #define CC_GC_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
+//CP_ME1_INT_STAT_DEBUG
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
+//CP_ME2_INT_STAT_DEBUG
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
 //CP_ME1_PIPE_PRIORITY_CNTS
 #define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
 #define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h
index 3973110f149c..a734abaa91a5 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h
@@ -4531,6 +4531,10 @@
 #define mmCP_GFX_QUEUE_INDEX_BASE_IDX                                                                  0
 #define mmCC_GC_EDC_CONFIG                                                                             0x1e38
 #define mmCC_GC_EDC_CONFIG_BASE_IDX                                                                    0
+#define mmCP_ME1_INT_STAT_DEBUG                                                                        0x1e35
+#define mmCP_ME1_INT_STAT_DEBUG_BASE_IDX                                                               0
+#define mmCP_ME2_INT_STAT_DEBUG                                                                        0x1e36
+#define mmCP_ME2_INT_STAT_DEBUG_BASE_IDX                                                               0
 #define mmCP_ME1_PIPE_PRIORITY_CNTS                                                                    0x1e39
 #define mmCP_ME1_PIPE_PRIORITY_CNTS_BASE_IDX                                                           0
 #define mmCP_ME1_PIPE0_PRIORITY                                                                        0x1e3a
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h
index d4e8ff22ecb8..d7a17bae2584 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h
@@ -17028,6 +17028,60 @@
 //CC_GC_EDC_CONFIG
 #define CC_GC_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
 #define CC_GC_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
+//CP_ME1_INT_STAT_DEBUG
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
+//CP_ME2_INT_STAT_DEBUG
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
 //CP_ME1_PIPE_PRIORITY_CNTS
 #define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
 #define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_3_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_3_offset.h
new file mode 100644
index 000000000000..3100de8b3881
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_3_offset.h
@@ -0,0 +1,7258 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _gc_9_4_3_OFFSET_HEADER
+#define _gc_9_4_3_OFFSET_HEADER
+
+
+
+// addressBlock: xcd0_gc_grbmdec
+// base address: 0x8000
+#define regGRBM_CNTL                                                                                    0x0000
+#define regGRBM_CNTL_BASE_IDX                                                                           0
+#define regGRBM_SKEW_CNTL                                                                               0x0001
+#define regGRBM_SKEW_CNTL_BASE_IDX                                                                      0
+#define regGRBM_STATUS2                                                                                 0x0002
+#define regGRBM_STATUS2_BASE_IDX                                                                        0
+#define regGRBM_PWR_CNTL                                                                                0x0003
+#define regGRBM_PWR_CNTL_BASE_IDX                                                                       0
+#define regGRBM_STATUS                                                                                  0x0004
+#define regGRBM_STATUS_BASE_IDX                                                                         0
+#define regGRBM_STATUS_SE0                                                                              0x0005
+#define regGRBM_STATUS_SE0_BASE_IDX                                                                     0
+#define regGRBM_STATUS_SE1                                                                              0x0006
+#define regGRBM_STATUS_SE1_BASE_IDX                                                                     0
+#define regGRBM_SOFT_RESET                                                                              0x0008
+#define regGRBM_SOFT_RESET_BASE_IDX                                                                     0
+#define regGRBM_GFX_CLKEN_CNTL                                                                          0x000c
+#define regGRBM_GFX_CLKEN_CNTL_BASE_IDX                                                                 0
+#define regGRBM_WAIT_IDLE_CLOCKS                                                                        0x000d
+#define regGRBM_WAIT_IDLE_CLOCKS_BASE_IDX                                                               0
+#define regGRBM_STATUS_SE2                                                                              0x000e
+#define regGRBM_STATUS_SE2_BASE_IDX                                                                     0
+#define regGRBM_STATUS_SE3                                                                              0x000f
+#define regGRBM_STATUS_SE3_BASE_IDX                                                                     0
+#define regGRBM_READ_ERROR                                                                              0x0016
+#define regGRBM_READ_ERROR_BASE_IDX                                                                     0
+#define regGRBM_READ_ERROR2                                                                             0x0017
+#define regGRBM_READ_ERROR2_BASE_IDX                                                                    0
+#define regGRBM_INT_CNTL                                                                                0x0018
+#define regGRBM_INT_CNTL_BASE_IDX                                                                       0
+#define regGRBM_TRAP_OP                                                                                 0x0019
+#define regGRBM_TRAP_OP_BASE_IDX                                                                        0
+#define regGRBM_TRAP_ADDR                                                                               0x001a
+#define regGRBM_TRAP_ADDR_BASE_IDX                                                                      0
+#define regGRBM_TRAP_ADDR_MSK                                                                           0x001b
+#define regGRBM_TRAP_ADDR_MSK_BASE_IDX                                                                  0
+#define regGRBM_TRAP_WD                                                                                 0x001c
+#define regGRBM_TRAP_WD_BASE_IDX                                                                        0
+#define regGRBM_TRAP_WD_MSK                                                                             0x001d
+#define regGRBM_TRAP_WD_MSK_BASE_IDX                                                                    0
+#define regGRBM_WRITE_ERROR                                                                             0x001f
+#define regGRBM_WRITE_ERROR_BASE_IDX                                                                    0
+#define regGRBM_IOV_ERROR                                                                               0x0020
+#define regGRBM_IOV_ERROR_BASE_IDX                                                                      0
+#define regGRBM_CHIP_REVISION                                                                           0x0021
+#define regGRBM_CHIP_REVISION_BASE_IDX                                                                  0
+#define regGRBM_GFX_CNTL                                                                                0x0022
+#define regGRBM_GFX_CNTL_BASE_IDX                                                                       0
+#define regGRBM_RSMU_CFG                                                                                0x0023
+#define regGRBM_RSMU_CFG_BASE_IDX                                                                       0
+#define regGRBM_IH_CREDIT                                                                               0x0024
+#define regGRBM_IH_CREDIT_BASE_IDX                                                                      0
+#define regGRBM_PWR_CNTL2                                                                               0x0025
+#define regGRBM_PWR_CNTL2_BASE_IDX                                                                      0
+#define regGRBM_UTCL2_INVAL_RANGE_START                                                                 0x0026
+#define regGRBM_UTCL2_INVAL_RANGE_START_BASE_IDX                                                        0
+#define regGRBM_UTCL2_INVAL_RANGE_END                                                                   0x0027
+#define regGRBM_UTCL2_INVAL_RANGE_END_BASE_IDX                                                          0
+#define regGRBM_RSMU_READ_ERROR                                                                         0x0028
+#define regGRBM_RSMU_READ_ERROR_BASE_IDX                                                                0
+#define regGRBM_CHICKEN_BITS                                                                            0x0029
+#define regGRBM_CHICKEN_BITS_BASE_IDX                                                                   0
+#define regGRBM_FENCE_RANGE0                                                                            0x002a
+#define regGRBM_FENCE_RANGE0_BASE_IDX                                                                   0
+#define regGRBM_FENCE_RANGE1                                                                            0x002b
+#define regGRBM_FENCE_RANGE1_BASE_IDX                                                                   0
+#define regGRBM_IOV_READ_ERROR                                                                          0x002c
+#define regGRBM_IOV_READ_ERROR_BASE_IDX                                                                 0
+#define regGRBM_NOWHERE                                                                                 0x003f
+#define regGRBM_NOWHERE_BASE_IDX                                                                        0
+#define regGRBM_SCRATCH_REG0                                                                            0x0040
+#define regGRBM_SCRATCH_REG0_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG1                                                                            0x0041
+#define regGRBM_SCRATCH_REG1_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG2                                                                            0x0042
+#define regGRBM_SCRATCH_REG2_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG3                                                                            0x0043
+#define regGRBM_SCRATCH_REG3_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG4                                                                            0x0044
+#define regGRBM_SCRATCH_REG4_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG5                                                                            0x0045
+#define regGRBM_SCRATCH_REG5_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG6                                                                            0x0046
+#define regGRBM_SCRATCH_REG6_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG7                                                                            0x0047
+#define regGRBM_SCRATCH_REG7_BASE_IDX                                                                   0
+#define regVIOLATION_DATA_ASYNC_VF_PROG                                                                 0x0048
+#define regVIOLATION_DATA_ASYNC_VF_PROG_BASE_IDX                                                        0
+
+
+// addressBlock: xcd0_gc_cpdec
+// base address: 0x8200
+#define regCP_CPC_DEBUG_CNTL                                                                            0x0080
+#define regCP_CPC_DEBUG_CNTL_BASE_IDX                                                                   0
+#define regCP_CPF_DEBUG_CNTL                                                                            0x0082
+#define regCP_CPF_DEBUG_CNTL_BASE_IDX                                                                   0
+#define regCP_CPC_STATUS                                                                                0x0084
+#define regCP_CPC_STATUS_BASE_IDX                                                                       0
+#define regCP_CPC_BUSY_STAT                                                                             0x0085
+#define regCP_CPC_BUSY_STAT_BASE_IDX                                                                    0
+#define regCP_CPC_STALLED_STAT1                                                                         0x0086
+#define regCP_CPC_STALLED_STAT1_BASE_IDX                                                                0
+#define regCP_CPF_STATUS                                                                                0x0087
+#define regCP_CPF_STATUS_BASE_IDX                                                                       0
+#define regCP_CPF_BUSY_STAT                                                                             0x0088
+#define regCP_CPF_BUSY_STAT_BASE_IDX                                                                    0
+#define regCP_CPF_STALLED_STAT1                                                                         0x0089
+#define regCP_CPF_STALLED_STAT1_BASE_IDX                                                                0
+#define regCP_CPC_GRBM_FREE_COUNT                                                                       0x008b
+#define regCP_CPC_GRBM_FREE_COUNT_BASE_IDX                                                              0
+#define regCP_CPC_PRIV_VIOLATION_ADDR                                                                   0x008c
+#define regCP_CPC_PRIV_VIOLATION_ADDR_BASE_IDX                                                          0
+#define regCP_MEC_CNTL                                                                                  0x008d
+#define regCP_MEC_CNTL_BASE_IDX                                                                         0
+#define regCP_MEC_ME1_HEADER_DUMP                                                                       0x008e
+#define regCP_MEC_ME1_HEADER_DUMP_BASE_IDX                                                              0
+#define regCP_MEC_ME2_HEADER_DUMP                                                                       0x008f
+#define regCP_MEC_ME2_HEADER_DUMP_BASE_IDX                                                              0
+#define regCP_CPC_SCRATCH_INDEX                                                                         0x0090
+#define regCP_CPC_SCRATCH_INDEX_BASE_IDX                                                                0
+#define regCP_CPC_SCRATCH_DATA                                                                          0x0091
+#define regCP_CPC_SCRATCH_DATA_BASE_IDX                                                                 0
+#define regCP_CPF_GRBM_FREE_COUNT                                                                       0x0092
+#define regCP_CPF_GRBM_FREE_COUNT_BASE_IDX                                                              0
+#define regCP_CPC_HALT_HYST_COUNT                                                                       0x00a7
+#define regCP_CPC_HALT_HYST_COUNT_BASE_IDX                                                              0
+#define regCP_CE_COMPARE_COUNT                                                                          0x00c0
+#define regCP_CE_COMPARE_COUNT_BASE_IDX                                                                 0
+#define regCP_CE_DE_COUNT                                                                               0x00c1
+#define regCP_CE_DE_COUNT_BASE_IDX                                                                      0
+#define regCP_DE_CE_COUNT                                                                               0x00c2
+#define regCP_DE_CE_COUNT_BASE_IDX                                                                      0
+#define regCP_DE_LAST_INVAL_COUNT                                                                       0x00c3
+#define regCP_DE_LAST_INVAL_COUNT_BASE_IDX                                                              0
+#define regCP_DE_DE_COUNT                                                                               0x00c4
+#define regCP_DE_DE_COUNT_BASE_IDX                                                                      0
+#define regCP_STALLED_STAT3                                                                             0x019c
+#define regCP_STALLED_STAT3_BASE_IDX                                                                    0
+#define regCP_STALLED_STAT1                                                                             0x019d
+#define regCP_STALLED_STAT1_BASE_IDX                                                                    0
+#define regCP_STALLED_STAT2                                                                             0x019e
+#define regCP_STALLED_STAT2_BASE_IDX                                                                    0
+#define regCP_BUSY_STAT                                                                                 0x019f
+#define regCP_BUSY_STAT_BASE_IDX                                                                        0
+#define regCP_STAT                                                                                      0x01a0
+#define regCP_STAT_BASE_IDX                                                                             0
+#define regCP_ME_HEADER_DUMP                                                                            0x01a1
+#define regCP_ME_HEADER_DUMP_BASE_IDX                                                                   0
+#define regCP_PFP_HEADER_DUMP                                                                           0x01a2
+#define regCP_PFP_HEADER_DUMP_BASE_IDX                                                                  0
+#define regCP_GRBM_FREE_COUNT                                                                           0x01a3
+#define regCP_GRBM_FREE_COUNT_BASE_IDX                                                                  0
+#define regCP_CE_HEADER_DUMP                                                                            0x01a4
+#define regCP_CE_HEADER_DUMP_BASE_IDX                                                                   0
+#define regCP_PFP_INSTR_PNTR                                                                            0x01a5
+#define regCP_PFP_INSTR_PNTR_BASE_IDX                                                                   0
+#define regCP_ME_INSTR_PNTR                                                                             0x01a6
+#define regCP_ME_INSTR_PNTR_BASE_IDX                                                                    0
+#define regCP_CE_INSTR_PNTR                                                                             0x01a7
+#define regCP_CE_INSTR_PNTR_BASE_IDX                                                                    0
+#define regCP_MEC1_INSTR_PNTR                                                                           0x01a8
+#define regCP_MEC1_INSTR_PNTR_BASE_IDX                                                                  0
+#define regCP_MEC2_INSTR_PNTR                                                                           0x01a9
+#define regCP_MEC2_INSTR_PNTR_BASE_IDX                                                                  0
+#define regCP_CSF_STAT                                                                                  0x01b4
+#define regCP_CSF_STAT_BASE_IDX                                                                         0
+#define regCP_ME_CNTL                                                                                   0x01b6
+#define regCP_ME_CNTL_BASE_IDX                                                                          0
+#define regCP_CNTX_STAT                                                                                 0x01b8
+#define regCP_CNTX_STAT_BASE_IDX                                                                        0
+#define regCP_ME_PREEMPTION                                                                             0x01b9
+#define regCP_ME_PREEMPTION_BASE_IDX                                                                    0
+#define regCP_ROQ_THRESHOLDS                                                                            0x01bc
+#define regCP_ROQ_THRESHOLDS_BASE_IDX                                                                   0
+#define regCP_MEQ_STQ_THRESHOLD                                                                         0x01bd
+#define regCP_MEQ_STQ_THRESHOLD_BASE_IDX                                                                0
+#define regCP_RB2_RPTR                                                                                  0x01be
+#define regCP_RB2_RPTR_BASE_IDX                                                                         0
+#define regCP_RB1_RPTR                                                                                  0x01bf
+#define regCP_RB1_RPTR_BASE_IDX                                                                         0
+#define regCP_RB0_RPTR                                                                                  0x01c0
+#define regCP_RB0_RPTR_BASE_IDX                                                                         0
+#define regCP_RB_RPTR                                                                                   0x01c0
+#define regCP_RB_RPTR_BASE_IDX                                                                          0
+#define regCP_RB_WPTR_DELAY                                                                             0x01c1
+#define regCP_RB_WPTR_DELAY_BASE_IDX                                                                    0
+#define regCP_RB_WPTR_POLL_CNTL                                                                         0x01c2
+#define regCP_RB_WPTR_POLL_CNTL_BASE_IDX                                                                0
+#define regCP_ROQ1_THRESHOLDS                                                                           0x01d5
+#define regCP_ROQ1_THRESHOLDS_BASE_IDX                                                                  0
+#define regCP_ROQ2_THRESHOLDS                                                                           0x01d6
+#define regCP_ROQ2_THRESHOLDS_BASE_IDX                                                                  0
+#define regCP_STQ_THRESHOLDS                                                                            0x01d7
+#define regCP_STQ_THRESHOLDS_BASE_IDX                                                                   0
+#define regCP_QUEUE_THRESHOLDS                                                                          0x01d8
+#define regCP_QUEUE_THRESHOLDS_BASE_IDX                                                                 0
+#define regCP_MEQ_THRESHOLDS                                                                            0x01d9
+#define regCP_MEQ_THRESHOLDS_BASE_IDX                                                                   0
+#define regCP_ROQ_AVAIL                                                                                 0x01da
+#define regCP_ROQ_AVAIL_BASE_IDX                                                                        0
+#define regCP_STQ_AVAIL                                                                                 0x01db
+#define regCP_STQ_AVAIL_BASE_IDX                                                                        0
+#define regCP_ROQ2_AVAIL                                                                                0x01dc
+#define regCP_ROQ2_AVAIL_BASE_IDX                                                                       0
+#define regCP_MEQ_AVAIL                                                                                 0x01dd
+#define regCP_MEQ_AVAIL_BASE_IDX                                                                        0
+#define regCP_CMD_INDEX                                                                                 0x01de
+#define regCP_CMD_INDEX_BASE_IDX                                                                        0
+#define regCP_CMD_DATA                                                                                  0x01df
+#define regCP_CMD_DATA_BASE_IDX                                                                         0
+#define regCP_ROQ_RB_STAT                                                                               0x01e0
+#define regCP_ROQ_RB_STAT_BASE_IDX                                                                      0
+#define regCP_ROQ_IB1_STAT                                                                              0x01e1
+#define regCP_ROQ_IB1_STAT_BASE_IDX                                                                     0
+#define regCP_ROQ_IB2_STAT                                                                              0x01e2
+#define regCP_ROQ_IB2_STAT_BASE_IDX                                                                     0
+#define regCP_STQ_STAT                                                                                  0x01e3
+#define regCP_STQ_STAT_BASE_IDX                                                                         0
+#define regCP_STQ_WR_STAT                                                                               0x01e4
+#define regCP_STQ_WR_STAT_BASE_IDX                                                                      0
+#define regCP_MEQ_STAT                                                                                  0x01e5
+#define regCP_MEQ_STAT_BASE_IDX                                                                         0
+#define regCP_CEQ1_AVAIL                                                                                0x01e6
+#define regCP_CEQ1_AVAIL_BASE_IDX                                                                       0
+#define regCP_CEQ2_AVAIL                                                                                0x01e7
+#define regCP_CEQ2_AVAIL_BASE_IDX                                                                       0
+#define regCP_CE_ROQ_RB_STAT                                                                            0x01e8
+#define regCP_CE_ROQ_RB_STAT_BASE_IDX                                                                   0
+#define regCP_CE_ROQ_IB1_STAT                                                                           0x01e9
+#define regCP_CE_ROQ_IB1_STAT_BASE_IDX                                                                  0
+#define regCP_CE_ROQ_IB2_STAT                                                                           0x01ea
+#define regCP_CE_ROQ_IB2_STAT_BASE_IDX                                                                  0
+#define regCP_INT_STAT_DEBUG                                                                            0x01f7
+#define regCP_INT_STAT_DEBUG_BASE_IDX                                                                   0
+#define regCP_DEBUG_CNTL                                                                                0x01f8
+#define regCP_DEBUG_CNTL_BASE_IDX                                                                       0
+#define regCP_PRIV_VIOLATION_ADDR                                                                       0x01fa
+#define regCP_PRIV_VIOLATION_ADDR_BASE_IDX                                                              0
+
+
+// addressBlock: xcd0_gc_padec
+// base address: 0x8800
+#define regVGT_VTX_VECT_EJECT_REG                                                                       0x022c
+#define regVGT_VTX_VECT_EJECT_REG_BASE_IDX                                                              0
+#define regVGT_DMA_DATA_FIFO_DEPTH                                                                      0x022d
+#define regVGT_DMA_DATA_FIFO_DEPTH_BASE_IDX                                                             0
+#define regVGT_DMA_REQ_FIFO_DEPTH                                                                       0x022e
+#define regVGT_DMA_REQ_FIFO_DEPTH_BASE_IDX                                                              0
+#define regVGT_DRAW_INIT_FIFO_DEPTH                                                                     0x022f
+#define regVGT_DRAW_INIT_FIFO_DEPTH_BASE_IDX                                                            0
+#define regVGT_LAST_COPY_STATE                                                                          0x0230
+#define regVGT_LAST_COPY_STATE_BASE_IDX                                                                 0
+#define regVGT_CACHE_INVALIDATION                                                                       0x0231
+#define regVGT_CACHE_INVALIDATION_BASE_IDX                                                              0
+#define regVGT_RESET_DEBUG                                                                              0x0232
+#define regVGT_RESET_DEBUG_BASE_IDX                                                                     0
+#define regVGT_STRMOUT_DELAY                                                                            0x0233
+#define regVGT_STRMOUT_DELAY_BASE_IDX                                                                   0
+#define regVGT_FIFO_DEPTHS                                                                              0x0234
+#define regVGT_FIFO_DEPTHS_BASE_IDX                                                                     0
+#define regVGT_GS_VERTEX_REUSE                                                                          0x0235
+#define regVGT_GS_VERTEX_REUSE_BASE_IDX                                                                 0
+#define regVGT_MC_LAT_CNTL                                                                              0x0236
+#define regVGT_MC_LAT_CNTL_BASE_IDX                                                                     0
+#define regIA_CNTL_STATUS                                                                               0x0237
+#define regIA_CNTL_STATUS_BASE_IDX                                                                      0
+#define regVGT_CNTL_STATUS                                                                              0x023c
+#define regVGT_CNTL_STATUS_BASE_IDX                                                                     0
+#define regWD_CNTL_STATUS                                                                               0x023f
+#define regWD_CNTL_STATUS_BASE_IDX                                                                      0
+#define regCC_GC_PRIM_CONFIG                                                                            0x0240
+#define regCC_GC_PRIM_CONFIG_BASE_IDX                                                                   0
+#define regGC_USER_PRIM_CONFIG                                                                          0x0241
+#define regGC_USER_PRIM_CONFIG_BASE_IDX                                                                 0
+#define regWD_QOS                                                                                       0x0242
+#define regWD_QOS_BASE_IDX                                                                              0
+#define regWD_UTCL1_CNTL                                                                                0x0243
+#define regWD_UTCL1_CNTL_BASE_IDX                                                                       0
+#define regWD_UTCL1_STATUS                                                                              0x0244
+#define regWD_UTCL1_STATUS_BASE_IDX                                                                     0
+#define regIA_UTCL1_CNTL                                                                                0x0246
+#define regIA_UTCL1_CNTL_BASE_IDX                                                                       0
+#define regIA_UTCL1_STATUS                                                                              0x0247
+#define regIA_UTCL1_STATUS_BASE_IDX                                                                     0
+#define regVGT_SYS_CONFIG                                                                               0x0263
+#define regVGT_SYS_CONFIG_BASE_IDX                                                                      0
+#define regVGT_VS_MAX_WAVE_ID                                                                           0x0268
+#define regVGT_VS_MAX_WAVE_ID_BASE_IDX                                                                  0
+#define regVGT_GS_MAX_WAVE_ID                                                                           0x0269
+#define regVGT_GS_MAX_WAVE_ID_BASE_IDX                                                                  0
+#define regGFX_PIPE_CONTROL                                                                             0x026d
+#define regGFX_PIPE_CONTROL_BASE_IDX                                                                    0
+#define regCC_GC_SHADER_ARRAY_CONFIG                                                                    0x026f
+#define regCC_GC_SHADER_ARRAY_CONFIG_BASE_IDX                                                           0
+#define regGC_USER_SHADER_ARRAY_CONFIG                                                                  0x0270
+#define regGC_USER_SHADER_ARRAY_CONFIG_BASE_IDX                                                         0
+#define regVGT_DMA_PRIMITIVE_TYPE                                                                       0x0271
+#define regVGT_DMA_PRIMITIVE_TYPE_BASE_IDX                                                              0
+#define regVGT_DMA_CONTROL                                                                              0x0272
+#define regVGT_DMA_CONTROL_BASE_IDX                                                                     0
+#define regVGT_DMA_LS_HS_CONFIG                                                                         0x0273
+#define regVGT_DMA_LS_HS_CONFIG_BASE_IDX                                                                0
+#define regWD_BUF_RESOURCE_1                                                                            0x0276
+#define regWD_BUF_RESOURCE_1_BASE_IDX                                                                   0
+#define regWD_BUF_RESOURCE_2                                                                            0x0277
+#define regWD_BUF_RESOURCE_2_BASE_IDX                                                                   0
+#define regPA_CL_CNTL_STATUS                                                                            0x0284
+#define regPA_CL_CNTL_STATUS_BASE_IDX                                                                   0
+#define regPA_CL_ENHANCE                                                                                0x0285
+#define regPA_CL_ENHANCE_BASE_IDX                                                                       0
+#define regPA_CL_RESET_DEBUG                                                                            0x0286
+#define regPA_CL_RESET_DEBUG_BASE_IDX                                                                   0
+#define regPA_SU_CNTL_STATUS                                                                            0x0294
+#define regPA_SU_CNTL_STATUS_BASE_IDX                                                                   0
+#define regPA_SC_FIFO_DEPTH_CNTL                                                                        0x0295
+#define regPA_SC_FIFO_DEPTH_CNTL_BASE_IDX                                                               0
+#define regPA_SC_P3D_TRAP_SCREEN_HV_LOCK                                                                0x02c0
+#define regPA_SC_P3D_TRAP_SCREEN_HV_LOCK_BASE_IDX                                                       0
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_LOCK                                                               0x02c1
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_LOCK_BASE_IDX                                                      0
+#define regPA_SC_TRAP_SCREEN_HV_LOCK                                                                    0x02c2
+#define regPA_SC_TRAP_SCREEN_HV_LOCK_BASE_IDX                                                           0
+#define regPA_SC_FORCE_EOV_MAX_CNTS                                                                     0x02c9
+#define regPA_SC_FORCE_EOV_MAX_CNTS_BASE_IDX                                                            0
+#define regPA_SC_BINNER_EVENT_CNTL_0                                                                    0x02cc
+#define regPA_SC_BINNER_EVENT_CNTL_0_BASE_IDX                                                           0
+#define regPA_SC_BINNER_EVENT_CNTL_1                                                                    0x02cd
+#define regPA_SC_BINNER_EVENT_CNTL_1_BASE_IDX                                                           0
+#define regPA_SC_BINNER_EVENT_CNTL_2                                                                    0x02ce
+#define regPA_SC_BINNER_EVENT_CNTL_2_BASE_IDX                                                           0
+#define regPA_SC_BINNER_EVENT_CNTL_3                                                                    0x02cf
+#define regPA_SC_BINNER_EVENT_CNTL_3_BASE_IDX                                                           0
+#define regPA_SC_BINNER_TIMEOUT_COUNTER                                                                 0x02d0
+#define regPA_SC_BINNER_TIMEOUT_COUNTER_BASE_IDX                                                        0
+#define regPA_SC_BINNER_PERF_CNTL_0                                                                     0x02d1
+#define regPA_SC_BINNER_PERF_CNTL_0_BASE_IDX                                                            0
+#define regPA_SC_BINNER_PERF_CNTL_1                                                                     0x02d2
+#define regPA_SC_BINNER_PERF_CNTL_1_BASE_IDX                                                            0
+#define regPA_SC_BINNER_PERF_CNTL_2                                                                     0x02d3
+#define regPA_SC_BINNER_PERF_CNTL_2_BASE_IDX                                                            0
+#define regPA_SC_BINNER_PERF_CNTL_3                                                                     0x02d4
+#define regPA_SC_BINNER_PERF_CNTL_3_BASE_IDX                                                            0
+#define regPA_SC_ENHANCE_2                                                                              0x02dc
+#define regPA_SC_ENHANCE_2_BASE_IDX                                                                     0
+#define regPA_SC_FIFO_SIZE                                                                              0x02f3
+#define regPA_SC_FIFO_SIZE_BASE_IDX                                                                     0
+#define regPA_SC_IF_FIFO_SIZE                                                                           0x02f5
+#define regPA_SC_IF_FIFO_SIZE_BASE_IDX                                                                  0
+#define regPA_SC_PKR_WAVE_TABLE_CNTL                                                                    0x02f8
+#define regPA_SC_PKR_WAVE_TABLE_CNTL_BASE_IDX                                                           0
+#define regPA_UTCL1_CNTL1                                                                               0x02f9
+#define regPA_UTCL1_CNTL1_BASE_IDX                                                                      0
+#define regPA_UTCL1_CNTL2                                                                               0x02fa
+#define regPA_UTCL1_CNTL2_BASE_IDX                                                                      0
+#define regPA_SIDEBAND_REQUEST_DELAYS                                                                   0x02fb
+#define regPA_SIDEBAND_REQUEST_DELAYS_BASE_IDX                                                          0
+#define regPA_SC_ENHANCE                                                                                0x02fc
+#define regPA_SC_ENHANCE_BASE_IDX                                                                       0
+#define regPA_SC_ENHANCE_1                                                                              0x02fd
+#define regPA_SC_ENHANCE_1_BASE_IDX                                                                     0
+#define regPA_SC_DSM_CNTL                                                                               0x02fe
+#define regPA_SC_DSM_CNTL_BASE_IDX                                                                      0
+#define regPA_SC_TILE_STEERING_CREST_OVERRIDE                                                           0x02ff
+#define regPA_SC_TILE_STEERING_CREST_OVERRIDE_BASE_IDX                                                  0
+
+
+// addressBlock: xcd0_gc_sqdec
+// base address: 0x8c00
+#define regSQ_CONFIG                                                                                    0x0300
+#define regSQ_CONFIG_BASE_IDX                                                                           0
+#define regSQC_CONFIG                                                                                   0x0301
+#define regSQC_CONFIG_BASE_IDX                                                                          0
+#define regLDS_CONFIG                                                                                   0x0302
+#define regLDS_CONFIG_BASE_IDX                                                                          0
+#define regSQ_RANDOM_WAVE_PRI                                                                           0x0303
+#define regSQ_RANDOM_WAVE_PRI_BASE_IDX                                                                  0
+#define regSQ_REG_CREDITS                                                                               0x0304
+#define regSQ_REG_CREDITS_BASE_IDX                                                                      0
+#define regSQ_FIFO_SIZES                                                                                0x0305
+#define regSQ_FIFO_SIZES_BASE_IDX                                                                       0
+#define regSQ_DSM_CNTL                                                                                  0x0306
+#define regSQ_DSM_CNTL_BASE_IDX                                                                         0
+#define regSQ_DSM_CNTL2                                                                                 0x0307
+#define regSQ_DSM_CNTL2_BASE_IDX                                                                        0
+#define regSQ_RUNTIME_CONFIG                                                                            0x0308
+#define regSQ_RUNTIME_CONFIG_BASE_IDX                                                                   0
+#define regSQ_DEBUG_STS_GLOBAL                                                                          0x0309
+#define regSQ_DEBUG_STS_GLOBAL_BASE_IDX                                                                 0
+#define regSH_MEM_BASES                                                                                 0x030a
+#define regSH_MEM_BASES_BASE_IDX                                                                        0
+#define regSQ_TIMEOUT_CONFIG                                                                            0x030b
+#define regSQ_TIMEOUT_CONFIG_BASE_IDX                                                                   0
+#define regSQ_TIMEOUT_STATUS                                                                            0x030c
+#define regSQ_TIMEOUT_STATUS_BASE_IDX                                                                   0
+#define regSH_MEM_CONFIG                                                                                0x030d
+#define regSH_MEM_CONFIG_BASE_IDX                                                                       0
+#define regSP_MFMA_PORTD_RD_CONFIG                                                                      0x030e
+#define regSP_MFMA_PORTD_RD_CONFIG_BASE_IDX                                                             0
+#define regSH_CAC_CONFIG                                                                                0x030f
+#define regSH_CAC_CONFIG_BASE_IDX                                                                       0
+#define regSQ_DEBUG_STS_GLOBAL2                                                                         0x0310
+#define regSQ_DEBUG_STS_GLOBAL2_BASE_IDX                                                                0
+#define regSQ_DEBUG_STS_GLOBAL3                                                                         0x0311
+#define regSQ_DEBUG_STS_GLOBAL3_BASE_IDX                                                                0
+#define regCC_GC_SHADER_RATE_CONFIG                                                                     0x0312
+#define regCC_GC_SHADER_RATE_CONFIG_BASE_IDX                                                            0
+#define regGC_USER_SHADER_RATE_CONFIG                                                                   0x0313
+#define regGC_USER_SHADER_RATE_CONFIG_BASE_IDX                                                          0
+#define regSQ_INTERRUPT_AUTO_MASK                                                                       0x0314
+#define regSQ_INTERRUPT_AUTO_MASK_BASE_IDX                                                              0
+#define regSQ_INTERRUPT_MSG_CTRL                                                                        0x0315
+#define regSQ_INTERRUPT_MSG_CTRL_BASE_IDX                                                               0
+#define regSQ_DEBUG_PERFCOUNT_TRAP                                                                      0x0316
+#define regSQ_DEBUG_PERFCOUNT_TRAP_BASE_IDX                                                             0
+#define regSQ_UTCL1_CNTL1                                                                               0x0317
+#define regSQ_UTCL1_CNTL1_BASE_IDX                                                                      0
+#define regSQ_UTCL1_CNTL2                                                                               0x0318
+#define regSQ_UTCL1_CNTL2_BASE_IDX                                                                      0
+#define regSQ_UTCL1_STATUS                                                                              0x0319
+#define regSQ_UTCL1_STATUS_BASE_IDX                                                                     0
+#define regSQ_FED_INTERRUPT_STATUS                                                                      0x031a
+#define regSQ_FED_INTERRUPT_STATUS_BASE_IDX                                                             0
+#define regSQ_CGTS_CONFIG                                                                               0x031b
+#define regSQ_CGTS_CONFIG_BASE_IDX                                                                      0
+#define regSQ_SHADER_TBA_LO                                                                             0x031c
+#define regSQ_SHADER_TBA_LO_BASE_IDX                                                                    0
+#define regSQ_SHADER_TBA_HI                                                                             0x031d
+#define regSQ_SHADER_TBA_HI_BASE_IDX                                                                    0
+#define regSQ_SHADER_TMA_LO                                                                             0x031e
+#define regSQ_SHADER_TMA_LO_BASE_IDX                                                                    0
+#define regSQ_SHADER_TMA_HI                                                                             0x031f
+#define regSQ_SHADER_TMA_HI_BASE_IDX                                                                    0
+#define regSQC_DSM_CNTL                                                                                 0x0320
+#define regSQC_DSM_CNTL_BASE_IDX                                                                        0
+#define regSQC_DSM_CNTLA                                                                                0x0321
+#define regSQC_DSM_CNTLA_BASE_IDX                                                                       0
+#define regSQC_DSM_CNTLB                                                                                0x0322
+#define regSQC_DSM_CNTLB_BASE_IDX                                                                       0
+#define regSQC_DSM_CNTL2                                                                                0x0325
+#define regSQC_DSM_CNTL2_BASE_IDX                                                                       0
+#define regSQC_DSM_CNTL2A                                                                               0x0326
+#define regSQC_DSM_CNTL2A_BASE_IDX                                                                      0
+#define regSQC_DSM_CNTL2B                                                                               0x0327
+#define regSQC_DSM_CNTL2B_BASE_IDX                                                                      0
+#define regSQC_DSM_CNTL2E                                                                               0x032a
+#define regSQC_DSM_CNTL2E_BASE_IDX                                                                      0
+#define regSQC_EDC_FUE_CNTL                                                                             0x032b
+#define regSQC_EDC_FUE_CNTL_BASE_IDX                                                                    0
+#define regSQC_EDC_CNT2                                                                                 0x032c
+#define regSQC_EDC_CNT2_BASE_IDX                                                                        0
+#define regSQC_EDC_CNT3                                                                                 0x032d
+#define regSQC_EDC_CNT3_BASE_IDX                                                                        0
+#define regSQC_EDC_PARITY_CNT3                                                                          0x032e
+#define regSQC_EDC_PARITY_CNT3_BASE_IDX                                                                 0
+#define regSQ_DEBUG                                                                                     0x0332
+#define regSQ_DEBUG_BASE_IDX                                                                            0
+#define regSQ_PERF_SNAPSHOT_CTRL                                                                        0x0334
+#define regSQ_PERF_SNAPSHOT_CTRL_BASE_IDX                                                               0
+#define regSQ_DEBUG_FOR_INTERNAL_CTRL                                                                   0x0335
+#define regSQ_DEBUG_FOR_INTERNAL_CTRL_BASE_IDX                                                          0
+#define regSQ_REG_TIMESTAMP                                                                             0x0374
+#define regSQ_REG_TIMESTAMP_BASE_IDX                                                                    0
+#define regSQ_CMD_TIMESTAMP                                                                             0x0375
+#define regSQ_CMD_TIMESTAMP_BASE_IDX                                                                    0
+#define regSQ_HOSTTRAP_STATUS                                                                           0x0376
+#define regSQ_HOSTTRAP_STATUS_BASE_IDX                                                                  0
+#define regSQ_IND_INDEX                                                                                 0x0378
+#define regSQ_IND_INDEX_BASE_IDX                                                                        0
+#define regSQ_IND_DATA                                                                                  0x0379
+#define regSQ_IND_DATA_BASE_IDX                                                                         0
+#define regSQ_CONFIG1                                                                                   0x037a
+#define regSQ_CONFIG1_BASE_IDX                                                                          0
+#define regSQ_CMD                                                                                       0x037b
+#define regSQ_CMD_BASE_IDX                                                                              0
+#define regSQ_TIME_HI                                                                                   0x037c
+#define regSQ_TIME_HI_BASE_IDX                                                                          0
+#define regSQ_TIME_LO                                                                                   0x037d
+#define regSQ_TIME_LO_BASE_IDX                                                                          0
+#define regSQ_DS_0                                                                                      0x037f
+#define regSQ_DS_0_BASE_IDX                                                                             0
+#define regSQ_DS_1                                                                                      0x037f
+#define regSQ_DS_1_BASE_IDX                                                                             0
+#define regSQ_EXP_0                                                                                     0x037f
+#define regSQ_EXP_0_BASE_IDX                                                                            0
+#define regSQ_EXP_1                                                                                     0x037f
+#define regSQ_EXP_1_BASE_IDX                                                                            0
+#define regSQ_FLAT_0                                                                                    0x037f
+#define regSQ_FLAT_0_BASE_IDX                                                                           0
+#define regSQ_FLAT_1                                                                                    0x037f
+#define regSQ_FLAT_1_BASE_IDX                                                                           0
+#define regSQ_GLBL_0                                                                                    0x037f
+#define regSQ_GLBL_0_BASE_IDX                                                                           0
+#define regSQ_GLBL_1                                                                                    0x037f
+#define regSQ_GLBL_1_BASE_IDX                                                                           0
+#define regSQ_INST                                                                                      0x037f
+#define regSQ_INST_BASE_IDX                                                                             0
+#define regSQ_MIMG_0                                                                                    0x037f
+#define regSQ_MIMG_0_BASE_IDX                                                                           0
+#define regSQ_MIMG_1                                                                                    0x037f
+#define regSQ_MIMG_1_BASE_IDX                                                                           0
+#define regSQ_MTBUF_0                                                                                   0x037f
+#define regSQ_MTBUF_0_BASE_IDX                                                                          0
+#define regSQ_MTBUF_1                                                                                   0x037f
+#define regSQ_MTBUF_1_BASE_IDX                                                                          0
+#define regSQ_MUBUF_0                                                                                   0x037f
+#define regSQ_MUBUF_0_BASE_IDX                                                                          0
+#define regSQ_MUBUF_1                                                                                   0x037f
+#define regSQ_MUBUF_1_BASE_IDX                                                                          0
+#define regSQ_SCRATCH_0                                                                                 0x037f
+#define regSQ_SCRATCH_0_BASE_IDX                                                                        0
+#define regSQ_SCRATCH_1                                                                                 0x037f
+#define regSQ_SCRATCH_1_BASE_IDX                                                                        0
+#define regSQ_SMEM_0                                                                                    0x037f
+#define regSQ_SMEM_0_BASE_IDX                                                                           0
+#define regSQ_SMEM_1                                                                                    0x037f
+#define regSQ_SMEM_1_BASE_IDX                                                                           0
+#define regSQ_SOP1                                                                                      0x037f
+#define regSQ_SOP1_BASE_IDX                                                                             0
+#define regSQ_SOP2                                                                                      0x037f
+#define regSQ_SOP2_BASE_IDX                                                                             0
+#define regSQ_SOPC                                                                                      0x037f
+#define regSQ_SOPC_BASE_IDX                                                                             0
+#define regSQ_SOPK                                                                                      0x037f
+#define regSQ_SOPK_BASE_IDX                                                                             0
+#define regSQ_SOPP                                                                                      0x037f
+#define regSQ_SOPP_BASE_IDX                                                                             0
+#define regSQ_VINTRP                                                                                    0x037f
+#define regSQ_VINTRP_BASE_IDX                                                                           0
+#define regSQ_VOP1                                                                                      0x037f
+#define regSQ_VOP1_BASE_IDX                                                                             0
+#define regSQ_VOP2                                                                                      0x037f
+#define regSQ_VOP2_BASE_IDX                                                                             0
+#define regSQ_VOP3P_0                                                                                   0x037f
+#define regSQ_VOP3P_0_BASE_IDX                                                                          0
+#define regSQ_VOP3P_1                                                                                   0x037f
+#define regSQ_VOP3P_1_BASE_IDX                                                                          0
+#define regSQ_VOP3P_MFMA_0                                                                              0x037f
+#define regSQ_VOP3P_MFMA_0_BASE_IDX                                                                     0
+#define regSQ_VOP3P_MFMA_1                                                                              0x037f
+#define regSQ_VOP3P_MFMA_1_BASE_IDX                                                                     0
+#define regSQ_VOP3_0                                                                                    0x037f
+#define regSQ_VOP3_0_BASE_IDX                                                                           0
+#define regSQ_VOP3_0_SDST_ENC                                                                           0x037f
+#define regSQ_VOP3_0_SDST_ENC_BASE_IDX                                                                  0
+#define regSQ_VOP3_1                                                                                    0x037f
+#define regSQ_VOP3_1_BASE_IDX                                                                           0
+#define regSQ_VOPC                                                                                      0x037f
+#define regSQ_VOPC_BASE_IDX                                                                             0
+#define regSQ_VOP_DPP                                                                                   0x037f
+#define regSQ_VOP_DPP_BASE_IDX                                                                          0
+#define regSQ_VOP_SDWA                                                                                  0x037f
+#define regSQ_VOP_SDWA_BASE_IDX                                                                         0
+#define regSQ_VOP_SDWA_SDST_ENC                                                                         0x037f
+#define regSQ_VOP_SDWA_SDST_ENC_BASE_IDX                                                                0
+#define regSQ_LB_CTR_CTRL                                                                               0x0398
+#define regSQ_LB_CTR_CTRL_BASE_IDX                                                                      0
+#define regSQ_LB_DATA0                                                                                  0x0399
+#define regSQ_LB_DATA0_BASE_IDX                                                                         0
+#define regSQ_LB_DATA1                                                                                  0x039a
+#define regSQ_LB_DATA1_BASE_IDX                                                                         0
+#define regSQ_LB_DATA2                                                                                  0x039b
+#define regSQ_LB_DATA2_BASE_IDX                                                                         0
+#define regSQ_LB_DATA3                                                                                  0x039c
+#define regSQ_LB_DATA3_BASE_IDX                                                                         0
+#define regSQ_LB_CTR_SEL                                                                                0x039d
+#define regSQ_LB_CTR_SEL_BASE_IDX                                                                       0
+#define regSQ_LB_CTR0_CU                                                                                0x039e
+#define regSQ_LB_CTR0_CU_BASE_IDX                                                                       0
+#define regSQ_LB_CTR1_CU                                                                                0x039f
+#define regSQ_LB_CTR1_CU_BASE_IDX                                                                       0
+#define regSQ_LB_CTR2_CU                                                                                0x03a0
+#define regSQ_LB_CTR2_CU_BASE_IDX                                                                       0
+#define regSQ_LB_CTR3_CU                                                                                0x03a1
+#define regSQ_LB_CTR3_CU_BASE_IDX                                                                       0
+#define regSQC_EDC_CNT                                                                                  0x03a2
+#define regSQC_EDC_CNT_BASE_IDX                                                                         0
+#define regSQ_EDC_SEC_CNT                                                                               0x03a3
+#define regSQ_EDC_SEC_CNT_BASE_IDX                                                                      0
+#define regSQ_EDC_DED_CNT                                                                               0x03a4
+#define regSQ_EDC_DED_CNT_BASE_IDX                                                                      0
+#define regSQ_EDC_INFO                                                                                  0x03a5
+#define regSQ_EDC_INFO_BASE_IDX                                                                         0
+#define regSQ_EDC_CNT                                                                                   0x03a6
+#define regSQ_EDC_CNT_BASE_IDX                                                                          0
+#define regSQ_EDC_FUE_CNTL                                                                              0x03a7
+#define regSQ_EDC_FUE_CNTL_BASE_IDX                                                                     0
+#define regSQ_THREAD_TRACE_WORD_CMN                                                                     0x03b0
+#define regSQ_THREAD_TRACE_WORD_CMN_BASE_IDX                                                            0
+#define regSQ_THREAD_TRACE_WORD_EVENT                                                                   0x03b0
+#define regSQ_THREAD_TRACE_WORD_EVENT_BASE_IDX                                                          0
+#define regSQ_THREAD_TRACE_WORD_INST                                                                    0x03b0
+#define regSQ_THREAD_TRACE_WORD_INST_BASE_IDX                                                           0
+#define regSQ_THREAD_TRACE_WORD_INST_PC_1_OF_2                                                          0x03b0
+#define regSQ_THREAD_TRACE_WORD_INST_PC_1_OF_2_BASE_IDX                                                 0
+#define regSQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2                                                    0x03b0
+#define regSQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2_BASE_IDX                                           0
+#define regSQ_THREAD_TRACE_WORD_ISSUE                                                                   0x03b0
+#define regSQ_THREAD_TRACE_WORD_ISSUE_BASE_IDX                                                          0
+#define regSQ_THREAD_TRACE_WORD_MISC                                                                    0x03b0
+#define regSQ_THREAD_TRACE_WORD_MISC_BASE_IDX                                                           0
+#define regSQ_THREAD_TRACE_WORD_PERF_1_OF_2                                                             0x03b0
+#define regSQ_THREAD_TRACE_WORD_PERF_1_OF_2_BASE_IDX                                                    0
+#define regSQ_THREAD_TRACE_WORD_REG_1_OF_2                                                              0x03b0
+#define regSQ_THREAD_TRACE_WORD_REG_1_OF_2_BASE_IDX                                                     0
+#define regSQ_THREAD_TRACE_WORD_REG_2_OF_2                                                              0x03b0
+#define regSQ_THREAD_TRACE_WORD_REG_2_OF_2_BASE_IDX                                                     0
+#define regSQ_THREAD_TRACE_WORD_REG_CS_1_OF_2                                                           0x03b0
+#define regSQ_THREAD_TRACE_WORD_REG_CS_1_OF_2_BASE_IDX                                                  0
+#define regSQ_THREAD_TRACE_WORD_REG_CS_2_OF_2                                                           0x03b0
+#define regSQ_THREAD_TRACE_WORD_REG_CS_2_OF_2_BASE_IDX                                                  0
+#define regSQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2                                                        0x03b0
+#define regSQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2_BASE_IDX                                               0
+#define regSQ_THREAD_TRACE_WORD_WAVE                                                                    0x03b0
+#define regSQ_THREAD_TRACE_WORD_WAVE_BASE_IDX                                                           0
+#define regSQ_THREAD_TRACE_WORD_WAVE_START                                                              0x03b0
+#define regSQ_THREAD_TRACE_WORD_WAVE_START_BASE_IDX                                                     0
+#define regSQ_THREAD_TRACE_WORD_INST_PC_2_OF_2                                                          0x03b1
+#define regSQ_THREAD_TRACE_WORD_INST_PC_2_OF_2_BASE_IDX                                                 0
+#define regSQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2                                                    0x03b1
+#define regSQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2_BASE_IDX                                           0
+#define regSQ_THREAD_TRACE_WORD_PERF_2_OF_2                                                             0x03b1
+#define regSQ_THREAD_TRACE_WORD_PERF_2_OF_2_BASE_IDX                                                    0
+#define regSQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2                                                        0x03b1
+#define regSQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2_BASE_IDX                                               0
+#define regSQ_WREXEC_EXEC_HI                                                                            0x03b1
+#define regSQ_WREXEC_EXEC_HI_BASE_IDX                                                                   0
+#define regSQ_WREXEC_EXEC_LO                                                                            0x03b1
+#define regSQ_WREXEC_EXEC_LO_BASE_IDX                                                                   0
+#define regSQ_BUF_RSRC_WORD0                                                                            0x03c0
+#define regSQ_BUF_RSRC_WORD0_BASE_IDX                                                                   0
+#define regSQ_BUF_RSRC_WORD1                                                                            0x03c1
+#define regSQ_BUF_RSRC_WORD1_BASE_IDX                                                                   0
+#define regSQ_BUF_RSRC_WORD2                                                                            0x03c2
+#define regSQ_BUF_RSRC_WORD2_BASE_IDX                                                                   0
+#define regSQ_BUF_RSRC_WORD3                                                                            0x03c3
+#define regSQ_BUF_RSRC_WORD3_BASE_IDX                                                                   0
+#define regSQ_IMG_RSRC_WORD0                                                                            0x03c4
+#define regSQ_IMG_RSRC_WORD0_BASE_IDX                                                                   0
+#define regSQ_IMG_RSRC_WORD1                                                                            0x03c5
+#define regSQ_IMG_RSRC_WORD1_BASE_IDX                                                                   0
+#define regSQ_IMG_RSRC_WORD2                                                                            0x03c6
+#define regSQ_IMG_RSRC_WORD2_BASE_IDX                                                                   0
+#define regSQ_IMG_RSRC_WORD3                                                                            0x03c7
+#define regSQ_IMG_RSRC_WORD3_BASE_IDX                                                                   0
+#define regSQ_IMG_RSRC_WORD4                                                                            0x03c8
+#define regSQ_IMG_RSRC_WORD4_BASE_IDX                                                                   0
+#define regSQ_IMG_RSRC_WORD5                                                                            0x03c9
+#define regSQ_IMG_RSRC_WORD5_BASE_IDX                                                                   0
+#define regSQ_IMG_RSRC_WORD6                                                                            0x03ca
+#define regSQ_IMG_RSRC_WORD6_BASE_IDX                                                                   0
+#define regSQ_IMG_RSRC_WORD7                                                                            0x03cb
+#define regSQ_IMG_RSRC_WORD7_BASE_IDX                                                                   0
+#define regSQ_IMG_SAMP_WORD0                                                                            0x03cc
+#define regSQ_IMG_SAMP_WORD0_BASE_IDX                                                                   0
+#define regSQ_IMG_SAMP_WORD1                                                                            0x03cd
+#define regSQ_IMG_SAMP_WORD1_BASE_IDX                                                                   0
+#define regSQ_IMG_SAMP_WORD2                                                                            0x03ce
+#define regSQ_IMG_SAMP_WORD2_BASE_IDX                                                                   0
+#define regSQ_IMG_SAMP_WORD3                                                                            0x03cf
+#define regSQ_IMG_SAMP_WORD3_BASE_IDX                                                                   0
+#define regSQ_FLAT_SCRATCH_WORD0                                                                        0x03d0
+#define regSQ_FLAT_SCRATCH_WORD0_BASE_IDX                                                               0
+#define regSQ_FLAT_SCRATCH_WORD1                                                                        0x03d1
+#define regSQ_FLAT_SCRATCH_WORD1_BASE_IDX                                                               0
+#define regSQ_M0_GPR_IDX_WORD                                                                           0x03d2
+#define regSQ_M0_GPR_IDX_WORD_BASE_IDX                                                                  0
+#define regSQC_ICACHE_UTCL1_CNTL1                                                                       0x03d3
+#define regSQC_ICACHE_UTCL1_CNTL1_BASE_IDX                                                              0
+#define regSQC_ICACHE_UTCL1_CNTL2                                                                       0x03d4
+#define regSQC_ICACHE_UTCL1_CNTL2_BASE_IDX                                                              0
+#define regSQC_DCACHE_UTCL1_CNTL1                                                                       0x03d5
+#define regSQC_DCACHE_UTCL1_CNTL1_BASE_IDX                                                              0
+#define regSQC_DCACHE_UTCL1_CNTL2                                                                       0x03d6
+#define regSQC_DCACHE_UTCL1_CNTL2_BASE_IDX                                                              0
+#define regSQC_ICACHE_UTCL1_STATUS                                                                      0x03d7
+#define regSQC_ICACHE_UTCL1_STATUS_BASE_IDX                                                             0
+#define regSQC_DCACHE_UTCL1_STATUS                                                                      0x03d8
+#define regSQC_DCACHE_UTCL1_STATUS_BASE_IDX                                                             0
+
+
+// addressBlock: xcd0_gc_shsdec
+// base address: 0x9000
+#define regSX_DEBUG_BUSY                                                                                0x0414
+#define regSX_DEBUG_BUSY_BASE_IDX                                                                       0
+#define regSX_DEBUG_1                                                                                   0x0419
+#define regSX_DEBUG_1_BASE_IDX                                                                          0
+#define regSPI_PS_MAX_WAVE_ID                                                                           0x043a
+#define regSPI_PS_MAX_WAVE_ID_BASE_IDX                                                                  0
+#define regSPI_START_PHASE                                                                              0x043b
+#define regSPI_START_PHASE_BASE_IDX                                                                     0
+#define regSPI_GFX_CNTL                                                                                 0x043c
+#define regSPI_GFX_CNTL_BASE_IDX                                                                        0
+#define regSPI_DEBUG_READ                                                                               0x0442
+#define regSPI_DEBUG_READ_BASE_IDX                                                                      0
+#define regSPI_DSM_CNTL                                                                                 0x0443
+#define regSPI_DSM_CNTL_BASE_IDX                                                                        0
+#define regSPI_DSM_CNTL2                                                                                0x0444
+#define regSPI_DSM_CNTL2_BASE_IDX                                                                       0
+#define regSPI_EDC_CNT                                                                                  0x0445
+#define regSPI_EDC_CNT_BASE_IDX                                                                         0
+#define regSPI_DEBUG_BUSY                                                                               0x0450
+#define regSPI_DEBUG_BUSY_BASE_IDX                                                                      0
+#define regSPI_CONFIG_PS_CU_EN                                                                          0x0452
+#define regSPI_CONFIG_PS_CU_EN_BASE_IDX                                                                 0
+#define regSPI_WF_LIFETIME_CNTL                                                                         0x04aa
+#define regSPI_WF_LIFETIME_CNTL_BASE_IDX                                                                0
+#define regSPI_WF_LIFETIME_LIMIT_0                                                                      0x04ab
+#define regSPI_WF_LIFETIME_LIMIT_0_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_1                                                                      0x04ac
+#define regSPI_WF_LIFETIME_LIMIT_1_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_2                                                                      0x04ad
+#define regSPI_WF_LIFETIME_LIMIT_2_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_3                                                                      0x04ae
+#define regSPI_WF_LIFETIME_LIMIT_3_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_4                                                                      0x04af
+#define regSPI_WF_LIFETIME_LIMIT_4_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_5                                                                      0x04b0
+#define regSPI_WF_LIFETIME_LIMIT_5_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_6                                                                      0x04b1
+#define regSPI_WF_LIFETIME_LIMIT_6_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_7                                                                      0x04b2
+#define regSPI_WF_LIFETIME_LIMIT_7_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_8                                                                      0x04b3
+#define regSPI_WF_LIFETIME_LIMIT_8_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_9                                                                      0x04b4
+#define regSPI_WF_LIFETIME_LIMIT_9_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_STATUS_0                                                                     0x04b5
+#define regSPI_WF_LIFETIME_STATUS_0_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_1                                                                     0x04b6
+#define regSPI_WF_LIFETIME_STATUS_1_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_2                                                                     0x04b7
+#define regSPI_WF_LIFETIME_STATUS_2_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_3                                                                     0x04b8
+#define regSPI_WF_LIFETIME_STATUS_3_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_4                                                                     0x04b9
+#define regSPI_WF_LIFETIME_STATUS_4_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_5                                                                     0x04ba
+#define regSPI_WF_LIFETIME_STATUS_5_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_6                                                                     0x04bb
+#define regSPI_WF_LIFETIME_STATUS_6_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_7                                                                     0x04bc
+#define regSPI_WF_LIFETIME_STATUS_7_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_8                                                                     0x04bd
+#define regSPI_WF_LIFETIME_STATUS_8_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_9                                                                     0x04be
+#define regSPI_WF_LIFETIME_STATUS_9_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_10                                                                    0x04bf
+#define regSPI_WF_LIFETIME_STATUS_10_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_11                                                                    0x04c0
+#define regSPI_WF_LIFETIME_STATUS_11_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_12                                                                    0x04c1
+#define regSPI_WF_LIFETIME_STATUS_12_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_13                                                                    0x04c2
+#define regSPI_WF_LIFETIME_STATUS_13_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_14                                                                    0x04c3
+#define regSPI_WF_LIFETIME_STATUS_14_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_15                                                                    0x04c4
+#define regSPI_WF_LIFETIME_STATUS_15_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_16                                                                    0x04c5
+#define regSPI_WF_LIFETIME_STATUS_16_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_17                                                                    0x04c6
+#define regSPI_WF_LIFETIME_STATUS_17_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_18                                                                    0x04c7
+#define regSPI_WF_LIFETIME_STATUS_18_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_19                                                                    0x04c8
+#define regSPI_WF_LIFETIME_STATUS_19_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_20                                                                    0x04c9
+#define regSPI_WF_LIFETIME_STATUS_20_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_DEBUG                                                                        0x04ca
+#define regSPI_WF_LIFETIME_DEBUG_BASE_IDX                                                               0
+#define regSPI_LB_CTR_CTRL                                                                              0x04d4
+#define regSPI_LB_CTR_CTRL_BASE_IDX                                                                     0
+#define regSPI_LB_CU_MASK                                                                               0x04d5
+#define regSPI_LB_CU_MASK_BASE_IDX                                                                      0
+#define regSPI_LB_DATA_REG                                                                              0x04d6
+#define regSPI_LB_DATA_REG_BASE_IDX                                                                     0
+#define regSPI_PG_ENABLE_STATIC_CU_MASK                                                                 0x04d7
+#define regSPI_PG_ENABLE_STATIC_CU_MASK_BASE_IDX                                                        0
+#define regSPI_GDS_CREDITS                                                                              0x04d8
+#define regSPI_GDS_CREDITS_BASE_IDX                                                                     0
+#define regSPI_SX_EXPORT_BUFFER_SIZES                                                                   0x04d9
+#define regSPI_SX_EXPORT_BUFFER_SIZES_BASE_IDX                                                          0
+#define regSPI_SX_SCOREBOARD_BUFFER_SIZES                                                               0x04da
+#define regSPI_SX_SCOREBOARD_BUFFER_SIZES_BASE_IDX                                                      0
+#define regSPI_CSQ_WF_ACTIVE_STATUS                                                                     0x04db
+#define regSPI_CSQ_WF_ACTIVE_STATUS_BASE_IDX                                                            0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_0                                                                    0x04dc
+#define regSPI_CSQ_WF_ACTIVE_COUNT_0_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_1                                                                    0x04dd
+#define regSPI_CSQ_WF_ACTIVE_COUNT_1_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_2                                                                    0x04de
+#define regSPI_CSQ_WF_ACTIVE_COUNT_2_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_3                                                                    0x04df
+#define regSPI_CSQ_WF_ACTIVE_COUNT_3_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_4                                                                    0x04e0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_4_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_5                                                                    0x04e1
+#define regSPI_CSQ_WF_ACTIVE_COUNT_5_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_6                                                                    0x04e2
+#define regSPI_CSQ_WF_ACTIVE_COUNT_6_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_7                                                                    0x04e3
+#define regSPI_CSQ_WF_ACTIVE_COUNT_7_BASE_IDX                                                           0
+#define regSPI_LB_DATA_WAVES                                                                            0x04e4
+#define regSPI_LB_DATA_WAVES_BASE_IDX                                                                   0
+#define regSPI_LB_DATA_PERCU_WAVE_HSGS                                                                  0x04e5
+#define regSPI_LB_DATA_PERCU_WAVE_HSGS_BASE_IDX                                                         0
+#define regSPI_LB_DATA_PERCU_WAVE_VSPS                                                                  0x04e6
+#define regSPI_LB_DATA_PERCU_WAVE_VSPS_BASE_IDX                                                         0
+#define regSPI_LB_DATA_PERCU_WAVE_CS                                                                    0x04e7
+#define regSPI_LB_DATA_PERCU_WAVE_CS_BASE_IDX                                                           0
+#define regSPIS_DEBUG_READ                                                                              0x04ea
+#define regSPIS_DEBUG_READ_BASE_IDX                                                                     0
+#define regBCI_DEBUG_READ                                                                               0x04eb
+#define regBCI_DEBUG_READ_BASE_IDX                                                                      0
+#define regSPI_P0_TRAP_SCREEN_PSBA_LO                                                                   0x04ec
+#define regSPI_P0_TRAP_SCREEN_PSBA_LO_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_PSBA_HI                                                                   0x04ed
+#define regSPI_P0_TRAP_SCREEN_PSBA_HI_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_PSMA_LO                                                                   0x04ee
+#define regSPI_P0_TRAP_SCREEN_PSMA_LO_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_PSMA_HI                                                                   0x04ef
+#define regSPI_P0_TRAP_SCREEN_PSMA_HI_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_GPR_MIN                                                                   0x04f0
+#define regSPI_P0_TRAP_SCREEN_GPR_MIN_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSBA_LO                                                                   0x04f1
+#define regSPI_P1_TRAP_SCREEN_PSBA_LO_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSBA_HI                                                                   0x04f2
+#define regSPI_P1_TRAP_SCREEN_PSBA_HI_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSMA_LO                                                                   0x04f3
+#define regSPI_P1_TRAP_SCREEN_PSMA_LO_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSMA_HI                                                                   0x04f4
+#define regSPI_P1_TRAP_SCREEN_PSMA_HI_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_GPR_MIN                                                                   0x04f5
+#define regSPI_P1_TRAP_SCREEN_GPR_MIN_BASE_IDX                                                          0
+
+
+// addressBlock: xcd0_gc_tpdec
+// base address: 0x9400
+#define regTD_CNTL                                                                                      0x0525
+#define regTD_CNTL_BASE_IDX                                                                             0
+#define regTD_STATUS                                                                                    0x0526
+#define regTD_STATUS_BASE_IDX                                                                           0
+#define regTD_POWER_CNTL                                                                                0x052a
+#define regTD_POWER_CNTL_BASE_IDX                                                                       0
+#define regTD_DSM_CNTL                                                                                  0x052f
+#define regTD_DSM_CNTL_BASE_IDX                                                                         0
+#define regTD_DSM_CNTL2                                                                                 0x0530
+#define regTD_DSM_CNTL2_BASE_IDX                                                                        0
+#define regTD_SCRATCH                                                                                   0x0533
+#define regTD_SCRATCH_BASE_IDX                                                                          0
+#define regTA_POWER_CNTL                                                                                0x0540
+#define regTA_POWER_CNTL_BASE_IDX                                                                       0
+#define regTA_CNTL                                                                                      0x0541
+#define regTA_CNTL_BASE_IDX                                                                             0
+#define regTA_CNTL_AUX                                                                                  0x0542
+#define regTA_CNTL_AUX_BASE_IDX                                                                         0
+#define regTA_FEATURE_CNTL                                                                              0x0543
+#define regTA_FEATURE_CNTL_BASE_IDX                                                                     0
+#define regTA_STATUS                                                                                    0x0548
+#define regTA_STATUS_BASE_IDX                                                                           0
+#define regTA_SCRATCH                                                                                   0x0564
+#define regTA_SCRATCH_BASE_IDX                                                                          0
+#define regTA_DSM_CNTL                                                                                  0x0584
+#define regTA_DSM_CNTL_BASE_IDX                                                                         0
+#define regTA_DSM_CNTL2                                                                                 0x0585
+#define regTA_DSM_CNTL2_BASE_IDX                                                                        0
+
+
+// addressBlock: xcd0_gc_gdsdec
+// base address: 0x9700
+#define regGDS_CONFIG                                                                                   0x05c0
+#define regGDS_CONFIG_BASE_IDX                                                                          0
+#define regGDS_CNTL_STATUS                                                                              0x05c1
+#define regGDS_CNTL_STATUS_BASE_IDX                                                                     0
+#define regGDS_ENHANCE2                                                                                 0x05c2
+#define regGDS_ENHANCE2_BASE_IDX                                                                        0
+#define regGDS_PROTECTION_FAULT                                                                         0x05c3
+#define regGDS_PROTECTION_FAULT_BASE_IDX                                                                0
+#define regGDS_VM_PROTECTION_FAULT                                                                      0x05c4
+#define regGDS_VM_PROTECTION_FAULT_BASE_IDX                                                             0
+#define regGDS_EDC_CNT                                                                                  0x05c5
+#define regGDS_EDC_CNT_BASE_IDX                                                                         0
+#define regGDS_EDC_GRBM_CNT                                                                             0x05c6
+#define regGDS_EDC_GRBM_CNT_BASE_IDX                                                                    0
+#define regGDS_EDC_OA_DED                                                                               0x05c7
+#define regGDS_EDC_OA_DED_BASE_IDX                                                                      0
+#define regGDS_DSM_CNTL                                                                                 0x05ca
+#define regGDS_DSM_CNTL_BASE_IDX                                                                        0
+#define regGDS_EDC_OA_PHY_CNT                                                                           0x05cb
+#define regGDS_EDC_OA_PHY_CNT_BASE_IDX                                                                  0
+#define regGDS_EDC_OA_PIPE_CNT                                                                          0x05cc
+#define regGDS_EDC_OA_PIPE_CNT_BASE_IDX                                                                 0
+#define regGDS_DSM_CNTL2                                                                                0x05cd
+#define regGDS_DSM_CNTL2_BASE_IDX                                                                       0
+#define regGDS_WD_GDS_CSB                                                                               0x05ce
+#define regGDS_WD_GDS_CSB_BASE_IDX                                                                      0
+
+
+// addressBlock: xcd0_gc_rbdec
+// base address: 0x9800
+#define regDB_DEBUG                                                                                     0x060c
+#define regDB_DEBUG_BASE_IDX                                                                            0
+#define regDB_DEBUG2                                                                                    0x060d
+#define regDB_DEBUG2_BASE_IDX                                                                           0
+#define regDB_DEBUG3                                                                                    0x060e
+#define regDB_DEBUG3_BASE_IDX                                                                           0
+#define regDB_DEBUG4                                                                                    0x060f
+#define regDB_DEBUG4_BASE_IDX                                                                           0
+#define regDB_CREDIT_LIMIT                                                                              0x0614
+#define regDB_CREDIT_LIMIT_BASE_IDX                                                                     0
+#define regDB_WATERMARKS                                                                                0x0615
+#define regDB_WATERMARKS_BASE_IDX                                                                       0
+#define regDB_SUBTILE_CONTROL                                                                           0x0616
+#define regDB_SUBTILE_CONTROL_BASE_IDX                                                                  0
+#define regDB_FREE_CACHELINES                                                                           0x0617
+#define regDB_FREE_CACHELINES_BASE_IDX                                                                  0
+#define regDB_FIFO_DEPTH1                                                                               0x0618
+#define regDB_FIFO_DEPTH1_BASE_IDX                                                                      0
+#define regDB_FIFO_DEPTH2                                                                               0x0619
+#define regDB_FIFO_DEPTH2_BASE_IDX                                                                      0
+#define regDB_EXCEPTION_CONTROL                                                                         0x061a
+#define regDB_EXCEPTION_CONTROL_BASE_IDX                                                                0
+#define regDB_RING_CONTROL                                                                              0x061b
+#define regDB_RING_CONTROL_BASE_IDX                                                                     0
+#define regDB_MEM_ARB_WATERMARKS                                                                        0x061c
+#define regDB_MEM_ARB_WATERMARKS_BASE_IDX                                                               0
+#define regDB_RMI_CACHE_POLICY                                                                          0x061e
+#define regDB_RMI_CACHE_POLICY_BASE_IDX                                                                 0
+#define regDB_DFSM_CONFIG                                                                               0x0630
+#define regDB_DFSM_CONFIG_BASE_IDX                                                                      0
+#define regDB_DFSM_WATERMARK                                                                            0x0631
+#define regDB_DFSM_WATERMARK_BASE_IDX                                                                   0
+#define regDB_DFSM_TILES_IN_FLIGHT                                                                      0x0632
+#define regDB_DFSM_TILES_IN_FLIGHT_BASE_IDX                                                             0
+#define regDB_DFSM_PRIMS_IN_FLIGHT                                                                      0x0633
+#define regDB_DFSM_PRIMS_IN_FLIGHT_BASE_IDX                                                             0
+#define regDB_DFSM_WATCHDOG                                                                             0x0634
+#define regDB_DFSM_WATCHDOG_BASE_IDX                                                                    0
+#define regDB_DFSM_FLUSH_ENABLE                                                                         0x0635
+#define regDB_DFSM_FLUSH_ENABLE_BASE_IDX                                                                0
+#define regDB_DFSM_FLUSH_AUX_EVENT                                                                      0x0636
+#define regDB_DFSM_FLUSH_AUX_EVENT_BASE_IDX                                                             0
+#define regCC_RB_REDUNDANCY                                                                             0x063c
+#define regCC_RB_REDUNDANCY_BASE_IDX                                                                    0
+#define regCC_RB_BACKEND_DISABLE                                                                        0x063d
+#define regCC_RB_BACKEND_DISABLE_BASE_IDX                                                               0
+#define regGB_ADDR_CONFIG                                                                               0x063e
+#define regGB_ADDR_CONFIG_BASE_IDX                                                                      0
+#define regGB_BACKEND_MAP                                                                               0x063f
+#define regGB_BACKEND_MAP_BASE_IDX                                                                      0
+#define regGB_GPU_ID                                                                                    0x0640
+#define regGB_GPU_ID_BASE_IDX                                                                           0
+#define regCC_RB_DAISY_CHAIN                                                                            0x0641
+#define regCC_RB_DAISY_CHAIN_BASE_IDX                                                                   0
+#define regGB_ADDR_CONFIG_READ                                                                          0x0642
+#define regGB_ADDR_CONFIG_READ_BASE_IDX                                                                 0
+#define regGB_TILE_MODE0                                                                                0x0644
+#define regGB_TILE_MODE0_BASE_IDX                                                                       0
+#define regGB_TILE_MODE1                                                                                0x0645
+#define regGB_TILE_MODE1_BASE_IDX                                                                       0
+#define regGB_TILE_MODE2                                                                                0x0646
+#define regGB_TILE_MODE2_BASE_IDX                                                                       0
+#define regGB_TILE_MODE3                                                                                0x0647
+#define regGB_TILE_MODE3_BASE_IDX                                                                       0
+#define regGB_TILE_MODE4                                                                                0x0648
+#define regGB_TILE_MODE4_BASE_IDX                                                                       0
+#define regGB_TILE_MODE5                                                                                0x0649
+#define regGB_TILE_MODE5_BASE_IDX                                                                       0
+#define regGB_TILE_MODE6                                                                                0x064a
+#define regGB_TILE_MODE6_BASE_IDX                                                                       0
+#define regGB_TILE_MODE7                                                                                0x064b
+#define regGB_TILE_MODE7_BASE_IDX                                                                       0
+#define regGB_TILE_MODE8                                                                                0x064c
+#define regGB_TILE_MODE8_BASE_IDX                                                                       0
+#define regGB_TILE_MODE9                                                                                0x064d
+#define regGB_TILE_MODE9_BASE_IDX                                                                       0
+#define regGB_TILE_MODE10                                                                               0x064e
+#define regGB_TILE_MODE10_BASE_IDX                                                                      0
+#define regGB_TILE_MODE11                                                                               0x064f
+#define regGB_TILE_MODE11_BASE_IDX                                                                      0
+#define regGB_TILE_MODE12                                                                               0x0650
+#define regGB_TILE_MODE12_BASE_IDX                                                                      0
+#define regGB_TILE_MODE13                                                                               0x0651
+#define regGB_TILE_MODE13_BASE_IDX                                                                      0
+#define regGB_TILE_MODE14                                                                               0x0652
+#define regGB_TILE_MODE14_BASE_IDX                                                                      0
+#define regGB_TILE_MODE15                                                                               0x0653
+#define regGB_TILE_MODE15_BASE_IDX                                                                      0
+#define regGB_TILE_MODE16                                                                               0x0654
+#define regGB_TILE_MODE16_BASE_IDX                                                                      0
+#define regGB_TILE_MODE17                                                                               0x0655
+#define regGB_TILE_MODE17_BASE_IDX                                                                      0
+#define regGB_TILE_MODE18                                                                               0x0656
+#define regGB_TILE_MODE18_BASE_IDX                                                                      0
+#define regGB_TILE_MODE19                                                                               0x0657
+#define regGB_TILE_MODE19_BASE_IDX                                                                      0
+#define regGB_TILE_MODE20                                                                               0x0658
+#define regGB_TILE_MODE20_BASE_IDX                                                                      0
+#define regGB_TILE_MODE21                                                                               0x0659
+#define regGB_TILE_MODE21_BASE_IDX                                                                      0
+#define regGB_TILE_MODE22                                                                               0x065a
+#define regGB_TILE_MODE22_BASE_IDX                                                                      0
+#define regGB_TILE_MODE23                                                                               0x065b
+#define regGB_TILE_MODE23_BASE_IDX                                                                      0
+#define regGB_TILE_MODE24                                                                               0x065c
+#define regGB_TILE_MODE24_BASE_IDX                                                                      0
+#define regGB_TILE_MODE25                                                                               0x065d
+#define regGB_TILE_MODE25_BASE_IDX                                                                      0
+#define regGB_TILE_MODE26                                                                               0x065e
+#define regGB_TILE_MODE26_BASE_IDX                                                                      0
+#define regGB_TILE_MODE27                                                                               0x065f
+#define regGB_TILE_MODE27_BASE_IDX                                                                      0
+#define regGB_TILE_MODE28                                                                               0x0660
+#define regGB_TILE_MODE28_BASE_IDX                                                                      0
+#define regGB_TILE_MODE29                                                                               0x0661
+#define regGB_TILE_MODE29_BASE_IDX                                                                      0
+#define regGB_TILE_MODE30                                                                               0x0662
+#define regGB_TILE_MODE30_BASE_IDX                                                                      0
+#define regGB_TILE_MODE31                                                                               0x0663
+#define regGB_TILE_MODE31_BASE_IDX                                                                      0
+#define regGB_MACROTILE_MODE0                                                                           0x0664
+#define regGB_MACROTILE_MODE0_BASE_IDX                                                                  0
+#define regGB_MACROTILE_MODE1                                                                           0x0665
+#define regGB_MACROTILE_MODE1_BASE_IDX                                                                  0
+#define regGB_MACROTILE_MODE2                                                                           0x0666
+#define regGB_MACROTILE_MODE2_BASE_IDX                                                                  0
+#define regGB_MACROTILE_MODE3                                                                           0x0667
+#define regGB_MACROTILE_MODE3_BASE_IDX                                                                  0
+#define regGB_MACROTILE_MODE4                                                                           0x0668
+#define regGB_MACROTILE_MODE4_BASE_IDX                                                                  0
+#define regGB_MACROTILE_MODE5                                                                           0x0669
+#define regGB_MACROTILE_MODE5_BASE_IDX                                                                  0
+#define regGB_MACROTILE_MODE6                                                                           0x066a
+#define regGB_MACROTILE_MODE6_BASE_IDX                                                                  0
+#define regGB_MACROTILE_MODE7                                                                           0x066b
+#define regGB_MACROTILE_MODE7_BASE_IDX                                                                  0
+#define regGB_MACROTILE_MODE8                                                                           0x066c
+#define regGB_MACROTILE_MODE8_BASE_IDX                                                                  0
+#define regGB_MACROTILE_MODE9                                                                           0x066d
+#define regGB_MACROTILE_MODE9_BASE_IDX                                                                  0
+#define regGB_MACROTILE_MODE10                                                                          0x066e
+#define regGB_MACROTILE_MODE10_BASE_IDX                                                                 0
+#define regGB_MACROTILE_MODE11                                                                          0x066f
+#define regGB_MACROTILE_MODE11_BASE_IDX                                                                 0
+#define regGB_MACROTILE_MODE12                                                                          0x0670
+#define regGB_MACROTILE_MODE12_BASE_IDX                                                                 0
+#define regGB_MACROTILE_MODE13                                                                          0x0671
+#define regGB_MACROTILE_MODE13_BASE_IDX                                                                 0
+#define regGB_MACROTILE_MODE14                                                                          0x0672
+#define regGB_MACROTILE_MODE14_BASE_IDX                                                                 0
+#define regGB_MACROTILE_MODE15                                                                          0x0673
+#define regGB_MACROTILE_MODE15_BASE_IDX                                                                 0
+#define regCB_HW_CONTROL                                                                                0x0680
+#define regCB_HW_CONTROL_BASE_IDX                                                                       0
+#define regCB_HW_CONTROL_1                                                                              0x0681
+#define regCB_HW_CONTROL_1_BASE_IDX                                                                     0
+#define regCB_HW_CONTROL_2                                                                              0x0682
+#define regCB_HW_CONTROL_2_BASE_IDX                                                                     0
+#define regCB_HW_CONTROL_3                                                                              0x0683
+#define regCB_HW_CONTROL_3_BASE_IDX                                                                     0
+#define regCB_HW_MEM_ARBITER_RD                                                                         0x0686
+#define regCB_HW_MEM_ARBITER_RD_BASE_IDX                                                                0
+#define regCB_HW_MEM_ARBITER_WR                                                                         0x0687
+#define regCB_HW_MEM_ARBITER_WR_BASE_IDX                                                                0
+#define regCB_DCC_CONFIG                                                                                0x0688
+#define regCB_DCC_CONFIG_BASE_IDX                                                                       0
+#define regGC_USER_RB_REDUNDANCY                                                                        0x06de
+#define regGC_USER_RB_REDUNDANCY_BASE_IDX                                                               0
+#define regGC_USER_RB_BACKEND_DISABLE                                                                   0x06df
+#define regGC_USER_RB_BACKEND_DISABLE_BASE_IDX                                                          0
+
+
+// addressBlock: xcd0_gc_ea_gceadec
+// base address: 0xa800
+#define regGCEA_DRAM_RD_CLI2GRP_MAP0                                                                    0x0a00
+#define regGCEA_DRAM_RD_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regGCEA_DRAM_RD_CLI2GRP_MAP1                                                                    0x0a01
+#define regGCEA_DRAM_RD_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regGCEA_DRAM_WR_CLI2GRP_MAP0                                                                    0x0a02
+#define regGCEA_DRAM_WR_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regGCEA_DRAM_WR_CLI2GRP_MAP1                                                                    0x0a03
+#define regGCEA_DRAM_WR_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regGCEA_DRAM_RD_GRP2VC_MAP                                                                      0x0a04
+#define regGCEA_DRAM_RD_GRP2VC_MAP_BASE_IDX                                                             0
+#define regGCEA_DRAM_WR_GRP2VC_MAP                                                                      0x0a05
+#define regGCEA_DRAM_WR_GRP2VC_MAP_BASE_IDX                                                             0
+#define regGCEA_DRAM_RD_LAZY                                                                            0x0a06
+#define regGCEA_DRAM_RD_LAZY_BASE_IDX                                                                   0
+#define regGCEA_DRAM_WR_LAZY                                                                            0x0a07
+#define regGCEA_DRAM_WR_LAZY_BASE_IDX                                                                   0
+#define regGCEA_DRAM_RD_CAM_CNTL                                                                        0x0a08
+#define regGCEA_DRAM_RD_CAM_CNTL_BASE_IDX                                                               0
+#define regGCEA_DRAM_WR_CAM_CNTL                                                                        0x0a09
+#define regGCEA_DRAM_WR_CAM_CNTL_BASE_IDX                                                               0
+#define regGCEA_DRAM_PAGE_BURST                                                                         0x0a0a
+#define regGCEA_DRAM_PAGE_BURST_BASE_IDX                                                                0
+#define regGCEA_DRAM_RD_PRI_AGE                                                                         0x0a0b
+#define regGCEA_DRAM_RD_PRI_AGE_BASE_IDX                                                                0
+#define regGCEA_DRAM_WR_PRI_AGE                                                                         0x0a0c
+#define regGCEA_DRAM_WR_PRI_AGE_BASE_IDX                                                                0
+#define regGCEA_DRAM_RD_PRI_QUEUING                                                                     0x0a0d
+#define regGCEA_DRAM_RD_PRI_QUEUING_BASE_IDX                                                            0
+#define regGCEA_DRAM_WR_PRI_QUEUING                                                                     0x0a0e
+#define regGCEA_DRAM_WR_PRI_QUEUING_BASE_IDX                                                            0
+#define regGCEA_DRAM_RD_PRI_FIXED                                                                       0x0a0f
+#define regGCEA_DRAM_RD_PRI_FIXED_BASE_IDX                                                              0
+#define regGCEA_DRAM_WR_PRI_FIXED                                                                       0x0a10
+#define regGCEA_DRAM_WR_PRI_FIXED_BASE_IDX                                                              0
+#define regGCEA_DRAM_RD_PRI_URGENCY                                                                     0x0a11
+#define regGCEA_DRAM_RD_PRI_URGENCY_BASE_IDX                                                            0
+#define regGCEA_DRAM_WR_PRI_URGENCY                                                                     0x0a12
+#define regGCEA_DRAM_WR_PRI_URGENCY_BASE_IDX                                                            0
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI1                                                                  0x0a13
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI2                                                                  0x0a14
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI3                                                                  0x0a15
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI1                                                                  0x0a16
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI2                                                                  0x0a17
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI3                                                                  0x0a18
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regGCEA_IO_RD_CLI2GRP_MAP0                                                                      0x0ad5
+#define regGCEA_IO_RD_CLI2GRP_MAP0_BASE_IDX                                                             0
+#define regGCEA_IO_RD_CLI2GRP_MAP1                                                                      0x0ad6
+#define regGCEA_IO_RD_CLI2GRP_MAP1_BASE_IDX                                                             0
+#define regGCEA_IO_WR_CLI2GRP_MAP0                                                                      0x0ad7
+#define regGCEA_IO_WR_CLI2GRP_MAP0_BASE_IDX                                                             0
+#define regGCEA_IO_WR_CLI2GRP_MAP1                                                                      0x0ad8
+#define regGCEA_IO_WR_CLI2GRP_MAP1_BASE_IDX                                                             0
+#define regGCEA_IO_RD_COMBINE_FLUSH                                                                     0x0ad9
+#define regGCEA_IO_RD_COMBINE_FLUSH_BASE_IDX                                                            0
+#define regGCEA_IO_WR_COMBINE_FLUSH                                                                     0x0ada
+#define regGCEA_IO_WR_COMBINE_FLUSH_BASE_IDX                                                            0
+#define regGCEA_IO_GROUP_BURST                                                                          0x0adb
+#define regGCEA_IO_GROUP_BURST_BASE_IDX                                                                 0
+#define regGCEA_IO_RD_PRI_AGE                                                                           0x0adc
+#define regGCEA_IO_RD_PRI_AGE_BASE_IDX                                                                  0
+#define regGCEA_IO_WR_PRI_AGE                                                                           0x0add
+#define regGCEA_IO_WR_PRI_AGE_BASE_IDX                                                                  0
+#define regGCEA_IO_RD_PRI_QUEUING                                                                       0x0ade
+#define regGCEA_IO_RD_PRI_QUEUING_BASE_IDX                                                              0
+#define regGCEA_IO_WR_PRI_QUEUING                                                                       0x0adf
+#define regGCEA_IO_WR_PRI_QUEUING_BASE_IDX                                                              0
+#define regGCEA_IO_RD_PRI_FIXED                                                                         0x0ae0
+#define regGCEA_IO_RD_PRI_FIXED_BASE_IDX                                                                0
+#define regGCEA_IO_WR_PRI_FIXED                                                                         0x0ae1
+#define regGCEA_IO_WR_PRI_FIXED_BASE_IDX                                                                0
+#define regGCEA_IO_RD_PRI_URGENCY                                                                       0x0ae2
+#define regGCEA_IO_RD_PRI_URGENCY_BASE_IDX                                                              0
+#define regGCEA_IO_WR_PRI_URGENCY                                                                       0x0ae3
+#define regGCEA_IO_WR_PRI_URGENCY_BASE_IDX                                                              0
+#define regGCEA_IO_RD_PRI_URGENCY_MASKING                                                               0x0ae4
+#define regGCEA_IO_RD_PRI_URGENCY_MASKING_BASE_IDX                                                      0
+#define regGCEA_IO_WR_PRI_URGENCY_MASKING                                                               0x0ae5
+#define regGCEA_IO_WR_PRI_URGENCY_MASKING_BASE_IDX                                                      0
+#define regGCEA_IO_RD_PRI_QUANT_PRI1                                                                    0x0ae6
+#define regGCEA_IO_RD_PRI_QUANT_PRI1_BASE_IDX                                                           0
+#define regGCEA_IO_RD_PRI_QUANT_PRI2                                                                    0x0ae7
+#define regGCEA_IO_RD_PRI_QUANT_PRI2_BASE_IDX                                                           0
+#define regGCEA_IO_RD_PRI_QUANT_PRI3                                                                    0x0ae8
+#define regGCEA_IO_RD_PRI_QUANT_PRI3_BASE_IDX                                                           0
+#define regGCEA_IO_WR_PRI_QUANT_PRI1                                                                    0x0ae9
+#define regGCEA_IO_WR_PRI_QUANT_PRI1_BASE_IDX                                                           0
+#define regGCEA_IO_WR_PRI_QUANT_PRI2                                                                    0x0aea
+#define regGCEA_IO_WR_PRI_QUANT_PRI2_BASE_IDX                                                           0
+#define regGCEA_IO_WR_PRI_QUANT_PRI3                                                                    0x0aeb
+#define regGCEA_IO_WR_PRI_QUANT_PRI3_BASE_IDX                                                           0
+#define regGCEA_SDP_ARB_DRAM                                                                            0x0aec
+#define regGCEA_SDP_ARB_DRAM_BASE_IDX                                                                   0
+#define regGCEA_SDP_ARB_FINAL                                                                           0x0aee
+#define regGCEA_SDP_ARB_FINAL_BASE_IDX                                                                  0
+#define regGCEA_SDP_DRAM_PRIORITY                                                                       0x0aef
+#define regGCEA_SDP_DRAM_PRIORITY_BASE_IDX                                                              0
+#define regGCEA_SDP_IO_PRIORITY                                                                         0x0af1
+#define regGCEA_SDP_IO_PRIORITY_BASE_IDX                                                                0
+#define regGCEA_SDP_CREDITS                                                                             0x0af2
+#define regGCEA_SDP_CREDITS_BASE_IDX                                                                    0
+#define regGCEA_SDP_TAG_RESERVE0                                                                        0x0af3
+#define regGCEA_SDP_TAG_RESERVE0_BASE_IDX                                                               0
+#define regGCEA_SDP_TAG_RESERVE1                                                                        0x0af4
+#define regGCEA_SDP_TAG_RESERVE1_BASE_IDX                                                               0
+#define regGCEA_SDP_VCC_RESERVE0                                                                        0x0af5
+#define regGCEA_SDP_VCC_RESERVE0_BASE_IDX                                                               0
+#define regGCEA_SDP_VCC_RESERVE1                                                                        0x0af6
+#define regGCEA_SDP_VCC_RESERVE1_BASE_IDX                                                               0
+#define regGCEA_SDP_VCD_RESERVE0                                                                        0x0af7
+#define regGCEA_SDP_VCD_RESERVE0_BASE_IDX                                                               0
+#define regGCEA_SDP_VCD_RESERVE1                                                                        0x0af8
+#define regGCEA_SDP_VCD_RESERVE1_BASE_IDX                                                               0
+#define regGCEA_SDP_REQ_CNTL                                                                            0x0af9
+#define regGCEA_SDP_REQ_CNTL_BASE_IDX                                                                   0
+#define regGCEA_MISC                                                                                    0x0afa
+#define regGCEA_MISC_BASE_IDX                                                                           0
+#define regGCEA_LATENCY_SAMPLING                                                                        0x0afb
+#define regGCEA_LATENCY_SAMPLING_BASE_IDX                                                               0
+#define regGCEA_PERFCOUNTER_LO                                                                          0x0afc
+#define regGCEA_PERFCOUNTER_LO_BASE_IDX                                                                 0
+#define regGCEA_PERFCOUNTER_HI                                                                          0x0afd
+#define regGCEA_PERFCOUNTER_HI_BASE_IDX                                                                 0
+#define regGCEA_PERFCOUNTER0_CFG                                                                        0x0afe
+#define regGCEA_PERFCOUNTER0_CFG_BASE_IDX                                                               0
+#define regGCEA_PERFCOUNTER1_CFG                                                                        0x0aff
+#define regGCEA_PERFCOUNTER1_CFG_BASE_IDX                                                               0
+
+
+// addressBlock: xcd0_gc_ea_gceadec2
+// base address: 0x9c00
+#define regGCEA_PERFCOUNTER_RSLT_CNTL                                                                   0x0700
+#define regGCEA_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                          0
+#define regGCEA_MAM_CTRL                                                                                0x0701
+#define regGCEA_MAM_CTRL_BASE_IDX                                                                       0
+#define regGCEA_MAM_CTRL2                                                                               0x0702
+#define regGCEA_MAM_CTRL2_BASE_IDX                                                                      0
+#define regGCEA_DSM_CNTL                                                                                0x0708
+#define regGCEA_DSM_CNTL_BASE_IDX                                                                       0
+#define regGCEA_DSM_CNTLA                                                                               0x0709
+#define regGCEA_DSM_CNTLA_BASE_IDX                                                                      0
+#define regGCEA_DSM_CNTLB                                                                               0x070a
+#define regGCEA_DSM_CNTLB_BASE_IDX                                                                      0
+#define regGCEA_DSM_CNTL2                                                                               0x070b
+#define regGCEA_DSM_CNTL2_BASE_IDX                                                                      0
+#define regGCEA_DSM_CNTL2A                                                                              0x070c
+#define regGCEA_DSM_CNTL2A_BASE_IDX                                                                     0
+#define regGCEA_DSM_CNTL2B                                                                              0x070d
+#define regGCEA_DSM_CNTL2B_BASE_IDX                                                                     0
+#define regGCEA_TCC_XBR_CREDITS                                                                         0x070e
+#define regGCEA_TCC_XBR_CREDITS_BASE_IDX                                                                0
+#define regGCEA_TCC_XBR_MAXBURST                                                                        0x070f
+#define regGCEA_TCC_XBR_MAXBURST_BASE_IDX                                                               0
+#define regGCEA_PROBE_CNTL                                                                              0x0710
+#define regGCEA_PROBE_CNTL_BASE_IDX                                                                     0
+#define regGCEA_PROBE_MAP                                                                               0x0711
+#define regGCEA_PROBE_MAP_BASE_IDX                                                                      0
+#define regGCEA_ERR_STATUS                                                                              0x0712
+#define regGCEA_ERR_STATUS_BASE_IDX                                                                     0
+#define regGCEA_MISC2                                                                                   0x0713
+#define regGCEA_MISC2_BASE_IDX                                                                          0
+#define regGCEA_SDP_BACKDOOR_CMDCREDITS0                                                                0x0715
+#define regGCEA_SDP_BACKDOOR_CMDCREDITS0_BASE_IDX                                                       0
+#define regGCEA_SDP_BACKDOOR_CMDCREDITS1                                                                0x0716
+#define regGCEA_SDP_BACKDOOR_CMDCREDITS1_BASE_IDX                                                       0
+#define regGCEA_SDP_BACKDOOR_DATACREDITS0                                                               0x0717
+#define regGCEA_SDP_BACKDOOR_DATACREDITS0_BASE_IDX                                                      0
+#define regGCEA_SDP_BACKDOOR_DATACREDITS1                                                               0x0718
+#define regGCEA_SDP_BACKDOOR_DATACREDITS1_BASE_IDX                                                      0
+#define regGCEA_SDP_BACKDOOR_MISCCREDITS                                                                0x0719
+#define regGCEA_SDP_BACKDOOR_MISCCREDITS_BASE_IDX                                                       0
+#define regGCEA_SDP_ENABLE                                                                              0x071f
+#define regGCEA_SDP_ENABLE_BASE_IDX                                                                     0
+
+
+// addressBlock: xcd0_gc_ea_pwrdec
+// base address: 0x3c000
+#define regGCEA_ICG_CTRL                                                                                0x50c4
+#define regGCEA_ICG_CTRL_BASE_IDX                                                                       1
+
+
+// addressBlock: xcd0_gc_rmi_rmidec
+// base address: 0x9e00
+#define regRMI_GENERAL_CNTL                                                                             0x0780
+#define regRMI_GENERAL_CNTL_BASE_IDX                                                                    0
+#define regRMI_GENERAL_CNTL1                                                                            0x0781
+#define regRMI_GENERAL_CNTL1_BASE_IDX                                                                   0
+#define regRMI_GENERAL_STATUS                                                                           0x0782
+#define regRMI_GENERAL_STATUS_BASE_IDX                                                                  0
+#define regRMI_SUBBLOCK_STATUS0                                                                         0x0783
+#define regRMI_SUBBLOCK_STATUS0_BASE_IDX                                                                0
+#define regRMI_SUBBLOCK_STATUS1                                                                         0x0784
+#define regRMI_SUBBLOCK_STATUS1_BASE_IDX                                                                0
+#define regRMI_SUBBLOCK_STATUS2                                                                         0x0785
+#define regRMI_SUBBLOCK_STATUS2_BASE_IDX                                                                0
+#define regRMI_SUBBLOCK_STATUS3                                                                         0x0786
+#define regRMI_SUBBLOCK_STATUS3_BASE_IDX                                                                0
+#define regRMI_XBAR_CONFIG                                                                              0x0787
+#define regRMI_XBAR_CONFIG_BASE_IDX                                                                     0
+#define regRMI_PROBE_POP_LOGIC_CNTL                                                                     0x0788
+#define regRMI_PROBE_POP_LOGIC_CNTL_BASE_IDX                                                            0
+#define regRMI_UTC_XNACK_N_MISC_CNTL                                                                    0x0789
+#define regRMI_UTC_XNACK_N_MISC_CNTL_BASE_IDX                                                           0
+#define regRMI_DEMUX_CNTL                                                                               0x078a
+#define regRMI_DEMUX_CNTL_BASE_IDX                                                                      0
+#define regRMI_UTCL1_CNTL1                                                                              0x078b
+#define regRMI_UTCL1_CNTL1_BASE_IDX                                                                     0
+#define regRMI_UTCL1_CNTL2                                                                              0x078c
+#define regRMI_UTCL1_CNTL2_BASE_IDX                                                                     0
+#define regRMI_UTC_UNIT_CONFIG                                                                          0x078d
+#define regRMI_UTC_UNIT_CONFIG_BASE_IDX                                                                 0
+#define regRMI_TCIW_FORMATTER0_CNTL                                                                     0x078e
+#define regRMI_TCIW_FORMATTER0_CNTL_BASE_IDX                                                            0
+#define regRMI_TCIW_FORMATTER1_CNTL                                                                     0x078f
+#define regRMI_TCIW_FORMATTER1_CNTL_BASE_IDX                                                            0
+#define regRMI_SCOREBOARD_CNTL                                                                          0x0790
+#define regRMI_SCOREBOARD_CNTL_BASE_IDX                                                                 0
+#define regRMI_SCOREBOARD_STATUS0                                                                       0x0791
+#define regRMI_SCOREBOARD_STATUS0_BASE_IDX                                                              0
+#define regRMI_SCOREBOARD_STATUS1                                                                       0x0792
+#define regRMI_SCOREBOARD_STATUS1_BASE_IDX                                                              0
+#define regRMI_SCOREBOARD_STATUS2                                                                       0x0793
+#define regRMI_SCOREBOARD_STATUS2_BASE_IDX                                                              0
+#define regRMI_XBAR_ARBITER_CONFIG                                                                      0x0794
+#define regRMI_XBAR_ARBITER_CONFIG_BASE_IDX                                                             0
+#define regRMI_XBAR_ARBITER_CONFIG_1                                                                    0x0795
+#define regRMI_XBAR_ARBITER_CONFIG_1_BASE_IDX                                                           0
+#define regRMI_CLOCK_CNTRL                                                                              0x0796
+#define regRMI_CLOCK_CNTRL_BASE_IDX                                                                     0
+#define regRMI_UTCL1_STATUS                                                                             0x0797
+#define regRMI_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regRMI_XNACK_DEBUG                                                                              0x079d
+#define regRMI_XNACK_DEBUG_BASE_IDX                                                                     0
+#define regRMI_SPARE                                                                                    0x079e
+#define regRMI_SPARE_BASE_IDX                                                                           0
+#define regRMI_SPARE_1                                                                                  0x079f
+#define regRMI_SPARE_1_BASE_IDX                                                                         0
+#define regRMI_SPARE_2                                                                                  0x07a0
+#define regRMI_SPARE_2_BASE_IDX                                                                         0
+
+
+// addressBlock: xcd0_gc_utcl2_atcl2dec
+// base address: 0xa000
+#define regATC_L2_CNTL                                                                                  0x0800
+#define regATC_L2_CNTL_BASE_IDX                                                                         0
+#define regATC_L2_CNTL2                                                                                 0x0801
+#define regATC_L2_CNTL2_BASE_IDX                                                                        0
+#define regATC_L2_CACHE_DATA0                                                                           0x0804
+#define regATC_L2_CACHE_DATA0_BASE_IDX                                                                  0
+#define regATC_L2_CACHE_DATA1                                                                           0x0805
+#define regATC_L2_CACHE_DATA1_BASE_IDX                                                                  0
+#define regATC_L2_CACHE_DATA2                                                                           0x0806
+#define regATC_L2_CACHE_DATA2_BASE_IDX                                                                  0
+#define regATC_L2_CACHE_DATA3                                                                           0x0807
+#define regATC_L2_CACHE_DATA3_BASE_IDX                                                                  0
+#define regATC_L2_CNTL3                                                                                 0x0808
+#define regATC_L2_CNTL3_BASE_IDX                                                                        0
+#define regATC_L2_STATUS                                                                                0x0809
+#define regATC_L2_STATUS_BASE_IDX                                                                       0
+#define regATC_L2_STATUS2                                                                               0x080a
+#define regATC_L2_STATUS2_BASE_IDX                                                                      0
+#define regATC_L2_MISC_CG                                                                               0x080b
+#define regATC_L2_MISC_CG_BASE_IDX                                                                      0
+#define regATC_L2_MEM_POWER_LS                                                                          0x080c
+#define regATC_L2_MEM_POWER_LS_BASE_IDX                                                                 0
+#define regATC_L2_CGTT_CLK_CTRL                                                                         0x080d
+#define regATC_L2_CGTT_CLK_CTRL_BASE_IDX                                                                0
+#define regATC_L2_CACHE_4K_DSM_INDEX                                                                    0x080f
+#define regATC_L2_CACHE_4K_DSM_INDEX_BASE_IDX                                                           0
+#define regATC_L2_CACHE_32K_DSM_INDEX                                                                   0x0810
+#define regATC_L2_CACHE_32K_DSM_INDEX_BASE_IDX                                                          0
+#define regATC_L2_CACHE_2M_DSM_INDEX                                                                    0x0811
+#define regATC_L2_CACHE_2M_DSM_INDEX_BASE_IDX                                                           0
+#define regATC_L2_CACHE_4K_DSM_CNTL                                                                     0x0812
+#define regATC_L2_CACHE_4K_DSM_CNTL_BASE_IDX                                                            0
+#define regATC_L2_CACHE_32K_DSM_CNTL                                                                    0x0813
+#define regATC_L2_CACHE_32K_DSM_CNTL_BASE_IDX                                                           0
+#define regATC_L2_CACHE_2M_DSM_CNTL                                                                     0x0814
+#define regATC_L2_CACHE_2M_DSM_CNTL_BASE_IDX                                                            0
+#define regATC_L2_CNTL4                                                                                 0x0815
+#define regATC_L2_CNTL4_BASE_IDX                                                                        0
+#define regATC_L2_MM_GROUP_RT_CLASSES                                                                   0x0816
+#define regATC_L2_MM_GROUP_RT_CLASSES_BASE_IDX                                                          0
+
+
+// addressBlock: xcd0_gc_utcl2_vml2pfdec
+// base address: 0xa080
+#define regVM_L2_CNTL                                                                                   0x0820
+#define regVM_L2_CNTL_BASE_IDX                                                                          0
+#define regVM_L2_CNTL2                                                                                  0x0821
+#define regVM_L2_CNTL2_BASE_IDX                                                                         0
+#define regVM_L2_CNTL3                                                                                  0x0822
+#define regVM_L2_CNTL3_BASE_IDX                                                                         0
+#define regVM_L2_STATUS                                                                                 0x0823
+#define regVM_L2_STATUS_BASE_IDX                                                                        0
+#define regVM_DUMMY_PAGE_FAULT_CNTL                                                                     0x0824
+#define regVM_DUMMY_PAGE_FAULT_CNTL_BASE_IDX                                                            0
+#define regVM_DUMMY_PAGE_FAULT_ADDR_LO32                                                                0x0825
+#define regVM_DUMMY_PAGE_FAULT_ADDR_LO32_BASE_IDX                                                       0
+#define regVM_DUMMY_PAGE_FAULT_ADDR_HI32                                                                0x0826
+#define regVM_DUMMY_PAGE_FAULT_ADDR_HI32_BASE_IDX                                                       0
+#define regVM_L2_PROTECTION_FAULT_CNTL                                                                  0x0827
+#define regVM_L2_PROTECTION_FAULT_CNTL_BASE_IDX                                                         0
+#define regVM_L2_PROTECTION_FAULT_CNTL2                                                                 0x0828
+#define regVM_L2_PROTECTION_FAULT_CNTL2_BASE_IDX                                                        0
+#define regVM_L2_PROTECTION_FAULT_MM_CNTL3                                                              0x0829
+#define regVM_L2_PROTECTION_FAULT_MM_CNTL3_BASE_IDX                                                     0
+#define regVM_L2_PROTECTION_FAULT_MM_CNTL4                                                              0x082a
+#define regVM_L2_PROTECTION_FAULT_MM_CNTL4_BASE_IDX                                                     0
+#define regVM_L2_PROTECTION_FAULT_STATUS                                                                0x082b
+#define regVM_L2_PROTECTION_FAULT_STATUS_BASE_IDX                                                       0
+#define regVM_L2_PROTECTION_FAULT_ADDR_LO32                                                             0x082c
+#define regVM_L2_PROTECTION_FAULT_ADDR_LO32_BASE_IDX                                                    0
+#define regVM_L2_PROTECTION_FAULT_ADDR_HI32                                                             0x082d
+#define regVM_L2_PROTECTION_FAULT_ADDR_HI32_BASE_IDX                                                    0
+#define regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32                                                     0x082e
+#define regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32_BASE_IDX                                            0
+#define regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32                                                     0x082f
+#define regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32_BASE_IDX                                            0
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32                                               0x0831
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32_BASE_IDX                                      0
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32                                               0x0832
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32_BASE_IDX                                      0
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32                                              0x0833
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32_BASE_IDX                                     0
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32                                              0x0834
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32_BASE_IDX                                     0
+#define regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32                                                  0x0835
+#define regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32_BASE_IDX                                         0
+#define regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32                                                  0x0836
+#define regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32_BASE_IDX                                         0
+#define regVM_L2_CNTL4                                                                                  0x0837
+#define regVM_L2_CNTL4_BASE_IDX                                                                         0
+#define regVM_L2_CNTL5                                                                                  0x0838
+#define regVM_L2_CNTL5_BASE_IDX                                                                         0
+#define regVM_L2_MM_GROUP_RT_CLASSES                                                                    0x0839
+#define regVM_L2_MM_GROUP_RT_CLASSES_BASE_IDX                                                           0
+#define regVM_L2_BANK_SELECT_RESERVED_CID                                                               0x083a
+#define regVM_L2_BANK_SELECT_RESERVED_CID_BASE_IDX                                                      0
+#define regVM_L2_BANK_SELECT_RESERVED_CID2                                                              0x083b
+#define regVM_L2_BANK_SELECT_RESERVED_CID2_BASE_IDX                                                     0
+#define regVM_L2_CACHE_PARITY_CNTL                                                                      0x083c
+#define regVM_L2_CACHE_PARITY_CNTL_BASE_IDX                                                             0
+#define regVM_L2_CGTT_CLK_CTRL                                                                          0x083d
+#define regVM_L2_CGTT_CLK_CTRL_BASE_IDX                                                                 0
+#define regVM_L2_CGTT_BUSY_CTRL                                                                         0x083e
+#define regVM_L2_CGTT_BUSY_CTRL_BASE_IDX                                                                0
+#define regVML2_MEM_ECC_INDEX                                                                           0x0842
+#define regVML2_MEM_ECC_INDEX_BASE_IDX                                                                  0
+#define regVML2_WALKER_MEM_ECC_INDEX                                                                    0x0843
+#define regVML2_WALKER_MEM_ECC_INDEX_BASE_IDX                                                           0
+#define regUTCL2_MEM_ECC_INDEX                                                                          0x0844
+#define regUTCL2_MEM_ECC_INDEX_BASE_IDX                                                                 0
+#define regVML2_MEM_ECC_CNTL                                                                            0x0845
+#define regVML2_MEM_ECC_CNTL_BASE_IDX                                                                   0
+#define regVML2_WALKER_MEM_ECC_CNTL                                                                     0x0846
+#define regVML2_WALKER_MEM_ECC_CNTL_BASE_IDX                                                            0
+#define regUTCL2_MEM_ECC_CNTL                                                                           0x0847
+#define regUTCL2_MEM_ECC_CNTL_BASE_IDX                                                                  0
+#define regVML2_MEM_ECC_STATUS                                                                          0x0848
+#define regVML2_MEM_ECC_STATUS_BASE_IDX                                                                 0
+#define regVML2_WALKER_MEM_ECC_STATUS                                                                   0x0849
+#define regVML2_WALKER_MEM_ECC_STATUS_BASE_IDX                                                          0
+#define regUTCL2_MEM_ECC_STATUS                                                                         0x084a
+#define regUTCL2_MEM_ECC_STATUS_BASE_IDX                                                                0
+#define regUTCL2_EDC_MODE                                                                               0x084b
+#define regUTCL2_EDC_MODE_BASE_IDX                                                                      0
+#define regUTCL2_EDC_CONFIG                                                                             0x084c
+#define regUTCL2_EDC_CONFIG_BASE_IDX                                                                    0
+
+
+// addressBlock: xcd0_gc_utcl2_vml2vcdec
+// base address: 0xa180
+#define regVM_CONTEXT0_CNTL                                                                             0x0860
+#define regVM_CONTEXT0_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT1_CNTL                                                                             0x0861
+#define regVM_CONTEXT1_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT2_CNTL                                                                             0x0862
+#define regVM_CONTEXT2_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT3_CNTL                                                                             0x0863
+#define regVM_CONTEXT3_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT4_CNTL                                                                             0x0864
+#define regVM_CONTEXT4_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT5_CNTL                                                                             0x0865
+#define regVM_CONTEXT5_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT6_CNTL                                                                             0x0866
+#define regVM_CONTEXT6_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT7_CNTL                                                                             0x0867
+#define regVM_CONTEXT7_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT8_CNTL                                                                             0x0868
+#define regVM_CONTEXT8_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT9_CNTL                                                                             0x0869
+#define regVM_CONTEXT9_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT10_CNTL                                                                            0x086a
+#define regVM_CONTEXT10_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXT11_CNTL                                                                            0x086b
+#define regVM_CONTEXT11_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXT12_CNTL                                                                            0x086c
+#define regVM_CONTEXT12_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXT13_CNTL                                                                            0x086d
+#define regVM_CONTEXT13_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXT14_CNTL                                                                            0x086e
+#define regVM_CONTEXT14_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXT15_CNTL                                                                            0x086f
+#define regVM_CONTEXT15_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXTS_DISABLE                                                                          0x0870
+#define regVM_CONTEXTS_DISABLE_BASE_IDX                                                                 0
+#define regVM_INVALIDATE_ENG0_SEM                                                                       0x0871
+#define regVM_INVALIDATE_ENG0_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG1_SEM                                                                       0x0872
+#define regVM_INVALIDATE_ENG1_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG2_SEM                                                                       0x0873
+#define regVM_INVALIDATE_ENG2_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG3_SEM                                                                       0x0874
+#define regVM_INVALIDATE_ENG3_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG4_SEM                                                                       0x0875
+#define regVM_INVALIDATE_ENG4_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG5_SEM                                                                       0x0876
+#define regVM_INVALIDATE_ENG5_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG6_SEM                                                                       0x0877
+#define regVM_INVALIDATE_ENG6_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG7_SEM                                                                       0x0878
+#define regVM_INVALIDATE_ENG7_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG8_SEM                                                                       0x0879
+#define regVM_INVALIDATE_ENG8_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG9_SEM                                                                       0x087a
+#define regVM_INVALIDATE_ENG9_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG10_SEM                                                                      0x087b
+#define regVM_INVALIDATE_ENG10_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG11_SEM                                                                      0x087c
+#define regVM_INVALIDATE_ENG11_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG12_SEM                                                                      0x087d
+#define regVM_INVALIDATE_ENG12_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG13_SEM                                                                      0x087e
+#define regVM_INVALIDATE_ENG13_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG14_SEM                                                                      0x087f
+#define regVM_INVALIDATE_ENG14_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG15_SEM                                                                      0x0880
+#define regVM_INVALIDATE_ENG15_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG16_SEM                                                                      0x0881
+#define regVM_INVALIDATE_ENG16_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG17_SEM                                                                      0x0882
+#define regVM_INVALIDATE_ENG17_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG0_REQ                                                                       0x0883
+#define regVM_INVALIDATE_ENG0_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG1_REQ                                                                       0x0884
+#define regVM_INVALIDATE_ENG1_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG2_REQ                                                                       0x0885
+#define regVM_INVALIDATE_ENG2_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG3_REQ                                                                       0x0886
+#define regVM_INVALIDATE_ENG3_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG4_REQ                                                                       0x0887
+#define regVM_INVALIDATE_ENG4_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG5_REQ                                                                       0x0888
+#define regVM_INVALIDATE_ENG5_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG6_REQ                                                                       0x0889
+#define regVM_INVALIDATE_ENG6_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG7_REQ                                                                       0x088a
+#define regVM_INVALIDATE_ENG7_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG8_REQ                                                                       0x088b
+#define regVM_INVALIDATE_ENG8_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG9_REQ                                                                       0x088c
+#define regVM_INVALIDATE_ENG9_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG10_REQ                                                                      0x088d
+#define regVM_INVALIDATE_ENG10_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG11_REQ                                                                      0x088e
+#define regVM_INVALIDATE_ENG11_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG12_REQ                                                                      0x088f
+#define regVM_INVALIDATE_ENG12_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG13_REQ                                                                      0x0890
+#define regVM_INVALIDATE_ENG13_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG14_REQ                                                                      0x0891
+#define regVM_INVALIDATE_ENG14_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG15_REQ                                                                      0x0892
+#define regVM_INVALIDATE_ENG15_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG16_REQ                                                                      0x0893
+#define regVM_INVALIDATE_ENG16_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG17_REQ                                                                      0x0894
+#define regVM_INVALIDATE_ENG17_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG0_ACK                                                                       0x0895
+#define regVM_INVALIDATE_ENG0_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG1_ACK                                                                       0x0896
+#define regVM_INVALIDATE_ENG1_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG2_ACK                                                                       0x0897
+#define regVM_INVALIDATE_ENG2_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG3_ACK                                                                       0x0898
+#define regVM_INVALIDATE_ENG3_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG4_ACK                                                                       0x0899
+#define regVM_INVALIDATE_ENG4_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG5_ACK                                                                       0x089a
+#define regVM_INVALIDATE_ENG5_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG6_ACK                                                                       0x089b
+#define regVM_INVALIDATE_ENG6_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG7_ACK                                                                       0x089c
+#define regVM_INVALIDATE_ENG7_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG8_ACK                                                                       0x089d
+#define regVM_INVALIDATE_ENG8_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG9_ACK                                                                       0x089e
+#define regVM_INVALIDATE_ENG9_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG10_ACK                                                                      0x089f
+#define regVM_INVALIDATE_ENG10_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG11_ACK                                                                      0x08a0
+#define regVM_INVALIDATE_ENG11_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG12_ACK                                                                      0x08a1
+#define regVM_INVALIDATE_ENG12_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG13_ACK                                                                      0x08a2
+#define regVM_INVALIDATE_ENG13_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG14_ACK                                                                      0x08a3
+#define regVM_INVALIDATE_ENG14_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG15_ACK                                                                      0x08a4
+#define regVM_INVALIDATE_ENG15_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG16_ACK                                                                      0x08a5
+#define regVM_INVALIDATE_ENG16_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG17_ACK                                                                      0x08a6
+#define regVM_INVALIDATE_ENG17_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32                                                           0x08a7
+#define regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32                                                           0x08a8
+#define regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32                                                           0x08a9
+#define regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG1_ADDR_RANGE_HI32                                                           0x08aa
+#define regVM_INVALIDATE_ENG1_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG2_ADDR_RANGE_LO32                                                           0x08ab
+#define regVM_INVALIDATE_ENG2_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG2_ADDR_RANGE_HI32                                                           0x08ac
+#define regVM_INVALIDATE_ENG2_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG3_ADDR_RANGE_LO32                                                           0x08ad
+#define regVM_INVALIDATE_ENG3_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG3_ADDR_RANGE_HI32                                                           0x08ae
+#define regVM_INVALIDATE_ENG3_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG4_ADDR_RANGE_LO32                                                           0x08af
+#define regVM_INVALIDATE_ENG4_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG4_ADDR_RANGE_HI32                                                           0x08b0
+#define regVM_INVALIDATE_ENG4_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG5_ADDR_RANGE_LO32                                                           0x08b1
+#define regVM_INVALIDATE_ENG5_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG5_ADDR_RANGE_HI32                                                           0x08b2
+#define regVM_INVALIDATE_ENG5_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG6_ADDR_RANGE_LO32                                                           0x08b3
+#define regVM_INVALIDATE_ENG6_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG6_ADDR_RANGE_HI32                                                           0x08b4
+#define regVM_INVALIDATE_ENG6_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG7_ADDR_RANGE_LO32                                                           0x08b5
+#define regVM_INVALIDATE_ENG7_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG7_ADDR_RANGE_HI32                                                           0x08b6
+#define regVM_INVALIDATE_ENG7_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG8_ADDR_RANGE_LO32                                                           0x08b7
+#define regVM_INVALIDATE_ENG8_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG8_ADDR_RANGE_HI32                                                           0x08b8
+#define regVM_INVALIDATE_ENG8_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG9_ADDR_RANGE_LO32                                                           0x08b9
+#define regVM_INVALIDATE_ENG9_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG9_ADDR_RANGE_HI32                                                           0x08ba
+#define regVM_INVALIDATE_ENG9_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG10_ADDR_RANGE_LO32                                                          0x08bb
+#define regVM_INVALIDATE_ENG10_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG10_ADDR_RANGE_HI32                                                          0x08bc
+#define regVM_INVALIDATE_ENG10_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG11_ADDR_RANGE_LO32                                                          0x08bd
+#define regVM_INVALIDATE_ENG11_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG11_ADDR_RANGE_HI32                                                          0x08be
+#define regVM_INVALIDATE_ENG11_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG12_ADDR_RANGE_LO32                                                          0x08bf
+#define regVM_INVALIDATE_ENG12_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG12_ADDR_RANGE_HI32                                                          0x08c0
+#define regVM_INVALIDATE_ENG12_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG13_ADDR_RANGE_LO32                                                          0x08c1
+#define regVM_INVALIDATE_ENG13_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG13_ADDR_RANGE_HI32                                                          0x08c2
+#define regVM_INVALIDATE_ENG13_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG14_ADDR_RANGE_LO32                                                          0x08c3
+#define regVM_INVALIDATE_ENG14_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG14_ADDR_RANGE_HI32                                                          0x08c4
+#define regVM_INVALIDATE_ENG14_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG15_ADDR_RANGE_LO32                                                          0x08c5
+#define regVM_INVALIDATE_ENG15_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG15_ADDR_RANGE_HI32                                                          0x08c6
+#define regVM_INVALIDATE_ENG15_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG16_ADDR_RANGE_LO32                                                          0x08c7
+#define regVM_INVALIDATE_ENG16_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG16_ADDR_RANGE_HI32                                                          0x08c8
+#define regVM_INVALIDATE_ENG16_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG17_ADDR_RANGE_LO32                                                          0x08c9
+#define regVM_INVALIDATE_ENG17_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG17_ADDR_RANGE_HI32                                                          0x08ca
+#define regVM_INVALIDATE_ENG17_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32                                                        0x08cb
+#define regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32                                                        0x08cc
+#define regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32                                                        0x08cd
+#define regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32                                                        0x08ce
+#define regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32                                                        0x08cf
+#define regVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32                                                        0x08d0
+#define regVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32                                                        0x08d1
+#define regVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32                                                        0x08d2
+#define regVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32                                                        0x08d3
+#define regVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32                                                        0x08d4
+#define regVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32                                                        0x08d5
+#define regVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32                                                        0x08d6
+#define regVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32                                                        0x08d7
+#define regVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32                                                        0x08d8
+#define regVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32                                                        0x08d9
+#define regVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32                                                        0x08da
+#define regVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32                                                        0x08db
+#define regVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32                                                        0x08dc
+#define regVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32                                                        0x08dd
+#define regVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32                                                        0x08de
+#define regVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32                                                       0x08df
+#define regVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32                                                       0x08e0
+#define regVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32                                                       0x08e1
+#define regVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32                                                       0x08e2
+#define regVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32                                                       0x08e3
+#define regVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32                                                       0x08e4
+#define regVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32                                                       0x08e5
+#define regVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32                                                       0x08e6
+#define regVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32                                                       0x08e7
+#define regVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32                                                       0x08e8
+#define regVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32                                                       0x08e9
+#define regVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32                                                       0x08ea
+#define regVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32                                                       0x08eb
+#define regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32                                                       0x08ec
+#define regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32                                                       0x08ed
+#define regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32                                                       0x08ee
+#define regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32                                                       0x08ef
+#define regVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32                                                       0x08f0
+#define regVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32                                                       0x08f1
+#define regVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32                                                       0x08f2
+#define regVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32                                                       0x08f3
+#define regVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32                                                       0x08f4
+#define regVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32                                                       0x08f5
+#define regVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32                                                       0x08f6
+#define regVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32                                                       0x08f7
+#define regVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32                                                       0x08f8
+#define regVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32                                                       0x08f9
+#define regVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32                                                       0x08fa
+#define regVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32                                                       0x08fb
+#define regVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32                                                       0x08fc
+#define regVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32                                                       0x08fd
+#define regVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32                                                       0x08fe
+#define regVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32                                                      0x08ff
+#define regVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32                                                      0x0900
+#define regVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32                                                      0x0901
+#define regVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32                                                      0x0902
+#define regVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32                                                      0x0903
+#define regVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32                                                      0x0904
+#define regVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32                                                      0x0905
+#define regVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32                                                      0x0906
+#define regVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32                                                      0x0907
+#define regVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32                                                      0x0908
+#define regVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32                                                      0x0909
+#define regVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32                                                      0x090a
+#define regVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32                                                         0x090b
+#define regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32                                                         0x090c
+#define regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32                                                         0x090d
+#define regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32                                                         0x090e
+#define regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32                                                         0x090f
+#define regVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32                                                         0x0910
+#define regVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32                                                         0x0911
+#define regVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32                                                         0x0912
+#define regVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32                                                         0x0913
+#define regVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32                                                         0x0914
+#define regVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32                                                         0x0915
+#define regVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32                                                         0x0916
+#define regVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32                                                         0x0917
+#define regVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32                                                         0x0918
+#define regVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32                                                         0x0919
+#define regVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32                                                         0x091a
+#define regVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32                                                         0x091b
+#define regVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32                                                         0x091c
+#define regVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32                                                         0x091d
+#define regVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32                                                         0x091e
+#define regVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32                                                        0x091f
+#define regVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32                                                        0x0920
+#define regVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32                                                        0x0921
+#define regVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32                                                        0x0922
+#define regVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32                                                        0x0923
+#define regVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32                                                        0x0924
+#define regVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32                                                        0x0925
+#define regVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32                                                        0x0926
+#define regVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32                                                        0x0927
+#define regVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32                                                        0x0928
+#define regVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32                                                        0x0929
+#define regVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32                                                        0x092a
+#define regVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+
+
+// addressBlock: xcd0_gc_utcl2_vmsharedpfdec
+// base address: 0xa500
+#define regMC_VM_NB_MMIOBASE                                                                            0x0940
+#define regMC_VM_NB_MMIOBASE_BASE_IDX                                                                   0
+#define regMC_VM_NB_MMIOLIMIT                                                                           0x0941
+#define regMC_VM_NB_MMIOLIMIT_BASE_IDX                                                                  0
+#define regMC_VM_NB_PCI_CTRL                                                                            0x0942
+#define regMC_VM_NB_PCI_CTRL_BASE_IDX                                                                   0
+#define regMC_VM_NB_PCI_ARB                                                                             0x0943
+#define regMC_VM_NB_PCI_ARB_BASE_IDX                                                                    0
+#define regMC_VM_NB_TOP_OF_DRAM_SLOT1                                                                   0x0944
+#define regMC_VM_NB_TOP_OF_DRAM_SLOT1_BASE_IDX                                                          0
+#define regMC_VM_NB_LOWER_TOP_OF_DRAM2                                                                  0x0945
+#define regMC_VM_NB_LOWER_TOP_OF_DRAM2_BASE_IDX                                                         0
+#define regMC_VM_NB_UPPER_TOP_OF_DRAM2                                                                  0x0946
+#define regMC_VM_NB_UPPER_TOP_OF_DRAM2_BASE_IDX                                                         0
+#define regMC_VM_FB_OFFSET                                                                              0x0947
+#define regMC_VM_FB_OFFSET_BASE_IDX                                                                     0
+#define regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB                                                       0x0948
+#define regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX                                              0
+#define regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB                                                       0x0949
+#define regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX                                              0
+#define regMC_VM_STEERING                                                                               0x094a
+#define regMC_VM_STEERING_BASE_IDX                                                                      0
+#define regMC_SHARED_VIRT_RESET_REQ                                                                     0x094b
+#define regMC_SHARED_VIRT_RESET_REQ_BASE_IDX                                                            0
+#define regMC_MEM_POWER_LS                                                                              0x094c
+#define regMC_MEM_POWER_LS_BASE_IDX                                                                     0
+#define regMC_VM_CACHEABLE_DRAM_ADDRESS_START                                                           0x094d
+#define regMC_VM_CACHEABLE_DRAM_ADDRESS_START_BASE_IDX                                                  0
+#define regMC_VM_CACHEABLE_DRAM_ADDRESS_END                                                             0x094e
+#define regMC_VM_CACHEABLE_DRAM_ADDRESS_END_BASE_IDX                                                    0
+#define regMC_VM_APT_CNTL                                                                               0x0951
+#define regMC_VM_APT_CNTL_BASE_IDX                                                                      0
+#define regMC_VM_LOCAL_HBM_ADDRESS_START                                                                0x0952
+#define regMC_VM_LOCAL_HBM_ADDRESS_START_BASE_IDX                                                       0
+#define regMC_VM_LOCAL_HBM_ADDRESS_END                                                                  0x0953
+#define regMC_VM_LOCAL_HBM_ADDRESS_END_BASE_IDX                                                         0
+#define regMC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL                                                            0x0954
+#define regMC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL_BASE_IDX                                                   0
+#define regUTCL2_CGTT_CLK_CTRL                                                                          0x0955
+#define regUTCL2_CGTT_CLK_CTRL_BASE_IDX                                                                 0
+#define regMC_VM_XGMI_LFB_CNTL                                                                          0x0957
+#define regMC_VM_XGMI_LFB_CNTL_BASE_IDX                                                                 0
+#define regMC_VM_XGMI_LFB_SIZE                                                                          0x0958
+#define regMC_VM_XGMI_LFB_SIZE_BASE_IDX                                                                 0
+#define regMC_VM_CACHEABLE_DRAM_CNTL                                                                    0x0959
+#define regMC_VM_CACHEABLE_DRAM_CNTL_BASE_IDX                                                           0
+#define regMC_VM_HOST_MAPPING                                                                           0x095a
+#define regMC_VM_HOST_MAPPING_BASE_IDX                                                                  0
+
+
+// addressBlock: xcd0_gc_utcl2_vmsharedvcdec
+// base address: 0xa570
+#define regMC_VM_FB_LOCATION_BASE                                                                       0x095c
+#define regMC_VM_FB_LOCATION_BASE_BASE_IDX                                                              0
+#define regMC_VM_FB_LOCATION_TOP                                                                        0x095d
+#define regMC_VM_FB_LOCATION_TOP_BASE_IDX                                                               0
+#define regMC_VM_AGP_TOP                                                                                0x095e
+#define regMC_VM_AGP_TOP_BASE_IDX                                                                       0
+#define regMC_VM_AGP_BOT                                                                                0x095f
+#define regMC_VM_AGP_BOT_BASE_IDX                                                                       0
+#define regMC_VM_AGP_BASE                                                                               0x0960
+#define regMC_VM_AGP_BASE_BASE_IDX                                                                      0
+#define regMC_VM_SYSTEM_APERTURE_LOW_ADDR                                                               0x0961
+#define regMC_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX                                                      0
+#define regMC_VM_SYSTEM_APERTURE_HIGH_ADDR                                                              0x0962
+#define regMC_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX                                                     0
+#define regMC_VM_MX_L1_TLB_CNTL                                                                         0x0963
+#define regMC_VM_MX_L1_TLB_CNTL_BASE_IDX                                                                0
+
+
+// addressBlock: xcd0_gc_utcl2_l2tlbdec
+// base address: 0xa5b0
+#define regL2TLB_TLB0_STATUS                                                                            0x096d
+#define regL2TLB_TLB0_STATUS_BASE_IDX                                                                   0
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO                                                 0x096f
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO_BASE_IDX                                        0
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI                                                 0x0970
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI_BASE_IDX                                        0
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO                                                0x0971
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO_BASE_IDX                                       0
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI                                                0x0972
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI_BASE_IDX                                       0
+
+
+// addressBlock: xcd0_gc_tcdec
+// base address: 0xac00
+#define regTCP_INVALIDATE                                                                               0x0b00
+#define regTCP_INVALIDATE_BASE_IDX                                                                      0
+#define regTCP_STATUS                                                                                   0x0b01
+#define regTCP_STATUS_BASE_IDX                                                                          0
+#define regTCP_CNTL                                                                                     0x0b02
+#define regTCP_CNTL_BASE_IDX                                                                            0
+#define regTCP_CHAN_STEER_0                                                                             0x0b03
+#define regTCP_CHAN_STEER_0_BASE_IDX                                                                    0
+#define regTCP_CHAN_STEER_1                                                                             0x0b04
+#define regTCP_CHAN_STEER_1_BASE_IDX                                                                    0
+#define regTCP_ADDR_CONFIG                                                                              0x0b05
+#define regTCP_ADDR_CONFIG_BASE_IDX                                                                     0
+#define regTCP_CREDIT                                                                                   0x0b06
+#define regTCP_CREDIT_BASE_IDX                                                                          0
+#define regTCP_BUFFER_ADDR_HASH_CNTL                                                                    0x0b16
+#define regTCP_BUFFER_ADDR_HASH_CNTL_BASE_IDX                                                           0
+#define regTC_CFG_L1_LOAD_POLICY0                                                                       0x0b1a
+#define regTC_CFG_L1_LOAD_POLICY0_BASE_IDX                                                              0
+#define regTC_CFG_L1_LOAD_POLICY1                                                                       0x0b1b
+#define regTC_CFG_L1_LOAD_POLICY1_BASE_IDX                                                              0
+#define regTC_CFG_L1_STORE_POLICY                                                                       0x0b1c
+#define regTC_CFG_L1_STORE_POLICY_BASE_IDX                                                              0
+#define regTC_CFG_L2_LOAD_POLICY0                                                                       0x0b1d
+#define regTC_CFG_L2_LOAD_POLICY0_BASE_IDX                                                              0
+#define regTC_CFG_L2_LOAD_POLICY1                                                                       0x0b1e
+#define regTC_CFG_L2_LOAD_POLICY1_BASE_IDX                                                              0
+#define regTC_CFG_L2_STORE_POLICY0                                                                      0x0b1f
+#define regTC_CFG_L2_STORE_POLICY0_BASE_IDX                                                             0
+#define regTC_CFG_L2_STORE_POLICY1                                                                      0x0b20
+#define regTC_CFG_L2_STORE_POLICY1_BASE_IDX                                                             0
+#define regTC_CFG_L2_ATOMIC_POLICY                                                                      0x0b21
+#define regTC_CFG_L2_ATOMIC_POLICY_BASE_IDX                                                             0
+#define regTC_CFG_L1_VOLATILE                                                                           0x0b22
+#define regTC_CFG_L1_VOLATILE_BASE_IDX                                                                  0
+#define regTC_CFG_L2_VOLATILE                                                                           0x0b23
+#define regTC_CFG_L2_VOLATILE_BASE_IDX                                                                  0
+#define regTCI_MISC                                                                                     0x0b5c
+#define regTCI_MISC_BASE_IDX                                                                            0
+#define regTCI_CNTL_3                                                                                   0x0b5d
+#define regTCI_CNTL_3_BASE_IDX                                                                          0
+#define regTCI_DSM_CNTL                                                                                 0x0b5e
+#define regTCI_DSM_CNTL_BASE_IDX                                                                        0
+#define regTCI_DSM_CNTL2                                                                                0x0b5f
+#define regTCI_DSM_CNTL2_BASE_IDX                                                                       0
+#define regTCI_STATUS                                                                                   0x0b61
+#define regTCI_STATUS_BASE_IDX                                                                          0
+#define regTCI_CNTL_1                                                                                   0x0b62
+#define regTCI_CNTL_1_BASE_IDX                                                                          0
+#define regTCI_CNTL_2                                                                                   0x0b63
+#define regTCI_CNTL_2_BASE_IDX                                                                          0
+#define regTCC_CTRL                                                                                     0x0b80
+#define regTCC_CTRL_BASE_IDX                                                                            0
+#define regTCC_CTRL2                                                                                    0x0b81
+#define regTCC_CTRL2_BASE_IDX                                                                           0
+#define regTCC_DSM_CNTL                                                                                 0x0b86
+#define regTCC_DSM_CNTL_BASE_IDX                                                                        0
+#define regTCC_DSM_CNTLA                                                                                0x0b87
+#define regTCC_DSM_CNTLA_BASE_IDX                                                                       0
+#define regTCC_DSM_CNTL2                                                                                0x0b88
+#define regTCC_DSM_CNTL2_BASE_IDX                                                                       0
+#define regTCC_DSM_CNTL2A                                                                               0x0b89
+#define regTCC_DSM_CNTL2A_BASE_IDX                                                                      0
+#define regTCC_DSM_CNTL2B                                                                               0x0b8a
+#define regTCC_DSM_CNTL2B_BASE_IDX                                                                      0
+#define regTCC_WBINVL2                                                                                  0x0b8b
+#define regTCC_WBINVL2_BASE_IDX                                                                         0
+#define regTCC_SOFT_RESET                                                                               0x0b8c
+#define regTCC_SOFT_RESET_BASE_IDX                                                                      0
+#define regTCC_DSM_CNTL3                                                                                0x0b8e
+#define regTCC_DSM_CNTL3_BASE_IDX                                                                       0
+#define regTCA_CTRL                                                                                     0x0bc0
+#define regTCA_CTRL_BASE_IDX                                                                            0
+#define regTCA_BURST_MASK                                                                               0x0bc1
+#define regTCA_BURST_MASK_BASE_IDX                                                                      0
+#define regTCA_BURST_CTRL                                                                               0x0bc2
+#define regTCA_BURST_CTRL_BASE_IDX                                                                      0
+#define regTCA_DSM_CNTL                                                                                 0x0bc3
+#define regTCA_DSM_CNTL_BASE_IDX                                                                        0
+#define regTCA_DSM_CNTL2                                                                                0x0bc4
+#define regTCA_DSM_CNTL2_BASE_IDX                                                                       0
+#define regTCX_CTRL                                                                                     0x0bc6
+#define regTCX_CTRL_BASE_IDX                                                                            0
+#define regTCX_DSM_CNTL                                                                                 0x0bc7
+#define regTCX_DSM_CNTL_BASE_IDX                                                                        0
+#define regTCX_DSM_CNTL2                                                                                0x0bc8
+#define regTCX_DSM_CNTL2_BASE_IDX                                                                       0
+
+
+// addressBlock: xcd0_gc_shdec
+// base address: 0xb000
+#define regSPI_SHADER_PGM_RSRC3_PS                                                                      0x0c07
+#define regSPI_SHADER_PGM_RSRC3_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_LO_PS                                                                         0x0c08
+#define regSPI_SHADER_PGM_LO_PS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_PS                                                                         0x0c09
+#define regSPI_SHADER_PGM_HI_PS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC1_PS                                                                      0x0c0a
+#define regSPI_SHADER_PGM_RSRC1_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC2_PS                                                                      0x0c0b
+#define regSPI_SHADER_PGM_RSRC2_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_PS_0                                                                    0x0c0c
+#define regSPI_SHADER_USER_DATA_PS_0_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_1                                                                    0x0c0d
+#define regSPI_SHADER_USER_DATA_PS_1_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_2                                                                    0x0c0e
+#define regSPI_SHADER_USER_DATA_PS_2_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_3                                                                    0x0c0f
+#define regSPI_SHADER_USER_DATA_PS_3_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_4                                                                    0x0c10
+#define regSPI_SHADER_USER_DATA_PS_4_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_5                                                                    0x0c11
+#define regSPI_SHADER_USER_DATA_PS_5_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_6                                                                    0x0c12
+#define regSPI_SHADER_USER_DATA_PS_6_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_7                                                                    0x0c13
+#define regSPI_SHADER_USER_DATA_PS_7_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_8                                                                    0x0c14
+#define regSPI_SHADER_USER_DATA_PS_8_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_9                                                                    0x0c15
+#define regSPI_SHADER_USER_DATA_PS_9_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_10                                                                   0x0c16
+#define regSPI_SHADER_USER_DATA_PS_10_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_11                                                                   0x0c17
+#define regSPI_SHADER_USER_DATA_PS_11_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_12                                                                   0x0c18
+#define regSPI_SHADER_USER_DATA_PS_12_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_13                                                                   0x0c19
+#define regSPI_SHADER_USER_DATA_PS_13_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_14                                                                   0x0c1a
+#define regSPI_SHADER_USER_DATA_PS_14_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_15                                                                   0x0c1b
+#define regSPI_SHADER_USER_DATA_PS_15_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_16                                                                   0x0c1c
+#define regSPI_SHADER_USER_DATA_PS_16_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_17                                                                   0x0c1d
+#define regSPI_SHADER_USER_DATA_PS_17_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_18                                                                   0x0c1e
+#define regSPI_SHADER_USER_DATA_PS_18_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_19                                                                   0x0c1f
+#define regSPI_SHADER_USER_DATA_PS_19_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_20                                                                   0x0c20
+#define regSPI_SHADER_USER_DATA_PS_20_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_21                                                                   0x0c21
+#define regSPI_SHADER_USER_DATA_PS_21_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_22                                                                   0x0c22
+#define regSPI_SHADER_USER_DATA_PS_22_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_23                                                                   0x0c23
+#define regSPI_SHADER_USER_DATA_PS_23_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_24                                                                   0x0c24
+#define regSPI_SHADER_USER_DATA_PS_24_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_25                                                                   0x0c25
+#define regSPI_SHADER_USER_DATA_PS_25_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_26                                                                   0x0c26
+#define regSPI_SHADER_USER_DATA_PS_26_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_27                                                                   0x0c27
+#define regSPI_SHADER_USER_DATA_PS_27_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_28                                                                   0x0c28
+#define regSPI_SHADER_USER_DATA_PS_28_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_29                                                                   0x0c29
+#define regSPI_SHADER_USER_DATA_PS_29_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_30                                                                   0x0c2a
+#define regSPI_SHADER_USER_DATA_PS_30_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_31                                                                   0x0c2b
+#define regSPI_SHADER_USER_DATA_PS_31_BASE_IDX                                                          0
+#define regSPI_SHADER_PGM_RSRC3_VS                                                                      0x0c46
+#define regSPI_SHADER_PGM_RSRC3_VS_BASE_IDX                                                             0
+#define regSPI_SHADER_LATE_ALLOC_VS                                                                     0x0c47
+#define regSPI_SHADER_LATE_ALLOC_VS_BASE_IDX                                                            0
+#define regSPI_SHADER_PGM_LO_VS                                                                         0x0c48
+#define regSPI_SHADER_PGM_LO_VS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_VS                                                                         0x0c49
+#define regSPI_SHADER_PGM_HI_VS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC1_VS                                                                      0x0c4a
+#define regSPI_SHADER_PGM_RSRC1_VS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC2_VS                                                                      0x0c4b
+#define regSPI_SHADER_PGM_RSRC2_VS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_VS_0                                                                    0x0c4c
+#define regSPI_SHADER_USER_DATA_VS_0_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_VS_1                                                                    0x0c4d
+#define regSPI_SHADER_USER_DATA_VS_1_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_VS_2                                                                    0x0c4e
+#define regSPI_SHADER_USER_DATA_VS_2_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_VS_3                                                                    0x0c4f
+#define regSPI_SHADER_USER_DATA_VS_3_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_VS_4                                                                    0x0c50
+#define regSPI_SHADER_USER_DATA_VS_4_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_VS_5                                                                    0x0c51
+#define regSPI_SHADER_USER_DATA_VS_5_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_VS_6                                                                    0x0c52
+#define regSPI_SHADER_USER_DATA_VS_6_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_VS_7                                                                    0x0c53
+#define regSPI_SHADER_USER_DATA_VS_7_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_VS_8                                                                    0x0c54
+#define regSPI_SHADER_USER_DATA_VS_8_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_VS_9                                                                    0x0c55
+#define regSPI_SHADER_USER_DATA_VS_9_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_VS_10                                                                   0x0c56
+#define regSPI_SHADER_USER_DATA_VS_10_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_11                                                                   0x0c57
+#define regSPI_SHADER_USER_DATA_VS_11_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_12                                                                   0x0c58
+#define regSPI_SHADER_USER_DATA_VS_12_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_13                                                                   0x0c59
+#define regSPI_SHADER_USER_DATA_VS_13_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_14                                                                   0x0c5a
+#define regSPI_SHADER_USER_DATA_VS_14_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_15                                                                   0x0c5b
+#define regSPI_SHADER_USER_DATA_VS_15_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_16                                                                   0x0c5c
+#define regSPI_SHADER_USER_DATA_VS_16_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_17                                                                   0x0c5d
+#define regSPI_SHADER_USER_DATA_VS_17_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_18                                                                   0x0c5e
+#define regSPI_SHADER_USER_DATA_VS_18_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_19                                                                   0x0c5f
+#define regSPI_SHADER_USER_DATA_VS_19_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_20                                                                   0x0c60
+#define regSPI_SHADER_USER_DATA_VS_20_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_21                                                                   0x0c61
+#define regSPI_SHADER_USER_DATA_VS_21_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_22                                                                   0x0c62
+#define regSPI_SHADER_USER_DATA_VS_22_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_23                                                                   0x0c63
+#define regSPI_SHADER_USER_DATA_VS_23_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_24                                                                   0x0c64
+#define regSPI_SHADER_USER_DATA_VS_24_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_25                                                                   0x0c65
+#define regSPI_SHADER_USER_DATA_VS_25_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_26                                                                   0x0c66
+#define regSPI_SHADER_USER_DATA_VS_26_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_27                                                                   0x0c67
+#define regSPI_SHADER_USER_DATA_VS_27_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_28                                                                   0x0c68
+#define regSPI_SHADER_USER_DATA_VS_28_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_29                                                                   0x0c69
+#define regSPI_SHADER_USER_DATA_VS_29_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_30                                                                   0x0c6a
+#define regSPI_SHADER_USER_DATA_VS_30_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_VS_31                                                                   0x0c6b
+#define regSPI_SHADER_USER_DATA_VS_31_BASE_IDX                                                          0
+#define regSPI_SHADER_PGM_RSRC2_GS_VS                                                                   0x0c7c
+#define regSPI_SHADER_PGM_RSRC2_GS_VS_BASE_IDX                                                          0
+#define regSPI_SHADER_PGM_RSRC4_GS                                                                      0x0c81
+#define regSPI_SHADER_PGM_RSRC4_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_ADDR_LO_GS                                                              0x0c82
+#define regSPI_SHADER_USER_DATA_ADDR_LO_GS_BASE_IDX                                                     0
+#define regSPI_SHADER_USER_DATA_ADDR_HI_GS                                                              0x0c83
+#define regSPI_SHADER_USER_DATA_ADDR_HI_GS_BASE_IDX                                                     0
+#define regSPI_SHADER_PGM_LO_ES                                                                         0x0c84
+#define regSPI_SHADER_PGM_LO_ES_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_ES                                                                         0x0c85
+#define regSPI_SHADER_PGM_HI_ES_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC3_GS                                                                      0x0c87
+#define regSPI_SHADER_PGM_RSRC3_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_LO_GS                                                                         0x0c88
+#define regSPI_SHADER_PGM_LO_GS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_GS                                                                         0x0c89
+#define regSPI_SHADER_PGM_HI_GS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC1_GS                                                                      0x0c8a
+#define regSPI_SHADER_PGM_RSRC1_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC2_GS                                                                      0x0c8b
+#define regSPI_SHADER_PGM_RSRC2_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_ES_0                                                                    0x0ccc
+#define regSPI_SHADER_USER_DATA_ES_0_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_ES_1                                                                    0x0ccd
+#define regSPI_SHADER_USER_DATA_ES_1_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_ES_2                                                                    0x0cce
+#define regSPI_SHADER_USER_DATA_ES_2_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_ES_3                                                                    0x0ccf
+#define regSPI_SHADER_USER_DATA_ES_3_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_ES_4                                                                    0x0cd0
+#define regSPI_SHADER_USER_DATA_ES_4_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_ES_5                                                                    0x0cd1
+#define regSPI_SHADER_USER_DATA_ES_5_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_ES_6                                                                    0x0cd2
+#define regSPI_SHADER_USER_DATA_ES_6_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_ES_7                                                                    0x0cd3
+#define regSPI_SHADER_USER_DATA_ES_7_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_ES_8                                                                    0x0cd4
+#define regSPI_SHADER_USER_DATA_ES_8_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_ES_9                                                                    0x0cd5
+#define regSPI_SHADER_USER_DATA_ES_9_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_ES_10                                                                   0x0cd6
+#define regSPI_SHADER_USER_DATA_ES_10_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_11                                                                   0x0cd7
+#define regSPI_SHADER_USER_DATA_ES_11_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_12                                                                   0x0cd8
+#define regSPI_SHADER_USER_DATA_ES_12_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_13                                                                   0x0cd9
+#define regSPI_SHADER_USER_DATA_ES_13_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_14                                                                   0x0cda
+#define regSPI_SHADER_USER_DATA_ES_14_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_15                                                                   0x0cdb
+#define regSPI_SHADER_USER_DATA_ES_15_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_16                                                                   0x0cdc
+#define regSPI_SHADER_USER_DATA_ES_16_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_17                                                                   0x0cdd
+#define regSPI_SHADER_USER_DATA_ES_17_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_18                                                                   0x0cde
+#define regSPI_SHADER_USER_DATA_ES_18_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_19                                                                   0x0cdf
+#define regSPI_SHADER_USER_DATA_ES_19_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_20                                                                   0x0ce0
+#define regSPI_SHADER_USER_DATA_ES_20_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_21                                                                   0x0ce1
+#define regSPI_SHADER_USER_DATA_ES_21_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_22                                                                   0x0ce2
+#define regSPI_SHADER_USER_DATA_ES_22_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_23                                                                   0x0ce3
+#define regSPI_SHADER_USER_DATA_ES_23_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_24                                                                   0x0ce4
+#define regSPI_SHADER_USER_DATA_ES_24_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_25                                                                   0x0ce5
+#define regSPI_SHADER_USER_DATA_ES_25_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_26                                                                   0x0ce6
+#define regSPI_SHADER_USER_DATA_ES_26_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_27                                                                   0x0ce7
+#define regSPI_SHADER_USER_DATA_ES_27_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_28                                                                   0x0ce8
+#define regSPI_SHADER_USER_DATA_ES_28_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_29                                                                   0x0ce9
+#define regSPI_SHADER_USER_DATA_ES_29_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_30                                                                   0x0cea
+#define regSPI_SHADER_USER_DATA_ES_30_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_ES_31                                                                   0x0ceb
+#define regSPI_SHADER_USER_DATA_ES_31_BASE_IDX                                                          0
+#define regSPI_SHADER_PGM_RSRC4_HS                                                                      0x0d01
+#define regSPI_SHADER_PGM_RSRC4_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_ADDR_LO_HS                                                              0x0d02
+#define regSPI_SHADER_USER_DATA_ADDR_LO_HS_BASE_IDX                                                     0
+#define regSPI_SHADER_USER_DATA_ADDR_HI_HS                                                              0x0d03
+#define regSPI_SHADER_USER_DATA_ADDR_HI_HS_BASE_IDX                                                     0
+#define regSPI_SHADER_PGM_LO_LS                                                                         0x0d04
+#define regSPI_SHADER_PGM_LO_LS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_LS                                                                         0x0d05
+#define regSPI_SHADER_PGM_HI_LS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC3_HS                                                                      0x0d07
+#define regSPI_SHADER_PGM_RSRC3_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_LO_HS                                                                         0x0d08
+#define regSPI_SHADER_PGM_LO_HS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_HS                                                                         0x0d09
+#define regSPI_SHADER_PGM_HI_HS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC1_HS                                                                      0x0d0a
+#define regSPI_SHADER_PGM_RSRC1_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC2_HS                                                                      0x0d0b
+#define regSPI_SHADER_PGM_RSRC2_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_LS_0                                                                    0x0d0c
+#define regSPI_SHADER_USER_DATA_LS_0_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_LS_1                                                                    0x0d0d
+#define regSPI_SHADER_USER_DATA_LS_1_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_LS_2                                                                    0x0d0e
+#define regSPI_SHADER_USER_DATA_LS_2_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_LS_3                                                                    0x0d0f
+#define regSPI_SHADER_USER_DATA_LS_3_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_LS_4                                                                    0x0d10
+#define regSPI_SHADER_USER_DATA_LS_4_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_LS_5                                                                    0x0d11
+#define regSPI_SHADER_USER_DATA_LS_5_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_LS_6                                                                    0x0d12
+#define regSPI_SHADER_USER_DATA_LS_6_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_LS_7                                                                    0x0d13
+#define regSPI_SHADER_USER_DATA_LS_7_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_LS_8                                                                    0x0d14
+#define regSPI_SHADER_USER_DATA_LS_8_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_LS_9                                                                    0x0d15
+#define regSPI_SHADER_USER_DATA_LS_9_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_LS_10                                                                   0x0d16
+#define regSPI_SHADER_USER_DATA_LS_10_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_11                                                                   0x0d17
+#define regSPI_SHADER_USER_DATA_LS_11_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_12                                                                   0x0d18
+#define regSPI_SHADER_USER_DATA_LS_12_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_13                                                                   0x0d19
+#define regSPI_SHADER_USER_DATA_LS_13_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_14                                                                   0x0d1a
+#define regSPI_SHADER_USER_DATA_LS_14_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_15                                                                   0x0d1b
+#define regSPI_SHADER_USER_DATA_LS_15_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_16                                                                   0x0d1c
+#define regSPI_SHADER_USER_DATA_LS_16_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_17                                                                   0x0d1d
+#define regSPI_SHADER_USER_DATA_LS_17_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_18                                                                   0x0d1e
+#define regSPI_SHADER_USER_DATA_LS_18_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_19                                                                   0x0d1f
+#define regSPI_SHADER_USER_DATA_LS_19_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_20                                                                   0x0d20
+#define regSPI_SHADER_USER_DATA_LS_20_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_21                                                                   0x0d21
+#define regSPI_SHADER_USER_DATA_LS_21_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_22                                                                   0x0d22
+#define regSPI_SHADER_USER_DATA_LS_22_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_23                                                                   0x0d23
+#define regSPI_SHADER_USER_DATA_LS_23_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_24                                                                   0x0d24
+#define regSPI_SHADER_USER_DATA_LS_24_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_25                                                                   0x0d25
+#define regSPI_SHADER_USER_DATA_LS_25_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_26                                                                   0x0d26
+#define regSPI_SHADER_USER_DATA_LS_26_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_27                                                                   0x0d27
+#define regSPI_SHADER_USER_DATA_LS_27_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_28                                                                   0x0d28
+#define regSPI_SHADER_USER_DATA_LS_28_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_29                                                                   0x0d29
+#define regSPI_SHADER_USER_DATA_LS_29_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_30                                                                   0x0d2a
+#define regSPI_SHADER_USER_DATA_LS_30_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_LS_31                                                                   0x0d2b
+#define regSPI_SHADER_USER_DATA_LS_31_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_COMMON_0                                                                0x0d4c
+#define regSPI_SHADER_USER_DATA_COMMON_0_BASE_IDX                                                       0
+#define regSPI_SHADER_USER_DATA_COMMON_1                                                                0x0d4d
+#define regSPI_SHADER_USER_DATA_COMMON_1_BASE_IDX                                                       0
+#define regSPI_SHADER_USER_DATA_COMMON_2                                                                0x0d4e
+#define regSPI_SHADER_USER_DATA_COMMON_2_BASE_IDX                                                       0
+#define regSPI_SHADER_USER_DATA_COMMON_3                                                                0x0d4f
+#define regSPI_SHADER_USER_DATA_COMMON_3_BASE_IDX                                                       0
+#define regSPI_SHADER_USER_DATA_COMMON_4                                                                0x0d50
+#define regSPI_SHADER_USER_DATA_COMMON_4_BASE_IDX                                                       0
+#define regSPI_SHADER_USER_DATA_COMMON_5                                                                0x0d51
+#define regSPI_SHADER_USER_DATA_COMMON_5_BASE_IDX                                                       0
+#define regSPI_SHADER_USER_DATA_COMMON_6                                                                0x0d52
+#define regSPI_SHADER_USER_DATA_COMMON_6_BASE_IDX                                                       0
+#define regSPI_SHADER_USER_DATA_COMMON_7                                                                0x0d53
+#define regSPI_SHADER_USER_DATA_COMMON_7_BASE_IDX                                                       0
+#define regSPI_SHADER_USER_DATA_COMMON_8                                                                0x0d54
+#define regSPI_SHADER_USER_DATA_COMMON_8_BASE_IDX                                                       0
+#define regSPI_SHADER_USER_DATA_COMMON_9                                                                0x0d55
+#define regSPI_SHADER_USER_DATA_COMMON_9_BASE_IDX                                                       0
+#define regSPI_SHADER_USER_DATA_COMMON_10                                                               0x0d56
+#define regSPI_SHADER_USER_DATA_COMMON_10_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_11                                                               0x0d57
+#define regSPI_SHADER_USER_DATA_COMMON_11_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_12                                                               0x0d58
+#define regSPI_SHADER_USER_DATA_COMMON_12_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_13                                                               0x0d59
+#define regSPI_SHADER_USER_DATA_COMMON_13_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_14                                                               0x0d5a
+#define regSPI_SHADER_USER_DATA_COMMON_14_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_15                                                               0x0d5b
+#define regSPI_SHADER_USER_DATA_COMMON_15_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_16                                                               0x0d5c
+#define regSPI_SHADER_USER_DATA_COMMON_16_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_17                                                               0x0d5d
+#define regSPI_SHADER_USER_DATA_COMMON_17_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_18                                                               0x0d5e
+#define regSPI_SHADER_USER_DATA_COMMON_18_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_19                                                               0x0d5f
+#define regSPI_SHADER_USER_DATA_COMMON_19_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_20                                                               0x0d60
+#define regSPI_SHADER_USER_DATA_COMMON_20_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_21                                                               0x0d61
+#define regSPI_SHADER_USER_DATA_COMMON_21_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_22                                                               0x0d62
+#define regSPI_SHADER_USER_DATA_COMMON_22_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_23                                                               0x0d63
+#define regSPI_SHADER_USER_DATA_COMMON_23_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_24                                                               0x0d64
+#define regSPI_SHADER_USER_DATA_COMMON_24_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_25                                                               0x0d65
+#define regSPI_SHADER_USER_DATA_COMMON_25_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_26                                                               0x0d66
+#define regSPI_SHADER_USER_DATA_COMMON_26_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_27                                                               0x0d67
+#define regSPI_SHADER_USER_DATA_COMMON_27_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_28                                                               0x0d68
+#define regSPI_SHADER_USER_DATA_COMMON_28_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_29                                                               0x0d69
+#define regSPI_SHADER_USER_DATA_COMMON_29_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_30                                                               0x0d6a
+#define regSPI_SHADER_USER_DATA_COMMON_30_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_DATA_COMMON_31                                                               0x0d6b
+#define regSPI_SHADER_USER_DATA_COMMON_31_BASE_IDX                                                      0
+#define regCOMPUTE_DISPATCH_INITIATOR                                                                   0x0e00
+#define regCOMPUTE_DISPATCH_INITIATOR_BASE_IDX                                                          0
+#define regCOMPUTE_DIM_X                                                                                0x0e01
+#define regCOMPUTE_DIM_X_BASE_IDX                                                                       0
+#define regCOMPUTE_DIM_Y                                                                                0x0e02
+#define regCOMPUTE_DIM_Y_BASE_IDX                                                                       0
+#define regCOMPUTE_DIM_Z                                                                                0x0e03
+#define regCOMPUTE_DIM_Z_BASE_IDX                                                                       0
+#define regCOMPUTE_START_X                                                                              0x0e04
+#define regCOMPUTE_START_X_BASE_IDX                                                                     0
+#define regCOMPUTE_START_Y                                                                              0x0e05
+#define regCOMPUTE_START_Y_BASE_IDX                                                                     0
+#define regCOMPUTE_START_Z                                                                              0x0e06
+#define regCOMPUTE_START_Z_BASE_IDX                                                                     0
+#define regCOMPUTE_NUM_THREAD_X                                                                         0x0e07
+#define regCOMPUTE_NUM_THREAD_X_BASE_IDX                                                                0
+#define regCOMPUTE_NUM_THREAD_Y                                                                         0x0e08
+#define regCOMPUTE_NUM_THREAD_Y_BASE_IDX                                                                0
+#define regCOMPUTE_NUM_THREAD_Z                                                                         0x0e09
+#define regCOMPUTE_NUM_THREAD_Z_BASE_IDX                                                                0
+#define regCOMPUTE_PIPELINESTAT_ENABLE                                                                  0x0e0a
+#define regCOMPUTE_PIPELINESTAT_ENABLE_BASE_IDX                                                         0
+#define regCOMPUTE_PERFCOUNT_ENABLE                                                                     0x0e0b
+#define regCOMPUTE_PERFCOUNT_ENABLE_BASE_IDX                                                            0
+#define regCOMPUTE_PGM_LO                                                                               0x0e0c
+#define regCOMPUTE_PGM_LO_BASE_IDX                                                                      0
+#define regCOMPUTE_PGM_HI                                                                               0x0e0d
+#define regCOMPUTE_PGM_HI_BASE_IDX                                                                      0
+#define regCOMPUTE_DISPATCH_PKT_ADDR_LO                                                                 0x0e0e
+#define regCOMPUTE_DISPATCH_PKT_ADDR_LO_BASE_IDX                                                        0
+#define regCOMPUTE_DISPATCH_PKT_ADDR_HI                                                                 0x0e0f
+#define regCOMPUTE_DISPATCH_PKT_ADDR_HI_BASE_IDX                                                        0
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_LO                                                             0x0e10
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_LO_BASE_IDX                                                    0
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_HI                                                             0x0e11
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_HI_BASE_IDX                                                    0
+#define regCOMPUTE_PGM_RSRC1                                                                            0x0e12
+#define regCOMPUTE_PGM_RSRC1_BASE_IDX                                                                   0
+#define regCOMPUTE_PGM_RSRC2                                                                            0x0e13
+#define regCOMPUTE_PGM_RSRC2_BASE_IDX                                                                   0
+#define regCOMPUTE_VMID                                                                                 0x0e14
+#define regCOMPUTE_VMID_BASE_IDX                                                                        0
+#define regCOMPUTE_RESOURCE_LIMITS                                                                      0x0e15
+#define regCOMPUTE_RESOURCE_LIMITS_BASE_IDX                                                             0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE0                                                               0x0e16
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE0_BASE_IDX                                                      0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE1                                                               0x0e17
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE1_BASE_IDX                                                      0
+#define regCOMPUTE_TMPRING_SIZE                                                                         0x0e18
+#define regCOMPUTE_TMPRING_SIZE_BASE_IDX                                                                0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE2                                                               0x0e19
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE2_BASE_IDX                                                      0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE3                                                               0x0e1a
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE3_BASE_IDX                                                      0
+#define regCOMPUTE_RESTART_X                                                                            0x0e1b
+#define regCOMPUTE_RESTART_X_BASE_IDX                                                                   0
+#define regCOMPUTE_RESTART_Y                                                                            0x0e1c
+#define regCOMPUTE_RESTART_Y_BASE_IDX                                                                   0
+#define regCOMPUTE_RESTART_Z                                                                            0x0e1d
+#define regCOMPUTE_RESTART_Z_BASE_IDX                                                                   0
+#define regCOMPUTE_THREAD_TRACE_ENABLE                                                                  0x0e1e
+#define regCOMPUTE_THREAD_TRACE_ENABLE_BASE_IDX                                                         0
+#define regCOMPUTE_MISC_RESERVED                                                                        0x0e1f
+#define regCOMPUTE_MISC_RESERVED_BASE_IDX                                                               0
+#define regCOMPUTE_DISPATCH_ID                                                                          0x0e20
+#define regCOMPUTE_DISPATCH_ID_BASE_IDX                                                                 0
+#define regCOMPUTE_THREADGROUP_ID                                                                       0x0e21
+#define regCOMPUTE_THREADGROUP_ID_BASE_IDX                                                              0
+#define regCOMPUTE_RELAUNCH                                                                             0x0e22
+#define regCOMPUTE_RELAUNCH_BASE_IDX                                                                    0
+#define regCOMPUTE_WAVE_RESTORE_ADDR_LO                                                                 0x0e23
+#define regCOMPUTE_WAVE_RESTORE_ADDR_LO_BASE_IDX                                                        0
+#define regCOMPUTE_WAVE_RESTORE_ADDR_HI                                                                 0x0e24
+#define regCOMPUTE_WAVE_RESTORE_ADDR_HI_BASE_IDX                                                        0
+#define regCOMPUTE_TG_CHUNK_SIZE                                                                        0x0e27
+#define regCOMPUTE_TG_CHUNK_SIZE_BASE_IDX                                                               0
+#define regCOMPUTE_SHADER_CHKSUM                                                                        0x0e2c
+#define regCOMPUTE_SHADER_CHKSUM_BASE_IDX                                                               0
+#define regCOMPUTE_PGM_RSRC3                                                                            0x0e2d
+#define regCOMPUTE_PGM_RSRC3_BASE_IDX                                                                   0
+#define regCOMPUTE_USER_DATA_0                                                                          0x0e40
+#define regCOMPUTE_USER_DATA_0_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_1                                                                          0x0e41
+#define regCOMPUTE_USER_DATA_1_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_2                                                                          0x0e42
+#define regCOMPUTE_USER_DATA_2_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_3                                                                          0x0e43
+#define regCOMPUTE_USER_DATA_3_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_4                                                                          0x0e44
+#define regCOMPUTE_USER_DATA_4_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_5                                                                          0x0e45
+#define regCOMPUTE_USER_DATA_5_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_6                                                                          0x0e46
+#define regCOMPUTE_USER_DATA_6_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_7                                                                          0x0e47
+#define regCOMPUTE_USER_DATA_7_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_8                                                                          0x0e48
+#define regCOMPUTE_USER_DATA_8_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_9                                                                          0x0e49
+#define regCOMPUTE_USER_DATA_9_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_10                                                                         0x0e4a
+#define regCOMPUTE_USER_DATA_10_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_11                                                                         0x0e4b
+#define regCOMPUTE_USER_DATA_11_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_12                                                                         0x0e4c
+#define regCOMPUTE_USER_DATA_12_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_13                                                                         0x0e4d
+#define regCOMPUTE_USER_DATA_13_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_14                                                                         0x0e4e
+#define regCOMPUTE_USER_DATA_14_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_15                                                                         0x0e4f
+#define regCOMPUTE_USER_DATA_15_BASE_IDX                                                                0
+#define regCOMPUTE_DISPATCH_END                                                                         0x0e7e
+#define regCOMPUTE_DISPATCH_END_BASE_IDX                                                                0
+#define regCOMPUTE_NOWHERE                                                                              0x0e7f
+#define regCOMPUTE_NOWHERE_BASE_IDX                                                                     0
+
+
+// addressBlock: xcd0_gc_cppdec
+// base address: 0xc080
+#define regCP_DFY_CNTL                                                                                  0x1020
+#define regCP_DFY_CNTL_BASE_IDX                                                                         0
+#define regCP_DFY_STAT                                                                                  0x1021
+#define regCP_DFY_STAT_BASE_IDX                                                                         0
+#define regCP_DFY_ADDR_HI                                                                               0x1022
+#define regCP_DFY_ADDR_HI_BASE_IDX                                                                      0
+#define regCP_DFY_ADDR_LO                                                                               0x1023
+#define regCP_DFY_ADDR_LO_BASE_IDX                                                                      0
+#define regCP_DFY_DATA_0                                                                                0x1024
+#define regCP_DFY_DATA_0_BASE_IDX                                                                       0
+#define regCP_DFY_DATA_1                                                                                0x1025
+#define regCP_DFY_DATA_1_BASE_IDX                                                                       0
+#define regCP_DFY_DATA_2                                                                                0x1026
+#define regCP_DFY_DATA_2_BASE_IDX                                                                       0
+#define regCP_DFY_DATA_3                                                                                0x1027
+#define regCP_DFY_DATA_3_BASE_IDX                                                                       0
+#define regCP_DFY_DATA_4                                                                                0x1028
+#define regCP_DFY_DATA_4_BASE_IDX                                                                       0
+#define regCP_DFY_DATA_5                                                                                0x1029
+#define regCP_DFY_DATA_5_BASE_IDX                                                                       0
+#define regCP_DFY_DATA_6                                                                                0x102a
+#define regCP_DFY_DATA_6_BASE_IDX                                                                       0
+#define regCP_DFY_DATA_7                                                                                0x102b
+#define regCP_DFY_DATA_7_BASE_IDX                                                                       0
+#define regCP_DFY_DATA_8                                                                                0x102c
+#define regCP_DFY_DATA_8_BASE_IDX                                                                       0
+#define regCP_DFY_DATA_9                                                                                0x102d
+#define regCP_DFY_DATA_9_BASE_IDX                                                                       0
+#define regCP_DFY_DATA_10                                                                               0x102e
+#define regCP_DFY_DATA_10_BASE_IDX                                                                      0
+#define regCP_DFY_DATA_11                                                                               0x102f
+#define regCP_DFY_DATA_11_BASE_IDX                                                                      0
+#define regCP_DFY_DATA_12                                                                               0x1030
+#define regCP_DFY_DATA_12_BASE_IDX                                                                      0
+#define regCP_DFY_DATA_13                                                                               0x1031
+#define regCP_DFY_DATA_13_BASE_IDX                                                                      0
+#define regCP_DFY_DATA_14                                                                               0x1032
+#define regCP_DFY_DATA_14_BASE_IDX                                                                      0
+#define regCP_DFY_DATA_15                                                                               0x1033
+#define regCP_DFY_DATA_15_BASE_IDX                                                                      0
+#define regCP_DFY_CMD                                                                                   0x1034
+#define regCP_DFY_CMD_BASE_IDX                                                                          0
+#define regCP_EOPQ_WAIT_TIME                                                                            0x1035
+#define regCP_EOPQ_WAIT_TIME_BASE_IDX                                                                   0
+#define regCP_CPC_MGCG_SYNC_CNTL                                                                        0x1036
+#define regCP_CPC_MGCG_SYNC_CNTL_BASE_IDX                                                               0
+#define regCPC_INT_INFO                                                                                 0x1037
+#define regCPC_INT_INFO_BASE_IDX                                                                        0
+#define regCP_VIRT_STATUS                                                                               0x1038
+#define regCP_VIRT_STATUS_BASE_IDX                                                                      0
+#define regCPC_INT_ADDR                                                                                 0x1039
+#define regCPC_INT_ADDR_BASE_IDX                                                                        0
+#define regCPC_INT_PASID                                                                                0x103a
+#define regCPC_INT_PASID_BASE_IDX                                                                       0
+#define regCP_GFX_ERROR                                                                                 0x103b
+#define regCP_GFX_ERROR_BASE_IDX                                                                        0
+#define regCPG_UTCL1_CNTL                                                                               0x103c
+#define regCPG_UTCL1_CNTL_BASE_IDX                                                                      0
+#define regCPC_UTCL1_CNTL                                                                               0x103d
+#define regCPC_UTCL1_CNTL_BASE_IDX                                                                      0
+#define regCPF_UTCL1_CNTL                                                                               0x103e
+#define regCPF_UTCL1_CNTL_BASE_IDX                                                                      0
+#define regCP_AQL_SMM_STATUS                                                                            0x103f
+#define regCP_AQL_SMM_STATUS_BASE_IDX                                                                   0
+#define regCP_RB0_BASE                                                                                  0x1040
+#define regCP_RB0_BASE_BASE_IDX                                                                         0
+#define regCP_RB_BASE                                                                                   0x1040
+#define regCP_RB_BASE_BASE_IDX                                                                          0
+#define regCP_RB0_CNTL                                                                                  0x1041
+#define regCP_RB0_CNTL_BASE_IDX                                                                         0
+#define regCP_RB_CNTL                                                                                   0x1041
+#define regCP_RB_CNTL_BASE_IDX                                                                          0
+#define regCP_RB_RPTR_WR                                                                                0x1042
+#define regCP_RB_RPTR_WR_BASE_IDX                                                                       0
+#define regCP_RB0_RPTR_ADDR                                                                             0x1043
+#define regCP_RB0_RPTR_ADDR_BASE_IDX                                                                    0
+#define regCP_RB_RPTR_ADDR                                                                              0x1043
+#define regCP_RB_RPTR_ADDR_BASE_IDX                                                                     0
+#define regCP_RB0_RPTR_ADDR_HI                                                                          0x1044
+#define regCP_RB0_RPTR_ADDR_HI_BASE_IDX                                                                 0
+#define regCP_RB_RPTR_ADDR_HI                                                                           0x1044
+#define regCP_RB_RPTR_ADDR_HI_BASE_IDX                                                                  0
+#define regCP_RB0_BUFSZ_MASK                                                                            0x1045
+#define regCP_RB0_BUFSZ_MASK_BASE_IDX                                                                   0
+#define regCP_RB_BUFSZ_MASK                                                                             0x1045
+#define regCP_RB_BUFSZ_MASK_BASE_IDX                                                                    0
+#define regCP_RB_WPTR_POLL_ADDR_LO                                                                      0x1046
+#define regCP_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                             0
+#define regCP_RB_WPTR_POLL_ADDR_HI                                                                      0x1047
+#define regCP_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                             0
+#define regGC_PRIV_MODE                                                                                 0x1048
+#define regGC_PRIV_MODE_BASE_IDX                                                                        0
+#define regCP_INT_CNTL                                                                                  0x1049
+#define regCP_INT_CNTL_BASE_IDX                                                                         0
+#define regCP_INT_STATUS                                                                                0x104a
+#define regCP_INT_STATUS_BASE_IDX                                                                       0
+#define regCP_DEVICE_ID                                                                                 0x104b
+#define regCP_DEVICE_ID_BASE_IDX                                                                        0
+#define regCP_ME0_PIPE_PRIORITY_CNTS                                                                    0x104c
+#define regCP_ME0_PIPE_PRIORITY_CNTS_BASE_IDX                                                           0
+#define regCP_RING_PRIORITY_CNTS                                                                        0x104c
+#define regCP_RING_PRIORITY_CNTS_BASE_IDX                                                               0
+#define regCP_ME0_PIPE0_PRIORITY                                                                        0x104d
+#define regCP_ME0_PIPE0_PRIORITY_BASE_IDX                                                               0
+#define regCP_RING0_PRIORITY                                                                            0x104d
+#define regCP_RING0_PRIORITY_BASE_IDX                                                                   0
+#define regCP_ME0_PIPE1_PRIORITY                                                                        0x104e
+#define regCP_ME0_PIPE1_PRIORITY_BASE_IDX                                                               0
+#define regCP_RING1_PRIORITY                                                                            0x104e
+#define regCP_RING1_PRIORITY_BASE_IDX                                                                   0
+#define regCP_ME0_PIPE2_PRIORITY                                                                        0x104f
+#define regCP_ME0_PIPE2_PRIORITY_BASE_IDX                                                               0
+#define regCP_RING2_PRIORITY                                                                            0x104f
+#define regCP_RING2_PRIORITY_BASE_IDX                                                                   0
+#define regCP_FATAL_ERROR                                                                               0x1050
+#define regCP_FATAL_ERROR_BASE_IDX                                                                      0
+#define regCP_RB_VMID                                                                                   0x1051
+#define regCP_RB_VMID_BASE_IDX                                                                          0
+#define regCP_ME0_PIPE0_VMID                                                                            0x1052
+#define regCP_ME0_PIPE0_VMID_BASE_IDX                                                                   0
+#define regCP_ME0_PIPE1_VMID                                                                            0x1053
+#define regCP_ME0_PIPE1_VMID_BASE_IDX                                                                   0
+#define regCP_RB0_WPTR                                                                                  0x1054
+#define regCP_RB0_WPTR_BASE_IDX                                                                         0
+#define regCP_RB_WPTR                                                                                   0x1054
+#define regCP_RB_WPTR_BASE_IDX                                                                          0
+#define regCP_RB0_WPTR_HI                                                                               0x1055
+#define regCP_RB0_WPTR_HI_BASE_IDX                                                                      0
+#define regCP_RB_WPTR_HI                                                                                0x1055
+#define regCP_RB_WPTR_HI_BASE_IDX                                                                       0
+#define regCP_RB1_WPTR                                                                                  0x1056
+#define regCP_RB1_WPTR_BASE_IDX                                                                         0
+#define regCP_RB1_WPTR_HI                                                                               0x1057
+#define regCP_RB1_WPTR_HI_BASE_IDX                                                                      0
+#define regCP_RB2_WPTR                                                                                  0x1058
+#define regCP_RB2_WPTR_BASE_IDX                                                                         0
+#define regCP_RB_DOORBELL_CONTROL                                                                       0x1059
+#define regCP_RB_DOORBELL_CONTROL_BASE_IDX                                                              0
+#define regCP_RB_DOORBELL_RANGE_LOWER                                                                   0x105a
+#define regCP_RB_DOORBELL_RANGE_LOWER_BASE_IDX                                                          0
+#define regCP_RB_DOORBELL_RANGE_UPPER                                                                   0x105b
+#define regCP_RB_DOORBELL_RANGE_UPPER_BASE_IDX                                                          0
+#define regCP_MEC_DOORBELL_RANGE_LOWER                                                                  0x105c
+#define regCP_MEC_DOORBELL_RANGE_LOWER_BASE_IDX                                                         0
+#define regCP_MEC_DOORBELL_RANGE_UPPER                                                                  0x105d
+#define regCP_MEC_DOORBELL_RANGE_UPPER_BASE_IDX                                                         0
+#define regCPG_UTCL1_ERROR                                                                              0x105e
+#define regCPG_UTCL1_ERROR_BASE_IDX                                                                     0
+#define regCPC_UTCL1_ERROR                                                                              0x105f
+#define regCPC_UTCL1_ERROR_BASE_IDX                                                                     0
+#define regCP_RB1_BASE                                                                                  0x1060
+#define regCP_RB1_BASE_BASE_IDX                                                                         0
+#define regCP_RB1_CNTL                                                                                  0x1061
+#define regCP_RB1_CNTL_BASE_IDX                                                                         0
+#define regCP_RB1_RPTR_ADDR                                                                             0x1062
+#define regCP_RB1_RPTR_ADDR_BASE_IDX                                                                    0
+#define regCP_RB1_RPTR_ADDR_HI                                                                          0x1063
+#define regCP_RB1_RPTR_ADDR_HI_BASE_IDX                                                                 0
+#define regCP_RB2_BASE                                                                                  0x1065
+#define regCP_RB2_BASE_BASE_IDX                                                                         0
+#define regCP_RB2_CNTL                                                                                  0x1066
+#define regCP_RB2_CNTL_BASE_IDX                                                                         0
+#define regCP_RB2_RPTR_ADDR                                                                             0x1067
+#define regCP_RB2_RPTR_ADDR_BASE_IDX                                                                    0
+#define regCP_RB2_RPTR_ADDR_HI                                                                          0x1068
+#define regCP_RB2_RPTR_ADDR_HI_BASE_IDX                                                                 0
+#define regCP_RB0_ACTIVE                                                                                0x1069
+#define regCP_RB0_ACTIVE_BASE_IDX                                                                       0
+#define regCP_RB_ACTIVE                                                                                 0x1069
+#define regCP_RB_ACTIVE_BASE_IDX                                                                        0
+#define regCP_INT_CNTL_RING0                                                                            0x106a
+#define regCP_INT_CNTL_RING0_BASE_IDX                                                                   0
+#define regCP_INT_CNTL_RING1                                                                            0x106b
+#define regCP_INT_CNTL_RING1_BASE_IDX                                                                   0
+#define regCP_INT_CNTL_RING2                                                                            0x106c
+#define regCP_INT_CNTL_RING2_BASE_IDX                                                                   0
+#define regCP_INT_STATUS_RING0                                                                          0x106d
+#define regCP_INT_STATUS_RING0_BASE_IDX                                                                 0
+#define regCP_INT_STATUS_RING1                                                                          0x106e
+#define regCP_INT_STATUS_RING1_BASE_IDX                                                                 0
+#define regCP_INT_STATUS_RING2                                                                          0x106f
+#define regCP_INT_STATUS_RING2_BASE_IDX                                                                 0
+#define regCP_ME_F32_INTERRUPT                                                                          0x1073
+#define regCP_ME_F32_INTERRUPT_BASE_IDX                                                                 0
+#define regCP_PFP_F32_INTERRUPT                                                                         0x1074
+#define regCP_PFP_F32_INTERRUPT_BASE_IDX                                                                0
+#define regCP_CE_F32_INTERRUPT                                                                          0x1075
+#define regCP_CE_F32_INTERRUPT_BASE_IDX                                                                 0
+#define regCP_MEC1_F32_INTERRUPT                                                                        0x1076
+#define regCP_MEC1_F32_INTERRUPT_BASE_IDX                                                               0
+#define regCP_MEC2_F32_INTERRUPT                                                                        0x1077
+#define regCP_MEC2_F32_INTERRUPT_BASE_IDX                                                               0
+#define regCP_PWR_CNTL                                                                                  0x1078
+#define regCP_PWR_CNTL_BASE_IDX                                                                         0
+#define regCP_MEM_SLP_CNTL                                                                              0x1079
+#define regCP_MEM_SLP_CNTL_BASE_IDX                                                                     0
+#define regCP_ECC_DMA_FIRST_OCCURRENCE                                                                  0x107a
+#define regCP_ECC_DMA_FIRST_OCCURRENCE_BASE_IDX                                                         0
+#define regCP_ECC_FIRSTOCCURRENCE                                                                       0x107a
+#define regCP_ECC_FIRSTOCCURRENCE_BASE_IDX                                                              0
+#define regCP_ECC_FIRSTOCCURRENCE_RING0                                                                 0x107b
+#define regCP_ECC_FIRSTOCCURRENCE_RING0_BASE_IDX                                                        0
+#define regCP_ECC_FIRSTOCCURRENCE_RING1                                                                 0x107c
+#define regCP_ECC_FIRSTOCCURRENCE_RING1_BASE_IDX                                                        0
+#define regCP_ECC_FIRSTOCCURRENCE_RING2                                                                 0x107d
+#define regCP_ECC_FIRSTOCCURRENCE_RING2_BASE_IDX                                                        0
+#define regGB_EDC_MODE                                                                                  0x107e
+#define regGB_EDC_MODE_BASE_IDX                                                                         0
+#define regCP_DEBUG                                                                                     0x107f
+#define regCP_DEBUG_BASE_IDX                                                                            0
+#define regCP_CPF_DEBUG                                                                                 0x1080
+#define regCP_CPF_DEBUG_BASE_IDX                                                                        0
+#define regCP_CPC_DEBUG                                                                                 0x1081
+#define regCP_CPC_DEBUG_BASE_IDX                                                                        0
+#define regCP_CPC_DEBUG_2                                                                               0x1082
+#define regCP_CPC_DEBUG_2_BASE_IDX                                                                      0
+#define regCP_PQ_WPTR_POLL_CNTL                                                                         0x1083
+#define regCP_PQ_WPTR_POLL_CNTL_BASE_IDX                                                                0
+#define regCP_PQ_WPTR_POLL_CNTL1                                                                        0x1084
+#define regCP_PQ_WPTR_POLL_CNTL1_BASE_IDX                                                               0
+#define regCP_ME1_PIPE0_INT_CNTL                                                                        0x1085
+#define regCP_ME1_PIPE0_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE1_INT_CNTL                                                                        0x1086
+#define regCP_ME1_PIPE1_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE2_INT_CNTL                                                                        0x1087
+#define regCP_ME1_PIPE2_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE3_INT_CNTL                                                                        0x1088
+#define regCP_ME1_PIPE3_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME2_PIPE0_INT_CNTL                                                                        0x1089
+#define regCP_ME2_PIPE0_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME2_PIPE1_INT_CNTL                                                                        0x108a
+#define regCP_ME2_PIPE1_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME2_PIPE2_INT_CNTL                                                                        0x108b
+#define regCP_ME2_PIPE2_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME2_PIPE3_INT_CNTL                                                                        0x108c
+#define regCP_ME2_PIPE3_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE0_INT_STATUS                                                                      0x108d
+#define regCP_ME1_PIPE0_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_PIPE1_INT_STATUS                                                                      0x108e
+#define regCP_ME1_PIPE1_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_PIPE2_INT_STATUS                                                                      0x108f
+#define regCP_ME1_PIPE2_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_PIPE3_INT_STATUS                                                                      0x1090
+#define regCP_ME1_PIPE3_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME2_PIPE0_INT_STATUS                                                                      0x1091
+#define regCP_ME2_PIPE0_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME2_PIPE1_INT_STATUS                                                                      0x1092
+#define regCP_ME2_PIPE1_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME2_PIPE2_INT_STATUS                                                                      0x1093
+#define regCP_ME2_PIPE2_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME2_PIPE3_INT_STATUS                                                                      0x1094
+#define regCP_ME2_PIPE3_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_INT_STAT_DEBUG                                                                        0x1095
+#define regCP_ME1_INT_STAT_DEBUG_BASE_IDX                                                               0
+#define regCP_ME2_INT_STAT_DEBUG                                                                        0x1096
+#define regCP_ME2_INT_STAT_DEBUG_BASE_IDX                                                               0
+#define regCC_GC_EDC_CONFIG                                                                             0x1098
+#define regCC_GC_EDC_CONFIG_BASE_IDX                                                                    0
+#define regCP_ME1_PIPE_PRIORITY_CNTS                                                                    0x1099
+#define regCP_ME1_PIPE_PRIORITY_CNTS_BASE_IDX                                                           0
+#define regCP_ME1_PIPE0_PRIORITY                                                                        0x109a
+#define regCP_ME1_PIPE0_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME1_PIPE1_PRIORITY                                                                        0x109b
+#define regCP_ME1_PIPE1_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME1_PIPE2_PRIORITY                                                                        0x109c
+#define regCP_ME1_PIPE2_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME1_PIPE3_PRIORITY                                                                        0x109d
+#define regCP_ME1_PIPE3_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME2_PIPE_PRIORITY_CNTS                                                                    0x109e
+#define regCP_ME2_PIPE_PRIORITY_CNTS_BASE_IDX                                                           0
+#define regCP_ME2_PIPE0_PRIORITY                                                                        0x109f
+#define regCP_ME2_PIPE0_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME2_PIPE1_PRIORITY                                                                        0x10a0
+#define regCP_ME2_PIPE1_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME2_PIPE2_PRIORITY                                                                        0x10a1
+#define regCP_ME2_PIPE2_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME2_PIPE3_PRIORITY                                                                        0x10a2
+#define regCP_ME2_PIPE3_PRIORITY_BASE_IDX                                                               0
+#define regCP_CE_PRGRM_CNTR_START                                                                       0x10a3
+#define regCP_CE_PRGRM_CNTR_START_BASE_IDX                                                              0
+#define regCP_PFP_PRGRM_CNTR_START                                                                      0x10a4
+#define regCP_PFP_PRGRM_CNTR_START_BASE_IDX                                                             0
+#define regCP_ME_PRGRM_CNTR_START                                                                       0x10a5
+#define regCP_ME_PRGRM_CNTR_START_BASE_IDX                                                              0
+#define regCP_MEC1_PRGRM_CNTR_START                                                                     0x10a6
+#define regCP_MEC1_PRGRM_CNTR_START_BASE_IDX                                                            0
+#define regCP_MEC2_PRGRM_CNTR_START                                                                     0x10a7
+#define regCP_MEC2_PRGRM_CNTR_START_BASE_IDX                                                            0
+#define regCP_CE_INTR_ROUTINE_START                                                                     0x10a8
+#define regCP_CE_INTR_ROUTINE_START_BASE_IDX                                                            0
+#define regCP_PFP_INTR_ROUTINE_START                                                                    0x10a9
+#define regCP_PFP_INTR_ROUTINE_START_BASE_IDX                                                           0
+#define regCP_ME_INTR_ROUTINE_START                                                                     0x10aa
+#define regCP_ME_INTR_ROUTINE_START_BASE_IDX                                                            0
+#define regCP_MEC1_INTR_ROUTINE_START                                                                   0x10ab
+#define regCP_MEC1_INTR_ROUTINE_START_BASE_IDX                                                          0
+#define regCP_MEC2_INTR_ROUTINE_START                                                                   0x10ac
+#define regCP_MEC2_INTR_ROUTINE_START_BASE_IDX                                                          0
+#define regCP_CONTEXT_CNTL                                                                              0x10ad
+#define regCP_CONTEXT_CNTL_BASE_IDX                                                                     0
+#define regCP_MAX_CONTEXT                                                                               0x10ae
+#define regCP_MAX_CONTEXT_BASE_IDX                                                                      0
+#define regCP_IQ_WAIT_TIME1                                                                             0x10af
+#define regCP_IQ_WAIT_TIME1_BASE_IDX                                                                    0
+#define regCP_IQ_WAIT_TIME2                                                                             0x10b0
+#define regCP_IQ_WAIT_TIME2_BASE_IDX                                                                    0
+#define regCP_RB0_BASE_HI                                                                               0x10b1
+#define regCP_RB0_BASE_HI_BASE_IDX                                                                      0
+#define regCP_RB1_BASE_HI                                                                               0x10b2
+#define regCP_RB1_BASE_HI_BASE_IDX                                                                      0
+#define regCP_VMID_RESET                                                                                0x10b3
+#define regCP_VMID_RESET_BASE_IDX                                                                       0
+#define regCPC_INT_CNTL                                                                                 0x10b4
+#define regCPC_INT_CNTL_BASE_IDX                                                                        0
+#define regCPC_INT_STATUS                                                                               0x10b5
+#define regCPC_INT_STATUS_BASE_IDX                                                                      0
+#define regCP_VMID_PREEMPT                                                                              0x10b6
+#define regCP_VMID_PREEMPT_BASE_IDX                                                                     0
+#define regCPC_INT_CNTX_ID                                                                              0x10b7
+#define regCPC_INT_CNTX_ID_BASE_IDX                                                                     0
+#define regCP_PQ_STATUS                                                                                 0x10b8
+#define regCP_PQ_STATUS_BASE_IDX                                                                        0
+#define regCP_CPC_IC_BASE_LO                                                                            0x10b9
+#define regCP_CPC_IC_BASE_LO_BASE_IDX                                                                   0
+#define regCP_CPC_IC_BASE_HI                                                                            0x10ba
+#define regCP_CPC_IC_BASE_HI_BASE_IDX                                                                   0
+#define regCP_CPC_IC_BASE_CNTL                                                                          0x10bb
+#define regCP_CPC_IC_BASE_CNTL_BASE_IDX                                                                 0
+#define regCP_CPC_IC_OP_CNTL                                                                            0x10bc
+#define regCP_CPC_IC_OP_CNTL_BASE_IDX                                                                   0
+#define regCP_MEC1_F32_INT_DIS                                                                          0x10bd
+#define regCP_MEC1_F32_INT_DIS_BASE_IDX                                                                 0
+#define regCP_MEC2_F32_INT_DIS                                                                          0x10be
+#define regCP_MEC2_F32_INT_DIS_BASE_IDX                                                                 0
+#define regCP_VMID_STATUS                                                                               0x10bf
+#define regCP_VMID_STATUS_BASE_IDX                                                                      0
+
+
+// addressBlock: xcd0_gc_cppdec2
+// base address: 0xc600
+#define regCP_RB_DOORBELL_CONTROL_SCH_0                                                                 0x1180
+#define regCP_RB_DOORBELL_CONTROL_SCH_0_BASE_IDX                                                        0
+#define regCP_RB_DOORBELL_CONTROL_SCH_1                                                                 0x1181
+#define regCP_RB_DOORBELL_CONTROL_SCH_1_BASE_IDX                                                        0
+#define regCP_RB_DOORBELL_CONTROL_SCH_2                                                                 0x1182
+#define regCP_RB_DOORBELL_CONTROL_SCH_2_BASE_IDX                                                        0
+#define regCP_RB_DOORBELL_CONTROL_SCH_3                                                                 0x1183
+#define regCP_RB_DOORBELL_CONTROL_SCH_3_BASE_IDX                                                        0
+#define regCP_RB_DOORBELL_CONTROL_SCH_4                                                                 0x1184
+#define regCP_RB_DOORBELL_CONTROL_SCH_4_BASE_IDX                                                        0
+#define regCP_RB_DOORBELL_CONTROL_SCH_5                                                                 0x1185
+#define regCP_RB_DOORBELL_CONTROL_SCH_5_BASE_IDX                                                        0
+#define regCP_RB_DOORBELL_CONTROL_SCH_6                                                                 0x1186
+#define regCP_RB_DOORBELL_CONTROL_SCH_6_BASE_IDX                                                        0
+#define regCP_RB_DOORBELL_CONTROL_SCH_7                                                                 0x1187
+#define regCP_RB_DOORBELL_CONTROL_SCH_7_BASE_IDX                                                        0
+#define regCP_RB_DOORBELL_CLEAR                                                                         0x1188
+#define regCP_RB_DOORBELL_CLEAR_BASE_IDX                                                                0
+#define regCP_CPF_DSM_CNTL                                                                              0x1194
+#define regCP_CPF_DSM_CNTL_BASE_IDX                                                                     0
+#define regCP_CPG_DSM_CNTL                                                                              0x1195
+#define regCP_CPG_DSM_CNTL_BASE_IDX                                                                     0
+#define regCP_CPC_DSM_CNTL                                                                              0x1196
+#define regCP_CPC_DSM_CNTL_BASE_IDX                                                                     0
+#define regCP_CPF_DSM_CNTL2                                                                             0x1197
+#define regCP_CPF_DSM_CNTL2_BASE_IDX                                                                    0
+#define regCP_CPG_DSM_CNTL2                                                                             0x1198
+#define regCP_CPG_DSM_CNTL2_BASE_IDX                                                                    0
+#define regCP_CPC_DSM_CNTL2                                                                             0x1199
+#define regCP_CPC_DSM_CNTL2_BASE_IDX                                                                    0
+#define regCP_CPF_DSM_CNTL2A                                                                            0x119a
+#define regCP_CPF_DSM_CNTL2A_BASE_IDX                                                                   0
+#define regCP_CPG_DSM_CNTL2A                                                                            0x119b
+#define regCP_CPG_DSM_CNTL2A_BASE_IDX                                                                   0
+#define regCP_CPC_DSM_CNTL2A                                                                            0x119c
+#define regCP_CPC_DSM_CNTL2A_BASE_IDX                                                                   0
+#define regCP_EDC_FUE_CNTL                                                                              0x119d
+#define regCP_EDC_FUE_CNTL_BASE_IDX                                                                     0
+#define regCP_GFX_MQD_CONTROL                                                                           0x11a0
+#define regCP_GFX_MQD_CONTROL_BASE_IDX                                                                  0
+#define regCP_GFX_MQD_BASE_ADDR                                                                         0x11a1
+#define regCP_GFX_MQD_BASE_ADDR_BASE_IDX                                                                0
+#define regCP_GFX_MQD_BASE_ADDR_HI                                                                      0x11a2
+#define regCP_GFX_MQD_BASE_ADDR_HI_BASE_IDX                                                             0
+#define regCP_RB_STATUS                                                                                 0x11a3
+#define regCP_RB_STATUS_BASE_IDX                                                                        0
+#define regCPG_UTCL1_STATUS                                                                             0x11b4
+#define regCPG_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regCPC_UTCL1_STATUS                                                                             0x11b5
+#define regCPC_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regCPF_UTCL1_STATUS                                                                             0x11b6
+#define regCPF_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regCP_SD_CNTL                                                                                   0x11b7
+#define regCP_SD_CNTL_BASE_IDX                                                                          0
+#define regCP_SOFT_RESET_CNTL                                                                           0x11b9
+#define regCP_SOFT_RESET_CNTL_BASE_IDX                                                                  0
+#define regCP_CPC_GFX_CNTL                                                                              0x11ba
+#define regCP_CPC_GFX_CNTL_BASE_IDX                                                                     0
+
+
+// addressBlock: xcd0_gc_spipdec
+// base address: 0xc700
+#define regSPI_ARB_PRIORITY                                                                             0x11c0
+#define regSPI_ARB_PRIORITY_BASE_IDX                                                                    0
+#define regSPI_ARB_CYCLES_0                                                                             0x11c1
+#define regSPI_ARB_CYCLES_0_BASE_IDX                                                                    0
+#define regSPI_ARB_CYCLES_1                                                                             0x11c2
+#define regSPI_ARB_CYCLES_1_BASE_IDX                                                                    0
+#define regSPI_CDBG_SYS_GFX                                                                             0x11c3
+#define regSPI_CDBG_SYS_GFX_BASE_IDX                                                                    0
+#define regSPI_CDBG_SYS_HP3D                                                                            0x11c4
+#define regSPI_CDBG_SYS_HP3D_BASE_IDX                                                                   0
+#define regSPI_CDBG_SYS_CS0                                                                             0x11c5
+#define regSPI_CDBG_SYS_CS0_BASE_IDX                                                                    0
+#define regSPI_CDBG_SYS_CS1                                                                             0x11c6
+#define regSPI_CDBG_SYS_CS1_BASE_IDX                                                                    0
+#define regSPI_WCL_PIPE_PERCENT_GFX                                                                     0x11c7
+#define regSPI_WCL_PIPE_PERCENT_GFX_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_HP3D                                                                    0x11c8
+#define regSPI_WCL_PIPE_PERCENT_HP3D_BASE_IDX                                                           0
+#define regSPI_WCL_PIPE_PERCENT_CS0                                                                     0x11c9
+#define regSPI_WCL_PIPE_PERCENT_CS0_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS1                                                                     0x11ca
+#define regSPI_WCL_PIPE_PERCENT_CS1_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS2                                                                     0x11cb
+#define regSPI_WCL_PIPE_PERCENT_CS2_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS3                                                                     0x11cc
+#define regSPI_WCL_PIPE_PERCENT_CS3_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS4                                                                     0x11cd
+#define regSPI_WCL_PIPE_PERCENT_CS4_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS5                                                                     0x11ce
+#define regSPI_WCL_PIPE_PERCENT_CS5_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS6                                                                     0x11cf
+#define regSPI_WCL_PIPE_PERCENT_CS6_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS7                                                                     0x11d0
+#define regSPI_WCL_PIPE_PERCENT_CS7_BASE_IDX                                                            0
+#define regSPI_GDBG_WAVE_CNTL                                                                           0x11d1
+#define regSPI_GDBG_WAVE_CNTL_BASE_IDX                                                                  0
+#define regSPI_GDBG_TRAP_CONFIG                                                                         0x11d2
+#define regSPI_GDBG_TRAP_CONFIG_BASE_IDX                                                                0
+#define regSPI_GDBG_PER_VMID_CNTL                                                                       0x11d3
+#define regSPI_GDBG_PER_VMID_CNTL_BASE_IDX                                                              0
+#define regSPI_GDBG_WAVE_CNTL3                                                                          0x11d5
+#define regSPI_GDBG_WAVE_CNTL3_BASE_IDX                                                                 0
+#define regSPI_SCRATCH_ADDR_CHECK                                                                       0x11d8
+#define regSPI_SCRATCH_ADDR_CHECK_BASE_IDX                                                              0
+#define regSPI_SCRATCH_ADDR_STATUS                                                                      0x11d9
+#define regSPI_SCRATCH_ADDR_STATUS_BASE_IDX                                                             0
+#define regSPI_RESET_DEBUG                                                                              0x11da
+#define regSPI_RESET_DEBUG_BASE_IDX                                                                     0
+#define regSPI_COMPUTE_QUEUE_RESET                                                                      0x11db
+#define regSPI_COMPUTE_QUEUE_RESET_BASE_IDX                                                             0
+#define regSPI_RESOURCE_RESERVE_CU_0                                                                    0x11dc
+#define regSPI_RESOURCE_RESERVE_CU_0_BASE_IDX                                                           0
+#define regSPI_RESOURCE_RESERVE_CU_1                                                                    0x11dd
+#define regSPI_RESOURCE_RESERVE_CU_1_BASE_IDX                                                           0
+#define regSPI_RESOURCE_RESERVE_CU_2                                                                    0x11de
+#define regSPI_RESOURCE_RESERVE_CU_2_BASE_IDX                                                           0
+#define regSPI_RESOURCE_RESERVE_CU_3                                                                    0x11df
+#define regSPI_RESOURCE_RESERVE_CU_3_BASE_IDX                                                           0
+#define regSPI_RESOURCE_RESERVE_CU_4                                                                    0x11e0
+#define regSPI_RESOURCE_RESERVE_CU_4_BASE_IDX                                                           0
+#define regSPI_RESOURCE_RESERVE_CU_5                                                                    0x11e1
+#define regSPI_RESOURCE_RESERVE_CU_5_BASE_IDX                                                           0
+#define regSPI_RESOURCE_RESERVE_CU_6                                                                    0x11e2
+#define regSPI_RESOURCE_RESERVE_CU_6_BASE_IDX                                                           0
+#define regSPI_RESOURCE_RESERVE_CU_7                                                                    0x11e3
+#define regSPI_RESOURCE_RESERVE_CU_7_BASE_IDX                                                           0
+#define regSPI_RESOURCE_RESERVE_CU_8                                                                    0x11e4
+#define regSPI_RESOURCE_RESERVE_CU_8_BASE_IDX                                                           0
+#define regSPI_RESOURCE_RESERVE_CU_9                                                                    0x11e5
+#define regSPI_RESOURCE_RESERVE_CU_9_BASE_IDX                                                           0
+#define regSPI_RESOURCE_RESERVE_EN_CU_0                                                                 0x11e6
+#define regSPI_RESOURCE_RESERVE_EN_CU_0_BASE_IDX                                                        0
+#define regSPI_RESOURCE_RESERVE_EN_CU_1                                                                 0x11e7
+#define regSPI_RESOURCE_RESERVE_EN_CU_1_BASE_IDX                                                        0
+#define regSPI_RESOURCE_RESERVE_EN_CU_2                                                                 0x11e8
+#define regSPI_RESOURCE_RESERVE_EN_CU_2_BASE_IDX                                                        0
+#define regSPI_RESOURCE_RESERVE_EN_CU_3                                                                 0x11e9
+#define regSPI_RESOURCE_RESERVE_EN_CU_3_BASE_IDX                                                        0
+#define regSPI_RESOURCE_RESERVE_EN_CU_4                                                                 0x11ea
+#define regSPI_RESOURCE_RESERVE_EN_CU_4_BASE_IDX                                                        0
+#define regSPI_RESOURCE_RESERVE_EN_CU_5                                                                 0x11eb
+#define regSPI_RESOURCE_RESERVE_EN_CU_5_BASE_IDX                                                        0
+#define regSPI_RESOURCE_RESERVE_EN_CU_6                                                                 0x11ec
+#define regSPI_RESOURCE_RESERVE_EN_CU_6_BASE_IDX                                                        0
+#define regSPI_RESOURCE_RESERVE_EN_CU_7                                                                 0x11ed
+#define regSPI_RESOURCE_RESERVE_EN_CU_7_BASE_IDX                                                        0
+#define regSPI_RESOURCE_RESERVE_EN_CU_8                                                                 0x11ee
+#define regSPI_RESOURCE_RESERVE_EN_CU_8_BASE_IDX                                                        0
+#define regSPI_RESOURCE_RESERVE_EN_CU_9                                                                 0x11ef
+#define regSPI_RESOURCE_RESERVE_EN_CU_9_BASE_IDX                                                        0
+#define regSPI_RESOURCE_RESERVE_CU_10                                                                   0x11f0
+#define regSPI_RESOURCE_RESERVE_CU_10_BASE_IDX                                                          0
+#define regSPI_RESOURCE_RESERVE_CU_11                                                                   0x11f1
+#define regSPI_RESOURCE_RESERVE_CU_11_BASE_IDX                                                          0
+#define regSPI_RESOURCE_RESERVE_EN_CU_10                                                                0x11f2
+#define regSPI_RESOURCE_RESERVE_EN_CU_10_BASE_IDX                                                       0
+#define regSPI_RESOURCE_RESERVE_EN_CU_11                                                                0x11f3
+#define regSPI_RESOURCE_RESERVE_EN_CU_11_BASE_IDX                                                       0
+#define regSPI_RESOURCE_RESERVE_CU_12                                                                   0x11f4
+#define regSPI_RESOURCE_RESERVE_CU_12_BASE_IDX                                                          0
+#define regSPI_RESOURCE_RESERVE_CU_13                                                                   0x11f5
+#define regSPI_RESOURCE_RESERVE_CU_13_BASE_IDX                                                          0
+#define regSPI_RESOURCE_RESERVE_CU_14                                                                   0x11f6
+#define regSPI_RESOURCE_RESERVE_CU_14_BASE_IDX                                                          0
+#define regSPI_RESOURCE_RESERVE_CU_15                                                                   0x11f7
+#define regSPI_RESOURCE_RESERVE_CU_15_BASE_IDX                                                          0
+#define regSPI_RESOURCE_RESERVE_EN_CU_12                                                                0x11f8
+#define regSPI_RESOURCE_RESERVE_EN_CU_12_BASE_IDX                                                       0
+#define regSPI_RESOURCE_RESERVE_EN_CU_13                                                                0x11f9
+#define regSPI_RESOURCE_RESERVE_EN_CU_13_BASE_IDX                                                       0
+#define regSPI_RESOURCE_RESERVE_EN_CU_14                                                                0x11fa
+#define regSPI_RESOURCE_RESERVE_EN_CU_14_BASE_IDX                                                       0
+#define regSPI_RESOURCE_RESERVE_EN_CU_15                                                                0x11fb
+#define regSPI_RESOURCE_RESERVE_EN_CU_15_BASE_IDX                                                       0
+#define regSPI_COMPUTE_WF_CTX_SAVE                                                                      0x11fc
+#define regSPI_COMPUTE_WF_CTX_SAVE_BASE_IDX                                                             0
+#define regSPI_ARB_CNTL_0                                                                               0x11fd
+#define regSPI_ARB_CNTL_0_BASE_IDX                                                                      0
+
+
+// addressBlock: xcd0_gc_cpphqddec
+// base address: 0xc800
+#define regCP_HQD_GFX_CONTROL                                                                           0x123e
+#define regCP_HQD_GFX_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_GFX_STATUS                                                                            0x123f
+#define regCP_HQD_GFX_STATUS_BASE_IDX                                                                   0
+#define regCP_HPD_ROQ_OFFSETS                                                                           0x1240
+#define regCP_HPD_ROQ_OFFSETS_BASE_IDX                                                                  0
+#define regCP_HPD_STATUS0                                                                               0x1241
+#define regCP_HPD_STATUS0_BASE_IDX                                                                      0
+#define regCP_HPD_UTCL1_CNTL                                                                            0x1242
+#define regCP_HPD_UTCL1_CNTL_BASE_IDX                                                                   0
+#define regCP_HPD_UTCL1_ERROR                                                                           0x1243
+#define regCP_HPD_UTCL1_ERROR_BASE_IDX                                                                  0
+#define regCP_HPD_UTCL1_ERROR_ADDR                                                                      0x1244
+#define regCP_HPD_UTCL1_ERROR_ADDR_BASE_IDX                                                             0
+#define regCP_MQD_BASE_ADDR                                                                             0x1245
+#define regCP_MQD_BASE_ADDR_BASE_IDX                                                                    0
+#define regCP_MQD_BASE_ADDR_HI                                                                          0x1246
+#define regCP_MQD_BASE_ADDR_HI_BASE_IDX                                                                 0
+#define regCP_HQD_ACTIVE                                                                                0x1247
+#define regCP_HQD_ACTIVE_BASE_IDX                                                                       0
+#define regCP_HQD_VMID                                                                                  0x1248
+#define regCP_HQD_VMID_BASE_IDX                                                                         0
+#define regCP_HQD_PERSISTENT_STATE                                                                      0x1249
+#define regCP_HQD_PERSISTENT_STATE_BASE_IDX                                                             0
+#define regCP_HQD_PIPE_PRIORITY                                                                         0x124a
+#define regCP_HQD_PIPE_PRIORITY_BASE_IDX                                                                0
+#define regCP_HQD_QUEUE_PRIORITY                                                                        0x124b
+#define regCP_HQD_QUEUE_PRIORITY_BASE_IDX                                                               0
+#define regCP_HQD_QUANTUM                                                                               0x124c
+#define regCP_HQD_QUANTUM_BASE_IDX                                                                      0
+#define regCP_HQD_PQ_BASE                                                                               0x124d
+#define regCP_HQD_PQ_BASE_BASE_IDX                                                                      0
+#define regCP_HQD_PQ_BASE_HI                                                                            0x124e
+#define regCP_HQD_PQ_BASE_HI_BASE_IDX                                                                   0
+#define regCP_HQD_PQ_RPTR                                                                               0x124f
+#define regCP_HQD_PQ_RPTR_BASE_IDX                                                                      0
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR                                                                   0x1250
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR_BASE_IDX                                                          0
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR_HI                                                                0x1251
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR_HI_BASE_IDX                                                       0
+#define regCP_HQD_PQ_WPTR_POLL_ADDR                                                                     0x1252
+#define regCP_HQD_PQ_WPTR_POLL_ADDR_BASE_IDX                                                            0
+#define regCP_HQD_PQ_WPTR_POLL_ADDR_HI                                                                  0x1253
+#define regCP_HQD_PQ_WPTR_POLL_ADDR_HI_BASE_IDX                                                         0
+#define regCP_HQD_PQ_DOORBELL_CONTROL                                                                   0x1254
+#define regCP_HQD_PQ_DOORBELL_CONTROL_BASE_IDX                                                          0
+#define regCP_HQD_PQ_CONTROL                                                                            0x1256
+#define regCP_HQD_PQ_CONTROL_BASE_IDX                                                                   0
+#define regCP_HQD_IB_BASE_ADDR                                                                          0x1257
+#define regCP_HQD_IB_BASE_ADDR_BASE_IDX                                                                 0
+#define regCP_HQD_IB_BASE_ADDR_HI                                                                       0x1258
+#define regCP_HQD_IB_BASE_ADDR_HI_BASE_IDX                                                              0
+#define regCP_HQD_IB_RPTR                                                                               0x1259
+#define regCP_HQD_IB_RPTR_BASE_IDX                                                                      0
+#define regCP_HQD_IB_CONTROL                                                                            0x125a
+#define regCP_HQD_IB_CONTROL_BASE_IDX                                                                   0
+#define regCP_HQD_IQ_TIMER                                                                              0x125b
+#define regCP_HQD_IQ_TIMER_BASE_IDX                                                                     0
+#define regCP_HQD_IQ_RPTR                                                                               0x125c
+#define regCP_HQD_IQ_RPTR_BASE_IDX                                                                      0
+#define regCP_HQD_DEQUEUE_REQUEST                                                                       0x125d
+#define regCP_HQD_DEQUEUE_REQUEST_BASE_IDX                                                              0
+#define regCP_HQD_DMA_OFFLOAD                                                                           0x125e
+#define regCP_HQD_DMA_OFFLOAD_BASE_IDX                                                                  0
+#define regCP_HQD_OFFLOAD                                                                               0x125e
+#define regCP_HQD_OFFLOAD_BASE_IDX                                                                      0
+#define regCP_HQD_SEMA_CMD                                                                              0x125f
+#define regCP_HQD_SEMA_CMD_BASE_IDX                                                                     0
+#define regCP_HQD_MSG_TYPE                                                                              0x1260
+#define regCP_HQD_MSG_TYPE_BASE_IDX                                                                     0
+#define regCP_HQD_ATOMIC0_PREOP_LO                                                                      0x1261
+#define regCP_HQD_ATOMIC0_PREOP_LO_BASE_IDX                                                             0
+#define regCP_HQD_ATOMIC0_PREOP_HI                                                                      0x1262
+#define regCP_HQD_ATOMIC0_PREOP_HI_BASE_IDX                                                             0
+#define regCP_HQD_ATOMIC1_PREOP_LO                                                                      0x1263
+#define regCP_HQD_ATOMIC1_PREOP_LO_BASE_IDX                                                             0
+#define regCP_HQD_ATOMIC1_PREOP_HI                                                                      0x1264
+#define regCP_HQD_ATOMIC1_PREOP_HI_BASE_IDX                                                             0
+#define regCP_HQD_HQ_SCHEDULER0                                                                         0x1265
+#define regCP_HQD_HQ_SCHEDULER0_BASE_IDX                                                                0
+#define regCP_HQD_HQ_STATUS0                                                                            0x1265
+#define regCP_HQD_HQ_STATUS0_BASE_IDX                                                                   0
+#define regCP_HQD_HQ_CONTROL0                                                                           0x1266
+#define regCP_HQD_HQ_CONTROL0_BASE_IDX                                                                  0
+#define regCP_HQD_HQ_SCHEDULER1                                                                         0x1266
+#define regCP_HQD_HQ_SCHEDULER1_BASE_IDX                                                                0
+#define regCP_MQD_CONTROL                                                                               0x1267
+#define regCP_MQD_CONTROL_BASE_IDX                                                                      0
+#define regCP_HQD_HQ_STATUS1                                                                            0x1268
+#define regCP_HQD_HQ_STATUS1_BASE_IDX                                                                   0
+#define regCP_HQD_HQ_CONTROL1                                                                           0x1269
+#define regCP_HQD_HQ_CONTROL1_BASE_IDX                                                                  0
+#define regCP_HQD_EOP_BASE_ADDR                                                                         0x126a
+#define regCP_HQD_EOP_BASE_ADDR_BASE_IDX                                                                0
+#define regCP_HQD_EOP_BASE_ADDR_HI                                                                      0x126b
+#define regCP_HQD_EOP_BASE_ADDR_HI_BASE_IDX                                                             0
+#define regCP_HQD_EOP_CONTROL                                                                           0x126c
+#define regCP_HQD_EOP_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_EOP_RPTR                                                                              0x126d
+#define regCP_HQD_EOP_RPTR_BASE_IDX                                                                     0
+#define regCP_HQD_EOP_WPTR                                                                              0x126e
+#define regCP_HQD_EOP_WPTR_BASE_IDX                                                                     0
+#define regCP_HQD_EOP_EVENTS                                                                            0x126f
+#define regCP_HQD_EOP_EVENTS_BASE_IDX                                                                   0
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_LO                                                                 0x1270
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_LO_BASE_IDX                                                        0
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_HI                                                                 0x1271
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_HI_BASE_IDX                                                        0
+#define regCP_HQD_CTX_SAVE_CONTROL                                                                      0x1272
+#define regCP_HQD_CTX_SAVE_CONTROL_BASE_IDX                                                             0
+#define regCP_HQD_CNTL_STACK_OFFSET                                                                     0x1273
+#define regCP_HQD_CNTL_STACK_OFFSET_BASE_IDX                                                            0
+#define regCP_HQD_CNTL_STACK_SIZE                                                                       0x1274
+#define regCP_HQD_CNTL_STACK_SIZE_BASE_IDX                                                              0
+#define regCP_HQD_WG_STATE_OFFSET                                                                       0x1275
+#define regCP_HQD_WG_STATE_OFFSET_BASE_IDX                                                              0
+#define regCP_HQD_CTX_SAVE_SIZE                                                                         0x1276
+#define regCP_HQD_CTX_SAVE_SIZE_BASE_IDX                                                                0
+#define regCP_HQD_GDS_RESOURCE_STATE                                                                    0x1277
+#define regCP_HQD_GDS_RESOURCE_STATE_BASE_IDX                                                           0
+#define regCP_HQD_ERROR                                                                                 0x1278
+#define regCP_HQD_ERROR_BASE_IDX                                                                        0
+#define regCP_HQD_EOP_WPTR_MEM                                                                          0x1279
+#define regCP_HQD_EOP_WPTR_MEM_BASE_IDX                                                                 0
+#define regCP_HQD_AQL_CONTROL                                                                           0x127a
+#define regCP_HQD_AQL_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_PQ_WPTR_LO                                                                            0x127b
+#define regCP_HQD_PQ_WPTR_LO_BASE_IDX                                                                   0
+#define regCP_HQD_PQ_WPTR_HI                                                                            0x127c
+#define regCP_HQD_PQ_WPTR_HI_BASE_IDX                                                                   0
+#define regCP_HQD_AQL_CONTROL_1                                                                         0x127d
+#define regCP_HQD_AQL_CONTROL_1_BASE_IDX                                                                0
+#define regCP_HQD_AQL_DISPATCH_ID                                                                       0x127e
+#define regCP_HQD_AQL_DISPATCH_ID_BASE_IDX                                                              0
+#define regCP_HQD_AQL_DISPATCH_ID_HI                                                                    0x127f
+#define regCP_HQD_AQL_DISPATCH_ID_HI_BASE_IDX                                                           0
+
+
+// addressBlock: xcd0_gc_tcpdec
+// base address: 0xca80
+#define regTCP_WATCH0_ADDR_H                                                                            0x12a0
+#define regTCP_WATCH0_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH0_ADDR_L                                                                            0x12a1
+#define regTCP_WATCH0_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH0_CNTL                                                                              0x12a2
+#define regTCP_WATCH0_CNTL_BASE_IDX                                                                     0
+#define regTCP_WATCH1_ADDR_H                                                                            0x12a3
+#define regTCP_WATCH1_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH1_ADDR_L                                                                            0x12a4
+#define regTCP_WATCH1_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH1_CNTL                                                                              0x12a5
+#define regTCP_WATCH1_CNTL_BASE_IDX                                                                     0
+#define regTCP_WATCH2_ADDR_H                                                                            0x12a6
+#define regTCP_WATCH2_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH2_ADDR_L                                                                            0x12a7
+#define regTCP_WATCH2_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH2_CNTL                                                                              0x12a8
+#define regTCP_WATCH2_CNTL_BASE_IDX                                                                     0
+#define regTCP_WATCH3_ADDR_H                                                                            0x12a9
+#define regTCP_WATCH3_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH3_ADDR_L                                                                            0x12aa
+#define regTCP_WATCH3_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH3_CNTL                                                                              0x12ab
+#define regTCP_WATCH3_CNTL_BASE_IDX                                                                     0
+#define regTCP_GATCL1_CNTL                                                                              0x12b0
+#define regTCP_GATCL1_CNTL_BASE_IDX                                                                     0
+#define regTCP_ATC_EDC_GATCL1_CNT                                                                       0x12b1
+#define regTCP_ATC_EDC_GATCL1_CNT_BASE_IDX                                                              0
+#define regTCP_GATCL1_DSM_CNTL                                                                          0x12b2
+#define regTCP_GATCL1_DSM_CNTL_BASE_IDX                                                                 0
+#define regTCP_DSM_CNTL                                                                                 0x12b3
+#define regTCP_DSM_CNTL_BASE_IDX                                                                        0
+#define regTCP_CNTL2                                                                                    0x12b4
+#define regTCP_CNTL2_BASE_IDX                                                                           0
+#define regTCP_UTCL1_CNTL1                                                                              0x12b5
+#define regTCP_UTCL1_CNTL1_BASE_IDX                                                                     0
+#define regTCP_UTCL1_CNTL2                                                                              0x12b6
+#define regTCP_UTCL1_CNTL2_BASE_IDX                                                                     0
+#define regTCP_UTCL1_STATUS                                                                             0x12b7
+#define regTCP_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regTCP_DSM_CNTL2                                                                                0x12b8
+#define regTCP_DSM_CNTL2_BASE_IDX                                                                       0
+#define regTCP_PERFCOUNTER_FILTER                                                                       0x12b9
+#define regTCP_PERFCOUNTER_FILTER_BASE_IDX                                                              0
+#define regTCP_PERFCOUNTER_FILTER_EN                                                                    0x12ba
+#define regTCP_PERFCOUNTER_FILTER_EN_BASE_IDX                                                           0
+
+
+// addressBlock: xcd0_gc_gdspdec
+// base address: 0xcc00
+#define regGDS_VMID0_BASE                                                                               0x1300
+#define regGDS_VMID0_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID0_SIZE                                                                               0x1301
+#define regGDS_VMID0_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID1_BASE                                                                               0x1302
+#define regGDS_VMID1_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID1_SIZE                                                                               0x1303
+#define regGDS_VMID1_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID2_BASE                                                                               0x1304
+#define regGDS_VMID2_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID2_SIZE                                                                               0x1305
+#define regGDS_VMID2_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID3_BASE                                                                               0x1306
+#define regGDS_VMID3_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID3_SIZE                                                                               0x1307
+#define regGDS_VMID3_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID4_BASE                                                                               0x1308
+#define regGDS_VMID4_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID4_SIZE                                                                               0x1309
+#define regGDS_VMID4_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID5_BASE                                                                               0x130a
+#define regGDS_VMID5_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID5_SIZE                                                                               0x130b
+#define regGDS_VMID5_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID6_BASE                                                                               0x130c
+#define regGDS_VMID6_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID6_SIZE                                                                               0x130d
+#define regGDS_VMID6_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID7_BASE                                                                               0x130e
+#define regGDS_VMID7_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID7_SIZE                                                                               0x130f
+#define regGDS_VMID7_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID8_BASE                                                                               0x1310
+#define regGDS_VMID8_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID8_SIZE                                                                               0x1311
+#define regGDS_VMID8_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID9_BASE                                                                               0x1312
+#define regGDS_VMID9_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID9_SIZE                                                                               0x1313
+#define regGDS_VMID9_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID10_BASE                                                                              0x1314
+#define regGDS_VMID10_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID10_SIZE                                                                              0x1315
+#define regGDS_VMID10_SIZE_BASE_IDX                                                                     0
+#define regGDS_VMID11_BASE                                                                              0x1316
+#define regGDS_VMID11_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID11_SIZE                                                                              0x1317
+#define regGDS_VMID11_SIZE_BASE_IDX                                                                     0
+#define regGDS_VMID12_BASE                                                                              0x1318
+#define regGDS_VMID12_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID12_SIZE                                                                              0x1319
+#define regGDS_VMID12_SIZE_BASE_IDX                                                                     0
+#define regGDS_VMID13_BASE                                                                              0x131a
+#define regGDS_VMID13_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID13_SIZE                                                                              0x131b
+#define regGDS_VMID13_SIZE_BASE_IDX                                                                     0
+#define regGDS_VMID14_BASE                                                                              0x131c
+#define regGDS_VMID14_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID14_SIZE                                                                              0x131d
+#define regGDS_VMID14_SIZE_BASE_IDX                                                                     0
+#define regGDS_VMID15_BASE                                                                              0x131e
+#define regGDS_VMID15_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID15_SIZE                                                                              0x131f
+#define regGDS_VMID15_SIZE_BASE_IDX                                                                     0
+#define regGDS_GWS_VMID0                                                                                0x1320
+#define regGDS_GWS_VMID0_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID1                                                                                0x1321
+#define regGDS_GWS_VMID1_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID2                                                                                0x1322
+#define regGDS_GWS_VMID2_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID3                                                                                0x1323
+#define regGDS_GWS_VMID3_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID4                                                                                0x1324
+#define regGDS_GWS_VMID4_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID5                                                                                0x1325
+#define regGDS_GWS_VMID5_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID6                                                                                0x1326
+#define regGDS_GWS_VMID6_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID7                                                                                0x1327
+#define regGDS_GWS_VMID7_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID8                                                                                0x1328
+#define regGDS_GWS_VMID8_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID9                                                                                0x1329
+#define regGDS_GWS_VMID9_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID10                                                                               0x132a
+#define regGDS_GWS_VMID10_BASE_IDX                                                                      0
+#define regGDS_GWS_VMID11                                                                               0x132b
+#define regGDS_GWS_VMID11_BASE_IDX                                                                      0
+#define regGDS_GWS_VMID12                                                                               0x132c
+#define regGDS_GWS_VMID12_BASE_IDX                                                                      0
+#define regGDS_GWS_VMID13                                                                               0x132d
+#define regGDS_GWS_VMID13_BASE_IDX                                                                      0
+#define regGDS_GWS_VMID14                                                                               0x132e
+#define regGDS_GWS_VMID14_BASE_IDX                                                                      0
+#define regGDS_GWS_VMID15                                                                               0x132f
+#define regGDS_GWS_VMID15_BASE_IDX                                                                      0
+#define regGDS_OA_VMID0                                                                                 0x1330
+#define regGDS_OA_VMID0_BASE_IDX                                                                        0
+#define regGDS_OA_VMID1                                                                                 0x1331
+#define regGDS_OA_VMID1_BASE_IDX                                                                        0
+#define regGDS_OA_VMID2                                                                                 0x1332
+#define regGDS_OA_VMID2_BASE_IDX                                                                        0
+#define regGDS_OA_VMID3                                                                                 0x1333
+#define regGDS_OA_VMID3_BASE_IDX                                                                        0
+#define regGDS_OA_VMID4                                                                                 0x1334
+#define regGDS_OA_VMID4_BASE_IDX                                                                        0
+#define regGDS_OA_VMID5                                                                                 0x1335
+#define regGDS_OA_VMID5_BASE_IDX                                                                        0
+#define regGDS_OA_VMID6                                                                                 0x1336
+#define regGDS_OA_VMID6_BASE_IDX                                                                        0
+#define regGDS_OA_VMID7                                                                                 0x1337
+#define regGDS_OA_VMID7_BASE_IDX                                                                        0
+#define regGDS_OA_VMID8                                                                                 0x1338
+#define regGDS_OA_VMID8_BASE_IDX                                                                        0
+#define regGDS_OA_VMID9                                                                                 0x1339
+#define regGDS_OA_VMID9_BASE_IDX                                                                        0
+#define regGDS_OA_VMID10                                                                                0x133a
+#define regGDS_OA_VMID10_BASE_IDX                                                                       0
+#define regGDS_OA_VMID11                                                                                0x133b
+#define regGDS_OA_VMID11_BASE_IDX                                                                       0
+#define regGDS_OA_VMID12                                                                                0x133c
+#define regGDS_OA_VMID12_BASE_IDX                                                                       0
+#define regGDS_OA_VMID13                                                                                0x133d
+#define regGDS_OA_VMID13_BASE_IDX                                                                       0
+#define regGDS_OA_VMID14                                                                                0x133e
+#define regGDS_OA_VMID14_BASE_IDX                                                                       0
+#define regGDS_OA_VMID15                                                                                0x133f
+#define regGDS_OA_VMID15_BASE_IDX                                                                       0
+#define regGDS_GWS_RESET0                                                                               0x1344
+#define regGDS_GWS_RESET0_BASE_IDX                                                                      0
+#define regGDS_GWS_RESET1                                                                               0x1345
+#define regGDS_GWS_RESET1_BASE_IDX                                                                      0
+#define regGDS_GWS_RESOURCE_RESET                                                                       0x1346
+#define regGDS_GWS_RESOURCE_RESET_BASE_IDX                                                              0
+#define regGDS_COMPUTE_MAX_WAVE_ID                                                                      0x1348
+#define regGDS_COMPUTE_MAX_WAVE_ID_BASE_IDX                                                             0
+#define regGDS_OA_RESET_MASK                                                                            0x1349
+#define regGDS_OA_RESET_MASK_BASE_IDX                                                                   0
+#define regGDS_OA_RESET                                                                                 0x134a
+#define regGDS_OA_RESET_BASE_IDX                                                                        0
+#define regGDS_ENHANCE                                                                                  0x134b
+#define regGDS_ENHANCE_BASE_IDX                                                                         0
+#define regGDS_OA_CGPG_RESTORE                                                                          0x134c
+#define regGDS_OA_CGPG_RESTORE_BASE_IDX                                                                 0
+#define regGDS_CS_CTXSW_STATUS                                                                          0x134d
+#define regGDS_CS_CTXSW_STATUS_BASE_IDX                                                                 0
+#define regGDS_CS_CTXSW_CNT0                                                                            0x134e
+#define regGDS_CS_CTXSW_CNT0_BASE_IDX                                                                   0
+#define regGDS_CS_CTXSW_CNT1                                                                            0x134f
+#define regGDS_CS_CTXSW_CNT1_BASE_IDX                                                                   0
+#define regGDS_CS_CTXSW_CNT2                                                                            0x1350
+#define regGDS_CS_CTXSW_CNT2_BASE_IDX                                                                   0
+#define regGDS_CS_CTXSW_CNT3                                                                            0x1351
+#define regGDS_CS_CTXSW_CNT3_BASE_IDX                                                                   0
+#define regGDS_GFX_CTXSW_STATUS                                                                         0x1352
+#define regGDS_GFX_CTXSW_STATUS_BASE_IDX                                                                0
+#define regGDS_VS_CTXSW_CNT0                                                                            0x1353
+#define regGDS_VS_CTXSW_CNT0_BASE_IDX                                                                   0
+#define regGDS_VS_CTXSW_CNT1                                                                            0x1354
+#define regGDS_VS_CTXSW_CNT1_BASE_IDX                                                                   0
+#define regGDS_VS_CTXSW_CNT2                                                                            0x1355
+#define regGDS_VS_CTXSW_CNT2_BASE_IDX                                                                   0
+#define regGDS_VS_CTXSW_CNT3                                                                            0x1356
+#define regGDS_VS_CTXSW_CNT3_BASE_IDX                                                                   0
+#define regGDS_PS0_CTXSW_CNT0                                                                           0x1357
+#define regGDS_PS0_CTXSW_CNT0_BASE_IDX                                                                  0
+#define regGDS_PS0_CTXSW_CNT1                                                                           0x1358
+#define regGDS_PS0_CTXSW_CNT1_BASE_IDX                                                                  0
+#define regGDS_PS0_CTXSW_CNT2                                                                           0x1359
+#define regGDS_PS0_CTXSW_CNT2_BASE_IDX                                                                  0
+#define regGDS_PS0_CTXSW_CNT3                                                                           0x135a
+#define regGDS_PS0_CTXSW_CNT3_BASE_IDX                                                                  0
+#define regGDS_PS1_CTXSW_CNT0                                                                           0x135b
+#define regGDS_PS1_CTXSW_CNT0_BASE_IDX                                                                  0
+#define regGDS_PS1_CTXSW_CNT1                                                                           0x135c
+#define regGDS_PS1_CTXSW_CNT1_BASE_IDX                                                                  0
+#define regGDS_PS1_CTXSW_CNT2                                                                           0x135d
+#define regGDS_PS1_CTXSW_CNT2_BASE_IDX                                                                  0
+#define regGDS_PS1_CTXSW_CNT3                                                                           0x135e
+#define regGDS_PS1_CTXSW_CNT3_BASE_IDX                                                                  0
+#define regGDS_PS2_CTXSW_CNT0                                                                           0x135f
+#define regGDS_PS2_CTXSW_CNT0_BASE_IDX                                                                  0
+#define regGDS_PS2_CTXSW_CNT1                                                                           0x1360
+#define regGDS_PS2_CTXSW_CNT1_BASE_IDX                                                                  0
+#define regGDS_PS2_CTXSW_CNT2                                                                           0x1361
+#define regGDS_PS2_CTXSW_CNT2_BASE_IDX                                                                  0
+#define regGDS_PS2_CTXSW_CNT3                                                                           0x1362
+#define regGDS_PS2_CTXSW_CNT3_BASE_IDX                                                                  0
+#define regGDS_PS3_CTXSW_CNT0                                                                           0x1363
+#define regGDS_PS3_CTXSW_CNT0_BASE_IDX                                                                  0
+#define regGDS_PS3_CTXSW_CNT1                                                                           0x1364
+#define regGDS_PS3_CTXSW_CNT1_BASE_IDX                                                                  0
+#define regGDS_PS3_CTXSW_CNT2                                                                           0x1365
+#define regGDS_PS3_CTXSW_CNT2_BASE_IDX                                                                  0
+#define regGDS_PS3_CTXSW_CNT3                                                                           0x1366
+#define regGDS_PS3_CTXSW_CNT3_BASE_IDX                                                                  0
+#define regGDS_PS4_CTXSW_CNT0                                                                           0x1367
+#define regGDS_PS4_CTXSW_CNT0_BASE_IDX                                                                  0
+#define regGDS_PS4_CTXSW_CNT1                                                                           0x1368
+#define regGDS_PS4_CTXSW_CNT1_BASE_IDX                                                                  0
+#define regGDS_PS4_CTXSW_CNT2                                                                           0x1369
+#define regGDS_PS4_CTXSW_CNT2_BASE_IDX                                                                  0
+#define regGDS_PS4_CTXSW_CNT3                                                                           0x136a
+#define regGDS_PS4_CTXSW_CNT3_BASE_IDX                                                                  0
+#define regGDS_PS5_CTXSW_CNT0                                                                           0x136b
+#define regGDS_PS5_CTXSW_CNT0_BASE_IDX                                                                  0
+#define regGDS_PS5_CTXSW_CNT1                                                                           0x136c
+#define regGDS_PS5_CTXSW_CNT1_BASE_IDX                                                                  0
+#define regGDS_PS5_CTXSW_CNT2                                                                           0x136d
+#define regGDS_PS5_CTXSW_CNT2_BASE_IDX                                                                  0
+#define regGDS_PS5_CTXSW_CNT3                                                                           0x136e
+#define regGDS_PS5_CTXSW_CNT3_BASE_IDX                                                                  0
+#define regGDS_PS6_CTXSW_CNT0                                                                           0x136f
+#define regGDS_PS6_CTXSW_CNT0_BASE_IDX                                                                  0
+#define regGDS_PS6_CTXSW_CNT1                                                                           0x1370
+#define regGDS_PS6_CTXSW_CNT1_BASE_IDX                                                                  0
+#define regGDS_PS6_CTXSW_CNT2                                                                           0x1371
+#define regGDS_PS6_CTXSW_CNT2_BASE_IDX                                                                  0
+#define regGDS_PS6_CTXSW_CNT3                                                                           0x1372
+#define regGDS_PS6_CTXSW_CNT3_BASE_IDX                                                                  0
+#define regGDS_PS7_CTXSW_CNT0                                                                           0x1373
+#define regGDS_PS7_CTXSW_CNT0_BASE_IDX                                                                  0
+#define regGDS_PS7_CTXSW_CNT1                                                                           0x1374
+#define regGDS_PS7_CTXSW_CNT1_BASE_IDX                                                                  0
+#define regGDS_PS7_CTXSW_CNT2                                                                           0x1375
+#define regGDS_PS7_CTXSW_CNT2_BASE_IDX                                                                  0
+#define regGDS_PS7_CTXSW_CNT3                                                                           0x1376
+#define regGDS_PS7_CTXSW_CNT3_BASE_IDX                                                                  0
+#define regGDS_GS_CTXSW_CNT0                                                                            0x1377
+#define regGDS_GS_CTXSW_CNT0_BASE_IDX                                                                   0
+#define regGDS_GS_CTXSW_CNT1                                                                            0x1378
+#define regGDS_GS_CTXSW_CNT1_BASE_IDX                                                                   0
+#define regGDS_GS_CTXSW_CNT2                                                                            0x1379
+#define regGDS_GS_CTXSW_CNT2_BASE_IDX                                                                   0
+#define regGDS_GS_CTXSW_CNT3                                                                            0x137a
+#define regGDS_GS_CTXSW_CNT3_BASE_IDX                                                                   0
+
+
+// addressBlock: xcd0_gc_rasdec
+// base address: 0xce00
+#define regRAS_SIGNATURE_CONTROL                                                                        0x1380
+#define regRAS_SIGNATURE_CONTROL_BASE_IDX                                                               0
+#define regRAS_SIGNATURE_MASK                                                                           0x1381
+#define regRAS_SIGNATURE_MASK_BASE_IDX                                                                  0
+#define regRAS_SX_SIGNATURE0                                                                            0x1382
+#define regRAS_SX_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_SX_SIGNATURE1                                                                            0x1383
+#define regRAS_SX_SIGNATURE1_BASE_IDX                                                                   0
+#define regRAS_SX_SIGNATURE2                                                                            0x1384
+#define regRAS_SX_SIGNATURE2_BASE_IDX                                                                   0
+#define regRAS_SX_SIGNATURE3                                                                            0x1385
+#define regRAS_SX_SIGNATURE3_BASE_IDX                                                                   0
+#define regRAS_DB_SIGNATURE0                                                                            0x138b
+#define regRAS_DB_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_PA_SIGNATURE0                                                                            0x138c
+#define regRAS_PA_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_VGT_SIGNATURE0                                                                           0x138d
+#define regRAS_VGT_SIGNATURE0_BASE_IDX                                                                  0
+#define regRAS_SQ_SIGNATURE0                                                                            0x138e
+#define regRAS_SQ_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE0                                                                            0x138f
+#define regRAS_SC_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE1                                                                            0x1390
+#define regRAS_SC_SIGNATURE1_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE2                                                                            0x1391
+#define regRAS_SC_SIGNATURE2_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE3                                                                            0x1392
+#define regRAS_SC_SIGNATURE3_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE4                                                                            0x1393
+#define regRAS_SC_SIGNATURE4_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE5                                                                            0x1394
+#define regRAS_SC_SIGNATURE5_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE6                                                                            0x1395
+#define regRAS_SC_SIGNATURE6_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE7                                                                            0x1396
+#define regRAS_SC_SIGNATURE7_BASE_IDX                                                                   0
+#define regRAS_IA_SIGNATURE0                                                                            0x1397
+#define regRAS_IA_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_IA_SIGNATURE1                                                                            0x1398
+#define regRAS_IA_SIGNATURE1_BASE_IDX                                                                   0
+#define regRAS_SPI_SIGNATURE0                                                                           0x1399
+#define regRAS_SPI_SIGNATURE0_BASE_IDX                                                                  0
+#define regRAS_SPI_SIGNATURE1                                                                           0x139a
+#define regRAS_SPI_SIGNATURE1_BASE_IDX                                                                  0
+#define regRAS_TA_SIGNATURE0                                                                            0x139b
+#define regRAS_TA_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_TD_SIGNATURE0                                                                            0x139c
+#define regRAS_TD_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_CB_SIGNATURE0                                                                            0x139d
+#define regRAS_CB_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_BCI_SIGNATURE0                                                                           0x139e
+#define regRAS_BCI_SIGNATURE0_BASE_IDX                                                                  0
+#define regRAS_BCI_SIGNATURE1                                                                           0x139f
+#define regRAS_BCI_SIGNATURE1_BASE_IDX                                                                  0
+#define regRAS_TA_SIGNATURE1                                                                            0x13a0
+#define regRAS_TA_SIGNATURE1_BASE_IDX                                                                   0
+
+
+// addressBlock: xcd0_gc_gfxdec0
+// base address: 0x28000
+#define regDB_RENDER_CONTROL                                                                            0x0000
+#define regDB_RENDER_CONTROL_BASE_IDX                                                                   1
+#define regDB_COUNT_CONTROL                                                                             0x0001
+#define regDB_COUNT_CONTROL_BASE_IDX                                                                    1
+#define regDB_DEPTH_VIEW                                                                                0x0002
+#define regDB_DEPTH_VIEW_BASE_IDX                                                                       1
+#define regDB_RENDER_OVERRIDE                                                                           0x0003
+#define regDB_RENDER_OVERRIDE_BASE_IDX                                                                  1
+#define regDB_RENDER_OVERRIDE2                                                                          0x0004
+#define regDB_RENDER_OVERRIDE2_BASE_IDX                                                                 1
+#define regDB_HTILE_DATA_BASE                                                                           0x0005
+#define regDB_HTILE_DATA_BASE_BASE_IDX                                                                  1
+#define regDB_HTILE_DATA_BASE_HI                                                                        0x0006
+#define regDB_HTILE_DATA_BASE_HI_BASE_IDX                                                               1
+#define regDB_DEPTH_SIZE                                                                                0x0007
+#define regDB_DEPTH_SIZE_BASE_IDX                                                                       1
+#define regDB_DEPTH_BOUNDS_MIN                                                                          0x0008
+#define regDB_DEPTH_BOUNDS_MIN_BASE_IDX                                                                 1
+#define regDB_DEPTH_BOUNDS_MAX                                                                          0x0009
+#define regDB_DEPTH_BOUNDS_MAX_BASE_IDX                                                                 1
+#define regDB_STENCIL_CLEAR                                                                             0x000a
+#define regDB_STENCIL_CLEAR_BASE_IDX                                                                    1
+#define regDB_DEPTH_CLEAR                                                                               0x000b
+#define regDB_DEPTH_CLEAR_BASE_IDX                                                                      1
+#define regPA_SC_SCREEN_SCISSOR_TL                                                                      0x000c
+#define regPA_SC_SCREEN_SCISSOR_TL_BASE_IDX                                                             1
+#define regPA_SC_SCREEN_SCISSOR_BR                                                                      0x000d
+#define regPA_SC_SCREEN_SCISSOR_BR_BASE_IDX                                                             1
+#define regDB_Z_INFO                                                                                    0x000e
+#define regDB_Z_INFO_BASE_IDX                                                                           1
+#define regDB_STENCIL_INFO                                                                              0x000f
+#define regDB_STENCIL_INFO_BASE_IDX                                                                     1
+#define regDB_Z_READ_BASE                                                                               0x0010
+#define regDB_Z_READ_BASE_BASE_IDX                                                                      1
+#define regDB_Z_READ_BASE_HI                                                                            0x0011
+#define regDB_Z_READ_BASE_HI_BASE_IDX                                                                   1
+#define regDB_STENCIL_READ_BASE                                                                         0x0012
+#define regDB_STENCIL_READ_BASE_BASE_IDX                                                                1
+#define regDB_STENCIL_READ_BASE_HI                                                                      0x0013
+#define regDB_STENCIL_READ_BASE_HI_BASE_IDX                                                             1
+#define regDB_Z_WRITE_BASE                                                                              0x0014
+#define regDB_Z_WRITE_BASE_BASE_IDX                                                                     1
+#define regDB_Z_WRITE_BASE_HI                                                                           0x0015
+#define regDB_Z_WRITE_BASE_HI_BASE_IDX                                                                  1
+#define regDB_STENCIL_WRITE_BASE                                                                        0x0016
+#define regDB_STENCIL_WRITE_BASE_BASE_IDX                                                               1
+#define regDB_STENCIL_WRITE_BASE_HI                                                                     0x0017
+#define regDB_STENCIL_WRITE_BASE_HI_BASE_IDX                                                            1
+#define regDB_DFSM_CONTROL                                                                              0x0018
+#define regDB_DFSM_CONTROL_BASE_IDX                                                                     1
+#define regDB_Z_INFO2                                                                                   0x001a
+#define regDB_Z_INFO2_BASE_IDX                                                                          1
+#define regDB_STENCIL_INFO2                                                                             0x001b
+#define regDB_STENCIL_INFO2_BASE_IDX                                                                    1
+#define regCOHER_DEST_BASE_HI_0                                                                         0x007a
+#define regCOHER_DEST_BASE_HI_0_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_HI_1                                                                         0x007b
+#define regCOHER_DEST_BASE_HI_1_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_HI_2                                                                         0x007c
+#define regCOHER_DEST_BASE_HI_2_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_HI_3                                                                         0x007d
+#define regCOHER_DEST_BASE_HI_3_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_2                                                                            0x007e
+#define regCOHER_DEST_BASE_2_BASE_IDX                                                                   1
+#define regCOHER_DEST_BASE_3                                                                            0x007f
+#define regCOHER_DEST_BASE_3_BASE_IDX                                                                   1
+#define regPA_SC_WINDOW_OFFSET                                                                          0x0080
+#define regPA_SC_WINDOW_OFFSET_BASE_IDX                                                                 1
+#define regPA_SC_WINDOW_SCISSOR_TL                                                                      0x0081
+#define regPA_SC_WINDOW_SCISSOR_TL_BASE_IDX                                                             1
+#define regPA_SC_WINDOW_SCISSOR_BR                                                                      0x0082
+#define regPA_SC_WINDOW_SCISSOR_BR_BASE_IDX                                                             1
+#define regPA_SC_CLIPRECT_RULE                                                                          0x0083
+#define regPA_SC_CLIPRECT_RULE_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_0_TL                                                                          0x0084
+#define regPA_SC_CLIPRECT_0_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_0_BR                                                                          0x0085
+#define regPA_SC_CLIPRECT_0_BR_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_1_TL                                                                          0x0086
+#define regPA_SC_CLIPRECT_1_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_1_BR                                                                          0x0087
+#define regPA_SC_CLIPRECT_1_BR_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_2_TL                                                                          0x0088
+#define regPA_SC_CLIPRECT_2_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_2_BR                                                                          0x0089
+#define regPA_SC_CLIPRECT_2_BR_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_3_TL                                                                          0x008a
+#define regPA_SC_CLIPRECT_3_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_3_BR                                                                          0x008b
+#define regPA_SC_CLIPRECT_3_BR_BASE_IDX                                                                 1
+#define regPA_SC_EDGERULE                                                                               0x008c
+#define regPA_SC_EDGERULE_BASE_IDX                                                                      1
+#define regPA_SU_HARDWARE_SCREEN_OFFSET                                                                 0x008d
+#define regPA_SU_HARDWARE_SCREEN_OFFSET_BASE_IDX                                                        1
+#define regCB_TARGET_MASK                                                                               0x008e
+#define regCB_TARGET_MASK_BASE_IDX                                                                      1
+#define regCB_SHADER_MASK                                                                               0x008f
+#define regCB_SHADER_MASK_BASE_IDX                                                                      1
+#define regPA_SC_GENERIC_SCISSOR_TL                                                                     0x0090
+#define regPA_SC_GENERIC_SCISSOR_TL_BASE_IDX                                                            1
+#define regPA_SC_GENERIC_SCISSOR_BR                                                                     0x0091
+#define regPA_SC_GENERIC_SCISSOR_BR_BASE_IDX                                                            1
+#define regCOHER_DEST_BASE_0                                                                            0x0092
+#define regCOHER_DEST_BASE_0_BASE_IDX                                                                   1
+#define regCOHER_DEST_BASE_1                                                                            0x0093
+#define regCOHER_DEST_BASE_1_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_SCISSOR_0_TL                                                                     0x0094
+#define regPA_SC_VPORT_SCISSOR_0_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_0_BR                                                                     0x0095
+#define regPA_SC_VPORT_SCISSOR_0_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_1_TL                                                                     0x0096
+#define regPA_SC_VPORT_SCISSOR_1_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_1_BR                                                                     0x0097
+#define regPA_SC_VPORT_SCISSOR_1_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_2_TL                                                                     0x0098
+#define regPA_SC_VPORT_SCISSOR_2_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_2_BR                                                                     0x0099
+#define regPA_SC_VPORT_SCISSOR_2_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_3_TL                                                                     0x009a
+#define regPA_SC_VPORT_SCISSOR_3_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_3_BR                                                                     0x009b
+#define regPA_SC_VPORT_SCISSOR_3_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_4_TL                                                                     0x009c
+#define regPA_SC_VPORT_SCISSOR_4_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_4_BR                                                                     0x009d
+#define regPA_SC_VPORT_SCISSOR_4_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_5_TL                                                                     0x009e
+#define regPA_SC_VPORT_SCISSOR_5_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_5_BR                                                                     0x009f
+#define regPA_SC_VPORT_SCISSOR_5_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_6_TL                                                                     0x00a0
+#define regPA_SC_VPORT_SCISSOR_6_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_6_BR                                                                     0x00a1
+#define regPA_SC_VPORT_SCISSOR_6_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_7_TL                                                                     0x00a2
+#define regPA_SC_VPORT_SCISSOR_7_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_7_BR                                                                     0x00a3
+#define regPA_SC_VPORT_SCISSOR_7_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_8_TL                                                                     0x00a4
+#define regPA_SC_VPORT_SCISSOR_8_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_8_BR                                                                     0x00a5
+#define regPA_SC_VPORT_SCISSOR_8_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_9_TL                                                                     0x00a6
+#define regPA_SC_VPORT_SCISSOR_9_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_9_BR                                                                     0x00a7
+#define regPA_SC_VPORT_SCISSOR_9_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_10_TL                                                                    0x00a8
+#define regPA_SC_VPORT_SCISSOR_10_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_10_BR                                                                    0x00a9
+#define regPA_SC_VPORT_SCISSOR_10_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_11_TL                                                                    0x00aa
+#define regPA_SC_VPORT_SCISSOR_11_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_11_BR                                                                    0x00ab
+#define regPA_SC_VPORT_SCISSOR_11_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_12_TL                                                                    0x00ac
+#define regPA_SC_VPORT_SCISSOR_12_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_12_BR                                                                    0x00ad
+#define regPA_SC_VPORT_SCISSOR_12_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_13_TL                                                                    0x00ae
+#define regPA_SC_VPORT_SCISSOR_13_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_13_BR                                                                    0x00af
+#define regPA_SC_VPORT_SCISSOR_13_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_14_TL                                                                    0x00b0
+#define regPA_SC_VPORT_SCISSOR_14_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_14_BR                                                                    0x00b1
+#define regPA_SC_VPORT_SCISSOR_14_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_15_TL                                                                    0x00b2
+#define regPA_SC_VPORT_SCISSOR_15_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_15_BR                                                                    0x00b3
+#define regPA_SC_VPORT_SCISSOR_15_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_ZMIN_0                                                                           0x00b4
+#define regPA_SC_VPORT_ZMIN_0_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_0                                                                           0x00b5
+#define regPA_SC_VPORT_ZMAX_0_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_1                                                                           0x00b6
+#define regPA_SC_VPORT_ZMIN_1_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_1                                                                           0x00b7
+#define regPA_SC_VPORT_ZMAX_1_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_2                                                                           0x00b8
+#define regPA_SC_VPORT_ZMIN_2_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_2                                                                           0x00b9
+#define regPA_SC_VPORT_ZMAX_2_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_3                                                                           0x00ba
+#define regPA_SC_VPORT_ZMIN_3_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_3                                                                           0x00bb
+#define regPA_SC_VPORT_ZMAX_3_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_4                                                                           0x00bc
+#define regPA_SC_VPORT_ZMIN_4_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_4                                                                           0x00bd
+#define regPA_SC_VPORT_ZMAX_4_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_5                                                                           0x00be
+#define regPA_SC_VPORT_ZMIN_5_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_5                                                                           0x00bf
+#define regPA_SC_VPORT_ZMAX_5_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_6                                                                           0x00c0
+#define regPA_SC_VPORT_ZMIN_6_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_6                                                                           0x00c1
+#define regPA_SC_VPORT_ZMAX_6_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_7                                                                           0x00c2
+#define regPA_SC_VPORT_ZMIN_7_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_7                                                                           0x00c3
+#define regPA_SC_VPORT_ZMAX_7_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_8                                                                           0x00c4
+#define regPA_SC_VPORT_ZMIN_8_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_8                                                                           0x00c5
+#define regPA_SC_VPORT_ZMAX_8_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_9                                                                           0x00c6
+#define regPA_SC_VPORT_ZMIN_9_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_9                                                                           0x00c7
+#define regPA_SC_VPORT_ZMAX_9_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_10                                                                          0x00c8
+#define regPA_SC_VPORT_ZMIN_10_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_10                                                                          0x00c9
+#define regPA_SC_VPORT_ZMAX_10_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_11                                                                          0x00ca
+#define regPA_SC_VPORT_ZMIN_11_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_11                                                                          0x00cb
+#define regPA_SC_VPORT_ZMAX_11_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_12                                                                          0x00cc
+#define regPA_SC_VPORT_ZMIN_12_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_12                                                                          0x00cd
+#define regPA_SC_VPORT_ZMAX_12_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_13                                                                          0x00ce
+#define regPA_SC_VPORT_ZMIN_13_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_13                                                                          0x00cf
+#define regPA_SC_VPORT_ZMAX_13_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_14                                                                          0x00d0
+#define regPA_SC_VPORT_ZMIN_14_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_14                                                                          0x00d1
+#define regPA_SC_VPORT_ZMAX_14_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_15                                                                          0x00d2
+#define regPA_SC_VPORT_ZMIN_15_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_15                                                                          0x00d3
+#define regPA_SC_VPORT_ZMAX_15_BASE_IDX                                                                 1
+#define regPA_SC_RASTER_CONFIG                                                                          0x00d4
+#define regPA_SC_RASTER_CONFIG_BASE_IDX                                                                 1
+#define regPA_SC_RASTER_CONFIG_1                                                                        0x00d5
+#define regPA_SC_RASTER_CONFIG_1_BASE_IDX                                                               1
+#define regPA_SC_SCREEN_EXTENT_CONTROL                                                                  0x00d6
+#define regPA_SC_SCREEN_EXTENT_CONTROL_BASE_IDX                                                         1
+#define regPA_SC_TILE_STEERING_OVERRIDE                                                                 0x00d7
+#define regPA_SC_TILE_STEERING_OVERRIDE_BASE_IDX                                                        1
+#define regCP_PERFMON_CNTX_CNTL                                                                         0x00d8
+#define regCP_PERFMON_CNTX_CNTL_BASE_IDX                                                                1
+#define regCP_PIPEID                                                                                    0x00d9
+#define regCP_PIPEID_BASE_IDX                                                                           1
+#define regCP_RINGID                                                                                    0x00d9
+#define regCP_RINGID_BASE_IDX                                                                           1
+#define regCP_VMID                                                                                      0x00da
+#define regCP_VMID_BASE_IDX                                                                             1
+#define regPA_SC_RIGHT_VERT_GRID                                                                        0x00e8
+#define regPA_SC_RIGHT_VERT_GRID_BASE_IDX                                                               1
+#define regPA_SC_LEFT_VERT_GRID                                                                         0x00e9
+#define regPA_SC_LEFT_VERT_GRID_BASE_IDX                                                                1
+#define regPA_SC_HORIZ_GRID                                                                             0x00ea
+#define regPA_SC_HORIZ_GRID_BASE_IDX                                                                    1
+#define regVGT_MULTI_PRIM_IB_RESET_INDX                                                                 0x0103
+#define regVGT_MULTI_PRIM_IB_RESET_INDX_BASE_IDX                                                        1
+#define regCB_BLEND_RED                                                                                 0x0105
+#define regCB_BLEND_RED_BASE_IDX                                                                        1
+#define regCB_BLEND_GREEN                                                                               0x0106
+#define regCB_BLEND_GREEN_BASE_IDX                                                                      1
+#define regCB_BLEND_BLUE                                                                                0x0107
+#define regCB_BLEND_BLUE_BASE_IDX                                                                       1
+#define regCB_BLEND_ALPHA                                                                               0x0108
+#define regCB_BLEND_ALPHA_BASE_IDX                                                                      1
+#define regCB_DCC_CONTROL                                                                               0x0109
+#define regCB_DCC_CONTROL_BASE_IDX                                                                      1
+#define regDB_STENCIL_CONTROL                                                                           0x010b
+#define regDB_STENCIL_CONTROL_BASE_IDX                                                                  1
+#define regDB_STENCILREFMASK                                                                            0x010c
+#define regDB_STENCILREFMASK_BASE_IDX                                                                   1
+#define regDB_STENCILREFMASK_BF                                                                         0x010d
+#define regDB_STENCILREFMASK_BF_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XSCALE                                                                           0x010f
+#define regPA_CL_VPORT_XSCALE_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XOFFSET                                                                          0x0110
+#define regPA_CL_VPORT_XOFFSET_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_YSCALE                                                                           0x0111
+#define regPA_CL_VPORT_YSCALE_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_YOFFSET                                                                          0x0112
+#define regPA_CL_VPORT_YOFFSET_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_ZSCALE                                                                           0x0113
+#define regPA_CL_VPORT_ZSCALE_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_ZOFFSET                                                                          0x0114
+#define regPA_CL_VPORT_ZOFFSET_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_XSCALE_1                                                                         0x0115
+#define regPA_CL_VPORT_XSCALE_1_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_1                                                                        0x0116
+#define regPA_CL_VPORT_XOFFSET_1_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_1                                                                         0x0117
+#define regPA_CL_VPORT_YSCALE_1_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_1                                                                        0x0118
+#define regPA_CL_VPORT_YOFFSET_1_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_1                                                                         0x0119
+#define regPA_CL_VPORT_ZSCALE_1_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_1                                                                        0x011a
+#define regPA_CL_VPORT_ZOFFSET_1_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_2                                                                         0x011b
+#define regPA_CL_VPORT_XSCALE_2_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_2                                                                        0x011c
+#define regPA_CL_VPORT_XOFFSET_2_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_2                                                                         0x011d
+#define regPA_CL_VPORT_YSCALE_2_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_2                                                                        0x011e
+#define regPA_CL_VPORT_YOFFSET_2_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_2                                                                         0x011f
+#define regPA_CL_VPORT_ZSCALE_2_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_2                                                                        0x0120
+#define regPA_CL_VPORT_ZOFFSET_2_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_3                                                                         0x0121
+#define regPA_CL_VPORT_XSCALE_3_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_3                                                                        0x0122
+#define regPA_CL_VPORT_XOFFSET_3_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_3                                                                         0x0123
+#define regPA_CL_VPORT_YSCALE_3_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_3                                                                        0x0124
+#define regPA_CL_VPORT_YOFFSET_3_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_3                                                                         0x0125
+#define regPA_CL_VPORT_ZSCALE_3_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_3                                                                        0x0126
+#define regPA_CL_VPORT_ZOFFSET_3_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_4                                                                         0x0127
+#define regPA_CL_VPORT_XSCALE_4_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_4                                                                        0x0128
+#define regPA_CL_VPORT_XOFFSET_4_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_4                                                                         0x0129
+#define regPA_CL_VPORT_YSCALE_4_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_4                                                                        0x012a
+#define regPA_CL_VPORT_YOFFSET_4_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_4                                                                         0x012b
+#define regPA_CL_VPORT_ZSCALE_4_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_4                                                                        0x012c
+#define regPA_CL_VPORT_ZOFFSET_4_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_5                                                                         0x012d
+#define regPA_CL_VPORT_XSCALE_5_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_5                                                                        0x012e
+#define regPA_CL_VPORT_XOFFSET_5_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_5                                                                         0x012f
+#define regPA_CL_VPORT_YSCALE_5_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_5                                                                        0x0130
+#define regPA_CL_VPORT_YOFFSET_5_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_5                                                                         0x0131
+#define regPA_CL_VPORT_ZSCALE_5_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_5                                                                        0x0132
+#define regPA_CL_VPORT_ZOFFSET_5_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_6                                                                         0x0133
+#define regPA_CL_VPORT_XSCALE_6_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_6                                                                        0x0134
+#define regPA_CL_VPORT_XOFFSET_6_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_6                                                                         0x0135
+#define regPA_CL_VPORT_YSCALE_6_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_6                                                                        0x0136
+#define regPA_CL_VPORT_YOFFSET_6_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_6                                                                         0x0137
+#define regPA_CL_VPORT_ZSCALE_6_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_6                                                                        0x0138
+#define regPA_CL_VPORT_ZOFFSET_6_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_7                                                                         0x0139
+#define regPA_CL_VPORT_XSCALE_7_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_7                                                                        0x013a
+#define regPA_CL_VPORT_XOFFSET_7_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_7                                                                         0x013b
+#define regPA_CL_VPORT_YSCALE_7_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_7                                                                        0x013c
+#define regPA_CL_VPORT_YOFFSET_7_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_7                                                                         0x013d
+#define regPA_CL_VPORT_ZSCALE_7_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_7                                                                        0x013e
+#define regPA_CL_VPORT_ZOFFSET_7_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_8                                                                         0x013f
+#define regPA_CL_VPORT_XSCALE_8_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_8                                                                        0x0140
+#define regPA_CL_VPORT_XOFFSET_8_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_8                                                                         0x0141
+#define regPA_CL_VPORT_YSCALE_8_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_8                                                                        0x0142
+#define regPA_CL_VPORT_YOFFSET_8_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_8                                                                         0x0143
+#define regPA_CL_VPORT_ZSCALE_8_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_8                                                                        0x0144
+#define regPA_CL_VPORT_ZOFFSET_8_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_9                                                                         0x0145
+#define regPA_CL_VPORT_XSCALE_9_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_9                                                                        0x0146
+#define regPA_CL_VPORT_XOFFSET_9_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_9                                                                         0x0147
+#define regPA_CL_VPORT_YSCALE_9_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_9                                                                        0x0148
+#define regPA_CL_VPORT_YOFFSET_9_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_9                                                                         0x0149
+#define regPA_CL_VPORT_ZSCALE_9_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_9                                                                        0x014a
+#define regPA_CL_VPORT_ZOFFSET_9_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_10                                                                        0x014b
+#define regPA_CL_VPORT_XSCALE_10_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_10                                                                       0x014c
+#define regPA_CL_VPORT_XOFFSET_10_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_10                                                                        0x014d
+#define regPA_CL_VPORT_YSCALE_10_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_10                                                                       0x014e
+#define regPA_CL_VPORT_YOFFSET_10_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_10                                                                        0x014f
+#define regPA_CL_VPORT_ZSCALE_10_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_10                                                                       0x0150
+#define regPA_CL_VPORT_ZOFFSET_10_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE_11                                                                        0x0151
+#define regPA_CL_VPORT_XSCALE_11_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_11                                                                       0x0152
+#define regPA_CL_VPORT_XOFFSET_11_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_11                                                                        0x0153
+#define regPA_CL_VPORT_YSCALE_11_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_11                                                                       0x0154
+#define regPA_CL_VPORT_YOFFSET_11_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_11                                                                        0x0155
+#define regPA_CL_VPORT_ZSCALE_11_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_11                                                                       0x0156
+#define regPA_CL_VPORT_ZOFFSET_11_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE_12                                                                        0x0157
+#define regPA_CL_VPORT_XSCALE_12_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_12                                                                       0x0158
+#define regPA_CL_VPORT_XOFFSET_12_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_12                                                                        0x0159
+#define regPA_CL_VPORT_YSCALE_12_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_12                                                                       0x015a
+#define regPA_CL_VPORT_YOFFSET_12_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_12                                                                        0x015b
+#define regPA_CL_VPORT_ZSCALE_12_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_12                                                                       0x015c
+#define regPA_CL_VPORT_ZOFFSET_12_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE_13                                                                        0x015d
+#define regPA_CL_VPORT_XSCALE_13_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_13                                                                       0x015e
+#define regPA_CL_VPORT_XOFFSET_13_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_13                                                                        0x015f
+#define regPA_CL_VPORT_YSCALE_13_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_13                                                                       0x0160
+#define regPA_CL_VPORT_YOFFSET_13_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_13                                                                        0x0161
+#define regPA_CL_VPORT_ZSCALE_13_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_13                                                                       0x0162
+#define regPA_CL_VPORT_ZOFFSET_13_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE_14                                                                        0x0163
+#define regPA_CL_VPORT_XSCALE_14_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_14                                                                       0x0164
+#define regPA_CL_VPORT_XOFFSET_14_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_14                                                                        0x0165
+#define regPA_CL_VPORT_YSCALE_14_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_14                                                                       0x0166
+#define regPA_CL_VPORT_YOFFSET_14_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_14                                                                        0x0167
+#define regPA_CL_VPORT_ZSCALE_14_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_14                                                                       0x0168
+#define regPA_CL_VPORT_ZOFFSET_14_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE_15                                                                        0x0169
+#define regPA_CL_VPORT_XSCALE_15_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_15                                                                       0x016a
+#define regPA_CL_VPORT_XOFFSET_15_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_15                                                                        0x016b
+#define regPA_CL_VPORT_YSCALE_15_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_15                                                                       0x016c
+#define regPA_CL_VPORT_YOFFSET_15_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_15                                                                        0x016d
+#define regPA_CL_VPORT_ZSCALE_15_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_15                                                                       0x016e
+#define regPA_CL_VPORT_ZOFFSET_15_BASE_IDX                                                              1
+#define regPA_CL_UCP_0_X                                                                                0x016f
+#define regPA_CL_UCP_0_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_0_Y                                                                                0x0170
+#define regPA_CL_UCP_0_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_0_Z                                                                                0x0171
+#define regPA_CL_UCP_0_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_0_W                                                                                0x0172
+#define regPA_CL_UCP_0_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_X                                                                                0x0173
+#define regPA_CL_UCP_1_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_Y                                                                                0x0174
+#define regPA_CL_UCP_1_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_Z                                                                                0x0175
+#define regPA_CL_UCP_1_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_W                                                                                0x0176
+#define regPA_CL_UCP_1_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_X                                                                                0x0177
+#define regPA_CL_UCP_2_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_Y                                                                                0x0178
+#define regPA_CL_UCP_2_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_Z                                                                                0x0179
+#define regPA_CL_UCP_2_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_W                                                                                0x017a
+#define regPA_CL_UCP_2_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_X                                                                                0x017b
+#define regPA_CL_UCP_3_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_Y                                                                                0x017c
+#define regPA_CL_UCP_3_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_Z                                                                                0x017d
+#define regPA_CL_UCP_3_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_W                                                                                0x017e
+#define regPA_CL_UCP_3_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_X                                                                                0x017f
+#define regPA_CL_UCP_4_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_Y                                                                                0x0180
+#define regPA_CL_UCP_4_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_Z                                                                                0x0181
+#define regPA_CL_UCP_4_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_W                                                                                0x0182
+#define regPA_CL_UCP_4_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_X                                                                                0x0183
+#define regPA_CL_UCP_5_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_Y                                                                                0x0184
+#define regPA_CL_UCP_5_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_Z                                                                                0x0185
+#define regPA_CL_UCP_5_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_W                                                                                0x0186
+#define regPA_CL_UCP_5_W_BASE_IDX                                                                       1
+#define regPA_CL_PROG_NEAR_CLIP_Z                                                                       0x0187
+#define regPA_CL_PROG_NEAR_CLIP_Z_BASE_IDX                                                              1
+#define regSPI_PS_INPUT_CNTL_0                                                                          0x0191
+#define regSPI_PS_INPUT_CNTL_0_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_1                                                                          0x0192
+#define regSPI_PS_INPUT_CNTL_1_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_2                                                                          0x0193
+#define regSPI_PS_INPUT_CNTL_2_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_3                                                                          0x0194
+#define regSPI_PS_INPUT_CNTL_3_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_4                                                                          0x0195
+#define regSPI_PS_INPUT_CNTL_4_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_5                                                                          0x0196
+#define regSPI_PS_INPUT_CNTL_5_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_6                                                                          0x0197
+#define regSPI_PS_INPUT_CNTL_6_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_7                                                                          0x0198
+#define regSPI_PS_INPUT_CNTL_7_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_8                                                                          0x0199
+#define regSPI_PS_INPUT_CNTL_8_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_9                                                                          0x019a
+#define regSPI_PS_INPUT_CNTL_9_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_10                                                                         0x019b
+#define regSPI_PS_INPUT_CNTL_10_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_11                                                                         0x019c
+#define regSPI_PS_INPUT_CNTL_11_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_12                                                                         0x019d
+#define regSPI_PS_INPUT_CNTL_12_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_13                                                                         0x019e
+#define regSPI_PS_INPUT_CNTL_13_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_14                                                                         0x019f
+#define regSPI_PS_INPUT_CNTL_14_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_15                                                                         0x01a0
+#define regSPI_PS_INPUT_CNTL_15_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_16                                                                         0x01a1
+#define regSPI_PS_INPUT_CNTL_16_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_17                                                                         0x01a2
+#define regSPI_PS_INPUT_CNTL_17_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_18                                                                         0x01a3
+#define regSPI_PS_INPUT_CNTL_18_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_19                                                                         0x01a4
+#define regSPI_PS_INPUT_CNTL_19_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_20                                                                         0x01a5
+#define regSPI_PS_INPUT_CNTL_20_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_21                                                                         0x01a6
+#define regSPI_PS_INPUT_CNTL_21_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_22                                                                         0x01a7
+#define regSPI_PS_INPUT_CNTL_22_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_23                                                                         0x01a8
+#define regSPI_PS_INPUT_CNTL_23_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_24                                                                         0x01a9
+#define regSPI_PS_INPUT_CNTL_24_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_25                                                                         0x01aa
+#define regSPI_PS_INPUT_CNTL_25_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_26                                                                         0x01ab
+#define regSPI_PS_INPUT_CNTL_26_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_27                                                                         0x01ac
+#define regSPI_PS_INPUT_CNTL_27_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_28                                                                         0x01ad
+#define regSPI_PS_INPUT_CNTL_28_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_29                                                                         0x01ae
+#define regSPI_PS_INPUT_CNTL_29_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_30                                                                         0x01af
+#define regSPI_PS_INPUT_CNTL_30_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_31                                                                         0x01b0
+#define regSPI_PS_INPUT_CNTL_31_BASE_IDX                                                                1
+#define regSPI_VS_OUT_CONFIG                                                                            0x01b1
+#define regSPI_VS_OUT_CONFIG_BASE_IDX                                                                   1
+#define regSPI_PS_INPUT_ENA                                                                             0x01b3
+#define regSPI_PS_INPUT_ENA_BASE_IDX                                                                    1
+#define regSPI_PS_INPUT_ADDR                                                                            0x01b4
+#define regSPI_PS_INPUT_ADDR_BASE_IDX                                                                   1
+#define regSPI_INTERP_CONTROL_0                                                                         0x01b5
+#define regSPI_INTERP_CONTROL_0_BASE_IDX                                                                1
+#define regSPI_PS_IN_CONTROL                                                                            0x01b6
+#define regSPI_PS_IN_CONTROL_BASE_IDX                                                                   1
+#define regSPI_BARYC_CNTL                                                                               0x01b8
+#define regSPI_BARYC_CNTL_BASE_IDX                                                                      1
+#define regSPI_TMPRING_SIZE                                                                             0x01ba
+#define regSPI_TMPRING_SIZE_BASE_IDX                                                                    1
+#define regSPI_SHADER_POS_FORMAT                                                                        0x01c3
+#define regSPI_SHADER_POS_FORMAT_BASE_IDX                                                               1
+#define regSPI_SHADER_Z_FORMAT                                                                          0x01c4
+#define regSPI_SHADER_Z_FORMAT_BASE_IDX                                                                 1
+#define regSPI_SHADER_COL_FORMAT                                                                        0x01c5
+#define regSPI_SHADER_COL_FORMAT_BASE_IDX                                                               1
+#define regCB_BLEND0_CONTROL                                                                            0x01e0
+#define regCB_BLEND0_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND1_CONTROL                                                                            0x01e1
+#define regCB_BLEND1_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND2_CONTROL                                                                            0x01e2
+#define regCB_BLEND2_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND3_CONTROL                                                                            0x01e3
+#define regCB_BLEND3_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND4_CONTROL                                                                            0x01e4
+#define regCB_BLEND4_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND5_CONTROL                                                                            0x01e5
+#define regCB_BLEND5_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND6_CONTROL                                                                            0x01e6
+#define regCB_BLEND6_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND7_CONTROL                                                                            0x01e7
+#define regCB_BLEND7_CONTROL_BASE_IDX                                                                   1
+#define regCB_MRT0_EPITCH                                                                               0x01e8
+#define regCB_MRT0_EPITCH_BASE_IDX                                                                      1
+#define regCB_MRT1_EPITCH                                                                               0x01e9
+#define regCB_MRT1_EPITCH_BASE_IDX                                                                      1
+#define regCB_MRT2_EPITCH                                                                               0x01ea
+#define regCB_MRT2_EPITCH_BASE_IDX                                                                      1
+#define regCB_MRT3_EPITCH                                                                               0x01eb
+#define regCB_MRT3_EPITCH_BASE_IDX                                                                      1
+#define regCB_MRT4_EPITCH                                                                               0x01ec
+#define regCB_MRT4_EPITCH_BASE_IDX                                                                      1
+#define regCB_MRT5_EPITCH                                                                               0x01ed
+#define regCB_MRT5_EPITCH_BASE_IDX                                                                      1
+#define regCB_MRT6_EPITCH                                                                               0x01ee
+#define regCB_MRT6_EPITCH_BASE_IDX                                                                      1
+#define regCB_MRT7_EPITCH                                                                               0x01ef
+#define regCB_MRT7_EPITCH_BASE_IDX                                                                      1
+#define regCS_COPY_STATE                                                                                0x01f3
+#define regCS_COPY_STATE_BASE_IDX                                                                       1
+#define regGFX_COPY_STATE                                                                               0x01f4
+#define regGFX_COPY_STATE_BASE_IDX                                                                      1
+#define regPA_CL_POINT_X_RAD                                                                            0x01f5
+#define regPA_CL_POINT_X_RAD_BASE_IDX                                                                   1
+#define regPA_CL_POINT_Y_RAD                                                                            0x01f6
+#define regPA_CL_POINT_Y_RAD_BASE_IDX                                                                   1
+#define regPA_CL_POINT_SIZE                                                                             0x01f7
+#define regPA_CL_POINT_SIZE_BASE_IDX                                                                    1
+#define regPA_CL_POINT_CULL_RAD                                                                         0x01f8
+#define regPA_CL_POINT_CULL_RAD_BASE_IDX                                                                1
+#define regVGT_DMA_BASE_HI                                                                              0x01f9
+#define regVGT_DMA_BASE_HI_BASE_IDX                                                                     1
+#define regVGT_DMA_BASE                                                                                 0x01fa
+#define regVGT_DMA_BASE_BASE_IDX                                                                        1
+#define regVGT_DRAW_INITIATOR                                                                           0x01fc
+#define regVGT_DRAW_INITIATOR_BASE_IDX                                                                  1
+#define regVGT_IMMED_DATA                                                                               0x01fd
+#define regVGT_IMMED_DATA_BASE_IDX                                                                      1
+#define regVGT_EVENT_ADDRESS_REG                                                                        0x01fe
+#define regVGT_EVENT_ADDRESS_REG_BASE_IDX                                                               1
+#define regDB_DEPTH_CONTROL                                                                             0x0200
+#define regDB_DEPTH_CONTROL_BASE_IDX                                                                    1
+#define regDB_EQAA                                                                                      0x0201
+#define regDB_EQAA_BASE_IDX                                                                             1
+#define regCB_COLOR_CONTROL                                                                             0x0202
+#define regCB_COLOR_CONTROL_BASE_IDX                                                                    1
+#define regDB_SHADER_CONTROL                                                                            0x0203
+#define regDB_SHADER_CONTROL_BASE_IDX                                                                   1
+#define regPA_CL_CLIP_CNTL                                                                              0x0204
+#define regPA_CL_CLIP_CNTL_BASE_IDX                                                                     1
+#define regPA_SU_SC_MODE_CNTL                                                                           0x0205
+#define regPA_SU_SC_MODE_CNTL_BASE_IDX                                                                  1
+#define regPA_CL_VTE_CNTL                                                                               0x0206
+#define regPA_CL_VTE_CNTL_BASE_IDX                                                                      1
+#define regPA_CL_VS_OUT_CNTL                                                                            0x0207
+#define regPA_CL_VS_OUT_CNTL_BASE_IDX                                                                   1
+#define regPA_CL_NANINF_CNTL                                                                            0x0208
+#define regPA_CL_NANINF_CNTL_BASE_IDX                                                                   1
+#define regPA_SU_LINE_STIPPLE_CNTL                                                                      0x0209
+#define regPA_SU_LINE_STIPPLE_CNTL_BASE_IDX                                                             1
+#define regPA_SU_LINE_STIPPLE_SCALE                                                                     0x020a
+#define regPA_SU_LINE_STIPPLE_SCALE_BASE_IDX                                                            1
+#define regPA_SU_PRIM_FILTER_CNTL                                                                       0x020b
+#define regPA_SU_PRIM_FILTER_CNTL_BASE_IDX                                                              1
+#define regPA_SU_SMALL_PRIM_FILTER_CNTL                                                                 0x020c
+#define regPA_SU_SMALL_PRIM_FILTER_CNTL_BASE_IDX                                                        1
+#define regPA_CL_OBJPRIM_ID_CNTL                                                                        0x020d
+#define regPA_CL_OBJPRIM_ID_CNTL_BASE_IDX                                                               1
+#define regPA_CL_NGG_CNTL                                                                               0x020e
+#define regPA_CL_NGG_CNTL_BASE_IDX                                                                      1
+#define regPA_SU_OVER_RASTERIZATION_CNTL                                                                0x020f
+#define regPA_SU_OVER_RASTERIZATION_CNTL_BASE_IDX                                                       1
+#define regPA_STEREO_CNTL                                                                               0x0210
+#define regPA_STEREO_CNTL_BASE_IDX                                                                      1
+#define regPA_SU_POINT_SIZE                                                                             0x0280
+#define regPA_SU_POINT_SIZE_BASE_IDX                                                                    1
+#define regPA_SU_POINT_MINMAX                                                                           0x0281
+#define regPA_SU_POINT_MINMAX_BASE_IDX                                                                  1
+#define regPA_SU_LINE_CNTL                                                                              0x0282
+#define regPA_SU_LINE_CNTL_BASE_IDX                                                                     1
+#define regPA_SC_LINE_STIPPLE                                                                           0x0283
+#define regPA_SC_LINE_STIPPLE_BASE_IDX                                                                  1
+#define regVGT_OUTPUT_PATH_CNTL                                                                         0x0284
+#define regVGT_OUTPUT_PATH_CNTL_BASE_IDX                                                                1
+#define regVGT_HOS_CNTL                                                                                 0x0285
+#define regVGT_HOS_CNTL_BASE_IDX                                                                        1
+#define regVGT_HOS_MAX_TESS_LEVEL                                                                       0x0286
+#define regVGT_HOS_MAX_TESS_LEVEL_BASE_IDX                                                              1
+#define regVGT_HOS_MIN_TESS_LEVEL                                                                       0x0287
+#define regVGT_HOS_MIN_TESS_LEVEL_BASE_IDX                                                              1
+#define regVGT_HOS_REUSE_DEPTH                                                                          0x0288
+#define regVGT_HOS_REUSE_DEPTH_BASE_IDX                                                                 1
+#define regVGT_GROUP_PRIM_TYPE                                                                          0x0289
+#define regVGT_GROUP_PRIM_TYPE_BASE_IDX                                                                 1
+#define regVGT_GROUP_FIRST_DECR                                                                         0x028a
+#define regVGT_GROUP_FIRST_DECR_BASE_IDX                                                                1
+#define regVGT_GROUP_DECR                                                                               0x028b
+#define regVGT_GROUP_DECR_BASE_IDX                                                                      1
+#define regVGT_GROUP_VECT_0_CNTL                                                                        0x028c
+#define regVGT_GROUP_VECT_0_CNTL_BASE_IDX                                                               1
+#define regVGT_GROUP_VECT_1_CNTL                                                                        0x028d
+#define regVGT_GROUP_VECT_1_CNTL_BASE_IDX                                                               1
+#define regVGT_GROUP_VECT_0_FMT_CNTL                                                                    0x028e
+#define regVGT_GROUP_VECT_0_FMT_CNTL_BASE_IDX                                                           1
+#define regVGT_GROUP_VECT_1_FMT_CNTL                                                                    0x028f
+#define regVGT_GROUP_VECT_1_FMT_CNTL_BASE_IDX                                                           1
+#define regVGT_GS_MODE                                                                                  0x0290
+#define regVGT_GS_MODE_BASE_IDX                                                                         1
+#define regVGT_GS_ONCHIP_CNTL                                                                           0x0291
+#define regVGT_GS_ONCHIP_CNTL_BASE_IDX                                                                  1
+#define regPA_SC_MODE_CNTL_0                                                                            0x0292
+#define regPA_SC_MODE_CNTL_0_BASE_IDX                                                                   1
+#define regPA_SC_MODE_CNTL_1                                                                            0x0293
+#define regPA_SC_MODE_CNTL_1_BASE_IDX                                                                   1
+#define regVGT_ENHANCE                                                                                  0x0294
+#define regVGT_ENHANCE_BASE_IDX                                                                         1
+#define regVGT_GS_PER_ES                                                                                0x0295
+#define regVGT_GS_PER_ES_BASE_IDX                                                                       1
+#define regVGT_ES_PER_GS                                                                                0x0296
+#define regVGT_ES_PER_GS_BASE_IDX                                                                       1
+#define regVGT_GS_PER_VS                                                                                0x0297
+#define regVGT_GS_PER_VS_BASE_IDX                                                                       1
+#define regVGT_GSVS_RING_OFFSET_1                                                                       0x0298
+#define regVGT_GSVS_RING_OFFSET_1_BASE_IDX                                                              1
+#define regVGT_GSVS_RING_OFFSET_2                                                                       0x0299
+#define regVGT_GSVS_RING_OFFSET_2_BASE_IDX                                                              1
+#define regVGT_GSVS_RING_OFFSET_3                                                                       0x029a
+#define regVGT_GSVS_RING_OFFSET_3_BASE_IDX                                                              1
+#define regVGT_GS_OUT_PRIM_TYPE                                                                         0x029b
+#define regVGT_GS_OUT_PRIM_TYPE_BASE_IDX                                                                1
+#define regIA_ENHANCE                                                                                   0x029c
+#define regIA_ENHANCE_BASE_IDX                                                                          1
+#define regVGT_DMA_SIZE                                                                                 0x029d
+#define regVGT_DMA_SIZE_BASE_IDX                                                                        1
+#define regVGT_DMA_MAX_SIZE                                                                             0x029e
+#define regVGT_DMA_MAX_SIZE_BASE_IDX                                                                    1
+#define regVGT_DMA_INDEX_TYPE                                                                           0x029f
+#define regVGT_DMA_INDEX_TYPE_BASE_IDX                                                                  1
+#define regWD_ENHANCE                                                                                   0x02a0
+#define regWD_ENHANCE_BASE_IDX                                                                          1
+#define regVGT_PRIMITIVEID_EN                                                                           0x02a1
+#define regVGT_PRIMITIVEID_EN_BASE_IDX                                                                  1
+#define regVGT_DMA_NUM_INSTANCES                                                                        0x02a2
+#define regVGT_DMA_NUM_INSTANCES_BASE_IDX                                                               1
+#define regVGT_PRIMITIVEID_RESET                                                                        0x02a3
+#define regVGT_PRIMITIVEID_RESET_BASE_IDX                                                               1
+#define regVGT_EVENT_INITIATOR                                                                          0x02a4
+#define regVGT_EVENT_INITIATOR_BASE_IDX                                                                 1
+#define regVGT_GS_MAX_PRIMS_PER_SUBGROUP                                                                0x02a5
+#define regVGT_GS_MAX_PRIMS_PER_SUBGROUP_BASE_IDX                                                       1
+#define regVGT_DRAW_PAYLOAD_CNTL                                                                        0x02a6
+#define regVGT_DRAW_PAYLOAD_CNTL_BASE_IDX                                                               1
+#define regVGT_INSTANCE_STEP_RATE_0                                                                     0x02a8
+#define regVGT_INSTANCE_STEP_RATE_0_BASE_IDX                                                            1
+#define regVGT_INSTANCE_STEP_RATE_1                                                                     0x02a9
+#define regVGT_INSTANCE_STEP_RATE_1_BASE_IDX                                                            1
+#define regIA_MULTI_VGT_PARAM_BC                                                                        0x02aa
+#define regIA_MULTI_VGT_PARAM_BC_BASE_IDX                                                               1
+#define regVGT_ESGS_RING_ITEMSIZE                                                                       0x02ab
+#define regVGT_ESGS_RING_ITEMSIZE_BASE_IDX                                                              1
+#define regVGT_GSVS_RING_ITEMSIZE                                                                       0x02ac
+#define regVGT_GSVS_RING_ITEMSIZE_BASE_IDX                                                              1
+#define regVGT_REUSE_OFF                                                                                0x02ad
+#define regVGT_REUSE_OFF_BASE_IDX                                                                       1
+#define regVGT_VTX_CNT_EN                                                                               0x02ae
+#define regVGT_VTX_CNT_EN_BASE_IDX                                                                      1
+#define regDB_HTILE_SURFACE                                                                             0x02af
+#define regDB_HTILE_SURFACE_BASE_IDX                                                                    1
+#define regDB_SRESULTS_COMPARE_STATE0                                                                   0x02b0
+#define regDB_SRESULTS_COMPARE_STATE0_BASE_IDX                                                          1
+#define regDB_SRESULTS_COMPARE_STATE1                                                                   0x02b1
+#define regDB_SRESULTS_COMPARE_STATE1_BASE_IDX                                                          1
+#define regDB_PRELOAD_CONTROL                                                                           0x02b2
+#define regDB_PRELOAD_CONTROL_BASE_IDX                                                                  1
+#define regVGT_STRMOUT_BUFFER_SIZE_0                                                                    0x02b4
+#define regVGT_STRMOUT_BUFFER_SIZE_0_BASE_IDX                                                           1
+#define regVGT_STRMOUT_VTX_STRIDE_0                                                                     0x02b5
+#define regVGT_STRMOUT_VTX_STRIDE_0_BASE_IDX                                                            1
+#define regVGT_STRMOUT_BUFFER_OFFSET_0                                                                  0x02b7
+#define regVGT_STRMOUT_BUFFER_OFFSET_0_BASE_IDX                                                         1
+#define regVGT_STRMOUT_BUFFER_SIZE_1                                                                    0x02b8
+#define regVGT_STRMOUT_BUFFER_SIZE_1_BASE_IDX                                                           1
+#define regVGT_STRMOUT_VTX_STRIDE_1                                                                     0x02b9
+#define regVGT_STRMOUT_VTX_STRIDE_1_BASE_IDX                                                            1
+#define regVGT_STRMOUT_BUFFER_OFFSET_1                                                                  0x02bb
+#define regVGT_STRMOUT_BUFFER_OFFSET_1_BASE_IDX                                                         1
+#define regVGT_STRMOUT_BUFFER_SIZE_2                                                                    0x02bc
+#define regVGT_STRMOUT_BUFFER_SIZE_2_BASE_IDX                                                           1
+#define regVGT_STRMOUT_VTX_STRIDE_2                                                                     0x02bd
+#define regVGT_STRMOUT_VTX_STRIDE_2_BASE_IDX                                                            1
+#define regVGT_STRMOUT_BUFFER_OFFSET_2                                                                  0x02bf
+#define regVGT_STRMOUT_BUFFER_OFFSET_2_BASE_IDX                                                         1
+#define regVGT_STRMOUT_BUFFER_SIZE_3                                                                    0x02c0
+#define regVGT_STRMOUT_BUFFER_SIZE_3_BASE_IDX                                                           1
+#define regVGT_STRMOUT_VTX_STRIDE_3                                                                     0x02c1
+#define regVGT_STRMOUT_VTX_STRIDE_3_BASE_IDX                                                            1
+#define regVGT_STRMOUT_BUFFER_OFFSET_3                                                                  0x02c3
+#define regVGT_STRMOUT_BUFFER_OFFSET_3_BASE_IDX                                                         1
+#define regVGT_STRMOUT_DRAW_OPAQUE_OFFSET                                                               0x02ca
+#define regVGT_STRMOUT_DRAW_OPAQUE_OFFSET_BASE_IDX                                                      1
+#define regVGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE                                                   0x02cb
+#define regVGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE_BASE_IDX                                          1
+#define regVGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE                                                        0x02cc
+#define regVGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE_BASE_IDX                                               1
+#define regVGT_GS_MAX_VERT_OUT                                                                          0x02ce
+#define regVGT_GS_MAX_VERT_OUT_BASE_IDX                                                                 1
+#define regVGT_TESS_DISTRIBUTION                                                                        0x02d4
+#define regVGT_TESS_DISTRIBUTION_BASE_IDX                                                               1
+#define regVGT_SHADER_STAGES_EN                                                                         0x02d5
+#define regVGT_SHADER_STAGES_EN_BASE_IDX                                                                1
+#define regVGT_LS_HS_CONFIG                                                                             0x02d6
+#define regVGT_LS_HS_CONFIG_BASE_IDX                                                                    1
+#define regVGT_GS_VERT_ITEMSIZE                                                                         0x02d7
+#define regVGT_GS_VERT_ITEMSIZE_BASE_IDX                                                                1
+#define regVGT_GS_VERT_ITEMSIZE_1                                                                       0x02d8
+#define regVGT_GS_VERT_ITEMSIZE_1_BASE_IDX                                                              1
+#define regVGT_GS_VERT_ITEMSIZE_2                                                                       0x02d9
+#define regVGT_GS_VERT_ITEMSIZE_2_BASE_IDX                                                              1
+#define regVGT_GS_VERT_ITEMSIZE_3                                                                       0x02da
+#define regVGT_GS_VERT_ITEMSIZE_3_BASE_IDX                                                              1
+#define regVGT_TF_PARAM                                                                                 0x02db
+#define regVGT_TF_PARAM_BASE_IDX                                                                        1
+#define regDB_ALPHA_TO_MASK                                                                             0x02dc
+#define regDB_ALPHA_TO_MASK_BASE_IDX                                                                    1
+#define regVGT_DISPATCH_DRAW_INDEX                                                                      0x02dd
+#define regVGT_DISPATCH_DRAW_INDEX_BASE_IDX                                                             1
+#define regPA_SU_POLY_OFFSET_DB_FMT_CNTL                                                                0x02de
+#define regPA_SU_POLY_OFFSET_DB_FMT_CNTL_BASE_IDX                                                       1
+#define regPA_SU_POLY_OFFSET_CLAMP                                                                      0x02df
+#define regPA_SU_POLY_OFFSET_CLAMP_BASE_IDX                                                             1
+#define regPA_SU_POLY_OFFSET_FRONT_SCALE                                                                0x02e0
+#define regPA_SU_POLY_OFFSET_FRONT_SCALE_BASE_IDX                                                       1
+#define regPA_SU_POLY_OFFSET_FRONT_OFFSET                                                               0x02e1
+#define regPA_SU_POLY_OFFSET_FRONT_OFFSET_BASE_IDX                                                      1
+#define regPA_SU_POLY_OFFSET_BACK_SCALE                                                                 0x02e2
+#define regPA_SU_POLY_OFFSET_BACK_SCALE_BASE_IDX                                                        1
+#define regPA_SU_POLY_OFFSET_BACK_OFFSET                                                                0x02e3
+#define regPA_SU_POLY_OFFSET_BACK_OFFSET_BASE_IDX                                                       1
+#define regVGT_GS_INSTANCE_CNT                                                                          0x02e4
+#define regVGT_GS_INSTANCE_CNT_BASE_IDX                                                                 1
+#define regVGT_STRMOUT_CONFIG                                                                           0x02e5
+#define regVGT_STRMOUT_CONFIG_BASE_IDX                                                                  1
+#define regVGT_STRMOUT_BUFFER_CONFIG                                                                    0x02e6
+#define regVGT_STRMOUT_BUFFER_CONFIG_BASE_IDX                                                           1
+#define regVGT_DMA_EVENT_INITIATOR                                                                      0x02e7
+#define regVGT_DMA_EVENT_INITIATOR_BASE_IDX                                                             1
+#define regPA_SC_CENTROID_PRIORITY_0                                                                    0x02f5
+#define regPA_SC_CENTROID_PRIORITY_0_BASE_IDX                                                           1
+#define regPA_SC_CENTROID_PRIORITY_1                                                                    0x02f6
+#define regPA_SC_CENTROID_PRIORITY_1_BASE_IDX                                                           1
+#define regPA_SC_LINE_CNTL                                                                              0x02f7
+#define regPA_SC_LINE_CNTL_BASE_IDX                                                                     1
+#define regPA_SC_AA_CONFIG                                                                              0x02f8
+#define regPA_SC_AA_CONFIG_BASE_IDX                                                                     1
+#define regPA_SU_VTX_CNTL                                                                               0x02f9
+#define regPA_SU_VTX_CNTL_BASE_IDX                                                                      1
+#define regPA_CL_GB_VERT_CLIP_ADJ                                                                       0x02fa
+#define regPA_CL_GB_VERT_CLIP_ADJ_BASE_IDX                                                              1
+#define regPA_CL_GB_VERT_DISC_ADJ                                                                       0x02fb
+#define regPA_CL_GB_VERT_DISC_ADJ_BASE_IDX                                                              1
+#define regPA_CL_GB_HORZ_CLIP_ADJ                                                                       0x02fc
+#define regPA_CL_GB_HORZ_CLIP_ADJ_BASE_IDX                                                              1
+#define regPA_CL_GB_HORZ_DISC_ADJ                                                                       0x02fd
+#define regPA_CL_GB_HORZ_DISC_ADJ_BASE_IDX                                                              1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0                                                            0x02fe
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1                                                            0x02ff
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2                                                            0x0300
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3                                                            0x0301
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0                                                            0x0302
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1                                                            0x0303
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2                                                            0x0304
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3                                                            0x0305
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0                                                            0x0306
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1                                                            0x0307
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2                                                            0x0308
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3                                                            0x0309
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0                                                            0x030a
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1                                                            0x030b
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2                                                            0x030c
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3                                                            0x030d
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3_BASE_IDX                                                   1
+#define regPA_SC_AA_MASK_X0Y0_X1Y0                                                                      0x030e
+#define regPA_SC_AA_MASK_X0Y0_X1Y0_BASE_IDX                                                             1
+#define regPA_SC_AA_MASK_X0Y1_X1Y1                                                                      0x030f
+#define regPA_SC_AA_MASK_X0Y1_X1Y1_BASE_IDX                                                             1
+#define regPA_SC_SHADER_CONTROL                                                                         0x0310
+#define regPA_SC_SHADER_CONTROL_BASE_IDX                                                                1
+#define regPA_SC_BINNER_CNTL_0                                                                          0x0311
+#define regPA_SC_BINNER_CNTL_0_BASE_IDX                                                                 1
+#define regPA_SC_BINNER_CNTL_1                                                                          0x0312
+#define regPA_SC_BINNER_CNTL_1_BASE_IDX                                                                 1
+#define regPA_SC_CONSERVATIVE_RASTERIZATION_CNTL                                                        0x0313
+#define regPA_SC_CONSERVATIVE_RASTERIZATION_CNTL_BASE_IDX                                               1
+#define regPA_SC_NGG_MODE_CNTL                                                                          0x0314
+#define regPA_SC_NGG_MODE_CNTL_BASE_IDX                                                                 1
+#define regVGT_VERTEX_REUSE_BLOCK_CNTL                                                                  0x0316
+#define regVGT_VERTEX_REUSE_BLOCK_CNTL_BASE_IDX                                                         1
+#define regVGT_OUT_DEALLOC_CNTL                                                                         0x0317
+#define regVGT_OUT_DEALLOC_CNTL_BASE_IDX                                                                1
+#define regCB_COLOR0_BASE                                                                               0x0318
+#define regCB_COLOR0_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR0_BASE_EXT                                                                           0x0319
+#define regCB_COLOR0_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR0_ATTRIB2                                                                            0x031a
+#define regCB_COLOR0_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR0_VIEW                                                                               0x031b
+#define regCB_COLOR0_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR0_INFO                                                                               0x031c
+#define regCB_COLOR0_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR0_ATTRIB                                                                             0x031d
+#define regCB_COLOR0_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR0_DCC_CONTROL                                                                        0x031e
+#define regCB_COLOR0_DCC_CONTROL_BASE_IDX                                                               1
+#define regCB_COLOR0_CMASK                                                                              0x031f
+#define regCB_COLOR0_CMASK_BASE_IDX                                                                     1
+#define regCB_COLOR0_CMASK_BASE_EXT                                                                     0x0320
+#define regCB_COLOR0_CMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR0_FMASK                                                                              0x0321
+#define regCB_COLOR0_FMASK_BASE_IDX                                                                     1
+#define regCB_COLOR0_FMASK_BASE_EXT                                                                     0x0322
+#define regCB_COLOR0_FMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR0_CLEAR_WORD0                                                                        0x0323
+#define regCB_COLOR0_CLEAR_WORD0_BASE_IDX                                                               1
+#define regCB_COLOR0_CLEAR_WORD1                                                                        0x0324
+#define regCB_COLOR0_CLEAR_WORD1_BASE_IDX                                                               1
+#define regCB_COLOR0_DCC_BASE                                                                           0x0325
+#define regCB_COLOR0_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR0_DCC_BASE_EXT                                                                       0x0326
+#define regCB_COLOR0_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR1_BASE                                                                               0x0327
+#define regCB_COLOR1_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR1_BASE_EXT                                                                           0x0328
+#define regCB_COLOR1_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR1_ATTRIB2                                                                            0x0329
+#define regCB_COLOR1_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR1_VIEW                                                                               0x032a
+#define regCB_COLOR1_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR1_INFO                                                                               0x032b
+#define regCB_COLOR1_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR1_ATTRIB                                                                             0x032c
+#define regCB_COLOR1_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR1_DCC_CONTROL                                                                        0x032d
+#define regCB_COLOR1_DCC_CONTROL_BASE_IDX                                                               1
+#define regCB_COLOR1_CMASK                                                                              0x032e
+#define regCB_COLOR1_CMASK_BASE_IDX                                                                     1
+#define regCB_COLOR1_CMASK_BASE_EXT                                                                     0x032f
+#define regCB_COLOR1_CMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR1_FMASK                                                                              0x0330
+#define regCB_COLOR1_FMASK_BASE_IDX                                                                     1
+#define regCB_COLOR1_FMASK_BASE_EXT                                                                     0x0331
+#define regCB_COLOR1_FMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR1_CLEAR_WORD0                                                                        0x0332
+#define regCB_COLOR1_CLEAR_WORD0_BASE_IDX                                                               1
+#define regCB_COLOR1_CLEAR_WORD1                                                                        0x0333
+#define regCB_COLOR1_CLEAR_WORD1_BASE_IDX                                                               1
+#define regCB_COLOR1_DCC_BASE                                                                           0x0334
+#define regCB_COLOR1_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR1_DCC_BASE_EXT                                                                       0x0335
+#define regCB_COLOR1_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR2_BASE                                                                               0x0336
+#define regCB_COLOR2_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR2_BASE_EXT                                                                           0x0337
+#define regCB_COLOR2_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR2_ATTRIB2                                                                            0x0338
+#define regCB_COLOR2_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR2_VIEW                                                                               0x0339
+#define regCB_COLOR2_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR2_INFO                                                                               0x033a
+#define regCB_COLOR2_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR2_ATTRIB                                                                             0x033b
+#define regCB_COLOR2_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR2_DCC_CONTROL                                                                        0x033c
+#define regCB_COLOR2_DCC_CONTROL_BASE_IDX                                                               1
+#define regCB_COLOR2_CMASK                                                                              0x033d
+#define regCB_COLOR2_CMASK_BASE_IDX                                                                     1
+#define regCB_COLOR2_CMASK_BASE_EXT                                                                     0x033e
+#define regCB_COLOR2_CMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR2_FMASK                                                                              0x033f
+#define regCB_COLOR2_FMASK_BASE_IDX                                                                     1
+#define regCB_COLOR2_FMASK_BASE_EXT                                                                     0x0340
+#define regCB_COLOR2_FMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR2_CLEAR_WORD0                                                                        0x0341
+#define regCB_COLOR2_CLEAR_WORD0_BASE_IDX                                                               1
+#define regCB_COLOR2_CLEAR_WORD1                                                                        0x0342
+#define regCB_COLOR2_CLEAR_WORD1_BASE_IDX                                                               1
+#define regCB_COLOR2_DCC_BASE                                                                           0x0343
+#define regCB_COLOR2_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR2_DCC_BASE_EXT                                                                       0x0344
+#define regCB_COLOR2_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR3_BASE                                                                               0x0345
+#define regCB_COLOR3_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR3_BASE_EXT                                                                           0x0346
+#define regCB_COLOR3_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR3_ATTRIB2                                                                            0x0347
+#define regCB_COLOR3_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR3_VIEW                                                                               0x0348
+#define regCB_COLOR3_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR3_INFO                                                                               0x0349
+#define regCB_COLOR3_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR3_ATTRIB                                                                             0x034a
+#define regCB_COLOR3_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR3_DCC_CONTROL                                                                        0x034b
+#define regCB_COLOR3_DCC_CONTROL_BASE_IDX                                                               1
+#define regCB_COLOR3_CMASK                                                                              0x034c
+#define regCB_COLOR3_CMASK_BASE_IDX                                                                     1
+#define regCB_COLOR3_CMASK_BASE_EXT                                                                     0x034d
+#define regCB_COLOR3_CMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR3_FMASK                                                                              0x034e
+#define regCB_COLOR3_FMASK_BASE_IDX                                                                     1
+#define regCB_COLOR3_FMASK_BASE_EXT                                                                     0x034f
+#define regCB_COLOR3_FMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR3_CLEAR_WORD0                                                                        0x0350
+#define regCB_COLOR3_CLEAR_WORD0_BASE_IDX                                                               1
+#define regCB_COLOR3_CLEAR_WORD1                                                                        0x0351
+#define regCB_COLOR3_CLEAR_WORD1_BASE_IDX                                                               1
+#define regCB_COLOR3_DCC_BASE                                                                           0x0352
+#define regCB_COLOR3_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR3_DCC_BASE_EXT                                                                       0x0353
+#define regCB_COLOR3_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR4_BASE                                                                               0x0354
+#define regCB_COLOR4_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR4_BASE_EXT                                                                           0x0355
+#define regCB_COLOR4_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR4_ATTRIB2                                                                            0x0356
+#define regCB_COLOR4_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR4_VIEW                                                                               0x0357
+#define regCB_COLOR4_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR4_INFO                                                                               0x0358
+#define regCB_COLOR4_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR4_ATTRIB                                                                             0x0359
+#define regCB_COLOR4_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR4_DCC_CONTROL                                                                        0x035a
+#define regCB_COLOR4_DCC_CONTROL_BASE_IDX                                                               1
+#define regCB_COLOR4_CMASK                                                                              0x035b
+#define regCB_COLOR4_CMASK_BASE_IDX                                                                     1
+#define regCB_COLOR4_CMASK_BASE_EXT                                                                     0x035c
+#define regCB_COLOR4_CMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR4_FMASK                                                                              0x035d
+#define regCB_COLOR4_FMASK_BASE_IDX                                                                     1
+#define regCB_COLOR4_FMASK_BASE_EXT                                                                     0x035e
+#define regCB_COLOR4_FMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR4_CLEAR_WORD0                                                                        0x035f
+#define regCB_COLOR4_CLEAR_WORD0_BASE_IDX                                                               1
+#define regCB_COLOR4_CLEAR_WORD1                                                                        0x0360
+#define regCB_COLOR4_CLEAR_WORD1_BASE_IDX                                                               1
+#define regCB_COLOR4_DCC_BASE                                                                           0x0361
+#define regCB_COLOR4_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR4_DCC_BASE_EXT                                                                       0x0362
+#define regCB_COLOR4_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR5_BASE                                                                               0x0363
+#define regCB_COLOR5_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR5_BASE_EXT                                                                           0x0364
+#define regCB_COLOR5_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR5_ATTRIB2                                                                            0x0365
+#define regCB_COLOR5_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR5_VIEW                                                                               0x0366
+#define regCB_COLOR5_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR5_INFO                                                                               0x0367
+#define regCB_COLOR5_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR5_ATTRIB                                                                             0x0368
+#define regCB_COLOR5_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR5_DCC_CONTROL                                                                        0x0369
+#define regCB_COLOR5_DCC_CONTROL_BASE_IDX                                                               1
+#define regCB_COLOR5_CMASK                                                                              0x036a
+#define regCB_COLOR5_CMASK_BASE_IDX                                                                     1
+#define regCB_COLOR5_CMASK_BASE_EXT                                                                     0x036b
+#define regCB_COLOR5_CMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR5_FMASK                                                                              0x036c
+#define regCB_COLOR5_FMASK_BASE_IDX                                                                     1
+#define regCB_COLOR5_FMASK_BASE_EXT                                                                     0x036d
+#define regCB_COLOR5_FMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR5_CLEAR_WORD0                                                                        0x036e
+#define regCB_COLOR5_CLEAR_WORD0_BASE_IDX                                                               1
+#define regCB_COLOR5_CLEAR_WORD1                                                                        0x036f
+#define regCB_COLOR5_CLEAR_WORD1_BASE_IDX                                                               1
+#define regCB_COLOR5_DCC_BASE                                                                           0x0370
+#define regCB_COLOR5_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR5_DCC_BASE_EXT                                                                       0x0371
+#define regCB_COLOR5_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR6_BASE                                                                               0x0372
+#define regCB_COLOR6_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR6_BASE_EXT                                                                           0x0373
+#define regCB_COLOR6_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR6_ATTRIB2                                                                            0x0374
+#define regCB_COLOR6_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR6_VIEW                                                                               0x0375
+#define regCB_COLOR6_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR6_INFO                                                                               0x0376
+#define regCB_COLOR6_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR6_ATTRIB                                                                             0x0377
+#define regCB_COLOR6_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR6_DCC_CONTROL                                                                        0x0378
+#define regCB_COLOR6_DCC_CONTROL_BASE_IDX                                                               1
+#define regCB_COLOR6_CMASK                                                                              0x0379
+#define regCB_COLOR6_CMASK_BASE_IDX                                                                     1
+#define regCB_COLOR6_CMASK_BASE_EXT                                                                     0x037a
+#define regCB_COLOR6_CMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR6_FMASK                                                                              0x037b
+#define regCB_COLOR6_FMASK_BASE_IDX                                                                     1
+#define regCB_COLOR6_FMASK_BASE_EXT                                                                     0x037c
+#define regCB_COLOR6_FMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR6_CLEAR_WORD0                                                                        0x037d
+#define regCB_COLOR6_CLEAR_WORD0_BASE_IDX                                                               1
+#define regCB_COLOR6_CLEAR_WORD1                                                                        0x037e
+#define regCB_COLOR6_CLEAR_WORD1_BASE_IDX                                                               1
+#define regCB_COLOR6_DCC_BASE                                                                           0x037f
+#define regCB_COLOR6_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR6_DCC_BASE_EXT                                                                       0x0380
+#define regCB_COLOR6_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR7_BASE                                                                               0x0381
+#define regCB_COLOR7_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR7_BASE_EXT                                                                           0x0382
+#define regCB_COLOR7_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR7_ATTRIB2                                                                            0x0383
+#define regCB_COLOR7_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR7_VIEW                                                                               0x0384
+#define regCB_COLOR7_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR7_INFO                                                                               0x0385
+#define regCB_COLOR7_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR7_ATTRIB                                                                             0x0386
+#define regCB_COLOR7_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR7_DCC_CONTROL                                                                        0x0387
+#define regCB_COLOR7_DCC_CONTROL_BASE_IDX                                                               1
+#define regCB_COLOR7_CMASK                                                                              0x0388
+#define regCB_COLOR7_CMASK_BASE_IDX                                                                     1
+#define regCB_COLOR7_CMASK_BASE_EXT                                                                     0x0389
+#define regCB_COLOR7_CMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR7_FMASK                                                                              0x038a
+#define regCB_COLOR7_FMASK_BASE_IDX                                                                     1
+#define regCB_COLOR7_FMASK_BASE_EXT                                                                     0x038b
+#define regCB_COLOR7_FMASK_BASE_EXT_BASE_IDX                                                            1
+#define regCB_COLOR7_CLEAR_WORD0                                                                        0x038c
+#define regCB_COLOR7_CLEAR_WORD0_BASE_IDX                                                               1
+#define regCB_COLOR7_CLEAR_WORD1                                                                        0x038d
+#define regCB_COLOR7_CLEAR_WORD1_BASE_IDX                                                               1
+#define regCB_COLOR7_DCC_BASE                                                                           0x038e
+#define regCB_COLOR7_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR7_DCC_BASE_EXT                                                                       0x038f
+#define regCB_COLOR7_DCC_BASE_EXT_BASE_IDX                                                              1
+
+
+// addressBlock: xcd0_gc_gfxudec
+// base address: 0x30000
+#define regCP_EOP_DONE_ADDR_LO                                                                          0x2000
+#define regCP_EOP_DONE_ADDR_LO_BASE_IDX                                                                 1
+#define regCP_EOP_DONE_ADDR_HI                                                                          0x2001
+#define regCP_EOP_DONE_ADDR_HI_BASE_IDX                                                                 1
+#define regCP_EOP_DONE_DATA_LO                                                                          0x2002
+#define regCP_EOP_DONE_DATA_LO_BASE_IDX                                                                 1
+#define regCP_EOP_DONE_DATA_HI                                                                          0x2003
+#define regCP_EOP_DONE_DATA_HI_BASE_IDX                                                                 1
+#define regCP_EOP_LAST_FENCE_LO                                                                         0x2004
+#define regCP_EOP_LAST_FENCE_LO_BASE_IDX                                                                1
+#define regCP_EOP_LAST_FENCE_HI                                                                         0x2005
+#define regCP_EOP_LAST_FENCE_HI_BASE_IDX                                                                1
+#define regCP_STREAM_OUT_ADDR_LO                                                                        0x2006
+#define regCP_STREAM_OUT_ADDR_LO_BASE_IDX                                                               1
+#define regCP_STREAM_OUT_ADDR_HI                                                                        0x2007
+#define regCP_STREAM_OUT_ADDR_HI_BASE_IDX                                                               1
+#define regCP_NUM_PRIM_WRITTEN_COUNT0_LO                                                                0x2008
+#define regCP_NUM_PRIM_WRITTEN_COUNT0_LO_BASE_IDX                                                       1
+#define regCP_NUM_PRIM_WRITTEN_COUNT0_HI                                                                0x2009
+#define regCP_NUM_PRIM_WRITTEN_COUNT0_HI_BASE_IDX                                                       1
+#define regCP_NUM_PRIM_NEEDED_COUNT0_LO                                                                 0x200a
+#define regCP_NUM_PRIM_NEEDED_COUNT0_LO_BASE_IDX                                                        1
+#define regCP_NUM_PRIM_NEEDED_COUNT0_HI                                                                 0x200b
+#define regCP_NUM_PRIM_NEEDED_COUNT0_HI_BASE_IDX                                                        1
+#define regCP_NUM_PRIM_WRITTEN_COUNT1_LO                                                                0x200c
+#define regCP_NUM_PRIM_WRITTEN_COUNT1_LO_BASE_IDX                                                       1
+#define regCP_NUM_PRIM_WRITTEN_COUNT1_HI                                                                0x200d
+#define regCP_NUM_PRIM_WRITTEN_COUNT1_HI_BASE_IDX                                                       1
+#define regCP_NUM_PRIM_NEEDED_COUNT1_LO                                                                 0x200e
+#define regCP_NUM_PRIM_NEEDED_COUNT1_LO_BASE_IDX                                                        1
+#define regCP_NUM_PRIM_NEEDED_COUNT1_HI                                                                 0x200f
+#define regCP_NUM_PRIM_NEEDED_COUNT1_HI_BASE_IDX                                                        1
+#define regCP_NUM_PRIM_WRITTEN_COUNT2_LO                                                                0x2010
+#define regCP_NUM_PRIM_WRITTEN_COUNT2_LO_BASE_IDX                                                       1
+#define regCP_NUM_PRIM_WRITTEN_COUNT2_HI                                                                0x2011
+#define regCP_NUM_PRIM_WRITTEN_COUNT2_HI_BASE_IDX                                                       1
+#define regCP_NUM_PRIM_NEEDED_COUNT2_LO                                                                 0x2012
+#define regCP_NUM_PRIM_NEEDED_COUNT2_LO_BASE_IDX                                                        1
+#define regCP_NUM_PRIM_NEEDED_COUNT2_HI                                                                 0x2013
+#define regCP_NUM_PRIM_NEEDED_COUNT2_HI_BASE_IDX                                                        1
+#define regCP_NUM_PRIM_WRITTEN_COUNT3_LO                                                                0x2014
+#define regCP_NUM_PRIM_WRITTEN_COUNT3_LO_BASE_IDX                                                       1
+#define regCP_NUM_PRIM_WRITTEN_COUNT3_HI                                                                0x2015
+#define regCP_NUM_PRIM_WRITTEN_COUNT3_HI_BASE_IDX                                                       1
+#define regCP_NUM_PRIM_NEEDED_COUNT3_LO                                                                 0x2016
+#define regCP_NUM_PRIM_NEEDED_COUNT3_LO_BASE_IDX                                                        1
+#define regCP_NUM_PRIM_NEEDED_COUNT3_HI                                                                 0x2017
+#define regCP_NUM_PRIM_NEEDED_COUNT3_HI_BASE_IDX                                                        1
+#define regCP_PIPE_STATS_ADDR_LO                                                                        0x2018
+#define regCP_PIPE_STATS_ADDR_LO_BASE_IDX                                                               1
+#define regCP_PIPE_STATS_ADDR_HI                                                                        0x2019
+#define regCP_PIPE_STATS_ADDR_HI_BASE_IDX                                                               1
+#define regCP_VGT_IAVERT_COUNT_LO                                                                       0x201a
+#define regCP_VGT_IAVERT_COUNT_LO_BASE_IDX                                                              1
+#define regCP_VGT_IAVERT_COUNT_HI                                                                       0x201b
+#define regCP_VGT_IAVERT_COUNT_HI_BASE_IDX                                                              1
+#define regCP_VGT_IAPRIM_COUNT_LO                                                                       0x201c
+#define regCP_VGT_IAPRIM_COUNT_LO_BASE_IDX                                                              1
+#define regCP_VGT_IAPRIM_COUNT_HI                                                                       0x201d
+#define regCP_VGT_IAPRIM_COUNT_HI_BASE_IDX                                                              1
+#define regCP_VGT_GSPRIM_COUNT_LO                                                                       0x201e
+#define regCP_VGT_GSPRIM_COUNT_LO_BASE_IDX                                                              1
+#define regCP_VGT_GSPRIM_COUNT_HI                                                                       0x201f
+#define regCP_VGT_GSPRIM_COUNT_HI_BASE_IDX                                                              1
+#define regCP_VGT_VSINVOC_COUNT_LO                                                                      0x2020
+#define regCP_VGT_VSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_VSINVOC_COUNT_HI                                                                      0x2021
+#define regCP_VGT_VSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_GSINVOC_COUNT_LO                                                                      0x2022
+#define regCP_VGT_GSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_GSINVOC_COUNT_HI                                                                      0x2023
+#define regCP_VGT_GSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_HSINVOC_COUNT_LO                                                                      0x2024
+#define regCP_VGT_HSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_HSINVOC_COUNT_HI                                                                      0x2025
+#define regCP_VGT_HSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_DSINVOC_COUNT_LO                                                                      0x2026
+#define regCP_VGT_DSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_DSINVOC_COUNT_HI                                                                      0x2027
+#define regCP_VGT_DSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_PA_CINVOC_COUNT_LO                                                                        0x2028
+#define regCP_PA_CINVOC_COUNT_LO_BASE_IDX                                                               1
+#define regCP_PA_CINVOC_COUNT_HI                                                                        0x2029
+#define regCP_PA_CINVOC_COUNT_HI_BASE_IDX                                                               1
+#define regCP_PA_CPRIM_COUNT_LO                                                                         0x202a
+#define regCP_PA_CPRIM_COUNT_LO_BASE_IDX                                                                1
+#define regCP_PA_CPRIM_COUNT_HI                                                                         0x202b
+#define regCP_PA_CPRIM_COUNT_HI_BASE_IDX                                                                1
+#define regCP_SC_PSINVOC_COUNT0_LO                                                                      0x202c
+#define regCP_SC_PSINVOC_COUNT0_LO_BASE_IDX                                                             1
+#define regCP_SC_PSINVOC_COUNT0_HI                                                                      0x202d
+#define regCP_SC_PSINVOC_COUNT0_HI_BASE_IDX                                                             1
+#define regCP_SC_PSINVOC_COUNT1_LO                                                                      0x202e
+#define regCP_SC_PSINVOC_COUNT1_LO_BASE_IDX                                                             1
+#define regCP_SC_PSINVOC_COUNT1_HI                                                                      0x202f
+#define regCP_SC_PSINVOC_COUNT1_HI_BASE_IDX                                                             1
+#define regCP_VGT_CSINVOC_COUNT_LO                                                                      0x2030
+#define regCP_VGT_CSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_CSINVOC_COUNT_HI                                                                      0x2031
+#define regCP_VGT_CSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_PIPE_STATS_CONTROL                                                                        0x203d
+#define regCP_PIPE_STATS_CONTROL_BASE_IDX                                                               1
+#define regCP_STREAM_OUT_CONTROL                                                                        0x203e
+#define regCP_STREAM_OUT_CONTROL_BASE_IDX                                                               1
+#define regCP_STRMOUT_CNTL                                                                              0x203f
+#define regCP_STRMOUT_CNTL_BASE_IDX                                                                     1
+#define regSCRATCH_REG0                                                                                 0x2040
+#define regSCRATCH_REG0_BASE_IDX                                                                        1
+#define regSCRATCH_REG1                                                                                 0x2041
+#define regSCRATCH_REG1_BASE_IDX                                                                        1
+#define regSCRATCH_REG2                                                                                 0x2042
+#define regSCRATCH_REG2_BASE_IDX                                                                        1
+#define regSCRATCH_REG3                                                                                 0x2043
+#define regSCRATCH_REG3_BASE_IDX                                                                        1
+#define regSCRATCH_REG4                                                                                 0x2044
+#define regSCRATCH_REG4_BASE_IDX                                                                        1
+#define regSCRATCH_REG5                                                                                 0x2045
+#define regSCRATCH_REG5_BASE_IDX                                                                        1
+#define regSCRATCH_REG6                                                                                 0x2046
+#define regSCRATCH_REG6_BASE_IDX                                                                        1
+#define regSCRATCH_REG7                                                                                 0x2047
+#define regSCRATCH_REG7_BASE_IDX                                                                        1
+#define regCP_APPEND_DATA_HI                                                                            0x204c
+#define regCP_APPEND_DATA_HI_BASE_IDX                                                                   1
+#define regCP_APPEND_LAST_CS_FENCE_HI                                                                   0x204d
+#define regCP_APPEND_LAST_CS_FENCE_HI_BASE_IDX                                                          1
+#define regCP_APPEND_LAST_PS_FENCE_HI                                                                   0x204e
+#define regCP_APPEND_LAST_PS_FENCE_HI_BASE_IDX                                                          1
+#define regSCRATCH_UMSK                                                                                 0x2050
+#define regSCRATCH_UMSK_BASE_IDX                                                                        1
+#define regSCRATCH_ADDR                                                                                 0x2051
+#define regSCRATCH_ADDR_BASE_IDX                                                                        1
+#define regCP_PFP_ATOMIC_PREOP_LO                                                                       0x2052
+#define regCP_PFP_ATOMIC_PREOP_LO_BASE_IDX                                                              1
+#define regCP_PFP_ATOMIC_PREOP_HI                                                                       0x2053
+#define regCP_PFP_ATOMIC_PREOP_HI_BASE_IDX                                                              1
+#define regCP_PFP_GDS_ATOMIC0_PREOP_LO                                                                  0x2054
+#define regCP_PFP_GDS_ATOMIC0_PREOP_LO_BASE_IDX                                                         1
+#define regCP_PFP_GDS_ATOMIC0_PREOP_HI                                                                  0x2055
+#define regCP_PFP_GDS_ATOMIC0_PREOP_HI_BASE_IDX                                                         1
+#define regCP_PFP_GDS_ATOMIC1_PREOP_LO                                                                  0x2056
+#define regCP_PFP_GDS_ATOMIC1_PREOP_LO_BASE_IDX                                                         1
+#define regCP_PFP_GDS_ATOMIC1_PREOP_HI                                                                  0x2057
+#define regCP_PFP_GDS_ATOMIC1_PREOP_HI_BASE_IDX                                                         1
+#define regCP_APPEND_ADDR_LO                                                                            0x2058
+#define regCP_APPEND_ADDR_LO_BASE_IDX                                                                   1
+#define regCP_APPEND_ADDR_HI                                                                            0x2059
+#define regCP_APPEND_ADDR_HI_BASE_IDX                                                                   1
+#define regCP_APPEND_DATA_LO                                                                            0x205a
+#define regCP_APPEND_DATA_LO_BASE_IDX                                                                   1
+#define regCP_APPEND_LAST_CS_FENCE_LO                                                                   0x205b
+#define regCP_APPEND_LAST_CS_FENCE_LO_BASE_IDX                                                          1
+#define regCP_APPEND_LAST_PS_FENCE_LO                                                                   0x205c
+#define regCP_APPEND_LAST_PS_FENCE_LO_BASE_IDX                                                          1
+#define regCP_ATOMIC_PREOP_LO                                                                           0x205d
+#define regCP_ATOMIC_PREOP_LO_BASE_IDX                                                                  1
+#define regCP_ME_ATOMIC_PREOP_LO                                                                        0x205d
+#define regCP_ME_ATOMIC_PREOP_LO_BASE_IDX                                                               1
+#define regCP_ATOMIC_PREOP_HI                                                                           0x205e
+#define regCP_ATOMIC_PREOP_HI_BASE_IDX                                                                  1
+#define regCP_ME_ATOMIC_PREOP_HI                                                                        0x205e
+#define regCP_ME_ATOMIC_PREOP_HI_BASE_IDX                                                               1
+#define regCP_GDS_ATOMIC0_PREOP_LO                                                                      0x205f
+#define regCP_GDS_ATOMIC0_PREOP_LO_BASE_IDX                                                             1
+#define regCP_ME_GDS_ATOMIC0_PREOP_LO                                                                   0x205f
+#define regCP_ME_GDS_ATOMIC0_PREOP_LO_BASE_IDX                                                          1
+#define regCP_GDS_ATOMIC0_PREOP_HI                                                                      0x2060
+#define regCP_GDS_ATOMIC0_PREOP_HI_BASE_IDX                                                             1
+#define regCP_ME_GDS_ATOMIC0_PREOP_HI                                                                   0x2060
+#define regCP_ME_GDS_ATOMIC0_PREOP_HI_BASE_IDX                                                          1
+#define regCP_GDS_ATOMIC1_PREOP_LO                                                                      0x2061
+#define regCP_GDS_ATOMIC1_PREOP_LO_BASE_IDX                                                             1
+#define regCP_ME_GDS_ATOMIC1_PREOP_LO                                                                   0x2061
+#define regCP_ME_GDS_ATOMIC1_PREOP_LO_BASE_IDX                                                          1
+#define regCP_GDS_ATOMIC1_PREOP_HI                                                                      0x2062
+#define regCP_GDS_ATOMIC1_PREOP_HI_BASE_IDX                                                             1
+#define regCP_ME_GDS_ATOMIC1_PREOP_HI                                                                   0x2062
+#define regCP_ME_GDS_ATOMIC1_PREOP_HI_BASE_IDX                                                          1
+#define regCP_ME_MC_WADDR_LO                                                                            0x2069
+#define regCP_ME_MC_WADDR_LO_BASE_IDX                                                                   1
+#define regCP_ME_MC_WADDR_HI                                                                            0x206a
+#define regCP_ME_MC_WADDR_HI_BASE_IDX                                                                   1
+#define regCP_ME_MC_WDATA_LO                                                                            0x206b
+#define regCP_ME_MC_WDATA_LO_BASE_IDX                                                                   1
+#define regCP_ME_MC_WDATA_HI                                                                            0x206c
+#define regCP_ME_MC_WDATA_HI_BASE_IDX                                                                   1
+#define regCP_ME_MC_RADDR_LO                                                                            0x206d
+#define regCP_ME_MC_RADDR_LO_BASE_IDX                                                                   1
+#define regCP_ME_MC_RADDR_HI                                                                            0x206e
+#define regCP_ME_MC_RADDR_HI_BASE_IDX                                                                   1
+#define regCP_SEM_WAIT_TIMER                                                                            0x206f
+#define regCP_SEM_WAIT_TIMER_BASE_IDX                                                                   1
+#define regCP_SIG_SEM_ADDR_LO                                                                           0x2070
+#define regCP_SIG_SEM_ADDR_LO_BASE_IDX                                                                  1
+#define regCP_SIG_SEM_ADDR_HI                                                                           0x2071
+#define regCP_SIG_SEM_ADDR_HI_BASE_IDX                                                                  1
+#define regCP_WAIT_REG_MEM_TIMEOUT                                                                      0x2074
+#define regCP_WAIT_REG_MEM_TIMEOUT_BASE_IDX                                                             1
+#define regCP_WAIT_SEM_ADDR_LO                                                                          0x2075
+#define regCP_WAIT_SEM_ADDR_LO_BASE_IDX                                                                 1
+#define regCP_WAIT_SEM_ADDR_HI                                                                          0x2076
+#define regCP_WAIT_SEM_ADDR_HI_BASE_IDX                                                                 1
+#define regCP_DMA_PFP_CONTROL                                                                           0x2077
+#define regCP_DMA_PFP_CONTROL_BASE_IDX                                                                  1
+#define regCP_DMA_ME_CONTROL                                                                            0x2078
+#define regCP_DMA_ME_CONTROL_BASE_IDX                                                                   1
+#define regCP_COHER_BASE_HI                                                                             0x2079
+#define regCP_COHER_BASE_HI_BASE_IDX                                                                    1
+#define regCP_COHER_START_DELAY                                                                         0x207b
+#define regCP_COHER_START_DELAY_BASE_IDX                                                                1
+#define regCP_COHER_CNTL                                                                                0x207c
+#define regCP_COHER_CNTL_BASE_IDX                                                                       1
+#define regCP_COHER_SIZE                                                                                0x207d
+#define regCP_COHER_SIZE_BASE_IDX                                                                       1
+#define regCP_COHER_BASE                                                                                0x207e
+#define regCP_COHER_BASE_BASE_IDX                                                                       1
+#define regCP_COHER_STATUS                                                                              0x207f
+#define regCP_COHER_STATUS_BASE_IDX                                                                     1
+#define regCP_DMA_ME_SRC_ADDR                                                                           0x2080
+#define regCP_DMA_ME_SRC_ADDR_BASE_IDX                                                                  1
+#define regCP_DMA_ME_SRC_ADDR_HI                                                                        0x2081
+#define regCP_DMA_ME_SRC_ADDR_HI_BASE_IDX                                                               1
+#define regCP_DMA_ME_DST_ADDR                                                                           0x2082
+#define regCP_DMA_ME_DST_ADDR_BASE_IDX                                                                  1
+#define regCP_DMA_ME_DST_ADDR_HI                                                                        0x2083
+#define regCP_DMA_ME_DST_ADDR_HI_BASE_IDX                                                               1
+#define regCP_DMA_ME_COMMAND                                                                            0x2084
+#define regCP_DMA_ME_COMMAND_BASE_IDX                                                                   1
+#define regCP_DMA_PFP_SRC_ADDR                                                                          0x2085
+#define regCP_DMA_PFP_SRC_ADDR_BASE_IDX                                                                 1
+#define regCP_DMA_PFP_SRC_ADDR_HI                                                                       0x2086
+#define regCP_DMA_PFP_SRC_ADDR_HI_BASE_IDX                                                              1
+#define regCP_DMA_PFP_DST_ADDR                                                                          0x2087
+#define regCP_DMA_PFP_DST_ADDR_BASE_IDX                                                                 1
+#define regCP_DMA_PFP_DST_ADDR_HI                                                                       0x2088
+#define regCP_DMA_PFP_DST_ADDR_HI_BASE_IDX                                                              1
+#define regCP_DMA_PFP_COMMAND                                                                           0x2089
+#define regCP_DMA_PFP_COMMAND_BASE_IDX                                                                  1
+#define regCP_DMA_CNTL                                                                                  0x208a
+#define regCP_DMA_CNTL_BASE_IDX                                                                         1
+#define regCP_DMA_READ_TAGS                                                                             0x208b
+#define regCP_DMA_READ_TAGS_BASE_IDX                                                                    1
+#define regCP_COHER_SIZE_HI                                                                             0x208c
+#define regCP_COHER_SIZE_HI_BASE_IDX                                                                    1
+#define regCP_PFP_IB_CONTROL                                                                            0x208d
+#define regCP_PFP_IB_CONTROL_BASE_IDX                                                                   1
+#define regCP_PFP_LOAD_CONTROL                                                                          0x208e
+#define regCP_PFP_LOAD_CONTROL_BASE_IDX                                                                 1
+#define regCP_SCRATCH_INDEX                                                                             0x208f
+#define regCP_SCRATCH_INDEX_BASE_IDX                                                                    1
+#define regCP_SCRATCH_DATA                                                                              0x2090
+#define regCP_SCRATCH_DATA_BASE_IDX                                                                     1
+#define regCP_RB_OFFSET                                                                                 0x2091
+#define regCP_RB_OFFSET_BASE_IDX                                                                        1
+#define regCP_IB1_OFFSET                                                                                0x2092
+#define regCP_IB1_OFFSET_BASE_IDX                                                                       1
+#define regCP_IB2_OFFSET                                                                                0x2093
+#define regCP_IB2_OFFSET_BASE_IDX                                                                       1
+#define regCP_IB1_PREAMBLE_BEGIN                                                                        0x2094
+#define regCP_IB1_PREAMBLE_BEGIN_BASE_IDX                                                               1
+#define regCP_IB1_PREAMBLE_END                                                                          0x2095
+#define regCP_IB1_PREAMBLE_END_BASE_IDX                                                                 1
+#define regCP_IB2_PREAMBLE_BEGIN                                                                        0x2096
+#define regCP_IB2_PREAMBLE_BEGIN_BASE_IDX                                                               1
+#define regCP_IB2_PREAMBLE_END                                                                          0x2097
+#define regCP_IB2_PREAMBLE_END_BASE_IDX                                                                 1
+#define regCP_CE_IB1_OFFSET                                                                             0x2098
+#define regCP_CE_IB1_OFFSET_BASE_IDX                                                                    1
+#define regCP_CE_IB2_OFFSET                                                                             0x2099
+#define regCP_CE_IB2_OFFSET_BASE_IDX                                                                    1
+#define regCP_CE_COUNTER                                                                                0x209a
+#define regCP_CE_COUNTER_BASE_IDX                                                                       1
+#define regCP_CE_RB_OFFSET                                                                              0x209b
+#define regCP_CE_RB_OFFSET_BASE_IDX                                                                     1
+#define regCP_CE_INIT_CMD_BUFSZ                                                                         0x20bd
+#define regCP_CE_INIT_CMD_BUFSZ_BASE_IDX                                                                1
+#define regCP_CE_IB1_CMD_BUFSZ                                                                          0x20be
+#define regCP_CE_IB1_CMD_BUFSZ_BASE_IDX                                                                 1
+#define regCP_CE_IB2_CMD_BUFSZ                                                                          0x20bf
+#define regCP_CE_IB2_CMD_BUFSZ_BASE_IDX                                                                 1
+#define regCP_IB1_CMD_BUFSZ                                                                             0x20c0
+#define regCP_IB1_CMD_BUFSZ_BASE_IDX                                                                    1
+#define regCP_IB2_CMD_BUFSZ                                                                             0x20c1
+#define regCP_IB2_CMD_BUFSZ_BASE_IDX                                                                    1
+#define regCP_ST_CMD_BUFSZ                                                                              0x20c2
+#define regCP_ST_CMD_BUFSZ_BASE_IDX                                                                     1
+#define regCP_CE_INIT_BASE_LO                                                                           0x20c3
+#define regCP_CE_INIT_BASE_LO_BASE_IDX                                                                  1
+#define regCP_CE_INIT_BASE_HI                                                                           0x20c4
+#define regCP_CE_INIT_BASE_HI_BASE_IDX                                                                  1
+#define regCP_CE_INIT_BUFSZ                                                                             0x20c5
+#define regCP_CE_INIT_BUFSZ_BASE_IDX                                                                    1
+#define regCP_CE_IB1_BASE_LO                                                                            0x20c6
+#define regCP_CE_IB1_BASE_LO_BASE_IDX                                                                   1
+#define regCP_CE_IB1_BASE_HI                                                                            0x20c7
+#define regCP_CE_IB1_BASE_HI_BASE_IDX                                                                   1
+#define regCP_CE_IB1_BUFSZ                                                                              0x20c8
+#define regCP_CE_IB1_BUFSZ_BASE_IDX                                                                     1
+#define regCP_CE_IB2_BASE_LO                                                                            0x20c9
+#define regCP_CE_IB2_BASE_LO_BASE_IDX                                                                   1
+#define regCP_CE_IB2_BASE_HI                                                                            0x20ca
+#define regCP_CE_IB2_BASE_HI_BASE_IDX                                                                   1
+#define regCP_CE_IB2_BUFSZ                                                                              0x20cb
+#define regCP_CE_IB2_BUFSZ_BASE_IDX                                                                     1
+#define regCP_IB1_BASE_LO                                                                               0x20cc
+#define regCP_IB1_BASE_LO_BASE_IDX                                                                      1
+#define regCP_IB1_BASE_HI                                                                               0x20cd
+#define regCP_IB1_BASE_HI_BASE_IDX                                                                      1
+#define regCP_IB1_BUFSZ                                                                                 0x20ce
+#define regCP_IB1_BUFSZ_BASE_IDX                                                                        1
+#define regCP_IB2_BASE_LO                                                                               0x20cf
+#define regCP_IB2_BASE_LO_BASE_IDX                                                                      1
+#define regCP_IB2_BASE_HI                                                                               0x20d0
+#define regCP_IB2_BASE_HI_BASE_IDX                                                                      1
+#define regCP_IB2_BUFSZ                                                                                 0x20d1
+#define regCP_IB2_BUFSZ_BASE_IDX                                                                        1
+#define regCP_ST_BASE_LO                                                                                0x20d2
+#define regCP_ST_BASE_LO_BASE_IDX                                                                       1
+#define regCP_ST_BASE_HI                                                                                0x20d3
+#define regCP_ST_BASE_HI_BASE_IDX                                                                       1
+#define regCP_ST_BUFSZ                                                                                  0x20d4
+#define regCP_ST_BUFSZ_BASE_IDX                                                                         1
+#define regCP_EOP_DONE_EVENT_CNTL                                                                       0x20d5
+#define regCP_EOP_DONE_EVENT_CNTL_BASE_IDX                                                              1
+#define regCP_EOP_DONE_DATA_CNTL                                                                        0x20d6
+#define regCP_EOP_DONE_DATA_CNTL_BASE_IDX                                                               1
+#define regCP_EOP_DONE_CNTX_ID                                                                          0x20d7
+#define regCP_EOP_DONE_CNTX_ID_BASE_IDX                                                                 1
+#define regCP_PFP_COMPLETION_STATUS                                                                     0x20ec
+#define regCP_PFP_COMPLETION_STATUS_BASE_IDX                                                            1
+#define regCP_CE_COMPLETION_STATUS                                                                      0x20ed
+#define regCP_CE_COMPLETION_STATUS_BASE_IDX                                                             1
+#define regCP_PRED_NOT_VISIBLE                                                                          0x20ee
+#define regCP_PRED_NOT_VISIBLE_BASE_IDX                                                                 1
+#define regCP_PFP_METADATA_BASE_ADDR                                                                    0x20f0
+#define regCP_PFP_METADATA_BASE_ADDR_BASE_IDX                                                           1
+#define regCP_PFP_METADATA_BASE_ADDR_HI                                                                 0x20f1
+#define regCP_PFP_METADATA_BASE_ADDR_HI_BASE_IDX                                                        1
+#define regCP_CE_METADATA_BASE_ADDR                                                                     0x20f2
+#define regCP_CE_METADATA_BASE_ADDR_BASE_IDX                                                            1
+#define regCP_CE_METADATA_BASE_ADDR_HI                                                                  0x20f3
+#define regCP_CE_METADATA_BASE_ADDR_HI_BASE_IDX                                                         1
+#define regCP_DRAW_INDX_INDR_ADDR                                                                       0x20f4
+#define regCP_DRAW_INDX_INDR_ADDR_BASE_IDX                                                              1
+#define regCP_DRAW_INDX_INDR_ADDR_HI                                                                    0x20f5
+#define regCP_DRAW_INDX_INDR_ADDR_HI_BASE_IDX                                                           1
+#define regCP_DISPATCH_INDR_ADDR                                                                        0x20f6
+#define regCP_DISPATCH_INDR_ADDR_BASE_IDX                                                               1
+#define regCP_DISPATCH_INDR_ADDR_HI                                                                     0x20f7
+#define regCP_DISPATCH_INDR_ADDR_HI_BASE_IDX                                                            1
+#define regCP_INDEX_BASE_ADDR                                                                           0x20f8
+#define regCP_INDEX_BASE_ADDR_BASE_IDX                                                                  1
+#define regCP_INDEX_BASE_ADDR_HI                                                                        0x20f9
+#define regCP_INDEX_BASE_ADDR_HI_BASE_IDX                                                               1
+#define regCP_INDEX_TYPE                                                                                0x20fa
+#define regCP_INDEX_TYPE_BASE_IDX                                                                       1
+#define regCP_GDS_BKUP_ADDR                                                                             0x20fb
+#define regCP_GDS_BKUP_ADDR_BASE_IDX                                                                    1
+#define regCP_GDS_BKUP_ADDR_HI                                                                          0x20fc
+#define regCP_GDS_BKUP_ADDR_HI_BASE_IDX                                                                 1
+#define regCP_SAMPLE_STATUS                                                                             0x20fd
+#define regCP_SAMPLE_STATUS_BASE_IDX                                                                    1
+#define regCP_ME_COHER_CNTL                                                                             0x20fe
+#define regCP_ME_COHER_CNTL_BASE_IDX                                                                    1
+#define regCP_ME_COHER_SIZE                                                                             0x20ff
+#define regCP_ME_COHER_SIZE_BASE_IDX                                                                    1
+#define regCP_ME_COHER_SIZE_HI                                                                          0x2100
+#define regCP_ME_COHER_SIZE_HI_BASE_IDX                                                                 1
+#define regCP_ME_COHER_BASE                                                                             0x2101
+#define regCP_ME_COHER_BASE_BASE_IDX                                                                    1
+#define regCP_ME_COHER_BASE_HI                                                                          0x2102
+#define regCP_ME_COHER_BASE_HI_BASE_IDX                                                                 1
+#define regCP_ME_COHER_STATUS                                                                           0x2103
+#define regCP_ME_COHER_STATUS_BASE_IDX                                                                  1
+#define regRLC_GPM_PERF_COUNT_0                                                                         0x2140
+#define regRLC_GPM_PERF_COUNT_0_BASE_IDX                                                                1
+#define regRLC_GPM_PERF_COUNT_1                                                                         0x2141
+#define regRLC_GPM_PERF_COUNT_1_BASE_IDX                                                                1
+#define regGRBM_GFX_INDEX                                                                               0x2200
+#define regGRBM_GFX_INDEX_BASE_IDX                                                                      1
+#define regVGT_GSVS_RING_SIZE                                                                           0x2241
+#define regVGT_GSVS_RING_SIZE_BASE_IDX                                                                  1
+#define regVGT_PRIMITIVE_TYPE                                                                           0x2242
+#define regVGT_PRIMITIVE_TYPE_BASE_IDX                                                                  1
+#define regVGT_INDEX_TYPE                                                                               0x2243
+#define regVGT_INDEX_TYPE_BASE_IDX                                                                      1
+#define regVGT_STRMOUT_BUFFER_FILLED_SIZE_0                                                             0x2244
+#define regVGT_STRMOUT_BUFFER_FILLED_SIZE_0_BASE_IDX                                                    1
+#define regVGT_STRMOUT_BUFFER_FILLED_SIZE_1                                                             0x2245
+#define regVGT_STRMOUT_BUFFER_FILLED_SIZE_1_BASE_IDX                                                    1
+#define regVGT_STRMOUT_BUFFER_FILLED_SIZE_2                                                             0x2246
+#define regVGT_STRMOUT_BUFFER_FILLED_SIZE_2_BASE_IDX                                                    1
+#define regVGT_STRMOUT_BUFFER_FILLED_SIZE_3                                                             0x2247
+#define regVGT_STRMOUT_BUFFER_FILLED_SIZE_3_BASE_IDX                                                    1
+#define regVGT_MAX_VTX_INDX                                                                             0x2248
+#define regVGT_MAX_VTX_INDX_BASE_IDX                                                                    1
+#define regVGT_MIN_VTX_INDX                                                                             0x2249
+#define regVGT_MIN_VTX_INDX_BASE_IDX                                                                    1
+#define regVGT_INDX_OFFSET                                                                              0x224a
+#define regVGT_INDX_OFFSET_BASE_IDX                                                                     1
+#define regVGT_MULTI_PRIM_IB_RESET_EN                                                                   0x224b
+#define regVGT_MULTI_PRIM_IB_RESET_EN_BASE_IDX                                                          1
+#define regVGT_NUM_INDICES                                                                              0x224c
+#define regVGT_NUM_INDICES_BASE_IDX                                                                     1
+#define regVGT_NUM_INSTANCES                                                                            0x224d
+#define regVGT_NUM_INSTANCES_BASE_IDX                                                                   1
+#define regVGT_TF_RING_SIZE                                                                             0x224e
+#define regVGT_TF_RING_SIZE_BASE_IDX                                                                    1
+#define regVGT_HS_OFFCHIP_PARAM                                                                         0x224f
+#define regVGT_HS_OFFCHIP_PARAM_BASE_IDX                                                                1
+#define regVGT_TF_MEMORY_BASE                                                                           0x2250
+#define regVGT_TF_MEMORY_BASE_BASE_IDX                                                                  1
+#define regVGT_TF_MEMORY_BASE_HI                                                                        0x2251
+#define regVGT_TF_MEMORY_BASE_HI_BASE_IDX                                                               1
+#define regWD_POS_BUF_BASE                                                                              0x2252
+#define regWD_POS_BUF_BASE_BASE_IDX                                                                     1
+#define regWD_POS_BUF_BASE_HI                                                                           0x2253
+#define regWD_POS_BUF_BASE_HI_BASE_IDX                                                                  1
+#define regWD_CNTL_SB_BUF_BASE                                                                          0x2254
+#define regWD_CNTL_SB_BUF_BASE_BASE_IDX                                                                 1
+#define regWD_CNTL_SB_BUF_BASE_HI                                                                       0x2255
+#define regWD_CNTL_SB_BUF_BASE_HI_BASE_IDX                                                              1
+#define regWD_INDEX_BUF_BASE                                                                            0x2256
+#define regWD_INDEX_BUF_BASE_BASE_IDX                                                                   1
+#define regWD_INDEX_BUF_BASE_HI                                                                         0x2257
+#define regWD_INDEX_BUF_BASE_HI_BASE_IDX                                                                1
+#define regIA_MULTI_VGT_PARAM                                                                           0x2258
+#define regIA_MULTI_VGT_PARAM_BASE_IDX                                                                  1
+#define regVGT_INSTANCE_BASE_ID                                                                         0x225a
+#define regVGT_INSTANCE_BASE_ID_BASE_IDX                                                                1
+#define regPA_SU_LINE_STIPPLE_VALUE                                                                     0x2280
+#define regPA_SU_LINE_STIPPLE_VALUE_BASE_IDX                                                            1
+#define regPA_SC_LINE_STIPPLE_STATE                                                                     0x2281
+#define regPA_SC_LINE_STIPPLE_STATE_BASE_IDX                                                            1
+#define regPA_SC_SCREEN_EXTENT_MIN_0                                                                    0x2284
+#define regPA_SC_SCREEN_EXTENT_MIN_0_BASE_IDX                                                           1
+#define regPA_SC_SCREEN_EXTENT_MAX_0                                                                    0x2285
+#define regPA_SC_SCREEN_EXTENT_MAX_0_BASE_IDX                                                           1
+#define regPA_SC_SCREEN_EXTENT_MIN_1                                                                    0x2286
+#define regPA_SC_SCREEN_EXTENT_MIN_1_BASE_IDX                                                           1
+#define regPA_SC_SCREEN_EXTENT_MAX_1                                                                    0x228b
+#define regPA_SC_SCREEN_EXTENT_MAX_1_BASE_IDX                                                           1
+#define regPA_SC_P3D_TRAP_SCREEN_HV_EN                                                                  0x22a0
+#define regPA_SC_P3D_TRAP_SCREEN_HV_EN_BASE_IDX                                                         1
+#define regPA_SC_P3D_TRAP_SCREEN_H                                                                      0x22a1
+#define regPA_SC_P3D_TRAP_SCREEN_H_BASE_IDX                                                             1
+#define regPA_SC_P3D_TRAP_SCREEN_V                                                                      0x22a2
+#define regPA_SC_P3D_TRAP_SCREEN_V_BASE_IDX                                                             1
+#define regPA_SC_P3D_TRAP_SCREEN_OCCURRENCE                                                             0x22a3
+#define regPA_SC_P3D_TRAP_SCREEN_OCCURRENCE_BASE_IDX                                                    1
+#define regPA_SC_P3D_TRAP_SCREEN_COUNT                                                                  0x22a4
+#define regPA_SC_P3D_TRAP_SCREEN_COUNT_BASE_IDX                                                         1
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_EN                                                                 0x22a8
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_EN_BASE_IDX                                                        1
+#define regPA_SC_HP3D_TRAP_SCREEN_H                                                                     0x22a9
+#define regPA_SC_HP3D_TRAP_SCREEN_H_BASE_IDX                                                            1
+#define regPA_SC_HP3D_TRAP_SCREEN_V                                                                     0x22aa
+#define regPA_SC_HP3D_TRAP_SCREEN_V_BASE_IDX                                                            1
+#define regPA_SC_HP3D_TRAP_SCREEN_OCCURRENCE                                                            0x22ab
+#define regPA_SC_HP3D_TRAP_SCREEN_OCCURRENCE_BASE_IDX                                                   1
+#define regPA_SC_HP3D_TRAP_SCREEN_COUNT                                                                 0x22ac
+#define regPA_SC_HP3D_TRAP_SCREEN_COUNT_BASE_IDX                                                        1
+#define regPA_SC_TRAP_SCREEN_HV_EN                                                                      0x22b0
+#define regPA_SC_TRAP_SCREEN_HV_EN_BASE_IDX                                                             1
+#define regPA_SC_TRAP_SCREEN_H                                                                          0x22b1
+#define regPA_SC_TRAP_SCREEN_H_BASE_IDX                                                                 1
+#define regPA_SC_TRAP_SCREEN_V                                                                          0x22b2
+#define regPA_SC_TRAP_SCREEN_V_BASE_IDX                                                                 1
+#define regPA_SC_TRAP_SCREEN_OCCURRENCE                                                                 0x22b3
+#define regPA_SC_TRAP_SCREEN_OCCURRENCE_BASE_IDX                                                        1
+#define regPA_SC_TRAP_SCREEN_COUNT                                                                      0x22b4
+#define regPA_SC_TRAP_SCREEN_COUNT_BASE_IDX                                                             1
+#define regPA_STATE_STEREO_X                                                                            0x22b5
+#define regPA_STATE_STEREO_X_BASE_IDX                                                                   1
+#define regSQ_THREAD_TRACE_BASE                                                                         0x2330
+#define regSQ_THREAD_TRACE_BASE_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_SIZE                                                                         0x2331
+#define regSQ_THREAD_TRACE_SIZE_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_MASK                                                                         0x2332
+#define regSQ_THREAD_TRACE_MASK_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_TOKEN_MASK                                                                   0x2333
+#define regSQ_THREAD_TRACE_TOKEN_MASK_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_PERF_MASK                                                                    0x2334
+#define regSQ_THREAD_TRACE_PERF_MASK_BASE_IDX                                                           1
+#define regSQ_THREAD_TRACE_CTRL                                                                         0x2335
+#define regSQ_THREAD_TRACE_CTRL_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_MODE                                                                         0x2336
+#define regSQ_THREAD_TRACE_MODE_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_BASE2                                                                        0x2337
+#define regSQ_THREAD_TRACE_BASE2_BASE_IDX                                                               1
+#define regSQ_THREAD_TRACE_TOKEN_MASK2                                                                  0x2338
+#define regSQ_THREAD_TRACE_TOKEN_MASK2_BASE_IDX                                                         1
+#define regSQ_THREAD_TRACE_WPTR                                                                         0x2339
+#define regSQ_THREAD_TRACE_WPTR_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_STATUS                                                                       0x233a
+#define regSQ_THREAD_TRACE_STATUS_BASE_IDX                                                              1
+#define regSQ_THREAD_TRACE_HIWATER                                                                      0x233b
+#define regSQ_THREAD_TRACE_HIWATER_BASE_IDX                                                             1
+#define regSQ_THREAD_TRACE_CNTR                                                                         0x233c
+#define regSQ_THREAD_TRACE_CNTR_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_USERDATA_0                                                                   0x2340
+#define regSQ_THREAD_TRACE_USERDATA_0_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_1                                                                   0x2341
+#define regSQ_THREAD_TRACE_USERDATA_1_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_2                                                                   0x2342
+#define regSQ_THREAD_TRACE_USERDATA_2_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_3                                                                   0x2343
+#define regSQ_THREAD_TRACE_USERDATA_3_BASE_IDX                                                          1
+#define regSQC_CACHES                                                                                   0x2348
+#define regSQC_CACHES_BASE_IDX                                                                          1
+#define regSQC_WRITEBACK                                                                                0x2349
+#define regSQC_WRITEBACK_BASE_IDX                                                                       1
+#define regDB_OCCLUSION_COUNT0_LOW                                                                      0x23c0
+#define regDB_OCCLUSION_COUNT0_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT0_HI                                                                       0x23c1
+#define regDB_OCCLUSION_COUNT0_HI_BASE_IDX                                                              1
+#define regDB_OCCLUSION_COUNT1_LOW                                                                      0x23c2
+#define regDB_OCCLUSION_COUNT1_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT1_HI                                                                       0x23c3
+#define regDB_OCCLUSION_COUNT1_HI_BASE_IDX                                                              1
+#define regDB_OCCLUSION_COUNT2_LOW                                                                      0x23c4
+#define regDB_OCCLUSION_COUNT2_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT2_HI                                                                       0x23c5
+#define regDB_OCCLUSION_COUNT2_HI_BASE_IDX                                                              1
+#define regDB_OCCLUSION_COUNT3_LOW                                                                      0x23c6
+#define regDB_OCCLUSION_COUNT3_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT3_HI                                                                       0x23c7
+#define regDB_OCCLUSION_COUNT3_HI_BASE_IDX                                                              1
+#define regDB_ZPASS_COUNT_LOW                                                                           0x23fe
+#define regDB_ZPASS_COUNT_LOW_BASE_IDX                                                                  1
+#define regDB_ZPASS_COUNT_HI                                                                            0x23ff
+#define regDB_ZPASS_COUNT_HI_BASE_IDX                                                                   1
+#define regGDS_RD_ADDR                                                                                  0x2400
+#define regGDS_RD_ADDR_BASE_IDX                                                                         1
+#define regGDS_RD_DATA                                                                                  0x2401
+#define regGDS_RD_DATA_BASE_IDX                                                                         1
+#define regGDS_RD_BURST_ADDR                                                                            0x2402
+#define regGDS_RD_BURST_ADDR_BASE_IDX                                                                   1
+#define regGDS_RD_BURST_COUNT                                                                           0x2403
+#define regGDS_RD_BURST_COUNT_BASE_IDX                                                                  1
+#define regGDS_RD_BURST_DATA                                                                            0x2404
+#define regGDS_RD_BURST_DATA_BASE_IDX                                                                   1
+#define regGDS_WR_ADDR                                                                                  0x2405
+#define regGDS_WR_ADDR_BASE_IDX                                                                         1
+#define regGDS_WR_DATA                                                                                  0x2406
+#define regGDS_WR_DATA_BASE_IDX                                                                         1
+#define regGDS_WR_BURST_ADDR                                                                            0x2407
+#define regGDS_WR_BURST_ADDR_BASE_IDX                                                                   1
+#define regGDS_WR_BURST_DATA                                                                            0x2408
+#define regGDS_WR_BURST_DATA_BASE_IDX                                                                   1
+#define regGDS_WRITE_COMPLETE                                                                           0x2409
+#define regGDS_WRITE_COMPLETE_BASE_IDX                                                                  1
+#define regGDS_ATOM_CNTL                                                                                0x240a
+#define regGDS_ATOM_CNTL_BASE_IDX                                                                       1
+#define regGDS_ATOM_COMPLETE                                                                            0x240b
+#define regGDS_ATOM_COMPLETE_BASE_IDX                                                                   1
+#define regGDS_ATOM_BASE                                                                                0x240c
+#define regGDS_ATOM_BASE_BASE_IDX                                                                       1
+#define regGDS_ATOM_SIZE                                                                                0x240d
+#define regGDS_ATOM_SIZE_BASE_IDX                                                                       1
+#define regGDS_ATOM_OFFSET0                                                                             0x240e
+#define regGDS_ATOM_OFFSET0_BASE_IDX                                                                    1
+#define regGDS_ATOM_OFFSET1                                                                             0x240f
+#define regGDS_ATOM_OFFSET1_BASE_IDX                                                                    1
+#define regGDS_ATOM_DST                                                                                 0x2410
+#define regGDS_ATOM_DST_BASE_IDX                                                                        1
+#define regGDS_ATOM_OP                                                                                  0x2411
+#define regGDS_ATOM_OP_BASE_IDX                                                                         1
+#define regGDS_ATOM_SRC0                                                                                0x2412
+#define regGDS_ATOM_SRC0_BASE_IDX                                                                       1
+#define regGDS_ATOM_SRC0_U                                                                              0x2413
+#define regGDS_ATOM_SRC0_U_BASE_IDX                                                                     1
+#define regGDS_ATOM_SRC1                                                                                0x2414
+#define regGDS_ATOM_SRC1_BASE_IDX                                                                       1
+#define regGDS_ATOM_SRC1_U                                                                              0x2415
+#define regGDS_ATOM_SRC1_U_BASE_IDX                                                                     1
+#define regGDS_ATOM_READ0                                                                               0x2416
+#define regGDS_ATOM_READ0_BASE_IDX                                                                      1
+#define regGDS_ATOM_READ0_U                                                                             0x2417
+#define regGDS_ATOM_READ0_U_BASE_IDX                                                                    1
+#define regGDS_ATOM_READ1                                                                               0x2418
+#define regGDS_ATOM_READ1_BASE_IDX                                                                      1
+#define regGDS_ATOM_READ1_U                                                                             0x2419
+#define regGDS_ATOM_READ1_U_BASE_IDX                                                                    1
+#define regGDS_GWS_RESOURCE_CNTL                                                                        0x241a
+#define regGDS_GWS_RESOURCE_CNTL_BASE_IDX                                                               1
+#define regGDS_GWS_RESOURCE                                                                             0x241b
+#define regGDS_GWS_RESOURCE_BASE_IDX                                                                    1
+#define regGDS_GWS_RESOURCE_CNT                                                                         0x241c
+#define regGDS_GWS_RESOURCE_CNT_BASE_IDX                                                                1
+#define regGDS_OA_CNTL                                                                                  0x241d
+#define regGDS_OA_CNTL_BASE_IDX                                                                         1
+#define regGDS_OA_COUNTER                                                                               0x241e
+#define regGDS_OA_COUNTER_BASE_IDX                                                                      1
+#define regGDS_OA_ADDRESS                                                                               0x241f
+#define regGDS_OA_ADDRESS_BASE_IDX                                                                      1
+#define regGDS_OA_INCDEC                                                                                0x2420
+#define regGDS_OA_INCDEC_BASE_IDX                                                                       1
+#define regGDS_OA_RING_SIZE                                                                             0x2421
+#define regGDS_OA_RING_SIZE_BASE_IDX                                                                    1
+#define regSPI_CONFIG_CNTL                                                                              0x2440
+#define regSPI_CONFIG_CNTL_BASE_IDX                                                                     1
+#define regSPI_CONFIG_CNTL_1                                                                            0x2441
+#define regSPI_CONFIG_CNTL_1_BASE_IDX                                                                   1
+#define regSPI_CONFIG_CNTL_2                                                                            0x2442
+#define regSPI_CONFIG_CNTL_2_BASE_IDX                                                                   1
+#define regSPI_WAVE_LIMIT_CNTL                                                                          0x2443
+#define regSPI_WAVE_LIMIT_CNTL_BASE_IDX                                                                 1
+
+
+// addressBlock: xcd0_gc_perfddec
+// base address: 0x34000
+#define regCPG_PERFCOUNTER1_LO                                                                          0x3000
+#define regCPG_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCPG_PERFCOUNTER1_HI                                                                          0x3001
+#define regCPG_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCPG_PERFCOUNTER0_LO                                                                          0x3002
+#define regCPG_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCPG_PERFCOUNTER0_HI                                                                          0x3003
+#define regCPG_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER1_LO                                                                          0x3004
+#define regCPC_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER1_HI                                                                          0x3005
+#define regCPC_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER0_LO                                                                          0x3006
+#define regCPC_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER0_HI                                                                          0x3007
+#define regCPC_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER1_LO                                                                          0x3008
+#define regCPF_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER1_HI                                                                          0x3009
+#define regCPF_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER0_LO                                                                          0x300a
+#define regCPF_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER0_HI                                                                          0x300b
+#define regCPF_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCPF_LATENCY_STATS_DATA                                                                       0x300c
+#define regCPF_LATENCY_STATS_DATA_BASE_IDX                                                              1
+#define regCPG_LATENCY_STATS_DATA                                                                       0x300d
+#define regCPG_LATENCY_STATS_DATA_BASE_IDX                                                              1
+#define regCPC_LATENCY_STATS_DATA                                                                       0x300e
+#define regCPC_LATENCY_STATS_DATA_BASE_IDX                                                              1
+#define regGRBM_PERFCOUNTER0_LO                                                                         0x3040
+#define regGRBM_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGRBM_PERFCOUNTER0_HI                                                                         0x3041
+#define regGRBM_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGRBM_PERFCOUNTER1_LO                                                                         0x3043
+#define regGRBM_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGRBM_PERFCOUNTER1_HI                                                                         0x3044
+#define regGRBM_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGRBM_SE0_PERFCOUNTER_LO                                                                      0x3045
+#define regGRBM_SE0_PERFCOUNTER_LO_BASE_IDX                                                             1
+#define regGRBM_SE0_PERFCOUNTER_HI                                                                      0x3046
+#define regGRBM_SE0_PERFCOUNTER_HI_BASE_IDX                                                             1
+#define regGRBM_SE1_PERFCOUNTER_LO                                                                      0x3047
+#define regGRBM_SE1_PERFCOUNTER_LO_BASE_IDX                                                             1
+#define regGRBM_SE1_PERFCOUNTER_HI                                                                      0x3048
+#define regGRBM_SE1_PERFCOUNTER_HI_BASE_IDX                                                             1
+#define regGRBM_SE2_PERFCOUNTER_LO                                                                      0x3049
+#define regGRBM_SE2_PERFCOUNTER_LO_BASE_IDX                                                             1
+#define regGRBM_SE2_PERFCOUNTER_HI                                                                      0x304a
+#define regGRBM_SE2_PERFCOUNTER_HI_BASE_IDX                                                             1
+#define regGRBM_SE3_PERFCOUNTER_LO                                                                      0x304b
+#define regGRBM_SE3_PERFCOUNTER_LO_BASE_IDX                                                             1
+#define regGRBM_SE3_PERFCOUNTER_HI                                                                      0x304c
+#define regGRBM_SE3_PERFCOUNTER_HI_BASE_IDX                                                             1
+#define regWD_PERFCOUNTER0_LO                                                                           0x3080
+#define regWD_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regWD_PERFCOUNTER0_HI                                                                           0x3081
+#define regWD_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regWD_PERFCOUNTER1_LO                                                                           0x3082
+#define regWD_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regWD_PERFCOUNTER1_HI                                                                           0x3083
+#define regWD_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regWD_PERFCOUNTER2_LO                                                                           0x3084
+#define regWD_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regWD_PERFCOUNTER2_HI                                                                           0x3085
+#define regWD_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regWD_PERFCOUNTER3_LO                                                                           0x3086
+#define regWD_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regWD_PERFCOUNTER3_HI                                                                           0x3087
+#define regWD_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regIA_PERFCOUNTER0_LO                                                                           0x3088
+#define regIA_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regIA_PERFCOUNTER0_HI                                                                           0x3089
+#define regIA_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regIA_PERFCOUNTER1_LO                                                                           0x308a
+#define regIA_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regIA_PERFCOUNTER1_HI                                                                           0x308b
+#define regIA_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regIA_PERFCOUNTER2_LO                                                                           0x308c
+#define regIA_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regIA_PERFCOUNTER2_HI                                                                           0x308d
+#define regIA_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regIA_PERFCOUNTER3_LO                                                                           0x308e
+#define regIA_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regIA_PERFCOUNTER3_HI                                                                           0x308f
+#define regIA_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regVGT_PERFCOUNTER0_LO                                                                          0x3090
+#define regVGT_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regVGT_PERFCOUNTER0_HI                                                                          0x3091
+#define regVGT_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regVGT_PERFCOUNTER1_LO                                                                          0x3092
+#define regVGT_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regVGT_PERFCOUNTER1_HI                                                                          0x3093
+#define regVGT_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regVGT_PERFCOUNTER2_LO                                                                          0x3094
+#define regVGT_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regVGT_PERFCOUNTER2_HI                                                                          0x3095
+#define regVGT_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regVGT_PERFCOUNTER3_LO                                                                          0x3096
+#define regVGT_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regVGT_PERFCOUNTER3_HI                                                                          0x3097
+#define regVGT_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regPA_SU_PERFCOUNTER0_LO                                                                        0x3100
+#define regPA_SU_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER0_HI                                                                        0x3101
+#define regPA_SU_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER1_LO                                                                        0x3102
+#define regPA_SU_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER1_HI                                                                        0x3103
+#define regPA_SU_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER2_LO                                                                        0x3104
+#define regPA_SU_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER2_HI                                                                        0x3105
+#define regPA_SU_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER3_LO                                                                        0x3106
+#define regPA_SU_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER3_HI                                                                        0x3107
+#define regPA_SU_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER0_LO                                                                        0x3140
+#define regPA_SC_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER0_HI                                                                        0x3141
+#define regPA_SC_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER1_LO                                                                        0x3142
+#define regPA_SC_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER1_HI                                                                        0x3143
+#define regPA_SC_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER2_LO                                                                        0x3144
+#define regPA_SC_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER2_HI                                                                        0x3145
+#define regPA_SC_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER3_LO                                                                        0x3146
+#define regPA_SC_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER3_HI                                                                        0x3147
+#define regPA_SC_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER4_LO                                                                        0x3148
+#define regPA_SC_PERFCOUNTER4_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER4_HI                                                                        0x3149
+#define regPA_SC_PERFCOUNTER4_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER5_LO                                                                        0x314a
+#define regPA_SC_PERFCOUNTER5_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER5_HI                                                                        0x314b
+#define regPA_SC_PERFCOUNTER5_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER6_LO                                                                        0x314c
+#define regPA_SC_PERFCOUNTER6_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER6_HI                                                                        0x314d
+#define regPA_SC_PERFCOUNTER6_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER7_LO                                                                        0x314e
+#define regPA_SC_PERFCOUNTER7_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER7_HI                                                                        0x314f
+#define regPA_SC_PERFCOUNTER7_HI_BASE_IDX                                                               1
+#define regSPI_PERFCOUNTER0_HI                                                                          0x3180
+#define regSPI_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER0_LO                                                                          0x3181
+#define regSPI_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER1_HI                                                                          0x3182
+#define regSPI_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER1_LO                                                                          0x3183
+#define regSPI_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER2_HI                                                                          0x3184
+#define regSPI_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER2_LO                                                                          0x3185
+#define regSPI_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER3_HI                                                                          0x3186
+#define regSPI_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER3_LO                                                                          0x3187
+#define regSPI_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER4_HI                                                                          0x3188
+#define regSPI_PERFCOUNTER4_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER4_LO                                                                          0x3189
+#define regSPI_PERFCOUNTER4_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER5_HI                                                                          0x318a
+#define regSPI_PERFCOUNTER5_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER5_LO                                                                          0x318b
+#define regSPI_PERFCOUNTER5_LO_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER0_LO                                                                           0x31c0
+#define regSQ_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER0_HI                                                                           0x31c1
+#define regSQ_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER1_LO                                                                           0x31c2
+#define regSQ_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER1_HI                                                                           0x31c3
+#define regSQ_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER2_LO                                                                           0x31c4
+#define regSQ_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER2_HI                                                                           0x31c5
+#define regSQ_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER3_LO                                                                           0x31c6
+#define regSQ_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER3_HI                                                                           0x31c7
+#define regSQ_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER4_LO                                                                           0x31c8
+#define regSQ_PERFCOUNTER4_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER4_HI                                                                           0x31c9
+#define regSQ_PERFCOUNTER4_HI_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER5_LO                                                                           0x31ca
+#define regSQ_PERFCOUNTER5_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER5_HI                                                                           0x31cb
+#define regSQ_PERFCOUNTER5_HI_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER6_LO                                                                           0x31cc
+#define regSQ_PERFCOUNTER6_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER6_HI                                                                           0x31cd
+#define regSQ_PERFCOUNTER6_HI_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER7_LO                                                                           0x31ce
+#define regSQ_PERFCOUNTER7_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER7_HI                                                                           0x31cf
+#define regSQ_PERFCOUNTER7_HI_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER8_LO                                                                           0x31d0
+#define regSQ_PERFCOUNTER8_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER8_HI                                                                           0x31d1
+#define regSQ_PERFCOUNTER8_HI_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER9_LO                                                                           0x31d2
+#define regSQ_PERFCOUNTER9_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER9_HI                                                                           0x31d3
+#define regSQ_PERFCOUNTER9_HI_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER10_LO                                                                          0x31d4
+#define regSQ_PERFCOUNTER10_LO_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER10_HI                                                                          0x31d5
+#define regSQ_PERFCOUNTER10_HI_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER11_LO                                                                          0x31d6
+#define regSQ_PERFCOUNTER11_LO_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER11_HI                                                                          0x31d7
+#define regSQ_PERFCOUNTER11_HI_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER12_LO                                                                          0x31d8
+#define regSQ_PERFCOUNTER12_LO_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER12_HI                                                                          0x31d9
+#define regSQ_PERFCOUNTER12_HI_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER13_LO                                                                          0x31da
+#define regSQ_PERFCOUNTER13_LO_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER13_HI                                                                          0x31db
+#define regSQ_PERFCOUNTER13_HI_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER14_LO                                                                          0x31dc
+#define regSQ_PERFCOUNTER14_LO_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER14_HI                                                                          0x31dd
+#define regSQ_PERFCOUNTER14_HI_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER15_LO                                                                          0x31de
+#define regSQ_PERFCOUNTER15_LO_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER15_HI                                                                          0x31df
+#define regSQ_PERFCOUNTER15_HI_BASE_IDX                                                                 1
+#define regSX_PERFCOUNTER0_LO                                                                           0x3240
+#define regSX_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER0_HI                                                                           0x3241
+#define regSX_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER1_LO                                                                           0x3242
+#define regSX_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER1_HI                                                                           0x3243
+#define regSX_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER2_LO                                                                           0x3244
+#define regSX_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER2_HI                                                                           0x3245
+#define regSX_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER3_LO                                                                           0x3246
+#define regSX_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER3_HI                                                                           0x3247
+#define regSX_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regGDS_PERFCOUNTER0_LO                                                                          0x3280
+#define regGDS_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER0_HI                                                                          0x3281
+#define regGDS_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER1_LO                                                                          0x3282
+#define regGDS_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER1_HI                                                                          0x3283
+#define regGDS_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER2_LO                                                                          0x3284
+#define regGDS_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER2_HI                                                                          0x3285
+#define regGDS_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER3_LO                                                                          0x3286
+#define regGDS_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER3_HI                                                                          0x3287
+#define regGDS_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regTA_PERFCOUNTER0_LO                                                                           0x32c0
+#define regTA_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regTA_PERFCOUNTER0_HI                                                                           0x32c1
+#define regTA_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regTA_PERFCOUNTER1_LO                                                                           0x32c2
+#define regTA_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regTA_PERFCOUNTER1_HI                                                                           0x32c3
+#define regTA_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER0_LO                                                                           0x3300
+#define regTD_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER0_HI                                                                           0x3301
+#define regTD_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER1_LO                                                                           0x3302
+#define regTD_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER1_HI                                                                           0x3303
+#define regTD_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regTCP_PERFCOUNTER0_LO                                                                          0x3340
+#define regTCP_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER0_HI                                                                          0x3341
+#define regTCP_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER1_LO                                                                          0x3342
+#define regTCP_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER1_HI                                                                          0x3343
+#define regTCP_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER2_LO                                                                          0x3344
+#define regTCP_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER2_HI                                                                          0x3345
+#define regTCP_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER3_LO                                                                          0x3346
+#define regTCP_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER3_HI                                                                          0x3347
+#define regTCP_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regTCC_PERFCOUNTER0_LO                                                                          0x3380
+#define regTCC_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regTCC_PERFCOUNTER0_HI                                                                          0x3381
+#define regTCC_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regTCC_PERFCOUNTER1_LO                                                                          0x3382
+#define regTCC_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regTCC_PERFCOUNTER1_HI                                                                          0x3383
+#define regTCC_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regTCC_PERFCOUNTER2_LO                                                                          0x3384
+#define regTCC_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regTCC_PERFCOUNTER2_HI                                                                          0x3385
+#define regTCC_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regTCC_PERFCOUNTER3_LO                                                                          0x3386
+#define regTCC_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regTCC_PERFCOUNTER3_HI                                                                          0x3387
+#define regTCC_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regTCA_PERFCOUNTER0_LO                                                                          0x3390
+#define regTCA_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regTCA_PERFCOUNTER0_HI                                                                          0x3391
+#define regTCA_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regTCA_PERFCOUNTER1_LO                                                                          0x3392
+#define regTCA_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regTCA_PERFCOUNTER1_HI                                                                          0x3393
+#define regTCA_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regTCA_PERFCOUNTER2_LO                                                                          0x3394
+#define regTCA_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regTCA_PERFCOUNTER2_HI                                                                          0x3395
+#define regTCA_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regTCA_PERFCOUNTER3_LO                                                                          0x3396
+#define regTCA_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regTCA_PERFCOUNTER3_HI                                                                          0x3397
+#define regTCA_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regCB_PERFCOUNTER0_LO                                                                           0x3406
+#define regCB_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER0_HI                                                                           0x3407
+#define regCB_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER1_LO                                                                           0x3408
+#define regCB_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER1_HI                                                                           0x3409
+#define regCB_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER2_LO                                                                           0x340a
+#define regCB_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER2_HI                                                                           0x340b
+#define regCB_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER3_LO                                                                           0x340c
+#define regCB_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER3_HI                                                                           0x340d
+#define regCB_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER0_LO                                                                           0x3440
+#define regDB_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER0_HI                                                                           0x3441
+#define regDB_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER1_LO                                                                           0x3442
+#define regDB_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER1_HI                                                                           0x3443
+#define regDB_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER2_LO                                                                           0x3444
+#define regDB_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER2_HI                                                                           0x3445
+#define regDB_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER3_LO                                                                           0x3446
+#define regDB_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER3_HI                                                                           0x3447
+#define regDB_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regRLC_PERFCOUNTER0_LO                                                                          0x3480
+#define regRLC_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regRLC_PERFCOUNTER0_HI                                                                          0x3481
+#define regRLC_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regRLC_PERFCOUNTER1_LO                                                                          0x3482
+#define regRLC_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regRLC_PERFCOUNTER1_HI                                                                          0x3483
+#define regRLC_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER0_LO                                                                          0x34c0
+#define regRMI_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER0_HI                                                                          0x34c1
+#define regRMI_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER1_LO                                                                          0x34c2
+#define regRMI_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER1_HI                                                                          0x34c3
+#define regRMI_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER2_LO                                                                          0x34c4
+#define regRMI_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER2_HI                                                                          0x34c5
+#define regRMI_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER3_LO                                                                          0x34c6
+#define regRMI_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER3_HI                                                                          0x34c7
+#define regRMI_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+
+
+// addressBlock: xcd0_gc_utcl2_atcl2pfcntrdec
+// base address: 0x35400
+#define regATC_L2_PERFCOUNTER_LO                                                                        0x3500
+#define regATC_L2_PERFCOUNTER_LO_BASE_IDX                                                               1
+#define regATC_L2_PERFCOUNTER_HI                                                                        0x3501
+#define regATC_L2_PERFCOUNTER_HI_BASE_IDX                                                               1
+
+
+// addressBlock: xcd0_gc_utcl2_vml2prdec
+// base address: 0x35408
+#define regMC_VM_L2_PERFCOUNTER_LO                                                                      0x3502
+#define regMC_VM_L2_PERFCOUNTER_LO_BASE_IDX                                                             1
+#define regMC_VM_L2_PERFCOUNTER_HI                                                                      0x3503
+#define regMC_VM_L2_PERFCOUNTER_HI_BASE_IDX                                                             1
+
+
+// addressBlock: xcd0_gc_utcl2_l2tlbprdec
+// base address: 0x35448
+#define regL2TLB_PERFCOUNTER_LO                                                                         0x3512
+#define regL2TLB_PERFCOUNTER_LO_BASE_IDX                                                                1
+#define regL2TLB_PERFCOUNTER_HI                                                                         0x3513
+#define regL2TLB_PERFCOUNTER_HI_BASE_IDX                                                                1
+
+
+// addressBlock: xcd0_gc_perfsdec
+// base address: 0x36000
+#define regCPG_PERFCOUNTER1_SELECT                                                                      0x3800
+#define regCPG_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCPG_PERFCOUNTER0_SELECT1                                                                     0x3801
+#define regCPG_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCPG_PERFCOUNTER0_SELECT                                                                      0x3802
+#define regCPG_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCPC_PERFCOUNTER1_SELECT                                                                      0x3803
+#define regCPC_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCPC_PERFCOUNTER0_SELECT1                                                                     0x3804
+#define regCPC_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCPF_PERFCOUNTER1_SELECT                                                                      0x3805
+#define regCPF_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCPF_PERFCOUNTER0_SELECT1                                                                     0x3806
+#define regCPF_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCPF_PERFCOUNTER0_SELECT                                                                      0x3807
+#define regCPF_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCP_PERFMON_CNTL                                                                              0x3808
+#define regCP_PERFMON_CNTL_BASE_IDX                                                                     1
+#define regCPC_PERFCOUNTER0_SELECT                                                                      0x3809
+#define regCPC_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCPF_TC_PERF_COUNTER_WINDOW_SELECT                                                            0x380a
+#define regCPF_TC_PERF_COUNTER_WINDOW_SELECT_BASE_IDX                                                   1
+#define regCPG_TC_PERF_COUNTER_WINDOW_SELECT                                                            0x380b
+#define regCPG_TC_PERF_COUNTER_WINDOW_SELECT_BASE_IDX                                                   1
+#define regCPF_LATENCY_STATS_SELECT                                                                     0x380c
+#define regCPF_LATENCY_STATS_SELECT_BASE_IDX                                                            1
+#define regCPG_LATENCY_STATS_SELECT                                                                     0x380d
+#define regCPG_LATENCY_STATS_SELECT_BASE_IDX                                                            1
+#define regCPC_LATENCY_STATS_SELECT                                                                     0x380e
+#define regCPC_LATENCY_STATS_SELECT_BASE_IDX                                                            1
+#define regCP_DRAW_OBJECT                                                                               0x3810
+#define regCP_DRAW_OBJECT_BASE_IDX                                                                      1
+#define regCP_DRAW_OBJECT_COUNTER                                                                       0x3811
+#define regCP_DRAW_OBJECT_COUNTER_BASE_IDX                                                              1
+#define regCP_DRAW_WINDOW_MASK_HI                                                                       0x3812
+#define regCP_DRAW_WINDOW_MASK_HI_BASE_IDX                                                              1
+#define regCP_DRAW_WINDOW_HI                                                                            0x3813
+#define regCP_DRAW_WINDOW_HI_BASE_IDX                                                                   1
+#define regCP_DRAW_WINDOW_LO                                                                            0x3814
+#define regCP_DRAW_WINDOW_LO_BASE_IDX                                                                   1
+#define regCP_DRAW_WINDOW_CNTL                                                                          0x3815
+#define regCP_DRAW_WINDOW_CNTL_BASE_IDX                                                                 1
+#define regGRBM_PERFCOUNTER0_SELECT                                                                     0x3840
+#define regGRBM_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGRBM_PERFCOUNTER1_SELECT                                                                     0x3841
+#define regGRBM_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGRBM_SE0_PERFCOUNTER_SELECT                                                                  0x3842
+#define regGRBM_SE0_PERFCOUNTER_SELECT_BASE_IDX                                                         1
+#define regGRBM_SE1_PERFCOUNTER_SELECT                                                                  0x3843
+#define regGRBM_SE1_PERFCOUNTER_SELECT_BASE_IDX                                                         1
+#define regGRBM_SE2_PERFCOUNTER_SELECT                                                                  0x3844
+#define regGRBM_SE2_PERFCOUNTER_SELECT_BASE_IDX                                                         1
+#define regGRBM_SE3_PERFCOUNTER_SELECT                                                                  0x3845
+#define regGRBM_SE3_PERFCOUNTER_SELECT_BASE_IDX                                                         1
+#define regWD_PERFCOUNTER0_SELECT                                                                       0x3880
+#define regWD_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regWD_PERFCOUNTER1_SELECT                                                                       0x3881
+#define regWD_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regWD_PERFCOUNTER2_SELECT                                                                       0x3882
+#define regWD_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regWD_PERFCOUNTER3_SELECT                                                                       0x3883
+#define regWD_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regIA_PERFCOUNTER0_SELECT                                                                       0x3884
+#define regIA_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regIA_PERFCOUNTER1_SELECT                                                                       0x3885
+#define regIA_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regIA_PERFCOUNTER2_SELECT                                                                       0x3886
+#define regIA_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regIA_PERFCOUNTER3_SELECT                                                                       0x3887
+#define regIA_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regIA_PERFCOUNTER0_SELECT1                                                                      0x3888
+#define regIA_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regVGT_PERFCOUNTER0_SELECT                                                                      0x388c
+#define regVGT_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regVGT_PERFCOUNTER1_SELECT                                                                      0x388d
+#define regVGT_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regVGT_PERFCOUNTER2_SELECT                                                                      0x388e
+#define regVGT_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regVGT_PERFCOUNTER3_SELECT                                                                      0x388f
+#define regVGT_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regVGT_PERFCOUNTER0_SELECT1                                                                     0x3890
+#define regVGT_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regVGT_PERFCOUNTER1_SELECT1                                                                     0x3891
+#define regVGT_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regVGT_PERFCOUNTER_SEID_MASK                                                                    0x3894
+#define regVGT_PERFCOUNTER_SEID_MASK_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER0_SELECT                                                                    0x3900
+#define regPA_SU_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER0_SELECT1                                                                   0x3901
+#define regPA_SU_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regPA_SU_PERFCOUNTER1_SELECT                                                                    0x3902
+#define regPA_SU_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER1_SELECT1                                                                   0x3903
+#define regPA_SU_PERFCOUNTER1_SELECT1_BASE_IDX                                                          1
+#define regPA_SU_PERFCOUNTER2_SELECT                                                                    0x3904
+#define regPA_SU_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER3_SELECT                                                                    0x3905
+#define regPA_SU_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER0_SELECT                                                                    0x3940
+#define regPA_SC_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER0_SELECT1                                                                   0x3941
+#define regPA_SC_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regPA_SC_PERFCOUNTER1_SELECT                                                                    0x3942
+#define regPA_SC_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER2_SELECT                                                                    0x3943
+#define regPA_SC_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER3_SELECT                                                                    0x3944
+#define regPA_SC_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER4_SELECT                                                                    0x3945
+#define regPA_SC_PERFCOUNTER4_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER5_SELECT                                                                    0x3946
+#define regPA_SC_PERFCOUNTER5_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER6_SELECT                                                                    0x3947
+#define regPA_SC_PERFCOUNTER6_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER7_SELECT                                                                    0x3948
+#define regPA_SC_PERFCOUNTER7_SELECT_BASE_IDX                                                           1
+#define regSPI_PERFCOUNTER0_SELECT                                                                      0x3980
+#define regSPI_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER1_SELECT                                                                      0x3981
+#define regSPI_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER2_SELECT                                                                      0x3982
+#define regSPI_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER3_SELECT                                                                      0x3983
+#define regSPI_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER0_SELECT1                                                                     0x3984
+#define regSPI_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER1_SELECT1                                                                     0x3985
+#define regSPI_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER2_SELECT1                                                                     0x3986
+#define regSPI_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER3_SELECT1                                                                     0x3987
+#define regSPI_PERFCOUNTER3_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER4_SELECT                                                                      0x3988
+#define regSPI_PERFCOUNTER4_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER5_SELECT                                                                      0x3989
+#define regSPI_PERFCOUNTER5_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER_BINS                                                                         0x398a
+#define regSPI_PERFCOUNTER_BINS_BASE_IDX                                                                1
+#define regSQ_PERFCOUNTER0_SELECT                                                                       0x39c0
+#define regSQ_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER1_SELECT                                                                       0x39c1
+#define regSQ_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER2_SELECT                                                                       0x39c2
+#define regSQ_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER3_SELECT                                                                       0x39c3
+#define regSQ_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER4_SELECT                                                                       0x39c4
+#define regSQ_PERFCOUNTER4_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER5_SELECT                                                                       0x39c5
+#define regSQ_PERFCOUNTER5_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER6_SELECT                                                                       0x39c6
+#define regSQ_PERFCOUNTER6_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER7_SELECT                                                                       0x39c7
+#define regSQ_PERFCOUNTER7_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER8_SELECT                                                                       0x39c8
+#define regSQ_PERFCOUNTER8_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER9_SELECT                                                                       0x39c9
+#define regSQ_PERFCOUNTER9_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER10_SELECT                                                                      0x39ca
+#define regSQ_PERFCOUNTER10_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER11_SELECT                                                                      0x39cb
+#define regSQ_PERFCOUNTER11_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER12_SELECT                                                                      0x39cc
+#define regSQ_PERFCOUNTER12_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER13_SELECT                                                                      0x39cd
+#define regSQ_PERFCOUNTER13_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER14_SELECT                                                                      0x39ce
+#define regSQ_PERFCOUNTER14_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER15_SELECT                                                                      0x39cf
+#define regSQ_PERFCOUNTER15_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER_CTRL                                                                          0x39e0
+#define regSQ_PERFCOUNTER_CTRL_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER_MASK                                                                          0x39e1
+#define regSQ_PERFCOUNTER_MASK_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER_CTRL2                                                                         0x39e2
+#define regSQ_PERFCOUNTER_CTRL2_BASE_IDX                                                                1
+#define regSX_PERFCOUNTER0_SELECT                                                                       0x3a40
+#define regSX_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER1_SELECT                                                                       0x3a41
+#define regSX_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER2_SELECT                                                                       0x3a42
+#define regSX_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER3_SELECT                                                                       0x3a43
+#define regSX_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER0_SELECT1                                                                      0x3a44
+#define regSX_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regSX_PERFCOUNTER1_SELECT1                                                                      0x3a45
+#define regSX_PERFCOUNTER1_SELECT1_BASE_IDX                                                             1
+#define regGDS_PERFCOUNTER0_SELECT                                                                      0x3a80
+#define regGDS_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regGDS_PERFCOUNTER1_SELECT                                                                      0x3a81
+#define regGDS_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regGDS_PERFCOUNTER2_SELECT                                                                      0x3a82
+#define regGDS_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regGDS_PERFCOUNTER3_SELECT                                                                      0x3a83
+#define regGDS_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regGDS_PERFCOUNTER0_SELECT1                                                                     0x3a84
+#define regGDS_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regTA_PERFCOUNTER0_SELECT                                                                       0x3ac0
+#define regTA_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regTA_PERFCOUNTER0_SELECT1                                                                      0x3ac1
+#define regTA_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regTA_PERFCOUNTER1_SELECT                                                                       0x3ac2
+#define regTA_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regTD_PERFCOUNTER0_SELECT                                                                       0x3b00
+#define regTD_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regTD_PERFCOUNTER0_SELECT1                                                                      0x3b01
+#define regTD_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regTD_PERFCOUNTER1_SELECT                                                                       0x3b02
+#define regTD_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regTCP_PERFCOUNTER0_SELECT                                                                      0x3b40
+#define regTCP_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER0_SELECT1                                                                     0x3b41
+#define regTCP_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regTCP_PERFCOUNTER1_SELECT                                                                      0x3b42
+#define regTCP_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER1_SELECT1                                                                     0x3b43
+#define regTCP_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regTCP_PERFCOUNTER2_SELECT                                                                      0x3b44
+#define regTCP_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER3_SELECT                                                                      0x3b45
+#define regTCP_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regTCC_PERFCOUNTER0_SELECT                                                                      0x3b80
+#define regTCC_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regTCC_PERFCOUNTER0_SELECT1                                                                     0x3b81
+#define regTCC_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regTCC_PERFCOUNTER1_SELECT                                                                      0x3b82
+#define regTCC_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regTCC_PERFCOUNTER1_SELECT1                                                                     0x3b83
+#define regTCC_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regTCC_PERFCOUNTER2_SELECT                                                                      0x3b84
+#define regTCC_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regTCC_PERFCOUNTER3_SELECT                                                                      0x3b85
+#define regTCC_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regTCA_PERFCOUNTER0_SELECT                                                                      0x3b90
+#define regTCA_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regTCA_PERFCOUNTER0_SELECT1                                                                     0x3b91
+#define regTCA_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regTCA_PERFCOUNTER1_SELECT                                                                      0x3b92
+#define regTCA_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regTCA_PERFCOUNTER1_SELECT1                                                                     0x3b93
+#define regTCA_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regTCA_PERFCOUNTER2_SELECT                                                                      0x3b94
+#define regTCA_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regTCA_PERFCOUNTER3_SELECT                                                                      0x3b95
+#define regTCA_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regCB_PERFCOUNTER_FILTER                                                                        0x3c00
+#define regCB_PERFCOUNTER_FILTER_BASE_IDX                                                               1
+#define regCB_PERFCOUNTER0_SELECT                                                                       0x3c01
+#define regCB_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regCB_PERFCOUNTER0_SELECT1                                                                      0x3c02
+#define regCB_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regCB_PERFCOUNTER1_SELECT                                                                       0x3c03
+#define regCB_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regCB_PERFCOUNTER2_SELECT                                                                       0x3c04
+#define regCB_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regCB_PERFCOUNTER3_SELECT                                                                       0x3c05
+#define regCB_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER0_SELECT                                                                       0x3c40
+#define regDB_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER0_SELECT1                                                                      0x3c41
+#define regDB_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regDB_PERFCOUNTER1_SELECT                                                                       0x3c42
+#define regDB_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER1_SELECT1                                                                      0x3c43
+#define regDB_PERFCOUNTER1_SELECT1_BASE_IDX                                                             1
+#define regDB_PERFCOUNTER2_SELECT                                                                       0x3c44
+#define regDB_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER3_SELECT                                                                       0x3c46
+#define regDB_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regRLC_SPM_PERFMON_CNTL                                                                         0x3c80
+#define regRLC_SPM_PERFMON_CNTL_BASE_IDX                                                                1
+#define regRLC_SPM_PERFMON_RING_BASE_LO                                                                 0x3c81
+#define regRLC_SPM_PERFMON_RING_BASE_LO_BASE_IDX                                                        1
+#define regRLC_SPM_PERFMON_RING_BASE_HI                                                                 0x3c82
+#define regRLC_SPM_PERFMON_RING_BASE_HI_BASE_IDX                                                        1
+#define regRLC_SPM_PERFMON_RING_SIZE                                                                    0x3c83
+#define regRLC_SPM_PERFMON_RING_SIZE_BASE_IDX                                                           1
+#define regRLC_SPM_PERFMON_SEGMENT_SIZE                                                                 0x3c84
+#define regRLC_SPM_PERFMON_SEGMENT_SIZE_BASE_IDX                                                        1
+#define regRLC_SPM_SE_MUXSEL_ADDR                                                                       0x3c85
+#define regRLC_SPM_SE_MUXSEL_ADDR_BASE_IDX                                                              1
+#define regRLC_SPM_SE_MUXSEL_DATA                                                                       0x3c86
+#define regRLC_SPM_SE_MUXSEL_DATA_BASE_IDX                                                              1
+#define regRLC_SPM_CPG_PERFMON_SAMPLE_DELAY                                                             0x3c87
+#define regRLC_SPM_CPG_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_CPC_PERFMON_SAMPLE_DELAY                                                             0x3c88
+#define regRLC_SPM_CPC_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_CPF_PERFMON_SAMPLE_DELAY                                                             0x3c89
+#define regRLC_SPM_CPF_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_CB_PERFMON_SAMPLE_DELAY                                                              0x3c8a
+#define regRLC_SPM_CB_PERFMON_SAMPLE_DELAY_BASE_IDX                                                     1
+#define regRLC_SPM_DB_PERFMON_SAMPLE_DELAY                                                              0x3c8b
+#define regRLC_SPM_DB_PERFMON_SAMPLE_DELAY_BASE_IDX                                                     1
+#define regRLC_SPM_PA_PERFMON_SAMPLE_DELAY                                                              0x3c8c
+#define regRLC_SPM_PA_PERFMON_SAMPLE_DELAY_BASE_IDX                                                     1
+#define regRLC_SPM_GDS_PERFMON_SAMPLE_DELAY                                                             0x3c8d
+#define regRLC_SPM_GDS_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_IA_PERFMON_SAMPLE_DELAY                                                              0x3c8e
+#define regRLC_SPM_IA_PERFMON_SAMPLE_DELAY_BASE_IDX                                                     1
+#define regRLC_SPM_SC_PERFMON_SAMPLE_DELAY                                                              0x3c90
+#define regRLC_SPM_SC_PERFMON_SAMPLE_DELAY_BASE_IDX                                                     1
+#define regRLC_SPM_TCC_PERFMON_SAMPLE_DELAY                                                             0x3c91
+#define regRLC_SPM_TCC_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_TCA_PERFMON_SAMPLE_DELAY                                                             0x3c92
+#define regRLC_SPM_TCA_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_TCP_PERFMON_SAMPLE_DELAY                                                             0x3c93
+#define regRLC_SPM_TCP_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_TA_PERFMON_SAMPLE_DELAY                                                              0x3c94
+#define regRLC_SPM_TA_PERFMON_SAMPLE_DELAY_BASE_IDX                                                     1
+#define regRLC_SPM_TD_PERFMON_SAMPLE_DELAY                                                              0x3c95
+#define regRLC_SPM_TD_PERFMON_SAMPLE_DELAY_BASE_IDX                                                     1
+#define regRLC_SPM_VGT_PERFMON_SAMPLE_DELAY                                                             0x3c96
+#define regRLC_SPM_VGT_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_SPI_PERFMON_SAMPLE_DELAY                                                             0x3c97
+#define regRLC_SPM_SPI_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_SQG_PERFMON_SAMPLE_DELAY                                                             0x3c98
+#define regRLC_SPM_SQG_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_SX_PERFMON_SAMPLE_DELAY                                                              0x3c9a
+#define regRLC_SPM_SX_PERFMON_SAMPLE_DELAY_BASE_IDX                                                     1
+#define regRLC_SPM_GLOBAL_MUXSEL_ADDR                                                                   0x3c9b
+#define regRLC_SPM_GLOBAL_MUXSEL_ADDR_BASE_IDX                                                          1
+#define regRLC_SPM_GLOBAL_MUXSEL_DATA                                                                   0x3c9c
+#define regRLC_SPM_GLOBAL_MUXSEL_DATA_BASE_IDX                                                          1
+#define regRLC_SPM_RING_RDPTR                                                                           0x3c9d
+#define regRLC_SPM_RING_RDPTR_BASE_IDX                                                                  1
+#define regRLC_SPM_SEGMENT_THRESHOLD                                                                    0x3c9e
+#define regRLC_SPM_SEGMENT_THRESHOLD_BASE_IDX                                                           1
+#define regRLC_SPM_RMI_PERFMON_SAMPLE_DELAY                                                             0x3ca3
+#define regRLC_SPM_RMI_PERFMON_SAMPLE_DELAY_BASE_IDX                                                    1
+#define regRLC_SPM_PERFMON_SAMPLE_DELAY_MAX                                                             0x3ca4
+#define regRLC_SPM_PERFMON_SAMPLE_DELAY_MAX_BASE_IDX                                                    1
+#define regRLC_PERFMON_CNTL                                                                             0x3cc0
+#define regRLC_PERFMON_CNTL_BASE_IDX                                                                    1
+#define regRLC_PERFCOUNTER0_SELECT                                                                      0x3cc1
+#define regRLC_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regRLC_PERFCOUNTER1_SELECT                                                                      0x3cc2
+#define regRLC_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regRLC_GPU_IOV_PERF_CNT_CNTL                                                                    0x3cc3
+#define regRLC_GPU_IOV_PERF_CNT_CNTL_BASE_IDX                                                           1
+#define regRLC_GPU_IOV_PERF_CNT_WR_ADDR                                                                 0x3cc4
+#define regRLC_GPU_IOV_PERF_CNT_WR_ADDR_BASE_IDX                                                        1
+#define regRLC_GPU_IOV_PERF_CNT_WR_DATA                                                                 0x3cc5
+#define regRLC_GPU_IOV_PERF_CNT_WR_DATA_BASE_IDX                                                        1
+#define regRLC_GPU_IOV_PERF_CNT_RD_ADDR                                                                 0x3cc6
+#define regRLC_GPU_IOV_PERF_CNT_RD_ADDR_BASE_IDX                                                        1
+#define regRLC_GPU_IOV_PERF_CNT_RD_DATA                                                                 0x3cc7
+#define regRLC_GPU_IOV_PERF_CNT_RD_DATA_BASE_IDX                                                        1
+#define regRMI_PERFCOUNTER0_SELECT                                                                      0x3d00
+#define regRMI_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regRMI_PERFCOUNTER0_SELECT1                                                                     0x3d01
+#define regRMI_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regRMI_PERFCOUNTER1_SELECT                                                                      0x3d02
+#define regRMI_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regRMI_PERFCOUNTER2_SELECT                                                                      0x3d03
+#define regRMI_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regRMI_PERFCOUNTER2_SELECT1                                                                     0x3d04
+#define regRMI_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regRMI_PERFCOUNTER3_SELECT                                                                      0x3d05
+#define regRMI_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regRMI_PERF_COUNTER_CNTL                                                                        0x3d06
+#define regRMI_PERF_COUNTER_CNTL_BASE_IDX                                                               1
+
+
+// addressBlock: xcd0_gc_utcl2_atcl2pfcntldec
+// base address: 0x37500
+#define regATC_L2_PERFCOUNTER0_CFG                                                                      0x3d40
+#define regATC_L2_PERFCOUNTER0_CFG_BASE_IDX                                                             1
+#define regATC_L2_PERFCOUNTER1_CFG                                                                      0x3d41
+#define regATC_L2_PERFCOUNTER1_CFG_BASE_IDX                                                             1
+#define regATC_L2_PERFCOUNTER_RSLT_CNTL                                                                 0x3d42
+#define regATC_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                        1
+
+
+// addressBlock: xcd0_gc_utcl2_vml2pldec
+// base address: 0x37518
+#define regMC_VM_L2_PERFCOUNTER0_CFG                                                                    0x3d46
+#define regMC_VM_L2_PERFCOUNTER0_CFG_BASE_IDX                                                           1
+#define regMC_VM_L2_PERFCOUNTER1_CFG                                                                    0x3d47
+#define regMC_VM_L2_PERFCOUNTER1_CFG_BASE_IDX                                                           1
+#define regMC_VM_L2_PERFCOUNTER2_CFG                                                                    0x3d48
+#define regMC_VM_L2_PERFCOUNTER2_CFG_BASE_IDX                                                           1
+#define regMC_VM_L2_PERFCOUNTER3_CFG                                                                    0x3d49
+#define regMC_VM_L2_PERFCOUNTER3_CFG_BASE_IDX                                                           1
+#define regMC_VM_L2_PERFCOUNTER4_CFG                                                                    0x3d4a
+#define regMC_VM_L2_PERFCOUNTER4_CFG_BASE_IDX                                                           1
+#define regMC_VM_L2_PERFCOUNTER5_CFG                                                                    0x3d4b
+#define regMC_VM_L2_PERFCOUNTER5_CFG_BASE_IDX                                                           1
+#define regMC_VM_L2_PERFCOUNTER6_CFG                                                                    0x3d4c
+#define regMC_VM_L2_PERFCOUNTER6_CFG_BASE_IDX                                                           1
+#define regMC_VM_L2_PERFCOUNTER7_CFG                                                                    0x3d4d
+#define regMC_VM_L2_PERFCOUNTER7_CFG_BASE_IDX                                                           1
+#define regMC_VM_L2_PERFCOUNTER_RSLT_CNTL                                                               0x3d56
+#define regMC_VM_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                      1
+
+
+// addressBlock: xcd0_gc_utcl2_l2tlbpldec
+// base address: 0x37578
+#define regL2TLB_PERFCOUNTER0_CFG                                                                       0x3d5e
+#define regL2TLB_PERFCOUNTER0_CFG_BASE_IDX                                                              1
+#define regL2TLB_PERFCOUNTER1_CFG                                                                       0x3d5f
+#define regL2TLB_PERFCOUNTER1_CFG_BASE_IDX                                                              1
+#define regL2TLB_PERFCOUNTER2_CFG                                                                       0x3d60
+#define regL2TLB_PERFCOUNTER2_CFG_BASE_IDX                                                              1
+#define regL2TLB_PERFCOUNTER3_CFG                                                                       0x3d61
+#define regL2TLB_PERFCOUNTER3_CFG_BASE_IDX                                                              1
+#define regL2TLB_PERFCOUNTER_RSLT_CNTL                                                                  0x3d62
+#define regL2TLB_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         1
+
+
+// addressBlock: xcd0_gc_gdflldec
+// base address: 0x3a000
+#define regGDFLL_EDC_HYSTERESIS_CNTL                                                                    0x481b
+#define regGDFLL_EDC_HYSTERESIS_CNTL_BASE_IDX                                                           1
+#define regGDFLL_EDC_HYSTERESIS_STAT                                                                    0x481c
+#define regGDFLL_EDC_HYSTERESIS_STAT_BASE_IDX                                                           1
+
+
+// addressBlock: xcd0_gc_rlcpdec
+// base address: 0x3b000
+#define regRLC_CNTL                                                                                     0x4c00
+#define regRLC_CNTL_BASE_IDX                                                                            1
+#define regRLC_CGCG_CGLS_CTRL_2                                                                         0x4c03
+#define regRLC_CGCG_CGLS_CTRL_2_BASE_IDX                                                                1
+#define regRLC_STAT                                                                                     0x4c04
+#define regRLC_STAT_BASE_IDX                                                                            1
+#define regRLC_SAFE_MODE                                                                                0x4c05
+#define regRLC_SAFE_MODE_BASE_IDX                                                                       1
+#define regRLC_MEM_SLP_CNTL                                                                             0x4c06
+#define regRLC_MEM_SLP_CNTL_BASE_IDX                                                                    1
+#define regSMU_RLC_RESPONSE                                                                             0x4c07
+#define regSMU_RLC_RESPONSE_BASE_IDX                                                                    1
+#define regRLC_RLCV_SAFE_MODE                                                                           0x4c08
+#define regRLC_RLCV_SAFE_MODE_BASE_IDX                                                                  1
+#define regRLC_SMU_SAFE_MODE                                                                            0x4c09
+#define regRLC_SMU_SAFE_MODE_BASE_IDX                                                                   1
+#define regRLC_RLCV_COMMAND                                                                             0x4c0a
+#define regRLC_RLCV_COMMAND_BASE_IDX                                                                    1
+#define regRLC_REFCLOCK_TIMESTAMP_LSB                                                                   0x4c0c
+#define regRLC_REFCLOCK_TIMESTAMP_LSB_BASE_IDX                                                          1
+#define regRLC_REFCLOCK_TIMESTAMP_MSB                                                                   0x4c0d
+#define regRLC_REFCLOCK_TIMESTAMP_MSB_BASE_IDX                                                          1
+#define regRLC_GPM_TIMER_INT_0                                                                          0x4c0e
+#define regRLC_GPM_TIMER_INT_0_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_INT_1                                                                          0x4c0f
+#define regRLC_GPM_TIMER_INT_1_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_INT_2                                                                          0x4c10
+#define regRLC_GPM_TIMER_INT_2_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_CTRL                                                                           0x4c11
+#define regRLC_GPM_TIMER_CTRL_BASE_IDX                                                                  1
+#define regRLC_LB_CNTR_MAX                                                                              0x4c12
+#define regRLC_LB_CNTR_MAX_BASE_IDX                                                                     1
+#define regRLC_GPM_TIMER_STAT                                                                           0x4c13
+#define regRLC_GPM_TIMER_STAT_BASE_IDX                                                                  1
+#define regRLC_GPM_TIMER_INT_3                                                                          0x4c15
+#define regRLC_GPM_TIMER_INT_3_BASE_IDX                                                                 1
+#define regRLC_SERDES_WR_NONCU_MASTER_MASK_1                                                            0x4c16
+#define regRLC_SERDES_WR_NONCU_MASTER_MASK_1_BASE_IDX                                                   1
+#define regRLC_SERDES_NONCU_MASTER_BUSY_1                                                               0x4c17
+#define regRLC_SERDES_NONCU_MASTER_BUSY_1_BASE_IDX                                                      1
+#define regRLC_INT_STAT                                                                                 0x4c18
+#define regRLC_INT_STAT_BASE_IDX                                                                        1
+#define regRLC_LB_CNTL                                                                                  0x4c19
+#define regRLC_LB_CNTL_BASE_IDX                                                                         1
+#define regRLC_MGCG_CTRL                                                                                0x4c1a
+#define regRLC_MGCG_CTRL_BASE_IDX                                                                       1
+#define regRLC_LB_CNTR_INIT                                                                             0x4c1b
+#define regRLC_LB_CNTR_INIT_BASE_IDX                                                                    1
+#define regRLC_LOAD_BALANCE_CNTR                                                                        0x4c1c
+#define regRLC_LOAD_BALANCE_CNTR_BASE_IDX                                                               1
+#define regRLC_JUMP_TABLE_RESTORE                                                                       0x4c1e
+#define regRLC_JUMP_TABLE_RESTORE_BASE_IDX                                                              1
+#define regRLC_PG_DELAY_2                                                                               0x4c1f
+#define regRLC_PG_DELAY_2_BASE_IDX                                                                      1
+#define regRLC_GPU_CLOCK_COUNT_LSB                                                                      0x4c24
+#define regRLC_GPU_CLOCK_COUNT_LSB_BASE_IDX                                                             1
+#define regRLC_GPU_CLOCK_COUNT_MSB                                                                      0x4c25
+#define regRLC_GPU_CLOCK_COUNT_MSB_BASE_IDX                                                             1
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT                                                                  0x4c26
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_BASE_IDX                                                         1
+#define regRLC_UCODE_CNTL                                                                               0x4c27
+#define regRLC_UCODE_CNTL_BASE_IDX                                                                      1
+#define regRLC_GPM_THREAD_RESET                                                                         0x4c28
+#define regRLC_GPM_THREAD_RESET_BASE_IDX                                                                1
+#define regRLC_GPM_CP_DMA_COMPLETE_T0                                                                   0x4c29
+#define regRLC_GPM_CP_DMA_COMPLETE_T0_BASE_IDX                                                          1
+#define regRLC_GPM_CP_DMA_COMPLETE_T1                                                                   0x4c2a
+#define regRLC_GPM_CP_DMA_COMPLETE_T1_BASE_IDX                                                          1
+#define regRLC_FIREWALL_VIOLATION                                                                       0x4c2b
+#define regRLC_FIREWALL_VIOLATION_BASE_IDX                                                              1
+#define regRLC_CLK_COUNT_GFXCLK_LSB                                                                     0x4c30
+#define regRLC_CLK_COUNT_GFXCLK_LSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_GFXCLK_MSB                                                                     0x4c31
+#define regRLC_CLK_COUNT_GFXCLK_MSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_REFCLK_LSB                                                                     0x4c32
+#define regRLC_CLK_COUNT_REFCLK_LSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_REFCLK_MSB                                                                     0x4c33
+#define regRLC_CLK_COUNT_REFCLK_MSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_CTRL                                                                           0x4c34
+#define regRLC_CLK_COUNT_CTRL_BASE_IDX                                                                  1
+#define regRLC_CLK_COUNT_STAT                                                                           0x4c35
+#define regRLC_CLK_COUNT_STAT_BASE_IDX                                                                  1
+#define regRLC_GPM_STAT                                                                                 0x4c40
+#define regRLC_GPM_STAT_BASE_IDX                                                                        1
+#define regRLC_GPU_CLOCK_32_RES_SEL                                                                     0x4c41
+#define regRLC_GPU_CLOCK_32_RES_SEL_BASE_IDX                                                            1
+#define regRLC_GPU_CLOCK_32                                                                             0x4c42
+#define regRLC_GPU_CLOCK_32_BASE_IDX                                                                    1
+#define regRLC_PG_CNTL                                                                                  0x4c43
+#define regRLC_PG_CNTL_BASE_IDX                                                                         1
+#define regRLC_GPM_THREAD_PRIORITY                                                                      0x4c44
+#define regRLC_GPM_THREAD_PRIORITY_BASE_IDX                                                             1
+#define regRLC_GPM_THREAD_ENABLE                                                                        0x4c45
+#define regRLC_GPM_THREAD_ENABLE_BASE_IDX                                                               1
+#define regRLC_CGTT_MGCG_OVERRIDE                                                                       0x4c48
+#define regRLC_CGTT_MGCG_OVERRIDE_BASE_IDX                                                              1
+#define regRLC_CGCG_CGLS_CTRL                                                                           0x4c49
+#define regRLC_CGCG_CGLS_CTRL_BASE_IDX                                                                  1
+#define regRLC_CGCG_RAMP_CTRL                                                                           0x4c4a
+#define regRLC_CGCG_RAMP_CTRL_BASE_IDX                                                                  1
+#define regRLC_DYN_PG_STATUS                                                                            0x4c4b
+#define regRLC_DYN_PG_STATUS_BASE_IDX                                                                   1
+#define regRLC_DYN_PG_REQUEST                                                                           0x4c4c
+#define regRLC_DYN_PG_REQUEST_BASE_IDX                                                                  1
+#define regRLC_PG_DELAY                                                                                 0x4c4d
+#define regRLC_PG_DELAY_BASE_IDX                                                                        1
+#define regRLC_CU_STATUS                                                                                0x4c4e
+#define regRLC_CU_STATUS_BASE_IDX                                                                       1
+#define regRLC_LB_INIT_CU_MASK                                                                          0x4c4f
+#define regRLC_LB_INIT_CU_MASK_BASE_IDX                                                                 1
+#define regRLC_LB_ALWAYS_ACTIVE_CU_MASK                                                                 0x4c50
+#define regRLC_LB_ALWAYS_ACTIVE_CU_MASK_BASE_IDX                                                        1
+#define regRLC_LB_PARAMS                                                                                0x4c51
+#define regRLC_LB_PARAMS_BASE_IDX                                                                       1
+#define regRLC_THREAD1_DELAY                                                                            0x4c52
+#define regRLC_THREAD1_DELAY_BASE_IDX                                                                   1
+#define regRLC_PG_ALWAYS_ON_CU_MASK                                                                     0x4c53
+#define regRLC_PG_ALWAYS_ON_CU_MASK_BASE_IDX                                                            1
+#define regRLC_MAX_PG_CU                                                                                0x4c54
+#define regRLC_MAX_PG_CU_BASE_IDX                                                                       1
+#define regRLC_AUTO_PG_CTRL                                                                             0x4c55
+#define regRLC_AUTO_PG_CTRL_BASE_IDX                                                                    1
+#define regRLC_SMU_GRBM_REG_SAVE_CTRL                                                                   0x4c56
+#define regRLC_SMU_GRBM_REG_SAVE_CTRL_BASE_IDX                                                          1
+#define regRLC_SERDES_RD_PENDING                                                                        0x4c58
+#define regRLC_SERDES_RD_PENDING_BASE_IDX                                                               1
+#define regRLC_SERDES_RD_MASTER_INDEX                                                                   0x4c59
+#define regRLC_SERDES_RD_MASTER_INDEX_BASE_IDX                                                          1
+#define regRLC_SERDES_RD_DATA_0                                                                         0x4c5a
+#define regRLC_SERDES_RD_DATA_0_BASE_IDX                                                                1
+#define regRLC_SERDES_RD_DATA_1                                                                         0x4c5b
+#define regRLC_SERDES_RD_DATA_1_BASE_IDX                                                                1
+#define regRLC_SERDES_RD_DATA_2                                                                         0x4c5c
+#define regRLC_SERDES_RD_DATA_2_BASE_IDX                                                                1
+#define regRLC_SERDES_WR_CU_MASTER_MASK                                                                 0x4c5d
+#define regRLC_SERDES_WR_CU_MASTER_MASK_BASE_IDX                                                        1
+#define regRLC_SERDES_WR_NONCU_MASTER_MASK                                                              0x4c5e
+#define regRLC_SERDES_WR_NONCU_MASTER_MASK_BASE_IDX                                                     1
+#define regRLC_SERDES_WR_CTRL                                                                           0x4c5f
+#define regRLC_SERDES_WR_CTRL_BASE_IDX                                                                  1
+#define regRLC_SERDES_WR_DATA                                                                           0x4c60
+#define regRLC_SERDES_WR_DATA_BASE_IDX                                                                  1
+#define regRLC_SERDES_CU_MASTER_BUSY                                                                    0x4c61
+#define regRLC_SERDES_CU_MASTER_BUSY_BASE_IDX                                                           1
+#define regRLC_SERDES_NONCU_MASTER_BUSY                                                                 0x4c62
+#define regRLC_SERDES_NONCU_MASTER_BUSY_BASE_IDX                                                        1
+#define regRLC_GPM_GENERAL_0                                                                            0x4c63
+#define regRLC_GPM_GENERAL_0_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_1                                                                            0x4c64
+#define regRLC_GPM_GENERAL_1_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_2                                                                            0x4c65
+#define regRLC_GPM_GENERAL_2_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_3                                                                            0x4c66
+#define regRLC_GPM_GENERAL_3_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_4                                                                            0x4c67
+#define regRLC_GPM_GENERAL_4_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_5                                                                            0x4c68
+#define regRLC_GPM_GENERAL_5_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_6                                                                            0x4c69
+#define regRLC_GPM_GENERAL_6_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_7                                                                            0x4c6a
+#define regRLC_GPM_GENERAL_7_BASE_IDX                                                                   1
+#define regRLC_GPM_SCRATCH_ADDR                                                                         0x4c6c
+#define regRLC_GPM_SCRATCH_ADDR_BASE_IDX                                                                1
+#define regRLC_GPM_SCRATCH_DATA                                                                         0x4c6d
+#define regRLC_GPM_SCRATCH_DATA_BASE_IDX                                                                1
+#define regRLC_STATIC_PG_STATUS                                                                         0x4c6e
+#define regRLC_STATIC_PG_STATUS_BASE_IDX                                                                1
+#define regRLC_SPM_MC_CNTL                                                                              0x4c71
+#define regRLC_SPM_MC_CNTL_BASE_IDX                                                                     1
+#define regRLC_SPM_INT_CNTL                                                                             0x4c72
+#define regRLC_SPM_INT_CNTL_BASE_IDX                                                                    1
+#define regRLC_SPM_INT_STATUS                                                                           0x4c73
+#define regRLC_SPM_INT_STATUS_BASE_IDX                                                                  1
+#define regRLC_SMU_MESSAGE                                                                              0x4c76
+#define regRLC_SMU_MESSAGE_BASE_IDX                                                                     1
+#define regRLC_GPM_LOG_SIZE                                                                             0x4c77
+#define regRLC_GPM_LOG_SIZE_BASE_IDX                                                                    1
+#define regRLC_PG_DELAY_3                                                                               0x4c78
+#define regRLC_PG_DELAY_3_BASE_IDX                                                                      1
+#define regRLC_GPR_REG1                                                                                 0x4c79
+#define regRLC_GPR_REG1_BASE_IDX                                                                        1
+#define regRLC_GPR_REG2                                                                                 0x4c7a
+#define regRLC_GPR_REG2_BASE_IDX                                                                        1
+#define regRLC_GPM_LOG_CONT                                                                             0x4c7b
+#define regRLC_GPM_LOG_CONT_BASE_IDX                                                                    1
+#define regRLC_GPM_INT_DISABLE_TH0                                                                      0x4c7c
+#define regRLC_GPM_INT_DISABLE_TH0_BASE_IDX                                                             1
+#define regRLC_GPM_INT_FORCE_TH0                                                                        0x4c7e
+#define regRLC_GPM_INT_FORCE_TH0_BASE_IDX                                                               1
+#define regRLC_GPM_INT_FORCE_TH1                                                                        0x4c7f
+#define regRLC_GPM_INT_FORCE_TH1_BASE_IDX                                                               1
+#define regRLC_SRM_CNTL                                                                                 0x4c80
+#define regRLC_SRM_CNTL_BASE_IDX                                                                        1
+#define regRLC_SRM_ARAM_ADDR                                                                            0x4c83
+#define regRLC_SRM_ARAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_SRM_ARAM_DATA                                                                            0x4c84
+#define regRLC_SRM_ARAM_DATA_BASE_IDX                                                                   1
+#define regRLC_SRM_DRAM_ADDR                                                                            0x4c85
+#define regRLC_SRM_DRAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_SRM_DRAM_DATA                                                                            0x4c86
+#define regRLC_SRM_DRAM_DATA_BASE_IDX                                                                   1
+#define regRLC_SRM_GPM_COMMAND                                                                          0x4c87
+#define regRLC_SRM_GPM_COMMAND_BASE_IDX                                                                 1
+#define regRLC_SRM_GPM_COMMAND_STATUS                                                                   0x4c88
+#define regRLC_SRM_GPM_COMMAND_STATUS_BASE_IDX                                                          1
+#define regRLC_SRM_RLCV_COMMAND                                                                         0x4c89
+#define regRLC_SRM_RLCV_COMMAND_BASE_IDX                                                                1
+#define regRLC_SRM_RLCV_COMMAND_STATUS                                                                  0x4c8a
+#define regRLC_SRM_RLCV_COMMAND_STATUS_BASE_IDX                                                         1
+#define regRLC_SRM_INDEX_CNTL_ADDR_0                                                                    0x4c8b
+#define regRLC_SRM_INDEX_CNTL_ADDR_0_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_1                                                                    0x4c8c
+#define regRLC_SRM_INDEX_CNTL_ADDR_1_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_2                                                                    0x4c8d
+#define regRLC_SRM_INDEX_CNTL_ADDR_2_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_3                                                                    0x4c8e
+#define regRLC_SRM_INDEX_CNTL_ADDR_3_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_4                                                                    0x4c8f
+#define regRLC_SRM_INDEX_CNTL_ADDR_4_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_5                                                                    0x4c90
+#define regRLC_SRM_INDEX_CNTL_ADDR_5_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_6                                                                    0x4c91
+#define regRLC_SRM_INDEX_CNTL_ADDR_6_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_7                                                                    0x4c92
+#define regRLC_SRM_INDEX_CNTL_ADDR_7_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_0                                                                    0x4c93
+#define regRLC_SRM_INDEX_CNTL_DATA_0_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_1                                                                    0x4c94
+#define regRLC_SRM_INDEX_CNTL_DATA_1_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_2                                                                    0x4c95
+#define regRLC_SRM_INDEX_CNTL_DATA_2_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_3                                                                    0x4c96
+#define regRLC_SRM_INDEX_CNTL_DATA_3_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_4                                                                    0x4c97
+#define regRLC_SRM_INDEX_CNTL_DATA_4_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_5                                                                    0x4c98
+#define regRLC_SRM_INDEX_CNTL_DATA_5_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_6                                                                    0x4c99
+#define regRLC_SRM_INDEX_CNTL_DATA_6_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_7                                                                    0x4c9a
+#define regRLC_SRM_INDEX_CNTL_DATA_7_BASE_IDX                                                           1
+#define regRLC_SRM_STAT                                                                                 0x4c9b
+#define regRLC_SRM_STAT_BASE_IDX                                                                        1
+#define regRLC_SRM_GPM_ABORT                                                                            0x4c9c
+#define regRLC_SRM_GPM_ABORT_BASE_IDX                                                                   1
+#define regRLC_CSIB_ADDR_LO                                                                             0x4ca2
+#define regRLC_CSIB_ADDR_LO_BASE_IDX                                                                    1
+#define regRLC_CSIB_ADDR_HI                                                                             0x4ca3
+#define regRLC_CSIB_ADDR_HI_BASE_IDX                                                                    1
+#define regRLC_CSIB_LENGTH                                                                              0x4ca4
+#define regRLC_CSIB_LENGTH_BASE_IDX                                                                     1
+#define regRLC_SMU_COMMAND                                                                              0x4ca9
+#define regRLC_SMU_COMMAND_BASE_IDX                                                                     1
+#define regRLC_CP_SCHEDULERS                                                                            0x4caa
+#define regRLC_CP_SCHEDULERS_BASE_IDX                                                                   1
+#define regRLC_SMU_ARGUMENT_1                                                                           0x4cab
+#define regRLC_SMU_ARGUMENT_1_BASE_IDX                                                                  1
+#define regRLC_SMU_ARGUMENT_2                                                                           0x4cac
+#define regRLC_SMU_ARGUMENT_2_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_8                                                                            0x4cad
+#define regRLC_GPM_GENERAL_8_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_9                                                                            0x4cae
+#define regRLC_GPM_GENERAL_9_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_10                                                                           0x4caf
+#define regRLC_GPM_GENERAL_10_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_11                                                                           0x4cb0
+#define regRLC_GPM_GENERAL_11_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_12                                                                           0x4cb1
+#define regRLC_GPM_GENERAL_12_BASE_IDX                                                                  1
+#define regRLC_GPM_UTCL1_CNTL_0                                                                         0x4cb2
+#define regRLC_GPM_UTCL1_CNTL_0_BASE_IDX                                                                1
+#define regRLC_GPM_UTCL1_CNTL_1                                                                         0x4cb3
+#define regRLC_GPM_UTCL1_CNTL_1_BASE_IDX                                                                1
+#define regRLC_GPM_UTCL1_CNTL_2                                                                         0x4cb4
+#define regRLC_GPM_UTCL1_CNTL_2_BASE_IDX                                                                1
+#define regRLC_SPM_UTCL1_CNTL                                                                           0x4cb5
+#define regRLC_SPM_UTCL1_CNTL_BASE_IDX                                                                  1
+#define regRLC_UTCL1_STATUS_2                                                                           0x4cb6
+#define regRLC_UTCL1_STATUS_2_BASE_IDX                                                                  1
+#define regRLC_LB_THR_CONFIG_2                                                                          0x4cb8
+#define regRLC_LB_THR_CONFIG_2_BASE_IDX                                                                 1
+#define regRLC_LB_THR_CONFIG_3                                                                          0x4cb9
+#define regRLC_LB_THR_CONFIG_3_BASE_IDX                                                                 1
+#define regRLC_LB_THR_CONFIG_4                                                                          0x4cba
+#define regRLC_LB_THR_CONFIG_4_BASE_IDX                                                                 1
+#define regRLC_SPM_UTCL1_ERROR_1                                                                        0x4cbc
+#define regRLC_SPM_UTCL1_ERROR_1_BASE_IDX                                                               1
+#define regRLC_SPM_UTCL1_ERROR_2                                                                        0x4cbd
+#define regRLC_SPM_UTCL1_ERROR_2_BASE_IDX                                                               1
+#define regRLC_GPM_UTCL1_TH0_ERROR_1                                                                    0x4cbe
+#define regRLC_GPM_UTCL1_TH0_ERROR_1_BASE_IDX                                                           1
+#define regRLC_LB_THR_CONFIG_1                                                                          0x4cbf
+#define regRLC_LB_THR_CONFIG_1_BASE_IDX                                                                 1
+#define regRLC_GPM_UTCL1_TH0_ERROR_2                                                                    0x4cc0
+#define regRLC_GPM_UTCL1_TH0_ERROR_2_BASE_IDX                                                           1
+#define regRLC_GPM_UTCL1_TH1_ERROR_1                                                                    0x4cc1
+#define regRLC_GPM_UTCL1_TH1_ERROR_1_BASE_IDX                                                           1
+#define regRLC_GPM_UTCL1_TH1_ERROR_2                                                                    0x4cc2
+#define regRLC_GPM_UTCL1_TH1_ERROR_2_BASE_IDX                                                           1
+#define regRLC_GPM_UTCL1_TH2_ERROR_1                                                                    0x4cc3
+#define regRLC_GPM_UTCL1_TH2_ERROR_1_BASE_IDX                                                           1
+#define regRLC_GPM_UTCL1_TH2_ERROR_2                                                                    0x4cc4
+#define regRLC_GPM_UTCL1_TH2_ERROR_2_BASE_IDX                                                           1
+#define regRLC_SEMAPHORE_0                                                                              0x4cc7
+#define regRLC_SEMAPHORE_0_BASE_IDX                                                                     1
+#define regRLC_SEMAPHORE_1                                                                              0x4cc8
+#define regRLC_SEMAPHORE_1_BASE_IDX                                                                     1
+#define regRLC_CP_EOF_INT                                                                               0x4cca
+#define regRLC_CP_EOF_INT_BASE_IDX                                                                      1
+#define regRLC_CP_EOF_INT_CNT                                                                           0x4ccb
+#define regRLC_CP_EOF_INT_CNT_BASE_IDX                                                                  1
+#define regRLC_SPARE_INT                                                                                0x4ccc
+#define regRLC_SPARE_INT_BASE_IDX                                                                       1
+#define regRLC_PREWALKER_UTCL1_CNTL                                                                     0x4ccd
+#define regRLC_PREWALKER_UTCL1_CNTL_BASE_IDX                                                            1
+#define regRLC_PREWALKER_UTCL1_TRIG                                                                     0x4cce
+#define regRLC_PREWALKER_UTCL1_TRIG_BASE_IDX                                                            1
+#define regRLC_PREWALKER_UTCL1_ADDR_LSB                                                                 0x4ccf
+#define regRLC_PREWALKER_UTCL1_ADDR_LSB_BASE_IDX                                                        1
+#define regRLC_PREWALKER_UTCL1_ADDR_MSB                                                                 0x4cd0
+#define regRLC_PREWALKER_UTCL1_ADDR_MSB_BASE_IDX                                                        1
+#define regRLC_PREWALKER_UTCL1_SIZE_LSB                                                                 0x4cd1
+#define regRLC_PREWALKER_UTCL1_SIZE_LSB_BASE_IDX                                                        1
+#define regRLC_PREWALKER_UTCL1_SIZE_MSB                                                                 0x4cd2
+#define regRLC_PREWALKER_UTCL1_SIZE_MSB_BASE_IDX                                                        1
+#define regRLC_DSM_TRIG                                                                                 0x4cd3
+#define regRLC_DSM_TRIG_BASE_IDX                                                                        1
+#define regRLC_UTCL1_STATUS                                                                             0x4cd4
+#define regRLC_UTCL1_STATUS_BASE_IDX                                                                    1
+#define regRLC_R2I_CNTL_0                                                                               0x4cd5
+#define regRLC_R2I_CNTL_0_BASE_IDX                                                                      1
+#define regRLC_R2I_CNTL_1                                                                               0x4cd6
+#define regRLC_R2I_CNTL_1_BASE_IDX                                                                      1
+#define regRLC_R2I_CNTL_2                                                                               0x4cd7
+#define regRLC_R2I_CNTL_2_BASE_IDX                                                                      1
+#define regRLC_R2I_CNTL_3                                                                               0x4cd8
+#define regRLC_R2I_CNTL_3_BASE_IDX                                                                      1
+#define regRLC_UTCL2_CNTL                                                                               0x4cd9
+#define regRLC_UTCL2_CNTL_BASE_IDX                                                                      1
+#define regRLC_LBPW_CU_STAT                                                                             0x4cda
+#define regRLC_LBPW_CU_STAT_BASE_IDX                                                                    1
+#define regRLC_DS_CNTL                                                                                  0x4cdb
+#define regRLC_DS_CNTL_BASE_IDX                                                                         1
+#define regRLC_GPM_INT_STAT_TH0                                                                         0x4cdc
+#define regRLC_GPM_INT_STAT_TH0_BASE_IDX                                                                1
+#define regRLC_GPM_GENERAL_13                                                                           0x4cdd
+#define regRLC_GPM_GENERAL_13_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_14                                                                           0x4cde
+#define regRLC_GPM_GENERAL_14_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_15                                                                           0x4cdf
+#define regRLC_GPM_GENERAL_15_BASE_IDX                                                                  1
+#define regRLC_SPARE_INT_1                                                                              0x4ce0
+#define regRLC_SPARE_INT_1_BASE_IDX                                                                     1
+#define regRLC_RLCV_SPARE_INT_1                                                                         0x4ce1
+#define regRLC_RLCV_SPARE_INT_1_BASE_IDX                                                                1
+#define regRLC_SEMAPHORE_2                                                                              0x4ce3
+#define regRLC_SEMAPHORE_2_BASE_IDX                                                                     1
+#define regRLC_SEMAPHORE_3                                                                              0x4ce4
+#define regRLC_SEMAPHORE_3_BASE_IDX                                                                     1
+#define regRLC_SMU_ARGUMENT_3                                                                           0x4ce5
+#define regRLC_SMU_ARGUMENT_3_BASE_IDX                                                                  1
+#define regRLC_SMU_ARGUMENT_4                                                                           0x4ce6
+#define regRLC_SMU_ARGUMENT_4_BASE_IDX                                                                  1
+#define regRLC_GPU_CLOCK_COUNT_LSB_1                                                                    0x4ce8
+#define regRLC_GPU_CLOCK_COUNT_LSB_1_BASE_IDX                                                           1
+#define regRLC_GPU_CLOCK_COUNT_MSB_1                                                                    0x4ce9
+#define regRLC_GPU_CLOCK_COUNT_MSB_1_BASE_IDX                                                           1
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_1                                                                0x4cea
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_1_BASE_IDX                                                       1
+#define regRLC_GPU_CLOCK_COUNT_LSB_2                                                                    0x4ceb
+#define regRLC_GPU_CLOCK_COUNT_LSB_2_BASE_IDX                                                           1
+#define regRLC_GPU_CLOCK_COUNT_MSB_2                                                                    0x4cec
+#define regRLC_GPU_CLOCK_COUNT_MSB_2_BASE_IDX                                                           1
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_2                                                                0x4cef
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_2_BASE_IDX                                                       1
+#define regRLC_CPG_STAT_INVAL                                                                           0x4d09
+#define regRLC_CPG_STAT_INVAL_BASE_IDX                                                                  1
+#define regRLC_DSM_CNTL                                                                                 0x4d42
+#define regRLC_DSM_CNTL_BASE_IDX                                                                        1
+#define regRLC_DSM_CNTLA                                                                                0x4d43
+#define regRLC_DSM_CNTLA_BASE_IDX                                                                       1
+#define regRLC_DSM_CNTL2                                                                                0x4d44
+#define regRLC_DSM_CNTL2_BASE_IDX                                                                       1
+#define regRLC_DSM_CNTL2A                                                                               0x4d45
+#define regRLC_DSM_CNTL2A_BASE_IDX                                                                      1
+#define regRLC_RLCV_SPARE_INT                                                                           0x4f30
+#define regRLC_RLCV_SPARE_INT_BASE_IDX                                                                  1
+#define regRLC_SMU_CLK_REQ                                                                              0x4f97
+#define regRLC_SMU_CLK_REQ_BASE_IDX                                                                     1
+
+
+// addressBlock: xcd0_gc_pwrdec
+// base address: 0x3c000
+#define regCGTS_SM_CTRL_REG                                                                             0x5000
+#define regCGTS_SM_CTRL_REG_BASE_IDX                                                                    1
+#define regCGTS_RD_CTRL_REG                                                                             0x5001
+#define regCGTS_RD_CTRL_REG_BASE_IDX                                                                    1
+#define regCGTS_RD_REG                                                                                  0x5002
+#define regCGTS_RD_REG_BASE_IDX                                                                         1
+#define regCGTS_TCC_DISABLE                                                                             0x5003
+#define regCGTS_TCC_DISABLE_BASE_IDX                                                                    1
+#define regCGTS_USER_TCC_DISABLE                                                                        0x5004
+#define regCGTS_USER_TCC_DISABLE_BASE_IDX                                                               1
+#define regCGTS_CU0_SP0_CTRL_REG                                                                        0x5008
+#define regCGTS_CU0_SP0_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU0_LDS_SQ_CTRL_REG                                                                     0x5009
+#define regCGTS_CU0_LDS_SQ_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU0_TA_SQC_CTRL_REG                                                                     0x500a
+#define regCGTS_CU0_TA_SQC_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU0_SP1_CTRL_REG                                                                        0x500b
+#define regCGTS_CU0_SP1_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU0_TD_TCP_CTRL_REG                                                                     0x500c
+#define regCGTS_CU0_TD_TCP_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU1_SP0_CTRL_REG                                                                        0x500d
+#define regCGTS_CU1_SP0_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU1_LDS_SQ_CTRL_REG                                                                     0x500e
+#define regCGTS_CU1_LDS_SQ_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU1_TA_SQC_CTRL_REG                                                                     0x500f
+#define regCGTS_CU1_TA_SQC_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU1_SP1_CTRL_REG                                                                        0x5010
+#define regCGTS_CU1_SP1_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU1_TD_TCP_CTRL_REG                                                                     0x5011
+#define regCGTS_CU1_TD_TCP_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU2_SP0_CTRL_REG                                                                        0x5012
+#define regCGTS_CU2_SP0_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU2_LDS_SQ_CTRL_REG                                                                     0x5013
+#define regCGTS_CU2_LDS_SQ_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU2_TA_SQC_CTRL_REG                                                                     0x5014
+#define regCGTS_CU2_TA_SQC_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU2_SP1_CTRL_REG                                                                        0x5015
+#define regCGTS_CU2_SP1_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU2_TD_TCP_CTRL_REG                                                                     0x5016
+#define regCGTS_CU2_TD_TCP_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU3_SP0_CTRL_REG                                                                        0x5017
+#define regCGTS_CU3_SP0_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU3_LDS_SQ_CTRL_REG                                                                     0x5018
+#define regCGTS_CU3_LDS_SQ_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU3_TA_SQC_CTRL_REG                                                                     0x5019
+#define regCGTS_CU3_TA_SQC_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU3_SP1_CTRL_REG                                                                        0x501a
+#define regCGTS_CU3_SP1_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU3_TD_TCP_CTRL_REG                                                                     0x501b
+#define regCGTS_CU3_TD_TCP_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU4_SP0_CTRL_REG                                                                        0x501c
+#define regCGTS_CU4_SP0_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU4_LDS_SQ_CTRL_REG                                                                     0x501d
+#define regCGTS_CU4_LDS_SQ_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU4_TA_SQC_CTRL_REG                                                                     0x501e
+#define regCGTS_CU4_TA_SQC_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU4_SP1_CTRL_REG                                                                        0x501f
+#define regCGTS_CU4_SP1_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU4_TD_TCP_CTRL_REG                                                                     0x5020
+#define regCGTS_CU4_TD_TCP_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU5_SP0_CTRL_REG                                                                        0x5021
+#define regCGTS_CU5_SP0_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU5_LDS_SQ_CTRL_REG                                                                     0x5022
+#define regCGTS_CU5_LDS_SQ_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU5_TA_SQC_CTRL_REG                                                                     0x5023
+#define regCGTS_CU5_TA_SQC_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU5_SP1_CTRL_REG                                                                        0x5024
+#define regCGTS_CU5_SP1_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU5_TD_TCP_CTRL_REG                                                                     0x5025
+#define regCGTS_CU5_TD_TCP_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU6_SP0_CTRL_REG                                                                        0x5026
+#define regCGTS_CU6_SP0_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU6_LDS_SQ_CTRL_REG                                                                     0x5027
+#define regCGTS_CU6_LDS_SQ_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU6_TA_SQC_CTRL_REG                                                                     0x5028
+#define regCGTS_CU6_TA_SQC_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU6_SP1_CTRL_REG                                                                        0x5029
+#define regCGTS_CU6_SP1_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU6_TD_TCP_CTRL_REG                                                                     0x502a
+#define regCGTS_CU6_TD_TCP_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU7_SP0_CTRL_REG                                                                        0x502b
+#define regCGTS_CU7_SP0_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU7_LDS_SQ_CTRL_REG                                                                     0x502c
+#define regCGTS_CU7_LDS_SQ_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU7_TA_SQC_CTRL_REG                                                                     0x502d
+#define regCGTS_CU7_TA_SQC_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU7_SP1_CTRL_REG                                                                        0x502e
+#define regCGTS_CU7_SP1_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU7_TD_TCP_CTRL_REG                                                                     0x502f
+#define regCGTS_CU7_TD_TCP_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU8_SP0_CTRL_REG                                                                        0x5030
+#define regCGTS_CU8_SP0_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU8_LDS_SQ_CTRL_REG                                                                     0x5031
+#define regCGTS_CU8_LDS_SQ_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU8_TA_SQC_CTRL_REG                                                                     0x5032
+#define regCGTS_CU8_TA_SQC_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU8_SP1_CTRL_REG                                                                        0x5033
+#define regCGTS_CU8_SP1_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU8_TD_TCP_CTRL_REG                                                                     0x5034
+#define regCGTS_CU8_TD_TCP_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU9_SP0_CTRL_REG                                                                        0x5035
+#define regCGTS_CU9_SP0_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU9_LDS_SQ_CTRL_REG                                                                     0x5036
+#define regCGTS_CU9_LDS_SQ_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU9_TA_SQC_CTRL_REG                                                                     0x5037
+#define regCGTS_CU9_TA_SQC_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU9_SP1_CTRL_REG                                                                        0x5038
+#define regCGTS_CU9_SP1_CTRL_REG_BASE_IDX                                                               1
+#define regCGTS_CU9_TD_TCP_CTRL_REG                                                                     0x5039
+#define regCGTS_CU9_TD_TCP_CTRL_REG_BASE_IDX                                                            1
+#define regCGTS_CU10_SP0_CTRL_REG                                                                       0x503a
+#define regCGTS_CU10_SP0_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU10_LDS_SQ_CTRL_REG                                                                    0x503b
+#define regCGTS_CU10_LDS_SQ_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU10_TA_SQC_CTRL_REG                                                                    0x503c
+#define regCGTS_CU10_TA_SQC_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU10_SP1_CTRL_REG                                                                       0x503d
+#define regCGTS_CU10_SP1_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU10_TD_TCP_CTRL_REG                                                                    0x503e
+#define regCGTS_CU10_TD_TCP_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU11_SP0_CTRL_REG                                                                       0x503f
+#define regCGTS_CU11_SP0_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU11_LDS_SQ_CTRL_REG                                                                    0x5040
+#define regCGTS_CU11_LDS_SQ_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU11_TA_SQC_CTRL_REG                                                                    0x5041
+#define regCGTS_CU11_TA_SQC_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU11_SP1_CTRL_REG                                                                       0x5042
+#define regCGTS_CU11_SP1_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU11_TD_TCP_CTRL_REG                                                                    0x5043
+#define regCGTS_CU11_TD_TCP_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU12_SP0_CTRL_REG                                                                       0x5044
+#define regCGTS_CU12_SP0_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU12_LDS_SQ_CTRL_REG                                                                    0x5045
+#define regCGTS_CU12_LDS_SQ_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU12_TA_SQC_CTRL_REG                                                                    0x5046
+#define regCGTS_CU12_TA_SQC_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU12_SP1_CTRL_REG                                                                       0x5047
+#define regCGTS_CU12_SP1_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU12_TD_TCP_CTRL_REG                                                                    0x5048
+#define regCGTS_CU12_TD_TCP_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU13_SP0_CTRL_REG                                                                       0x5049
+#define regCGTS_CU13_SP0_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU13_LDS_SQ_CTRL_REG                                                                    0x504a
+#define regCGTS_CU13_LDS_SQ_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU13_TA_SQC_CTRL_REG                                                                    0x504b
+#define regCGTS_CU13_TA_SQC_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU13_SP1_CTRL_REG                                                                       0x504c
+#define regCGTS_CU13_SP1_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU13_TD_TCP_CTRL_REG                                                                    0x504d
+#define regCGTS_CU13_TD_TCP_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU14_SP0_CTRL_REG                                                                       0x504e
+#define regCGTS_CU14_SP0_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU14_LDS_SQ_CTRL_REG                                                                    0x504f
+#define regCGTS_CU14_LDS_SQ_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU14_TA_SQC_CTRL_REG                                                                    0x5050
+#define regCGTS_CU14_TA_SQC_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU14_SP1_CTRL_REG                                                                       0x5051
+#define regCGTS_CU14_SP1_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU14_TD_TCP_CTRL_REG                                                                    0x5052
+#define regCGTS_CU14_TD_TCP_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU15_SP0_CTRL_REG                                                                       0x5053
+#define regCGTS_CU15_SP0_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU15_LDS_SQ_CTRL_REG                                                                    0x5054
+#define regCGTS_CU15_LDS_SQ_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU15_TA_SQC_CTRL_REG                                                                    0x5055
+#define regCGTS_CU15_TA_SQC_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU15_SP1_CTRL_REG                                                                       0x5056
+#define regCGTS_CU15_SP1_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU15_TD_TCP_CTRL_REG                                                                    0x5057
+#define regCGTS_CU15_TD_TCP_CTRL_REG_BASE_IDX                                                           1
+#define regCGTS_CU0_TCPI_CTRL_REG                                                                       0x5058
+#define regCGTS_CU0_TCPI_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU1_TCPI_CTRL_REG                                                                       0x5059
+#define regCGTS_CU1_TCPI_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU2_TCPI_CTRL_REG                                                                       0x505a
+#define regCGTS_CU2_TCPI_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU3_TCPI_CTRL_REG                                                                       0x505b
+#define regCGTS_CU3_TCPI_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU4_TCPI_CTRL_REG                                                                       0x505c
+#define regCGTS_CU4_TCPI_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU5_TCPI_CTRL_REG                                                                       0x505d
+#define regCGTS_CU5_TCPI_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU6_TCPI_CTRL_REG                                                                       0x505e
+#define regCGTS_CU6_TCPI_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU7_TCPI_CTRL_REG                                                                       0x505f
+#define regCGTS_CU7_TCPI_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU8_TCPI_CTRL_REG                                                                       0x5060
+#define regCGTS_CU8_TCPI_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU9_TCPI_CTRL_REG                                                                       0x5061
+#define regCGTS_CU9_TCPI_CTRL_REG_BASE_IDX                                                              1
+#define regCGTS_CU10_TCPI_CTRL_REG                                                                      0x5062
+#define regCGTS_CU10_TCPI_CTRL_REG_BASE_IDX                                                             1
+#define regCGTS_CU11_TCPI_CTRL_REG                                                                      0x5063
+#define regCGTS_CU11_TCPI_CTRL_REG_BASE_IDX                                                             1
+#define regCGTS_CU12_TCPI_CTRL_REG                                                                      0x5064
+#define regCGTS_CU12_TCPI_CTRL_REG_BASE_IDX                                                             1
+#define regCGTS_CU13_TCPI_CTRL_REG                                                                      0x5065
+#define regCGTS_CU13_TCPI_CTRL_REG_BASE_IDX                                                             1
+#define regCGTS_CU14_TCPI_CTRL_REG                                                                      0x5066
+#define regCGTS_CU14_TCPI_CTRL_REG_BASE_IDX                                                             1
+#define regCGTS_CU15_TCPI_CTRL_REG                                                                      0x5067
+#define regCGTS_CU15_TCPI_CTRL_REG_BASE_IDX                                                             1
+#define regCGTT_SPI_PS_CLK_CTRL                                                                         0x507d
+#define regCGTT_SPI_PS_CLK_CTRL_BASE_IDX                                                                1
+#define regCGTT_SPIS_CLK_CTRL                                                                           0x507e
+#define regCGTT_SPIS_CLK_CTRL_BASE_IDX                                                                  1
+#define regCGTX_SPI_DEBUG_CLK_CTRL                                                                      0x507f
+#define regCGTX_SPI_DEBUG_CLK_CTRL_BASE_IDX                                                             1
+#define regCGTT_SPI_CLK_CTRL                                                                            0x5080
+#define regCGTT_SPI_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_PC_CLK_CTRL                                                                             0x5081
+#define regCGTT_PC_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_BCI_CLK_CTRL                                                                            0x5082
+#define regCGTT_BCI_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_VGT_CLK_CTRL                                                                            0x5084
+#define regCGTT_VGT_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_IA_CLK_CTRL                                                                             0x5085
+#define regCGTT_IA_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_WD_CLK_CTRL                                                                             0x5086
+#define regCGTT_WD_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_PA_CLK_CTRL                                                                             0x5088
+#define regCGTT_PA_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_SC_CLK_CTRL0                                                                            0x5089
+#define regCGTT_SC_CLK_CTRL0_BASE_IDX                                                                   1
+#define regCGTT_SC_CLK_CTRL1                                                                            0x508a
+#define regCGTT_SC_CLK_CTRL1_BASE_IDX                                                                   1
+#define regCGTT_SC_CLK_CTRL2                                                                            0x508b
+#define regCGTT_SC_CLK_CTRL2_BASE_IDX                                                                   1
+#define regCGTT_SQ_CLK_CTRL                                                                             0x508c
+#define regCGTT_SQ_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_SQG_CLK_CTRL                                                                            0x508d
+#define regCGTT_SQG_CLK_CTRL_BASE_IDX                                                                   1
+#define regSQ_ALU_CLK_CTRL                                                                              0x508e
+#define regSQ_ALU_CLK_CTRL_BASE_IDX                                                                     1
+#define regSQ_TEX_CLK_CTRL                                                                              0x508f
+#define regSQ_TEX_CLK_CTRL_BASE_IDX                                                                     1
+#define regSQ_LDS_CLK_CTRL                                                                              0x5090
+#define regSQ_LDS_CLK_CTRL_BASE_IDX                                                                     1
+#define regSQ_POWER_THROTTLE                                                                            0x5091
+#define regSQ_POWER_THROTTLE_BASE_IDX                                                                   1
+#define regSQ_POWER_THROTTLE2                                                                           0x5092
+#define regSQ_POWER_THROTTLE2_BASE_IDX                                                                  1
+#define regTD_CGTT_CTRL                                                                                 0x509c
+#define regTD_CGTT_CTRL_BASE_IDX                                                                        1
+#define regTA_CGTT_CTRL                                                                                 0x509d
+#define regTA_CGTT_CTRL_BASE_IDX                                                                        1
+#define regCGTT_TCPI_CLK_CTRL                                                                           0x509e
+#define regCGTT_TCPI_CLK_CTRL_BASE_IDX                                                                  1
+#define regTCX_CGTT_SCLK_CTRL                                                                           0x50a3
+#define regTCX_CGTT_SCLK_CTRL_BASE_IDX                                                                  1
+#define regDB_CGTT_CLK_CTRL_0                                                                           0x50a4
+#define regDB_CGTT_CLK_CTRL_0_BASE_IDX                                                                  1
+#define regCB_CGTT_SCLK_CTRL                                                                            0x50a8
+#define regCB_CGTT_SCLK_CTRL_BASE_IDX                                                                   1
+#define regTCC_CGTT_SCLK_CTRL                                                                           0x50ac
+#define regTCC_CGTT_SCLK_CTRL_BASE_IDX                                                                  1
+#define regTCC_CGTT_SCLK_CTRL2                                                                          0x50ad
+#define regTCC_CGTT_SCLK_CTRL2_BASE_IDX                                                                 1
+#define regTCC_CGTT_SCLK_CTRL3                                                                          0x50ae
+#define regTCC_CGTT_SCLK_CTRL3_BASE_IDX                                                                 1
+#define regTCA_CGTT_SCLK_CTRL                                                                           0x50af
+#define regTCA_CGTT_SCLK_CTRL_BASE_IDX                                                                  1
+#define regCGTT_CP_CLK_CTRL                                                                             0x50b0
+#define regCGTT_CP_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_CPC_CLK_CTRL                                                                            0x50b2
+#define regCGTT_CPC_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_RLC_CLK_CTRL                                                                            0x50b5
+#define regCGTT_RLC_CLK_CTRL_BASE_IDX                                                                   1
+#define regRLC_GFX_RM_CNTL                                                                              0x50b6
+#define regRLC_GFX_RM_CNTL_BASE_IDX                                                                     1
+#define regRMI_CGTT_SCLK_CTRL                                                                           0x50c0
+#define regRMI_CGTT_SCLK_CTRL_BASE_IDX                                                                  1
+#define regCGTT_TCPF_CLK_CTRL                                                                           0x50c1
+#define regCGTT_TCPF_CLK_CTRL_BASE_IDX                                                                  1
+
+
+// addressBlock: xcd0_gc_hypdec
+// base address: 0x3e000
+#define regCP_HYP_PFP_UCODE_ADDR                                                                        0x5814
+#define regCP_HYP_PFP_UCODE_ADDR_BASE_IDX                                                               1
+#define regCP_PFP_UCODE_ADDR                                                                            0x5814
+#define regCP_PFP_UCODE_ADDR_BASE_IDX                                                                   1
+#define regCP_HYP_PFP_UCODE_DATA                                                                        0x5815
+#define regCP_HYP_PFP_UCODE_DATA_BASE_IDX                                                               1
+#define regCP_PFP_UCODE_DATA                                                                            0x5815
+#define regCP_PFP_UCODE_DATA_BASE_IDX                                                                   1
+#define regCP_HYP_ME_UCODE_ADDR                                                                         0x5816
+#define regCP_HYP_ME_UCODE_ADDR_BASE_IDX                                                                1
+#define regCP_ME_RAM_RADDR                                                                              0x5816
+#define regCP_ME_RAM_RADDR_BASE_IDX                                                                     1
+#define regCP_ME_RAM_WADDR                                                                              0x5816
+#define regCP_ME_RAM_WADDR_BASE_IDX                                                                     1
+#define regCP_HYP_ME_UCODE_DATA                                                                         0x5817
+#define regCP_HYP_ME_UCODE_DATA_BASE_IDX                                                                1
+#define regCP_ME_RAM_DATA                                                                               0x5817
+#define regCP_ME_RAM_DATA_BASE_IDX                                                                      1
+#define regCP_CE_UCODE_ADDR                                                                             0x5818
+#define regCP_CE_UCODE_ADDR_BASE_IDX                                                                    1
+#define regCP_HYP_CE_UCODE_ADDR                                                                         0x5818
+#define regCP_HYP_CE_UCODE_ADDR_BASE_IDX                                                                1
+#define regCP_CE_UCODE_DATA                                                                             0x5819
+#define regCP_CE_UCODE_DATA_BASE_IDX                                                                    1
+#define regCP_HYP_CE_UCODE_DATA                                                                         0x5819
+#define regCP_HYP_CE_UCODE_DATA_BASE_IDX                                                                1
+#define regCP_HYP_MEC1_UCODE_ADDR                                                                       0x581a
+#define regCP_HYP_MEC1_UCODE_ADDR_BASE_IDX                                                              1
+#define regCP_MEC_ME1_UCODE_ADDR                                                                        0x581a
+#define regCP_MEC_ME1_UCODE_ADDR_BASE_IDX                                                               1
+#define regCP_HYP_MEC1_UCODE_DATA                                                                       0x581b
+#define regCP_HYP_MEC1_UCODE_DATA_BASE_IDX                                                              1
+#define regCP_MEC_ME1_UCODE_DATA                                                                        0x581b
+#define regCP_MEC_ME1_UCODE_DATA_BASE_IDX                                                               1
+#define regCP_HYP_MEC2_UCODE_ADDR                                                                       0x581c
+#define regCP_HYP_MEC2_UCODE_ADDR_BASE_IDX                                                              1
+#define regCP_MEC_ME2_UCODE_ADDR                                                                        0x581c
+#define regCP_MEC_ME2_UCODE_ADDR_BASE_IDX                                                               1
+#define regCP_HYP_MEC2_UCODE_DATA                                                                       0x581d
+#define regCP_HYP_MEC2_UCODE_DATA_BASE_IDX                                                              1
+#define regCP_MEC_ME2_UCODE_DATA                                                                        0x581d
+#define regCP_MEC_ME2_UCODE_DATA_BASE_IDX                                                               1
+#define regCP_HYP_PFP_UCODE_CHKSUM                                                                      0x581e
+#define regCP_HYP_PFP_UCODE_CHKSUM_BASE_IDX                                                             1
+#define regCP_HYP_CE_UCODE_CHKSUM                                                                       0x581f
+#define regCP_HYP_CE_UCODE_CHKSUM_BASE_IDX                                                              1
+#define regCP_HYP_ME_UCODE_CHKSUM                                                                       0x5820
+#define regCP_HYP_ME_UCODE_CHKSUM_BASE_IDX                                                              1
+#define regCP_HYP_MEC_ME1_UCODE_CHKSUM                                                                  0x5821
+#define regCP_HYP_MEC_ME1_UCODE_CHKSUM_BASE_IDX                                                         1
+#define regCP_HYP_MEC_ME2_UCODE_CHKSUM                                                                  0x5822
+#define regCP_HYP_MEC_ME2_UCODE_CHKSUM_BASE_IDX                                                         1
+#define regCP_HYP_XCP_CTL                                                                               0x5828
+#define regCP_HYP_XCP_CTL_BASE_IDX                                                                      1
+#define regRLC_GPM_UCODE_ADDR                                                                           0x583c
+#define regRLC_GPM_UCODE_ADDR_BASE_IDX                                                                  1
+#define regRLC_GPM_UCODE_DATA                                                                           0x583d
+#define regRLC_GPM_UCODE_DATA_BASE_IDX                                                                  1
+#define regGRBM_GFX_INDEX_SR_SELECT                                                                     0x5a00
+#define regGRBM_GFX_INDEX_SR_SELECT_BASE_IDX                                                            1
+#define regGRBM_GFX_INDEX_SR_DATA                                                                       0x5a01
+#define regGRBM_GFX_INDEX_SR_DATA_BASE_IDX                                                              1
+#define regGRBM_GFX_CNTL_SR_SELECT                                                                      0x5a02
+#define regGRBM_GFX_CNTL_SR_SELECT_BASE_IDX                                                             1
+#define regGRBM_GFX_CNTL_SR_DATA                                                                        0x5a03
+#define regGRBM_GFX_CNTL_SR_DATA_BASE_IDX                                                               1
+#define regGRBM_MCM_ADDR                                                                                0x5a07
+#define regGRBM_MCM_ADDR_BASE_IDX                                                                       1
+#define regRLC_GPU_IOV_VF_ENABLE                                                                        0x5b00
+#define regRLC_GPU_IOV_VF_ENABLE_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_CFG_REG6                                                                         0x5b06
+#define regRLC_GPU_IOV_CFG_REG6_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_CFG_REG8                                                                         0x5b20
+#define regRLC_GPU_IOV_CFG_REG8_BASE_IDX                                                                1
+#define regRLC_RLCV_TIMER_INT_0                                                                         0x5b25
+#define regRLC_RLCV_TIMER_INT_0_BASE_IDX                                                                1
+#define regRLC_RLCV_TIMER_CTRL                                                                          0x5b26
+#define regRLC_RLCV_TIMER_CTRL_BASE_IDX                                                                 1
+#define regRLC_RLCV_TIMER_STAT                                                                          0x5b27
+#define regRLC_RLCV_TIMER_STAT_BASE_IDX                                                                 1
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS                                                               0x5b2a
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_BASE_IDX                                                      1
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_SET                                                           0x5b2b
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_SET_BASE_IDX                                                  1
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_CLR                                                           0x5b2c
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_CLR_BASE_IDX                                                  1
+#define regRLC_GPU_IOV_VF_MASK                                                                          0x5b2d
+#define regRLC_GPU_IOV_VF_MASK_BASE_IDX                                                                 1
+#define regRLC_HYP_SEMAPHORE_0                                                                          0x5b2e
+#define regRLC_HYP_SEMAPHORE_0_BASE_IDX                                                                 1
+#define regRLC_HYP_SEMAPHORE_1                                                                          0x5b2f
+#define regRLC_HYP_SEMAPHORE_1_BASE_IDX                                                                 1
+#define regRLC_CLK_CNTL                                                                                 0x5b31
+#define regRLC_CLK_CNTL_BASE_IDX                                                                        1
+#define regRLC_GPU_IOV_SCH_BLOCK                                                                        0x5b34
+#define regRLC_GPU_IOV_SCH_BLOCK_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_CFG_REG1                                                                         0x5b35
+#define regRLC_GPU_IOV_CFG_REG1_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_CFG_REG2                                                                         0x5b36
+#define regRLC_GPU_IOV_CFG_REG2_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_VM_BUSY_STATUS                                                                   0x5b37
+#define regRLC_GPU_IOV_VM_BUSY_STATUS_BASE_IDX                                                          1
+#define regRLC_GPU_IOV_SCH_0                                                                            0x5b38
+#define regRLC_GPU_IOV_SCH_0_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_ACTIVE_FCN_ID                                                                    0x5b39
+#define regRLC_GPU_IOV_ACTIVE_FCN_ID_BASE_IDX                                                           1
+#define regRLC_GPU_IOV_SCH_3                                                                            0x5b3a
+#define regRLC_GPU_IOV_SCH_3_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_SCH_1                                                                            0x5b3b
+#define regRLC_GPU_IOV_SCH_1_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_SCH_2                                                                            0x5b3c
+#define regRLC_GPU_IOV_SCH_2_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_INT_STAT                                                                         0x5b3f
+#define regRLC_GPU_IOV_INT_STAT_BASE_IDX                                                                1
+#define regRLC_RLCV_TIMER_INT_1                                                                         0x5b40
+#define regRLC_RLCV_TIMER_INT_1_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_UCODE_ADDR                                                                       0x5b42
+#define regRLC_GPU_IOV_UCODE_ADDR_BASE_IDX                                                              1
+#define regRLC_GPU_IOV_UCODE_DATA                                                                       0x5b43
+#define regRLC_GPU_IOV_UCODE_DATA_BASE_IDX                                                              1
+#define regRLC_GPU_IOV_SCRATCH_ADDR                                                                     0x5b44
+#define regRLC_GPU_IOV_SCRATCH_ADDR_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SCRATCH_DATA                                                                     0x5b45
+#define regRLC_GPU_IOV_SCRATCH_DATA_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_F32_CNTL                                                                         0x5b46
+#define regRLC_GPU_IOV_F32_CNTL_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_F32_RESET                                                                        0x5b47
+#define regRLC_GPU_IOV_F32_RESET_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_SDMA0_STATUS                                                                     0x5b48
+#define regRLC_GPU_IOV_SDMA0_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA1_STATUS                                                                     0x5b49
+#define regRLC_GPU_IOV_SDMA1_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SMU_RESPONSE                                                                     0x5b4a
+#define regRLC_GPU_IOV_SMU_RESPONSE_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_VIRT_RESET_REQ                                                                   0x5b4c
+#define regRLC_GPU_IOV_VIRT_RESET_REQ_BASE_IDX                                                          1
+#define regRLC_GPU_IOV_RLC_RESPONSE                                                                     0x5b4d
+#define regRLC_GPU_IOV_RLC_RESPONSE_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_INT_DISABLE                                                                      0x5b4e
+#define regRLC_GPU_IOV_INT_DISABLE_BASE_IDX                                                             1
+#define regRLC_GPU_IOV_INT_FORCE                                                                        0x5b4f
+#define regRLC_GPU_IOV_INT_FORCE_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_SDMA0_BUSY_STATUS                                                                0x5b50
+#define regRLC_GPU_IOV_SDMA0_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA1_BUSY_STATUS                                                                0x5b51
+#define regRLC_GPU_IOV_SDMA1_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_HYP_SEMAPHORE_2                                                                          0x5b52
+#define regRLC_HYP_SEMAPHORE_2_BASE_IDX                                                                 1
+#define regRLC_HYP_SEMAPHORE_3                                                                          0x5b53
+#define regRLC_HYP_SEMAPHORE_3_BASE_IDX                                                                 1
+#define regRLC_GPU_IOV_SDMA2_STATUS                                                                     0x5b54
+#define regRLC_GPU_IOV_SDMA2_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA3_STATUS                                                                     0x5b55
+#define regRLC_GPU_IOV_SDMA3_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA4_STATUS                                                                     0x5b56
+#define regRLC_GPU_IOV_SDMA4_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA5_STATUS                                                                     0x5b57
+#define regRLC_GPU_IOV_SDMA5_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA6_STATUS                                                                     0x5b58
+#define regRLC_GPU_IOV_SDMA6_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA7_STATUS                                                                     0x5b59
+#define regRLC_GPU_IOV_SDMA7_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA2_BUSY_STATUS                                                                0x5b5a
+#define regRLC_GPU_IOV_SDMA2_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA3_BUSY_STATUS                                                                0x5b5b
+#define regRLC_GPU_IOV_SDMA3_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA4_BUSY_STATUS                                                                0x5b5c
+#define regRLC_GPU_IOV_SDMA4_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA5_BUSY_STATUS                                                                0x5b5d
+#define regRLC_GPU_IOV_SDMA5_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA6_BUSY_STATUS                                                                0x5b5e
+#define regRLC_GPU_IOV_SDMA6_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA7_BUSY_STATUS                                                                0x5b5f
+#define regRLC_GPU_IOV_SDMA7_BUSY_STATUS_BASE_IDX                                                       1
+
+
+// addressBlock: xcd0_gc_utcl2_vmsharedhvdec
+// base address: 0x3ea00
+#define regMC_VM_FB_SIZE_OFFSET_VF0                                                                     0x5a80
+#define regMC_VM_FB_SIZE_OFFSET_VF0_BASE_IDX                                                            1
+#define regMC_VM_FB_SIZE_OFFSET_VF1                                                                     0x5a81
+#define regMC_VM_FB_SIZE_OFFSET_VF1_BASE_IDX                                                            1
+#define regMC_VM_FB_SIZE_OFFSET_VF2                                                                     0x5a82
+#define regMC_VM_FB_SIZE_OFFSET_VF2_BASE_IDX                                                            1
+#define regMC_VM_FB_SIZE_OFFSET_VF3                                                                     0x5a83
+#define regMC_VM_FB_SIZE_OFFSET_VF3_BASE_IDX                                                            1
+#define regMC_VM_FB_SIZE_OFFSET_VF4                                                                     0x5a84
+#define regMC_VM_FB_SIZE_OFFSET_VF4_BASE_IDX                                                            1
+#define regMC_VM_FB_SIZE_OFFSET_VF5                                                                     0x5a85
+#define regMC_VM_FB_SIZE_OFFSET_VF5_BASE_IDX                                                            1
+#define regMC_VM_FB_SIZE_OFFSET_VF6                                                                     0x5a86
+#define regMC_VM_FB_SIZE_OFFSET_VF6_BASE_IDX                                                            1
+#define regMC_VM_FB_SIZE_OFFSET_VF7                                                                     0x5a87
+#define regMC_VM_FB_SIZE_OFFSET_VF7_BASE_IDX                                                            1
+#define regMC_VM_FB_SIZE_OFFSET_VF8                                                                     0x5a88
+#define regMC_VM_FB_SIZE_OFFSET_VF8_BASE_IDX                                                            1
+#define regMC_VM_FB_SIZE_OFFSET_VF9                                                                     0x5a89
+#define regMC_VM_FB_SIZE_OFFSET_VF9_BASE_IDX                                                            1
+#define regMC_VM_FB_SIZE_OFFSET_VF10                                                                    0x5a8a
+#define regMC_VM_FB_SIZE_OFFSET_VF10_BASE_IDX                                                           1
+#define regMC_VM_FB_SIZE_OFFSET_VF11                                                                    0x5a8b
+#define regMC_VM_FB_SIZE_OFFSET_VF11_BASE_IDX                                                           1
+#define regMC_VM_FB_SIZE_OFFSET_VF12                                                                    0x5a8c
+#define regMC_VM_FB_SIZE_OFFSET_VF12_BASE_IDX                                                           1
+#define regMC_VM_FB_SIZE_OFFSET_VF13                                                                    0x5a8d
+#define regMC_VM_FB_SIZE_OFFSET_VF13_BASE_IDX                                                           1
+#define regMC_VM_FB_SIZE_OFFSET_VF14                                                                    0x5a8e
+#define regMC_VM_FB_SIZE_OFFSET_VF14_BASE_IDX                                                           1
+#define regMC_VM_FB_SIZE_OFFSET_VF15                                                                    0x5a8f
+#define regMC_VM_FB_SIZE_OFFSET_VF15_BASE_IDX                                                           1
+#define regVM_IOMMU_MMIO_CNTRL_1                                                                        0x5a90
+#define regVM_IOMMU_MMIO_CNTRL_1_BASE_IDX                                                               1
+#define regMC_VM_MARC_BASE_LO_0                                                                         0x5a91
+#define regMC_VM_MARC_BASE_LO_0_BASE_IDX                                                                1
+#define regMC_VM_MARC_BASE_LO_1                                                                         0x5a92
+#define regMC_VM_MARC_BASE_LO_1_BASE_IDX                                                                1
+#define regMC_VM_MARC_BASE_LO_2                                                                         0x5a93
+#define regMC_VM_MARC_BASE_LO_2_BASE_IDX                                                                1
+#define regMC_VM_MARC_BASE_LO_3                                                                         0x5a94
+#define regMC_VM_MARC_BASE_LO_3_BASE_IDX                                                                1
+#define regMC_VM_MARC_BASE_HI_0                                                                         0x5a95
+#define regMC_VM_MARC_BASE_HI_0_BASE_IDX                                                                1
+#define regMC_VM_MARC_BASE_HI_1                                                                         0x5a96
+#define regMC_VM_MARC_BASE_HI_1_BASE_IDX                                                                1
+#define regMC_VM_MARC_BASE_HI_2                                                                         0x5a97
+#define regMC_VM_MARC_BASE_HI_2_BASE_IDX                                                                1
+#define regMC_VM_MARC_BASE_HI_3                                                                         0x5a98
+#define regMC_VM_MARC_BASE_HI_3_BASE_IDX                                                                1
+#define regMC_VM_MARC_RELOC_LO_0                                                                        0x5a99
+#define regMC_VM_MARC_RELOC_LO_0_BASE_IDX                                                               1
+#define regMC_VM_MARC_RELOC_LO_1                                                                        0x5a9a
+#define regMC_VM_MARC_RELOC_LO_1_BASE_IDX                                                               1
+#define regMC_VM_MARC_RELOC_LO_2                                                                        0x5a9b
+#define regMC_VM_MARC_RELOC_LO_2_BASE_IDX                                                               1
+#define regMC_VM_MARC_RELOC_LO_3                                                                        0x5a9c
+#define regMC_VM_MARC_RELOC_LO_3_BASE_IDX                                                               1
+#define regMC_VM_MARC_RELOC_HI_0                                                                        0x5a9d
+#define regMC_VM_MARC_RELOC_HI_0_BASE_IDX                                                               1
+#define regMC_VM_MARC_RELOC_HI_1                                                                        0x5a9e
+#define regMC_VM_MARC_RELOC_HI_1_BASE_IDX                                                               1
+#define regMC_VM_MARC_RELOC_HI_2                                                                        0x5a9f
+#define regMC_VM_MARC_RELOC_HI_2_BASE_IDX                                                               1
+#define regMC_VM_MARC_RELOC_HI_3                                                                        0x5aa0
+#define regMC_VM_MARC_RELOC_HI_3_BASE_IDX                                                               1
+#define regMC_VM_MARC_LEN_LO_0                                                                          0x5aa1
+#define regMC_VM_MARC_LEN_LO_0_BASE_IDX                                                                 1
+#define regMC_VM_MARC_LEN_LO_1                                                                          0x5aa2
+#define regMC_VM_MARC_LEN_LO_1_BASE_IDX                                                                 1
+#define regMC_VM_MARC_LEN_LO_2                                                                          0x5aa3
+#define regMC_VM_MARC_LEN_LO_2_BASE_IDX                                                                 1
+#define regMC_VM_MARC_LEN_LO_3                                                                          0x5aa4
+#define regMC_VM_MARC_LEN_LO_3_BASE_IDX                                                                 1
+#define regMC_VM_MARC_LEN_HI_0                                                                          0x5aa5
+#define regMC_VM_MARC_LEN_HI_0_BASE_IDX                                                                 1
+#define regMC_VM_MARC_LEN_HI_1                                                                          0x5aa6
+#define regMC_VM_MARC_LEN_HI_1_BASE_IDX                                                                 1
+#define regMC_VM_MARC_LEN_HI_2                                                                          0x5aa7
+#define regMC_VM_MARC_LEN_HI_2_BASE_IDX                                                                 1
+#define regMC_VM_MARC_LEN_HI_3                                                                          0x5aa8
+#define regMC_VM_MARC_LEN_HI_3_BASE_IDX                                                                 1
+#define regVM_IOMMU_CONTROL_REGISTER                                                                    0x5aa9
+#define regVM_IOMMU_CONTROL_REGISTER_BASE_IDX                                                           1
+#define regVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER                                           0x5aaa
+#define regVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER_BASE_IDX                                  1
+#define regVM_PCIE_ATS_CNTL                                                                             0x5aab
+#define regVM_PCIE_ATS_CNTL_BASE_IDX                                                                    1
+#define regVM_PCIE_ATS_CNTL_VF_0                                                                        0x5aac
+#define regVM_PCIE_ATS_CNTL_VF_0_BASE_IDX                                                               1
+#define regVM_PCIE_ATS_CNTL_VF_1                                                                        0x5aad
+#define regVM_PCIE_ATS_CNTL_VF_1_BASE_IDX                                                               1
+#define regVM_PCIE_ATS_CNTL_VF_2                                                                        0x5aae
+#define regVM_PCIE_ATS_CNTL_VF_2_BASE_IDX                                                               1
+#define regVM_PCIE_ATS_CNTL_VF_3                                                                        0x5aaf
+#define regVM_PCIE_ATS_CNTL_VF_3_BASE_IDX                                                               1
+#define regVM_PCIE_ATS_CNTL_VF_4                                                                        0x5ab0
+#define regVM_PCIE_ATS_CNTL_VF_4_BASE_IDX                                                               1
+#define regVM_PCIE_ATS_CNTL_VF_5                                                                        0x5ab1
+#define regVM_PCIE_ATS_CNTL_VF_5_BASE_IDX                                                               1
+#define regVM_PCIE_ATS_CNTL_VF_6                                                                        0x5ab2
+#define regVM_PCIE_ATS_CNTL_VF_6_BASE_IDX                                                               1
+#define regVM_PCIE_ATS_CNTL_VF_7                                                                        0x5ab3
+#define regVM_PCIE_ATS_CNTL_VF_7_BASE_IDX                                                               1
+#define regVM_PCIE_ATS_CNTL_VF_8                                                                        0x5ab4
+#define regVM_PCIE_ATS_CNTL_VF_8_BASE_IDX                                                               1
+#define regVM_PCIE_ATS_CNTL_VF_9                                                                        0x5ab5
+#define regVM_PCIE_ATS_CNTL_VF_9_BASE_IDX                                                               1
+#define regVM_PCIE_ATS_CNTL_VF_10                                                                       0x5ab6
+#define regVM_PCIE_ATS_CNTL_VF_10_BASE_IDX                                                              1
+#define regVM_PCIE_ATS_CNTL_VF_11                                                                       0x5ab7
+#define regVM_PCIE_ATS_CNTL_VF_11_BASE_IDX                                                              1
+#define regVM_PCIE_ATS_CNTL_VF_12                                                                       0x5ab8
+#define regVM_PCIE_ATS_CNTL_VF_12_BASE_IDX                                                              1
+#define regVM_PCIE_ATS_CNTL_VF_13                                                                       0x5ab9
+#define regVM_PCIE_ATS_CNTL_VF_13_BASE_IDX                                                              1
+#define regVM_PCIE_ATS_CNTL_VF_14                                                                       0x5aba
+#define regVM_PCIE_ATS_CNTL_VF_14_BASE_IDX                                                              1
+#define regVM_PCIE_ATS_CNTL_VF_15                                                                       0x5abb
+#define regVM_PCIE_ATS_CNTL_VF_15_BASE_IDX                                                              1
+#define regMC_SHARED_ACTIVE_FCN_ID                                                                      0x5abc
+#define regMC_SHARED_ACTIVE_FCN_ID_BASE_IDX                                                             1
+#define regMC_VM_XGMI_GPUIOV_ENABLE                                                                     0x5abd
+#define regMC_VM_XGMI_GPUIOV_ENABLE_BASE_IDX                                                            1
+
+
+// addressBlock: xcd0_gc_pspdec
+// base address: 0x3f000
+#define regCPG_PSP_DEBUG                                                                                0x5c30
+#define regCPG_PSP_DEBUG_BASE_IDX                                                                       1
+#define regCPC_PSP_DEBUG                                                                                0x5c31
+#define regCPC_PSP_DEBUG_BASE_IDX                                                                       1
+#define regCP_PSP_XCP_CTL                                                                               0x5c34
+#define regCP_PSP_XCP_CTL_BASE_IDX                                                                      1
+#define regGRBM_SEC_CNTL                                                                                0x5e0b
+#define regGRBM_SEC_CNTL_BASE_IDX                                                                       1
+#define regGRBM_IOV_ERROR_FIFO_DATA                                                                     0x5e12
+#define regGRBM_IOV_ERROR_FIFO_DATA_BASE_IDX                                                            1
+#define regGRBM_DSM_BYPASS                                                                              0x5e13
+#define regGRBM_DSM_BYPASS_BASE_IDX                                                                     1
+#define regGRBM_CAM_INDEX                                                                               0x5e16
+#define regGRBM_CAM_INDEX_BASE_IDX                                                                      1
+#define regGRBM_HYP_CAM_INDEX                                                                           0x5e16
+#define regGRBM_HYP_CAM_INDEX_BASE_IDX                                                                  1
+#define regGRBM_CAM_DATA                                                                                0x5e17
+#define regGRBM_CAM_DATA_BASE_IDX                                                                       1
+#define regGRBM_HYP_CAM_DATA                                                                            0x5e17
+#define regGRBM_HYP_CAM_DATA_BASE_IDX                                                                   1
+#define regRLC_FWL_FIRST_VIOL_ADDR                                                                      0x5f37
+#define regRLC_FWL_FIRST_VIOL_ADDR_BASE_IDX                                                             1
+
+
+// addressBlock: sqind
+// base address: 0x0
+#define ixSQ_DEBUG_STS_LOCAL                                                                           0x0008
+#define ixSQ_DEBUG_CTRL_LOCAL                                                                          0x0009
+#define ixSQ_WAVE_VALID_AND_IDLE                                                                       0x000a
+#define ixSQ_WAVE_MODE                                                                                 0x0011
+#define ixSQ_WAVE_STATUS                                                                               0x0012
+#define ixSQ_WAVE_TRAPSTS                                                                              0x0013
+#define ixSQ_WAVE_HW_ID                                                                                0x0014
+#define ixSQ_WAVE_GPR_ALLOC                                                                            0x0015
+#define ixSQ_WAVE_LDS_ALLOC                                                                            0x0016
+#define ixSQ_WAVE_IB_STS                                                                               0x0017
+#define ixSQ_WAVE_PC_LO                                                                                0x0018
+#define ixSQ_WAVE_PC_HI                                                                                0x0019
+#define ixSQ_WAVE_INST_DW0                                                                             0x001a
+#define ixSQ_WAVE_INST_DW1                                                                             0x001b
+#define ixSQ_WAVE_IB_DBG0                                                                              0x001c
+#define ixSQ_WAVE_IB_DBG1                                                                              0x001d
+#define ixSQ_WAVE_FLUSH_IB                                                                             0x001e
+#define ixSQ_WAVE_TTMP0                                                                                0x026c
+#define ixSQ_WAVE_TTMP1                                                                                0x026d
+#define ixSQ_WAVE_TTMP2                                                                                0x026e
+#define ixSQ_WAVE_TTMP3                                                                                0x026f
+#define ixSQ_WAVE_TTMP4                                                                                0x0270
+#define ixSQ_WAVE_TTMP5                                                                                0x0271
+#define ixSQ_WAVE_TTMP6                                                                                0x0272
+#define ixSQ_WAVE_TTMP7                                                                                0x0273
+#define ixSQ_WAVE_TTMP8                                                                                0x0274
+#define ixSQ_WAVE_TTMP9                                                                                0x0275
+#define ixSQ_WAVE_TTMP10                                                                               0x0276
+#define ixSQ_WAVE_TTMP11                                                                               0x0277
+#define ixSQ_WAVE_TTMP12                                                                               0x0278
+#define ixSQ_WAVE_TTMP13                                                                               0x0279
+#define ixSQ_WAVE_TTMP14                                                                               0x027a
+#define ixSQ_WAVE_TTMP15                                                                               0x027b
+#define ixSQ_WAVE_M0                                                                                   0x027c
+#define ixSQ_WAVE_EXEC_LO                                                                              0x027e
+#define ixSQ_WAVE_EXEC_HI                                                                              0x027f
+#define ixSQ_INTERRUPT_WORD_AUTO_CTXID                                                                 0x20c0
+#define ixSQ_INTERRUPT_WORD_AUTO_HI                                                                    0x20c0
+#define ixSQ_INTERRUPT_WORD_AUTO_LO                                                                    0x20c0
+#define ixSQ_INTERRUPT_WORD_CMN_CTXID                                                                  0x20c0
+#define ixSQ_INTERRUPT_WORD_CMN_HI                                                                     0x20c0
+#define ixSQ_INTERRUPT_WORD_WAVE_CTXID                                                                 0x20c0
+#define ixSQ_INTERRUPT_WORD_WAVE_HI                                                                    0x20c0
+#define ixSQ_INTERRUPT_WORD_WAVE_LO                                                                    0x20c0
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_3_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_3_sh_mask.h
new file mode 100644
index 000000000000..84a75b58347f
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_3_sh_mask.h
@@ -0,0 +1,30535 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _gc_9_4_3_SH_MASK_HEADER
+#define _gc_9_4_3_SH_MASK_HEADER
+
+
+// addressBlock: xcd0_gc_grbmdec
+//GRBM_CNTL
+#define GRBM_CNTL__READ_TIMEOUT__SHIFT                                                                        0x0
+#define GRBM_CNTL__REPORT_LAST_RDERR__SHIFT                                                                   0x1f
+#define GRBM_CNTL__READ_TIMEOUT_MASK                                                                          0x000000FFL
+#define GRBM_CNTL__REPORT_LAST_RDERR_MASK                                                                     0x80000000L
+//GRBM_SKEW_CNTL
+#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD__SHIFT                                                             0x0
+#define GRBM_SKEW_CNTL__SKEW_COUNT__SHIFT                                                                     0x6
+#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD_MASK                                                               0x0000003FL
+#define GRBM_SKEW_CNTL__SKEW_COUNT_MASK                                                                       0x00000FC0L
+//GRBM_STATUS2
+#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL__SHIFT                                                           0x0
+#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING__SHIFT                                                           0x4
+#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING__SHIFT                                                           0x5
+#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING__SHIFT                                                              0x6
+#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING__SHIFT                                                              0x7
+#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING__SHIFT                                                              0x8
+#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING__SHIFT                                                              0x9
+#define GRBM_STATUS2__ME2PIPE0_RQ_PENDING__SHIFT                                                              0xa
+#define GRBM_STATUS2__ME2PIPE1_RQ_PENDING__SHIFT                                                              0xb
+#define GRBM_STATUS2__ME2PIPE2_RQ_PENDING__SHIFT                                                              0xc
+#define GRBM_STATUS2__ME2PIPE3_RQ_PENDING__SHIFT                                                              0xd
+#define GRBM_STATUS2__RLC_RQ_PENDING__SHIFT                                                                   0xe
+#define GRBM_STATUS2__UTCL2_BUSY__SHIFT                                                                       0xf
+#define GRBM_STATUS2__EA_BUSY__SHIFT                                                                          0x10
+#define GRBM_STATUS2__RMI_BUSY__SHIFT                                                                         0x11
+#define GRBM_STATUS2__UTCL2_RQ_PENDING__SHIFT                                                                 0x12
+#define GRBM_STATUS2__CPF_RQ_PENDING__SHIFT                                                                   0x13
+#define GRBM_STATUS2__EA_LINK_BUSY__SHIFT                                                                     0x14
+#define GRBM_STATUS2__CANE_BUSY__SHIFT                                                                        0x15
+#define GRBM_STATUS2__CANE_LINK_BUSY__SHIFT                                                                   0x16
+#define GRBM_STATUS2__RLC_BUSY__SHIFT                                                                         0x18
+#define GRBM_STATUS2__TC_BUSY__SHIFT                                                                          0x19
+#define GRBM_STATUS2__TCC_CC_RESIDENT__SHIFT                                                                  0x1a
+#define GRBM_STATUS2__CPF_BUSY__SHIFT                                                                         0x1c
+#define GRBM_STATUS2__CPC_BUSY__SHIFT                                                                         0x1d
+#define GRBM_STATUS2__CPG_BUSY__SHIFT                                                                         0x1e
+#define GRBM_STATUS2__CPAXI_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL_MASK                                                             0x0000000FL
+#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING_MASK                                                             0x00000010L
+#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING_MASK                                                             0x00000020L
+#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING_MASK                                                                0x00000040L
+#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING_MASK                                                                0x00000080L
+#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING_MASK                                                                0x00000100L
+#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING_MASK                                                                0x00000200L
+#define GRBM_STATUS2__ME2PIPE0_RQ_PENDING_MASK                                                                0x00000400L
+#define GRBM_STATUS2__ME2PIPE1_RQ_PENDING_MASK                                                                0x00000800L
+#define GRBM_STATUS2__ME2PIPE2_RQ_PENDING_MASK                                                                0x00001000L
+#define GRBM_STATUS2__ME2PIPE3_RQ_PENDING_MASK                                                                0x00002000L
+#define GRBM_STATUS2__RLC_RQ_PENDING_MASK                                                                     0x00004000L
+#define GRBM_STATUS2__UTCL2_BUSY_MASK                                                                         0x00008000L
+#define GRBM_STATUS2__EA_BUSY_MASK                                                                            0x00010000L
+#define GRBM_STATUS2__RMI_BUSY_MASK                                                                           0x00020000L
+#define GRBM_STATUS2__UTCL2_RQ_PENDING_MASK                                                                   0x00040000L
+#define GRBM_STATUS2__CPF_RQ_PENDING_MASK                                                                     0x00080000L
+#define GRBM_STATUS2__EA_LINK_BUSY_MASK                                                                       0x00100000L
+#define GRBM_STATUS2__CANE_BUSY_MASK                                                                          0x00200000L
+#define GRBM_STATUS2__CANE_LINK_BUSY_MASK                                                                     0x00400000L
+#define GRBM_STATUS2__RLC_BUSY_MASK                                                                           0x01000000L
+#define GRBM_STATUS2__TC_BUSY_MASK                                                                            0x02000000L
+#define GRBM_STATUS2__TCC_CC_RESIDENT_MASK                                                                    0x04000000L
+#define GRBM_STATUS2__CPF_BUSY_MASK                                                                           0x10000000L
+#define GRBM_STATUS2__CPC_BUSY_MASK                                                                           0x20000000L
+#define GRBM_STATUS2__CPG_BUSY_MASK                                                                           0x40000000L
+#define GRBM_STATUS2__CPAXI_BUSY_MASK                                                                         0x80000000L
+//GRBM_PWR_CNTL
+#define GRBM_PWR_CNTL__ALL_REQ_TYPE__SHIFT                                                                    0x0
+#define GRBM_PWR_CNTL__GFX_REQ_TYPE__SHIFT                                                                    0x2
+#define GRBM_PWR_CNTL__ALL_RSP_TYPE__SHIFT                                                                    0x4
+#define GRBM_PWR_CNTL__GFX_RSP_TYPE__SHIFT                                                                    0x6
+#define GRBM_PWR_CNTL__GFX_REQ_EN__SHIFT                                                                      0xe
+#define GRBM_PWR_CNTL__ALL_REQ_EN__SHIFT                                                                      0xf
+#define GRBM_PWR_CNTL__ALL_REQ_TYPE_MASK                                                                      0x00000003L
+#define GRBM_PWR_CNTL__GFX_REQ_TYPE_MASK                                                                      0x0000000CL
+#define GRBM_PWR_CNTL__ALL_RSP_TYPE_MASK                                                                      0x00000030L
+#define GRBM_PWR_CNTL__GFX_RSP_TYPE_MASK                                                                      0x000000C0L
+#define GRBM_PWR_CNTL__GFX_REQ_EN_MASK                                                                        0x00004000L
+#define GRBM_PWR_CNTL__ALL_REQ_EN_MASK                                                                        0x00008000L
+//GRBM_STATUS
+#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL__SHIFT                                                            0x0
+#define GRBM_STATUS__RSMU_RQ_PENDING__SHIFT                                                                   0x5
+#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING__SHIFT                                                            0x7
+#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING__SHIFT                                                            0x8
+#define GRBM_STATUS__GDS_DMA_RQ_PENDING__SHIFT                                                                0x9
+#define GRBM_STATUS__DB_CLEAN__SHIFT                                                                          0xc
+#define GRBM_STATUS__CB_CLEAN__SHIFT                                                                          0xd
+#define GRBM_STATUS__TA_BUSY__SHIFT                                                                           0xe
+#define GRBM_STATUS__GDS_BUSY__SHIFT                                                                          0xf
+#define GRBM_STATUS__WD_BUSY_NO_DMA__SHIFT                                                                    0x10
+#define GRBM_STATUS__VGT_BUSY__SHIFT                                                                          0x11
+#define GRBM_STATUS__IA_BUSY_NO_DMA__SHIFT                                                                    0x12
+#define GRBM_STATUS__IA_BUSY__SHIFT                                                                           0x13
+#define GRBM_STATUS__SX_BUSY__SHIFT                                                                           0x14
+#define GRBM_STATUS__WD_BUSY__SHIFT                                                                           0x15
+#define GRBM_STATUS__SPI_BUSY__SHIFT                                                                          0x16
+#define GRBM_STATUS__BCI_BUSY__SHIFT                                                                          0x17
+#define GRBM_STATUS__SC_BUSY__SHIFT                                                                           0x18
+#define GRBM_STATUS__PA_BUSY__SHIFT                                                                           0x19
+#define GRBM_STATUS__DB_BUSY__SHIFT                                                                           0x1a
+#define GRBM_STATUS__CP_COHERENCY_BUSY__SHIFT                                                                 0x1c
+#define GRBM_STATUS__CP_BUSY__SHIFT                                                                           0x1d
+#define GRBM_STATUS__CB_BUSY__SHIFT                                                                           0x1e
+#define GRBM_STATUS__GUI_ACTIVE__SHIFT                                                                        0x1f
+#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL_MASK                                                              0x0000000FL
+#define GRBM_STATUS__RSMU_RQ_PENDING_MASK                                                                     0x00000020L
+#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING_MASK                                                              0x00000080L
+#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING_MASK                                                              0x00000100L
+#define GRBM_STATUS__GDS_DMA_RQ_PENDING_MASK                                                                  0x00000200L
+#define GRBM_STATUS__DB_CLEAN_MASK                                                                            0x00001000L
+#define GRBM_STATUS__CB_CLEAN_MASK                                                                            0x00002000L
+#define GRBM_STATUS__TA_BUSY_MASK                                                                             0x00004000L
+#define GRBM_STATUS__GDS_BUSY_MASK                                                                            0x00008000L
+#define GRBM_STATUS__WD_BUSY_NO_DMA_MASK                                                                      0x00010000L
+#define GRBM_STATUS__VGT_BUSY_MASK                                                                            0x00020000L
+#define GRBM_STATUS__IA_BUSY_NO_DMA_MASK                                                                      0x00040000L
+#define GRBM_STATUS__IA_BUSY_MASK                                                                             0x00080000L
+#define GRBM_STATUS__SX_BUSY_MASK                                                                             0x00100000L
+#define GRBM_STATUS__WD_BUSY_MASK                                                                             0x00200000L
+#define GRBM_STATUS__SPI_BUSY_MASK                                                                            0x00400000L
+#define GRBM_STATUS__BCI_BUSY_MASK                                                                            0x00800000L
+#define GRBM_STATUS__SC_BUSY_MASK                                                                             0x01000000L
+#define GRBM_STATUS__PA_BUSY_MASK                                                                             0x02000000L
+#define GRBM_STATUS__DB_BUSY_MASK                                                                             0x04000000L
+#define GRBM_STATUS__CP_COHERENCY_BUSY_MASK                                                                   0x10000000L
+#define GRBM_STATUS__CP_BUSY_MASK                                                                             0x20000000L
+#define GRBM_STATUS__CB_BUSY_MASK                                                                             0x40000000L
+#define GRBM_STATUS__GUI_ACTIVE_MASK                                                                          0x80000000L
+//GRBM_STATUS_SE0
+#define GRBM_STATUS_SE0__DB_CLEAN__SHIFT                                                                      0x1
+#define GRBM_STATUS_SE0__CB_CLEAN__SHIFT                                                                      0x2
+#define GRBM_STATUS_SE0__RMI_BUSY__SHIFT                                                                      0x15
+#define GRBM_STATUS_SE0__BCI_BUSY__SHIFT                                                                      0x16
+#define GRBM_STATUS_SE0__VGT_BUSY__SHIFT                                                                      0x17
+#define GRBM_STATUS_SE0__PA_BUSY__SHIFT                                                                       0x18
+#define GRBM_STATUS_SE0__TA_BUSY__SHIFT                                                                       0x19
+#define GRBM_STATUS_SE0__SX_BUSY__SHIFT                                                                       0x1a
+#define GRBM_STATUS_SE0__SPI_BUSY__SHIFT                                                                      0x1b
+#define GRBM_STATUS_SE0__SC_BUSY__SHIFT                                                                       0x1d
+#define GRBM_STATUS_SE0__DB_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS_SE0__CB_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS_SE0__DB_CLEAN_MASK                                                                        0x00000002L
+#define GRBM_STATUS_SE0__CB_CLEAN_MASK                                                                        0x00000004L
+#define GRBM_STATUS_SE0__RMI_BUSY_MASK                                                                        0x00200000L
+#define GRBM_STATUS_SE0__BCI_BUSY_MASK                                                                        0x00400000L
+#define GRBM_STATUS_SE0__VGT_BUSY_MASK                                                                        0x00800000L
+#define GRBM_STATUS_SE0__PA_BUSY_MASK                                                                         0x01000000L
+#define GRBM_STATUS_SE0__TA_BUSY_MASK                                                                         0x02000000L
+#define GRBM_STATUS_SE0__SX_BUSY_MASK                                                                         0x04000000L
+#define GRBM_STATUS_SE0__SPI_BUSY_MASK                                                                        0x08000000L
+#define GRBM_STATUS_SE0__SC_BUSY_MASK                                                                         0x20000000L
+#define GRBM_STATUS_SE0__DB_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS_SE0__CB_BUSY_MASK                                                                         0x80000000L
+//GRBM_STATUS_SE1
+#define GRBM_STATUS_SE1__DB_CLEAN__SHIFT                                                                      0x1
+#define GRBM_STATUS_SE1__CB_CLEAN__SHIFT                                                                      0x2
+#define GRBM_STATUS_SE1__RMI_BUSY__SHIFT                                                                      0x15
+#define GRBM_STATUS_SE1__BCI_BUSY__SHIFT                                                                      0x16
+#define GRBM_STATUS_SE1__VGT_BUSY__SHIFT                                                                      0x17
+#define GRBM_STATUS_SE1__PA_BUSY__SHIFT                                                                       0x18
+#define GRBM_STATUS_SE1__TA_BUSY__SHIFT                                                                       0x19
+#define GRBM_STATUS_SE1__SX_BUSY__SHIFT                                                                       0x1a
+#define GRBM_STATUS_SE1__SPI_BUSY__SHIFT                                                                      0x1b
+#define GRBM_STATUS_SE1__SC_BUSY__SHIFT                                                                       0x1d
+#define GRBM_STATUS_SE1__DB_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS_SE1__CB_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS_SE1__DB_CLEAN_MASK                                                                        0x00000002L
+#define GRBM_STATUS_SE1__CB_CLEAN_MASK                                                                        0x00000004L
+#define GRBM_STATUS_SE1__RMI_BUSY_MASK                                                                        0x00200000L
+#define GRBM_STATUS_SE1__BCI_BUSY_MASK                                                                        0x00400000L
+#define GRBM_STATUS_SE1__VGT_BUSY_MASK                                                                        0x00800000L
+#define GRBM_STATUS_SE1__PA_BUSY_MASK                                                                         0x01000000L
+#define GRBM_STATUS_SE1__TA_BUSY_MASK                                                                         0x02000000L
+#define GRBM_STATUS_SE1__SX_BUSY_MASK                                                                         0x04000000L
+#define GRBM_STATUS_SE1__SPI_BUSY_MASK                                                                        0x08000000L
+#define GRBM_STATUS_SE1__SC_BUSY_MASK                                                                         0x20000000L
+#define GRBM_STATUS_SE1__DB_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS_SE1__CB_BUSY_MASK                                                                         0x80000000L
+//GRBM_SOFT_RESET
+#define GRBM_SOFT_RESET__SOFT_RESET_CP__SHIFT                                                                 0x0
+#define GRBM_SOFT_RESET__SOFT_RESET_RLC__SHIFT                                                                0x2
+#define GRBM_SOFT_RESET__SOFT_RESET_GFX__SHIFT                                                                0x10
+#define GRBM_SOFT_RESET__SOFT_RESET_CPF__SHIFT                                                                0x11
+#define GRBM_SOFT_RESET__SOFT_RESET_CPC__SHIFT                                                                0x12
+#define GRBM_SOFT_RESET__SOFT_RESET_CPG__SHIFT                                                                0x13
+#define GRBM_SOFT_RESET__SOFT_RESET_CAC__SHIFT                                                                0x14
+#define GRBM_SOFT_RESET__SOFT_RESET_CANE__SHIFT                                                               0x15
+#define GRBM_SOFT_RESET__SOFT_RESET_EA__SHIFT                                                                 0x16
+#define GRBM_SOFT_RESET__SOFT_RESET_UTCL2__SHIFT                                                              0x17
+#define GRBM_SOFT_RESET__SOFT_RESET_CP_MASK                                                                   0x00000001L
+#define GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK                                                                  0x00000004L
+#define GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK                                                                  0x00010000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPF_MASK                                                                  0x00020000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPC_MASK                                                                  0x00040000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPG_MASK                                                                  0x00080000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CAC_MASK                                                                  0x00100000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CANE_MASK                                                                 0x00200000L
+#define GRBM_SOFT_RESET__SOFT_RESET_EA_MASK                                                                   0x00400000L
+#define GRBM_SOFT_RESET__SOFT_RESET_UTCL2_MASK                                                                0x00800000L
+//GRBM_GFX_CLKEN_CNTL
+#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT                                                          0x0
+#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT__SHIFT                                                            0x8
+#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK                                                            0x0000000FL
+#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT_MASK                                                              0x00001F00L
+//GRBM_WAIT_IDLE_CLOCKS
+#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS__SHIFT                                                        0x0
+#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS_MASK                                                          0x000000FFL
+//GRBM_STATUS_SE2
+#define GRBM_STATUS_SE2__DB_CLEAN__SHIFT                                                                      0x1
+#define GRBM_STATUS_SE2__CB_CLEAN__SHIFT                                                                      0x2
+#define GRBM_STATUS_SE2__RMI_BUSY__SHIFT                                                                      0x15
+#define GRBM_STATUS_SE2__BCI_BUSY__SHIFT                                                                      0x16
+#define GRBM_STATUS_SE2__VGT_BUSY__SHIFT                                                                      0x17
+#define GRBM_STATUS_SE2__PA_BUSY__SHIFT                                                                       0x18
+#define GRBM_STATUS_SE2__TA_BUSY__SHIFT                                                                       0x19
+#define GRBM_STATUS_SE2__SX_BUSY__SHIFT                                                                       0x1a
+#define GRBM_STATUS_SE2__SPI_BUSY__SHIFT                                                                      0x1b
+#define GRBM_STATUS_SE2__SC_BUSY__SHIFT                                                                       0x1d
+#define GRBM_STATUS_SE2__DB_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS_SE2__CB_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS_SE2__DB_CLEAN_MASK                                                                        0x00000002L
+#define GRBM_STATUS_SE2__CB_CLEAN_MASK                                                                        0x00000004L
+#define GRBM_STATUS_SE2__RMI_BUSY_MASK                                                                        0x00200000L
+#define GRBM_STATUS_SE2__BCI_BUSY_MASK                                                                        0x00400000L
+#define GRBM_STATUS_SE2__VGT_BUSY_MASK                                                                        0x00800000L
+#define GRBM_STATUS_SE2__PA_BUSY_MASK                                                                         0x01000000L
+#define GRBM_STATUS_SE2__TA_BUSY_MASK                                                                         0x02000000L
+#define GRBM_STATUS_SE2__SX_BUSY_MASK                                                                         0x04000000L
+#define GRBM_STATUS_SE2__SPI_BUSY_MASK                                                                        0x08000000L
+#define GRBM_STATUS_SE2__SC_BUSY_MASK                                                                         0x20000000L
+#define GRBM_STATUS_SE2__DB_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS_SE2__CB_BUSY_MASK                                                                         0x80000000L
+//GRBM_STATUS_SE3
+#define GRBM_STATUS_SE3__DB_CLEAN__SHIFT                                                                      0x1
+#define GRBM_STATUS_SE3__CB_CLEAN__SHIFT                                                                      0x2
+#define GRBM_STATUS_SE3__RMI_BUSY__SHIFT                                                                      0x15
+#define GRBM_STATUS_SE3__BCI_BUSY__SHIFT                                                                      0x16
+#define GRBM_STATUS_SE3__VGT_BUSY__SHIFT                                                                      0x17
+#define GRBM_STATUS_SE3__PA_BUSY__SHIFT                                                                       0x18
+#define GRBM_STATUS_SE3__TA_BUSY__SHIFT                                                                       0x19
+#define GRBM_STATUS_SE3__SX_BUSY__SHIFT                                                                       0x1a
+#define GRBM_STATUS_SE3__SPI_BUSY__SHIFT                                                                      0x1b
+#define GRBM_STATUS_SE3__SC_BUSY__SHIFT                                                                       0x1d
+#define GRBM_STATUS_SE3__DB_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS_SE3__CB_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS_SE3__DB_CLEAN_MASK                                                                        0x00000002L
+#define GRBM_STATUS_SE3__CB_CLEAN_MASK                                                                        0x00000004L
+#define GRBM_STATUS_SE3__RMI_BUSY_MASK                                                                        0x00200000L
+#define GRBM_STATUS_SE3__BCI_BUSY_MASK                                                                        0x00400000L
+#define GRBM_STATUS_SE3__VGT_BUSY_MASK                                                                        0x00800000L
+#define GRBM_STATUS_SE3__PA_BUSY_MASK                                                                         0x01000000L
+#define GRBM_STATUS_SE3__TA_BUSY_MASK                                                                         0x02000000L
+#define GRBM_STATUS_SE3__SX_BUSY_MASK                                                                         0x04000000L
+#define GRBM_STATUS_SE3__SPI_BUSY_MASK                                                                        0x08000000L
+#define GRBM_STATUS_SE3__SC_BUSY_MASK                                                                         0x20000000L
+#define GRBM_STATUS_SE3__DB_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS_SE3__CB_BUSY_MASK                                                                         0x80000000L
+//GRBM_READ_ERROR
+#define GRBM_READ_ERROR__READ_ADDRESS__SHIFT                                                                  0x2
+#define GRBM_READ_ERROR__READ_PIPEID__SHIFT                                                                   0x14
+#define GRBM_READ_ERROR__READ_MEID__SHIFT                                                                     0x16
+#define GRBM_READ_ERROR__READ_ERROR__SHIFT                                                                    0x1f
+#define GRBM_READ_ERROR__READ_ADDRESS_MASK                                                                    0x0003FFFCL
+#define GRBM_READ_ERROR__READ_PIPEID_MASK                                                                     0x00300000L
+#define GRBM_READ_ERROR__READ_MEID_MASK                                                                       0x00C00000L
+#define GRBM_READ_ERROR__READ_ERROR_MASK                                                                      0x80000000L
+//GRBM_READ_ERROR2
+#define GRBM_READ_ERROR2__READ_REQUESTER_CPF__SHIFT                                                           0x10
+#define GRBM_READ_ERROR2__READ_REQUESTER_RSMU__SHIFT                                                          0x11
+#define GRBM_READ_ERROR2__READ_REQUESTER_RLC__SHIFT                                                           0x12
+#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA__SHIFT                                                       0x13
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF__SHIFT                                                   0x14
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF__SHIFT                                                   0x15
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF__SHIFT                                                   0x16
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF__SHIFT                                                   0x17
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0__SHIFT                                                      0x18
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1__SHIFT                                                      0x19
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2__SHIFT                                                      0x1a
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3__SHIFT                                                      0x1b
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0__SHIFT                                                      0x1c
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1__SHIFT                                                      0x1d
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2__SHIFT                                                      0x1e
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3__SHIFT                                                      0x1f
+#define GRBM_READ_ERROR2__READ_REQUESTER_CPF_MASK                                                             0x00010000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_RSMU_MASK                                                            0x00020000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_RLC_MASK                                                             0x00040000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA_MASK                                                         0x00080000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF_MASK                                                     0x00100000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF_MASK                                                     0x00200000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF_MASK                                                     0x00400000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF_MASK                                                     0x00800000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0_MASK                                                        0x01000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1_MASK                                                        0x02000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2_MASK                                                        0x04000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3_MASK                                                        0x08000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0_MASK                                                        0x10000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1_MASK                                                        0x20000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2_MASK                                                        0x40000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3_MASK                                                        0x80000000L
+//GRBM_INT_CNTL
+#define GRBM_INT_CNTL__RDERR_INT_ENABLE__SHIFT                                                                0x0
+#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE__SHIFT                                                             0x13
+#define GRBM_INT_CNTL__RDERR_INT_ENABLE_MASK                                                                  0x00000001L
+#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE_MASK                                                               0x00080000L
+//GRBM_TRAP_OP
+#define GRBM_TRAP_OP__RW__SHIFT                                                                               0x0
+#define GRBM_TRAP_OP__RW_MASK                                                                                 0x00000001L
+//GRBM_TRAP_ADDR
+#define GRBM_TRAP_ADDR__DATA__SHIFT                                                                           0x0
+#define GRBM_TRAP_ADDR__DATA_MASK                                                                             0x0003FFFFL
+//GRBM_TRAP_ADDR_MSK
+#define GRBM_TRAP_ADDR_MSK__DATA__SHIFT                                                                       0x0
+#define GRBM_TRAP_ADDR_MSK__DATA_MASK                                                                         0x0003FFFFL
+//GRBM_TRAP_WD
+#define GRBM_TRAP_WD__DATA__SHIFT                                                                             0x0
+#define GRBM_TRAP_WD__DATA_MASK                                                                               0xFFFFFFFFL
+//GRBM_TRAP_WD_MSK
+#define GRBM_TRAP_WD_MSK__DATA__SHIFT                                                                         0x0
+#define GRBM_TRAP_WD_MSK__DATA_MASK                                                                           0xFFFFFFFFL
+//GRBM_WRITE_ERROR
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RLC__SHIFT                                                          0x0
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RSMU__SHIFT                                                         0x1
+#define GRBM_WRITE_ERROR__WRITE_SSRCID__SHIFT                                                                 0x2
+#define GRBM_WRITE_ERROR__WRITE_VFID__SHIFT                                                                   0x5
+#define GRBM_WRITE_ERROR__WRITE_VF__SHIFT                                                                     0xc
+#define GRBM_WRITE_ERROR__WRITE_VMID__SHIFT                                                                   0xd
+#define GRBM_WRITE_ERROR__TMZ__SHIFT                                                                          0x11
+#define GRBM_WRITE_ERROR__CP_SECURE_WR_ILLEGAL__SHIFT                                                         0x12
+#define GRBM_WRITE_ERROR__WRITE_PIPEID__SHIFT                                                                 0x14
+#define GRBM_WRITE_ERROR__WRITE_MEID__SHIFT                                                                   0x16
+#define GRBM_WRITE_ERROR__WRITE_ERROR__SHIFT                                                                  0x1f
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RLC_MASK                                                            0x00000001L
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RSMU_MASK                                                           0x00000002L
+#define GRBM_WRITE_ERROR__WRITE_SSRCID_MASK                                                                   0x0000001CL
+#define GRBM_WRITE_ERROR__WRITE_VFID_MASK                                                                     0x000001E0L
+#define GRBM_WRITE_ERROR__WRITE_VF_MASK                                                                       0x00001000L
+#define GRBM_WRITE_ERROR__WRITE_VMID_MASK                                                                     0x0001E000L
+#define GRBM_WRITE_ERROR__TMZ_MASK                                                                            0x00020000L
+#define GRBM_WRITE_ERROR__CP_SECURE_WR_ILLEGAL_MASK                                                           0x00040000L
+#define GRBM_WRITE_ERROR__WRITE_PIPEID_MASK                                                                   0x00300000L
+#define GRBM_WRITE_ERROR__WRITE_MEID_MASK                                                                     0x00C00000L
+#define GRBM_WRITE_ERROR__WRITE_ERROR_MASK                                                                    0x80000000L
+//GRBM_IOV_ERROR
+#define GRBM_IOV_ERROR__IOV_ADDR__SHIFT                                                                       0x2
+#define GRBM_IOV_ERROR__IOV_VFID__SHIFT                                                                       0x14
+#define GRBM_IOV_ERROR__IOV_VF__SHIFT                                                                         0x1a
+#define GRBM_IOV_ERROR__IOV_OP__SHIFT                                                                         0x1b
+#define GRBM_IOV_ERROR__IOV_ERROR__SHIFT                                                                      0x1f
+#define GRBM_IOV_ERROR__IOV_ADDR_MASK                                                                         0x000FFFFCL
+#define GRBM_IOV_ERROR__IOV_VFID_MASK                                                                         0x03F00000L
+#define GRBM_IOV_ERROR__IOV_VF_MASK                                                                           0x04000000L
+#define GRBM_IOV_ERROR__IOV_OP_MASK                                                                           0x08000000L
+#define GRBM_IOV_ERROR__IOV_ERROR_MASK                                                                        0x80000000L
+//GRBM_CHIP_REVISION
+#define GRBM_CHIP_REVISION__CHIP_REVISION__SHIFT                                                              0x0
+#define GRBM_CHIP_REVISION__CHIP_REVISION_MASK                                                                0x000000FFL
+//GRBM_GFX_CNTL
+#define GRBM_GFX_CNTL__PIPEID__SHIFT                                                                          0x0
+#define GRBM_GFX_CNTL__MEID__SHIFT                                                                            0x2
+#define GRBM_GFX_CNTL__VMID__SHIFT                                                                            0x4
+#define GRBM_GFX_CNTL__QUEUEID__SHIFT                                                                         0x8
+#define GRBM_GFX_CNTL__PIPEID_MASK                                                                            0x00000003L
+#define GRBM_GFX_CNTL__MEID_MASK                                                                              0x0000000CL
+#define GRBM_GFX_CNTL__VMID_MASK                                                                              0x000000F0L
+#define GRBM_GFX_CNTL__QUEUEID_MASK                                                                           0x00000700L
+//GRBM_RSMU_CFG
+#define GRBM_RSMU_CFG__APERTURE_ID__SHIFT                                                                     0x0
+#define GRBM_RSMU_CFG__QOS__SHIFT                                                                             0xc
+#define GRBM_RSMU_CFG__POSTED_WR__SHIFT                                                                       0x10
+#define GRBM_RSMU_CFG__DEBUG_MASK__SHIFT                                                                      0x11
+#define GRBM_RSMU_CFG__APERTURE_ID_MASK                                                                       0x00000FFFL
+#define GRBM_RSMU_CFG__QOS_MASK                                                                               0x0000F000L
+#define GRBM_RSMU_CFG__POSTED_WR_MASK                                                                         0x00010000L
+#define GRBM_RSMU_CFG__DEBUG_MASK_MASK                                                                        0x00020000L
+//GRBM_IH_CREDIT
+#define GRBM_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                   0x0
+#define GRBM_IH_CREDIT__IH_CLIENT_ID__SHIFT                                                                   0x10
+#define GRBM_IH_CREDIT__CREDIT_VALUE_MASK                                                                     0x00000003L
+#define GRBM_IH_CREDIT__IH_CLIENT_ID_MASK                                                                     0x00FF0000L
+//GRBM_PWR_CNTL2
+#define GRBM_PWR_CNTL2__PWR_REQUEST_HALT__SHIFT                                                               0x10
+#define GRBM_PWR_CNTL2__PWR_GFX3D_REQUEST_HALT__SHIFT                                                         0x14
+#define GRBM_PWR_CNTL2__PWR_REQUEST_HALT_MASK                                                                 0x00010000L
+#define GRBM_PWR_CNTL2__PWR_GFX3D_REQUEST_HALT_MASK                                                           0x00100000L
+//GRBM_UTCL2_INVAL_RANGE_START
+#define GRBM_UTCL2_INVAL_RANGE_START__DATA__SHIFT                                                             0x0
+#define GRBM_UTCL2_INVAL_RANGE_START__DATA_MASK                                                               0x0003FFFFL
+//GRBM_UTCL2_INVAL_RANGE_END
+#define GRBM_UTCL2_INVAL_RANGE_END__DATA__SHIFT                                                               0x0
+#define GRBM_UTCL2_INVAL_RANGE_END__DATA_MASK                                                                 0x0003FFFFL
+//GRBM_RSMU_READ_ERROR
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ADDRESS__SHIFT                                                        0x2
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VF__SHIFT                                                             0x14
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VFID__SHIFT                                                           0x15
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_TYPE__SHIFT                                                     0x1b
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR__SHIFT                                                          0x1f
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ADDRESS_MASK                                                          0x000FFFFCL
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VF_MASK                                                               0x00100000L
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VFID_MASK                                                             0x07E00000L
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_TYPE_MASK                                                       0x08000000L
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_MASK                                                            0x80000000L
+//GRBM_CHICKEN_BITS
+#define GRBM_CHICKEN_BITS__DISABLE_CP_VMID_RESET_REQ__SHIFT                                                   0x0
+#define GRBM_CHICKEN_BITS__DISABLE_CP_VMID_RESET_REQ_MASK                                                     0x00000001L
+//GRBM_FENCE_RANGE0
+#define GRBM_FENCE_RANGE0__START__SHIFT                                                                       0x0
+#define GRBM_FENCE_RANGE0__END__SHIFT                                                                         0x10
+#define GRBM_FENCE_RANGE0__START_MASK                                                                         0x0000FFFFL
+#define GRBM_FENCE_RANGE0__END_MASK                                                                           0xFFFF0000L
+//GRBM_FENCE_RANGE1
+#define GRBM_FENCE_RANGE1__START__SHIFT                                                                       0x0
+#define GRBM_FENCE_RANGE1__END__SHIFT                                                                         0x10
+#define GRBM_FENCE_RANGE1__START_MASK                                                                         0x0000FFFFL
+#define GRBM_FENCE_RANGE1__END_MASK                                                                           0xFFFF0000L
+//GRBM_IOV_READ_ERROR
+#define GRBM_IOV_READ_ERROR__IOV_ADDR__SHIFT                                                                  0x2
+#define GRBM_IOV_READ_ERROR__IOV_VFID__SHIFT                                                                  0x14
+#define GRBM_IOV_READ_ERROR__IOV_VF__SHIFT                                                                    0x1a
+#define GRBM_IOV_READ_ERROR__IOV_OP__SHIFT                                                                    0x1b
+#define GRBM_IOV_READ_ERROR__IOV_ERROR__SHIFT                                                                 0x1f
+#define GRBM_IOV_READ_ERROR__IOV_ADDR_MASK                                                                    0x000FFFFCL
+#define GRBM_IOV_READ_ERROR__IOV_VFID_MASK                                                                    0x03F00000L
+#define GRBM_IOV_READ_ERROR__IOV_VF_MASK                                                                      0x04000000L
+#define GRBM_IOV_READ_ERROR__IOV_OP_MASK                                                                      0x08000000L
+#define GRBM_IOV_READ_ERROR__IOV_ERROR_MASK                                                                   0x80000000L
+//GRBM_NOWHERE
+#define GRBM_NOWHERE__DATA__SHIFT                                                                             0x0
+#define GRBM_NOWHERE__DATA_MASK                                                                               0xFFFFFFFFL
+//GRBM_SCRATCH_REG0
+#define GRBM_SCRATCH_REG0__SCRATCH_REG0__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG0__SCRATCH_REG0_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG1
+#define GRBM_SCRATCH_REG1__SCRATCH_REG1__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG1__SCRATCH_REG1_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG2
+#define GRBM_SCRATCH_REG2__SCRATCH_REG2__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG2__SCRATCH_REG2_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG3
+#define GRBM_SCRATCH_REG3__SCRATCH_REG3__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG3__SCRATCH_REG3_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG4
+#define GRBM_SCRATCH_REG4__SCRATCH_REG4__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG4__SCRATCH_REG4_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG5
+#define GRBM_SCRATCH_REG5__SCRATCH_REG5__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG5__SCRATCH_REG5_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG6
+#define GRBM_SCRATCH_REG6__SCRATCH_REG6__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG6__SCRATCH_REG6_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG7
+#define GRBM_SCRATCH_REG7__SCRATCH_REG7__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG7__SCRATCH_REG7_MASK                                                                  0xFFFFFFFFL
+//VIOLATION_DATA_ASYNC_VF_PROG
+#define VIOLATION_DATA_ASYNC_VF_PROG__SSRCID__SHIFT                                                           0x0
+#define VIOLATION_DATA_ASYNC_VF_PROG__VFID__SHIFT                                                             0x4
+#define VIOLATION_DATA_ASYNC_VF_PROG__VIOLATION_ERROR__SHIFT                                                  0x1f
+#define VIOLATION_DATA_ASYNC_VF_PROG__SSRCID_MASK                                                             0x0000000FL
+#define VIOLATION_DATA_ASYNC_VF_PROG__VFID_MASK                                                               0x000003F0L
+#define VIOLATION_DATA_ASYNC_VF_PROG__VIOLATION_ERROR_MASK                                                    0x80000000L
+
+
+// addressBlock: xcd0_gc_cpdec
+//CP_CPC_DEBUG_CNTL
+#define CP_CPC_DEBUG_CNTL__DEBUG_INDX__SHIFT                                                                  0x0
+#define CP_CPC_DEBUG_CNTL__DEBUG_BUS_DC_GD_SEL__SHIFT                                                         0x8
+#define CP_CPC_DEBUG_CNTL__DEBUG_BUS_SELECT_BITS__SHIFT                                                       0x10
+#define CP_CPC_DEBUG_CNTL__DEBUG_BUS_FLOP_EN__SHIFT                                                           0x1f
+#define CP_CPC_DEBUG_CNTL__DEBUG_INDX_MASK                                                                    0x0000007FL
+#define CP_CPC_DEBUG_CNTL__DEBUG_BUS_DC_GD_SEL_MASK                                                           0x00000700L
+#define CP_CPC_DEBUG_CNTL__DEBUG_BUS_SELECT_BITS_MASK                                                         0x003F0000L
+#define CP_CPC_DEBUG_CNTL__DEBUG_BUS_FLOP_EN_MASK                                                             0x80000000L
+//CP_CPF_DEBUG_CNTL
+#define CP_CPF_DEBUG_CNTL__DEBUG_INDX__SHIFT                                                                  0x0
+#define CP_CPF_DEBUG_CNTL__DEBUG_BUS_SELECT_BITS__SHIFT                                                       0x10
+#define CP_CPF_DEBUG_CNTL__DEBUG_BUS_FLOP_EN__SHIFT                                                           0x1f
+#define CP_CPF_DEBUG_CNTL__DEBUG_INDX_MASK                                                                    0x0000007FL
+#define CP_CPF_DEBUG_CNTL__DEBUG_BUS_SELECT_BITS_MASK                                                         0x003F0000L
+#define CP_CPF_DEBUG_CNTL__DEBUG_BUS_FLOP_EN_MASK                                                             0x80000000L
+//CP_CPC_STATUS
+#define CP_CPC_STATUS__MEC1_BUSY__SHIFT                                                                       0x0
+#define CP_CPC_STATUS__MEC2_BUSY__SHIFT                                                                       0x1
+#define CP_CPC_STATUS__DC0_BUSY__SHIFT                                                                        0x2
+#define CP_CPC_STATUS__DC1_BUSY__SHIFT                                                                        0x3
+#define CP_CPC_STATUS__RCIU1_BUSY__SHIFT                                                                      0x4
+#define CP_CPC_STATUS__RCIU2_BUSY__SHIFT                                                                      0x5
+#define CP_CPC_STATUS__ROQ1_BUSY__SHIFT                                                                       0x6
+#define CP_CPC_STATUS__ROQ2_BUSY__SHIFT                                                                       0x7
+#define CP_CPC_STATUS__TCIU_BUSY__SHIFT                                                                       0xa
+#define CP_CPC_STATUS__SCRATCH_RAM_BUSY__SHIFT                                                                0xb
+#define CP_CPC_STATUS__QU_BUSY__SHIFT                                                                         0xc
+#define CP_CPC_STATUS__UTCL2IU_BUSY__SHIFT                                                                    0xd
+#define CP_CPC_STATUS__SAVE_RESTORE_BUSY__SHIFT                                                               0xe
+#define CP_CPC_STATUS__CPG_CPC_BUSY__SHIFT                                                                    0x1d
+#define CP_CPC_STATUS__CPF_CPC_BUSY__SHIFT                                                                    0x1e
+#define CP_CPC_STATUS__CPC_BUSY__SHIFT                                                                        0x1f
+#define CP_CPC_STATUS__MEC1_BUSY_MASK                                                                         0x00000001L
+#define CP_CPC_STATUS__MEC2_BUSY_MASK                                                                         0x00000002L
+#define CP_CPC_STATUS__DC0_BUSY_MASK                                                                          0x00000004L
+#define CP_CPC_STATUS__DC1_BUSY_MASK                                                                          0x00000008L
+#define CP_CPC_STATUS__RCIU1_BUSY_MASK                                                                        0x00000010L
+#define CP_CPC_STATUS__RCIU2_BUSY_MASK                                                                        0x00000020L
+#define CP_CPC_STATUS__ROQ1_BUSY_MASK                                                                         0x00000040L
+#define CP_CPC_STATUS__ROQ2_BUSY_MASK                                                                         0x00000080L
+#define CP_CPC_STATUS__TCIU_BUSY_MASK                                                                         0x00000400L
+#define CP_CPC_STATUS__SCRATCH_RAM_BUSY_MASK                                                                  0x00000800L
+#define CP_CPC_STATUS__QU_BUSY_MASK                                                                           0x00001000L
+#define CP_CPC_STATUS__UTCL2IU_BUSY_MASK                                                                      0x00002000L
+#define CP_CPC_STATUS__SAVE_RESTORE_BUSY_MASK                                                                 0x00004000L
+#define CP_CPC_STATUS__CPG_CPC_BUSY_MASK                                                                      0x20000000L
+#define CP_CPC_STATUS__CPF_CPC_BUSY_MASK                                                                      0x40000000L
+#define CP_CPC_STATUS__CPC_BUSY_MASK                                                                          0x80000000L
+//CP_CPC_BUSY_STAT
+#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY__SHIFT                                                               0x0
+#define CP_CPC_BUSY_STAT__MEC1_SEMAPOHRE_BUSY__SHIFT                                                          0x1
+#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY__SHIFT                                                              0x2
+#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY__SHIFT                                                            0x3
+#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY__SHIFT                                                          0x4
+#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY__SHIFT                                                           0x5
+#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY__SHIFT                                                           0x6
+#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY__SHIFT                                                                 0x7
+#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY__SHIFT                                                                0x8
+#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY__SHIFT                                                      0x9
+#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY__SHIFT                                                              0xa
+#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY__SHIFT                                                              0xb
+#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY__SHIFT                                                              0xc
+#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY__SHIFT                                                              0xd
+#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY__SHIFT                                                               0x10
+#define CP_CPC_BUSY_STAT__MEC2_SEMAPOHRE_BUSY__SHIFT                                                          0x11
+#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY__SHIFT                                                              0x12
+#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY__SHIFT                                                            0x13
+#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY__SHIFT                                                          0x14
+#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY__SHIFT                                                           0x15
+#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY__SHIFT                                                           0x16
+#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY__SHIFT                                                                 0x17
+#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY__SHIFT                                                                0x18
+#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY__SHIFT                                                      0x19
+#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY__SHIFT                                                              0x1a
+#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY__SHIFT                                                              0x1b
+#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY__SHIFT                                                              0x1c
+#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY__SHIFT                                                              0x1d
+#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY_MASK                                                                 0x00000001L
+#define CP_CPC_BUSY_STAT__MEC1_SEMAPOHRE_BUSY_MASK                                                            0x00000002L
+#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY_MASK                                                                0x00000004L
+#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY_MASK                                                              0x00000008L
+#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY_MASK                                                            0x00000010L
+#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY_MASK                                                             0x00000020L
+#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY_MASK                                                             0x00000040L
+#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY_MASK                                                                   0x00000080L
+#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY_MASK                                                                  0x00000100L
+#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY_MASK                                                        0x00000200L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY_MASK                                                                0x00000400L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY_MASK                                                                0x00000800L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY_MASK                                                                0x00001000L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY_MASK                                                                0x00002000L
+#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY_MASK                                                                 0x00010000L
+#define CP_CPC_BUSY_STAT__MEC2_SEMAPOHRE_BUSY_MASK                                                            0x00020000L
+#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY_MASK                                                                0x00040000L
+#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY_MASK                                                              0x00080000L
+#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY_MASK                                                            0x00100000L
+#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY_MASK                                                             0x00200000L
+#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY_MASK                                                             0x00400000L
+#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY_MASK                                                                   0x00800000L
+#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY_MASK                                                                  0x01000000L
+#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY_MASK                                                        0x02000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY_MASK                                                                0x04000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY_MASK                                                                0x08000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY_MASK                                                                0x10000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY_MASK                                                                0x20000000L
+//CP_CPC_STALLED_STAT1
+#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL__SHIFT                                                       0x3
+#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION__SHIFT                                                      0x4
+#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL__SHIFT                                                       0x6
+#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET__SHIFT                                                     0x8
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU__SHIFT                                                        0x9
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ__SHIFT                                                   0xa
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA__SHIFT                                                    0xd
+#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET__SHIFT                                                     0x10
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU__SHIFT                                                        0x11
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ__SHIFT                                                   0x12
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA__SHIFT                                                    0x15
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE__SHIFT                                                  0x16
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                  0x17
+#define CP_CPC_STALLED_STAT1__UTCL1_WAITING_ON_TRANS__SHIFT                                                   0x18
+#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL_MASK                                                         0x00000008L
+#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION_MASK                                                        0x00000010L
+#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL_MASK                                                         0x00000040L
+#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET_MASK                                                       0x00000100L
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_MASK                                                          0x00000200L
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ_MASK                                                     0x00000400L
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA_MASK                                                      0x00002000L
+#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET_MASK                                                       0x00010000L
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_MASK                                                          0x00020000L
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ_MASK                                                     0x00040000L
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA_MASK                                                      0x00200000L
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE_MASK                                                    0x00400000L
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS_MASK                                                    0x00800000L
+#define CP_CPC_STALLED_STAT1__UTCL1_WAITING_ON_TRANS_MASK                                                     0x01000000L
+//CP_CPF_STATUS
+#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY__SHIFT                                                              0x0
+#define CP_CPF_STATUS__CSF_BUSY__SHIFT                                                                        0x1
+#define CP_CPF_STATUS__ROQ_ALIGN_BUSY__SHIFT                                                                  0x4
+#define CP_CPF_STATUS__ROQ_RING_BUSY__SHIFT                                                                   0x5
+#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY__SHIFT                                                              0x6
+#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY__SHIFT                                                              0x7
+#define CP_CPF_STATUS__ROQ_STATE_BUSY__SHIFT                                                                  0x8
+#define CP_CPF_STATUS__ROQ_CE_RING_BUSY__SHIFT                                                                0x9
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY__SHIFT                                                           0xa
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY__SHIFT                                                           0xb
+#define CP_CPF_STATUS__SEMAPHORE_BUSY__SHIFT                                                                  0xc
+#define CP_CPF_STATUS__INTERRUPT_BUSY__SHIFT                                                                  0xd
+#define CP_CPF_STATUS__TCIU_BUSY__SHIFT                                                                       0xe
+#define CP_CPF_STATUS__HQD_BUSY__SHIFT                                                                        0xf
+#define CP_CPF_STATUS__PRT_BUSY__SHIFT                                                                        0x10
+#define CP_CPF_STATUS__UTCL2IU_BUSY__SHIFT                                                                    0x11
+#define CP_CPF_STATUS__CPF_GFX_BUSY__SHIFT                                                                    0x1a
+#define CP_CPF_STATUS__CPF_CMP_BUSY__SHIFT                                                                    0x1b
+#define CP_CPF_STATUS__GRBM_CPF_STAT_BUSY__SHIFT                                                              0x1c
+#define CP_CPF_STATUS__CPC_CPF_BUSY__SHIFT                                                                    0x1e
+#define CP_CPF_STATUS__CPF_BUSY__SHIFT                                                                        0x1f
+#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY_MASK                                                                0x00000001L
+#define CP_CPF_STATUS__CSF_BUSY_MASK                                                                          0x00000002L
+#define CP_CPF_STATUS__ROQ_ALIGN_BUSY_MASK                                                                    0x00000010L
+#define CP_CPF_STATUS__ROQ_RING_BUSY_MASK                                                                     0x00000020L
+#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY_MASK                                                                0x00000040L
+#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY_MASK                                                                0x00000080L
+#define CP_CPF_STATUS__ROQ_STATE_BUSY_MASK                                                                    0x00000100L
+#define CP_CPF_STATUS__ROQ_CE_RING_BUSY_MASK                                                                  0x00000200L
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY_MASK                                                             0x00000400L
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY_MASK                                                             0x00000800L
+#define CP_CPF_STATUS__SEMAPHORE_BUSY_MASK                                                                    0x00001000L
+#define CP_CPF_STATUS__INTERRUPT_BUSY_MASK                                                                    0x00002000L
+#define CP_CPF_STATUS__TCIU_BUSY_MASK                                                                         0x00004000L
+#define CP_CPF_STATUS__HQD_BUSY_MASK                                                                          0x00008000L
+#define CP_CPF_STATUS__PRT_BUSY_MASK                                                                          0x00010000L
+#define CP_CPF_STATUS__UTCL2IU_BUSY_MASK                                                                      0x00020000L
+#define CP_CPF_STATUS__CPF_GFX_BUSY_MASK                                                                      0x04000000L
+#define CP_CPF_STATUS__CPF_CMP_BUSY_MASK                                                                      0x08000000L
+#define CP_CPF_STATUS__GRBM_CPF_STAT_BUSY_MASK                                                                0x30000000L
+#define CP_CPF_STATUS__CPC_CPF_BUSY_MASK                                                                      0x40000000L
+#define CP_CPF_STATUS__CPF_BUSY_MASK                                                                          0x80000000L
+//CP_CPF_BUSY_STAT
+#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT                                                            0x0
+#define CP_CPF_BUSY_STAT__CSF_RING_BUSY__SHIFT                                                                0x1
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY__SHIFT                                                           0x2
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY__SHIFT                                                           0x3
+#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY__SHIFT                                                               0x4
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY__SHIFT                                                            0x5
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY__SHIFT                                                            0x6
+#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY__SHIFT                                                             0x7
+#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY__SHIFT                                                               0x8
+#define CP_CPF_BUSY_STAT__OUTSTANDING_READ_TAGS__SHIFT                                                        0x9
+#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY__SHIFT                                                      0xb
+#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY__SHIFT                                                            0xc
+#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY__SHIFT                                                            0xd
+#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY__SHIFT                                                         0xe
+#define CP_CPF_BUSY_STAT__HQD_WAIT_SEMAPHORE_BUSY__SHIFT                                                      0xf
+#define CP_CPF_BUSY_STAT__HQD_SIGNAL_SEMAPHORE_BUSY__SHIFT                                                    0x10
+#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY__SHIFT                                                             0x11
+#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY__SHIFT                                                          0x12
+#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY__SHIFT                                                          0x13
+#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY__SHIFT                                                          0x14
+#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY__SHIFT                                                         0x15
+#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY__SHIFT                                                       0x16
+#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY__SHIFT                                                         0x17
+#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY__SHIFT                                                           0x18
+#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY__SHIFT                                                             0x19
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY__SHIFT                                                              0x1a
+#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY__SHIFT                                                              0x1b
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY__SHIFT                                                              0x1c
+#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY__SHIFT                                                           0x1d
+#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY__SHIFT                                                                  0x1e
+#define CP_CPF_BUSY_STAT__HQD_IB_BUSY__SHIFT                                                                  0x1f
+#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK                                                              0x00000001L
+#define CP_CPF_BUSY_STAT__CSF_RING_BUSY_MASK                                                                  0x00000002L
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY_MASK                                                             0x00000004L
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY_MASK                                                             0x00000008L
+#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY_MASK                                                                 0x00000010L
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY_MASK                                                              0x00000020L
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY_MASK                                                              0x00000040L
+#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY_MASK                                                               0x00000080L
+#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY_MASK                                                                 0x00000100L
+#define CP_CPF_BUSY_STAT__OUTSTANDING_READ_TAGS_MASK                                                          0x00000200L
+#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY_MASK                                                        0x00000800L
+#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY_MASK                                                              0x00001000L
+#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY_MASK                                                              0x00002000L
+#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY_MASK                                                           0x00004000L
+#define CP_CPF_BUSY_STAT__HQD_WAIT_SEMAPHORE_BUSY_MASK                                                        0x00008000L
+#define CP_CPF_BUSY_STAT__HQD_SIGNAL_SEMAPHORE_BUSY_MASK                                                      0x00010000L
+#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY_MASK                                                               0x00020000L
+#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY_MASK                                                            0x00040000L
+#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY_MASK                                                            0x00080000L
+#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY_MASK                                                            0x00100000L
+#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY_MASK                                                           0x00200000L
+#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY_MASK                                                         0x00400000L
+#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY_MASK                                                           0x00800000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY_MASK                                                             0x01000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY_MASK                                                               0x02000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY_MASK                                                                0x04000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY_MASK                                                                0x08000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY_MASK                                                                0x10000000L
+#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY_MASK                                                             0x20000000L
+#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY_MASK                                                                    0x40000000L
+#define CP_CPF_BUSY_STAT__HQD_IB_BUSY_MASK                                                                    0x80000000L
+//CP_CPF_STALLED_STAT1
+#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA__SHIFT                                                       0x0
+#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA__SHIFT                                                      0x1
+#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA__SHIFT                                                      0x2
+#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA__SHIFT                                                      0x3
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE__SHIFT                                                     0x5
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS__SHIFT                                                     0x6
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE__SHIFT                                                  0x7
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                  0x8
+#define CP_CPF_STALLED_STAT1__GFX_UTCL1_WAITING_ON_TRANS__SHIFT                                               0x9
+#define CP_CPF_STALLED_STAT1__CMP_UTCL1_WAITING_ON_TRANS__SHIFT                                               0xa
+#define CP_CPF_STALLED_STAT1__RCIU_WAITING_ON_FREE__SHIFT                                                     0xb
+#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA_MASK                                                         0x00000001L
+#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA_MASK                                                        0x00000002L
+#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA_MASK                                                        0x00000004L
+#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA_MASK                                                        0x00000008L
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE_MASK                                                       0x00000020L
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS_MASK                                                       0x00000040L
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE_MASK                                                    0x00000080L
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS_MASK                                                    0x00000100L
+#define CP_CPF_STALLED_STAT1__GFX_UTCL1_WAITING_ON_TRANS_MASK                                                 0x00000200L
+#define CP_CPF_STALLED_STAT1__CMP_UTCL1_WAITING_ON_TRANS_MASK                                                 0x00000400L
+#define CP_CPF_STALLED_STAT1__RCIU_WAITING_ON_FREE_MASK                                                       0x00000800L
+//CP_CPC_GRBM_FREE_COUNT
+#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                             0x0
+#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                               0x0000003FL
+//CP_CPC_PRIV_VIOLATION_ADDR
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_STATUS__SHIFT                                              0x0
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_OP__SHIFT                                                  0x1
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR__SHIFT                                                0x2
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_APERTURE_ID__SHIFT                                         0x14
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_STATUS_MASK                                                0x00000001L
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_OP_MASK                                                    0x00000002L
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR_MASK                                                  0x000FFFFCL
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_APERTURE_ID_MASK                                           0xFFF00000L
+//CP_MEC_CNTL
+#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE__SHIFT                                                             0x4
+#define CP_MEC_CNTL__MEC_ME1_PIPE0_RESET__SHIFT                                                               0x10
+#define CP_MEC_CNTL__MEC_ME1_PIPE1_RESET__SHIFT                                                               0x11
+#define CP_MEC_CNTL__MEC_ME1_PIPE2_RESET__SHIFT                                                               0x12
+#define CP_MEC_CNTL__MEC_ME1_PIPE3_RESET__SHIFT                                                               0x13
+#define CP_MEC_CNTL__MEC_ME2_PIPE0_RESET__SHIFT                                                               0x14
+#define CP_MEC_CNTL__MEC_ME2_PIPE1_RESET__SHIFT                                                               0x15
+#define CP_MEC_CNTL__MEC_ME2_HALT__SHIFT                                                                      0x1c
+#define CP_MEC_CNTL__MEC_ME2_STEP__SHIFT                                                                      0x1d
+#define CP_MEC_CNTL__MEC_ME1_HALT__SHIFT                                                                      0x1e
+#define CP_MEC_CNTL__MEC_ME1_STEP__SHIFT                                                                      0x1f
+#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK                                                               0x00000010L
+#define CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK                                                                 0x00010000L
+#define CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK                                                                 0x00020000L
+#define CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK                                                                 0x00040000L
+#define CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK                                                                 0x00080000L
+#define CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK                                                                 0x00100000L
+#define CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK                                                                 0x00200000L
+#define CP_MEC_CNTL__MEC_ME2_HALT_MASK                                                                        0x10000000L
+#define CP_MEC_CNTL__MEC_ME2_STEP_MASK                                                                        0x20000000L
+#define CP_MEC_CNTL__MEC_ME1_HALT_MASK                                                                        0x40000000L
+#define CP_MEC_CNTL__MEC_ME1_STEP_MASK                                                                        0x80000000L
+//CP_MEC_ME1_HEADER_DUMP
+#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP__SHIFT                                                            0x0
+#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP_MASK                                                              0xFFFFFFFFL
+//CP_MEC_ME2_HEADER_DUMP
+#define CP_MEC_ME2_HEADER_DUMP__HEADER_DUMP__SHIFT                                                            0x0
+#define CP_MEC_ME2_HEADER_DUMP__HEADER_DUMP_MASK                                                              0xFFFFFFFFL
+//CP_CPC_SCRATCH_INDEX
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                            0x0
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                              0x000003FFL
+//CP_CPC_SCRATCH_DATA
+#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                              0x0
+#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                0xFFFFFFFFL
+//CP_CPF_GRBM_FREE_COUNT
+#define CP_CPF_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                             0x0
+#define CP_CPF_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                               0x00000007L
+//CP_CPC_HALT_HYST_COUNT
+#define CP_CPC_HALT_HYST_COUNT__COUNT__SHIFT                                                                  0x0
+#define CP_CPC_HALT_HYST_COUNT__COUNT_MASK                                                                    0x0000000FL
+//CP_CE_COMPARE_COUNT
+#define CP_CE_COMPARE_COUNT__COMPARE_COUNT__SHIFT                                                             0x0
+#define CP_CE_COMPARE_COUNT__COMPARE_COUNT_MASK                                                               0xFFFFFFFFL
+//CP_CE_DE_COUNT
+#define CP_CE_DE_COUNT__DRAW_ENGINE_COUNT__SHIFT                                                              0x0
+#define CP_CE_DE_COUNT__DRAW_ENGINE_COUNT_MASK                                                                0xFFFFFFFFL
+//CP_DE_CE_COUNT
+#define CP_DE_CE_COUNT__CONST_ENGINE_COUNT__SHIFT                                                             0x0
+#define CP_DE_CE_COUNT__CONST_ENGINE_COUNT_MASK                                                               0xFFFFFFFFL
+//CP_DE_LAST_INVAL_COUNT
+#define CP_DE_LAST_INVAL_COUNT__LAST_INVAL_COUNT__SHIFT                                                       0x0
+#define CP_DE_LAST_INVAL_COUNT__LAST_INVAL_COUNT_MASK                                                         0xFFFFFFFFL
+//CP_DE_DE_COUNT
+#define CP_DE_DE_COUNT__DRAW_ENGINE_COUNT__SHIFT                                                              0x0
+#define CP_DE_DE_COUNT__DRAW_ENGINE_COUNT_MASK                                                                0xFFFFFFFFL
+//CP_STALLED_STAT3
+#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV__SHIFT                                                     0x0
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV__SHIFT                                        0x1
+#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER__SHIFT                                     0x2
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY__SHIFT                                                       0x3
+#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY__SHIFT                                                       0x4
+#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY__SHIFT                                                      0x5
+#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV__SHIFT                                                0x6
+#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV__SHIFT                                                 0x7
+#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA__SHIFT                                                    0xa
+#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG__SHIFT                                                 0xb
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER__SHIFT                                                     0xc
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW__SHIFT                                           0xd
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE__SHIFT                                                         0xe
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS__SHIFT                                                         0xf
+#define CP_STALLED_STAT3__CE_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                  0x10
+#define CP_STALLED_STAT3__CE_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                                0x11
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_FREE__SHIFT                                                      0x12
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                      0x13
+#define CP_STALLED_STAT3__UTCL1_WAITING_ON_TRANS__SHIFT                                                       0x14
+#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV_MASK                                                       0x00000001L
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV_MASK                                          0x00000002L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER_MASK                                       0x00000004L
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY_MASK                                                         0x00000008L
+#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY_MASK                                                         0x00000010L
+#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY_MASK                                                        0x00000020L
+#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV_MASK                                                  0x00000040L
+#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV_MASK                                                   0x00000080L
+#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA_MASK                                                      0x00000400L
+#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG_MASK                                                   0x00000800L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_MASK                                                       0x00001000L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW_MASK                                             0x00002000L
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE_MASK                                                           0x00004000L
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS_MASK                                                           0x00008000L
+#define CP_STALLED_STAT3__CE_STALLED_ON_TC_WR_CONFIRM_MASK                                                    0x00010000L
+#define CP_STALLED_STAT3__CE_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                  0x00020000L
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_FREE_MASK                                                        0x00040000L
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_TAGS_MASK                                                        0x00080000L
+#define CP_STALLED_STAT3__UTCL1_WAITING_ON_TRANS_MASK                                                         0x00100000L
+//CP_STALLED_STAT1
+#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV__SHIFT                                                   0x0
+#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV__SHIFT                                                   0x2
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV__SHIFT                                                 0x4
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG__SHIFT                                                 0xa
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG__SHIFT                                                 0xb
+#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                  0xc
+#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                                0xd
+#define CP_STALLED_STAT1__ME_WAITING_ON_TC_READ_DATA__SHIFT                                                   0xe
+#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA__SHIFT                                                  0xf
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE__SHIFT                                                     0x17
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE__SHIFT                                                    0x18
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE__SHIFT                                                     0x19
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ__SHIFT                                                      0x1a
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ__SHIFT                                                     0x1b
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ__SHIFT                                                  0x1c
+#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION__SHIFT                                                 0x1d
+#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV_MASK                                                     0x00000001L
+#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV_MASK                                                     0x00000004L
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_MASK                                                   0x00000010L
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG_MASK                                                   0x00000400L
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG_MASK                                                   0x00000800L
+#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM_MASK                                                    0x00001000L
+#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                  0x00002000L
+#define CP_STALLED_STAT1__ME_WAITING_ON_TC_READ_DATA_MASK                                                     0x00004000L
+#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA_MASK                                                    0x00008000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE_MASK                                                       0x00800000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE_MASK                                                      0x01000000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE_MASK                                                       0x02000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ_MASK                                                        0x04000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ_MASK                                                       0x08000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ_MASK                                                    0x10000000L
+#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION_MASK                                                   0x20000000L
+//CP_STALLED_STAT2
+#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV__SHIFT                                                    0x0
+#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV__SHIFT                                                    0x1
+#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV__SHIFT                                                   0x2
+#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING__SHIFT                                                    0x4
+#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING__SHIFT                                                        0x5
+#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA__SHIFT                                                   0x8
+#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER__SHIFT                                                        0x9
+#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER__SHIFT                                                      0xa
+#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME__SHIFT                                                     0xb
+#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV__SHIFT                                                       0xc
+#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV__SHIFT                                                   0xd
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP__SHIFT                                                     0xe
+#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH__SHIFT                                                  0xf
+#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV__SHIFT                                                     0x10
+#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV__SHIFT                                                     0x11
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ__SHIFT                                                     0x12
+#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                 0x13
+#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                               0x14
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_SC_EOP_DONE__SHIFT                                                  0x15
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_WR_CONFIRM__SHIFT                                                   0x16
+#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING__SHIFT                                                0x17
+#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING__SHIFT                                                   0x18
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE__SHIFT                                                   0x19
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE__SHIFT                                                   0x1a
+#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM__SHIFT                                                    0x1b
+#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION__SHIFT                                                      0x1c
+#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE__SHIFT                                              0x1d
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS__SHIFT                                                   0x1e
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN__SHIFT                                                    0x1f
+#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV_MASK                                                      0x00000001L
+#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV_MASK                                                      0x00000002L
+#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV_MASK                                                     0x00000004L
+#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING_MASK                                                      0x00000010L
+#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING_MASK                                                          0x00000020L
+#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA_MASK                                                     0x00000100L
+#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER_MASK                                                          0x00000200L
+#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER_MASK                                                        0x00000400L
+#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME_MASK                                                       0x00000800L
+#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV_MASK                                                         0x00001000L
+#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV_MASK                                                     0x00002000L
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP_MASK                                                       0x00004000L
+#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH_MASK                                                    0x00008000L
+#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV_MASK                                                       0x00010000L
+#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV_MASK                                                       0x00020000L
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ_MASK                                                       0x00040000L
+#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM_MASK                                                   0x00080000L
+#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                 0x00100000L
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_SC_EOP_DONE_MASK                                                    0x00200000L
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_WR_CONFIRM_MASK                                                     0x00400000L
+#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING_MASK                                                  0x00800000L
+#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING_MASK                                                     0x01000000L
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE_MASK                                                     0x02000000L
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE_MASK                                                     0x04000000L
+#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM_MASK                                                      0x08000000L
+#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION_MASK                                                        0x10000000L
+#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE_MASK                                                0x20000000L
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS_MASK                                                     0x40000000L
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN_MASK                                                      0x80000000L
+//CP_BUSY_STAT
+#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT                                                                0x0
+#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO__SHIFT                                                               0x6
+#define CP_BUSY_STAT__PFP_PARSING_PACKETS__SHIFT                                                              0x7
+#define CP_BUSY_STAT__ME_PARSING_PACKETS__SHIFT                                                               0x8
+#define CP_BUSY_STAT__RCIU_PFP_BUSY__SHIFT                                                                    0x9
+#define CP_BUSY_STAT__RCIU_ME_BUSY__SHIFT                                                                     0xa
+#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY__SHIFT                                                            0xc
+#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING__SHIFT                                                           0xd
+#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS__SHIFT                                                             0xe
+#define CP_BUSY_STAT__GFX_CONTEXT_BUSY__SHIFT                                                                 0xf
+#define CP_BUSY_STAT__ME_PARSER_BUSY__SHIFT                                                                   0x11
+#define CP_BUSY_STAT__EOP_DONE_BUSY__SHIFT                                                                    0x12
+#define CP_BUSY_STAT__STRM_OUT_BUSY__SHIFT                                                                    0x13
+#define CP_BUSY_STAT__PIPE_STATS_BUSY__SHIFT                                                                  0x14
+#define CP_BUSY_STAT__RCIU_CE_BUSY__SHIFT                                                                     0x15
+#define CP_BUSY_STAT__CE_PARSING_PACKETS__SHIFT                                                               0x16
+#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK                                                                  0x00000001L
+#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO_MASK                                                                 0x00000040L
+#define CP_BUSY_STAT__PFP_PARSING_PACKETS_MASK                                                                0x00000080L
+#define CP_BUSY_STAT__ME_PARSING_PACKETS_MASK                                                                 0x00000100L
+#define CP_BUSY_STAT__RCIU_PFP_BUSY_MASK                                                                      0x00000200L
+#define CP_BUSY_STAT__RCIU_ME_BUSY_MASK                                                                       0x00000400L
+#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY_MASK                                                              0x00001000L
+#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING_MASK                                                             0x00002000L
+#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS_MASK                                                               0x00004000L
+#define CP_BUSY_STAT__GFX_CONTEXT_BUSY_MASK                                                                   0x00008000L
+#define CP_BUSY_STAT__ME_PARSER_BUSY_MASK                                                                     0x00020000L
+#define CP_BUSY_STAT__EOP_DONE_BUSY_MASK                                                                      0x00040000L
+#define CP_BUSY_STAT__STRM_OUT_BUSY_MASK                                                                      0x00080000L
+#define CP_BUSY_STAT__PIPE_STATS_BUSY_MASK                                                                    0x00100000L
+#define CP_BUSY_STAT__RCIU_CE_BUSY_MASK                                                                       0x00200000L
+#define CP_BUSY_STAT__CE_PARSING_PACKETS_MASK                                                                 0x00400000L
+//CP_STAT
+#define CP_STAT__ROQ_RING_BUSY__SHIFT                                                                         0x9
+#define CP_STAT__ROQ_INDIRECT1_BUSY__SHIFT                                                                    0xa
+#define CP_STAT__ROQ_INDIRECT2_BUSY__SHIFT                                                                    0xb
+#define CP_STAT__ROQ_STATE_BUSY__SHIFT                                                                        0xc
+#define CP_STAT__DC_BUSY__SHIFT                                                                               0xd
+#define CP_STAT__UTCL2IU_BUSY__SHIFT                                                                          0xe
+#define CP_STAT__PFP_BUSY__SHIFT                                                                              0xf
+#define CP_STAT__MEQ_BUSY__SHIFT                                                                              0x10
+#define CP_STAT__ME_BUSY__SHIFT                                                                               0x11
+#define CP_STAT__QUERY_BUSY__SHIFT                                                                            0x12
+#define CP_STAT__SEMAPHORE_BUSY__SHIFT                                                                        0x13
+#define CP_STAT__INTERRUPT_BUSY__SHIFT                                                                        0x14
+#define CP_STAT__SURFACE_SYNC_BUSY__SHIFT                                                                     0x15
+#define CP_STAT__DMA_BUSY__SHIFT                                                                              0x16
+#define CP_STAT__RCIU_BUSY__SHIFT                                                                             0x17
+#define CP_STAT__SCRATCH_RAM_BUSY__SHIFT                                                                      0x18
+#define CP_STAT__CE_BUSY__SHIFT                                                                               0x1a
+#define CP_STAT__TCIU_BUSY__SHIFT                                                                             0x1b
+#define CP_STAT__ROQ_CE_RING_BUSY__SHIFT                                                                      0x1c
+#define CP_STAT__ROQ_CE_INDIRECT1_BUSY__SHIFT                                                                 0x1d
+#define CP_STAT__ROQ_CE_INDIRECT2_BUSY__SHIFT                                                                 0x1e
+#define CP_STAT__CP_BUSY__SHIFT                                                                               0x1f
+#define CP_STAT__ROQ_RING_BUSY_MASK                                                                           0x00000200L
+#define CP_STAT__ROQ_INDIRECT1_BUSY_MASK                                                                      0x00000400L
+#define CP_STAT__ROQ_INDIRECT2_BUSY_MASK                                                                      0x00000800L
+#define CP_STAT__ROQ_STATE_BUSY_MASK                                                                          0x00001000L
+#define CP_STAT__DC_BUSY_MASK                                                                                 0x00002000L
+#define CP_STAT__UTCL2IU_BUSY_MASK                                                                            0x00004000L
+#define CP_STAT__PFP_BUSY_MASK                                                                                0x00008000L
+#define CP_STAT__MEQ_BUSY_MASK                                                                                0x00010000L
+#define CP_STAT__ME_BUSY_MASK                                                                                 0x00020000L
+#define CP_STAT__QUERY_BUSY_MASK                                                                              0x00040000L
+#define CP_STAT__SEMAPHORE_BUSY_MASK                                                                          0x00080000L
+#define CP_STAT__INTERRUPT_BUSY_MASK                                                                          0x00100000L
+#define CP_STAT__SURFACE_SYNC_BUSY_MASK                                                                       0x00200000L
+#define CP_STAT__DMA_BUSY_MASK                                                                                0x00400000L
+#define CP_STAT__RCIU_BUSY_MASK                                                                               0x00800000L
+#define CP_STAT__SCRATCH_RAM_BUSY_MASK                                                                        0x01000000L
+#define CP_STAT__CE_BUSY_MASK                                                                                 0x04000000L
+#define CP_STAT__TCIU_BUSY_MASK                                                                               0x08000000L
+#define CP_STAT__ROQ_CE_RING_BUSY_MASK                                                                        0x10000000L
+#define CP_STAT__ROQ_CE_INDIRECT1_BUSY_MASK                                                                   0x20000000L
+#define CP_STAT__ROQ_CE_INDIRECT2_BUSY_MASK                                                                   0x40000000L
+#define CP_STAT__CP_BUSY_MASK                                                                                 0x80000000L
+//CP_ME_HEADER_DUMP
+#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP__SHIFT                                                              0x0
+#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP_MASK                                                                0xFFFFFFFFL
+//CP_PFP_HEADER_DUMP
+#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP__SHIFT                                                            0x0
+#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP_MASK                                                              0xFFFFFFFFL
+//CP_GRBM_FREE_COUNT
+#define CP_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                                 0x0
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS__SHIFT                                                             0x8
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP__SHIFT                                                             0x10
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                                   0x0000003FL
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS_MASK                                                               0x00003F00L
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP_MASK                                                               0x003F0000L
+//CP_CE_HEADER_DUMP
+#define CP_CE_HEADER_DUMP__CE_HEADER_DUMP__SHIFT                                                              0x0
+#define CP_CE_HEADER_DUMP__CE_HEADER_DUMP_MASK                                                                0xFFFFFFFFL
+//CP_PFP_INSTR_PNTR
+#define CP_PFP_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                  0x0
+#define CP_PFP_INSTR_PNTR__INSTR_PNTR_MASK                                                                    0x0000FFFFL
+//CP_ME_INSTR_PNTR
+#define CP_ME_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                   0x0
+#define CP_ME_INSTR_PNTR__INSTR_PNTR_MASK                                                                     0x0000FFFFL
+//CP_CE_INSTR_PNTR
+#define CP_CE_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                   0x0
+#define CP_CE_INSTR_PNTR__INSTR_PNTR_MASK                                                                     0x0000FFFFL
+//CP_MEC1_INSTR_PNTR
+#define CP_MEC1_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                 0x0
+#define CP_MEC1_INSTR_PNTR__INSTR_PNTR_MASK                                                                   0x0000FFFFL
+//CP_MEC2_INSTR_PNTR
+#define CP_MEC2_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                 0x0
+#define CP_MEC2_INSTR_PNTR__INSTR_PNTR_MASK                                                                   0x0000FFFFL
+//CP_CSF_STAT
+#define CP_CSF_STAT__BUFFER_REQUEST_COUNT__SHIFT                                                              0x8
+#define CP_CSF_STAT__BUFFER_REQUEST_COUNT_MASK                                                                0x0001FF00L
+//CP_ME_CNTL
+#define CP_ME_CNTL__CE_INVALIDATE_ICACHE__SHIFT                                                               0x4
+#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE__SHIFT                                                              0x6
+#define CP_ME_CNTL__ME_INVALIDATE_ICACHE__SHIFT                                                               0x8
+#define CP_ME_CNTL__CE_PIPE0_RESET__SHIFT                                                                     0x10
+#define CP_ME_CNTL__CE_PIPE1_RESET__SHIFT                                                                     0x11
+#define CP_ME_CNTL__PFP_PIPE0_RESET__SHIFT                                                                    0x12
+#define CP_ME_CNTL__PFP_PIPE1_RESET__SHIFT                                                                    0x13
+#define CP_ME_CNTL__ME_PIPE0_RESET__SHIFT                                                                     0x14
+#define CP_ME_CNTL__ME_PIPE1_RESET__SHIFT                                                                     0x15
+#define CP_ME_CNTL__CE_HALT__SHIFT                                                                            0x18
+#define CP_ME_CNTL__CE_STEP__SHIFT                                                                            0x19
+#define CP_ME_CNTL__PFP_HALT__SHIFT                                                                           0x1a
+#define CP_ME_CNTL__PFP_STEP__SHIFT                                                                           0x1b
+#define CP_ME_CNTL__ME_HALT__SHIFT                                                                            0x1c
+#define CP_ME_CNTL__ME_STEP__SHIFT                                                                            0x1d
+#define CP_ME_CNTL__CE_INVALIDATE_ICACHE_MASK                                                                 0x00000010L
+#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE_MASK                                                                0x00000040L
+#define CP_ME_CNTL__ME_INVALIDATE_ICACHE_MASK                                                                 0x00000100L
+#define CP_ME_CNTL__CE_PIPE0_RESET_MASK                                                                       0x00010000L
+#define CP_ME_CNTL__CE_PIPE1_RESET_MASK                                                                       0x00020000L
+#define CP_ME_CNTL__PFP_PIPE0_RESET_MASK                                                                      0x00040000L
+#define CP_ME_CNTL__PFP_PIPE1_RESET_MASK                                                                      0x00080000L
+#define CP_ME_CNTL__ME_PIPE0_RESET_MASK                                                                       0x00100000L
+#define CP_ME_CNTL__ME_PIPE1_RESET_MASK                                                                       0x00200000L
+#define CP_ME_CNTL__CE_HALT_MASK                                                                              0x01000000L
+#define CP_ME_CNTL__CE_STEP_MASK                                                                              0x02000000L
+#define CP_ME_CNTL__PFP_HALT_MASK                                                                             0x04000000L
+#define CP_ME_CNTL__PFP_STEP_MASK                                                                             0x08000000L
+#define CP_ME_CNTL__ME_HALT_MASK                                                                              0x10000000L
+#define CP_ME_CNTL__ME_STEP_MASK                                                                              0x20000000L
+//CP_CNTX_STAT
+#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS__SHIFT                                                             0x0
+#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT__SHIFT                                                             0x8
+#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS__SHIFT                                                              0x14
+#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT__SHIFT                                                              0x1c
+#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS_MASK                                                               0x000000FFL
+#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT_MASK                                                               0x00000700L
+#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS_MASK                                                                0x0FF00000L
+#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT_MASK                                                                0x70000000L
+//CP_ME_PREEMPTION
+#define CP_ME_PREEMPTION__OBSOLETE__SHIFT                                                                     0x0
+#define CP_ME_PREEMPTION__OBSOLETE_MASK                                                                       0x00000001L
+//CP_ROQ_THRESHOLDS
+#define CP_ROQ_THRESHOLDS__IB1_START__SHIFT                                                                   0x0
+#define CP_ROQ_THRESHOLDS__IB2_START__SHIFT                                                                   0x8
+#define CP_ROQ_THRESHOLDS__IB1_START_MASK                                                                     0x000000FFL
+#define CP_ROQ_THRESHOLDS__IB2_START_MASK                                                                     0x0000FF00L
+//CP_MEQ_STQ_THRESHOLD
+#define CP_MEQ_STQ_THRESHOLD__STQ_START__SHIFT                                                                0x0
+#define CP_MEQ_STQ_THRESHOLD__STQ_START_MASK                                                                  0x000000FFL
+//CP_RB2_RPTR
+#define CP_RB2_RPTR__RB_RPTR__SHIFT                                                                           0x0
+#define CP_RB2_RPTR__RB_RPTR_MASK                                                                             0x000FFFFFL
+//CP_RB1_RPTR
+#define CP_RB1_RPTR__RB_RPTR__SHIFT                                                                           0x0
+#define CP_RB1_RPTR__RB_RPTR_MASK                                                                             0x000FFFFFL
+//CP_RB0_RPTR
+#define CP_RB0_RPTR__RB_RPTR__SHIFT                                                                           0x0
+#define CP_RB0_RPTR__RB_RPTR_MASK                                                                             0x000FFFFFL
+//CP_RB_RPTR
+#define CP_RB_RPTR__RB_RPTR__SHIFT                                                                            0x0
+#define CP_RB_RPTR__RB_RPTR_MASK                                                                              0x000FFFFFL
+//CP_RB_WPTR_DELAY
+#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER__SHIFT                                                              0x0
+#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT__SHIFT                                                              0x1c
+#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER_MASK                                                                0x0FFFFFFFL
+#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT_MASK                                                                0xF0000000L
+//CP_RB_WPTR_POLL_CNTL
+#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT                                                           0x0
+#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                          0x10
+#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK                                                             0x0000FFFFL
+#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                            0xFFFF0000L
+//CP_ROQ1_THRESHOLDS
+#define CP_ROQ1_THRESHOLDS__RB1_START__SHIFT                                                                  0x0
+#define CP_ROQ1_THRESHOLDS__RB2_START__SHIFT                                                                  0x8
+#define CP_ROQ1_THRESHOLDS__R0_IB1_START__SHIFT                                                               0x10
+#define CP_ROQ1_THRESHOLDS__R1_IB1_START__SHIFT                                                               0x18
+#define CP_ROQ1_THRESHOLDS__RB1_START_MASK                                                                    0x000000FFL
+#define CP_ROQ1_THRESHOLDS__RB2_START_MASK                                                                    0x0000FF00L
+#define CP_ROQ1_THRESHOLDS__R0_IB1_START_MASK                                                                 0x00FF0000L
+#define CP_ROQ1_THRESHOLDS__R1_IB1_START_MASK                                                                 0xFF000000L
+//CP_ROQ2_THRESHOLDS
+#define CP_ROQ2_THRESHOLDS__R2_IB1_START__SHIFT                                                               0x0
+#define CP_ROQ2_THRESHOLDS__R0_IB2_START__SHIFT                                                               0x8
+#define CP_ROQ2_THRESHOLDS__R1_IB2_START__SHIFT                                                               0x10
+#define CP_ROQ2_THRESHOLDS__R2_IB2_START__SHIFT                                                               0x18
+#define CP_ROQ2_THRESHOLDS__R2_IB1_START_MASK                                                                 0x000000FFL
+#define CP_ROQ2_THRESHOLDS__R0_IB2_START_MASK                                                                 0x0000FF00L
+#define CP_ROQ2_THRESHOLDS__R1_IB2_START_MASK                                                                 0x00FF0000L
+#define CP_ROQ2_THRESHOLDS__R2_IB2_START_MASK                                                                 0xFF000000L
+//CP_STQ_THRESHOLDS
+#define CP_STQ_THRESHOLDS__STQ0_START__SHIFT                                                                  0x0
+#define CP_STQ_THRESHOLDS__STQ1_START__SHIFT                                                                  0x8
+#define CP_STQ_THRESHOLDS__STQ2_START__SHIFT                                                                  0x10
+#define CP_STQ_THRESHOLDS__STQ0_START_MASK                                                                    0x000000FFL
+#define CP_STQ_THRESHOLDS__STQ1_START_MASK                                                                    0x0000FF00L
+#define CP_STQ_THRESHOLDS__STQ2_START_MASK                                                                    0x00FF0000L
+//CP_QUEUE_THRESHOLDS
+#define CP_QUEUE_THRESHOLDS__ROQ_IB1_START__SHIFT                                                             0x0
+#define CP_QUEUE_THRESHOLDS__ROQ_IB2_START__SHIFT                                                             0x8
+#define CP_QUEUE_THRESHOLDS__ROQ_IB1_START_MASK                                                               0x0000003FL
+#define CP_QUEUE_THRESHOLDS__ROQ_IB2_START_MASK                                                               0x00003F00L
+//CP_MEQ_THRESHOLDS
+#define CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT                                                                  0x0
+#define CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT                                                                  0x8
+#define CP_MEQ_THRESHOLDS__MEQ1_START_MASK                                                                    0x000000FFL
+#define CP_MEQ_THRESHOLDS__MEQ2_START_MASK                                                                    0x0000FF00L
+//CP_ROQ_AVAIL
+#define CP_ROQ_AVAIL__ROQ_CNT_RING__SHIFT                                                                     0x0
+#define CP_ROQ_AVAIL__ROQ_CNT_IB1__SHIFT                                                                      0x10
+#define CP_ROQ_AVAIL__ROQ_CNT_RING_MASK                                                                       0x000007FFL
+#define CP_ROQ_AVAIL__ROQ_CNT_IB1_MASK                                                                        0x07FF0000L
+//CP_STQ_AVAIL
+#define CP_STQ_AVAIL__STQ_CNT__SHIFT                                                                          0x0
+#define CP_STQ_AVAIL__STQ_CNT_MASK                                                                            0x000001FFL
+//CP_ROQ2_AVAIL
+#define CP_ROQ2_AVAIL__ROQ_CNT_IB2__SHIFT                                                                     0x0
+#define CP_ROQ2_AVAIL__ROQ_CNT_IB2_MASK                                                                       0x000007FFL
+//CP_MEQ_AVAIL
+#define CP_MEQ_AVAIL__MEQ_CNT__SHIFT                                                                          0x0
+#define CP_MEQ_AVAIL__MEQ_CNT_MASK                                                                            0x000003FFL
+//CP_CMD_INDEX
+#define CP_CMD_INDEX__CMD_INDEX__SHIFT                                                                        0x0
+#define CP_CMD_INDEX__CMD_ME_SEL__SHIFT                                                                       0xc
+#define CP_CMD_INDEX__CMD_QUEUE_SEL__SHIFT                                                                    0x10
+#define CP_CMD_INDEX__CMD_INDEX_MASK                                                                          0x000007FFL
+#define CP_CMD_INDEX__CMD_ME_SEL_MASK                                                                         0x00003000L
+#define CP_CMD_INDEX__CMD_QUEUE_SEL_MASK                                                                      0x00070000L
+//CP_CMD_DATA
+#define CP_CMD_DATA__CMD_DATA__SHIFT                                                                          0x0
+#define CP_CMD_DATA__CMD_DATA_MASK                                                                            0xFFFFFFFFL
+//CP_ROQ_RB_STAT
+#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY__SHIFT                                                               0x0
+#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY__SHIFT                                                               0x10
+#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY_MASK                                                                 0x000003FFL
+#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY_MASK                                                                 0x03FF0000L
+//CP_ROQ_IB1_STAT
+#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1__SHIFT                                                            0x0
+#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1__SHIFT                                                            0x10
+#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1_MASK                                                              0x000003FFL
+#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1_MASK                                                              0x03FF0000L
+//CP_ROQ_IB2_STAT
+#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2__SHIFT                                                            0x0
+#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2__SHIFT                                                            0x10
+#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2_MASK                                                              0x000003FFL
+#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2_MASK                                                              0x03FF0000L
+//CP_STQ_STAT
+#define CP_STQ_STAT__STQ_RPTR__SHIFT                                                                          0x0
+#define CP_STQ_STAT__STQ_RPTR_MASK                                                                            0x000003FFL
+//CP_STQ_WR_STAT
+#define CP_STQ_WR_STAT__STQ_WPTR__SHIFT                                                                       0x0
+#define CP_STQ_WR_STAT__STQ_WPTR_MASK                                                                         0x000003FFL
+//CP_MEQ_STAT
+#define CP_MEQ_STAT__MEQ_RPTR__SHIFT                                                                          0x0
+#define CP_MEQ_STAT__MEQ_WPTR__SHIFT                                                                          0x10
+#define CP_MEQ_STAT__MEQ_RPTR_MASK                                                                            0x000003FFL
+#define CP_MEQ_STAT__MEQ_WPTR_MASK                                                                            0x03FF0000L
+//CP_CEQ1_AVAIL
+#define CP_CEQ1_AVAIL__CEQ_CNT_RING__SHIFT                                                                    0x0
+#define CP_CEQ1_AVAIL__CEQ_CNT_IB1__SHIFT                                                                     0x10
+#define CP_CEQ1_AVAIL__CEQ_CNT_RING_MASK                                                                      0x000007FFL
+#define CP_CEQ1_AVAIL__CEQ_CNT_IB1_MASK                                                                       0x07FF0000L
+//CP_CEQ2_AVAIL
+#define CP_CEQ2_AVAIL__CEQ_CNT_IB2__SHIFT                                                                     0x0
+#define CP_CEQ2_AVAIL__CEQ_CNT_IB2_MASK                                                                       0x000007FFL
+//CP_CE_ROQ_RB_STAT
+#define CP_CE_ROQ_RB_STAT__CEQ_RPTR_PRIMARY__SHIFT                                                            0x0
+#define CP_CE_ROQ_RB_STAT__CEQ_WPTR_PRIMARY__SHIFT                                                            0x10
+#define CP_CE_ROQ_RB_STAT__CEQ_RPTR_PRIMARY_MASK                                                              0x000003FFL
+#define CP_CE_ROQ_RB_STAT__CEQ_WPTR_PRIMARY_MASK                                                              0x03FF0000L
+//CP_CE_ROQ_IB1_STAT
+#define CP_CE_ROQ_IB1_STAT__CEQ_RPTR_INDIRECT1__SHIFT                                                         0x0
+#define CP_CE_ROQ_IB1_STAT__CEQ_WPTR_INDIRECT1__SHIFT                                                         0x10
+#define CP_CE_ROQ_IB1_STAT__CEQ_RPTR_INDIRECT1_MASK                                                           0x000003FFL
+#define CP_CE_ROQ_IB1_STAT__CEQ_WPTR_INDIRECT1_MASK                                                           0x03FF0000L
+//CP_CE_ROQ_IB2_STAT
+#define CP_CE_ROQ_IB2_STAT__CEQ_RPTR_INDIRECT2__SHIFT                                                         0x0
+#define CP_CE_ROQ_IB2_STAT__CEQ_WPTR_INDIRECT2__SHIFT                                                         0x10
+#define CP_CE_ROQ_IB2_STAT__CEQ_RPTR_INDIRECT2_MASK                                                           0x000003FFL
+#define CP_CE_ROQ_IB2_STAT__CEQ_WPTR_INDIRECT2_MASK                                                           0x03FF0000L
+//CP_INT_STAT_DEBUG
+#define CP_INT_STAT_DEBUG__CP_VM_DOORBELL_WR_INT_ASSERTED__SHIFT                                              0xb
+#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                                   0xe
+#define CP_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                            0x10
+#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                               0x11
+#define CP_INT_STAT_DEBUG__CMP_BUSY_INT_ASSERTED__SHIFT                                                       0x12
+#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED__SHIFT                                                      0x13
+#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED__SHIFT                                                     0x14
+#define CP_INT_STAT_DEBUG__GFX_IDLE_INT_ASSERTED__SHIFT                                                       0x15
+#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED__SHIFT                                                     0x16
+#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                       0x17
+#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                                   0x18
+#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                     0x1a
+#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                             0x1b
+#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                       0x1d
+#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                       0x1e
+#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                       0x1f
+#define CP_INT_STAT_DEBUG__CP_VM_DOORBELL_WR_INT_ASSERTED_MASK                                                0x00000800L
+#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                     0x00004000L
+#define CP_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                              0x00010000L
+#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                                 0x00020000L
+#define CP_INT_STAT_DEBUG__CMP_BUSY_INT_ASSERTED_MASK                                                         0x00040000L
+#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED_MASK                                                        0x00080000L
+#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED_MASK                                                       0x00100000L
+#define CP_INT_STAT_DEBUG__GFX_IDLE_INT_ASSERTED_MASK                                                         0x00200000L
+#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED_MASK                                                       0x00400000L
+#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                         0x00800000L
+#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                     0x01000000L
+#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                       0x04000000L
+#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                               0x08000000L
+#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                         0x20000000L
+#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                         0x40000000L
+#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                         0x80000000L
+//CP_DEBUG_CNTL
+#define CP_DEBUG_CNTL__DEBUG_INDX__SHIFT                                                                      0x0
+#define CP_DEBUG_CNTL__DEBUG_BUS_SELECT_BITS__SHIFT                                                           0x10
+#define CP_DEBUG_CNTL__DEBUG_BUS_FLOP_EN__SHIFT                                                               0x1f
+#define CP_DEBUG_CNTL__DEBUG_INDX_MASK                                                                        0x0000007FL
+#define CP_DEBUG_CNTL__DEBUG_BUS_SELECT_BITS_MASK                                                             0x003F0000L
+#define CP_DEBUG_CNTL__DEBUG_BUS_FLOP_EN_MASK                                                                 0x80000000L
+//CP_PRIV_VIOLATION_ADDR
+#define CP_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR__SHIFT                                                    0x0
+#define CP_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR_MASK                                                      0x0000FFFFL
+
+
+// addressBlock: xcd0_gc_padec
+//VGT_VTX_VECT_EJECT_REG
+#define VGT_VTX_VECT_EJECT_REG__PRIM_COUNT__SHIFT                                                             0x0
+#define VGT_VTX_VECT_EJECT_REG__PRIM_COUNT_MASK                                                               0x0000007FL
+//VGT_DMA_DATA_FIFO_DEPTH
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH__SHIFT                                                   0x0
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA2DRAW_FIFO_DEPTH__SHIFT                                                   0x9
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH_MASK                                                     0x000001FFL
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA2DRAW_FIFO_DEPTH_MASK                                                     0x0007FE00L
+//VGT_DMA_REQ_FIFO_DEPTH
+#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH__SHIFT                                                     0x0
+#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH_MASK                                                       0x0000003FL
+//VGT_DRAW_INIT_FIFO_DEPTH
+#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH__SHIFT                                                 0x0
+#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH_MASK                                                   0x0000003FL
+//VGT_LAST_COPY_STATE
+#define VGT_LAST_COPY_STATE__SRC_STATE_ID__SHIFT                                                              0x0
+#define VGT_LAST_COPY_STATE__DST_STATE_ID__SHIFT                                                              0x10
+#define VGT_LAST_COPY_STATE__SRC_STATE_ID_MASK                                                                0x00000007L
+#define VGT_LAST_COPY_STATE__DST_STATE_ID_MASK                                                                0x00070000L
+//VGT_CACHE_INVALIDATION
+#define VGT_CACHE_INVALIDATION__CACHE_INVALIDATION__SHIFT                                                     0x0
+#define VGT_CACHE_INVALIDATION__DIS_INSTANCING_OPT__SHIFT                                                     0x4
+#define VGT_CACHE_INVALIDATION__VS_NO_EXTRA_BUFFER__SHIFT                                                     0x5
+#define VGT_CACHE_INVALIDATION__AUTO_INVLD_EN__SHIFT                                                          0x6
+#define VGT_CACHE_INVALIDATION__USE_GS_DONE__SHIFT                                                            0x9
+#define VGT_CACHE_INVALIDATION__DIS_RANGE_FULL_INVLD__SHIFT                                                   0xb
+#define VGT_CACHE_INVALIDATION__GS_LATE_ALLOC_EN__SHIFT                                                       0xc
+#define VGT_CACHE_INVALIDATION__STREAMOUT_FULL_FLUSH__SHIFT                                                   0xd
+#define VGT_CACHE_INVALIDATION__ES_LIMIT__SHIFT                                                               0x10
+#define VGT_CACHE_INVALIDATION__ENABLE_PING_PONG__SHIFT                                                       0x15
+#define VGT_CACHE_INVALIDATION__OPT_FLOW_CNTL_1__SHIFT                                                        0x16
+#define VGT_CACHE_INVALIDATION__OPT_FLOW_CNTL_2__SHIFT                                                        0x19
+#define VGT_CACHE_INVALIDATION__EN_WAVE_MERGE__SHIFT                                                          0x1c
+#define VGT_CACHE_INVALIDATION__ENABLE_PING_PONG_EOI__SHIFT                                                   0x1d
+#define VGT_CACHE_INVALIDATION__CACHE_INVALIDATION_MASK                                                       0x00000003L
+#define VGT_CACHE_INVALIDATION__DIS_INSTANCING_OPT_MASK                                                       0x00000010L
+#define VGT_CACHE_INVALIDATION__VS_NO_EXTRA_BUFFER_MASK                                                       0x00000020L
+#define VGT_CACHE_INVALIDATION__AUTO_INVLD_EN_MASK                                                            0x000000C0L
+#define VGT_CACHE_INVALIDATION__USE_GS_DONE_MASK                                                              0x00000200L
+#define VGT_CACHE_INVALIDATION__DIS_RANGE_FULL_INVLD_MASK                                                     0x00000800L
+#define VGT_CACHE_INVALIDATION__GS_LATE_ALLOC_EN_MASK                                                         0x00001000L
+#define VGT_CACHE_INVALIDATION__STREAMOUT_FULL_FLUSH_MASK                                                     0x00002000L
+#define VGT_CACHE_INVALIDATION__ES_LIMIT_MASK                                                                 0x001F0000L
+#define VGT_CACHE_INVALIDATION__ENABLE_PING_PONG_MASK                                                         0x00200000L
+#define VGT_CACHE_INVALIDATION__OPT_FLOW_CNTL_1_MASK                                                          0x01C00000L
+#define VGT_CACHE_INVALIDATION__OPT_FLOW_CNTL_2_MASK                                                          0x0E000000L
+#define VGT_CACHE_INVALIDATION__EN_WAVE_MERGE_MASK                                                            0x10000000L
+#define VGT_CACHE_INVALIDATION__ENABLE_PING_PONG_EOI_MASK                                                     0x20000000L
+//VGT_RESET_DEBUG
+#define VGT_RESET_DEBUG__GS_DISABLE__SHIFT                                                                    0x0
+#define VGT_RESET_DEBUG__TESS_DISABLE__SHIFT                                                                  0x1
+#define VGT_RESET_DEBUG__WD_DISABLE__SHIFT                                                                    0x2
+#define VGT_RESET_DEBUG__GS_DISABLE_MASK                                                                      0x00000001L
+#define VGT_RESET_DEBUG__TESS_DISABLE_MASK                                                                    0x00000002L
+#define VGT_RESET_DEBUG__WD_DISABLE_MASK                                                                      0x00000004L
+//VGT_STRMOUT_DELAY
+#define VGT_STRMOUT_DELAY__SKIP_DELAY__SHIFT                                                                  0x0
+#define VGT_STRMOUT_DELAY__SE0_WD_DELAY__SHIFT                                                                0x8
+#define VGT_STRMOUT_DELAY__SE1_WD_DELAY__SHIFT                                                                0xb
+#define VGT_STRMOUT_DELAY__SE2_WD_DELAY__SHIFT                                                                0xe
+#define VGT_STRMOUT_DELAY__SE3_WD_DELAY__SHIFT                                                                0x11
+#define VGT_STRMOUT_DELAY__SKIP_DELAY_MASK                                                                    0x000000FFL
+#define VGT_STRMOUT_DELAY__SE0_WD_DELAY_MASK                                                                  0x00000700L
+#define VGT_STRMOUT_DELAY__SE1_WD_DELAY_MASK                                                                  0x00003800L
+#define VGT_STRMOUT_DELAY__SE2_WD_DELAY_MASK                                                                  0x0001C000L
+#define VGT_STRMOUT_DELAY__SE3_WD_DELAY_MASK                                                                  0x000E0000L
+//VGT_FIFO_DEPTHS
+#define VGT_FIFO_DEPTHS__VS_DEALLOC_TBL_DEPTH__SHIFT                                                          0x0
+#define VGT_FIFO_DEPTHS__RESERVED_0__SHIFT                                                                    0x7
+#define VGT_FIFO_DEPTHS__CLIPP_FIFO_DEPTH__SHIFT                                                              0x8
+#define VGT_FIFO_DEPTHS__HSINPUT_FIFO_DEPTH__SHIFT                                                            0x16
+#define VGT_FIFO_DEPTHS__VS_DEALLOC_TBL_DEPTH_MASK                                                            0x0000007FL
+#define VGT_FIFO_DEPTHS__RESERVED_0_MASK                                                                      0x00000080L
+#define VGT_FIFO_DEPTHS__CLIPP_FIFO_DEPTH_MASK                                                                0x003FFF00L
+#define VGT_FIFO_DEPTHS__HSINPUT_FIFO_DEPTH_MASK                                                              0x0FC00000L
+//VGT_GS_VERTEX_REUSE
+#define VGT_GS_VERTEX_REUSE__VERT_REUSE__SHIFT                                                                0x0
+#define VGT_GS_VERTEX_REUSE__VERT_REUSE_MASK                                                                  0x0000001FL
+//VGT_MC_LAT_CNTL
+#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES__SHIFT                                                             0x0
+#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES_MASK                                                               0x0000000FL
+//IA_CNTL_STATUS
+#define IA_CNTL_STATUS__IA_BUSY__SHIFT                                                                        0x0
+#define IA_CNTL_STATUS__IA_DMA_BUSY__SHIFT                                                                    0x1
+#define IA_CNTL_STATUS__IA_DMA_REQ_BUSY__SHIFT                                                                0x2
+#define IA_CNTL_STATUS__IA_GRP_BUSY__SHIFT                                                                    0x3
+#define IA_CNTL_STATUS__IA_ADC_BUSY__SHIFT                                                                    0x4
+#define IA_CNTL_STATUS__IA_BUSY_MASK                                                                          0x00000001L
+#define IA_CNTL_STATUS__IA_DMA_BUSY_MASK                                                                      0x00000002L
+#define IA_CNTL_STATUS__IA_DMA_REQ_BUSY_MASK                                                                  0x00000004L
+#define IA_CNTL_STATUS__IA_GRP_BUSY_MASK                                                                      0x00000008L
+#define IA_CNTL_STATUS__IA_ADC_BUSY_MASK                                                                      0x00000010L
+//VGT_CNTL_STATUS
+#define VGT_CNTL_STATUS__VGT_BUSY__SHIFT                                                                      0x0
+#define VGT_CNTL_STATUS__VGT_OUT_INDX_BUSY__SHIFT                                                             0x1
+#define VGT_CNTL_STATUS__VGT_OUT_BUSY__SHIFT                                                                  0x2
+#define VGT_CNTL_STATUS__VGT_PT_BUSY__SHIFT                                                                   0x3
+#define VGT_CNTL_STATUS__VGT_TE_BUSY__SHIFT                                                                   0x4
+#define VGT_CNTL_STATUS__VGT_VR_BUSY__SHIFT                                                                   0x5
+#define VGT_CNTL_STATUS__VGT_PI_BUSY__SHIFT                                                                   0x6
+#define VGT_CNTL_STATUS__VGT_GS_BUSY__SHIFT                                                                   0x7
+#define VGT_CNTL_STATUS__VGT_HS_BUSY__SHIFT                                                                   0x8
+#define VGT_CNTL_STATUS__VGT_TE11_BUSY__SHIFT                                                                 0x9
+#define VGT_CNTL_STATUS__VGT_PRIMGEN_BUSY__SHIFT                                                              0xa
+#define VGT_CNTL_STATUS__VGT_BUSY_MASK                                                                        0x00000001L
+#define VGT_CNTL_STATUS__VGT_OUT_INDX_BUSY_MASK                                                               0x00000002L
+#define VGT_CNTL_STATUS__VGT_OUT_BUSY_MASK                                                                    0x00000004L
+#define VGT_CNTL_STATUS__VGT_PT_BUSY_MASK                                                                     0x00000008L
+#define VGT_CNTL_STATUS__VGT_TE_BUSY_MASK                                                                     0x00000010L
+#define VGT_CNTL_STATUS__VGT_VR_BUSY_MASK                                                                     0x00000020L
+#define VGT_CNTL_STATUS__VGT_PI_BUSY_MASK                                                                     0x00000040L
+#define VGT_CNTL_STATUS__VGT_GS_BUSY_MASK                                                                     0x00000080L
+#define VGT_CNTL_STATUS__VGT_HS_BUSY_MASK                                                                     0x00000100L
+#define VGT_CNTL_STATUS__VGT_TE11_BUSY_MASK                                                                   0x00000200L
+#define VGT_CNTL_STATUS__VGT_PRIMGEN_BUSY_MASK                                                                0x00000400L
+//WD_CNTL_STATUS
+#define WD_CNTL_STATUS__WD_BUSY__SHIFT                                                                        0x0
+#define WD_CNTL_STATUS__WD_SPL_DMA_BUSY__SHIFT                                                                0x1
+#define WD_CNTL_STATUS__WD_SPL_DI_BUSY__SHIFT                                                                 0x2
+#define WD_CNTL_STATUS__WD_ADC_BUSY__SHIFT                                                                    0x3
+#define WD_CNTL_STATUS__WD_BUSY_MASK                                                                          0x00000001L
+#define WD_CNTL_STATUS__WD_SPL_DMA_BUSY_MASK                                                                  0x00000002L
+#define WD_CNTL_STATUS__WD_SPL_DI_BUSY_MASK                                                                   0x00000004L
+#define WD_CNTL_STATUS__WD_ADC_BUSY_MASK                                                                      0x00000008L
+//CC_GC_PRIM_CONFIG
+#define CC_GC_PRIM_CONFIG__WRITE_DIS__SHIFT                                                                   0x0
+#define CC_GC_PRIM_CONFIG__INACTIVE_IA__SHIFT                                                                 0x10
+#define CC_GC_PRIM_CONFIG__INACTIVE_VGT_PA__SHIFT                                                             0x18
+#define CC_GC_PRIM_CONFIG__WRITE_DIS_MASK                                                                     0x00000001L
+#define CC_GC_PRIM_CONFIG__INACTIVE_IA_MASK                                                                   0x00030000L
+#define CC_GC_PRIM_CONFIG__INACTIVE_VGT_PA_MASK                                                               0x0F000000L
+//GC_USER_PRIM_CONFIG
+#define GC_USER_PRIM_CONFIG__INACTIVE_IA__SHIFT                                                               0x10
+#define GC_USER_PRIM_CONFIG__INACTIVE_VGT_PA__SHIFT                                                           0x18
+#define GC_USER_PRIM_CONFIG__INACTIVE_IA_MASK                                                                 0x00030000L
+#define GC_USER_PRIM_CONFIG__INACTIVE_VGT_PA_MASK                                                             0x0F000000L
+//WD_QOS
+#define WD_QOS__DRAW_STALL__SHIFT                                                                             0x0
+#define WD_QOS__DRAW_STALL_MASK                                                                               0x00000001L
+//WD_UTCL1_CNTL
+#define WD_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                            0x0
+#define WD_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                 0x17
+#define WD_UTCL1_CNTL__DROP_MODE__SHIFT                                                                       0x18
+#define WD_UTCL1_CNTL__BYPASS__SHIFT                                                                          0x19
+#define WD_UTCL1_CNTL__INVALIDATE__SHIFT                                                                      0x1a
+#define WD_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                 0x1b
+#define WD_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                     0x1c
+#define WD_UTCL1_CNTL__FORCE_SD_VMID_DIRTY__SHIFT                                                             0x1d
+#define WD_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                              0x000FFFFFL
+#define WD_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                   0x00800000L
+#define WD_UTCL1_CNTL__DROP_MODE_MASK                                                                         0x01000000L
+#define WD_UTCL1_CNTL__BYPASS_MASK                                                                            0x02000000L
+#define WD_UTCL1_CNTL__INVALIDATE_MASK                                                                        0x04000000L
+#define WD_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                   0x08000000L
+#define WD_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                       0x10000000L
+#define WD_UTCL1_CNTL__FORCE_SD_VMID_DIRTY_MASK                                                               0x20000000L
+//WD_UTCL1_STATUS
+#define WD_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                                0x0
+#define WD_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                                0x1
+#define WD_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                  0x2
+#define WD_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                 0x8
+#define WD_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                 0x10
+#define WD_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                   0x18
+#define WD_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                  0x00000001L
+#define WD_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                  0x00000002L
+#define WD_UTCL1_STATUS__PRT_DETECTED_MASK                                                                    0x00000004L
+#define WD_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                   0x00003F00L
+#define WD_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                   0x003F0000L
+#define WD_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                     0x3F000000L
+//IA_UTCL1_CNTL
+#define IA_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                            0x0
+#define IA_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                 0x17
+#define IA_UTCL1_CNTL__DROP_MODE__SHIFT                                                                       0x18
+#define IA_UTCL1_CNTL__BYPASS__SHIFT                                                                          0x19
+#define IA_UTCL1_CNTL__INVALIDATE__SHIFT                                                                      0x1a
+#define IA_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                 0x1b
+#define IA_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                     0x1c
+#define IA_UTCL1_CNTL__FORCE_SD_VMID_DIRTY__SHIFT                                                             0x1d
+#define IA_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                              0x000FFFFFL
+#define IA_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                   0x00800000L
+#define IA_UTCL1_CNTL__DROP_MODE_MASK                                                                         0x01000000L
+#define IA_UTCL1_CNTL__BYPASS_MASK                                                                            0x02000000L
+#define IA_UTCL1_CNTL__INVALIDATE_MASK                                                                        0x04000000L
+#define IA_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                   0x08000000L
+#define IA_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                       0x10000000L
+#define IA_UTCL1_CNTL__FORCE_SD_VMID_DIRTY_MASK                                                               0x20000000L
+//IA_UTCL1_STATUS
+#define IA_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                                0x0
+#define IA_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                                0x1
+#define IA_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                  0x2
+#define IA_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                 0x8
+#define IA_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                 0x10
+#define IA_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                   0x18
+#define IA_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                  0x00000001L
+#define IA_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                  0x00000002L
+#define IA_UTCL1_STATUS__PRT_DETECTED_MASK                                                                    0x00000004L
+#define IA_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                   0x00003F00L
+#define IA_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                   0x003F0000L
+#define IA_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                     0x3F000000L
+//VGT_SYS_CONFIG
+#define VGT_SYS_CONFIG__DUAL_CORE_EN__SHIFT                                                                   0x0
+#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP__SHIFT                                                               0x1
+#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE__SHIFT                                                       0x7
+#define VGT_SYS_CONFIG__DUAL_CORE_EN_MASK                                                                     0x00000001L
+#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP_MASK                                                                 0x0000007EL
+#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE_MASK                                                         0x00000080L
+//VGT_VS_MAX_WAVE_ID
+#define VGT_VS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                                0x0
+#define VGT_VS_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                                  0x00000FFFL
+//VGT_GS_MAX_WAVE_ID
+#define VGT_GS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                                0x0
+#define VGT_GS_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                                  0x00000FFFL
+//GFX_PIPE_CONTROL
+#define GFX_PIPE_CONTROL__HYSTERESIS_CNT__SHIFT                                                               0x0
+#define GFX_PIPE_CONTROL__RESERVED__SHIFT                                                                     0xd
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN__SHIFT                                                           0x10
+#define GFX_PIPE_CONTROL__HYSTERESIS_CNT_MASK                                                                 0x00001FFFL
+#define GFX_PIPE_CONTROL__RESERVED_MASK                                                                       0x0000E000L
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN_MASK                                                             0x00010000L
+//CC_GC_SHADER_ARRAY_CONFIG
+#define CC_GC_SHADER_ARRAY_CONFIG__WRITE_DIS__SHIFT                                                           0x0
+#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT                                                        0x10
+#define CC_GC_SHADER_ARRAY_CONFIG__WRITE_DIS_MASK                                                             0x00000001L
+#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK                                                          0xFFFF0000L
+//GC_USER_SHADER_ARRAY_CONFIG
+#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT                                                      0x10
+#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK                                                        0xFFFF0000L
+//VGT_DMA_PRIMITIVE_TYPE
+#define VGT_DMA_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT                                                              0x0
+#define VGT_DMA_PRIMITIVE_TYPE__PRIM_TYPE_MASK                                                                0x0000003FL
+//VGT_DMA_CONTROL
+#define VGT_DMA_CONTROL__PRIMGROUP_SIZE__SHIFT                                                                0x0
+#define VGT_DMA_CONTROL__IA_SWITCH_ON_EOP__SHIFT                                                              0x11
+#define VGT_DMA_CONTROL__SWITCH_ON_EOI__SHIFT                                                                 0x13
+#define VGT_DMA_CONTROL__WD_SWITCH_ON_EOP__SHIFT                                                              0x14
+#define VGT_DMA_CONTROL__EN_INST_OPT_BASIC__SHIFT                                                             0x15
+#define VGT_DMA_CONTROL__EN_INST_OPT_ADV__SHIFT                                                               0x16
+#define VGT_DMA_CONTROL__HW_USE_ONLY__SHIFT                                                                   0x17
+#define VGT_DMA_CONTROL__PRIMGROUP_SIZE_MASK                                                                  0x0000FFFFL
+#define VGT_DMA_CONTROL__IA_SWITCH_ON_EOP_MASK                                                                0x00020000L
+#define VGT_DMA_CONTROL__SWITCH_ON_EOI_MASK                                                                   0x00080000L
+#define VGT_DMA_CONTROL__WD_SWITCH_ON_EOP_MASK                                                                0x00100000L
+#define VGT_DMA_CONTROL__EN_INST_OPT_BASIC_MASK                                                               0x00200000L
+#define VGT_DMA_CONTROL__EN_INST_OPT_ADV_MASK                                                                 0x00400000L
+#define VGT_DMA_CONTROL__HW_USE_ONLY_MASK                                                                     0x00800000L
+//VGT_DMA_LS_HS_CONFIG
+#define VGT_DMA_LS_HS_CONFIG__HS_NUM_INPUT_CP__SHIFT                                                          0x8
+#define VGT_DMA_LS_HS_CONFIG__HS_NUM_INPUT_CP_MASK                                                            0x00003F00L
+//WD_BUF_RESOURCE_1
+#define WD_BUF_RESOURCE_1__POS_BUF_SIZE__SHIFT                                                                0x0
+#define WD_BUF_RESOURCE_1__INDEX_BUF_SIZE__SHIFT                                                              0x10
+#define WD_BUF_RESOURCE_1__POS_BUF_SIZE_MASK                                                                  0x0000FFFFL
+#define WD_BUF_RESOURCE_1__INDEX_BUF_SIZE_MASK                                                                0xFFFF0000L
+//WD_BUF_RESOURCE_2
+#define WD_BUF_RESOURCE_2__PARAM_BUF_SIZE__SHIFT                                                              0x0
+#define WD_BUF_RESOURCE_2__ADDR_MODE__SHIFT                                                                   0xf
+#define WD_BUF_RESOURCE_2__CNTL_SB_BUF_SIZE__SHIFT                                                            0x10
+#define WD_BUF_RESOURCE_2__PARAM_BUF_SIZE_MASK                                                                0x00001FFFL
+#define WD_BUF_RESOURCE_2__ADDR_MODE_MASK                                                                     0x00008000L
+#define WD_BUF_RESOURCE_2__CNTL_SB_BUF_SIZE_MASK                                                              0xFFFF0000L
+//PA_CL_CNTL_STATUS
+#define PA_CL_CNTL_STATUS__UTC_FAULT_DETECTED__SHIFT                                                          0x0
+#define PA_CL_CNTL_STATUS__UTC_RETRY_DETECTED__SHIFT                                                          0x1
+#define PA_CL_CNTL_STATUS__UTC_PRT_DETECTED__SHIFT                                                            0x2
+#define PA_CL_CNTL_STATUS__UTC_FAULT_DETECTED_MASK                                                            0x00000001L
+#define PA_CL_CNTL_STATUS__UTC_RETRY_DETECTED_MASK                                                            0x00000002L
+#define PA_CL_CNTL_STATUS__UTC_PRT_DETECTED_MASK                                                              0x00000004L
+//PA_CL_ENHANCE
+#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA__SHIFT                                                            0x0
+#define PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT                                                                    0x1
+#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL__SHIFT                                                          0x3
+#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE__SHIFT                                                             0x4
+#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL__SHIFT                                                              0x5
+#define PA_CL_ENHANCE__IGNORE_PIPELINE_RESET__SHIFT                                                           0x6
+#define PA_CL_ENHANCE__KILL_INNER_EDGE_FLAGS__SHIFT                                                           0x7
+#define PA_CL_ENHANCE__NGG_PA_TO_ALL_SC__SHIFT                                                                0x8
+#define PA_CL_ENHANCE__TC_LATENCY_TIME_STAMP_RESOLUTION__SHIFT                                                0x9
+#define PA_CL_ENHANCE__NGG_BYPASS_PRIM_FILTER__SHIFT                                                          0xb
+#define PA_CL_ENHANCE__NGG_SIDEBAND_MEMORY_DEPTH__SHIFT                                                       0xc
+#define PA_CL_ENHANCE__NGG_PRIM_INDICES_FIFO_DEPTH__SHIFT                                                     0xe
+#define PA_CL_ENHANCE__PROG_NEAR_CLIP_PLANE_ENABLE__SHIFT                                                     0x11
+#define PA_CL_ENHANCE__OUTPUT_SWITCH_TO_LEGACY_EVENT__SHIFT                                                   0x12
+#define PA_CL_ENHANCE__NO_SWITCH_TO_LEGACY_AFTER_VMID_RESET__SHIFT                                            0x13
+#define PA_CL_ENHANCE__POLY_INNER_EDGE_FLAG_DISABLE__SHIFT                                                    0x14
+#define PA_CL_ENHANCE__TC_REQUEST_PERF_CNTR_ENABLE__SHIFT                                                     0x15
+#define PA_CL_ENHANCE__ECO_SPARE3__SHIFT                                                                      0x1c
+#define PA_CL_ENHANCE__ECO_SPARE2__SHIFT                                                                      0x1d
+#define PA_CL_ENHANCE__ECO_SPARE1__SHIFT                                                                      0x1e
+#define PA_CL_ENHANCE__ECO_SPARE0__SHIFT                                                                      0x1f
+#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK                                                              0x00000001L
+#define PA_CL_ENHANCE__NUM_CLIP_SEQ_MASK                                                                      0x00000006L
+#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL_MASK                                                            0x00000008L
+#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE_MASK                                                               0x00000010L
+#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL_MASK                                                                0x00000020L
+#define PA_CL_ENHANCE__IGNORE_PIPELINE_RESET_MASK                                                             0x00000040L
+#define PA_CL_ENHANCE__KILL_INNER_EDGE_FLAGS_MASK                                                             0x00000080L
+#define PA_CL_ENHANCE__NGG_PA_TO_ALL_SC_MASK                                                                  0x00000100L
+#define PA_CL_ENHANCE__TC_LATENCY_TIME_STAMP_RESOLUTION_MASK                                                  0x00000600L
+#define PA_CL_ENHANCE__NGG_BYPASS_PRIM_FILTER_MASK                                                            0x00000800L
+#define PA_CL_ENHANCE__NGG_SIDEBAND_MEMORY_DEPTH_MASK                                                         0x00003000L
+#define PA_CL_ENHANCE__NGG_PRIM_INDICES_FIFO_DEPTH_MASK                                                       0x0001C000L
+#define PA_CL_ENHANCE__PROG_NEAR_CLIP_PLANE_ENABLE_MASK                                                       0x00020000L
+#define PA_CL_ENHANCE__OUTPUT_SWITCH_TO_LEGACY_EVENT_MASK                                                     0x00040000L
+#define PA_CL_ENHANCE__NO_SWITCH_TO_LEGACY_AFTER_VMID_RESET_MASK                                              0x00080000L
+#define PA_CL_ENHANCE__POLY_INNER_EDGE_FLAG_DISABLE_MASK                                                      0x00100000L
+#define PA_CL_ENHANCE__TC_REQUEST_PERF_CNTR_ENABLE_MASK                                                       0x00200000L
+#define PA_CL_ENHANCE__ECO_SPARE3_MASK                                                                        0x10000000L
+#define PA_CL_ENHANCE__ECO_SPARE2_MASK                                                                        0x20000000L
+#define PA_CL_ENHANCE__ECO_SPARE1_MASK                                                                        0x40000000L
+#define PA_CL_ENHANCE__ECO_SPARE0_MASK                                                                        0x80000000L
+//PA_CL_RESET_DEBUG
+#define PA_CL_RESET_DEBUG__CL_TRIV_DISC_DISABLE__SHIFT                                                        0x0
+#define PA_CL_RESET_DEBUG__CL_TRIV_DISC_DISABLE_MASK                                                          0x00000001L
+//PA_SU_CNTL_STATUS
+#define PA_SU_CNTL_STATUS__SU_BUSY__SHIFT                                                                     0x1f
+#define PA_SU_CNTL_STATUS__SU_BUSY_MASK                                                                       0x80000000L
+//PA_SC_FIFO_DEPTH_CNTL
+#define PA_SC_FIFO_DEPTH_CNTL__DEPTH__SHIFT                                                                   0x0
+#define PA_SC_FIFO_DEPTH_CNTL__DEPTH_MASK                                                                     0x000003FFL
+//PA_SC_P3D_TRAP_SCREEN_HV_LOCK
+#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                         0x0
+#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                           0x00000001L
+//PA_SC_HP3D_TRAP_SCREEN_HV_LOCK
+#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                        0x0
+#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                          0x00000001L
+//PA_SC_TRAP_SCREEN_HV_LOCK
+#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                             0x0
+#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                               0x00000001L
+//PA_SC_FORCE_EOV_MAX_CNTS
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT                                                0x0
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT                                                0x10
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT_MASK                                                  0x0000FFFFL
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT_MASK                                                  0xFFFF0000L
+//PA_SC_BINNER_EVENT_CNTL_0
+#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_0__SHIFT                                                          0x0
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS1__SHIFT                                              0x2
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS2__SHIFT                                              0x4
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS3__SHIFT                                              0x6
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_TS__SHIFT                                                      0x8
+#define PA_SC_BINNER_EVENT_CNTL_0__CONTEXT_DONE__SHIFT                                                        0xa
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH__SHIFT                                                         0xc
+#define PA_SC_BINNER_EVENT_CNTL_0__CS_PARTIAL_FLUSH__SHIFT                                                    0xe
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_SYNC__SHIFT                                                  0x10
+#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_9__SHIFT                                                          0x12
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_RESET__SHIFT                                                 0x14
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_INCR_DE__SHIFT                                                 0x16
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_IB_END__SHIFT                                                  0x18
+#define PA_SC_BINNER_EVENT_CNTL_0__RST_PIX_CNT__SHIFT                                                         0x1a
+#define PA_SC_BINNER_EVENT_CNTL_0__BREAK_BATCH__SHIFT                                                         0x1c
+#define PA_SC_BINNER_EVENT_CNTL_0__VS_PARTIAL_FLUSH__SHIFT                                                    0x1e
+#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_0_MASK                                                            0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS1_MASK                                                0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS2_MASK                                                0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS3_MASK                                                0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_TS_MASK                                                        0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_0__CONTEXT_DONE_MASK                                                          0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_MASK                                                           0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_0__CS_PARTIAL_FLUSH_MASK                                                      0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_SYNC_MASK                                                    0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_9_MASK                                                            0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_RESET_MASK                                                   0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_INCR_DE_MASK                                                   0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_IB_END_MASK                                                    0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_0__RST_PIX_CNT_MASK                                                           0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_0__BREAK_BATCH_MASK                                                           0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_0__VS_PARTIAL_FLUSH_MASK                                                      0xC0000000L
+//PA_SC_BINNER_EVENT_CNTL_1
+#define PA_SC_BINNER_EVENT_CNTL_1__PS_PARTIAL_FLUSH__SHIFT                                                    0x0
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_HS_OUTPUT__SHIFT                                                     0x2
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_DFSM__SHIFT                                                          0x4
+#define PA_SC_BINNER_EVENT_CNTL_1__RESET_TO_LOWEST_VGT__SHIFT                                                 0x6
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_TS_EVENT__SHIFT                                        0x8
+#define PA_SC_BINNER_EVENT_CNTL_1__ZPASS_DONE__SHIFT                                                          0xa
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_EVENT__SHIFT                                           0xc
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_START__SHIFT                                                   0xe
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_STOP__SHIFT                                                    0x10
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_START__SHIFT                                                  0x12
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_STOP__SHIFT                                                   0x14
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_SAMPLE__SHIFT                                                  0x16
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_ES_OUTPUT__SHIFT                                                     0x18
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_GS_OUTPUT__SHIFT                                                     0x1a
+#define PA_SC_BINNER_EVENT_CNTL_1__SAMPLE_PIPELINESTAT__SHIFT                                                 0x1c
+#define PA_SC_BINNER_EVENT_CNTL_1__SO_VGTSTREAMOUT_FLUSH__SHIFT                                               0x1e
+#define PA_SC_BINNER_EVENT_CNTL_1__PS_PARTIAL_FLUSH_MASK                                                      0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_HS_OUTPUT_MASK                                                       0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_DFSM_MASK                                                            0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_1__RESET_TO_LOWEST_VGT_MASK                                                   0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_TS_EVENT_MASK                                          0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_1__ZPASS_DONE_MASK                                                            0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_EVENT_MASK                                             0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_START_MASK                                                     0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_STOP_MASK                                                      0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_START_MASK                                                    0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_STOP_MASK                                                     0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_SAMPLE_MASK                                                    0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_ES_OUTPUT_MASK                                                       0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_GS_OUTPUT_MASK                                                       0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_1__SAMPLE_PIPELINESTAT_MASK                                                   0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_1__SO_VGTSTREAMOUT_FLUSH_MASK                                                 0xC0000000L
+//PA_SC_BINNER_EVENT_CNTL_2
+#define PA_SC_BINNER_EVENT_CNTL_2__SAMPLE_STREAMOUTSTATS__SHIFT                                               0x0
+#define PA_SC_BINNER_EVENT_CNTL_2__RESET_VTX_CNT__SHIFT                                                       0x2
+#define PA_SC_BINNER_EVENT_CNTL_2__BLOCK_CONTEXT_DONE__SHIFT                                                  0x4
+#define PA_SC_BINNER_EVENT_CNTL_2__CS_CONTEXT_DONE__SHIFT                                                     0x6
+#define PA_SC_BINNER_EVENT_CNTL_2__VGT_FLUSH__SHIFT                                                           0x8
+#define PA_SC_BINNER_EVENT_CNTL_2__TGID_ROLLOVER__SHIFT                                                       0xa
+#define PA_SC_BINNER_EVENT_CNTL_2__SQ_NON_EVENT__SHIFT                                                        0xc
+#define PA_SC_BINNER_EVENT_CNTL_2__SC_SEND_DB_VPZ__SHIFT                                                      0xe
+#define PA_SC_BINNER_EVENT_CNTL_2__BOTTOM_OF_PIPE_TS__SHIFT                                                   0x10
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_SX_TS__SHIFT                                                         0x12
+#define PA_SC_BINNER_EVENT_CNTL_2__DB_CACHE_FLUSH_AND_INV__SHIFT                                              0x14
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_DATA_TS__SHIFT                                            0x16
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_META__SHIFT                                               0x18
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_DATA_TS__SHIFT                                            0x1a
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_META__SHIFT                                               0x1c
+#define PA_SC_BINNER_EVENT_CNTL_2__CS_DONE__SHIFT                                                             0x1e
+#define PA_SC_BINNER_EVENT_CNTL_2__SAMPLE_STREAMOUTSTATS_MASK                                                 0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_2__RESET_VTX_CNT_MASK                                                         0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_2__BLOCK_CONTEXT_DONE_MASK                                                    0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_2__CS_CONTEXT_DONE_MASK                                                       0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_2__VGT_FLUSH_MASK                                                             0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_2__TGID_ROLLOVER_MASK                                                         0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_2__SQ_NON_EVENT_MASK                                                          0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_2__SC_SEND_DB_VPZ_MASK                                                        0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_2__BOTTOM_OF_PIPE_TS_MASK                                                     0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_SX_TS_MASK                                                           0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_2__DB_CACHE_FLUSH_AND_INV_MASK                                                0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_DATA_TS_MASK                                              0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_META_MASK                                                 0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_DATA_TS_MASK                                              0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_META_MASK                                                 0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_2__CS_DONE_MASK                                                               0xC0000000L
+//PA_SC_BINNER_EVENT_CNTL_3
+#define PA_SC_BINNER_EVENT_CNTL_3__PS_DONE__SHIFT                                                             0x0
+#define PA_SC_BINNER_EVENT_CNTL_3__FLUSH_AND_INV_CB_PIXEL_DATA__SHIFT                                         0x2
+#define PA_SC_BINNER_EVENT_CNTL_3__SX_CB_RAT_ACK_REQUEST__SHIFT                                               0x4
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_START__SHIFT                                                  0x6
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_STOP__SHIFT                                                   0x8
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_MARKER__SHIFT                                                 0xa
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FLUSH__SHIFT                                                  0xc
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FINISH__SHIFT                                                 0xe
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_CONTROL__SHIFT                                             0x10
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_DUMP__SHIFT                                                0x12
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_RESET__SHIFT                                               0x14
+#define PA_SC_BINNER_EVENT_CNTL_3__CONTEXT_SUSPEND__SHIFT                                                     0x16
+#define PA_SC_BINNER_EVENT_CNTL_3__OFFCHIP_HS_DEALLOC__SHIFT                                                  0x18
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_NGG_PIPELINE__SHIFT                                                 0x1a
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_LEGACY_PIPELINE__SHIFT                                              0x1c
+#define PA_SC_BINNER_EVENT_CNTL_3__RESERVED_63__SHIFT                                                         0x1e
+#define PA_SC_BINNER_EVENT_CNTL_3__PS_DONE_MASK                                                               0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_3__FLUSH_AND_INV_CB_PIXEL_DATA_MASK                                           0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_3__SX_CB_RAT_ACK_REQUEST_MASK                                                 0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_START_MASK                                                    0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_STOP_MASK                                                     0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_MARKER_MASK                                                   0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FLUSH_MASK                                                    0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FINISH_MASK                                                   0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_CONTROL_MASK                                               0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_DUMP_MASK                                                  0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_RESET_MASK                                                 0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_3__CONTEXT_SUSPEND_MASK                                                       0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_3__OFFCHIP_HS_DEALLOC_MASK                                                    0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_NGG_PIPELINE_MASK                                                   0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_LEGACY_PIPELINE_MASK                                                0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_3__RESERVED_63_MASK                                                           0xC0000000L
+//PA_SC_BINNER_TIMEOUT_COUNTER
+#define PA_SC_BINNER_TIMEOUT_COUNTER__THRESHOLD__SHIFT                                                        0x0
+#define PA_SC_BINNER_TIMEOUT_COUNTER__THRESHOLD_MASK                                                          0xFFFFFFFFL
+//PA_SC_BINNER_PERF_CNTL_0
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_PRIMS_THRESHOLD__SHIFT                                         0x0
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_PRIMS_THRESHOLD__SHIFT                                       0xa
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_CONTEXT_THRESHOLD__SHIFT                                       0x14
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_CONTEXT_THRESHOLD__SHIFT                                     0x17
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_PRIMS_THRESHOLD_MASK                                           0x000003FFL
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_PRIMS_THRESHOLD_MASK                                         0x000FFC00L
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_CONTEXT_THRESHOLD_MASK                                         0x00700000L
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_CONTEXT_THRESHOLD_MASK                                       0x03800000L
+//PA_SC_BINNER_PERF_CNTL_1
+#define PA_SC_BINNER_PERF_CNTL_1__BIN_HIST_NUM_PERSISTENT_STATE_THRESHOLD__SHIFT                              0x0
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_PERSISTENT_STATE_THRESHOLD__SHIFT                            0x5
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_TRIV_REJECTED_PRIMS_THRESHOLD__SHIFT                         0xa
+#define PA_SC_BINNER_PERF_CNTL_1__BIN_HIST_NUM_PERSISTENT_STATE_THRESHOLD_MASK                                0x0000001FL
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_PERSISTENT_STATE_THRESHOLD_MASK                              0x000003E0L
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_TRIV_REJECTED_PRIMS_THRESHOLD_MASK                           0x03FFFC00L
+//PA_SC_BINNER_PERF_CNTL_2
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_ROWS_PER_PRIM_THRESHOLD__SHIFT                               0x0
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_COLUMNS_PER_ROW_THRESHOLD__SHIFT                             0xb
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_ROWS_PER_PRIM_THRESHOLD_MASK                                 0x000007FFL
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_COLUMNS_PER_ROW_THRESHOLD_MASK                               0x003FF800L
+//PA_SC_BINNER_PERF_CNTL_3
+#define PA_SC_BINNER_PERF_CNTL_3__BATCH_HIST_NUM_PS_WAVE_BREAKS_THRESHOLD__SHIFT                              0x0
+#define PA_SC_BINNER_PERF_CNTL_3__BATCH_HIST_NUM_PS_WAVE_BREAKS_THRESHOLD_MASK                                0xFFFFFFFFL
+//PA_SC_ENHANCE_2
+#define PA_SC_ENHANCE_2__RESERVED_0__SHIFT                                                                    0x0
+#define PA_SC_ENHANCE_2__RESERVED_1__SHIFT                                                                    0x1
+#define PA_SC_ENHANCE_2__RESERVED_2__SHIFT                                                                    0x2
+#define PA_SC_ENHANCE_2__RESERVED_3__SHIFT                                                                    0x3
+#define PA_SC_ENHANCE_2__RESERVED_4__SHIFT                                                                    0x4
+#define PA_SC_ENHANCE_2__RESERVED_5__SHIFT                                                                    0x5
+#define PA_SC_ENHANCE_2__ENABLE_SC_SEND_DB_VPZ_FOR_COMPOUND_INDEX_EN__SHIFT                                   0x6
+#define PA_SC_ENHANCE_2__ENABLE_SC_SEND_DB_VPZ_FOR_EN_PIPELINE_PRIMID__SHIFT                                  0x7
+#define PA_SC_ENHANCE_2__RSVD__SHIFT                                                                          0x8
+#define PA_SC_ENHANCE_2__RESERVED_0_MASK                                                                      0x00000001L
+#define PA_SC_ENHANCE_2__RESERVED_1_MASK                                                                      0x00000002L
+#define PA_SC_ENHANCE_2__RESERVED_2_MASK                                                                      0x00000004L
+#define PA_SC_ENHANCE_2__RESERVED_3_MASK                                                                      0x00000008L
+#define PA_SC_ENHANCE_2__RESERVED_4_MASK                                                                      0x00000010L
+#define PA_SC_ENHANCE_2__RESERVED_5_MASK                                                                      0x00000020L
+#define PA_SC_ENHANCE_2__ENABLE_SC_SEND_DB_VPZ_FOR_COMPOUND_INDEX_EN_MASK                                     0x00000040L
+#define PA_SC_ENHANCE_2__ENABLE_SC_SEND_DB_VPZ_FOR_EN_PIPELINE_PRIMID_MASK                                    0x00000080L
+#define PA_SC_ENHANCE_2__RSVD_MASK                                                                            0xFFFFFF00L
+//PA_SC_FIFO_SIZE
+#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT                                                    0x0
+#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT                                                     0x6
+#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT                                                         0xf
+#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT                                                      0x15
+#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE_MASK                                                      0x0000003FL
+#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE_MASK                                                       0x00007FC0L
+#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE_MASK                                                           0x001F8000L
+#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE_MASK                                                        0xFFE00000L
+//PA_SC_IF_FIFO_SIZE
+#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE__SHIFT                                                    0x0
+#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE__SHIFT                                                    0x6
+#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE__SHIFT                                                        0xc
+#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE__SHIFT                                                        0x12
+#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE_MASK                                                      0x0000003FL
+#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE_MASK                                                      0x00000FC0L
+#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE_MASK                                                          0x0003F000L
+#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE_MASK                                                          0x00FC0000L
+//PA_SC_PKR_WAVE_TABLE_CNTL
+#define PA_SC_PKR_WAVE_TABLE_CNTL__SIZE__SHIFT                                                                0x0
+#define PA_SC_PKR_WAVE_TABLE_CNTL__SIZE_MASK                                                                  0x0000003FL
+//PA_UTCL1_CNTL1
+#define PA_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                               0x0
+#define PA_UTCL1_CNTL1__GPUVM_64K_DEFAULT__SHIFT                                                              0x1
+#define PA_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                                0x2
+#define PA_UTCL1_CNTL1__RESP_MODE__SHIFT                                                                      0x3
+#define PA_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                                0x5
+#define PA_UTCL1_CNTL1__CLIENTID__SHIFT                                                                       0x7
+#define PA_UTCL1_CNTL1__SPARE__SHIFT                                                                          0x10
+#define PA_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                              0x11
+#define PA_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                           0x12
+#define PA_UTCL1_CNTL1__REG_INV_VMID__SHIFT                                                                   0x13
+#define PA_UTCL1_CNTL1__REG_INV_ALL_VMID__SHIFT                                                               0x17
+#define PA_UTCL1_CNTL1__REG_INV_TOGGLE__SHIFT                                                                 0x18
+#define PA_UTCL1_CNTL1__INVALIDATE_ALL_VMID__SHIFT                                                            0x19
+#define PA_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                                     0x1a
+#define PA_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                                 0x1b
+#define PA_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                         0x1c
+#define PA_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                         0x1e
+#define PA_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                                 0x00000001L
+#define PA_UTCL1_CNTL1__GPUVM_64K_DEFAULT_MASK                                                                0x00000002L
+#define PA_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                                  0x00000004L
+#define PA_UTCL1_CNTL1__RESP_MODE_MASK                                                                        0x00000018L
+#define PA_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                                  0x00000060L
+#define PA_UTCL1_CNTL1__CLIENTID_MASK                                                                         0x0000FF80L
+#define PA_UTCL1_CNTL1__SPARE_MASK                                                                            0x00010000L
+#define PA_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                                0x00020000L
+#define PA_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                             0x00040000L
+#define PA_UTCL1_CNTL1__REG_INV_VMID_MASK                                                                     0x00780000L
+#define PA_UTCL1_CNTL1__REG_INV_ALL_VMID_MASK                                                                 0x00800000L
+#define PA_UTCL1_CNTL1__REG_INV_TOGGLE_MASK                                                                   0x01000000L
+#define PA_UTCL1_CNTL1__INVALIDATE_ALL_VMID_MASK                                                              0x02000000L
+#define PA_UTCL1_CNTL1__FORCE_MISS_MASK                                                                       0x04000000L
+#define PA_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                                   0x08000000L
+#define PA_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                           0x30000000L
+#define PA_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                           0xC0000000L
+//PA_UTCL1_CNTL2
+#define PA_UTCL1_CNTL2__SPARE1__SHIFT                                                                         0x0
+#define PA_UTCL1_CNTL2__SPARE2__SHIFT                                                                         0x8
+#define PA_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                 0x9
+#define PA_UTCL1_CNTL2__LINE_VALID__SHIFT                                                                     0xa
+#define PA_UTCL1_CNTL2__SPARE3__SHIFT                                                                         0xb
+#define PA_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                                 0xc
+#define PA_UTCL1_CNTL2__ENABLE_SHOOTDOWN_OPT__SHIFT                                                           0xd
+#define PA_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                                    0xe
+#define PA_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                            0xf
+#define PA_UTCL1_CNTL2__SPARE4__SHIFT                                                                         0x10
+#define PA_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR__SHIFT                                                        0x12
+#define PA_UTCL1_CNTL2__PERF_EVENT_RD_WR__SHIFT                                                               0x13
+#define PA_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID__SHIFT                                                         0x14
+#define PA_UTCL1_CNTL2__PERF_EVENT_VMID__SHIFT                                                                0x15
+#define PA_UTCL1_CNTL2__SPARE5__SHIFT                                                                         0x19
+#define PA_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                           0x1a
+#define PA_UTCL1_CNTL2__RESERVED__SHIFT                                                                       0x1b
+#define PA_UTCL1_CNTL2__SPARE1_MASK                                                                           0x000000FFL
+#define PA_UTCL1_CNTL2__SPARE2_MASK                                                                           0x00000100L
+#define PA_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                                   0x00000200L
+#define PA_UTCL1_CNTL2__LINE_VALID_MASK                                                                       0x00000400L
+#define PA_UTCL1_CNTL2__SPARE3_MASK                                                                           0x00000800L
+#define PA_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                                   0x00001000L
+#define PA_UTCL1_CNTL2__ENABLE_SHOOTDOWN_OPT_MASK                                                             0x00002000L
+#define PA_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                                      0x00004000L
+#define PA_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                              0x00008000L
+#define PA_UTCL1_CNTL2__SPARE4_MASK                                                                           0x00030000L
+#define PA_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR_MASK                                                          0x00040000L
+#define PA_UTCL1_CNTL2__PERF_EVENT_RD_WR_MASK                                                                 0x00080000L
+#define PA_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID_MASK                                                           0x00100000L
+#define PA_UTCL1_CNTL2__PERF_EVENT_VMID_MASK                                                                  0x01E00000L
+#define PA_UTCL1_CNTL2__SPARE5_MASK                                                                           0x02000000L
+#define PA_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                             0x04000000L
+#define PA_UTCL1_CNTL2__RESERVED_MASK                                                                         0xF8000000L
+//PA_SIDEBAND_REQUEST_DELAYS
+#define PA_SIDEBAND_REQUEST_DELAYS__RETRY_DELAY__SHIFT                                                        0x0
+#define PA_SIDEBAND_REQUEST_DELAYS__INITIAL_DELAY__SHIFT                                                      0x10
+#define PA_SIDEBAND_REQUEST_DELAYS__RETRY_DELAY_MASK                                                          0x0000FFFFL
+#define PA_SIDEBAND_REQUEST_DELAYS__INITIAL_DELAY_MASK                                                        0xFFFF0000L
+//PA_SC_ENHANCE
+#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER__SHIFT                                                       0x0
+#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX__SHIFT                                                          0x1
+#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX__SHIFT                                                        0x2
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS__SHIFT                                                  0x3
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID__SHIFT                                               0x4
+#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX__SHIFT                                                             0x5
+#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER__SHIFT                                                     0x6
+#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION__SHIFT                                              0x7
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM__SHIFT                                                   0x8
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE__SHIFT                                              0x9
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE__SHIFT                                                   0xa
+#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE__SHIFT                                                          0xb
+#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS__SHIFT                                          0xc
+#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE__SHIFT                                                 0xd
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE__SHIFT                                             0xe
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE__SHIFT                                                   0xf
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST__SHIFT                                   0x10
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING__SHIFT                                        0x11
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY__SHIFT                               0x12
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING__SHIFT                               0x13
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING__SHIFT                              0x14
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS__SHIFT                                 0x15
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID__SHIFT                                   0x16
+#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO__SHIFT                           0x17
+#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT__SHIFT                                          0x18
+#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING__SHIFT                                       0x19
+#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET__SHIFT                                                  0x1a
+#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET__SHIFT                                              0x1b
+#define PA_SC_ENHANCE__IOO_DISABLE_SCAN_UNSELECTED_FIFOS_FOR_DUAL_GFX_RING_CHANGE__SHIFT                      0x1c
+#define PA_SC_ENHANCE__OOO_USE_ABSOLUTE_FIFO_COUNT_IN_THRESHOLD_SWITCHING__SHIFT                              0x1d
+#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER_MASK                                                         0x00000001L
+#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX_MASK                                                            0x00000002L
+#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX_MASK                                                          0x00000004L
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS_MASK                                                    0x00000008L
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID_MASK                                                 0x00000010L
+#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX_MASK                                                               0x00000020L
+#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER_MASK                                                       0x00000040L
+#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION_MASK                                                0x00000080L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM_MASK                                                     0x00000100L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE_MASK                                                0x00000200L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE_MASK                                                     0x00000400L
+#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE_MASK                                                            0x00000800L
+#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS_MASK                                            0x00001000L
+#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE_MASK                                                   0x00002000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE_MASK                                               0x00004000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE_MASK                                                     0x00008000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST_MASK                                     0x00010000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING_MASK                                          0x00020000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY_MASK                                 0x00040000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING_MASK                                 0x00080000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING_MASK                                0x00100000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS_MASK                                   0x00200000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID_MASK                                     0x00400000L
+#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO_MASK                             0x00800000L
+#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT_MASK                                            0x01000000L
+#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING_MASK                                         0x02000000L
+#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET_MASK                                                    0x04000000L
+#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET_MASK                                                0x08000000L
+#define PA_SC_ENHANCE__IOO_DISABLE_SCAN_UNSELECTED_FIFOS_FOR_DUAL_GFX_RING_CHANGE_MASK                        0x10000000L
+#define PA_SC_ENHANCE__OOO_USE_ABSOLUTE_FIFO_COUNT_IN_THRESHOLD_SWITCHING_MASK                                0x20000000L
+//PA_SC_ENHANCE_1
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_ENABLE__SHIFT                                                0x0
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE__SHIFT                                                       0x1
+#define PA_SC_ENHANCE_1__DISABLE_SC_BINNING__SHIFT                                                            0x3
+#define PA_SC_ENHANCE_1__BYPASS_PBB__SHIFT                                                                    0x4
+#define PA_SC_ENHANCE_1__ECO_SPARE0__SHIFT                                                                    0x5
+#define PA_SC_ENHANCE_1__ECO_SPARE1__SHIFT                                                                    0x6
+#define PA_SC_ENHANCE_1__ECO_SPARE2__SHIFT                                                                    0x7
+#define PA_SC_ENHANCE_1__ECO_SPARE3__SHIFT                                                                    0x8
+#define PA_SC_ENHANCE_1__DISABLE_SC_PROCESS_RESET_PBB__SHIFT                                                  0x9
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_OPT__SHIFT                                                       0xa
+#define PA_SC_ENHANCE_1__ENABLE_DFSM_FLUSH_EVENT_TO_FLUSH_POPS_CAM__SHIFT                                     0xb
+#define PA_SC_ENHANCE_1__DEBUG_PIXEL_PICKER_XY_UNPACK__SHIFT                                                  0xc
+#define PA_SC_ENHANCE_1__DISABLE_PACKER_GRAD_FDCE_ENHANCE__SHIFT                                              0xd
+#define PA_SC_ENHANCE_1__DISABLE_SC_DB_TILE_INTF_FINE_CLOCK_GATE__SHIFT                                       0xe
+#define PA_SC_ENHANCE_1__DISABLE_SC_PIPELINE_RESET_LEGACY_MODE_TRANSITION__SHIFT                              0xf
+#define PA_SC_ENHANCE_1__DISABLE_PACKER_ODC_ENHANCE__SHIFT                                                    0x10
+#define PA_SC_ENHANCE_1__ALLOW_SCALE_LINE_WIDTH_PAD_WITH_BINNING__SHIFT                                       0x11
+#define PA_SC_ENHANCE_1__OPTIMAL_BIN_SELECTION__SHIFT                                                         0x12
+#define PA_SC_ENHANCE_1__DISABLE_FORCE_SOP_ALL_EVENTS__SHIFT                                                  0x13
+#define PA_SC_ENHANCE_1__DISABLE_PBB_CLK_OPTIMIZATION__SHIFT                                                  0x14
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_CLK_OPTIMIZATION__SHIFT                                          0x15
+#define PA_SC_ENHANCE_1__DISABLE_PBB_BINNING_CLK_OPTIMIZATION__SHIFT                                          0x16
+#define PA_SC_ENHANCE_1__DISABLE_INTF_CG__SHIFT                                                               0x17
+#define PA_SC_ENHANCE_1__IOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT__SHIFT                                        0x18
+#define PA_SC_ENHANCE_1__DISABLE_SHADER_PROFILING_FOR_POWER__SHIFT                                            0x19
+#define PA_SC_ENHANCE_1__FLUSH_ON_BINNING_TRANSITION__SHIFT                                                   0x1a
+#define PA_SC_ENHANCE_1__DISABLE_QUAD_PROC_FDCE_ENHANCE__SHIFT                                                0x1b
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX__SHIFT                                                  0x1c
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX_1__SHIFT                                                0x1d
+#define PA_SC_ENHANCE_1__PASS_VPZ_EVENT_TO_SPI__SHIFT                                                         0x1e
+#define PA_SC_ENHANCE_1__RSVD__SHIFT                                                                          0x1f
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_ENABLE_MASK                                                  0x00000001L
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_MASK                                                         0x00000006L
+#define PA_SC_ENHANCE_1__DISABLE_SC_BINNING_MASK                                                              0x00000008L
+#define PA_SC_ENHANCE_1__BYPASS_PBB_MASK                                                                      0x00000010L
+#define PA_SC_ENHANCE_1__ECO_SPARE0_MASK                                                                      0x00000020L
+#define PA_SC_ENHANCE_1__ECO_SPARE1_MASK                                                                      0x00000040L
+#define PA_SC_ENHANCE_1__ECO_SPARE2_MASK                                                                      0x00000080L
+#define PA_SC_ENHANCE_1__ECO_SPARE3_MASK                                                                      0x00000100L
+#define PA_SC_ENHANCE_1__DISABLE_SC_PROCESS_RESET_PBB_MASK                                                    0x00000200L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_OPT_MASK                                                         0x00000400L
+#define PA_SC_ENHANCE_1__ENABLE_DFSM_FLUSH_EVENT_TO_FLUSH_POPS_CAM_MASK                                       0x00000800L
+#define PA_SC_ENHANCE_1__DEBUG_PIXEL_PICKER_XY_UNPACK_MASK                                                    0x00001000L
+#define PA_SC_ENHANCE_1__DISABLE_PACKER_GRAD_FDCE_ENHANCE_MASK                                                0x00002000L
+#define PA_SC_ENHANCE_1__DISABLE_SC_DB_TILE_INTF_FINE_CLOCK_GATE_MASK                                         0x00004000L
+#define PA_SC_ENHANCE_1__DISABLE_SC_PIPELINE_RESET_LEGACY_MODE_TRANSITION_MASK                                0x00008000L
+#define PA_SC_ENHANCE_1__DISABLE_PACKER_ODC_ENHANCE_MASK                                                      0x00010000L
+#define PA_SC_ENHANCE_1__ALLOW_SCALE_LINE_WIDTH_PAD_WITH_BINNING_MASK                                         0x00020000L
+#define PA_SC_ENHANCE_1__OPTIMAL_BIN_SELECTION_MASK                                                           0x00040000L
+#define PA_SC_ENHANCE_1__DISABLE_FORCE_SOP_ALL_EVENTS_MASK                                                    0x00080000L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_CLK_OPTIMIZATION_MASK                                                    0x00100000L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_CLK_OPTIMIZATION_MASK                                            0x00200000L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_BINNING_CLK_OPTIMIZATION_MASK                                            0x00400000L
+#define PA_SC_ENHANCE_1__DISABLE_INTF_CG_MASK                                                                 0x00800000L
+#define PA_SC_ENHANCE_1__IOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT_MASK                                          0x01000000L
+#define PA_SC_ENHANCE_1__DISABLE_SHADER_PROFILING_FOR_POWER_MASK                                              0x02000000L
+#define PA_SC_ENHANCE_1__FLUSH_ON_BINNING_TRANSITION_MASK                                                     0x04000000L
+#define PA_SC_ENHANCE_1__DISABLE_QUAD_PROC_FDCE_ENHANCE_MASK                                                  0x08000000L
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX_MASK                                                    0x10000000L
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX_1_MASK                                                  0x20000000L
+#define PA_SC_ENHANCE_1__PASS_VPZ_EVENT_TO_SPI_MASK                                                           0x40000000L
+#define PA_SC_ENHANCE_1__RSVD_MASK                                                                            0x80000000L
+//PA_SC_DSM_CNTL
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_0__SHIFT                                                                0x0
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_1__SHIFT                                                                0x1
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_0_MASK                                                                  0x00000001L
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_1_MASK                                                                  0x00000002L
+//PA_SC_TILE_STEERING_CREST_OVERRIDE
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__ONE_RB_MODE_ENABLE__SHIFT                                         0x0
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SE_SELECT__SHIFT                                                  0x1
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__RB_SELECT__SHIFT                                                  0x5
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__ONE_RB_MODE_ENABLE_MASK                                           0x00000001L
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SE_SELECT_MASK                                                    0x00000006L
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__RB_SELECT_MASK                                                    0x00000060L
+
+
+// addressBlock: xcd0_gc_sqdec
+//SQ_CONFIG
+#define SQ_CONFIG__DISABLE_BARRIER_WAITCNT__SHIFT                                                             0x0
+#define SQ_CONFIG__DISABLE_REPEATER_FGCG_CLOCK_GATING__SHIFT                                                  0x1
+#define SQ_CONFIG__DISABLE_SPIPRIO_OVER_USERPRIO__SHIFT                                                       0x2
+#define SQ_CONFIG__OVERRIDE_SP_MAI_ALU_BUSY__SHIFT                                                            0x3
+#define SQ_CONFIG__DISABLE_RAM_CLOCK_GATING__SHIFT                                                            0x4
+#define SQ_CONFIG__DISABLE_MAI_CO_EXEC__SHIFT                                                                 0x5
+#define SQ_CONFIG__OVERRIDE_MAI_ALU_BUSY__SHIFT                                                               0x6
+#define SQ_CONFIG__OVERRIDE_ALU_BUSY__SHIFT                                                                   0x7
+#define SQ_CONFIG__DEBUG_EN__SHIFT                                                                            0x8
+#define SQ_CONFIG__DEBUG_SINGLE_MEMOP__SHIFT                                                                  0x9
+#define SQ_CONFIG__DEBUG_ONE_INST_CLAUSE__SHIFT                                                               0xa
+#define SQ_CONFIG__OVERRIDE_LDS_IDX_BUSY__SHIFT                                                               0xb
+#define SQ_CONFIG__EARLY_TA_DONE_DISABLE__SHIFT                                                               0xc
+#define SQ_CONFIG__DUA_FLAT_LOCK_ENABLE__SHIFT                                                                0xd
+#define SQ_CONFIG__DUA_LDS_BYPASS_DISABLE__SHIFT                                                              0xe
+#define SQ_CONFIG__DUA_FLAT_LDS_PINGPONG_DISABLE__SHIFT                                                       0xf
+#define SQ_CONFIG__DISABLE_VMEM_SOFT_CLAUSE__SHIFT                                                            0x10
+#define SQ_CONFIG__DISABLE_SMEM_SOFT_CLAUSE__SHIFT                                                            0x11
+#define SQ_CONFIG__ENABLE_HIPRIO_ON_EXP_RDY_VS__SHIFT                                                         0x12
+#define SQ_CONFIG__PRIO_VAL_ON_EXP_RDY_VS__SHIFT                                                              0x13
+#define SQ_CONFIG__REPLAY_SLEEP_CNT__SHIFT                                                                    0x15
+#define SQ_CONFIG__DISABLE_SP_VGPR_WRITE_SKIP__SHIFT                                                          0x1c
+#define SQ_CONFIG__DISABLE_SP_REDUNDANT_THREAD_GATING__SHIFT                                                  0x1d
+#define SQ_CONFIG__DISABLE_FLAT_SOFT_CLAUSE__SHIFT                                                            0x1e
+#define SQ_CONFIG__DISABLE_MIMG_SOFT_CLAUSE__SHIFT                                                            0x1f
+#define SQ_CONFIG__DISABLE_BARRIER_WAITCNT_MASK                                                               0x00000001L
+#define SQ_CONFIG__DISABLE_REPEATER_FGCG_CLOCK_GATING_MASK                                                    0x00000002L
+#define SQ_CONFIG__DISABLE_SPIPRIO_OVER_USERPRIO_MASK                                                         0x00000004L
+#define SQ_CONFIG__OVERRIDE_SP_MAI_ALU_BUSY_MASK                                                              0x00000008L
+#define SQ_CONFIG__DISABLE_RAM_CLOCK_GATING_MASK                                                              0x00000010L
+#define SQ_CONFIG__DISABLE_MAI_CO_EXEC_MASK                                                                   0x00000020L
+#define SQ_CONFIG__OVERRIDE_MAI_ALU_BUSY_MASK                                                                 0x00000040L
+#define SQ_CONFIG__OVERRIDE_ALU_BUSY_MASK                                                                     0x00000080L
+#define SQ_CONFIG__DEBUG_EN_MASK                                                                              0x00000100L
+#define SQ_CONFIG__DEBUG_SINGLE_MEMOP_MASK                                                                    0x00000200L
+#define SQ_CONFIG__DEBUG_ONE_INST_CLAUSE_MASK                                                                 0x00000400L
+#define SQ_CONFIG__OVERRIDE_LDS_IDX_BUSY_MASK                                                                 0x00000800L
+#define SQ_CONFIG__EARLY_TA_DONE_DISABLE_MASK                                                                 0x00001000L
+#define SQ_CONFIG__DUA_FLAT_LOCK_ENABLE_MASK                                                                  0x00002000L
+#define SQ_CONFIG__DUA_LDS_BYPASS_DISABLE_MASK                                                                0x00004000L
+#define SQ_CONFIG__DUA_FLAT_LDS_PINGPONG_DISABLE_MASK                                                         0x00008000L
+#define SQ_CONFIG__DISABLE_VMEM_SOFT_CLAUSE_MASK                                                              0x00010000L
+#define SQ_CONFIG__DISABLE_SMEM_SOFT_CLAUSE_MASK                                                              0x00020000L
+#define SQ_CONFIG__ENABLE_HIPRIO_ON_EXP_RDY_VS_MASK                                                           0x00040000L
+#define SQ_CONFIG__PRIO_VAL_ON_EXP_RDY_VS_MASK                                                                0x00180000L
+#define SQ_CONFIG__REPLAY_SLEEP_CNT_MASK                                                                      0x0FE00000L
+#define SQ_CONFIG__DISABLE_SP_VGPR_WRITE_SKIP_MASK                                                            0x10000000L
+#define SQ_CONFIG__DISABLE_SP_REDUNDANT_THREAD_GATING_MASK                                                    0x20000000L
+#define SQ_CONFIG__DISABLE_FLAT_SOFT_CLAUSE_MASK                                                              0x40000000L
+#define SQ_CONFIG__DISABLE_MIMG_SOFT_CLAUSE_MASK                                                              0x80000000L
+//SQC_CONFIG
+#define SQC_CONFIG__INST_CACHE_SIZE__SHIFT                                                                    0x0
+#define SQC_CONFIG__DATA_CACHE_SIZE__SHIFT                                                                    0x2
+#define SQC_CONFIG__MISS_FIFO_DEPTH__SHIFT                                                                    0x4
+#define SQC_CONFIG__HIT_FIFO_DEPTH__SHIFT                                                                     0x6
+#define SQC_CONFIG__FORCE_ALWAYS_MISS__SHIFT                                                                  0x7
+#define SQC_CONFIG__FORCE_IN_ORDER__SHIFT                                                                     0x8
+#define SQC_CONFIG__IDENTITY_HASH_BANK__SHIFT                                                                 0x9
+#define SQC_CONFIG__IDENTITY_HASH_SET__SHIFT                                                                  0xa
+#define SQC_CONFIG__PER_VMID_INV_DISABLE__SHIFT                                                               0xb
+#define SQC_CONFIG__EVICT_LRU__SHIFT                                                                          0xc
+#define SQC_CONFIG__FORCE_2_BANK__SHIFT                                                                       0xe
+#define SQC_CONFIG__FORCE_1_BANK__SHIFT                                                                       0xf
+#define SQC_CONFIG__LS_DISABLE_CLOCKS__SHIFT                                                                  0x10
+#define SQC_CONFIG__INST_PRF_COUNT__SHIFT                                                                     0x18
+#define SQC_CONFIG__INST_PRF_FILTER_DIS__SHIFT                                                                0x1d
+#define SQC_CONFIG__DISABLE_PREFETCH_CROSS_4K_BOUNDARY_CHECK__SHIFT                                           0x1e
+#define SQC_CONFIG__MEM_LS_DISABLE__SHIFT                                                                     0x1f
+#define SQC_CONFIG__INST_CACHE_SIZE_MASK                                                                      0x00000003L
+#define SQC_CONFIG__DATA_CACHE_SIZE_MASK                                                                      0x0000000CL
+#define SQC_CONFIG__MISS_FIFO_DEPTH_MASK                                                                      0x00000030L
+#define SQC_CONFIG__HIT_FIFO_DEPTH_MASK                                                                       0x00000040L
+#define SQC_CONFIG__FORCE_ALWAYS_MISS_MASK                                                                    0x00000080L
+#define SQC_CONFIG__FORCE_IN_ORDER_MASK                                                                       0x00000100L
+#define SQC_CONFIG__IDENTITY_HASH_BANK_MASK                                                                   0x00000200L
+#define SQC_CONFIG__IDENTITY_HASH_SET_MASK                                                                    0x00000400L
+#define SQC_CONFIG__PER_VMID_INV_DISABLE_MASK                                                                 0x00000800L
+#define SQC_CONFIG__EVICT_LRU_MASK                                                                            0x00003000L
+#define SQC_CONFIG__FORCE_2_BANK_MASK                                                                         0x00004000L
+#define SQC_CONFIG__FORCE_1_BANK_MASK                                                                         0x00008000L
+#define SQC_CONFIG__LS_DISABLE_CLOCKS_MASK                                                                    0x00FF0000L
+#define SQC_CONFIG__INST_PRF_COUNT_MASK                                                                       0x1F000000L
+#define SQC_CONFIG__INST_PRF_FILTER_DIS_MASK                                                                  0x20000000L
+#define SQC_CONFIG__DISABLE_PREFETCH_CROSS_4K_BOUNDARY_CHECK_MASK                                             0x40000000L
+#define SQC_CONFIG__MEM_LS_DISABLE_MASK                                                                       0x80000000L
+//LDS_CONFIG
+#define LDS_CONFIG__ADDR_OUT_OF_RANGE_REPORTING__SHIFT                                                        0x0
+#define LDS_CONFIG__TMZ_VIOLATION_REPORTING__SHIFT                                                            0x1
+#define LDS_CONFIG__DISABLE_RAM_CLOCK_GATING__SHIFT                                                           0x2
+#define LDS_CONFIG__DISABLE_IDXCLK_MGCG__SHIFT                                                                0x3
+#define LDS_CONFIG__DISABLE_MEMCLK_MGCG__SHIFT                                                                0x4
+#define LDS_CONFIG__DISABLE_ATTRCLK_MGCG__SHIFT                                                               0x5
+#define LDS_CONFIG__DISABLE_ATODFPCLK_MGCG__SHIFT                                                             0x6
+#define LDS_CONFIG__DISABLE_PHASE_FGCG__SHIFT                                                                 0x7
+#define LDS_CONFIG__DISABLE_LDS_SP_READ_FGCG__SHIFT                                                           0x8
+#define LDS_CONFIG__DISABLE_SP_DATA_CLOCK_GATING__SHIFT                                                       0x9
+#define LDS_CONFIG__DISABLE_TD_DATA_CLOCK_GATING__SHIFT                                                       0xa
+#define LDS_CONFIG__ADDR_OUT_OF_RANGE_REPORTING_MASK                                                          0x00000001L
+#define LDS_CONFIG__TMZ_VIOLATION_REPORTING_MASK                                                              0x00000002L
+#define LDS_CONFIG__DISABLE_RAM_CLOCK_GATING_MASK                                                             0x00000004L
+#define LDS_CONFIG__DISABLE_IDXCLK_MGCG_MASK                                                                  0x00000008L
+#define LDS_CONFIG__DISABLE_MEMCLK_MGCG_MASK                                                                  0x00000010L
+#define LDS_CONFIG__DISABLE_ATTRCLK_MGCG_MASK                                                                 0x00000020L
+#define LDS_CONFIG__DISABLE_ATODFPCLK_MGCG_MASK                                                               0x00000040L
+#define LDS_CONFIG__DISABLE_PHASE_FGCG_MASK                                                                   0x00000080L
+#define LDS_CONFIG__DISABLE_LDS_SP_READ_FGCG_MASK                                                             0x00000100L
+#define LDS_CONFIG__DISABLE_SP_DATA_CLOCK_GATING_MASK                                                         0x00000200L
+#define LDS_CONFIG__DISABLE_TD_DATA_CLOCK_GATING_MASK                                                         0x00000400L
+//SQ_RANDOM_WAVE_PRI
+#define SQ_RANDOM_WAVE_PRI__RET__SHIFT                                                                        0x0
+#define SQ_RANDOM_WAVE_PRI__RUI__SHIFT                                                                        0x7
+#define SQ_RANDOM_WAVE_PRI__RNG__SHIFT                                                                        0xa
+#define SQ_RANDOM_WAVE_PRI__RET_MASK                                                                          0x0000007FL
+#define SQ_RANDOM_WAVE_PRI__RUI_MASK                                                                          0x00000380L
+#define SQ_RANDOM_WAVE_PRI__RNG_MASK                                                                          0x007FFC00L
+//SQ_REG_CREDITS
+#define SQ_REG_CREDITS__SRBM_CREDITS__SHIFT                                                                   0x0
+#define SQ_REG_CREDITS__CMD_CREDITS__SHIFT                                                                    0x8
+#define SQ_REG_CREDITS__REG_BUSY__SHIFT                                                                       0x1c
+#define SQ_REG_CREDITS__SRBM_OVERFLOW__SHIFT                                                                  0x1d
+#define SQ_REG_CREDITS__IMMED_OVERFLOW__SHIFT                                                                 0x1e
+#define SQ_REG_CREDITS__CMD_OVERFLOW__SHIFT                                                                   0x1f
+#define SQ_REG_CREDITS__SRBM_CREDITS_MASK                                                                     0x0000003FL
+#define SQ_REG_CREDITS__CMD_CREDITS_MASK                                                                      0x00000F00L
+#define SQ_REG_CREDITS__REG_BUSY_MASK                                                                         0x10000000L
+#define SQ_REG_CREDITS__SRBM_OVERFLOW_MASK                                                                    0x20000000L
+#define SQ_REG_CREDITS__IMMED_OVERFLOW_MASK                                                                   0x40000000L
+#define SQ_REG_CREDITS__CMD_OVERFLOW_MASK                                                                     0x80000000L
+//SQ_FIFO_SIZES
+#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE__SHIFT                                                             0x0
+#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE__SHIFT                                                                0x8
+#define SQ_FIFO_SIZES__EXPORT_BUF_SIZE__SHIFT                                                                 0x10
+#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE__SHIFT                                                             0x12
+#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE_MASK                                                               0x0000000FL
+#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE_MASK                                                                  0x00000F00L
+#define SQ_FIFO_SIZES__EXPORT_BUF_SIZE_MASK                                                                   0x00030000L
+#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE_MASK                                                               0x000C0000L
+//SQ_DSM_CNTL
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_0__SHIFT                                                                 0x0
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_1__SHIFT                                                                 0x1
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_0__SHIFT                                                                0x2
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_1__SHIFT                                                                0x3
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA0__SHIFT                                                      0x8
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA1__SHIFT                                                      0x9
+#define SQ_DSM_CNTL__SGPR_ENABLE_SINGLE_WRITE__SHIFT                                                          0xa
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA0__SHIFT                                                       0x10
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA1__SHIFT                                                       0x11
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE01__SHIFT                                                         0x12
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA2__SHIFT                                                       0x13
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA3__SHIFT                                                       0x14
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE23__SHIFT                                                         0x15
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA0__SHIFT                                                        0x18
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA1__SHIFT                                                        0x19
+#define SQ_DSM_CNTL__SP_ENABLE_SINGLE_WRITE__SHIFT                                                            0x1a
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_0_MASK                                                                   0x00000001L
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_1_MASK                                                                   0x00000002L
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_0_MASK                                                                  0x00000004L
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_1_MASK                                                                  0x00000008L
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA0_MASK                                                        0x00000100L
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA1_MASK                                                        0x00000200L
+#define SQ_DSM_CNTL__SGPR_ENABLE_SINGLE_WRITE_MASK                                                            0x00000400L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA0_MASK                                                         0x00010000L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA1_MASK                                                         0x00020000L
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE01_MASK                                                           0x00040000L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA2_MASK                                                         0x00080000L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA3_MASK                                                         0x00100000L
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE23_MASK                                                           0x00200000L
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA0_MASK                                                          0x01000000L
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA1_MASK                                                          0x02000000L
+#define SQ_DSM_CNTL__SP_ENABLE_SINGLE_WRITE_MASK                                                              0x04000000L
+//SQ_DSM_CNTL2
+#define SQ_DSM_CNTL2__SGPR_ENABLE_ERROR_INJECT__SHIFT                                                         0x0
+#define SQ_DSM_CNTL2__SGPR_SELECT_INJECT_DELAY__SHIFT                                                         0x2
+#define SQ_DSM_CNTL2__LDS_D_ENABLE_ERROR_INJECT__SHIFT                                                        0x3
+#define SQ_DSM_CNTL2__LDS_D_SELECT_INJECT_DELAY__SHIFT                                                        0x5
+#define SQ_DSM_CNTL2__LDS_I_ENABLE_ERROR_INJECT__SHIFT                                                        0x6
+#define SQ_DSM_CNTL2__LDS_I_SELECT_INJECT_DELAY__SHIFT                                                        0x8
+#define SQ_DSM_CNTL2__SP_ENABLE_ERROR_INJECT__SHIFT                                                           0x9
+#define SQ_DSM_CNTL2__SP_SELECT_INJECT_DELAY__SHIFT                                                           0xb
+#define SQ_DSM_CNTL2__LDS_INJECT_DELAY__SHIFT                                                                 0xe
+#define SQ_DSM_CNTL2__SP_INJECT_DELAY__SHIFT                                                                  0x14
+#define SQ_DSM_CNTL2__SQ_INJECT_DELAY__SHIFT                                                                  0x1a
+#define SQ_DSM_CNTL2__SGPR_ENABLE_ERROR_INJECT_MASK                                                           0x00000003L
+#define SQ_DSM_CNTL2__SGPR_SELECT_INJECT_DELAY_MASK                                                           0x00000004L
+#define SQ_DSM_CNTL2__LDS_D_ENABLE_ERROR_INJECT_MASK                                                          0x00000018L
+#define SQ_DSM_CNTL2__LDS_D_SELECT_INJECT_DELAY_MASK                                                          0x00000020L
+#define SQ_DSM_CNTL2__LDS_I_ENABLE_ERROR_INJECT_MASK                                                          0x000000C0L
+#define SQ_DSM_CNTL2__LDS_I_SELECT_INJECT_DELAY_MASK                                                          0x00000100L
+#define SQ_DSM_CNTL2__SP_ENABLE_ERROR_INJECT_MASK                                                             0x00000600L
+#define SQ_DSM_CNTL2__SP_SELECT_INJECT_DELAY_MASK                                                             0x00000800L
+#define SQ_DSM_CNTL2__LDS_INJECT_DELAY_MASK                                                                   0x000FC000L
+#define SQ_DSM_CNTL2__SP_INJECT_DELAY_MASK                                                                    0x03F00000L
+#define SQ_DSM_CNTL2__SQ_INJECT_DELAY_MASK                                                                    0xFC000000L
+//SQ_RUNTIME_CONFIG
+#define SQ_RUNTIME_CONFIG__ENABLE_TEX_ARB_OLDEST__SHIFT                                                       0x0
+#define SQ_RUNTIME_CONFIG__ENABLE_TEX_ARB_OLDEST_MASK                                                         0x00000001L
+//SQ_DEBUG_STS_GLOBAL
+#define SQ_DEBUG_STS_GLOBAL__BUSY__SHIFT                                                                      0x0
+#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_MSG_BUSY__SHIFT                                                        0x1
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH0__SHIFT                                                            0x4
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH1__SHIFT                                                            0x10
+#define SQ_DEBUG_STS_GLOBAL__BUSY_MASK                                                                        0x00000001L
+#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_MSG_BUSY_MASK                                                          0x00000002L
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH0_MASK                                                              0x0000FFF0L
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH1_MASK                                                              0x0FFF0000L
+//SH_MEM_BASES
+#define SH_MEM_BASES__PRIVATE_BASE__SHIFT                                                                     0x0
+#define SH_MEM_BASES__SHARED_BASE__SHIFT                                                                      0x10
+#define SH_MEM_BASES__PRIVATE_BASE_MASK                                                                       0x0000FFFFL
+#define SH_MEM_BASES__SHARED_BASE_MASK                                                                        0xFFFF0000L
+//SQ_TIMEOUT_CONFIG
+#define SQ_TIMEOUT_CONFIG__PERIOD_SEL__SHIFT                                                                  0x0
+#define SQ_TIMEOUT_CONFIG__TIMEOUT_FATAL_DISABLE__SHIFT                                                       0x6
+#define SQ_TIMEOUT_CONFIG__TIMER_LONGER_SEL__SHIFT                                                            0x7
+#define SQ_TIMEOUT_CONFIG__TIMEOUT_CONDITIONS_MASK__SHIFT                                                     0x8
+#define SQ_TIMEOUT_CONFIG__PERIOD_SEL_MASK                                                                    0x0000003FL
+#define SQ_TIMEOUT_CONFIG__TIMEOUT_FATAL_DISABLE_MASK                                                         0x00000040L
+#define SQ_TIMEOUT_CONFIG__TIMER_LONGER_SEL_MASK                                                              0x00000080L
+#define SQ_TIMEOUT_CONFIG__TIMEOUT_CONDITIONS_MASK_MASK                                                       0x07FFFF00L
+//SQ_TIMEOUT_STATUS
+#define SQ_TIMEOUT_STATUS__WAVE_TIMEOUT__SHIFT                                                                0x0
+#define SQ_TIMEOUT_STATUS__WAVE_TIMEOUT_MASK                                                                  0xFFFFFFFFL
+//SH_MEM_CONFIG
+#define SH_MEM_CONFIG__ADDRESS_MODE__SHIFT                                                                    0x0
+#define SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT                                                                  0x3
+#define SH_MEM_CONFIG__F8_MODE__SHIFT                                                                         0x8
+#define SH_MEM_CONFIG__RETRY_DISABLE__SHIFT                                                                   0xc
+#define SH_MEM_CONFIG__PRIVATE_NV__SHIFT                                                                      0xd
+#define SH_MEM_CONFIG__ADDRESS_MODE_MASK                                                                      0x00000001L
+#define SH_MEM_CONFIG__ALIGNMENT_MODE_MASK                                                                    0x00000018L
+#define SH_MEM_CONFIG__F8_MODE_MASK                                                                           0x00000100L
+#define SH_MEM_CONFIG__RETRY_DISABLE_MASK                                                                     0x00001000L
+#define SH_MEM_CONFIG__PRIVATE_NV_MASK                                                                        0x00002000L
+//SP_MFMA_PORTD_RD_CONFIG
+#define SP_MFMA_PORTD_RD_CONFIG__SET__SHIFT                                                                   0x0
+#define SP_MFMA_PORTD_RD_CONFIG__TYPE__SHIFT                                                                  0x1
+#define SP_MFMA_PORTD_RD_CONFIG__LAST_PASS__SHIFT                                                             0x4
+#define SP_MFMA_PORTD_RD_CONFIG__PORTD_PATTERN__SHIFT                                                         0x9
+#define SP_MFMA_PORTD_RD_CONFIG__SET_MASK                                                                     0x00000001L
+#define SP_MFMA_PORTD_RD_CONFIG__TYPE_MASK                                                                    0x0000000EL
+#define SP_MFMA_PORTD_RD_CONFIG__LAST_PASS_MASK                                                               0x000001F0L
+#define SP_MFMA_PORTD_RD_CONFIG__PORTD_PATTERN_MASK                                                           0x1FFFFE00L
+//SH_CAC_CONFIG
+#define SH_CAC_CONFIG__SQG_UTCL1_REPEATER_FGCG_DISABLE__SHIFT                                                 0x0
+#define SH_CAC_CONFIG__SQC_UTCL1_REPEATER_FGCG_DISABLE__SHIFT                                                 0x1
+#define SH_CAC_CONFIG__SPI_SQ_CMD_REPEATER_FGCG_DISABLE__SHIFT                                                0x2
+#define SH_CAC_CONFIG__SQ_MSG_REPEATER_FGCG_DISABLE__SHIFT                                                    0x3
+#define SH_CAC_CONFIG__SQC_TC_REPEATER_FGCG_DISABLE__SHIFT                                                    0x4
+#define SH_CAC_CONFIG__SQC_SQ_REPEATER_FGCG_DISABLE__SHIFT                                                    0x5
+#define SH_CAC_CONFIG__SQG_TC_REPEATER_FGCG_DISABLE__SHIFT                                                    0x6
+#define SH_CAC_CONFIG__SQC_DISABLE_RAM_CLOCK_GATING__SHIFT                                                    0x8
+#define SH_CAC_CONFIG__SQG_DISABLE_RAM_CLOCK_GATING__SHIFT                                                    0x9
+#define SH_CAC_CONFIG__SQC_MGCG_CLOCK_OFF_DELAY_CNT__SHIFT                                                    0x10
+#define SH_CAC_CONFIG__SQC_MGCG_DISABLE__SHIFT                                                                0x14
+#define SH_CAC_CONFIG__SQC_TC_REQ_CLKEN_CHICKENBIT__SHIFT                                                     0x1c
+#define SH_CAC_CONFIG__SQC_ICACHE_CTRL_MGCG_DISABLE__SHIFT                                                    0x1d
+#define SH_CAC_CONFIG__SQC_DCACHE_CTRL_MGCG_DISABLE__SHIFT                                                    0x1e
+#define SH_CAC_CONFIG__SQC_DISABLE_UTCL1_FGCG_PADDR__SHIFT                                                    0x1f
+#define SH_CAC_CONFIG__SQG_UTCL1_REPEATER_FGCG_DISABLE_MASK                                                   0x00000001L
+#define SH_CAC_CONFIG__SQC_UTCL1_REPEATER_FGCG_DISABLE_MASK                                                   0x00000002L
+#define SH_CAC_CONFIG__SPI_SQ_CMD_REPEATER_FGCG_DISABLE_MASK                                                  0x00000004L
+#define SH_CAC_CONFIG__SQ_MSG_REPEATER_FGCG_DISABLE_MASK                                                      0x00000008L
+#define SH_CAC_CONFIG__SQC_TC_REPEATER_FGCG_DISABLE_MASK                                                      0x00000010L
+#define SH_CAC_CONFIG__SQC_SQ_REPEATER_FGCG_DISABLE_MASK                                                      0x00000020L
+#define SH_CAC_CONFIG__SQG_TC_REPEATER_FGCG_DISABLE_MASK                                                      0x00000040L
+#define SH_CAC_CONFIG__SQC_DISABLE_RAM_CLOCK_GATING_MASK                                                      0x00000100L
+#define SH_CAC_CONFIG__SQG_DISABLE_RAM_CLOCK_GATING_MASK                                                      0x00000200L
+#define SH_CAC_CONFIG__SQC_MGCG_CLOCK_OFF_DELAY_CNT_MASK                                                      0x000F0000L
+#define SH_CAC_CONFIG__SQC_MGCG_DISABLE_MASK                                                                  0x0FF00000L
+#define SH_CAC_CONFIG__SQC_TC_REQ_CLKEN_CHICKENBIT_MASK                                                       0x10000000L
+#define SH_CAC_CONFIG__SQC_ICACHE_CTRL_MGCG_DISABLE_MASK                                                      0x20000000L
+#define SH_CAC_CONFIG__SQC_DCACHE_CTRL_MGCG_DISABLE_MASK                                                      0x40000000L
+#define SH_CAC_CONFIG__SQC_DISABLE_UTCL1_FGCG_PADDR_MASK                                                      0x80000000L
+//SQ_DEBUG_STS_GLOBAL2
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX0__SHIFT                                                          0x0
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX1__SHIFT                                                          0x8
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_IMMED__SHIFT                                                         0x10
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_HOST__SHIFT                                                          0x18
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX0_MASK                                                            0x000000FFL
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX1_MASK                                                            0x0000FF00L
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_IMMED_MASK                                                           0x00FF0000L
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_HOST_MASK                                                            0xFF000000L
+//SQ_DEBUG_STS_GLOBAL3
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_CMD__SHIFT                                                      0x0
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_REG__SHIFT                                                      0x4
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_CMD_MASK                                                        0x0000000FL
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_REG_MASK                                                        0x000003F0L
+//CC_GC_SHADER_RATE_CONFIG
+#define CC_GC_SHADER_RATE_CONFIG__WRITE_DIS__SHIFT                                                            0x0
+#define CC_GC_SHADER_RATE_CONFIG__DPFP_RATE__SHIFT                                                            0x1
+#define CC_GC_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE__SHIFT                                                  0x3
+#define CC_GC_SHADER_RATE_CONFIG__HALF_LDS__SHIFT                                                             0x4
+#define CC_GC_SHADER_RATE_CONFIG__WRITE_DIS_MASK                                                              0x00000001L
+#define CC_GC_SHADER_RATE_CONFIG__DPFP_RATE_MASK                                                              0x00000006L
+#define CC_GC_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE_MASK                                                    0x00000008L
+#define CC_GC_SHADER_RATE_CONFIG__HALF_LDS_MASK                                                               0x00000010L
+//GC_USER_SHADER_RATE_CONFIG
+#define GC_USER_SHADER_RATE_CONFIG__DPFP_RATE__SHIFT                                                          0x1
+#define GC_USER_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE__SHIFT                                                0x3
+#define GC_USER_SHADER_RATE_CONFIG__HALF_LDS__SHIFT                                                           0x4
+#define GC_USER_SHADER_RATE_CONFIG__DPFP_RATE_MASK                                                            0x00000006L
+#define GC_USER_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE_MASK                                                  0x00000008L
+#define GC_USER_SHADER_RATE_CONFIG__HALF_LDS_MASK                                                             0x00000010L
+//SQ_INTERRUPT_AUTO_MASK
+#define SQ_INTERRUPT_AUTO_MASK__MASK__SHIFT                                                                   0x0
+#define SQ_INTERRUPT_AUTO_MASK__MASK_MASK                                                                     0x00FFFFFFL
+//SQ_INTERRUPT_MSG_CTRL
+#define SQ_INTERRUPT_MSG_CTRL__STALL__SHIFT                                                                   0x0
+#define SQ_INTERRUPT_MSG_CTRL__STALL_MASK                                                                     0x00000001L
+//SQ_DEBUG_PERFCOUNT_TRAP
+#define SQ_DEBUG_PERFCOUNT_TRAP__ENABLE__SHIFT                                                                0x0
+#define SQ_DEBUG_PERFCOUNT_TRAP__COUNTER__SHIFT                                                               0x1
+#define SQ_DEBUG_PERFCOUNT_TRAP__LIMIT__SHIFT                                                                 0x4
+#define SQ_DEBUG_PERFCOUNT_TRAP__ENABLE_MASK                                                                  0x00000001L
+#define SQ_DEBUG_PERFCOUNT_TRAP__COUNTER_MASK                                                                 0x0000000EL
+#define SQ_DEBUG_PERFCOUNT_TRAP__LIMIT_MASK                                                                   0x0FFFFFF0L
+//SQ_UTCL1_CNTL1
+#define SQ_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                               0x0
+#define SQ_UTCL1_CNTL1__GPUVM_64K_DEF__SHIFT                                                                  0x1
+#define SQ_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                                0x2
+#define SQ_UTCL1_CNTL1__RESP_MODE__SHIFT                                                                      0x3
+#define SQ_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                                0x5
+#define SQ_UTCL1_CNTL1__CLIENTID__SHIFT                                                                       0x7
+#define SQ_UTCL1_CNTL1__USERVM_DIS__SHIFT                                                                     0x10
+#define SQ_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                              0x11
+#define SQ_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                           0x12
+#define SQ_UTCL1_CNTL1__REG_INVALIDATE_VMID__SHIFT                                                            0x13
+#define SQ_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID__SHIFT                                                        0x17
+#define SQ_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE__SHIFT                                                          0x18
+#define SQ_UTCL1_CNTL1__REG_INVALIDATE_ALL__SHIFT                                                             0x19
+#define SQ_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                                     0x1a
+#define SQ_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                                 0x1b
+#define SQ_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                         0x1c
+#define SQ_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                         0x1e
+#define SQ_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                                 0x00000001L
+#define SQ_UTCL1_CNTL1__GPUVM_64K_DEF_MASK                                                                    0x00000002L
+#define SQ_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                                  0x00000004L
+#define SQ_UTCL1_CNTL1__RESP_MODE_MASK                                                                        0x00000018L
+#define SQ_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                                  0x00000060L
+#define SQ_UTCL1_CNTL1__CLIENTID_MASK                                                                         0x0000FF80L
+#define SQ_UTCL1_CNTL1__USERVM_DIS_MASK                                                                       0x00010000L
+#define SQ_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                                0x00020000L
+#define SQ_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                             0x00040000L
+#define SQ_UTCL1_CNTL1__REG_INVALIDATE_VMID_MASK                                                              0x00780000L
+#define SQ_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID_MASK                                                          0x00800000L
+#define SQ_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE_MASK                                                            0x01000000L
+#define SQ_UTCL1_CNTL1__REG_INVALIDATE_ALL_MASK                                                               0x02000000L
+#define SQ_UTCL1_CNTL1__FORCE_MISS_MASK                                                                       0x04000000L
+#define SQ_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                                   0x08000000L
+#define SQ_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                           0x30000000L
+#define SQ_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                           0xC0000000L
+//SQ_UTCL1_CNTL2
+#define SQ_UTCL1_CNTL2__SPARE__SHIFT                                                                          0x0
+#define SQ_UTCL1_CNTL2__LFIFO_SCAN_DISABLE__SHIFT                                                             0x8
+#define SQ_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                 0x9
+#define SQ_UTCL1_CNTL2__LINE_VALID__SHIFT                                                                     0xa
+#define SQ_UTCL1_CNTL2__DIS_EDC__SHIFT                                                                        0xb
+#define SQ_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                                 0xc
+#define SQ_UTCL1_CNTL2__SHOOTDOWN_OPT__SHIFT                                                                  0xd
+#define SQ_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                                    0xe
+#define SQ_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                            0xf
+#define SQ_UTCL1_CNTL2__RETRY_TIMER__SHIFT                                                                    0x10
+#define SQ_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                           0x1a
+#define SQ_UTCL1_CNTL2__PREFETCH_PAGE__SHIFT                                                                  0x1c
+#define SQ_UTCL1_CNTL2__SPARE_MASK                                                                            0x000000FFL
+#define SQ_UTCL1_CNTL2__LFIFO_SCAN_DISABLE_MASK                                                               0x00000100L
+#define SQ_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                                   0x00000200L
+#define SQ_UTCL1_CNTL2__LINE_VALID_MASK                                                                       0x00000400L
+#define SQ_UTCL1_CNTL2__DIS_EDC_MASK                                                                          0x00000800L
+#define SQ_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                                   0x00001000L
+#define SQ_UTCL1_CNTL2__SHOOTDOWN_OPT_MASK                                                                    0x00002000L
+#define SQ_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                                      0x00004000L
+#define SQ_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                              0x00008000L
+#define SQ_UTCL1_CNTL2__RETRY_TIMER_MASK                                                                      0x007F0000L
+#define SQ_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                             0x04000000L
+#define SQ_UTCL1_CNTL2__PREFETCH_PAGE_MASK                                                                    0xF0000000L
+//SQ_UTCL1_STATUS
+#define SQ_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                                0x0
+#define SQ_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                                0x1
+#define SQ_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                  0x2
+#define SQ_UTCL1_STATUS__RESERVED__SHIFT                                                                      0x3
+#define SQ_UTCL1_STATUS__UNUSED__SHIFT                                                                        0x10
+#define SQ_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                  0x00000001L
+#define SQ_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                  0x00000002L
+#define SQ_UTCL1_STATUS__PRT_DETECTED_MASK                                                                    0x00000004L
+#define SQ_UTCL1_STATUS__RESERVED_MASK                                                                        0x0000FFF8L
+#define SQ_UTCL1_STATUS__UNUSED_MASK                                                                          0xFFFF0000L
+//SQ_FED_INTERRUPT_STATUS
+#define SQ_FED_INTERRUPT_STATUS__INTERRUPT_STATUS__SHIFT                                                      0x0
+#define SQ_FED_INTERRUPT_STATUS__INTERRUPT_SIMD_ID__SHIFT                                                     0x2
+#define SQ_FED_INTERRUPT_STATUS__INTERRUPT_WAVE_ID__SHIFT                                                     0x4
+#define SQ_FED_INTERRUPT_STATUS__INTERRUPT_CU_ID__SHIFT                                                       0x8
+#define SQ_FED_INTERRUPT_STATUS__INTERRUPT_VM_ID__SHIFT                                                       0xc
+#define SQ_FED_INTERRUPT_STATUS__TO_RSMU_DISABLE__SHIFT                                                       0x10
+#define SQ_FED_INTERRUPT_STATUS__TO_IH_DISABLE__SHIFT                                                         0x11
+#define SQ_FED_INTERRUPT_STATUS__FED_HALT_DISABLE__SHIFT                                                      0x12
+#define SQ_FED_INTERRUPT_STATUS__INTERRUPT_STATUS_MASK                                                        0x00000001L
+#define SQ_FED_INTERRUPT_STATUS__INTERRUPT_SIMD_ID_MASK                                                       0x0000000CL
+#define SQ_FED_INTERRUPT_STATUS__INTERRUPT_WAVE_ID_MASK                                                       0x000000F0L
+#define SQ_FED_INTERRUPT_STATUS__INTERRUPT_CU_ID_MASK                                                         0x00000F00L
+#define SQ_FED_INTERRUPT_STATUS__INTERRUPT_VM_ID_MASK                                                         0x0000F000L
+#define SQ_FED_INTERRUPT_STATUS__TO_RSMU_DISABLE_MASK                                                         0x00010000L
+#define SQ_FED_INTERRUPT_STATUS__TO_IH_DISABLE_MASK                                                           0x00020000L
+#define SQ_FED_INTERRUPT_STATUS__FED_HALT_DISABLE_MASK                                                        0x00040000L
+//SQ_CGTS_CONFIG
+#define SQ_CGTS_CONFIG__DGEMM_EXTRA_BUSY_PASS__SHIFT                                                          0x0
+#define SQ_CGTS_CONFIG__XDL_EXTRA_BUSY_PASS__SHIFT                                                            0x4
+#define SQ_CGTS_CONFIG__VALU_EXTRA_BUSY_PASS__SHIFT                                                           0x8
+#define SQ_CGTS_CONFIG__DLOP_EXTRA_BUSY_PASS__SHIFT                                                           0xc
+#define SQ_CGTS_CONFIG__XDL_EXTRA_GAP_PASS__SHIFT                                                             0x10
+#define SQ_CGTS_CONFIG__DGEMM_EXTRA_GAP_PASS__SHIFT                                                           0x12
+#define SQ_CGTS_CONFIG__DLOP_EXTRA_GAP_PASS__SHIFT                                                            0x14
+#define SQ_CGTS_CONFIG__DGEMM_EXTRA_BUSY_PASS_MASK                                                            0x0000000FL
+#define SQ_CGTS_CONFIG__XDL_EXTRA_BUSY_PASS_MASK                                                              0x000000F0L
+#define SQ_CGTS_CONFIG__VALU_EXTRA_BUSY_PASS_MASK                                                             0x00000F00L
+#define SQ_CGTS_CONFIG__DLOP_EXTRA_BUSY_PASS_MASK                                                             0x0000F000L
+#define SQ_CGTS_CONFIG__XDL_EXTRA_GAP_PASS_MASK                                                               0x00030000L
+#define SQ_CGTS_CONFIG__DGEMM_EXTRA_GAP_PASS_MASK                                                             0x000C0000L
+#define SQ_CGTS_CONFIG__DLOP_EXTRA_GAP_PASS_MASK                                                              0x00300000L
+//SQ_SHADER_TBA_LO
+#define SQ_SHADER_TBA_LO__ADDR_LO__SHIFT                                                                      0x0
+#define SQ_SHADER_TBA_LO__ADDR_LO_MASK                                                                        0xFFFFFFFFL
+//SQ_SHADER_TBA_HI
+#define SQ_SHADER_TBA_HI__ADDR_HI__SHIFT                                                                      0x0
+#define SQ_SHADER_TBA_HI__ADDR_HI_MASK                                                                        0x000000FFL
+//SQ_SHADER_TMA_LO
+#define SQ_SHADER_TMA_LO__ADDR_LO__SHIFT                                                                      0x0
+#define SQ_SHADER_TMA_LO__ADDR_LO_MASK                                                                        0xFFFFFFFFL
+//SQ_SHADER_TMA_HI
+#define SQ_SHADER_TMA_HI__ADDR_HI__SHIFT                                                                      0x0
+#define SQ_SHADER_TMA_HI__ADDR_HI_MASK                                                                        0x000000FFL
+//SQC_DSM_CNTL
+#define SQC_DSM_CNTL__INST_UTCL1_LFIFO_DSM_IRRITATOR_DATA__SHIFT                                              0x0
+#define SQC_DSM_CNTL__INST_UTCL1_LFIFO_ENABLE_SINGLE_WRITE__SHIFT                                             0x2
+#define SQC_DSM_CNTL__DATA_CU0_WRITE_DATA_BUF_DSM_IRRITATOR_DATA__SHIFT                                       0x3
+#define SQC_DSM_CNTL__DATA_CU0_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE__SHIFT                                      0x5
+#define SQC_DSM_CNTL__DATA_CU0_UTCL1_LFIFO_DSM_IRRITATOR_DATA__SHIFT                                          0x6
+#define SQC_DSM_CNTL__DATA_CU0_UTCL1_LFIFO_ENABLE_SINGLE_WRITE__SHIFT                                         0x8
+#define SQC_DSM_CNTL__DATA_CU1_WRITE_DATA_BUF_DSM_IRRITATOR_DATA__SHIFT                                       0x9
+#define SQC_DSM_CNTL__DATA_CU1_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE__SHIFT                                      0xb
+#define SQC_DSM_CNTL__DATA_CU1_UTCL1_LFIFO_DSM_IRRITATOR_DATA__SHIFT                                          0xc
+#define SQC_DSM_CNTL__DATA_CU1_UTCL1_LFIFO_ENABLE_SINGLE_WRITE__SHIFT                                         0xe
+#define SQC_DSM_CNTL__DATA_CU2_WRITE_DATA_BUF_DSM_IRRITATOR_DATA__SHIFT                                       0xf
+#define SQC_DSM_CNTL__DATA_CU2_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE__SHIFT                                      0x11
+#define SQC_DSM_CNTL__DATA_CU2_UTCL1_LFIFO_DSM_IRRITATOR_DATA__SHIFT                                          0x12
+#define SQC_DSM_CNTL__DATA_CU2_UTCL1_LFIFO_ENABLE_SINGLE_WRITE__SHIFT                                         0x14
+#define SQC_DSM_CNTL__DATA_CU3_WRITE_DATA_BUF_DSM_IRRITATOR_DATA__SHIFT                                       0x15
+#define SQC_DSM_CNTL__DATA_CU3_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE__SHIFT                                      0x17
+#define SQC_DSM_CNTL__DATA_CU3_UTCL1_LFIFO_DSM_IRRITATOR_DATA__SHIFT                                          0x18
+#define SQC_DSM_CNTL__DATA_CU3_UTCL1_LFIFO_ENABLE_SINGLE_WRITE__SHIFT                                         0x1a
+#define SQC_DSM_CNTL__INST_UTCL1_LFIFO_DSM_IRRITATOR_DATA_MASK                                                0x00000003L
+#define SQC_DSM_CNTL__INST_UTCL1_LFIFO_ENABLE_SINGLE_WRITE_MASK                                               0x00000004L
+#define SQC_DSM_CNTL__DATA_CU0_WRITE_DATA_BUF_DSM_IRRITATOR_DATA_MASK                                         0x00000018L
+#define SQC_DSM_CNTL__DATA_CU0_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE_MASK                                        0x00000020L
+#define SQC_DSM_CNTL__DATA_CU0_UTCL1_LFIFO_DSM_IRRITATOR_DATA_MASK                                            0x000000C0L
+#define SQC_DSM_CNTL__DATA_CU0_UTCL1_LFIFO_ENABLE_SINGLE_WRITE_MASK                                           0x00000100L
+#define SQC_DSM_CNTL__DATA_CU1_WRITE_DATA_BUF_DSM_IRRITATOR_DATA_MASK                                         0x00000600L
+#define SQC_DSM_CNTL__DATA_CU1_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE_MASK                                        0x00000800L
+#define SQC_DSM_CNTL__DATA_CU1_UTCL1_LFIFO_DSM_IRRITATOR_DATA_MASK                                            0x00003000L
+#define SQC_DSM_CNTL__DATA_CU1_UTCL1_LFIFO_ENABLE_SINGLE_WRITE_MASK                                           0x00004000L
+#define SQC_DSM_CNTL__DATA_CU2_WRITE_DATA_BUF_DSM_IRRITATOR_DATA_MASK                                         0x00018000L
+#define SQC_DSM_CNTL__DATA_CU2_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE_MASK                                        0x00020000L
+#define SQC_DSM_CNTL__DATA_CU2_UTCL1_LFIFO_DSM_IRRITATOR_DATA_MASK                                            0x000C0000L
+#define SQC_DSM_CNTL__DATA_CU2_UTCL1_LFIFO_ENABLE_SINGLE_WRITE_MASK                                           0x00100000L
+#define SQC_DSM_CNTL__DATA_CU3_WRITE_DATA_BUF_DSM_IRRITATOR_DATA_MASK                                         0x00600000L
+#define SQC_DSM_CNTL__DATA_CU3_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE_MASK                                        0x00800000L
+#define SQC_DSM_CNTL__DATA_CU3_UTCL1_LFIFO_DSM_IRRITATOR_DATA_MASK                                            0x03000000L
+#define SQC_DSM_CNTL__DATA_CU3_UTCL1_LFIFO_ENABLE_SINGLE_WRITE_MASK                                           0x04000000L
+//SQC_DSM_CNTLA
+#define SQC_DSM_CNTLA__INST_TAG_RAM_DSM_IRRITATOR_DATA__SHIFT                                                 0x0
+#define SQC_DSM_CNTLA__INST_TAG_RAM_ENABLE_SINGLE_WRITE__SHIFT                                                0x2
+#define SQC_DSM_CNTLA__INST_UTCL1_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                         0x3
+#define SQC_DSM_CNTLA__INST_UTCL1_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                        0x5
+#define SQC_DSM_CNTLA__INST_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                               0x6
+#define SQC_DSM_CNTLA__INST_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                              0x8
+#define SQC_DSM_CNTLA__INST_BANK_RAM_DSM_IRRITATOR_DATA__SHIFT                                                0x9
+#define SQC_DSM_CNTLA__INST_BANK_RAM_ENABLE_SINGLE_WRITE__SHIFT                                               0xb
+#define SQC_DSM_CNTLA__DATA_TAG_RAM_DSM_IRRITATOR_DATA__SHIFT                                                 0xc
+#define SQC_DSM_CNTLA__DATA_TAG_RAM_ENABLE_SINGLE_WRITE__SHIFT                                                0xe
+#define SQC_DSM_CNTLA__DATA_HIT_FIFO_DSM_IRRITATOR_DATA__SHIFT                                                0xf
+#define SQC_DSM_CNTLA__DATA_HIT_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                               0x11
+#define SQC_DSM_CNTLA__DATA_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                               0x12
+#define SQC_DSM_CNTLA__DATA_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                              0x14
+#define SQC_DSM_CNTLA__DATA_DIRTY_BIT_RAM_DSM_IRRITATOR_DATA__SHIFT                                           0x15
+#define SQC_DSM_CNTLA__DATA_DIRTY_BIT_RAM_ENABLE_SINGLE_WRITE__SHIFT                                          0x17
+#define SQC_DSM_CNTLA__DATA_BANK_RAM_DSM_IRRITATOR_DATA__SHIFT                                                0x18
+#define SQC_DSM_CNTLA__DATA_BANK_RAM_ENABLE_SINGLE_WRITE__SHIFT                                               0x1a
+#define SQC_DSM_CNTLA__INST_TAG_RAM_DSM_IRRITATOR_DATA_MASK                                                   0x00000003L
+#define SQC_DSM_CNTLA__INST_TAG_RAM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000004L
+#define SQC_DSM_CNTLA__INST_UTCL1_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                           0x00000018L
+#define SQC_DSM_CNTLA__INST_UTCL1_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                          0x00000020L
+#define SQC_DSM_CNTLA__INST_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                                 0x000000C0L
+#define SQC_DSM_CNTLA__INST_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                                0x00000100L
+#define SQC_DSM_CNTLA__INST_BANK_RAM_DSM_IRRITATOR_DATA_MASK                                                  0x00000600L
+#define SQC_DSM_CNTLA__INST_BANK_RAM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000800L
+#define SQC_DSM_CNTLA__DATA_TAG_RAM_DSM_IRRITATOR_DATA_MASK                                                   0x00003000L
+#define SQC_DSM_CNTLA__DATA_TAG_RAM_ENABLE_SINGLE_WRITE_MASK                                                  0x00004000L
+#define SQC_DSM_CNTLA__DATA_HIT_FIFO_DSM_IRRITATOR_DATA_MASK                                                  0x00018000L
+#define SQC_DSM_CNTLA__DATA_HIT_FIFO_ENABLE_SINGLE_WRITE_MASK                                                 0x00020000L
+#define SQC_DSM_CNTLA__DATA_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                                 0x000C0000L
+#define SQC_DSM_CNTLA__DATA_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                                0x00100000L
+#define SQC_DSM_CNTLA__DATA_DIRTY_BIT_RAM_DSM_IRRITATOR_DATA_MASK                                             0x00600000L
+#define SQC_DSM_CNTLA__DATA_DIRTY_BIT_RAM_ENABLE_SINGLE_WRITE_MASK                                            0x00800000L
+#define SQC_DSM_CNTLA__DATA_BANK_RAM_DSM_IRRITATOR_DATA_MASK                                                  0x03000000L
+#define SQC_DSM_CNTLA__DATA_BANK_RAM_ENABLE_SINGLE_WRITE_MASK                                                 0x04000000L
+//SQC_DSM_CNTLB
+#define SQC_DSM_CNTLB__INST_TAG_RAM_DSM_IRRITATOR_DATA__SHIFT                                                 0x0
+#define SQC_DSM_CNTLB__INST_TAG_RAM_ENABLE_SINGLE_WRITE__SHIFT                                                0x2
+#define SQC_DSM_CNTLB__INST_UTCL1_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                         0x3
+#define SQC_DSM_CNTLB__INST_UTCL1_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                        0x5
+#define SQC_DSM_CNTLB__INST_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                               0x6
+#define SQC_DSM_CNTLB__INST_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                              0x8
+#define SQC_DSM_CNTLB__INST_BANK_RAM_DSM_IRRITATOR_DATA__SHIFT                                                0x9
+#define SQC_DSM_CNTLB__INST_BANK_RAM_ENABLE_SINGLE_WRITE__SHIFT                                               0xb
+#define SQC_DSM_CNTLB__DATA_TAG_RAM_DSM_IRRITATOR_DATA__SHIFT                                                 0xc
+#define SQC_DSM_CNTLB__DATA_TAG_RAM_ENABLE_SINGLE_WRITE__SHIFT                                                0xe
+#define SQC_DSM_CNTLB__DATA_HIT_FIFO_DSM_IRRITATOR_DATA__SHIFT                                                0xf
+#define SQC_DSM_CNTLB__DATA_HIT_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                               0x11
+#define SQC_DSM_CNTLB__DATA_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                               0x12
+#define SQC_DSM_CNTLB__DATA_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                              0x14
+#define SQC_DSM_CNTLB__DATA_DIRTY_BIT_RAM_DSM_IRRITATOR_DATA__SHIFT                                           0x15
+#define SQC_DSM_CNTLB__DATA_DIRTY_BIT_RAM_ENABLE_SINGLE_WRITE__SHIFT                                          0x17
+#define SQC_DSM_CNTLB__DATA_BANK_RAM_DSM_IRRITATOR_DATA__SHIFT                                                0x18
+#define SQC_DSM_CNTLB__DATA_BANK_RAM_ENABLE_SINGLE_WRITE__SHIFT                                               0x1a
+#define SQC_DSM_CNTLB__INST_TAG_RAM_DSM_IRRITATOR_DATA_MASK                                                   0x00000003L
+#define SQC_DSM_CNTLB__INST_TAG_RAM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000004L
+#define SQC_DSM_CNTLB__INST_UTCL1_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                           0x00000018L
+#define SQC_DSM_CNTLB__INST_UTCL1_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                          0x00000020L
+#define SQC_DSM_CNTLB__INST_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                                 0x000000C0L
+#define SQC_DSM_CNTLB__INST_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                                0x00000100L
+#define SQC_DSM_CNTLB__INST_BANK_RAM_DSM_IRRITATOR_DATA_MASK                                                  0x00000600L
+#define SQC_DSM_CNTLB__INST_BANK_RAM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000800L
+#define SQC_DSM_CNTLB__DATA_TAG_RAM_DSM_IRRITATOR_DATA_MASK                                                   0x00003000L
+#define SQC_DSM_CNTLB__DATA_TAG_RAM_ENABLE_SINGLE_WRITE_MASK                                                  0x00004000L
+#define SQC_DSM_CNTLB__DATA_HIT_FIFO_DSM_IRRITATOR_DATA_MASK                                                  0x00018000L
+#define SQC_DSM_CNTLB__DATA_HIT_FIFO_ENABLE_SINGLE_WRITE_MASK                                                 0x00020000L
+#define SQC_DSM_CNTLB__DATA_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                                 0x000C0000L
+#define SQC_DSM_CNTLB__DATA_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                                0x00100000L
+#define SQC_DSM_CNTLB__DATA_DIRTY_BIT_RAM_DSM_IRRITATOR_DATA_MASK                                             0x00600000L
+#define SQC_DSM_CNTLB__DATA_DIRTY_BIT_RAM_ENABLE_SINGLE_WRITE_MASK                                            0x00800000L
+#define SQC_DSM_CNTLB__DATA_BANK_RAM_DSM_IRRITATOR_DATA_MASK                                                  0x03000000L
+#define SQC_DSM_CNTLB__DATA_BANK_RAM_ENABLE_SINGLE_WRITE_MASK                                                 0x04000000L
+//SQC_DSM_CNTL2
+#define SQC_DSM_CNTL2__INST_UTCL1_LFIFO_ENABLE_ERROR_INJECT__SHIFT                                            0x0
+#define SQC_DSM_CNTL2__INST_UTCL1_LFIFO_SELECT_INJECT_DELAY__SHIFT                                            0x2
+#define SQC_DSM_CNTL2__DATA_CU0_WRITE_DATA_BUF_ENABLE_ERROR_INJECT__SHIFT                                     0x3
+#define SQC_DSM_CNTL2__DATA_CU0_WRITE_DATA_BUF_SELECT_INJECT_DELAY__SHIFT                                     0x5
+#define SQC_DSM_CNTL2__DATA_CU0_UTCL1_LFIFO_ENABLE_ERROR_INJECT__SHIFT                                        0x6
+#define SQC_DSM_CNTL2__DATA_CU0_UTCL1_LFIFO_SELECT_INJECT_DELAY__SHIFT                                        0x8
+#define SQC_DSM_CNTL2__DATA_CU1_WRITE_DATA_BUF_ENABLE_ERROR_INJECT__SHIFT                                     0x9
+#define SQC_DSM_CNTL2__DATA_CU1_WRITE_DATA_BUF_SELECT_INJECT_DELAY__SHIFT                                     0xb
+#define SQC_DSM_CNTL2__DATA_CU1_UTCL1_LFIFO_ENABLE_ERROR_INJECT__SHIFT                                        0xc
+#define SQC_DSM_CNTL2__DATA_CU1_UTCL1_LFIFO_SELECT_INJECT_DELAY__SHIFT                                        0xe
+#define SQC_DSM_CNTL2__DATA_CU2_WRITE_DATA_BUF_ENABLE_ERROR_INJECT__SHIFT                                     0xf
+#define SQC_DSM_CNTL2__DATA_CU2_WRITE_DATA_BUF_SELECT_INJECT_DELAY__SHIFT                                     0x11
+#define SQC_DSM_CNTL2__DATA_CU2_UTCL1_LFIFO_ENABLE_ERROR_INJECT__SHIFT                                        0x12
+#define SQC_DSM_CNTL2__DATA_CU2_UTCL1_LFIFO_SELECT_INJECT_DELAY__SHIFT                                        0x14
+#define SQC_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                    0x1a
+#define SQC_DSM_CNTL2__INST_UTCL1_LFIFO_ENABLE_ERROR_INJECT_MASK                                              0x00000003L
+#define SQC_DSM_CNTL2__INST_UTCL1_LFIFO_SELECT_INJECT_DELAY_MASK                                              0x00000004L
+#define SQC_DSM_CNTL2__DATA_CU0_WRITE_DATA_BUF_ENABLE_ERROR_INJECT_MASK                                       0x00000018L
+#define SQC_DSM_CNTL2__DATA_CU0_WRITE_DATA_BUF_SELECT_INJECT_DELAY_MASK                                       0x00000020L
+#define SQC_DSM_CNTL2__DATA_CU0_UTCL1_LFIFO_ENABLE_ERROR_INJECT_MASK                                          0x000000C0L
+#define SQC_DSM_CNTL2__DATA_CU0_UTCL1_LFIFO_SELECT_INJECT_DELAY_MASK                                          0x00000100L
+#define SQC_DSM_CNTL2__DATA_CU1_WRITE_DATA_BUF_ENABLE_ERROR_INJECT_MASK                                       0x00000600L
+#define SQC_DSM_CNTL2__DATA_CU1_WRITE_DATA_BUF_SELECT_INJECT_DELAY_MASK                                       0x00000800L
+#define SQC_DSM_CNTL2__DATA_CU1_UTCL1_LFIFO_ENABLE_ERROR_INJECT_MASK                                          0x00003000L
+#define SQC_DSM_CNTL2__DATA_CU1_UTCL1_LFIFO_SELECT_INJECT_DELAY_MASK                                          0x00004000L
+#define SQC_DSM_CNTL2__DATA_CU2_WRITE_DATA_BUF_ENABLE_ERROR_INJECT_MASK                                       0x00018000L
+#define SQC_DSM_CNTL2__DATA_CU2_WRITE_DATA_BUF_SELECT_INJECT_DELAY_MASK                                       0x00020000L
+#define SQC_DSM_CNTL2__DATA_CU2_UTCL1_LFIFO_ENABLE_ERROR_INJECT_MASK                                          0x000C0000L
+#define SQC_DSM_CNTL2__DATA_CU2_UTCL1_LFIFO_SELECT_INJECT_DELAY_MASK                                          0x00100000L
+#define SQC_DSM_CNTL2__INJECT_DELAY_MASK                                                                      0xFC000000L
+//SQC_DSM_CNTL2A
+#define SQC_DSM_CNTL2A__INST_TAG_RAM_ENABLE_ERROR_INJECT__SHIFT                                               0x0
+#define SQC_DSM_CNTL2A__INST_TAG_RAM_SELECT_INJECT_DELAY__SHIFT                                               0x2
+#define SQC_DSM_CNTL2A__INST_UTCL1_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                       0x3
+#define SQC_DSM_CNTL2A__INST_UTCL1_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                       0x5
+#define SQC_DSM_CNTL2A__INST_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                             0x6
+#define SQC_DSM_CNTL2A__INST_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                             0x8
+#define SQC_DSM_CNTL2A__INST_BANK_RAM_ENABLE_ERROR_INJECT__SHIFT                                              0x9
+#define SQC_DSM_CNTL2A__INST_BANK_RAM_SELECT_INJECT_DELAY__SHIFT                                              0xb
+#define SQC_DSM_CNTL2A__DATA_TAG_RAM_ENABLE_ERROR_INJECT__SHIFT                                               0xc
+#define SQC_DSM_CNTL2A__DATA_TAG_RAM_SELECT_INJECT_DELAY__SHIFT                                               0xe
+#define SQC_DSM_CNTL2A__DATA_HIT_FIFO_ENABLE_ERROR_INJECT__SHIFT                                              0xf
+#define SQC_DSM_CNTL2A__DATA_HIT_FIFO_SELECT_INJECT_DELAY__SHIFT                                              0x11
+#define SQC_DSM_CNTL2A__DATA_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                             0x12
+#define SQC_DSM_CNTL2A__DATA_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                             0x14
+#define SQC_DSM_CNTL2A__DATA_DIRTY_BIT_RAM_ENABLE_ERROR_INJECT__SHIFT                                         0x15
+#define SQC_DSM_CNTL2A__DATA_DIRTY_BIT_RAM_SELECT_INJECT_DELAY__SHIFT                                         0x17
+#define SQC_DSM_CNTL2A__DATA_BANK_RAM_ENABLE_ERROR_INJECT__SHIFT                                              0x18
+#define SQC_DSM_CNTL2A__DATA_BANK_RAM_SELECT_INJECT_DELAY__SHIFT                                              0x1a
+#define SQC_DSM_CNTL2A__INST_TAG_RAM_ENABLE_ERROR_INJECT_MASK                                                 0x00000003L
+#define SQC_DSM_CNTL2A__INST_TAG_RAM_SELECT_INJECT_DELAY_MASK                                                 0x00000004L
+#define SQC_DSM_CNTL2A__INST_UTCL1_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                         0x00000018L
+#define SQC_DSM_CNTL2A__INST_UTCL1_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                         0x00000020L
+#define SQC_DSM_CNTL2A__INST_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                               0x000000C0L
+#define SQC_DSM_CNTL2A__INST_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                               0x00000100L
+#define SQC_DSM_CNTL2A__INST_BANK_RAM_ENABLE_ERROR_INJECT_MASK                                                0x00000600L
+#define SQC_DSM_CNTL2A__INST_BANK_RAM_SELECT_INJECT_DELAY_MASK                                                0x00000800L
+#define SQC_DSM_CNTL2A__DATA_TAG_RAM_ENABLE_ERROR_INJECT_MASK                                                 0x00003000L
+#define SQC_DSM_CNTL2A__DATA_TAG_RAM_SELECT_INJECT_DELAY_MASK                                                 0x00004000L
+#define SQC_DSM_CNTL2A__DATA_HIT_FIFO_ENABLE_ERROR_INJECT_MASK                                                0x00018000L
+#define SQC_DSM_CNTL2A__DATA_HIT_FIFO_SELECT_INJECT_DELAY_MASK                                                0x00020000L
+#define SQC_DSM_CNTL2A__DATA_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                               0x000C0000L
+#define SQC_DSM_CNTL2A__DATA_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                               0x00100000L
+#define SQC_DSM_CNTL2A__DATA_DIRTY_BIT_RAM_ENABLE_ERROR_INJECT_MASK                                           0x00600000L
+#define SQC_DSM_CNTL2A__DATA_DIRTY_BIT_RAM_SELECT_INJECT_DELAY_MASK                                           0x00800000L
+#define SQC_DSM_CNTL2A__DATA_BANK_RAM_ENABLE_ERROR_INJECT_MASK                                                0x03000000L
+#define SQC_DSM_CNTL2A__DATA_BANK_RAM_SELECT_INJECT_DELAY_MASK                                                0x04000000L
+//SQC_DSM_CNTL2B
+#define SQC_DSM_CNTL2B__INST_TAG_RAM_ENABLE_ERROR_INJECT__SHIFT                                               0x0
+#define SQC_DSM_CNTL2B__INST_TAG_RAM_SELECT_INJECT_DELAY__SHIFT                                               0x2
+#define SQC_DSM_CNTL2B__INST_UTCL1_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                       0x3
+#define SQC_DSM_CNTL2B__INST_UTCL1_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                       0x5
+#define SQC_DSM_CNTL2B__INST_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                             0x6
+#define SQC_DSM_CNTL2B__INST_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                             0x8
+#define SQC_DSM_CNTL2B__INST_BANK_RAM_ENABLE_ERROR_INJECT__SHIFT                                              0x9
+#define SQC_DSM_CNTL2B__INST_BANK_RAM_SELECT_INJECT_DELAY__SHIFT                                              0xb
+#define SQC_DSM_CNTL2B__DATA_TAG_RAM_ENABLE_ERROR_INJECT__SHIFT                                               0xc
+#define SQC_DSM_CNTL2B__DATA_TAG_RAM_SELECT_INJECT_DELAY__SHIFT                                               0xe
+#define SQC_DSM_CNTL2B__DATA_HIT_FIFO_ENABLE_ERROR_INJECT__SHIFT                                              0xf
+#define SQC_DSM_CNTL2B__DATA_HIT_FIFO_SELECT_INJECT_DELAY__SHIFT                                              0x11
+#define SQC_DSM_CNTL2B__DATA_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                             0x12
+#define SQC_DSM_CNTL2B__DATA_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                             0x14
+#define SQC_DSM_CNTL2B__DATA_DIRTY_BIT_RAM_ENABLE_ERROR_INJECT__SHIFT                                         0x15
+#define SQC_DSM_CNTL2B__DATA_DIRTY_BIT_RAM_SELECT_INJECT_DELAY__SHIFT                                         0x17
+#define SQC_DSM_CNTL2B__DATA_BANK_RAM_ENABLE_ERROR_INJECT__SHIFT                                              0x18
+#define SQC_DSM_CNTL2B__DATA_BANK_RAM_SELECT_INJECT_DELAY__SHIFT                                              0x1a
+#define SQC_DSM_CNTL2B__INST_TAG_RAM_ENABLE_ERROR_INJECT_MASK                                                 0x00000003L
+#define SQC_DSM_CNTL2B__INST_TAG_RAM_SELECT_INJECT_DELAY_MASK                                                 0x00000004L
+#define SQC_DSM_CNTL2B__INST_UTCL1_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                         0x00000018L
+#define SQC_DSM_CNTL2B__INST_UTCL1_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                         0x00000020L
+#define SQC_DSM_CNTL2B__INST_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                               0x000000C0L
+#define SQC_DSM_CNTL2B__INST_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                               0x00000100L
+#define SQC_DSM_CNTL2B__INST_BANK_RAM_ENABLE_ERROR_INJECT_MASK                                                0x00000600L
+#define SQC_DSM_CNTL2B__INST_BANK_RAM_SELECT_INJECT_DELAY_MASK                                                0x00000800L
+#define SQC_DSM_CNTL2B__DATA_TAG_RAM_ENABLE_ERROR_INJECT_MASK                                                 0x00003000L
+#define SQC_DSM_CNTL2B__DATA_TAG_RAM_SELECT_INJECT_DELAY_MASK                                                 0x00004000L
+#define SQC_DSM_CNTL2B__DATA_HIT_FIFO_ENABLE_ERROR_INJECT_MASK                                                0x00018000L
+#define SQC_DSM_CNTL2B__DATA_HIT_FIFO_SELECT_INJECT_DELAY_MASK                                                0x00020000L
+#define SQC_DSM_CNTL2B__DATA_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                               0x000C0000L
+#define SQC_DSM_CNTL2B__DATA_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                               0x00100000L
+#define SQC_DSM_CNTL2B__DATA_DIRTY_BIT_RAM_ENABLE_ERROR_INJECT_MASK                                           0x00600000L
+#define SQC_DSM_CNTL2B__DATA_DIRTY_BIT_RAM_SELECT_INJECT_DELAY_MASK                                           0x00800000L
+#define SQC_DSM_CNTL2B__DATA_BANK_RAM_ENABLE_ERROR_INJECT_MASK                                                0x03000000L
+#define SQC_DSM_CNTL2B__DATA_BANK_RAM_SELECT_INJECT_DELAY_MASK                                                0x04000000L
+//SQC_DSM_CNTL2E
+#define SQC_DSM_CNTL2E__DATA_CU3_WRITE_DATA_BUF_ENABLE_ERROR_INJECT__SHIFT                                    0x0
+#define SQC_DSM_CNTL2E__DATA_CU3_WRITE_DATA_BUF_SELECT_INJECT_DELAY__SHIFT                                    0x2
+#define SQC_DSM_CNTL2E__DATA_CU3_UTCL1_LFIFO_ENABLE_ERROR_INJECT__SHIFT                                       0x3
+#define SQC_DSM_CNTL2E__DATA_CU3_UTCL1_LFIFO_SELECT_INJECT_DELAY__SHIFT                                       0x5
+#define SQC_DSM_CNTL2E__DATA_CU3_WRITE_DATA_BUF_ENABLE_ERROR_INJECT_MASK                                      0x00000003L
+#define SQC_DSM_CNTL2E__DATA_CU3_WRITE_DATA_BUF_SELECT_INJECT_DELAY_MASK                                      0x00000004L
+#define SQC_DSM_CNTL2E__DATA_CU3_UTCL1_LFIFO_ENABLE_ERROR_INJECT_MASK                                         0x00000018L
+#define SQC_DSM_CNTL2E__DATA_CU3_UTCL1_LFIFO_SELECT_INJECT_DELAY_MASK                                         0x00000020L
+//SQC_EDC_FUE_CNTL
+#define SQC_EDC_FUE_CNTL__BLOCK_FUE_FLAGS__SHIFT                                                              0x0
+#define SQC_EDC_FUE_CNTL__FUE_INTERRUPT_ENABLES__SHIFT                                                        0x10
+#define SQC_EDC_FUE_CNTL__BLOCK_FUE_FLAGS_MASK                                                                0x0000FFFFL
+#define SQC_EDC_FUE_CNTL__FUE_INTERRUPT_ENABLES_MASK                                                          0xFFFF0000L
+//SQC_EDC_CNT2
+#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_SEC_COUNT__SHIFT                                                     0x0
+#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_DED_COUNT__SHIFT                                                     0x2
+#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_SEC_COUNT__SHIFT                                                    0x4
+#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_DED_COUNT__SHIFT                                                    0x6
+#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_SEC_COUNT__SHIFT                                                     0x8
+#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_DED_COUNT__SHIFT                                                     0xa
+#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_SEC_COUNT__SHIFT                                                    0xc
+#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_DED_COUNT__SHIFT                                                    0xe
+#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_SEC_COUNT__SHIFT                                                       0x10
+#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_DED_COUNT__SHIFT                                                       0x12
+#define SQC_EDC_CNT2__DATA_BANKA_DIRTY_BIT_RAM_SEC_COUNT__SHIFT                                               0x14
+#define SQC_EDC_CNT2__DATA_BANKA_DIRTY_BIT_RAM_DED_COUNT__SHIFT                                               0x16
+#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_SEC_COUNT_MASK                                                       0x00000003L
+#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_DED_COUNT_MASK                                                       0x0000000CL
+#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_SEC_COUNT_MASK                                                      0x00000030L
+#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_DED_COUNT_MASK                                                      0x000000C0L
+#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_SEC_COUNT_MASK                                                       0x00000300L
+#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_DED_COUNT_MASK                                                       0x00000C00L
+#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_SEC_COUNT_MASK                                                      0x00003000L
+#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_DED_COUNT_MASK                                                      0x0000C000L
+#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_SEC_COUNT_MASK                                                         0x00030000L
+#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_DED_COUNT_MASK                                                         0x000C0000L
+#define SQC_EDC_CNT2__DATA_BANKA_DIRTY_BIT_RAM_SEC_COUNT_MASK                                                 0x00300000L
+#define SQC_EDC_CNT2__DATA_BANKA_DIRTY_BIT_RAM_DED_COUNT_MASK                                                 0x00C00000L
+//SQC_EDC_CNT3
+#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_SEC_COUNT__SHIFT                                                     0x0
+#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_DED_COUNT__SHIFT                                                     0x2
+#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_SEC_COUNT__SHIFT                                                    0x4
+#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_DED_COUNT__SHIFT                                                    0x6
+#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_SEC_COUNT__SHIFT                                                     0x8
+#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_DED_COUNT__SHIFT                                                     0xa
+#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_SEC_COUNT__SHIFT                                                    0xc
+#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_DED_COUNT__SHIFT                                                    0xe
+#define SQC_EDC_CNT3__DATA_BANKB_DIRTY_BIT_RAM_SEC_COUNT__SHIFT                                               0x10
+#define SQC_EDC_CNT3__DATA_BANKB_DIRTY_BIT_RAM_DED_COUNT__SHIFT                                               0x12
+#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_SEC_COUNT_MASK                                                       0x00000003L
+#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_DED_COUNT_MASK                                                       0x0000000CL
+#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_SEC_COUNT_MASK                                                      0x00000030L
+#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_DED_COUNT_MASK                                                      0x000000C0L
+#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_SEC_COUNT_MASK                                                       0x00000300L
+#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_DED_COUNT_MASK                                                       0x00000C00L
+#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_SEC_COUNT_MASK                                                      0x00003000L
+#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_DED_COUNT_MASK                                                      0x0000C000L
+#define SQC_EDC_CNT3__DATA_BANKB_DIRTY_BIT_RAM_SEC_COUNT_MASK                                                 0x00030000L
+#define SQC_EDC_CNT3__DATA_BANKB_DIRTY_BIT_RAM_DED_COUNT_MASK                                                 0x000C0000L
+//SQC_EDC_PARITY_CNT3
+#define SQC_EDC_PARITY_CNT3__INST_BANKA_UTCL1_MISS_FIFO_SEC_COUNT__SHIFT                                      0x0
+#define SQC_EDC_PARITY_CNT3__INST_BANKA_UTCL1_MISS_FIFO_DED_COUNT__SHIFT                                      0x2
+#define SQC_EDC_PARITY_CNT3__INST_BANKA_MISS_FIFO_SEC_COUNT__SHIFT                                            0x4
+#define SQC_EDC_PARITY_CNT3__INST_BANKA_MISS_FIFO_DED_COUNT__SHIFT                                            0x6
+#define SQC_EDC_PARITY_CNT3__DATA_BANKA_HIT_FIFO_SEC_COUNT__SHIFT                                             0x8
+#define SQC_EDC_PARITY_CNT3__DATA_BANKA_HIT_FIFO_DED_COUNT__SHIFT                                             0xa
+#define SQC_EDC_PARITY_CNT3__DATA_BANKA_MISS_FIFO_SEC_COUNT__SHIFT                                            0xc
+#define SQC_EDC_PARITY_CNT3__DATA_BANKA_MISS_FIFO_DED_COUNT__SHIFT                                            0xe
+#define SQC_EDC_PARITY_CNT3__INST_BANKB_UTCL1_MISS_FIFO_SEC_COUNT__SHIFT                                      0x10
+#define SQC_EDC_PARITY_CNT3__INST_BANKB_UTCL1_MISS_FIFO_DED_COUNT__SHIFT                                      0x12
+#define SQC_EDC_PARITY_CNT3__INST_BANKB_MISS_FIFO_SEC_COUNT__SHIFT                                            0x14
+#define SQC_EDC_PARITY_CNT3__INST_BANKB_MISS_FIFO_DED_COUNT__SHIFT                                            0x16
+#define SQC_EDC_PARITY_CNT3__DATA_BANKB_HIT_FIFO_SEC_COUNT__SHIFT                                             0x18
+#define SQC_EDC_PARITY_CNT3__DATA_BANKB_HIT_FIFO_DED_COUNT__SHIFT                                             0x1a
+#define SQC_EDC_PARITY_CNT3__DATA_BANKB_MISS_FIFO_SEC_COUNT__SHIFT                                            0x1c
+#define SQC_EDC_PARITY_CNT3__DATA_BANKB_MISS_FIFO_DED_COUNT__SHIFT                                            0x1e
+#define SQC_EDC_PARITY_CNT3__INST_BANKA_UTCL1_MISS_FIFO_SEC_COUNT_MASK                                        0x00000003L
+#define SQC_EDC_PARITY_CNT3__INST_BANKA_UTCL1_MISS_FIFO_DED_COUNT_MASK                                        0x0000000CL
+#define SQC_EDC_PARITY_CNT3__INST_BANKA_MISS_FIFO_SEC_COUNT_MASK                                              0x00000030L
+#define SQC_EDC_PARITY_CNT3__INST_BANKA_MISS_FIFO_DED_COUNT_MASK                                              0x000000C0L
+#define SQC_EDC_PARITY_CNT3__DATA_BANKA_HIT_FIFO_SEC_COUNT_MASK                                               0x00000300L
+#define SQC_EDC_PARITY_CNT3__DATA_BANKA_HIT_FIFO_DED_COUNT_MASK                                               0x00000C00L
+#define SQC_EDC_PARITY_CNT3__DATA_BANKA_MISS_FIFO_SEC_COUNT_MASK                                              0x00003000L
+#define SQC_EDC_PARITY_CNT3__DATA_BANKA_MISS_FIFO_DED_COUNT_MASK                                              0x0000C000L
+#define SQC_EDC_PARITY_CNT3__INST_BANKB_UTCL1_MISS_FIFO_SEC_COUNT_MASK                                        0x00030000L
+#define SQC_EDC_PARITY_CNT3__INST_BANKB_UTCL1_MISS_FIFO_DED_COUNT_MASK                                        0x000C0000L
+#define SQC_EDC_PARITY_CNT3__INST_BANKB_MISS_FIFO_SEC_COUNT_MASK                                              0x00300000L
+#define SQC_EDC_PARITY_CNT3__INST_BANKB_MISS_FIFO_DED_COUNT_MASK                                              0x00C00000L
+#define SQC_EDC_PARITY_CNT3__DATA_BANKB_HIT_FIFO_SEC_COUNT_MASK                                               0x03000000L
+#define SQC_EDC_PARITY_CNT3__DATA_BANKB_HIT_FIFO_DED_COUNT_MASK                                               0x0C000000L
+#define SQC_EDC_PARITY_CNT3__DATA_BANKB_MISS_FIFO_SEC_COUNT_MASK                                              0x30000000L
+#define SQC_EDC_PARITY_CNT3__DATA_BANKB_MISS_FIFO_DED_COUNT_MASK                                              0xC0000000L
+//SQ_DEBUG
+#define SQ_DEBUG__SINGLE_MEMOP__SHIFT                                                                         0x0
+#define SQ_DEBUG__SINGLE_ALU_OP__SHIFT                                                                        0x1
+#define SQ_DEBUG__SINGLE_MEMOP_MASK                                                                           0x00000001L
+#define SQ_DEBUG__SINGLE_ALU_OP_MASK                                                                          0x00000002L
+//SQ_PERF_SNAPSHOT_CTRL
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_INTVAL__SHIFT                                                            0x0
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_SEL__SHIFT                                                               0x5
+#define SQ_PERF_SNAPSHOT_CTRL__VMID_MASK__SHIFT                                                               0x6
+#define SQ_PERF_SNAPSHOT_CTRL__ENABLE__SHIFT                                                                  0x16
+#define SQ_PERF_SNAPSHOT_CTRL__TEST_MODE__SHIFT                                                               0x17
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_INTVAL_MASK                                                              0x0000001FL
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_SEL_MASK                                                                 0x00000020L
+#define SQ_PERF_SNAPSHOT_CTRL__VMID_MASK_MASK                                                                 0x003FFFC0L
+#define SQ_PERF_SNAPSHOT_CTRL__ENABLE_MASK                                                                    0x00400000L
+#define SQ_PERF_SNAPSHOT_CTRL__TEST_MODE_MASK                                                                 0x00800000L
+//SQ_DEBUG_FOR_INTERNAL_CTRL
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__FLAG_WR_ADDR_MATCH_FLAT_SCRATCH__SHIFT                                    0x0
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__FLAG_RD_FLAT_SCRATCH_RETURN_ZERO__SHIFT                                   0x1
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__DISABLE_FLAT_SCRATCH_WRITE_PROTECT__SHIFT                                 0x2
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__DISABLE_FLAT_SCRATCH_READ_PROTECT__SHIFT                                  0x3
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__DISABLE_SNAPSHOT_TRAP_ON_BARRIER__SHIFT                                   0x4
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__ENABLE_DED_TRIGGER_HALT__SHIFT                                            0x5
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__FLAG_WR_ADDR_MATCH_FLAT_SCRATCH_MASK                                      0x00000001L
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__FLAG_RD_FLAT_SCRATCH_RETURN_ZERO_MASK                                     0x00000002L
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__DISABLE_FLAT_SCRATCH_WRITE_PROTECT_MASK                                   0x00000004L
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__DISABLE_FLAT_SCRATCH_READ_PROTECT_MASK                                    0x00000008L
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__DISABLE_SNAPSHOT_TRAP_ON_BARRIER_MASK                                     0x00000010L
+#define SQ_DEBUG_FOR_INTERNAL_CTRL__ENABLE_DED_TRIGGER_HALT_MASK                                              0x00000020L
+//SQ_REG_TIMESTAMP
+#define SQ_REG_TIMESTAMP__TIMESTAMP__SHIFT                                                                    0x0
+#define SQ_REG_TIMESTAMP__TIMESTAMP_MASK                                                                      0x000000FFL
+//SQ_CMD_TIMESTAMP
+#define SQ_CMD_TIMESTAMP__TIMESTAMP__SHIFT                                                                    0x0
+#define SQ_CMD_TIMESTAMP__TIMESTAMP_MASK                                                                      0x000000FFL
+//SQ_HOSTTRAP_STATUS
+#define SQ_HOSTTRAP_STATUS__HTPENDINGCOUNT__SHIFT                                                             0x0
+#define SQ_HOSTTRAP_STATUS__HTPENDING_OVERRIDE__SHIFT                                                         0x8
+#define SQ_HOSTTRAP_STATUS__HTPENDINGCOUNT_MASK                                                               0x000000FFL
+#define SQ_HOSTTRAP_STATUS__HTPENDING_OVERRIDE_MASK                                                           0x00000100L
+//SQ_IND_INDEX
+#define SQ_IND_INDEX__WAVE_ID__SHIFT                                                                          0x0
+#define SQ_IND_INDEX__SIMD_ID__SHIFT                                                                          0x4
+#define SQ_IND_INDEX__THREAD_ID__SHIFT                                                                        0x6
+#define SQ_IND_INDEX__AUTO_INCR__SHIFT                                                                        0xc
+#define SQ_IND_INDEX__FORCE_READ__SHIFT                                                                       0xd
+#define SQ_IND_INDEX__READ_TIMEOUT__SHIFT                                                                     0xe
+#define SQ_IND_INDEX__UNINDEXED__SHIFT                                                                        0xf
+#define SQ_IND_INDEX__INDEX__SHIFT                                                                            0x10
+#define SQ_IND_INDEX__WAVE_ID_MASK                                                                            0x0000000FL
+#define SQ_IND_INDEX__SIMD_ID_MASK                                                                            0x00000030L
+#define SQ_IND_INDEX__THREAD_ID_MASK                                                                          0x00000FC0L
+#define SQ_IND_INDEX__AUTO_INCR_MASK                                                                          0x00001000L
+#define SQ_IND_INDEX__FORCE_READ_MASK                                                                         0x00002000L
+#define SQ_IND_INDEX__READ_TIMEOUT_MASK                                                                       0x00004000L
+#define SQ_IND_INDEX__UNINDEXED_MASK                                                                          0x00008000L
+#define SQ_IND_INDEX__INDEX_MASK                                                                              0xFFFF0000L
+//SQ_IND_DATA
+#define SQ_IND_DATA__DATA__SHIFT                                                                              0x0
+#define SQ_IND_DATA__DATA_MASK                                                                                0xFFFFFFFFL
+//SQ_CONFIG1
+#define SQ_CONFIG1__DISABLE_XDL_PORTD_CO_EXEC__SHIFT                                                          0x0
+#define SQ_CONFIG1__DISABLE_MGCG_ON_IBUF__SHIFT                                                               0x1
+#define SQ_CONFIG1__DISABLE_MGCG_ON_PERF__SHIFT                                                               0x2
+#define SQ_CONFIG1__DISABLE_MGCG_ON_EXP__SHIFT                                                                0x3
+#define SQ_CONFIG1__DISABLE_MGCG_ON_SCA__SHIFT                                                                0x4
+#define SQ_CONFIG1__DISABLE_MGCG_ON_SREG__SHIFT                                                               0x5
+#define SQ_CONFIG1__DISABLE_MGCG_ON_VDEC__SHIFT                                                               0x6
+#define SQ_CONFIG1__EXTRA_DGEMM_PROTECT_EN__SHIFT                                                             0x7
+#define SQ_CONFIG1__DISABLE_VALU_COEXEC__SHIFT                                                                0x8
+#define SQ_CONFIG1__DISABLE_WAVE_VALU_COEXEC__SHIFT                                                           0x9
+#define SQ_CONFIG1__VGPR_ARB_PLUS1__SHIFT                                                                     0xa
+#define SQ_CONFIG1__DISABLE_VGPR_COLLAPSE__SHIFT                                                              0xb
+#define SQ_CONFIG1__DISABLE_XNACK_CHECK_IN_RETRY_DISABLE__SHIFT                                               0xc
+#define SQ_CONFIG1__DISABLE_BARRIER_ADDR_WATCH__SHIFT                                                         0xd
+#define SQ_CONFIG1__DISABLE_BARRIER_MEMVIOL_WAIT__SHIFT                                                       0xe
+#define SQ_CONFIG1__DISABLE_BARRIER_MEMVIOL_BACKOFF__SHIFT                                                    0xf
+#define SQ_CONFIG1__EXPAND_SP_CMD_FGCG__SHIFT                                                                 0x10
+#define SQ_CONFIG1__EXPAND_SP_EXPORT_FGCG__SHIFT                                                              0x11
+#define SQ_CONFIG1__DISABLE_MGCG_ON_PERF_SNAP__SHIFT                                                          0x12
+#define SQ_CONFIG1__DISABLE_VALU_COEXEC_MODE_AUTO_CLEAN__SHIFT                                                0x13
+#define SQ_CONFIG1__SP_CORE1_MGCG_OVERRIDE__SHIFT                                                             0x14
+#define SQ_CONFIG1__SP_CORE4_MGCG_OVERRIDE__SHIFT                                                             0x15
+#define SQ_CONFIG1__SP_CORE5_MGCG_OVERRIDE__SHIFT                                                             0x16
+#define SQ_CONFIG1__DISABLE_SP_VGPR_COLLAPSE__SHIFT                                                           0x17
+#define SQ_CONFIG1__SP_FGCG_REP_OVERRIDE__SHIFT                                                               0x18
+#define SQ_CONFIG1__DPMACC_MGCG_OVERRIDE__SHIFT                                                               0x19
+#define SQ_CONFIG1__XDLMACC_MGCG_OVERRIDE__SHIFT                                                              0x1a
+#define SQ_CONFIG1__TRANSMACC_MGCG_OVERRIDE__SHIFT                                                            0x1b
+#define SQ_CONFIG1__SPMACC_MGCG_OVERRIDE__SHIFT                                                               0x1c
+#define SQ_CONFIG1__DPMACC_DGEMM2X_MGCG_OVERRIDE__SHIFT                                                       0x1d
+#define SQ_CONFIG1__DISABLE_SP_VGPR_READ_SKIP__SHIFT                                                          0x1e
+#define SQ_CONFIG1__SP_SRC_1ST_BUFFER_MGCG_OVERRIDE__SHIFT                                                    0x1f
+#define SQ_CONFIG1__DISABLE_XDL_PORTD_CO_EXEC_MASK                                                            0x00000001L
+#define SQ_CONFIG1__DISABLE_MGCG_ON_IBUF_MASK                                                                 0x00000002L
+#define SQ_CONFIG1__DISABLE_MGCG_ON_PERF_MASK                                                                 0x00000004L
+#define SQ_CONFIG1__DISABLE_MGCG_ON_EXP_MASK                                                                  0x00000008L
+#define SQ_CONFIG1__DISABLE_MGCG_ON_SCA_MASK                                                                  0x00000010L
+#define SQ_CONFIG1__DISABLE_MGCG_ON_SREG_MASK                                                                 0x00000020L
+#define SQ_CONFIG1__DISABLE_MGCG_ON_VDEC_MASK                                                                 0x00000040L
+#define SQ_CONFIG1__EXTRA_DGEMM_PROTECT_EN_MASK                                                               0x00000080L
+#define SQ_CONFIG1__DISABLE_VALU_COEXEC_MASK                                                                  0x00000100L
+#define SQ_CONFIG1__DISABLE_WAVE_VALU_COEXEC_MASK                                                             0x00000200L
+#define SQ_CONFIG1__VGPR_ARB_PLUS1_MASK                                                                       0x00000400L
+#define SQ_CONFIG1__DISABLE_VGPR_COLLAPSE_MASK                                                                0x00000800L
+#define SQ_CONFIG1__DISABLE_XNACK_CHECK_IN_RETRY_DISABLE_MASK                                                 0x00001000L
+#define SQ_CONFIG1__DISABLE_BARRIER_ADDR_WATCH_MASK                                                           0x00002000L
+#define SQ_CONFIG1__DISABLE_BARRIER_MEMVIOL_WAIT_MASK                                                         0x00004000L
+#define SQ_CONFIG1__DISABLE_BARRIER_MEMVIOL_BACKOFF_MASK                                                      0x00008000L
+#define SQ_CONFIG1__EXPAND_SP_CMD_FGCG_MASK                                                                   0x00010000L
+#define SQ_CONFIG1__EXPAND_SP_EXPORT_FGCG_MASK                                                                0x00020000L
+#define SQ_CONFIG1__DISABLE_MGCG_ON_PERF_SNAP_MASK                                                            0x00040000L
+#define SQ_CONFIG1__DISABLE_VALU_COEXEC_MODE_AUTO_CLEAN_MASK                                                  0x00080000L
+#define SQ_CONFIG1__SP_CORE1_MGCG_OVERRIDE_MASK                                                               0x00100000L
+#define SQ_CONFIG1__SP_CORE4_MGCG_OVERRIDE_MASK                                                               0x00200000L
+#define SQ_CONFIG1__SP_CORE5_MGCG_OVERRIDE_MASK                                                               0x00400000L
+#define SQ_CONFIG1__DISABLE_SP_VGPR_COLLAPSE_MASK                                                             0x00800000L
+#define SQ_CONFIG1__SP_FGCG_REP_OVERRIDE_MASK                                                                 0x01000000L
+#define SQ_CONFIG1__DPMACC_MGCG_OVERRIDE_MASK                                                                 0x02000000L
+#define SQ_CONFIG1__XDLMACC_MGCG_OVERRIDE_MASK                                                                0x04000000L
+#define SQ_CONFIG1__TRANSMACC_MGCG_OVERRIDE_MASK                                                              0x08000000L
+#define SQ_CONFIG1__SPMACC_MGCG_OVERRIDE_MASK                                                                 0x10000000L
+#define SQ_CONFIG1__DPMACC_DGEMM2X_MGCG_OVERRIDE_MASK                                                         0x20000000L
+#define SQ_CONFIG1__DISABLE_SP_VGPR_READ_SKIP_MASK                                                            0x40000000L
+#define SQ_CONFIG1__SP_SRC_1ST_BUFFER_MGCG_OVERRIDE_MASK                                                      0x80000000L
+//SQ_CMD
+#define SQ_CMD__CMD__SHIFT                                                                                    0x0
+#define SQ_CMD__MODE__SHIFT                                                                                   0x4
+#define SQ_CMD__CHECK_VMID__SHIFT                                                                             0x7
+#define SQ_CMD__DATA__SHIFT                                                                                   0x8
+#define SQ_CMD__WAVE_ID__SHIFT                                                                                0x10
+#define SQ_CMD__SIMD_ID__SHIFT                                                                                0x14
+#define SQ_CMD__QUEUE_ID__SHIFT                                                                               0x18
+#define SQ_CMD__VM_ID__SHIFT                                                                                  0x1c
+#define SQ_CMD__CMD_MASK                                                                                      0x00000007L
+#define SQ_CMD__MODE_MASK                                                                                     0x00000070L
+#define SQ_CMD__CHECK_VMID_MASK                                                                               0x00000080L
+#define SQ_CMD__DATA_MASK                                                                                     0x00000F00L
+#define SQ_CMD__WAVE_ID_MASK                                                                                  0x000F0000L
+#define SQ_CMD__SIMD_ID_MASK                                                                                  0x00300000L
+#define SQ_CMD__QUEUE_ID_MASK                                                                                 0x07000000L
+#define SQ_CMD__VM_ID_MASK                                                                                    0xF0000000L
+//SQ_TIME_HI
+#define SQ_TIME_HI__TIME__SHIFT                                                                               0x0
+#define SQ_TIME_HI__TIME_MASK                                                                                 0xFFFFFFFFL
+//SQ_TIME_LO
+#define SQ_TIME_LO__TIME__SHIFT                                                                               0x0
+#define SQ_TIME_LO__TIME_MASK                                                                                 0xFFFFFFFFL
+//SQ_DS_0
+#define SQ_DS_0__OFFSET0__SHIFT                                                                               0x0
+#define SQ_DS_0__OFFSET1__SHIFT                                                                               0x8
+#define SQ_DS_0__GDS__SHIFT                                                                                   0x10
+#define SQ_DS_0__OP__SHIFT                                                                                    0x11
+#define SQ_DS_0__ACC__SHIFT                                                                                   0x19
+#define SQ_DS_0__ENCODING__SHIFT                                                                              0x1a
+#define SQ_DS_0__OFFSET0_MASK                                                                                 0x000000FFL
+#define SQ_DS_0__OFFSET1_MASK                                                                                 0x0000FF00L
+#define SQ_DS_0__GDS_MASK                                                                                     0x00010000L
+#define SQ_DS_0__OP_MASK                                                                                      0x01FE0000L
+#define SQ_DS_0__ACC_MASK                                                                                     0x02000000L
+#define SQ_DS_0__ENCODING_MASK                                                                                0xFC000000L
+//SQ_DS_1
+#define SQ_DS_1__ADDR__SHIFT                                                                                  0x0
+#define SQ_DS_1__DATA0__SHIFT                                                                                 0x8
+#define SQ_DS_1__DATA1__SHIFT                                                                                 0x10
+#define SQ_DS_1__VDST__SHIFT                                                                                  0x18
+#define SQ_DS_1__ADDR_MASK                                                                                    0x000000FFL
+#define SQ_DS_1__DATA0_MASK                                                                                   0x0000FF00L
+#define SQ_DS_1__DATA1_MASK                                                                                   0x00FF0000L
+#define SQ_DS_1__VDST_MASK                                                                                    0xFF000000L
+//SQ_EXP_0
+#define SQ_EXP_0__EN__SHIFT                                                                                   0x0
+#define SQ_EXP_0__TGT__SHIFT                                                                                  0x4
+#define SQ_EXP_0__COMPR__SHIFT                                                                                0xa
+#define SQ_EXP_0__DONE__SHIFT                                                                                 0xb
+#define SQ_EXP_0__VM__SHIFT                                                                                   0xc
+#define SQ_EXP_0__ENCODING__SHIFT                                                                             0x1a
+#define SQ_EXP_0__EN_MASK                                                                                     0x0000000FL
+#define SQ_EXP_0__TGT_MASK                                                                                    0x000003F0L
+#define SQ_EXP_0__COMPR_MASK                                                                                  0x00000400L
+#define SQ_EXP_0__DONE_MASK                                                                                   0x00000800L
+#define SQ_EXP_0__VM_MASK                                                                                     0x00001000L
+#define SQ_EXP_0__ENCODING_MASK                                                                               0xFC000000L
+//SQ_EXP_1
+#define SQ_EXP_1__VSRC0__SHIFT                                                                                0x0
+#define SQ_EXP_1__VSRC1__SHIFT                                                                                0x8
+#define SQ_EXP_1__VSRC2__SHIFT                                                                                0x10
+#define SQ_EXP_1__VSRC3__SHIFT                                                                                0x18
+#define SQ_EXP_1__VSRC0_MASK                                                                                  0x000000FFL
+#define SQ_EXP_1__VSRC1_MASK                                                                                  0x0000FF00L
+#define SQ_EXP_1__VSRC2_MASK                                                                                  0x00FF0000L
+#define SQ_EXP_1__VSRC3_MASK                                                                                  0xFF000000L
+//SQ_FLAT_0
+#define SQ_FLAT_0__OFFSET__SHIFT                                                                              0x0
+#define SQ_FLAT_0__SVE__SHIFT                                                                                 0xd
+#define SQ_FLAT_0__SEG__SHIFT                                                                                 0xe
+#define SQ_FLAT_0__SC0__SHIFT                                                                                 0x10
+#define SQ_FLAT_0__NT__SHIFT                                                                                  0x11
+#define SQ_FLAT_0__OP__SHIFT                                                                                  0x12
+#define SQ_FLAT_0__SC1__SHIFT                                                                                 0x19
+#define SQ_FLAT_0__ENCODING__SHIFT                                                                            0x1a
+#define SQ_FLAT_0__OFFSET_MASK                                                                                0x00000FFFL
+#define SQ_FLAT_0__SVE_MASK                                                                                   0x00002000L
+#define SQ_FLAT_0__SEG_MASK                                                                                   0x0000C000L
+#define SQ_FLAT_0__SC0_MASK                                                                                   0x00010000L
+#define SQ_FLAT_0__NT_MASK                                                                                    0x00020000L
+#define SQ_FLAT_0__OP_MASK                                                                                    0x01FC0000L
+#define SQ_FLAT_0__SC1_MASK                                                                                   0x02000000L
+#define SQ_FLAT_0__ENCODING_MASK                                                                              0xFC000000L
+//SQ_FLAT_1
+#define SQ_FLAT_1__ADDR__SHIFT                                                                                0x0
+#define SQ_FLAT_1__DATA__SHIFT                                                                                0x8
+#define SQ_FLAT_1__SADDR__SHIFT                                                                               0x10
+#define SQ_FLAT_1__ACC__SHIFT                                                                                 0x17
+#define SQ_FLAT_1__VDST__SHIFT                                                                                0x18
+#define SQ_FLAT_1__ADDR_MASK                                                                                  0x000000FFL
+#define SQ_FLAT_1__DATA_MASK                                                                                  0x0000FF00L
+#define SQ_FLAT_1__SADDR_MASK                                                                                 0x007F0000L
+#define SQ_FLAT_1__ACC_MASK                                                                                   0x00800000L
+#define SQ_FLAT_1__VDST_MASK                                                                                  0xFF000000L
+//SQ_GLBL_0
+#define SQ_GLBL_0__OFFSET__SHIFT                                                                              0x0
+#define SQ_GLBL_0__SVE__SHIFT                                                                                 0xd
+#define SQ_GLBL_0__SEG__SHIFT                                                                                 0xe
+#define SQ_GLBL_0__SC0__SHIFT                                                                                 0x10
+#define SQ_GLBL_0__NT__SHIFT                                                                                  0x11
+#define SQ_GLBL_0__OP__SHIFT                                                                                  0x12
+#define SQ_GLBL_0__SC1__SHIFT                                                                                 0x19
+#define SQ_GLBL_0__ENCODING__SHIFT                                                                            0x1a
+#define SQ_GLBL_0__OFFSET_MASK                                                                                0x00001FFFL
+#define SQ_GLBL_0__SVE_MASK                                                                                   0x00002000L
+#define SQ_GLBL_0__SEG_MASK                                                                                   0x0000C000L
+#define SQ_GLBL_0__SC0_MASK                                                                                   0x00010000L
+#define SQ_GLBL_0__NT_MASK                                                                                    0x00020000L
+#define SQ_GLBL_0__OP_MASK                                                                                    0x01FC0000L
+#define SQ_GLBL_0__SC1_MASK                                                                                   0x02000000L
+#define SQ_GLBL_0__ENCODING_MASK                                                                              0xFC000000L
+//SQ_GLBL_1
+#define SQ_GLBL_1__ADDR__SHIFT                                                                                0x0
+#define SQ_GLBL_1__DATA__SHIFT                                                                                0x8
+#define SQ_GLBL_1__SADDR__SHIFT                                                                               0x10
+#define SQ_GLBL_1__ACC__SHIFT                                                                                 0x17
+#define SQ_GLBL_1__VDST__SHIFT                                                                                0x18
+#define SQ_GLBL_1__ADDR_MASK                                                                                  0x000000FFL
+#define SQ_GLBL_1__DATA_MASK                                                                                  0x0000FF00L
+#define SQ_GLBL_1__SADDR_MASK                                                                                 0x007F0000L
+#define SQ_GLBL_1__ACC_MASK                                                                                   0x00800000L
+#define SQ_GLBL_1__VDST_MASK                                                                                  0xFF000000L
+//SQ_INST
+#define SQ_INST__ENCODING__SHIFT                                                                              0x0
+#define SQ_INST__ENCODING_MASK                                                                                0xFFFFFFFFL
+//SQ_MIMG_0
+#define SQ_MIMG_0__OPM__SHIFT                                                                                 0x0
+#define SQ_MIMG_0__SC1__SHIFT                                                                                 0x7
+#define SQ_MIMG_0__DMASK__SHIFT                                                                               0x8
+#define SQ_MIMG_0__UNORM__SHIFT                                                                               0xc
+#define SQ_MIMG_0__SC0__SHIFT                                                                                 0xd
+#define SQ_MIMG_0__DA__SHIFT                                                                                  0xe
+#define SQ_MIMG_0__A16__SHIFT                                                                                 0xf
+#define SQ_MIMG_0__ACC__SHIFT                                                                                 0x10
+#define SQ_MIMG_0__LWE__SHIFT                                                                                 0x11
+#define SQ_MIMG_0__OP__SHIFT                                                                                  0x12
+#define SQ_MIMG_0__NT__SHIFT                                                                                  0x19
+#define SQ_MIMG_0__ENCODING__SHIFT                                                                            0x1a
+#define SQ_MIMG_0__OPM_MASK                                                                                   0x00000001L
+#define SQ_MIMG_0__SC1_MASK                                                                                   0x00000080L
+#define SQ_MIMG_0__DMASK_MASK                                                                                 0x00000F00L
+#define SQ_MIMG_0__UNORM_MASK                                                                                 0x00001000L
+#define SQ_MIMG_0__SC0_MASK                                                                                   0x00002000L
+#define SQ_MIMG_0__DA_MASK                                                                                    0x00004000L
+#define SQ_MIMG_0__A16_MASK                                                                                   0x00008000L
+#define SQ_MIMG_0__ACC_MASK                                                                                   0x00010000L
+#define SQ_MIMG_0__LWE_MASK                                                                                   0x00020000L
+#define SQ_MIMG_0__OP_MASK                                                                                    0x01FC0000L
+#define SQ_MIMG_0__NT_MASK                                                                                    0x02000000L
+#define SQ_MIMG_0__ENCODING_MASK                                                                              0xFC000000L
+//SQ_MIMG_1
+#define SQ_MIMG_1__VADDR__SHIFT                                                                               0x0
+#define SQ_MIMG_1__VDATA__SHIFT                                                                               0x8
+#define SQ_MIMG_1__SRSRC__SHIFT                                                                               0x10
+#define SQ_MIMG_1__SSAMP__SHIFT                                                                               0x15
+#define SQ_MIMG_1__D16__SHIFT                                                                                 0x1f
+#define SQ_MIMG_1__VADDR_MASK                                                                                 0x000000FFL
+#define SQ_MIMG_1__VDATA_MASK                                                                                 0x0000FF00L
+#define SQ_MIMG_1__SRSRC_MASK                                                                                 0x001F0000L
+#define SQ_MIMG_1__SSAMP_MASK                                                                                 0x03E00000L
+#define SQ_MIMG_1__D16_MASK                                                                                   0x80000000L
+//SQ_MTBUF_0
+#define SQ_MTBUF_0__OFFSET__SHIFT                                                                             0x0
+#define SQ_MTBUF_0__OFFEN__SHIFT                                                                              0xc
+#define SQ_MTBUF_0__IDXEN__SHIFT                                                                              0xd
+#define SQ_MTBUF_0__SC0__SHIFT                                                                                0xe
+#define SQ_MTBUF_0__OP__SHIFT                                                                                 0xf
+#define SQ_MTBUF_0__DFMT__SHIFT                                                                               0x13
+#define SQ_MTBUF_0__NFMT__SHIFT                                                                               0x17
+#define SQ_MTBUF_0__ENCODING__SHIFT                                                                           0x1a
+#define SQ_MTBUF_0__OFFSET_MASK                                                                               0x00000FFFL
+#define SQ_MTBUF_0__OFFEN_MASK                                                                                0x00001000L
+#define SQ_MTBUF_0__IDXEN_MASK                                                                                0x00002000L
+#define SQ_MTBUF_0__SC0_MASK                                                                                  0x00004000L
+#define SQ_MTBUF_0__OP_MASK                                                                                   0x00078000L
+#define SQ_MTBUF_0__DFMT_MASK                                                                                 0x00780000L
+#define SQ_MTBUF_0__NFMT_MASK                                                                                 0x03800000L
+#define SQ_MTBUF_0__ENCODING_MASK                                                                             0xFC000000L
+//SQ_MTBUF_1
+#define SQ_MTBUF_1__VADDR__SHIFT                                                                              0x0
+#define SQ_MTBUF_1__VDATA__SHIFT                                                                              0x8
+#define SQ_MTBUF_1__SRSRC__SHIFT                                                                              0x10
+#define SQ_MTBUF_1__SC1__SHIFT                                                                                0x15
+#define SQ_MTBUF_1__NT__SHIFT                                                                                 0x16
+#define SQ_MTBUF_1__ACC__SHIFT                                                                                0x17
+#define SQ_MTBUF_1__SOFFSET__SHIFT                                                                            0x18
+#define SQ_MTBUF_1__VADDR_MASK                                                                                0x000000FFL
+#define SQ_MTBUF_1__VDATA_MASK                                                                                0x0000FF00L
+#define SQ_MTBUF_1__SRSRC_MASK                                                                                0x001F0000L
+#define SQ_MTBUF_1__SC1_MASK                                                                                  0x00200000L
+#define SQ_MTBUF_1__NT_MASK                                                                                   0x00400000L
+#define SQ_MTBUF_1__ACC_MASK                                                                                  0x00800000L
+#define SQ_MTBUF_1__SOFFSET_MASK                                                                              0xFF000000L
+//SQ_MUBUF_0
+#define SQ_MUBUF_0__OFFSET__SHIFT                                                                             0x0
+#define SQ_MUBUF_0__OFFEN__SHIFT                                                                              0xc
+#define SQ_MUBUF_0__IDXEN__SHIFT                                                                              0xd
+#define SQ_MUBUF_0__SC0__SHIFT                                                                                0xe
+#define SQ_MUBUF_0__SC1__SHIFT                                                                                0xf
+#define SQ_MUBUF_0__LDS__SHIFT                                                                                0x10
+#define SQ_MUBUF_0__NT__SHIFT                                                                                 0x11
+#define SQ_MUBUF_0__OP__SHIFT                                                                                 0x12
+#define SQ_MUBUF_0__ENCODING__SHIFT                                                                           0x1a
+#define SQ_MUBUF_0__OFFSET_MASK                                                                               0x00000FFFL
+#define SQ_MUBUF_0__OFFEN_MASK                                                                                0x00001000L
+#define SQ_MUBUF_0__IDXEN_MASK                                                                                0x00002000L
+#define SQ_MUBUF_0__SC0_MASK                                                                                  0x00004000L
+#define SQ_MUBUF_0__SC1_MASK                                                                                  0x00008000L
+#define SQ_MUBUF_0__LDS_MASK                                                                                  0x00010000L
+#define SQ_MUBUF_0__NT_MASK                                                                                   0x00020000L
+#define SQ_MUBUF_0__OP_MASK                                                                                   0x01FC0000L
+#define SQ_MUBUF_0__ENCODING_MASK                                                                             0xFC000000L
+//SQ_MUBUF_1
+#define SQ_MUBUF_1__VADDR__SHIFT                                                                              0x0
+#define SQ_MUBUF_1__VDATA__SHIFT                                                                              0x8
+#define SQ_MUBUF_1__SRSRC__SHIFT                                                                              0x10
+#define SQ_MUBUF_1__ACC__SHIFT                                                                                0x17
+#define SQ_MUBUF_1__SOFFSET__SHIFT                                                                            0x18
+#define SQ_MUBUF_1__VADDR_MASK                                                                                0x000000FFL
+#define SQ_MUBUF_1__VDATA_MASK                                                                                0x0000FF00L
+#define SQ_MUBUF_1__SRSRC_MASK                                                                                0x001F0000L
+#define SQ_MUBUF_1__ACC_MASK                                                                                  0x00800000L
+#define SQ_MUBUF_1__SOFFSET_MASK                                                                              0xFF000000L
+//SQ_SCRATCH_0
+#define SQ_SCRATCH_0__OFFSET__SHIFT                                                                           0x0
+#define SQ_SCRATCH_0__SVE__SHIFT                                                                              0xd
+#define SQ_SCRATCH_0__SEG__SHIFT                                                                              0xe
+#define SQ_SCRATCH_0__SC0__SHIFT                                                                              0x10
+#define SQ_SCRATCH_0__NT__SHIFT                                                                               0x11
+#define SQ_SCRATCH_0__OP__SHIFT                                                                               0x12
+#define SQ_SCRATCH_0__SC1__SHIFT                                                                              0x19
+#define SQ_SCRATCH_0__ENCODING__SHIFT                                                                         0x1a
+#define SQ_SCRATCH_0__OFFSET_MASK                                                                             0x00001FFFL
+#define SQ_SCRATCH_0__SVE_MASK                                                                                0x00002000L
+#define SQ_SCRATCH_0__SEG_MASK                                                                                0x0000C000L
+#define SQ_SCRATCH_0__SC0_MASK                                                                                0x00010000L
+#define SQ_SCRATCH_0__NT_MASK                                                                                 0x00020000L
+#define SQ_SCRATCH_0__OP_MASK                                                                                 0x01FC0000L
+#define SQ_SCRATCH_0__SC1_MASK                                                                                0x02000000L
+#define SQ_SCRATCH_0__ENCODING_MASK                                                                           0xFC000000L
+//SQ_SCRATCH_1
+#define SQ_SCRATCH_1__ADDR__SHIFT                                                                             0x0
+#define SQ_SCRATCH_1__DATA__SHIFT                                                                             0x8
+#define SQ_SCRATCH_1__SADDR__SHIFT                                                                            0x10
+#define SQ_SCRATCH_1__ACC__SHIFT                                                                              0x17
+#define SQ_SCRATCH_1__VDST__SHIFT                                                                             0x18
+#define SQ_SCRATCH_1__ADDR_MASK                                                                               0x000000FFL
+#define SQ_SCRATCH_1__DATA_MASK                                                                               0x0000FF00L
+#define SQ_SCRATCH_1__SADDR_MASK                                                                              0x007F0000L
+#define SQ_SCRATCH_1__ACC_MASK                                                                                0x00800000L
+#define SQ_SCRATCH_1__VDST_MASK                                                                               0xFF000000L
+//SQ_SMEM_0
+#define SQ_SMEM_0__SBASE__SHIFT                                                                               0x0
+#define SQ_SMEM_0__SDATA__SHIFT                                                                               0x6
+#define SQ_SMEM_0__SOFFSET_EN__SHIFT                                                                          0xe
+#define SQ_SMEM_0__NV__SHIFT                                                                                  0xf
+#define SQ_SMEM_0__GLC__SHIFT                                                                                 0x10
+#define SQ_SMEM_0__IMM__SHIFT                                                                                 0x11
+#define SQ_SMEM_0__OP__SHIFT                                                                                  0x12
+#define SQ_SMEM_0__ENCODING__SHIFT                                                                            0x1a
+#define SQ_SMEM_0__SBASE_MASK                                                                                 0x0000003FL
+#define SQ_SMEM_0__SDATA_MASK                                                                                 0x00001FC0L
+#define SQ_SMEM_0__SOFFSET_EN_MASK                                                                            0x00004000L
+#define SQ_SMEM_0__NV_MASK                                                                                    0x00008000L
+#define SQ_SMEM_0__GLC_MASK                                                                                   0x00010000L
+#define SQ_SMEM_0__IMM_MASK                                                                                   0x00020000L
+#define SQ_SMEM_0__OP_MASK                                                                                    0x03FC0000L
+#define SQ_SMEM_0__ENCODING_MASK                                                                              0xFC000000L
+//SQ_SMEM_1
+#define SQ_SMEM_1__OFFSET__SHIFT                                                                              0x0
+#define SQ_SMEM_1__SOFFSET__SHIFT                                                                             0x19
+#define SQ_SMEM_1__OFFSET_MASK                                                                                0x001FFFFFL
+#define SQ_SMEM_1__SOFFSET_MASK                                                                               0xFE000000L
+//SQ_SOP1
+#define SQ_SOP1__SSRC0__SHIFT                                                                                 0x0
+#define SQ_SOP1__OP__SHIFT                                                                                    0x8
+#define SQ_SOP1__SDST__SHIFT                                                                                  0x10
+#define SQ_SOP1__ENCODING__SHIFT                                                                              0x17
+#define SQ_SOP1__SSRC0_MASK                                                                                   0x000000FFL
+#define SQ_SOP1__OP_MASK                                                                                      0x0000FF00L
+#define SQ_SOP1__SDST_MASK                                                                                    0x007F0000L
+#define SQ_SOP1__ENCODING_MASK                                                                                0xFF800000L
+//SQ_SOP2
+#define SQ_SOP2__SSRC0__SHIFT                                                                                 0x0
+#define SQ_SOP2__SSRC1__SHIFT                                                                                 0x8
+#define SQ_SOP2__SDST__SHIFT                                                                                  0x10
+#define SQ_SOP2__OP__SHIFT                                                                                    0x17
+#define SQ_SOP2__ENCODING__SHIFT                                                                              0x1e
+#define SQ_SOP2__SSRC0_MASK                                                                                   0x000000FFL
+#define SQ_SOP2__SSRC1_MASK                                                                                   0x0000FF00L
+#define SQ_SOP2__SDST_MASK                                                                                    0x007F0000L
+#define SQ_SOP2__OP_MASK                                                                                      0x3F800000L
+#define SQ_SOP2__ENCODING_MASK                                                                                0xC0000000L
+//SQ_SOPC
+#define SQ_SOPC__SSRC0__SHIFT                                                                                 0x0
+#define SQ_SOPC__SSRC1__SHIFT                                                                                 0x8
+#define SQ_SOPC__OP__SHIFT                                                                                    0x10
+#define SQ_SOPC__ENCODING__SHIFT                                                                              0x17
+#define SQ_SOPC__SSRC0_MASK                                                                                   0x000000FFL
+#define SQ_SOPC__SSRC1_MASK                                                                                   0x0000FF00L
+#define SQ_SOPC__OP_MASK                                                                                      0x007F0000L
+#define SQ_SOPC__ENCODING_MASK                                                                                0xFF800000L
+//SQ_SOPK
+#define SQ_SOPK__SIMM16__SHIFT                                                                                0x0
+#define SQ_SOPK__SDST__SHIFT                                                                                  0x10
+#define SQ_SOPK__OP__SHIFT                                                                                    0x17
+#define SQ_SOPK__ENCODING__SHIFT                                                                              0x1c
+#define SQ_SOPK__SIMM16_MASK                                                                                  0x0000FFFFL
+#define SQ_SOPK__SDST_MASK                                                                                    0x007F0000L
+#define SQ_SOPK__OP_MASK                                                                                      0x0F800000L
+#define SQ_SOPK__ENCODING_MASK                                                                                0xF0000000L
+//SQ_SOPP
+#define SQ_SOPP__SIMM16__SHIFT                                                                                0x0
+#define SQ_SOPP__OP__SHIFT                                                                                    0x10
+#define SQ_SOPP__ENCODING__SHIFT                                                                              0x17
+#define SQ_SOPP__SIMM16_MASK                                                                                  0x0000FFFFL
+#define SQ_SOPP__OP_MASK                                                                                      0x007F0000L
+#define SQ_SOPP__ENCODING_MASK                                                                                0xFF800000L
+//SQ_VINTRP
+#define SQ_VINTRP__VSRC__SHIFT                                                                                0x0
+#define SQ_VINTRP__ATTRCHAN__SHIFT                                                                            0x8
+#define SQ_VINTRP__ATTR__SHIFT                                                                                0xa
+#define SQ_VINTRP__OP__SHIFT                                                                                  0x10
+#define SQ_VINTRP__VDST__SHIFT                                                                                0x12
+#define SQ_VINTRP__ENCODING__SHIFT                                                                            0x1a
+#define SQ_VINTRP__VSRC_MASK                                                                                  0x000000FFL
+#define SQ_VINTRP__ATTRCHAN_MASK                                                                              0x00000300L
+#define SQ_VINTRP__ATTR_MASK                                                                                  0x0000FC00L
+#define SQ_VINTRP__OP_MASK                                                                                    0x00030000L
+#define SQ_VINTRP__VDST_MASK                                                                                  0x03FC0000L
+#define SQ_VINTRP__ENCODING_MASK                                                                              0xFC000000L
+//SQ_VOP1
+#define SQ_VOP1__SRC0__SHIFT                                                                                  0x0
+#define SQ_VOP1__OP__SHIFT                                                                                    0x9
+#define SQ_VOP1__VDST__SHIFT                                                                                  0x11
+#define SQ_VOP1__ENCODING__SHIFT                                                                              0x19
+#define SQ_VOP1__SRC0_MASK                                                                                    0x000001FFL
+#define SQ_VOP1__OP_MASK                                                                                      0x0001FE00L
+#define SQ_VOP1__VDST_MASK                                                                                    0x01FE0000L
+#define SQ_VOP1__ENCODING_MASK                                                                                0xFE000000L
+//SQ_VOP2
+#define SQ_VOP2__SRC0__SHIFT                                                                                  0x0
+#define SQ_VOP2__VSRC1__SHIFT                                                                                 0x9
+#define SQ_VOP2__VDST__SHIFT                                                                                  0x11
+#define SQ_VOP2__OP__SHIFT                                                                                    0x19
+#define SQ_VOP2__ENCODING__SHIFT                                                                              0x1f
+#define SQ_VOP2__SRC0_MASK                                                                                    0x000001FFL
+#define SQ_VOP2__VSRC1_MASK                                                                                   0x0001FE00L
+#define SQ_VOP2__VDST_MASK                                                                                    0x01FE0000L
+#define SQ_VOP2__OP_MASK                                                                                      0x7E000000L
+#define SQ_VOP2__ENCODING_MASK                                                                                0x80000000L
+//SQ_VOP3P_0
+#define SQ_VOP3P_0__VDST__SHIFT                                                                               0x0
+#define SQ_VOP3P_0__NEG_HI__SHIFT                                                                             0x8
+#define SQ_VOP3P_0__OP_SEL__SHIFT                                                                             0xb
+#define SQ_VOP3P_0__OP_SEL_HI_2__SHIFT                                                                        0xe
+#define SQ_VOP3P_0__CLAMP__SHIFT                                                                              0xf
+#define SQ_VOP3P_0__OP__SHIFT                                                                                 0x10
+#define SQ_VOP3P_0__ENCODING__SHIFT                                                                           0x17
+#define SQ_VOP3P_0__VDST_MASK                                                                                 0x000000FFL
+#define SQ_VOP3P_0__NEG_HI_MASK                                                                               0x00000700L
+#define SQ_VOP3P_0__OP_SEL_MASK                                                                               0x00003800L
+#define SQ_VOP3P_0__OP_SEL_HI_2_MASK                                                                          0x00004000L
+#define SQ_VOP3P_0__CLAMP_MASK                                                                                0x00008000L
+#define SQ_VOP3P_0__OP_MASK                                                                                   0x007F0000L
+#define SQ_VOP3P_0__ENCODING_MASK                                                                             0xFF800000L
+//SQ_VOP3P_1
+#define SQ_VOP3P_1__SRC0__SHIFT                                                                               0x0
+#define SQ_VOP3P_1__SRC1__SHIFT                                                                               0x9
+#define SQ_VOP3P_1__SRC2__SHIFT                                                                               0x12
+#define SQ_VOP3P_1__OP_SEL_HI__SHIFT                                                                          0x1b
+#define SQ_VOP3P_1__NEG__SHIFT                                                                                0x1d
+#define SQ_VOP3P_1__SRC0_MASK                                                                                 0x000001FFL
+#define SQ_VOP3P_1__SRC1_MASK                                                                                 0x0003FE00L
+#define SQ_VOP3P_1__SRC2_MASK                                                                                 0x07FC0000L
+#define SQ_VOP3P_1__OP_SEL_HI_MASK                                                                            0x18000000L
+#define SQ_VOP3P_1__NEG_MASK                                                                                  0xE0000000L
+//SQ_VOP3P_MFMA_0
+#define SQ_VOP3P_MFMA_0__VDST__SHIFT                                                                          0x0
+#define SQ_VOP3P_MFMA_0__CBSZ__SHIFT                                                                          0x8
+#define SQ_VOP3P_MFMA_0__ABID__SHIFT                                                                          0xb
+#define SQ_VOP3P_MFMA_0__ACC_CD__SHIFT                                                                        0xf
+#define SQ_VOP3P_MFMA_0__OP__SHIFT                                                                            0x10
+#define SQ_VOP3P_MFMA_0__ENCODING__SHIFT                                                                      0x17
+#define SQ_VOP3P_MFMA_0__VDST_MASK                                                                            0x000000FFL
+#define SQ_VOP3P_MFMA_0__CBSZ_MASK                                                                            0x00000700L
+#define SQ_VOP3P_MFMA_0__ABID_MASK                                                                            0x00007800L
+#define SQ_VOP3P_MFMA_0__ACC_CD_MASK                                                                          0x00008000L
+#define SQ_VOP3P_MFMA_0__OP_MASK                                                                              0x007F0000L
+#define SQ_VOP3P_MFMA_0__ENCODING_MASK                                                                        0xFF800000L
+//SQ_VOP3P_MFMA_1
+#define SQ_VOP3P_MFMA_1__SRC0__SHIFT                                                                          0x0
+#define SQ_VOP3P_MFMA_1__SRC1__SHIFT                                                                          0x9
+#define SQ_VOP3P_MFMA_1__SRC2__SHIFT                                                                          0x12
+#define SQ_VOP3P_MFMA_1__ACC__SHIFT                                                                           0x1b
+#define SQ_VOP3P_MFMA_1__BLGP__SHIFT                                                                          0x1d
+#define SQ_VOP3P_MFMA_1__SRC0_MASK                                                                            0x000001FFL
+#define SQ_VOP3P_MFMA_1__SRC1_MASK                                                                            0x0003FE00L
+#define SQ_VOP3P_MFMA_1__SRC2_MASK                                                                            0x07FC0000L
+#define SQ_VOP3P_MFMA_1__ACC_MASK                                                                             0x18000000L
+#define SQ_VOP3P_MFMA_1__BLGP_MASK                                                                            0xE0000000L
+//SQ_VOP3_0
+#define SQ_VOP3_0__VDST__SHIFT                                                                                0x0
+#define SQ_VOP3_0__ABS__SHIFT                                                                                 0x8
+#define SQ_VOP3_0__OP_SEL__SHIFT                                                                              0xb
+#define SQ_VOP3_0__CLAMP__SHIFT                                                                               0xf
+#define SQ_VOP3_0__OP__SHIFT                                                                                  0x10
+#define SQ_VOP3_0__ENCODING__SHIFT                                                                            0x1a
+#define SQ_VOP3_0__VDST_MASK                                                                                  0x000000FFL
+#define SQ_VOP3_0__ABS_MASK                                                                                   0x00000700L
+#define SQ_VOP3_0__OP_SEL_MASK                                                                                0x00007800L
+#define SQ_VOP3_0__CLAMP_MASK                                                                                 0x00008000L
+#define SQ_VOP3_0__OP_MASK                                                                                    0x03FF0000L
+#define SQ_VOP3_0__ENCODING_MASK                                                                              0xFC000000L
+//SQ_VOP3_0_SDST_ENC
+#define SQ_VOP3_0_SDST_ENC__VDST__SHIFT                                                                       0x0
+#define SQ_VOP3_0_SDST_ENC__SDST__SHIFT                                                                       0x8
+#define SQ_VOP3_0_SDST_ENC__CLAMP__SHIFT                                                                      0xf
+#define SQ_VOP3_0_SDST_ENC__OP__SHIFT                                                                         0x10
+#define SQ_VOP3_0_SDST_ENC__ENCODING__SHIFT                                                                   0x1a
+#define SQ_VOP3_0_SDST_ENC__VDST_MASK                                                                         0x000000FFL
+#define SQ_VOP3_0_SDST_ENC__SDST_MASK                                                                         0x00007F00L
+#define SQ_VOP3_0_SDST_ENC__CLAMP_MASK                                                                        0x00008000L
+#define SQ_VOP3_0_SDST_ENC__OP_MASK                                                                           0x03FF0000L
+#define SQ_VOP3_0_SDST_ENC__ENCODING_MASK                                                                     0xFC000000L
+//SQ_VOP3_1
+#define SQ_VOP3_1__SRC0__SHIFT                                                                                0x0
+#define SQ_VOP3_1__SRC1__SHIFT                                                                                0x9
+#define SQ_VOP3_1__SRC2__SHIFT                                                                                0x12
+#define SQ_VOP3_1__OMOD__SHIFT                                                                                0x1b
+#define SQ_VOP3_1__NEG__SHIFT                                                                                 0x1d
+#define SQ_VOP3_1__SRC0_MASK                                                                                  0x000001FFL
+#define SQ_VOP3_1__SRC1_MASK                                                                                  0x0003FE00L
+#define SQ_VOP3_1__SRC2_MASK                                                                                  0x07FC0000L
+#define SQ_VOP3_1__OMOD_MASK                                                                                  0x18000000L
+#define SQ_VOP3_1__NEG_MASK                                                                                   0xE0000000L
+//SQ_VOPC
+#define SQ_VOPC__SRC0__SHIFT                                                                                  0x0
+#define SQ_VOPC__VSRC1__SHIFT                                                                                 0x9
+#define SQ_VOPC__OP__SHIFT                                                                                    0x11
+#define SQ_VOPC__ENCODING__SHIFT                                                                              0x19
+#define SQ_VOPC__SRC0_MASK                                                                                    0x000001FFL
+#define SQ_VOPC__VSRC1_MASK                                                                                   0x0001FE00L
+#define SQ_VOPC__OP_MASK                                                                                      0x01FE0000L
+#define SQ_VOPC__ENCODING_MASK                                                                                0xFE000000L
+//SQ_VOP_DPP
+#define SQ_VOP_DPP__SRC0__SHIFT                                                                               0x0
+#define SQ_VOP_DPP__DPP_CTRL__SHIFT                                                                           0x8
+#define SQ_VOP_DPP__BOUND_CTRL__SHIFT                                                                         0x13
+#define SQ_VOP_DPP__SRC0_NEG__SHIFT                                                                           0x14
+#define SQ_VOP_DPP__SRC0_ABS__SHIFT                                                                           0x15
+#define SQ_VOP_DPP__SRC1_NEG__SHIFT                                                                           0x16
+#define SQ_VOP_DPP__SRC1_ABS__SHIFT                                                                           0x17
+#define SQ_VOP_DPP__BANK_MASK__SHIFT                                                                          0x18
+#define SQ_VOP_DPP__ROW_MASK__SHIFT                                                                           0x1c
+#define SQ_VOP_DPP__SRC0_MASK                                                                                 0x000000FFL
+#define SQ_VOP_DPP__DPP_CTRL_MASK                                                                             0x0001FF00L
+#define SQ_VOP_DPP__BOUND_CTRL_MASK                                                                           0x00080000L
+#define SQ_VOP_DPP__SRC0_NEG_MASK                                                                             0x00100000L
+#define SQ_VOP_DPP__SRC0_ABS_MASK                                                                             0x00200000L
+#define SQ_VOP_DPP__SRC1_NEG_MASK                                                                             0x00400000L
+#define SQ_VOP_DPP__SRC1_ABS_MASK                                                                             0x00800000L
+#define SQ_VOP_DPP__BANK_MASK_MASK                                                                            0x0F000000L
+#define SQ_VOP_DPP__ROW_MASK_MASK                                                                             0xF0000000L
+//SQ_VOP_SDWA
+#define SQ_VOP_SDWA__SRC0__SHIFT                                                                              0x0
+#define SQ_VOP_SDWA__DST_SEL__SHIFT                                                                           0x8
+#define SQ_VOP_SDWA__DST_UNUSED__SHIFT                                                                        0xb
+#define SQ_VOP_SDWA__CLAMP__SHIFT                                                                             0xd
+#define SQ_VOP_SDWA__OMOD__SHIFT                                                                              0xe
+#define SQ_VOP_SDWA__SRC0_SEL__SHIFT                                                                          0x10
+#define SQ_VOP_SDWA__SRC0_SEXT__SHIFT                                                                         0x13
+#define SQ_VOP_SDWA__SRC0_NEG__SHIFT                                                                          0x14
+#define SQ_VOP_SDWA__SRC0_ABS__SHIFT                                                                          0x15
+#define SQ_VOP_SDWA__S0__SHIFT                                                                                0x17
+#define SQ_VOP_SDWA__SRC1_SEL__SHIFT                                                                          0x18
+#define SQ_VOP_SDWA__SRC1_SEXT__SHIFT                                                                         0x1b
+#define SQ_VOP_SDWA__SRC1_NEG__SHIFT                                                                          0x1c
+#define SQ_VOP_SDWA__SRC1_ABS__SHIFT                                                                          0x1d
+#define SQ_VOP_SDWA__S1__SHIFT                                                                                0x1f
+#define SQ_VOP_SDWA__SRC0_MASK                                                                                0x000000FFL
+#define SQ_VOP_SDWA__DST_SEL_MASK                                                                             0x00000700L
+#define SQ_VOP_SDWA__DST_UNUSED_MASK                                                                          0x00001800L
+#define SQ_VOP_SDWA__CLAMP_MASK                                                                               0x00002000L
+#define SQ_VOP_SDWA__OMOD_MASK                                                                                0x0000C000L
+#define SQ_VOP_SDWA__SRC0_SEL_MASK                                                                            0x00070000L
+#define SQ_VOP_SDWA__SRC0_SEXT_MASK                                                                           0x00080000L
+#define SQ_VOP_SDWA__SRC0_NEG_MASK                                                                            0x00100000L
+#define SQ_VOP_SDWA__SRC0_ABS_MASK                                                                            0x00200000L
+#define SQ_VOP_SDWA__S0_MASK                                                                                  0x00800000L
+#define SQ_VOP_SDWA__SRC1_SEL_MASK                                                                            0x07000000L
+#define SQ_VOP_SDWA__SRC1_SEXT_MASK                                                                           0x08000000L
+#define SQ_VOP_SDWA__SRC1_NEG_MASK                                                                            0x10000000L
+#define SQ_VOP_SDWA__SRC1_ABS_MASK                                                                            0x20000000L
+#define SQ_VOP_SDWA__S1_MASK                                                                                  0x80000000L
+//SQ_VOP_SDWA_SDST_ENC
+#define SQ_VOP_SDWA_SDST_ENC__SRC0__SHIFT                                                                     0x0
+#define SQ_VOP_SDWA_SDST_ENC__SDST__SHIFT                                                                     0x8
+#define SQ_VOP_SDWA_SDST_ENC__SD__SHIFT                                                                       0xf
+#define SQ_VOP_SDWA_SDST_ENC__SRC0_SEL__SHIFT                                                                 0x10
+#define SQ_VOP_SDWA_SDST_ENC__SRC0_SEXT__SHIFT                                                                0x13
+#define SQ_VOP_SDWA_SDST_ENC__SRC0_NEG__SHIFT                                                                 0x14
+#define SQ_VOP_SDWA_SDST_ENC__SRC0_ABS__SHIFT                                                                 0x15
+#define SQ_VOP_SDWA_SDST_ENC__S0__SHIFT                                                                       0x17
+#define SQ_VOP_SDWA_SDST_ENC__SRC1_SEL__SHIFT                                                                 0x18
+#define SQ_VOP_SDWA_SDST_ENC__SRC1_SEXT__SHIFT                                                                0x1b
+#define SQ_VOP_SDWA_SDST_ENC__SRC1_NEG__SHIFT                                                                 0x1c
+#define SQ_VOP_SDWA_SDST_ENC__SRC1_ABS__SHIFT                                                                 0x1d
+#define SQ_VOP_SDWA_SDST_ENC__S1__SHIFT                                                                       0x1f
+#define SQ_VOP_SDWA_SDST_ENC__SRC0_MASK                                                                       0x000000FFL
+#define SQ_VOP_SDWA_SDST_ENC__SDST_MASK                                                                       0x00007F00L
+#define SQ_VOP_SDWA_SDST_ENC__SD_MASK                                                                         0x00008000L
+#define SQ_VOP_SDWA_SDST_ENC__SRC0_SEL_MASK                                                                   0x00070000L
+#define SQ_VOP_SDWA_SDST_ENC__SRC0_SEXT_MASK                                                                  0x00080000L
+#define SQ_VOP_SDWA_SDST_ENC__SRC0_NEG_MASK                                                                   0x00100000L
+#define SQ_VOP_SDWA_SDST_ENC__SRC0_ABS_MASK                                                                   0x00200000L
+#define SQ_VOP_SDWA_SDST_ENC__S0_MASK                                                                         0x00800000L
+#define SQ_VOP_SDWA_SDST_ENC__SRC1_SEL_MASK                                                                   0x07000000L
+#define SQ_VOP_SDWA_SDST_ENC__SRC1_SEXT_MASK                                                                  0x08000000L
+#define SQ_VOP_SDWA_SDST_ENC__SRC1_NEG_MASK                                                                   0x10000000L
+#define SQ_VOP_SDWA_SDST_ENC__SRC1_ABS_MASK                                                                   0x20000000L
+#define SQ_VOP_SDWA_SDST_ENC__S1_MASK                                                                         0x80000000L
+//SQ_LB_CTR_CTRL
+#define SQ_LB_CTR_CTRL__START__SHIFT                                                                          0x0
+#define SQ_LB_CTR_CTRL__LOAD__SHIFT                                                                           0x1
+#define SQ_LB_CTR_CTRL__CLEAR__SHIFT                                                                          0x2
+#define SQ_LB_CTR_CTRL__START_MASK                                                                            0x00000001L
+#define SQ_LB_CTR_CTRL__LOAD_MASK                                                                             0x00000002L
+#define SQ_LB_CTR_CTRL__CLEAR_MASK                                                                            0x00000004L
+//SQ_LB_DATA0
+#define SQ_LB_DATA0__DATA__SHIFT                                                                              0x0
+#define SQ_LB_DATA0__DATA_MASK                                                                                0xFFFFFFFFL
+//SQ_LB_DATA1
+#define SQ_LB_DATA1__DATA__SHIFT                                                                              0x0
+#define SQ_LB_DATA1__DATA_MASK                                                                                0xFFFFFFFFL
+//SQ_LB_DATA2
+#define SQ_LB_DATA2__DATA__SHIFT                                                                              0x0
+#define SQ_LB_DATA2__DATA_MASK                                                                                0xFFFFFFFFL
+//SQ_LB_DATA3
+#define SQ_LB_DATA3__DATA__SHIFT                                                                              0x0
+#define SQ_LB_DATA3__DATA_MASK                                                                                0xFFFFFFFFL
+//SQ_LB_CTR_SEL
+#define SQ_LB_CTR_SEL__SEL0__SHIFT                                                                            0x0
+#define SQ_LB_CTR_SEL__SEL1__SHIFT                                                                            0x4
+#define SQ_LB_CTR_SEL__SEL2__SHIFT                                                                            0x8
+#define SQ_LB_CTR_SEL__SEL3__SHIFT                                                                            0xc
+#define SQ_LB_CTR_SEL__SEL0_MASK                                                                              0x0000000FL
+#define SQ_LB_CTR_SEL__SEL1_MASK                                                                              0x000000F0L
+#define SQ_LB_CTR_SEL__SEL2_MASK                                                                              0x00000F00L
+#define SQ_LB_CTR_SEL__SEL3_MASK                                                                              0x0000F000L
+//SQ_LB_CTR0_CU
+#define SQ_LB_CTR0_CU__SH0_MASK__SHIFT                                                                        0x0
+#define SQ_LB_CTR0_CU__SH1_MASK__SHIFT                                                                        0x10
+#define SQ_LB_CTR0_CU__SH0_MASK_MASK                                                                          0x0000FFFFL
+#define SQ_LB_CTR0_CU__SH1_MASK_MASK                                                                          0xFFFF0000L
+//SQ_LB_CTR1_CU
+#define SQ_LB_CTR1_CU__SH0_MASK__SHIFT                                                                        0x0
+#define SQ_LB_CTR1_CU__SH1_MASK__SHIFT                                                                        0x10
+#define SQ_LB_CTR1_CU__SH0_MASK_MASK                                                                          0x0000FFFFL
+#define SQ_LB_CTR1_CU__SH1_MASK_MASK                                                                          0xFFFF0000L
+//SQ_LB_CTR2_CU
+#define SQ_LB_CTR2_CU__SH0_MASK__SHIFT                                                                        0x0
+#define SQ_LB_CTR2_CU__SH1_MASK__SHIFT                                                                        0x10
+#define SQ_LB_CTR2_CU__SH0_MASK_MASK                                                                          0x0000FFFFL
+#define SQ_LB_CTR2_CU__SH1_MASK_MASK                                                                          0xFFFF0000L
+//SQ_LB_CTR3_CU
+#define SQ_LB_CTR3_CU__SH0_MASK__SHIFT                                                                        0x0
+#define SQ_LB_CTR3_CU__SH1_MASK__SHIFT                                                                        0x10
+#define SQ_LB_CTR3_CU__SH0_MASK_MASK                                                                          0x0000FFFFL
+#define SQ_LB_CTR3_CU__SH1_MASK_MASK                                                                          0xFFFF0000L
+//SQC_EDC_CNT
+#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_SEC_COUNT__SHIFT                                                 0x0
+#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_DED_COUNT__SHIFT                                                 0x2
+#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_SEC_COUNT__SHIFT                                                    0x4
+#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_DED_COUNT__SHIFT                                                    0x6
+#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_SEC_COUNT__SHIFT                                                 0x8
+#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_DED_COUNT__SHIFT                                                 0xa
+#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_SEC_COUNT__SHIFT                                                    0xc
+#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_DED_COUNT__SHIFT                                                    0xe
+#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_SEC_COUNT__SHIFT                                                 0x10
+#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_DED_COUNT__SHIFT                                                 0x12
+#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_SEC_COUNT__SHIFT                                                    0x14
+#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_DED_COUNT__SHIFT                                                    0x16
+#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_SEC_COUNT__SHIFT                                                 0x18
+#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_DED_COUNT__SHIFT                                                 0x1a
+#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_SEC_COUNT__SHIFT                                                    0x1c
+#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_DED_COUNT__SHIFT                                                    0x1e
+#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_SEC_COUNT_MASK                                                   0x00000003L
+#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_DED_COUNT_MASK                                                   0x0000000CL
+#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_SEC_COUNT_MASK                                                      0x00000030L
+#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_DED_COUNT_MASK                                                      0x000000C0L
+#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_SEC_COUNT_MASK                                                   0x00000300L
+#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_DED_COUNT_MASK                                                   0x00000C00L
+#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_SEC_COUNT_MASK                                                      0x00003000L
+#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_DED_COUNT_MASK                                                      0x0000C000L
+#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_SEC_COUNT_MASK                                                   0x00030000L
+#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_DED_COUNT_MASK                                                   0x000C0000L
+#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_SEC_COUNT_MASK                                                      0x00300000L
+#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_DED_COUNT_MASK                                                      0x00C00000L
+#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_SEC_COUNT_MASK                                                   0x03000000L
+#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_DED_COUNT_MASK                                                   0x0C000000L
+#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_SEC_COUNT_MASK                                                      0x30000000L
+#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_DED_COUNT_MASK                                                      0xC0000000L
+//SQ_EDC_SEC_CNT
+#define SQ_EDC_SEC_CNT__LDS_SEC__SHIFT                                                                        0x0
+#define SQ_EDC_SEC_CNT__SGPR_SEC__SHIFT                                                                       0x8
+#define SQ_EDC_SEC_CNT__VGPR_SEC__SHIFT                                                                       0x10
+#define SQ_EDC_SEC_CNT__LDS_SEC_MASK                                                                          0x000000FFL
+#define SQ_EDC_SEC_CNT__SGPR_SEC_MASK                                                                         0x0000FF00L
+#define SQ_EDC_SEC_CNT__VGPR_SEC_MASK                                                                         0x00FF0000L
+//SQ_EDC_DED_CNT
+#define SQ_EDC_DED_CNT__LDS_DED__SHIFT                                                                        0x0
+#define SQ_EDC_DED_CNT__SGPR_DED__SHIFT                                                                       0x8
+#define SQ_EDC_DED_CNT__VGPR_DED__SHIFT                                                                       0x10
+#define SQ_EDC_DED_CNT__LDS_DED_MASK                                                                          0x000000FFL
+#define SQ_EDC_DED_CNT__SGPR_DED_MASK                                                                         0x0000FF00L
+#define SQ_EDC_DED_CNT__VGPR_DED_MASK                                                                         0x00FF0000L
+//SQ_EDC_INFO
+#define SQ_EDC_INFO__WAVE_ID__SHIFT                                                                           0x0
+#define SQ_EDC_INFO__SIMD_ID__SHIFT                                                                           0x4
+#define SQ_EDC_INFO__SOURCE__SHIFT                                                                            0x6
+#define SQ_EDC_INFO__VM_ID__SHIFT                                                                             0x9
+#define SQ_EDC_INFO__WAVE_ID_MASK                                                                             0x0000000FL
+#define SQ_EDC_INFO__SIMD_ID_MASK                                                                             0x00000030L
+#define SQ_EDC_INFO__SOURCE_MASK                                                                              0x000001C0L
+#define SQ_EDC_INFO__VM_ID_MASK                                                                               0x00001E00L
+//SQ_EDC_CNT
+#define SQ_EDC_CNT__LDS_D_SEC_COUNT__SHIFT                                                                    0x0
+#define SQ_EDC_CNT__LDS_D_DED_COUNT__SHIFT                                                                    0x2
+#define SQ_EDC_CNT__LDS_I_SEC_COUNT__SHIFT                                                                    0x4
+#define SQ_EDC_CNT__LDS_I_DED_COUNT__SHIFT                                                                    0x6
+#define SQ_EDC_CNT__SGPR_SEC_COUNT__SHIFT                                                                     0x8
+#define SQ_EDC_CNT__SGPR_DED_COUNT__SHIFT                                                                     0xa
+#define SQ_EDC_CNT__VGPR0_SEC_COUNT__SHIFT                                                                    0xc
+#define SQ_EDC_CNT__VGPR0_DED_COUNT__SHIFT                                                                    0xe
+#define SQ_EDC_CNT__VGPR1_SEC_COUNT__SHIFT                                                                    0x10
+#define SQ_EDC_CNT__VGPR1_DED_COUNT__SHIFT                                                                    0x12
+#define SQ_EDC_CNT__VGPR2_SEC_COUNT__SHIFT                                                                    0x14
+#define SQ_EDC_CNT__VGPR2_DED_COUNT__SHIFT                                                                    0x16
+#define SQ_EDC_CNT__VGPR3_SEC_COUNT__SHIFT                                                                    0x18
+#define SQ_EDC_CNT__VGPR3_DED_COUNT__SHIFT                                                                    0x1a
+#define SQ_EDC_CNT__LDS_D_SEC_COUNT_MASK                                                                      0x00000003L
+#define SQ_EDC_CNT__LDS_D_DED_COUNT_MASK                                                                      0x0000000CL
+#define SQ_EDC_CNT__LDS_I_SEC_COUNT_MASK                                                                      0x00000030L
+#define SQ_EDC_CNT__LDS_I_DED_COUNT_MASK                                                                      0x000000C0L
+#define SQ_EDC_CNT__SGPR_SEC_COUNT_MASK                                                                       0x00000300L
+#define SQ_EDC_CNT__SGPR_DED_COUNT_MASK                                                                       0x00000C00L
+#define SQ_EDC_CNT__VGPR0_SEC_COUNT_MASK                                                                      0x00003000L
+#define SQ_EDC_CNT__VGPR0_DED_COUNT_MASK                                                                      0x0000C000L
+#define SQ_EDC_CNT__VGPR1_SEC_COUNT_MASK                                                                      0x00030000L
+#define SQ_EDC_CNT__VGPR1_DED_COUNT_MASK                                                                      0x000C0000L
+#define SQ_EDC_CNT__VGPR2_SEC_COUNT_MASK                                                                      0x00300000L
+#define SQ_EDC_CNT__VGPR2_DED_COUNT_MASK                                                                      0x00C00000L
+#define SQ_EDC_CNT__VGPR3_SEC_COUNT_MASK                                                                      0x03000000L
+#define SQ_EDC_CNT__VGPR3_DED_COUNT_MASK                                                                      0x0C000000L
+//SQ_EDC_FUE_CNTL
+#define SQ_EDC_FUE_CNTL__BLOCK_FUE_FLAGS__SHIFT                                                               0x0
+#define SQ_EDC_FUE_CNTL__FUE_INTERRUPT_ENABLES__SHIFT                                                         0x10
+#define SQ_EDC_FUE_CNTL__BLOCK_FUE_FLAGS_MASK                                                                 0x0000FFFFL
+#define SQ_EDC_FUE_CNTL__FUE_INTERRUPT_ENABLES_MASK                                                           0xFFFF0000L
+//SQ_THREAD_TRACE_WORD_CMN
+#define SQ_THREAD_TRACE_WORD_CMN__TOKEN_TYPE__SHIFT                                                           0x0
+#define SQ_THREAD_TRACE_WORD_CMN__TIME_DELTA__SHIFT                                                           0x4
+#define SQ_THREAD_TRACE_WORD_CMN__TOKEN_TYPE_MASK                                                             0x000FL
+#define SQ_THREAD_TRACE_WORD_CMN__TIME_DELTA_MASK                                                             0x0010L
+//SQ_THREAD_TRACE_WORD_EVENT
+#define SQ_THREAD_TRACE_WORD_EVENT__TOKEN_TYPE__SHIFT                                                         0x0
+#define SQ_THREAD_TRACE_WORD_EVENT__TIME_DELTA__SHIFT                                                         0x4
+#define SQ_THREAD_TRACE_WORD_EVENT__SH_ID__SHIFT                                                              0x5
+#define SQ_THREAD_TRACE_WORD_EVENT__STAGE__SHIFT                                                              0x6
+#define SQ_THREAD_TRACE_WORD_EVENT__EVENT_TYPE__SHIFT                                                         0xa
+#define SQ_THREAD_TRACE_WORD_EVENT__TOKEN_TYPE_MASK                                                           0x000FL
+#define SQ_THREAD_TRACE_WORD_EVENT__TIME_DELTA_MASK                                                           0x0010L
+#define SQ_THREAD_TRACE_WORD_EVENT__SH_ID_MASK                                                                0x0020L
+#define SQ_THREAD_TRACE_WORD_EVENT__STAGE_MASK                                                                0x01C0L
+#define SQ_THREAD_TRACE_WORD_EVENT__EVENT_TYPE_MASK                                                           0xFC00L
+//SQ_THREAD_TRACE_WORD_INST
+#define SQ_THREAD_TRACE_WORD_INST__TOKEN_TYPE__SHIFT                                                          0x0
+#define SQ_THREAD_TRACE_WORD_INST__TIME_DELTA__SHIFT                                                          0x4
+#define SQ_THREAD_TRACE_WORD_INST__WAVE_ID__SHIFT                                                             0x5
+#define SQ_THREAD_TRACE_WORD_INST__SIMD_ID__SHIFT                                                             0x9
+#define SQ_THREAD_TRACE_WORD_INST__INST_TYPE__SHIFT                                                           0xb
+#define SQ_THREAD_TRACE_WORD_INST__TOKEN_TYPE_MASK                                                            0x000FL
+#define SQ_THREAD_TRACE_WORD_INST__TIME_DELTA_MASK                                                            0x0010L
+#define SQ_THREAD_TRACE_WORD_INST__WAVE_ID_MASK                                                               0x01E0L
+#define SQ_THREAD_TRACE_WORD_INST__SIMD_ID_MASK                                                               0x0600L
+#define SQ_THREAD_TRACE_WORD_INST__INST_TYPE_MASK                                                             0xF800L
+//SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TOKEN_TYPE__SHIFT                                                0x0
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TIME_DELTA__SHIFT                                                0x4
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__WAVE_ID__SHIFT                                                   0x5
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__SIMD_ID__SHIFT                                                   0x9
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TRAP_ERROR__SHIFT                                                0xf
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__PC_LO__SHIFT                                                     0x10
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TOKEN_TYPE_MASK                                                  0x0000000FL
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TIME_DELTA_MASK                                                  0x00000010L
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__WAVE_ID_MASK                                                     0x000001E0L
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__SIMD_ID_MASK                                                     0x00000600L
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TRAP_ERROR_MASK                                                  0x00008000L
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__PC_LO_MASK                                                       0xFFFF0000L
+//SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TOKEN_TYPE__SHIFT                                          0x0
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TIME_DELTA__SHIFT                                          0x4
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__PRIV__SHIFT                                                0x5
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__CU_ID__SHIFT                                               0x6
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__WAVE_ID__SHIFT                                             0xa
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SIMD_ID__SHIFT                                             0xe
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__DATA_LO__SHIFT                                             0x10
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TOKEN_TYPE_MASK                                            0x0000000FL
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TIME_DELTA_MASK                                            0x00000010L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__PRIV_MASK                                                  0x00000020L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__CU_ID_MASK                                                 0x000003C0L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__WAVE_ID_MASK                                               0x00003C00L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SIMD_ID_MASK                                               0x0000C000L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__DATA_LO_MASK                                               0xFFFF0000L
+//SQ_THREAD_TRACE_WORD_ISSUE
+#define SQ_THREAD_TRACE_WORD_ISSUE__TOKEN_TYPE__SHIFT                                                         0x0
+#define SQ_THREAD_TRACE_WORD_ISSUE__TIME_DELTA__SHIFT                                                         0x4
+#define SQ_THREAD_TRACE_WORD_ISSUE__SIMD_ID__SHIFT                                                            0x5
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST0__SHIFT                                                              0x8
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST1__SHIFT                                                              0xa
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST2__SHIFT                                                              0xc
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST3__SHIFT                                                              0xe
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST4__SHIFT                                                              0x10
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST5__SHIFT                                                              0x12
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST6__SHIFT                                                              0x14
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST7__SHIFT                                                              0x16
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST8__SHIFT                                                              0x18
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST9__SHIFT                                                              0x1a
+#define SQ_THREAD_TRACE_WORD_ISSUE__TOKEN_TYPE_MASK                                                           0x0000000FL
+#define SQ_THREAD_TRACE_WORD_ISSUE__TIME_DELTA_MASK                                                           0x00000010L
+#define SQ_THREAD_TRACE_WORD_ISSUE__SIMD_ID_MASK                                                              0x00000060L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST0_MASK                                                                0x00000300L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST1_MASK                                                                0x00000C00L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST2_MASK                                                                0x00003000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST3_MASK                                                                0x0000C000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST4_MASK                                                                0x00030000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST5_MASK                                                                0x000C0000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST6_MASK                                                                0x00300000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST7_MASK                                                                0x00C00000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST8_MASK                                                                0x03000000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST9_MASK                                                                0x0C000000L
+//SQ_THREAD_TRACE_WORD_MISC
+#define SQ_THREAD_TRACE_WORD_MISC__TOKEN_TYPE__SHIFT                                                          0x0
+#define SQ_THREAD_TRACE_WORD_MISC__TIME_DELTA__SHIFT                                                          0x4
+#define SQ_THREAD_TRACE_WORD_MISC__SH_ID__SHIFT                                                               0xc
+#define SQ_THREAD_TRACE_WORD_MISC__MISC_TOKEN_TYPE__SHIFT                                                     0xd
+#define SQ_THREAD_TRACE_WORD_MISC__TOKEN_TYPE_MASK                                                            0x000FL
+#define SQ_THREAD_TRACE_WORD_MISC__TIME_DELTA_MASK                                                            0x0FF0L
+#define SQ_THREAD_TRACE_WORD_MISC__SH_ID_MASK                                                                 0x1000L
+#define SQ_THREAD_TRACE_WORD_MISC__MISC_TOKEN_TYPE_MASK                                                       0xE000L
+//SQ_THREAD_TRACE_WORD_PERF_1_OF_2
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TOKEN_TYPE__SHIFT                                                   0x0
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TIME_DELTA__SHIFT                                                   0x4
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__SH_ID__SHIFT                                                        0x5
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CU_ID__SHIFT                                                        0x6
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR_BANK__SHIFT                                                    0xa
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR0__SHIFT                                                        0xc
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR1_LO__SHIFT                                                     0x19
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TOKEN_TYPE_MASK                                                     0x0000000FL
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TIME_DELTA_MASK                                                     0x00000010L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__SH_ID_MASK                                                          0x00000020L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CU_ID_MASK                                                          0x000003C0L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR_BANK_MASK                                                      0x00000C00L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR0_MASK                                                          0x01FFF000L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR1_LO_MASK                                                       0xFE000000L
+//SQ_THREAD_TRACE_WORD_REG_1_OF_2
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TOKEN_TYPE__SHIFT                                                    0x0
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TIME_DELTA__SHIFT                                                    0x4
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__PIPE_ID__SHIFT                                                       0x5
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__ME_ID__SHIFT                                                         0x7
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_DROPPED_PREV__SHIFT                                              0x9
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_TYPE__SHIFT                                                      0xa
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_PRIV__SHIFT                                                      0xe
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_OP__SHIFT                                                        0xf
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_ADDR__SHIFT                                                      0x10
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TOKEN_TYPE_MASK                                                      0x0000000FL
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TIME_DELTA_MASK                                                      0x00000010L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__PIPE_ID_MASK                                                         0x00000060L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__ME_ID_MASK                                                           0x00000180L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_DROPPED_PREV_MASK                                                0x00000200L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_TYPE_MASK                                                        0x00001C00L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_PRIV_MASK                                                        0x00004000L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_OP_MASK                                                          0x00008000L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_ADDR_MASK                                                        0xFFFF0000L
+//SQ_THREAD_TRACE_WORD_REG_2_OF_2
+#define SQ_THREAD_TRACE_WORD_REG_2_OF_2__DATA__SHIFT                                                          0x0
+#define SQ_THREAD_TRACE_WORD_REG_2_OF_2__DATA_MASK                                                            0xFFFFFFFFL
+//SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TOKEN_TYPE__SHIFT                                                 0x0
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TIME_DELTA__SHIFT                                                 0x4
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__PIPE_ID__SHIFT                                                    0x5
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__ME_ID__SHIFT                                                      0x7
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__REG_ADDR__SHIFT                                                   0x9
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__DATA_LO__SHIFT                                                    0x10
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TOKEN_TYPE_MASK                                                   0x0000000FL
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TIME_DELTA_MASK                                                   0x00000010L
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__PIPE_ID_MASK                                                      0x00000060L
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__ME_ID_MASK                                                        0x00000180L
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__REG_ADDR_MASK                                                     0x0000FE00L
+#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__DATA_LO_MASK                                                      0xFFFF0000L
+//SQ_THREAD_TRACE_WORD_REG_CS_2_OF_2
+#define SQ_THREAD_TRACE_WORD_REG_CS_2_OF_2__DATA_HI__SHIFT                                                    0x0
+#define SQ_THREAD_TRACE_WORD_REG_CS_2_OF_2__DATA_HI_MASK                                                      0x0000FFFFL
+//SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TOKEN_TYPE__SHIFT                                              0x0
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TIME_LO__SHIFT                                                 0x10
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TOKEN_TYPE_MASK                                                0x0000000FL
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TIME_LO_MASK                                                   0xFFFF0000L
+//SQ_THREAD_TRACE_WORD_WAVE
+#define SQ_THREAD_TRACE_WORD_WAVE__TOKEN_TYPE__SHIFT                                                          0x0
+#define SQ_THREAD_TRACE_WORD_WAVE__TIME_DELTA__SHIFT                                                          0x4
+#define SQ_THREAD_TRACE_WORD_WAVE__SH_ID__SHIFT                                                               0x5
+#define SQ_THREAD_TRACE_WORD_WAVE__CU_ID__SHIFT                                                               0x6
+#define SQ_THREAD_TRACE_WORD_WAVE__WAVE_ID__SHIFT                                                             0xa
+#define SQ_THREAD_TRACE_WORD_WAVE__SIMD_ID__SHIFT                                                             0xe
+#define SQ_THREAD_TRACE_WORD_WAVE__TOKEN_TYPE_MASK                                                            0x000FL
+#define SQ_THREAD_TRACE_WORD_WAVE__TIME_DELTA_MASK                                                            0x0010L
+#define SQ_THREAD_TRACE_WORD_WAVE__SH_ID_MASK                                                                 0x0020L
+#define SQ_THREAD_TRACE_WORD_WAVE__CU_ID_MASK                                                                 0x03C0L
+#define SQ_THREAD_TRACE_WORD_WAVE__WAVE_ID_MASK                                                               0x3C00L
+#define SQ_THREAD_TRACE_WORD_WAVE__SIMD_ID_MASK                                                               0xC000L
+//SQ_THREAD_TRACE_WORD_WAVE_START
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TOKEN_TYPE__SHIFT                                                    0x0
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TIME_DELTA__SHIFT                                                    0x4
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SH_ID__SHIFT                                                         0x5
+#define SQ_THREAD_TRACE_WORD_WAVE_START__CU_ID__SHIFT                                                         0x6
+#define SQ_THREAD_TRACE_WORD_WAVE_START__WAVE_ID__SHIFT                                                       0xa
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SIMD_ID__SHIFT                                                       0xe
+#define SQ_THREAD_TRACE_WORD_WAVE_START__DISPATCHER__SHIFT                                                    0x10
+#define SQ_THREAD_TRACE_WORD_WAVE_START__VS_NO_ALLOC_OR_GROUPED__SHIFT                                        0x15
+#define SQ_THREAD_TRACE_WORD_WAVE_START__COUNT__SHIFT                                                         0x16
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TG_ID__SHIFT                                                         0x1d
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TOKEN_TYPE_MASK                                                      0x0000000FL
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TIME_DELTA_MASK                                                      0x00000010L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SH_ID_MASK                                                           0x00000020L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__CU_ID_MASK                                                           0x000003C0L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__WAVE_ID_MASK                                                         0x00003C00L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SIMD_ID_MASK                                                         0x0000C000L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__DISPATCHER_MASK                                                      0x001F0000L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__VS_NO_ALLOC_OR_GROUPED_MASK                                          0x00200000L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__COUNT_MASK                                                           0x1FC00000L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TG_ID_MASK                                                           0xE0000000L
+//SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2
+#define SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2__PC_HI__SHIFT                                                     0x0
+#define SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2__PC_HI_MASK                                                       0x00FFFFFFL
+//SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2__DATA_HI__SHIFT                                             0x0
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2__DATA_HI_MASK                                               0xFFFFL
+//SQ_THREAD_TRACE_WORD_PERF_2_OF_2
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR1_HI__SHIFT                                                     0x0
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR2__SHIFT                                                        0x6
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR3__SHIFT                                                        0x13
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR1_HI_MASK                                                       0x0000003FL
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR2_MASK                                                          0x0007FFC0L
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR3_MASK                                                          0xFFF80000L
+//SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2__TIME_HI__SHIFT                                                 0x0
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2__TIME_HI_MASK                                                   0xFFFFFFFFL
+//SQ_WREXEC_EXEC_HI
+#define SQ_WREXEC_EXEC_HI__ADDR_HI__SHIFT                                                                     0x0
+#define SQ_WREXEC_EXEC_HI__FIRST_WAVE__SHIFT                                                                  0x1a
+#define SQ_WREXEC_EXEC_HI__ATC__SHIFT                                                                         0x1b
+#define SQ_WREXEC_EXEC_HI__MTYPE__SHIFT                                                                       0x1c
+#define SQ_WREXEC_EXEC_HI__MSB__SHIFT                                                                         0x1f
+#define SQ_WREXEC_EXEC_HI__ADDR_HI_MASK                                                                       0x0000FFFFL
+#define SQ_WREXEC_EXEC_HI__FIRST_WAVE_MASK                                                                    0x04000000L
+#define SQ_WREXEC_EXEC_HI__ATC_MASK                                                                           0x08000000L
+#define SQ_WREXEC_EXEC_HI__MTYPE_MASK                                                                         0x70000000L
+#define SQ_WREXEC_EXEC_HI__MSB_MASK                                                                           0x80000000L
+//SQ_WREXEC_EXEC_LO
+#define SQ_WREXEC_EXEC_LO__ADDR_LO__SHIFT                                                                     0x0
+#define SQ_WREXEC_EXEC_LO__ADDR_LO_MASK                                                                       0xFFFFFFFFL
+//SQ_BUF_RSRC_WORD0
+#define SQ_BUF_RSRC_WORD0__BASE_ADDRESS__SHIFT                                                                0x0
+#define SQ_BUF_RSRC_WORD0__BASE_ADDRESS_MASK                                                                  0xFFFFFFFFL
+//SQ_BUF_RSRC_WORD1
+#define SQ_BUF_RSRC_WORD1__BASE_ADDRESS_HI__SHIFT                                                             0x0
+#define SQ_BUF_RSRC_WORD1__STRIDE__SHIFT                                                                      0x10
+#define SQ_BUF_RSRC_WORD1__CACHE_SWIZZLE__SHIFT                                                               0x1e
+#define SQ_BUF_RSRC_WORD1__SWIZZLE_ENABLE__SHIFT                                                              0x1f
+#define SQ_BUF_RSRC_WORD1__BASE_ADDRESS_HI_MASK                                                               0x0000FFFFL
+#define SQ_BUF_RSRC_WORD1__STRIDE_MASK                                                                        0x3FFF0000L
+#define SQ_BUF_RSRC_WORD1__CACHE_SWIZZLE_MASK                                                                 0x40000000L
+#define SQ_BUF_RSRC_WORD1__SWIZZLE_ENABLE_MASK                                                                0x80000000L
+//SQ_BUF_RSRC_WORD2
+#define SQ_BUF_RSRC_WORD2__NUM_RECORDS__SHIFT                                                                 0x0
+#define SQ_BUF_RSRC_WORD2__NUM_RECORDS_MASK                                                                   0xFFFFFFFFL
+//SQ_BUF_RSRC_WORD3
+#define SQ_BUF_RSRC_WORD3__DST_SEL_X__SHIFT                                                                   0x0
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Y__SHIFT                                                                   0x3
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Z__SHIFT                                                                   0x6
+#define SQ_BUF_RSRC_WORD3__DST_SEL_W__SHIFT                                                                   0x9
+#define SQ_BUF_RSRC_WORD3__NUM_FORMAT__SHIFT                                                                  0xc
+#define SQ_BUF_RSRC_WORD3__DATA_FORMAT__SHIFT                                                                 0xf
+#define SQ_BUF_RSRC_WORD3__USER_VM_ENABLE__SHIFT                                                              0x13
+#define SQ_BUF_RSRC_WORD3__USER_VM_MODE__SHIFT                                                                0x14
+#define SQ_BUF_RSRC_WORD3__INDEX_STRIDE__SHIFT                                                                0x15
+#define SQ_BUF_RSRC_WORD3__ADD_TID_ENABLE__SHIFT                                                              0x17
+#define SQ_BUF_RSRC_WORD3__NV__SHIFT                                                                          0x1b
+#define SQ_BUF_RSRC_WORD3__TYPE__SHIFT                                                                        0x1e
+#define SQ_BUF_RSRC_WORD3__DST_SEL_X_MASK                                                                     0x00000007L
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Y_MASK                                                                     0x00000038L
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Z_MASK                                                                     0x000001C0L
+#define SQ_BUF_RSRC_WORD3__DST_SEL_W_MASK                                                                     0x00000E00L
+#define SQ_BUF_RSRC_WORD3__NUM_FORMAT_MASK                                                                    0x00007000L
+#define SQ_BUF_RSRC_WORD3__DATA_FORMAT_MASK                                                                   0x00078000L
+#define SQ_BUF_RSRC_WORD3__USER_VM_ENABLE_MASK                                                                0x00080000L
+#define SQ_BUF_RSRC_WORD3__USER_VM_MODE_MASK                                                                  0x00100000L
+#define SQ_BUF_RSRC_WORD3__INDEX_STRIDE_MASK                                                                  0x00600000L
+#define SQ_BUF_RSRC_WORD3__ADD_TID_ENABLE_MASK                                                                0x00800000L
+#define SQ_BUF_RSRC_WORD3__NV_MASK                                                                            0x08000000L
+#define SQ_BUF_RSRC_WORD3__TYPE_MASK                                                                          0xC0000000L
+//SQ_IMG_RSRC_WORD0
+#define SQ_IMG_RSRC_WORD0__BASE_ADDRESS__SHIFT                                                                0x0
+#define SQ_IMG_RSRC_WORD0__BASE_ADDRESS_MASK                                                                  0xFFFFFFFFL
+//SQ_IMG_RSRC_WORD1
+#define SQ_IMG_RSRC_WORD1__BASE_ADDRESS_HI__SHIFT                                                             0x0
+#define SQ_IMG_RSRC_WORD1__MIN_LOD__SHIFT                                                                     0x8
+#define SQ_IMG_RSRC_WORD1__DATA_FORMAT__SHIFT                                                                 0x14
+#define SQ_IMG_RSRC_WORD1__NUM_FORMAT__SHIFT                                                                  0x1a
+#define SQ_IMG_RSRC_WORD1__NV__SHIFT                                                                          0x1e
+#define SQ_IMG_RSRC_WORD1__META_DIRECT__SHIFT                                                                 0x1f
+#define SQ_IMG_RSRC_WORD1__BASE_ADDRESS_HI_MASK                                                               0x000000FFL
+#define SQ_IMG_RSRC_WORD1__MIN_LOD_MASK                                                                       0x000FFF00L
+#define SQ_IMG_RSRC_WORD1__DATA_FORMAT_MASK                                                                   0x03F00000L
+#define SQ_IMG_RSRC_WORD1__NUM_FORMAT_MASK                                                                    0x3C000000L
+#define SQ_IMG_RSRC_WORD1__NV_MASK                                                                            0x40000000L
+#define SQ_IMG_RSRC_WORD1__META_DIRECT_MASK                                                                   0x80000000L
+//SQ_IMG_RSRC_WORD2
+#define SQ_IMG_RSRC_WORD2__WIDTH__SHIFT                                                                       0x0
+#define SQ_IMG_RSRC_WORD2__HEIGHT__SHIFT                                                                      0xe
+#define SQ_IMG_RSRC_WORD2__PERF_MOD__SHIFT                                                                    0x1c
+#define SQ_IMG_RSRC_WORD2__WIDTH_MASK                                                                         0x00003FFFL
+#define SQ_IMG_RSRC_WORD2__HEIGHT_MASK                                                                        0x0FFFC000L
+#define SQ_IMG_RSRC_WORD2__PERF_MOD_MASK                                                                      0x70000000L
+//SQ_IMG_RSRC_WORD3
+#define SQ_IMG_RSRC_WORD3__DST_SEL_X__SHIFT                                                                   0x0
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Y__SHIFT                                                                   0x3
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Z__SHIFT                                                                   0x6
+#define SQ_IMG_RSRC_WORD3__DST_SEL_W__SHIFT                                                                   0x9
+#define SQ_IMG_RSRC_WORD3__BASE_LEVEL__SHIFT                                                                  0xc
+#define SQ_IMG_RSRC_WORD3__LAST_LEVEL__SHIFT                                                                  0x10
+#define SQ_IMG_RSRC_WORD3__SW_MODE__SHIFT                                                                     0x14
+#define SQ_IMG_RSRC_WORD3__TYPE__SHIFT                                                                        0x1c
+#define SQ_IMG_RSRC_WORD3__DST_SEL_X_MASK                                                                     0x00000007L
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Y_MASK                                                                     0x00000038L
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Z_MASK                                                                     0x000001C0L
+#define SQ_IMG_RSRC_WORD3__DST_SEL_W_MASK                                                                     0x00000E00L
+#define SQ_IMG_RSRC_WORD3__BASE_LEVEL_MASK                                                                    0x0000F000L
+#define SQ_IMG_RSRC_WORD3__LAST_LEVEL_MASK                                                                    0x000F0000L
+#define SQ_IMG_RSRC_WORD3__SW_MODE_MASK                                                                       0x01F00000L
+#define SQ_IMG_RSRC_WORD3__TYPE_MASK                                                                          0xF0000000L
+//SQ_IMG_RSRC_WORD4
+#define SQ_IMG_RSRC_WORD4__DEPTH__SHIFT                                                                       0x0
+#define SQ_IMG_RSRC_WORD4__PITCH__SHIFT                                                                       0xd
+#define SQ_IMG_RSRC_WORD4__BC_SWIZZLE__SHIFT                                                                  0x1d
+#define SQ_IMG_RSRC_WORD4__DEPTH_MASK                                                                         0x00001FFFL
+#define SQ_IMG_RSRC_WORD4__PITCH_MASK                                                                         0x1FFFE000L
+#define SQ_IMG_RSRC_WORD4__BC_SWIZZLE_MASK                                                                    0xE0000000L
+//SQ_IMG_RSRC_WORD5
+#define SQ_IMG_RSRC_WORD5__BASE_ARRAY__SHIFT                                                                  0x0
+#define SQ_IMG_RSRC_WORD5__ARRAY_PITCH__SHIFT                                                                 0xd
+#define SQ_IMG_RSRC_WORD5__META_DATA_ADDRESS__SHIFT                                                           0x11
+#define SQ_IMG_RSRC_WORD5__META_LINEAR__SHIFT                                                                 0x19
+#define SQ_IMG_RSRC_WORD5__META_PIPE_ALIGNED__SHIFT                                                           0x1a
+#define SQ_IMG_RSRC_WORD5__META_RB_ALIGNED__SHIFT                                                             0x1b
+#define SQ_IMG_RSRC_WORD5__MAX_MIP__SHIFT                                                                     0x1c
+#define SQ_IMG_RSRC_WORD5__BASE_ARRAY_MASK                                                                    0x00001FFFL
+#define SQ_IMG_RSRC_WORD5__ARRAY_PITCH_MASK                                                                   0x0001E000L
+#define SQ_IMG_RSRC_WORD5__META_DATA_ADDRESS_MASK                                                             0x01FE0000L
+#define SQ_IMG_RSRC_WORD5__META_LINEAR_MASK                                                                   0x02000000L
+#define SQ_IMG_RSRC_WORD5__META_PIPE_ALIGNED_MASK                                                             0x04000000L
+#define SQ_IMG_RSRC_WORD5__META_RB_ALIGNED_MASK                                                               0x08000000L
+#define SQ_IMG_RSRC_WORD5__MAX_MIP_MASK                                                                       0xF0000000L
+//SQ_IMG_RSRC_WORD6
+#define SQ_IMG_RSRC_WORD6__MIN_LOD_WARN__SHIFT                                                                0x0
+#define SQ_IMG_RSRC_WORD6__COUNTER_BANK_ID__SHIFT                                                             0xc
+#define SQ_IMG_RSRC_WORD6__LOD_HDW_CNT_EN__SHIFT                                                              0x14
+#define SQ_IMG_RSRC_WORD6__COMPRESSION_EN__SHIFT                                                              0x15
+#define SQ_IMG_RSRC_WORD6__ALPHA_IS_ON_MSB__SHIFT                                                             0x16
+#define SQ_IMG_RSRC_WORD6__COLOR_TRANSFORM__SHIFT                                                             0x17
+#define SQ_IMG_RSRC_WORD6__LOST_ALPHA_BITS__SHIFT                                                             0x18
+#define SQ_IMG_RSRC_WORD6__LOST_COLOR_BITS__SHIFT                                                             0x1c
+#define SQ_IMG_RSRC_WORD6__MIN_LOD_WARN_MASK                                                                  0x00000FFFL
+#define SQ_IMG_RSRC_WORD6__COUNTER_BANK_ID_MASK                                                               0x000FF000L
+#define SQ_IMG_RSRC_WORD6__LOD_HDW_CNT_EN_MASK                                                                0x00100000L
+#define SQ_IMG_RSRC_WORD6__COMPRESSION_EN_MASK                                                                0x00200000L
+#define SQ_IMG_RSRC_WORD6__ALPHA_IS_ON_MSB_MASK                                                               0x00400000L
+#define SQ_IMG_RSRC_WORD6__COLOR_TRANSFORM_MASK                                                               0x00800000L
+#define SQ_IMG_RSRC_WORD6__LOST_ALPHA_BITS_MASK                                                               0x0F000000L
+#define SQ_IMG_RSRC_WORD6__LOST_COLOR_BITS_MASK                                                               0xF0000000L
+//SQ_IMG_RSRC_WORD7
+#define SQ_IMG_RSRC_WORD7__META_DATA_ADDRESS__SHIFT                                                           0x0
+#define SQ_IMG_RSRC_WORD7__META_DATA_ADDRESS_MASK                                                             0xFFFFFFFFL
+//SQ_IMG_SAMP_WORD0
+#define SQ_IMG_SAMP_WORD0__CLAMP_X__SHIFT                                                                     0x0
+#define SQ_IMG_SAMP_WORD0__CLAMP_Y__SHIFT                                                                     0x3
+#define SQ_IMG_SAMP_WORD0__CLAMP_Z__SHIFT                                                                     0x6
+#define SQ_IMG_SAMP_WORD0__MAX_ANISO_RATIO__SHIFT                                                             0x9
+#define SQ_IMG_SAMP_WORD0__DEPTH_COMPARE_FUNC__SHIFT                                                          0xc
+#define SQ_IMG_SAMP_WORD0__FORCE_UNNORMALIZED__SHIFT                                                          0xf
+#define SQ_IMG_SAMP_WORD0__ANISO_THRESHOLD__SHIFT                                                             0x10
+#define SQ_IMG_SAMP_WORD0__MC_COORD_TRUNC__SHIFT                                                              0x13
+#define SQ_IMG_SAMP_WORD0__FORCE_DEGAMMA__SHIFT                                                               0x14
+#define SQ_IMG_SAMP_WORD0__ANISO_BIAS__SHIFT                                                                  0x15
+#define SQ_IMG_SAMP_WORD0__TRUNC_COORD__SHIFT                                                                 0x1b
+#define SQ_IMG_SAMP_WORD0__DISABLE_CUBE_WRAP__SHIFT                                                           0x1c
+#define SQ_IMG_SAMP_WORD0__FILTER_MODE__SHIFT                                                                 0x1d
+#define SQ_IMG_SAMP_WORD0__COMPAT_MODE__SHIFT                                                                 0x1f
+#define SQ_IMG_SAMP_WORD0__CLAMP_X_MASK                                                                       0x00000007L
+#define SQ_IMG_SAMP_WORD0__CLAMP_Y_MASK                                                                       0x00000038L
+#define SQ_IMG_SAMP_WORD0__CLAMP_Z_MASK                                                                       0x000001C0L
+#define SQ_IMG_SAMP_WORD0__MAX_ANISO_RATIO_MASK                                                               0x00000E00L
+#define SQ_IMG_SAMP_WORD0__DEPTH_COMPARE_FUNC_MASK                                                            0x00007000L
+#define SQ_IMG_SAMP_WORD0__FORCE_UNNORMALIZED_MASK                                                            0x00008000L
+#define SQ_IMG_SAMP_WORD0__ANISO_THRESHOLD_MASK                                                               0x00070000L
+#define SQ_IMG_SAMP_WORD0__MC_COORD_TRUNC_MASK                                                                0x00080000L
+#define SQ_IMG_SAMP_WORD0__FORCE_DEGAMMA_MASK                                                                 0x00100000L
+#define SQ_IMG_SAMP_WORD0__ANISO_BIAS_MASK                                                                    0x07E00000L
+#define SQ_IMG_SAMP_WORD0__TRUNC_COORD_MASK                                                                   0x08000000L
+#define SQ_IMG_SAMP_WORD0__DISABLE_CUBE_WRAP_MASK                                                             0x10000000L
+#define SQ_IMG_SAMP_WORD0__FILTER_MODE_MASK                                                                   0x60000000L
+#define SQ_IMG_SAMP_WORD0__COMPAT_MODE_MASK                                                                   0x80000000L
+//SQ_IMG_SAMP_WORD1
+#define SQ_IMG_SAMP_WORD1__MIN_LOD__SHIFT                                                                     0x0
+#define SQ_IMG_SAMP_WORD1__MAX_LOD__SHIFT                                                                     0xc
+#define SQ_IMG_SAMP_WORD1__PERF_MIP__SHIFT                                                                    0x18
+#define SQ_IMG_SAMP_WORD1__PERF_Z__SHIFT                                                                      0x1c
+#define SQ_IMG_SAMP_WORD1__MIN_LOD_MASK                                                                       0x00000FFFL
+#define SQ_IMG_SAMP_WORD1__MAX_LOD_MASK                                                                       0x00FFF000L
+#define SQ_IMG_SAMP_WORD1__PERF_MIP_MASK                                                                      0x0F000000L
+#define SQ_IMG_SAMP_WORD1__PERF_Z_MASK                                                                        0xF0000000L
+//SQ_IMG_SAMP_WORD2
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS__SHIFT                                                                    0x0
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS_SEC__SHIFT                                                                0xe
+#define SQ_IMG_SAMP_WORD2__XY_MAG_FILTER__SHIFT                                                               0x14
+#define SQ_IMG_SAMP_WORD2__XY_MIN_FILTER__SHIFT                                                               0x16
+#define SQ_IMG_SAMP_WORD2__Z_FILTER__SHIFT                                                                    0x18
+#define SQ_IMG_SAMP_WORD2__MIP_FILTER__SHIFT                                                                  0x1a
+#define SQ_IMG_SAMP_WORD2__MIP_POINT_PRECLAMP__SHIFT                                                          0x1c
+#define SQ_IMG_SAMP_WORD2__BLEND_ZERO_PRT__SHIFT                                                              0x1d
+#define SQ_IMG_SAMP_WORD2__FILTER_PREC_FIX__SHIFT                                                             0x1e
+#define SQ_IMG_SAMP_WORD2__ANISO_OVERRIDE__SHIFT                                                              0x1f
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS_MASK                                                                      0x00003FFFL
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS_SEC_MASK                                                                  0x000FC000L
+#define SQ_IMG_SAMP_WORD2__XY_MAG_FILTER_MASK                                                                 0x00300000L
+#define SQ_IMG_SAMP_WORD2__XY_MIN_FILTER_MASK                                                                 0x00C00000L
+#define SQ_IMG_SAMP_WORD2__Z_FILTER_MASK                                                                      0x03000000L
+#define SQ_IMG_SAMP_WORD2__MIP_FILTER_MASK                                                                    0x0C000000L
+#define SQ_IMG_SAMP_WORD2__MIP_POINT_PRECLAMP_MASK                                                            0x10000000L
+#define SQ_IMG_SAMP_WORD2__BLEND_ZERO_PRT_MASK                                                                0x20000000L
+#define SQ_IMG_SAMP_WORD2__FILTER_PREC_FIX_MASK                                                               0x40000000L
+#define SQ_IMG_SAMP_WORD2__ANISO_OVERRIDE_MASK                                                                0x80000000L
+//SQ_IMG_SAMP_WORD3
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_PTR__SHIFT                                                            0x0
+#define SQ_IMG_SAMP_WORD3__SKIP_DEGAMMA__SHIFT                                                                0xc
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_TYPE__SHIFT                                                           0x1e
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_PTR_MASK                                                              0x00000FFFL
+#define SQ_IMG_SAMP_WORD3__SKIP_DEGAMMA_MASK                                                                  0x00001000L
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_TYPE_MASK                                                             0xC0000000L
+//SQ_FLAT_SCRATCH_WORD0
+#define SQ_FLAT_SCRATCH_WORD0__SIZE__SHIFT                                                                    0x0
+#define SQ_FLAT_SCRATCH_WORD0__SIZE_MASK                                                                      0x0007FFFFL
+//SQ_FLAT_SCRATCH_WORD1
+#define SQ_FLAT_SCRATCH_WORD1__OFFSET__SHIFT                                                                  0x0
+#define SQ_FLAT_SCRATCH_WORD1__OFFSET_MASK                                                                    0x00FFFFFFL
+//SQ_M0_GPR_IDX_WORD
+#define SQ_M0_GPR_IDX_WORD__INDEX__SHIFT                                                                      0x0
+#define SQ_M0_GPR_IDX_WORD__VSRC0_REL__SHIFT                                                                  0xc
+#define SQ_M0_GPR_IDX_WORD__VSRC1_REL__SHIFT                                                                  0xd
+#define SQ_M0_GPR_IDX_WORD__VSRC2_REL__SHIFT                                                                  0xe
+#define SQ_M0_GPR_IDX_WORD__VDST_REL__SHIFT                                                                   0xf
+#define SQ_M0_GPR_IDX_WORD__INDEX_MASK                                                                        0x000000FFL
+#define SQ_M0_GPR_IDX_WORD__VSRC0_REL_MASK                                                                    0x00001000L
+#define SQ_M0_GPR_IDX_WORD__VSRC1_REL_MASK                                                                    0x00002000L
+#define SQ_M0_GPR_IDX_WORD__VSRC2_REL_MASK                                                                    0x00004000L
+#define SQ_M0_GPR_IDX_WORD__VDST_REL_MASK                                                                     0x00008000L
+//SQC_ICACHE_UTCL1_CNTL1
+#define SQC_ICACHE_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                       0x0
+#define SQC_ICACHE_UTCL1_CNTL1__GPUVM_64K_DEF__SHIFT                                                          0x1
+#define SQC_ICACHE_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                        0x2
+#define SQC_ICACHE_UTCL1_CNTL1__RESP_MODE__SHIFT                                                              0x3
+#define SQC_ICACHE_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                        0x5
+#define SQC_ICACHE_UTCL1_CNTL1__CLIENTID__SHIFT                                                               0x7
+#define SQC_ICACHE_UTCL1_CNTL1__RESERVED__SHIFT                                                               0x10
+#define SQC_ICACHE_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                      0x11
+#define SQC_ICACHE_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                   0x12
+#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_VMID__SHIFT                                                    0x13
+#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID__SHIFT                                                0x17
+#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE__SHIFT                                                  0x18
+#define SQC_ICACHE_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID__SHIFT                                             0x19
+#define SQC_ICACHE_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                             0x1a
+#define SQC_ICACHE_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                         0x1b
+#define SQC_ICACHE_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                 0x1c
+#define SQC_ICACHE_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                 0x1e
+#define SQC_ICACHE_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                         0x00000001L
+#define SQC_ICACHE_UTCL1_CNTL1__GPUVM_64K_DEF_MASK                                                            0x00000002L
+#define SQC_ICACHE_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                          0x00000004L
+#define SQC_ICACHE_UTCL1_CNTL1__RESP_MODE_MASK                                                                0x00000018L
+#define SQC_ICACHE_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                          0x00000060L
+#define SQC_ICACHE_UTCL1_CNTL1__CLIENTID_MASK                                                                 0x0000FF80L
+#define SQC_ICACHE_UTCL1_CNTL1__RESERVED_MASK                                                                 0x00010000L
+#define SQC_ICACHE_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                        0x00020000L
+#define SQC_ICACHE_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                     0x00040000L
+#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_VMID_MASK                                                      0x00780000L
+#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID_MASK                                                  0x00800000L
+#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE_MASK                                                    0x01000000L
+#define SQC_ICACHE_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID_MASK                                               0x02000000L
+#define SQC_ICACHE_UTCL1_CNTL1__FORCE_MISS_MASK                                                               0x04000000L
+#define SQC_ICACHE_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                           0x08000000L
+#define SQC_ICACHE_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                   0x30000000L
+#define SQC_ICACHE_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                   0xC0000000L
+//SQC_ICACHE_UTCL1_CNTL2
+#define SQC_ICACHE_UTCL1_CNTL2__SPARE__SHIFT                                                                  0x0
+#define SQC_ICACHE_UTCL1_CNTL2__LFIFO_SCAN_DISABLE__SHIFT                                                     0x8
+#define SQC_ICACHE_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                         0x9
+#define SQC_ICACHE_UTCL1_CNTL2__LINE_VALID__SHIFT                                                             0xa
+#define SQC_ICACHE_UTCL1_CNTL2__DIS_EDC__SHIFT                                                                0xb
+#define SQC_ICACHE_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                         0xc
+#define SQC_ICACHE_UTCL1_CNTL2__SHOOTDOWN_OPT__SHIFT                                                          0xd
+#define SQC_ICACHE_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                            0xe
+#define SQC_ICACHE_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                    0xf
+#define SQC_ICACHE_UTCL1_CNTL2__ARB_BURST_MODE__SHIFT                                                         0x10
+#define SQC_ICACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR__SHIFT                                                0x12
+#define SQC_ICACHE_UTCL1_CNTL2__PERF_EVENT_RD_WR__SHIFT                                                       0x13
+#define SQC_ICACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID__SHIFT                                                 0x14
+#define SQC_ICACHE_UTCL1_CNTL2__PERF_EVENT_VMID__SHIFT                                                        0x15
+#define SQC_ICACHE_UTCL1_CNTL2__RESERVED__SHIFT                                                               0x19
+#define SQC_ICACHE_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                   0x1a
+#define SQC_ICACHE_UTCL1_CNTL2__SPARE_MASK                                                                    0x000000FFL
+#define SQC_ICACHE_UTCL1_CNTL2__LFIFO_SCAN_DISABLE_MASK                                                       0x00000100L
+#define SQC_ICACHE_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                           0x00000200L
+#define SQC_ICACHE_UTCL1_CNTL2__LINE_VALID_MASK                                                               0x00000400L
+#define SQC_ICACHE_UTCL1_CNTL2__DIS_EDC_MASK                                                                  0x00000800L
+#define SQC_ICACHE_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                           0x00001000L
+#define SQC_ICACHE_UTCL1_CNTL2__SHOOTDOWN_OPT_MASK                                                            0x00002000L
+#define SQC_ICACHE_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                              0x00004000L
+#define SQC_ICACHE_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                      0x00008000L
+#define SQC_ICACHE_UTCL1_CNTL2__ARB_BURST_MODE_MASK                                                           0x00030000L
+#define SQC_ICACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR_MASK                                                  0x00040000L
+#define SQC_ICACHE_UTCL1_CNTL2__PERF_EVENT_RD_WR_MASK                                                         0x00080000L
+#define SQC_ICACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID_MASK                                                   0x00100000L
+#define SQC_ICACHE_UTCL1_CNTL2__PERF_EVENT_VMID_MASK                                                          0x01E00000L
+#define SQC_ICACHE_UTCL1_CNTL2__RESERVED_MASK                                                                 0x02000000L
+#define SQC_ICACHE_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                     0x04000000L
+//SQC_DCACHE_UTCL1_CNTL1
+#define SQC_DCACHE_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                       0x0
+#define SQC_DCACHE_UTCL1_CNTL1__GPUVM_64K_DEF__SHIFT                                                          0x1
+#define SQC_DCACHE_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                        0x2
+#define SQC_DCACHE_UTCL1_CNTL1__RESP_MODE__SHIFT                                                              0x3
+#define SQC_DCACHE_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                        0x5
+#define SQC_DCACHE_UTCL1_CNTL1__CLIENTID__SHIFT                                                               0x7
+#define SQC_DCACHE_UTCL1_CNTL1__RESERVED__SHIFT                                                               0x10
+#define SQC_DCACHE_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                      0x11
+#define SQC_DCACHE_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                   0x12
+#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_VMID__SHIFT                                                    0x13
+#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID__SHIFT                                                0x17
+#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE__SHIFT                                                  0x18
+#define SQC_DCACHE_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID__SHIFT                                             0x19
+#define SQC_DCACHE_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                             0x1a
+#define SQC_DCACHE_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                         0x1b
+#define SQC_DCACHE_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                 0x1c
+#define SQC_DCACHE_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                 0x1e
+#define SQC_DCACHE_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                         0x00000001L
+#define SQC_DCACHE_UTCL1_CNTL1__GPUVM_64K_DEF_MASK                                                            0x00000002L
+#define SQC_DCACHE_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                          0x00000004L
+#define SQC_DCACHE_UTCL1_CNTL1__RESP_MODE_MASK                                                                0x00000018L
+#define SQC_DCACHE_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                          0x00000060L
+#define SQC_DCACHE_UTCL1_CNTL1__CLIENTID_MASK                                                                 0x0000FF80L
+#define SQC_DCACHE_UTCL1_CNTL1__RESERVED_MASK                                                                 0x00010000L
+#define SQC_DCACHE_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                        0x00020000L
+#define SQC_DCACHE_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                     0x00040000L
+#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_VMID_MASK                                                      0x00780000L
+#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID_MASK                                                  0x00800000L
+#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE_MASK                                                    0x01000000L
+#define SQC_DCACHE_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID_MASK                                               0x02000000L
+#define SQC_DCACHE_UTCL1_CNTL1__FORCE_MISS_MASK                                                               0x04000000L
+#define SQC_DCACHE_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                           0x08000000L
+#define SQC_DCACHE_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                   0x30000000L
+#define SQC_DCACHE_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                   0xC0000000L
+//SQC_DCACHE_UTCL1_CNTL2
+#define SQC_DCACHE_UTCL1_CNTL2__SPARE__SHIFT                                                                  0x0
+#define SQC_DCACHE_UTCL1_CNTL2__LFIFO_SCAN_DISABLE__SHIFT                                                     0x8
+#define SQC_DCACHE_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                         0x9
+#define SQC_DCACHE_UTCL1_CNTL2__LINE_VALID__SHIFT                                                             0xa
+#define SQC_DCACHE_UTCL1_CNTL2__DIS_EDC__SHIFT                                                                0xb
+#define SQC_DCACHE_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                         0xc
+#define SQC_DCACHE_UTCL1_CNTL2__SHOOTDOWN_OPT__SHIFT                                                          0xd
+#define SQC_DCACHE_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                            0xe
+#define SQC_DCACHE_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                    0xf
+#define SQC_DCACHE_UTCL1_CNTL2__ARB_BURST_MODE__SHIFT                                                         0x10
+#define SQC_DCACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR__SHIFT                                                0x12
+#define SQC_DCACHE_UTCL1_CNTL2__PERF_EVENT_RD_WR__SHIFT                                                       0x13
+#define SQC_DCACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID__SHIFT                                                 0x14
+#define SQC_DCACHE_UTCL1_CNTL2__PERF_EVENT_VMID__SHIFT                                                        0x15
+#define SQC_DCACHE_UTCL1_CNTL2__RESERVED__SHIFT                                                               0x19
+#define SQC_DCACHE_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                   0x1a
+#define SQC_DCACHE_UTCL1_CNTL2__SPARE_MASK                                                                    0x000000FFL
+#define SQC_DCACHE_UTCL1_CNTL2__LFIFO_SCAN_DISABLE_MASK                                                       0x00000100L
+#define SQC_DCACHE_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                           0x00000200L
+#define SQC_DCACHE_UTCL1_CNTL2__LINE_VALID_MASK                                                               0x00000400L
+#define SQC_DCACHE_UTCL1_CNTL2__DIS_EDC_MASK                                                                  0x00000800L
+#define SQC_DCACHE_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                           0x00001000L
+#define SQC_DCACHE_UTCL1_CNTL2__SHOOTDOWN_OPT_MASK                                                            0x00002000L
+#define SQC_DCACHE_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                              0x00004000L
+#define SQC_DCACHE_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                      0x00008000L
+#define SQC_DCACHE_UTCL1_CNTL2__ARB_BURST_MODE_MASK                                                           0x00030000L
+#define SQC_DCACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR_MASK                                                  0x00040000L
+#define SQC_DCACHE_UTCL1_CNTL2__PERF_EVENT_RD_WR_MASK                                                         0x00080000L
+#define SQC_DCACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID_MASK                                                   0x00100000L
+#define SQC_DCACHE_UTCL1_CNTL2__PERF_EVENT_VMID_MASK                                                          0x01E00000L
+#define SQC_DCACHE_UTCL1_CNTL2__RESERVED_MASK                                                                 0x02000000L
+#define SQC_DCACHE_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                     0x04000000L
+//SQC_ICACHE_UTCL1_STATUS
+#define SQC_ICACHE_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                        0x0
+#define SQC_ICACHE_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                        0x1
+#define SQC_ICACHE_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                          0x2
+#define SQC_ICACHE_UTCL1_STATUS__FAULT_DETECTED_MASK                                                          0x00000001L
+#define SQC_ICACHE_UTCL1_STATUS__RETRY_DETECTED_MASK                                                          0x00000002L
+#define SQC_ICACHE_UTCL1_STATUS__PRT_DETECTED_MASK                                                            0x00000004L
+//SQC_DCACHE_UTCL1_STATUS
+#define SQC_DCACHE_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                        0x0
+#define SQC_DCACHE_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                        0x1
+#define SQC_DCACHE_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                          0x2
+#define SQC_DCACHE_UTCL1_STATUS__FAULT_DETECTED_MASK                                                          0x00000001L
+#define SQC_DCACHE_UTCL1_STATUS__RETRY_DETECTED_MASK                                                          0x00000002L
+#define SQC_DCACHE_UTCL1_STATUS__PRT_DETECTED_MASK                                                            0x00000004L
+
+
+// addressBlock: xcd0_gc_shsdec
+//SX_DEBUG_BUSY
+#define SX_DEBUG_BUSY__RESERVED__SHIFT                                                                        0x0
+#define SX_DEBUG_BUSY__PCCMD_VALID__SHIFT                                                                     0x1b
+#define SX_DEBUG_BUSY__VDATA1_VALID__SHIFT                                                                    0x1c
+#define SX_DEBUG_BUSY__VDATA0_VALID__SHIFT                                                                    0x1d
+#define SX_DEBUG_BUSY__CMD_BUSYORVAL__SHIFT                                                                   0x1e
+#define SX_DEBUG_BUSY__PCDATA_VALID__SHIFT                                                                    0x1f
+#define SX_DEBUG_BUSY__RESERVED_MASK                                                                          0x07FFFFFFL
+#define SX_DEBUG_BUSY__PCCMD_VALID_MASK                                                                       0x08000000L
+#define SX_DEBUG_BUSY__VDATA1_VALID_MASK                                                                      0x10000000L
+#define SX_DEBUG_BUSY__VDATA0_VALID_MASK                                                                      0x20000000L
+#define SX_DEBUG_BUSY__CMD_BUSYORVAL_MASK                                                                     0x40000000L
+#define SX_DEBUG_BUSY__PCDATA_VALID_MASK                                                                      0x80000000L
+//SX_DEBUG_1
+#define SX_DEBUG_1__RESERVED__SHIFT                                                                           0x0
+#define SX_DEBUG_1__DISABLE_REP_FGCG__SHIFT                                                                   0xd
+#define SX_DEBUG_1__RESERVED_MASK                                                                             0x00001FFFL
+#define SX_DEBUG_1__DISABLE_REP_FGCG_MASK                                                                     0x00002000L
+//SPI_PS_MAX_WAVE_ID
+#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                                0x0
+#define SPI_PS_MAX_WAVE_ID__MAX_COLLISION_WAVE_ID__SHIFT                                                      0x10
+#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                                  0x00000FFFL
+#define SPI_PS_MAX_WAVE_ID__MAX_COLLISION_WAVE_ID_MASK                                                        0x03FF0000L
+//SPI_START_PHASE
+#define SPI_START_PHASE__VGPR_START_PHASE__SHIFT                                                              0x0
+#define SPI_START_PHASE__SGPR_START_PHASE__SHIFT                                                              0x2
+#define SPI_START_PHASE__WAVE_START_PHASE__SHIFT                                                              0x4
+#define SPI_START_PHASE__SPI_TD_GAP__SHIFT                                                                    0x6
+#define SPI_START_PHASE__VGPR_START_PHASE_MASK                                                                0x00000003L
+#define SPI_START_PHASE__SGPR_START_PHASE_MASK                                                                0x0000000CL
+#define SPI_START_PHASE__WAVE_START_PHASE_MASK                                                                0x00000030L
+#define SPI_START_PHASE__SPI_TD_GAP_MASK                                                                      0x000003C0L
+//SPI_GFX_CNTL
+#define SPI_GFX_CNTL__RESET_COUNTS__SHIFT                                                                     0x0
+#define SPI_GFX_CNTL__RESET_COUNTS_MASK                                                                       0x00000001L
+//SPI_DEBUG_READ
+#define SPI_DEBUG_READ__DATA__SHIFT                                                                           0x0
+#define SPI_DEBUG_READ__DATA_MASK                                                                             0xFFFFFFFFL
+//SPI_DSM_CNTL
+#define SPI_DSM_CNTL__SPI_SR_MEM_DSM_IRRITATOR_DATA__SHIFT                                                    0x0
+#define SPI_DSM_CNTL__SPI_SR_MEM_ENABLE_SINGLE_WRITE__SHIFT                                                   0x2
+#define SPI_DSM_CNTL__SPI_GDS_EXPREQ_MEM_DSM_IRRITATOR_DATA__SHIFT                                            0x3
+#define SPI_DSM_CNTL__SPI_GDS_EXPREQ_MEM_ENABLE_SINGLE_WRITE__SHIFT                                           0x5
+#define SPI_DSM_CNTL__SPI_WB_GRANT_30_MEM_DSM_IRRITATOR_DATA__SHIFT                                           0x6
+#define SPI_DSM_CNTL__SPI_WB_GRANT_30_MEM_ENABLE_SINGLE_WRITE__SHIFT                                          0x8
+#define SPI_DSM_CNTL__RESERVED__SHIFT                                                                         0x9
+#define SPI_DSM_CNTL__SPI_LIFE_CNT_MEM_DSM_IRRITATOR_DATA__SHIFT                                              0xc
+#define SPI_DSM_CNTL__SPI_LIFE_CNT_MEM_ENABLE_SINGLE_WRITE__SHIFT                                             0xe
+#define SPI_DSM_CNTL__UNUSED__SHIFT                                                                           0xf
+#define SPI_DSM_CNTL__SPI_SR_MEM_DSM_IRRITATOR_DATA_MASK                                                      0x00000003L
+#define SPI_DSM_CNTL__SPI_SR_MEM_ENABLE_SINGLE_WRITE_MASK                                                     0x00000004L
+#define SPI_DSM_CNTL__SPI_GDS_EXPREQ_MEM_DSM_IRRITATOR_DATA_MASK                                              0x00000018L
+#define SPI_DSM_CNTL__SPI_GDS_EXPREQ_MEM_ENABLE_SINGLE_WRITE_MASK                                             0x00000020L
+#define SPI_DSM_CNTL__SPI_WB_GRANT_30_MEM_DSM_IRRITATOR_DATA_MASK                                             0x000000C0L
+#define SPI_DSM_CNTL__SPI_WB_GRANT_30_MEM_ENABLE_SINGLE_WRITE_MASK                                            0x00000100L
+#define SPI_DSM_CNTL__RESERVED_MASK                                                                           0x00000E00L
+#define SPI_DSM_CNTL__SPI_LIFE_CNT_MEM_DSM_IRRITATOR_DATA_MASK                                                0x00003000L
+#define SPI_DSM_CNTL__SPI_LIFE_CNT_MEM_ENABLE_SINGLE_WRITE_MASK                                               0x00004000L
+#define SPI_DSM_CNTL__UNUSED_MASK                                                                             0xFFFF8000L
+//SPI_DSM_CNTL2
+#define SPI_DSM_CNTL2__SPI_SR_MEM_ENABLE_ERROR_INJECT__SHIFT                                                  0x0
+#define SPI_DSM_CNTL2__SPI_SR_MEM_SELECT_INJECT_DELAY__SHIFT                                                  0x2
+#define SPI_DSM_CNTL2__SPI_SR_MEM_INJECT_DELAY__SHIFT                                                         0x4
+#define SPI_DSM_CNTL2__SPI_GDS_EXPREQ_MEM_ENABLE_ERROR_INJECT__SHIFT                                          0xa
+#define SPI_DSM_CNTL2__SPI_GDS_EXPREQ_MEM_SELECT_INJECT_DELAY__SHIFT                                          0xc
+#define SPI_DSM_CNTL2__SPI_WB_GRANT_30_MEM_ENABLE_ERROR_INJECT__SHIFT                                         0xd
+#define SPI_DSM_CNTL2__SPI_WB_GRANT_30_MEM_SELECT_INJECT_DELAY__SHIFT                                         0xf
+#define SPI_DSM_CNTL2__RESERVED__SHIFT                                                                        0x10
+#define SPI_DSM_CNTL2__SPI_LIFE_CNT_MEM_ENABLE_ERROR_INJECT__SHIFT                                            0x13
+#define SPI_DSM_CNTL2__SPI_LIFE_CNT_MEM_SELECT_INJECT_DELAY__SHIFT                                            0x15
+#define SPI_DSM_CNTL2__UNUSED__SHIFT                                                                          0x16
+#define SPI_DSM_CNTL2__SPI_SR_MEM_ENABLE_ERROR_INJECT_MASK                                                    0x00000003L
+#define SPI_DSM_CNTL2__SPI_SR_MEM_SELECT_INJECT_DELAY_MASK                                                    0x00000004L
+#define SPI_DSM_CNTL2__SPI_SR_MEM_INJECT_DELAY_MASK                                                           0x000003F0L
+#define SPI_DSM_CNTL2__SPI_GDS_EXPREQ_MEM_ENABLE_ERROR_INJECT_MASK                                            0x00000C00L
+#define SPI_DSM_CNTL2__SPI_GDS_EXPREQ_MEM_SELECT_INJECT_DELAY_MASK                                            0x00001000L
+#define SPI_DSM_CNTL2__SPI_WB_GRANT_30_MEM_ENABLE_ERROR_INJECT_MASK                                           0x00006000L
+#define SPI_DSM_CNTL2__SPI_WB_GRANT_30_MEM_SELECT_INJECT_DELAY_MASK                                           0x00008000L
+#define SPI_DSM_CNTL2__RESERVED_MASK                                                                          0x00070000L
+#define SPI_DSM_CNTL2__SPI_LIFE_CNT_MEM_ENABLE_ERROR_INJECT_MASK                                              0x00180000L
+#define SPI_DSM_CNTL2__SPI_LIFE_CNT_MEM_SELECT_INJECT_DELAY_MASK                                              0x00200000L
+#define SPI_DSM_CNTL2__UNUSED_MASK                                                                            0xFFC00000L
+//SPI_EDC_CNT
+#define SPI_EDC_CNT__SPI_SR_MEM_SEC_COUNT__SHIFT                                                              0x0
+#define SPI_EDC_CNT__SPI_SR_MEM_DED_COUNT__SHIFT                                                              0x2
+#define SPI_EDC_CNT__SPI_GDS_EXPREQ_SEC_COUNT__SHIFT                                                          0x4
+#define SPI_EDC_CNT__SPI_GDS_EXPREQ_DED_COUNT__SHIFT                                                          0x6
+#define SPI_EDC_CNT__SPI_WB_GRANT_30_SEC_COUNT__SHIFT                                                         0x8
+#define SPI_EDC_CNT__SPI_WB_GRANT_30_DED_COUNT__SHIFT                                                         0xa
+#define SPI_EDC_CNT__RESERVED__SHIFT                                                                          0xc
+#define SPI_EDC_CNT__SPI_LIFE_CNT_SEC_COUNT__SHIFT                                                            0x10
+#define SPI_EDC_CNT__SPI_LIFE_CNT_DED_COUNT__SHIFT                                                            0x12
+#define SPI_EDC_CNT__UNUSED__SHIFT                                                                            0x14
+#define SPI_EDC_CNT__SPI_SR_MEM_SEC_COUNT_MASK                                                                0x00000003L
+#define SPI_EDC_CNT__SPI_SR_MEM_DED_COUNT_MASK                                                                0x0000000CL
+#define SPI_EDC_CNT__SPI_GDS_EXPREQ_SEC_COUNT_MASK                                                            0x00000030L
+#define SPI_EDC_CNT__SPI_GDS_EXPREQ_DED_COUNT_MASK                                                            0x000000C0L
+#define SPI_EDC_CNT__SPI_WB_GRANT_30_SEC_COUNT_MASK                                                           0x00000300L
+#define SPI_EDC_CNT__SPI_WB_GRANT_30_DED_COUNT_MASK                                                           0x00000C00L
+#define SPI_EDC_CNT__RESERVED_MASK                                                                            0x0000F000L
+#define SPI_EDC_CNT__SPI_LIFE_CNT_SEC_COUNT_MASK                                                              0x00030000L
+#define SPI_EDC_CNT__SPI_LIFE_CNT_DED_COUNT_MASK                                                              0x000C0000L
+#define SPI_EDC_CNT__UNUSED_MASK                                                                              0xFFF00000L
+//SPI_DEBUG_BUSY
+#define SPI_DEBUG_BUSY__HS_BUSY__SHIFT                                                                        0x0
+#define SPI_DEBUG_BUSY__GS_BUSY__SHIFT                                                                        0x1
+#define SPI_DEBUG_BUSY__VS_BUSY__SHIFT                                                                        0x2
+#define SPI_DEBUG_BUSY__PS0_BUSY__SHIFT                                                                       0x3
+#define SPI_DEBUG_BUSY__PS1_BUSY__SHIFT                                                                       0x4
+#define SPI_DEBUG_BUSY__CSG_BUSY__SHIFT                                                                       0x5
+#define SPI_DEBUG_BUSY__CS0_BUSY__SHIFT                                                                       0x6
+#define SPI_DEBUG_BUSY__CS1_BUSY__SHIFT                                                                       0x7
+#define SPI_DEBUG_BUSY__CS2_BUSY__SHIFT                                                                       0x8
+#define SPI_DEBUG_BUSY__CS3_BUSY__SHIFT                                                                       0x9
+#define SPI_DEBUG_BUSY__CS4_BUSY__SHIFT                                                                       0xa
+#define SPI_DEBUG_BUSY__CS5_BUSY__SHIFT                                                                       0xb
+#define SPI_DEBUG_BUSY__CS6_BUSY__SHIFT                                                                       0xc
+#define SPI_DEBUG_BUSY__CS7_BUSY__SHIFT                                                                       0xd
+#define SPI_DEBUG_BUSY__LDS_WR_CTL0_BUSY__SHIFT                                                               0xe
+#define SPI_DEBUG_BUSY__LDS_WR_CTL1_BUSY__SHIFT                                                               0xf
+#define SPI_DEBUG_BUSY__RSRC_ALLOC0_BUSY__SHIFT                                                               0x10
+#define SPI_DEBUG_BUSY__RSRC_ALLOC1_BUSY__SHIFT                                                               0x11
+#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY__SHIFT                                                                0x12
+#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY__SHIFT                                                               0x13
+#define SPI_DEBUG_BUSY__GRBM_BUSY__SHIFT                                                                      0x14
+#define SPI_DEBUG_BUSY__SPIS_BUSY__SHIFT                                                                      0x15
+#define SPI_DEBUG_BUSY__HS_BUSY_MASK                                                                          0x00000001L
+#define SPI_DEBUG_BUSY__GS_BUSY_MASK                                                                          0x00000002L
+#define SPI_DEBUG_BUSY__VS_BUSY_MASK                                                                          0x00000004L
+#define SPI_DEBUG_BUSY__PS0_BUSY_MASK                                                                         0x00000008L
+#define SPI_DEBUG_BUSY__PS1_BUSY_MASK                                                                         0x00000010L
+#define SPI_DEBUG_BUSY__CSG_BUSY_MASK                                                                         0x00000020L
+#define SPI_DEBUG_BUSY__CS0_BUSY_MASK                                                                         0x00000040L
+#define SPI_DEBUG_BUSY__CS1_BUSY_MASK                                                                         0x00000080L
+#define SPI_DEBUG_BUSY__CS2_BUSY_MASK                                                                         0x00000100L
+#define SPI_DEBUG_BUSY__CS3_BUSY_MASK                                                                         0x00000200L
+#define SPI_DEBUG_BUSY__CS4_BUSY_MASK                                                                         0x00000400L
+#define SPI_DEBUG_BUSY__CS5_BUSY_MASK                                                                         0x00000800L
+#define SPI_DEBUG_BUSY__CS6_BUSY_MASK                                                                         0x00001000L
+#define SPI_DEBUG_BUSY__CS7_BUSY_MASK                                                                         0x00002000L
+#define SPI_DEBUG_BUSY__LDS_WR_CTL0_BUSY_MASK                                                                 0x00004000L
+#define SPI_DEBUG_BUSY__LDS_WR_CTL1_BUSY_MASK                                                                 0x00008000L
+#define SPI_DEBUG_BUSY__RSRC_ALLOC0_BUSY_MASK                                                                 0x00010000L
+#define SPI_DEBUG_BUSY__RSRC_ALLOC1_BUSY_MASK                                                                 0x00020000L
+#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY_MASK                                                                  0x00040000L
+#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY_MASK                                                                 0x00080000L
+#define SPI_DEBUG_BUSY__GRBM_BUSY_MASK                                                                        0x00100000L
+#define SPI_DEBUG_BUSY__SPIS_BUSY_MASK                                                                        0x00200000L
+//SPI_CONFIG_PS_CU_EN
+#define SPI_CONFIG_PS_CU_EN__ENABLE__SHIFT                                                                    0x0
+#define SPI_CONFIG_PS_CU_EN__PKR0_CU_EN__SHIFT                                                                0x1
+#define SPI_CONFIG_PS_CU_EN__PKR1_CU_EN__SHIFT                                                                0x10
+#define SPI_CONFIG_PS_CU_EN__ENABLE_MASK                                                                      0x00000001L
+#define SPI_CONFIG_PS_CU_EN__PKR0_CU_EN_MASK                                                                  0x0000FFFEL
+#define SPI_CONFIG_PS_CU_EN__PKR1_CU_EN_MASK                                                                  0xFFFF0000L
+//SPI_WF_LIFETIME_CNTL
+#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD__SHIFT                                                            0x0
+#define SPI_WF_LIFETIME_CNTL__EN__SHIFT                                                                       0x4
+#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD_MASK                                                              0x0000000FL
+#define SPI_WF_LIFETIME_CNTL__EN_MASK                                                                         0x00000010L
+//SPI_WF_LIFETIME_LIMIT_0
+#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_1
+#define SPI_WF_LIFETIME_LIMIT_1__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_1__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_1__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_1__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_2
+#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_3
+#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_4
+#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_5
+#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_6
+#define SPI_WF_LIFETIME_LIMIT_6__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_6__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_6__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_6__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_7
+#define SPI_WF_LIFETIME_LIMIT_7__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_7__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_7__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_7__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_8
+#define SPI_WF_LIFETIME_LIMIT_8__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_8__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_8__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_8__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_9
+#define SPI_WF_LIFETIME_LIMIT_9__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_9__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_9__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_9__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_STATUS_0
+#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_0__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_0__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_1
+#define SPI_WF_LIFETIME_STATUS_1__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_1__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_1__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_1__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_2
+#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_2__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_2__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_3
+#define SPI_WF_LIFETIME_STATUS_3__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_3__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_3__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_3__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_4
+#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_4__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_4__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_5
+#define SPI_WF_LIFETIME_STATUS_5__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_5__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_5__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_5__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_6
+#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_6__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_6__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_7
+#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_7__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_7__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_8
+#define SPI_WF_LIFETIME_STATUS_8__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_8__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_8__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_8__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_9
+#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_9__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_9__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_10
+#define SPI_WF_LIFETIME_STATUS_10__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_10__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_10__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_10__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_11
+#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_11__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_11__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_12
+#define SPI_WF_LIFETIME_STATUS_12__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_12__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_12__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_12__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_13
+#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_13__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_13__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_14
+#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_14__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_14__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_15
+#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_15__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_15__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_16
+#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_16__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_16__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_17
+#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_17__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_17__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_18
+#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_18__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_18__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_19
+#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_19__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_19__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_20
+#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_20__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_20__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_DEBUG
+#define SPI_WF_LIFETIME_DEBUG__START_VALUE__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_DEBUG__OVERRIDE_EN__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_DEBUG__START_VALUE_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_DEBUG__OVERRIDE_EN_MASK                                                               0x80000000L
+//SPI_LB_CTR_CTRL
+#define SPI_LB_CTR_CTRL__LOAD__SHIFT                                                                          0x0
+#define SPI_LB_CTR_CTRL__WAVES_SELECT__SHIFT                                                                  0x1
+#define SPI_LB_CTR_CTRL__CLEAR_ON_READ__SHIFT                                                                 0x3
+#define SPI_LB_CTR_CTRL__RESET_COUNTS__SHIFT                                                                  0x4
+#define SPI_LB_CTR_CTRL__LOAD_MASK                                                                            0x00000001L
+#define SPI_LB_CTR_CTRL__WAVES_SELECT_MASK                                                                    0x00000006L
+#define SPI_LB_CTR_CTRL__CLEAR_ON_READ_MASK                                                                   0x00000008L
+#define SPI_LB_CTR_CTRL__RESET_COUNTS_MASK                                                                    0x00000010L
+//SPI_LB_CU_MASK
+#define SPI_LB_CU_MASK__CU_MASK__SHIFT                                                                        0x0
+#define SPI_LB_CU_MASK__CU_MASK_MASK                                                                          0xFFFFL
+//SPI_LB_DATA_REG
+#define SPI_LB_DATA_REG__CNT_DATA__SHIFT                                                                      0x0
+#define SPI_LB_DATA_REG__CNT_DATA_MASK                                                                        0xFFFFFFFFL
+//SPI_PG_ENABLE_STATIC_CU_MASK
+#define SPI_PG_ENABLE_STATIC_CU_MASK__CU_MASK__SHIFT                                                          0x0
+#define SPI_PG_ENABLE_STATIC_CU_MASK__CU_MASK_MASK                                                            0xFFFFL
+//SPI_GDS_CREDITS
+#define SPI_GDS_CREDITS__DS_DATA_CREDITS__SHIFT                                                               0x0
+#define SPI_GDS_CREDITS__DS_CMD_CREDITS__SHIFT                                                                0x8
+#define SPI_GDS_CREDITS__UNUSED__SHIFT                                                                        0x10
+#define SPI_GDS_CREDITS__DS_DATA_CREDITS_MASK                                                                 0x000000FFL
+#define SPI_GDS_CREDITS__DS_CMD_CREDITS_MASK                                                                  0x0000FF00L
+#define SPI_GDS_CREDITS__UNUSED_MASK                                                                          0xFFFF0000L
+//SPI_SX_EXPORT_BUFFER_SIZES
+#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE__SHIFT                                                  0x0
+#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE__SHIFT                                               0x10
+#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE_MASK                                                    0x0000FFFFL
+#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE_MASK                                                 0xFFFF0000L
+//SPI_SX_SCOREBOARD_BUFFER_SIZES
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE__SHIFT                                          0x0
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE__SHIFT                                       0x10
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE_MASK                                            0x0000FFFFL
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE_MASK                                         0xFFFF0000L
+//SPI_CSQ_WF_ACTIVE_STATUS
+#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE_MASK                                                                 0xFFFFFFFFL
+//SPI_CSQ_WF_ACTIVE_COUNT_0
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK                                                                 0x000001FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__EVENTS_MASK                                                                0x01FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_1
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT_MASK                                                                 0x000001FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__EVENTS_MASK                                                                0x01FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_2
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT_MASK                                                                 0x000001FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__EVENTS_MASK                                                                0x01FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_3
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT_MASK                                                                 0x000001FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__EVENTS_MASK                                                                0x01FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_4
+#define SPI_CSQ_WF_ACTIVE_COUNT_4__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_4__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_4__COUNT_MASK                                                                 0x000001FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_4__EVENTS_MASK                                                                0x01FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_5
+#define SPI_CSQ_WF_ACTIVE_COUNT_5__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_5__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_5__COUNT_MASK                                                                 0x000001FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_5__EVENTS_MASK                                                                0x01FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_6
+#define SPI_CSQ_WF_ACTIVE_COUNT_6__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_6__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_6__COUNT_MASK                                                                 0x000001FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_6__EVENTS_MASK                                                                0x01FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_7
+#define SPI_CSQ_WF_ACTIVE_COUNT_7__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_7__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_7__COUNT_MASK                                                                 0x000001FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_7__EVENTS_MASK                                                                0x01FF0000L
+//SPI_LB_DATA_WAVES
+#define SPI_LB_DATA_WAVES__COUNT0__SHIFT                                                                      0x0
+#define SPI_LB_DATA_WAVES__COUNT1__SHIFT                                                                      0x10
+#define SPI_LB_DATA_WAVES__COUNT0_MASK                                                                        0x0000FFFFL
+#define SPI_LB_DATA_WAVES__COUNT1_MASK                                                                        0xFFFF0000L
+//SPI_LB_DATA_PERCU_WAVE_HSGS
+#define SPI_LB_DATA_PERCU_WAVE_HSGS__CU_USED_HS__SHIFT                                                        0x0
+#define SPI_LB_DATA_PERCU_WAVE_HSGS__CU_USED_GS__SHIFT                                                        0x10
+#define SPI_LB_DATA_PERCU_WAVE_HSGS__CU_USED_HS_MASK                                                          0x0000FFFFL
+#define SPI_LB_DATA_PERCU_WAVE_HSGS__CU_USED_GS_MASK                                                          0xFFFF0000L
+//SPI_LB_DATA_PERCU_WAVE_VSPS
+#define SPI_LB_DATA_PERCU_WAVE_VSPS__CU_USED_VS__SHIFT                                                        0x0
+#define SPI_LB_DATA_PERCU_WAVE_VSPS__CU_USED_PS__SHIFT                                                        0x10
+#define SPI_LB_DATA_PERCU_WAVE_VSPS__CU_USED_VS_MASK                                                          0x0000FFFFL
+#define SPI_LB_DATA_PERCU_WAVE_VSPS__CU_USED_PS_MASK                                                          0xFFFF0000L
+//SPI_LB_DATA_PERCU_WAVE_CS
+#define SPI_LB_DATA_PERCU_WAVE_CS__ACTIVE__SHIFT                                                              0x0
+#define SPI_LB_DATA_PERCU_WAVE_CS__ACTIVE_MASK                                                                0xFFFFL
+//SPIS_DEBUG_READ
+#define SPIS_DEBUG_READ__DATA__SHIFT                                                                          0x0
+#define SPIS_DEBUG_READ__DATA_MASK                                                                            0xFFFFFFFFL
+//BCI_DEBUG_READ
+#define BCI_DEBUG_READ__DATA__SHIFT                                                                           0x0
+#define BCI_DEBUG_READ__DATA_MASK                                                                             0xFFFFFFL
+//SPI_P0_TRAP_SCREEN_PSBA_LO
+#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P0_TRAP_SCREEN_PSBA_HI
+#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK                                                             0xFFL
+//SPI_P0_TRAP_SCREEN_PSMA_LO
+#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P0_TRAP_SCREEN_PSMA_HI
+#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK                                                             0xFFL
+//SPI_P0_TRAP_SCREEN_GPR_MIN
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT                                                           0x6
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK                                                             0x003FL
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK                                                             0x03C0L
+//SPI_P1_TRAP_SCREEN_PSBA_LO
+#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P1_TRAP_SCREEN_PSBA_HI
+#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK                                                             0xFFL
+//SPI_P1_TRAP_SCREEN_PSMA_LO
+#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P1_TRAP_SCREEN_PSMA_HI
+#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK                                                             0xFFL
+//SPI_P1_TRAP_SCREEN_GPR_MIN
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT                                                           0x6
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK                                                             0x003FL
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK                                                             0x03C0L
+
+
+// addressBlock: xcd0_gc_tpdec
+//TD_CNTL
+#define TD_CNTL__SYNC_PHASE_SH__SHIFT                                                                         0x0
+#define TD_CNTL__TD_IN_CAC_CHICKENBITS__SHIFT                                                                 0x2
+#define TD_CNTL__TD_OUT_CAC_CHICKENBITS__SHIFT                                                                0x6
+#define TD_CNTL__EXTEND_LDS_STALL__SHIFT                                                                      0x9
+#define TD_CNTL__LDS_STALL_PHASE_ADJUST__SHIFT                                                                0xb
+#define TD_CNTL__DISABLE_POWER_THROTTLE__SHIFT                                                                0x14
+#define TD_CNTL__ENABLE_ROUND_TO_ZERO__SHIFT                                                                  0x15
+#define TD_CNTL__DISABLE_2BIT_SIGNED_FORMAT__SHIFT                                                            0x17
+#define TD_CNTL__SRAM_FGCG_TD_KCLARR_LG__SHIFT                                                                0x1b
+#define TD_CNTL__RFGCG_CHICKEN__SHIFT                                                                         0x1c
+#define TD_CNTL__SYNC_PHASE_SH_MASK                                                                           0x00000003L
+#define TD_CNTL__TD_IN_CAC_CHICKENBITS_MASK                                                                   0x0000000CL
+#define TD_CNTL__TD_OUT_CAC_CHICKENBITS_MASK                                                                  0x000000C0L
+#define TD_CNTL__EXTEND_LDS_STALL_MASK                                                                        0x00000600L
+#define TD_CNTL__LDS_STALL_PHASE_ADJUST_MASK                                                                  0x00001800L
+#define TD_CNTL__DISABLE_POWER_THROTTLE_MASK                                                                  0x00100000L
+#define TD_CNTL__ENABLE_ROUND_TO_ZERO_MASK                                                                    0x00200000L
+#define TD_CNTL__DISABLE_2BIT_SIGNED_FORMAT_MASK                                                              0x00800000L
+#define TD_CNTL__SRAM_FGCG_TD_KCLARR_LG_MASK                                                                  0x08000000L
+#define TD_CNTL__RFGCG_CHICKEN_MASK                                                                           0x70000000L
+//TD_STATUS
+#define TD_STATUS__BUSY__SHIFT                                                                                0x1f
+#define TD_STATUS__BUSY_MASK                                                                                  0x80000000L
+//TD_POWER_CNTL
+#define TD_POWER_CNTL__MGCG_OUTPUTSTAGE__SHIFT                                                                0x1
+#define TD_POWER_CNTL__MID0_THREAD_DATA__SHIFT                                                                0x2
+#define TD_POWER_CNTL__MID2_ACCUM_DATA__SHIFT                                                                 0x3
+#define TD_POWER_CNTL__MGCG_OUTPUTSTAGE_MASK                                                                  0x00000002L
+#define TD_POWER_CNTL__MID0_THREAD_DATA_MASK                                                                  0x00000004L
+#define TD_POWER_CNTL__MID2_ACCUM_DATA_MASK                                                                   0x00000008L
+//TD_DSM_CNTL
+#define TD_DSM_CNTL__TD_SS_FIFO_LO_DSM_IRRITATOR_DATA__SHIFT                                                  0x0
+#define TD_DSM_CNTL__TD_SS_FIFO_LO_ENABLE_SINGLE_WRITE__SHIFT                                                 0x2
+#define TD_DSM_CNTL__TD_SS_FIFO_HI_DSM_IRRITATOR_DATA__SHIFT                                                  0x3
+#define TD_DSM_CNTL__TD_SS_FIFO_HI_ENABLE_SINGLE_WRITE__SHIFT                                                 0x5
+#define TD_DSM_CNTL__TD_CS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                                     0x6
+#define TD_DSM_CNTL__TD_CS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                                    0x8
+#define TD_DSM_CNTL__TD_SS_FIFO_LO_DSM_IRRITATOR_DATA_MASK                                                    0x00000003L
+#define TD_DSM_CNTL__TD_SS_FIFO_LO_ENABLE_SINGLE_WRITE_MASK                                                   0x00000004L
+#define TD_DSM_CNTL__TD_SS_FIFO_HI_DSM_IRRITATOR_DATA_MASK                                                    0x00000018L
+#define TD_DSM_CNTL__TD_SS_FIFO_HI_ENABLE_SINGLE_WRITE_MASK                                                   0x00000020L
+#define TD_DSM_CNTL__TD_CS_FIFO_DSM_IRRITATOR_DATA_MASK                                                       0x000000C0L
+#define TD_DSM_CNTL__TD_CS_FIFO_ENABLE_SINGLE_WRITE_MASK                                                      0x00000100L
+//TD_DSM_CNTL2
+#define TD_DSM_CNTL2__TD_SS_FIFO_LO_ENABLE_ERROR_INJECT__SHIFT                                                0x0
+#define TD_DSM_CNTL2__TD_SS_FIFO_LO_SELECT_INJECT_DELAY__SHIFT                                                0x2
+#define TD_DSM_CNTL2__TD_SS_FIFO_HI_ENABLE_ERROR_INJECT__SHIFT                                                0x3
+#define TD_DSM_CNTL2__TD_SS_FIFO_HI_SELECT_INJECT_DELAY__SHIFT                                                0x5
+#define TD_DSM_CNTL2__TD_CS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                                   0x6
+#define TD_DSM_CNTL2__TD_CS_FIFO_SELECT_INJECT_DELAY__SHIFT                                                   0x8
+#define TD_DSM_CNTL2__TD_INJECT_DELAY__SHIFT                                                                  0x1a
+#define TD_DSM_CNTL2__TD_SS_FIFO_LO_ENABLE_ERROR_INJECT_MASK                                                  0x00000003L
+#define TD_DSM_CNTL2__TD_SS_FIFO_LO_SELECT_INJECT_DELAY_MASK                                                  0x00000004L
+#define TD_DSM_CNTL2__TD_SS_FIFO_HI_ENABLE_ERROR_INJECT_MASK                                                  0x00000018L
+#define TD_DSM_CNTL2__TD_SS_FIFO_HI_SELECT_INJECT_DELAY_MASK                                                  0x00000020L
+#define TD_DSM_CNTL2__TD_CS_FIFO_ENABLE_ERROR_INJECT_MASK                                                     0x000000C0L
+#define TD_DSM_CNTL2__TD_CS_FIFO_SELECT_INJECT_DELAY_MASK                                                     0x00000100L
+#define TD_DSM_CNTL2__TD_INJECT_DELAY_MASK                                                                    0xFC000000L
+//TD_SCRATCH
+#define TD_SCRATCH__SCRATCH__SHIFT                                                                            0x0
+#define TD_SCRATCH__SCRATCH_MASK                                                                              0xFFFFFFFFL
+//TA_POWER_CNTL
+#define TA_POWER_CNTL__INPUT_CLK_EN_MODE__SHIFT                                                               0x0
+#define TA_POWER_CNTL__LOD_CLK_EN_MODE__SHIFT                                                                 0x1
+#define TA_POWER_CNTL__WDP_CLK_EN_MODE__SHIFT                                                                 0x2
+#define TA_POWER_CNTL__INPUT_CLK_EN_MODE_MASK                                                                 0x00000001L
+#define TA_POWER_CNTL__LOD_CLK_EN_MODE_MASK                                                                   0x00000002L
+#define TA_POWER_CNTL__WDP_CLK_EN_MODE_MASK                                                                   0x00000004L
+//TA_CNTL
+#define TA_CNTL__FX_XNACK_CREDIT__SHIFT                                                                       0x0
+#define TA_CNTL__SQ_XNACK_CREDIT__SHIFT                                                                       0x9
+#define TA_CNTL__TC_DATA_CREDIT__SHIFT                                                                        0xd
+#define TA_CNTL__ALIGNER_CREDIT__SHIFT                                                                        0x10
+#define TA_CNTL__TD_FIFO_CREDIT__SHIFT                                                                        0x16
+#define TA_CNTL__FX_XNACK_CREDIT_MASK                                                                         0x0000007FL
+#define TA_CNTL__SQ_XNACK_CREDIT_MASK                                                                         0x00001E00L
+#define TA_CNTL__TC_DATA_CREDIT_MASK                                                                          0x0000E000L
+#define TA_CNTL__ALIGNER_CREDIT_MASK                                                                          0x001F0000L
+#define TA_CNTL__TD_FIFO_CREDIT_MASK                                                                          0xFFC00000L
+//TA_CNTL_AUX
+#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N__SHIFT                                                                  0x0
+#define TA_CNTL_AUX__RESERVED__SHIFT                                                                          0x1
+#define TA_CNTL_AUX__TFAULT_EN_OVERRIDE__SHIFT                                                                0x5
+#define TA_CNTL_AUX__DISABLE_GATHER4_BC_SWIZZLE__SHIFT                                                        0x7
+#define TA_CNTL_AUX__DETERMINISM_RESERVED_DISABLE__SHIFT                                                      0x14
+#define TA_CNTL_AUX__DETERMINISM_OPCODE_STRICT_DISABLE__SHIFT                                                 0x15
+#define TA_CNTL_AUX__DETERMINISM_MISC_DISABLE__SHIFT                                                          0x16
+#define TA_CNTL_AUX__DETERMINISM_SAMPLE_C_DFMT_DISABLE__SHIFT                                                 0x17
+#define TA_CNTL_AUX__DETERMINISM_SAMPLER_MSAA_DISABLE__SHIFT                                                  0x18
+#define TA_CNTL_AUX__DETERMINISM_WRITEOP_READFMT_DISABLE__SHIFT                                               0x19
+#define TA_CNTL_AUX__DETERMINISM_DFMT_NFMT_DISABLE__SHIFT                                                     0x1a
+#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N_MASK                                                                    0x00000001L
+#define TA_CNTL_AUX__RESERVED_MASK                                                                            0x0000000EL
+#define TA_CNTL_AUX__TFAULT_EN_OVERRIDE_MASK                                                                  0x00000020L
+#define TA_CNTL_AUX__DISABLE_GATHER4_BC_SWIZZLE_MASK                                                          0x00000080L
+#define TA_CNTL_AUX__DETERMINISM_RESERVED_DISABLE_MASK                                                        0x00100000L
+#define TA_CNTL_AUX__DETERMINISM_OPCODE_STRICT_DISABLE_MASK                                                   0x00200000L
+#define TA_CNTL_AUX__DETERMINISM_MISC_DISABLE_MASK                                                            0x00400000L
+#define TA_CNTL_AUX__DETERMINISM_SAMPLE_C_DFMT_DISABLE_MASK                                                   0x00800000L
+#define TA_CNTL_AUX__DETERMINISM_SAMPLER_MSAA_DISABLE_MASK                                                    0x01000000L
+#define TA_CNTL_AUX__DETERMINISM_WRITEOP_READFMT_DISABLE_MASK                                                 0x02000000L
+#define TA_CNTL_AUX__DETERMINISM_DFMT_NFMT_DISABLE_MASK                                                       0x04000000L
+//TA_FEATURE_CNTL
+#define TA_FEATURE_CNTL__ATOMIC_COALESCING_EN__SHIFT                                                          0x4
+#define TA_FEATURE_CNTL__TA_ACFIFO_CHICKEN__SHIFT                                                             0xb
+#define TA_FEATURE_CNTL__TA_CAC_CHICKEN__SHIFT                                                                0xc
+#define TA_FEATURE_CNTL__AFIFO_SPLIT_CHICKEN__SHIFT                                                           0xd
+#define TA_FEATURE_CNTL__TA_DXFIFO_CHICKEN__SHIFT                                                             0xe
+#define TA_FEATURE_CNTL__ATOMIC_COALESCING_EN_MASK                                                            0x00000030L
+#define TA_FEATURE_CNTL__TA_ACFIFO_CHICKEN_MASK                                                               0x00000800L
+#define TA_FEATURE_CNTL__TA_CAC_CHICKEN_MASK                                                                  0x00001000L
+#define TA_FEATURE_CNTL__AFIFO_SPLIT_CHICKEN_MASK                                                             0x00002000L
+#define TA_FEATURE_CNTL__TA_DXFIFO_CHICKEN_MASK                                                               0x00004000L
+//TA_STATUS
+#define TA_STATUS__FG_PFIFO_EMPTYB__SHIFT                                                                     0xc
+#define TA_STATUS__FL_PFIFO_EMPTYB__SHIFT                                                                     0x10
+#define TA_STATUS__FA_PFIFO_EMPTYB__SHIFT                                                                     0x14
+#define TA_STATUS__IN_BUSY__SHIFT                                                                             0x18
+#define TA_STATUS__FG_BUSY__SHIFT                                                                             0x19
+#define TA_STATUS__TA_BUSY__SHIFT                                                                             0x1c
+#define TA_STATUS__FA_BUSY__SHIFT                                                                             0x1d
+#define TA_STATUS__AL_BUSY__SHIFT                                                                             0x1e
+#define TA_STATUS__BUSY__SHIFT                                                                                0x1f
+#define TA_STATUS__FG_PFIFO_EMPTYB_MASK                                                                       0x00001000L
+#define TA_STATUS__FL_PFIFO_EMPTYB_MASK                                                                       0x00010000L
+#define TA_STATUS__FA_PFIFO_EMPTYB_MASK                                                                       0x00100000L
+#define TA_STATUS__IN_BUSY_MASK                                                                               0x01000000L
+#define TA_STATUS__FG_BUSY_MASK                                                                               0x02000000L
+#define TA_STATUS__TA_BUSY_MASK                                                                               0x10000000L
+#define TA_STATUS__FA_BUSY_MASK                                                                               0x20000000L
+#define TA_STATUS__AL_BUSY_MASK                                                                               0x40000000L
+#define TA_STATUS__BUSY_MASK                                                                                  0x80000000L
+//TA_SCRATCH
+#define TA_SCRATCH__SCRATCH__SHIFT                                                                            0x0
+#define TA_SCRATCH__SCRATCH_MASK                                                                              0xFFFFFFFFL
+//TA_DSM_CNTL
+#define TA_DSM_CNTL__TA_FS_DFIFO_DSM_IRRITATOR_DATA__SHIFT                                                    0x0
+#define TA_DSM_CNTL__TA_FS_DFIFO_ENABLE_SINGLE_WRITE__SHIFT                                                   0x2
+#define TA_DSM_CNTL__TA_FX_LFIFO_DSM_IRRITATOR_DATA__SHIFT                                                    0x9
+#define TA_DSM_CNTL__TA_FX_LFIFO_ENABLE_SINGLE_WRITE__SHIFT                                                   0xb
+#define TA_DSM_CNTL__TA_FS_CFIFO_DSM_IRRITATOR_DATA__SHIFT                                                    0xc
+#define TA_DSM_CNTL__TA_FS_CFIFO_ENABLE_SINGLE_WRITE__SHIFT                                                   0xe
+#define TA_DSM_CNTL__TA_FS_AFIFO_LO_DSM_IRRITATOR_DATA__SHIFT                                                 0xf
+#define TA_DSM_CNTL__TA_FS_AFIFO_LO_ENABLE_SINGLE_WRITE__SHIFT                                                0x11
+#define TA_DSM_CNTL__TA_FS_AFIFO_HI_DSM_IRRITATOR_DATA__SHIFT                                                 0x12
+#define TA_DSM_CNTL__TA_FS_AFIFO_HI_ENABLE_SINGLE_WRITE__SHIFT                                                0x14
+#define TA_DSM_CNTL__TA_FS_DFIFO_DSM_IRRITATOR_DATA_MASK                                                      0x00000003L
+#define TA_DSM_CNTL__TA_FS_DFIFO_ENABLE_SINGLE_WRITE_MASK                                                     0x00000004L
+#define TA_DSM_CNTL__TA_FX_LFIFO_DSM_IRRITATOR_DATA_MASK                                                      0x00000600L
+#define TA_DSM_CNTL__TA_FX_LFIFO_ENABLE_SINGLE_WRITE_MASK                                                     0x00000800L
+#define TA_DSM_CNTL__TA_FS_CFIFO_DSM_IRRITATOR_DATA_MASK                                                      0x00003000L
+#define TA_DSM_CNTL__TA_FS_CFIFO_ENABLE_SINGLE_WRITE_MASK                                                     0x00004000L
+#define TA_DSM_CNTL__TA_FS_AFIFO_LO_DSM_IRRITATOR_DATA_MASK                                                   0x00018000L
+#define TA_DSM_CNTL__TA_FS_AFIFO_LO_ENABLE_SINGLE_WRITE_MASK                                                  0x00020000L
+#define TA_DSM_CNTL__TA_FS_AFIFO_HI_DSM_IRRITATOR_DATA_MASK                                                   0x000C0000L
+#define TA_DSM_CNTL__TA_FS_AFIFO_HI_ENABLE_SINGLE_WRITE_MASK                                                  0x00100000L
+//TA_DSM_CNTL2
+#define TA_DSM_CNTL2__TA_FS_DFIFO_ENABLE_ERROR_INJECT__SHIFT                                                  0x0
+#define TA_DSM_CNTL2__TA_FS_DFIFO_SELECT_INJECT_DELAY__SHIFT                                                  0x2
+#define TA_DSM_CNTL2__TA_FX_LFIFO_ENABLE_ERROR_INJECT__SHIFT                                                  0x9
+#define TA_DSM_CNTL2__TA_FX_LFIFO_SELECT_INJECT_DELAY__SHIFT                                                  0xb
+#define TA_DSM_CNTL2__TA_FS_CFIFO_ENABLE_ERROR_INJECT__SHIFT                                                  0xc
+#define TA_DSM_CNTL2__TA_FS_CFIFO_SELECT_INJECT_DELAY__SHIFT                                                  0xe
+#define TA_DSM_CNTL2__TA_FS_AFIFO_LO_ENABLE_ERROR_INJECT__SHIFT                                               0xf
+#define TA_DSM_CNTL2__TA_FS_AFIFO_LO_SELECT_INJECT_DELAY__SHIFT                                               0x11
+#define TA_DSM_CNTL2__TA_FS_AFIFO_HI_ENABLE_ERROR_INJECT__SHIFT                                               0x12
+#define TA_DSM_CNTL2__TA_FS_AFIFO_HI_SELECT_INJECT_DELAY__SHIFT                                               0x14
+#define TA_DSM_CNTL2__TA_INJECT_DELAY__SHIFT                                                                  0x1a
+#define TA_DSM_CNTL2__TA_FS_DFIFO_ENABLE_ERROR_INJECT_MASK                                                    0x00000003L
+#define TA_DSM_CNTL2__TA_FS_DFIFO_SELECT_INJECT_DELAY_MASK                                                    0x00000004L
+#define TA_DSM_CNTL2__TA_FX_LFIFO_ENABLE_ERROR_INJECT_MASK                                                    0x00000600L
+#define TA_DSM_CNTL2__TA_FX_LFIFO_SELECT_INJECT_DELAY_MASK                                                    0x00000800L
+#define TA_DSM_CNTL2__TA_FS_CFIFO_ENABLE_ERROR_INJECT_MASK                                                    0x00003000L
+#define TA_DSM_CNTL2__TA_FS_CFIFO_SELECT_INJECT_DELAY_MASK                                                    0x00004000L
+#define TA_DSM_CNTL2__TA_FS_AFIFO_LO_ENABLE_ERROR_INJECT_MASK                                                 0x00018000L
+#define TA_DSM_CNTL2__TA_FS_AFIFO_LO_SELECT_INJECT_DELAY_MASK                                                 0x00020000L
+#define TA_DSM_CNTL2__TA_FS_AFIFO_HI_ENABLE_ERROR_INJECT_MASK                                                 0x000C0000L
+#define TA_DSM_CNTL2__TA_FS_AFIFO_HI_SELECT_INJECT_DELAY_MASK                                                 0x00100000L
+#define TA_DSM_CNTL2__TA_INJECT_DELAY_MASK                                                                    0xFC000000L
+
+
+// addressBlock: xcd0_gc_gdsdec
+//GDS_CONFIG
+#define GDS_CONFIG__WRITE_DIS__SHIFT                                                                          0x0
+#define GDS_CONFIG__SH0_GPR_PHASE_SEL__SHIFT                                                                  0x1
+#define GDS_CONFIG__SH1_GPR_PHASE_SEL__SHIFT                                                                  0x3
+#define GDS_CONFIG__SH2_GPR_PHASE_SEL__SHIFT                                                                  0x5
+#define GDS_CONFIG__SH3_GPR_PHASE_SEL__SHIFT                                                                  0x7
+#define GDS_CONFIG__SH4_GPR_PHASE_SEL__SHIFT                                                                  0x9
+#define GDS_CONFIG__SH5_GPR_PHASE_SEL__SHIFT                                                                  0xb
+#define GDS_CONFIG__SH6_GPR_PHASE_SEL__SHIFT                                                                  0xd
+#define GDS_CONFIG__SH7_GPR_PHASE_SEL__SHIFT                                                                  0xf
+#define GDS_CONFIG__WRITE_DIS_MASK                                                                            0x00000001L
+#define GDS_CONFIG__SH0_GPR_PHASE_SEL_MASK                                                                    0x00000006L
+#define GDS_CONFIG__SH1_GPR_PHASE_SEL_MASK                                                                    0x00000018L
+#define GDS_CONFIG__SH2_GPR_PHASE_SEL_MASK                                                                    0x00000060L
+#define GDS_CONFIG__SH3_GPR_PHASE_SEL_MASK                                                                    0x00000180L
+#define GDS_CONFIG__SH4_GPR_PHASE_SEL_MASK                                                                    0x00000600L
+#define GDS_CONFIG__SH5_GPR_PHASE_SEL_MASK                                                                    0x00001800L
+#define GDS_CONFIG__SH6_GPR_PHASE_SEL_MASK                                                                    0x00006000L
+#define GDS_CONFIG__SH7_GPR_PHASE_SEL_MASK                                                                    0x00018000L
+//GDS_CNTL_STATUS
+#define GDS_CNTL_STATUS__GDS_BUSY__SHIFT                                                                      0x0
+#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY__SHIFT                                                                0x1
+#define GDS_CNTL_STATUS__ORD_APP_BUSY__SHIFT                                                                  0x2
+#define GDS_CNTL_STATUS__DS_BANK_CONFLICT__SHIFT                                                              0x3
+#define GDS_CNTL_STATUS__DS_ADDR_CONFLICT__SHIFT                                                              0x4
+#define GDS_CNTL_STATUS__DS_WR_CLAMP__SHIFT                                                                   0x5
+#define GDS_CNTL_STATUS__DS_RD_CLAMP__SHIFT                                                                   0x6
+#define GDS_CNTL_STATUS__GRBM_RBUF_BUSY__SHIFT                                                                0x7
+#define GDS_CNTL_STATUS__DS_BUSY__SHIFT                                                                       0x8
+#define GDS_CNTL_STATUS__GWS_BUSY__SHIFT                                                                      0x9
+#define GDS_CNTL_STATUS__ORD_FIFO_BUSY__SHIFT                                                                 0xa
+#define GDS_CNTL_STATUS__CREDIT_BUSY0__SHIFT                                                                  0xb
+#define GDS_CNTL_STATUS__CREDIT_BUSY1__SHIFT                                                                  0xc
+#define GDS_CNTL_STATUS__CREDIT_BUSY2__SHIFT                                                                  0xd
+#define GDS_CNTL_STATUS__CREDIT_BUSY3__SHIFT                                                                  0xe
+#define GDS_CNTL_STATUS__CREDIT_BUSY4__SHIFT                                                                  0xf
+#define GDS_CNTL_STATUS__CREDIT_BUSY5__SHIFT                                                                  0x10
+#define GDS_CNTL_STATUS__CREDIT_BUSY6__SHIFT                                                                  0x11
+#define GDS_CNTL_STATUS__CREDIT_BUSY7__SHIFT                                                                  0x12
+#define GDS_CNTL_STATUS__GDS_BUSY_MASK                                                                        0x00000001L
+#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY_MASK                                                                  0x00000002L
+#define GDS_CNTL_STATUS__ORD_APP_BUSY_MASK                                                                    0x00000004L
+#define GDS_CNTL_STATUS__DS_BANK_CONFLICT_MASK                                                                0x00000008L
+#define GDS_CNTL_STATUS__DS_ADDR_CONFLICT_MASK                                                                0x00000010L
+#define GDS_CNTL_STATUS__DS_WR_CLAMP_MASK                                                                     0x00000020L
+#define GDS_CNTL_STATUS__DS_RD_CLAMP_MASK                                                                     0x00000040L
+#define GDS_CNTL_STATUS__GRBM_RBUF_BUSY_MASK                                                                  0x00000080L
+#define GDS_CNTL_STATUS__DS_BUSY_MASK                                                                         0x00000100L
+#define GDS_CNTL_STATUS__GWS_BUSY_MASK                                                                        0x00000200L
+#define GDS_CNTL_STATUS__ORD_FIFO_BUSY_MASK                                                                   0x00000400L
+#define GDS_CNTL_STATUS__CREDIT_BUSY0_MASK                                                                    0x00000800L
+#define GDS_CNTL_STATUS__CREDIT_BUSY1_MASK                                                                    0x00001000L
+#define GDS_CNTL_STATUS__CREDIT_BUSY2_MASK                                                                    0x00002000L
+#define GDS_CNTL_STATUS__CREDIT_BUSY3_MASK                                                                    0x00004000L
+#define GDS_CNTL_STATUS__CREDIT_BUSY4_MASK                                                                    0x00008000L
+#define GDS_CNTL_STATUS__CREDIT_BUSY5_MASK                                                                    0x00010000L
+#define GDS_CNTL_STATUS__CREDIT_BUSY6_MASK                                                                    0x00020000L
+#define GDS_CNTL_STATUS__CREDIT_BUSY7_MASK                                                                    0x00040000L
+//GDS_ENHANCE2
+#define GDS_ENHANCE2__MISC__SHIFT                                                                             0x0
+#define GDS_ENHANCE2__GDS_TD_INTERFACES_FGCG_OVERRIDE__SHIFT                                                  0x10
+#define GDS_ENHANCE2__GDS_PHY_CMD_RAM_FGCG_OVERRIDE__SHIFT                                                    0x11
+#define GDS_ENHANCE2__GDS_FED_IN_PROPAGATE__SHIFT                                                             0x12
+#define GDS_ENHANCE2__UNUSED__SHIFT                                                                           0x13
+#define GDS_ENHANCE2__MISC_MASK                                                                               0x0000FFFFL
+#define GDS_ENHANCE2__GDS_TD_INTERFACES_FGCG_OVERRIDE_MASK                                                    0x00010000L
+#define GDS_ENHANCE2__GDS_PHY_CMD_RAM_FGCG_OVERRIDE_MASK                                                      0x00020000L
+#define GDS_ENHANCE2__GDS_FED_IN_PROPAGATE_MASK                                                               0x00040000L
+#define GDS_ENHANCE2__UNUSED_MASK                                                                             0xFFF80000L
+//GDS_PROTECTION_FAULT
+#define GDS_PROTECTION_FAULT__WRITE_DIS__SHIFT                                                                0x0
+#define GDS_PROTECTION_FAULT__FAULT_DETECTED__SHIFT                                                           0x1
+#define GDS_PROTECTION_FAULT__GRBM__SHIFT                                                                     0x2
+#define GDS_PROTECTION_FAULT__SH_ID__SHIFT                                                                    0x3
+#define GDS_PROTECTION_FAULT__CU_ID__SHIFT                                                                    0x6
+#define GDS_PROTECTION_FAULT__SIMD_ID__SHIFT                                                                  0xa
+#define GDS_PROTECTION_FAULT__WAVE_ID__SHIFT                                                                  0xc
+#define GDS_PROTECTION_FAULT__ADDRESS__SHIFT                                                                  0x10
+#define GDS_PROTECTION_FAULT__WRITE_DIS_MASK                                                                  0x00000001L
+#define GDS_PROTECTION_FAULT__FAULT_DETECTED_MASK                                                             0x00000002L
+#define GDS_PROTECTION_FAULT__GRBM_MASK                                                                       0x00000004L
+#define GDS_PROTECTION_FAULT__SH_ID_MASK                                                                      0x00000038L
+#define GDS_PROTECTION_FAULT__CU_ID_MASK                                                                      0x000003C0L
+#define GDS_PROTECTION_FAULT__SIMD_ID_MASK                                                                    0x00000C00L
+#define GDS_PROTECTION_FAULT__WAVE_ID_MASK                                                                    0x0000F000L
+#define GDS_PROTECTION_FAULT__ADDRESS_MASK                                                                    0xFFFF0000L
+//GDS_VM_PROTECTION_FAULT
+#define GDS_VM_PROTECTION_FAULT__WRITE_DIS__SHIFT                                                             0x0
+#define GDS_VM_PROTECTION_FAULT__FAULT_DETECTED__SHIFT                                                        0x1
+#define GDS_VM_PROTECTION_FAULT__GWS__SHIFT                                                                   0x2
+#define GDS_VM_PROTECTION_FAULT__OA__SHIFT                                                                    0x3
+#define GDS_VM_PROTECTION_FAULT__GRBM__SHIFT                                                                  0x4
+#define GDS_VM_PROTECTION_FAULT__TMZ__SHIFT                                                                   0x5
+#define GDS_VM_PROTECTION_FAULT__VMID__SHIFT                                                                  0x8
+#define GDS_VM_PROTECTION_FAULT__ADDRESS__SHIFT                                                               0x10
+#define GDS_VM_PROTECTION_FAULT__WRITE_DIS_MASK                                                               0x00000001L
+#define GDS_VM_PROTECTION_FAULT__FAULT_DETECTED_MASK                                                          0x00000002L
+#define GDS_VM_PROTECTION_FAULT__GWS_MASK                                                                     0x00000004L
+#define GDS_VM_PROTECTION_FAULT__OA_MASK                                                                      0x00000008L
+#define GDS_VM_PROTECTION_FAULT__GRBM_MASK                                                                    0x00000010L
+#define GDS_VM_PROTECTION_FAULT__TMZ_MASK                                                                     0x00000020L
+#define GDS_VM_PROTECTION_FAULT__VMID_MASK                                                                    0x00000F00L
+#define GDS_VM_PROTECTION_FAULT__ADDRESS_MASK                                                                 0xFFFF0000L
+//GDS_EDC_CNT
+#define GDS_EDC_CNT__GDS_MEM_DED__SHIFT                                                                       0x0
+#define GDS_EDC_CNT__GDS_MEM_SEC__SHIFT                                                                       0x4
+#define GDS_EDC_CNT__UNUSED__SHIFT                                                                            0x6
+#define GDS_EDC_CNT__GDS_MEM_DED_MASK                                                                         0x00000003L
+#define GDS_EDC_CNT__GDS_MEM_SEC_MASK                                                                         0x00000030L
+#define GDS_EDC_CNT__UNUSED_MASK                                                                              0xFFFFFFC0L
+//GDS_EDC_GRBM_CNT
+#define GDS_EDC_GRBM_CNT__DED__SHIFT                                                                          0x0
+#define GDS_EDC_GRBM_CNT__SEC__SHIFT                                                                          0x2
+#define GDS_EDC_GRBM_CNT__UNUSED__SHIFT                                                                       0x4
+#define GDS_EDC_GRBM_CNT__DED_MASK                                                                            0x00000003L
+#define GDS_EDC_GRBM_CNT__SEC_MASK                                                                            0x0000000CL
+#define GDS_EDC_GRBM_CNT__UNUSED_MASK                                                                         0xFFFFFFF0L
+//GDS_EDC_OA_DED
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_PIX_DED__SHIFT                                                            0x0
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_VTX_DED__SHIFT                                                            0x1
+#define GDS_EDC_OA_DED__ME0_CS_DED__SHIFT                                                                     0x2
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_GS_DED__SHIFT                                                             0x3
+#define GDS_EDC_OA_DED__ME1_PIPE0_DED__SHIFT                                                                  0x4
+#define GDS_EDC_OA_DED__ME1_PIPE1_DED__SHIFT                                                                  0x5
+#define GDS_EDC_OA_DED__ME1_PIPE2_DED__SHIFT                                                                  0x6
+#define GDS_EDC_OA_DED__ME1_PIPE3_DED__SHIFT                                                                  0x7
+#define GDS_EDC_OA_DED__ME2_PIPE0_DED__SHIFT                                                                  0x8
+#define GDS_EDC_OA_DED__ME2_PIPE1_DED__SHIFT                                                                  0x9
+#define GDS_EDC_OA_DED__ME2_PIPE2_DED__SHIFT                                                                  0xa
+#define GDS_EDC_OA_DED__ME2_PIPE3_DED__SHIFT                                                                  0xb
+#define GDS_EDC_OA_DED__UNUSED1__SHIFT                                                                        0xc
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_PIX_DED_MASK                                                              0x00000001L
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_VTX_DED_MASK                                                              0x00000002L
+#define GDS_EDC_OA_DED__ME0_CS_DED_MASK                                                                       0x00000004L
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_GS_DED_MASK                                                               0x00000008L
+#define GDS_EDC_OA_DED__ME1_PIPE0_DED_MASK                                                                    0x00000010L
+#define GDS_EDC_OA_DED__ME1_PIPE1_DED_MASK                                                                    0x00000020L
+#define GDS_EDC_OA_DED__ME1_PIPE2_DED_MASK                                                                    0x00000040L
+#define GDS_EDC_OA_DED__ME1_PIPE3_DED_MASK                                                                    0x00000080L
+#define GDS_EDC_OA_DED__ME2_PIPE0_DED_MASK                                                                    0x00000100L
+#define GDS_EDC_OA_DED__ME2_PIPE1_DED_MASK                                                                    0x00000200L
+#define GDS_EDC_OA_DED__ME2_PIPE2_DED_MASK                                                                    0x00000400L
+#define GDS_EDC_OA_DED__ME2_PIPE3_DED_MASK                                                                    0x00000800L
+#define GDS_EDC_OA_DED__UNUSED1_MASK                                                                          0xFFFFF000L
+//GDS_DSM_CNTL
+#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_0__SHIFT                                                 0x0
+#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_1__SHIFT                                                 0x1
+#define GDS_DSM_CNTL__GDS_MEM_ENABLE_SINGLE_WRITE__SHIFT                                                      0x2
+#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_0__SHIFT                                         0x3
+#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_1__SHIFT                                         0x4
+#define GDS_DSM_CNTL__GDS_INPUT_QUEUE_ENABLE_SINGLE_WRITE__SHIFT                                              0x5
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_0__SHIFT                                         0x6
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_1__SHIFT                                         0x7
+#define GDS_DSM_CNTL__GDS_PHY_CMD_RAM_ENABLE_SINGLE_WRITE__SHIFT                                              0x8
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_0__SHIFT                                        0x9
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_1__SHIFT                                        0xa
+#define GDS_DSM_CNTL__GDS_PHY_DATA_RAM_ENABLE_SINGLE_WRITE__SHIFT                                             0xb
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_0__SHIFT                                            0xc
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_1__SHIFT                                            0xd
+#define GDS_DSM_CNTL__GDS_PIPE_MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xe
+#define GDS_DSM_CNTL__UNUSED__SHIFT                                                                           0xf
+#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_0_MASK                                                   0x00000001L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_1_MASK                                                   0x00000002L
+#define GDS_DSM_CNTL__GDS_MEM_ENABLE_SINGLE_WRITE_MASK                                                        0x00000004L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_0_MASK                                           0x00000008L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_1_MASK                                           0x00000010L
+#define GDS_DSM_CNTL__GDS_INPUT_QUEUE_ENABLE_SINGLE_WRITE_MASK                                                0x00000020L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_0_MASK                                           0x00000040L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_1_MASK                                           0x00000080L
+#define GDS_DSM_CNTL__GDS_PHY_CMD_RAM_ENABLE_SINGLE_WRITE_MASK                                                0x00000100L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_0_MASK                                          0x00000200L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_1_MASK                                          0x00000400L
+#define GDS_DSM_CNTL__GDS_PHY_DATA_RAM_ENABLE_SINGLE_WRITE_MASK                                               0x00000800L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_0_MASK                                              0x00001000L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_1_MASK                                              0x00002000L
+#define GDS_DSM_CNTL__GDS_PIPE_MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00004000L
+#define GDS_DSM_CNTL__UNUSED_MASK                                                                             0xFFFF8000L
+//GDS_EDC_OA_PHY_CNT
+#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_SEC__SHIFT                                                        0x0
+#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_DED__SHIFT                                                        0x2
+#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_SEC__SHIFT                                                        0x4
+#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_DED__SHIFT                                                        0x6
+#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_SEC__SHIFT                                                       0x8
+#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_DED__SHIFT                                                       0xa
+#define GDS_EDC_OA_PHY_CNT__UNUSED1__SHIFT                                                                    0xc
+#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_SEC_MASK                                                          0x00000003L
+#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_DED_MASK                                                          0x0000000CL
+#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_SEC_MASK                                                          0x00000030L
+#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_DED_MASK                                                          0x000000C0L
+#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_SEC_MASK                                                         0x00000300L
+#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_DED_MASK                                                         0x00000C00L
+#define GDS_EDC_OA_PHY_CNT__UNUSED1_MASK                                                                      0xFFFFF000L
+//GDS_EDC_OA_PIPE_CNT
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_SEC__SHIFT                                                    0x0
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_DED__SHIFT                                                    0x2
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_SEC__SHIFT                                                    0x4
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_DED__SHIFT                                                    0x6
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_SEC__SHIFT                                                    0x8
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_DED__SHIFT                                                    0xa
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_SEC__SHIFT                                                    0xc
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_DED__SHIFT                                                    0xe
+#define GDS_EDC_OA_PIPE_CNT__UNUSED__SHIFT                                                                    0x10
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_SEC_MASK                                                      0x00000003L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_DED_MASK                                                      0x0000000CL
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_SEC_MASK                                                      0x00000030L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_DED_MASK                                                      0x000000C0L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_SEC_MASK                                                      0x00000300L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_DED_MASK                                                      0x00000C00L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_SEC_MASK                                                      0x00003000L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_DED_MASK                                                      0x0000C000L
+#define GDS_EDC_OA_PIPE_CNT__UNUSED_MASK                                                                      0xFFFF0000L
+//GDS_DSM_CNTL2
+#define GDS_DSM_CNTL2__GDS_MEM_ENABLE_ERROR_INJECT__SHIFT                                                     0x0
+#define GDS_DSM_CNTL2__GDS_MEM_SELECT_INJECT_DELAY__SHIFT                                                     0x2
+#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_ENABLE_ERROR_INJECT__SHIFT                                             0x3
+#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_SELECT_INJECT_DELAY__SHIFT                                             0x5
+#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_ENABLE_ERROR_INJECT__SHIFT                                             0x6
+#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_SELECT_INJECT_DELAY__SHIFT                                             0x8
+#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_ENABLE_ERROR_INJECT__SHIFT                                            0x9
+#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_SELECT_INJECT_DELAY__SHIFT                                            0xb
+#define GDS_DSM_CNTL2__GDS_PIPE_MEM_ENABLE_ERROR_INJECT__SHIFT                                                0xc
+#define GDS_DSM_CNTL2__GDS_PIPE_MEM_SELECT_INJECT_DELAY__SHIFT                                                0xe
+#define GDS_DSM_CNTL2__UNUSED__SHIFT                                                                          0xf
+#define GDS_DSM_CNTL2__GDS_INJECT_DELAY__SHIFT                                                                0x1a
+#define GDS_DSM_CNTL2__GDS_MEM_ENABLE_ERROR_INJECT_MASK                                                       0x00000003L
+#define GDS_DSM_CNTL2__GDS_MEM_SELECT_INJECT_DELAY_MASK                                                       0x00000004L
+#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_ENABLE_ERROR_INJECT_MASK                                               0x00000018L
+#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_SELECT_INJECT_DELAY_MASK                                               0x00000020L
+#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_ENABLE_ERROR_INJECT_MASK                                               0x000000C0L
+#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_SELECT_INJECT_DELAY_MASK                                               0x00000100L
+#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_ENABLE_ERROR_INJECT_MASK                                              0x00000600L
+#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_SELECT_INJECT_DELAY_MASK                                              0x00000800L
+#define GDS_DSM_CNTL2__GDS_PIPE_MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00003000L
+#define GDS_DSM_CNTL2__GDS_PIPE_MEM_SELECT_INJECT_DELAY_MASK                                                  0x00004000L
+#define GDS_DSM_CNTL2__UNUSED_MASK                                                                            0x03FF8000L
+#define GDS_DSM_CNTL2__GDS_INJECT_DELAY_MASK                                                                  0xFC000000L
+//GDS_WD_GDS_CSB
+#define GDS_WD_GDS_CSB__COUNTER__SHIFT                                                                        0x0
+#define GDS_WD_GDS_CSB__UNUSED__SHIFT                                                                         0xd
+#define GDS_WD_GDS_CSB__COUNTER_MASK                                                                          0x00001FFFL
+#define GDS_WD_GDS_CSB__UNUSED_MASK                                                                           0xFFFFE000L
+
+
+// addressBlock: xcd0_gc_rbdec
+//DB_DEBUG
+#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE__SHIFT                                                       0x0
+#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE__SHIFT                                                         0x1
+#define DB_DEBUG__FETCH_FULL_Z_TILE__SHIFT                                                                    0x2
+#define DB_DEBUG__FETCH_FULL_STENCIL_TILE__SHIFT                                                              0x3
+#define DB_DEBUG__FORCE_Z_MODE__SHIFT                                                                         0x4
+#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ__SHIFT                                                               0x6
+#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ__SHIFT                                                             0x7
+#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE__SHIFT                                                               0x8
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0__SHIFT                                                              0xa
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1__SHIFT                                                              0xc
+#define DB_DEBUG__DEBUG_FAST_Z_DISABLE__SHIFT                                                                 0xe
+#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE__SHIFT                                                           0xf
+#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE__SHIFT                                                              0x10
+#define DB_DEBUG__DISABLE_SUMM_SQUADS__SHIFT                                                                  0x11
+#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS__SHIFT                                                               0x12
+#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE__SHIFT                                                             0x13
+#define DB_DEBUG__NEVER_FREE_Z_ONLY__SHIFT                                                                    0x15
+#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS__SHIFT                                                0x16
+#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION__SHIFT                                                    0x17
+#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES__SHIFT                                                           0x18
+#define DB_DEBUG__ONE_FREE_IN_FLIGHT__SHIFT                                                                   0x1c
+#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT__SHIFT                                                           0x1d
+#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC__SHIFT                                                           0x1e
+#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC__SHIFT                                                           0x1f
+#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE_MASK                                                         0x00000001L
+#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE_MASK                                                           0x00000002L
+#define DB_DEBUG__FETCH_FULL_Z_TILE_MASK                                                                      0x00000004L
+#define DB_DEBUG__FETCH_FULL_STENCIL_TILE_MASK                                                                0x00000008L
+#define DB_DEBUG__FORCE_Z_MODE_MASK                                                                           0x00000030L
+#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ_MASK                                                                 0x00000040L
+#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ_MASK                                                               0x00000080L
+#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE_MASK                                                                 0x00000300L
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0_MASK                                                                0x00000C00L
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1_MASK                                                                0x00003000L
+#define DB_DEBUG__DEBUG_FAST_Z_DISABLE_MASK                                                                   0x00004000L
+#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE_MASK                                                             0x00008000L
+#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE_MASK                                                                0x00010000L
+#define DB_DEBUG__DISABLE_SUMM_SQUADS_MASK                                                                    0x00020000L
+#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS_MASK                                                                 0x00040000L
+#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE_MASK                                                               0x00180000L
+#define DB_DEBUG__NEVER_FREE_Z_ONLY_MASK                                                                      0x00200000L
+#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS_MASK                                                  0x00400000L
+#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION_MASK                                                      0x00800000L
+#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES_MASK                                                             0x0F000000L
+#define DB_DEBUG__ONE_FREE_IN_FLIGHT_MASK                                                                     0x10000000L
+#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT_MASK                                                             0x20000000L
+#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC_MASK                                                             0x40000000L
+#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC_MASK                                                             0x80000000L
+//DB_DEBUG2
+#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING__SHIFT                                                            0x0
+#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE__SHIFT                                                          0x1
+#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE__SHIFT                                                            0x2
+#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB__SHIFT                                                                 0x3
+#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM__SHIFT                                                        0x4
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL__SHIFT                                                            0x5
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_REZ__SHIFT                                                        0x6
+#define DB_DEBUG2__ENABLE_VIEWPORT_STALL_ON_ALL__SHIFT                                                        0x7
+#define DB_DEBUG2__OPTIMIZE_HIZ_MATCHES_FB_DISABLE__SHIFT                                                     0x8
+#define DB_DEBUG2__CLK_OFF_DELAY__SHIFT                                                                       0x9
+#define DB_DEBUG2__DISABLE_TILE_COVERED_FOR_PS_ITER__SHIFT                                                    0xe
+#define DB_DEBUG2__ENABLE_SUBTILE_GROUPING__SHIFT                                                             0xf
+#define DB_DEBUG2__RESERVED__SHIFT                                                                            0x10
+#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING__SHIFT                                                         0x11
+#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING__SHIFT                                                         0x12
+#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL__SHIFT                                                        0x13
+#define DB_DEBUG2__DISABLE_VR_OBJ_PRIM_ID__SHIFT                                                              0x1a
+#define DB_DEBUG2__DISABLE_VR_PS_INVOKE__SHIFT                                                                0x1b
+#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM__SHIFT                                                             0x1c
+#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL__SHIFT                                                        0x1d
+#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM__SHIFT                                                    0x1e
+#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT__SHIFT                                                0x1f
+#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING_MASK                                                              0x00000001L
+#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE_MASK                                                            0x00000002L
+#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE_MASK                                                              0x00000004L
+#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB_MASK                                                                   0x00000008L
+#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM_MASK                                                          0x00000010L
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_MASK                                                              0x00000020L
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_REZ_MASK                                                          0x00000040L
+#define DB_DEBUG2__ENABLE_VIEWPORT_STALL_ON_ALL_MASK                                                          0x00000080L
+#define DB_DEBUG2__OPTIMIZE_HIZ_MATCHES_FB_DISABLE_MASK                                                       0x00000100L
+#define DB_DEBUG2__CLK_OFF_DELAY_MASK                                                                         0x00003E00L
+#define DB_DEBUG2__DISABLE_TILE_COVERED_FOR_PS_ITER_MASK                                                      0x00004000L
+#define DB_DEBUG2__ENABLE_SUBTILE_GROUPING_MASK                                                               0x00008000L
+#define DB_DEBUG2__RESERVED_MASK                                                                              0x00010000L
+#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING_MASK                                                           0x00020000L
+#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING_MASK                                                           0x00040000L
+#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL_MASK                                                          0x00080000L
+#define DB_DEBUG2__DISABLE_VR_OBJ_PRIM_ID_MASK                                                                0x04000000L
+#define DB_DEBUG2__DISABLE_VR_PS_INVOKE_MASK                                                                  0x08000000L
+#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM_MASK                                                               0x10000000L
+#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL_MASK                                                          0x20000000L
+#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM_MASK                                                      0x40000000L
+#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT_MASK                                                  0x80000000L
+//DB_DEBUG3
+#define DB_DEBUG3__DISABLE_CLEAR_ZRANGE_CORRECTION__SHIFT                                                     0x0
+#define DB_DEBUG3__ROUND_ZRANGE_CORRECTION__SHIFT                                                             0x1
+#define DB_DEBUG3__FORCE_DB_IS_GOOD__SHIFT                                                                    0x2
+#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION__SHIFT                                                     0x3
+#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP__SHIFT                                                          0x4
+#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z__SHIFT                                                             0x5
+#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z__SHIFT                                                              0x6
+#define DB_DEBUG3__DISABLE_TCP_CAM_BYPASS__SHIFT                                                              0x7
+#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION__SHIFT                                                      0x8
+#define DB_DEBUG3__DISABLE_REDUNDANT_PLANE_FLUSHES_OPT__SHIFT                                                 0x9
+#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP__SHIFT                                            0xa
+#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS__SHIFT                                                        0xb
+#define DB_DEBUG3__DISABLE_OP_Z_DATA_FORWARDING__SHIFT                                                        0xc
+#define DB_DEBUG3__DISABLE_OP_DF_BYPASS__SHIFT                                                                0xd
+#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE__SHIFT                                                         0xe
+#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK__SHIFT                                                       0xf
+#define DB_DEBUG3__ALLOW_RF2P_RW_COLLISION__SHIFT                                                             0x10
+#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE__SHIFT                                                         0x11
+#define DB_DEBUG3__DISABLE_OP_S_DATA_FORWARDING__SHIFT                                                        0x12
+#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE__SHIFT                                                     0x13
+#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE__SHIFT                                                         0x14
+#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT__SHIFT                                                0x15
+#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB__SHIFT                                                        0x16
+#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD__SHIFT                                                  0x17
+#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT__SHIFT                                                           0x18
+#define DB_DEBUG3__DISABLE_DI_DT_STALL__SHIFT                                                                 0x19
+#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET__SHIFT                                                             0x1a
+#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX__SHIFT                                                       0x1b
+#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND__SHIFT                                                         0x1c
+#define DB_DEBUG3__DONT_DELETE_CONTEXT_SUSPEND__SHIFT                                                         0x1d
+#define DB_DEBUG3__DISABLE_4XAA_2P_DELAYED_WRITE__SHIFT                                                       0x1e
+#define DB_DEBUG3__DISABLE_4XAA_2P_INTERLEAVED_PMASK__SHIFT                                                   0x1f
+#define DB_DEBUG3__DISABLE_CLEAR_ZRANGE_CORRECTION_MASK                                                       0x00000001L
+#define DB_DEBUG3__ROUND_ZRANGE_CORRECTION_MASK                                                               0x00000002L
+#define DB_DEBUG3__FORCE_DB_IS_GOOD_MASK                                                                      0x00000004L
+#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION_MASK                                                       0x00000008L
+#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP_MASK                                                            0x00000010L
+#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z_MASK                                                               0x00000020L
+#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z_MASK                                                                0x00000040L
+#define DB_DEBUG3__DISABLE_TCP_CAM_BYPASS_MASK                                                                0x00000080L
+#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION_MASK                                                        0x00000100L
+#define DB_DEBUG3__DISABLE_REDUNDANT_PLANE_FLUSHES_OPT_MASK                                                   0x00000200L
+#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP_MASK                                              0x00000400L
+#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS_MASK                                                          0x00000800L
+#define DB_DEBUG3__DISABLE_OP_Z_DATA_FORWARDING_MASK                                                          0x00001000L
+#define DB_DEBUG3__DISABLE_OP_DF_BYPASS_MASK                                                                  0x00002000L
+#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE_MASK                                                           0x00004000L
+#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK_MASK                                                         0x00008000L
+#define DB_DEBUG3__ALLOW_RF2P_RW_COLLISION_MASK                                                               0x00010000L
+#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE_MASK                                                           0x00020000L
+#define DB_DEBUG3__DISABLE_OP_S_DATA_FORWARDING_MASK                                                          0x00040000L
+#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE_MASK                                                       0x00080000L
+#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE_MASK                                                           0x00100000L
+#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT_MASK                                                  0x00200000L
+#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB_MASK                                                          0x00400000L
+#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD_MASK                                                    0x00800000L
+#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT_MASK                                                             0x01000000L
+#define DB_DEBUG3__DISABLE_DI_DT_STALL_MASK                                                                   0x02000000L
+#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET_MASK                                                               0x04000000L
+#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX_MASK                                                         0x08000000L
+#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND_MASK                                                           0x10000000L
+#define DB_DEBUG3__DONT_DELETE_CONTEXT_SUSPEND_MASK                                                           0x20000000L
+#define DB_DEBUG3__DISABLE_4XAA_2P_DELAYED_WRITE_MASK                                                         0x40000000L
+#define DB_DEBUG3__DISABLE_4XAA_2P_INTERLEAVED_PMASK_MASK                                                     0x80000000L
+//DB_DEBUG4
+#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION__SHIFT                                                         0x0
+#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION__SHIFT                                                   0x1
+#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL__SHIFT                                                    0x2
+#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL__SHIFT                                             0x3
+#define DB_DEBUG4__DISABLE_4XAA_2P_ZD_HOLDOFF__SHIFT                                                          0x4
+#define DB_DEBUG4__ENABLE_A2M_DQUAD_OPTIMIZATION__SHIFT                                                       0x5
+#define DB_DEBUG4__ENABLE_DBCB_SLOW_FORMAT_COLLAPSE__SHIFT                                                    0x6
+#define DB_DEBUG4__ALWAYS_ON_RMI_CLK_EN__SHIFT                                                                0x7
+#define DB_DEBUG4__DFSM_CONVERT_PASSTHROUGH_TO_BYPASS__SHIFT                                                  0x8
+#define DB_DEBUG4__DISABLE_UNMAPPED_Z_INDICATOR__SHIFT                                                        0x9
+#define DB_DEBUG4__DISABLE_UNMAPPED_S_INDICATOR__SHIFT                                                        0xa
+#define DB_DEBUG4__DISABLE_UNMAPPED_H_INDICATOR__SHIFT                                                        0xb
+#define DB_DEBUG4__DISABLE_SEPARATE_DFSM_CLK__SHIFT                                                           0xc
+#define DB_DEBUG4__DISABLE_DTT_FAST_HTILENACK_LOOKUP__SHIFT                                                   0xd
+#define DB_DEBUG4__DISABLE_RESCHECK_MEMCOHER_OPTIMIZATION__SHIFT                                              0xe
+#define DB_DEBUG4__DISABLE_TS_WRITE_L0__SHIFT                                                                 0xf
+#define DB_DEBUG4__DISABLE_DYNAMIC_RAM_LIGHT_SLEEP_MODE__SHIFT                                                0x10
+#define DB_DEBUG4__DISABLE_HIZ_Q1_TS_COLLISION_DETECT__SHIFT                                                  0x11
+#define DB_DEBUG4__DISABLE_HIZ_Q2_TS_COLLISION_DETECT__SHIFT                                                  0x12
+#define DB_DEBUG4__DB_EXTRA_DEBUG4__SHIFT                                                                     0x13
+#define DB_DEBUG4__DISABLE_8PPC_OBJPRIMID_WHEN_NO_SHADER_EXPORTS__SHIFT                                       0x1e
+#define DB_DEBUG4__FULL_TILE_CACHE_EVICT_ON_HALF_FULL__SHIFT                                                  0x1f
+#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION_MASK                                                           0x00000001L
+#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION_MASK                                                     0x00000002L
+#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL_MASK                                                      0x00000004L
+#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL_MASK                                               0x00000008L
+#define DB_DEBUG4__DISABLE_4XAA_2P_ZD_HOLDOFF_MASK                                                            0x00000010L
+#define DB_DEBUG4__ENABLE_A2M_DQUAD_OPTIMIZATION_MASK                                                         0x00000020L
+#define DB_DEBUG4__ENABLE_DBCB_SLOW_FORMAT_COLLAPSE_MASK                                                      0x00000040L
+#define DB_DEBUG4__ALWAYS_ON_RMI_CLK_EN_MASK                                                                  0x00000080L
+#define DB_DEBUG4__DFSM_CONVERT_PASSTHROUGH_TO_BYPASS_MASK                                                    0x00000100L
+#define DB_DEBUG4__DISABLE_UNMAPPED_Z_INDICATOR_MASK                                                          0x00000200L
+#define DB_DEBUG4__DISABLE_UNMAPPED_S_INDICATOR_MASK                                                          0x00000400L
+#define DB_DEBUG4__DISABLE_UNMAPPED_H_INDICATOR_MASK                                                          0x00000800L
+#define DB_DEBUG4__DISABLE_SEPARATE_DFSM_CLK_MASK                                                             0x00001000L
+#define DB_DEBUG4__DISABLE_DTT_FAST_HTILENACK_LOOKUP_MASK                                                     0x00002000L
+#define DB_DEBUG4__DISABLE_RESCHECK_MEMCOHER_OPTIMIZATION_MASK                                                0x00004000L
+#define DB_DEBUG4__DISABLE_TS_WRITE_L0_MASK                                                                   0x00008000L
+#define DB_DEBUG4__DISABLE_DYNAMIC_RAM_LIGHT_SLEEP_MODE_MASK                                                  0x00010000L
+#define DB_DEBUG4__DISABLE_HIZ_Q1_TS_COLLISION_DETECT_MASK                                                    0x00020000L
+#define DB_DEBUG4__DISABLE_HIZ_Q2_TS_COLLISION_DETECT_MASK                                                    0x00040000L
+#define DB_DEBUG4__DB_EXTRA_DEBUG4_MASK                                                                       0x3FF80000L
+#define DB_DEBUG4__DISABLE_8PPC_OBJPRIMID_WHEN_NO_SHADER_EXPORTS_MASK                                         0x40000000L
+#define DB_DEBUG4__FULL_TILE_CACHE_EVICT_ON_HALF_FULL_MASK                                                    0x80000000L
+//DB_CREDIT_LIMIT
+#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS__SHIFT                                                            0x0
+#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS__SHIFT                                                            0x5
+#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS__SHIFT                                                           0xa
+#define DB_CREDIT_LIMIT__DB_CB_TILE_CREDITS__SHIFT                                                            0x18
+#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS_MASK                                                              0x0000001FL
+#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS_MASK                                                              0x000003E0L
+#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS_MASK                                                             0x00001C00L
+#define DB_CREDIT_LIMIT__DB_CB_TILE_CREDITS_MASK                                                              0x7F000000L
+//DB_WATERMARKS
+#define DB_WATERMARKS__DEPTH_FREE__SHIFT                                                                      0x0
+#define DB_WATERMARKS__DEPTH_FLUSH__SHIFT                                                                     0x5
+#define DB_WATERMARKS__FORCE_SUMMARIZE__SHIFT                                                                 0xb
+#define DB_WATERMARKS__DEPTH_PENDING_FREE__SHIFT                                                              0xf
+#define DB_WATERMARKS__DEPTH_CACHELINE_FREE__SHIFT                                                            0x14
+#define DB_WATERMARKS__AUTO_FLUSH_HTILE__SHIFT                                                                0x1e
+#define DB_WATERMARKS__AUTO_FLUSH_QUAD__SHIFT                                                                 0x1f
+#define DB_WATERMARKS__DEPTH_FREE_MASK                                                                        0x0000001FL
+#define DB_WATERMARKS__DEPTH_FLUSH_MASK                                                                       0x000007E0L
+#define DB_WATERMARKS__FORCE_SUMMARIZE_MASK                                                                   0x00007800L
+#define DB_WATERMARKS__DEPTH_PENDING_FREE_MASK                                                                0x000F8000L
+#define DB_WATERMARKS__DEPTH_CACHELINE_FREE_MASK                                                              0x0FF00000L
+#define DB_WATERMARKS__AUTO_FLUSH_HTILE_MASK                                                                  0x40000000L
+#define DB_WATERMARKS__AUTO_FLUSH_QUAD_MASK                                                                   0x80000000L
+//DB_SUBTILE_CONTROL
+#define DB_SUBTILE_CONTROL__MSAA1_X__SHIFT                                                                    0x0
+#define DB_SUBTILE_CONTROL__MSAA1_Y__SHIFT                                                                    0x2
+#define DB_SUBTILE_CONTROL__MSAA2_X__SHIFT                                                                    0x4
+#define DB_SUBTILE_CONTROL__MSAA2_Y__SHIFT                                                                    0x6
+#define DB_SUBTILE_CONTROL__MSAA4_X__SHIFT                                                                    0x8
+#define DB_SUBTILE_CONTROL__MSAA4_Y__SHIFT                                                                    0xa
+#define DB_SUBTILE_CONTROL__MSAA8_X__SHIFT                                                                    0xc
+#define DB_SUBTILE_CONTROL__MSAA8_Y__SHIFT                                                                    0xe
+#define DB_SUBTILE_CONTROL__MSAA16_X__SHIFT                                                                   0x10
+#define DB_SUBTILE_CONTROL__MSAA16_Y__SHIFT                                                                   0x12
+#define DB_SUBTILE_CONTROL__MSAA1_X_MASK                                                                      0x00000003L
+#define DB_SUBTILE_CONTROL__MSAA1_Y_MASK                                                                      0x0000000CL
+#define DB_SUBTILE_CONTROL__MSAA2_X_MASK                                                                      0x00000030L
+#define DB_SUBTILE_CONTROL__MSAA2_Y_MASK                                                                      0x000000C0L
+#define DB_SUBTILE_CONTROL__MSAA4_X_MASK                                                                      0x00000300L
+#define DB_SUBTILE_CONTROL__MSAA4_Y_MASK                                                                      0x00000C00L
+#define DB_SUBTILE_CONTROL__MSAA8_X_MASK                                                                      0x00003000L
+#define DB_SUBTILE_CONTROL__MSAA8_Y_MASK                                                                      0x0000C000L
+#define DB_SUBTILE_CONTROL__MSAA16_X_MASK                                                                     0x00030000L
+#define DB_SUBTILE_CONTROL__MSAA16_Y_MASK                                                                     0x000C0000L
+//DB_FREE_CACHELINES
+#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH__SHIFT                                                           0x0
+#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH__SHIFT                                                           0x7
+#define DB_FREE_CACHELINES__FREE_Z_DEPTH__SHIFT                                                               0xe
+#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH__SHIFT                                                           0x14
+#define DB_FREE_CACHELINES__QUAD_READ_REQS__SHIFT                                                             0x18
+#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH_MASK                                                             0x0000007FL
+#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH_MASK                                                             0x00003F80L
+#define DB_FREE_CACHELINES__FREE_Z_DEPTH_MASK                                                                 0x000FC000L
+#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH_MASK                                                             0x00F00000L
+#define DB_FREE_CACHELINES__QUAD_READ_REQS_MASK                                                               0xFF000000L
+//DB_FIFO_DEPTH1
+#define DB_FIFO_DEPTH1__DB_RMI_RDREQ_CREDITS__SHIFT                                                           0x0
+#define DB_FIFO_DEPTH1__DB_RMI_WRREQ_CREDITS__SHIFT                                                           0x5
+#define DB_FIFO_DEPTH1__MCC_DEPTH__SHIFT                                                                      0xa
+#define DB_FIFO_DEPTH1__QC_DEPTH__SHIFT                                                                       0x10
+#define DB_FIFO_DEPTH1__LTILE_PROBE_FIFO_DEPTH__SHIFT                                                         0x15
+#define DB_FIFO_DEPTH1__DB_RMI_RDREQ_CREDITS_MASK                                                             0x0000001FL
+#define DB_FIFO_DEPTH1__DB_RMI_WRREQ_CREDITS_MASK                                                             0x000003E0L
+#define DB_FIFO_DEPTH1__MCC_DEPTH_MASK                                                                        0x0000FC00L
+#define DB_FIFO_DEPTH1__QC_DEPTH_MASK                                                                         0x001F0000L
+#define DB_FIFO_DEPTH1__LTILE_PROBE_FIFO_DEPTH_MASK                                                           0x1FE00000L
+//DB_FIFO_DEPTH2
+#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH__SHIFT                                                               0x0
+#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH__SHIFT                                                            0x8
+#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH__SHIFT                                                               0xf
+#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH__SHIFT                                                            0x19
+#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH_MASK                                                                 0x000000FFL
+#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH_MASK                                                              0x00007F00L
+#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH_MASK                                                                 0x01FF8000L
+#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH_MASK                                                              0xFE000000L
+//DB_EXCEPTION_CONTROL
+#define DB_EXCEPTION_CONTROL__EARLY_Z_PANIC_DISABLE__SHIFT                                                    0x0
+#define DB_EXCEPTION_CONTROL__LATE_Z_PANIC_DISABLE__SHIFT                                                     0x1
+#define DB_EXCEPTION_CONTROL__RE_Z_PANIC_DISABLE__SHIFT                                                       0x2
+#define DB_EXCEPTION_CONTROL__EARLY_Z_PANIC_DISABLE_MASK                                                      0x00000001L
+#define DB_EXCEPTION_CONTROL__LATE_Z_PANIC_DISABLE_MASK                                                       0x00000002L
+#define DB_EXCEPTION_CONTROL__RE_Z_PANIC_DISABLE_MASK                                                         0x00000004L
+//DB_RING_CONTROL
+#define DB_RING_CONTROL__COUNTER_CONTROL__SHIFT                                                               0x0
+#define DB_RING_CONTROL__COUNTER_CONTROL_MASK                                                                 0x00000003L
+//DB_MEM_ARB_WATERMARKS
+#define DB_MEM_ARB_WATERMARKS__CLIENT0_WATERMARK__SHIFT                                                       0x0
+#define DB_MEM_ARB_WATERMARKS__CLIENT1_WATERMARK__SHIFT                                                       0x8
+#define DB_MEM_ARB_WATERMARKS__CLIENT2_WATERMARK__SHIFT                                                       0x10
+#define DB_MEM_ARB_WATERMARKS__CLIENT3_WATERMARK__SHIFT                                                       0x18
+#define DB_MEM_ARB_WATERMARKS__CLIENT0_WATERMARK_MASK                                                         0x00000007L
+#define DB_MEM_ARB_WATERMARKS__CLIENT1_WATERMARK_MASK                                                         0x00000700L
+#define DB_MEM_ARB_WATERMARKS__CLIENT2_WATERMARK_MASK                                                         0x00070000L
+#define DB_MEM_ARB_WATERMARKS__CLIENT3_WATERMARK_MASK                                                         0x07000000L
+//DB_RMI_CACHE_POLICY
+#define DB_RMI_CACHE_POLICY__Z_RD__SHIFT                                                                      0x0
+#define DB_RMI_CACHE_POLICY__S_RD__SHIFT                                                                      0x1
+#define DB_RMI_CACHE_POLICY__HTILE_RD__SHIFT                                                                  0x2
+#define DB_RMI_CACHE_POLICY__Z_WR__SHIFT                                                                      0x8
+#define DB_RMI_CACHE_POLICY__S_WR__SHIFT                                                                      0x9
+#define DB_RMI_CACHE_POLICY__HTILE_WR__SHIFT                                                                  0xa
+#define DB_RMI_CACHE_POLICY__ZPCPSD_WR__SHIFT                                                                 0xb
+#define DB_RMI_CACHE_POLICY__CC_RD__SHIFT                                                                     0x10
+#define DB_RMI_CACHE_POLICY__FMASK_RD__SHIFT                                                                  0x11
+#define DB_RMI_CACHE_POLICY__CMASK_RD__SHIFT                                                                  0x12
+#define DB_RMI_CACHE_POLICY__DCC_RD__SHIFT                                                                    0x13
+#define DB_RMI_CACHE_POLICY__CC_WR__SHIFT                                                                     0x18
+#define DB_RMI_CACHE_POLICY__FMASK_WR__SHIFT                                                                  0x19
+#define DB_RMI_CACHE_POLICY__CMASK_WR__SHIFT                                                                  0x1a
+#define DB_RMI_CACHE_POLICY__DCC_WR__SHIFT                                                                    0x1b
+#define DB_RMI_CACHE_POLICY__Z_RD_MASK                                                                        0x00000001L
+#define DB_RMI_CACHE_POLICY__S_RD_MASK                                                                        0x00000002L
+#define DB_RMI_CACHE_POLICY__HTILE_RD_MASK                                                                    0x00000004L
+#define DB_RMI_CACHE_POLICY__Z_WR_MASK                                                                        0x00000100L
+#define DB_RMI_CACHE_POLICY__S_WR_MASK                                                                        0x00000200L
+#define DB_RMI_CACHE_POLICY__HTILE_WR_MASK                                                                    0x00000400L
+#define DB_RMI_CACHE_POLICY__ZPCPSD_WR_MASK                                                                   0x00000800L
+#define DB_RMI_CACHE_POLICY__CC_RD_MASK                                                                       0x00010000L
+#define DB_RMI_CACHE_POLICY__FMASK_RD_MASK                                                                    0x00020000L
+#define DB_RMI_CACHE_POLICY__CMASK_RD_MASK                                                                    0x00040000L
+#define DB_RMI_CACHE_POLICY__DCC_RD_MASK                                                                      0x00080000L
+#define DB_RMI_CACHE_POLICY__CC_WR_MASK                                                                       0x01000000L
+#define DB_RMI_CACHE_POLICY__FMASK_WR_MASK                                                                    0x02000000L
+#define DB_RMI_CACHE_POLICY__CMASK_WR_MASK                                                                    0x04000000L
+#define DB_RMI_CACHE_POLICY__DCC_WR_MASK                                                                      0x08000000L
+//DB_DFSM_CONFIG
+#define DB_DFSM_CONFIG__BYPASS_DFSM__SHIFT                                                                    0x0
+#define DB_DFSM_CONFIG__DISABLE_PUNCHOUT__SHIFT                                                               0x1
+#define DB_DFSM_CONFIG__DISABLE_POPS__SHIFT                                                                   0x2
+#define DB_DFSM_CONFIG__FORCE_FLUSH__SHIFT                                                                    0x3
+#define DB_DFSM_CONFIG__MIDDLE_PIPE_MAX_DEPTH__SHIFT                                                          0x8
+#define DB_DFSM_CONFIG__BYPASS_DFSM_MASK                                                                      0x00000001L
+#define DB_DFSM_CONFIG__DISABLE_PUNCHOUT_MASK                                                                 0x00000002L
+#define DB_DFSM_CONFIG__DISABLE_POPS_MASK                                                                     0x00000004L
+#define DB_DFSM_CONFIG__FORCE_FLUSH_MASK                                                                      0x00000008L
+#define DB_DFSM_CONFIG__MIDDLE_PIPE_MAX_DEPTH_MASK                                                            0x00007F00L
+//DB_DFSM_WATERMARK
+#define DB_DFSM_WATERMARK__DFSM_HIGH_WATERMARK__SHIFT                                                         0x0
+#define DB_DFSM_WATERMARK__POPS_HIGH_WATERMARK__SHIFT                                                         0x10
+#define DB_DFSM_WATERMARK__DFSM_HIGH_WATERMARK_MASK                                                           0x0000FFFFL
+#define DB_DFSM_WATERMARK__POPS_HIGH_WATERMARK_MASK                                                           0xFFFF0000L
+//DB_DFSM_TILES_IN_FLIGHT
+#define DB_DFSM_TILES_IN_FLIGHT__HIGH_WATERMARK__SHIFT                                                        0x0
+#define DB_DFSM_TILES_IN_FLIGHT__HARD_LIMIT__SHIFT                                                            0x10
+#define DB_DFSM_TILES_IN_FLIGHT__HIGH_WATERMARK_MASK                                                          0x0000FFFFL
+#define DB_DFSM_TILES_IN_FLIGHT__HARD_LIMIT_MASK                                                              0xFFFF0000L
+//DB_DFSM_PRIMS_IN_FLIGHT
+#define DB_DFSM_PRIMS_IN_FLIGHT__HIGH_WATERMARK__SHIFT                                                        0x0
+#define DB_DFSM_PRIMS_IN_FLIGHT__HARD_LIMIT__SHIFT                                                            0x10
+#define DB_DFSM_PRIMS_IN_FLIGHT__HIGH_WATERMARK_MASK                                                          0x0000FFFFL
+#define DB_DFSM_PRIMS_IN_FLIGHT__HARD_LIMIT_MASK                                                              0xFFFF0000L
+//DB_DFSM_WATCHDOG
+#define DB_DFSM_WATCHDOG__TIMER_TARGET__SHIFT                                                                 0x0
+#define DB_DFSM_WATCHDOG__TIMER_TARGET_MASK                                                                   0xFFFFFFFFL
+//DB_DFSM_FLUSH_ENABLE
+#define DB_DFSM_FLUSH_ENABLE__PRIMARY_EVENTS__SHIFT                                                           0x0
+#define DB_DFSM_FLUSH_ENABLE__AUX_FORCE_PASSTHRU__SHIFT                                                       0x18
+#define DB_DFSM_FLUSH_ENABLE__AUX_EVENTS__SHIFT                                                               0x1c
+#define DB_DFSM_FLUSH_ENABLE__PRIMARY_EVENTS_MASK                                                             0x000003FFL
+#define DB_DFSM_FLUSH_ENABLE__AUX_FORCE_PASSTHRU_MASK                                                         0x0F000000L
+#define DB_DFSM_FLUSH_ENABLE__AUX_EVENTS_MASK                                                                 0xF0000000L
+//DB_DFSM_FLUSH_AUX_EVENT
+#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_A__SHIFT                                                               0x0
+#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_B__SHIFT                                                               0x8
+#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_C__SHIFT                                                               0x10
+#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_D__SHIFT                                                               0x18
+#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_A_MASK                                                                 0x000000FFL
+#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_B_MASK                                                                 0x0000FF00L
+#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_C_MASK                                                                 0x00FF0000L
+#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_D_MASK                                                                 0xFF000000L
+//CC_RB_REDUNDANCY
+#define CC_RB_REDUNDANCY__WRITE_DIS__SHIFT                                                                    0x0
+#define CC_RB_REDUNDANCY__FAILED_RB0__SHIFT                                                                   0x8
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT                                                               0xc
+#define CC_RB_REDUNDANCY__FAILED_RB1__SHIFT                                                                   0x10
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT                                                               0x14
+#define CC_RB_REDUNDANCY__WRITE_DIS_MASK                                                                      0x00000001L
+#define CC_RB_REDUNDANCY__FAILED_RB0_MASK                                                                     0x00000F00L
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY0_MASK                                                                 0x00001000L
+#define CC_RB_REDUNDANCY__FAILED_RB1_MASK                                                                     0x000F0000L
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY1_MASK                                                                 0x00100000L
+//CC_RB_BACKEND_DISABLE
+#define CC_RB_BACKEND_DISABLE__WRITE_DIS__SHIFT                                                               0x0
+#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT                                                         0x10
+#define CC_RB_BACKEND_DISABLE__WRITE_DIS_MASK                                                                 0x00000001L
+#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK                                                           0x00FF0000L
+//GB_ADDR_CONFIG
+#define GB_ADDR_CONFIG__NUM_PIPES__SHIFT                                                                      0x0
+#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                           0x3
+#define GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS__SHIFT                                                           0x6
+#define GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT                                                           0x8
+#define GB_ADDR_CONFIG__NUM_BANKS__SHIFT                                                                      0xc
+#define GB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT                                                        0x10
+#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT                                                             0x13
+#define GB_ADDR_CONFIG__NUM_GPUS__SHIFT                                                                       0x15
+#define GB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT                                                            0x18
+#define GB_ADDR_CONFIG__NUM_RB_PER_SE__SHIFT                                                                  0x1a
+#define GB_ADDR_CONFIG__ROW_SIZE__SHIFT                                                                       0x1c
+#define GB_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT                                                                0x1e
+#define GB_ADDR_CONFIG__SE_ENABLE__SHIFT                                                                      0x1f
+#define GB_ADDR_CONFIG__NUM_PIPES_MASK                                                                        0x00000007L
+#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                             0x00000038L
+#define GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS_MASK                                                             0x000000C0L
+#define GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK                                                             0x00000700L
+#define GB_ADDR_CONFIG__NUM_BANKS_MASK                                                                        0x00007000L
+#define GB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK                                                          0x00070000L
+#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK                                                               0x00180000L
+#define GB_ADDR_CONFIG__NUM_GPUS_MASK                                                                         0x00E00000L
+#define GB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK                                                              0x03000000L
+#define GB_ADDR_CONFIG__NUM_RB_PER_SE_MASK                                                                    0x0C000000L
+#define GB_ADDR_CONFIG__ROW_SIZE_MASK                                                                         0x30000000L
+#define GB_ADDR_CONFIG__NUM_LOWER_PIPES_MASK                                                                  0x40000000L
+#define GB_ADDR_CONFIG__SE_ENABLE_MASK                                                                        0x80000000L
+//GB_BACKEND_MAP
+#define GB_BACKEND_MAP__BACKEND_MAP__SHIFT                                                                    0x0
+#define GB_BACKEND_MAP__BACKEND_MAP_MASK                                                                      0xFFFFFFFFL
+//GB_GPU_ID
+#define GB_GPU_ID__GPU_ID__SHIFT                                                                              0x0
+#define GB_GPU_ID__GPU_ID_MASK                                                                                0x0000000FL
+//CC_RB_DAISY_CHAIN
+#define CC_RB_DAISY_CHAIN__RB_0__SHIFT                                                                        0x0
+#define CC_RB_DAISY_CHAIN__RB_1__SHIFT                                                                        0x4
+#define CC_RB_DAISY_CHAIN__RB_2__SHIFT                                                                        0x8
+#define CC_RB_DAISY_CHAIN__RB_3__SHIFT                                                                        0xc
+#define CC_RB_DAISY_CHAIN__RB_4__SHIFT                                                                        0x10
+#define CC_RB_DAISY_CHAIN__RB_5__SHIFT                                                                        0x14
+#define CC_RB_DAISY_CHAIN__RB_6__SHIFT                                                                        0x18
+#define CC_RB_DAISY_CHAIN__RB_7__SHIFT                                                                        0x1c
+#define CC_RB_DAISY_CHAIN__RB_0_MASK                                                                          0x0000000FL
+#define CC_RB_DAISY_CHAIN__RB_1_MASK                                                                          0x000000F0L
+#define CC_RB_DAISY_CHAIN__RB_2_MASK                                                                          0x00000F00L
+#define CC_RB_DAISY_CHAIN__RB_3_MASK                                                                          0x0000F000L
+#define CC_RB_DAISY_CHAIN__RB_4_MASK                                                                          0x000F0000L
+#define CC_RB_DAISY_CHAIN__RB_5_MASK                                                                          0x00F00000L
+#define CC_RB_DAISY_CHAIN__RB_6_MASK                                                                          0x0F000000L
+#define CC_RB_DAISY_CHAIN__RB_7_MASK                                                                          0xF0000000L
+//GB_ADDR_CONFIG_READ
+#define GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                                 0x0
+#define GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                      0x3
+#define GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS__SHIFT                                                      0x6
+#define GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE__SHIFT                                                      0x8
+#define GB_ADDR_CONFIG_READ__NUM_BANKS__SHIFT                                                                 0xc
+#define GB_ADDR_CONFIG_READ__SHADER_ENGINE_TILE_SIZE__SHIFT                                                   0x10
+#define GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                        0x13
+#define GB_ADDR_CONFIG_READ__NUM_GPUS__SHIFT                                                                  0x15
+#define GB_ADDR_CONFIG_READ__MULTI_GPU_TILE_SIZE__SHIFT                                                       0x18
+#define GB_ADDR_CONFIG_READ__NUM_RB_PER_SE__SHIFT                                                             0x1a
+#define GB_ADDR_CONFIG_READ__ROW_SIZE__SHIFT                                                                  0x1c
+#define GB_ADDR_CONFIG_READ__NUM_LOWER_PIPES__SHIFT                                                           0x1e
+#define GB_ADDR_CONFIG_READ__SE_ENABLE__SHIFT                                                                 0x1f
+#define GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                                   0x00000007L
+#define GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                        0x00000038L
+#define GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS_MASK                                                        0x000000C0L
+#define GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE_MASK                                                        0x00000700L
+#define GB_ADDR_CONFIG_READ__NUM_BANKS_MASK                                                                   0x00007000L
+#define GB_ADDR_CONFIG_READ__SHADER_ENGINE_TILE_SIZE_MASK                                                     0x00070000L
+#define GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                          0x00180000L
+#define GB_ADDR_CONFIG_READ__NUM_GPUS_MASK                                                                    0x00E00000L
+#define GB_ADDR_CONFIG_READ__MULTI_GPU_TILE_SIZE_MASK                                                         0x03000000L
+#define GB_ADDR_CONFIG_READ__NUM_RB_PER_SE_MASK                                                               0x0C000000L
+#define GB_ADDR_CONFIG_READ__ROW_SIZE_MASK                                                                    0x30000000L
+#define GB_ADDR_CONFIG_READ__NUM_LOWER_PIPES_MASK                                                             0x40000000L
+#define GB_ADDR_CONFIG_READ__SE_ENABLE_MASK                                                                   0x80000000L
+//GB_TILE_MODE0
+#define GB_TILE_MODE0__ARRAY_MODE__SHIFT                                                                      0x2
+#define GB_TILE_MODE0__PIPE_CONFIG__SHIFT                                                                     0x6
+#define GB_TILE_MODE0__TILE_SPLIT__SHIFT                                                                      0xb
+#define GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
+#define GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT                                                                    0x19
+#define GB_TILE_MODE0__ARRAY_MODE_MASK                                                                        0x0000003CL
+#define GB_TILE_MODE0__PIPE_CONFIG_MASK                                                                       0x000007C0L
+#define GB_TILE_MODE0__TILE_SPLIT_MASK                                                                        0x00003800L
+#define GB_TILE_MODE0__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
+#define GB_TILE_MODE0__SAMPLE_SPLIT_MASK                                                                      0x06000000L
+//GB_TILE_MODE1
+#define GB_TILE_MODE1__ARRAY_MODE__SHIFT                                                                      0x2
+#define GB_TILE_MODE1__PIPE_CONFIG__SHIFT                                                                     0x6
+#define GB_TILE_MODE1__TILE_SPLIT__SHIFT                                                                      0xb
+#define GB_TILE_MODE1__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
+#define GB_TILE_MODE1__SAMPLE_SPLIT__SHIFT                                                                    0x19
+#define GB_TILE_MODE1__ARRAY_MODE_MASK                                                                        0x0000003CL
+#define GB_TILE_MODE1__PIPE_CONFIG_MASK                                                                       0x000007C0L
+#define GB_TILE_MODE1__TILE_SPLIT_MASK                                                                        0x00003800L
+#define GB_TILE_MODE1__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
+#define GB_TILE_MODE1__SAMPLE_SPLIT_MASK                                                                      0x06000000L
+//GB_TILE_MODE2
+#define GB_TILE_MODE2__ARRAY_MODE__SHIFT                                                                      0x2
+#define GB_TILE_MODE2__PIPE_CONFIG__SHIFT                                                                     0x6
+#define GB_TILE_MODE2__TILE_SPLIT__SHIFT                                                                      0xb
+#define GB_TILE_MODE2__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
+#define GB_TILE_MODE2__SAMPLE_SPLIT__SHIFT                                                                    0x19
+#define GB_TILE_MODE2__ARRAY_MODE_MASK                                                                        0x0000003CL
+#define GB_TILE_MODE2__PIPE_CONFIG_MASK                                                                       0x000007C0L
+#define GB_TILE_MODE2__TILE_SPLIT_MASK                                                                        0x00003800L
+#define GB_TILE_MODE2__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
+#define GB_TILE_MODE2__SAMPLE_SPLIT_MASK                                                                      0x06000000L
+//GB_TILE_MODE3
+#define GB_TILE_MODE3__ARRAY_MODE__SHIFT                                                                      0x2
+#define GB_TILE_MODE3__PIPE_CONFIG__SHIFT                                                                     0x6
+#define GB_TILE_MODE3__TILE_SPLIT__SHIFT                                                                      0xb
+#define GB_TILE_MODE3__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
+#define GB_TILE_MODE3__SAMPLE_SPLIT__SHIFT                                                                    0x19
+#define GB_TILE_MODE3__ARRAY_MODE_MASK                                                                        0x0000003CL
+#define GB_TILE_MODE3__PIPE_CONFIG_MASK                                                                       0x000007C0L
+#define GB_TILE_MODE3__TILE_SPLIT_MASK                                                                        0x00003800L
+#define GB_TILE_MODE3__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
+#define GB_TILE_MODE3__SAMPLE_SPLIT_MASK                                                                      0x06000000L
+//GB_TILE_MODE4
+#define GB_TILE_MODE4__ARRAY_MODE__SHIFT                                                                      0x2
+#define GB_TILE_MODE4__PIPE_CONFIG__SHIFT                                                                     0x6
+#define GB_TILE_MODE4__TILE_SPLIT__SHIFT                                                                      0xb
+#define GB_TILE_MODE4__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
+#define GB_TILE_MODE4__SAMPLE_SPLIT__SHIFT                                                                    0x19
+#define GB_TILE_MODE4__ARRAY_MODE_MASK                                                                        0x0000003CL
+#define GB_TILE_MODE4__PIPE_CONFIG_MASK                                                                       0x000007C0L
+#define GB_TILE_MODE4__TILE_SPLIT_MASK                                                                        0x00003800L
+#define GB_TILE_MODE4__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
+#define GB_TILE_MODE4__SAMPLE_SPLIT_MASK                                                                      0x06000000L
+//GB_TILE_MODE5
+#define GB_TILE_MODE5__ARRAY_MODE__SHIFT                                                                      0x2
+#define GB_TILE_MODE5__PIPE_CONFIG__SHIFT                                                                     0x6
+#define GB_TILE_MODE5__TILE_SPLIT__SHIFT                                                                      0xb
+#define GB_TILE_MODE5__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
+#define GB_TILE_MODE5__SAMPLE_SPLIT__SHIFT                                                                    0x19
+#define GB_TILE_MODE5__ARRAY_MODE_MASK                                                                        0x0000003CL
+#define GB_TILE_MODE5__PIPE_CONFIG_MASK                                                                       0x000007C0L
+#define GB_TILE_MODE5__TILE_SPLIT_MASK                                                                        0x00003800L
+#define GB_TILE_MODE5__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
+#define GB_TILE_MODE5__SAMPLE_SPLIT_MASK                                                                      0x06000000L
+//GB_TILE_MODE6
+#define GB_TILE_MODE6__ARRAY_MODE__SHIFT                                                                      0x2
+#define GB_TILE_MODE6__PIPE_CONFIG__SHIFT                                                                     0x6
+#define GB_TILE_MODE6__TILE_SPLIT__SHIFT                                                                      0xb
+#define GB_TILE_MODE6__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
+#define GB_TILE_MODE6__SAMPLE_SPLIT__SHIFT                                                                    0x19
+#define GB_TILE_MODE6__ARRAY_MODE_MASK                                                                        0x0000003CL
+#define GB_TILE_MODE6__PIPE_CONFIG_MASK                                                                       0x000007C0L
+#define GB_TILE_MODE6__TILE_SPLIT_MASK                                                                        0x00003800L
+#define GB_TILE_MODE6__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
+#define GB_TILE_MODE6__SAMPLE_SPLIT_MASK                                                                      0x06000000L
+//GB_TILE_MODE7
+#define GB_TILE_MODE7__ARRAY_MODE__SHIFT                                                                      0x2
+#define GB_TILE_MODE7__PIPE_CONFIG__SHIFT                                                                     0x6
+#define GB_TILE_MODE7__TILE_SPLIT__SHIFT                                                                      0xb
+#define GB_TILE_MODE7__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
+#define GB_TILE_MODE7__SAMPLE_SPLIT__SHIFT                                                                    0x19
+#define GB_TILE_MODE7__ARRAY_MODE_MASK                                                                        0x0000003CL
+#define GB_TILE_MODE7__PIPE_CONFIG_MASK                                                                       0x000007C0L
+#define GB_TILE_MODE7__TILE_SPLIT_MASK                                                                        0x00003800L
+#define GB_TILE_MODE7__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
+#define GB_TILE_MODE7__SAMPLE_SPLIT_MASK                                                                      0x06000000L
+//GB_TILE_MODE8
+#define GB_TILE_MODE8__ARRAY_MODE__SHIFT                                                                      0x2
+#define GB_TILE_MODE8__PIPE_CONFIG__SHIFT                                                                     0x6
+#define GB_TILE_MODE8__TILE_SPLIT__SHIFT                                                                      0xb
+#define GB_TILE_MODE8__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
+#define GB_TILE_MODE8__SAMPLE_SPLIT__SHIFT                                                                    0x19
+#define GB_TILE_MODE8__ARRAY_MODE_MASK                                                                        0x0000003CL
+#define GB_TILE_MODE8__PIPE_CONFIG_MASK                                                                       0x000007C0L
+#define GB_TILE_MODE8__TILE_SPLIT_MASK                                                                        0x00003800L
+#define GB_TILE_MODE8__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
+#define GB_TILE_MODE8__SAMPLE_SPLIT_MASK                                                                      0x06000000L
+//GB_TILE_MODE9
+#define GB_TILE_MODE9__ARRAY_MODE__SHIFT                                                                      0x2
+#define GB_TILE_MODE9__PIPE_CONFIG__SHIFT                                                                     0x6
+#define GB_TILE_MODE9__TILE_SPLIT__SHIFT                                                                      0xb
+#define GB_TILE_MODE9__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
+#define GB_TILE_MODE9__SAMPLE_SPLIT__SHIFT                                                                    0x19
+#define GB_TILE_MODE9__ARRAY_MODE_MASK                                                                        0x0000003CL
+#define GB_TILE_MODE9__PIPE_CONFIG_MASK                                                                       0x000007C0L
+#define GB_TILE_MODE9__TILE_SPLIT_MASK                                                                        0x00003800L
+#define GB_TILE_MODE9__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
+#define GB_TILE_MODE9__SAMPLE_SPLIT_MASK                                                                      0x06000000L
+//GB_TILE_MODE10
+#define GB_TILE_MODE10__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE10__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE10__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE10__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE10__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE10__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE10__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE10__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE10__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE10__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE11
+#define GB_TILE_MODE11__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE11__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE11__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE11__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE11__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE11__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE11__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE11__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE11__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE11__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE12
+#define GB_TILE_MODE12__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE12__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE12__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE12__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE12__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE12__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE12__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE12__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE12__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE12__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE13
+#define GB_TILE_MODE13__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE13__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE13__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE13__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE13__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE13__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE13__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE13__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE13__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE13__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE14
+#define GB_TILE_MODE14__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE14__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE14__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE14__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE14__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE14__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE14__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE14__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE14__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE14__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE15
+#define GB_TILE_MODE15__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE15__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE15__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE15__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE15__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE15__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE15__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE15__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE15__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE15__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE16
+#define GB_TILE_MODE16__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE16__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE16__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE16__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE16__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE16__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE16__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE16__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE16__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE16__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE17
+#define GB_TILE_MODE17__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE17__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE17__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE17__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE17__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE17__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE17__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE17__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE17__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE17__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE18
+#define GB_TILE_MODE18__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE18__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE18__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE18__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE18__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE18__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE18__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE18__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE18__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE18__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE19
+#define GB_TILE_MODE19__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE19__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE19__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE19__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE19__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE19__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE19__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE19__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE19__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE19__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE20
+#define GB_TILE_MODE20__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE20__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE20__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE20__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE20__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE20__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE20__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE20__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE20__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE20__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE21
+#define GB_TILE_MODE21__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE21__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE21__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE21__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE21__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE21__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE21__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE21__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE21__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE21__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE22
+#define GB_TILE_MODE22__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE22__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE22__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE22__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE22__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE22__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE22__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE22__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE22__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE22__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE23
+#define GB_TILE_MODE23__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE23__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE23__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE23__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE23__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE23__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE23__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE23__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE23__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE23__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE24
+#define GB_TILE_MODE24__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE24__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE24__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE24__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE24__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE24__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE24__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE24__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE24__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE24__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE25
+#define GB_TILE_MODE25__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE25__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE25__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE25__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE25__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE25__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE25__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE25__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE25__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE25__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE26
+#define GB_TILE_MODE26__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE26__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE26__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE26__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE26__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE26__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE26__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE26__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE26__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE26__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE27
+#define GB_TILE_MODE27__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE27__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE27__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE27__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE27__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE27__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE27__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE27__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE27__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE27__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE28
+#define GB_TILE_MODE28__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE28__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE28__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE28__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE28__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE28__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE28__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE28__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE28__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE28__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE29
+#define GB_TILE_MODE29__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE29__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE29__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE29__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE29__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE29__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE29__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE29__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE29__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE29__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE30
+#define GB_TILE_MODE30__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE30__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE30__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE30__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE30__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE30__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE30__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE30__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE30__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE30__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_TILE_MODE31
+#define GB_TILE_MODE31__ARRAY_MODE__SHIFT                                                                     0x2
+#define GB_TILE_MODE31__PIPE_CONFIG__SHIFT                                                                    0x6
+#define GB_TILE_MODE31__TILE_SPLIT__SHIFT                                                                     0xb
+#define GB_TILE_MODE31__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
+#define GB_TILE_MODE31__SAMPLE_SPLIT__SHIFT                                                                   0x19
+#define GB_TILE_MODE31__ARRAY_MODE_MASK                                                                       0x0000003CL
+#define GB_TILE_MODE31__PIPE_CONFIG_MASK                                                                      0x000007C0L
+#define GB_TILE_MODE31__TILE_SPLIT_MASK                                                                       0x00003800L
+#define GB_TILE_MODE31__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
+#define GB_TILE_MODE31__SAMPLE_SPLIT_MASK                                                                     0x06000000L
+//GB_MACROTILE_MODE0
+#define GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT                                                                 0x0
+#define GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT                                                                0x2
+#define GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT                                                          0x4
+#define GB_MACROTILE_MODE0__NUM_BANKS__SHIFT                                                                  0x6
+#define GB_MACROTILE_MODE0__BANK_WIDTH_MASK                                                                   0x00000003L
+#define GB_MACROTILE_MODE0__BANK_HEIGHT_MASK                                                                  0x0000000CL
+#define GB_MACROTILE_MODE0__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
+#define GB_MACROTILE_MODE0__NUM_BANKS_MASK                                                                    0x000000C0L
+//GB_MACROTILE_MODE1
+#define GB_MACROTILE_MODE1__BANK_WIDTH__SHIFT                                                                 0x0
+#define GB_MACROTILE_MODE1__BANK_HEIGHT__SHIFT                                                                0x2
+#define GB_MACROTILE_MODE1__MACRO_TILE_ASPECT__SHIFT                                                          0x4
+#define GB_MACROTILE_MODE1__NUM_BANKS__SHIFT                                                                  0x6
+#define GB_MACROTILE_MODE1__BANK_WIDTH_MASK                                                                   0x00000003L
+#define GB_MACROTILE_MODE1__BANK_HEIGHT_MASK                                                                  0x0000000CL
+#define GB_MACROTILE_MODE1__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
+#define GB_MACROTILE_MODE1__NUM_BANKS_MASK                                                                    0x000000C0L
+//GB_MACROTILE_MODE2
+#define GB_MACROTILE_MODE2__BANK_WIDTH__SHIFT                                                                 0x0
+#define GB_MACROTILE_MODE2__BANK_HEIGHT__SHIFT                                                                0x2
+#define GB_MACROTILE_MODE2__MACRO_TILE_ASPECT__SHIFT                                                          0x4
+#define GB_MACROTILE_MODE2__NUM_BANKS__SHIFT                                                                  0x6
+#define GB_MACROTILE_MODE2__BANK_WIDTH_MASK                                                                   0x00000003L
+#define GB_MACROTILE_MODE2__BANK_HEIGHT_MASK                                                                  0x0000000CL
+#define GB_MACROTILE_MODE2__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
+#define GB_MACROTILE_MODE2__NUM_BANKS_MASK                                                                    0x000000C0L
+//GB_MACROTILE_MODE3
+#define GB_MACROTILE_MODE3__BANK_WIDTH__SHIFT                                                                 0x0
+#define GB_MACROTILE_MODE3__BANK_HEIGHT__SHIFT                                                                0x2
+#define GB_MACROTILE_MODE3__MACRO_TILE_ASPECT__SHIFT                                                          0x4
+#define GB_MACROTILE_MODE3__NUM_BANKS__SHIFT                                                                  0x6
+#define GB_MACROTILE_MODE3__BANK_WIDTH_MASK                                                                   0x00000003L
+#define GB_MACROTILE_MODE3__BANK_HEIGHT_MASK                                                                  0x0000000CL
+#define GB_MACROTILE_MODE3__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
+#define GB_MACROTILE_MODE3__NUM_BANKS_MASK                                                                    0x000000C0L
+//GB_MACROTILE_MODE4
+#define GB_MACROTILE_MODE4__BANK_WIDTH__SHIFT                                                                 0x0
+#define GB_MACROTILE_MODE4__BANK_HEIGHT__SHIFT                                                                0x2
+#define GB_MACROTILE_MODE4__MACRO_TILE_ASPECT__SHIFT                                                          0x4
+#define GB_MACROTILE_MODE4__NUM_BANKS__SHIFT                                                                  0x6
+#define GB_MACROTILE_MODE4__BANK_WIDTH_MASK                                                                   0x00000003L
+#define GB_MACROTILE_MODE4__BANK_HEIGHT_MASK                                                                  0x0000000CL
+#define GB_MACROTILE_MODE4__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
+#define GB_MACROTILE_MODE4__NUM_BANKS_MASK                                                                    0x000000C0L
+//GB_MACROTILE_MODE5
+#define GB_MACROTILE_MODE5__BANK_WIDTH__SHIFT                                                                 0x0
+#define GB_MACROTILE_MODE5__BANK_HEIGHT__SHIFT                                                                0x2
+#define GB_MACROTILE_MODE5__MACRO_TILE_ASPECT__SHIFT                                                          0x4
+#define GB_MACROTILE_MODE5__NUM_BANKS__SHIFT                                                                  0x6
+#define GB_MACROTILE_MODE5__BANK_WIDTH_MASK                                                                   0x00000003L
+#define GB_MACROTILE_MODE5__BANK_HEIGHT_MASK                                                                  0x0000000CL
+#define GB_MACROTILE_MODE5__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
+#define GB_MACROTILE_MODE5__NUM_BANKS_MASK                                                                    0x000000C0L
+//GB_MACROTILE_MODE6
+#define GB_MACROTILE_MODE6__BANK_WIDTH__SHIFT                                                                 0x0
+#define GB_MACROTILE_MODE6__BANK_HEIGHT__SHIFT                                                                0x2
+#define GB_MACROTILE_MODE6__MACRO_TILE_ASPECT__SHIFT                                                          0x4
+#define GB_MACROTILE_MODE6__NUM_BANKS__SHIFT                                                                  0x6
+#define GB_MACROTILE_MODE6__BANK_WIDTH_MASK                                                                   0x00000003L
+#define GB_MACROTILE_MODE6__BANK_HEIGHT_MASK                                                                  0x0000000CL
+#define GB_MACROTILE_MODE6__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
+#define GB_MACROTILE_MODE6__NUM_BANKS_MASK                                                                    0x000000C0L
+//GB_MACROTILE_MODE7
+#define GB_MACROTILE_MODE7__BANK_WIDTH__SHIFT                                                                 0x0
+#define GB_MACROTILE_MODE7__BANK_HEIGHT__SHIFT                                                                0x2
+#define GB_MACROTILE_MODE7__MACRO_TILE_ASPECT__SHIFT                                                          0x4
+#define GB_MACROTILE_MODE7__NUM_BANKS__SHIFT                                                                  0x6
+#define GB_MACROTILE_MODE7__BANK_WIDTH_MASK                                                                   0x00000003L
+#define GB_MACROTILE_MODE7__BANK_HEIGHT_MASK                                                                  0x0000000CL
+#define GB_MACROTILE_MODE7__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
+#define GB_MACROTILE_MODE7__NUM_BANKS_MASK                                                                    0x000000C0L
+//GB_MACROTILE_MODE8
+#define GB_MACROTILE_MODE8__BANK_WIDTH__SHIFT                                                                 0x0
+#define GB_MACROTILE_MODE8__BANK_HEIGHT__SHIFT                                                                0x2
+#define GB_MACROTILE_MODE8__MACRO_TILE_ASPECT__SHIFT                                                          0x4
+#define GB_MACROTILE_MODE8__NUM_BANKS__SHIFT                                                                  0x6
+#define GB_MACROTILE_MODE8__BANK_WIDTH_MASK                                                                   0x00000003L
+#define GB_MACROTILE_MODE8__BANK_HEIGHT_MASK                                                                  0x0000000CL
+#define GB_MACROTILE_MODE8__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
+#define GB_MACROTILE_MODE8__NUM_BANKS_MASK                                                                    0x000000C0L
+//GB_MACROTILE_MODE9
+#define GB_MACROTILE_MODE9__BANK_WIDTH__SHIFT                                                                 0x0
+#define GB_MACROTILE_MODE9__BANK_HEIGHT__SHIFT                                                                0x2
+#define GB_MACROTILE_MODE9__MACRO_TILE_ASPECT__SHIFT                                                          0x4
+#define GB_MACROTILE_MODE9__NUM_BANKS__SHIFT                                                                  0x6
+#define GB_MACROTILE_MODE9__BANK_WIDTH_MASK                                                                   0x00000003L
+#define GB_MACROTILE_MODE9__BANK_HEIGHT_MASK                                                                  0x0000000CL
+#define GB_MACROTILE_MODE9__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
+#define GB_MACROTILE_MODE9__NUM_BANKS_MASK                                                                    0x000000C0L
+//GB_MACROTILE_MODE10
+#define GB_MACROTILE_MODE10__BANK_WIDTH__SHIFT                                                                0x0
+#define GB_MACROTILE_MODE10__BANK_HEIGHT__SHIFT                                                               0x2
+#define GB_MACROTILE_MODE10__MACRO_TILE_ASPECT__SHIFT                                                         0x4
+#define GB_MACROTILE_MODE10__NUM_BANKS__SHIFT                                                                 0x6
+#define GB_MACROTILE_MODE10__BANK_WIDTH_MASK                                                                  0x00000003L
+#define GB_MACROTILE_MODE10__BANK_HEIGHT_MASK                                                                 0x0000000CL
+#define GB_MACROTILE_MODE10__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
+#define GB_MACROTILE_MODE10__NUM_BANKS_MASK                                                                   0x000000C0L
+//GB_MACROTILE_MODE11
+#define GB_MACROTILE_MODE11__BANK_WIDTH__SHIFT                                                                0x0
+#define GB_MACROTILE_MODE11__BANK_HEIGHT__SHIFT                                                               0x2
+#define GB_MACROTILE_MODE11__MACRO_TILE_ASPECT__SHIFT                                                         0x4
+#define GB_MACROTILE_MODE11__NUM_BANKS__SHIFT                                                                 0x6
+#define GB_MACROTILE_MODE11__BANK_WIDTH_MASK                                                                  0x00000003L
+#define GB_MACROTILE_MODE11__BANK_HEIGHT_MASK                                                                 0x0000000CL
+#define GB_MACROTILE_MODE11__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
+#define GB_MACROTILE_MODE11__NUM_BANKS_MASK                                                                   0x000000C0L
+//GB_MACROTILE_MODE12
+#define GB_MACROTILE_MODE12__BANK_WIDTH__SHIFT                                                                0x0
+#define GB_MACROTILE_MODE12__BANK_HEIGHT__SHIFT                                                               0x2
+#define GB_MACROTILE_MODE12__MACRO_TILE_ASPECT__SHIFT                                                         0x4
+#define GB_MACROTILE_MODE12__NUM_BANKS__SHIFT                                                                 0x6
+#define GB_MACROTILE_MODE12__BANK_WIDTH_MASK                                                                  0x00000003L
+#define GB_MACROTILE_MODE12__BANK_HEIGHT_MASK                                                                 0x0000000CL
+#define GB_MACROTILE_MODE12__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
+#define GB_MACROTILE_MODE12__NUM_BANKS_MASK                                                                   0x000000C0L
+//GB_MACROTILE_MODE13
+#define GB_MACROTILE_MODE13__BANK_WIDTH__SHIFT                                                                0x0
+#define GB_MACROTILE_MODE13__BANK_HEIGHT__SHIFT                                                               0x2
+#define GB_MACROTILE_MODE13__MACRO_TILE_ASPECT__SHIFT                                                         0x4
+#define GB_MACROTILE_MODE13__NUM_BANKS__SHIFT                                                                 0x6
+#define GB_MACROTILE_MODE13__BANK_WIDTH_MASK                                                                  0x00000003L
+#define GB_MACROTILE_MODE13__BANK_HEIGHT_MASK                                                                 0x0000000CL
+#define GB_MACROTILE_MODE13__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
+#define GB_MACROTILE_MODE13__NUM_BANKS_MASK                                                                   0x000000C0L
+//GB_MACROTILE_MODE14
+#define GB_MACROTILE_MODE14__BANK_WIDTH__SHIFT                                                                0x0
+#define GB_MACROTILE_MODE14__BANK_HEIGHT__SHIFT                                                               0x2
+#define GB_MACROTILE_MODE14__MACRO_TILE_ASPECT__SHIFT                                                         0x4
+#define GB_MACROTILE_MODE14__NUM_BANKS__SHIFT                                                                 0x6
+#define GB_MACROTILE_MODE14__BANK_WIDTH_MASK                                                                  0x00000003L
+#define GB_MACROTILE_MODE14__BANK_HEIGHT_MASK                                                                 0x0000000CL
+#define GB_MACROTILE_MODE14__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
+#define GB_MACROTILE_MODE14__NUM_BANKS_MASK                                                                   0x000000C0L
+//GB_MACROTILE_MODE15
+#define GB_MACROTILE_MODE15__BANK_WIDTH__SHIFT                                                                0x0
+#define GB_MACROTILE_MODE15__BANK_HEIGHT__SHIFT                                                               0x2
+#define GB_MACROTILE_MODE15__MACRO_TILE_ASPECT__SHIFT                                                         0x4
+#define GB_MACROTILE_MODE15__NUM_BANKS__SHIFT                                                                 0x6
+#define GB_MACROTILE_MODE15__BANK_WIDTH_MASK                                                                  0x00000003L
+#define GB_MACROTILE_MODE15__BANK_HEIGHT_MASK                                                                 0x0000000CL
+#define GB_MACROTILE_MODE15__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
+#define GB_MACROTILE_MODE15__NUM_BANKS_MASK                                                                   0x000000C0L
+//CB_HW_CONTROL
+#define CB_HW_CONTROL__CM_CACHE_EVICT_POINT__SHIFT                                                            0x0
+#define CB_HW_CONTROL__FC_CACHE_EVICT_POINT__SHIFT                                                            0x6
+#define CB_HW_CONTROL__CC_CACHE_EVICT_POINT__SHIFT                                                            0xc
+#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE__SHIFT                                                      0x10
+#define CB_HW_CONTROL__DISABLE_INTNORM_LE11BPC_CLAMPING__SHIFT                                                0x12
+#define CB_HW_CONTROL__FORCE_NEEDS_DST__SHIFT                                                                 0x13
+#define CB_HW_CONTROL__FORCE_ALWAYS_TOGGLE__SHIFT                                                             0x14
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST__SHIFT                                                0x15
+#define CB_HW_CONTROL__DISABLE_FULL_WRITE_MASK__SHIFT                                                         0x16
+#define CB_HW_CONTROL__DISABLE_RESOLVE_OPT_FOR_SINGLE_FRAG__SHIFT                                             0x17
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT                                                   0x18
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS__SHIFT                                                        0x19
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT                                                 0x1a
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED__SHIFT                                0x1b
+#define CB_HW_CONTROL__PRIORITIZE_FC_WR_OVER_FC_RD_ON_CMASK_CONFLICT__SHIFT                                   0x1c
+#define CB_HW_CONTROL__PRIORITIZE_FC_EVICT_OVER_FOP_RD_ON_BANK_CONFLICT__SHIFT                                0x1d
+#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT__SHIFT                                              0x1e
+#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE__SHIFT                                    0x1f
+#define CB_HW_CONTROL__CM_CACHE_EVICT_POINT_MASK                                                              0x0000000FL
+#define CB_HW_CONTROL__FC_CACHE_EVICT_POINT_MASK                                                              0x000003C0L
+#define CB_HW_CONTROL__CC_CACHE_EVICT_POINT_MASK                                                              0x0000F000L
+#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE_MASK                                                        0x00010000L
+#define CB_HW_CONTROL__DISABLE_INTNORM_LE11BPC_CLAMPING_MASK                                                  0x00040000L
+#define CB_HW_CONTROL__FORCE_NEEDS_DST_MASK                                                                   0x00080000L
+#define CB_HW_CONTROL__FORCE_ALWAYS_TOGGLE_MASK                                                               0x00100000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST_MASK                                                  0x00200000L
+#define CB_HW_CONTROL__DISABLE_FULL_WRITE_MASK_MASK                                                           0x00400000L
+#define CB_HW_CONTROL__DISABLE_RESOLVE_OPT_FOR_SINGLE_FRAG_MASK                                               0x00800000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST_MASK                                                     0x01000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS_MASK                                                          0x02000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK                                                   0x04000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED_MASK                                  0x08000000L
+#define CB_HW_CONTROL__PRIORITIZE_FC_WR_OVER_FC_RD_ON_CMASK_CONFLICT_MASK                                     0x10000000L
+#define CB_HW_CONTROL__PRIORITIZE_FC_EVICT_OVER_FOP_RD_ON_BANK_CONFLICT_MASK                                  0x20000000L
+#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT_MASK                                                0x40000000L
+#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE_MASK                                      0x80000000L
+//CB_HW_CONTROL_1
+#define CB_HW_CONTROL_1__CM_CACHE_NUM_TAGS__SHIFT                                                             0x0
+#define CB_HW_CONTROL_1__FC_CACHE_NUM_TAGS__SHIFT                                                             0x5
+#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS__SHIFT                                                             0xb
+#define CB_HW_CONTROL_1__CM_TILE_FIFO_DEPTH__SHIFT                                                            0x11
+#define CB_HW_CONTROL_1__RMI_CREDITS__SHIFT                                                                   0x1a
+#define CB_HW_CONTROL_1__CM_CACHE_NUM_TAGS_MASK                                                               0x0000001FL
+#define CB_HW_CONTROL_1__FC_CACHE_NUM_TAGS_MASK                                                               0x000007E0L
+#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS_MASK                                                               0x0001F800L
+#define CB_HW_CONTROL_1__CM_TILE_FIFO_DEPTH_MASK                                                              0x03FE0000L
+#define CB_HW_CONTROL_1__RMI_CREDITS_MASK                                                                     0xFC000000L
+//CB_HW_CONTROL_2
+#define CB_HW_CONTROL_2__CC_EVEN_ODD_FIFO_DEPTH__SHIFT                                                        0x0
+#define CB_HW_CONTROL_2__FC_RDLAT_TILE_FIFO_DEPTH__SHIFT                                                      0x8
+#define CB_HW_CONTROL_2__FC_RDLAT_QUAD_FIFO_DEPTH__SHIFT                                                      0xf
+#define CB_HW_CONTROL_2__DRR_ASSUMED_FIFO_DEPTH_DIV8__SHIFT                                                   0x18
+#define CB_HW_CONTROL_2__CHICKEN_BITS__SHIFT                                                                  0x1c
+#define CB_HW_CONTROL_2__CC_EVEN_ODD_FIFO_DEPTH_MASK                                                          0x000000FFL
+#define CB_HW_CONTROL_2__FC_RDLAT_TILE_FIFO_DEPTH_MASK                                                        0x00007F00L
+#define CB_HW_CONTROL_2__FC_RDLAT_QUAD_FIFO_DEPTH_MASK                                                        0x007F8000L
+#define CB_HW_CONTROL_2__DRR_ASSUMED_FIFO_DEPTH_DIV8_MASK                                                     0x0F000000L
+#define CB_HW_CONTROL_2__CHICKEN_BITS_MASK                                                                    0xF0000000L
+//CB_HW_CONTROL_3
+#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL__SHIFT                                        0x0
+#define CB_HW_CONTROL_3__RAM_ADDRESS_CONFLICTS_DISALLOWED__SHIFT                                              0x1
+#define CB_HW_CONTROL_3__DISABLE_FAST_CLEAR_FETCH_OPT__SHIFT                                                  0x2
+#define CB_HW_CONTROL_3__DISABLE_QUAD_MARKER_DROP_STOP__SHIFT                                                 0x3
+#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_CAM_CLR__SHIFT                                            0x4
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_STATUS_ACCUM__SHIFT                                            0x5
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_KEY_MOD__SHIFT                                                 0x6
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_PANIC_GATING__SHIFT                                                 0x7
+#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_TARGET_MASK_VALIDATION__SHIFT                             0x8
+#define CB_HW_CONTROL_3__SPLIT_ALL_FAST_MODE_TRANSFERS__SHIFT                                                 0x9
+#define CB_HW_CONTROL_3__DISABLE_SHADER_BLEND_OPTS__SHIFT                                                     0xa
+#define CB_HW_CONTROL_3__DISABLE_CMASK_LAST_QUAD_INSERTION__SHIFT                                             0xb
+#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_511967__SHIFT                                              0xc
+#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_520657__SHIFT                                              0xd
+#define CB_HW_CONTROL_3__DISABLE_OC_FIXES_OF_BUG_522542__SHIFT                                                0xe
+#define CB_HW_CONTROL_3__FORCE_RMI_LAST_HIGH__SHIFT                                                           0xf
+#define CB_HW_CONTROL_3__FORCE_RMI_CLKEN_HIGH__SHIFT                                                          0x10
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_CC__SHIFT                                                       0x11
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_FC__SHIFT                                                       0x12
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_DC__SHIFT                                                       0x13
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_CM__SHIFT                                                       0x14
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CC__SHIFT                                                    0x15
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_FC__SHIFT                                                    0x16
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_DC__SHIFT                                                    0x17
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CM__SHIFT                                                    0x18
+#define CB_HW_CONTROL_3__DISABLE_NACK_COLOR_RD_WR_OPT__SHIFT                                                  0x19
+#define CB_HW_CONTROL_3__DISABLE_BLENDER_CLOCK_GATING__SHIFT                                                  0x1a
+#define CB_HW_CONTROL_3__DISABLE_DUALSRC_WITH_OBJPRIMID_FIX__SHIFT                                            0x1b
+#define CB_HW_CONTROL_3__COLOR_CACHE_PREFETCH_NUM_CLS__SHIFT                                                  0x1c
+#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL_MASK                                          0x00000001L
+#define CB_HW_CONTROL_3__RAM_ADDRESS_CONFLICTS_DISALLOWED_MASK                                                0x00000002L
+#define CB_HW_CONTROL_3__DISABLE_FAST_CLEAR_FETCH_OPT_MASK                                                    0x00000004L
+#define CB_HW_CONTROL_3__DISABLE_QUAD_MARKER_DROP_STOP_MASK                                                   0x00000008L
+#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_CAM_CLR_MASK                                              0x00000010L
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_STATUS_ACCUM_MASK                                              0x00000020L
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_KEY_MOD_MASK                                                   0x00000040L
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_PANIC_GATING_MASK                                                   0x00000080L
+#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_TARGET_MASK_VALIDATION_MASK                               0x00000100L
+#define CB_HW_CONTROL_3__SPLIT_ALL_FAST_MODE_TRANSFERS_MASK                                                   0x00000200L
+#define CB_HW_CONTROL_3__DISABLE_SHADER_BLEND_OPTS_MASK                                                       0x00000400L
+#define CB_HW_CONTROL_3__DISABLE_CMASK_LAST_QUAD_INSERTION_MASK                                               0x00000800L
+#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_511967_MASK                                                0x00001000L
+#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_520657_MASK                                                0x00002000L
+#define CB_HW_CONTROL_3__DISABLE_OC_FIXES_OF_BUG_522542_MASK                                                  0x00004000L
+#define CB_HW_CONTROL_3__FORCE_RMI_LAST_HIGH_MASK                                                             0x00008000L
+#define CB_HW_CONTROL_3__FORCE_RMI_CLKEN_HIGH_MASK                                                            0x00010000L
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_CC_MASK                                                         0x00020000L
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_FC_MASK                                                         0x00040000L
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_DC_MASK                                                         0x00080000L
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_CM_MASK                                                         0x00100000L
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CC_MASK                                                      0x00200000L
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_FC_MASK                                                      0x00400000L
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_DC_MASK                                                      0x00800000L
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CM_MASK                                                      0x01000000L
+#define CB_HW_CONTROL_3__DISABLE_NACK_COLOR_RD_WR_OPT_MASK                                                    0x02000000L
+#define CB_HW_CONTROL_3__DISABLE_BLENDER_CLOCK_GATING_MASK                                                    0x04000000L
+#define CB_HW_CONTROL_3__DISABLE_DUALSRC_WITH_OBJPRIMID_FIX_MASK                                              0x08000000L
+#define CB_HW_CONTROL_3__COLOR_CACHE_PREFETCH_NUM_CLS_MASK                                                    0x30000000L
+//CB_HW_MEM_ARBITER_RD
+#define CB_HW_MEM_ARBITER_RD__MODE__SHIFT                                                                     0x0
+#define CB_HW_MEM_ARBITER_RD__IGNORE_URGENT_AGE__SHIFT                                                        0x2
+#define CB_HW_MEM_ARBITER_RD__BREAK_GROUP_AGE__SHIFT                                                          0x6
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_CC__SHIFT                                                                0xa
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_FC__SHIFT                                                                0xc
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_CM__SHIFT                                                                0xe
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DC__SHIFT                                                                0x10
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_REQS__SHIFT                                                        0x12
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_NOREQS__SHIFT                                                      0x14
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_IGNORE_NUM_TIDS__SHIFT                                                   0x16
+#define CB_HW_MEM_ARBITER_RD__SCALE_AGE__SHIFT                                                                0x17
+#define CB_HW_MEM_ARBITER_RD__SCALE_WEIGHT__SHIFT                                                             0x1a
+#define CB_HW_MEM_ARBITER_RD__SEND_LASTS_WITHIN_GROUPS__SHIFT                                                 0x1d
+#define CB_HW_MEM_ARBITER_RD__MODE_MASK                                                                       0x00000003L
+#define CB_HW_MEM_ARBITER_RD__IGNORE_URGENT_AGE_MASK                                                          0x0000003CL
+#define CB_HW_MEM_ARBITER_RD__BREAK_GROUP_AGE_MASK                                                            0x000003C0L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_CC_MASK                                                                  0x00000C00L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_FC_MASK                                                                  0x00003000L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_CM_MASK                                                                  0x0000C000L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DC_MASK                                                                  0x00030000L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_REQS_MASK                                                          0x000C0000L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_NOREQS_MASK                                                        0x00300000L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_IGNORE_NUM_TIDS_MASK                                                     0x00400000L
+#define CB_HW_MEM_ARBITER_RD__SCALE_AGE_MASK                                                                  0x03800000L
+#define CB_HW_MEM_ARBITER_RD__SCALE_WEIGHT_MASK                                                               0x1C000000L
+#define CB_HW_MEM_ARBITER_RD__SEND_LASTS_WITHIN_GROUPS_MASK                                                   0x20000000L
+//CB_HW_MEM_ARBITER_WR
+#define CB_HW_MEM_ARBITER_WR__MODE__SHIFT                                                                     0x0
+#define CB_HW_MEM_ARBITER_WR__IGNORE_URGENT_AGE__SHIFT                                                        0x2
+#define CB_HW_MEM_ARBITER_WR__BREAK_GROUP_AGE__SHIFT                                                          0x6
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_CC__SHIFT                                                                0xa
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_FC__SHIFT                                                                0xc
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_CM__SHIFT                                                                0xe
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DC__SHIFT                                                                0x10
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_REQS__SHIFT                                                        0x12
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_NOREQS__SHIFT                                                      0x14
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_IGNORE_BYTE_MASK__SHIFT                                                  0x16
+#define CB_HW_MEM_ARBITER_WR__SCALE_AGE__SHIFT                                                                0x17
+#define CB_HW_MEM_ARBITER_WR__SCALE_WEIGHT__SHIFT                                                             0x1a
+#define CB_HW_MEM_ARBITER_WR__SEND_LASTS_WITHIN_GROUPS__SHIFT                                                 0x1d
+#define CB_HW_MEM_ARBITER_WR__MODE_MASK                                                                       0x00000003L
+#define CB_HW_MEM_ARBITER_WR__IGNORE_URGENT_AGE_MASK                                                          0x0000003CL
+#define CB_HW_MEM_ARBITER_WR__BREAK_GROUP_AGE_MASK                                                            0x000003C0L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_CC_MASK                                                                  0x00000C00L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_FC_MASK                                                                  0x00003000L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_CM_MASK                                                                  0x0000C000L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DC_MASK                                                                  0x00030000L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_REQS_MASK                                                          0x000C0000L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_NOREQS_MASK                                                        0x00300000L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_IGNORE_BYTE_MASK_MASK                                                    0x00400000L
+#define CB_HW_MEM_ARBITER_WR__SCALE_AGE_MASK                                                                  0x03800000L
+#define CB_HW_MEM_ARBITER_WR__SCALE_WEIGHT_MASK                                                               0x1C000000L
+#define CB_HW_MEM_ARBITER_WR__SEND_LASTS_WITHIN_GROUPS_MASK                                                   0x20000000L
+//CB_DCC_CONFIG
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DEPTH__SHIFT                                                        0x0
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DISABLE__SHIFT                                                      0x5
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_CC_POP_DISABLE__SHIFT                                               0x6
+#define CB_DCC_CONFIG__DISABLE_CONSTANT_ENCODE__SHIFT                                                         0x7
+#define CB_DCC_CONFIG__FC_RDLAT_KEYID_FIFO_DEPTH__SHIFT                                                       0x8
+#define CB_DCC_CONFIG__READ_RETURN_SKID_FIFO_DEPTH__SHIFT                                                     0x10
+#define CB_DCC_CONFIG__DCC_CACHE_EVICT_POINT__SHIFT                                                           0x18
+#define CB_DCC_CONFIG__DCC_CACHE_NUM_TAGS__SHIFT                                                              0x1c
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DEPTH_MASK                                                          0x0000001FL
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DISABLE_MASK                                                        0x00000020L
+#define CB_DCC_CONFIG__OVERWRITE_COMBINER_CC_POP_DISABLE_MASK                                                 0x00000040L
+#define CB_DCC_CONFIG__DISABLE_CONSTANT_ENCODE_MASK                                                           0x00000080L
+#define CB_DCC_CONFIG__FC_RDLAT_KEYID_FIFO_DEPTH_MASK                                                         0x0000FF00L
+#define CB_DCC_CONFIG__READ_RETURN_SKID_FIFO_DEPTH_MASK                                                       0x007F0000L
+#define CB_DCC_CONFIG__DCC_CACHE_EVICT_POINT_MASK                                                             0x0F000000L
+#define CB_DCC_CONFIG__DCC_CACHE_NUM_TAGS_MASK                                                                0xF0000000L
+//GC_USER_RB_REDUNDANCY
+#define GC_USER_RB_REDUNDANCY__FAILED_RB0__SHIFT                                                              0x8
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT                                                          0xc
+#define GC_USER_RB_REDUNDANCY__FAILED_RB1__SHIFT                                                              0x10
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT                                                          0x14
+#define GC_USER_RB_REDUNDANCY__FAILED_RB0_MASK                                                                0x00000F00L
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY0_MASK                                                            0x00001000L
+#define GC_USER_RB_REDUNDANCY__FAILED_RB1_MASK                                                                0x000F0000L
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY1_MASK                                                            0x00100000L
+//GC_USER_RB_BACKEND_DISABLE
+#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT                                                    0x10
+#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK                                                      0x00FF0000L
+
+
+// addressBlock: xcd0_gc_ea_gceadec
+//GCEA_DRAM_RD_CLI2GRP_MAP0
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//GCEA_DRAM_RD_CLI2GRP_MAP1
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//GCEA_DRAM_WR_CLI2GRP_MAP0
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//GCEA_DRAM_WR_CLI2GRP_MAP1
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//GCEA_DRAM_RD_GRP2VC_MAP
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//GCEA_DRAM_WR_GRP2VC_MAP
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//GCEA_DRAM_RD_LAZY
+#define GCEA_DRAM_RD_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define GCEA_DRAM_RD_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define GCEA_DRAM_RD_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define GCEA_DRAM_RD_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define GCEA_DRAM_RD_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define GCEA_DRAM_RD_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define GCEA_DRAM_RD_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define GCEA_DRAM_RD_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//GCEA_DRAM_WR_LAZY
+#define GCEA_DRAM_WR_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define GCEA_DRAM_WR_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define GCEA_DRAM_WR_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define GCEA_DRAM_WR_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define GCEA_DRAM_WR_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define GCEA_DRAM_WR_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define GCEA_DRAM_WR_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define GCEA_DRAM_WR_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//GCEA_DRAM_RD_CAM_CNTL
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define GCEA_DRAM_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define GCEA_DRAM_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+//GCEA_DRAM_WR_CAM_CNTL
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define GCEA_DRAM_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define GCEA_DRAM_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+//GCEA_DRAM_PAGE_BURST
+#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//GCEA_DRAM_RD_PRI_AGE
+#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//GCEA_DRAM_WR_PRI_AGE
+#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//GCEA_DRAM_RD_PRI_QUEUING
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//GCEA_DRAM_WR_PRI_QUEUING
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//GCEA_DRAM_RD_PRI_FIXED
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//GCEA_DRAM_WR_PRI_FIXED
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//GCEA_DRAM_RD_PRI_URGENCY
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//GCEA_DRAM_WR_PRI_URGENCY
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//GCEA_DRAM_RD_PRI_QUANT_PRI1
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_DRAM_RD_PRI_QUANT_PRI2
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_DRAM_RD_PRI_QUANT_PRI3
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_DRAM_WR_PRI_QUANT_PRI1
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_DRAM_WR_PRI_QUANT_PRI2
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_DRAM_WR_PRI_QUANT_PRI3
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_IO_RD_CLI2GRP_MAP0
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                            0x0
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                            0x2
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                            0x4
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                            0x6
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                            0x8
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                            0xa
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                            0xc
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                            0xe
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                            0x10
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                            0x12
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                           0x14
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                           0x16
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                           0x18
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                           0x1a
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                           0x1c
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                           0x1e
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                              0x00000003L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                              0x0000000CL
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                              0x00000030L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                              0x000000C0L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                              0x00000300L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                              0x00000C00L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                              0x00003000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                              0x0000C000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                              0x00030000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                              0x000C0000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                             0x00300000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                             0x00C00000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                             0x03000000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                             0x0C000000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                             0x30000000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                             0xC0000000L
+//GCEA_IO_RD_CLI2GRP_MAP1
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                           0x0
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                           0x2
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                           0x4
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                           0x6
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                           0x8
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                           0xa
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                           0xc
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                           0xe
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                           0x10
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                           0x12
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                           0x14
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                           0x16
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                           0x18
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                           0x1a
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                           0x1c
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                           0x1e
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                             0x00000003L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                             0x0000000CL
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                             0x00000030L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                             0x000000C0L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                             0x00000300L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                             0x00000C00L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                             0x00003000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                             0x0000C000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                             0x00030000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                             0x000C0000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                             0x00300000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                             0x00C00000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                             0x03000000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                             0x0C000000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                             0x30000000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                             0xC0000000L
+//GCEA_IO_WR_CLI2GRP_MAP0
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                            0x0
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                            0x2
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                            0x4
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                            0x6
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                            0x8
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                            0xa
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                            0xc
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                            0xe
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                            0x10
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                            0x12
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                           0x14
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                           0x16
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                           0x18
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                           0x1a
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                           0x1c
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                           0x1e
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                              0x00000003L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                              0x0000000CL
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                              0x00000030L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                              0x000000C0L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                              0x00000300L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                              0x00000C00L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                              0x00003000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                              0x0000C000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                              0x00030000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                              0x000C0000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                             0x00300000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                             0x00C00000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                             0x03000000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                             0x0C000000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                             0x30000000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                             0xC0000000L
+//GCEA_IO_WR_CLI2GRP_MAP1
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                           0x0
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                           0x2
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                           0x4
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                           0x6
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                           0x8
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                           0xa
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                           0xc
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                           0xe
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                           0x10
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                           0x12
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                           0x14
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                           0x16
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                           0x18
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                           0x1a
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                           0x1c
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                           0x1e
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                             0x00000003L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                             0x0000000CL
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                             0x00000030L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                             0x000000C0L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                             0x00000300L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                             0x00000C00L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                             0x00003000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                             0x0000C000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                             0x00030000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                             0x000C0000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                             0x00300000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                             0x00C00000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                             0x03000000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                             0x0C000000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                             0x30000000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                             0xC0000000L
+//GCEA_IO_RD_COMBINE_FLUSH
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                         0x0
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                         0x4
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                         0x8
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                         0xc
+#define GCEA_IO_RD_COMBINE_FLUSH__COMB_MODE__SHIFT                                                            0x10
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                           0x0000000FL
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                           0x000000F0L
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                           0x00000F00L
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                           0x0000F000L
+#define GCEA_IO_RD_COMBINE_FLUSH__COMB_MODE_MASK                                                              0x00030000L
+//GCEA_IO_WR_COMBINE_FLUSH
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                         0x0
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                         0x4
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                         0x8
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                         0xc
+#define GCEA_IO_WR_COMBINE_FLUSH__COMB_MODE__SHIFT                                                            0x10
+#define GCEA_IO_WR_COMBINE_FLUSH__DISABLE_MAM_CHAINING__SHIFT                                                 0x12
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                           0x0000000FL
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                           0x000000F0L
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                           0x00000F00L
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                           0x0000F000L
+#define GCEA_IO_WR_COMBINE_FLUSH__COMB_MODE_MASK                                                              0x00030000L
+#define GCEA_IO_WR_COMBINE_FLUSH__DISABLE_MAM_CHAINING_MASK                                                   0x00040000L
+//GCEA_IO_GROUP_BURST
+#define GCEA_IO_GROUP_BURST__RD_LIMIT_LO__SHIFT                                                               0x0
+#define GCEA_IO_GROUP_BURST__RD_LIMIT_HI__SHIFT                                                               0x8
+#define GCEA_IO_GROUP_BURST__WR_LIMIT_LO__SHIFT                                                               0x10
+#define GCEA_IO_GROUP_BURST__WR_LIMIT_HI__SHIFT                                                               0x18
+#define GCEA_IO_GROUP_BURST__RD_LIMIT_LO_MASK                                                                 0x000000FFL
+#define GCEA_IO_GROUP_BURST__RD_LIMIT_HI_MASK                                                                 0x0000FF00L
+#define GCEA_IO_GROUP_BURST__WR_LIMIT_LO_MASK                                                                 0x00FF0000L
+#define GCEA_IO_GROUP_BURST__WR_LIMIT_HI_MASK                                                                 0xFF000000L
+//GCEA_IO_RD_PRI_AGE
+#define GCEA_IO_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                          0x0
+#define GCEA_IO_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                          0x3
+#define GCEA_IO_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                          0x6
+#define GCEA_IO_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                          0x9
+#define GCEA_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                     0xc
+#define GCEA_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                     0xf
+#define GCEA_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                     0x12
+#define GCEA_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                     0x15
+#define GCEA_IO_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                            0x00000007L
+#define GCEA_IO_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                            0x00000038L
+#define GCEA_IO_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                            0x000001C0L
+#define GCEA_IO_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                            0x00000E00L
+#define GCEA_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                       0x00007000L
+#define GCEA_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                       0x00038000L
+#define GCEA_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                       0x001C0000L
+#define GCEA_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                       0x00E00000L
+//GCEA_IO_WR_PRI_AGE
+#define GCEA_IO_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                          0x0
+#define GCEA_IO_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                          0x3
+#define GCEA_IO_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                          0x6
+#define GCEA_IO_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                          0x9
+#define GCEA_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                     0xc
+#define GCEA_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                     0xf
+#define GCEA_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                     0x12
+#define GCEA_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                     0x15
+#define GCEA_IO_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                            0x00000007L
+#define GCEA_IO_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                            0x00000038L
+#define GCEA_IO_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                            0x000001C0L
+#define GCEA_IO_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                            0x00000E00L
+#define GCEA_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                       0x00007000L
+#define GCEA_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                       0x00038000L
+#define GCEA_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                       0x001C0000L
+#define GCEA_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                       0x00E00000L
+//GCEA_IO_RD_PRI_QUEUING
+#define GCEA_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                             0x0
+#define GCEA_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                             0x3
+#define GCEA_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                             0x6
+#define GCEA_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                             0x9
+#define GCEA_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                               0x00000007L
+#define GCEA_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                               0x00000038L
+#define GCEA_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                               0x000001C0L
+#define GCEA_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                               0x00000E00L
+//GCEA_IO_WR_PRI_QUEUING
+#define GCEA_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                             0x0
+#define GCEA_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                             0x3
+#define GCEA_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                             0x6
+#define GCEA_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                             0x9
+#define GCEA_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                               0x00000007L
+#define GCEA_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                               0x00000038L
+#define GCEA_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                               0x000001C0L
+#define GCEA_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                               0x00000E00L
+//GCEA_IO_RD_PRI_FIXED
+#define GCEA_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                 0x0
+#define GCEA_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                 0x3
+#define GCEA_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                 0x6
+#define GCEA_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                 0x9
+#define GCEA_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                   0x00000007L
+#define GCEA_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                   0x00000038L
+#define GCEA_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                   0x000001C0L
+#define GCEA_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                   0x00000E00L
+//GCEA_IO_WR_PRI_FIXED
+#define GCEA_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                 0x0
+#define GCEA_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                 0x3
+#define GCEA_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                 0x6
+#define GCEA_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                 0x9
+#define GCEA_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                   0x00000007L
+#define GCEA_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                   0x00000038L
+#define GCEA_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                   0x000001C0L
+#define GCEA_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                   0x00000E00L
+//GCEA_IO_RD_PRI_URGENCY
+#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                             0x0
+#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                             0x3
+#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                             0x6
+#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                             0x9
+#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                    0xc
+#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                    0xd
+#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                    0xe
+#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                    0xf
+#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                               0x00000007L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                               0x00000038L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                               0x000001C0L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                               0x00000E00L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                      0x00001000L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                      0x00002000L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                      0x00004000L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                      0x00008000L
+//GCEA_IO_WR_PRI_URGENCY
+#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                             0x0
+#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                             0x3
+#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                             0x6
+#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                             0x9
+#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                    0xc
+#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                    0xd
+#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                    0xe
+#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                    0xf
+#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                               0x00000007L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                               0x00000038L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                               0x000001C0L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                               0x00000E00L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                      0x00001000L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                      0x00002000L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                      0x00004000L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                      0x00008000L
+//GCEA_IO_RD_PRI_URGENCY_MASKING
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                      0x0
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                      0x1
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                      0x2
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                      0x3
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                      0x4
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                      0x5
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                      0x6
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                      0x7
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                      0x8
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                      0x9
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                     0xa
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                     0xb
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                     0xc
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                     0xd
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                     0xe
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                     0xf
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                     0x10
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                     0x11
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                     0x12
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                     0x13
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                     0x14
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                     0x15
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                     0x16
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                     0x17
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                     0x18
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                     0x19
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                     0x1a
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                     0x1b
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                     0x1c
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                     0x1d
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                     0x1e
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                     0x1f
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                        0x00000001L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                        0x00000002L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                        0x00000004L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                        0x00000008L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                        0x00000010L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                        0x00000020L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                        0x00000040L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                        0x00000080L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                        0x00000100L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                        0x00000200L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                       0x00000400L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                       0x00000800L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                       0x00001000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                       0x00002000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                       0x00004000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                       0x00008000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                       0x00010000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                       0x00020000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                       0x00040000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                       0x00080000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                       0x00100000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                       0x00200000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                       0x00400000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                       0x00800000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                       0x01000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                       0x02000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                       0x04000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                       0x08000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                       0x10000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                       0x20000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                       0x40000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                       0x80000000L
+//GCEA_IO_WR_PRI_URGENCY_MASKING
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                      0x0
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                      0x1
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                      0x2
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                      0x3
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                      0x4
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                      0x5
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                      0x6
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                      0x7
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                      0x8
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                      0x9
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                     0xa
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                     0xb
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                     0xc
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                     0xd
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                     0xe
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                     0xf
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                     0x10
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                     0x11
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                     0x12
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                     0x13
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                     0x14
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                     0x15
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                     0x16
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                     0x17
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                     0x18
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                     0x19
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                     0x1a
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                     0x1b
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                     0x1c
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                     0x1d
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                     0x1e
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                     0x1f
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                        0x00000001L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                        0x00000002L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                        0x00000004L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                        0x00000008L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                        0x00000010L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                        0x00000020L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                        0x00000040L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                        0x00000080L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                        0x00000100L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                        0x00000200L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                       0x00000400L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                       0x00000800L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                       0x00001000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                       0x00002000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                       0x00004000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                       0x00008000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                       0x00010000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                       0x00020000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                       0x00040000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                       0x00080000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                       0x00100000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                       0x00200000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                       0x00400000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                       0x00800000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                       0x01000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                       0x02000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                       0x04000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                       0x08000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                       0x10000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                       0x20000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                       0x40000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                       0x80000000L
+//GCEA_IO_RD_PRI_QUANT_PRI1
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_IO_RD_PRI_QUANT_PRI2
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_IO_RD_PRI_QUANT_PRI3
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_IO_WR_PRI_QUANT_PRI1
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_IO_WR_PRI_QUANT_PRI2
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_IO_WR_PRI_QUANT_PRI3
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_SDP_ARB_DRAM
+#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL__SHIFT                                                       0x0
+#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA__SHIFT                                                       0x8
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI__SHIFT                                                          0x10
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI__SHIFT                                                          0x11
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES__SHIFT                                                          0x12
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES__SHIFT                                                          0x13
+#define GCEA_SDP_ARB_DRAM__EOB_ON_EXPIRE__SHIFT                                                               0x14
+#define GCEA_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                      0x15
+#define GCEA_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING__SHIFT                                                        0x16
+#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL_MASK                                                         0x0000007FL
+#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA_MASK                                                         0x00007F00L
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI_MASK                                                            0x00010000L
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI_MASK                                                            0x00020000L
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES_MASK                                                            0x00040000L
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES_MASK                                                            0x00080000L
+#define GCEA_SDP_ARB_DRAM__EOB_ON_EXPIRE_MASK                                                                 0x00100000L
+#define GCEA_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE_MASK                                                        0x00200000L
+#define GCEA_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING_MASK                                                          0x00400000L
+//GCEA_SDP_ARB_FINAL
+#define GCEA_SDP_ARB_FINAL__DRAM_BURST_LIMIT__SHIFT                                                           0x0
+#define GCEA_SDP_ARB_FINAL__GMI_BURST_LIMIT__SHIFT                                                            0x5
+#define GCEA_SDP_ARB_FINAL__IO_BURST_LIMIT__SHIFT                                                             0xa
+#define GCEA_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER__SHIFT                                                     0xf
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC0__SHIFT                                                                 0x11
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC1__SHIFT                                                                 0x12
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC2__SHIFT                                                                 0x13
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC3__SHIFT                                                                 0x14
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC4__SHIFT                                                                 0x15
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC5__SHIFT                                                                 0x16
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC6__SHIFT                                                                 0x17
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC7__SHIFT                                                                 0x18
+#define GCEA_SDP_ARB_FINAL__ERREVENT_ON_ERROR__SHIFT                                                          0x19
+#define GCEA_SDP_ARB_FINAL__HALTREQ_ON_ERROR__SHIFT                                                           0x1a
+#define GCEA_SDP_ARB_FINAL__DRAM_BURST_STRETCH__SHIFT                                                         0x1b
+#define GCEA_SDP_ARB_FINAL__GMI_BURST_STRETCH__SHIFT                                                          0x1c
+#define GCEA_SDP_ARB_FINAL__DRAM_BURST_LIMIT_MASK                                                             0x0000001FL
+#define GCEA_SDP_ARB_FINAL__GMI_BURST_LIMIT_MASK                                                              0x000003E0L
+#define GCEA_SDP_ARB_FINAL__IO_BURST_LIMIT_MASK                                                               0x00007C00L
+#define GCEA_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER_MASK                                                       0x00018000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC0_MASK                                                                   0x00020000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC1_MASK                                                                   0x00040000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC2_MASK                                                                   0x00080000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC3_MASK                                                                   0x00100000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC4_MASK                                                                   0x00200000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC5_MASK                                                                   0x00400000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC6_MASK                                                                   0x00800000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC7_MASK                                                                   0x01000000L
+#define GCEA_SDP_ARB_FINAL__ERREVENT_ON_ERROR_MASK                                                            0x02000000L
+#define GCEA_SDP_ARB_FINAL__HALTREQ_ON_ERROR_MASK                                                             0x04000000L
+#define GCEA_SDP_ARB_FINAL__DRAM_BURST_STRETCH_MASK                                                           0x08000000L
+#define GCEA_SDP_ARB_FINAL__GMI_BURST_STRETCH_MASK                                                            0x10000000L
+//GCEA_SDP_DRAM_PRIORITY
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                     0x0
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                     0x4
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                     0x8
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                     0xc
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                     0x10
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                     0x14
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                     0x18
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                     0x1c
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                       0x0000000FL
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                       0x000000F0L
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                       0x00000F00L
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                       0x0000F000L
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                       0x000F0000L
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                       0x00F00000L
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                       0x0F000000L
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                       0xF0000000L
+//GCEA_SDP_IO_PRIORITY
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                       0x0
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                       0x4
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                       0x8
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                       0xc
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                       0x10
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                       0x14
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                       0x18
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                       0x1c
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                         0x0000000FL
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                         0x000000F0L
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                         0x00000F00L
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                         0x0000F000L
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                         0x000F0000L
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                         0x00F00000L
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                         0x0F000000L
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                         0xF0000000L
+//GCEA_SDP_CREDITS
+#define GCEA_SDP_CREDITS__TAG_LIMIT__SHIFT                                                                    0x0
+#define GCEA_SDP_CREDITS__WR_RESP_CREDITS__SHIFT                                                              0x8
+#define GCEA_SDP_CREDITS__RD_RESP_CREDITS__SHIFT                                                              0x10
+#define GCEA_SDP_CREDITS__PRB_REQ_CREDITS__SHIFT                                                              0x18
+#define GCEA_SDP_CREDITS__TAG_LIMIT_MASK                                                                      0x000000FFL
+#define GCEA_SDP_CREDITS__WR_RESP_CREDITS_MASK                                                                0x00007F00L
+#define GCEA_SDP_CREDITS__RD_RESP_CREDITS_MASK                                                                0x007F0000L
+#define GCEA_SDP_CREDITS__PRB_REQ_CREDITS_MASK                                                                0x3F000000L
+//GCEA_SDP_TAG_RESERVE0
+#define GCEA_SDP_TAG_RESERVE0__VC0__SHIFT                                                                     0x0
+#define GCEA_SDP_TAG_RESERVE0__VC1__SHIFT                                                                     0x8
+#define GCEA_SDP_TAG_RESERVE0__VC2__SHIFT                                                                     0x10
+#define GCEA_SDP_TAG_RESERVE0__VC3__SHIFT                                                                     0x18
+#define GCEA_SDP_TAG_RESERVE0__VC0_MASK                                                                       0x000000FFL
+#define GCEA_SDP_TAG_RESERVE0__VC1_MASK                                                                       0x0000FF00L
+#define GCEA_SDP_TAG_RESERVE0__VC2_MASK                                                                       0x00FF0000L
+#define GCEA_SDP_TAG_RESERVE0__VC3_MASK                                                                       0xFF000000L
+//GCEA_SDP_TAG_RESERVE1
+#define GCEA_SDP_TAG_RESERVE1__VC4__SHIFT                                                                     0x0
+#define GCEA_SDP_TAG_RESERVE1__VC5__SHIFT                                                                     0x8
+#define GCEA_SDP_TAG_RESERVE1__VC6__SHIFT                                                                     0x10
+#define GCEA_SDP_TAG_RESERVE1__VC7__SHIFT                                                                     0x18
+#define GCEA_SDP_TAG_RESERVE1__VC4_MASK                                                                       0x000000FFL
+#define GCEA_SDP_TAG_RESERVE1__VC5_MASK                                                                       0x0000FF00L
+#define GCEA_SDP_TAG_RESERVE1__VC6_MASK                                                                       0x00FF0000L
+#define GCEA_SDP_TAG_RESERVE1__VC7_MASK                                                                       0xFF000000L
+//GCEA_SDP_VCC_RESERVE0
+#define GCEA_SDP_VCC_RESERVE0__VC0_CREDITS__SHIFT                                                             0x0
+#define GCEA_SDP_VCC_RESERVE0__VC1_CREDITS__SHIFT                                                             0x6
+#define GCEA_SDP_VCC_RESERVE0__VC2_CREDITS__SHIFT                                                             0xc
+#define GCEA_SDP_VCC_RESERVE0__VC3_CREDITS__SHIFT                                                             0x12
+#define GCEA_SDP_VCC_RESERVE0__VC4_CREDITS__SHIFT                                                             0x18
+#define GCEA_SDP_VCC_RESERVE0__VC0_CREDITS_MASK                                                               0x0000003FL
+#define GCEA_SDP_VCC_RESERVE0__VC1_CREDITS_MASK                                                               0x00000FC0L
+#define GCEA_SDP_VCC_RESERVE0__VC2_CREDITS_MASK                                                               0x0003F000L
+#define GCEA_SDP_VCC_RESERVE0__VC3_CREDITS_MASK                                                               0x00FC0000L
+#define GCEA_SDP_VCC_RESERVE0__VC4_CREDITS_MASK                                                               0x3F000000L
+//GCEA_SDP_VCC_RESERVE1
+#define GCEA_SDP_VCC_RESERVE1__VC5_CREDITS__SHIFT                                                             0x0
+#define GCEA_SDP_VCC_RESERVE1__VC6_CREDITS__SHIFT                                                             0x6
+#define GCEA_SDP_VCC_RESERVE1__VC7_CREDITS__SHIFT                                                             0xc
+#define GCEA_SDP_VCC_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                         0x1f
+#define GCEA_SDP_VCC_RESERVE1__VC5_CREDITS_MASK                                                               0x0000003FL
+#define GCEA_SDP_VCC_RESERVE1__VC6_CREDITS_MASK                                                               0x00000FC0L
+#define GCEA_SDP_VCC_RESERVE1__VC7_CREDITS_MASK                                                               0x0003F000L
+#define GCEA_SDP_VCC_RESERVE1__DISTRIBUTE_POOL_MASK                                                           0x80000000L
+//GCEA_SDP_VCD_RESERVE0
+#define GCEA_SDP_VCD_RESERVE0__VC0_CREDITS__SHIFT                                                             0x0
+#define GCEA_SDP_VCD_RESERVE0__VC1_CREDITS__SHIFT                                                             0x6
+#define GCEA_SDP_VCD_RESERVE0__VC2_CREDITS__SHIFT                                                             0xc
+#define GCEA_SDP_VCD_RESERVE0__VC3_CREDITS__SHIFT                                                             0x12
+#define GCEA_SDP_VCD_RESERVE0__VC4_CREDITS__SHIFT                                                             0x18
+#define GCEA_SDP_VCD_RESERVE0__VC0_CREDITS_MASK                                                               0x0000003FL
+#define GCEA_SDP_VCD_RESERVE0__VC1_CREDITS_MASK                                                               0x00000FC0L
+#define GCEA_SDP_VCD_RESERVE0__VC2_CREDITS_MASK                                                               0x0003F000L
+#define GCEA_SDP_VCD_RESERVE0__VC3_CREDITS_MASK                                                               0x00FC0000L
+#define GCEA_SDP_VCD_RESERVE0__VC4_CREDITS_MASK                                                               0x3F000000L
+//GCEA_SDP_VCD_RESERVE1
+#define GCEA_SDP_VCD_RESERVE1__VC5_CREDITS__SHIFT                                                             0x0
+#define GCEA_SDP_VCD_RESERVE1__VC6_CREDITS__SHIFT                                                             0x6
+#define GCEA_SDP_VCD_RESERVE1__VC7_CREDITS__SHIFT                                                             0xc
+#define GCEA_SDP_VCD_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                         0x1f
+#define GCEA_SDP_VCD_RESERVE1__VC5_CREDITS_MASK                                                               0x0000003FL
+#define GCEA_SDP_VCD_RESERVE1__VC6_CREDITS_MASK                                                               0x00000FC0L
+#define GCEA_SDP_VCD_RESERVE1__VC7_CREDITS_MASK                                                               0x0003F000L
+#define GCEA_SDP_VCD_RESERVE1__DISTRIBUTE_POOL_MASK                                                           0x80000000L
+//GCEA_SDP_REQ_CNTL
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ__SHIFT                                                   0x0
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE__SHIFT                                                  0x1
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC__SHIFT                                                 0x2
+#define GCEA_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM__SHIFT                                                     0x3
+#define GCEA_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI__SHIFT                                                      0x4
+#define GCEA_SDP_REQ_CNTL__INNER_DOMAIN_MODE__SHIFT                                                           0x5
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ__SHIFT                                                        0x6
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE__SHIFT                                                       0x8
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC__SHIFT                                                      0xa
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ_MASK                                                     0x00000001L
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE_MASK                                                    0x00000002L
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC_MASK                                                   0x00000004L
+#define GCEA_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM_MASK                                                       0x00000008L
+#define GCEA_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI_MASK                                                        0x00000010L
+#define GCEA_SDP_REQ_CNTL__INNER_DOMAIN_MODE_MASK                                                             0x00000020L
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ_MASK                                                          0x000000C0L
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE_MASK                                                         0x00000300L
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC_MASK                                                        0x00000C00L
+//GCEA_MISC
+#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB__SHIFT                                                         0x0
+#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB__SHIFT                                                         0x1
+#define GCEA_MISC__RELATIVE_PRI_IN_GMI_RD_ARB__SHIFT                                                          0x2
+#define GCEA_MISC__RELATIVE_PRI_IN_GMI_WR_ARB__SHIFT                                                          0x3
+#define GCEA_MISC__RELATIVE_PRI_IN_IO_RD_ARB__SHIFT                                                           0x4
+#define GCEA_MISC__RELATIVE_PRI_IN_IO_WR_ARB__SHIFT                                                           0x5
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC0__SHIFT                                                               0x6
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC1__SHIFT                                                               0x7
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC2__SHIFT                                                               0x8
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC3__SHIFT                                                               0x9
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC4__SHIFT                                                               0xa
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC5__SHIFT                                                               0xb
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC6__SHIFT                                                               0xc
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC7__SHIFT                                                               0xd
+#define GCEA_MISC__EARLY_SDP_ORIGDATA__SHIFT                                                                  0xe
+#define GCEA_MISC__LINKMGR_DYNAMIC_MODE__SHIFT                                                                0xf
+#define GCEA_MISC__LINKMGR_HALT_THRESHOLD__SHIFT                                                              0x11
+#define GCEA_MISC__LINKMGR_RECONNECT_DELAY__SHIFT                                                             0x13
+#define GCEA_MISC__LINKMGR_IDLE_THRESHOLD__SHIFT                                                              0x15
+#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB__SHIFT                                                      0x1a
+#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB__SHIFT                                                       0x1b
+#define GCEA_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB__SHIFT                                                          0x1c
+#define GCEA_MISC__FAVOUR_LAST_CS_IN_GMI_ARB__SHIFT                                                           0x1d
+#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB__SHIFT                                                        0x1e
+#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB__SHIFT                                                         0x1f
+#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB_MASK                                                           0x00000001L
+#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB_MASK                                                           0x00000002L
+#define GCEA_MISC__RELATIVE_PRI_IN_GMI_RD_ARB_MASK                                                            0x00000004L
+#define GCEA_MISC__RELATIVE_PRI_IN_GMI_WR_ARB_MASK                                                            0x00000008L
+#define GCEA_MISC__RELATIVE_PRI_IN_IO_RD_ARB_MASK                                                             0x00000010L
+#define GCEA_MISC__RELATIVE_PRI_IN_IO_WR_ARB_MASK                                                             0x00000020L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC0_MASK                                                                 0x00000040L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC1_MASK                                                                 0x00000080L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC2_MASK                                                                 0x00000100L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC3_MASK                                                                 0x00000200L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC4_MASK                                                                 0x00000400L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC5_MASK                                                                 0x00000800L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC6_MASK                                                                 0x00001000L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC7_MASK                                                                 0x00002000L
+#define GCEA_MISC__EARLY_SDP_ORIGDATA_MASK                                                                    0x00004000L
+#define GCEA_MISC__LINKMGR_DYNAMIC_MODE_MASK                                                                  0x00018000L
+#define GCEA_MISC__LINKMGR_HALT_THRESHOLD_MASK                                                                0x00060000L
+#define GCEA_MISC__LINKMGR_RECONNECT_DELAY_MASK                                                               0x00180000L
+#define GCEA_MISC__LINKMGR_IDLE_THRESHOLD_MASK                                                                0x03E00000L
+#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB_MASK                                                        0x04000000L
+#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB_MASK                                                         0x08000000L
+#define GCEA_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB_MASK                                                            0x10000000L
+#define GCEA_MISC__FAVOUR_LAST_CS_IN_GMI_ARB_MASK                                                             0x20000000L
+#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB_MASK                                                          0x40000000L
+#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB_MASK                                                           0x80000000L
+//GCEA_LATENCY_SAMPLING
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_DRAM__SHIFT                                                           0x0
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_DRAM__SHIFT                                                           0x1
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_GMI__SHIFT                                                            0x2
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_GMI__SHIFT                                                            0x3
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_IO__SHIFT                                                             0x4
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_IO__SHIFT                                                             0x5
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_READ__SHIFT                                                           0x6
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_READ__SHIFT                                                           0x7
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_WRITE__SHIFT                                                          0x8
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_WRITE__SHIFT                                                          0x9
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET__SHIFT                                                     0xa
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET__SHIFT                                                     0xb
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET__SHIFT                                                   0xc
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET__SHIFT                                                   0xd
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_VC__SHIFT                                                             0xe
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_VC__SHIFT                                                             0x16
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_DRAM_MASK                                                             0x00000001L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_DRAM_MASK                                                             0x00000002L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_GMI_MASK                                                              0x00000004L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_GMI_MASK                                                              0x00000008L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_IO_MASK                                                               0x00000010L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_IO_MASK                                                               0x00000020L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_READ_MASK                                                             0x00000040L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_READ_MASK                                                             0x00000080L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_WRITE_MASK                                                            0x00000100L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_WRITE_MASK                                                            0x00000200L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET_MASK                                                       0x00000400L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET_MASK                                                       0x00000800L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET_MASK                                                     0x00001000L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET_MASK                                                     0x00002000L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_VC_MASK                                                               0x003FC000L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_VC_MASK                                                               0x3FC00000L
+//GCEA_PERFCOUNTER_LO
+#define GCEA_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                                0x0
+#define GCEA_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                  0xFFFFFFFFL
+//GCEA_PERFCOUNTER_HI
+#define GCEA_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                                0x0
+#define GCEA_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                             0x10
+#define GCEA_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                  0x0000FFFFL
+#define GCEA_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                               0xFFFF0000L
+//GCEA_PERFCOUNTER0_CFG
+#define GCEA_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                                0x0
+#define GCEA_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                            0x8
+#define GCEA_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                               0x18
+#define GCEA_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                  0x1c
+#define GCEA_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                   0x1d
+#define GCEA_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                  0x000000FFL
+#define GCEA_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                              0x0000FF00L
+#define GCEA_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                 0x0F000000L
+#define GCEA_PERFCOUNTER0_CFG__ENABLE_MASK                                                                    0x10000000L
+#define GCEA_PERFCOUNTER0_CFG__CLEAR_MASK                                                                     0x20000000L
+//GCEA_PERFCOUNTER1_CFG
+#define GCEA_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                                0x0
+#define GCEA_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                            0x8
+#define GCEA_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                               0x18
+#define GCEA_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                  0x1c
+#define GCEA_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                   0x1d
+#define GCEA_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                  0x000000FFL
+#define GCEA_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                              0x0000FF00L
+#define GCEA_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                 0x0F000000L
+#define GCEA_PERFCOUNTER1_CFG__ENABLE_MASK                                                                    0x10000000L
+#define GCEA_PERFCOUNTER1_CFG__CLEAR_MASK                                                                     0x20000000L
+
+
+// addressBlock: xcd0_gc_ea_gceadec2
+//GCEA_PERFCOUNTER_RSLT_CNTL
+#define GCEA_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                                0x0
+#define GCEA_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                      0x8
+#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                       0x10
+#define GCEA_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                         0x18
+#define GCEA_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                          0x19
+#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                               0x1a
+#define GCEA_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                  0x0000000FL
+#define GCEA_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                        0x0000FF00L
+#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                         0x00FF0000L
+#define GCEA_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                           0x01000000L
+#define GCEA_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                            0x02000000L
+#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                 0x04000000L
+//GCEA_MAM_CTRL
+#define GCEA_MAM_CTRL__ADRAM_MODE__SHIFT                                                                      0x0
+#define GCEA_MAM_CTRL__ADRAM_COALESCE_DISABLE__SHIFT                                                          0x2
+#define GCEA_MAM_CTRL__ARAM_FLUSH_CNTR_THRESHOLD__SHIFT                                                       0x3
+#define GCEA_MAM_CTRL__ARAM_FLUSH_CNTR_DISABLE__SHIFT                                                         0x6
+#define GCEA_MAM_CTRL__ARAM_FORCE_FLUSH__SHIFT                                                                0x7
+#define GCEA_MAM_CTRL__ALOG_MODE1_FILTER1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_MAM_CTRL__ALOG_MODE1_FILTER2_BYPASS__SHIFT                                                       0xb
+#define GCEA_MAM_CTRL__ALOG_ACTIVE__SHIFT                                                                     0xc
+#define GCEA_MAM_CTRL__SDP_PRIORITY__SHIFT                                                                    0xd
+#define GCEA_MAM_CTRL__CLIENT_ID__SHIFT                                                                       0x11
+#define GCEA_MAM_CTRL__MAM_DISABLE__SHIFT                                                                     0x16
+#define GCEA_MAM_CTRL__ARAM_FLUSH_RB_SIZE__SHIFT                                                              0x17
+#define GCEA_MAM_CTRL__ALOG_MODE__SHIFT                                                                       0x1b
+#define GCEA_MAM_CTRL__ALOG_MODE2_LOCK_WINDOW__SHIFT                                                          0x1c
+#define GCEA_MAM_CTRL__ALOG_TRACK_2M_SEGMENT__SHIFT                                                           0x1f
+#define GCEA_MAM_CTRL__ADRAM_MODE_MASK                                                                        0x00000003L
+#define GCEA_MAM_CTRL__ADRAM_COALESCE_DISABLE_MASK                                                            0x00000004L
+#define GCEA_MAM_CTRL__ARAM_FLUSH_CNTR_THRESHOLD_MASK                                                         0x00000038L
+#define GCEA_MAM_CTRL__ARAM_FLUSH_CNTR_DISABLE_MASK                                                           0x00000040L
+#define GCEA_MAM_CTRL__ARAM_FORCE_FLUSH_MASK                                                                  0x00000080L
+#define GCEA_MAM_CTRL__ALOG_MODE1_FILTER1_THRESHOLD_MASK                                                      0x00000700L
+#define GCEA_MAM_CTRL__ALOG_MODE1_FILTER2_BYPASS_MASK                                                         0x00000800L
+#define GCEA_MAM_CTRL__ALOG_ACTIVE_MASK                                                                       0x00001000L
+#define GCEA_MAM_CTRL__SDP_PRIORITY_MASK                                                                      0x0001E000L
+#define GCEA_MAM_CTRL__CLIENT_ID_MASK                                                                         0x003E0000L
+#define GCEA_MAM_CTRL__MAM_DISABLE_MASK                                                                       0x00400000L
+#define GCEA_MAM_CTRL__ARAM_FLUSH_RB_SIZE_MASK                                                                0x07800000L
+#define GCEA_MAM_CTRL__ALOG_MODE_MASK                                                                         0x08000000L
+#define GCEA_MAM_CTRL__ALOG_MODE2_LOCK_WINDOW_MASK                                                            0x70000000L
+#define GCEA_MAM_CTRL__ALOG_TRACK_2M_SEGMENT_MASK                                                             0x80000000L
+//GCEA_MAM_CTRL2
+#define GCEA_MAM_CTRL2__ALOG_MODE2_INTR_THRESHOLD__SHIFT                                                      0x0
+#define GCEA_MAM_CTRL2__ALOG_SPACE_EN__SHIFT                                                                  0x2
+#define GCEA_MAM_CTRL2__ARAM_FLUSH_SNOOP_EN__SHIFT                                                            0x5
+#define GCEA_MAM_CTRL2__ARAM_FLUSH_NOALLOC__SHIFT                                                             0x6
+#define GCEA_MAM_CTRL2__RESERVED_FIELD__SHIFT                                                                 0x7
+#define GCEA_MAM_CTRL2__ADDR_HI__SHIFT                                                                        0x18
+#define GCEA_MAM_CTRL2__ALOG_MODE2_INTR_THRESHOLD_MASK                                                        0x00000003L
+#define GCEA_MAM_CTRL2__ALOG_SPACE_EN_MASK                                                                    0x0000001CL
+#define GCEA_MAM_CTRL2__ARAM_FLUSH_SNOOP_EN_MASK                                                              0x00000020L
+#define GCEA_MAM_CTRL2__ARAM_FLUSH_NOALLOC_MASK                                                               0x00000040L
+#define GCEA_MAM_CTRL2__RESERVED_FIELD_MASK                                                                   0x00FFFF80L
+#define GCEA_MAM_CTRL2__ADDR_HI_MASK                                                                          0xFF000000L
+//GCEA_DSM_CNTL
+#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x0
+#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x2
+#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x3
+#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x5
+#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x6
+#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x8
+#define GCEA_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x9
+#define GCEA_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xb
+#define GCEA_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                  0xc
+#define GCEA_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xe
+#define GCEA_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0xf
+#define GCEA_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0x11
+#define GCEA_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x12
+#define GCEA_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0x14
+#define GCEA_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x15
+#define GCEA_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x17
+#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000003L
+#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000004L
+#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000018L
+#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000020L
+#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x000000C0L
+#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000100L
+#define GCEA_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000600L
+#define GCEA_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000800L
+#define GCEA_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                    0x00003000L
+#define GCEA_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00004000L
+#define GCEA_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00018000L
+#define GCEA_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00020000L
+#define GCEA_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x000C0000L
+#define GCEA_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00100000L
+#define GCEA_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00600000L
+#define GCEA_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00800000L
+//GCEA_DSM_CNTLA
+#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x0
+#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x2
+#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x3
+#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x5
+#define GCEA_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x6
+#define GCEA_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0x8
+#define GCEA_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x9
+#define GCEA_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xb
+#define GCEA_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                                0xc
+#define GCEA_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0xe
+#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                               0xf
+#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x11
+#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x12
+#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x14
+#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00000003L
+#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000004L
+#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00000018L
+#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000020L
+#define GCEA_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x000000C0L
+#define GCEA_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000100L
+#define GCEA_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00000600L
+#define GCEA_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000800L
+#define GCEA_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00003000L
+#define GCEA_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00004000L
+#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00018000L
+#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00020000L
+#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                 0x000C0000L
+#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00100000L
+//GCEA_DSM_CNTLB
+#define GCEA_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x0
+#define GCEA_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x2
+#define GCEA_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x3
+#define GCEA_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x5
+#define GCEA_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x6
+#define GCEA_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x8
+#define GCEA_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x9
+#define GCEA_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0xb
+#define GCEA_DSM_CNTLB__MAM_A0MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0xc
+#define GCEA_DSM_CNTLB__MAM_A0MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0xe
+#define GCEA_DSM_CNTLB__MAM_A1MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0xf
+#define GCEA_DSM_CNTLB__MAM_A1MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x11
+#define GCEA_DSM_CNTLB__MAM_A2MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x12
+#define GCEA_DSM_CNTLB__MAM_A2MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x14
+#define GCEA_DSM_CNTLB__MAM_A3MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x15
+#define GCEA_DSM_CNTLB__MAM_A3MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x17
+#define GCEA_DSM_CNTLB__MAM_AFMEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x18
+#define GCEA_DSM_CNTLB__MAM_AFMEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x1a
+#define GCEA_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00000003L
+#define GCEA_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00000004L
+#define GCEA_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00000018L
+#define GCEA_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00000020L
+#define GCEA_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA_MASK                                                     0x000000C0L
+#define GCEA_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00000100L
+#define GCEA_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00000600L
+#define GCEA_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00000800L
+#define GCEA_DSM_CNTLB__MAM_A0MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00003000L
+#define GCEA_DSM_CNTLB__MAM_A0MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00004000L
+#define GCEA_DSM_CNTLB__MAM_A1MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00018000L
+#define GCEA_DSM_CNTLB__MAM_A1MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00020000L
+#define GCEA_DSM_CNTLB__MAM_A2MEM_DSM_IRRITATOR_DATA_MASK                                                     0x000C0000L
+#define GCEA_DSM_CNTLB__MAM_A2MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00100000L
+#define GCEA_DSM_CNTLB__MAM_A3MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00600000L
+#define GCEA_DSM_CNTLB__MAM_A3MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00800000L
+#define GCEA_DSM_CNTLB__MAM_AFMEM_DSM_IRRITATOR_DATA_MASK                                                     0x03000000L
+#define GCEA_DSM_CNTLB__MAM_AFMEM_ENABLE_SINGLE_WRITE_MASK                                                    0x04000000L
+//GCEA_DSM_CNTL2
+#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x0
+#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x2
+#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x3
+#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x5
+#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x6
+#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0x8
+#define GCEA_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                                0x9
+#define GCEA_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                                0xb
+#define GCEA_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                                0xc
+#define GCEA_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                                0xe
+#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0xf
+#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0x11
+#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x12
+#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0x14
+#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x15
+#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                              0x17
+#define GCEA_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                   0x1a
+#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000003L
+#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000004L
+#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000018L
+#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000020L
+#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x000000C0L
+#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00000100L
+#define GCEA_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000600L
+#define GCEA_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                  0x00000800L
+#define GCEA_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                  0x00003000L
+#define GCEA_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                  0x00004000L
+#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00018000L
+#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00020000L
+#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x000C0000L
+#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00100000L
+#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                                0x00600000L
+#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                                0x00800000L
+#define GCEA_DSM_CNTL2__INJECT_DELAY_MASK                                                                     0xFC000000L
+//GCEA_DSM_CNTL2A
+#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x0
+#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x2
+#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x3
+#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x5
+#define GCEA_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x6
+#define GCEA_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0x8
+#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x9
+#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0xb
+#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                              0xc
+#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                              0xe
+#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                             0xf
+#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                             0x11
+#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x12
+#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                             0x14
+#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00000003L
+#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00000004L
+#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00000018L
+#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00000020L
+#define GCEA_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x000000C0L
+#define GCEA_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000100L
+#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00000600L
+#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000800L
+#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                                0x00003000L
+#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                                0x00004000L
+#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                               0x00018000L
+#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                               0x00020000L
+#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                               0x000C0000L
+#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                               0x00100000L
+//GCEA_DSM_CNTL2B
+#define GCEA_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x0
+#define GCEA_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x2
+#define GCEA_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x3
+#define GCEA_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x5
+#define GCEA_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x6
+#define GCEA_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x8
+#define GCEA_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x9
+#define GCEA_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY__SHIFT                                                 0xb
+#define GCEA_DSM_CNTL2B__MAM_A0MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0xc
+#define GCEA_DSM_CNTL2B__MAM_A0MEM_SELECT_INJECT_DELAY__SHIFT                                                 0xe
+#define GCEA_DSM_CNTL2B__MAM_A1MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0xf
+#define GCEA_DSM_CNTL2B__MAM_A1MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x11
+#define GCEA_DSM_CNTL2B__MAM_A2MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x12
+#define GCEA_DSM_CNTL2B__MAM_A2MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x14
+#define GCEA_DSM_CNTL2B__MAM_A3MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x15
+#define GCEA_DSM_CNTL2B__MAM_A3MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x17
+#define GCEA_DSM_CNTL2B__MAM_AFMEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x18
+#define GCEA_DSM_CNTL2B__MAM_AFMEM_SELECT_INJECT_DELAY__SHIFT                                                 0x1a
+#define GCEA_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00000003L
+#define GCEA_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY_MASK                                                   0x00000004L
+#define GCEA_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00000018L
+#define GCEA_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY_MASK                                                   0x00000020L
+#define GCEA_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT_MASK                                                   0x000000C0L
+#define GCEA_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY_MASK                                                   0x00000100L
+#define GCEA_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00000600L
+#define GCEA_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY_MASK                                                   0x00000800L
+#define GCEA_DSM_CNTL2B__MAM_A0MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00003000L
+#define GCEA_DSM_CNTL2B__MAM_A0MEM_SELECT_INJECT_DELAY_MASK                                                   0x00004000L
+#define GCEA_DSM_CNTL2B__MAM_A1MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00018000L
+#define GCEA_DSM_CNTL2B__MAM_A1MEM_SELECT_INJECT_DELAY_MASK                                                   0x00020000L
+#define GCEA_DSM_CNTL2B__MAM_A2MEM_ENABLE_ERROR_INJECT_MASK                                                   0x000C0000L
+#define GCEA_DSM_CNTL2B__MAM_A2MEM_SELECT_INJECT_DELAY_MASK                                                   0x00100000L
+#define GCEA_DSM_CNTL2B__MAM_A3MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00600000L
+#define GCEA_DSM_CNTL2B__MAM_A3MEM_SELECT_INJECT_DELAY_MASK                                                   0x00800000L
+#define GCEA_DSM_CNTL2B__MAM_AFMEM_ENABLE_ERROR_INJECT_MASK                                                   0x03000000L
+#define GCEA_DSM_CNTL2B__MAM_AFMEM_SELECT_INJECT_DELAY_MASK                                                   0x04000000L
+//GCEA_TCC_XBR_CREDITS
+#define GCEA_TCC_XBR_CREDITS__DRAM_RD_LIMIT__SHIFT                                                            0x0
+#define GCEA_TCC_XBR_CREDITS__DRAM_RD_RESERVE__SHIFT                                                          0x6
+#define GCEA_TCC_XBR_CREDITS__IO_RD_LIMIT__SHIFT                                                              0x8
+#define GCEA_TCC_XBR_CREDITS__IO_RD_RESERVE__SHIFT                                                            0xe
+#define GCEA_TCC_XBR_CREDITS__DRAM_WR_LIMIT__SHIFT                                                            0x10
+#define GCEA_TCC_XBR_CREDITS__DRAM_WR_RESERVE__SHIFT                                                          0x16
+#define GCEA_TCC_XBR_CREDITS__IO_WR_LIMIT__SHIFT                                                              0x18
+#define GCEA_TCC_XBR_CREDITS__IO_WR_RESERVE__SHIFT                                                            0x1e
+#define GCEA_TCC_XBR_CREDITS__DRAM_RD_LIMIT_MASK                                                              0x0000003FL
+#define GCEA_TCC_XBR_CREDITS__DRAM_RD_RESERVE_MASK                                                            0x000000C0L
+#define GCEA_TCC_XBR_CREDITS__IO_RD_LIMIT_MASK                                                                0x00003F00L
+#define GCEA_TCC_XBR_CREDITS__IO_RD_RESERVE_MASK                                                              0x0000C000L
+#define GCEA_TCC_XBR_CREDITS__DRAM_WR_LIMIT_MASK                                                              0x003F0000L
+#define GCEA_TCC_XBR_CREDITS__DRAM_WR_RESERVE_MASK                                                            0x00C00000L
+#define GCEA_TCC_XBR_CREDITS__IO_WR_LIMIT_MASK                                                                0x3F000000L
+#define GCEA_TCC_XBR_CREDITS__IO_WR_RESERVE_MASK                                                              0xC0000000L
+//GCEA_TCC_XBR_MAXBURST
+#define GCEA_TCC_XBR_MAXBURST__DRAM_RD__SHIFT                                                                 0x0
+#define GCEA_TCC_XBR_MAXBURST__IO_RD__SHIFT                                                                   0x4
+#define GCEA_TCC_XBR_MAXBURST__DRAM_WR__SHIFT                                                                 0x8
+#define GCEA_TCC_XBR_MAXBURST__IO_WR__SHIFT                                                                   0xc
+#define GCEA_TCC_XBR_MAXBURST__DRAM_RD_MASK                                                                   0x0000000FL
+#define GCEA_TCC_XBR_MAXBURST__IO_RD_MASK                                                                     0x000000F0L
+#define GCEA_TCC_XBR_MAXBURST__DRAM_WR_MASK                                                                   0x00000F00L
+#define GCEA_TCC_XBR_MAXBURST__IO_WR_MASK                                                                     0x0000F000L
+//GCEA_PROBE_CNTL
+#define GCEA_PROBE_CNTL__REQ2RSP_DELAY__SHIFT                                                                 0x0
+#define GCEA_PROBE_CNTL__PRB_FILTER_DISABLE__SHIFT                                                            0x5
+#define GCEA_PROBE_CNTL__REQ2RSP_DELAY_MASK                                                                   0x0000001FL
+#define GCEA_PROBE_CNTL__PRB_FILTER_DISABLE_MASK                                                              0x00000020L
+//GCEA_PROBE_MAP
+#define GCEA_PROBE_MAP__CHADDR0_TO_RIGHTTCC__SHIFT                                                            0x0
+#define GCEA_PROBE_MAP__CHADDR1_TO_RIGHTTCC__SHIFT                                                            0x1
+#define GCEA_PROBE_MAP__CHADDR2_TO_RIGHTTCC__SHIFT                                                            0x2
+#define GCEA_PROBE_MAP__CHADDR3_TO_RIGHTTCC__SHIFT                                                            0x3
+#define GCEA_PROBE_MAP__CHADDR4_TO_RIGHTTCC__SHIFT                                                            0x4
+#define GCEA_PROBE_MAP__CHADDR5_TO_RIGHTTCC__SHIFT                                                            0x5
+#define GCEA_PROBE_MAP__CHADDR6_TO_RIGHTTCC__SHIFT                                                            0x6
+#define GCEA_PROBE_MAP__CHADDR7_TO_RIGHTTCC__SHIFT                                                            0x7
+#define GCEA_PROBE_MAP__CHADDR8_TO_RIGHTTCC__SHIFT                                                            0x8
+#define GCEA_PROBE_MAP__CHADDR9_TO_RIGHTTCC__SHIFT                                                            0x9
+#define GCEA_PROBE_MAP__CHADDR10_TO_RIGHTTCC__SHIFT                                                           0xa
+#define GCEA_PROBE_MAP__CHADDR11_TO_RIGHTTCC__SHIFT                                                           0xb
+#define GCEA_PROBE_MAP__CHADDR12_TO_RIGHTTCC__SHIFT                                                           0xc
+#define GCEA_PROBE_MAP__CHADDR13_TO_RIGHTTCC__SHIFT                                                           0xd
+#define GCEA_PROBE_MAP__CHADDR14_TO_RIGHTTCC__SHIFT                                                           0xe
+#define GCEA_PROBE_MAP__CHADDR15_TO_RIGHTTCC__SHIFT                                                           0xf
+#define GCEA_PROBE_MAP__INTLV_SIZE__SHIFT                                                                     0x10
+#define GCEA_PROBE_MAP__CHADDR0_TO_RIGHTTCC_MASK                                                              0x00000001L
+#define GCEA_PROBE_MAP__CHADDR1_TO_RIGHTTCC_MASK                                                              0x00000002L
+#define GCEA_PROBE_MAP__CHADDR2_TO_RIGHTTCC_MASK                                                              0x00000004L
+#define GCEA_PROBE_MAP__CHADDR3_TO_RIGHTTCC_MASK                                                              0x00000008L
+#define GCEA_PROBE_MAP__CHADDR4_TO_RIGHTTCC_MASK                                                              0x00000010L
+#define GCEA_PROBE_MAP__CHADDR5_TO_RIGHTTCC_MASK                                                              0x00000020L
+#define GCEA_PROBE_MAP__CHADDR6_TO_RIGHTTCC_MASK                                                              0x00000040L
+#define GCEA_PROBE_MAP__CHADDR7_TO_RIGHTTCC_MASK                                                              0x00000080L
+#define GCEA_PROBE_MAP__CHADDR8_TO_RIGHTTCC_MASK                                                              0x00000100L
+#define GCEA_PROBE_MAP__CHADDR9_TO_RIGHTTCC_MASK                                                              0x00000200L
+#define GCEA_PROBE_MAP__CHADDR10_TO_RIGHTTCC_MASK                                                             0x00000400L
+#define GCEA_PROBE_MAP__CHADDR11_TO_RIGHTTCC_MASK                                                             0x00000800L
+#define GCEA_PROBE_MAP__CHADDR12_TO_RIGHTTCC_MASK                                                             0x00001000L
+#define GCEA_PROBE_MAP__CHADDR13_TO_RIGHTTCC_MASK                                                             0x00002000L
+#define GCEA_PROBE_MAP__CHADDR14_TO_RIGHTTCC_MASK                                                             0x00004000L
+#define GCEA_PROBE_MAP__CHADDR15_TO_RIGHTTCC_MASK                                                             0x00008000L
+#define GCEA_PROBE_MAP__INTLV_SIZE_MASK                                                                       0x00030000L
+//GCEA_ERR_STATUS
+#define GCEA_ERR_STATUS__SDP_RDRSP_STATUS__SHIFT                                                              0x0
+#define GCEA_ERR_STATUS__SDP_WRRSP_STATUS__SHIFT                                                              0x4
+#define GCEA_ERR_STATUS__SDP_RDRSP_DATASTATUS__SHIFT                                                          0x8
+#define GCEA_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR__SHIFT                                                    0xa
+#define GCEA_ERR_STATUS__CLEAR_ERROR_STATUS__SHIFT                                                            0xb
+#define GCEA_ERR_STATUS__BUSY_ON_ERROR__SHIFT                                                                 0xc
+#define GCEA_ERR_STATUS__FUE_FLAG__SHIFT                                                                      0xd
+#define GCEA_ERR_STATUS__IGNORE_RDRSP_FED__SHIFT                                                              0xe
+#define GCEA_ERR_STATUS__INTERRUPT_ON_FATAL__SHIFT                                                            0xf
+#define GCEA_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL__SHIFT                                                    0x10
+#define GCEA_ERR_STATUS__LEVEL_INTERRUPT__SHIFT                                                               0x11
+#define GCEA_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR__SHIFT                                                      0x12
+#define GCEA_ERR_STATUS__FUE_FLAG_CLIENT__SHIFT                                                               0x13
+#define GCEA_ERR_STATUS__SDP_RDRSP_STATUS_MASK                                                                0x0000000FL
+#define GCEA_ERR_STATUS__SDP_WRRSP_STATUS_MASK                                                                0x000000F0L
+#define GCEA_ERR_STATUS__SDP_RDRSP_DATASTATUS_MASK                                                            0x00000300L
+#define GCEA_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR_MASK                                                      0x00000400L
+#define GCEA_ERR_STATUS__CLEAR_ERROR_STATUS_MASK                                                              0x00000800L
+#define GCEA_ERR_STATUS__BUSY_ON_ERROR_MASK                                                                   0x00001000L
+#define GCEA_ERR_STATUS__FUE_FLAG_MASK                                                                        0x00002000L
+#define GCEA_ERR_STATUS__IGNORE_RDRSP_FED_MASK                                                                0x00004000L
+#define GCEA_ERR_STATUS__INTERRUPT_ON_FATAL_MASK                                                              0x00008000L
+#define GCEA_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL_MASK                                                      0x00010000L
+#define GCEA_ERR_STATUS__LEVEL_INTERRUPT_MASK                                                                 0x00020000L
+#define GCEA_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR_MASK                                                        0x00040000L
+#define GCEA_ERR_STATUS__FUE_FLAG_CLIENT_MASK                                                                 0x00080000L
+//GCEA_MISC2
+#define GCEA_MISC2__CSGROUP_SWAP_IN_DRAM_ARB__SHIFT                                                           0x0
+#define GCEA_MISC2__CSGROUP_SWAP_IN_GMI_ARB__SHIFT                                                            0x1
+#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM__SHIFT                                                        0x2
+#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_GMI__SHIFT                                                         0x7
+#define GCEA_MISC2__IO_RDWR_PRIORITY_ENABLE__SHIFT                                                            0xc
+#define GCEA_MISC2__BLOCK_REQUESTS__SHIFT                                                                     0xd
+#define GCEA_MISC2__REQUESTS_BLOCKED__SHIFT                                                                   0xe
+#define GCEA_MISC2__FGCLKEN_OVERRIDE__SHIFT                                                                   0xf
+#define GCEA_MISC2__DRAM_RD_THROTTLE__SHIFT                                                                   0x10
+#define GCEA_MISC2__DRAM_WR_THROTTLE__SHIFT                                                                   0x11
+#define GCEA_MISC2__GMI_RD_THROTTLE__SHIFT                                                                    0x12
+#define GCEA_MISC2__GMI_WR_THROTTLE__SHIFT                                                                    0x13
+#define GCEA_MISC2__CONTAIN_ILLEGAL_OP__SHIFT                                                                 0x14
+#define GCEA_MISC2__REPORT_ILLEGAL_OP__SHIFT                                                                  0x15
+#define GCEA_MISC2__CSGROUP_SWAP_IN_DRAM_ARB_MASK                                                             0x00000001L
+#define GCEA_MISC2__CSGROUP_SWAP_IN_GMI_ARB_MASK                                                              0x00000002L
+#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM_MASK                                                          0x0000007CL
+#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_GMI_MASK                                                           0x00000F80L
+#define GCEA_MISC2__IO_RDWR_PRIORITY_ENABLE_MASK                                                              0x00001000L
+#define GCEA_MISC2__BLOCK_REQUESTS_MASK                                                                       0x00002000L
+#define GCEA_MISC2__REQUESTS_BLOCKED_MASK                                                                     0x00004000L
+#define GCEA_MISC2__FGCLKEN_OVERRIDE_MASK                                                                     0x00008000L
+#define GCEA_MISC2__DRAM_RD_THROTTLE_MASK                                                                     0x00010000L
+#define GCEA_MISC2__DRAM_WR_THROTTLE_MASK                                                                     0x00020000L
+#define GCEA_MISC2__GMI_RD_THROTTLE_MASK                                                                      0x00040000L
+#define GCEA_MISC2__GMI_WR_THROTTLE_MASK                                                                      0x00080000L
+#define GCEA_MISC2__CONTAIN_ILLEGAL_OP_MASK                                                                   0x00100000L
+#define GCEA_MISC2__REPORT_ILLEGAL_OP_MASK                                                                    0x00200000L
+//GCEA_SDP_BACKDOOR_CMDCREDITS0
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC0_CREDITS_RECEIVED__SHIFT                                            0x0
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC1_CREDITS_RECEIVED__SHIFT                                            0x7
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC2_CREDITS_RECEIVED__SHIFT                                            0xe
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC3_CREDITS_RECEIVED__SHIFT                                            0x15
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC4_CREDITS_RECEIVED__SHIFT                                            0x1c
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC0_CREDITS_RECEIVED_MASK                                              0x0000007FL
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC1_CREDITS_RECEIVED_MASK                                              0x00003F80L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC2_CREDITS_RECEIVED_MASK                                              0x001FC000L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC3_CREDITS_RECEIVED_MASK                                              0x0FE00000L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC4_CREDITS_RECEIVED_MASK                                              0xF0000000L
+//GCEA_SDP_BACKDOOR_CMDCREDITS1
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC4_CREDITS_RECEIVED__SHIFT                                            0x0
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC5_CREDITS_RECEIVED__SHIFT                                            0x3
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC6_CREDITS_RECEIVED__SHIFT                                            0xa
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC7_CREDITS_RECEIVED__SHIFT                                            0x11
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__POOL_CREDITS_RECEIVED__SHIFT                                           0x18
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC4_CREDITS_RECEIVED_MASK                                              0x00000007L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC5_CREDITS_RECEIVED_MASK                                              0x000003F8L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC6_CREDITS_RECEIVED_MASK                                              0x0001FC00L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC7_CREDITS_RECEIVED_MASK                                              0x00FE0000L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__POOL_CREDITS_RECEIVED_MASK                                             0x7F000000L
+//GCEA_SDP_BACKDOOR_DATACREDITS0
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC0_CREDITS_RECEIVED__SHIFT                                           0x0
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC1_CREDITS_RECEIVED__SHIFT                                           0x7
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC2_CREDITS_RECEIVED__SHIFT                                           0xe
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC3_CREDITS_RECEIVED__SHIFT                                           0x15
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC4_CREDITS_RECEIVED__SHIFT                                           0x1c
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC0_CREDITS_RECEIVED_MASK                                             0x0000007FL
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC1_CREDITS_RECEIVED_MASK                                             0x00003F80L
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC2_CREDITS_RECEIVED_MASK                                             0x001FC000L
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC3_CREDITS_RECEIVED_MASK                                             0x0FE00000L
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC4_CREDITS_RECEIVED_MASK                                             0xF0000000L
+//GCEA_SDP_BACKDOOR_DATACREDITS1
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC4_CREDITS_RECEIVED__SHIFT                                           0x0
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC5_CREDITS_RECEIVED__SHIFT                                           0x3
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC6_CREDITS_RECEIVED__SHIFT                                           0xa
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC7_CREDITS_RECEIVED__SHIFT                                           0x11
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__POOL_CREDITS_RECEIVED__SHIFT                                          0x18
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC4_CREDITS_RECEIVED_MASK                                             0x00000007L
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC5_CREDITS_RECEIVED_MASK                                             0x000003F8L
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC6_CREDITS_RECEIVED_MASK                                             0x0001FC00L
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC7_CREDITS_RECEIVED_MASK                                             0x00FE0000L
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__POOL_CREDITS_RECEIVED_MASK                                            0x7F000000L
+//GCEA_SDP_BACKDOOR_MISCCREDITS
+#define GCEA_SDP_BACKDOOR_MISCCREDITS__RDRSP_CREDITS_RELEASED__SHIFT                                          0x0
+#define GCEA_SDP_BACKDOOR_MISCCREDITS__WRRSP_CREDITS_RELEASED__SHIFT                                          0x8
+#define GCEA_SDP_BACKDOOR_MISCCREDITS__PRB_REQ_CREDITS_RELEASED__SHIFT                                        0x10
+#define GCEA_SDP_BACKDOOR_MISCCREDITS__PRB_RSP_CREDITS_RECEIVED__SHIFT                                        0x17
+#define GCEA_SDP_BACKDOOR_MISCCREDITS__RDRSP_CREDITS_RELEASED_MASK                                            0x000000FFL
+#define GCEA_SDP_BACKDOOR_MISCCREDITS__WRRSP_CREDITS_RELEASED_MASK                                            0x0000FF00L
+#define GCEA_SDP_BACKDOOR_MISCCREDITS__PRB_REQ_CREDITS_RELEASED_MASK                                          0x007F0000L
+#define GCEA_SDP_BACKDOOR_MISCCREDITS__PRB_RSP_CREDITS_RECEIVED_MASK                                          0x3F800000L
+//GCEA_SDP_ENABLE
+#define GCEA_SDP_ENABLE__ENABLE__SHIFT                                                                        0x0
+#define GCEA_SDP_ENABLE__ENABLE_MASK                                                                          0x00000001L
+
+
+// addressBlock: xcd0_gc_ea_pwrdec
+//GCEA_ICG_CTRL
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                            0x0
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_READ__SHIFT                                                              0x1
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_WRITE__SHIFT                                                             0x2
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                          0x3
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                              0x00000001L
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_READ_MASK                                                                0x00000002L
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_WRITE_MASK                                                               0x00000004L
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                            0x00000008L
+
+
+// addressBlock: xcd0_gc_rmi_rmidec
+//RMI_GENERAL_CNTL
+#define RMI_GENERAL_CNTL__BURST_DISABLE__SHIFT                                                                0x0
+#define RMI_GENERAL_CNTL__VMID_BYPASS_ENABLE__SHIFT                                                           0x1
+#define RMI_GENERAL_CNTL__XBAR_MUX_CONFIG__SHIFT                                                              0x11
+#define RMI_GENERAL_CNTL__RB0_HARVEST_EN__SHIFT                                                               0x13
+#define RMI_GENERAL_CNTL__RB1_HARVEST_EN__SHIFT                                                               0x14
+#define RMI_GENERAL_CNTL__LOOPBACK_DIS_BY_REQ_TYPE__SHIFT                                                     0x15
+#define RMI_GENERAL_CNTL__XBAR_MUX_CONFIG_UPDATE__SHIFT                                                       0x19
+#define RMI_GENERAL_CNTL__SKID_FIFO_0_OVERFLOW_ERROR_MASK__SHIFT                                              0x1a
+#define RMI_GENERAL_CNTL__SKID_FIFO_0_UNDERFLOW_ERROR_MASK__SHIFT                                             0x1b
+#define RMI_GENERAL_CNTL__SKID_FIFO_1_OVERFLOW_ERROR_MASK__SHIFT                                              0x1c
+#define RMI_GENERAL_CNTL__SKID_FIFO_1_UNDERFLOW_ERROR_MASK__SHIFT                                             0x1d
+#define RMI_GENERAL_CNTL__SKID_FIFO_FREESPACE_IS_ZERO_ERROR_MASK__SHIFT                                       0x1e
+#define RMI_GENERAL_CNTL__BURST_DISABLE_MASK                                                                  0x00000001L
+#define RMI_GENERAL_CNTL__VMID_BYPASS_ENABLE_MASK                                                             0x0001FFFEL
+#define RMI_GENERAL_CNTL__XBAR_MUX_CONFIG_MASK                                                                0x00060000L
+#define RMI_GENERAL_CNTL__RB0_HARVEST_EN_MASK                                                                 0x00080000L
+#define RMI_GENERAL_CNTL__RB1_HARVEST_EN_MASK                                                                 0x00100000L
+#define RMI_GENERAL_CNTL__LOOPBACK_DIS_BY_REQ_TYPE_MASK                                                       0x01E00000L
+#define RMI_GENERAL_CNTL__XBAR_MUX_CONFIG_UPDATE_MASK                                                         0x02000000L
+#define RMI_GENERAL_CNTL__SKID_FIFO_0_OVERFLOW_ERROR_MASK_MASK                                                0x04000000L
+#define RMI_GENERAL_CNTL__SKID_FIFO_0_UNDERFLOW_ERROR_MASK_MASK                                               0x08000000L
+#define RMI_GENERAL_CNTL__SKID_FIFO_1_OVERFLOW_ERROR_MASK_MASK                                                0x10000000L
+#define RMI_GENERAL_CNTL__SKID_FIFO_1_UNDERFLOW_ERROR_MASK_MASK                                               0x20000000L
+#define RMI_GENERAL_CNTL__SKID_FIFO_FREESPACE_IS_ZERO_ERROR_MASK_MASK                                         0x40000000L
+//RMI_GENERAL_CNTL1
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_ENABLE_PER_MTYPE__SHIFT                                                0x0
+#define RMI_GENERAL_CNTL1__TCIW0_64B_RD_STALL_MODE__SHIFT                                                     0x4
+#define RMI_GENERAL_CNTL1__TCIW1_64B_RD_STALL_MODE__SHIFT                                                     0x6
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_DISABLE_FOR_LOOPBACK__SHIFT                                            0x8
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_VALUE__SHIFT                                                       0x9
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE__SHIFT                                                             0xa
+#define RMI_GENERAL_CNTL1__UTCL1_PROBE0_RR_ARB_BURST_HINT_EN__SHIFT                                           0xb
+#define RMI_GENERAL_CNTL1__UTCL1_PROBE1_RR_ARB_BURST_HINT_EN__SHIFT                                           0xc
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_ENABLE_PER_MTYPE_MASK                                                  0x0000000FL
+#define RMI_GENERAL_CNTL1__TCIW0_64B_RD_STALL_MODE_MASK                                                       0x00000030L
+#define RMI_GENERAL_CNTL1__TCIW1_64B_RD_STALL_MODE_MASK                                                       0x000000C0L
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_DISABLE_FOR_LOOPBACK_MASK                                              0x00000100L
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_VALUE_MASK                                                         0x00000200L
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_MASK                                                               0x00000400L
+#define RMI_GENERAL_CNTL1__UTCL1_PROBE0_RR_ARB_BURST_HINT_EN_MASK                                             0x00000800L
+#define RMI_GENERAL_CNTL1__UTCL1_PROBE1_RR_ARB_BURST_HINT_EN_MASK                                             0x00001000L
+//RMI_GENERAL_STATUS
+#define RMI_GENERAL_STATUS__GENERAL_RMI_ERRORS_COMBINED__SHIFT                                                0x0
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_OVERFLOW_ERROR__SHIFT                                                 0x1
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_UNDERFLOW_ERROR__SHIFT                                                0x2
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_OVERFLOW_ERROR__SHIFT                                                 0x3
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_UNDERFLOW_ERROR__SHIFT                                                0x4
+#define RMI_GENERAL_STATUS__RMI_XBAR_BUSY__SHIFT                                                              0x5
+#define RMI_GENERAL_STATUS__RMI_UTCL1_BUSY__SHIFT                                                             0x6
+#define RMI_GENERAL_STATUS__RMI_SCOREBOARD_BUSY__SHIFT                                                        0x7
+#define RMI_GENERAL_STATUS__TCIW0_PRT_FIFO_BUSY__SHIFT                                                        0x8
+#define RMI_GENERAL_STATUS__TCIW_FRMTR0_BUSY__SHIFT                                                           0x9
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR0_BUSY__SHIFT                                                       0xa
+#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_0_BUSY__SHIFT                                                 0xb
+#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_0_BUSY__SHIFT                                                 0xc
+#define RMI_GENERAL_STATUS__TCIW1_PRT_FIFO_BUSY__SHIFT                                                        0xd
+#define RMI_GENERAL_STATUS__TCIW_FRMTR1_BUSY__SHIFT                                                           0xe
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR1_BUSY__SHIFT                                                       0xf
+#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_1_BUSY__SHIFT                                                 0x10
+#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_1_BUSY__SHIFT                                                 0x11
+#define RMI_GENERAL_STATUS__UTC_PROBE1_BUSY__SHIFT                                                            0x12
+#define RMI_GENERAL_STATUS__UTC_PROBE0_BUSY__SHIFT                                                            0x13
+#define RMI_GENERAL_STATUS__RMI_XNACK_BUSY__SHIFT                                                             0x14
+#define RMI_GENERAL_STATUS__XNACK_FIFO_NUM_USED__SHIFT                                                        0x15
+#define RMI_GENERAL_STATUS__XNACK_FIFO_EMPTY__SHIFT                                                           0x1d
+#define RMI_GENERAL_STATUS__XNACK_FIFO_FULL__SHIFT                                                            0x1e
+#define RMI_GENERAL_STATUS__SKID_FIFO_FREESPACE_IS_ZERO_ERROR__SHIFT                                          0x1f
+#define RMI_GENERAL_STATUS__GENERAL_RMI_ERRORS_COMBINED_MASK                                                  0x00000001L
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_OVERFLOW_ERROR_MASK                                                   0x00000002L
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_UNDERFLOW_ERROR_MASK                                                  0x00000004L
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_OVERFLOW_ERROR_MASK                                                   0x00000008L
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_UNDERFLOW_ERROR_MASK                                                  0x00000010L
+#define RMI_GENERAL_STATUS__RMI_XBAR_BUSY_MASK                                                                0x00000020L
+#define RMI_GENERAL_STATUS__RMI_UTCL1_BUSY_MASK                                                               0x00000040L
+#define RMI_GENERAL_STATUS__RMI_SCOREBOARD_BUSY_MASK                                                          0x00000080L
+#define RMI_GENERAL_STATUS__TCIW0_PRT_FIFO_BUSY_MASK                                                          0x00000100L
+#define RMI_GENERAL_STATUS__TCIW_FRMTR0_BUSY_MASK                                                             0x00000200L
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR0_BUSY_MASK                                                         0x00000400L
+#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_0_BUSY_MASK                                                   0x00000800L
+#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_0_BUSY_MASK                                                   0x00001000L
+#define RMI_GENERAL_STATUS__TCIW1_PRT_FIFO_BUSY_MASK                                                          0x00002000L
+#define RMI_GENERAL_STATUS__TCIW_FRMTR1_BUSY_MASK                                                             0x00004000L
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR1_BUSY_MASK                                                         0x00008000L
+#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_1_BUSY_MASK                                                   0x00010000L
+#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_1_BUSY_MASK                                                   0x00020000L
+#define RMI_GENERAL_STATUS__UTC_PROBE1_BUSY_MASK                                                              0x00040000L
+#define RMI_GENERAL_STATUS__UTC_PROBE0_BUSY_MASK                                                              0x00080000L
+#define RMI_GENERAL_STATUS__RMI_XNACK_BUSY_MASK                                                               0x00100000L
+#define RMI_GENERAL_STATUS__XNACK_FIFO_NUM_USED_MASK                                                          0x1FE00000L
+#define RMI_GENERAL_STATUS__XNACK_FIFO_EMPTY_MASK                                                             0x20000000L
+#define RMI_GENERAL_STATUS__XNACK_FIFO_FULL_MASK                                                              0x40000000L
+#define RMI_GENERAL_STATUS__SKID_FIFO_FREESPACE_IS_ZERO_ERROR_MASK                                            0x80000000L
+//RMI_SUBBLOCK_STATUS0
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE0__SHIFT                                     0x0
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE0__SHIFT                                         0x7
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE0__SHIFT                                        0x8
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE1__SHIFT                                     0x9
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE1__SHIFT                                         0x10
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE1__SHIFT                                        0x11
+#define RMI_SUBBLOCK_STATUS0__TCIW0_INFLIGHT_CNT__SHIFT                                                       0x12
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE0_MASK                                       0x0000007FL
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE0_MASK                                           0x00000080L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE0_MASK                                          0x00000100L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE1_MASK                                       0x0000FE00L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE1_MASK                                           0x00010000L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE1_MASK                                          0x00020000L
+#define RMI_SUBBLOCK_STATUS0__TCIW0_INFLIGHT_CNT_MASK                                                         0x0FFC0000L
+//RMI_SUBBLOCK_STATUS1
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_0_FREE_SPACE__SHIFT                                                   0x0
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_1_FREE_SPACE__SHIFT                                                   0xa
+#define RMI_SUBBLOCK_STATUS1__TCIW1_INFLIGHT_CNT__SHIFT                                                       0x14
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_0_FREE_SPACE_MASK                                                     0x000003FFL
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_1_FREE_SPACE_MASK                                                     0x000FFC00L
+#define RMI_SUBBLOCK_STATUS1__TCIW1_INFLIGHT_CNT_MASK                                                         0x3FF00000L
+//RMI_SUBBLOCK_STATUS2
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_0_NUM_USED__SHIFT                                                      0x0
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_1_NUM_USED__SHIFT                                                      0x9
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_0_NUM_USED_MASK                                                        0x000001FFL
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_1_NUM_USED_MASK                                                        0x0003FE00L
+//RMI_SUBBLOCK_STATUS3
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_0_FREE_SPACE_TOTAL__SHIFT                                             0x0
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_1_FREE_SPACE_TOTAL__SHIFT                                             0xa
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_0_FREE_SPACE_TOTAL_MASK                                               0x000003FFL
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_1_FREE_SPACE_TOTAL_MASK                                               0x000FFC00L
+//RMI_XBAR_CONFIG
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_OVERRIDE__SHIFT                                                      0x0
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_REQ_TYPE_OVERRIDE__SHIFT                                             0x2
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_CB_DB_OVERRIDE__SHIFT                                                0x6
+#define RMI_XBAR_CONFIG__ARBITER_DIS__SHIFT                                                                   0x7
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ__SHIFT                                                                0x8
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_OVERRIDE__SHIFT                                                       0xc
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB0__SHIFT                                                                0xd
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB1__SHIFT                                                                0xe
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_OVERRIDE_MASK                                                        0x00000003L
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_REQ_TYPE_OVERRIDE_MASK                                               0x0000003CL
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_CB_DB_OVERRIDE_MASK                                                  0x00000040L
+#define RMI_XBAR_CONFIG__ARBITER_DIS_MASK                                                                     0x00000080L
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_MASK                                                                  0x00000F00L
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_OVERRIDE_MASK                                                         0x00001000L
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB0_MASK                                                                  0x00002000L
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB1_MASK                                                                  0x00004000L
+//RMI_PROBE_POP_LOGIC_CNTL
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_0_MAX_DEPTH__SHIFT                                             0x0
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE0_DIS__SHIFT                                                    0x7
+#define RMI_PROBE_POP_LOGIC_CNTL__REDUCE_MAX_XLAT_CHAIN_SIZE_BY_2__SHIFT                                      0x8
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_1_MAX_DEPTH__SHIFT                                             0xa
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE1_DIS__SHIFT                                                    0x11
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_0_MAX_DEPTH_MASK                                               0x0000007FL
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE0_DIS_MASK                                                      0x00000080L
+#define RMI_PROBE_POP_LOGIC_CNTL__REDUCE_MAX_XLAT_CHAIN_SIZE_BY_2_MASK                                        0x00000300L
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_1_MAX_DEPTH_MASK                                               0x0001FC00L
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE1_DIS_MASK                                                      0x00020000L
+//RMI_UTC_XNACK_N_MISC_CNTL
+#define RMI_UTC_XNACK_N_MISC_CNTL__MASTER_XNACK_TIMER_INC__SHIFT                                              0x0
+#define RMI_UTC_XNACK_N_MISC_CNTL__IND_XNACK_TIMER_START_VALUE__SHIFT                                         0x8
+#define RMI_UTC_XNACK_N_MISC_CNTL__UTCL1_PERM_MODE__SHIFT                                                     0xc
+#define RMI_UTC_XNACK_N_MISC_CNTL__CP_VMID_RESET_REQUEST_DISABLE__SHIFT                                       0xd
+#define RMI_UTC_XNACK_N_MISC_CNTL__MASTER_XNACK_TIMER_INC_MASK                                                0x000000FFL
+#define RMI_UTC_XNACK_N_MISC_CNTL__IND_XNACK_TIMER_START_VALUE_MASK                                           0x00000F00L
+#define RMI_UTC_XNACK_N_MISC_CNTL__UTCL1_PERM_MODE_MASK                                                       0x00001000L
+#define RMI_UTC_XNACK_N_MISC_CNTL__CP_VMID_RESET_REQUEST_DISABLE_MASK                                         0x00002000L
+//RMI_DEMUX_CNTL
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL__SHIFT                                                               0x0
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_BREAK_LOB_ON_IDLEIN__SHIFT                                                 0x1
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_OVERRIDE__SHIFT                                                0x4
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_START_VALUE__SHIFT                                             0x6
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE__SHIFT                                                                0xe
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL__SHIFT                                                               0x10
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_BREAK_LOB_ON_IDLEIN__SHIFT                                                 0x11
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_OVERRIDE__SHIFT                                                0x14
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_START_VALUE__SHIFT                                             0x16
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE__SHIFT                                                                0x1e
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_MASK                                                                 0x00000001L
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_BREAK_LOB_ON_IDLEIN_MASK                                                   0x00000002L
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_OVERRIDE_MASK                                                  0x00000030L
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_START_VALUE_MASK                                               0x00003FC0L
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE_MASK                                                                  0x0000C000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_MASK                                                                 0x00010000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_BREAK_LOB_ON_IDLEIN_MASK                                                   0x00020000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_OVERRIDE_MASK                                                  0x00300000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_START_VALUE_MASK                                               0x3FC00000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE_MASK                                                                  0xC0000000L
+//RMI_UTCL1_CNTL1
+#define RMI_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                              0x0
+#define RMI_UTCL1_CNTL1__GPUVM_64K_DEF__SHIFT                                                                 0x1
+#define RMI_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                               0x2
+#define RMI_UTCL1_CNTL1__RESP_MODE__SHIFT                                                                     0x3
+#define RMI_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                               0x5
+#define RMI_UTCL1_CNTL1__CLIENTID__SHIFT                                                                      0x7
+#define RMI_UTCL1_CNTL1__USERVM_DIS__SHIFT                                                                    0x10
+#define RMI_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                             0x11
+#define RMI_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                          0x12
+#define RMI_UTCL1_CNTL1__REG_INV_VMID__SHIFT                                                                  0x13
+#define RMI_UTCL1_CNTL1__REG_INV_ALL_VMID__SHIFT                                                              0x17
+#define RMI_UTCL1_CNTL1__REG_INV_TOGGLE__SHIFT                                                                0x18
+#define RMI_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID__SHIFT                                                    0x19
+#define RMI_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                                    0x1a
+#define RMI_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                                0x1b
+#define RMI_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1c
+#define RMI_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1e
+#define RMI_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                                0x00000001L
+#define RMI_UTCL1_CNTL1__GPUVM_64K_DEF_MASK                                                                   0x00000002L
+#define RMI_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                                 0x00000004L
+#define RMI_UTCL1_CNTL1__RESP_MODE_MASK                                                                       0x00000018L
+#define RMI_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                                 0x00000060L
+#define RMI_UTCL1_CNTL1__CLIENTID_MASK                                                                        0x0000FF80L
+#define RMI_UTCL1_CNTL1__USERVM_DIS_MASK                                                                      0x00010000L
+#define RMI_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                               0x00020000L
+#define RMI_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                            0x00040000L
+#define RMI_UTCL1_CNTL1__REG_INV_VMID_MASK                                                                    0x00780000L
+#define RMI_UTCL1_CNTL1__REG_INV_ALL_VMID_MASK                                                                0x00800000L
+#define RMI_UTCL1_CNTL1__REG_INV_TOGGLE_MASK                                                                  0x01000000L
+#define RMI_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID_MASK                                                      0x02000000L
+#define RMI_UTCL1_CNTL1__FORCE_MISS_MASK                                                                      0x04000000L
+#define RMI_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                                  0x08000000L
+#define RMI_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x30000000L
+#define RMI_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xC0000000L
+//RMI_UTCL1_CNTL2
+#define RMI_UTCL1_CNTL2__UTC_SPARE__SHIFT                                                                     0x0
+#define RMI_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                0x9
+#define RMI_UTCL1_CNTL2__LINE_VALID__SHIFT                                                                    0xa
+#define RMI_UTCL1_CNTL2__DIS_EDC__SHIFT                                                                       0xb
+#define RMI_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                                0xc
+#define RMI_UTCL1_CNTL2__SHOOTDOWN_OPT__SHIFT                                                                 0xd
+#define RMI_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                                   0xe
+#define RMI_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                           0xf
+#define RMI_UTCL1_CNTL2__UTCL1_ARB_BURST_MODE__SHIFT                                                          0x10
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_RD_WR__SHIFT                                                 0x12
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_RD_WR__SHIFT                                                        0x13
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_VMID__SHIFT                                                  0x14
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_VMID__SHIFT                                                         0x15
+#define RMI_UTCL1_CNTL2__UTCL1_DIS_DUAL_L2_REQ__SHIFT                                                         0x19
+#define RMI_UTCL1_CNTL2__UTCL1_FORCE_FRAG_2M_TO_64K__SHIFT                                                    0x1a
+#define RMI_UTCL1_CNTL2__UTC_SPARE_MASK                                                                       0x000000FFL
+#define RMI_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                                  0x00000200L
+#define RMI_UTCL1_CNTL2__LINE_VALID_MASK                                                                      0x00000400L
+#define RMI_UTCL1_CNTL2__DIS_EDC_MASK                                                                         0x00000800L
+#define RMI_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                                  0x00001000L
+#define RMI_UTCL1_CNTL2__SHOOTDOWN_OPT_MASK                                                                   0x00002000L
+#define RMI_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                                     0x00004000L
+#define RMI_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                             0x00008000L
+#define RMI_UTCL1_CNTL2__UTCL1_ARB_BURST_MODE_MASK                                                            0x00030000L
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_RD_WR_MASK                                                   0x00040000L
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_RD_WR_MASK                                                          0x00080000L
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_VMID_MASK                                                    0x00100000L
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_VMID_MASK                                                           0x01E00000L
+#define RMI_UTCL1_CNTL2__UTCL1_DIS_DUAL_L2_REQ_MASK                                                           0x02000000L
+#define RMI_UTCL1_CNTL2__UTCL1_FORCE_FRAG_2M_TO_64K_MASK                                                      0x04000000L
+//RMI_UTC_UNIT_CONFIG
+#define RMI_UTC_UNIT_CONFIG__TMZ_REQ_EN__SHIFT                                                                0x0
+#define RMI_UTC_UNIT_CONFIG__TMZ_REQ_EN_MASK                                                                  0x0000FFFFL
+//RMI_TCIW_FORMATTER0_CNTL
+#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_DIS_OVERRIDE__SHIFT                                             0x0
+#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_TIME_OUT_WINDOW__SHIFT                                          0x1
+#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_MAX_ALLOWED_INFLIGHT_REQ__SHIFT                                       0x9
+#define RMI_TCIW_FORMATTER0_CNTL__SKID_FIFO_0_FREE_SPACE_DELTA__SHIFT                                         0x13
+#define RMI_TCIW_FORMATTER0_CNTL__SKID_FIFO_0_FREE_SPACE_DELTA_UPDATE__SHIFT                                  0x1b
+#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_REQ_SAFE_MODE__SHIFT                                                  0x1c
+#define RMI_TCIW_FORMATTER0_CNTL__RMI_IN0_REORDER_DIS__SHIFT                                                  0x1d
+#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_DIS_AT_LAST_OF_BURST__SHIFT                                     0x1e
+#define RMI_TCIW_FORMATTER0_CNTL__ALL_FAULT_RET0_DATA__SHIFT                                                  0x1f
+#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_DIS_OVERRIDE_MASK                                               0x00000001L
+#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_TIME_OUT_WINDOW_MASK                                            0x000001FEL
+#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_MAX_ALLOWED_INFLIGHT_REQ_MASK                                         0x0007FE00L
+#define RMI_TCIW_FORMATTER0_CNTL__SKID_FIFO_0_FREE_SPACE_DELTA_MASK                                           0x07F80000L
+#define RMI_TCIW_FORMATTER0_CNTL__SKID_FIFO_0_FREE_SPACE_DELTA_UPDATE_MASK                                    0x08000000L
+#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_REQ_SAFE_MODE_MASK                                                    0x10000000L
+#define RMI_TCIW_FORMATTER0_CNTL__RMI_IN0_REORDER_DIS_MASK                                                    0x20000000L
+#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_DIS_AT_LAST_OF_BURST_MASK                                       0x40000000L
+#define RMI_TCIW_FORMATTER0_CNTL__ALL_FAULT_RET0_DATA_MASK                                                    0x80000000L
+//RMI_TCIW_FORMATTER1_CNTL
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_OVERRIDE__SHIFT                                             0x0
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_TIME_OUT_WINDOW__SHIFT                                          0x1
+#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_MAX_ALLOWED_INFLIGHT_REQ__SHIFT                                       0x9
+#define RMI_TCIW_FORMATTER1_CNTL__SKID_FIFO_1_FREE_SPACE_DELTA__SHIFT                                         0x13
+#define RMI_TCIW_FORMATTER1_CNTL__SKID_FIFO_1_FREE_SPACE_DELTA_UPDATE__SHIFT                                  0x1b
+#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_REQ_SAFE_MODE__SHIFT                                                  0x1c
+#define RMI_TCIW_FORMATTER1_CNTL__RMI_IN1_REORDER_DIS__SHIFT                                                  0x1d
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_AT_LAST_OF_BURST__SHIFT                                     0x1e
+#define RMI_TCIW_FORMATTER1_CNTL__ALL_FAULT_RET1_DATA__SHIFT                                                  0x1f
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_OVERRIDE_MASK                                               0x00000001L
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_TIME_OUT_WINDOW_MASK                                            0x000001FEL
+#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_MAX_ALLOWED_INFLIGHT_REQ_MASK                                         0x0007FE00L
+#define RMI_TCIW_FORMATTER1_CNTL__SKID_FIFO_1_FREE_SPACE_DELTA_MASK                                           0x07F80000L
+#define RMI_TCIW_FORMATTER1_CNTL__SKID_FIFO_1_FREE_SPACE_DELTA_UPDATE_MASK                                    0x08000000L
+#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_REQ_SAFE_MODE_MASK                                                    0x10000000L
+#define RMI_TCIW_FORMATTER1_CNTL__RMI_IN1_REORDER_DIS_MASK                                                    0x20000000L
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_AT_LAST_OF_BURST_MASK                                       0x40000000L
+#define RMI_TCIW_FORMATTER1_CNTL__ALL_FAULT_RET1_DATA_MASK                                                    0x80000000L
+//RMI_SCOREBOARD_CNTL
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB0_FLUSH__SHIFT                                                        0x0
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB0__SHIFT                                              0x1
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB1_FLUSH__SHIFT                                                        0x2
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB1__SHIFT                                              0x3
+#define RMI_SCOREBOARD_CNTL__TIME_STAMP_FLUSH_RB1__SHIFT                                                      0x4
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_EN__SHIFT                                         0x5
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_VALUE__SHIFT                                      0x6
+#define RMI_SCOREBOARD_CNTL__TIME_STAMP_FLUSH_RB0__SHIFT                                                      0x7
+#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_EN__SHIFT                                                  0x8
+#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_TIMER_START_VALUE__SHIFT                                   0x9
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB0_FLUSH_MASK                                                          0x00000001L
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB0_MASK                                                0x00000002L
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB1_FLUSH_MASK                                                          0x00000004L
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB1_MASK                                                0x00000008L
+#define RMI_SCOREBOARD_CNTL__TIME_STAMP_FLUSH_RB1_MASK                                                        0x00000010L
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_EN_MASK                                           0x00000020L
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_VALUE_MASK                                        0x00000040L
+#define RMI_SCOREBOARD_CNTL__TIME_STAMP_FLUSH_RB0_MASK                                                        0x00000080L
+#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_EN_MASK                                                    0x00000100L
+#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_TIMER_START_VALUE_MASK                                     0x001FFE00L
+//RMI_SCOREBOARD_STATUS0
+#define RMI_SCOREBOARD_STATUS0__CURRENT_SESSION_ID__SHIFT                                                     0x0
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_IN_PROG__SHIFT                                                    0x1
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_REQ_VMID__SHIFT                                                   0x2
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_UTC_DONE__SHIFT                                                   0x12
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_DONE__SHIFT                                                       0x13
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_FLUSH_TYPE__SHIFT                                                 0x14
+#define RMI_SCOREBOARD_STATUS0__FORCE_VMID_INV_DONE__SHIFT                                                    0x15
+#define RMI_SCOREBOARD_STATUS0__CURRENT_SESSION_ID_MASK                                                       0x00000001L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_IN_PROG_MASK                                                      0x00000002L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_REQ_VMID_MASK                                                     0x0003FFFCL
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_UTC_DONE_MASK                                                     0x00040000L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_DONE_MASK                                                         0x00080000L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_FLUSH_TYPE_MASK                                                   0x00100000L
+#define RMI_SCOREBOARD_STATUS0__FORCE_VMID_INV_DONE_MASK                                                      0x00200000L
+//RMI_SCOREBOARD_STATUS1
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB0__SHIFT                                                        0x0
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB0__SHIFT                                              0xc
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB0__SHIFT                                               0xd
+#define RMI_SCOREBOARD_STATUS1__MULTI_VMID_INVAL_FROM_CP_DETECTED__SHIFT                                      0xe
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB1__SHIFT                                                        0xf
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB1__SHIFT                                              0x1b
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB1__SHIFT                                               0x1c
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB1__SHIFT                                                  0x1d
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB0__SHIFT                                                  0x1e
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB0_MASK                                                          0x00000FFFL
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB0_MASK                                                0x00001000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB0_MASK                                                 0x00002000L
+#define RMI_SCOREBOARD_STATUS1__MULTI_VMID_INVAL_FROM_CP_DETECTED_MASK                                        0x00004000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB1_MASK                                                          0x07FF8000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB1_MASK                                                0x08000000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB1_MASK                                                 0x10000000L
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB1_MASK                                                    0x20000000L
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB0_MASK                                                    0x40000000L
+//RMI_SCOREBOARD_STATUS2
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB0__SHIFT                                                       0x0
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB0__SHIFT                                             0xc
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB1__SHIFT                                                       0xd
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB1__SHIFT                                             0x19
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB1__SHIFT                                                     0x1a
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB0__SHIFT                                                     0x1b
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB0__SHIFT                                           0x1c
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB1__SHIFT                                           0x1d
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB0__SHIFT                                              0x1e
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB1__SHIFT                                              0x1f
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB0_MASK                                                         0x00000FFFL
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB0_MASK                                               0x00001000L
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB1_MASK                                                         0x01FFE000L
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB1_MASK                                               0x02000000L
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB1_MASK                                                       0x04000000L
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB0_MASK                                                       0x08000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB0_MASK                                             0x10000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB1_MASK                                             0x20000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB0_MASK                                                0x40000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB1_MASK                                                0x80000000L
+//RMI_XBAR_ARBITER_CONFIG
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE__SHIFT                                                        0x0
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_WEIGHTEDRR__SHIFT                                     0x2
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL__SHIFT                                                       0x3
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_IDLEIN__SHIFT                                         0x4
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_OVERRIDE__SHIFT                                        0x6
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_START_VALUE__SHIFT                                     0x8
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE__SHIFT                                                        0x10
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_WEIGHTEDRR__SHIFT                                     0x12
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL__SHIFT                                                       0x13
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_IDLEIN__SHIFT                                         0x14
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_OVERRIDE__SHIFT                                        0x16
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_START_VALUE__SHIFT                                     0x18
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE_MASK                                                          0x00000003L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_WEIGHTEDRR_MASK                                       0x00000004L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_MASK                                                         0x00000008L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_IDLEIN_MASK                                           0x00000010L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_OVERRIDE_MASK                                          0x000000C0L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_START_VALUE_MASK                                       0x0000FF00L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE_MASK                                                          0x00030000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_WEIGHTEDRR_MASK                                       0x00040000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_MASK                                                         0x00080000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_IDLEIN_MASK                                           0x00100000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_OVERRIDE_MASK                                          0x00C00000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_START_VALUE_MASK                                       0xFF000000L
+//RMI_XBAR_ARBITER_CONFIG_1
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_RD__SHIFT                                  0x0
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_WR__SHIFT                                  0x8
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB1_RD__SHIFT                                  0x10
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB1_WR__SHIFT                                  0x18
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_RD_MASK                                    0x000000FFL
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_WR_MASK                                    0x0000FF00L
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB1_RD_MASK                                    0x00FF0000L
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB1_WR_MASK                                    0xFF000000L
+//RMI_CLOCK_CNTRL
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_BUSY_MASK__SHIFT                                                         0x0
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_BUSY_MASK__SHIFT                                                         0x5
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_WAKEUP_MASK__SHIFT                                                       0xa
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_WAKEUP_MASK__SHIFT                                                       0xf
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB1_BUSY_MASK__SHIFT                                                         0x14
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB1_WAKEUP_MASK__SHIFT                                                       0x19
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_BUSY_MASK_MASK                                                           0x0000001FL
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_BUSY_MASK_MASK                                                           0x000003E0L
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_WAKEUP_MASK_MASK                                                         0x00007C00L
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_WAKEUP_MASK_MASK                                                         0x000F8000L
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB1_BUSY_MASK_MASK                                                           0x01F00000L
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB1_WAKEUP_MASK_MASK                                                         0x3E000000L
+//RMI_UTCL1_STATUS
+#define RMI_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define RMI_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define RMI_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define RMI_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define RMI_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define RMI_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+//RMI_XNACK_DEBUG
+#define RMI_XNACK_DEBUG__XNACK_PER_VMID__SHIFT                                                                0x0
+#define RMI_XNACK_DEBUG__XNACK_PER_VMID_MASK                                                                  0x0000FFFFL
+//RMI_SPARE
+#define RMI_SPARE__RMI_ARBITER_STALL_TIMER_ENABLED_ALLOW_STREAMING__SHIFT                                     0x0
+#define RMI_SPARE__SPARE_BIT_1__SHIFT                                                                         0x1
+#define RMI_SPARE__SPARE_BIT_2__SHIFT                                                                         0x2
+#define RMI_SPARE__SPARE_BIT_3__SHIFT                                                                         0x3
+#define RMI_SPARE__SPARE_BIT_4__SHIFT                                                                         0x4
+#define RMI_SPARE__SPARE_BIT_5__SHIFT                                                                         0x5
+#define RMI_SPARE__SPARE_BIT_6__SHIFT                                                                         0x6
+#define RMI_SPARE__SPARE_BIT_7__SHIFT                                                                         0x7
+#define RMI_SPARE__SPARE_BIT_8_0__SHIFT                                                                       0x8
+#define RMI_SPARE__SPARE_BIT_16_0__SHIFT                                                                      0x10
+#define RMI_SPARE__RMI_ARBITER_STALL_TIMER_ENABLED_ALLOW_STREAMING_MASK                                       0x00000001L
+#define RMI_SPARE__SPARE_BIT_1_MASK                                                                           0x00000002L
+#define RMI_SPARE__SPARE_BIT_2_MASK                                                                           0x00000004L
+#define RMI_SPARE__SPARE_BIT_3_MASK                                                                           0x00000008L
+#define RMI_SPARE__SPARE_BIT_4_MASK                                                                           0x00000010L
+#define RMI_SPARE__SPARE_BIT_5_MASK                                                                           0x00000020L
+#define RMI_SPARE__SPARE_BIT_6_MASK                                                                           0x00000040L
+#define RMI_SPARE__SPARE_BIT_7_MASK                                                                           0x00000080L
+#define RMI_SPARE__SPARE_BIT_8_0_MASK                                                                         0x0000FF00L
+#define RMI_SPARE__SPARE_BIT_16_0_MASK                                                                        0xFFFF0000L
+//RMI_SPARE_1
+#define RMI_SPARE_1__SPARE_BIT_8__SHIFT                                                                       0x0
+#define RMI_SPARE_1__SPARE_BIT_9__SHIFT                                                                       0x1
+#define RMI_SPARE_1__SPARE_BIT_10__SHIFT                                                                      0x2
+#define RMI_SPARE_1__SPARE_BIT_11__SHIFT                                                                      0x3
+#define RMI_SPARE_1__SPARE_BIT_12__SHIFT                                                                      0x4
+#define RMI_SPARE_1__SPARE_BIT_13__SHIFT                                                                      0x5
+#define RMI_SPARE_1__SPARE_BIT_14__SHIFT                                                                      0x6
+#define RMI_SPARE_1__SPARE_BIT_15__SHIFT                                                                      0x7
+#define RMI_SPARE_1__SPARE_BIT_8_1__SHIFT                                                                     0x8
+#define RMI_SPARE_1__SPARE_BIT_16_1__SHIFT                                                                    0x10
+#define RMI_SPARE_1__SPARE_BIT_8_MASK                                                                         0x00000001L
+#define RMI_SPARE_1__SPARE_BIT_9_MASK                                                                         0x00000002L
+#define RMI_SPARE_1__SPARE_BIT_10_MASK                                                                        0x00000004L
+#define RMI_SPARE_1__SPARE_BIT_11_MASK                                                                        0x00000008L
+#define RMI_SPARE_1__SPARE_BIT_12_MASK                                                                        0x00000010L
+#define RMI_SPARE_1__SPARE_BIT_13_MASK                                                                        0x00000020L
+#define RMI_SPARE_1__SPARE_BIT_14_MASK                                                                        0x00000040L
+#define RMI_SPARE_1__SPARE_BIT_15_MASK                                                                        0x00000080L
+#define RMI_SPARE_1__SPARE_BIT_8_1_MASK                                                                       0x0000FF00L
+#define RMI_SPARE_1__SPARE_BIT_16_1_MASK                                                                      0xFFFF0000L
+//RMI_SPARE_2
+#define RMI_SPARE_2__SPARE_BIT_16__SHIFT                                                                      0x0
+#define RMI_SPARE_2__SPARE_BIT_17__SHIFT                                                                      0x1
+#define RMI_SPARE_2__SPARE_BIT_18__SHIFT                                                                      0x2
+#define RMI_SPARE_2__SPARE_BIT_19__SHIFT                                                                      0x3
+#define RMI_SPARE_2__SPARE_BIT_20__SHIFT                                                                      0x4
+#define RMI_SPARE_2__SPARE_BIT_21__SHIFT                                                                      0x5
+#define RMI_SPARE_2__SPARE_BIT_22__SHIFT                                                                      0x6
+#define RMI_SPARE_2__SPARE_BIT_23__SHIFT                                                                      0x7
+#define RMI_SPARE_2__SPARE_BIT_4_0__SHIFT                                                                     0x8
+#define RMI_SPARE_2__SPARE_BIT_4_1__SHIFT                                                                     0xc
+#define RMI_SPARE_2__SPARE_BIT_8_2__SHIFT                                                                     0x10
+#define RMI_SPARE_2__SPARE_BIT_8_3__SHIFT                                                                     0x18
+#define RMI_SPARE_2__SPARE_BIT_16_MASK                                                                        0x00000001L
+#define RMI_SPARE_2__SPARE_BIT_17_MASK                                                                        0x00000002L
+#define RMI_SPARE_2__SPARE_BIT_18_MASK                                                                        0x00000004L
+#define RMI_SPARE_2__SPARE_BIT_19_MASK                                                                        0x00000008L
+#define RMI_SPARE_2__SPARE_BIT_20_MASK                                                                        0x00000010L
+#define RMI_SPARE_2__SPARE_BIT_21_MASK                                                                        0x00000020L
+#define RMI_SPARE_2__SPARE_BIT_22_MASK                                                                        0x00000040L
+#define RMI_SPARE_2__SPARE_BIT_23_MASK                                                                        0x00000080L
+#define RMI_SPARE_2__SPARE_BIT_4_0_MASK                                                                       0x00000F00L
+#define RMI_SPARE_2__SPARE_BIT_4_1_MASK                                                                       0x0000F000L
+#define RMI_SPARE_2__SPARE_BIT_8_2_MASK                                                                       0x00FF0000L
+#define RMI_SPARE_2__SPARE_BIT_8_3_MASK                                                                       0xFF000000L
+
+
+// addressBlock: xcd0_gc_utcl2_atcl2dec
+//ATC_L2_CNTL
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS__SHIFT                                               0x0
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS__SHIFT                                              0x3
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                                   0x6
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                                  0x7
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READ_REQUESTS__SHIFT                                          0x8
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITE_REQUESTS__SHIFT                                         0xb
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                              0xe
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                             0xf
+#define ATC_L2_CNTL__CACHE_INVALIDATE_MODE__SHIFT                                                             0x10
+#define ATC_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY__SHIFT                                          0x13
+#define ATC_L2_CNTL__FRAG_APT_INTXN_MODE__SHIFT                                                               0x14
+#define ATC_L2_CNTL__CLI_GPA_REQ_FRAG_SIZE__SHIFT                                                             0x16
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS_MASK                                                 0x00000003L
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS_MASK                                                0x00000018L
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                                     0x00000040L
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                                    0x00000080L
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READ_REQUESTS_MASK                                            0x00000300L
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITE_REQUESTS_MASK                                           0x00001800L
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                                0x00004000L
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                               0x00008000L
+#define ATC_L2_CNTL__CACHE_INVALIDATE_MODE_MASK                                                               0x00070000L
+#define ATC_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY_MASK                                            0x00080000L
+#define ATC_L2_CNTL__FRAG_APT_INTXN_MODE_MASK                                                                 0x00300000L
+#define ATC_L2_CNTL__CLI_GPA_REQ_FRAG_SIZE_MASK                                                               0x0FC00000L
+//ATC_L2_CNTL2
+#define ATC_L2_CNTL2__BANK_SELECT__SHIFT                                                                      0x0
+#define ATC_L2_CNTL2__NUM_BANKS_LOG2__SHIFT                                                                   0x6
+#define ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE__SHIFT                                                             0x9
+#define ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                              0xb
+#define ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS__SHIFT                                                     0xc
+#define ATC_L2_CNTL2__L2_CACHE_VMID_MODE__SHIFT                                                               0xf
+#define ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                         0x12
+#define ATC_L2_CNTL2__BANK_SELECT_MASK                                                                        0x0000003FL
+#define ATC_L2_CNTL2__NUM_BANKS_LOG2_MASK                                                                     0x000001C0L
+#define ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE_MASK                                                               0x00000600L
+#define ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE_MASK                                                0x00000800L
+#define ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS_MASK                                                       0x00007000L
+#define ATC_L2_CNTL2__L2_CACHE_VMID_MODE_MASK                                                                 0x00038000L
+#define ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                           0x00FC0000L
+//ATC_L2_CACHE_DATA0
+#define ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID__SHIFT                                                        0x0
+#define ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID__SHIFT                                                          0x1
+#define ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES__SHIFT                                                          0x2
+#define ATC_L2_CACHE_DATA0__VIRTUAL_PAGE_ADDRESS_HIGH__SHIFT                                                  0x17
+#define ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID_MASK                                                          0x00000001L
+#define ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID_MASK                                                            0x00000002L
+#define ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES_MASK                                                            0x007FFFFCL
+#define ATC_L2_CACHE_DATA0__VIRTUAL_PAGE_ADDRESS_HIGH_MASK                                                    0x07800000L
+//ATC_L2_CACHE_DATA1
+#define ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW__SHIFT                                                   0x0
+#define ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW_MASK                                                     0xFFFFFFFFL
+//ATC_L2_CACHE_DATA2
+#define ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS__SHIFT                                                      0x0
+#define ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS_MASK                                                        0xFFFFFFFFL
+//ATC_L2_CACHE_DATA3
+#define ATC_L2_CACHE_DATA3__PHYSICAL_PAGE_ADDRESS__SHIFT                                                      0x0
+#define ATC_L2_CACHE_DATA3__PHYSICAL_PAGE_ADDRESS_MASK                                                        0xFFFFFFFFL
+//ATC_L2_CNTL3
+#define ATC_L2_CNTL3__L2_SMALLK_FRAGMENT_SIZE__SHIFT                                                          0x0
+#define ATC_L2_CNTL3__L2_MIDK_FRAGMENT_SIZE__SHIFT                                                            0x6
+#define ATC_L2_CNTL3__L2_BIGK_FRAGMENT_SIZE__SHIFT                                                            0xc
+#define ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST__SHIFT                                                  0x12
+#define ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1__SHIFT                                                        0x15
+#define ATC_L2_CNTL3__COMPCLKREQ_OFF_HYSTERESIS__SHIFT                                                        0x1b
+#define ATC_L2_CNTL3__REPEATER_FGCG_OFF__SHIFT                                                                0x1e
+#define ATC_L2_CNTL3__L2_SMALLK_FRAGMENT_SIZE_MASK                                                            0x0000003FL
+#define ATC_L2_CNTL3__L2_MIDK_FRAGMENT_SIZE_MASK                                                              0x00000FC0L
+#define ATC_L2_CNTL3__L2_BIGK_FRAGMENT_SIZE_MASK                                                              0x0003F000L
+#define ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST_MASK                                                    0x001C0000L
+#define ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1_MASK                                                          0x07E00000L
+#define ATC_L2_CNTL3__COMPCLKREQ_OFF_HYSTERESIS_MASK                                                          0x38000000L
+#define ATC_L2_CNTL3__REPEATER_FGCG_OFF_MASK                                                                  0x40000000L
+//ATC_L2_STATUS
+#define ATC_L2_STATUS__BUSY__SHIFT                                                                            0x0
+#define ATC_L2_STATUS__NO_OUTSTANDING_AT_REQUESTS__SHIFT                                                      0x1
+#define ATC_L2_STATUS__BUSY_MASK                                                                              0x00000001L
+#define ATC_L2_STATUS__NO_OUTSTANDING_AT_REQUESTS_MASK                                                        0x00000002L
+//ATC_L2_STATUS2
+#define ATC_L2_STATUS2__UCE_MEM_ADDR__SHIFT                                                                   0x0
+#define ATC_L2_STATUS2__UCE_MEM_INST__SHIFT                                                                   0xc
+#define ATC_L2_STATUS2__UCE_SRT_CACHE__SHIFT                                                                  0x14
+#define ATC_L2_STATUS2__UCE__SHIFT                                                                            0x15
+#define ATC_L2_STATUS2__UCE_MEM_ADDR_MASK                                                                     0x00000FFFL
+#define ATC_L2_STATUS2__UCE_MEM_INST_MASK                                                                     0x000FF000L
+#define ATC_L2_STATUS2__UCE_SRT_CACHE_MASK                                                                    0x00100000L
+#define ATC_L2_STATUS2__UCE_MASK                                                                              0x00200000L
+//ATC_L2_MISC_CG
+#define ATC_L2_MISC_CG__OFFDLY__SHIFT                                                                         0x6
+#define ATC_L2_MISC_CG__ENABLE__SHIFT                                                                         0x12
+#define ATC_L2_MISC_CG__MEM_LS_ENABLE__SHIFT                                                                  0x13
+#define ATC_L2_MISC_CG__OFFDLY_MASK                                                                           0x00000FC0L
+#define ATC_L2_MISC_CG__ENABLE_MASK                                                                           0x00040000L
+#define ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK                                                                    0x00080000L
+//ATC_L2_MEM_POWER_LS
+#define ATC_L2_MEM_POWER_LS__LS_SETUP__SHIFT                                                                  0x0
+#define ATC_L2_MEM_POWER_LS__LS_HOLD__SHIFT                                                                   0x6
+#define ATC_L2_MEM_POWER_LS__LS_SETUP_MASK                                                                    0x0000003FL
+#define ATC_L2_MEM_POWER_LS__LS_HOLD_MASK                                                                     0x00000FC0L
+//ATC_L2_CGTT_CLK_CTRL
+#define ATC_L2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
+#define ATC_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
+#define ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                            0xf
+#define ATC_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE__SHIFT                                                      0x10
+#define ATC_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                            0x18
+#define ATC_L2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
+#define ATC_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
+#define ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK                                                              0x00008000L
+#define ATC_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_MASK                                                        0x00FF0000L
+#define ATC_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                              0xFF000000L
+//ATC_L2_CACHE_4K_DSM_INDEX
+#define ATC_L2_CACHE_4K_DSM_INDEX__INDEX__SHIFT                                                               0x0
+#define ATC_L2_CACHE_4K_DSM_INDEX__INDEX_MASK                                                                 0x000000FFL
+//ATC_L2_CACHE_32K_DSM_INDEX
+#define ATC_L2_CACHE_32K_DSM_INDEX__INDEX__SHIFT                                                              0x0
+#define ATC_L2_CACHE_32K_DSM_INDEX__INDEX_MASK                                                                0x000000FFL
+//ATC_L2_CACHE_2M_DSM_INDEX
+#define ATC_L2_CACHE_2M_DSM_INDEX__INDEX__SHIFT                                                               0x0
+#define ATC_L2_CACHE_2M_DSM_INDEX__INDEX_MASK                                                                 0x000000FFL
+//ATC_L2_CACHE_4K_DSM_CNTL
+#define ATC_L2_CACHE_4K_DSM_CNTL__INJECT_DELAY__SHIFT                                                         0x0
+#define ATC_L2_CACHE_4K_DSM_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                   0x6
+#define ATC_L2_CACHE_4K_DSM_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                  0x8
+#define ATC_L2_CACHE_4K_DSM_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                  0x9
+#define ATC_L2_CACHE_4K_DSM_CNTL__SELECT_INJECT_DELAY__SHIFT                                                  0xb
+#define ATC_L2_CACHE_4K_DSM_CNTL__WRITE_COUNTERS__SHIFT                                                       0xc
+#define ATC_L2_CACHE_4K_DSM_CNTL__SEC_COUNT__SHIFT                                                            0xd
+#define ATC_L2_CACHE_4K_DSM_CNTL__DED_COUNT__SHIFT                                                            0xf
+#define ATC_L2_CACHE_4K_DSM_CNTL__TEST_FUE__SHIFT                                                             0x11
+#define ATC_L2_CACHE_4K_DSM_CNTL__INJECT_DELAY_MASK                                                           0x0000003FL
+#define ATC_L2_CACHE_4K_DSM_CNTL__DSM_IRRITATOR_DATA_MASK                                                     0x000000C0L
+#define ATC_L2_CACHE_4K_DSM_CNTL__ENABLE_SINGLE_WRITE_MASK                                                    0x00000100L
+#define ATC_L2_CACHE_4K_DSM_CNTL__ENABLE_ERROR_INJECT_MASK                                                    0x00000600L
+#define ATC_L2_CACHE_4K_DSM_CNTL__SELECT_INJECT_DELAY_MASK                                                    0x00000800L
+#define ATC_L2_CACHE_4K_DSM_CNTL__WRITE_COUNTERS_MASK                                                         0x00001000L
+#define ATC_L2_CACHE_4K_DSM_CNTL__SEC_COUNT_MASK                                                              0x00006000L
+#define ATC_L2_CACHE_4K_DSM_CNTL__DED_COUNT_MASK                                                              0x00018000L
+#define ATC_L2_CACHE_4K_DSM_CNTL__TEST_FUE_MASK                                                               0x00020000L
+//ATC_L2_CACHE_32K_DSM_CNTL
+#define ATC_L2_CACHE_32K_DSM_CNTL__INJECT_DELAY__SHIFT                                                        0x0
+#define ATC_L2_CACHE_32K_DSM_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                  0x6
+#define ATC_L2_CACHE_32K_DSM_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                 0x8
+#define ATC_L2_CACHE_32K_DSM_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                 0x9
+#define ATC_L2_CACHE_32K_DSM_CNTL__SELECT_INJECT_DELAY__SHIFT                                                 0xb
+#define ATC_L2_CACHE_32K_DSM_CNTL__WRITE_COUNTERS__SHIFT                                                      0xc
+#define ATC_L2_CACHE_32K_DSM_CNTL__SEC_COUNT__SHIFT                                                           0xd
+#define ATC_L2_CACHE_32K_DSM_CNTL__DED_COUNT__SHIFT                                                           0xf
+#define ATC_L2_CACHE_32K_DSM_CNTL__TEST_FUE__SHIFT                                                            0x11
+#define ATC_L2_CACHE_32K_DSM_CNTL__INJECT_DELAY_MASK                                                          0x0000003FL
+#define ATC_L2_CACHE_32K_DSM_CNTL__DSM_IRRITATOR_DATA_MASK                                                    0x000000C0L
+#define ATC_L2_CACHE_32K_DSM_CNTL__ENABLE_SINGLE_WRITE_MASK                                                   0x00000100L
+#define ATC_L2_CACHE_32K_DSM_CNTL__ENABLE_ERROR_INJECT_MASK                                                   0x00000600L
+#define ATC_L2_CACHE_32K_DSM_CNTL__SELECT_INJECT_DELAY_MASK                                                   0x00000800L
+#define ATC_L2_CACHE_32K_DSM_CNTL__WRITE_COUNTERS_MASK                                                        0x00001000L
+#define ATC_L2_CACHE_32K_DSM_CNTL__SEC_COUNT_MASK                                                             0x00006000L
+#define ATC_L2_CACHE_32K_DSM_CNTL__DED_COUNT_MASK                                                             0x00018000L
+#define ATC_L2_CACHE_32K_DSM_CNTL__TEST_FUE_MASK                                                              0x00020000L
+//ATC_L2_CACHE_2M_DSM_CNTL
+#define ATC_L2_CACHE_2M_DSM_CNTL__INJECT_DELAY__SHIFT                                                         0x0
+#define ATC_L2_CACHE_2M_DSM_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                   0x6
+#define ATC_L2_CACHE_2M_DSM_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                  0x8
+#define ATC_L2_CACHE_2M_DSM_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                  0x9
+#define ATC_L2_CACHE_2M_DSM_CNTL__SELECT_INJECT_DELAY__SHIFT                                                  0xb
+#define ATC_L2_CACHE_2M_DSM_CNTL__WRITE_COUNTERS__SHIFT                                                       0xc
+#define ATC_L2_CACHE_2M_DSM_CNTL__SEC_COUNT__SHIFT                                                            0xd
+#define ATC_L2_CACHE_2M_DSM_CNTL__DED_COUNT__SHIFT                                                            0xf
+#define ATC_L2_CACHE_2M_DSM_CNTL__TEST_FUE__SHIFT                                                             0x11
+#define ATC_L2_CACHE_2M_DSM_CNTL__INJECT_DELAY_MASK                                                           0x0000003FL
+#define ATC_L2_CACHE_2M_DSM_CNTL__DSM_IRRITATOR_DATA_MASK                                                     0x000000C0L
+#define ATC_L2_CACHE_2M_DSM_CNTL__ENABLE_SINGLE_WRITE_MASK                                                    0x00000100L
+#define ATC_L2_CACHE_2M_DSM_CNTL__ENABLE_ERROR_INJECT_MASK                                                    0x00000600L
+#define ATC_L2_CACHE_2M_DSM_CNTL__SELECT_INJECT_DELAY_MASK                                                    0x00000800L
+#define ATC_L2_CACHE_2M_DSM_CNTL__WRITE_COUNTERS_MASK                                                         0x00001000L
+#define ATC_L2_CACHE_2M_DSM_CNTL__SEC_COUNT_MASK                                                              0x00006000L
+#define ATC_L2_CACHE_2M_DSM_CNTL__DED_COUNT_MASK                                                              0x00018000L
+#define ATC_L2_CACHE_2M_DSM_CNTL__TEST_FUE_MASK                                                               0x00020000L
+//ATC_L2_CNTL4
+#define ATC_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                          0x0
+#define ATC_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                         0xa
+#define ATC_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                            0x000003FFL
+#define ATC_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                           0x000FFC00L
+//ATC_L2_MM_GROUP_RT_CLASSES
+#define ATC_L2_MM_GROUP_RT_CLASSES__GROUP_RT_CLASS__SHIFT                                                     0x0
+#define ATC_L2_MM_GROUP_RT_CLASSES__GROUP_RT_CLASS_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: xcd0_gc_utcl2_vml2pfdec
+//VM_L2_CNTL
+#define VM_L2_CNTL__ENABLE_L2_CACHE__SHIFT                                                                    0x0
+#define VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING__SHIFT                                                      0x1
+#define VM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE__SHIFT                                                      0x2
+#define VM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE__SHIFT                                                      0x4
+#define VM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE__SHIFT                                                  0x8
+#define VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                            0x9
+#define VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                           0xa
+#define VM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY__SHIFT                                           0xb
+#define VM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE__SHIFT                                                           0xc
+#define VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT                                                            0xf
+#define VM_L2_CNTL__PDE_FAULT_CLASSIFICATION__SHIFT                                                           0x12
+#define VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT                                                      0x13
+#define VM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE__SHIFT                                                        0x15
+#define VM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE__SHIFT                                                             0x1a
+#define VM_L2_CNTL__ENABLE_L2_CACHE_MASK                                                                      0x00000001L
+#define VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK                                                        0x00000002L
+#define VM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE_MASK                                                        0x0000000CL
+#define VM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE_MASK                                                        0x00000030L
+#define VM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE_MASK                                                    0x00000100L
+#define VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK                                              0x00000200L
+#define VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK                                             0x00000400L
+#define VM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY_MASK                                             0x00000800L
+#define VM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE_MASK                                                             0x00007000L
+#define VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE_MASK                                                              0x00038000L
+#define VM_L2_CNTL__PDE_FAULT_CLASSIFICATION_MASK                                                             0x00040000L
+#define VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE_MASK                                                        0x00180000L
+#define VM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE_MASK                                                          0x03E00000L
+#define VM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE_MASK                                                               0x0C000000L
+//VM_L2_CNTL2
+#define VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS__SHIFT                                                            0x0
+#define VM_L2_CNTL2__INVALIDATE_L2_CACHE__SHIFT                                                               0x1
+#define VM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN__SHIFT                                                     0x15
+#define VM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION__SHIFT                                                   0x16
+#define VM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE__SHIFT                                                            0x17
+#define VM_L2_CNTL2__INVALIDATE_CACHE_MODE__SHIFT                                                             0x1a
+#define VM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE__SHIFT                                                          0x1c
+#define VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK                                                              0x00000001L
+#define VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK                                                                 0x00000002L
+#define VM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN_MASK                                                       0x00200000L
+#define VM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION_MASK                                                     0x00400000L
+#define VM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE_MASK                                                              0x03800000L
+#define VM_L2_CNTL2__INVALIDATE_CACHE_MODE_MASK                                                               0x0C000000L
+#define VM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE_MASK                                                            0x70000000L
+//VM_L2_CNTL3
+#define VM_L2_CNTL3__BANK_SELECT__SHIFT                                                                       0x0
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_MODE__SHIFT                                                              0x6
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                          0x8
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                                                       0xf
+#define VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY__SHIFT                                                       0x14
+#define VM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE__SHIFT                                                        0x15
+#define VM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE__SHIFT                                                      0x18
+#define VM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS__SHIFT                                                            0x1c
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS__SHIFT                                                          0x1d
+#define VM_L2_CNTL3__PDE_CACHE_FORCE_MISS__SHIFT                                                              0x1e
+#define VM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY__SHIFT                                                         0x1f
+#define VM_L2_CNTL3__BANK_SELECT_MASK                                                                         0x0000003FL
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_MODE_MASK                                                                0x000000C0L
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                            0x00001F00L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                                                         0x000F8000L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK                                                         0x00100000L
+#define VM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE_MASK                                                          0x00E00000L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE_MASK                                                        0x0F000000L
+#define VM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS_MASK                                                              0x10000000L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS_MASK                                                            0x20000000L
+#define VM_L2_CNTL3__PDE_CACHE_FORCE_MISS_MASK                                                                0x40000000L
+#define VM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY_MASK                                                           0x80000000L
+//VM_L2_STATUS
+#define VM_L2_STATUS__L2_BUSY__SHIFT                                                                          0x0
+#define VM_L2_STATUS__CONTEXT_DOMAIN_BUSY__SHIFT                                                              0x1
+#define VM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS__SHIFT                                                 0x11
+#define VM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS__SHIFT                                               0x12
+#define VM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS__SHIFT                                                   0x13
+#define VM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS__SHIFT                                                   0x14
+#define VM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS__SHIFT                                                   0x15
+#define VM_L2_STATUS__L2_BUSY_MASK                                                                            0x00000001L
+#define VM_L2_STATUS__CONTEXT_DOMAIN_BUSY_MASK                                                                0x0001FFFEL
+#define VM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS_MASK                                                   0x00020000L
+#define VM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS_MASK                                                 0x00040000L
+#define VM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS_MASK                                                     0x00080000L
+#define VM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS_MASK                                                     0x00100000L
+#define VM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS_MASK                                                     0x00200000L
+//VM_DUMMY_PAGE_FAULT_CNTL
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE__SHIFT                                              0x0
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL__SHIFT                                           0x1
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS__SHIFT                                              0x2
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE_MASK                                                0x00000001L
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL_MASK                                             0x00000002L
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS_MASK                                                0x000000FCL
+//VM_DUMMY_PAGE_FAULT_ADDR_LO32
+#define VM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32__SHIFT                                            0x0
+#define VM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32_MASK                                              0xFFFFFFFFL
+//VM_DUMMY_PAGE_FAULT_ADDR_HI32
+#define VM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI4__SHIFT                                             0x0
+#define VM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI4_MASK                                               0x0000000FL
+//VM_L2_PROTECTION_FAULT_CNTL
+#define VM_L2_PROTECTION_FAULT_CNTL__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                0x0
+#define VM_L2_PROTECTION_FAULT_CNTL__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES__SHIFT             0x1
+#define VM_L2_PROTECTION_FAULT_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                             0x2
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x3
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x4
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x5
+#define VM_L2_PROTECTION_FAULT_CNTL__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                 0x6
+#define VM_L2_PROTECTION_FAULT_CNTL__NACK_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x7
+#define VM_L2_PROTECTION_FAULT_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                        0x8
+#define VM_L2_PROTECTION_FAULT_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                             0x9
+#define VM_L2_PROTECTION_FAULT_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0xa
+#define VM_L2_PROTECTION_FAULT_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                             0xb
+#define VM_L2_PROTECTION_FAULT_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                           0xc
+#define VM_L2_PROTECTION_FAULT_CNTL__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                                0xd
+#define VM_L2_PROTECTION_FAULT_CNTL__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                          0x1d
+#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_NO_RETRY_FAULT__SHIFT                                           0x1e
+#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_RETRY_FAULT__SHIFT                                              0x1f
+#define VM_L2_PROTECTION_FAULT_CNTL__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                  0x00000001L
+#define VM_L2_PROTECTION_FAULT_CNTL__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES_MASK               0x00000002L
+#define VM_L2_PROTECTION_FAULT_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                               0x00000004L
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000008L
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000010L
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000020L
+#define VM_L2_PROTECTION_FAULT_CNTL__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                   0x00000040L
+#define VM_L2_PROTECTION_FAULT_CNTL__NACK_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000080L
+#define VM_L2_PROTECTION_FAULT_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                          0x00000100L
+#define VM_L2_PROTECTION_FAULT_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                               0x00000200L
+#define VM_L2_PROTECTION_FAULT_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000400L
+#define VM_L2_PROTECTION_FAULT_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                               0x00000800L
+#define VM_L2_PROTECTION_FAULT_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                             0x00001000L
+#define VM_L2_PROTECTION_FAULT_CNTL__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                                  0x1FFFE000L
+#define VM_L2_PROTECTION_FAULT_CNTL__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                            0x20000000L
+#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_NO_RETRY_FAULT_MASK                                             0x40000000L
+#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_RETRY_FAULT_MASK                                                0x80000000L
+//VM_L2_PROTECTION_FAULT_CNTL2
+#define VM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                                    0x0
+#define VM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                              0x10
+#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE__SHIFT                                        0x11
+#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY__SHIFT                             0x12
+#define VM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT__SHIFT                                     0x13
+#define VM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                                      0x0000FFFFL
+#define VM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                                0x00010000L
+#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_MASK                                          0x00020000L
+#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY_MASK                               0x00040000L
+#define VM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT_MASK                                       0x00080000L
+//VM_L2_PROTECTION_FAULT_MM_CNTL3
+#define VM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                  0x0
+#define VM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                    0xFFFFFFFFL
+//VM_L2_PROTECTION_FAULT_MM_CNTL4
+#define VM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                 0x0
+#define VM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                   0xFFFFFFFFL
+//VM_L2_PROTECTION_FAULT_STATUS
+#define VM_L2_PROTECTION_FAULT_STATUS__MORE_FAULTS__SHIFT                                                     0x0
+#define VM_L2_PROTECTION_FAULT_STATUS__WALKER_ERROR__SHIFT                                                    0x1
+#define VM_L2_PROTECTION_FAULT_STATUS__PERMISSION_FAULTS__SHIFT                                               0x4
+#define VM_L2_PROTECTION_FAULT_STATUS__MAPPING_ERROR__SHIFT                                                   0x8
+#define VM_L2_PROTECTION_FAULT_STATUS__CID__SHIFT                                                             0x9
+#define VM_L2_PROTECTION_FAULT_STATUS__RW__SHIFT                                                              0x12
+#define VM_L2_PROTECTION_FAULT_STATUS__ATOMIC__SHIFT                                                          0x13
+#define VM_L2_PROTECTION_FAULT_STATUS__VMID__SHIFT                                                            0x14
+#define VM_L2_PROTECTION_FAULT_STATUS__VF__SHIFT                                                              0x18
+#define VM_L2_PROTECTION_FAULT_STATUS__VFID__SHIFT                                                            0x19
+#define VM_L2_PROTECTION_FAULT_STATUS__UCE__SHIFT                                                             0x1d
+#define VM_L2_PROTECTION_FAULT_STATUS__FED__SHIFT                                                             0x1e
+#define VM_L2_PROTECTION_FAULT_STATUS__MORE_FAULTS_MASK                                                       0x00000001L
+#define VM_L2_PROTECTION_FAULT_STATUS__WALKER_ERROR_MASK                                                      0x0000000EL
+#define VM_L2_PROTECTION_FAULT_STATUS__PERMISSION_FAULTS_MASK                                                 0x000000F0L
+#define VM_L2_PROTECTION_FAULT_STATUS__MAPPING_ERROR_MASK                                                     0x00000100L
+#define VM_L2_PROTECTION_FAULT_STATUS__CID_MASK                                                               0x0003FE00L
+#define VM_L2_PROTECTION_FAULT_STATUS__RW_MASK                                                                0x00040000L
+#define VM_L2_PROTECTION_FAULT_STATUS__ATOMIC_MASK                                                            0x00080000L
+#define VM_L2_PROTECTION_FAULT_STATUS__VMID_MASK                                                              0x00F00000L
+#define VM_L2_PROTECTION_FAULT_STATUS__VF_MASK                                                                0x01000000L
+#define VM_L2_PROTECTION_FAULT_STATUS__VFID_MASK                                                              0x1E000000L
+#define VM_L2_PROTECTION_FAULT_STATUS__UCE_MASK                                                               0x20000000L
+#define VM_L2_PROTECTION_FAULT_STATUS__FED_MASK                                                               0x40000000L
+//VM_L2_PROTECTION_FAULT_ADDR_LO32
+#define VM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32__SHIFT                                       0x0
+#define VM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32_MASK                                         0xFFFFFFFFL
+//VM_L2_PROTECTION_FAULT_ADDR_HI32
+#define VM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI4__SHIFT                                        0x0
+#define VM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI4_MASK                                          0x0000000FL
+//VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32
+#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32__SHIFT                              0x0
+#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32_MASK                                0xFFFFFFFFL
+//VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32
+#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI4__SHIFT                               0x0
+#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI4_MASK                                 0x0000000FL
+//VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                       0x0
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                         0xFFFFFFFFL
+//VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                        0x0
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                          0x0000000FL
+//VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                      0x0
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                        0xFFFFFFFFL
+//VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                       0x0
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                         0x0000000FL
+//VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32
+#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32__SHIFT                         0x0
+#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32_MASK                           0xFFFFFFFFL
+//VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32
+#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI4__SHIFT                          0x0
+#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI4_MASK                            0x0000000FL
+//VM_L2_CNTL4
+#define VM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT__SHIFT                                                       0x0
+#define VM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL__SHIFT                                                      0x6
+#define VM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL__SHIFT                                                      0x7
+#define VM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                           0x8
+#define VM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                          0x12
+#define VM_L2_CNTL4__BPM_CGCGLS_OVERRIDE__SHIFT                                                               0x1c
+#define VM_L2_CNTL4__GC_CH_FGCG_OFF__SHIFT                                                                    0x1d
+#define VM_L2_CNTL4__VFIFO_HEAD_OF_QUEUE__SHIFT                                                               0x1e
+#define VM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT_MASK                                                         0x0000003FL
+#define VM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL_MASK                                                        0x00000040L
+#define VM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL_MASK                                                        0x00000080L
+#define VM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                             0x0003FF00L
+#define VM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                            0x0FFC0000L
+#define VM_L2_CNTL4__BPM_CGCGLS_OVERRIDE_MASK                                                                 0x10000000L
+#define VM_L2_CNTL4__GC_CH_FGCG_OFF_MASK                                                                      0x20000000L
+#define VM_L2_CNTL4__VFIFO_HEAD_OF_QUEUE_MASK                                                                 0x40000000L
+//VM_L2_CNTL5
+#define VM_L2_CNTL5__WALKER_FETCH_PDE_MTYPE_ENABLE__SHIFT                                                     0x0
+#define VM_L2_CNTL5__WALKER_FETCH_PDE_NOALLOC_ENABLE__SHIFT                                                   0x1
+#define VM_L2_CNTL5__WALKER_FETCH_PDE_MTYPE_ENABLE_MASK                                                       0x00000001L
+#define VM_L2_CNTL5__WALKER_FETCH_PDE_NOALLOC_ENABLE_MASK                                                     0x00000002L
+//VM_L2_MM_GROUP_RT_CLASSES
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS__SHIFT                                                    0x0
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS__SHIFT                                                    0x1
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS__SHIFT                                                    0x2
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS__SHIFT                                                    0x3
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS__SHIFT                                                    0x4
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS__SHIFT                                                    0x5
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS__SHIFT                                                    0x6
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS__SHIFT                                                    0x7
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS__SHIFT                                                    0x8
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS__SHIFT                                                    0x9
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS__SHIFT                                                   0xa
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS__SHIFT                                                   0xb
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS__SHIFT                                                   0xc
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS__SHIFT                                                   0xd
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS__SHIFT                                                   0xe
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS__SHIFT                                                   0xf
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS__SHIFT                                                   0x10
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS__SHIFT                                                   0x11
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS__SHIFT                                                   0x12
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS__SHIFT                                                   0x13
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS__SHIFT                                                   0x14
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS__SHIFT                                                   0x15
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS__SHIFT                                                   0x16
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS__SHIFT                                                   0x17
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS__SHIFT                                                   0x18
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS__SHIFT                                                   0x19
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS__SHIFT                                                   0x1a
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS__SHIFT                                                   0x1b
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS__SHIFT                                                   0x1c
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS__SHIFT                                                   0x1d
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS__SHIFT                                                   0x1e
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS__SHIFT                                                   0x1f
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS_MASK                                                      0x00000001L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS_MASK                                                      0x00000002L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS_MASK                                                      0x00000004L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS_MASK                                                      0x00000008L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS_MASK                                                      0x00000010L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS_MASK                                                      0x00000020L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS_MASK                                                      0x00000040L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS_MASK                                                      0x00000080L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS_MASK                                                      0x00000100L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS_MASK                                                      0x00000200L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS_MASK                                                     0x00000400L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS_MASK                                                     0x00000800L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS_MASK                                                     0x00001000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS_MASK                                                     0x00002000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS_MASK                                                     0x00004000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS_MASK                                                     0x00008000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS_MASK                                                     0x00010000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS_MASK                                                     0x00020000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS_MASK                                                     0x00040000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS_MASK                                                     0x00080000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS_MASK                                                     0x00100000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS_MASK                                                     0x00200000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS_MASK                                                     0x00400000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS_MASK                                                     0x00800000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS_MASK                                                     0x01000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS_MASK                                                     0x02000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS_MASK                                                     0x04000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS_MASK                                                     0x08000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS_MASK                                                     0x10000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS_MASK                                                     0x20000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS_MASK                                                     0x40000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS_MASK                                                     0x80000000L
+//VM_L2_BANK_SELECT_RESERVED_CID
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID__SHIFT                                        0x0
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID__SHIFT                                       0xa
+#define VM_L2_BANK_SELECT_RESERVED_CID__ENABLE__SHIFT                                                         0x14
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                               0x18
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                            0x19
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID_MASK                                          0x000001FFL
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID_MASK                                         0x0007FC00L
+#define VM_L2_BANK_SELECT_RESERVED_CID__ENABLE_MASK                                                           0x00100000L
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE_MASK                                 0x01000000L
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                              0x02000000L
+//VM_L2_BANK_SELECT_RESERVED_CID2
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID__SHIFT                                       0x0
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID__SHIFT                                      0xa
+#define VM_L2_BANK_SELECT_RESERVED_CID2__ENABLE__SHIFT                                                        0x14
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                              0x18
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                           0x19
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID_MASK                                         0x000001FFL
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID_MASK                                        0x0007FC00L
+#define VM_L2_BANK_SELECT_RESERVED_CID2__ENABLE_MASK                                                          0x00100000L
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE_MASK                                0x01000000L
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                             0x02000000L
+//VM_L2_CACHE_PARITY_CNTL
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES__SHIFT                                 0x0
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES__SHIFT                               0x1
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES__SHIFT                                    0x2
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE__SHIFT                                 0x3
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE__SHIFT                               0x4
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE__SHIFT                                    0x5
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK__SHIFT                                                      0x6
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER__SHIFT                                                    0x9
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC__SHIFT                                                     0xc
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES_MASK                                   0x00000001L
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES_MASK                                 0x00000002L
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES_MASK                                      0x00000004L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE_MASK                                   0x00000008L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE_MASK                                 0x00000010L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE_MASK                                      0x00000020L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK_MASK                                                        0x000001C0L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER_MASK                                                      0x00000E00L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC_MASK                                                       0x0000F000L
+//VM_L2_CGTT_CLK_CTRL
+#define VM_L2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define VM_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define VM_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                             0xf
+#define VM_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE__SHIFT                                                       0x10
+#define VM_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                             0x18
+#define VM_L2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define VM_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define VM_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK                                                               0x00008000L
+#define VM_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_MASK                                                         0x00FF0000L
+#define VM_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                               0xFF000000L
+//VM_L2_CGTT_BUSY_CTRL
+#define VM_L2_CGTT_BUSY_CTRL__READ_DELAY__SHIFT                                                               0x0
+#define VM_L2_CGTT_BUSY_CTRL__ALWAYS_BUSY__SHIFT                                                              0x4
+#define VM_L2_CGTT_BUSY_CTRL__READ_DELAY_MASK                                                                 0x0000000FL
+#define VM_L2_CGTT_BUSY_CTRL__ALWAYS_BUSY_MASK                                                                0x00000010L
+//VML2_MEM_ECC_INDEX
+#define VML2_MEM_ECC_INDEX__INDEX__SHIFT                                                                      0x0
+#define VML2_MEM_ECC_INDEX__INDEX_MASK                                                                        0x000000FFL
+//VML2_WALKER_MEM_ECC_INDEX
+#define VML2_WALKER_MEM_ECC_INDEX__INDEX__SHIFT                                                               0x0
+#define VML2_WALKER_MEM_ECC_INDEX__INDEX_MASK                                                                 0x000000FFL
+//UTCL2_MEM_ECC_INDEX
+#define UTCL2_MEM_ECC_INDEX__INDEX__SHIFT                                                                     0x0
+#define UTCL2_MEM_ECC_INDEX__INDEX_MASK                                                                       0x000000FFL
+//VML2_MEM_ECC_CNTL
+#define VML2_MEM_ECC_CNTL__INJECT_DELAY__SHIFT                                                                0x0
+#define VML2_MEM_ECC_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                          0x6
+#define VML2_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                         0x8
+#define VML2_MEM_ECC_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                         0x9
+#define VML2_MEM_ECC_CNTL__SELECT_INJECT_DELAY__SHIFT                                                         0xb
+#define VML2_MEM_ECC_CNTL__SEC_COUNT__SHIFT                                                                   0xc
+#define VML2_MEM_ECC_CNTL__DED_COUNT__SHIFT                                                                   0xe
+#define VML2_MEM_ECC_CNTL__WRITE_COUNTERS__SHIFT                                                              0x10
+#define VML2_MEM_ECC_CNTL__TEST_FUE__SHIFT                                                                    0x11
+#define VML2_MEM_ECC_CNTL__INJECT_DELAY_MASK                                                                  0x0000003FL
+#define VML2_MEM_ECC_CNTL__DSM_IRRITATOR_DATA_MASK                                                            0x000000C0L
+#define VML2_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE_MASK                                                           0x00000100L
+#define VML2_MEM_ECC_CNTL__ENABLE_ERROR_INJECT_MASK                                                           0x00000600L
+#define VML2_MEM_ECC_CNTL__SELECT_INJECT_DELAY_MASK                                                           0x00000800L
+#define VML2_MEM_ECC_CNTL__SEC_COUNT_MASK                                                                     0x00003000L
+#define VML2_MEM_ECC_CNTL__DED_COUNT_MASK                                                                     0x0000C000L
+#define VML2_MEM_ECC_CNTL__WRITE_COUNTERS_MASK                                                                0x00010000L
+#define VML2_MEM_ECC_CNTL__TEST_FUE_MASK                                                                      0x00020000L
+//VML2_WALKER_MEM_ECC_CNTL
+#define VML2_WALKER_MEM_ECC_CNTL__INJECT_DELAY__SHIFT                                                         0x0
+#define VML2_WALKER_MEM_ECC_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                   0x6
+#define VML2_WALKER_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                  0x8
+#define VML2_WALKER_MEM_ECC_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                  0x9
+#define VML2_WALKER_MEM_ECC_CNTL__SELECT_INJECT_DELAY__SHIFT                                                  0xb
+#define VML2_WALKER_MEM_ECC_CNTL__SEC_COUNT__SHIFT                                                            0xc
+#define VML2_WALKER_MEM_ECC_CNTL__DED_COUNT__SHIFT                                                            0xe
+#define VML2_WALKER_MEM_ECC_CNTL__WRITE_COUNTERS__SHIFT                                                       0x10
+#define VML2_WALKER_MEM_ECC_CNTL__TEST_FUE__SHIFT                                                             0x11
+#define VML2_WALKER_MEM_ECC_CNTL__INJECT_DELAY_MASK                                                           0x0000003FL
+#define VML2_WALKER_MEM_ECC_CNTL__DSM_IRRITATOR_DATA_MASK                                                     0x000000C0L
+#define VML2_WALKER_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE_MASK                                                    0x00000100L
+#define VML2_WALKER_MEM_ECC_CNTL__ENABLE_ERROR_INJECT_MASK                                                    0x00000600L
+#define VML2_WALKER_MEM_ECC_CNTL__SELECT_INJECT_DELAY_MASK                                                    0x00000800L
+#define VML2_WALKER_MEM_ECC_CNTL__SEC_COUNT_MASK                                                              0x00003000L
+#define VML2_WALKER_MEM_ECC_CNTL__DED_COUNT_MASK                                                              0x0000C000L
+#define VML2_WALKER_MEM_ECC_CNTL__WRITE_COUNTERS_MASK                                                         0x00010000L
+#define VML2_WALKER_MEM_ECC_CNTL__TEST_FUE_MASK                                                               0x00020000L
+//UTCL2_MEM_ECC_CNTL
+#define UTCL2_MEM_ECC_CNTL__INJECT_DELAY__SHIFT                                                               0x0
+#define UTCL2_MEM_ECC_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                         0x6
+#define UTCL2_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                        0x8
+#define UTCL2_MEM_ECC_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                        0x9
+#define UTCL2_MEM_ECC_CNTL__SELECT_INJECT_DELAY__SHIFT                                                        0xb
+#define UTCL2_MEM_ECC_CNTL__SEC_COUNT__SHIFT                                                                  0xc
+#define UTCL2_MEM_ECC_CNTL__DED_COUNT__SHIFT                                                                  0xe
+#define UTCL2_MEM_ECC_CNTL__WRITE_COUNTERS__SHIFT                                                             0x10
+#define UTCL2_MEM_ECC_CNTL__TEST_FUE__SHIFT                                                                   0x11
+#define UTCL2_MEM_ECC_CNTL__INJECT_DELAY_MASK                                                                 0x0000003FL
+#define UTCL2_MEM_ECC_CNTL__DSM_IRRITATOR_DATA_MASK                                                           0x000000C0L
+#define UTCL2_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE_MASK                                                          0x00000100L
+#define UTCL2_MEM_ECC_CNTL__ENABLE_ERROR_INJECT_MASK                                                          0x00000600L
+#define UTCL2_MEM_ECC_CNTL__SELECT_INJECT_DELAY_MASK                                                          0x00000800L
+#define UTCL2_MEM_ECC_CNTL__SEC_COUNT_MASK                                                                    0x00003000L
+#define UTCL2_MEM_ECC_CNTL__DED_COUNT_MASK                                                                    0x0000C000L
+#define UTCL2_MEM_ECC_CNTL__WRITE_COUNTERS_MASK                                                               0x00010000L
+#define UTCL2_MEM_ECC_CNTL__TEST_FUE_MASK                                                                     0x00020000L
+//VML2_MEM_ECC_STATUS
+#define VML2_MEM_ECC_STATUS__UCE__SHIFT                                                                       0x0
+#define VML2_MEM_ECC_STATUS__FED__SHIFT                                                                       0x1
+#define VML2_MEM_ECC_STATUS__UCE_MASK                                                                         0x00000001L
+#define VML2_MEM_ECC_STATUS__FED_MASK                                                                         0x00000002L
+//VML2_WALKER_MEM_ECC_STATUS
+#define VML2_WALKER_MEM_ECC_STATUS__UCE__SHIFT                                                                0x0
+#define VML2_WALKER_MEM_ECC_STATUS__FED__SHIFT                                                                0x1
+#define VML2_WALKER_MEM_ECC_STATUS__UCE_MASK                                                                  0x00000001L
+#define VML2_WALKER_MEM_ECC_STATUS__FED_MASK                                                                  0x00000002L
+//UTCL2_MEM_ECC_STATUS
+#define UTCL2_MEM_ECC_STATUS__UCE__SHIFT                                                                      0x0
+#define UTCL2_MEM_ECC_STATUS__FED__SHIFT                                                                      0x1
+#define UTCL2_MEM_ECC_STATUS__UCE_MASK                                                                        0x00000001L
+#define UTCL2_MEM_ECC_STATUS__FED_MASK                                                                        0x00000002L
+//UTCL2_EDC_MODE
+#define UTCL2_EDC_MODE__FORCE_SEC_ON_DED__SHIFT                                                               0xf
+#define UTCL2_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                  0x10
+#define UTCL2_EDC_MODE__GATE_FUE__SHIFT                                                                       0x11
+#define UTCL2_EDC_MODE__DED_MODE__SHIFT                                                                       0x14
+#define UTCL2_EDC_MODE__PROP_FED__SHIFT                                                                       0x1d
+#define UTCL2_EDC_MODE__BYPASS__SHIFT                                                                         0x1f
+#define UTCL2_EDC_MODE__FORCE_SEC_ON_DED_MASK                                                                 0x00008000L
+#define UTCL2_EDC_MODE__COUNT_FED_OUT_MASK                                                                    0x00010000L
+#define UTCL2_EDC_MODE__GATE_FUE_MASK                                                                         0x00020000L
+#define UTCL2_EDC_MODE__DED_MODE_MASK                                                                         0x00300000L
+#define UTCL2_EDC_MODE__PROP_FED_MASK                                                                         0x20000000L
+#define UTCL2_EDC_MODE__BYPASS_MASK                                                                           0x80000000L
+//UTCL2_EDC_CONFIG
+#define UTCL2_EDC_CONFIG__WRITE_DIS__SHIFT                                                                    0x0
+#define UTCL2_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
+#define UTCL2_EDC_CONFIG__WRITE_DIS_MASK                                                                      0x00000001L
+#define UTCL2_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
+
+
+// addressBlock: xcd0_gc_utcl2_vml2vcdec
+//VM_CONTEXT0_CNTL
+#define VM_CONTEXT0_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT0_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT0_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT1_CNTL
+#define VM_CONTEXT1_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT1_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT1_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT2_CNTL
+#define VM_CONTEXT2_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT2_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT2_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT2_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT3_CNTL
+#define VM_CONTEXT3_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT3_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT3_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT3_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT4_CNTL
+#define VM_CONTEXT4_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT4_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT4_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT4_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT5_CNTL
+#define VM_CONTEXT5_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT5_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT5_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT5_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT6_CNTL
+#define VM_CONTEXT6_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT6_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT6_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT6_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT7_CNTL
+#define VM_CONTEXT7_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT7_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT7_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT7_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT8_CNTL
+#define VM_CONTEXT8_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT8_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT8_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT8_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT9_CNTL
+#define VM_CONTEXT9_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT9_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT9_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT9_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT10_CNTL
+#define VM_CONTEXT10_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT10_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT10_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT10_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXT11_CNTL
+#define VM_CONTEXT11_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT11_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT11_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT11_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXT12_CNTL
+#define VM_CONTEXT12_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT12_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT12_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT12_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXT13_CNTL
+#define VM_CONTEXT13_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT13_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT13_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT13_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXT14_CNTL
+#define VM_CONTEXT14_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT14_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT14_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT14_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXT15_CNTL
+#define VM_CONTEXT15_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT15_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT15_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT15_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXTS_DISABLE
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0__SHIFT                                                         0x0
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1__SHIFT                                                         0x1
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2__SHIFT                                                         0x2
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3__SHIFT                                                         0x3
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4__SHIFT                                                         0x4
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5__SHIFT                                                         0x5
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6__SHIFT                                                         0x6
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7__SHIFT                                                         0x7
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8__SHIFT                                                         0x8
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9__SHIFT                                                         0x9
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10__SHIFT                                                        0xa
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11__SHIFT                                                        0xb
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12__SHIFT                                                        0xc
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13__SHIFT                                                        0xd
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14__SHIFT                                                        0xe
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15__SHIFT                                                        0xf
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0_MASK                                                           0x00000001L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1_MASK                                                           0x00000002L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2_MASK                                                           0x00000004L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3_MASK                                                           0x00000008L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4_MASK                                                           0x00000010L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5_MASK                                                           0x00000020L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6_MASK                                                           0x00000040L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7_MASK                                                           0x00000080L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8_MASK                                                           0x00000100L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9_MASK                                                           0x00000200L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10_MASK                                                          0x00000400L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11_MASK                                                          0x00000800L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12_MASK                                                          0x00001000L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13_MASK                                                          0x00002000L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14_MASK                                                          0x00004000L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15_MASK                                                          0x00008000L
+//VM_INVALIDATE_ENG0_SEM
+#define VM_INVALIDATE_ENG0_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG0_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG1_SEM
+#define VM_INVALIDATE_ENG1_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG1_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG2_SEM
+#define VM_INVALIDATE_ENG2_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG2_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG3_SEM
+#define VM_INVALIDATE_ENG3_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG3_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG4_SEM
+#define VM_INVALIDATE_ENG4_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG4_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG5_SEM
+#define VM_INVALIDATE_ENG5_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG5_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG6_SEM
+#define VM_INVALIDATE_ENG6_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG6_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG7_SEM
+#define VM_INVALIDATE_ENG7_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG7_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG8_SEM
+#define VM_INVALIDATE_ENG8_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG8_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG9_SEM
+#define VM_INVALIDATE_ENG9_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG9_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG10_SEM
+#define VM_INVALIDATE_ENG10_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG10_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG11_SEM
+#define VM_INVALIDATE_ENG11_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG11_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG12_SEM
+#define VM_INVALIDATE_ENG12_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG12_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG13_SEM
+#define VM_INVALIDATE_ENG13_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG13_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG14_SEM
+#define VM_INVALIDATE_ENG14_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG14_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG15_SEM
+#define VM_INVALIDATE_ENG15_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG15_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG16_SEM
+#define VM_INVALIDATE_ENG16_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG16_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG17_SEM
+#define VM_INVALIDATE_ENG17_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG17_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG0_REQ
+#define VM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG0_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG0_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG0_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG0_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG1_REQ
+#define VM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG1_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG1_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG1_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG1_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG2_REQ
+#define VM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG2_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG2_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG2_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG2_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG3_REQ
+#define VM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG3_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG3_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG3_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG3_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG4_REQ
+#define VM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG4_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG4_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG4_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG4_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG5_REQ
+#define VM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG5_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG5_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG5_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG5_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG6_REQ
+#define VM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG6_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG6_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG6_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG6_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG7_REQ
+#define VM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG7_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG7_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG7_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG7_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG8_REQ
+#define VM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG8_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG8_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG8_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG8_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG9_REQ
+#define VM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG9_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG9_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG9_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG9_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG10_REQ
+#define VM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG10_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG10_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG10_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG10_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG11_REQ
+#define VM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG11_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG11_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG11_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG11_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG12_REQ
+#define VM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG12_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG12_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG12_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG12_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG13_REQ
+#define VM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG13_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG13_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG13_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG13_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG14_REQ
+#define VM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG14_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG14_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG14_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG14_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG15_REQ
+#define VM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG15_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG15_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG15_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG15_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG16_REQ
+#define VM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG16_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG16_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG16_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG16_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG17_REQ
+#define VM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG17_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG17_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG17_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG17_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG0_ACK
+#define VM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG0_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG0_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG1_ACK
+#define VM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG1_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG1_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG2_ACK
+#define VM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG2_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG2_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG3_ACK
+#define VM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG3_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG3_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG4_ACK
+#define VM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG4_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG4_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG5_ACK
+#define VM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG5_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG5_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG6_ACK
+#define VM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG6_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG6_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG7_ACK
+#define VM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG7_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG7_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG8_ACK
+#define VM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG8_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG8_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG9_ACK
+#define VM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG9_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG9_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG10_ACK
+#define VM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG10_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG10_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG11_ACK
+#define VM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG11_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG11_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG12_ACK
+#define VM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG12_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG12_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG13_ACK
+#define VM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG13_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG13_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG14_ACK
+#define VM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG14_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG14_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG15_ACK
+#define VM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG15_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG15_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG16_ACK
+#define VM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG16_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG16_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG17_ACK
+#define VM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG17_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG17_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG0_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG0_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG1_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG1_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG2_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG2_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG3_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG3_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG4_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG4_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG5_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG5_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG6_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG6_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG7_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG7_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG8_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG8_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG9_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG9_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG10_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG10_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG11_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG11_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG12_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG12_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG13_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG13_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG14_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG14_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG15_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG15_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG16_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG16_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG17_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG17_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+//VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+//VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+//VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+//VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+//VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+
+
+// addressBlock: xcd0_gc_utcl2_vmsharedpfdec
+//MC_VM_NB_MMIOBASE
+#define MC_VM_NB_MMIOBASE__MMIOBASE__SHIFT                                                                    0x0
+#define MC_VM_NB_MMIOBASE__MMIOBASE_MASK                                                                      0xFFFFFFFFL
+//MC_VM_NB_MMIOLIMIT
+#define MC_VM_NB_MMIOLIMIT__MMIOLIMIT__SHIFT                                                                  0x0
+#define MC_VM_NB_MMIOLIMIT__MMIOLIMIT_MASK                                                                    0xFFFFFFFFL
+//MC_VM_NB_PCI_CTRL
+#define MC_VM_NB_PCI_CTRL__MMIOENABLE__SHIFT                                                                  0x17
+#define MC_VM_NB_PCI_CTRL__MMIOENABLE_MASK                                                                    0x00800000L
+//MC_VM_NB_PCI_ARB
+#define MC_VM_NB_PCI_ARB__VGA_HOLE__SHIFT                                                                     0x3
+#define MC_VM_NB_PCI_ARB__VGA_HOLE_MASK                                                                       0x00000008L
+//MC_VM_NB_TOP_OF_DRAM_SLOT1
+#define MC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM__SHIFT                                                        0x17
+#define MC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM_MASK                                                          0xFF800000L
+//MC_VM_NB_LOWER_TOP_OF_DRAM2
+#define MC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE__SHIFT                                                            0x0
+#define MC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2__SHIFT                                                        0x17
+#define MC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE_MASK                                                              0x00000001L
+#define MC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2_MASK                                                          0xFF800000L
+//MC_VM_NB_UPPER_TOP_OF_DRAM2
+#define MC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2__SHIFT                                                        0x0
+#define MC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2_MASK                                                          0x0000FFFFL
+//MC_VM_FB_OFFSET
+#define MC_VM_FB_OFFSET__FB_OFFSET__SHIFT                                                                     0x0
+#define MC_VM_FB_OFFSET__FB_OFFSET_MASK                                                                       0x00FFFFFFL
+//MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB__SHIFT                               0x0
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB_MASK                                 0xFFFFFFFFL
+//MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB__SHIFT                               0x0
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB_MASK                                 0x0000000FL
+//MC_VM_STEERING
+#define MC_VM_STEERING__DEFAULT_STEERING__SHIFT                                                               0x0
+#define MC_VM_STEERING__DEFAULT_STEERING_MASK                                                                 0x00000003L
+//MC_SHARED_VIRT_RESET_REQ
+#define MC_SHARED_VIRT_RESET_REQ__VF__SHIFT                                                                   0x0
+#define MC_SHARED_VIRT_RESET_REQ__PF__SHIFT                                                                   0x1f
+#define MC_SHARED_VIRT_RESET_REQ__VF_MASK                                                                     0x0000FFFFL
+#define MC_SHARED_VIRT_RESET_REQ__PF_MASK                                                                     0x80000000L
+//MC_MEM_POWER_LS
+#define MC_MEM_POWER_LS__LS_SETUP__SHIFT                                                                      0x0
+#define MC_MEM_POWER_LS__LS_HOLD__SHIFT                                                                       0x6
+#define MC_MEM_POWER_LS__LS_SETUP_MASK                                                                        0x0000003FL
+#define MC_MEM_POWER_LS__LS_HOLD_MASK                                                                         0x00000FC0L
+//MC_VM_CACHEABLE_DRAM_ADDRESS_START
+#define MC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS__SHIFT                                                    0x0
+#define MC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS_MASK                                                      0x00FFFFFFL
+//MC_VM_CACHEABLE_DRAM_ADDRESS_END
+#define MC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS__SHIFT                                                      0x0
+#define MC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS_MASK                                                        0x00FFFFFFL
+//MC_VM_APT_CNTL
+#define MC_VM_APT_CNTL__FORCE_MTYPE_UC__SHIFT                                                                 0x0
+#define MC_VM_APT_CNTL__DIRECT_SYSTEM_EN__SHIFT                                                               0x1
+#define MC_VM_APT_CNTL__CHECK_IS_LOCAL__SHIFT                                                                 0x2
+#define MC_VM_APT_CNTL__PERMS_GRANTED__SHIFT                                                                  0x3
+#define MC_VM_APT_CNTL__LOCAL_SYSMEM_APERTURE_CNTL__SHIFT                                                     0x4
+#define MC_VM_APT_CNTL__FORCE_MTYPE_UC_MASK                                                                   0x00000001L
+#define MC_VM_APT_CNTL__DIRECT_SYSTEM_EN_MASK                                                                 0x00000002L
+#define MC_VM_APT_CNTL__CHECK_IS_LOCAL_MASK                                                                   0x00000004L
+#define MC_VM_APT_CNTL__PERMS_GRANTED_MASK                                                                    0x00000008L
+#define MC_VM_APT_CNTL__LOCAL_SYSMEM_APERTURE_CNTL_MASK                                                       0x00000030L
+//MC_VM_LOCAL_HBM_ADDRESS_START
+#define MC_VM_LOCAL_HBM_ADDRESS_START__ADDRESS__SHIFT                                                         0x0
+#define MC_VM_LOCAL_HBM_ADDRESS_START__ADDRESS_MASK                                                           0x00FFFFFFL
+//MC_VM_LOCAL_HBM_ADDRESS_END
+#define MC_VM_LOCAL_HBM_ADDRESS_END__ADDRESS__SHIFT                                                           0x0
+#define MC_VM_LOCAL_HBM_ADDRESS_END__ADDRESS_MASK                                                             0x00FFFFFFL
+//MC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL
+#define MC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL__LOCK__SHIFT                                                        0x0
+#define MC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL__LOCK_MASK                                                          0x00000001L
+//UTCL2_CGTT_CLK_CTRL
+#define UTCL2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define UTCL2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE_EXTRA__SHIFT                                                       0xc
+#define UTCL2_CGTT_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                             0xf
+#define UTCL2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE__SHIFT                                                       0x10
+#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                             0x18
+#define UTCL2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define UTCL2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE_EXTRA_MASK                                                         0x00007000L
+#define UTCL2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK                                                               0x00008000L
+#define UTCL2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_MASK                                                         0x00FF0000L
+#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                               0xFF000000L
+//MC_VM_XGMI_LFB_CNTL
+#define MC_VM_XGMI_LFB_CNTL__PF_LFB_REGION__SHIFT                                                             0x0
+#define MC_VM_XGMI_LFB_CNTL__PF_MAX_REGION__SHIFT                                                             0x4
+#define MC_VM_XGMI_LFB_CNTL__PF_LFB_REGION_MASK                                                               0x0000000FL
+#define MC_VM_XGMI_LFB_CNTL__PF_MAX_REGION_MASK                                                               0x000000F0L
+//MC_VM_XGMI_LFB_SIZE
+#define MC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE__SHIFT                                                               0x0
+#define MC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE_MASK                                                                 0x0001FFFFL
+//MC_VM_CACHEABLE_DRAM_CNTL
+#define MC_VM_CACHEABLE_DRAM_CNTL__ENABLE_CACHEABLE_DRAM_ADDRESS_APERTURE__SHIFT                              0x0
+#define MC_VM_CACHEABLE_DRAM_CNTL__ENABLE_CACHEABLE_DRAM_ADDRESS_APERTURE_MASK                                0x00000001L
+//MC_VM_HOST_MAPPING
+#define MC_VM_HOST_MAPPING__MODE__SHIFT                                                                       0x0
+#define MC_VM_HOST_MAPPING__MODE_MASK                                                                         0x00000001L
+
+
+// addressBlock: xcd0_gc_utcl2_vmsharedvcdec
+//MC_VM_FB_LOCATION_BASE
+#define MC_VM_FB_LOCATION_BASE__FB_BASE__SHIFT                                                                0x0
+#define MC_VM_FB_LOCATION_BASE__FB_BASE_MASK                                                                  0x00FFFFFFL
+//MC_VM_FB_LOCATION_TOP
+#define MC_VM_FB_LOCATION_TOP__FB_TOP__SHIFT                                                                  0x0
+#define MC_VM_FB_LOCATION_TOP__FB_TOP_MASK                                                                    0x00FFFFFFL
+//MC_VM_AGP_TOP
+#define MC_VM_AGP_TOP__AGP_TOP__SHIFT                                                                         0x0
+#define MC_VM_AGP_TOP__AGP_TOP_MASK                                                                           0x00FFFFFFL
+//MC_VM_AGP_BOT
+#define MC_VM_AGP_BOT__AGP_BOT__SHIFT                                                                         0x0
+#define MC_VM_AGP_BOT__AGP_BOT_MASK                                                                           0x00FFFFFFL
+//MC_VM_AGP_BASE
+#define MC_VM_AGP_BASE__AGP_BASE__SHIFT                                                                       0x0
+#define MC_VM_AGP_BASE__AGP_BASE_MASK                                                                         0x00FFFFFFL
+//MC_VM_SYSTEM_APERTURE_LOW_ADDR
+#define MC_VM_SYSTEM_APERTURE_LOW_ADDR__LOGICAL_ADDR__SHIFT                                                   0x0
+#define MC_VM_SYSTEM_APERTURE_LOW_ADDR__LOGICAL_ADDR_MASK                                                     0x3FFFFFFFL
+//MC_VM_SYSTEM_APERTURE_HIGH_ADDR
+#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR__LOGICAL_ADDR__SHIFT                                                  0x0
+#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR__LOGICAL_ADDR_MASK                                                    0x3FFFFFFFL
+//MC_VM_MX_L1_TLB_CNTL
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT                                                            0x0
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT                                                       0x3
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT                                          0x5
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT                                             0x6
+#define MC_VM_MX_L1_TLB_CNTL__ECO_BITS__SHIFT                                                                 0x7
+#define MC_VM_MX_L1_TLB_CNTL__MTYPE__SHIFT                                                                    0xb
+#define MC_VM_MX_L1_TLB_CNTL__ATC_EN__SHIFT                                                                   0xd
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK                                                              0x00000001L
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK                                                         0x00000018L
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK                                            0x00000020L
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK                                               0x00000040L
+#define MC_VM_MX_L1_TLB_CNTL__ECO_BITS_MASK                                                                   0x00000780L
+#define MC_VM_MX_L1_TLB_CNTL__MTYPE_MASK                                                                      0x00001800L
+#define MC_VM_MX_L1_TLB_CNTL__ATC_EN_MASK                                                                     0x00002000L
+
+
+// addressBlock: xcd0_gc_utcl2_l2tlbdec
+//L2TLB_TLB0_STATUS
+#define L2TLB_TLB0_STATUS__BUSY__SHIFT                                                                        0x0
+#define L2TLB_TLB0_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                         0x1
+#define L2TLB_TLB0_STATUS__BUSY_MASK                                                                          0x00000001L
+#define L2TLB_TLB0_STATUS__FOUND_PARITY_ERRORS_MASK                                                           0x00000002L
+//UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO__ADDR__SHIFT                                             0x0
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO__ADDR_MASK                                               0xFFFFFFFFL
+//UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__ADDR__SHIFT                                             0x0
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VMID__SHIFT                                             0x4
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VFID__SHIFT                                             0x9
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VF__SHIFT                                               0xd
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__GPA__SHIFT                                              0xe
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__RD_PERM__SHIFT                                          0x10
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__WR_PERM__SHIFT                                          0x11
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__EX_PERM__SHIFT                                          0x12
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__CLIENT_ID__SHIFT                                        0x13
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__REQ__SHIFT                                              0x1f
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__ADDR_MASK                                               0x0000000FL
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VMID_MASK                                               0x000000F0L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VFID_MASK                                               0x00001E00L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VF_MASK                                                 0x00002000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__GPA_MASK                                                0x0000C000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__RD_PERM_MASK                                            0x00010000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__WR_PERM_MASK                                            0x00020000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__EX_PERM_MASK                                            0x00040000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__CLIENT_ID_MASK                                          0x0FF80000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__REQ_MASK                                                0x80000000L
+//UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO__ADDR__SHIFT                                            0x0
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO__ADDR_MASK                                              0xFFFFFFFFL
+//UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ADDR__SHIFT                                            0x0
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PERMS__SHIFT                                           0x4
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__FRAGMENT_SIZE__SHIFT                                   0x7
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SNOOP__SHIFT                                           0xd
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SPA__SHIFT                                             0xe
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__IO__SHIFT                                              0xf
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PTE_TMZ__SHIFT                                         0x10
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NO_PTE__SHIFT                                          0x11
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MTYPE__SHIFT                                           0x12
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MEMLOG__SHIFT                                          0x14
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__LLC_NOALLOC__SHIFT                                     0x15
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NACK__SHIFT                                            0x16
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ACK__SHIFT                                             0x1f
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ADDR_MASK                                              0x0000000FL
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PERMS_MASK                                             0x00000070L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__FRAGMENT_SIZE_MASK                                     0x00001F80L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SNOOP_MASK                                             0x00002000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SPA_MASK                                               0x00004000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__IO_MASK                                                0x00008000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PTE_TMZ_MASK                                           0x00010000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NO_PTE_MASK                                            0x00020000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MTYPE_MASK                                             0x000C0000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MEMLOG_MASK                                            0x00100000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__LLC_NOALLOC_MASK                                       0x00200000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NACK_MASK                                              0x00C00000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ACK_MASK                                               0x80000000L
+
+
+// addressBlock: xcd0_gc_tcdec
+//TCP_INVALIDATE
+#define TCP_INVALIDATE__START__SHIFT                                                                          0x0
+#define TCP_INVALIDATE__START_MASK                                                                            0x00000001L
+//TCP_STATUS
+#define TCP_STATUS__TCP_BUSY__SHIFT                                                                           0x0
+#define TCP_STATUS__INPUT_BUSY__SHIFT                                                                         0x1
+#define TCP_STATUS__ADRS_BUSY__SHIFT                                                                          0x2
+#define TCP_STATUS__TAGRAMS_BUSY__SHIFT                                                                       0x3
+#define TCP_STATUS__CNTRL_BUSY__SHIFT                                                                         0x4
+#define TCP_STATUS__LFIFO_BUSY__SHIFT                                                                         0x5
+#define TCP_STATUS__READ_BUSY__SHIFT                                                                          0x6
+#define TCP_STATUS__FORMAT_BUSY__SHIFT                                                                        0x7
+#define TCP_STATUS__VM_BUSY__SHIFT                                                                            0x8
+#define TCP_STATUS__TCP_BUSY_MASK                                                                             0x00000001L
+#define TCP_STATUS__INPUT_BUSY_MASK                                                                           0x00000002L
+#define TCP_STATUS__ADRS_BUSY_MASK                                                                            0x00000004L
+#define TCP_STATUS__TAGRAMS_BUSY_MASK                                                                         0x00000008L
+#define TCP_STATUS__CNTRL_BUSY_MASK                                                                           0x00000010L
+#define TCP_STATUS__LFIFO_BUSY_MASK                                                                           0x00000020L
+#define TCP_STATUS__READ_BUSY_MASK                                                                            0x00000040L
+#define TCP_STATUS__FORMAT_BUSY_MASK                                                                          0x00000080L
+#define TCP_STATUS__VM_BUSY_MASK                                                                              0x00000100L
+//TCP_CNTL
+#define TCP_CNTL__FORCE_HIT__SHIFT                                                                            0x0
+#define TCP_CNTL__FORCE_MISS__SHIFT                                                                           0x1
+#define TCP_CNTL__L1_SIZE__SHIFT                                                                              0x2
+#define TCP_CNTL__FLAT_BUF_HASH_ENABLE__SHIFT                                                                 0x4
+#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE__SHIFT                                                               0x5
+#define TCP_CNTL__FORCE_EOW_TOTAL_CNT__SHIFT                                                                  0xf
+#define TCP_CNTL__FORCE_EOW_TAGRAM_CNT__SHIFT                                                                 0x16
+#define TCP_CNTL__DISABLE_Z_MAP__SHIFT                                                                        0x1c
+#define TCP_CNTL__FORCE_HIT_MASK                                                                              0x00000001L
+#define TCP_CNTL__FORCE_MISS_MASK                                                                             0x00000002L
+#define TCP_CNTL__L1_SIZE_MASK                                                                                0x0000000CL
+#define TCP_CNTL__FLAT_BUF_HASH_ENABLE_MASK                                                                   0x00000010L
+#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE_MASK                                                                 0x00000020L
+#define TCP_CNTL__FORCE_EOW_TOTAL_CNT_MASK                                                                    0x001F8000L
+#define TCP_CNTL__FORCE_EOW_TAGRAM_CNT_MASK                                                                   0x0FC00000L
+#define TCP_CNTL__DISABLE_Z_MAP_MASK                                                                          0x10000000L
+//TCP_CHAN_STEER_0
+#define TCP_CHAN_STEER_0__CHAN0__SHIFT                                                                        0x0
+#define TCP_CHAN_STEER_0__CHAN1__SHIFT                                                                        0x4
+#define TCP_CHAN_STEER_0__CHAN2__SHIFT                                                                        0x8
+#define TCP_CHAN_STEER_0__CHAN3__SHIFT                                                                        0xc
+#define TCP_CHAN_STEER_0__CHAN4__SHIFT                                                                        0x10
+#define TCP_CHAN_STEER_0__CHAN5__SHIFT                                                                        0x14
+#define TCP_CHAN_STEER_0__CHAN6__SHIFT                                                                        0x18
+#define TCP_CHAN_STEER_0__CHAN7__SHIFT                                                                        0x1c
+#define TCP_CHAN_STEER_0__CHAN0_MASK                                                                          0x0000000FL
+#define TCP_CHAN_STEER_0__CHAN1_MASK                                                                          0x000000F0L
+#define TCP_CHAN_STEER_0__CHAN2_MASK                                                                          0x00000F00L
+#define TCP_CHAN_STEER_0__CHAN3_MASK                                                                          0x0000F000L
+#define TCP_CHAN_STEER_0__CHAN4_MASK                                                                          0x000F0000L
+#define TCP_CHAN_STEER_0__CHAN5_MASK                                                                          0x00F00000L
+#define TCP_CHAN_STEER_0__CHAN6_MASK                                                                          0x0F000000L
+#define TCP_CHAN_STEER_0__CHAN7_MASK                                                                          0xF0000000L
+//TCP_CHAN_STEER_1
+#define TCP_CHAN_STEER_1__CHAN8__SHIFT                                                                        0x0
+#define TCP_CHAN_STEER_1__CHAN9__SHIFT                                                                        0x4
+#define TCP_CHAN_STEER_1__CHANA__SHIFT                                                                        0x8
+#define TCP_CHAN_STEER_1__CHANB__SHIFT                                                                        0xc
+#define TCP_CHAN_STEER_1__CHANC__SHIFT                                                                        0x10
+#define TCP_CHAN_STEER_1__CHAND__SHIFT                                                                        0x14
+#define TCP_CHAN_STEER_1__CHANE__SHIFT                                                                        0x18
+#define TCP_CHAN_STEER_1__CHANF__SHIFT                                                                        0x1c
+#define TCP_CHAN_STEER_1__CHAN8_MASK                                                                          0x0000000FL
+#define TCP_CHAN_STEER_1__CHAN9_MASK                                                                          0x000000F0L
+#define TCP_CHAN_STEER_1__CHANA_MASK                                                                          0x00000F00L
+#define TCP_CHAN_STEER_1__CHANB_MASK                                                                          0x0000F000L
+#define TCP_CHAN_STEER_1__CHANC_MASK                                                                          0x000F0000L
+#define TCP_CHAN_STEER_1__CHAND_MASK                                                                          0x00F00000L
+#define TCP_CHAN_STEER_1__CHANE_MASK                                                                          0x0F000000L
+#define TCP_CHAN_STEER_1__CHANF_MASK                                                                          0xF0000000L
+//TCP_ADDR_CONFIG
+#define TCP_ADDR_CONFIG__NUM_TCC_BANKS__SHIFT                                                                 0x0
+#define TCP_ADDR_CONFIG__ENABLE64KHASH__SHIFT                                                                 0xb
+#define TCP_ADDR_CONFIG__ENABLE2MHASH__SHIFT                                                                  0xc
+#define TCP_ADDR_CONFIG__ENABLE1GHASH__SHIFT                                                                  0xd
+#define TCP_ADDR_CONFIG__ENABLE1THASH__SHIFT                                                                  0xe
+#define TCP_ADDR_CONFIG__ENABLE4KHASH__SHIFT                                                                  0xf
+#define TCP_ADDR_CONFIG__NUM_TCC_BANKS_MASK                                                                   0x0000001FL
+#define TCP_ADDR_CONFIG__ENABLE64KHASH_MASK                                                                   0x00000800L
+#define TCP_ADDR_CONFIG__ENABLE2MHASH_MASK                                                                    0x00001000L
+#define TCP_ADDR_CONFIG__ENABLE1GHASH_MASK                                                                    0x00002000L
+#define TCP_ADDR_CONFIG__ENABLE1THASH_MASK                                                                    0x00004000L
+#define TCP_ADDR_CONFIG__ENABLE4KHASH_MASK                                                                    0x00008000L
+//TCP_CREDIT
+#define TCP_CREDIT__LFIFO_CREDIT__SHIFT                                                                       0x0
+#define TCP_CREDIT__REQ_FIFO_CREDIT__SHIFT                                                                    0x10
+#define TCP_CREDIT__TD_CREDIT__SHIFT                                                                          0x1d
+#define TCP_CREDIT__LFIFO_CREDIT_MASK                                                                         0x000007FFL
+#define TCP_CREDIT__REQ_FIFO_CREDIT_MASK                                                                      0x007F0000L
+#define TCP_CREDIT__TD_CREDIT_MASK                                                                            0xE0000000L
+//TCP_BUFFER_ADDR_HASH_CNTL
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_BITS__SHIFT                                                        0x0
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_BITS__SHIFT                                                           0x8
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_XOR_COUNT__SHIFT                                                   0x10
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_XOR_COUNT__SHIFT                                                      0x18
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_BITS_MASK                                                          0x00000007L
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_BITS_MASK                                                             0x00000700L
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_XOR_COUNT_MASK                                                     0x00070000L
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_XOR_COUNT_MASK                                                        0x07000000L
+//TC_CFG_L1_LOAD_POLICY0
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_0__SHIFT                                                               0x0
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_1__SHIFT                                                               0x2
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_2__SHIFT                                                               0x4
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_3__SHIFT                                                               0x6
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_4__SHIFT                                                               0x8
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_5__SHIFT                                                               0xa
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_6__SHIFT                                                               0xc
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_7__SHIFT                                                               0xe
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_8__SHIFT                                                               0x10
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_9__SHIFT                                                               0x12
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_10__SHIFT                                                              0x14
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_11__SHIFT                                                              0x16
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_12__SHIFT                                                              0x18
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_13__SHIFT                                                              0x1a
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_14__SHIFT                                                              0x1c
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_15__SHIFT                                                              0x1e
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_0_MASK                                                                 0x00000003L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_1_MASK                                                                 0x0000000CL
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_2_MASK                                                                 0x00000030L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_3_MASK                                                                 0x000000C0L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_4_MASK                                                                 0x00000300L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_5_MASK                                                                 0x00000C00L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_6_MASK                                                                 0x00003000L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_7_MASK                                                                 0x0000C000L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_8_MASK                                                                 0x00030000L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_9_MASK                                                                 0x000C0000L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_10_MASK                                                                0x00300000L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_11_MASK                                                                0x00C00000L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_12_MASK                                                                0x03000000L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_13_MASK                                                                0x0C000000L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_14_MASK                                                                0x30000000L
+#define TC_CFG_L1_LOAD_POLICY0__POLICY_15_MASK                                                                0xC0000000L
+//TC_CFG_L1_LOAD_POLICY1
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_16__SHIFT                                                              0x0
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_17__SHIFT                                                              0x2
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_18__SHIFT                                                              0x4
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_19__SHIFT                                                              0x6
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_20__SHIFT                                                              0x8
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_21__SHIFT                                                              0xa
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_22__SHIFT                                                              0xc
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_23__SHIFT                                                              0xe
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_24__SHIFT                                                              0x10
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_25__SHIFT                                                              0x12
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_26__SHIFT                                                              0x14
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_27__SHIFT                                                              0x16
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_28__SHIFT                                                              0x18
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_29__SHIFT                                                              0x1a
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_30__SHIFT                                                              0x1c
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_31__SHIFT                                                              0x1e
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_16_MASK                                                                0x00000003L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_17_MASK                                                                0x0000000CL
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_18_MASK                                                                0x00000030L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_19_MASK                                                                0x000000C0L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_20_MASK                                                                0x00000300L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_21_MASK                                                                0x00000C00L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_22_MASK                                                                0x00003000L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_23_MASK                                                                0x0000C000L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_24_MASK                                                                0x00030000L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_25_MASK                                                                0x000C0000L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_26_MASK                                                                0x00300000L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_27_MASK                                                                0x00C00000L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_28_MASK                                                                0x03000000L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_29_MASK                                                                0x0C000000L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_30_MASK                                                                0x30000000L
+#define TC_CFG_L1_LOAD_POLICY1__POLICY_31_MASK                                                                0xC0000000L
+//TC_CFG_L1_STORE_POLICY
+#define TC_CFG_L1_STORE_POLICY__POLICY_0__SHIFT                                                               0x0
+#define TC_CFG_L1_STORE_POLICY__POLICY_1__SHIFT                                                               0x1
+#define TC_CFG_L1_STORE_POLICY__POLICY_2__SHIFT                                                               0x2
+#define TC_CFG_L1_STORE_POLICY__POLICY_3__SHIFT                                                               0x3
+#define TC_CFG_L1_STORE_POLICY__POLICY_4__SHIFT                                                               0x4
+#define TC_CFG_L1_STORE_POLICY__POLICY_5__SHIFT                                                               0x5
+#define TC_CFG_L1_STORE_POLICY__POLICY_6__SHIFT                                                               0x6
+#define TC_CFG_L1_STORE_POLICY__POLICY_7__SHIFT                                                               0x7
+#define TC_CFG_L1_STORE_POLICY__POLICY_8__SHIFT                                                               0x8
+#define TC_CFG_L1_STORE_POLICY__POLICY_9__SHIFT                                                               0x9
+#define TC_CFG_L1_STORE_POLICY__POLICY_10__SHIFT                                                              0xa
+#define TC_CFG_L1_STORE_POLICY__POLICY_11__SHIFT                                                              0xb
+#define TC_CFG_L1_STORE_POLICY__POLICY_12__SHIFT                                                              0xc
+#define TC_CFG_L1_STORE_POLICY__POLICY_13__SHIFT                                                              0xd
+#define TC_CFG_L1_STORE_POLICY__POLICY_14__SHIFT                                                              0xe
+#define TC_CFG_L1_STORE_POLICY__POLICY_15__SHIFT                                                              0xf
+#define TC_CFG_L1_STORE_POLICY__POLICY_16__SHIFT                                                              0x10
+#define TC_CFG_L1_STORE_POLICY__POLICY_17__SHIFT                                                              0x11
+#define TC_CFG_L1_STORE_POLICY__POLICY_18__SHIFT                                                              0x12
+#define TC_CFG_L1_STORE_POLICY__POLICY_19__SHIFT                                                              0x13
+#define TC_CFG_L1_STORE_POLICY__POLICY_20__SHIFT                                                              0x14
+#define TC_CFG_L1_STORE_POLICY__POLICY_21__SHIFT                                                              0x15
+#define TC_CFG_L1_STORE_POLICY__POLICY_22__SHIFT                                                              0x16
+#define TC_CFG_L1_STORE_POLICY__POLICY_23__SHIFT                                                              0x17
+#define TC_CFG_L1_STORE_POLICY__POLICY_24__SHIFT                                                              0x18
+#define TC_CFG_L1_STORE_POLICY__POLICY_25__SHIFT                                                              0x19
+#define TC_CFG_L1_STORE_POLICY__POLICY_26__SHIFT                                                              0x1a
+#define TC_CFG_L1_STORE_POLICY__POLICY_27__SHIFT                                                              0x1b
+#define TC_CFG_L1_STORE_POLICY__POLICY_28__SHIFT                                                              0x1c
+#define TC_CFG_L1_STORE_POLICY__POLICY_29__SHIFT                                                              0x1d
+#define TC_CFG_L1_STORE_POLICY__POLICY_30__SHIFT                                                              0x1e
+#define TC_CFG_L1_STORE_POLICY__POLICY_31__SHIFT                                                              0x1f
+#define TC_CFG_L1_STORE_POLICY__POLICY_0_MASK                                                                 0x00000001L
+#define TC_CFG_L1_STORE_POLICY__POLICY_1_MASK                                                                 0x00000002L
+#define TC_CFG_L1_STORE_POLICY__POLICY_2_MASK                                                                 0x00000004L
+#define TC_CFG_L1_STORE_POLICY__POLICY_3_MASK                                                                 0x00000008L
+#define TC_CFG_L1_STORE_POLICY__POLICY_4_MASK                                                                 0x00000010L
+#define TC_CFG_L1_STORE_POLICY__POLICY_5_MASK                                                                 0x00000020L
+#define TC_CFG_L1_STORE_POLICY__POLICY_6_MASK                                                                 0x00000040L
+#define TC_CFG_L1_STORE_POLICY__POLICY_7_MASK                                                                 0x00000080L
+#define TC_CFG_L1_STORE_POLICY__POLICY_8_MASK                                                                 0x00000100L
+#define TC_CFG_L1_STORE_POLICY__POLICY_9_MASK                                                                 0x00000200L
+#define TC_CFG_L1_STORE_POLICY__POLICY_10_MASK                                                                0x00000400L
+#define TC_CFG_L1_STORE_POLICY__POLICY_11_MASK                                                                0x00000800L
+#define TC_CFG_L1_STORE_POLICY__POLICY_12_MASK                                                                0x00001000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_13_MASK                                                                0x00002000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_14_MASK                                                                0x00004000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_15_MASK                                                                0x00008000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_16_MASK                                                                0x00010000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_17_MASK                                                                0x00020000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_18_MASK                                                                0x00040000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_19_MASK                                                                0x00080000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_20_MASK                                                                0x00100000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_21_MASK                                                                0x00200000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_22_MASK                                                                0x00400000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_23_MASK                                                                0x00800000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_24_MASK                                                                0x01000000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_25_MASK                                                                0x02000000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_26_MASK                                                                0x04000000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_27_MASK                                                                0x08000000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_28_MASK                                                                0x10000000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_29_MASK                                                                0x20000000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_30_MASK                                                                0x40000000L
+#define TC_CFG_L1_STORE_POLICY__POLICY_31_MASK                                                                0x80000000L
+//TC_CFG_L2_LOAD_POLICY0
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_0__SHIFT                                                               0x0
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_1__SHIFT                                                               0x2
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_2__SHIFT                                                               0x4
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_3__SHIFT                                                               0x6
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_4__SHIFT                                                               0x8
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_5__SHIFT                                                               0xa
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_6__SHIFT                                                               0xc
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_7__SHIFT                                                               0xe
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_8__SHIFT                                                               0x10
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_9__SHIFT                                                               0x12
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_10__SHIFT                                                              0x14
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_11__SHIFT                                                              0x16
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_12__SHIFT                                                              0x18
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_13__SHIFT                                                              0x1a
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_14__SHIFT                                                              0x1c
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_15__SHIFT                                                              0x1e
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_0_MASK                                                                 0x00000003L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_1_MASK                                                                 0x0000000CL
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_2_MASK                                                                 0x00000030L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_3_MASK                                                                 0x000000C0L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_4_MASK                                                                 0x00000300L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_5_MASK                                                                 0x00000C00L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_6_MASK                                                                 0x00003000L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_7_MASK                                                                 0x0000C000L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_8_MASK                                                                 0x00030000L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_9_MASK                                                                 0x000C0000L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_10_MASK                                                                0x00300000L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_11_MASK                                                                0x00C00000L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_12_MASK                                                                0x03000000L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_13_MASK                                                                0x0C000000L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_14_MASK                                                                0x30000000L
+#define TC_CFG_L2_LOAD_POLICY0__POLICY_15_MASK                                                                0xC0000000L
+//TC_CFG_L2_LOAD_POLICY1
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_16__SHIFT                                                              0x0
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_17__SHIFT                                                              0x2
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_18__SHIFT                                                              0x4
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_19__SHIFT                                                              0x6
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_20__SHIFT                                                              0x8
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_21__SHIFT                                                              0xa
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_22__SHIFT                                                              0xc
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_23__SHIFT                                                              0xe
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_24__SHIFT                                                              0x10
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_25__SHIFT                                                              0x12
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_26__SHIFT                                                              0x14
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_27__SHIFT                                                              0x16
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_28__SHIFT                                                              0x18
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_29__SHIFT                                                              0x1a
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_30__SHIFT                                                              0x1c
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_31__SHIFT                                                              0x1e
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_16_MASK                                                                0x00000003L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_17_MASK                                                                0x0000000CL
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_18_MASK                                                                0x00000030L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_19_MASK                                                                0x000000C0L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_20_MASK                                                                0x00000300L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_21_MASK                                                                0x00000C00L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_22_MASK                                                                0x00003000L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_23_MASK                                                                0x0000C000L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_24_MASK                                                                0x00030000L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_25_MASK                                                                0x000C0000L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_26_MASK                                                                0x00300000L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_27_MASK                                                                0x00C00000L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_28_MASK                                                                0x03000000L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_29_MASK                                                                0x0C000000L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_30_MASK                                                                0x30000000L
+#define TC_CFG_L2_LOAD_POLICY1__POLICY_31_MASK                                                                0xC0000000L
+//TC_CFG_L2_STORE_POLICY0
+#define TC_CFG_L2_STORE_POLICY0__POLICY_0__SHIFT                                                              0x0
+#define TC_CFG_L2_STORE_POLICY0__POLICY_1__SHIFT                                                              0x2
+#define TC_CFG_L2_STORE_POLICY0__POLICY_2__SHIFT                                                              0x4
+#define TC_CFG_L2_STORE_POLICY0__POLICY_3__SHIFT                                                              0x6
+#define TC_CFG_L2_STORE_POLICY0__POLICY_4__SHIFT                                                              0x8
+#define TC_CFG_L2_STORE_POLICY0__POLICY_5__SHIFT                                                              0xa
+#define TC_CFG_L2_STORE_POLICY0__POLICY_6__SHIFT                                                              0xc
+#define TC_CFG_L2_STORE_POLICY0__POLICY_7__SHIFT                                                              0xe
+#define TC_CFG_L2_STORE_POLICY0__POLICY_8__SHIFT                                                              0x10
+#define TC_CFG_L2_STORE_POLICY0__POLICY_9__SHIFT                                                              0x12
+#define TC_CFG_L2_STORE_POLICY0__POLICY_10__SHIFT                                                             0x14
+#define TC_CFG_L2_STORE_POLICY0__POLICY_11__SHIFT                                                             0x16
+#define TC_CFG_L2_STORE_POLICY0__POLICY_12__SHIFT                                                             0x18
+#define TC_CFG_L2_STORE_POLICY0__POLICY_13__SHIFT                                                             0x1a
+#define TC_CFG_L2_STORE_POLICY0__POLICY_14__SHIFT                                                             0x1c
+#define TC_CFG_L2_STORE_POLICY0__POLICY_15__SHIFT                                                             0x1e
+#define TC_CFG_L2_STORE_POLICY0__POLICY_0_MASK                                                                0x00000003L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_1_MASK                                                                0x0000000CL
+#define TC_CFG_L2_STORE_POLICY0__POLICY_2_MASK                                                                0x00000030L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_3_MASK                                                                0x000000C0L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_4_MASK                                                                0x00000300L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_5_MASK                                                                0x00000C00L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_6_MASK                                                                0x00003000L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_7_MASK                                                                0x0000C000L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_8_MASK                                                                0x00030000L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_9_MASK                                                                0x000C0000L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_10_MASK                                                               0x00300000L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_11_MASK                                                               0x00C00000L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_12_MASK                                                               0x03000000L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_13_MASK                                                               0x0C000000L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_14_MASK                                                               0x30000000L
+#define TC_CFG_L2_STORE_POLICY0__POLICY_15_MASK                                                               0xC0000000L
+//TC_CFG_L2_STORE_POLICY1
+#define TC_CFG_L2_STORE_POLICY1__POLICY_16__SHIFT                                                             0x0
+#define TC_CFG_L2_STORE_POLICY1__POLICY_17__SHIFT                                                             0x2
+#define TC_CFG_L2_STORE_POLICY1__POLICY_18__SHIFT                                                             0x4
+#define TC_CFG_L2_STORE_POLICY1__POLICY_19__SHIFT                                                             0x6
+#define TC_CFG_L2_STORE_POLICY1__POLICY_20__SHIFT                                                             0x8
+#define TC_CFG_L2_STORE_POLICY1__POLICY_21__SHIFT                                                             0xa
+#define TC_CFG_L2_STORE_POLICY1__POLICY_22__SHIFT                                                             0xc
+#define TC_CFG_L2_STORE_POLICY1__POLICY_23__SHIFT                                                             0xe
+#define TC_CFG_L2_STORE_POLICY1__POLICY_24__SHIFT                                                             0x10
+#define TC_CFG_L2_STORE_POLICY1__POLICY_25__SHIFT                                                             0x12
+#define TC_CFG_L2_STORE_POLICY1__POLICY_26__SHIFT                                                             0x14
+#define TC_CFG_L2_STORE_POLICY1__POLICY_27__SHIFT                                                             0x16
+#define TC_CFG_L2_STORE_POLICY1__POLICY_28__SHIFT                                                             0x18
+#define TC_CFG_L2_STORE_POLICY1__POLICY_29__SHIFT                                                             0x1a
+#define TC_CFG_L2_STORE_POLICY1__POLICY_30__SHIFT                                                             0x1c
+#define TC_CFG_L2_STORE_POLICY1__POLICY_31__SHIFT                                                             0x1e
+#define TC_CFG_L2_STORE_POLICY1__POLICY_16_MASK                                                               0x00000003L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_17_MASK                                                               0x0000000CL
+#define TC_CFG_L2_STORE_POLICY1__POLICY_18_MASK                                                               0x00000030L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_19_MASK                                                               0x000000C0L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_20_MASK                                                               0x00000300L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_21_MASK                                                               0x00000C00L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_22_MASK                                                               0x00003000L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_23_MASK                                                               0x0000C000L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_24_MASK                                                               0x00030000L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_25_MASK                                                               0x000C0000L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_26_MASK                                                               0x00300000L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_27_MASK                                                               0x00C00000L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_28_MASK                                                               0x03000000L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_29_MASK                                                               0x0C000000L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_30_MASK                                                               0x30000000L
+#define TC_CFG_L2_STORE_POLICY1__POLICY_31_MASK                                                               0xC0000000L
+//TC_CFG_L2_ATOMIC_POLICY
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_0__SHIFT                                                              0x0
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_1__SHIFT                                                              0x2
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_2__SHIFT                                                              0x4
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_3__SHIFT                                                              0x6
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_4__SHIFT                                                              0x8
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_5__SHIFT                                                              0xa
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_6__SHIFT                                                              0xc
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_7__SHIFT                                                              0xe
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_8__SHIFT                                                              0x10
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_9__SHIFT                                                              0x12
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_10__SHIFT                                                             0x14
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_11__SHIFT                                                             0x16
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_12__SHIFT                                                             0x18
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_13__SHIFT                                                             0x1a
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_14__SHIFT                                                             0x1c
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_15__SHIFT                                                             0x1e
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_0_MASK                                                                0x00000003L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_1_MASK                                                                0x0000000CL
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_2_MASK                                                                0x00000030L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_3_MASK                                                                0x000000C0L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_4_MASK                                                                0x00000300L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_5_MASK                                                                0x00000C00L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_6_MASK                                                                0x00003000L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_7_MASK                                                                0x0000C000L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_8_MASK                                                                0x00030000L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_9_MASK                                                                0x000C0000L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_10_MASK                                                               0x00300000L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_11_MASK                                                               0x00C00000L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_12_MASK                                                               0x03000000L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_13_MASK                                                               0x0C000000L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_14_MASK                                                               0x30000000L
+#define TC_CFG_L2_ATOMIC_POLICY__POLICY_15_MASK                                                               0xC0000000L
+//TC_CFG_L1_VOLATILE
+#define TC_CFG_L1_VOLATILE__VOL__SHIFT                                                                        0x0
+#define TC_CFG_L1_VOLATILE__VOL_MASK                                                                          0x0000000FL
+//TC_CFG_L2_VOLATILE
+#define TC_CFG_L2_VOLATILE__VOL__SHIFT                                                                        0x0
+#define TC_CFG_L2_VOLATILE__VOL_MASK                                                                          0x0000000FL
+//TCI_MISC
+#define TCI_MISC__FGCG_REPEATER_DISABLE__SHIFT                                                                0x0
+#define TCI_MISC__LEGACY_MGCG_DISABLE__SHIFT                                                                  0x1
+#define TCI_MISC__FGCG_REPEATER_DISABLE_MASK                                                                  0x00000001L
+#define TCI_MISC__LEGACY_MGCG_DISABLE_MASK                                                                    0x00000002L
+//TCI_CNTL_3
+#define TCI_CNTL_3__DISABLE_DOUBLING_L2_BANDWIDTH__SHIFT                                                      0x0
+#define TCI_CNTL_3__COMBINING_DELAY_WINDOW__SHIFT                                                             0x2
+#define TCI_CNTL_3__CHICKEN_BIT_TCR_MGCG__SHIFT                                                               0x4
+#define TCI_CNTL_3__TCR_FGCG_REPEATER_DISABLE__SHIFT                                                          0x7
+#define TCI_CNTL_3__DISABLE_DOUBLING_L2_BANDWIDTH_MASK                                                        0x00000003L
+#define TCI_CNTL_3__COMBINING_DELAY_WINDOW_MASK                                                               0x0000000CL
+#define TCI_CNTL_3__CHICKEN_BIT_TCR_MGCG_MASK                                                                 0x00000070L
+#define TCI_CNTL_3__TCR_FGCG_REPEATER_DISABLE_MASK                                                            0x00000080L
+//TCI_DSM_CNTL
+#define TCI_DSM_CNTL__WRITE_RAM_IRRITATOR_DATA_SEL__SHIFT                                                     0x0
+#define TCI_DSM_CNTL__WRITE_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                                 0x2
+#define TCI_DSM_CNTL__WRITE_RAM_IRRITATOR_DATA_SEL_MASK                                                       0x00000003L
+#define TCI_DSM_CNTL__WRITE_RAM_IRRITATOR_SINGLE_WRITE_MASK                                                   0x00000004L
+//TCI_DSM_CNTL2
+#define TCI_DSM_CNTL2__WRITE_RAM_ENABLE_ERROR_INJECT__SHIFT                                                   0x0
+#define TCI_DSM_CNTL2__WRITE_RAM_SELECT_INJECT_DELAY__SHIFT                                                   0x2
+#define TCI_DSM_CNTL2__TCI_INJECT_DELAY__SHIFT                                                                0x1a
+#define TCI_DSM_CNTL2__WRITE_RAM_ENABLE_ERROR_INJECT_MASK                                                     0x00000003L
+#define TCI_DSM_CNTL2__WRITE_RAM_SELECT_INJECT_DELAY_MASK                                                     0x00000004L
+#define TCI_DSM_CNTL2__TCI_INJECT_DELAY_MASK                                                                  0xFC000000L
+//TCI_STATUS
+#define TCI_STATUS__TCI_BUSY__SHIFT                                                                           0x0
+#define TCI_STATUS__TCI_BUSY_MASK                                                                             0x00000001L
+//TCI_CNTL_1
+#define TCI_CNTL_1__WBINVL1_NUM_CYCLES__SHIFT                                                                 0x0
+#define TCI_CNTL_1__REQ_FIFO_DEPTH__SHIFT                                                                     0x10
+#define TCI_CNTL_1__WDATA_RAM_DEPTH__SHIFT                                                                    0x18
+#define TCI_CNTL_1__WBINVL1_NUM_CYCLES_MASK                                                                   0x0000FFFFL
+#define TCI_CNTL_1__REQ_FIFO_DEPTH_MASK                                                                       0x00FF0000L
+#define TCI_CNTL_1__WDATA_RAM_DEPTH_MASK                                                                      0xFF000000L
+//TCI_CNTL_2
+#define TCI_CNTL_2__L1_INVAL_ON_WBINVL2__SHIFT                                                                0x0
+#define TCI_CNTL_2__TCA_MAX_CREDIT__SHIFT                                                                     0x1
+#define TCI_CNTL_2__L1_INVAL_ON_WBINVL2_MASK                                                                  0x00000001L
+#define TCI_CNTL_2__TCA_MAX_CREDIT_MASK                                                                       0x000001FEL
+//TCC_CTRL
+#define TCC_CTRL__CACHE_SIZE__SHIFT                                                                           0x0
+#define TCC_CTRL__RATE__SHIFT                                                                                 0x2
+#define TCC_CTRL__WRITEBACK_MARGIN__SHIFT                                                                     0x4
+#define TCC_CTRL__SRC_FIFO_SIZE__SHIFT                                                                        0xc
+#define TCC_CTRL__LATENCY_FIFO_SIZE__SHIFT                                                                    0x10
+#define TCC_CTRL__LINEAR_SET_HASH__SHIFT                                                                      0x15
+#define TCC_CTRL__OUTPUT_FIFO_CLK_MODE__SHIFT                                                                 0x16
+#define TCC_CTRL__EXECUTE_CLK_MODE__SHIFT                                                                     0x17
+#define TCC_CTRL__RETURN_BUFFER_CLK_MODE__SHIFT                                                               0x19
+#define TCC_CTRL__SRC_FIFO_CLK_MODE__SHIFT                                                                    0x1a
+#define TCC_CTRL__MC_WRITE_CLK_MODE__SHIFT                                                                    0x1b
+#define TCC_CTRL__LATENCY_FIFO_CLK_MODE__SHIFT                                                                0x1c
+#define TCC_CTRL__DISABLE_SHARED_128B_READ_REQUEST__SHIFT                                                     0x1d
+#define TCC_CTRL__CACHE_SIZE_MASK                                                                             0x00000003L
+#define TCC_CTRL__RATE_MASK                                                                                   0x0000000CL
+#define TCC_CTRL__WRITEBACK_MARGIN_MASK                                                                       0x000000F0L
+#define TCC_CTRL__SRC_FIFO_SIZE_MASK                                                                          0x0000F000L
+#define TCC_CTRL__LATENCY_FIFO_SIZE_MASK                                                                      0x000F0000L
+#define TCC_CTRL__LINEAR_SET_HASH_MASK                                                                        0x00200000L
+#define TCC_CTRL__OUTPUT_FIFO_CLK_MODE_MASK                                                                   0x00400000L
+#define TCC_CTRL__EXECUTE_CLK_MODE_MASK                                                                       0x01800000L
+#define TCC_CTRL__RETURN_BUFFER_CLK_MODE_MASK                                                                 0x02000000L
+#define TCC_CTRL__SRC_FIFO_CLK_MODE_MASK                                                                      0x04000000L
+#define TCC_CTRL__MC_WRITE_CLK_MODE_MASK                                                                      0x08000000L
+#define TCC_CTRL__LATENCY_FIFO_CLK_MODE_MASK                                                                  0x10000000L
+#define TCC_CTRL__DISABLE_SHARED_128B_READ_REQUEST_MASK                                                       0x20000000L
+//TCC_CTRL2
+#define TCC_CTRL2__PROBE_FIFO_SIZE__SHIFT                                                                     0x0
+#define TCC_CTRL2__INF_NAN_CLAMP__SHIFT                                                                       0x10
+#define TCC_CTRL2__PROBE_FILTER_CTRL__SHIFT                                                                   0x11
+#define TCC_CTRL2__WAIT_CLK_STABLE_CNT__SHIFT                                                                 0x12
+#define TCC_CTRL2__TCC_TCX_REPEATER_FGCG_DISABLE__SHIFT                                                       0x17
+#define TCC_CTRL2__TCC_EA0_RDREQ_FGCG_DISABLE__SHIFT                                                          0x18
+#define TCC_CTRL2__TCC_EA0_WRREQ_FGCG_DISABLE__SHIFT                                                          0x19
+#define TCC_CTRL2__TCC_TCX_ACK_REPEATER_FGCG_DISABLE__SHIFT                                                   0x1a
+#define TCC_CTRL2__TCC_TCA_HOLE_REPEATER_FGCG_DISABLE__SHIFT                                                  0x1b
+#define TCC_CTRL2__TCC_TCA_RTN_REPEATER_FGCG_DISABLE__SHIFT                                                   0x1c
+#define TCC_CTRL2__USE_EA_EARLYWRRET_ON_WRITEBACK__SHIFT                                                      0x1d
+#define TCC_CTRL2__Enable_TCC_64K_2M_1G_1T_hash__SHIFT                                                        0x1e
+#define TCC_CTRL2__PROBE_FIFO_SIZE_MASK                                                                       0x0000000FL
+#define TCC_CTRL2__INF_NAN_CLAMP_MASK                                                                         0x00010000L
+#define TCC_CTRL2__PROBE_FILTER_CTRL_MASK                                                                     0x00020000L
+#define TCC_CTRL2__WAIT_CLK_STABLE_CNT_MASK                                                                   0x007C0000L
+#define TCC_CTRL2__TCC_TCX_REPEATER_FGCG_DISABLE_MASK                                                         0x00800000L
+#define TCC_CTRL2__TCC_EA0_RDREQ_FGCG_DISABLE_MASK                                                            0x01000000L
+#define TCC_CTRL2__TCC_EA0_WRREQ_FGCG_DISABLE_MASK                                                            0x02000000L
+#define TCC_CTRL2__TCC_TCX_ACK_REPEATER_FGCG_DISABLE_MASK                                                     0x04000000L
+#define TCC_CTRL2__TCC_TCA_HOLE_REPEATER_FGCG_DISABLE_MASK                                                    0x08000000L
+#define TCC_CTRL2__TCC_TCA_RTN_REPEATER_FGCG_DISABLE_MASK                                                     0x10000000L
+#define TCC_CTRL2__USE_EA_EARLYWRRET_ON_WRITEBACK_MASK                                                        0x20000000L
+#define TCC_CTRL2__Enable_TCC_64K_2M_1G_1T_hash_MASK                                                          0x40000000L
+//TCC_DSM_CNTL
+#define TCC_DSM_CNTL__CACHE_DATA_IRRITATOR_DATA_SEL__SHIFT                                                    0x0
+#define TCC_DSM_CNTL__CACHE_DATA_IRRITATOR_SINGLE_WRITE__SHIFT                                                0x2
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_0_1_IRRITATOR_DATA_SEL__SHIFT                                           0x3
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_0_1_IRRITATOR_SINGLE_WRITE__SHIFT                                       0x5
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_0_IRRITATOR_DATA_SEL__SHIFT                                           0x6
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_0_IRRITATOR_SINGLE_WRITE__SHIFT                                       0x8
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_1_IRRITATOR_DATA_SEL__SHIFT                                           0x9
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_1_IRRITATOR_SINGLE_WRITE__SHIFT                                       0xb
+#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_0_IRRITATOR_DATA_SEL__SHIFT                                            0xc
+#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_0_IRRITATOR_SINGLE_WRITE__SHIFT                                        0xe
+#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_1_IRRITATOR_DATA_SEL__SHIFT                                            0xf
+#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_1_IRRITATOR_SINGLE_WRITE__SHIFT                                        0x11
+#define TCC_DSM_CNTL__HIGH_RATE_TAG_IRRITATOR_DATA_SEL__SHIFT                                                 0x12
+#define TCC_DSM_CNTL__HIGH_RATE_TAG_IRRITATOR_SINGLE_WRITE__SHIFT                                             0x14
+#define TCC_DSM_CNTL__LOW_RATE_TAG_IRRITATOR_DATA_SEL__SHIFT                                                  0x15
+#define TCC_DSM_CNTL__LOW_RATE_TAG_IRRITATOR_SINGLE_WRITE__SHIFT                                              0x17
+#define TCC_DSM_CNTL__IN_USE_DEC_IRRITATOR_DATA_SEL__SHIFT                                                    0x18
+#define TCC_DSM_CNTL__IN_USE_DEC_IRRITATOR_SINGLE_WRITE__SHIFT                                                0x1a
+#define TCC_DSM_CNTL__IN_USE_TRANSFER_IRRITATOR_DATA_SEL__SHIFT                                               0x1b
+#define TCC_DSM_CNTL__IN_USE_TRANSFER_IRRITATOR_SINGLE_WRITE__SHIFT                                           0x1d
+#define TCC_DSM_CNTL__CACHE_DATA_IRRITATOR_DATA_SEL_MASK                                                      0x00000003L
+#define TCC_DSM_CNTL__CACHE_DATA_IRRITATOR_SINGLE_WRITE_MASK                                                  0x00000004L
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_0_1_IRRITATOR_DATA_SEL_MASK                                             0x00000018L
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_0_1_IRRITATOR_SINGLE_WRITE_MASK                                         0x00000020L
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_0_IRRITATOR_DATA_SEL_MASK                                             0x000000C0L
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_0_IRRITATOR_SINGLE_WRITE_MASK                                         0x00000100L
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_1_IRRITATOR_DATA_SEL_MASK                                             0x00000600L
+#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_1_IRRITATOR_SINGLE_WRITE_MASK                                         0x00000800L
+#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_0_IRRITATOR_DATA_SEL_MASK                                              0x00003000L
+#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_0_IRRITATOR_SINGLE_WRITE_MASK                                          0x00004000L
+#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_1_IRRITATOR_DATA_SEL_MASK                                              0x00018000L
+#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_1_IRRITATOR_SINGLE_WRITE_MASK                                          0x00020000L
+#define TCC_DSM_CNTL__HIGH_RATE_TAG_IRRITATOR_DATA_SEL_MASK                                                   0x000C0000L
+#define TCC_DSM_CNTL__HIGH_RATE_TAG_IRRITATOR_SINGLE_WRITE_MASK                                               0x00100000L
+#define TCC_DSM_CNTL__LOW_RATE_TAG_IRRITATOR_DATA_SEL_MASK                                                    0x00600000L
+#define TCC_DSM_CNTL__LOW_RATE_TAG_IRRITATOR_SINGLE_WRITE_MASK                                                0x00800000L
+#define TCC_DSM_CNTL__IN_USE_DEC_IRRITATOR_DATA_SEL_MASK                                                      0x03000000L
+#define TCC_DSM_CNTL__IN_USE_DEC_IRRITATOR_SINGLE_WRITE_MASK                                                  0x04000000L
+#define TCC_DSM_CNTL__IN_USE_TRANSFER_IRRITATOR_DATA_SEL_MASK                                                 0x18000000L
+#define TCC_DSM_CNTL__IN_USE_TRANSFER_IRRITATOR_SINGLE_WRITE_MASK                                             0x20000000L
+//TCC_DSM_CNTLA
+#define TCC_DSM_CNTLA__SRC_FIFO_IRRITATOR_DATA_SEL__SHIFT                                                     0x0
+#define TCC_DSM_CNTLA__SRC_FIFO_IRRITATOR_SINGLE_WRITE__SHIFT                                                 0x2
+#define TCC_DSM_CNTLA__UC_ATOMIC_FIFO_IRRITATOR_DATA_SEL__SHIFT                                               0x3
+#define TCC_DSM_CNTLA__UC_ATOMIC_FIFO_IRRITATOR_SINGLE_WRITE__SHIFT                                           0x5
+#define TCC_DSM_CNTLA__WRITE_RETURN_IRRITATOR_DATA_SEL__SHIFT                                                 0x6
+#define TCC_DSM_CNTLA__WRITE_RETURN_IRRITATOR_SINGLE_WRITE__SHIFT                                             0x8
+#define TCC_DSM_CNTLA__WRITE_CACHE_READ_IRRITATOR_DATA_SEL__SHIFT                                             0x9
+#define TCC_DSM_CNTLA__WRITE_CACHE_READ_IRRITATOR_SINGLE_WRITE__SHIFT                                         0xb
+#define TCC_DSM_CNTLA__LATENCY_FIFO_NEXT_RAM_IRRITATOR_DATA_SEL__SHIFT                                        0xc
+#define TCC_DSM_CNTLA__LATENCY_FIFO_NEXT_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                    0xe
+#define TCC_DSM_CNTLA__CACHE_TAG_PROBE_FIFO_IRRITATOR_DATA_SEL__SHIFT                                         0xf
+#define TCC_DSM_CNTLA__CACHE_TAG_PROBE_FIFO_IRRITATOR_SINGLE_WRITE__SHIFT                                     0x11
+#define TCC_DSM_CNTLA__LATENCY_FIFO_IRRITATOR_DATA_SEL__SHIFT                                                 0x12
+#define TCC_DSM_CNTLA__LATENCY_FIFO_IRRITATOR_SINGLE_WRITE__SHIFT                                             0x14
+#define TCC_DSM_CNTLA__RETURN_DATA_IRRITATOR_DATA_SEL__SHIFT                                                  0x15
+#define TCC_DSM_CNTLA__RETURN_DATA_IRRITATOR_SINGLE_WRITE__SHIFT                                              0x17
+#define TCC_DSM_CNTLA__RETURN_CONTROL_IRRITATOR_DATA_SEL__SHIFT                                               0x18
+#define TCC_DSM_CNTLA__RETURN_CONTROL_IRRITATOR_SINGLE_WRITE__SHIFT                                           0x1a
+#define TCC_DSM_CNTLA__OUTPUT_FIFOS_IRRITATOR_DATA_SEL__SHIFT                                                 0x1b
+#define TCC_DSM_CNTLA__OUTPUT_FIFOS_IRRITATOR_SINGLE_WRITE__SHIFT                                             0x1d
+#define TCC_DSM_CNTLA__SRC_FIFO_IRRITATOR_DATA_SEL_MASK                                                       0x00000003L
+#define TCC_DSM_CNTLA__SRC_FIFO_IRRITATOR_SINGLE_WRITE_MASK                                                   0x00000004L
+#define TCC_DSM_CNTLA__UC_ATOMIC_FIFO_IRRITATOR_DATA_SEL_MASK                                                 0x00000018L
+#define TCC_DSM_CNTLA__UC_ATOMIC_FIFO_IRRITATOR_SINGLE_WRITE_MASK                                             0x00000020L
+#define TCC_DSM_CNTLA__WRITE_RETURN_IRRITATOR_DATA_SEL_MASK                                                   0x000000C0L
+#define TCC_DSM_CNTLA__WRITE_RETURN_IRRITATOR_SINGLE_WRITE_MASK                                               0x00000100L
+#define TCC_DSM_CNTLA__WRITE_CACHE_READ_IRRITATOR_DATA_SEL_MASK                                               0x00000600L
+#define TCC_DSM_CNTLA__WRITE_CACHE_READ_IRRITATOR_SINGLE_WRITE_MASK                                           0x00000800L
+#define TCC_DSM_CNTLA__LATENCY_FIFO_NEXT_RAM_IRRITATOR_DATA_SEL_MASK                                          0x00003000L
+#define TCC_DSM_CNTLA__LATENCY_FIFO_NEXT_RAM_IRRITATOR_SINGLE_WRITE_MASK                                      0x00004000L
+#define TCC_DSM_CNTLA__CACHE_TAG_PROBE_FIFO_IRRITATOR_DATA_SEL_MASK                                           0x00018000L
+#define TCC_DSM_CNTLA__CACHE_TAG_PROBE_FIFO_IRRITATOR_SINGLE_WRITE_MASK                                       0x00020000L
+#define TCC_DSM_CNTLA__LATENCY_FIFO_IRRITATOR_DATA_SEL_MASK                                                   0x000C0000L
+#define TCC_DSM_CNTLA__LATENCY_FIFO_IRRITATOR_SINGLE_WRITE_MASK                                               0x00100000L
+#define TCC_DSM_CNTLA__RETURN_DATA_IRRITATOR_DATA_SEL_MASK                                                    0x00600000L
+#define TCC_DSM_CNTLA__RETURN_DATA_IRRITATOR_SINGLE_WRITE_MASK                                                0x00800000L
+#define TCC_DSM_CNTLA__RETURN_CONTROL_IRRITATOR_DATA_SEL_MASK                                                 0x03000000L
+#define TCC_DSM_CNTLA__RETURN_CONTROL_IRRITATOR_SINGLE_WRITE_MASK                                             0x04000000L
+#define TCC_DSM_CNTLA__OUTPUT_FIFOS_IRRITATOR_DATA_SEL_MASK                                                   0x18000000L
+#define TCC_DSM_CNTLA__OUTPUT_FIFOS_IRRITATOR_SINGLE_WRITE_MASK                                               0x20000000L
+//TCC_DSM_CNTL2
+#define TCC_DSM_CNTL2__CACHE_DATA_ENABLE_ERROR_INJECT__SHIFT                                                  0x0
+#define TCC_DSM_CNTL2__CACHE_DATA_SELECT_INJECT_DELAY__SHIFT                                                  0x2
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_0_1_ENABLE_ERROR_INJECT__SHIFT                                         0x3
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_0_1_SELECT_INJECT_DELAY__SHIFT                                         0x5
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_0_ENABLE_ERROR_INJECT__SHIFT                                         0x6
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_0_SELECT_INJECT_DELAY__SHIFT                                         0x8
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_1_ENABLE_ERROR_INJECT__SHIFT                                         0x9
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_1_SELECT_INJECT_DELAY__SHIFT                                         0xb
+#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_0_ENABLE_ERROR_INJECT__SHIFT                                          0xc
+#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_0_SELECT_INJECT_DELAY__SHIFT                                          0xe
+#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_1_ENABLE_ERROR_INJECT__SHIFT                                          0xf
+#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_1_SELECT_INJECT_DELAY__SHIFT                                          0x11
+#define TCC_DSM_CNTL2__HIGH_RATE_TAG_ENABLE_ERROR_INJECT__SHIFT                                               0x12
+#define TCC_DSM_CNTL2__HIGH_RATE_TAG_SELECT_INJECT_DELAY__SHIFT                                               0x14
+#define TCC_DSM_CNTL2__LOW_RATE_TAG_ENABLE_ERROR_INJECT__SHIFT                                                0x15
+#define TCC_DSM_CNTL2__LOW_RATE_TAG_SELECT_INJECT_DELAY__SHIFT                                                0x17
+#define TCC_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                    0x1a
+#define TCC_DSM_CNTL2__CACHE_DATA_ENABLE_ERROR_INJECT_MASK                                                    0x00000003L
+#define TCC_DSM_CNTL2__CACHE_DATA_SELECT_INJECT_DELAY_MASK                                                    0x00000004L
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_0_1_ENABLE_ERROR_INJECT_MASK                                           0x00000018L
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_0_1_SELECT_INJECT_DELAY_MASK                                           0x00000020L
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_0_ENABLE_ERROR_INJECT_MASK                                           0x000000C0L
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_0_SELECT_INJECT_DELAY_MASK                                           0x00000100L
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_1_ENABLE_ERROR_INJECT_MASK                                           0x00000600L
+#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_1_SELECT_INJECT_DELAY_MASK                                           0x00000800L
+#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_0_ENABLE_ERROR_INJECT_MASK                                            0x00003000L
+#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_0_SELECT_INJECT_DELAY_MASK                                            0x00004000L
+#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_1_ENABLE_ERROR_INJECT_MASK                                            0x00018000L
+#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_1_SELECT_INJECT_DELAY_MASK                                            0x00020000L
+#define TCC_DSM_CNTL2__HIGH_RATE_TAG_ENABLE_ERROR_INJECT_MASK                                                 0x000C0000L
+#define TCC_DSM_CNTL2__HIGH_RATE_TAG_SELECT_INJECT_DELAY_MASK                                                 0x00100000L
+#define TCC_DSM_CNTL2__LOW_RATE_TAG_ENABLE_ERROR_INJECT_MASK                                                  0x00600000L
+#define TCC_DSM_CNTL2__LOW_RATE_TAG_SELECT_INJECT_DELAY_MASK                                                  0x00800000L
+#define TCC_DSM_CNTL2__INJECT_DELAY_MASK                                                                      0xFC000000L
+//TCC_DSM_CNTL2A
+#define TCC_DSM_CNTL2A__IN_USE_DEC_ENABLE_ERROR_INJECT__SHIFT                                                 0x0
+#define TCC_DSM_CNTL2A__IN_USE_DEC_SELECT_INJECT_DELAY__SHIFT                                                 0x2
+#define TCC_DSM_CNTL2A__IN_USE_TRANSFER_ENABLE_ERROR_INJECT__SHIFT                                            0x3
+#define TCC_DSM_CNTL2A__IN_USE_TRANSFER_SELECT_INJECT_DELAY__SHIFT                                            0x5
+#define TCC_DSM_CNTL2A__RETURN_DATA_ENABLE_ERROR_INJECT__SHIFT                                                0x6
+#define TCC_DSM_CNTL2A__RETURN_DATA_SELECT_INJECT_DELAY__SHIFT                                                0x8
+#define TCC_DSM_CNTL2A__RETURN_CONTROL_ENABLE_ERROR_INJECT__SHIFT                                             0x9
+#define TCC_DSM_CNTL2A__RETURN_CONTROL_SELECT_INJECT_DELAY__SHIFT                                             0xb
+#define TCC_DSM_CNTL2A__UC_ATOMIC_FIFO_ENABLE_ERROR_INJECT__SHIFT                                             0xc
+#define TCC_DSM_CNTL2A__UC_ATOMIC_FIFO_SELECT_INJECT_DELAY__SHIFT                                             0xe
+#define TCC_DSM_CNTL2A__WRITE_RETURN_ENABLE_ERROR_INJECT__SHIFT                                               0xf
+#define TCC_DSM_CNTL2A__WRITE_RETURN_SELECT_INJECT_DELAY__SHIFT                                               0x11
+#define TCC_DSM_CNTL2A__WRITE_CACHE_READ_ENABLE_ERROR_INJECT__SHIFT                                           0x12
+#define TCC_DSM_CNTL2A__WRITE_CACHE_READ_SELECT_INJECT_DELAY__SHIFT                                           0x14
+#define TCC_DSM_CNTL2A__SRC_FIFO_ENABLE_ERROR_INJECT__SHIFT                                                   0x15
+#define TCC_DSM_CNTL2A__SRC_FIFO_SELECT_INJECT_DELAY__SHIFT                                                   0x17
+#define TCC_DSM_CNTL2A__CACHE_TAG_PROBE_FIFO_ENABLE_ERROR_INJECT__SHIFT                                       0x18
+#define TCC_DSM_CNTL2A__CACHE_TAG_PROBE_FIFO_SELECT_INJECT_DELAY__SHIFT                                       0x1a
+#define TCC_DSM_CNTL2A__OUTPUT_FIFOS_ENABLE_ERROR_INJECT__SHIFT                                               0x1b
+#define TCC_DSM_CNTL2A__OUTPUT_FIFOS_SELECT_INJECT_DELAY__SHIFT                                               0x1d
+#define TCC_DSM_CNTL2A__IN_USE_DEC_ENABLE_ERROR_INJECT_MASK                                                   0x00000003L
+#define TCC_DSM_CNTL2A__IN_USE_DEC_SELECT_INJECT_DELAY_MASK                                                   0x00000004L
+#define TCC_DSM_CNTL2A__IN_USE_TRANSFER_ENABLE_ERROR_INJECT_MASK                                              0x00000018L
+#define TCC_DSM_CNTL2A__IN_USE_TRANSFER_SELECT_INJECT_DELAY_MASK                                              0x00000020L
+#define TCC_DSM_CNTL2A__RETURN_DATA_ENABLE_ERROR_INJECT_MASK                                                  0x000000C0L
+#define TCC_DSM_CNTL2A__RETURN_DATA_SELECT_INJECT_DELAY_MASK                                                  0x00000100L
+#define TCC_DSM_CNTL2A__RETURN_CONTROL_ENABLE_ERROR_INJECT_MASK                                               0x00000600L
+#define TCC_DSM_CNTL2A__RETURN_CONTROL_SELECT_INJECT_DELAY_MASK                                               0x00000800L
+#define TCC_DSM_CNTL2A__UC_ATOMIC_FIFO_ENABLE_ERROR_INJECT_MASK                                               0x00003000L
+#define TCC_DSM_CNTL2A__UC_ATOMIC_FIFO_SELECT_INJECT_DELAY_MASK                                               0x00004000L
+#define TCC_DSM_CNTL2A__WRITE_RETURN_ENABLE_ERROR_INJECT_MASK                                                 0x00018000L
+#define TCC_DSM_CNTL2A__WRITE_RETURN_SELECT_INJECT_DELAY_MASK                                                 0x00020000L
+#define TCC_DSM_CNTL2A__WRITE_CACHE_READ_ENABLE_ERROR_INJECT_MASK                                             0x000C0000L
+#define TCC_DSM_CNTL2A__WRITE_CACHE_READ_SELECT_INJECT_DELAY_MASK                                             0x00100000L
+#define TCC_DSM_CNTL2A__SRC_FIFO_ENABLE_ERROR_INJECT_MASK                                                     0x00600000L
+#define TCC_DSM_CNTL2A__SRC_FIFO_SELECT_INJECT_DELAY_MASK                                                     0x00800000L
+#define TCC_DSM_CNTL2A__CACHE_TAG_PROBE_FIFO_ENABLE_ERROR_INJECT_MASK                                         0x03000000L
+#define TCC_DSM_CNTL2A__CACHE_TAG_PROBE_FIFO_SELECT_INJECT_DELAY_MASK                                         0x04000000L
+#define TCC_DSM_CNTL2A__OUTPUT_FIFOS_ENABLE_ERROR_INJECT_MASK                                                 0x18000000L
+#define TCC_DSM_CNTL2A__OUTPUT_FIFOS_SELECT_INJECT_DELAY_MASK                                                 0x20000000L
+//TCC_DSM_CNTL2B
+#define TCC_DSM_CNTL2B__LATENCY_FIFO_ENABLE_ERROR_INJECT__SHIFT                                               0x0
+#define TCC_DSM_CNTL2B__LATENCY_FIFO_SELECT_INJECT_DELAY__SHIFT                                               0x2
+#define TCC_DSM_CNTL2B__LATENCY_FIFO_NEXT_RAM_ENABLE_ERROR_INJECT__SHIFT                                      0x3
+#define TCC_DSM_CNTL2B__LATENCY_FIFO_NEXT_RAM_SELECT_INJECT_DELAY__SHIFT                                      0x5
+#define TCC_DSM_CNTL2B__WRITE_EARLY_RETURN_ENABLE_ERROR_INJECT__SHIFT                                         0xc
+#define TCC_DSM_CNTL2B__WRITE_EARLY_RETURN_SELECT_INJECT_DELAY__SHIFT                                         0xe
+#define TCC_DSM_CNTL2B__WRITE_EARLY_RETURN_IRRITATOR_DATA_SEL__SHIFT                                          0xf
+#define TCC_DSM_CNTL2B__WRITE_EARLY_RETURN_IRRITATOR_SINGLE_WRITE__SHIFT                                      0x11
+#define TCC_DSM_CNTL2B__RETURN_BUFFER_LEVEL_BUBBLE_THRESHOLD__SHIFT                                           0x12
+#define TCC_DSM_CNTL2B__LATENCY_FIFO_ENABLE_ERROR_INJECT_MASK                                                 0x00000003L
+#define TCC_DSM_CNTL2B__LATENCY_FIFO_SELECT_INJECT_DELAY_MASK                                                 0x00000004L
+#define TCC_DSM_CNTL2B__LATENCY_FIFO_NEXT_RAM_ENABLE_ERROR_INJECT_MASK                                        0x00000018L
+#define TCC_DSM_CNTL2B__LATENCY_FIFO_NEXT_RAM_SELECT_INJECT_DELAY_MASK                                        0x00000020L
+#define TCC_DSM_CNTL2B__WRITE_EARLY_RETURN_ENABLE_ERROR_INJECT_MASK                                           0x00003000L
+#define TCC_DSM_CNTL2B__WRITE_EARLY_RETURN_SELECT_INJECT_DELAY_MASK                                           0x00004000L
+#define TCC_DSM_CNTL2B__WRITE_EARLY_RETURN_IRRITATOR_DATA_SEL_MASK                                            0x00018000L
+#define TCC_DSM_CNTL2B__WRITE_EARLY_RETURN_IRRITATOR_SINGLE_WRITE_MASK                                        0x00020000L
+#define TCC_DSM_CNTL2B__RETURN_BUFFER_LEVEL_BUBBLE_THRESHOLD_MASK                                             0x00FC0000L
+//TCC_WBINVL2
+#define TCC_WBINVL2__DONE__SHIFT                                                                              0x4
+#define TCC_WBINVL2__DONE_MASK                                                                                0x00000010L
+//TCC_SOFT_RESET
+#define TCC_SOFT_RESET__HALT_FOR_RESET__SHIFT                                                                 0x0
+#define TCC_SOFT_RESET__HALT_FOR_RESET_MASK                                                                   0x00000001L
+//TCC_DSM_CNTL3
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_2_IRRITATOR_DATA_SEL__SHIFT                                          0x0
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_2_IRRITATOR_SINGLE_WRITE__SHIFT                                      0x2
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_3_IRRITATOR_DATA_SEL__SHIFT                                          0x3
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_3_IRRITATOR_SINGLE_WRITE__SHIFT                                      0x5
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_2_IRRITATOR_DATA_SEL__SHIFT                                          0x6
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_2_IRRITATOR_SINGLE_WRITE__SHIFT                                      0x8
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_3_IRRITATOR_DATA_SEL__SHIFT                                          0x9
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_3_IRRITATOR_SINGLE_WRITE__SHIFT                                      0xb
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_2_ENABLE_ERROR_INJECT__SHIFT                                         0xc
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_2_SELECT_INJECT_DELAY__SHIFT                                         0xe
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_3_ENABLE_ERROR_INJECT__SHIFT                                         0xf
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_3_SELECT_INJECT_DELAY__SHIFT                                         0x11
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_2_ENABLE_ERROR_INJECT__SHIFT                                         0x12
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_2_SELECT_INJECT_DELAY__SHIFT                                         0x14
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_3_ENABLE_ERROR_INJECT__SHIFT                                         0x15
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_3_SELECT_INJECT_DELAY__SHIFT                                         0x17
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_2_IRRITATOR_DATA_SEL_MASK                                            0x00000003L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_2_IRRITATOR_SINGLE_WRITE_MASK                                        0x00000004L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_3_IRRITATOR_DATA_SEL_MASK                                            0x00000018L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_3_IRRITATOR_SINGLE_WRITE_MASK                                        0x00000020L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_2_IRRITATOR_DATA_SEL_MASK                                            0x000000C0L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_2_IRRITATOR_SINGLE_WRITE_MASK                                        0x00000100L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_3_IRRITATOR_DATA_SEL_MASK                                            0x00000600L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_3_IRRITATOR_SINGLE_WRITE_MASK                                        0x00000800L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_2_ENABLE_ERROR_INJECT_MASK                                           0x00003000L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_2_SELECT_INJECT_DELAY_MASK                                           0x00004000L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_3_ENABLE_ERROR_INJECT_MASK                                           0x00018000L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_0_3_SELECT_INJECT_DELAY_MASK                                           0x00020000L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_2_ENABLE_ERROR_INJECT_MASK                                           0x000C0000L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_2_SELECT_INJECT_DELAY_MASK                                           0x00100000L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_3_ENABLE_ERROR_INJECT_MASK                                           0x00600000L
+#define TCC_DSM_CNTL3__CACHE_DATA_BANK_1_3_SELECT_INJECT_DELAY_MASK                                           0x00800000L
+//TCA_CTRL
+#define TCA_CTRL__HOLE_TIMEOUT__SHIFT                                                                         0x0
+#define TCA_CTRL__RB_STILL_4_PHASE__SHIFT                                                                     0x4
+#define TCA_CTRL__RB_AS_TCI__SHIFT                                                                            0x5
+#define TCA_CTRL__DISABLE_UTCL2_PRIORITY__SHIFT                                                               0x6
+#define TCA_CTRL__DISABLE_RB_ONLY_TCA_ARBITER__SHIFT                                                          0x7
+#define TCA_CTRL__TCA_TCC_FGCG_DISABLE__SHIFT                                                                 0x8
+#define TCA_CTRL__TCA_TCA_FGCG_DISABLE__SHIFT                                                                 0x9
+#define TCA_CTRL__TCA_TCH_FGCG_DISABLE__SHIFT                                                                 0xa
+#define TCA_CTRL__TCA_TCX_FGCG_DISABLE__SHIFT                                                                 0xb
+#define TCA_CTRL__TCA_RANDOM_REVERSE_PRIORITY_ENABLE__SHIFT                                                   0xc
+#define TCA_CTRL__RTN_CREDIT_THRESHOLD__SHIFT                                                                 0xd
+#define TCA_CTRL__ACK_CREDIT_THRESHOLD__SHIFT                                                                 0x10
+#define TCA_CTRL__RTN_ARB_MODE__SHIFT                                                                         0x13
+#define TCA_CTRL__HOLE_ARB_SHARE_THRESHOLD__SHIFT                                                             0x18
+#define TCA_CTRL__HOLE_ARB_LANE_THRESHOLD__SHIFT                                                              0x1c
+#define TCA_CTRL__HOLE_TIMEOUT_MASK                                                                           0x0000000FL
+#define TCA_CTRL__RB_STILL_4_PHASE_MASK                                                                       0x00000010L
+#define TCA_CTRL__RB_AS_TCI_MASK                                                                              0x00000020L
+#define TCA_CTRL__DISABLE_UTCL2_PRIORITY_MASK                                                                 0x00000040L
+#define TCA_CTRL__DISABLE_RB_ONLY_TCA_ARBITER_MASK                                                            0x00000080L
+#define TCA_CTRL__TCA_TCC_FGCG_DISABLE_MASK                                                                   0x00000100L
+#define TCA_CTRL__TCA_TCA_FGCG_DISABLE_MASK                                                                   0x00000200L
+#define TCA_CTRL__TCA_TCH_FGCG_DISABLE_MASK                                                                   0x00000400L
+#define TCA_CTRL__TCA_TCX_FGCG_DISABLE_MASK                                                                   0x00000800L
+#define TCA_CTRL__TCA_RANDOM_REVERSE_PRIORITY_ENABLE_MASK                                                     0x00001000L
+#define TCA_CTRL__RTN_CREDIT_THRESHOLD_MASK                                                                   0x0000E000L
+#define TCA_CTRL__ACK_CREDIT_THRESHOLD_MASK                                                                   0x00070000L
+#define TCA_CTRL__RTN_ARB_MODE_MASK                                                                           0x00080000L
+#define TCA_CTRL__HOLE_ARB_SHARE_THRESHOLD_MASK                                                               0x07000000L
+#define TCA_CTRL__HOLE_ARB_LANE_THRESHOLD_MASK                                                                0x70000000L
+//TCA_BURST_MASK
+#define TCA_BURST_MASK__ADDR_MASK__SHIFT                                                                      0x0
+#define TCA_BURST_MASK__ADDR_MASK_MASK                                                                        0xFFFFFFFFL
+//TCA_BURST_CTRL
+#define TCA_BURST_CTRL__MAX_BURST__SHIFT                                                                      0x0
+#define TCA_BURST_CTRL__TCP_DISABLE__SHIFT                                                                    0x4
+#define TCA_BURST_CTRL__SQC_DISABLE__SHIFT                                                                    0x5
+#define TCA_BURST_CTRL__CPF_DISABLE__SHIFT                                                                    0x6
+#define TCA_BURST_CTRL__CPG_DISABLE__SHIFT                                                                    0x7
+#define TCA_BURST_CTRL__SQG_DISABLE__SHIFT                                                                    0xa
+#define TCA_BURST_CTRL__UTCL2_DISABLE__SHIFT                                                                  0xb
+#define TCA_BURST_CTRL__TPI_DISABLE__SHIFT                                                                    0xc
+#define TCA_BURST_CTRL__RLC_DISABLE__SHIFT                                                                    0xd
+#define TCA_BURST_CTRL__MAX_BURST_MASK                                                                        0x00000007L
+#define TCA_BURST_CTRL__TCP_DISABLE_MASK                                                                      0x00000010L
+#define TCA_BURST_CTRL__SQC_DISABLE_MASK                                                                      0x00000020L
+#define TCA_BURST_CTRL__CPF_DISABLE_MASK                                                                      0x00000040L
+#define TCA_BURST_CTRL__CPG_DISABLE_MASK                                                                      0x00000080L
+#define TCA_BURST_CTRL__SQG_DISABLE_MASK                                                                      0x00000400L
+#define TCA_BURST_CTRL__UTCL2_DISABLE_MASK                                                                    0x00000800L
+#define TCA_BURST_CTRL__TPI_DISABLE_MASK                                                                      0x00001000L
+#define TCA_BURST_CTRL__RLC_DISABLE_MASK                                                                      0x00002000L
+//TCA_DSM_CNTL
+#define TCA_DSM_CNTL__HOLE_FIFO_SED_IRRITATOR_DATA_SEL__SHIFT                                                 0x0
+#define TCA_DSM_CNTL__HOLE_FIFO_SED_IRRITATOR_SINGLE_WRITE__SHIFT                                             0x2
+#define TCA_DSM_CNTL__REQ_FIFO_SED_IRRITATOR_DATA_SEL__SHIFT                                                  0x3
+#define TCA_DSM_CNTL__REQ_FIFO_SED_IRRITATOR_SINGLE_WRITE__SHIFT                                              0x5
+#define TCA_DSM_CNTL__HOLE_FIFO_SED_IRRITATOR_DATA_SEL_MASK                                                   0x00000003L
+#define TCA_DSM_CNTL__HOLE_FIFO_SED_IRRITATOR_SINGLE_WRITE_MASK                                               0x00000004L
+#define TCA_DSM_CNTL__REQ_FIFO_SED_IRRITATOR_DATA_SEL_MASK                                                    0x00000018L
+#define TCA_DSM_CNTL__REQ_FIFO_SED_IRRITATOR_SINGLE_WRITE_MASK                                                0x00000020L
+//TCA_DSM_CNTL2
+#define TCA_DSM_CNTL2__HOLE_FIFO_SED_ENABLE_ERROR_INJECT__SHIFT                                               0x0
+#define TCA_DSM_CNTL2__HOLE_FIFO_SED_SELECT_INJECT_DELAY__SHIFT                                               0x2
+#define TCA_DSM_CNTL2__REQ_FIFO_SED_ENABLE_ERROR_INJECT__SHIFT                                                0x3
+#define TCA_DSM_CNTL2__REQ_FIFO_SED_SELECT_INJECT_DELAY__SHIFT                                                0x5
+#define TCA_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                    0x1a
+#define TCA_DSM_CNTL2__HOLE_FIFO_SED_ENABLE_ERROR_INJECT_MASK                                                 0x00000003L
+#define TCA_DSM_CNTL2__HOLE_FIFO_SED_SELECT_INJECT_DELAY_MASK                                                 0x00000004L
+#define TCA_DSM_CNTL2__REQ_FIFO_SED_ENABLE_ERROR_INJECT_MASK                                                  0x00000018L
+#define TCA_DSM_CNTL2__REQ_FIFO_SED_SELECT_INJECT_DELAY_MASK                                                  0x00000020L
+#define TCA_DSM_CNTL2__INJECT_DELAY_MASK                                                                      0xFC000000L
+//TCX_CTRL
+#define TCX_CTRL__TCX_TCX_FGCG_DISABLE__SHIFT                                                                 0x0
+#define TCX_CTRL__TCX_TCR_FGCG_DISABLE__SHIFT                                                                 0x1
+#define TCX_CTRL__TCX_TCC_FGCG_DISABLE__SHIFT                                                                 0x2
+#define TCX_CTRL__TCX_TCX_FGCG_DISABLE_MASK                                                                   0x00000001L
+#define TCX_CTRL__TCX_TCR_FGCG_DISABLE_MASK                                                                   0x00000002L
+#define TCX_CTRL__TCX_TCC_FGCG_DISABLE_MASK                                                                   0x00000004L
+//TCX_DSM_CNTL
+#define TCX_DSM_CNTL__GROUP0_SED_IRRITATOR_DATA_SEL__SHIFT                                                    0x0
+#define TCX_DSM_CNTL__GROUP1_SED_IRRITATOR_DATA_SEL__SHIFT                                                    0x2
+#define TCX_DSM_CNTL__GROUP2_SED_IRRITATOR_DATA_SEL__SHIFT                                                    0x4
+#define TCX_DSM_CNTL__GROUP4_SED_IRRITATOR_DATA_SEL__SHIFT                                                    0x8
+#define TCX_DSM_CNTL__GROUP5_SED_IRRITATOR_DATA_SEL__SHIFT                                                    0xa
+#define TCX_DSM_CNTL__GROUP6_SED_IRRITATOR_DATA_SEL__SHIFT                                                    0xc
+#define TCX_DSM_CNTL__GROUP8_SED_IRRITATOR_DATA_SEL__SHIFT                                                    0x10
+#define TCX_DSM_CNTL__GROUP9_SED_IRRITATOR_DATA_SEL__SHIFT                                                    0x12
+#define TCX_DSM_CNTL__GROUP10_SED_IRRITATOR_DATA_SEL__SHIFT                                                   0x14
+#define TCX_DSM_CNTL__GROUP13_SED_IRRITATOR_DATA_SEL__SHIFT                                                   0x1a
+#define TCX_DSM_CNTL__GROUP14_SED_IRRITATOR_DATA_SEL__SHIFT                                                   0x1c
+#define TCX_DSM_CNTL__SED_IRRITATOR_SINGLE_WRITE__SHIFT                                                       0x1e
+#define TCX_DSM_CNTL__GROUP0_SED_IRRITATOR_DATA_SEL_MASK                                                      0x00000003L
+#define TCX_DSM_CNTL__GROUP1_SED_IRRITATOR_DATA_SEL_MASK                                                      0x0000000CL
+#define TCX_DSM_CNTL__GROUP2_SED_IRRITATOR_DATA_SEL_MASK                                                      0x00000030L
+#define TCX_DSM_CNTL__GROUP4_SED_IRRITATOR_DATA_SEL_MASK                                                      0x00000300L
+#define TCX_DSM_CNTL__GROUP5_SED_IRRITATOR_DATA_SEL_MASK                                                      0x00000C00L
+#define TCX_DSM_CNTL__GROUP6_SED_IRRITATOR_DATA_SEL_MASK                                                      0x00003000L
+#define TCX_DSM_CNTL__GROUP8_SED_IRRITATOR_DATA_SEL_MASK                                                      0x00030000L
+#define TCX_DSM_CNTL__GROUP9_SED_IRRITATOR_DATA_SEL_MASK                                                      0x000C0000L
+#define TCX_DSM_CNTL__GROUP10_SED_IRRITATOR_DATA_SEL_MASK                                                     0x00300000L
+#define TCX_DSM_CNTL__GROUP13_SED_IRRITATOR_DATA_SEL_MASK                                                     0x0C000000L
+#define TCX_DSM_CNTL__GROUP14_SED_IRRITATOR_DATA_SEL_MASK                                                     0x30000000L
+#define TCX_DSM_CNTL__SED_IRRITATOR_SINGLE_WRITE_MASK                                                         0x40000000L
+//TCX_DSM_CNTL2
+#define TCX_DSM_CNTL2__SED_ENABLE_ERROR_INJECT__SHIFT                                                         0x0
+#define TCX_DSM_CNTL2__SED_SELECT_INJECT_DELAY__SHIFT                                                         0x2
+#define TCX_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                    0x1a
+#define TCX_DSM_CNTL2__SED_ENABLE_ERROR_INJECT_MASK                                                           0x00000003L
+#define TCX_DSM_CNTL2__SED_SELECT_INJECT_DELAY_MASK                                                           0x00000004L
+#define TCX_DSM_CNTL2__INJECT_DELAY_MASK                                                                      0xFC000000L
+
+
+// addressBlock: xcd0_gc_shdec
+//SPI_SHADER_PGM_RSRC3_PS
+#define SPI_SHADER_PGM_RSRC3_PS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC3_PS__WAVE_LIMIT__SHIFT                                                            0x10
+#define SPI_SHADER_PGM_RSRC3_PS__LOCK_LOW_THRESHOLD__SHIFT                                                    0x16
+#define SPI_SHADER_PGM_RSRC3_PS__SIMD_DISABLE__SHIFT                                                          0x1a
+#define SPI_SHADER_PGM_RSRC3_PS__CU_EN_MASK                                                                   0x0000FFFFL
+#define SPI_SHADER_PGM_RSRC3_PS__WAVE_LIMIT_MASK                                                              0x003F0000L
+#define SPI_SHADER_PGM_RSRC3_PS__LOCK_LOW_THRESHOLD_MASK                                                      0x03C00000L
+#define SPI_SHADER_PGM_RSRC3_PS__SIMD_DISABLE_MASK                                                            0x3C000000L
+//SPI_SHADER_PGM_LO_PS
+#define SPI_SHADER_PGM_LO_PS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_PS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_PS
+#define SPI_SHADER_PGM_HI_PS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_PS__MEM_BASE_MASK                                                                   0xFFL
+//SPI_SHADER_PGM_RSRC1_PS
+#define SPI_SHADER_PGM_RSRC1_PS__VGPRS__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC1_PS__SGPRS__SHIFT                                                                 0x6
+#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY__SHIFT                                                              0xa
+#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE__SHIFT                                                            0xc
+#define SPI_SHADER_PGM_RSRC1_PS__PRIV__SHIFT                                                                  0x14
+#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP__SHIFT                                                            0x15
+#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE__SHIFT                                                             0x17
+#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE__SHIFT                                                      0x18
+#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER__SHIFT                                                             0x1c
+#define SPI_SHADER_PGM_RSRC1_PS__FP16_OVFL__SHIFT                                                             0x1d
+#define SPI_SHADER_PGM_RSRC1_PS__VGPRS_MASK                                                                   0x0000003FL
+#define SPI_SHADER_PGM_RSRC1_PS__SGPRS_MASK                                                                   0x000003C0L
+#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY_MASK                                                                0x00000C00L
+#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE_MASK                                                              0x000FF000L
+#define SPI_SHADER_PGM_RSRC1_PS__PRIV_MASK                                                                    0x00100000L
+#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP_MASK                                                              0x00200000L
+#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE_MASK                                                               0x00800000L
+#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE_MASK                                                        0x01000000L
+#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER_MASK                                                               0x10000000L
+#define SPI_SHADER_PGM_RSRC1_PS__FP16_OVFL_MASK                                                               0x20000000L
+//SPI_SHADER_PGM_RSRC2_PS
+#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR__SHIFT                                                             0x1
+#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT__SHIFT                                                          0x6
+#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN__SHIFT                                                           0x7
+#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE__SHIFT                                                        0x8
+#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN__SHIFT                                                               0x10
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_COLLISION_WAVEID__SHIFT                                                 0x19
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_INTRAWAVE_COLLISION__SHIFT                                              0x1a
+#define SPI_SHADER_PGM_RSRC2_PS__SKIP_USGPR0__SHIFT                                                           0x1b
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MSB__SHIFT                                                         0x1c
+#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN_MASK                                                              0x00000001L
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MASK                                                               0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT_MASK                                                            0x00000040L
+#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN_MASK                                                             0x00000080L
+#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE_MASK                                                          0x0000FF00L
+#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN_MASK                                                                 0x01FF0000L
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_COLLISION_WAVEID_MASK                                                   0x02000000L
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_INTRAWAVE_COLLISION_MASK                                                0x04000000L
+#define SPI_SHADER_PGM_RSRC2_PS__SKIP_USGPR0_MASK                                                             0x08000000L
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MSB_MASK                                                           0x10000000L
+//SPI_SHADER_USER_DATA_PS_0
+#define SPI_SHADER_USER_DATA_PS_0__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_0__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_1
+#define SPI_SHADER_USER_DATA_PS_1__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_1__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_2
+#define SPI_SHADER_USER_DATA_PS_2__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_2__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_3
+#define SPI_SHADER_USER_DATA_PS_3__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_3__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_4
+#define SPI_SHADER_USER_DATA_PS_4__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_4__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_5
+#define SPI_SHADER_USER_DATA_PS_5__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_5__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_6
+#define SPI_SHADER_USER_DATA_PS_6__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_6__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_7
+#define SPI_SHADER_USER_DATA_PS_7__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_7__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_8
+#define SPI_SHADER_USER_DATA_PS_8__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_8__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_9
+#define SPI_SHADER_USER_DATA_PS_9__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_9__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_10
+#define SPI_SHADER_USER_DATA_PS_10__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_10__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_11
+#define SPI_SHADER_USER_DATA_PS_11__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_11__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_12
+#define SPI_SHADER_USER_DATA_PS_12__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_12__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_13
+#define SPI_SHADER_USER_DATA_PS_13__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_13__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_14
+#define SPI_SHADER_USER_DATA_PS_14__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_14__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_15
+#define SPI_SHADER_USER_DATA_PS_15__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_15__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_16
+#define SPI_SHADER_USER_DATA_PS_16__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_16__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_17
+#define SPI_SHADER_USER_DATA_PS_17__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_17__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_18
+#define SPI_SHADER_USER_DATA_PS_18__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_18__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_19
+#define SPI_SHADER_USER_DATA_PS_19__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_19__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_20
+#define SPI_SHADER_USER_DATA_PS_20__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_20__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_21
+#define SPI_SHADER_USER_DATA_PS_21__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_21__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_22
+#define SPI_SHADER_USER_DATA_PS_22__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_22__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_23
+#define SPI_SHADER_USER_DATA_PS_23__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_23__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_24
+#define SPI_SHADER_USER_DATA_PS_24__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_24__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_25
+#define SPI_SHADER_USER_DATA_PS_25__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_25__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_26
+#define SPI_SHADER_USER_DATA_PS_26__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_26__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_27
+#define SPI_SHADER_USER_DATA_PS_27__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_27__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_28
+#define SPI_SHADER_USER_DATA_PS_28__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_28__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_29
+#define SPI_SHADER_USER_DATA_PS_29__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_29__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_30
+#define SPI_SHADER_USER_DATA_PS_30__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_30__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_31
+#define SPI_SHADER_USER_DATA_PS_31__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_31__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC3_VS
+#define SPI_SHADER_PGM_RSRC3_VS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC3_VS__WAVE_LIMIT__SHIFT                                                            0x10
+#define SPI_SHADER_PGM_RSRC3_VS__LOCK_LOW_THRESHOLD__SHIFT                                                    0x16
+#define SPI_SHADER_PGM_RSRC3_VS__SIMD_DISABLE__SHIFT                                                          0x1a
+#define SPI_SHADER_PGM_RSRC3_VS__CU_EN_MASK                                                                   0x0000FFFFL
+#define SPI_SHADER_PGM_RSRC3_VS__WAVE_LIMIT_MASK                                                              0x003F0000L
+#define SPI_SHADER_PGM_RSRC3_VS__LOCK_LOW_THRESHOLD_MASK                                                      0x03C00000L
+#define SPI_SHADER_PGM_RSRC3_VS__SIMD_DISABLE_MASK                                                            0x3C000000L
+//SPI_SHADER_LATE_ALLOC_VS
+#define SPI_SHADER_LATE_ALLOC_VS__LIMIT__SHIFT                                                                0x0
+#define SPI_SHADER_LATE_ALLOC_VS__LIMIT_MASK                                                                  0x0000003FL
+//SPI_SHADER_PGM_LO_VS
+#define SPI_SHADER_PGM_LO_VS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_VS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_VS
+#define SPI_SHADER_PGM_HI_VS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_VS__MEM_BASE_MASK                                                                   0xFFL
+//SPI_SHADER_PGM_RSRC1_VS
+#define SPI_SHADER_PGM_RSRC1_VS__VGPRS__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC1_VS__SGPRS__SHIFT                                                                 0x6
+#define SPI_SHADER_PGM_RSRC1_VS__PRIORITY__SHIFT                                                              0xa
+#define SPI_SHADER_PGM_RSRC1_VS__FLOAT_MODE__SHIFT                                                            0xc
+#define SPI_SHADER_PGM_RSRC1_VS__PRIV__SHIFT                                                                  0x14
+#define SPI_SHADER_PGM_RSRC1_VS__DX10_CLAMP__SHIFT                                                            0x15
+#define SPI_SHADER_PGM_RSRC1_VS__DEBUG_MODE__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC1_VS__IEEE_MODE__SHIFT                                                             0x17
+#define SPI_SHADER_PGM_RSRC1_VS__VGPR_COMP_CNT__SHIFT                                                         0x18
+#define SPI_SHADER_PGM_RSRC1_VS__CU_GROUP_ENABLE__SHIFT                                                       0x1a
+#define SPI_SHADER_PGM_RSRC1_VS__CDBG_USER__SHIFT                                                             0x1e
+#define SPI_SHADER_PGM_RSRC1_VS__FP16_OVFL__SHIFT                                                             0x1f
+#define SPI_SHADER_PGM_RSRC1_VS__VGPRS_MASK                                                                   0x0000003FL
+#define SPI_SHADER_PGM_RSRC1_VS__SGPRS_MASK                                                                   0x000003C0L
+#define SPI_SHADER_PGM_RSRC1_VS__PRIORITY_MASK                                                                0x00000C00L
+#define SPI_SHADER_PGM_RSRC1_VS__FLOAT_MODE_MASK                                                              0x000FF000L
+#define SPI_SHADER_PGM_RSRC1_VS__PRIV_MASK                                                                    0x00100000L
+#define SPI_SHADER_PGM_RSRC1_VS__DX10_CLAMP_MASK                                                              0x00200000L
+#define SPI_SHADER_PGM_RSRC1_VS__DEBUG_MODE_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC1_VS__IEEE_MODE_MASK                                                               0x00800000L
+#define SPI_SHADER_PGM_RSRC1_VS__VGPR_COMP_CNT_MASK                                                           0x03000000L
+#define SPI_SHADER_PGM_RSRC1_VS__CU_GROUP_ENABLE_MASK                                                         0x04000000L
+#define SPI_SHADER_PGM_RSRC1_VS__CDBG_USER_MASK                                                               0x40000000L
+#define SPI_SHADER_PGM_RSRC1_VS__FP16_OVFL_MASK                                                               0x80000000L
+//SPI_SHADER_PGM_RSRC2_VS
+#define SPI_SHADER_PGM_RSRC2_VS__SCRATCH_EN__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR__SHIFT                                                             0x1
+#define SPI_SHADER_PGM_RSRC2_VS__TRAP_PRESENT__SHIFT                                                          0x6
+#define SPI_SHADER_PGM_RSRC2_VS__OC_LDS_EN__SHIFT                                                             0x7
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE0_EN__SHIFT                                                           0x8
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE1_EN__SHIFT                                                           0x9
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE2_EN__SHIFT                                                           0xa
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE3_EN__SHIFT                                                           0xb
+#define SPI_SHADER_PGM_RSRC2_VS__SO_EN__SHIFT                                                                 0xc
+#define SPI_SHADER_PGM_RSRC2_VS__EXCP_EN__SHIFT                                                               0xd
+#define SPI_SHADER_PGM_RSRC2_VS__PC_BASE_EN__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC2_VS__DISPATCH_DRAW_EN__SHIFT                                                      0x18
+#define SPI_SHADER_PGM_RSRC2_VS__SKIP_USGPR0__SHIFT                                                           0x1b
+#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR_MSB__SHIFT                                                         0x1c
+#define SPI_SHADER_PGM_RSRC2_VS__SCRATCH_EN_MASK                                                              0x00000001L
+#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR_MASK                                                               0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_VS__TRAP_PRESENT_MASK                                                            0x00000040L
+#define SPI_SHADER_PGM_RSRC2_VS__OC_LDS_EN_MASK                                                               0x00000080L
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE0_EN_MASK                                                             0x00000100L
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE1_EN_MASK                                                             0x00000200L
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE2_EN_MASK                                                             0x00000400L
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE3_EN_MASK                                                             0x00000800L
+#define SPI_SHADER_PGM_RSRC2_VS__SO_EN_MASK                                                                   0x00001000L
+#define SPI_SHADER_PGM_RSRC2_VS__EXCP_EN_MASK                                                                 0x003FE000L
+#define SPI_SHADER_PGM_RSRC2_VS__PC_BASE_EN_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC2_VS__DISPATCH_DRAW_EN_MASK                                                        0x01000000L
+#define SPI_SHADER_PGM_RSRC2_VS__SKIP_USGPR0_MASK                                                             0x08000000L
+#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR_MSB_MASK                                                           0x10000000L
+//SPI_SHADER_USER_DATA_VS_0
+#define SPI_SHADER_USER_DATA_VS_0__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_VS_0__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_1
+#define SPI_SHADER_USER_DATA_VS_1__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_VS_1__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_2
+#define SPI_SHADER_USER_DATA_VS_2__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_VS_2__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_3
+#define SPI_SHADER_USER_DATA_VS_3__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_VS_3__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_4
+#define SPI_SHADER_USER_DATA_VS_4__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_VS_4__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_5
+#define SPI_SHADER_USER_DATA_VS_5__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_VS_5__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_6
+#define SPI_SHADER_USER_DATA_VS_6__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_VS_6__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_7
+#define SPI_SHADER_USER_DATA_VS_7__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_VS_7__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_8
+#define SPI_SHADER_USER_DATA_VS_8__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_VS_8__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_9
+#define SPI_SHADER_USER_DATA_VS_9__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_VS_9__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_10
+#define SPI_SHADER_USER_DATA_VS_10__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_10__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_11
+#define SPI_SHADER_USER_DATA_VS_11__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_11__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_12
+#define SPI_SHADER_USER_DATA_VS_12__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_12__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_13
+#define SPI_SHADER_USER_DATA_VS_13__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_13__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_14
+#define SPI_SHADER_USER_DATA_VS_14__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_14__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_15
+#define SPI_SHADER_USER_DATA_VS_15__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_15__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_16
+#define SPI_SHADER_USER_DATA_VS_16__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_16__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_17
+#define SPI_SHADER_USER_DATA_VS_17__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_17__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_18
+#define SPI_SHADER_USER_DATA_VS_18__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_18__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_19
+#define SPI_SHADER_USER_DATA_VS_19__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_19__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_20
+#define SPI_SHADER_USER_DATA_VS_20__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_20__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_21
+#define SPI_SHADER_USER_DATA_VS_21__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_21__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_22
+#define SPI_SHADER_USER_DATA_VS_22__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_22__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_23
+#define SPI_SHADER_USER_DATA_VS_23__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_23__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_24
+#define SPI_SHADER_USER_DATA_VS_24__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_24__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_25
+#define SPI_SHADER_USER_DATA_VS_25__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_25__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_26
+#define SPI_SHADER_USER_DATA_VS_26__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_26__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_27
+#define SPI_SHADER_USER_DATA_VS_27__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_27__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_28
+#define SPI_SHADER_USER_DATA_VS_28__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_28__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_29
+#define SPI_SHADER_USER_DATA_VS_29__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_29__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_30
+#define SPI_SHADER_USER_DATA_VS_30__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_30__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_VS_31
+#define SPI_SHADER_USER_DATA_VS_31__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_VS_31__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC2_GS_VS
+#define SPI_SHADER_PGM_RSRC2_GS_VS__SCRATCH_EN__SHIFT                                                         0x0
+#define SPI_SHADER_PGM_RSRC2_GS_VS__USER_SGPR__SHIFT                                                          0x1
+#define SPI_SHADER_PGM_RSRC2_GS_VS__TRAP_PRESENT__SHIFT                                                       0x6
+#define SPI_SHADER_PGM_RSRC2_GS_VS__EXCP_EN__SHIFT                                                            0x7
+#define SPI_SHADER_PGM_RSRC2_GS_VS__VGPR_COMP_CNT__SHIFT                                                      0x10
+#define SPI_SHADER_PGM_RSRC2_GS_VS__OC_LDS_EN__SHIFT                                                          0x12
+#define SPI_SHADER_PGM_RSRC2_GS_VS__LDS_SIZE__SHIFT                                                           0x13
+#define SPI_SHADER_PGM_RSRC2_GS_VS__SKIP_USGPR0__SHIFT                                                        0x1b
+#define SPI_SHADER_PGM_RSRC2_GS_VS__USER_SGPR_MSB__SHIFT                                                      0x1c
+#define SPI_SHADER_PGM_RSRC2_GS_VS__SCRATCH_EN_MASK                                                           0x00000001L
+#define SPI_SHADER_PGM_RSRC2_GS_VS__USER_SGPR_MASK                                                            0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_GS_VS__TRAP_PRESENT_MASK                                                         0x00000040L
+#define SPI_SHADER_PGM_RSRC2_GS_VS__EXCP_EN_MASK                                                              0x0000FF80L
+#define SPI_SHADER_PGM_RSRC2_GS_VS__VGPR_COMP_CNT_MASK                                                        0x00030000L
+#define SPI_SHADER_PGM_RSRC2_GS_VS__OC_LDS_EN_MASK                                                            0x00040000L
+#define SPI_SHADER_PGM_RSRC2_GS_VS__LDS_SIZE_MASK                                                             0x07F80000L
+#define SPI_SHADER_PGM_RSRC2_GS_VS__SKIP_USGPR0_MASK                                                          0x08000000L
+#define SPI_SHADER_PGM_RSRC2_GS_VS__USER_SGPR_MSB_MASK                                                        0x10000000L
+//SPI_SHADER_PGM_RSRC4_GS
+#define SPI_SHADER_PGM_RSRC4_GS__GROUP_FIFO_DEPTH__SHIFT                                                      0x0
+#define SPI_SHADER_PGM_RSRC4_GS__SPI_SHADER_LATE_ALLOC_GS__SHIFT                                              0x7
+#define SPI_SHADER_PGM_RSRC4_GS__GROUP_FIFO_DEPTH_MASK                                                        0x0000007FL
+#define SPI_SHADER_PGM_RSRC4_GS__SPI_SHADER_LATE_ALLOC_GS_MASK                                                0x00003F80L
+//SPI_SHADER_USER_DATA_ADDR_LO_GS
+#define SPI_SHADER_USER_DATA_ADDR_LO_GS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_LO_GS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ADDR_HI_GS
+#define SPI_SHADER_USER_DATA_ADDR_HI_GS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_HI_GS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_PGM_LO_ES
+#define SPI_SHADER_PGM_LO_ES__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_ES__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_ES
+#define SPI_SHADER_PGM_HI_ES__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_ES__MEM_BASE_MASK                                                                   0xFFL
+//SPI_SHADER_PGM_RSRC3_GS
+#define SPI_SHADER_PGM_RSRC3_GS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC3_GS__WAVE_LIMIT__SHIFT                                                            0x10
+#define SPI_SHADER_PGM_RSRC3_GS__LOCK_LOW_THRESHOLD__SHIFT                                                    0x16
+#define SPI_SHADER_PGM_RSRC3_GS__SIMD_DISABLE__SHIFT                                                          0x1a
+#define SPI_SHADER_PGM_RSRC3_GS__CU_EN_MASK                                                                   0x0000FFFFL
+#define SPI_SHADER_PGM_RSRC3_GS__WAVE_LIMIT_MASK                                                              0x003F0000L
+#define SPI_SHADER_PGM_RSRC3_GS__LOCK_LOW_THRESHOLD_MASK                                                      0x03C00000L
+#define SPI_SHADER_PGM_RSRC3_GS__SIMD_DISABLE_MASK                                                            0x3C000000L
+//SPI_SHADER_PGM_LO_GS
+#define SPI_SHADER_PGM_LO_GS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_GS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_GS
+#define SPI_SHADER_PGM_HI_GS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_GS__MEM_BASE_MASK                                                                   0xFFL
+//SPI_SHADER_PGM_RSRC1_GS
+#define SPI_SHADER_PGM_RSRC1_GS__VGPRS__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC1_GS__SGPRS__SHIFT                                                                 0x6
+#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY__SHIFT                                                              0xa
+#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE__SHIFT                                                            0xc
+#define SPI_SHADER_PGM_RSRC1_GS__PRIV__SHIFT                                                                  0x14
+#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP__SHIFT                                                            0x15
+#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE__SHIFT                                                             0x17
+#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE__SHIFT                                                       0x18
+#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER__SHIFT                                                             0x1c
+#define SPI_SHADER_PGM_RSRC1_GS__GS_VGPR_COMP_CNT__SHIFT                                                      0x1d
+#define SPI_SHADER_PGM_RSRC1_GS__FP16_OVFL__SHIFT                                                             0x1f
+#define SPI_SHADER_PGM_RSRC1_GS__VGPRS_MASK                                                                   0x0000003FL
+#define SPI_SHADER_PGM_RSRC1_GS__SGPRS_MASK                                                                   0x000003C0L
+#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY_MASK                                                                0x00000C00L
+#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE_MASK                                                              0x000FF000L
+#define SPI_SHADER_PGM_RSRC1_GS__PRIV_MASK                                                                    0x00100000L
+#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP_MASK                                                              0x00200000L
+#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE_MASK                                                               0x00800000L
+#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE_MASK                                                         0x01000000L
+#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER_MASK                                                               0x10000000L
+#define SPI_SHADER_PGM_RSRC1_GS__GS_VGPR_COMP_CNT_MASK                                                        0x60000000L
+#define SPI_SHADER_PGM_RSRC1_GS__FP16_OVFL_MASK                                                               0x80000000L
+//SPI_SHADER_PGM_RSRC2_GS
+#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR__SHIFT                                                             0x1
+#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT__SHIFT                                                          0x6
+#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN__SHIFT                                                               0x7
+#define SPI_SHADER_PGM_RSRC2_GS__ES_VGPR_COMP_CNT__SHIFT                                                      0x10
+#define SPI_SHADER_PGM_RSRC2_GS__OC_LDS_EN__SHIFT                                                             0x12
+#define SPI_SHADER_PGM_RSRC2_GS__LDS_SIZE__SHIFT                                                              0x13
+#define SPI_SHADER_PGM_RSRC2_GS__SKIP_USGPR0__SHIFT                                                           0x1b
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MSB__SHIFT                                                         0x1c
+#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN_MASK                                                              0x00000001L
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MASK                                                               0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT_MASK                                                            0x00000040L
+#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN_MASK                                                                 0x0000FF80L
+#define SPI_SHADER_PGM_RSRC2_GS__ES_VGPR_COMP_CNT_MASK                                                        0x00030000L
+#define SPI_SHADER_PGM_RSRC2_GS__OC_LDS_EN_MASK                                                               0x00040000L
+#define SPI_SHADER_PGM_RSRC2_GS__LDS_SIZE_MASK                                                                0x07F80000L
+#define SPI_SHADER_PGM_RSRC2_GS__SKIP_USGPR0_MASK                                                             0x08000000L
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MSB_MASK                                                           0x10000000L
+//SPI_SHADER_USER_DATA_ES_0
+#define SPI_SHADER_USER_DATA_ES_0__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_ES_0__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_1
+#define SPI_SHADER_USER_DATA_ES_1__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_ES_1__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_2
+#define SPI_SHADER_USER_DATA_ES_2__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_ES_2__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_3
+#define SPI_SHADER_USER_DATA_ES_3__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_ES_3__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_4
+#define SPI_SHADER_USER_DATA_ES_4__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_ES_4__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_5
+#define SPI_SHADER_USER_DATA_ES_5__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_ES_5__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_6
+#define SPI_SHADER_USER_DATA_ES_6__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_ES_6__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_7
+#define SPI_SHADER_USER_DATA_ES_7__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_ES_7__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_8
+#define SPI_SHADER_USER_DATA_ES_8__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_ES_8__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_9
+#define SPI_SHADER_USER_DATA_ES_9__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_ES_9__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_10
+#define SPI_SHADER_USER_DATA_ES_10__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_10__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_11
+#define SPI_SHADER_USER_DATA_ES_11__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_11__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_12
+#define SPI_SHADER_USER_DATA_ES_12__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_12__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_13
+#define SPI_SHADER_USER_DATA_ES_13__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_13__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_14
+#define SPI_SHADER_USER_DATA_ES_14__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_14__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_15
+#define SPI_SHADER_USER_DATA_ES_15__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_15__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_16
+#define SPI_SHADER_USER_DATA_ES_16__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_16__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_17
+#define SPI_SHADER_USER_DATA_ES_17__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_17__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_18
+#define SPI_SHADER_USER_DATA_ES_18__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_18__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_19
+#define SPI_SHADER_USER_DATA_ES_19__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_19__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_20
+#define SPI_SHADER_USER_DATA_ES_20__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_20__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_21
+#define SPI_SHADER_USER_DATA_ES_21__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_21__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_22
+#define SPI_SHADER_USER_DATA_ES_22__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_22__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_23
+#define SPI_SHADER_USER_DATA_ES_23__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_23__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_24
+#define SPI_SHADER_USER_DATA_ES_24__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_24__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_25
+#define SPI_SHADER_USER_DATA_ES_25__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_25__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_26
+#define SPI_SHADER_USER_DATA_ES_26__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_26__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_27
+#define SPI_SHADER_USER_DATA_ES_27__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_27__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_28
+#define SPI_SHADER_USER_DATA_ES_28__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_28__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_29
+#define SPI_SHADER_USER_DATA_ES_29__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_29__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_30
+#define SPI_SHADER_USER_DATA_ES_30__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_30__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ES_31
+#define SPI_SHADER_USER_DATA_ES_31__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_ES_31__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC4_HS
+#define SPI_SHADER_PGM_RSRC4_HS__GROUP_FIFO_DEPTH__SHIFT                                                      0x0
+#define SPI_SHADER_PGM_RSRC4_HS__GROUP_FIFO_DEPTH_MASK                                                        0x0000007FL
+//SPI_SHADER_USER_DATA_ADDR_LO_HS
+#define SPI_SHADER_USER_DATA_ADDR_LO_HS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_LO_HS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ADDR_HI_HS
+#define SPI_SHADER_USER_DATA_ADDR_HI_HS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_HI_HS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_PGM_LO_LS
+#define SPI_SHADER_PGM_LO_LS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_LS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_LS
+#define SPI_SHADER_PGM_HI_LS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_LS__MEM_BASE_MASK                                                                   0xFFL
+//SPI_SHADER_PGM_RSRC3_HS
+#define SPI_SHADER_PGM_RSRC3_HS__WAVE_LIMIT__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC3_HS__LOCK_LOW_THRESHOLD__SHIFT                                                    0x6
+#define SPI_SHADER_PGM_RSRC3_HS__SIMD_DISABLE__SHIFT                                                          0xa
+#define SPI_SHADER_PGM_RSRC3_HS__CU_EN__SHIFT                                                                 0x10
+#define SPI_SHADER_PGM_RSRC3_HS__WAVE_LIMIT_MASK                                                              0x0000003FL
+#define SPI_SHADER_PGM_RSRC3_HS__LOCK_LOW_THRESHOLD_MASK                                                      0x000003C0L
+#define SPI_SHADER_PGM_RSRC3_HS__SIMD_DISABLE_MASK                                                            0x00003C00L
+#define SPI_SHADER_PGM_RSRC3_HS__CU_EN_MASK                                                                   0xFFFF0000L
+//SPI_SHADER_PGM_LO_HS
+#define SPI_SHADER_PGM_LO_HS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_HS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_HS
+#define SPI_SHADER_PGM_HI_HS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_HS__MEM_BASE_MASK                                                                   0xFFL
+//SPI_SHADER_PGM_RSRC1_HS
+#define SPI_SHADER_PGM_RSRC1_HS__VGPRS__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC1_HS__SGPRS__SHIFT                                                                 0x6
+#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY__SHIFT                                                              0xa
+#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE__SHIFT                                                            0xc
+#define SPI_SHADER_PGM_RSRC1_HS__PRIV__SHIFT                                                                  0x14
+#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP__SHIFT                                                            0x15
+#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE__SHIFT                                                             0x17
+#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER__SHIFT                                                             0x1b
+#define SPI_SHADER_PGM_RSRC1_HS__LS_VGPR_COMP_CNT__SHIFT                                                      0x1c
+#define SPI_SHADER_PGM_RSRC1_HS__FP16_OVFL__SHIFT                                                             0x1e
+#define SPI_SHADER_PGM_RSRC1_HS__VGPRS_MASK                                                                   0x0000003FL
+#define SPI_SHADER_PGM_RSRC1_HS__SGPRS_MASK                                                                   0x000003C0L
+#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY_MASK                                                                0x00000C00L
+#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE_MASK                                                              0x000FF000L
+#define SPI_SHADER_PGM_RSRC1_HS__PRIV_MASK                                                                    0x00100000L
+#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP_MASK                                                              0x00200000L
+#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE_MASK                                                               0x00800000L
+#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER_MASK                                                               0x08000000L
+#define SPI_SHADER_PGM_RSRC1_HS__LS_VGPR_COMP_CNT_MASK                                                        0x30000000L
+#define SPI_SHADER_PGM_RSRC1_HS__FP16_OVFL_MASK                                                               0x40000000L
+//SPI_SHADER_PGM_RSRC2_HS
+#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR__SHIFT                                                             0x1
+#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT__SHIFT                                                          0x6
+#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN__SHIFT                                                               0x7
+#define SPI_SHADER_PGM_RSRC2_HS__LDS_SIZE__SHIFT                                                              0x10
+#define SPI_SHADER_PGM_RSRC2_HS__SKIP_USGPR0__SHIFT                                                           0x1b
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MSB__SHIFT                                                         0x1c
+#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN_MASK                                                              0x00000001L
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MASK                                                               0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT_MASK                                                            0x00000040L
+#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN_MASK                                                                 0x0000FF80L
+#define SPI_SHADER_PGM_RSRC2_HS__LDS_SIZE_MASK                                                                0x01FF0000L
+#define SPI_SHADER_PGM_RSRC2_HS__SKIP_USGPR0_MASK                                                             0x08000000L
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MSB_MASK                                                           0x10000000L
+//SPI_SHADER_USER_DATA_LS_0
+#define SPI_SHADER_USER_DATA_LS_0__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_LS_0__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_1
+#define SPI_SHADER_USER_DATA_LS_1__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_LS_1__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_2
+#define SPI_SHADER_USER_DATA_LS_2__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_LS_2__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_3
+#define SPI_SHADER_USER_DATA_LS_3__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_LS_3__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_4
+#define SPI_SHADER_USER_DATA_LS_4__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_LS_4__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_5
+#define SPI_SHADER_USER_DATA_LS_5__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_LS_5__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_6
+#define SPI_SHADER_USER_DATA_LS_6__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_LS_6__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_7
+#define SPI_SHADER_USER_DATA_LS_7__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_LS_7__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_8
+#define SPI_SHADER_USER_DATA_LS_8__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_LS_8__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_9
+#define SPI_SHADER_USER_DATA_LS_9__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_LS_9__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_10
+#define SPI_SHADER_USER_DATA_LS_10__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_10__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_11
+#define SPI_SHADER_USER_DATA_LS_11__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_11__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_12
+#define SPI_SHADER_USER_DATA_LS_12__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_12__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_13
+#define SPI_SHADER_USER_DATA_LS_13__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_13__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_14
+#define SPI_SHADER_USER_DATA_LS_14__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_14__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_15
+#define SPI_SHADER_USER_DATA_LS_15__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_15__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_16
+#define SPI_SHADER_USER_DATA_LS_16__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_16__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_17
+#define SPI_SHADER_USER_DATA_LS_17__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_17__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_18
+#define SPI_SHADER_USER_DATA_LS_18__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_18__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_19
+#define SPI_SHADER_USER_DATA_LS_19__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_19__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_20
+#define SPI_SHADER_USER_DATA_LS_20__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_20__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_21
+#define SPI_SHADER_USER_DATA_LS_21__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_21__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_22
+#define SPI_SHADER_USER_DATA_LS_22__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_22__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_23
+#define SPI_SHADER_USER_DATA_LS_23__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_23__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_24
+#define SPI_SHADER_USER_DATA_LS_24__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_24__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_25
+#define SPI_SHADER_USER_DATA_LS_25__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_25__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_26
+#define SPI_SHADER_USER_DATA_LS_26__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_26__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_27
+#define SPI_SHADER_USER_DATA_LS_27__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_27__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_28
+#define SPI_SHADER_USER_DATA_LS_28__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_28__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_29
+#define SPI_SHADER_USER_DATA_LS_29__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_29__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_30
+#define SPI_SHADER_USER_DATA_LS_30__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_30__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_LS_31
+#define SPI_SHADER_USER_DATA_LS_31__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_LS_31__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_0
+#define SPI_SHADER_USER_DATA_COMMON_0__DATA__SHIFT                                                            0x0
+#define SPI_SHADER_USER_DATA_COMMON_0__DATA_MASK                                                              0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_1
+#define SPI_SHADER_USER_DATA_COMMON_1__DATA__SHIFT                                                            0x0
+#define SPI_SHADER_USER_DATA_COMMON_1__DATA_MASK                                                              0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_2
+#define SPI_SHADER_USER_DATA_COMMON_2__DATA__SHIFT                                                            0x0
+#define SPI_SHADER_USER_DATA_COMMON_2__DATA_MASK                                                              0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_3
+#define SPI_SHADER_USER_DATA_COMMON_3__DATA__SHIFT                                                            0x0
+#define SPI_SHADER_USER_DATA_COMMON_3__DATA_MASK                                                              0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_4
+#define SPI_SHADER_USER_DATA_COMMON_4__DATA__SHIFT                                                            0x0
+#define SPI_SHADER_USER_DATA_COMMON_4__DATA_MASK                                                              0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_5
+#define SPI_SHADER_USER_DATA_COMMON_5__DATA__SHIFT                                                            0x0
+#define SPI_SHADER_USER_DATA_COMMON_5__DATA_MASK                                                              0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_6
+#define SPI_SHADER_USER_DATA_COMMON_6__DATA__SHIFT                                                            0x0
+#define SPI_SHADER_USER_DATA_COMMON_6__DATA_MASK                                                              0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_7
+#define SPI_SHADER_USER_DATA_COMMON_7__DATA__SHIFT                                                            0x0
+#define SPI_SHADER_USER_DATA_COMMON_7__DATA_MASK                                                              0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_8
+#define SPI_SHADER_USER_DATA_COMMON_8__DATA__SHIFT                                                            0x0
+#define SPI_SHADER_USER_DATA_COMMON_8__DATA_MASK                                                              0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_9
+#define SPI_SHADER_USER_DATA_COMMON_9__DATA__SHIFT                                                            0x0
+#define SPI_SHADER_USER_DATA_COMMON_9__DATA_MASK                                                              0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_10
+#define SPI_SHADER_USER_DATA_COMMON_10__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_10__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_11
+#define SPI_SHADER_USER_DATA_COMMON_11__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_11__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_12
+#define SPI_SHADER_USER_DATA_COMMON_12__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_12__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_13
+#define SPI_SHADER_USER_DATA_COMMON_13__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_13__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_14
+#define SPI_SHADER_USER_DATA_COMMON_14__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_14__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_15
+#define SPI_SHADER_USER_DATA_COMMON_15__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_15__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_16
+#define SPI_SHADER_USER_DATA_COMMON_16__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_16__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_17
+#define SPI_SHADER_USER_DATA_COMMON_17__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_17__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_18
+#define SPI_SHADER_USER_DATA_COMMON_18__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_18__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_19
+#define SPI_SHADER_USER_DATA_COMMON_19__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_19__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_20
+#define SPI_SHADER_USER_DATA_COMMON_20__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_20__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_21
+#define SPI_SHADER_USER_DATA_COMMON_21__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_21__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_22
+#define SPI_SHADER_USER_DATA_COMMON_22__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_22__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_23
+#define SPI_SHADER_USER_DATA_COMMON_23__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_23__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_24
+#define SPI_SHADER_USER_DATA_COMMON_24__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_24__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_25
+#define SPI_SHADER_USER_DATA_COMMON_25__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_25__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_26
+#define SPI_SHADER_USER_DATA_COMMON_26__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_26__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_27
+#define SPI_SHADER_USER_DATA_COMMON_27__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_27__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_28
+#define SPI_SHADER_USER_DATA_COMMON_28__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_28__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_29
+#define SPI_SHADER_USER_DATA_COMMON_29__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_29__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_30
+#define SPI_SHADER_USER_DATA_COMMON_30__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_30__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_COMMON_31
+#define SPI_SHADER_USER_DATA_COMMON_31__DATA__SHIFT                                                           0x0
+#define SPI_SHADER_USER_DATA_COMMON_31__DATA_MASK                                                             0xFFFFFFFFL
+//COMPUTE_DISPATCH_INITIATOR
+#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN__SHIFT                                                  0x0
+#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN__SHIFT                                                      0x1
+#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000__SHIFT                                                 0x2
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL__SHIFT                                                0x3
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE__SHIFT                                                0x4
+#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS__SHIFT                                              0x5
+#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE__SHIFT                                                         0x6
+#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL__SHIFT                                                  0xa
+#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL__SHIFT                                                  0xb
+#define COMPUTE_DISPATCH_INITIATOR__RESERVED__SHIFT                                                           0xc
+#define COMPUTE_DISPATCH_INITIATOR__RESTORE__SHIFT                                                            0xe
+#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN_MASK                                                    0x00000001L
+#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN_MASK                                                        0x00000002L
+#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000_MASK                                                   0x00000004L
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL_MASK                                                  0x00000008L
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE_MASK                                                  0x00000010L
+#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS_MASK                                                0x00000020L
+#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE_MASK                                                           0x00000040L
+#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL_MASK                                                    0x00000400L
+#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL_MASK                                                    0x00000800L
+#define COMPUTE_DISPATCH_INITIATOR__RESERVED_MASK                                                             0x00001000L
+#define COMPUTE_DISPATCH_INITIATOR__RESTORE_MASK                                                              0x00004000L
+//COMPUTE_DIM_X
+#define COMPUTE_DIM_X__SIZE__SHIFT                                                                            0x0
+#define COMPUTE_DIM_X__SIZE_MASK                                                                              0xFFFFFFFFL
+//COMPUTE_DIM_Y
+#define COMPUTE_DIM_Y__SIZE__SHIFT                                                                            0x0
+#define COMPUTE_DIM_Y__SIZE_MASK                                                                              0xFFFFFFFFL
+//COMPUTE_DIM_Z
+#define COMPUTE_DIM_Z__SIZE__SHIFT                                                                            0x0
+#define COMPUTE_DIM_Z__SIZE_MASK                                                                              0xFFFFFFFFL
+//COMPUTE_START_X
+#define COMPUTE_START_X__START__SHIFT                                                                         0x0
+#define COMPUTE_START_X__START_MASK                                                                           0xFFFFFFFFL
+//COMPUTE_START_Y
+#define COMPUTE_START_Y__START__SHIFT                                                                         0x0
+#define COMPUTE_START_Y__START_MASK                                                                           0xFFFFFFFFL
+//COMPUTE_START_Z
+#define COMPUTE_START_Z__START__SHIFT                                                                         0x0
+#define COMPUTE_START_Z__START_MASK                                                                           0xFFFFFFFFL
+//COMPUTE_NUM_THREAD_X
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL__SHIFT                                                          0x0
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL_MASK                                                            0x0000FFFFL
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
+//COMPUTE_NUM_THREAD_Y
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL__SHIFT                                                          0x0
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL_MASK                                                            0x0000FFFFL
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
+//COMPUTE_NUM_THREAD_Z
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL__SHIFT                                                          0x0
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL_MASK                                                            0x0000FFFFL
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
+//COMPUTE_PIPELINESTAT_ENABLE
+#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE__SHIFT                                               0x0
+#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE_MASK                                                 0x00000001L
+//COMPUTE_PERFCOUNT_ENABLE
+#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE__SHIFT                                                     0x0
+#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE_MASK                                                       0x00000001L
+//COMPUTE_PGM_LO
+#define COMPUTE_PGM_LO__DATA__SHIFT                                                                           0x0
+#define COMPUTE_PGM_LO__DATA_MASK                                                                             0xFFFFFFFFL
+//COMPUTE_PGM_HI
+#define COMPUTE_PGM_HI__DATA__SHIFT                                                                           0x0
+#define COMPUTE_PGM_HI__DATA_MASK                                                                             0x000000FFL
+//COMPUTE_DISPATCH_PKT_ADDR_LO
+#define COMPUTE_DISPATCH_PKT_ADDR_LO__DATA__SHIFT                                                             0x0
+#define COMPUTE_DISPATCH_PKT_ADDR_LO__DATA_MASK                                                               0xFFFFFFFFL
+//COMPUTE_DISPATCH_PKT_ADDR_HI
+#define COMPUTE_DISPATCH_PKT_ADDR_HI__DATA__SHIFT                                                             0x0
+#define COMPUTE_DISPATCH_PKT_ADDR_HI__DATA_MASK                                                               0x000000FFL
+//COMPUTE_DISPATCH_SCRATCH_BASE_LO
+#define COMPUTE_DISPATCH_SCRATCH_BASE_LO__DATA__SHIFT                                                         0x0
+#define COMPUTE_DISPATCH_SCRATCH_BASE_LO__DATA_MASK                                                           0xFFFFFFFFL
+//COMPUTE_DISPATCH_SCRATCH_BASE_HI
+#define COMPUTE_DISPATCH_SCRATCH_BASE_HI__DATA__SHIFT                                                         0x0
+#define COMPUTE_DISPATCH_SCRATCH_BASE_HI__DATA_MASK                                                           0x000000FFL
+//COMPUTE_PGM_RSRC1
+#define COMPUTE_PGM_RSRC1__VGPRS__SHIFT                                                                       0x0
+#define COMPUTE_PGM_RSRC1__SGPRS__SHIFT                                                                       0x6
+#define COMPUTE_PGM_RSRC1__PRIORITY__SHIFT                                                                    0xa
+#define COMPUTE_PGM_RSRC1__FLOAT_MODE__SHIFT                                                                  0xc
+#define COMPUTE_PGM_RSRC1__PRIV__SHIFT                                                                        0x14
+#define COMPUTE_PGM_RSRC1__DX10_CLAMP__SHIFT                                                                  0x15
+#define COMPUTE_PGM_RSRC1__DEBUG_MODE__SHIFT                                                                  0x16
+#define COMPUTE_PGM_RSRC1__IEEE_MODE__SHIFT                                                                   0x17
+#define COMPUTE_PGM_RSRC1__BULKY__SHIFT                                                                       0x18
+#define COMPUTE_PGM_RSRC1__CDBG_USER__SHIFT                                                                   0x19
+#define COMPUTE_PGM_RSRC1__FP16_OVFL__SHIFT                                                                   0x1a
+#define COMPUTE_PGM_RSRC1__VGPRS_MASK                                                                         0x0000003FL
+#define COMPUTE_PGM_RSRC1__SGPRS_MASK                                                                         0x000003C0L
+#define COMPUTE_PGM_RSRC1__PRIORITY_MASK                                                                      0x00000C00L
+#define COMPUTE_PGM_RSRC1__FLOAT_MODE_MASK                                                                    0x000FF000L
+#define COMPUTE_PGM_RSRC1__PRIV_MASK                                                                          0x00100000L
+#define COMPUTE_PGM_RSRC1__DX10_CLAMP_MASK                                                                    0x00200000L
+#define COMPUTE_PGM_RSRC1__DEBUG_MODE_MASK                                                                    0x00400000L
+#define COMPUTE_PGM_RSRC1__IEEE_MODE_MASK                                                                     0x00800000L
+#define COMPUTE_PGM_RSRC1__BULKY_MASK                                                                         0x01000000L
+#define COMPUTE_PGM_RSRC1__CDBG_USER_MASK                                                                     0x02000000L
+#define COMPUTE_PGM_RSRC1__FP16_OVFL_MASK                                                                     0x04000000L
+//COMPUTE_PGM_RSRC2
+#define COMPUTE_PGM_RSRC2__SCRATCH_EN__SHIFT                                                                  0x0
+#define COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT                                                                   0x1
+#define COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT                                                                0x6
+#define COMPUTE_PGM_RSRC2__TGID_X_EN__SHIFT                                                                   0x7
+#define COMPUTE_PGM_RSRC2__TGID_Y_EN__SHIFT                                                                   0x8
+#define COMPUTE_PGM_RSRC2__TGID_Z_EN__SHIFT                                                                   0x9
+#define COMPUTE_PGM_RSRC2__TG_SIZE_EN__SHIFT                                                                  0xa
+#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT__SHIFT                                                              0xb
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT                                                                 0xd
+#define COMPUTE_PGM_RSRC2__LDS_SIZE__SHIFT                                                                    0xf
+#define COMPUTE_PGM_RSRC2__EXCP_EN__SHIFT                                                                     0x18
+#define COMPUTE_PGM_RSRC2__SKIP_USGPR0__SHIFT                                                                 0x1f
+#define COMPUTE_PGM_RSRC2__SCRATCH_EN_MASK                                                                    0x00000001L
+#define COMPUTE_PGM_RSRC2__USER_SGPR_MASK                                                                     0x0000003EL
+#define COMPUTE_PGM_RSRC2__TRAP_PRESENT_MASK                                                                  0x00000040L
+#define COMPUTE_PGM_RSRC2__TGID_X_EN_MASK                                                                     0x00000080L
+#define COMPUTE_PGM_RSRC2__TGID_Y_EN_MASK                                                                     0x00000100L
+#define COMPUTE_PGM_RSRC2__TGID_Z_EN_MASK                                                                     0x00000200L
+#define COMPUTE_PGM_RSRC2__TG_SIZE_EN_MASK                                                                    0x00000400L
+#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT_MASK                                                                0x00001800L
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK                                                                   0x00006000L
+#define COMPUTE_PGM_RSRC2__LDS_SIZE_MASK                                                                      0x00FF8000L
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MASK                                                                       0x7F000000L
+#define COMPUTE_PGM_RSRC2__SKIP_USGPR0_MASK                                                                   0x80000000L
+//COMPUTE_VMID
+#define COMPUTE_VMID__DATA__SHIFT                                                                             0x0
+#define COMPUTE_VMID__DATA_MASK                                                                               0x0000000FL
+//COMPUTE_RESOURCE_LIMITS
+#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH__SHIFT                                                          0x0
+#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU__SHIFT                                                             0xc
+#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD__SHIFT                                                        0x10
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL__SHIFT                                                        0x16
+#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST__SHIFT                                                       0x17
+#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT__SHIFT                                                        0x18
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DISABLE__SHIFT                                                          0x1b
+#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH_MASK                                                            0x000003FFL
+#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU_MASK                                                               0x0000F000L
+#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD_MASK                                                          0x003F0000L
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL_MASK                                                          0x00400000L
+#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK                                                         0x00800000L
+#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT_MASK                                                          0x07000000L
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DISABLE_MASK                                                            0x78000000L
+//COMPUTE_STATIC_THREAD_MGMT_SE0
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_STATIC_THREAD_MGMT_SE1
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_TMPRING_SIZE
+#define COMPUTE_TMPRING_SIZE__WAVES__SHIFT                                                                    0x0
+#define COMPUTE_TMPRING_SIZE__WAVESIZE__SHIFT                                                                 0xc
+#define COMPUTE_TMPRING_SIZE__WAVES_MASK                                                                      0x00000FFFL
+#define COMPUTE_TMPRING_SIZE__WAVESIZE_MASK                                                                   0x01FFF000L
+//COMPUTE_STATIC_THREAD_MGMT_SE2
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_STATIC_THREAD_MGMT_SE3
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_RESTART_X
+#define COMPUTE_RESTART_X__RESTART__SHIFT                                                                     0x0
+#define COMPUTE_RESTART_X__RESTART_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_RESTART_Y
+#define COMPUTE_RESTART_Y__RESTART__SHIFT                                                                     0x0
+#define COMPUTE_RESTART_Y__RESTART_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_RESTART_Z
+#define COMPUTE_RESTART_Z__RESTART__SHIFT                                                                     0x0
+#define COMPUTE_RESTART_Z__RESTART_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_THREAD_TRACE_ENABLE
+#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE__SHIFT                                               0x0
+#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE_MASK                                                 0x00000001L
+//COMPUTE_MISC_RESERVED
+#define COMPUTE_MISC_RESERVED__SEND_SEID__SHIFT                                                               0x0
+#define COMPUTE_MISC_RESERVED__WAVE_ID_BASE__SHIFT                                                            0x5
+#define COMPUTE_MISC_RESERVED__SEND_SEID_MASK                                                                 0x00000003L
+#define COMPUTE_MISC_RESERVED__WAVE_ID_BASE_MASK                                                              0x0001FFE0L
+//COMPUTE_DISPATCH_ID
+#define COMPUTE_DISPATCH_ID__DISPATCH_ID__SHIFT                                                               0x0
+#define COMPUTE_DISPATCH_ID__DISPATCH_ID_MASK                                                                 0xFFFFFFFFL
+//COMPUTE_THREADGROUP_ID
+#define COMPUTE_THREADGROUP_ID__THREADGROUP_ID__SHIFT                                                         0x0
+#define COMPUTE_THREADGROUP_ID__THREADGROUP_ID_MASK                                                           0xFFFFFFFFL
+//COMPUTE_RELAUNCH
+#define COMPUTE_RELAUNCH__PAYLOAD__SHIFT                                                                      0x0
+#define COMPUTE_RELAUNCH__IS_EVENT__SHIFT                                                                     0x1e
+#define COMPUTE_RELAUNCH__IS_STATE__SHIFT                                                                     0x1f
+#define COMPUTE_RELAUNCH__PAYLOAD_MASK                                                                        0x3FFFFFFFL
+#define COMPUTE_RELAUNCH__IS_EVENT_MASK                                                                       0x40000000L
+#define COMPUTE_RELAUNCH__IS_STATE_MASK                                                                       0x80000000L
+//COMPUTE_WAVE_RESTORE_ADDR_LO
+#define COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR__SHIFT                                                             0x0
+#define COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFFL
+//COMPUTE_WAVE_RESTORE_ADDR_HI
+#define COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR_MASK                                                               0xFFFFL
+//COMPUTE_TG_CHUNK_SIZE
+#define COMPUTE_TG_CHUNK_SIZE__TG_CHUNK_SIZE__SHIFT                                                           0x0
+#define COMPUTE_TG_CHUNK_SIZE__TG_CHUNK_SIZE_MASK                                                             0x0000FFFFL
+//COMPUTE_SHADER_CHKSUM
+#define COMPUTE_SHADER_CHKSUM__CHECKSUM__SHIFT                                                                0x0
+#define COMPUTE_SHADER_CHKSUM__CHECKSUM_MASK                                                                  0xFFFFFFFFL
+//COMPUTE_PGM_RSRC3
+#define COMPUTE_PGM_RSRC3__ACCUM_OFFSET__SHIFT                                                                0x0
+#define COMPUTE_PGM_RSRC3__TRAP_ON_START__SHIFT                                                               0xa
+#define COMPUTE_PGM_RSRC3__TRAP_ON_END__SHIFT                                                                 0xb
+#define COMPUTE_PGM_RSRC3__TG_SPLIT__SHIFT                                                                    0x10
+#define COMPUTE_PGM_RSRC3__ACCUM_OFFSET_MASK                                                                  0x0000003FL
+#define COMPUTE_PGM_RSRC3__TRAP_ON_START_MASK                                                                 0x00000400L
+#define COMPUTE_PGM_RSRC3__TRAP_ON_END_MASK                                                                   0x00000800L
+#define COMPUTE_PGM_RSRC3__TG_SPLIT_MASK                                                                      0x00010000L
+//COMPUTE_USER_DATA_0
+#define COMPUTE_USER_DATA_0__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_0__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_1
+#define COMPUTE_USER_DATA_1__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_1__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_2
+#define COMPUTE_USER_DATA_2__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_2__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_3
+#define COMPUTE_USER_DATA_3__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_3__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_4
+#define COMPUTE_USER_DATA_4__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_4__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_5
+#define COMPUTE_USER_DATA_5__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_5__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_6
+#define COMPUTE_USER_DATA_6__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_6__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_7
+#define COMPUTE_USER_DATA_7__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_7__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_8
+#define COMPUTE_USER_DATA_8__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_8__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_9
+#define COMPUTE_USER_DATA_9__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_9__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_10
+#define COMPUTE_USER_DATA_10__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_10__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_11
+#define COMPUTE_USER_DATA_11__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_11__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_12
+#define COMPUTE_USER_DATA_12__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_12__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_13
+#define COMPUTE_USER_DATA_13__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_13__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_14
+#define COMPUTE_USER_DATA_14__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_14__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_15
+#define COMPUTE_USER_DATA_15__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_15__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_DISPATCH_END
+#define COMPUTE_DISPATCH_END__DATA__SHIFT                                                                     0x0
+#define COMPUTE_DISPATCH_END__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_NOWHERE
+#define COMPUTE_NOWHERE__DATA__SHIFT                                                                          0x0
+#define COMPUTE_NOWHERE__DATA_MASK                                                                            0xFFFFFFFFL
+
+
+// addressBlock: xcd0_gc_cppdec
+//CP_DFY_CNTL
+#define CP_DFY_CNTL__POLICY__SHIFT                                                                            0x0
+#define CP_DFY_CNTL__MTYPE__SHIFT                                                                             0x2
+#define CP_DFY_CNTL__WRITE_DIS__SHIFT                                                                         0x1b
+#define CP_DFY_CNTL__LFSR_RESET__SHIFT                                                                        0x1c
+#define CP_DFY_CNTL__MODE__SHIFT                                                                              0x1d
+#define CP_DFY_CNTL__ENABLE__SHIFT                                                                            0x1f
+#define CP_DFY_CNTL__POLICY_MASK                                                                              0x00000001L
+#define CP_DFY_CNTL__MTYPE_MASK                                                                               0x0000000CL
+#define CP_DFY_CNTL__WRITE_DIS_MASK                                                                           0x08000000L
+#define CP_DFY_CNTL__LFSR_RESET_MASK                                                                          0x10000000L
+#define CP_DFY_CNTL__MODE_MASK                                                                                0x60000000L
+#define CP_DFY_CNTL__ENABLE_MASK                                                                              0x80000000L
+//CP_DFY_STAT
+#define CP_DFY_STAT__BURST_COUNT__SHIFT                                                                       0x0
+#define CP_DFY_STAT__TAGS_PENDING__SHIFT                                                                      0x10
+#define CP_DFY_STAT__BUSY__SHIFT                                                                              0x1f
+#define CP_DFY_STAT__BURST_COUNT_MASK                                                                         0x0000FFFFL
+#define CP_DFY_STAT__TAGS_PENDING_MASK                                                                        0x07FF0000L
+#define CP_DFY_STAT__BUSY_MASK                                                                                0x80000000L
+//CP_DFY_ADDR_HI
+#define CP_DFY_ADDR_HI__ADDR_HI__SHIFT                                                                        0x0
+#define CP_DFY_ADDR_HI__ADDR_HI_MASK                                                                          0xFFFFFFFFL
+//CP_DFY_ADDR_LO
+#define CP_DFY_ADDR_LO__ADDR_LO__SHIFT                                                                        0x5
+#define CP_DFY_ADDR_LO__ADDR_LO_MASK                                                                          0xFFFFFFE0L
+//CP_DFY_DATA_0
+#define CP_DFY_DATA_0__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_0__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_1
+#define CP_DFY_DATA_1__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_1__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_2
+#define CP_DFY_DATA_2__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_2__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_3
+#define CP_DFY_DATA_3__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_3__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_4
+#define CP_DFY_DATA_4__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_4__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_5
+#define CP_DFY_DATA_5__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_5__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_6
+#define CP_DFY_DATA_6__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_6__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_7
+#define CP_DFY_DATA_7__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_7__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_8
+#define CP_DFY_DATA_8__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_8__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_9
+#define CP_DFY_DATA_9__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_9__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_10
+#define CP_DFY_DATA_10__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_10__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_11
+#define CP_DFY_DATA_11__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_11__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_12
+#define CP_DFY_DATA_12__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_12__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_13
+#define CP_DFY_DATA_13__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_13__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_14
+#define CP_DFY_DATA_14__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_14__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_15
+#define CP_DFY_DATA_15__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_15__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_CMD
+#define CP_DFY_CMD__OFFSET__SHIFT                                                                             0x0
+#define CP_DFY_CMD__SIZE__SHIFT                                                                               0x10
+#define CP_DFY_CMD__OFFSET_MASK                                                                               0x000001FFL
+#define CP_DFY_CMD__SIZE_MASK                                                                                 0xFFFF0000L
+//CP_EOPQ_WAIT_TIME
+#define CP_EOPQ_WAIT_TIME__WAIT_TIME__SHIFT                                                                   0x0
+#define CP_EOPQ_WAIT_TIME__SCALE_COUNT__SHIFT                                                                 0xa
+#define CP_EOPQ_WAIT_TIME__WAIT_TIME_MASK                                                                     0x000003FFL
+#define CP_EOPQ_WAIT_TIME__SCALE_COUNT_MASK                                                                   0x0003FC00L
+//CP_CPC_MGCG_SYNC_CNTL
+#define CP_CPC_MGCG_SYNC_CNTL__COOLDOWN_PERIOD__SHIFT                                                         0x0
+#define CP_CPC_MGCG_SYNC_CNTL__WARMUP_PERIOD__SHIFT                                                           0x8
+#define CP_CPC_MGCG_SYNC_CNTL__COOLDOWN_PERIOD_MASK                                                           0x000000FFL
+#define CP_CPC_MGCG_SYNC_CNTL__WARMUP_PERIOD_MASK                                                             0x0000FF00L
+//CPC_INT_INFO
+#define CPC_INT_INFO__ADDR_HI__SHIFT                                                                          0x0
+#define CPC_INT_INFO__TYPE__SHIFT                                                                             0x10
+#define CPC_INT_INFO__VMID__SHIFT                                                                             0x14
+#define CPC_INT_INFO__QUEUE_ID__SHIFT                                                                         0x1c
+#define CPC_INT_INFO__ADDR_HI_MASK                                                                            0x0000FFFFL
+#define CPC_INT_INFO__TYPE_MASK                                                                               0x00010000L
+#define CPC_INT_INFO__VMID_MASK                                                                               0x00F00000L
+#define CPC_INT_INFO__QUEUE_ID_MASK                                                                           0x70000000L
+//CP_VIRT_STATUS
+#define CP_VIRT_STATUS__VIRT_STATUS__SHIFT                                                                    0x0
+#define CP_VIRT_STATUS__VIRT_STATUS_MASK                                                                      0xFFFFFFFFL
+//CPC_INT_ADDR
+#define CPC_INT_ADDR__ADDR__SHIFT                                                                             0x0
+#define CPC_INT_ADDR__ADDR_MASK                                                                               0xFFFFFFFFL
+//CPC_INT_PASID
+#define CPC_INT_PASID__PASID__SHIFT                                                                           0x0
+#define CPC_INT_PASID__PASID_MASK                                                                             0x0000FFFFL
+//CP_GFX_ERROR
+#define CP_GFX_ERROR__EDC_ERROR_ID__SHIFT                                                                     0x0
+#define CP_GFX_ERROR__SUA_ERROR__SHIFT                                                                        0x4
+#define CP_GFX_ERROR__RSVD1_ERROR__SHIFT                                                                      0x5
+#define CP_GFX_ERROR__RSVD2_ERROR__SHIFT                                                                      0x6
+#define CP_GFX_ERROR__SEM_UTCL1_ERROR__SHIFT                                                                  0x7
+#define CP_GFX_ERROR__QU_STRM_UTCL1_ERROR__SHIFT                                                              0x8
+#define CP_GFX_ERROR__QU_EOP_UTCL1_ERROR__SHIFT                                                               0x9
+#define CP_GFX_ERROR__QU_PIPE_UTCL1_ERROR__SHIFT                                                              0xa
+#define CP_GFX_ERROR__QU_READ_UTCL1_ERROR__SHIFT                                                              0xb
+#define CP_GFX_ERROR__SYNC_MEMRD_UTCL1_ERROR__SHIFT                                                           0xc
+#define CP_GFX_ERROR__SYNC_MEMWR_UTCL1_ERROR__SHIFT                                                           0xd
+#define CP_GFX_ERROR__SHADOW_UTCL1_ERROR__SHIFT                                                               0xe
+#define CP_GFX_ERROR__APPEND_UTCL1_ERROR__SHIFT                                                               0xf
+#define CP_GFX_ERROR__CE_DMA_UTCL1_ERROR__SHIFT                                                               0x10
+#define CP_GFX_ERROR__PFP_VGTDMA_UTCL1_ERROR__SHIFT                                                           0x11
+#define CP_GFX_ERROR__DMA_SRC_UTCL1_ERROR__SHIFT                                                              0x12
+#define CP_GFX_ERROR__DMA_DST_UTCL1_ERROR__SHIFT                                                              0x13
+#define CP_GFX_ERROR__PFP_TC_UTCL1_ERROR__SHIFT                                                               0x14
+#define CP_GFX_ERROR__ME_TC_UTCL1_ERROR__SHIFT                                                                0x15
+#define CP_GFX_ERROR__CE_TC_UTCL1_ERROR__SHIFT                                                                0x16
+#define CP_GFX_ERROR__PRT_LOD_UTCL1_ERROR__SHIFT                                                              0x17
+#define CP_GFX_ERROR__RDPTR_RPT_UTCL1_ERROR__SHIFT                                                            0x18
+#define CP_GFX_ERROR__RB_FETCHER_UTCL1_ERROR__SHIFT                                                           0x19
+#define CP_GFX_ERROR__I1_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1a
+#define CP_GFX_ERROR__I2_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1b
+#define CP_GFX_ERROR__C1_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1c
+#define CP_GFX_ERROR__C2_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1d
+#define CP_GFX_ERROR__ST_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1e
+#define CP_GFX_ERROR__CE_INIT_UTCL1_ERROR__SHIFT                                                              0x1f
+#define CP_GFX_ERROR__EDC_ERROR_ID_MASK                                                                       0x0000000FL
+#define CP_GFX_ERROR__SUA_ERROR_MASK                                                                          0x00000010L
+#define CP_GFX_ERROR__RSVD1_ERROR_MASK                                                                        0x00000020L
+#define CP_GFX_ERROR__RSVD2_ERROR_MASK                                                                        0x00000040L
+#define CP_GFX_ERROR__SEM_UTCL1_ERROR_MASK                                                                    0x00000080L
+#define CP_GFX_ERROR__QU_STRM_UTCL1_ERROR_MASK                                                                0x00000100L
+#define CP_GFX_ERROR__QU_EOP_UTCL1_ERROR_MASK                                                                 0x00000200L
+#define CP_GFX_ERROR__QU_PIPE_UTCL1_ERROR_MASK                                                                0x00000400L
+#define CP_GFX_ERROR__QU_READ_UTCL1_ERROR_MASK                                                                0x00000800L
+#define CP_GFX_ERROR__SYNC_MEMRD_UTCL1_ERROR_MASK                                                             0x00001000L
+#define CP_GFX_ERROR__SYNC_MEMWR_UTCL1_ERROR_MASK                                                             0x00002000L
+#define CP_GFX_ERROR__SHADOW_UTCL1_ERROR_MASK                                                                 0x00004000L
+#define CP_GFX_ERROR__APPEND_UTCL1_ERROR_MASK                                                                 0x00008000L
+#define CP_GFX_ERROR__CE_DMA_UTCL1_ERROR_MASK                                                                 0x00010000L
+#define CP_GFX_ERROR__PFP_VGTDMA_UTCL1_ERROR_MASK                                                             0x00020000L
+#define CP_GFX_ERROR__DMA_SRC_UTCL1_ERROR_MASK                                                                0x00040000L
+#define CP_GFX_ERROR__DMA_DST_UTCL1_ERROR_MASK                                                                0x00080000L
+#define CP_GFX_ERROR__PFP_TC_UTCL1_ERROR_MASK                                                                 0x00100000L
+#define CP_GFX_ERROR__ME_TC_UTCL1_ERROR_MASK                                                                  0x00200000L
+#define CP_GFX_ERROR__CE_TC_UTCL1_ERROR_MASK                                                                  0x00400000L
+#define CP_GFX_ERROR__PRT_LOD_UTCL1_ERROR_MASK                                                                0x00800000L
+#define CP_GFX_ERROR__RDPTR_RPT_UTCL1_ERROR_MASK                                                              0x01000000L
+#define CP_GFX_ERROR__RB_FETCHER_UTCL1_ERROR_MASK                                                             0x02000000L
+#define CP_GFX_ERROR__I1_FETCHER_UTCL1_ERROR_MASK                                                             0x04000000L
+#define CP_GFX_ERROR__I2_FETCHER_UTCL1_ERROR_MASK                                                             0x08000000L
+#define CP_GFX_ERROR__C1_FETCHER_UTCL1_ERROR_MASK                                                             0x10000000L
+#define CP_GFX_ERROR__C2_FETCHER_UTCL1_ERROR_MASK                                                             0x20000000L
+#define CP_GFX_ERROR__ST_FETCHER_UTCL1_ERROR_MASK                                                             0x40000000L
+#define CP_GFX_ERROR__CE_INIT_UTCL1_ERROR_MASK                                                                0x80000000L
+//CPG_UTCL1_CNTL
+#define CPG_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
+#define CPG_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                0x17
+#define CPG_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
+#define CPG_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
+#define CPG_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
+#define CPG_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
+#define CPG_UTCL1_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                          0x1d
+#define CPG_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
+#define CPG_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
+#define CPG_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                  0x00800000L
+#define CPG_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
+#define CPG_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
+#define CPG_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
+#define CPG_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
+#define CPG_UTCL1_CNTL__IGNORE_PTE_PERMISSION_MASK                                                            0x20000000L
+#define CPG_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
+//CPC_UTCL1_CNTL
+#define CPC_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
+#define CPC_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
+#define CPC_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
+#define CPC_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
+#define CPC_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
+#define CPC_UTCL1_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                          0x1d
+#define CPC_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
+#define CPC_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
+#define CPC_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
+#define CPC_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
+#define CPC_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
+#define CPC_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
+#define CPC_UTCL1_CNTL__IGNORE_PTE_PERMISSION_MASK                                                            0x20000000L
+#define CPC_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
+//CPF_UTCL1_CNTL
+#define CPF_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
+#define CPF_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                0x17
+#define CPF_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
+#define CPF_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
+#define CPF_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
+#define CPF_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
+#define CPF_UTCL1_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                          0x1d
+#define CPF_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
+#define CPF_UTCL1_CNTL__FORCE_NO_EXE__SHIFT                                                                   0x1f
+#define CPF_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
+#define CPF_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                  0x00800000L
+#define CPF_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
+#define CPF_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
+#define CPF_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
+#define CPF_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
+#define CPF_UTCL1_CNTL__IGNORE_PTE_PERMISSION_MASK                                                            0x20000000L
+#define CPF_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
+#define CPF_UTCL1_CNTL__FORCE_NO_EXE_MASK                                                                     0x80000000L
+//CP_AQL_SMM_STATUS
+#define CP_AQL_SMM_STATUS__AQL_QUEUE_SMM__SHIFT                                                               0x0
+#define CP_AQL_SMM_STATUS__AQL_QUEUE_SMM_MASK                                                                 0xFFFFFFFFL
+//CP_RB0_BASE
+#define CP_RB0_BASE__RB_BASE__SHIFT                                                                           0x0
+#define CP_RB0_BASE__RB_BASE_MASK                                                                             0xFFFFFFFFL
+//CP_RB_BASE
+#define CP_RB_BASE__RB_BASE__SHIFT                                                                            0x0
+#define CP_RB_BASE__RB_BASE_MASK                                                                              0xFFFFFFFFL
+//CP_RB0_CNTL
+#define CP_RB0_CNTL__RB_BUFSZ__SHIFT                                                                          0x0
+#define CP_RB0_CNTL__RB_BLKSZ__SHIFT                                                                          0x8
+#define CP_RB0_CNTL__BUF_SWAP__SHIFT                                                                          0x11
+#define CP_RB0_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x14
+#define CP_RB0_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                    0x16
+#define CP_RB0_CNTL__CACHE_POLICY__SHIFT                                                                      0x18
+#define CP_RB0_CNTL__RB_NO_UPDATE__SHIFT                                                                      0x1b
+#define CP_RB0_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                    0x1f
+#define CP_RB0_CNTL__RB_BUFSZ_MASK                                                                            0x0000003FL
+#define CP_RB0_CNTL__RB_BLKSZ_MASK                                                                            0x00003F00L
+#define CP_RB0_CNTL__BUF_SWAP_MASK                                                                            0x00060000L
+#define CP_RB0_CNTL__MIN_AVAILSZ_MASK                                                                         0x00300000L
+#define CP_RB0_CNTL__MIN_IB_AVAILSZ_MASK                                                                      0x00C00000L
+#define CP_RB0_CNTL__CACHE_POLICY_MASK                                                                        0x01000000L
+#define CP_RB0_CNTL__RB_NO_UPDATE_MASK                                                                        0x08000000L
+#define CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK                                                                      0x80000000L
+//CP_RB_CNTL
+#define CP_RB_CNTL__RB_BUFSZ__SHIFT                                                                           0x0
+#define CP_RB_CNTL__RB_BLKSZ__SHIFT                                                                           0x8
+#define CP_RB_CNTL__MIN_AVAILSZ__SHIFT                                                                        0x14
+#define CP_RB_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                     0x16
+#define CP_RB_CNTL__CACHE_POLICY__SHIFT                                                                       0x18
+#define CP_RB_CNTL__RB_NO_UPDATE__SHIFT                                                                       0x1b
+#define CP_RB_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                     0x1f
+#define CP_RB_CNTL__RB_BUFSZ_MASK                                                                             0x0000003FL
+#define CP_RB_CNTL__RB_BLKSZ_MASK                                                                             0x00003F00L
+#define CP_RB_CNTL__MIN_AVAILSZ_MASK                                                                          0x00300000L
+#define CP_RB_CNTL__MIN_IB_AVAILSZ_MASK                                                                       0x00C00000L
+#define CP_RB_CNTL__CACHE_POLICY_MASK                                                                         0x01000000L
+#define CP_RB_CNTL__RB_NO_UPDATE_MASK                                                                         0x08000000L
+#define CP_RB_CNTL__RB_RPTR_WR_ENA_MASK                                                                       0x80000000L
+//CP_RB_RPTR_WR
+#define CP_RB_RPTR_WR__RB_RPTR_WR__SHIFT                                                                      0x0
+#define CP_RB_RPTR_WR__RB_RPTR_WR_MASK                                                                        0x000FFFFFL
+//CP_RB0_RPTR_ADDR
+#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                 0x2
+#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                   0xFFFFFFFCL
+//CP_RB_RPTR_ADDR
+#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                  0x2
+#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                    0xFFFFFFFCL
+//CP_RB0_RPTR_ADDR_HI
+#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                           0x0
+#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                             0x0000FFFFL
+//CP_RB_RPTR_ADDR_HI
+#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                            0x0
+#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                              0x0000FFFFL
+//CP_RB0_BUFSZ_MASK
+#define CP_RB0_BUFSZ_MASK__DATA__SHIFT                                                                        0x0
+#define CP_RB0_BUFSZ_MASK__DATA_MASK                                                                          0x000FFFFFL
+//CP_RB_BUFSZ_MASK
+#define CP_RB_BUFSZ_MASK__DATA__SHIFT                                                                         0x0
+#define CP_RB_BUFSZ_MASK__DATA_MASK                                                                           0x000FFFFFL
+//CP_RB_WPTR_POLL_ADDR_LO
+#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO__SHIFT                                                  0x2
+#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO_MASK                                                    0xFFFFFFFCL
+//CP_RB_WPTR_POLL_ADDR_HI
+#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI__SHIFT                                                  0x0
+#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI_MASK                                                    0x0000FFFFL
+//GC_PRIV_MODE
+#define GC_PRIV_MODE__MC_PRIV_MODE__SHIFT                                                                     0x0
+#define GC_PRIV_MODE__MC_PRIV_MODE_MASK                                                                       0x00000001L
+//CP_INT_CNTL
+#define CP_INT_CNTL__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                      0xb
+#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                           0xe
+#define CP_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                                    0x10
+#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                       0x11
+#define CP_INT_CNTL__CMP_BUSY_INT_ENABLE__SHIFT                                                               0x12
+#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE__SHIFT                                                              0x13
+#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE__SHIFT                                                             0x14
+#define CP_INT_CNTL__GFX_IDLE_INT_ENABLE__SHIFT                                                               0x15
+#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE__SHIFT                                                             0x16
+#define CP_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                               0x17
+#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                           0x18
+#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                             0x1a
+#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                                     0x1b
+#define CP_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                               0x1d
+#define CP_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                               0x1e
+#define CP_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                               0x1f
+#define CP_INT_CNTL__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                        0x00000800L
+#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                             0x00004000L
+#define CP_INT_CNTL__GPF_INT_ENABLE_MASK                                                                      0x00010000L
+#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                         0x00020000L
+#define CP_INT_CNTL__CMP_BUSY_INT_ENABLE_MASK                                                                 0x00040000L
+#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK                                                                0x00080000L
+#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK                                                               0x00100000L
+#define CP_INT_CNTL__GFX_IDLE_INT_ENABLE_MASK                                                                 0x00200000L
+#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE_MASK                                                               0x00400000L
+#define CP_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                                 0x00800000L
+#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                             0x01000000L
+#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                               0x04000000L
+#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                       0x08000000L
+#define CP_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                                 0x20000000L
+#define CP_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                                 0x40000000L
+#define CP_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                                 0x80000000L
+//CP_INT_STATUS
+#define CP_INT_STATUS__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                      0xb
+#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT__SHIFT                                                           0xe
+#define CP_INT_STATUS__GPF_INT_STAT__SHIFT                                                                    0x10
+#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                       0x11
+#define CP_INT_STATUS__CMP_BUSY_INT_STAT__SHIFT                                                               0x12
+#define CP_INT_STATUS__CNTX_BUSY_INT_STAT__SHIFT                                                              0x13
+#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT__SHIFT                                                             0x14
+#define CP_INT_STATUS__GFX_IDLE_INT_STAT__SHIFT                                                               0x15
+#define CP_INT_STATUS__PRIV_INSTR_INT_STAT__SHIFT                                                             0x16
+#define CP_INT_STATUS__PRIV_REG_INT_STAT__SHIFT                                                               0x17
+#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT__SHIFT                                                           0x18
+#define CP_INT_STATUS__TIME_STAMP_INT_STAT__SHIFT                                                             0x1a
+#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                                     0x1b
+#define CP_INT_STATUS__GENERIC2_INT_STAT__SHIFT                                                               0x1d
+#define CP_INT_STATUS__GENERIC1_INT_STAT__SHIFT                                                               0x1e
+#define CP_INT_STATUS__GENERIC0_INT_STAT__SHIFT                                                               0x1f
+#define CP_INT_STATUS__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                        0x00000800L
+#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT_MASK                                                             0x00004000L
+#define CP_INT_STATUS__GPF_INT_STAT_MASK                                                                      0x00010000L
+#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                         0x00020000L
+#define CP_INT_STATUS__CMP_BUSY_INT_STAT_MASK                                                                 0x00040000L
+#define CP_INT_STATUS__CNTX_BUSY_INT_STAT_MASK                                                                0x00080000L
+#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT_MASK                                                               0x00100000L
+#define CP_INT_STATUS__GFX_IDLE_INT_STAT_MASK                                                                 0x00200000L
+#define CP_INT_STATUS__PRIV_INSTR_INT_STAT_MASK                                                               0x00400000L
+#define CP_INT_STATUS__PRIV_REG_INT_STAT_MASK                                                                 0x00800000L
+#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT_MASK                                                             0x01000000L
+#define CP_INT_STATUS__TIME_STAMP_INT_STAT_MASK                                                               0x04000000L
+#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT_MASK                                                       0x08000000L
+#define CP_INT_STATUS__GENERIC2_INT_STAT_MASK                                                                 0x20000000L
+#define CP_INT_STATUS__GENERIC1_INT_STAT_MASK                                                                 0x40000000L
+#define CP_INT_STATUS__GENERIC0_INT_STAT_MASK                                                                 0x80000000L
+//CP_DEVICE_ID
+#define CP_DEVICE_ID__DEVICE_ID__SHIFT                                                                        0x0
+#define CP_DEVICE_ID__DEVICE_ID_MASK                                                                          0x000000FFL
+//CP_ME0_PIPE_PRIORITY_CNTS
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
+//CP_RING_PRIORITY_CNTS
+#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                           0x0
+#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                          0x8
+#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                          0x10
+#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                           0x18
+#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                             0x000000FFL
+#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                            0x0000FF00L
+#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                            0x00FF0000L
+#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                             0xFF000000L
+//CP_ME0_PIPE0_PRIORITY
+#define CP_ME0_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME0_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_RING0_PRIORITY
+#define CP_RING0_PRIORITY__PRIORITY__SHIFT                                                                    0x0
+#define CP_RING0_PRIORITY__PRIORITY_MASK                                                                      0x00000003L
+//CP_ME0_PIPE1_PRIORITY
+#define CP_ME0_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME0_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_RING1_PRIORITY
+#define CP_RING1_PRIORITY__PRIORITY__SHIFT                                                                    0x0
+#define CP_RING1_PRIORITY__PRIORITY_MASK                                                                      0x00000003L
+//CP_ME0_PIPE2_PRIORITY
+#define CP_ME0_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME0_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_RING2_PRIORITY
+#define CP_RING2_PRIORITY__PRIORITY__SHIFT                                                                    0x0
+#define CP_RING2_PRIORITY__PRIORITY_MASK                                                                      0x00000003L
+//CP_FATAL_ERROR
+#define CP_FATAL_ERROR__CPF_FATAL_ERROR__SHIFT                                                                0x0
+#define CP_FATAL_ERROR__CPG_FATAL_ERROR__SHIFT                                                                0x1
+#define CP_FATAL_ERROR__GFX_HALT_PROC__SHIFT                                                                  0x2
+#define CP_FATAL_ERROR__DIS_CPG_FATAL_ERROR__SHIFT                                                            0x3
+#define CP_FATAL_ERROR__CPG_TAG_FATAL_ERROR_EN__SHIFT                                                         0x4
+#define CP_FATAL_ERROR__CPF_FATAL_ERROR_MASK                                                                  0x00000001L
+#define CP_FATAL_ERROR__CPG_FATAL_ERROR_MASK                                                                  0x00000002L
+#define CP_FATAL_ERROR__GFX_HALT_PROC_MASK                                                                    0x00000004L
+#define CP_FATAL_ERROR__DIS_CPG_FATAL_ERROR_MASK                                                              0x00000008L
+#define CP_FATAL_ERROR__CPG_TAG_FATAL_ERROR_EN_MASK                                                           0x00000010L
+//CP_RB_VMID
+#define CP_RB_VMID__RB0_VMID__SHIFT                                                                           0x0
+#define CP_RB_VMID__RB1_VMID__SHIFT                                                                           0x8
+#define CP_RB_VMID__RB2_VMID__SHIFT                                                                           0x10
+#define CP_RB_VMID__RB0_VMID_MASK                                                                             0x0000000FL
+#define CP_RB_VMID__RB1_VMID_MASK                                                                             0x00000F00L
+#define CP_RB_VMID__RB2_VMID_MASK                                                                             0x000F0000L
+//CP_ME0_PIPE0_VMID
+#define CP_ME0_PIPE0_VMID__VMID__SHIFT                                                                        0x0
+#define CP_ME0_PIPE0_VMID__VMID_MASK                                                                          0x0000000FL
+//CP_ME0_PIPE1_VMID
+#define CP_ME0_PIPE1_VMID__VMID__SHIFT                                                                        0x0
+#define CP_ME0_PIPE1_VMID__VMID_MASK                                                                          0x0000000FL
+//CP_RB0_WPTR
+#define CP_RB0_WPTR__RB_WPTR__SHIFT                                                                           0x0
+#define CP_RB0_WPTR__RB_WPTR_MASK                                                                             0xFFFFFFFFL
+//CP_RB_WPTR
+#define CP_RB_WPTR__RB_WPTR__SHIFT                                                                            0x0
+#define CP_RB_WPTR__RB_WPTR_MASK                                                                              0xFFFFFFFFL
+//CP_RB0_WPTR_HI
+#define CP_RB0_WPTR_HI__RB_WPTR__SHIFT                                                                        0x0
+#define CP_RB0_WPTR_HI__RB_WPTR_MASK                                                                          0xFFFFFFFFL
+//CP_RB_WPTR_HI
+#define CP_RB_WPTR_HI__RB_WPTR__SHIFT                                                                         0x0
+#define CP_RB_WPTR_HI__RB_WPTR_MASK                                                                           0xFFFFFFFFL
+//CP_RB1_WPTR
+#define CP_RB1_WPTR__RB_WPTR__SHIFT                                                                           0x0
+#define CP_RB1_WPTR__RB_WPTR_MASK                                                                             0xFFFFFFFFL
+//CP_RB1_WPTR_HI
+#define CP_RB1_WPTR_HI__RB_WPTR__SHIFT                                                                        0x0
+#define CP_RB1_WPTR_HI__RB_WPTR_MASK                                                                          0xFFFFFFFFL
+//CP_RB2_WPTR
+#define CP_RB2_WPTR__RB_WPTR__SHIFT                                                                           0x0
+#define CP_RB2_WPTR__RB_WPTR_MASK                                                                             0x000FFFFFL
+//CP_RB_DOORBELL_CONTROL
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT                                                      0x1
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT                                                        0x2
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_EN__SHIFT                                                            0x1e
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT                                                           0x1f
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_BIF_DROP_MASK                                                        0x00000002L
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK                                                          0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_EN_MASK                                                              0x40000000L
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_HIT_MASK                                                             0x80000000L
+//CP_RB_DOORBELL_RANGE_LOWER
+#define CP_RB_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER__SHIFT                                               0x2
+#define CP_RB_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER_MASK                                                 0x0FFFFFFCL
+//CP_RB_DOORBELL_RANGE_UPPER
+#define CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER__SHIFT                                               0x2
+#define CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK                                                 0x0FFFFFFCL
+//CP_MEC_DOORBELL_RANGE_LOWER
+#define CP_MEC_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER__SHIFT                                              0x2
+#define CP_MEC_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER_MASK                                                0x0FFFFFFCL
+//CP_MEC_DOORBELL_RANGE_UPPER
+#define CP_MEC_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER__SHIFT                                              0x2
+#define CP_MEC_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK                                                0x0FFFFFFCL
+//CPG_UTCL1_ERROR
+#define CPG_UTCL1_ERROR__ERROR_DETECTED_HALT__SHIFT                                                           0x0
+#define CPG_UTCL1_ERROR__ERROR_DETECTED_HALT_MASK                                                             0x00000001L
+//CPC_UTCL1_ERROR
+#define CPC_UTCL1_ERROR__ERROR_DETECTED_HALT__SHIFT                                                           0x0
+#define CPC_UTCL1_ERROR__ERROR_DETECTED_HALT_MASK                                                             0x00000001L
+//CP_RB1_BASE
+#define CP_RB1_BASE__RB_BASE__SHIFT                                                                           0x0
+#define CP_RB1_BASE__RB_BASE_MASK                                                                             0xFFFFFFFFL
+//CP_RB1_CNTL
+#define CP_RB1_CNTL__RB_BUFSZ__SHIFT                                                                          0x0
+#define CP_RB1_CNTL__RB_BLKSZ__SHIFT                                                                          0x8
+#define CP_RB1_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x14
+#define CP_RB1_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                    0x16
+#define CP_RB1_CNTL__CACHE_POLICY__SHIFT                                                                      0x18
+#define CP_RB1_CNTL__RB_NO_UPDATE__SHIFT                                                                      0x1b
+#define CP_RB1_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                    0x1f
+#define CP_RB1_CNTL__RB_BUFSZ_MASK                                                                            0x0000003FL
+#define CP_RB1_CNTL__RB_BLKSZ_MASK                                                                            0x00003F00L
+#define CP_RB1_CNTL__MIN_AVAILSZ_MASK                                                                         0x00300000L
+#define CP_RB1_CNTL__MIN_IB_AVAILSZ_MASK                                                                      0x00C00000L
+#define CP_RB1_CNTL__CACHE_POLICY_MASK                                                                        0x01000000L
+#define CP_RB1_CNTL__RB_NO_UPDATE_MASK                                                                        0x08000000L
+#define CP_RB1_CNTL__RB_RPTR_WR_ENA_MASK                                                                      0x80000000L
+//CP_RB1_RPTR_ADDR
+#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                 0x2
+#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                   0xFFFFFFFCL
+//CP_RB1_RPTR_ADDR_HI
+#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                           0x0
+#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                             0x0000FFFFL
+//CP_RB2_BASE
+#define CP_RB2_BASE__RB_BASE__SHIFT                                                                           0x0
+#define CP_RB2_BASE__RB_BASE_MASK                                                                             0xFFFFFFFFL
+//CP_RB2_CNTL
+#define CP_RB2_CNTL__RB_BUFSZ__SHIFT                                                                          0x0
+#define CP_RB2_CNTL__RB_BLKSZ__SHIFT                                                                          0x8
+#define CP_RB2_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x14
+#define CP_RB2_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                    0x16
+#define CP_RB2_CNTL__CACHE_POLICY__SHIFT                                                                      0x18
+#define CP_RB2_CNTL__RB_NO_UPDATE__SHIFT                                                                      0x1b
+#define CP_RB2_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                    0x1f
+#define CP_RB2_CNTL__RB_BUFSZ_MASK                                                                            0x0000003FL
+#define CP_RB2_CNTL__RB_BLKSZ_MASK                                                                            0x00003F00L
+#define CP_RB2_CNTL__MIN_AVAILSZ_MASK                                                                         0x00300000L
+#define CP_RB2_CNTL__MIN_IB_AVAILSZ_MASK                                                                      0x00C00000L
+#define CP_RB2_CNTL__CACHE_POLICY_MASK                                                                        0x01000000L
+#define CP_RB2_CNTL__RB_NO_UPDATE_MASK                                                                        0x08000000L
+#define CP_RB2_CNTL__RB_RPTR_WR_ENA_MASK                                                                      0x80000000L
+//CP_RB2_RPTR_ADDR
+#define CP_RB2_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                 0x2
+#define CP_RB2_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                   0xFFFFFFFCL
+//CP_RB2_RPTR_ADDR_HI
+#define CP_RB2_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                           0x0
+#define CP_RB2_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                             0x0000FFFFL
+//CP_RB0_ACTIVE
+#define CP_RB0_ACTIVE__ACTIVE__SHIFT                                                                          0x0
+#define CP_RB0_ACTIVE__ACTIVE_MASK                                                                            0x00000001L
+//CP_RB_ACTIVE
+#define CP_RB_ACTIVE__ACTIVE__SHIFT                                                                           0x0
+#define CP_RB_ACTIVE__ACTIVE_MASK                                                                             0x00000001L
+//CP_INT_CNTL_RING0
+#define CP_INT_CNTL_RING0__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                0xb
+#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                     0xe
+#define CP_INT_CNTL_RING0__GPF_INT_ENABLE__SHIFT                                                              0x10
+#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                 0x11
+#define CP_INT_CNTL_RING0__CMP_BUSY_INT_ENABLE__SHIFT                                                         0x12
+#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE__SHIFT                                                        0x13
+#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE__SHIFT                                                       0x14
+#define CP_INT_CNTL_RING0__GFX_IDLE_INT_ENABLE__SHIFT                                                         0x15
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE__SHIFT                                                       0x16
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE__SHIFT                                                         0x17
+#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE__SHIFT                                                     0x18
+#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE__SHIFT                                                       0x1a
+#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                               0x1b
+#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE__SHIFT                                                         0x1d
+#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE__SHIFT                                                         0x1e
+#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE__SHIFT                                                         0x1f
+#define CP_INT_CNTL_RING0__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                  0x00000800L
+#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE_MASK                                                       0x00004000L
+#define CP_INT_CNTL_RING0__GPF_INT_ENABLE_MASK                                                                0x00010000L
+#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                   0x00020000L
+#define CP_INT_CNTL_RING0__CMP_BUSY_INT_ENABLE_MASK                                                           0x00040000L
+#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK                                                          0x00080000L
+#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK                                                         0x00100000L
+#define CP_INT_CNTL_RING0__GFX_IDLE_INT_ENABLE_MASK                                                           0x00200000L
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK                                                         0x00400000L
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK                                                           0x00800000L
+#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK                                                       0x01000000L
+#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK                                                         0x04000000L
+#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                 0x08000000L
+#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE_MASK                                                           0x20000000L
+#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE_MASK                                                           0x40000000L
+#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE_MASK                                                           0x80000000L
+//CP_INT_CNTL_RING1
+#define CP_INT_CNTL_RING1__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                0xb
+#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                     0xe
+#define CP_INT_CNTL_RING1__GPF_INT_ENABLE__SHIFT                                                              0x10
+#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                 0x11
+#define CP_INT_CNTL_RING1__CMP_BUSY_INT_ENABLE__SHIFT                                                         0x12
+#define CP_INT_CNTL_RING1__CNTX_BUSY_INT_ENABLE__SHIFT                                                        0x13
+#define CP_INT_CNTL_RING1__CNTX_EMPTY_INT_ENABLE__SHIFT                                                       0x14
+#define CP_INT_CNTL_RING1__GFX_IDLE_INT_ENABLE__SHIFT                                                         0x15
+#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE__SHIFT                                                       0x16
+#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE__SHIFT                                                         0x17
+#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE__SHIFT                                                     0x18
+#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE__SHIFT                                                       0x1a
+#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                               0x1b
+#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE__SHIFT                                                         0x1d
+#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE__SHIFT                                                         0x1e
+#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE__SHIFT                                                         0x1f
+#define CP_INT_CNTL_RING1__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                  0x00000800L
+#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE_MASK                                                       0x00004000L
+#define CP_INT_CNTL_RING1__GPF_INT_ENABLE_MASK                                                                0x00010000L
+#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                   0x00020000L
+#define CP_INT_CNTL_RING1__CMP_BUSY_INT_ENABLE_MASK                                                           0x00040000L
+#define CP_INT_CNTL_RING1__CNTX_BUSY_INT_ENABLE_MASK                                                          0x00080000L
+#define CP_INT_CNTL_RING1__CNTX_EMPTY_INT_ENABLE_MASK                                                         0x00100000L
+#define CP_INT_CNTL_RING1__GFX_IDLE_INT_ENABLE_MASK                                                           0x00200000L
+#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE_MASK                                                         0x00400000L
+#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE_MASK                                                           0x00800000L
+#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE_MASK                                                       0x01000000L
+#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE_MASK                                                         0x04000000L
+#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                 0x08000000L
+#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE_MASK                                                           0x20000000L
+#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE_MASK                                                           0x40000000L
+#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE_MASK                                                           0x80000000L
+//CP_INT_CNTL_RING2
+#define CP_INT_CNTL_RING2__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                0xb
+#define CP_INT_CNTL_RING2__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                     0xe
+#define CP_INT_CNTL_RING2__GPF_INT_ENABLE__SHIFT                                                              0x10
+#define CP_INT_CNTL_RING2__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                 0x11
+#define CP_INT_CNTL_RING2__CMP_BUSY_INT_ENABLE__SHIFT                                                         0x12
+#define CP_INT_CNTL_RING2__CNTX_BUSY_INT_ENABLE__SHIFT                                                        0x13
+#define CP_INT_CNTL_RING2__CNTX_EMPTY_INT_ENABLE__SHIFT                                                       0x14
+#define CP_INT_CNTL_RING2__GFX_IDLE_INT_ENABLE__SHIFT                                                         0x15
+#define CP_INT_CNTL_RING2__PRIV_INSTR_INT_ENABLE__SHIFT                                                       0x16
+#define CP_INT_CNTL_RING2__PRIV_REG_INT_ENABLE__SHIFT                                                         0x17
+#define CP_INT_CNTL_RING2__OPCODE_ERROR_INT_ENABLE__SHIFT                                                     0x18
+#define CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE__SHIFT                                                       0x1a
+#define CP_INT_CNTL_RING2__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                               0x1b
+#define CP_INT_CNTL_RING2__GENERIC2_INT_ENABLE__SHIFT                                                         0x1d
+#define CP_INT_CNTL_RING2__GENERIC1_INT_ENABLE__SHIFT                                                         0x1e
+#define CP_INT_CNTL_RING2__GENERIC0_INT_ENABLE__SHIFT                                                         0x1f
+#define CP_INT_CNTL_RING2__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                  0x00000800L
+#define CP_INT_CNTL_RING2__CP_ECC_ERROR_INT_ENABLE_MASK                                                       0x00004000L
+#define CP_INT_CNTL_RING2__GPF_INT_ENABLE_MASK                                                                0x00010000L
+#define CP_INT_CNTL_RING2__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                   0x00020000L
+#define CP_INT_CNTL_RING2__CMP_BUSY_INT_ENABLE_MASK                                                           0x00040000L
+#define CP_INT_CNTL_RING2__CNTX_BUSY_INT_ENABLE_MASK                                                          0x00080000L
+#define CP_INT_CNTL_RING2__CNTX_EMPTY_INT_ENABLE_MASK                                                         0x00100000L
+#define CP_INT_CNTL_RING2__GFX_IDLE_INT_ENABLE_MASK                                                           0x00200000L
+#define CP_INT_CNTL_RING2__PRIV_INSTR_INT_ENABLE_MASK                                                         0x00400000L
+#define CP_INT_CNTL_RING2__PRIV_REG_INT_ENABLE_MASK                                                           0x00800000L
+#define CP_INT_CNTL_RING2__OPCODE_ERROR_INT_ENABLE_MASK                                                       0x01000000L
+#define CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE_MASK                                                         0x04000000L
+#define CP_INT_CNTL_RING2__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                 0x08000000L
+#define CP_INT_CNTL_RING2__GENERIC2_INT_ENABLE_MASK                                                           0x20000000L
+#define CP_INT_CNTL_RING2__GENERIC1_INT_ENABLE_MASK                                                           0x40000000L
+#define CP_INT_CNTL_RING2__GENERIC0_INT_ENABLE_MASK                                                           0x80000000L
+//CP_INT_STATUS_RING0
+#define CP_INT_STATUS_RING0__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                0xb
+#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT__SHIFT                                                     0xe
+#define CP_INT_STATUS_RING0__GPF_INT_STAT__SHIFT                                                              0x10
+#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                 0x11
+#define CP_INT_STATUS_RING0__CMP_BUSY_INT_STAT__SHIFT                                                         0x12
+#define CP_INT_STATUS_RING0__GCNTX_BUSY_INT_STAT__SHIFT                                                       0x13
+#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT__SHIFT                                                       0x14
+#define CP_INT_STATUS_RING0__GFX_IDLE_INT_STAT__SHIFT                                                         0x15
+#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT__SHIFT                                                       0x16
+#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT__SHIFT                                                         0x17
+#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT__SHIFT                                                     0x18
+#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT__SHIFT                                                       0x1a
+#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                               0x1b
+#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT__SHIFT                                                         0x1d
+#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT__SHIFT                                                         0x1e
+#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT__SHIFT                                                         0x1f
+#define CP_INT_STATUS_RING0__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                  0x00000800L
+#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT_MASK                                                       0x00004000L
+#define CP_INT_STATUS_RING0__GPF_INT_STAT_MASK                                                                0x00010000L
+#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                   0x00020000L
+#define CP_INT_STATUS_RING0__CMP_BUSY_INT_STAT_MASK                                                           0x00040000L
+#define CP_INT_STATUS_RING0__GCNTX_BUSY_INT_STAT_MASK                                                         0x00080000L
+#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT_MASK                                                         0x00100000L
+#define CP_INT_STATUS_RING0__GFX_IDLE_INT_STAT_MASK                                                           0x00200000L
+#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT_MASK                                                         0x00400000L
+#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT_MASK                                                           0x00800000L
+#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT_MASK                                                       0x01000000L
+#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT_MASK                                                         0x04000000L
+#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT_MASK                                                 0x08000000L
+#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT_MASK                                                           0x20000000L
+#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT_MASK                                                           0x40000000L
+#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT_MASK                                                           0x80000000L
+//CP_INT_STATUS_RING1
+#define CP_INT_STATUS_RING1__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                0xb
+#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT__SHIFT                                                     0xe
+#define CP_INT_STATUS_RING1__GPF_INT_STAT__SHIFT                                                              0x10
+#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                 0x11
+#define CP_INT_STATUS_RING1__CMP_BUSY_INT_STAT__SHIFT                                                         0x12
+#define CP_INT_STATUS_RING1__CNTX_BUSY_INT_STAT__SHIFT                                                        0x13
+#define CP_INT_STATUS_RING1__CNTX_EMPTY_INT_STAT__SHIFT                                                       0x14
+#define CP_INT_STATUS_RING1__GFX_IDLE_INT_STAT__SHIFT                                                         0x15
+#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT__SHIFT                                                       0x16
+#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT__SHIFT                                                         0x17
+#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT__SHIFT                                                     0x18
+#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT__SHIFT                                                       0x1a
+#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                               0x1b
+#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT__SHIFT                                                         0x1d
+#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT__SHIFT                                                         0x1e
+#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT__SHIFT                                                         0x1f
+#define CP_INT_STATUS_RING1__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                  0x00000800L
+#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT_MASK                                                       0x00004000L
+#define CP_INT_STATUS_RING1__GPF_INT_STAT_MASK                                                                0x00010000L
+#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                   0x00020000L
+#define CP_INT_STATUS_RING1__CMP_BUSY_INT_STAT_MASK                                                           0x00040000L
+#define CP_INT_STATUS_RING1__CNTX_BUSY_INT_STAT_MASK                                                          0x00080000L
+#define CP_INT_STATUS_RING1__CNTX_EMPTY_INT_STAT_MASK                                                         0x00100000L
+#define CP_INT_STATUS_RING1__GFX_IDLE_INT_STAT_MASK                                                           0x00200000L
+#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT_MASK                                                         0x00400000L
+#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT_MASK                                                           0x00800000L
+#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT_MASK                                                       0x01000000L
+#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT_MASK                                                         0x04000000L
+#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT_MASK                                                 0x08000000L
+#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT_MASK                                                           0x20000000L
+#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT_MASK                                                           0x40000000L
+#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT_MASK                                                           0x80000000L
+//CP_INT_STATUS_RING2
+#define CP_INT_STATUS_RING2__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                0xb
+#define CP_INT_STATUS_RING2__CP_ECC_ERROR_INT_STAT__SHIFT                                                     0xe
+#define CP_INT_STATUS_RING2__GPF_INT_STAT__SHIFT                                                              0x10
+#define CP_INT_STATUS_RING2__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                 0x11
+#define CP_INT_STATUS_RING2__CMP_BUSY_INT_STAT__SHIFT                                                         0x12
+#define CP_INT_STATUS_RING2__CNTX_BUSY_INT_STAT__SHIFT                                                        0x13
+#define CP_INT_STATUS_RING2__CNTX_EMPTY_INT_STAT__SHIFT                                                       0x14
+#define CP_INT_STATUS_RING2__GFX_IDLE_INT_STAT__SHIFT                                                         0x15
+#define CP_INT_STATUS_RING2__PRIV_INSTR_INT_STAT__SHIFT                                                       0x16
+#define CP_INT_STATUS_RING2__PRIV_REG_INT_STAT__SHIFT                                                         0x17
+#define CP_INT_STATUS_RING2__OPCODE_ERROR_INT_STAT__SHIFT                                                     0x18
+#define CP_INT_STATUS_RING2__TIME_STAMP_INT_STAT__SHIFT                                                       0x1a
+#define CP_INT_STATUS_RING2__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                               0x1b
+#define CP_INT_STATUS_RING2__GENERIC2_INT_STAT__SHIFT                                                         0x1d
+#define CP_INT_STATUS_RING2__GENERIC1_INT_STAT__SHIFT                                                         0x1e
+#define CP_INT_STATUS_RING2__GENERIC0_INT_STAT__SHIFT                                                         0x1f
+#define CP_INT_STATUS_RING2__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                  0x00000800L
+#define CP_INT_STATUS_RING2__CP_ECC_ERROR_INT_STAT_MASK                                                       0x00004000L
+#define CP_INT_STATUS_RING2__GPF_INT_STAT_MASK                                                                0x00010000L
+#define CP_INT_STATUS_RING2__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                   0x00020000L
+#define CP_INT_STATUS_RING2__CMP_BUSY_INT_STAT_MASK                                                           0x00040000L
+#define CP_INT_STATUS_RING2__CNTX_BUSY_INT_STAT_MASK                                                          0x00080000L
+#define CP_INT_STATUS_RING2__CNTX_EMPTY_INT_STAT_MASK                                                         0x00100000L
+#define CP_INT_STATUS_RING2__GFX_IDLE_INT_STAT_MASK                                                           0x00200000L
+#define CP_INT_STATUS_RING2__PRIV_INSTR_INT_STAT_MASK                                                         0x00400000L
+#define CP_INT_STATUS_RING2__PRIV_REG_INT_STAT_MASK                                                           0x00800000L
+#define CP_INT_STATUS_RING2__OPCODE_ERROR_INT_STAT_MASK                                                       0x01000000L
+#define CP_INT_STATUS_RING2__TIME_STAMP_INT_STAT_MASK                                                         0x04000000L
+#define CP_INT_STATUS_RING2__RESERVED_BIT_ERROR_INT_STAT_MASK                                                 0x08000000L
+#define CP_INT_STATUS_RING2__GENERIC2_INT_STAT_MASK                                                           0x20000000L
+#define CP_INT_STATUS_RING2__GENERIC1_INT_STAT_MASK                                                           0x40000000L
+#define CP_INT_STATUS_RING2__GENERIC0_INT_STAT_MASK                                                           0x80000000L
+//CP_ME_F32_INTERRUPT
+#define CP_ME_F32_INTERRUPT__ECC_ERROR_INT__SHIFT                                                             0x0
+#define CP_ME_F32_INTERRUPT__TIME_STAMP_INT__SHIFT                                                            0x1
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_2__SHIFT                                                              0x2
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_3__SHIFT                                                              0x3
+#define CP_ME_F32_INTERRUPT__ECC_ERROR_INT_MASK                                                               0x00000001L
+#define CP_ME_F32_INTERRUPT__TIME_STAMP_INT_MASK                                                              0x00000002L
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_2_MASK                                                                0x00000004L
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_3_MASK                                                                0x00000008L
+//CP_PFP_F32_INTERRUPT
+#define CP_PFP_F32_INTERRUPT__ECC_ERROR_INT__SHIFT                                                            0x0
+#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                             0x1
+#define CP_PFP_F32_INTERRUPT__RESERVED_BIT_ERR_INT__SHIFT                                                     0x2
+#define CP_PFP_F32_INTERRUPT__PFP_F32_INT_3__SHIFT                                                            0x3
+#define CP_PFP_F32_INTERRUPT__ECC_ERROR_INT_MASK                                                              0x00000001L
+#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT_MASK                                                               0x00000002L
+#define CP_PFP_F32_INTERRUPT__RESERVED_BIT_ERR_INT_MASK                                                       0x00000004L
+#define CP_PFP_F32_INTERRUPT__PFP_F32_INT_3_MASK                                                              0x00000008L
+//CP_CE_F32_INTERRUPT
+#define CP_CE_F32_INTERRUPT__ECC_ERROR_INT__SHIFT                                                             0x0
+#define CP_CE_F32_INTERRUPT__RESERVED_BIT_ERR_INT__SHIFT                                                      0x1
+#define CP_CE_F32_INTERRUPT__CE_F32_INT_2__SHIFT                                                              0x2
+#define CP_CE_F32_INTERRUPT__CE_F32_INT_3__SHIFT                                                              0x3
+#define CP_CE_F32_INTERRUPT__ECC_ERROR_INT_MASK                                                               0x00000001L
+#define CP_CE_F32_INTERRUPT__RESERVED_BIT_ERR_INT_MASK                                                        0x00000002L
+#define CP_CE_F32_INTERRUPT__CE_F32_INT_2_MASK                                                                0x00000004L
+#define CP_CE_F32_INTERRUPT__CE_F32_INT_3_MASK                                                                0x00000008L
+//CP_MEC1_F32_INTERRUPT
+#define CP_MEC1_F32_INTERRUPT__EDC_ROQ_FED_INT__SHIFT                                                         0x0
+#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                            0x1
+#define CP_MEC1_F32_INTERRUPT__RESERVED_BIT_ERR_INT__SHIFT                                                    0x2
+#define CP_MEC1_F32_INTERRUPT__EDC_TC_FED_INT__SHIFT                                                          0x3
+#define CP_MEC1_F32_INTERRUPT__EDC_GDS_FED_INT__SHIFT                                                         0x4
+#define CP_MEC1_F32_INTERRUPT__EDC_SCRATCH_FED_INT__SHIFT                                                     0x5
+#define CP_MEC1_F32_INTERRUPT__WAVE_RESTORE_INT__SHIFT                                                        0x6
+#define CP_MEC1_F32_INTERRUPT__SUA_VIOLATION_INT__SHIFT                                                       0x7
+#define CP_MEC1_F32_INTERRUPT__EDC_DMA_FED_INT__SHIFT                                                         0x8
+#define CP_MEC1_F32_INTERRUPT__IQ_TIMER_INT__SHIFT                                                            0x9
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPF__SHIFT                                                             0xa
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_DMA__SHIFT                                                             0xb
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPC__SHIFT                                                             0xc
+#define CP_MEC1_F32_INTERRUPT__EDC_SR_MEM_FED_INT__SHIFT                                                      0xd
+#define CP_MEC1_F32_INTERRUPT__QUEUE_MESSAGE_INT__SHIFT                                                       0xe
+#define CP_MEC1_F32_INTERRUPT__FATAL_EDC_ERROR_INT__SHIFT                                                     0xf
+#define CP_MEC1_F32_INTERRUPT__EDC_ROQ_FED_INT_MASK                                                           0x00000001L
+#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT_MASK                                                              0x00000002L
+#define CP_MEC1_F32_INTERRUPT__RESERVED_BIT_ERR_INT_MASK                                                      0x00000004L
+#define CP_MEC1_F32_INTERRUPT__EDC_TC_FED_INT_MASK                                                            0x00000008L
+#define CP_MEC1_F32_INTERRUPT__EDC_GDS_FED_INT_MASK                                                           0x00000010L
+#define CP_MEC1_F32_INTERRUPT__EDC_SCRATCH_FED_INT_MASK                                                       0x00000020L
+#define CP_MEC1_F32_INTERRUPT__WAVE_RESTORE_INT_MASK                                                          0x00000040L
+#define CP_MEC1_F32_INTERRUPT__SUA_VIOLATION_INT_MASK                                                         0x00000080L
+#define CP_MEC1_F32_INTERRUPT__EDC_DMA_FED_INT_MASK                                                           0x00000100L
+#define CP_MEC1_F32_INTERRUPT__IQ_TIMER_INT_MASK                                                              0x00000200L
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPF_MASK                                                               0x00000400L
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_DMA_MASK                                                               0x00000800L
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPC_MASK                                                               0x00001000L
+#define CP_MEC1_F32_INTERRUPT__EDC_SR_MEM_FED_INT_MASK                                                        0x00002000L
+#define CP_MEC1_F32_INTERRUPT__QUEUE_MESSAGE_INT_MASK                                                         0x00004000L
+#define CP_MEC1_F32_INTERRUPT__FATAL_EDC_ERROR_INT_MASK                                                       0x00008000L
+//CP_MEC2_F32_INTERRUPT
+#define CP_MEC2_F32_INTERRUPT__EDC_ROQ_FED_INT__SHIFT                                                         0x0
+#define CP_MEC2_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                            0x1
+#define CP_MEC2_F32_INTERRUPT__RESERVED_BIT_ERR_INT__SHIFT                                                    0x2
+#define CP_MEC2_F32_INTERRUPT__EDC_TC_FED_INT__SHIFT                                                          0x3
+#define CP_MEC2_F32_INTERRUPT__EDC_GDS_FED_INT__SHIFT                                                         0x4
+#define CP_MEC2_F32_INTERRUPT__EDC_SCRATCH_FED_INT__SHIFT                                                     0x5
+#define CP_MEC2_F32_INTERRUPT__WAVE_RESTORE_INT__SHIFT                                                        0x6
+#define CP_MEC2_F32_INTERRUPT__SUA_VIOLATION_INT__SHIFT                                                       0x7
+#define CP_MEC2_F32_INTERRUPT__EDC_DMA_FED_INT__SHIFT                                                         0x8
+#define CP_MEC2_F32_INTERRUPT__IQ_TIMER_INT__SHIFT                                                            0x9
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_CPF__SHIFT                                                             0xa
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_DMA__SHIFT                                                             0xb
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_CPC__SHIFT                                                             0xc
+#define CP_MEC2_F32_INTERRUPT__EDC_SR_MEM_FED_INT__SHIFT                                                      0xd
+#define CP_MEC2_F32_INTERRUPT__QUEUE_MESSAGE_INT__SHIFT                                                       0xe
+#define CP_MEC2_F32_INTERRUPT__FATAL_EDC_ERROR_INT__SHIFT                                                     0xf
+#define CP_MEC2_F32_INTERRUPT__EDC_ROQ_FED_INT_MASK                                                           0x00000001L
+#define CP_MEC2_F32_INTERRUPT__PRIV_REG_INT_MASK                                                              0x00000002L
+#define CP_MEC2_F32_INTERRUPT__RESERVED_BIT_ERR_INT_MASK                                                      0x00000004L
+#define CP_MEC2_F32_INTERRUPT__EDC_TC_FED_INT_MASK                                                            0x00000008L
+#define CP_MEC2_F32_INTERRUPT__EDC_GDS_FED_INT_MASK                                                           0x00000010L
+#define CP_MEC2_F32_INTERRUPT__EDC_SCRATCH_FED_INT_MASK                                                       0x00000020L
+#define CP_MEC2_F32_INTERRUPT__WAVE_RESTORE_INT_MASK                                                          0x00000040L
+#define CP_MEC2_F32_INTERRUPT__SUA_VIOLATION_INT_MASK                                                         0x00000080L
+#define CP_MEC2_F32_INTERRUPT__EDC_DMA_FED_INT_MASK                                                           0x00000100L
+#define CP_MEC2_F32_INTERRUPT__IQ_TIMER_INT_MASK                                                              0x00000200L
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_CPF_MASK                                                               0x00000400L
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_DMA_MASK                                                               0x00000800L
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_CPC_MASK                                                               0x00001000L
+#define CP_MEC2_F32_INTERRUPT__EDC_SR_MEM_FED_INT_MASK                                                        0x00002000L
+#define CP_MEC2_F32_INTERRUPT__QUEUE_MESSAGE_INT_MASK                                                         0x00004000L
+#define CP_MEC2_F32_INTERRUPT__FATAL_EDC_ERROR_INT_MASK                                                       0x00008000L
+//CP_PWR_CNTL
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE0__SHIFT                                                            0x0
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE1__SHIFT                                                            0x1
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE0__SHIFT                                                            0x8
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE1__SHIFT                                                            0x9
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE2__SHIFT                                                            0xa
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE3__SHIFT                                                            0xb
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE0__SHIFT                                                            0x10
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE1__SHIFT                                                            0x11
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE2__SHIFT                                                            0x12
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE3__SHIFT                                                            0x13
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE0_MASK                                                              0x00000001L
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE1_MASK                                                              0x00000002L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE0_MASK                                                              0x00000100L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE1_MASK                                                              0x00000200L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE2_MASK                                                              0x00000400L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE3_MASK                                                              0x00000800L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE0_MASK                                                              0x00010000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE1_MASK                                                              0x00020000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE2_MASK                                                              0x00040000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE3_MASK                                                              0x00080000L
+//CP_MEM_SLP_CNTL
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_EN__SHIFT                                                                  0x0
+#define CP_MEM_SLP_CNTL__CP_MEM_DS_EN__SHIFT                                                                  0x1
+#define CP_MEM_SLP_CNTL__RESERVED__SHIFT                                                                      0x2
+#define CP_MEM_SLP_CNTL__CP_LS_DS_BUSY_OVERRIDE__SHIFT                                                        0x7
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_ON_DELAY__SHIFT                                                            0x8
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_OFF_DELAY__SHIFT                                                           0x10
+#define CP_MEM_SLP_CNTL__RESERVED1__SHIFT                                                                     0x18
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK                                                                    0x00000001L
+#define CP_MEM_SLP_CNTL__CP_MEM_DS_EN_MASK                                                                    0x00000002L
+#define CP_MEM_SLP_CNTL__RESERVED_MASK                                                                        0x0000007CL
+#define CP_MEM_SLP_CNTL__CP_LS_DS_BUSY_OVERRIDE_MASK                                                          0x00000080L
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_ON_DELAY_MASK                                                              0x0000FF00L
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_OFF_DELAY_MASK                                                             0x00FF0000L
+#define CP_MEM_SLP_CNTL__RESERVED1_MASK                                                                       0xFF000000L
+//CP_ECC_DMA_FIRST_OCCURRENCE
+#define CP_ECC_DMA_FIRST_OCCURRENCE__INTERFACE__SHIFT                                                         0x0
+#define CP_ECC_DMA_FIRST_OCCURRENCE__CLIENT__SHIFT                                                            0x4
+#define CP_ECC_DMA_FIRST_OCCURRENCE__ME__SHIFT                                                                0x8
+#define CP_ECC_DMA_FIRST_OCCURRENCE__PIPE__SHIFT                                                              0xa
+#define CP_ECC_DMA_FIRST_OCCURRENCE__VMID__SHIFT                                                              0x10
+#define CP_ECC_DMA_FIRST_OCCURRENCE__INTERFACE_MASK                                                           0x00000003L
+#define CP_ECC_DMA_FIRST_OCCURRENCE__CLIENT_MASK                                                              0x000000F0L
+#define CP_ECC_DMA_FIRST_OCCURRENCE__ME_MASK                                                                  0x00000300L
+#define CP_ECC_DMA_FIRST_OCCURRENCE__PIPE_MASK                                                                0x00000C00L
+#define CP_ECC_DMA_FIRST_OCCURRENCE__VMID_MASK                                                                0x000F0000L
+//CP_ECC_FIRSTOCCURRENCE
+#define CP_ECC_FIRSTOCCURRENCE__INTERFACE__SHIFT                                                              0x0
+#define CP_ECC_FIRSTOCCURRENCE__CLIENT__SHIFT                                                                 0x4
+#define CP_ECC_FIRSTOCCURRENCE__ME__SHIFT                                                                     0x8
+#define CP_ECC_FIRSTOCCURRENCE__PIPE__SHIFT                                                                   0xa
+#define CP_ECC_FIRSTOCCURRENCE__QUEUE__SHIFT                                                                  0xc
+#define CP_ECC_FIRSTOCCURRENCE__VMID__SHIFT                                                                   0x10
+#define CP_ECC_FIRSTOCCURRENCE__INTERFACE_MASK                                                                0x00000003L
+#define CP_ECC_FIRSTOCCURRENCE__CLIENT_MASK                                                                   0x000000F0L
+#define CP_ECC_FIRSTOCCURRENCE__ME_MASK                                                                       0x00000300L
+#define CP_ECC_FIRSTOCCURRENCE__PIPE_MASK                                                                     0x00000C00L
+#define CP_ECC_FIRSTOCCURRENCE__QUEUE_MASK                                                                    0x00007000L
+#define CP_ECC_FIRSTOCCURRENCE__VMID_MASK                                                                     0x000F0000L
+//CP_ECC_FIRSTOCCURRENCE_RING0
+#define CP_ECC_FIRSTOCCURRENCE_RING0__OBSOLETE__SHIFT                                                         0x0
+#define CP_ECC_FIRSTOCCURRENCE_RING0__OBSOLETE_MASK                                                           0xFFFFFFFFL
+//CP_ECC_FIRSTOCCURRENCE_RING1
+#define CP_ECC_FIRSTOCCURRENCE_RING1__OBSOLETE__SHIFT                                                         0x0
+#define CP_ECC_FIRSTOCCURRENCE_RING1__OBSOLETE_MASK                                                           0xFFFFFFFFL
+//CP_ECC_FIRSTOCCURRENCE_RING2
+#define CP_ECC_FIRSTOCCURRENCE_RING2__OBSOLETE__SHIFT                                                         0x0
+#define CP_ECC_FIRSTOCCURRENCE_RING2__OBSOLETE_MASK                                                           0xFFFFFFFFL
+//GB_EDC_MODE
+#define GB_EDC_MODE__FORCE_SEC_ON_DED__SHIFT                                                                  0xf
+#define GB_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                     0x10
+#define GB_EDC_MODE__GATE_FUE__SHIFT                                                                          0x11
+#define GB_EDC_MODE__DED_MODE__SHIFT                                                                          0x14
+#define GB_EDC_MODE__PROP_FED__SHIFT                                                                          0x1d
+#define GB_EDC_MODE__BYPASS__SHIFT                                                                            0x1f
+#define GB_EDC_MODE__FORCE_SEC_ON_DED_MASK                                                                    0x00008000L
+#define GB_EDC_MODE__COUNT_FED_OUT_MASK                                                                       0x00010000L
+#define GB_EDC_MODE__GATE_FUE_MASK                                                                            0x00020000L
+#define GB_EDC_MODE__DED_MODE_MASK                                                                            0x00300000L
+#define GB_EDC_MODE__PROP_FED_MASK                                                                            0x20000000L
+#define GB_EDC_MODE__BYPASS_MASK                                                                              0x80000000L
+//CP_DEBUG
+#define CP_DEBUG__CPG_REPEATER_FGCG_OVERRIDE__SHIFT                                                           0x9
+#define CP_DEBUG__CPG_RAM_CLK_GATING_DISABLE__SHIFT                                                           0xe
+#define CP_DEBUG__DISABLE_GFX_HALT_ON_UTCL1_ERROR__SHIFT                                                      0xf
+#define CP_DEBUG__SURFSYNC_CNTX_RDADDR__SHIFT                                                                 0x10
+#define CP_DEBUG__BUSY_EXTENDER__SHIFT                                                                        0x13
+#define CP_DEBUG__PRIV_REG_INTERRUPT_ENABLE__SHIFT                                                            0x15
+#define CP_DEBUG__INTERRUPT_ENABLE__SHIFT                                                                     0x16
+#define CP_DEBUG__PREDICATE_DISABLE__SHIFT                                                                    0x17
+#define CP_DEBUG__UNDERFLOW_BUSY_DISABLE__SHIFT                                                               0x18
+#define CP_DEBUG__OVERFLOW_BUSY_DISABLE__SHIFT                                                                0x19
+#define CP_DEBUG__EVENT_FILT_DISABLE__SHIFT                                                                   0x1a
+#define CP_DEBUG__CPG_TC_MTYPE_OVERRIDE__SHIFT                                                                0x1b
+#define CP_DEBUG__CPG_TC_ONE_CYCLE_WRITE_DISABLE__SHIFT                                                       0x1c
+#define CP_DEBUG__CS_STATE_FILT_DISABLE__SHIFT                                                                0x1d
+#define CP_DEBUG__CS_PIPELINE_RESET_DISABLE__SHIFT                                                            0x1e
+#define CP_DEBUG__IB_PACKET_INJECTOR_DISABLE__SHIFT                                                           0x1f
+#define CP_DEBUG__CPG_REPEATER_FGCG_OVERRIDE_MASK                                                             0x00000200L
+#define CP_DEBUG__CPG_RAM_CLK_GATING_DISABLE_MASK                                                             0x00004000L
+#define CP_DEBUG__DISABLE_GFX_HALT_ON_UTCL1_ERROR_MASK                                                        0x00008000L
+#define CP_DEBUG__SURFSYNC_CNTX_RDADDR_MASK                                                                   0x00070000L
+#define CP_DEBUG__BUSY_EXTENDER_MASK                                                                          0x00180000L
+#define CP_DEBUG__PRIV_REG_INTERRUPT_ENABLE_MASK                                                              0x00200000L
+#define CP_DEBUG__INTERRUPT_ENABLE_MASK                                                                       0x00400000L
+#define CP_DEBUG__PREDICATE_DISABLE_MASK                                                                      0x00800000L
+#define CP_DEBUG__UNDERFLOW_BUSY_DISABLE_MASK                                                                 0x01000000L
+#define CP_DEBUG__OVERFLOW_BUSY_DISABLE_MASK                                                                  0x02000000L
+#define CP_DEBUG__EVENT_FILT_DISABLE_MASK                                                                     0x04000000L
+#define CP_DEBUG__CPG_TC_MTYPE_OVERRIDE_MASK                                                                  0x08000000L
+#define CP_DEBUG__CPG_TC_ONE_CYCLE_WRITE_DISABLE_MASK                                                         0x10000000L
+#define CP_DEBUG__CS_STATE_FILT_DISABLE_MASK                                                                  0x20000000L
+#define CP_DEBUG__CS_PIPELINE_RESET_DISABLE_MASK                                                              0x40000000L
+#define CP_DEBUG__IB_PACKET_INJECTOR_DISABLE_MASK                                                             0x80000000L
+//CP_CPF_DEBUG
+#define CP_CPF_DEBUG__QUE_MANAGER_CLR_DBELLUPD_DIS__SHIFT                                                     0x6
+#define CP_CPF_DEBUG__CPF_REPEATER_FGCG_OVERRIDE__SHIFT                                                       0x10
+#define CP_CPF_DEBUG__CPF_RAM_CLK_GATING_DISABLE__SHIFT                                                       0x12
+#define CP_CPF_DEBUG__BUSY_EXTENDER__SHIFT                                                                    0x13
+#define CP_CPF_DEBUG__UNDERFLOW_BUSY_DISABLE__SHIFT                                                           0x18
+#define CP_CPF_DEBUG__OVERFLOW_BUSY_DISABLE__SHIFT                                                            0x19
+#define CP_CPF_DEBUG__CPF_PRIORITY_YIELD_ACTIVE_DIS__SHIFT                                                    0x1d
+#define CP_CPF_DEBUG__CPF_TC_MTYPE_OVERRIDE__SHIFT                                                            0x1e
+#define CP_CPF_DEBUG__DBGU_TRIGGER__SHIFT                                                                     0x1f
+#define CP_CPF_DEBUG__QUE_MANAGER_CLR_DBELLUPD_DIS_MASK                                                       0x00000040L
+#define CP_CPF_DEBUG__CPF_REPEATER_FGCG_OVERRIDE_MASK                                                         0x00010000L
+#define CP_CPF_DEBUG__CPF_RAM_CLK_GATING_DISABLE_MASK                                                         0x00040000L
+#define CP_CPF_DEBUG__BUSY_EXTENDER_MASK                                                                      0x00180000L
+#define CP_CPF_DEBUG__UNDERFLOW_BUSY_DISABLE_MASK                                                             0x01000000L
+#define CP_CPF_DEBUG__OVERFLOW_BUSY_DISABLE_MASK                                                              0x02000000L
+#define CP_CPF_DEBUG__CPF_PRIORITY_YIELD_ACTIVE_DIS_MASK                                                      0x20000000L
+#define CP_CPF_DEBUG__CPF_TC_MTYPE_OVERRIDE_MASK                                                              0x40000000L
+#define CP_CPF_DEBUG__DBGU_TRIGGER_MASK                                                                       0x80000000L
+//CP_CPC_DEBUG
+#define CP_CPC_DEBUG__CPC_PIPE_SEL__SHIFT                                                                     0x0
+#define CP_CPC_DEBUG__CPC_ROLLOVER_EVENT_DEBUG_EN__SHIFT                                                      0x3
+#define CP_CPC_DEBUG__CPC_HARVESTING_CONTINUE_DISABLE__SHIFT                                                  0xb
+#define CP_CPC_DEBUG__CPC_HARVESTING_RELAUNCH_DISABLE__SHIFT                                                  0xc
+#define CP_CPC_DEBUG__CPC_HARVEST_AUTO_DISABLE__SHIFT                                                         0xd
+#define CP_CPC_DEBUG__CPC_HARVESTING_DISPATCH_DISABLE__SHIFT                                                  0xe
+#define CP_CPC_DEBUG__CPC_REPEATER_FGCG_OVERRIDE__SHIFT                                                       0xf
+#define CP_CPC_DEBUG__CPC_RAM_CLK_GATING_DISABLE__SHIFT                                                       0x12
+#define CP_CPC_DEBUG__BUSY_EXTENDER__SHIFT                                                                    0x13
+#define CP_CPC_DEBUG__UCODE_ECC_ERROR_DISABLE__SHIFT                                                          0x15
+#define CP_CPC_DEBUG__RESERVED_INTERRUPT_ENABLE__SHIFT                                                        0x16
+#define CP_CPC_DEBUG__RESTORE_FIFO_EMPTY_SEL__SHIFT                                                           0x17
+#define CP_CPC_DEBUG__UNDERFLOW_BUSY_DISABLE__SHIFT                                                           0x18
+#define CP_CPC_DEBUG__OVERFLOW_BUSY_DISABLE__SHIFT                                                            0x19
+#define CP_CPC_DEBUG__EVENT_FILT_DISABLE__SHIFT                                                               0x1a
+#define CP_CPC_DEBUG__PRIV_REG_INTERRUPT_ENABLE__SHIFT                                                        0x1b
+#define CP_CPC_DEBUG__CPC_TC_ONE_CYCLE_WRITE_DISABLE__SHIFT                                                   0x1c
+#define CP_CPC_DEBUG__CS_STATE_FILT_DISABLE__SHIFT                                                            0x1d
+#define CP_CPC_DEBUG__ME2_UCODE_RAM_ENABLE__SHIFT                                                             0x1f
+#define CP_CPC_DEBUG__CPC_PIPE_SEL_MASK                                                                       0x00000003L
+#define CP_CPC_DEBUG__CPC_ROLLOVER_EVENT_DEBUG_EN_MASK                                                        0x00000008L
+#define CP_CPC_DEBUG__CPC_HARVESTING_CONTINUE_DISABLE_MASK                                                    0x00000800L
+#define CP_CPC_DEBUG__CPC_HARVESTING_RELAUNCH_DISABLE_MASK                                                    0x00001000L
+#define CP_CPC_DEBUG__CPC_HARVEST_AUTO_DISABLE_MASK                                                           0x00002000L
+#define CP_CPC_DEBUG__CPC_HARVESTING_DISPATCH_DISABLE_MASK                                                    0x00004000L
+#define CP_CPC_DEBUG__CPC_REPEATER_FGCG_OVERRIDE_MASK                                                         0x00008000L
+#define CP_CPC_DEBUG__CPC_RAM_CLK_GATING_DISABLE_MASK                                                         0x00040000L
+#define CP_CPC_DEBUG__BUSY_EXTENDER_MASK                                                                      0x00180000L
+#define CP_CPC_DEBUG__UCODE_ECC_ERROR_DISABLE_MASK                                                            0x00200000L
+#define CP_CPC_DEBUG__RESERVED_INTERRUPT_ENABLE_MASK                                                          0x00400000L
+#define CP_CPC_DEBUG__RESTORE_FIFO_EMPTY_SEL_MASK                                                             0x00800000L
+#define CP_CPC_DEBUG__UNDERFLOW_BUSY_DISABLE_MASK                                                             0x01000000L
+#define CP_CPC_DEBUG__OVERFLOW_BUSY_DISABLE_MASK                                                              0x02000000L
+#define CP_CPC_DEBUG__EVENT_FILT_DISABLE_MASK                                                                 0x04000000L
+#define CP_CPC_DEBUG__PRIV_REG_INTERRUPT_ENABLE_MASK                                                          0x08000000L
+#define CP_CPC_DEBUG__CPC_TC_ONE_CYCLE_WRITE_DISABLE_MASK                                                     0x10000000L
+#define CP_CPC_DEBUG__CS_STATE_FILT_DISABLE_MASK                                                              0x20000000L
+#define CP_CPC_DEBUG__ME2_UCODE_RAM_ENABLE_MASK                                                               0x80000000L
+//CP_CPC_DEBUG_2
+#define CP_CPC_DEBUG_2__DC_GD_PIPE3_QSWITCH_CNT__SHIFT                                                        0x0
+#define CP_CPC_DEBUG_2__DC_GD_PIPE2_QSWITCH_CNT__SHIFT                                                        0x8
+#define CP_CPC_DEBUG_2__DC_GD_PIPE1_QSWITCH_CNT__SHIFT                                                        0x10
+#define CP_CPC_DEBUG_2__DC_GD_PIPE0_QSWITCH_CNT__SHIFT                                                        0x18
+#define CP_CPC_DEBUG_2__DC_GD_PIPE3_QSWITCH_CNT_MASK                                                          0x000000FFL
+#define CP_CPC_DEBUG_2__DC_GD_PIPE2_QSWITCH_CNT_MASK                                                          0x0000FF00L
+#define CP_CPC_DEBUG_2__DC_GD_PIPE1_QSWITCH_CNT_MASK                                                          0x00FF0000L
+#define CP_CPC_DEBUG_2__DC_GD_PIPE0_QSWITCH_CNT_MASK                                                          0xFF000000L
+//CP_PQ_WPTR_POLL_CNTL
+#define CP_PQ_WPTR_POLL_CNTL__PERIOD__SHIFT                                                                   0x0
+#define CP_PQ_WPTR_POLL_CNTL__DISABLE_PEND_REQ_ONE_SHOT__SHIFT                                                0x1d
+#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE__SHIFT                                                              0x1e
+#define CP_PQ_WPTR_POLL_CNTL__EN__SHIFT                                                                       0x1f
+#define CP_PQ_WPTR_POLL_CNTL__PERIOD_MASK                                                                     0x000000FFL
+#define CP_PQ_WPTR_POLL_CNTL__DISABLE_PEND_REQ_ONE_SHOT_MASK                                                  0x20000000L
+#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE_MASK                                                                0x40000000L
+#define CP_PQ_WPTR_POLL_CNTL__EN_MASK                                                                         0x80000000L
+//CP_PQ_WPTR_POLL_CNTL1
+#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK__SHIFT                                                              0x0
+#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK_MASK                                                                0xFFFFFFFFL
+//CP_ME1_PIPE0_INT_CNTL
+#define CP_ME1_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE0_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE0_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE1_INT_CNTL
+#define CP_ME1_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE1_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE1_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE2_INT_CNTL
+#define CP_ME1_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE2_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE2_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE3_INT_CNTL
+#define CP_ME1_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE3_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE3_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME2_PIPE0_INT_CNTL
+#define CP_ME2_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME2_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME2_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME2_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME2_PIPE0_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME2_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME2_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME2_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME2_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME2_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME2_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME2_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME2_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME2_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME2_PIPE0_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME2_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME2_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME2_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME2_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME2_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME2_PIPE1_INT_CNTL
+#define CP_ME2_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME2_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME2_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME2_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME2_PIPE1_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME2_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME2_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME2_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME2_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME2_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME2_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME2_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME2_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME2_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME2_PIPE1_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME2_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME2_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME2_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME2_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME2_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME2_PIPE2_INT_CNTL
+#define CP_ME2_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME2_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME2_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME2_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME2_PIPE2_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME2_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME2_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME2_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME2_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME2_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME2_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME2_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME2_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME2_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME2_PIPE2_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME2_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME2_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME2_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME2_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME2_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME2_PIPE3_INT_CNTL
+#define CP_ME2_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME2_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME2_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME2_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME2_PIPE3_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME2_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME2_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME2_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME2_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME2_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME2_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME2_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME2_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME2_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME2_PIPE3_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME2_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME2_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME2_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME2_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME2_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE0_INT_STATUS
+#define CP_ME1_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE0_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE0_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_PIPE1_INT_STATUS
+#define CP_ME1_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE1_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE1_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_PIPE2_INT_STATUS
+#define CP_ME1_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE2_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE2_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_PIPE3_INT_STATUS
+#define CP_ME1_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE3_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE3_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME2_PIPE0_INT_STATUS
+#define CP_ME2_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME2_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME2_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME2_PIPE0_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME2_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME2_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME2_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME2_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME2_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME2_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME2_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME2_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME2_PIPE0_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME2_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME2_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME2_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME2_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME2_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME2_PIPE1_INT_STATUS
+#define CP_ME2_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME2_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME2_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME2_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME2_PIPE1_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME2_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME2_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME2_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME2_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME2_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME2_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME2_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME2_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME2_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME2_PIPE1_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME2_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME2_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME2_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME2_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME2_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME2_PIPE2_INT_STATUS
+#define CP_ME2_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME2_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME2_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME2_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME2_PIPE2_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME2_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME2_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME2_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME2_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME2_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME2_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME2_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME2_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME2_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME2_PIPE2_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME2_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME2_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME2_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME2_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME2_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME2_PIPE3_INT_STATUS
+#define CP_ME2_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME2_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME2_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME2_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME2_PIPE3_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME2_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME2_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME2_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME2_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME2_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME2_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME2_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME2_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME2_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME2_PIPE3_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME2_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME2_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME2_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME2_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME2_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_INT_STAT_DEBUG
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
+//CP_ME2_INT_STAT_DEBUG
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
+//CC_GC_EDC_CONFIG
+#define CC_GC_EDC_CONFIG__WRITE_DIS__SHIFT                                                                    0x0
+#define CC_GC_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
+#define CC_GC_EDC_CONFIG__ENABLE_IRRITATOR_CLK__SHIFT                                                         0x2
+#define CC_GC_EDC_CONFIG__WRITE_DIS_MASK                                                                      0x00000001L
+#define CC_GC_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
+#define CC_GC_EDC_CONFIG__ENABLE_IRRITATOR_CLK_MASK                                                           0x00000004L
+//CP_ME1_PIPE_PRIORITY_CNTS
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
+//CP_ME1_PIPE0_PRIORITY
+#define CP_ME1_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME1_PIPE1_PRIORITY
+#define CP_ME1_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME1_PIPE2_PRIORITY
+#define CP_ME1_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME1_PIPE3_PRIORITY
+#define CP_ME1_PIPE3_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE3_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME2_PIPE_PRIORITY_CNTS
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
+//CP_ME2_PIPE0_PRIORITY
+#define CP_ME2_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME2_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME2_PIPE1_PRIORITY
+#define CP_ME2_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME2_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME2_PIPE2_PRIORITY
+#define CP_ME2_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME2_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME2_PIPE3_PRIORITY
+#define CP_ME2_PIPE3_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME2_PIPE3_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_CE_PRGRM_CNTR_START
+#define CP_CE_PRGRM_CNTR_START__IP_START__SHIFT                                                               0x0
+#define CP_CE_PRGRM_CNTR_START__IP_START_MASK                                                                 0x000007FFL
+//CP_PFP_PRGRM_CNTR_START
+#define CP_PFP_PRGRM_CNTR_START__IP_START__SHIFT                                                              0x0
+#define CP_PFP_PRGRM_CNTR_START__IP_START_MASK                                                                0x00001FFFL
+//CP_ME_PRGRM_CNTR_START
+#define CP_ME_PRGRM_CNTR_START__IP_START__SHIFT                                                               0x0
+#define CP_ME_PRGRM_CNTR_START__IP_START_MASK                                                                 0x00000FFFL
+//CP_MEC1_PRGRM_CNTR_START
+#define CP_MEC1_PRGRM_CNTR_START__IP_START__SHIFT                                                             0x0
+#define CP_MEC1_PRGRM_CNTR_START__IP_START_MASK                                                               0x0000FFFFL
+//CP_MEC2_PRGRM_CNTR_START
+#define CP_MEC2_PRGRM_CNTR_START__IP_START__SHIFT                                                             0x0
+#define CP_MEC2_PRGRM_CNTR_START__IP_START_MASK                                                               0x0000FFFFL
+//CP_CE_INTR_ROUTINE_START
+#define CP_CE_INTR_ROUTINE_START__IR_START__SHIFT                                                             0x0
+#define CP_CE_INTR_ROUTINE_START__IR_START_MASK                                                               0x000007FFL
+//CP_PFP_INTR_ROUTINE_START
+#define CP_PFP_INTR_ROUTINE_START__IR_START__SHIFT                                                            0x0
+#define CP_PFP_INTR_ROUTINE_START__IR_START_MASK                                                              0x00001FFFL
+//CP_ME_INTR_ROUTINE_START
+#define CP_ME_INTR_ROUTINE_START__IR_START__SHIFT                                                             0x0
+#define CP_ME_INTR_ROUTINE_START__IR_START_MASK                                                               0x00000FFFL
+//CP_MEC1_INTR_ROUTINE_START
+#define CP_MEC1_INTR_ROUTINE_START__IR_START__SHIFT                                                           0x0
+#define CP_MEC1_INTR_ROUTINE_START__IR_START_MASK                                                             0x0000FFFFL
+//CP_MEC2_INTR_ROUTINE_START
+#define CP_MEC2_INTR_ROUTINE_START__IR_START__SHIFT                                                           0x0
+#define CP_MEC2_INTR_ROUTINE_START__IR_START_MASK                                                             0x0000FFFFL
+//CP_CONTEXT_CNTL
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_WD_CNTX__SHIFT                                                          0x0
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX__SHIFT                                                        0x4
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_WD_CNTX__SHIFT                                                          0x10
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX__SHIFT                                                        0x14
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_WD_CNTX_MASK                                                            0x00000007L
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX_MASK                                                          0x00000070L
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_WD_CNTX_MASK                                                            0x00070000L
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX_MASK                                                          0x00700000L
+//CP_MAX_CONTEXT
+#define CP_MAX_CONTEXT__MAX_CONTEXT__SHIFT                                                                    0x0
+#define CP_MAX_CONTEXT__MAX_CONTEXT_MASK                                                                      0x00000007L
+//CP_IQ_WAIT_TIME1
+#define CP_IQ_WAIT_TIME1__IB_OFFLOAD__SHIFT                                                                   0x0
+#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD__SHIFT                                                               0x8
+#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD__SHIFT                                                                  0x10
+#define CP_IQ_WAIT_TIME1__GWS__SHIFT                                                                          0x18
+#define CP_IQ_WAIT_TIME1__IB_OFFLOAD_MASK                                                                     0x000000FFL
+#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD_MASK                                                                 0x0000FF00L
+#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD_MASK                                                                    0x00FF0000L
+#define CP_IQ_WAIT_TIME1__GWS_MASK                                                                            0xFF000000L
+//CP_IQ_WAIT_TIME2
+#define CP_IQ_WAIT_TIME2__QUE_SLEEP__SHIFT                                                                    0x0
+#define CP_IQ_WAIT_TIME2__SCH_WAVE__SHIFT                                                                     0x8
+#define CP_IQ_WAIT_TIME2__SEM_REARM__SHIFT                                                                    0x10
+#define CP_IQ_WAIT_TIME2__DEQ_RETRY__SHIFT                                                                    0x18
+#define CP_IQ_WAIT_TIME2__QUE_SLEEP_MASK                                                                      0x000000FFL
+#define CP_IQ_WAIT_TIME2__SCH_WAVE_MASK                                                                       0x0000FF00L
+#define CP_IQ_WAIT_TIME2__SEM_REARM_MASK                                                                      0x00FF0000L
+#define CP_IQ_WAIT_TIME2__DEQ_RETRY_MASK                                                                      0xFF000000L
+//CP_RB0_BASE_HI
+#define CP_RB0_BASE_HI__RB_BASE_HI__SHIFT                                                                     0x0
+#define CP_RB0_BASE_HI__RB_BASE_HI_MASK                                                                       0x000000FFL
+//CP_RB1_BASE_HI
+#define CP_RB1_BASE_HI__RB_BASE_HI__SHIFT                                                                     0x0
+#define CP_RB1_BASE_HI__RB_BASE_HI_MASK                                                                       0x000000FFL
+//CP_VMID_RESET
+#define CP_VMID_RESET__RESET_REQUEST__SHIFT                                                                   0x0
+#define CP_VMID_RESET__RESET_REQUEST_MASK                                                                     0x0000FFFFL
+//CPC_INT_CNTL
+#define CPC_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                                      0xc
+#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                                       0xd
+#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                          0xe
+#define CPC_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                         0xf
+#define CPC_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                                   0x10
+#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                      0x11
+#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                              0x17
+#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                          0x18
+#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                            0x1a
+#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                                    0x1b
+#define CPC_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                              0x1d
+#define CPC_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                              0x1e
+#define CPC_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                              0x1f
+#define CPC_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                                        0x00001000L
+#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                         0x00002000L
+#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                            0x00004000L
+#define CPC_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                           0x00008000L
+#define CPC_INT_CNTL__GPF_INT_ENABLE_MASK                                                                     0x00010000L
+#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                        0x00020000L
+#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                                0x00800000L
+#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                            0x01000000L
+#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                              0x04000000L
+#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                      0x08000000L
+#define CPC_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                                0x20000000L
+#define CPC_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                                0x40000000L
+#define CPC_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                                0x80000000L
+//CPC_INT_STATUS
+#define CPC_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                                    0xc
+#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                                     0xd
+#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                                        0xe
+#define CPC_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                                       0xf
+#define CPC_INT_STATUS__GPF_INT_STATUS__SHIFT                                                                 0x10
+#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                                    0x11
+#define CPC_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                            0x17
+#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                                        0x18
+#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                          0x1a
+#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                                  0x1b
+#define CPC_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                            0x1d
+#define CPC_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                            0x1e
+#define CPC_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                            0x1f
+#define CPC_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                                      0x00001000L
+#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                                       0x00002000L
+#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                          0x00004000L
+#define CPC_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                         0x00008000L
+#define CPC_INT_STATUS__GPF_INT_STATUS_MASK                                                                   0x00010000L
+#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                                      0x00020000L
+#define CPC_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                              0x00800000L
+#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                          0x01000000L
+#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                            0x04000000L
+#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                                    0x08000000L
+#define CPC_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                              0x20000000L
+#define CPC_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                              0x40000000L
+#define CPC_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                              0x80000000L
+//CP_VMID_PREEMPT
+#define CP_VMID_PREEMPT__PREEMPT_REQUEST__SHIFT                                                               0x0
+#define CP_VMID_PREEMPT__VIRT_COMMAND__SHIFT                                                                  0x10
+#define CP_VMID_PREEMPT__PREEMPT_REQUEST_MASK                                                                 0x0000FFFFL
+#define CP_VMID_PREEMPT__VIRT_COMMAND_MASK                                                                    0x000F0000L
+//CPC_INT_CNTX_ID
+#define CPC_INT_CNTX_ID__CNTX_ID__SHIFT                                                                       0x0
+#define CPC_INT_CNTX_ID__CNTX_ID_MASK                                                                         0xFFFFFFFFL
+//CP_PQ_STATUS
+#define CP_PQ_STATUS__DOORBELL_UPDATED__SHIFT                                                                 0x0
+#define CP_PQ_STATUS__DOORBELL_ENABLE__SHIFT                                                                  0x1
+#define CP_PQ_STATUS__DOORBELL_UPDATED_EN__SHIFT                                                              0x2
+#define CP_PQ_STATUS__DOORBELL_UPDATED_MASK                                                                   0x00000001L
+#define CP_PQ_STATUS__DOORBELL_ENABLE_MASK                                                                    0x00000002L
+#define CP_PQ_STATUS__DOORBELL_UPDATED_EN_MASK                                                                0x00000004L
+//CP_CPC_IC_BASE_LO
+#define CP_CPC_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                  0xc
+#define CP_CPC_IC_BASE_LO__IC_BASE_LO_MASK                                                                    0xFFFFF000L
+//CP_CPC_IC_BASE_HI
+#define CP_CPC_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                  0x0
+#define CP_CPC_IC_BASE_HI__IC_BASE_HI_MASK                                                                    0x0000FFFFL
+//CP_CPC_IC_BASE_CNTL
+#define CP_CPC_IC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_CPC_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_CPC_IC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_CPC_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x01000000L
+//CP_CPC_IC_OP_CNTL
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                            0x0
+#define CP_CPC_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                0x4
+#define CP_CPC_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                               0x5
+#define CP_CPC_IC_OP_CNTL__ICACHE_INVALIDATED__SHIFT                                                          0x6
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                              0x00000001L
+#define CP_CPC_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                  0x00000010L
+#define CP_CPC_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                 0x00000020L
+#define CP_CPC_IC_OP_CNTL__ICACHE_INVALIDATED_MASK                                                            0x00000040L
+//CP_MEC1_F32_INT_DIS
+#define CP_MEC1_F32_INT_DIS__EDC_ROQ_FED_INT__SHIFT                                                           0x0
+#define CP_MEC1_F32_INT_DIS__PRIV_REG_INT__SHIFT                                                              0x1
+#define CP_MEC1_F32_INT_DIS__RESERVED_BIT_ERR_INT__SHIFT                                                      0x2
+#define CP_MEC1_F32_INT_DIS__EDC_TC_FED_INT__SHIFT                                                            0x3
+#define CP_MEC1_F32_INT_DIS__EDC_GDS_FED_INT__SHIFT                                                           0x4
+#define CP_MEC1_F32_INT_DIS__EDC_SCRATCH_FED_INT__SHIFT                                                       0x5
+#define CP_MEC1_F32_INT_DIS__WAVE_RESTORE_INT__SHIFT                                                          0x6
+#define CP_MEC1_F32_INT_DIS__SUA_VIOLATION_INT__SHIFT                                                         0x7
+#define CP_MEC1_F32_INT_DIS__EDC_DMA_FED_INT__SHIFT                                                           0x8
+#define CP_MEC1_F32_INT_DIS__IQ_TIMER_INT__SHIFT                                                              0x9
+#define CP_MEC1_F32_INT_DIS__GPF_INT_CPF__SHIFT                                                               0xa
+#define CP_MEC1_F32_INT_DIS__GPF_INT_DMA__SHIFT                                                               0xb
+#define CP_MEC1_F32_INT_DIS__GPF_INT_CPC__SHIFT                                                               0xc
+#define CP_MEC1_F32_INT_DIS__EDC_SR_MEM_FED_INT__SHIFT                                                        0xd
+#define CP_MEC1_F32_INT_DIS__QUEUE_MESSAGE_INT__SHIFT                                                         0xe
+#define CP_MEC1_F32_INT_DIS__FATAL_EDC_ERROR_INT__SHIFT                                                       0xf
+#define CP_MEC1_F32_INT_DIS__EDC_ROQ_FED_INT_MASK                                                             0x00000001L
+#define CP_MEC1_F32_INT_DIS__PRIV_REG_INT_MASK                                                                0x00000002L
+#define CP_MEC1_F32_INT_DIS__RESERVED_BIT_ERR_INT_MASK                                                        0x00000004L
+#define CP_MEC1_F32_INT_DIS__EDC_TC_FED_INT_MASK                                                              0x00000008L
+#define CP_MEC1_F32_INT_DIS__EDC_GDS_FED_INT_MASK                                                             0x00000010L
+#define CP_MEC1_F32_INT_DIS__EDC_SCRATCH_FED_INT_MASK                                                         0x00000020L
+#define CP_MEC1_F32_INT_DIS__WAVE_RESTORE_INT_MASK                                                            0x00000040L
+#define CP_MEC1_F32_INT_DIS__SUA_VIOLATION_INT_MASK                                                           0x00000080L
+#define CP_MEC1_F32_INT_DIS__EDC_DMA_FED_INT_MASK                                                             0x00000100L
+#define CP_MEC1_F32_INT_DIS__IQ_TIMER_INT_MASK                                                                0x00000200L
+#define CP_MEC1_F32_INT_DIS__GPF_INT_CPF_MASK                                                                 0x00000400L
+#define CP_MEC1_F32_INT_DIS__GPF_INT_DMA_MASK                                                                 0x00000800L
+#define CP_MEC1_F32_INT_DIS__GPF_INT_CPC_MASK                                                                 0x00001000L
+#define CP_MEC1_F32_INT_DIS__EDC_SR_MEM_FED_INT_MASK                                                          0x00002000L
+#define CP_MEC1_F32_INT_DIS__QUEUE_MESSAGE_INT_MASK                                                           0x00004000L
+#define CP_MEC1_F32_INT_DIS__FATAL_EDC_ERROR_INT_MASK                                                         0x00008000L
+//CP_MEC2_F32_INT_DIS
+#define CP_MEC2_F32_INT_DIS__EDC_ROQ_FED_INT__SHIFT                                                           0x0
+#define CP_MEC2_F32_INT_DIS__PRIV_REG_INT__SHIFT                                                              0x1
+#define CP_MEC2_F32_INT_DIS__RESERVED_BIT_ERR_INT__SHIFT                                                      0x2
+#define CP_MEC2_F32_INT_DIS__EDC_TC_FED_INT__SHIFT                                                            0x3
+#define CP_MEC2_F32_INT_DIS__EDC_GDS_FED_INT__SHIFT                                                           0x4
+#define CP_MEC2_F32_INT_DIS__EDC_SCRATCH_FED_INT__SHIFT                                                       0x5
+#define CP_MEC2_F32_INT_DIS__WAVE_RESTORE_INT__SHIFT                                                          0x6
+#define CP_MEC2_F32_INT_DIS__SUA_VIOLATION_INT__SHIFT                                                         0x7
+#define CP_MEC2_F32_INT_DIS__EDC_DMA_FED_INT__SHIFT                                                           0x8
+#define CP_MEC2_F32_INT_DIS__IQ_TIMER_INT__SHIFT                                                              0x9
+#define CP_MEC2_F32_INT_DIS__GPF_INT_CPF__SHIFT                                                               0xa
+#define CP_MEC2_F32_INT_DIS__GPF_INT_DMA__SHIFT                                                               0xb
+#define CP_MEC2_F32_INT_DIS__GPF_INT_CPC__SHIFT                                                               0xc
+#define CP_MEC2_F32_INT_DIS__EDC_SR_MEM_FED_INT__SHIFT                                                        0xd
+#define CP_MEC2_F32_INT_DIS__QUEUE_MESSAGE_INT__SHIFT                                                         0xe
+#define CP_MEC2_F32_INT_DIS__FATAL_EDC_ERROR_INT__SHIFT                                                       0xf
+#define CP_MEC2_F32_INT_DIS__EDC_ROQ_FED_INT_MASK                                                             0x00000001L
+#define CP_MEC2_F32_INT_DIS__PRIV_REG_INT_MASK                                                                0x00000002L
+#define CP_MEC2_F32_INT_DIS__RESERVED_BIT_ERR_INT_MASK                                                        0x00000004L
+#define CP_MEC2_F32_INT_DIS__EDC_TC_FED_INT_MASK                                                              0x00000008L
+#define CP_MEC2_F32_INT_DIS__EDC_GDS_FED_INT_MASK                                                             0x00000010L
+#define CP_MEC2_F32_INT_DIS__EDC_SCRATCH_FED_INT_MASK                                                         0x00000020L
+#define CP_MEC2_F32_INT_DIS__WAVE_RESTORE_INT_MASK                                                            0x00000040L
+#define CP_MEC2_F32_INT_DIS__SUA_VIOLATION_INT_MASK                                                           0x00000080L
+#define CP_MEC2_F32_INT_DIS__EDC_DMA_FED_INT_MASK                                                             0x00000100L
+#define CP_MEC2_F32_INT_DIS__IQ_TIMER_INT_MASK                                                                0x00000200L
+#define CP_MEC2_F32_INT_DIS__GPF_INT_CPF_MASK                                                                 0x00000400L
+#define CP_MEC2_F32_INT_DIS__GPF_INT_DMA_MASK                                                                 0x00000800L
+#define CP_MEC2_F32_INT_DIS__GPF_INT_CPC_MASK                                                                 0x00001000L
+#define CP_MEC2_F32_INT_DIS__EDC_SR_MEM_FED_INT_MASK                                                          0x00002000L
+#define CP_MEC2_F32_INT_DIS__QUEUE_MESSAGE_INT_MASK                                                           0x00004000L
+#define CP_MEC2_F32_INT_DIS__FATAL_EDC_ERROR_INT_MASK                                                         0x00008000L
+//CP_VMID_STATUS
+#define CP_VMID_STATUS__PREEMPT_DE_STATUS__SHIFT                                                              0x0
+#define CP_VMID_STATUS__PREEMPT_CE_STATUS__SHIFT                                                              0x10
+#define CP_VMID_STATUS__PREEMPT_DE_STATUS_MASK                                                                0x0000FFFFL
+#define CP_VMID_STATUS__PREEMPT_CE_STATUS_MASK                                                                0xFFFF0000L
+
+
+// addressBlock: xcd0_gc_cppdec2
+//CP_RB_DOORBELL_CONTROL_SCH_0
+#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_OFFSET__SHIFT                                                  0x2
+#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_EN__SHIFT                                                      0x1e
+#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_HIT__SHIFT                                                     0x1f
+#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_EN_MASK                                                        0x40000000L
+#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_HIT_MASK                                                       0x80000000L
+//CP_RB_DOORBELL_CONTROL_SCH_1
+#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_OFFSET__SHIFT                                                  0x2
+#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_EN__SHIFT                                                      0x1e
+#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_HIT__SHIFT                                                     0x1f
+#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_EN_MASK                                                        0x40000000L
+#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_HIT_MASK                                                       0x80000000L
+//CP_RB_DOORBELL_CONTROL_SCH_2
+#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_OFFSET__SHIFT                                                  0x2
+#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_EN__SHIFT                                                      0x1e
+#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_HIT__SHIFT                                                     0x1f
+#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_EN_MASK                                                        0x40000000L
+#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_HIT_MASK                                                       0x80000000L
+//CP_RB_DOORBELL_CONTROL_SCH_3
+#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_OFFSET__SHIFT                                                  0x2
+#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_EN__SHIFT                                                      0x1e
+#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_HIT__SHIFT                                                     0x1f
+#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_EN_MASK                                                        0x40000000L
+#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_HIT_MASK                                                       0x80000000L
+//CP_RB_DOORBELL_CONTROL_SCH_4
+#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_OFFSET__SHIFT                                                  0x2
+#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_EN__SHIFT                                                      0x1e
+#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_HIT__SHIFT                                                     0x1f
+#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_EN_MASK                                                        0x40000000L
+#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_HIT_MASK                                                       0x80000000L
+//CP_RB_DOORBELL_CONTROL_SCH_5
+#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_OFFSET__SHIFT                                                  0x2
+#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_EN__SHIFT                                                      0x1e
+#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_HIT__SHIFT                                                     0x1f
+#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_EN_MASK                                                        0x40000000L
+#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_HIT_MASK                                                       0x80000000L
+//CP_RB_DOORBELL_CONTROL_SCH_6
+#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_OFFSET__SHIFT                                                  0x2
+#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_EN__SHIFT                                                      0x1e
+#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_HIT__SHIFT                                                     0x1f
+#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_EN_MASK                                                        0x40000000L
+#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_HIT_MASK                                                       0x80000000L
+//CP_RB_DOORBELL_CONTROL_SCH_7
+#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_OFFSET__SHIFT                                                  0x2
+#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_EN__SHIFT                                                      0x1e
+#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_HIT__SHIFT                                                     0x1f
+#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_EN_MASK                                                        0x40000000L
+#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_HIT_MASK                                                       0x80000000L
+//CP_RB_DOORBELL_CLEAR
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUEUE__SHIFT                                                             0x0
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_EN_CLEAR__SHIFT                                             0x8
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_HIT_CLEAR__SHIFT                                            0x9
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_EN_CLEAR__SHIFT                                                 0xa
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_HIT_CLEAR__SHIFT                                                0xb
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_EN_CLEAR__SHIFT                                                 0xc
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_HIT_CLEAR__SHIFT                                                0xd
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUEUE_MASK                                                               0x00000007L
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_EN_CLEAR_MASK                                               0x00000100L
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_HIT_CLEAR_MASK                                              0x00000200L
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_EN_CLEAR_MASK                                                   0x00000400L
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_HIT_CLEAR_MASK                                                  0x00000800L
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_EN_CLEAR_MASK                                                   0x00001000L
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_HIT_CLEAR_MASK                                                  0x00002000L
+//CP_CPF_DSM_CNTL
+#define CP_CPF_DSM_CNTL__CPF0_DSM_IRRITATOR_DATA__SHIFT                                                       0x0
+#define CP_CPF_DSM_CNTL__CPF0_ENABLE_SINGLE_WRITE__SHIFT                                                      0x2
+#define CP_CPF_DSM_CNTL__CPF1_DSM_IRRITATOR_DATA__SHIFT                                                       0x3
+#define CP_CPF_DSM_CNTL__CPF1_ENABLE_SINGLE_WRITE__SHIFT                                                      0x5
+#define CP_CPF_DSM_CNTL__CPF2_DSM_IRRITATOR_DATA__SHIFT                                                       0x6
+#define CP_CPF_DSM_CNTL__CPF2_ENABLE_SINGLE_WRITE__SHIFT                                                      0x8
+#define CP_CPF_DSM_CNTL__CPF0_DSM_IRRITATOR_DATA_MASK                                                         0x00000003L
+#define CP_CPF_DSM_CNTL__CPF0_ENABLE_SINGLE_WRITE_MASK                                                        0x00000004L
+#define CP_CPF_DSM_CNTL__CPF1_DSM_IRRITATOR_DATA_MASK                                                         0x00000018L
+#define CP_CPF_DSM_CNTL__CPF1_ENABLE_SINGLE_WRITE_MASK                                                        0x00000020L
+#define CP_CPF_DSM_CNTL__CPF2_DSM_IRRITATOR_DATA_MASK                                                         0x000000C0L
+#define CP_CPF_DSM_CNTL__CPF2_ENABLE_SINGLE_WRITE_MASK                                                        0x00000100L
+//CP_CPG_DSM_CNTL
+#define CP_CPG_DSM_CNTL__CPG0_DSM_IRRITATOR_DATA__SHIFT                                                       0x0
+#define CP_CPG_DSM_CNTL__CPG0_ENABLE_SINGLE_WRITE__SHIFT                                                      0x2
+#define CP_CPG_DSM_CNTL__CPG1_DSM_IRRITATOR_DATA__SHIFT                                                       0x3
+#define CP_CPG_DSM_CNTL__CPG1_ENABLE_SINGLE_WRITE__SHIFT                                                      0x5
+#define CP_CPG_DSM_CNTL__CPG2_DSM_IRRITATOR_DATA__SHIFT                                                       0x6
+#define CP_CPG_DSM_CNTL__CPG2_ENABLE_SINGLE_WRITE__SHIFT                                                      0x8
+#define CP_CPG_DSM_CNTL__CPG0_DSM_IRRITATOR_DATA_MASK                                                         0x00000003L
+#define CP_CPG_DSM_CNTL__CPG0_ENABLE_SINGLE_WRITE_MASK                                                        0x00000004L
+#define CP_CPG_DSM_CNTL__CPG1_DSM_IRRITATOR_DATA_MASK                                                         0x00000018L
+#define CP_CPG_DSM_CNTL__CPG1_ENABLE_SINGLE_WRITE_MASK                                                        0x00000020L
+#define CP_CPG_DSM_CNTL__CPG2_DSM_IRRITATOR_DATA_MASK                                                         0x000000C0L
+#define CP_CPG_DSM_CNTL__CPG2_ENABLE_SINGLE_WRITE_MASK                                                        0x00000100L
+//CP_CPC_DSM_CNTL
+#define CP_CPC_DSM_CNTL__CPC0_DSM_IRRITATOR_DATA__SHIFT                                                       0x0
+#define CP_CPC_DSM_CNTL__CPC0_ENABLE_SINGLE_WRITE__SHIFT                                                      0x2
+#define CP_CPC_DSM_CNTL__CPC1_DSM_IRRITATOR_DATA__SHIFT                                                       0x3
+#define CP_CPC_DSM_CNTL__CPC1_ENABLE_SINGLE_WRITE__SHIFT                                                      0x5
+#define CP_CPC_DSM_CNTL__CPC2_DSM_IRRITATOR_DATA__SHIFT                                                       0x6
+#define CP_CPC_DSM_CNTL__CPC2_ENABLE_SINGLE_WRITE__SHIFT                                                      0x8
+#define CP_CPC_DSM_CNTL__CPC3_DSM_IRRITATOR_DATA__SHIFT                                                       0x9
+#define CP_CPC_DSM_CNTL__CPC3_ENABLE_SINGLE_WRITE__SHIFT                                                      0xb
+#define CP_CPC_DSM_CNTL__CPC4_DSM_IRRITATOR_DATA__SHIFT                                                       0xc
+#define CP_CPC_DSM_CNTL__CPC4_ENABLE_SINGLE_WRITE__SHIFT                                                      0xe
+#define CP_CPC_DSM_CNTL__CPC5_DSM_IRRITATOR_DATA__SHIFT                                                       0xf
+#define CP_CPC_DSM_CNTL__CPC5_ENABLE_SINGLE_WRITE__SHIFT                                                      0x11
+#define CP_CPC_DSM_CNTL__CPC6_DSM_IRRITATOR_DATA__SHIFT                                                       0x12
+#define CP_CPC_DSM_CNTL__CPC6_ENABLE_SINGLE_WRITE__SHIFT                                                      0x14
+#define CP_CPC_DSM_CNTL__CPC7_DSM_IRRITATOR_DATA__SHIFT                                                       0x15
+#define CP_CPC_DSM_CNTL__CPC7_ENABLE_SINGLE_WRITE__SHIFT                                                      0x17
+#define CP_CPC_DSM_CNTL__CPC8_DSM_IRRITATOR_DATA__SHIFT                                                       0x18
+#define CP_CPC_DSM_CNTL__CPC8_ENABLE_SINGLE_WRITE__SHIFT                                                      0x1a
+#define CP_CPC_DSM_CNTL__CPC0_DSM_IRRITATOR_DATA_MASK                                                         0x00000003L
+#define CP_CPC_DSM_CNTL__CPC0_ENABLE_SINGLE_WRITE_MASK                                                        0x00000004L
+#define CP_CPC_DSM_CNTL__CPC1_DSM_IRRITATOR_DATA_MASK                                                         0x00000018L
+#define CP_CPC_DSM_CNTL__CPC1_ENABLE_SINGLE_WRITE_MASK                                                        0x00000020L
+#define CP_CPC_DSM_CNTL__CPC2_DSM_IRRITATOR_DATA_MASK                                                         0x000000C0L
+#define CP_CPC_DSM_CNTL__CPC2_ENABLE_SINGLE_WRITE_MASK                                                        0x00000100L
+#define CP_CPC_DSM_CNTL__CPC3_DSM_IRRITATOR_DATA_MASK                                                         0x00000600L
+#define CP_CPC_DSM_CNTL__CPC3_ENABLE_SINGLE_WRITE_MASK                                                        0x00000800L
+#define CP_CPC_DSM_CNTL__CPC4_DSM_IRRITATOR_DATA_MASK                                                         0x00003000L
+#define CP_CPC_DSM_CNTL__CPC4_ENABLE_SINGLE_WRITE_MASK                                                        0x00004000L
+#define CP_CPC_DSM_CNTL__CPC5_DSM_IRRITATOR_DATA_MASK                                                         0x00018000L
+#define CP_CPC_DSM_CNTL__CPC5_ENABLE_SINGLE_WRITE_MASK                                                        0x00020000L
+#define CP_CPC_DSM_CNTL__CPC6_DSM_IRRITATOR_DATA_MASK                                                         0x000C0000L
+#define CP_CPC_DSM_CNTL__CPC6_ENABLE_SINGLE_WRITE_MASK                                                        0x00100000L
+#define CP_CPC_DSM_CNTL__CPC7_DSM_IRRITATOR_DATA_MASK                                                         0x00600000L
+#define CP_CPC_DSM_CNTL__CPC7_ENABLE_SINGLE_WRITE_MASK                                                        0x00800000L
+#define CP_CPC_DSM_CNTL__CPC8_DSM_IRRITATOR_DATA_MASK                                                         0x03000000L
+#define CP_CPC_DSM_CNTL__CPC8_ENABLE_SINGLE_WRITE_MASK                                                        0x04000000L
+//CP_CPF_DSM_CNTL2
+#define CP_CPF_DSM_CNTL2__CPF0_ENABLE_ERROR_INJECT__SHIFT                                                     0x0
+#define CP_CPF_DSM_CNTL2__CPF0_SELECT_INJECT_DELAY__SHIFT                                                     0x2
+#define CP_CPF_DSM_CNTL2__CPF1_ENABLE_ERROR_INJECT__SHIFT                                                     0x3
+#define CP_CPF_DSM_CNTL2__CPF1_SELECT_INJECT_DELAY__SHIFT                                                     0x5
+#define CP_CPF_DSM_CNTL2__CPF2_ENABLE_ERROR_INJECT__SHIFT                                                     0x6
+#define CP_CPF_DSM_CNTL2__CPF2_SELECT_INJECT_DELAY__SHIFT                                                     0x8
+#define CP_CPF_DSM_CNTL2__CPF0_ENABLE_ERROR_INJECT_MASK                                                       0x00000003L
+#define CP_CPF_DSM_CNTL2__CPF0_SELECT_INJECT_DELAY_MASK                                                       0x00000004L
+#define CP_CPF_DSM_CNTL2__CPF1_ENABLE_ERROR_INJECT_MASK                                                       0x00000018L
+#define CP_CPF_DSM_CNTL2__CPF1_SELECT_INJECT_DELAY_MASK                                                       0x00000020L
+#define CP_CPF_DSM_CNTL2__CPF2_ENABLE_ERROR_INJECT_MASK                                                       0x000000C0L
+#define CP_CPF_DSM_CNTL2__CPF2_SELECT_INJECT_DELAY_MASK                                                       0x00000100L
+//CP_CPG_DSM_CNTL2
+#define CP_CPG_DSM_CNTL2__CPG0_ENABLE_ERROR_INJECT__SHIFT                                                     0x0
+#define CP_CPG_DSM_CNTL2__CPG0_SELECT_INJECT_DELAY__SHIFT                                                     0x2
+#define CP_CPG_DSM_CNTL2__CPG1_ENABLE_ERROR_INJECT__SHIFT                                                     0x3
+#define CP_CPG_DSM_CNTL2__CPG1_SELECT_INJECT_DELAY__SHIFT                                                     0x5
+#define CP_CPG_DSM_CNTL2__CPG2_ENABLE_ERROR_INJECT__SHIFT                                                     0x6
+#define CP_CPG_DSM_CNTL2__CPG2_SELECT_INJECT_DELAY__SHIFT                                                     0x8
+#define CP_CPG_DSM_CNTL2__CPG0_ENABLE_ERROR_INJECT_MASK                                                       0x00000003L
+#define CP_CPG_DSM_CNTL2__CPG0_SELECT_INJECT_DELAY_MASK                                                       0x00000004L
+#define CP_CPG_DSM_CNTL2__CPG1_ENABLE_ERROR_INJECT_MASK                                                       0x00000018L
+#define CP_CPG_DSM_CNTL2__CPG1_SELECT_INJECT_DELAY_MASK                                                       0x00000020L
+#define CP_CPG_DSM_CNTL2__CPG2_ENABLE_ERROR_INJECT_MASK                                                       0x000000C0L
+#define CP_CPG_DSM_CNTL2__CPG2_SELECT_INJECT_DELAY_MASK                                                       0x00000100L
+//CP_CPC_DSM_CNTL2
+#define CP_CPC_DSM_CNTL2__CPC0_ENABLE_ERROR_INJECT__SHIFT                                                     0x0
+#define CP_CPC_DSM_CNTL2__CPC0_SELECT_INJECT_DELAY__SHIFT                                                     0x2
+#define CP_CPC_DSM_CNTL2__CPC1_ENABLE_ERROR_INJECT__SHIFT                                                     0x3
+#define CP_CPC_DSM_CNTL2__CPC1_SELECT_INJECT_DELAY__SHIFT                                                     0x5
+#define CP_CPC_DSM_CNTL2__CPC2_ENABLE_ERROR_INJECT__SHIFT                                                     0x6
+#define CP_CPC_DSM_CNTL2__CPC2_SELECT_INJECT_DELAY__SHIFT                                                     0x8
+#define CP_CPC_DSM_CNTL2__CPC3_ENABLE_ERROR_INJECT__SHIFT                                                     0x9
+#define CP_CPC_DSM_CNTL2__CPC3_SELECT_INJECT_DELAY__SHIFT                                                     0xb
+#define CP_CPC_DSM_CNTL2__CPC4_ENABLE_ERROR_INJECT__SHIFT                                                     0xc
+#define CP_CPC_DSM_CNTL2__CPC4_SELECT_INJECT_DELAY__SHIFT                                                     0xe
+#define CP_CPC_DSM_CNTL2__CPC5_ENABLE_ERROR_INJECT__SHIFT                                                     0xf
+#define CP_CPC_DSM_CNTL2__CPC5_SELECT_INJECT_DELAY__SHIFT                                                     0x11
+#define CP_CPC_DSM_CNTL2__CPC6_ENABLE_ERROR_INJECT__SHIFT                                                     0x12
+#define CP_CPC_DSM_CNTL2__CPC6_SELECT_INJECT_DELAY__SHIFT                                                     0x14
+#define CP_CPC_DSM_CNTL2__CPC7_ENABLE_ERROR_INJECT__SHIFT                                                     0x15
+#define CP_CPC_DSM_CNTL2__CPC7_SELECT_INJECT_DELAY__SHIFT                                                     0x17
+#define CP_CPC_DSM_CNTL2__CPC8_ENABLE_ERROR_INJECT__SHIFT                                                     0x18
+#define CP_CPC_DSM_CNTL2__CPC8_SELECT_INJECT_DELAY__SHIFT                                                     0x1a
+#define CP_CPC_DSM_CNTL2__CPC0_ENABLE_ERROR_INJECT_MASK                                                       0x00000003L
+#define CP_CPC_DSM_CNTL2__CPC0_SELECT_INJECT_DELAY_MASK                                                       0x00000004L
+#define CP_CPC_DSM_CNTL2__CPC1_ENABLE_ERROR_INJECT_MASK                                                       0x00000018L
+#define CP_CPC_DSM_CNTL2__CPC1_SELECT_INJECT_DELAY_MASK                                                       0x00000020L
+#define CP_CPC_DSM_CNTL2__CPC2_ENABLE_ERROR_INJECT_MASK                                                       0x000000C0L
+#define CP_CPC_DSM_CNTL2__CPC2_SELECT_INJECT_DELAY_MASK                                                       0x00000100L
+#define CP_CPC_DSM_CNTL2__CPC3_ENABLE_ERROR_INJECT_MASK                                                       0x00000600L
+#define CP_CPC_DSM_CNTL2__CPC3_SELECT_INJECT_DELAY_MASK                                                       0x00000800L
+#define CP_CPC_DSM_CNTL2__CPC4_ENABLE_ERROR_INJECT_MASK                                                       0x00003000L
+#define CP_CPC_DSM_CNTL2__CPC4_SELECT_INJECT_DELAY_MASK                                                       0x00004000L
+#define CP_CPC_DSM_CNTL2__CPC5_ENABLE_ERROR_INJECT_MASK                                                       0x00018000L
+#define CP_CPC_DSM_CNTL2__CPC5_SELECT_INJECT_DELAY_MASK                                                       0x00020000L
+#define CP_CPC_DSM_CNTL2__CPC6_ENABLE_ERROR_INJECT_MASK                                                       0x000C0000L
+#define CP_CPC_DSM_CNTL2__CPC6_SELECT_INJECT_DELAY_MASK                                                       0x00100000L
+#define CP_CPC_DSM_CNTL2__CPC7_ENABLE_ERROR_INJECT_MASK                                                       0x00600000L
+#define CP_CPC_DSM_CNTL2__CPC7_SELECT_INJECT_DELAY_MASK                                                       0x00800000L
+#define CP_CPC_DSM_CNTL2__CPC8_ENABLE_ERROR_INJECT_MASK                                                       0x03000000L
+#define CP_CPC_DSM_CNTL2__CPC8_SELECT_INJECT_DELAY_MASK                                                       0x04000000L
+//CP_CPF_DSM_CNTL2A
+#define CP_CPF_DSM_CNTL2A__CPF_INJECT_DELAY__SHIFT                                                            0x0
+#define CP_CPF_DSM_CNTL2A__CPF_INJECT_DELAY_MASK                                                              0x0000003FL
+//CP_CPG_DSM_CNTL2A
+#define CP_CPG_DSM_CNTL2A__CPG_INJECT_DELAY__SHIFT                                                            0x0
+#define CP_CPG_DSM_CNTL2A__CPG_INJECT_DELAY_MASK                                                              0x0000003FL
+//CP_CPC_DSM_CNTL2A
+#define CP_CPC_DSM_CNTL2A__CPC_INJECT_DELAY__SHIFT                                                            0x0
+#define CP_CPC_DSM_CNTL2A__CPC_INJECT_DELAY_MASK                                                              0x0000003FL
+//CP_EDC_FUE_CNTL
+#define CP_EDC_FUE_CNTL__CP_FUE_MASK__SHIFT                                                                   0x0
+#define CP_EDC_FUE_CNTL__SPI_FUE_MASK__SHIFT                                                                  0x1
+#define CP_EDC_FUE_CNTL__GDS_FUE_MASK__SHIFT                                                                  0x2
+#define CP_EDC_FUE_CNTL__TC_RLC_FUE_MASK__SHIFT                                                               0x3
+#define CP_EDC_FUE_CNTL__TC_CPG_FUE_MASK__SHIFT                                                               0x4
+#define CP_EDC_FUE_CNTL__TCA_FUE_MASK__SHIFT                                                                  0x5
+#define CP_EDC_FUE_CNTL__TCC_FUE_MASK__SHIFT                                                                  0x6
+#define CP_EDC_FUE_CNTL__UTCL2_FUE_MASK__SHIFT                                                                0x7
+#define CP_EDC_FUE_CNTL__CP_FUE_FLAG__SHIFT                                                                   0x10
+#define CP_EDC_FUE_CNTL__SPI_FUE_FLAG__SHIFT                                                                  0x11
+#define CP_EDC_FUE_CNTL__GDS_FUE_FLAG__SHIFT                                                                  0x12
+#define CP_EDC_FUE_CNTL__TC_RLC_FUE_FLAG__SHIFT                                                               0x13
+#define CP_EDC_FUE_CNTL__TC_CPG_FUE_FLAG__SHIFT                                                               0x14
+#define CP_EDC_FUE_CNTL__TCA_FUE_FLAG__SHIFT                                                                  0x15
+#define CP_EDC_FUE_CNTL__TCC_FUE_FLAG__SHIFT                                                                  0x16
+#define CP_EDC_FUE_CNTL__UTCL2_FUE_FLAG__SHIFT                                                                0x17
+#define CP_EDC_FUE_CNTL__CP_FUE_MASK_MASK                                                                     0x00000001L
+#define CP_EDC_FUE_CNTL__SPI_FUE_MASK_MASK                                                                    0x00000002L
+#define CP_EDC_FUE_CNTL__GDS_FUE_MASK_MASK                                                                    0x00000004L
+#define CP_EDC_FUE_CNTL__TC_RLC_FUE_MASK_MASK                                                                 0x00000008L
+#define CP_EDC_FUE_CNTL__TC_CPG_FUE_MASK_MASK                                                                 0x00000010L
+#define CP_EDC_FUE_CNTL__TCA_FUE_MASK_MASK                                                                    0x00000020L
+#define CP_EDC_FUE_CNTL__TCC_FUE_MASK_MASK                                                                    0x00000040L
+#define CP_EDC_FUE_CNTL__UTCL2_FUE_MASK_MASK                                                                  0x00000080L
+#define CP_EDC_FUE_CNTL__CP_FUE_FLAG_MASK                                                                     0x00010000L
+#define CP_EDC_FUE_CNTL__SPI_FUE_FLAG_MASK                                                                    0x00020000L
+#define CP_EDC_FUE_CNTL__GDS_FUE_FLAG_MASK                                                                    0x00040000L
+#define CP_EDC_FUE_CNTL__TC_RLC_FUE_FLAG_MASK                                                                 0x00080000L
+#define CP_EDC_FUE_CNTL__TC_CPG_FUE_FLAG_MASK                                                                 0x00100000L
+#define CP_EDC_FUE_CNTL__TCA_FUE_FLAG_MASK                                                                    0x00200000L
+#define CP_EDC_FUE_CNTL__TCC_FUE_FLAG_MASK                                                                    0x00400000L
+#define CP_EDC_FUE_CNTL__UTCL2_FUE_FLAG_MASK                                                                  0x00800000L
+//CP_GFX_MQD_CONTROL
+#define CP_GFX_MQD_CONTROL__VMID__SHIFT                                                                       0x0
+#define CP_GFX_MQD_CONTROL__PRIV_STATE__SHIFT                                                                 0x8
+#define CP_GFX_MQD_CONTROL__EXE_DISABLE__SHIFT                                                                0x17
+#define CP_GFX_MQD_CONTROL__CACHE_POLICY__SHIFT                                                               0x18
+#define CP_GFX_MQD_CONTROL__VMID_MASK                                                                         0x0000000FL
+#define CP_GFX_MQD_CONTROL__PRIV_STATE_MASK                                                                   0x00000100L
+#define CP_GFX_MQD_CONTROL__EXE_DISABLE_MASK                                                                  0x00800000L
+#define CP_GFX_MQD_CONTROL__CACHE_POLICY_MASK                                                                 0x01000000L
+//CP_GFX_MQD_BASE_ADDR
+#define CP_GFX_MQD_BASE_ADDR__BASE_ADDR__SHIFT                                                                0x2
+#define CP_GFX_MQD_BASE_ADDR__BASE_ADDR_MASK                                                                  0xFFFFFFFCL
+//CP_GFX_MQD_BASE_ADDR_HI
+#define CP_GFX_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                          0x0
+#define CP_GFX_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                            0x0000FFFFL
+//CP_RB_STATUS
+#define CP_RB_STATUS__DOORBELL_UPDATED__SHIFT                                                                 0x0
+#define CP_RB_STATUS__DOORBELL_ENABLE__SHIFT                                                                  0x1
+#define CP_RB_STATUS__DOORBELL_UPDATED_MASK                                                                   0x00000001L
+#define CP_RB_STATUS__DOORBELL_ENABLE_MASK                                                                    0x00000002L
+//CPG_UTCL1_STATUS
+#define CPG_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define CPG_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define CPG_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define CPG_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define CPG_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define CPG_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define CPG_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define CPG_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define CPG_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define CPG_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define CPG_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define CPG_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+//CPC_UTCL1_STATUS
+#define CPC_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define CPC_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define CPC_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define CPC_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define CPC_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define CPC_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define CPC_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define CPC_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define CPC_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define CPC_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define CPC_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define CPC_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+//CPF_UTCL1_STATUS
+#define CPF_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define CPF_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define CPF_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define CPF_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define CPF_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define CPF_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define CPF_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define CPF_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define CPF_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define CPF_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define CPF_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define CPF_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+//CP_SD_CNTL
+#define CP_SD_CNTL__CPF_EN__SHIFT                                                                             0x0
+#define CP_SD_CNTL__CPG_EN__SHIFT                                                                             0x1
+#define CP_SD_CNTL__CPC_EN__SHIFT                                                                             0x2
+#define CP_SD_CNTL__RLC_EN__SHIFT                                                                             0x3
+#define CP_SD_CNTL__SPI_EN__SHIFT                                                                             0x4
+#define CP_SD_CNTL__WD_EN__SHIFT                                                                              0x5
+#define CP_SD_CNTL__IA_EN__SHIFT                                                                              0x6
+#define CP_SD_CNTL__PA_EN__SHIFT                                                                              0x7
+#define CP_SD_CNTL__RMI_EN__SHIFT                                                                             0x8
+#define CP_SD_CNTL__EA_EN__SHIFT                                                                              0x9
+#define CP_SD_CNTL__CPF_EN_MASK                                                                               0x00000001L
+#define CP_SD_CNTL__CPG_EN_MASK                                                                               0x00000002L
+#define CP_SD_CNTL__CPC_EN_MASK                                                                               0x00000004L
+#define CP_SD_CNTL__RLC_EN_MASK                                                                               0x00000008L
+#define CP_SD_CNTL__SPI_EN_MASK                                                                               0x00000010L
+#define CP_SD_CNTL__WD_EN_MASK                                                                                0x00000020L
+#define CP_SD_CNTL__IA_EN_MASK                                                                                0x00000040L
+#define CP_SD_CNTL__PA_EN_MASK                                                                                0x00000080L
+#define CP_SD_CNTL__RMI_EN_MASK                                                                               0x00000100L
+#define CP_SD_CNTL__EA_EN_MASK                                                                                0x00000200L
+//CP_SOFT_RESET_CNTL
+#define CP_SOFT_RESET_CNTL__CMP_ONLY_SOFT_RESET__SHIFT                                                        0x0
+#define CP_SOFT_RESET_CNTL__GFX_ONLY_SOFT_RESET__SHIFT                                                        0x1
+#define CP_SOFT_RESET_CNTL__CMP_HQD_REG_RESET__SHIFT                                                          0x2
+#define CP_SOFT_RESET_CNTL__CMP_INTR_REG_RESET__SHIFT                                                         0x3
+#define CP_SOFT_RESET_CNTL__CMP_HQD_QUEUE_DOORBELL_RESET__SHIFT                                               0x4
+#define CP_SOFT_RESET_CNTL__GFX_RB_DOORBELL_RESET__SHIFT                                                      0x5
+#define CP_SOFT_RESET_CNTL__GFX_INTR_REG_RESET__SHIFT                                                         0x6
+#define CP_SOFT_RESET_CNTL__CMP_ONLY_SOFT_RESET_MASK                                                          0x00000001L
+#define CP_SOFT_RESET_CNTL__GFX_ONLY_SOFT_RESET_MASK                                                          0x00000002L
+#define CP_SOFT_RESET_CNTL__CMP_HQD_REG_RESET_MASK                                                            0x00000004L
+#define CP_SOFT_RESET_CNTL__CMP_INTR_REG_RESET_MASK                                                           0x00000008L
+#define CP_SOFT_RESET_CNTL__CMP_HQD_QUEUE_DOORBELL_RESET_MASK                                                 0x00000010L
+#define CP_SOFT_RESET_CNTL__GFX_RB_DOORBELL_RESET_MASK                                                        0x00000020L
+#define CP_SOFT_RESET_CNTL__GFX_INTR_REG_RESET_MASK                                                           0x00000040L
+//CP_CPC_GFX_CNTL
+#define CP_CPC_GFX_CNTL__QUEUEID__SHIFT                                                                       0x0
+#define CP_CPC_GFX_CNTL__PIPEID__SHIFT                                                                        0x3
+#define CP_CPC_GFX_CNTL__MEID__SHIFT                                                                          0x5
+#define CP_CPC_GFX_CNTL__VALID__SHIFT                                                                         0x7
+#define CP_CPC_GFX_CNTL__QUEUEID_MASK                                                                         0x00000007L
+#define CP_CPC_GFX_CNTL__PIPEID_MASK                                                                          0x00000018L
+#define CP_CPC_GFX_CNTL__MEID_MASK                                                                            0x00000060L
+#define CP_CPC_GFX_CNTL__VALID_MASK                                                                           0x00000080L
+
+
+// addressBlock: xcd0_gc_spipdec
+//SPI_ARB_PRIORITY
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0__SHIFT                                                               0x0
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1__SHIFT                                                               0x3
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2__SHIFT                                                               0x6
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3__SHIFT                                                               0x9
+#define SPI_ARB_PRIORITY__TS0_DUR_MULT__SHIFT                                                                 0xc
+#define SPI_ARB_PRIORITY__TS1_DUR_MULT__SHIFT                                                                 0xe
+#define SPI_ARB_PRIORITY__TS2_DUR_MULT__SHIFT                                                                 0x10
+#define SPI_ARB_PRIORITY__TS3_DUR_MULT__SHIFT                                                                 0x12
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0_MASK                                                                 0x00000007L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1_MASK                                                                 0x00000038L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2_MASK                                                                 0x000001C0L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3_MASK                                                                 0x00000E00L
+#define SPI_ARB_PRIORITY__TS0_DUR_MULT_MASK                                                                   0x00003000L
+#define SPI_ARB_PRIORITY__TS1_DUR_MULT_MASK                                                                   0x0000C000L
+#define SPI_ARB_PRIORITY__TS2_DUR_MULT_MASK                                                                   0x00030000L
+#define SPI_ARB_PRIORITY__TS3_DUR_MULT_MASK                                                                   0x000C0000L
+//SPI_ARB_CYCLES_0
+#define SPI_ARB_CYCLES_0__TS0_DURATION__SHIFT                                                                 0x0
+#define SPI_ARB_CYCLES_0__TS1_DURATION__SHIFT                                                                 0x10
+#define SPI_ARB_CYCLES_0__TS0_DURATION_MASK                                                                   0x0000FFFFL
+#define SPI_ARB_CYCLES_0__TS1_DURATION_MASK                                                                   0xFFFF0000L
+//SPI_ARB_CYCLES_1
+#define SPI_ARB_CYCLES_1__TS2_DURATION__SHIFT                                                                 0x0
+#define SPI_ARB_CYCLES_1__TS3_DURATION__SHIFT                                                                 0x10
+#define SPI_ARB_CYCLES_1__TS2_DURATION_MASK                                                                   0x0000FFFFL
+#define SPI_ARB_CYCLES_1__TS3_DURATION_MASK                                                                   0xFFFF0000L
+//SPI_CDBG_SYS_GFX
+#define SPI_CDBG_SYS_GFX__PS_EN__SHIFT                                                                        0x0
+#define SPI_CDBG_SYS_GFX__VS_EN__SHIFT                                                                        0x1
+#define SPI_CDBG_SYS_GFX__GS_EN__SHIFT                                                                        0x2
+#define SPI_CDBG_SYS_GFX__ES_EN__SHIFT                                                                        0x3
+#define SPI_CDBG_SYS_GFX__HS_EN__SHIFT                                                                        0x4
+#define SPI_CDBG_SYS_GFX__LS_EN__SHIFT                                                                        0x5
+#define SPI_CDBG_SYS_GFX__CS_EN__SHIFT                                                                        0x6
+#define SPI_CDBG_SYS_GFX__PS_EN_MASK                                                                          0x0001L
+#define SPI_CDBG_SYS_GFX__VS_EN_MASK                                                                          0x0002L
+#define SPI_CDBG_SYS_GFX__GS_EN_MASK                                                                          0x0004L
+#define SPI_CDBG_SYS_GFX__ES_EN_MASK                                                                          0x0008L
+#define SPI_CDBG_SYS_GFX__HS_EN_MASK                                                                          0x0010L
+#define SPI_CDBG_SYS_GFX__LS_EN_MASK                                                                          0x0020L
+#define SPI_CDBG_SYS_GFX__CS_EN_MASK                                                                          0x0040L
+//SPI_CDBG_SYS_HP3D
+#define SPI_CDBG_SYS_HP3D__PS_EN__SHIFT                                                                       0x0
+#define SPI_CDBG_SYS_HP3D__VS_EN__SHIFT                                                                       0x1
+#define SPI_CDBG_SYS_HP3D__GS_EN__SHIFT                                                                       0x2
+#define SPI_CDBG_SYS_HP3D__ES_EN__SHIFT                                                                       0x3
+#define SPI_CDBG_SYS_HP3D__HS_EN__SHIFT                                                                       0x4
+#define SPI_CDBG_SYS_HP3D__LS_EN__SHIFT                                                                       0x5
+#define SPI_CDBG_SYS_HP3D__PS_EN_MASK                                                                         0x0001L
+#define SPI_CDBG_SYS_HP3D__VS_EN_MASK                                                                         0x0002L
+#define SPI_CDBG_SYS_HP3D__GS_EN_MASK                                                                         0x0004L
+#define SPI_CDBG_SYS_HP3D__ES_EN_MASK                                                                         0x0008L
+#define SPI_CDBG_SYS_HP3D__HS_EN_MASK                                                                         0x0010L
+#define SPI_CDBG_SYS_HP3D__LS_EN_MASK                                                                         0x0020L
+//SPI_CDBG_SYS_CS0
+#define SPI_CDBG_SYS_CS0__PIPE0__SHIFT                                                                        0x0
+#define SPI_CDBG_SYS_CS0__PIPE1__SHIFT                                                                        0x8
+#define SPI_CDBG_SYS_CS0__PIPE2__SHIFT                                                                        0x10
+#define SPI_CDBG_SYS_CS0__PIPE3__SHIFT                                                                        0x18
+#define SPI_CDBG_SYS_CS0__PIPE0_MASK                                                                          0x000000FFL
+#define SPI_CDBG_SYS_CS0__PIPE1_MASK                                                                          0x0000FF00L
+#define SPI_CDBG_SYS_CS0__PIPE2_MASK                                                                          0x00FF0000L
+#define SPI_CDBG_SYS_CS0__PIPE3_MASK                                                                          0xFF000000L
+//SPI_CDBG_SYS_CS1
+#define SPI_CDBG_SYS_CS1__PIPE0__SHIFT                                                                        0x0
+#define SPI_CDBG_SYS_CS1__PIPE1__SHIFT                                                                        0x8
+#define SPI_CDBG_SYS_CS1__PIPE2__SHIFT                                                                        0x10
+#define SPI_CDBG_SYS_CS1__PIPE3__SHIFT                                                                        0x18
+#define SPI_CDBG_SYS_CS1__PIPE0_MASK                                                                          0x000000FFL
+#define SPI_CDBG_SYS_CS1__PIPE1_MASK                                                                          0x0000FF00L
+#define SPI_CDBG_SYS_CS1__PIPE2_MASK                                                                          0x00FF0000L
+#define SPI_CDBG_SYS_CS1__PIPE3_MASK                                                                          0xFF000000L
+//SPI_WCL_PIPE_PERCENT_GFX
+#define SPI_WCL_PIPE_PERCENT_GFX__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_GFX__LS_GRP_VALUE__SHIFT                                                         0x7
+#define SPI_WCL_PIPE_PERCENT_GFX__HS_GRP_VALUE__SHIFT                                                         0xc
+#define SPI_WCL_PIPE_PERCENT_GFX__ES_GRP_VALUE__SHIFT                                                         0x11
+#define SPI_WCL_PIPE_PERCENT_GFX__GS_GRP_VALUE__SHIFT                                                         0x16
+#define SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK                                                                  0x0000007FL
+#define SPI_WCL_PIPE_PERCENT_GFX__LS_GRP_VALUE_MASK                                                           0x00000F80L
+#define SPI_WCL_PIPE_PERCENT_GFX__HS_GRP_VALUE_MASK                                                           0x0001F000L
+#define SPI_WCL_PIPE_PERCENT_GFX__ES_GRP_VALUE_MASK                                                           0x003E0000L
+#define SPI_WCL_PIPE_PERCENT_GFX__GS_GRP_VALUE_MASK                                                           0x07C00000L
+//SPI_WCL_PIPE_PERCENT_HP3D
+#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE__SHIFT                                                               0x0
+#define SPI_WCL_PIPE_PERCENT_HP3D__HS_GRP_VALUE__SHIFT                                                        0xc
+#define SPI_WCL_PIPE_PERCENT_HP3D__GS_GRP_VALUE__SHIFT                                                        0x16
+#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE_MASK                                                                 0x0000007FL
+#define SPI_WCL_PIPE_PERCENT_HP3D__HS_GRP_VALUE_MASK                                                          0x0001F000L
+#define SPI_WCL_PIPE_PERCENT_HP3D__GS_GRP_VALUE_MASK                                                          0x07C00000L
+//SPI_WCL_PIPE_PERCENT_CS0
+#define SPI_WCL_PIPE_PERCENT_CS0__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS0__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS1
+#define SPI_WCL_PIPE_PERCENT_CS1__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS1__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS2
+#define SPI_WCL_PIPE_PERCENT_CS2__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS2__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS3
+#define SPI_WCL_PIPE_PERCENT_CS3__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS3__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS4
+#define SPI_WCL_PIPE_PERCENT_CS4__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS4__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS5
+#define SPI_WCL_PIPE_PERCENT_CS5__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS5__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS6
+#define SPI_WCL_PIPE_PERCENT_CS6__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS6__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS7
+#define SPI_WCL_PIPE_PERCENT_CS7__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS7__VALUE_MASK                                                                  0x7FL
+//SPI_GDBG_WAVE_CNTL
+#define SPI_GDBG_WAVE_CNTL__STALL_RA__SHIFT                                                                   0x0
+#define SPI_GDBG_WAVE_CNTL__STALL_RA_MASK                                                                     0x01L
+//SPI_GDBG_TRAP_CONFIG
+#define SPI_GDBG_TRAP_CONFIG__PIPE0_EN__SHIFT                                                                 0x0
+#define SPI_GDBG_TRAP_CONFIG__PIPE1_EN__SHIFT                                                                 0x8
+#define SPI_GDBG_TRAP_CONFIG__PIPE2_EN__SHIFT                                                                 0x10
+#define SPI_GDBG_TRAP_CONFIG__PIPE3_EN__SHIFT                                                                 0x18
+#define SPI_GDBG_TRAP_CONFIG__PIPE0_EN_MASK                                                                   0x000000FFL
+#define SPI_GDBG_TRAP_CONFIG__PIPE1_EN_MASK                                                                   0x0000FF00L
+#define SPI_GDBG_TRAP_CONFIG__PIPE2_EN_MASK                                                                   0x00FF0000L
+#define SPI_GDBG_TRAP_CONFIG__PIPE3_EN_MASK                                                                   0xFF000000L
+//SPI_GDBG_PER_VMID_CNTL
+#define SPI_GDBG_PER_VMID_CNTL__STALL_VMID__SHIFT                                                             0x0
+#define SPI_GDBG_PER_VMID_CNTL__LAUNCH_MODE__SHIFT                                                            0x1
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_EN__SHIFT                                                                0x3
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_EN__SHIFT                                                                0x4
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_REPLACE__SHIFT                                                           0xd
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_ON_START__SHIFT                                                          0xe
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_ON_END__SHIFT                                                            0xf
+#define SPI_GDBG_PER_VMID_CNTL__STALL_VMID_MASK                                                               0x0001L
+#define SPI_GDBG_PER_VMID_CNTL__LAUNCH_MODE_MASK                                                              0x0006L
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_EN_MASK                                                                  0x0008L
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_EN_MASK                                                                  0x1FF0L
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_REPLACE_MASK                                                             0x2000L
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_ON_START_MASK                                                            0x4000L
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_ON_END_MASK                                                              0x8000L
+//SPI_GDBG_WAVE_CNTL3
+#define SPI_GDBG_WAVE_CNTL3__STALL_PS__SHIFT                                                                  0x0
+#define SPI_GDBG_WAVE_CNTL3__STALL_VS__SHIFT                                                                  0x1
+#define SPI_GDBG_WAVE_CNTL3__STALL_GS__SHIFT                                                                  0x2
+#define SPI_GDBG_WAVE_CNTL3__STALL_HS__SHIFT                                                                  0x3
+#define SPI_GDBG_WAVE_CNTL3__STALL_CSG__SHIFT                                                                 0x4
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS0__SHIFT                                                                 0x5
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS1__SHIFT                                                                 0x6
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS2__SHIFT                                                                 0x7
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS3__SHIFT                                                                 0x8
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS4__SHIFT                                                                 0x9
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS5__SHIFT                                                                 0xa
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS6__SHIFT                                                                 0xb
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS7__SHIFT                                                                 0xc
+#define SPI_GDBG_WAVE_CNTL3__STALL_DURATION__SHIFT                                                            0xd
+#define SPI_GDBG_WAVE_CNTL3__STALL_MULT__SHIFT                                                                0x1c
+#define SPI_GDBG_WAVE_CNTL3__STALL_PS_MASK                                                                    0x00000001L
+#define SPI_GDBG_WAVE_CNTL3__STALL_VS_MASK                                                                    0x00000002L
+#define SPI_GDBG_WAVE_CNTL3__STALL_GS_MASK                                                                    0x00000004L
+#define SPI_GDBG_WAVE_CNTL3__STALL_HS_MASK                                                                    0x00000008L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CSG_MASK                                                                   0x00000010L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS0_MASK                                                                   0x00000020L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS1_MASK                                                                   0x00000040L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS2_MASK                                                                   0x00000080L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS3_MASK                                                                   0x00000100L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS4_MASK                                                                   0x00000200L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS5_MASK                                                                   0x00000400L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS6_MASK                                                                   0x00000800L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS7_MASK                                                                   0x00001000L
+#define SPI_GDBG_WAVE_CNTL3__STALL_DURATION_MASK                                                              0x0FFFE000L
+#define SPI_GDBG_WAVE_CNTL3__STALL_MULT_MASK                                                                  0x10000000L
+//SPI_SCRATCH_ADDR_CHECK
+#define SPI_SCRATCH_ADDR_CHECK__RESERVED__SHIFT                                                               0x0
+#define SPI_SCRATCH_ADDR_CHECK__RESERVED_MASK                                                                 0x0FL
+//SPI_SCRATCH_ADDR_STATUS
+#define SPI_SCRATCH_ADDR_STATUS__WRITE_DIS__SHIFT                                                             0x0
+#define SPI_SCRATCH_ADDR_STATUS__OVERFLOW_DETECTED__SHIFT                                                     0x1
+#define SPI_SCRATCH_ADDR_STATUS__ME_ID__SHIFT                                                                 0x2
+#define SPI_SCRATCH_ADDR_STATUS__PIPE_ID__SHIFT                                                               0x4
+#define SPI_SCRATCH_ADDR_STATUS__WRITE_DIS_MASK                                                               0x01L
+#define SPI_SCRATCH_ADDR_STATUS__OVERFLOW_DETECTED_MASK                                                       0x02L
+#define SPI_SCRATCH_ADDR_STATUS__ME_ID_MASK                                                                   0x0CL
+#define SPI_SCRATCH_ADDR_STATUS__PIPE_ID_MASK                                                                 0x30L
+//SPI_RESET_DEBUG
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET__SHIFT                                                             0x0
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PER_VMID__SHIFT                                                    0x1
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_ALL_VMID__SHIFT                                                    0x2
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_RESOURCE__SHIFT                                                    0x3
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PRIORITY__SHIFT                                                    0x4
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_MASK                                                               0x01L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PER_VMID_MASK                                                      0x02L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_ALL_VMID_MASK                                                      0x04L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_RESOURCE_MASK                                                      0x08L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PRIORITY_MASK                                                      0x10L
+//SPI_COMPUTE_QUEUE_RESET
+#define SPI_COMPUTE_QUEUE_RESET__RESET__SHIFT                                                                 0x0
+#define SPI_COMPUTE_QUEUE_RESET__RESET_MASK                                                                   0x01L
+//SPI_RESOURCE_RESERVE_CU_0
+#define SPI_RESOURCE_RESERVE_CU_0__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_0__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_0__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_0__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_0__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_0__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_0__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_0__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_1
+#define SPI_RESOURCE_RESERVE_CU_1__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_1__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_1__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_1__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_1__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_1__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_1__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_1__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_2
+#define SPI_RESOURCE_RESERVE_CU_2__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_2__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_2__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_2__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_2__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_2__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_2__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_2__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_3
+#define SPI_RESOURCE_RESERVE_CU_3__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_3__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_3__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_3__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_3__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_3__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_3__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_3__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_4
+#define SPI_RESOURCE_RESERVE_CU_4__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_4__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_4__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_4__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_4__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_4__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_4__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_4__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_5
+#define SPI_RESOURCE_RESERVE_CU_5__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_5__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_5__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_5__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_5__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_5__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_5__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_5__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_6
+#define SPI_RESOURCE_RESERVE_CU_6__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_6__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_6__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_6__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_6__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_6__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_6__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_6__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_7
+#define SPI_RESOURCE_RESERVE_CU_7__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_7__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_7__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_7__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_7__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_7__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_7__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_7__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_8
+#define SPI_RESOURCE_RESERVE_CU_8__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_8__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_8__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_8__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_8__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_8__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_8__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_8__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_9
+#define SPI_RESOURCE_RESERVE_CU_9__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_9__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_9__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_9__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_9__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_9__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_9__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_9__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_EN_CU_0
+#define SPI_RESOURCE_RESERVE_EN_CU_0__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_0__RESERVE_SPACE_ONLY__SHIFT                                               0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_0__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK_MASK                                                         0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_0__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_1
+#define SPI_RESOURCE_RESERVE_EN_CU_1__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_1__RESERVE_SPACE_ONLY__SHIFT                                               0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_1__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK_MASK                                                         0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_1__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_2
+#define SPI_RESOURCE_RESERVE_EN_CU_2__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_2__RESERVE_SPACE_ONLY__SHIFT                                               0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_2__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK_MASK                                                         0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_2__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_3
+#define SPI_RESOURCE_RESERVE_EN_CU_3__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_3__RESERVE_SPACE_ONLY__SHIFT                                               0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_3__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK_MASK                                                         0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_3__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_4
+#define SPI_RESOURCE_RESERVE_EN_CU_4__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_4__RESERVE_SPACE_ONLY__SHIFT                                               0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_4__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK_MASK                                                         0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_4__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_5
+#define SPI_RESOURCE_RESERVE_EN_CU_5__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_5__RESERVE_SPACE_ONLY__SHIFT                                               0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_5__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK_MASK                                                         0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_5__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_6
+#define SPI_RESOURCE_RESERVE_EN_CU_6__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_6__RESERVE_SPACE_ONLY__SHIFT                                               0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_6__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK_MASK                                                         0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_6__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_7
+#define SPI_RESOURCE_RESERVE_EN_CU_7__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_7__RESERVE_SPACE_ONLY__SHIFT                                               0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_7__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK_MASK                                                         0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_7__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_8
+#define SPI_RESOURCE_RESERVE_EN_CU_8__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_8__RESERVE_SPACE_ONLY__SHIFT                                               0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_8__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK_MASK                                                         0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_8__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_9
+#define SPI_RESOURCE_RESERVE_EN_CU_9__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_9__RESERVE_SPACE_ONLY__SHIFT                                               0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_9__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK_MASK                                                         0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_9__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
+//SPI_RESOURCE_RESERVE_CU_10
+#define SPI_RESOURCE_RESERVE_CU_10__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_10__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_10__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_10__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_10__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_10__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_10__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_10__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_11
+#define SPI_RESOURCE_RESERVE_CU_11__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_11__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_11__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_11__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_11__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_11__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_11__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_11__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_EN_CU_10
+#define SPI_RESOURCE_RESERVE_EN_CU_10__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_10__RESERVE_SPACE_ONLY__SHIFT                                              0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_10__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK_MASK                                                        0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_10__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_11
+#define SPI_RESOURCE_RESERVE_EN_CU_11__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_11__RESERVE_SPACE_ONLY__SHIFT                                              0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_11__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK_MASK                                                        0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_11__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
+//SPI_RESOURCE_RESERVE_CU_12
+#define SPI_RESOURCE_RESERVE_CU_12__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_12__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_12__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_12__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_12__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_12__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_12__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_12__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_12__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_12__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_13
+#define SPI_RESOURCE_RESERVE_CU_13__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_13__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_13__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_13__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_13__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_13__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_13__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_13__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_13__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_13__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_14
+#define SPI_RESOURCE_RESERVE_CU_14__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_14__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_14__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_14__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_14__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_14__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_14__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_14__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_14__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_14__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_15
+#define SPI_RESOURCE_RESERVE_CU_15__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_15__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_15__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_15__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_15__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_15__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_15__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_15__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_15__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_15__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_EN_CU_12
+#define SPI_RESOURCE_RESERVE_EN_CU_12__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_12__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_12__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_12__RESERVE_SPACE_ONLY__SHIFT                                              0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_12__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_12__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_12__QUEUE_MASK_MASK                                                        0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_12__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_13
+#define SPI_RESOURCE_RESERVE_EN_CU_13__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_13__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_13__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_13__RESERVE_SPACE_ONLY__SHIFT                                              0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_13__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_13__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_13__QUEUE_MASK_MASK                                                        0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_13__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_14
+#define SPI_RESOURCE_RESERVE_EN_CU_14__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_14__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_14__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_14__RESERVE_SPACE_ONLY__SHIFT                                              0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_14__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_14__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_14__QUEUE_MASK_MASK                                                        0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_14__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
+//SPI_RESOURCE_RESERVE_EN_CU_15
+#define SPI_RESOURCE_RESERVE_EN_CU_15__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_15__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_15__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_15__RESERVE_SPACE_ONLY__SHIFT                                              0x18
+#define SPI_RESOURCE_RESERVE_EN_CU_15__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_15__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_15__QUEUE_MASK_MASK                                                        0x00FF0000L
+#define SPI_RESOURCE_RESERVE_EN_CU_15__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
+//SPI_COMPUTE_WF_CTX_SAVE
+#define SPI_COMPUTE_WF_CTX_SAVE__INITIATE__SHIFT                                                              0x0
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_INTERRUPT_EN__SHIFT                                                      0x1
+#define SPI_COMPUTE_WF_CTX_SAVE__DONE_INTERRUPT_EN__SHIFT                                                     0x2
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_REQ_BUSY__SHIFT                                                          0x1e
+#define SPI_COMPUTE_WF_CTX_SAVE__SAVE_BUSY__SHIFT                                                             0x1f
+#define SPI_COMPUTE_WF_CTX_SAVE__INITIATE_MASK                                                                0x00000001L
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_INTERRUPT_EN_MASK                                                        0x00000002L
+#define SPI_COMPUTE_WF_CTX_SAVE__DONE_INTERRUPT_EN_MASK                                                       0x00000004L
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_REQ_BUSY_MASK                                                            0x40000000L
+#define SPI_COMPUTE_WF_CTX_SAVE__SAVE_BUSY_MASK                                                               0x80000000L
+//SPI_ARB_CNTL_0
+#define SPI_ARB_CNTL_0__EXP_ARB_COL_WT__SHIFT                                                                 0x0
+#define SPI_ARB_CNTL_0__EXP_ARB_POS_WT__SHIFT                                                                 0x4
+#define SPI_ARB_CNTL_0__EXP_ARB_GDS_WT__SHIFT                                                                 0x8
+#define SPI_ARB_CNTL_0__EXP_ARB_COL_WT_MASK                                                                   0x0000000FL
+#define SPI_ARB_CNTL_0__EXP_ARB_POS_WT_MASK                                                                   0x000000F0L
+#define SPI_ARB_CNTL_0__EXP_ARB_GDS_WT_MASK                                                                   0x00000F00L
+
+
+// addressBlock: xcd0_gc_cpphqddec
+//CP_HQD_GFX_CONTROL
+#define CP_HQD_GFX_CONTROL__MESSAGE__SHIFT                                                                    0x0
+#define CP_HQD_GFX_CONTROL__MISC__SHIFT                                                                       0x4
+#define CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT                                                          0xf
+#define CP_HQD_GFX_CONTROL__MESSAGE_MASK                                                                      0x0000000FL
+#define CP_HQD_GFX_CONTROL__MISC_MASK                                                                         0x00007FF0L
+#define CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN_MASK                                                            0x00008000L
+//CP_HQD_GFX_STATUS
+#define CP_HQD_GFX_STATUS__STATUS__SHIFT                                                                      0x0
+#define CP_HQD_GFX_STATUS__STATUS_MASK                                                                        0x0000FFFFL
+//CP_HPD_ROQ_OFFSETS
+#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET__SHIFT                                                                  0x0
+#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET__SHIFT                                                                  0x8
+#define CP_HPD_ROQ_OFFSETS__IB_OFFSET__SHIFT                                                                  0x10
+#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET_MASK                                                                    0x00000007L
+#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET_MASK                                                                    0x00003F00L
+#define CP_HPD_ROQ_OFFSETS__IB_OFFSET_MASK                                                                    0x003F0000L
+//CP_HPD_STATUS0
+#define CP_HPD_STATUS0__QUEUE_STATE__SHIFT                                                                    0x0
+#define CP_HPD_STATUS0__MAPPED_QUEUE__SHIFT                                                                   0x5
+#define CP_HPD_STATUS0__QUEUE_AVAILABLE__SHIFT                                                                0x8
+#define CP_HPD_STATUS0__FETCHING_MQD__SHIFT                                                                   0x10
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_PQIB__SHIFT                                                           0x11
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_IQ__SHIFT                                                             0x12
+#define CP_HPD_STATUS0__FORCE_QUEUE_STATE__SHIFT                                                              0x14
+#define CP_HPD_STATUS0__ENABLE_OFFLOAD_CHECK__SHIFT                                                           0x1d
+#define CP_HPD_STATUS0__FORCE_QUEUE__SHIFT                                                                    0x1f
+#define CP_HPD_STATUS0__QUEUE_STATE_MASK                                                                      0x0000001FL
+#define CP_HPD_STATUS0__MAPPED_QUEUE_MASK                                                                     0x000000E0L
+#define CP_HPD_STATUS0__QUEUE_AVAILABLE_MASK                                                                  0x0000FF00L
+#define CP_HPD_STATUS0__FETCHING_MQD_MASK                                                                     0x00010000L
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_PQIB_MASK                                                             0x00020000L
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_IQ_MASK                                                               0x00040000L
+#define CP_HPD_STATUS0__FORCE_QUEUE_STATE_MASK                                                                0x01F00000L
+#define CP_HPD_STATUS0__ENABLE_OFFLOAD_CHECK_MASK                                                             0x20000000L
+#define CP_HPD_STATUS0__FORCE_QUEUE_MASK                                                                      0x80000000L
+//CP_HPD_UTCL1_CNTL
+#define CP_HPD_UTCL1_CNTL__SELECT__SHIFT                                                                      0x0
+#define CP_HPD_UTCL1_CNTL__SELECT_MASK                                                                        0x0000000FL
+//CP_HPD_UTCL1_ERROR
+#define CP_HPD_UTCL1_ERROR__ADDR_HI__SHIFT                                                                    0x0
+#define CP_HPD_UTCL1_ERROR__TYPE__SHIFT                                                                       0x10
+#define CP_HPD_UTCL1_ERROR__VMID__SHIFT                                                                       0x14
+#define CP_HPD_UTCL1_ERROR__ADDR_HI_MASK                                                                      0x0000FFFFL
+#define CP_HPD_UTCL1_ERROR__TYPE_MASK                                                                         0x00010000L
+#define CP_HPD_UTCL1_ERROR__VMID_MASK                                                                         0x00F00000L
+//CP_HPD_UTCL1_ERROR_ADDR
+#define CP_HPD_UTCL1_ERROR_ADDR__ADDR__SHIFT                                                                  0xc
+#define CP_HPD_UTCL1_ERROR_ADDR__ADDR_MASK                                                                    0xFFFFF000L
+//CP_MQD_BASE_ADDR
+#define CP_MQD_BASE_ADDR__BASE_ADDR__SHIFT                                                                    0x2
+#define CP_MQD_BASE_ADDR__BASE_ADDR_MASK                                                                      0xFFFFFFFCL
+//CP_MQD_BASE_ADDR_HI
+#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                              0x0
+#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                                0x0000FFFFL
+//CP_HQD_ACTIVE
+#define CP_HQD_ACTIVE__ACTIVE__SHIFT                                                                          0x0
+#define CP_HQD_ACTIVE__BUSY_GATE__SHIFT                                                                       0x1
+#define CP_HQD_ACTIVE__ACTIVE_MASK                                                                            0x00000001L
+#define CP_HQD_ACTIVE__BUSY_GATE_MASK                                                                         0x00000002L
+//CP_HQD_VMID
+#define CP_HQD_VMID__VMID__SHIFT                                                                              0x0
+#define CP_HQD_VMID__IB_VMID__SHIFT                                                                           0x8
+#define CP_HQD_VMID__VQID__SHIFT                                                                              0x10
+#define CP_HQD_VMID__VMID_MASK                                                                                0x0000000FL
+#define CP_HQD_VMID__IB_VMID_MASK                                                                             0x00000F00L
+#define CP_HQD_VMID__VQID_MASK                                                                                0x03FF0000L
+//CP_HQD_PERSISTENT_STATE
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ__SHIFT                                                           0x0
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT                                                          0x8
+#define CP_HQD_PERSISTENT_STATE__WPP_SWITCH_QOS_EN__SHIFT                                                     0x15
+#define CP_HQD_PERSISTENT_STATE__IQ_SWITCH_QOS_EN__SHIFT                                                      0x16
+#define CP_HQD_PERSISTENT_STATE__IB_SWITCH_QOS_EN__SHIFT                                                      0x17
+#define CP_HQD_PERSISTENT_STATE__EOP_SWITCH_QOS_EN__SHIFT                                                     0x18
+#define CP_HQD_PERSISTENT_STATE__PQ_SWITCH_QOS_EN__SHIFT                                                      0x19
+#define CP_HQD_PERSISTENT_STATE__TC_OFFLOAD_QOS_EN__SHIFT                                                     0x1a
+#define CP_HQD_PERSISTENT_STATE__CACHE_FULL_PACKET_EN__SHIFT                                                  0x1b
+#define CP_HQD_PERSISTENT_STATE__RESTORE_ACTIVE__SHIFT                                                        0x1c
+#define CP_HQD_PERSISTENT_STATE__RELAUNCH_WAVES__SHIFT                                                        0x1d
+#define CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT                                                          0x1e
+#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE__SHIFT                                                           0x1f
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK                                                             0x00000001L
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE_MASK                                                            0x0003FF00L
+#define CP_HQD_PERSISTENT_STATE__WPP_SWITCH_QOS_EN_MASK                                                       0x00200000L
+#define CP_HQD_PERSISTENT_STATE__IQ_SWITCH_QOS_EN_MASK                                                        0x00400000L
+#define CP_HQD_PERSISTENT_STATE__IB_SWITCH_QOS_EN_MASK                                                        0x00800000L
+#define CP_HQD_PERSISTENT_STATE__EOP_SWITCH_QOS_EN_MASK                                                       0x01000000L
+#define CP_HQD_PERSISTENT_STATE__PQ_SWITCH_QOS_EN_MASK                                                        0x02000000L
+#define CP_HQD_PERSISTENT_STATE__TC_OFFLOAD_QOS_EN_MASK                                                       0x04000000L
+#define CP_HQD_PERSISTENT_STATE__CACHE_FULL_PACKET_EN_MASK                                                    0x08000000L
+#define CP_HQD_PERSISTENT_STATE__RESTORE_ACTIVE_MASK                                                          0x10000000L
+#define CP_HQD_PERSISTENT_STATE__RELAUNCH_WAVES_MASK                                                          0x20000000L
+#define CP_HQD_PERSISTENT_STATE__QSWITCH_MODE_MASK                                                            0x40000000L
+#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE_MASK                                                             0x80000000L
+//CP_HQD_PIPE_PRIORITY
+#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY__SHIFT                                                            0x0
+#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY_MASK                                                              0x00000003L
+//CP_HQD_QUEUE_PRIORITY
+#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL__SHIFT                                                          0x0
+#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL_MASK                                                            0x0000000FL
+//CP_HQD_QUANTUM
+#define CP_HQD_QUANTUM__QUANTUM_EN__SHIFT                                                                     0x0
+#define CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT                                                                  0x4
+#define CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT                                                               0x8
+#define CP_HQD_QUANTUM__QUANTUM_ACTIVE__SHIFT                                                                 0x1f
+#define CP_HQD_QUANTUM__QUANTUM_EN_MASK                                                                       0x00000001L
+#define CP_HQD_QUANTUM__QUANTUM_SCALE_MASK                                                                    0x00000010L
+#define CP_HQD_QUANTUM__QUANTUM_DURATION_MASK                                                                 0x00003F00L
+#define CP_HQD_QUANTUM__QUANTUM_ACTIVE_MASK                                                                   0x80000000L
+//CP_HQD_PQ_BASE
+#define CP_HQD_PQ_BASE__ADDR__SHIFT                                                                           0x0
+#define CP_HQD_PQ_BASE__ADDR_MASK                                                                             0xFFFFFFFFL
+//CP_HQD_PQ_BASE_HI
+#define CP_HQD_PQ_BASE_HI__ADDR_HI__SHIFT                                                                     0x0
+#define CP_HQD_PQ_BASE_HI__ADDR_HI_MASK                                                                       0x000000FFL
+//CP_HQD_PQ_RPTR
+#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET__SHIFT                                                                0x0
+#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET_MASK                                                                  0xFFFFFFFFL
+//CP_HQD_PQ_RPTR_REPORT_ADDR
+#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR__SHIFT                                                   0x2
+#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR_MASK                                                     0xFFFFFFFCL
+//CP_HQD_PQ_RPTR_REPORT_ADDR_HI
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI__SHIFT                                             0x0
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI_MASK                                               0x0000FFFFL
+//CP_HQD_PQ_WPTR_POLL_ADDR
+#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR__SHIFT                                                            0x3
+#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR_MASK                                                              0xFFFFFFF8L
+//CP_HQD_PQ_WPTR_POLL_ADDR_HI
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI__SHIFT                                                      0x0
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI_MASK                                                        0x0000FFFFL
+//CP_HQD_PQ_DOORBELL_CONTROL
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE__SHIFT                                                      0x0
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT                                                  0x1
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT                                                    0x2
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE__SHIFT                                                    0x1c
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT__SHIFT                                                  0x1d
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT                                                        0x1e
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT                                                       0x1f
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE_MASK                                                        0x00000001L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP_MASK                                                    0x00000002L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK                                                      0x0FFFFFFCL
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE_MASK                                                      0x10000000L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT_MASK                                                    0x20000000L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK                                                          0x40000000L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT_MASK                                                         0x80000000L
+//CP_HQD_PQ_CONTROL
+#define CP_HQD_PQ_CONTROL__QUEUE_SIZE__SHIFT                                                                  0x0
+#define CP_HQD_PQ_CONTROL__WPTR_CARRY__SHIFT                                                                  0x6
+#define CP_HQD_PQ_CONTROL__RPTR_CARRY__SHIFT                                                                  0x7
+#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT                                                             0x8
+#define CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT                                                               0xe
+#define CP_HQD_PQ_CONTROL__PQ_EMPTY__SHIFT                                                                    0xf
+#define CP_HQD_PQ_CONTROL__WPP_CLAMP_EN__SHIFT                                                                0x10
+#define CP_HQD_PQ_CONTROL__ENDIAN_SWAP__SHIFT                                                                 0x11
+#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE__SHIFT                                                              0x14
+#define CP_HQD_PQ_CONTROL__TMZ__SHIFT                                                                         0x16
+#define CP_HQD_PQ_CONTROL__EXE_DISABLE__SHIFT                                                                 0x17
+#define CP_HQD_PQ_CONTROL__CACHE_POLICY__SHIFT                                                                0x18
+#define CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT                                                             0x19
+#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR__SHIFT                                                              0x1b
+#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH__SHIFT                                                              0x1c
+#define CP_HQD_PQ_CONTROL__ROQ_PQ_IB_FLIP__SHIFT                                                              0x1d
+#define CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT                                                                  0x1e
+#define CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT                                                                   0x1f
+#define CP_HQD_PQ_CONTROL__QUEUE_SIZE_MASK                                                                    0x0000003FL
+#define CP_HQD_PQ_CONTROL__WPTR_CARRY_MASK                                                                    0x00000040L
+#define CP_HQD_PQ_CONTROL__RPTR_CARRY_MASK                                                                    0x00000080L
+#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE_MASK                                                               0x00003F00L
+#define CP_HQD_PQ_CONTROL__QUEUE_FULL_EN_MASK                                                                 0x00004000L
+#define CP_HQD_PQ_CONTROL__PQ_EMPTY_MASK                                                                      0x00008000L
+#define CP_HQD_PQ_CONTROL__WPP_CLAMP_EN_MASK                                                                  0x00010000L
+#define CP_HQD_PQ_CONTROL__ENDIAN_SWAP_MASK                                                                   0x00060000L
+#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE_MASK                                                                0x00300000L
+#define CP_HQD_PQ_CONTROL__TMZ_MASK                                                                           0x00400000L
+#define CP_HQD_PQ_CONTROL__EXE_DISABLE_MASK                                                                   0x00800000L
+#define CP_HQD_PQ_CONTROL__CACHE_POLICY_MASK                                                                  0x01000000L
+#define CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR_MASK                                                               0x06000000L
+#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK                                                                0x08000000L
+#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK                                                                0x10000000L
+#define CP_HQD_PQ_CONTROL__ROQ_PQ_IB_FLIP_MASK                                                                0x20000000L
+#define CP_HQD_PQ_CONTROL__PRIV_STATE_MASK                                                                    0x40000000L
+#define CP_HQD_PQ_CONTROL__KMD_QUEUE_MASK                                                                     0x80000000L
+//CP_HQD_IB_BASE_ADDR
+#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR__SHIFT                                                              0x2
+#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR_MASK                                                                0xFFFFFFFCL
+//CP_HQD_IB_BASE_ADDR_HI
+#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI__SHIFT                                                        0x0
+#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI_MASK                                                          0x0000FFFFL
+//CP_HQD_IB_RPTR
+#define CP_HQD_IB_RPTR__CONSUMED_OFFSET__SHIFT                                                                0x0
+#define CP_HQD_IB_RPTR__CONSUMED_OFFSET_MASK                                                                  0x000FFFFFL
+//CP_HQD_IB_CONTROL
+#define CP_HQD_IB_CONTROL__IB_SIZE__SHIFT                                                                     0x0
+#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT                                                           0x14
+#define CP_HQD_IB_CONTROL__IB_EXE_DISABLE__SHIFT                                                              0x17
+#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY__SHIFT                                                             0x18
+#define CP_HQD_IB_CONTROL__IB_PRIV_STATE__SHIFT                                                               0x1e
+#define CP_HQD_IB_CONTROL__PROCESSING_IB__SHIFT                                                               0x1f
+#define CP_HQD_IB_CONTROL__IB_SIZE_MASK                                                                       0x000FFFFFL
+#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE_MASK                                                             0x00300000L
+#define CP_HQD_IB_CONTROL__IB_EXE_DISABLE_MASK                                                                0x00800000L
+#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY_MASK                                                               0x01000000L
+#define CP_HQD_IB_CONTROL__IB_PRIV_STATE_MASK                                                                 0x40000000L
+#define CP_HQD_IB_CONTROL__PROCESSING_IB_MASK                                                                 0x80000000L
+//CP_HQD_IQ_TIMER
+#define CP_HQD_IQ_TIMER__WAIT_TIME__SHIFT                                                                     0x0
+#define CP_HQD_IQ_TIMER__RETRY_TYPE__SHIFT                                                                    0x8
+#define CP_HQD_IQ_TIMER__IMMEDIATE_EXPIRE__SHIFT                                                              0xb
+#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE__SHIFT                                                                0xc
+#define CP_HQD_IQ_TIMER__CLOCK_COUNT__SHIFT                                                                   0xe
+#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE__SHIFT                                                                0x10
+#define CP_HQD_IQ_TIMER__QUANTUM_TIMER__SHIFT                                                                 0x16
+#define CP_HQD_IQ_TIMER__EXE_DISABLE__SHIFT                                                                   0x17
+#define CP_HQD_IQ_TIMER__CACHE_POLICY__SHIFT                                                                  0x18
+#define CP_HQD_IQ_TIMER__QUEUE_TYPE__SHIFT                                                                    0x19
+#define CP_HQD_IQ_TIMER__REARM_TIMER__SHIFT                                                                   0x1c
+#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN__SHIFT                                                                 0x1d
+#define CP_HQD_IQ_TIMER__PROCESSING_IQ__SHIFT                                                                 0x1e
+#define CP_HQD_IQ_TIMER__ACTIVE__SHIFT                                                                        0x1f
+#define CP_HQD_IQ_TIMER__WAIT_TIME_MASK                                                                       0x000000FFL
+#define CP_HQD_IQ_TIMER__RETRY_TYPE_MASK                                                                      0x00000700L
+#define CP_HQD_IQ_TIMER__IMMEDIATE_EXPIRE_MASK                                                                0x00000800L
+#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE_MASK                                                                  0x00003000L
+#define CP_HQD_IQ_TIMER__CLOCK_COUNT_MASK                                                                     0x0000C000L
+#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE_MASK                                                                  0x003F0000L
+#define CP_HQD_IQ_TIMER__QUANTUM_TIMER_MASK                                                                   0x00400000L
+#define CP_HQD_IQ_TIMER__EXE_DISABLE_MASK                                                                     0x00800000L
+#define CP_HQD_IQ_TIMER__CACHE_POLICY_MASK                                                                    0x01000000L
+#define CP_HQD_IQ_TIMER__QUEUE_TYPE_MASK                                                                      0x02000000L
+#define CP_HQD_IQ_TIMER__REARM_TIMER_MASK                                                                     0x10000000L
+#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN_MASK                                                                   0x20000000L
+#define CP_HQD_IQ_TIMER__PROCESSING_IQ_MASK                                                                   0x40000000L
+#define CP_HQD_IQ_TIMER__ACTIVE_MASK                                                                          0x80000000L
+//CP_HQD_IQ_RPTR
+#define CP_HQD_IQ_RPTR__OFFSET__SHIFT                                                                         0x0
+#define CP_HQD_IQ_RPTR__OFFSET_MASK                                                                           0x0000003FL
+//CP_HQD_DEQUEUE_REQUEST
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ__SHIFT                                                            0x0
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND__SHIFT                                                            0x4
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT__SHIFT                                                            0x8
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN__SHIFT                                                         0x9
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN__SHIFT                                                         0xa
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_MASK                                                              0x00000007L
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK                                                              0x00000010L
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT_MASK                                                              0x00000100L
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN_MASK                                                           0x00000200L
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN_MASK                                                           0x00000400L
+//CP_HQD_DMA_OFFLOAD
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD__SHIFT                                                                0x0
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD_MASK                                                                  0x00000001L
+//CP_HQD_OFFLOAD
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD__SHIFT                                                                    0x0
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_EN__SHIFT                                                                 0x1
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD__SHIFT                                                                    0x2
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD_EN__SHIFT                                                                 0x3
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD__SHIFT                                                                    0x4
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_EN__SHIFT                                                                 0x5
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_MASK                                                                      0x00000001L
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_EN_MASK                                                                   0x00000002L
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD_MASK                                                                      0x00000004L
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD_EN_MASK                                                                   0x00000008L
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_MASK                                                                      0x00000010L
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_EN_MASK                                                                   0x00000020L
+//CP_HQD_SEMA_CMD
+#define CP_HQD_SEMA_CMD__RETRY__SHIFT                                                                         0x0
+#define CP_HQD_SEMA_CMD__RESULT__SHIFT                                                                        0x1
+#define CP_HQD_SEMA_CMD__RETRY_MASK                                                                           0x00000001L
+#define CP_HQD_SEMA_CMD__RESULT_MASK                                                                          0x00000006L
+//CP_HQD_MSG_TYPE
+#define CP_HQD_MSG_TYPE__ACTION__SHIFT                                                                        0x0
+#define CP_HQD_MSG_TYPE__SAVE_STATE__SHIFT                                                                    0x4
+#define CP_HQD_MSG_TYPE__ACTION_MASK                                                                          0x00000007L
+#define CP_HQD_MSG_TYPE__SAVE_STATE_MASK                                                                      0x00000070L
+//CP_HQD_ATOMIC0_PREOP_LO
+#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO_MASK                                                        0xFFFFFFFFL
+//CP_HQD_ATOMIC0_PREOP_HI
+#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI_MASK                                                        0xFFFFFFFFL
+//CP_HQD_ATOMIC1_PREOP_LO
+#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO_MASK                                                        0xFFFFFFFFL
+//CP_HQD_ATOMIC1_PREOP_HI
+#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI_MASK                                                        0xFFFFFFFFL
+//CP_HQD_HQ_SCHEDULER0
+#define CP_HQD_HQ_SCHEDULER0__SCHEDULER__SHIFT                                                                0x0
+#define CP_HQD_HQ_SCHEDULER0__SCHEDULER_MASK                                                                  0xFFFFFFFFL
+//CP_HQD_HQ_STATUS0
+#define CP_HQD_HQ_STATUS0__DEQUEUE_STATUS__SHIFT                                                              0x0
+#define CP_HQD_HQ_STATUS0__DEQUEUE_RETRY_CNT__SHIFT                                                           0x2
+#define CP_HQD_HQ_STATUS0__RSV_6_4__SHIFT                                                                     0x4
+#define CP_HQD_HQ_STATUS0__SCRATCH_RAM_INIT__SHIFT                                                            0x7
+#define CP_HQD_HQ_STATUS0__TCL2_DIRTY__SHIFT                                                                  0x8
+#define CP_HQD_HQ_STATUS0__PG_ACTIVATED__SHIFT                                                                0x9
+#define CP_HQD_HQ_STATUS0__RSVR_29_10__SHIFT                                                                  0xa
+#define CP_HQD_HQ_STATUS0__QUEUE_IDLE__SHIFT                                                                  0x1e
+#define CP_HQD_HQ_STATUS0__DB_UPDATED_MSG_EN__SHIFT                                                           0x1f
+#define CP_HQD_HQ_STATUS0__DEQUEUE_STATUS_MASK                                                                0x00000003L
+#define CP_HQD_HQ_STATUS0__DEQUEUE_RETRY_CNT_MASK                                                             0x0000000CL
+#define CP_HQD_HQ_STATUS0__RSV_6_4_MASK                                                                       0x00000070L
+#define CP_HQD_HQ_STATUS0__SCRATCH_RAM_INIT_MASK                                                              0x00000080L
+#define CP_HQD_HQ_STATUS0__TCL2_DIRTY_MASK                                                                    0x00000100L
+#define CP_HQD_HQ_STATUS0__PG_ACTIVATED_MASK                                                                  0x00000200L
+#define CP_HQD_HQ_STATUS0__RSVR_29_10_MASK                                                                    0x3FFFFC00L
+#define CP_HQD_HQ_STATUS0__QUEUE_IDLE_MASK                                                                    0x40000000L
+#define CP_HQD_HQ_STATUS0__DB_UPDATED_MSG_EN_MASK                                                             0x80000000L
+//CP_HQD_HQ_CONTROL0
+#define CP_HQD_HQ_CONTROL0__CONTROL__SHIFT                                                                    0x0
+#define CP_HQD_HQ_CONTROL0__CONTROL_MASK                                                                      0xFFFFFFFFL
+//CP_HQD_HQ_SCHEDULER1
+#define CP_HQD_HQ_SCHEDULER1__SCHEDULER__SHIFT                                                                0x0
+#define CP_HQD_HQ_SCHEDULER1__SCHEDULER_MASK                                                                  0xFFFFFFFFL
+//CP_MQD_CONTROL
+#define CP_MQD_CONTROL__VMID__SHIFT                                                                           0x0
+#define CP_MQD_CONTROL__PRIV_STATE__SHIFT                                                                     0x8
+#define CP_MQD_CONTROL__PROCESSING_MQD__SHIFT                                                                 0xc
+#define CP_MQD_CONTROL__PROCESSING_MQD_EN__SHIFT                                                              0xd
+#define CP_MQD_CONTROL__EXE_DISABLE__SHIFT                                                                    0x17
+#define CP_MQD_CONTROL__CACHE_POLICY__SHIFT                                                                   0x18
+#define CP_MQD_CONTROL__VMID_MASK                                                                             0x0000000FL
+#define CP_MQD_CONTROL__PRIV_STATE_MASK                                                                       0x00000100L
+#define CP_MQD_CONTROL__PROCESSING_MQD_MASK                                                                   0x00001000L
+#define CP_MQD_CONTROL__PROCESSING_MQD_EN_MASK                                                                0x00002000L
+#define CP_MQD_CONTROL__EXE_DISABLE_MASK                                                                      0x00800000L
+#define CP_MQD_CONTROL__CACHE_POLICY_MASK                                                                     0x01000000L
+//CP_HQD_HQ_STATUS1
+#define CP_HQD_HQ_STATUS1__STATUS__SHIFT                                                                      0x0
+#define CP_HQD_HQ_STATUS1__STATUS_MASK                                                                        0xFFFFFFFFL
+//CP_HQD_HQ_CONTROL1
+#define CP_HQD_HQ_CONTROL1__CONTROL__SHIFT                                                                    0x0
+#define CP_HQD_HQ_CONTROL1__CONTROL_MASK                                                                      0xFFFFFFFFL
+//CP_HQD_EOP_BASE_ADDR
+#define CP_HQD_EOP_BASE_ADDR__BASE_ADDR__SHIFT                                                                0x0
+#define CP_HQD_EOP_BASE_ADDR__BASE_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_HQD_EOP_BASE_ADDR_HI
+#define CP_HQD_EOP_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                          0x0
+#define CP_HQD_EOP_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                            0x000000FFL
+//CP_HQD_EOP_CONTROL
+#define CP_HQD_EOP_CONTROL__EOP_SIZE__SHIFT                                                                   0x0
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOP__SHIFT                                                             0x8
+#define CP_HQD_EOP_CONTROL__PROCESS_EOP_EN__SHIFT                                                             0xc
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOPIB__SHIFT                                                           0xd
+#define CP_HQD_EOP_CONTROL__PROCESS_EOPIB_EN__SHIFT                                                           0xe
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER__SHIFT                                                               0x15
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER_EN__SHIFT                                                            0x16
+#define CP_HQD_EOP_CONTROL__EXE_DISABLE__SHIFT                                                                0x17
+#define CP_HQD_EOP_CONTROL__CACHE_POLICY__SHIFT                                                               0x18
+#define CP_HQD_EOP_CONTROL__SIG_SEM_RESULT__SHIFT                                                             0x1d
+#define CP_HQD_EOP_CONTROL__PEND_SIG_SEM__SHIFT                                                               0x1f
+#define CP_HQD_EOP_CONTROL__EOP_SIZE_MASK                                                                     0x0000003FL
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOP_MASK                                                               0x00000100L
+#define CP_HQD_EOP_CONTROL__PROCESS_EOP_EN_MASK                                                               0x00001000L
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOPIB_MASK                                                             0x00002000L
+#define CP_HQD_EOP_CONTROL__PROCESS_EOPIB_EN_MASK                                                             0x00004000L
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER_MASK                                                                 0x00200000L
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER_EN_MASK                                                              0x00400000L
+#define CP_HQD_EOP_CONTROL__EXE_DISABLE_MASK                                                                  0x00800000L
+#define CP_HQD_EOP_CONTROL__CACHE_POLICY_MASK                                                                 0x01000000L
+#define CP_HQD_EOP_CONTROL__SIG_SEM_RESULT_MASK                                                               0x60000000L
+#define CP_HQD_EOP_CONTROL__PEND_SIG_SEM_MASK                                                                 0x80000000L
+//CP_HQD_EOP_RPTR
+#define CP_HQD_EOP_RPTR__RPTR__SHIFT                                                                          0x0
+#define CP_HQD_EOP_RPTR__RESET_FETCHER__SHIFT                                                                 0x1c
+#define CP_HQD_EOP_RPTR__DEQUEUE_PEND__SHIFT                                                                  0x1d
+#define CP_HQD_EOP_RPTR__RPTR_EQ_CSMD_WPTR__SHIFT                                                             0x1e
+#define CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT                                                                  0x1f
+#define CP_HQD_EOP_RPTR__RPTR_MASK                                                                            0x00001FFFL
+#define CP_HQD_EOP_RPTR__RESET_FETCHER_MASK                                                                   0x10000000L
+#define CP_HQD_EOP_RPTR__DEQUEUE_PEND_MASK                                                                    0x20000000L
+#define CP_HQD_EOP_RPTR__RPTR_EQ_CSMD_WPTR_MASK                                                               0x40000000L
+#define CP_HQD_EOP_RPTR__INIT_FETCHER_MASK                                                                    0x80000000L
+//CP_HQD_EOP_WPTR
+#define CP_HQD_EOP_WPTR__WPTR__SHIFT                                                                          0x0
+#define CP_HQD_EOP_WPTR__EOP_EMPTY__SHIFT                                                                     0xf
+#define CP_HQD_EOP_WPTR__EOP_AVAIL__SHIFT                                                                     0x10
+#define CP_HQD_EOP_WPTR__WPTR_MASK                                                                            0x00001FFFL
+#define CP_HQD_EOP_WPTR__EOP_EMPTY_MASK                                                                       0x00008000L
+#define CP_HQD_EOP_WPTR__EOP_AVAIL_MASK                                                                       0x1FFF0000L
+//CP_HQD_EOP_EVENTS
+#define CP_HQD_EOP_EVENTS__EVENT_COUNT__SHIFT                                                                 0x0
+#define CP_HQD_EOP_EVENTS__CS_PARTIAL_FLUSH_PEND__SHIFT                                                       0x10
+#define CP_HQD_EOP_EVENTS__EVENT_COUNT_MASK                                                                   0x00000FFFL
+#define CP_HQD_EOP_EVENTS__CS_PARTIAL_FLUSH_PEND_MASK                                                         0x00010000L
+//CP_HQD_CTX_SAVE_BASE_ADDR_LO
+#define CP_HQD_CTX_SAVE_BASE_ADDR_LO__ADDR__SHIFT                                                             0xc
+#define CP_HQD_CTX_SAVE_BASE_ADDR_LO__ADDR_MASK                                                               0xFFFFF000L
+//CP_HQD_CTX_SAVE_BASE_ADDR_HI
+#define CP_HQD_CTX_SAVE_BASE_ADDR_HI__ADDR_HI__SHIFT                                                          0x0
+#define CP_HQD_CTX_SAVE_BASE_ADDR_HI__ADDR_HI_MASK                                                            0x0000FFFFL
+//CP_HQD_CTX_SAVE_CONTROL
+#define CP_HQD_CTX_SAVE_CONTROL__POLICY__SHIFT                                                                0x3
+#define CP_HQD_CTX_SAVE_CONTROL__EXE_DISABLE__SHIFT                                                           0x17
+#define CP_HQD_CTX_SAVE_CONTROL__POLICY_MASK                                                                  0x00000008L
+#define CP_HQD_CTX_SAVE_CONTROL__EXE_DISABLE_MASK                                                             0x00800000L
+//CP_HQD_CNTL_STACK_OFFSET
+#define CP_HQD_CNTL_STACK_OFFSET__OFFSET__SHIFT                                                               0x2
+#define CP_HQD_CNTL_STACK_OFFSET__OFFSET_MASK                                                                 0x0000FFFCL
+//CP_HQD_CNTL_STACK_SIZE
+#define CP_HQD_CNTL_STACK_SIZE__SIZE__SHIFT                                                                   0xc
+#define CP_HQD_CNTL_STACK_SIZE__SIZE_MASK                                                                     0x0000F000L
+//CP_HQD_WG_STATE_OFFSET
+#define CP_HQD_WG_STATE_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define CP_HQD_WG_STATE_OFFSET__OFFSET_MASK                                                                   0x07FFFFFCL
+//CP_HQD_CTX_SAVE_SIZE
+#define CP_HQD_CTX_SAVE_SIZE__SIZE__SHIFT                                                                     0xc
+#define CP_HQD_CTX_SAVE_SIZE__SIZE_MASK                                                                       0x07FFF000L
+//CP_HQD_GDS_RESOURCE_STATE
+#define CP_HQD_GDS_RESOURCE_STATE__OA_REQUIRED__SHIFT                                                         0x0
+#define CP_HQD_GDS_RESOURCE_STATE__OA_ACQUIRED__SHIFT                                                         0x1
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_SIZE__SHIFT                                                            0x4
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_PNTR__SHIFT                                                            0xc
+#define CP_HQD_GDS_RESOURCE_STATE__OA_REQUIRED_MASK                                                           0x00000001L
+#define CP_HQD_GDS_RESOURCE_STATE__OA_ACQUIRED_MASK                                                           0x00000002L
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_SIZE_MASK                                                              0x000003F0L
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_PNTR_MASK                                                              0x0003F000L
+//CP_HQD_ERROR
+#define CP_HQD_ERROR__EDC_ERROR_ID__SHIFT                                                                     0x0
+#define CP_HQD_ERROR__SUA_ERROR__SHIFT                                                                        0x4
+#define CP_HQD_ERROR__AQL_ERROR__SHIFT                                                                        0x5
+#define CP_HQD_ERROR__PQ_UTCL1_ERROR__SHIFT                                                                   0x8
+#define CP_HQD_ERROR__IB_UTCL1_ERROR__SHIFT                                                                   0x9
+#define CP_HQD_ERROR__EOP_UTCL1_ERROR__SHIFT                                                                  0xa
+#define CP_HQD_ERROR__IQ_UTCL1_ERROR__SHIFT                                                                   0xb
+#define CP_HQD_ERROR__RRPT_UTCL1_ERROR__SHIFT                                                                 0xc
+#define CP_HQD_ERROR__WPP_UTCL1_ERROR__SHIFT                                                                  0xd
+#define CP_HQD_ERROR__SEM_UTCL1_ERROR__SHIFT                                                                  0xe
+#define CP_HQD_ERROR__DMA_SRC_UTCL1_ERROR__SHIFT                                                              0xf
+#define CP_HQD_ERROR__DMA_DST_UTCL1_ERROR__SHIFT                                                              0x10
+#define CP_HQD_ERROR__SR_UTCL1_ERROR__SHIFT                                                                   0x11
+#define CP_HQD_ERROR__QU_UTCL1_ERROR__SHIFT                                                                   0x12
+#define CP_HQD_ERROR__TC_UTCL1_ERROR__SHIFT                                                                   0x13
+#define CP_HQD_ERROR__EDC_ERROR_ID_MASK                                                                       0x0000000FL
+#define CP_HQD_ERROR__SUA_ERROR_MASK                                                                          0x00000010L
+#define CP_HQD_ERROR__AQL_ERROR_MASK                                                                          0x00000020L
+#define CP_HQD_ERROR__PQ_UTCL1_ERROR_MASK                                                                     0x00000100L
+#define CP_HQD_ERROR__IB_UTCL1_ERROR_MASK                                                                     0x00000200L
+#define CP_HQD_ERROR__EOP_UTCL1_ERROR_MASK                                                                    0x00000400L
+#define CP_HQD_ERROR__IQ_UTCL1_ERROR_MASK                                                                     0x00000800L
+#define CP_HQD_ERROR__RRPT_UTCL1_ERROR_MASK                                                                   0x00001000L
+#define CP_HQD_ERROR__WPP_UTCL1_ERROR_MASK                                                                    0x00002000L
+#define CP_HQD_ERROR__SEM_UTCL1_ERROR_MASK                                                                    0x00004000L
+#define CP_HQD_ERROR__DMA_SRC_UTCL1_ERROR_MASK                                                                0x00008000L
+#define CP_HQD_ERROR__DMA_DST_UTCL1_ERROR_MASK                                                                0x00010000L
+#define CP_HQD_ERROR__SR_UTCL1_ERROR_MASK                                                                     0x00020000L
+#define CP_HQD_ERROR__QU_UTCL1_ERROR_MASK                                                                     0x00040000L
+#define CP_HQD_ERROR__TC_UTCL1_ERROR_MASK                                                                     0x00080000L
+//CP_HQD_EOP_WPTR_MEM
+#define CP_HQD_EOP_WPTR_MEM__WPTR__SHIFT                                                                      0x0
+#define CP_HQD_EOP_WPTR_MEM__WPTR_MASK                                                                        0x00001FFFL
+//CP_HQD_AQL_CONTROL
+#define CP_HQD_AQL_CONTROL__CONTROL0__SHIFT                                                                   0x0
+#define CP_HQD_AQL_CONTROL__CONTROL0_EN__SHIFT                                                                0xf
+#define CP_HQD_AQL_CONTROL__CONTROL1__SHIFT                                                                   0x10
+#define CP_HQD_AQL_CONTROL__CONTROL1_EN__SHIFT                                                                0x1f
+#define CP_HQD_AQL_CONTROL__CONTROL0_MASK                                                                     0x00007FFFL
+#define CP_HQD_AQL_CONTROL__CONTROL0_EN_MASK                                                                  0x00008000L
+#define CP_HQD_AQL_CONTROL__CONTROL1_MASK                                                                     0x7FFF0000L
+#define CP_HQD_AQL_CONTROL__CONTROL1_EN_MASK                                                                  0x80000000L
+//CP_HQD_PQ_WPTR_LO
+#define CP_HQD_PQ_WPTR_LO__OFFSET__SHIFT                                                                      0x0
+#define CP_HQD_PQ_WPTR_LO__OFFSET_MASK                                                                        0xFFFFFFFFL
+//CP_HQD_PQ_WPTR_HI
+#define CP_HQD_PQ_WPTR_HI__DATA__SHIFT                                                                        0x0
+#define CP_HQD_PQ_WPTR_HI__DATA_MASK                                                                          0xFFFFFFFFL
+//CP_HQD_AQL_CONTROL_1
+#define CP_HQD_AQL_CONTROL_1__RESERVED__SHIFT                                                                 0x0
+#define CP_HQD_AQL_CONTROL_1__RESERVED_MASK                                                                   0xFFFFFFFFL
+//CP_HQD_AQL_DISPATCH_ID
+#define CP_HQD_AQL_DISPATCH_ID__CONSUMED_OFFSET__SHIFT                                                        0x0
+#define CP_HQD_AQL_DISPATCH_ID__CONSUMED_OFFSET_MASK                                                          0xFFFFFFFFL
+//CP_HQD_AQL_DISPATCH_ID_HI
+#define CP_HQD_AQL_DISPATCH_ID_HI__CONSUMED_OFFSET__SHIFT                                                     0x0
+#define CP_HQD_AQL_DISPATCH_ID_HI__CONSUMED_OFFSET_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: xcd0_gc_tcpdec
+//TCP_WATCH0_ADDR_H
+#define TCP_WATCH0_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH0_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH0_ADDR_L
+#define TCP_WATCH0_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH0_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH0_CNTL
+#define TCP_WATCH0_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH0_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH0_CNTL__ATC__SHIFT                                                                           0x1c
+#define TCP_WATCH0_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH0_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH0_CNTL__MASK_MASK                                                                            0x00FFFFFFL
+#define TCP_WATCH0_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH0_CNTL__ATC_MASK                                                                             0x10000000L
+#define TCP_WATCH0_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH0_CNTL__VALID_MASK                                                                           0x80000000L
+//TCP_WATCH1_ADDR_H
+#define TCP_WATCH1_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH1_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH1_ADDR_L
+#define TCP_WATCH1_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH1_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH1_CNTL
+#define TCP_WATCH1_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH1_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH1_CNTL__ATC__SHIFT                                                                           0x1c
+#define TCP_WATCH1_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH1_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH1_CNTL__MASK_MASK                                                                            0x00FFFFFFL
+#define TCP_WATCH1_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH1_CNTL__ATC_MASK                                                                             0x10000000L
+#define TCP_WATCH1_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH1_CNTL__VALID_MASK                                                                           0x80000000L
+//TCP_WATCH2_ADDR_H
+#define TCP_WATCH2_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH2_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH2_ADDR_L
+#define TCP_WATCH2_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH2_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH2_CNTL
+#define TCP_WATCH2_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH2_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH2_CNTL__ATC__SHIFT                                                                           0x1c
+#define TCP_WATCH2_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH2_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH2_CNTL__MASK_MASK                                                                            0x00FFFFFFL
+#define TCP_WATCH2_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH2_CNTL__ATC_MASK                                                                             0x10000000L
+#define TCP_WATCH2_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH2_CNTL__VALID_MASK                                                                           0x80000000L
+//TCP_WATCH3_ADDR_H
+#define TCP_WATCH3_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH3_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH3_ADDR_L
+#define TCP_WATCH3_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH3_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH3_CNTL
+#define TCP_WATCH3_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH3_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH3_CNTL__ATC__SHIFT                                                                           0x1c
+#define TCP_WATCH3_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH3_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH3_CNTL__MASK_MASK                                                                            0x00FFFFFFL
+#define TCP_WATCH3_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH3_CNTL__ATC_MASK                                                                             0x10000000L
+#define TCP_WATCH3_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH3_CNTL__VALID_MASK                                                                           0x80000000L
+//TCP_GATCL1_CNTL
+#define TCP_GATCL1_CNTL__INVALIDATE_ALL_VMID__SHIFT                                                           0x19
+#define TCP_GATCL1_CNTL__FORCE_MISS__SHIFT                                                                    0x1a
+#define TCP_GATCL1_CNTL__FORCE_IN_ORDER__SHIFT                                                                0x1b
+#define TCP_GATCL1_CNTL__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1c
+#define TCP_GATCL1_CNTL__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1e
+#define TCP_GATCL1_CNTL__INVALIDATE_ALL_VMID_MASK                                                             0x02000000L
+#define TCP_GATCL1_CNTL__FORCE_MISS_MASK                                                                      0x04000000L
+#define TCP_GATCL1_CNTL__FORCE_IN_ORDER_MASK                                                                  0x08000000L
+#define TCP_GATCL1_CNTL__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x30000000L
+#define TCP_GATCL1_CNTL__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xC0000000L
+//TCP_ATC_EDC_GATCL1_CNT
+#define TCP_ATC_EDC_GATCL1_CNT__DATA_SEC__SHIFT                                                               0x0
+#define TCP_ATC_EDC_GATCL1_CNT__DATA_SEC_MASK                                                                 0x000000FFL
+//TCP_GATCL1_DSM_CNTL
+#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A0__SHIFT                                      0x0
+#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A1__SHIFT                                      0x1
+#define TCP_GATCL1_DSM_CNTL__TCP_GATCL1_ENABLE_SINGLE_WRITE_A__SHIFT                                          0x2
+#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A0_MASK                                        0x00000001L
+#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A1_MASK                                        0x00000002L
+#define TCP_GATCL1_DSM_CNTL__TCP_GATCL1_ENABLE_SINGLE_WRITE_A_MASK                                            0x00000004L
+//TCP_DSM_CNTL
+#define TCP_DSM_CNTL__CACHE_RAM_IRRITATOR_DATA_SEL__SHIFT                                                     0x0
+#define TCP_DSM_CNTL__CACHE_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                                 0x2
+#define TCP_DSM_CNTL__LFIFO_RAM_IRRITATOR_DATA_SEL__SHIFT                                                     0x3
+#define TCP_DSM_CNTL__LFIFO_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                                 0x5
+#define TCP_DSM_CNTL__CMD_FIFO_IRRITATOR_DATA_SEL__SHIFT                                                      0x6
+#define TCP_DSM_CNTL__CMD_FIFO_IRRITATOR_SINGLE_WRITE__SHIFT                                                  0x8
+#define TCP_DSM_CNTL__VM_FIFO_IRRITATOR_DATA_SEL__SHIFT                                                       0x9
+#define TCP_DSM_CNTL__VM_FIFO_IRRITATOR_SINGLE_WRITE__SHIFT                                                   0xb
+#define TCP_DSM_CNTL__DB_RAM_IRRITATOR_DATA_SEL__SHIFT                                                        0xc
+#define TCP_DSM_CNTL__DB_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                                    0xe
+#define TCP_DSM_CNTL__UTCL1_LFIFO0_IRRITATOR_DATA_SEL__SHIFT                                                  0xf
+#define TCP_DSM_CNTL__UTCL1_LFIFO0_IRRITATOR_SINGLE_WRITE__SHIFT                                              0x11
+#define TCP_DSM_CNTL__UTCL1_LFIFO1_IRRITATOR_DATA_SEL__SHIFT                                                  0x12
+#define TCP_DSM_CNTL__UTCL1_LFIFO1_IRRITATOR_SINGLE_WRITE__SHIFT                                              0x14
+#define TCP_DSM_CNTL__CACHE_RAM_IRRITATOR_DATA_SEL_MASK                                                       0x00000003L
+#define TCP_DSM_CNTL__CACHE_RAM_IRRITATOR_SINGLE_WRITE_MASK                                                   0x00000004L
+#define TCP_DSM_CNTL__LFIFO_RAM_IRRITATOR_DATA_SEL_MASK                                                       0x00000018L
+#define TCP_DSM_CNTL__LFIFO_RAM_IRRITATOR_SINGLE_WRITE_MASK                                                   0x00000020L
+#define TCP_DSM_CNTL__CMD_FIFO_IRRITATOR_DATA_SEL_MASK                                                        0x000000C0L
+#define TCP_DSM_CNTL__CMD_FIFO_IRRITATOR_SINGLE_WRITE_MASK                                                    0x00000100L
+#define TCP_DSM_CNTL__VM_FIFO_IRRITATOR_DATA_SEL_MASK                                                         0x00000600L
+#define TCP_DSM_CNTL__VM_FIFO_IRRITATOR_SINGLE_WRITE_MASK                                                     0x00000800L
+#define TCP_DSM_CNTL__DB_RAM_IRRITATOR_DATA_SEL_MASK                                                          0x00003000L
+#define TCP_DSM_CNTL__DB_RAM_IRRITATOR_SINGLE_WRITE_MASK                                                      0x00004000L
+#define TCP_DSM_CNTL__UTCL1_LFIFO0_IRRITATOR_DATA_SEL_MASK                                                    0x00018000L
+#define TCP_DSM_CNTL__UTCL1_LFIFO0_IRRITATOR_SINGLE_WRITE_MASK                                                0x00020000L
+#define TCP_DSM_CNTL__UTCL1_LFIFO1_IRRITATOR_DATA_SEL_MASK                                                    0x000C0000L
+#define TCP_DSM_CNTL__UTCL1_LFIFO1_IRRITATOR_SINGLE_WRITE_MASK                                                0x00100000L
+//TCP_CNTL2
+#define TCP_CNTL2__LS_DISABLE_CLOCKS__SHIFT                                                                   0x0
+#define TCP_CNTL2__TCPF_FMT_MGCG_DISABLE__SHIFT                                                               0x8
+#define TCP_CNTL2__TCPI_NEW_MGCG_CTRL_BIT__SHIFT                                                              0xa
+#define TCP_CNTL2__MISS_CLK_DISABLE__SHIFT                                                                    0xb
+#define TCP_CNTL2__ADRS_CLK_DISABLE__SHIFT                                                                    0xc
+#define TCP_CNTL2__VM_CLK_DISABLE__SHIFT                                                                      0xd
+#define TCP_CNTL2__TAGRAM_CLK_DISABLE__SHIFT                                                                  0xe
+#define TCP_CNTL2__LEGACY_MGCG_DISABLE__SHIFT                                                                 0xf
+#define TCP_CNTL2__MEM_MID_GATE_MGCG_DISABLE__SHIFT                                                           0x13
+#define TCP_CNTL2__UTCL1_MID_GATE_MGCG_DISABLE__SHIFT                                                         0x14
+#define TCP_CNTL2__LS_DISABLE_CLOCKS_MASK                                                                     0x000000FFL
+#define TCP_CNTL2__TCPF_FMT_MGCG_DISABLE_MASK                                                                 0x00000100L
+#define TCP_CNTL2__TCPI_NEW_MGCG_CTRL_BIT_MASK                                                                0x00000400L
+#define TCP_CNTL2__MISS_CLK_DISABLE_MASK                                                                      0x00000800L
+#define TCP_CNTL2__ADRS_CLK_DISABLE_MASK                                                                      0x00001000L
+#define TCP_CNTL2__VM_CLK_DISABLE_MASK                                                                        0x00002000L
+#define TCP_CNTL2__TAGRAM_CLK_DISABLE_MASK                                                                    0x00004000L
+#define TCP_CNTL2__LEGACY_MGCG_DISABLE_MASK                                                                   0x00008000L
+#define TCP_CNTL2__MEM_MID_GATE_MGCG_DISABLE_MASK                                                             0x00080000L
+#define TCP_CNTL2__UTCL1_MID_GATE_MGCG_DISABLE_MASK                                                           0x00100000L
+//TCP_UTCL1_CNTL1
+#define TCP_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                              0x0
+#define TCP_UTCL1_CNTL1__GPUVM_64K_DEFAULT__SHIFT                                                             0x1
+#define TCP_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                               0x2
+#define TCP_UTCL1_CNTL1__RESP_MODE__SHIFT                                                                     0x3
+#define TCP_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                               0x5
+#define TCP_UTCL1_CNTL1__CLIENTID__SHIFT                                                                      0x7
+#define TCP_UTCL1_CNTL1__UTCL1_FGCG_REPEATER_DISABLE__SHIFT                                                   0x10
+#define TCP_UTCL1_CNTL1__REG_INV_VMID__SHIFT                                                                  0x13
+#define TCP_UTCL1_CNTL1__REG_INV_ALL_VMID__SHIFT                                                              0x17
+#define TCP_UTCL1_CNTL1__REG_INV_TOGGLE__SHIFT                                                                0x18
+#define TCP_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID__SHIFT                                                    0x19
+#define TCP_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                                    0x1a
+#define TCP_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1c
+#define TCP_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1e
+#define TCP_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                                0x00000001L
+#define TCP_UTCL1_CNTL1__GPUVM_64K_DEFAULT_MASK                                                               0x00000002L
+#define TCP_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                                 0x00000004L
+#define TCP_UTCL1_CNTL1__RESP_MODE_MASK                                                                       0x00000018L
+#define TCP_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                                 0x00000060L
+#define TCP_UTCL1_CNTL1__CLIENTID_MASK                                                                        0x0000FF80L
+#define TCP_UTCL1_CNTL1__UTCL1_FGCG_REPEATER_DISABLE_MASK                                                     0x00010000L
+#define TCP_UTCL1_CNTL1__REG_INV_VMID_MASK                                                                    0x00780000L
+#define TCP_UTCL1_CNTL1__REG_INV_ALL_VMID_MASK                                                                0x00800000L
+#define TCP_UTCL1_CNTL1__REG_INV_TOGGLE_MASK                                                                  0x01000000L
+#define TCP_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID_MASK                                                      0x02000000L
+#define TCP_UTCL1_CNTL1__FORCE_MISS_MASK                                                                      0x04000000L
+#define TCP_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x30000000L
+#define TCP_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xC0000000L
+//TCP_UTCL1_CNTL2
+#define TCP_UTCL1_CNTL2__SPARE__SHIFT                                                                         0x0
+#define TCP_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                0x9
+#define TCP_UTCL1_CNTL2__ANY_LINE_VALID__SHIFT                                                                0xa
+#define TCP_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                                0xc
+#define TCP_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                                   0xe
+#define TCP_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                           0xf
+#define TCP_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                          0x1a
+#define TCP_UTCL1_CNTL2__THRASHING_TIMEOUT_PROTECT_ENABLE__SHIFT                                              0x1b
+#define TCP_UTCL1_CNTL2__THRASHING_ENABLE__SHIFT                                                              0x1c
+#define TCP_UTCL1_CNTL2__SPARE_MASK                                                                           0x000000FFL
+#define TCP_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                                  0x00000200L
+#define TCP_UTCL1_CNTL2__ANY_LINE_VALID_MASK                                                                  0x00000400L
+#define TCP_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                                  0x00001000L
+#define TCP_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                                     0x00004000L
+#define TCP_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                             0x00008000L
+#define TCP_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                            0x04000000L
+#define TCP_UTCL1_CNTL2__THRASHING_TIMEOUT_PROTECT_ENABLE_MASK                                                0x08000000L
+#define TCP_UTCL1_CNTL2__THRASHING_ENABLE_MASK                                                                0x10000000L
+//TCP_UTCL1_STATUS
+#define TCP_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define TCP_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define TCP_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define TCP_UTCL1_STATUS__TIMEOUT_DETECTED__SHIFT                                                             0x3
+#define TCP_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define TCP_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define TCP_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define TCP_UTCL1_STATUS__TIMEOUT_DETECTED_MASK                                                               0x00000008L
+//TCP_DSM_CNTL2
+#define TCP_DSM_CNTL2__CACHE_RAM_ENABLE_ERROR_INJECT__SHIFT                                                   0x0
+#define TCP_DSM_CNTL2__CACHE_RAM_SELECT_INJECT_DELAY__SHIFT                                                   0x2
+#define TCP_DSM_CNTL2__LFIFO_RAM_ENABLE_ERROR_INJECT__SHIFT                                                   0x3
+#define TCP_DSM_CNTL2__LFIFO_RAM_SELECT_INJECT_DELAY__SHIFT                                                   0x5
+#define TCP_DSM_CNTL2__CMD_FIFO_ENABLE_ERROR_INJECT__SHIFT                                                    0x6
+#define TCP_DSM_CNTL2__CMD_FIFO_SELECT_INJECT_DELAY__SHIFT                                                    0x8
+#define TCP_DSM_CNTL2__VM_FIFO_ENABLE_ERROR_INJECT__SHIFT                                                     0x9
+#define TCP_DSM_CNTL2__VM_FIFO_SELECT_INJECT_DELAY__SHIFT                                                     0xb
+#define TCP_DSM_CNTL2__DB_RAM_ENABLE_ERROR_INJECT__SHIFT                                                      0xc
+#define TCP_DSM_CNTL2__DB_RAM_SELECT_INJECT_DELAY__SHIFT                                                      0xe
+#define TCP_DSM_CNTL2__UTCL1_LFIFO0_ENABLE_ERROR_INJECT__SHIFT                                                0xf
+#define TCP_DSM_CNTL2__UTCL1_LFIFO0_SELECT_INJECT_DELAY__SHIFT                                                0x11
+#define TCP_DSM_CNTL2__UTCL1_LFIFO1_ENABLE_ERROR_INJECT__SHIFT                                                0x12
+#define TCP_DSM_CNTL2__UTCL1_LFIFO1_SELECT_INJECT_DELAY__SHIFT                                                0x14
+#define TCP_DSM_CNTL2__TCP_INJECT_DELAY__SHIFT                                                                0x1a
+#define TCP_DSM_CNTL2__CACHE_RAM_ENABLE_ERROR_INJECT_MASK                                                     0x00000003L
+#define TCP_DSM_CNTL2__CACHE_RAM_SELECT_INJECT_DELAY_MASK                                                     0x00000004L
+#define TCP_DSM_CNTL2__LFIFO_RAM_ENABLE_ERROR_INJECT_MASK                                                     0x00000018L
+#define TCP_DSM_CNTL2__LFIFO_RAM_SELECT_INJECT_DELAY_MASK                                                     0x00000020L
+#define TCP_DSM_CNTL2__CMD_FIFO_ENABLE_ERROR_INJECT_MASK                                                      0x000000C0L
+#define TCP_DSM_CNTL2__CMD_FIFO_SELECT_INJECT_DELAY_MASK                                                      0x00000100L
+#define TCP_DSM_CNTL2__VM_FIFO_ENABLE_ERROR_INJECT_MASK                                                       0x00000600L
+#define TCP_DSM_CNTL2__VM_FIFO_SELECT_INJECT_DELAY_MASK                                                       0x00000800L
+#define TCP_DSM_CNTL2__DB_RAM_ENABLE_ERROR_INJECT_MASK                                                        0x00003000L
+#define TCP_DSM_CNTL2__DB_RAM_SELECT_INJECT_DELAY_MASK                                                        0x00004000L
+#define TCP_DSM_CNTL2__UTCL1_LFIFO0_ENABLE_ERROR_INJECT_MASK                                                  0x00018000L
+#define TCP_DSM_CNTL2__UTCL1_LFIFO0_SELECT_INJECT_DELAY_MASK                                                  0x00020000L
+#define TCP_DSM_CNTL2__UTCL1_LFIFO1_ENABLE_ERROR_INJECT_MASK                                                  0x000C0000L
+#define TCP_DSM_CNTL2__UTCL1_LFIFO1_SELECT_INJECT_DELAY_MASK                                                  0x00100000L
+#define TCP_DSM_CNTL2__TCP_INJECT_DELAY_MASK                                                                  0xFC000000L
+//TCP_PERFCOUNTER_FILTER
+#define TCP_PERFCOUNTER_FILTER__BUFFER__SHIFT                                                                 0x0
+#define TCP_PERFCOUNTER_FILTER__FLAT__SHIFT                                                                   0x1
+#define TCP_PERFCOUNTER_FILTER__DIM__SHIFT                                                                    0x2
+#define TCP_PERFCOUNTER_FILTER__DATA_FORMAT__SHIFT                                                            0x5
+#define TCP_PERFCOUNTER_FILTER__NUM_FORMAT__SHIFT                                                             0xb
+#define TCP_PERFCOUNTER_FILTER__SW_MODE__SHIFT                                                                0xf
+#define TCP_PERFCOUNTER_FILTER__NUM_SAMPLES__SHIFT                                                            0x14
+#define TCP_PERFCOUNTER_FILTER__OPCODE_TYPE__SHIFT                                                            0x16
+#define TCP_PERFCOUNTER_FILTER__GLC__SHIFT                                                                    0x19
+#define TCP_PERFCOUNTER_FILTER__SLC__SHIFT                                                                    0x1a
+#define TCP_PERFCOUNTER_FILTER__COMPRESSION_ENABLE__SHIFT                                                     0x1b
+#define TCP_PERFCOUNTER_FILTER__ADDR_MODE__SHIFT                                                              0x1c
+#define TCP_PERFCOUNTER_FILTER__BUFFER_MASK                                                                   0x00000001L
+#define TCP_PERFCOUNTER_FILTER__FLAT_MASK                                                                     0x00000002L
+#define TCP_PERFCOUNTER_FILTER__DIM_MASK                                                                      0x0000001CL
+#define TCP_PERFCOUNTER_FILTER__DATA_FORMAT_MASK                                                              0x000007E0L
+#define TCP_PERFCOUNTER_FILTER__NUM_FORMAT_MASK                                                               0x00007800L
+#define TCP_PERFCOUNTER_FILTER__SW_MODE_MASK                                                                  0x000F8000L
+#define TCP_PERFCOUNTER_FILTER__NUM_SAMPLES_MASK                                                              0x00300000L
+#define TCP_PERFCOUNTER_FILTER__OPCODE_TYPE_MASK                                                              0x01C00000L
+#define TCP_PERFCOUNTER_FILTER__GLC_MASK                                                                      0x02000000L
+#define TCP_PERFCOUNTER_FILTER__SLC_MASK                                                                      0x04000000L
+#define TCP_PERFCOUNTER_FILTER__COMPRESSION_ENABLE_MASK                                                       0x08000000L
+#define TCP_PERFCOUNTER_FILTER__ADDR_MODE_MASK                                                                0x70000000L
+//TCP_PERFCOUNTER_FILTER_EN
+#define TCP_PERFCOUNTER_FILTER_EN__BUFFER__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER_FILTER_EN__FLAT__SHIFT                                                                0x1
+#define TCP_PERFCOUNTER_FILTER_EN__DIM__SHIFT                                                                 0x2
+#define TCP_PERFCOUNTER_FILTER_EN__DATA_FORMAT__SHIFT                                                         0x3
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_FORMAT__SHIFT                                                          0x4
+#define TCP_PERFCOUNTER_FILTER_EN__SW_MODE__SHIFT                                                             0x5
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_SAMPLES__SHIFT                                                         0x6
+#define TCP_PERFCOUNTER_FILTER_EN__OPCODE_TYPE__SHIFT                                                         0x7
+#define TCP_PERFCOUNTER_FILTER_EN__GLC__SHIFT                                                                 0x8
+#define TCP_PERFCOUNTER_FILTER_EN__SLC__SHIFT                                                                 0x9
+#define TCP_PERFCOUNTER_FILTER_EN__COMPRESSION_ENABLE__SHIFT                                                  0xa
+#define TCP_PERFCOUNTER_FILTER_EN__ADDR_MODE__SHIFT                                                           0xb
+#define TCP_PERFCOUNTER_FILTER_EN__BUFFER_MASK                                                                0x00000001L
+#define TCP_PERFCOUNTER_FILTER_EN__FLAT_MASK                                                                  0x00000002L
+#define TCP_PERFCOUNTER_FILTER_EN__DIM_MASK                                                                   0x00000004L
+#define TCP_PERFCOUNTER_FILTER_EN__DATA_FORMAT_MASK                                                           0x00000008L
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_FORMAT_MASK                                                            0x00000010L
+#define TCP_PERFCOUNTER_FILTER_EN__SW_MODE_MASK                                                               0x00000020L
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_SAMPLES_MASK                                                           0x00000040L
+#define TCP_PERFCOUNTER_FILTER_EN__OPCODE_TYPE_MASK                                                           0x00000080L
+#define TCP_PERFCOUNTER_FILTER_EN__GLC_MASK                                                                   0x00000100L
+#define TCP_PERFCOUNTER_FILTER_EN__SLC_MASK                                                                   0x00000200L
+#define TCP_PERFCOUNTER_FILTER_EN__COMPRESSION_ENABLE_MASK                                                    0x00000400L
+#define TCP_PERFCOUNTER_FILTER_EN__ADDR_MODE_MASK                                                             0x00000800L
+
+
+// addressBlock: xcd0_gc_gdspdec
+//GDS_VMID0_BASE
+#define GDS_VMID0_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID0_BASE__BASE_MASK                                                                             0x0000FFFFL
+//GDS_VMID0_SIZE
+#define GDS_VMID0_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID0_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+//GDS_VMID1_BASE
+#define GDS_VMID1_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID1_BASE__BASE_MASK                                                                             0x0000FFFFL
+//GDS_VMID1_SIZE
+#define GDS_VMID1_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID1_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+//GDS_VMID2_BASE
+#define GDS_VMID2_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID2_BASE__BASE_MASK                                                                             0x0000FFFFL
+//GDS_VMID2_SIZE
+#define GDS_VMID2_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID2_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+//GDS_VMID3_BASE
+#define GDS_VMID3_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID3_BASE__BASE_MASK                                                                             0x0000FFFFL
+//GDS_VMID3_SIZE
+#define GDS_VMID3_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID3_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+//GDS_VMID4_BASE
+#define GDS_VMID4_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID4_BASE__BASE_MASK                                                                             0x0000FFFFL
+//GDS_VMID4_SIZE
+#define GDS_VMID4_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID4_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+//GDS_VMID5_BASE
+#define GDS_VMID5_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID5_BASE__BASE_MASK                                                                             0x0000FFFFL
+//GDS_VMID5_SIZE
+#define GDS_VMID5_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID5_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+//GDS_VMID6_BASE
+#define GDS_VMID6_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID6_BASE__BASE_MASK                                                                             0x0000FFFFL
+//GDS_VMID6_SIZE
+#define GDS_VMID6_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID6_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+//GDS_VMID7_BASE
+#define GDS_VMID7_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID7_BASE__BASE_MASK                                                                             0x0000FFFFL
+//GDS_VMID7_SIZE
+#define GDS_VMID7_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID7_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+//GDS_VMID8_BASE
+#define GDS_VMID8_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID8_BASE__BASE_MASK                                                                             0x0000FFFFL
+//GDS_VMID8_SIZE
+#define GDS_VMID8_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID8_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+//GDS_VMID9_BASE
+#define GDS_VMID9_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID9_BASE__BASE_MASK                                                                             0x0000FFFFL
+//GDS_VMID9_SIZE
+#define GDS_VMID9_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID9_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+//GDS_VMID10_BASE
+#define GDS_VMID10_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID10_BASE__BASE_MASK                                                                            0x0000FFFFL
+//GDS_VMID10_SIZE
+#define GDS_VMID10_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID10_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+//GDS_VMID11_BASE
+#define GDS_VMID11_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID11_BASE__BASE_MASK                                                                            0x0000FFFFL
+//GDS_VMID11_SIZE
+#define GDS_VMID11_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID11_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+//GDS_VMID12_BASE
+#define GDS_VMID12_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID12_BASE__BASE_MASK                                                                            0x0000FFFFL
+//GDS_VMID12_SIZE
+#define GDS_VMID12_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID12_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+//GDS_VMID13_BASE
+#define GDS_VMID13_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID13_BASE__BASE_MASK                                                                            0x0000FFFFL
+//GDS_VMID13_SIZE
+#define GDS_VMID13_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID13_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+//GDS_VMID14_BASE
+#define GDS_VMID14_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID14_BASE__BASE_MASK                                                                            0x0000FFFFL
+//GDS_VMID14_SIZE
+#define GDS_VMID14_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID14_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+//GDS_VMID15_BASE
+#define GDS_VMID15_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID15_BASE__BASE_MASK                                                                            0x0000FFFFL
+//GDS_VMID15_SIZE
+#define GDS_VMID15_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID15_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+//GDS_GWS_VMID0
+#define GDS_GWS_VMID0__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID0__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID0__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID0__SIZE_MASK                                                                              0x007F0000L
+//GDS_GWS_VMID1
+#define GDS_GWS_VMID1__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID1__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID1__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID1__SIZE_MASK                                                                              0x007F0000L
+//GDS_GWS_VMID2
+#define GDS_GWS_VMID2__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID2__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID2__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID2__SIZE_MASK                                                                              0x007F0000L
+//GDS_GWS_VMID3
+#define GDS_GWS_VMID3__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID3__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID3__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID3__SIZE_MASK                                                                              0x007F0000L
+//GDS_GWS_VMID4
+#define GDS_GWS_VMID4__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID4__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID4__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID4__SIZE_MASK                                                                              0x007F0000L
+//GDS_GWS_VMID5
+#define GDS_GWS_VMID5__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID5__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID5__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID5__SIZE_MASK                                                                              0x007F0000L
+//GDS_GWS_VMID6
+#define GDS_GWS_VMID6__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID6__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID6__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID6__SIZE_MASK                                                                              0x007F0000L
+//GDS_GWS_VMID7
+#define GDS_GWS_VMID7__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID7__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID7__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID7__SIZE_MASK                                                                              0x007F0000L
+//GDS_GWS_VMID8
+#define GDS_GWS_VMID8__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID8__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID8__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID8__SIZE_MASK                                                                              0x007F0000L
+//GDS_GWS_VMID9
+#define GDS_GWS_VMID9__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID9__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID9__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID9__SIZE_MASK                                                                              0x007F0000L
+//GDS_GWS_VMID10
+#define GDS_GWS_VMID10__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID10__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID10__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID10__SIZE_MASK                                                                             0x007F0000L
+//GDS_GWS_VMID11
+#define GDS_GWS_VMID11__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID11__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID11__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID11__SIZE_MASK                                                                             0x007F0000L
+//GDS_GWS_VMID12
+#define GDS_GWS_VMID12__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID12__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID12__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID12__SIZE_MASK                                                                             0x007F0000L
+//GDS_GWS_VMID13
+#define GDS_GWS_VMID13__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID13__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID13__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID13__SIZE_MASK                                                                             0x007F0000L
+//GDS_GWS_VMID14
+#define GDS_GWS_VMID14__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID14__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID14__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID14__SIZE_MASK                                                                             0x007F0000L
+//GDS_GWS_VMID15
+#define GDS_GWS_VMID15__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID15__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID15__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID15__SIZE_MASK                                                                             0x007F0000L
+//GDS_OA_VMID0
+#define GDS_OA_VMID0__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID0__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID0__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID0__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID1
+#define GDS_OA_VMID1__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID1__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID1__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID1__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID2
+#define GDS_OA_VMID2__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID2__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID2__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID2__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID3
+#define GDS_OA_VMID3__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID3__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID3__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID3__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID4
+#define GDS_OA_VMID4__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID4__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID4__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID4__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID5
+#define GDS_OA_VMID5__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID5__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID5__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID5__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID6
+#define GDS_OA_VMID6__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID6__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID6__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID6__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID7
+#define GDS_OA_VMID7__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID7__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID7__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID7__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID8
+#define GDS_OA_VMID8__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID8__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID8__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID8__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID9
+#define GDS_OA_VMID9__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID9__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID9__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID9__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID10
+#define GDS_OA_VMID10__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID10__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID10__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID10__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_OA_VMID11
+#define GDS_OA_VMID11__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID11__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID11__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID11__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_OA_VMID12
+#define GDS_OA_VMID12__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID12__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID12__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID12__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_OA_VMID13
+#define GDS_OA_VMID13__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID13__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID13__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID13__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_OA_VMID14
+#define GDS_OA_VMID14__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID14__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID14__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID14__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_OA_VMID15
+#define GDS_OA_VMID15__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID15__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID15__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID15__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_GWS_RESET0
+#define GDS_GWS_RESET0__RESOURCE0_RESET__SHIFT                                                                0x0
+#define GDS_GWS_RESET0__RESOURCE1_RESET__SHIFT                                                                0x1
+#define GDS_GWS_RESET0__RESOURCE2_RESET__SHIFT                                                                0x2
+#define GDS_GWS_RESET0__RESOURCE3_RESET__SHIFT                                                                0x3
+#define GDS_GWS_RESET0__RESOURCE4_RESET__SHIFT                                                                0x4
+#define GDS_GWS_RESET0__RESOURCE5_RESET__SHIFT                                                                0x5
+#define GDS_GWS_RESET0__RESOURCE6_RESET__SHIFT                                                                0x6
+#define GDS_GWS_RESET0__RESOURCE7_RESET__SHIFT                                                                0x7
+#define GDS_GWS_RESET0__RESOURCE8_RESET__SHIFT                                                                0x8
+#define GDS_GWS_RESET0__RESOURCE9_RESET__SHIFT                                                                0x9
+#define GDS_GWS_RESET0__RESOURCE10_RESET__SHIFT                                                               0xa
+#define GDS_GWS_RESET0__RESOURCE11_RESET__SHIFT                                                               0xb
+#define GDS_GWS_RESET0__RESOURCE12_RESET__SHIFT                                                               0xc
+#define GDS_GWS_RESET0__RESOURCE13_RESET__SHIFT                                                               0xd
+#define GDS_GWS_RESET0__RESOURCE14_RESET__SHIFT                                                               0xe
+#define GDS_GWS_RESET0__RESOURCE15_RESET__SHIFT                                                               0xf
+#define GDS_GWS_RESET0__RESOURCE16_RESET__SHIFT                                                               0x10
+#define GDS_GWS_RESET0__RESOURCE17_RESET__SHIFT                                                               0x11
+#define GDS_GWS_RESET0__RESOURCE18_RESET__SHIFT                                                               0x12
+#define GDS_GWS_RESET0__RESOURCE19_RESET__SHIFT                                                               0x13
+#define GDS_GWS_RESET0__RESOURCE20_RESET__SHIFT                                                               0x14
+#define GDS_GWS_RESET0__RESOURCE21_RESET__SHIFT                                                               0x15
+#define GDS_GWS_RESET0__RESOURCE22_RESET__SHIFT                                                               0x16
+#define GDS_GWS_RESET0__RESOURCE23_RESET__SHIFT                                                               0x17
+#define GDS_GWS_RESET0__RESOURCE24_RESET__SHIFT                                                               0x18
+#define GDS_GWS_RESET0__RESOURCE25_RESET__SHIFT                                                               0x19
+#define GDS_GWS_RESET0__RESOURCE26_RESET__SHIFT                                                               0x1a
+#define GDS_GWS_RESET0__RESOURCE27_RESET__SHIFT                                                               0x1b
+#define GDS_GWS_RESET0__RESOURCE28_RESET__SHIFT                                                               0x1c
+#define GDS_GWS_RESET0__RESOURCE29_RESET__SHIFT                                                               0x1d
+#define GDS_GWS_RESET0__RESOURCE30_RESET__SHIFT                                                               0x1e
+#define GDS_GWS_RESET0__RESOURCE31_RESET__SHIFT                                                               0x1f
+#define GDS_GWS_RESET0__RESOURCE0_RESET_MASK                                                                  0x00000001L
+#define GDS_GWS_RESET0__RESOURCE1_RESET_MASK                                                                  0x00000002L
+#define GDS_GWS_RESET0__RESOURCE2_RESET_MASK                                                                  0x00000004L
+#define GDS_GWS_RESET0__RESOURCE3_RESET_MASK                                                                  0x00000008L
+#define GDS_GWS_RESET0__RESOURCE4_RESET_MASK                                                                  0x00000010L
+#define GDS_GWS_RESET0__RESOURCE5_RESET_MASK                                                                  0x00000020L
+#define GDS_GWS_RESET0__RESOURCE6_RESET_MASK                                                                  0x00000040L
+#define GDS_GWS_RESET0__RESOURCE7_RESET_MASK                                                                  0x00000080L
+#define GDS_GWS_RESET0__RESOURCE8_RESET_MASK                                                                  0x00000100L
+#define GDS_GWS_RESET0__RESOURCE9_RESET_MASK                                                                  0x00000200L
+#define GDS_GWS_RESET0__RESOURCE10_RESET_MASK                                                                 0x00000400L
+#define GDS_GWS_RESET0__RESOURCE11_RESET_MASK                                                                 0x00000800L
+#define GDS_GWS_RESET0__RESOURCE12_RESET_MASK                                                                 0x00001000L
+#define GDS_GWS_RESET0__RESOURCE13_RESET_MASK                                                                 0x00002000L
+#define GDS_GWS_RESET0__RESOURCE14_RESET_MASK                                                                 0x00004000L
+#define GDS_GWS_RESET0__RESOURCE15_RESET_MASK                                                                 0x00008000L
+#define GDS_GWS_RESET0__RESOURCE16_RESET_MASK                                                                 0x00010000L
+#define GDS_GWS_RESET0__RESOURCE17_RESET_MASK                                                                 0x00020000L
+#define GDS_GWS_RESET0__RESOURCE18_RESET_MASK                                                                 0x00040000L
+#define GDS_GWS_RESET0__RESOURCE19_RESET_MASK                                                                 0x00080000L
+#define GDS_GWS_RESET0__RESOURCE20_RESET_MASK                                                                 0x00100000L
+#define GDS_GWS_RESET0__RESOURCE21_RESET_MASK                                                                 0x00200000L
+#define GDS_GWS_RESET0__RESOURCE22_RESET_MASK                                                                 0x00400000L
+#define GDS_GWS_RESET0__RESOURCE23_RESET_MASK                                                                 0x00800000L
+#define GDS_GWS_RESET0__RESOURCE24_RESET_MASK                                                                 0x01000000L
+#define GDS_GWS_RESET0__RESOURCE25_RESET_MASK                                                                 0x02000000L
+#define GDS_GWS_RESET0__RESOURCE26_RESET_MASK                                                                 0x04000000L
+#define GDS_GWS_RESET0__RESOURCE27_RESET_MASK                                                                 0x08000000L
+#define GDS_GWS_RESET0__RESOURCE28_RESET_MASK                                                                 0x10000000L
+#define GDS_GWS_RESET0__RESOURCE29_RESET_MASK                                                                 0x20000000L
+#define GDS_GWS_RESET0__RESOURCE30_RESET_MASK                                                                 0x40000000L
+#define GDS_GWS_RESET0__RESOURCE31_RESET_MASK                                                                 0x80000000L
+//GDS_GWS_RESET1
+#define GDS_GWS_RESET1__RESOURCE32_RESET__SHIFT                                                               0x0
+#define GDS_GWS_RESET1__RESOURCE33_RESET__SHIFT                                                               0x1
+#define GDS_GWS_RESET1__RESOURCE34_RESET__SHIFT                                                               0x2
+#define GDS_GWS_RESET1__RESOURCE35_RESET__SHIFT                                                               0x3
+#define GDS_GWS_RESET1__RESOURCE36_RESET__SHIFT                                                               0x4
+#define GDS_GWS_RESET1__RESOURCE37_RESET__SHIFT                                                               0x5
+#define GDS_GWS_RESET1__RESOURCE38_RESET__SHIFT                                                               0x6
+#define GDS_GWS_RESET1__RESOURCE39_RESET__SHIFT                                                               0x7
+#define GDS_GWS_RESET1__RESOURCE40_RESET__SHIFT                                                               0x8
+#define GDS_GWS_RESET1__RESOURCE41_RESET__SHIFT                                                               0x9
+#define GDS_GWS_RESET1__RESOURCE42_RESET__SHIFT                                                               0xa
+#define GDS_GWS_RESET1__RESOURCE43_RESET__SHIFT                                                               0xb
+#define GDS_GWS_RESET1__RESOURCE44_RESET__SHIFT                                                               0xc
+#define GDS_GWS_RESET1__RESOURCE45_RESET__SHIFT                                                               0xd
+#define GDS_GWS_RESET1__RESOURCE46_RESET__SHIFT                                                               0xe
+#define GDS_GWS_RESET1__RESOURCE47_RESET__SHIFT                                                               0xf
+#define GDS_GWS_RESET1__RESOURCE48_RESET__SHIFT                                                               0x10
+#define GDS_GWS_RESET1__RESOURCE49_RESET__SHIFT                                                               0x11
+#define GDS_GWS_RESET1__RESOURCE50_RESET__SHIFT                                                               0x12
+#define GDS_GWS_RESET1__RESOURCE51_RESET__SHIFT                                                               0x13
+#define GDS_GWS_RESET1__RESOURCE52_RESET__SHIFT                                                               0x14
+#define GDS_GWS_RESET1__RESOURCE53_RESET__SHIFT                                                               0x15
+#define GDS_GWS_RESET1__RESOURCE54_RESET__SHIFT                                                               0x16
+#define GDS_GWS_RESET1__RESOURCE55_RESET__SHIFT                                                               0x17
+#define GDS_GWS_RESET1__RESOURCE56_RESET__SHIFT                                                               0x18
+#define GDS_GWS_RESET1__RESOURCE57_RESET__SHIFT                                                               0x19
+#define GDS_GWS_RESET1__RESOURCE58_RESET__SHIFT                                                               0x1a
+#define GDS_GWS_RESET1__RESOURCE59_RESET__SHIFT                                                               0x1b
+#define GDS_GWS_RESET1__RESOURCE60_RESET__SHIFT                                                               0x1c
+#define GDS_GWS_RESET1__RESOURCE61_RESET__SHIFT                                                               0x1d
+#define GDS_GWS_RESET1__RESOURCE62_RESET__SHIFT                                                               0x1e
+#define GDS_GWS_RESET1__RESOURCE63_RESET__SHIFT                                                               0x1f
+#define GDS_GWS_RESET1__RESOURCE32_RESET_MASK                                                                 0x00000001L
+#define GDS_GWS_RESET1__RESOURCE33_RESET_MASK                                                                 0x00000002L
+#define GDS_GWS_RESET1__RESOURCE34_RESET_MASK                                                                 0x00000004L
+#define GDS_GWS_RESET1__RESOURCE35_RESET_MASK                                                                 0x00000008L
+#define GDS_GWS_RESET1__RESOURCE36_RESET_MASK                                                                 0x00000010L
+#define GDS_GWS_RESET1__RESOURCE37_RESET_MASK                                                                 0x00000020L
+#define GDS_GWS_RESET1__RESOURCE38_RESET_MASK                                                                 0x00000040L
+#define GDS_GWS_RESET1__RESOURCE39_RESET_MASK                                                                 0x00000080L
+#define GDS_GWS_RESET1__RESOURCE40_RESET_MASK                                                                 0x00000100L
+#define GDS_GWS_RESET1__RESOURCE41_RESET_MASK                                                                 0x00000200L
+#define GDS_GWS_RESET1__RESOURCE42_RESET_MASK                                                                 0x00000400L
+#define GDS_GWS_RESET1__RESOURCE43_RESET_MASK                                                                 0x00000800L
+#define GDS_GWS_RESET1__RESOURCE44_RESET_MASK                                                                 0x00001000L
+#define GDS_GWS_RESET1__RESOURCE45_RESET_MASK                                                                 0x00002000L
+#define GDS_GWS_RESET1__RESOURCE46_RESET_MASK                                                                 0x00004000L
+#define GDS_GWS_RESET1__RESOURCE47_RESET_MASK                                                                 0x00008000L
+#define GDS_GWS_RESET1__RESOURCE48_RESET_MASK                                                                 0x00010000L
+#define GDS_GWS_RESET1__RESOURCE49_RESET_MASK                                                                 0x00020000L
+#define GDS_GWS_RESET1__RESOURCE50_RESET_MASK                                                                 0x00040000L
+#define GDS_GWS_RESET1__RESOURCE51_RESET_MASK                                                                 0x00080000L
+#define GDS_GWS_RESET1__RESOURCE52_RESET_MASK                                                                 0x00100000L
+#define GDS_GWS_RESET1__RESOURCE53_RESET_MASK                                                                 0x00200000L
+#define GDS_GWS_RESET1__RESOURCE54_RESET_MASK                                                                 0x00400000L
+#define GDS_GWS_RESET1__RESOURCE55_RESET_MASK                                                                 0x00800000L
+#define GDS_GWS_RESET1__RESOURCE56_RESET_MASK                                                                 0x01000000L
+#define GDS_GWS_RESET1__RESOURCE57_RESET_MASK                                                                 0x02000000L
+#define GDS_GWS_RESET1__RESOURCE58_RESET_MASK                                                                 0x04000000L
+#define GDS_GWS_RESET1__RESOURCE59_RESET_MASK                                                                 0x08000000L
+#define GDS_GWS_RESET1__RESOURCE60_RESET_MASK                                                                 0x10000000L
+#define GDS_GWS_RESET1__RESOURCE61_RESET_MASK                                                                 0x20000000L
+#define GDS_GWS_RESET1__RESOURCE62_RESET_MASK                                                                 0x40000000L
+#define GDS_GWS_RESET1__RESOURCE63_RESET_MASK                                                                 0x80000000L
+//GDS_GWS_RESOURCE_RESET
+#define GDS_GWS_RESOURCE_RESET__RESET__SHIFT                                                                  0x0
+#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID__SHIFT                                                            0x8
+#define GDS_GWS_RESOURCE_RESET__RESET_MASK                                                                    0x00000001L
+#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID_MASK                                                              0x0000FF00L
+//GDS_COMPUTE_MAX_WAVE_ID
+#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                           0x0
+#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                             0x00000FFFL
+//GDS_OA_RESET_MASK
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_PIX_RESET__SHIFT                                                       0x0
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_VTX_RESET__SHIFT                                                       0x1
+#define GDS_OA_RESET_MASK__ME0_CS_RESET__SHIFT                                                                0x2
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_GS_RESET__SHIFT                                                        0x3
+#define GDS_OA_RESET_MASK__ME1_PIPE0_RESET__SHIFT                                                             0x4
+#define GDS_OA_RESET_MASK__ME1_PIPE1_RESET__SHIFT                                                             0x5
+#define GDS_OA_RESET_MASK__ME1_PIPE2_RESET__SHIFT                                                             0x6
+#define GDS_OA_RESET_MASK__ME1_PIPE3_RESET__SHIFT                                                             0x7
+#define GDS_OA_RESET_MASK__ME2_PIPE0_RESET__SHIFT                                                             0x8
+#define GDS_OA_RESET_MASK__ME2_PIPE1_RESET__SHIFT                                                             0x9
+#define GDS_OA_RESET_MASK__ME2_PIPE2_RESET__SHIFT                                                             0xa
+#define GDS_OA_RESET_MASK__ME2_PIPE3_RESET__SHIFT                                                             0xb
+#define GDS_OA_RESET_MASK__UNUSED1__SHIFT                                                                     0xc
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_PIX_RESET_MASK                                                         0x00000001L
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_VTX_RESET_MASK                                                         0x00000002L
+#define GDS_OA_RESET_MASK__ME0_CS_RESET_MASK                                                                  0x00000004L
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_GS_RESET_MASK                                                          0x00000008L
+#define GDS_OA_RESET_MASK__ME1_PIPE0_RESET_MASK                                                               0x00000010L
+#define GDS_OA_RESET_MASK__ME1_PIPE1_RESET_MASK                                                               0x00000020L
+#define GDS_OA_RESET_MASK__ME1_PIPE2_RESET_MASK                                                               0x00000040L
+#define GDS_OA_RESET_MASK__ME1_PIPE3_RESET_MASK                                                               0x00000080L
+#define GDS_OA_RESET_MASK__ME2_PIPE0_RESET_MASK                                                               0x00000100L
+#define GDS_OA_RESET_MASK__ME2_PIPE1_RESET_MASK                                                               0x00000200L
+#define GDS_OA_RESET_MASK__ME2_PIPE2_RESET_MASK                                                               0x00000400L
+#define GDS_OA_RESET_MASK__ME2_PIPE3_RESET_MASK                                                               0x00000800L
+#define GDS_OA_RESET_MASK__UNUSED1_MASK                                                                       0xFFFFF000L
+//GDS_OA_RESET
+#define GDS_OA_RESET__RESET__SHIFT                                                                            0x0
+#define GDS_OA_RESET__PIPE_ID__SHIFT                                                                          0x8
+#define GDS_OA_RESET__RESET_MASK                                                                              0x00000001L
+#define GDS_OA_RESET__PIPE_ID_MASK                                                                            0x0000FF00L
+//GDS_ENHANCE
+#define GDS_ENHANCE__MISC__SHIFT                                                                              0x0
+#define GDS_ENHANCE__AUTO_INC_INDEX__SHIFT                                                                    0x10
+#define GDS_ENHANCE__CGPG_RESTORE__SHIFT                                                                      0x11
+#define GDS_ENHANCE__RD_BUF_TAG_MISS__SHIFT                                                                   0x12
+#define GDS_ENHANCE__GDSA_PC_CGTS_DIS__SHIFT                                                                  0x13
+#define GDS_ENHANCE__GDSO_PC_CGTS_DIS__SHIFT                                                                  0x14
+#define GDS_ENHANCE__WD_GDS_CSB_OVERRIDE__SHIFT                                                               0x15
+#define GDS_ENHANCE__GDS_CLK_ENHANCE_DIS__SHIFT                                                               0x16
+#define GDS_ENHANCE__DS_MEM_CLK_GATE_DIS__SHIFT                                                               0x17
+#define GDS_ENHANCE__UNUSED__SHIFT                                                                            0x18
+#define GDS_ENHANCE__MISC_MASK                                                                                0x0000FFFFL
+#define GDS_ENHANCE__AUTO_INC_INDEX_MASK                                                                      0x00010000L
+#define GDS_ENHANCE__CGPG_RESTORE_MASK                                                                        0x00020000L
+#define GDS_ENHANCE__RD_BUF_TAG_MISS_MASK                                                                     0x00040000L
+#define GDS_ENHANCE__GDSA_PC_CGTS_DIS_MASK                                                                    0x00080000L
+#define GDS_ENHANCE__GDSO_PC_CGTS_DIS_MASK                                                                    0x00100000L
+#define GDS_ENHANCE__WD_GDS_CSB_OVERRIDE_MASK                                                                 0x00200000L
+#define GDS_ENHANCE__GDS_CLK_ENHANCE_DIS_MASK                                                                 0x00400000L
+#define GDS_ENHANCE__DS_MEM_CLK_GATE_DIS_MASK                                                                 0x00800000L
+#define GDS_ENHANCE__UNUSED_MASK                                                                              0xFF000000L
+//GDS_OA_CGPG_RESTORE
+#define GDS_OA_CGPG_RESTORE__VMID__SHIFT                                                                      0x0
+#define GDS_OA_CGPG_RESTORE__MEID__SHIFT                                                                      0x8
+#define GDS_OA_CGPG_RESTORE__PIPEID__SHIFT                                                                    0xc
+#define GDS_OA_CGPG_RESTORE__QUEUEID__SHIFT                                                                   0x10
+#define GDS_OA_CGPG_RESTORE__UNUSED__SHIFT                                                                    0x14
+#define GDS_OA_CGPG_RESTORE__VMID_MASK                                                                        0x000000FFL
+#define GDS_OA_CGPG_RESTORE__MEID_MASK                                                                        0x00000F00L
+#define GDS_OA_CGPG_RESTORE__PIPEID_MASK                                                                      0x0000F000L
+#define GDS_OA_CGPG_RESTORE__QUEUEID_MASK                                                                     0x000F0000L
+#define GDS_OA_CGPG_RESTORE__UNUSED_MASK                                                                      0xFFF00000L
+//GDS_CS_CTXSW_STATUS
+#define GDS_CS_CTXSW_STATUS__R__SHIFT                                                                         0x0
+#define GDS_CS_CTXSW_STATUS__W__SHIFT                                                                         0x1
+#define GDS_CS_CTXSW_STATUS__UNUSED__SHIFT                                                                    0x2
+#define GDS_CS_CTXSW_STATUS__R_MASK                                                                           0x00000001L
+#define GDS_CS_CTXSW_STATUS__W_MASK                                                                           0x00000002L
+#define GDS_CS_CTXSW_STATUS__UNUSED_MASK                                                                      0xFFFFFFFCL
+//GDS_CS_CTXSW_CNT0
+#define GDS_CS_CTXSW_CNT0__UPDN__SHIFT                                                                        0x0
+#define GDS_CS_CTXSW_CNT0__PTR__SHIFT                                                                         0x10
+#define GDS_CS_CTXSW_CNT0__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_CS_CTXSW_CNT0__PTR_MASK                                                                           0xFFFF0000L
+//GDS_CS_CTXSW_CNT1
+#define GDS_CS_CTXSW_CNT1__UPDN__SHIFT                                                                        0x0
+#define GDS_CS_CTXSW_CNT1__PTR__SHIFT                                                                         0x10
+#define GDS_CS_CTXSW_CNT1__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_CS_CTXSW_CNT1__PTR_MASK                                                                           0xFFFF0000L
+//GDS_CS_CTXSW_CNT2
+#define GDS_CS_CTXSW_CNT2__UPDN__SHIFT                                                                        0x0
+#define GDS_CS_CTXSW_CNT2__PTR__SHIFT                                                                         0x10
+#define GDS_CS_CTXSW_CNT2__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_CS_CTXSW_CNT2__PTR_MASK                                                                           0xFFFF0000L
+//GDS_CS_CTXSW_CNT3
+#define GDS_CS_CTXSW_CNT3__UPDN__SHIFT                                                                        0x0
+#define GDS_CS_CTXSW_CNT3__PTR__SHIFT                                                                         0x10
+#define GDS_CS_CTXSW_CNT3__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_CS_CTXSW_CNT3__PTR_MASK                                                                           0xFFFF0000L
+//GDS_GFX_CTXSW_STATUS
+#define GDS_GFX_CTXSW_STATUS__R__SHIFT                                                                        0x0
+#define GDS_GFX_CTXSW_STATUS__W__SHIFT                                                                        0x1
+#define GDS_GFX_CTXSW_STATUS__UNUSED__SHIFT                                                                   0x2
+#define GDS_GFX_CTXSW_STATUS__R_MASK                                                                          0x00000001L
+#define GDS_GFX_CTXSW_STATUS__W_MASK                                                                          0x00000002L
+#define GDS_GFX_CTXSW_STATUS__UNUSED_MASK                                                                     0xFFFFFFFCL
+//GDS_VS_CTXSW_CNT0
+#define GDS_VS_CTXSW_CNT0__UPDN__SHIFT                                                                        0x0
+#define GDS_VS_CTXSW_CNT0__PTR__SHIFT                                                                         0x10
+#define GDS_VS_CTXSW_CNT0__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_VS_CTXSW_CNT0__PTR_MASK                                                                           0xFFFF0000L
+//GDS_VS_CTXSW_CNT1
+#define GDS_VS_CTXSW_CNT1__UPDN__SHIFT                                                                        0x0
+#define GDS_VS_CTXSW_CNT1__PTR__SHIFT                                                                         0x10
+#define GDS_VS_CTXSW_CNT1__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_VS_CTXSW_CNT1__PTR_MASK                                                                           0xFFFF0000L
+//GDS_VS_CTXSW_CNT2
+#define GDS_VS_CTXSW_CNT2__UPDN__SHIFT                                                                        0x0
+#define GDS_VS_CTXSW_CNT2__PTR__SHIFT                                                                         0x10
+#define GDS_VS_CTXSW_CNT2__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_VS_CTXSW_CNT2__PTR_MASK                                                                           0xFFFF0000L
+//GDS_VS_CTXSW_CNT3
+#define GDS_VS_CTXSW_CNT3__UPDN__SHIFT                                                                        0x0
+#define GDS_VS_CTXSW_CNT3__PTR__SHIFT                                                                         0x10
+#define GDS_VS_CTXSW_CNT3__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_VS_CTXSW_CNT3__PTR_MASK                                                                           0xFFFF0000L
+//GDS_PS0_CTXSW_CNT0
+#define GDS_PS0_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
+#define GDS_PS0_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
+#define GDS_PS0_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS0_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS0_CTXSW_CNT1
+#define GDS_PS0_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
+#define GDS_PS0_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
+#define GDS_PS0_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS0_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS0_CTXSW_CNT2
+#define GDS_PS0_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
+#define GDS_PS0_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
+#define GDS_PS0_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS0_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS0_CTXSW_CNT3
+#define GDS_PS0_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
+#define GDS_PS0_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
+#define GDS_PS0_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS0_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS1_CTXSW_CNT0
+#define GDS_PS1_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
+#define GDS_PS1_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
+#define GDS_PS1_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS1_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS1_CTXSW_CNT1
+#define GDS_PS1_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
+#define GDS_PS1_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
+#define GDS_PS1_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS1_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS1_CTXSW_CNT2
+#define GDS_PS1_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
+#define GDS_PS1_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
+#define GDS_PS1_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS1_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS1_CTXSW_CNT3
+#define GDS_PS1_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
+#define GDS_PS1_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
+#define GDS_PS1_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS1_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS2_CTXSW_CNT0
+#define GDS_PS2_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
+#define GDS_PS2_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
+#define GDS_PS2_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS2_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS2_CTXSW_CNT1
+#define GDS_PS2_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
+#define GDS_PS2_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
+#define GDS_PS2_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS2_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS2_CTXSW_CNT2
+#define GDS_PS2_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
+#define GDS_PS2_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
+#define GDS_PS2_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS2_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS2_CTXSW_CNT3
+#define GDS_PS2_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
+#define GDS_PS2_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
+#define GDS_PS2_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS2_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS3_CTXSW_CNT0
+#define GDS_PS3_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
+#define GDS_PS3_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
+#define GDS_PS3_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS3_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS3_CTXSW_CNT1
+#define GDS_PS3_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
+#define GDS_PS3_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
+#define GDS_PS3_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS3_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS3_CTXSW_CNT2
+#define GDS_PS3_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
+#define GDS_PS3_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
+#define GDS_PS3_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS3_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS3_CTXSW_CNT3
+#define GDS_PS3_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
+#define GDS_PS3_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
+#define GDS_PS3_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS3_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS4_CTXSW_CNT0
+#define GDS_PS4_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
+#define GDS_PS4_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
+#define GDS_PS4_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS4_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS4_CTXSW_CNT1
+#define GDS_PS4_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
+#define GDS_PS4_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
+#define GDS_PS4_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS4_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS4_CTXSW_CNT2
+#define GDS_PS4_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
+#define GDS_PS4_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
+#define GDS_PS4_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS4_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS4_CTXSW_CNT3
+#define GDS_PS4_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
+#define GDS_PS4_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
+#define GDS_PS4_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS4_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS5_CTXSW_CNT0
+#define GDS_PS5_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
+#define GDS_PS5_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
+#define GDS_PS5_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS5_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS5_CTXSW_CNT1
+#define GDS_PS5_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
+#define GDS_PS5_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
+#define GDS_PS5_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS5_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS5_CTXSW_CNT2
+#define GDS_PS5_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
+#define GDS_PS5_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
+#define GDS_PS5_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS5_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS5_CTXSW_CNT3
+#define GDS_PS5_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
+#define GDS_PS5_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
+#define GDS_PS5_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS5_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS6_CTXSW_CNT0
+#define GDS_PS6_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
+#define GDS_PS6_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
+#define GDS_PS6_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS6_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS6_CTXSW_CNT1
+#define GDS_PS6_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
+#define GDS_PS6_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
+#define GDS_PS6_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS6_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS6_CTXSW_CNT2
+#define GDS_PS6_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
+#define GDS_PS6_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
+#define GDS_PS6_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS6_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS6_CTXSW_CNT3
+#define GDS_PS6_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
+#define GDS_PS6_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
+#define GDS_PS6_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS6_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS7_CTXSW_CNT0
+#define GDS_PS7_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
+#define GDS_PS7_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
+#define GDS_PS7_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS7_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS7_CTXSW_CNT1
+#define GDS_PS7_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
+#define GDS_PS7_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
+#define GDS_PS7_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS7_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS7_CTXSW_CNT2
+#define GDS_PS7_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
+#define GDS_PS7_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
+#define GDS_PS7_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS7_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
+//GDS_PS7_CTXSW_CNT3
+#define GDS_PS7_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
+#define GDS_PS7_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
+#define GDS_PS7_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
+#define GDS_PS7_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
+//GDS_GS_CTXSW_CNT0
+#define GDS_GS_CTXSW_CNT0__UPDN__SHIFT                                                                        0x0
+#define GDS_GS_CTXSW_CNT0__PTR__SHIFT                                                                         0x10
+#define GDS_GS_CTXSW_CNT0__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_GS_CTXSW_CNT0__PTR_MASK                                                                           0xFFFF0000L
+//GDS_GS_CTXSW_CNT1
+#define GDS_GS_CTXSW_CNT1__UPDN__SHIFT                                                                        0x0
+#define GDS_GS_CTXSW_CNT1__PTR__SHIFT                                                                         0x10
+#define GDS_GS_CTXSW_CNT1__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_GS_CTXSW_CNT1__PTR_MASK                                                                           0xFFFF0000L
+//GDS_GS_CTXSW_CNT2
+#define GDS_GS_CTXSW_CNT2__UPDN__SHIFT                                                                        0x0
+#define GDS_GS_CTXSW_CNT2__PTR__SHIFT                                                                         0x10
+#define GDS_GS_CTXSW_CNT2__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_GS_CTXSW_CNT2__PTR_MASK                                                                           0xFFFF0000L
+//GDS_GS_CTXSW_CNT3
+#define GDS_GS_CTXSW_CNT3__UPDN__SHIFT                                                                        0x0
+#define GDS_GS_CTXSW_CNT3__PTR__SHIFT                                                                         0x10
+#define GDS_GS_CTXSW_CNT3__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_GS_CTXSW_CNT3__PTR_MASK                                                                           0xFFFF0000L
+
+
+// addressBlock: xcd0_gc_rasdec
+//RAS_SIGNATURE_CONTROL
+#define RAS_SIGNATURE_CONTROL__ENABLE__SHIFT                                                                  0x0
+#define RAS_SIGNATURE_CONTROL__ENABLE_MASK                                                                    0x00000001L
+//RAS_SIGNATURE_MASK
+#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK__SHIFT                                                             0x0
+#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK_MASK                                                               0xFFFFFFFFL
+//RAS_SX_SIGNATURE0
+#define RAS_SX_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SX_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SX_SIGNATURE1
+#define RAS_SX_SIGNATURE1__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SX_SIGNATURE1__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SX_SIGNATURE2
+#define RAS_SX_SIGNATURE2__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SX_SIGNATURE2__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SX_SIGNATURE3
+#define RAS_SX_SIGNATURE3__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SX_SIGNATURE3__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_DB_SIGNATURE0
+#define RAS_DB_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_DB_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_PA_SIGNATURE0
+#define RAS_PA_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_PA_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_VGT_SIGNATURE0
+#define RAS_VGT_SIGNATURE0__SIGNATURE__SHIFT                                                                  0x0
+#define RAS_VGT_SIGNATURE0__SIGNATURE_MASK                                                                    0xFFFFFFFFL
+//RAS_SQ_SIGNATURE0
+#define RAS_SQ_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SQ_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE0
+#define RAS_SC_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE1
+#define RAS_SC_SIGNATURE1__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE1__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE2
+#define RAS_SC_SIGNATURE2__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE2__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE3
+#define RAS_SC_SIGNATURE3__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE3__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE4
+#define RAS_SC_SIGNATURE4__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE4__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE5
+#define RAS_SC_SIGNATURE5__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE5__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE6
+#define RAS_SC_SIGNATURE6__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE6__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE7
+#define RAS_SC_SIGNATURE7__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE7__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_IA_SIGNATURE0
+#define RAS_IA_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_IA_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_IA_SIGNATURE1
+#define RAS_IA_SIGNATURE1__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_IA_SIGNATURE1__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SPI_SIGNATURE0
+#define RAS_SPI_SIGNATURE0__SIGNATURE__SHIFT                                                                  0x0
+#define RAS_SPI_SIGNATURE0__SIGNATURE_MASK                                                                    0xFFFFFFFFL
+//RAS_SPI_SIGNATURE1
+#define RAS_SPI_SIGNATURE1__SIGNATURE__SHIFT                                                                  0x0
+#define RAS_SPI_SIGNATURE1__SIGNATURE_MASK                                                                    0xFFFFFFFFL
+//RAS_TA_SIGNATURE0
+#define RAS_TA_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_TA_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_TD_SIGNATURE0
+#define RAS_TD_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_TD_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_CB_SIGNATURE0
+#define RAS_CB_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_CB_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_BCI_SIGNATURE0
+#define RAS_BCI_SIGNATURE0__SIGNATURE__SHIFT                                                                  0x0
+#define RAS_BCI_SIGNATURE0__SIGNATURE_MASK                                                                    0xFFFFFFFFL
+//RAS_BCI_SIGNATURE1
+#define RAS_BCI_SIGNATURE1__SIGNATURE__SHIFT                                                                  0x0
+#define RAS_BCI_SIGNATURE1__SIGNATURE_MASK                                                                    0xFFFFFFFFL
+//RAS_TA_SIGNATURE1
+#define RAS_TA_SIGNATURE1__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_TA_SIGNATURE1__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+
+
+// addressBlock: xcd0_gc_gfxdec0
+//DB_RENDER_CONTROL
+#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE__SHIFT                                                          0x0
+#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE__SHIFT                                                        0x1
+#define DB_RENDER_CONTROL__DEPTH_COPY__SHIFT                                                                  0x2
+#define DB_RENDER_CONTROL__STENCIL_COPY__SHIFT                                                                0x3
+#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE__SHIFT                                                          0x4
+#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE__SHIFT                                                    0x5
+#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE__SHIFT                                                      0x6
+#define DB_RENDER_CONTROL__COPY_CENTROID__SHIFT                                                               0x7
+#define DB_RENDER_CONTROL__COPY_SAMPLE__SHIFT                                                                 0x8
+#define DB_RENDER_CONTROL__DECOMPRESS_ENABLE__SHIFT                                                           0xc
+#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE_MASK                                                            0x00000001L
+#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define DB_RENDER_CONTROL__DEPTH_COPY_MASK                                                                    0x00000004L
+#define DB_RENDER_CONTROL__STENCIL_COPY_MASK                                                                  0x00000008L
+#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE_MASK                                                            0x00000010L
+#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE_MASK                                                      0x00000020L
+#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE_MASK                                                        0x00000040L
+#define DB_RENDER_CONTROL__COPY_CENTROID_MASK                                                                 0x00000080L
+#define DB_RENDER_CONTROL__COPY_SAMPLE_MASK                                                                   0x00000F00L
+#define DB_RENDER_CONTROL__DECOMPRESS_ENABLE_MASK                                                             0x00001000L
+//DB_COUNT_CONTROL
+#define DB_COUNT_CONTROL__ZPASS_INCREMENT_DISABLE__SHIFT                                                      0x0
+#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS__SHIFT                                                         0x1
+#define DB_COUNT_CONTROL__SAMPLE_RATE__SHIFT                                                                  0x4
+#define DB_COUNT_CONTROL__ZPASS_ENABLE__SHIFT                                                                 0x8
+#define DB_COUNT_CONTROL__ZFAIL_ENABLE__SHIFT                                                                 0xc
+#define DB_COUNT_CONTROL__SFAIL_ENABLE__SHIFT                                                                 0x10
+#define DB_COUNT_CONTROL__DBFAIL_ENABLE__SHIFT                                                                0x14
+#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE__SHIFT                                                            0x18
+#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE__SHIFT                                                             0x1c
+#define DB_COUNT_CONTROL__ZPASS_INCREMENT_DISABLE_MASK                                                        0x00000001L
+#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS_MASK                                                           0x00000002L
+#define DB_COUNT_CONTROL__SAMPLE_RATE_MASK                                                                    0x00000070L
+#define DB_COUNT_CONTROL__ZPASS_ENABLE_MASK                                                                   0x00000F00L
+#define DB_COUNT_CONTROL__ZFAIL_ENABLE_MASK                                                                   0x0000F000L
+#define DB_COUNT_CONTROL__SFAIL_ENABLE_MASK                                                                   0x000F0000L
+#define DB_COUNT_CONTROL__DBFAIL_ENABLE_MASK                                                                  0x00F00000L
+#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE_MASK                                                              0x0F000000L
+#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE_MASK                                                               0xF0000000L
+//DB_DEPTH_VIEW
+#define DB_DEPTH_VIEW__SLICE_START__SHIFT                                                                     0x0
+#define DB_DEPTH_VIEW__SLICE_MAX__SHIFT                                                                       0xd
+#define DB_DEPTH_VIEW__Z_READ_ONLY__SHIFT                                                                     0x18
+#define DB_DEPTH_VIEW__STENCIL_READ_ONLY__SHIFT                                                               0x19
+#define DB_DEPTH_VIEW__MIPID__SHIFT                                                                           0x1a
+#define DB_DEPTH_VIEW__SLICE_START_MASK                                                                       0x000007FFL
+#define DB_DEPTH_VIEW__SLICE_MAX_MASK                                                                         0x00FFE000L
+#define DB_DEPTH_VIEW__Z_READ_ONLY_MASK                                                                       0x01000000L
+#define DB_DEPTH_VIEW__STENCIL_READ_ONLY_MASK                                                                 0x02000000L
+#define DB_DEPTH_VIEW__MIPID_MASK                                                                             0x3C000000L
+//DB_RENDER_OVERRIDE
+#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE__SHIFT                                                           0x0
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0__SHIFT                                                          0x2
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1__SHIFT                                                          0x4
+#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER__SHIFT                                                       0x6
+#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE__SHIFT                                                             0x7
+#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE__SHIFT                                                       0x8
+#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE__SHIFT                                                          0x9
+#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL__SHIFT                                                           0xa
+#define DB_RENDER_OVERRIDE__FORCE_Z_READ__SHIFT                                                               0xb
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ__SHIFT                                                         0xc
+#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE__SHIFT                                                         0xd
+#define DB_RENDER_OVERRIDE__FORCE_QC_SMASK_CONFLICT__SHIFT                                                    0xf
+#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP__SHIFT                                                     0x10
+#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE__SHIFT                                                           0x11
+#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED__SHIFT                                                      0x12
+#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM__SHIFT                                                         0x13
+#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT__SHIFT                                                           0x15
+#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES__SHIFT                                                    0x1a
+#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY__SHIFT                                                              0x1b
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY__SHIFT                                                        0x1c
+#define DB_RENDER_OVERRIDE__FORCE_Z_VALID__SHIFT                                                              0x1d
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID__SHIFT                                                        0x1e
+#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION__SHIFT                                                       0x1f
+#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE_MASK                                                             0x00000003L
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0_MASK                                                            0x0000000CL
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1_MASK                                                            0x00000030L
+#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER_MASK                                                         0x00000040L
+#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE_MASK                                                               0x00000080L
+#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE_MASK                                                         0x00000100L
+#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE_MASK                                                            0x00000200L
+#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL_MASK                                                             0x00000400L
+#define DB_RENDER_OVERRIDE__FORCE_Z_READ_MASK                                                                 0x00000800L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ_MASK                                                           0x00001000L
+#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE_MASK                                                           0x00006000L
+#define DB_RENDER_OVERRIDE__FORCE_QC_SMASK_CONFLICT_MASK                                                      0x00008000L
+#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP_MASK                                                       0x00010000L
+#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE_MASK                                                             0x00020000L
+#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED_MASK                                                        0x00040000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM_MASK                                                           0x00180000L
+#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT_MASK                                                             0x03E00000L
+#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES_MASK                                                      0x04000000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY_MASK                                                                0x08000000L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY_MASK                                                          0x10000000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_VALID_MASK                                                                0x20000000L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID_MASK                                                          0x40000000L
+#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION_MASK                                                         0x80000000L
+//DB_RENDER_OVERRIDE2
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL__SHIFT                                              0x0
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN__SHIFT                                            0x2
+#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION__SHIFT                                       0x5
+#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION__SHIFT                                        0x6
+#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION__SHIFT                                               0x7
+#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH__SHIFT                                                     0x8
+#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP__SHIFT                                                         0x9
+#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE__SHIFT                                           0xa
+#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE__SHIFT                                                 0xb
+#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC__SHIFT                                                                 0xc
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF__SHIFT                                                              0xf
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF__SHIFT                                                              0x12
+#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE__SHIFT                                                           0x15
+#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS__SHIFT                                                         0x16
+#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS__SHIFT                                                         0x17
+#define DB_RENDER_OVERRIDE2__ALLOW_PARTIAL_RES_HIER_KILL__SHIFT                                               0x19
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL_MASK                                                0x00000003L
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN_MASK                                              0x0000001CL
+#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION_MASK                                         0x00000020L
+#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION_MASK                                          0x00000040L
+#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION_MASK                                                 0x00000080L
+#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH_MASK                                                       0x00000100L
+#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP_MASK                                                           0x00000200L
+#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE_MASK                                             0x00000400L
+#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE_MASK                                                   0x00000800L
+#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC_MASK                                                                   0x00007000L
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF_MASK                                                                0x00038000L
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF_MASK                                                                0x001C0000L
+#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE_MASK                                                             0x00200000L
+#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS_MASK                                                           0x00400000L
+#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS_MASK                                                           0x00800000L
+#define DB_RENDER_OVERRIDE2__ALLOW_PARTIAL_RES_HIER_KILL_MASK                                                 0x02000000L
+//DB_HTILE_DATA_BASE
+#define DB_HTILE_DATA_BASE__BASE_256B__SHIFT                                                                  0x0
+#define DB_HTILE_DATA_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//DB_HTILE_DATA_BASE_HI
+#define DB_HTILE_DATA_BASE_HI__BASE_HI__SHIFT                                                                 0x0
+#define DB_HTILE_DATA_BASE_HI__BASE_HI_MASK                                                                   0x000000FFL
+//DB_DEPTH_SIZE
+#define DB_DEPTH_SIZE__X_MAX__SHIFT                                                                           0x0
+#define DB_DEPTH_SIZE__Y_MAX__SHIFT                                                                           0x10
+#define DB_DEPTH_SIZE__X_MAX_MASK                                                                             0x00003FFFL
+#define DB_DEPTH_SIZE__Y_MAX_MASK                                                                             0x3FFF0000L
+//DB_DEPTH_BOUNDS_MIN
+#define DB_DEPTH_BOUNDS_MIN__MIN__SHIFT                                                                       0x0
+#define DB_DEPTH_BOUNDS_MIN__MIN_MASK                                                                         0xFFFFFFFFL
+//DB_DEPTH_BOUNDS_MAX
+#define DB_DEPTH_BOUNDS_MAX__MAX__SHIFT                                                                       0x0
+#define DB_DEPTH_BOUNDS_MAX__MAX_MASK                                                                         0xFFFFFFFFL
+//DB_STENCIL_CLEAR
+#define DB_STENCIL_CLEAR__CLEAR__SHIFT                                                                        0x0
+#define DB_STENCIL_CLEAR__CLEAR_MASK                                                                          0x000000FFL
+//DB_DEPTH_CLEAR
+#define DB_DEPTH_CLEAR__DEPTH_CLEAR__SHIFT                                                                    0x0
+#define DB_DEPTH_CLEAR__DEPTH_CLEAR_MASK                                                                      0xFFFFFFFFL
+//PA_SC_SCREEN_SCISSOR_TL
+#define PA_SC_SCREEN_SCISSOR_TL__TL_X__SHIFT                                                                  0x0
+#define PA_SC_SCREEN_SCISSOR_TL__TL_Y__SHIFT                                                                  0x10
+#define PA_SC_SCREEN_SCISSOR_TL__TL_X_MASK                                                                    0x0000FFFFL
+#define PA_SC_SCREEN_SCISSOR_TL__TL_Y_MASK                                                                    0xFFFF0000L
+//PA_SC_SCREEN_SCISSOR_BR
+#define PA_SC_SCREEN_SCISSOR_BR__BR_X__SHIFT                                                                  0x0
+#define PA_SC_SCREEN_SCISSOR_BR__BR_Y__SHIFT                                                                  0x10
+#define PA_SC_SCREEN_SCISSOR_BR__BR_X_MASK                                                                    0x0000FFFFL
+#define PA_SC_SCREEN_SCISSOR_BR__BR_Y_MASK                                                                    0xFFFF0000L
+//DB_Z_INFO
+#define DB_Z_INFO__FORMAT__SHIFT                                                                              0x0
+#define DB_Z_INFO__NUM_SAMPLES__SHIFT                                                                         0x2
+#define DB_Z_INFO__SW_MODE__SHIFT                                                                             0x4
+#define DB_Z_INFO__PARTIALLY_RESIDENT__SHIFT                                                                  0xc
+#define DB_Z_INFO__FAULT_BEHAVIOR__SHIFT                                                                      0xd
+#define DB_Z_INFO__ITERATE_FLUSH__SHIFT                                                                       0xf
+#define DB_Z_INFO__MAXMIP__SHIFT                                                                              0x10
+#define DB_Z_INFO__DECOMPRESS_ON_N_ZPLANES__SHIFT                                                             0x17
+#define DB_Z_INFO__ALLOW_EXPCLEAR__SHIFT                                                                      0x1b
+#define DB_Z_INFO__READ_SIZE__SHIFT                                                                           0x1c
+#define DB_Z_INFO__TILE_SURFACE_ENABLE__SHIFT                                                                 0x1d
+#define DB_Z_INFO__CLEAR_DISALLOWED__SHIFT                                                                    0x1e
+#define DB_Z_INFO__ZRANGE_PRECISION__SHIFT                                                                    0x1f
+#define DB_Z_INFO__FORMAT_MASK                                                                                0x00000003L
+#define DB_Z_INFO__NUM_SAMPLES_MASK                                                                           0x0000000CL
+#define DB_Z_INFO__SW_MODE_MASK                                                                               0x000001F0L
+#define DB_Z_INFO__PARTIALLY_RESIDENT_MASK                                                                    0x00001000L
+#define DB_Z_INFO__FAULT_BEHAVIOR_MASK                                                                        0x00006000L
+#define DB_Z_INFO__ITERATE_FLUSH_MASK                                                                         0x00008000L
+#define DB_Z_INFO__MAXMIP_MASK                                                                                0x000F0000L
+#define DB_Z_INFO__DECOMPRESS_ON_N_ZPLANES_MASK                                                               0x07800000L
+#define DB_Z_INFO__ALLOW_EXPCLEAR_MASK                                                                        0x08000000L
+#define DB_Z_INFO__READ_SIZE_MASK                                                                             0x10000000L
+#define DB_Z_INFO__TILE_SURFACE_ENABLE_MASK                                                                   0x20000000L
+#define DB_Z_INFO__CLEAR_DISALLOWED_MASK                                                                      0x40000000L
+#define DB_Z_INFO__ZRANGE_PRECISION_MASK                                                                      0x80000000L
+//DB_STENCIL_INFO
+#define DB_STENCIL_INFO__FORMAT__SHIFT                                                                        0x0
+#define DB_STENCIL_INFO__SW_MODE__SHIFT                                                                       0x4
+#define DB_STENCIL_INFO__PARTIALLY_RESIDENT__SHIFT                                                            0xc
+#define DB_STENCIL_INFO__FAULT_BEHAVIOR__SHIFT                                                                0xd
+#define DB_STENCIL_INFO__ITERATE_FLUSH__SHIFT                                                                 0xf
+#define DB_STENCIL_INFO__ALLOW_EXPCLEAR__SHIFT                                                                0x1b
+#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE__SHIFT                                                          0x1d
+#define DB_STENCIL_INFO__CLEAR_DISALLOWED__SHIFT                                                              0x1e
+#define DB_STENCIL_INFO__FORMAT_MASK                                                                          0x00000001L
+#define DB_STENCIL_INFO__SW_MODE_MASK                                                                         0x000001F0L
+#define DB_STENCIL_INFO__PARTIALLY_RESIDENT_MASK                                                              0x00001000L
+#define DB_STENCIL_INFO__FAULT_BEHAVIOR_MASK                                                                  0x00006000L
+#define DB_STENCIL_INFO__ITERATE_FLUSH_MASK                                                                   0x00008000L
+#define DB_STENCIL_INFO__ALLOW_EXPCLEAR_MASK                                                                  0x08000000L
+#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE_MASK                                                            0x20000000L
+#define DB_STENCIL_INFO__CLEAR_DISALLOWED_MASK                                                                0x40000000L
+//DB_Z_READ_BASE
+#define DB_Z_READ_BASE__BASE_256B__SHIFT                                                                      0x0
+#define DB_Z_READ_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//DB_Z_READ_BASE_HI
+#define DB_Z_READ_BASE_HI__BASE_HI__SHIFT                                                                     0x0
+#define DB_Z_READ_BASE_HI__BASE_HI_MASK                                                                       0x000000FFL
+//DB_STENCIL_READ_BASE
+#define DB_STENCIL_READ_BASE__BASE_256B__SHIFT                                                                0x0
+#define DB_STENCIL_READ_BASE__BASE_256B_MASK                                                                  0xFFFFFFFFL
+//DB_STENCIL_READ_BASE_HI
+#define DB_STENCIL_READ_BASE_HI__BASE_HI__SHIFT                                                               0x0
+#define DB_STENCIL_READ_BASE_HI__BASE_HI_MASK                                                                 0x000000FFL
+//DB_Z_WRITE_BASE
+#define DB_Z_WRITE_BASE__BASE_256B__SHIFT                                                                     0x0
+#define DB_Z_WRITE_BASE__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//DB_Z_WRITE_BASE_HI
+#define DB_Z_WRITE_BASE_HI__BASE_HI__SHIFT                                                                    0x0
+#define DB_Z_WRITE_BASE_HI__BASE_HI_MASK                                                                      0x000000FFL
+//DB_STENCIL_WRITE_BASE
+#define DB_STENCIL_WRITE_BASE__BASE_256B__SHIFT                                                               0x0
+#define DB_STENCIL_WRITE_BASE__BASE_256B_MASK                                                                 0xFFFFFFFFL
+//DB_STENCIL_WRITE_BASE_HI
+#define DB_STENCIL_WRITE_BASE_HI__BASE_HI__SHIFT                                                              0x0
+#define DB_STENCIL_WRITE_BASE_HI__BASE_HI_MASK                                                                0x000000FFL
+//DB_DFSM_CONTROL
+#define DB_DFSM_CONTROL__PUNCHOUT_MODE__SHIFT                                                                 0x0
+#define DB_DFSM_CONTROL__POPS_DRAIN_PS_ON_OVERLAP__SHIFT                                                      0x2
+#define DB_DFSM_CONTROL__DISALLOW_OVERFLOW__SHIFT                                                             0x3
+#define DB_DFSM_CONTROL__PUNCHOUT_MODE_MASK                                                                   0x00000003L
+#define DB_DFSM_CONTROL__POPS_DRAIN_PS_ON_OVERLAP_MASK                                                        0x00000004L
+#define DB_DFSM_CONTROL__DISALLOW_OVERFLOW_MASK                                                               0x00000008L
+//DB_Z_INFO2
+#define DB_Z_INFO2__EPITCH__SHIFT                                                                             0x0
+#define DB_Z_INFO2__EPITCH_MASK                                                                               0x0000FFFFL
+//DB_STENCIL_INFO2
+#define DB_STENCIL_INFO2__EPITCH__SHIFT                                                                       0x0
+#define DB_STENCIL_INFO2__EPITCH_MASK                                                                         0x0000FFFFL
+//COHER_DEST_BASE_HI_0
+#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_HI_1
+#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_HI_2
+#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_HI_3
+#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_2
+#define COHER_DEST_BASE_2__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_2__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//COHER_DEST_BASE_3
+#define COHER_DEST_BASE_3__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_3__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//PA_SC_WINDOW_OFFSET
+#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET__SHIFT                                                           0x0
+#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET__SHIFT                                                           0x10
+#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET_MASK                                                             0x0000FFFFL
+#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET_MASK                                                             0xFFFF0000L
+//PA_SC_WINDOW_SCISSOR_TL
+#define PA_SC_WINDOW_SCISSOR_TL__TL_X__SHIFT                                                                  0x0
+#define PA_SC_WINDOW_SCISSOR_TL__TL_Y__SHIFT                                                                  0x10
+#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                 0x1f
+#define PA_SC_WINDOW_SCISSOR_TL__TL_X_MASK                                                                    0x00007FFFL
+#define PA_SC_WINDOW_SCISSOR_TL__TL_Y_MASK                                                                    0x7FFF0000L
+#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK                                                   0x80000000L
+//PA_SC_WINDOW_SCISSOR_BR
+#define PA_SC_WINDOW_SCISSOR_BR__BR_X__SHIFT                                                                  0x0
+#define PA_SC_WINDOW_SCISSOR_BR__BR_Y__SHIFT                                                                  0x10
+#define PA_SC_WINDOW_SCISSOR_BR__BR_X_MASK                                                                    0x00007FFFL
+#define PA_SC_WINDOW_SCISSOR_BR__BR_Y_MASK                                                                    0x7FFF0000L
+//PA_SC_CLIPRECT_RULE
+#define PA_SC_CLIPRECT_RULE__CLIP_RULE__SHIFT                                                                 0x0
+#define PA_SC_CLIPRECT_RULE__CLIP_RULE_MASK                                                                   0x0000FFFFL
+//PA_SC_CLIPRECT_0_TL
+#define PA_SC_CLIPRECT_0_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_0_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_0_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_0_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_0_BR
+#define PA_SC_CLIPRECT_0_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_0_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_0_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_0_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_1_TL
+#define PA_SC_CLIPRECT_1_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_1_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_1_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_1_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_1_BR
+#define PA_SC_CLIPRECT_1_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_1_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_1_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_1_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_2_TL
+#define PA_SC_CLIPRECT_2_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_2_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_2_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_2_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_2_BR
+#define PA_SC_CLIPRECT_2_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_2_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_2_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_2_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_3_TL
+#define PA_SC_CLIPRECT_3_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_3_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_3_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_3_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_3_BR
+#define PA_SC_CLIPRECT_3_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_3_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_3_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_3_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_EDGERULE
+#define PA_SC_EDGERULE__ER_TRI__SHIFT                                                                         0x0
+#define PA_SC_EDGERULE__ER_POINT__SHIFT                                                                       0x4
+#define PA_SC_EDGERULE__ER_RECT__SHIFT                                                                        0x8
+#define PA_SC_EDGERULE__ER_LINE_LR__SHIFT                                                                     0xc
+#define PA_SC_EDGERULE__ER_LINE_RL__SHIFT                                                                     0x12
+#define PA_SC_EDGERULE__ER_LINE_TB__SHIFT                                                                     0x18
+#define PA_SC_EDGERULE__ER_LINE_BT__SHIFT                                                                     0x1c
+#define PA_SC_EDGERULE__ER_TRI_MASK                                                                           0x0000000FL
+#define PA_SC_EDGERULE__ER_POINT_MASK                                                                         0x000000F0L
+#define PA_SC_EDGERULE__ER_RECT_MASK                                                                          0x00000F00L
+#define PA_SC_EDGERULE__ER_LINE_LR_MASK                                                                       0x0003F000L
+#define PA_SC_EDGERULE__ER_LINE_RL_MASK                                                                       0x00FC0000L
+#define PA_SC_EDGERULE__ER_LINE_TB_MASK                                                                       0x0F000000L
+#define PA_SC_EDGERULE__ER_LINE_BT_MASK                                                                       0xF0000000L
+//PA_SU_HARDWARE_SCREEN_OFFSET
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X__SHIFT                                               0x0
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y__SHIFT                                               0x10
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X_MASK                                                 0x000001FFL
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y_MASK                                                 0x01FF0000L
+//CB_TARGET_MASK
+#define CB_TARGET_MASK__TARGET0_ENABLE__SHIFT                                                                 0x0
+#define CB_TARGET_MASK__TARGET1_ENABLE__SHIFT                                                                 0x4
+#define CB_TARGET_MASK__TARGET2_ENABLE__SHIFT                                                                 0x8
+#define CB_TARGET_MASK__TARGET3_ENABLE__SHIFT                                                                 0xc
+#define CB_TARGET_MASK__TARGET4_ENABLE__SHIFT                                                                 0x10
+#define CB_TARGET_MASK__TARGET5_ENABLE__SHIFT                                                                 0x14
+#define CB_TARGET_MASK__TARGET6_ENABLE__SHIFT                                                                 0x18
+#define CB_TARGET_MASK__TARGET7_ENABLE__SHIFT                                                                 0x1c
+#define CB_TARGET_MASK__TARGET0_ENABLE_MASK                                                                   0x0000000FL
+#define CB_TARGET_MASK__TARGET1_ENABLE_MASK                                                                   0x000000F0L
+#define CB_TARGET_MASK__TARGET2_ENABLE_MASK                                                                   0x00000F00L
+#define CB_TARGET_MASK__TARGET3_ENABLE_MASK                                                                   0x0000F000L
+#define CB_TARGET_MASK__TARGET4_ENABLE_MASK                                                                   0x000F0000L
+#define CB_TARGET_MASK__TARGET5_ENABLE_MASK                                                                   0x00F00000L
+#define CB_TARGET_MASK__TARGET6_ENABLE_MASK                                                                   0x0F000000L
+#define CB_TARGET_MASK__TARGET7_ENABLE_MASK                                                                   0xF0000000L
+//CB_SHADER_MASK
+#define CB_SHADER_MASK__OUTPUT0_ENABLE__SHIFT                                                                 0x0
+#define CB_SHADER_MASK__OUTPUT1_ENABLE__SHIFT                                                                 0x4
+#define CB_SHADER_MASK__OUTPUT2_ENABLE__SHIFT                                                                 0x8
+#define CB_SHADER_MASK__OUTPUT3_ENABLE__SHIFT                                                                 0xc
+#define CB_SHADER_MASK__OUTPUT4_ENABLE__SHIFT                                                                 0x10
+#define CB_SHADER_MASK__OUTPUT5_ENABLE__SHIFT                                                                 0x14
+#define CB_SHADER_MASK__OUTPUT6_ENABLE__SHIFT                                                                 0x18
+#define CB_SHADER_MASK__OUTPUT7_ENABLE__SHIFT                                                                 0x1c
+#define CB_SHADER_MASK__OUTPUT0_ENABLE_MASK                                                                   0x0000000FL
+#define CB_SHADER_MASK__OUTPUT1_ENABLE_MASK                                                                   0x000000F0L
+#define CB_SHADER_MASK__OUTPUT2_ENABLE_MASK                                                                   0x00000F00L
+#define CB_SHADER_MASK__OUTPUT3_ENABLE_MASK                                                                   0x0000F000L
+#define CB_SHADER_MASK__OUTPUT4_ENABLE_MASK                                                                   0x000F0000L
+#define CB_SHADER_MASK__OUTPUT5_ENABLE_MASK                                                                   0x00F00000L
+#define CB_SHADER_MASK__OUTPUT6_ENABLE_MASK                                                                   0x0F000000L
+#define CB_SHADER_MASK__OUTPUT7_ENABLE_MASK                                                                   0xF0000000L
+//PA_SC_GENERIC_SCISSOR_TL
+#define PA_SC_GENERIC_SCISSOR_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_GENERIC_SCISSOR_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_GENERIC_SCISSOR_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_GENERIC_SCISSOR_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_GENERIC_SCISSOR_BR
+#define PA_SC_GENERIC_SCISSOR_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_GENERIC_SCISSOR_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_GENERIC_SCISSOR_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_GENERIC_SCISSOR_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//COHER_DEST_BASE_0
+#define COHER_DEST_BASE_0__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_0__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//COHER_DEST_BASE_1
+#define COHER_DEST_BASE_1__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_1__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//PA_SC_VPORT_SCISSOR_0_TL
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_0_BR
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_1_TL
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_1_BR
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_2_TL
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_2_BR
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_3_TL
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_3_BR
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_4_TL
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_4_BR
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_5_TL
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_5_BR
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_6_TL
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_6_BR
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_7_TL
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_7_BR
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_8_TL
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_8_BR
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_9_TL
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_9_BR
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_10_TL
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_10_BR
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_11_TL
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_11_BR
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_12_TL
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_12_BR
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_13_TL
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_13_BR
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_14_TL
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_14_BR
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_15_TL
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_15_BR
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_ZMIN_0
+#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_0
+#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_1
+#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_1
+#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_2
+#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_2
+#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_3
+#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_3
+#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_4
+#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_4
+#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_5
+#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_5
+#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_6
+#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_6
+#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_7
+#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_7
+#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_8
+#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_8
+#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_9
+#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_9
+#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_10
+#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_10
+#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_11
+#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_11
+#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_12
+#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_12
+#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_13
+#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_13
+#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_14
+#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_14
+#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_15
+#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_15
+#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_RASTER_CONFIG
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT                                                               0x0
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT                                                               0x2
+#define PA_SC_RASTER_CONFIG__RB_XSEL2__SHIFT                                                                  0x4
+#define PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT                                                                   0x6
+#define PA_SC_RASTER_CONFIG__RB_YSEL__SHIFT                                                                   0x7
+#define PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT                                                                   0x8
+#define PA_SC_RASTER_CONFIG__PKR_XSEL__SHIFT                                                                  0xa
+#define PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT                                                                  0xc
+#define PA_SC_RASTER_CONFIG__PKR_XSEL2__SHIFT                                                                 0xe
+#define PA_SC_RASTER_CONFIG__SC_MAP__SHIFT                                                                    0x10
+#define PA_SC_RASTER_CONFIG__SC_XSEL__SHIFT                                                                   0x12
+#define PA_SC_RASTER_CONFIG__SC_YSEL__SHIFT                                                                   0x14
+#define PA_SC_RASTER_CONFIG__SE_MAP__SHIFT                                                                    0x18
+#define PA_SC_RASTER_CONFIG__SE_XSEL__SHIFT                                                                   0x1a
+#define PA_SC_RASTER_CONFIG__SE_YSEL__SHIFT                                                                   0x1d
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0_MASK                                                                 0x00000003L
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1_MASK                                                                 0x0000000CL
+#define PA_SC_RASTER_CONFIG__RB_XSEL2_MASK                                                                    0x00000030L
+#define PA_SC_RASTER_CONFIG__RB_XSEL_MASK                                                                     0x00000040L
+#define PA_SC_RASTER_CONFIG__RB_YSEL_MASK                                                                     0x00000080L
+#define PA_SC_RASTER_CONFIG__PKR_MAP_MASK                                                                     0x00000300L
+#define PA_SC_RASTER_CONFIG__PKR_XSEL_MASK                                                                    0x00000C00L
+#define PA_SC_RASTER_CONFIG__PKR_YSEL_MASK                                                                    0x00003000L
+#define PA_SC_RASTER_CONFIG__PKR_XSEL2_MASK                                                                   0x0000C000L
+#define PA_SC_RASTER_CONFIG__SC_MAP_MASK                                                                      0x00030000L
+#define PA_SC_RASTER_CONFIG__SC_XSEL_MASK                                                                     0x000C0000L
+#define PA_SC_RASTER_CONFIG__SC_YSEL_MASK                                                                     0x00300000L
+#define PA_SC_RASTER_CONFIG__SE_MAP_MASK                                                                      0x03000000L
+#define PA_SC_RASTER_CONFIG__SE_XSEL_MASK                                                                     0x1C000000L
+#define PA_SC_RASTER_CONFIG__SE_YSEL_MASK                                                                     0xE0000000L
+//PA_SC_RASTER_CONFIG_1
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP__SHIFT                                                             0x0
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL__SHIFT                                                            0x2
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL__SHIFT                                                            0x5
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP_MASK                                                               0x00000003L
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL_MASK                                                              0x0000001CL
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL_MASK                                                              0x000000E0L
+//PA_SC_SCREEN_EXTENT_CONTROL
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE__SHIFT                                                 0x0
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE__SHIFT                                                  0x2
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE_MASK                                                   0x00000003L
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE_MASK                                                    0x0000000CL
+//PA_SC_TILE_STEERING_OVERRIDE
+#define PA_SC_TILE_STEERING_OVERRIDE__ENABLE__SHIFT                                                           0x0
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_SE__SHIFT                                                           0x1
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SE__SHIFT                                                    0x5
+#define PA_SC_TILE_STEERING_OVERRIDE__DISABLE_SRBSL_DB_OPTIMIZED_PACKING__SHIFT                               0x8
+#define PA_SC_TILE_STEERING_OVERRIDE__ENABLE_MASK                                                             0x00000001L
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_SE_MASK                                                             0x00000006L
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SE_MASK                                                      0x00000060L
+#define PA_SC_TILE_STEERING_OVERRIDE__DISABLE_SRBSL_DB_OPTIMIZED_PACKING_MASK                                 0x00000100L
+//CP_PERFMON_CNTX_CNTL
+#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE__SHIFT                                                           0x1f
+#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE_MASK                                                             0x80000000L
+//CP_PIPEID
+#define CP_PIPEID__PIPE_ID__SHIFT                                                                             0x0
+#define CP_PIPEID__PIPE_ID_MASK                                                                               0x00000003L
+//CP_RINGID
+#define CP_RINGID__RINGID__SHIFT                                                                              0x0
+#define CP_RINGID__RINGID_MASK                                                                                0x00000003L
+//CP_VMID
+#define CP_VMID__VMID__SHIFT                                                                                  0x0
+#define CP_VMID__VMID_MASK                                                                                    0x0000000FL
+//PA_SC_RIGHT_VERT_GRID
+#define PA_SC_RIGHT_VERT_GRID__LEFT_QTR__SHIFT                                                                0x0
+#define PA_SC_RIGHT_VERT_GRID__LEFT_HALF__SHIFT                                                               0x8
+#define PA_SC_RIGHT_VERT_GRID__RIGHT_HALF__SHIFT                                                              0x10
+#define PA_SC_RIGHT_VERT_GRID__RIGHT_QTR__SHIFT                                                               0x18
+#define PA_SC_RIGHT_VERT_GRID__LEFT_QTR_MASK                                                                  0x000000FFL
+#define PA_SC_RIGHT_VERT_GRID__LEFT_HALF_MASK                                                                 0x0000FF00L
+#define PA_SC_RIGHT_VERT_GRID__RIGHT_HALF_MASK                                                                0x00FF0000L
+#define PA_SC_RIGHT_VERT_GRID__RIGHT_QTR_MASK                                                                 0xFF000000L
+//PA_SC_LEFT_VERT_GRID
+#define PA_SC_LEFT_VERT_GRID__LEFT_QTR__SHIFT                                                                 0x0
+#define PA_SC_LEFT_VERT_GRID__LEFT_HALF__SHIFT                                                                0x8
+#define PA_SC_LEFT_VERT_GRID__RIGHT_HALF__SHIFT                                                               0x10
+#define PA_SC_LEFT_VERT_GRID__RIGHT_QTR__SHIFT                                                                0x18
+#define PA_SC_LEFT_VERT_GRID__LEFT_QTR_MASK                                                                   0x000000FFL
+#define PA_SC_LEFT_VERT_GRID__LEFT_HALF_MASK                                                                  0x0000FF00L
+#define PA_SC_LEFT_VERT_GRID__RIGHT_HALF_MASK                                                                 0x00FF0000L
+#define PA_SC_LEFT_VERT_GRID__RIGHT_QTR_MASK                                                                  0xFF000000L
+//PA_SC_HORIZ_GRID
+#define PA_SC_HORIZ_GRID__TOP_QTR__SHIFT                                                                      0x0
+#define PA_SC_HORIZ_GRID__TOP_HALF__SHIFT                                                                     0x8
+#define PA_SC_HORIZ_GRID__BOT_HALF__SHIFT                                                                     0x10
+#define PA_SC_HORIZ_GRID__BOT_QTR__SHIFT                                                                      0x18
+#define PA_SC_HORIZ_GRID__TOP_QTR_MASK                                                                        0x000000FFL
+#define PA_SC_HORIZ_GRID__TOP_HALF_MASK                                                                       0x0000FF00L
+#define PA_SC_HORIZ_GRID__BOT_HALF_MASK                                                                       0x00FF0000L
+#define PA_SC_HORIZ_GRID__BOT_QTR_MASK                                                                        0xFF000000L
+//VGT_MULTI_PRIM_IB_RESET_INDX
+#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX__SHIFT                                                       0x0
+#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX_MASK                                                         0xFFFFFFFFL
+//CB_BLEND_RED
+#define CB_BLEND_RED__BLEND_RED__SHIFT                                                                        0x0
+#define CB_BLEND_RED__BLEND_RED_MASK                                                                          0xFFFFFFFFL
+//CB_BLEND_GREEN
+#define CB_BLEND_GREEN__BLEND_GREEN__SHIFT                                                                    0x0
+#define CB_BLEND_GREEN__BLEND_GREEN_MASK                                                                      0xFFFFFFFFL
+//CB_BLEND_BLUE
+#define CB_BLEND_BLUE__BLEND_BLUE__SHIFT                                                                      0x0
+#define CB_BLEND_BLUE__BLEND_BLUE_MASK                                                                        0xFFFFFFFFL
+//CB_BLEND_ALPHA
+#define CB_BLEND_ALPHA__BLEND_ALPHA__SHIFT                                                                    0x0
+#define CB_BLEND_ALPHA__BLEND_ALPHA_MASK                                                                      0xFFFFFFFFL
+//CB_DCC_CONTROL
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                                     0x0
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_MRT_SHARING_DISABLE__SHIFT                                         0x1
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_WATERMARK__SHIFT                                                   0x2
+#define CB_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_AC01__SHIFT                                                   0x8
+#define CB_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_SINGLE__SHIFT                                                 0x9
+#define CB_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                                    0xa
+#define CB_DCC_CONTROL__DISABLE_ELIMFC_SKIP_OF_AC01__SHIFT                                                    0xc
+#define CB_DCC_CONTROL__DISABLE_ELIMFC_SKIP_OF_SINGLE__SHIFT                                                  0xd
+#define CB_DCC_CONTROL__ENABLE_ELIMFC_SKIP_OF_REG__SHIFT                                                      0xe
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                       0x00000001L
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_MRT_SHARING_DISABLE_MASK                                           0x00000002L
+#define CB_DCC_CONTROL__OVERWRITE_COMBINER_WATERMARK_MASK                                                     0x0000007CL
+#define CB_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_AC01_MASK                                                     0x00000100L
+#define CB_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_SINGLE_MASK                                                   0x00000200L
+#define CB_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                                      0x00000400L
+#define CB_DCC_CONTROL__DISABLE_ELIMFC_SKIP_OF_AC01_MASK                                                      0x00001000L
+#define CB_DCC_CONTROL__DISABLE_ELIMFC_SKIP_OF_SINGLE_MASK                                                    0x00002000L
+#define CB_DCC_CONTROL__ENABLE_ELIMFC_SKIP_OF_REG_MASK                                                        0x00004000L
+//DB_STENCIL_CONTROL
+#define DB_STENCIL_CONTROL__STENCILFAIL__SHIFT                                                                0x0
+#define DB_STENCIL_CONTROL__STENCILZPASS__SHIFT                                                               0x4
+#define DB_STENCIL_CONTROL__STENCILZFAIL__SHIFT                                                               0x8
+#define DB_STENCIL_CONTROL__STENCILFAIL_BF__SHIFT                                                             0xc
+#define DB_STENCIL_CONTROL__STENCILZPASS_BF__SHIFT                                                            0x10
+#define DB_STENCIL_CONTROL__STENCILZFAIL_BF__SHIFT                                                            0x14
+#define DB_STENCIL_CONTROL__STENCILFAIL_MASK                                                                  0x0000000FL
+#define DB_STENCIL_CONTROL__STENCILZPASS_MASK                                                                 0x000000F0L
+#define DB_STENCIL_CONTROL__STENCILZFAIL_MASK                                                                 0x00000F00L
+#define DB_STENCIL_CONTROL__STENCILFAIL_BF_MASK                                                               0x0000F000L
+#define DB_STENCIL_CONTROL__STENCILZPASS_BF_MASK                                                              0x000F0000L
+#define DB_STENCIL_CONTROL__STENCILZFAIL_BF_MASK                                                              0x00F00000L
+//DB_STENCILREFMASK
+#define DB_STENCILREFMASK__STENCILTESTVAL__SHIFT                                                              0x0
+#define DB_STENCILREFMASK__STENCILMASK__SHIFT                                                                 0x8
+#define DB_STENCILREFMASK__STENCILWRITEMASK__SHIFT                                                            0x10
+#define DB_STENCILREFMASK__STENCILOPVAL__SHIFT                                                                0x18
+#define DB_STENCILREFMASK__STENCILTESTVAL_MASK                                                                0x000000FFL
+#define DB_STENCILREFMASK__STENCILMASK_MASK                                                                   0x0000FF00L
+#define DB_STENCILREFMASK__STENCILWRITEMASK_MASK                                                              0x00FF0000L
+#define DB_STENCILREFMASK__STENCILOPVAL_MASK                                                                  0xFF000000L
+//DB_STENCILREFMASK_BF
+#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF__SHIFT                                                        0x0
+#define DB_STENCILREFMASK_BF__STENCILMASK_BF__SHIFT                                                           0x8
+#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF__SHIFT                                                      0x10
+#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF__SHIFT                                                          0x18
+#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF_MASK                                                          0x000000FFL
+#define DB_STENCILREFMASK_BF__STENCILMASK_BF_MASK                                                             0x0000FF00L
+#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF_MASK                                                        0x00FF0000L
+#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF_MASK                                                            0xFF000000L
+//PA_CL_VPORT_XSCALE
+#define PA_CL_VPORT_XSCALE__VPORT_XSCALE__SHIFT                                                               0x0
+#define PA_CL_VPORT_XSCALE__VPORT_XSCALE_MASK                                                                 0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET
+#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET__SHIFT                                                             0x0
+#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE
+#define PA_CL_VPORT_YSCALE__VPORT_YSCALE__SHIFT                                                               0x0
+#define PA_CL_VPORT_YSCALE__VPORT_YSCALE_MASK                                                                 0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET
+#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET__SHIFT                                                             0x0
+#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE
+#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE__SHIFT                                                               0x0
+#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE_MASK                                                                 0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET
+#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_1
+#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_1
+#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_1
+#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_1
+#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_1
+#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_1
+#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_2
+#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_2
+#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_2
+#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_2
+#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_2
+#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_2
+#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_3
+#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_3
+#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_3
+#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_3
+#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_3
+#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_3
+#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_4
+#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_4
+#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_4
+#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_4
+#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_4
+#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_4
+#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_5
+#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_5
+#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_5
+#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_5
+#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_5
+#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_5
+#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_6
+#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_6
+#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_6
+#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_6
+#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_6
+#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_6
+#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_7
+#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_7
+#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_7
+#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_7
+#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_7
+#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_7
+#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_8
+#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_8
+#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_8
+#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_8
+#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_8
+#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_8
+#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_9
+#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_9
+#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_9
+#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_9
+#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_9
+#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_9
+#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_10
+#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_10
+#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_10
+#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_10
+#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_10
+#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_10
+#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_11
+#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_11
+#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_11
+#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_11
+#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_11
+#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_11
+#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_12
+#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_12
+#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_12
+#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_12
+#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_12
+#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_12
+#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_13
+#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_13
+#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_13
+#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_13
+#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_13
+#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_13
+#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_14
+#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_14
+#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_14
+#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_14
+#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_14
+#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_14
+#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_15
+#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_15
+#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_15
+#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_15
+#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_15
+#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_15
+#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_UCP_0_X
+#define PA_CL_UCP_0_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_0_Y
+#define PA_CL_UCP_0_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_0_Z
+#define PA_CL_UCP_0_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_0_W
+#define PA_CL_UCP_0_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_X
+#define PA_CL_UCP_1_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_Y
+#define PA_CL_UCP_1_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_Z
+#define PA_CL_UCP_1_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_W
+#define PA_CL_UCP_1_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_X
+#define PA_CL_UCP_2_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_Y
+#define PA_CL_UCP_2_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_Z
+#define PA_CL_UCP_2_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_W
+#define PA_CL_UCP_2_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_X
+#define PA_CL_UCP_3_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_Y
+#define PA_CL_UCP_3_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_Z
+#define PA_CL_UCP_3_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_W
+#define PA_CL_UCP_3_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_X
+#define PA_CL_UCP_4_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_Y
+#define PA_CL_UCP_4_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_Z
+#define PA_CL_UCP_4_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_W
+#define PA_CL_UCP_4_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_X
+#define PA_CL_UCP_5_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_Y
+#define PA_CL_UCP_5_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_Z
+#define PA_CL_UCP_5_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_W
+#define PA_CL_UCP_5_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_PROG_NEAR_CLIP_Z
+#define PA_CL_PROG_NEAR_CLIP_Z__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_PROG_NEAR_CLIP_Z__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//SPI_PS_INPUT_CNTL_0
+#define SPI_PS_INPUT_CNTL_0__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_0__CYL_WRAP__SHIFT                                                                  0xd
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_0__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_0__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_0__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_0__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_0__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_0__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_0__CYL_WRAP_MASK                                                                    0x0001E000L
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_0__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_0__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_0__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_0__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_0__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_1
+#define SPI_PS_INPUT_CNTL_1__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_1__CYL_WRAP__SHIFT                                                                  0xd
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_1__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_1__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_1__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_1__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_1__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_1__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_1__CYL_WRAP_MASK                                                                    0x0001E000L
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_1__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_1__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_1__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_1__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_1__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_2
+#define SPI_PS_INPUT_CNTL_2__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_2__CYL_WRAP__SHIFT                                                                  0xd
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_2__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_2__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_2__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_2__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_2__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_2__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_2__CYL_WRAP_MASK                                                                    0x0001E000L
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_2__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_2__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_2__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_2__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_2__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_3
+#define SPI_PS_INPUT_CNTL_3__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_3__CYL_WRAP__SHIFT                                                                  0xd
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_3__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_3__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_3__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_3__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_3__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_3__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_3__CYL_WRAP_MASK                                                                    0x0001E000L
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_3__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_3__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_3__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_3__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_3__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_4
+#define SPI_PS_INPUT_CNTL_4__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_4__CYL_WRAP__SHIFT                                                                  0xd
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_4__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_4__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_4__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_4__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_4__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_4__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_4__CYL_WRAP_MASK                                                                    0x0001E000L
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_4__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_4__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_4__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_4__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_4__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_5
+#define SPI_PS_INPUT_CNTL_5__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_5__CYL_WRAP__SHIFT                                                                  0xd
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_5__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_5__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_5__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_5__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_5__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_5__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_5__CYL_WRAP_MASK                                                                    0x0001E000L
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_5__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_5__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_5__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_5__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_5__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_6
+#define SPI_PS_INPUT_CNTL_6__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_6__CYL_WRAP__SHIFT                                                                  0xd
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_6__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_6__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_6__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_6__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_6__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_6__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_6__CYL_WRAP_MASK                                                                    0x0001E000L
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_6__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_6__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_6__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_6__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_6__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_7
+#define SPI_PS_INPUT_CNTL_7__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_7__CYL_WRAP__SHIFT                                                                  0xd
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_7__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_7__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_7__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_7__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_7__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_7__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_7__CYL_WRAP_MASK                                                                    0x0001E000L
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_7__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_7__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_7__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_7__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_7__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_8
+#define SPI_PS_INPUT_CNTL_8__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_8__CYL_WRAP__SHIFT                                                                  0xd
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_8__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_8__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_8__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_8__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_8__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_8__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_8__CYL_WRAP_MASK                                                                    0x0001E000L
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_8__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_8__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_8__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_8__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_8__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_9
+#define SPI_PS_INPUT_CNTL_9__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_9__CYL_WRAP__SHIFT                                                                  0xd
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_9__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_9__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_9__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_9__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_9__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_9__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_9__CYL_WRAP_MASK                                                                    0x0001E000L
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_9__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_9__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_9__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_9__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_9__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_10
+#define SPI_PS_INPUT_CNTL_10__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_10__CYL_WRAP__SHIFT                                                                 0xd
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_10__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_10__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_10__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_10__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_10__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_10__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_10__CYL_WRAP_MASK                                                                   0x0001E000L
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_10__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_10__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_10__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_10__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_10__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_11
+#define SPI_PS_INPUT_CNTL_11__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_11__CYL_WRAP__SHIFT                                                                 0xd
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_11__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_11__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_11__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_11__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_11__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_11__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_11__CYL_WRAP_MASK                                                                   0x0001E000L
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_11__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_11__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_11__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_11__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_11__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_12
+#define SPI_PS_INPUT_CNTL_12__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_12__CYL_WRAP__SHIFT                                                                 0xd
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_12__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_12__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_12__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_12__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_12__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_12__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_12__CYL_WRAP_MASK                                                                   0x0001E000L
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_12__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_12__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_12__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_12__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_12__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_13
+#define SPI_PS_INPUT_CNTL_13__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_13__CYL_WRAP__SHIFT                                                                 0xd
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_13__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_13__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_13__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_13__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_13__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_13__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_13__CYL_WRAP_MASK                                                                   0x0001E000L
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_13__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_13__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_13__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_13__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_13__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_14
+#define SPI_PS_INPUT_CNTL_14__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_14__CYL_WRAP__SHIFT                                                                 0xd
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_14__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_14__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_14__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_14__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_14__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_14__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_14__CYL_WRAP_MASK                                                                   0x0001E000L
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_14__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_14__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_14__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_14__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_14__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_15
+#define SPI_PS_INPUT_CNTL_15__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_15__CYL_WRAP__SHIFT                                                                 0xd
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_15__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_15__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_15__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_15__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_15__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_15__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_15__CYL_WRAP_MASK                                                                   0x0001E000L
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_15__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_15__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_15__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_15__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_15__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_16
+#define SPI_PS_INPUT_CNTL_16__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_16__CYL_WRAP__SHIFT                                                                 0xd
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_16__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_16__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_16__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_16__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_16__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_16__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_16__CYL_WRAP_MASK                                                                   0x0001E000L
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_16__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_16__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_16__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_16__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_16__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_17
+#define SPI_PS_INPUT_CNTL_17__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_17__CYL_WRAP__SHIFT                                                                 0xd
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_17__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_17__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_17__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_17__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_17__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_17__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_17__CYL_WRAP_MASK                                                                   0x0001E000L
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_17__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_17__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_17__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_17__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_17__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_18
+#define SPI_PS_INPUT_CNTL_18__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_18__CYL_WRAP__SHIFT                                                                 0xd
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_18__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_18__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_18__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_18__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_18__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_18__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_18__CYL_WRAP_MASK                                                                   0x0001E000L
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_18__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_18__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_18__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_18__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_18__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_19
+#define SPI_PS_INPUT_CNTL_19__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_19__CYL_WRAP__SHIFT                                                                 0xd
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_19__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_19__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_19__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_19__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_19__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_19__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_19__CYL_WRAP_MASK                                                                   0x0001E000L
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_19__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_19__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_19__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_19__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_19__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_20
+#define SPI_PS_INPUT_CNTL_20__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_20__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_20__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_20__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_20__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_20__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_20__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_20__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_20__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_20__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_20__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_20__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_21
+#define SPI_PS_INPUT_CNTL_21__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_21__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_21__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_21__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_21__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_21__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_21__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_21__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_21__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_21__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_21__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_21__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_22
+#define SPI_PS_INPUT_CNTL_22__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_22__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_22__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_22__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_22__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_22__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_22__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_22__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_22__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_22__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_22__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_22__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_23
+#define SPI_PS_INPUT_CNTL_23__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_23__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_23__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_23__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_23__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_23__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_23__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_23__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_23__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_23__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_23__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_23__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_24
+#define SPI_PS_INPUT_CNTL_24__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_24__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_24__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_24__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_24__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_24__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_24__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_24__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_24__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_24__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_24__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_24__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_25
+#define SPI_PS_INPUT_CNTL_25__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_25__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_25__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_25__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_25__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_25__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_25__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_25__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_25__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_25__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_25__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_25__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_26
+#define SPI_PS_INPUT_CNTL_26__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_26__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_26__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_26__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_26__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_26__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_26__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_26__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_26__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_26__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_26__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_26__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_27
+#define SPI_PS_INPUT_CNTL_27__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_27__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_27__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_27__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_27__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_27__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_27__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_27__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_27__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_27__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_27__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_27__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_28
+#define SPI_PS_INPUT_CNTL_28__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_28__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_28__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_28__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_28__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_28__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_28__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_28__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_28__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_28__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_28__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_28__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_29
+#define SPI_PS_INPUT_CNTL_29__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_29__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_29__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_29__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_29__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_29__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_29__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_29__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_29__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_29__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_29__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_29__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_30
+#define SPI_PS_INPUT_CNTL_30__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_30__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_30__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_30__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_30__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_30__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_30__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_30__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_30__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_30__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_30__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_30__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_31
+#define SPI_PS_INPUT_CNTL_31__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_31__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_31__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_31__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_31__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_31__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_31__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_31__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_31__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_31__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_31__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_31__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_VS_OUT_CONFIG
+#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT__SHIFT                                                             0x1
+#define SPI_VS_OUT_CONFIG__VS_HALF_PACK__SHIFT                                                                0x6
+#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT_MASK                                                               0x0000003EL
+#define SPI_VS_OUT_CONFIG__VS_HALF_PACK_MASK                                                                  0x00000040L
+//SPI_PS_INPUT_ENA
+#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA__SHIFT                                                             0x0
+#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA__SHIFT                                                             0x1
+#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA__SHIFT                                                           0x2
+#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA__SHIFT                                                         0x3
+#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA__SHIFT                                                            0x4
+#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA__SHIFT                                                            0x5
+#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA__SHIFT                                                          0x6
+#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA__SHIFT                                                         0x7
+#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA__SHIFT                                                              0x8
+#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA__SHIFT                                                              0x9
+#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA__SHIFT                                                              0xa
+#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA__SHIFT                                                              0xb
+#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA__SHIFT                                                               0xc
+#define SPI_PS_INPUT_ENA__ANCILLARY_ENA__SHIFT                                                                0xd
+#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA__SHIFT                                                          0xe
+#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA__SHIFT                                                             0xf
+#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA_MASK                                                               0x00000001L
+#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA_MASK                                                               0x00000002L
+#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA_MASK                                                             0x00000004L
+#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA_MASK                                                           0x00000008L
+#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA_MASK                                                              0x00000010L
+#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA_MASK                                                              0x00000020L
+#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA_MASK                                                            0x00000040L
+#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA_MASK                                                           0x00000080L
+#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA_MASK                                                                0x00000100L
+#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA_MASK                                                                0x00000200L
+#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA_MASK                                                                0x00000400L
+#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA_MASK                                                                0x00000800L
+#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA_MASK                                                                 0x00001000L
+#define SPI_PS_INPUT_ENA__ANCILLARY_ENA_MASK                                                                  0x00002000L
+#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA_MASK                                                            0x00004000L
+#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA_MASK                                                               0x00008000L
+//SPI_PS_INPUT_ADDR
+#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA__SHIFT                                                            0x0
+#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA__SHIFT                                                            0x1
+#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA__SHIFT                                                          0x2
+#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA__SHIFT                                                        0x3
+#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA__SHIFT                                                           0x4
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA__SHIFT                                                           0x5
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA__SHIFT                                                         0x6
+#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA__SHIFT                                                        0x7
+#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA__SHIFT                                                             0x8
+#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA__SHIFT                                                             0x9
+#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA__SHIFT                                                             0xa
+#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA__SHIFT                                                             0xb
+#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA__SHIFT                                                              0xc
+#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA__SHIFT                                                               0xd
+#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA__SHIFT                                                         0xe
+#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA__SHIFT                                                            0xf
+#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA_MASK                                                              0x00000001L
+#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA_MASK                                                              0x00000002L
+#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA_MASK                                                            0x00000004L
+#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA_MASK                                                          0x00000008L
+#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA_MASK                                                             0x00000010L
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA_MASK                                                             0x00000020L
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA_MASK                                                           0x00000040L
+#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA_MASK                                                          0x00000080L
+#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA_MASK                                                               0x00000100L
+#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA_MASK                                                               0x00000200L
+#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA_MASK                                                               0x00000400L
+#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA_MASK                                                                0x00001000L
+#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA_MASK                                                                 0x00002000L
+#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA_MASK                                                           0x00004000L
+#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA_MASK                                                              0x00008000L
+//SPI_INTERP_CONTROL_0
+#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA__SHIFT                                                           0x0
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA__SHIFT                                                           0x1
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X__SHIFT                                                        0x2
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y__SHIFT                                                        0x5
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z__SHIFT                                                        0x8
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W__SHIFT                                                        0xb
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1__SHIFT                                                         0xe
+#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA_MASK                                                             0x00000001L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA_MASK                                                             0x00000002L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X_MASK                                                          0x0000001CL
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y_MASK                                                          0x000000E0L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z_MASK                                                          0x00000700L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W_MASK                                                          0x00003800L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1_MASK                                                           0x00004000L
+//SPI_PS_IN_CONTROL
+#define SPI_PS_IN_CONTROL__NUM_INTERP__SHIFT                                                                  0x0
+#define SPI_PS_IN_CONTROL__PARAM_GEN__SHIFT                                                                   0x6
+#define SPI_PS_IN_CONTROL__OFFCHIP_PARAM_EN__SHIFT                                                            0x7
+#define SPI_PS_IN_CONTROL__LATE_PC_DEALLOC__SHIFT                                                             0x8
+#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE__SHIFT                                                         0xe
+#define SPI_PS_IN_CONTROL__NUM_INTERP_MASK                                                                    0x0000003FL
+#define SPI_PS_IN_CONTROL__PARAM_GEN_MASK                                                                     0x00000040L
+#define SPI_PS_IN_CONTROL__OFFCHIP_PARAM_EN_MASK                                                              0x00000080L
+#define SPI_PS_IN_CONTROL__LATE_PC_DEALLOC_MASK                                                               0x00000100L
+#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE_MASK                                                           0x00004000L
+//SPI_BARYC_CNTL
+#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL__SHIFT                                                              0x0
+#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL__SHIFT                                                            0x4
+#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL__SHIFT                                                             0x8
+#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL__SHIFT                                                           0xc
+#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION__SHIFT                                                             0x10
+#define SPI_BARYC_CNTL__POS_FLOAT_ULC__SHIFT                                                                  0x14
+#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS__SHIFT                                                            0x18
+#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL_MASK                                                                0x00000001L
+#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL_MASK                                                              0x00000010L
+#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL_MASK                                                               0x00000100L
+#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL_MASK                                                             0x00001000L
+#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION_MASK                                                               0x00030000L
+#define SPI_BARYC_CNTL__POS_FLOAT_ULC_MASK                                                                    0x00100000L
+#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS_MASK                                                              0x01000000L
+//SPI_TMPRING_SIZE
+#define SPI_TMPRING_SIZE__WAVES__SHIFT                                                                        0x0
+#define SPI_TMPRING_SIZE__WAVESIZE__SHIFT                                                                     0xc
+#define SPI_TMPRING_SIZE__WAVES_MASK                                                                          0x00000FFFL
+#define SPI_TMPRING_SIZE__WAVESIZE_MASK                                                                       0x01FFF000L
+//SPI_SHADER_POS_FORMAT
+#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT__SHIFT                                                      0x0
+#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT__SHIFT                                                      0x4
+#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT__SHIFT                                                      0x8
+#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT__SHIFT                                                      0xc
+#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT_MASK                                                        0x0000000FL
+#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT_MASK                                                        0x000000F0L
+#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT_MASK                                                        0x00000F00L
+#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT_MASK                                                        0x0000F000L
+//SPI_SHADER_Z_FORMAT
+#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT__SHIFT                                                           0x0
+#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT_MASK                                                             0x0000000FL
+//SPI_SHADER_COL_FORMAT
+#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT__SHIFT                                                      0x0
+#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT__SHIFT                                                      0x4
+#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT__SHIFT                                                      0x8
+#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT__SHIFT                                                      0xc
+#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT__SHIFT                                                      0x10
+#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT__SHIFT                                                      0x14
+#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT__SHIFT                                                      0x18
+#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT__SHIFT                                                      0x1c
+#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT_MASK                                                        0x0000000FL
+#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT_MASK                                                        0x000000F0L
+#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT_MASK                                                        0x00000F00L
+#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT_MASK                                                        0x0000F000L
+#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT_MASK                                                        0x000F0000L
+#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT_MASK                                                        0x00F00000L
+#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT_MASK                                                        0x0F000000L
+#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT_MASK                                                        0xF0000000L
+//CB_BLEND0_CONTROL
+#define CB_BLEND0_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND0_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND0_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND0_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND0_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND0_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND0_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND0_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND0_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND0_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND1_CONTROL
+#define CB_BLEND1_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND1_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND1_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND1_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND1_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND1_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND1_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND1_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND1_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND1_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND2_CONTROL
+#define CB_BLEND2_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND2_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND2_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND2_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND2_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND2_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND2_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND2_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND2_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND2_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND3_CONTROL
+#define CB_BLEND3_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND3_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND3_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND3_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND3_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND3_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND3_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND3_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND3_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND3_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND4_CONTROL
+#define CB_BLEND4_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND4_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND4_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND4_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND4_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND4_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND4_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND4_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND4_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND4_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND5_CONTROL
+#define CB_BLEND5_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND5_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND5_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND5_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND5_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND5_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND5_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND5_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND5_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND5_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND6_CONTROL
+#define CB_BLEND6_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND6_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND6_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND6_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND6_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND6_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND6_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND6_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND6_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND6_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND7_CONTROL
+#define CB_BLEND7_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND7_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND7_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND7_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND7_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND7_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND7_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND7_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND7_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND7_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_MRT0_EPITCH
+#define CB_MRT0_EPITCH__EPITCH__SHIFT                                                                         0x0
+#define CB_MRT0_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
+//CB_MRT1_EPITCH
+#define CB_MRT1_EPITCH__EPITCH__SHIFT                                                                         0x0
+#define CB_MRT1_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
+//CB_MRT2_EPITCH
+#define CB_MRT2_EPITCH__EPITCH__SHIFT                                                                         0x0
+#define CB_MRT2_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
+//CB_MRT3_EPITCH
+#define CB_MRT3_EPITCH__EPITCH__SHIFT                                                                         0x0
+#define CB_MRT3_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
+//CB_MRT4_EPITCH
+#define CB_MRT4_EPITCH__EPITCH__SHIFT                                                                         0x0
+#define CB_MRT4_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
+//CB_MRT5_EPITCH
+#define CB_MRT5_EPITCH__EPITCH__SHIFT                                                                         0x0
+#define CB_MRT5_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
+//CB_MRT6_EPITCH
+#define CB_MRT6_EPITCH__EPITCH__SHIFT                                                                         0x0
+#define CB_MRT6_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
+//CB_MRT7_EPITCH
+#define CB_MRT7_EPITCH__EPITCH__SHIFT                                                                         0x0
+#define CB_MRT7_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
+//CS_COPY_STATE
+#define CS_COPY_STATE__SRC_STATE_ID__SHIFT                                                                    0x0
+#define CS_COPY_STATE__SRC_STATE_ID_MASK                                                                      0x00000007L
+//GFX_COPY_STATE
+#define GFX_COPY_STATE__SRC_STATE_ID__SHIFT                                                                   0x0
+#define GFX_COPY_STATE__SRC_STATE_ID_MASK                                                                     0x00000007L
+//PA_CL_POINT_X_RAD
+#define PA_CL_POINT_X_RAD__DATA_REGISTER__SHIFT                                                               0x0
+#define PA_CL_POINT_X_RAD__DATA_REGISTER_MASK                                                                 0xFFFFFFFFL
+//PA_CL_POINT_Y_RAD
+#define PA_CL_POINT_Y_RAD__DATA_REGISTER__SHIFT                                                               0x0
+#define PA_CL_POINT_Y_RAD__DATA_REGISTER_MASK                                                                 0xFFFFFFFFL
+//PA_CL_POINT_SIZE
+#define PA_CL_POINT_SIZE__DATA_REGISTER__SHIFT                                                                0x0
+#define PA_CL_POINT_SIZE__DATA_REGISTER_MASK                                                                  0xFFFFFFFFL
+//PA_CL_POINT_CULL_RAD
+#define PA_CL_POINT_CULL_RAD__DATA_REGISTER__SHIFT                                                            0x0
+#define PA_CL_POINT_CULL_RAD__DATA_REGISTER_MASK                                                              0xFFFFFFFFL
+//VGT_DMA_BASE_HI
+#define VGT_DMA_BASE_HI__BASE_ADDR__SHIFT                                                                     0x0
+#define VGT_DMA_BASE_HI__BASE_ADDR_MASK                                                                       0x0000FFFFL
+//VGT_DMA_BASE
+#define VGT_DMA_BASE__BASE_ADDR__SHIFT                                                                        0x0
+#define VGT_DMA_BASE__BASE_ADDR_MASK                                                                          0xFFFFFFFFL
+//VGT_DRAW_INITIATOR
+#define VGT_DRAW_INITIATOR__SOURCE_SELECT__SHIFT                                                              0x0
+#define VGT_DRAW_INITIATOR__MAJOR_MODE__SHIFT                                                                 0x2
+#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX__SHIFT                                                             0x4
+#define VGT_DRAW_INITIATOR__NOT_EOP__SHIFT                                                                    0x5
+#define VGT_DRAW_INITIATOR__USE_OPAQUE__SHIFT                                                                 0x6
+#define VGT_DRAW_INITIATOR__UNROLLED_INST__SHIFT                                                              0x7
+#define VGT_DRAW_INITIATOR__GRBM_SKEW_NO_DEC__SHIFT                                                           0x8
+#define VGT_DRAW_INITIATOR__REG_RT_INDEX__SHIFT                                                               0x1d
+#define VGT_DRAW_INITIATOR__SOURCE_SELECT_MASK                                                                0x00000003L
+#define VGT_DRAW_INITIATOR__MAJOR_MODE_MASK                                                                   0x0000000CL
+#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX_MASK                                                               0x00000010L
+#define VGT_DRAW_INITIATOR__NOT_EOP_MASK                                                                      0x00000020L
+#define VGT_DRAW_INITIATOR__USE_OPAQUE_MASK                                                                   0x00000040L
+#define VGT_DRAW_INITIATOR__UNROLLED_INST_MASK                                                                0x00000080L
+#define VGT_DRAW_INITIATOR__GRBM_SKEW_NO_DEC_MASK                                                             0x00000100L
+#define VGT_DRAW_INITIATOR__REG_RT_INDEX_MASK                                                                 0xE0000000L
+//VGT_IMMED_DATA
+#define VGT_IMMED_DATA__DATA__SHIFT                                                                           0x0
+#define VGT_IMMED_DATA__DATA_MASK                                                                             0xFFFFFFFFL
+//VGT_EVENT_ADDRESS_REG
+#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW__SHIFT                                                             0x0
+#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW_MASK                                                               0x0FFFFFFFL
+//DB_DEPTH_CONTROL
+#define DB_DEPTH_CONTROL__STENCIL_ENABLE__SHIFT                                                               0x0
+#define DB_DEPTH_CONTROL__Z_ENABLE__SHIFT                                                                     0x1
+#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE__SHIFT                                                               0x2
+#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE__SHIFT                                                          0x3
+#define DB_DEPTH_CONTROL__ZFUNC__SHIFT                                                                        0x4
+#define DB_DEPTH_CONTROL__BACKFACE_ENABLE__SHIFT                                                              0x7
+#define DB_DEPTH_CONTROL__STENCILFUNC__SHIFT                                                                  0x8
+#define DB_DEPTH_CONTROL__STENCILFUNC_BF__SHIFT                                                               0x14
+#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL__SHIFT                                            0x1e
+#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS__SHIFT                                           0x1f
+#define DB_DEPTH_CONTROL__STENCIL_ENABLE_MASK                                                                 0x00000001L
+#define DB_DEPTH_CONTROL__Z_ENABLE_MASK                                                                       0x00000002L
+#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE_MASK                                                                 0x00000004L
+#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE_MASK                                                            0x00000008L
+#define DB_DEPTH_CONTROL__ZFUNC_MASK                                                                          0x00000070L
+#define DB_DEPTH_CONTROL__BACKFACE_ENABLE_MASK                                                                0x00000080L
+#define DB_DEPTH_CONTROL__STENCILFUNC_MASK                                                                    0x00000700L
+#define DB_DEPTH_CONTROL__STENCILFUNC_BF_MASK                                                                 0x00700000L
+#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL_MASK                                              0x40000000L
+#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS_MASK                                             0x80000000L
+//DB_EQAA
+#define DB_EQAA__MAX_ANCHOR_SAMPLES__SHIFT                                                                    0x0
+#define DB_EQAA__PS_ITER_SAMPLES__SHIFT                                                                       0x4
+#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES__SHIFT                                                               0x8
+#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES__SHIFT                                                             0xc
+#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS__SHIFT                                                            0x10
+#define DB_EQAA__INCOHERENT_EQAA_READS__SHIFT                                                                 0x11
+#define DB_EQAA__INTERPOLATE_COMP_Z__SHIFT                                                                    0x12
+#define DB_EQAA__INTERPOLATE_SRC_Z__SHIFT                                                                     0x13
+#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS__SHIFT                                                            0x14
+#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE__SHIFT                                                            0x15
+#define DB_EQAA__OVERRASTERIZATION_AMOUNT__SHIFT                                                              0x18
+#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION__SHIFT                                                        0x1b
+#define DB_EQAA__MAX_ANCHOR_SAMPLES_MASK                                                                      0x00000007L
+#define DB_EQAA__PS_ITER_SAMPLES_MASK                                                                         0x00000070L
+#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES_MASK                                                                 0x00000700L
+#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES_MASK                                                               0x00007000L
+#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS_MASK                                                              0x00010000L
+#define DB_EQAA__INCOHERENT_EQAA_READS_MASK                                                                   0x00020000L
+#define DB_EQAA__INTERPOLATE_COMP_Z_MASK                                                                      0x00040000L
+#define DB_EQAA__INTERPOLATE_SRC_Z_MASK                                                                       0x00080000L
+#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS_MASK                                                              0x00100000L
+#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE_MASK                                                              0x00200000L
+#define DB_EQAA__OVERRASTERIZATION_AMOUNT_MASK                                                                0x07000000L
+#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION_MASK                                                          0x08000000L
+//CB_COLOR_CONTROL
+#define CB_COLOR_CONTROL__DISABLE_DUAL_QUAD__SHIFT                                                            0x0
+#define CB_COLOR_CONTROL__DEGAMMA_ENABLE__SHIFT                                                               0x3
+#define CB_COLOR_CONTROL__MODE__SHIFT                                                                         0x4
+#define CB_COLOR_CONTROL__ROP3__SHIFT                                                                         0x10
+#define CB_COLOR_CONTROL__DISABLE_DUAL_QUAD_MASK                                                              0x00000001L
+#define CB_COLOR_CONTROL__DEGAMMA_ENABLE_MASK                                                                 0x00000008L
+#define CB_COLOR_CONTROL__MODE_MASK                                                                           0x00000070L
+#define CB_COLOR_CONTROL__ROP3_MASK                                                                           0x00FF0000L
+//DB_SHADER_CONTROL
+#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE__SHIFT                                                             0x0
+#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE__SHIFT                                              0x1
+#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE__SHIFT                                                0x2
+#define DB_SHADER_CONTROL__Z_ORDER__SHIFT                                                                     0x4
+#define DB_SHADER_CONTROL__KILL_ENABLE__SHIFT                                                                 0x6
+#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE__SHIFT                                                     0x7
+#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE__SHIFT                                                          0x8
+#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL__SHIFT                                                           0x9
+#define DB_SHADER_CONTROL__EXEC_ON_NOOP__SHIFT                                                                0xa
+#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE__SHIFT                                                       0xb
+#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER__SHIFT                                                         0xc
+#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT__SHIFT                                                       0xd
+#define DB_SHADER_CONTROL__DUAL_QUAD_DISABLE__SHIFT                                                           0xf
+#define DB_SHADER_CONTROL__PRIMITIVE_ORDERED_PIXEL_SHADER__SHIFT                                              0x10
+#define DB_SHADER_CONTROL__EXEC_IF_OVERLAPPED__SHIFT                                                          0x11
+#define DB_SHADER_CONTROL__POPS_OVERLAP_NUM_SAMPLES__SHIFT                                                    0x14
+#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE_MASK                                                               0x00000001L
+#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE_MASK                                                0x00000002L
+#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE_MASK                                                  0x00000004L
+#define DB_SHADER_CONTROL__Z_ORDER_MASK                                                                       0x00000030L
+#define DB_SHADER_CONTROL__KILL_ENABLE_MASK                                                                   0x00000040L
+#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE_MASK                                                       0x00000080L
+#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE_MASK                                                            0x00000100L
+#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL_MASK                                                             0x00000200L
+#define DB_SHADER_CONTROL__EXEC_ON_NOOP_MASK                                                                  0x00000400L
+#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE_MASK                                                         0x00000800L
+#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER_MASK                                                           0x00001000L
+#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT_MASK                                                         0x00006000L
+#define DB_SHADER_CONTROL__DUAL_QUAD_DISABLE_MASK                                                             0x00008000L
+#define DB_SHADER_CONTROL__PRIMITIVE_ORDERED_PIXEL_SHADER_MASK                                                0x00010000L
+#define DB_SHADER_CONTROL__EXEC_IF_OVERLAPPED_MASK                                                            0x00020000L
+#define DB_SHADER_CONTROL__POPS_OVERLAP_NUM_SAMPLES_MASK                                                      0x00700000L
+//PA_CL_CLIP_CNTL
+#define PA_CL_CLIP_CNTL__UCP_ENA_0__SHIFT                                                                     0x0
+#define PA_CL_CLIP_CNTL__UCP_ENA_1__SHIFT                                                                     0x1
+#define PA_CL_CLIP_CNTL__UCP_ENA_2__SHIFT                                                                     0x2
+#define PA_CL_CLIP_CNTL__UCP_ENA_3__SHIFT                                                                     0x3
+#define PA_CL_CLIP_CNTL__UCP_ENA_4__SHIFT                                                                     0x4
+#define PA_CL_CLIP_CNTL__UCP_ENA_5__SHIFT                                                                     0x5
+#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG__SHIFT                                                            0xd
+#define PA_CL_CLIP_CNTL__PS_UCP_MODE__SHIFT                                                                   0xe
+#define PA_CL_CLIP_CNTL__CLIP_DISABLE__SHIFT                                                                  0x10
+#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA__SHIFT                                                             0x11
+#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA__SHIFT                                                        0x12
+#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF__SHIFT                                                             0x13
+#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT__SHIFT                                                           0x14
+#define PA_CL_CLIP_CNTL__VTX_KILL_OR__SHIFT                                                                   0x15
+#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL__SHIFT                                                         0x16
+#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA__SHIFT                                                       0x18
+#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE__SHIFT                                                     0x19
+#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE__SHIFT                                                            0x1a
+#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE__SHIFT                                                             0x1b
+#define PA_CL_CLIP_CNTL__ZCLIP_PROG_NEAR_ENA__SHIFT                                                           0x1c
+#define PA_CL_CLIP_CNTL__UCP_ENA_0_MASK                                                                       0x00000001L
+#define PA_CL_CLIP_CNTL__UCP_ENA_1_MASK                                                                       0x00000002L
+#define PA_CL_CLIP_CNTL__UCP_ENA_2_MASK                                                                       0x00000004L
+#define PA_CL_CLIP_CNTL__UCP_ENA_3_MASK                                                                       0x00000008L
+#define PA_CL_CLIP_CNTL__UCP_ENA_4_MASK                                                                       0x00000010L
+#define PA_CL_CLIP_CNTL__UCP_ENA_5_MASK                                                                       0x00000020L
+#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG_MASK                                                              0x00002000L
+#define PA_CL_CLIP_CNTL__PS_UCP_MODE_MASK                                                                     0x0000C000L
+#define PA_CL_CLIP_CNTL__CLIP_DISABLE_MASK                                                                    0x00010000L
+#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA_MASK                                                               0x00020000L
+#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA_MASK                                                          0x00040000L
+#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF_MASK                                                               0x00080000L
+#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT_MASK                                                             0x00100000L
+#define PA_CL_CLIP_CNTL__VTX_KILL_OR_MASK                                                                     0x00200000L
+#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL_MASK                                                           0x00400000L
+#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA_MASK                                                         0x01000000L
+#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE_MASK                                                       0x02000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE_MASK                                                              0x04000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE_MASK                                                               0x08000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_PROG_NEAR_ENA_MASK                                                             0x10000000L
+//PA_SU_SC_MODE_CNTL
+#define PA_SU_SC_MODE_CNTL__CULL_FRONT__SHIFT                                                                 0x0
+#define PA_SU_SC_MODE_CNTL__CULL_BACK__SHIFT                                                                  0x1
+#define PA_SU_SC_MODE_CNTL__FACE__SHIFT                                                                       0x2
+#define PA_SU_SC_MODE_CNTL__POLY_MODE__SHIFT                                                                  0x3
+#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE__SHIFT                                                       0x5
+#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE__SHIFT                                                        0x8
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE__SHIFT                                                   0xb
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE__SHIFT                                                    0xc
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE__SHIFT                                                    0xd
+#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE__SHIFT                                                   0x10
+#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST__SHIFT                                                         0x13
+#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS__SHIFT                                                             0x14
+#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA__SHIFT                                                          0x15
+#define PA_SU_SC_MODE_CNTL__RIGHT_TRIANGLE_ALTERNATE_GRADIENT_REF__SHIFT                                      0x16
+#define PA_SU_SC_MODE_CNTL__NEW_QUAD_DECOMPOSITION__SHIFT                                                     0x17
+#define PA_SU_SC_MODE_CNTL__CULL_FRONT_MASK                                                                   0x00000001L
+#define PA_SU_SC_MODE_CNTL__CULL_BACK_MASK                                                                    0x00000002L
+#define PA_SU_SC_MODE_CNTL__FACE_MASK                                                                         0x00000004L
+#define PA_SU_SC_MODE_CNTL__POLY_MODE_MASK                                                                    0x00000018L
+#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE_MASK                                                         0x000000E0L
+#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE_MASK                                                          0x00000700L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE_MASK                                                     0x00000800L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE_MASK                                                      0x00001000L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE_MASK                                                      0x00002000L
+#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE_MASK                                                     0x00010000L
+#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST_MASK                                                           0x00080000L
+#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS_MASK                                                               0x00100000L
+#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA_MASK                                                            0x00200000L
+#define PA_SU_SC_MODE_CNTL__RIGHT_TRIANGLE_ALTERNATE_GRADIENT_REF_MASK                                        0x00400000L
+#define PA_SU_SC_MODE_CNTL__NEW_QUAD_DECOMPOSITION_MASK                                                       0x00800000L
+//PA_CL_VTE_CNTL
+#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA__SHIFT                                                              0x0
+#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA__SHIFT                                                             0x1
+#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA__SHIFT                                                              0x2
+#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA__SHIFT                                                             0x3
+#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA__SHIFT                                                              0x4
+#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA__SHIFT                                                             0x5
+#define PA_CL_VTE_CNTL__VTX_XY_FMT__SHIFT                                                                     0x8
+#define PA_CL_VTE_CNTL__VTX_Z_FMT__SHIFT                                                                      0x9
+#define PA_CL_VTE_CNTL__VTX_W0_FMT__SHIFT                                                                     0xa
+#define PA_CL_VTE_CNTL__PERFCOUNTER_REF__SHIFT                                                                0xb
+#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA_MASK                                                                0x00000001L
+#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA_MASK                                                               0x00000002L
+#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA_MASK                                                                0x00000004L
+#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA_MASK                                                               0x00000008L
+#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA_MASK                                                                0x00000010L
+#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA_MASK                                                               0x00000020L
+#define PA_CL_VTE_CNTL__VTX_XY_FMT_MASK                                                                       0x00000100L
+#define PA_CL_VTE_CNTL__VTX_Z_FMT_MASK                                                                        0x00000200L
+#define PA_CL_VTE_CNTL__VTX_W0_FMT_MASK                                                                       0x00000400L
+#define PA_CL_VTE_CNTL__PERFCOUNTER_REF_MASK                                                                  0x00000800L
+//PA_CL_VS_OUT_CNTL
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0__SHIFT                                                             0x0
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1__SHIFT                                                             0x1
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2__SHIFT                                                             0x2
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3__SHIFT                                                             0x3
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4__SHIFT                                                             0x4
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5__SHIFT                                                             0x5
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6__SHIFT                                                             0x6
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7__SHIFT                                                             0x7
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0__SHIFT                                                             0x8
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1__SHIFT                                                             0x9
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2__SHIFT                                                             0xa
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3__SHIFT                                                             0xb
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4__SHIFT                                                             0xc
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5__SHIFT                                                             0xd
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6__SHIFT                                                             0xe
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7__SHIFT                                                             0xf
+#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE__SHIFT                                                          0x10
+#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG__SHIFT                                                           0x11
+#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX__SHIFT                                                  0x12
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX__SHIFT                                                       0x13
+#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG__SHIFT                                                           0x14
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA__SHIFT                                                         0x15
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA__SHIFT                                                      0x16
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA__SHIFT                                                      0x17
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA__SHIFT                                                    0x18
+#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG__SHIFT                                                         0x19
+#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH__SHIFT                                                          0x1a
+#define PA_CL_VS_OUT_CNTL__USE_VTX_SHD_OBJPRIM_ID__SHIFT                                                      0x1b
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0_MASK                                                               0x00000001L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1_MASK                                                               0x00000002L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2_MASK                                                               0x00000004L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3_MASK                                                               0x00000008L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4_MASK                                                               0x00000010L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5_MASK                                                               0x00000020L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6_MASK                                                               0x00000040L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7_MASK                                                               0x00000080L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0_MASK                                                               0x00000100L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1_MASK                                                               0x00000200L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2_MASK                                                               0x00000400L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3_MASK                                                               0x00000800L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4_MASK                                                               0x00001000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5_MASK                                                               0x00002000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6_MASK                                                               0x00004000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7_MASK                                                               0x00008000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE_MASK                                                            0x00010000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG_MASK                                                             0x00020000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX_MASK                                                    0x00040000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX_MASK                                                         0x00080000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG_MASK                                                             0x00100000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA_MASK                                                           0x00200000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA_MASK                                                        0x00400000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA_MASK                                                        0x00800000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA_MASK                                                      0x01000000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG_MASK                                                           0x02000000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH_MASK                                                            0x04000000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_SHD_OBJPRIM_ID_MASK                                                        0x08000000L
+//PA_CL_NANINF_CNTL
+#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD__SHIFT                                                          0x0
+#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD__SHIFT                                                           0x1
+#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD__SHIFT                                                           0x2
+#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0__SHIFT                                                           0x3
+#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN__SHIFT                                                           0x4
+#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN__SHIFT                                                            0x5
+#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN__SHIFT                                                            0x6
+#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0__SHIFT                                                        0x7
+#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF__SHIFT                                                            0x8
+#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN__SHIFT                                                            0x9
+#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF__SHIFT                                                             0xa
+#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN__SHIFT                                                             0xb
+#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF__SHIFT                                                             0xc
+#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN__SHIFT                                                             0xd
+#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD__SHIFT                                                    0xe
+#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0__SHIFT                                                         0x14
+#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD_MASK                                                            0x00000001L
+#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD_MASK                                                             0x00000002L
+#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD_MASK                                                             0x00000004L
+#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0_MASK                                                             0x00000008L
+#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN_MASK                                                             0x00000010L
+#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN_MASK                                                              0x00000020L
+#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN_MASK                                                              0x00000040L
+#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0_MASK                                                          0x00000080L
+#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF_MASK                                                              0x00000100L
+#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN_MASK                                                              0x00000200L
+#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF_MASK                                                               0x00000400L
+#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN_MASK                                                               0x00000800L
+#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF_MASK                                                               0x00001000L
+#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN_MASK                                                               0x00002000L
+#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD_MASK                                                      0x00004000L
+#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0_MASK                                                           0x00100000L
+//PA_SU_LINE_STIPPLE_CNTL
+#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET__SHIFT                                                    0x0
+#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH__SHIFT                                                    0x2
+#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM__SHIFT                                                      0x3
+#define PA_SU_LINE_STIPPLE_CNTL__DIAMOND_ADJUST__SHIFT                                                        0x4
+#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET_MASK                                                      0x00000003L
+#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH_MASK                                                      0x00000004L
+#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM_MASK                                                        0x00000008L
+#define PA_SU_LINE_STIPPLE_CNTL__DIAMOND_ADJUST_MASK                                                          0x00000010L
+//PA_SU_LINE_STIPPLE_SCALE
+#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE__SHIFT                                                   0x0
+#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE_MASK                                                     0xFFFFFFFFL
+//PA_SU_PRIM_FILTER_CNTL
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT                                                0x0
+#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT                                                    0x1
+#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT                                                   0x2
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT                                               0x3
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA__SHIFT                                                    0x4
+#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA__SHIFT                                                        0x5
+#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA__SHIFT                                                       0x6
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA__SHIFT                                                   0x7
+#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT__SHIFT                                                   0x8
+#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION__SHIFT                                                   0x1e
+#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION__SHIFT                                                  0x1f
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK                                                  0x00000001L
+#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK                                                      0x00000002L
+#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK                                                     0x00000004L
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK                                                 0x00000008L
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA_MASK                                                      0x00000010L
+#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA_MASK                                                          0x00000020L
+#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA_MASK                                                         0x00000040L
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA_MASK                                                     0x00000080L
+#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT_MASK                                                     0x0000FF00L
+#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION_MASK                                                     0x40000000L
+#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION_MASK                                                    0x80000000L
+//PA_SU_SMALL_PRIM_FILTER_CNTL
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SMALL_PRIM_FILTER_ENABLE__SHIFT                                         0x0
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT                                          0x1
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT                                              0x2
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT                                             0x3
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT                                         0x4
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SRBSL_ENABLE__SHIFT                                                     0x5
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SC_1XMSAA_COMPATIBLE_DISABLE__SHIFT                                     0x6
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SMALL_PRIM_FILTER_ENABLE_MASK                                           0x00000001L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK                                            0x00000002L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK                                                0x00000004L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK                                               0x00000008L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK                                           0x00000010L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SRBSL_ENABLE_MASK                                                       0x00000020L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SC_1XMSAA_COMPATIBLE_DISABLE_MASK                                       0x00000040L
+//PA_CL_OBJPRIM_ID_CNTL
+#define PA_CL_OBJPRIM_ID_CNTL__OBJ_ID_SEL__SHIFT                                                              0x0
+#define PA_CL_OBJPRIM_ID_CNTL__ADD_PIPED_PRIM_ID__SHIFT                                                       0x1
+#define PA_CL_OBJPRIM_ID_CNTL__EN_32BIT_OBJPRIMID__SHIFT                                                      0x2
+#define PA_CL_OBJPRIM_ID_CNTL__OBJ_ID_SEL_MASK                                                                0x00000001L
+#define PA_CL_OBJPRIM_ID_CNTL__ADD_PIPED_PRIM_ID_MASK                                                         0x00000002L
+#define PA_CL_OBJPRIM_ID_CNTL__EN_32BIT_OBJPRIMID_MASK                                                        0x00000004L
+//PA_CL_NGG_CNTL
+#define PA_CL_NGG_CNTL__VERTEX_REUSE_OFF__SHIFT                                                               0x0
+#define PA_CL_NGG_CNTL__INDEX_BUF_EDGE_FLAG_ENA__SHIFT                                                        0x1
+#define PA_CL_NGG_CNTL__VERTEX_REUSE_OFF_MASK                                                                 0x00000001L
+#define PA_CL_NGG_CNTL__INDEX_BUF_EDGE_FLAG_ENA_MASK                                                          0x00000002L
+//PA_SU_OVER_RASTERIZATION_CNTL
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_TRIANGLES__SHIFT                                        0x0
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_LINES__SHIFT                                            0x1
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_POINTS__SHIFT                                           0x2
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_RECTANGLES__SHIFT                                       0x3
+#define PA_SU_OVER_RASTERIZATION_CNTL__USE_PROVOKING_ZW__SHIFT                                                0x4
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_TRIANGLES_MASK                                          0x00000001L
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_LINES_MASK                                              0x00000002L
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_POINTS_MASK                                             0x00000004L
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_RECTANGLES_MASK                                         0x00000008L
+#define PA_SU_OVER_RASTERIZATION_CNTL__USE_PROVOKING_ZW_MASK                                                  0x00000010L
+//PA_STEREO_CNTL
+#define PA_STEREO_CNTL__EN_STEREO__SHIFT                                                                      0x0
+#define PA_STEREO_CNTL__STEREO_MODE__SHIFT                                                                    0x1
+#define PA_STEREO_CNTL__RT_SLICE_MODE__SHIFT                                                                  0x5
+#define PA_STEREO_CNTL__RT_SLICE_OFFSET__SHIFT                                                                0x8
+#define PA_STEREO_CNTL__VP_ID_MODE__SHIFT                                                                     0xa
+#define PA_STEREO_CNTL__VP_ID_OFFSET__SHIFT                                                                   0xd
+#define PA_STEREO_CNTL__EN_STEREO_MASK                                                                        0x00000001L
+#define PA_STEREO_CNTL__STEREO_MODE_MASK                                                                      0x0000001EL
+#define PA_STEREO_CNTL__RT_SLICE_MODE_MASK                                                                    0x000000E0L
+#define PA_STEREO_CNTL__RT_SLICE_OFFSET_MASK                                                                  0x00000300L
+#define PA_STEREO_CNTL__VP_ID_MODE_MASK                                                                       0x00001C00L
+#define PA_STEREO_CNTL__VP_ID_OFFSET_MASK                                                                     0x0001E000L
+//PA_SU_POINT_SIZE
+#define PA_SU_POINT_SIZE__HEIGHT__SHIFT                                                                       0x0
+#define PA_SU_POINT_SIZE__WIDTH__SHIFT                                                                        0x10
+#define PA_SU_POINT_SIZE__HEIGHT_MASK                                                                         0x0000FFFFL
+#define PA_SU_POINT_SIZE__WIDTH_MASK                                                                          0xFFFF0000L
+//PA_SU_POINT_MINMAX
+#define PA_SU_POINT_MINMAX__MIN_SIZE__SHIFT                                                                   0x0
+#define PA_SU_POINT_MINMAX__MAX_SIZE__SHIFT                                                                   0x10
+#define PA_SU_POINT_MINMAX__MIN_SIZE_MASK                                                                     0x0000FFFFL
+#define PA_SU_POINT_MINMAX__MAX_SIZE_MASK                                                                     0xFFFF0000L
+//PA_SU_LINE_CNTL
+#define PA_SU_LINE_CNTL__WIDTH__SHIFT                                                                         0x0
+#define PA_SU_LINE_CNTL__WIDTH_MASK                                                                           0x0000FFFFL
+//PA_SC_LINE_STIPPLE
+#define PA_SC_LINE_STIPPLE__LINE_PATTERN__SHIFT                                                               0x0
+#define PA_SC_LINE_STIPPLE__REPEAT_COUNT__SHIFT                                                               0x10
+#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER__SHIFT                                                          0x1c
+#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL__SHIFT                                                            0x1d
+#define PA_SC_LINE_STIPPLE__LINE_PATTERN_MASK                                                                 0x0000FFFFL
+#define PA_SC_LINE_STIPPLE__REPEAT_COUNT_MASK                                                                 0x00FF0000L
+#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER_MASK                                                            0x10000000L
+#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL_MASK                                                              0x60000000L
+//VGT_OUTPUT_PATH_CNTL
+#define VGT_OUTPUT_PATH_CNTL__PATH_SELECT__SHIFT                                                              0x0
+#define VGT_OUTPUT_PATH_CNTL__PATH_SELECT_MASK                                                                0x00000007L
+//VGT_HOS_CNTL
+#define VGT_HOS_CNTL__TESS_MODE__SHIFT                                                                        0x0
+#define VGT_HOS_CNTL__TESS_MODE_MASK                                                                          0x00000003L
+//VGT_HOS_MAX_TESS_LEVEL
+#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS__SHIFT                                                               0x0
+#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS_MASK                                                                 0xFFFFFFFFL
+//VGT_HOS_MIN_TESS_LEVEL
+#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS__SHIFT                                                               0x0
+#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS_MASK                                                                 0xFFFFFFFFL
+//VGT_HOS_REUSE_DEPTH
+#define VGT_HOS_REUSE_DEPTH__REUSE_DEPTH__SHIFT                                                               0x0
+#define VGT_HOS_REUSE_DEPTH__REUSE_DEPTH_MASK                                                                 0x000000FFL
+//VGT_GROUP_PRIM_TYPE
+#define VGT_GROUP_PRIM_TYPE__PRIM_TYPE__SHIFT                                                                 0x0
+#define VGT_GROUP_PRIM_TYPE__RETAIN_ORDER__SHIFT                                                              0xe
+#define VGT_GROUP_PRIM_TYPE__RETAIN_QUADS__SHIFT                                                              0xf
+#define VGT_GROUP_PRIM_TYPE__PRIM_ORDER__SHIFT                                                                0x10
+#define VGT_GROUP_PRIM_TYPE__PRIM_TYPE_MASK                                                                   0x0000001FL
+#define VGT_GROUP_PRIM_TYPE__RETAIN_ORDER_MASK                                                                0x00004000L
+#define VGT_GROUP_PRIM_TYPE__RETAIN_QUADS_MASK                                                                0x00008000L
+#define VGT_GROUP_PRIM_TYPE__PRIM_ORDER_MASK                                                                  0x00070000L
+//VGT_GROUP_FIRST_DECR
+#define VGT_GROUP_FIRST_DECR__FIRST_DECR__SHIFT                                                               0x0
+#define VGT_GROUP_FIRST_DECR__FIRST_DECR_MASK                                                                 0x0000000FL
+//VGT_GROUP_DECR
+#define VGT_GROUP_DECR__DECR__SHIFT                                                                           0x0
+#define VGT_GROUP_DECR__DECR_MASK                                                                             0x0000000FL
+//VGT_GROUP_VECT_0_CNTL
+#define VGT_GROUP_VECT_0_CNTL__COMP_X_EN__SHIFT                                                               0x0
+#define VGT_GROUP_VECT_0_CNTL__COMP_Y_EN__SHIFT                                                               0x1
+#define VGT_GROUP_VECT_0_CNTL__COMP_Z_EN__SHIFT                                                               0x2
+#define VGT_GROUP_VECT_0_CNTL__COMP_W_EN__SHIFT                                                               0x3
+#define VGT_GROUP_VECT_0_CNTL__STRIDE__SHIFT                                                                  0x8
+#define VGT_GROUP_VECT_0_CNTL__SHIFT__SHIFT                                                                   0x10
+#define VGT_GROUP_VECT_0_CNTL__COMP_X_EN_MASK                                                                 0x00000001L
+#define VGT_GROUP_VECT_0_CNTL__COMP_Y_EN_MASK                                                                 0x00000002L
+#define VGT_GROUP_VECT_0_CNTL__COMP_Z_EN_MASK                                                                 0x00000004L
+#define VGT_GROUP_VECT_0_CNTL__COMP_W_EN_MASK                                                                 0x00000008L
+#define VGT_GROUP_VECT_0_CNTL__STRIDE_MASK                                                                    0x0000FF00L
+#define VGT_GROUP_VECT_0_CNTL__SHIFT_MASK                                                                     0x00FF0000L
+//VGT_GROUP_VECT_1_CNTL
+#define VGT_GROUP_VECT_1_CNTL__COMP_X_EN__SHIFT                                                               0x0
+#define VGT_GROUP_VECT_1_CNTL__COMP_Y_EN__SHIFT                                                               0x1
+#define VGT_GROUP_VECT_1_CNTL__COMP_Z_EN__SHIFT                                                               0x2
+#define VGT_GROUP_VECT_1_CNTL__COMP_W_EN__SHIFT                                                               0x3
+#define VGT_GROUP_VECT_1_CNTL__STRIDE__SHIFT                                                                  0x8
+#define VGT_GROUP_VECT_1_CNTL__SHIFT__SHIFT                                                                   0x10
+#define VGT_GROUP_VECT_1_CNTL__COMP_X_EN_MASK                                                                 0x00000001L
+#define VGT_GROUP_VECT_1_CNTL__COMP_Y_EN_MASK                                                                 0x00000002L
+#define VGT_GROUP_VECT_1_CNTL__COMP_Z_EN_MASK                                                                 0x00000004L
+#define VGT_GROUP_VECT_1_CNTL__COMP_W_EN_MASK                                                                 0x00000008L
+#define VGT_GROUP_VECT_1_CNTL__STRIDE_MASK                                                                    0x0000FF00L
+#define VGT_GROUP_VECT_1_CNTL__SHIFT_MASK                                                                     0x00FF0000L
+//VGT_GROUP_VECT_0_FMT_CNTL
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_CONV__SHIFT                                                              0x0
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_OFFSET__SHIFT                                                            0x4
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_CONV__SHIFT                                                              0x8
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_OFFSET__SHIFT                                                            0xc
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_CONV__SHIFT                                                              0x10
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_OFFSET__SHIFT                                                            0x14
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_CONV__SHIFT                                                              0x18
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_OFFSET__SHIFT                                                            0x1c
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_CONV_MASK                                                                0x0000000FL
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_OFFSET_MASK                                                              0x000000F0L
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_CONV_MASK                                                                0x00000F00L
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_OFFSET_MASK                                                              0x0000F000L
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_CONV_MASK                                                                0x000F0000L
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_OFFSET_MASK                                                              0x00F00000L
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_CONV_MASK                                                                0x0F000000L
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_OFFSET_MASK                                                              0xF0000000L
+//VGT_GROUP_VECT_1_FMT_CNTL
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_CONV__SHIFT                                                              0x0
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_OFFSET__SHIFT                                                            0x4
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_CONV__SHIFT                                                              0x8
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_OFFSET__SHIFT                                                            0xc
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_CONV__SHIFT                                                              0x10
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_OFFSET__SHIFT                                                            0x14
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_CONV__SHIFT                                                              0x18
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_OFFSET__SHIFT                                                            0x1c
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_CONV_MASK                                                                0x0000000FL
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_OFFSET_MASK                                                              0x000000F0L
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_CONV_MASK                                                                0x00000F00L
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_OFFSET_MASK                                                              0x0000F000L
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_CONV_MASK                                                                0x000F0000L
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_OFFSET_MASK                                                              0x00F00000L
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_CONV_MASK                                                                0x0F000000L
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_OFFSET_MASK                                                              0xF0000000L
+//VGT_GS_MODE
+#define VGT_GS_MODE__MODE__SHIFT                                                                              0x0
+#define VGT_GS_MODE__RESERVED_0__SHIFT                                                                        0x3
+#define VGT_GS_MODE__CUT_MODE__SHIFT                                                                          0x4
+#define VGT_GS_MODE__RESERVED_1__SHIFT                                                                        0x6
+#define VGT_GS_MODE__GS_C_PACK_EN__SHIFT                                                                      0xb
+#define VGT_GS_MODE__RESERVED_2__SHIFT                                                                        0xc
+#define VGT_GS_MODE__ES_PASSTHRU__SHIFT                                                                       0xd
+#define VGT_GS_MODE__RESERVED_3__SHIFT                                                                        0xe
+#define VGT_GS_MODE__RESERVED_4__SHIFT                                                                        0xf
+#define VGT_GS_MODE__RESERVED_5__SHIFT                                                                        0x10
+#define VGT_GS_MODE__PARTIAL_THD_AT_EOI__SHIFT                                                                0x11
+#define VGT_GS_MODE__SUPPRESS_CUTS__SHIFT                                                                     0x12
+#define VGT_GS_MODE__ES_WRITE_OPTIMIZE__SHIFT                                                                 0x13
+#define VGT_GS_MODE__GS_WRITE_OPTIMIZE__SHIFT                                                                 0x14
+#define VGT_GS_MODE__ONCHIP__SHIFT                                                                            0x15
+#define VGT_GS_MODE__MODE_MASK                                                                                0x00000007L
+#define VGT_GS_MODE__RESERVED_0_MASK                                                                          0x00000008L
+#define VGT_GS_MODE__CUT_MODE_MASK                                                                            0x00000030L
+#define VGT_GS_MODE__RESERVED_1_MASK                                                                          0x000007C0L
+#define VGT_GS_MODE__GS_C_PACK_EN_MASK                                                                        0x00000800L
+#define VGT_GS_MODE__RESERVED_2_MASK                                                                          0x00001000L
+#define VGT_GS_MODE__ES_PASSTHRU_MASK                                                                         0x00002000L
+#define VGT_GS_MODE__RESERVED_3_MASK                                                                          0x00004000L
+#define VGT_GS_MODE__RESERVED_4_MASK                                                                          0x00008000L
+#define VGT_GS_MODE__RESERVED_5_MASK                                                                          0x00010000L
+#define VGT_GS_MODE__PARTIAL_THD_AT_EOI_MASK                                                                  0x00020000L
+#define VGT_GS_MODE__SUPPRESS_CUTS_MASK                                                                       0x00040000L
+#define VGT_GS_MODE__ES_WRITE_OPTIMIZE_MASK                                                                   0x00080000L
+#define VGT_GS_MODE__GS_WRITE_OPTIMIZE_MASK                                                                   0x00100000L
+#define VGT_GS_MODE__ONCHIP_MASK                                                                              0x00600000L
+//VGT_GS_ONCHIP_CNTL
+#define VGT_GS_ONCHIP_CNTL__ES_VERTS_PER_SUBGRP__SHIFT                                                        0x0
+#define VGT_GS_ONCHIP_CNTL__GS_PRIMS_PER_SUBGRP__SHIFT                                                        0xb
+#define VGT_GS_ONCHIP_CNTL__GS_INST_PRIMS_IN_SUBGRP__SHIFT                                                    0x16
+#define VGT_GS_ONCHIP_CNTL__ES_VERTS_PER_SUBGRP_MASK                                                          0x000007FFL
+#define VGT_GS_ONCHIP_CNTL__GS_PRIMS_PER_SUBGRP_MASK                                                          0x003FF800L
+#define VGT_GS_ONCHIP_CNTL__GS_INST_PRIMS_IN_SUBGRP_MASK                                                      0xFFC00000L
+//PA_SC_MODE_CNTL_0
+#define PA_SC_MODE_CNTL_0__MSAA_ENABLE__SHIFT                                                                 0x0
+#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE__SHIFT                                                        0x1
+#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE__SHIFT                                                         0x2
+#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR__SHIFT                                                    0x3
+#define PA_SC_MODE_CNTL_0__SCALE_LINE_WIDTH_PAD__SHIFT                                                        0x4
+#define PA_SC_MODE_CNTL_0__ALTERNATE_RBS_PER_TILE__SHIFT                                                      0x5
+#define PA_SC_MODE_CNTL_0__COARSE_TILE_STARTS_ON_EVEN_RB__SHIFT                                               0x6
+#define PA_SC_MODE_CNTL_0__MSAA_ENABLE_MASK                                                                   0x00000001L
+#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE_MASK                                                          0x00000002L
+#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE_MASK                                                           0x00000004L
+#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR_MASK                                                      0x00000008L
+#define PA_SC_MODE_CNTL_0__SCALE_LINE_WIDTH_PAD_MASK                                                          0x00000010L
+#define PA_SC_MODE_CNTL_0__ALTERNATE_RBS_PER_TILE_MASK                                                        0x00000020L
+#define PA_SC_MODE_CNTL_0__COARSE_TILE_STARTS_ON_EVEN_RB_MASK                                                 0x00000040L
+//PA_SC_MODE_CNTL_1
+#define PA_SC_MODE_CNTL_1__WALK_SIZE__SHIFT                                                                   0x0
+#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT__SHIFT                                                              0x1
+#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST__SHIFT                                                    0x2
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE__SHIFT                                                           0x3
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE__SHIFT                                                             0x4
+#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE__SHIFT                                                 0x7
+#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE__SHIFT                                                      0x8
+#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE__SHIFT                                                          0x9
+#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR__SHIFT                                                       0xa
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT__SHIFT                                                             0xb
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET__SHIFT                                                             0xc
+#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT__SHIFT                                                             0xd
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z__SHIFT                                                          0xe
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK__SHIFT                                                   0xf
+#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE__SHIFT                                                              0x10
+#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE__SHIFT                                     0x11
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE__SHIFT                                                  0x12
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE__SHIFT                                                      0x13
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE__SHIFT                                                             0x14
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE__SHIFT                                               0x18
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE__SHIFT                                                     0x19
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE__SHIFT                                                        0x1a
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE__SHIFT                                               0x1b
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK__SHIFT                                                     0x1c
+#define PA_SC_MODE_CNTL_1__WALK_SIZE_MASK                                                                     0x00000001L
+#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT_MASK                                                                0x00000002L
+#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST_MASK                                                      0x00000004L
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE_MASK                                                             0x00000008L
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE_MASK                                                               0x00000070L
+#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE_MASK                                                   0x00000080L
+#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE_MASK                                                        0x00000100L
+#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE_MASK                                                            0x00000200L
+#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR_MASK                                                         0x00000400L
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT_MASK                                                               0x00000800L
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET_MASK                                                               0x00001000L
+#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT_MASK                                                               0x00002000L
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z_MASK                                                            0x00004000L
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK_MASK                                                     0x00008000L
+#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE_MASK                                                                0x00010000L
+#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE_MASK                                       0x00020000L
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE_MASK                                                    0x00040000L
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE_MASK                                                        0x00080000L
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_MASK                                                               0x00F00000L
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE_MASK                                                 0x01000000L
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE_MASK                                                       0x02000000L
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE_MASK                                                          0x04000000L
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE_MASK                                                 0x08000000L
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK_MASK                                                       0x70000000L
+//VGT_ENHANCE
+#define VGT_ENHANCE__MISC__SHIFT                                                                              0x0
+#define VGT_ENHANCE__MISC_MASK                                                                                0xFFFFFFFFL
+//VGT_GS_PER_ES
+#define VGT_GS_PER_ES__GS_PER_ES__SHIFT                                                                       0x0
+#define VGT_GS_PER_ES__GS_PER_ES_MASK                                                                         0x000007FFL
+//VGT_ES_PER_GS
+#define VGT_ES_PER_GS__ES_PER_GS__SHIFT                                                                       0x0
+#define VGT_ES_PER_GS__ES_PER_GS_MASK                                                                         0x000007FFL
+//VGT_GS_PER_VS
+#define VGT_GS_PER_VS__GS_PER_VS__SHIFT                                                                       0x0
+#define VGT_GS_PER_VS__GS_PER_VS_MASK                                                                         0x0000000FL
+//VGT_GSVS_RING_OFFSET_1
+#define VGT_GSVS_RING_OFFSET_1__OFFSET__SHIFT                                                                 0x0
+#define VGT_GSVS_RING_OFFSET_1__OFFSET_MASK                                                                   0x00007FFFL
+//VGT_GSVS_RING_OFFSET_2
+#define VGT_GSVS_RING_OFFSET_2__OFFSET__SHIFT                                                                 0x0
+#define VGT_GSVS_RING_OFFSET_2__OFFSET_MASK                                                                   0x00007FFFL
+//VGT_GSVS_RING_OFFSET_3
+#define VGT_GSVS_RING_OFFSET_3__OFFSET__SHIFT                                                                 0x0
+#define VGT_GSVS_RING_OFFSET_3__OFFSET_MASK                                                                   0x00007FFFL
+//VGT_GS_OUT_PRIM_TYPE
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE__SHIFT                                                             0x0
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_1__SHIFT                                                           0x8
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_2__SHIFT                                                           0x10
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_3__SHIFT                                                           0x16
+#define VGT_GS_OUT_PRIM_TYPE__UNIQUE_TYPE_PER_STREAM__SHIFT                                                   0x1f
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_MASK                                                               0x0000003FL
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_1_MASK                                                             0x00003F00L
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_2_MASK                                                             0x003F0000L
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_3_MASK                                                             0x0FC00000L
+#define VGT_GS_OUT_PRIM_TYPE__UNIQUE_TYPE_PER_STREAM_MASK                                                     0x80000000L
+//IA_ENHANCE
+#define IA_ENHANCE__MISC__SHIFT                                                                               0x0
+#define IA_ENHANCE__MISC_MASK                                                                                 0xFFFFFFFFL
+//VGT_DMA_SIZE
+#define VGT_DMA_SIZE__NUM_INDICES__SHIFT                                                                      0x0
+#define VGT_DMA_SIZE__NUM_INDICES_MASK                                                                        0xFFFFFFFFL
+//VGT_DMA_MAX_SIZE
+#define VGT_DMA_MAX_SIZE__MAX_SIZE__SHIFT                                                                     0x0
+#define VGT_DMA_MAX_SIZE__MAX_SIZE_MASK                                                                       0xFFFFFFFFL
+//VGT_DMA_INDEX_TYPE
+#define VGT_DMA_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                 0x0
+#define VGT_DMA_INDEX_TYPE__SWAP_MODE__SHIFT                                                                  0x2
+#define VGT_DMA_INDEX_TYPE__BUF_TYPE__SHIFT                                                                   0x4
+#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY__SHIFT                                                               0x6
+#define VGT_DMA_INDEX_TYPE__PRIMGEN_EN__SHIFT                                                                 0x8
+#define VGT_DMA_INDEX_TYPE__NOT_EOP__SHIFT                                                                    0x9
+#define VGT_DMA_INDEX_TYPE__REQ_PATH__SHIFT                                                                   0xa
+#define VGT_DMA_INDEX_TYPE__INDEX_TYPE_MASK                                                                   0x00000003L
+#define VGT_DMA_INDEX_TYPE__SWAP_MODE_MASK                                                                    0x0000000CL
+#define VGT_DMA_INDEX_TYPE__BUF_TYPE_MASK                                                                     0x00000030L
+#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY_MASK                                                                 0x00000040L
+#define VGT_DMA_INDEX_TYPE__PRIMGEN_EN_MASK                                                                   0x00000100L
+#define VGT_DMA_INDEX_TYPE__NOT_EOP_MASK                                                                      0x00000200L
+#define VGT_DMA_INDEX_TYPE__REQ_PATH_MASK                                                                     0x00000400L
+//WD_ENHANCE
+#define WD_ENHANCE__MISC__SHIFT                                                                               0x0
+#define WD_ENHANCE__MISC_MASK                                                                                 0xFFFFFFFFL
+//VGT_PRIMITIVEID_EN
+#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN__SHIFT                                                             0x0
+#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI__SHIFT                                                       0x1
+#define VGT_PRIMITIVEID_EN__NGG_DISABLE_PROVOK_REUSE__SHIFT                                                   0x2
+#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN_MASK                                                               0x00000001L
+#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI_MASK                                                         0x00000002L
+#define VGT_PRIMITIVEID_EN__NGG_DISABLE_PROVOK_REUSE_MASK                                                     0x00000004L
+//VGT_DMA_NUM_INSTANCES
+#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES__SHIFT                                                           0x0
+#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES_MASK                                                             0xFFFFFFFFL
+//VGT_PRIMITIVEID_RESET
+#define VGT_PRIMITIVEID_RESET__VALUE__SHIFT                                                                   0x0
+#define VGT_PRIMITIVEID_RESET__VALUE_MASK                                                                     0xFFFFFFFFL
+//VGT_EVENT_INITIATOR
+#define VGT_EVENT_INITIATOR__EVENT_TYPE__SHIFT                                                                0x0
+#define VGT_EVENT_INITIATOR__ADDRESS_HI__SHIFT                                                                0xa
+#define VGT_EVENT_INITIATOR__EXTENDED_EVENT__SHIFT                                                            0x1b
+#define VGT_EVENT_INITIATOR__EVENT_TYPE_MASK                                                                  0x0000003FL
+#define VGT_EVENT_INITIATOR__ADDRESS_HI_MASK                                                                  0x07FFFC00L
+#define VGT_EVENT_INITIATOR__EXTENDED_EVENT_MASK                                                              0x08000000L
+//VGT_GS_MAX_PRIMS_PER_SUBGROUP
+#define VGT_GS_MAX_PRIMS_PER_SUBGROUP__MAX_PRIMS_PER_SUBGROUP__SHIFT                                          0x0
+#define VGT_GS_MAX_PRIMS_PER_SUBGROUP__MAX_PRIMS_PER_SUBGROUP_MASK                                            0x0000FFFFL
+//VGT_DRAW_PAYLOAD_CNTL
+#define VGT_DRAW_PAYLOAD_CNTL__OBJPRIM_ID_EN__SHIFT                                                           0x0
+#define VGT_DRAW_PAYLOAD_CNTL__EN_REG_RT_INDEX__SHIFT                                                         0x1
+#define VGT_DRAW_PAYLOAD_CNTL__EN_PIPELINE_PRIMID__SHIFT                                                      0x2
+#define VGT_DRAW_PAYLOAD_CNTL__OBJECT_ID_INST_EN__SHIFT                                                       0x3
+#define VGT_DRAW_PAYLOAD_CNTL__OBJPRIM_ID_EN_MASK                                                             0x00000001L
+#define VGT_DRAW_PAYLOAD_CNTL__EN_REG_RT_INDEX_MASK                                                           0x00000002L
+#define VGT_DRAW_PAYLOAD_CNTL__EN_PIPELINE_PRIMID_MASK                                                        0x00000004L
+#define VGT_DRAW_PAYLOAD_CNTL__OBJECT_ID_INST_EN_MASK                                                         0x00000008L
+//VGT_INSTANCE_STEP_RATE_0
+#define VGT_INSTANCE_STEP_RATE_0__STEP_RATE__SHIFT                                                            0x0
+#define VGT_INSTANCE_STEP_RATE_0__STEP_RATE_MASK                                                              0xFFFFFFFFL
+//VGT_INSTANCE_STEP_RATE_1
+#define VGT_INSTANCE_STEP_RATE_1__STEP_RATE__SHIFT                                                            0x0
+#define VGT_INSTANCE_STEP_RATE_1__STEP_RATE_MASK                                                              0xFFFFFFFFL
+//IA_MULTI_VGT_PARAM_BC
+//VGT_ESGS_RING_ITEMSIZE
+#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE__SHIFT                                                               0x0
+#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE_MASK                                                                 0x00007FFFL
+//VGT_GSVS_RING_ITEMSIZE
+#define VGT_GSVS_RING_ITEMSIZE__ITEMSIZE__SHIFT                                                               0x0
+#define VGT_GSVS_RING_ITEMSIZE__ITEMSIZE_MASK                                                                 0x00007FFFL
+//VGT_REUSE_OFF
+#define VGT_REUSE_OFF__REUSE_OFF__SHIFT                                                                       0x0
+#define VGT_REUSE_OFF__REUSE_OFF_MASK                                                                         0x00000001L
+//VGT_VTX_CNT_EN
+#define VGT_VTX_CNT_EN__VTX_CNT_EN__SHIFT                                                                     0x0
+#define VGT_VTX_CNT_EN__VTX_CNT_EN_MASK                                                                       0x00000001L
+//DB_HTILE_SURFACE
+#define DB_HTILE_SURFACE__FULL_CACHE__SHIFT                                                                   0x1
+#define DB_HTILE_SURFACE__HTILE_USES_PRELOAD_WIN__SHIFT                                                       0x2
+#define DB_HTILE_SURFACE__PRELOAD__SHIFT                                                                      0x3
+#define DB_HTILE_SURFACE__PREFETCH_WIDTH__SHIFT                                                               0x4
+#define DB_HTILE_SURFACE__PREFETCH_HEIGHT__SHIFT                                                              0xa
+#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE__SHIFT                                                      0x10
+#define DB_HTILE_SURFACE__PIPE_ALIGNED__SHIFT                                                                 0x12
+#define DB_HTILE_SURFACE__RB_ALIGNED__SHIFT                                                                   0x13
+#define DB_HTILE_SURFACE__FULL_CACHE_MASK                                                                     0x00000002L
+#define DB_HTILE_SURFACE__HTILE_USES_PRELOAD_WIN_MASK                                                         0x00000004L
+#define DB_HTILE_SURFACE__PRELOAD_MASK                                                                        0x00000008L
+#define DB_HTILE_SURFACE__PREFETCH_WIDTH_MASK                                                                 0x000003F0L
+#define DB_HTILE_SURFACE__PREFETCH_HEIGHT_MASK                                                                0x0000FC00L
+#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE_MASK                                                        0x00010000L
+#define DB_HTILE_SURFACE__PIPE_ALIGNED_MASK                                                                   0x00040000L
+#define DB_HTILE_SURFACE__RB_ALIGNED_MASK                                                                     0x00080000L
+//DB_SRESULTS_COMPARE_STATE0
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0__SHIFT                                                       0x0
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0__SHIFT                                                      0x4
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0__SHIFT                                                       0xc
+#define DB_SRESULTS_COMPARE_STATE0__ENABLE0__SHIFT                                                            0x18
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0_MASK                                                         0x00000007L
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0_MASK                                                        0x00000FF0L
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0_MASK                                                         0x000FF000L
+#define DB_SRESULTS_COMPARE_STATE0__ENABLE0_MASK                                                              0x01000000L
+//DB_SRESULTS_COMPARE_STATE1
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1__SHIFT                                                       0x0
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1__SHIFT                                                      0x4
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1__SHIFT                                                       0xc
+#define DB_SRESULTS_COMPARE_STATE1__ENABLE1__SHIFT                                                            0x18
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1_MASK                                                         0x00000007L
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1_MASK                                                        0x00000FF0L
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1_MASK                                                         0x000FF000L
+#define DB_SRESULTS_COMPARE_STATE1__ENABLE1_MASK                                                              0x01000000L
+//DB_PRELOAD_CONTROL
+#define DB_PRELOAD_CONTROL__START_X__SHIFT                                                                    0x0
+#define DB_PRELOAD_CONTROL__START_Y__SHIFT                                                                    0x8
+#define DB_PRELOAD_CONTROL__MAX_X__SHIFT                                                                      0x10
+#define DB_PRELOAD_CONTROL__MAX_Y__SHIFT                                                                      0x18
+#define DB_PRELOAD_CONTROL__START_X_MASK                                                                      0x000000FFL
+#define DB_PRELOAD_CONTROL__START_Y_MASK                                                                      0x0000FF00L
+#define DB_PRELOAD_CONTROL__MAX_X_MASK                                                                        0x00FF0000L
+#define DB_PRELOAD_CONTROL__MAX_Y_MASK                                                                        0xFF000000L
+//VGT_STRMOUT_BUFFER_SIZE_0
+#define VGT_STRMOUT_BUFFER_SIZE_0__SIZE__SHIFT                                                                0x0
+#define VGT_STRMOUT_BUFFER_SIZE_0__SIZE_MASK                                                                  0xFFFFFFFFL
+//VGT_STRMOUT_VTX_STRIDE_0
+#define VGT_STRMOUT_VTX_STRIDE_0__STRIDE__SHIFT                                                               0x0
+#define VGT_STRMOUT_VTX_STRIDE_0__STRIDE_MASK                                                                 0x000003FFL
+//VGT_STRMOUT_BUFFER_OFFSET_0
+#define VGT_STRMOUT_BUFFER_OFFSET_0__OFFSET__SHIFT                                                            0x0
+#define VGT_STRMOUT_BUFFER_OFFSET_0__OFFSET_MASK                                                              0xFFFFFFFFL
+//VGT_STRMOUT_BUFFER_SIZE_1
+#define VGT_STRMOUT_BUFFER_SIZE_1__SIZE__SHIFT                                                                0x0
+#define VGT_STRMOUT_BUFFER_SIZE_1__SIZE_MASK                                                                  0xFFFFFFFFL
+//VGT_STRMOUT_VTX_STRIDE_1
+#define VGT_STRMOUT_VTX_STRIDE_1__STRIDE__SHIFT                                                               0x0
+#define VGT_STRMOUT_VTX_STRIDE_1__STRIDE_MASK                                                                 0x000003FFL
+//VGT_STRMOUT_BUFFER_OFFSET_1
+#define VGT_STRMOUT_BUFFER_OFFSET_1__OFFSET__SHIFT                                                            0x0
+#define VGT_STRMOUT_BUFFER_OFFSET_1__OFFSET_MASK                                                              0xFFFFFFFFL
+//VGT_STRMOUT_BUFFER_SIZE_2
+#define VGT_STRMOUT_BUFFER_SIZE_2__SIZE__SHIFT                                                                0x0
+#define VGT_STRMOUT_BUFFER_SIZE_2__SIZE_MASK                                                                  0xFFFFFFFFL
+//VGT_STRMOUT_VTX_STRIDE_2
+#define VGT_STRMOUT_VTX_STRIDE_2__STRIDE__SHIFT                                                               0x0
+#define VGT_STRMOUT_VTX_STRIDE_2__STRIDE_MASK                                                                 0x000003FFL
+//VGT_STRMOUT_BUFFER_OFFSET_2
+#define VGT_STRMOUT_BUFFER_OFFSET_2__OFFSET__SHIFT                                                            0x0
+#define VGT_STRMOUT_BUFFER_OFFSET_2__OFFSET_MASK                                                              0xFFFFFFFFL
+//VGT_STRMOUT_BUFFER_SIZE_3
+#define VGT_STRMOUT_BUFFER_SIZE_3__SIZE__SHIFT                                                                0x0
+#define VGT_STRMOUT_BUFFER_SIZE_3__SIZE_MASK                                                                  0xFFFFFFFFL
+//VGT_STRMOUT_VTX_STRIDE_3
+#define VGT_STRMOUT_VTX_STRIDE_3__STRIDE__SHIFT                                                               0x0
+#define VGT_STRMOUT_VTX_STRIDE_3__STRIDE_MASK                                                                 0x000003FFL
+//VGT_STRMOUT_BUFFER_OFFSET_3
+#define VGT_STRMOUT_BUFFER_OFFSET_3__OFFSET__SHIFT                                                            0x0
+#define VGT_STRMOUT_BUFFER_OFFSET_3__OFFSET_MASK                                                              0xFFFFFFFFL
+//VGT_STRMOUT_DRAW_OPAQUE_OFFSET
+#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET__SHIFT                                                         0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET_MASK                                                           0xFFFFFFFFL
+//VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE
+#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE__SHIFT                                               0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE_MASK                                                 0xFFFFFFFFL
+//VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE
+#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE__SHIFT                                           0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE_MASK                                             0x000001FFL
+//VGT_GS_MAX_VERT_OUT
+#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT__SHIFT                                                              0x0
+#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT_MASK                                                                0x000007FFL
+//VGT_TESS_DISTRIBUTION
+#define VGT_TESS_DISTRIBUTION__ACCUM_ISOLINE__SHIFT                                                           0x0
+#define VGT_TESS_DISTRIBUTION__ACCUM_TRI__SHIFT                                                               0x8
+#define VGT_TESS_DISTRIBUTION__ACCUM_QUAD__SHIFT                                                              0x10
+#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT__SHIFT                                                             0x18
+#define VGT_TESS_DISTRIBUTION__TRAP_SPLIT__SHIFT                                                              0x1d
+#define VGT_TESS_DISTRIBUTION__ACCUM_ISOLINE_MASK                                                             0x000000FFL
+#define VGT_TESS_DISTRIBUTION__ACCUM_TRI_MASK                                                                 0x0000FF00L
+#define VGT_TESS_DISTRIBUTION__ACCUM_QUAD_MASK                                                                0x00FF0000L
+#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT_MASK                                                               0x1F000000L
+#define VGT_TESS_DISTRIBUTION__TRAP_SPLIT_MASK                                                                0xE0000000L
+//VGT_SHADER_STAGES_EN
+#define VGT_SHADER_STAGES_EN__LS_EN__SHIFT                                                                    0x0
+#define VGT_SHADER_STAGES_EN__HS_EN__SHIFT                                                                    0x2
+#define VGT_SHADER_STAGES_EN__ES_EN__SHIFT                                                                    0x3
+#define VGT_SHADER_STAGES_EN__GS_EN__SHIFT                                                                    0x5
+#define VGT_SHADER_STAGES_EN__VS_EN__SHIFT                                                                    0x6
+#define VGT_SHADER_STAGES_EN__DISPATCH_DRAW_EN__SHIFT                                                         0x9
+#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_0__SHIFT                                                      0xa
+#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_1__SHIFT                                                      0xb
+#define VGT_SHADER_STAGES_EN__VS_WAVE_ID_EN__SHIFT                                                            0xc
+#define VGT_SHADER_STAGES_EN__PRIMGEN_EN__SHIFT                                                               0xd
+#define VGT_SHADER_STAGES_EN__ORDERED_ID_MODE__SHIFT                                                          0xe
+#define VGT_SHADER_STAGES_EN__MAX_PRIMGRP_IN_WAVE__SHIFT                                                      0xf
+#define VGT_SHADER_STAGES_EN__GS_FAST_LAUNCH__SHIFT                                                           0x13
+#define VGT_SHADER_STAGES_EN__LS_EN_MASK                                                                      0x00000003L
+#define VGT_SHADER_STAGES_EN__HS_EN_MASK                                                                      0x00000004L
+#define VGT_SHADER_STAGES_EN__ES_EN_MASK                                                                      0x00000018L
+#define VGT_SHADER_STAGES_EN__GS_EN_MASK                                                                      0x00000020L
+#define VGT_SHADER_STAGES_EN__VS_EN_MASK                                                                      0x000000C0L
+#define VGT_SHADER_STAGES_EN__DISPATCH_DRAW_EN_MASK                                                           0x00000200L
+#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_0_MASK                                                        0x00000400L
+#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_1_MASK                                                        0x00000800L
+#define VGT_SHADER_STAGES_EN__VS_WAVE_ID_EN_MASK                                                              0x00001000L
+#define VGT_SHADER_STAGES_EN__PRIMGEN_EN_MASK                                                                 0x00002000L
+#define VGT_SHADER_STAGES_EN__ORDERED_ID_MODE_MASK                                                            0x00004000L
+#define VGT_SHADER_STAGES_EN__MAX_PRIMGRP_IN_WAVE_MASK                                                        0x00078000L
+#define VGT_SHADER_STAGES_EN__GS_FAST_LAUNCH_MASK                                                             0x00180000L
+//VGT_LS_HS_CONFIG
+#define VGT_LS_HS_CONFIG__NUM_PATCHES__SHIFT                                                                  0x0
+#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP__SHIFT                                                              0x8
+#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP__SHIFT                                                             0xe
+#define VGT_LS_HS_CONFIG__NUM_PATCHES_MASK                                                                    0x000000FFL
+#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP_MASK                                                                0x00003F00L
+#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP_MASK                                                               0x000FC000L
+//VGT_GS_VERT_ITEMSIZE
+#define VGT_GS_VERT_ITEMSIZE__ITEMSIZE__SHIFT                                                                 0x0
+#define VGT_GS_VERT_ITEMSIZE__ITEMSIZE_MASK                                                                   0x00007FFFL
+//VGT_GS_VERT_ITEMSIZE_1
+#define VGT_GS_VERT_ITEMSIZE_1__ITEMSIZE__SHIFT                                                               0x0
+#define VGT_GS_VERT_ITEMSIZE_1__ITEMSIZE_MASK                                                                 0x00007FFFL
+//VGT_GS_VERT_ITEMSIZE_2
+#define VGT_GS_VERT_ITEMSIZE_2__ITEMSIZE__SHIFT                                                               0x0
+#define VGT_GS_VERT_ITEMSIZE_2__ITEMSIZE_MASK                                                                 0x00007FFFL
+//VGT_GS_VERT_ITEMSIZE_3
+#define VGT_GS_VERT_ITEMSIZE_3__ITEMSIZE__SHIFT                                                               0x0
+#define VGT_GS_VERT_ITEMSIZE_3__ITEMSIZE_MASK                                                                 0x00007FFFL
+//VGT_TF_PARAM
+#define VGT_TF_PARAM__TYPE__SHIFT                                                                             0x0
+#define VGT_TF_PARAM__PARTITIONING__SHIFT                                                                     0x2
+#define VGT_TF_PARAM__TOPOLOGY__SHIFT                                                                         0x5
+#define VGT_TF_PARAM__RESERVED_REDUC_AXIS__SHIFT                                                              0x8
+#define VGT_TF_PARAM__DEPRECATED__SHIFT                                                                       0x9
+#define VGT_TF_PARAM__DISABLE_DONUTS__SHIFT                                                                   0xe
+#define VGT_TF_PARAM__RDREQ_POLICY__SHIFT                                                                     0xf
+#define VGT_TF_PARAM__DISTRIBUTION_MODE__SHIFT                                                                0x11
+#define VGT_TF_PARAM__TYPE_MASK                                                                               0x00000003L
+#define VGT_TF_PARAM__PARTITIONING_MASK                                                                       0x0000001CL
+#define VGT_TF_PARAM__TOPOLOGY_MASK                                                                           0x000000E0L
+#define VGT_TF_PARAM__RESERVED_REDUC_AXIS_MASK                                                                0x00000100L
+#define VGT_TF_PARAM__DEPRECATED_MASK                                                                         0x00000200L
+#define VGT_TF_PARAM__DISABLE_DONUTS_MASK                                                                     0x00004000L
+#define VGT_TF_PARAM__RDREQ_POLICY_MASK                                                                       0x00008000L
+#define VGT_TF_PARAM__DISTRIBUTION_MODE_MASK                                                                  0x00060000L
+//DB_ALPHA_TO_MASK
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE__SHIFT                                                         0x0
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0__SHIFT                                                        0x8
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1__SHIFT                                                        0xa
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2__SHIFT                                                        0xc
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3__SHIFT                                                        0xe
+#define DB_ALPHA_TO_MASK__OFFSET_ROUND__SHIFT                                                                 0x10
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE_MASK                                                           0x00000001L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0_MASK                                                          0x00000300L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1_MASK                                                          0x00000C00L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2_MASK                                                          0x00003000L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3_MASK                                                          0x0000C000L
+#define DB_ALPHA_TO_MASK__OFFSET_ROUND_MASK                                                                   0x00010000L
+//VGT_DISPATCH_DRAW_INDEX
+#define VGT_DISPATCH_DRAW_INDEX__MATCH_INDEX__SHIFT                                                           0x0
+#define VGT_DISPATCH_DRAW_INDEX__MATCH_INDEX_MASK                                                             0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_DB_FMT_CNTL
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS__SHIFT                                     0x0
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT__SHIFT                                     0x8
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS_MASK                                       0x000000FFL
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT_MASK                                       0x00000100L
+//PA_SU_POLY_OFFSET_CLAMP
+#define PA_SU_POLY_OFFSET_CLAMP__CLAMP__SHIFT                                                                 0x0
+#define PA_SU_POLY_OFFSET_CLAMP__CLAMP_MASK                                                                   0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_FRONT_SCALE
+#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE__SHIFT                                                           0x0
+#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE_MASK                                                             0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_FRONT_OFFSET
+#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET__SHIFT                                                         0x0
+#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET_MASK                                                           0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_BACK_SCALE
+#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE__SHIFT                                                            0x0
+#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE_MASK                                                              0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_BACK_OFFSET
+#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET__SHIFT                                                          0x0
+#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET_MASK                                                            0xFFFFFFFFL
+//VGT_GS_INSTANCE_CNT
+#define VGT_GS_INSTANCE_CNT__ENABLE__SHIFT                                                                    0x0
+#define VGT_GS_INSTANCE_CNT__CNT__SHIFT                                                                       0x2
+#define VGT_GS_INSTANCE_CNT__ENABLE_MASK                                                                      0x00000001L
+#define VGT_GS_INSTANCE_CNT__CNT_MASK                                                                         0x000001FCL
+//VGT_STRMOUT_CONFIG
+#define VGT_STRMOUT_CONFIG__STREAMOUT_0_EN__SHIFT                                                             0x0
+#define VGT_STRMOUT_CONFIG__STREAMOUT_1_EN__SHIFT                                                             0x1
+#define VGT_STRMOUT_CONFIG__STREAMOUT_2_EN__SHIFT                                                             0x2
+#define VGT_STRMOUT_CONFIG__STREAMOUT_3_EN__SHIFT                                                             0x3
+#define VGT_STRMOUT_CONFIG__RAST_STREAM__SHIFT                                                                0x4
+#define VGT_STRMOUT_CONFIG__EN_PRIMS_NEEDED_CNT__SHIFT                                                        0x7
+#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK__SHIFT                                                           0x8
+#define VGT_STRMOUT_CONFIG__USE_RAST_STREAM_MASK__SHIFT                                                       0x1f
+#define VGT_STRMOUT_CONFIG__STREAMOUT_0_EN_MASK                                                               0x00000001L
+#define VGT_STRMOUT_CONFIG__STREAMOUT_1_EN_MASK                                                               0x00000002L
+#define VGT_STRMOUT_CONFIG__STREAMOUT_2_EN_MASK                                                               0x00000004L
+#define VGT_STRMOUT_CONFIG__STREAMOUT_3_EN_MASK                                                               0x00000008L
+#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK                                                                  0x00000070L
+#define VGT_STRMOUT_CONFIG__EN_PRIMS_NEEDED_CNT_MASK                                                          0x00000080L
+#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK_MASK                                                             0x00000F00L
+#define VGT_STRMOUT_CONFIG__USE_RAST_STREAM_MASK_MASK                                                         0x80000000L
+//VGT_STRMOUT_BUFFER_CONFIG
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_0_BUFFER_EN__SHIFT                                                  0x0
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_1_BUFFER_EN__SHIFT                                                  0x4
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_2_BUFFER_EN__SHIFT                                                  0x8
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_3_BUFFER_EN__SHIFT                                                  0xc
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_0_BUFFER_EN_MASK                                                    0x0000000FL
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_1_BUFFER_EN_MASK                                                    0x000000F0L
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_2_BUFFER_EN_MASK                                                    0x00000F00L
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_3_BUFFER_EN_MASK                                                    0x0000F000L
+//VGT_DMA_EVENT_INITIATOR
+#define VGT_DMA_EVENT_INITIATOR__EVENT_TYPE__SHIFT                                                            0x0
+#define VGT_DMA_EVENT_INITIATOR__ADDRESS_HI__SHIFT                                                            0xa
+#define VGT_DMA_EVENT_INITIATOR__EXTENDED_EVENT__SHIFT                                                        0x1b
+#define VGT_DMA_EVENT_INITIATOR__EVENT_TYPE_MASK                                                              0x0000003FL
+#define VGT_DMA_EVENT_INITIATOR__ADDRESS_HI_MASK                                                              0x07FFFC00L
+#define VGT_DMA_EVENT_INITIATOR__EXTENDED_EVENT_MASK                                                          0x08000000L
+//PA_SC_CENTROID_PRIORITY_0
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0__SHIFT                                                          0x0
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1__SHIFT                                                          0x4
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2__SHIFT                                                          0x8
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3__SHIFT                                                          0xc
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4__SHIFT                                                          0x10
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5__SHIFT                                                          0x14
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6__SHIFT                                                          0x18
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7__SHIFT                                                          0x1c
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0_MASK                                                            0x0000000FL
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1_MASK                                                            0x000000F0L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2_MASK                                                            0x00000F00L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3_MASK                                                            0x0000F000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4_MASK                                                            0x000F0000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5_MASK                                                            0x00F00000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6_MASK                                                            0x0F000000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7_MASK                                                            0xF0000000L
+//PA_SC_CENTROID_PRIORITY_1
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8__SHIFT                                                          0x0
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9__SHIFT                                                          0x4
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10__SHIFT                                                         0x8
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11__SHIFT                                                         0xc
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12__SHIFT                                                         0x10
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13__SHIFT                                                         0x14
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14__SHIFT                                                         0x18
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15__SHIFT                                                         0x1c
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8_MASK                                                            0x0000000FL
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9_MASK                                                            0x000000F0L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10_MASK                                                           0x00000F00L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11_MASK                                                           0x0000F000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12_MASK                                                           0x000F0000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13_MASK                                                           0x00F00000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14_MASK                                                           0x0F000000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15_MASK                                                           0xF0000000L
+//PA_SC_LINE_CNTL
+#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH__SHIFT                                                             0x9
+#define PA_SC_LINE_CNTL__LAST_PIXEL__SHIFT                                                                    0xa
+#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA__SHIFT                                                      0xb
+#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA__SHIFT                                                         0xc
+#define PA_SC_LINE_CNTL__EXTRA_DX_DY_PRECISION__SHIFT                                                         0xd
+#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH_MASK                                                               0x00000200L
+#define PA_SC_LINE_CNTL__LAST_PIXEL_MASK                                                                      0x00000400L
+#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA_MASK                                                        0x00000800L
+#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA_MASK                                                           0x00001000L
+#define PA_SC_LINE_CNTL__EXTRA_DX_DY_PRECISION_MASK                                                           0x00002000L
+//PA_SC_AA_CONFIG
+#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES__SHIFT                                                              0x0
+#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN__SHIFT                                                         0x4
+#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST__SHIFT                                                               0xd
+#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES__SHIFT                                                          0x14
+#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE__SHIFT                                                        0x18
+#define PA_SC_AA_CONFIG__COVERAGE_TO_SHADER_SELECT__SHIFT                                                     0x1a
+#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES_MASK                                                                0x00000007L
+#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN_MASK                                                           0x00000010L
+#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST_MASK                                                                 0x0001E000L
+#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES_MASK                                                            0x00700000L
+#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE_MASK                                                          0x03000000L
+#define PA_SC_AA_CONFIG__COVERAGE_TO_SHADER_SELECT_MASK                                                       0x0C000000L
+//PA_SU_VTX_CNTL
+#define PA_SU_VTX_CNTL__PIX_CENTER__SHIFT                                                                     0x0
+#define PA_SU_VTX_CNTL__ROUND_MODE__SHIFT                                                                     0x1
+#define PA_SU_VTX_CNTL__QUANT_MODE__SHIFT                                                                     0x3
+#define PA_SU_VTX_CNTL__PIX_CENTER_MASK                                                                       0x00000001L
+#define PA_SU_VTX_CNTL__ROUND_MODE_MASK                                                                       0x00000006L
+#define PA_SU_VTX_CNTL__QUANT_MODE_MASK                                                                       0x00000038L
+//PA_CL_GB_VERT_CLIP_ADJ
+#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_CL_GB_VERT_DISC_ADJ
+#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_CL_GB_HORZ_CLIP_ADJ
+#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_CL_GB_HORZ_DISC_ADJ
+#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_MASK_X0Y0_X1Y0
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0__SHIFT                                                          0x0
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0__SHIFT                                                          0x10
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0_MASK                                                            0x0000FFFFL
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0_MASK                                                            0xFFFF0000L
+//PA_SC_AA_MASK_X0Y1_X1Y1
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1__SHIFT                                                          0x0
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1__SHIFT                                                          0x10
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1_MASK                                                            0x0000FFFFL
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1_MASK                                                            0xFFFF0000L
+//PA_SC_SHADER_CONTROL
+#define PA_SC_SHADER_CONTROL__REALIGN_DQUADS_AFTER_N_WAVES__SHIFT                                             0x0
+#define PA_SC_SHADER_CONTROL__LOAD_COLLISION_WAVEID__SHIFT                                                    0x2
+#define PA_SC_SHADER_CONTROL__LOAD_INTRAWAVE_COLLISION__SHIFT                                                 0x3
+#define PA_SC_SHADER_CONTROL__REALIGN_DQUADS_AFTER_N_WAVES_MASK                                               0x00000003L
+#define PA_SC_SHADER_CONTROL__LOAD_COLLISION_WAVEID_MASK                                                      0x00000004L
+#define PA_SC_SHADER_CONTROL__LOAD_INTRAWAVE_COLLISION_MASK                                                   0x00000008L
+//PA_SC_BINNER_CNTL_0
+#define PA_SC_BINNER_CNTL_0__BINNING_MODE__SHIFT                                                              0x0
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X__SHIFT                                                                0x2
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y__SHIFT                                                                0x3
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_EXTEND__SHIFT                                                         0x4
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_EXTEND__SHIFT                                                         0x7
+#define PA_SC_BINNER_CNTL_0__CONTEXT_STATES_PER_BIN__SHIFT                                                    0xa
+#define PA_SC_BINNER_CNTL_0__PERSISTENT_STATES_PER_BIN__SHIFT                                                 0xd
+#define PA_SC_BINNER_CNTL_0__DISABLE_START_OF_PRIM__SHIFT                                                     0x12
+#define PA_SC_BINNER_CNTL_0__FPOVS_PER_BATCH__SHIFT                                                           0x13
+#define PA_SC_BINNER_CNTL_0__OPTIMAL_BIN_SELECTION__SHIFT                                                     0x1b
+#define PA_SC_BINNER_CNTL_0__FLUSH_ON_BINNING_TRANSITION__SHIFT                                               0x1c
+#define PA_SC_BINNER_CNTL_0__BINNING_MODE_MASK                                                                0x00000003L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_MASK                                                                  0x00000004L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_MASK                                                                  0x00000008L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_EXTEND_MASK                                                           0x00000070L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_EXTEND_MASK                                                           0x00000380L
+#define PA_SC_BINNER_CNTL_0__CONTEXT_STATES_PER_BIN_MASK                                                      0x00001C00L
+#define PA_SC_BINNER_CNTL_0__PERSISTENT_STATES_PER_BIN_MASK                                                   0x0003E000L
+#define PA_SC_BINNER_CNTL_0__DISABLE_START_OF_PRIM_MASK                                                       0x00040000L
+#define PA_SC_BINNER_CNTL_0__FPOVS_PER_BATCH_MASK                                                             0x07F80000L
+#define PA_SC_BINNER_CNTL_0__OPTIMAL_BIN_SELECTION_MASK                                                       0x08000000L
+#define PA_SC_BINNER_CNTL_0__FLUSH_ON_BINNING_TRANSITION_MASK                                                 0x10000000L
+//PA_SC_BINNER_CNTL_1
+#define PA_SC_BINNER_CNTL_1__MAX_ALLOC_COUNT__SHIFT                                                           0x0
+#define PA_SC_BINNER_CNTL_1__MAX_PRIM_PER_BATCH__SHIFT                                                        0x10
+#define PA_SC_BINNER_CNTL_1__MAX_ALLOC_COUNT_MASK                                                             0x0000FFFFL
+#define PA_SC_BINNER_CNTL_1__MAX_PRIM_PER_BATCH_MASK                                                          0xFFFF0000L
+//PA_SC_CONSERVATIVE_RASTERIZATION_CNTL
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_ENABLE__SHIFT                                        0x0
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_SAMPLE_SELECT__SHIFT                                 0x1
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_ENABLE__SHIFT                                       0x5
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_SAMPLE_SELECT__SHIFT                                0x6
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PBB_UNCERTAINTY_REGION_ENABLE__SHIFT                           0xa
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_EXTENT__SHIFT                                          0xb
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_OFFSET__SHIFT                                          0xc
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_OVER_RAST_INNER_TO_NORMAL__SHIFT                      0xd
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_UNDER_RAST_INNER_TO_NORMAL__SHIFT                     0xe
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__DEGENERATE_OVERRIDE_INNER_TO_NORMAL_DISABLE__SHIFT             0xf
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MODE__SHIFT                                 0x10
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OUTER_UNCERTAINTY_EDGERULE_OVERRIDE__SHIFT                     0x12
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__INNER_UNCERTAINTY_EDGERULE_OVERRIDE__SHIFT                     0x13
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__NULL_SQUAD_AA_MASK_ENABLE__SHIFT                               0x14
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__COVERAGE_AA_MASK_ENABLE__SHIFT                                 0x15
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PREZ_AA_MASK_ENABLE__SHIFT                                     0x16
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__POSTZ_AA_MASK_ENABLE__SHIFT                                    0x17
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__CENTROID_SAMPLE_OVERRIDE__SHIFT                                0x18
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_ENABLE_MASK                                          0x00000001L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_SAMPLE_SELECT_MASK                                   0x0000001EL
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_ENABLE_MASK                                         0x00000020L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_SAMPLE_SELECT_MASK                                  0x000003C0L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PBB_UNCERTAINTY_REGION_ENABLE_MASK                             0x00000400L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_EXTENT_MASK                                            0x00000800L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_OFFSET_MASK                                            0x00001000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_OVER_RAST_INNER_TO_NORMAL_MASK                        0x00002000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_UNDER_RAST_INNER_TO_NORMAL_MASK                       0x00004000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__DEGENERATE_OVERRIDE_INNER_TO_NORMAL_DISABLE_MASK               0x00008000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MODE_MASK                                   0x00030000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OUTER_UNCERTAINTY_EDGERULE_OVERRIDE_MASK                       0x00040000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__INNER_UNCERTAINTY_EDGERULE_OVERRIDE_MASK                       0x00080000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__NULL_SQUAD_AA_MASK_ENABLE_MASK                                 0x00100000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__COVERAGE_AA_MASK_ENABLE_MASK                                   0x00200000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PREZ_AA_MASK_ENABLE_MASK                                       0x00400000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__POSTZ_AA_MASK_ENABLE_MASK                                      0x00800000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__CENTROID_SAMPLE_OVERRIDE_MASK                                  0x01000000L
+//PA_SC_NGG_MODE_CNTL
+#define PA_SC_NGG_MODE_CNTL__MAX_DEALLOCS_IN_WAVE__SHIFT                                                      0x0
+#define PA_SC_NGG_MODE_CNTL__MAX_DEALLOCS_IN_WAVE_MASK                                                        0x000007FFL
+//VGT_VERTEX_REUSE_BLOCK_CNTL
+#define VGT_VERTEX_REUSE_BLOCK_CNTL__VTX_REUSE_DEPTH__SHIFT                                                   0x0
+#define VGT_VERTEX_REUSE_BLOCK_CNTL__VTX_REUSE_DEPTH_MASK                                                     0x000000FFL
+//VGT_OUT_DEALLOC_CNTL
+#define VGT_OUT_DEALLOC_CNTL__DEALLOC_DIST__SHIFT                                                             0x0
+#define VGT_OUT_DEALLOC_CNTL__DEALLOC_DIST_MASK                                                               0x0000007FL
+//CB_COLOR0_BASE
+#define CB_COLOR0_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR0_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR0_BASE_EXT
+#define CB_COLOR0_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR0_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR0_ATTRIB2
+#define CB_COLOR0_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR0_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR0_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR0_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR0_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR0_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR0_VIEW
+#define CB_COLOR0_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR0_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR0_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
+#define CB_COLOR0_VIEW__SLICE_START_MASK                                                                      0x000007FFL
+#define CB_COLOR0_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
+#define CB_COLOR0_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
+//CB_COLOR0_INFO
+#define CB_COLOR0_INFO__ENDIAN__SHIFT                                                                         0x0
+#define CB_COLOR0_INFO__FORMAT__SHIFT                                                                         0x2
+#define CB_COLOR0_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR0_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR0_INFO__FAST_CLEAR__SHIFT                                                                     0xd
+#define CB_COLOR0_INFO__COMPRESSION__SHIFT                                                                    0xe
+#define CB_COLOR0_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR0_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR0_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR0_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR0_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
+#define CB_COLOR0_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
+#define CB_COLOR0_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
+#define CB_COLOR0_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
+#define CB_COLOR0_INFO__ENDIAN_MASK                                                                           0x00000003L
+#define CB_COLOR0_INFO__FORMAT_MASK                                                                           0x0000007CL
+#define CB_COLOR0_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR0_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR0_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
+#define CB_COLOR0_INFO__COMPRESSION_MASK                                                                      0x00004000L
+#define CB_COLOR0_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR0_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR0_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR0_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+#define CB_COLOR0_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
+#define CB_COLOR0_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
+#define CB_COLOR0_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
+#define CB_COLOR0_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
+//CB_COLOR0_ATTRIB
+#define CB_COLOR0_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
+#define CB_COLOR0_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
+#define CB_COLOR0_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
+#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
+#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
+#define CB_COLOR0_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
+#define CB_COLOR0_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
+#define CB_COLOR0_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
+#define CB_COLOR0_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
+#define CB_COLOR0_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
+#define CB_COLOR0_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
+#define CB_COLOR0_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
+#define CB_COLOR0_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
+#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
+#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
+#define CB_COLOR0_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
+#define CB_COLOR0_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
+#define CB_COLOR0_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
+#define CB_COLOR0_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
+#define CB_COLOR0_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
+//CB_COLOR0_DCC_CONTROL
+#define CB_COLOR0_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
+#define CB_COLOR0_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
+#define CB_COLOR0_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
+#define CB_COLOR0_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
+#define CB_COLOR0_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
+#define CB_COLOR0_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
+#define CB_COLOR0_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
+#define CB_COLOR0_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
+#define CB_COLOR0_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
+#define CB_COLOR0_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                             0x12
+#define CB_COLOR0_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                        0x13
+#define CB_COLOR0_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
+#define CB_COLOR0_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define CB_COLOR0_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
+#define CB_COLOR0_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
+#define CB_COLOR0_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
+#define CB_COLOR0_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
+#define CB_COLOR0_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
+#define CB_COLOR0_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
+#define CB_COLOR0_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
+#define CB_COLOR0_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                               0x00040000L
+#define CB_COLOR0_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                          0x00080000L
+//CB_COLOR0_CMASK
+#define CB_COLOR0_CMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR0_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR0_CMASK_BASE_EXT
+#define CB_COLOR0_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR0_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR0_FMASK
+#define CB_COLOR0_FMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR0_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR0_FMASK_BASE_EXT
+#define CB_COLOR0_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR0_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR0_CLEAR_WORD0
+#define CB_COLOR0_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
+#define CB_COLOR0_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
+//CB_COLOR0_CLEAR_WORD1
+#define CB_COLOR0_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
+#define CB_COLOR0_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
+//CB_COLOR0_DCC_BASE
+#define CB_COLOR0_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR0_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR0_DCC_BASE_EXT
+#define CB_COLOR0_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR0_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR1_BASE
+#define CB_COLOR1_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR1_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR1_BASE_EXT
+#define CB_COLOR1_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR1_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR1_ATTRIB2
+#define CB_COLOR1_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR1_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR1_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR1_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR1_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR1_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR1_VIEW
+#define CB_COLOR1_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR1_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR1_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
+#define CB_COLOR1_VIEW__SLICE_START_MASK                                                                      0x000007FFL
+#define CB_COLOR1_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
+#define CB_COLOR1_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
+//CB_COLOR1_INFO
+#define CB_COLOR1_INFO__ENDIAN__SHIFT                                                                         0x0
+#define CB_COLOR1_INFO__FORMAT__SHIFT                                                                         0x2
+#define CB_COLOR1_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR1_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR1_INFO__FAST_CLEAR__SHIFT                                                                     0xd
+#define CB_COLOR1_INFO__COMPRESSION__SHIFT                                                                    0xe
+#define CB_COLOR1_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR1_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR1_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR1_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR1_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
+#define CB_COLOR1_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
+#define CB_COLOR1_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
+#define CB_COLOR1_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
+#define CB_COLOR1_INFO__ENDIAN_MASK                                                                           0x00000003L
+#define CB_COLOR1_INFO__FORMAT_MASK                                                                           0x0000007CL
+#define CB_COLOR1_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR1_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR1_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
+#define CB_COLOR1_INFO__COMPRESSION_MASK                                                                      0x00004000L
+#define CB_COLOR1_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR1_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR1_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR1_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+#define CB_COLOR1_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
+#define CB_COLOR1_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
+#define CB_COLOR1_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
+#define CB_COLOR1_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
+//CB_COLOR1_ATTRIB
+#define CB_COLOR1_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
+#define CB_COLOR1_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
+#define CB_COLOR1_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
+#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
+#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
+#define CB_COLOR1_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
+#define CB_COLOR1_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
+#define CB_COLOR1_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
+#define CB_COLOR1_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
+#define CB_COLOR1_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
+#define CB_COLOR1_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
+#define CB_COLOR1_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
+#define CB_COLOR1_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
+#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
+#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
+#define CB_COLOR1_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
+#define CB_COLOR1_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
+#define CB_COLOR1_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
+#define CB_COLOR1_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
+#define CB_COLOR1_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
+//CB_COLOR1_DCC_CONTROL
+#define CB_COLOR1_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
+#define CB_COLOR1_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
+#define CB_COLOR1_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
+#define CB_COLOR1_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
+#define CB_COLOR1_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
+#define CB_COLOR1_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
+#define CB_COLOR1_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
+#define CB_COLOR1_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
+#define CB_COLOR1_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
+#define CB_COLOR1_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                             0x12
+#define CB_COLOR1_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                        0x13
+#define CB_COLOR1_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
+#define CB_COLOR1_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define CB_COLOR1_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
+#define CB_COLOR1_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
+#define CB_COLOR1_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
+#define CB_COLOR1_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
+#define CB_COLOR1_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
+#define CB_COLOR1_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
+#define CB_COLOR1_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
+#define CB_COLOR1_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                               0x00040000L
+#define CB_COLOR1_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                          0x00080000L
+//CB_COLOR1_CMASK
+#define CB_COLOR1_CMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR1_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR1_CMASK_BASE_EXT
+#define CB_COLOR1_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR1_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR1_FMASK
+#define CB_COLOR1_FMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR1_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR1_FMASK_BASE_EXT
+#define CB_COLOR1_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR1_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR1_CLEAR_WORD0
+#define CB_COLOR1_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
+#define CB_COLOR1_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
+//CB_COLOR1_CLEAR_WORD1
+#define CB_COLOR1_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
+#define CB_COLOR1_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
+//CB_COLOR1_DCC_BASE
+#define CB_COLOR1_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR1_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR1_DCC_BASE_EXT
+#define CB_COLOR1_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR1_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR2_BASE
+#define CB_COLOR2_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR2_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR2_BASE_EXT
+#define CB_COLOR2_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR2_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR2_ATTRIB2
+#define CB_COLOR2_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR2_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR2_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR2_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR2_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR2_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR2_VIEW
+#define CB_COLOR2_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR2_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR2_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
+#define CB_COLOR2_VIEW__SLICE_START_MASK                                                                      0x000007FFL
+#define CB_COLOR2_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
+#define CB_COLOR2_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
+//CB_COLOR2_INFO
+#define CB_COLOR2_INFO__ENDIAN__SHIFT                                                                         0x0
+#define CB_COLOR2_INFO__FORMAT__SHIFT                                                                         0x2
+#define CB_COLOR2_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR2_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR2_INFO__FAST_CLEAR__SHIFT                                                                     0xd
+#define CB_COLOR2_INFO__COMPRESSION__SHIFT                                                                    0xe
+#define CB_COLOR2_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR2_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR2_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR2_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR2_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
+#define CB_COLOR2_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
+#define CB_COLOR2_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
+#define CB_COLOR2_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
+#define CB_COLOR2_INFO__ENDIAN_MASK                                                                           0x00000003L
+#define CB_COLOR2_INFO__FORMAT_MASK                                                                           0x0000007CL
+#define CB_COLOR2_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR2_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR2_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
+#define CB_COLOR2_INFO__COMPRESSION_MASK                                                                      0x00004000L
+#define CB_COLOR2_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR2_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR2_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR2_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+#define CB_COLOR2_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
+#define CB_COLOR2_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
+#define CB_COLOR2_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
+#define CB_COLOR2_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
+//CB_COLOR2_ATTRIB
+#define CB_COLOR2_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
+#define CB_COLOR2_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
+#define CB_COLOR2_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
+#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
+#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
+#define CB_COLOR2_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
+#define CB_COLOR2_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
+#define CB_COLOR2_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
+#define CB_COLOR2_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
+#define CB_COLOR2_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
+#define CB_COLOR2_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
+#define CB_COLOR2_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
+#define CB_COLOR2_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
+#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
+#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
+#define CB_COLOR2_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
+#define CB_COLOR2_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
+#define CB_COLOR2_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
+#define CB_COLOR2_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
+#define CB_COLOR2_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
+//CB_COLOR2_DCC_CONTROL
+#define CB_COLOR2_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
+#define CB_COLOR2_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
+#define CB_COLOR2_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
+#define CB_COLOR2_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
+#define CB_COLOR2_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
+#define CB_COLOR2_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
+#define CB_COLOR2_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
+#define CB_COLOR2_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
+#define CB_COLOR2_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
+#define CB_COLOR2_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                             0x12
+#define CB_COLOR2_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                        0x13
+#define CB_COLOR2_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
+#define CB_COLOR2_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define CB_COLOR2_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
+#define CB_COLOR2_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
+#define CB_COLOR2_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
+#define CB_COLOR2_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
+#define CB_COLOR2_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
+#define CB_COLOR2_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
+#define CB_COLOR2_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
+#define CB_COLOR2_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                               0x00040000L
+#define CB_COLOR2_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                          0x00080000L
+//CB_COLOR2_CMASK
+#define CB_COLOR2_CMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR2_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR2_CMASK_BASE_EXT
+#define CB_COLOR2_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR2_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR2_FMASK
+#define CB_COLOR2_FMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR2_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR2_FMASK_BASE_EXT
+#define CB_COLOR2_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR2_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR2_CLEAR_WORD0
+#define CB_COLOR2_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
+#define CB_COLOR2_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
+//CB_COLOR2_CLEAR_WORD1
+#define CB_COLOR2_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
+#define CB_COLOR2_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
+//CB_COLOR2_DCC_BASE
+#define CB_COLOR2_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR2_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR2_DCC_BASE_EXT
+#define CB_COLOR2_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR2_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR3_BASE
+#define CB_COLOR3_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR3_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR3_BASE_EXT
+#define CB_COLOR3_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR3_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR3_ATTRIB2
+#define CB_COLOR3_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR3_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR3_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR3_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR3_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR3_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR3_VIEW
+#define CB_COLOR3_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR3_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR3_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
+#define CB_COLOR3_VIEW__SLICE_START_MASK                                                                      0x000007FFL
+#define CB_COLOR3_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
+#define CB_COLOR3_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
+//CB_COLOR3_INFO
+#define CB_COLOR3_INFO__ENDIAN__SHIFT                                                                         0x0
+#define CB_COLOR3_INFO__FORMAT__SHIFT                                                                         0x2
+#define CB_COLOR3_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR3_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR3_INFO__FAST_CLEAR__SHIFT                                                                     0xd
+#define CB_COLOR3_INFO__COMPRESSION__SHIFT                                                                    0xe
+#define CB_COLOR3_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR3_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR3_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR3_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR3_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
+#define CB_COLOR3_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
+#define CB_COLOR3_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
+#define CB_COLOR3_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
+#define CB_COLOR3_INFO__ENDIAN_MASK                                                                           0x00000003L
+#define CB_COLOR3_INFO__FORMAT_MASK                                                                           0x0000007CL
+#define CB_COLOR3_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR3_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR3_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
+#define CB_COLOR3_INFO__COMPRESSION_MASK                                                                      0x00004000L
+#define CB_COLOR3_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR3_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR3_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR3_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+#define CB_COLOR3_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
+#define CB_COLOR3_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
+#define CB_COLOR3_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
+#define CB_COLOR3_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
+//CB_COLOR3_ATTRIB
+#define CB_COLOR3_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
+#define CB_COLOR3_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
+#define CB_COLOR3_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
+#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
+#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
+#define CB_COLOR3_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
+#define CB_COLOR3_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
+#define CB_COLOR3_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
+#define CB_COLOR3_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
+#define CB_COLOR3_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
+#define CB_COLOR3_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
+#define CB_COLOR3_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
+#define CB_COLOR3_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
+#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
+#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
+#define CB_COLOR3_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
+#define CB_COLOR3_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
+#define CB_COLOR3_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
+#define CB_COLOR3_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
+#define CB_COLOR3_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
+//CB_COLOR3_DCC_CONTROL
+#define CB_COLOR3_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
+#define CB_COLOR3_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
+#define CB_COLOR3_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
+#define CB_COLOR3_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
+#define CB_COLOR3_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
+#define CB_COLOR3_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
+#define CB_COLOR3_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
+#define CB_COLOR3_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
+#define CB_COLOR3_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
+#define CB_COLOR3_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                             0x12
+#define CB_COLOR3_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                        0x13
+#define CB_COLOR3_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
+#define CB_COLOR3_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define CB_COLOR3_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
+#define CB_COLOR3_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
+#define CB_COLOR3_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
+#define CB_COLOR3_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
+#define CB_COLOR3_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
+#define CB_COLOR3_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
+#define CB_COLOR3_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
+#define CB_COLOR3_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                               0x00040000L
+#define CB_COLOR3_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                          0x00080000L
+//CB_COLOR3_CMASK
+#define CB_COLOR3_CMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR3_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR3_CMASK_BASE_EXT
+#define CB_COLOR3_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR3_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR3_FMASK
+#define CB_COLOR3_FMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR3_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR3_FMASK_BASE_EXT
+#define CB_COLOR3_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR3_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR3_CLEAR_WORD0
+#define CB_COLOR3_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
+#define CB_COLOR3_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
+//CB_COLOR3_CLEAR_WORD1
+#define CB_COLOR3_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
+#define CB_COLOR3_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
+//CB_COLOR3_DCC_BASE
+#define CB_COLOR3_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR3_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR3_DCC_BASE_EXT
+#define CB_COLOR3_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR3_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR4_BASE
+#define CB_COLOR4_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR4_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR4_BASE_EXT
+#define CB_COLOR4_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR4_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR4_ATTRIB2
+#define CB_COLOR4_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR4_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR4_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR4_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR4_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR4_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR4_VIEW
+#define CB_COLOR4_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR4_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR4_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
+#define CB_COLOR4_VIEW__SLICE_START_MASK                                                                      0x000007FFL
+#define CB_COLOR4_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
+#define CB_COLOR4_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
+//CB_COLOR4_INFO
+#define CB_COLOR4_INFO__ENDIAN__SHIFT                                                                         0x0
+#define CB_COLOR4_INFO__FORMAT__SHIFT                                                                         0x2
+#define CB_COLOR4_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR4_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR4_INFO__FAST_CLEAR__SHIFT                                                                     0xd
+#define CB_COLOR4_INFO__COMPRESSION__SHIFT                                                                    0xe
+#define CB_COLOR4_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR4_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR4_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR4_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR4_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
+#define CB_COLOR4_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
+#define CB_COLOR4_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
+#define CB_COLOR4_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
+#define CB_COLOR4_INFO__ENDIAN_MASK                                                                           0x00000003L
+#define CB_COLOR4_INFO__FORMAT_MASK                                                                           0x0000007CL
+#define CB_COLOR4_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR4_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR4_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
+#define CB_COLOR4_INFO__COMPRESSION_MASK                                                                      0x00004000L
+#define CB_COLOR4_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR4_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR4_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR4_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+#define CB_COLOR4_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
+#define CB_COLOR4_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
+#define CB_COLOR4_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
+#define CB_COLOR4_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
+//CB_COLOR4_ATTRIB
+#define CB_COLOR4_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
+#define CB_COLOR4_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
+#define CB_COLOR4_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
+#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
+#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
+#define CB_COLOR4_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
+#define CB_COLOR4_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
+#define CB_COLOR4_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
+#define CB_COLOR4_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
+#define CB_COLOR4_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
+#define CB_COLOR4_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
+#define CB_COLOR4_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
+#define CB_COLOR4_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
+#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
+#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
+#define CB_COLOR4_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
+#define CB_COLOR4_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
+#define CB_COLOR4_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
+#define CB_COLOR4_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
+#define CB_COLOR4_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
+//CB_COLOR4_DCC_CONTROL
+#define CB_COLOR4_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
+#define CB_COLOR4_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
+#define CB_COLOR4_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
+#define CB_COLOR4_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
+#define CB_COLOR4_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
+#define CB_COLOR4_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
+#define CB_COLOR4_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
+#define CB_COLOR4_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
+#define CB_COLOR4_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
+#define CB_COLOR4_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                             0x12
+#define CB_COLOR4_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                        0x13
+#define CB_COLOR4_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
+#define CB_COLOR4_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define CB_COLOR4_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
+#define CB_COLOR4_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
+#define CB_COLOR4_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
+#define CB_COLOR4_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
+#define CB_COLOR4_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
+#define CB_COLOR4_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
+#define CB_COLOR4_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
+#define CB_COLOR4_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                               0x00040000L
+#define CB_COLOR4_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                          0x00080000L
+//CB_COLOR4_CMASK
+#define CB_COLOR4_CMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR4_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR4_CMASK_BASE_EXT
+#define CB_COLOR4_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR4_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR4_FMASK
+#define CB_COLOR4_FMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR4_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR4_FMASK_BASE_EXT
+#define CB_COLOR4_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR4_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR4_CLEAR_WORD0
+#define CB_COLOR4_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
+#define CB_COLOR4_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
+//CB_COLOR4_CLEAR_WORD1
+#define CB_COLOR4_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
+#define CB_COLOR4_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
+//CB_COLOR4_DCC_BASE
+#define CB_COLOR4_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR4_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR4_DCC_BASE_EXT
+#define CB_COLOR4_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR4_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR5_BASE
+#define CB_COLOR5_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR5_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR5_BASE_EXT
+#define CB_COLOR5_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR5_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR5_ATTRIB2
+#define CB_COLOR5_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR5_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR5_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR5_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR5_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR5_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR5_VIEW
+#define CB_COLOR5_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR5_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR5_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
+#define CB_COLOR5_VIEW__SLICE_START_MASK                                                                      0x000007FFL
+#define CB_COLOR5_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
+#define CB_COLOR5_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
+//CB_COLOR5_INFO
+#define CB_COLOR5_INFO__ENDIAN__SHIFT                                                                         0x0
+#define CB_COLOR5_INFO__FORMAT__SHIFT                                                                         0x2
+#define CB_COLOR5_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR5_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR5_INFO__FAST_CLEAR__SHIFT                                                                     0xd
+#define CB_COLOR5_INFO__COMPRESSION__SHIFT                                                                    0xe
+#define CB_COLOR5_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR5_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR5_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR5_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR5_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
+#define CB_COLOR5_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
+#define CB_COLOR5_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
+#define CB_COLOR5_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
+#define CB_COLOR5_INFO__ENDIAN_MASK                                                                           0x00000003L
+#define CB_COLOR5_INFO__FORMAT_MASK                                                                           0x0000007CL
+#define CB_COLOR5_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR5_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR5_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
+#define CB_COLOR5_INFO__COMPRESSION_MASK                                                                      0x00004000L
+#define CB_COLOR5_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR5_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR5_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR5_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+#define CB_COLOR5_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
+#define CB_COLOR5_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
+#define CB_COLOR5_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
+#define CB_COLOR5_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
+//CB_COLOR5_ATTRIB
+#define CB_COLOR5_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
+#define CB_COLOR5_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
+#define CB_COLOR5_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
+#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
+#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
+#define CB_COLOR5_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
+#define CB_COLOR5_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
+#define CB_COLOR5_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
+#define CB_COLOR5_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
+#define CB_COLOR5_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
+#define CB_COLOR5_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
+#define CB_COLOR5_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
+#define CB_COLOR5_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
+#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
+#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
+#define CB_COLOR5_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
+#define CB_COLOR5_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
+#define CB_COLOR5_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
+#define CB_COLOR5_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
+#define CB_COLOR5_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
+//CB_COLOR5_DCC_CONTROL
+#define CB_COLOR5_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
+#define CB_COLOR5_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
+#define CB_COLOR5_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
+#define CB_COLOR5_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
+#define CB_COLOR5_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
+#define CB_COLOR5_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
+#define CB_COLOR5_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
+#define CB_COLOR5_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
+#define CB_COLOR5_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
+#define CB_COLOR5_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                             0x12
+#define CB_COLOR5_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                        0x13
+#define CB_COLOR5_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
+#define CB_COLOR5_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define CB_COLOR5_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
+#define CB_COLOR5_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
+#define CB_COLOR5_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
+#define CB_COLOR5_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
+#define CB_COLOR5_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
+#define CB_COLOR5_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
+#define CB_COLOR5_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
+#define CB_COLOR5_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                               0x00040000L
+#define CB_COLOR5_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                          0x00080000L
+//CB_COLOR5_CMASK
+#define CB_COLOR5_CMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR5_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR5_CMASK_BASE_EXT
+#define CB_COLOR5_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR5_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR5_FMASK
+#define CB_COLOR5_FMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR5_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR5_FMASK_BASE_EXT
+#define CB_COLOR5_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR5_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR5_CLEAR_WORD0
+#define CB_COLOR5_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
+#define CB_COLOR5_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
+//CB_COLOR5_CLEAR_WORD1
+#define CB_COLOR5_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
+#define CB_COLOR5_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
+//CB_COLOR5_DCC_BASE
+#define CB_COLOR5_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR5_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR5_DCC_BASE_EXT
+#define CB_COLOR5_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR5_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR6_BASE
+#define CB_COLOR6_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR6_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR6_BASE_EXT
+#define CB_COLOR6_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR6_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR6_ATTRIB2
+#define CB_COLOR6_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR6_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR6_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR6_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR6_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR6_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR6_VIEW
+#define CB_COLOR6_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR6_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR6_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
+#define CB_COLOR6_VIEW__SLICE_START_MASK                                                                      0x000007FFL
+#define CB_COLOR6_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
+#define CB_COLOR6_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
+//CB_COLOR6_INFO
+#define CB_COLOR6_INFO__ENDIAN__SHIFT                                                                         0x0
+#define CB_COLOR6_INFO__FORMAT__SHIFT                                                                         0x2
+#define CB_COLOR6_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR6_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR6_INFO__FAST_CLEAR__SHIFT                                                                     0xd
+#define CB_COLOR6_INFO__COMPRESSION__SHIFT                                                                    0xe
+#define CB_COLOR6_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR6_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR6_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR6_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR6_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
+#define CB_COLOR6_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
+#define CB_COLOR6_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
+#define CB_COLOR6_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
+#define CB_COLOR6_INFO__ENDIAN_MASK                                                                           0x00000003L
+#define CB_COLOR6_INFO__FORMAT_MASK                                                                           0x0000007CL
+#define CB_COLOR6_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR6_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR6_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
+#define CB_COLOR6_INFO__COMPRESSION_MASK                                                                      0x00004000L
+#define CB_COLOR6_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR6_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR6_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR6_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+#define CB_COLOR6_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
+#define CB_COLOR6_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
+#define CB_COLOR6_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
+#define CB_COLOR6_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
+//CB_COLOR6_ATTRIB
+#define CB_COLOR6_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
+#define CB_COLOR6_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
+#define CB_COLOR6_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
+#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
+#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
+#define CB_COLOR6_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
+#define CB_COLOR6_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
+#define CB_COLOR6_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
+#define CB_COLOR6_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
+#define CB_COLOR6_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
+#define CB_COLOR6_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
+#define CB_COLOR6_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
+#define CB_COLOR6_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
+#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
+#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
+#define CB_COLOR6_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
+#define CB_COLOR6_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
+#define CB_COLOR6_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
+#define CB_COLOR6_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
+#define CB_COLOR6_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
+//CB_COLOR6_DCC_CONTROL
+#define CB_COLOR6_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
+#define CB_COLOR6_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
+#define CB_COLOR6_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
+#define CB_COLOR6_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
+#define CB_COLOR6_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
+#define CB_COLOR6_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
+#define CB_COLOR6_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
+#define CB_COLOR6_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
+#define CB_COLOR6_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
+#define CB_COLOR6_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                             0x12
+#define CB_COLOR6_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                        0x13
+#define CB_COLOR6_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
+#define CB_COLOR6_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define CB_COLOR6_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
+#define CB_COLOR6_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
+#define CB_COLOR6_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
+#define CB_COLOR6_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
+#define CB_COLOR6_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
+#define CB_COLOR6_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
+#define CB_COLOR6_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
+#define CB_COLOR6_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                               0x00040000L
+#define CB_COLOR6_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                          0x00080000L
+//CB_COLOR6_CMASK
+#define CB_COLOR6_CMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR6_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR6_CMASK_BASE_EXT
+#define CB_COLOR6_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR6_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR6_FMASK
+#define CB_COLOR6_FMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR6_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR6_FMASK_BASE_EXT
+#define CB_COLOR6_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR6_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR6_CLEAR_WORD0
+#define CB_COLOR6_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
+#define CB_COLOR6_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
+//CB_COLOR6_CLEAR_WORD1
+#define CB_COLOR6_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
+#define CB_COLOR6_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
+//CB_COLOR6_DCC_BASE
+#define CB_COLOR6_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR6_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR6_DCC_BASE_EXT
+#define CB_COLOR6_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR6_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR7_BASE
+#define CB_COLOR7_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR7_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR7_BASE_EXT
+#define CB_COLOR7_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR7_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR7_ATTRIB2
+#define CB_COLOR7_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR7_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR7_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR7_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR7_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR7_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR7_VIEW
+#define CB_COLOR7_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR7_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR7_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
+#define CB_COLOR7_VIEW__SLICE_START_MASK                                                                      0x000007FFL
+#define CB_COLOR7_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
+#define CB_COLOR7_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
+//CB_COLOR7_INFO
+#define CB_COLOR7_INFO__ENDIAN__SHIFT                                                                         0x0
+#define CB_COLOR7_INFO__FORMAT__SHIFT                                                                         0x2
+#define CB_COLOR7_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR7_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR7_INFO__FAST_CLEAR__SHIFT                                                                     0xd
+#define CB_COLOR7_INFO__COMPRESSION__SHIFT                                                                    0xe
+#define CB_COLOR7_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR7_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR7_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR7_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR7_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
+#define CB_COLOR7_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
+#define CB_COLOR7_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
+#define CB_COLOR7_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
+#define CB_COLOR7_INFO__ENDIAN_MASK                                                                           0x00000003L
+#define CB_COLOR7_INFO__FORMAT_MASK                                                                           0x0000007CL
+#define CB_COLOR7_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR7_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR7_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
+#define CB_COLOR7_INFO__COMPRESSION_MASK                                                                      0x00004000L
+#define CB_COLOR7_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR7_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR7_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR7_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+#define CB_COLOR7_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
+#define CB_COLOR7_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
+#define CB_COLOR7_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
+#define CB_COLOR7_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
+//CB_COLOR7_ATTRIB
+#define CB_COLOR7_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
+#define CB_COLOR7_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
+#define CB_COLOR7_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
+#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
+#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
+#define CB_COLOR7_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
+#define CB_COLOR7_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
+#define CB_COLOR7_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
+#define CB_COLOR7_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
+#define CB_COLOR7_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
+#define CB_COLOR7_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
+#define CB_COLOR7_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
+#define CB_COLOR7_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
+#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
+#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
+#define CB_COLOR7_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
+#define CB_COLOR7_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
+#define CB_COLOR7_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
+#define CB_COLOR7_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
+#define CB_COLOR7_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
+//CB_COLOR7_DCC_CONTROL
+#define CB_COLOR7_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
+#define CB_COLOR7_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
+#define CB_COLOR7_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
+#define CB_COLOR7_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
+#define CB_COLOR7_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
+#define CB_COLOR7_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
+#define CB_COLOR7_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
+#define CB_COLOR7_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
+#define CB_COLOR7_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
+#define CB_COLOR7_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                             0x12
+#define CB_COLOR7_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                        0x13
+#define CB_COLOR7_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
+#define CB_COLOR7_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define CB_COLOR7_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
+#define CB_COLOR7_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
+#define CB_COLOR7_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
+#define CB_COLOR7_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
+#define CB_COLOR7_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
+#define CB_COLOR7_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
+#define CB_COLOR7_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
+#define CB_COLOR7_DCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                               0x00040000L
+#define CB_COLOR7_DCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                          0x00080000L
+//CB_COLOR7_CMASK
+#define CB_COLOR7_CMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR7_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR7_CMASK_BASE_EXT
+#define CB_COLOR7_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR7_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR7_FMASK
+#define CB_COLOR7_FMASK__BASE_256B__SHIFT                                                                     0x0
+#define CB_COLOR7_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//CB_COLOR7_FMASK_BASE_EXT
+#define CB_COLOR7_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
+#define CB_COLOR7_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
+//CB_COLOR7_CLEAR_WORD0
+#define CB_COLOR7_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
+#define CB_COLOR7_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
+//CB_COLOR7_CLEAR_WORD1
+#define CB_COLOR7_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
+#define CB_COLOR7_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
+//CB_COLOR7_DCC_BASE
+#define CB_COLOR7_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR7_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR7_DCC_BASE_EXT
+#define CB_COLOR7_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR7_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+
+
+// addressBlock: xcd0_gc_gfxudec
+//CP_EOP_DONE_ADDR_LO
+#define CP_EOP_DONE_ADDR_LO__ADDR_LO__SHIFT                                                                   0x2
+#define CP_EOP_DONE_ADDR_LO__ADDR_LO_MASK                                                                     0xFFFFFFFCL
+//CP_EOP_DONE_ADDR_HI
+#define CP_EOP_DONE_ADDR_HI__ADDR_HI__SHIFT                                                                   0x0
+#define CP_EOP_DONE_ADDR_HI__ADDR_HI_MASK                                                                     0x0000FFFFL
+//CP_EOP_DONE_DATA_LO
+#define CP_EOP_DONE_DATA_LO__DATA_LO__SHIFT                                                                   0x0
+#define CP_EOP_DONE_DATA_LO__DATA_LO_MASK                                                                     0xFFFFFFFFL
+//CP_EOP_DONE_DATA_HI
+#define CP_EOP_DONE_DATA_HI__DATA_HI__SHIFT                                                                   0x0
+#define CP_EOP_DONE_DATA_HI__DATA_HI_MASK                                                                     0xFFFFFFFFL
+//CP_EOP_LAST_FENCE_LO
+#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO__SHIFT                                                            0x0
+#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO_MASK                                                              0xFFFFFFFFL
+//CP_EOP_LAST_FENCE_HI
+#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI__SHIFT                                                            0x0
+#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI_MASK                                                              0xFFFFFFFFL
+//CP_STREAM_OUT_ADDR_LO
+#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_LO__SHIFT                                                      0x2
+#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_LO_MASK                                                        0xFFFFFFFCL
+//CP_STREAM_OUT_ADDR_HI
+#define CP_STREAM_OUT_ADDR_HI__STREAM_OUT_ADDR_HI__SHIFT                                                      0x0
+#define CP_STREAM_OUT_ADDR_HI__STREAM_OUT_ADDR_HI_MASK                                                        0x0000FFFFL
+//CP_NUM_PRIM_WRITTEN_COUNT0_LO
+#define CP_NUM_PRIM_WRITTEN_COUNT0_LO__NUM_PRIM_WRITTEN_CNT0_LO__SHIFT                                        0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT0_LO__NUM_PRIM_WRITTEN_CNT0_LO_MASK                                          0xFFFFFFFFL
+//CP_NUM_PRIM_WRITTEN_COUNT0_HI
+#define CP_NUM_PRIM_WRITTEN_COUNT0_HI__NUM_PRIM_WRITTEN_CNT0_HI__SHIFT                                        0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT0_HI__NUM_PRIM_WRITTEN_CNT0_HI_MASK                                          0xFFFFFFFFL
+//CP_NUM_PRIM_NEEDED_COUNT0_LO
+#define CP_NUM_PRIM_NEEDED_COUNT0_LO__NUM_PRIM_NEEDED_CNT0_LO__SHIFT                                          0x0
+#define CP_NUM_PRIM_NEEDED_COUNT0_LO__NUM_PRIM_NEEDED_CNT0_LO_MASK                                            0xFFFFFFFFL
+//CP_NUM_PRIM_NEEDED_COUNT0_HI
+#define CP_NUM_PRIM_NEEDED_COUNT0_HI__NUM_PRIM_NEEDED_CNT0_HI__SHIFT                                          0x0
+#define CP_NUM_PRIM_NEEDED_COUNT0_HI__NUM_PRIM_NEEDED_CNT0_HI_MASK                                            0xFFFFFFFFL
+//CP_NUM_PRIM_WRITTEN_COUNT1_LO
+#define CP_NUM_PRIM_WRITTEN_COUNT1_LO__NUM_PRIM_WRITTEN_CNT1_LO__SHIFT                                        0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT1_LO__NUM_PRIM_WRITTEN_CNT1_LO_MASK                                          0xFFFFFFFFL
+//CP_NUM_PRIM_WRITTEN_COUNT1_HI
+#define CP_NUM_PRIM_WRITTEN_COUNT1_HI__NUM_PRIM_WRITTEN_CNT1_HI__SHIFT                                        0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT1_HI__NUM_PRIM_WRITTEN_CNT1_HI_MASK                                          0xFFFFFFFFL
+//CP_NUM_PRIM_NEEDED_COUNT1_LO
+#define CP_NUM_PRIM_NEEDED_COUNT1_LO__NUM_PRIM_NEEDED_CNT1_LO__SHIFT                                          0x0
+#define CP_NUM_PRIM_NEEDED_COUNT1_LO__NUM_PRIM_NEEDED_CNT1_LO_MASK                                            0xFFFFFFFFL
+//CP_NUM_PRIM_NEEDED_COUNT1_HI
+#define CP_NUM_PRIM_NEEDED_COUNT1_HI__NUM_PRIM_NEEDED_CNT1_HI__SHIFT                                          0x0
+#define CP_NUM_PRIM_NEEDED_COUNT1_HI__NUM_PRIM_NEEDED_CNT1_HI_MASK                                            0xFFFFFFFFL
+//CP_NUM_PRIM_WRITTEN_COUNT2_LO
+#define CP_NUM_PRIM_WRITTEN_COUNT2_LO__NUM_PRIM_WRITTEN_CNT2_LO__SHIFT                                        0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT2_LO__NUM_PRIM_WRITTEN_CNT2_LO_MASK                                          0xFFFFFFFFL
+//CP_NUM_PRIM_WRITTEN_COUNT2_HI
+#define CP_NUM_PRIM_WRITTEN_COUNT2_HI__NUM_PRIM_WRITTEN_CNT2_HI__SHIFT                                        0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT2_HI__NUM_PRIM_WRITTEN_CNT2_HI_MASK                                          0xFFFFFFFFL
+//CP_NUM_PRIM_NEEDED_COUNT2_LO
+#define CP_NUM_PRIM_NEEDED_COUNT2_LO__NUM_PRIM_NEEDED_CNT2_LO__SHIFT                                          0x0
+#define CP_NUM_PRIM_NEEDED_COUNT2_LO__NUM_PRIM_NEEDED_CNT2_LO_MASK                                            0xFFFFFFFFL
+//CP_NUM_PRIM_NEEDED_COUNT2_HI
+#define CP_NUM_PRIM_NEEDED_COUNT2_HI__NUM_PRIM_NEEDED_CNT2_HI__SHIFT                                          0x0
+#define CP_NUM_PRIM_NEEDED_COUNT2_HI__NUM_PRIM_NEEDED_CNT2_HI_MASK                                            0xFFFFFFFFL
+//CP_NUM_PRIM_WRITTEN_COUNT3_LO
+#define CP_NUM_PRIM_WRITTEN_COUNT3_LO__NUM_PRIM_WRITTEN_CNT3_LO__SHIFT                                        0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT3_LO__NUM_PRIM_WRITTEN_CNT3_LO_MASK                                          0xFFFFFFFFL
+//CP_NUM_PRIM_WRITTEN_COUNT3_HI
+#define CP_NUM_PRIM_WRITTEN_COUNT3_HI__NUM_PRIM_WRITTEN_CNT3_HI__SHIFT                                        0x0
+#define CP_NUM_PRIM_WRITTEN_COUNT3_HI__NUM_PRIM_WRITTEN_CNT3_HI_MASK                                          0xFFFFFFFFL
+//CP_NUM_PRIM_NEEDED_COUNT3_LO
+#define CP_NUM_PRIM_NEEDED_COUNT3_LO__NUM_PRIM_NEEDED_CNT3_LO__SHIFT                                          0x0
+#define CP_NUM_PRIM_NEEDED_COUNT3_LO__NUM_PRIM_NEEDED_CNT3_LO_MASK                                            0xFFFFFFFFL
+//CP_NUM_PRIM_NEEDED_COUNT3_HI
+#define CP_NUM_PRIM_NEEDED_COUNT3_HI__NUM_PRIM_NEEDED_CNT3_HI__SHIFT                                          0x0
+#define CP_NUM_PRIM_NEEDED_COUNT3_HI__NUM_PRIM_NEEDED_CNT3_HI_MASK                                            0xFFFFFFFFL
+//CP_PIPE_STATS_ADDR_LO
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO__SHIFT                                                      0x2
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO_MASK                                                        0xFFFFFFFCL
+//CP_PIPE_STATS_ADDR_HI
+#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI__SHIFT                                                      0x0
+#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI_MASK                                                        0x0000FFFFL
+//CP_VGT_IAVERT_COUNT_LO
+#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO__SHIFT                                                        0x0
+#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO_MASK                                                          0xFFFFFFFFL
+//CP_VGT_IAVERT_COUNT_HI
+#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI__SHIFT                                                        0x0
+#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI_MASK                                                          0xFFFFFFFFL
+//CP_VGT_IAPRIM_COUNT_LO
+#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO__SHIFT                                                        0x0
+#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO_MASK                                                          0xFFFFFFFFL
+//CP_VGT_IAPRIM_COUNT_HI
+#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI__SHIFT                                                        0x0
+#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI_MASK                                                          0xFFFFFFFFL
+//CP_VGT_GSPRIM_COUNT_LO
+#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO__SHIFT                                                        0x0
+#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO_MASK                                                          0xFFFFFFFFL
+//CP_VGT_GSPRIM_COUNT_HI
+#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI__SHIFT                                                        0x0
+#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI_MASK                                                          0xFFFFFFFFL
+//CP_VGT_VSINVOC_COUNT_LO
+#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_VSINVOC_COUNT_HI
+#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_GSINVOC_COUNT_LO
+#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_GSINVOC_COUNT_HI
+#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_HSINVOC_COUNT_LO
+#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_HSINVOC_COUNT_HI
+#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_DSINVOC_COUNT_LO
+#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_DSINVOC_COUNT_HI
+#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_PA_CINVOC_COUNT_LO
+#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO__SHIFT                                                         0x0
+#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO_MASK                                                           0xFFFFFFFFL
+//CP_PA_CINVOC_COUNT_HI
+#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI__SHIFT                                                         0x0
+#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI_MASK                                                           0xFFFFFFFFL
+//CP_PA_CPRIM_COUNT_LO
+#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO__SHIFT                                                           0x0
+#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO_MASK                                                             0xFFFFFFFFL
+//CP_PA_CPRIM_COUNT_HI
+#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI__SHIFT                                                           0x0
+#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI_MASK                                                             0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT0_LO
+#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO__SHIFT                                                     0x0
+#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO_MASK                                                       0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT0_HI
+#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI__SHIFT                                                     0x0
+#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI_MASK                                                       0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT1_LO
+#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE__SHIFT                                                              0x0
+#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE_MASK                                                                0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT1_HI
+#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE__SHIFT                                                              0x0
+#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE_MASK                                                                0xFFFFFFFFL
+//CP_VGT_CSINVOC_COUNT_LO
+#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_CSINVOC_COUNT_HI
+#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_PIPE_STATS_CONTROL
+#define CP_PIPE_STATS_CONTROL__CACHE_POLICY__SHIFT                                                            0x19
+#define CP_PIPE_STATS_CONTROL__CACHE_POLICY_MASK                                                              0x02000000L
+//CP_STREAM_OUT_CONTROL
+#define CP_STREAM_OUT_CONTROL__CACHE_POLICY__SHIFT                                                            0x19
+#define CP_STREAM_OUT_CONTROL__CACHE_POLICY_MASK                                                              0x02000000L
+//CP_STRMOUT_CNTL
+#define CP_STRMOUT_CNTL__OFFSET_UPDATE_DONE__SHIFT                                                            0x0
+#define CP_STRMOUT_CNTL__OFFSET_UPDATE_DONE_MASK                                                              0x00000001L
+//SCRATCH_REG0
+#define SCRATCH_REG0__SCRATCH_REG0__SHIFT                                                                     0x0
+#define SCRATCH_REG0__SCRATCH_REG0_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG1
+#define SCRATCH_REG1__SCRATCH_REG1__SHIFT                                                                     0x0
+#define SCRATCH_REG1__SCRATCH_REG1_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG2
+#define SCRATCH_REG2__SCRATCH_REG2__SHIFT                                                                     0x0
+#define SCRATCH_REG2__SCRATCH_REG2_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG3
+#define SCRATCH_REG3__SCRATCH_REG3__SHIFT                                                                     0x0
+#define SCRATCH_REG3__SCRATCH_REG3_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG4
+#define SCRATCH_REG4__SCRATCH_REG4__SHIFT                                                                     0x0
+#define SCRATCH_REG4__SCRATCH_REG4_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG5
+#define SCRATCH_REG5__SCRATCH_REG5__SHIFT                                                                     0x0
+#define SCRATCH_REG5__SCRATCH_REG5_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG6
+#define SCRATCH_REG6__SCRATCH_REG6__SHIFT                                                                     0x0
+#define SCRATCH_REG6__SCRATCH_REG6_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG7
+#define SCRATCH_REG7__SCRATCH_REG7__SHIFT                                                                     0x0
+#define SCRATCH_REG7__SCRATCH_REG7_MASK                                                                       0xFFFFFFFFL
+//CP_APPEND_DATA_HI
+#define CP_APPEND_DATA_HI__DATA__SHIFT                                                                        0x0
+#define CP_APPEND_DATA_HI__DATA_MASK                                                                          0xFFFFFFFFL
+//CP_APPEND_LAST_CS_FENCE_HI
+#define CP_APPEND_LAST_CS_FENCE_HI__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_CS_FENCE_HI__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_APPEND_LAST_PS_FENCE_HI
+#define CP_APPEND_LAST_PS_FENCE_HI__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_PS_FENCE_HI__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//SCRATCH_UMSK
+#define SCRATCH_UMSK__OBSOLETE_UMSK__SHIFT                                                                    0x0
+#define SCRATCH_UMSK__OBSOLETE_SWAP__SHIFT                                                                    0x10
+#define SCRATCH_UMSK__OBSOLETE_UMSK_MASK                                                                      0x000000FFL
+#define SCRATCH_UMSK__OBSOLETE_SWAP_MASK                                                                      0x00030000L
+//SCRATCH_ADDR
+#define SCRATCH_ADDR__OBSOLETE_ADDR__SHIFT                                                                    0x0
+#define SCRATCH_ADDR__OBSOLETE_ADDR_MASK                                                                      0xFFFFFFFFL
+//CP_PFP_ATOMIC_PREOP_LO
+#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                        0x0
+#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                          0xFFFFFFFFL
+//CP_PFP_ATOMIC_PREOP_HI
+#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                        0x0
+#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                          0xFFFFFFFFL
+//CP_PFP_GDS_ATOMIC0_PREOP_LO
+#define CP_PFP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT                                              0x0
+#define CP_PFP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK                                                0xFFFFFFFFL
+//CP_PFP_GDS_ATOMIC0_PREOP_HI
+#define CP_PFP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT                                              0x0
+#define CP_PFP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK                                                0xFFFFFFFFL
+//CP_PFP_GDS_ATOMIC1_PREOP_LO
+#define CP_PFP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT                                              0x0
+#define CP_PFP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK                                                0xFFFFFFFFL
+//CP_PFP_GDS_ATOMIC1_PREOP_HI
+#define CP_PFP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT                                              0x0
+#define CP_PFP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK                                                0xFFFFFFFFL
+//CP_APPEND_ADDR_LO
+#define CP_APPEND_ADDR_LO__MEM_ADDR_LO__SHIFT                                                                 0x2
+#define CP_APPEND_ADDR_LO__MEM_ADDR_LO_MASK                                                                   0xFFFFFFFCL
+//CP_APPEND_ADDR_HI
+#define CP_APPEND_ADDR_HI__MEM_ADDR_HI__SHIFT                                                                 0x0
+#define CP_APPEND_ADDR_HI__CS_PS_SEL__SHIFT                                                                   0x10
+#define CP_APPEND_ADDR_HI__CACHE_POLICY__SHIFT                                                                0x19
+#define CP_APPEND_ADDR_HI__COMMAND__SHIFT                                                                     0x1d
+#define CP_APPEND_ADDR_HI__MEM_ADDR_HI_MASK                                                                   0x0000FFFFL
+#define CP_APPEND_ADDR_HI__CS_PS_SEL_MASK                                                                     0x00010000L
+#define CP_APPEND_ADDR_HI__CACHE_POLICY_MASK                                                                  0x02000000L
+#define CP_APPEND_ADDR_HI__COMMAND_MASK                                                                       0xE0000000L
+//CP_APPEND_DATA_LO
+#define CP_APPEND_DATA_LO__DATA__SHIFT                                                                        0x0
+#define CP_APPEND_DATA_LO__DATA_MASK                                                                          0xFFFFFFFFL
+//CP_APPEND_LAST_CS_FENCE_LO
+#define CP_APPEND_LAST_CS_FENCE_LO__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_CS_FENCE_LO__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_APPEND_LAST_PS_FENCE_LO
+#define CP_APPEND_LAST_PS_FENCE_LO__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_PS_FENCE_LO__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_ATOMIC_PREOP_LO
+#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                            0x0
+#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                              0xFFFFFFFFL
+//CP_ME_ATOMIC_PREOP_LO
+#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                         0x0
+#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                           0xFFFFFFFFL
+//CP_ATOMIC_PREOP_HI
+#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                            0x0
+#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                              0xFFFFFFFFL
+//CP_ME_ATOMIC_PREOP_HI
+#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                         0x0
+#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                           0xFFFFFFFFL
+//CP_GDS_ATOMIC0_PREOP_LO
+#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT                                                  0x0
+#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK                                                    0xFFFFFFFFL
+//CP_ME_GDS_ATOMIC0_PREOP_LO
+#define CP_ME_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT                                               0x0
+#define CP_ME_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK                                                 0xFFFFFFFFL
+//CP_GDS_ATOMIC0_PREOP_HI
+#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT                                                  0x0
+#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK                                                    0xFFFFFFFFL
+//CP_ME_GDS_ATOMIC0_PREOP_HI
+#define CP_ME_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT                                               0x0
+#define CP_ME_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK                                                 0xFFFFFFFFL
+//CP_GDS_ATOMIC1_PREOP_LO
+#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT                                                  0x0
+#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK                                                    0xFFFFFFFFL
+//CP_ME_GDS_ATOMIC1_PREOP_LO
+#define CP_ME_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT                                               0x0
+#define CP_ME_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK                                                 0xFFFFFFFFL
+//CP_GDS_ATOMIC1_PREOP_HI
+#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT                                                  0x0
+#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK                                                    0xFFFFFFFFL
+//CP_ME_GDS_ATOMIC1_PREOP_HI
+#define CP_ME_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT                                               0x0
+#define CP_ME_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK                                                 0xFFFFFFFFL
+//CP_ME_MC_WADDR_LO
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO__SHIFT                                                              0x2
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO_MASK                                                                0xFFFFFFFCL
+//CP_ME_MC_WADDR_HI
+#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI__SHIFT                                                              0x0
+#define CP_ME_MC_WADDR_HI__CACHE_POLICY__SHIFT                                                                0x16
+#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI_MASK                                                                0x0000FFFFL
+#define CP_ME_MC_WADDR_HI__CACHE_POLICY_MASK                                                                  0x00400000L
+//CP_ME_MC_WDATA_LO
+#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO__SHIFT                                                              0x0
+#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO_MASK                                                                0xFFFFFFFFL
+//CP_ME_MC_WDATA_HI
+#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI__SHIFT                                                              0x0
+#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI_MASK                                                                0xFFFFFFFFL
+//CP_ME_MC_RADDR_LO
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO__SHIFT                                                              0x2
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO_MASK                                                                0xFFFFFFFCL
+//CP_ME_MC_RADDR_HI
+#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI__SHIFT                                                              0x0
+#define CP_ME_MC_RADDR_HI__CACHE_POLICY__SHIFT                                                                0x16
+#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI_MASK                                                                0x0000FFFFL
+#define CP_ME_MC_RADDR_HI__CACHE_POLICY_MASK                                                                  0x00400000L
+//CP_SEM_WAIT_TIMER
+#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER__SHIFT                                                              0x0
+#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER_MASK                                                                0xFFFFFFFFL
+//CP_SIG_SEM_ADDR_LO
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_SWAP__SHIFT                                                              0x0
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT                                                                0x3
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_SWAP_MASK                                                                0x00000003L
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO_MASK                                                                  0xFFFFFFF8L
+//CP_SIG_SEM_ADDR_HI
+#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT                                                                0x0
+#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT                                                            0x10
+#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT                                                            0x14
+#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT                                                            0x18
+#define CP_SIG_SEM_ADDR_HI__SEM_SELECT__SHIFT                                                                 0x1d
+#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI_MASK                                                                  0x0000FFFFL
+#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK                                                              0x00010000L
+#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK                                                              0x00100000L
+#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK                                                              0x03000000L
+#define CP_SIG_SEM_ADDR_HI__SEM_SELECT_MASK                                                                   0xE0000000L
+//CP_WAIT_REG_MEM_TIMEOUT
+#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT__SHIFT                                                  0x0
+#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT_MASK                                                    0xFFFFFFFFL
+//CP_WAIT_SEM_ADDR_LO
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_SWAP__SHIFT                                                             0x0
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT                                                               0x3
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_SWAP_MASK                                                               0x00000003L
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO_MASK                                                                 0xFFFFFFF8L
+//CP_WAIT_SEM_ADDR_HI
+#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT                                                               0x0
+#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT                                                           0x10
+#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT                                                           0x14
+#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT                                                           0x18
+#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT__SHIFT                                                                0x1d
+#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI_MASK                                                                 0x0000FFFFL
+#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK                                                             0x00010000L
+#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK                                                             0x00100000L
+#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK                                                             0x03000000L
+#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT_MASK                                                                  0xE0000000L
+//CP_DMA_PFP_CONTROL
+#define CP_DMA_PFP_CONTROL__MEMLOG_CLEAR__SHIFT                                                               0xa
+#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY__SHIFT                                                           0xd
+#define CP_DMA_PFP_CONTROL__DST_SELECT__SHIFT                                                                 0x14
+#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY__SHIFT                                                           0x19
+#define CP_DMA_PFP_CONTROL__SRC_SELECT__SHIFT                                                                 0x1d
+#define CP_DMA_PFP_CONTROL__MEMLOG_CLEAR_MASK                                                                 0x00000400L
+#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY_MASK                                                             0x00002000L
+#define CP_DMA_PFP_CONTROL__DST_SELECT_MASK                                                                   0x00300000L
+#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY_MASK                                                             0x02000000L
+#define CP_DMA_PFP_CONTROL__SRC_SELECT_MASK                                                                   0x60000000L
+//CP_DMA_ME_CONTROL
+#define CP_DMA_ME_CONTROL__MEMLOG_CLEAR__SHIFT                                                                0xa
+#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY__SHIFT                                                            0xd
+#define CP_DMA_ME_CONTROL__DST_SELECT__SHIFT                                                                  0x14
+#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY__SHIFT                                                            0x19
+#define CP_DMA_ME_CONTROL__SRC_SELECT__SHIFT                                                                  0x1d
+#define CP_DMA_ME_CONTROL__MEMLOG_CLEAR_MASK                                                                  0x00000400L
+#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY_MASK                                                              0x00002000L
+#define CP_DMA_ME_CONTROL__DST_SELECT_MASK                                                                    0x00300000L
+#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY_MASK                                                              0x02000000L
+#define CP_DMA_ME_CONTROL__SRC_SELECT_MASK                                                                    0x60000000L
+//CP_COHER_BASE_HI
+#define CP_COHER_BASE_HI__COHER_BASE_HI_256B__SHIFT                                                           0x0
+#define CP_COHER_BASE_HI__COHER_BASE_HI_256B_MASK                                                             0x000000FFL
+//CP_COHER_START_DELAY
+#define CP_COHER_START_DELAY__START_DELAY_COUNT__SHIFT                                                        0x0
+#define CP_COHER_START_DELAY__START_DELAY_COUNT_MASK                                                          0x0000003FL
+//CP_COHER_CNTL
+#define CP_COHER_CNTL__TC_NC_ACTION_ENA__SHIFT                                                                0x3
+#define CP_COHER_CNTL__TC_WC_ACTION_ENA__SHIFT                                                                0x4
+#define CP_COHER_CNTL__TC_INV_METADATA_ACTION_ENA__SHIFT                                                      0x5
+#define CP_COHER_CNTL__TCL1_VOL_ACTION_ENA__SHIFT                                                             0xf
+#define CP_COHER_CNTL__TC_WB_ACTION_ENA__SHIFT                                                                0x12
+#define CP_COHER_CNTL__TCL1_ACTION_ENA__SHIFT                                                                 0x16
+#define CP_COHER_CNTL__TC_ACTION_ENA__SHIFT                                                                   0x17
+#define CP_COHER_CNTL__CB_ACTION_ENA__SHIFT                                                                   0x19
+#define CP_COHER_CNTL__DB_ACTION_ENA__SHIFT                                                                   0x1a
+#define CP_COHER_CNTL__SH_KCACHE_ACTION_ENA__SHIFT                                                            0x1b
+#define CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA__SHIFT                                                        0x1c
+#define CP_COHER_CNTL__SH_ICACHE_ACTION_ENA__SHIFT                                                            0x1d
+#define CP_COHER_CNTL__SH_KCACHE_WB_ACTION_ENA__SHIFT                                                         0x1e
+#define CP_COHER_CNTL__TC_NC_ACTION_ENA_MASK                                                                  0x00000008L
+#define CP_COHER_CNTL__TC_WC_ACTION_ENA_MASK                                                                  0x00000010L
+#define CP_COHER_CNTL__TC_INV_METADATA_ACTION_ENA_MASK                                                        0x00000020L
+#define CP_COHER_CNTL__TCL1_VOL_ACTION_ENA_MASK                                                               0x00008000L
+#define CP_COHER_CNTL__TC_WB_ACTION_ENA_MASK                                                                  0x00040000L
+#define CP_COHER_CNTL__TCL1_ACTION_ENA_MASK                                                                   0x00400000L
+#define CP_COHER_CNTL__TC_ACTION_ENA_MASK                                                                     0x00800000L
+#define CP_COHER_CNTL__CB_ACTION_ENA_MASK                                                                     0x02000000L
+#define CP_COHER_CNTL__DB_ACTION_ENA_MASK                                                                     0x04000000L
+#define CP_COHER_CNTL__SH_KCACHE_ACTION_ENA_MASK                                                              0x08000000L
+#define CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA_MASK                                                          0x10000000L
+#define CP_COHER_CNTL__SH_ICACHE_ACTION_ENA_MASK                                                              0x20000000L
+#define CP_COHER_CNTL__SH_KCACHE_WB_ACTION_ENA_MASK                                                           0x40000000L
+//CP_COHER_SIZE
+#define CP_COHER_SIZE__COHER_SIZE_256B__SHIFT                                                                 0x0
+#define CP_COHER_SIZE__COHER_SIZE_256B_MASK                                                                   0xFFFFFFFFL
+//CP_COHER_BASE
+#define CP_COHER_BASE__COHER_BASE_256B__SHIFT                                                                 0x0
+#define CP_COHER_BASE__COHER_BASE_256B_MASK                                                                   0xFFFFFFFFL
+//CP_COHER_STATUS
+#define CP_COHER_STATUS__MEID__SHIFT                                                                          0x18
+#define CP_COHER_STATUS__STATUS__SHIFT                                                                        0x1f
+#define CP_COHER_STATUS__MEID_MASK                                                                            0x03000000L
+#define CP_COHER_STATUS__STATUS_MASK                                                                          0x80000000L
+//CP_DMA_ME_SRC_ADDR
+#define CP_DMA_ME_SRC_ADDR__SRC_ADDR__SHIFT                                                                   0x0
+#define CP_DMA_ME_SRC_ADDR__SRC_ADDR_MASK                                                                     0xFFFFFFFFL
+//CP_DMA_ME_SRC_ADDR_HI
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT                                                             0x0
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI_MASK                                                               0x0000FFFFL
+//CP_DMA_ME_DST_ADDR
+#define CP_DMA_ME_DST_ADDR__DST_ADDR__SHIFT                                                                   0x0
+#define CP_DMA_ME_DST_ADDR__DST_ADDR_MASK                                                                     0xFFFFFFFFL
+//CP_DMA_ME_DST_ADDR_HI
+#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI__SHIFT                                                             0x0
+#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI_MASK                                                               0x0000FFFFL
+//CP_DMA_ME_COMMAND
+#define CP_DMA_ME_COMMAND__BYTE_COUNT__SHIFT                                                                  0x0
+#define CP_DMA_ME_COMMAND__SAS__SHIFT                                                                         0x1a
+#define CP_DMA_ME_COMMAND__DAS__SHIFT                                                                         0x1b
+#define CP_DMA_ME_COMMAND__SAIC__SHIFT                                                                        0x1c
+#define CP_DMA_ME_COMMAND__DAIC__SHIFT                                                                        0x1d
+#define CP_DMA_ME_COMMAND__RAW_WAIT__SHIFT                                                                    0x1e
+#define CP_DMA_ME_COMMAND__DIS_WC__SHIFT                                                                      0x1f
+#define CP_DMA_ME_COMMAND__BYTE_COUNT_MASK                                                                    0x03FFFFFFL
+#define CP_DMA_ME_COMMAND__SAS_MASK                                                                           0x04000000L
+#define CP_DMA_ME_COMMAND__DAS_MASK                                                                           0x08000000L
+#define CP_DMA_ME_COMMAND__SAIC_MASK                                                                          0x10000000L
+#define CP_DMA_ME_COMMAND__DAIC_MASK                                                                          0x20000000L
+#define CP_DMA_ME_COMMAND__RAW_WAIT_MASK                                                                      0x40000000L
+#define CP_DMA_ME_COMMAND__DIS_WC_MASK                                                                        0x80000000L
+//CP_DMA_PFP_SRC_ADDR
+#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR__SHIFT                                                                  0x0
+#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR_MASK                                                                    0xFFFFFFFFL
+//CP_DMA_PFP_SRC_ADDR_HI
+#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT                                                            0x0
+#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI_MASK                                                              0x0000FFFFL
+//CP_DMA_PFP_DST_ADDR
+#define CP_DMA_PFP_DST_ADDR__DST_ADDR__SHIFT                                                                  0x0
+#define CP_DMA_PFP_DST_ADDR__DST_ADDR_MASK                                                                    0xFFFFFFFFL
+//CP_DMA_PFP_DST_ADDR_HI
+#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI__SHIFT                                                            0x0
+#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI_MASK                                                              0x0000FFFFL
+//CP_DMA_PFP_COMMAND
+#define CP_DMA_PFP_COMMAND__BYTE_COUNT__SHIFT                                                                 0x0
+#define CP_DMA_PFP_COMMAND__SAS__SHIFT                                                                        0x1a
+#define CP_DMA_PFP_COMMAND__DAS__SHIFT                                                                        0x1b
+#define CP_DMA_PFP_COMMAND__SAIC__SHIFT                                                                       0x1c
+#define CP_DMA_PFP_COMMAND__DAIC__SHIFT                                                                       0x1d
+#define CP_DMA_PFP_COMMAND__RAW_WAIT__SHIFT                                                                   0x1e
+#define CP_DMA_PFP_COMMAND__DIS_WC__SHIFT                                                                     0x1f
+#define CP_DMA_PFP_COMMAND__BYTE_COUNT_MASK                                                                   0x03FFFFFFL
+#define CP_DMA_PFP_COMMAND__SAS_MASK                                                                          0x04000000L
+#define CP_DMA_PFP_COMMAND__DAS_MASK                                                                          0x08000000L
+#define CP_DMA_PFP_COMMAND__SAIC_MASK                                                                         0x10000000L
+#define CP_DMA_PFP_COMMAND__DAIC_MASK                                                                         0x20000000L
+#define CP_DMA_PFP_COMMAND__RAW_WAIT_MASK                                                                     0x40000000L
+#define CP_DMA_PFP_COMMAND__DIS_WC_MASK                                                                       0x80000000L
+//CP_DMA_CNTL
+#define CP_DMA_CNTL__UTCL1_FAULT_CONTROL__SHIFT                                                               0x0
+#define CP_DMA_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x4
+#define CP_DMA_CNTL__BUFFER_DEPTH__SHIFT                                                                      0x10
+#define CP_DMA_CNTL__PIO_FIFO_EMPTY__SHIFT                                                                    0x1c
+#define CP_DMA_CNTL__PIO_FIFO_FULL__SHIFT                                                                     0x1d
+#define CP_DMA_CNTL__PIO_COUNT__SHIFT                                                                         0x1e
+#define CP_DMA_CNTL__UTCL1_FAULT_CONTROL_MASK                                                                 0x00000001L
+#define CP_DMA_CNTL__MIN_AVAILSZ_MASK                                                                         0x00000030L
+#define CP_DMA_CNTL__BUFFER_DEPTH_MASK                                                                        0x000F0000L
+#define CP_DMA_CNTL__PIO_FIFO_EMPTY_MASK                                                                      0x10000000L
+#define CP_DMA_CNTL__PIO_FIFO_FULL_MASK                                                                       0x20000000L
+#define CP_DMA_CNTL__PIO_COUNT_MASK                                                                           0xC0000000L
+//CP_DMA_READ_TAGS
+#define CP_DMA_READ_TAGS__DMA_READ_TAG__SHIFT                                                                 0x0
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID__SHIFT                                                           0x1c
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_MASK                                                                   0x03FFFFFFL
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID_MASK                                                             0x10000000L
+//CP_COHER_SIZE_HI
+#define CP_COHER_SIZE_HI__COHER_SIZE_HI_256B__SHIFT                                                           0x0
+#define CP_COHER_SIZE_HI__COHER_SIZE_HI_256B_MASK                                                             0x000000FFL
+//CP_PFP_IB_CONTROL
+#define CP_PFP_IB_CONTROL__IB_EN__SHIFT                                                                       0x0
+#define CP_PFP_IB_CONTROL__IB_EN_MASK                                                                         0x000000FFL
+//CP_PFP_LOAD_CONTROL
+#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN__SHIFT                                                             0x0
+#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN__SHIFT                                                               0x1
+#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN__SHIFT                                                             0x10
+#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN__SHIFT                                                              0x18
+#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN_MASK                                                               0x00000001L
+#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN_MASK                                                                 0x00000002L
+#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN_MASK                                                               0x00010000L
+#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN_MASK                                                                0x01000000L
+//CP_SCRATCH_INDEX
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                                0x0
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                                  0x000000FFL
+//CP_SCRATCH_DATA
+#define CP_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                                  0x0
+#define CP_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                    0xFFFFFFFFL
+//CP_RB_OFFSET
+#define CP_RB_OFFSET__RB_OFFSET__SHIFT                                                                        0x0
+#define CP_RB_OFFSET__RB_OFFSET_MASK                                                                          0x000FFFFFL
+//CP_IB1_OFFSET
+#define CP_IB1_OFFSET__IB1_OFFSET__SHIFT                                                                      0x0
+#define CP_IB1_OFFSET__IB1_OFFSET_MASK                                                                        0x000FFFFFL
+//CP_IB2_OFFSET
+#define CP_IB2_OFFSET__IB2_OFFSET__SHIFT                                                                      0x0
+#define CP_IB2_OFFSET__IB2_OFFSET_MASK                                                                        0x000FFFFFL
+//CP_IB1_PREAMBLE_BEGIN
+#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN__SHIFT                                                      0x0
+#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN_MASK                                                        0x000FFFFFL
+//CP_IB1_PREAMBLE_END
+#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END__SHIFT                                                          0x0
+#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END_MASK                                                            0x000FFFFFL
+//CP_IB2_PREAMBLE_BEGIN
+#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN__SHIFT                                                      0x0
+#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN_MASK                                                        0x000FFFFFL
+//CP_IB2_PREAMBLE_END
+#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END__SHIFT                                                          0x0
+#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END_MASK                                                            0x000FFFFFL
+//CP_CE_IB1_OFFSET
+#define CP_CE_IB1_OFFSET__IB1_OFFSET__SHIFT                                                                   0x0
+#define CP_CE_IB1_OFFSET__IB1_OFFSET_MASK                                                                     0x000FFFFFL
+//CP_CE_IB2_OFFSET
+#define CP_CE_IB2_OFFSET__IB2_OFFSET__SHIFT                                                                   0x0
+#define CP_CE_IB2_OFFSET__IB2_OFFSET_MASK                                                                     0x000FFFFFL
+//CP_CE_COUNTER
+#define CP_CE_COUNTER__CONST_ENGINE_COUNT__SHIFT                                                              0x0
+#define CP_CE_COUNTER__CONST_ENGINE_COUNT_MASK                                                                0xFFFFFFFFL
+//CP_CE_RB_OFFSET
+#define CP_CE_RB_OFFSET__RB_OFFSET__SHIFT                                                                     0x0
+#define CP_CE_RB_OFFSET__RB_OFFSET_MASK                                                                       0x000FFFFFL
+//CP_CE_INIT_CMD_BUFSZ
+#define CP_CE_INIT_CMD_BUFSZ__INIT_CMD_REQSZ__SHIFT                                                           0x0
+#define CP_CE_INIT_CMD_BUFSZ__INIT_CMD_REQSZ_MASK                                                             0x00000FFFL
+//CP_CE_IB1_CMD_BUFSZ
+#define CP_CE_IB1_CMD_BUFSZ__IB1_CMD_REQSZ__SHIFT                                                             0x0
+#define CP_CE_IB1_CMD_BUFSZ__IB1_CMD_REQSZ_MASK                                                               0x000FFFFFL
+//CP_CE_IB2_CMD_BUFSZ
+#define CP_CE_IB2_CMD_BUFSZ__IB2_CMD_REQSZ__SHIFT                                                             0x0
+#define CP_CE_IB2_CMD_BUFSZ__IB2_CMD_REQSZ_MASK                                                               0x000FFFFFL
+//CP_IB1_CMD_BUFSZ
+#define CP_IB1_CMD_BUFSZ__IB1_CMD_REQSZ__SHIFT                                                                0x0
+#define CP_IB1_CMD_BUFSZ__IB1_CMD_REQSZ_MASK                                                                  0x000FFFFFL
+//CP_IB2_CMD_BUFSZ
+#define CP_IB2_CMD_BUFSZ__IB2_CMD_REQSZ__SHIFT                                                                0x0
+#define CP_IB2_CMD_BUFSZ__IB2_CMD_REQSZ_MASK                                                                  0x000FFFFFL
+//CP_ST_CMD_BUFSZ
+#define CP_ST_CMD_BUFSZ__ST_CMD_REQSZ__SHIFT                                                                  0x0
+#define CP_ST_CMD_BUFSZ__ST_CMD_REQSZ_MASK                                                                    0x000FFFFFL
+//CP_CE_INIT_BASE_LO
+#define CP_CE_INIT_BASE_LO__INIT_BASE_LO__SHIFT                                                               0x5
+#define CP_CE_INIT_BASE_LO__INIT_BASE_LO_MASK                                                                 0xFFFFFFE0L
+//CP_CE_INIT_BASE_HI
+#define CP_CE_INIT_BASE_HI__INIT_BASE_HI__SHIFT                                                               0x0
+#define CP_CE_INIT_BASE_HI__INIT_BASE_HI_MASK                                                                 0x0000FFFFL
+//CP_CE_INIT_BUFSZ
+#define CP_CE_INIT_BUFSZ__INIT_BUFSZ__SHIFT                                                                   0x0
+#define CP_CE_INIT_BUFSZ__INIT_BUFSZ_MASK                                                                     0x00000FFFL
+//CP_CE_IB1_BASE_LO
+#define CP_CE_IB1_BASE_LO__IB1_BASE_LO__SHIFT                                                                 0x2
+#define CP_CE_IB1_BASE_LO__IB1_BASE_LO_MASK                                                                   0xFFFFFFFCL
+//CP_CE_IB1_BASE_HI
+#define CP_CE_IB1_BASE_HI__IB1_BASE_HI__SHIFT                                                                 0x0
+#define CP_CE_IB1_BASE_HI__IB1_BASE_HI_MASK                                                                   0x0000FFFFL
+//CP_CE_IB1_BUFSZ
+#define CP_CE_IB1_BUFSZ__IB1_BUFSZ__SHIFT                                                                     0x0
+#define CP_CE_IB1_BUFSZ__IB1_BUFSZ_MASK                                                                       0x000FFFFFL
+//CP_CE_IB2_BASE_LO
+#define CP_CE_IB2_BASE_LO__IB2_BASE_LO__SHIFT                                                                 0x2
+#define CP_CE_IB2_BASE_LO__IB2_BASE_LO_MASK                                                                   0xFFFFFFFCL
+//CP_CE_IB2_BASE_HI
+#define CP_CE_IB2_BASE_HI__IB2_BASE_HI__SHIFT                                                                 0x0
+#define CP_CE_IB2_BASE_HI__IB2_BASE_HI_MASK                                                                   0x0000FFFFL
+//CP_CE_IB2_BUFSZ
+#define CP_CE_IB2_BUFSZ__IB2_BUFSZ__SHIFT                                                                     0x0
+#define CP_CE_IB2_BUFSZ__IB2_BUFSZ_MASK                                                                       0x000FFFFFL
+//CP_IB1_BASE_LO
+#define CP_IB1_BASE_LO__IB1_BASE_LO__SHIFT                                                                    0x2
+#define CP_IB1_BASE_LO__IB1_BASE_LO_MASK                                                                      0xFFFFFFFCL
+//CP_IB1_BASE_HI
+#define CP_IB1_BASE_HI__IB1_BASE_HI__SHIFT                                                                    0x0
+#define CP_IB1_BASE_HI__IB1_BASE_HI_MASK                                                                      0x0000FFFFL
+//CP_IB1_BUFSZ
+#define CP_IB1_BUFSZ__IB1_BUFSZ__SHIFT                                                                        0x0
+#define CP_IB1_BUFSZ__IB1_BUFSZ_MASK                                                                          0x000FFFFFL
+//CP_IB2_BASE_LO
+#define CP_IB2_BASE_LO__IB2_BASE_LO__SHIFT                                                                    0x2
+#define CP_IB2_BASE_LO__IB2_BASE_LO_MASK                                                                      0xFFFFFFFCL
+//CP_IB2_BASE_HI
+#define CP_IB2_BASE_HI__IB2_BASE_HI__SHIFT                                                                    0x0
+#define CP_IB2_BASE_HI__IB2_BASE_HI_MASK                                                                      0x0000FFFFL
+//CP_IB2_BUFSZ
+#define CP_IB2_BUFSZ__IB2_BUFSZ__SHIFT                                                                        0x0
+#define CP_IB2_BUFSZ__IB2_BUFSZ_MASK                                                                          0x000FFFFFL
+//CP_ST_BASE_LO
+#define CP_ST_BASE_LO__ST_BASE_LO__SHIFT                                                                      0x2
+#define CP_ST_BASE_LO__ST_BASE_LO_MASK                                                                        0xFFFFFFFCL
+//CP_ST_BASE_HI
+#define CP_ST_BASE_HI__ST_BASE_HI__SHIFT                                                                      0x0
+#define CP_ST_BASE_HI__ST_BASE_HI_MASK                                                                        0x0000FFFFL
+//CP_ST_BUFSZ
+#define CP_ST_BUFSZ__ST_BUFSZ__SHIFT                                                                          0x0
+#define CP_ST_BUFSZ__ST_BUFSZ_MASK                                                                            0x000FFFFFL
+//CP_EOP_DONE_EVENT_CNTL
+#define CP_EOP_DONE_EVENT_CNTL__WBINV_TC_OP__SHIFT                                                            0x0
+#define CP_EOP_DONE_EVENT_CNTL__WBINV_ACTION_ENA__SHIFT                                                       0xc
+#define CP_EOP_DONE_EVENT_CNTL__CACHE_POLICY__SHIFT                                                           0x19
+#define CP_EOP_DONE_EVENT_CNTL__EXECUTE__SHIFT                                                                0x1c
+#define CP_EOP_DONE_EVENT_CNTL__WBINV_TC_OP_MASK                                                              0x0000007FL
+#define CP_EOP_DONE_EVENT_CNTL__WBINV_ACTION_ENA_MASK                                                         0x0003F000L
+#define CP_EOP_DONE_EVENT_CNTL__CACHE_POLICY_MASK                                                             0x02000000L
+#define CP_EOP_DONE_EVENT_CNTL__EXECUTE_MASK                                                                  0x10000000L
+//CP_EOP_DONE_DATA_CNTL
+#define CP_EOP_DONE_DATA_CNTL__DST_SEL__SHIFT                                                                 0x10
+#define CP_EOP_DONE_DATA_CNTL__INT_SEL__SHIFT                                                                 0x18
+#define CP_EOP_DONE_DATA_CNTL__DATA_SEL__SHIFT                                                                0x1d
+#define CP_EOP_DONE_DATA_CNTL__DST_SEL_MASK                                                                   0x00030000L
+#define CP_EOP_DONE_DATA_CNTL__INT_SEL_MASK                                                                   0x07000000L
+#define CP_EOP_DONE_DATA_CNTL__DATA_SEL_MASK                                                                  0xE0000000L
+//CP_EOP_DONE_CNTX_ID
+#define CP_EOP_DONE_CNTX_ID__CNTX_ID__SHIFT                                                                   0x0
+#define CP_EOP_DONE_CNTX_ID__CNTX_ID_MASK                                                                     0xFFFFFFFFL
+//CP_PFP_COMPLETION_STATUS
+#define CP_PFP_COMPLETION_STATUS__STATUS__SHIFT                                                               0x0
+#define CP_PFP_COMPLETION_STATUS__STATUS_MASK                                                                 0x00000003L
+//CP_CE_COMPLETION_STATUS
+#define CP_CE_COMPLETION_STATUS__STATUS__SHIFT                                                                0x0
+#define CP_CE_COMPLETION_STATUS__STATUS_MASK                                                                  0x00000003L
+//CP_PRED_NOT_VISIBLE
+#define CP_PRED_NOT_VISIBLE__NOT_VISIBLE__SHIFT                                                               0x0
+#define CP_PRED_NOT_VISIBLE__NOT_VISIBLE_MASK                                                                 0x00000001L
+//CP_PFP_METADATA_BASE_ADDR
+#define CP_PFP_METADATA_BASE_ADDR__ADDR_LO__SHIFT                                                             0x0
+#define CP_PFP_METADATA_BASE_ADDR__ADDR_LO_MASK                                                               0xFFFFFFFFL
+//CP_PFP_METADATA_BASE_ADDR_HI
+#define CP_PFP_METADATA_BASE_ADDR_HI__ADDR_HI__SHIFT                                                          0x0
+#define CP_PFP_METADATA_BASE_ADDR_HI__ADDR_HI_MASK                                                            0x0000FFFFL
+//CP_CE_METADATA_BASE_ADDR
+#define CP_CE_METADATA_BASE_ADDR__ADDR_LO__SHIFT                                                              0x0
+#define CP_CE_METADATA_BASE_ADDR__ADDR_LO_MASK                                                                0xFFFFFFFFL
+//CP_CE_METADATA_BASE_ADDR_HI
+#define CP_CE_METADATA_BASE_ADDR_HI__ADDR_HI__SHIFT                                                           0x0
+#define CP_CE_METADATA_BASE_ADDR_HI__ADDR_HI_MASK                                                             0x0000FFFFL
+//CP_DRAW_INDX_INDR_ADDR
+#define CP_DRAW_INDX_INDR_ADDR__ADDR_LO__SHIFT                                                                0x0
+#define CP_DRAW_INDX_INDR_ADDR__ADDR_LO_MASK                                                                  0xFFFFFFFFL
+//CP_DRAW_INDX_INDR_ADDR_HI
+#define CP_DRAW_INDX_INDR_ADDR_HI__ADDR_HI__SHIFT                                                             0x0
+#define CP_DRAW_INDX_INDR_ADDR_HI__ADDR_HI_MASK                                                               0x0000FFFFL
+//CP_DISPATCH_INDR_ADDR
+#define CP_DISPATCH_INDR_ADDR__ADDR_LO__SHIFT                                                                 0x0
+#define CP_DISPATCH_INDR_ADDR__ADDR_LO_MASK                                                                   0xFFFFFFFFL
+//CP_DISPATCH_INDR_ADDR_HI
+#define CP_DISPATCH_INDR_ADDR_HI__ADDR_HI__SHIFT                                                              0x0
+#define CP_DISPATCH_INDR_ADDR_HI__ADDR_HI_MASK                                                                0x0000FFFFL
+//CP_INDEX_BASE_ADDR
+#define CP_INDEX_BASE_ADDR__ADDR_LO__SHIFT                                                                    0x0
+#define CP_INDEX_BASE_ADDR__ADDR_LO_MASK                                                                      0xFFFFFFFFL
+//CP_INDEX_BASE_ADDR_HI
+#define CP_INDEX_BASE_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_INDEX_BASE_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
+//CP_INDEX_TYPE
+#define CP_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                      0x0
+#define CP_INDEX_TYPE__INDEX_TYPE_MASK                                                                        0x00000003L
+//CP_GDS_BKUP_ADDR
+#define CP_GDS_BKUP_ADDR__ADDR_LO__SHIFT                                                                      0x0
+#define CP_GDS_BKUP_ADDR__ADDR_LO_MASK                                                                        0xFFFFFFFFL
+//CP_GDS_BKUP_ADDR_HI
+#define CP_GDS_BKUP_ADDR_HI__ADDR_HI__SHIFT                                                                   0x0
+#define CP_GDS_BKUP_ADDR_HI__ADDR_HI_MASK                                                                     0x0000FFFFL
+//CP_SAMPLE_STATUS
+#define CP_SAMPLE_STATUS__Z_PASS_ACITVE__SHIFT                                                                0x0
+#define CP_SAMPLE_STATUS__STREAMOUT_ACTIVE__SHIFT                                                             0x1
+#define CP_SAMPLE_STATUS__PIPELINE_ACTIVE__SHIFT                                                              0x2
+#define CP_SAMPLE_STATUS__STIPPLE_ACTIVE__SHIFT                                                               0x3
+#define CP_SAMPLE_STATUS__VGT_BUFFERS_ACTIVE__SHIFT                                                           0x4
+#define CP_SAMPLE_STATUS__SCREEN_EXT_ACTIVE__SHIFT                                                            0x5
+#define CP_SAMPLE_STATUS__DRAW_INDIRECT_ACTIVE__SHIFT                                                         0x6
+#define CP_SAMPLE_STATUS__DISP_INDIRECT_ACTIVE__SHIFT                                                         0x7
+#define CP_SAMPLE_STATUS__Z_PASS_ACITVE_MASK                                                                  0x00000001L
+#define CP_SAMPLE_STATUS__STREAMOUT_ACTIVE_MASK                                                               0x00000002L
+#define CP_SAMPLE_STATUS__PIPELINE_ACTIVE_MASK                                                                0x00000004L
+#define CP_SAMPLE_STATUS__STIPPLE_ACTIVE_MASK                                                                 0x00000008L
+#define CP_SAMPLE_STATUS__VGT_BUFFERS_ACTIVE_MASK                                                             0x00000010L
+#define CP_SAMPLE_STATUS__SCREEN_EXT_ACTIVE_MASK                                                              0x00000020L
+#define CP_SAMPLE_STATUS__DRAW_INDIRECT_ACTIVE_MASK                                                           0x00000040L
+#define CP_SAMPLE_STATUS__DISP_INDIRECT_ACTIVE_MASK                                                           0x00000080L
+//CP_ME_COHER_CNTL
+#define CP_ME_COHER_CNTL__DEST_BASE_0_ENA__SHIFT                                                              0x0
+#define CP_ME_COHER_CNTL__DEST_BASE_1_ENA__SHIFT                                                              0x1
+#define CP_ME_COHER_CNTL__CB0_DEST_BASE_ENA__SHIFT                                                            0x6
+#define CP_ME_COHER_CNTL__CB1_DEST_BASE_ENA__SHIFT                                                            0x7
+#define CP_ME_COHER_CNTL__CB2_DEST_BASE_ENA__SHIFT                                                            0x8
+#define CP_ME_COHER_CNTL__CB3_DEST_BASE_ENA__SHIFT                                                            0x9
+#define CP_ME_COHER_CNTL__CB4_DEST_BASE_ENA__SHIFT                                                            0xa
+#define CP_ME_COHER_CNTL__CB5_DEST_BASE_ENA__SHIFT                                                            0xb
+#define CP_ME_COHER_CNTL__CB6_DEST_BASE_ENA__SHIFT                                                            0xc
+#define CP_ME_COHER_CNTL__CB7_DEST_BASE_ENA__SHIFT                                                            0xd
+#define CP_ME_COHER_CNTL__DB_DEST_BASE_ENA__SHIFT                                                             0xe
+#define CP_ME_COHER_CNTL__DEST_BASE_2_ENA__SHIFT                                                              0x13
+#define CP_ME_COHER_CNTL__DEST_BASE_3_ENA__SHIFT                                                              0x15
+#define CP_ME_COHER_CNTL__DEST_BASE_0_ENA_MASK                                                                0x00000001L
+#define CP_ME_COHER_CNTL__DEST_BASE_1_ENA_MASK                                                                0x00000002L
+#define CP_ME_COHER_CNTL__CB0_DEST_BASE_ENA_MASK                                                              0x00000040L
+#define CP_ME_COHER_CNTL__CB1_DEST_BASE_ENA_MASK                                                              0x00000080L
+#define CP_ME_COHER_CNTL__CB2_DEST_BASE_ENA_MASK                                                              0x00000100L
+#define CP_ME_COHER_CNTL__CB3_DEST_BASE_ENA_MASK                                                              0x00000200L
+#define CP_ME_COHER_CNTL__CB4_DEST_BASE_ENA_MASK                                                              0x00000400L
+#define CP_ME_COHER_CNTL__CB5_DEST_BASE_ENA_MASK                                                              0x00000800L
+#define CP_ME_COHER_CNTL__CB6_DEST_BASE_ENA_MASK                                                              0x00001000L
+#define CP_ME_COHER_CNTL__CB7_DEST_BASE_ENA_MASK                                                              0x00002000L
+#define CP_ME_COHER_CNTL__DB_DEST_BASE_ENA_MASK                                                               0x00004000L
+#define CP_ME_COHER_CNTL__DEST_BASE_2_ENA_MASK                                                                0x00080000L
+#define CP_ME_COHER_CNTL__DEST_BASE_3_ENA_MASK                                                                0x00200000L
+//CP_ME_COHER_SIZE
+#define CP_ME_COHER_SIZE__COHER_SIZE_256B__SHIFT                                                              0x0
+#define CP_ME_COHER_SIZE__COHER_SIZE_256B_MASK                                                                0xFFFFFFFFL
+//CP_ME_COHER_SIZE_HI
+#define CP_ME_COHER_SIZE_HI__COHER_SIZE_HI_256B__SHIFT                                                        0x0
+#define CP_ME_COHER_SIZE_HI__COHER_SIZE_HI_256B_MASK                                                          0x000000FFL
+//CP_ME_COHER_BASE
+#define CP_ME_COHER_BASE__COHER_BASE_256B__SHIFT                                                              0x0
+#define CP_ME_COHER_BASE__COHER_BASE_256B_MASK                                                                0xFFFFFFFFL
+//CP_ME_COHER_BASE_HI
+#define CP_ME_COHER_BASE_HI__COHER_BASE_HI_256B__SHIFT                                                        0x0
+#define CP_ME_COHER_BASE_HI__COHER_BASE_HI_256B_MASK                                                          0x000000FFL
+//CP_ME_COHER_STATUS
+#define CP_ME_COHER_STATUS__MATCHING_GFX_CNTX__SHIFT                                                          0x0
+#define CP_ME_COHER_STATUS__STATUS__SHIFT                                                                     0x1f
+#define CP_ME_COHER_STATUS__MATCHING_GFX_CNTX_MASK                                                            0x000000FFL
+#define CP_ME_COHER_STATUS__STATUS_MASK                                                                       0x80000000L
+//RLC_GPM_PERF_COUNT_0
+#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL__SHIFT                                                              0x0
+#define RLC_GPM_PERF_COUNT_0__SE_INDEX__SHIFT                                                                 0x4
+#define RLC_GPM_PERF_COUNT_0__SH_INDEX__SHIFT                                                                 0x8
+#define RLC_GPM_PERF_COUNT_0__CU_INDEX__SHIFT                                                                 0xc
+#define RLC_GPM_PERF_COUNT_0__EVENT_SEL__SHIFT                                                                0x10
+#define RLC_GPM_PERF_COUNT_0__UNUSED__SHIFT                                                                   0x12
+#define RLC_GPM_PERF_COUNT_0__ENABLE__SHIFT                                                                   0x14
+#define RLC_GPM_PERF_COUNT_0__RESERVED__SHIFT                                                                 0x15
+#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL_MASK                                                                0x0000000FL
+#define RLC_GPM_PERF_COUNT_0__SE_INDEX_MASK                                                                   0x000000F0L
+#define RLC_GPM_PERF_COUNT_0__SH_INDEX_MASK                                                                   0x00000F00L
+#define RLC_GPM_PERF_COUNT_0__CU_INDEX_MASK                                                                   0x0000F000L
+#define RLC_GPM_PERF_COUNT_0__EVENT_SEL_MASK                                                                  0x00030000L
+#define RLC_GPM_PERF_COUNT_0__UNUSED_MASK                                                                     0x000C0000L
+#define RLC_GPM_PERF_COUNT_0__ENABLE_MASK                                                                     0x00100000L
+#define RLC_GPM_PERF_COUNT_0__RESERVED_MASK                                                                   0xFFE00000L
+//RLC_GPM_PERF_COUNT_1
+#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL__SHIFT                                                              0x0
+#define RLC_GPM_PERF_COUNT_1__SE_INDEX__SHIFT                                                                 0x4
+#define RLC_GPM_PERF_COUNT_1__SH_INDEX__SHIFT                                                                 0x8
+#define RLC_GPM_PERF_COUNT_1__CU_INDEX__SHIFT                                                                 0xc
+#define RLC_GPM_PERF_COUNT_1__EVENT_SEL__SHIFT                                                                0x10
+#define RLC_GPM_PERF_COUNT_1__UNUSED__SHIFT                                                                   0x12
+#define RLC_GPM_PERF_COUNT_1__ENABLE__SHIFT                                                                   0x14
+#define RLC_GPM_PERF_COUNT_1__RESERVED__SHIFT                                                                 0x15
+#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL_MASK                                                                0x0000000FL
+#define RLC_GPM_PERF_COUNT_1__SE_INDEX_MASK                                                                   0x000000F0L
+#define RLC_GPM_PERF_COUNT_1__SH_INDEX_MASK                                                                   0x00000F00L
+#define RLC_GPM_PERF_COUNT_1__CU_INDEX_MASK                                                                   0x0000F000L
+#define RLC_GPM_PERF_COUNT_1__EVENT_SEL_MASK                                                                  0x00030000L
+#define RLC_GPM_PERF_COUNT_1__UNUSED_MASK                                                                     0x000C0000L
+#define RLC_GPM_PERF_COUNT_1__ENABLE_MASK                                                                     0x00100000L
+#define RLC_GPM_PERF_COUNT_1__RESERVED_MASK                                                                   0xFFE00000L
+//GRBM_GFX_INDEX
+#define GRBM_GFX_INDEX__INSTANCE_INDEX__SHIFT                                                                 0x0
+#define GRBM_GFX_INDEX__SH_INDEX__SHIFT                                                                       0x8
+#define GRBM_GFX_INDEX__SE_INDEX__SHIFT                                                                       0x10
+#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES__SHIFT                                                            0x1d
+#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES__SHIFT                                                      0x1e
+#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT                                                            0x1f
+#define GRBM_GFX_INDEX__INSTANCE_INDEX_MASK                                                                   0x000000FFL
+#define GRBM_GFX_INDEX__SH_INDEX_MASK                                                                         0x0000FF00L
+#define GRBM_GFX_INDEX__SE_INDEX_MASK                                                                         0x00FF0000L
+#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK                                                              0x20000000L
+#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK                                                        0x40000000L
+#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK                                                              0x80000000L
+//VGT_GSVS_RING_SIZE
+#define VGT_GSVS_RING_SIZE__MEM_SIZE__SHIFT                                                                   0x0
+#define VGT_GSVS_RING_SIZE__MEM_SIZE_MASK                                                                     0xFFFFFFFFL
+//VGT_PRIMITIVE_TYPE
+#define VGT_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT                                                                  0x0
+#define VGT_PRIMITIVE_TYPE__PRIM_TYPE_MASK                                                                    0x0000003FL
+//VGT_INDEX_TYPE
+#define VGT_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                     0x0
+#define VGT_INDEX_TYPE__PRIMGEN_EN__SHIFT                                                                     0x8
+#define VGT_INDEX_TYPE__INDEX_TYPE_MASK                                                                       0x00000003L
+#define VGT_INDEX_TYPE__PRIMGEN_EN_MASK                                                                       0x00000100L
+//VGT_STRMOUT_BUFFER_FILLED_SIZE_0
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_0__SIZE__SHIFT                                                         0x0
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_0__SIZE_MASK                                                           0xFFFFFFFFL
+//VGT_STRMOUT_BUFFER_FILLED_SIZE_1
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_1__SIZE__SHIFT                                                         0x0
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_1__SIZE_MASK                                                           0xFFFFFFFFL
+//VGT_STRMOUT_BUFFER_FILLED_SIZE_2
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_2__SIZE__SHIFT                                                         0x0
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_2__SIZE_MASK                                                           0xFFFFFFFFL
+//VGT_STRMOUT_BUFFER_FILLED_SIZE_3
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_3__SIZE__SHIFT                                                         0x0
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_3__SIZE_MASK                                                           0xFFFFFFFFL
+//VGT_MAX_VTX_INDX
+#define VGT_MAX_VTX_INDX__MAX_INDX__SHIFT                                                                     0x0
+#define VGT_MAX_VTX_INDX__MAX_INDX_MASK                                                                       0xFFFFFFFFL
+//VGT_MIN_VTX_INDX
+#define VGT_MIN_VTX_INDX__MIN_INDX__SHIFT                                                                     0x0
+#define VGT_MIN_VTX_INDX__MIN_INDX_MASK                                                                       0xFFFFFFFFL
+//VGT_INDX_OFFSET
+#define VGT_INDX_OFFSET__INDX_OFFSET__SHIFT                                                                   0x0
+#define VGT_INDX_OFFSET__INDX_OFFSET_MASK                                                                     0xFFFFFFFFL
+//VGT_MULTI_PRIM_IB_RESET_EN
+#define VGT_MULTI_PRIM_IB_RESET_EN__RESET_EN__SHIFT                                                           0x0
+#define VGT_MULTI_PRIM_IB_RESET_EN__MATCH_ALL_BITS__SHIFT                                                     0x1
+#define VGT_MULTI_PRIM_IB_RESET_EN__RESET_EN_MASK                                                             0x00000001L
+#define VGT_MULTI_PRIM_IB_RESET_EN__MATCH_ALL_BITS_MASK                                                       0x00000002L
+//VGT_NUM_INDICES
+#define VGT_NUM_INDICES__NUM_INDICES__SHIFT                                                                   0x0
+#define VGT_NUM_INDICES__NUM_INDICES_MASK                                                                     0xFFFFFFFFL
+//VGT_NUM_INSTANCES
+#define VGT_NUM_INSTANCES__NUM_INSTANCES__SHIFT                                                               0x0
+#define VGT_NUM_INSTANCES__NUM_INSTANCES_MASK                                                                 0xFFFFFFFFL
+//VGT_TF_RING_SIZE
+#define VGT_TF_RING_SIZE__SIZE__SHIFT                                                                         0x0
+#define VGT_TF_RING_SIZE__SIZE_MASK                                                                           0x0000FFFFL
+//VGT_HS_OFFCHIP_PARAM
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING__SHIFT                                                        0x0
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY__SHIFT                                                      0x9
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING_MASK                                                          0x000001FFL
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY_MASK                                                        0x00000600L
+//VGT_TF_MEMORY_BASE
+#define VGT_TF_MEMORY_BASE__BASE__SHIFT                                                                       0x0
+#define VGT_TF_MEMORY_BASE__BASE_MASK                                                                         0xFFFFFFFFL
+//VGT_TF_MEMORY_BASE_HI
+#define VGT_TF_MEMORY_BASE_HI__BASE_HI__SHIFT                                                                 0x0
+#define VGT_TF_MEMORY_BASE_HI__BASE_HI_MASK                                                                   0x000000FFL
+//WD_POS_BUF_BASE
+#define WD_POS_BUF_BASE__BASE__SHIFT                                                                          0x0
+#define WD_POS_BUF_BASE__BASE_MASK                                                                            0xFFFFFFFFL
+//WD_POS_BUF_BASE_HI
+#define WD_POS_BUF_BASE_HI__BASE_HI__SHIFT                                                                    0x0
+#define WD_POS_BUF_BASE_HI__BASE_HI_MASK                                                                      0x000000FFL
+//WD_CNTL_SB_BUF_BASE
+#define WD_CNTL_SB_BUF_BASE__BASE__SHIFT                                                                      0x0
+#define WD_CNTL_SB_BUF_BASE__BASE_MASK                                                                        0xFFFFFFFFL
+//WD_CNTL_SB_BUF_BASE_HI
+#define WD_CNTL_SB_BUF_BASE_HI__BASE_HI__SHIFT                                                                0x0
+#define WD_CNTL_SB_BUF_BASE_HI__BASE_HI_MASK                                                                  0x000000FFL
+//WD_INDEX_BUF_BASE
+#define WD_INDEX_BUF_BASE__BASE__SHIFT                                                                        0x0
+#define WD_INDEX_BUF_BASE__BASE_MASK                                                                          0xFFFFFFFFL
+//WD_INDEX_BUF_BASE_HI
+#define WD_INDEX_BUF_BASE_HI__BASE_HI__SHIFT                                                                  0x0
+#define WD_INDEX_BUF_BASE_HI__BASE_HI_MASK                                                                    0x000000FFL
+//IA_MULTI_VGT_PARAM
+#define IA_MULTI_VGT_PARAM__PRIMGROUP_SIZE__SHIFT                                                             0x0
+#define IA_MULTI_VGT_PARAM__PARTIAL_VS_WAVE_ON__SHIFT                                                         0x10
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOP__SHIFT                                                              0x11
+#define IA_MULTI_VGT_PARAM__PARTIAL_ES_WAVE_ON__SHIFT                                                         0x12
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOI__SHIFT                                                              0x13
+#define IA_MULTI_VGT_PARAM__WD_SWITCH_ON_EOP__SHIFT                                                           0x14
+#define IA_MULTI_VGT_PARAM__EN_INST_OPT_BASIC__SHIFT                                                          0x15
+#define IA_MULTI_VGT_PARAM__EN_INST_OPT_ADV__SHIFT                                                            0x16
+#define IA_MULTI_VGT_PARAM__HW_USE_ONLY__SHIFT                                                                0x17
+#define IA_MULTI_VGT_PARAM__PRIMGROUP_SIZE_MASK                                                               0x0000FFFFL
+#define IA_MULTI_VGT_PARAM__PARTIAL_VS_WAVE_ON_MASK                                                           0x00010000L
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOP_MASK                                                                0x00020000L
+#define IA_MULTI_VGT_PARAM__PARTIAL_ES_WAVE_ON_MASK                                                           0x00040000L
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOI_MASK                                                                0x00080000L
+#define IA_MULTI_VGT_PARAM__WD_SWITCH_ON_EOP_MASK                                                             0x00100000L
+#define IA_MULTI_VGT_PARAM__EN_INST_OPT_BASIC_MASK                                                            0x00200000L
+#define IA_MULTI_VGT_PARAM__EN_INST_OPT_ADV_MASK                                                              0x00400000L
+#define IA_MULTI_VGT_PARAM__HW_USE_ONLY_MASK                                                                  0x00800000L
+//VGT_INSTANCE_BASE_ID
+#define VGT_INSTANCE_BASE_ID__INSTANCE_BASE_ID__SHIFT                                                         0x0
+#define VGT_INSTANCE_BASE_ID__INSTANCE_BASE_ID_MASK                                                           0xFFFFFFFFL
+//PA_SU_LINE_STIPPLE_VALUE
+#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE__SHIFT                                                   0x0
+#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE_MASK                                                     0x00FFFFFFL
+//PA_SC_LINE_STIPPLE_STATE
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR__SHIFT                                                          0x0
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT__SHIFT                                                        0x8
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR_MASK                                                            0x0000000FL
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT_MASK                                                          0x0000FF00L
+//PA_SC_SCREEN_EXTENT_MIN_0
+#define PA_SC_SCREEN_EXTENT_MIN_0__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MIN_0__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MIN_0__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MIN_0__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_SCREEN_EXTENT_MAX_0
+#define PA_SC_SCREEN_EXTENT_MAX_0__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MAX_0__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MAX_0__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MAX_0__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_SCREEN_EXTENT_MIN_1
+#define PA_SC_SCREEN_EXTENT_MIN_1__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MIN_1__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MIN_1__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MIN_1__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_SCREEN_EXTENT_MAX_1
+#define PA_SC_SCREEN_EXTENT_MAX_1__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MAX_1__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MAX_1__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MAX_1__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_P3D_TRAP_SCREEN_HV_EN
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                              0x0
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                       0x1
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                                0x00000001L
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                         0x00000002L
+//PA_SC_P3D_TRAP_SCREEN_H
+#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD__SHIFT                                                               0x0
+#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD_MASK                                                                 0x00003FFFL
+//PA_SC_P3D_TRAP_SCREEN_V
+#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD__SHIFT                                                               0x0
+#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD_MASK                                                                 0x00003FFFL
+//PA_SC_P3D_TRAP_SCREEN_OCCURRENCE
+#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                        0x0
+#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                          0x0000FFFFL
+//PA_SC_P3D_TRAP_SCREEN_COUNT
+#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                             0x0
+#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT_MASK                                                               0x0000FFFFL
+//PA_SC_HP3D_TRAP_SCREEN_HV_EN
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                             0x0
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                      0x1
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                               0x00000001L
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                        0x00000002L
+//PA_SC_HP3D_TRAP_SCREEN_H
+#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD__SHIFT                                                              0x0
+#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD_MASK                                                                0x00003FFFL
+//PA_SC_HP3D_TRAP_SCREEN_V
+#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD__SHIFT                                                              0x0
+#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD_MASK                                                                0x00003FFFL
+//PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE
+#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                       0x0
+#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                         0x0000FFFFL
+//PA_SC_HP3D_TRAP_SCREEN_COUNT
+#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                            0x0
+#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT_MASK                                                              0x0000FFFFL
+//PA_SC_TRAP_SCREEN_HV_EN
+#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                                  0x0
+#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                           0x1
+#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                                    0x00000001L
+#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                             0x00000002L
+//PA_SC_TRAP_SCREEN_H
+#define PA_SC_TRAP_SCREEN_H__X_COORD__SHIFT                                                                   0x0
+#define PA_SC_TRAP_SCREEN_H__X_COORD_MASK                                                                     0x00003FFFL
+//PA_SC_TRAP_SCREEN_V
+#define PA_SC_TRAP_SCREEN_V__Y_COORD__SHIFT                                                                   0x0
+#define PA_SC_TRAP_SCREEN_V__Y_COORD_MASK                                                                     0x00003FFFL
+//PA_SC_TRAP_SCREEN_OCCURRENCE
+#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                            0x0
+#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                              0x0000FFFFL
+//PA_SC_TRAP_SCREEN_COUNT
+#define PA_SC_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                                 0x0
+#define PA_SC_TRAP_SCREEN_COUNT__COUNT_MASK                                                                   0x0000FFFFL
+//PA_STATE_STEREO_X
+#define PA_STATE_STEREO_X__STEREO_X_OFFSET__SHIFT                                                             0x0
+#define PA_STATE_STEREO_X__STEREO_X_OFFSET_MASK                                                               0xFFFFFFFFL
+//SQ_THREAD_TRACE_BASE
+#define SQ_THREAD_TRACE_BASE__ADDR__SHIFT                                                                     0x0
+#define SQ_THREAD_TRACE_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SQ_THREAD_TRACE_SIZE
+#define SQ_THREAD_TRACE_SIZE__SIZE__SHIFT                                                                     0x0
+#define SQ_THREAD_TRACE_SIZE__SIZE_MASK                                                                       0x003FFFFFL
+//SQ_THREAD_TRACE_MASK
+#define SQ_THREAD_TRACE_MASK__CU_SEL__SHIFT                                                                   0x0
+#define SQ_THREAD_TRACE_MASK__SH_SEL__SHIFT                                                                   0x5
+#define SQ_THREAD_TRACE_MASK__REG_STALL_EN__SHIFT                                                             0x7
+#define SQ_THREAD_TRACE_MASK__SIMD_EN__SHIFT                                                                  0x8
+#define SQ_THREAD_TRACE_MASK__VM_ID_MASK__SHIFT                                                               0xc
+#define SQ_THREAD_TRACE_MASK__SPI_STALL_EN__SHIFT                                                             0xe
+#define SQ_THREAD_TRACE_MASK__SQ_STALL_EN__SHIFT                                                              0xf
+#define SQ_THREAD_TRACE_MASK__CU_SEL_MASK                                                                     0x0000001FL
+#define SQ_THREAD_TRACE_MASK__SH_SEL_MASK                                                                     0x00000020L
+#define SQ_THREAD_TRACE_MASK__REG_STALL_EN_MASK                                                               0x00000080L
+#define SQ_THREAD_TRACE_MASK__SIMD_EN_MASK                                                                    0x00000F00L
+#define SQ_THREAD_TRACE_MASK__VM_ID_MASK_MASK                                                                 0x00003000L
+#define SQ_THREAD_TRACE_MASK__SPI_STALL_EN_MASK                                                               0x00004000L
+#define SQ_THREAD_TRACE_MASK__SQ_STALL_EN_MASK                                                                0x00008000L
+//SQ_THREAD_TRACE_TOKEN_MASK
+#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_MASK__SHIFT                                                         0x0
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_MASK__SHIFT                                                           0x10
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DROP_ON_STALL__SHIFT                                                  0x18
+#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_MASK_MASK                                                           0x0000FFFFL
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_MASK_MASK                                                             0x00FF0000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DROP_ON_STALL_MASK                                                    0x01000000L
+//SQ_THREAD_TRACE_PERF_MASK
+#define SQ_THREAD_TRACE_PERF_MASK__SH0_MASK__SHIFT                                                            0x0
+#define SQ_THREAD_TRACE_PERF_MASK__SH1_MASK__SHIFT                                                            0x10
+#define SQ_THREAD_TRACE_PERF_MASK__SH0_MASK_MASK                                                              0x0000FFFFL
+#define SQ_THREAD_TRACE_PERF_MASK__SH1_MASK_MASK                                                              0xFFFF0000L
+//SQ_THREAD_TRACE_CTRL
+#define SQ_THREAD_TRACE_CTRL__RESET_BUFFER__SHIFT                                                             0x1f
+#define SQ_THREAD_TRACE_CTRL__RESET_BUFFER_MASK                                                               0x80000000L
+//SQ_THREAD_TRACE_MODE
+#define SQ_THREAD_TRACE_MODE__MASK_PS__SHIFT                                                                  0x0
+#define SQ_THREAD_TRACE_MODE__MASK_VS__SHIFT                                                                  0x3
+#define SQ_THREAD_TRACE_MODE__MASK_GS__SHIFT                                                                  0x6
+#define SQ_THREAD_TRACE_MODE__MASK_ES__SHIFT                                                                  0x9
+#define SQ_THREAD_TRACE_MODE__MASK_HS__SHIFT                                                                  0xc
+#define SQ_THREAD_TRACE_MODE__MASK_LS__SHIFT                                                                  0xf
+#define SQ_THREAD_TRACE_MODE__MASK_CS__SHIFT                                                                  0x12
+#define SQ_THREAD_TRACE_MODE__MODE__SHIFT                                                                     0x15
+#define SQ_THREAD_TRACE_MODE__CAPTURE_MODE__SHIFT                                                             0x17
+#define SQ_THREAD_TRACE_MODE__AUTOFLUSH_EN__SHIFT                                                             0x19
+#define SQ_THREAD_TRACE_MODE__TC_PERF_EN__SHIFT                                                               0x1a
+#define SQ_THREAD_TRACE_MODE__ISSUE_MASK__SHIFT                                                               0x1b
+#define SQ_THREAD_TRACE_MODE__TEST_MODE__SHIFT                                                                0x1d
+#define SQ_THREAD_TRACE_MODE__INTERRUPT_EN__SHIFT                                                             0x1e
+#define SQ_THREAD_TRACE_MODE__WRAP__SHIFT                                                                     0x1f
+#define SQ_THREAD_TRACE_MODE__MASK_PS_MASK                                                                    0x00000007L
+#define SQ_THREAD_TRACE_MODE__MASK_VS_MASK                                                                    0x00000038L
+#define SQ_THREAD_TRACE_MODE__MASK_GS_MASK                                                                    0x000001C0L
+#define SQ_THREAD_TRACE_MODE__MASK_ES_MASK                                                                    0x00000E00L
+#define SQ_THREAD_TRACE_MODE__MASK_HS_MASK                                                                    0x00007000L
+#define SQ_THREAD_TRACE_MODE__MASK_LS_MASK                                                                    0x00038000L
+#define SQ_THREAD_TRACE_MODE__MASK_CS_MASK                                                                    0x001C0000L
+#define SQ_THREAD_TRACE_MODE__MODE_MASK                                                                       0x00600000L
+#define SQ_THREAD_TRACE_MODE__CAPTURE_MODE_MASK                                                               0x01800000L
+#define SQ_THREAD_TRACE_MODE__AUTOFLUSH_EN_MASK                                                               0x02000000L
+#define SQ_THREAD_TRACE_MODE__TC_PERF_EN_MASK                                                                 0x04000000L
+#define SQ_THREAD_TRACE_MODE__ISSUE_MASK_MASK                                                                 0x18000000L
+#define SQ_THREAD_TRACE_MODE__TEST_MODE_MASK                                                                  0x20000000L
+#define SQ_THREAD_TRACE_MODE__INTERRUPT_EN_MASK                                                               0x40000000L
+#define SQ_THREAD_TRACE_MODE__WRAP_MASK                                                                       0x80000000L
+//SQ_THREAD_TRACE_BASE2
+#define SQ_THREAD_TRACE_BASE2__ADDR_HI__SHIFT                                                                 0x0
+#define SQ_THREAD_TRACE_BASE2__ADDR_HI_MASK                                                                   0x0000000FL
+//SQ_THREAD_TRACE_TOKEN_MASK2
+#define SQ_THREAD_TRACE_TOKEN_MASK2__INST_MASK__SHIFT                                                         0x0
+#define SQ_THREAD_TRACE_TOKEN_MASK2__INST_MASK_MASK                                                           0xFFFFFFFFL
+//SQ_THREAD_TRACE_WPTR
+#define SQ_THREAD_TRACE_WPTR__WPTR__SHIFT                                                                     0x0
+#define SQ_THREAD_TRACE_WPTR__READ_OFFSET__SHIFT                                                              0x1e
+#define SQ_THREAD_TRACE_WPTR__WPTR_MASK                                                                       0x3FFFFFFFL
+#define SQ_THREAD_TRACE_WPTR__READ_OFFSET_MASK                                                                0xC0000000L
+//SQ_THREAD_TRACE_STATUS
+#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING__SHIFT                                                         0x0
+#define SQ_THREAD_TRACE_STATUS__FINISH_DONE__SHIFT                                                            0x10
+#define SQ_THREAD_TRACE_STATUS__UTC_ERROR__SHIFT                                                              0x1c
+#define SQ_THREAD_TRACE_STATUS__NEW_BUF__SHIFT                                                                0x1d
+#define SQ_THREAD_TRACE_STATUS__BUSY__SHIFT                                                                   0x1e
+#define SQ_THREAD_TRACE_STATUS__FULL__SHIFT                                                                   0x1f
+#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING_MASK                                                           0x000003FFL
+#define SQ_THREAD_TRACE_STATUS__FINISH_DONE_MASK                                                              0x03FF0000L
+#define SQ_THREAD_TRACE_STATUS__UTC_ERROR_MASK                                                                0x10000000L
+#define SQ_THREAD_TRACE_STATUS__NEW_BUF_MASK                                                                  0x20000000L
+#define SQ_THREAD_TRACE_STATUS__BUSY_MASK                                                                     0x40000000L
+#define SQ_THREAD_TRACE_STATUS__FULL_MASK                                                                     0x80000000L
+//SQ_THREAD_TRACE_HIWATER
+#define SQ_THREAD_TRACE_HIWATER__HIWATER__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_HIWATER__HIWATER_MASK                                                                 0x00000007L
+//SQ_THREAD_TRACE_CNTR
+#define SQ_THREAD_TRACE_CNTR__CNTR__SHIFT                                                                     0x0
+#define SQ_THREAD_TRACE_CNTR__CNTR_MASK                                                                       0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_0
+#define SQ_THREAD_TRACE_USERDATA_0__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_0__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_1
+#define SQ_THREAD_TRACE_USERDATA_1__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_1__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_2
+#define SQ_THREAD_TRACE_USERDATA_2__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_2__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_3
+#define SQ_THREAD_TRACE_USERDATA_3__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_3__DATA_MASK                                                                 0xFFFFFFFFL
+//SQC_CACHES
+#define SQC_CACHES__TARGET_INST__SHIFT                                                                        0x0
+#define SQC_CACHES__TARGET_DATA__SHIFT                                                                        0x1
+#define SQC_CACHES__INVALIDATE__SHIFT                                                                         0x2
+#define SQC_CACHES__WRITEBACK__SHIFT                                                                          0x3
+#define SQC_CACHES__VOL__SHIFT                                                                                0x4
+#define SQC_CACHES__COMPLETE__SHIFT                                                                           0x10
+#define SQC_CACHES__TARGET_INST_MASK                                                                          0x00000001L
+#define SQC_CACHES__TARGET_DATA_MASK                                                                          0x00000002L
+#define SQC_CACHES__INVALIDATE_MASK                                                                           0x00000004L
+#define SQC_CACHES__WRITEBACK_MASK                                                                            0x00000008L
+#define SQC_CACHES__VOL_MASK                                                                                  0x00000010L
+#define SQC_CACHES__COMPLETE_MASK                                                                             0x00010000L
+//SQC_WRITEBACK
+#define SQC_WRITEBACK__DWB__SHIFT                                                                             0x0
+#define SQC_WRITEBACK__DIRTY__SHIFT                                                                           0x1
+#define SQC_WRITEBACK__DWB_MASK                                                                               0x00000001L
+#define SQC_WRITEBACK__DIRTY_MASK                                                                             0x00000002L
+//DB_OCCLUSION_COUNT0_LOW
+#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT0_HI
+#define DB_OCCLUSION_COUNT0_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT0_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//DB_OCCLUSION_COUNT1_LOW
+#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT1_HI
+#define DB_OCCLUSION_COUNT1_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT1_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//DB_OCCLUSION_COUNT2_LOW
+#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT2_HI
+#define DB_OCCLUSION_COUNT2_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT2_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//DB_OCCLUSION_COUNT3_LOW
+#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT3_HI
+#define DB_OCCLUSION_COUNT3_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT3_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//DB_ZPASS_COUNT_LOW
+#define DB_ZPASS_COUNT_LOW__COUNT_LOW__SHIFT                                                                  0x0
+#define DB_ZPASS_COUNT_LOW__COUNT_LOW_MASK                                                                    0xFFFFFFFFL
+//DB_ZPASS_COUNT_HI
+#define DB_ZPASS_COUNT_HI__COUNT_HI__SHIFT                                                                    0x0
+#define DB_ZPASS_COUNT_HI__COUNT_HI_MASK                                                                      0x7FFFFFFFL
+//GDS_RD_ADDR
+#define GDS_RD_ADDR__READ_ADDR__SHIFT                                                                         0x0
+#define GDS_RD_ADDR__READ_ADDR_MASK                                                                           0xFFFFFFFFL
+//GDS_RD_DATA
+#define GDS_RD_DATA__READ_DATA__SHIFT                                                                         0x0
+#define GDS_RD_DATA__READ_DATA_MASK                                                                           0xFFFFFFFFL
+//GDS_RD_BURST_ADDR
+#define GDS_RD_BURST_ADDR__BURST_ADDR__SHIFT                                                                  0x0
+#define GDS_RD_BURST_ADDR__BURST_ADDR_MASK                                                                    0xFFFFFFFFL
+//GDS_RD_BURST_COUNT
+#define GDS_RD_BURST_COUNT__BURST_COUNT__SHIFT                                                                0x0
+#define GDS_RD_BURST_COUNT__BURST_COUNT_MASK                                                                  0xFFFFFFFFL
+//GDS_RD_BURST_DATA
+#define GDS_RD_BURST_DATA__BURST_DATA__SHIFT                                                                  0x0
+#define GDS_RD_BURST_DATA__BURST_DATA_MASK                                                                    0xFFFFFFFFL
+//GDS_WR_ADDR
+#define GDS_WR_ADDR__WRITE_ADDR__SHIFT                                                                        0x0
+#define GDS_WR_ADDR__WRITE_ADDR_MASK                                                                          0xFFFFFFFFL
+//GDS_WR_DATA
+#define GDS_WR_DATA__WRITE_DATA__SHIFT                                                                        0x0
+#define GDS_WR_DATA__WRITE_DATA_MASK                                                                          0xFFFFFFFFL
+//GDS_WR_BURST_ADDR
+#define GDS_WR_BURST_ADDR__WRITE_ADDR__SHIFT                                                                  0x0
+#define GDS_WR_BURST_ADDR__WRITE_ADDR_MASK                                                                    0xFFFFFFFFL
+//GDS_WR_BURST_DATA
+#define GDS_WR_BURST_DATA__WRITE_DATA__SHIFT                                                                  0x0
+#define GDS_WR_BURST_DATA__WRITE_DATA_MASK                                                                    0xFFFFFFFFL
+//GDS_WRITE_COMPLETE
+#define GDS_WRITE_COMPLETE__WRITE_COMPLETE__SHIFT                                                             0x0
+#define GDS_WRITE_COMPLETE__WRITE_COMPLETE_MASK                                                               0xFFFFFFFFL
+//GDS_ATOM_CNTL
+#define GDS_ATOM_CNTL__AINC__SHIFT                                                                            0x0
+#define GDS_ATOM_CNTL__UNUSED1__SHIFT                                                                         0x6
+#define GDS_ATOM_CNTL__DMODE__SHIFT                                                                           0x8
+#define GDS_ATOM_CNTL__UNUSED2__SHIFT                                                                         0xa
+#define GDS_ATOM_CNTL__AINC_MASK                                                                              0x0000003FL
+#define GDS_ATOM_CNTL__UNUSED1_MASK                                                                           0x000000C0L
+#define GDS_ATOM_CNTL__DMODE_MASK                                                                             0x00000300L
+#define GDS_ATOM_CNTL__UNUSED2_MASK                                                                           0xFFFFFC00L
+//GDS_ATOM_COMPLETE
+#define GDS_ATOM_COMPLETE__COMPLETE__SHIFT                                                                    0x0
+#define GDS_ATOM_COMPLETE__UNUSED__SHIFT                                                                      0x1
+#define GDS_ATOM_COMPLETE__COMPLETE_MASK                                                                      0x00000001L
+#define GDS_ATOM_COMPLETE__UNUSED_MASK                                                                        0xFFFFFFFEL
+//GDS_ATOM_BASE
+#define GDS_ATOM_BASE__BASE__SHIFT                                                                            0x0
+#define GDS_ATOM_BASE__UNUSED__SHIFT                                                                          0x10
+#define GDS_ATOM_BASE__BASE_MASK                                                                              0x0000FFFFL
+#define GDS_ATOM_BASE__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_ATOM_SIZE
+#define GDS_ATOM_SIZE__SIZE__SHIFT                                                                            0x0
+#define GDS_ATOM_SIZE__UNUSED__SHIFT                                                                          0x10
+#define GDS_ATOM_SIZE__SIZE_MASK                                                                              0x0000FFFFL
+#define GDS_ATOM_SIZE__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_ATOM_OFFSET0
+#define GDS_ATOM_OFFSET0__OFFSET0__SHIFT                                                                      0x0
+#define GDS_ATOM_OFFSET0__UNUSED__SHIFT                                                                       0x8
+#define GDS_ATOM_OFFSET0__OFFSET0_MASK                                                                        0x000000FFL
+#define GDS_ATOM_OFFSET0__UNUSED_MASK                                                                         0xFFFFFF00L
+//GDS_ATOM_OFFSET1
+#define GDS_ATOM_OFFSET1__OFFSET1__SHIFT                                                                      0x0
+#define GDS_ATOM_OFFSET1__UNUSED__SHIFT                                                                       0x8
+#define GDS_ATOM_OFFSET1__OFFSET1_MASK                                                                        0x000000FFL
+#define GDS_ATOM_OFFSET1__UNUSED_MASK                                                                         0xFFFFFF00L
+//GDS_ATOM_DST
+#define GDS_ATOM_DST__DST__SHIFT                                                                              0x0
+#define GDS_ATOM_DST__DST_MASK                                                                                0xFFFFFFFFL
+//GDS_ATOM_OP
+#define GDS_ATOM_OP__OP__SHIFT                                                                                0x0
+#define GDS_ATOM_OP__UNUSED__SHIFT                                                                            0x8
+#define GDS_ATOM_OP__OP_MASK                                                                                  0x000000FFL
+#define GDS_ATOM_OP__UNUSED_MASK                                                                              0xFFFFFF00L
+//GDS_ATOM_SRC0
+#define GDS_ATOM_SRC0__DATA__SHIFT                                                                            0x0
+#define GDS_ATOM_SRC0__DATA_MASK                                                                              0xFFFFFFFFL
+//GDS_ATOM_SRC0_U
+#define GDS_ATOM_SRC0_U__DATA__SHIFT                                                                          0x0
+#define GDS_ATOM_SRC0_U__DATA_MASK                                                                            0xFFFFFFFFL
+//GDS_ATOM_SRC1
+#define GDS_ATOM_SRC1__DATA__SHIFT                                                                            0x0
+#define GDS_ATOM_SRC1__DATA_MASK                                                                              0xFFFFFFFFL
+//GDS_ATOM_SRC1_U
+#define GDS_ATOM_SRC1_U__DATA__SHIFT                                                                          0x0
+#define GDS_ATOM_SRC1_U__DATA_MASK                                                                            0xFFFFFFFFL
+//GDS_ATOM_READ0
+#define GDS_ATOM_READ0__DATA__SHIFT                                                                           0x0
+#define GDS_ATOM_READ0__DATA_MASK                                                                             0xFFFFFFFFL
+//GDS_ATOM_READ0_U
+#define GDS_ATOM_READ0_U__DATA__SHIFT                                                                         0x0
+#define GDS_ATOM_READ0_U__DATA_MASK                                                                           0xFFFFFFFFL
+//GDS_ATOM_READ1
+#define GDS_ATOM_READ1__DATA__SHIFT                                                                           0x0
+#define GDS_ATOM_READ1__DATA_MASK                                                                             0xFFFFFFFFL
+//GDS_ATOM_READ1_U
+#define GDS_ATOM_READ1_U__DATA__SHIFT                                                                         0x0
+#define GDS_ATOM_READ1_U__DATA_MASK                                                                           0xFFFFFFFFL
+//GDS_GWS_RESOURCE_CNTL
+#define GDS_GWS_RESOURCE_CNTL__INDEX__SHIFT                                                                   0x0
+#define GDS_GWS_RESOURCE_CNTL__UNUSED__SHIFT                                                                  0x6
+#define GDS_GWS_RESOURCE_CNTL__INDEX_MASK                                                                     0x0000003FL
+#define GDS_GWS_RESOURCE_CNTL__UNUSED_MASK                                                                    0xFFFFFFC0L
+//GDS_GWS_RESOURCE
+#define GDS_GWS_RESOURCE__FLAG__SHIFT                                                                         0x0
+#define GDS_GWS_RESOURCE__COUNTER__SHIFT                                                                      0x1
+#define GDS_GWS_RESOURCE__TYPE__SHIFT                                                                         0xe
+#define GDS_GWS_RESOURCE__DED__SHIFT                                                                          0xf
+#define GDS_GWS_RESOURCE__RELEASE_ALL__SHIFT                                                                  0x10
+#define GDS_GWS_RESOURCE__HEAD_QUEUE__SHIFT                                                                   0x11
+#define GDS_GWS_RESOURCE__HEAD_VALID__SHIFT                                                                   0x1d
+#define GDS_GWS_RESOURCE__HEAD_FLAG__SHIFT                                                                    0x1e
+#define GDS_GWS_RESOURCE__HALTED__SHIFT                                                                       0x1f
+#define GDS_GWS_RESOURCE__FLAG_MASK                                                                           0x00000001L
+#define GDS_GWS_RESOURCE__COUNTER_MASK                                                                        0x00003FFEL
+#define GDS_GWS_RESOURCE__TYPE_MASK                                                                           0x00004000L
+#define GDS_GWS_RESOURCE__DED_MASK                                                                            0x00008000L
+#define GDS_GWS_RESOURCE__RELEASE_ALL_MASK                                                                    0x00010000L
+#define GDS_GWS_RESOURCE__HEAD_QUEUE_MASK                                                                     0x1FFE0000L
+#define GDS_GWS_RESOURCE__HEAD_VALID_MASK                                                                     0x20000000L
+#define GDS_GWS_RESOURCE__HEAD_FLAG_MASK                                                                      0x40000000L
+#define GDS_GWS_RESOURCE__HALTED_MASK                                                                         0x80000000L
+//GDS_GWS_RESOURCE_CNT
+#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT__SHIFT                                                             0x0
+#define GDS_GWS_RESOURCE_CNT__UNUSED__SHIFT                                                                   0x10
+#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT_MASK                                                               0x0000FFFFL
+#define GDS_GWS_RESOURCE_CNT__UNUSED_MASK                                                                     0xFFFF0000L
+//GDS_OA_CNTL
+#define GDS_OA_CNTL__INDEX__SHIFT                                                                             0x0
+#define GDS_OA_CNTL__UNUSED__SHIFT                                                                            0x4
+#define GDS_OA_CNTL__INDEX_MASK                                                                               0x0000000FL
+#define GDS_OA_CNTL__UNUSED_MASK                                                                              0xFFFFFFF0L
+//GDS_OA_COUNTER
+#define GDS_OA_COUNTER__SPACE_AVAILABLE__SHIFT                                                                0x0
+#define GDS_OA_COUNTER__SPACE_AVAILABLE_MASK                                                                  0xFFFFFFFFL
+//GDS_OA_ADDRESS
+#define GDS_OA_ADDRESS__DS_ADDRESS__SHIFT                                                                     0x0
+#define GDS_OA_ADDRESS__CRAWLER__SHIFT                                                                        0x10
+#define GDS_OA_ADDRESS__CRAWLER_TYPE__SHIFT                                                                   0x14
+#define GDS_OA_ADDRESS__UNUSED__SHIFT                                                                         0x16
+#define GDS_OA_ADDRESS__NO_ALLOC__SHIFT                                                                       0x1e
+#define GDS_OA_ADDRESS__ENABLE__SHIFT                                                                         0x1f
+#define GDS_OA_ADDRESS__DS_ADDRESS_MASK                                                                       0x0000FFFFL
+#define GDS_OA_ADDRESS__CRAWLER_MASK                                                                          0x000F0000L
+#define GDS_OA_ADDRESS__CRAWLER_TYPE_MASK                                                                     0x00300000L
+#define GDS_OA_ADDRESS__UNUSED_MASK                                                                           0x3FC00000L
+#define GDS_OA_ADDRESS__NO_ALLOC_MASK                                                                         0x40000000L
+#define GDS_OA_ADDRESS__ENABLE_MASK                                                                           0x80000000L
+//GDS_OA_INCDEC
+#define GDS_OA_INCDEC__VALUE__SHIFT                                                                           0x0
+#define GDS_OA_INCDEC__INCDEC__SHIFT                                                                          0x1f
+#define GDS_OA_INCDEC__VALUE_MASK                                                                             0x7FFFFFFFL
+#define GDS_OA_INCDEC__INCDEC_MASK                                                                            0x80000000L
+//GDS_OA_RING_SIZE
+#define GDS_OA_RING_SIZE__RING_SIZE__SHIFT                                                                    0x0
+#define GDS_OA_RING_SIZE__RING_SIZE_MASK                                                                      0xFFFFFFFFL
+//SPI_CONFIG_CNTL
+#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY__SHIFT                                                            0x0
+#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER__SHIFT                                                            0x15
+#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS__SHIFT                                                         0x18
+#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS__SHIFT                                                         0x19
+#define SPI_CONFIG_CNTL__RSRC_MGMT_RESET__SHIFT                                                               0x1a
+#define SPI_CONFIG_CNTL__TTRACE_STALL_ALL__SHIFT                                                              0x1b
+#define SPI_CONFIG_CNTL__ALLOC_ARB_LRU_ENA__SHIFT                                                             0x1c
+#define SPI_CONFIG_CNTL__EXP_ARB_LRU_ENA__SHIFT                                                               0x1d
+#define SPI_CONFIG_CNTL__PS_PKR_PRIORITY_CNTL__SHIFT                                                          0x1e
+#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY_MASK                                                              0x001FFFFFL
+#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER_MASK                                                              0x00E00000L
+#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS_MASK                                                           0x01000000L
+#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS_MASK                                                           0x02000000L
+#define SPI_CONFIG_CNTL__RSRC_MGMT_RESET_MASK                                                                 0x04000000L
+#define SPI_CONFIG_CNTL__TTRACE_STALL_ALL_MASK                                                                0x08000000L
+#define SPI_CONFIG_CNTL__ALLOC_ARB_LRU_ENA_MASK                                                               0x10000000L
+#define SPI_CONFIG_CNTL__EXP_ARB_LRU_ENA_MASK                                                                 0x20000000L
+#define SPI_CONFIG_CNTL__PS_PKR_PRIORITY_CNTL_MASK                                                            0xC0000000L
+//SPI_CONFIG_CNTL_1
+#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT                                                              0x0
+#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW__SHIFT                                                     0x4
+#define SPI_CONFIG_CNTL_1__BATON_RESET_DISABLE__SHIFT                                                         0x5
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE__SHIFT                                                             0x6
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT__SHIFT                                                             0x7
+#define SPI_CONFIG_CNTL_1__CRC_SIMD_ID_WADDR_DISABLE__SHIFT                                                   0x8
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_MODE__SHIFT                                                            0x9
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT__SHIFT                                                             0xa
+#define SPI_CONFIG_CNTL_1__CSC_PWR_SAVE_DISABLE__SHIFT                                                        0xe
+#define SPI_CONFIG_CNTL_1__CSG_PWR_SAVE_DISABLE__SHIFT                                                        0xf
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_SIZE__SHIFT                                                               0x10
+#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY_MASK                                                                0x0000000FL
+#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW_MASK                                                       0x00000010L
+#define SPI_CONFIG_CNTL_1__BATON_RESET_DISABLE_MASK                                                           0x00000020L
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE_MASK                                                               0x00000040L
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT_MASK                                                               0x00000080L
+#define SPI_CONFIG_CNTL_1__CRC_SIMD_ID_WADDR_DISABLE_MASK                                                     0x00000100L
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_MODE_MASK                                                              0x00000200L
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT_MASK                                                               0x00003C00L
+#define SPI_CONFIG_CNTL_1__CSC_PWR_SAVE_DISABLE_MASK                                                          0x00004000L
+#define SPI_CONFIG_CNTL_1__CSG_PWR_SAVE_DISABLE_MASK                                                          0x00008000L
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_SIZE_MASK                                                                 0xFFFF0000L
+//SPI_CONFIG_CNTL_2
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_REQUEST_CYCLE_OVHD__SHIFT                                    0x0
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_GRANT_CYCLE_OVHD__SHIFT                                      0x4
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_REQUEST_CYCLE_OVHD_MASK                                      0x0000000FL
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_GRANT_CYCLE_OVHD_MASK                                        0x000000F0L
+//SPI_WAVE_LIMIT_CNTL
+#define SPI_WAVE_LIMIT_CNTL__PS_WAVE_GRAN__SHIFT                                                              0x0
+#define SPI_WAVE_LIMIT_CNTL__VS_WAVE_GRAN__SHIFT                                                              0x2
+#define SPI_WAVE_LIMIT_CNTL__GS_WAVE_GRAN__SHIFT                                                              0x4
+#define SPI_WAVE_LIMIT_CNTL__HS_WAVE_GRAN__SHIFT                                                              0x6
+#define SPI_WAVE_LIMIT_CNTL__PS_WAVE_GRAN_MASK                                                                0x00000003L
+#define SPI_WAVE_LIMIT_CNTL__VS_WAVE_GRAN_MASK                                                                0x0000000CL
+#define SPI_WAVE_LIMIT_CNTL__GS_WAVE_GRAN_MASK                                                                0x00000030L
+#define SPI_WAVE_LIMIT_CNTL__HS_WAVE_GRAN_MASK                                                                0x000000C0L
+
+
+// addressBlock: xcd0_gc_perfddec
+//CPG_PERFCOUNTER1_LO
+#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPG_PERFCOUNTER1_HI
+#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPG_PERFCOUNTER0_LO
+#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPG_PERFCOUNTER0_HI
+#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER1_LO
+#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER1_HI
+#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER0_LO
+#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER0_HI
+#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER1_LO
+#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER1_HI
+#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER0_LO
+#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER0_HI
+#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPF_LATENCY_STATS_DATA
+#define CPF_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
+#define CPF_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//CPG_LATENCY_STATS_DATA
+#define CPG_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
+#define CPG_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//CPC_LATENCY_STATS_DATA
+#define CPC_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
+#define CPC_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//GRBM_PERFCOUNTER0_LO
+#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GRBM_PERFCOUNTER0_HI
+#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GRBM_PERFCOUNTER1_LO
+#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GRBM_PERFCOUNTER1_HI
+#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GRBM_SE0_PERFCOUNTER_LO
+#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
+#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
+//GRBM_SE0_PERFCOUNTER_HI
+#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
+#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
+//GRBM_SE1_PERFCOUNTER_LO
+#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
+#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
+//GRBM_SE1_PERFCOUNTER_HI
+#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
+#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
+//GRBM_SE2_PERFCOUNTER_LO
+#define GRBM_SE2_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
+#define GRBM_SE2_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
+//GRBM_SE2_PERFCOUNTER_HI
+#define GRBM_SE2_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
+#define GRBM_SE2_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
+//GRBM_SE3_PERFCOUNTER_LO
+#define GRBM_SE3_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
+#define GRBM_SE3_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
+//GRBM_SE3_PERFCOUNTER_HI
+#define GRBM_SE3_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
+#define GRBM_SE3_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
+//WD_PERFCOUNTER0_LO
+#define WD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define WD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//WD_PERFCOUNTER0_HI
+#define WD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define WD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//WD_PERFCOUNTER1_LO
+#define WD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define WD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//WD_PERFCOUNTER1_HI
+#define WD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define WD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//WD_PERFCOUNTER2_LO
+#define WD_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define WD_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//WD_PERFCOUNTER2_HI
+#define WD_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define WD_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//WD_PERFCOUNTER3_LO
+#define WD_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define WD_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//WD_PERFCOUNTER3_HI
+#define WD_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define WD_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//IA_PERFCOUNTER0_LO
+#define IA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define IA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//IA_PERFCOUNTER0_HI
+#define IA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define IA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//IA_PERFCOUNTER1_LO
+#define IA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define IA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//IA_PERFCOUNTER1_HI
+#define IA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define IA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//IA_PERFCOUNTER2_LO
+#define IA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define IA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//IA_PERFCOUNTER2_HI
+#define IA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define IA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//IA_PERFCOUNTER3_LO
+#define IA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define IA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//IA_PERFCOUNTER3_HI
+#define IA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define IA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//VGT_PERFCOUNTER0_LO
+#define VGT_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define VGT_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//VGT_PERFCOUNTER0_HI
+#define VGT_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define VGT_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//VGT_PERFCOUNTER1_LO
+#define VGT_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define VGT_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//VGT_PERFCOUNTER1_HI
+#define VGT_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define VGT_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//VGT_PERFCOUNTER2_LO
+#define VGT_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define VGT_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//VGT_PERFCOUNTER2_HI
+#define VGT_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define VGT_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//VGT_PERFCOUNTER3_LO
+#define VGT_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define VGT_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//VGT_PERFCOUNTER3_HI
+#define VGT_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define VGT_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//PA_SU_PERFCOUNTER0_LO
+#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER0_HI
+#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0x0000FFFFL
+//PA_SU_PERFCOUNTER1_LO
+#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER1_HI
+#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0x0000FFFFL
+//PA_SU_PERFCOUNTER2_LO
+#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER2_HI
+#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0x0000FFFFL
+//PA_SU_PERFCOUNTER3_LO
+#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER3_HI
+#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0x0000FFFFL
+//PA_SC_PERFCOUNTER0_LO
+#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER0_HI
+#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER1_LO
+#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER1_HI
+#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER2_LO
+#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER2_HI
+#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER3_LO
+#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER3_HI
+#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER4_LO
+#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER4_HI
+#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER5_LO
+#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER5_HI
+#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER6_LO
+#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER6_HI
+#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER7_LO
+#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER7_HI
+#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//SPI_PERFCOUNTER0_HI
+#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER0_LO
+#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER1_HI
+#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER1_LO
+#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER2_HI
+#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER2_LO
+#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER3_HI
+#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER3_LO
+#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER4_HI
+#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER4_LO
+#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER5_HI
+#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER5_LO
+#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER0_LO
+#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER0_HI
+#define SQ_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER1_LO
+#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER1_HI
+#define SQ_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER2_LO
+#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER2_HI
+#define SQ_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER3_LO
+#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER3_HI
+#define SQ_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER4_LO
+#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER4_HI
+#define SQ_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER5_LO
+#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER5_HI
+#define SQ_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER6_LO
+#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER6_HI
+#define SQ_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER7_LO
+#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER7_HI
+#define SQ_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER8_LO
+#define SQ_PERFCOUNTER8_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER8_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER8_HI
+#define SQ_PERFCOUNTER8_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER8_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER9_LO
+#define SQ_PERFCOUNTER9_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER9_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER9_HI
+#define SQ_PERFCOUNTER9_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER9_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER10_LO
+#define SQ_PERFCOUNTER10_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER10_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER10_HI
+#define SQ_PERFCOUNTER10_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER10_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER11_LO
+#define SQ_PERFCOUNTER11_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER11_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER11_HI
+#define SQ_PERFCOUNTER11_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER11_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER12_LO
+#define SQ_PERFCOUNTER12_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER12_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER12_HI
+#define SQ_PERFCOUNTER12_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER12_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER13_LO
+#define SQ_PERFCOUNTER13_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER13_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER13_HI
+#define SQ_PERFCOUNTER13_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER13_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER14_LO
+#define SQ_PERFCOUNTER14_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER14_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER14_HI
+#define SQ_PERFCOUNTER14_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER14_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER15_LO
+#define SQ_PERFCOUNTER15_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER15_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQ_PERFCOUNTER15_HI
+#define SQ_PERFCOUNTER15_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQ_PERFCOUNTER15_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SX_PERFCOUNTER0_LO
+#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER0_HI
+#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER1_LO
+#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER1_HI
+#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER2_LO
+#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER2_HI
+#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER3_LO
+#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER3_HI
+#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//GDS_PERFCOUNTER0_LO
+#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER0_HI
+#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER1_LO
+#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER1_HI
+#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER2_LO
+#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER2_HI
+#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER3_LO
+#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER3_HI
+#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TA_PERFCOUNTER0_LO
+#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TA_PERFCOUNTER0_HI
+#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TA_PERFCOUNTER1_LO
+#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TA_PERFCOUNTER1_HI
+#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER0_LO
+#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER0_HI
+#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER1_LO
+#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER1_HI
+#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TCP_PERFCOUNTER0_LO
+#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER0_HI
+#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER1_LO
+#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER1_HI
+#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER2_LO
+#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER2_HI
+#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER3_LO
+#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER3_HI
+#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCC_PERFCOUNTER0_LO
+#define TCC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCC_PERFCOUNTER0_HI
+#define TCC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCC_PERFCOUNTER1_LO
+#define TCC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCC_PERFCOUNTER1_HI
+#define TCC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCC_PERFCOUNTER2_LO
+#define TCC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCC_PERFCOUNTER2_HI
+#define TCC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCC_PERFCOUNTER3_LO
+#define TCC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCC_PERFCOUNTER3_HI
+#define TCC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCA_PERFCOUNTER0_LO
+#define TCA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCA_PERFCOUNTER0_HI
+#define TCA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCA_PERFCOUNTER1_LO
+#define TCA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCA_PERFCOUNTER1_HI
+#define TCA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCA_PERFCOUNTER2_LO
+#define TCA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCA_PERFCOUNTER2_HI
+#define TCA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCA_PERFCOUNTER3_LO
+#define TCA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCA_PERFCOUNTER3_HI
+#define TCA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CB_PERFCOUNTER0_LO
+#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER0_HI
+#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER1_LO
+#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER1_HI
+#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER2_LO
+#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER2_HI
+#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER3_LO
+#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER3_HI
+#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER0_LO
+#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER0_HI
+#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER1_LO
+#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER1_HI
+#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER2_LO
+#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER2_HI
+#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER3_LO
+#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER3_HI
+#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//RLC_PERFCOUNTER0_LO
+#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RLC_PERFCOUNTER0_HI
+#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RLC_PERFCOUNTER1_LO
+#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RLC_PERFCOUNTER1_HI
+#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER0_LO
+#define RMI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER0_HI
+#define RMI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER1_LO
+#define RMI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER1_HI
+#define RMI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER2_LO
+#define RMI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER2_HI
+#define RMI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER3_LO
+#define RMI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER3_HI
+#define RMI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+
+
+// addressBlock: xcd0_gc_utcl2_atcl2pfcntrdec
+//ATC_L2_PERFCOUNTER_LO
+#define ATC_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                              0x0
+#define ATC_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                0xFFFFFFFFL
+//ATC_L2_PERFCOUNTER_HI
+#define ATC_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                              0x0
+#define ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                           0x10
+#define ATC_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                0x0000FFFFL
+#define ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                             0xFFFF0000L
+
+
+// addressBlock: xcd0_gc_utcl2_vml2prdec
+//MC_VM_L2_PERFCOUNTER_LO
+#define MC_VM_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                              0xFFFFFFFFL
+//MC_VM_L2_PERFCOUNTER_HI
+#define MC_VM_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                         0x10
+#define MC_VM_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                              0x0000FFFFL
+#define MC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                           0xFFFF0000L
+
+
+// addressBlock: xcd0_gc_utcl2_l2tlbprdec
+//L2TLB_PERFCOUNTER_LO
+#define L2TLB_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//L2TLB_PERFCOUNTER_HI
+#define L2TLB_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define L2TLB_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define L2TLB_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+
+
+// addressBlock: xcd0_gc_perfsdec
+//CPG_PERFCOUNTER1_SELECT
+#define CPG_PERFCOUNTER1_SELECT__CNTR_SEL0__SHIFT                                                             0x0
+#define CPG_PERFCOUNTER1_SELECT__CNTR_SEL1__SHIFT                                                             0xa
+#define CPG_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPG_PERFCOUNTER1_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
+#define CPG_PERFCOUNTER1_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
+#define CPG_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CPG_PERFCOUNTER0_SELECT1
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_SEL2__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_SEL3__SHIFT                                                            0xa
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_SEL2_MASK                                                              0x000003FFL
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_SEL3_MASK                                                              0x000FFC00L
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
+//CPG_PERFCOUNTER0_SELECT
+#define CPG_PERFCOUNTER0_SELECT__CNTR_SEL0__SHIFT                                                             0x0
+#define CPG_PERFCOUNTER0_SELECT__CNTR_SEL1__SHIFT                                                             0xa
+#define CPG_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPG_PERFCOUNTER0_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
+#define CPG_PERFCOUNTER0_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
+#define CPG_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CPC_PERFCOUNTER1_SELECT
+#define CPC_PERFCOUNTER1_SELECT__CNTR_SEL0__SHIFT                                                             0x0
+#define CPC_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPC_PERFCOUNTER1_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPC_PERFCOUNTER1_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
+#define CPC_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPC_PERFCOUNTER1_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CPC_PERFCOUNTER0_SELECT1
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_SEL2__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_SEL3__SHIFT                                                            0xa
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_SEL2_MASK                                                              0x000003FFL
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_SEL3_MASK                                                              0x000FFC00L
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
+//CPF_PERFCOUNTER1_SELECT
+#define CPF_PERFCOUNTER1_SELECT__CNTR_SEL0__SHIFT                                                             0x0
+#define CPF_PERFCOUNTER1_SELECT__CNTR_SEL1__SHIFT                                                             0xa
+#define CPF_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPF_PERFCOUNTER1_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
+#define CPF_PERFCOUNTER1_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
+#define CPF_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CPF_PERFCOUNTER0_SELECT1
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_SEL2__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_SEL3__SHIFT                                                            0xa
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_SEL2_MASK                                                              0x000003FFL
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_SEL3_MASK                                                              0x000FFC00L
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
+//CPF_PERFCOUNTER0_SELECT
+#define CPF_PERFCOUNTER0_SELECT__CNTR_SEL0__SHIFT                                                             0x0
+#define CPF_PERFCOUNTER0_SELECT__CNTR_SEL1__SHIFT                                                             0xa
+#define CPF_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPF_PERFCOUNTER0_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
+#define CPF_PERFCOUNTER0_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
+#define CPF_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CP_PERFMON_CNTL
+#define CP_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                                 0x0
+#define CP_PERFMON_CNTL__SPM_PERFMON_STATE__SHIFT                                                             0x4
+#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE__SHIFT                                                           0x8
+#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT                                                         0xa
+#define CP_PERFMON_CNTL__PERFMON_STATE_MASK                                                                   0x0000000FL
+#define CP_PERFMON_CNTL__SPM_PERFMON_STATE_MASK                                                               0x000000F0L
+#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE_MASK                                                             0x00000300L
+#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK                                                           0x00000400L
+//CPC_PERFCOUNTER0_SELECT
+#define CPC_PERFCOUNTER0_SELECT__CNTR_SEL0__SHIFT                                                             0x0
+#define CPC_PERFCOUNTER0_SELECT__CNTR_SEL1__SHIFT                                                             0xa
+#define CPC_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPC_PERFCOUNTER0_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
+#define CPC_PERFCOUNTER0_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
+#define CPC_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CPF_TC_PERF_COUNTER_WINDOW_SELECT
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x00000007L
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
+//CPG_TC_PERF_COUNTER_WINDOW_SELECT
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x0000001FL
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
+//CPF_LATENCY_STATS_SELECT
+#define CPF_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
+#define CPF_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
+#define CPF_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
+#define CPF_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000000FL
+#define CPF_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
+#define CPF_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
+//CPG_LATENCY_STATS_SELECT
+#define CPG_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
+#define CPG_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
+#define CPG_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
+#define CPG_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000001FL
+#define CPG_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
+#define CPG_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
+//CPC_LATENCY_STATS_SELECT
+#define CPC_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
+#define CPC_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
+#define CPC_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
+#define CPC_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x00000007L
+#define CPC_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
+#define CPC_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
+//CP_DRAW_OBJECT
+#define CP_DRAW_OBJECT__OBJECT__SHIFT                                                                         0x0
+#define CP_DRAW_OBJECT__OBJECT_MASK                                                                           0xFFFFFFFFL
+//CP_DRAW_OBJECT_COUNTER
+#define CP_DRAW_OBJECT_COUNTER__COUNT__SHIFT                                                                  0x0
+#define CP_DRAW_OBJECT_COUNTER__COUNT_MASK                                                                    0x0000FFFFL
+//CP_DRAW_WINDOW_MASK_HI
+#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI__SHIFT                                                         0x0
+#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI_MASK                                                           0xFFFFFFFFL
+//CP_DRAW_WINDOW_HI
+#define CP_DRAW_WINDOW_HI__WINDOW_HI__SHIFT                                                                   0x0
+#define CP_DRAW_WINDOW_HI__WINDOW_HI_MASK                                                                     0xFFFFFFFFL
+//CP_DRAW_WINDOW_LO
+#define CP_DRAW_WINDOW_LO__MIN__SHIFT                                                                         0x0
+#define CP_DRAW_WINDOW_LO__MAX__SHIFT                                                                         0x10
+#define CP_DRAW_WINDOW_LO__MIN_MASK                                                                           0x0000FFFFL
+#define CP_DRAW_WINDOW_LO__MAX_MASK                                                                           0xFFFF0000L
+//CP_DRAW_WINDOW_CNTL
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX__SHIFT                                                0x0
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN__SHIFT                                                0x1
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI__SHIFT                                                    0x2
+#define CP_DRAW_WINDOW_CNTL__MODE__SHIFT                                                                      0x8
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX_MASK                                                  0x00000001L
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN_MASK                                                  0x00000002L
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI_MASK                                                      0x00000004L
+#define CP_DRAW_WINDOW_CNTL__MODE_MASK                                                                        0x00000100L
+//GRBM_PERFCOUNTER0_SELECT
+#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xa
+#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xb
+#define GRBM_PERFCOUNTER0_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                           0xc
+#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                            0xd
+#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                            0xe
+#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x10
+#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                            0x11
+#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x12
+#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT                                          0x13
+#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x14
+#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x15
+#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT                                            0x16
+#define GRBM_PERFCOUNTER0_SELECT__IA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x17
+#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT                                           0x18
+#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x19
+#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1a
+#define GRBM_PERFCOUNTER0_SELECT__TC_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1b
+#define GRBM_PERFCOUNTER0_SELECT__WD_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1c
+#define GRBM_PERFCOUNTER0_SELECT__UTCL2_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1d
+#define GRBM_PERFCOUNTER0_SELECT__EA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1e
+#define GRBM_PERFCOUNTER0_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1f
+#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x0000003FL
+#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000400L
+#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000800L
+#define GRBM_PERFCOUNTER0_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                             0x00001000L
+#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                              0x00002000L
+#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                              0x00004000L
+#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                             0x00010000L
+#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                              0x00020000L
+#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                              0x00040000L
+#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK                                            0x00080000L
+#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                              0x00100000L
+#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                              0x00200000L
+#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK                                              0x00400000L
+#define GRBM_PERFCOUNTER0_SELECT__IA_BUSY_USER_DEFINED_MASK_MASK                                              0x00800000L
+#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK                                             0x01000000L
+#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                             0x02000000L
+#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                             0x04000000L
+#define GRBM_PERFCOUNTER0_SELECT__TC_BUSY_USER_DEFINED_MASK_MASK                                              0x08000000L
+#define GRBM_PERFCOUNTER0_SELECT__WD_BUSY_USER_DEFINED_MASK_MASK                                              0x10000000L
+#define GRBM_PERFCOUNTER0_SELECT__UTCL2_BUSY_USER_DEFINED_MASK_MASK                                           0x20000000L
+#define GRBM_PERFCOUNTER0_SELECT__EA_BUSY_USER_DEFINED_MASK_MASK                                              0x40000000L
+#define GRBM_PERFCOUNTER0_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                             0x80000000L
+//GRBM_PERFCOUNTER1_SELECT
+#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xa
+#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xb
+#define GRBM_PERFCOUNTER1_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                           0xc
+#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                            0xd
+#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                            0xe
+#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x10
+#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                            0x11
+#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x12
+#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT                                          0x13
+#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x14
+#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x15
+#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT                                            0x16
+#define GRBM_PERFCOUNTER1_SELECT__IA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x17
+#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT                                           0x18
+#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x19
+#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1a
+#define GRBM_PERFCOUNTER1_SELECT__TC_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1b
+#define GRBM_PERFCOUNTER1_SELECT__WD_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1c
+#define GRBM_PERFCOUNTER1_SELECT__UTCL2_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1d
+#define GRBM_PERFCOUNTER1_SELECT__EA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1e
+#define GRBM_PERFCOUNTER1_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1f
+#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x0000003FL
+#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000400L
+#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000800L
+#define GRBM_PERFCOUNTER1_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                             0x00001000L
+#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                              0x00002000L
+#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                              0x00004000L
+#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                             0x00010000L
+#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                              0x00020000L
+#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                              0x00040000L
+#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK                                            0x00080000L
+#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                              0x00100000L
+#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                              0x00200000L
+#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK                                              0x00400000L
+#define GRBM_PERFCOUNTER1_SELECT__IA_BUSY_USER_DEFINED_MASK_MASK                                              0x00800000L
+#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK                                             0x01000000L
+#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                             0x02000000L
+#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                             0x04000000L
+#define GRBM_PERFCOUNTER1_SELECT__TC_BUSY_USER_DEFINED_MASK_MASK                                              0x08000000L
+#define GRBM_PERFCOUNTER1_SELECT__WD_BUSY_USER_DEFINED_MASK_MASK                                              0x10000000L
+#define GRBM_PERFCOUNTER1_SELECT__UTCL2_BUSY_USER_DEFINED_MASK_MASK                                           0x20000000L
+#define GRBM_PERFCOUNTER1_SELECT__EA_BUSY_USER_DEFINED_MASK_MASK                                              0x40000000L
+#define GRBM_PERFCOUNTER1_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                             0x80000000L
+//GRBM_SE0_PERFCOUNTER_SELECT
+#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
+#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
+#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
+#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
+#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
+#define GRBM_SE0_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                        0x13
+#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
+#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
+#define GRBM_SE0_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
+#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
+#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                          0x00080000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
+//GRBM_SE1_PERFCOUNTER_SELECT
+#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
+#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
+#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
+#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
+#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
+#define GRBM_SE1_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                        0x13
+#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
+#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
+#define GRBM_SE1_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
+#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
+#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                          0x00080000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
+//GRBM_SE2_PERFCOUNTER_SELECT
+#define GRBM_SE2_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
+#define GRBM_SE2_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
+#define GRBM_SE2_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
+#define GRBM_SE2_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
+#define GRBM_SE2_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
+#define GRBM_SE2_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                        0x13
+#define GRBM_SE2_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
+#define GRBM_SE2_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
+#define GRBM_SE2_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
+#define GRBM_SE2_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
+#define GRBM_SE2_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                          0x00080000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
+//GRBM_SE3_PERFCOUNTER_SELECT
+#define GRBM_SE3_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
+#define GRBM_SE3_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
+#define GRBM_SE3_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
+#define GRBM_SE3_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
+#define GRBM_SE3_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
+#define GRBM_SE3_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                        0x13
+#define GRBM_SE3_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
+#define GRBM_SE3_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
+#define GRBM_SE3_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
+#define GRBM_SE3_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
+#define GRBM_SE3_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                          0x00080000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
+//WD_PERFCOUNTER0_SELECT
+#define WD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define WD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define WD_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
+#define WD_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//WD_PERFCOUNTER1_SELECT
+#define WD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define WD_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define WD_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
+#define WD_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//WD_PERFCOUNTER2_SELECT
+#define WD_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define WD_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define WD_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
+#define WD_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//WD_PERFCOUNTER3_SELECT
+#define WD_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define WD_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define WD_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
+#define WD_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//IA_PERFCOUNTER0_SELECT
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define IA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define IA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//IA_PERFCOUNTER1_SELECT
+#define IA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define IA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define IA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
+#define IA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//IA_PERFCOUNTER2_SELECT
+#define IA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define IA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define IA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
+#define IA_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//IA_PERFCOUNTER3_SELECT
+#define IA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define IA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define IA_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
+#define IA_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//IA_PERFCOUNTER0_SELECT1
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//VGT_PERFCOUNTER0_SELECT
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define VGT_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define VGT_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//VGT_PERFCOUNTER1_SELECT
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define VGT_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define VGT_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//VGT_PERFCOUNTER2_SELECT
+#define VGT_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define VGT_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define VGT_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000000FFL
+#define VGT_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//VGT_PERFCOUNTER3_SELECT
+#define VGT_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define VGT_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define VGT_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000000FFL
+#define VGT_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//VGT_PERFCOUNTER0_SELECT1
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//VGT_PERFCOUNTER1_SELECT1
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//VGT_PERFCOUNTER_SEID_MASK
+#define VGT_PERFCOUNTER_SEID_MASK__PERF_SEID_IGNORE_MASK__SHIFT                                               0x0
+#define VGT_PERFCOUNTER_SEID_MASK__PERF_SEID_IGNORE_MASK_MASK                                                 0x000000FFL
+//PA_SU_PERFCOUNTER0_SELECT
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER0_SELECT1
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SU_PERFCOUNTER1_SELECT
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER1_SELECT1
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SU_PERFCOUNTER2_SELECT
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER3_SELECT
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SC_PERFCOUNTER0_SELECT
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SC_PERFCOUNTER0_SELECT1
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SC_PERFCOUNTER1_SELECT
+#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER2_SELECT
+#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER3_SELECT
+#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER4_SELECT
+#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER5_SELECT
+#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER6_SELECT
+#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER7_SELECT
+#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//SPI_PERFCOUNTER0_SELECT
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER1_SELECT
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER2_SELECT
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER3_SELECT
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER0_SELECT1
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER1_SELECT1
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER2_SELECT1
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER3_SELECT1
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER4_SELECT
+#define SPI_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                0x000000FFL
+//SPI_PERFCOUNTER5_SELECT
+#define SPI_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                0x000000FFL
+//SPI_PERFCOUNTER_BINS
+#define SPI_PERFCOUNTER_BINS__BIN0_MIN__SHIFT                                                                 0x0
+#define SPI_PERFCOUNTER_BINS__BIN0_MAX__SHIFT                                                                 0x4
+#define SPI_PERFCOUNTER_BINS__BIN1_MIN__SHIFT                                                                 0x8
+#define SPI_PERFCOUNTER_BINS__BIN1_MAX__SHIFT                                                                 0xc
+#define SPI_PERFCOUNTER_BINS__BIN2_MIN__SHIFT                                                                 0x10
+#define SPI_PERFCOUNTER_BINS__BIN2_MAX__SHIFT                                                                 0x14
+#define SPI_PERFCOUNTER_BINS__BIN3_MIN__SHIFT                                                                 0x18
+#define SPI_PERFCOUNTER_BINS__BIN3_MAX__SHIFT                                                                 0x1c
+#define SPI_PERFCOUNTER_BINS__BIN0_MIN_MASK                                                                   0x0000000FL
+#define SPI_PERFCOUNTER_BINS__BIN0_MAX_MASK                                                                   0x000000F0L
+#define SPI_PERFCOUNTER_BINS__BIN1_MIN_MASK                                                                   0x00000F00L
+#define SPI_PERFCOUNTER_BINS__BIN1_MAX_MASK                                                                   0x0000F000L
+#define SPI_PERFCOUNTER_BINS__BIN2_MIN_MASK                                                                   0x000F0000L
+#define SPI_PERFCOUNTER_BINS__BIN2_MAX_MASK                                                                   0x00F00000L
+#define SPI_PERFCOUNTER_BINS__BIN3_MIN_MASK                                                                   0x0F000000L
+#define SPI_PERFCOUNTER_BINS__BIN3_MAX_MASK                                                                   0xF0000000L
+//SQ_PERFCOUNTER0_SELECT
+#define SQ_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER0_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
+#define SQ_PERFCOUNTER0_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
+#define SQ_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER0_SELECT__SIMD_MASK__SHIFT                                                              0x18
+#define SQ_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER0_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
+#define SQ_PERFCOUNTER0_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
+#define SQ_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER0_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
+#define SQ_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER1_SELECT
+#define SQ_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER1_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
+#define SQ_PERFCOUNTER1_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
+#define SQ_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER1_SELECT__SIMD_MASK__SHIFT                                                              0x18
+#define SQ_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER1_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
+#define SQ_PERFCOUNTER1_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
+#define SQ_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER1_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
+#define SQ_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER2_SELECT
+#define SQ_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER2_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
+#define SQ_PERFCOUNTER2_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
+#define SQ_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER2_SELECT__SIMD_MASK__SHIFT                                                              0x18
+#define SQ_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER2_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
+#define SQ_PERFCOUNTER2_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
+#define SQ_PERFCOUNTER2_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER2_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
+#define SQ_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER3_SELECT
+#define SQ_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER3_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
+#define SQ_PERFCOUNTER3_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
+#define SQ_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER3_SELECT__SIMD_MASK__SHIFT                                                              0x18
+#define SQ_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER3_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
+#define SQ_PERFCOUNTER3_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
+#define SQ_PERFCOUNTER3_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER3_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
+#define SQ_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER4_SELECT
+#define SQ_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER4_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
+#define SQ_PERFCOUNTER4_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
+#define SQ_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER4_SELECT__SIMD_MASK__SHIFT                                                              0x18
+#define SQ_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER4_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
+#define SQ_PERFCOUNTER4_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
+#define SQ_PERFCOUNTER4_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER4_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
+#define SQ_PERFCOUNTER4_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER5_SELECT
+#define SQ_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER5_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
+#define SQ_PERFCOUNTER5_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
+#define SQ_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER5_SELECT__SIMD_MASK__SHIFT                                                              0x18
+#define SQ_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER5_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
+#define SQ_PERFCOUNTER5_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
+#define SQ_PERFCOUNTER5_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER5_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
+#define SQ_PERFCOUNTER5_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER6_SELECT
+#define SQ_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER6_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
+#define SQ_PERFCOUNTER6_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
+#define SQ_PERFCOUNTER6_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER6_SELECT__SIMD_MASK__SHIFT                                                              0x18
+#define SQ_PERFCOUNTER6_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER6_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
+#define SQ_PERFCOUNTER6_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
+#define SQ_PERFCOUNTER6_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER6_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
+#define SQ_PERFCOUNTER6_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER7_SELECT
+#define SQ_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER7_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
+#define SQ_PERFCOUNTER7_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
+#define SQ_PERFCOUNTER7_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER7_SELECT__SIMD_MASK__SHIFT                                                              0x18
+#define SQ_PERFCOUNTER7_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER7_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
+#define SQ_PERFCOUNTER7_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
+#define SQ_PERFCOUNTER7_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER7_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
+#define SQ_PERFCOUNTER7_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER8_SELECT
+#define SQ_PERFCOUNTER8_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER8_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
+#define SQ_PERFCOUNTER8_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
+#define SQ_PERFCOUNTER8_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER8_SELECT__SIMD_MASK__SHIFT                                                              0x18
+#define SQ_PERFCOUNTER8_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER8_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER8_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
+#define SQ_PERFCOUNTER8_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
+#define SQ_PERFCOUNTER8_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER8_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
+#define SQ_PERFCOUNTER8_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER9_SELECT
+#define SQ_PERFCOUNTER9_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER9_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
+#define SQ_PERFCOUNTER9_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
+#define SQ_PERFCOUNTER9_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER9_SELECT__SIMD_MASK__SHIFT                                                              0x18
+#define SQ_PERFCOUNTER9_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER9_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER9_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
+#define SQ_PERFCOUNTER9_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
+#define SQ_PERFCOUNTER9_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER9_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
+#define SQ_PERFCOUNTER9_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER10_SELECT
+#define SQ_PERFCOUNTER10_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER10_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
+#define SQ_PERFCOUNTER10_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
+#define SQ_PERFCOUNTER10_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER10_SELECT__SIMD_MASK__SHIFT                                                             0x18
+#define SQ_PERFCOUNTER10_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER10_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER10_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
+#define SQ_PERFCOUNTER10_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
+#define SQ_PERFCOUNTER10_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER10_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
+#define SQ_PERFCOUNTER10_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER11_SELECT
+#define SQ_PERFCOUNTER11_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER11_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
+#define SQ_PERFCOUNTER11_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
+#define SQ_PERFCOUNTER11_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER11_SELECT__SIMD_MASK__SHIFT                                                             0x18
+#define SQ_PERFCOUNTER11_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER11_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER11_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
+#define SQ_PERFCOUNTER11_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
+#define SQ_PERFCOUNTER11_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER11_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
+#define SQ_PERFCOUNTER11_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER12_SELECT
+#define SQ_PERFCOUNTER12_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER12_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
+#define SQ_PERFCOUNTER12_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
+#define SQ_PERFCOUNTER12_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER12_SELECT__SIMD_MASK__SHIFT                                                             0x18
+#define SQ_PERFCOUNTER12_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER12_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER12_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
+#define SQ_PERFCOUNTER12_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
+#define SQ_PERFCOUNTER12_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER12_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
+#define SQ_PERFCOUNTER12_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER13_SELECT
+#define SQ_PERFCOUNTER13_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER13_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
+#define SQ_PERFCOUNTER13_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
+#define SQ_PERFCOUNTER13_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER13_SELECT__SIMD_MASK__SHIFT                                                             0x18
+#define SQ_PERFCOUNTER13_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER13_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER13_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
+#define SQ_PERFCOUNTER13_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
+#define SQ_PERFCOUNTER13_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER13_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
+#define SQ_PERFCOUNTER13_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER14_SELECT
+#define SQ_PERFCOUNTER14_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER14_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
+#define SQ_PERFCOUNTER14_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
+#define SQ_PERFCOUNTER14_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER14_SELECT__SIMD_MASK__SHIFT                                                             0x18
+#define SQ_PERFCOUNTER14_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER14_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER14_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
+#define SQ_PERFCOUNTER14_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
+#define SQ_PERFCOUNTER14_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER14_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
+#define SQ_PERFCOUNTER14_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER15_SELECT
+#define SQ_PERFCOUNTER15_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER15_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
+#define SQ_PERFCOUNTER15_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
+#define SQ_PERFCOUNTER15_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER15_SELECT__SIMD_MASK__SHIFT                                                             0x18
+#define SQ_PERFCOUNTER15_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER15_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER15_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
+#define SQ_PERFCOUNTER15_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
+#define SQ_PERFCOUNTER15_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER15_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
+#define SQ_PERFCOUNTER15_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER_CTRL
+#define SQ_PERFCOUNTER_CTRL__PS_EN__SHIFT                                                                     0x0
+#define SQ_PERFCOUNTER_CTRL__VS_EN__SHIFT                                                                     0x1
+#define SQ_PERFCOUNTER_CTRL__GS_EN__SHIFT                                                                     0x2
+#define SQ_PERFCOUNTER_CTRL__ES_EN__SHIFT                                                                     0x3
+#define SQ_PERFCOUNTER_CTRL__HS_EN__SHIFT                                                                     0x4
+#define SQ_PERFCOUNTER_CTRL__LS_EN__SHIFT                                                                     0x5
+#define SQ_PERFCOUNTER_CTRL__CS_EN__SHIFT                                                                     0x6
+#define SQ_PERFCOUNTER_CTRL__CNTR_RATE__SHIFT                                                                 0x8
+#define SQ_PERFCOUNTER_CTRL__DISABLE_FLUSH__SHIFT                                                             0xd
+#define SQ_PERFCOUNTER_CTRL__VMID_MASK__SHIFT                                                                 0x10
+#define SQ_PERFCOUNTER_CTRL__PS_EN_MASK                                                                       0x00000001L
+#define SQ_PERFCOUNTER_CTRL__VS_EN_MASK                                                                       0x00000002L
+#define SQ_PERFCOUNTER_CTRL__GS_EN_MASK                                                                       0x00000004L
+#define SQ_PERFCOUNTER_CTRL__ES_EN_MASK                                                                       0x00000008L
+#define SQ_PERFCOUNTER_CTRL__HS_EN_MASK                                                                       0x00000010L
+#define SQ_PERFCOUNTER_CTRL__LS_EN_MASK                                                                       0x00000020L
+#define SQ_PERFCOUNTER_CTRL__CS_EN_MASK                                                                       0x00000040L
+#define SQ_PERFCOUNTER_CTRL__CNTR_RATE_MASK                                                                   0x00001F00L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_FLUSH_MASK                                                               0x00002000L
+#define SQ_PERFCOUNTER_CTRL__VMID_MASK_MASK                                                                   0xFFFF0000L
+//SQ_PERFCOUNTER_MASK
+#define SQ_PERFCOUNTER_MASK__SH0_MASK__SHIFT                                                                  0x0
+#define SQ_PERFCOUNTER_MASK__SH1_MASK__SHIFT                                                                  0x10
+#define SQ_PERFCOUNTER_MASK__SH0_MASK_MASK                                                                    0x0000FFFFL
+#define SQ_PERFCOUNTER_MASK__SH1_MASK_MASK                                                                    0xFFFF0000L
+//SQ_PERFCOUNTER_CTRL2
+#define SQ_PERFCOUNTER_CTRL2__FORCE_EN__SHIFT                                                                 0x0
+#define SQ_PERFCOUNTER_CTRL2__FORCE_EN_MASK                                                                   0x00000001L
+//SX_PERFCOUNTER0_SELECT
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define SX_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define SX_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER1_SELECT
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define SX_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define SX_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER2_SELECT
+#define SX_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER3_SELECT
+#define SX_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER0_SELECT1
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//SX_PERFCOUNTER1_SELECT1
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//GDS_PERFCOUNTER0_SELECT
+#define GDS_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GDS_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GDS_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GDS_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GDS_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GDS_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GDS_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GDS_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GDS_PERFCOUNTER1_SELECT
+#define GDS_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GDS_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GDS_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GDS_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GDS_PERFCOUNTER2_SELECT
+#define GDS_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GDS_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GDS_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GDS_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GDS_PERFCOUNTER3_SELECT
+#define GDS_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GDS_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GDS_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GDS_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GDS_PERFCOUNTER0_SELECT1
+#define GDS_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GDS_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GDS_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GDS_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GDS_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GDS_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GDS_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//TA_PERFCOUNTER0_SELECT
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define TA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x0000007FL
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x0001FC00L
+#define TA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TA_PERFCOUNTER0_SELECT1
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x0000007FL
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x0001FC00L
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//TA_PERFCOUNTER1_SELECT
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x0000007FL
+#define TA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TD_PERFCOUNTER0_SELECT
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define TD_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x0000003FL
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x0000FC00L
+#define TD_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TD_PERFCOUNTER0_SELECT1
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x0000003FL
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x0000FC00L
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//TD_PERFCOUNTER1_SELECT
+#define TD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TD_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TD_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TD_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x0000003FL
+#define TD_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TD_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TCP_PERFCOUNTER0_SELECT
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x0000007FL
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x0001FC00L
+#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER0_SELECT1
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x0000007FL
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x0001FC00L
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//TCP_PERFCOUNTER1_SELECT
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x0000007FL
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x0001FC00L
+#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER1_SELECT1
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x0000007FL
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x0001FC00L
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//TCP_PERFCOUNTER2_SELECT
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x0000007FL
+#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER3_SELECT
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x0000007FL
+#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCC_PERFCOUNTER0_SELECT
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define TCC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCC_PERFCOUNTER0_SELECT1
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x18
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x1c
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0x0F000000L
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0xF0000000L
+//TCC_PERFCOUNTER1_SELECT
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define TCC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCC_PERFCOUNTER1_SELECT1
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x18
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x1c
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0x0F000000L
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0xF0000000L
+//TCC_PERFCOUNTER2_SELECT
+#define TCC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCC_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCC_PERFCOUNTER3_SELECT
+#define TCC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCC_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCA_PERFCOUNTER0_SELECT
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define TCA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCA_PERFCOUNTER0_SELECT1
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x18
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x1c
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0x0F000000L
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0xF0000000L
+//TCA_PERFCOUNTER1_SELECT
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define TCA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCA_PERFCOUNTER1_SELECT1
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x18
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x1c
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0x0F000000L
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0xF0000000L
+//TCA_PERFCOUNTER2_SELECT
+#define TCA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCA_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCA_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCA_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCA_PERFCOUNTER3_SELECT
+#define TCA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCA_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCA_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCA_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCA_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CB_PERFCOUNTER_FILTER
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE__SHIFT                                                        0x0
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL__SHIFT                                                           0x1
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE__SHIFT                                                    0x4
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL__SHIFT                                                       0x5
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE__SHIFT                                                     0xa
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL__SHIFT                                                        0xb
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE__SHIFT                                                       0xc
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL__SHIFT                                                          0xd
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE__SHIFT                                               0x11
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL__SHIFT                                                  0x12
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE__SHIFT                                             0x15
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL__SHIFT                                                0x16
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE_MASK                                                          0x00000001L
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL_MASK                                                             0x0000000EL
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE_MASK                                                      0x00000010L
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL_MASK                                                         0x000003E0L
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE_MASK                                                       0x00000400L
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL_MASK                                                          0x00000800L
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE_MASK                                                         0x00001000L
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL_MASK                                                            0x0000E000L
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE_MASK                                                 0x00020000L
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL_MASK                                                    0x001C0000L
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE_MASK                                               0x00200000L
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL_MASK                                                  0x00C00000L
+//CB_PERFCOUNTER0_SELECT
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define CB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x0007FC00L
+#define CB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//CB_PERFCOUNTER0_SELECT1
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000001FFL
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x0007FC00L
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//CB_PERFCOUNTER1_SELECT
+#define CB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define CB_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//CB_PERFCOUNTER2_SELECT
+#define CB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define CB_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//CB_PERFCOUNTER3_SELECT
+#define CB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define CB_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER0_SELECT
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER0_SELECT1
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//DB_PERFCOUNTER1_SELECT
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER1_SELECT1
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//DB_PERFCOUNTER2_SELECT
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER3_SELECT
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//RLC_SPM_PERFMON_CNTL
+#define RLC_SPM_PERFMON_CNTL__RESERVED1__SHIFT                                                                0x0
+#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE__SHIFT                                                        0xc
+#define RLC_SPM_PERFMON_CNTL__RESERVED__SHIFT                                                                 0xe
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL__SHIFT                                                  0x10
+#define RLC_SPM_PERFMON_CNTL__RESERVED1_MASK                                                                  0x00000FFFL
+#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE_MASK                                                          0x00003000L
+#define RLC_SPM_PERFMON_CNTL__RESERVED_MASK                                                                   0x0000C000L
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL_MASK                                                    0xFFFF0000L
+//RLC_SPM_PERFMON_RING_BASE_LO
+#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO__SHIFT                                                     0x0
+#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO_MASK                                                       0xFFFFFFFFL
+//RLC_SPM_PERFMON_RING_BASE_HI
+#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI__SHIFT                                                     0x0
+#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED__SHIFT                                                         0x10
+#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI_MASK                                                       0x0000FFFFL
+#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED_MASK                                                           0xFFFF0000L
+//RLC_SPM_PERFMON_RING_SIZE
+#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE__SHIFT                                                      0x0
+#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE_MASK                                                        0xFFFFFFFFL
+//RLC_SPM_PERFMON_SEGMENT_SIZE
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__PERFMON_SEGMENT_SIZE__SHIFT                                             0x0
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED1__SHIFT                                                        0x8
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_LINE__SHIFT                                                  0xb
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE0_NUM_LINE__SHIFT                                                     0x10
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE1_NUM_LINE__SHIFT                                                     0x15
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE2_NUM_LINE__SHIFT                                                     0x1a
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED__SHIFT                                                         0x1f
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__PERFMON_SEGMENT_SIZE_MASK                                               0x000000FFL
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED1_MASK                                                          0x00000700L
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_LINE_MASK                                                    0x0000F800L
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE0_NUM_LINE_MASK                                                       0x001F0000L
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE1_NUM_LINE_MASK                                                       0x03E00000L
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE2_NUM_LINE_MASK                                                       0x7C000000L
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED_MASK                                                           0x80000000L
+//RLC_SPM_SE_MUXSEL_ADDR
+#define RLC_SPM_SE_MUXSEL_ADDR__PERFMON_SEL_ADDR__SHIFT                                                       0x0
+#define RLC_SPM_SE_MUXSEL_ADDR__RESERVED__SHIFT                                                               0x8
+#define RLC_SPM_SE_MUXSEL_ADDR__PERFMON_SEL_ADDR_MASK                                                         0x000000FFL
+#define RLC_SPM_SE_MUXSEL_ADDR__RESERVED_MASK                                                                 0xFFFFFF00L
+//RLC_SPM_SE_MUXSEL_DATA
+#define RLC_SPM_SE_MUXSEL_DATA__PERFMON_SEL_DATA__SHIFT                                                       0x0
+#define RLC_SPM_SE_MUXSEL_DATA__PERFMON_SEL_DATA_MASK                                                         0xFFFFFFFFL
+//RLC_SPM_CPG_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_CPC_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_CPF_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_CB_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
+#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
+#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
+#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
+//RLC_SPM_DB_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
+#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
+#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
+#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
+//RLC_SPM_PA_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
+#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
+#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
+#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
+//RLC_SPM_GDS_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_IA_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
+#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
+#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
+#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
+//RLC_SPM_SC_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
+#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
+#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
+#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
+//RLC_SPM_TCC_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_TCA_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_TCP_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_TA_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
+#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
+#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
+#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
+//RLC_SPM_TD_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
+#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
+#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
+#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
+//RLC_SPM_VGT_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_SPI_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_SQG_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_SX_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
+#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
+#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
+#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
+//RLC_SPM_GLOBAL_MUXSEL_ADDR
+#define RLC_SPM_GLOBAL_MUXSEL_ADDR__PERFMON_SEL_ADDR__SHIFT                                                   0x0
+#define RLC_SPM_GLOBAL_MUXSEL_ADDR__RESERVED__SHIFT                                                           0x7
+#define RLC_SPM_GLOBAL_MUXSEL_ADDR__PERFMON_SEL_ADDR_MASK                                                     0x0000007FL
+#define RLC_SPM_GLOBAL_MUXSEL_ADDR__RESERVED_MASK                                                             0xFFFFFF80L
+//RLC_SPM_GLOBAL_MUXSEL_DATA
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__PERFMON_SEL_DATA__SHIFT                                                   0x0
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__PERFMON_SEL_DATA_MASK                                                     0xFFFFFFFFL
+//RLC_SPM_RING_RDPTR
+#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR__SHIFT                                                         0x0
+#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR_MASK                                                           0xFFFFFFFFL
+//RLC_SPM_SEGMENT_THRESHOLD
+#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD__SHIFT                                               0x0
+#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD_MASK                                                 0xFFFFFFFFL
+//RLC_SPM_RMI_PERFMON_SAMPLE_DELAY
+#define RLC_SPM_RMI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
+#define RLC_SPM_RMI_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_RMI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
+#define RLC_SPM_RMI_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_SPM_PERFMON_SAMPLE_DELAY_MAX
+#define RLC_SPM_PERFMON_SAMPLE_DELAY_MAX__PERFMON_MAX_SAMPLE_DELAY__SHIFT                                     0x0
+#define RLC_SPM_PERFMON_SAMPLE_DELAY_MAX__RESERVED__SHIFT                                                     0x8
+#define RLC_SPM_PERFMON_SAMPLE_DELAY_MAX__PERFMON_MAX_SAMPLE_DELAY_MASK                                       0x000000FFL
+#define RLC_SPM_PERFMON_SAMPLE_DELAY_MAX__RESERVED_MASK                                                       0xFFFFFF00L
+//RLC_PERFMON_CNTL
+#define RLC_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                                0x0
+#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT                                                        0xa
+#define RLC_PERFMON_CNTL__PERFMON_STATE_MASK                                                                  0x00000007L
+#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK                                                          0x00000400L
+//RLC_PERFCOUNTER0_SELECT
+#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
+#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x00FFL
+//RLC_PERFCOUNTER1_SELECT
+#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
+#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x00FFL
+//RLC_GPU_IOV_PERF_CNT_CNTL
+#define RLC_GPU_IOV_PERF_CNT_CNTL__ENABLE__SHIFT                                                              0x0
+#define RLC_GPU_IOV_PERF_CNT_CNTL__MODE_SELECT__SHIFT                                                         0x1
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESET__SHIFT                                                               0x2
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESERVED__SHIFT                                                            0x3
+#define RLC_GPU_IOV_PERF_CNT_CNTL__ENABLE_MASK                                                                0x00000001L
+#define RLC_GPU_IOV_PERF_CNT_CNTL__MODE_SELECT_MASK                                                           0x00000002L
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESET_MASK                                                                 0x00000004L
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESERVED_MASK                                                              0xFFFFFFF8L
+//RLC_GPU_IOV_PERF_CNT_WR_ADDR
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__VFID__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__CNT_ID__SHIFT                                                           0x4
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__RESERVED__SHIFT                                                         0x6
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__VFID_MASK                                                               0x0000000FL
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__CNT_ID_MASK                                                             0x00000030L
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__RESERVED_MASK                                                           0xFFFFFFC0L
+//RLC_GPU_IOV_PERF_CNT_WR_DATA
+#define RLC_GPU_IOV_PERF_CNT_WR_DATA__DATA__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_WR_DATA__DATA_MASK                                                               0x0000000FL
+//RLC_GPU_IOV_PERF_CNT_RD_ADDR
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__VFID__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__CNT_ID__SHIFT                                                           0x4
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__RESERVED__SHIFT                                                         0x6
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__VFID_MASK                                                               0x0000000FL
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__CNT_ID_MASK                                                             0x00000030L
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__RESERVED_MASK                                                           0xFFFFFFC0L
+//RLC_GPU_IOV_PERF_CNT_RD_DATA
+#define RLC_GPU_IOV_PERF_CNT_RD_DATA__DATA__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_RD_DATA__DATA_MASK                                                               0x0000000FL
+//RMI_PERFCOUNTER0_SELECT
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define RMI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x0007FC00L
+#define RMI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERFCOUNTER0_SELECT1
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000001FFL
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x0007FC00L
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//RMI_PERFCOUNTER1_SELECT
+#define RMI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define RMI_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERFCOUNTER2_SELECT
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define RMI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x0007FC00L
+#define RMI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERFCOUNTER2_SELECT1
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000001FFL
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x0007FC00L
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//RMI_PERFCOUNTER3_SELECT
+#define RMI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define RMI_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERF_COUNTER_CNTL
+#define RMI_PERF_COUNTER_CNTL__TRANS_BASED_PERF_EN_SEL__SHIFT                                                 0x0
+#define RMI_PERF_COUNTER_CNTL__EVENT_BASED_PERF_EN_SEL__SHIFT                                                 0x2
+#define RMI_PERF_COUNTER_CNTL__TC_PERF_EN_SEL__SHIFT                                                          0x4
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK0__SHIFT                                                 0x6
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK1__SHIFT                                                 0x8
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_CID__SHIFT                                                        0xa
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_VMID__SHIFT                                                       0xe
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_BURST_LENGTH_THRESHOLD__SHIFT                                     0x13
+#define RMI_PERF_COUNTER_CNTL__PERF_SOFT_RESET__SHIFT                                                         0x19
+#define RMI_PERF_COUNTER_CNTL__PERF_CNTR_SPM_SEL__SHIFT                                                       0x1a
+#define RMI_PERF_COUNTER_CNTL__TRANS_BASED_PERF_EN_SEL_MASK                                                   0x00000003L
+#define RMI_PERF_COUNTER_CNTL__EVENT_BASED_PERF_EN_SEL_MASK                                                   0x0000000CL
+#define RMI_PERF_COUNTER_CNTL__TC_PERF_EN_SEL_MASK                                                            0x00000030L
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK0_MASK                                                   0x000000C0L
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK1_MASK                                                   0x00000300L
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_CID_MASK                                                          0x00003C00L
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_VMID_MASK                                                         0x0007C000L
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_BURST_LENGTH_THRESHOLD_MASK                                       0x01F80000L
+#define RMI_PERF_COUNTER_CNTL__PERF_SOFT_RESET_MASK                                                           0x02000000L
+#define RMI_PERF_COUNTER_CNTL__PERF_CNTR_SPM_SEL_MASK                                                         0x04000000L
+
+
+// addressBlock: xcd0_gc_utcl2_atcl2pfcntldec
+//ATC_L2_PERFCOUNTER0_CFG
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                              0x0
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                          0x8
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                             0x18
+#define ATC_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                0x1c
+#define ATC_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                 0x1d
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                0x000000FFL
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                            0x0000FF00L
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                               0x0F000000L
+#define ATC_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                  0x10000000L
+#define ATC_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                   0x20000000L
+//ATC_L2_PERFCOUNTER1_CFG
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                              0x0
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                          0x8
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                             0x18
+#define ATC_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                0x1c
+#define ATC_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                 0x1d
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                0x000000FFL
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                            0x0000FF00L
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                               0x0F000000L
+#define ATC_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                  0x10000000L
+#define ATC_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                   0x20000000L
+//ATC_L2_PERFCOUNTER_RSLT_CNTL
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                              0x0
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                    0x8
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                     0x10
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                       0x18
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                        0x19
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                             0x1a
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                0x0000000FL
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                      0x0000FF00L
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                       0x00FF0000L
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                         0x01000000L
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                          0x02000000L
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                               0x04000000L
+
+
+// addressBlock: xcd0_gc_utcl2_vml2pldec
+//MC_VM_L2_PERFCOUNTER0_CFG
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER1_CFG
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER2_CFG
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER2_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER2_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER3_CFG
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER3_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER3_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER4_CFG
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER4_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER4_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER4_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER4_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER5_CFG
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER5_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER5_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER5_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER5_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER6_CFG
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER6_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER6_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER6_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER6_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER7_CFG
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER7_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER7_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER7_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER7_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER_RSLT_CNTL
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                            0x0
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                  0x8
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                   0x10
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                     0x18
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                      0x19
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                           0x1a
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                              0x0000000FL
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                    0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                     0x00FF0000L
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                       0x01000000L
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                        0x02000000L
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                             0x04000000L
+
+
+// addressBlock: xcd0_gc_utcl2_l2tlbpldec
+//L2TLB_PERFCOUNTER0_CFG
+#define L2TLB_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define L2TLB_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define L2TLB_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define L2TLB_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define L2TLB_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define L2TLB_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define L2TLB_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define L2TLB_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define L2TLB_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//L2TLB_PERFCOUNTER1_CFG
+#define L2TLB_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define L2TLB_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define L2TLB_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define L2TLB_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define L2TLB_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define L2TLB_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define L2TLB_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define L2TLB_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define L2TLB_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//L2TLB_PERFCOUNTER2_CFG
+#define L2TLB_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define L2TLB_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                              0x18
+#define L2TLB_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                 0x1c
+#define L2TLB_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                  0x1d
+#define L2TLB_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define L2TLB_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define L2TLB_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define L2TLB_PERFCOUNTER2_CFG__ENABLE_MASK                                                                   0x10000000L
+#define L2TLB_PERFCOUNTER2_CFG__CLEAR_MASK                                                                    0x20000000L
+//L2TLB_PERFCOUNTER3_CFG
+#define L2TLB_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define L2TLB_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                              0x18
+#define L2TLB_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                                 0x1c
+#define L2TLB_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                                  0x1d
+#define L2TLB_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define L2TLB_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define L2TLB_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define L2TLB_PERFCOUNTER3_CFG__ENABLE_MASK                                                                   0x10000000L
+#define L2TLB_PERFCOUNTER3_CFG__CLEAR_MASK                                                                    0x20000000L
+//L2TLB_PERFCOUNTER_RSLT_CNTL
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+
+
+// addressBlock: xcd0_gc_gdflldec
+//GDFLL_EDC_HYSTERESIS_CNTL
+#define GDFLL_EDC_HYSTERESIS_CNTL__MAX_HYSTERESIS__SHIFT                                                      0x0
+#define GDFLL_EDC_HYSTERESIS_CNTL__MAX_HYSTERESIS_MASK                                                        0x000000FFL
+//GDFLL_EDC_HYSTERESIS_STAT
+#define GDFLL_EDC_HYSTERESIS_STAT__HYSTERESIS_CNT__SHIFT                                                      0x0
+#define GDFLL_EDC_HYSTERESIS_STAT__EDC__SHIFT                                                                 0x8
+#define GDFLL_EDC_HYSTERESIS_STAT__HYSTERESIS_CNT_MASK                                                        0x000000FFL
+#define GDFLL_EDC_HYSTERESIS_STAT__EDC_MASK                                                                   0x00000100L
+
+
+// addressBlock: xcd0_gc_rlcpdec
+//RLC_CNTL
+#define RLC_CNTL__RLC_ENABLE_F32__SHIFT                                                                       0x0
+#define RLC_CNTL__FORCE_RETRY__SHIFT                                                                          0x1
+#define RLC_CNTL__READ_CACHE_DISABLE__SHIFT                                                                   0x2
+#define RLC_CNTL__RLC_STEP_F32__SHIFT                                                                         0x3
+#define RLC_CNTL__RESERVED__SHIFT                                                                             0x4
+#define RLC_CNTL__RLC_ENABLE_F32_MASK                                                                         0x00000001L
+#define RLC_CNTL__FORCE_RETRY_MASK                                                                            0x00000002L
+#define RLC_CNTL__READ_CACHE_DISABLE_MASK                                                                     0x00000004L
+#define RLC_CNTL__RLC_STEP_F32_MASK                                                                           0x00000008L
+#define RLC_CNTL__RESERVED_MASK                                                                               0xFFFFFFF0L
+//RLC_CGCG_CGLS_CTRL_2
+#define RLC_CGCG_CGLS_CTRL_2__CANE_STAT_BUSY_MASK__SHIFT                                                      0x0
+#define RLC_CGCG_CGLS_CTRL_2__RESERVED__SHIFT                                                                 0x1
+#define RLC_CGCG_CGLS_CTRL_2__CANE_STAT_BUSY_MASK_MASK                                                        0x00000001L
+#define RLC_CGCG_CGLS_CTRL_2__RESERVED_MASK                                                                   0xFFFFFFFEL
+//RLC_STAT
+#define RLC_STAT__RLC_BUSY__SHIFT                                                                             0x0
+#define RLC_STAT__RLC_SRM_BUSY__SHIFT                                                                         0x1
+#define RLC_STAT__RLC_GPM_BUSY__SHIFT                                                                         0x2
+#define RLC_STAT__RLC_SPM_BUSY__SHIFT                                                                         0x3
+#define RLC_STAT__MC_BUSY__SHIFT                                                                              0x4
+#define RLC_STAT__RLC_THREAD_0_BUSY__SHIFT                                                                    0x5
+#define RLC_STAT__RLC_THREAD_1_BUSY__SHIFT                                                                    0x6
+#define RLC_STAT__RLC_THREAD_2_BUSY__SHIFT                                                                    0x7
+#define RLC_STAT__RESERVED__SHIFT                                                                             0x8
+#define RLC_STAT__RLC_BUSY_MASK                                                                               0x00000001L
+#define RLC_STAT__RLC_SRM_BUSY_MASK                                                                           0x00000002L
+#define RLC_STAT__RLC_GPM_BUSY_MASK                                                                           0x00000004L
+#define RLC_STAT__RLC_SPM_BUSY_MASK                                                                           0x00000008L
+#define RLC_STAT__MC_BUSY_MASK                                                                                0x00000010L
+#define RLC_STAT__RLC_THREAD_0_BUSY_MASK                                                                      0x00000020L
+#define RLC_STAT__RLC_THREAD_1_BUSY_MASK                                                                      0x00000040L
+#define RLC_STAT__RLC_THREAD_2_BUSY_MASK                                                                      0x00000080L
+#define RLC_STAT__RESERVED_MASK                                                                               0xFFFFFF00L
+//RLC_SAFE_MODE
+#define RLC_SAFE_MODE__CMD__SHIFT                                                                             0x0
+#define RLC_SAFE_MODE__MESSAGE__SHIFT                                                                         0x1
+#define RLC_SAFE_MODE__RESERVED1__SHIFT                                                                       0x5
+#define RLC_SAFE_MODE__RESPONSE__SHIFT                                                                        0x8
+#define RLC_SAFE_MODE__RESERVED__SHIFT                                                                        0xc
+#define RLC_SAFE_MODE__CMD_MASK                                                                               0x00000001L
+#define RLC_SAFE_MODE__MESSAGE_MASK                                                                           0x0000001EL
+#define RLC_SAFE_MODE__RESERVED1_MASK                                                                         0x000000E0L
+#define RLC_SAFE_MODE__RESPONSE_MASK                                                                          0x00000F00L
+#define RLC_SAFE_MODE__RESERVED_MASK                                                                          0xFFFFF000L
+//RLC_MEM_SLP_CNTL
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN__SHIFT                                                                0x0
+#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN__SHIFT                                                                0x1
+#define RLC_MEM_SLP_CNTL__RESERVED__SHIFT                                                                     0x2
+#define RLC_MEM_SLP_CNTL__RLC_LS_DS_BUSY_OVERRIDE__SHIFT                                                      0x7
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY__SHIFT                                                          0x8
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY__SHIFT                                                         0x10
+#define RLC_MEM_SLP_CNTL__RESERVED1__SHIFT                                                                    0x18
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK                                                                  0x00000001L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN_MASK                                                                  0x00000002L
+#define RLC_MEM_SLP_CNTL__RESERVED_MASK                                                                       0x0000007CL
+#define RLC_MEM_SLP_CNTL__RLC_LS_DS_BUSY_OVERRIDE_MASK                                                        0x00000080L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY_MASK                                                            0x0000FF00L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY_MASK                                                           0x00FF0000L
+#define RLC_MEM_SLP_CNTL__RESERVED1_MASK                                                                      0xFF000000L
+//SMU_RLC_RESPONSE
+#define SMU_RLC_RESPONSE__RESP__SHIFT                                                                         0x0
+#define SMU_RLC_RESPONSE__RESP_MASK                                                                           0xFFFFFFFFL
+//RLC_RLCV_SAFE_MODE
+#define RLC_RLCV_SAFE_MODE__CMD__SHIFT                                                                        0x0
+#define RLC_RLCV_SAFE_MODE__MESSAGE__SHIFT                                                                    0x1
+#define RLC_RLCV_SAFE_MODE__RESERVED1__SHIFT                                                                  0x5
+#define RLC_RLCV_SAFE_MODE__RESPONSE__SHIFT                                                                   0x8
+#define RLC_RLCV_SAFE_MODE__RESERVED__SHIFT                                                                   0xc
+#define RLC_RLCV_SAFE_MODE__CMD_MASK                                                                          0x00000001L
+#define RLC_RLCV_SAFE_MODE__MESSAGE_MASK                                                                      0x0000001EL
+#define RLC_RLCV_SAFE_MODE__RESERVED1_MASK                                                                    0x000000E0L
+#define RLC_RLCV_SAFE_MODE__RESPONSE_MASK                                                                     0x00000F00L
+#define RLC_RLCV_SAFE_MODE__RESERVED_MASK                                                                     0xFFFFF000L
+//RLC_SMU_SAFE_MODE
+#define RLC_SMU_SAFE_MODE__CMD__SHIFT                                                                         0x0
+#define RLC_SMU_SAFE_MODE__MESSAGE__SHIFT                                                                     0x1
+#define RLC_SMU_SAFE_MODE__RESERVED1__SHIFT                                                                   0x5
+#define RLC_SMU_SAFE_MODE__RESPONSE__SHIFT                                                                    0x8
+#define RLC_SMU_SAFE_MODE__RESERVED__SHIFT                                                                    0xc
+#define RLC_SMU_SAFE_MODE__CMD_MASK                                                                           0x00000001L
+#define RLC_SMU_SAFE_MODE__MESSAGE_MASK                                                                       0x0000001EL
+#define RLC_SMU_SAFE_MODE__RESERVED1_MASK                                                                     0x000000E0L
+#define RLC_SMU_SAFE_MODE__RESPONSE_MASK                                                                      0x00000F00L
+#define RLC_SMU_SAFE_MODE__RESERVED_MASK                                                                      0xFFFFF000L
+//RLC_RLCV_COMMAND
+#define RLC_RLCV_COMMAND__CMD__SHIFT                                                                          0x0
+#define RLC_RLCV_COMMAND__RESERVED__SHIFT                                                                     0x4
+#define RLC_RLCV_COMMAND__CMD_MASK                                                                            0x0000000FL
+#define RLC_RLCV_COMMAND__RESERVED_MASK                                                                       0xFFFFFFF0L
+//RLC_REFCLOCK_TIMESTAMP_LSB
+#define RLC_REFCLOCK_TIMESTAMP_LSB__TIMESTAMP_LSB__SHIFT                                                      0x0
+#define RLC_REFCLOCK_TIMESTAMP_LSB__TIMESTAMP_LSB_MASK                                                        0xFFFFFFFFL
+//RLC_REFCLOCK_TIMESTAMP_MSB
+#define RLC_REFCLOCK_TIMESTAMP_MSB__TIMESTAMP_MSB__SHIFT                                                      0x0
+#define RLC_REFCLOCK_TIMESTAMP_MSB__TIMESTAMP_MSB_MASK                                                        0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_0
+#define RLC_GPM_TIMER_INT_0__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_0__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_1
+#define RLC_GPM_TIMER_INT_1__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_1__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_2
+#define RLC_GPM_TIMER_INT_2__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_2__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_CTRL
+#define RLC_GPM_TIMER_CTRL__TIMER_0_EN__SHIFT                                                                 0x0
+#define RLC_GPM_TIMER_CTRL__TIMER_1_EN__SHIFT                                                                 0x1
+#define RLC_GPM_TIMER_CTRL__TIMER_2_EN__SHIFT                                                                 0x2
+#define RLC_GPM_TIMER_CTRL__TIMER_3_EN__SHIFT                                                                 0x3
+#define RLC_GPM_TIMER_CTRL__RESERVED__SHIFT                                                                   0x4
+#define RLC_GPM_TIMER_CTRL__TIMER_0_EN_MASK                                                                   0x00000001L
+#define RLC_GPM_TIMER_CTRL__TIMER_1_EN_MASK                                                                   0x00000002L
+#define RLC_GPM_TIMER_CTRL__TIMER_2_EN_MASK                                                                   0x00000004L
+#define RLC_GPM_TIMER_CTRL__TIMER_3_EN_MASK                                                                   0x00000008L
+#define RLC_GPM_TIMER_CTRL__RESERVED_MASK                                                                     0xFFFFFFF0L
+//RLC_LB_CNTR_MAX
+#define RLC_LB_CNTR_MAX__LB_CNTR_MAX__SHIFT                                                                   0x0
+#define RLC_LB_CNTR_MAX__LB_CNTR_MAX_MASK                                                                     0xFFFFFFFFL
+//RLC_GPM_TIMER_STAT
+#define RLC_GPM_TIMER_STAT__TIMER_0_STAT__SHIFT                                                               0x0
+#define RLC_GPM_TIMER_STAT__TIMER_1_STAT__SHIFT                                                               0x1
+#define RLC_GPM_TIMER_STAT__TIMER_2_STAT__SHIFT                                                               0x2
+#define RLC_GPM_TIMER_STAT__TIMER_3_STAT__SHIFT                                                               0x3
+#define RLC_GPM_TIMER_STAT__TIMER_0_ENABLE_SYNC__SHIFT                                                        0x8
+#define RLC_GPM_TIMER_STAT__TIMER_1_ENABLE_SYNC__SHIFT                                                        0x9
+#define RLC_GPM_TIMER_STAT__TIMER_2_ENABLE_SYNC__SHIFT                                                        0xa
+#define RLC_GPM_TIMER_STAT__TIMER_3_ENABLE_SYNC__SHIFT                                                        0xb
+#define RLC_GPM_TIMER_STAT__RESERVED__SHIFT                                                                   0xc
+#define RLC_GPM_TIMER_STAT__TIMER_0_STAT_MASK                                                                 0x00000001L
+#define RLC_GPM_TIMER_STAT__TIMER_1_STAT_MASK                                                                 0x00000002L
+#define RLC_GPM_TIMER_STAT__TIMER_2_STAT_MASK                                                                 0x00000004L
+#define RLC_GPM_TIMER_STAT__TIMER_3_STAT_MASK                                                                 0x00000008L
+#define RLC_GPM_TIMER_STAT__TIMER_0_ENABLE_SYNC_MASK                                                          0x00000100L
+#define RLC_GPM_TIMER_STAT__TIMER_1_ENABLE_SYNC_MASK                                                          0x00000200L
+#define RLC_GPM_TIMER_STAT__TIMER_2_ENABLE_SYNC_MASK                                                          0x00000400L
+#define RLC_GPM_TIMER_STAT__TIMER_3_ENABLE_SYNC_MASK                                                          0x00000800L
+#define RLC_GPM_TIMER_STAT__RESERVED_MASK                                                                     0xFFFFF000L
+//RLC_GPM_TIMER_INT_3
+#define RLC_GPM_TIMER_INT_3__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_3__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_WR_NONCU_MASTER_MASK_1
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SE_MASTER_MASK_1__SHIFT                                            0x0
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__GC_MASTER_MASK_1__SHIFT                                            0x10
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__GC_GFX_MASTER_MASK_1__SHIFT                                        0x11
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__TC0_1_MASTER_MASK__SHIFT                                           0x12
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__RESERVED_1__SHIFT                                                  0x13
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE4_MASTER_MASK__SHIFT                                          0x14
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE5_MASTER_MASK__SHIFT                                          0x15
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE6_MASTER_MASK__SHIFT                                          0x16
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE7_MASTER_MASK__SHIFT                                          0x17
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__EA_1_MASTER_MASK__SHIFT                                            0x18
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__RESERVED__SHIFT                                                    0x19
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SE_MASTER_MASK_1_MASK                                              0x0000FFFFL
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__GC_MASTER_MASK_1_MASK                                              0x00010000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__GC_GFX_MASTER_MASK_1_MASK                                          0x00020000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__TC0_1_MASTER_MASK_MASK                                             0x00040000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__RESERVED_1_MASK                                                    0x00080000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE4_MASTER_MASK_MASK                                            0x00100000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE5_MASTER_MASK_MASK                                            0x00200000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE6_MASTER_MASK_MASK                                            0x00400000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE7_MASTER_MASK_MASK                                            0x00800000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__EA_1_MASTER_MASK_MASK                                              0x01000000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__RESERVED_MASK                                                      0xFE000000L
+//RLC_SERDES_NONCU_MASTER_BUSY_1
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__SE_MASTER_BUSY_1__SHIFT                                               0x0
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__GC_MASTER_BUSY_1__SHIFT                                               0x10
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__GC_GFX_MASTER_BUSY_1__SHIFT                                           0x11
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__TC0_MASTER_BUSY_1__SHIFT                                              0x12
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__RESERVED_1__SHIFT                                                     0x13
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE4_MASTER_BUSY__SHIFT                                             0x14
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE5_MASTER_BUSY__SHIFT                                             0x15
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE6_MASTER_BUSY__SHIFT                                             0x16
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE7_MASTER_BUSY__SHIFT                                             0x17
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__EA_1_MASTER_BUSY__SHIFT                                               0x18
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__RESERVED__SHIFT                                                       0x19
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__SE_MASTER_BUSY_1_MASK                                                 0x0000FFFFL
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__GC_MASTER_BUSY_1_MASK                                                 0x00010000L
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__GC_GFX_MASTER_BUSY_1_MASK                                             0x00020000L
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__TC0_MASTER_BUSY_1_MASK                                                0x00040000L
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__RESERVED_1_MASK                                                       0x00080000L
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE4_MASTER_BUSY_MASK                                               0x00100000L
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE5_MASTER_BUSY_MASK                                               0x00200000L
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE6_MASTER_BUSY_MASK                                               0x00400000L
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE7_MASTER_BUSY_MASK                                               0x00800000L
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__EA_1_MASTER_BUSY_MASK                                                 0x01000000L
+#define RLC_SERDES_NONCU_MASTER_BUSY_1__RESERVED_MASK                                                         0xFE000000L
+//RLC_INT_STAT
+#define RLC_INT_STAT__LAST_CP_RLC_INT_ID__SHIFT                                                               0x0
+#define RLC_INT_STAT__CP_RLC_INT_PENDING__SHIFT                                                               0x8
+#define RLC_INT_STAT__RESERVED__SHIFT                                                                         0x9
+#define RLC_INT_STAT__LAST_CP_RLC_INT_ID_MASK                                                                 0x000000FFL
+#define RLC_INT_STAT__CP_RLC_INT_PENDING_MASK                                                                 0x00000100L
+#define RLC_INT_STAT__RESERVED_MASK                                                                           0xFFFFFE00L
+//RLC_LB_CNTL
+#define RLC_LB_CNTL__LOAD_BALANCE_ENABLE__SHIFT                                                               0x0
+#define RLC_LB_CNTL__LB_CNT_CP_BUSY__SHIFT                                                                    0x1
+#define RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE__SHIFT                                                                0x2
+#define RLC_LB_CNTL__LB_CNT_REG_INC__SHIFT                                                                    0x3
+#define RLC_LB_CNTL__CU_MASK_USED_OFF_HYST__SHIFT                                                             0x4
+#define RLC_LB_CNTL__RESERVED__SHIFT                                                                          0xc
+#define RLC_LB_CNTL__LOAD_BALANCE_ENABLE_MASK                                                                 0x00000001L
+#define RLC_LB_CNTL__LB_CNT_CP_BUSY_MASK                                                                      0x00000002L
+#define RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK                                                                  0x00000004L
+#define RLC_LB_CNTL__LB_CNT_REG_INC_MASK                                                                      0x00000008L
+#define RLC_LB_CNTL__CU_MASK_USED_OFF_HYST_MASK                                                               0x00000FF0L
+#define RLC_LB_CNTL__RESERVED_MASK                                                                            0xFFFFF000L
+//RLC_MGCG_CTRL
+#define RLC_MGCG_CTRL__MGCG_EN__SHIFT                                                                         0x0
+#define RLC_MGCG_CTRL__SILICON_EN__SHIFT                                                                      0x1
+#define RLC_MGCG_CTRL__SIMULATION_EN__SHIFT                                                                   0x2
+#define RLC_MGCG_CTRL__ON_DELAY__SHIFT                                                                        0x3
+#define RLC_MGCG_CTRL__OFF_HYSTERESIS__SHIFT                                                                  0x7
+#define RLC_MGCG_CTRL__GC_CAC_MGCG_CLK_CNTL__SHIFT                                                            0xf
+#define RLC_MGCG_CTRL__SE_CAC_MGCG_CLK_CNTL__SHIFT                                                            0x10
+#define RLC_MGCG_CTRL__SPARE__SHIFT                                                                           0x11
+#define RLC_MGCG_CTRL__MGCG_EN_MASK                                                                           0x00000001L
+#define RLC_MGCG_CTRL__SILICON_EN_MASK                                                                        0x00000002L
+#define RLC_MGCG_CTRL__SIMULATION_EN_MASK                                                                     0x00000004L
+#define RLC_MGCG_CTRL__ON_DELAY_MASK                                                                          0x00000078L
+#define RLC_MGCG_CTRL__OFF_HYSTERESIS_MASK                                                                    0x00007F80L
+#define RLC_MGCG_CTRL__GC_CAC_MGCG_CLK_CNTL_MASK                                                              0x00008000L
+#define RLC_MGCG_CTRL__SE_CAC_MGCG_CLK_CNTL_MASK                                                              0x00010000L
+#define RLC_MGCG_CTRL__SPARE_MASK                                                                             0xFFFE0000L
+//RLC_LB_CNTR_INIT
+#define RLC_LB_CNTR_INIT__LB_CNTR_INIT__SHIFT                                                                 0x0
+#define RLC_LB_CNTR_INIT__LB_CNTR_INIT_MASK                                                                   0xFFFFFFFFL
+//RLC_LOAD_BALANCE_CNTR
+#define RLC_LOAD_BALANCE_CNTR__RLC_LOAD_BALANCE_CNTR__SHIFT                                                   0x0
+#define RLC_LOAD_BALANCE_CNTR__RLC_LOAD_BALANCE_CNTR_MASK                                                     0xFFFFFFFFL
+//RLC_JUMP_TABLE_RESTORE
+#define RLC_JUMP_TABLE_RESTORE__ADDR__SHIFT                                                                   0x0
+#define RLC_JUMP_TABLE_RESTORE__ADDR_MASK                                                                     0xFFFFFFFFL
+//RLC_PG_DELAY_2
+#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE__SHIFT                                                           0x0
+#define RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT                                                               0x8
+#define RLC_PG_DELAY_2__PERCU_TIMEOUT_VALUE__SHIFT                                                            0x10
+#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE_MASK                                                             0x000000FFL
+#define RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK                                                                 0x0000FF00L
+#define RLC_PG_DELAY_2__PERCU_TIMEOUT_VALUE_MASK                                                              0xFFFF0000L
+//RLC_GPU_CLOCK_COUNT_LSB
+#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB__SHIFT                                                        0x0
+#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB_MASK                                                          0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_MSB
+#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB__SHIFT                                                        0x0
+#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB_MASK                                                          0xFFFFFFFFL
+//RLC_CAPTURE_GPU_CLOCK_COUNT
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE__SHIFT                                                           0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED__SHIFT                                                          0x1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE_MASK                                                             0x00000001L
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED_MASK                                                            0xFFFFFFFEL
+//RLC_UCODE_CNTL
+#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS__SHIFT                                                                0x0
+#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS_MASK                                                                  0xFFFFFFFFL
+//RLC_GPM_THREAD_RESET
+#define RLC_GPM_THREAD_RESET__THREAD0_RESET__SHIFT                                                            0x0
+#define RLC_GPM_THREAD_RESET__THREAD1_RESET__SHIFT                                                            0x1
+#define RLC_GPM_THREAD_RESET__THREAD2_RESET__SHIFT                                                            0x2
+#define RLC_GPM_THREAD_RESET__THREAD3_RESET__SHIFT                                                            0x3
+#define RLC_GPM_THREAD_RESET__RESERVED__SHIFT                                                                 0x4
+#define RLC_GPM_THREAD_RESET__THREAD0_RESET_MASK                                                              0x00000001L
+#define RLC_GPM_THREAD_RESET__THREAD1_RESET_MASK                                                              0x00000002L
+#define RLC_GPM_THREAD_RESET__THREAD2_RESET_MASK                                                              0x00000004L
+#define RLC_GPM_THREAD_RESET__THREAD3_RESET_MASK                                                              0x00000008L
+#define RLC_GPM_THREAD_RESET__RESERVED_MASK                                                                   0xFFFFFFF0L
+//RLC_GPM_CP_DMA_COMPLETE_T0
+#define RLC_GPM_CP_DMA_COMPLETE_T0__DATA__SHIFT                                                               0x0
+#define RLC_GPM_CP_DMA_COMPLETE_T0__RESERVED__SHIFT                                                           0x1
+#define RLC_GPM_CP_DMA_COMPLETE_T0__DATA_MASK                                                                 0x00000001L
+#define RLC_GPM_CP_DMA_COMPLETE_T0__RESERVED_MASK                                                             0xFFFFFFFEL
+//RLC_GPM_CP_DMA_COMPLETE_T1
+#define RLC_GPM_CP_DMA_COMPLETE_T1__DATA__SHIFT                                                               0x0
+#define RLC_GPM_CP_DMA_COMPLETE_T1__RESERVED__SHIFT                                                           0x1
+#define RLC_GPM_CP_DMA_COMPLETE_T1__DATA_MASK                                                                 0x00000001L
+#define RLC_GPM_CP_DMA_COMPLETE_T1__RESERVED_MASK                                                             0xFFFFFFFEL
+//RLC_FIREWALL_VIOLATION
+#define RLC_FIREWALL_VIOLATION__ADDR__SHIFT                                                                   0x0
+#define RLC_FIREWALL_VIOLATION__ADDR_MASK                                                                     0xFFFFFFFFL
+//RLC_CLK_COUNT_GFXCLK_LSB
+#define RLC_CLK_COUNT_GFXCLK_LSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_GFXCLK_LSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_GFXCLK_MSB
+#define RLC_CLK_COUNT_GFXCLK_MSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_GFXCLK_MSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_REFCLK_LSB
+#define RLC_CLK_COUNT_REFCLK_LSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_REFCLK_LSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_REFCLK_MSB
+#define RLC_CLK_COUNT_REFCLK_MSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_REFCLK_MSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_CTRL
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RUN__SHIFT                                                                 0x0
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RESET__SHIFT                                                               0x1
+#define RLC_CLK_COUNT_CTRL__GFXCLK_SAMPLE__SHIFT                                                              0x2
+#define RLC_CLK_COUNT_CTRL__REFCLK_RUN__SHIFT                                                                 0x3
+#define RLC_CLK_COUNT_CTRL__REFCLK_RESET__SHIFT                                                               0x4
+#define RLC_CLK_COUNT_CTRL__REFCLK_SAMPLE__SHIFT                                                              0x5
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RUN_MASK                                                                   0x00000001L
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RESET_MASK                                                                 0x00000002L
+#define RLC_CLK_COUNT_CTRL__GFXCLK_SAMPLE_MASK                                                                0x00000004L
+#define RLC_CLK_COUNT_CTRL__REFCLK_RUN_MASK                                                                   0x00000008L
+#define RLC_CLK_COUNT_CTRL__REFCLK_RESET_MASK                                                                 0x00000010L
+#define RLC_CLK_COUNT_CTRL__REFCLK_SAMPLE_MASK                                                                0x00000020L
+//RLC_CLK_COUNT_STAT
+#define RLC_CLK_COUNT_STAT__GFXCLK_VALID__SHIFT                                                               0x0
+#define RLC_CLK_COUNT_STAT__REFCLK_VALID__SHIFT                                                               0x1
+#define RLC_CLK_COUNT_STAT__REFCLK_RUN_RESYNC__SHIFT                                                          0x2
+#define RLC_CLK_COUNT_STAT__REFCLK_RESET_RESYNC__SHIFT                                                        0x3
+#define RLC_CLK_COUNT_STAT__REFCLK_SAMPLE_RESYNC__SHIFT                                                       0x4
+#define RLC_CLK_COUNT_STAT__RESERVED__SHIFT                                                                   0x5
+#define RLC_CLK_COUNT_STAT__GFXCLK_VALID_MASK                                                                 0x00000001L
+#define RLC_CLK_COUNT_STAT__REFCLK_VALID_MASK                                                                 0x00000002L
+#define RLC_CLK_COUNT_STAT__REFCLK_RUN_RESYNC_MASK                                                            0x00000004L
+#define RLC_CLK_COUNT_STAT__REFCLK_RESET_RESYNC_MASK                                                          0x00000008L
+#define RLC_CLK_COUNT_STAT__REFCLK_SAMPLE_RESYNC_MASK                                                         0x00000010L
+#define RLC_CLK_COUNT_STAT__RESERVED_MASK                                                                     0xFFFFFFE0L
+//RLC_GPM_STAT
+#define RLC_GPM_STAT__RLC_BUSY__SHIFT                                                                         0x0
+#define RLC_GPM_STAT__GFX_POWER_STATUS__SHIFT                                                                 0x1
+#define RLC_GPM_STAT__GFX_CLOCK_STATUS__SHIFT                                                                 0x2
+#define RLC_GPM_STAT__GFX_LS_STATUS__SHIFT                                                                    0x3
+#define RLC_GPM_STAT__GFX_PIPELINE_POWER_STATUS__SHIFT                                                        0x4
+#define RLC_GPM_STAT__CNTX_IDLE_BEING_PROCESSED__SHIFT                                                        0x5
+#define RLC_GPM_STAT__CNTX_BUSY_BEING_PROCESSED__SHIFT                                                        0x6
+#define RLC_GPM_STAT__GFX_IDLE_BEING_PROCESSED__SHIFT                                                         0x7
+#define RLC_GPM_STAT__CMP_BUSY_BEING_PROCESSED__SHIFT                                                         0x8
+#define RLC_GPM_STAT__SAVING_REGISTERS__SHIFT                                                                 0x9
+#define RLC_GPM_STAT__RESTORING_REGISTERS__SHIFT                                                              0xa
+#define RLC_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE__SHIFT                                                0xb
+#define RLC_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE__SHIFT                                                  0xc
+#define RLC_GPM_STAT__STATIC_CU_POWERING_UP__SHIFT                                                            0xd
+#define RLC_GPM_STAT__STATIC_CU_POWERING_DOWN__SHIFT                                                          0xe
+#define RLC_GPM_STAT__DYN_CU_POWERING_UP__SHIFT                                                               0xf
+#define RLC_GPM_STAT__DYN_CU_POWERING_DOWN__SHIFT                                                             0x10
+#define RLC_GPM_STAT__ABORTED_PD_SEQUENCE__SHIFT                                                              0x11
+#define RLC_GPM_STAT__CMP_power_status__SHIFT                                                                 0x12
+#define RLC_GPM_STAT__RESERVED_1__SHIFT                                                                       0x13
+#define RLC_GPM_STAT__MGCG_OVERRIDE_STATUS__SHIFT                                                             0x15
+#define RLC_GPM_STAT__RLC_EXEC_ROM_CODE__SHIFT                                                                0x16
+#define RLC_GPM_STAT__FGCG_OVERRIDE_STATUS__SHIFT                                                             0x17
+#define RLC_GPM_STAT__PG_ERROR_STATUS__SHIFT                                                                  0x18
+#define RLC_GPM_STAT__RLC_BUSY_MASK                                                                           0x00000001L
+#define RLC_GPM_STAT__GFX_POWER_STATUS_MASK                                                                   0x00000002L
+#define RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK                                                                   0x00000004L
+#define RLC_GPM_STAT__GFX_LS_STATUS_MASK                                                                      0x00000008L
+#define RLC_GPM_STAT__GFX_PIPELINE_POWER_STATUS_MASK                                                          0x00000010L
+#define RLC_GPM_STAT__CNTX_IDLE_BEING_PROCESSED_MASK                                                          0x00000020L
+#define RLC_GPM_STAT__CNTX_BUSY_BEING_PROCESSED_MASK                                                          0x00000040L
+#define RLC_GPM_STAT__GFX_IDLE_BEING_PROCESSED_MASK                                                           0x00000080L
+#define RLC_GPM_STAT__CMP_BUSY_BEING_PROCESSED_MASK                                                           0x00000100L
+#define RLC_GPM_STAT__SAVING_REGISTERS_MASK                                                                   0x00000200L
+#define RLC_GPM_STAT__RESTORING_REGISTERS_MASK                                                                0x00000400L
+#define RLC_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE_MASK                                                  0x00000800L
+#define RLC_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE_MASK                                                    0x00001000L
+#define RLC_GPM_STAT__STATIC_CU_POWERING_UP_MASK                                                              0x00002000L
+#define RLC_GPM_STAT__STATIC_CU_POWERING_DOWN_MASK                                                            0x00004000L
+#define RLC_GPM_STAT__DYN_CU_POWERING_UP_MASK                                                                 0x00008000L
+#define RLC_GPM_STAT__DYN_CU_POWERING_DOWN_MASK                                                               0x00010000L
+#define RLC_GPM_STAT__ABORTED_PD_SEQUENCE_MASK                                                                0x00020000L
+#define RLC_GPM_STAT__CMP_power_status_MASK                                                                   0x00040000L
+#define RLC_GPM_STAT__RESERVED_1_MASK                                                                         0x00180000L
+#define RLC_GPM_STAT__MGCG_OVERRIDE_STATUS_MASK                                                               0x00200000L
+#define RLC_GPM_STAT__RLC_EXEC_ROM_CODE_MASK                                                                  0x00400000L
+#define RLC_GPM_STAT__FGCG_OVERRIDE_STATUS_MASK                                                               0x00800000L
+#define RLC_GPM_STAT__PG_ERROR_STATUS_MASK                                                                    0xFF000000L
+//RLC_GPU_CLOCK_32_RES_SEL
+#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL__SHIFT                                                              0x0
+#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED__SHIFT                                                             0x6
+#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL_MASK                                                                0x0000003FL
+#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED_MASK                                                               0xFFFFFFC0L
+//RLC_GPU_CLOCK_32
+#define RLC_GPU_CLOCK_32__GPU_CLOCK_32__SHIFT                                                                 0x0
+#define RLC_GPU_CLOCK_32__GPU_CLOCK_32_MASK                                                                   0xFFFFFFFFL
+//RLC_PG_CNTL
+#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE__SHIFT                                                           0x0
+#define RLC_PG_CNTL__GFX_POWER_GATING_SRC__SHIFT                                                              0x1
+#define RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE__SHIFT                                                              0x2
+#define RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE__SHIFT                                                           0x3
+#define RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE__SHIFT                                                            0x4
+#define RLC_PG_CNTL__RESERVED__SHIFT                                                                          0x5
+#define RLC_PG_CNTL__PG_OVERRIDE__SHIFT                                                                       0xe
+#define RLC_PG_CNTL__CP_PG_DISABLE__SHIFT                                                                     0xf
+#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE__SHIFT                                                             0x10
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE__SHIFT                                                     0x11
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE__SHIFT                                                     0x12
+#define RLC_PG_CNTL__RESERVED1__SHIFT                                                                         0x13
+#define RLC_PG_CNTL__Ultra_Low_Voltage_Enable__SHIFT                                                          0x15
+#define RLC_PG_CNTL__RESERVED2__SHIFT                                                                         0x16
+#define RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE__SHIFT                                                             0x17
+#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK                                                             0x00000001L
+#define RLC_PG_CNTL__GFX_POWER_GATING_SRC_MASK                                                                0x00000002L
+#define RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK                                                                0x00000004L
+#define RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK                                                             0x00000008L
+#define RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK                                                              0x00000010L
+#define RLC_PG_CNTL__RESERVED_MASK                                                                            0x00003FE0L
+#define RLC_PG_CNTL__PG_OVERRIDE_MASK                                                                         0x00004000L
+#define RLC_PG_CNTL__CP_PG_DISABLE_MASK                                                                       0x00008000L
+#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE_MASK                                                               0x00010000L
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK                                                       0x00020000L
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK                                                       0x00040000L
+#define RLC_PG_CNTL__RESERVED1_MASK                                                                           0x00180000L
+#define RLC_PG_CNTL__Ultra_Low_Voltage_Enable_MASK                                                            0x00200000L
+#define RLC_PG_CNTL__RESERVED2_MASK                                                                           0x00400000L
+#define RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK                                                               0x00800000L
+//RLC_GPM_THREAD_PRIORITY
+#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY__SHIFT                                                      0x0
+#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY__SHIFT                                                      0x8
+#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY__SHIFT                                                      0x10
+#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY__SHIFT                                                      0x18
+#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY_MASK                                                        0x000000FFL
+#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY_MASK                                                        0x0000FF00L
+#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY_MASK                                                        0x00FF0000L
+#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY_MASK                                                        0xFF000000L
+//RLC_GPM_THREAD_ENABLE
+#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE__SHIFT                                                          0x0
+#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE__SHIFT                                                          0x1
+#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE__SHIFT                                                          0x2
+#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE__SHIFT                                                          0x3
+#define RLC_GPM_THREAD_ENABLE__RESERVED__SHIFT                                                                0x4
+#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE_MASK                                                            0x00000001L
+#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE_MASK                                                            0x00000002L
+#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE_MASK                                                            0x00000004L
+#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE_MASK                                                            0x00000008L
+#define RLC_GPM_THREAD_ENABLE__RESERVED_MASK                                                                  0xFFFFFFF0L
+//RLC_CGTT_MGCG_OVERRIDE
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_0__SHIFT                                                             0x0
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE__SHIFT                                                 0x1
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE__SHIFT                                                    0x2
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE__SHIFT                                                    0x3
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE__SHIFT                                                    0x4
+#define RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE__SHIFT                                                0x5
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE__SHIFT                                                    0x6
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE__SHIFT                                                0x7
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE__SHIFT                                                    0x8
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_REP_FGCG_OVERRIDE__SHIFT                                                0x9
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_PERF_CLK_EN__SHIFT                                                      0xa
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_15_11__SHIFT                                                         0xb
+#define RLC_CGTT_MGCG_OVERRIDE__ENABLE_CGTS_LEGACY__SHIFT                                                     0x10
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_31_17__SHIFT                                                         0x11
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_0_MASK                                                               0x00000001L
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK                                                   0x00000002L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK                                                      0x00000004L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK                                                      0x00000008L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK                                                      0x00000010L
+#define RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK                                                  0x00000020L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK                                                      0x00000040L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK                                                  0x00000080L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK                                                      0x00000100L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_REP_FGCG_OVERRIDE_MASK                                                  0x00000200L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_PERF_CLK_EN_MASK                                                        0x00000400L
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_15_11_MASK                                                           0x0000F800L
+#define RLC_CGTT_MGCG_OVERRIDE__ENABLE_CGTS_LEGACY_MASK                                                       0x00010000L
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_31_17_MASK                                                           0xFFFE0000L
+//RLC_CGCG_CGLS_CTRL
+#define RLC_CGCG_CGLS_CTRL__CGCG_EN__SHIFT                                                                    0x0
+#define RLC_CGCG_CGLS_CTRL__CGLS_EN__SHIFT                                                                    0x1
+#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT                                                   0x2
+#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT                                                    0x8
+#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER__SHIFT                                                            0x1b
+#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL__SHIFT                                                              0x1c
+#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE__SHIFT                                                                 0x1d
+#define RLC_CGCG_CGLS_CTRL__SIM_SILICON_EN__SHIFT                                                             0x1f
+#define RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK                                                                      0x00000001L
+#define RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK                                                                      0x00000002L
+#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK                                                     0x000000FCL
+#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK                                                      0x07FFFF00L
+#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER_MASK                                                              0x08000000L
+#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL_MASK                                                                0x10000000L
+#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE_MASK                                                                   0x60000000L
+#define RLC_CGCG_CGLS_CTRL__SIM_SILICON_EN_MASK                                                               0x80000000L
+//RLC_CGCG_RAMP_CTRL
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT__SHIFT                                                        0x0
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT__SHIFT                                                         0x4
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT__SHIFT                                                          0x8
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT__SHIFT                                                           0xc
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT__SHIFT                                                             0x10
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT__SHIFT                                                            0x1c
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT_MASK                                                          0x0000000FL
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT_MASK                                                           0x000000F0L
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT_MASK                                                            0x00000F00L
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT_MASK                                                             0x0000F000L
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT_MASK                                                               0x0FFF0000L
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT_MASK                                                              0xF0000000L
+//RLC_DYN_PG_STATUS
+#define RLC_DYN_PG_STATUS__PG_STATUS_CU_MASK__SHIFT                                                           0x0
+#define RLC_DYN_PG_STATUS__PG_STATUS_CU_MASK_MASK                                                             0xFFFFFFFFL
+//RLC_DYN_PG_REQUEST
+#define RLC_DYN_PG_REQUEST__PG_REQUEST_CU_MASK__SHIFT                                                         0x0
+#define RLC_DYN_PG_REQUEST__PG_REQUEST_CU_MASK_MASK                                                           0xFFFFFFFFL
+//RLC_PG_DELAY
+#define RLC_PG_DELAY__POWER_UP_DELAY__SHIFT                                                                   0x0
+#define RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT                                                                 0x8
+#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT                                                              0x10
+#define RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT                                                                  0x18
+#define RLC_PG_DELAY__POWER_UP_DELAY_MASK                                                                     0x000000FFL
+#define RLC_PG_DELAY__POWER_DOWN_DELAY_MASK                                                                   0x0000FF00L
+#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY_MASK                                                                0x00FF0000L
+#define RLC_PG_DELAY__MEM_SLEEP_DELAY_MASK                                                                    0xFF000000L
+//RLC_CU_STATUS
+#define RLC_CU_STATUS__WORK_PENDING__SHIFT                                                                    0x0
+#define RLC_CU_STATUS__WORK_PENDING_MASK                                                                      0xFFFFFFFFL
+//RLC_LB_INIT_CU_MASK
+#define RLC_LB_INIT_CU_MASK__INIT_CU_MASK__SHIFT                                                              0x0
+#define RLC_LB_INIT_CU_MASK__INIT_CU_MASK_MASK                                                                0xFFFFFFFFL
+//RLC_LB_ALWAYS_ACTIVE_CU_MASK
+#define RLC_LB_ALWAYS_ACTIVE_CU_MASK__ALWAYS_ACTIVE_CU_MASK__SHIFT                                            0x0
+#define RLC_LB_ALWAYS_ACTIVE_CU_MASK__ALWAYS_ACTIVE_CU_MASK_MASK                                              0xFFFFFFFFL
+//RLC_LB_PARAMS
+#define RLC_LB_PARAMS__SKIP_L2_CHECK__SHIFT                                                                   0x0
+#define RLC_LB_PARAMS__FIFO_SAMPLES__SHIFT                                                                    0x1
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLES__SHIFT                                                                 0x8
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLE_INTERVAL__SHIFT                                                         0x10
+#define RLC_LB_PARAMS__SKIP_L2_CHECK_MASK                                                                     0x00000001L
+#define RLC_LB_PARAMS__FIFO_SAMPLES_MASK                                                                      0x000000FEL
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLES_MASK                                                                   0x0000FF00L
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLE_INTERVAL_MASK                                                           0xFFFF0000L
+//RLC_THREAD1_DELAY
+#define RLC_THREAD1_DELAY__CU_IDEL_DELAY__SHIFT                                                               0x0
+#define RLC_THREAD1_DELAY__LBPW_INNER_LOOP_DELAY__SHIFT                                                       0x8
+#define RLC_THREAD1_DELAY__LBPW_OUTER_LOOP_DELAY__SHIFT                                                       0x10
+#define RLC_THREAD1_DELAY__SPARE__SHIFT                                                                       0x18
+#define RLC_THREAD1_DELAY__CU_IDEL_DELAY_MASK                                                                 0x000000FFL
+#define RLC_THREAD1_DELAY__LBPW_INNER_LOOP_DELAY_MASK                                                         0x0000FF00L
+#define RLC_THREAD1_DELAY__LBPW_OUTER_LOOP_DELAY_MASK                                                         0x00FF0000L
+#define RLC_THREAD1_DELAY__SPARE_MASK                                                                         0xFF000000L
+//RLC_PG_ALWAYS_ON_CU_MASK
+#define RLC_PG_ALWAYS_ON_CU_MASK__AON_CU_MASK__SHIFT                                                          0x0
+#define RLC_PG_ALWAYS_ON_CU_MASK__AON_CU_MASK_MASK                                                            0xFFFFFFFFL
+//RLC_MAX_PG_CU
+#define RLC_MAX_PG_CU__MAX_POWERED_UP_CU__SHIFT                                                               0x0
+#define RLC_MAX_PG_CU__SPARE__SHIFT                                                                           0x8
+#define RLC_MAX_PG_CU__MAX_POWERED_UP_CU_MASK                                                                 0x000000FFL
+#define RLC_MAX_PG_CU__SPARE_MASK                                                                             0xFFFFFF00L
+//RLC_AUTO_PG_CTRL
+#define RLC_AUTO_PG_CTRL__AUTO_PG_EN__SHIFT                                                                   0x0
+#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN__SHIFT                                                0x1
+#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN__SHIFT                                                              0x2
+#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT                                             0x3
+#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD__SHIFT                                             0x13
+#define RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK                                                                     0x00000001L
+#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN_MASK                                                  0x00000002L
+#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN_MASK                                                                0x00000004L
+#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK                                               0x0007FFF8L
+#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD_MASK                                               0xFFF80000L
+//RLC_SMU_GRBM_REG_SAVE_CTRL
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__START_GRBM_REG_SAVE__SHIFT                                                0x0
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__SPARE__SHIFT                                                              0x1
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__START_GRBM_REG_SAVE_MASK                                                  0x00000001L
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__SPARE_MASK                                                                0xFFFFFFFEL
+//RLC_SERDES_RD_PENDING
+#define RLC_SERDES_RD_PENDING__RD_PENDING__SHIFT                                                              0x0
+#define RLC_SERDES_RD_PENDING__RD_PENDING_MASK                                                                0x00000001L
+//RLC_SERDES_RD_MASTER_INDEX
+#define RLC_SERDES_RD_MASTER_INDEX__CU_ID__SHIFT                                                              0x0
+#define RLC_SERDES_RD_MASTER_INDEX__SH_ID__SHIFT                                                              0x4
+#define RLC_SERDES_RD_MASTER_INDEX__SE_ID__SHIFT                                                              0x6
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_ID__SHIFT                                                        0x9
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU__SHIFT                                                           0xc
+#define RLC_SERDES_RD_MASTER_INDEX__NON_SE__SHIFT                                                             0xd
+#define RLC_SERDES_RD_MASTER_INDEX__DATA_REG_ID__SHIFT                                                        0x11
+#define RLC_SERDES_RD_MASTER_INDEX__SPARE__SHIFT                                                              0x13
+#define RLC_SERDES_RD_MASTER_INDEX__CU_ID_MASK                                                                0x0000000FL
+#define RLC_SERDES_RD_MASTER_INDEX__SH_ID_MASK                                                                0x00000030L
+#define RLC_SERDES_RD_MASTER_INDEX__SE_ID_MASK                                                                0x000001C0L
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_ID_MASK                                                          0x00000E00L
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_MASK                                                             0x00001000L
+#define RLC_SERDES_RD_MASTER_INDEX__NON_SE_MASK                                                               0x0001E000L
+#define RLC_SERDES_RD_MASTER_INDEX__DATA_REG_ID_MASK                                                          0x00060000L
+#define RLC_SERDES_RD_MASTER_INDEX__SPARE_MASK                                                                0xFFF80000L
+//RLC_SERDES_RD_DATA_0
+#define RLC_SERDES_RD_DATA_0__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_0__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_RD_DATA_1
+#define RLC_SERDES_RD_DATA_1__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_1__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_RD_DATA_2
+#define RLC_SERDES_RD_DATA_2__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_2__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_WR_CU_MASTER_MASK
+#define RLC_SERDES_WR_CU_MASTER_MASK__MASTER_MASK__SHIFT                                                      0x0
+#define RLC_SERDES_WR_CU_MASTER_MASK__MASTER_MASK_MASK                                                        0xFFFFFFFFL
+//RLC_SERDES_WR_NONCU_MASTER_MASK
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SE_MASTER_MASK__SHIFT                                                0x0
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_MASTER_MASK__SHIFT                                                0x10
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_GFX_MASTER_MASK__SHIFT                                            0x11
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC0_MASTER_MASK__SHIFT                                               0x12
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC1_MASTER_MASK__SHIFT                                               0x13
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE0_MASTER_MASK__SHIFT                                            0x14
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE1_MASTER_MASK__SHIFT                                            0x15
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE2_MASTER_MASK__SHIFT                                            0x16
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE3_MASTER_MASK__SHIFT                                            0x17
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__EA_0_MASTER_MASK__SHIFT                                              0x18
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC2_MASTER_MASK__SHIFT                                               0x19
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__RESERVED__SHIFT                                                      0x1a
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SE_MASTER_MASK_MASK                                                  0x0000FFFFL
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_MASTER_MASK_MASK                                                  0x00010000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_GFX_MASTER_MASK_MASK                                              0x00020000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC0_MASTER_MASK_MASK                                                 0x00040000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC1_MASTER_MASK_MASK                                                 0x00080000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE0_MASTER_MASK_MASK                                              0x00100000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE1_MASTER_MASK_MASK                                              0x00200000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE2_MASTER_MASK_MASK                                              0x00400000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE3_MASTER_MASK_MASK                                              0x00800000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__EA_0_MASTER_MASK_MASK                                                0x01000000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC2_MASTER_MASK_MASK                                                 0x02000000L
+#define RLC_SERDES_WR_NONCU_MASTER_MASK__RESERVED_MASK                                                        0xFC000000L
+//RLC_SERDES_WR_CTRL
+#define RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT                                                                   0x0
+#define RLC_SERDES_WR_CTRL__POWER_DOWN__SHIFT                                                                 0x8
+#define RLC_SERDES_WR_CTRL__POWER_UP__SHIFT                                                                   0x9
+#define RLC_SERDES_WR_CTRL__P1_SELECT__SHIFT                                                                  0xa
+#define RLC_SERDES_WR_CTRL__P2_SELECT__SHIFT                                                                  0xb
+#define RLC_SERDES_WR_CTRL__WRITE_COMMAND__SHIFT                                                              0xc
+#define RLC_SERDES_WR_CTRL__READ_COMMAND__SHIFT                                                               0xd
+#define RLC_SERDES_WR_CTRL__RDDATA_RESET__SHIFT                                                               0xe
+#define RLC_SERDES_WR_CTRL__SHORT_FORMAT__SHIFT                                                               0xf
+#define RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT                                                                   0x10
+#define RLC_SERDES_WR_CTRL__SRBM_OVERRIDE__SHIFT                                                              0x1a
+#define RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR__SHIFT                                                              0x1b
+#define RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT                                                                   0x1c
+#define RLC_SERDES_WR_CTRL__BPM_ADDR_MASK                                                                     0x000000FFL
+#define RLC_SERDES_WR_CTRL__POWER_DOWN_MASK                                                                   0x00000100L
+#define RLC_SERDES_WR_CTRL__POWER_UP_MASK                                                                     0x00000200L
+#define RLC_SERDES_WR_CTRL__P1_SELECT_MASK                                                                    0x00000400L
+#define RLC_SERDES_WR_CTRL__P2_SELECT_MASK                                                                    0x00000800L
+#define RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK                                                                0x00001000L
+#define RLC_SERDES_WR_CTRL__READ_COMMAND_MASK                                                                 0x00002000L
+#define RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK                                                                 0x00004000L
+#define RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK                                                                 0x00008000L
+#define RLC_SERDES_WR_CTRL__BPM_DATA_MASK                                                                     0x03FF0000L
+#define RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK                                                                0x04000000L
+#define RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK                                                                0x08000000L
+#define RLC_SERDES_WR_CTRL__REG_ADDR_MASK                                                                     0xF0000000L
+//RLC_SERDES_WR_DATA
+#define RLC_SERDES_WR_DATA__DATA__SHIFT                                                                       0x0
+#define RLC_SERDES_WR_DATA__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_SERDES_CU_MASTER_BUSY
+#define RLC_SERDES_CU_MASTER_BUSY__BUSY_BUSY__SHIFT                                                           0x0
+#define RLC_SERDES_CU_MASTER_BUSY__BUSY_BUSY_MASK                                                             0xFFFFFFFFL
+//RLC_SERDES_NONCU_MASTER_BUSY
+#define RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY__SHIFT                                                   0x0
+#define RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY__SHIFT                                                   0x10
+#define RLC_SERDES_NONCU_MASTER_BUSY__GC_GFX_MASTER_BUSY__SHIFT                                               0x11
+#define RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY__SHIFT                                                  0x12
+#define RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY__SHIFT                                                  0x13
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE0_MASTER_BUSY__SHIFT                                               0x14
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE1_MASTER_BUSY__SHIFT                                               0x15
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE2_MASTER_BUSY__SHIFT                                               0x16
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE3_MASTER_BUSY__SHIFT                                               0x17
+#define RLC_SERDES_NONCU_MASTER_BUSY__EA_0_MASTER_BUSY__SHIFT                                                 0x18
+#define RLC_SERDES_NONCU_MASTER_BUSY__TC2_MASTER_BUSY__SHIFT                                                  0x19
+#define RLC_SERDES_NONCU_MASTER_BUSY__RESERVED__SHIFT                                                         0x1a
+#define RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK                                                     0x0000FFFFL
+#define RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK                                                     0x00010000L
+#define RLC_SERDES_NONCU_MASTER_BUSY__GC_GFX_MASTER_BUSY_MASK                                                 0x00020000L
+#define RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK                                                    0x00040000L
+#define RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK                                                    0x00080000L
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE0_MASTER_BUSY_MASK                                                 0x00100000L
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE1_MASTER_BUSY_MASK                                                 0x00200000L
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE2_MASTER_BUSY_MASK                                                 0x00400000L
+#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE3_MASTER_BUSY_MASK                                                 0x00800000L
+#define RLC_SERDES_NONCU_MASTER_BUSY__EA_0_MASTER_BUSY_MASK                                                   0x01000000L
+#define RLC_SERDES_NONCU_MASTER_BUSY__TC2_MASTER_BUSY_MASK                                                    0x02000000L
+#define RLC_SERDES_NONCU_MASTER_BUSY__RESERVED_MASK                                                           0xFC000000L
+//RLC_GPM_GENERAL_0
+#define RLC_GPM_GENERAL_0__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_0__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_1
+#define RLC_GPM_GENERAL_1__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_1__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_2
+#define RLC_GPM_GENERAL_2__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_2__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_3
+#define RLC_GPM_GENERAL_3__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_3__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_4
+#define RLC_GPM_GENERAL_4__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_4__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_5
+#define RLC_GPM_GENERAL_5__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_5__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_6
+#define RLC_GPM_GENERAL_6__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_6__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_7
+#define RLC_GPM_GENERAL_7__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_7__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_SCRATCH_ADDR
+#define RLC_GPM_SCRATCH_ADDR__ADDR__SHIFT                                                                     0x0
+#define RLC_GPM_SCRATCH_ADDR__RESERVED__SHIFT                                                                 0x9
+#define RLC_GPM_SCRATCH_ADDR__ADDR_MASK                                                                       0x000001FFL
+#define RLC_GPM_SCRATCH_ADDR__RESERVED_MASK                                                                   0xFFFFFE00L
+//RLC_GPM_SCRATCH_DATA
+#define RLC_GPM_SCRATCH_DATA__DATA__SHIFT                                                                     0x0
+#define RLC_GPM_SCRATCH_DATA__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_STATIC_PG_STATUS
+#define RLC_STATIC_PG_STATUS__PG_STATUS_CU_MASK__SHIFT                                                        0x0
+#define RLC_STATIC_PG_STATUS__PG_STATUS_CU_MASK_MASK                                                          0xFFFFFFFFL
+//RLC_SPM_MC_CNTL
+#define RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT                                                                  0x0
+#define RLC_SPM_MC_CNTL__RLC_SPM_POLICY__SHIFT                                                                0x4
+#define RLC_SPM_MC_CNTL__RLC_SPM_PERF_CNTR__SHIFT                                                             0x5
+#define RLC_SPM_MC_CNTL__RLC_SPM_FED__SHIFT                                                                   0x6
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_OVER__SHIFT                                                            0x7
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE__SHIFT                                                                 0x8
+#define RLC_SPM_MC_CNTL__RESERVED__SHIFT                                                                      0xa
+#define RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK                                                                    0x0000000FL
+#define RLC_SPM_MC_CNTL__RLC_SPM_POLICY_MASK                                                                  0x00000010L
+#define RLC_SPM_MC_CNTL__RLC_SPM_PERF_CNTR_MASK                                                               0x00000020L
+#define RLC_SPM_MC_CNTL__RLC_SPM_FED_MASK                                                                     0x00000040L
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_OVER_MASK                                                              0x00000080L
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_MASK                                                                   0x00000300L
+#define RLC_SPM_MC_CNTL__RESERVED_MASK                                                                        0xFFFFFC00L
+//RLC_SPM_INT_CNTL
+#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL__SHIFT                                                             0x0
+#define RLC_SPM_INT_CNTL__RESERVED__SHIFT                                                                     0x1
+#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL_MASK                                                               0x00000001L
+#define RLC_SPM_INT_CNTL__RESERVED_MASK                                                                       0xFFFFFFFEL
+//RLC_SPM_INT_STATUS
+#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS__SHIFT                                                         0x0
+#define RLC_SPM_INT_STATUS__RESERVED__SHIFT                                                                   0x1
+#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS_MASK                                                           0x00000001L
+#define RLC_SPM_INT_STATUS__RESERVED_MASK                                                                     0xFFFFFFFEL
+//RLC_SMU_MESSAGE
+#define RLC_SMU_MESSAGE__CMD__SHIFT                                                                           0x0
+#define RLC_SMU_MESSAGE__CMD_MASK                                                                             0xFFFFFFFFL
+//RLC_GPM_LOG_SIZE
+#define RLC_GPM_LOG_SIZE__SIZE__SHIFT                                                                         0x0
+#define RLC_GPM_LOG_SIZE__SIZE_MASK                                                                           0xFFFFFFFFL
+//RLC_PG_DELAY_3
+#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT                                                        0x0
+#define RLC_PG_DELAY_3__RESERVED__SHIFT                                                                       0x8
+#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK                                                          0x000000FFL
+#define RLC_PG_DELAY_3__RESERVED_MASK                                                                         0xFFFFFF00L
+//RLC_GPR_REG1
+#define RLC_GPR_REG1__DATA__SHIFT                                                                             0x0
+#define RLC_GPR_REG1__DATA_MASK                                                                               0xFFFFFFFFL
+//RLC_GPR_REG2
+#define RLC_GPR_REG2__DATA__SHIFT                                                                             0x0
+#define RLC_GPR_REG2__DATA_MASK                                                                               0xFFFFFFFFL
+//RLC_GPM_LOG_CONT
+#define RLC_GPM_LOG_CONT__CONT__SHIFT                                                                         0x0
+#define RLC_GPM_LOG_CONT__CONT_MASK                                                                           0xFFFFFFFFL
+//RLC_GPM_INT_DISABLE_TH0
+#define RLC_GPM_INT_DISABLE_TH0__DISABLE__SHIFT                                                               0x0
+#define RLC_GPM_INT_DISABLE_TH0__DISABLE_MASK                                                                 0xFFFFFFFFL
+//RLC_GPM_INT_FORCE_TH0
+#define RLC_GPM_INT_FORCE_TH0__FORCE__SHIFT                                                                   0x0
+#define RLC_GPM_INT_FORCE_TH0__FORCE_MASK                                                                     0xFFFFFFFFL
+//RLC_GPM_INT_FORCE_TH1
+#define RLC_GPM_INT_FORCE_TH1__FORCE__SHIFT                                                                   0x0
+#define RLC_GPM_INT_FORCE_TH1__FORCE_MASK                                                                     0xFFFFFFFFL
+//RLC_SRM_CNTL
+#define RLC_SRM_CNTL__SRM_ENABLE__SHIFT                                                                       0x0
+#define RLC_SRM_CNTL__AUTO_INCR_ADDR__SHIFT                                                                   0x1
+#define RLC_SRM_CNTL__RESERVED__SHIFT                                                                         0x2
+#define RLC_SRM_CNTL__SRM_ENABLE_MASK                                                                         0x00000001L
+#define RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK                                                                     0x00000002L
+#define RLC_SRM_CNTL__RESERVED_MASK                                                                           0xFFFFFFFCL
+//RLC_SRM_ARAM_ADDR
+#define RLC_SRM_ARAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_SRM_ARAM_ADDR__RESERVED__SHIFT                                                                    0xb
+#define RLC_SRM_ARAM_ADDR__ADDR_MASK                                                                          0x000007FFL
+#define RLC_SRM_ARAM_ADDR__RESERVED_MASK                                                                      0xFFFFF800L
+//RLC_SRM_ARAM_DATA
+#define RLC_SRM_ARAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_SRM_ARAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_SRM_DRAM_ADDR
+#define RLC_SRM_DRAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_SRM_DRAM_ADDR__RESERVED__SHIFT                                                                    0xb
+#define RLC_SRM_DRAM_ADDR__ADDR_MASK                                                                          0x000007FFL
+#define RLC_SRM_DRAM_ADDR__RESERVED_MASK                                                                      0xFFFFF800L
+//RLC_SRM_DRAM_DATA
+#define RLC_SRM_DRAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_SRM_DRAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_SRM_GPM_COMMAND
+#define RLC_SRM_GPM_COMMAND__OP__SHIFT                                                                        0x0
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL__SHIFT                                                                0x1
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_NUM__SHIFT                                                            0x2
+#define RLC_SRM_GPM_COMMAND__SIZE__SHIFT                                                                      0x5
+#define RLC_SRM_GPM_COMMAND__RESERVED_16__SHIFT                                                               0x10
+#define RLC_SRM_GPM_COMMAND__START_OFFSET__SHIFT                                                              0x11
+#define RLC_SRM_GPM_COMMAND__RESERVED_30_29__SHIFT                                                            0x1d
+#define RLC_SRM_GPM_COMMAND__DEST_MEMORY__SHIFT                                                               0x1f
+#define RLC_SRM_GPM_COMMAND__OP_MASK                                                                          0x00000001L
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_MASK                                                                  0x00000002L
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_NUM_MASK                                                              0x0000001CL
+#define RLC_SRM_GPM_COMMAND__SIZE_MASK                                                                        0x0000FFE0L
+#define RLC_SRM_GPM_COMMAND__RESERVED_16_MASK                                                                 0x00010000L
+#define RLC_SRM_GPM_COMMAND__START_OFFSET_MASK                                                                0x0FFE0000L
+#define RLC_SRM_GPM_COMMAND__RESERVED_30_29_MASK                                                              0x60000000L
+#define RLC_SRM_GPM_COMMAND__DEST_MEMORY_MASK                                                                 0x80000000L
+//RLC_SRM_GPM_COMMAND_STATUS
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_EMPTY__SHIFT                                                         0x0
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_FULL__SHIFT                                                          0x1
+#define RLC_SRM_GPM_COMMAND_STATUS__RESERVED__SHIFT                                                           0x2
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_EMPTY_MASK                                                           0x00000001L
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_FULL_MASK                                                            0x00000002L
+#define RLC_SRM_GPM_COMMAND_STATUS__RESERVED_MASK                                                             0xFFFFFFFCL
+//RLC_SRM_RLCV_COMMAND
+#define RLC_SRM_RLCV_COMMAND__OP__SHIFT                                                                       0x0
+#define RLC_SRM_RLCV_COMMAND__INDEX_CNTL__SHIFT                                                               0x1
+#define RLC_SRM_RLCV_COMMAND__INDEX_CNTL_NUM__SHIFT                                                           0x2
+#define RLC_SRM_RLCV_COMMAND__SIZE__SHIFT                                                                     0x5
+#define RLC_SRM_RLCV_COMMAND__RESERVED_16__SHIFT                                                              0x10
+#define RLC_SRM_RLCV_COMMAND__START_OFFSET__SHIFT                                                             0x11
+#define RLC_SRM_RLCV_COMMAND__RESERVED1__SHIFT                                                                0x1c
+#define RLC_SRM_RLCV_COMMAND__DEST_MEMORY__SHIFT                                                              0x1f
+#define RLC_SRM_RLCV_COMMAND__OP_MASK                                                                         0x00000001L
+#define RLC_SRM_RLCV_COMMAND__INDEX_CNTL_MASK                                                                 0x00000002L
+#define RLC_SRM_RLCV_COMMAND__INDEX_CNTL_NUM_MASK                                                             0x0000001CL
+#define RLC_SRM_RLCV_COMMAND__SIZE_MASK                                                                       0x0000FFE0L
+#define RLC_SRM_RLCV_COMMAND__RESERVED_16_MASK                                                                0x00010000L
+#define RLC_SRM_RLCV_COMMAND__START_OFFSET_MASK                                                               0x0FFE0000L
+#define RLC_SRM_RLCV_COMMAND__RESERVED1_MASK                                                                  0x70000000L
+#define RLC_SRM_RLCV_COMMAND__DEST_MEMORY_MASK                                                                0x80000000L
+//RLC_SRM_RLCV_COMMAND_STATUS
+#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_EMPTY__SHIFT                                                        0x0
+#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_FULL__SHIFT                                                         0x1
+#define RLC_SRM_RLCV_COMMAND_STATUS__RESERVED__SHIFT                                                          0x2
+#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_EMPTY_MASK                                                          0x00000001L
+#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_FULL_MASK                                                           0x00000002L
+#define RLC_SRM_RLCV_COMMAND_STATUS__RESERVED_MASK                                                            0xFFFFFFFCL
+//RLC_SRM_INDEX_CNTL_ADDR_0
+#define RLC_SRM_INDEX_CNTL_ADDR_0__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_0__RESERVED__SHIFT                                                            0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_0__ADDRESS_MASK                                                               0x0000FFFFL
+#define RLC_SRM_INDEX_CNTL_ADDR_0__RESERVED_MASK                                                              0xFFFF0000L
+//RLC_SRM_INDEX_CNTL_ADDR_1
+#define RLC_SRM_INDEX_CNTL_ADDR_1__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_1__RESERVED__SHIFT                                                            0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_1__ADDRESS_MASK                                                               0x0000FFFFL
+#define RLC_SRM_INDEX_CNTL_ADDR_1__RESERVED_MASK                                                              0xFFFF0000L
+//RLC_SRM_INDEX_CNTL_ADDR_2
+#define RLC_SRM_INDEX_CNTL_ADDR_2__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_2__RESERVED__SHIFT                                                            0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_2__ADDRESS_MASK                                                               0x0000FFFFL
+#define RLC_SRM_INDEX_CNTL_ADDR_2__RESERVED_MASK                                                              0xFFFF0000L
+//RLC_SRM_INDEX_CNTL_ADDR_3
+#define RLC_SRM_INDEX_CNTL_ADDR_3__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_3__RESERVED__SHIFT                                                            0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_3__ADDRESS_MASK                                                               0x0000FFFFL
+#define RLC_SRM_INDEX_CNTL_ADDR_3__RESERVED_MASK                                                              0xFFFF0000L
+//RLC_SRM_INDEX_CNTL_ADDR_4
+#define RLC_SRM_INDEX_CNTL_ADDR_4__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_4__RESERVED__SHIFT                                                            0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_4__ADDRESS_MASK                                                               0x0000FFFFL
+#define RLC_SRM_INDEX_CNTL_ADDR_4__RESERVED_MASK                                                              0xFFFF0000L
+//RLC_SRM_INDEX_CNTL_ADDR_5
+#define RLC_SRM_INDEX_CNTL_ADDR_5__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_5__RESERVED__SHIFT                                                            0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_5__ADDRESS_MASK                                                               0x0000FFFFL
+#define RLC_SRM_INDEX_CNTL_ADDR_5__RESERVED_MASK                                                              0xFFFF0000L
+//RLC_SRM_INDEX_CNTL_ADDR_6
+#define RLC_SRM_INDEX_CNTL_ADDR_6__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_6__RESERVED__SHIFT                                                            0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_6__ADDRESS_MASK                                                               0x0000FFFFL
+#define RLC_SRM_INDEX_CNTL_ADDR_6__RESERVED_MASK                                                              0xFFFF0000L
+//RLC_SRM_INDEX_CNTL_ADDR_7
+#define RLC_SRM_INDEX_CNTL_ADDR_7__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_7__RESERVED__SHIFT                                                            0x10
+#define RLC_SRM_INDEX_CNTL_ADDR_7__ADDRESS_MASK                                                               0x0000FFFFL
+#define RLC_SRM_INDEX_CNTL_ADDR_7__RESERVED_MASK                                                              0xFFFF0000L
+//RLC_SRM_INDEX_CNTL_DATA_0
+#define RLC_SRM_INDEX_CNTL_DATA_0__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_0__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_1
+#define RLC_SRM_INDEX_CNTL_DATA_1__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_1__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_2
+#define RLC_SRM_INDEX_CNTL_DATA_2__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_2__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_3
+#define RLC_SRM_INDEX_CNTL_DATA_3__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_3__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_4
+#define RLC_SRM_INDEX_CNTL_DATA_4__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_4__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_5
+#define RLC_SRM_INDEX_CNTL_DATA_5__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_5__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_6
+#define RLC_SRM_INDEX_CNTL_DATA_6__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_6__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_7
+#define RLC_SRM_INDEX_CNTL_DATA_7__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_7__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_STAT
+#define RLC_SRM_STAT__SRM_BUSY__SHIFT                                                                         0x0
+#define RLC_SRM_STAT__SRM_BUSY_DELAY__SHIFT                                                                   0x1
+#define RLC_SRM_STAT__RESERVED__SHIFT                                                                         0x2
+#define RLC_SRM_STAT__SRM_BUSY_MASK                                                                           0x00000001L
+#define RLC_SRM_STAT__SRM_BUSY_DELAY_MASK                                                                     0x00000002L
+#define RLC_SRM_STAT__RESERVED_MASK                                                                           0xFFFFFFFCL
+//RLC_SRM_GPM_ABORT
+#define RLC_SRM_GPM_ABORT__ABORT__SHIFT                                                                       0x0
+#define RLC_SRM_GPM_ABORT__RESERVED__SHIFT                                                                    0x1
+#define RLC_SRM_GPM_ABORT__ABORT_MASK                                                                         0x00000001L
+#define RLC_SRM_GPM_ABORT__RESERVED_MASK                                                                      0xFFFFFFFEL
+//RLC_CSIB_ADDR_LO
+#define RLC_CSIB_ADDR_LO__ADDRESS__SHIFT                                                                      0x0
+#define RLC_CSIB_ADDR_LO__ADDRESS_MASK                                                                        0xFFFFFFFFL
+//RLC_CSIB_ADDR_HI
+#define RLC_CSIB_ADDR_HI__ADDRESS__SHIFT                                                                      0x0
+#define RLC_CSIB_ADDR_HI__ADDRESS_MASK                                                                        0x0000FFFFL
+//RLC_CSIB_LENGTH
+#define RLC_CSIB_LENGTH__LENGTH__SHIFT                                                                        0x0
+#define RLC_CSIB_LENGTH__LENGTH_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_COMMAND
+#define RLC_SMU_COMMAND__CMD__SHIFT                                                                           0x0
+#define RLC_SMU_COMMAND__CMD_MASK                                                                             0xFFFFFFFFL
+//RLC_CP_SCHEDULERS
+#define RLC_CP_SCHEDULERS__scheduler0__SHIFT                                                                  0x0
+#define RLC_CP_SCHEDULERS__scheduler1__SHIFT                                                                  0x8
+#define RLC_CP_SCHEDULERS__scheduler2__SHIFT                                                                  0x10
+#define RLC_CP_SCHEDULERS__scheduler3__SHIFT                                                                  0x18
+#define RLC_CP_SCHEDULERS__scheduler0_MASK                                                                    0x000000FFL
+#define RLC_CP_SCHEDULERS__scheduler1_MASK                                                                    0x0000FF00L
+#define RLC_CP_SCHEDULERS__scheduler2_MASK                                                                    0x00FF0000L
+#define RLC_CP_SCHEDULERS__scheduler3_MASK                                                                    0xFF000000L
+//RLC_SMU_ARGUMENT_1
+#define RLC_SMU_ARGUMENT_1__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_1__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_2
+#define RLC_SMU_ARGUMENT_2__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_2__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_8
+#define RLC_GPM_GENERAL_8__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_8__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_9
+#define RLC_GPM_GENERAL_9__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_9__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_10
+#define RLC_GPM_GENERAL_10__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_10__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_11
+#define RLC_GPM_GENERAL_11__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_11__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_12
+#define RLC_GPM_GENERAL_12__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_12__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_UTCL1_CNTL_0
+#define RLC_GPM_UTCL1_CNTL_0__XNACK_REDO_TIMER_CNT__SHIFT                                                     0x0
+#define RLC_GPM_UTCL1_CNTL_0__DROP_MODE__SHIFT                                                                0x18
+#define RLC_GPM_UTCL1_CNTL_0__RESERVED__SHIFT                                                                 0x19
+#define RLC_GPM_UTCL1_CNTL_0__INVALIDATE__SHIFT                                                               0x1a
+#define RLC_GPM_UTCL1_CNTL_0__FRAG_LIMIT_MODE__SHIFT                                                          0x1b
+#define RLC_GPM_UTCL1_CNTL_0__FORCE_SNOOP__SHIFT                                                              0x1c
+#define RLC_GPM_UTCL1_CNTL_0__XNACK_REDO_TIMER_CNT_MASK                                                       0x000FFFFFL
+#define RLC_GPM_UTCL1_CNTL_0__DROP_MODE_MASK                                                                  0x01000000L
+#define RLC_GPM_UTCL1_CNTL_0__RESERVED_MASK                                                                   0x02000000L
+#define RLC_GPM_UTCL1_CNTL_0__INVALIDATE_MASK                                                                 0x04000000L
+#define RLC_GPM_UTCL1_CNTL_0__FRAG_LIMIT_MODE_MASK                                                            0x08000000L
+#define RLC_GPM_UTCL1_CNTL_0__FORCE_SNOOP_MASK                                                                0x10000000L
+//RLC_GPM_UTCL1_CNTL_1
+#define RLC_GPM_UTCL1_CNTL_1__XNACK_REDO_TIMER_CNT__SHIFT                                                     0x0
+#define RLC_GPM_UTCL1_CNTL_1__DROP_MODE__SHIFT                                                                0x18
+#define RLC_GPM_UTCL1_CNTL_1__RESERVED__SHIFT                                                                 0x19
+#define RLC_GPM_UTCL1_CNTL_1__INVALIDATE__SHIFT                                                               0x1a
+#define RLC_GPM_UTCL1_CNTL_1__FRAG_LIMIT_MODE__SHIFT                                                          0x1b
+#define RLC_GPM_UTCL1_CNTL_1__FORCE_SNOOP__SHIFT                                                              0x1c
+#define RLC_GPM_UTCL1_CNTL_1__XNACK_REDO_TIMER_CNT_MASK                                                       0x000FFFFFL
+#define RLC_GPM_UTCL1_CNTL_1__DROP_MODE_MASK                                                                  0x01000000L
+#define RLC_GPM_UTCL1_CNTL_1__RESERVED_MASK                                                                   0x02000000L
+#define RLC_GPM_UTCL1_CNTL_1__INVALIDATE_MASK                                                                 0x04000000L
+#define RLC_GPM_UTCL1_CNTL_1__FRAG_LIMIT_MODE_MASK                                                            0x08000000L
+#define RLC_GPM_UTCL1_CNTL_1__FORCE_SNOOP_MASK                                                                0x10000000L
+//RLC_GPM_UTCL1_CNTL_2
+#define RLC_GPM_UTCL1_CNTL_2__XNACK_REDO_TIMER_CNT__SHIFT                                                     0x0
+#define RLC_GPM_UTCL1_CNTL_2__DROP_MODE__SHIFT                                                                0x18
+#define RLC_GPM_UTCL1_CNTL_2__RESERVED__SHIFT                                                                 0x19
+#define RLC_GPM_UTCL1_CNTL_2__INVALIDATE__SHIFT                                                               0x1a
+#define RLC_GPM_UTCL1_CNTL_2__FRAG_LIMIT_MODE__SHIFT                                                          0x1b
+#define RLC_GPM_UTCL1_CNTL_2__FORCE_SNOOP__SHIFT                                                              0x1c
+#define RLC_GPM_UTCL1_CNTL_2__XNACK_REDO_TIMER_CNT_MASK                                                       0x000FFFFFL
+#define RLC_GPM_UTCL1_CNTL_2__DROP_MODE_MASK                                                                  0x01000000L
+#define RLC_GPM_UTCL1_CNTL_2__RESERVED_MASK                                                                   0x02000000L
+#define RLC_GPM_UTCL1_CNTL_2__INVALIDATE_MASK                                                                 0x04000000L
+#define RLC_GPM_UTCL1_CNTL_2__FRAG_LIMIT_MODE_MASK                                                            0x08000000L
+#define RLC_GPM_UTCL1_CNTL_2__FORCE_SNOOP_MASK                                                                0x10000000L
+//RLC_SPM_UTCL1_CNTL
+#define RLC_SPM_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                       0x0
+#define RLC_SPM_UTCL1_CNTL__DROP_MODE__SHIFT                                                                  0x18
+#define RLC_SPM_UTCL1_CNTL__RESERVED__SHIFT                                                                   0x19
+#define RLC_SPM_UTCL1_CNTL__INVALIDATE__SHIFT                                                                 0x1a
+#define RLC_SPM_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                            0x1b
+#define RLC_SPM_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                0x1c
+#define RLC_SPM_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                         0x000FFFFFL
+#define RLC_SPM_UTCL1_CNTL__DROP_MODE_MASK                                                                    0x01000000L
+#define RLC_SPM_UTCL1_CNTL__RESERVED_MASK                                                                     0x02000000L
+#define RLC_SPM_UTCL1_CNTL__INVALIDATE_MASK                                                                   0x04000000L
+#define RLC_SPM_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                              0x08000000L
+#define RLC_SPM_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                  0x10000000L
+//RLC_UTCL1_STATUS_2
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_BUSY__SHIFT                                                         0x0
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_BUSY__SHIFT                                                         0x1
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_BUSY__SHIFT                                                         0x2
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_BUSY__SHIFT                                                             0x3
+#define RLC_UTCL1_STATUS_2__PREWALKER_UTCL1_BUSY__SHIFT                                                       0x4
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_StallOnTrans__SHIFT                                                 0x5
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_StallOnTrans__SHIFT                                                 0x6
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_StallOnTrans__SHIFT                                                 0x7
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_StallOnTrans__SHIFT                                                     0x8
+#define RLC_UTCL1_STATUS_2__PREWALKER_UTCL1_StallOnTrans__SHIFT                                               0x9
+#define RLC_UTCL1_STATUS_2__RESERVED__SHIFT                                                                   0xa
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_BUSY_MASK                                                           0x00000001L
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_BUSY_MASK                                                           0x00000002L
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_BUSY_MASK                                                           0x00000004L
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_BUSY_MASK                                                               0x00000008L
+#define RLC_UTCL1_STATUS_2__PREWALKER_UTCL1_BUSY_MASK                                                         0x00000010L
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_StallOnTrans_MASK                                                   0x00000020L
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_StallOnTrans_MASK                                                   0x00000040L
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_StallOnTrans_MASK                                                   0x00000080L
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_StallOnTrans_MASK                                                       0x00000100L
+#define RLC_UTCL1_STATUS_2__PREWALKER_UTCL1_StallOnTrans_MASK                                                 0x00000200L
+#define RLC_UTCL1_STATUS_2__RESERVED_MASK                                                                     0xFFFFFC00L
+//RLC_LB_THR_CONFIG_2
+#define RLC_LB_THR_CONFIG_2__DATA__SHIFT                                                                      0x0
+#define RLC_LB_THR_CONFIG_2__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_LB_THR_CONFIG_3
+#define RLC_LB_THR_CONFIG_3__DATA__SHIFT                                                                      0x0
+#define RLC_LB_THR_CONFIG_3__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_LB_THR_CONFIG_4
+#define RLC_LB_THR_CONFIG_4__DATA__SHIFT                                                                      0x0
+#define RLC_LB_THR_CONFIG_4__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_SPM_UTCL1_ERROR_1
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqError__SHIFT                                                     0x0
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorVmid__SHIFT                                                 0x2
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                             0x6
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqError_MASK                                                       0x00000003L
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorVmid_MASK                                                   0x0000003CL
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                               0x000003C0L
+//RLC_SPM_UTCL1_ERROR_2
+#define RLC_SPM_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                             0x0
+#define RLC_SPM_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                               0xFFFFFFFFL
+//RLC_GPM_UTCL1_TH0_ERROR_1
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqError__SHIFT                                                 0x0
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorVmid__SHIFT                                             0x2
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                         0x6
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqError_MASK                                                   0x00000003L
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorVmid_MASK                                               0x0000003CL
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                           0x000003C0L
+//RLC_LB_THR_CONFIG_1
+#define RLC_LB_THR_CONFIG_1__DATA__SHIFT                                                                      0x0
+#define RLC_LB_THR_CONFIG_1__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_GPM_UTCL1_TH0_ERROR_2
+#define RLC_GPM_UTCL1_TH0_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                         0x0
+#define RLC_GPM_UTCL1_TH0_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                           0xFFFFFFFFL
+//RLC_GPM_UTCL1_TH1_ERROR_1
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqError__SHIFT                                                 0x0
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorVmid__SHIFT                                             0x2
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                         0x6
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqError_MASK                                                   0x00000003L
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorVmid_MASK                                               0x0000003CL
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                           0x000003C0L
+//RLC_GPM_UTCL1_TH1_ERROR_2
+#define RLC_GPM_UTCL1_TH1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                         0x0
+#define RLC_GPM_UTCL1_TH1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                           0xFFFFFFFFL
+//RLC_GPM_UTCL1_TH2_ERROR_1
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqError__SHIFT                                                 0x0
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorVmid__SHIFT                                             0x2
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                         0x6
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqError_MASK                                                   0x00000003L
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorVmid_MASK                                               0x0000003CL
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                           0x000003C0L
+//RLC_GPM_UTCL1_TH2_ERROR_2
+#define RLC_GPM_UTCL1_TH2_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                         0x0
+#define RLC_GPM_UTCL1_TH2_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                           0xFFFFFFFFL
+//RLC_SEMAPHORE_0
+#define RLC_SEMAPHORE_0__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_0__RESERVED__SHIFT                                                                      0x5
+#define RLC_SEMAPHORE_0__CLIENT_ID_MASK                                                                       0x0000001FL
+#define RLC_SEMAPHORE_0__RESERVED_MASK                                                                        0xFFFFFFE0L
+//RLC_SEMAPHORE_1
+#define RLC_SEMAPHORE_1__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_1__RESERVED__SHIFT                                                                      0x5
+#define RLC_SEMAPHORE_1__CLIENT_ID_MASK                                                                       0x0000001FL
+#define RLC_SEMAPHORE_1__RESERVED_MASK                                                                        0xFFFFFFE0L
+//RLC_CP_EOF_INT
+#define RLC_CP_EOF_INT__INTERRUPT__SHIFT                                                                      0x0
+#define RLC_CP_EOF_INT__RESERVED__SHIFT                                                                       0x1
+#define RLC_CP_EOF_INT__INTERRUPT_MASK                                                                        0x00000001L
+#define RLC_CP_EOF_INT__RESERVED_MASK                                                                         0xFFFFFFFEL
+//RLC_CP_EOF_INT_CNT
+#define RLC_CP_EOF_INT_CNT__CNT__SHIFT                                                                        0x0
+#define RLC_CP_EOF_INT_CNT__CNT_MASK                                                                          0xFFFFFFFFL
+//RLC_SPARE_INT
+#define RLC_SPARE_INT__INTERRUPT__SHIFT                                                                       0x0
+#define RLC_SPARE_INT__RESERVED__SHIFT                                                                        0x1
+#define RLC_SPARE_INT__INTERRUPT_MASK                                                                         0x00000001L
+#define RLC_SPARE_INT__RESERVED_MASK                                                                          0xFFFFFFFEL
+//RLC_PREWALKER_UTCL1_CNTL
+#define RLC_PREWALKER_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                 0x0
+#define RLC_PREWALKER_UTCL1_CNTL__DROP_MODE__SHIFT                                                            0x18
+#define RLC_PREWALKER_UTCL1_CNTL__RESERVED__SHIFT                                                             0x19
+#define RLC_PREWALKER_UTCL1_CNTL__INVALIDATE__SHIFT                                                           0x1a
+#define RLC_PREWALKER_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                      0x1b
+#define RLC_PREWALKER_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                          0x1c
+#define RLC_PREWALKER_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                   0x000FFFFFL
+#define RLC_PREWALKER_UTCL1_CNTL__DROP_MODE_MASK                                                              0x01000000L
+#define RLC_PREWALKER_UTCL1_CNTL__RESERVED_MASK                                                               0x02000000L
+#define RLC_PREWALKER_UTCL1_CNTL__INVALIDATE_MASK                                                             0x04000000L
+#define RLC_PREWALKER_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                        0x08000000L
+#define RLC_PREWALKER_UTCL1_CNTL__FORCE_SNOOP_MASK                                                            0x10000000L
+//RLC_PREWALKER_UTCL1_TRIG
+#define RLC_PREWALKER_UTCL1_TRIG__VALID__SHIFT                                                                0x0
+#define RLC_PREWALKER_UTCL1_TRIG__VMID__SHIFT                                                                 0x1
+#define RLC_PREWALKER_UTCL1_TRIG__PRIME_MODE__SHIFT                                                           0x5
+#define RLC_PREWALKER_UTCL1_TRIG__READ_PERM__SHIFT                                                            0x6
+#define RLC_PREWALKER_UTCL1_TRIG__WRITE_PERM__SHIFT                                                           0x7
+#define RLC_PREWALKER_UTCL1_TRIG__EXEC_PERM__SHIFT                                                            0x8
+#define RLC_PREWALKER_UTCL1_TRIG__RESERVED__SHIFT                                                             0x9
+#define RLC_PREWALKER_UTCL1_TRIG__READY__SHIFT                                                                0x1f
+#define RLC_PREWALKER_UTCL1_TRIG__VALID_MASK                                                                  0x00000001L
+#define RLC_PREWALKER_UTCL1_TRIG__VMID_MASK                                                                   0x0000001EL
+#define RLC_PREWALKER_UTCL1_TRIG__PRIME_MODE_MASK                                                             0x00000020L
+#define RLC_PREWALKER_UTCL1_TRIG__READ_PERM_MASK                                                              0x00000040L
+#define RLC_PREWALKER_UTCL1_TRIG__WRITE_PERM_MASK                                                             0x00000080L
+#define RLC_PREWALKER_UTCL1_TRIG__EXEC_PERM_MASK                                                              0x00000100L
+#define RLC_PREWALKER_UTCL1_TRIG__RESERVED_MASK                                                               0x7FFFFE00L
+#define RLC_PREWALKER_UTCL1_TRIG__READY_MASK                                                                  0x80000000L
+//RLC_PREWALKER_UTCL1_ADDR_LSB
+#define RLC_PREWALKER_UTCL1_ADDR_LSB__ADDR_LSB__SHIFT                                                         0x0
+#define RLC_PREWALKER_UTCL1_ADDR_LSB__ADDR_LSB_MASK                                                           0xFFFFFFFFL
+//RLC_PREWALKER_UTCL1_ADDR_MSB
+#define RLC_PREWALKER_UTCL1_ADDR_MSB__ADDR_MSB__SHIFT                                                         0x0
+#define RLC_PREWALKER_UTCL1_ADDR_MSB__ADDR_MSB_MASK                                                           0x0000FFFFL
+//RLC_PREWALKER_UTCL1_SIZE_LSB
+#define RLC_PREWALKER_UTCL1_SIZE_LSB__SIZE_LSB__SHIFT                                                         0x0
+#define RLC_PREWALKER_UTCL1_SIZE_LSB__SIZE_LSB_MASK                                                           0xFFFFFFFFL
+//RLC_PREWALKER_UTCL1_SIZE_MSB
+#define RLC_PREWALKER_UTCL1_SIZE_MSB__SIZE_MSB__SHIFT                                                         0x0
+#define RLC_PREWALKER_UTCL1_SIZE_MSB__SIZE_MSB_MASK                                                           0x00000003L
+//RLC_DSM_TRIG
+#define RLC_DSM_TRIG__START__SHIFT                                                                            0x0
+#define RLC_DSM_TRIG__START_MASK                                                                              0x00000001L
+//RLC_UTCL1_STATUS
+#define RLC_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define RLC_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define RLC_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define RLC_UTCL1_STATUS__RESERVED__SHIFT                                                                     0x3
+#define RLC_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define RLC_UTCL1_STATUS__RESERVED_1__SHIFT                                                                   0xe
+#define RLC_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define RLC_UTCL1_STATUS__RESERVED_2__SHIFT                                                                   0x16
+#define RLC_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define RLC_UTCL1_STATUS__RESERVED_3__SHIFT                                                                   0x1e
+#define RLC_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define RLC_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define RLC_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define RLC_UTCL1_STATUS__RESERVED_MASK                                                                       0x000000F8L
+#define RLC_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define RLC_UTCL1_STATUS__RESERVED_1_MASK                                                                     0x0000C000L
+#define RLC_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define RLC_UTCL1_STATUS__RESERVED_2_MASK                                                                     0x00C00000L
+#define RLC_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+#define RLC_UTCL1_STATUS__RESERVED_3_MASK                                                                     0xC0000000L
+//RLC_R2I_CNTL_0
+#define RLC_R2I_CNTL_0__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_0__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_R2I_CNTL_1
+#define RLC_R2I_CNTL_1__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_1__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_R2I_CNTL_2
+#define RLC_R2I_CNTL_2__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_2__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_R2I_CNTL_3
+#define RLC_R2I_CNTL_3__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_3__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_UTCL2_CNTL
+#define RLC_UTCL2_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x0
+#define RLC_UTCL2_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                          0x1
+#define RLC_UTCL2_CNTL__RESERVED__SHIFT                                                                       0x2
+#define RLC_UTCL2_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x00000001L
+#define RLC_UTCL2_CNTL__IGNORE_PTE_PERMISSION_MASK                                                            0x00000002L
+#define RLC_UTCL2_CNTL__RESERVED_MASK                                                                         0xFFFFFFFCL
+//RLC_LBPW_CU_STAT
+#define RLC_LBPW_CU_STAT__MAX_CU__SHIFT                                                                       0x0
+#define RLC_LBPW_CU_STAT__ON_CU__SHIFT                                                                        0x10
+#define RLC_LBPW_CU_STAT__MAX_CU_MASK                                                                         0x0000FFFFL
+#define RLC_LBPW_CU_STAT__ON_CU_MASK                                                                          0xFFFF0000L
+//RLC_DS_CNTL
+#define RLC_DS_CNTL__GFX_CLK_DS_RLC_BUSY_MASK__SHIFT                                                          0x0
+#define RLC_DS_CNTL__GFX_CLK_DS_CP_BUSY_MASK__SHIFT                                                           0x1
+#define RLC_DS_CNTL__GFX_CLK_TCC_RLC_GRBM_CC_RESIDENT_MASK__SHIFT                                             0x2
+#define RLC_DS_CNTL__GFX_CLK_EA_RLC_GRBM_STAT_BUSY_MASK__SHIFT                                                0x3
+#define RLC_DS_CNTL__RESRVED__SHIFT                                                                           0x4
+#define RLC_DS_CNTL__SOC_CLK_TCC_RLC_GRBM_CC_RESIDENT_MASK__SHIFT                                             0xe
+#define RLC_DS_CNTL__SOC_CLK_EA_RLC_GRBM_STAT_BUSY_MASK__SHIFT                                                0xf
+#define RLC_DS_CNTL__SOC_CLK_DS_RLC_BUSY_MASK__SHIFT                                                          0x10
+#define RLC_DS_CNTL__SOC_CLK_DS_CP_BUSY_MASK__SHIFT                                                           0x11
+#define RLC_DS_CNTL__RESRVED_1__SHIFT                                                                         0x12
+#define RLC_DS_CNTL__GFX_CLK_DS_RLC_BUSY_MASK_MASK                                                            0x00000001L
+#define RLC_DS_CNTL__GFX_CLK_DS_CP_BUSY_MASK_MASK                                                             0x00000002L
+#define RLC_DS_CNTL__GFX_CLK_TCC_RLC_GRBM_CC_RESIDENT_MASK_MASK                                               0x00000004L
+#define RLC_DS_CNTL__GFX_CLK_EA_RLC_GRBM_STAT_BUSY_MASK_MASK                                                  0x00000008L
+#define RLC_DS_CNTL__RESRVED_MASK                                                                             0x00003FF0L
+#define RLC_DS_CNTL__SOC_CLK_TCC_RLC_GRBM_CC_RESIDENT_MASK_MASK                                               0x00004000L
+#define RLC_DS_CNTL__SOC_CLK_EA_RLC_GRBM_STAT_BUSY_MASK_MASK                                                  0x00008000L
+#define RLC_DS_CNTL__SOC_CLK_DS_RLC_BUSY_MASK_MASK                                                            0x00010000L
+#define RLC_DS_CNTL__SOC_CLK_DS_CP_BUSY_MASK_MASK                                                             0x00020000L
+#define RLC_DS_CNTL__RESRVED_1_MASK                                                                           0xFFFC0000L
+//RLC_GPM_INT_STAT_TH0
+#define RLC_GPM_INT_STAT_TH0__STATUS__SHIFT                                                                   0x0
+#define RLC_GPM_INT_STAT_TH0__STATUS_MASK                                                                     0xFFFFFFFFL
+//RLC_GPM_GENERAL_13
+#define RLC_GPM_GENERAL_13__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_13__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_14
+#define RLC_GPM_GENERAL_14__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_14__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_15
+#define RLC_GPM_GENERAL_15__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_15__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_SPARE_INT_1
+#define RLC_SPARE_INT_1__INTERRUPT__SHIFT                                                                     0x0
+#define RLC_SPARE_INT_1__RESERVED__SHIFT                                                                      0x1
+#define RLC_SPARE_INT_1__INTERRUPT_MASK                                                                       0x00000001L
+#define RLC_SPARE_INT_1__RESERVED_MASK                                                                        0xFFFFFFFEL
+//RLC_RLCV_SPARE_INT_1
+#define RLC_RLCV_SPARE_INT_1__INTERRUPT__SHIFT                                                                0x0
+#define RLC_RLCV_SPARE_INT_1__RESERVED__SHIFT                                                                 0x1
+#define RLC_RLCV_SPARE_INT_1__INTERRUPT_MASK                                                                  0x00000001L
+#define RLC_RLCV_SPARE_INT_1__RESERVED_MASK                                                                   0xFFFFFFFEL
+//RLC_SEMAPHORE_2
+#define RLC_SEMAPHORE_2__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_2__RESERVED__SHIFT                                                                      0x5
+#define RLC_SEMAPHORE_2__CLIENT_ID_MASK                                                                       0x0000001FL
+#define RLC_SEMAPHORE_2__RESERVED_MASK                                                                        0xFFFFFFE0L
+//RLC_SEMAPHORE_3
+#define RLC_SEMAPHORE_3__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_3__RESERVED__SHIFT                                                                      0x5
+#define RLC_SEMAPHORE_3__CLIENT_ID_MASK                                                                       0x0000001FL
+#define RLC_SEMAPHORE_3__RESERVED_MASK                                                                        0xFFFFFFE0L
+//RLC_SMU_ARGUMENT_3
+#define RLC_SMU_ARGUMENT_3__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_3__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_4
+#define RLC_SMU_ARGUMENT_4__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_4__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_LSB_1
+#define RLC_GPU_CLOCK_COUNT_LSB_1__GPU_CLOCKS_LSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_LSB_1__GPU_CLOCKS_LSB_MASK                                                        0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_MSB_1
+#define RLC_GPU_CLOCK_COUNT_MSB_1__GPU_CLOCKS_MSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_MSB_1__GPU_CLOCKS_MSB_MASK                                                        0xFFFFFFFFL
+//RLC_CAPTURE_GPU_CLOCK_COUNT_1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__CAPTURE__SHIFT                                                         0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__RESERVED__SHIFT                                                        0x1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__CAPTURE_MASK                                                           0x00000001L
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__RESERVED_MASK                                                          0xFFFFFFFEL
+//RLC_GPU_CLOCK_COUNT_LSB_2
+#define RLC_GPU_CLOCK_COUNT_LSB_2__GPU_CLOCKS_LSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_LSB_2__GPU_CLOCKS_LSB_MASK                                                        0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_MSB_2
+#define RLC_GPU_CLOCK_COUNT_MSB_2__GPU_CLOCKS_MSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_MSB_2__GPU_CLOCKS_MSB_MASK                                                        0xFFFFFFFFL
+//RLC_CAPTURE_GPU_CLOCK_COUNT_2
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__CAPTURE__SHIFT                                                         0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__RESERVED__SHIFT                                                        0x1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__CAPTURE_MASK                                                           0x00000001L
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__RESERVED_MASK                                                          0xFFFFFFFEL
+//RLC_CPG_STAT_INVAL
+#define RLC_CPG_STAT_INVAL__CPG_stat_inval__SHIFT                                                             0x0
+#define RLC_CPG_STAT_INVAL__CPG_stat_inval_MASK                                                               0x00000001L
+//RLC_DSM_CNTL
+#define RLC_DSM_CNTL__RLCG_INSTR_RAM_IRRITATOR_DATA_SEL__SHIFT                                                0x0
+#define RLC_DSM_CNTL__RLCG_INSTR_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                            0x2
+#define RLC_DSM_CNTL__RLCG_SCRATCH_RAM_IRRITATOR_DATA_SEL__SHIFT                                              0x3
+#define RLC_DSM_CNTL__RLCG_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                          0x5
+#define RLC_DSM_CNTL__RLCV_INSTR_RAM_IRRITATOR_DATA_SEL__SHIFT                                                0x6
+#define RLC_DSM_CNTL__RLCV_INSTR_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                            0x8
+#define RLC_DSM_CNTL__RLCV_SCRATCH_RAM_IRRITATOR_DATA_SEL__SHIFT                                              0x9
+#define RLC_DSM_CNTL__RLCV_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                          0xb
+#define RLC_DSM_CNTL__RLC_TCTAG_RAM_IRRITATOR_DATA_SEL__SHIFT                                                 0xc
+#define RLC_DSM_CNTL__RLC_TCTAG_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                             0xe
+#define RLC_DSM_CNTL__RLC_SPM_SCRATCH_RAM_IRRITATOR_DATA_SEL__SHIFT                                           0xf
+#define RLC_DSM_CNTL__RLC_SPM_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                       0x11
+#define RLC_DSM_CNTL__RLC_SRM_DATA_RAM_IRRITATOR_DATA_SEL__SHIFT                                              0x12
+#define RLC_DSM_CNTL__RLC_SRM_DATA_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                          0x14
+#define RLC_DSM_CNTL__RLC_SRM_ADDR_RAM_IRRITATOR_DATA_SEL__SHIFT                                              0x15
+#define RLC_DSM_CNTL__RLC_SRM_ADDR_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                          0x17
+#define RLC_DSM_CNTL__RLCG_INSTR_RAM_IRRITATOR_DATA_SEL_MASK                                                  0x00000003L
+#define RLC_DSM_CNTL__RLCG_INSTR_RAM_IRRITATOR_SINGLE_WRITE_MASK                                              0x00000004L
+#define RLC_DSM_CNTL__RLCG_SCRATCH_RAM_IRRITATOR_DATA_SEL_MASK                                                0x00000018L
+#define RLC_DSM_CNTL__RLCG_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE_MASK                                            0x00000020L
+#define RLC_DSM_CNTL__RLCV_INSTR_RAM_IRRITATOR_DATA_SEL_MASK                                                  0x000000C0L
+#define RLC_DSM_CNTL__RLCV_INSTR_RAM_IRRITATOR_SINGLE_WRITE_MASK                                              0x00000100L
+#define RLC_DSM_CNTL__RLCV_SCRATCH_RAM_IRRITATOR_DATA_SEL_MASK                                                0x00000600L
+#define RLC_DSM_CNTL__RLCV_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE_MASK                                            0x00000800L
+#define RLC_DSM_CNTL__RLC_TCTAG_RAM_IRRITATOR_DATA_SEL_MASK                                                   0x00003000L
+#define RLC_DSM_CNTL__RLC_TCTAG_RAM_IRRITATOR_SINGLE_WRITE_MASK                                               0x00004000L
+#define RLC_DSM_CNTL__RLC_SPM_SCRATCH_RAM_IRRITATOR_DATA_SEL_MASK                                             0x00018000L
+#define RLC_DSM_CNTL__RLC_SPM_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE_MASK                                         0x00020000L
+#define RLC_DSM_CNTL__RLC_SRM_DATA_RAM_IRRITATOR_DATA_SEL_MASK                                                0x000C0000L
+#define RLC_DSM_CNTL__RLC_SRM_DATA_RAM_IRRITATOR_SINGLE_WRITE_MASK                                            0x00100000L
+#define RLC_DSM_CNTL__RLC_SRM_ADDR_RAM_IRRITATOR_DATA_SEL_MASK                                                0x00600000L
+#define RLC_DSM_CNTL__RLC_SRM_ADDR_RAM_IRRITATOR_SINGLE_WRITE_MASK                                            0x00800000L
+//RLC_DSM_CNTLA
+#define RLC_DSM_CNTLA__RLC_SPM_SE0_SCRATCH_RAM_IRRITATOR_DATA_SEL__SHIFT                                      0x0
+#define RLC_DSM_CNTLA__RLC_SPM_SE0_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                  0x2
+#define RLC_DSM_CNTLA__RLC_SPM_SE1_SCRATCH_RAM_IRRITATOR_DATA_SEL__SHIFT                                      0x3
+#define RLC_DSM_CNTLA__RLC_SPM_SE1_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                  0x5
+#define RLC_DSM_CNTLA__RLC_SPM_SE2_SCRATCH_RAM_IRRITATOR_DATA_SEL__SHIFT                                      0x6
+#define RLC_DSM_CNTLA__RLC_SPM_SE2_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                  0x8
+#define RLC_DSM_CNTLA__RLC_SPM_SE3_SCRATCH_RAM_IRRITATOR_DATA_SEL__SHIFT                                      0x9
+#define RLC_DSM_CNTLA__RLC_SPM_SE3_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                  0xb
+#define RLC_DSM_CNTLA__RLC_SPM_SE0_SCRATCH_RAM_IRRITATOR_DATA_SEL_MASK                                        0x00000003L
+#define RLC_DSM_CNTLA__RLC_SPM_SE0_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE_MASK                                    0x00000004L
+#define RLC_DSM_CNTLA__RLC_SPM_SE1_SCRATCH_RAM_IRRITATOR_DATA_SEL_MASK                                        0x00000018L
+#define RLC_DSM_CNTLA__RLC_SPM_SE1_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE_MASK                                    0x00000020L
+#define RLC_DSM_CNTLA__RLC_SPM_SE2_SCRATCH_RAM_IRRITATOR_DATA_SEL_MASK                                        0x000000C0L
+#define RLC_DSM_CNTLA__RLC_SPM_SE2_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE_MASK                                    0x00000100L
+#define RLC_DSM_CNTLA__RLC_SPM_SE3_SCRATCH_RAM_IRRITATOR_DATA_SEL_MASK                                        0x00000600L
+#define RLC_DSM_CNTLA__RLC_SPM_SE3_SCRATCH_RAM_IRRITATOR_SINGLE_WRITE_MASK                                    0x00000800L
+//RLC_DSM_CNTL2
+#define RLC_DSM_CNTL2__RLCG_INSTR_RAM_ENABLE_ERROR_INJECT__SHIFT                                              0x0
+#define RLC_DSM_CNTL2__RLCG_INSTR_RAM_SELECT_INJECT_DELAY__SHIFT                                              0x2
+#define RLC_DSM_CNTL2__RLCG_SCRATCH_RAM_ENABLE_ERROR_INJECT__SHIFT                                            0x3
+#define RLC_DSM_CNTL2__RLCG_SCRATCH_RAM_SELECT_INJECT_DELAY__SHIFT                                            0x5
+#define RLC_DSM_CNTL2__RLCV_INSTR_RAM_ENABLE_ERROR_INJECT__SHIFT                                              0x6
+#define RLC_DSM_CNTL2__RLCV_INSTR_RAM_SELECT_INJECT_DELAY__SHIFT                                              0x8
+#define RLC_DSM_CNTL2__RLCV_SCRATCH_RAM_ENABLE_ERROR_INJECT__SHIFT                                            0x9
+#define RLC_DSM_CNTL2__RLCV_SCRATCH_RAM_SELECT_INJECT_DELAY__SHIFT                                            0xb
+#define RLC_DSM_CNTL2__RLC_TCTAG_RAM_ENABLE_ERROR_INJECT__SHIFT                                               0xc
+#define RLC_DSM_CNTL2__RLC_TCTAG_RAM_SELECT_INJECT_DELAY__SHIFT                                               0xe
+#define RLC_DSM_CNTL2__RLC_SPM_SCRATCH_RAM_ENABLE_ERROR_INJECT__SHIFT                                         0xf
+#define RLC_DSM_CNTL2__RLC_SPM_SCRATCH_RAM_SELECT_INJECT_DELAY__SHIFT                                         0x11
+#define RLC_DSM_CNTL2__RLC_SRM_DATA_RAM_ENABLE_ERROR_INJECT__SHIFT                                            0x12
+#define RLC_DSM_CNTL2__RLC_SRM_DATA_RAM_SELECT_INJECT_DELAY__SHIFT                                            0x14
+#define RLC_DSM_CNTL2__RLC_SRM_ADDR_RAM_ENABLE_ERROR_INJECT__SHIFT                                            0x15
+#define RLC_DSM_CNTL2__RLC_SRM_ADDR_RAM_SELECT_INJECT_DELAY__SHIFT                                            0x17
+#define RLC_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                    0x1a
+#define RLC_DSM_CNTL2__RLCG_INSTR_RAM_ENABLE_ERROR_INJECT_MASK                                                0x00000003L
+#define RLC_DSM_CNTL2__RLCG_INSTR_RAM_SELECT_INJECT_DELAY_MASK                                                0x00000004L
+#define RLC_DSM_CNTL2__RLCG_SCRATCH_RAM_ENABLE_ERROR_INJECT_MASK                                              0x00000018L
+#define RLC_DSM_CNTL2__RLCG_SCRATCH_RAM_SELECT_INJECT_DELAY_MASK                                              0x00000020L
+#define RLC_DSM_CNTL2__RLCV_INSTR_RAM_ENABLE_ERROR_INJECT_MASK                                                0x000000C0L
+#define RLC_DSM_CNTL2__RLCV_INSTR_RAM_SELECT_INJECT_DELAY_MASK                                                0x00000100L
+#define RLC_DSM_CNTL2__RLCV_SCRATCH_RAM_ENABLE_ERROR_INJECT_MASK                                              0x00000600L
+#define RLC_DSM_CNTL2__RLCV_SCRATCH_RAM_SELECT_INJECT_DELAY_MASK                                              0x00000800L
+#define RLC_DSM_CNTL2__RLC_TCTAG_RAM_ENABLE_ERROR_INJECT_MASK                                                 0x00003000L
+#define RLC_DSM_CNTL2__RLC_TCTAG_RAM_SELECT_INJECT_DELAY_MASK                                                 0x00004000L
+#define RLC_DSM_CNTL2__RLC_SPM_SCRATCH_RAM_ENABLE_ERROR_INJECT_MASK                                           0x00018000L
+#define RLC_DSM_CNTL2__RLC_SPM_SCRATCH_RAM_SELECT_INJECT_DELAY_MASK                                           0x00020000L
+#define RLC_DSM_CNTL2__RLC_SRM_DATA_RAM_ENABLE_ERROR_INJECT_MASK                                              0x000C0000L
+#define RLC_DSM_CNTL2__RLC_SRM_DATA_RAM_SELECT_INJECT_DELAY_MASK                                              0x00100000L
+#define RLC_DSM_CNTL2__RLC_SRM_ADDR_RAM_ENABLE_ERROR_INJECT_MASK                                              0x00600000L
+#define RLC_DSM_CNTL2__RLC_SRM_ADDR_RAM_SELECT_INJECT_DELAY_MASK                                              0x00800000L
+#define RLC_DSM_CNTL2__INJECT_DELAY_MASK                                                                      0xFC000000L
+//RLC_DSM_CNTL2A
+#define RLC_DSM_CNTL2A__RLC_SPM_SE0_SCRATCH_RAM_ENABLE_ERROR_INJECT__SHIFT                                    0x0
+#define RLC_DSM_CNTL2A__RLC_SPM_SE0_SCRATCH_RAM_SELECT_INJECT_DELAY__SHIFT                                    0x2
+#define RLC_DSM_CNTL2A__RLC_SPM_SE1_SCRATCH_RAM_ENABLE_ERROR_INJECT__SHIFT                                    0x3
+#define RLC_DSM_CNTL2A__RLC_SPM_SE1_SCRATCH_RAM_SELECT_INJECT_DELAY__SHIFT                                    0x5
+#define RLC_DSM_CNTL2A__RLC_SPM_SE2_SCRATCH_RAM_ENABLE_ERROR_INJECT__SHIFT                                    0x6
+#define RLC_DSM_CNTL2A__RLC_SPM_SE2_SCRATCH_RAM_SELECT_INJECT_DELAY__SHIFT                                    0x8
+#define RLC_DSM_CNTL2A__RLC_SPM_SE3_SCRATCH_RAM_ENABLE_ERROR_INJECT__SHIFT                                    0x9
+#define RLC_DSM_CNTL2A__RLC_SPM_SE3_SCRATCH_RAM_SELECT_INJECT_DELAY__SHIFT                                    0xb
+#define RLC_DSM_CNTL2A__RLC_SPM_SE0_SCRATCH_RAM_ENABLE_ERROR_INJECT_MASK                                      0x00000003L
+#define RLC_DSM_CNTL2A__RLC_SPM_SE0_SCRATCH_RAM_SELECT_INJECT_DELAY_MASK                                      0x00000004L
+#define RLC_DSM_CNTL2A__RLC_SPM_SE1_SCRATCH_RAM_ENABLE_ERROR_INJECT_MASK                                      0x00000018L
+#define RLC_DSM_CNTL2A__RLC_SPM_SE1_SCRATCH_RAM_SELECT_INJECT_DELAY_MASK                                      0x00000020L
+#define RLC_DSM_CNTL2A__RLC_SPM_SE2_SCRATCH_RAM_ENABLE_ERROR_INJECT_MASK                                      0x000000C0L
+#define RLC_DSM_CNTL2A__RLC_SPM_SE2_SCRATCH_RAM_SELECT_INJECT_DELAY_MASK                                      0x00000100L
+#define RLC_DSM_CNTL2A__RLC_SPM_SE3_SCRATCH_RAM_ENABLE_ERROR_INJECT_MASK                                      0x00000600L
+#define RLC_DSM_CNTL2A__RLC_SPM_SE3_SCRATCH_RAM_SELECT_INJECT_DELAY_MASK                                      0x00000800L
+//RLC_RLCV_SPARE_INT
+#define RLC_RLCV_SPARE_INT__INTERRUPT__SHIFT                                                                  0x0
+#define RLC_RLCV_SPARE_INT__RESERVED__SHIFT                                                                   0x1
+#define RLC_RLCV_SPARE_INT__INTERRUPT_MASK                                                                    0x00000001L
+#define RLC_RLCV_SPARE_INT__RESERVED_MASK                                                                     0xFFFFFFFEL
+//RLC_SMU_CLK_REQ
+#define RLC_SMU_CLK_REQ__VALID__SHIFT                                                                         0x0
+#define RLC_SMU_CLK_REQ__VALID_MASK                                                                           0x00000001L
+
+
+// addressBlock: xcd0_gc_pwrdec
+//CGTS_SM_CTRL_REG
+#define CGTS_SM_CTRL_REG__ON_SEQ_DELAY__SHIFT                                                                 0x0
+#define CGTS_SM_CTRL_REG__OFF_SEQ_DELAY__SHIFT                                                                0x4
+#define CGTS_SM_CTRL_REG__MGCG_ENABLED__SHIFT                                                                 0xc
+#define CGTS_SM_CTRL_REG__BASE_MODE__SHIFT                                                                    0x10
+#define CGTS_SM_CTRL_REG__SM_MODE__SHIFT                                                                      0x11
+#define CGTS_SM_CTRL_REG__SM_MODE_ENABLE__SHIFT                                                               0x14
+#define CGTS_SM_CTRL_REG__OVERRIDE__SHIFT                                                                     0x15
+#define CGTS_SM_CTRL_REG__LS_OVERRIDE__SHIFT                                                                  0x16
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN__SHIFT                                                            0x17
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT                                                               0x18
+#define CGTS_SM_CTRL_REG__ON_SEQ_DELAY_MASK                                                                   0x0000000FL
+#define CGTS_SM_CTRL_REG__OFF_SEQ_DELAY_MASK                                                                  0x00000FF0L
+#define CGTS_SM_CTRL_REG__MGCG_ENABLED_MASK                                                                   0x00001000L
+#define CGTS_SM_CTRL_REG__BASE_MODE_MASK                                                                      0x00010000L
+#define CGTS_SM_CTRL_REG__SM_MODE_MASK                                                                        0x000E0000L
+#define CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK                                                                 0x00100000L
+#define CGTS_SM_CTRL_REG__OVERRIDE_MASK                                                                       0x00200000L
+#define CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK                                                                    0x00400000L
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK                                                              0x00800000L
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_MASK                                                                 0xFF000000L
+//CGTS_RD_CTRL_REG
+#define CGTS_RD_CTRL_REG__ROW_MUX_SEL__SHIFT                                                                  0x0
+#define CGTS_RD_CTRL_REG__REG_MUX_SEL__SHIFT                                                                  0x8
+#define CGTS_RD_CTRL_REG__ROW_MUX_SEL_MASK                                                                    0x0000001FL
+#define CGTS_RD_CTRL_REG__REG_MUX_SEL_MASK                                                                    0x00001F00L
+//CGTS_RD_REG
+#define CGTS_RD_REG__READ_DATA__SHIFT                                                                         0x0
+#define CGTS_RD_REG__READ_DATA_MASK                                                                           0x00003FFFL
+//CGTS_TCC_DISABLE
+#define CGTS_TCC_DISABLE__WRITE_DIS__SHIFT                                                                    0x0
+#define CGTS_TCC_DISABLE__TCC_DISABLE__SHIFT                                                                  0x10
+#define CGTS_TCC_DISABLE__WRITE_DIS_MASK                                                                      0x00000001L
+#define CGTS_TCC_DISABLE__TCC_DISABLE_MASK                                                                    0xFFFF0000L
+//CGTS_USER_TCC_DISABLE
+#define CGTS_USER_TCC_DISABLE__TCC_DISABLE__SHIFT                                                             0x10
+#define CGTS_USER_TCC_DISABLE__TCC_DISABLE_MASK                                                               0xFFFF0000L
+//CGTS_CU0_SP0_CTRL_REG
+#define CGTS_CU0_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
+#define CGTS_CU0_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU0_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU0_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU0_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU0_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
+#define CGTS_CU0_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU0_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU0_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU0_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU0_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
+#define CGTS_CU0_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU0_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU0_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU0_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU0_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
+#define CGTS_CU0_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU0_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU0_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU0_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU0_LDS_SQ_CTRL_REG
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU0_TA_SQC_CTRL_REG
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC__SHIFT                                                                  0x10
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU0_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_MASK                                                                    0x007F0000L
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU0_SP1_CTRL_REG
+#define CGTS_CU0_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
+#define CGTS_CU0_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU0_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU0_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU0_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU0_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
+#define CGTS_CU0_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU0_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU0_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU0_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU0_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
+#define CGTS_CU0_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU0_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU0_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU0_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU0_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
+#define CGTS_CU0_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU0_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU0_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU0_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU0_TD_TCP_CTRL_REG
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU0_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU1_SP0_CTRL_REG
+#define CGTS_CU1_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
+#define CGTS_CU1_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU1_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU1_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU1_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU1_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
+#define CGTS_CU1_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU1_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU1_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU1_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU1_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
+#define CGTS_CU1_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU1_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU1_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU1_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU1_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
+#define CGTS_CU1_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU1_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU1_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU1_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU1_LDS_SQ_CTRL_REG
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU1_TA_SQC_CTRL_REG
+#define CGTS_CU1_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
+#define CGTS_CU1_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU1_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU1_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU1_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU1_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
+#define CGTS_CU1_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU1_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU1_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU1_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+//CGTS_CU1_SP1_CTRL_REG
+#define CGTS_CU1_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
+#define CGTS_CU1_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU1_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU1_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU1_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU1_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
+#define CGTS_CU1_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU1_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU1_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU1_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU1_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
+#define CGTS_CU1_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU1_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU1_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU1_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU1_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
+#define CGTS_CU1_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU1_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU1_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU1_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU1_TD_TCP_CTRL_REG
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU1_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU2_SP0_CTRL_REG
+#define CGTS_CU2_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
+#define CGTS_CU2_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU2_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU2_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU2_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU2_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
+#define CGTS_CU2_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU2_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU2_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU2_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU2_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
+#define CGTS_CU2_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU2_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU2_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU2_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU2_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
+#define CGTS_CU2_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU2_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU2_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU2_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU2_LDS_SQ_CTRL_REG
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU2_TA_SQC_CTRL_REG
+#define CGTS_CU2_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
+#define CGTS_CU2_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU2_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU2_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU2_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU2_TA_SQC_CTRL_REG__SQC__SHIFT                                                                  0x10
+#define CGTS_CU2_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU2_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU2_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU2_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU2_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
+#define CGTS_CU2_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU2_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU2_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU2_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU2_TA_SQC_CTRL_REG__SQC_MASK                                                                    0x007F0000L
+#define CGTS_CU2_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU2_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU2_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU2_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU2_SP1_CTRL_REG
+#define CGTS_CU2_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
+#define CGTS_CU2_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU2_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU2_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU2_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU2_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
+#define CGTS_CU2_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU2_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU2_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU2_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU2_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
+#define CGTS_CU2_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU2_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU2_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU2_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU2_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
+#define CGTS_CU2_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU2_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU2_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU2_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU2_TD_TCP_CTRL_REG
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU2_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU3_SP0_CTRL_REG
+#define CGTS_CU3_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
+#define CGTS_CU3_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU3_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU3_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU3_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU3_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
+#define CGTS_CU3_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU3_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU3_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU3_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU3_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
+#define CGTS_CU3_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU3_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU3_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU3_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU3_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
+#define CGTS_CU3_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU3_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU3_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU3_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU3_LDS_SQ_CTRL_REG
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU3_TA_SQC_CTRL_REG
+#define CGTS_CU3_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
+#define CGTS_CU3_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU3_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU3_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU3_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU3_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
+#define CGTS_CU3_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU3_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU3_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU3_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+//CGTS_CU3_SP1_CTRL_REG
+#define CGTS_CU3_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
+#define CGTS_CU3_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU3_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU3_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU3_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU3_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
+#define CGTS_CU3_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU3_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU3_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU3_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU3_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
+#define CGTS_CU3_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU3_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU3_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU3_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU3_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
+#define CGTS_CU3_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU3_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU3_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU3_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU3_TD_TCP_CTRL_REG
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU3_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU4_SP0_CTRL_REG
+#define CGTS_CU4_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
+#define CGTS_CU4_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU4_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU4_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU4_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU4_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
+#define CGTS_CU4_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU4_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU4_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU4_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU4_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
+#define CGTS_CU4_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU4_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU4_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU4_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU4_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
+#define CGTS_CU4_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU4_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU4_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU4_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU4_LDS_SQ_CTRL_REG
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU4_TA_SQC_CTRL_REG
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC__SHIFT                                                                  0x10
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU4_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_MASK                                                                    0x007F0000L
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU4_SP1_CTRL_REG
+#define CGTS_CU4_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
+#define CGTS_CU4_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU4_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU4_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU4_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU4_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
+#define CGTS_CU4_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU4_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU4_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU4_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU4_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
+#define CGTS_CU4_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU4_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU4_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU4_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU4_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
+#define CGTS_CU4_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU4_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU4_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU4_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU4_TD_TCP_CTRL_REG
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU4_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU5_SP0_CTRL_REG
+#define CGTS_CU5_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
+#define CGTS_CU5_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU5_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU5_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU5_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU5_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
+#define CGTS_CU5_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU5_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU5_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU5_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU5_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
+#define CGTS_CU5_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU5_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU5_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU5_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU5_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
+#define CGTS_CU5_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU5_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU5_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU5_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU5_LDS_SQ_CTRL_REG
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU5_TA_SQC_CTRL_REG
+#define CGTS_CU5_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
+#define CGTS_CU5_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU5_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU5_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU5_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU5_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
+#define CGTS_CU5_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU5_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU5_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU5_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+//CGTS_CU5_SP1_CTRL_REG
+#define CGTS_CU5_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
+#define CGTS_CU5_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU5_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU5_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU5_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU5_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
+#define CGTS_CU5_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU5_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU5_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU5_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU5_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
+#define CGTS_CU5_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU5_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU5_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU5_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU5_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
+#define CGTS_CU5_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU5_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU5_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU5_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU5_TD_TCP_CTRL_REG
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU5_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU6_SP0_CTRL_REG
+#define CGTS_CU6_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
+#define CGTS_CU6_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU6_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU6_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU6_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU6_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
+#define CGTS_CU6_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU6_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU6_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU6_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU6_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
+#define CGTS_CU6_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU6_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU6_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU6_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU6_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
+#define CGTS_CU6_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU6_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU6_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU6_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU6_LDS_SQ_CTRL_REG
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU6_TA_SQC_CTRL_REG
+#define CGTS_CU6_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
+#define CGTS_CU6_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU6_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU6_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU6_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU6_TA_SQC_CTRL_REG__SQC__SHIFT                                                                  0x10
+#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU6_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
+#define CGTS_CU6_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU6_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU6_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU6_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_MASK                                                                    0x007F0000L
+#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU6_SP1_CTRL_REG
+#define CGTS_CU6_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
+#define CGTS_CU6_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU6_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU6_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU6_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU6_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
+#define CGTS_CU6_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU6_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU6_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU6_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU6_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
+#define CGTS_CU6_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU6_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU6_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU6_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU6_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
+#define CGTS_CU6_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU6_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU6_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU6_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU6_TD_TCP_CTRL_REG
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU6_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU7_SP0_CTRL_REG
+#define CGTS_CU7_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
+#define CGTS_CU7_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU7_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU7_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU7_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU7_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
+#define CGTS_CU7_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU7_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU7_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU7_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU7_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
+#define CGTS_CU7_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU7_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU7_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU7_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU7_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
+#define CGTS_CU7_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU7_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU7_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU7_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU7_LDS_SQ_CTRL_REG
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU7_TA_SQC_CTRL_REG
+#define CGTS_CU7_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
+#define CGTS_CU7_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU7_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU7_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU7_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU7_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
+#define CGTS_CU7_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU7_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU7_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU7_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+//CGTS_CU7_SP1_CTRL_REG
+#define CGTS_CU7_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
+#define CGTS_CU7_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU7_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU7_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU7_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU7_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
+#define CGTS_CU7_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU7_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU7_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU7_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU7_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
+#define CGTS_CU7_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU7_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU7_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU7_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU7_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
+#define CGTS_CU7_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU7_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU7_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU7_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU7_TD_TCP_CTRL_REG
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU7_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU8_SP0_CTRL_REG
+#define CGTS_CU8_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
+#define CGTS_CU8_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU8_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU8_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU8_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU8_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
+#define CGTS_CU8_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU8_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU8_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU8_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU8_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
+#define CGTS_CU8_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU8_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU8_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU8_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU8_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
+#define CGTS_CU8_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU8_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU8_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU8_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU8_LDS_SQ_CTRL_REG
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU8_TA_SQC_CTRL_REG
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC__SHIFT                                                                  0x10
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU8_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_MASK                                                                    0x007F0000L
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU8_SP1_CTRL_REG
+#define CGTS_CU8_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
+#define CGTS_CU8_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU8_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU8_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU8_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU8_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
+#define CGTS_CU8_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU8_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU8_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU8_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU8_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
+#define CGTS_CU8_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU8_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU8_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU8_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU8_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
+#define CGTS_CU8_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU8_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU8_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU8_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU8_TD_TCP_CTRL_REG
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU8_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU9_SP0_CTRL_REG
+#define CGTS_CU9_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
+#define CGTS_CU9_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU9_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU9_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU9_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU9_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
+#define CGTS_CU9_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU9_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU9_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU9_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU9_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
+#define CGTS_CU9_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU9_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU9_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU9_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU9_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
+#define CGTS_CU9_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU9_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU9_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU9_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU9_LDS_SQ_CTRL_REG
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU9_TA_SQC_CTRL_REG
+#define CGTS_CU9_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
+#define CGTS_CU9_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU9_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU9_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU9_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU9_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
+#define CGTS_CU9_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU9_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU9_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU9_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+//CGTS_CU9_SP1_CTRL_REG
+#define CGTS_CU9_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
+#define CGTS_CU9_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
+#define CGTS_CU9_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
+#define CGTS_CU9_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
+#define CGTS_CU9_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
+#define CGTS_CU9_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
+#define CGTS_CU9_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
+#define CGTS_CU9_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
+#define CGTS_CU9_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
+#define CGTS_CU9_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
+#define CGTS_CU9_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
+#define CGTS_CU9_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
+#define CGTS_CU9_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
+#define CGTS_CU9_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
+#define CGTS_CU9_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
+#define CGTS_CU9_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
+#define CGTS_CU9_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
+#define CGTS_CU9_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
+#define CGTS_CU9_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
+#define CGTS_CU9_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
+//CGTS_CU9_TD_TCP_CTRL_REG
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU9_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU10_SP0_CTRL_REG
+#define CGTS_CU10_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
+#define CGTS_CU10_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU10_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU10_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU10_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU10_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
+#define CGTS_CU10_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU10_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU10_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU10_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU10_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
+#define CGTS_CU10_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU10_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU10_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU10_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU10_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
+#define CGTS_CU10_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU10_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU10_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU10_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU10_LDS_SQ_CTRL_REG
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
+#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU10_TA_SQC_CTRL_REG
+#define CGTS_CU10_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
+#define CGTS_CU10_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU10_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU10_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU10_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU10_TA_SQC_CTRL_REG__SQC__SHIFT                                                                 0x10
+#define CGTS_CU10_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU10_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU10_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU10_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU10_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
+#define CGTS_CU10_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU10_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU10_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU10_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU10_TA_SQC_CTRL_REG__SQC_MASK                                                                   0x007F0000L
+#define CGTS_CU10_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU10_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU10_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU10_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU10_SP1_CTRL_REG
+#define CGTS_CU10_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
+#define CGTS_CU10_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU10_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU10_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU10_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU10_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
+#define CGTS_CU10_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU10_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU10_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU10_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU10_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
+#define CGTS_CU10_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU10_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU10_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU10_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU10_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
+#define CGTS_CU10_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU10_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU10_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU10_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU10_TD_TCP_CTRL_REG
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU10_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
+#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
+//CGTS_CU11_SP0_CTRL_REG
+#define CGTS_CU11_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
+#define CGTS_CU11_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU11_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU11_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU11_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU11_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
+#define CGTS_CU11_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU11_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU11_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU11_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU11_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
+#define CGTS_CU11_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU11_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU11_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU11_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU11_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
+#define CGTS_CU11_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU11_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU11_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU11_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU11_LDS_SQ_CTRL_REG
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
+#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU11_TA_SQC_CTRL_REG
+#define CGTS_CU11_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
+#define CGTS_CU11_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU11_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU11_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU11_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU11_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
+#define CGTS_CU11_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU11_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU11_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU11_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+//CGTS_CU11_SP1_CTRL_REG
+#define CGTS_CU11_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
+#define CGTS_CU11_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU11_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU11_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU11_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU11_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
+#define CGTS_CU11_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU11_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU11_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU11_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU11_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
+#define CGTS_CU11_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU11_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU11_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU11_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU11_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
+#define CGTS_CU11_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU11_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU11_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU11_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU11_TD_TCP_CTRL_REG
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU11_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
+#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
+//CGTS_CU12_SP0_CTRL_REG
+#define CGTS_CU12_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
+#define CGTS_CU12_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU12_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU12_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU12_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU12_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
+#define CGTS_CU12_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU12_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU12_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU12_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU12_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
+#define CGTS_CU12_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU12_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU12_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU12_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU12_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
+#define CGTS_CU12_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU12_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU12_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU12_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU12_LDS_SQ_CTRL_REG
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
+#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU12_TA_SQC_CTRL_REG
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC__SHIFT                                                                 0x10
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU12_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_MASK                                                                   0x007F0000L
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU12_SP1_CTRL_REG
+#define CGTS_CU12_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
+#define CGTS_CU12_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU12_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU12_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU12_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU12_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
+#define CGTS_CU12_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU12_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU12_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU12_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU12_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
+#define CGTS_CU12_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU12_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU12_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU12_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU12_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
+#define CGTS_CU12_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU12_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU12_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU12_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU12_TD_TCP_CTRL_REG
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU12_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
+#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
+//CGTS_CU13_SP0_CTRL_REG
+#define CGTS_CU13_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
+#define CGTS_CU13_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU13_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU13_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU13_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU13_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
+#define CGTS_CU13_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU13_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU13_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU13_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU13_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
+#define CGTS_CU13_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU13_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU13_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU13_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU13_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
+#define CGTS_CU13_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU13_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU13_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU13_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU13_LDS_SQ_CTRL_REG
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
+#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU13_TA_SQC_CTRL_REG
+#define CGTS_CU13_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
+#define CGTS_CU13_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU13_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU13_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU13_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU13_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
+#define CGTS_CU13_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU13_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU13_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU13_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+//CGTS_CU13_SP1_CTRL_REG
+#define CGTS_CU13_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
+#define CGTS_CU13_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU13_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU13_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU13_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU13_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
+#define CGTS_CU13_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU13_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU13_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU13_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU13_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
+#define CGTS_CU13_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU13_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU13_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU13_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU13_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
+#define CGTS_CU13_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU13_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU13_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU13_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU13_TD_TCP_CTRL_REG
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU13_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
+#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
+//CGTS_CU14_SP0_CTRL_REG
+#define CGTS_CU14_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
+#define CGTS_CU14_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU14_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU14_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU14_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU14_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
+#define CGTS_CU14_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU14_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU14_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU14_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU14_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
+#define CGTS_CU14_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU14_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU14_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU14_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU14_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
+#define CGTS_CU14_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU14_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU14_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU14_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU14_LDS_SQ_CTRL_REG
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
+#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU14_TA_SQC_CTRL_REG
+#define CGTS_CU14_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
+#define CGTS_CU14_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU14_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU14_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU14_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU14_TA_SQC_CTRL_REG__SQC__SHIFT                                                                 0x10
+#define CGTS_CU14_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                        0x17
+#define CGTS_CU14_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                   0x18
+#define CGTS_CU14_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                     0x1a
+#define CGTS_CU14_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
+#define CGTS_CU14_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
+#define CGTS_CU14_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU14_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU14_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU14_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU14_TA_SQC_CTRL_REG__SQC_MASK                                                                   0x007F0000L
+#define CGTS_CU14_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                          0x00800000L
+#define CGTS_CU14_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                     0x03000000L
+#define CGTS_CU14_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                       0x04000000L
+#define CGTS_CU14_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
+//CGTS_CU14_SP1_CTRL_REG
+#define CGTS_CU14_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
+#define CGTS_CU14_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU14_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU14_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU14_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU14_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
+#define CGTS_CU14_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU14_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU14_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU14_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU14_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
+#define CGTS_CU14_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU14_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU14_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU14_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU14_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
+#define CGTS_CU14_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU14_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU14_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU14_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU14_TD_TCP_CTRL_REG
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU14_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
+#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
+//CGTS_CU15_SP0_CTRL_REG
+#define CGTS_CU15_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
+#define CGTS_CU15_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU15_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU15_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU15_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU15_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
+#define CGTS_CU15_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU15_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU15_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU15_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU15_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
+#define CGTS_CU15_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU15_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU15_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU15_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU15_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
+#define CGTS_CU15_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU15_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU15_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU15_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU15_LDS_SQ_CTRL_REG
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
+#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
+#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
+//CGTS_CU15_TA_SQC_CTRL_REG
+#define CGTS_CU15_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
+#define CGTS_CU15_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU15_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU15_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU15_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU15_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
+#define CGTS_CU15_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU15_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU15_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU15_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+//CGTS_CU15_SP1_CTRL_REG
+#define CGTS_CU15_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
+#define CGTS_CU15_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU15_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU15_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU15_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU15_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
+#define CGTS_CU15_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
+#define CGTS_CU15_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
+#define CGTS_CU15_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
+#define CGTS_CU15_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
+#define CGTS_CU15_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
+#define CGTS_CU15_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU15_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU15_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU15_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU15_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
+#define CGTS_CU15_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
+#define CGTS_CU15_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
+#define CGTS_CU15_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
+#define CGTS_CU15_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
+//CGTS_CU15_TD_TCP_CTRL_REG
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU15_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
+#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
+//CGTS_CU0_TCPI_CTRL_REG
+#define CGTS_CU0_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
+#define CGTS_CU0_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU0_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU0_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU0_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU0_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
+#define CGTS_CU0_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
+#define CGTS_CU0_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU0_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU0_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU0_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU0_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
+//CGTS_CU1_TCPI_CTRL_REG
+#define CGTS_CU1_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
+#define CGTS_CU1_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU1_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU1_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU1_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU1_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
+#define CGTS_CU1_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
+#define CGTS_CU1_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU1_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU1_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU1_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU1_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
+//CGTS_CU2_TCPI_CTRL_REG
+#define CGTS_CU2_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
+#define CGTS_CU2_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU2_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU2_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU2_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU2_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
+#define CGTS_CU2_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
+#define CGTS_CU2_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU2_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU2_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU2_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU2_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
+//CGTS_CU3_TCPI_CTRL_REG
+#define CGTS_CU3_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
+#define CGTS_CU3_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU3_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU3_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU3_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU3_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
+#define CGTS_CU3_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
+#define CGTS_CU3_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU3_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU3_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU3_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU3_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
+//CGTS_CU4_TCPI_CTRL_REG
+#define CGTS_CU4_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
+#define CGTS_CU4_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU4_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU4_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU4_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU4_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
+#define CGTS_CU4_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
+#define CGTS_CU4_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU4_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU4_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU4_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU4_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
+//CGTS_CU5_TCPI_CTRL_REG
+#define CGTS_CU5_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
+#define CGTS_CU5_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU5_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU5_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU5_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU5_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
+#define CGTS_CU5_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
+#define CGTS_CU5_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU5_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU5_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU5_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU5_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
+//CGTS_CU6_TCPI_CTRL_REG
+#define CGTS_CU6_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
+#define CGTS_CU6_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU6_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU6_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU6_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU6_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
+#define CGTS_CU6_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
+#define CGTS_CU6_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU6_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU6_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU6_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU6_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
+//CGTS_CU7_TCPI_CTRL_REG
+#define CGTS_CU7_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
+#define CGTS_CU7_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU7_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU7_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU7_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU7_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
+#define CGTS_CU7_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
+#define CGTS_CU7_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU7_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU7_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU7_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU7_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
+//CGTS_CU8_TCPI_CTRL_REG
+#define CGTS_CU8_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
+#define CGTS_CU8_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU8_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU8_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU8_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU8_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
+#define CGTS_CU8_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
+#define CGTS_CU8_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU8_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU8_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU8_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU8_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
+//CGTS_CU9_TCPI_CTRL_REG
+#define CGTS_CU9_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
+#define CGTS_CU9_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
+#define CGTS_CU9_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
+#define CGTS_CU9_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
+#define CGTS_CU9_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
+#define CGTS_CU9_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
+#define CGTS_CU9_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
+#define CGTS_CU9_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
+#define CGTS_CU9_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
+#define CGTS_CU9_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
+#define CGTS_CU9_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
+#define CGTS_CU9_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
+//CGTS_CU10_TCPI_CTRL_REG
+#define CGTS_CU10_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
+#define CGTS_CU10_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU10_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU10_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU10_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU10_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
+#define CGTS_CU10_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
+#define CGTS_CU10_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU10_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU10_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU10_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU10_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
+//CGTS_CU11_TCPI_CTRL_REG
+#define CGTS_CU11_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
+#define CGTS_CU11_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU11_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU11_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU11_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU11_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
+#define CGTS_CU11_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
+#define CGTS_CU11_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU11_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU11_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU11_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU11_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
+//CGTS_CU12_TCPI_CTRL_REG
+#define CGTS_CU12_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
+#define CGTS_CU12_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU12_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU12_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU12_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU12_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
+#define CGTS_CU12_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
+#define CGTS_CU12_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU12_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU12_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU12_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU12_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
+//CGTS_CU13_TCPI_CTRL_REG
+#define CGTS_CU13_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
+#define CGTS_CU13_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU13_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU13_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU13_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU13_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
+#define CGTS_CU13_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
+#define CGTS_CU13_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU13_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU13_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU13_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU13_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
+//CGTS_CU14_TCPI_CTRL_REG
+#define CGTS_CU14_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
+#define CGTS_CU14_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU14_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU14_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU14_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU14_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
+#define CGTS_CU14_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
+#define CGTS_CU14_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU14_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU14_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU14_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU14_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
+//CGTS_CU15_TCPI_CTRL_REG
+#define CGTS_CU15_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
+#define CGTS_CU15_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
+#define CGTS_CU15_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
+#define CGTS_CU15_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
+#define CGTS_CU15_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
+#define CGTS_CU15_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
+#define CGTS_CU15_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
+#define CGTS_CU15_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
+#define CGTS_CU15_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
+#define CGTS_CU15_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
+#define CGTS_CU15_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
+#define CGTS_CU15_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
+//CGTT_SPI_PS_CLK_CTRL
+#define CGTT_SPI_PS_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
+#define CGTT_SPI_PS_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                     0x10
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                     0x11
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                     0x12
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                     0x13
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                     0x14
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                     0x15
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                     0x16
+#define CGTT_SPI_PS_CLK_CTRL__GRP6_OVERRIDE__SHIFT                                                            0x18
+#define CGTT_SPI_PS_CLK_CTRL__GRP5_OVERRIDE__SHIFT                                                            0x19
+#define CGTT_SPI_PS_CLK_CTRL__GRP4_OVERRIDE__SHIFT                                                            0x1a
+#define CGTT_SPI_PS_CLK_CTRL__GRP3_OVERRIDE__SHIFT                                                            0x1b
+#define CGTT_SPI_PS_CLK_CTRL__GRP2_OVERRIDE__SHIFT                                                            0x1c
+#define CGTT_SPI_PS_CLK_CTRL__GRP1_OVERRIDE__SHIFT                                                            0x1d
+#define CGTT_SPI_PS_CLK_CTRL__GRP0_OVERRIDE__SHIFT                                                            0x1e
+#define CGTT_SPI_PS_CLK_CTRL__REG_OVERRIDE__SHIFT                                                             0x1f
+#define CGTT_SPI_PS_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
+#define CGTT_SPI_PS_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                       0x00010000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                       0x00020000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                       0x00040000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                       0x00080000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                       0x00100000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                       0x00200000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                       0x00400000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP6_OVERRIDE_MASK                                                              0x01000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP5_OVERRIDE_MASK                                                              0x02000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP4_OVERRIDE_MASK                                                              0x04000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP3_OVERRIDE_MASK                                                              0x08000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP2_OVERRIDE_MASK                                                              0x10000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP1_OVERRIDE_MASK                                                              0x20000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP0_OVERRIDE_MASK                                                              0x40000000L
+#define CGTT_SPI_PS_CLK_CTRL__REG_OVERRIDE_MASK                                                               0x80000000L
+//CGTT_SPIS_CLK_CTRL
+#define CGTT_SPIS_CLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
+#define CGTT_SPIS_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x10
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x11
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x12
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x13
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x14
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x15
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x16
+#define CGTT_SPIS_CLK_CTRL__GRP6_OVERRIDE__SHIFT                                                              0x18
+#define CGTT_SPIS_CLK_CTRL__GRP5_OVERRIDE__SHIFT                                                              0x19
+#define CGTT_SPIS_CLK_CTRL__GRP4_OVERRIDE__SHIFT                                                              0x1a
+#define CGTT_SPIS_CLK_CTRL__GRP3_OVERRIDE__SHIFT                                                              0x1b
+#define CGTT_SPIS_CLK_CTRL__GRP2_OVERRIDE__SHIFT                                                              0x1c
+#define CGTT_SPIS_CLK_CTRL__GRP1_OVERRIDE__SHIFT                                                              0x1d
+#define CGTT_SPIS_CLK_CTRL__GRP0_OVERRIDE__SHIFT                                                              0x1e
+#define CGTT_SPIS_CLK_CTRL__REG_OVERRIDE__SHIFT                                                               0x1f
+#define CGTT_SPIS_CLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
+#define CGTT_SPIS_CLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00010000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00020000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00040000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00080000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00100000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00200000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00400000L
+#define CGTT_SPIS_CLK_CTRL__GRP6_OVERRIDE_MASK                                                                0x01000000L
+#define CGTT_SPIS_CLK_CTRL__GRP5_OVERRIDE_MASK                                                                0x02000000L
+#define CGTT_SPIS_CLK_CTRL__GRP4_OVERRIDE_MASK                                                                0x04000000L
+#define CGTT_SPIS_CLK_CTRL__GRP3_OVERRIDE_MASK                                                                0x08000000L
+#define CGTT_SPIS_CLK_CTRL__GRP2_OVERRIDE_MASK                                                                0x10000000L
+#define CGTT_SPIS_CLK_CTRL__GRP1_OVERRIDE_MASK                                                                0x20000000L
+#define CGTT_SPIS_CLK_CTRL__GRP0_OVERRIDE_MASK                                                                0x40000000L
+#define CGTT_SPIS_CLK_CTRL__REG_OVERRIDE_MASK                                                                 0x80000000L
+//CGTX_SPI_DEBUG_CLK_CTRL
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT                                                      0x0
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT                                                      0x6
+#define CGTX_SPI_DEBUG_CLK_CTRL__ALL_CLK_ON_OVERRIDE__SHIFT                                                   0x7
+#define CGTX_SPI_DEBUG_CLK_CTRL__SPI_SH_CLK_CONTROL__SHIFT                                                    0x8
+#define CGTX_SPI_DEBUG_CLK_CTRL__SPI_REPEATER_FGCG_OVERRIDE__SHIFT                                            0x9
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OFF_HYST_MASK                                                        0x0000003FL
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OVERRIDE_MASK                                                        0x00000040L
+#define CGTX_SPI_DEBUG_CLK_CTRL__ALL_CLK_ON_OVERRIDE_MASK                                                     0x00000080L
+#define CGTX_SPI_DEBUG_CLK_CTRL__SPI_SH_CLK_CONTROL_MASK                                                      0x00000100L
+#define CGTX_SPI_DEBUG_CLK_CTRL__SPI_REPEATER_FGCG_OVERRIDE_MASK                                              0x00000200L
+//CGTT_SPI_CLK_CTRL
+#define CGTT_SPI_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x14
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x15
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x16
+#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE__SHIFT                                                               0x1c
+#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE__SHIFT                                                               0x1d
+#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE__SHIFT                                                               0x1e
+#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_SPI_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00100000L
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00200000L
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00400000L
+#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE_MASK                                                                 0x10000000L
+#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE_MASK                                                                 0x20000000L
+#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE_MASK                                                                 0x40000000L
+#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//CGTT_PC_CLK_CTRL
+#define CGTT_PC_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_PC_CLK_CTRL__PC_RAM_FGCG_OVERRIDE__SHIFT                                                         0x11
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT                                                             0x12
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT                                                             0x18
+#define CGTT_PC_CLK_CTRL__PC_WRITE_CLK_EN_OVERRIDE__SHIFT                                                     0x19
+#define CGTT_PC_CLK_CTRL__PC_READ_CLK_EN_OVERRIDE__SHIFT                                                      0x1a
+#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE__SHIFT                                                               0x1b
+#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE__SHIFT                                                               0x1c
+#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE__SHIFT                                                               0x1d
+#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE__SHIFT                                                               0x1e
+#define CGTT_PC_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_PC_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_PC_CLK_CTRL__PC_RAM_FGCG_OVERRIDE_MASK                                                           0x00020000L
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OFF_HYST_MASK                                                               0x00FC0000L
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OVERRIDE_MASK                                                               0x01000000L
+#define CGTT_PC_CLK_CTRL__PC_WRITE_CLK_EN_OVERRIDE_MASK                                                       0x02000000L
+#define CGTT_PC_CLK_CTRL__PC_READ_CLK_EN_OVERRIDE_MASK                                                        0x04000000L
+#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE_MASK                                                                 0x08000000L
+#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE_MASK                                                                 0x10000000L
+#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE_MASK                                                                 0x20000000L
+#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE_MASK                                                                 0x40000000L
+#define CGTT_PC_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_BCI_CLK_CTRL
+#define CGTT_BCI_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_BCI_CLK_CTRL__RESERVED__SHIFT                                                                    0xc
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE__SHIFT                                                              0x18
+#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE__SHIFT                                                              0x19
+#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE__SHIFT                                                              0x1a
+#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE__SHIFT                                                              0x1b
+#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE__SHIFT                                                              0x1c
+#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE__SHIFT                                                              0x1d
+#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE__SHIFT                                                              0x1e
+#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_BCI_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_BCI_CLK_CTRL__RESERVED_MASK                                                                      0x0000F000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE_MASK                                                                0x01000000L
+#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE_MASK                                                                0x02000000L
+#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE_MASK                                                                0x04000000L
+#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE_MASK                                                                0x08000000L
+#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE_MASK                                                                0x10000000L
+#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE_MASK                                                                0x20000000L
+#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE_MASK                                                                0x40000000L
+#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//CGTT_VGT_CLK_CTRL
+#define CGTT_VGT_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE__SHIFT                                                                 0xf
+#define CGTT_VGT_CLK_CTRL__DBG_ENABLE__SHIFT                                                                  0x10
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE9__SHIFT                                                              0x18
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE8__SHIFT                                                              0x19
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                              0x1a
+#define CGTT_VGT_CLK_CTRL__PRIMGEN_OVERRIDE__SHIFT                                                            0x1b
+#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE__SHIFT                                                               0x1c
+#define CGTT_VGT_CLK_CTRL__GS_OVERRIDE__SHIFT                                                                 0x1d
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                               0x1e
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_VGT_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE_MASK                                                                   0x00008000L
+#define CGTT_VGT_CLK_CTRL__DBG_ENABLE_MASK                                                                    0x00010000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE9_MASK                                                                0x01000000L
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE8_MASK                                                                0x02000000L
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                0x04000000L
+#define CGTT_VGT_CLK_CTRL__PRIMGEN_OVERRIDE_MASK                                                              0x08000000L
+#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE_MASK                                                                 0x10000000L
+#define CGTT_VGT_CLK_CTRL__GS_OVERRIDE_MASK                                                                   0x20000000L
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE_MASK                                                                 0x40000000L
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//CGTT_IA_CLK_CTRL
+#define CGTT_IA_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x18
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE__SHIFT                                                                  0x19
+#define CGTT_IA_CLK_CTRL__DBG_ENABLE__SHIFT                                                                   0x1a
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                               0x1b
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                               0x1c
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                               0x1d
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1e
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_IA_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x01000000L
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE_MASK                                                                    0x02000000L
+#define CGTT_IA_CLK_CTRL__DBG_ENABLE_MASK                                                                     0x04000000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                 0x08000000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                 0x10000000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                                 0x20000000L
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x40000000L
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_WD_CLK_CTRL
+#define CGTT_WD_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_WD_CLK_CTRL__PERF_ENABLE__SHIFT                                                                  0xf
+#define CGTT_WD_CLK_CTRL__DBG_ENABLE__SHIFT                                                                   0x10
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE8__SHIFT                                                               0x19
+#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x1a
+#define CGTT_WD_CLK_CTRL__PRIMGEN_OVERRIDE__SHIFT                                                             0x1b
+#define CGTT_WD_CLK_CTRL__TESS_OVERRIDE__SHIFT                                                                0x1c
+#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1d
+#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE__SHIFT                                                          0x1e
+#define CGTT_WD_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_WD_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_WD_CLK_CTRL__PERF_ENABLE_MASK                                                                    0x00008000L
+#define CGTT_WD_CLK_CTRL__DBG_ENABLE_MASK                                                                     0x00010000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE8_MASK                                                                 0x02000000L
+#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x04000000L
+#define CGTT_WD_CLK_CTRL__PRIMGEN_OVERRIDE_MASK                                                               0x08000000L
+#define CGTT_WD_CLK_CTRL__TESS_OVERRIDE_MASK                                                                  0x10000000L
+#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x20000000L
+#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE_MASK                                                            0x40000000L
+#define CGTT_WD_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_PA_CLK_CTRL
+#define CGTT_PA_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_PA_CLK_CTRL__DEBUG_BUS_EN__SHIFT                                                                 0x17
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x18
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                               0x19
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                               0x1a
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                               0x1b
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                               0x1c
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE__SHIFT                                                              0x1d
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE__SHIFT                                                              0x1e
+#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE__SHIFT                                                             0x1f
+#define CGTT_PA_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_PA_CLK_CTRL__DEBUG_BUS_EN_MASK                                                                   0x00800000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x01000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                 0x02000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                 0x04000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                 0x08000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                 0x10000000L
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE_MASK                                                                0x20000000L
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE_MASK                                                                0x40000000L
+#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE_MASK                                                               0x80000000L
+//CGTT_SC_CLK_CTRL0
+#define CGTT_SC_CLK_CTRL0__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_STALL_OVERRIDE__SHIFT                                              0x10
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x11
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x12
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x13
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x14
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x15
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x16
+#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE__SHIFT                                                      0x17
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_OVERRIDE__SHIFT                                                    0x18
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT                                                              0x19
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT                                                              0x1a
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT                                                              0x1b
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT                                                              0x1c
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT                                                              0x1d
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT                                                              0x1e
+#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE__SHIFT                                                            0x1f
+#define CGTT_SC_CLK_CTRL0__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_STALL_OVERRIDE_MASK                                                0x00010000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE5_MASK                                                          0x00020000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE4_MASK                                                          0x00040000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE3_MASK                                                          0x00080000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE2_MASK                                                          0x00100000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE1_MASK                                                          0x00200000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE0_MASK                                                          0x00400000L
+#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE_MASK                                                        0x00800000L
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_OVERRIDE_MASK                                                      0x01000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE5_MASK                                                                0x02000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE4_MASK                                                                0x04000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE3_MASK                                                                0x08000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE2_MASK                                                                0x10000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE1_MASK                                                                0x20000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE0_MASK                                                                0x40000000L
+#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE_MASK                                                              0x80000000L
+//CGTT_SC_CLK_CTRL1
+#define CGTT_SC_CLK_CTRL1__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SC_CLK_CTRL1__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE__SHIFT                                              0x11
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE__SHIFT                                              0x12
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_STALL_OVERRIDE__SHIFT                                     0x13
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_STALL_OVERRIDE__SHIFT                                           0x14
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE__SHIFT                                            0x15
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE__SHIFT                                                      0x16
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE__SHIFT                                                    0x19
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE__SHIFT                                                    0x1a
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_OVERRIDE__SHIFT                                           0x1b
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE__SHIFT                                                 0x1c
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE__SHIFT                                                  0x1d
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_OVERRIDE__SHIFT                                                            0x1e
+#define CGTT_SC_CLK_CTRL1__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SC_CLK_CTRL1__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE_MASK                                                0x00020000L
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE_MASK                                                0x00040000L
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_STALL_OVERRIDE_MASK                                       0x00080000L
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_STALL_OVERRIDE_MASK                                             0x00100000L
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE_MASK                                              0x00200000L
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE_MASK                                                        0x00400000L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE_MASK                                                      0x02000000L
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE_MASK                                                      0x04000000L
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_OVERRIDE_MASK                                             0x08000000L
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE_MASK                                                   0x10000000L
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE_MASK                                                    0x20000000L
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_OVERRIDE_MASK                                                              0x40000000L
+//CGTT_SC_CLK_CTRL2
+#define CGTT_SC_CLK_CTRL2__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SC_CLK_CTRL2__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SC_CLK_CTRL2__SCF_SCB_INTF_CLK_OVERRIDE__SHIFT                                                   0x1b
+#define CGTT_SC_CLK_CTRL2__SC_PKR_INTF_CLK_OVERRIDE__SHIFT                                                    0x1c
+#define CGTT_SC_CLK_CTRL2__SC_DB_INTF_CLK_OVERRIDE__SHIFT                                                     0x1d
+#define CGTT_SC_CLK_CTRL2__PA_SC_INTF_CLK_OVERRIDE__SHIFT                                                     0x1e
+#define CGTT_SC_CLK_CTRL2__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SC_CLK_CTRL2__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SC_CLK_CTRL2__SCF_SCB_INTF_CLK_OVERRIDE_MASK                                                     0x08000000L
+#define CGTT_SC_CLK_CTRL2__SC_PKR_INTF_CLK_OVERRIDE_MASK                                                      0x10000000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_INTF_CLK_OVERRIDE_MASK                                                       0x20000000L
+#define CGTT_SC_CLK_CTRL2__PA_SC_INTF_CLK_OVERRIDE_MASK                                                       0x40000000L
+//CGTT_SQ_CLK_CTRL
+#define CGTT_SQ_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_SQ_CLK_CTRL__PERFMON_OVERRIDE__SHIFT                                                             0x1d
+#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1e
+#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_SQ_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_SQ_CLK_CTRL__PERFMON_OVERRIDE_MASK                                                               0x20000000L
+#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x40000000L
+#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_SQG_CLK_CTRL
+#define CGTT_SQG_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE__SHIFT                                                             0x1c
+#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE__SHIFT                                                            0x1d
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                               0x1e
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_SQG_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE_MASK                                                               0x10000000L
+#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE_MASK                                                              0x20000000L
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE_MASK                                                                 0x40000000L
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//SQ_ALU_CLK_CTRL
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT                                                               0x0
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT                                                               0x10
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH0_MASK                                                                 0x0000FFFFL
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH1_MASK                                                                 0xFFFF0000L
+//SQ_TEX_CLK_CTRL
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT                                                               0x0
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT                                                               0x10
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH0_MASK                                                                 0x0000FFFFL
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH1_MASK                                                                 0xFFFF0000L
+//SQ_LDS_CLK_CTRL
+#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT                                                               0x0
+#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT                                                               0x10
+#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH0_MASK                                                                 0x0000FFFFL
+#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH1_MASK                                                                 0xFFFF0000L
+//SQ_POWER_THROTTLE
+#define SQ_POWER_THROTTLE__MIN_POWER__SHIFT                                                                   0x0
+#define SQ_POWER_THROTTLE__MAX_POWER__SHIFT                                                                   0x10
+#define SQ_POWER_THROTTLE__PHASE_OFFSET__SHIFT                                                                0x1e
+#define SQ_POWER_THROTTLE__MIN_POWER_MASK                                                                     0x00003FFFL
+#define SQ_POWER_THROTTLE__MAX_POWER_MASK                                                                     0x3FFF0000L
+#define SQ_POWER_THROTTLE__PHASE_OFFSET_MASK                                                                  0xC0000000L
+//SQ_POWER_THROTTLE2
+#define SQ_POWER_THROTTLE2__MAX_POWER_DELTA__SHIFT                                                            0x0
+#define SQ_POWER_THROTTLE2__SHORT_TERM_INTERVAL_SIZE__SHIFT                                                   0x10
+#define SQ_POWER_THROTTLE2__LONG_TERM_INTERVAL_RATIO__SHIFT                                                   0x1b
+#define SQ_POWER_THROTTLE2__USE_REF_CLOCK__SHIFT                                                              0x1f
+#define SQ_POWER_THROTTLE2__MAX_POWER_DELTA_MASK                                                              0x00003FFFL
+#define SQ_POWER_THROTTLE2__SHORT_TERM_INTERVAL_SIZE_MASK                                                     0x03FF0000L
+#define SQ_POWER_THROTTLE2__LONG_TERM_INTERVAL_RATIO_MASK                                                     0x78000000L
+#define SQ_POWER_THROTTLE2__USE_REF_CLOCK_MASK                                                                0x80000000L
+//TD_CGTT_CTRL
+#define TD_CGTT_CTRL__ON_DELAY__SHIFT                                                                         0x0
+#define TD_CGTT_CTRL__OFF_HYSTERESIS__SHIFT                                                                   0x4
+#define TD_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT                                                                   0x18
+#define TD_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT                                                                   0x19
+#define TD_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT                                                                   0x1a
+#define TD_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT                                                                   0x1b
+#define TD_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT                                                                   0x1c
+#define TD_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT                                                                   0x1d
+#define TD_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT                                                                   0x1e
+#define TD_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT                                                                   0x1f
+#define TD_CGTT_CTRL__ON_DELAY_MASK                                                                           0x0000000FL
+#define TD_CGTT_CTRL__OFF_HYSTERESIS_MASK                                                                     0x00000FF0L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE7_MASK                                                                     0x01000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE6_MASK                                                                     0x02000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE5_MASK                                                                     0x04000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE4_MASK                                                                     0x08000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE3_MASK                                                                     0x10000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE2_MASK                                                                     0x20000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE1_MASK                                                                     0x40000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE0_MASK                                                                     0x80000000L
+//TA_CGTT_CTRL
+#define TA_CGTT_CTRL__ON_DELAY__SHIFT                                                                         0x0
+#define TA_CGTT_CTRL__OFF_HYSTERESIS__SHIFT                                                                   0x4
+#define TA_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT                                                                   0x18
+#define TA_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT                                                                   0x19
+#define TA_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT                                                                   0x1a
+#define TA_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT                                                                   0x1b
+#define TA_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT                                                                   0x1c
+#define TA_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT                                                                   0x1d
+#define TA_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT                                                                   0x1e
+#define TA_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT                                                                   0x1f
+#define TA_CGTT_CTRL__ON_DELAY_MASK                                                                           0x0000000FL
+#define TA_CGTT_CTRL__OFF_HYSTERESIS_MASK                                                                     0x00000FF0L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE7_MASK                                                                     0x01000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE6_MASK                                                                     0x02000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE5_MASK                                                                     0x04000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE4_MASK                                                                     0x08000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE3_MASK                                                                     0x10000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE2_MASK                                                                     0x20000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE1_MASK                                                                     0x40000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE0_MASK                                                                     0x80000000L
+//CGTT_TCPI_CLK_CTRL
+#define CGTT_TCPI_CLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
+#define CGTT_TCPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define CGTT_TCPI_CLK_CTRL__SPARE__SHIFT                                                                      0xc
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                             0x18
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define CGTT_TCPI_CLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
+#define CGTT_TCPI_CLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define CGTT_TCPI_CLK_CTRL__SPARE_MASK                                                                        0x0000F000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                               0x01000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+//TCX_CGTT_SCLK_CTRL
+#define TCX_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define TCX_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
+#define TCX_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
+#define TCX_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
+#define TCX_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
+#define TCX_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define TCX_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define TCX_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
+#define TCX_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
+#define TCX_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
+#define TCX_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
+#define TCX_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+//DB_CGTT_CLK_CTRL_0
+#define DB_CGTT_CLK_CTRL_0__ON_DELAY__SHIFT                                                                   0x0
+#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define DB_CGTT_CLK_CTRL_0__RESERVED__SHIFT                                                                   0xc
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7__SHIFT                                                             0x18
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6__SHIFT                                                             0x19
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5__SHIFT                                                             0x1a
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4__SHIFT                                                             0x1b
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3__SHIFT                                                             0x1c
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2__SHIFT                                                             0x1d
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define DB_CGTT_CLK_CTRL_0__ON_DELAY_MASK                                                                     0x0000000FL
+#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define DB_CGTT_CLK_CTRL_0__RESERVED_MASK                                                                     0x0000F000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7_MASK                                                               0x01000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6_MASK                                                               0x02000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5_MASK                                                               0x04000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4_MASK                                                               0x08000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3_MASK                                                               0x10000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2_MASK                                                               0x20000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+//CB_CGTT_SCLK_CTRL
+#define CB_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                              0x1f
+#define CB_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                                0x80000000L
+//TCC_CGTT_SCLK_CTRL
+#define TCC_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define TCC_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+//TCC_CGTT_SCLK_CTRL2
+#define TCC_CGTT_SCLK_CTRL2__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define TCC_CGTT_SCLK_CTRL2__SOFT_OVERRIDE4__SHIFT                                                            0x1b
+#define TCC_CGTT_SCLK_CTRL2__SOFT_OVERRIDE3__SHIFT                                                            0x1c
+#define TCC_CGTT_SCLK_CTRL2__SOFT_OVERRIDE2__SHIFT                                                            0x1d
+#define TCC_CGTT_SCLK_CTRL2__SOFT_OVERRIDE1__SHIFT                                                            0x1e
+#define TCC_CGTT_SCLK_CTRL2__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define TCC_CGTT_SCLK_CTRL2__SOFT_OVERRIDE4_MASK                                                              0x08000000L
+#define TCC_CGTT_SCLK_CTRL2__SOFT_OVERRIDE3_MASK                                                              0x10000000L
+#define TCC_CGTT_SCLK_CTRL2__SOFT_OVERRIDE2_MASK                                                              0x20000000L
+#define TCC_CGTT_SCLK_CTRL2__SOFT_OVERRIDE1_MASK                                                              0x40000000L
+//TCC_CGTT_SCLK_CTRL3
+#define TCC_CGTT_SCLK_CTRL3__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE18__SHIFT                                                           0xc
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE17__SHIFT                                                           0xd
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE16__SHIFT                                                           0xe
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE15__SHIFT                                                           0xf
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE14__SHIFT                                                           0x10
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE13__SHIFT                                                           0x11
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE12__SHIFT                                                           0x12
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE11__SHIFT                                                           0x13
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE10__SHIFT                                                           0x14
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE9__SHIFT                                                            0x15
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE8__SHIFT                                                            0x17
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE7__SHIFT                                                            0x18
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE6__SHIFT                                                            0x19
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE5__SHIFT                                                            0x1a
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE4__SHIFT                                                            0x1b
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE3__SHIFT                                                            0x1c
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE2__SHIFT                                                            0x1d
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE1__SHIFT                                                            0x1e
+#define TCC_CGTT_SCLK_CTRL3__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE18_MASK                                                             0x00001000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE17_MASK                                                             0x00002000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE16_MASK                                                             0x00004000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE15_MASK                                                             0x00008000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE14_MASK                                                             0x00010000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE13_MASK                                                             0x00020000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE12_MASK                                                             0x00040000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE11_MASK                                                             0x00080000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE10_MASK                                                             0x00100000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE9_MASK                                                              0x00200000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE8_MASK                                                              0x00800000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE7_MASK                                                              0x01000000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE6_MASK                                                              0x02000000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE5_MASK                                                              0x04000000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE4_MASK                                                              0x08000000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE3_MASK                                                              0x10000000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE2_MASK                                                              0x20000000L
+#define TCC_CGTT_SCLK_CTRL3__SOFT_OVERRIDE1_MASK                                                              0x40000000L
+//TCA_CGTT_SCLK_CTRL
+#define TCA_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define TCA_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+//CGTT_CP_CLK_CTRL
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                                0xf
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                        0x1d
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                            0x1e
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                            0x1f
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                  0x00008000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                          0x20000000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                              0x40000000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                              0x80000000L
+//CGTT_CPC_CLK_CTRL
+#define CGTT_CPC_CLK_CTRL__REG_CLK_OFF_HYSTERESIS__SHIFT                                                      0x0
+#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                               0xf
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1d
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
+#define CGTT_CPC_CLK_CTRL__REG_CLK_OFF_HYSTERESIS_MASK                                                        0x0000000FL
+#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                 0x00008000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x20000000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
+//CGTT_RLC_CLK_CTRL
+#define CGTT_RLC_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
+#define CGTT_RLC_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
+//RLC_GFX_RM_CNTL
+#define RLC_GFX_RM_CNTL__RLC_GFX_RM_VALID__SHIFT                                                              0x0
+#define RLC_GFX_RM_CNTL__RESERVED__SHIFT                                                                      0x1
+#define RLC_GFX_RM_CNTL__RLC_GFX_RM_VALID_MASK                                                                0x00000001L
+#define RLC_GFX_RM_CNTL__RESERVED_MASK                                                                        0xFFFFFFFEL
+//RMI_CGTT_SCLK_CTRL
+#define RMI_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
+#define RMI_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define RMI_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
+#define RMI_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+//CGTT_TCPF_CLK_CTRL
+#define CGTT_TCPF_CLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
+#define CGTT_TCPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define CGTT_TCPF_CLK_CTRL__SPARE__SHIFT                                                                      0xc
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                             0x18
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define CGTT_TCPF_CLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
+#define CGTT_TCPF_CLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define CGTT_TCPF_CLK_CTRL__SPARE_MASK                                                                        0x0000F000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                               0x01000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+
+
+// addressBlock: xcd0_gc_hypdec
+//CP_HYP_PFP_UCODE_ADDR
+#define CP_HYP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
+#define CP_HYP_PFP_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x00003FFFL
+//CP_PFP_UCODE_ADDR
+#define CP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                  0x0
+#define CP_PFP_UCODE_ADDR__UCODE_ADDR_MASK                                                                    0x00003FFFL
+//CP_HYP_PFP_UCODE_DATA
+#define CP_HYP_PFP_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
+#define CP_HYP_PFP_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
+//CP_PFP_UCODE_DATA
+#define CP_PFP_UCODE_DATA__UCODE_DATA__SHIFT                                                                  0x0
+#define CP_PFP_UCODE_DATA__UCODE_DATA_MASK                                                                    0xFFFFFFFFL
+//CP_HYP_ME_UCODE_ADDR
+#define CP_HYP_ME_UCODE_ADDR__UCODE_ADDR__SHIFT                                                               0x0
+#define CP_HYP_ME_UCODE_ADDR__UCODE_ADDR_MASK                                                                 0x00001FFFL
+//CP_ME_RAM_RADDR
+#define CP_ME_RAM_RADDR__ME_RAM_RADDR__SHIFT                                                                  0x0
+#define CP_ME_RAM_RADDR__ME_RAM_RADDR_MASK                                                                    0x00001FFFL
+//CP_ME_RAM_WADDR
+#define CP_ME_RAM_WADDR__ME_RAM_WADDR__SHIFT                                                                  0x0
+#define CP_ME_RAM_WADDR__ME_RAM_WADDR_MASK                                                                    0x00001FFFL
+//CP_HYP_ME_UCODE_DATA
+#define CP_HYP_ME_UCODE_DATA__UCODE_DATA__SHIFT                                                               0x0
+#define CP_HYP_ME_UCODE_DATA__UCODE_DATA_MASK                                                                 0xFFFFFFFFL
+//CP_ME_RAM_DATA
+#define CP_ME_RAM_DATA__ME_RAM_DATA__SHIFT                                                                    0x0
+#define CP_ME_RAM_DATA__ME_RAM_DATA_MASK                                                                      0xFFFFFFFFL
+//CP_CE_UCODE_ADDR
+#define CP_CE_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                   0x0
+#define CP_CE_UCODE_ADDR__UCODE_ADDR_MASK                                                                     0x00000FFFL
+//CP_HYP_CE_UCODE_ADDR
+#define CP_HYP_CE_UCODE_ADDR__UCODE_ADDR__SHIFT                                                               0x0
+#define CP_HYP_CE_UCODE_ADDR__UCODE_ADDR_MASK                                                                 0x00000FFFL
+//CP_CE_UCODE_DATA
+#define CP_CE_UCODE_DATA__UCODE_DATA__SHIFT                                                                   0x0
+#define CP_CE_UCODE_DATA__UCODE_DATA_MASK                                                                     0xFFFFFFFFL
+//CP_HYP_CE_UCODE_DATA
+#define CP_HYP_CE_UCODE_DATA__UCODE_DATA__SHIFT                                                               0x0
+#define CP_HYP_CE_UCODE_DATA__UCODE_DATA_MASK                                                                 0xFFFFFFFFL
+//CP_HYP_MEC1_UCODE_ADDR
+#define CP_HYP_MEC1_UCODE_ADDR__UCODE_ADDR__SHIFT                                                             0x0
+#define CP_HYP_MEC1_UCODE_ADDR__UCODE_ADDR_MASK                                                               0x0001FFFFL
+//CP_MEC_ME1_UCODE_ADDR
+#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
+#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x0001FFFFL
+//CP_HYP_MEC1_UCODE_DATA
+#define CP_HYP_MEC1_UCODE_DATA__UCODE_DATA__SHIFT                                                             0x0
+#define CP_HYP_MEC1_UCODE_DATA__UCODE_DATA_MASK                                                               0xFFFFFFFFL
+//CP_MEC_ME1_UCODE_DATA
+#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
+#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
+//CP_HYP_MEC2_UCODE_ADDR
+#define CP_HYP_MEC2_UCODE_ADDR__UCODE_ADDR__SHIFT                                                             0x0
+#define CP_HYP_MEC2_UCODE_ADDR__UCODE_ADDR_MASK                                                               0x0001FFFFL
+//CP_MEC_ME2_UCODE_ADDR
+#define CP_MEC_ME2_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
+#define CP_MEC_ME2_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x0001FFFFL
+//CP_HYP_MEC2_UCODE_DATA
+#define CP_HYP_MEC2_UCODE_DATA__UCODE_DATA__SHIFT                                                             0x0
+#define CP_HYP_MEC2_UCODE_DATA__UCODE_DATA_MASK                                                               0xFFFFFFFFL
+//CP_MEC_ME2_UCODE_DATA
+#define CP_MEC_ME2_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
+#define CP_MEC_ME2_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
+//CP_HYP_PFP_UCODE_CHKSUM
+#define CP_HYP_PFP_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                          0x0
+#define CP_HYP_PFP_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                            0xFFFFFFFFL
+//CP_HYP_CE_UCODE_CHKSUM
+#define CP_HYP_CE_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                           0x0
+#define CP_HYP_CE_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                             0xFFFFFFFFL
+//CP_HYP_ME_UCODE_CHKSUM
+#define CP_HYP_ME_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                           0x0
+#define CP_HYP_ME_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                             0xFFFFFFFFL
+//CP_HYP_MEC_ME1_UCODE_CHKSUM
+#define CP_HYP_MEC_ME1_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                      0x0
+#define CP_HYP_MEC_ME1_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                        0xFFFFFFFFL
+//CP_HYP_MEC_ME2_UCODE_CHKSUM
+#define CP_HYP_MEC_ME2_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                      0x0
+#define CP_HYP_MEC_ME2_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                        0xFFFFFFFFL
+//CP_HYP_XCP_CTL
+#define CP_HYP_XCP_CTL__VIRTUAL_XCC_ID__SHIFT                                                                 0x0
+#define CP_HYP_XCP_CTL__NUM_XCC_IN_XCP__SHIFT                                                                 0x3
+#define CP_HYP_XCP_CTL__VIRTUAL_XCC_ID_MASK                                                                   0x00000007L
+#define CP_HYP_XCP_CTL__NUM_XCC_IN_XCP_MASK                                                                   0x00000078L
+//RLC_GPM_UCODE_ADDR
+#define RLC_GPM_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                 0x0
+#define RLC_GPM_UCODE_ADDR__RESERVED__SHIFT                                                                   0xe
+#define RLC_GPM_UCODE_ADDR__UCODE_ADDR_MASK                                                                   0x00003FFFL
+#define RLC_GPM_UCODE_ADDR__RESERVED_MASK                                                                     0xFFFFC000L
+//RLC_GPM_UCODE_DATA
+#define RLC_GPM_UCODE_DATA__UCODE_DATA__SHIFT                                                                 0x0
+#define RLC_GPM_UCODE_DATA__UCODE_DATA_MASK                                                                   0xFFFFFFFFL
+//GRBM_GFX_INDEX_SR_SELECT
+#define GRBM_GFX_INDEX_SR_SELECT__INDEX__SHIFT                                                                0x0
+#define GRBM_GFX_INDEX_SR_SELECT__VF_PF__SHIFT                                                                0x1f
+#define GRBM_GFX_INDEX_SR_SELECT__INDEX_MASK                                                                  0x00000007L
+#define GRBM_GFX_INDEX_SR_SELECT__VF_PF_MASK                                                                  0x80000000L
+//GRBM_GFX_INDEX_SR_DATA
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_INDEX__SHIFT                                                         0x0
+#define GRBM_GFX_INDEX_SR_DATA__SH_INDEX__SHIFT                                                               0x8
+#define GRBM_GFX_INDEX_SR_DATA__SE_INDEX__SHIFT                                                               0x10
+#define GRBM_GFX_INDEX_SR_DATA__SH_BROADCAST_WRITES__SHIFT                                                    0x1d
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_BROADCAST_WRITES__SHIFT                                              0x1e
+#define GRBM_GFX_INDEX_SR_DATA__SE_BROADCAST_WRITES__SHIFT                                                    0x1f
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_INDEX_MASK                                                           0x000000FFL
+#define GRBM_GFX_INDEX_SR_DATA__SH_INDEX_MASK                                                                 0x0000FF00L
+#define GRBM_GFX_INDEX_SR_DATA__SE_INDEX_MASK                                                                 0x00FF0000L
+#define GRBM_GFX_INDEX_SR_DATA__SH_BROADCAST_WRITES_MASK                                                      0x20000000L
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_BROADCAST_WRITES_MASK                                                0x40000000L
+#define GRBM_GFX_INDEX_SR_DATA__SE_BROADCAST_WRITES_MASK                                                      0x80000000L
+//GRBM_GFX_CNTL_SR_SELECT
+#define GRBM_GFX_CNTL_SR_SELECT__INDEX__SHIFT                                                                 0x0
+#define GRBM_GFX_CNTL_SR_SELECT__VF_PF__SHIFT                                                                 0x1f
+#define GRBM_GFX_CNTL_SR_SELECT__INDEX_MASK                                                                   0x00000007L
+#define GRBM_GFX_CNTL_SR_SELECT__VF_PF_MASK                                                                   0x80000000L
+//GRBM_GFX_CNTL_SR_DATA
+#define GRBM_GFX_CNTL_SR_DATA__PIPEID__SHIFT                                                                  0x0
+#define GRBM_GFX_CNTL_SR_DATA__MEID__SHIFT                                                                    0x2
+#define GRBM_GFX_CNTL_SR_DATA__VMID__SHIFT                                                                    0x4
+#define GRBM_GFX_CNTL_SR_DATA__QUEUEID__SHIFT                                                                 0x8
+#define GRBM_GFX_CNTL_SR_DATA__PIPEID_MASK                                                                    0x00000003L
+#define GRBM_GFX_CNTL_SR_DATA__MEID_MASK                                                                      0x0000000CL
+#define GRBM_GFX_CNTL_SR_DATA__VMID_MASK                                                                      0x000000F0L
+#define GRBM_GFX_CNTL_SR_DATA__QUEUEID_MASK                                                                   0x00000700L
+//GRBM_MCM_ADDR
+#define GRBM_MCM_ADDR__MCM_ADDR_IH__SHIFT                                                                     0x0
+#define GRBM_MCM_ADDR__MCM_ADDR_IH_MASK                                                                       0x000000FFL
+//RLC_GPU_IOV_VF_ENABLE
+#define RLC_GPU_IOV_VF_ENABLE__VF_ENABLE__SHIFT                                                               0x0
+#define RLC_GPU_IOV_VF_ENABLE__RESERVED__SHIFT                                                                0x1
+#define RLC_GPU_IOV_VF_ENABLE__VF_NUM__SHIFT                                                                  0x10
+#define RLC_GPU_IOV_VF_ENABLE__VF_ENABLE_MASK                                                                 0x00000001L
+#define RLC_GPU_IOV_VF_ENABLE__RESERVED_MASK                                                                  0x0000FFFEL
+#define RLC_GPU_IOV_VF_ENABLE__VF_NUM_MASK                                                                    0xFFFF0000L
+//RLC_GPU_IOV_CFG_REG6
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_SIZE__SHIFT                                                               0x0
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_LOCATION__SHIFT                                                           0x7
+#define RLC_GPU_IOV_CFG_REG6__RESERVED__SHIFT                                                                 0x8
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_OFFSET__SHIFT                                                             0xa
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_SIZE_MASK                                                                 0x0000007FL
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_LOCATION_MASK                                                             0x00000080L
+#define RLC_GPU_IOV_CFG_REG6__RESERVED_MASK                                                                   0x00000300L
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_OFFSET_MASK                                                               0xFFFFFC00L
+//RLC_GPU_IOV_CFG_REG8
+#define RLC_GPU_IOV_CFG_REG8__VM_BUSY_STATUS__SHIFT                                                           0x0
+#define RLC_GPU_IOV_CFG_REG8__VM_BUSY_STATUS_MASK                                                             0xFFFFFFFFL
+//RLC_RLCV_TIMER_INT_0
+#define RLC_RLCV_TIMER_INT_0__TIMER__SHIFT                                                                    0x0
+#define RLC_RLCV_TIMER_INT_0__TIMER_MASK                                                                      0xFFFFFFFFL
+//RLC_RLCV_TIMER_CTRL
+#define RLC_RLCV_TIMER_CTRL__TIMER_0_EN__SHIFT                                                                0x0
+#define RLC_RLCV_TIMER_CTRL__TIMER_1_EN__SHIFT                                                                0x1
+#define RLC_RLCV_TIMER_CTRL__RESERVED__SHIFT                                                                  0x2
+#define RLC_RLCV_TIMER_CTRL__TIMER_0_EN_MASK                                                                  0x00000001L
+#define RLC_RLCV_TIMER_CTRL__TIMER_1_EN_MASK                                                                  0x00000002L
+#define RLC_RLCV_TIMER_CTRL__RESERVED_MASK                                                                    0xFFFFFFFCL
+//RLC_RLCV_TIMER_STAT
+#define RLC_RLCV_TIMER_STAT__TIMER_0_STAT__SHIFT                                                              0x0
+#define RLC_RLCV_TIMER_STAT__TIMER_1_STAT__SHIFT                                                              0x1
+#define RLC_RLCV_TIMER_STAT__RESERVED__SHIFT                                                                  0x2
+#define RLC_RLCV_TIMER_STAT__TIMER_0_ENABLE_SYNC__SHIFT                                                       0x8
+#define RLC_RLCV_TIMER_STAT__TIMER_1_ENABLE_SYNC__SHIFT                                                       0x9
+#define RLC_RLCV_TIMER_STAT__TIMER_0_STAT_MASK                                                                0x00000001L
+#define RLC_RLCV_TIMER_STAT__TIMER_1_STAT_MASK                                                                0x00000002L
+#define RLC_RLCV_TIMER_STAT__RESERVED_MASK                                                                    0x000000FCL
+#define RLC_RLCV_TIMER_STAT__TIMER_0_ENABLE_SYNC_MASK                                                         0x00000100L
+#define RLC_RLCV_TIMER_STAT__TIMER_1_ENABLE_SYNC_MASK                                                         0x00000200L
+//RLC_GPU_IOV_VF_DOORBELL_STATUS
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS__SHIFT                                             0x0
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__RESERVED__SHIFT                                                       0x10
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__PF_DOORBELL_STATUS__SHIFT                                             0x1f
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_MASK                                               0x0000FFFFL
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__RESERVED_MASK                                                         0x7FFF0000L
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__PF_DOORBELL_STATUS_MASK                                               0x80000000L
+//RLC_GPU_IOV_VF_DOORBELL_STATUS_SET
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__VF_DOORBELL_STATUS_SET__SHIFT                                     0x0
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__RESERVED__SHIFT                                                   0x10
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__PF_DOORBELL_STATUS_SET__SHIFT                                     0x1f
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__VF_DOORBELL_STATUS_SET_MASK                                       0x0000FFFFL
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__RESERVED_MASK                                                     0x7FFF0000L
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__PF_DOORBELL_STATUS_SET_MASK                                       0x80000000L
+//RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__VF_DOORBELL_STATUS_CLR__SHIFT                                     0x0
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__RESERVED__SHIFT                                                   0x10
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__PF_DOORBELL_STATUS_CLR__SHIFT                                     0x1f
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__VF_DOORBELL_STATUS_CLR_MASK                                       0x0000FFFFL
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__RESERVED_MASK                                                     0x7FFF0000L
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__PF_DOORBELL_STATUS_CLR_MASK                                       0x80000000L
+//RLC_GPU_IOV_VF_MASK
+#define RLC_GPU_IOV_VF_MASK__VF_MASK__SHIFT                                                                   0x0
+#define RLC_GPU_IOV_VF_MASK__RESERVED__SHIFT                                                                  0x10
+#define RLC_GPU_IOV_VF_MASK__VF_MASK_MASK                                                                     0x0000FFFFL
+#define RLC_GPU_IOV_VF_MASK__RESERVED_MASK                                                                    0xFFFF0000L
+//RLC_HYP_SEMAPHORE_0
+#define RLC_HYP_SEMAPHORE_0__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_0__RESERVED__SHIFT                                                                  0x5
+#define RLC_HYP_SEMAPHORE_0__CLIENT_ID_MASK                                                                   0x0000001FL
+#define RLC_HYP_SEMAPHORE_0__RESERVED_MASK                                                                    0xFFFFFFE0L
+//RLC_HYP_SEMAPHORE_1
+#define RLC_HYP_SEMAPHORE_1__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_1__RESERVED__SHIFT                                                                  0x5
+#define RLC_HYP_SEMAPHORE_1__CLIENT_ID_MASK                                                                   0x0000001FL
+#define RLC_HYP_SEMAPHORE_1__RESERVED_MASK                                                                    0xFFFFFFE0L
+//RLC_CLK_CNTL
+#define RLC_CLK_CNTL__RLC_SRM_CLK_CNTL__SHIFT                                                                 0x0
+#define RLC_CLK_CNTL__RLC_SPM_CLK_CNTL__SHIFT                                                                 0x2
+#define RLC_CLK_CNTL__RLC_GPM_CLK_CNTL__SHIFT                                                                 0x4
+#define RLC_CLK_CNTL__RLC_CMN_CLK_CNTL__SHIFT                                                                 0x5
+#define RLC_CLK_CNTL__RLC_TC_CLK_CNTL__SHIFT                                                                  0x6
+#define RLC_CLK_CNTL__RLC_SPP_CLK_CNTL__SHIFT                                                                 0x7
+#define RLC_CLK_CNTL__RLC_SRAM_CLK_GATER_OVERRIDE__SHIFT                                                      0x8
+#define RLC_CLK_CNTL__RLC_EDC_OVERRIDE__SHIFT                                                                 0x9
+#define RLC_CLK_CNTL__RESERVED_11_10__SHIFT                                                                   0xa
+#define RLC_CLK_CNTL__RLC_TC_FGCG_REP_OVERRIDE__SHIFT                                                         0xc
+#define RLC_CLK_CNTL__RLC_DFLL_CLK_CNTL__SHIFT                                                                0xd
+#define RLC_CLK_CNTL__DBGBUS_CLK_ACTIVE_OVERRIDE__SHIFT                                                       0xe
+#define RLC_CLK_CNTL__RLC_CAC2_CLK_CNTL__SHIFT                                                                0xf
+#define RLC_CLK_CNTL__RESERVED_1__SHIFT                                                                       0x11
+#define RLC_CLK_CNTL__RLC_UTCL2_FGCG_OVERRIDE__SHIFT                                                          0x12
+#define RLC_CLK_CNTL__RESERVED__SHIFT                                                                         0x13
+#define RLC_CLK_CNTL__RLC_SRM_CLK_CNTL_MASK                                                                   0x00000003L
+#define RLC_CLK_CNTL__RLC_SPM_CLK_CNTL_MASK                                                                   0x0000000CL
+#define RLC_CLK_CNTL__RLC_GPM_CLK_CNTL_MASK                                                                   0x00000010L
+#define RLC_CLK_CNTL__RLC_CMN_CLK_CNTL_MASK                                                                   0x00000020L
+#define RLC_CLK_CNTL__RLC_TC_CLK_CNTL_MASK                                                                    0x00000040L
+#define RLC_CLK_CNTL__RLC_SPP_CLK_CNTL_MASK                                                                   0x00000080L
+#define RLC_CLK_CNTL__RLC_SRAM_CLK_GATER_OVERRIDE_MASK                                                        0x00000100L
+#define RLC_CLK_CNTL__RLC_EDC_OVERRIDE_MASK                                                                   0x00000200L
+#define RLC_CLK_CNTL__RESERVED_11_10_MASK                                                                     0x00000C00L
+#define RLC_CLK_CNTL__RLC_TC_FGCG_REP_OVERRIDE_MASK                                                           0x00001000L
+#define RLC_CLK_CNTL__RLC_DFLL_CLK_CNTL_MASK                                                                  0x00002000L
+#define RLC_CLK_CNTL__DBGBUS_CLK_ACTIVE_OVERRIDE_MASK                                                         0x00004000L
+#define RLC_CLK_CNTL__RLC_CAC2_CLK_CNTL_MASK                                                                  0x00018000L
+#define RLC_CLK_CNTL__RESERVED_1_MASK                                                                         0x00020000L
+#define RLC_CLK_CNTL__RLC_UTCL2_FGCG_OVERRIDE_MASK                                                            0x00040000L
+#define RLC_CLK_CNTL__RESERVED_MASK                                                                           0xFFF80000L
+//RLC_GPU_IOV_SCH_BLOCK
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_ID__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Ver__SHIFT                                                           0x4
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Size__SHIFT                                                          0x8
+#define RLC_GPU_IOV_SCH_BLOCK__RESERVED__SHIFT                                                                0x10
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_ID_MASK                                                              0x0000000FL
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Ver_MASK                                                             0x000000F0L
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Size_MASK                                                            0x00007F00L
+#define RLC_GPU_IOV_SCH_BLOCK__RESERVED_MASK                                                                  0x7FFF0000L
+//RLC_GPU_IOV_CFG_REG1
+#define RLC_GPU_IOV_CFG_REG1__CMD_TYPE__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE__SHIFT                                                              0x4
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_INTR_EN__SHIFT                                                      0x5
+#define RLC_GPU_IOV_CFG_REG1__RESERVED__SHIFT                                                                 0x6
+#define RLC_GPU_IOV_CFG_REG1__FCN_ID__SHIFT                                                                   0x8
+#define RLC_GPU_IOV_CFG_REG1__NEXT_FCN_ID__SHIFT                                                              0x10
+#define RLC_GPU_IOV_CFG_REG1__RESERVED1__SHIFT                                                                0x18
+#define RLC_GPU_IOV_CFG_REG1__CMD_TYPE_MASK                                                                   0x0000000FL
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_MASK                                                                0x00000010L
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_INTR_EN_MASK                                                        0x00000020L
+#define RLC_GPU_IOV_CFG_REG1__RESERVED_MASK                                                                   0x000000C0L
+#define RLC_GPU_IOV_CFG_REG1__FCN_ID_MASK                                                                     0x0000FF00L
+#define RLC_GPU_IOV_CFG_REG1__NEXT_FCN_ID_MASK                                                                0x00FF0000L
+#define RLC_GPU_IOV_CFG_REG1__RESERVED1_MASK                                                                  0xFF000000L
+//RLC_GPU_IOV_CFG_REG2
+#define RLC_GPU_IOV_CFG_REG2__CMD_STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_CFG_REG2__RESERVED__SHIFT                                                                 0x4
+#define RLC_GPU_IOV_CFG_REG2__CMD_STATUS_MASK                                                                 0x0000000FL
+#define RLC_GPU_IOV_CFG_REG2__RESERVED_MASK                                                                   0xFFFFFFF0L
+//RLC_GPU_IOV_VM_BUSY_STATUS
+#define RLC_GPU_IOV_VM_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                     0x0
+#define RLC_GPU_IOV_VM_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                       0xFFFFFFFFL
+//RLC_GPU_IOV_SCH_0
+#define RLC_GPU_IOV_SCH_0__ACTIVE_FUNCTIONS__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SCH_0__ACTIVE_FUNCTIONS_MASK                                                              0xFFFFFFFFL
+//RLC_GPU_IOV_ACTIVE_FCN_ID
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__VF_ID__SHIFT                                                               0x0
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED__SHIFT                                                            0x4
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__PF_VF__SHIFT                                                               0x1f
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__VF_ID_MASK                                                                 0x0000000FL
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_MASK                                                              0x7FFFFFF0L
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__PF_VF_MASK                                                                 0x80000000L
+//RLC_GPU_IOV_SCH_3
+#define RLC_GPU_IOV_SCH_3__Time_Quanta_Def__SHIFT                                                             0x0
+#define RLC_GPU_IOV_SCH_3__Time_Quanta_Def_MASK                                                               0xFFFFFFFFL
+//RLC_GPU_IOV_SCH_1
+#define RLC_GPU_IOV_SCH_1__DATA__SHIFT                                                                        0x0
+#define RLC_GPU_IOV_SCH_1__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPU_IOV_SCH_2
+#define RLC_GPU_IOV_SCH_2__DATA__SHIFT                                                                        0x0
+#define RLC_GPU_IOV_SCH_2__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPU_IOV_INT_STAT
+#define RLC_GPU_IOV_INT_STAT__STATUS__SHIFT                                                                   0x0
+#define RLC_GPU_IOV_INT_STAT__STATUS_MASK                                                                     0xFFFFFFFFL
+//RLC_RLCV_TIMER_INT_1
+#define RLC_RLCV_TIMER_INT_1__TIMER__SHIFT                                                                    0x0
+#define RLC_RLCV_TIMER_INT_1__TIMER_MASK                                                                      0xFFFFFFFFL
+//RLC_GPU_IOV_UCODE_ADDR
+#define RLC_GPU_IOV_UCODE_ADDR__UCODE_ADDR__SHIFT                                                             0x0
+#define RLC_GPU_IOV_UCODE_ADDR__RESERVED__SHIFT                                                               0xc
+#define RLC_GPU_IOV_UCODE_ADDR__UCODE_ADDR_MASK                                                               0x00000FFFL
+#define RLC_GPU_IOV_UCODE_ADDR__RESERVED_MASK                                                                 0xFFFFF000L
+//RLC_GPU_IOV_UCODE_DATA
+#define RLC_GPU_IOV_UCODE_DATA__UCODE_DATA__SHIFT                                                             0x0
+#define RLC_GPU_IOV_UCODE_DATA__UCODE_DATA_MASK                                                               0xFFFFFFFFL
+//RLC_GPU_IOV_SCRATCH_ADDR
+#define RLC_GPU_IOV_SCRATCH_ADDR__ADDR__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_SCRATCH_ADDR__RESERVED__SHIFT                                                             0x9
+#define RLC_GPU_IOV_SCRATCH_ADDR__ADDR_MASK                                                                   0x000001FFL
+#define RLC_GPU_IOV_SCRATCH_ADDR__RESERVED_MASK                                                               0xFFFFFE00L
+//RLC_GPU_IOV_SCRATCH_DATA
+#define RLC_GPU_IOV_SCRATCH_DATA__DATA__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_SCRATCH_DATA__DATA_MASK                                                                   0xFFFFFFFFL
+//RLC_GPU_IOV_F32_CNTL
+#define RLC_GPU_IOV_F32_CNTL__ENABLE__SHIFT                                                                   0x0
+#define RLC_GPU_IOV_F32_CNTL__RESERVED__SHIFT                                                                 0x1
+#define RLC_GPU_IOV_F32_CNTL__ENABLE_MASK                                                                     0x00000001L
+#define RLC_GPU_IOV_F32_CNTL__RESERVED_MASK                                                                   0xFFFFFFFEL
+//RLC_GPU_IOV_F32_RESET
+#define RLC_GPU_IOV_F32_RESET__RESET__SHIFT                                                                   0x0
+#define RLC_GPU_IOV_F32_RESET__RESERVED__SHIFT                                                                0x1
+#define RLC_GPU_IOV_F32_RESET__RESET_MASK                                                                     0x00000001L
+#define RLC_GPU_IOV_F32_RESET__RESERVED_MASK                                                                  0xFFFFFFFEL
+//RLC_GPU_IOV_SDMA0_STATUS
+#define RLC_GPU_IOV_SDMA0_STATUS__PREEMPTED__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED__SHIFT                                                             0x1
+#define RLC_GPU_IOV_SDMA0_STATUS__SAVED__SHIFT                                                                0x8
+#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED1__SHIFT                                                            0x9
+#define RLC_GPU_IOV_SDMA0_STATUS__RESTORED__SHIFT                                                             0xc
+#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED2__SHIFT                                                            0xd
+#define RLC_GPU_IOV_SDMA0_STATUS__PREEMPTED_MASK                                                              0x00000001L
+#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED_MASK                                                               0x000000FEL
+#define RLC_GPU_IOV_SDMA0_STATUS__SAVED_MASK                                                                  0x00000100L
+#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED1_MASK                                                              0x00000E00L
+#define RLC_GPU_IOV_SDMA0_STATUS__RESTORED_MASK                                                               0x00001000L
+#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED2_MASK                                                              0xFFFFE000L
+//RLC_GPU_IOV_SDMA1_STATUS
+#define RLC_GPU_IOV_SDMA1_STATUS__PREEMPTED__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED__SHIFT                                                             0x1
+#define RLC_GPU_IOV_SDMA1_STATUS__SAVED__SHIFT                                                                0x8
+#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED1__SHIFT                                                            0x9
+#define RLC_GPU_IOV_SDMA1_STATUS__RESTORED__SHIFT                                                             0xc
+#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED2__SHIFT                                                            0xd
+#define RLC_GPU_IOV_SDMA1_STATUS__PREEMPTED_MASK                                                              0x00000001L
+#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED_MASK                                                               0x000000FEL
+#define RLC_GPU_IOV_SDMA1_STATUS__SAVED_MASK                                                                  0x00000100L
+#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED1_MASK                                                              0x00000E00L
+#define RLC_GPU_IOV_SDMA1_STATUS__RESTORED_MASK                                                               0x00001000L
+#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED2_MASK                                                              0xFFFFE000L
+//RLC_GPU_IOV_SMU_RESPONSE
+#define RLC_GPU_IOV_SMU_RESPONSE__RESP__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_SMU_RESPONSE__RESP_MASK                                                                   0xFFFFFFFFL
+//RLC_GPU_IOV_VIRT_RESET_REQ
+#define RLC_GPU_IOV_VIRT_RESET_REQ__VF_FLR__SHIFT                                                             0x0
+#define RLC_GPU_IOV_VIRT_RESET_REQ__RESERVED__SHIFT                                                           0x10
+#define RLC_GPU_IOV_VIRT_RESET_REQ__SOFT_PF_FLR__SHIFT                                                        0x1f
+#define RLC_GPU_IOV_VIRT_RESET_REQ__VF_FLR_MASK                                                               0x0000FFFFL
+#define RLC_GPU_IOV_VIRT_RESET_REQ__RESERVED_MASK                                                             0x7FFF0000L
+#define RLC_GPU_IOV_VIRT_RESET_REQ__SOFT_PF_FLR_MASK                                                          0x80000000L
+//RLC_GPU_IOV_RLC_RESPONSE
+#define RLC_GPU_IOV_RLC_RESPONSE__RESP__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_RLC_RESPONSE__RESP_MASK                                                                   0xFFFFFFFFL
+//RLC_GPU_IOV_INT_DISABLE
+#define RLC_GPU_IOV_INT_DISABLE__DISABLE__SHIFT                                                               0x0
+#define RLC_GPU_IOV_INT_DISABLE__DISABLE_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_INT_FORCE
+#define RLC_GPU_IOV_INT_FORCE__FORCE__SHIFT                                                                   0x0
+#define RLC_GPU_IOV_INT_FORCE__FORCE_MASK                                                                     0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA0_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA0_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA0_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA1_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA1_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA1_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_HYP_SEMAPHORE_2
+#define RLC_HYP_SEMAPHORE_2__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_2__RESERVED__SHIFT                                                                  0x5
+#define RLC_HYP_SEMAPHORE_2__CLIENT_ID_MASK                                                                   0x0000001FL
+#define RLC_HYP_SEMAPHORE_2__RESERVED_MASK                                                                    0xFFFFFFE0L
+//RLC_HYP_SEMAPHORE_3
+#define RLC_HYP_SEMAPHORE_3__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_3__RESERVED__SHIFT                                                                  0x5
+#define RLC_HYP_SEMAPHORE_3__CLIENT_ID_MASK                                                                   0x0000001FL
+#define RLC_HYP_SEMAPHORE_3__RESERVED_MASK                                                                    0xFFFFFFE0L
+//RLC_GPU_IOV_SDMA2_STATUS
+#define RLC_GPU_IOV_SDMA2_STATUS__PREEMPTED__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SDMA2_STATUS__RESERVED__SHIFT                                                             0x1
+#define RLC_GPU_IOV_SDMA2_STATUS__SAVED__SHIFT                                                                0x8
+#define RLC_GPU_IOV_SDMA2_STATUS__RESERVED1__SHIFT                                                            0x9
+#define RLC_GPU_IOV_SDMA2_STATUS__RESTORED__SHIFT                                                             0xc
+#define RLC_GPU_IOV_SDMA2_STATUS__RESERVED2__SHIFT                                                            0xd
+#define RLC_GPU_IOV_SDMA2_STATUS__PREEMPTED_MASK                                                              0x00000001L
+#define RLC_GPU_IOV_SDMA2_STATUS__RESERVED_MASK                                                               0x000000FEL
+#define RLC_GPU_IOV_SDMA2_STATUS__SAVED_MASK                                                                  0x00000100L
+#define RLC_GPU_IOV_SDMA2_STATUS__RESERVED1_MASK                                                              0x00000E00L
+#define RLC_GPU_IOV_SDMA2_STATUS__RESTORED_MASK                                                               0x00001000L
+#define RLC_GPU_IOV_SDMA2_STATUS__RESERVED2_MASK                                                              0xFFFFE000L
+//RLC_GPU_IOV_SDMA3_STATUS
+#define RLC_GPU_IOV_SDMA3_STATUS__PREEMPTED__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SDMA3_STATUS__RESERVED__SHIFT                                                             0x1
+#define RLC_GPU_IOV_SDMA3_STATUS__SAVED__SHIFT                                                                0x8
+#define RLC_GPU_IOV_SDMA3_STATUS__RESERVED1__SHIFT                                                            0x9
+#define RLC_GPU_IOV_SDMA3_STATUS__RESTORED__SHIFT                                                             0xc
+#define RLC_GPU_IOV_SDMA3_STATUS__RESERVED2__SHIFT                                                            0xd
+#define RLC_GPU_IOV_SDMA3_STATUS__PREEMPTED_MASK                                                              0x00000001L
+#define RLC_GPU_IOV_SDMA3_STATUS__RESERVED_MASK                                                               0x000000FEL
+#define RLC_GPU_IOV_SDMA3_STATUS__SAVED_MASK                                                                  0x00000100L
+#define RLC_GPU_IOV_SDMA3_STATUS__RESERVED1_MASK                                                              0x00000E00L
+#define RLC_GPU_IOV_SDMA3_STATUS__RESTORED_MASK                                                               0x00001000L
+#define RLC_GPU_IOV_SDMA3_STATUS__RESERVED2_MASK                                                              0xFFFFE000L
+//RLC_GPU_IOV_SDMA4_STATUS
+#define RLC_GPU_IOV_SDMA4_STATUS__PREEMPTED__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SDMA4_STATUS__RESERVED__SHIFT                                                             0x1
+#define RLC_GPU_IOV_SDMA4_STATUS__SAVED__SHIFT                                                                0x8
+#define RLC_GPU_IOV_SDMA4_STATUS__RESERVED1__SHIFT                                                            0x9
+#define RLC_GPU_IOV_SDMA4_STATUS__RESTORED__SHIFT                                                             0xc
+#define RLC_GPU_IOV_SDMA4_STATUS__RESERVED2__SHIFT                                                            0xd
+#define RLC_GPU_IOV_SDMA4_STATUS__PREEMPTED_MASK                                                              0x00000001L
+#define RLC_GPU_IOV_SDMA4_STATUS__RESERVED_MASK                                                               0x000000FEL
+#define RLC_GPU_IOV_SDMA4_STATUS__SAVED_MASK                                                                  0x00000100L
+#define RLC_GPU_IOV_SDMA4_STATUS__RESERVED1_MASK                                                              0x00000E00L
+#define RLC_GPU_IOV_SDMA4_STATUS__RESTORED_MASK                                                               0x00001000L
+#define RLC_GPU_IOV_SDMA4_STATUS__RESERVED2_MASK                                                              0xFFFFE000L
+//RLC_GPU_IOV_SDMA5_STATUS
+#define RLC_GPU_IOV_SDMA5_STATUS__PREEMPTED__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SDMA5_STATUS__RESERVED__SHIFT                                                             0x1
+#define RLC_GPU_IOV_SDMA5_STATUS__SAVED__SHIFT                                                                0x8
+#define RLC_GPU_IOV_SDMA5_STATUS__RESERVED1__SHIFT                                                            0x9
+#define RLC_GPU_IOV_SDMA5_STATUS__RESTORED__SHIFT                                                             0xc
+#define RLC_GPU_IOV_SDMA5_STATUS__RESERVED2__SHIFT                                                            0xd
+#define RLC_GPU_IOV_SDMA5_STATUS__PREEMPTED_MASK                                                              0x00000001L
+#define RLC_GPU_IOV_SDMA5_STATUS__RESERVED_MASK                                                               0x000000FEL
+#define RLC_GPU_IOV_SDMA5_STATUS__SAVED_MASK                                                                  0x00000100L
+#define RLC_GPU_IOV_SDMA5_STATUS__RESERVED1_MASK                                                              0x00000E00L
+#define RLC_GPU_IOV_SDMA5_STATUS__RESTORED_MASK                                                               0x00001000L
+#define RLC_GPU_IOV_SDMA5_STATUS__RESERVED2_MASK                                                              0xFFFFE000L
+//RLC_GPU_IOV_SDMA6_STATUS
+#define RLC_GPU_IOV_SDMA6_STATUS__PREEMPTED__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SDMA6_STATUS__RESERVED__SHIFT                                                             0x1
+#define RLC_GPU_IOV_SDMA6_STATUS__SAVED__SHIFT                                                                0x8
+#define RLC_GPU_IOV_SDMA6_STATUS__RESERVED1__SHIFT                                                            0x9
+#define RLC_GPU_IOV_SDMA6_STATUS__RESTORED__SHIFT                                                             0xc
+#define RLC_GPU_IOV_SDMA6_STATUS__RESERVED2__SHIFT                                                            0xd
+#define RLC_GPU_IOV_SDMA6_STATUS__PREEMPTED_MASK                                                              0x00000001L
+#define RLC_GPU_IOV_SDMA6_STATUS__RESERVED_MASK                                                               0x000000FEL
+#define RLC_GPU_IOV_SDMA6_STATUS__SAVED_MASK                                                                  0x00000100L
+#define RLC_GPU_IOV_SDMA6_STATUS__RESERVED1_MASK                                                              0x00000E00L
+#define RLC_GPU_IOV_SDMA6_STATUS__RESTORED_MASK                                                               0x00001000L
+#define RLC_GPU_IOV_SDMA6_STATUS__RESERVED2_MASK                                                              0xFFFFE000L
+//RLC_GPU_IOV_SDMA7_STATUS
+#define RLC_GPU_IOV_SDMA7_STATUS__PREEMPTED__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SDMA7_STATUS__RESERVED__SHIFT                                                             0x1
+#define RLC_GPU_IOV_SDMA7_STATUS__SAVED__SHIFT                                                                0x8
+#define RLC_GPU_IOV_SDMA7_STATUS__RESERVED1__SHIFT                                                            0x9
+#define RLC_GPU_IOV_SDMA7_STATUS__RESTORED__SHIFT                                                             0xc
+#define RLC_GPU_IOV_SDMA7_STATUS__RESERVED2__SHIFT                                                            0xd
+#define RLC_GPU_IOV_SDMA7_STATUS__PREEMPTED_MASK                                                              0x00000001L
+#define RLC_GPU_IOV_SDMA7_STATUS__RESERVED_MASK                                                               0x000000FEL
+#define RLC_GPU_IOV_SDMA7_STATUS__SAVED_MASK                                                                  0x00000100L
+#define RLC_GPU_IOV_SDMA7_STATUS__RESERVED1_MASK                                                              0x00000E00L
+#define RLC_GPU_IOV_SDMA7_STATUS__RESTORED_MASK                                                               0x00001000L
+#define RLC_GPU_IOV_SDMA7_STATUS__RESERVED2_MASK                                                              0xFFFFE000L
+//RLC_GPU_IOV_SDMA2_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA2_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA2_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA3_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA3_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA3_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA4_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA4_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA4_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA5_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA5_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA5_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA6_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA6_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA6_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA7_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA7_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA7_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+
+
+// addressBlock: xcd0_gc_utcl2_vmsharedhvdec
+//MC_VM_FB_SIZE_OFFSET_VF0
+#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF1
+#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF2
+#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF3
+#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF4
+#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF5
+#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF6
+#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF7
+#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF8
+#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF9
+#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF10
+#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF11
+#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF12
+#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF13
+#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF14
+#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF15
+#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//VM_IOMMU_MMIO_CNTRL_1
+#define VM_IOMMU_MMIO_CNTRL_1__MARC_EN__SHIFT                                                                 0x8
+#define VM_IOMMU_MMIO_CNTRL_1__MARC_EN_MASK                                                                   0x00000100L
+//MC_VM_MARC_BASE_LO_0
+#define MC_VM_MARC_BASE_LO_0__MARC_BASE_LO_0__SHIFT                                                           0xc
+#define MC_VM_MARC_BASE_LO_0__MARC_BASE_LO_0_MASK                                                             0xFFFFF000L
+//MC_VM_MARC_BASE_LO_1
+#define MC_VM_MARC_BASE_LO_1__MARC_BASE_LO_1__SHIFT                                                           0xc
+#define MC_VM_MARC_BASE_LO_1__MARC_BASE_LO_1_MASK                                                             0xFFFFF000L
+//MC_VM_MARC_BASE_LO_2
+#define MC_VM_MARC_BASE_LO_2__MARC_BASE_LO_2__SHIFT                                                           0xc
+#define MC_VM_MARC_BASE_LO_2__MARC_BASE_LO_2_MASK                                                             0xFFFFF000L
+//MC_VM_MARC_BASE_LO_3
+#define MC_VM_MARC_BASE_LO_3__MARC_BASE_LO_3__SHIFT                                                           0xc
+#define MC_VM_MARC_BASE_LO_3__MARC_BASE_LO_3_MASK                                                             0xFFFFF000L
+//MC_VM_MARC_BASE_HI_0
+#define MC_VM_MARC_BASE_HI_0__MARC_BASE_HI_0__SHIFT                                                           0x0
+#define MC_VM_MARC_BASE_HI_0__MARC_BASE_HI_0_MASK                                                             0x000FFFFFL
+//MC_VM_MARC_BASE_HI_1
+#define MC_VM_MARC_BASE_HI_1__MARC_BASE_HI_1__SHIFT                                                           0x0
+#define MC_VM_MARC_BASE_HI_1__MARC_BASE_HI_1_MASK                                                             0x000FFFFFL
+//MC_VM_MARC_BASE_HI_2
+#define MC_VM_MARC_BASE_HI_2__MARC_BASE_HI_2__SHIFT                                                           0x0
+#define MC_VM_MARC_BASE_HI_2__MARC_BASE_HI_2_MASK                                                             0x000FFFFFL
+//MC_VM_MARC_BASE_HI_3
+#define MC_VM_MARC_BASE_HI_3__MARC_BASE_HI_3__SHIFT                                                           0x0
+#define MC_VM_MARC_BASE_HI_3__MARC_BASE_HI_3_MASK                                                             0x000FFFFFL
+//MC_VM_MARC_RELOC_LO_0
+#define MC_VM_MARC_RELOC_LO_0__MARC_ENABLE_0__SHIFT                                                           0x0
+#define MC_VM_MARC_RELOC_LO_0__MARC_READONLY_0__SHIFT                                                         0x1
+#define MC_VM_MARC_RELOC_LO_0__MARC_RELOC_LO_0__SHIFT                                                         0xc
+#define MC_VM_MARC_RELOC_LO_0__MARC_ENABLE_0_MASK                                                             0x00000001L
+#define MC_VM_MARC_RELOC_LO_0__MARC_READONLY_0_MASK                                                           0x00000002L
+#define MC_VM_MARC_RELOC_LO_0__MARC_RELOC_LO_0_MASK                                                           0xFFFFF000L
+//MC_VM_MARC_RELOC_LO_1
+#define MC_VM_MARC_RELOC_LO_1__MARC_ENABLE_1__SHIFT                                                           0x0
+#define MC_VM_MARC_RELOC_LO_1__MARC_READONLY_1__SHIFT                                                         0x1
+#define MC_VM_MARC_RELOC_LO_1__MARC_RELOC_LO_1__SHIFT                                                         0xc
+#define MC_VM_MARC_RELOC_LO_1__MARC_ENABLE_1_MASK                                                             0x00000001L
+#define MC_VM_MARC_RELOC_LO_1__MARC_READONLY_1_MASK                                                           0x00000002L
+#define MC_VM_MARC_RELOC_LO_1__MARC_RELOC_LO_1_MASK                                                           0xFFFFF000L
+//MC_VM_MARC_RELOC_LO_2
+#define MC_VM_MARC_RELOC_LO_2__MARC_ENABLE_2__SHIFT                                                           0x0
+#define MC_VM_MARC_RELOC_LO_2__MARC_READONLY_2__SHIFT                                                         0x1
+#define MC_VM_MARC_RELOC_LO_2__MARC_RELOC_LO_2__SHIFT                                                         0xc
+#define MC_VM_MARC_RELOC_LO_2__MARC_ENABLE_2_MASK                                                             0x00000001L
+#define MC_VM_MARC_RELOC_LO_2__MARC_READONLY_2_MASK                                                           0x00000002L
+#define MC_VM_MARC_RELOC_LO_2__MARC_RELOC_LO_2_MASK                                                           0xFFFFF000L
+//MC_VM_MARC_RELOC_LO_3
+#define MC_VM_MARC_RELOC_LO_3__MARC_ENABLE_3__SHIFT                                                           0x0
+#define MC_VM_MARC_RELOC_LO_3__MARC_READONLY_3__SHIFT                                                         0x1
+#define MC_VM_MARC_RELOC_LO_3__MARC_RELOC_LO_3__SHIFT                                                         0xc
+#define MC_VM_MARC_RELOC_LO_3__MARC_ENABLE_3_MASK                                                             0x00000001L
+#define MC_VM_MARC_RELOC_LO_3__MARC_READONLY_3_MASK                                                           0x00000002L
+#define MC_VM_MARC_RELOC_LO_3__MARC_RELOC_LO_3_MASK                                                           0xFFFFF000L
+//MC_VM_MARC_RELOC_HI_0
+#define MC_VM_MARC_RELOC_HI_0__MARC_RELOC_HI_0__SHIFT                                                         0x0
+#define MC_VM_MARC_RELOC_HI_0__MARC_RELOC_HI_0_MASK                                                           0x000FFFFFL
+//MC_VM_MARC_RELOC_HI_1
+#define MC_VM_MARC_RELOC_HI_1__MARC_RELOC_HI_1__SHIFT                                                         0x0
+#define MC_VM_MARC_RELOC_HI_1__MARC_RELOC_HI_1_MASK                                                           0x000FFFFFL
+//MC_VM_MARC_RELOC_HI_2
+#define MC_VM_MARC_RELOC_HI_2__MARC_RELOC_HI_2__SHIFT                                                         0x0
+#define MC_VM_MARC_RELOC_HI_2__MARC_RELOC_HI_2_MASK                                                           0x000FFFFFL
+//MC_VM_MARC_RELOC_HI_3
+#define MC_VM_MARC_RELOC_HI_3__MARC_RELOC_HI_3__SHIFT                                                         0x0
+#define MC_VM_MARC_RELOC_HI_3__MARC_RELOC_HI_3_MASK                                                           0x000FFFFFL
+//MC_VM_MARC_LEN_LO_0
+#define MC_VM_MARC_LEN_LO_0__MARC_LEN_LO_0__SHIFT                                                             0xc
+#define MC_VM_MARC_LEN_LO_0__MARC_LEN_LO_0_MASK                                                               0xFFFFF000L
+//MC_VM_MARC_LEN_LO_1
+#define MC_VM_MARC_LEN_LO_1__MARC_LEN_LO_1__SHIFT                                                             0xc
+#define MC_VM_MARC_LEN_LO_1__MARC_LEN_LO_1_MASK                                                               0xFFFFF000L
+//MC_VM_MARC_LEN_LO_2
+#define MC_VM_MARC_LEN_LO_2__MARC_LEN_LO_2__SHIFT                                                             0xc
+#define MC_VM_MARC_LEN_LO_2__MARC_LEN_LO_2_MASK                                                               0xFFFFF000L
+//MC_VM_MARC_LEN_LO_3
+#define MC_VM_MARC_LEN_LO_3__MARC_LEN_LO_3__SHIFT                                                             0xc
+#define MC_VM_MARC_LEN_LO_3__MARC_LEN_LO_3_MASK                                                               0xFFFFF000L
+//MC_VM_MARC_LEN_HI_0
+#define MC_VM_MARC_LEN_HI_0__MARC_LEN_HI_0__SHIFT                                                             0x0
+#define MC_VM_MARC_LEN_HI_0__MARC_LEN_HI_0_MASK                                                               0x000FFFFFL
+//MC_VM_MARC_LEN_HI_1
+#define MC_VM_MARC_LEN_HI_1__MARC_LEN_HI_1__SHIFT                                                             0x0
+#define MC_VM_MARC_LEN_HI_1__MARC_LEN_HI_1_MASK                                                               0x000FFFFFL
+//MC_VM_MARC_LEN_HI_2
+#define MC_VM_MARC_LEN_HI_2__MARC_LEN_HI_2__SHIFT                                                             0x0
+#define MC_VM_MARC_LEN_HI_2__MARC_LEN_HI_2_MASK                                                               0x000FFFFFL
+//MC_VM_MARC_LEN_HI_3
+#define MC_VM_MARC_LEN_HI_3__MARC_LEN_HI_3__SHIFT                                                             0x0
+#define MC_VM_MARC_LEN_HI_3__MARC_LEN_HI_3_MASK                                                               0x000FFFFFL
+//VM_IOMMU_CONTROL_REGISTER
+#define VM_IOMMU_CONTROL_REGISTER__IOMMUEN__SHIFT                                                             0x0
+#define VM_IOMMU_CONTROL_REGISTER__IOMMUEN_MASK                                                               0x00000001L
+//VM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER
+#define VM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN__SHIFT                                  0xd
+#define VM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN_MASK                                    0x00002000L
+//VM_PCIE_ATS_CNTL
+#define VM_PCIE_ATS_CNTL__STU__SHIFT                                                                          0x10
+#define VM_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                                   0x1f
+#define VM_PCIE_ATS_CNTL__STU_MASK                                                                            0x001F0000L
+#define VM_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                                     0x80000000L
+//VM_PCIE_ATS_CNTL_VF_0
+#define VM_PCIE_ATS_CNTL_VF_0__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_0__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_1
+#define VM_PCIE_ATS_CNTL_VF_1__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_1__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_2
+#define VM_PCIE_ATS_CNTL_VF_2__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_2__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_3
+#define VM_PCIE_ATS_CNTL_VF_3__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_3__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_4
+#define VM_PCIE_ATS_CNTL_VF_4__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_4__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_5
+#define VM_PCIE_ATS_CNTL_VF_5__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_5__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_6
+#define VM_PCIE_ATS_CNTL_VF_6__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_6__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_7
+#define VM_PCIE_ATS_CNTL_VF_7__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_7__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_8
+#define VM_PCIE_ATS_CNTL_VF_8__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_8__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_9
+#define VM_PCIE_ATS_CNTL_VF_9__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_9__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_10
+#define VM_PCIE_ATS_CNTL_VF_10__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_10__ATC_ENABLE_MASK                                                               0x80000000L
+//VM_PCIE_ATS_CNTL_VF_11
+#define VM_PCIE_ATS_CNTL_VF_11__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_11__ATC_ENABLE_MASK                                                               0x80000000L
+//VM_PCIE_ATS_CNTL_VF_12
+#define VM_PCIE_ATS_CNTL_VF_12__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_12__ATC_ENABLE_MASK                                                               0x80000000L
+//VM_PCIE_ATS_CNTL_VF_13
+#define VM_PCIE_ATS_CNTL_VF_13__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_13__ATC_ENABLE_MASK                                                               0x80000000L
+//VM_PCIE_ATS_CNTL_VF_14
+#define VM_PCIE_ATS_CNTL_VF_14__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_14__ATC_ENABLE_MASK                                                               0x80000000L
+//VM_PCIE_ATS_CNTL_VF_15
+#define VM_PCIE_ATS_CNTL_VF_15__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_15__ATC_ENABLE_MASK                                                               0x80000000L
+//MC_SHARED_ACTIVE_FCN_ID
+#define MC_SHARED_ACTIVE_FCN_ID__VFID__SHIFT                                                                  0x0
+#define MC_SHARED_ACTIVE_FCN_ID__VF__SHIFT                                                                    0x1f
+#define MC_SHARED_ACTIVE_FCN_ID__VFID_MASK                                                                    0x0000000FL
+#define MC_SHARED_ACTIVE_FCN_ID__VF_MASK                                                                      0x80000000L
+//MC_VM_XGMI_GPUIOV_ENABLE
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF0__SHIFT                                                           0x0
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF1__SHIFT                                                           0x1
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF2__SHIFT                                                           0x2
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF3__SHIFT                                                           0x3
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF4__SHIFT                                                           0x4
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF5__SHIFT                                                           0x5
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF6__SHIFT                                                           0x6
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF7__SHIFT                                                           0x7
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF8__SHIFT                                                           0x8
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF9__SHIFT                                                           0x9
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF10__SHIFT                                                          0xa
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF11__SHIFT                                                          0xb
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF12__SHIFT                                                          0xc
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF13__SHIFT                                                          0xd
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF14__SHIFT                                                          0xe
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF15__SHIFT                                                          0xf
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_PF__SHIFT                                                            0x1f
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF0_MASK                                                             0x00000001L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF1_MASK                                                             0x00000002L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF2_MASK                                                             0x00000004L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF3_MASK                                                             0x00000008L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF4_MASK                                                             0x00000010L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF5_MASK                                                             0x00000020L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF6_MASK                                                             0x00000040L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF7_MASK                                                             0x00000080L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF8_MASK                                                             0x00000100L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF9_MASK                                                             0x00000200L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF10_MASK                                                            0x00000400L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF11_MASK                                                            0x00000800L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF12_MASK                                                            0x00001000L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF13_MASK                                                            0x00002000L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF14_MASK                                                            0x00004000L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF15_MASK                                                            0x00008000L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_PF_MASK                                                              0x80000000L
+
+
+// addressBlock: xcd0_gc_pspdec
+//CPG_PSP_DEBUG
+#define CPG_PSP_DEBUG__PRIV_VIOLATION_CNTL__SHIFT                                                             0x0
+#define CPG_PSP_DEBUG__VMID_VIOLATION_CNTL__SHIFT                                                             0x2
+#define CPG_PSP_DEBUG__PRIV_VIOLATION_CNTL_MASK                                                               0x00000003L
+#define CPG_PSP_DEBUG__VMID_VIOLATION_CNTL_MASK                                                               0x00000004L
+//CPC_PSP_DEBUG
+#define CPC_PSP_DEBUG__PRIV_VIOLATION_CNTL__SHIFT                                                             0x0
+#define CPC_PSP_DEBUG__VMID_VIOLATION_CNTL__SHIFT                                                             0x2
+#define CPC_PSP_DEBUG__UTCL2IUGPAOVERRIDE__SHIFT                                                              0x3
+#define CPC_PSP_DEBUG__CPC_DC_FIX_DISABLE__SHIFT                                                              0x4
+#define CPC_PSP_DEBUG__PRIV_VIOLATION_CNTL_MASK                                                               0x00000003L
+#define CPC_PSP_DEBUG__VMID_VIOLATION_CNTL_MASK                                                               0x00000004L
+#define CPC_PSP_DEBUG__UTCL2IUGPAOVERRIDE_MASK                                                                0x00000008L
+#define CPC_PSP_DEBUG__CPC_DC_FIX_DISABLE_MASK                                                                0x00000010L
+//CP_PSP_XCP_CTL
+#define CP_PSP_XCP_CTL__PHYSICAL_XCC_ID__SHIFT                                                                0x0
+#define CP_PSP_XCP_CTL__XCC_DIE_ID__SHIFT                                                                     0x3
+#define CP_PSP_XCP_CTL__PHYSICAL_XCC_ID_MASK                                                                  0x00000007L
+#define CP_PSP_XCP_CTL__XCC_DIE_ID_MASK                                                                       0x00000038L
+//GRBM_SEC_CNTL
+#define GRBM_SEC_CNTL__DEBUG_ENABLE__SHIFT                                                                    0x0
+#define GRBM_SEC_CNTL__DEBUG_ENABLE_MASK                                                                      0x00000001L
+//GRBM_IOV_ERROR_FIFO_DATA
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_ADDR__SHIFT                                                             0x0
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_VFID__SHIFT                                                             0x12
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_SSRCID__SHIFT                                                           0x18
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_OP__SHIFT                                                               0x1c
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_VF__SHIFT                                                               0x1d
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_OVERFLOW__SHIFT                                                         0x1e
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_READ_VALID__SHIFT                                                       0x1f
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_ADDR_MASK                                                               0x0003FFFFL
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_VFID_MASK                                                               0x00FC0000L
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_SSRCID_MASK                                                             0x0F000000L
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_OP_MASK                                                                 0x10000000L
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_VF_MASK                                                                 0x20000000L
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_OVERFLOW_MASK                                                           0x40000000L
+#define GRBM_IOV_ERROR_FIFO_DATA__IOV_READ_VALID_MASK                                                         0x80000000L
+//GRBM_DSM_BYPASS
+#define GRBM_DSM_BYPASS__BYPASS_BITS__SHIFT                                                                   0x0
+#define GRBM_DSM_BYPASS__BYPASS_EN__SHIFT                                                                     0x2
+#define GRBM_DSM_BYPASS__BYPASS_BITS_MASK                                                                     0x00000003L
+#define GRBM_DSM_BYPASS__BYPASS_EN_MASK                                                                       0x00000004L
+//GRBM_CAM_INDEX
+#define GRBM_CAM_INDEX__CAM_INDEX__SHIFT                                                                      0x0
+#define GRBM_CAM_INDEX__CAM_INDEX_MASK                                                                        0x00000007L
+//GRBM_HYP_CAM_INDEX
+#define GRBM_HYP_CAM_INDEX__CAM_INDEX__SHIFT                                                                  0x0
+#define GRBM_HYP_CAM_INDEX__CAM_INDEX_MASK                                                                    0x00000007L
+//GRBM_CAM_DATA
+#define GRBM_CAM_DATA__CAM_ADDR__SHIFT                                                                        0x0
+#define GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT                                                                   0x10
+#define GRBM_CAM_DATA__CAM_ADDR_MASK                                                                          0x0000FFFFL
+#define GRBM_CAM_DATA__CAM_REMAPADDR_MASK                                                                     0xFFFF0000L
+//GRBM_HYP_CAM_DATA
+#define GRBM_HYP_CAM_DATA__CAM_ADDR__SHIFT                                                                    0x0
+#define GRBM_HYP_CAM_DATA__CAM_REMAPADDR__SHIFT                                                               0x10
+#define GRBM_HYP_CAM_DATA__CAM_ADDR_MASK                                                                      0x0000FFFFL
+#define GRBM_HYP_CAM_DATA__CAM_REMAPADDR_MASK                                                                 0xFFFF0000L
+//RLC_FWL_FIRST_VIOL_ADDR
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_STATUS__SHIFT                                                           0x0
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_OP__SHIFT                                                               0x1
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_ADDR__SHIFT                                                             0x2
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_APERTURE_ID__SHIFT                                                      0x14
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_STATUS_MASK                                                             0x00000001L
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_OP_MASK                                                                 0x00000002L
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_ADDR_MASK                                                               0x000FFFFCL
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_APERTURE_ID_MASK                                                        0xFFF00000L
+
+
+// addressBlock: sqind
+//SQ_DEBUG_STS_LOCAL
+#define SQ_DEBUG_STS_LOCAL__BUSY__SHIFT                                                                       0x0
+#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL__SHIFT                                                                 0x4
+#define SQ_DEBUG_STS_LOCAL__BUSY_MASK                                                                         0x00000001L
+#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL_MASK                                                                   0x000003F0L
+//SQ_DEBUG_CTRL_LOCAL
+#define SQ_DEBUG_CTRL_LOCAL__UNUSED__SHIFT                                                                    0x0
+#define SQ_DEBUG_CTRL_LOCAL__PERF_SEL_INSTS_VALU_MFMA_NON_WAVE__SHIFT                                         0x8
+#define SQ_DEBUG_CTRL_LOCAL__PERF_SEL_INSTS_VALU_MFMA_MOPS_NON_WAVE__SHIFT                                    0x9
+#define SQ_DEBUG_CTRL_LOCAL__UNUSED_MASK                                                                      0x000000FFL
+#define SQ_DEBUG_CTRL_LOCAL__PERF_SEL_INSTS_VALU_MFMA_NON_WAVE_MASK                                           0x00000100L
+#define SQ_DEBUG_CTRL_LOCAL__PERF_SEL_INSTS_VALU_MFMA_MOPS_NON_WAVE_MASK                                      0x00000200L
+//SQ_WAVE_VALID_AND_IDLE
+#define SQ_WAVE_VALID_AND_IDLE__WAVE_SLOT__SHIFT                                                              0x0
+#define SQ_WAVE_VALID_AND_IDLE__WAVE_SLOT_MASK                                                                0xFFFFFFFFL
+//SQ_PERF_SNAPSHOT_DATA
+//SQ_PERF_SNAPSHOT_DATA1
+//SQ_PERF_SNAPSHOT_PC_LO
+//SQ_PERF_SNAPSHOT_PC_HI
+//SQ_WAVE_MODE
+#define SQ_WAVE_MODE__FP_ROUND__SHIFT                                                                         0x0
+#define SQ_WAVE_MODE__FP_DENORM__SHIFT                                                                        0x4
+#define SQ_WAVE_MODE__DX10_CLAMP__SHIFT                                                                       0x8
+#define SQ_WAVE_MODE__IEEE__SHIFT                                                                             0x9
+#define SQ_WAVE_MODE__LOD_CLAMPED__SHIFT                                                                      0xa
+#define SQ_WAVE_MODE__DEBUG_EN__SHIFT                                                                         0xb
+#define SQ_WAVE_MODE__EXCP_EN__SHIFT                                                                          0xc
+#define SQ_WAVE_MODE__FP16_OVFL__SHIFT                                                                        0x17
+#define SQ_WAVE_MODE__POPS_PACKER0__SHIFT                                                                     0x18
+#define SQ_WAVE_MODE__POPS_PACKER1__SHIFT                                                                     0x19
+#define SQ_WAVE_MODE__DISABLE_PERF__SHIFT                                                                     0x1a
+#define SQ_WAVE_MODE__GPR_IDX_EN__SHIFT                                                                       0x1b
+#define SQ_WAVE_MODE__VSKIP__SHIFT                                                                            0x1c
+#define SQ_WAVE_MODE__CSP__SHIFT                                                                              0x1d
+#define SQ_WAVE_MODE__FP_ROUND_MASK                                                                           0x0000000FL
+#define SQ_WAVE_MODE__FP_DENORM_MASK                                                                          0x000000F0L
+#define SQ_WAVE_MODE__DX10_CLAMP_MASK                                                                         0x00000100L
+#define SQ_WAVE_MODE__IEEE_MASK                                                                               0x00000200L
+#define SQ_WAVE_MODE__LOD_CLAMPED_MASK                                                                        0x00000400L
+#define SQ_WAVE_MODE__DEBUG_EN_MASK                                                                           0x00000800L
+#define SQ_WAVE_MODE__EXCP_EN_MASK                                                                            0x001FF000L
+#define SQ_WAVE_MODE__FP16_OVFL_MASK                                                                          0x00800000L
+#define SQ_WAVE_MODE__POPS_PACKER0_MASK                                                                       0x01000000L
+#define SQ_WAVE_MODE__POPS_PACKER1_MASK                                                                       0x02000000L
+#define SQ_WAVE_MODE__DISABLE_PERF_MASK                                                                       0x04000000L
+#define SQ_WAVE_MODE__GPR_IDX_EN_MASK                                                                         0x08000000L
+#define SQ_WAVE_MODE__VSKIP_MASK                                                                              0x10000000L
+#define SQ_WAVE_MODE__CSP_MASK                                                                                0xE0000000L
+//SQ_WAVE_STATUS
+#define SQ_WAVE_STATUS__SCC__SHIFT                                                                            0x0
+#define SQ_WAVE_STATUS__SPI_PRIO__SHIFT                                                                       0x1
+#define SQ_WAVE_STATUS__USER_PRIO__SHIFT                                                                      0x3
+#define SQ_WAVE_STATUS__PRIV__SHIFT                                                                           0x5
+#define SQ_WAVE_STATUS__TRAP_EN__SHIFT                                                                        0x6
+#define SQ_WAVE_STATUS__TTRACE_EN__SHIFT                                                                      0x7
+#define SQ_WAVE_STATUS__EXPORT_RDY__SHIFT                                                                     0x8
+#define SQ_WAVE_STATUS__EXECZ__SHIFT                                                                          0x9
+#define SQ_WAVE_STATUS__VCCZ__SHIFT                                                                           0xa
+#define SQ_WAVE_STATUS__IN_TG__SHIFT                                                                          0xb
+#define SQ_WAVE_STATUS__IN_BARRIER__SHIFT                                                                     0xc
+#define SQ_WAVE_STATUS__HALT__SHIFT                                                                           0xd
+#define SQ_WAVE_STATUS__TRAP__SHIFT                                                                           0xe
+#define SQ_WAVE_STATUS__TTRACE_CU_EN__SHIFT                                                                   0xf
+#define SQ_WAVE_STATUS__VALID__SHIFT                                                                          0x10
+#define SQ_WAVE_STATUS__ECC_ERR__SHIFT                                                                        0x11
+#define SQ_WAVE_STATUS__SKIP_EXPORT__SHIFT                                                                    0x12
+#define SQ_WAVE_STATUS__PERF_EN__SHIFT                                                                        0x13
+#define SQ_WAVE_STATUS__COND_DBG_USER__SHIFT                                                                  0x14
+#define SQ_WAVE_STATUS__COND_DBG_SYS__SHIFT                                                                   0x15
+#define SQ_WAVE_STATUS__ALLOW_REPLAY__SHIFT                                                                   0x16
+#define SQ_WAVE_STATUS__FATAL_HALT__SHIFT                                                                     0x17
+#define SQ_WAVE_STATUS__MUST_EXPORT__SHIFT                                                                    0x1b
+#define SQ_WAVE_STATUS__SCRATCH_EN__SHIFT                                                                     0x1c
+#define SQ_WAVE_STATUS__IDLE__SHIFT                                                                           0x1f
+#define SQ_WAVE_STATUS__SCC_MASK                                                                              0x00000001L
+#define SQ_WAVE_STATUS__SPI_PRIO_MASK                                                                         0x00000006L
+#define SQ_WAVE_STATUS__USER_PRIO_MASK                                                                        0x00000018L
+#define SQ_WAVE_STATUS__PRIV_MASK                                                                             0x00000020L
+#define SQ_WAVE_STATUS__TRAP_EN_MASK                                                                          0x00000040L
+#define SQ_WAVE_STATUS__TTRACE_EN_MASK                                                                        0x00000080L
+#define SQ_WAVE_STATUS__EXPORT_RDY_MASK                                                                       0x00000100L
+#define SQ_WAVE_STATUS__EXECZ_MASK                                                                            0x00000200L
+#define SQ_WAVE_STATUS__VCCZ_MASK                                                                             0x00000400L
+#define SQ_WAVE_STATUS__IN_TG_MASK                                                                            0x00000800L
+#define SQ_WAVE_STATUS__IN_BARRIER_MASK                                                                       0x00001000L
+#define SQ_WAVE_STATUS__HALT_MASK                                                                             0x00002000L
+#define SQ_WAVE_STATUS__TRAP_MASK                                                                             0x00004000L
+#define SQ_WAVE_STATUS__TTRACE_CU_EN_MASK                                                                     0x00008000L
+#define SQ_WAVE_STATUS__VALID_MASK                                                                            0x00010000L
+#define SQ_WAVE_STATUS__ECC_ERR_MASK                                                                          0x00020000L
+#define SQ_WAVE_STATUS__SKIP_EXPORT_MASK                                                                      0x00040000L
+#define SQ_WAVE_STATUS__PERF_EN_MASK                                                                          0x00080000L
+#define SQ_WAVE_STATUS__COND_DBG_USER_MASK                                                                    0x00100000L
+#define SQ_WAVE_STATUS__COND_DBG_SYS_MASK                                                                     0x00200000L
+#define SQ_WAVE_STATUS__ALLOW_REPLAY_MASK                                                                     0x00400000L
+#define SQ_WAVE_STATUS__FATAL_HALT_MASK                                                                       0x00800000L
+#define SQ_WAVE_STATUS__MUST_EXPORT_MASK                                                                      0x08000000L
+#define SQ_WAVE_STATUS__SCRATCH_EN_MASK                                                                       0x10000000L
+#define SQ_WAVE_STATUS__IDLE_MASK                                                                             0x80000000L
+//SQ_WAVE_TRAPSTS
+#define SQ_WAVE_TRAPSTS__EXCP__SHIFT                                                                          0x0
+#define SQ_WAVE_TRAPSTS__SAVECTX__SHIFT                                                                       0xa
+#define SQ_WAVE_TRAPSTS__ILLEGAL_INST__SHIFT                                                                  0xb
+#define SQ_WAVE_TRAPSTS__EXCP_HI__SHIFT                                                                       0xc
+#define SQ_WAVE_TRAPSTS__EXCP_CYCLE__SHIFT                                                                    0x10
+#define SQ_WAVE_TRAPSTS__HOST_TRAP__SHIFT                                                                     0x16
+#define SQ_WAVE_TRAPSTS__WAVE_END__SHIFT                                                                      0x18
+#define SQ_WAVE_TRAPSTS__TRAP_AFTER_INST__SHIFT                                                               0x19
+#define SQ_WAVE_TRAPSTS__PERF_SNAPSHOT__SHIFT                                                                 0x1a
+#define SQ_WAVE_TRAPSTS__XNACK_ERROR__SHIFT                                                                   0x1c
+#define SQ_WAVE_TRAPSTS__DP_RATE__SHIFT                                                                       0x1d
+#define SQ_WAVE_TRAPSTS__EXCP_MASK                                                                            0x000001FFL
+#define SQ_WAVE_TRAPSTS__SAVECTX_MASK                                                                         0x00000400L
+#define SQ_WAVE_TRAPSTS__ILLEGAL_INST_MASK                                                                    0x00000800L
+#define SQ_WAVE_TRAPSTS__EXCP_HI_MASK                                                                         0x00007000L
+#define SQ_WAVE_TRAPSTS__EXCP_CYCLE_MASK                                                                      0x003F0000L
+#define SQ_WAVE_TRAPSTS__HOST_TRAP_MASK                                                                       0x00400000L
+#define SQ_WAVE_TRAPSTS__WAVE_END_MASK                                                                        0x01000000L
+#define SQ_WAVE_TRAPSTS__TRAP_AFTER_INST_MASK                                                                 0x02000000L
+#define SQ_WAVE_TRAPSTS__PERF_SNAPSHOT_MASK                                                                   0x04000000L
+#define SQ_WAVE_TRAPSTS__XNACK_ERROR_MASK                                                                     0x10000000L
+#define SQ_WAVE_TRAPSTS__DP_RATE_MASK                                                                         0xE0000000L
+//SQ_WAVE_HW_ID
+#define SQ_WAVE_HW_ID__WAVE_ID__SHIFT                                                                         0x0
+#define SQ_WAVE_HW_ID__SIMD_ID__SHIFT                                                                         0x4
+#define SQ_WAVE_HW_ID__PIPE_ID__SHIFT                                                                         0x6
+#define SQ_WAVE_HW_ID__CU_ID__SHIFT                                                                           0x8
+#define SQ_WAVE_HW_ID__SH_ID__SHIFT                                                                           0xc
+#define SQ_WAVE_HW_ID__SE_ID__SHIFT                                                                           0xd
+#define SQ_WAVE_HW_ID__TG_ID__SHIFT                                                                           0x10
+#define SQ_WAVE_HW_ID__VM_ID__SHIFT                                                                           0x14
+#define SQ_WAVE_HW_ID__QUEUE_ID__SHIFT                                                                        0x18
+#define SQ_WAVE_HW_ID__STATE_ID__SHIFT                                                                        0x1b
+#define SQ_WAVE_HW_ID__ME_ID__SHIFT                                                                           0x1e
+#define SQ_WAVE_HW_ID__WAVE_ID_MASK                                                                           0x0000000FL
+#define SQ_WAVE_HW_ID__SIMD_ID_MASK                                                                           0x00000030L
+#define SQ_WAVE_HW_ID__PIPE_ID_MASK                                                                           0x000000C0L
+#define SQ_WAVE_HW_ID__CU_ID_MASK                                                                             0x00000F00L
+#define SQ_WAVE_HW_ID__SH_ID_MASK                                                                             0x00001000L
+#define SQ_WAVE_HW_ID__SE_ID_MASK                                                                             0x0000E000L
+#define SQ_WAVE_HW_ID__TG_ID_MASK                                                                             0x000F0000L
+#define SQ_WAVE_HW_ID__VM_ID_MASK                                                                             0x00F00000L
+#define SQ_WAVE_HW_ID__QUEUE_ID_MASK                                                                          0x07000000L
+#define SQ_WAVE_HW_ID__STATE_ID_MASK                                                                          0x38000000L
+#define SQ_WAVE_HW_ID__ME_ID_MASK                                                                             0xC0000000L
+//SQ_WAVE_GPR_ALLOC
+#define SQ_WAVE_GPR_ALLOC__VGPR_BASE__SHIFT                                                                   0x0
+#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE__SHIFT                                                                   0x6
+#define SQ_WAVE_GPR_ALLOC__ACCV_OFFSET__SHIFT                                                                 0xc
+#define SQ_WAVE_GPR_ALLOC__SGPR_BASE__SHIFT                                                                   0x12
+#define SQ_WAVE_GPR_ALLOC__SGPR_SIZE__SHIFT                                                                   0x18
+#define SQ_WAVE_GPR_ALLOC__VGPR_BASE_MASK                                                                     0x0000003FL
+#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE_MASK                                                                     0x00000FC0L
+#define SQ_WAVE_GPR_ALLOC__ACCV_OFFSET_MASK                                                                   0x0003F000L
+#define SQ_WAVE_GPR_ALLOC__SGPR_BASE_MASK                                                                     0x00FC0000L
+#define SQ_WAVE_GPR_ALLOC__SGPR_SIZE_MASK                                                                     0x0F000000L
+//SQ_WAVE_LDS_ALLOC
+#define SQ_WAVE_LDS_ALLOC__LDS_BASE__SHIFT                                                                    0x0
+#define SQ_WAVE_LDS_ALLOC__LDS_SIZE__SHIFT                                                                    0xc
+#define SQ_WAVE_LDS_ALLOC__LDS_BASE_MASK                                                                      0x000000FFL
+#define SQ_WAVE_LDS_ALLOC__LDS_SIZE_MASK                                                                      0x001FF000L
+//SQ_WAVE_IB_STS
+#define SQ_WAVE_IB_STS__VM_CNT__SHIFT                                                                         0x0
+#define SQ_WAVE_IB_STS__EXP_CNT__SHIFT                                                                        0x4
+#define SQ_WAVE_IB_STS__LGKM_CNT__SHIFT                                                                       0x8
+#define SQ_WAVE_IB_STS__VALU_CNT__SHIFT                                                                       0xc
+#define SQ_WAVE_IB_STS__FIRST_REPLAY__SHIFT                                                                   0xf
+#define SQ_WAVE_IB_STS__RCNT__SHIFT                                                                           0x10
+#define SQ_WAVE_IB_STS__VM_CNT_HI__SHIFT                                                                      0x16
+#define SQ_WAVE_IB_STS__VM_CNT_MASK                                                                           0x0000000FL
+#define SQ_WAVE_IB_STS__EXP_CNT_MASK                                                                          0x00000070L
+#define SQ_WAVE_IB_STS__LGKM_CNT_MASK                                                                         0x00000F00L
+#define SQ_WAVE_IB_STS__VALU_CNT_MASK                                                                         0x00007000L
+#define SQ_WAVE_IB_STS__FIRST_REPLAY_MASK                                                                     0x00008000L
+#define SQ_WAVE_IB_STS__RCNT_MASK                                                                             0x001F0000L
+#define SQ_WAVE_IB_STS__VM_CNT_HI_MASK                                                                        0x00C00000L
+//SQ_WAVE_PC_LO
+#define SQ_WAVE_PC_LO__PC_LO__SHIFT                                                                           0x0
+#define SQ_WAVE_PC_LO__PC_LO_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_PC_HI
+#define SQ_WAVE_PC_HI__PC_HI__SHIFT                                                                           0x0
+#define SQ_WAVE_PC_HI__PC_HI_MASK                                                                             0x0000FFFFL
+//SQ_WAVE_INST_DW0
+#define SQ_WAVE_INST_DW0__INST_DW0__SHIFT                                                                     0x0
+#define SQ_WAVE_INST_DW0__INST_DW0_MASK                                                                       0xFFFFFFFFL
+//SQ_WAVE_INST_DW1
+#define SQ_WAVE_INST_DW1__INST_DW1__SHIFT                                                                     0x0
+#define SQ_WAVE_INST_DW1__INST_DW1_MASK                                                                       0xFFFFFFFFL
+//SQ_WAVE_IB_DBG0
+#define SQ_WAVE_IB_DBG0__IBUF_ST__SHIFT                                                                       0x0
+#define SQ_WAVE_IB_DBG0__PC_INVALID__SHIFT                                                                    0x3
+#define SQ_WAVE_IB_DBG0__NEED_NEXT_DW__SHIFT                                                                  0x4
+#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT__SHIFT                                                               0x5
+#define SQ_WAVE_IB_DBG0__IBUF_RPTR__SHIFT                                                                     0x8
+#define SQ_WAVE_IB_DBG0__IBUF_WPTR__SHIFT                                                                     0xa
+#define SQ_WAVE_IB_DBG0__INST_STR_ST__SHIFT                                                                   0x10
+#define SQ_WAVE_IB_DBG0__ECC_ST__SHIFT                                                                        0x18
+#define SQ_WAVE_IB_DBG0__IS_HYB__SHIFT                                                                        0x1a
+#define SQ_WAVE_IB_DBG0__HYB_CNT__SHIFT                                                                       0x1b
+#define SQ_WAVE_IB_DBG0__KILL__SHIFT                                                                          0x1d
+#define SQ_WAVE_IB_DBG0__NEED_KILL_IFETCH__SHIFT                                                              0x1e
+#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT_HI__SHIFT                                                            0x1f
+#define SQ_WAVE_IB_DBG0__IBUF_ST_MASK                                                                         0x00000007L
+#define SQ_WAVE_IB_DBG0__PC_INVALID_MASK                                                                      0x00000008L
+#define SQ_WAVE_IB_DBG0__NEED_NEXT_DW_MASK                                                                    0x00000010L
+#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT_MASK                                                                 0x000000E0L
+#define SQ_WAVE_IB_DBG0__IBUF_RPTR_MASK                                                                       0x00000300L
+#define SQ_WAVE_IB_DBG0__IBUF_WPTR_MASK                                                                       0x00000C00L
+#define SQ_WAVE_IB_DBG0__INST_STR_ST_MASK                                                                     0x000F0000L
+#define SQ_WAVE_IB_DBG0__ECC_ST_MASK                                                                          0x03000000L
+#define SQ_WAVE_IB_DBG0__IS_HYB_MASK                                                                          0x04000000L
+#define SQ_WAVE_IB_DBG0__HYB_CNT_MASK                                                                         0x18000000L
+#define SQ_WAVE_IB_DBG0__KILL_MASK                                                                            0x20000000L
+#define SQ_WAVE_IB_DBG0__NEED_KILL_IFETCH_MASK                                                                0x40000000L
+#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT_HI_MASK                                                              0x80000000L
+//SQ_WAVE_IB_DBG1
+#define SQ_WAVE_IB_DBG1__IXNACK__SHIFT                                                                        0x0
+#define SQ_WAVE_IB_DBG1__XNACK__SHIFT                                                                         0x1
+#define SQ_WAVE_IB_DBG1__TA_NEED_RESET__SHIFT                                                                 0x2
+#define SQ_WAVE_IB_DBG1__XCNT__SHIFT                                                                          0x4
+#define SQ_WAVE_IB_DBG1__QCNT__SHIFT                                                                          0xb
+#define SQ_WAVE_IB_DBG1__RCNT__SHIFT                                                                          0x12
+#define SQ_WAVE_IB_DBG1__MISC_CNT__SHIFT                                                                      0x19
+#define SQ_WAVE_IB_DBG1__IXNACK_MASK                                                                          0x00000001L
+#define SQ_WAVE_IB_DBG1__XNACK_MASK                                                                           0x00000002L
+#define SQ_WAVE_IB_DBG1__TA_NEED_RESET_MASK                                                                   0x00000004L
+#define SQ_WAVE_IB_DBG1__XCNT_MASK                                                                            0x000001F0L
+#define SQ_WAVE_IB_DBG1__QCNT_MASK                                                                            0x0000F800L
+#define SQ_WAVE_IB_DBG1__RCNT_MASK                                                                            0x007C0000L
+#define SQ_WAVE_IB_DBG1__MISC_CNT_MASK                                                                        0xFE000000L
+//SQ_WAVE_FLUSH_IB
+#define SQ_WAVE_FLUSH_IB__UNUSED__SHIFT                                                                       0x0
+#define SQ_WAVE_FLUSH_IB__UNUSED_MASK                                                                         0xFFFFFFFFL
+//SQ_WAVE_TTMP0
+#define SQ_WAVE_TTMP0__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP0__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP1
+#define SQ_WAVE_TTMP1__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP1__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP2
+#define SQ_WAVE_TTMP2__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP2__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP3
+#define SQ_WAVE_TTMP3__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP3__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP4
+#define SQ_WAVE_TTMP4__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP4__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP5
+#define SQ_WAVE_TTMP5__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP5__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP6
+#define SQ_WAVE_TTMP6__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP6__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP7
+#define SQ_WAVE_TTMP7__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP7__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP8
+#define SQ_WAVE_TTMP8__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP8__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP9
+#define SQ_WAVE_TTMP9__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP9__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP10
+#define SQ_WAVE_TTMP10__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP10__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP11
+#define SQ_WAVE_TTMP11__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP11__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP12
+#define SQ_WAVE_TTMP12__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP12__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP13
+#define SQ_WAVE_TTMP13__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP13__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP14
+#define SQ_WAVE_TTMP14__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP14__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP15
+#define SQ_WAVE_TTMP15__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP15__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_M0
+#define SQ_WAVE_M0__M0__SHIFT                                                                                 0x0
+#define SQ_WAVE_M0__M0_MASK                                                                                   0xFFFFFFFFL
+//SQ_WAVE_EXEC_LO
+#define SQ_WAVE_EXEC_LO__EXEC_LO__SHIFT                                                                       0x0
+#define SQ_WAVE_EXEC_LO__EXEC_LO_MASK                                                                         0xFFFFFFFFL
+//SQ_WAVE_EXEC_HI
+#define SQ_WAVE_EXEC_HI__EXEC_HI__SHIFT                                                                       0x0
+#define SQ_WAVE_EXEC_HI__EXEC_HI_MASK                                                                         0xFFFFFFFFL
+//SQ_INTERRUPT_WORD_AUTO_CTXID
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE__SHIFT                                                     0x0
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT__SHIFT                                                              0x1
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL__SHIFT                                            0x2
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP__SHIFT                                                    0x3
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP__SHIFT                                                    0x4
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW__SHIFT                                                0x5
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW__SHIFT                                                0x6
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW__SHIFT                                                   0x7
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR__SHIFT                                           0x8
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID__SHIFT                                                            0x18
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING__SHIFT                                                         0x1a
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_MASK                                                       0x0000001L
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT_MASK                                                                0x0000002L
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL_MASK                                              0x0000004L
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP_MASK                                                      0x0000008L
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP_MASK                                                      0x0000010L
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW_MASK                                                  0x0000020L
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW_MASK                                                  0x0000040L
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW_MASK                                                     0x0000080L
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR_MASK                                             0x0000100L
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID_MASK                                                              0x3000000L
+#define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING_MASK                                                           0xC000000L
+//SQ_INTERRUPT_WORD_AUTO_HI
+#define SQ_INTERRUPT_WORD_AUTO_HI__SE_ID__SHIFT                                                               0x8
+#define SQ_INTERRUPT_WORD_AUTO_HI__ENCODING__SHIFT                                                            0xa
+#define SQ_INTERRUPT_WORD_AUTO_HI__SE_ID_MASK                                                                 0x300L
+#define SQ_INTERRUPT_WORD_AUTO_HI__ENCODING_MASK                                                              0xC00L
+//SQ_INTERRUPT_WORD_AUTO_LO
+#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE__SHIFT                                                        0x0
+#define SQ_INTERRUPT_WORD_AUTO_LO__WLT__SHIFT                                                                 0x1
+#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE_BUF_FULL__SHIFT                                               0x2
+#define SQ_INTERRUPT_WORD_AUTO_LO__REG_TIMESTAMP__SHIFT                                                       0x3
+#define SQ_INTERRUPT_WORD_AUTO_LO__CMD_TIMESTAMP__SHIFT                                                       0x4
+#define SQ_INTERRUPT_WORD_AUTO_LO__HOST_CMD_OVERFLOW__SHIFT                                                   0x5
+#define SQ_INTERRUPT_WORD_AUTO_LO__HOST_REG_OVERFLOW__SHIFT                                                   0x6
+#define SQ_INTERRUPT_WORD_AUTO_LO__IMMED_OVERFLOW__SHIFT                                                      0x7
+#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE_UTC_ERROR__SHIFT                                              0x8
+#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE_MASK                                                          0x001L
+#define SQ_INTERRUPT_WORD_AUTO_LO__WLT_MASK                                                                   0x002L
+#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE_BUF_FULL_MASK                                                 0x004L
+#define SQ_INTERRUPT_WORD_AUTO_LO__REG_TIMESTAMP_MASK                                                         0x008L
+#define SQ_INTERRUPT_WORD_AUTO_LO__CMD_TIMESTAMP_MASK                                                         0x010L
+#define SQ_INTERRUPT_WORD_AUTO_LO__HOST_CMD_OVERFLOW_MASK                                                     0x020L
+#define SQ_INTERRUPT_WORD_AUTO_LO__HOST_REG_OVERFLOW_MASK                                                     0x040L
+#define SQ_INTERRUPT_WORD_AUTO_LO__IMMED_OVERFLOW_MASK                                                        0x080L
+#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE_UTC_ERROR_MASK                                                0x100L
+//SQ_INTERRUPT_WORD_CMN_CTXID
+#define SQ_INTERRUPT_WORD_CMN_CTXID__SE_ID__SHIFT                                                             0x18
+#define SQ_INTERRUPT_WORD_CMN_CTXID__ENCODING__SHIFT                                                          0x1a
+#define SQ_INTERRUPT_WORD_CMN_CTXID__SE_ID_MASK                                                               0x3000000L
+#define SQ_INTERRUPT_WORD_CMN_CTXID__ENCODING_MASK                                                            0xC000000L
+//SQ_INTERRUPT_WORD_CMN_HI
+#define SQ_INTERRUPT_WORD_CMN_HI__SE_ID__SHIFT                                                                0x8
+#define SQ_INTERRUPT_WORD_CMN_HI__ENCODING__SHIFT                                                             0xa
+#define SQ_INTERRUPT_WORD_CMN_HI__SE_ID_MASK                                                                  0x300L
+#define SQ_INTERRUPT_WORD_CMN_HI__ENCODING_MASK                                                               0xC00L
+//SQ_INTERRUPT_WORD_WAVE_CTXID
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA__SHIFT                                                             0x0
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID__SHIFT                                                            0xc
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV__SHIFT                                                             0xd
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID__SHIFT                                                          0xe
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID__SHIFT                                                          0x12
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID__SHIFT                                                            0x14
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID__SHIFT                                                            0x18
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING__SHIFT                                                         0x1a
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA_MASK                                                               0x0000FFFL
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID_MASK                                                              0x0001000L
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV_MASK                                                               0x0002000L
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID_MASK                                                            0x003C000L
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID_MASK                                                            0x00C0000L
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID_MASK                                                              0x0F00000L
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID_MASK                                                              0x3000000L
+#define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING_MASK                                                           0xC000000L
+//SQ_INTERRUPT_WORD_WAVE_HI
+#define SQ_INTERRUPT_WORD_WAVE_HI__CU_ID__SHIFT                                                               0x0
+#define SQ_INTERRUPT_WORD_WAVE_HI__VM_ID__SHIFT                                                               0x4
+#define SQ_INTERRUPT_WORD_WAVE_HI__SE_ID__SHIFT                                                               0x8
+#define SQ_INTERRUPT_WORD_WAVE_HI__ENCODING__SHIFT                                                            0xa
+#define SQ_INTERRUPT_WORD_WAVE_HI__CU_ID_MASK                                                                 0x00FL
+#define SQ_INTERRUPT_WORD_WAVE_HI__VM_ID_MASK                                                                 0x0F0L
+#define SQ_INTERRUPT_WORD_WAVE_HI__SE_ID_MASK                                                                 0x300L
+#define SQ_INTERRUPT_WORD_WAVE_HI__ENCODING_MASK                                                              0xC00L
+//SQ_INTERRUPT_WORD_WAVE_LO
+#define SQ_INTERRUPT_WORD_WAVE_LO__DATA__SHIFT                                                                0x0
+#define SQ_INTERRUPT_WORD_WAVE_LO__SH_ID__SHIFT                                                               0x18
+#define SQ_INTERRUPT_WORD_WAVE_LO__PRIV__SHIFT                                                                0x19
+#define SQ_INTERRUPT_WORD_WAVE_LO__WAVE_ID__SHIFT                                                             0x1a
+#define SQ_INTERRUPT_WORD_WAVE_LO__SIMD_ID__SHIFT                                                             0x1e
+#define SQ_INTERRUPT_WORD_WAVE_LO__DATA_MASK                                                                  0x00FFFFFFL
+#define SQ_INTERRUPT_WORD_WAVE_LO__SH_ID_MASK                                                                 0x01000000L
+#define SQ_INTERRUPT_WORD_WAVE_LO__PRIV_MASK                                                                  0x02000000L
+#define SQ_INTERRUPT_WORD_WAVE_LO__WAVE_ID_MASK                                                               0x3C000000L
+#define SQ_INTERRUPT_WORD_WAVE_LO__SIMD_ID_MASK                                                               0xC0000000L
+
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_4_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_4_2_offset.h
new file mode 100644
index 000000000000..546b043ccdf5
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_4_2_offset.h
@@ -0,0 +1,219 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _hdp_4_4_2_OFFSET_HEADER
+#define _hdp_4_4_2_OFFSET_HEADER
+
+
+
+// addressBlock: aid_hdp_hdpdec
+// base address: 0x3c80
+#define regHDP_MMHUB_TLVL                                                                               0x0000
+#define regHDP_MMHUB_TLVL_BASE_IDX                                                                      0
+#define regHDP_MMHUB_UNITID                                                                             0x0001
+#define regHDP_MMHUB_UNITID_BASE_IDX                                                                    0
+#define regHDP_NONSURFACE_BASE                                                                          0x0040
+#define regHDP_NONSURFACE_BASE_BASE_IDX                                                                 0
+#define regHDP_NONSURFACE_INFO                                                                          0x0041
+#define regHDP_NONSURFACE_INFO_BASE_IDX                                                                 0
+#define regHDP_NONSURFACE_BASE_HI                                                                       0x0042
+#define regHDP_NONSURFACE_BASE_HI_BASE_IDX                                                              0
+#define regHDP_SURFACE_WRITE_FLAGS                                                                      0x00c4
+#define regHDP_SURFACE_WRITE_FLAGS_BASE_IDX                                                             0
+#define regHDP_SURFACE_READ_FLAGS                                                                       0x00c5
+#define regHDP_SURFACE_READ_FLAGS_BASE_IDX                                                              0
+#define regHDP_SURFACE_WRITE_FLAGS_CLR                                                                  0x00c6
+#define regHDP_SURFACE_WRITE_FLAGS_CLR_BASE_IDX                                                         0
+#define regHDP_SURFACE_READ_FLAGS_CLR                                                                   0x00c7
+#define regHDP_SURFACE_READ_FLAGS_CLR_BASE_IDX                                                          0
+#define regHDP_NONSURF_FLAGS                                                                            0x00c8
+#define regHDP_NONSURF_FLAGS_BASE_IDX                                                                   0
+#define regHDP_NONSURF_FLAGS_CLR                                                                        0x00c9
+#define regHDP_NONSURF_FLAGS_CLR_BASE_IDX                                                               0
+#define regHDP_HOST_PATH_CNTL                                                                           0x00cc
+#define regHDP_HOST_PATH_CNTL_BASE_IDX                                                                  0
+#define regHDP_SW_SEMAPHORE                                                                             0x00cd
+#define regHDP_SW_SEMAPHORE_BASE_IDX                                                                    0
+#define regHDP_DEBUG0                                                                                   0x00ce
+#define regHDP_DEBUG0_BASE_IDX                                                                          0
+#define regHDP_LAST_SURFACE_HIT                                                                         0x00d0
+#define regHDP_LAST_SURFACE_HIT_BASE_IDX                                                                0
+#define regHDP_OUTSTANDING_REQ                                                                          0x00d2
+#define regHDP_OUTSTANDING_REQ_BASE_IDX                                                                 0
+#define regHDP_MISC_CNTL                                                                                0x00d3
+#define regHDP_MISC_CNTL_BASE_IDX                                                                       0
+#define regHDP_MEM_POWER_CTRL                                                                           0x00d4
+#define regHDP_MEM_POWER_CTRL_BASE_IDX                                                                  0
+#define regHDP_MMHUB_CNTL                                                                               0x00d5
+#define regHDP_MMHUB_CNTL_BASE_IDX                                                                      0
+#define regHDP_EDC_CNT                                                                                  0x00d6
+#define regHDP_EDC_CNT_BASE_IDX                                                                         0
+#define regHDP_VERSION                                                                                  0x00d7
+#define regHDP_VERSION_BASE_IDX                                                                         0
+#define regHDP_CLK_CNTL                                                                                 0x00d8
+#define regHDP_CLK_CNTL_BASE_IDX                                                                        0
+#define regHDP_MEMIO_CNTL                                                                               0x00f6
+#define regHDP_MEMIO_CNTL_BASE_IDX                                                                      0
+#define regHDP_MEMIO_ADDR                                                                               0x00f7
+#define regHDP_MEMIO_ADDR_BASE_IDX                                                                      0
+#define regHDP_MEMIO_STATUS                                                                             0x00f8
+#define regHDP_MEMIO_STATUS_BASE_IDX                                                                    0
+#define regHDP_MEMIO_WR_DATA                                                                            0x00f9
+#define regHDP_MEMIO_WR_DATA_BASE_IDX                                                                   0
+#define regHDP_MEMIO_RD_DATA                                                                            0x00fa
+#define regHDP_MEMIO_RD_DATA_BASE_IDX                                                                   0
+#define regHDP_XDP_DIRECT2HDP_FIRST                                                                     0x0100
+#define regHDP_XDP_DIRECT2HDP_FIRST_BASE_IDX                                                            0
+#define regHDP_XDP_D2H_FLUSH                                                                            0x0101
+#define regHDP_XDP_D2H_FLUSH_BASE_IDX                                                                   0
+#define regHDP_XDP_D2H_BAR_UPDATE                                                                       0x0102
+#define regHDP_XDP_D2H_BAR_UPDATE_BASE_IDX                                                              0
+#define regHDP_XDP_D2H_RSVD_3                                                                           0x0103
+#define regHDP_XDP_D2H_RSVD_3_BASE_IDX                                                                  0
+#define regHDP_XDP_D2H_RSVD_4                                                                           0x0104
+#define regHDP_XDP_D2H_RSVD_4_BASE_IDX                                                                  0
+#define regHDP_XDP_D2H_RSVD_5                                                                           0x0105
+#define regHDP_XDP_D2H_RSVD_5_BASE_IDX                                                                  0
+#define regHDP_XDP_D2H_RSVD_6                                                                           0x0106
+#define regHDP_XDP_D2H_RSVD_6_BASE_IDX                                                                  0
+#define regHDP_XDP_D2H_RSVD_7                                                                           0x0107
+#define regHDP_XDP_D2H_RSVD_7_BASE_IDX                                                                  0
+#define regHDP_XDP_D2H_RSVD_8                                                                           0x0108
+#define regHDP_XDP_D2H_RSVD_8_BASE_IDX                                                                  0
+#define regHDP_XDP_D2H_RSVD_9                                                                           0x0109
+#define regHDP_XDP_D2H_RSVD_9_BASE_IDX                                                                  0
+#define regHDP_XDP_D2H_RSVD_10                                                                          0x010a
+#define regHDP_XDP_D2H_RSVD_10_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_11                                                                          0x010b
+#define regHDP_XDP_D2H_RSVD_11_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_12                                                                          0x010c
+#define regHDP_XDP_D2H_RSVD_12_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_13                                                                          0x010d
+#define regHDP_XDP_D2H_RSVD_13_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_14                                                                          0x010e
+#define regHDP_XDP_D2H_RSVD_14_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_15                                                                          0x010f
+#define regHDP_XDP_D2H_RSVD_15_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_16                                                                          0x0110
+#define regHDP_XDP_D2H_RSVD_16_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_17                                                                          0x0111
+#define regHDP_XDP_D2H_RSVD_17_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_18                                                                          0x0112
+#define regHDP_XDP_D2H_RSVD_18_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_19                                                                          0x0113
+#define regHDP_XDP_D2H_RSVD_19_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_20                                                                          0x0114
+#define regHDP_XDP_D2H_RSVD_20_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_21                                                                          0x0115
+#define regHDP_XDP_D2H_RSVD_21_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_22                                                                          0x0116
+#define regHDP_XDP_D2H_RSVD_22_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_23                                                                          0x0117
+#define regHDP_XDP_D2H_RSVD_23_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_24                                                                          0x0118
+#define regHDP_XDP_D2H_RSVD_24_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_25                                                                          0x0119
+#define regHDP_XDP_D2H_RSVD_25_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_26                                                                          0x011a
+#define regHDP_XDP_D2H_RSVD_26_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_27                                                                          0x011b
+#define regHDP_XDP_D2H_RSVD_27_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_28                                                                          0x011c
+#define regHDP_XDP_D2H_RSVD_28_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_29                                                                          0x011d
+#define regHDP_XDP_D2H_RSVD_29_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_30                                                                          0x011e
+#define regHDP_XDP_D2H_RSVD_30_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_31                                                                          0x011f
+#define regHDP_XDP_D2H_RSVD_31_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_32                                                                          0x0120
+#define regHDP_XDP_D2H_RSVD_32_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_33                                                                          0x0121
+#define regHDP_XDP_D2H_RSVD_33_BASE_IDX                                                                 0
+#define regHDP_XDP_D2H_RSVD_34                                                                          0x0122
+#define regHDP_XDP_D2H_RSVD_34_BASE_IDX                                                                 0
+#define regHDP_XDP_DIRECT2HDP_LAST                                                                      0x0123
+#define regHDP_XDP_DIRECT2HDP_LAST_BASE_IDX                                                             0
+#define regHDP_XDP_P2P_BAR_CFG                                                                          0x0124
+#define regHDP_XDP_P2P_BAR_CFG_BASE_IDX                                                                 0
+#define regHDP_XDP_P2P_MBX_OFFSET                                                                       0x0125
+#define regHDP_XDP_P2P_MBX_OFFSET_BASE_IDX                                                              0
+#define regHDP_XDP_P2P_MBX_ADDR0                                                                        0x0126
+#define regHDP_XDP_P2P_MBX_ADDR0_BASE_IDX                                                               0
+#define regHDP_XDP_P2P_MBX_ADDR1                                                                        0x0127
+#define regHDP_XDP_P2P_MBX_ADDR1_BASE_IDX                                                               0
+#define regHDP_XDP_P2P_MBX_ADDR2                                                                        0x0128
+#define regHDP_XDP_P2P_MBX_ADDR2_BASE_IDX                                                               0
+#define regHDP_XDP_P2P_MBX_ADDR3                                                                        0x0129
+#define regHDP_XDP_P2P_MBX_ADDR3_BASE_IDX                                                               0
+#define regHDP_XDP_P2P_MBX_ADDR4                                                                        0x012a
+#define regHDP_XDP_P2P_MBX_ADDR4_BASE_IDX                                                               0
+#define regHDP_XDP_P2P_MBX_ADDR5                                                                        0x012b
+#define regHDP_XDP_P2P_MBX_ADDR5_BASE_IDX                                                               0
+#define regHDP_XDP_P2P_MBX_ADDR6                                                                        0x012c
+#define regHDP_XDP_P2P_MBX_ADDR6_BASE_IDX                                                               0
+#define regHDP_XDP_HDP_MBX_MC_CFG                                                                       0x012d
+#define regHDP_XDP_HDP_MBX_MC_CFG_BASE_IDX                                                              0
+#define regHDP_XDP_HDP_MC_CFG                                                                           0x012e
+#define regHDP_XDP_HDP_MC_CFG_BASE_IDX                                                                  0
+#define regHDP_XDP_HST_CFG                                                                              0x012f
+#define regHDP_XDP_HST_CFG_BASE_IDX                                                                     0
+#define regHDP_XDP_HDP_IPH_CFG                                                                          0x0131
+#define regHDP_XDP_HDP_IPH_CFG_BASE_IDX                                                                 0
+#define regHDP_XDP_P2P_BAR0                                                                             0x0134
+#define regHDP_XDP_P2P_BAR0_BASE_IDX                                                                    0
+#define regHDP_XDP_P2P_BAR1                                                                             0x0135
+#define regHDP_XDP_P2P_BAR1_BASE_IDX                                                                    0
+#define regHDP_XDP_P2P_BAR2                                                                             0x0136
+#define regHDP_XDP_P2P_BAR2_BASE_IDX                                                                    0
+#define regHDP_XDP_P2P_BAR3                                                                             0x0137
+#define regHDP_XDP_P2P_BAR3_BASE_IDX                                                                    0
+#define regHDP_XDP_P2P_BAR4                                                                             0x0138
+#define regHDP_XDP_P2P_BAR4_BASE_IDX                                                                    0
+#define regHDP_XDP_P2P_BAR5                                                                             0x0139
+#define regHDP_XDP_P2P_BAR5_BASE_IDX                                                                    0
+#define regHDP_XDP_P2P_BAR6                                                                             0x013a
+#define regHDP_XDP_P2P_BAR6_BASE_IDX                                                                    0
+#define regHDP_XDP_P2P_BAR7                                                                             0x013b
+#define regHDP_XDP_P2P_BAR7_BASE_IDX                                                                    0
+#define regHDP_XDP_FLUSH_ARMED_STS                                                                      0x013c
+#define regHDP_XDP_FLUSH_ARMED_STS_BASE_IDX                                                             0
+#define regHDP_XDP_FLUSH_CNTR0_STS                                                                      0x013d
+#define regHDP_XDP_FLUSH_CNTR0_STS_BASE_IDX                                                             0
+#define regHDP_XDP_BUSY_STS                                                                             0x013e
+#define regHDP_XDP_BUSY_STS_BASE_IDX                                                                    0
+#define regHDP_XDP_STICKY                                                                               0x013f
+#define regHDP_XDP_STICKY_BASE_IDX                                                                      0
+#define regHDP_XDP_CHKN                                                                                 0x0140
+#define regHDP_XDP_CHKN_BASE_IDX                                                                        0
+#define regHDP_XDP_BARS_ADDR_39_36                                                                      0x0144
+#define regHDP_XDP_BARS_ADDR_39_36_BASE_IDX                                                             0
+#define regHDP_XDP_MC_VM_FB_LOCATION_BASE                                                               0x0145
+#define regHDP_XDP_MC_VM_FB_LOCATION_BASE_BASE_IDX                                                      0
+#define regHDP_XDP_GPU_IOV_VIOLATION_LOG                                                                0x0148
+#define regHDP_XDP_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                       0
+#define regHDP_XDP_GPU_IOV_VIOLATION_LOG2                                                               0x0149
+#define regHDP_XDP_GPU_IOV_VIOLATION_LOG2_BASE_IDX                                                      0
+#define regHDP_XDP_MMHUB_ERROR                                                                          0x014a
+#define regHDP_XDP_MMHUB_ERROR_BASE_IDX                                                                 0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_4_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_4_2_sh_mask.h
new file mode 100644
index 000000000000..3ccd2797936e
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_4_2_sh_mask.h
@@ -0,0 +1,663 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _hdp_4_4_2_SH_MASK_HEADER
+#define _hdp_4_4_2_SH_MASK_HEADER
+
+
+// addressBlock: aid_hdp_hdpdec
+//HDP_MMHUB_TLVL
+#define HDP_MMHUB_TLVL__HDP_WR_TLVL__SHIFT                                                                    0x0
+#define HDP_MMHUB_TLVL__HDP_RD_TLVL__SHIFT                                                                    0x4
+#define HDP_MMHUB_TLVL__XDP_WR_TLVL__SHIFT                                                                    0x8
+#define HDP_MMHUB_TLVL__XDP_RD_TLVL__SHIFT                                                                    0xc
+#define HDP_MMHUB_TLVL__XDP_MBX_WR_TLVL__SHIFT                                                                0x10
+#define HDP_MMHUB_TLVL__HDP_WR_TLVL_MASK                                                                      0x0000000FL
+#define HDP_MMHUB_TLVL__HDP_RD_TLVL_MASK                                                                      0x000000F0L
+#define HDP_MMHUB_TLVL__XDP_WR_TLVL_MASK                                                                      0x00000F00L
+#define HDP_MMHUB_TLVL__XDP_RD_TLVL_MASK                                                                      0x0000F000L
+#define HDP_MMHUB_TLVL__XDP_MBX_WR_TLVL_MASK                                                                  0x000F0000L
+//HDP_MMHUB_UNITID
+#define HDP_MMHUB_UNITID__HDP_UNITID__SHIFT                                                                   0x0
+#define HDP_MMHUB_UNITID__XDP_UNITID__SHIFT                                                                   0x8
+#define HDP_MMHUB_UNITID__XDP_MBX_UNITID__SHIFT                                                               0x10
+#define HDP_MMHUB_UNITID__HDP_UNITID_MASK                                                                     0x0000003FL
+#define HDP_MMHUB_UNITID__XDP_UNITID_MASK                                                                     0x00003F00L
+#define HDP_MMHUB_UNITID__XDP_MBX_UNITID_MASK                                                                 0x003F0000L
+//HDP_NONSURFACE_BASE
+#define HDP_NONSURFACE_BASE__NONSURF_BASE_39_8__SHIFT                                                         0x0
+#define HDP_NONSURFACE_BASE__NONSURF_BASE_39_8_MASK                                                           0xFFFFFFFFL
+//HDP_NONSURFACE_INFO
+#define HDP_NONSURFACE_INFO__NONSURF_SWAP__SHIFT                                                              0x4
+#define HDP_NONSURFACE_INFO__NONSURF_VMID__SHIFT                                                              0x8
+#define HDP_NONSURFACE_INFO__NONSURF_SWAP_MASK                                                                0x00000030L
+#define HDP_NONSURFACE_INFO__NONSURF_VMID_MASK                                                                0x00000F00L
+//HDP_NONSURFACE_BASE_HI
+#define HDP_NONSURFACE_BASE_HI__NONSURF_BASE_47_40__SHIFT                                                     0x0
+#define HDP_NONSURFACE_BASE_HI__NONSURF_BASE_47_40_MASK                                                       0x000000FFL
+//HDP_SURFACE_WRITE_FLAGS
+#define HDP_SURFACE_WRITE_FLAGS__SURF0_WRITE_FLAG__SHIFT                                                      0x0
+#define HDP_SURFACE_WRITE_FLAGS__SURF1_WRITE_FLAG__SHIFT                                                      0x1
+#define HDP_SURFACE_WRITE_FLAGS__SURF0_WRITE_FLAG_MASK                                                        0x00000001L
+#define HDP_SURFACE_WRITE_FLAGS__SURF1_WRITE_FLAG_MASK                                                        0x00000002L
+//HDP_SURFACE_READ_FLAGS
+#define HDP_SURFACE_READ_FLAGS__SURF0_READ_FLAG__SHIFT                                                        0x0
+#define HDP_SURFACE_READ_FLAGS__SURF1_READ_FLAG__SHIFT                                                        0x1
+#define HDP_SURFACE_READ_FLAGS__SURF0_READ_FLAG_MASK                                                          0x00000001L
+#define HDP_SURFACE_READ_FLAGS__SURF1_READ_FLAG_MASK                                                          0x00000002L
+//HDP_SURFACE_WRITE_FLAGS_CLR
+#define HDP_SURFACE_WRITE_FLAGS_CLR__SURF0_WRITE_FLAG_CLR__SHIFT                                              0x0
+#define HDP_SURFACE_WRITE_FLAGS_CLR__SURF1_WRITE_FLAG_CLR__SHIFT                                              0x1
+#define HDP_SURFACE_WRITE_FLAGS_CLR__SURF0_WRITE_FLAG_CLR_MASK                                                0x00000001L
+#define HDP_SURFACE_WRITE_FLAGS_CLR__SURF1_WRITE_FLAG_CLR_MASK                                                0x00000002L
+//HDP_SURFACE_READ_FLAGS_CLR
+#define HDP_SURFACE_READ_FLAGS_CLR__SURF0_READ_FLAG_CLR__SHIFT                                                0x0
+#define HDP_SURFACE_READ_FLAGS_CLR__SURF1_READ_FLAG_CLR__SHIFT                                                0x1
+#define HDP_SURFACE_READ_FLAGS_CLR__SURF0_READ_FLAG_CLR_MASK                                                  0x00000001L
+#define HDP_SURFACE_READ_FLAGS_CLR__SURF1_READ_FLAG_CLR_MASK                                                  0x00000002L
+//HDP_NONSURF_FLAGS
+#define HDP_NONSURF_FLAGS__NONSURF_WRITE_FLAG__SHIFT                                                          0x0
+#define HDP_NONSURF_FLAGS__NONSURF_READ_FLAG__SHIFT                                                           0x1
+#define HDP_NONSURF_FLAGS__NONSURF_WRITE_FLAG_MASK                                                            0x00000001L
+#define HDP_NONSURF_FLAGS__NONSURF_READ_FLAG_MASK                                                             0x00000002L
+//HDP_NONSURF_FLAGS_CLR
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_WRITE_FLAG_CLR__SHIFT                                                  0x0
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_READ_FLAG_CLR__SHIFT                                                   0x1
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_WRITE_FLAG_CLR_MASK                                                    0x00000001L
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_READ_FLAG_CLR_MASK                                                     0x00000002L
+//HDP_HOST_PATH_CNTL
+#define HDP_HOST_PATH_CNTL__WR_STALL_TIMER__SHIFT                                                             0x9
+#define HDP_HOST_PATH_CNTL__RD_STALL_TIMER__SHIFT                                                             0xb
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER_PRELOAD_CFG__SHIFT                                            0x12
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER__SHIFT                                                        0x13
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_EN__SHIFT                                                           0x15
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_64B_EN__SHIFT                                                       0x16
+#define HDP_HOST_PATH_CNTL__ALL_SURFACES_DIS__SHIFT                                                           0x1d
+#define HDP_HOST_PATH_CNTL__WR_STALL_TIMER_MASK                                                               0x00000600L
+#define HDP_HOST_PATH_CNTL__RD_STALL_TIMER_MASK                                                               0x00001800L
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER_PRELOAD_CFG_MASK                                              0x00040000L
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER_MASK                                                          0x00180000L
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_EN_MASK                                                             0x00200000L
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_64B_EN_MASK                                                         0x00400000L
+#define HDP_HOST_PATH_CNTL__ALL_SURFACES_DIS_MASK                                                             0x20000000L
+//HDP_SW_SEMAPHORE
+#define HDP_SW_SEMAPHORE__SW_SEMAPHORE__SHIFT                                                                 0x0
+#define HDP_SW_SEMAPHORE__SW_SEMAPHORE_MASK                                                                   0xFFFFFFFFL
+//HDP_DEBUG0
+#define HDP_DEBUG0__HDP_DEBUG__SHIFT                                                                          0x0
+#define HDP_DEBUG0__HDP_DEBUG_MASK                                                                            0xFFFFFFFFL
+//HDP_LAST_SURFACE_HIT
+#define HDP_LAST_SURFACE_HIT__LAST_SURFACE_HIT__SHIFT                                                         0x0
+#define HDP_LAST_SURFACE_HIT__LAST_SURFACE_HIT_MASK                                                           0x00000003L
+//HDP_OUTSTANDING_REQ
+#define HDP_OUTSTANDING_REQ__WRITE_REQ__SHIFT                                                                 0x0
+#define HDP_OUTSTANDING_REQ__READ_REQ__SHIFT                                                                  0x8
+#define HDP_OUTSTANDING_REQ__WRITE_REQ_MASK                                                                   0x000000FFL
+#define HDP_OUTSTANDING_REQ__READ_REQ_MASK                                                                    0x0000FF00L
+//HDP_MISC_CNTL
+#define HDP_MISC_CNTL__IDLE_HYSTERESIS_CNTL__SHIFT                                                            0x2
+#define HDP_MISC_CNTL__ATOMIC_BUFFER_PROTECT_ENABLE__SHIFT                                                    0x4
+#define HDP_MISC_CNTL__OUTSTANDING_WRITE_COUNT_1024__SHIFT                                                    0x5
+#define HDP_MISC_CNTL__RAW_ADDR_CAM_ENABLE__SHIFT                                                             0x7
+#define HDP_MISC_CNTL__MMHUB_EARLY_WRACK_ENABLE__SHIFT                                                        0x8
+#define HDP_MISC_CNTL__EARLY_WRACK_MISSING_PROTECT_ENABLE__SHIFT                                              0x9
+#define HDP_MISC_CNTL__SIMULTANEOUS_READS_WRITES__SHIFT                                                       0xb
+#define HDP_MISC_CNTL__SYSHUB_CHANNEL_PRIORITY__SHIFT                                                         0xc
+#define HDP_MISC_CNTL__READ_BUFFER_WATERMARK__SHIFT                                                           0xe
+#define HDP_MISC_CNTL__SRAM_ECC_ENABLE__SHIFT                                                                 0x14
+#define HDP_MISC_CNTL__FED_ENABLE__SHIFT                                                                      0x15
+#define HDP_MISC_CNTL__ATOMIC_FED_ENABLE__SHIFT                                                               0x16
+#define HDP_MISC_CNTL__MMHUB_WRBURST_ENABLE__SHIFT                                                            0x18
+#define HDP_MISC_CNTL__HDP_MMHUB_PENDING_WR_TAG_CHECK__SHIFT                                                  0x1a
+#define HDP_MISC_CNTL__XDP_MMHUB_PENDING_WR_TAG_CHECK__SHIFT                                                  0x1b
+#define HDP_MISC_CNTL__MMHUB_WRBURST_SIZE__SHIFT                                                              0x1e
+#define HDP_MISC_CNTL__IDLE_HYSTERESIS_CNTL_MASK                                                              0x0000000CL
+#define HDP_MISC_CNTL__ATOMIC_BUFFER_PROTECT_ENABLE_MASK                                                      0x00000010L
+#define HDP_MISC_CNTL__OUTSTANDING_WRITE_COUNT_1024_MASK                                                      0x00000020L
+#define HDP_MISC_CNTL__RAW_ADDR_CAM_ENABLE_MASK                                                               0x00000080L
+#define HDP_MISC_CNTL__MMHUB_EARLY_WRACK_ENABLE_MASK                                                          0x00000100L
+#define HDP_MISC_CNTL__EARLY_WRACK_MISSING_PROTECT_ENABLE_MASK                                                0x00000200L
+#define HDP_MISC_CNTL__SIMULTANEOUS_READS_WRITES_MASK                                                         0x00000800L
+#define HDP_MISC_CNTL__SYSHUB_CHANNEL_PRIORITY_MASK                                                           0x00003000L
+#define HDP_MISC_CNTL__READ_BUFFER_WATERMARK_MASK                                                             0x0000C000L
+#define HDP_MISC_CNTL__SRAM_ECC_ENABLE_MASK                                                                   0x00100000L
+#define HDP_MISC_CNTL__FED_ENABLE_MASK                                                                        0x00200000L
+#define HDP_MISC_CNTL__ATOMIC_FED_ENABLE_MASK                                                                 0x00400000L
+#define HDP_MISC_CNTL__MMHUB_WRBURST_ENABLE_MASK                                                              0x01000000L
+#define HDP_MISC_CNTL__HDP_MMHUB_PENDING_WR_TAG_CHECK_MASK                                                    0x04000000L
+#define HDP_MISC_CNTL__XDP_MMHUB_PENDING_WR_TAG_CHECK_MASK                                                    0x08000000L
+#define HDP_MISC_CNTL__MMHUB_WRBURST_SIZE_MASK                                                                0x40000000L
+//HDP_MEM_POWER_CTRL
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN__SHIFT                                                      0x0
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN__SHIFT                                                        0x1
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_DS_EN__SHIFT                                                        0x2
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_SD_EN__SHIFT                                                        0x3
+#define HDP_MEM_POWER_CTRL__IPH_MEM_IDLE_HYSTERESIS__SHIFT                                                    0x4
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_UP_RECOVER_DELAY__SHIFT                                             0x8
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_DOWN_LS_ENTER_DELAY__SHIFT                                          0xe
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN__SHIFT                                                       0x10
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN__SHIFT                                                         0x11
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_DS_EN__SHIFT                                                         0x12
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_SD_EN__SHIFT                                                         0x13
+#define HDP_MEM_POWER_CTRL__RC_MEM_IDLE_HYSTERESIS__SHIFT                                                     0x14
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_UP_RECOVER_DELAY__SHIFT                                              0x18
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_DOWN_LS_ENTER_DELAY__SHIFT                                           0x1e
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK                                                        0x00000001L
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK                                                          0x00000002L
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_DS_EN_MASK                                                          0x00000004L
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_SD_EN_MASK                                                          0x00000008L
+#define HDP_MEM_POWER_CTRL__IPH_MEM_IDLE_HYSTERESIS_MASK                                                      0x00000070L
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_UP_RECOVER_DELAY_MASK                                               0x00003F00L
+#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_DOWN_LS_ENTER_DELAY_MASK                                            0x0000C000L
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK                                                         0x00010000L
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK                                                           0x00020000L
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_DS_EN_MASK                                                           0x00040000L
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_SD_EN_MASK                                                           0x00080000L
+#define HDP_MEM_POWER_CTRL__RC_MEM_IDLE_HYSTERESIS_MASK                                                       0x00700000L
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_UP_RECOVER_DELAY_MASK                                                0x3F000000L
+#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_DOWN_LS_ENTER_DELAY_MASK                                             0xC0000000L
+//HDP_MMHUB_CNTL
+#define HDP_MMHUB_CNTL__HDP_MMHUB_RO__SHIFT                                                                   0x0
+#define HDP_MMHUB_CNTL__HDP_MMHUB_GCC__SHIFT                                                                  0x1
+#define HDP_MMHUB_CNTL__HDP_MMHUB_SNOOP__SHIFT                                                                0x2
+#define HDP_MMHUB_CNTL__HDP_MMHUB_RO_OVERRIDE__SHIFT                                                          0x4
+#define HDP_MMHUB_CNTL__HDP_MMHUB_GCC_OVERRIDE__SHIFT                                                         0x5
+#define HDP_MMHUB_CNTL__HDP_MMHUB_SNOOP_OVERRIDE__SHIFT                                                       0x6
+#define HDP_MMHUB_CNTL__HDP_MMHUB_RO_MASK                                                                     0x00000001L
+#define HDP_MMHUB_CNTL__HDP_MMHUB_GCC_MASK                                                                    0x00000002L
+#define HDP_MMHUB_CNTL__HDP_MMHUB_SNOOP_MASK                                                                  0x00000004L
+#define HDP_MMHUB_CNTL__HDP_MMHUB_RO_OVERRIDE_MASK                                                            0x00000010L
+#define HDP_MMHUB_CNTL__HDP_MMHUB_GCC_OVERRIDE_MASK                                                           0x00000020L
+#define HDP_MMHUB_CNTL__HDP_MMHUB_SNOOP_OVERRIDE_MASK                                                         0x00000040L
+//HDP_EDC_CNT
+#define HDP_EDC_CNT__MEM0_SED_COUNT__SHIFT                                                                    0x0
+#define HDP_EDC_CNT__MEM0_SED_COUNT_MASK                                                                      0x00000003L
+//HDP_VERSION
+#define HDP_VERSION__MINVER__SHIFT                                                                            0x0
+#define HDP_VERSION__MAJVER__SHIFT                                                                            0x8
+#define HDP_VERSION__REV__SHIFT                                                                               0x10
+#define HDP_VERSION__MINVER_MASK                                                                              0x000000FFL
+#define HDP_VERSION__MAJVER_MASK                                                                              0x0000FF00L
+#define HDP_VERSION__REV_MASK                                                                                 0x00FF0000L
+//HDP_CLK_CNTL
+#define HDP_CLK_CNTL__REG_CLK_ENABLE_COUNT__SHIFT                                                             0x0
+#define HDP_CLK_CNTL__REG_WAKE_DYN_CLK__SHIFT                                                                 0x4
+#define HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE__SHIFT                                                        0x1a
+#define HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE__SHIFT                                                         0x1b
+#define HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE__SHIFT                                                           0x1c
+#define HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE__SHIFT                                                            0x1d
+#define HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE__SHIFT                                                        0x1e
+#define HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE__SHIFT                                                        0x1f
+#define HDP_CLK_CNTL__REG_CLK_ENABLE_COUNT_MASK                                                               0x0000000FL
+#define HDP_CLK_CNTL__REG_WAKE_DYN_CLK_MASK                                                                   0x00000010L
+#define HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK                                                          0x04000000L
+#define HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK                                                           0x08000000L
+#define HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK                                                             0x10000000L
+#define HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK                                                              0x20000000L
+#define HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK                                                          0x40000000L
+#define HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK                                                          0x80000000L
+//HDP_MEMIO_CNTL
+#define HDP_MEMIO_CNTL__MEMIO_SEND__SHIFT                                                                     0x0
+#define HDP_MEMIO_CNTL__MEMIO_OP__SHIFT                                                                       0x1
+#define HDP_MEMIO_CNTL__MEMIO_BE__SHIFT                                                                       0x2
+#define HDP_MEMIO_CNTL__MEMIO_WR_STROBE__SHIFT                                                                0x6
+#define HDP_MEMIO_CNTL__MEMIO_RD_STROBE__SHIFT                                                                0x7
+#define HDP_MEMIO_CNTL__MEMIO_ADDR_UPPER__SHIFT                                                               0x8
+#define HDP_MEMIO_CNTL__MEMIO_CLR_WR_ERROR__SHIFT                                                             0xe
+#define HDP_MEMIO_CNTL__MEMIO_CLR_RD_ERROR__SHIFT                                                             0xf
+#define HDP_MEMIO_CNTL__MEMIO_VF__SHIFT                                                                       0x10
+#define HDP_MEMIO_CNTL__MEMIO_VFID__SHIFT                                                                     0x11
+#define HDP_MEMIO_CNTL__MEMIO_SEND_MASK                                                                       0x00000001L
+#define HDP_MEMIO_CNTL__MEMIO_OP_MASK                                                                         0x00000002L
+#define HDP_MEMIO_CNTL__MEMIO_BE_MASK                                                                         0x0000003CL
+#define HDP_MEMIO_CNTL__MEMIO_WR_STROBE_MASK                                                                  0x00000040L
+#define HDP_MEMIO_CNTL__MEMIO_RD_STROBE_MASK                                                                  0x00000080L
+#define HDP_MEMIO_CNTL__MEMIO_ADDR_UPPER_MASK                                                                 0x00003F00L
+#define HDP_MEMIO_CNTL__MEMIO_CLR_WR_ERROR_MASK                                                               0x00004000L
+#define HDP_MEMIO_CNTL__MEMIO_CLR_RD_ERROR_MASK                                                               0x00008000L
+#define HDP_MEMIO_CNTL__MEMIO_VF_MASK                                                                         0x00010000L
+#define HDP_MEMIO_CNTL__MEMIO_VFID_MASK                                                                       0x003E0000L
+//HDP_MEMIO_ADDR
+#define HDP_MEMIO_ADDR__MEMIO_ADDR_LOWER__SHIFT                                                               0x0
+#define HDP_MEMIO_ADDR__MEMIO_ADDR_LOWER_MASK                                                                 0xFFFFFFFFL
+//HDP_MEMIO_STATUS
+#define HDP_MEMIO_STATUS__MEMIO_WR_STATUS__SHIFT                                                              0x0
+#define HDP_MEMIO_STATUS__MEMIO_RD_STATUS__SHIFT                                                              0x1
+#define HDP_MEMIO_STATUS__MEMIO_WR_ERROR__SHIFT                                                               0x2
+#define HDP_MEMIO_STATUS__MEMIO_RD_ERROR__SHIFT                                                               0x3
+#define HDP_MEMIO_STATUS__MEMIO_WR_STATUS_MASK                                                                0x00000001L
+#define HDP_MEMIO_STATUS__MEMIO_RD_STATUS_MASK                                                                0x00000002L
+#define HDP_MEMIO_STATUS__MEMIO_WR_ERROR_MASK                                                                 0x00000004L
+#define HDP_MEMIO_STATUS__MEMIO_RD_ERROR_MASK                                                                 0x00000008L
+//HDP_MEMIO_WR_DATA
+#define HDP_MEMIO_WR_DATA__MEMIO_WR_DATA__SHIFT                                                               0x0
+#define HDP_MEMIO_WR_DATA__MEMIO_WR_DATA_MASK                                                                 0xFFFFFFFFL
+//HDP_MEMIO_RD_DATA
+#define HDP_MEMIO_RD_DATA__MEMIO_RD_DATA__SHIFT                                                               0x0
+#define HDP_MEMIO_RD_DATA__MEMIO_RD_DATA_MASK                                                                 0xFFFFFFFFL
+//HDP_XDP_DIRECT2HDP_FIRST
+#define HDP_XDP_DIRECT2HDP_FIRST__RESERVED__SHIFT                                                             0x0
+#define HDP_XDP_DIRECT2HDP_FIRST__RESERVED_MASK                                                               0xFFFFFFFFL
+//HDP_XDP_D2H_FLUSH
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_FLUSH_NUM__SHIFT                                                         0x0
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ENC_DATA__SHIFT                                                      0x4
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ADDR_SEL__SHIFT                                                      0x8
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_XPB_CLG__SHIFT                                                           0xb
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_SEND_HOST__SHIFT                                                         0x10
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_ALTER_FLUSH_NUM__SHIFT                                                   0x12
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_0__SHIFT                                                            0x13
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_1__SHIFT                                                            0x14
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_FLUSH_NUM_MASK                                                           0x0000000FL
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ENC_DATA_MASK                                                        0x000000F0L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ADDR_SEL_MASK                                                        0x00000700L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_XPB_CLG_MASK                                                             0x0000F800L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_SEND_HOST_MASK                                                           0x00010000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_ALTER_FLUSH_NUM_MASK                                                     0x00040000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_0_MASK                                                              0x00080000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_1_MASK                                                              0x00100000L
+//HDP_XDP_D2H_BAR_UPDATE
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_ADDR__SHIFT                                                    0x0
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_FLUSH_NUM__SHIFT                                               0x10
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_BAR_NUM__SHIFT                                                 0x14
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_ADDR_MASK                                                      0x0000FFFFL
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_FLUSH_NUM_MASK                                                 0x000F0000L
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_BAR_NUM_MASK                                                   0x00700000L
+//HDP_XDP_D2H_RSVD_3
+#define HDP_XDP_D2H_RSVD_3__RESERVED__SHIFT                                                                   0x0
+#define HDP_XDP_D2H_RSVD_3__RESERVED_MASK                                                                     0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_4
+#define HDP_XDP_D2H_RSVD_4__RESERVED__SHIFT                                                                   0x0
+#define HDP_XDP_D2H_RSVD_4__RESERVED_MASK                                                                     0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_5
+#define HDP_XDP_D2H_RSVD_5__RESERVED__SHIFT                                                                   0x0
+#define HDP_XDP_D2H_RSVD_5__RESERVED_MASK                                                                     0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_6
+#define HDP_XDP_D2H_RSVD_6__RESERVED__SHIFT                                                                   0x0
+#define HDP_XDP_D2H_RSVD_6__RESERVED_MASK                                                                     0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_7
+#define HDP_XDP_D2H_RSVD_7__RESERVED__SHIFT                                                                   0x0
+#define HDP_XDP_D2H_RSVD_7__RESERVED_MASK                                                                     0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_8
+#define HDP_XDP_D2H_RSVD_8__RESERVED__SHIFT                                                                   0x0
+#define HDP_XDP_D2H_RSVD_8__RESERVED_MASK                                                                     0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_9
+#define HDP_XDP_D2H_RSVD_9__RESERVED__SHIFT                                                                   0x0
+#define HDP_XDP_D2H_RSVD_9__RESERVED_MASK                                                                     0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_10
+#define HDP_XDP_D2H_RSVD_10__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_10__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_11
+#define HDP_XDP_D2H_RSVD_11__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_11__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_12
+#define HDP_XDP_D2H_RSVD_12__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_12__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_13
+#define HDP_XDP_D2H_RSVD_13__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_13__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_14
+#define HDP_XDP_D2H_RSVD_14__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_14__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_15
+#define HDP_XDP_D2H_RSVD_15__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_15__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_16
+#define HDP_XDP_D2H_RSVD_16__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_16__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_17
+#define HDP_XDP_D2H_RSVD_17__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_17__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_18
+#define HDP_XDP_D2H_RSVD_18__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_18__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_19
+#define HDP_XDP_D2H_RSVD_19__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_19__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_20
+#define HDP_XDP_D2H_RSVD_20__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_20__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_21
+#define HDP_XDP_D2H_RSVD_21__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_21__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_22
+#define HDP_XDP_D2H_RSVD_22__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_22__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_23
+#define HDP_XDP_D2H_RSVD_23__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_23__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_24
+#define HDP_XDP_D2H_RSVD_24__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_24__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_25
+#define HDP_XDP_D2H_RSVD_25__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_25__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_26
+#define HDP_XDP_D2H_RSVD_26__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_26__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_27
+#define HDP_XDP_D2H_RSVD_27__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_27__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_28
+#define HDP_XDP_D2H_RSVD_28__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_28__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_29
+#define HDP_XDP_D2H_RSVD_29__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_29__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_30
+#define HDP_XDP_D2H_RSVD_30__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_30__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_31
+#define HDP_XDP_D2H_RSVD_31__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_31__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_32
+#define HDP_XDP_D2H_RSVD_32__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_32__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_33
+#define HDP_XDP_D2H_RSVD_33__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_33__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_34
+#define HDP_XDP_D2H_RSVD_34__RESERVED__SHIFT                                                                  0x0
+#define HDP_XDP_D2H_RSVD_34__RESERVED_MASK                                                                    0xFFFFFFFFL
+//HDP_XDP_DIRECT2HDP_LAST
+#define HDP_XDP_DIRECT2HDP_LAST__RESERVED__SHIFT                                                              0x0
+#define HDP_XDP_DIRECT2HDP_LAST__RESERVED_MASK                                                                0xFFFFFFFFL
+//HDP_XDP_P2P_BAR_CFG
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_ADDR_SIZE__SHIFT                                                     0x0
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_BAR_FROM__SHIFT                                                      0x4
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_ADDR_SIZE_MASK                                                       0x0000000FL
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_BAR_FROM_MASK                                                        0x00000030L
+//HDP_XDP_P2P_MBX_OFFSET
+#define HDP_XDP_P2P_MBX_OFFSET__P2P_MBX_OFFSET__SHIFT                                                         0x0
+#define HDP_XDP_P2P_MBX_OFFSET__P2P_MBX_OFFSET_MASK                                                           0x0001FFFFL
+//HDP_XDP_P2P_MBX_ADDR0
+#define HDP_XDP_P2P_MBX_ADDR0__VALID__SHIFT                                                                   0x0
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_35_19__SHIFT                                                              0x3
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_39_36__SHIFT                                                              0x14
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_47_40__SHIFT                                                              0x18
+#define HDP_XDP_P2P_MBX_ADDR0__VALID_MASK                                                                     0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_35_19_MASK                                                                0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_39_36_MASK                                                                0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_47_40_MASK                                                                0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR1
+#define HDP_XDP_P2P_MBX_ADDR1__VALID__SHIFT                                                                   0x0
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_35_19__SHIFT                                                              0x3
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_39_36__SHIFT                                                              0x14
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_47_40__SHIFT                                                              0x18
+#define HDP_XDP_P2P_MBX_ADDR1__VALID_MASK                                                                     0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_35_19_MASK                                                                0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_39_36_MASK                                                                0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_47_40_MASK                                                                0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR2
+#define HDP_XDP_P2P_MBX_ADDR2__VALID__SHIFT                                                                   0x0
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_35_19__SHIFT                                                              0x3
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_39_36__SHIFT                                                              0x14
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_47_40__SHIFT                                                              0x18
+#define HDP_XDP_P2P_MBX_ADDR2__VALID_MASK                                                                     0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_35_19_MASK                                                                0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_39_36_MASK                                                                0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_47_40_MASK                                                                0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR3
+#define HDP_XDP_P2P_MBX_ADDR3__VALID__SHIFT                                                                   0x0
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_35_19__SHIFT                                                              0x3
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_39_36__SHIFT                                                              0x14
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_47_40__SHIFT                                                              0x18
+#define HDP_XDP_P2P_MBX_ADDR3__VALID_MASK                                                                     0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_35_19_MASK                                                                0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_39_36_MASK                                                                0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_47_40_MASK                                                                0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR4
+#define HDP_XDP_P2P_MBX_ADDR4__VALID__SHIFT                                                                   0x0
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_35_19__SHIFT                                                              0x3
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_39_36__SHIFT                                                              0x14
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_47_40__SHIFT                                                              0x18
+#define HDP_XDP_P2P_MBX_ADDR4__VALID_MASK                                                                     0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_35_19_MASK                                                                0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_39_36_MASK                                                                0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_47_40_MASK                                                                0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR5
+#define HDP_XDP_P2P_MBX_ADDR5__VALID__SHIFT                                                                   0x0
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_35_19__SHIFT                                                              0x3
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_39_36__SHIFT                                                              0x14
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_47_40__SHIFT                                                              0x18
+#define HDP_XDP_P2P_MBX_ADDR5__VALID_MASK                                                                     0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_35_19_MASK                                                                0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_39_36_MASK                                                                0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_47_40_MASK                                                                0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR6
+#define HDP_XDP_P2P_MBX_ADDR6__VALID__SHIFT                                                                   0x0
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_35_19__SHIFT                                                              0x3
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_39_36__SHIFT                                                              0x14
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_47_40__SHIFT                                                              0x18
+#define HDP_XDP_P2P_MBX_ADDR6__VALID_MASK                                                                     0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_35_19_MASK                                                                0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_39_36_MASK                                                                0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_47_40_MASK                                                                0xFF000000L
+//HDP_XDP_HDP_MBX_MC_CFG
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_QOS__SHIFT                                           0x0
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SWAP__SHIFT                                          0x4
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_VMID__SHIFT                                          0x8
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_RO__SHIFT                                            0xc
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_GCC__SHIFT                                           0xd
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SNOOP__SHIFT                                         0xe
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_QOS_MASK                                             0x0000000FL
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SWAP_MASK                                            0x00000030L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_VMID_MASK                                            0x00000F00L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_RO_MASK                                              0x00001000L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_GCC_MASK                                             0x00002000L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SNOOP_MASK                                           0x00004000L
+//HDP_XDP_HDP_MC_CFG
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SNOOP_OVERRIDE__SHIFT                                      0x0
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_GCC_OVERRIDE__SHIFT                                        0x1
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_RO_OVERRIDE__SHIFT                                         0x2
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SNOOP__SHIFT                                               0x3
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SWAP__SHIFT                                                0x4
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_VMID__SHIFT                                                0x8
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_RO__SHIFT                                                  0xc
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_GCC__SHIFT                                                 0xd
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_XDP_HIGHER_PRI_THRESH__SHIFT                                           0xe
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SNOOP_OVERRIDE_MASK                                        0x00000001L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_GCC_OVERRIDE_MASK                                          0x00000002L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_RO_OVERRIDE_MASK                                           0x00000004L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SNOOP_MASK                                                 0x00000008L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SWAP_MASK                                                  0x00000030L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_VMID_MASK                                                  0x00000F00L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_RO_MASK                                                    0x00001000L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_GCC_MASK                                                   0x00002000L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_XDP_HIGHER_PRI_THRESH_MASK                                             0x000FC000L
+//HDP_XDP_HST_CFG
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_EN__SHIFT                                                         0x0
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER__SHIFT                                                      0x1
+#define HDP_XDP_HST_CFG__HST_CFG_WR_BURST_EN__SHIFT                                                           0x3
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_64B_EN__SHIFT                                                     0x4
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER_PRELOAD_CFG__SHIFT                                          0x5
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_EN_MASK                                                           0x00000001L
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER_MASK                                                        0x00000006L
+#define HDP_XDP_HST_CFG__HST_CFG_WR_BURST_EN_MASK                                                             0x00000008L
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_64B_EN_MASK                                                       0x00000010L
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER_PRELOAD_CFG_MASK                                            0x00000020L
+//HDP_XDP_HDP_IPH_CFG
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_SYS_FIFO_DEPTH_OVERRIDE__SHIFT                                       0x0
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_XDP_FIFO_DEPTH_OVERRIDE__SHIFT                                       0x6
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_INVERSE_PEER_TAG_MATCHING__SHIFT                                     0xc
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_P2P_RD_EN__SHIFT                                                     0xd
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_SYS_FIFO_DEPTH_OVERRIDE_MASK                                         0x0000003FL
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_XDP_FIFO_DEPTH_OVERRIDE_MASK                                         0x00000FC0L
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_INVERSE_PEER_TAG_MATCHING_MASK                                       0x00001000L
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_P2P_RD_EN_MASK                                                       0x00002000L
+//HDP_XDP_P2P_BAR0
+#define HDP_XDP_P2P_BAR0__ADDR__SHIFT                                                                         0x0
+#define HDP_XDP_P2P_BAR0__FLUSH__SHIFT                                                                        0x10
+#define HDP_XDP_P2P_BAR0__VALID__SHIFT                                                                        0x14
+#define HDP_XDP_P2P_BAR0__ADDR_MASK                                                                           0x0000FFFFL
+#define HDP_XDP_P2P_BAR0__FLUSH_MASK                                                                          0x000F0000L
+#define HDP_XDP_P2P_BAR0__VALID_MASK                                                                          0x00100000L
+//HDP_XDP_P2P_BAR1
+#define HDP_XDP_P2P_BAR1__ADDR__SHIFT                                                                         0x0
+#define HDP_XDP_P2P_BAR1__FLUSH__SHIFT                                                                        0x10
+#define HDP_XDP_P2P_BAR1__VALID__SHIFT                                                                        0x14
+#define HDP_XDP_P2P_BAR1__ADDR_MASK                                                                           0x0000FFFFL
+#define HDP_XDP_P2P_BAR1__FLUSH_MASK                                                                          0x000F0000L
+#define HDP_XDP_P2P_BAR1__VALID_MASK                                                                          0x00100000L
+//HDP_XDP_P2P_BAR2
+#define HDP_XDP_P2P_BAR2__ADDR__SHIFT                                                                         0x0
+#define HDP_XDP_P2P_BAR2__FLUSH__SHIFT                                                                        0x10
+#define HDP_XDP_P2P_BAR2__VALID__SHIFT                                                                        0x14
+#define HDP_XDP_P2P_BAR2__ADDR_MASK                                                                           0x0000FFFFL
+#define HDP_XDP_P2P_BAR2__FLUSH_MASK                                                                          0x000F0000L
+#define HDP_XDP_P2P_BAR2__VALID_MASK                                                                          0x00100000L
+//HDP_XDP_P2P_BAR3
+#define HDP_XDP_P2P_BAR3__ADDR__SHIFT                                                                         0x0
+#define HDP_XDP_P2P_BAR3__FLUSH__SHIFT                                                                        0x10
+#define HDP_XDP_P2P_BAR3__VALID__SHIFT                                                                        0x14
+#define HDP_XDP_P2P_BAR3__ADDR_MASK                                                                           0x0000FFFFL
+#define HDP_XDP_P2P_BAR3__FLUSH_MASK                                                                          0x000F0000L
+#define HDP_XDP_P2P_BAR3__VALID_MASK                                                                          0x00100000L
+//HDP_XDP_P2P_BAR4
+#define HDP_XDP_P2P_BAR4__ADDR__SHIFT                                                                         0x0
+#define HDP_XDP_P2P_BAR4__FLUSH__SHIFT                                                                        0x10
+#define HDP_XDP_P2P_BAR4__VALID__SHIFT                                                                        0x14
+#define HDP_XDP_P2P_BAR4__ADDR_MASK                                                                           0x0000FFFFL
+#define HDP_XDP_P2P_BAR4__FLUSH_MASK                                                                          0x000F0000L
+#define HDP_XDP_P2P_BAR4__VALID_MASK                                                                          0x00100000L
+//HDP_XDP_P2P_BAR5
+#define HDP_XDP_P2P_BAR5__ADDR__SHIFT                                                                         0x0
+#define HDP_XDP_P2P_BAR5__FLUSH__SHIFT                                                                        0x10
+#define HDP_XDP_P2P_BAR5__VALID__SHIFT                                                                        0x14
+#define HDP_XDP_P2P_BAR5__ADDR_MASK                                                                           0x0000FFFFL
+#define HDP_XDP_P2P_BAR5__FLUSH_MASK                                                                          0x000F0000L
+#define HDP_XDP_P2P_BAR5__VALID_MASK                                                                          0x00100000L
+//HDP_XDP_P2P_BAR6
+#define HDP_XDP_P2P_BAR6__ADDR__SHIFT                                                                         0x0
+#define HDP_XDP_P2P_BAR6__FLUSH__SHIFT                                                                        0x10
+#define HDP_XDP_P2P_BAR6__VALID__SHIFT                                                                        0x14
+#define HDP_XDP_P2P_BAR6__ADDR_MASK                                                                           0x0000FFFFL
+#define HDP_XDP_P2P_BAR6__FLUSH_MASK                                                                          0x000F0000L
+#define HDP_XDP_P2P_BAR6__VALID_MASK                                                                          0x00100000L
+//HDP_XDP_P2P_BAR7
+#define HDP_XDP_P2P_BAR7__ADDR__SHIFT                                                                         0x0
+#define HDP_XDP_P2P_BAR7__FLUSH__SHIFT                                                                        0x10
+#define HDP_XDP_P2P_BAR7__VALID__SHIFT                                                                        0x14
+#define HDP_XDP_P2P_BAR7__ADDR_MASK                                                                           0x0000FFFFL
+#define HDP_XDP_P2P_BAR7__FLUSH_MASK                                                                          0x000F0000L
+#define HDP_XDP_P2P_BAR7__VALID_MASK                                                                          0x00100000L
+//HDP_XDP_FLUSH_ARMED_STS
+#define HDP_XDP_FLUSH_ARMED_STS__FLUSH_ARMED_STS__SHIFT                                                       0x0
+#define HDP_XDP_FLUSH_ARMED_STS__FLUSH_ARMED_STS_MASK                                                         0xFFFFFFFFL
+//HDP_XDP_FLUSH_CNTR0_STS
+#define HDP_XDP_FLUSH_CNTR0_STS__FLUSH_CNTR0_STS__SHIFT                                                       0x0
+#define HDP_XDP_FLUSH_CNTR0_STS__FLUSH_CNTR0_STS_MASK                                                         0x03FFFFFFL
+//HDP_XDP_BUSY_STS
+#define HDP_XDP_BUSY_STS__BUSY_BITS__SHIFT                                                                    0x0
+#define HDP_XDP_BUSY_STS__BUSY_BITS_MASK                                                                      0x00FFFFFFL
+//HDP_XDP_STICKY
+#define HDP_XDP_STICKY__STICKY_STS__SHIFT                                                                     0x0
+#define HDP_XDP_STICKY__STICKY_W1C__SHIFT                                                                     0x10
+#define HDP_XDP_STICKY__STICKY_STS_MASK                                                                       0x0000FFFFL
+#define HDP_XDP_STICKY__STICKY_W1C_MASK                                                                       0xFFFF0000L
+//HDP_XDP_CHKN
+#define HDP_XDP_CHKN__CHKN_0_RSVD__SHIFT                                                                      0x0
+#define HDP_XDP_CHKN__CHKN_1_RSVD__SHIFT                                                                      0x8
+#define HDP_XDP_CHKN__CHKN_2_RSVD__SHIFT                                                                      0x10
+#define HDP_XDP_CHKN__CHKN_3_RSVD__SHIFT                                                                      0x18
+#define HDP_XDP_CHKN__CHKN_0_RSVD_MASK                                                                        0x000000FFL
+#define HDP_XDP_CHKN__CHKN_1_RSVD_MASK                                                                        0x0000FF00L
+#define HDP_XDP_CHKN__CHKN_2_RSVD_MASK                                                                        0x00FF0000L
+#define HDP_XDP_CHKN__CHKN_3_RSVD_MASK                                                                        0xFF000000L
+//HDP_XDP_BARS_ADDR_39_36
+#define HDP_XDP_BARS_ADDR_39_36__BAR0_ADDR_39_36__SHIFT                                                       0x0
+#define HDP_XDP_BARS_ADDR_39_36__BAR1_ADDR_39_36__SHIFT                                                       0x4
+#define HDP_XDP_BARS_ADDR_39_36__BAR2_ADDR_39_36__SHIFT                                                       0x8
+#define HDP_XDP_BARS_ADDR_39_36__BAR3_ADDR_39_36__SHIFT                                                       0xc
+#define HDP_XDP_BARS_ADDR_39_36__BAR4_ADDR_39_36__SHIFT                                                       0x10
+#define HDP_XDP_BARS_ADDR_39_36__BAR5_ADDR_39_36__SHIFT                                                       0x14
+#define HDP_XDP_BARS_ADDR_39_36__BAR6_ADDR_39_36__SHIFT                                                       0x18
+#define HDP_XDP_BARS_ADDR_39_36__BAR7_ADDR_39_36__SHIFT                                                       0x1c
+#define HDP_XDP_BARS_ADDR_39_36__BAR0_ADDR_39_36_MASK                                                         0x0000000FL
+#define HDP_XDP_BARS_ADDR_39_36__BAR1_ADDR_39_36_MASK                                                         0x000000F0L
+#define HDP_XDP_BARS_ADDR_39_36__BAR2_ADDR_39_36_MASK                                                         0x00000F00L
+#define HDP_XDP_BARS_ADDR_39_36__BAR3_ADDR_39_36_MASK                                                         0x0000F000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR4_ADDR_39_36_MASK                                                         0x000F0000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR5_ADDR_39_36_MASK                                                         0x00F00000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR6_ADDR_39_36_MASK                                                         0x0F000000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR7_ADDR_39_36_MASK                                                         0xF0000000L
+//HDP_XDP_MC_VM_FB_LOCATION_BASE
+#define HDP_XDP_MC_VM_FB_LOCATION_BASE__FB_BASE__SHIFT                                                        0x0
+#define HDP_XDP_MC_VM_FB_LOCATION_BASE__FB_BASE_MASK                                                          0x03FFFFFFL
+//HDP_XDP_GPU_IOV_VIOLATION_LOG
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                0x0
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                       0x1
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                         0x2
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__OPCODE__SHIFT                                                          0x12
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                              0x13
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VFID__SHIFT                                                            0x14
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                  0x00000001L
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                         0x00000002L
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                           0x0003FFFCL
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__OPCODE_MASK                                                            0x00040000L
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VF_MASK                                                                0x00080000L
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VFID_MASK                                                              0x00F00000L
+//HDP_XDP_GPU_IOV_VIOLATION_LOG2
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID__SHIFT                                                   0x0
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID_MASK                                                     0x000003FFL
+//HDP_XDP_MMHUB_ERROR
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_01__SHIFT                                                              0x1
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_10__SHIFT                                                              0x2
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_11__SHIFT                                                              0x3
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_FED__SHIFT                                                             0x4
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_01__SHIFT                                                         0x5
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_10__SHIFT                                                         0x6
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_11__SHIFT                                                         0x7
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_01__SHIFT                                                              0x9
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_10__SHIFT                                                              0xa
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_11__SHIFT                                                              0xb
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_FED__SHIFT                                                             0xc
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_01__SHIFT                                                         0xd
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_10__SHIFT                                                         0xe
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_11__SHIFT                                                         0xf
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_01__SHIFT                                                              0x11
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_10__SHIFT                                                              0x12
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_11__SHIFT                                                              0x13
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_01__SHIFT                                                         0x15
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_10__SHIFT                                                         0x16
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_11__SHIFT                                                         0x17
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_01_MASK                                                                0x00000002L
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_10_MASK                                                                0x00000004L
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_11_MASK                                                                0x00000008L
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_FED_MASK                                                               0x00000010L
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_01_MASK                                                           0x00000020L
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_10_MASK                                                           0x00000040L
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_11_MASK                                                           0x00000080L
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_01_MASK                                                                0x00000200L
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_10_MASK                                                                0x00000400L
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_11_MASK                                                                0x00000800L
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_FED_MASK                                                               0x00001000L
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_01_MASK                                                           0x00002000L
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_10_MASK                                                           0x00004000L
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_11_MASK                                                           0x00008000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_01_MASK                                                                0x00020000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_10_MASK                                                                0x00040000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_11_MASK                                                                0x00080000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_01_MASK                                                           0x00200000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_10_MASK                                                           0x00400000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_11_MASK                                                           0x00800000L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_8_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_8_0_offset.h
new file mode 100644
index 000000000000..8bcc81f2dfc0
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_8_0_offset.h
@@ -0,0 +1,3314 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _mmhub_1_8_0_OFFSET_HEADER
+#define _mmhub_1_8_0_OFFSET_HEADER
+
+
+
+// addressBlock: aid_mmhub_dagb_dagbdec0
+// base address: 0x60000
+#define regDAGB0_RDCLI0                                                                                 0x0000
+#define regDAGB0_RDCLI0_BASE_IDX                                                                        0
+#define regDAGB0_RDCLI1                                                                                 0x0001
+#define regDAGB0_RDCLI1_BASE_IDX                                                                        0
+#define regDAGB0_RDCLI2                                                                                 0x0002
+#define regDAGB0_RDCLI2_BASE_IDX                                                                        0
+#define regDAGB0_RDCLI3                                                                                 0x0003
+#define regDAGB0_RDCLI3_BASE_IDX                                                                        0
+#define regDAGB0_RDCLI4                                                                                 0x0004
+#define regDAGB0_RDCLI4_BASE_IDX                                                                        0
+#define regDAGB0_RDCLI5                                                                                 0x0005
+#define regDAGB0_RDCLI5_BASE_IDX                                                                        0
+#define regDAGB0_RDCLI6                                                                                 0x0006
+#define regDAGB0_RDCLI6_BASE_IDX                                                                        0
+#define regDAGB0_RDCLI7                                                                                 0x0007
+#define regDAGB0_RDCLI7_BASE_IDX                                                                        0
+#define regDAGB0_RDCLI8                                                                                 0x0008
+#define regDAGB0_RDCLI8_BASE_IDX                                                                        0
+#define regDAGB0_RDCLI9                                                                                 0x0009
+#define regDAGB0_RDCLI9_BASE_IDX                                                                        0
+#define regDAGB0_RDCLI10                                                                                0x000a
+#define regDAGB0_RDCLI10_BASE_IDX                                                                       0
+#define regDAGB0_RDCLI11                                                                                0x000b
+#define regDAGB0_RDCLI11_BASE_IDX                                                                       0
+#define regDAGB0_RDCLI12                                                                                0x000c
+#define regDAGB0_RDCLI12_BASE_IDX                                                                       0
+#define regDAGB0_RDCLI13                                                                                0x000d
+#define regDAGB0_RDCLI13_BASE_IDX                                                                       0
+#define regDAGB0_RDCLI14                                                                                0x000e
+#define regDAGB0_RDCLI14_BASE_IDX                                                                       0
+#define regDAGB0_RDCLI15                                                                                0x000f
+#define regDAGB0_RDCLI15_BASE_IDX                                                                       0
+#define regDAGB0_RD_CNTL                                                                                0x0010
+#define regDAGB0_RD_CNTL_BASE_IDX                                                                       0
+#define regDAGB0_RD_GMI_CNTL                                                                            0x0011
+#define regDAGB0_RD_GMI_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_RD_ADDR_DAGB                                                                           0x0012
+#define regDAGB0_RD_ADDR_DAGB_BASE_IDX                                                                  0
+#define regDAGB0_RD_OUTPUT_DAGB_MAX_BURST                                                               0x0013
+#define regDAGB0_RD_OUTPUT_DAGB_MAX_BURST_BASE_IDX                                                      0
+#define regDAGB0_RD_OUTPUT_DAGB_LAZY_TIMER                                                              0x0014
+#define regDAGB0_RD_OUTPUT_DAGB_LAZY_TIMER_BASE_IDX                                                     0
+#define regDAGB0_RD_CGTT_CLK_CTRL                                                                       0x0015
+#define regDAGB0_RD_CGTT_CLK_CTRL_BASE_IDX                                                              0
+#define regDAGB0_L1TLB_RD_CGTT_CLK_CTRL                                                                 0x0016
+#define regDAGB0_L1TLB_RD_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB0_ATCVM_RD_CGTT_CLK_CTRL                                                                 0x0017
+#define regDAGB0_ATCVM_RD_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB0_RD_ADDR_DAGB_MAX_BURST0                                                                0x0018
+#define regDAGB0_RD_ADDR_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB0_RD_ADDR_DAGB_LAZY_TIMER0                                                               0x0019
+#define regDAGB0_RD_ADDR_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB0_RD_ADDR_DAGB_MAX_BURST1                                                                0x001a
+#define regDAGB0_RD_ADDR_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB0_RD_ADDR_DAGB_LAZY_TIMER1                                                               0x001b
+#define regDAGB0_RD_ADDR_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB0_RD_VC0_CNTL                                                                            0x001c
+#define regDAGB0_RD_VC0_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_RD_VC1_CNTL                                                                            0x001d
+#define regDAGB0_RD_VC1_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_RD_VC2_CNTL                                                                            0x001e
+#define regDAGB0_RD_VC2_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_RD_VC3_CNTL                                                                            0x001f
+#define regDAGB0_RD_VC3_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_RD_VC4_CNTL                                                                            0x0020
+#define regDAGB0_RD_VC4_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_RD_VC5_CNTL                                                                            0x0021
+#define regDAGB0_RD_VC5_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_RD_VC6_CNTL                                                                            0x0022
+#define regDAGB0_RD_VC6_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_RD_VC7_CNTL                                                                            0x0023
+#define regDAGB0_RD_VC7_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_RD_CNTL_MISC                                                                           0x0024
+#define regDAGB0_RD_CNTL_MISC_BASE_IDX                                                                  0
+#define regDAGB0_RD_TLB_CREDIT                                                                          0x0025
+#define regDAGB0_RD_TLB_CREDIT_BASE_IDX                                                                 0
+#define regDAGB0_RD_RDRET_CREDIT_CNTL                                                                   0x0026
+#define regDAGB0_RD_RDRET_CREDIT_CNTL_BASE_IDX                                                          0
+#define regDAGB0_RD_RDRET_CREDIT_CNTL2                                                                  0x0027
+#define regDAGB0_RD_RDRET_CREDIT_CNTL2_BASE_IDX                                                         0
+#define regDAGB0_RDCLI_ASK_PENDING                                                                      0x0028
+#define regDAGB0_RDCLI_ASK_PENDING_BASE_IDX                                                             0
+#define regDAGB0_RDCLI_GO_PENDING                                                                       0x0029
+#define regDAGB0_RDCLI_GO_PENDING_BASE_IDX                                                              0
+#define regDAGB0_RDCLI_GBLSEND_PENDING                                                                  0x002a
+#define regDAGB0_RDCLI_GBLSEND_PENDING_BASE_IDX                                                         0
+#define regDAGB0_RDCLI_TLB_PENDING                                                                      0x002b
+#define regDAGB0_RDCLI_TLB_PENDING_BASE_IDX                                                             0
+#define regDAGB0_RDCLI_OARB_PENDING                                                                     0x002c
+#define regDAGB0_RDCLI_OARB_PENDING_BASE_IDX                                                            0
+#define regDAGB0_RDCLI_OSD_PENDING                                                                      0x002d
+#define regDAGB0_RDCLI_OSD_PENDING_BASE_IDX                                                             0
+#define regDAGB0_RDCLI_NOALLOC_OVERRIDE                                                                 0x002e
+#define regDAGB0_RDCLI_NOALLOC_OVERRIDE_BASE_IDX                                                        0
+#define regDAGB0_RDCLI_NOALLOC_OVERRIDE_VALUE                                                           0x002f
+#define regDAGB0_RDCLI_NOALLOC_OVERRIDE_VALUE_BASE_IDX                                                  0
+#define regDAGB0_WRCLI0                                                                                 0x0030
+#define regDAGB0_WRCLI0_BASE_IDX                                                                        0
+#define regDAGB0_WRCLI1                                                                                 0x0031
+#define regDAGB0_WRCLI1_BASE_IDX                                                                        0
+#define regDAGB0_WRCLI2                                                                                 0x0032
+#define regDAGB0_WRCLI2_BASE_IDX                                                                        0
+#define regDAGB0_WRCLI3                                                                                 0x0033
+#define regDAGB0_WRCLI3_BASE_IDX                                                                        0
+#define regDAGB0_WRCLI4                                                                                 0x0034
+#define regDAGB0_WRCLI4_BASE_IDX                                                                        0
+#define regDAGB0_WRCLI5                                                                                 0x0035
+#define regDAGB0_WRCLI5_BASE_IDX                                                                        0
+#define regDAGB0_WRCLI6                                                                                 0x0036
+#define regDAGB0_WRCLI6_BASE_IDX                                                                        0
+#define regDAGB0_WRCLI7                                                                                 0x0037
+#define regDAGB0_WRCLI7_BASE_IDX                                                                        0
+#define regDAGB0_WRCLI8                                                                                 0x0038
+#define regDAGB0_WRCLI8_BASE_IDX                                                                        0
+#define regDAGB0_WRCLI9                                                                                 0x0039
+#define regDAGB0_WRCLI9_BASE_IDX                                                                        0
+#define regDAGB0_WRCLI10                                                                                0x003a
+#define regDAGB0_WRCLI10_BASE_IDX                                                                       0
+#define regDAGB0_WRCLI11                                                                                0x003b
+#define regDAGB0_WRCLI11_BASE_IDX                                                                       0
+#define regDAGB0_WRCLI12                                                                                0x003c
+#define regDAGB0_WRCLI12_BASE_IDX                                                                       0
+#define regDAGB0_WRCLI13                                                                                0x003d
+#define regDAGB0_WRCLI13_BASE_IDX                                                                       0
+#define regDAGB0_WRCLI14                                                                                0x003e
+#define regDAGB0_WRCLI14_BASE_IDX                                                                       0
+#define regDAGB0_WRCLI15                                                                                0x003f
+#define regDAGB0_WRCLI15_BASE_IDX                                                                       0
+#define regDAGB0_WR_CNTL                                                                                0x0040
+#define regDAGB0_WR_CNTL_BASE_IDX                                                                       0
+#define regDAGB0_WR_GMI_CNTL                                                                            0x0041
+#define regDAGB0_WR_GMI_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_WR_ADDR_DAGB                                                                           0x0042
+#define regDAGB0_WR_ADDR_DAGB_BASE_IDX                                                                  0
+#define regDAGB0_WR_OUTPUT_DAGB_MAX_BURST                                                               0x0043
+#define regDAGB0_WR_OUTPUT_DAGB_MAX_BURST_BASE_IDX                                                      0
+#define regDAGB0_WR_OUTPUT_DAGB_LAZY_TIMER                                                              0x0044
+#define regDAGB0_WR_OUTPUT_DAGB_LAZY_TIMER_BASE_IDX                                                     0
+#define regDAGB0_WR_CGTT_CLK_CTRL                                                                       0x0045
+#define regDAGB0_WR_CGTT_CLK_CTRL_BASE_IDX                                                              0
+#define regDAGB0_L1TLB_WR_CGTT_CLK_CTRL                                                                 0x0046
+#define regDAGB0_L1TLB_WR_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB0_ATCVM_WR_CGTT_CLK_CTRL                                                                 0x0047
+#define regDAGB0_ATCVM_WR_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB0_WR_ADDR_DAGB_MAX_BURST0                                                                0x0048
+#define regDAGB0_WR_ADDR_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB0_WR_ADDR_DAGB_LAZY_TIMER0                                                               0x0049
+#define regDAGB0_WR_ADDR_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB0_WR_ADDR_DAGB_MAX_BURST1                                                                0x004a
+#define regDAGB0_WR_ADDR_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB0_WR_ADDR_DAGB_LAZY_TIMER1                                                               0x004b
+#define regDAGB0_WR_ADDR_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB0_WR_DATA_DAGB                                                                           0x004c
+#define regDAGB0_WR_DATA_DAGB_BASE_IDX                                                                  0
+#define regDAGB0_WR_DATA_DAGB_MAX_BURST0                                                                0x004d
+#define regDAGB0_WR_DATA_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB0_WR_DATA_DAGB_LAZY_TIMER0                                                               0x004e
+#define regDAGB0_WR_DATA_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB0_WR_DATA_DAGB_MAX_BURST1                                                                0x004f
+#define regDAGB0_WR_DATA_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB0_WR_DATA_DAGB_LAZY_TIMER1                                                               0x0050
+#define regDAGB0_WR_DATA_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB0_WR_VC0_CNTL                                                                            0x0051
+#define regDAGB0_WR_VC0_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_WR_VC1_CNTL                                                                            0x0052
+#define regDAGB0_WR_VC1_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_WR_VC2_CNTL                                                                            0x0053
+#define regDAGB0_WR_VC2_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_WR_VC3_CNTL                                                                            0x0054
+#define regDAGB0_WR_VC3_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_WR_VC4_CNTL                                                                            0x0055
+#define regDAGB0_WR_VC4_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_WR_VC5_CNTL                                                                            0x0056
+#define regDAGB0_WR_VC5_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_WR_VC6_CNTL                                                                            0x0057
+#define regDAGB0_WR_VC6_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_WR_VC7_CNTL                                                                            0x0058
+#define regDAGB0_WR_VC7_CNTL_BASE_IDX                                                                   0
+#define regDAGB0_WR_CNTL_MISC                                                                           0x0059
+#define regDAGB0_WR_CNTL_MISC_BASE_IDX                                                                  0
+#define regDAGB0_WR_TLB_CREDIT                                                                          0x005a
+#define regDAGB0_WR_TLB_CREDIT_BASE_IDX                                                                 0
+#define regDAGB0_WR_DATA_CREDIT                                                                         0x005b
+#define regDAGB0_WR_DATA_CREDIT_BASE_IDX                                                                0
+#define regDAGB0_WR_MISC_CREDIT                                                                         0x005c
+#define regDAGB0_WR_MISC_CREDIT_BASE_IDX                                                                0
+#define regDAGB0_WR_OSD_CREDIT_CNTL1                                                                    0x005d
+#define regDAGB0_WR_OSD_CREDIT_CNTL1_BASE_IDX                                                           0
+#define regDAGB0_WR_OSD_CREDIT_CNTL2                                                                    0x005e
+#define regDAGB0_WR_OSD_CREDIT_CNTL2_BASE_IDX                                                           0
+#define regDAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1                                                            0x005f
+#define regDAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1_BASE_IDX                                                   0
+#define regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE                                                               0x0060
+#define regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX                                                      0
+#define regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE                                                         0x0061
+#define regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX                                                0
+#define regDAGB0_WRCLI_ASK_PENDING                                                                      0x0062
+#define regDAGB0_WRCLI_ASK_PENDING_BASE_IDX                                                             0
+#define regDAGB0_WRCLI_GO_PENDING                                                                       0x0063
+#define regDAGB0_WRCLI_GO_PENDING_BASE_IDX                                                              0
+#define regDAGB0_WRCLI_GBLSEND_PENDING                                                                  0x0064
+#define regDAGB0_WRCLI_GBLSEND_PENDING_BASE_IDX                                                         0
+#define regDAGB0_WRCLI_TLB_PENDING                                                                      0x0065
+#define regDAGB0_WRCLI_TLB_PENDING_BASE_IDX                                                             0
+#define regDAGB0_WRCLI_OARB_PENDING                                                                     0x0066
+#define regDAGB0_WRCLI_OARB_PENDING_BASE_IDX                                                            0
+#define regDAGB0_WRCLI_OSD_PENDING                                                                      0x0067
+#define regDAGB0_WRCLI_OSD_PENDING_BASE_IDX                                                             0
+#define regDAGB0_WRCLI_DBUS_ASK_PENDING                                                                 0x0068
+#define regDAGB0_WRCLI_DBUS_ASK_PENDING_BASE_IDX                                                        0
+#define regDAGB0_WRCLI_DBUS_GO_PENDING                                                                  0x0069
+#define regDAGB0_WRCLI_DBUS_GO_PENDING_BASE_IDX                                                         0
+#define regDAGB0_WRCLI_NOALLOC_OVERRIDE                                                                 0x006a
+#define regDAGB0_WRCLI_NOALLOC_OVERRIDE_BASE_IDX                                                        0
+#define regDAGB0_WRCLI_NOALLOC_OVERRIDE_VALUE                                                           0x006b
+#define regDAGB0_WRCLI_NOALLOC_OVERRIDE_VALUE_BASE_IDX                                                  0
+#define regDAGB0_DAGB_DLY                                                                               0x006c
+#define regDAGB0_DAGB_DLY_BASE_IDX                                                                      0
+#define regDAGB0_CNTL_MISC                                                                              0x006d
+#define regDAGB0_CNTL_MISC_BASE_IDX                                                                     0
+#define regDAGB0_CNTL_MISC2                                                                             0x006e
+#define regDAGB0_CNTL_MISC2_BASE_IDX                                                                    0
+#define regDAGB0_FATAL_ERROR_CNTL                                                                       0x006f
+#define regDAGB0_FATAL_ERROR_CNTL_BASE_IDX                                                              0
+#define regDAGB0_FATAL_ERROR_CLEAR                                                                      0x0070
+#define regDAGB0_FATAL_ERROR_CLEAR_BASE_IDX                                                             0
+#define regDAGB0_FATAL_ERROR_STATUS0                                                                    0x0071
+#define regDAGB0_FATAL_ERROR_STATUS0_BASE_IDX                                                           0
+#define regDAGB0_FATAL_ERROR_STATUS1                                                                    0x0072
+#define regDAGB0_FATAL_ERROR_STATUS1_BASE_IDX                                                           0
+#define regDAGB0_FATAL_ERROR_STATUS2                                                                    0x0073
+#define regDAGB0_FATAL_ERROR_STATUS2_BASE_IDX                                                           0
+#define regDAGB0_FATAL_ERROR_STATUS3                                                                    0x0074
+#define regDAGB0_FATAL_ERROR_STATUS3_BASE_IDX                                                           0
+#define regDAGB0_FIFO_EMPTY                                                                             0x0075
+#define regDAGB0_FIFO_EMPTY_BASE_IDX                                                                    0
+#define regDAGB0_FIFO_FULL                                                                              0x0076
+#define regDAGB0_FIFO_FULL_BASE_IDX                                                                     0
+#define regDAGB0_WR_CREDITS_FULL                                                                        0x0077
+#define regDAGB0_WR_CREDITS_FULL_BASE_IDX                                                               0
+#define regDAGB0_RD_CREDITS_FULL                                                                        0x0078
+#define regDAGB0_RD_CREDITS_FULL_BASE_IDX                                                               0
+#define regDAGB0_PERFCOUNTER_LO                                                                         0x0079
+#define regDAGB0_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regDAGB0_PERFCOUNTER_HI                                                                         0x007a
+#define regDAGB0_PERFCOUNTER_HI_BASE_IDX                                                                0
+#define regDAGB0_PERFCOUNTER0_CFG                                                                       0x007b
+#define regDAGB0_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regDAGB0_PERFCOUNTER1_CFG                                                                       0x007c
+#define regDAGB0_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regDAGB0_PERFCOUNTER2_CFG                                                                       0x007d
+#define regDAGB0_PERFCOUNTER2_CFG_BASE_IDX                                                              0
+#define regDAGB0_PERFCOUNTER_RSLT_CNTL                                                                  0x007e
+#define regDAGB0_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+#define regDAGB0_L1TLB_REG_RW                                                                           0x007f
+#define regDAGB0_L1TLB_REG_RW_BASE_IDX                                                                  0
+
+
+// addressBlock: aid_mmhub_dagb_dagbdec1
+// base address: 0x60200
+#define regDAGB1_RDCLI0                                                                                 0x0080
+#define regDAGB1_RDCLI0_BASE_IDX                                                                        0
+#define regDAGB1_RDCLI1                                                                                 0x0081
+#define regDAGB1_RDCLI1_BASE_IDX                                                                        0
+#define regDAGB1_RDCLI2                                                                                 0x0082
+#define regDAGB1_RDCLI2_BASE_IDX                                                                        0
+#define regDAGB1_RDCLI3                                                                                 0x0083
+#define regDAGB1_RDCLI3_BASE_IDX                                                                        0
+#define regDAGB1_RDCLI4                                                                                 0x0084
+#define regDAGB1_RDCLI4_BASE_IDX                                                                        0
+#define regDAGB1_RDCLI5                                                                                 0x0085
+#define regDAGB1_RDCLI5_BASE_IDX                                                                        0
+#define regDAGB1_RDCLI6                                                                                 0x0086
+#define regDAGB1_RDCLI6_BASE_IDX                                                                        0
+#define regDAGB1_RDCLI7                                                                                 0x0087
+#define regDAGB1_RDCLI7_BASE_IDX                                                                        0
+#define regDAGB1_RDCLI8                                                                                 0x0088
+#define regDAGB1_RDCLI8_BASE_IDX                                                                        0
+#define regDAGB1_RDCLI9                                                                                 0x0089
+#define regDAGB1_RDCLI9_BASE_IDX                                                                        0
+#define regDAGB1_RDCLI10                                                                                0x008a
+#define regDAGB1_RDCLI10_BASE_IDX                                                                       0
+#define regDAGB1_RDCLI11                                                                                0x008b
+#define regDAGB1_RDCLI11_BASE_IDX                                                                       0
+#define regDAGB1_RDCLI12                                                                                0x008c
+#define regDAGB1_RDCLI12_BASE_IDX                                                                       0
+#define regDAGB1_RDCLI13                                                                                0x008d
+#define regDAGB1_RDCLI13_BASE_IDX                                                                       0
+#define regDAGB1_RDCLI14                                                                                0x008e
+#define regDAGB1_RDCLI14_BASE_IDX                                                                       0
+#define regDAGB1_RDCLI15                                                                                0x008f
+#define regDAGB1_RDCLI15_BASE_IDX                                                                       0
+#define regDAGB1_RD_CNTL                                                                                0x0090
+#define regDAGB1_RD_CNTL_BASE_IDX                                                                       0
+#define regDAGB1_RD_GMI_CNTL                                                                            0x0091
+#define regDAGB1_RD_GMI_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_RD_ADDR_DAGB                                                                           0x0092
+#define regDAGB1_RD_ADDR_DAGB_BASE_IDX                                                                  0
+#define regDAGB1_RD_OUTPUT_DAGB_MAX_BURST                                                               0x0093
+#define regDAGB1_RD_OUTPUT_DAGB_MAX_BURST_BASE_IDX                                                      0
+#define regDAGB1_RD_OUTPUT_DAGB_LAZY_TIMER                                                              0x0094
+#define regDAGB1_RD_OUTPUT_DAGB_LAZY_TIMER_BASE_IDX                                                     0
+#define regDAGB1_RD_CGTT_CLK_CTRL                                                                       0x0095
+#define regDAGB1_RD_CGTT_CLK_CTRL_BASE_IDX                                                              0
+#define regDAGB1_L1TLB_RD_CGTT_CLK_CTRL                                                                 0x0096
+#define regDAGB1_L1TLB_RD_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB1_ATCVM_RD_CGTT_CLK_CTRL                                                                 0x0097
+#define regDAGB1_ATCVM_RD_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB1_RD_ADDR_DAGB_MAX_BURST0                                                                0x0098
+#define regDAGB1_RD_ADDR_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB1_RD_ADDR_DAGB_LAZY_TIMER0                                                               0x0099
+#define regDAGB1_RD_ADDR_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB1_RD_ADDR_DAGB_MAX_BURST1                                                                0x009a
+#define regDAGB1_RD_ADDR_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB1_RD_ADDR_DAGB_LAZY_TIMER1                                                               0x009b
+#define regDAGB1_RD_ADDR_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB1_RD_VC0_CNTL                                                                            0x009c
+#define regDAGB1_RD_VC0_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_RD_VC1_CNTL                                                                            0x009d
+#define regDAGB1_RD_VC1_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_RD_VC2_CNTL                                                                            0x009e
+#define regDAGB1_RD_VC2_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_RD_VC3_CNTL                                                                            0x009f
+#define regDAGB1_RD_VC3_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_RD_VC4_CNTL                                                                            0x00a0
+#define regDAGB1_RD_VC4_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_RD_VC5_CNTL                                                                            0x00a1
+#define regDAGB1_RD_VC5_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_RD_VC6_CNTL                                                                            0x00a2
+#define regDAGB1_RD_VC6_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_RD_VC7_CNTL                                                                            0x00a3
+#define regDAGB1_RD_VC7_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_RD_CNTL_MISC                                                                           0x00a4
+#define regDAGB1_RD_CNTL_MISC_BASE_IDX                                                                  0
+#define regDAGB1_RD_TLB_CREDIT                                                                          0x00a5
+#define regDAGB1_RD_TLB_CREDIT_BASE_IDX                                                                 0
+#define regDAGB1_RD_RDRET_CREDIT_CNTL                                                                   0x00a6
+#define regDAGB1_RD_RDRET_CREDIT_CNTL_BASE_IDX                                                          0
+#define regDAGB1_RD_RDRET_CREDIT_CNTL2                                                                  0x00a7
+#define regDAGB1_RD_RDRET_CREDIT_CNTL2_BASE_IDX                                                         0
+#define regDAGB1_RDCLI_ASK_PENDING                                                                      0x00a8
+#define regDAGB1_RDCLI_ASK_PENDING_BASE_IDX                                                             0
+#define regDAGB1_RDCLI_GO_PENDING                                                                       0x00a9
+#define regDAGB1_RDCLI_GO_PENDING_BASE_IDX                                                              0
+#define regDAGB1_RDCLI_GBLSEND_PENDING                                                                  0x00aa
+#define regDAGB1_RDCLI_GBLSEND_PENDING_BASE_IDX                                                         0
+#define regDAGB1_RDCLI_TLB_PENDING                                                                      0x00ab
+#define regDAGB1_RDCLI_TLB_PENDING_BASE_IDX                                                             0
+#define regDAGB1_RDCLI_OARB_PENDING                                                                     0x00ac
+#define regDAGB1_RDCLI_OARB_PENDING_BASE_IDX                                                            0
+#define regDAGB1_RDCLI_OSD_PENDING                                                                      0x00ad
+#define regDAGB1_RDCLI_OSD_PENDING_BASE_IDX                                                             0
+#define regDAGB1_RDCLI_NOALLOC_OVERRIDE                                                                 0x00ae
+#define regDAGB1_RDCLI_NOALLOC_OVERRIDE_BASE_IDX                                                        0
+#define regDAGB1_RDCLI_NOALLOC_OVERRIDE_VALUE                                                           0x00af
+#define regDAGB1_RDCLI_NOALLOC_OVERRIDE_VALUE_BASE_IDX                                                  0
+#define regDAGB1_WRCLI0                                                                                 0x00b0
+#define regDAGB1_WRCLI0_BASE_IDX                                                                        0
+#define regDAGB1_WRCLI1                                                                                 0x00b1
+#define regDAGB1_WRCLI1_BASE_IDX                                                                        0
+#define regDAGB1_WRCLI2                                                                                 0x00b2
+#define regDAGB1_WRCLI2_BASE_IDX                                                                        0
+#define regDAGB1_WRCLI3                                                                                 0x00b3
+#define regDAGB1_WRCLI3_BASE_IDX                                                                        0
+#define regDAGB1_WRCLI4                                                                                 0x00b4
+#define regDAGB1_WRCLI4_BASE_IDX                                                                        0
+#define regDAGB1_WRCLI5                                                                                 0x00b5
+#define regDAGB1_WRCLI5_BASE_IDX                                                                        0
+#define regDAGB1_WRCLI6                                                                                 0x00b6
+#define regDAGB1_WRCLI6_BASE_IDX                                                                        0
+#define regDAGB1_WRCLI7                                                                                 0x00b7
+#define regDAGB1_WRCLI7_BASE_IDX                                                                        0
+#define regDAGB1_WRCLI8                                                                                 0x00b8
+#define regDAGB1_WRCLI8_BASE_IDX                                                                        0
+#define regDAGB1_WRCLI9                                                                                 0x00b9
+#define regDAGB1_WRCLI9_BASE_IDX                                                                        0
+#define regDAGB1_WRCLI10                                                                                0x00ba
+#define regDAGB1_WRCLI10_BASE_IDX                                                                       0
+#define regDAGB1_WRCLI11                                                                                0x00bb
+#define regDAGB1_WRCLI11_BASE_IDX                                                                       0
+#define regDAGB1_WRCLI12                                                                                0x00bc
+#define regDAGB1_WRCLI12_BASE_IDX                                                                       0
+#define regDAGB1_WRCLI13                                                                                0x00bd
+#define regDAGB1_WRCLI13_BASE_IDX                                                                       0
+#define regDAGB1_WRCLI14                                                                                0x00be
+#define regDAGB1_WRCLI14_BASE_IDX                                                                       0
+#define regDAGB1_WRCLI15                                                                                0x00bf
+#define regDAGB1_WRCLI15_BASE_IDX                                                                       0
+#define regDAGB1_WR_CNTL                                                                                0x00c0
+#define regDAGB1_WR_CNTL_BASE_IDX                                                                       0
+#define regDAGB1_WR_GMI_CNTL                                                                            0x00c1
+#define regDAGB1_WR_GMI_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_WR_ADDR_DAGB                                                                           0x00c2
+#define regDAGB1_WR_ADDR_DAGB_BASE_IDX                                                                  0
+#define regDAGB1_WR_OUTPUT_DAGB_MAX_BURST                                                               0x00c3
+#define regDAGB1_WR_OUTPUT_DAGB_MAX_BURST_BASE_IDX                                                      0
+#define regDAGB1_WR_OUTPUT_DAGB_LAZY_TIMER                                                              0x00c4
+#define regDAGB1_WR_OUTPUT_DAGB_LAZY_TIMER_BASE_IDX                                                     0
+#define regDAGB1_WR_CGTT_CLK_CTRL                                                                       0x00c5
+#define regDAGB1_WR_CGTT_CLK_CTRL_BASE_IDX                                                              0
+#define regDAGB1_L1TLB_WR_CGTT_CLK_CTRL                                                                 0x00c6
+#define regDAGB1_L1TLB_WR_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB1_ATCVM_WR_CGTT_CLK_CTRL                                                                 0x00c7
+#define regDAGB1_ATCVM_WR_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB1_WR_ADDR_DAGB_MAX_BURST0                                                                0x00c8
+#define regDAGB1_WR_ADDR_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB1_WR_ADDR_DAGB_LAZY_TIMER0                                                               0x00c9
+#define regDAGB1_WR_ADDR_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB1_WR_ADDR_DAGB_MAX_BURST1                                                                0x00ca
+#define regDAGB1_WR_ADDR_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB1_WR_ADDR_DAGB_LAZY_TIMER1                                                               0x00cb
+#define regDAGB1_WR_ADDR_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB1_WR_DATA_DAGB                                                                           0x00cc
+#define regDAGB1_WR_DATA_DAGB_BASE_IDX                                                                  0
+#define regDAGB1_WR_DATA_DAGB_MAX_BURST0                                                                0x00cd
+#define regDAGB1_WR_DATA_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB1_WR_DATA_DAGB_LAZY_TIMER0                                                               0x00ce
+#define regDAGB1_WR_DATA_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB1_WR_DATA_DAGB_MAX_BURST1                                                                0x00cf
+#define regDAGB1_WR_DATA_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB1_WR_DATA_DAGB_LAZY_TIMER1                                                               0x00d0
+#define regDAGB1_WR_DATA_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB1_WR_VC0_CNTL                                                                            0x00d1
+#define regDAGB1_WR_VC0_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_WR_VC1_CNTL                                                                            0x00d2
+#define regDAGB1_WR_VC1_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_WR_VC2_CNTL                                                                            0x00d3
+#define regDAGB1_WR_VC2_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_WR_VC3_CNTL                                                                            0x00d4
+#define regDAGB1_WR_VC3_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_WR_VC4_CNTL                                                                            0x00d5
+#define regDAGB1_WR_VC4_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_WR_VC5_CNTL                                                                            0x00d6
+#define regDAGB1_WR_VC5_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_WR_VC6_CNTL                                                                            0x00d7
+#define regDAGB1_WR_VC6_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_WR_VC7_CNTL                                                                            0x00d8
+#define regDAGB1_WR_VC7_CNTL_BASE_IDX                                                                   0
+#define regDAGB1_WR_CNTL_MISC                                                                           0x00d9
+#define regDAGB1_WR_CNTL_MISC_BASE_IDX                                                                  0
+#define regDAGB1_WR_TLB_CREDIT                                                                          0x00da
+#define regDAGB1_WR_TLB_CREDIT_BASE_IDX                                                                 0
+#define regDAGB1_WR_DATA_CREDIT                                                                         0x00db
+#define regDAGB1_WR_DATA_CREDIT_BASE_IDX                                                                0
+#define regDAGB1_WR_MISC_CREDIT                                                                         0x00dc
+#define regDAGB1_WR_MISC_CREDIT_BASE_IDX                                                                0
+#define regDAGB1_WR_OSD_CREDIT_CNTL1                                                                    0x00dd
+#define regDAGB1_WR_OSD_CREDIT_CNTL1_BASE_IDX                                                           0
+#define regDAGB1_WR_OSD_CREDIT_CNTL2                                                                    0x00de
+#define regDAGB1_WR_OSD_CREDIT_CNTL2_BASE_IDX                                                           0
+#define regDAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1                                                            0x00df
+#define regDAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1_BASE_IDX                                                   0
+#define regDAGB1_WRCLI_GPU_SNOOP_OVERRIDE                                                               0x00e0
+#define regDAGB1_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX                                                      0
+#define regDAGB1_WRCLI_GPU_SNOOP_OVERRIDE_VALUE                                                         0x00e1
+#define regDAGB1_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX                                                0
+#define regDAGB1_WRCLI_ASK_PENDING                                                                      0x00e2
+#define regDAGB1_WRCLI_ASK_PENDING_BASE_IDX                                                             0
+#define regDAGB1_WRCLI_GO_PENDING                                                                       0x00e3
+#define regDAGB1_WRCLI_GO_PENDING_BASE_IDX                                                              0
+#define regDAGB1_WRCLI_GBLSEND_PENDING                                                                  0x00e4
+#define regDAGB1_WRCLI_GBLSEND_PENDING_BASE_IDX                                                         0
+#define regDAGB1_WRCLI_TLB_PENDING                                                                      0x00e5
+#define regDAGB1_WRCLI_TLB_PENDING_BASE_IDX                                                             0
+#define regDAGB1_WRCLI_OARB_PENDING                                                                     0x00e6
+#define regDAGB1_WRCLI_OARB_PENDING_BASE_IDX                                                            0
+#define regDAGB1_WRCLI_OSD_PENDING                                                                      0x00e7
+#define regDAGB1_WRCLI_OSD_PENDING_BASE_IDX                                                             0
+#define regDAGB1_WRCLI_DBUS_ASK_PENDING                                                                 0x00e8
+#define regDAGB1_WRCLI_DBUS_ASK_PENDING_BASE_IDX                                                        0
+#define regDAGB1_WRCLI_DBUS_GO_PENDING                                                                  0x00e9
+#define regDAGB1_WRCLI_DBUS_GO_PENDING_BASE_IDX                                                         0
+#define regDAGB1_DAGB_DLY                                                                               0x00ec
+#define regDAGB1_DAGB_DLY_BASE_IDX                                                                      0
+#define regDAGB1_CNTL_MISC                                                                              0x00ed
+#define regDAGB1_CNTL_MISC_BASE_IDX                                                                     0
+#define regDAGB1_CNTL_MISC2                                                                             0x00ee
+#define regDAGB1_CNTL_MISC2_BASE_IDX                                                                    0
+#define regDAGB1_FATAL_ERROR_CNTL                                                                       0x00ef
+#define regDAGB1_FATAL_ERROR_CNTL_BASE_IDX                                                              0
+#define regDAGB1_FATAL_ERROR_CLEAR                                                                      0x00f0
+#define regDAGB1_FATAL_ERROR_CLEAR_BASE_IDX                                                             0
+#define regDAGB1_FATAL_ERROR_STATUS0                                                                    0x00f1
+#define regDAGB1_FATAL_ERROR_STATUS0_BASE_IDX                                                           0
+#define regDAGB1_FATAL_ERROR_STATUS1                                                                    0x00f2
+#define regDAGB1_FATAL_ERROR_STATUS1_BASE_IDX                                                           0
+#define regDAGB1_FATAL_ERROR_STATUS2                                                                    0x00f3
+#define regDAGB1_FATAL_ERROR_STATUS2_BASE_IDX                                                           0
+#define regDAGB1_FATAL_ERROR_STATUS3                                                                    0x00f4
+#define regDAGB1_FATAL_ERROR_STATUS3_BASE_IDX                                                           0
+#define regDAGB1_FIFO_EMPTY                                                                             0x00f5
+#define regDAGB1_FIFO_EMPTY_BASE_IDX                                                                    0
+#define regDAGB1_FIFO_FULL                                                                              0x00f6
+#define regDAGB1_FIFO_FULL_BASE_IDX                                                                     0
+#define regDAGB1_WR_CREDITS_FULL                                                                        0x00f7
+#define regDAGB1_WR_CREDITS_FULL_BASE_IDX                                                               0
+#define regDAGB1_RD_CREDITS_FULL                                                                        0x00f8
+#define regDAGB1_RD_CREDITS_FULL_BASE_IDX                                                               0
+#define regDAGB1_PERFCOUNTER_LO                                                                         0x00f9
+#define regDAGB1_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regDAGB1_PERFCOUNTER_HI                                                                         0x00fa
+#define regDAGB1_PERFCOUNTER_HI_BASE_IDX                                                                0
+#define regDAGB1_PERFCOUNTER0_CFG                                                                       0x00fb
+#define regDAGB1_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regDAGB1_PERFCOUNTER1_CFG                                                                       0x00fc
+#define regDAGB1_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regDAGB1_PERFCOUNTER2_CFG                                                                       0x00fd
+#define regDAGB1_PERFCOUNTER2_CFG_BASE_IDX                                                              0
+#define regDAGB1_PERFCOUNTER_RSLT_CNTL                                                                  0x00fe
+#define regDAGB1_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+#define regDAGB1_L1TLB_REG_RW                                                                           0x00ff
+#define regDAGB1_L1TLB_REG_RW_BASE_IDX                                                                  0
+
+
+// addressBlock: aid_mmhub_dagb_dagbdec2
+// base address: 0x60400
+#define regDAGB2_RDCLI0                                                                                 0x0100
+#define regDAGB2_RDCLI0_BASE_IDX                                                                        0
+#define regDAGB2_RDCLI1                                                                                 0x0101
+#define regDAGB2_RDCLI1_BASE_IDX                                                                        0
+#define regDAGB2_RDCLI2                                                                                 0x0102
+#define regDAGB2_RDCLI2_BASE_IDX                                                                        0
+#define regDAGB2_RDCLI3                                                                                 0x0103
+#define regDAGB2_RDCLI3_BASE_IDX                                                                        0
+#define regDAGB2_RDCLI4                                                                                 0x0104
+#define regDAGB2_RDCLI4_BASE_IDX                                                                        0
+#define regDAGB2_RDCLI5                                                                                 0x0105
+#define regDAGB2_RDCLI5_BASE_IDX                                                                        0
+#define regDAGB2_RDCLI6                                                                                 0x0106
+#define regDAGB2_RDCLI6_BASE_IDX                                                                        0
+#define regDAGB2_RDCLI7                                                                                 0x0107
+#define regDAGB2_RDCLI7_BASE_IDX                                                                        0
+#define regDAGB2_RDCLI8                                                                                 0x0108
+#define regDAGB2_RDCLI8_BASE_IDX                                                                        0
+#define regDAGB2_RDCLI9                                                                                 0x0109
+#define regDAGB2_RDCLI9_BASE_IDX                                                                        0
+#define regDAGB2_RDCLI10                                                                                0x010a
+#define regDAGB2_RDCLI10_BASE_IDX                                                                       0
+#define regDAGB2_RDCLI11                                                                                0x010b
+#define regDAGB2_RDCLI11_BASE_IDX                                                                       0
+#define regDAGB2_RDCLI12                                                                                0x010c
+#define regDAGB2_RDCLI12_BASE_IDX                                                                       0
+#define regDAGB2_RDCLI13                                                                                0x010d
+#define regDAGB2_RDCLI13_BASE_IDX                                                                       0
+#define regDAGB2_RDCLI14                                                                                0x010e
+#define regDAGB2_RDCLI14_BASE_IDX                                                                       0
+#define regDAGB2_RDCLI15                                                                                0x010f
+#define regDAGB2_RDCLI15_BASE_IDX                                                                       0
+#define regDAGB2_RD_CNTL                                                                                0x0110
+#define regDAGB2_RD_CNTL_BASE_IDX                                                                       0
+#define regDAGB2_RD_GMI_CNTL                                                                            0x0111
+#define regDAGB2_RD_GMI_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_RD_ADDR_DAGB                                                                           0x0112
+#define regDAGB2_RD_ADDR_DAGB_BASE_IDX                                                                  0
+#define regDAGB2_RD_OUTPUT_DAGB_MAX_BURST                                                               0x0113
+#define regDAGB2_RD_OUTPUT_DAGB_MAX_BURST_BASE_IDX                                                      0
+#define regDAGB2_RD_OUTPUT_DAGB_LAZY_TIMER                                                              0x0114
+#define regDAGB2_RD_OUTPUT_DAGB_LAZY_TIMER_BASE_IDX                                                     0
+#define regDAGB2_RD_CGTT_CLK_CTRL                                                                       0x0115
+#define regDAGB2_RD_CGTT_CLK_CTRL_BASE_IDX                                                              0
+#define regDAGB2_L1TLB_RD_CGTT_CLK_CTRL                                                                 0x0116
+#define regDAGB2_L1TLB_RD_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB2_ATCVM_RD_CGTT_CLK_CTRL                                                                 0x0117
+#define regDAGB2_ATCVM_RD_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB2_RD_ADDR_DAGB_MAX_BURST0                                                                0x0118
+#define regDAGB2_RD_ADDR_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB2_RD_ADDR_DAGB_LAZY_TIMER0                                                               0x0119
+#define regDAGB2_RD_ADDR_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB2_RD_ADDR_DAGB_MAX_BURST1                                                                0x011a
+#define regDAGB2_RD_ADDR_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB2_RD_ADDR_DAGB_LAZY_TIMER1                                                               0x011b
+#define regDAGB2_RD_ADDR_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB2_RD_VC0_CNTL                                                                            0x011c
+#define regDAGB2_RD_VC0_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_RD_VC1_CNTL                                                                            0x011d
+#define regDAGB2_RD_VC1_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_RD_VC2_CNTL                                                                            0x011e
+#define regDAGB2_RD_VC2_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_RD_VC3_CNTL                                                                            0x011f
+#define regDAGB2_RD_VC3_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_RD_VC4_CNTL                                                                            0x0120
+#define regDAGB2_RD_VC4_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_RD_VC5_CNTL                                                                            0x0121
+#define regDAGB2_RD_VC5_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_RD_VC6_CNTL                                                                            0x0122
+#define regDAGB2_RD_VC6_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_RD_VC7_CNTL                                                                            0x0123
+#define regDAGB2_RD_VC7_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_RD_CNTL_MISC                                                                           0x0124
+#define regDAGB2_RD_CNTL_MISC_BASE_IDX                                                                  0
+#define regDAGB2_RD_TLB_CREDIT                                                                          0x0125
+#define regDAGB2_RD_TLB_CREDIT_BASE_IDX                                                                 0
+#define regDAGB2_RD_RDRET_CREDIT_CNTL                                                                   0x0126
+#define regDAGB2_RD_RDRET_CREDIT_CNTL_BASE_IDX                                                          0
+#define regDAGB2_RD_RDRET_CREDIT_CNTL2                                                                  0x0127
+#define regDAGB2_RD_RDRET_CREDIT_CNTL2_BASE_IDX                                                         0
+#define regDAGB2_RDCLI_ASK_PENDING                                                                      0x0128
+#define regDAGB2_RDCLI_ASK_PENDING_BASE_IDX                                                             0
+#define regDAGB2_RDCLI_GO_PENDING                                                                       0x0129
+#define regDAGB2_RDCLI_GO_PENDING_BASE_IDX                                                              0
+#define regDAGB2_RDCLI_GBLSEND_PENDING                                                                  0x012a
+#define regDAGB2_RDCLI_GBLSEND_PENDING_BASE_IDX                                                         0
+#define regDAGB2_RDCLI_TLB_PENDING                                                                      0x012b
+#define regDAGB2_RDCLI_TLB_PENDING_BASE_IDX                                                             0
+#define regDAGB2_RDCLI_OARB_PENDING                                                                     0x012c
+#define regDAGB2_RDCLI_OARB_PENDING_BASE_IDX                                                            0
+#define regDAGB2_RDCLI_OSD_PENDING                                                                      0x012d
+#define regDAGB2_RDCLI_OSD_PENDING_BASE_IDX                                                             0
+#define regDAGB2_WRCLI0                                                                                 0x0130
+#define regDAGB2_WRCLI0_BASE_IDX                                                                        0
+#define regDAGB2_WRCLI1                                                                                 0x0131
+#define regDAGB2_WRCLI1_BASE_IDX                                                                        0
+#define regDAGB2_WRCLI2                                                                                 0x0132
+#define regDAGB2_WRCLI2_BASE_IDX                                                                        0
+#define regDAGB2_WRCLI3                                                                                 0x0133
+#define regDAGB2_WRCLI3_BASE_IDX                                                                        0
+#define regDAGB2_WRCLI4                                                                                 0x0134
+#define regDAGB2_WRCLI4_BASE_IDX                                                                        0
+#define regDAGB2_WRCLI5                                                                                 0x0135
+#define regDAGB2_WRCLI5_BASE_IDX                                                                        0
+#define regDAGB2_WRCLI6                                                                                 0x0136
+#define regDAGB2_WRCLI6_BASE_IDX                                                                        0
+#define regDAGB2_WRCLI7                                                                                 0x0137
+#define regDAGB2_WRCLI7_BASE_IDX                                                                        0
+#define regDAGB2_WRCLI8                                                                                 0x0138
+#define regDAGB2_WRCLI8_BASE_IDX                                                                        0
+#define regDAGB2_WRCLI9                                                                                 0x0139
+#define regDAGB2_WRCLI9_BASE_IDX                                                                        0
+#define regDAGB2_WRCLI10                                                                                0x013a
+#define regDAGB2_WRCLI10_BASE_IDX                                                                       0
+#define regDAGB2_WRCLI11                                                                                0x013b
+#define regDAGB2_WRCLI11_BASE_IDX                                                                       0
+#define regDAGB2_WRCLI12                                                                                0x013c
+#define regDAGB2_WRCLI12_BASE_IDX                                                                       0
+#define regDAGB2_WRCLI13                                                                                0x013d
+#define regDAGB2_WRCLI13_BASE_IDX                                                                       0
+#define regDAGB2_WRCLI14                                                                                0x013e
+#define regDAGB2_WRCLI14_BASE_IDX                                                                       0
+#define regDAGB2_WRCLI15                                                                                0x013f
+#define regDAGB2_WRCLI15_BASE_IDX                                                                       0
+#define regDAGB2_WR_CNTL                                                                                0x0140
+#define regDAGB2_WR_CNTL_BASE_IDX                                                                       0
+#define regDAGB2_WR_GMI_CNTL                                                                            0x0141
+#define regDAGB2_WR_GMI_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_WR_ADDR_DAGB                                                                           0x0142
+#define regDAGB2_WR_ADDR_DAGB_BASE_IDX                                                                  0
+#define regDAGB2_WR_OUTPUT_DAGB_MAX_BURST                                                               0x0143
+#define regDAGB2_WR_OUTPUT_DAGB_MAX_BURST_BASE_IDX                                                      0
+#define regDAGB2_WR_OUTPUT_DAGB_LAZY_TIMER                                                              0x0144
+#define regDAGB2_WR_OUTPUT_DAGB_LAZY_TIMER_BASE_IDX                                                     0
+#define regDAGB2_WR_CGTT_CLK_CTRL                                                                       0x0145
+#define regDAGB2_WR_CGTT_CLK_CTRL_BASE_IDX                                                              0
+#define regDAGB2_L1TLB_WR_CGTT_CLK_CTRL                                                                 0x0146
+#define regDAGB2_L1TLB_WR_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB2_ATCVM_WR_CGTT_CLK_CTRL                                                                 0x0147
+#define regDAGB2_ATCVM_WR_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB2_WR_ADDR_DAGB_MAX_BURST0                                                                0x0148
+#define regDAGB2_WR_ADDR_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB2_WR_ADDR_DAGB_LAZY_TIMER0                                                               0x0149
+#define regDAGB2_WR_ADDR_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB2_WR_ADDR_DAGB_MAX_BURST1                                                                0x014a
+#define regDAGB2_WR_ADDR_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB2_WR_ADDR_DAGB_LAZY_TIMER1                                                               0x014b
+#define regDAGB2_WR_ADDR_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB2_WR_DATA_DAGB                                                                           0x014c
+#define regDAGB2_WR_DATA_DAGB_BASE_IDX                                                                  0
+#define regDAGB2_WR_DATA_DAGB_MAX_BURST0                                                                0x014d
+#define regDAGB2_WR_DATA_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB2_WR_DATA_DAGB_LAZY_TIMER0                                                               0x014e
+#define regDAGB2_WR_DATA_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB2_WR_DATA_DAGB_MAX_BURST1                                                                0x014f
+#define regDAGB2_WR_DATA_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB2_WR_DATA_DAGB_LAZY_TIMER1                                                               0x0150
+#define regDAGB2_WR_DATA_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB2_WR_VC0_CNTL                                                                            0x0151
+#define regDAGB2_WR_VC0_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_WR_VC1_CNTL                                                                            0x0152
+#define regDAGB2_WR_VC1_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_WR_VC2_CNTL                                                                            0x0153
+#define regDAGB2_WR_VC2_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_WR_VC3_CNTL                                                                            0x0154
+#define regDAGB2_WR_VC3_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_WR_VC4_CNTL                                                                            0x0155
+#define regDAGB2_WR_VC4_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_WR_VC5_CNTL                                                                            0x0156
+#define regDAGB2_WR_VC5_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_WR_VC6_CNTL                                                                            0x0157
+#define regDAGB2_WR_VC6_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_WR_VC7_CNTL                                                                            0x0158
+#define regDAGB2_WR_VC7_CNTL_BASE_IDX                                                                   0
+#define regDAGB2_WR_CNTL_MISC                                                                           0x0159
+#define regDAGB2_WR_CNTL_MISC_BASE_IDX                                                                  0
+#define regDAGB2_WR_TLB_CREDIT                                                                          0x015a
+#define regDAGB2_WR_TLB_CREDIT_BASE_IDX                                                                 0
+#define regDAGB2_WR_DATA_CREDIT                                                                         0x015b
+#define regDAGB2_WR_DATA_CREDIT_BASE_IDX                                                                0
+#define regDAGB2_WR_MISC_CREDIT                                                                         0x015c
+#define regDAGB2_WR_MISC_CREDIT_BASE_IDX                                                                0
+#define regDAGB2_WR_OSD_CREDIT_CNTL1                                                                    0x015d
+#define regDAGB2_WR_OSD_CREDIT_CNTL1_BASE_IDX                                                           0
+#define regDAGB2_WR_OSD_CREDIT_CNTL2                                                                    0x015e
+#define regDAGB2_WR_OSD_CREDIT_CNTL2_BASE_IDX                                                           0
+#define regDAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1                                                            0x015f
+#define regDAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1_BASE_IDX                                                   0
+#define regDAGB2_WRCLI_GPU_SNOOP_OVERRIDE                                                               0x0160
+#define regDAGB2_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX                                                      0
+#define regDAGB2_WRCLI_GPU_SNOOP_OVERRIDE_VALUE                                                         0x0161
+#define regDAGB2_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX                                                0
+#define regDAGB2_WRCLI_ASK_PENDING                                                                      0x0162
+#define regDAGB2_WRCLI_ASK_PENDING_BASE_IDX                                                             0
+#define regDAGB2_WRCLI_GO_PENDING                                                                       0x0163
+#define regDAGB2_WRCLI_GO_PENDING_BASE_IDX                                                              0
+#define regDAGB2_WRCLI_GBLSEND_PENDING                                                                  0x0164
+#define regDAGB2_WRCLI_GBLSEND_PENDING_BASE_IDX                                                         0
+#define regDAGB2_WRCLI_TLB_PENDING                                                                      0x0165
+#define regDAGB2_WRCLI_TLB_PENDING_BASE_IDX                                                             0
+#define regDAGB2_WRCLI_OARB_PENDING                                                                     0x0166
+#define regDAGB2_WRCLI_OARB_PENDING_BASE_IDX                                                            0
+#define regDAGB2_WRCLI_OSD_PENDING                                                                      0x0167
+#define regDAGB2_WRCLI_OSD_PENDING_BASE_IDX                                                             0
+#define regDAGB2_WRCLI_DBUS_ASK_PENDING                                                                 0x0168
+#define regDAGB2_WRCLI_DBUS_ASK_PENDING_BASE_IDX                                                        0
+#define regDAGB2_WRCLI_DBUS_GO_PENDING                                                                  0x0169
+#define regDAGB2_WRCLI_DBUS_GO_PENDING_BASE_IDX                                                         0
+#define regDAGB2_DAGB_DLY                                                                               0x016c
+#define regDAGB2_DAGB_DLY_BASE_IDX                                                                      0
+#define regDAGB2_CNTL_MISC                                                                              0x016d
+#define regDAGB2_CNTL_MISC_BASE_IDX                                                                     0
+#define regDAGB2_CNTL_MISC2                                                                             0x016e
+#define regDAGB2_CNTL_MISC2_BASE_IDX                                                                    0
+#define regDAGB2_FATAL_ERROR_CNTL                                                                       0x016f
+#define regDAGB2_FATAL_ERROR_CNTL_BASE_IDX                                                              0
+#define regDAGB2_FATAL_ERROR_CLEAR                                                                      0x0170
+#define regDAGB2_FATAL_ERROR_CLEAR_BASE_IDX                                                             0
+#define regDAGB2_FATAL_ERROR_STATUS0                                                                    0x0171
+#define regDAGB2_FATAL_ERROR_STATUS0_BASE_IDX                                                           0
+#define regDAGB2_FATAL_ERROR_STATUS1                                                                    0x0172
+#define regDAGB2_FATAL_ERROR_STATUS1_BASE_IDX                                                           0
+#define regDAGB2_FATAL_ERROR_STATUS2                                                                    0x0173
+#define regDAGB2_FATAL_ERROR_STATUS2_BASE_IDX                                                           0
+#define regDAGB2_FATAL_ERROR_STATUS3                                                                    0x0174
+#define regDAGB2_FATAL_ERROR_STATUS3_BASE_IDX                                                           0
+#define regDAGB2_FIFO_EMPTY                                                                             0x0175
+#define regDAGB2_FIFO_EMPTY_BASE_IDX                                                                    0
+#define regDAGB2_FIFO_FULL                                                                              0x0176
+#define regDAGB2_FIFO_FULL_BASE_IDX                                                                     0
+#define regDAGB2_WR_CREDITS_FULL                                                                        0x0177
+#define regDAGB2_WR_CREDITS_FULL_BASE_IDX                                                               0
+#define regDAGB2_RD_CREDITS_FULL                                                                        0x0178
+#define regDAGB2_RD_CREDITS_FULL_BASE_IDX                                                               0
+#define regDAGB2_PERFCOUNTER_LO                                                                         0x0179
+#define regDAGB2_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regDAGB2_PERFCOUNTER_HI                                                                         0x017a
+#define regDAGB2_PERFCOUNTER_HI_BASE_IDX                                                                0
+#define regDAGB2_PERFCOUNTER0_CFG                                                                       0x017b
+#define regDAGB2_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regDAGB2_PERFCOUNTER1_CFG                                                                       0x017c
+#define regDAGB2_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regDAGB2_PERFCOUNTER2_CFG                                                                       0x017d
+#define regDAGB2_PERFCOUNTER2_CFG_BASE_IDX                                                              0
+#define regDAGB2_PERFCOUNTER_RSLT_CNTL                                                                  0x017e
+#define regDAGB2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+#define regDAGB2_L1TLB_REG_RW                                                                           0x017f
+#define regDAGB2_L1TLB_REG_RW_BASE_IDX                                                                  0
+
+
+// addressBlock: aid_mmhub_dagb_dagbdec3
+// base address: 0x60600
+#define regDAGB3_RDCLI0                                                                                 0x0180
+#define regDAGB3_RDCLI0_BASE_IDX                                                                        0
+#define regDAGB3_RDCLI1                                                                                 0x0181
+#define regDAGB3_RDCLI1_BASE_IDX                                                                        0
+#define regDAGB3_RDCLI2                                                                                 0x0182
+#define regDAGB3_RDCLI2_BASE_IDX                                                                        0
+#define regDAGB3_RDCLI3                                                                                 0x0183
+#define regDAGB3_RDCLI3_BASE_IDX                                                                        0
+#define regDAGB3_RDCLI4                                                                                 0x0184
+#define regDAGB3_RDCLI4_BASE_IDX                                                                        0
+#define regDAGB3_RDCLI5                                                                                 0x0185
+#define regDAGB3_RDCLI5_BASE_IDX                                                                        0
+#define regDAGB3_RDCLI6                                                                                 0x0186
+#define regDAGB3_RDCLI6_BASE_IDX                                                                        0
+#define regDAGB3_RDCLI7                                                                                 0x0187
+#define regDAGB3_RDCLI7_BASE_IDX                                                                        0
+#define regDAGB3_RDCLI8                                                                                 0x0188
+#define regDAGB3_RDCLI8_BASE_IDX                                                                        0
+#define regDAGB3_RDCLI9                                                                                 0x0189
+#define regDAGB3_RDCLI9_BASE_IDX                                                                        0
+#define regDAGB3_RDCLI10                                                                                0x018a
+#define regDAGB3_RDCLI10_BASE_IDX                                                                       0
+#define regDAGB3_RDCLI11                                                                                0x018b
+#define regDAGB3_RDCLI11_BASE_IDX                                                                       0
+#define regDAGB3_RDCLI12                                                                                0x018c
+#define regDAGB3_RDCLI12_BASE_IDX                                                                       0
+#define regDAGB3_RDCLI13                                                                                0x018d
+#define regDAGB3_RDCLI13_BASE_IDX                                                                       0
+#define regDAGB3_RDCLI14                                                                                0x018e
+#define regDAGB3_RDCLI14_BASE_IDX                                                                       0
+#define regDAGB3_RDCLI15                                                                                0x018f
+#define regDAGB3_RDCLI15_BASE_IDX                                                                       0
+#define regDAGB3_RD_CNTL                                                                                0x0190
+#define regDAGB3_RD_CNTL_BASE_IDX                                                                       0
+#define regDAGB3_RD_GMI_CNTL                                                                            0x0191
+#define regDAGB3_RD_GMI_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_RD_ADDR_DAGB                                                                           0x0192
+#define regDAGB3_RD_ADDR_DAGB_BASE_IDX                                                                  0
+#define regDAGB3_RD_OUTPUT_DAGB_MAX_BURST                                                               0x0193
+#define regDAGB3_RD_OUTPUT_DAGB_MAX_BURST_BASE_IDX                                                      0
+#define regDAGB3_RD_OUTPUT_DAGB_LAZY_TIMER                                                              0x0194
+#define regDAGB3_RD_OUTPUT_DAGB_LAZY_TIMER_BASE_IDX                                                     0
+#define regDAGB3_RD_CGTT_CLK_CTRL                                                                       0x0195
+#define regDAGB3_RD_CGTT_CLK_CTRL_BASE_IDX                                                              0
+#define regDAGB3_L1TLB_RD_CGTT_CLK_CTRL                                                                 0x0196
+#define regDAGB3_L1TLB_RD_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB3_ATCVM_RD_CGTT_CLK_CTRL                                                                 0x0197
+#define regDAGB3_ATCVM_RD_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB3_RD_ADDR_DAGB_MAX_BURST0                                                                0x0198
+#define regDAGB3_RD_ADDR_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB3_RD_ADDR_DAGB_LAZY_TIMER0                                                               0x0199
+#define regDAGB3_RD_ADDR_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB3_RD_ADDR_DAGB_MAX_BURST1                                                                0x019a
+#define regDAGB3_RD_ADDR_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB3_RD_ADDR_DAGB_LAZY_TIMER1                                                               0x019b
+#define regDAGB3_RD_ADDR_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB3_RD_VC0_CNTL                                                                            0x019c
+#define regDAGB3_RD_VC0_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_RD_VC1_CNTL                                                                            0x019d
+#define regDAGB3_RD_VC1_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_RD_VC2_CNTL                                                                            0x019e
+#define regDAGB3_RD_VC2_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_RD_VC3_CNTL                                                                            0x019f
+#define regDAGB3_RD_VC3_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_RD_VC4_CNTL                                                                            0x01a0
+#define regDAGB3_RD_VC4_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_RD_VC5_CNTL                                                                            0x01a1
+#define regDAGB3_RD_VC5_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_RD_VC6_CNTL                                                                            0x01a2
+#define regDAGB3_RD_VC6_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_RD_VC7_CNTL                                                                            0x01a3
+#define regDAGB3_RD_VC7_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_RD_CNTL_MISC                                                                           0x01a4
+#define regDAGB3_RD_CNTL_MISC_BASE_IDX                                                                  0
+#define regDAGB3_RD_TLB_CREDIT                                                                          0x01a5
+#define regDAGB3_RD_TLB_CREDIT_BASE_IDX                                                                 0
+#define regDAGB3_RD_RDRET_CREDIT_CNTL                                                                   0x01a6
+#define regDAGB3_RD_RDRET_CREDIT_CNTL_BASE_IDX                                                          0
+#define regDAGB3_RD_RDRET_CREDIT_CNTL2                                                                  0x01a7
+#define regDAGB3_RD_RDRET_CREDIT_CNTL2_BASE_IDX                                                         0
+#define regDAGB3_RDCLI_ASK_PENDING                                                                      0x01a8
+#define regDAGB3_RDCLI_ASK_PENDING_BASE_IDX                                                             0
+#define regDAGB3_RDCLI_GO_PENDING                                                                       0x01a9
+#define regDAGB3_RDCLI_GO_PENDING_BASE_IDX                                                              0
+#define regDAGB3_RDCLI_GBLSEND_PENDING                                                                  0x01aa
+#define regDAGB3_RDCLI_GBLSEND_PENDING_BASE_IDX                                                         0
+#define regDAGB3_RDCLI_TLB_PENDING                                                                      0x01ab
+#define regDAGB3_RDCLI_TLB_PENDING_BASE_IDX                                                             0
+#define regDAGB3_RDCLI_OARB_PENDING                                                                     0x01ac
+#define regDAGB3_RDCLI_OARB_PENDING_BASE_IDX                                                            0
+#define regDAGB3_RDCLI_OSD_PENDING                                                                      0x01ad
+#define regDAGB3_RDCLI_OSD_PENDING_BASE_IDX                                                             0
+#define regDAGB3_WRCLI0                                                                                 0x01b0
+#define regDAGB3_WRCLI0_BASE_IDX                                                                        0
+#define regDAGB3_WRCLI1                                                                                 0x01b1
+#define regDAGB3_WRCLI1_BASE_IDX                                                                        0
+#define regDAGB3_WRCLI2                                                                                 0x01b2
+#define regDAGB3_WRCLI2_BASE_IDX                                                                        0
+#define regDAGB3_WRCLI3                                                                                 0x01b3
+#define regDAGB3_WRCLI3_BASE_IDX                                                                        0
+#define regDAGB3_WRCLI4                                                                                 0x01b4
+#define regDAGB3_WRCLI4_BASE_IDX                                                                        0
+#define regDAGB3_WRCLI5                                                                                 0x01b5
+#define regDAGB3_WRCLI5_BASE_IDX                                                                        0
+#define regDAGB3_WRCLI6                                                                                 0x01b6
+#define regDAGB3_WRCLI6_BASE_IDX                                                                        0
+#define regDAGB3_WRCLI7                                                                                 0x01b7
+#define regDAGB3_WRCLI7_BASE_IDX                                                                        0
+#define regDAGB3_WRCLI8                                                                                 0x01b8
+#define regDAGB3_WRCLI8_BASE_IDX                                                                        0
+#define regDAGB3_WRCLI9                                                                                 0x01b9
+#define regDAGB3_WRCLI9_BASE_IDX                                                                        0
+#define regDAGB3_WRCLI10                                                                                0x01ba
+#define regDAGB3_WRCLI10_BASE_IDX                                                                       0
+#define regDAGB3_WRCLI11                                                                                0x01bb
+#define regDAGB3_WRCLI11_BASE_IDX                                                                       0
+#define regDAGB3_WRCLI12                                                                                0x01bc
+#define regDAGB3_WRCLI12_BASE_IDX                                                                       0
+#define regDAGB3_WRCLI13                                                                                0x01bd
+#define regDAGB3_WRCLI13_BASE_IDX                                                                       0
+#define regDAGB3_WRCLI14                                                                                0x01be
+#define regDAGB3_WRCLI14_BASE_IDX                                                                       0
+#define regDAGB3_WRCLI15                                                                                0x01bf
+#define regDAGB3_WRCLI15_BASE_IDX                                                                       0
+#define regDAGB3_WR_CNTL                                                                                0x01c0
+#define regDAGB3_WR_CNTL_BASE_IDX                                                                       0
+#define regDAGB3_WR_GMI_CNTL                                                                            0x01c1
+#define regDAGB3_WR_GMI_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_WR_ADDR_DAGB                                                                           0x01c2
+#define regDAGB3_WR_ADDR_DAGB_BASE_IDX                                                                  0
+#define regDAGB3_WR_OUTPUT_DAGB_MAX_BURST                                                               0x01c3
+#define regDAGB3_WR_OUTPUT_DAGB_MAX_BURST_BASE_IDX                                                      0
+#define regDAGB3_WR_OUTPUT_DAGB_LAZY_TIMER                                                              0x01c4
+#define regDAGB3_WR_OUTPUT_DAGB_LAZY_TIMER_BASE_IDX                                                     0
+#define regDAGB3_WR_CGTT_CLK_CTRL                                                                       0x01c5
+#define regDAGB3_WR_CGTT_CLK_CTRL_BASE_IDX                                                              0
+#define regDAGB3_L1TLB_WR_CGTT_CLK_CTRL                                                                 0x01c6
+#define regDAGB3_L1TLB_WR_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB3_ATCVM_WR_CGTT_CLK_CTRL                                                                 0x01c7
+#define regDAGB3_ATCVM_WR_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB3_WR_ADDR_DAGB_MAX_BURST0                                                                0x01c8
+#define regDAGB3_WR_ADDR_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB3_WR_ADDR_DAGB_LAZY_TIMER0                                                               0x01c9
+#define regDAGB3_WR_ADDR_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB3_WR_ADDR_DAGB_MAX_BURST1                                                                0x01ca
+#define regDAGB3_WR_ADDR_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB3_WR_ADDR_DAGB_LAZY_TIMER1                                                               0x01cb
+#define regDAGB3_WR_ADDR_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB3_WR_DATA_DAGB                                                                           0x01cc
+#define regDAGB3_WR_DATA_DAGB_BASE_IDX                                                                  0
+#define regDAGB3_WR_DATA_DAGB_MAX_BURST0                                                                0x01cd
+#define regDAGB3_WR_DATA_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB3_WR_DATA_DAGB_LAZY_TIMER0                                                               0x01ce
+#define regDAGB3_WR_DATA_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB3_WR_DATA_DAGB_MAX_BURST1                                                                0x01cf
+#define regDAGB3_WR_DATA_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB3_WR_DATA_DAGB_LAZY_TIMER1                                                               0x01d0
+#define regDAGB3_WR_DATA_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB3_WR_VC0_CNTL                                                                            0x01d1
+#define regDAGB3_WR_VC0_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_WR_VC1_CNTL                                                                            0x01d2
+#define regDAGB3_WR_VC1_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_WR_VC2_CNTL                                                                            0x01d3
+#define regDAGB3_WR_VC2_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_WR_VC3_CNTL                                                                            0x01d4
+#define regDAGB3_WR_VC3_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_WR_VC4_CNTL                                                                            0x01d5
+#define regDAGB3_WR_VC4_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_WR_VC5_CNTL                                                                            0x01d6
+#define regDAGB3_WR_VC5_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_WR_VC6_CNTL                                                                            0x01d7
+#define regDAGB3_WR_VC6_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_WR_VC7_CNTL                                                                            0x01d8
+#define regDAGB3_WR_VC7_CNTL_BASE_IDX                                                                   0
+#define regDAGB3_WR_CNTL_MISC                                                                           0x01d9
+#define regDAGB3_WR_CNTL_MISC_BASE_IDX                                                                  0
+#define regDAGB3_WR_TLB_CREDIT                                                                          0x01da
+#define regDAGB3_WR_TLB_CREDIT_BASE_IDX                                                                 0
+#define regDAGB3_WR_DATA_CREDIT                                                                         0x01db
+#define regDAGB3_WR_DATA_CREDIT_BASE_IDX                                                                0
+#define regDAGB3_WR_MISC_CREDIT                                                                         0x01dc
+#define regDAGB3_WR_MISC_CREDIT_BASE_IDX                                                                0
+#define regDAGB3_WR_OSD_CREDIT_CNTL1                                                                    0x01dd
+#define regDAGB3_WR_OSD_CREDIT_CNTL1_BASE_IDX                                                           0
+#define regDAGB3_WR_OSD_CREDIT_CNTL2                                                                    0x01de
+#define regDAGB3_WR_OSD_CREDIT_CNTL2_BASE_IDX                                                           0
+#define regDAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1                                                            0x01df
+#define regDAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1_BASE_IDX                                                   0
+#define regDAGB3_WRCLI_GPU_SNOOP_OVERRIDE                                                               0x01e0
+#define regDAGB3_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX                                                      0
+#define regDAGB3_WRCLI_GPU_SNOOP_OVERRIDE_VALUE                                                         0x01e1
+#define regDAGB3_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX                                                0
+#define regDAGB3_WRCLI_ASK_PENDING                                                                      0x01e2
+#define regDAGB3_WRCLI_ASK_PENDING_BASE_IDX                                                             0
+#define regDAGB3_WRCLI_GO_PENDING                                                                       0x01e3
+#define regDAGB3_WRCLI_GO_PENDING_BASE_IDX                                                              0
+#define regDAGB3_WRCLI_GBLSEND_PENDING                                                                  0x01e4
+#define regDAGB3_WRCLI_GBLSEND_PENDING_BASE_IDX                                                         0
+#define regDAGB3_WRCLI_TLB_PENDING                                                                      0x01e5
+#define regDAGB3_WRCLI_TLB_PENDING_BASE_IDX                                                             0
+#define regDAGB3_WRCLI_OARB_PENDING                                                                     0x01e6
+#define regDAGB3_WRCLI_OARB_PENDING_BASE_IDX                                                            0
+#define regDAGB3_WRCLI_OSD_PENDING                                                                      0x01e7
+#define regDAGB3_WRCLI_OSD_PENDING_BASE_IDX                                                             0
+#define regDAGB3_WRCLI_DBUS_ASK_PENDING                                                                 0x01e8
+#define regDAGB3_WRCLI_DBUS_ASK_PENDING_BASE_IDX                                                        0
+#define regDAGB3_WRCLI_DBUS_GO_PENDING                                                                  0x01e9
+#define regDAGB3_WRCLI_DBUS_GO_PENDING_BASE_IDX                                                         0
+#define regDAGB3_DAGB_DLY                                                                               0x01ec
+#define regDAGB3_DAGB_DLY_BASE_IDX                                                                      0
+#define regDAGB3_CNTL_MISC                                                                              0x01ed
+#define regDAGB3_CNTL_MISC_BASE_IDX                                                                     0
+#define regDAGB3_CNTL_MISC2                                                                             0x01ee
+#define regDAGB3_CNTL_MISC2_BASE_IDX                                                                    0
+#define regDAGB3_FATAL_ERROR_CNTL                                                                       0x01ef
+#define regDAGB3_FATAL_ERROR_CNTL_BASE_IDX                                                              0
+#define regDAGB3_FATAL_ERROR_CLEAR                                                                      0x01f0
+#define regDAGB3_FATAL_ERROR_CLEAR_BASE_IDX                                                             0
+#define regDAGB3_FATAL_ERROR_STATUS0                                                                    0x01f1
+#define regDAGB3_FATAL_ERROR_STATUS0_BASE_IDX                                                           0
+#define regDAGB3_FATAL_ERROR_STATUS1                                                                    0x01f2
+#define regDAGB3_FATAL_ERROR_STATUS1_BASE_IDX                                                           0
+#define regDAGB3_FATAL_ERROR_STATUS2                                                                    0x01f3
+#define regDAGB3_FATAL_ERROR_STATUS2_BASE_IDX                                                           0
+#define regDAGB3_FATAL_ERROR_STATUS3                                                                    0x01f4
+#define regDAGB3_FATAL_ERROR_STATUS3_BASE_IDX                                                           0
+#define regDAGB3_FIFO_EMPTY                                                                             0x01f5
+#define regDAGB3_FIFO_EMPTY_BASE_IDX                                                                    0
+#define regDAGB3_FIFO_FULL                                                                              0x01f6
+#define regDAGB3_FIFO_FULL_BASE_IDX                                                                     0
+#define regDAGB3_WR_CREDITS_FULL                                                                        0x01f7
+#define regDAGB3_WR_CREDITS_FULL_BASE_IDX                                                               0
+#define regDAGB3_RD_CREDITS_FULL                                                                        0x01f8
+#define regDAGB3_RD_CREDITS_FULL_BASE_IDX                                                               0
+#define regDAGB3_PERFCOUNTER_LO                                                                         0x01f9
+#define regDAGB3_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regDAGB3_PERFCOUNTER_HI                                                                         0x01fa
+#define regDAGB3_PERFCOUNTER_HI_BASE_IDX                                                                0
+#define regDAGB3_PERFCOUNTER0_CFG                                                                       0x01fb
+#define regDAGB3_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regDAGB3_PERFCOUNTER1_CFG                                                                       0x01fc
+#define regDAGB3_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regDAGB3_PERFCOUNTER2_CFG                                                                       0x01fd
+#define regDAGB3_PERFCOUNTER2_CFG_BASE_IDX                                                              0
+#define regDAGB3_PERFCOUNTER_RSLT_CNTL                                                                  0x01fe
+#define regDAGB3_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+#define regDAGB3_L1TLB_REG_RW                                                                           0x01ff
+#define regDAGB3_L1TLB_REG_RW_BASE_IDX                                                                  0
+
+
+// addressBlock: aid_mmhub_dagb_dagbdec4
+// base address: 0x60800
+#define regDAGB4_RDCLI0                                                                                 0x0200
+#define regDAGB4_RDCLI0_BASE_IDX                                                                        0
+#define regDAGB4_RDCLI1                                                                                 0x0201
+#define regDAGB4_RDCLI1_BASE_IDX                                                                        0
+#define regDAGB4_RDCLI2                                                                                 0x0202
+#define regDAGB4_RDCLI2_BASE_IDX                                                                        0
+#define regDAGB4_RDCLI3                                                                                 0x0203
+#define regDAGB4_RDCLI3_BASE_IDX                                                                        0
+#define regDAGB4_RDCLI4                                                                                 0x0204
+#define regDAGB4_RDCLI4_BASE_IDX                                                                        0
+#define regDAGB4_RDCLI5                                                                                 0x0205
+#define regDAGB4_RDCLI5_BASE_IDX                                                                        0
+#define regDAGB4_RDCLI6                                                                                 0x0206
+#define regDAGB4_RDCLI6_BASE_IDX                                                                        0
+#define regDAGB4_RDCLI7                                                                                 0x0207
+#define regDAGB4_RDCLI7_BASE_IDX                                                                        0
+#define regDAGB4_RDCLI8                                                                                 0x0208
+#define regDAGB4_RDCLI8_BASE_IDX                                                                        0
+#define regDAGB4_RDCLI9                                                                                 0x0209
+#define regDAGB4_RDCLI9_BASE_IDX                                                                        0
+#define regDAGB4_RDCLI10                                                                                0x020a
+#define regDAGB4_RDCLI10_BASE_IDX                                                                       0
+#define regDAGB4_RDCLI11                                                                                0x020b
+#define regDAGB4_RDCLI11_BASE_IDX                                                                       0
+#define regDAGB4_RDCLI12                                                                                0x020c
+#define regDAGB4_RDCLI12_BASE_IDX                                                                       0
+#define regDAGB4_RDCLI13                                                                                0x020d
+#define regDAGB4_RDCLI13_BASE_IDX                                                                       0
+#define regDAGB4_RDCLI14                                                                                0x020e
+#define regDAGB4_RDCLI14_BASE_IDX                                                                       0
+#define regDAGB4_RDCLI15                                                                                0x020f
+#define regDAGB4_RDCLI15_BASE_IDX                                                                       0
+#define regDAGB4_RD_CNTL                                                                                0x0210
+#define regDAGB4_RD_CNTL_BASE_IDX                                                                       0
+#define regDAGB4_RD_GMI_CNTL                                                                            0x0211
+#define regDAGB4_RD_GMI_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_RD_ADDR_DAGB                                                                           0x0212
+#define regDAGB4_RD_ADDR_DAGB_BASE_IDX                                                                  0
+#define regDAGB4_RD_OUTPUT_DAGB_MAX_BURST                                                               0x0213
+#define regDAGB4_RD_OUTPUT_DAGB_MAX_BURST_BASE_IDX                                                      0
+#define regDAGB4_RD_OUTPUT_DAGB_LAZY_TIMER                                                              0x0214
+#define regDAGB4_RD_OUTPUT_DAGB_LAZY_TIMER_BASE_IDX                                                     0
+#define regDAGB4_RD_CGTT_CLK_CTRL                                                                       0x0215
+#define regDAGB4_RD_CGTT_CLK_CTRL_BASE_IDX                                                              0
+#define regDAGB4_L1TLB_RD_CGTT_CLK_CTRL                                                                 0x0216
+#define regDAGB4_L1TLB_RD_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB4_ATCVM_RD_CGTT_CLK_CTRL                                                                 0x0217
+#define regDAGB4_ATCVM_RD_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB4_RD_ADDR_DAGB_MAX_BURST0                                                                0x0218
+#define regDAGB4_RD_ADDR_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB4_RD_ADDR_DAGB_LAZY_TIMER0                                                               0x0219
+#define regDAGB4_RD_ADDR_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB4_RD_ADDR_DAGB_MAX_BURST1                                                                0x021a
+#define regDAGB4_RD_ADDR_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB4_RD_ADDR_DAGB_LAZY_TIMER1                                                               0x021b
+#define regDAGB4_RD_ADDR_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB4_RD_VC0_CNTL                                                                            0x021c
+#define regDAGB4_RD_VC0_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_RD_VC1_CNTL                                                                            0x021d
+#define regDAGB4_RD_VC1_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_RD_VC2_CNTL                                                                            0x021e
+#define regDAGB4_RD_VC2_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_RD_VC3_CNTL                                                                            0x021f
+#define regDAGB4_RD_VC3_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_RD_VC4_CNTL                                                                            0x0220
+#define regDAGB4_RD_VC4_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_RD_VC5_CNTL                                                                            0x0221
+#define regDAGB4_RD_VC5_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_RD_VC6_CNTL                                                                            0x0222
+#define regDAGB4_RD_VC6_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_RD_VC7_CNTL                                                                            0x0223
+#define regDAGB4_RD_VC7_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_RD_CNTL_MISC                                                                           0x0224
+#define regDAGB4_RD_CNTL_MISC_BASE_IDX                                                                  0
+#define regDAGB4_RD_TLB_CREDIT                                                                          0x0225
+#define regDAGB4_RD_TLB_CREDIT_BASE_IDX                                                                 0
+#define regDAGB4_RD_RDRET_CREDIT_CNTL                                                                   0x0226
+#define regDAGB4_RD_RDRET_CREDIT_CNTL_BASE_IDX                                                          0
+#define regDAGB4_RD_RDRET_CREDIT_CNTL2                                                                  0x0227
+#define regDAGB4_RD_RDRET_CREDIT_CNTL2_BASE_IDX                                                         0
+#define regDAGB4_RDCLI_ASK_PENDING                                                                      0x0228
+#define regDAGB4_RDCLI_ASK_PENDING_BASE_IDX                                                             0
+#define regDAGB4_RDCLI_GO_PENDING                                                                       0x0229
+#define regDAGB4_RDCLI_GO_PENDING_BASE_IDX                                                              0
+#define regDAGB4_RDCLI_GBLSEND_PENDING                                                                  0x022a
+#define regDAGB4_RDCLI_GBLSEND_PENDING_BASE_IDX                                                         0
+#define regDAGB4_RDCLI_TLB_PENDING                                                                      0x022b
+#define regDAGB4_RDCLI_TLB_PENDING_BASE_IDX                                                             0
+#define regDAGB4_RDCLI_OARB_PENDING                                                                     0x022c
+#define regDAGB4_RDCLI_OARB_PENDING_BASE_IDX                                                            0
+#define regDAGB4_RDCLI_OSD_PENDING                                                                      0x022d
+#define regDAGB4_RDCLI_OSD_PENDING_BASE_IDX                                                             0
+#define regDAGB4_WRCLI0                                                                                 0x0230
+#define regDAGB4_WRCLI0_BASE_IDX                                                                        0
+#define regDAGB4_WRCLI1                                                                                 0x0231
+#define regDAGB4_WRCLI1_BASE_IDX                                                                        0
+#define regDAGB4_WRCLI2                                                                                 0x0232
+#define regDAGB4_WRCLI2_BASE_IDX                                                                        0
+#define regDAGB4_WRCLI3                                                                                 0x0233
+#define regDAGB4_WRCLI3_BASE_IDX                                                                        0
+#define regDAGB4_WRCLI4                                                                                 0x0234
+#define regDAGB4_WRCLI4_BASE_IDX                                                                        0
+#define regDAGB4_WRCLI5                                                                                 0x0235
+#define regDAGB4_WRCLI5_BASE_IDX                                                                        0
+#define regDAGB4_WRCLI6                                                                                 0x0236
+#define regDAGB4_WRCLI6_BASE_IDX                                                                        0
+#define regDAGB4_WRCLI7                                                                                 0x0237
+#define regDAGB4_WRCLI7_BASE_IDX                                                                        0
+#define regDAGB4_WRCLI8                                                                                 0x0238
+#define regDAGB4_WRCLI8_BASE_IDX                                                                        0
+#define regDAGB4_WRCLI9                                                                                 0x0239
+#define regDAGB4_WRCLI9_BASE_IDX                                                                        0
+#define regDAGB4_WRCLI10                                                                                0x023a
+#define regDAGB4_WRCLI10_BASE_IDX                                                                       0
+#define regDAGB4_WRCLI11                                                                                0x023b
+#define regDAGB4_WRCLI11_BASE_IDX                                                                       0
+#define regDAGB4_WRCLI12                                                                                0x023c
+#define regDAGB4_WRCLI12_BASE_IDX                                                                       0
+#define regDAGB4_WRCLI13                                                                                0x023d
+#define regDAGB4_WRCLI13_BASE_IDX                                                                       0
+#define regDAGB4_WRCLI14                                                                                0x023e
+#define regDAGB4_WRCLI14_BASE_IDX                                                                       0
+#define regDAGB4_WRCLI15                                                                                0x023f
+#define regDAGB4_WRCLI15_BASE_IDX                                                                       0
+#define regDAGB4_WR_CNTL                                                                                0x0240
+#define regDAGB4_WR_CNTL_BASE_IDX                                                                       0
+#define regDAGB4_WR_GMI_CNTL                                                                            0x0241
+#define regDAGB4_WR_GMI_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_WR_ADDR_DAGB                                                                           0x0242
+#define regDAGB4_WR_ADDR_DAGB_BASE_IDX                                                                  0
+#define regDAGB4_WR_OUTPUT_DAGB_MAX_BURST                                                               0x0243
+#define regDAGB4_WR_OUTPUT_DAGB_MAX_BURST_BASE_IDX                                                      0
+#define regDAGB4_WR_OUTPUT_DAGB_LAZY_TIMER                                                              0x0244
+#define regDAGB4_WR_OUTPUT_DAGB_LAZY_TIMER_BASE_IDX                                                     0
+#define regDAGB4_WR_CGTT_CLK_CTRL                                                                       0x0245
+#define regDAGB4_WR_CGTT_CLK_CTRL_BASE_IDX                                                              0
+#define regDAGB4_L1TLB_WR_CGTT_CLK_CTRL                                                                 0x0246
+#define regDAGB4_L1TLB_WR_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB4_ATCVM_WR_CGTT_CLK_CTRL                                                                 0x0247
+#define regDAGB4_ATCVM_WR_CGTT_CLK_CTRL_BASE_IDX                                                        0
+#define regDAGB4_WR_ADDR_DAGB_MAX_BURST0                                                                0x0248
+#define regDAGB4_WR_ADDR_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB4_WR_ADDR_DAGB_LAZY_TIMER0                                                               0x0249
+#define regDAGB4_WR_ADDR_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB4_WR_ADDR_DAGB_MAX_BURST1                                                                0x024a
+#define regDAGB4_WR_ADDR_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB4_WR_ADDR_DAGB_LAZY_TIMER1                                                               0x024b
+#define regDAGB4_WR_ADDR_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB4_WR_DATA_DAGB                                                                           0x024c
+#define regDAGB4_WR_DATA_DAGB_BASE_IDX                                                                  0
+#define regDAGB4_WR_DATA_DAGB_MAX_BURST0                                                                0x024d
+#define regDAGB4_WR_DATA_DAGB_MAX_BURST0_BASE_IDX                                                       0
+#define regDAGB4_WR_DATA_DAGB_LAZY_TIMER0                                                               0x024e
+#define regDAGB4_WR_DATA_DAGB_LAZY_TIMER0_BASE_IDX                                                      0
+#define regDAGB4_WR_DATA_DAGB_MAX_BURST1                                                                0x024f
+#define regDAGB4_WR_DATA_DAGB_MAX_BURST1_BASE_IDX                                                       0
+#define regDAGB4_WR_DATA_DAGB_LAZY_TIMER1                                                               0x0250
+#define regDAGB4_WR_DATA_DAGB_LAZY_TIMER1_BASE_IDX                                                      0
+#define regDAGB4_WR_VC0_CNTL                                                                            0x0251
+#define regDAGB4_WR_VC0_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_WR_VC1_CNTL                                                                            0x0252
+#define regDAGB4_WR_VC1_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_WR_VC2_CNTL                                                                            0x0253
+#define regDAGB4_WR_VC2_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_WR_VC3_CNTL                                                                            0x0254
+#define regDAGB4_WR_VC3_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_WR_VC4_CNTL                                                                            0x0255
+#define regDAGB4_WR_VC4_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_WR_VC5_CNTL                                                                            0x0256
+#define regDAGB4_WR_VC5_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_WR_VC6_CNTL                                                                            0x0257
+#define regDAGB4_WR_VC6_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_WR_VC7_CNTL                                                                            0x0258
+#define regDAGB4_WR_VC7_CNTL_BASE_IDX                                                                   0
+#define regDAGB4_WR_CNTL_MISC                                                                           0x0259
+#define regDAGB4_WR_CNTL_MISC_BASE_IDX                                                                  0
+#define regDAGB4_WR_TLB_CREDIT                                                                          0x025a
+#define regDAGB4_WR_TLB_CREDIT_BASE_IDX                                                                 0
+#define regDAGB4_WR_DATA_CREDIT                                                                         0x025b
+#define regDAGB4_WR_DATA_CREDIT_BASE_IDX                                                                0
+#define regDAGB4_WR_MISC_CREDIT                                                                         0x025c
+#define regDAGB4_WR_MISC_CREDIT_BASE_IDX                                                                0
+#define regDAGB4_WR_OSD_CREDIT_CNTL1                                                                    0x025d
+#define regDAGB4_WR_OSD_CREDIT_CNTL1_BASE_IDX                                                           0
+#define regDAGB4_WR_OSD_CREDIT_CNTL2                                                                    0x025e
+#define regDAGB4_WR_OSD_CREDIT_CNTL2_BASE_IDX                                                           0
+#define regDAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1                                                            0x025f
+#define regDAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1_BASE_IDX                                                   0
+#define regDAGB4_WRCLI_GPU_SNOOP_OVERRIDE                                                               0x0260
+#define regDAGB4_WRCLI_GPU_SNOOP_OVERRIDE_BASE_IDX                                                      0
+#define regDAGB4_WRCLI_GPU_SNOOP_OVERRIDE_VALUE                                                         0x0261
+#define regDAGB4_WRCLI_GPU_SNOOP_OVERRIDE_VALUE_BASE_IDX                                                0
+#define regDAGB4_WRCLI_ASK_PENDING                                                                      0x0262
+#define regDAGB4_WRCLI_ASK_PENDING_BASE_IDX                                                             0
+#define regDAGB4_WRCLI_GO_PENDING                                                                       0x0263
+#define regDAGB4_WRCLI_GO_PENDING_BASE_IDX                                                              0
+#define regDAGB4_WRCLI_GBLSEND_PENDING                                                                  0x0264
+#define regDAGB4_WRCLI_GBLSEND_PENDING_BASE_IDX                                                         0
+#define regDAGB4_WRCLI_TLB_PENDING                                                                      0x0265
+#define regDAGB4_WRCLI_TLB_PENDING_BASE_IDX                                                             0
+#define regDAGB4_WRCLI_OARB_PENDING                                                                     0x0266
+#define regDAGB4_WRCLI_OARB_PENDING_BASE_IDX                                                            0
+#define regDAGB4_WRCLI_OSD_PENDING                                                                      0x0267
+#define regDAGB4_WRCLI_OSD_PENDING_BASE_IDX                                                             0
+#define regDAGB4_WRCLI_DBUS_ASK_PENDING                                                                 0x0268
+#define regDAGB4_WRCLI_DBUS_ASK_PENDING_BASE_IDX                                                        0
+#define regDAGB4_WRCLI_DBUS_GO_PENDING                                                                  0x0269
+#define regDAGB4_WRCLI_DBUS_GO_PENDING_BASE_IDX                                                         0
+#define regDAGB4_DAGB_DLY                                                                               0x026c
+#define regDAGB4_DAGB_DLY_BASE_IDX                                                                      0
+#define regDAGB4_CNTL_MISC                                                                              0x026d
+#define regDAGB4_CNTL_MISC_BASE_IDX                                                                     0
+#define regDAGB4_CNTL_MISC2                                                                             0x026e
+#define regDAGB4_CNTL_MISC2_BASE_IDX                                                                    0
+#define regDAGB4_FATAL_ERROR_CNTL                                                                       0x026f
+#define regDAGB4_FATAL_ERROR_CNTL_BASE_IDX                                                              0
+#define regDAGB4_FATAL_ERROR_CLEAR                                                                      0x0270
+#define regDAGB4_FATAL_ERROR_CLEAR_BASE_IDX                                                             0
+#define regDAGB4_FATAL_ERROR_STATUS0                                                                    0x0271
+#define regDAGB4_FATAL_ERROR_STATUS0_BASE_IDX                                                           0
+#define regDAGB4_FATAL_ERROR_STATUS1                                                                    0x0272
+#define regDAGB4_FATAL_ERROR_STATUS1_BASE_IDX                                                           0
+#define regDAGB4_FATAL_ERROR_STATUS2                                                                    0x0273
+#define regDAGB4_FATAL_ERROR_STATUS2_BASE_IDX                                                           0
+#define regDAGB4_FATAL_ERROR_STATUS3                                                                    0x0274
+#define regDAGB4_FATAL_ERROR_STATUS3_BASE_IDX                                                           0
+#define regDAGB4_FIFO_EMPTY                                                                             0x0275
+#define regDAGB4_FIFO_EMPTY_BASE_IDX                                                                    0
+#define regDAGB4_FIFO_FULL                                                                              0x0276
+#define regDAGB4_FIFO_FULL_BASE_IDX                                                                     0
+#define regDAGB4_WR_CREDITS_FULL                                                                        0x0277
+#define regDAGB4_WR_CREDITS_FULL_BASE_IDX                                                               0
+#define regDAGB4_RD_CREDITS_FULL                                                                        0x0278
+#define regDAGB4_RD_CREDITS_FULL_BASE_IDX                                                               0
+#define regDAGB4_PERFCOUNTER_LO                                                                         0x0279
+#define regDAGB4_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regDAGB4_PERFCOUNTER_HI                                                                         0x027a
+#define regDAGB4_PERFCOUNTER_HI_BASE_IDX                                                                0
+#define regDAGB4_PERFCOUNTER0_CFG                                                                       0x027b
+#define regDAGB4_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regDAGB4_PERFCOUNTER1_CFG                                                                       0x027c
+#define regDAGB4_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regDAGB4_PERFCOUNTER2_CFG                                                                       0x027d
+#define regDAGB4_PERFCOUNTER2_CFG_BASE_IDX                                                              0
+#define regDAGB4_PERFCOUNTER_RSLT_CNTL                                                                  0x027e
+#define regDAGB4_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+#define regDAGB4_L1TLB_REG_RW                                                                           0x027f
+#define regDAGB4_L1TLB_REG_RW_BASE_IDX                                                                  0
+
+
+// addressBlock: aid_mmhub_ea_mmeadec0
+// base address: 0x60c00
+#define regMMEA0_DRAM_RD_CLI2GRP_MAP0                                                                   0x0300
+#define regMMEA0_DRAM_RD_CLI2GRP_MAP0_BASE_IDX                                                          0
+#define regMMEA0_DRAM_RD_CLI2GRP_MAP1                                                                   0x0301
+#define regMMEA0_DRAM_RD_CLI2GRP_MAP1_BASE_IDX                                                          0
+#define regMMEA0_DRAM_WR_CLI2GRP_MAP0                                                                   0x0302
+#define regMMEA0_DRAM_WR_CLI2GRP_MAP0_BASE_IDX                                                          0
+#define regMMEA0_DRAM_WR_CLI2GRP_MAP1                                                                   0x0303
+#define regMMEA0_DRAM_WR_CLI2GRP_MAP1_BASE_IDX                                                          0
+#define regMMEA0_DRAM_RD_GRP2VC_MAP                                                                     0x0304
+#define regMMEA0_DRAM_RD_GRP2VC_MAP_BASE_IDX                                                            0
+#define regMMEA0_DRAM_WR_GRP2VC_MAP                                                                     0x0305
+#define regMMEA0_DRAM_WR_GRP2VC_MAP_BASE_IDX                                                            0
+#define regMMEA0_DRAM_RD_LAZY                                                                           0x0306
+#define regMMEA0_DRAM_RD_LAZY_BASE_IDX                                                                  0
+#define regMMEA0_DRAM_WR_LAZY                                                                           0x0307
+#define regMMEA0_DRAM_WR_LAZY_BASE_IDX                                                                  0
+#define regMMEA0_DRAM_RD_CAM_CNTL                                                                       0x0308
+#define regMMEA0_DRAM_RD_CAM_CNTL_BASE_IDX                                                              0
+#define regMMEA0_DRAM_WR_CAM_CNTL                                                                       0x0309
+#define regMMEA0_DRAM_WR_CAM_CNTL_BASE_IDX                                                              0
+#define regMMEA0_DRAM_PAGE_BURST                                                                        0x030a
+#define regMMEA0_DRAM_PAGE_BURST_BASE_IDX                                                               0
+#define regMMEA0_DRAM_RD_PRI_AGE                                                                        0x030b
+#define regMMEA0_DRAM_RD_PRI_AGE_BASE_IDX                                                               0
+#define regMMEA0_DRAM_WR_PRI_AGE                                                                        0x030c
+#define regMMEA0_DRAM_WR_PRI_AGE_BASE_IDX                                                               0
+#define regMMEA0_DRAM_RD_PRI_QUEUING                                                                    0x030d
+#define regMMEA0_DRAM_RD_PRI_QUEUING_BASE_IDX                                                           0
+#define regMMEA0_DRAM_WR_PRI_QUEUING                                                                    0x030e
+#define regMMEA0_DRAM_WR_PRI_QUEUING_BASE_IDX                                                           0
+#define regMMEA0_DRAM_RD_PRI_FIXED                                                                      0x030f
+#define regMMEA0_DRAM_RD_PRI_FIXED_BASE_IDX                                                             0
+#define regMMEA0_DRAM_WR_PRI_FIXED                                                                      0x0310
+#define regMMEA0_DRAM_WR_PRI_FIXED_BASE_IDX                                                             0
+#define regMMEA0_DRAM_RD_PRI_URGENCY                                                                    0x0311
+#define regMMEA0_DRAM_RD_PRI_URGENCY_BASE_IDX                                                           0
+#define regMMEA0_DRAM_WR_PRI_URGENCY                                                                    0x0312
+#define regMMEA0_DRAM_WR_PRI_URGENCY_BASE_IDX                                                           0
+#define regMMEA0_DRAM_RD_PRI_QUANT_PRI1                                                                 0x0313
+#define regMMEA0_DRAM_RD_PRI_QUANT_PRI1_BASE_IDX                                                        0
+#define regMMEA0_DRAM_RD_PRI_QUANT_PRI2                                                                 0x0314
+#define regMMEA0_DRAM_RD_PRI_QUANT_PRI2_BASE_IDX                                                        0
+#define regMMEA0_DRAM_RD_PRI_QUANT_PRI3                                                                 0x0315
+#define regMMEA0_DRAM_RD_PRI_QUANT_PRI3_BASE_IDX                                                        0
+#define regMMEA0_DRAM_WR_PRI_QUANT_PRI1                                                                 0x0316
+#define regMMEA0_DRAM_WR_PRI_QUANT_PRI1_BASE_IDX                                                        0
+#define regMMEA0_DRAM_WR_PRI_QUANT_PRI2                                                                 0x0317
+#define regMMEA0_DRAM_WR_PRI_QUANT_PRI2_BASE_IDX                                                        0
+#define regMMEA0_DRAM_WR_PRI_QUANT_PRI3                                                                 0x0318
+#define regMMEA0_DRAM_WR_PRI_QUANT_PRI3_BASE_IDX                                                        0
+#define regMMEA0_GMI_RD_CLI2GRP_MAP0                                                                    0x0319
+#define regMMEA0_GMI_RD_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regMMEA0_GMI_RD_CLI2GRP_MAP1                                                                    0x031a
+#define regMMEA0_GMI_RD_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regMMEA0_GMI_WR_CLI2GRP_MAP0                                                                    0x031b
+#define regMMEA0_GMI_WR_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regMMEA0_GMI_WR_CLI2GRP_MAP1                                                                    0x031c
+#define regMMEA0_GMI_WR_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regMMEA0_GMI_RD_GRP2VC_MAP                                                                      0x031d
+#define regMMEA0_GMI_RD_GRP2VC_MAP_BASE_IDX                                                             0
+#define regMMEA0_GMI_WR_GRP2VC_MAP                                                                      0x031e
+#define regMMEA0_GMI_WR_GRP2VC_MAP_BASE_IDX                                                             0
+#define regMMEA0_GMI_RD_LAZY                                                                            0x031f
+#define regMMEA0_GMI_RD_LAZY_BASE_IDX                                                                   0
+#define regMMEA0_GMI_WR_LAZY                                                                            0x0320
+#define regMMEA0_GMI_WR_LAZY_BASE_IDX                                                                   0
+#define regMMEA0_GMI_RD_CAM_CNTL                                                                        0x0321
+#define regMMEA0_GMI_RD_CAM_CNTL_BASE_IDX                                                               0
+#define regMMEA0_GMI_WR_CAM_CNTL                                                                        0x0322
+#define regMMEA0_GMI_WR_CAM_CNTL_BASE_IDX                                                               0
+#define regMMEA0_GMI_PAGE_BURST                                                                         0x0323
+#define regMMEA0_GMI_PAGE_BURST_BASE_IDX                                                                0
+#define regMMEA0_GMI_RD_PRI_AGE                                                                         0x0324
+#define regMMEA0_GMI_RD_PRI_AGE_BASE_IDX                                                                0
+#define regMMEA0_GMI_WR_PRI_AGE                                                                         0x0325
+#define regMMEA0_GMI_WR_PRI_AGE_BASE_IDX                                                                0
+#define regMMEA0_GMI_RD_PRI_QUEUING                                                                     0x0326
+#define regMMEA0_GMI_RD_PRI_QUEUING_BASE_IDX                                                            0
+#define regMMEA0_GMI_WR_PRI_QUEUING                                                                     0x0327
+#define regMMEA0_GMI_WR_PRI_QUEUING_BASE_IDX                                                            0
+#define regMMEA0_GMI_RD_PRI_FIXED                                                                       0x0328
+#define regMMEA0_GMI_RD_PRI_FIXED_BASE_IDX                                                              0
+#define regMMEA0_GMI_WR_PRI_FIXED                                                                       0x0329
+#define regMMEA0_GMI_WR_PRI_FIXED_BASE_IDX                                                              0
+#define regMMEA0_GMI_RD_PRI_URGENCY                                                                     0x032a
+#define regMMEA0_GMI_RD_PRI_URGENCY_BASE_IDX                                                            0
+#define regMMEA0_GMI_WR_PRI_URGENCY                                                                     0x032b
+#define regMMEA0_GMI_WR_PRI_URGENCY_BASE_IDX                                                            0
+#define regMMEA0_GMI_RD_PRI_URGENCY_MASKING                                                             0x032c
+#define regMMEA0_GMI_RD_PRI_URGENCY_MASKING_BASE_IDX                                                    0
+#define regMMEA0_GMI_WR_PRI_URGENCY_MASKING                                                             0x032d
+#define regMMEA0_GMI_WR_PRI_URGENCY_MASKING_BASE_IDX                                                    0
+#define regMMEA0_GMI_RD_PRI_QUANT_PRI1                                                                  0x032e
+#define regMMEA0_GMI_RD_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regMMEA0_GMI_RD_PRI_QUANT_PRI2                                                                  0x032f
+#define regMMEA0_GMI_RD_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regMMEA0_GMI_RD_PRI_QUANT_PRI3                                                                  0x0330
+#define regMMEA0_GMI_RD_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regMMEA0_GMI_WR_PRI_QUANT_PRI1                                                                  0x0331
+#define regMMEA0_GMI_WR_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regMMEA0_GMI_WR_PRI_QUANT_PRI2                                                                  0x0332
+#define regMMEA0_GMI_WR_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regMMEA0_GMI_WR_PRI_QUANT_PRI3                                                                  0x0333
+#define regMMEA0_GMI_WR_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regMMEA0_IO_RD_CLI2GRP_MAP0                                                                     0x03d5
+#define regMMEA0_IO_RD_CLI2GRP_MAP0_BASE_IDX                                                            0
+#define regMMEA0_IO_RD_CLI2GRP_MAP1                                                                     0x03d6
+#define regMMEA0_IO_RD_CLI2GRP_MAP1_BASE_IDX                                                            0
+#define regMMEA0_IO_WR_CLI2GRP_MAP0                                                                     0x03d7
+#define regMMEA0_IO_WR_CLI2GRP_MAP0_BASE_IDX                                                            0
+#define regMMEA0_IO_WR_CLI2GRP_MAP1                                                                     0x03d8
+#define regMMEA0_IO_WR_CLI2GRP_MAP1_BASE_IDX                                                            0
+#define regMMEA0_IO_RD_COMBINE_FLUSH                                                                    0x03d9
+#define regMMEA0_IO_RD_COMBINE_FLUSH_BASE_IDX                                                           0
+#define regMMEA0_IO_WR_COMBINE_FLUSH                                                                    0x03da
+#define regMMEA0_IO_WR_COMBINE_FLUSH_BASE_IDX                                                           0
+#define regMMEA0_IO_GROUP_BURST                                                                         0x03db
+#define regMMEA0_IO_GROUP_BURST_BASE_IDX                                                                0
+#define regMMEA0_IO_RD_PRI_AGE                                                                          0x03dc
+#define regMMEA0_IO_RD_PRI_AGE_BASE_IDX                                                                 0
+#define regMMEA0_IO_WR_PRI_AGE                                                                          0x03dd
+#define regMMEA0_IO_WR_PRI_AGE_BASE_IDX                                                                 0
+#define regMMEA0_IO_RD_PRI_QUEUING                                                                      0x03de
+#define regMMEA0_IO_RD_PRI_QUEUING_BASE_IDX                                                             0
+#define regMMEA0_IO_WR_PRI_QUEUING                                                                      0x03df
+#define regMMEA0_IO_WR_PRI_QUEUING_BASE_IDX                                                             0
+#define regMMEA0_IO_RD_PRI_FIXED                                                                        0x03e0
+#define regMMEA0_IO_RD_PRI_FIXED_BASE_IDX                                                               0
+#define regMMEA0_IO_WR_PRI_FIXED                                                                        0x03e1
+#define regMMEA0_IO_WR_PRI_FIXED_BASE_IDX                                                               0
+#define regMMEA0_IO_RD_PRI_URGENCY                                                                      0x03e2
+#define regMMEA0_IO_RD_PRI_URGENCY_BASE_IDX                                                             0
+#define regMMEA0_IO_WR_PRI_URGENCY                                                                      0x03e3
+#define regMMEA0_IO_WR_PRI_URGENCY_BASE_IDX                                                             0
+#define regMMEA0_IO_RD_PRI_URGENCY_MASKING                                                              0x03e4
+#define regMMEA0_IO_RD_PRI_URGENCY_MASKING_BASE_IDX                                                     0
+#define regMMEA0_IO_WR_PRI_URGENCY_MASKING                                                              0x03e5
+#define regMMEA0_IO_WR_PRI_URGENCY_MASKING_BASE_IDX                                                     0
+#define regMMEA0_IO_RD_PRI_QUANT_PRI1                                                                   0x03e6
+#define regMMEA0_IO_RD_PRI_QUANT_PRI1_BASE_IDX                                                          0
+#define regMMEA0_IO_RD_PRI_QUANT_PRI2                                                                   0x03e7
+#define regMMEA0_IO_RD_PRI_QUANT_PRI2_BASE_IDX                                                          0
+#define regMMEA0_IO_RD_PRI_QUANT_PRI3                                                                   0x03e8
+#define regMMEA0_IO_RD_PRI_QUANT_PRI3_BASE_IDX                                                          0
+#define regMMEA0_IO_WR_PRI_QUANT_PRI1                                                                   0x03e9
+#define regMMEA0_IO_WR_PRI_QUANT_PRI1_BASE_IDX                                                          0
+#define regMMEA0_IO_WR_PRI_QUANT_PRI2                                                                   0x03ea
+#define regMMEA0_IO_WR_PRI_QUANT_PRI2_BASE_IDX                                                          0
+#define regMMEA0_IO_WR_PRI_QUANT_PRI3                                                                   0x03eb
+#define regMMEA0_IO_WR_PRI_QUANT_PRI3_BASE_IDX                                                          0
+#define regMMEA0_SDP_ARB_DRAM                                                                           0x03ec
+#define regMMEA0_SDP_ARB_DRAM_BASE_IDX                                                                  0
+#define regMMEA0_SDP_ARB_GMI                                                                            0x03ed
+#define regMMEA0_SDP_ARB_GMI_BASE_IDX                                                                   0
+#define regMMEA0_SDP_ARB_FINAL                                                                          0x03ee
+#define regMMEA0_SDP_ARB_FINAL_BASE_IDX                                                                 0
+#define regMMEA0_SDP_DRAM_PRIORITY                                                                      0x03ef
+#define regMMEA0_SDP_DRAM_PRIORITY_BASE_IDX                                                             0
+#define regMMEA0_SDP_GMI_PRIORITY                                                                       0x03f0
+#define regMMEA0_SDP_GMI_PRIORITY_BASE_IDX                                                              0
+#define regMMEA0_SDP_IO_PRIORITY                                                                        0x03f1
+#define regMMEA0_SDP_IO_PRIORITY_BASE_IDX                                                               0
+#define regMMEA0_SDP_CREDITS                                                                            0x03f2
+#define regMMEA0_SDP_CREDITS_BASE_IDX                                                                   0
+#define regMMEA0_SDP_TAG_RESERVE0                                                                       0x03f3
+#define regMMEA0_SDP_TAG_RESERVE0_BASE_IDX                                                              0
+#define regMMEA0_SDP_TAG_RESERVE1                                                                       0x03f4
+#define regMMEA0_SDP_TAG_RESERVE1_BASE_IDX                                                              0
+#define regMMEA0_SDP_VCC_RESERVE0                                                                       0x03f5
+#define regMMEA0_SDP_VCC_RESERVE0_BASE_IDX                                                              0
+#define regMMEA0_SDP_VCC_RESERVE1                                                                       0x03f6
+#define regMMEA0_SDP_VCC_RESERVE1_BASE_IDX                                                              0
+#define regMMEA0_SDP_VCD_RESERVE0                                                                       0x03f7
+#define regMMEA0_SDP_VCD_RESERVE0_BASE_IDX                                                              0
+#define regMMEA0_SDP_VCD_RESERVE1                                                                       0x03f8
+#define regMMEA0_SDP_VCD_RESERVE1_BASE_IDX                                                              0
+#define regMMEA0_SDP_REQ_CNTL                                                                           0x03f9
+#define regMMEA0_SDP_REQ_CNTL_BASE_IDX                                                                  0
+#define regMMEA0_MISC                                                                                   0x03fa
+#define regMMEA0_MISC_BASE_IDX                                                                          0
+#define regMMEA0_LATENCY_SAMPLING                                                                       0x03fb
+#define regMMEA0_LATENCY_SAMPLING_BASE_IDX                                                              0
+#define regMMEA0_PERFCOUNTER_LO                                                                         0x03fc
+#define regMMEA0_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regMMEA0_PERFCOUNTER_HI                                                                         0x03fd
+#define regMMEA0_PERFCOUNTER_HI_BASE_IDX                                                                0
+#define regMMEA0_PERFCOUNTER0_CFG                                                                       0x03fe
+#define regMMEA0_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regMMEA0_PERFCOUNTER1_CFG                                                                       0x03ff
+#define regMMEA0_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regMMEA0_PERFCOUNTER_RSLT_CNTL                                                                  0x0400
+#define regMMEA0_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+#define regMMEA0_DSM_CNTL                                                                               0x0408
+#define regMMEA0_DSM_CNTL_BASE_IDX                                                                      0
+#define regMMEA0_DSM_CNTLA                                                                              0x0409
+#define regMMEA0_DSM_CNTLA_BASE_IDX                                                                     0
+#define regMMEA0_DSM_CNTLB                                                                              0x040a
+#define regMMEA0_DSM_CNTLB_BASE_IDX                                                                     0
+#define regMMEA0_DSM_CNTL2                                                                              0x040b
+#define regMMEA0_DSM_CNTL2_BASE_IDX                                                                     0
+#define regMMEA0_DSM_CNTL2A                                                                             0x040c
+#define regMMEA0_DSM_CNTL2A_BASE_IDX                                                                    0
+#define regMMEA0_DSM_CNTL2B                                                                             0x040d
+#define regMMEA0_DSM_CNTL2B_BASE_IDX                                                                    0
+#define regMMEA0_CGTT_CLK_CTRL                                                                          0x040f
+#define regMMEA0_CGTT_CLK_CTRL_BASE_IDX                                                                 0
+#define regMMEA0_EDC_MODE                                                                               0x0410
+#define regMMEA0_EDC_MODE_BASE_IDX                                                                      0
+#define regMMEA0_ERR_STATUS                                                                             0x0411
+#define regMMEA0_ERR_STATUS_BASE_IDX                                                                    0
+#define regMMEA0_MISC2                                                                                  0x0412
+#define regMMEA0_MISC2_BASE_IDX                                                                         0
+#define regMMEA0_MISC_AON                                                                               0x0415
+#define regMMEA0_MISC_AON_BASE_IDX                                                                      0
+
+
+// addressBlock: aid_mmhub_ea_mmeadec1
+// base address: 0x61100
+#define regMMEA1_DRAM_RD_CLI2GRP_MAP0                                                                   0x0440
+#define regMMEA1_DRAM_RD_CLI2GRP_MAP0_BASE_IDX                                                          0
+#define regMMEA1_DRAM_RD_CLI2GRP_MAP1                                                                   0x0441
+#define regMMEA1_DRAM_RD_CLI2GRP_MAP1_BASE_IDX                                                          0
+#define regMMEA1_DRAM_WR_CLI2GRP_MAP0                                                                   0x0442
+#define regMMEA1_DRAM_WR_CLI2GRP_MAP0_BASE_IDX                                                          0
+#define regMMEA1_DRAM_WR_CLI2GRP_MAP1                                                                   0x0443
+#define regMMEA1_DRAM_WR_CLI2GRP_MAP1_BASE_IDX                                                          0
+#define regMMEA1_DRAM_RD_GRP2VC_MAP                                                                     0x0444
+#define regMMEA1_DRAM_RD_GRP2VC_MAP_BASE_IDX                                                            0
+#define regMMEA1_DRAM_WR_GRP2VC_MAP                                                                     0x0445
+#define regMMEA1_DRAM_WR_GRP2VC_MAP_BASE_IDX                                                            0
+#define regMMEA1_DRAM_RD_LAZY                                                                           0x0446
+#define regMMEA1_DRAM_RD_LAZY_BASE_IDX                                                                  0
+#define regMMEA1_DRAM_WR_LAZY                                                                           0x0447
+#define regMMEA1_DRAM_WR_LAZY_BASE_IDX                                                                  0
+#define regMMEA1_DRAM_RD_CAM_CNTL                                                                       0x0448
+#define regMMEA1_DRAM_RD_CAM_CNTL_BASE_IDX                                                              0
+#define regMMEA1_DRAM_WR_CAM_CNTL                                                                       0x0449
+#define regMMEA1_DRAM_WR_CAM_CNTL_BASE_IDX                                                              0
+#define regMMEA1_DRAM_PAGE_BURST                                                                        0x044a
+#define regMMEA1_DRAM_PAGE_BURST_BASE_IDX                                                               0
+#define regMMEA1_DRAM_RD_PRI_AGE                                                                        0x044b
+#define regMMEA1_DRAM_RD_PRI_AGE_BASE_IDX                                                               0
+#define regMMEA1_DRAM_WR_PRI_AGE                                                                        0x044c
+#define regMMEA1_DRAM_WR_PRI_AGE_BASE_IDX                                                               0
+#define regMMEA1_DRAM_RD_PRI_QUEUING                                                                    0x044d
+#define regMMEA1_DRAM_RD_PRI_QUEUING_BASE_IDX                                                           0
+#define regMMEA1_DRAM_WR_PRI_QUEUING                                                                    0x044e
+#define regMMEA1_DRAM_WR_PRI_QUEUING_BASE_IDX                                                           0
+#define regMMEA1_DRAM_RD_PRI_FIXED                                                                      0x044f
+#define regMMEA1_DRAM_RD_PRI_FIXED_BASE_IDX                                                             0
+#define regMMEA1_DRAM_WR_PRI_FIXED                                                                      0x0450
+#define regMMEA1_DRAM_WR_PRI_FIXED_BASE_IDX                                                             0
+#define regMMEA1_DRAM_RD_PRI_URGENCY                                                                    0x0451
+#define regMMEA1_DRAM_RD_PRI_URGENCY_BASE_IDX                                                           0
+#define regMMEA1_DRAM_WR_PRI_URGENCY                                                                    0x0452
+#define regMMEA1_DRAM_WR_PRI_URGENCY_BASE_IDX                                                           0
+#define regMMEA1_DRAM_RD_PRI_QUANT_PRI1                                                                 0x0453
+#define regMMEA1_DRAM_RD_PRI_QUANT_PRI1_BASE_IDX                                                        0
+#define regMMEA1_DRAM_RD_PRI_QUANT_PRI2                                                                 0x0454
+#define regMMEA1_DRAM_RD_PRI_QUANT_PRI2_BASE_IDX                                                        0
+#define regMMEA1_DRAM_RD_PRI_QUANT_PRI3                                                                 0x0455
+#define regMMEA1_DRAM_RD_PRI_QUANT_PRI3_BASE_IDX                                                        0
+#define regMMEA1_DRAM_WR_PRI_QUANT_PRI1                                                                 0x0456
+#define regMMEA1_DRAM_WR_PRI_QUANT_PRI1_BASE_IDX                                                        0
+#define regMMEA1_DRAM_WR_PRI_QUANT_PRI2                                                                 0x0457
+#define regMMEA1_DRAM_WR_PRI_QUANT_PRI2_BASE_IDX                                                        0
+#define regMMEA1_DRAM_WR_PRI_QUANT_PRI3                                                                 0x0458
+#define regMMEA1_DRAM_WR_PRI_QUANT_PRI3_BASE_IDX                                                        0
+#define regMMEA1_GMI_RD_CLI2GRP_MAP0                                                                    0x0459
+#define regMMEA1_GMI_RD_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regMMEA1_GMI_RD_CLI2GRP_MAP1                                                                    0x045a
+#define regMMEA1_GMI_RD_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regMMEA1_GMI_WR_CLI2GRP_MAP0                                                                    0x045b
+#define regMMEA1_GMI_WR_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regMMEA1_GMI_WR_CLI2GRP_MAP1                                                                    0x045c
+#define regMMEA1_GMI_WR_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regMMEA1_GMI_RD_GRP2VC_MAP                                                                      0x045d
+#define regMMEA1_GMI_RD_GRP2VC_MAP_BASE_IDX                                                             0
+#define regMMEA1_GMI_WR_GRP2VC_MAP                                                                      0x045e
+#define regMMEA1_GMI_WR_GRP2VC_MAP_BASE_IDX                                                             0
+#define regMMEA1_GMI_RD_LAZY                                                                            0x045f
+#define regMMEA1_GMI_RD_LAZY_BASE_IDX                                                                   0
+#define regMMEA1_GMI_WR_LAZY                                                                            0x0460
+#define regMMEA1_GMI_WR_LAZY_BASE_IDX                                                                   0
+#define regMMEA1_GMI_RD_CAM_CNTL                                                                        0x0461
+#define regMMEA1_GMI_RD_CAM_CNTL_BASE_IDX                                                               0
+#define regMMEA1_GMI_WR_CAM_CNTL                                                                        0x0462
+#define regMMEA1_GMI_WR_CAM_CNTL_BASE_IDX                                                               0
+#define regMMEA1_GMI_PAGE_BURST                                                                         0x0463
+#define regMMEA1_GMI_PAGE_BURST_BASE_IDX                                                                0
+#define regMMEA1_GMI_RD_PRI_AGE                                                                         0x0464
+#define regMMEA1_GMI_RD_PRI_AGE_BASE_IDX                                                                0
+#define regMMEA1_GMI_WR_PRI_AGE                                                                         0x0465
+#define regMMEA1_GMI_WR_PRI_AGE_BASE_IDX                                                                0
+#define regMMEA1_GMI_RD_PRI_QUEUING                                                                     0x0466
+#define regMMEA1_GMI_RD_PRI_QUEUING_BASE_IDX                                                            0
+#define regMMEA1_GMI_WR_PRI_QUEUING                                                                     0x0467
+#define regMMEA1_GMI_WR_PRI_QUEUING_BASE_IDX                                                            0
+#define regMMEA1_GMI_RD_PRI_FIXED                                                                       0x0468
+#define regMMEA1_GMI_RD_PRI_FIXED_BASE_IDX                                                              0
+#define regMMEA1_GMI_WR_PRI_FIXED                                                                       0x0469
+#define regMMEA1_GMI_WR_PRI_FIXED_BASE_IDX                                                              0
+#define regMMEA1_GMI_RD_PRI_URGENCY                                                                     0x046a
+#define regMMEA1_GMI_RD_PRI_URGENCY_BASE_IDX                                                            0
+#define regMMEA1_GMI_WR_PRI_URGENCY                                                                     0x046b
+#define regMMEA1_GMI_WR_PRI_URGENCY_BASE_IDX                                                            0
+#define regMMEA1_GMI_RD_PRI_URGENCY_MASKING                                                             0x046c
+#define regMMEA1_GMI_RD_PRI_URGENCY_MASKING_BASE_IDX                                                    0
+#define regMMEA1_GMI_WR_PRI_URGENCY_MASKING                                                             0x046d
+#define regMMEA1_GMI_WR_PRI_URGENCY_MASKING_BASE_IDX                                                    0
+#define regMMEA1_GMI_RD_PRI_QUANT_PRI1                                                                  0x046e
+#define regMMEA1_GMI_RD_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regMMEA1_GMI_RD_PRI_QUANT_PRI2                                                                  0x046f
+#define regMMEA1_GMI_RD_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regMMEA1_GMI_RD_PRI_QUANT_PRI3                                                                  0x0470
+#define regMMEA1_GMI_RD_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regMMEA1_GMI_WR_PRI_QUANT_PRI1                                                                  0x0471
+#define regMMEA1_GMI_WR_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regMMEA1_GMI_WR_PRI_QUANT_PRI2                                                                  0x0472
+#define regMMEA1_GMI_WR_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regMMEA1_GMI_WR_PRI_QUANT_PRI3                                                                  0x0473
+#define regMMEA1_GMI_WR_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regMMEA1_IO_RD_CLI2GRP_MAP0                                                                     0x0515
+#define regMMEA1_IO_RD_CLI2GRP_MAP0_BASE_IDX                                                            0
+#define regMMEA1_IO_RD_CLI2GRP_MAP1                                                                     0x0516
+#define regMMEA1_IO_RD_CLI2GRP_MAP1_BASE_IDX                                                            0
+#define regMMEA1_IO_WR_CLI2GRP_MAP0                                                                     0x0517
+#define regMMEA1_IO_WR_CLI2GRP_MAP0_BASE_IDX                                                            0
+#define regMMEA1_IO_WR_CLI2GRP_MAP1                                                                     0x0518
+#define regMMEA1_IO_WR_CLI2GRP_MAP1_BASE_IDX                                                            0
+#define regMMEA1_IO_RD_COMBINE_FLUSH                                                                    0x0519
+#define regMMEA1_IO_RD_COMBINE_FLUSH_BASE_IDX                                                           0
+#define regMMEA1_IO_WR_COMBINE_FLUSH                                                                    0x051a
+#define regMMEA1_IO_WR_COMBINE_FLUSH_BASE_IDX                                                           0
+#define regMMEA1_IO_GROUP_BURST                                                                         0x051b
+#define regMMEA1_IO_GROUP_BURST_BASE_IDX                                                                0
+#define regMMEA1_IO_RD_PRI_AGE                                                                          0x051c
+#define regMMEA1_IO_RD_PRI_AGE_BASE_IDX                                                                 0
+#define regMMEA1_IO_WR_PRI_AGE                                                                          0x051d
+#define regMMEA1_IO_WR_PRI_AGE_BASE_IDX                                                                 0
+#define regMMEA1_IO_RD_PRI_QUEUING                                                                      0x051e
+#define regMMEA1_IO_RD_PRI_QUEUING_BASE_IDX                                                             0
+#define regMMEA1_IO_WR_PRI_QUEUING                                                                      0x051f
+#define regMMEA1_IO_WR_PRI_QUEUING_BASE_IDX                                                             0
+#define regMMEA1_IO_RD_PRI_FIXED                                                                        0x0520
+#define regMMEA1_IO_RD_PRI_FIXED_BASE_IDX                                                               0
+#define regMMEA1_IO_WR_PRI_FIXED                                                                        0x0521
+#define regMMEA1_IO_WR_PRI_FIXED_BASE_IDX                                                               0
+#define regMMEA1_IO_RD_PRI_URGENCY                                                                      0x0522
+#define regMMEA1_IO_RD_PRI_URGENCY_BASE_IDX                                                             0
+#define regMMEA1_IO_WR_PRI_URGENCY                                                                      0x0523
+#define regMMEA1_IO_WR_PRI_URGENCY_BASE_IDX                                                             0
+#define regMMEA1_IO_RD_PRI_URGENCY_MASKING                                                              0x0524
+#define regMMEA1_IO_RD_PRI_URGENCY_MASKING_BASE_IDX                                                     0
+#define regMMEA1_IO_WR_PRI_URGENCY_MASKING                                                              0x0525
+#define regMMEA1_IO_WR_PRI_URGENCY_MASKING_BASE_IDX                                                     0
+#define regMMEA1_IO_RD_PRI_QUANT_PRI1                                                                   0x0526
+#define regMMEA1_IO_RD_PRI_QUANT_PRI1_BASE_IDX                                                          0
+#define regMMEA1_IO_RD_PRI_QUANT_PRI2                                                                   0x0527
+#define regMMEA1_IO_RD_PRI_QUANT_PRI2_BASE_IDX                                                          0
+#define regMMEA1_IO_RD_PRI_QUANT_PRI3                                                                   0x0528
+#define regMMEA1_IO_RD_PRI_QUANT_PRI3_BASE_IDX                                                          0
+#define regMMEA1_IO_WR_PRI_QUANT_PRI1                                                                   0x0529
+#define regMMEA1_IO_WR_PRI_QUANT_PRI1_BASE_IDX                                                          0
+#define regMMEA1_IO_WR_PRI_QUANT_PRI2                                                                   0x052a
+#define regMMEA1_IO_WR_PRI_QUANT_PRI2_BASE_IDX                                                          0
+#define regMMEA1_IO_WR_PRI_QUANT_PRI3                                                                   0x052b
+#define regMMEA1_IO_WR_PRI_QUANT_PRI3_BASE_IDX                                                          0
+#define regMMEA1_SDP_ARB_DRAM                                                                           0x052c
+#define regMMEA1_SDP_ARB_DRAM_BASE_IDX                                                                  0
+#define regMMEA1_SDP_ARB_GMI                                                                            0x052d
+#define regMMEA1_SDP_ARB_GMI_BASE_IDX                                                                   0
+#define regMMEA1_SDP_ARB_FINAL                                                                          0x052e
+#define regMMEA1_SDP_ARB_FINAL_BASE_IDX                                                                 0
+#define regMMEA1_SDP_DRAM_PRIORITY                                                                      0x052f
+#define regMMEA1_SDP_DRAM_PRIORITY_BASE_IDX                                                             0
+#define regMMEA1_SDP_GMI_PRIORITY                                                                       0x0530
+#define regMMEA1_SDP_GMI_PRIORITY_BASE_IDX                                                              0
+#define regMMEA1_SDP_IO_PRIORITY                                                                        0x0531
+#define regMMEA1_SDP_IO_PRIORITY_BASE_IDX                                                               0
+#define regMMEA1_SDP_CREDITS                                                                            0x0532
+#define regMMEA1_SDP_CREDITS_BASE_IDX                                                                   0
+#define regMMEA1_SDP_TAG_RESERVE0                                                                       0x0533
+#define regMMEA1_SDP_TAG_RESERVE0_BASE_IDX                                                              0
+#define regMMEA1_SDP_TAG_RESERVE1                                                                       0x0534
+#define regMMEA1_SDP_TAG_RESERVE1_BASE_IDX                                                              0
+#define regMMEA1_SDP_VCC_RESERVE0                                                                       0x0535
+#define regMMEA1_SDP_VCC_RESERVE0_BASE_IDX                                                              0
+#define regMMEA1_SDP_VCC_RESERVE1                                                                       0x0536
+#define regMMEA1_SDP_VCC_RESERVE1_BASE_IDX                                                              0
+#define regMMEA1_SDP_VCD_RESERVE0                                                                       0x0537
+#define regMMEA1_SDP_VCD_RESERVE0_BASE_IDX                                                              0
+#define regMMEA1_SDP_VCD_RESERVE1                                                                       0x0538
+#define regMMEA1_SDP_VCD_RESERVE1_BASE_IDX                                                              0
+#define regMMEA1_SDP_REQ_CNTL                                                                           0x0539
+#define regMMEA1_SDP_REQ_CNTL_BASE_IDX                                                                  0
+#define regMMEA1_MISC                                                                                   0x053a
+#define regMMEA1_MISC_BASE_IDX                                                                          0
+#define regMMEA1_LATENCY_SAMPLING                                                                       0x053b
+#define regMMEA1_LATENCY_SAMPLING_BASE_IDX                                                              0
+#define regMMEA1_PERFCOUNTER_LO                                                                         0x053c
+#define regMMEA1_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regMMEA1_PERFCOUNTER_HI                                                                         0x053d
+#define regMMEA1_PERFCOUNTER_HI_BASE_IDX                                                                0
+#define regMMEA1_PERFCOUNTER0_CFG                                                                       0x053e
+#define regMMEA1_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regMMEA1_PERFCOUNTER1_CFG                                                                       0x053f
+#define regMMEA1_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regMMEA1_PERFCOUNTER_RSLT_CNTL                                                                  0x0540
+#define regMMEA1_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+#define regMMEA1_DSM_CNTL                                                                               0x0548
+#define regMMEA1_DSM_CNTL_BASE_IDX                                                                      0
+#define regMMEA1_DSM_CNTLA                                                                              0x0549
+#define regMMEA1_DSM_CNTLA_BASE_IDX                                                                     0
+#define regMMEA1_DSM_CNTLB                                                                              0x054a
+#define regMMEA1_DSM_CNTLB_BASE_IDX                                                                     0
+#define regMMEA1_DSM_CNTL2                                                                              0x054b
+#define regMMEA1_DSM_CNTL2_BASE_IDX                                                                     0
+#define regMMEA1_DSM_CNTL2A                                                                             0x054c
+#define regMMEA1_DSM_CNTL2A_BASE_IDX                                                                    0
+#define regMMEA1_DSM_CNTL2B                                                                             0x054d
+#define regMMEA1_DSM_CNTL2B_BASE_IDX                                                                    0
+#define regMMEA1_CGTT_CLK_CTRL                                                                          0x054f
+#define regMMEA1_CGTT_CLK_CTRL_BASE_IDX                                                                 0
+#define regMMEA1_EDC_MODE                                                                               0x0550
+#define regMMEA1_EDC_MODE_BASE_IDX                                                                      0
+#define regMMEA1_ERR_STATUS                                                                             0x0551
+#define regMMEA1_ERR_STATUS_BASE_IDX                                                                    0
+#define regMMEA1_MISC2                                                                                  0x0552
+#define regMMEA1_MISC2_BASE_IDX                                                                         0
+#define regMMEA1_MISC_AON                                                                               0x0555
+#define regMMEA1_MISC_AON_BASE_IDX                                                                      0
+
+
+// addressBlock: aid_mmhub_ea_mmeadec2
+// base address: 0x61600
+#define regMMEA2_DRAM_RD_CLI2GRP_MAP0                                                                   0x0580
+#define regMMEA2_DRAM_RD_CLI2GRP_MAP0_BASE_IDX                                                          0
+#define regMMEA2_DRAM_RD_CLI2GRP_MAP1                                                                   0x0581
+#define regMMEA2_DRAM_RD_CLI2GRP_MAP1_BASE_IDX                                                          0
+#define regMMEA2_DRAM_WR_CLI2GRP_MAP0                                                                   0x0582
+#define regMMEA2_DRAM_WR_CLI2GRP_MAP0_BASE_IDX                                                          0
+#define regMMEA2_DRAM_WR_CLI2GRP_MAP1                                                                   0x0583
+#define regMMEA2_DRAM_WR_CLI2GRP_MAP1_BASE_IDX                                                          0
+#define regMMEA2_DRAM_RD_GRP2VC_MAP                                                                     0x0584
+#define regMMEA2_DRAM_RD_GRP2VC_MAP_BASE_IDX                                                            0
+#define regMMEA2_DRAM_WR_GRP2VC_MAP                                                                     0x0585
+#define regMMEA2_DRAM_WR_GRP2VC_MAP_BASE_IDX                                                            0
+#define regMMEA2_DRAM_RD_LAZY                                                                           0x0586
+#define regMMEA2_DRAM_RD_LAZY_BASE_IDX                                                                  0
+#define regMMEA2_DRAM_WR_LAZY                                                                           0x0587
+#define regMMEA2_DRAM_WR_LAZY_BASE_IDX                                                                  0
+#define regMMEA2_DRAM_RD_CAM_CNTL                                                                       0x0588
+#define regMMEA2_DRAM_RD_CAM_CNTL_BASE_IDX                                                              0
+#define regMMEA2_DRAM_WR_CAM_CNTL                                                                       0x0589
+#define regMMEA2_DRAM_WR_CAM_CNTL_BASE_IDX                                                              0
+#define regMMEA2_DRAM_PAGE_BURST                                                                        0x058a
+#define regMMEA2_DRAM_PAGE_BURST_BASE_IDX                                                               0
+#define regMMEA2_DRAM_RD_PRI_AGE                                                                        0x058b
+#define regMMEA2_DRAM_RD_PRI_AGE_BASE_IDX                                                               0
+#define regMMEA2_DRAM_WR_PRI_AGE                                                                        0x058c
+#define regMMEA2_DRAM_WR_PRI_AGE_BASE_IDX                                                               0
+#define regMMEA2_DRAM_RD_PRI_QUEUING                                                                    0x058d
+#define regMMEA2_DRAM_RD_PRI_QUEUING_BASE_IDX                                                           0
+#define regMMEA2_DRAM_WR_PRI_QUEUING                                                                    0x058e
+#define regMMEA2_DRAM_WR_PRI_QUEUING_BASE_IDX                                                           0
+#define regMMEA2_DRAM_RD_PRI_FIXED                                                                      0x058f
+#define regMMEA2_DRAM_RD_PRI_FIXED_BASE_IDX                                                             0
+#define regMMEA2_DRAM_WR_PRI_FIXED                                                                      0x0590
+#define regMMEA2_DRAM_WR_PRI_FIXED_BASE_IDX                                                             0
+#define regMMEA2_DRAM_RD_PRI_URGENCY                                                                    0x0591
+#define regMMEA2_DRAM_RD_PRI_URGENCY_BASE_IDX                                                           0
+#define regMMEA2_DRAM_WR_PRI_URGENCY                                                                    0x0592
+#define regMMEA2_DRAM_WR_PRI_URGENCY_BASE_IDX                                                           0
+#define regMMEA2_DRAM_RD_PRI_QUANT_PRI1                                                                 0x0593
+#define regMMEA2_DRAM_RD_PRI_QUANT_PRI1_BASE_IDX                                                        0
+#define regMMEA2_DRAM_RD_PRI_QUANT_PRI2                                                                 0x0594
+#define regMMEA2_DRAM_RD_PRI_QUANT_PRI2_BASE_IDX                                                        0
+#define regMMEA2_DRAM_RD_PRI_QUANT_PRI3                                                                 0x0595
+#define regMMEA2_DRAM_RD_PRI_QUANT_PRI3_BASE_IDX                                                        0
+#define regMMEA2_DRAM_WR_PRI_QUANT_PRI1                                                                 0x0596
+#define regMMEA2_DRAM_WR_PRI_QUANT_PRI1_BASE_IDX                                                        0
+#define regMMEA2_DRAM_WR_PRI_QUANT_PRI2                                                                 0x0597
+#define regMMEA2_DRAM_WR_PRI_QUANT_PRI2_BASE_IDX                                                        0
+#define regMMEA2_DRAM_WR_PRI_QUANT_PRI3                                                                 0x0598
+#define regMMEA2_DRAM_WR_PRI_QUANT_PRI3_BASE_IDX                                                        0
+#define regMMEA2_GMI_RD_CLI2GRP_MAP0                                                                    0x0599
+#define regMMEA2_GMI_RD_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regMMEA2_GMI_RD_CLI2GRP_MAP1                                                                    0x059a
+#define regMMEA2_GMI_RD_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regMMEA2_GMI_WR_CLI2GRP_MAP0                                                                    0x059b
+#define regMMEA2_GMI_WR_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regMMEA2_GMI_WR_CLI2GRP_MAP1                                                                    0x059c
+#define regMMEA2_GMI_WR_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regMMEA2_GMI_RD_GRP2VC_MAP                                                                      0x059d
+#define regMMEA2_GMI_RD_GRP2VC_MAP_BASE_IDX                                                             0
+#define regMMEA2_GMI_WR_GRP2VC_MAP                                                                      0x059e
+#define regMMEA2_GMI_WR_GRP2VC_MAP_BASE_IDX                                                             0
+#define regMMEA2_GMI_RD_LAZY                                                                            0x059f
+#define regMMEA2_GMI_RD_LAZY_BASE_IDX                                                                   0
+#define regMMEA2_GMI_WR_LAZY                                                                            0x05a0
+#define regMMEA2_GMI_WR_LAZY_BASE_IDX                                                                   0
+#define regMMEA2_GMI_RD_CAM_CNTL                                                                        0x05a1
+#define regMMEA2_GMI_RD_CAM_CNTL_BASE_IDX                                                               0
+#define regMMEA2_GMI_WR_CAM_CNTL                                                                        0x05a2
+#define regMMEA2_GMI_WR_CAM_CNTL_BASE_IDX                                                               0
+#define regMMEA2_GMI_PAGE_BURST                                                                         0x05a3
+#define regMMEA2_GMI_PAGE_BURST_BASE_IDX                                                                0
+#define regMMEA2_GMI_RD_PRI_AGE                                                                         0x05a4
+#define regMMEA2_GMI_RD_PRI_AGE_BASE_IDX                                                                0
+#define regMMEA2_GMI_WR_PRI_AGE                                                                         0x05a5
+#define regMMEA2_GMI_WR_PRI_AGE_BASE_IDX                                                                0
+#define regMMEA2_GMI_RD_PRI_QUEUING                                                                     0x05a6
+#define regMMEA2_GMI_RD_PRI_QUEUING_BASE_IDX                                                            0
+#define regMMEA2_GMI_WR_PRI_QUEUING                                                                     0x05a7
+#define regMMEA2_GMI_WR_PRI_QUEUING_BASE_IDX                                                            0
+#define regMMEA2_GMI_RD_PRI_FIXED                                                                       0x05a8
+#define regMMEA2_GMI_RD_PRI_FIXED_BASE_IDX                                                              0
+#define regMMEA2_GMI_WR_PRI_FIXED                                                                       0x05a9
+#define regMMEA2_GMI_WR_PRI_FIXED_BASE_IDX                                                              0
+#define regMMEA2_GMI_RD_PRI_URGENCY                                                                     0x05aa
+#define regMMEA2_GMI_RD_PRI_URGENCY_BASE_IDX                                                            0
+#define regMMEA2_GMI_WR_PRI_URGENCY                                                                     0x05ab
+#define regMMEA2_GMI_WR_PRI_URGENCY_BASE_IDX                                                            0
+#define regMMEA2_GMI_RD_PRI_URGENCY_MASKING                                                             0x05ac
+#define regMMEA2_GMI_RD_PRI_URGENCY_MASKING_BASE_IDX                                                    0
+#define regMMEA2_GMI_WR_PRI_URGENCY_MASKING                                                             0x05ad
+#define regMMEA2_GMI_WR_PRI_URGENCY_MASKING_BASE_IDX                                                    0
+#define regMMEA2_GMI_RD_PRI_QUANT_PRI1                                                                  0x05ae
+#define regMMEA2_GMI_RD_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regMMEA2_GMI_RD_PRI_QUANT_PRI2                                                                  0x05af
+#define regMMEA2_GMI_RD_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regMMEA2_GMI_RD_PRI_QUANT_PRI3                                                                  0x05b0
+#define regMMEA2_GMI_RD_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regMMEA2_GMI_WR_PRI_QUANT_PRI1                                                                  0x05b1
+#define regMMEA2_GMI_WR_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regMMEA2_GMI_WR_PRI_QUANT_PRI2                                                                  0x05b2
+#define regMMEA2_GMI_WR_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regMMEA2_GMI_WR_PRI_QUANT_PRI3                                                                  0x05b3
+#define regMMEA2_GMI_WR_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regMMEA2_IO_RD_CLI2GRP_MAP0                                                                     0x0655
+#define regMMEA2_IO_RD_CLI2GRP_MAP0_BASE_IDX                                                            0
+#define regMMEA2_IO_RD_CLI2GRP_MAP1                                                                     0x0656
+#define regMMEA2_IO_RD_CLI2GRP_MAP1_BASE_IDX                                                            0
+#define regMMEA2_IO_WR_CLI2GRP_MAP0                                                                     0x0657
+#define regMMEA2_IO_WR_CLI2GRP_MAP0_BASE_IDX                                                            0
+#define regMMEA2_IO_WR_CLI2GRP_MAP1                                                                     0x0658
+#define regMMEA2_IO_WR_CLI2GRP_MAP1_BASE_IDX                                                            0
+#define regMMEA2_IO_RD_COMBINE_FLUSH                                                                    0x0659
+#define regMMEA2_IO_RD_COMBINE_FLUSH_BASE_IDX                                                           0
+#define regMMEA2_IO_WR_COMBINE_FLUSH                                                                    0x065a
+#define regMMEA2_IO_WR_COMBINE_FLUSH_BASE_IDX                                                           0
+#define regMMEA2_IO_GROUP_BURST                                                                         0x065b
+#define regMMEA2_IO_GROUP_BURST_BASE_IDX                                                                0
+#define regMMEA2_IO_RD_PRI_AGE                                                                          0x065c
+#define regMMEA2_IO_RD_PRI_AGE_BASE_IDX                                                                 0
+#define regMMEA2_IO_WR_PRI_AGE                                                                          0x065d
+#define regMMEA2_IO_WR_PRI_AGE_BASE_IDX                                                                 0
+#define regMMEA2_IO_RD_PRI_QUEUING                                                                      0x065e
+#define regMMEA2_IO_RD_PRI_QUEUING_BASE_IDX                                                             0
+#define regMMEA2_IO_WR_PRI_QUEUING                                                                      0x065f
+#define regMMEA2_IO_WR_PRI_QUEUING_BASE_IDX                                                             0
+#define regMMEA2_IO_RD_PRI_FIXED                                                                        0x0660
+#define regMMEA2_IO_RD_PRI_FIXED_BASE_IDX                                                               0
+#define regMMEA2_IO_WR_PRI_FIXED                                                                        0x0661
+#define regMMEA2_IO_WR_PRI_FIXED_BASE_IDX                                                               0
+#define regMMEA2_IO_RD_PRI_URGENCY                                                                      0x0662
+#define regMMEA2_IO_RD_PRI_URGENCY_BASE_IDX                                                             0
+#define regMMEA2_IO_WR_PRI_URGENCY                                                                      0x0663
+#define regMMEA2_IO_WR_PRI_URGENCY_BASE_IDX                                                             0
+#define regMMEA2_IO_RD_PRI_URGENCY_MASKING                                                              0x0664
+#define regMMEA2_IO_RD_PRI_URGENCY_MASKING_BASE_IDX                                                     0
+#define regMMEA2_IO_WR_PRI_URGENCY_MASKING                                                              0x0665
+#define regMMEA2_IO_WR_PRI_URGENCY_MASKING_BASE_IDX                                                     0
+#define regMMEA2_IO_RD_PRI_QUANT_PRI1                                                                   0x0666
+#define regMMEA2_IO_RD_PRI_QUANT_PRI1_BASE_IDX                                                          0
+#define regMMEA2_IO_RD_PRI_QUANT_PRI2                                                                   0x0667
+#define regMMEA2_IO_RD_PRI_QUANT_PRI2_BASE_IDX                                                          0
+#define regMMEA2_IO_RD_PRI_QUANT_PRI3                                                                   0x0668
+#define regMMEA2_IO_RD_PRI_QUANT_PRI3_BASE_IDX                                                          0
+#define regMMEA2_IO_WR_PRI_QUANT_PRI1                                                                   0x0669
+#define regMMEA2_IO_WR_PRI_QUANT_PRI1_BASE_IDX                                                          0
+#define regMMEA2_IO_WR_PRI_QUANT_PRI2                                                                   0x066a
+#define regMMEA2_IO_WR_PRI_QUANT_PRI2_BASE_IDX                                                          0
+#define regMMEA2_IO_WR_PRI_QUANT_PRI3                                                                   0x066b
+#define regMMEA2_IO_WR_PRI_QUANT_PRI3_BASE_IDX                                                          0
+#define regMMEA2_SDP_ARB_DRAM                                                                           0x066c
+#define regMMEA2_SDP_ARB_DRAM_BASE_IDX                                                                  0
+#define regMMEA2_SDP_ARB_GMI                                                                            0x066d
+#define regMMEA2_SDP_ARB_GMI_BASE_IDX                                                                   0
+#define regMMEA2_SDP_ARB_FINAL                                                                          0x066e
+#define regMMEA2_SDP_ARB_FINAL_BASE_IDX                                                                 0
+#define regMMEA2_SDP_DRAM_PRIORITY                                                                      0x066f
+#define regMMEA2_SDP_DRAM_PRIORITY_BASE_IDX                                                             0
+#define regMMEA2_SDP_GMI_PRIORITY                                                                       0x0670
+#define regMMEA2_SDP_GMI_PRIORITY_BASE_IDX                                                              0
+#define regMMEA2_SDP_IO_PRIORITY                                                                        0x0671
+#define regMMEA2_SDP_IO_PRIORITY_BASE_IDX                                                               0
+#define regMMEA2_SDP_CREDITS                                                                            0x0672
+#define regMMEA2_SDP_CREDITS_BASE_IDX                                                                   0
+#define regMMEA2_SDP_TAG_RESERVE0                                                                       0x0673
+#define regMMEA2_SDP_TAG_RESERVE0_BASE_IDX                                                              0
+#define regMMEA2_SDP_TAG_RESERVE1                                                                       0x0674
+#define regMMEA2_SDP_TAG_RESERVE1_BASE_IDX                                                              0
+#define regMMEA2_SDP_VCC_RESERVE0                                                                       0x0675
+#define regMMEA2_SDP_VCC_RESERVE0_BASE_IDX                                                              0
+#define regMMEA2_SDP_VCC_RESERVE1                                                                       0x0676
+#define regMMEA2_SDP_VCC_RESERVE1_BASE_IDX                                                              0
+#define regMMEA2_SDP_VCD_RESERVE0                                                                       0x0677
+#define regMMEA2_SDP_VCD_RESERVE0_BASE_IDX                                                              0
+#define regMMEA2_SDP_VCD_RESERVE1                                                                       0x0678
+#define regMMEA2_SDP_VCD_RESERVE1_BASE_IDX                                                              0
+#define regMMEA2_SDP_REQ_CNTL                                                                           0x0679
+#define regMMEA2_SDP_REQ_CNTL_BASE_IDX                                                                  0
+#define regMMEA2_MISC                                                                                   0x067a
+#define regMMEA2_MISC_BASE_IDX                                                                          0
+#define regMMEA2_LATENCY_SAMPLING                                                                       0x067b
+#define regMMEA2_LATENCY_SAMPLING_BASE_IDX                                                              0
+#define regMMEA2_PERFCOUNTER_LO                                                                         0x067c
+#define regMMEA2_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regMMEA2_PERFCOUNTER_HI                                                                         0x067d
+#define regMMEA2_PERFCOUNTER_HI_BASE_IDX                                                                0
+#define regMMEA2_PERFCOUNTER0_CFG                                                                       0x067e
+#define regMMEA2_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regMMEA2_PERFCOUNTER1_CFG                                                                       0x067f
+#define regMMEA2_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regMMEA2_PERFCOUNTER_RSLT_CNTL                                                                  0x0680
+#define regMMEA2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+#define regMMEA2_DSM_CNTL                                                                               0x0688
+#define regMMEA2_DSM_CNTL_BASE_IDX                                                                      0
+#define regMMEA2_DSM_CNTLA                                                                              0x0689
+#define regMMEA2_DSM_CNTLA_BASE_IDX                                                                     0
+#define regMMEA2_DSM_CNTLB                                                                              0x068a
+#define regMMEA2_DSM_CNTLB_BASE_IDX                                                                     0
+#define regMMEA2_DSM_CNTL2                                                                              0x068b
+#define regMMEA2_DSM_CNTL2_BASE_IDX                                                                     0
+#define regMMEA2_DSM_CNTL2A                                                                             0x068c
+#define regMMEA2_DSM_CNTL2A_BASE_IDX                                                                    0
+#define regMMEA2_DSM_CNTL2B                                                                             0x068d
+#define regMMEA2_DSM_CNTL2B_BASE_IDX                                                                    0
+#define regMMEA2_CGTT_CLK_CTRL                                                                          0x068f
+#define regMMEA2_CGTT_CLK_CTRL_BASE_IDX                                                                 0
+#define regMMEA2_EDC_MODE                                                                               0x0690
+#define regMMEA2_EDC_MODE_BASE_IDX                                                                      0
+#define regMMEA2_ERR_STATUS                                                                             0x0691
+#define regMMEA2_ERR_STATUS_BASE_IDX                                                                    0
+#define regMMEA2_MISC2                                                                                  0x0692
+#define regMMEA2_MISC2_BASE_IDX                                                                         0
+#define regMMEA2_MISC_AON                                                                               0x0695
+#define regMMEA2_MISC_AON_BASE_IDX                                                                      0
+
+
+// addressBlock: aid_mmhub_ea_mmeadec3
+// base address: 0x61b00
+#define regMMEA3_DRAM_RD_CLI2GRP_MAP0                                                                   0x06c0
+#define regMMEA3_DRAM_RD_CLI2GRP_MAP0_BASE_IDX                                                          0
+#define regMMEA3_DRAM_RD_CLI2GRP_MAP1                                                                   0x06c1
+#define regMMEA3_DRAM_RD_CLI2GRP_MAP1_BASE_IDX                                                          0
+#define regMMEA3_DRAM_WR_CLI2GRP_MAP0                                                                   0x06c2
+#define regMMEA3_DRAM_WR_CLI2GRP_MAP0_BASE_IDX                                                          0
+#define regMMEA3_DRAM_WR_CLI2GRP_MAP1                                                                   0x06c3
+#define regMMEA3_DRAM_WR_CLI2GRP_MAP1_BASE_IDX                                                          0
+#define regMMEA3_DRAM_RD_GRP2VC_MAP                                                                     0x06c4
+#define regMMEA3_DRAM_RD_GRP2VC_MAP_BASE_IDX                                                            0
+#define regMMEA3_DRAM_WR_GRP2VC_MAP                                                                     0x06c5
+#define regMMEA3_DRAM_WR_GRP2VC_MAP_BASE_IDX                                                            0
+#define regMMEA3_DRAM_RD_LAZY                                                                           0x06c6
+#define regMMEA3_DRAM_RD_LAZY_BASE_IDX                                                                  0
+#define regMMEA3_DRAM_WR_LAZY                                                                           0x06c7
+#define regMMEA3_DRAM_WR_LAZY_BASE_IDX                                                                  0
+#define regMMEA3_DRAM_RD_CAM_CNTL                                                                       0x06c8
+#define regMMEA3_DRAM_RD_CAM_CNTL_BASE_IDX                                                              0
+#define regMMEA3_DRAM_WR_CAM_CNTL                                                                       0x06c9
+#define regMMEA3_DRAM_WR_CAM_CNTL_BASE_IDX                                                              0
+#define regMMEA3_DRAM_PAGE_BURST                                                                        0x06ca
+#define regMMEA3_DRAM_PAGE_BURST_BASE_IDX                                                               0
+#define regMMEA3_DRAM_RD_PRI_AGE                                                                        0x06cb
+#define regMMEA3_DRAM_RD_PRI_AGE_BASE_IDX                                                               0
+#define regMMEA3_DRAM_WR_PRI_AGE                                                                        0x06cc
+#define regMMEA3_DRAM_WR_PRI_AGE_BASE_IDX                                                               0
+#define regMMEA3_DRAM_RD_PRI_QUEUING                                                                    0x06cd
+#define regMMEA3_DRAM_RD_PRI_QUEUING_BASE_IDX                                                           0
+#define regMMEA3_DRAM_WR_PRI_QUEUING                                                                    0x06ce
+#define regMMEA3_DRAM_WR_PRI_QUEUING_BASE_IDX                                                           0
+#define regMMEA3_DRAM_RD_PRI_FIXED                                                                      0x06cf
+#define regMMEA3_DRAM_RD_PRI_FIXED_BASE_IDX                                                             0
+#define regMMEA3_DRAM_WR_PRI_FIXED                                                                      0x06d0
+#define regMMEA3_DRAM_WR_PRI_FIXED_BASE_IDX                                                             0
+#define regMMEA3_DRAM_RD_PRI_URGENCY                                                                    0x06d1
+#define regMMEA3_DRAM_RD_PRI_URGENCY_BASE_IDX                                                           0
+#define regMMEA3_DRAM_WR_PRI_URGENCY                                                                    0x06d2
+#define regMMEA3_DRAM_WR_PRI_URGENCY_BASE_IDX                                                           0
+#define regMMEA3_DRAM_RD_PRI_QUANT_PRI1                                                                 0x06d3
+#define regMMEA3_DRAM_RD_PRI_QUANT_PRI1_BASE_IDX                                                        0
+#define regMMEA3_DRAM_RD_PRI_QUANT_PRI2                                                                 0x06d4
+#define regMMEA3_DRAM_RD_PRI_QUANT_PRI2_BASE_IDX                                                        0
+#define regMMEA3_DRAM_RD_PRI_QUANT_PRI3                                                                 0x06d5
+#define regMMEA3_DRAM_RD_PRI_QUANT_PRI3_BASE_IDX                                                        0
+#define regMMEA3_DRAM_WR_PRI_QUANT_PRI1                                                                 0x06d6
+#define regMMEA3_DRAM_WR_PRI_QUANT_PRI1_BASE_IDX                                                        0
+#define regMMEA3_DRAM_WR_PRI_QUANT_PRI2                                                                 0x06d7
+#define regMMEA3_DRAM_WR_PRI_QUANT_PRI2_BASE_IDX                                                        0
+#define regMMEA3_DRAM_WR_PRI_QUANT_PRI3                                                                 0x06d8
+#define regMMEA3_DRAM_WR_PRI_QUANT_PRI3_BASE_IDX                                                        0
+#define regMMEA3_GMI_RD_CLI2GRP_MAP0                                                                    0x06d9
+#define regMMEA3_GMI_RD_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regMMEA3_GMI_RD_CLI2GRP_MAP1                                                                    0x06da
+#define regMMEA3_GMI_RD_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regMMEA3_GMI_WR_CLI2GRP_MAP0                                                                    0x06db
+#define regMMEA3_GMI_WR_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regMMEA3_GMI_WR_CLI2GRP_MAP1                                                                    0x06dc
+#define regMMEA3_GMI_WR_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regMMEA3_GMI_RD_GRP2VC_MAP                                                                      0x06dd
+#define regMMEA3_GMI_RD_GRP2VC_MAP_BASE_IDX                                                             0
+#define regMMEA3_GMI_WR_GRP2VC_MAP                                                                      0x06de
+#define regMMEA3_GMI_WR_GRP2VC_MAP_BASE_IDX                                                             0
+#define regMMEA3_GMI_RD_LAZY                                                                            0x06df
+#define regMMEA3_GMI_RD_LAZY_BASE_IDX                                                                   0
+#define regMMEA3_GMI_WR_LAZY                                                                            0x06e0
+#define regMMEA3_GMI_WR_LAZY_BASE_IDX                                                                   0
+#define regMMEA3_GMI_RD_CAM_CNTL                                                                        0x06e1
+#define regMMEA3_GMI_RD_CAM_CNTL_BASE_IDX                                                               0
+#define regMMEA3_GMI_WR_CAM_CNTL                                                                        0x06e2
+#define regMMEA3_GMI_WR_CAM_CNTL_BASE_IDX                                                               0
+#define regMMEA3_GMI_PAGE_BURST                                                                         0x06e3
+#define regMMEA3_GMI_PAGE_BURST_BASE_IDX                                                                0
+#define regMMEA3_GMI_RD_PRI_AGE                                                                         0x06e4
+#define regMMEA3_GMI_RD_PRI_AGE_BASE_IDX                                                                0
+#define regMMEA3_GMI_WR_PRI_AGE                                                                         0x06e5
+#define regMMEA3_GMI_WR_PRI_AGE_BASE_IDX                                                                0
+#define regMMEA3_GMI_RD_PRI_QUEUING                                                                     0x06e6
+#define regMMEA3_GMI_RD_PRI_QUEUING_BASE_IDX                                                            0
+#define regMMEA3_GMI_WR_PRI_QUEUING                                                                     0x06e7
+#define regMMEA3_GMI_WR_PRI_QUEUING_BASE_IDX                                                            0
+#define regMMEA3_GMI_RD_PRI_FIXED                                                                       0x06e8
+#define regMMEA3_GMI_RD_PRI_FIXED_BASE_IDX                                                              0
+#define regMMEA3_GMI_WR_PRI_FIXED                                                                       0x06e9
+#define regMMEA3_GMI_WR_PRI_FIXED_BASE_IDX                                                              0
+#define regMMEA3_GMI_RD_PRI_URGENCY                                                                     0x06ea
+#define regMMEA3_GMI_RD_PRI_URGENCY_BASE_IDX                                                            0
+#define regMMEA3_GMI_WR_PRI_URGENCY                                                                     0x06eb
+#define regMMEA3_GMI_WR_PRI_URGENCY_BASE_IDX                                                            0
+#define regMMEA3_GMI_RD_PRI_URGENCY_MASKING                                                             0x06ec
+#define regMMEA3_GMI_RD_PRI_URGENCY_MASKING_BASE_IDX                                                    0
+#define regMMEA3_GMI_WR_PRI_URGENCY_MASKING                                                             0x06ed
+#define regMMEA3_GMI_WR_PRI_URGENCY_MASKING_BASE_IDX                                                    0
+#define regMMEA3_GMI_RD_PRI_QUANT_PRI1                                                                  0x06ee
+#define regMMEA3_GMI_RD_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regMMEA3_GMI_RD_PRI_QUANT_PRI2                                                                  0x06ef
+#define regMMEA3_GMI_RD_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regMMEA3_GMI_RD_PRI_QUANT_PRI3                                                                  0x06f0
+#define regMMEA3_GMI_RD_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regMMEA3_GMI_WR_PRI_QUANT_PRI1                                                                  0x06f1
+#define regMMEA3_GMI_WR_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regMMEA3_GMI_WR_PRI_QUANT_PRI2                                                                  0x06f2
+#define regMMEA3_GMI_WR_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regMMEA3_GMI_WR_PRI_QUANT_PRI3                                                                  0x06f3
+#define regMMEA3_GMI_WR_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regMMEA3_IO_RD_CLI2GRP_MAP0                                                                     0x0795
+#define regMMEA3_IO_RD_CLI2GRP_MAP0_BASE_IDX                                                            0
+#define regMMEA3_IO_RD_CLI2GRP_MAP1                                                                     0x0796
+#define regMMEA3_IO_RD_CLI2GRP_MAP1_BASE_IDX                                                            0
+#define regMMEA3_IO_WR_CLI2GRP_MAP0                                                                     0x0797
+#define regMMEA3_IO_WR_CLI2GRP_MAP0_BASE_IDX                                                            0
+#define regMMEA3_IO_WR_CLI2GRP_MAP1                                                                     0x0798
+#define regMMEA3_IO_WR_CLI2GRP_MAP1_BASE_IDX                                                            0
+#define regMMEA3_IO_RD_COMBINE_FLUSH                                                                    0x0799
+#define regMMEA3_IO_RD_COMBINE_FLUSH_BASE_IDX                                                           0
+#define regMMEA3_IO_WR_COMBINE_FLUSH                                                                    0x079a
+#define regMMEA3_IO_WR_COMBINE_FLUSH_BASE_IDX                                                           0
+#define regMMEA3_IO_GROUP_BURST                                                                         0x079b
+#define regMMEA3_IO_GROUP_BURST_BASE_IDX                                                                0
+#define regMMEA3_IO_RD_PRI_AGE                                                                          0x079c
+#define regMMEA3_IO_RD_PRI_AGE_BASE_IDX                                                                 0
+#define regMMEA3_IO_WR_PRI_AGE                                                                          0x079d
+#define regMMEA3_IO_WR_PRI_AGE_BASE_IDX                                                                 0
+#define regMMEA3_IO_RD_PRI_QUEUING                                                                      0x079e
+#define regMMEA3_IO_RD_PRI_QUEUING_BASE_IDX                                                             0
+#define regMMEA3_IO_WR_PRI_QUEUING                                                                      0x079f
+#define regMMEA3_IO_WR_PRI_QUEUING_BASE_IDX                                                             0
+#define regMMEA3_IO_RD_PRI_FIXED                                                                        0x07a0
+#define regMMEA3_IO_RD_PRI_FIXED_BASE_IDX                                                               0
+#define regMMEA3_IO_WR_PRI_FIXED                                                                        0x07a1
+#define regMMEA3_IO_WR_PRI_FIXED_BASE_IDX                                                               0
+#define regMMEA3_IO_RD_PRI_URGENCY                                                                      0x07a2
+#define regMMEA3_IO_RD_PRI_URGENCY_BASE_IDX                                                             0
+#define regMMEA3_IO_WR_PRI_URGENCY                                                                      0x07a3
+#define regMMEA3_IO_WR_PRI_URGENCY_BASE_IDX                                                             0
+#define regMMEA3_IO_RD_PRI_URGENCY_MASKING                                                              0x07a4
+#define regMMEA3_IO_RD_PRI_URGENCY_MASKING_BASE_IDX                                                     0
+#define regMMEA3_IO_WR_PRI_URGENCY_MASKING                                                              0x07a5
+#define regMMEA3_IO_WR_PRI_URGENCY_MASKING_BASE_IDX                                                     0
+#define regMMEA3_IO_RD_PRI_QUANT_PRI1                                                                   0x07a6
+#define regMMEA3_IO_RD_PRI_QUANT_PRI1_BASE_IDX                                                          0
+#define regMMEA3_IO_RD_PRI_QUANT_PRI2                                                                   0x07a7
+#define regMMEA3_IO_RD_PRI_QUANT_PRI2_BASE_IDX                                                          0
+#define regMMEA3_IO_RD_PRI_QUANT_PRI3                                                                   0x07a8
+#define regMMEA3_IO_RD_PRI_QUANT_PRI3_BASE_IDX                                                          0
+#define regMMEA3_IO_WR_PRI_QUANT_PRI1                                                                   0x07a9
+#define regMMEA3_IO_WR_PRI_QUANT_PRI1_BASE_IDX                                                          0
+#define regMMEA3_IO_WR_PRI_QUANT_PRI2                                                                   0x07aa
+#define regMMEA3_IO_WR_PRI_QUANT_PRI2_BASE_IDX                                                          0
+#define regMMEA3_IO_WR_PRI_QUANT_PRI3                                                                   0x07ab
+#define regMMEA3_IO_WR_PRI_QUANT_PRI3_BASE_IDX                                                          0
+#define regMMEA3_SDP_ARB_DRAM                                                                           0x07ac
+#define regMMEA3_SDP_ARB_DRAM_BASE_IDX                                                                  0
+#define regMMEA3_SDP_ARB_GMI                                                                            0x07ad
+#define regMMEA3_SDP_ARB_GMI_BASE_IDX                                                                   0
+#define regMMEA3_SDP_ARB_FINAL                                                                          0x07ae
+#define regMMEA3_SDP_ARB_FINAL_BASE_IDX                                                                 0
+#define regMMEA3_SDP_DRAM_PRIORITY                                                                      0x07af
+#define regMMEA3_SDP_DRAM_PRIORITY_BASE_IDX                                                             0
+#define regMMEA3_SDP_GMI_PRIORITY                                                                       0x07b0
+#define regMMEA3_SDP_GMI_PRIORITY_BASE_IDX                                                              0
+#define regMMEA3_SDP_IO_PRIORITY                                                                        0x07b1
+#define regMMEA3_SDP_IO_PRIORITY_BASE_IDX                                                               0
+#define regMMEA3_SDP_CREDITS                                                                            0x07b2
+#define regMMEA3_SDP_CREDITS_BASE_IDX                                                                   0
+#define regMMEA3_SDP_TAG_RESERVE0                                                                       0x07b3
+#define regMMEA3_SDP_TAG_RESERVE0_BASE_IDX                                                              0
+#define regMMEA3_SDP_TAG_RESERVE1                                                                       0x07b4
+#define regMMEA3_SDP_TAG_RESERVE1_BASE_IDX                                                              0
+#define regMMEA3_SDP_VCC_RESERVE0                                                                       0x07b5
+#define regMMEA3_SDP_VCC_RESERVE0_BASE_IDX                                                              0
+#define regMMEA3_SDP_VCC_RESERVE1                                                                       0x07b6
+#define regMMEA3_SDP_VCC_RESERVE1_BASE_IDX                                                              0
+#define regMMEA3_SDP_VCD_RESERVE0                                                                       0x07b7
+#define regMMEA3_SDP_VCD_RESERVE0_BASE_IDX                                                              0
+#define regMMEA3_SDP_VCD_RESERVE1                                                                       0x07b8
+#define regMMEA3_SDP_VCD_RESERVE1_BASE_IDX                                                              0
+#define regMMEA3_SDP_REQ_CNTL                                                                           0x07b9
+#define regMMEA3_SDP_REQ_CNTL_BASE_IDX                                                                  0
+#define regMMEA3_MISC                                                                                   0x07ba
+#define regMMEA3_MISC_BASE_IDX                                                                          0
+#define regMMEA3_LATENCY_SAMPLING                                                                       0x07bb
+#define regMMEA3_LATENCY_SAMPLING_BASE_IDX                                                              0
+#define regMMEA3_PERFCOUNTER_LO                                                                         0x07bc
+#define regMMEA3_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regMMEA3_PERFCOUNTER_HI                                                                         0x07bd
+#define regMMEA3_PERFCOUNTER_HI_BASE_IDX                                                                0
+#define regMMEA3_PERFCOUNTER0_CFG                                                                       0x07be
+#define regMMEA3_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regMMEA3_PERFCOUNTER1_CFG                                                                       0x07bf
+#define regMMEA3_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regMMEA3_PERFCOUNTER_RSLT_CNTL                                                                  0x07c0
+#define regMMEA3_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+#define regMMEA3_DSM_CNTL                                                                               0x07c8
+#define regMMEA3_DSM_CNTL_BASE_IDX                                                                      0
+#define regMMEA3_DSM_CNTLA                                                                              0x07c9
+#define regMMEA3_DSM_CNTLA_BASE_IDX                                                                     0
+#define regMMEA3_DSM_CNTLB                                                                              0x07ca
+#define regMMEA3_DSM_CNTLB_BASE_IDX                                                                     0
+#define regMMEA3_DSM_CNTL2                                                                              0x07cb
+#define regMMEA3_DSM_CNTL2_BASE_IDX                                                                     0
+#define regMMEA3_DSM_CNTL2A                                                                             0x07cc
+#define regMMEA3_DSM_CNTL2A_BASE_IDX                                                                    0
+#define regMMEA3_DSM_CNTL2B                                                                             0x07cd
+#define regMMEA3_DSM_CNTL2B_BASE_IDX                                                                    0
+#define regMMEA3_CGTT_CLK_CTRL                                                                          0x07cf
+#define regMMEA3_CGTT_CLK_CTRL_BASE_IDX                                                                 0
+#define regMMEA3_EDC_MODE                                                                               0x07d0
+#define regMMEA3_EDC_MODE_BASE_IDX                                                                      0
+#define regMMEA3_ERR_STATUS                                                                             0x07d1
+#define regMMEA3_ERR_STATUS_BASE_IDX                                                                    0
+#define regMMEA3_MISC2                                                                                  0x07d2
+#define regMMEA3_MISC2_BASE_IDX                                                                         0
+#define regMMEA3_MISC_AON                                                                               0x07d5
+#define regMMEA3_MISC_AON_BASE_IDX                                                                      0
+
+
+// addressBlock: aid_mmhub_ea_mmeadec4
+// base address: 0x62000
+#define regMMEA4_DRAM_RD_CLI2GRP_MAP0                                                                   0x0800
+#define regMMEA4_DRAM_RD_CLI2GRP_MAP0_BASE_IDX                                                          0
+#define regMMEA4_DRAM_RD_CLI2GRP_MAP1                                                                   0x0801
+#define regMMEA4_DRAM_RD_CLI2GRP_MAP1_BASE_IDX                                                          0
+#define regMMEA4_DRAM_WR_CLI2GRP_MAP0                                                                   0x0802
+#define regMMEA4_DRAM_WR_CLI2GRP_MAP0_BASE_IDX                                                          0
+#define regMMEA4_DRAM_WR_CLI2GRP_MAP1                                                                   0x0803
+#define regMMEA4_DRAM_WR_CLI2GRP_MAP1_BASE_IDX                                                          0
+#define regMMEA4_DRAM_RD_GRP2VC_MAP                                                                     0x0804
+#define regMMEA4_DRAM_RD_GRP2VC_MAP_BASE_IDX                                                            0
+#define regMMEA4_DRAM_WR_GRP2VC_MAP                                                                     0x0805
+#define regMMEA4_DRAM_WR_GRP2VC_MAP_BASE_IDX                                                            0
+#define regMMEA4_DRAM_RD_LAZY                                                                           0x0806
+#define regMMEA4_DRAM_RD_LAZY_BASE_IDX                                                                  0
+#define regMMEA4_DRAM_WR_LAZY                                                                           0x0807
+#define regMMEA4_DRAM_WR_LAZY_BASE_IDX                                                                  0
+#define regMMEA4_DRAM_RD_CAM_CNTL                                                                       0x0808
+#define regMMEA4_DRAM_RD_CAM_CNTL_BASE_IDX                                                              0
+#define regMMEA4_DRAM_WR_CAM_CNTL                                                                       0x0809
+#define regMMEA4_DRAM_WR_CAM_CNTL_BASE_IDX                                                              0
+#define regMMEA4_DRAM_PAGE_BURST                                                                        0x080a
+#define regMMEA4_DRAM_PAGE_BURST_BASE_IDX                                                               0
+#define regMMEA4_DRAM_RD_PRI_AGE                                                                        0x080b
+#define regMMEA4_DRAM_RD_PRI_AGE_BASE_IDX                                                               0
+#define regMMEA4_DRAM_WR_PRI_AGE                                                                        0x080c
+#define regMMEA4_DRAM_WR_PRI_AGE_BASE_IDX                                                               0
+#define regMMEA4_DRAM_RD_PRI_QUEUING                                                                    0x080d
+#define regMMEA4_DRAM_RD_PRI_QUEUING_BASE_IDX                                                           0
+#define regMMEA4_DRAM_WR_PRI_QUEUING                                                                    0x080e
+#define regMMEA4_DRAM_WR_PRI_QUEUING_BASE_IDX                                                           0
+#define regMMEA4_DRAM_RD_PRI_FIXED                                                                      0x080f
+#define regMMEA4_DRAM_RD_PRI_FIXED_BASE_IDX                                                             0
+#define regMMEA4_DRAM_WR_PRI_FIXED                                                                      0x0810
+#define regMMEA4_DRAM_WR_PRI_FIXED_BASE_IDX                                                             0
+#define regMMEA4_DRAM_RD_PRI_URGENCY                                                                    0x0811
+#define regMMEA4_DRAM_RD_PRI_URGENCY_BASE_IDX                                                           0
+#define regMMEA4_DRAM_WR_PRI_URGENCY                                                                    0x0812
+#define regMMEA4_DRAM_WR_PRI_URGENCY_BASE_IDX                                                           0
+#define regMMEA4_DRAM_RD_PRI_QUANT_PRI1                                                                 0x0813
+#define regMMEA4_DRAM_RD_PRI_QUANT_PRI1_BASE_IDX                                                        0
+#define regMMEA4_DRAM_RD_PRI_QUANT_PRI2                                                                 0x0814
+#define regMMEA4_DRAM_RD_PRI_QUANT_PRI2_BASE_IDX                                                        0
+#define regMMEA4_DRAM_RD_PRI_QUANT_PRI3                                                                 0x0815
+#define regMMEA4_DRAM_RD_PRI_QUANT_PRI3_BASE_IDX                                                        0
+#define regMMEA4_DRAM_WR_PRI_QUANT_PRI1                                                                 0x0816
+#define regMMEA4_DRAM_WR_PRI_QUANT_PRI1_BASE_IDX                                                        0
+#define regMMEA4_DRAM_WR_PRI_QUANT_PRI2                                                                 0x0817
+#define regMMEA4_DRAM_WR_PRI_QUANT_PRI2_BASE_IDX                                                        0
+#define regMMEA4_DRAM_WR_PRI_QUANT_PRI3                                                                 0x0818
+#define regMMEA4_DRAM_WR_PRI_QUANT_PRI3_BASE_IDX                                                        0
+#define regMMEA4_GMI_RD_CLI2GRP_MAP0                                                                    0x0819
+#define regMMEA4_GMI_RD_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regMMEA4_GMI_RD_CLI2GRP_MAP1                                                                    0x081a
+#define regMMEA4_GMI_RD_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regMMEA4_GMI_WR_CLI2GRP_MAP0                                                                    0x081b
+#define regMMEA4_GMI_WR_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regMMEA4_GMI_WR_CLI2GRP_MAP1                                                                    0x081c
+#define regMMEA4_GMI_WR_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regMMEA4_GMI_RD_GRP2VC_MAP                                                                      0x081d
+#define regMMEA4_GMI_RD_GRP2VC_MAP_BASE_IDX                                                             0
+#define regMMEA4_GMI_WR_GRP2VC_MAP                                                                      0x081e
+#define regMMEA4_GMI_WR_GRP2VC_MAP_BASE_IDX                                                             0
+#define regMMEA4_GMI_RD_LAZY                                                                            0x081f
+#define regMMEA4_GMI_RD_LAZY_BASE_IDX                                                                   0
+#define regMMEA4_GMI_WR_LAZY                                                                            0x0820
+#define regMMEA4_GMI_WR_LAZY_BASE_IDX                                                                   0
+#define regMMEA4_GMI_RD_CAM_CNTL                                                                        0x0821
+#define regMMEA4_GMI_RD_CAM_CNTL_BASE_IDX                                                               0
+#define regMMEA4_GMI_WR_CAM_CNTL                                                                        0x0822
+#define regMMEA4_GMI_WR_CAM_CNTL_BASE_IDX                                                               0
+#define regMMEA4_GMI_PAGE_BURST                                                                         0x0823
+#define regMMEA4_GMI_PAGE_BURST_BASE_IDX                                                                0
+#define regMMEA4_GMI_RD_PRI_AGE                                                                         0x0824
+#define regMMEA4_GMI_RD_PRI_AGE_BASE_IDX                                                                0
+#define regMMEA4_GMI_WR_PRI_AGE                                                                         0x0825
+#define regMMEA4_GMI_WR_PRI_AGE_BASE_IDX                                                                0
+#define regMMEA4_GMI_RD_PRI_QUEUING                                                                     0x0826
+#define regMMEA4_GMI_RD_PRI_QUEUING_BASE_IDX                                                            0
+#define regMMEA4_GMI_WR_PRI_QUEUING                                                                     0x0827
+#define regMMEA4_GMI_WR_PRI_QUEUING_BASE_IDX                                                            0
+#define regMMEA4_GMI_RD_PRI_FIXED                                                                       0x0828
+#define regMMEA4_GMI_RD_PRI_FIXED_BASE_IDX                                                              0
+#define regMMEA4_GMI_WR_PRI_FIXED                                                                       0x0829
+#define regMMEA4_GMI_WR_PRI_FIXED_BASE_IDX                                                              0
+#define regMMEA4_GMI_RD_PRI_URGENCY                                                                     0x082a
+#define regMMEA4_GMI_RD_PRI_URGENCY_BASE_IDX                                                            0
+#define regMMEA4_GMI_WR_PRI_URGENCY                                                                     0x082b
+#define regMMEA4_GMI_WR_PRI_URGENCY_BASE_IDX                                                            0
+#define regMMEA4_GMI_RD_PRI_URGENCY_MASKING                                                             0x082c
+#define regMMEA4_GMI_RD_PRI_URGENCY_MASKING_BASE_IDX                                                    0
+#define regMMEA4_GMI_WR_PRI_URGENCY_MASKING                                                             0x082d
+#define regMMEA4_GMI_WR_PRI_URGENCY_MASKING_BASE_IDX                                                    0
+#define regMMEA4_GMI_RD_PRI_QUANT_PRI1                                                                  0x082e
+#define regMMEA4_GMI_RD_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regMMEA4_GMI_RD_PRI_QUANT_PRI2                                                                  0x082f
+#define regMMEA4_GMI_RD_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regMMEA4_GMI_RD_PRI_QUANT_PRI3                                                                  0x0830
+#define regMMEA4_GMI_RD_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regMMEA4_GMI_WR_PRI_QUANT_PRI1                                                                  0x0831
+#define regMMEA4_GMI_WR_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regMMEA4_GMI_WR_PRI_QUANT_PRI2                                                                  0x0832
+#define regMMEA4_GMI_WR_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regMMEA4_GMI_WR_PRI_QUANT_PRI3                                                                  0x0833
+#define regMMEA4_GMI_WR_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regMMEA4_IO_RD_CLI2GRP_MAP0                                                                     0x08d5
+#define regMMEA4_IO_RD_CLI2GRP_MAP0_BASE_IDX                                                            0
+#define regMMEA4_IO_RD_CLI2GRP_MAP1                                                                     0x08d6
+#define regMMEA4_IO_RD_CLI2GRP_MAP1_BASE_IDX                                                            0
+#define regMMEA4_IO_WR_CLI2GRP_MAP0                                                                     0x08d7
+#define regMMEA4_IO_WR_CLI2GRP_MAP0_BASE_IDX                                                            0
+#define regMMEA4_IO_WR_CLI2GRP_MAP1                                                                     0x08d8
+#define regMMEA4_IO_WR_CLI2GRP_MAP1_BASE_IDX                                                            0
+#define regMMEA4_IO_RD_COMBINE_FLUSH                                                                    0x08d9
+#define regMMEA4_IO_RD_COMBINE_FLUSH_BASE_IDX                                                           0
+#define regMMEA4_IO_WR_COMBINE_FLUSH                                                                    0x08da
+#define regMMEA4_IO_WR_COMBINE_FLUSH_BASE_IDX                                                           0
+#define regMMEA4_IO_GROUP_BURST                                                                         0x08db
+#define regMMEA4_IO_GROUP_BURST_BASE_IDX                                                                0
+#define regMMEA4_IO_RD_PRI_AGE                                                                          0x08dc
+#define regMMEA4_IO_RD_PRI_AGE_BASE_IDX                                                                 0
+#define regMMEA4_IO_WR_PRI_AGE                                                                          0x08dd
+#define regMMEA4_IO_WR_PRI_AGE_BASE_IDX                                                                 0
+#define regMMEA4_IO_RD_PRI_QUEUING                                                                      0x08de
+#define regMMEA4_IO_RD_PRI_QUEUING_BASE_IDX                                                             0
+#define regMMEA4_IO_WR_PRI_QUEUING                                                                      0x08df
+#define regMMEA4_IO_WR_PRI_QUEUING_BASE_IDX                                                             0
+#define regMMEA4_IO_RD_PRI_FIXED                                                                        0x08e0
+#define regMMEA4_IO_RD_PRI_FIXED_BASE_IDX                                                               0
+#define regMMEA4_IO_WR_PRI_FIXED                                                                        0x08e1
+#define regMMEA4_IO_WR_PRI_FIXED_BASE_IDX                                                               0
+#define regMMEA4_IO_RD_PRI_URGENCY                                                                      0x08e2
+#define regMMEA4_IO_RD_PRI_URGENCY_BASE_IDX                                                             0
+#define regMMEA4_IO_WR_PRI_URGENCY                                                                      0x08e3
+#define regMMEA4_IO_WR_PRI_URGENCY_BASE_IDX                                                             0
+#define regMMEA4_IO_RD_PRI_URGENCY_MASKING                                                              0x08e4
+#define regMMEA4_IO_RD_PRI_URGENCY_MASKING_BASE_IDX                                                     0
+#define regMMEA4_IO_WR_PRI_URGENCY_MASKING                                                              0x08e5
+#define regMMEA4_IO_WR_PRI_URGENCY_MASKING_BASE_IDX                                                     0
+#define regMMEA4_IO_RD_PRI_QUANT_PRI1                                                                   0x08e6
+#define regMMEA4_IO_RD_PRI_QUANT_PRI1_BASE_IDX                                                          0
+#define regMMEA4_IO_RD_PRI_QUANT_PRI2                                                                   0x08e7
+#define regMMEA4_IO_RD_PRI_QUANT_PRI2_BASE_IDX                                                          0
+#define regMMEA4_IO_RD_PRI_QUANT_PRI3                                                                   0x08e8
+#define regMMEA4_IO_RD_PRI_QUANT_PRI3_BASE_IDX                                                          0
+#define regMMEA4_IO_WR_PRI_QUANT_PRI1                                                                   0x08e9
+#define regMMEA4_IO_WR_PRI_QUANT_PRI1_BASE_IDX                                                          0
+#define regMMEA4_IO_WR_PRI_QUANT_PRI2                                                                   0x08ea
+#define regMMEA4_IO_WR_PRI_QUANT_PRI2_BASE_IDX                                                          0
+#define regMMEA4_IO_WR_PRI_QUANT_PRI3                                                                   0x08eb
+#define regMMEA4_IO_WR_PRI_QUANT_PRI3_BASE_IDX                                                          0
+#define regMMEA4_SDP_ARB_DRAM                                                                           0x08ec
+#define regMMEA4_SDP_ARB_DRAM_BASE_IDX                                                                  0
+#define regMMEA4_SDP_ARB_GMI                                                                            0x08ed
+#define regMMEA4_SDP_ARB_GMI_BASE_IDX                                                                   0
+#define regMMEA4_SDP_ARB_FINAL                                                                          0x08ee
+#define regMMEA4_SDP_ARB_FINAL_BASE_IDX                                                                 0
+#define regMMEA4_SDP_DRAM_PRIORITY                                                                      0x08ef
+#define regMMEA4_SDP_DRAM_PRIORITY_BASE_IDX                                                             0
+#define regMMEA4_SDP_GMI_PRIORITY                                                                       0x08f0
+#define regMMEA4_SDP_GMI_PRIORITY_BASE_IDX                                                              0
+#define regMMEA4_SDP_IO_PRIORITY                                                                        0x08f1
+#define regMMEA4_SDP_IO_PRIORITY_BASE_IDX                                                               0
+#define regMMEA4_SDP_CREDITS                                                                            0x08f2
+#define regMMEA4_SDP_CREDITS_BASE_IDX                                                                   0
+#define regMMEA4_SDP_TAG_RESERVE0                                                                       0x08f3
+#define regMMEA4_SDP_TAG_RESERVE0_BASE_IDX                                                              0
+#define regMMEA4_SDP_TAG_RESERVE1                                                                       0x08f4
+#define regMMEA4_SDP_TAG_RESERVE1_BASE_IDX                                                              0
+#define regMMEA4_SDP_VCC_RESERVE0                                                                       0x08f5
+#define regMMEA4_SDP_VCC_RESERVE0_BASE_IDX                                                              0
+#define regMMEA4_SDP_VCC_RESERVE1                                                                       0x08f6
+#define regMMEA4_SDP_VCC_RESERVE1_BASE_IDX                                                              0
+#define regMMEA4_SDP_VCD_RESERVE0                                                                       0x08f7
+#define regMMEA4_SDP_VCD_RESERVE0_BASE_IDX                                                              0
+#define regMMEA4_SDP_VCD_RESERVE1                                                                       0x08f8
+#define regMMEA4_SDP_VCD_RESERVE1_BASE_IDX                                                              0
+#define regMMEA4_SDP_REQ_CNTL                                                                           0x08f9
+#define regMMEA4_SDP_REQ_CNTL_BASE_IDX                                                                  0
+#define regMMEA4_MISC                                                                                   0x08fa
+#define regMMEA4_MISC_BASE_IDX                                                                          0
+#define regMMEA4_LATENCY_SAMPLING                                                                       0x08fb
+#define regMMEA4_LATENCY_SAMPLING_BASE_IDX                                                              0
+#define regMMEA4_PERFCOUNTER_LO                                                                         0x08fc
+#define regMMEA4_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regMMEA4_PERFCOUNTER_HI                                                                         0x08fd
+#define regMMEA4_PERFCOUNTER_HI_BASE_IDX                                                                0
+#define regMMEA4_PERFCOUNTER0_CFG                                                                       0x08fe
+#define regMMEA4_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regMMEA4_PERFCOUNTER1_CFG                                                                       0x08ff
+#define regMMEA4_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regMMEA4_PERFCOUNTER_RSLT_CNTL                                                                  0x0900
+#define regMMEA4_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+#define regMMEA4_DSM_CNTL                                                                               0x0908
+#define regMMEA4_DSM_CNTL_BASE_IDX                                                                      0
+#define regMMEA4_DSM_CNTLA                                                                              0x0909
+#define regMMEA4_DSM_CNTLA_BASE_IDX                                                                     0
+#define regMMEA4_DSM_CNTLB                                                                              0x090a
+#define regMMEA4_DSM_CNTLB_BASE_IDX                                                                     0
+#define regMMEA4_DSM_CNTL2                                                                              0x090b
+#define regMMEA4_DSM_CNTL2_BASE_IDX                                                                     0
+#define regMMEA4_DSM_CNTL2A                                                                             0x090c
+#define regMMEA4_DSM_CNTL2A_BASE_IDX                                                                    0
+#define regMMEA4_DSM_CNTL2B                                                                             0x090d
+#define regMMEA4_DSM_CNTL2B_BASE_IDX                                                                    0
+#define regMMEA4_CGTT_CLK_CTRL                                                                          0x090f
+#define regMMEA4_CGTT_CLK_CTRL_BASE_IDX                                                                 0
+#define regMMEA4_EDC_MODE                                                                               0x0910
+#define regMMEA4_EDC_MODE_BASE_IDX                                                                      0
+#define regMMEA4_ERR_STATUS                                                                             0x0911
+#define regMMEA4_ERR_STATUS_BASE_IDX                                                                    0
+#define regMMEA4_MISC2                                                                                  0x0912
+#define regMMEA4_MISC2_BASE_IDX                                                                         0
+#define regMMEA4_MISC_AON                                                                               0x0915
+#define regMMEA4_MISC_AON_BASE_IDX                                                                      0
+
+
+// addressBlock: aid_mmhub_pctldec0
+// base address: 0x62a00
+#define regPCTL0_CTRL                                                                                   0x0a80
+#define regPCTL0_CTRL_BASE_IDX                                                                          0
+#define regPCTL0_MMHUB_DEEPSLEEP_IB                                                                     0x0a81
+#define regPCTL0_MMHUB_DEEPSLEEP_IB_BASE_IDX                                                            0
+#define regPCTL0_MMHUB_DEEPSLEEP_OVERRIDE                                                               0x0a82
+#define regPCTL0_MMHUB_DEEPSLEEP_OVERRIDE_BASE_IDX                                                      0
+#define regPCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB                                                            0x0a83
+#define regPCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB_BASE_IDX                                                   0
+#define regPCTL0_PG_IGNORE_DEEPSLEEP                                                                    0x0a84
+#define regPCTL0_PG_IGNORE_DEEPSLEEP_BASE_IDX                                                           0
+#define regPCTL0_PG_IGNORE_DEEPSLEEP_IB                                                                 0x0a85
+#define regPCTL0_PG_IGNORE_DEEPSLEEP_IB_BASE_IDX                                                        0
+#define regPCTL0_SLICE0_CFG_DAGB_BUSY                                                                   0x0a86
+#define regPCTL0_SLICE0_CFG_DAGB_BUSY_BASE_IDX                                                          0
+#define regPCTL0_SLICE0_CFG_DS_ALLOW                                                                    0x0a87
+#define regPCTL0_SLICE0_CFG_DS_ALLOW_BASE_IDX                                                           0
+#define regPCTL0_SLICE0_CFG_DS_ALLOW_IB                                                                 0x0a88
+#define regPCTL0_SLICE0_CFG_DS_ALLOW_IB_BASE_IDX                                                        0
+#define regPCTL0_SLICE1_CFG_DAGB_BUSY                                                                   0x0a89
+#define regPCTL0_SLICE1_CFG_DAGB_BUSY_BASE_IDX                                                          0
+#define regPCTL0_SLICE1_CFG_DS_ALLOW                                                                    0x0a8a
+#define regPCTL0_SLICE1_CFG_DS_ALLOW_BASE_IDX                                                           0
+#define regPCTL0_SLICE1_CFG_DS_ALLOW_IB                                                                 0x0a8b
+#define regPCTL0_SLICE1_CFG_DS_ALLOW_IB_BASE_IDX                                                        0
+#define regPCTL0_SLICE2_CFG_DAGB_BUSY                                                                   0x0a8c
+#define regPCTL0_SLICE2_CFG_DAGB_BUSY_BASE_IDX                                                          0
+#define regPCTL0_SLICE2_CFG_DS_ALLOW                                                                    0x0a8d
+#define regPCTL0_SLICE2_CFG_DS_ALLOW_BASE_IDX                                                           0
+#define regPCTL0_SLICE2_CFG_DS_ALLOW_IB                                                                 0x0a8e
+#define regPCTL0_SLICE2_CFG_DS_ALLOW_IB_BASE_IDX                                                        0
+#define regPCTL0_SLICE3_CFG_DAGB_BUSY                                                                   0x0a8f
+#define regPCTL0_SLICE3_CFG_DAGB_BUSY_BASE_IDX                                                          0
+#define regPCTL0_SLICE3_CFG_DS_ALLOW                                                                    0x0a90
+#define regPCTL0_SLICE3_CFG_DS_ALLOW_BASE_IDX                                                           0
+#define regPCTL0_SLICE3_CFG_DS_ALLOW_IB                                                                 0x0a91
+#define regPCTL0_SLICE3_CFG_DS_ALLOW_IB_BASE_IDX                                                        0
+#define regPCTL0_SLICE4_CFG_DAGB_BUSY                                                                   0x0a92
+#define regPCTL0_SLICE4_CFG_DAGB_BUSY_BASE_IDX                                                          0
+#define regPCTL0_SLICE4_CFG_DS_ALLOW                                                                    0x0a93
+#define regPCTL0_SLICE4_CFG_DS_ALLOW_BASE_IDX                                                           0
+#define regPCTL0_SLICE4_CFG_DS_ALLOW_IB                                                                 0x0a94
+#define regPCTL0_SLICE4_CFG_DS_ALLOW_IB_BASE_IDX                                                        0
+#define regPCTL0_UTCL2_MISC                                                                             0x0a95
+#define regPCTL0_UTCL2_MISC_BASE_IDX                                                                    0
+#define regPCTL0_SLICE0_MISC                                                                            0x0a96
+#define regPCTL0_SLICE0_MISC_BASE_IDX                                                                   0
+#define regPCTL0_SLICE1_MISC                                                                            0x0a97
+#define regPCTL0_SLICE1_MISC_BASE_IDX                                                                   0
+#define regPCTL0_SLICE2_MISC                                                                            0x0a98
+#define regPCTL0_SLICE2_MISC_BASE_IDX                                                                   0
+#define regPCTL0_SLICE3_MISC                                                                            0x0a99
+#define regPCTL0_SLICE3_MISC_BASE_IDX                                                                   0
+#define regPCTL0_SLICE4_MISC                                                                            0x0a9a
+#define regPCTL0_SLICE4_MISC_BASE_IDX                                                                   0
+
+
+// addressBlock: aid_mmhub_l1tlb_vml1dec
+// base address: 0x62c00
+#define regMC_VM_MX_L1_TLB0_STATUS                                                                      0x0b08
+#define regMC_VM_MX_L1_TLB0_STATUS_BASE_IDX                                                             0
+#define regMC_VM_MX_L1_TLB1_STATUS                                                                      0x0b09
+#define regMC_VM_MX_L1_TLB1_STATUS_BASE_IDX                                                             0
+#define regMC_VM_MX_L1_TLB2_STATUS                                                                      0x0b0a
+#define regMC_VM_MX_L1_TLB2_STATUS_BASE_IDX                                                             0
+#define regMC_VM_MX_L1_TLB3_STATUS                                                                      0x0b0b
+#define regMC_VM_MX_L1_TLB3_STATUS_BASE_IDX                                                             0
+#define regMC_VM_MX_L1_TLB4_STATUS                                                                      0x0b0c
+#define regMC_VM_MX_L1_TLB4_STATUS_BASE_IDX                                                             0
+#define regMC_VM_MX_L1_TLB5_STATUS                                                                      0x0b0d
+#define regMC_VM_MX_L1_TLB5_STATUS_BASE_IDX                                                             0
+#define regMC_VM_MX_L1_TLB6_STATUS                                                                      0x0b0e
+#define regMC_VM_MX_L1_TLB6_STATUS_BASE_IDX                                                             0
+#define regMC_VM_MX_L1_TLB7_STATUS                                                                      0x0b0f
+#define regMC_VM_MX_L1_TLB7_STATUS_BASE_IDX                                                             0
+
+
+// addressBlock: aid_mmhub_l1tlb_vml1pldec
+// base address: 0x62c80
+#define regMC_VM_MX_L1_PERFCOUNTER0_CFG                                                                 0x0b20
+#define regMC_VM_MX_L1_PERFCOUNTER0_CFG_BASE_IDX                                                        0
+#define regMC_VM_MX_L1_PERFCOUNTER1_CFG                                                                 0x0b21
+#define regMC_VM_MX_L1_PERFCOUNTER1_CFG_BASE_IDX                                                        0
+#define regMC_VM_MX_L1_PERFCOUNTER2_CFG                                                                 0x0b22
+#define regMC_VM_MX_L1_PERFCOUNTER2_CFG_BASE_IDX                                                        0
+#define regMC_VM_MX_L1_PERFCOUNTER3_CFG                                                                 0x0b23
+#define regMC_VM_MX_L1_PERFCOUNTER3_CFG_BASE_IDX                                                        0
+#define regMC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL                                                            0x0b24
+#define regMC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                   0
+
+
+// addressBlock: aid_mmhub_l1tlb_vml1prdec
+// base address: 0x62cc0
+#define regMC_VM_MX_L1_PERFCOUNTER_LO                                                                   0x0b30
+#define regMC_VM_MX_L1_PERFCOUNTER_LO_BASE_IDX                                                          0
+#define regMC_VM_MX_L1_PERFCOUNTER_HI                                                                   0x0b31
+#define regMC_VM_MX_L1_PERFCOUNTER_HI_BASE_IDX                                                          0
+
+
+// addressBlock: aid_mmhub_utcl2_atcl2dec
+// base address: 0x62d00
+#define regATC_L2_CNTL                                                                                  0x0b40
+#define regATC_L2_CNTL_BASE_IDX                                                                         0
+#define regATC_L2_CNTL2                                                                                 0x0b41
+#define regATC_L2_CNTL2_BASE_IDX                                                                        0
+#define regATC_L2_CACHE_DATA0                                                                           0x0b44
+#define regATC_L2_CACHE_DATA0_BASE_IDX                                                                  0
+#define regATC_L2_CACHE_DATA1                                                                           0x0b45
+#define regATC_L2_CACHE_DATA1_BASE_IDX                                                                  0
+#define regATC_L2_CACHE_DATA2                                                                           0x0b46
+#define regATC_L2_CACHE_DATA2_BASE_IDX                                                                  0
+#define regATC_L2_CACHE_DATA3                                                                           0x0b47
+#define regATC_L2_CACHE_DATA3_BASE_IDX                                                                  0
+#define regATC_L2_CNTL3                                                                                 0x0b48
+#define regATC_L2_CNTL3_BASE_IDX                                                                        0
+#define regATC_L2_STATUS                                                                                0x0b49
+#define regATC_L2_STATUS_BASE_IDX                                                                       0
+#define regATC_L2_STATUS2                                                                               0x0b4a
+#define regATC_L2_STATUS2_BASE_IDX                                                                      0
+#define regATC_L2_MISC_CG                                                                               0x0b4b
+#define regATC_L2_MISC_CG_BASE_IDX                                                                      0
+#define regATC_L2_MEM_POWER_LS                                                                          0x0b4c
+#define regATC_L2_MEM_POWER_LS_BASE_IDX                                                                 0
+#define regATC_L2_CGTT_CLK_CTRL                                                                         0x0b4d
+#define regATC_L2_CGTT_CLK_CTRL_BASE_IDX                                                                0
+#define regATC_L2_CACHE_4K_DSM_INDEX                                                                    0x0b4f
+#define regATC_L2_CACHE_4K_DSM_INDEX_BASE_IDX                                                           0
+#define regATC_L2_CACHE_32K_DSM_INDEX                                                                   0x0b50
+#define regATC_L2_CACHE_32K_DSM_INDEX_BASE_IDX                                                          0
+#define regATC_L2_CACHE_2M_DSM_INDEX                                                                    0x0b51
+#define regATC_L2_CACHE_2M_DSM_INDEX_BASE_IDX                                                           0
+#define regATC_L2_CACHE_4K_DSM_CNTL                                                                     0x0b52
+#define regATC_L2_CACHE_4K_DSM_CNTL_BASE_IDX                                                            0
+#define regATC_L2_CACHE_32K_DSM_CNTL                                                                    0x0b53
+#define regATC_L2_CACHE_32K_DSM_CNTL_BASE_IDX                                                           0
+#define regATC_L2_CACHE_2M_DSM_CNTL                                                                     0x0b54
+#define regATC_L2_CACHE_2M_DSM_CNTL_BASE_IDX                                                            0
+#define regATC_L2_CNTL4                                                                                 0x0b55
+#define regATC_L2_CNTL4_BASE_IDX                                                                        0
+#define regATC_L2_MM_GROUP_RT_CLASSES                                                                   0x0b56
+#define regATC_L2_MM_GROUP_RT_CLASSES_BASE_IDX                                                          0
+
+
+// addressBlock: aid_mmhub_utcl2_vml2pfdec
+// base address: 0x62d80
+#define regVM_L2_CNTL                                                                                   0x0b60
+#define regVM_L2_CNTL_BASE_IDX                                                                          0
+#define regVM_L2_CNTL2                                                                                  0x0b61
+#define regVM_L2_CNTL2_BASE_IDX                                                                         0
+#define regVM_L2_CNTL3                                                                                  0x0b62
+#define regVM_L2_CNTL3_BASE_IDX                                                                         0
+#define regVM_L2_STATUS                                                                                 0x0b63
+#define regVM_L2_STATUS_BASE_IDX                                                                        0
+#define regVM_DUMMY_PAGE_FAULT_CNTL                                                                     0x0b64
+#define regVM_DUMMY_PAGE_FAULT_CNTL_BASE_IDX                                                            0
+#define regVM_DUMMY_PAGE_FAULT_ADDR_LO32                                                                0x0b65
+#define regVM_DUMMY_PAGE_FAULT_ADDR_LO32_BASE_IDX                                                       0
+#define regVM_DUMMY_PAGE_FAULT_ADDR_HI32                                                                0x0b66
+#define regVM_DUMMY_PAGE_FAULT_ADDR_HI32_BASE_IDX                                                       0
+#define regVM_L2_PROTECTION_FAULT_CNTL                                                                  0x0b67
+#define regVM_L2_PROTECTION_FAULT_CNTL_BASE_IDX                                                         0
+#define regVM_L2_PROTECTION_FAULT_CNTL2                                                                 0x0b68
+#define regVM_L2_PROTECTION_FAULT_CNTL2_BASE_IDX                                                        0
+#define regVM_L2_PROTECTION_FAULT_MM_CNTL3                                                              0x0b69
+#define regVM_L2_PROTECTION_FAULT_MM_CNTL3_BASE_IDX                                                     0
+#define regVM_L2_PROTECTION_FAULT_MM_CNTL4                                                              0x0b6a
+#define regVM_L2_PROTECTION_FAULT_MM_CNTL4_BASE_IDX                                                     0
+#define regVM_L2_PROTECTION_FAULT_STATUS                                                                0x0b6b
+#define regVM_L2_PROTECTION_FAULT_STATUS_BASE_IDX                                                       0
+#define regVM_L2_PROTECTION_FAULT_ADDR_LO32                                                             0x0b6c
+#define regVM_L2_PROTECTION_FAULT_ADDR_LO32_BASE_IDX                                                    0
+#define regVM_L2_PROTECTION_FAULT_ADDR_HI32                                                             0x0b6d
+#define regVM_L2_PROTECTION_FAULT_ADDR_HI32_BASE_IDX                                                    0
+#define regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32                                                     0x0b6e
+#define regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32_BASE_IDX                                            0
+#define regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32                                                     0x0b6f
+#define regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32_BASE_IDX                                            0
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32                                               0x0b71
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32_BASE_IDX                                      0
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32                                               0x0b72
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32_BASE_IDX                                      0
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32                                              0x0b73
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32_BASE_IDX                                     0
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32                                              0x0b74
+#define regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32_BASE_IDX                                     0
+#define regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32                                                  0x0b75
+#define regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32_BASE_IDX                                         0
+#define regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32                                                  0x0b76
+#define regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32_BASE_IDX                                         0
+#define regVM_L2_CNTL4                                                                                  0x0b77
+#define regVM_L2_CNTL4_BASE_IDX                                                                         0
+#define regVM_L2_CNTL5                                                                                  0x0b78
+#define regVM_L2_CNTL5_BASE_IDX                                                                         0
+#define regVM_L2_MM_GROUP_RT_CLASSES                                                                    0x0b79
+#define regVM_L2_MM_GROUP_RT_CLASSES_BASE_IDX                                                           0
+#define regVM_L2_BANK_SELECT_RESERVED_CID                                                               0x0b7a
+#define regVM_L2_BANK_SELECT_RESERVED_CID_BASE_IDX                                                      0
+#define regVM_L2_BANK_SELECT_RESERVED_CID2                                                              0x0b7b
+#define regVM_L2_BANK_SELECT_RESERVED_CID2_BASE_IDX                                                     0
+#define regVM_L2_CACHE_PARITY_CNTL                                                                      0x0b7c
+#define regVM_L2_CACHE_PARITY_CNTL_BASE_IDX                                                             0
+#define regVM_L2_CGTT_CLK_CTRL                                                                          0x0b7d
+#define regVM_L2_CGTT_CLK_CTRL_BASE_IDX                                                                 0
+#define regVM_L2_CGTT_BUSY_CTRL                                                                         0x0b7e
+#define regVM_L2_CGTT_BUSY_CTRL_BASE_IDX                                                                0
+#define regVML2_MEM_ECC_INDEX                                                                           0x0b82
+#define regVML2_MEM_ECC_INDEX_BASE_IDX                                                                  0
+#define regVML2_WALKER_MEM_ECC_INDEX                                                                    0x0b83
+#define regVML2_WALKER_MEM_ECC_INDEX_BASE_IDX                                                           0
+#define regUTCL2_MEM_ECC_INDEX                                                                          0x0b84
+#define regUTCL2_MEM_ECC_INDEX_BASE_IDX                                                                 0
+#define regVML2_MEM_ECC_CNTL                                                                            0x0b85
+#define regVML2_MEM_ECC_CNTL_BASE_IDX                                                                   0
+#define regVML2_WALKER_MEM_ECC_CNTL                                                                     0x0b86
+#define regVML2_WALKER_MEM_ECC_CNTL_BASE_IDX                                                            0
+#define regUTCL2_MEM_ECC_CNTL                                                                           0x0b87
+#define regUTCL2_MEM_ECC_CNTL_BASE_IDX                                                                  0
+#define regVML2_MEM_ECC_STATUS                                                                          0x0b88
+#define regVML2_MEM_ECC_STATUS_BASE_IDX                                                                 0
+#define regVML2_WALKER_MEM_ECC_STATUS                                                                   0x0b89
+#define regVML2_WALKER_MEM_ECC_STATUS_BASE_IDX                                                          0
+#define regUTCL2_MEM_ECC_STATUS                                                                         0x0b8a
+#define regUTCL2_MEM_ECC_STATUS_BASE_IDX                                                                0
+#define regUTCL2_EDC_MODE                                                                               0x0b8b
+#define regUTCL2_EDC_MODE_BASE_IDX                                                                      0
+#define regUTCL2_EDC_CONFIG                                                                             0x0b8c
+#define regUTCL2_EDC_CONFIG_BASE_IDX                                                                    0
+
+
+// addressBlock: aid_mmhub_utcl2_vml2vcdec
+// base address: 0x62e80
+#define regVM_CONTEXT0_CNTL                                                                             0x0ba0
+#define regVM_CONTEXT0_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT1_CNTL                                                                             0x0ba1
+#define regVM_CONTEXT1_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT2_CNTL                                                                             0x0ba2
+#define regVM_CONTEXT2_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT3_CNTL                                                                             0x0ba3
+#define regVM_CONTEXT3_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT4_CNTL                                                                             0x0ba4
+#define regVM_CONTEXT4_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT5_CNTL                                                                             0x0ba5
+#define regVM_CONTEXT5_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT6_CNTL                                                                             0x0ba6
+#define regVM_CONTEXT6_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT7_CNTL                                                                             0x0ba7
+#define regVM_CONTEXT7_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT8_CNTL                                                                             0x0ba8
+#define regVM_CONTEXT8_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT9_CNTL                                                                             0x0ba9
+#define regVM_CONTEXT9_CNTL_BASE_IDX                                                                    0
+#define regVM_CONTEXT10_CNTL                                                                            0x0baa
+#define regVM_CONTEXT10_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXT11_CNTL                                                                            0x0bab
+#define regVM_CONTEXT11_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXT12_CNTL                                                                            0x0bac
+#define regVM_CONTEXT12_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXT13_CNTL                                                                            0x0bad
+#define regVM_CONTEXT13_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXT14_CNTL                                                                            0x0bae
+#define regVM_CONTEXT14_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXT15_CNTL                                                                            0x0baf
+#define regVM_CONTEXT15_CNTL_BASE_IDX                                                                   0
+#define regVM_CONTEXTS_DISABLE                                                                          0x0bb0
+#define regVM_CONTEXTS_DISABLE_BASE_IDX                                                                 0
+#define regVM_INVALIDATE_ENG0_SEM                                                                       0x0bb1
+#define regVM_INVALIDATE_ENG0_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG1_SEM                                                                       0x0bb2
+#define regVM_INVALIDATE_ENG1_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG2_SEM                                                                       0x0bb3
+#define regVM_INVALIDATE_ENG2_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG3_SEM                                                                       0x0bb4
+#define regVM_INVALIDATE_ENG3_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG4_SEM                                                                       0x0bb5
+#define regVM_INVALIDATE_ENG4_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG5_SEM                                                                       0x0bb6
+#define regVM_INVALIDATE_ENG5_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG6_SEM                                                                       0x0bb7
+#define regVM_INVALIDATE_ENG6_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG7_SEM                                                                       0x0bb8
+#define regVM_INVALIDATE_ENG7_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG8_SEM                                                                       0x0bb9
+#define regVM_INVALIDATE_ENG8_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG9_SEM                                                                       0x0bba
+#define regVM_INVALIDATE_ENG9_SEM_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG10_SEM                                                                      0x0bbb
+#define regVM_INVALIDATE_ENG10_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG11_SEM                                                                      0x0bbc
+#define regVM_INVALIDATE_ENG11_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG12_SEM                                                                      0x0bbd
+#define regVM_INVALIDATE_ENG12_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG13_SEM                                                                      0x0bbe
+#define regVM_INVALIDATE_ENG13_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG14_SEM                                                                      0x0bbf
+#define regVM_INVALIDATE_ENG14_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG15_SEM                                                                      0x0bc0
+#define regVM_INVALIDATE_ENG15_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG16_SEM                                                                      0x0bc1
+#define regVM_INVALIDATE_ENG16_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG17_SEM                                                                      0x0bc2
+#define regVM_INVALIDATE_ENG17_SEM_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG0_REQ                                                                       0x0bc3
+#define regVM_INVALIDATE_ENG0_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG1_REQ                                                                       0x0bc4
+#define regVM_INVALIDATE_ENG1_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG2_REQ                                                                       0x0bc5
+#define regVM_INVALIDATE_ENG2_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG3_REQ                                                                       0x0bc6
+#define regVM_INVALIDATE_ENG3_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG4_REQ                                                                       0x0bc7
+#define regVM_INVALIDATE_ENG4_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG5_REQ                                                                       0x0bc8
+#define regVM_INVALIDATE_ENG5_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG6_REQ                                                                       0x0bc9
+#define regVM_INVALIDATE_ENG6_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG7_REQ                                                                       0x0bca
+#define regVM_INVALIDATE_ENG7_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG8_REQ                                                                       0x0bcb
+#define regVM_INVALIDATE_ENG8_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG9_REQ                                                                       0x0bcc
+#define regVM_INVALIDATE_ENG9_REQ_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG10_REQ                                                                      0x0bcd
+#define regVM_INVALIDATE_ENG10_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG11_REQ                                                                      0x0bce
+#define regVM_INVALIDATE_ENG11_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG12_REQ                                                                      0x0bcf
+#define regVM_INVALIDATE_ENG12_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG13_REQ                                                                      0x0bd0
+#define regVM_INVALIDATE_ENG13_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG14_REQ                                                                      0x0bd1
+#define regVM_INVALIDATE_ENG14_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG15_REQ                                                                      0x0bd2
+#define regVM_INVALIDATE_ENG15_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG16_REQ                                                                      0x0bd3
+#define regVM_INVALIDATE_ENG16_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG17_REQ                                                                      0x0bd4
+#define regVM_INVALIDATE_ENG17_REQ_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG0_ACK                                                                       0x0bd5
+#define regVM_INVALIDATE_ENG0_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG1_ACK                                                                       0x0bd6
+#define regVM_INVALIDATE_ENG1_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG2_ACK                                                                       0x0bd7
+#define regVM_INVALIDATE_ENG2_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG3_ACK                                                                       0x0bd8
+#define regVM_INVALIDATE_ENG3_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG4_ACK                                                                       0x0bd9
+#define regVM_INVALIDATE_ENG4_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG5_ACK                                                                       0x0bda
+#define regVM_INVALIDATE_ENG5_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG6_ACK                                                                       0x0bdb
+#define regVM_INVALIDATE_ENG6_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG7_ACK                                                                       0x0bdc
+#define regVM_INVALIDATE_ENG7_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG8_ACK                                                                       0x0bdd
+#define regVM_INVALIDATE_ENG8_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG9_ACK                                                                       0x0bde
+#define regVM_INVALIDATE_ENG9_ACK_BASE_IDX                                                              0
+#define regVM_INVALIDATE_ENG10_ACK                                                                      0x0bdf
+#define regVM_INVALIDATE_ENG10_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG11_ACK                                                                      0x0be0
+#define regVM_INVALIDATE_ENG11_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG12_ACK                                                                      0x0be1
+#define regVM_INVALIDATE_ENG12_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG13_ACK                                                                      0x0be2
+#define regVM_INVALIDATE_ENG13_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG14_ACK                                                                      0x0be3
+#define regVM_INVALIDATE_ENG14_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG15_ACK                                                                      0x0be4
+#define regVM_INVALIDATE_ENG15_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG16_ACK                                                                      0x0be5
+#define regVM_INVALIDATE_ENG16_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG17_ACK                                                                      0x0be6
+#define regVM_INVALIDATE_ENG17_ACK_BASE_IDX                                                             0
+#define regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32                                                           0x0be7
+#define regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32                                                           0x0be8
+#define regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32                                                           0x0be9
+#define regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG1_ADDR_RANGE_HI32                                                           0x0bea
+#define regVM_INVALIDATE_ENG1_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG2_ADDR_RANGE_LO32                                                           0x0beb
+#define regVM_INVALIDATE_ENG2_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG2_ADDR_RANGE_HI32                                                           0x0bec
+#define regVM_INVALIDATE_ENG2_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG3_ADDR_RANGE_LO32                                                           0x0bed
+#define regVM_INVALIDATE_ENG3_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG3_ADDR_RANGE_HI32                                                           0x0bee
+#define regVM_INVALIDATE_ENG3_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG4_ADDR_RANGE_LO32                                                           0x0bef
+#define regVM_INVALIDATE_ENG4_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG4_ADDR_RANGE_HI32                                                           0x0bf0
+#define regVM_INVALIDATE_ENG4_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG5_ADDR_RANGE_LO32                                                           0x0bf1
+#define regVM_INVALIDATE_ENG5_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG5_ADDR_RANGE_HI32                                                           0x0bf2
+#define regVM_INVALIDATE_ENG5_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG6_ADDR_RANGE_LO32                                                           0x0bf3
+#define regVM_INVALIDATE_ENG6_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG6_ADDR_RANGE_HI32                                                           0x0bf4
+#define regVM_INVALIDATE_ENG6_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG7_ADDR_RANGE_LO32                                                           0x0bf5
+#define regVM_INVALIDATE_ENG7_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG7_ADDR_RANGE_HI32                                                           0x0bf6
+#define regVM_INVALIDATE_ENG7_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG8_ADDR_RANGE_LO32                                                           0x0bf7
+#define regVM_INVALIDATE_ENG8_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG8_ADDR_RANGE_HI32                                                           0x0bf8
+#define regVM_INVALIDATE_ENG8_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG9_ADDR_RANGE_LO32                                                           0x0bf9
+#define regVM_INVALIDATE_ENG9_ADDR_RANGE_LO32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG9_ADDR_RANGE_HI32                                                           0x0bfa
+#define regVM_INVALIDATE_ENG9_ADDR_RANGE_HI32_BASE_IDX                                                  0
+#define regVM_INVALIDATE_ENG10_ADDR_RANGE_LO32                                                          0x0bfb
+#define regVM_INVALIDATE_ENG10_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG10_ADDR_RANGE_HI32                                                          0x0bfc
+#define regVM_INVALIDATE_ENG10_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG11_ADDR_RANGE_LO32                                                          0x0bfd
+#define regVM_INVALIDATE_ENG11_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG11_ADDR_RANGE_HI32                                                          0x0bfe
+#define regVM_INVALIDATE_ENG11_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG12_ADDR_RANGE_LO32                                                          0x0bff
+#define regVM_INVALIDATE_ENG12_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG12_ADDR_RANGE_HI32                                                          0x0c00
+#define regVM_INVALIDATE_ENG12_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG13_ADDR_RANGE_LO32                                                          0x0c01
+#define regVM_INVALIDATE_ENG13_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG13_ADDR_RANGE_HI32                                                          0x0c02
+#define regVM_INVALIDATE_ENG13_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG14_ADDR_RANGE_LO32                                                          0x0c03
+#define regVM_INVALIDATE_ENG14_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG14_ADDR_RANGE_HI32                                                          0x0c04
+#define regVM_INVALIDATE_ENG14_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG15_ADDR_RANGE_LO32                                                          0x0c05
+#define regVM_INVALIDATE_ENG15_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG15_ADDR_RANGE_HI32                                                          0x0c06
+#define regVM_INVALIDATE_ENG15_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG16_ADDR_RANGE_LO32                                                          0x0c07
+#define regVM_INVALIDATE_ENG16_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG16_ADDR_RANGE_HI32                                                          0x0c08
+#define regVM_INVALIDATE_ENG16_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG17_ADDR_RANGE_LO32                                                          0x0c09
+#define regVM_INVALIDATE_ENG17_ADDR_RANGE_LO32_BASE_IDX                                                 0
+#define regVM_INVALIDATE_ENG17_ADDR_RANGE_HI32                                                          0x0c0a
+#define regVM_INVALIDATE_ENG17_ADDR_RANGE_HI32_BASE_IDX                                                 0
+#define regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32                                                        0x0c0b
+#define regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32                                                        0x0c0c
+#define regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32                                                        0x0c0d
+#define regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32                                                        0x0c0e
+#define regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32                                                        0x0c0f
+#define regVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32                                                        0x0c10
+#define regVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32                                                        0x0c11
+#define regVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32                                                        0x0c12
+#define regVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32                                                        0x0c13
+#define regVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32                                                        0x0c14
+#define regVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32                                                        0x0c15
+#define regVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32                                                        0x0c16
+#define regVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32                                                        0x0c17
+#define regVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32                                                        0x0c18
+#define regVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32                                                        0x0c19
+#define regVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32                                                        0x0c1a
+#define regVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32                                                        0x0c1b
+#define regVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32                                                        0x0c1c
+#define regVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32                                                        0x0c1d
+#define regVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32                                                        0x0c1e
+#define regVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32                                                       0x0c1f
+#define regVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32                                                       0x0c20
+#define regVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32                                                       0x0c21
+#define regVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32                                                       0x0c22
+#define regVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32                                                       0x0c23
+#define regVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32                                                       0x0c24
+#define regVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32                                                       0x0c25
+#define regVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32                                                       0x0c26
+#define regVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32                                                       0x0c27
+#define regVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32                                                       0x0c28
+#define regVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32                                                       0x0c29
+#define regVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32                                                       0x0c2a
+#define regVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32                                                       0x0c2b
+#define regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32                                                       0x0c2c
+#define regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32                                                       0x0c2d
+#define regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32                                                       0x0c2e
+#define regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32                                                       0x0c2f
+#define regVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32                                                       0x0c30
+#define regVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32                                                       0x0c31
+#define regVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32                                                       0x0c32
+#define regVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32                                                       0x0c33
+#define regVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32                                                       0x0c34
+#define regVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32                                                       0x0c35
+#define regVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32                                                       0x0c36
+#define regVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32                                                       0x0c37
+#define regVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32                                                       0x0c38
+#define regVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32                                                       0x0c39
+#define regVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32                                                       0x0c3a
+#define regVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32                                                       0x0c3b
+#define regVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32                                                       0x0c3c
+#define regVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32                                                       0x0c3d
+#define regVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                              0
+#define regVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32                                                       0x0c3e
+#define regVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                              0
+#define regVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32                                                      0x0c3f
+#define regVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32                                                      0x0c40
+#define regVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32                                                      0x0c41
+#define regVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32                                                      0x0c42
+#define regVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32                                                      0x0c43
+#define regVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32                                                      0x0c44
+#define regVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32                                                      0x0c45
+#define regVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32                                                      0x0c46
+#define regVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32                                                      0x0c47
+#define regVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32                                                      0x0c48
+#define regVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32                                                      0x0c49
+#define regVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                             0
+#define regVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32                                                      0x0c4a
+#define regVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                             0
+#define regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32                                                         0x0c4b
+#define regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32                                                         0x0c4c
+#define regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32                                                         0x0c4d
+#define regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32                                                         0x0c4e
+#define regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32                                                         0x0c4f
+#define regVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32                                                         0x0c50
+#define regVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32                                                         0x0c51
+#define regVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32                                                         0x0c52
+#define regVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32                                                         0x0c53
+#define regVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32                                                         0x0c54
+#define regVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32                                                         0x0c55
+#define regVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32                                                         0x0c56
+#define regVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32                                                         0x0c57
+#define regVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32                                                         0x0c58
+#define regVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32                                                         0x0c59
+#define regVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32                                                         0x0c5a
+#define regVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32                                                         0x0c5b
+#define regVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32                                                         0x0c5c
+#define regVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32                                                         0x0c5d
+#define regVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                                0
+#define regVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32                                                         0x0c5e
+#define regVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                                0
+#define regVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32                                                        0x0c5f
+#define regVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32                                                        0x0c60
+#define regVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32                                                        0x0c61
+#define regVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32                                                        0x0c62
+#define regVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32                                                        0x0c63
+#define regVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32                                                        0x0c64
+#define regVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32                                                        0x0c65
+#define regVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32                                                        0x0c66
+#define regVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32                                                        0x0c67
+#define regVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32                                                        0x0c68
+#define regVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+#define regVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32                                                        0x0c69
+#define regVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                               0
+#define regVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32                                                        0x0c6a
+#define regVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                               0
+
+
+// addressBlock: aid_mmhub_utcl2_vmsharedpfdec
+// base address: 0x63200
+#define regMC_VM_NB_MMIOBASE                                                                            0x0c80
+#define regMC_VM_NB_MMIOBASE_BASE_IDX                                                                   0
+#define regMC_VM_NB_MMIOLIMIT                                                                           0x0c81
+#define regMC_VM_NB_MMIOLIMIT_BASE_IDX                                                                  0
+#define regMC_VM_NB_PCI_CTRL                                                                            0x0c82
+#define regMC_VM_NB_PCI_CTRL_BASE_IDX                                                                   0
+#define regMC_VM_NB_PCI_ARB                                                                             0x0c83
+#define regMC_VM_NB_PCI_ARB_BASE_IDX                                                                    0
+#define regMC_VM_NB_TOP_OF_DRAM_SLOT1                                                                   0x0c84
+#define regMC_VM_NB_TOP_OF_DRAM_SLOT1_BASE_IDX                                                          0
+#define regMC_VM_NB_LOWER_TOP_OF_DRAM2                                                                  0x0c85
+#define regMC_VM_NB_LOWER_TOP_OF_DRAM2_BASE_IDX                                                         0
+#define regMC_VM_NB_UPPER_TOP_OF_DRAM2                                                                  0x0c86
+#define regMC_VM_NB_UPPER_TOP_OF_DRAM2_BASE_IDX                                                         0
+#define regMC_VM_FB_OFFSET                                                                              0x0c87
+#define regMC_VM_FB_OFFSET_BASE_IDX                                                                     0
+#define regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB                                                       0x0c88
+#define regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX                                              0
+#define regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB                                                       0x0c89
+#define regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX                                              0
+#define regMC_VM_STEERING                                                                               0x0c8a
+#define regMC_VM_STEERING_BASE_IDX                                                                      0
+#define regMC_SHARED_VIRT_RESET_REQ                                                                     0x0c8b
+#define regMC_SHARED_VIRT_RESET_REQ_BASE_IDX                                                            0
+#define regMC_MEM_POWER_LS                                                                              0x0c8c
+#define regMC_MEM_POWER_LS_BASE_IDX                                                                     0
+#define regMC_VM_CACHEABLE_DRAM_ADDRESS_START                                                           0x0c8d
+#define regMC_VM_CACHEABLE_DRAM_ADDRESS_START_BASE_IDX                                                  0
+#define regMC_VM_CACHEABLE_DRAM_ADDRESS_END                                                             0x0c8e
+#define regMC_VM_CACHEABLE_DRAM_ADDRESS_END_BASE_IDX                                                    0
+#define regMC_VM_APT_CNTL                                                                               0x0c91
+#define regMC_VM_APT_CNTL_BASE_IDX                                                                      0
+#define regMC_VM_LOCAL_HBM_ADDRESS_START                                                                0x0c92
+#define regMC_VM_LOCAL_HBM_ADDRESS_START_BASE_IDX                                                       0
+#define regMC_VM_LOCAL_HBM_ADDRESS_END                                                                  0x0c93
+#define regMC_VM_LOCAL_HBM_ADDRESS_END_BASE_IDX                                                         0
+#define regMC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL                                                            0x0c94
+#define regMC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL_BASE_IDX                                                   0
+#define regUTCL2_CGTT_CLK_CTRL                                                                          0x0c95
+#define regUTCL2_CGTT_CLK_CTRL_BASE_IDX                                                                 0
+#define regMC_VM_XGMI_LFB_CNTL                                                                          0x0c97
+#define regMC_VM_XGMI_LFB_CNTL_BASE_IDX                                                                 0
+#define regMC_VM_XGMI_LFB_SIZE                                                                          0x0c98
+#define regMC_VM_XGMI_LFB_SIZE_BASE_IDX                                                                 0
+#define regMC_VM_CACHEABLE_DRAM_CNTL                                                                    0x0c99
+#define regMC_VM_CACHEABLE_DRAM_CNTL_BASE_IDX                                                           0
+#define regMC_VM_HOST_MAPPING                                                                           0x0c9a
+#define regMC_VM_HOST_MAPPING_BASE_IDX                                                                  0
+
+
+// addressBlock: aid_mmhub_utcl2_vmsharedvcdec
+// base address: 0x63270
+#define regMC_VM_FB_LOCATION_BASE                                                                       0x0c9c
+#define regMC_VM_FB_LOCATION_BASE_BASE_IDX                                                              0
+#define regMC_VM_FB_LOCATION_TOP                                                                        0x0c9d
+#define regMC_VM_FB_LOCATION_TOP_BASE_IDX                                                               0
+#define regMC_VM_AGP_TOP                                                                                0x0c9e
+#define regMC_VM_AGP_TOP_BASE_IDX                                                                       0
+#define regMC_VM_AGP_BOT                                                                                0x0c9f
+#define regMC_VM_AGP_BOT_BASE_IDX                                                                       0
+#define regMC_VM_AGP_BASE                                                                               0x0ca0
+#define regMC_VM_AGP_BASE_BASE_IDX                                                                      0
+#define regMC_VM_SYSTEM_APERTURE_LOW_ADDR                                                               0x0ca1
+#define regMC_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX                                                      0
+#define regMC_VM_SYSTEM_APERTURE_HIGH_ADDR                                                              0x0ca2
+#define regMC_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX                                                     0
+#define regMC_VM_MX_L1_TLB_CNTL                                                                         0x0ca3
+#define regMC_VM_MX_L1_TLB_CNTL_BASE_IDX                                                                0
+
+
+// addressBlock: aid_mmhub_utcl2_vmsharedhvdec
+// base address: 0x632b0
+#define regMC_VM_FB_SIZE_OFFSET_VF0                                                                     0x0cac
+#define regMC_VM_FB_SIZE_OFFSET_VF0_BASE_IDX                                                            0
+#define regMC_VM_FB_SIZE_OFFSET_VF1                                                                     0x0cad
+#define regMC_VM_FB_SIZE_OFFSET_VF1_BASE_IDX                                                            0
+#define regMC_VM_FB_SIZE_OFFSET_VF2                                                                     0x0cae
+#define regMC_VM_FB_SIZE_OFFSET_VF2_BASE_IDX                                                            0
+#define regMC_VM_FB_SIZE_OFFSET_VF3                                                                     0x0caf
+#define regMC_VM_FB_SIZE_OFFSET_VF3_BASE_IDX                                                            0
+#define regMC_VM_FB_SIZE_OFFSET_VF4                                                                     0x0cb0
+#define regMC_VM_FB_SIZE_OFFSET_VF4_BASE_IDX                                                            0
+#define regMC_VM_FB_SIZE_OFFSET_VF5                                                                     0x0cb1
+#define regMC_VM_FB_SIZE_OFFSET_VF5_BASE_IDX                                                            0
+#define regMC_VM_FB_SIZE_OFFSET_VF6                                                                     0x0cb2
+#define regMC_VM_FB_SIZE_OFFSET_VF6_BASE_IDX                                                            0
+#define regMC_VM_FB_SIZE_OFFSET_VF7                                                                     0x0cb3
+#define regMC_VM_FB_SIZE_OFFSET_VF7_BASE_IDX                                                            0
+#define regMC_VM_FB_SIZE_OFFSET_VF8                                                                     0x0cb4
+#define regMC_VM_FB_SIZE_OFFSET_VF8_BASE_IDX                                                            0
+#define regMC_VM_FB_SIZE_OFFSET_VF9                                                                     0x0cb5
+#define regMC_VM_FB_SIZE_OFFSET_VF9_BASE_IDX                                                            0
+#define regMC_VM_FB_SIZE_OFFSET_VF10                                                                    0x0cb6
+#define regMC_VM_FB_SIZE_OFFSET_VF10_BASE_IDX                                                           0
+#define regMC_VM_FB_SIZE_OFFSET_VF11                                                                    0x0cb7
+#define regMC_VM_FB_SIZE_OFFSET_VF11_BASE_IDX                                                           0
+#define regMC_VM_FB_SIZE_OFFSET_VF12                                                                    0x0cb8
+#define regMC_VM_FB_SIZE_OFFSET_VF12_BASE_IDX                                                           0
+#define regMC_VM_FB_SIZE_OFFSET_VF13                                                                    0x0cb9
+#define regMC_VM_FB_SIZE_OFFSET_VF13_BASE_IDX                                                           0
+#define regMC_VM_FB_SIZE_OFFSET_VF14                                                                    0x0cba
+#define regMC_VM_FB_SIZE_OFFSET_VF14_BASE_IDX                                                           0
+#define regMC_VM_FB_SIZE_OFFSET_VF15                                                                    0x0cbb
+#define regMC_VM_FB_SIZE_OFFSET_VF15_BASE_IDX                                                           0
+#define regVM_IOMMU_MMIO_CNTRL_1                                                                        0x0cbc
+#define regVM_IOMMU_MMIO_CNTRL_1_BASE_IDX                                                               0
+#define regMC_VM_MARC_BASE_LO_0                                                                         0x0cbd
+#define regMC_VM_MARC_BASE_LO_0_BASE_IDX                                                                0
+#define regMC_VM_MARC_BASE_LO_1                                                                         0x0cbe
+#define regMC_VM_MARC_BASE_LO_1_BASE_IDX                                                                0
+#define regMC_VM_MARC_BASE_LO_2                                                                         0x0cbf
+#define regMC_VM_MARC_BASE_LO_2_BASE_IDX                                                                0
+#define regMC_VM_MARC_BASE_LO_3                                                                         0x0cc0
+#define regMC_VM_MARC_BASE_LO_3_BASE_IDX                                                                0
+#define regMC_VM_MARC_BASE_HI_0                                                                         0x0cc1
+#define regMC_VM_MARC_BASE_HI_0_BASE_IDX                                                                0
+#define regMC_VM_MARC_BASE_HI_1                                                                         0x0cc2
+#define regMC_VM_MARC_BASE_HI_1_BASE_IDX                                                                0
+#define regMC_VM_MARC_BASE_HI_2                                                                         0x0cc3
+#define regMC_VM_MARC_BASE_HI_2_BASE_IDX                                                                0
+#define regMC_VM_MARC_BASE_HI_3                                                                         0x0cc4
+#define regMC_VM_MARC_BASE_HI_3_BASE_IDX                                                                0
+#define regMC_VM_MARC_RELOC_LO_0                                                                        0x0cc5
+#define regMC_VM_MARC_RELOC_LO_0_BASE_IDX                                                               0
+#define regMC_VM_MARC_RELOC_LO_1                                                                        0x0cc6
+#define regMC_VM_MARC_RELOC_LO_1_BASE_IDX                                                               0
+#define regMC_VM_MARC_RELOC_LO_2                                                                        0x0cc7
+#define regMC_VM_MARC_RELOC_LO_2_BASE_IDX                                                               0
+#define regMC_VM_MARC_RELOC_LO_3                                                                        0x0cc8
+#define regMC_VM_MARC_RELOC_LO_3_BASE_IDX                                                               0
+#define regMC_VM_MARC_RELOC_HI_0                                                                        0x0cc9
+#define regMC_VM_MARC_RELOC_HI_0_BASE_IDX                                                               0
+#define regMC_VM_MARC_RELOC_HI_1                                                                        0x0cca
+#define regMC_VM_MARC_RELOC_HI_1_BASE_IDX                                                               0
+#define regMC_VM_MARC_RELOC_HI_2                                                                        0x0ccb
+#define regMC_VM_MARC_RELOC_HI_2_BASE_IDX                                                               0
+#define regMC_VM_MARC_RELOC_HI_3                                                                        0x0ccc
+#define regMC_VM_MARC_RELOC_HI_3_BASE_IDX                                                               0
+#define regMC_VM_MARC_LEN_LO_0                                                                          0x0ccd
+#define regMC_VM_MARC_LEN_LO_0_BASE_IDX                                                                 0
+#define regMC_VM_MARC_LEN_LO_1                                                                          0x0cce
+#define regMC_VM_MARC_LEN_LO_1_BASE_IDX                                                                 0
+#define regMC_VM_MARC_LEN_LO_2                                                                          0x0ccf
+#define regMC_VM_MARC_LEN_LO_2_BASE_IDX                                                                 0
+#define regMC_VM_MARC_LEN_LO_3                                                                          0x0cd0
+#define regMC_VM_MARC_LEN_LO_3_BASE_IDX                                                                 0
+#define regMC_VM_MARC_LEN_HI_0                                                                          0x0cd1
+#define regMC_VM_MARC_LEN_HI_0_BASE_IDX                                                                 0
+#define regMC_VM_MARC_LEN_HI_1                                                                          0x0cd2
+#define regMC_VM_MARC_LEN_HI_1_BASE_IDX                                                                 0
+#define regMC_VM_MARC_LEN_HI_2                                                                          0x0cd3
+#define regMC_VM_MARC_LEN_HI_2_BASE_IDX                                                                 0
+#define regMC_VM_MARC_LEN_HI_3                                                                          0x0cd4
+#define regMC_VM_MARC_LEN_HI_3_BASE_IDX                                                                 0
+#define regVM_IOMMU_CONTROL_REGISTER                                                                    0x0cd5
+#define regVM_IOMMU_CONTROL_REGISTER_BASE_IDX                                                           0
+#define regVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER                                           0x0cd6
+#define regVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER_BASE_IDX                                  0
+#define regVM_PCIE_ATS_CNTL                                                                             0x0cd7
+#define regVM_PCIE_ATS_CNTL_BASE_IDX                                                                    0
+#define regVM_PCIE_ATS_CNTL_VF_0                                                                        0x0cd8
+#define regVM_PCIE_ATS_CNTL_VF_0_BASE_IDX                                                               0
+#define regVM_PCIE_ATS_CNTL_VF_1                                                                        0x0cd9
+#define regVM_PCIE_ATS_CNTL_VF_1_BASE_IDX                                                               0
+#define regVM_PCIE_ATS_CNTL_VF_2                                                                        0x0cda
+#define regVM_PCIE_ATS_CNTL_VF_2_BASE_IDX                                                               0
+#define regVM_PCIE_ATS_CNTL_VF_3                                                                        0x0cdb
+#define regVM_PCIE_ATS_CNTL_VF_3_BASE_IDX                                                               0
+#define regVM_PCIE_ATS_CNTL_VF_4                                                                        0x0cdc
+#define regVM_PCIE_ATS_CNTL_VF_4_BASE_IDX                                                               0
+#define regVM_PCIE_ATS_CNTL_VF_5                                                                        0x0cdd
+#define regVM_PCIE_ATS_CNTL_VF_5_BASE_IDX                                                               0
+#define regVM_PCIE_ATS_CNTL_VF_6                                                                        0x0cde
+#define regVM_PCIE_ATS_CNTL_VF_6_BASE_IDX                                                               0
+#define regVM_PCIE_ATS_CNTL_VF_7                                                                        0x0cdf
+#define regVM_PCIE_ATS_CNTL_VF_7_BASE_IDX                                                               0
+#define regVM_PCIE_ATS_CNTL_VF_8                                                                        0x0ce0
+#define regVM_PCIE_ATS_CNTL_VF_8_BASE_IDX                                                               0
+#define regVM_PCIE_ATS_CNTL_VF_9                                                                        0x0ce1
+#define regVM_PCIE_ATS_CNTL_VF_9_BASE_IDX                                                               0
+#define regVM_PCIE_ATS_CNTL_VF_10                                                                       0x0ce2
+#define regVM_PCIE_ATS_CNTL_VF_10_BASE_IDX                                                              0
+#define regVM_PCIE_ATS_CNTL_VF_11                                                                       0x0ce3
+#define regVM_PCIE_ATS_CNTL_VF_11_BASE_IDX                                                              0
+#define regVM_PCIE_ATS_CNTL_VF_12                                                                       0x0ce4
+#define regVM_PCIE_ATS_CNTL_VF_12_BASE_IDX                                                              0
+#define regVM_PCIE_ATS_CNTL_VF_13                                                                       0x0ce5
+#define regVM_PCIE_ATS_CNTL_VF_13_BASE_IDX                                                              0
+#define regVM_PCIE_ATS_CNTL_VF_14                                                                       0x0ce6
+#define regVM_PCIE_ATS_CNTL_VF_14_BASE_IDX                                                              0
+#define regVM_PCIE_ATS_CNTL_VF_15                                                                       0x0ce7
+#define regVM_PCIE_ATS_CNTL_VF_15_BASE_IDX                                                              0
+#define regMC_SHARED_ACTIVE_FCN_ID                                                                      0x0ce8
+#define regMC_SHARED_ACTIVE_FCN_ID_BASE_IDX                                                             0
+#define regMC_VM_XGMI_GPUIOV_ENABLE                                                                     0x0ce9
+#define regMC_VM_XGMI_GPUIOV_ENABLE_BASE_IDX                                                            0
+
+
+// addressBlock: aid_mmhub_utcl2_atcl2pfcntrdec
+// base address: 0x633b0
+#define regATC_L2_PERFCOUNTER_LO                                                                        0x0cec
+#define regATC_L2_PERFCOUNTER_LO_BASE_IDX                                                               0
+#define regATC_L2_PERFCOUNTER_HI                                                                        0x0ced
+#define regATC_L2_PERFCOUNTER_HI_BASE_IDX                                                               0
+
+
+// addressBlock: aid_mmhub_utcl2_atcl2pfcntldec
+// base address: 0x633b8
+#define regATC_L2_PERFCOUNTER0_CFG                                                                      0x0cee
+#define regATC_L2_PERFCOUNTER0_CFG_BASE_IDX                                                             0
+#define regATC_L2_PERFCOUNTER1_CFG                                                                      0x0cef
+#define regATC_L2_PERFCOUNTER1_CFG_BASE_IDX                                                             0
+#define regATC_L2_PERFCOUNTER_RSLT_CNTL                                                                 0x0cf0
+#define regATC_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                        0
+
+
+// addressBlock: aid_mmhub_utcl2_vml2pldec
+// base address: 0x633d0
+#define regMC_VM_L2_PERFCOUNTER0_CFG                                                                    0x0cf4
+#define regMC_VM_L2_PERFCOUNTER0_CFG_BASE_IDX                                                           0
+#define regMC_VM_L2_PERFCOUNTER1_CFG                                                                    0x0cf5
+#define regMC_VM_L2_PERFCOUNTER1_CFG_BASE_IDX                                                           0
+#define regMC_VM_L2_PERFCOUNTER2_CFG                                                                    0x0cf6
+#define regMC_VM_L2_PERFCOUNTER2_CFG_BASE_IDX                                                           0
+#define regMC_VM_L2_PERFCOUNTER3_CFG                                                                    0x0cf7
+#define regMC_VM_L2_PERFCOUNTER3_CFG_BASE_IDX                                                           0
+#define regMC_VM_L2_PERFCOUNTER4_CFG                                                                    0x0cf8
+#define regMC_VM_L2_PERFCOUNTER4_CFG_BASE_IDX                                                           0
+#define regMC_VM_L2_PERFCOUNTER5_CFG                                                                    0x0cf9
+#define regMC_VM_L2_PERFCOUNTER5_CFG_BASE_IDX                                                           0
+#define regMC_VM_L2_PERFCOUNTER6_CFG                                                                    0x0cfa
+#define regMC_VM_L2_PERFCOUNTER6_CFG_BASE_IDX                                                           0
+#define regMC_VM_L2_PERFCOUNTER7_CFG                                                                    0x0cfb
+#define regMC_VM_L2_PERFCOUNTER7_CFG_BASE_IDX                                                           0
+#define regMC_VM_L2_PERFCOUNTER_RSLT_CNTL                                                               0x0d04
+#define regMC_VM_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                      0
+
+
+// addressBlock: aid_mmhub_utcl2_vml2prdec
+// base address: 0x63430
+#define regMC_VM_L2_PERFCOUNTER_LO                                                                      0x0d0c
+#define regMC_VM_L2_PERFCOUNTER_LO_BASE_IDX                                                             0
+#define regMC_VM_L2_PERFCOUNTER_HI                                                                      0x0d0d
+#define regMC_VM_L2_PERFCOUNTER_HI_BASE_IDX                                                             0
+
+
+// addressBlock: aid_mmhub_utcl2_l2tlbdec
+// base address: 0x63470
+#define regL2TLB_TLB0_STATUS                                                                            0x0d1d
+#define regL2TLB_TLB0_STATUS_BASE_IDX                                                                   0
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO                                                 0x0d1f
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO_BASE_IDX                                        0
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI                                                 0x0d20
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI_BASE_IDX                                        0
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO                                                0x0d21
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO_BASE_IDX                                       0
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI                                                0x0d22
+#define regUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI_BASE_IDX                                       0
+
+
+// addressBlock: aid_mmhub_utcl2_l2tlbpldec
+// base address: 0x63490
+#define regL2TLB_PERFCOUNTER0_CFG                                                                       0x0d24
+#define regL2TLB_PERFCOUNTER0_CFG_BASE_IDX                                                              0
+#define regL2TLB_PERFCOUNTER1_CFG                                                                       0x0d25
+#define regL2TLB_PERFCOUNTER1_CFG_BASE_IDX                                                              0
+#define regL2TLB_PERFCOUNTER2_CFG                                                                       0x0d26
+#define regL2TLB_PERFCOUNTER2_CFG_BASE_IDX                                                              0
+#define regL2TLB_PERFCOUNTER3_CFG                                                                       0x0d27
+#define regL2TLB_PERFCOUNTER3_CFG_BASE_IDX                                                              0
+#define regL2TLB_PERFCOUNTER_RSLT_CNTL                                                                  0x0d28
+#define regL2TLB_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                         0
+
+
+// addressBlock: aid_mmhub_utcl2_l2tlbprdec
+// base address: 0x634b0
+#define regL2TLB_PERFCOUNTER_LO                                                                         0x0d2c
+#define regL2TLB_PERFCOUNTER_LO_BASE_IDX                                                                0
+#define regL2TLB_PERFCOUNTER_HI                                                                         0x0d2d
+#define regL2TLB_PERFCOUNTER_HI_BASE_IDX                                                                0
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_8_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_8_0_sh_mask.h
new file mode 100644
index 000000000000..af41468ce69f
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_8_0_sh_mask.h
@@ -0,0 +1,22315 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _mmhub_1_8_0_SH_MASK_HEADER
+#define _mmhub_1_8_0_SH_MASK_HEADER
+
+
+// addressBlock: aid_mmhub_dagb_dagbdec0
+//DAGB0_RDCLI0
+#define DAGB0_RDCLI0__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_RDCLI0__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_RDCLI0__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_RDCLI0__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_RDCLI0__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_RDCLI0__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_RDCLI0__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_RDCLI0__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_RDCLI0__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_RDCLI0__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_RDCLI0__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_RDCLI0__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_RDCLI0__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_RDCLI0__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_RDCLI0__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_RDCLI0__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_RDCLI0__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_RDCLI0__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_RDCLI0__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_RDCLI0__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_RDCLI1
+#define DAGB0_RDCLI1__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_RDCLI1__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_RDCLI1__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_RDCLI1__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_RDCLI1__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_RDCLI1__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_RDCLI1__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_RDCLI1__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_RDCLI1__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_RDCLI1__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_RDCLI1__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_RDCLI1__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_RDCLI1__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_RDCLI1__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_RDCLI1__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_RDCLI1__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_RDCLI1__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_RDCLI1__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_RDCLI1__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_RDCLI1__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_RDCLI2
+#define DAGB0_RDCLI2__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_RDCLI2__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_RDCLI2__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_RDCLI2__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_RDCLI2__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_RDCLI2__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_RDCLI2__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_RDCLI2__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_RDCLI2__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_RDCLI2__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_RDCLI2__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_RDCLI2__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_RDCLI2__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_RDCLI2__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_RDCLI2__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_RDCLI2__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_RDCLI2__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_RDCLI2__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_RDCLI2__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_RDCLI2__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_RDCLI3
+#define DAGB0_RDCLI3__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_RDCLI3__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_RDCLI3__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_RDCLI3__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_RDCLI3__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_RDCLI3__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_RDCLI3__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_RDCLI3__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_RDCLI3__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_RDCLI3__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_RDCLI3__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_RDCLI3__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_RDCLI3__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_RDCLI3__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_RDCLI3__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_RDCLI3__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_RDCLI3__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_RDCLI3__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_RDCLI3__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_RDCLI3__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_RDCLI4
+#define DAGB0_RDCLI4__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_RDCLI4__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_RDCLI4__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_RDCLI4__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_RDCLI4__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_RDCLI4__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_RDCLI4__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_RDCLI4__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_RDCLI4__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_RDCLI4__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_RDCLI4__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_RDCLI4__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_RDCLI4__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_RDCLI4__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_RDCLI4__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_RDCLI4__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_RDCLI4__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_RDCLI4__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_RDCLI4__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_RDCLI4__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_RDCLI5
+#define DAGB0_RDCLI5__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_RDCLI5__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_RDCLI5__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_RDCLI5__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_RDCLI5__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_RDCLI5__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_RDCLI5__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_RDCLI5__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_RDCLI5__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_RDCLI5__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_RDCLI5__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_RDCLI5__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_RDCLI5__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_RDCLI5__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_RDCLI5__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_RDCLI5__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_RDCLI5__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_RDCLI5__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_RDCLI5__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_RDCLI5__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_RDCLI6
+#define DAGB0_RDCLI6__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_RDCLI6__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_RDCLI6__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_RDCLI6__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_RDCLI6__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_RDCLI6__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_RDCLI6__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_RDCLI6__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_RDCLI6__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_RDCLI6__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_RDCLI6__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_RDCLI6__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_RDCLI6__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_RDCLI6__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_RDCLI6__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_RDCLI6__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_RDCLI6__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_RDCLI6__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_RDCLI6__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_RDCLI6__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_RDCLI7
+#define DAGB0_RDCLI7__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_RDCLI7__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_RDCLI7__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_RDCLI7__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_RDCLI7__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_RDCLI7__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_RDCLI7__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_RDCLI7__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_RDCLI7__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_RDCLI7__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_RDCLI7__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_RDCLI7__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_RDCLI7__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_RDCLI7__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_RDCLI7__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_RDCLI7__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_RDCLI7__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_RDCLI7__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_RDCLI7__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_RDCLI7__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_RDCLI8
+#define DAGB0_RDCLI8__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_RDCLI8__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_RDCLI8__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_RDCLI8__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_RDCLI8__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_RDCLI8__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_RDCLI8__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_RDCLI8__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_RDCLI8__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_RDCLI8__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_RDCLI8__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_RDCLI8__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_RDCLI8__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_RDCLI8__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_RDCLI8__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_RDCLI8__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_RDCLI8__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_RDCLI8__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_RDCLI8__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_RDCLI8__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_RDCLI9
+#define DAGB0_RDCLI9__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_RDCLI9__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_RDCLI9__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_RDCLI9__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_RDCLI9__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_RDCLI9__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_RDCLI9__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_RDCLI9__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_RDCLI9__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_RDCLI9__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_RDCLI9__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_RDCLI9__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_RDCLI9__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_RDCLI9__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_RDCLI9__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_RDCLI9__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_RDCLI9__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_RDCLI9__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_RDCLI9__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_RDCLI9__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_RDCLI10
+#define DAGB0_RDCLI10__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_RDCLI10__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_RDCLI10__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_RDCLI10__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_RDCLI10__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_RDCLI10__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_RDCLI10__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_RDCLI10__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_RDCLI10__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_RDCLI10__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_RDCLI10__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_RDCLI10__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_RDCLI10__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_RDCLI10__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_RDCLI10__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_RDCLI10__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_RDCLI10__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_RDCLI10__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_RDCLI10__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_RDCLI10__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_RDCLI11
+#define DAGB0_RDCLI11__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_RDCLI11__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_RDCLI11__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_RDCLI11__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_RDCLI11__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_RDCLI11__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_RDCLI11__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_RDCLI11__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_RDCLI11__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_RDCLI11__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_RDCLI11__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_RDCLI11__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_RDCLI11__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_RDCLI11__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_RDCLI11__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_RDCLI11__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_RDCLI11__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_RDCLI11__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_RDCLI11__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_RDCLI11__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_RDCLI12
+#define DAGB0_RDCLI12__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_RDCLI12__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_RDCLI12__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_RDCLI12__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_RDCLI12__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_RDCLI12__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_RDCLI12__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_RDCLI12__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_RDCLI12__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_RDCLI12__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_RDCLI12__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_RDCLI12__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_RDCLI12__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_RDCLI12__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_RDCLI12__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_RDCLI12__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_RDCLI12__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_RDCLI12__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_RDCLI12__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_RDCLI12__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_RDCLI13
+#define DAGB0_RDCLI13__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_RDCLI13__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_RDCLI13__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_RDCLI13__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_RDCLI13__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_RDCLI13__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_RDCLI13__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_RDCLI13__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_RDCLI13__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_RDCLI13__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_RDCLI13__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_RDCLI13__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_RDCLI13__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_RDCLI13__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_RDCLI13__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_RDCLI13__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_RDCLI13__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_RDCLI13__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_RDCLI13__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_RDCLI13__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_RDCLI14
+#define DAGB0_RDCLI14__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_RDCLI14__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_RDCLI14__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_RDCLI14__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_RDCLI14__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_RDCLI14__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_RDCLI14__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_RDCLI14__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_RDCLI14__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_RDCLI14__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_RDCLI14__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_RDCLI14__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_RDCLI14__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_RDCLI14__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_RDCLI14__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_RDCLI14__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_RDCLI14__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_RDCLI14__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_RDCLI14__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_RDCLI14__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_RDCLI15
+#define DAGB0_RDCLI15__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_RDCLI15__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_RDCLI15__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_RDCLI15__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_RDCLI15__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_RDCLI15__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_RDCLI15__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_RDCLI15__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_RDCLI15__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_RDCLI15__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_RDCLI15__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_RDCLI15__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_RDCLI15__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_RDCLI15__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_RDCLI15__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_RDCLI15__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_RDCLI15__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_RDCLI15__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_RDCLI15__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_RDCLI15__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_RD_CNTL
+#define DAGB0_RD_CNTL__SCLK_FREQ__SHIFT                                                                       0x0
+#define DAGB0_RD_CNTL__CLI_MAX_BW_WINDOW__SHIFT                                                               0x4
+#define DAGB0_RD_CNTL__VC_MAX_BW_WINDOW__SHIFT                                                                0xa
+#define DAGB0_RD_CNTL__IO_LEVEL_OVERRIDE_ENABLE__SHIFT                                                        0x10
+#define DAGB0_RD_CNTL__IO_LEVEL__SHIFT                                                                        0x11
+#define DAGB0_RD_CNTL__IO_LEVEL_COMPLY_VC__SHIFT                                                              0x14
+#define DAGB0_RD_CNTL__SHARE_VC_NUM__SHIFT                                                                    0x17
+#define DAGB0_RD_CNTL__FIX_JUMP__SHIFT                                                                        0x1a
+#define DAGB0_RD_CNTL__SCLK_FREQ_MASK                                                                         0x0000000FL
+#define DAGB0_RD_CNTL__CLI_MAX_BW_WINDOW_MASK                                                                 0x000003F0L
+#define DAGB0_RD_CNTL__VC_MAX_BW_WINDOW_MASK                                                                  0x0000FC00L
+#define DAGB0_RD_CNTL__IO_LEVEL_OVERRIDE_ENABLE_MASK                                                          0x00010000L
+#define DAGB0_RD_CNTL__IO_LEVEL_MASK                                                                          0x000E0000L
+#define DAGB0_RD_CNTL__IO_LEVEL_COMPLY_VC_MASK                                                                0x00700000L
+#define DAGB0_RD_CNTL__SHARE_VC_NUM_MASK                                                                      0x03800000L
+#define DAGB0_RD_CNTL__FIX_JUMP_MASK                                                                          0x04000000L
+//DAGB0_RD_GMI_CNTL
+#define DAGB0_RD_GMI_CNTL__EA_CREDIT__SHIFT                                                                   0x0
+#define DAGB0_RD_GMI_CNTL__LEVEL__SHIFT                                                                       0x6
+#define DAGB0_RD_GMI_CNTL__MAX_BURST__SHIFT                                                                   0x9
+#define DAGB0_RD_GMI_CNTL__LAZY_TIMER__SHIFT                                                                  0xd
+#define DAGB0_RD_GMI_CNTL__EA_CREDIT_MASK                                                                     0x0000003FL
+#define DAGB0_RD_GMI_CNTL__LEVEL_MASK                                                                         0x000001C0L
+#define DAGB0_RD_GMI_CNTL__MAX_BURST_MASK                                                                     0x00001E00L
+#define DAGB0_RD_GMI_CNTL__LAZY_TIMER_MASK                                                                    0x0001E000L
+//DAGB0_RD_ADDR_DAGB
+#define DAGB0_RD_ADDR_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB0_RD_ADDR_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB0_RD_ADDR_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB0_RD_ADDR_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB0_RD_ADDR_DAGB__JUMP_MODE__SHIFT                                                                  0xd
+#define DAGB0_RD_ADDR_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB0_RD_ADDR_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB0_RD_ADDR_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB0_RD_ADDR_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+#define DAGB0_RD_ADDR_DAGB__JUMP_MODE_MASK                                                                    0x00002000L
+//DAGB0_RD_OUTPUT_DAGB_MAX_BURST
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC0__SHIFT                                                            0x0
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC1__SHIFT                                                            0x4
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC2__SHIFT                                                            0x8
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC3__SHIFT                                                            0xc
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC4__SHIFT                                                            0x10
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC5__SHIFT                                                            0x14
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC6__SHIFT                                                            0x18
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC7__SHIFT                                                            0x1c
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC0_MASK                                                              0x0000000FL
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC1_MASK                                                              0x000000F0L
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC2_MASK                                                              0x00000F00L
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC3_MASK                                                              0x0000F000L
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC4_MASK                                                              0x000F0000L
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC5_MASK                                                              0x00F00000L
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC6_MASK                                                              0x0F000000L
+#define DAGB0_RD_OUTPUT_DAGB_MAX_BURST__VC7_MASK                                                              0xF0000000L
+//DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC0__SHIFT                                                           0x0
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC1__SHIFT                                                           0x4
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC2__SHIFT                                                           0x8
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC3__SHIFT                                                           0xc
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC4__SHIFT                                                           0x10
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC5__SHIFT                                                           0x14
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC6__SHIFT                                                           0x18
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC7__SHIFT                                                           0x1c
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC0_MASK                                                             0x0000000FL
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC1_MASK                                                             0x000000F0L
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC2_MASK                                                             0x00000F00L
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC3_MASK                                                             0x0000F000L
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC4_MASK                                                             0x000F0000L
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC5_MASK                                                             0x00F00000L
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC6_MASK                                                             0x0F000000L
+#define DAGB0_RD_OUTPUT_DAGB_LAZY_TIMER__VC7_MASK                                                             0xF0000000L
+//DAGB0_RD_CGTT_CLK_CTRL
+#define DAGB0_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                               0x0
+#define DAGB0_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                         0x4
+#define DAGB0_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                   0xc
+#define DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                            0x1e
+#define DAGB0_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                          0x1f
+#define DAGB0_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                 0x0000000FL
+#define DAGB0_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                           0x00000FF0L
+#define DAGB0_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                     0x0FFFF000L
+#define DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                              0x40000000L
+#define DAGB0_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                            0x80000000L
+//DAGB0_L1TLB_RD_CGTT_CLK_CTRL
+#define DAGB0_L1TLB_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB0_L1TLB_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB0_L1TLB_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB0_L1TLB_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB0_L1TLB_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB0_L1TLB_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB0_L1TLB_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB0_L1TLB_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB0_L1TLB_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB0_L1TLB_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB0_ATCVM_RD_CGTT_CLK_CTRL
+#define DAGB0_ATCVM_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB0_ATCVM_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB0_ATCVM_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB0_ATCVM_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB0_ATCVM_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB0_ATCVM_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB0_ATCVM_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB0_ATCVM_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB0_ATCVM_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB0_ATCVM_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB0_RD_ADDR_DAGB_MAX_BURST0
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB0_RD_ADDR_DAGB_LAZY_TIMER0
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB0_RD_ADDR_DAGB_MAX_BURST1
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB0_RD_ADDR_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB0_RD_ADDR_DAGB_LAZY_TIMER1
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB0_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB0_RD_VC0_CNTL
+#define DAGB0_RD_VC0_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_RD_VC0_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_RD_VC0_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_RD_VC0_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_RD_VC0_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_RD_VC0_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_RD_VC0_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_RD_VC0_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_RD_VC0_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_RD_VC0_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_RD_VC0_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_RD_VC0_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_RD_VC0_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_RD_VC0_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_RD_VC0_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_RD_VC0_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_RD_VC1_CNTL
+#define DAGB0_RD_VC1_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_RD_VC1_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_RD_VC1_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_RD_VC1_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_RD_VC1_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_RD_VC1_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_RD_VC1_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_RD_VC1_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_RD_VC1_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_RD_VC1_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_RD_VC1_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_RD_VC1_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_RD_VC1_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_RD_VC1_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_RD_VC1_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_RD_VC1_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_RD_VC2_CNTL
+#define DAGB0_RD_VC2_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_RD_VC2_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_RD_VC2_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_RD_VC2_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_RD_VC2_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_RD_VC2_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_RD_VC2_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_RD_VC2_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_RD_VC2_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_RD_VC2_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_RD_VC2_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_RD_VC2_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_RD_VC2_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_RD_VC2_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_RD_VC2_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_RD_VC2_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_RD_VC3_CNTL
+#define DAGB0_RD_VC3_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_RD_VC3_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_RD_VC3_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_RD_VC3_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_RD_VC3_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_RD_VC3_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_RD_VC3_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_RD_VC3_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_RD_VC3_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_RD_VC3_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_RD_VC3_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_RD_VC3_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_RD_VC3_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_RD_VC3_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_RD_VC3_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_RD_VC3_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_RD_VC4_CNTL
+#define DAGB0_RD_VC4_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_RD_VC4_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_RD_VC4_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_RD_VC4_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_RD_VC4_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_RD_VC4_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_RD_VC4_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_RD_VC4_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_RD_VC4_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_RD_VC4_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_RD_VC4_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_RD_VC4_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_RD_VC4_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_RD_VC4_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_RD_VC4_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_RD_VC4_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_RD_VC5_CNTL
+#define DAGB0_RD_VC5_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_RD_VC5_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_RD_VC5_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_RD_VC5_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_RD_VC5_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_RD_VC5_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_RD_VC5_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_RD_VC5_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_RD_VC5_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_RD_VC5_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_RD_VC5_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_RD_VC5_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_RD_VC5_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_RD_VC5_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_RD_VC5_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_RD_VC5_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_RD_VC6_CNTL
+#define DAGB0_RD_VC6_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_RD_VC6_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_RD_VC6_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_RD_VC6_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_RD_VC6_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_RD_VC6_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_RD_VC6_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_RD_VC6_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_RD_VC6_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_RD_VC6_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_RD_VC6_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_RD_VC6_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_RD_VC6_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_RD_VC6_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_RD_VC6_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_RD_VC6_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_RD_VC7_CNTL
+#define DAGB0_RD_VC7_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_RD_VC7_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_RD_VC7_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_RD_VC7_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_RD_VC7_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_RD_VC7_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_RD_VC7_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_RD_VC7_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_RD_VC7_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_RD_VC7_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_RD_VC7_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_RD_VC7_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_RD_VC7_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_RD_VC7_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_RD_VC7_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_RD_VC7_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_RD_CNTL_MISC
+#define DAGB0_RD_CNTL_MISC__STOR_POOL_CREDIT__SHIFT                                                           0x0
+#define DAGB0_RD_CNTL_MISC__EA_POOL_CREDIT__SHIFT                                                             0x6
+#define DAGB0_RD_CNTL_MISC__IO_EA_CREDIT__SHIFT                                                               0xd
+#define DAGB0_RD_CNTL_MISC__STOR_CC_NEW_MODE__SHIFT                                                           0x13
+#define DAGB0_RD_CNTL_MISC__EA_CC_LEGACY_MODE__SHIFT                                                          0x14
+#define DAGB0_RD_CNTL_MISC__UTCL2_CID__SHIFT                                                                  0x15
+#define DAGB0_RD_CNTL_MISC__RDRET_FIFO_CREDITS__SHIFT                                                         0x1a
+#define DAGB0_RD_CNTL_MISC__STOR_POOL_CREDIT_MASK                                                             0x0000003FL
+#define DAGB0_RD_CNTL_MISC__EA_POOL_CREDIT_MASK                                                               0x00001FC0L
+#define DAGB0_RD_CNTL_MISC__IO_EA_CREDIT_MASK                                                                 0x0007E000L
+#define DAGB0_RD_CNTL_MISC__STOR_CC_NEW_MODE_MASK                                                             0x00080000L
+#define DAGB0_RD_CNTL_MISC__EA_CC_LEGACY_MODE_MASK                                                            0x00100000L
+#define DAGB0_RD_CNTL_MISC__UTCL2_CID_MASK                                                                    0x03E00000L
+#define DAGB0_RD_CNTL_MISC__RDRET_FIFO_CREDITS_MASK                                                           0xFC000000L
+//DAGB0_RD_TLB_CREDIT
+#define DAGB0_RD_TLB_CREDIT__TLB0__SHIFT                                                                      0x0
+#define DAGB0_RD_TLB_CREDIT__TLB1__SHIFT                                                                      0x5
+#define DAGB0_RD_TLB_CREDIT__TLB2__SHIFT                                                                      0xa
+#define DAGB0_RD_TLB_CREDIT__TLB3__SHIFT                                                                      0xf
+#define DAGB0_RD_TLB_CREDIT__TLB4__SHIFT                                                                      0x14
+#define DAGB0_RD_TLB_CREDIT__TLB5__SHIFT                                                                      0x19
+#define DAGB0_RD_TLB_CREDIT__TLB0_MASK                                                                        0x0000001FL
+#define DAGB0_RD_TLB_CREDIT__TLB1_MASK                                                                        0x000003E0L
+#define DAGB0_RD_TLB_CREDIT__TLB2_MASK                                                                        0x00007C00L
+#define DAGB0_RD_TLB_CREDIT__TLB3_MASK                                                                        0x000F8000L
+#define DAGB0_RD_TLB_CREDIT__TLB4_MASK                                                                        0x01F00000L
+#define DAGB0_RD_TLB_CREDIT__TLB5_MASK                                                                        0x3E000000L
+//DAGB0_RD_RDRET_CREDIT_CNTL
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC0_CREDIT__SHIFT                                                         0x0
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC1_CREDIT__SHIFT                                                         0x6
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC2_CREDIT__SHIFT                                                         0xc
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC3_CREDIT__SHIFT                                                         0x12
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC4_CREDIT__SHIFT                                                         0x18
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC_MODE__SHIFT                                                            0x1e
+#define DAGB0_RD_RDRET_CREDIT_CNTL__FIX_EQ__SHIFT                                                             0x1f
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC0_CREDIT_MASK                                                           0x0000003FL
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC1_CREDIT_MASK                                                           0x00000FC0L
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC2_CREDIT_MASK                                                           0x0003F000L
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC3_CREDIT_MASK                                                           0x00FC0000L
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC4_CREDIT_MASK                                                           0x3F000000L
+#define DAGB0_RD_RDRET_CREDIT_CNTL__VC_MODE_MASK                                                              0x40000000L
+#define DAGB0_RD_RDRET_CREDIT_CNTL__FIX_EQ_MASK                                                               0x80000000L
+//DAGB0_RD_RDRET_CREDIT_CNTL2
+#define DAGB0_RD_RDRET_CREDIT_CNTL2__IO_CREDIT__SHIFT                                                         0x0
+#define DAGB0_RD_RDRET_CREDIT_CNTL2__GMI_CREDIT__SHIFT                                                        0x6
+#define DAGB0_RD_RDRET_CREDIT_CNTL2__POOL_CREDIT__SHIFT                                                       0xc
+#define DAGB0_RD_RDRET_CREDIT_CNTL2__IO_CREDIT_MASK                                                           0x0000003FL
+#define DAGB0_RD_RDRET_CREDIT_CNTL2__GMI_CREDIT_MASK                                                          0x00000FC0L
+#define DAGB0_RD_RDRET_CREDIT_CNTL2__POOL_CREDIT_MASK                                                         0x0007F000L
+//DAGB0_RDCLI_ASK_PENDING
+#define DAGB0_RDCLI_ASK_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB0_RDCLI_ASK_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB0_RDCLI_GO_PENDING
+#define DAGB0_RDCLI_GO_PENDING__BUSY__SHIFT                                                                   0x0
+#define DAGB0_RDCLI_GO_PENDING__BUSY_MASK                                                                     0xFFFFFFFFL
+//DAGB0_RDCLI_GBLSEND_PENDING
+#define DAGB0_RDCLI_GBLSEND_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB0_RDCLI_GBLSEND_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB0_RDCLI_TLB_PENDING
+#define DAGB0_RDCLI_TLB_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB0_RDCLI_TLB_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB0_RDCLI_OARB_PENDING
+#define DAGB0_RDCLI_OARB_PENDING__BUSY__SHIFT                                                                 0x0
+#define DAGB0_RDCLI_OARB_PENDING__BUSY_MASK                                                                   0xFFFFFFFFL
+//DAGB0_RDCLI_OSD_PENDING
+#define DAGB0_RDCLI_OSD_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB0_RDCLI_OSD_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB0_RDCLI_NOALLOC_OVERRIDE
+#define DAGB0_RDCLI_NOALLOC_OVERRIDE__ENABLE__SHIFT                                                           0x0
+#define DAGB0_RDCLI_NOALLOC_OVERRIDE__ENABLE_MASK                                                             0x0000FFFFL
+//DAGB0_RDCLI_NOALLOC_OVERRIDE_VALUE
+#define DAGB0_RDCLI_NOALLOC_OVERRIDE_VALUE__VALUE__SHIFT                                                      0x0
+#define DAGB0_RDCLI_NOALLOC_OVERRIDE_VALUE__VALUE_MASK                                                        0x0000FFFFL
+//DAGB0_WRCLI0
+#define DAGB0_WRCLI0__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_WRCLI0__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_WRCLI0__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_WRCLI0__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_WRCLI0__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_WRCLI0__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_WRCLI0__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_WRCLI0__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_WRCLI0__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_WRCLI0__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_WRCLI0__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_WRCLI0__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_WRCLI0__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_WRCLI0__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_WRCLI0__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_WRCLI0__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_WRCLI0__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_WRCLI0__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_WRCLI0__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_WRCLI0__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_WRCLI1
+#define DAGB0_WRCLI1__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_WRCLI1__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_WRCLI1__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_WRCLI1__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_WRCLI1__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_WRCLI1__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_WRCLI1__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_WRCLI1__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_WRCLI1__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_WRCLI1__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_WRCLI1__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_WRCLI1__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_WRCLI1__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_WRCLI1__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_WRCLI1__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_WRCLI1__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_WRCLI1__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_WRCLI1__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_WRCLI1__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_WRCLI1__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_WRCLI2
+#define DAGB0_WRCLI2__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_WRCLI2__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_WRCLI2__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_WRCLI2__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_WRCLI2__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_WRCLI2__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_WRCLI2__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_WRCLI2__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_WRCLI2__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_WRCLI2__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_WRCLI2__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_WRCLI2__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_WRCLI2__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_WRCLI2__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_WRCLI2__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_WRCLI2__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_WRCLI2__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_WRCLI2__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_WRCLI2__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_WRCLI2__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_WRCLI3
+#define DAGB0_WRCLI3__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_WRCLI3__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_WRCLI3__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_WRCLI3__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_WRCLI3__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_WRCLI3__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_WRCLI3__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_WRCLI3__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_WRCLI3__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_WRCLI3__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_WRCLI3__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_WRCLI3__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_WRCLI3__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_WRCLI3__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_WRCLI3__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_WRCLI3__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_WRCLI3__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_WRCLI3__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_WRCLI3__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_WRCLI3__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_WRCLI4
+#define DAGB0_WRCLI4__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_WRCLI4__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_WRCLI4__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_WRCLI4__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_WRCLI4__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_WRCLI4__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_WRCLI4__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_WRCLI4__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_WRCLI4__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_WRCLI4__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_WRCLI4__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_WRCLI4__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_WRCLI4__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_WRCLI4__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_WRCLI4__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_WRCLI4__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_WRCLI4__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_WRCLI4__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_WRCLI4__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_WRCLI4__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_WRCLI5
+#define DAGB0_WRCLI5__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_WRCLI5__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_WRCLI5__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_WRCLI5__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_WRCLI5__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_WRCLI5__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_WRCLI5__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_WRCLI5__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_WRCLI5__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_WRCLI5__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_WRCLI5__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_WRCLI5__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_WRCLI5__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_WRCLI5__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_WRCLI5__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_WRCLI5__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_WRCLI5__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_WRCLI5__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_WRCLI5__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_WRCLI5__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_WRCLI6
+#define DAGB0_WRCLI6__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_WRCLI6__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_WRCLI6__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_WRCLI6__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_WRCLI6__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_WRCLI6__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_WRCLI6__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_WRCLI6__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_WRCLI6__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_WRCLI6__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_WRCLI6__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_WRCLI6__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_WRCLI6__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_WRCLI6__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_WRCLI6__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_WRCLI6__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_WRCLI6__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_WRCLI6__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_WRCLI6__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_WRCLI6__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_WRCLI7
+#define DAGB0_WRCLI7__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_WRCLI7__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_WRCLI7__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_WRCLI7__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_WRCLI7__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_WRCLI7__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_WRCLI7__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_WRCLI7__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_WRCLI7__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_WRCLI7__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_WRCLI7__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_WRCLI7__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_WRCLI7__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_WRCLI7__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_WRCLI7__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_WRCLI7__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_WRCLI7__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_WRCLI7__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_WRCLI7__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_WRCLI7__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_WRCLI8
+#define DAGB0_WRCLI8__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_WRCLI8__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_WRCLI8__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_WRCLI8__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_WRCLI8__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_WRCLI8__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_WRCLI8__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_WRCLI8__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_WRCLI8__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_WRCLI8__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_WRCLI8__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_WRCLI8__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_WRCLI8__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_WRCLI8__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_WRCLI8__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_WRCLI8__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_WRCLI8__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_WRCLI8__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_WRCLI8__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_WRCLI8__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_WRCLI9
+#define DAGB0_WRCLI9__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB0_WRCLI9__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB0_WRCLI9__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB0_WRCLI9__URG_LOW__SHIFT                                                                          0x8
+#define DAGB0_WRCLI9__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB0_WRCLI9__MAX_BW__SHIFT                                                                           0xd
+#define DAGB0_WRCLI9__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB0_WRCLI9__MIN_BW__SHIFT                                                                           0x16
+#define DAGB0_WRCLI9__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB0_WRCLI9__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB0_WRCLI9__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB0_WRCLI9__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB0_WRCLI9__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB0_WRCLI9__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB0_WRCLI9__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB0_WRCLI9__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB0_WRCLI9__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB0_WRCLI9__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB0_WRCLI9__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB0_WRCLI9__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB0_WRCLI10
+#define DAGB0_WRCLI10__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_WRCLI10__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_WRCLI10__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_WRCLI10__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_WRCLI10__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_WRCLI10__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_WRCLI10__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_WRCLI10__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_WRCLI10__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_WRCLI10__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_WRCLI10__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_WRCLI10__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_WRCLI10__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_WRCLI10__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_WRCLI10__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_WRCLI10__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_WRCLI10__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_WRCLI10__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_WRCLI10__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_WRCLI10__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_WRCLI11
+#define DAGB0_WRCLI11__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_WRCLI11__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_WRCLI11__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_WRCLI11__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_WRCLI11__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_WRCLI11__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_WRCLI11__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_WRCLI11__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_WRCLI11__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_WRCLI11__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_WRCLI11__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_WRCLI11__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_WRCLI11__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_WRCLI11__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_WRCLI11__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_WRCLI11__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_WRCLI11__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_WRCLI11__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_WRCLI11__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_WRCLI11__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_WRCLI12
+#define DAGB0_WRCLI12__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_WRCLI12__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_WRCLI12__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_WRCLI12__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_WRCLI12__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_WRCLI12__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_WRCLI12__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_WRCLI12__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_WRCLI12__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_WRCLI12__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_WRCLI12__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_WRCLI12__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_WRCLI12__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_WRCLI12__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_WRCLI12__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_WRCLI12__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_WRCLI12__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_WRCLI12__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_WRCLI12__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_WRCLI12__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_WRCLI13
+#define DAGB0_WRCLI13__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_WRCLI13__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_WRCLI13__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_WRCLI13__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_WRCLI13__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_WRCLI13__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_WRCLI13__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_WRCLI13__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_WRCLI13__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_WRCLI13__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_WRCLI13__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_WRCLI13__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_WRCLI13__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_WRCLI13__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_WRCLI13__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_WRCLI13__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_WRCLI13__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_WRCLI13__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_WRCLI13__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_WRCLI13__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_WRCLI14
+#define DAGB0_WRCLI14__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_WRCLI14__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_WRCLI14__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_WRCLI14__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_WRCLI14__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_WRCLI14__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_WRCLI14__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_WRCLI14__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_WRCLI14__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_WRCLI14__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_WRCLI14__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_WRCLI14__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_WRCLI14__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_WRCLI14__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_WRCLI14__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_WRCLI14__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_WRCLI14__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_WRCLI14__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_WRCLI14__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_WRCLI14__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_WRCLI15
+#define DAGB0_WRCLI15__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB0_WRCLI15__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB0_WRCLI15__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB0_WRCLI15__URG_LOW__SHIFT                                                                         0x8
+#define DAGB0_WRCLI15__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB0_WRCLI15__MAX_BW__SHIFT                                                                          0xd
+#define DAGB0_WRCLI15__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB0_WRCLI15__MIN_BW__SHIFT                                                                          0x16
+#define DAGB0_WRCLI15__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB0_WRCLI15__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB0_WRCLI15__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB0_WRCLI15__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB0_WRCLI15__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB0_WRCLI15__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB0_WRCLI15__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB0_WRCLI15__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB0_WRCLI15__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB0_WRCLI15__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB0_WRCLI15__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB0_WRCLI15__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB0_WR_CNTL
+#define DAGB0_WR_CNTL__SCLK_FREQ__SHIFT                                                                       0x0
+#define DAGB0_WR_CNTL__CLI_MAX_BW_WINDOW__SHIFT                                                               0x4
+#define DAGB0_WR_CNTL__VC_MAX_BW_WINDOW__SHIFT                                                                0xa
+#define DAGB0_WR_CNTL__IO_LEVEL_OVERRIDE_ENABLE__SHIFT                                                        0x10
+#define DAGB0_WR_CNTL__IO_LEVEL__SHIFT                                                                        0x11
+#define DAGB0_WR_CNTL__IO_LEVEL_COMPLY_VC__SHIFT                                                              0x14
+#define DAGB0_WR_CNTL__SHARE_VC_NUM__SHIFT                                                                    0x17
+#define DAGB0_WR_CNTL__FIX_JUMP__SHIFT                                                                        0x1a
+#define DAGB0_WR_CNTL__SCLK_FREQ_MASK                                                                         0x0000000FL
+#define DAGB0_WR_CNTL__CLI_MAX_BW_WINDOW_MASK                                                                 0x000003F0L
+#define DAGB0_WR_CNTL__VC_MAX_BW_WINDOW_MASK                                                                  0x0000FC00L
+#define DAGB0_WR_CNTL__IO_LEVEL_OVERRIDE_ENABLE_MASK                                                          0x00010000L
+#define DAGB0_WR_CNTL__IO_LEVEL_MASK                                                                          0x000E0000L
+#define DAGB0_WR_CNTL__IO_LEVEL_COMPLY_VC_MASK                                                                0x00700000L
+#define DAGB0_WR_CNTL__SHARE_VC_NUM_MASK                                                                      0x03800000L
+#define DAGB0_WR_CNTL__FIX_JUMP_MASK                                                                          0x04000000L
+//DAGB0_WR_GMI_CNTL
+#define DAGB0_WR_GMI_CNTL__EA_CREDIT__SHIFT                                                                   0x0
+#define DAGB0_WR_GMI_CNTL__LEVEL__SHIFT                                                                       0x6
+#define DAGB0_WR_GMI_CNTL__MAX_BURST__SHIFT                                                                   0x9
+#define DAGB0_WR_GMI_CNTL__LAZY_TIMER__SHIFT                                                                  0xd
+#define DAGB0_WR_GMI_CNTL__EA_CREDIT_MASK                                                                     0x0000003FL
+#define DAGB0_WR_GMI_CNTL__LEVEL_MASK                                                                         0x000001C0L
+#define DAGB0_WR_GMI_CNTL__MAX_BURST_MASK                                                                     0x00001E00L
+#define DAGB0_WR_GMI_CNTL__LAZY_TIMER_MASK                                                                    0x0001E000L
+//DAGB0_WR_ADDR_DAGB
+#define DAGB0_WR_ADDR_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB0_WR_ADDR_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB0_WR_ADDR_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB0_WR_ADDR_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB0_WR_ADDR_DAGB__JUMP_MODE__SHIFT                                                                  0xd
+#define DAGB0_WR_ADDR_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB0_WR_ADDR_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB0_WR_ADDR_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB0_WR_ADDR_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+#define DAGB0_WR_ADDR_DAGB__JUMP_MODE_MASK                                                                    0x00002000L
+//DAGB0_WR_OUTPUT_DAGB_MAX_BURST
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC0__SHIFT                                                            0x0
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC1__SHIFT                                                            0x4
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC2__SHIFT                                                            0x8
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC3__SHIFT                                                            0xc
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC4__SHIFT                                                            0x10
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC5__SHIFT                                                            0x14
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC6__SHIFT                                                            0x18
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC7__SHIFT                                                            0x1c
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC0_MASK                                                              0x0000000FL
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC1_MASK                                                              0x000000F0L
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC2_MASK                                                              0x00000F00L
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC3_MASK                                                              0x0000F000L
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC4_MASK                                                              0x000F0000L
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC5_MASK                                                              0x00F00000L
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC6_MASK                                                              0x0F000000L
+#define DAGB0_WR_OUTPUT_DAGB_MAX_BURST__VC7_MASK                                                              0xF0000000L
+//DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC0__SHIFT                                                           0x0
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC1__SHIFT                                                           0x4
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC2__SHIFT                                                           0x8
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC3__SHIFT                                                           0xc
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC4__SHIFT                                                           0x10
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC5__SHIFT                                                           0x14
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC6__SHIFT                                                           0x18
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC7__SHIFT                                                           0x1c
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC0_MASK                                                             0x0000000FL
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC1_MASK                                                             0x000000F0L
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC2_MASK                                                             0x00000F00L
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC3_MASK                                                             0x0000F000L
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC4_MASK                                                             0x000F0000L
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC5_MASK                                                             0x00F00000L
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC6_MASK                                                             0x0F000000L
+#define DAGB0_WR_OUTPUT_DAGB_LAZY_TIMER__VC7_MASK                                                             0xF0000000L
+//DAGB0_WR_CGTT_CLK_CTRL
+#define DAGB0_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                               0x0
+#define DAGB0_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                         0x4
+#define DAGB0_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                   0xc
+#define DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                            0x1e
+#define DAGB0_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                          0x1f
+#define DAGB0_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                 0x0000000FL
+#define DAGB0_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                           0x00000FF0L
+#define DAGB0_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                     0x0FFFF000L
+#define DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                              0x40000000L
+#define DAGB0_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                            0x80000000L
+//DAGB0_L1TLB_WR_CGTT_CLK_CTRL
+#define DAGB0_L1TLB_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB0_L1TLB_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB0_L1TLB_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB0_L1TLB_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB0_L1TLB_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB0_L1TLB_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB0_L1TLB_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB0_L1TLB_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB0_L1TLB_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB0_L1TLB_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB0_ATCVM_WR_CGTT_CLK_CTRL
+#define DAGB0_ATCVM_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB0_ATCVM_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB0_ATCVM_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB0_ATCVM_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB0_ATCVM_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB0_ATCVM_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB0_ATCVM_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB0_ATCVM_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB0_ATCVM_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB0_ATCVM_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB0_WR_ADDR_DAGB_MAX_BURST0
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB0_WR_ADDR_DAGB_LAZY_TIMER0
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB0_WR_ADDR_DAGB_MAX_BURST1
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB0_WR_ADDR_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB0_WR_ADDR_DAGB_LAZY_TIMER1
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB0_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB0_WR_DATA_DAGB
+#define DAGB0_WR_DATA_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB0_WR_DATA_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB0_WR_DATA_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB0_WR_DATA_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB0_WR_DATA_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB0_WR_DATA_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB0_WR_DATA_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB0_WR_DATA_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+//DAGB0_WR_DATA_DAGB_MAX_BURST0
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB0_WR_DATA_DAGB_LAZY_TIMER0
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB0_WR_DATA_DAGB_MAX_BURST1
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB0_WR_DATA_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB0_WR_DATA_DAGB_LAZY_TIMER1
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB0_WR_DATA_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB0_WR_VC0_CNTL
+#define DAGB0_WR_VC0_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_WR_VC0_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_WR_VC0_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_WR_VC0_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_WR_VC0_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_WR_VC0_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_WR_VC0_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_WR_VC0_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_WR_VC0_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_WR_VC0_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_WR_VC0_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_WR_VC0_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_WR_VC0_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_WR_VC0_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_WR_VC0_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_WR_VC0_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_WR_VC1_CNTL
+#define DAGB0_WR_VC1_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_WR_VC1_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_WR_VC1_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_WR_VC1_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_WR_VC1_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_WR_VC1_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_WR_VC1_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_WR_VC1_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_WR_VC1_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_WR_VC1_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_WR_VC1_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_WR_VC1_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_WR_VC1_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_WR_VC1_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_WR_VC1_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_WR_VC1_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_WR_VC2_CNTL
+#define DAGB0_WR_VC2_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_WR_VC2_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_WR_VC2_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_WR_VC2_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_WR_VC2_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_WR_VC2_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_WR_VC2_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_WR_VC2_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_WR_VC2_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_WR_VC2_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_WR_VC2_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_WR_VC2_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_WR_VC2_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_WR_VC2_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_WR_VC2_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_WR_VC2_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_WR_VC3_CNTL
+#define DAGB0_WR_VC3_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_WR_VC3_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_WR_VC3_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_WR_VC3_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_WR_VC3_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_WR_VC3_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_WR_VC3_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_WR_VC3_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_WR_VC3_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_WR_VC3_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_WR_VC3_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_WR_VC3_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_WR_VC3_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_WR_VC3_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_WR_VC3_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_WR_VC3_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_WR_VC4_CNTL
+#define DAGB0_WR_VC4_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_WR_VC4_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_WR_VC4_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_WR_VC4_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_WR_VC4_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_WR_VC4_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_WR_VC4_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_WR_VC4_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_WR_VC4_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_WR_VC4_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_WR_VC4_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_WR_VC4_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_WR_VC4_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_WR_VC4_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_WR_VC4_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_WR_VC4_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_WR_VC5_CNTL
+#define DAGB0_WR_VC5_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_WR_VC5_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_WR_VC5_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_WR_VC5_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_WR_VC5_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_WR_VC5_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_WR_VC5_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_WR_VC5_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_WR_VC5_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_WR_VC5_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_WR_VC5_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_WR_VC5_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_WR_VC5_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_WR_VC5_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_WR_VC5_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_WR_VC5_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_WR_VC6_CNTL
+#define DAGB0_WR_VC6_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_WR_VC6_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_WR_VC6_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_WR_VC6_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_WR_VC6_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_WR_VC6_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_WR_VC6_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_WR_VC6_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_WR_VC6_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_WR_VC6_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_WR_VC6_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_WR_VC6_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_WR_VC6_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_WR_VC6_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_WR_VC6_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_WR_VC6_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_WR_VC7_CNTL
+#define DAGB0_WR_VC7_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB0_WR_VC7_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB0_WR_VC7_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB0_WR_VC7_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB0_WR_VC7_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB0_WR_VC7_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB0_WR_VC7_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB0_WR_VC7_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB0_WR_VC7_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB0_WR_VC7_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB0_WR_VC7_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB0_WR_VC7_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB0_WR_VC7_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB0_WR_VC7_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB0_WR_VC7_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB0_WR_VC7_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB0_WR_CNTL_MISC
+#define DAGB0_WR_CNTL_MISC__STOR_POOL_CREDIT__SHIFT                                                           0x0
+#define DAGB0_WR_CNTL_MISC__EA_POOL_CREDIT__SHIFT                                                             0x6
+#define DAGB0_WR_CNTL_MISC__IO_EA_CREDIT__SHIFT                                                               0xd
+#define DAGB0_WR_CNTL_MISC__STOR_CC_NEW_MODE__SHIFT                                                           0x13
+#define DAGB0_WR_CNTL_MISC__EA_CC_LEGACY_MODE__SHIFT                                                          0x14
+#define DAGB0_WR_CNTL_MISC__UTCL2_CID__SHIFT                                                                  0x15
+#define DAGB0_WR_CNTL_MISC__RDRET_FIFO_CREDITS__SHIFT                                                         0x1a
+#define DAGB0_WR_CNTL_MISC__STOR_POOL_CREDIT_MASK                                                             0x0000003FL
+#define DAGB0_WR_CNTL_MISC__EA_POOL_CREDIT_MASK                                                               0x00001FC0L
+#define DAGB0_WR_CNTL_MISC__IO_EA_CREDIT_MASK                                                                 0x0007E000L
+#define DAGB0_WR_CNTL_MISC__STOR_CC_NEW_MODE_MASK                                                             0x00080000L
+#define DAGB0_WR_CNTL_MISC__EA_CC_LEGACY_MODE_MASK                                                            0x00100000L
+#define DAGB0_WR_CNTL_MISC__UTCL2_CID_MASK                                                                    0x03E00000L
+#define DAGB0_WR_CNTL_MISC__RDRET_FIFO_CREDITS_MASK                                                           0xFC000000L
+//DAGB0_WR_TLB_CREDIT
+#define DAGB0_WR_TLB_CREDIT__TLB0__SHIFT                                                                      0x0
+#define DAGB0_WR_TLB_CREDIT__TLB1__SHIFT                                                                      0x5
+#define DAGB0_WR_TLB_CREDIT__TLB2__SHIFT                                                                      0xa
+#define DAGB0_WR_TLB_CREDIT__TLB3__SHIFT                                                                      0xf
+#define DAGB0_WR_TLB_CREDIT__TLB4__SHIFT                                                                      0x14
+#define DAGB0_WR_TLB_CREDIT__TLB5__SHIFT                                                                      0x19
+#define DAGB0_WR_TLB_CREDIT__TLB0_MASK                                                                        0x0000001FL
+#define DAGB0_WR_TLB_CREDIT__TLB1_MASK                                                                        0x000003E0L
+#define DAGB0_WR_TLB_CREDIT__TLB2_MASK                                                                        0x00007C00L
+#define DAGB0_WR_TLB_CREDIT__TLB3_MASK                                                                        0x000F8000L
+#define DAGB0_WR_TLB_CREDIT__TLB4_MASK                                                                        0x01F00000L
+#define DAGB0_WR_TLB_CREDIT__TLB5_MASK                                                                        0x3E000000L
+//DAGB0_WR_DATA_CREDIT
+#define DAGB0_WR_DATA_CREDIT__DLOCK_VC_CREDITS__SHIFT                                                         0x0
+#define DAGB0_WR_DATA_CREDIT__LARGE_BURST_CREDITS__SHIFT                                                      0x8
+#define DAGB0_WR_DATA_CREDIT__MIDDLE_BURST_CREDITS__SHIFT                                                     0x10
+#define DAGB0_WR_DATA_CREDIT__SMALL_BURST_CREDITS__SHIFT                                                      0x18
+#define DAGB0_WR_DATA_CREDIT__DLOCK_VC_CREDITS_MASK                                                           0x000000FFL
+#define DAGB0_WR_DATA_CREDIT__LARGE_BURST_CREDITS_MASK                                                        0x0000FF00L
+#define DAGB0_WR_DATA_CREDIT__MIDDLE_BURST_CREDITS_MASK                                                       0x00FF0000L
+#define DAGB0_WR_DATA_CREDIT__SMALL_BURST_CREDITS_MASK                                                        0xFF000000L
+//DAGB0_WR_MISC_CREDIT
+#define DAGB0_WR_MISC_CREDIT__ATOMIC_CREDIT__SHIFT                                                            0x0
+#define DAGB0_WR_MISC_CREDIT__DLOCK_VC_NUM__SHIFT                                                             0x6
+#define DAGB0_WR_MISC_CREDIT__OSD_CREDIT__SHIFT                                                               0x9
+#define DAGB0_WR_MISC_CREDIT__OSD_DLOCK_CREDIT__SHIFT                                                         0x10
+#define DAGB0_WR_MISC_CREDIT__ATOMIC_CREDIT_MASK                                                              0x0000003FL
+#define DAGB0_WR_MISC_CREDIT__DLOCK_VC_NUM_MASK                                                               0x000001C0L
+#define DAGB0_WR_MISC_CREDIT__OSD_CREDIT_MASK                                                                 0x0000FE00L
+#define DAGB0_WR_MISC_CREDIT__OSD_DLOCK_CREDIT_MASK                                                           0x007F0000L
+//DAGB0_WR_OSD_CREDIT_CNTL1
+#define DAGB0_WR_OSD_CREDIT_CNTL1__VC0_CREDIT__SHIFT                                                          0x0
+#define DAGB0_WR_OSD_CREDIT_CNTL1__VC1_CREDIT__SHIFT                                                          0x4
+#define DAGB0_WR_OSD_CREDIT_CNTL1__VC2_CREDIT__SHIFT                                                          0x8
+#define DAGB0_WR_OSD_CREDIT_CNTL1__VC3_CREDIT__SHIFT                                                          0xc
+#define DAGB0_WR_OSD_CREDIT_CNTL1__IO_CREDIT__SHIFT                                                           0x10
+#define DAGB0_WR_OSD_CREDIT_CNTL1__GMI_CREDIT__SHIFT                                                          0x14
+#define DAGB0_WR_OSD_CREDIT_CNTL1__POOL_CREDIT__SHIFT                                                         0x18
+#define DAGB0_WR_OSD_CREDIT_CNTL1__VC0_CREDIT_MASK                                                            0x0000000FL
+#define DAGB0_WR_OSD_CREDIT_CNTL1__VC1_CREDIT_MASK                                                            0x000000F0L
+#define DAGB0_WR_OSD_CREDIT_CNTL1__VC2_CREDIT_MASK                                                            0x00000F00L
+#define DAGB0_WR_OSD_CREDIT_CNTL1__VC3_CREDIT_MASK                                                            0x0000F000L
+#define DAGB0_WR_OSD_CREDIT_CNTL1__IO_CREDIT_MASK                                                             0x000F0000L
+#define DAGB0_WR_OSD_CREDIT_CNTL1__GMI_CREDIT_MASK                                                            0x00F00000L
+#define DAGB0_WR_OSD_CREDIT_CNTL1__POOL_CREDIT_MASK                                                           0x3F000000L
+//DAGB0_WR_OSD_CREDIT_CNTL2
+#define DAGB0_WR_OSD_CREDIT_CNTL2__CREDIT_MARGIN__SHIFT                                                       0x0
+#define DAGB0_WR_OSD_CREDIT_CNTL2__ENABLE_LEGACY__SHIFT                                                       0x4
+#define DAGB0_WR_OSD_CREDIT_CNTL2__CREDIT_MARGIN_MASK                                                         0x0000000FL
+#define DAGB0_WR_OSD_CREDIT_CNTL2__ENABLE_LEGACY_MASK                                                         0x00000010L
+//DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC0_CREDIT__SHIFT                                                  0x0
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC1_CREDIT__SHIFT                                                  0x5
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC2_CREDIT__SHIFT                                                  0xa
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC3_CREDIT__SHIFT                                                  0xf
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__POOL_CREDIT__SHIFT                                                 0x14
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC_MODE__SHIFT                                                     0x19
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX_EQ__SHIFT                                                      0x1a
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX0__SHIFT                                                        0x1b
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX1__SHIFT                                                        0x1c
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX2__SHIFT                                                        0x1d
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC0_CREDIT_MASK                                                    0x0000001FL
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC1_CREDIT_MASK                                                    0x000003E0L
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC2_CREDIT_MASK                                                    0x00007C00L
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC3_CREDIT_MASK                                                    0x000F8000L
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__POOL_CREDIT_MASK                                                   0x01F00000L
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC_MODE_MASK                                                       0x02000000L
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX_EQ_MASK                                                        0x04000000L
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX0_MASK                                                          0x08000000L
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX1_MASK                                                          0x10000000L
+#define DAGB0_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX2_MASK                                                          0x20000000L
+//DAGB0_WRCLI_GPU_SNOOP_OVERRIDE
+#define DAGB0_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT                                                         0x0
+#define DAGB0_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK                                                           0x0000FFFFL
+//DAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE
+#define DAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT                                                   0x0
+#define DAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK                                                     0x0000FFFFL
+//DAGB0_WRCLI_ASK_PENDING
+#define DAGB0_WRCLI_ASK_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB0_WRCLI_ASK_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB0_WRCLI_GO_PENDING
+#define DAGB0_WRCLI_GO_PENDING__BUSY__SHIFT                                                                   0x0
+#define DAGB0_WRCLI_GO_PENDING__BUSY_MASK                                                                     0xFFFFFFFFL
+//DAGB0_WRCLI_GBLSEND_PENDING
+#define DAGB0_WRCLI_GBLSEND_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB0_WRCLI_GBLSEND_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB0_WRCLI_TLB_PENDING
+#define DAGB0_WRCLI_TLB_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB0_WRCLI_TLB_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB0_WRCLI_OARB_PENDING
+#define DAGB0_WRCLI_OARB_PENDING__BUSY__SHIFT                                                                 0x0
+#define DAGB0_WRCLI_OARB_PENDING__BUSY_MASK                                                                   0xFFFFFFFFL
+//DAGB0_WRCLI_OSD_PENDING
+#define DAGB0_WRCLI_OSD_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB0_WRCLI_OSD_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB0_WRCLI_DBUS_ASK_PENDING
+#define DAGB0_WRCLI_DBUS_ASK_PENDING__BUSY__SHIFT                                                             0x0
+#define DAGB0_WRCLI_DBUS_ASK_PENDING__BUSY_MASK                                                               0xFFFFFFFFL
+//DAGB0_WRCLI_DBUS_GO_PENDING
+#define DAGB0_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB0_WRCLI_DBUS_GO_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB0_WRCLI_NOALLOC_OVERRIDE
+#define DAGB0_WRCLI_NOALLOC_OVERRIDE__ENABLE__SHIFT                                                           0x0
+#define DAGB0_WRCLI_NOALLOC_OVERRIDE__ENABLE_MASK                                                             0x0000FFFFL
+//DAGB0_WRCLI_NOALLOC_OVERRIDE_VALUE
+#define DAGB0_WRCLI_NOALLOC_OVERRIDE_VALUE__VALUE__SHIFT                                                      0x0
+#define DAGB0_WRCLI_NOALLOC_OVERRIDE_VALUE__VALUE_MASK                                                        0x0000FFFFL
+//DAGB0_DAGB_DLY
+#define DAGB0_DAGB_DLY__DLY__SHIFT                                                                            0x0
+#define DAGB0_DAGB_DLY__CLI__SHIFT                                                                            0x8
+#define DAGB0_DAGB_DLY__POS__SHIFT                                                                            0x10
+#define DAGB0_DAGB_DLY__DLY_MASK                                                                              0x000000FFL
+#define DAGB0_DAGB_DLY__CLI_MASK                                                                              0x0000FF00L
+#define DAGB0_DAGB_DLY__POS_MASK                                                                              0x000F0000L
+//DAGB0_CNTL_MISC
+#define DAGB0_CNTL_MISC__EA_VC0_REMAP__SHIFT                                                                  0x0
+#define DAGB0_CNTL_MISC__EA_VC1_REMAP__SHIFT                                                                  0x3
+#define DAGB0_CNTL_MISC__EA_VC2_REMAP__SHIFT                                                                  0x6
+#define DAGB0_CNTL_MISC__EA_VC3_REMAP__SHIFT                                                                  0x9
+#define DAGB0_CNTL_MISC__EA_VC4_REMAP__SHIFT                                                                  0xc
+#define DAGB0_CNTL_MISC__EA_VC5_REMAP__SHIFT                                                                  0xf
+#define DAGB0_CNTL_MISC__EA_VC6_REMAP__SHIFT                                                                  0x12
+#define DAGB0_CNTL_MISC__EA_VC7_REMAP__SHIFT                                                                  0x15
+#define DAGB0_CNTL_MISC__BW_INIT_CYCLE__SHIFT                                                                 0x18
+#define DAGB0_CNTL_MISC__BW_RW_GAP_CYCLE__SHIFT                                                               0x1e
+#define DAGB0_CNTL_MISC__EA_VC0_REMAP_MASK                                                                    0x00000007L
+#define DAGB0_CNTL_MISC__EA_VC1_REMAP_MASK                                                                    0x00000038L
+#define DAGB0_CNTL_MISC__EA_VC2_REMAP_MASK                                                                    0x000001C0L
+#define DAGB0_CNTL_MISC__EA_VC3_REMAP_MASK                                                                    0x00000E00L
+#define DAGB0_CNTL_MISC__EA_VC4_REMAP_MASK                                                                    0x00007000L
+#define DAGB0_CNTL_MISC__EA_VC5_REMAP_MASK                                                                    0x00038000L
+#define DAGB0_CNTL_MISC__EA_VC6_REMAP_MASK                                                                    0x001C0000L
+#define DAGB0_CNTL_MISC__EA_VC7_REMAP_MASK                                                                    0x00E00000L
+#define DAGB0_CNTL_MISC__BW_INIT_CYCLE_MASK                                                                   0x3F000000L
+#define DAGB0_CNTL_MISC__BW_RW_GAP_CYCLE_MASK                                                                 0xC0000000L
+//DAGB0_CNTL_MISC2
+#define DAGB0_CNTL_MISC2__URG_BOOST_ENABLE__SHIFT                                                             0x0
+#define DAGB0_CNTL_MISC2__URG_HALT_ENABLE__SHIFT                                                              0x1
+#define DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG__SHIFT                                                             0x2
+#define DAGB0_CNTL_MISC2__DISABLE_WRRET_CG__SHIFT                                                             0x3
+#define DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG__SHIFT                                                             0x4
+#define DAGB0_CNTL_MISC2__DISABLE_RDRET_CG__SHIFT                                                             0x5
+#define DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG__SHIFT                                                             0x6
+#define DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG__SHIFT                                                             0x7
+#define DAGB0_CNTL_MISC2__DISABLE_EAWRREQ_BUSY__SHIFT                                                         0x8
+#define DAGB0_CNTL_MISC2__DISABLE_EARDREQ_BUSY__SHIFT                                                         0x9
+#define DAGB0_CNTL_MISC2__SWAP_CTL__SHIFT                                                                     0xa
+#define DAGB0_CNTL_MISC2__RDRET_FIFO_PERF__SHIFT                                                              0xb
+#define DAGB0_CNTL_MISC2__HDP_CID__SHIFT                                                                      0xc
+#define DAGB0_CNTL_MISC2__RDRET_FIFO_DLOCK_CREDITS__SHIFT                                                     0x10
+#define DAGB0_CNTL_MISC2__URG_BOOST_ENABLE_MASK                                                               0x00000001L
+#define DAGB0_CNTL_MISC2__URG_HALT_ENABLE_MASK                                                                0x00000002L
+#define DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK                                                               0x00000004L
+#define DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK                                                               0x00000008L
+#define DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK                                                               0x00000010L
+#define DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK                                                               0x00000020L
+#define DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK                                                               0x00000040L
+#define DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK                                                               0x00000080L
+#define DAGB0_CNTL_MISC2__DISABLE_EAWRREQ_BUSY_MASK                                                           0x00000100L
+#define DAGB0_CNTL_MISC2__DISABLE_EARDREQ_BUSY_MASK                                                           0x00000200L
+#define DAGB0_CNTL_MISC2__SWAP_CTL_MASK                                                                       0x00000400L
+#define DAGB0_CNTL_MISC2__RDRET_FIFO_PERF_MASK                                                                0x00000800L
+#define DAGB0_CNTL_MISC2__HDP_CID_MASK                                                                        0x0000F000L
+#define DAGB0_CNTL_MISC2__RDRET_FIFO_DLOCK_CREDITS_MASK                                                       0x003F0000L
+//DAGB0_FATAL_ERROR_CNTL
+#define DAGB0_FATAL_ERROR_CNTL__FILTER_NUM__SHIFT                                                             0x0
+#define DAGB0_FATAL_ERROR_CNTL__FILTER_NUM_MASK                                                               0x000003FFL
+//DAGB0_FATAL_ERROR_CLEAR
+#define DAGB0_FATAL_ERROR_CLEAR__CLEAR__SHIFT                                                                 0x0
+#define DAGB0_FATAL_ERROR_CLEAR__CLEAR_MASK                                                                   0x00000001L
+//DAGB0_FATAL_ERROR_STATUS0
+#define DAGB0_FATAL_ERROR_STATUS0__VALID__SHIFT                                                               0x0
+#define DAGB0_FATAL_ERROR_STATUS0__CID__SHIFT                                                                 0x1
+#define DAGB0_FATAL_ERROR_STATUS0__ADDR_LO__SHIFT                                                             0x6
+#define DAGB0_FATAL_ERROR_STATUS0__VALID_MASK                                                                 0x00000001L
+#define DAGB0_FATAL_ERROR_STATUS0__CID_MASK                                                                   0x0000003EL
+#define DAGB0_FATAL_ERROR_STATUS0__ADDR_LO_MASK                                                               0xFFFFFFC0L
+//DAGB0_FATAL_ERROR_STATUS1
+#define DAGB0_FATAL_ERROR_STATUS1__ADDR_HI__SHIFT                                                             0x0
+#define DAGB0_FATAL_ERROR_STATUS1__ADDR_HI_MASK                                                               0x0001FFFFL
+//DAGB0_FATAL_ERROR_STATUS2
+#define DAGB0_FATAL_ERROR_STATUS2__TAG__SHIFT                                                                 0x0
+#define DAGB0_FATAL_ERROR_STATUS2__VFID__SHIFT                                                                0x10
+#define DAGB0_FATAL_ERROR_STATUS2__VF__SHIFT                                                                  0x14
+#define DAGB0_FATAL_ERROR_STATUS2__SPACE__SHIFT                                                               0x15
+#define DAGB0_FATAL_ERROR_STATUS2__IO__SHIFT                                                                  0x16
+#define DAGB0_FATAL_ERROR_STATUS2__SIZE__SHIFT                                                                0x17
+#define DAGB0_FATAL_ERROR_STATUS2__DBGMSK__SHIFT                                                              0x18
+#define DAGB0_FATAL_ERROR_STATUS2__FED__SHIFT                                                                 0x19
+#define DAGB0_FATAL_ERROR_STATUS2__TAG_MASK                                                                   0x0000FFFFL
+#define DAGB0_FATAL_ERROR_STATUS2__VFID_MASK                                                                  0x000F0000L
+#define DAGB0_FATAL_ERROR_STATUS2__VF_MASK                                                                    0x00100000L
+#define DAGB0_FATAL_ERROR_STATUS2__SPACE_MASK                                                                 0x00200000L
+#define DAGB0_FATAL_ERROR_STATUS2__IO_MASK                                                                    0x00400000L
+#define DAGB0_FATAL_ERROR_STATUS2__SIZE_MASK                                                                  0x00800000L
+#define DAGB0_FATAL_ERROR_STATUS2__DBGMSK_MASK                                                                0x01000000L
+#define DAGB0_FATAL_ERROR_STATUS2__FED_MASK                                                                   0x02000000L
+//DAGB0_FATAL_ERROR_STATUS3
+#define DAGB0_FATAL_ERROR_STATUS3__NOALLOC__SHIFT                                                             0x0
+#define DAGB0_FATAL_ERROR_STATUS3__UNITID__SHIFT                                                              0x1
+#define DAGB0_FATAL_ERROR_STATUS3__OP__SHIFT                                                                  0x7
+#define DAGB0_FATAL_ERROR_STATUS3__SECLEVEL__SHIFT                                                            0xe
+#define DAGB0_FATAL_ERROR_STATUS3__WRTMZ__SHIFT                                                               0x11
+#define DAGB0_FATAL_ERROR_STATUS3__RDTMZ__SHIFT                                                               0x12
+#define DAGB0_FATAL_ERROR_STATUS3__SNOOP__SHIFT                                                               0x13
+#define DAGB0_FATAL_ERROR_STATUS3__INVAL__SHIFT                                                               0x14
+#define DAGB0_FATAL_ERROR_STATUS3__NACK__SHIFT                                                                0x15
+#define DAGB0_FATAL_ERROR_STATUS3__RO__SHIFT                                                                  0x17
+#define DAGB0_FATAL_ERROR_STATUS3__MEMLOG__SHIFT                                                              0x18
+#define DAGB0_FATAL_ERROR_STATUS3__EOP__SHIFT                                                                 0x19
+#define DAGB0_FATAL_ERROR_STATUS3__NOALLOC_MASK                                                               0x00000001L
+#define DAGB0_FATAL_ERROR_STATUS3__UNITID_MASK                                                                0x0000007EL
+#define DAGB0_FATAL_ERROR_STATUS3__OP_MASK                                                                    0x00003F80L
+#define DAGB0_FATAL_ERROR_STATUS3__SECLEVEL_MASK                                                              0x0001C000L
+#define DAGB0_FATAL_ERROR_STATUS3__WRTMZ_MASK                                                                 0x00020000L
+#define DAGB0_FATAL_ERROR_STATUS3__RDTMZ_MASK                                                                 0x00040000L
+#define DAGB0_FATAL_ERROR_STATUS3__SNOOP_MASK                                                                 0x00080000L
+#define DAGB0_FATAL_ERROR_STATUS3__INVAL_MASK                                                                 0x00100000L
+#define DAGB0_FATAL_ERROR_STATUS3__NACK_MASK                                                                  0x00600000L
+#define DAGB0_FATAL_ERROR_STATUS3__RO_MASK                                                                    0x00800000L
+#define DAGB0_FATAL_ERROR_STATUS3__MEMLOG_MASK                                                                0x01000000L
+#define DAGB0_FATAL_ERROR_STATUS3__EOP_MASK                                                                   0x02000000L
+//DAGB0_FIFO_EMPTY
+#define DAGB0_FIFO_EMPTY__EMPTY__SHIFT                                                                        0x0
+#define DAGB0_FIFO_EMPTY__EMPTY_MASK                                                                          0x00FFFFFFL
+//DAGB0_FIFO_FULL
+#define DAGB0_FIFO_FULL__FULL__SHIFT                                                                          0x0
+#define DAGB0_FIFO_FULL__FULL_MASK                                                                            0x007FFFFFL
+//DAGB0_WR_CREDITS_FULL
+#define DAGB0_WR_CREDITS_FULL__FULL__SHIFT                                                                    0x0
+#define DAGB0_WR_CREDITS_FULL__FULL_MASK                                                                      0x1FFFFFFFL
+//DAGB0_RD_CREDITS_FULL
+#define DAGB0_RD_CREDITS_FULL__FULL__SHIFT                                                                    0x0
+#define DAGB0_RD_CREDITS_FULL__FULL_MASK                                                                      0x0003FFFFL
+//DAGB0_PERFCOUNTER_LO
+#define DAGB0_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define DAGB0_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//DAGB0_PERFCOUNTER_HI
+#define DAGB0_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define DAGB0_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define DAGB0_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define DAGB0_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+//DAGB0_PERFCOUNTER0_CFG
+#define DAGB0_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB0_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB0_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB0_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB0_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB0_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB0_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB0_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB0_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB0_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB0_PERFCOUNTER1_CFG
+#define DAGB0_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB0_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB0_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB0_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB0_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB0_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB0_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB0_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB0_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB0_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB0_PERFCOUNTER2_CFG
+#define DAGB0_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB0_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB0_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB0_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB0_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB0_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB0_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB0_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB0_PERFCOUNTER2_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB0_PERFCOUNTER2_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB0_PERFCOUNTER_RSLT_CNTL
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define DAGB0_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+//DAGB0_L1TLB_REG_RW
+#define DAGB0_L1TLB_REG_RW__REG_WRITE_L1TLB_CTRL__SHIFT                                                       0x0
+#define DAGB0_L1TLB_REG_RW__REG_READ_L1TLB_CTRL__SHIFT                                                        0x1
+#define DAGB0_L1TLB_REG_RW__VMID_EXCEP_INT_CTRL__SHIFT                                                        0x2
+#define DAGB0_L1TLB_REG_RW__WDAT_PARITY_CHECK__SHIFT                                                          0x4
+#define DAGB0_L1TLB_REG_RW__DISABLE_RDRET_CHECK__SHIFT                                                        0x5
+#define DAGB0_L1TLB_REG_RW__RESERVE__SHIFT                                                                    0x6
+#define DAGB0_L1TLB_REG_RW__REG_WRITE_L1TLB_CTRL_MASK                                                         0x00000001L
+#define DAGB0_L1TLB_REG_RW__REG_READ_L1TLB_CTRL_MASK                                                          0x00000002L
+#define DAGB0_L1TLB_REG_RW__VMID_EXCEP_INT_CTRL_MASK                                                          0x00000004L
+#define DAGB0_L1TLB_REG_RW__WDAT_PARITY_CHECK_MASK                                                            0x00000010L
+#define DAGB0_L1TLB_REG_RW__DISABLE_RDRET_CHECK_MASK                                                          0x00000020L
+#define DAGB0_L1TLB_REG_RW__RESERVE_MASK                                                                      0xFFFFFFC0L
+
+
+// addressBlock: aid_mmhub_dagb_dagbdec1
+//DAGB1_RDCLI0
+#define DAGB1_RDCLI0__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_RDCLI0__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_RDCLI0__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_RDCLI0__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_RDCLI0__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_RDCLI0__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_RDCLI0__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_RDCLI0__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_RDCLI0__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_RDCLI0__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_RDCLI0__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_RDCLI0__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_RDCLI0__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_RDCLI0__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_RDCLI0__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_RDCLI0__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_RDCLI0__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_RDCLI0__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_RDCLI0__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_RDCLI0__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_RDCLI1
+#define DAGB1_RDCLI1__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_RDCLI1__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_RDCLI1__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_RDCLI1__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_RDCLI1__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_RDCLI1__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_RDCLI1__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_RDCLI1__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_RDCLI1__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_RDCLI1__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_RDCLI1__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_RDCLI1__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_RDCLI1__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_RDCLI1__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_RDCLI1__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_RDCLI1__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_RDCLI1__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_RDCLI1__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_RDCLI1__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_RDCLI1__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_RDCLI2
+#define DAGB1_RDCLI2__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_RDCLI2__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_RDCLI2__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_RDCLI2__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_RDCLI2__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_RDCLI2__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_RDCLI2__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_RDCLI2__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_RDCLI2__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_RDCLI2__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_RDCLI2__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_RDCLI2__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_RDCLI2__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_RDCLI2__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_RDCLI2__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_RDCLI2__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_RDCLI2__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_RDCLI2__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_RDCLI2__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_RDCLI2__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_RDCLI3
+#define DAGB1_RDCLI3__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_RDCLI3__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_RDCLI3__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_RDCLI3__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_RDCLI3__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_RDCLI3__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_RDCLI3__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_RDCLI3__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_RDCLI3__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_RDCLI3__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_RDCLI3__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_RDCLI3__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_RDCLI3__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_RDCLI3__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_RDCLI3__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_RDCLI3__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_RDCLI3__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_RDCLI3__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_RDCLI3__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_RDCLI3__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_RDCLI4
+#define DAGB1_RDCLI4__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_RDCLI4__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_RDCLI4__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_RDCLI4__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_RDCLI4__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_RDCLI4__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_RDCLI4__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_RDCLI4__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_RDCLI4__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_RDCLI4__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_RDCLI4__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_RDCLI4__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_RDCLI4__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_RDCLI4__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_RDCLI4__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_RDCLI4__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_RDCLI4__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_RDCLI4__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_RDCLI4__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_RDCLI4__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_RDCLI5
+#define DAGB1_RDCLI5__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_RDCLI5__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_RDCLI5__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_RDCLI5__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_RDCLI5__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_RDCLI5__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_RDCLI5__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_RDCLI5__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_RDCLI5__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_RDCLI5__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_RDCLI5__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_RDCLI5__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_RDCLI5__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_RDCLI5__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_RDCLI5__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_RDCLI5__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_RDCLI5__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_RDCLI5__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_RDCLI5__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_RDCLI5__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_RDCLI6
+#define DAGB1_RDCLI6__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_RDCLI6__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_RDCLI6__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_RDCLI6__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_RDCLI6__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_RDCLI6__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_RDCLI6__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_RDCLI6__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_RDCLI6__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_RDCLI6__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_RDCLI6__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_RDCLI6__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_RDCLI6__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_RDCLI6__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_RDCLI6__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_RDCLI6__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_RDCLI6__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_RDCLI6__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_RDCLI6__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_RDCLI6__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_RDCLI7
+#define DAGB1_RDCLI7__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_RDCLI7__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_RDCLI7__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_RDCLI7__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_RDCLI7__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_RDCLI7__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_RDCLI7__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_RDCLI7__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_RDCLI7__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_RDCLI7__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_RDCLI7__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_RDCLI7__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_RDCLI7__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_RDCLI7__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_RDCLI7__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_RDCLI7__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_RDCLI7__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_RDCLI7__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_RDCLI7__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_RDCLI7__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_RDCLI8
+#define DAGB1_RDCLI8__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_RDCLI8__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_RDCLI8__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_RDCLI8__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_RDCLI8__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_RDCLI8__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_RDCLI8__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_RDCLI8__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_RDCLI8__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_RDCLI8__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_RDCLI8__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_RDCLI8__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_RDCLI8__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_RDCLI8__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_RDCLI8__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_RDCLI8__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_RDCLI8__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_RDCLI8__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_RDCLI8__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_RDCLI8__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_RDCLI9
+#define DAGB1_RDCLI9__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_RDCLI9__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_RDCLI9__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_RDCLI9__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_RDCLI9__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_RDCLI9__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_RDCLI9__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_RDCLI9__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_RDCLI9__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_RDCLI9__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_RDCLI9__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_RDCLI9__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_RDCLI9__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_RDCLI9__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_RDCLI9__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_RDCLI9__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_RDCLI9__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_RDCLI9__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_RDCLI9__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_RDCLI9__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_RDCLI10
+#define DAGB1_RDCLI10__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_RDCLI10__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_RDCLI10__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_RDCLI10__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_RDCLI10__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_RDCLI10__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_RDCLI10__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_RDCLI10__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_RDCLI10__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_RDCLI10__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_RDCLI10__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_RDCLI10__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_RDCLI10__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_RDCLI10__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_RDCLI10__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_RDCLI10__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_RDCLI10__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_RDCLI10__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_RDCLI10__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_RDCLI10__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_RDCLI11
+#define DAGB1_RDCLI11__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_RDCLI11__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_RDCLI11__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_RDCLI11__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_RDCLI11__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_RDCLI11__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_RDCLI11__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_RDCLI11__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_RDCLI11__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_RDCLI11__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_RDCLI11__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_RDCLI11__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_RDCLI11__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_RDCLI11__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_RDCLI11__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_RDCLI11__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_RDCLI11__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_RDCLI11__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_RDCLI11__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_RDCLI11__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_RDCLI12
+#define DAGB1_RDCLI12__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_RDCLI12__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_RDCLI12__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_RDCLI12__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_RDCLI12__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_RDCLI12__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_RDCLI12__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_RDCLI12__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_RDCLI12__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_RDCLI12__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_RDCLI12__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_RDCLI12__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_RDCLI12__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_RDCLI12__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_RDCLI12__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_RDCLI12__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_RDCLI12__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_RDCLI12__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_RDCLI12__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_RDCLI12__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_RDCLI13
+#define DAGB1_RDCLI13__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_RDCLI13__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_RDCLI13__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_RDCLI13__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_RDCLI13__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_RDCLI13__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_RDCLI13__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_RDCLI13__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_RDCLI13__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_RDCLI13__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_RDCLI13__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_RDCLI13__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_RDCLI13__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_RDCLI13__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_RDCLI13__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_RDCLI13__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_RDCLI13__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_RDCLI13__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_RDCLI13__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_RDCLI13__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_RDCLI14
+#define DAGB1_RDCLI14__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_RDCLI14__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_RDCLI14__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_RDCLI14__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_RDCLI14__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_RDCLI14__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_RDCLI14__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_RDCLI14__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_RDCLI14__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_RDCLI14__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_RDCLI14__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_RDCLI14__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_RDCLI14__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_RDCLI14__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_RDCLI14__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_RDCLI14__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_RDCLI14__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_RDCLI14__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_RDCLI14__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_RDCLI14__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_RDCLI15
+#define DAGB1_RDCLI15__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_RDCLI15__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_RDCLI15__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_RDCLI15__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_RDCLI15__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_RDCLI15__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_RDCLI15__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_RDCLI15__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_RDCLI15__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_RDCLI15__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_RDCLI15__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_RDCLI15__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_RDCLI15__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_RDCLI15__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_RDCLI15__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_RDCLI15__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_RDCLI15__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_RDCLI15__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_RDCLI15__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_RDCLI15__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_RD_CNTL
+#define DAGB1_RD_CNTL__SCLK_FREQ__SHIFT                                                                       0x0
+#define DAGB1_RD_CNTL__CLI_MAX_BW_WINDOW__SHIFT                                                               0x4
+#define DAGB1_RD_CNTL__VC_MAX_BW_WINDOW__SHIFT                                                                0xa
+#define DAGB1_RD_CNTL__IO_LEVEL_OVERRIDE_ENABLE__SHIFT                                                        0x10
+#define DAGB1_RD_CNTL__IO_LEVEL__SHIFT                                                                        0x11
+#define DAGB1_RD_CNTL__IO_LEVEL_COMPLY_VC__SHIFT                                                              0x14
+#define DAGB1_RD_CNTL__SHARE_VC_NUM__SHIFT                                                                    0x17
+#define DAGB1_RD_CNTL__FIX_JUMP__SHIFT                                                                        0x1a
+#define DAGB1_RD_CNTL__SCLK_FREQ_MASK                                                                         0x0000000FL
+#define DAGB1_RD_CNTL__CLI_MAX_BW_WINDOW_MASK                                                                 0x000003F0L
+#define DAGB1_RD_CNTL__VC_MAX_BW_WINDOW_MASK                                                                  0x0000FC00L
+#define DAGB1_RD_CNTL__IO_LEVEL_OVERRIDE_ENABLE_MASK                                                          0x00010000L
+#define DAGB1_RD_CNTL__IO_LEVEL_MASK                                                                          0x000E0000L
+#define DAGB1_RD_CNTL__IO_LEVEL_COMPLY_VC_MASK                                                                0x00700000L
+#define DAGB1_RD_CNTL__SHARE_VC_NUM_MASK                                                                      0x03800000L
+#define DAGB1_RD_CNTL__FIX_JUMP_MASK                                                                          0x04000000L
+//DAGB1_RD_GMI_CNTL
+#define DAGB1_RD_GMI_CNTL__EA_CREDIT__SHIFT                                                                   0x0
+#define DAGB1_RD_GMI_CNTL__LEVEL__SHIFT                                                                       0x6
+#define DAGB1_RD_GMI_CNTL__MAX_BURST__SHIFT                                                                   0x9
+#define DAGB1_RD_GMI_CNTL__LAZY_TIMER__SHIFT                                                                  0xd
+#define DAGB1_RD_GMI_CNTL__EA_CREDIT_MASK                                                                     0x0000003FL
+#define DAGB1_RD_GMI_CNTL__LEVEL_MASK                                                                         0x000001C0L
+#define DAGB1_RD_GMI_CNTL__MAX_BURST_MASK                                                                     0x00001E00L
+#define DAGB1_RD_GMI_CNTL__LAZY_TIMER_MASK                                                                    0x0001E000L
+//DAGB1_RD_ADDR_DAGB
+#define DAGB1_RD_ADDR_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB1_RD_ADDR_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB1_RD_ADDR_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB1_RD_ADDR_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB1_RD_ADDR_DAGB__JUMP_MODE__SHIFT                                                                  0xd
+#define DAGB1_RD_ADDR_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB1_RD_ADDR_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB1_RD_ADDR_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB1_RD_ADDR_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+#define DAGB1_RD_ADDR_DAGB__JUMP_MODE_MASK                                                                    0x00002000L
+//DAGB1_RD_OUTPUT_DAGB_MAX_BURST
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC0__SHIFT                                                            0x0
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC1__SHIFT                                                            0x4
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC2__SHIFT                                                            0x8
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC3__SHIFT                                                            0xc
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC4__SHIFT                                                            0x10
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC5__SHIFT                                                            0x14
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC6__SHIFT                                                            0x18
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC7__SHIFT                                                            0x1c
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC0_MASK                                                              0x0000000FL
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC1_MASK                                                              0x000000F0L
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC2_MASK                                                              0x00000F00L
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC3_MASK                                                              0x0000F000L
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC4_MASK                                                              0x000F0000L
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC5_MASK                                                              0x00F00000L
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC6_MASK                                                              0x0F000000L
+#define DAGB1_RD_OUTPUT_DAGB_MAX_BURST__VC7_MASK                                                              0xF0000000L
+//DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC0__SHIFT                                                           0x0
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC1__SHIFT                                                           0x4
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC2__SHIFT                                                           0x8
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC3__SHIFT                                                           0xc
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC4__SHIFT                                                           0x10
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC5__SHIFT                                                           0x14
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC6__SHIFT                                                           0x18
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC7__SHIFT                                                           0x1c
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC0_MASK                                                             0x0000000FL
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC1_MASK                                                             0x000000F0L
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC2_MASK                                                             0x00000F00L
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC3_MASK                                                             0x0000F000L
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC4_MASK                                                             0x000F0000L
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC5_MASK                                                             0x00F00000L
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC6_MASK                                                             0x0F000000L
+#define DAGB1_RD_OUTPUT_DAGB_LAZY_TIMER__VC7_MASK                                                             0xF0000000L
+//DAGB1_RD_CGTT_CLK_CTRL
+#define DAGB1_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                               0x0
+#define DAGB1_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                         0x4
+#define DAGB1_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                   0xc
+#define DAGB1_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                            0x1e
+#define DAGB1_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                          0x1f
+#define DAGB1_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                 0x0000000FL
+#define DAGB1_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                           0x00000FF0L
+#define DAGB1_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                     0x0FFFF000L
+#define DAGB1_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                              0x40000000L
+#define DAGB1_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                            0x80000000L
+//DAGB1_L1TLB_RD_CGTT_CLK_CTRL
+#define DAGB1_L1TLB_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB1_L1TLB_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB1_L1TLB_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB1_L1TLB_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB1_L1TLB_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB1_L1TLB_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB1_L1TLB_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB1_L1TLB_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB1_L1TLB_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB1_L1TLB_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB1_ATCVM_RD_CGTT_CLK_CTRL
+#define DAGB1_ATCVM_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB1_ATCVM_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB1_ATCVM_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB1_ATCVM_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB1_ATCVM_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB1_ATCVM_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB1_ATCVM_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB1_ATCVM_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB1_ATCVM_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB1_ATCVM_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB1_RD_ADDR_DAGB_MAX_BURST0
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB1_RD_ADDR_DAGB_LAZY_TIMER0
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB1_RD_ADDR_DAGB_MAX_BURST1
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB1_RD_ADDR_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB1_RD_ADDR_DAGB_LAZY_TIMER1
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB1_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB1_RD_VC0_CNTL
+#define DAGB1_RD_VC0_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_RD_VC0_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_RD_VC0_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_RD_VC0_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_RD_VC0_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_RD_VC0_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_RD_VC0_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_RD_VC0_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_RD_VC0_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_RD_VC0_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_RD_VC0_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_RD_VC0_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_RD_VC0_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_RD_VC0_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_RD_VC0_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_RD_VC0_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_RD_VC1_CNTL
+#define DAGB1_RD_VC1_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_RD_VC1_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_RD_VC1_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_RD_VC1_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_RD_VC1_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_RD_VC1_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_RD_VC1_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_RD_VC1_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_RD_VC1_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_RD_VC1_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_RD_VC1_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_RD_VC1_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_RD_VC1_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_RD_VC1_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_RD_VC1_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_RD_VC1_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_RD_VC2_CNTL
+#define DAGB1_RD_VC2_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_RD_VC2_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_RD_VC2_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_RD_VC2_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_RD_VC2_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_RD_VC2_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_RD_VC2_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_RD_VC2_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_RD_VC2_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_RD_VC2_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_RD_VC2_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_RD_VC2_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_RD_VC2_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_RD_VC2_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_RD_VC2_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_RD_VC2_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_RD_VC3_CNTL
+#define DAGB1_RD_VC3_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_RD_VC3_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_RD_VC3_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_RD_VC3_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_RD_VC3_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_RD_VC3_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_RD_VC3_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_RD_VC3_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_RD_VC3_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_RD_VC3_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_RD_VC3_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_RD_VC3_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_RD_VC3_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_RD_VC3_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_RD_VC3_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_RD_VC3_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_RD_VC4_CNTL
+#define DAGB1_RD_VC4_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_RD_VC4_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_RD_VC4_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_RD_VC4_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_RD_VC4_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_RD_VC4_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_RD_VC4_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_RD_VC4_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_RD_VC4_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_RD_VC4_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_RD_VC4_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_RD_VC4_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_RD_VC4_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_RD_VC4_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_RD_VC4_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_RD_VC4_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_RD_VC5_CNTL
+#define DAGB1_RD_VC5_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_RD_VC5_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_RD_VC5_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_RD_VC5_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_RD_VC5_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_RD_VC5_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_RD_VC5_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_RD_VC5_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_RD_VC5_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_RD_VC5_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_RD_VC5_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_RD_VC5_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_RD_VC5_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_RD_VC5_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_RD_VC5_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_RD_VC5_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_RD_VC6_CNTL
+#define DAGB1_RD_VC6_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_RD_VC6_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_RD_VC6_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_RD_VC6_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_RD_VC6_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_RD_VC6_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_RD_VC6_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_RD_VC6_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_RD_VC6_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_RD_VC6_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_RD_VC6_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_RD_VC6_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_RD_VC6_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_RD_VC6_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_RD_VC6_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_RD_VC6_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_RD_VC7_CNTL
+#define DAGB1_RD_VC7_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_RD_VC7_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_RD_VC7_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_RD_VC7_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_RD_VC7_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_RD_VC7_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_RD_VC7_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_RD_VC7_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_RD_VC7_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_RD_VC7_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_RD_VC7_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_RD_VC7_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_RD_VC7_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_RD_VC7_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_RD_VC7_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_RD_VC7_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_RD_CNTL_MISC
+#define DAGB1_RD_CNTL_MISC__STOR_POOL_CREDIT__SHIFT                                                           0x0
+#define DAGB1_RD_CNTL_MISC__EA_POOL_CREDIT__SHIFT                                                             0x6
+#define DAGB1_RD_CNTL_MISC__IO_EA_CREDIT__SHIFT                                                               0xd
+#define DAGB1_RD_CNTL_MISC__STOR_CC_NEW_MODE__SHIFT                                                           0x13
+#define DAGB1_RD_CNTL_MISC__EA_CC_LEGACY_MODE__SHIFT                                                          0x14
+#define DAGB1_RD_CNTL_MISC__UTCL2_CID__SHIFT                                                                  0x15
+#define DAGB1_RD_CNTL_MISC__RDRET_FIFO_CREDITS__SHIFT                                                         0x1a
+#define DAGB1_RD_CNTL_MISC__STOR_POOL_CREDIT_MASK                                                             0x0000003FL
+#define DAGB1_RD_CNTL_MISC__EA_POOL_CREDIT_MASK                                                               0x00001FC0L
+#define DAGB1_RD_CNTL_MISC__IO_EA_CREDIT_MASK                                                                 0x0007E000L
+#define DAGB1_RD_CNTL_MISC__STOR_CC_NEW_MODE_MASK                                                             0x00080000L
+#define DAGB1_RD_CNTL_MISC__EA_CC_LEGACY_MODE_MASK                                                            0x00100000L
+#define DAGB1_RD_CNTL_MISC__UTCL2_CID_MASK                                                                    0x03E00000L
+#define DAGB1_RD_CNTL_MISC__RDRET_FIFO_CREDITS_MASK                                                           0xFC000000L
+//DAGB1_RD_TLB_CREDIT
+#define DAGB1_RD_TLB_CREDIT__TLB0__SHIFT                                                                      0x0
+#define DAGB1_RD_TLB_CREDIT__TLB1__SHIFT                                                                      0x5
+#define DAGB1_RD_TLB_CREDIT__TLB2__SHIFT                                                                      0xa
+#define DAGB1_RD_TLB_CREDIT__TLB3__SHIFT                                                                      0xf
+#define DAGB1_RD_TLB_CREDIT__TLB4__SHIFT                                                                      0x14
+#define DAGB1_RD_TLB_CREDIT__TLB5__SHIFT                                                                      0x19
+#define DAGB1_RD_TLB_CREDIT__TLB0_MASK                                                                        0x0000001FL
+#define DAGB1_RD_TLB_CREDIT__TLB1_MASK                                                                        0x000003E0L
+#define DAGB1_RD_TLB_CREDIT__TLB2_MASK                                                                        0x00007C00L
+#define DAGB1_RD_TLB_CREDIT__TLB3_MASK                                                                        0x000F8000L
+#define DAGB1_RD_TLB_CREDIT__TLB4_MASK                                                                        0x01F00000L
+#define DAGB1_RD_TLB_CREDIT__TLB5_MASK                                                                        0x3E000000L
+//DAGB1_RD_RDRET_CREDIT_CNTL
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC0_CREDIT__SHIFT                                                         0x0
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC1_CREDIT__SHIFT                                                         0x6
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC2_CREDIT__SHIFT                                                         0xc
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC3_CREDIT__SHIFT                                                         0x12
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC4_CREDIT__SHIFT                                                         0x18
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC_MODE__SHIFT                                                            0x1e
+#define DAGB1_RD_RDRET_CREDIT_CNTL__FIX_EQ__SHIFT                                                             0x1f
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC0_CREDIT_MASK                                                           0x0000003FL
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC1_CREDIT_MASK                                                           0x00000FC0L
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC2_CREDIT_MASK                                                           0x0003F000L
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC3_CREDIT_MASK                                                           0x00FC0000L
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC4_CREDIT_MASK                                                           0x3F000000L
+#define DAGB1_RD_RDRET_CREDIT_CNTL__VC_MODE_MASK                                                              0x40000000L
+#define DAGB1_RD_RDRET_CREDIT_CNTL__FIX_EQ_MASK                                                               0x80000000L
+//DAGB1_RD_RDRET_CREDIT_CNTL2
+#define DAGB1_RD_RDRET_CREDIT_CNTL2__IO_CREDIT__SHIFT                                                         0x0
+#define DAGB1_RD_RDRET_CREDIT_CNTL2__GMI_CREDIT__SHIFT                                                        0x6
+#define DAGB1_RD_RDRET_CREDIT_CNTL2__POOL_CREDIT__SHIFT                                                       0xc
+#define DAGB1_RD_RDRET_CREDIT_CNTL2__IO_CREDIT_MASK                                                           0x0000003FL
+#define DAGB1_RD_RDRET_CREDIT_CNTL2__GMI_CREDIT_MASK                                                          0x00000FC0L
+#define DAGB1_RD_RDRET_CREDIT_CNTL2__POOL_CREDIT_MASK                                                         0x0007F000L
+//DAGB1_RDCLI_ASK_PENDING
+#define DAGB1_RDCLI_ASK_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB1_RDCLI_ASK_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB1_RDCLI_GO_PENDING
+#define DAGB1_RDCLI_GO_PENDING__BUSY__SHIFT                                                                   0x0
+#define DAGB1_RDCLI_GO_PENDING__BUSY_MASK                                                                     0xFFFFFFFFL
+//DAGB1_RDCLI_GBLSEND_PENDING
+#define DAGB1_RDCLI_GBLSEND_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB1_RDCLI_GBLSEND_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB1_RDCLI_TLB_PENDING
+#define DAGB1_RDCLI_TLB_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB1_RDCLI_TLB_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB1_RDCLI_OARB_PENDING
+#define DAGB1_RDCLI_OARB_PENDING__BUSY__SHIFT                                                                 0x0
+#define DAGB1_RDCLI_OARB_PENDING__BUSY_MASK                                                                   0xFFFFFFFFL
+//DAGB1_RDCLI_OSD_PENDING
+#define DAGB1_RDCLI_OSD_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB1_RDCLI_OSD_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB1_RDCLI_NOALLOC_OVERRIDE
+#define DAGB1_RDCLI_NOALLOC_OVERRIDE__ENABLE__SHIFT                                                           0x0
+#define DAGB1_RDCLI_NOALLOC_OVERRIDE__ENABLE_MASK                                                             0x0000FFFFL
+//DAGB1_RDCLI_NOALLOC_OVERRIDE_VALUE
+#define DAGB1_RDCLI_NOALLOC_OVERRIDE_VALUE__VALUE__SHIFT                                                      0x0
+#define DAGB1_RDCLI_NOALLOC_OVERRIDE_VALUE__VALUE_MASK                                                        0x0000FFFFL
+//DAGB1_WRCLI0
+#define DAGB1_WRCLI0__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_WRCLI0__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_WRCLI0__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_WRCLI0__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_WRCLI0__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_WRCLI0__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_WRCLI0__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_WRCLI0__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_WRCLI0__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_WRCLI0__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_WRCLI0__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_WRCLI0__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_WRCLI0__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_WRCLI0__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_WRCLI0__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_WRCLI0__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_WRCLI0__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_WRCLI0__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_WRCLI0__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_WRCLI0__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_WRCLI1
+#define DAGB1_WRCLI1__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_WRCLI1__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_WRCLI1__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_WRCLI1__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_WRCLI1__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_WRCLI1__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_WRCLI1__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_WRCLI1__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_WRCLI1__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_WRCLI1__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_WRCLI1__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_WRCLI1__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_WRCLI1__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_WRCLI1__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_WRCLI1__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_WRCLI1__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_WRCLI1__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_WRCLI1__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_WRCLI1__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_WRCLI1__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_WRCLI2
+#define DAGB1_WRCLI2__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_WRCLI2__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_WRCLI2__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_WRCLI2__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_WRCLI2__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_WRCLI2__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_WRCLI2__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_WRCLI2__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_WRCLI2__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_WRCLI2__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_WRCLI2__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_WRCLI2__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_WRCLI2__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_WRCLI2__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_WRCLI2__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_WRCLI2__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_WRCLI2__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_WRCLI2__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_WRCLI2__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_WRCLI2__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_WRCLI3
+#define DAGB1_WRCLI3__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_WRCLI3__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_WRCLI3__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_WRCLI3__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_WRCLI3__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_WRCLI3__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_WRCLI3__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_WRCLI3__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_WRCLI3__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_WRCLI3__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_WRCLI3__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_WRCLI3__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_WRCLI3__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_WRCLI3__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_WRCLI3__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_WRCLI3__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_WRCLI3__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_WRCLI3__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_WRCLI3__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_WRCLI3__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_WRCLI4
+#define DAGB1_WRCLI4__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_WRCLI4__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_WRCLI4__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_WRCLI4__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_WRCLI4__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_WRCLI4__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_WRCLI4__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_WRCLI4__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_WRCLI4__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_WRCLI4__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_WRCLI4__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_WRCLI4__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_WRCLI4__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_WRCLI4__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_WRCLI4__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_WRCLI4__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_WRCLI4__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_WRCLI4__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_WRCLI4__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_WRCLI4__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_WRCLI5
+#define DAGB1_WRCLI5__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_WRCLI5__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_WRCLI5__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_WRCLI5__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_WRCLI5__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_WRCLI5__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_WRCLI5__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_WRCLI5__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_WRCLI5__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_WRCLI5__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_WRCLI5__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_WRCLI5__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_WRCLI5__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_WRCLI5__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_WRCLI5__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_WRCLI5__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_WRCLI5__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_WRCLI5__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_WRCLI5__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_WRCLI5__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_WRCLI6
+#define DAGB1_WRCLI6__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_WRCLI6__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_WRCLI6__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_WRCLI6__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_WRCLI6__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_WRCLI6__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_WRCLI6__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_WRCLI6__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_WRCLI6__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_WRCLI6__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_WRCLI6__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_WRCLI6__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_WRCLI6__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_WRCLI6__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_WRCLI6__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_WRCLI6__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_WRCLI6__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_WRCLI6__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_WRCLI6__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_WRCLI6__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_WRCLI7
+#define DAGB1_WRCLI7__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_WRCLI7__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_WRCLI7__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_WRCLI7__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_WRCLI7__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_WRCLI7__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_WRCLI7__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_WRCLI7__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_WRCLI7__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_WRCLI7__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_WRCLI7__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_WRCLI7__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_WRCLI7__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_WRCLI7__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_WRCLI7__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_WRCLI7__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_WRCLI7__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_WRCLI7__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_WRCLI7__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_WRCLI7__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_WRCLI8
+#define DAGB1_WRCLI8__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_WRCLI8__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_WRCLI8__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_WRCLI8__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_WRCLI8__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_WRCLI8__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_WRCLI8__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_WRCLI8__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_WRCLI8__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_WRCLI8__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_WRCLI8__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_WRCLI8__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_WRCLI8__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_WRCLI8__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_WRCLI8__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_WRCLI8__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_WRCLI8__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_WRCLI8__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_WRCLI8__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_WRCLI8__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_WRCLI9
+#define DAGB1_WRCLI9__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB1_WRCLI9__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB1_WRCLI9__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB1_WRCLI9__URG_LOW__SHIFT                                                                          0x8
+#define DAGB1_WRCLI9__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB1_WRCLI9__MAX_BW__SHIFT                                                                           0xd
+#define DAGB1_WRCLI9__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB1_WRCLI9__MIN_BW__SHIFT                                                                           0x16
+#define DAGB1_WRCLI9__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB1_WRCLI9__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB1_WRCLI9__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB1_WRCLI9__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB1_WRCLI9__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB1_WRCLI9__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB1_WRCLI9__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB1_WRCLI9__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB1_WRCLI9__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB1_WRCLI9__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB1_WRCLI9__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB1_WRCLI9__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB1_WRCLI10
+#define DAGB1_WRCLI10__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_WRCLI10__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_WRCLI10__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_WRCLI10__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_WRCLI10__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_WRCLI10__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_WRCLI10__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_WRCLI10__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_WRCLI10__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_WRCLI10__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_WRCLI10__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_WRCLI10__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_WRCLI10__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_WRCLI10__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_WRCLI10__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_WRCLI10__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_WRCLI10__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_WRCLI10__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_WRCLI10__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_WRCLI10__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_WRCLI11
+#define DAGB1_WRCLI11__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_WRCLI11__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_WRCLI11__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_WRCLI11__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_WRCLI11__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_WRCLI11__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_WRCLI11__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_WRCLI11__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_WRCLI11__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_WRCLI11__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_WRCLI11__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_WRCLI11__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_WRCLI11__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_WRCLI11__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_WRCLI11__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_WRCLI11__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_WRCLI11__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_WRCLI11__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_WRCLI11__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_WRCLI11__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_WRCLI12
+#define DAGB1_WRCLI12__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_WRCLI12__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_WRCLI12__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_WRCLI12__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_WRCLI12__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_WRCLI12__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_WRCLI12__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_WRCLI12__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_WRCLI12__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_WRCLI12__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_WRCLI12__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_WRCLI12__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_WRCLI12__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_WRCLI12__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_WRCLI12__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_WRCLI12__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_WRCLI12__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_WRCLI12__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_WRCLI12__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_WRCLI12__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_WRCLI13
+#define DAGB1_WRCLI13__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_WRCLI13__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_WRCLI13__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_WRCLI13__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_WRCLI13__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_WRCLI13__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_WRCLI13__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_WRCLI13__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_WRCLI13__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_WRCLI13__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_WRCLI13__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_WRCLI13__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_WRCLI13__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_WRCLI13__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_WRCLI13__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_WRCLI13__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_WRCLI13__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_WRCLI13__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_WRCLI13__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_WRCLI13__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_WRCLI14
+#define DAGB1_WRCLI14__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_WRCLI14__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_WRCLI14__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_WRCLI14__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_WRCLI14__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_WRCLI14__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_WRCLI14__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_WRCLI14__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_WRCLI14__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_WRCLI14__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_WRCLI14__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_WRCLI14__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_WRCLI14__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_WRCLI14__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_WRCLI14__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_WRCLI14__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_WRCLI14__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_WRCLI14__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_WRCLI14__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_WRCLI14__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_WRCLI15
+#define DAGB1_WRCLI15__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB1_WRCLI15__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB1_WRCLI15__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB1_WRCLI15__URG_LOW__SHIFT                                                                         0x8
+#define DAGB1_WRCLI15__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB1_WRCLI15__MAX_BW__SHIFT                                                                          0xd
+#define DAGB1_WRCLI15__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB1_WRCLI15__MIN_BW__SHIFT                                                                          0x16
+#define DAGB1_WRCLI15__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB1_WRCLI15__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB1_WRCLI15__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB1_WRCLI15__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB1_WRCLI15__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB1_WRCLI15__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB1_WRCLI15__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB1_WRCLI15__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB1_WRCLI15__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB1_WRCLI15__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB1_WRCLI15__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB1_WRCLI15__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB1_WR_CNTL
+#define DAGB1_WR_CNTL__SCLK_FREQ__SHIFT                                                                       0x0
+#define DAGB1_WR_CNTL__CLI_MAX_BW_WINDOW__SHIFT                                                               0x4
+#define DAGB1_WR_CNTL__VC_MAX_BW_WINDOW__SHIFT                                                                0xa
+#define DAGB1_WR_CNTL__IO_LEVEL_OVERRIDE_ENABLE__SHIFT                                                        0x10
+#define DAGB1_WR_CNTL__IO_LEVEL__SHIFT                                                                        0x11
+#define DAGB1_WR_CNTL__IO_LEVEL_COMPLY_VC__SHIFT                                                              0x14
+#define DAGB1_WR_CNTL__SHARE_VC_NUM__SHIFT                                                                    0x17
+#define DAGB1_WR_CNTL__FIX_JUMP__SHIFT                                                                        0x1a
+#define DAGB1_WR_CNTL__SCLK_FREQ_MASK                                                                         0x0000000FL
+#define DAGB1_WR_CNTL__CLI_MAX_BW_WINDOW_MASK                                                                 0x000003F0L
+#define DAGB1_WR_CNTL__VC_MAX_BW_WINDOW_MASK                                                                  0x0000FC00L
+#define DAGB1_WR_CNTL__IO_LEVEL_OVERRIDE_ENABLE_MASK                                                          0x00010000L
+#define DAGB1_WR_CNTL__IO_LEVEL_MASK                                                                          0x000E0000L
+#define DAGB1_WR_CNTL__IO_LEVEL_COMPLY_VC_MASK                                                                0x00700000L
+#define DAGB1_WR_CNTL__SHARE_VC_NUM_MASK                                                                      0x03800000L
+#define DAGB1_WR_CNTL__FIX_JUMP_MASK                                                                          0x04000000L
+//DAGB1_WR_GMI_CNTL
+#define DAGB1_WR_GMI_CNTL__EA_CREDIT__SHIFT                                                                   0x0
+#define DAGB1_WR_GMI_CNTL__LEVEL__SHIFT                                                                       0x6
+#define DAGB1_WR_GMI_CNTL__MAX_BURST__SHIFT                                                                   0x9
+#define DAGB1_WR_GMI_CNTL__LAZY_TIMER__SHIFT                                                                  0xd
+#define DAGB1_WR_GMI_CNTL__EA_CREDIT_MASK                                                                     0x0000003FL
+#define DAGB1_WR_GMI_CNTL__LEVEL_MASK                                                                         0x000001C0L
+#define DAGB1_WR_GMI_CNTL__MAX_BURST_MASK                                                                     0x00001E00L
+#define DAGB1_WR_GMI_CNTL__LAZY_TIMER_MASK                                                                    0x0001E000L
+//DAGB1_WR_ADDR_DAGB
+#define DAGB1_WR_ADDR_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB1_WR_ADDR_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB1_WR_ADDR_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB1_WR_ADDR_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB1_WR_ADDR_DAGB__JUMP_MODE__SHIFT                                                                  0xd
+#define DAGB1_WR_ADDR_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB1_WR_ADDR_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB1_WR_ADDR_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB1_WR_ADDR_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+#define DAGB1_WR_ADDR_DAGB__JUMP_MODE_MASK                                                                    0x00002000L
+//DAGB1_WR_OUTPUT_DAGB_MAX_BURST
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC0__SHIFT                                                            0x0
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC1__SHIFT                                                            0x4
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC2__SHIFT                                                            0x8
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC3__SHIFT                                                            0xc
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC4__SHIFT                                                            0x10
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC5__SHIFT                                                            0x14
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC6__SHIFT                                                            0x18
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC7__SHIFT                                                            0x1c
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC0_MASK                                                              0x0000000FL
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC1_MASK                                                              0x000000F0L
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC2_MASK                                                              0x00000F00L
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC3_MASK                                                              0x0000F000L
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC4_MASK                                                              0x000F0000L
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC5_MASK                                                              0x00F00000L
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC6_MASK                                                              0x0F000000L
+#define DAGB1_WR_OUTPUT_DAGB_MAX_BURST__VC7_MASK                                                              0xF0000000L
+//DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC0__SHIFT                                                           0x0
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC1__SHIFT                                                           0x4
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC2__SHIFT                                                           0x8
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC3__SHIFT                                                           0xc
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC4__SHIFT                                                           0x10
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC5__SHIFT                                                           0x14
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC6__SHIFT                                                           0x18
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC7__SHIFT                                                           0x1c
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC0_MASK                                                             0x0000000FL
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC1_MASK                                                             0x000000F0L
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC2_MASK                                                             0x00000F00L
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC3_MASK                                                             0x0000F000L
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC4_MASK                                                             0x000F0000L
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC5_MASK                                                             0x00F00000L
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC6_MASK                                                             0x0F000000L
+#define DAGB1_WR_OUTPUT_DAGB_LAZY_TIMER__VC7_MASK                                                             0xF0000000L
+//DAGB1_WR_CGTT_CLK_CTRL
+#define DAGB1_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                               0x0
+#define DAGB1_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                         0x4
+#define DAGB1_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                   0xc
+#define DAGB1_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                            0x1e
+#define DAGB1_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                          0x1f
+#define DAGB1_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                 0x0000000FL
+#define DAGB1_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                           0x00000FF0L
+#define DAGB1_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                     0x0FFFF000L
+#define DAGB1_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                              0x40000000L
+#define DAGB1_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                            0x80000000L
+//DAGB1_L1TLB_WR_CGTT_CLK_CTRL
+#define DAGB1_L1TLB_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB1_L1TLB_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB1_L1TLB_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB1_L1TLB_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB1_L1TLB_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB1_L1TLB_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB1_L1TLB_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB1_L1TLB_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB1_L1TLB_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB1_L1TLB_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB1_ATCVM_WR_CGTT_CLK_CTRL
+#define DAGB1_ATCVM_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB1_ATCVM_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB1_ATCVM_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB1_ATCVM_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB1_ATCVM_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB1_ATCVM_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB1_ATCVM_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB1_ATCVM_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB1_ATCVM_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB1_ATCVM_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB1_WR_ADDR_DAGB_MAX_BURST0
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB1_WR_ADDR_DAGB_LAZY_TIMER0
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB1_WR_ADDR_DAGB_MAX_BURST1
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB1_WR_ADDR_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB1_WR_ADDR_DAGB_LAZY_TIMER1
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB1_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB1_WR_DATA_DAGB
+#define DAGB1_WR_DATA_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB1_WR_DATA_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB1_WR_DATA_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB1_WR_DATA_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB1_WR_DATA_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB1_WR_DATA_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB1_WR_DATA_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB1_WR_DATA_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+//DAGB1_WR_DATA_DAGB_MAX_BURST0
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB1_WR_DATA_DAGB_LAZY_TIMER0
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB1_WR_DATA_DAGB_MAX_BURST1
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB1_WR_DATA_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB1_WR_DATA_DAGB_LAZY_TIMER1
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB1_WR_DATA_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB1_WR_VC0_CNTL
+#define DAGB1_WR_VC0_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_WR_VC0_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_WR_VC0_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_WR_VC0_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_WR_VC0_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_WR_VC0_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_WR_VC0_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_WR_VC0_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_WR_VC0_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_WR_VC0_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_WR_VC0_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_WR_VC0_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_WR_VC0_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_WR_VC0_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_WR_VC0_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_WR_VC0_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_WR_VC1_CNTL
+#define DAGB1_WR_VC1_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_WR_VC1_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_WR_VC1_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_WR_VC1_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_WR_VC1_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_WR_VC1_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_WR_VC1_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_WR_VC1_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_WR_VC1_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_WR_VC1_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_WR_VC1_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_WR_VC1_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_WR_VC1_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_WR_VC1_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_WR_VC1_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_WR_VC1_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_WR_VC2_CNTL
+#define DAGB1_WR_VC2_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_WR_VC2_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_WR_VC2_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_WR_VC2_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_WR_VC2_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_WR_VC2_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_WR_VC2_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_WR_VC2_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_WR_VC2_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_WR_VC2_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_WR_VC2_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_WR_VC2_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_WR_VC2_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_WR_VC2_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_WR_VC2_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_WR_VC2_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_WR_VC3_CNTL
+#define DAGB1_WR_VC3_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_WR_VC3_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_WR_VC3_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_WR_VC3_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_WR_VC3_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_WR_VC3_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_WR_VC3_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_WR_VC3_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_WR_VC3_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_WR_VC3_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_WR_VC3_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_WR_VC3_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_WR_VC3_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_WR_VC3_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_WR_VC3_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_WR_VC3_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_WR_VC4_CNTL
+#define DAGB1_WR_VC4_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_WR_VC4_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_WR_VC4_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_WR_VC4_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_WR_VC4_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_WR_VC4_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_WR_VC4_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_WR_VC4_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_WR_VC4_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_WR_VC4_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_WR_VC4_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_WR_VC4_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_WR_VC4_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_WR_VC4_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_WR_VC4_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_WR_VC4_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_WR_VC5_CNTL
+#define DAGB1_WR_VC5_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_WR_VC5_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_WR_VC5_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_WR_VC5_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_WR_VC5_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_WR_VC5_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_WR_VC5_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_WR_VC5_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_WR_VC5_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_WR_VC5_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_WR_VC5_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_WR_VC5_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_WR_VC5_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_WR_VC5_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_WR_VC5_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_WR_VC5_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_WR_VC6_CNTL
+#define DAGB1_WR_VC6_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_WR_VC6_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_WR_VC6_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_WR_VC6_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_WR_VC6_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_WR_VC6_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_WR_VC6_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_WR_VC6_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_WR_VC6_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_WR_VC6_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_WR_VC6_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_WR_VC6_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_WR_VC6_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_WR_VC6_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_WR_VC6_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_WR_VC6_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_WR_VC7_CNTL
+#define DAGB1_WR_VC7_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB1_WR_VC7_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB1_WR_VC7_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB1_WR_VC7_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB1_WR_VC7_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB1_WR_VC7_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB1_WR_VC7_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB1_WR_VC7_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB1_WR_VC7_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB1_WR_VC7_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB1_WR_VC7_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB1_WR_VC7_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB1_WR_VC7_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB1_WR_VC7_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB1_WR_VC7_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB1_WR_VC7_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB1_WR_CNTL_MISC
+#define DAGB1_WR_CNTL_MISC__STOR_POOL_CREDIT__SHIFT                                                           0x0
+#define DAGB1_WR_CNTL_MISC__EA_POOL_CREDIT__SHIFT                                                             0x6
+#define DAGB1_WR_CNTL_MISC__IO_EA_CREDIT__SHIFT                                                               0xd
+#define DAGB1_WR_CNTL_MISC__STOR_CC_NEW_MODE__SHIFT                                                           0x13
+#define DAGB1_WR_CNTL_MISC__EA_CC_LEGACY_MODE__SHIFT                                                          0x14
+#define DAGB1_WR_CNTL_MISC__UTCL2_CID__SHIFT                                                                  0x15
+#define DAGB1_WR_CNTL_MISC__RDRET_FIFO_CREDITS__SHIFT                                                         0x1a
+#define DAGB1_WR_CNTL_MISC__STOR_POOL_CREDIT_MASK                                                             0x0000003FL
+#define DAGB1_WR_CNTL_MISC__EA_POOL_CREDIT_MASK                                                               0x00001FC0L
+#define DAGB1_WR_CNTL_MISC__IO_EA_CREDIT_MASK                                                                 0x0007E000L
+#define DAGB1_WR_CNTL_MISC__STOR_CC_NEW_MODE_MASK                                                             0x00080000L
+#define DAGB1_WR_CNTL_MISC__EA_CC_LEGACY_MODE_MASK                                                            0x00100000L
+#define DAGB1_WR_CNTL_MISC__UTCL2_CID_MASK                                                                    0x03E00000L
+#define DAGB1_WR_CNTL_MISC__RDRET_FIFO_CREDITS_MASK                                                           0xFC000000L
+//DAGB1_WR_TLB_CREDIT
+#define DAGB1_WR_TLB_CREDIT__TLB0__SHIFT                                                                      0x0
+#define DAGB1_WR_TLB_CREDIT__TLB1__SHIFT                                                                      0x5
+#define DAGB1_WR_TLB_CREDIT__TLB2__SHIFT                                                                      0xa
+#define DAGB1_WR_TLB_CREDIT__TLB3__SHIFT                                                                      0xf
+#define DAGB1_WR_TLB_CREDIT__TLB4__SHIFT                                                                      0x14
+#define DAGB1_WR_TLB_CREDIT__TLB5__SHIFT                                                                      0x19
+#define DAGB1_WR_TLB_CREDIT__TLB0_MASK                                                                        0x0000001FL
+#define DAGB1_WR_TLB_CREDIT__TLB1_MASK                                                                        0x000003E0L
+#define DAGB1_WR_TLB_CREDIT__TLB2_MASK                                                                        0x00007C00L
+#define DAGB1_WR_TLB_CREDIT__TLB3_MASK                                                                        0x000F8000L
+#define DAGB1_WR_TLB_CREDIT__TLB4_MASK                                                                        0x01F00000L
+#define DAGB1_WR_TLB_CREDIT__TLB5_MASK                                                                        0x3E000000L
+//DAGB1_WR_DATA_CREDIT
+#define DAGB1_WR_DATA_CREDIT__DLOCK_VC_CREDITS__SHIFT                                                         0x0
+#define DAGB1_WR_DATA_CREDIT__LARGE_BURST_CREDITS__SHIFT                                                      0x8
+#define DAGB1_WR_DATA_CREDIT__MIDDLE_BURST_CREDITS__SHIFT                                                     0x10
+#define DAGB1_WR_DATA_CREDIT__SMALL_BURST_CREDITS__SHIFT                                                      0x18
+#define DAGB1_WR_DATA_CREDIT__DLOCK_VC_CREDITS_MASK                                                           0x000000FFL
+#define DAGB1_WR_DATA_CREDIT__LARGE_BURST_CREDITS_MASK                                                        0x0000FF00L
+#define DAGB1_WR_DATA_CREDIT__MIDDLE_BURST_CREDITS_MASK                                                       0x00FF0000L
+#define DAGB1_WR_DATA_CREDIT__SMALL_BURST_CREDITS_MASK                                                        0xFF000000L
+//DAGB1_WR_MISC_CREDIT
+#define DAGB1_WR_MISC_CREDIT__ATOMIC_CREDIT__SHIFT                                                            0x0
+#define DAGB1_WR_MISC_CREDIT__DLOCK_VC_NUM__SHIFT                                                             0x6
+#define DAGB1_WR_MISC_CREDIT__OSD_CREDIT__SHIFT                                                               0x9
+#define DAGB1_WR_MISC_CREDIT__OSD_DLOCK_CREDIT__SHIFT                                                         0x10
+#define DAGB1_WR_MISC_CREDIT__ATOMIC_CREDIT_MASK                                                              0x0000003FL
+#define DAGB1_WR_MISC_CREDIT__DLOCK_VC_NUM_MASK                                                               0x000001C0L
+#define DAGB1_WR_MISC_CREDIT__OSD_CREDIT_MASK                                                                 0x0000FE00L
+#define DAGB1_WR_MISC_CREDIT__OSD_DLOCK_CREDIT_MASK                                                           0x007F0000L
+//DAGB1_WR_OSD_CREDIT_CNTL1
+#define DAGB1_WR_OSD_CREDIT_CNTL1__VC0_CREDIT__SHIFT                                                          0x0
+#define DAGB1_WR_OSD_CREDIT_CNTL1__VC1_CREDIT__SHIFT                                                          0x4
+#define DAGB1_WR_OSD_CREDIT_CNTL1__VC2_CREDIT__SHIFT                                                          0x8
+#define DAGB1_WR_OSD_CREDIT_CNTL1__VC3_CREDIT__SHIFT                                                          0xc
+#define DAGB1_WR_OSD_CREDIT_CNTL1__IO_CREDIT__SHIFT                                                           0x10
+#define DAGB1_WR_OSD_CREDIT_CNTL1__GMI_CREDIT__SHIFT                                                          0x14
+#define DAGB1_WR_OSD_CREDIT_CNTL1__POOL_CREDIT__SHIFT                                                         0x18
+#define DAGB1_WR_OSD_CREDIT_CNTL1__VC0_CREDIT_MASK                                                            0x0000000FL
+#define DAGB1_WR_OSD_CREDIT_CNTL1__VC1_CREDIT_MASK                                                            0x000000F0L
+#define DAGB1_WR_OSD_CREDIT_CNTL1__VC2_CREDIT_MASK                                                            0x00000F00L
+#define DAGB1_WR_OSD_CREDIT_CNTL1__VC3_CREDIT_MASK                                                            0x0000F000L
+#define DAGB1_WR_OSD_CREDIT_CNTL1__IO_CREDIT_MASK                                                             0x000F0000L
+#define DAGB1_WR_OSD_CREDIT_CNTL1__GMI_CREDIT_MASK                                                            0x00F00000L
+#define DAGB1_WR_OSD_CREDIT_CNTL1__POOL_CREDIT_MASK                                                           0x3F000000L
+//DAGB1_WR_OSD_CREDIT_CNTL2
+#define DAGB1_WR_OSD_CREDIT_CNTL2__CREDIT_MARGIN__SHIFT                                                       0x0
+#define DAGB1_WR_OSD_CREDIT_CNTL2__ENABLE_LEGACY__SHIFT                                                       0x4
+#define DAGB1_WR_OSD_CREDIT_CNTL2__CREDIT_MARGIN_MASK                                                         0x0000000FL
+#define DAGB1_WR_OSD_CREDIT_CNTL2__ENABLE_LEGACY_MASK                                                         0x00000010L
+//DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC0_CREDIT__SHIFT                                                  0x0
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC1_CREDIT__SHIFT                                                  0x5
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC2_CREDIT__SHIFT                                                  0xa
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC3_CREDIT__SHIFT                                                  0xf
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__POOL_CREDIT__SHIFT                                                 0x14
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC_MODE__SHIFT                                                     0x19
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX_EQ__SHIFT                                                      0x1a
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX0__SHIFT                                                        0x1b
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX1__SHIFT                                                        0x1c
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX2__SHIFT                                                        0x1d
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC0_CREDIT_MASK                                                    0x0000001FL
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC1_CREDIT_MASK                                                    0x000003E0L
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC2_CREDIT_MASK                                                    0x00007C00L
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC3_CREDIT_MASK                                                    0x000F8000L
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__POOL_CREDIT_MASK                                                   0x01F00000L
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC_MODE_MASK                                                       0x02000000L
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX_EQ_MASK                                                        0x04000000L
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX0_MASK                                                          0x08000000L
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX1_MASK                                                          0x10000000L
+#define DAGB1_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX2_MASK                                                          0x20000000L
+//DAGB1_WRCLI_GPU_SNOOP_OVERRIDE
+#define DAGB1_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT                                                         0x0
+#define DAGB1_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK                                                           0x0000FFFFL
+//DAGB1_WRCLI_GPU_SNOOP_OVERRIDE_VALUE
+#define DAGB1_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT                                                   0x0
+#define DAGB1_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK                                                     0x0000FFFFL
+//DAGB1_WRCLI_ASK_PENDING
+#define DAGB1_WRCLI_ASK_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB1_WRCLI_ASK_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB1_WRCLI_GO_PENDING
+#define DAGB1_WRCLI_GO_PENDING__BUSY__SHIFT                                                                   0x0
+#define DAGB1_WRCLI_GO_PENDING__BUSY_MASK                                                                     0xFFFFFFFFL
+//DAGB1_WRCLI_GBLSEND_PENDING
+#define DAGB1_WRCLI_GBLSEND_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB1_WRCLI_GBLSEND_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB1_WRCLI_TLB_PENDING
+#define DAGB1_WRCLI_TLB_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB1_WRCLI_TLB_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB1_WRCLI_OARB_PENDING
+#define DAGB1_WRCLI_OARB_PENDING__BUSY__SHIFT                                                                 0x0
+#define DAGB1_WRCLI_OARB_PENDING__BUSY_MASK                                                                   0xFFFFFFFFL
+//DAGB1_WRCLI_OSD_PENDING
+#define DAGB1_WRCLI_OSD_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB1_WRCLI_OSD_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB1_WRCLI_DBUS_ASK_PENDING
+#define DAGB1_WRCLI_DBUS_ASK_PENDING__BUSY__SHIFT                                                             0x0
+#define DAGB1_WRCLI_DBUS_ASK_PENDING__BUSY_MASK                                                               0xFFFFFFFFL
+//DAGB1_WRCLI_DBUS_GO_PENDING
+#define DAGB1_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB1_WRCLI_DBUS_GO_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB1_DAGB_DLY
+#define DAGB1_DAGB_DLY__DLY__SHIFT                                                                            0x0
+#define DAGB1_DAGB_DLY__CLI__SHIFT                                                                            0x8
+#define DAGB1_DAGB_DLY__POS__SHIFT                                                                            0x10
+#define DAGB1_DAGB_DLY__DLY_MASK                                                                              0x000000FFL
+#define DAGB1_DAGB_DLY__CLI_MASK                                                                              0x0000FF00L
+#define DAGB1_DAGB_DLY__POS_MASK                                                                              0x000F0000L
+//DAGB1_CNTL_MISC
+#define DAGB1_CNTL_MISC__EA_VC0_REMAP__SHIFT                                                                  0x0
+#define DAGB1_CNTL_MISC__EA_VC1_REMAP__SHIFT                                                                  0x3
+#define DAGB1_CNTL_MISC__EA_VC2_REMAP__SHIFT                                                                  0x6
+#define DAGB1_CNTL_MISC__EA_VC3_REMAP__SHIFT                                                                  0x9
+#define DAGB1_CNTL_MISC__EA_VC4_REMAP__SHIFT                                                                  0xc
+#define DAGB1_CNTL_MISC__EA_VC5_REMAP__SHIFT                                                                  0xf
+#define DAGB1_CNTL_MISC__EA_VC6_REMAP__SHIFT                                                                  0x12
+#define DAGB1_CNTL_MISC__EA_VC7_REMAP__SHIFT                                                                  0x15
+#define DAGB1_CNTL_MISC__BW_INIT_CYCLE__SHIFT                                                                 0x18
+#define DAGB1_CNTL_MISC__BW_RW_GAP_CYCLE__SHIFT                                                               0x1e
+#define DAGB1_CNTL_MISC__EA_VC0_REMAP_MASK                                                                    0x00000007L
+#define DAGB1_CNTL_MISC__EA_VC1_REMAP_MASK                                                                    0x00000038L
+#define DAGB1_CNTL_MISC__EA_VC2_REMAP_MASK                                                                    0x000001C0L
+#define DAGB1_CNTL_MISC__EA_VC3_REMAP_MASK                                                                    0x00000E00L
+#define DAGB1_CNTL_MISC__EA_VC4_REMAP_MASK                                                                    0x00007000L
+#define DAGB1_CNTL_MISC__EA_VC5_REMAP_MASK                                                                    0x00038000L
+#define DAGB1_CNTL_MISC__EA_VC6_REMAP_MASK                                                                    0x001C0000L
+#define DAGB1_CNTL_MISC__EA_VC7_REMAP_MASK                                                                    0x00E00000L
+#define DAGB1_CNTL_MISC__BW_INIT_CYCLE_MASK                                                                   0x3F000000L
+#define DAGB1_CNTL_MISC__BW_RW_GAP_CYCLE_MASK                                                                 0xC0000000L
+//DAGB1_CNTL_MISC2
+#define DAGB1_CNTL_MISC2__URG_BOOST_ENABLE__SHIFT                                                             0x0
+#define DAGB1_CNTL_MISC2__URG_HALT_ENABLE__SHIFT                                                              0x1
+#define DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG__SHIFT                                                             0x2
+#define DAGB1_CNTL_MISC2__DISABLE_WRRET_CG__SHIFT                                                             0x3
+#define DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG__SHIFT                                                             0x4
+#define DAGB1_CNTL_MISC2__DISABLE_RDRET_CG__SHIFT                                                             0x5
+#define DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG__SHIFT                                                             0x6
+#define DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG__SHIFT                                                             0x7
+#define DAGB1_CNTL_MISC2__DISABLE_EAWRREQ_BUSY__SHIFT                                                         0x8
+#define DAGB1_CNTL_MISC2__DISABLE_EARDREQ_BUSY__SHIFT                                                         0x9
+#define DAGB1_CNTL_MISC2__SWAP_CTL__SHIFT                                                                     0xa
+#define DAGB1_CNTL_MISC2__RDRET_FIFO_PERF__SHIFT                                                              0xb
+#define DAGB1_CNTL_MISC2__HDP_CID__SHIFT                                                                      0xc
+#define DAGB1_CNTL_MISC2__RDRET_FIFO_DLOCK_CREDITS__SHIFT                                                     0x10
+#define DAGB1_CNTL_MISC2__URG_BOOST_ENABLE_MASK                                                               0x00000001L
+#define DAGB1_CNTL_MISC2__URG_HALT_ENABLE_MASK                                                                0x00000002L
+#define DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK                                                               0x00000004L
+#define DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK                                                               0x00000008L
+#define DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK                                                               0x00000010L
+#define DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK                                                               0x00000020L
+#define DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK                                                               0x00000040L
+#define DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK                                                               0x00000080L
+#define DAGB1_CNTL_MISC2__DISABLE_EAWRREQ_BUSY_MASK                                                           0x00000100L
+#define DAGB1_CNTL_MISC2__DISABLE_EARDREQ_BUSY_MASK                                                           0x00000200L
+#define DAGB1_CNTL_MISC2__SWAP_CTL_MASK                                                                       0x00000400L
+#define DAGB1_CNTL_MISC2__RDRET_FIFO_PERF_MASK                                                                0x00000800L
+#define DAGB1_CNTL_MISC2__HDP_CID_MASK                                                                        0x0000F000L
+#define DAGB1_CNTL_MISC2__RDRET_FIFO_DLOCK_CREDITS_MASK                                                       0x003F0000L
+//DAGB1_FATAL_ERROR_CNTL
+#define DAGB1_FATAL_ERROR_CNTL__FILTER_NUM__SHIFT                                                             0x0
+#define DAGB1_FATAL_ERROR_CNTL__FILTER_NUM_MASK                                                               0x000003FFL
+//DAGB1_FATAL_ERROR_CLEAR
+#define DAGB1_FATAL_ERROR_CLEAR__CLEAR__SHIFT                                                                 0x0
+#define DAGB1_FATAL_ERROR_CLEAR__CLEAR_MASK                                                                   0x00000001L
+//DAGB1_FATAL_ERROR_STATUS0
+#define DAGB1_FATAL_ERROR_STATUS0__VALID__SHIFT                                                               0x0
+#define DAGB1_FATAL_ERROR_STATUS0__CID__SHIFT                                                                 0x1
+#define DAGB1_FATAL_ERROR_STATUS0__ADDR_LO__SHIFT                                                             0x6
+#define DAGB1_FATAL_ERROR_STATUS0__VALID_MASK                                                                 0x00000001L
+#define DAGB1_FATAL_ERROR_STATUS0__CID_MASK                                                                   0x0000003EL
+#define DAGB1_FATAL_ERROR_STATUS0__ADDR_LO_MASK                                                               0xFFFFFFC0L
+//DAGB1_FATAL_ERROR_STATUS1
+#define DAGB1_FATAL_ERROR_STATUS1__ADDR_HI__SHIFT                                                             0x0
+#define DAGB1_FATAL_ERROR_STATUS1__ADDR_HI_MASK                                                               0x0001FFFFL
+//DAGB1_FATAL_ERROR_STATUS2
+#define DAGB1_FATAL_ERROR_STATUS2__TAG__SHIFT                                                                 0x0
+#define DAGB1_FATAL_ERROR_STATUS2__VFID__SHIFT                                                                0x10
+#define DAGB1_FATAL_ERROR_STATUS2__VF__SHIFT                                                                  0x14
+#define DAGB1_FATAL_ERROR_STATUS2__SPACE__SHIFT                                                               0x15
+#define DAGB1_FATAL_ERROR_STATUS2__IO__SHIFT                                                                  0x16
+#define DAGB1_FATAL_ERROR_STATUS2__SIZE__SHIFT                                                                0x17
+#define DAGB1_FATAL_ERROR_STATUS2__DBGMSK__SHIFT                                                              0x18
+#define DAGB1_FATAL_ERROR_STATUS2__FED__SHIFT                                                                 0x19
+#define DAGB1_FATAL_ERROR_STATUS2__TAG_MASK                                                                   0x0000FFFFL
+#define DAGB1_FATAL_ERROR_STATUS2__VFID_MASK                                                                  0x000F0000L
+#define DAGB1_FATAL_ERROR_STATUS2__VF_MASK                                                                    0x00100000L
+#define DAGB1_FATAL_ERROR_STATUS2__SPACE_MASK                                                                 0x00200000L
+#define DAGB1_FATAL_ERROR_STATUS2__IO_MASK                                                                    0x00400000L
+#define DAGB1_FATAL_ERROR_STATUS2__SIZE_MASK                                                                  0x00800000L
+#define DAGB1_FATAL_ERROR_STATUS2__DBGMSK_MASK                                                                0x01000000L
+#define DAGB1_FATAL_ERROR_STATUS2__FED_MASK                                                                   0x02000000L
+//DAGB1_FATAL_ERROR_STATUS3
+#define DAGB1_FATAL_ERROR_STATUS3__NOALLOC__SHIFT                                                             0x0
+#define DAGB1_FATAL_ERROR_STATUS3__UNITID__SHIFT                                                              0x1
+#define DAGB1_FATAL_ERROR_STATUS3__OP__SHIFT                                                                  0x7
+#define DAGB1_FATAL_ERROR_STATUS3__SECLEVEL__SHIFT                                                            0xe
+#define DAGB1_FATAL_ERROR_STATUS3__WRTMZ__SHIFT                                                               0x11
+#define DAGB1_FATAL_ERROR_STATUS3__RDTMZ__SHIFT                                                               0x12
+#define DAGB1_FATAL_ERROR_STATUS3__SNOOP__SHIFT                                                               0x13
+#define DAGB1_FATAL_ERROR_STATUS3__INVAL__SHIFT                                                               0x14
+#define DAGB1_FATAL_ERROR_STATUS3__NACK__SHIFT                                                                0x15
+#define DAGB1_FATAL_ERROR_STATUS3__RO__SHIFT                                                                  0x17
+#define DAGB1_FATAL_ERROR_STATUS3__MEMLOG__SHIFT                                                              0x18
+#define DAGB1_FATAL_ERROR_STATUS3__EOP__SHIFT                                                                 0x19
+#define DAGB1_FATAL_ERROR_STATUS3__NOALLOC_MASK                                                               0x00000001L
+#define DAGB1_FATAL_ERROR_STATUS3__UNITID_MASK                                                                0x0000007EL
+#define DAGB1_FATAL_ERROR_STATUS3__OP_MASK                                                                    0x00003F80L
+#define DAGB1_FATAL_ERROR_STATUS3__SECLEVEL_MASK                                                              0x0001C000L
+#define DAGB1_FATAL_ERROR_STATUS3__WRTMZ_MASK                                                                 0x00020000L
+#define DAGB1_FATAL_ERROR_STATUS3__RDTMZ_MASK                                                                 0x00040000L
+#define DAGB1_FATAL_ERROR_STATUS3__SNOOP_MASK                                                                 0x00080000L
+#define DAGB1_FATAL_ERROR_STATUS3__INVAL_MASK                                                                 0x00100000L
+#define DAGB1_FATAL_ERROR_STATUS3__NACK_MASK                                                                  0x00600000L
+#define DAGB1_FATAL_ERROR_STATUS3__RO_MASK                                                                    0x00800000L
+#define DAGB1_FATAL_ERROR_STATUS3__MEMLOG_MASK                                                                0x01000000L
+#define DAGB1_FATAL_ERROR_STATUS3__EOP_MASK                                                                   0x02000000L
+//DAGB1_FIFO_EMPTY
+#define DAGB1_FIFO_EMPTY__EMPTY__SHIFT                                                                        0x0
+#define DAGB1_FIFO_EMPTY__EMPTY_MASK                                                                          0x00FFFFFFL
+//DAGB1_FIFO_FULL
+#define DAGB1_FIFO_FULL__FULL__SHIFT                                                                          0x0
+#define DAGB1_FIFO_FULL__FULL_MASK                                                                            0x007FFFFFL
+//DAGB1_WR_CREDITS_FULL
+#define DAGB1_WR_CREDITS_FULL__FULL__SHIFT                                                                    0x0
+#define DAGB1_WR_CREDITS_FULL__FULL_MASK                                                                      0x1FFFFFFFL
+//DAGB1_RD_CREDITS_FULL
+#define DAGB1_RD_CREDITS_FULL__FULL__SHIFT                                                                    0x0
+#define DAGB1_RD_CREDITS_FULL__FULL_MASK                                                                      0x0003FFFFL
+//DAGB1_PERFCOUNTER_LO
+#define DAGB1_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define DAGB1_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//DAGB1_PERFCOUNTER_HI
+#define DAGB1_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define DAGB1_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define DAGB1_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define DAGB1_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+//DAGB1_PERFCOUNTER0_CFG
+#define DAGB1_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB1_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB1_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB1_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB1_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB1_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB1_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB1_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB1_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB1_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB1_PERFCOUNTER1_CFG
+#define DAGB1_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB1_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB1_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB1_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB1_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB1_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB1_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB1_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB1_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB1_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB1_PERFCOUNTER2_CFG
+#define DAGB1_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB1_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB1_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB1_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB1_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB1_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB1_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB1_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB1_PERFCOUNTER2_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB1_PERFCOUNTER2_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB1_PERFCOUNTER_RSLT_CNTL
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define DAGB1_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+//DAGB1_L1TLB_REG_RW
+#define DAGB1_L1TLB_REG_RW__REG_WRITE_L1TLB_CTRL__SHIFT                                                       0x0
+#define DAGB1_L1TLB_REG_RW__REG_READ_L1TLB_CTRL__SHIFT                                                        0x1
+#define DAGB1_L1TLB_REG_RW__VMID_EXCEP_INT_CTRL__SHIFT                                                        0x2
+#define DAGB1_L1TLB_REG_RW__WDAT_PARITY_CHECK__SHIFT                                                          0x4
+#define DAGB1_L1TLB_REG_RW__DISABLE_RDRET_CHECK__SHIFT                                                        0x5
+#define DAGB1_L1TLB_REG_RW__RESERVE__SHIFT                                                                    0x6
+#define DAGB1_L1TLB_REG_RW__REG_WRITE_L1TLB_CTRL_MASK                                                         0x00000001L
+#define DAGB1_L1TLB_REG_RW__REG_READ_L1TLB_CTRL_MASK                                                          0x00000002L
+#define DAGB1_L1TLB_REG_RW__VMID_EXCEP_INT_CTRL_MASK                                                          0x00000004L
+#define DAGB1_L1TLB_REG_RW__WDAT_PARITY_CHECK_MASK                                                            0x00000010L
+#define DAGB1_L1TLB_REG_RW__DISABLE_RDRET_CHECK_MASK                                                          0x00000020L
+#define DAGB1_L1TLB_REG_RW__RESERVE_MASK                                                                      0xFFFFFFC0L
+
+
+// addressBlock: aid_mmhub_dagb_dagbdec2
+//DAGB2_RDCLI0
+#define DAGB2_RDCLI0__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_RDCLI0__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_RDCLI0__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_RDCLI0__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_RDCLI0__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_RDCLI0__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_RDCLI0__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_RDCLI0__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_RDCLI0__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_RDCLI0__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_RDCLI0__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_RDCLI0__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_RDCLI0__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_RDCLI0__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_RDCLI0__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_RDCLI0__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_RDCLI0__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_RDCLI0__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_RDCLI0__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_RDCLI0__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_RDCLI1
+#define DAGB2_RDCLI1__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_RDCLI1__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_RDCLI1__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_RDCLI1__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_RDCLI1__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_RDCLI1__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_RDCLI1__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_RDCLI1__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_RDCLI1__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_RDCLI1__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_RDCLI1__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_RDCLI1__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_RDCLI1__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_RDCLI1__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_RDCLI1__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_RDCLI1__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_RDCLI1__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_RDCLI1__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_RDCLI1__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_RDCLI1__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_RDCLI2
+#define DAGB2_RDCLI2__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_RDCLI2__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_RDCLI2__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_RDCLI2__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_RDCLI2__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_RDCLI2__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_RDCLI2__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_RDCLI2__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_RDCLI2__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_RDCLI2__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_RDCLI2__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_RDCLI2__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_RDCLI2__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_RDCLI2__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_RDCLI2__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_RDCLI2__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_RDCLI2__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_RDCLI2__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_RDCLI2__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_RDCLI2__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_RDCLI3
+#define DAGB2_RDCLI3__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_RDCLI3__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_RDCLI3__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_RDCLI3__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_RDCLI3__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_RDCLI3__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_RDCLI3__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_RDCLI3__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_RDCLI3__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_RDCLI3__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_RDCLI3__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_RDCLI3__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_RDCLI3__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_RDCLI3__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_RDCLI3__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_RDCLI3__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_RDCLI3__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_RDCLI3__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_RDCLI3__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_RDCLI3__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_RDCLI4
+#define DAGB2_RDCLI4__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_RDCLI4__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_RDCLI4__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_RDCLI4__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_RDCLI4__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_RDCLI4__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_RDCLI4__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_RDCLI4__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_RDCLI4__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_RDCLI4__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_RDCLI4__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_RDCLI4__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_RDCLI4__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_RDCLI4__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_RDCLI4__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_RDCLI4__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_RDCLI4__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_RDCLI4__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_RDCLI4__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_RDCLI4__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_RDCLI5
+#define DAGB2_RDCLI5__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_RDCLI5__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_RDCLI5__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_RDCLI5__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_RDCLI5__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_RDCLI5__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_RDCLI5__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_RDCLI5__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_RDCLI5__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_RDCLI5__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_RDCLI5__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_RDCLI5__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_RDCLI5__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_RDCLI5__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_RDCLI5__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_RDCLI5__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_RDCLI5__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_RDCLI5__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_RDCLI5__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_RDCLI5__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_RDCLI6
+#define DAGB2_RDCLI6__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_RDCLI6__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_RDCLI6__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_RDCLI6__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_RDCLI6__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_RDCLI6__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_RDCLI6__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_RDCLI6__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_RDCLI6__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_RDCLI6__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_RDCLI6__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_RDCLI6__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_RDCLI6__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_RDCLI6__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_RDCLI6__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_RDCLI6__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_RDCLI6__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_RDCLI6__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_RDCLI6__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_RDCLI6__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_RDCLI7
+#define DAGB2_RDCLI7__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_RDCLI7__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_RDCLI7__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_RDCLI7__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_RDCLI7__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_RDCLI7__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_RDCLI7__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_RDCLI7__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_RDCLI7__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_RDCLI7__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_RDCLI7__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_RDCLI7__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_RDCLI7__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_RDCLI7__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_RDCLI7__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_RDCLI7__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_RDCLI7__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_RDCLI7__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_RDCLI7__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_RDCLI7__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_RDCLI8
+#define DAGB2_RDCLI8__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_RDCLI8__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_RDCLI8__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_RDCLI8__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_RDCLI8__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_RDCLI8__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_RDCLI8__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_RDCLI8__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_RDCLI8__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_RDCLI8__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_RDCLI8__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_RDCLI8__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_RDCLI8__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_RDCLI8__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_RDCLI8__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_RDCLI8__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_RDCLI8__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_RDCLI8__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_RDCLI8__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_RDCLI8__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_RDCLI9
+#define DAGB2_RDCLI9__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_RDCLI9__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_RDCLI9__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_RDCLI9__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_RDCLI9__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_RDCLI9__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_RDCLI9__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_RDCLI9__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_RDCLI9__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_RDCLI9__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_RDCLI9__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_RDCLI9__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_RDCLI9__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_RDCLI9__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_RDCLI9__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_RDCLI9__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_RDCLI9__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_RDCLI9__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_RDCLI9__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_RDCLI9__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_RDCLI10
+#define DAGB2_RDCLI10__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_RDCLI10__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_RDCLI10__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_RDCLI10__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_RDCLI10__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_RDCLI10__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_RDCLI10__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_RDCLI10__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_RDCLI10__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_RDCLI10__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_RDCLI10__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_RDCLI10__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_RDCLI10__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_RDCLI10__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_RDCLI10__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_RDCLI10__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_RDCLI10__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_RDCLI10__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_RDCLI10__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_RDCLI10__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_RDCLI11
+#define DAGB2_RDCLI11__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_RDCLI11__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_RDCLI11__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_RDCLI11__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_RDCLI11__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_RDCLI11__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_RDCLI11__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_RDCLI11__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_RDCLI11__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_RDCLI11__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_RDCLI11__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_RDCLI11__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_RDCLI11__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_RDCLI11__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_RDCLI11__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_RDCLI11__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_RDCLI11__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_RDCLI11__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_RDCLI11__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_RDCLI11__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_RDCLI12
+#define DAGB2_RDCLI12__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_RDCLI12__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_RDCLI12__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_RDCLI12__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_RDCLI12__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_RDCLI12__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_RDCLI12__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_RDCLI12__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_RDCLI12__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_RDCLI12__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_RDCLI12__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_RDCLI12__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_RDCLI12__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_RDCLI12__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_RDCLI12__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_RDCLI12__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_RDCLI12__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_RDCLI12__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_RDCLI12__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_RDCLI12__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_RDCLI13
+#define DAGB2_RDCLI13__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_RDCLI13__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_RDCLI13__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_RDCLI13__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_RDCLI13__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_RDCLI13__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_RDCLI13__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_RDCLI13__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_RDCLI13__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_RDCLI13__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_RDCLI13__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_RDCLI13__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_RDCLI13__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_RDCLI13__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_RDCLI13__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_RDCLI13__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_RDCLI13__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_RDCLI13__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_RDCLI13__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_RDCLI13__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_RDCLI14
+#define DAGB2_RDCLI14__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_RDCLI14__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_RDCLI14__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_RDCLI14__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_RDCLI14__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_RDCLI14__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_RDCLI14__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_RDCLI14__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_RDCLI14__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_RDCLI14__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_RDCLI14__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_RDCLI14__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_RDCLI14__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_RDCLI14__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_RDCLI14__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_RDCLI14__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_RDCLI14__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_RDCLI14__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_RDCLI14__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_RDCLI14__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_RDCLI15
+#define DAGB2_RDCLI15__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_RDCLI15__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_RDCLI15__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_RDCLI15__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_RDCLI15__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_RDCLI15__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_RDCLI15__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_RDCLI15__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_RDCLI15__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_RDCLI15__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_RDCLI15__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_RDCLI15__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_RDCLI15__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_RDCLI15__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_RDCLI15__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_RDCLI15__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_RDCLI15__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_RDCLI15__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_RDCLI15__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_RDCLI15__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_RD_CNTL
+#define DAGB2_RD_CNTL__SCLK_FREQ__SHIFT                                                                       0x0
+#define DAGB2_RD_CNTL__CLI_MAX_BW_WINDOW__SHIFT                                                               0x4
+#define DAGB2_RD_CNTL__VC_MAX_BW_WINDOW__SHIFT                                                                0xa
+#define DAGB2_RD_CNTL__IO_LEVEL_OVERRIDE_ENABLE__SHIFT                                                        0x10
+#define DAGB2_RD_CNTL__IO_LEVEL__SHIFT                                                                        0x11
+#define DAGB2_RD_CNTL__IO_LEVEL_COMPLY_VC__SHIFT                                                              0x14
+#define DAGB2_RD_CNTL__SHARE_VC_NUM__SHIFT                                                                    0x17
+#define DAGB2_RD_CNTL__FIX_JUMP__SHIFT                                                                        0x1a
+#define DAGB2_RD_CNTL__SCLK_FREQ_MASK                                                                         0x0000000FL
+#define DAGB2_RD_CNTL__CLI_MAX_BW_WINDOW_MASK                                                                 0x000003F0L
+#define DAGB2_RD_CNTL__VC_MAX_BW_WINDOW_MASK                                                                  0x0000FC00L
+#define DAGB2_RD_CNTL__IO_LEVEL_OVERRIDE_ENABLE_MASK                                                          0x00010000L
+#define DAGB2_RD_CNTL__IO_LEVEL_MASK                                                                          0x000E0000L
+#define DAGB2_RD_CNTL__IO_LEVEL_COMPLY_VC_MASK                                                                0x00700000L
+#define DAGB2_RD_CNTL__SHARE_VC_NUM_MASK                                                                      0x03800000L
+#define DAGB2_RD_CNTL__FIX_JUMP_MASK                                                                          0x04000000L
+//DAGB2_RD_GMI_CNTL
+#define DAGB2_RD_GMI_CNTL__EA_CREDIT__SHIFT                                                                   0x0
+#define DAGB2_RD_GMI_CNTL__LEVEL__SHIFT                                                                       0x6
+#define DAGB2_RD_GMI_CNTL__MAX_BURST__SHIFT                                                                   0x9
+#define DAGB2_RD_GMI_CNTL__LAZY_TIMER__SHIFT                                                                  0xd
+#define DAGB2_RD_GMI_CNTL__EA_CREDIT_MASK                                                                     0x0000003FL
+#define DAGB2_RD_GMI_CNTL__LEVEL_MASK                                                                         0x000001C0L
+#define DAGB2_RD_GMI_CNTL__MAX_BURST_MASK                                                                     0x00001E00L
+#define DAGB2_RD_GMI_CNTL__LAZY_TIMER_MASK                                                                    0x0001E000L
+//DAGB2_RD_ADDR_DAGB
+#define DAGB2_RD_ADDR_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB2_RD_ADDR_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB2_RD_ADDR_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB2_RD_ADDR_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB2_RD_ADDR_DAGB__JUMP_MODE__SHIFT                                                                  0xd
+#define DAGB2_RD_ADDR_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB2_RD_ADDR_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB2_RD_ADDR_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB2_RD_ADDR_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+#define DAGB2_RD_ADDR_DAGB__JUMP_MODE_MASK                                                                    0x00002000L
+//DAGB2_RD_OUTPUT_DAGB_MAX_BURST
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC0__SHIFT                                                            0x0
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC1__SHIFT                                                            0x4
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC2__SHIFT                                                            0x8
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC3__SHIFT                                                            0xc
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC4__SHIFT                                                            0x10
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC5__SHIFT                                                            0x14
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC6__SHIFT                                                            0x18
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC7__SHIFT                                                            0x1c
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC0_MASK                                                              0x0000000FL
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC1_MASK                                                              0x000000F0L
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC2_MASK                                                              0x00000F00L
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC3_MASK                                                              0x0000F000L
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC4_MASK                                                              0x000F0000L
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC5_MASK                                                              0x00F00000L
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC6_MASK                                                              0x0F000000L
+#define DAGB2_RD_OUTPUT_DAGB_MAX_BURST__VC7_MASK                                                              0xF0000000L
+//DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC0__SHIFT                                                           0x0
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC1__SHIFT                                                           0x4
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC2__SHIFT                                                           0x8
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC3__SHIFT                                                           0xc
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC4__SHIFT                                                           0x10
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC5__SHIFT                                                           0x14
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC6__SHIFT                                                           0x18
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC7__SHIFT                                                           0x1c
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC0_MASK                                                             0x0000000FL
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC1_MASK                                                             0x000000F0L
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC2_MASK                                                             0x00000F00L
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC3_MASK                                                             0x0000F000L
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC4_MASK                                                             0x000F0000L
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC5_MASK                                                             0x00F00000L
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC6_MASK                                                             0x0F000000L
+#define DAGB2_RD_OUTPUT_DAGB_LAZY_TIMER__VC7_MASK                                                             0xF0000000L
+//DAGB2_RD_CGTT_CLK_CTRL
+#define DAGB2_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                               0x0
+#define DAGB2_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                         0x4
+#define DAGB2_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                   0xc
+#define DAGB2_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                            0x1e
+#define DAGB2_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                          0x1f
+#define DAGB2_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                 0x0000000FL
+#define DAGB2_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                           0x00000FF0L
+#define DAGB2_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                     0x0FFFF000L
+#define DAGB2_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                              0x40000000L
+#define DAGB2_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                            0x80000000L
+//DAGB2_L1TLB_RD_CGTT_CLK_CTRL
+#define DAGB2_L1TLB_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB2_L1TLB_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB2_L1TLB_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB2_L1TLB_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB2_L1TLB_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB2_L1TLB_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB2_L1TLB_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB2_L1TLB_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB2_L1TLB_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB2_L1TLB_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB2_ATCVM_RD_CGTT_CLK_CTRL
+#define DAGB2_ATCVM_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB2_ATCVM_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB2_ATCVM_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB2_ATCVM_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB2_ATCVM_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB2_ATCVM_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB2_ATCVM_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB2_ATCVM_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB2_ATCVM_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB2_ATCVM_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB2_RD_ADDR_DAGB_MAX_BURST0
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB2_RD_ADDR_DAGB_LAZY_TIMER0
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB2_RD_ADDR_DAGB_MAX_BURST1
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB2_RD_ADDR_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB2_RD_ADDR_DAGB_LAZY_TIMER1
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB2_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB2_RD_VC0_CNTL
+#define DAGB2_RD_VC0_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_RD_VC0_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_RD_VC0_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_RD_VC0_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_RD_VC0_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_RD_VC0_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_RD_VC0_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_RD_VC0_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_RD_VC0_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_RD_VC0_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_RD_VC0_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_RD_VC0_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_RD_VC0_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_RD_VC0_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_RD_VC0_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_RD_VC0_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_RD_VC1_CNTL
+#define DAGB2_RD_VC1_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_RD_VC1_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_RD_VC1_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_RD_VC1_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_RD_VC1_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_RD_VC1_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_RD_VC1_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_RD_VC1_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_RD_VC1_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_RD_VC1_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_RD_VC1_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_RD_VC1_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_RD_VC1_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_RD_VC1_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_RD_VC1_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_RD_VC1_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_RD_VC2_CNTL
+#define DAGB2_RD_VC2_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_RD_VC2_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_RD_VC2_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_RD_VC2_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_RD_VC2_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_RD_VC2_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_RD_VC2_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_RD_VC2_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_RD_VC2_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_RD_VC2_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_RD_VC2_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_RD_VC2_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_RD_VC2_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_RD_VC2_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_RD_VC2_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_RD_VC2_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_RD_VC3_CNTL
+#define DAGB2_RD_VC3_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_RD_VC3_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_RD_VC3_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_RD_VC3_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_RD_VC3_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_RD_VC3_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_RD_VC3_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_RD_VC3_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_RD_VC3_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_RD_VC3_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_RD_VC3_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_RD_VC3_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_RD_VC3_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_RD_VC3_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_RD_VC3_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_RD_VC3_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_RD_VC4_CNTL
+#define DAGB2_RD_VC4_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_RD_VC4_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_RD_VC4_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_RD_VC4_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_RD_VC4_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_RD_VC4_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_RD_VC4_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_RD_VC4_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_RD_VC4_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_RD_VC4_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_RD_VC4_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_RD_VC4_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_RD_VC4_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_RD_VC4_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_RD_VC4_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_RD_VC4_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_RD_VC5_CNTL
+#define DAGB2_RD_VC5_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_RD_VC5_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_RD_VC5_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_RD_VC5_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_RD_VC5_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_RD_VC5_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_RD_VC5_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_RD_VC5_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_RD_VC5_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_RD_VC5_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_RD_VC5_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_RD_VC5_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_RD_VC5_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_RD_VC5_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_RD_VC5_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_RD_VC5_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_RD_VC6_CNTL
+#define DAGB2_RD_VC6_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_RD_VC6_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_RD_VC6_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_RD_VC6_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_RD_VC6_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_RD_VC6_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_RD_VC6_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_RD_VC6_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_RD_VC6_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_RD_VC6_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_RD_VC6_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_RD_VC6_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_RD_VC6_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_RD_VC6_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_RD_VC6_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_RD_VC6_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_RD_VC7_CNTL
+#define DAGB2_RD_VC7_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_RD_VC7_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_RD_VC7_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_RD_VC7_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_RD_VC7_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_RD_VC7_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_RD_VC7_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_RD_VC7_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_RD_VC7_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_RD_VC7_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_RD_VC7_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_RD_VC7_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_RD_VC7_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_RD_VC7_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_RD_VC7_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_RD_VC7_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_RD_CNTL_MISC
+#define DAGB2_RD_CNTL_MISC__STOR_POOL_CREDIT__SHIFT                                                           0x0
+#define DAGB2_RD_CNTL_MISC__EA_POOL_CREDIT__SHIFT                                                             0x6
+#define DAGB2_RD_CNTL_MISC__IO_EA_CREDIT__SHIFT                                                               0xd
+#define DAGB2_RD_CNTL_MISC__STOR_CC_NEW_MODE__SHIFT                                                           0x13
+#define DAGB2_RD_CNTL_MISC__EA_CC_LEGACY_MODE__SHIFT                                                          0x14
+#define DAGB2_RD_CNTL_MISC__UTCL2_CID__SHIFT                                                                  0x15
+#define DAGB2_RD_CNTL_MISC__RDRET_FIFO_CREDITS__SHIFT                                                         0x1a
+#define DAGB2_RD_CNTL_MISC__STOR_POOL_CREDIT_MASK                                                             0x0000003FL
+#define DAGB2_RD_CNTL_MISC__EA_POOL_CREDIT_MASK                                                               0x00001FC0L
+#define DAGB2_RD_CNTL_MISC__IO_EA_CREDIT_MASK                                                                 0x0007E000L
+#define DAGB2_RD_CNTL_MISC__STOR_CC_NEW_MODE_MASK                                                             0x00080000L
+#define DAGB2_RD_CNTL_MISC__EA_CC_LEGACY_MODE_MASK                                                            0x00100000L
+#define DAGB2_RD_CNTL_MISC__UTCL2_CID_MASK                                                                    0x03E00000L
+#define DAGB2_RD_CNTL_MISC__RDRET_FIFO_CREDITS_MASK                                                           0xFC000000L
+//DAGB2_RD_TLB_CREDIT
+#define DAGB2_RD_TLB_CREDIT__TLB0__SHIFT                                                                      0x0
+#define DAGB2_RD_TLB_CREDIT__TLB1__SHIFT                                                                      0x5
+#define DAGB2_RD_TLB_CREDIT__TLB2__SHIFT                                                                      0xa
+#define DAGB2_RD_TLB_CREDIT__TLB3__SHIFT                                                                      0xf
+#define DAGB2_RD_TLB_CREDIT__TLB4__SHIFT                                                                      0x14
+#define DAGB2_RD_TLB_CREDIT__TLB5__SHIFT                                                                      0x19
+#define DAGB2_RD_TLB_CREDIT__TLB0_MASK                                                                        0x0000001FL
+#define DAGB2_RD_TLB_CREDIT__TLB1_MASK                                                                        0x000003E0L
+#define DAGB2_RD_TLB_CREDIT__TLB2_MASK                                                                        0x00007C00L
+#define DAGB2_RD_TLB_CREDIT__TLB3_MASK                                                                        0x000F8000L
+#define DAGB2_RD_TLB_CREDIT__TLB4_MASK                                                                        0x01F00000L
+#define DAGB2_RD_TLB_CREDIT__TLB5_MASK                                                                        0x3E000000L
+//DAGB2_RD_RDRET_CREDIT_CNTL
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC0_CREDIT__SHIFT                                                         0x0
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC1_CREDIT__SHIFT                                                         0x6
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC2_CREDIT__SHIFT                                                         0xc
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC3_CREDIT__SHIFT                                                         0x12
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC4_CREDIT__SHIFT                                                         0x18
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC_MODE__SHIFT                                                            0x1e
+#define DAGB2_RD_RDRET_CREDIT_CNTL__FIX_EQ__SHIFT                                                             0x1f
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC0_CREDIT_MASK                                                           0x0000003FL
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC1_CREDIT_MASK                                                           0x00000FC0L
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC2_CREDIT_MASK                                                           0x0003F000L
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC3_CREDIT_MASK                                                           0x00FC0000L
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC4_CREDIT_MASK                                                           0x3F000000L
+#define DAGB2_RD_RDRET_CREDIT_CNTL__VC_MODE_MASK                                                              0x40000000L
+#define DAGB2_RD_RDRET_CREDIT_CNTL__FIX_EQ_MASK                                                               0x80000000L
+//DAGB2_RD_RDRET_CREDIT_CNTL2
+#define DAGB2_RD_RDRET_CREDIT_CNTL2__IO_CREDIT__SHIFT                                                         0x0
+#define DAGB2_RD_RDRET_CREDIT_CNTL2__GMI_CREDIT__SHIFT                                                        0x6
+#define DAGB2_RD_RDRET_CREDIT_CNTL2__POOL_CREDIT__SHIFT                                                       0xc
+#define DAGB2_RD_RDRET_CREDIT_CNTL2__IO_CREDIT_MASK                                                           0x0000003FL
+#define DAGB2_RD_RDRET_CREDIT_CNTL2__GMI_CREDIT_MASK                                                          0x00000FC0L
+#define DAGB2_RD_RDRET_CREDIT_CNTL2__POOL_CREDIT_MASK                                                         0x0007F000L
+//DAGB2_RDCLI_ASK_PENDING
+#define DAGB2_RDCLI_ASK_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB2_RDCLI_ASK_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB2_RDCLI_GO_PENDING
+#define DAGB2_RDCLI_GO_PENDING__BUSY__SHIFT                                                                   0x0
+#define DAGB2_RDCLI_GO_PENDING__BUSY_MASK                                                                     0xFFFFFFFFL
+//DAGB2_RDCLI_GBLSEND_PENDING
+#define DAGB2_RDCLI_GBLSEND_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB2_RDCLI_GBLSEND_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB2_RDCLI_TLB_PENDING
+#define DAGB2_RDCLI_TLB_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB2_RDCLI_TLB_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB2_RDCLI_OARB_PENDING
+#define DAGB2_RDCLI_OARB_PENDING__BUSY__SHIFT                                                                 0x0
+#define DAGB2_RDCLI_OARB_PENDING__BUSY_MASK                                                                   0xFFFFFFFFL
+//DAGB2_RDCLI_OSD_PENDING
+#define DAGB2_RDCLI_OSD_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB2_RDCLI_OSD_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB2_WRCLI0
+#define DAGB2_WRCLI0__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_WRCLI0__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_WRCLI0__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_WRCLI0__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_WRCLI0__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_WRCLI0__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_WRCLI0__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_WRCLI0__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_WRCLI0__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_WRCLI0__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_WRCLI0__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_WRCLI0__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_WRCLI0__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_WRCLI0__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_WRCLI0__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_WRCLI0__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_WRCLI0__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_WRCLI0__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_WRCLI0__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_WRCLI0__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_WRCLI1
+#define DAGB2_WRCLI1__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_WRCLI1__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_WRCLI1__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_WRCLI1__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_WRCLI1__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_WRCLI1__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_WRCLI1__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_WRCLI1__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_WRCLI1__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_WRCLI1__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_WRCLI1__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_WRCLI1__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_WRCLI1__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_WRCLI1__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_WRCLI1__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_WRCLI1__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_WRCLI1__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_WRCLI1__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_WRCLI1__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_WRCLI1__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_WRCLI2
+#define DAGB2_WRCLI2__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_WRCLI2__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_WRCLI2__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_WRCLI2__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_WRCLI2__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_WRCLI2__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_WRCLI2__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_WRCLI2__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_WRCLI2__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_WRCLI2__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_WRCLI2__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_WRCLI2__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_WRCLI2__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_WRCLI2__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_WRCLI2__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_WRCLI2__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_WRCLI2__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_WRCLI2__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_WRCLI2__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_WRCLI2__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_WRCLI3
+#define DAGB2_WRCLI3__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_WRCLI3__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_WRCLI3__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_WRCLI3__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_WRCLI3__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_WRCLI3__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_WRCLI3__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_WRCLI3__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_WRCLI3__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_WRCLI3__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_WRCLI3__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_WRCLI3__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_WRCLI3__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_WRCLI3__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_WRCLI3__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_WRCLI3__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_WRCLI3__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_WRCLI3__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_WRCLI3__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_WRCLI3__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_WRCLI4
+#define DAGB2_WRCLI4__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_WRCLI4__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_WRCLI4__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_WRCLI4__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_WRCLI4__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_WRCLI4__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_WRCLI4__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_WRCLI4__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_WRCLI4__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_WRCLI4__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_WRCLI4__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_WRCLI4__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_WRCLI4__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_WRCLI4__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_WRCLI4__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_WRCLI4__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_WRCLI4__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_WRCLI4__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_WRCLI4__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_WRCLI4__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_WRCLI5
+#define DAGB2_WRCLI5__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_WRCLI5__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_WRCLI5__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_WRCLI5__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_WRCLI5__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_WRCLI5__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_WRCLI5__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_WRCLI5__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_WRCLI5__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_WRCLI5__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_WRCLI5__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_WRCLI5__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_WRCLI5__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_WRCLI5__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_WRCLI5__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_WRCLI5__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_WRCLI5__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_WRCLI5__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_WRCLI5__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_WRCLI5__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_WRCLI6
+#define DAGB2_WRCLI6__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_WRCLI6__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_WRCLI6__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_WRCLI6__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_WRCLI6__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_WRCLI6__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_WRCLI6__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_WRCLI6__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_WRCLI6__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_WRCLI6__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_WRCLI6__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_WRCLI6__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_WRCLI6__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_WRCLI6__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_WRCLI6__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_WRCLI6__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_WRCLI6__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_WRCLI6__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_WRCLI6__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_WRCLI6__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_WRCLI7
+#define DAGB2_WRCLI7__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_WRCLI7__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_WRCLI7__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_WRCLI7__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_WRCLI7__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_WRCLI7__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_WRCLI7__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_WRCLI7__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_WRCLI7__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_WRCLI7__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_WRCLI7__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_WRCLI7__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_WRCLI7__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_WRCLI7__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_WRCLI7__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_WRCLI7__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_WRCLI7__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_WRCLI7__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_WRCLI7__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_WRCLI7__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_WRCLI8
+#define DAGB2_WRCLI8__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_WRCLI8__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_WRCLI8__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_WRCLI8__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_WRCLI8__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_WRCLI8__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_WRCLI8__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_WRCLI8__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_WRCLI8__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_WRCLI8__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_WRCLI8__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_WRCLI8__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_WRCLI8__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_WRCLI8__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_WRCLI8__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_WRCLI8__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_WRCLI8__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_WRCLI8__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_WRCLI8__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_WRCLI8__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_WRCLI9
+#define DAGB2_WRCLI9__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB2_WRCLI9__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB2_WRCLI9__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB2_WRCLI9__URG_LOW__SHIFT                                                                          0x8
+#define DAGB2_WRCLI9__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB2_WRCLI9__MAX_BW__SHIFT                                                                           0xd
+#define DAGB2_WRCLI9__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB2_WRCLI9__MIN_BW__SHIFT                                                                           0x16
+#define DAGB2_WRCLI9__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB2_WRCLI9__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB2_WRCLI9__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB2_WRCLI9__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB2_WRCLI9__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB2_WRCLI9__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB2_WRCLI9__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB2_WRCLI9__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB2_WRCLI9__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB2_WRCLI9__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB2_WRCLI9__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB2_WRCLI9__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB2_WRCLI10
+#define DAGB2_WRCLI10__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_WRCLI10__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_WRCLI10__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_WRCLI10__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_WRCLI10__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_WRCLI10__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_WRCLI10__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_WRCLI10__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_WRCLI10__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_WRCLI10__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_WRCLI10__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_WRCLI10__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_WRCLI10__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_WRCLI10__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_WRCLI10__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_WRCLI10__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_WRCLI10__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_WRCLI10__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_WRCLI10__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_WRCLI10__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_WRCLI11
+#define DAGB2_WRCLI11__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_WRCLI11__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_WRCLI11__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_WRCLI11__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_WRCLI11__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_WRCLI11__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_WRCLI11__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_WRCLI11__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_WRCLI11__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_WRCLI11__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_WRCLI11__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_WRCLI11__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_WRCLI11__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_WRCLI11__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_WRCLI11__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_WRCLI11__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_WRCLI11__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_WRCLI11__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_WRCLI11__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_WRCLI11__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_WRCLI12
+#define DAGB2_WRCLI12__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_WRCLI12__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_WRCLI12__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_WRCLI12__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_WRCLI12__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_WRCLI12__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_WRCLI12__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_WRCLI12__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_WRCLI12__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_WRCLI12__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_WRCLI12__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_WRCLI12__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_WRCLI12__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_WRCLI12__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_WRCLI12__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_WRCLI12__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_WRCLI12__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_WRCLI12__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_WRCLI12__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_WRCLI12__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_WRCLI13
+#define DAGB2_WRCLI13__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_WRCLI13__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_WRCLI13__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_WRCLI13__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_WRCLI13__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_WRCLI13__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_WRCLI13__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_WRCLI13__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_WRCLI13__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_WRCLI13__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_WRCLI13__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_WRCLI13__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_WRCLI13__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_WRCLI13__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_WRCLI13__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_WRCLI13__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_WRCLI13__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_WRCLI13__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_WRCLI13__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_WRCLI13__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_WRCLI14
+#define DAGB2_WRCLI14__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_WRCLI14__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_WRCLI14__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_WRCLI14__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_WRCLI14__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_WRCLI14__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_WRCLI14__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_WRCLI14__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_WRCLI14__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_WRCLI14__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_WRCLI14__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_WRCLI14__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_WRCLI14__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_WRCLI14__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_WRCLI14__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_WRCLI14__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_WRCLI14__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_WRCLI14__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_WRCLI14__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_WRCLI14__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_WRCLI15
+#define DAGB2_WRCLI15__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB2_WRCLI15__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB2_WRCLI15__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB2_WRCLI15__URG_LOW__SHIFT                                                                         0x8
+#define DAGB2_WRCLI15__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB2_WRCLI15__MAX_BW__SHIFT                                                                          0xd
+#define DAGB2_WRCLI15__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB2_WRCLI15__MIN_BW__SHIFT                                                                          0x16
+#define DAGB2_WRCLI15__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB2_WRCLI15__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB2_WRCLI15__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB2_WRCLI15__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB2_WRCLI15__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB2_WRCLI15__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB2_WRCLI15__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB2_WRCLI15__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB2_WRCLI15__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB2_WRCLI15__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB2_WRCLI15__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB2_WRCLI15__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB2_WR_CNTL
+#define DAGB2_WR_CNTL__SCLK_FREQ__SHIFT                                                                       0x0
+#define DAGB2_WR_CNTL__CLI_MAX_BW_WINDOW__SHIFT                                                               0x4
+#define DAGB2_WR_CNTL__VC_MAX_BW_WINDOW__SHIFT                                                                0xa
+#define DAGB2_WR_CNTL__IO_LEVEL_OVERRIDE_ENABLE__SHIFT                                                        0x10
+#define DAGB2_WR_CNTL__IO_LEVEL__SHIFT                                                                        0x11
+#define DAGB2_WR_CNTL__IO_LEVEL_COMPLY_VC__SHIFT                                                              0x14
+#define DAGB2_WR_CNTL__SHARE_VC_NUM__SHIFT                                                                    0x17
+#define DAGB2_WR_CNTL__FIX_JUMP__SHIFT                                                                        0x1a
+#define DAGB2_WR_CNTL__SCLK_FREQ_MASK                                                                         0x0000000FL
+#define DAGB2_WR_CNTL__CLI_MAX_BW_WINDOW_MASK                                                                 0x000003F0L
+#define DAGB2_WR_CNTL__VC_MAX_BW_WINDOW_MASK                                                                  0x0000FC00L
+#define DAGB2_WR_CNTL__IO_LEVEL_OVERRIDE_ENABLE_MASK                                                          0x00010000L
+#define DAGB2_WR_CNTL__IO_LEVEL_MASK                                                                          0x000E0000L
+#define DAGB2_WR_CNTL__IO_LEVEL_COMPLY_VC_MASK                                                                0x00700000L
+#define DAGB2_WR_CNTL__SHARE_VC_NUM_MASK                                                                      0x03800000L
+#define DAGB2_WR_CNTL__FIX_JUMP_MASK                                                                          0x04000000L
+//DAGB2_WR_GMI_CNTL
+#define DAGB2_WR_GMI_CNTL__EA_CREDIT__SHIFT                                                                   0x0
+#define DAGB2_WR_GMI_CNTL__LEVEL__SHIFT                                                                       0x6
+#define DAGB2_WR_GMI_CNTL__MAX_BURST__SHIFT                                                                   0x9
+#define DAGB2_WR_GMI_CNTL__LAZY_TIMER__SHIFT                                                                  0xd
+#define DAGB2_WR_GMI_CNTL__EA_CREDIT_MASK                                                                     0x0000003FL
+#define DAGB2_WR_GMI_CNTL__LEVEL_MASK                                                                         0x000001C0L
+#define DAGB2_WR_GMI_CNTL__MAX_BURST_MASK                                                                     0x00001E00L
+#define DAGB2_WR_GMI_CNTL__LAZY_TIMER_MASK                                                                    0x0001E000L
+//DAGB2_WR_ADDR_DAGB
+#define DAGB2_WR_ADDR_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB2_WR_ADDR_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB2_WR_ADDR_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB2_WR_ADDR_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB2_WR_ADDR_DAGB__JUMP_MODE__SHIFT                                                                  0xd
+#define DAGB2_WR_ADDR_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB2_WR_ADDR_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB2_WR_ADDR_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB2_WR_ADDR_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+#define DAGB2_WR_ADDR_DAGB__JUMP_MODE_MASK                                                                    0x00002000L
+//DAGB2_WR_OUTPUT_DAGB_MAX_BURST
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC0__SHIFT                                                            0x0
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC1__SHIFT                                                            0x4
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC2__SHIFT                                                            0x8
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC3__SHIFT                                                            0xc
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC4__SHIFT                                                            0x10
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC5__SHIFT                                                            0x14
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC6__SHIFT                                                            0x18
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC7__SHIFT                                                            0x1c
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC0_MASK                                                              0x0000000FL
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC1_MASK                                                              0x000000F0L
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC2_MASK                                                              0x00000F00L
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC3_MASK                                                              0x0000F000L
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC4_MASK                                                              0x000F0000L
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC5_MASK                                                              0x00F00000L
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC6_MASK                                                              0x0F000000L
+#define DAGB2_WR_OUTPUT_DAGB_MAX_BURST__VC7_MASK                                                              0xF0000000L
+//DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC0__SHIFT                                                           0x0
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC1__SHIFT                                                           0x4
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC2__SHIFT                                                           0x8
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC3__SHIFT                                                           0xc
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC4__SHIFT                                                           0x10
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC5__SHIFT                                                           0x14
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC6__SHIFT                                                           0x18
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC7__SHIFT                                                           0x1c
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC0_MASK                                                             0x0000000FL
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC1_MASK                                                             0x000000F0L
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC2_MASK                                                             0x00000F00L
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC3_MASK                                                             0x0000F000L
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC4_MASK                                                             0x000F0000L
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC5_MASK                                                             0x00F00000L
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC6_MASK                                                             0x0F000000L
+#define DAGB2_WR_OUTPUT_DAGB_LAZY_TIMER__VC7_MASK                                                             0xF0000000L
+//DAGB2_WR_CGTT_CLK_CTRL
+#define DAGB2_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                               0x0
+#define DAGB2_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                         0x4
+#define DAGB2_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                   0xc
+#define DAGB2_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                            0x1e
+#define DAGB2_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                          0x1f
+#define DAGB2_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                 0x0000000FL
+#define DAGB2_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                           0x00000FF0L
+#define DAGB2_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                     0x0FFFF000L
+#define DAGB2_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                              0x40000000L
+#define DAGB2_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                            0x80000000L
+//DAGB2_L1TLB_WR_CGTT_CLK_CTRL
+#define DAGB2_L1TLB_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB2_L1TLB_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB2_L1TLB_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB2_L1TLB_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB2_L1TLB_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB2_L1TLB_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB2_L1TLB_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB2_L1TLB_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB2_L1TLB_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB2_L1TLB_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB2_ATCVM_WR_CGTT_CLK_CTRL
+#define DAGB2_ATCVM_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB2_ATCVM_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB2_ATCVM_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB2_ATCVM_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB2_ATCVM_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB2_ATCVM_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB2_ATCVM_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB2_ATCVM_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB2_ATCVM_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB2_ATCVM_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB2_WR_ADDR_DAGB_MAX_BURST0
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB2_WR_ADDR_DAGB_LAZY_TIMER0
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB2_WR_ADDR_DAGB_MAX_BURST1
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB2_WR_ADDR_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB2_WR_ADDR_DAGB_LAZY_TIMER1
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB2_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB2_WR_DATA_DAGB
+#define DAGB2_WR_DATA_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB2_WR_DATA_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB2_WR_DATA_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB2_WR_DATA_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB2_WR_DATA_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB2_WR_DATA_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB2_WR_DATA_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB2_WR_DATA_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+//DAGB2_WR_DATA_DAGB_MAX_BURST0
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB2_WR_DATA_DAGB_LAZY_TIMER0
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB2_WR_DATA_DAGB_MAX_BURST1
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB2_WR_DATA_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB2_WR_DATA_DAGB_LAZY_TIMER1
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB2_WR_DATA_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB2_WR_VC0_CNTL
+#define DAGB2_WR_VC0_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_WR_VC0_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_WR_VC0_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_WR_VC0_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_WR_VC0_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_WR_VC0_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_WR_VC0_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_WR_VC0_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_WR_VC0_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_WR_VC0_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_WR_VC0_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_WR_VC0_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_WR_VC0_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_WR_VC0_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_WR_VC0_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_WR_VC0_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_WR_VC1_CNTL
+#define DAGB2_WR_VC1_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_WR_VC1_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_WR_VC1_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_WR_VC1_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_WR_VC1_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_WR_VC1_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_WR_VC1_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_WR_VC1_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_WR_VC1_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_WR_VC1_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_WR_VC1_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_WR_VC1_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_WR_VC1_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_WR_VC1_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_WR_VC1_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_WR_VC1_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_WR_VC2_CNTL
+#define DAGB2_WR_VC2_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_WR_VC2_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_WR_VC2_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_WR_VC2_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_WR_VC2_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_WR_VC2_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_WR_VC2_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_WR_VC2_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_WR_VC2_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_WR_VC2_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_WR_VC2_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_WR_VC2_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_WR_VC2_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_WR_VC2_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_WR_VC2_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_WR_VC2_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_WR_VC3_CNTL
+#define DAGB2_WR_VC3_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_WR_VC3_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_WR_VC3_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_WR_VC3_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_WR_VC3_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_WR_VC3_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_WR_VC3_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_WR_VC3_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_WR_VC3_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_WR_VC3_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_WR_VC3_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_WR_VC3_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_WR_VC3_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_WR_VC3_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_WR_VC3_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_WR_VC3_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_WR_VC4_CNTL
+#define DAGB2_WR_VC4_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_WR_VC4_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_WR_VC4_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_WR_VC4_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_WR_VC4_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_WR_VC4_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_WR_VC4_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_WR_VC4_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_WR_VC4_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_WR_VC4_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_WR_VC4_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_WR_VC4_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_WR_VC4_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_WR_VC4_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_WR_VC4_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_WR_VC4_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_WR_VC5_CNTL
+#define DAGB2_WR_VC5_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_WR_VC5_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_WR_VC5_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_WR_VC5_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_WR_VC5_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_WR_VC5_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_WR_VC5_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_WR_VC5_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_WR_VC5_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_WR_VC5_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_WR_VC5_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_WR_VC5_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_WR_VC5_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_WR_VC5_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_WR_VC5_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_WR_VC5_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_WR_VC6_CNTL
+#define DAGB2_WR_VC6_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_WR_VC6_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_WR_VC6_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_WR_VC6_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_WR_VC6_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_WR_VC6_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_WR_VC6_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_WR_VC6_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_WR_VC6_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_WR_VC6_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_WR_VC6_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_WR_VC6_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_WR_VC6_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_WR_VC6_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_WR_VC6_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_WR_VC6_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_WR_VC7_CNTL
+#define DAGB2_WR_VC7_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB2_WR_VC7_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB2_WR_VC7_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB2_WR_VC7_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB2_WR_VC7_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB2_WR_VC7_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB2_WR_VC7_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB2_WR_VC7_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB2_WR_VC7_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB2_WR_VC7_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB2_WR_VC7_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB2_WR_VC7_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB2_WR_VC7_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB2_WR_VC7_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB2_WR_VC7_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB2_WR_VC7_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB2_WR_CNTL_MISC
+#define DAGB2_WR_CNTL_MISC__STOR_POOL_CREDIT__SHIFT                                                           0x0
+#define DAGB2_WR_CNTL_MISC__EA_POOL_CREDIT__SHIFT                                                             0x6
+#define DAGB2_WR_CNTL_MISC__IO_EA_CREDIT__SHIFT                                                               0xd
+#define DAGB2_WR_CNTL_MISC__STOR_CC_NEW_MODE__SHIFT                                                           0x13
+#define DAGB2_WR_CNTL_MISC__EA_CC_LEGACY_MODE__SHIFT                                                          0x14
+#define DAGB2_WR_CNTL_MISC__UTCL2_CID__SHIFT                                                                  0x15
+#define DAGB2_WR_CNTL_MISC__RDRET_FIFO_CREDITS__SHIFT                                                         0x1a
+#define DAGB2_WR_CNTL_MISC__STOR_POOL_CREDIT_MASK                                                             0x0000003FL
+#define DAGB2_WR_CNTL_MISC__EA_POOL_CREDIT_MASK                                                               0x00001FC0L
+#define DAGB2_WR_CNTL_MISC__IO_EA_CREDIT_MASK                                                                 0x0007E000L
+#define DAGB2_WR_CNTL_MISC__STOR_CC_NEW_MODE_MASK                                                             0x00080000L
+#define DAGB2_WR_CNTL_MISC__EA_CC_LEGACY_MODE_MASK                                                            0x00100000L
+#define DAGB2_WR_CNTL_MISC__UTCL2_CID_MASK                                                                    0x03E00000L
+#define DAGB2_WR_CNTL_MISC__RDRET_FIFO_CREDITS_MASK                                                           0xFC000000L
+//DAGB2_WR_TLB_CREDIT
+#define DAGB2_WR_TLB_CREDIT__TLB0__SHIFT                                                                      0x0
+#define DAGB2_WR_TLB_CREDIT__TLB1__SHIFT                                                                      0x5
+#define DAGB2_WR_TLB_CREDIT__TLB2__SHIFT                                                                      0xa
+#define DAGB2_WR_TLB_CREDIT__TLB3__SHIFT                                                                      0xf
+#define DAGB2_WR_TLB_CREDIT__TLB4__SHIFT                                                                      0x14
+#define DAGB2_WR_TLB_CREDIT__TLB5__SHIFT                                                                      0x19
+#define DAGB2_WR_TLB_CREDIT__TLB0_MASK                                                                        0x0000001FL
+#define DAGB2_WR_TLB_CREDIT__TLB1_MASK                                                                        0x000003E0L
+#define DAGB2_WR_TLB_CREDIT__TLB2_MASK                                                                        0x00007C00L
+#define DAGB2_WR_TLB_CREDIT__TLB3_MASK                                                                        0x000F8000L
+#define DAGB2_WR_TLB_CREDIT__TLB4_MASK                                                                        0x01F00000L
+#define DAGB2_WR_TLB_CREDIT__TLB5_MASK                                                                        0x3E000000L
+//DAGB2_WR_DATA_CREDIT
+#define DAGB2_WR_DATA_CREDIT__DLOCK_VC_CREDITS__SHIFT                                                         0x0
+#define DAGB2_WR_DATA_CREDIT__LARGE_BURST_CREDITS__SHIFT                                                      0x8
+#define DAGB2_WR_DATA_CREDIT__MIDDLE_BURST_CREDITS__SHIFT                                                     0x10
+#define DAGB2_WR_DATA_CREDIT__SMALL_BURST_CREDITS__SHIFT                                                      0x18
+#define DAGB2_WR_DATA_CREDIT__DLOCK_VC_CREDITS_MASK                                                           0x000000FFL
+#define DAGB2_WR_DATA_CREDIT__LARGE_BURST_CREDITS_MASK                                                        0x0000FF00L
+#define DAGB2_WR_DATA_CREDIT__MIDDLE_BURST_CREDITS_MASK                                                       0x00FF0000L
+#define DAGB2_WR_DATA_CREDIT__SMALL_BURST_CREDITS_MASK                                                        0xFF000000L
+//DAGB2_WR_MISC_CREDIT
+#define DAGB2_WR_MISC_CREDIT__ATOMIC_CREDIT__SHIFT                                                            0x0
+#define DAGB2_WR_MISC_CREDIT__DLOCK_VC_NUM__SHIFT                                                             0x6
+#define DAGB2_WR_MISC_CREDIT__OSD_CREDIT__SHIFT                                                               0x9
+#define DAGB2_WR_MISC_CREDIT__OSD_DLOCK_CREDIT__SHIFT                                                         0x10
+#define DAGB2_WR_MISC_CREDIT__ATOMIC_CREDIT_MASK                                                              0x0000003FL
+#define DAGB2_WR_MISC_CREDIT__DLOCK_VC_NUM_MASK                                                               0x000001C0L
+#define DAGB2_WR_MISC_CREDIT__OSD_CREDIT_MASK                                                                 0x0000FE00L
+#define DAGB2_WR_MISC_CREDIT__OSD_DLOCK_CREDIT_MASK                                                           0x007F0000L
+//DAGB2_WR_OSD_CREDIT_CNTL1
+#define DAGB2_WR_OSD_CREDIT_CNTL1__VC0_CREDIT__SHIFT                                                          0x0
+#define DAGB2_WR_OSD_CREDIT_CNTL1__VC1_CREDIT__SHIFT                                                          0x4
+#define DAGB2_WR_OSD_CREDIT_CNTL1__VC2_CREDIT__SHIFT                                                          0x8
+#define DAGB2_WR_OSD_CREDIT_CNTL1__VC3_CREDIT__SHIFT                                                          0xc
+#define DAGB2_WR_OSD_CREDIT_CNTL1__IO_CREDIT__SHIFT                                                           0x10
+#define DAGB2_WR_OSD_CREDIT_CNTL1__GMI_CREDIT__SHIFT                                                          0x14
+#define DAGB2_WR_OSD_CREDIT_CNTL1__POOL_CREDIT__SHIFT                                                         0x18
+#define DAGB2_WR_OSD_CREDIT_CNTL1__VC0_CREDIT_MASK                                                            0x0000000FL
+#define DAGB2_WR_OSD_CREDIT_CNTL1__VC1_CREDIT_MASK                                                            0x000000F0L
+#define DAGB2_WR_OSD_CREDIT_CNTL1__VC2_CREDIT_MASK                                                            0x00000F00L
+#define DAGB2_WR_OSD_CREDIT_CNTL1__VC3_CREDIT_MASK                                                            0x0000F000L
+#define DAGB2_WR_OSD_CREDIT_CNTL1__IO_CREDIT_MASK                                                             0x000F0000L
+#define DAGB2_WR_OSD_CREDIT_CNTL1__GMI_CREDIT_MASK                                                            0x00F00000L
+#define DAGB2_WR_OSD_CREDIT_CNTL1__POOL_CREDIT_MASK                                                           0x3F000000L
+//DAGB2_WR_OSD_CREDIT_CNTL2
+#define DAGB2_WR_OSD_CREDIT_CNTL2__CREDIT_MARGIN__SHIFT                                                       0x0
+#define DAGB2_WR_OSD_CREDIT_CNTL2__ENABLE_LEGACY__SHIFT                                                       0x4
+#define DAGB2_WR_OSD_CREDIT_CNTL2__CREDIT_MARGIN_MASK                                                         0x0000000FL
+#define DAGB2_WR_OSD_CREDIT_CNTL2__ENABLE_LEGACY_MASK                                                         0x00000010L
+//DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC0_CREDIT__SHIFT                                                  0x0
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC1_CREDIT__SHIFT                                                  0x5
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC2_CREDIT__SHIFT                                                  0xa
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC3_CREDIT__SHIFT                                                  0xf
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__POOL_CREDIT__SHIFT                                                 0x14
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC_MODE__SHIFT                                                     0x19
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX_EQ__SHIFT                                                      0x1a
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX0__SHIFT                                                        0x1b
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX1__SHIFT                                                        0x1c
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX2__SHIFT                                                        0x1d
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC0_CREDIT_MASK                                                    0x0000001FL
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC1_CREDIT_MASK                                                    0x000003E0L
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC2_CREDIT_MASK                                                    0x00007C00L
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC3_CREDIT_MASK                                                    0x000F8000L
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__POOL_CREDIT_MASK                                                   0x01F00000L
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC_MODE_MASK                                                       0x02000000L
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX_EQ_MASK                                                        0x04000000L
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX0_MASK                                                          0x08000000L
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX1_MASK                                                          0x10000000L
+#define DAGB2_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX2_MASK                                                          0x20000000L
+//DAGB2_WRCLI_GPU_SNOOP_OVERRIDE
+#define DAGB2_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT                                                         0x0
+#define DAGB2_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK                                                           0x0000FFFFL
+//DAGB2_WRCLI_GPU_SNOOP_OVERRIDE_VALUE
+#define DAGB2_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT                                                   0x0
+#define DAGB2_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK                                                     0x0000FFFFL
+//DAGB2_WRCLI_ASK_PENDING
+#define DAGB2_WRCLI_ASK_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB2_WRCLI_ASK_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB2_WRCLI_GO_PENDING
+#define DAGB2_WRCLI_GO_PENDING__BUSY__SHIFT                                                                   0x0
+#define DAGB2_WRCLI_GO_PENDING__BUSY_MASK                                                                     0xFFFFFFFFL
+//DAGB2_WRCLI_GBLSEND_PENDING
+#define DAGB2_WRCLI_GBLSEND_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB2_WRCLI_GBLSEND_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB2_WRCLI_TLB_PENDING
+#define DAGB2_WRCLI_TLB_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB2_WRCLI_TLB_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB2_WRCLI_OARB_PENDING
+#define DAGB2_WRCLI_OARB_PENDING__BUSY__SHIFT                                                                 0x0
+#define DAGB2_WRCLI_OARB_PENDING__BUSY_MASK                                                                   0xFFFFFFFFL
+//DAGB2_WRCLI_OSD_PENDING
+#define DAGB2_WRCLI_OSD_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB2_WRCLI_OSD_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB2_WRCLI_DBUS_ASK_PENDING
+#define DAGB2_WRCLI_DBUS_ASK_PENDING__BUSY__SHIFT                                                             0x0
+#define DAGB2_WRCLI_DBUS_ASK_PENDING__BUSY_MASK                                                               0xFFFFFFFFL
+//DAGB2_WRCLI_DBUS_GO_PENDING
+#define DAGB2_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB2_WRCLI_DBUS_GO_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB2_DAGB_DLY
+#define DAGB2_DAGB_DLY__DLY__SHIFT                                                                            0x0
+#define DAGB2_DAGB_DLY__CLI__SHIFT                                                                            0x8
+#define DAGB2_DAGB_DLY__POS__SHIFT                                                                            0x10
+#define DAGB2_DAGB_DLY__DLY_MASK                                                                              0x000000FFL
+#define DAGB2_DAGB_DLY__CLI_MASK                                                                              0x0000FF00L
+#define DAGB2_DAGB_DLY__POS_MASK                                                                              0x000F0000L
+//DAGB2_CNTL_MISC
+#define DAGB2_CNTL_MISC__EA_VC0_REMAP__SHIFT                                                                  0x0
+#define DAGB2_CNTL_MISC__EA_VC1_REMAP__SHIFT                                                                  0x3
+#define DAGB2_CNTL_MISC__EA_VC2_REMAP__SHIFT                                                                  0x6
+#define DAGB2_CNTL_MISC__EA_VC3_REMAP__SHIFT                                                                  0x9
+#define DAGB2_CNTL_MISC__EA_VC4_REMAP__SHIFT                                                                  0xc
+#define DAGB2_CNTL_MISC__EA_VC5_REMAP__SHIFT                                                                  0xf
+#define DAGB2_CNTL_MISC__EA_VC6_REMAP__SHIFT                                                                  0x12
+#define DAGB2_CNTL_MISC__EA_VC7_REMAP__SHIFT                                                                  0x15
+#define DAGB2_CNTL_MISC__BW_INIT_CYCLE__SHIFT                                                                 0x18
+#define DAGB2_CNTL_MISC__BW_RW_GAP_CYCLE__SHIFT                                                               0x1e
+#define DAGB2_CNTL_MISC__EA_VC0_REMAP_MASK                                                                    0x00000007L
+#define DAGB2_CNTL_MISC__EA_VC1_REMAP_MASK                                                                    0x00000038L
+#define DAGB2_CNTL_MISC__EA_VC2_REMAP_MASK                                                                    0x000001C0L
+#define DAGB2_CNTL_MISC__EA_VC3_REMAP_MASK                                                                    0x00000E00L
+#define DAGB2_CNTL_MISC__EA_VC4_REMAP_MASK                                                                    0x00007000L
+#define DAGB2_CNTL_MISC__EA_VC5_REMAP_MASK                                                                    0x00038000L
+#define DAGB2_CNTL_MISC__EA_VC6_REMAP_MASK                                                                    0x001C0000L
+#define DAGB2_CNTL_MISC__EA_VC7_REMAP_MASK                                                                    0x00E00000L
+#define DAGB2_CNTL_MISC__BW_INIT_CYCLE_MASK                                                                   0x3F000000L
+#define DAGB2_CNTL_MISC__BW_RW_GAP_CYCLE_MASK                                                                 0xC0000000L
+//DAGB2_CNTL_MISC2
+#define DAGB2_CNTL_MISC2__URG_BOOST_ENABLE__SHIFT                                                             0x0
+#define DAGB2_CNTL_MISC2__URG_HALT_ENABLE__SHIFT                                                              0x1
+#define DAGB2_CNTL_MISC2__DISABLE_WRREQ_CG__SHIFT                                                             0x2
+#define DAGB2_CNTL_MISC2__DISABLE_WRRET_CG__SHIFT                                                             0x3
+#define DAGB2_CNTL_MISC2__DISABLE_RDREQ_CG__SHIFT                                                             0x4
+#define DAGB2_CNTL_MISC2__DISABLE_RDRET_CG__SHIFT                                                             0x5
+#define DAGB2_CNTL_MISC2__DISABLE_TLBWR_CG__SHIFT                                                             0x6
+#define DAGB2_CNTL_MISC2__DISABLE_TLBRD_CG__SHIFT                                                             0x7
+#define DAGB2_CNTL_MISC2__DISABLE_EAWRREQ_BUSY__SHIFT                                                         0x8
+#define DAGB2_CNTL_MISC2__DISABLE_EARDREQ_BUSY__SHIFT                                                         0x9
+#define DAGB2_CNTL_MISC2__SWAP_CTL__SHIFT                                                                     0xa
+#define DAGB2_CNTL_MISC2__RDRET_FIFO_PERF__SHIFT                                                              0xb
+#define DAGB2_CNTL_MISC2__HDP_CID__SHIFT                                                                      0xc
+#define DAGB2_CNTL_MISC2__RDRET_FIFO_DLOCK_CREDITS__SHIFT                                                     0x10
+#define DAGB2_CNTL_MISC2__URG_BOOST_ENABLE_MASK                                                               0x00000001L
+#define DAGB2_CNTL_MISC2__URG_HALT_ENABLE_MASK                                                                0x00000002L
+#define DAGB2_CNTL_MISC2__DISABLE_WRREQ_CG_MASK                                                               0x00000004L
+#define DAGB2_CNTL_MISC2__DISABLE_WRRET_CG_MASK                                                               0x00000008L
+#define DAGB2_CNTL_MISC2__DISABLE_RDREQ_CG_MASK                                                               0x00000010L
+#define DAGB2_CNTL_MISC2__DISABLE_RDRET_CG_MASK                                                               0x00000020L
+#define DAGB2_CNTL_MISC2__DISABLE_TLBWR_CG_MASK                                                               0x00000040L
+#define DAGB2_CNTL_MISC2__DISABLE_TLBRD_CG_MASK                                                               0x00000080L
+#define DAGB2_CNTL_MISC2__DISABLE_EAWRREQ_BUSY_MASK                                                           0x00000100L
+#define DAGB2_CNTL_MISC2__DISABLE_EARDREQ_BUSY_MASK                                                           0x00000200L
+#define DAGB2_CNTL_MISC2__SWAP_CTL_MASK                                                                       0x00000400L
+#define DAGB2_CNTL_MISC2__RDRET_FIFO_PERF_MASK                                                                0x00000800L
+#define DAGB2_CNTL_MISC2__HDP_CID_MASK                                                                        0x0000F000L
+#define DAGB2_CNTL_MISC2__RDRET_FIFO_DLOCK_CREDITS_MASK                                                       0x003F0000L
+//DAGB2_FATAL_ERROR_CNTL
+#define DAGB2_FATAL_ERROR_CNTL__FILTER_NUM__SHIFT                                                             0x0
+#define DAGB2_FATAL_ERROR_CNTL__FILTER_NUM_MASK                                                               0x000003FFL
+//DAGB2_FATAL_ERROR_CLEAR
+#define DAGB2_FATAL_ERROR_CLEAR__CLEAR__SHIFT                                                                 0x0
+#define DAGB2_FATAL_ERROR_CLEAR__CLEAR_MASK                                                                   0x00000001L
+//DAGB2_FATAL_ERROR_STATUS0
+#define DAGB2_FATAL_ERROR_STATUS0__VALID__SHIFT                                                               0x0
+#define DAGB2_FATAL_ERROR_STATUS0__CID__SHIFT                                                                 0x1
+#define DAGB2_FATAL_ERROR_STATUS0__ADDR_LO__SHIFT                                                             0x6
+#define DAGB2_FATAL_ERROR_STATUS0__VALID_MASK                                                                 0x00000001L
+#define DAGB2_FATAL_ERROR_STATUS0__CID_MASK                                                                   0x0000003EL
+#define DAGB2_FATAL_ERROR_STATUS0__ADDR_LO_MASK                                                               0xFFFFFFC0L
+//DAGB2_FATAL_ERROR_STATUS1
+#define DAGB2_FATAL_ERROR_STATUS1__ADDR_HI__SHIFT                                                             0x0
+#define DAGB2_FATAL_ERROR_STATUS1__ADDR_HI_MASK                                                               0x0001FFFFL
+//DAGB2_FATAL_ERROR_STATUS2
+#define DAGB2_FATAL_ERROR_STATUS2__TAG__SHIFT                                                                 0x0
+#define DAGB2_FATAL_ERROR_STATUS2__VFID__SHIFT                                                                0x10
+#define DAGB2_FATAL_ERROR_STATUS2__VF__SHIFT                                                                  0x14
+#define DAGB2_FATAL_ERROR_STATUS2__SPACE__SHIFT                                                               0x15
+#define DAGB2_FATAL_ERROR_STATUS2__IO__SHIFT                                                                  0x16
+#define DAGB2_FATAL_ERROR_STATUS2__SIZE__SHIFT                                                                0x17
+#define DAGB2_FATAL_ERROR_STATUS2__DBGMSK__SHIFT                                                              0x18
+#define DAGB2_FATAL_ERROR_STATUS2__FED__SHIFT                                                                 0x19
+#define DAGB2_FATAL_ERROR_STATUS2__TAG_MASK                                                                   0x0000FFFFL
+#define DAGB2_FATAL_ERROR_STATUS2__VFID_MASK                                                                  0x000F0000L
+#define DAGB2_FATAL_ERROR_STATUS2__VF_MASK                                                                    0x00100000L
+#define DAGB2_FATAL_ERROR_STATUS2__SPACE_MASK                                                                 0x00200000L
+#define DAGB2_FATAL_ERROR_STATUS2__IO_MASK                                                                    0x00400000L
+#define DAGB2_FATAL_ERROR_STATUS2__SIZE_MASK                                                                  0x00800000L
+#define DAGB2_FATAL_ERROR_STATUS2__DBGMSK_MASK                                                                0x01000000L
+#define DAGB2_FATAL_ERROR_STATUS2__FED_MASK                                                                   0x02000000L
+//DAGB2_FATAL_ERROR_STATUS3
+#define DAGB2_FATAL_ERROR_STATUS3__NOALLOC__SHIFT                                                             0x0
+#define DAGB2_FATAL_ERROR_STATUS3__UNITID__SHIFT                                                              0x1
+#define DAGB2_FATAL_ERROR_STATUS3__OP__SHIFT                                                                  0x7
+#define DAGB2_FATAL_ERROR_STATUS3__SECLEVEL__SHIFT                                                            0xe
+#define DAGB2_FATAL_ERROR_STATUS3__WRTMZ__SHIFT                                                               0x11
+#define DAGB2_FATAL_ERROR_STATUS3__RDTMZ__SHIFT                                                               0x12
+#define DAGB2_FATAL_ERROR_STATUS3__SNOOP__SHIFT                                                               0x13
+#define DAGB2_FATAL_ERROR_STATUS3__INVAL__SHIFT                                                               0x14
+#define DAGB2_FATAL_ERROR_STATUS3__NACK__SHIFT                                                                0x15
+#define DAGB2_FATAL_ERROR_STATUS3__RO__SHIFT                                                                  0x17
+#define DAGB2_FATAL_ERROR_STATUS3__MEMLOG__SHIFT                                                              0x18
+#define DAGB2_FATAL_ERROR_STATUS3__EOP__SHIFT                                                                 0x19
+#define DAGB2_FATAL_ERROR_STATUS3__NOALLOC_MASK                                                               0x00000001L
+#define DAGB2_FATAL_ERROR_STATUS3__UNITID_MASK                                                                0x0000007EL
+#define DAGB2_FATAL_ERROR_STATUS3__OP_MASK                                                                    0x00003F80L
+#define DAGB2_FATAL_ERROR_STATUS3__SECLEVEL_MASK                                                              0x0001C000L
+#define DAGB2_FATAL_ERROR_STATUS3__WRTMZ_MASK                                                                 0x00020000L
+#define DAGB2_FATAL_ERROR_STATUS3__RDTMZ_MASK                                                                 0x00040000L
+#define DAGB2_FATAL_ERROR_STATUS3__SNOOP_MASK                                                                 0x00080000L
+#define DAGB2_FATAL_ERROR_STATUS3__INVAL_MASK                                                                 0x00100000L
+#define DAGB2_FATAL_ERROR_STATUS3__NACK_MASK                                                                  0x00600000L
+#define DAGB2_FATAL_ERROR_STATUS3__RO_MASK                                                                    0x00800000L
+#define DAGB2_FATAL_ERROR_STATUS3__MEMLOG_MASK                                                                0x01000000L
+#define DAGB2_FATAL_ERROR_STATUS3__EOP_MASK                                                                   0x02000000L
+//DAGB2_FIFO_EMPTY
+#define DAGB2_FIFO_EMPTY__EMPTY__SHIFT                                                                        0x0
+#define DAGB2_FIFO_EMPTY__EMPTY_MASK                                                                          0x00FFFFFFL
+//DAGB2_FIFO_FULL
+#define DAGB2_FIFO_FULL__FULL__SHIFT                                                                          0x0
+#define DAGB2_FIFO_FULL__FULL_MASK                                                                            0x007FFFFFL
+//DAGB2_WR_CREDITS_FULL
+#define DAGB2_WR_CREDITS_FULL__FULL__SHIFT                                                                    0x0
+#define DAGB2_WR_CREDITS_FULL__FULL_MASK                                                                      0x1FFFFFFFL
+//DAGB2_RD_CREDITS_FULL
+#define DAGB2_RD_CREDITS_FULL__FULL__SHIFT                                                                    0x0
+#define DAGB2_RD_CREDITS_FULL__FULL_MASK                                                                      0x0003FFFFL
+//DAGB2_PERFCOUNTER_LO
+#define DAGB2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define DAGB2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//DAGB2_PERFCOUNTER_HI
+#define DAGB2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define DAGB2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define DAGB2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define DAGB2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+//DAGB2_PERFCOUNTER0_CFG
+#define DAGB2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB2_PERFCOUNTER1_CFG
+#define DAGB2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB2_PERFCOUNTER2_CFG
+#define DAGB2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB2_PERFCOUNTER2_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB2_PERFCOUNTER2_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB2_PERFCOUNTER_RSLT_CNTL
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define DAGB2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+//DAGB2_L1TLB_REG_RW
+#define DAGB2_L1TLB_REG_RW__REG_WRITE_L1TLB_CTRL__SHIFT                                                       0x0
+#define DAGB2_L1TLB_REG_RW__REG_READ_L1TLB_CTRL__SHIFT                                                        0x1
+#define DAGB2_L1TLB_REG_RW__VMID_EXCEP_INT_CTRL__SHIFT                                                        0x2
+#define DAGB2_L1TLB_REG_RW__WDAT_PARITY_CHECK__SHIFT                                                          0x4
+#define DAGB2_L1TLB_REG_RW__DISABLE_RDRET_CHECK__SHIFT                                                        0x5
+#define DAGB2_L1TLB_REG_RW__RESERVE__SHIFT                                                                    0x6
+#define DAGB2_L1TLB_REG_RW__REG_WRITE_L1TLB_CTRL_MASK                                                         0x00000001L
+#define DAGB2_L1TLB_REG_RW__REG_READ_L1TLB_CTRL_MASK                                                          0x00000002L
+#define DAGB2_L1TLB_REG_RW__VMID_EXCEP_INT_CTRL_MASK                                                          0x00000004L
+#define DAGB2_L1TLB_REG_RW__WDAT_PARITY_CHECK_MASK                                                            0x00000010L
+#define DAGB2_L1TLB_REG_RW__DISABLE_RDRET_CHECK_MASK                                                          0x00000020L
+#define DAGB2_L1TLB_REG_RW__RESERVE_MASK                                                                      0xFFFFFFC0L
+
+
+// addressBlock: aid_mmhub_dagb_dagbdec3
+//DAGB3_RDCLI0
+#define DAGB3_RDCLI0__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_RDCLI0__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_RDCLI0__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_RDCLI0__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_RDCLI0__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_RDCLI0__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_RDCLI0__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_RDCLI0__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_RDCLI0__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_RDCLI0__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_RDCLI0__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_RDCLI0__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_RDCLI0__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_RDCLI0__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_RDCLI0__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_RDCLI0__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_RDCLI0__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_RDCLI0__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_RDCLI0__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_RDCLI0__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_RDCLI1
+#define DAGB3_RDCLI1__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_RDCLI1__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_RDCLI1__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_RDCLI1__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_RDCLI1__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_RDCLI1__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_RDCLI1__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_RDCLI1__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_RDCLI1__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_RDCLI1__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_RDCLI1__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_RDCLI1__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_RDCLI1__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_RDCLI1__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_RDCLI1__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_RDCLI1__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_RDCLI1__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_RDCLI1__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_RDCLI1__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_RDCLI1__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_RDCLI2
+#define DAGB3_RDCLI2__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_RDCLI2__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_RDCLI2__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_RDCLI2__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_RDCLI2__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_RDCLI2__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_RDCLI2__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_RDCLI2__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_RDCLI2__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_RDCLI2__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_RDCLI2__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_RDCLI2__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_RDCLI2__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_RDCLI2__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_RDCLI2__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_RDCLI2__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_RDCLI2__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_RDCLI2__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_RDCLI2__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_RDCLI2__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_RDCLI3
+#define DAGB3_RDCLI3__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_RDCLI3__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_RDCLI3__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_RDCLI3__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_RDCLI3__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_RDCLI3__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_RDCLI3__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_RDCLI3__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_RDCLI3__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_RDCLI3__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_RDCLI3__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_RDCLI3__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_RDCLI3__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_RDCLI3__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_RDCLI3__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_RDCLI3__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_RDCLI3__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_RDCLI3__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_RDCLI3__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_RDCLI3__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_RDCLI4
+#define DAGB3_RDCLI4__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_RDCLI4__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_RDCLI4__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_RDCLI4__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_RDCLI4__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_RDCLI4__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_RDCLI4__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_RDCLI4__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_RDCLI4__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_RDCLI4__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_RDCLI4__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_RDCLI4__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_RDCLI4__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_RDCLI4__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_RDCLI4__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_RDCLI4__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_RDCLI4__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_RDCLI4__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_RDCLI4__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_RDCLI4__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_RDCLI5
+#define DAGB3_RDCLI5__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_RDCLI5__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_RDCLI5__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_RDCLI5__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_RDCLI5__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_RDCLI5__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_RDCLI5__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_RDCLI5__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_RDCLI5__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_RDCLI5__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_RDCLI5__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_RDCLI5__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_RDCLI5__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_RDCLI5__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_RDCLI5__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_RDCLI5__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_RDCLI5__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_RDCLI5__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_RDCLI5__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_RDCLI5__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_RDCLI6
+#define DAGB3_RDCLI6__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_RDCLI6__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_RDCLI6__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_RDCLI6__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_RDCLI6__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_RDCLI6__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_RDCLI6__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_RDCLI6__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_RDCLI6__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_RDCLI6__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_RDCLI6__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_RDCLI6__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_RDCLI6__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_RDCLI6__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_RDCLI6__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_RDCLI6__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_RDCLI6__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_RDCLI6__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_RDCLI6__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_RDCLI6__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_RDCLI7
+#define DAGB3_RDCLI7__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_RDCLI7__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_RDCLI7__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_RDCLI7__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_RDCLI7__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_RDCLI7__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_RDCLI7__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_RDCLI7__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_RDCLI7__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_RDCLI7__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_RDCLI7__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_RDCLI7__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_RDCLI7__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_RDCLI7__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_RDCLI7__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_RDCLI7__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_RDCLI7__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_RDCLI7__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_RDCLI7__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_RDCLI7__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_RDCLI8
+#define DAGB3_RDCLI8__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_RDCLI8__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_RDCLI8__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_RDCLI8__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_RDCLI8__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_RDCLI8__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_RDCLI8__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_RDCLI8__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_RDCLI8__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_RDCLI8__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_RDCLI8__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_RDCLI8__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_RDCLI8__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_RDCLI8__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_RDCLI8__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_RDCLI8__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_RDCLI8__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_RDCLI8__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_RDCLI8__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_RDCLI8__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_RDCLI9
+#define DAGB3_RDCLI9__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_RDCLI9__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_RDCLI9__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_RDCLI9__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_RDCLI9__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_RDCLI9__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_RDCLI9__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_RDCLI9__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_RDCLI9__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_RDCLI9__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_RDCLI9__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_RDCLI9__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_RDCLI9__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_RDCLI9__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_RDCLI9__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_RDCLI9__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_RDCLI9__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_RDCLI9__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_RDCLI9__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_RDCLI9__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_RDCLI10
+#define DAGB3_RDCLI10__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_RDCLI10__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_RDCLI10__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_RDCLI10__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_RDCLI10__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_RDCLI10__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_RDCLI10__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_RDCLI10__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_RDCLI10__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_RDCLI10__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_RDCLI10__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_RDCLI10__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_RDCLI10__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_RDCLI10__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_RDCLI10__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_RDCLI10__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_RDCLI10__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_RDCLI10__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_RDCLI10__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_RDCLI10__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_RDCLI11
+#define DAGB3_RDCLI11__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_RDCLI11__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_RDCLI11__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_RDCLI11__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_RDCLI11__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_RDCLI11__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_RDCLI11__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_RDCLI11__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_RDCLI11__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_RDCLI11__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_RDCLI11__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_RDCLI11__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_RDCLI11__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_RDCLI11__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_RDCLI11__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_RDCLI11__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_RDCLI11__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_RDCLI11__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_RDCLI11__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_RDCLI11__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_RDCLI12
+#define DAGB3_RDCLI12__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_RDCLI12__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_RDCLI12__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_RDCLI12__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_RDCLI12__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_RDCLI12__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_RDCLI12__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_RDCLI12__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_RDCLI12__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_RDCLI12__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_RDCLI12__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_RDCLI12__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_RDCLI12__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_RDCLI12__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_RDCLI12__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_RDCLI12__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_RDCLI12__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_RDCLI12__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_RDCLI12__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_RDCLI12__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_RDCLI13
+#define DAGB3_RDCLI13__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_RDCLI13__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_RDCLI13__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_RDCLI13__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_RDCLI13__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_RDCLI13__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_RDCLI13__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_RDCLI13__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_RDCLI13__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_RDCLI13__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_RDCLI13__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_RDCLI13__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_RDCLI13__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_RDCLI13__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_RDCLI13__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_RDCLI13__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_RDCLI13__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_RDCLI13__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_RDCLI13__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_RDCLI13__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_RDCLI14
+#define DAGB3_RDCLI14__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_RDCLI14__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_RDCLI14__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_RDCLI14__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_RDCLI14__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_RDCLI14__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_RDCLI14__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_RDCLI14__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_RDCLI14__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_RDCLI14__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_RDCLI14__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_RDCLI14__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_RDCLI14__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_RDCLI14__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_RDCLI14__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_RDCLI14__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_RDCLI14__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_RDCLI14__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_RDCLI14__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_RDCLI14__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_RDCLI15
+#define DAGB3_RDCLI15__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_RDCLI15__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_RDCLI15__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_RDCLI15__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_RDCLI15__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_RDCLI15__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_RDCLI15__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_RDCLI15__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_RDCLI15__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_RDCLI15__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_RDCLI15__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_RDCLI15__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_RDCLI15__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_RDCLI15__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_RDCLI15__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_RDCLI15__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_RDCLI15__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_RDCLI15__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_RDCLI15__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_RDCLI15__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_RD_CNTL
+#define DAGB3_RD_CNTL__SCLK_FREQ__SHIFT                                                                       0x0
+#define DAGB3_RD_CNTL__CLI_MAX_BW_WINDOW__SHIFT                                                               0x4
+#define DAGB3_RD_CNTL__VC_MAX_BW_WINDOW__SHIFT                                                                0xa
+#define DAGB3_RD_CNTL__IO_LEVEL_OVERRIDE_ENABLE__SHIFT                                                        0x10
+#define DAGB3_RD_CNTL__IO_LEVEL__SHIFT                                                                        0x11
+#define DAGB3_RD_CNTL__IO_LEVEL_COMPLY_VC__SHIFT                                                              0x14
+#define DAGB3_RD_CNTL__SHARE_VC_NUM__SHIFT                                                                    0x17
+#define DAGB3_RD_CNTL__FIX_JUMP__SHIFT                                                                        0x1a
+#define DAGB3_RD_CNTL__SCLK_FREQ_MASK                                                                         0x0000000FL
+#define DAGB3_RD_CNTL__CLI_MAX_BW_WINDOW_MASK                                                                 0x000003F0L
+#define DAGB3_RD_CNTL__VC_MAX_BW_WINDOW_MASK                                                                  0x0000FC00L
+#define DAGB3_RD_CNTL__IO_LEVEL_OVERRIDE_ENABLE_MASK                                                          0x00010000L
+#define DAGB3_RD_CNTL__IO_LEVEL_MASK                                                                          0x000E0000L
+#define DAGB3_RD_CNTL__IO_LEVEL_COMPLY_VC_MASK                                                                0x00700000L
+#define DAGB3_RD_CNTL__SHARE_VC_NUM_MASK                                                                      0x03800000L
+#define DAGB3_RD_CNTL__FIX_JUMP_MASK                                                                          0x04000000L
+//DAGB3_RD_GMI_CNTL
+#define DAGB3_RD_GMI_CNTL__EA_CREDIT__SHIFT                                                                   0x0
+#define DAGB3_RD_GMI_CNTL__LEVEL__SHIFT                                                                       0x6
+#define DAGB3_RD_GMI_CNTL__MAX_BURST__SHIFT                                                                   0x9
+#define DAGB3_RD_GMI_CNTL__LAZY_TIMER__SHIFT                                                                  0xd
+#define DAGB3_RD_GMI_CNTL__EA_CREDIT_MASK                                                                     0x0000003FL
+#define DAGB3_RD_GMI_CNTL__LEVEL_MASK                                                                         0x000001C0L
+#define DAGB3_RD_GMI_CNTL__MAX_BURST_MASK                                                                     0x00001E00L
+#define DAGB3_RD_GMI_CNTL__LAZY_TIMER_MASK                                                                    0x0001E000L
+//DAGB3_RD_ADDR_DAGB
+#define DAGB3_RD_ADDR_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB3_RD_ADDR_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB3_RD_ADDR_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB3_RD_ADDR_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB3_RD_ADDR_DAGB__JUMP_MODE__SHIFT                                                                  0xd
+#define DAGB3_RD_ADDR_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB3_RD_ADDR_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB3_RD_ADDR_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB3_RD_ADDR_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+#define DAGB3_RD_ADDR_DAGB__JUMP_MODE_MASK                                                                    0x00002000L
+//DAGB3_RD_OUTPUT_DAGB_MAX_BURST
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC0__SHIFT                                                            0x0
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC1__SHIFT                                                            0x4
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC2__SHIFT                                                            0x8
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC3__SHIFT                                                            0xc
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC4__SHIFT                                                            0x10
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC5__SHIFT                                                            0x14
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC6__SHIFT                                                            0x18
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC7__SHIFT                                                            0x1c
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC0_MASK                                                              0x0000000FL
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC1_MASK                                                              0x000000F0L
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC2_MASK                                                              0x00000F00L
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC3_MASK                                                              0x0000F000L
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC4_MASK                                                              0x000F0000L
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC5_MASK                                                              0x00F00000L
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC6_MASK                                                              0x0F000000L
+#define DAGB3_RD_OUTPUT_DAGB_MAX_BURST__VC7_MASK                                                              0xF0000000L
+//DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC0__SHIFT                                                           0x0
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC1__SHIFT                                                           0x4
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC2__SHIFT                                                           0x8
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC3__SHIFT                                                           0xc
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC4__SHIFT                                                           0x10
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC5__SHIFT                                                           0x14
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC6__SHIFT                                                           0x18
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC7__SHIFT                                                           0x1c
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC0_MASK                                                             0x0000000FL
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC1_MASK                                                             0x000000F0L
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC2_MASK                                                             0x00000F00L
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC3_MASK                                                             0x0000F000L
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC4_MASK                                                             0x000F0000L
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC5_MASK                                                             0x00F00000L
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC6_MASK                                                             0x0F000000L
+#define DAGB3_RD_OUTPUT_DAGB_LAZY_TIMER__VC7_MASK                                                             0xF0000000L
+//DAGB3_RD_CGTT_CLK_CTRL
+#define DAGB3_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                               0x0
+#define DAGB3_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                         0x4
+#define DAGB3_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                   0xc
+#define DAGB3_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                            0x1e
+#define DAGB3_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                          0x1f
+#define DAGB3_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                 0x0000000FL
+#define DAGB3_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                           0x00000FF0L
+#define DAGB3_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                     0x0FFFF000L
+#define DAGB3_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                              0x40000000L
+#define DAGB3_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                            0x80000000L
+//DAGB3_L1TLB_RD_CGTT_CLK_CTRL
+#define DAGB3_L1TLB_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB3_L1TLB_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB3_L1TLB_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB3_L1TLB_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB3_L1TLB_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB3_L1TLB_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB3_L1TLB_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB3_L1TLB_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB3_L1TLB_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB3_L1TLB_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB3_ATCVM_RD_CGTT_CLK_CTRL
+#define DAGB3_ATCVM_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB3_ATCVM_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB3_ATCVM_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB3_ATCVM_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB3_ATCVM_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB3_ATCVM_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB3_ATCVM_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB3_ATCVM_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB3_ATCVM_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB3_ATCVM_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB3_RD_ADDR_DAGB_MAX_BURST0
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB3_RD_ADDR_DAGB_LAZY_TIMER0
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB3_RD_ADDR_DAGB_MAX_BURST1
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB3_RD_ADDR_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB3_RD_ADDR_DAGB_LAZY_TIMER1
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB3_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB3_RD_VC0_CNTL
+#define DAGB3_RD_VC0_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_RD_VC0_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_RD_VC0_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_RD_VC0_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_RD_VC0_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_RD_VC0_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_RD_VC0_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_RD_VC0_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_RD_VC0_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_RD_VC0_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_RD_VC0_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_RD_VC0_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_RD_VC0_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_RD_VC0_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_RD_VC0_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_RD_VC0_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_RD_VC1_CNTL
+#define DAGB3_RD_VC1_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_RD_VC1_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_RD_VC1_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_RD_VC1_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_RD_VC1_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_RD_VC1_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_RD_VC1_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_RD_VC1_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_RD_VC1_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_RD_VC1_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_RD_VC1_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_RD_VC1_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_RD_VC1_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_RD_VC1_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_RD_VC1_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_RD_VC1_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_RD_VC2_CNTL
+#define DAGB3_RD_VC2_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_RD_VC2_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_RD_VC2_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_RD_VC2_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_RD_VC2_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_RD_VC2_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_RD_VC2_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_RD_VC2_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_RD_VC2_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_RD_VC2_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_RD_VC2_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_RD_VC2_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_RD_VC2_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_RD_VC2_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_RD_VC2_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_RD_VC2_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_RD_VC3_CNTL
+#define DAGB3_RD_VC3_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_RD_VC3_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_RD_VC3_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_RD_VC3_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_RD_VC3_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_RD_VC3_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_RD_VC3_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_RD_VC3_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_RD_VC3_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_RD_VC3_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_RD_VC3_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_RD_VC3_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_RD_VC3_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_RD_VC3_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_RD_VC3_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_RD_VC3_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_RD_VC4_CNTL
+#define DAGB3_RD_VC4_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_RD_VC4_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_RD_VC4_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_RD_VC4_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_RD_VC4_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_RD_VC4_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_RD_VC4_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_RD_VC4_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_RD_VC4_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_RD_VC4_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_RD_VC4_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_RD_VC4_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_RD_VC4_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_RD_VC4_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_RD_VC4_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_RD_VC4_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_RD_VC5_CNTL
+#define DAGB3_RD_VC5_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_RD_VC5_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_RD_VC5_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_RD_VC5_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_RD_VC5_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_RD_VC5_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_RD_VC5_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_RD_VC5_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_RD_VC5_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_RD_VC5_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_RD_VC5_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_RD_VC5_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_RD_VC5_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_RD_VC5_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_RD_VC5_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_RD_VC5_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_RD_VC6_CNTL
+#define DAGB3_RD_VC6_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_RD_VC6_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_RD_VC6_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_RD_VC6_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_RD_VC6_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_RD_VC6_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_RD_VC6_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_RD_VC6_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_RD_VC6_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_RD_VC6_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_RD_VC6_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_RD_VC6_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_RD_VC6_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_RD_VC6_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_RD_VC6_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_RD_VC6_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_RD_VC7_CNTL
+#define DAGB3_RD_VC7_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_RD_VC7_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_RD_VC7_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_RD_VC7_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_RD_VC7_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_RD_VC7_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_RD_VC7_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_RD_VC7_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_RD_VC7_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_RD_VC7_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_RD_VC7_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_RD_VC7_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_RD_VC7_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_RD_VC7_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_RD_VC7_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_RD_VC7_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_RD_CNTL_MISC
+#define DAGB3_RD_CNTL_MISC__STOR_POOL_CREDIT__SHIFT                                                           0x0
+#define DAGB3_RD_CNTL_MISC__EA_POOL_CREDIT__SHIFT                                                             0x6
+#define DAGB3_RD_CNTL_MISC__IO_EA_CREDIT__SHIFT                                                               0xd
+#define DAGB3_RD_CNTL_MISC__STOR_CC_NEW_MODE__SHIFT                                                           0x13
+#define DAGB3_RD_CNTL_MISC__EA_CC_LEGACY_MODE__SHIFT                                                          0x14
+#define DAGB3_RD_CNTL_MISC__UTCL2_CID__SHIFT                                                                  0x15
+#define DAGB3_RD_CNTL_MISC__RDRET_FIFO_CREDITS__SHIFT                                                         0x1a
+#define DAGB3_RD_CNTL_MISC__STOR_POOL_CREDIT_MASK                                                             0x0000003FL
+#define DAGB3_RD_CNTL_MISC__EA_POOL_CREDIT_MASK                                                               0x00001FC0L
+#define DAGB3_RD_CNTL_MISC__IO_EA_CREDIT_MASK                                                                 0x0007E000L
+#define DAGB3_RD_CNTL_MISC__STOR_CC_NEW_MODE_MASK                                                             0x00080000L
+#define DAGB3_RD_CNTL_MISC__EA_CC_LEGACY_MODE_MASK                                                            0x00100000L
+#define DAGB3_RD_CNTL_MISC__UTCL2_CID_MASK                                                                    0x03E00000L
+#define DAGB3_RD_CNTL_MISC__RDRET_FIFO_CREDITS_MASK                                                           0xFC000000L
+//DAGB3_RD_TLB_CREDIT
+#define DAGB3_RD_TLB_CREDIT__TLB0__SHIFT                                                                      0x0
+#define DAGB3_RD_TLB_CREDIT__TLB1__SHIFT                                                                      0x5
+#define DAGB3_RD_TLB_CREDIT__TLB2__SHIFT                                                                      0xa
+#define DAGB3_RD_TLB_CREDIT__TLB3__SHIFT                                                                      0xf
+#define DAGB3_RD_TLB_CREDIT__TLB4__SHIFT                                                                      0x14
+#define DAGB3_RD_TLB_CREDIT__TLB5__SHIFT                                                                      0x19
+#define DAGB3_RD_TLB_CREDIT__TLB0_MASK                                                                        0x0000001FL
+#define DAGB3_RD_TLB_CREDIT__TLB1_MASK                                                                        0x000003E0L
+#define DAGB3_RD_TLB_CREDIT__TLB2_MASK                                                                        0x00007C00L
+#define DAGB3_RD_TLB_CREDIT__TLB3_MASK                                                                        0x000F8000L
+#define DAGB3_RD_TLB_CREDIT__TLB4_MASK                                                                        0x01F00000L
+#define DAGB3_RD_TLB_CREDIT__TLB5_MASK                                                                        0x3E000000L
+//DAGB3_RD_RDRET_CREDIT_CNTL
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC0_CREDIT__SHIFT                                                         0x0
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC1_CREDIT__SHIFT                                                         0x6
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC2_CREDIT__SHIFT                                                         0xc
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC3_CREDIT__SHIFT                                                         0x12
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC4_CREDIT__SHIFT                                                         0x18
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC_MODE__SHIFT                                                            0x1e
+#define DAGB3_RD_RDRET_CREDIT_CNTL__FIX_EQ__SHIFT                                                             0x1f
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC0_CREDIT_MASK                                                           0x0000003FL
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC1_CREDIT_MASK                                                           0x00000FC0L
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC2_CREDIT_MASK                                                           0x0003F000L
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC3_CREDIT_MASK                                                           0x00FC0000L
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC4_CREDIT_MASK                                                           0x3F000000L
+#define DAGB3_RD_RDRET_CREDIT_CNTL__VC_MODE_MASK                                                              0x40000000L
+#define DAGB3_RD_RDRET_CREDIT_CNTL__FIX_EQ_MASK                                                               0x80000000L
+//DAGB3_RD_RDRET_CREDIT_CNTL2
+#define DAGB3_RD_RDRET_CREDIT_CNTL2__IO_CREDIT__SHIFT                                                         0x0
+#define DAGB3_RD_RDRET_CREDIT_CNTL2__GMI_CREDIT__SHIFT                                                        0x6
+#define DAGB3_RD_RDRET_CREDIT_CNTL2__POOL_CREDIT__SHIFT                                                       0xc
+#define DAGB3_RD_RDRET_CREDIT_CNTL2__IO_CREDIT_MASK                                                           0x0000003FL
+#define DAGB3_RD_RDRET_CREDIT_CNTL2__GMI_CREDIT_MASK                                                          0x00000FC0L
+#define DAGB3_RD_RDRET_CREDIT_CNTL2__POOL_CREDIT_MASK                                                         0x0007F000L
+//DAGB3_RDCLI_ASK_PENDING
+#define DAGB3_RDCLI_ASK_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB3_RDCLI_ASK_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB3_RDCLI_GO_PENDING
+#define DAGB3_RDCLI_GO_PENDING__BUSY__SHIFT                                                                   0x0
+#define DAGB3_RDCLI_GO_PENDING__BUSY_MASK                                                                     0xFFFFFFFFL
+//DAGB3_RDCLI_GBLSEND_PENDING
+#define DAGB3_RDCLI_GBLSEND_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB3_RDCLI_GBLSEND_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB3_RDCLI_TLB_PENDING
+#define DAGB3_RDCLI_TLB_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB3_RDCLI_TLB_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB3_RDCLI_OARB_PENDING
+#define DAGB3_RDCLI_OARB_PENDING__BUSY__SHIFT                                                                 0x0
+#define DAGB3_RDCLI_OARB_PENDING__BUSY_MASK                                                                   0xFFFFFFFFL
+//DAGB3_RDCLI_OSD_PENDING
+#define DAGB3_RDCLI_OSD_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB3_RDCLI_OSD_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB3_WRCLI0
+#define DAGB3_WRCLI0__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_WRCLI0__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_WRCLI0__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_WRCLI0__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_WRCLI0__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_WRCLI0__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_WRCLI0__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_WRCLI0__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_WRCLI0__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_WRCLI0__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_WRCLI0__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_WRCLI0__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_WRCLI0__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_WRCLI0__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_WRCLI0__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_WRCLI0__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_WRCLI0__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_WRCLI0__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_WRCLI0__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_WRCLI0__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_WRCLI1
+#define DAGB3_WRCLI1__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_WRCLI1__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_WRCLI1__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_WRCLI1__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_WRCLI1__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_WRCLI1__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_WRCLI1__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_WRCLI1__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_WRCLI1__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_WRCLI1__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_WRCLI1__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_WRCLI1__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_WRCLI1__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_WRCLI1__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_WRCLI1__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_WRCLI1__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_WRCLI1__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_WRCLI1__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_WRCLI1__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_WRCLI1__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_WRCLI2
+#define DAGB3_WRCLI2__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_WRCLI2__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_WRCLI2__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_WRCLI2__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_WRCLI2__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_WRCLI2__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_WRCLI2__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_WRCLI2__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_WRCLI2__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_WRCLI2__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_WRCLI2__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_WRCLI2__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_WRCLI2__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_WRCLI2__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_WRCLI2__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_WRCLI2__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_WRCLI2__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_WRCLI2__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_WRCLI2__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_WRCLI2__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_WRCLI3
+#define DAGB3_WRCLI3__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_WRCLI3__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_WRCLI3__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_WRCLI3__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_WRCLI3__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_WRCLI3__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_WRCLI3__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_WRCLI3__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_WRCLI3__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_WRCLI3__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_WRCLI3__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_WRCLI3__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_WRCLI3__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_WRCLI3__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_WRCLI3__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_WRCLI3__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_WRCLI3__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_WRCLI3__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_WRCLI3__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_WRCLI3__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_WRCLI4
+#define DAGB3_WRCLI4__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_WRCLI4__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_WRCLI4__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_WRCLI4__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_WRCLI4__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_WRCLI4__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_WRCLI4__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_WRCLI4__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_WRCLI4__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_WRCLI4__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_WRCLI4__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_WRCLI4__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_WRCLI4__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_WRCLI4__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_WRCLI4__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_WRCLI4__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_WRCLI4__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_WRCLI4__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_WRCLI4__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_WRCLI4__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_WRCLI5
+#define DAGB3_WRCLI5__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_WRCLI5__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_WRCLI5__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_WRCLI5__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_WRCLI5__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_WRCLI5__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_WRCLI5__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_WRCLI5__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_WRCLI5__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_WRCLI5__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_WRCLI5__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_WRCLI5__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_WRCLI5__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_WRCLI5__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_WRCLI5__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_WRCLI5__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_WRCLI5__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_WRCLI5__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_WRCLI5__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_WRCLI5__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_WRCLI6
+#define DAGB3_WRCLI6__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_WRCLI6__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_WRCLI6__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_WRCLI6__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_WRCLI6__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_WRCLI6__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_WRCLI6__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_WRCLI6__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_WRCLI6__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_WRCLI6__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_WRCLI6__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_WRCLI6__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_WRCLI6__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_WRCLI6__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_WRCLI6__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_WRCLI6__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_WRCLI6__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_WRCLI6__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_WRCLI6__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_WRCLI6__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_WRCLI7
+#define DAGB3_WRCLI7__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_WRCLI7__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_WRCLI7__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_WRCLI7__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_WRCLI7__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_WRCLI7__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_WRCLI7__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_WRCLI7__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_WRCLI7__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_WRCLI7__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_WRCLI7__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_WRCLI7__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_WRCLI7__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_WRCLI7__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_WRCLI7__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_WRCLI7__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_WRCLI7__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_WRCLI7__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_WRCLI7__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_WRCLI7__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_WRCLI8
+#define DAGB3_WRCLI8__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_WRCLI8__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_WRCLI8__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_WRCLI8__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_WRCLI8__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_WRCLI8__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_WRCLI8__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_WRCLI8__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_WRCLI8__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_WRCLI8__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_WRCLI8__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_WRCLI8__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_WRCLI8__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_WRCLI8__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_WRCLI8__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_WRCLI8__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_WRCLI8__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_WRCLI8__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_WRCLI8__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_WRCLI8__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_WRCLI9
+#define DAGB3_WRCLI9__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB3_WRCLI9__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB3_WRCLI9__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB3_WRCLI9__URG_LOW__SHIFT                                                                          0x8
+#define DAGB3_WRCLI9__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB3_WRCLI9__MAX_BW__SHIFT                                                                           0xd
+#define DAGB3_WRCLI9__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB3_WRCLI9__MIN_BW__SHIFT                                                                           0x16
+#define DAGB3_WRCLI9__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB3_WRCLI9__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB3_WRCLI9__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB3_WRCLI9__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB3_WRCLI9__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB3_WRCLI9__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB3_WRCLI9__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB3_WRCLI9__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB3_WRCLI9__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB3_WRCLI9__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB3_WRCLI9__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB3_WRCLI9__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB3_WRCLI10
+#define DAGB3_WRCLI10__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_WRCLI10__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_WRCLI10__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_WRCLI10__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_WRCLI10__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_WRCLI10__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_WRCLI10__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_WRCLI10__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_WRCLI10__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_WRCLI10__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_WRCLI10__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_WRCLI10__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_WRCLI10__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_WRCLI10__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_WRCLI10__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_WRCLI10__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_WRCLI10__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_WRCLI10__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_WRCLI10__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_WRCLI10__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_WRCLI11
+#define DAGB3_WRCLI11__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_WRCLI11__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_WRCLI11__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_WRCLI11__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_WRCLI11__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_WRCLI11__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_WRCLI11__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_WRCLI11__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_WRCLI11__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_WRCLI11__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_WRCLI11__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_WRCLI11__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_WRCLI11__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_WRCLI11__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_WRCLI11__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_WRCLI11__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_WRCLI11__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_WRCLI11__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_WRCLI11__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_WRCLI11__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_WRCLI12
+#define DAGB3_WRCLI12__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_WRCLI12__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_WRCLI12__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_WRCLI12__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_WRCLI12__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_WRCLI12__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_WRCLI12__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_WRCLI12__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_WRCLI12__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_WRCLI12__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_WRCLI12__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_WRCLI12__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_WRCLI12__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_WRCLI12__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_WRCLI12__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_WRCLI12__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_WRCLI12__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_WRCLI12__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_WRCLI12__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_WRCLI12__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_WRCLI13
+#define DAGB3_WRCLI13__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_WRCLI13__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_WRCLI13__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_WRCLI13__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_WRCLI13__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_WRCLI13__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_WRCLI13__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_WRCLI13__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_WRCLI13__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_WRCLI13__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_WRCLI13__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_WRCLI13__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_WRCLI13__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_WRCLI13__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_WRCLI13__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_WRCLI13__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_WRCLI13__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_WRCLI13__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_WRCLI13__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_WRCLI13__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_WRCLI14
+#define DAGB3_WRCLI14__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_WRCLI14__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_WRCLI14__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_WRCLI14__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_WRCLI14__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_WRCLI14__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_WRCLI14__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_WRCLI14__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_WRCLI14__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_WRCLI14__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_WRCLI14__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_WRCLI14__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_WRCLI14__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_WRCLI14__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_WRCLI14__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_WRCLI14__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_WRCLI14__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_WRCLI14__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_WRCLI14__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_WRCLI14__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_WRCLI15
+#define DAGB3_WRCLI15__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB3_WRCLI15__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB3_WRCLI15__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB3_WRCLI15__URG_LOW__SHIFT                                                                         0x8
+#define DAGB3_WRCLI15__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB3_WRCLI15__MAX_BW__SHIFT                                                                          0xd
+#define DAGB3_WRCLI15__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB3_WRCLI15__MIN_BW__SHIFT                                                                          0x16
+#define DAGB3_WRCLI15__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB3_WRCLI15__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB3_WRCLI15__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB3_WRCLI15__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB3_WRCLI15__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB3_WRCLI15__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB3_WRCLI15__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB3_WRCLI15__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB3_WRCLI15__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB3_WRCLI15__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB3_WRCLI15__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB3_WRCLI15__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB3_WR_CNTL
+#define DAGB3_WR_CNTL__SCLK_FREQ__SHIFT                                                                       0x0
+#define DAGB3_WR_CNTL__CLI_MAX_BW_WINDOW__SHIFT                                                               0x4
+#define DAGB3_WR_CNTL__VC_MAX_BW_WINDOW__SHIFT                                                                0xa
+#define DAGB3_WR_CNTL__IO_LEVEL_OVERRIDE_ENABLE__SHIFT                                                        0x10
+#define DAGB3_WR_CNTL__IO_LEVEL__SHIFT                                                                        0x11
+#define DAGB3_WR_CNTL__IO_LEVEL_COMPLY_VC__SHIFT                                                              0x14
+#define DAGB3_WR_CNTL__SHARE_VC_NUM__SHIFT                                                                    0x17
+#define DAGB3_WR_CNTL__FIX_JUMP__SHIFT                                                                        0x1a
+#define DAGB3_WR_CNTL__SCLK_FREQ_MASK                                                                         0x0000000FL
+#define DAGB3_WR_CNTL__CLI_MAX_BW_WINDOW_MASK                                                                 0x000003F0L
+#define DAGB3_WR_CNTL__VC_MAX_BW_WINDOW_MASK                                                                  0x0000FC00L
+#define DAGB3_WR_CNTL__IO_LEVEL_OVERRIDE_ENABLE_MASK                                                          0x00010000L
+#define DAGB3_WR_CNTL__IO_LEVEL_MASK                                                                          0x000E0000L
+#define DAGB3_WR_CNTL__IO_LEVEL_COMPLY_VC_MASK                                                                0x00700000L
+#define DAGB3_WR_CNTL__SHARE_VC_NUM_MASK                                                                      0x03800000L
+#define DAGB3_WR_CNTL__FIX_JUMP_MASK                                                                          0x04000000L
+//DAGB3_WR_GMI_CNTL
+#define DAGB3_WR_GMI_CNTL__EA_CREDIT__SHIFT                                                                   0x0
+#define DAGB3_WR_GMI_CNTL__LEVEL__SHIFT                                                                       0x6
+#define DAGB3_WR_GMI_CNTL__MAX_BURST__SHIFT                                                                   0x9
+#define DAGB3_WR_GMI_CNTL__LAZY_TIMER__SHIFT                                                                  0xd
+#define DAGB3_WR_GMI_CNTL__EA_CREDIT_MASK                                                                     0x0000003FL
+#define DAGB3_WR_GMI_CNTL__LEVEL_MASK                                                                         0x000001C0L
+#define DAGB3_WR_GMI_CNTL__MAX_BURST_MASK                                                                     0x00001E00L
+#define DAGB3_WR_GMI_CNTL__LAZY_TIMER_MASK                                                                    0x0001E000L
+//DAGB3_WR_ADDR_DAGB
+#define DAGB3_WR_ADDR_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB3_WR_ADDR_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB3_WR_ADDR_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB3_WR_ADDR_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB3_WR_ADDR_DAGB__JUMP_MODE__SHIFT                                                                  0xd
+#define DAGB3_WR_ADDR_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB3_WR_ADDR_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB3_WR_ADDR_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB3_WR_ADDR_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+#define DAGB3_WR_ADDR_DAGB__JUMP_MODE_MASK                                                                    0x00002000L
+//DAGB3_WR_OUTPUT_DAGB_MAX_BURST
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC0__SHIFT                                                            0x0
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC1__SHIFT                                                            0x4
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC2__SHIFT                                                            0x8
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC3__SHIFT                                                            0xc
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC4__SHIFT                                                            0x10
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC5__SHIFT                                                            0x14
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC6__SHIFT                                                            0x18
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC7__SHIFT                                                            0x1c
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC0_MASK                                                              0x0000000FL
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC1_MASK                                                              0x000000F0L
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC2_MASK                                                              0x00000F00L
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC3_MASK                                                              0x0000F000L
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC4_MASK                                                              0x000F0000L
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC5_MASK                                                              0x00F00000L
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC6_MASK                                                              0x0F000000L
+#define DAGB3_WR_OUTPUT_DAGB_MAX_BURST__VC7_MASK                                                              0xF0000000L
+//DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC0__SHIFT                                                           0x0
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC1__SHIFT                                                           0x4
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC2__SHIFT                                                           0x8
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC3__SHIFT                                                           0xc
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC4__SHIFT                                                           0x10
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC5__SHIFT                                                           0x14
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC6__SHIFT                                                           0x18
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC7__SHIFT                                                           0x1c
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC0_MASK                                                             0x0000000FL
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC1_MASK                                                             0x000000F0L
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC2_MASK                                                             0x00000F00L
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC3_MASK                                                             0x0000F000L
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC4_MASK                                                             0x000F0000L
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC5_MASK                                                             0x00F00000L
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC6_MASK                                                             0x0F000000L
+#define DAGB3_WR_OUTPUT_DAGB_LAZY_TIMER__VC7_MASK                                                             0xF0000000L
+//DAGB3_WR_CGTT_CLK_CTRL
+#define DAGB3_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                               0x0
+#define DAGB3_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                         0x4
+#define DAGB3_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                   0xc
+#define DAGB3_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                            0x1e
+#define DAGB3_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                          0x1f
+#define DAGB3_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                 0x0000000FL
+#define DAGB3_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                           0x00000FF0L
+#define DAGB3_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                     0x0FFFF000L
+#define DAGB3_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                              0x40000000L
+#define DAGB3_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                            0x80000000L
+//DAGB3_L1TLB_WR_CGTT_CLK_CTRL
+#define DAGB3_L1TLB_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB3_L1TLB_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB3_L1TLB_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB3_L1TLB_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB3_L1TLB_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB3_L1TLB_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB3_L1TLB_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB3_L1TLB_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB3_L1TLB_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB3_L1TLB_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB3_ATCVM_WR_CGTT_CLK_CTRL
+#define DAGB3_ATCVM_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB3_ATCVM_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB3_ATCVM_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB3_ATCVM_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB3_ATCVM_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB3_ATCVM_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB3_ATCVM_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB3_ATCVM_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB3_ATCVM_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB3_ATCVM_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB3_WR_ADDR_DAGB_MAX_BURST0
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB3_WR_ADDR_DAGB_LAZY_TIMER0
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB3_WR_ADDR_DAGB_MAX_BURST1
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB3_WR_ADDR_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB3_WR_ADDR_DAGB_LAZY_TIMER1
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB3_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB3_WR_DATA_DAGB
+#define DAGB3_WR_DATA_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB3_WR_DATA_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB3_WR_DATA_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB3_WR_DATA_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB3_WR_DATA_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB3_WR_DATA_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB3_WR_DATA_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB3_WR_DATA_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+//DAGB3_WR_DATA_DAGB_MAX_BURST0
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB3_WR_DATA_DAGB_LAZY_TIMER0
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB3_WR_DATA_DAGB_MAX_BURST1
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB3_WR_DATA_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB3_WR_DATA_DAGB_LAZY_TIMER1
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB3_WR_DATA_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB3_WR_VC0_CNTL
+#define DAGB3_WR_VC0_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_WR_VC0_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_WR_VC0_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_WR_VC0_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_WR_VC0_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_WR_VC0_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_WR_VC0_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_WR_VC0_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_WR_VC0_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_WR_VC0_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_WR_VC0_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_WR_VC0_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_WR_VC0_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_WR_VC0_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_WR_VC0_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_WR_VC0_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_WR_VC1_CNTL
+#define DAGB3_WR_VC1_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_WR_VC1_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_WR_VC1_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_WR_VC1_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_WR_VC1_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_WR_VC1_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_WR_VC1_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_WR_VC1_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_WR_VC1_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_WR_VC1_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_WR_VC1_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_WR_VC1_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_WR_VC1_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_WR_VC1_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_WR_VC1_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_WR_VC1_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_WR_VC2_CNTL
+#define DAGB3_WR_VC2_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_WR_VC2_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_WR_VC2_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_WR_VC2_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_WR_VC2_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_WR_VC2_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_WR_VC2_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_WR_VC2_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_WR_VC2_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_WR_VC2_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_WR_VC2_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_WR_VC2_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_WR_VC2_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_WR_VC2_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_WR_VC2_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_WR_VC2_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_WR_VC3_CNTL
+#define DAGB3_WR_VC3_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_WR_VC3_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_WR_VC3_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_WR_VC3_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_WR_VC3_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_WR_VC3_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_WR_VC3_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_WR_VC3_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_WR_VC3_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_WR_VC3_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_WR_VC3_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_WR_VC3_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_WR_VC3_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_WR_VC3_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_WR_VC3_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_WR_VC3_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_WR_VC4_CNTL
+#define DAGB3_WR_VC4_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_WR_VC4_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_WR_VC4_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_WR_VC4_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_WR_VC4_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_WR_VC4_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_WR_VC4_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_WR_VC4_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_WR_VC4_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_WR_VC4_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_WR_VC4_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_WR_VC4_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_WR_VC4_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_WR_VC4_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_WR_VC4_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_WR_VC4_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_WR_VC5_CNTL
+#define DAGB3_WR_VC5_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_WR_VC5_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_WR_VC5_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_WR_VC5_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_WR_VC5_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_WR_VC5_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_WR_VC5_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_WR_VC5_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_WR_VC5_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_WR_VC5_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_WR_VC5_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_WR_VC5_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_WR_VC5_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_WR_VC5_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_WR_VC5_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_WR_VC5_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_WR_VC6_CNTL
+#define DAGB3_WR_VC6_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_WR_VC6_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_WR_VC6_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_WR_VC6_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_WR_VC6_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_WR_VC6_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_WR_VC6_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_WR_VC6_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_WR_VC6_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_WR_VC6_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_WR_VC6_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_WR_VC6_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_WR_VC6_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_WR_VC6_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_WR_VC6_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_WR_VC6_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_WR_VC7_CNTL
+#define DAGB3_WR_VC7_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB3_WR_VC7_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB3_WR_VC7_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB3_WR_VC7_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB3_WR_VC7_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB3_WR_VC7_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB3_WR_VC7_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB3_WR_VC7_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB3_WR_VC7_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB3_WR_VC7_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB3_WR_VC7_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB3_WR_VC7_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB3_WR_VC7_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB3_WR_VC7_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB3_WR_VC7_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB3_WR_VC7_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB3_WR_CNTL_MISC
+#define DAGB3_WR_CNTL_MISC__STOR_POOL_CREDIT__SHIFT                                                           0x0
+#define DAGB3_WR_CNTL_MISC__EA_POOL_CREDIT__SHIFT                                                             0x6
+#define DAGB3_WR_CNTL_MISC__IO_EA_CREDIT__SHIFT                                                               0xd
+#define DAGB3_WR_CNTL_MISC__STOR_CC_NEW_MODE__SHIFT                                                           0x13
+#define DAGB3_WR_CNTL_MISC__EA_CC_LEGACY_MODE__SHIFT                                                          0x14
+#define DAGB3_WR_CNTL_MISC__UTCL2_CID__SHIFT                                                                  0x15
+#define DAGB3_WR_CNTL_MISC__RDRET_FIFO_CREDITS__SHIFT                                                         0x1a
+#define DAGB3_WR_CNTL_MISC__STOR_POOL_CREDIT_MASK                                                             0x0000003FL
+#define DAGB3_WR_CNTL_MISC__EA_POOL_CREDIT_MASK                                                               0x00001FC0L
+#define DAGB3_WR_CNTL_MISC__IO_EA_CREDIT_MASK                                                                 0x0007E000L
+#define DAGB3_WR_CNTL_MISC__STOR_CC_NEW_MODE_MASK                                                             0x00080000L
+#define DAGB3_WR_CNTL_MISC__EA_CC_LEGACY_MODE_MASK                                                            0x00100000L
+#define DAGB3_WR_CNTL_MISC__UTCL2_CID_MASK                                                                    0x03E00000L
+#define DAGB3_WR_CNTL_MISC__RDRET_FIFO_CREDITS_MASK                                                           0xFC000000L
+//DAGB3_WR_TLB_CREDIT
+#define DAGB3_WR_TLB_CREDIT__TLB0__SHIFT                                                                      0x0
+#define DAGB3_WR_TLB_CREDIT__TLB1__SHIFT                                                                      0x5
+#define DAGB3_WR_TLB_CREDIT__TLB2__SHIFT                                                                      0xa
+#define DAGB3_WR_TLB_CREDIT__TLB3__SHIFT                                                                      0xf
+#define DAGB3_WR_TLB_CREDIT__TLB4__SHIFT                                                                      0x14
+#define DAGB3_WR_TLB_CREDIT__TLB5__SHIFT                                                                      0x19
+#define DAGB3_WR_TLB_CREDIT__TLB0_MASK                                                                        0x0000001FL
+#define DAGB3_WR_TLB_CREDIT__TLB1_MASK                                                                        0x000003E0L
+#define DAGB3_WR_TLB_CREDIT__TLB2_MASK                                                                        0x00007C00L
+#define DAGB3_WR_TLB_CREDIT__TLB3_MASK                                                                        0x000F8000L
+#define DAGB3_WR_TLB_CREDIT__TLB4_MASK                                                                        0x01F00000L
+#define DAGB3_WR_TLB_CREDIT__TLB5_MASK                                                                        0x3E000000L
+//DAGB3_WR_DATA_CREDIT
+#define DAGB3_WR_DATA_CREDIT__DLOCK_VC_CREDITS__SHIFT                                                         0x0
+#define DAGB3_WR_DATA_CREDIT__LARGE_BURST_CREDITS__SHIFT                                                      0x8
+#define DAGB3_WR_DATA_CREDIT__MIDDLE_BURST_CREDITS__SHIFT                                                     0x10
+#define DAGB3_WR_DATA_CREDIT__SMALL_BURST_CREDITS__SHIFT                                                      0x18
+#define DAGB3_WR_DATA_CREDIT__DLOCK_VC_CREDITS_MASK                                                           0x000000FFL
+#define DAGB3_WR_DATA_CREDIT__LARGE_BURST_CREDITS_MASK                                                        0x0000FF00L
+#define DAGB3_WR_DATA_CREDIT__MIDDLE_BURST_CREDITS_MASK                                                       0x00FF0000L
+#define DAGB3_WR_DATA_CREDIT__SMALL_BURST_CREDITS_MASK                                                        0xFF000000L
+//DAGB3_WR_MISC_CREDIT
+#define DAGB3_WR_MISC_CREDIT__ATOMIC_CREDIT__SHIFT                                                            0x0
+#define DAGB3_WR_MISC_CREDIT__DLOCK_VC_NUM__SHIFT                                                             0x6
+#define DAGB3_WR_MISC_CREDIT__OSD_CREDIT__SHIFT                                                               0x9
+#define DAGB3_WR_MISC_CREDIT__OSD_DLOCK_CREDIT__SHIFT                                                         0x10
+#define DAGB3_WR_MISC_CREDIT__ATOMIC_CREDIT_MASK                                                              0x0000003FL
+#define DAGB3_WR_MISC_CREDIT__DLOCK_VC_NUM_MASK                                                               0x000001C0L
+#define DAGB3_WR_MISC_CREDIT__OSD_CREDIT_MASK                                                                 0x0000FE00L
+#define DAGB3_WR_MISC_CREDIT__OSD_DLOCK_CREDIT_MASK                                                           0x007F0000L
+//DAGB3_WR_OSD_CREDIT_CNTL1
+#define DAGB3_WR_OSD_CREDIT_CNTL1__VC0_CREDIT__SHIFT                                                          0x0
+#define DAGB3_WR_OSD_CREDIT_CNTL1__VC1_CREDIT__SHIFT                                                          0x4
+#define DAGB3_WR_OSD_CREDIT_CNTL1__VC2_CREDIT__SHIFT                                                          0x8
+#define DAGB3_WR_OSD_CREDIT_CNTL1__VC3_CREDIT__SHIFT                                                          0xc
+#define DAGB3_WR_OSD_CREDIT_CNTL1__IO_CREDIT__SHIFT                                                           0x10
+#define DAGB3_WR_OSD_CREDIT_CNTL1__GMI_CREDIT__SHIFT                                                          0x14
+#define DAGB3_WR_OSD_CREDIT_CNTL1__POOL_CREDIT__SHIFT                                                         0x18
+#define DAGB3_WR_OSD_CREDIT_CNTL1__VC0_CREDIT_MASK                                                            0x0000000FL
+#define DAGB3_WR_OSD_CREDIT_CNTL1__VC1_CREDIT_MASK                                                            0x000000F0L
+#define DAGB3_WR_OSD_CREDIT_CNTL1__VC2_CREDIT_MASK                                                            0x00000F00L
+#define DAGB3_WR_OSD_CREDIT_CNTL1__VC3_CREDIT_MASK                                                            0x0000F000L
+#define DAGB3_WR_OSD_CREDIT_CNTL1__IO_CREDIT_MASK                                                             0x000F0000L
+#define DAGB3_WR_OSD_CREDIT_CNTL1__GMI_CREDIT_MASK                                                            0x00F00000L
+#define DAGB3_WR_OSD_CREDIT_CNTL1__POOL_CREDIT_MASK                                                           0x3F000000L
+//DAGB3_WR_OSD_CREDIT_CNTL2
+#define DAGB3_WR_OSD_CREDIT_CNTL2__CREDIT_MARGIN__SHIFT                                                       0x0
+#define DAGB3_WR_OSD_CREDIT_CNTL2__ENABLE_LEGACY__SHIFT                                                       0x4
+#define DAGB3_WR_OSD_CREDIT_CNTL2__CREDIT_MARGIN_MASK                                                         0x0000000FL
+#define DAGB3_WR_OSD_CREDIT_CNTL2__ENABLE_LEGACY_MASK                                                         0x00000010L
+//DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC0_CREDIT__SHIFT                                                  0x0
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC1_CREDIT__SHIFT                                                  0x5
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC2_CREDIT__SHIFT                                                  0xa
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC3_CREDIT__SHIFT                                                  0xf
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__POOL_CREDIT__SHIFT                                                 0x14
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC_MODE__SHIFT                                                     0x19
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX_EQ__SHIFT                                                      0x1a
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX0__SHIFT                                                        0x1b
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX1__SHIFT                                                        0x1c
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX2__SHIFT                                                        0x1d
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC0_CREDIT_MASK                                                    0x0000001FL
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC1_CREDIT_MASK                                                    0x000003E0L
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC2_CREDIT_MASK                                                    0x00007C00L
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC3_CREDIT_MASK                                                    0x000F8000L
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__POOL_CREDIT_MASK                                                   0x01F00000L
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC_MODE_MASK                                                       0x02000000L
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX_EQ_MASK                                                        0x04000000L
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX0_MASK                                                          0x08000000L
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX1_MASK                                                          0x10000000L
+#define DAGB3_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX2_MASK                                                          0x20000000L
+//DAGB3_WRCLI_GPU_SNOOP_OVERRIDE
+#define DAGB3_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT                                                         0x0
+#define DAGB3_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK                                                           0x0000FFFFL
+//DAGB3_WRCLI_GPU_SNOOP_OVERRIDE_VALUE
+#define DAGB3_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT                                                   0x0
+#define DAGB3_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK                                                     0x0000FFFFL
+//DAGB3_WRCLI_ASK_PENDING
+#define DAGB3_WRCLI_ASK_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB3_WRCLI_ASK_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB3_WRCLI_GO_PENDING
+#define DAGB3_WRCLI_GO_PENDING__BUSY__SHIFT                                                                   0x0
+#define DAGB3_WRCLI_GO_PENDING__BUSY_MASK                                                                     0xFFFFFFFFL
+//DAGB3_WRCLI_GBLSEND_PENDING
+#define DAGB3_WRCLI_GBLSEND_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB3_WRCLI_GBLSEND_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB3_WRCLI_TLB_PENDING
+#define DAGB3_WRCLI_TLB_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB3_WRCLI_TLB_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB3_WRCLI_OARB_PENDING
+#define DAGB3_WRCLI_OARB_PENDING__BUSY__SHIFT                                                                 0x0
+#define DAGB3_WRCLI_OARB_PENDING__BUSY_MASK                                                                   0xFFFFFFFFL
+//DAGB3_WRCLI_OSD_PENDING
+#define DAGB3_WRCLI_OSD_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB3_WRCLI_OSD_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB3_WRCLI_DBUS_ASK_PENDING
+#define DAGB3_WRCLI_DBUS_ASK_PENDING__BUSY__SHIFT                                                             0x0
+#define DAGB3_WRCLI_DBUS_ASK_PENDING__BUSY_MASK                                                               0xFFFFFFFFL
+//DAGB3_WRCLI_DBUS_GO_PENDING
+#define DAGB3_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB3_WRCLI_DBUS_GO_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB3_DAGB_DLY
+#define DAGB3_DAGB_DLY__DLY__SHIFT                                                                            0x0
+#define DAGB3_DAGB_DLY__CLI__SHIFT                                                                            0x8
+#define DAGB3_DAGB_DLY__POS__SHIFT                                                                            0x10
+#define DAGB3_DAGB_DLY__DLY_MASK                                                                              0x000000FFL
+#define DAGB3_DAGB_DLY__CLI_MASK                                                                              0x0000FF00L
+#define DAGB3_DAGB_DLY__POS_MASK                                                                              0x000F0000L
+//DAGB3_CNTL_MISC
+#define DAGB3_CNTL_MISC__EA_VC0_REMAP__SHIFT                                                                  0x0
+#define DAGB3_CNTL_MISC__EA_VC1_REMAP__SHIFT                                                                  0x3
+#define DAGB3_CNTL_MISC__EA_VC2_REMAP__SHIFT                                                                  0x6
+#define DAGB3_CNTL_MISC__EA_VC3_REMAP__SHIFT                                                                  0x9
+#define DAGB3_CNTL_MISC__EA_VC4_REMAP__SHIFT                                                                  0xc
+#define DAGB3_CNTL_MISC__EA_VC5_REMAP__SHIFT                                                                  0xf
+#define DAGB3_CNTL_MISC__EA_VC6_REMAP__SHIFT                                                                  0x12
+#define DAGB3_CNTL_MISC__EA_VC7_REMAP__SHIFT                                                                  0x15
+#define DAGB3_CNTL_MISC__BW_INIT_CYCLE__SHIFT                                                                 0x18
+#define DAGB3_CNTL_MISC__BW_RW_GAP_CYCLE__SHIFT                                                               0x1e
+#define DAGB3_CNTL_MISC__EA_VC0_REMAP_MASK                                                                    0x00000007L
+#define DAGB3_CNTL_MISC__EA_VC1_REMAP_MASK                                                                    0x00000038L
+#define DAGB3_CNTL_MISC__EA_VC2_REMAP_MASK                                                                    0x000001C0L
+#define DAGB3_CNTL_MISC__EA_VC3_REMAP_MASK                                                                    0x00000E00L
+#define DAGB3_CNTL_MISC__EA_VC4_REMAP_MASK                                                                    0x00007000L
+#define DAGB3_CNTL_MISC__EA_VC5_REMAP_MASK                                                                    0x00038000L
+#define DAGB3_CNTL_MISC__EA_VC6_REMAP_MASK                                                                    0x001C0000L
+#define DAGB3_CNTL_MISC__EA_VC7_REMAP_MASK                                                                    0x00E00000L
+#define DAGB3_CNTL_MISC__BW_INIT_CYCLE_MASK                                                                   0x3F000000L
+#define DAGB3_CNTL_MISC__BW_RW_GAP_CYCLE_MASK                                                                 0xC0000000L
+//DAGB3_CNTL_MISC2
+#define DAGB3_CNTL_MISC2__URG_BOOST_ENABLE__SHIFT                                                             0x0
+#define DAGB3_CNTL_MISC2__URG_HALT_ENABLE__SHIFT                                                              0x1
+#define DAGB3_CNTL_MISC2__DISABLE_WRREQ_CG__SHIFT                                                             0x2
+#define DAGB3_CNTL_MISC2__DISABLE_WRRET_CG__SHIFT                                                             0x3
+#define DAGB3_CNTL_MISC2__DISABLE_RDREQ_CG__SHIFT                                                             0x4
+#define DAGB3_CNTL_MISC2__DISABLE_RDRET_CG__SHIFT                                                             0x5
+#define DAGB3_CNTL_MISC2__DISABLE_TLBWR_CG__SHIFT                                                             0x6
+#define DAGB3_CNTL_MISC2__DISABLE_TLBRD_CG__SHIFT                                                             0x7
+#define DAGB3_CNTL_MISC2__DISABLE_EAWRREQ_BUSY__SHIFT                                                         0x8
+#define DAGB3_CNTL_MISC2__DISABLE_EARDREQ_BUSY__SHIFT                                                         0x9
+#define DAGB3_CNTL_MISC2__SWAP_CTL__SHIFT                                                                     0xa
+#define DAGB3_CNTL_MISC2__RDRET_FIFO_PERF__SHIFT                                                              0xb
+#define DAGB3_CNTL_MISC2__HDP_CID__SHIFT                                                                      0xc
+#define DAGB3_CNTL_MISC2__RDRET_FIFO_DLOCK_CREDITS__SHIFT                                                     0x10
+#define DAGB3_CNTL_MISC2__URG_BOOST_ENABLE_MASK                                                               0x00000001L
+#define DAGB3_CNTL_MISC2__URG_HALT_ENABLE_MASK                                                                0x00000002L
+#define DAGB3_CNTL_MISC2__DISABLE_WRREQ_CG_MASK                                                               0x00000004L
+#define DAGB3_CNTL_MISC2__DISABLE_WRRET_CG_MASK                                                               0x00000008L
+#define DAGB3_CNTL_MISC2__DISABLE_RDREQ_CG_MASK                                                               0x00000010L
+#define DAGB3_CNTL_MISC2__DISABLE_RDRET_CG_MASK                                                               0x00000020L
+#define DAGB3_CNTL_MISC2__DISABLE_TLBWR_CG_MASK                                                               0x00000040L
+#define DAGB3_CNTL_MISC2__DISABLE_TLBRD_CG_MASK                                                               0x00000080L
+#define DAGB3_CNTL_MISC2__DISABLE_EAWRREQ_BUSY_MASK                                                           0x00000100L
+#define DAGB3_CNTL_MISC2__DISABLE_EARDREQ_BUSY_MASK                                                           0x00000200L
+#define DAGB3_CNTL_MISC2__SWAP_CTL_MASK                                                                       0x00000400L
+#define DAGB3_CNTL_MISC2__RDRET_FIFO_PERF_MASK                                                                0x00000800L
+#define DAGB3_CNTL_MISC2__HDP_CID_MASK                                                                        0x0000F000L
+#define DAGB3_CNTL_MISC2__RDRET_FIFO_DLOCK_CREDITS_MASK                                                       0x003F0000L
+//DAGB3_FATAL_ERROR_CNTL
+#define DAGB3_FATAL_ERROR_CNTL__FILTER_NUM__SHIFT                                                             0x0
+#define DAGB3_FATAL_ERROR_CNTL__FILTER_NUM_MASK                                                               0x000003FFL
+//DAGB3_FATAL_ERROR_CLEAR
+#define DAGB3_FATAL_ERROR_CLEAR__CLEAR__SHIFT                                                                 0x0
+#define DAGB3_FATAL_ERROR_CLEAR__CLEAR_MASK                                                                   0x00000001L
+//DAGB3_FATAL_ERROR_STATUS0
+#define DAGB3_FATAL_ERROR_STATUS0__VALID__SHIFT                                                               0x0
+#define DAGB3_FATAL_ERROR_STATUS0__CID__SHIFT                                                                 0x1
+#define DAGB3_FATAL_ERROR_STATUS0__ADDR_LO__SHIFT                                                             0x6
+#define DAGB3_FATAL_ERROR_STATUS0__VALID_MASK                                                                 0x00000001L
+#define DAGB3_FATAL_ERROR_STATUS0__CID_MASK                                                                   0x0000003EL
+#define DAGB3_FATAL_ERROR_STATUS0__ADDR_LO_MASK                                                               0xFFFFFFC0L
+//DAGB3_FATAL_ERROR_STATUS1
+#define DAGB3_FATAL_ERROR_STATUS1__ADDR_HI__SHIFT                                                             0x0
+#define DAGB3_FATAL_ERROR_STATUS1__ADDR_HI_MASK                                                               0x0001FFFFL
+//DAGB3_FATAL_ERROR_STATUS2
+#define DAGB3_FATAL_ERROR_STATUS2__TAG__SHIFT                                                                 0x0
+#define DAGB3_FATAL_ERROR_STATUS2__VFID__SHIFT                                                                0x10
+#define DAGB3_FATAL_ERROR_STATUS2__VF__SHIFT                                                                  0x14
+#define DAGB3_FATAL_ERROR_STATUS2__SPACE__SHIFT                                                               0x15
+#define DAGB3_FATAL_ERROR_STATUS2__IO__SHIFT                                                                  0x16
+#define DAGB3_FATAL_ERROR_STATUS2__SIZE__SHIFT                                                                0x17
+#define DAGB3_FATAL_ERROR_STATUS2__DBGMSK__SHIFT                                                              0x18
+#define DAGB3_FATAL_ERROR_STATUS2__FED__SHIFT                                                                 0x19
+#define DAGB3_FATAL_ERROR_STATUS2__TAG_MASK                                                                   0x0000FFFFL
+#define DAGB3_FATAL_ERROR_STATUS2__VFID_MASK                                                                  0x000F0000L
+#define DAGB3_FATAL_ERROR_STATUS2__VF_MASK                                                                    0x00100000L
+#define DAGB3_FATAL_ERROR_STATUS2__SPACE_MASK                                                                 0x00200000L
+#define DAGB3_FATAL_ERROR_STATUS2__IO_MASK                                                                    0x00400000L
+#define DAGB3_FATAL_ERROR_STATUS2__SIZE_MASK                                                                  0x00800000L
+#define DAGB3_FATAL_ERROR_STATUS2__DBGMSK_MASK                                                                0x01000000L
+#define DAGB3_FATAL_ERROR_STATUS2__FED_MASK                                                                   0x02000000L
+//DAGB3_FATAL_ERROR_STATUS3
+#define DAGB3_FATAL_ERROR_STATUS3__NOALLOC__SHIFT                                                             0x0
+#define DAGB3_FATAL_ERROR_STATUS3__UNITID__SHIFT                                                              0x1
+#define DAGB3_FATAL_ERROR_STATUS3__OP__SHIFT                                                                  0x7
+#define DAGB3_FATAL_ERROR_STATUS3__SECLEVEL__SHIFT                                                            0xe
+#define DAGB3_FATAL_ERROR_STATUS3__WRTMZ__SHIFT                                                               0x11
+#define DAGB3_FATAL_ERROR_STATUS3__RDTMZ__SHIFT                                                               0x12
+#define DAGB3_FATAL_ERROR_STATUS3__SNOOP__SHIFT                                                               0x13
+#define DAGB3_FATAL_ERROR_STATUS3__INVAL__SHIFT                                                               0x14
+#define DAGB3_FATAL_ERROR_STATUS3__NACK__SHIFT                                                                0x15
+#define DAGB3_FATAL_ERROR_STATUS3__RO__SHIFT                                                                  0x17
+#define DAGB3_FATAL_ERROR_STATUS3__MEMLOG__SHIFT                                                              0x18
+#define DAGB3_FATAL_ERROR_STATUS3__EOP__SHIFT                                                                 0x19
+#define DAGB3_FATAL_ERROR_STATUS3__NOALLOC_MASK                                                               0x00000001L
+#define DAGB3_FATAL_ERROR_STATUS3__UNITID_MASK                                                                0x0000007EL
+#define DAGB3_FATAL_ERROR_STATUS3__OP_MASK                                                                    0x00003F80L
+#define DAGB3_FATAL_ERROR_STATUS3__SECLEVEL_MASK                                                              0x0001C000L
+#define DAGB3_FATAL_ERROR_STATUS3__WRTMZ_MASK                                                                 0x00020000L
+#define DAGB3_FATAL_ERROR_STATUS3__RDTMZ_MASK                                                                 0x00040000L
+#define DAGB3_FATAL_ERROR_STATUS3__SNOOP_MASK                                                                 0x00080000L
+#define DAGB3_FATAL_ERROR_STATUS3__INVAL_MASK                                                                 0x00100000L
+#define DAGB3_FATAL_ERROR_STATUS3__NACK_MASK                                                                  0x00600000L
+#define DAGB3_FATAL_ERROR_STATUS3__RO_MASK                                                                    0x00800000L
+#define DAGB3_FATAL_ERROR_STATUS3__MEMLOG_MASK                                                                0x01000000L
+#define DAGB3_FATAL_ERROR_STATUS3__EOP_MASK                                                                   0x02000000L
+//DAGB3_FIFO_EMPTY
+#define DAGB3_FIFO_EMPTY__EMPTY__SHIFT                                                                        0x0
+#define DAGB3_FIFO_EMPTY__EMPTY_MASK                                                                          0x00FFFFFFL
+//DAGB3_FIFO_FULL
+#define DAGB3_FIFO_FULL__FULL__SHIFT                                                                          0x0
+#define DAGB3_FIFO_FULL__FULL_MASK                                                                            0x007FFFFFL
+//DAGB3_WR_CREDITS_FULL
+#define DAGB3_WR_CREDITS_FULL__FULL__SHIFT                                                                    0x0
+#define DAGB3_WR_CREDITS_FULL__FULL_MASK                                                                      0x1FFFFFFFL
+//DAGB3_RD_CREDITS_FULL
+#define DAGB3_RD_CREDITS_FULL__FULL__SHIFT                                                                    0x0
+#define DAGB3_RD_CREDITS_FULL__FULL_MASK                                                                      0x0003FFFFL
+//DAGB3_PERFCOUNTER_LO
+#define DAGB3_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define DAGB3_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//DAGB3_PERFCOUNTER_HI
+#define DAGB3_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define DAGB3_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define DAGB3_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define DAGB3_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+//DAGB3_PERFCOUNTER0_CFG
+#define DAGB3_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB3_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB3_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB3_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB3_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB3_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB3_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB3_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB3_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB3_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB3_PERFCOUNTER1_CFG
+#define DAGB3_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB3_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB3_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB3_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB3_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB3_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB3_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB3_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB3_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB3_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB3_PERFCOUNTER2_CFG
+#define DAGB3_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB3_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB3_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB3_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB3_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB3_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB3_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB3_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB3_PERFCOUNTER2_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB3_PERFCOUNTER2_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB3_PERFCOUNTER_RSLT_CNTL
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define DAGB3_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+//DAGB3_L1TLB_REG_RW
+#define DAGB3_L1TLB_REG_RW__REG_WRITE_L1TLB_CTRL__SHIFT                                                       0x0
+#define DAGB3_L1TLB_REG_RW__REG_READ_L1TLB_CTRL__SHIFT                                                        0x1
+#define DAGB3_L1TLB_REG_RW__VMID_EXCEP_INT_CTRL__SHIFT                                                        0x2
+#define DAGB3_L1TLB_REG_RW__WDAT_PARITY_CHECK__SHIFT                                                          0x4
+#define DAGB3_L1TLB_REG_RW__DISABLE_RDRET_CHECK__SHIFT                                                        0x5
+#define DAGB3_L1TLB_REG_RW__RESERVE__SHIFT                                                                    0x6
+#define DAGB3_L1TLB_REG_RW__REG_WRITE_L1TLB_CTRL_MASK                                                         0x00000001L
+#define DAGB3_L1TLB_REG_RW__REG_READ_L1TLB_CTRL_MASK                                                          0x00000002L
+#define DAGB3_L1TLB_REG_RW__VMID_EXCEP_INT_CTRL_MASK                                                          0x00000004L
+#define DAGB3_L1TLB_REG_RW__WDAT_PARITY_CHECK_MASK                                                            0x00000010L
+#define DAGB3_L1TLB_REG_RW__DISABLE_RDRET_CHECK_MASK                                                          0x00000020L
+#define DAGB3_L1TLB_REG_RW__RESERVE_MASK                                                                      0xFFFFFFC0L
+
+
+// addressBlock: aid_mmhub_dagb_dagbdec4
+//DAGB4_RDCLI0
+#define DAGB4_RDCLI0__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_RDCLI0__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_RDCLI0__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_RDCLI0__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_RDCLI0__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_RDCLI0__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_RDCLI0__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_RDCLI0__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_RDCLI0__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_RDCLI0__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_RDCLI0__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_RDCLI0__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_RDCLI0__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_RDCLI0__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_RDCLI0__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_RDCLI0__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_RDCLI0__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_RDCLI0__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_RDCLI0__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_RDCLI0__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_RDCLI1
+#define DAGB4_RDCLI1__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_RDCLI1__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_RDCLI1__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_RDCLI1__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_RDCLI1__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_RDCLI1__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_RDCLI1__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_RDCLI1__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_RDCLI1__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_RDCLI1__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_RDCLI1__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_RDCLI1__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_RDCLI1__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_RDCLI1__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_RDCLI1__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_RDCLI1__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_RDCLI1__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_RDCLI1__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_RDCLI1__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_RDCLI1__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_RDCLI2
+#define DAGB4_RDCLI2__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_RDCLI2__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_RDCLI2__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_RDCLI2__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_RDCLI2__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_RDCLI2__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_RDCLI2__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_RDCLI2__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_RDCLI2__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_RDCLI2__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_RDCLI2__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_RDCLI2__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_RDCLI2__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_RDCLI2__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_RDCLI2__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_RDCLI2__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_RDCLI2__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_RDCLI2__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_RDCLI2__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_RDCLI2__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_RDCLI3
+#define DAGB4_RDCLI3__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_RDCLI3__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_RDCLI3__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_RDCLI3__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_RDCLI3__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_RDCLI3__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_RDCLI3__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_RDCLI3__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_RDCLI3__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_RDCLI3__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_RDCLI3__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_RDCLI3__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_RDCLI3__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_RDCLI3__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_RDCLI3__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_RDCLI3__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_RDCLI3__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_RDCLI3__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_RDCLI3__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_RDCLI3__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_RDCLI4
+#define DAGB4_RDCLI4__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_RDCLI4__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_RDCLI4__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_RDCLI4__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_RDCLI4__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_RDCLI4__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_RDCLI4__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_RDCLI4__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_RDCLI4__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_RDCLI4__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_RDCLI4__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_RDCLI4__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_RDCLI4__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_RDCLI4__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_RDCLI4__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_RDCLI4__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_RDCLI4__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_RDCLI4__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_RDCLI4__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_RDCLI4__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_RDCLI5
+#define DAGB4_RDCLI5__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_RDCLI5__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_RDCLI5__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_RDCLI5__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_RDCLI5__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_RDCLI5__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_RDCLI5__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_RDCLI5__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_RDCLI5__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_RDCLI5__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_RDCLI5__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_RDCLI5__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_RDCLI5__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_RDCLI5__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_RDCLI5__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_RDCLI5__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_RDCLI5__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_RDCLI5__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_RDCLI5__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_RDCLI5__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_RDCLI6
+#define DAGB4_RDCLI6__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_RDCLI6__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_RDCLI6__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_RDCLI6__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_RDCLI6__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_RDCLI6__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_RDCLI6__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_RDCLI6__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_RDCLI6__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_RDCLI6__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_RDCLI6__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_RDCLI6__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_RDCLI6__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_RDCLI6__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_RDCLI6__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_RDCLI6__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_RDCLI6__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_RDCLI6__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_RDCLI6__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_RDCLI6__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_RDCLI7
+#define DAGB4_RDCLI7__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_RDCLI7__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_RDCLI7__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_RDCLI7__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_RDCLI7__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_RDCLI7__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_RDCLI7__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_RDCLI7__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_RDCLI7__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_RDCLI7__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_RDCLI7__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_RDCLI7__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_RDCLI7__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_RDCLI7__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_RDCLI7__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_RDCLI7__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_RDCLI7__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_RDCLI7__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_RDCLI7__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_RDCLI7__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_RDCLI8
+#define DAGB4_RDCLI8__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_RDCLI8__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_RDCLI8__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_RDCLI8__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_RDCLI8__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_RDCLI8__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_RDCLI8__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_RDCLI8__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_RDCLI8__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_RDCLI8__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_RDCLI8__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_RDCLI8__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_RDCLI8__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_RDCLI8__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_RDCLI8__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_RDCLI8__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_RDCLI8__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_RDCLI8__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_RDCLI8__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_RDCLI8__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_RDCLI9
+#define DAGB4_RDCLI9__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_RDCLI9__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_RDCLI9__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_RDCLI9__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_RDCLI9__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_RDCLI9__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_RDCLI9__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_RDCLI9__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_RDCLI9__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_RDCLI9__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_RDCLI9__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_RDCLI9__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_RDCLI9__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_RDCLI9__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_RDCLI9__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_RDCLI9__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_RDCLI9__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_RDCLI9__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_RDCLI9__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_RDCLI9__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_RDCLI10
+#define DAGB4_RDCLI10__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_RDCLI10__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_RDCLI10__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_RDCLI10__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_RDCLI10__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_RDCLI10__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_RDCLI10__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_RDCLI10__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_RDCLI10__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_RDCLI10__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_RDCLI10__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_RDCLI10__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_RDCLI10__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_RDCLI10__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_RDCLI10__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_RDCLI10__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_RDCLI10__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_RDCLI10__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_RDCLI10__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_RDCLI10__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_RDCLI11
+#define DAGB4_RDCLI11__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_RDCLI11__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_RDCLI11__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_RDCLI11__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_RDCLI11__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_RDCLI11__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_RDCLI11__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_RDCLI11__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_RDCLI11__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_RDCLI11__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_RDCLI11__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_RDCLI11__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_RDCLI11__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_RDCLI11__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_RDCLI11__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_RDCLI11__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_RDCLI11__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_RDCLI11__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_RDCLI11__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_RDCLI11__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_RDCLI12
+#define DAGB4_RDCLI12__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_RDCLI12__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_RDCLI12__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_RDCLI12__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_RDCLI12__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_RDCLI12__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_RDCLI12__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_RDCLI12__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_RDCLI12__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_RDCLI12__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_RDCLI12__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_RDCLI12__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_RDCLI12__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_RDCLI12__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_RDCLI12__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_RDCLI12__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_RDCLI12__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_RDCLI12__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_RDCLI12__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_RDCLI12__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_RDCLI13
+#define DAGB4_RDCLI13__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_RDCLI13__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_RDCLI13__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_RDCLI13__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_RDCLI13__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_RDCLI13__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_RDCLI13__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_RDCLI13__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_RDCLI13__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_RDCLI13__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_RDCLI13__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_RDCLI13__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_RDCLI13__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_RDCLI13__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_RDCLI13__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_RDCLI13__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_RDCLI13__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_RDCLI13__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_RDCLI13__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_RDCLI13__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_RDCLI14
+#define DAGB4_RDCLI14__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_RDCLI14__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_RDCLI14__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_RDCLI14__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_RDCLI14__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_RDCLI14__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_RDCLI14__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_RDCLI14__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_RDCLI14__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_RDCLI14__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_RDCLI14__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_RDCLI14__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_RDCLI14__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_RDCLI14__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_RDCLI14__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_RDCLI14__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_RDCLI14__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_RDCLI14__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_RDCLI14__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_RDCLI14__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_RDCLI15
+#define DAGB4_RDCLI15__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_RDCLI15__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_RDCLI15__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_RDCLI15__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_RDCLI15__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_RDCLI15__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_RDCLI15__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_RDCLI15__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_RDCLI15__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_RDCLI15__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_RDCLI15__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_RDCLI15__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_RDCLI15__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_RDCLI15__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_RDCLI15__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_RDCLI15__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_RDCLI15__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_RDCLI15__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_RDCLI15__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_RDCLI15__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_RD_CNTL
+#define DAGB4_RD_CNTL__SCLK_FREQ__SHIFT                                                                       0x0
+#define DAGB4_RD_CNTL__CLI_MAX_BW_WINDOW__SHIFT                                                               0x4
+#define DAGB4_RD_CNTL__VC_MAX_BW_WINDOW__SHIFT                                                                0xa
+#define DAGB4_RD_CNTL__IO_LEVEL_OVERRIDE_ENABLE__SHIFT                                                        0x10
+#define DAGB4_RD_CNTL__IO_LEVEL__SHIFT                                                                        0x11
+#define DAGB4_RD_CNTL__IO_LEVEL_COMPLY_VC__SHIFT                                                              0x14
+#define DAGB4_RD_CNTL__SHARE_VC_NUM__SHIFT                                                                    0x17
+#define DAGB4_RD_CNTL__FIX_JUMP__SHIFT                                                                        0x1a
+#define DAGB4_RD_CNTL__SCLK_FREQ_MASK                                                                         0x0000000FL
+#define DAGB4_RD_CNTL__CLI_MAX_BW_WINDOW_MASK                                                                 0x000003F0L
+#define DAGB4_RD_CNTL__VC_MAX_BW_WINDOW_MASK                                                                  0x0000FC00L
+#define DAGB4_RD_CNTL__IO_LEVEL_OVERRIDE_ENABLE_MASK                                                          0x00010000L
+#define DAGB4_RD_CNTL__IO_LEVEL_MASK                                                                          0x000E0000L
+#define DAGB4_RD_CNTL__IO_LEVEL_COMPLY_VC_MASK                                                                0x00700000L
+#define DAGB4_RD_CNTL__SHARE_VC_NUM_MASK                                                                      0x03800000L
+#define DAGB4_RD_CNTL__FIX_JUMP_MASK                                                                          0x04000000L
+//DAGB4_RD_GMI_CNTL
+#define DAGB4_RD_GMI_CNTL__EA_CREDIT__SHIFT                                                                   0x0
+#define DAGB4_RD_GMI_CNTL__LEVEL__SHIFT                                                                       0x6
+#define DAGB4_RD_GMI_CNTL__MAX_BURST__SHIFT                                                                   0x9
+#define DAGB4_RD_GMI_CNTL__LAZY_TIMER__SHIFT                                                                  0xd
+#define DAGB4_RD_GMI_CNTL__EA_CREDIT_MASK                                                                     0x0000003FL
+#define DAGB4_RD_GMI_CNTL__LEVEL_MASK                                                                         0x000001C0L
+#define DAGB4_RD_GMI_CNTL__MAX_BURST_MASK                                                                     0x00001E00L
+#define DAGB4_RD_GMI_CNTL__LAZY_TIMER_MASK                                                                    0x0001E000L
+//DAGB4_RD_ADDR_DAGB
+#define DAGB4_RD_ADDR_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB4_RD_ADDR_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB4_RD_ADDR_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB4_RD_ADDR_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB4_RD_ADDR_DAGB__JUMP_MODE__SHIFT                                                                  0xd
+#define DAGB4_RD_ADDR_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB4_RD_ADDR_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB4_RD_ADDR_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB4_RD_ADDR_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+#define DAGB4_RD_ADDR_DAGB__JUMP_MODE_MASK                                                                    0x00002000L
+//DAGB4_RD_OUTPUT_DAGB_MAX_BURST
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC0__SHIFT                                                            0x0
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC1__SHIFT                                                            0x4
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC2__SHIFT                                                            0x8
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC3__SHIFT                                                            0xc
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC4__SHIFT                                                            0x10
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC5__SHIFT                                                            0x14
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC6__SHIFT                                                            0x18
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC7__SHIFT                                                            0x1c
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC0_MASK                                                              0x0000000FL
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC1_MASK                                                              0x000000F0L
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC2_MASK                                                              0x00000F00L
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC3_MASK                                                              0x0000F000L
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC4_MASK                                                              0x000F0000L
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC5_MASK                                                              0x00F00000L
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC6_MASK                                                              0x0F000000L
+#define DAGB4_RD_OUTPUT_DAGB_MAX_BURST__VC7_MASK                                                              0xF0000000L
+//DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC0__SHIFT                                                           0x0
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC1__SHIFT                                                           0x4
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC2__SHIFT                                                           0x8
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC3__SHIFT                                                           0xc
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC4__SHIFT                                                           0x10
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC5__SHIFT                                                           0x14
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC6__SHIFT                                                           0x18
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC7__SHIFT                                                           0x1c
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC0_MASK                                                             0x0000000FL
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC1_MASK                                                             0x000000F0L
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC2_MASK                                                             0x00000F00L
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC3_MASK                                                             0x0000F000L
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC4_MASK                                                             0x000F0000L
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC5_MASK                                                             0x00F00000L
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC6_MASK                                                             0x0F000000L
+#define DAGB4_RD_OUTPUT_DAGB_LAZY_TIMER__VC7_MASK                                                             0xF0000000L
+//DAGB4_RD_CGTT_CLK_CTRL
+#define DAGB4_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                               0x0
+#define DAGB4_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                         0x4
+#define DAGB4_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                   0xc
+#define DAGB4_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                            0x1e
+#define DAGB4_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                          0x1f
+#define DAGB4_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                 0x0000000FL
+#define DAGB4_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                           0x00000FF0L
+#define DAGB4_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                     0x0FFFF000L
+#define DAGB4_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                              0x40000000L
+#define DAGB4_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                            0x80000000L
+//DAGB4_L1TLB_RD_CGTT_CLK_CTRL
+#define DAGB4_L1TLB_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB4_L1TLB_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB4_L1TLB_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB4_L1TLB_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB4_L1TLB_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB4_L1TLB_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB4_L1TLB_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB4_L1TLB_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB4_L1TLB_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB4_L1TLB_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB4_ATCVM_RD_CGTT_CLK_CTRL
+#define DAGB4_ATCVM_RD_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB4_ATCVM_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB4_ATCVM_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB4_ATCVM_RD_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB4_ATCVM_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB4_ATCVM_RD_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB4_ATCVM_RD_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB4_ATCVM_RD_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB4_ATCVM_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB4_ATCVM_RD_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB4_RD_ADDR_DAGB_MAX_BURST0
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB4_RD_ADDR_DAGB_LAZY_TIMER0
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB4_RD_ADDR_DAGB_MAX_BURST1
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB4_RD_ADDR_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB4_RD_ADDR_DAGB_LAZY_TIMER1
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB4_RD_ADDR_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB4_RD_VC0_CNTL
+#define DAGB4_RD_VC0_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_RD_VC0_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_RD_VC0_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_RD_VC0_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_RD_VC0_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_RD_VC0_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_RD_VC0_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_RD_VC0_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_RD_VC0_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_RD_VC0_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_RD_VC0_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_RD_VC0_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_RD_VC0_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_RD_VC0_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_RD_VC0_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_RD_VC0_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_RD_VC1_CNTL
+#define DAGB4_RD_VC1_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_RD_VC1_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_RD_VC1_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_RD_VC1_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_RD_VC1_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_RD_VC1_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_RD_VC1_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_RD_VC1_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_RD_VC1_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_RD_VC1_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_RD_VC1_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_RD_VC1_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_RD_VC1_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_RD_VC1_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_RD_VC1_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_RD_VC1_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_RD_VC2_CNTL
+#define DAGB4_RD_VC2_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_RD_VC2_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_RD_VC2_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_RD_VC2_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_RD_VC2_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_RD_VC2_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_RD_VC2_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_RD_VC2_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_RD_VC2_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_RD_VC2_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_RD_VC2_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_RD_VC2_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_RD_VC2_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_RD_VC2_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_RD_VC2_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_RD_VC2_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_RD_VC3_CNTL
+#define DAGB4_RD_VC3_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_RD_VC3_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_RD_VC3_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_RD_VC3_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_RD_VC3_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_RD_VC3_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_RD_VC3_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_RD_VC3_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_RD_VC3_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_RD_VC3_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_RD_VC3_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_RD_VC3_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_RD_VC3_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_RD_VC3_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_RD_VC3_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_RD_VC3_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_RD_VC4_CNTL
+#define DAGB4_RD_VC4_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_RD_VC4_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_RD_VC4_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_RD_VC4_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_RD_VC4_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_RD_VC4_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_RD_VC4_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_RD_VC4_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_RD_VC4_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_RD_VC4_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_RD_VC4_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_RD_VC4_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_RD_VC4_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_RD_VC4_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_RD_VC4_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_RD_VC4_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_RD_VC5_CNTL
+#define DAGB4_RD_VC5_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_RD_VC5_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_RD_VC5_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_RD_VC5_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_RD_VC5_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_RD_VC5_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_RD_VC5_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_RD_VC5_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_RD_VC5_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_RD_VC5_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_RD_VC5_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_RD_VC5_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_RD_VC5_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_RD_VC5_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_RD_VC5_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_RD_VC5_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_RD_VC6_CNTL
+#define DAGB4_RD_VC6_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_RD_VC6_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_RD_VC6_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_RD_VC6_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_RD_VC6_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_RD_VC6_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_RD_VC6_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_RD_VC6_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_RD_VC6_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_RD_VC6_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_RD_VC6_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_RD_VC6_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_RD_VC6_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_RD_VC6_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_RD_VC6_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_RD_VC6_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_RD_VC7_CNTL
+#define DAGB4_RD_VC7_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_RD_VC7_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_RD_VC7_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_RD_VC7_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_RD_VC7_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_RD_VC7_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_RD_VC7_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_RD_VC7_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_RD_VC7_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_RD_VC7_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_RD_VC7_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_RD_VC7_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_RD_VC7_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_RD_VC7_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_RD_VC7_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_RD_VC7_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_RD_CNTL_MISC
+#define DAGB4_RD_CNTL_MISC__STOR_POOL_CREDIT__SHIFT                                                           0x0
+#define DAGB4_RD_CNTL_MISC__EA_POOL_CREDIT__SHIFT                                                             0x6
+#define DAGB4_RD_CNTL_MISC__IO_EA_CREDIT__SHIFT                                                               0xd
+#define DAGB4_RD_CNTL_MISC__STOR_CC_NEW_MODE__SHIFT                                                           0x13
+#define DAGB4_RD_CNTL_MISC__EA_CC_LEGACY_MODE__SHIFT                                                          0x14
+#define DAGB4_RD_CNTL_MISC__UTCL2_CID__SHIFT                                                                  0x15
+#define DAGB4_RD_CNTL_MISC__RDRET_FIFO_CREDITS__SHIFT                                                         0x1a
+#define DAGB4_RD_CNTL_MISC__STOR_POOL_CREDIT_MASK                                                             0x0000003FL
+#define DAGB4_RD_CNTL_MISC__EA_POOL_CREDIT_MASK                                                               0x00001FC0L
+#define DAGB4_RD_CNTL_MISC__IO_EA_CREDIT_MASK                                                                 0x0007E000L
+#define DAGB4_RD_CNTL_MISC__STOR_CC_NEW_MODE_MASK                                                             0x00080000L
+#define DAGB4_RD_CNTL_MISC__EA_CC_LEGACY_MODE_MASK                                                            0x00100000L
+#define DAGB4_RD_CNTL_MISC__UTCL2_CID_MASK                                                                    0x03E00000L
+#define DAGB4_RD_CNTL_MISC__RDRET_FIFO_CREDITS_MASK                                                           0xFC000000L
+//DAGB4_RD_TLB_CREDIT
+#define DAGB4_RD_TLB_CREDIT__TLB0__SHIFT                                                                      0x0
+#define DAGB4_RD_TLB_CREDIT__TLB1__SHIFT                                                                      0x5
+#define DAGB4_RD_TLB_CREDIT__TLB2__SHIFT                                                                      0xa
+#define DAGB4_RD_TLB_CREDIT__TLB3__SHIFT                                                                      0xf
+#define DAGB4_RD_TLB_CREDIT__TLB4__SHIFT                                                                      0x14
+#define DAGB4_RD_TLB_CREDIT__TLB5__SHIFT                                                                      0x19
+#define DAGB4_RD_TLB_CREDIT__TLB0_MASK                                                                        0x0000001FL
+#define DAGB4_RD_TLB_CREDIT__TLB1_MASK                                                                        0x000003E0L
+#define DAGB4_RD_TLB_CREDIT__TLB2_MASK                                                                        0x00007C00L
+#define DAGB4_RD_TLB_CREDIT__TLB3_MASK                                                                        0x000F8000L
+#define DAGB4_RD_TLB_CREDIT__TLB4_MASK                                                                        0x01F00000L
+#define DAGB4_RD_TLB_CREDIT__TLB5_MASK                                                                        0x3E000000L
+//DAGB4_RD_RDRET_CREDIT_CNTL
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC0_CREDIT__SHIFT                                                         0x0
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC1_CREDIT__SHIFT                                                         0x6
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC2_CREDIT__SHIFT                                                         0xc
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC3_CREDIT__SHIFT                                                         0x12
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC4_CREDIT__SHIFT                                                         0x18
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC_MODE__SHIFT                                                            0x1e
+#define DAGB4_RD_RDRET_CREDIT_CNTL__FIX_EQ__SHIFT                                                             0x1f
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC0_CREDIT_MASK                                                           0x0000003FL
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC1_CREDIT_MASK                                                           0x00000FC0L
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC2_CREDIT_MASK                                                           0x0003F000L
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC3_CREDIT_MASK                                                           0x00FC0000L
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC4_CREDIT_MASK                                                           0x3F000000L
+#define DAGB4_RD_RDRET_CREDIT_CNTL__VC_MODE_MASK                                                              0x40000000L
+#define DAGB4_RD_RDRET_CREDIT_CNTL__FIX_EQ_MASK                                                               0x80000000L
+//DAGB4_RD_RDRET_CREDIT_CNTL2
+#define DAGB4_RD_RDRET_CREDIT_CNTL2__IO_CREDIT__SHIFT                                                         0x0
+#define DAGB4_RD_RDRET_CREDIT_CNTL2__GMI_CREDIT__SHIFT                                                        0x6
+#define DAGB4_RD_RDRET_CREDIT_CNTL2__POOL_CREDIT__SHIFT                                                       0xc
+#define DAGB4_RD_RDRET_CREDIT_CNTL2__IO_CREDIT_MASK                                                           0x0000003FL
+#define DAGB4_RD_RDRET_CREDIT_CNTL2__GMI_CREDIT_MASK                                                          0x00000FC0L
+#define DAGB4_RD_RDRET_CREDIT_CNTL2__POOL_CREDIT_MASK                                                         0x0007F000L
+//DAGB4_RDCLI_ASK_PENDING
+#define DAGB4_RDCLI_ASK_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB4_RDCLI_ASK_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB4_RDCLI_GO_PENDING
+#define DAGB4_RDCLI_GO_PENDING__BUSY__SHIFT                                                                   0x0
+#define DAGB4_RDCLI_GO_PENDING__BUSY_MASK                                                                     0xFFFFFFFFL
+//DAGB4_RDCLI_GBLSEND_PENDING
+#define DAGB4_RDCLI_GBLSEND_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB4_RDCLI_GBLSEND_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB4_RDCLI_TLB_PENDING
+#define DAGB4_RDCLI_TLB_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB4_RDCLI_TLB_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB4_RDCLI_OARB_PENDING
+#define DAGB4_RDCLI_OARB_PENDING__BUSY__SHIFT                                                                 0x0
+#define DAGB4_RDCLI_OARB_PENDING__BUSY_MASK                                                                   0xFFFFFFFFL
+//DAGB4_RDCLI_OSD_PENDING
+#define DAGB4_RDCLI_OSD_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB4_RDCLI_OSD_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB4_WRCLI0
+#define DAGB4_WRCLI0__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_WRCLI0__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_WRCLI0__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_WRCLI0__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_WRCLI0__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_WRCLI0__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_WRCLI0__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_WRCLI0__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_WRCLI0__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_WRCLI0__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_WRCLI0__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_WRCLI0__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_WRCLI0__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_WRCLI0__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_WRCLI0__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_WRCLI0__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_WRCLI0__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_WRCLI0__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_WRCLI0__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_WRCLI0__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_WRCLI1
+#define DAGB4_WRCLI1__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_WRCLI1__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_WRCLI1__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_WRCLI1__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_WRCLI1__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_WRCLI1__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_WRCLI1__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_WRCLI1__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_WRCLI1__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_WRCLI1__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_WRCLI1__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_WRCLI1__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_WRCLI1__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_WRCLI1__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_WRCLI1__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_WRCLI1__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_WRCLI1__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_WRCLI1__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_WRCLI1__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_WRCLI1__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_WRCLI2
+#define DAGB4_WRCLI2__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_WRCLI2__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_WRCLI2__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_WRCLI2__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_WRCLI2__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_WRCLI2__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_WRCLI2__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_WRCLI2__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_WRCLI2__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_WRCLI2__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_WRCLI2__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_WRCLI2__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_WRCLI2__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_WRCLI2__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_WRCLI2__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_WRCLI2__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_WRCLI2__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_WRCLI2__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_WRCLI2__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_WRCLI2__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_WRCLI3
+#define DAGB4_WRCLI3__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_WRCLI3__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_WRCLI3__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_WRCLI3__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_WRCLI3__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_WRCLI3__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_WRCLI3__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_WRCLI3__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_WRCLI3__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_WRCLI3__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_WRCLI3__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_WRCLI3__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_WRCLI3__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_WRCLI3__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_WRCLI3__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_WRCLI3__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_WRCLI3__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_WRCLI3__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_WRCLI3__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_WRCLI3__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_WRCLI4
+#define DAGB4_WRCLI4__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_WRCLI4__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_WRCLI4__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_WRCLI4__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_WRCLI4__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_WRCLI4__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_WRCLI4__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_WRCLI4__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_WRCLI4__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_WRCLI4__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_WRCLI4__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_WRCLI4__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_WRCLI4__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_WRCLI4__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_WRCLI4__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_WRCLI4__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_WRCLI4__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_WRCLI4__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_WRCLI4__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_WRCLI4__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_WRCLI5
+#define DAGB4_WRCLI5__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_WRCLI5__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_WRCLI5__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_WRCLI5__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_WRCLI5__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_WRCLI5__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_WRCLI5__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_WRCLI5__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_WRCLI5__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_WRCLI5__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_WRCLI5__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_WRCLI5__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_WRCLI5__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_WRCLI5__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_WRCLI5__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_WRCLI5__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_WRCLI5__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_WRCLI5__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_WRCLI5__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_WRCLI5__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_WRCLI6
+#define DAGB4_WRCLI6__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_WRCLI6__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_WRCLI6__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_WRCLI6__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_WRCLI6__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_WRCLI6__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_WRCLI6__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_WRCLI6__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_WRCLI6__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_WRCLI6__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_WRCLI6__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_WRCLI6__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_WRCLI6__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_WRCLI6__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_WRCLI6__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_WRCLI6__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_WRCLI6__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_WRCLI6__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_WRCLI6__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_WRCLI6__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_WRCLI7
+#define DAGB4_WRCLI7__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_WRCLI7__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_WRCLI7__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_WRCLI7__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_WRCLI7__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_WRCLI7__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_WRCLI7__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_WRCLI7__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_WRCLI7__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_WRCLI7__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_WRCLI7__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_WRCLI7__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_WRCLI7__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_WRCLI7__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_WRCLI7__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_WRCLI7__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_WRCLI7__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_WRCLI7__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_WRCLI7__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_WRCLI7__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_WRCLI8
+#define DAGB4_WRCLI8__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_WRCLI8__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_WRCLI8__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_WRCLI8__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_WRCLI8__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_WRCLI8__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_WRCLI8__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_WRCLI8__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_WRCLI8__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_WRCLI8__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_WRCLI8__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_WRCLI8__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_WRCLI8__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_WRCLI8__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_WRCLI8__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_WRCLI8__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_WRCLI8__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_WRCLI8__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_WRCLI8__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_WRCLI8__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_WRCLI9
+#define DAGB4_WRCLI9__VIRT_CHAN__SHIFT                                                                        0x0
+#define DAGB4_WRCLI9__CHECK_TLB_CREDIT__SHIFT                                                                 0x3
+#define DAGB4_WRCLI9__URG_HIGH__SHIFT                                                                         0x4
+#define DAGB4_WRCLI9__URG_LOW__SHIFT                                                                          0x8
+#define DAGB4_WRCLI9__MAX_BW_ENABLE__SHIFT                                                                    0xc
+#define DAGB4_WRCLI9__MAX_BW__SHIFT                                                                           0xd
+#define DAGB4_WRCLI9__MIN_BW_ENABLE__SHIFT                                                                    0x15
+#define DAGB4_WRCLI9__MIN_BW__SHIFT                                                                           0x16
+#define DAGB4_WRCLI9__OSD_LIMITER_ENABLE__SHIFT                                                               0x19
+#define DAGB4_WRCLI9__MAX_OSD__SHIFT                                                                          0x1a
+#define DAGB4_WRCLI9__VIRT_CHAN_MASK                                                                          0x00000007L
+#define DAGB4_WRCLI9__CHECK_TLB_CREDIT_MASK                                                                   0x00000008L
+#define DAGB4_WRCLI9__URG_HIGH_MASK                                                                           0x000000F0L
+#define DAGB4_WRCLI9__URG_LOW_MASK                                                                            0x00000F00L
+#define DAGB4_WRCLI9__MAX_BW_ENABLE_MASK                                                                      0x00001000L
+#define DAGB4_WRCLI9__MAX_BW_MASK                                                                             0x001FE000L
+#define DAGB4_WRCLI9__MIN_BW_ENABLE_MASK                                                                      0x00200000L
+#define DAGB4_WRCLI9__MIN_BW_MASK                                                                             0x01C00000L
+#define DAGB4_WRCLI9__OSD_LIMITER_ENABLE_MASK                                                                 0x02000000L
+#define DAGB4_WRCLI9__MAX_OSD_MASK                                                                            0xFC000000L
+//DAGB4_WRCLI10
+#define DAGB4_WRCLI10__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_WRCLI10__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_WRCLI10__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_WRCLI10__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_WRCLI10__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_WRCLI10__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_WRCLI10__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_WRCLI10__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_WRCLI10__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_WRCLI10__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_WRCLI10__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_WRCLI10__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_WRCLI10__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_WRCLI10__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_WRCLI10__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_WRCLI10__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_WRCLI10__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_WRCLI10__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_WRCLI10__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_WRCLI10__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_WRCLI11
+#define DAGB4_WRCLI11__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_WRCLI11__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_WRCLI11__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_WRCLI11__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_WRCLI11__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_WRCLI11__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_WRCLI11__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_WRCLI11__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_WRCLI11__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_WRCLI11__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_WRCLI11__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_WRCLI11__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_WRCLI11__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_WRCLI11__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_WRCLI11__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_WRCLI11__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_WRCLI11__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_WRCLI11__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_WRCLI11__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_WRCLI11__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_WRCLI12
+#define DAGB4_WRCLI12__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_WRCLI12__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_WRCLI12__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_WRCLI12__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_WRCLI12__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_WRCLI12__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_WRCLI12__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_WRCLI12__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_WRCLI12__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_WRCLI12__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_WRCLI12__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_WRCLI12__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_WRCLI12__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_WRCLI12__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_WRCLI12__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_WRCLI12__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_WRCLI12__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_WRCLI12__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_WRCLI12__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_WRCLI12__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_WRCLI13
+#define DAGB4_WRCLI13__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_WRCLI13__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_WRCLI13__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_WRCLI13__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_WRCLI13__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_WRCLI13__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_WRCLI13__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_WRCLI13__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_WRCLI13__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_WRCLI13__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_WRCLI13__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_WRCLI13__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_WRCLI13__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_WRCLI13__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_WRCLI13__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_WRCLI13__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_WRCLI13__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_WRCLI13__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_WRCLI13__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_WRCLI13__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_WRCLI14
+#define DAGB4_WRCLI14__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_WRCLI14__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_WRCLI14__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_WRCLI14__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_WRCLI14__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_WRCLI14__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_WRCLI14__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_WRCLI14__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_WRCLI14__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_WRCLI14__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_WRCLI14__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_WRCLI14__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_WRCLI14__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_WRCLI14__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_WRCLI14__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_WRCLI14__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_WRCLI14__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_WRCLI14__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_WRCLI14__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_WRCLI14__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_WRCLI15
+#define DAGB4_WRCLI15__VIRT_CHAN__SHIFT                                                                       0x0
+#define DAGB4_WRCLI15__CHECK_TLB_CREDIT__SHIFT                                                                0x3
+#define DAGB4_WRCLI15__URG_HIGH__SHIFT                                                                        0x4
+#define DAGB4_WRCLI15__URG_LOW__SHIFT                                                                         0x8
+#define DAGB4_WRCLI15__MAX_BW_ENABLE__SHIFT                                                                   0xc
+#define DAGB4_WRCLI15__MAX_BW__SHIFT                                                                          0xd
+#define DAGB4_WRCLI15__MIN_BW_ENABLE__SHIFT                                                                   0x15
+#define DAGB4_WRCLI15__MIN_BW__SHIFT                                                                          0x16
+#define DAGB4_WRCLI15__OSD_LIMITER_ENABLE__SHIFT                                                              0x19
+#define DAGB4_WRCLI15__MAX_OSD__SHIFT                                                                         0x1a
+#define DAGB4_WRCLI15__VIRT_CHAN_MASK                                                                         0x00000007L
+#define DAGB4_WRCLI15__CHECK_TLB_CREDIT_MASK                                                                  0x00000008L
+#define DAGB4_WRCLI15__URG_HIGH_MASK                                                                          0x000000F0L
+#define DAGB4_WRCLI15__URG_LOW_MASK                                                                           0x00000F00L
+#define DAGB4_WRCLI15__MAX_BW_ENABLE_MASK                                                                     0x00001000L
+#define DAGB4_WRCLI15__MAX_BW_MASK                                                                            0x001FE000L
+#define DAGB4_WRCLI15__MIN_BW_ENABLE_MASK                                                                     0x00200000L
+#define DAGB4_WRCLI15__MIN_BW_MASK                                                                            0x01C00000L
+#define DAGB4_WRCLI15__OSD_LIMITER_ENABLE_MASK                                                                0x02000000L
+#define DAGB4_WRCLI15__MAX_OSD_MASK                                                                           0xFC000000L
+//DAGB4_WR_CNTL
+#define DAGB4_WR_CNTL__SCLK_FREQ__SHIFT                                                                       0x0
+#define DAGB4_WR_CNTL__CLI_MAX_BW_WINDOW__SHIFT                                                               0x4
+#define DAGB4_WR_CNTL__VC_MAX_BW_WINDOW__SHIFT                                                                0xa
+#define DAGB4_WR_CNTL__IO_LEVEL_OVERRIDE_ENABLE__SHIFT                                                        0x10
+#define DAGB4_WR_CNTL__IO_LEVEL__SHIFT                                                                        0x11
+#define DAGB4_WR_CNTL__IO_LEVEL_COMPLY_VC__SHIFT                                                              0x14
+#define DAGB4_WR_CNTL__SHARE_VC_NUM__SHIFT                                                                    0x17
+#define DAGB4_WR_CNTL__FIX_JUMP__SHIFT                                                                        0x1a
+#define DAGB4_WR_CNTL__SCLK_FREQ_MASK                                                                         0x0000000FL
+#define DAGB4_WR_CNTL__CLI_MAX_BW_WINDOW_MASK                                                                 0x000003F0L
+#define DAGB4_WR_CNTL__VC_MAX_BW_WINDOW_MASK                                                                  0x0000FC00L
+#define DAGB4_WR_CNTL__IO_LEVEL_OVERRIDE_ENABLE_MASK                                                          0x00010000L
+#define DAGB4_WR_CNTL__IO_LEVEL_MASK                                                                          0x000E0000L
+#define DAGB4_WR_CNTL__IO_LEVEL_COMPLY_VC_MASK                                                                0x00700000L
+#define DAGB4_WR_CNTL__SHARE_VC_NUM_MASK                                                                      0x03800000L
+#define DAGB4_WR_CNTL__FIX_JUMP_MASK                                                                          0x04000000L
+//DAGB4_WR_GMI_CNTL
+#define DAGB4_WR_GMI_CNTL__EA_CREDIT__SHIFT                                                                   0x0
+#define DAGB4_WR_GMI_CNTL__LEVEL__SHIFT                                                                       0x6
+#define DAGB4_WR_GMI_CNTL__MAX_BURST__SHIFT                                                                   0x9
+#define DAGB4_WR_GMI_CNTL__LAZY_TIMER__SHIFT                                                                  0xd
+#define DAGB4_WR_GMI_CNTL__EA_CREDIT_MASK                                                                     0x0000003FL
+#define DAGB4_WR_GMI_CNTL__LEVEL_MASK                                                                         0x000001C0L
+#define DAGB4_WR_GMI_CNTL__MAX_BURST_MASK                                                                     0x00001E00L
+#define DAGB4_WR_GMI_CNTL__LAZY_TIMER_MASK                                                                    0x0001E000L
+//DAGB4_WR_ADDR_DAGB
+#define DAGB4_WR_ADDR_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB4_WR_ADDR_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB4_WR_ADDR_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB4_WR_ADDR_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB4_WR_ADDR_DAGB__JUMP_MODE__SHIFT                                                                  0xd
+#define DAGB4_WR_ADDR_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB4_WR_ADDR_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB4_WR_ADDR_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB4_WR_ADDR_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+#define DAGB4_WR_ADDR_DAGB__JUMP_MODE_MASK                                                                    0x00002000L
+//DAGB4_WR_OUTPUT_DAGB_MAX_BURST
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC0__SHIFT                                                            0x0
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC1__SHIFT                                                            0x4
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC2__SHIFT                                                            0x8
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC3__SHIFT                                                            0xc
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC4__SHIFT                                                            0x10
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC5__SHIFT                                                            0x14
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC6__SHIFT                                                            0x18
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC7__SHIFT                                                            0x1c
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC0_MASK                                                              0x0000000FL
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC1_MASK                                                              0x000000F0L
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC2_MASK                                                              0x00000F00L
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC3_MASK                                                              0x0000F000L
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC4_MASK                                                              0x000F0000L
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC5_MASK                                                              0x00F00000L
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC6_MASK                                                              0x0F000000L
+#define DAGB4_WR_OUTPUT_DAGB_MAX_BURST__VC7_MASK                                                              0xF0000000L
+//DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC0__SHIFT                                                           0x0
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC1__SHIFT                                                           0x4
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC2__SHIFT                                                           0x8
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC3__SHIFT                                                           0xc
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC4__SHIFT                                                           0x10
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC5__SHIFT                                                           0x14
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC6__SHIFT                                                           0x18
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC7__SHIFT                                                           0x1c
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC0_MASK                                                             0x0000000FL
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC1_MASK                                                             0x000000F0L
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC2_MASK                                                             0x00000F00L
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC3_MASK                                                             0x0000F000L
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC4_MASK                                                             0x000F0000L
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC5_MASK                                                             0x00F00000L
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC6_MASK                                                             0x0F000000L
+#define DAGB4_WR_OUTPUT_DAGB_LAZY_TIMER__VC7_MASK                                                             0xF0000000L
+//DAGB4_WR_CGTT_CLK_CTRL
+#define DAGB4_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                               0x0
+#define DAGB4_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                         0x4
+#define DAGB4_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                   0xc
+#define DAGB4_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                            0x1e
+#define DAGB4_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                          0x1f
+#define DAGB4_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                 0x0000000FL
+#define DAGB4_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                           0x00000FF0L
+#define DAGB4_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                     0x0FFFF000L
+#define DAGB4_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                              0x40000000L
+#define DAGB4_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                            0x80000000L
+//DAGB4_L1TLB_WR_CGTT_CLK_CTRL
+#define DAGB4_L1TLB_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB4_L1TLB_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB4_L1TLB_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB4_L1TLB_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB4_L1TLB_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB4_L1TLB_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB4_L1TLB_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB4_L1TLB_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB4_L1TLB_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB4_L1TLB_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB4_ATCVM_WR_CGTT_CLK_CTRL
+#define DAGB4_ATCVM_WR_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                         0x0
+#define DAGB4_ATCVM_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                   0x4
+#define DAGB4_ATCVM_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                             0xc
+#define DAGB4_ATCVM_WR_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                      0x1e
+#define DAGB4_ATCVM_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                    0x1f
+#define DAGB4_ATCVM_WR_CGTT_CLK_CTRL__ON_DELAY_MASK                                                           0x0000000FL
+#define DAGB4_ATCVM_WR_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                     0x00000FF0L
+#define DAGB4_ATCVM_WR_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                               0x0FFFF000L
+#define DAGB4_ATCVM_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                        0x40000000L
+#define DAGB4_ATCVM_WR_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                      0x80000000L
+//DAGB4_WR_ADDR_DAGB_MAX_BURST0
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB4_WR_ADDR_DAGB_LAZY_TIMER0
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB4_WR_ADDR_DAGB_MAX_BURST1
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB4_WR_ADDR_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB4_WR_ADDR_DAGB_LAZY_TIMER1
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB4_WR_ADDR_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB4_WR_DATA_DAGB
+#define DAGB4_WR_DATA_DAGB__DAGB_ENABLE__SHIFT                                                                0x0
+#define DAGB4_WR_DATA_DAGB__ENABLE_JUMP_AHEAD__SHIFT                                                          0x3
+#define DAGB4_WR_DATA_DAGB__DISABLE_SELF_INIT__SHIFT                                                          0x6
+#define DAGB4_WR_DATA_DAGB__WHOAMI__SHIFT                                                                     0x7
+#define DAGB4_WR_DATA_DAGB__DAGB_ENABLE_MASK                                                                  0x00000007L
+#define DAGB4_WR_DATA_DAGB__ENABLE_JUMP_AHEAD_MASK                                                            0x00000038L
+#define DAGB4_WR_DATA_DAGB__DISABLE_SELF_INIT_MASK                                                            0x00000040L
+#define DAGB4_WR_DATA_DAGB__WHOAMI_MASK                                                                       0x00001F80L
+//DAGB4_WR_DATA_DAGB_MAX_BURST0
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT0__SHIFT                                                         0x0
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT1__SHIFT                                                         0x4
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT2__SHIFT                                                         0x8
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT3__SHIFT                                                         0xc
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT4__SHIFT                                                         0x10
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT5__SHIFT                                                         0x14
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT6__SHIFT                                                         0x18
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT7__SHIFT                                                         0x1c
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT0_MASK                                                           0x0000000FL
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT1_MASK                                                           0x000000F0L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT2_MASK                                                           0x00000F00L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT3_MASK                                                           0x0000F000L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT4_MASK                                                           0x000F0000L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT5_MASK                                                           0x00F00000L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT6_MASK                                                           0x0F000000L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST0__CLIENT7_MASK                                                           0xF0000000L
+//DAGB4_WR_DATA_DAGB_LAZY_TIMER0
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT0__SHIFT                                                        0x0
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT1__SHIFT                                                        0x4
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT2__SHIFT                                                        0x8
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT3__SHIFT                                                        0xc
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT4__SHIFT                                                        0x10
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT5__SHIFT                                                        0x14
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT6__SHIFT                                                        0x18
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT7__SHIFT                                                        0x1c
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT0_MASK                                                          0x0000000FL
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT1_MASK                                                          0x000000F0L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT2_MASK                                                          0x00000F00L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT3_MASK                                                          0x0000F000L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT4_MASK                                                          0x000F0000L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT5_MASK                                                          0x00F00000L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT6_MASK                                                          0x0F000000L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER0__CLIENT7_MASK                                                          0xF0000000L
+//DAGB4_WR_DATA_DAGB_MAX_BURST1
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT8__SHIFT                                                         0x0
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT9__SHIFT                                                         0x4
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT10__SHIFT                                                        0x8
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT11__SHIFT                                                        0xc
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT12__SHIFT                                                        0x10
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT13__SHIFT                                                        0x14
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT14__SHIFT                                                        0x18
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT15__SHIFT                                                        0x1c
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT8_MASK                                                           0x0000000FL
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT9_MASK                                                           0x000000F0L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT10_MASK                                                          0x00000F00L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT11_MASK                                                          0x0000F000L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT12_MASK                                                          0x000F0000L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT13_MASK                                                          0x00F00000L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT14_MASK                                                          0x0F000000L
+#define DAGB4_WR_DATA_DAGB_MAX_BURST1__CLIENT15_MASK                                                          0xF0000000L
+//DAGB4_WR_DATA_DAGB_LAZY_TIMER1
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT8__SHIFT                                                        0x0
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT9__SHIFT                                                        0x4
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT10__SHIFT                                                       0x8
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT11__SHIFT                                                       0xc
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT12__SHIFT                                                       0x10
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT13__SHIFT                                                       0x14
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT14__SHIFT                                                       0x18
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT15__SHIFT                                                       0x1c
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT8_MASK                                                          0x0000000FL
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT9_MASK                                                          0x000000F0L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT10_MASK                                                         0x00000F00L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT11_MASK                                                         0x0000F000L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT12_MASK                                                         0x000F0000L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT13_MASK                                                         0x00F00000L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT14_MASK                                                         0x0F000000L
+#define DAGB4_WR_DATA_DAGB_LAZY_TIMER1__CLIENT15_MASK                                                         0xF0000000L
+//DAGB4_WR_VC0_CNTL
+#define DAGB4_WR_VC0_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_WR_VC0_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_WR_VC0_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_WR_VC0_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_WR_VC0_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_WR_VC0_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_WR_VC0_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_WR_VC0_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_WR_VC0_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_WR_VC0_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_WR_VC0_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_WR_VC0_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_WR_VC0_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_WR_VC0_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_WR_VC0_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_WR_VC0_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_WR_VC1_CNTL
+#define DAGB4_WR_VC1_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_WR_VC1_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_WR_VC1_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_WR_VC1_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_WR_VC1_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_WR_VC1_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_WR_VC1_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_WR_VC1_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_WR_VC1_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_WR_VC1_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_WR_VC1_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_WR_VC1_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_WR_VC1_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_WR_VC1_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_WR_VC1_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_WR_VC1_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_WR_VC2_CNTL
+#define DAGB4_WR_VC2_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_WR_VC2_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_WR_VC2_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_WR_VC2_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_WR_VC2_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_WR_VC2_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_WR_VC2_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_WR_VC2_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_WR_VC2_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_WR_VC2_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_WR_VC2_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_WR_VC2_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_WR_VC2_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_WR_VC2_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_WR_VC2_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_WR_VC2_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_WR_VC3_CNTL
+#define DAGB4_WR_VC3_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_WR_VC3_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_WR_VC3_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_WR_VC3_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_WR_VC3_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_WR_VC3_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_WR_VC3_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_WR_VC3_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_WR_VC3_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_WR_VC3_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_WR_VC3_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_WR_VC3_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_WR_VC3_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_WR_VC3_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_WR_VC3_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_WR_VC3_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_WR_VC4_CNTL
+#define DAGB4_WR_VC4_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_WR_VC4_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_WR_VC4_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_WR_VC4_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_WR_VC4_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_WR_VC4_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_WR_VC4_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_WR_VC4_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_WR_VC4_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_WR_VC4_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_WR_VC4_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_WR_VC4_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_WR_VC4_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_WR_VC4_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_WR_VC4_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_WR_VC4_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_WR_VC5_CNTL
+#define DAGB4_WR_VC5_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_WR_VC5_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_WR_VC5_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_WR_VC5_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_WR_VC5_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_WR_VC5_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_WR_VC5_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_WR_VC5_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_WR_VC5_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_WR_VC5_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_WR_VC5_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_WR_VC5_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_WR_VC5_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_WR_VC5_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_WR_VC5_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_WR_VC5_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_WR_VC6_CNTL
+#define DAGB4_WR_VC6_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_WR_VC6_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_WR_VC6_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_WR_VC6_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_WR_VC6_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_WR_VC6_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_WR_VC6_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_WR_VC6_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_WR_VC6_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_WR_VC6_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_WR_VC6_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_WR_VC6_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_WR_VC6_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_WR_VC6_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_WR_VC6_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_WR_VC6_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_WR_VC7_CNTL
+#define DAGB4_WR_VC7_CNTL__STOR_CREDIT__SHIFT                                                                 0x0
+#define DAGB4_WR_VC7_CNTL__EA_CREDIT__SHIFT                                                                   0x5
+#define DAGB4_WR_VC7_CNTL__MAX_BW_ENABLE__SHIFT                                                               0xb
+#define DAGB4_WR_VC7_CNTL__MAX_BW__SHIFT                                                                      0xc
+#define DAGB4_WR_VC7_CNTL__MIN_BW_ENABLE__SHIFT                                                               0x14
+#define DAGB4_WR_VC7_CNTL__MIN_BW__SHIFT                                                                      0x15
+#define DAGB4_WR_VC7_CNTL__OSD_LIMITER_ENABLE__SHIFT                                                          0x18
+#define DAGB4_WR_VC7_CNTL__MAX_OSD__SHIFT                                                                     0x19
+#define DAGB4_WR_VC7_CNTL__STOR_CREDIT_MASK                                                                   0x0000001FL
+#define DAGB4_WR_VC7_CNTL__EA_CREDIT_MASK                                                                     0x000007E0L
+#define DAGB4_WR_VC7_CNTL__MAX_BW_ENABLE_MASK                                                                 0x00000800L
+#define DAGB4_WR_VC7_CNTL__MAX_BW_MASK                                                                        0x000FF000L
+#define DAGB4_WR_VC7_CNTL__MIN_BW_ENABLE_MASK                                                                 0x00100000L
+#define DAGB4_WR_VC7_CNTL__MIN_BW_MASK                                                                        0x00E00000L
+#define DAGB4_WR_VC7_CNTL__OSD_LIMITER_ENABLE_MASK                                                            0x01000000L
+#define DAGB4_WR_VC7_CNTL__MAX_OSD_MASK                                                                       0xFE000000L
+//DAGB4_WR_CNTL_MISC
+#define DAGB4_WR_CNTL_MISC__STOR_POOL_CREDIT__SHIFT                                                           0x0
+#define DAGB4_WR_CNTL_MISC__EA_POOL_CREDIT__SHIFT                                                             0x6
+#define DAGB4_WR_CNTL_MISC__IO_EA_CREDIT__SHIFT                                                               0xd
+#define DAGB4_WR_CNTL_MISC__STOR_CC_NEW_MODE__SHIFT                                                           0x13
+#define DAGB4_WR_CNTL_MISC__EA_CC_LEGACY_MODE__SHIFT                                                          0x14
+#define DAGB4_WR_CNTL_MISC__UTCL2_CID__SHIFT                                                                  0x15
+#define DAGB4_WR_CNTL_MISC__RDRET_FIFO_CREDITS__SHIFT                                                         0x1a
+#define DAGB4_WR_CNTL_MISC__STOR_POOL_CREDIT_MASK                                                             0x0000003FL
+#define DAGB4_WR_CNTL_MISC__EA_POOL_CREDIT_MASK                                                               0x00001FC0L
+#define DAGB4_WR_CNTL_MISC__IO_EA_CREDIT_MASK                                                                 0x0007E000L
+#define DAGB4_WR_CNTL_MISC__STOR_CC_NEW_MODE_MASK                                                             0x00080000L
+#define DAGB4_WR_CNTL_MISC__EA_CC_LEGACY_MODE_MASK                                                            0x00100000L
+#define DAGB4_WR_CNTL_MISC__UTCL2_CID_MASK                                                                    0x03E00000L
+#define DAGB4_WR_CNTL_MISC__RDRET_FIFO_CREDITS_MASK                                                           0xFC000000L
+//DAGB4_WR_TLB_CREDIT
+#define DAGB4_WR_TLB_CREDIT__TLB0__SHIFT                                                                      0x0
+#define DAGB4_WR_TLB_CREDIT__TLB1__SHIFT                                                                      0x5
+#define DAGB4_WR_TLB_CREDIT__TLB2__SHIFT                                                                      0xa
+#define DAGB4_WR_TLB_CREDIT__TLB3__SHIFT                                                                      0xf
+#define DAGB4_WR_TLB_CREDIT__TLB4__SHIFT                                                                      0x14
+#define DAGB4_WR_TLB_CREDIT__TLB5__SHIFT                                                                      0x19
+#define DAGB4_WR_TLB_CREDIT__TLB0_MASK                                                                        0x0000001FL
+#define DAGB4_WR_TLB_CREDIT__TLB1_MASK                                                                        0x000003E0L
+#define DAGB4_WR_TLB_CREDIT__TLB2_MASK                                                                        0x00007C00L
+#define DAGB4_WR_TLB_CREDIT__TLB3_MASK                                                                        0x000F8000L
+#define DAGB4_WR_TLB_CREDIT__TLB4_MASK                                                                        0x01F00000L
+#define DAGB4_WR_TLB_CREDIT__TLB5_MASK                                                                        0x3E000000L
+//DAGB4_WR_DATA_CREDIT
+#define DAGB4_WR_DATA_CREDIT__DLOCK_VC_CREDITS__SHIFT                                                         0x0
+#define DAGB4_WR_DATA_CREDIT__LARGE_BURST_CREDITS__SHIFT                                                      0x8
+#define DAGB4_WR_DATA_CREDIT__MIDDLE_BURST_CREDITS__SHIFT                                                     0x10
+#define DAGB4_WR_DATA_CREDIT__SMALL_BURST_CREDITS__SHIFT                                                      0x18
+#define DAGB4_WR_DATA_CREDIT__DLOCK_VC_CREDITS_MASK                                                           0x000000FFL
+#define DAGB4_WR_DATA_CREDIT__LARGE_BURST_CREDITS_MASK                                                        0x0000FF00L
+#define DAGB4_WR_DATA_CREDIT__MIDDLE_BURST_CREDITS_MASK                                                       0x00FF0000L
+#define DAGB4_WR_DATA_CREDIT__SMALL_BURST_CREDITS_MASK                                                        0xFF000000L
+//DAGB4_WR_MISC_CREDIT
+#define DAGB4_WR_MISC_CREDIT__ATOMIC_CREDIT__SHIFT                                                            0x0
+#define DAGB4_WR_MISC_CREDIT__DLOCK_VC_NUM__SHIFT                                                             0x6
+#define DAGB4_WR_MISC_CREDIT__OSD_CREDIT__SHIFT                                                               0x9
+#define DAGB4_WR_MISC_CREDIT__OSD_DLOCK_CREDIT__SHIFT                                                         0x10
+#define DAGB4_WR_MISC_CREDIT__ATOMIC_CREDIT_MASK                                                              0x0000003FL
+#define DAGB4_WR_MISC_CREDIT__DLOCK_VC_NUM_MASK                                                               0x000001C0L
+#define DAGB4_WR_MISC_CREDIT__OSD_CREDIT_MASK                                                                 0x0000FE00L
+#define DAGB4_WR_MISC_CREDIT__OSD_DLOCK_CREDIT_MASK                                                           0x007F0000L
+//DAGB4_WR_OSD_CREDIT_CNTL1
+#define DAGB4_WR_OSD_CREDIT_CNTL1__VC0_CREDIT__SHIFT                                                          0x0
+#define DAGB4_WR_OSD_CREDIT_CNTL1__VC1_CREDIT__SHIFT                                                          0x4
+#define DAGB4_WR_OSD_CREDIT_CNTL1__VC2_CREDIT__SHIFT                                                          0x8
+#define DAGB4_WR_OSD_CREDIT_CNTL1__VC3_CREDIT__SHIFT                                                          0xc
+#define DAGB4_WR_OSD_CREDIT_CNTL1__IO_CREDIT__SHIFT                                                           0x10
+#define DAGB4_WR_OSD_CREDIT_CNTL1__GMI_CREDIT__SHIFT                                                          0x14
+#define DAGB4_WR_OSD_CREDIT_CNTL1__POOL_CREDIT__SHIFT                                                         0x18
+#define DAGB4_WR_OSD_CREDIT_CNTL1__VC0_CREDIT_MASK                                                            0x0000000FL
+#define DAGB4_WR_OSD_CREDIT_CNTL1__VC1_CREDIT_MASK                                                            0x000000F0L
+#define DAGB4_WR_OSD_CREDIT_CNTL1__VC2_CREDIT_MASK                                                            0x00000F00L
+#define DAGB4_WR_OSD_CREDIT_CNTL1__VC3_CREDIT_MASK                                                            0x0000F000L
+#define DAGB4_WR_OSD_CREDIT_CNTL1__IO_CREDIT_MASK                                                             0x000F0000L
+#define DAGB4_WR_OSD_CREDIT_CNTL1__GMI_CREDIT_MASK                                                            0x00F00000L
+#define DAGB4_WR_OSD_CREDIT_CNTL1__POOL_CREDIT_MASK                                                           0x3F000000L
+//DAGB4_WR_OSD_CREDIT_CNTL2
+#define DAGB4_WR_OSD_CREDIT_CNTL2__CREDIT_MARGIN__SHIFT                                                       0x0
+#define DAGB4_WR_OSD_CREDIT_CNTL2__ENABLE_LEGACY__SHIFT                                                       0x4
+#define DAGB4_WR_OSD_CREDIT_CNTL2__CREDIT_MARGIN_MASK                                                         0x0000000FL
+#define DAGB4_WR_OSD_CREDIT_CNTL2__ENABLE_LEGACY_MASK                                                         0x00000010L
+//DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC0_CREDIT__SHIFT                                                  0x0
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC1_CREDIT__SHIFT                                                  0x5
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC2_CREDIT__SHIFT                                                  0xa
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC3_CREDIT__SHIFT                                                  0xf
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__POOL_CREDIT__SHIFT                                                 0x14
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC_MODE__SHIFT                                                     0x19
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX_EQ__SHIFT                                                      0x1a
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX0__SHIFT                                                        0x1b
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX1__SHIFT                                                        0x1c
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX2__SHIFT                                                        0x1d
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC0_CREDIT_MASK                                                    0x0000001FL
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC1_CREDIT_MASK                                                    0x000003E0L
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC2_CREDIT_MASK                                                    0x00007C00L
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC3_CREDIT_MASK                                                    0x000F8000L
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__POOL_CREDIT_MASK                                                   0x01F00000L
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__VC_MODE_MASK                                                       0x02000000L
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX_EQ_MASK                                                        0x04000000L
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX0_MASK                                                          0x08000000L
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX1_MASK                                                          0x10000000L
+#define DAGB4_WR_ATOMIC_FIFO_CREDIT_CNTL1__FIX2_MASK                                                          0x20000000L
+//DAGB4_WRCLI_GPU_SNOOP_OVERRIDE
+#define DAGB4_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE__SHIFT                                                         0x0
+#define DAGB4_WRCLI_GPU_SNOOP_OVERRIDE__ENABLE_MASK                                                           0x0000FFFFL
+//DAGB4_WRCLI_GPU_SNOOP_OVERRIDE_VALUE
+#define DAGB4_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE__SHIFT                                                   0x0
+#define DAGB4_WRCLI_GPU_SNOOP_OVERRIDE_VALUE__ENABLE_MASK                                                     0x0000FFFFL
+//DAGB4_WRCLI_ASK_PENDING
+#define DAGB4_WRCLI_ASK_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB4_WRCLI_ASK_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB4_WRCLI_GO_PENDING
+#define DAGB4_WRCLI_GO_PENDING__BUSY__SHIFT                                                                   0x0
+#define DAGB4_WRCLI_GO_PENDING__BUSY_MASK                                                                     0xFFFFFFFFL
+//DAGB4_WRCLI_GBLSEND_PENDING
+#define DAGB4_WRCLI_GBLSEND_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB4_WRCLI_GBLSEND_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB4_WRCLI_TLB_PENDING
+#define DAGB4_WRCLI_TLB_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB4_WRCLI_TLB_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB4_WRCLI_OARB_PENDING
+#define DAGB4_WRCLI_OARB_PENDING__BUSY__SHIFT                                                                 0x0
+#define DAGB4_WRCLI_OARB_PENDING__BUSY_MASK                                                                   0xFFFFFFFFL
+//DAGB4_WRCLI_OSD_PENDING
+#define DAGB4_WRCLI_OSD_PENDING__BUSY__SHIFT                                                                  0x0
+#define DAGB4_WRCLI_OSD_PENDING__BUSY_MASK                                                                    0xFFFFFFFFL
+//DAGB4_WRCLI_DBUS_ASK_PENDING
+#define DAGB4_WRCLI_DBUS_ASK_PENDING__BUSY__SHIFT                                                             0x0
+#define DAGB4_WRCLI_DBUS_ASK_PENDING__BUSY_MASK                                                               0xFFFFFFFFL
+//DAGB4_WRCLI_DBUS_GO_PENDING
+#define DAGB4_WRCLI_DBUS_GO_PENDING__BUSY__SHIFT                                                              0x0
+#define DAGB4_WRCLI_DBUS_GO_PENDING__BUSY_MASK                                                                0xFFFFFFFFL
+//DAGB4_DAGB_DLY
+#define DAGB4_DAGB_DLY__DLY__SHIFT                                                                            0x0
+#define DAGB4_DAGB_DLY__CLI__SHIFT                                                                            0x8
+#define DAGB4_DAGB_DLY__POS__SHIFT                                                                            0x10
+#define DAGB4_DAGB_DLY__DLY_MASK                                                                              0x000000FFL
+#define DAGB4_DAGB_DLY__CLI_MASK                                                                              0x0000FF00L
+#define DAGB4_DAGB_DLY__POS_MASK                                                                              0x000F0000L
+//DAGB4_CNTL_MISC
+#define DAGB4_CNTL_MISC__EA_VC0_REMAP__SHIFT                                                                  0x0
+#define DAGB4_CNTL_MISC__EA_VC1_REMAP__SHIFT                                                                  0x3
+#define DAGB4_CNTL_MISC__EA_VC2_REMAP__SHIFT                                                                  0x6
+#define DAGB4_CNTL_MISC__EA_VC3_REMAP__SHIFT                                                                  0x9
+#define DAGB4_CNTL_MISC__EA_VC4_REMAP__SHIFT                                                                  0xc
+#define DAGB4_CNTL_MISC__EA_VC5_REMAP__SHIFT                                                                  0xf
+#define DAGB4_CNTL_MISC__EA_VC6_REMAP__SHIFT                                                                  0x12
+#define DAGB4_CNTL_MISC__EA_VC7_REMAP__SHIFT                                                                  0x15
+#define DAGB4_CNTL_MISC__BW_INIT_CYCLE__SHIFT                                                                 0x18
+#define DAGB4_CNTL_MISC__BW_RW_GAP_CYCLE__SHIFT                                                               0x1e
+#define DAGB4_CNTL_MISC__EA_VC0_REMAP_MASK                                                                    0x00000007L
+#define DAGB4_CNTL_MISC__EA_VC1_REMAP_MASK                                                                    0x00000038L
+#define DAGB4_CNTL_MISC__EA_VC2_REMAP_MASK                                                                    0x000001C0L
+#define DAGB4_CNTL_MISC__EA_VC3_REMAP_MASK                                                                    0x00000E00L
+#define DAGB4_CNTL_MISC__EA_VC4_REMAP_MASK                                                                    0x00007000L
+#define DAGB4_CNTL_MISC__EA_VC5_REMAP_MASK                                                                    0x00038000L
+#define DAGB4_CNTL_MISC__EA_VC6_REMAP_MASK                                                                    0x001C0000L
+#define DAGB4_CNTL_MISC__EA_VC7_REMAP_MASK                                                                    0x00E00000L
+#define DAGB4_CNTL_MISC__BW_INIT_CYCLE_MASK                                                                   0x3F000000L
+#define DAGB4_CNTL_MISC__BW_RW_GAP_CYCLE_MASK                                                                 0xC0000000L
+//DAGB4_CNTL_MISC2
+#define DAGB4_CNTL_MISC2__URG_BOOST_ENABLE__SHIFT                                                             0x0
+#define DAGB4_CNTL_MISC2__URG_HALT_ENABLE__SHIFT                                                              0x1
+#define DAGB4_CNTL_MISC2__DISABLE_WRREQ_CG__SHIFT                                                             0x2
+#define DAGB4_CNTL_MISC2__DISABLE_WRRET_CG__SHIFT                                                             0x3
+#define DAGB4_CNTL_MISC2__DISABLE_RDREQ_CG__SHIFT                                                             0x4
+#define DAGB4_CNTL_MISC2__DISABLE_RDRET_CG__SHIFT                                                             0x5
+#define DAGB4_CNTL_MISC2__DISABLE_TLBWR_CG__SHIFT                                                             0x6
+#define DAGB4_CNTL_MISC2__DISABLE_TLBRD_CG__SHIFT                                                             0x7
+#define DAGB4_CNTL_MISC2__DISABLE_EAWRREQ_BUSY__SHIFT                                                         0x8
+#define DAGB4_CNTL_MISC2__DISABLE_EARDREQ_BUSY__SHIFT                                                         0x9
+#define DAGB4_CNTL_MISC2__SWAP_CTL__SHIFT                                                                     0xa
+#define DAGB4_CNTL_MISC2__RDRET_FIFO_PERF__SHIFT                                                              0xb
+#define DAGB4_CNTL_MISC2__HDP_CID__SHIFT                                                                      0xc
+#define DAGB4_CNTL_MISC2__RDRET_FIFO_DLOCK_CREDITS__SHIFT                                                     0x10
+#define DAGB4_CNTL_MISC2__URG_BOOST_ENABLE_MASK                                                               0x00000001L
+#define DAGB4_CNTL_MISC2__URG_HALT_ENABLE_MASK                                                                0x00000002L
+#define DAGB4_CNTL_MISC2__DISABLE_WRREQ_CG_MASK                                                               0x00000004L
+#define DAGB4_CNTL_MISC2__DISABLE_WRRET_CG_MASK                                                               0x00000008L
+#define DAGB4_CNTL_MISC2__DISABLE_RDREQ_CG_MASK                                                               0x00000010L
+#define DAGB4_CNTL_MISC2__DISABLE_RDRET_CG_MASK                                                               0x00000020L
+#define DAGB4_CNTL_MISC2__DISABLE_TLBWR_CG_MASK                                                               0x00000040L
+#define DAGB4_CNTL_MISC2__DISABLE_TLBRD_CG_MASK                                                               0x00000080L
+#define DAGB4_CNTL_MISC2__DISABLE_EAWRREQ_BUSY_MASK                                                           0x00000100L
+#define DAGB4_CNTL_MISC2__DISABLE_EARDREQ_BUSY_MASK                                                           0x00000200L
+#define DAGB4_CNTL_MISC2__SWAP_CTL_MASK                                                                       0x00000400L
+#define DAGB4_CNTL_MISC2__RDRET_FIFO_PERF_MASK                                                                0x00000800L
+#define DAGB4_CNTL_MISC2__HDP_CID_MASK                                                                        0x0000F000L
+#define DAGB4_CNTL_MISC2__RDRET_FIFO_DLOCK_CREDITS_MASK                                                       0x003F0000L
+//DAGB4_FATAL_ERROR_CNTL
+#define DAGB4_FATAL_ERROR_CNTL__FILTER_NUM__SHIFT                                                             0x0
+#define DAGB4_FATAL_ERROR_CNTL__FILTER_NUM_MASK                                                               0x000003FFL
+//DAGB4_FATAL_ERROR_CLEAR
+#define DAGB4_FATAL_ERROR_CLEAR__CLEAR__SHIFT                                                                 0x0
+#define DAGB4_FATAL_ERROR_CLEAR__CLEAR_MASK                                                                   0x00000001L
+//DAGB4_FATAL_ERROR_STATUS0
+#define DAGB4_FATAL_ERROR_STATUS0__VALID__SHIFT                                                               0x0
+#define DAGB4_FATAL_ERROR_STATUS0__CID__SHIFT                                                                 0x1
+#define DAGB4_FATAL_ERROR_STATUS0__ADDR_LO__SHIFT                                                             0x6
+#define DAGB4_FATAL_ERROR_STATUS0__VALID_MASK                                                                 0x00000001L
+#define DAGB4_FATAL_ERROR_STATUS0__CID_MASK                                                                   0x0000003EL
+#define DAGB4_FATAL_ERROR_STATUS0__ADDR_LO_MASK                                                               0xFFFFFFC0L
+//DAGB4_FATAL_ERROR_STATUS1
+#define DAGB4_FATAL_ERROR_STATUS1__ADDR_HI__SHIFT                                                             0x0
+#define DAGB4_FATAL_ERROR_STATUS1__ADDR_HI_MASK                                                               0x0001FFFFL
+//DAGB4_FATAL_ERROR_STATUS2
+#define DAGB4_FATAL_ERROR_STATUS2__TAG__SHIFT                                                                 0x0
+#define DAGB4_FATAL_ERROR_STATUS2__VFID__SHIFT                                                                0x10
+#define DAGB4_FATAL_ERROR_STATUS2__VF__SHIFT                                                                  0x14
+#define DAGB4_FATAL_ERROR_STATUS2__SPACE__SHIFT                                                               0x15
+#define DAGB4_FATAL_ERROR_STATUS2__IO__SHIFT                                                                  0x16
+#define DAGB4_FATAL_ERROR_STATUS2__SIZE__SHIFT                                                                0x17
+#define DAGB4_FATAL_ERROR_STATUS2__DBGMSK__SHIFT                                                              0x18
+#define DAGB4_FATAL_ERROR_STATUS2__FED__SHIFT                                                                 0x19
+#define DAGB4_FATAL_ERROR_STATUS2__TAG_MASK                                                                   0x0000FFFFL
+#define DAGB4_FATAL_ERROR_STATUS2__VFID_MASK                                                                  0x000F0000L
+#define DAGB4_FATAL_ERROR_STATUS2__VF_MASK                                                                    0x00100000L
+#define DAGB4_FATAL_ERROR_STATUS2__SPACE_MASK                                                                 0x00200000L
+#define DAGB4_FATAL_ERROR_STATUS2__IO_MASK                                                                    0x00400000L
+#define DAGB4_FATAL_ERROR_STATUS2__SIZE_MASK                                                                  0x00800000L
+#define DAGB4_FATAL_ERROR_STATUS2__DBGMSK_MASK                                                                0x01000000L
+#define DAGB4_FATAL_ERROR_STATUS2__FED_MASK                                                                   0x02000000L
+//DAGB4_FATAL_ERROR_STATUS3
+#define DAGB4_FATAL_ERROR_STATUS3__NOALLOC__SHIFT                                                             0x0
+#define DAGB4_FATAL_ERROR_STATUS3__UNITID__SHIFT                                                              0x1
+#define DAGB4_FATAL_ERROR_STATUS3__OP__SHIFT                                                                  0x7
+#define DAGB4_FATAL_ERROR_STATUS3__SECLEVEL__SHIFT                                                            0xe
+#define DAGB4_FATAL_ERROR_STATUS3__WRTMZ__SHIFT                                                               0x11
+#define DAGB4_FATAL_ERROR_STATUS3__RDTMZ__SHIFT                                                               0x12
+#define DAGB4_FATAL_ERROR_STATUS3__SNOOP__SHIFT                                                               0x13
+#define DAGB4_FATAL_ERROR_STATUS3__INVAL__SHIFT                                                               0x14
+#define DAGB4_FATAL_ERROR_STATUS3__NACK__SHIFT                                                                0x15
+#define DAGB4_FATAL_ERROR_STATUS3__RO__SHIFT                                                                  0x17
+#define DAGB4_FATAL_ERROR_STATUS3__MEMLOG__SHIFT                                                              0x18
+#define DAGB4_FATAL_ERROR_STATUS3__EOP__SHIFT                                                                 0x19
+#define DAGB4_FATAL_ERROR_STATUS3__NOALLOC_MASK                                                               0x00000001L
+#define DAGB4_FATAL_ERROR_STATUS3__UNITID_MASK                                                                0x0000007EL
+#define DAGB4_FATAL_ERROR_STATUS3__OP_MASK                                                                    0x00003F80L
+#define DAGB4_FATAL_ERROR_STATUS3__SECLEVEL_MASK                                                              0x0001C000L
+#define DAGB4_FATAL_ERROR_STATUS3__WRTMZ_MASK                                                                 0x00020000L
+#define DAGB4_FATAL_ERROR_STATUS3__RDTMZ_MASK                                                                 0x00040000L
+#define DAGB4_FATAL_ERROR_STATUS3__SNOOP_MASK                                                                 0x00080000L
+#define DAGB4_FATAL_ERROR_STATUS3__INVAL_MASK                                                                 0x00100000L
+#define DAGB4_FATAL_ERROR_STATUS3__NACK_MASK                                                                  0x00600000L
+#define DAGB4_FATAL_ERROR_STATUS3__RO_MASK                                                                    0x00800000L
+#define DAGB4_FATAL_ERROR_STATUS3__MEMLOG_MASK                                                                0x01000000L
+#define DAGB4_FATAL_ERROR_STATUS3__EOP_MASK                                                                   0x02000000L
+//DAGB4_FIFO_EMPTY
+#define DAGB4_FIFO_EMPTY__EMPTY__SHIFT                                                                        0x0
+#define DAGB4_FIFO_EMPTY__EMPTY_MASK                                                                          0x00FFFFFFL
+//DAGB4_FIFO_FULL
+#define DAGB4_FIFO_FULL__FULL__SHIFT                                                                          0x0
+#define DAGB4_FIFO_FULL__FULL_MASK                                                                            0x007FFFFFL
+//DAGB4_WR_CREDITS_FULL
+#define DAGB4_WR_CREDITS_FULL__FULL__SHIFT                                                                    0x0
+#define DAGB4_WR_CREDITS_FULL__FULL_MASK                                                                      0x1FFFFFFFL
+//DAGB4_RD_CREDITS_FULL
+#define DAGB4_RD_CREDITS_FULL__FULL__SHIFT                                                                    0x0
+#define DAGB4_RD_CREDITS_FULL__FULL_MASK                                                                      0x0003FFFFL
+//DAGB4_PERFCOUNTER_LO
+#define DAGB4_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define DAGB4_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//DAGB4_PERFCOUNTER_HI
+#define DAGB4_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define DAGB4_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define DAGB4_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define DAGB4_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+//DAGB4_PERFCOUNTER0_CFG
+#define DAGB4_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB4_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB4_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB4_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB4_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB4_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB4_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB4_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB4_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB4_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB4_PERFCOUNTER1_CFG
+#define DAGB4_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB4_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB4_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB4_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB4_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB4_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB4_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB4_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB4_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB4_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB4_PERFCOUNTER2_CFG
+#define DAGB4_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                               0x0
+#define DAGB4_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define DAGB4_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                              0x18
+#define DAGB4_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                 0x1c
+#define DAGB4_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                  0x1d
+#define DAGB4_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define DAGB4_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define DAGB4_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define DAGB4_PERFCOUNTER2_CFG__ENABLE_MASK                                                                   0x10000000L
+#define DAGB4_PERFCOUNTER2_CFG__CLEAR_MASK                                                                    0x20000000L
+//DAGB4_PERFCOUNTER_RSLT_CNTL
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define DAGB4_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+//DAGB4_L1TLB_REG_RW
+#define DAGB4_L1TLB_REG_RW__REG_WRITE_L1TLB_CTRL__SHIFT                                                       0x0
+#define DAGB4_L1TLB_REG_RW__REG_READ_L1TLB_CTRL__SHIFT                                                        0x1
+#define DAGB4_L1TLB_REG_RW__VMID_EXCEP_INT_CTRL__SHIFT                                                        0x2
+#define DAGB4_L1TLB_REG_RW__WDAT_PARITY_CHECK__SHIFT                                                          0x4
+#define DAGB4_L1TLB_REG_RW__DISABLE_RDRET_CHECK__SHIFT                                                        0x5
+#define DAGB4_L1TLB_REG_RW__RESERVE__SHIFT                                                                    0x6
+#define DAGB4_L1TLB_REG_RW__REG_WRITE_L1TLB_CTRL_MASK                                                         0x00000001L
+#define DAGB4_L1TLB_REG_RW__REG_READ_L1TLB_CTRL_MASK                                                          0x00000002L
+#define DAGB4_L1TLB_REG_RW__VMID_EXCEP_INT_CTRL_MASK                                                          0x00000004L
+#define DAGB4_L1TLB_REG_RW__WDAT_PARITY_CHECK_MASK                                                            0x00000010L
+#define DAGB4_L1TLB_REG_RW__DISABLE_RDRET_CHECK_MASK                                                          0x00000020L
+#define DAGB4_L1TLB_REG_RW__RESERVE_MASK                                                                      0xFFFFFFC0L
+
+
+// addressBlock: aid_mmhub_ea_mmeadec0
+//MMEA0_DRAM_RD_CLI2GRP_MAP0
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                         0x0
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                         0x2
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                         0x4
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                         0x6
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                         0x8
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                         0xa
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                         0xc
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                         0xe
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                         0x10
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                         0x12
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                        0x14
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                        0x16
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                        0x18
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                        0x1a
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                        0x1c
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                        0x1e
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                           0x00000003L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                           0x0000000CL
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                           0x00000030L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                           0x000000C0L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                           0x00000300L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                           0x00000C00L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                           0x00003000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                           0x0000C000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                           0x00030000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                           0x000C0000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                          0x00300000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                          0x00C00000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                          0x03000000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                          0x0C000000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                          0x30000000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                          0xC0000000L
+//MMEA0_DRAM_RD_CLI2GRP_MAP1
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                        0x0
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                        0x2
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                        0x4
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                        0x6
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                        0x8
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                        0xa
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                        0xc
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                        0xe
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                        0x10
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                        0x12
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                        0x14
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                        0x16
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                        0x18
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                        0x1a
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                        0x1c
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                        0x1e
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                          0x00000003L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                          0x0000000CL
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                          0x00000030L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                          0x000000C0L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                          0x00000300L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                          0x00000C00L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                          0x00003000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                          0x0000C000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                          0x00030000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                          0x000C0000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                          0x00300000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                          0x00C00000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                          0x03000000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                          0x0C000000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                          0x30000000L
+#define MMEA0_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                          0xC0000000L
+//MMEA0_DRAM_WR_CLI2GRP_MAP0
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                         0x0
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                         0x2
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                         0x4
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                         0x6
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                         0x8
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                         0xa
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                         0xc
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                         0xe
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                         0x10
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                         0x12
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                        0x14
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                        0x16
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                        0x18
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                        0x1a
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                        0x1c
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                        0x1e
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                           0x00000003L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                           0x0000000CL
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                           0x00000030L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                           0x000000C0L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                           0x00000300L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                           0x00000C00L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                           0x00003000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                           0x0000C000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                           0x00030000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                           0x000C0000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                          0x00300000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                          0x00C00000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                          0x03000000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                          0x0C000000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                          0x30000000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                          0xC0000000L
+//MMEA0_DRAM_WR_CLI2GRP_MAP1
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                        0x0
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                        0x2
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                        0x4
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                        0x6
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                        0x8
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                        0xa
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                        0xc
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                        0xe
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                        0x10
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                        0x12
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                        0x14
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                        0x16
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                        0x18
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                        0x1a
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                        0x1c
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                        0x1e
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                          0x00000003L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                          0x0000000CL
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                          0x00000030L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                          0x000000C0L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                          0x00000300L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                          0x00000C00L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                          0x00003000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                          0x0000C000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                          0x00030000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                          0x000C0000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                          0x00300000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                          0x00C00000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                          0x03000000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                          0x0C000000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                          0x30000000L
+#define MMEA0_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                          0xC0000000L
+//MMEA0_DRAM_RD_GRP2VC_MAP
+#define MMEA0_DRAM_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                            0x0
+#define MMEA0_DRAM_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                            0x3
+#define MMEA0_DRAM_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                            0x6
+#define MMEA0_DRAM_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                            0x9
+#define MMEA0_DRAM_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                              0x00000007L
+#define MMEA0_DRAM_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                              0x00000038L
+#define MMEA0_DRAM_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                              0x000001C0L
+#define MMEA0_DRAM_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                              0x00000E00L
+//MMEA0_DRAM_WR_GRP2VC_MAP
+#define MMEA0_DRAM_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                            0x0
+#define MMEA0_DRAM_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                            0x3
+#define MMEA0_DRAM_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                            0x6
+#define MMEA0_DRAM_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                            0x9
+#define MMEA0_DRAM_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                              0x00000007L
+#define MMEA0_DRAM_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                              0x00000038L
+#define MMEA0_DRAM_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                              0x000001C0L
+#define MMEA0_DRAM_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                              0x00000E00L
+//MMEA0_DRAM_RD_LAZY
+#define MMEA0_DRAM_RD_LAZY__GROUP0_DELAY__SHIFT                                                               0x0
+#define MMEA0_DRAM_RD_LAZY__GROUP1_DELAY__SHIFT                                                               0x3
+#define MMEA0_DRAM_RD_LAZY__GROUP2_DELAY__SHIFT                                                               0x6
+#define MMEA0_DRAM_RD_LAZY__GROUP3_DELAY__SHIFT                                                               0x9
+#define MMEA0_DRAM_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                           0xc
+#define MMEA0_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                          0x14
+#define MMEA0_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                          0x1b
+#define MMEA0_DRAM_RD_LAZY__GROUP0_DELAY_MASK                                                                 0x00000007L
+#define MMEA0_DRAM_RD_LAZY__GROUP1_DELAY_MASK                                                                 0x00000038L
+#define MMEA0_DRAM_RD_LAZY__GROUP2_DELAY_MASK                                                                 0x000001C0L
+#define MMEA0_DRAM_RD_LAZY__GROUP3_DELAY_MASK                                                                 0x00000E00L
+#define MMEA0_DRAM_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                             0x0003F000L
+#define MMEA0_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                            0x07F00000L
+#define MMEA0_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                            0x78000000L
+//MMEA0_DRAM_WR_LAZY
+#define MMEA0_DRAM_WR_LAZY__GROUP0_DELAY__SHIFT                                                               0x0
+#define MMEA0_DRAM_WR_LAZY__GROUP1_DELAY__SHIFT                                                               0x3
+#define MMEA0_DRAM_WR_LAZY__GROUP2_DELAY__SHIFT                                                               0x6
+#define MMEA0_DRAM_WR_LAZY__GROUP3_DELAY__SHIFT                                                               0x9
+#define MMEA0_DRAM_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                           0xc
+#define MMEA0_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                          0x14
+#define MMEA0_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                          0x1b
+#define MMEA0_DRAM_WR_LAZY__GROUP0_DELAY_MASK                                                                 0x00000007L
+#define MMEA0_DRAM_WR_LAZY__GROUP1_DELAY_MASK                                                                 0x00000038L
+#define MMEA0_DRAM_WR_LAZY__GROUP2_DELAY_MASK                                                                 0x000001C0L
+#define MMEA0_DRAM_WR_LAZY__GROUP3_DELAY_MASK                                                                 0x00000E00L
+#define MMEA0_DRAM_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                             0x0003F000L
+#define MMEA0_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                            0x07F00000L
+#define MMEA0_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                            0x78000000L
+//MMEA0_DRAM_RD_CAM_CNTL
+#define MMEA0_DRAM_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                           0x0
+#define MMEA0_DRAM_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                           0x4
+#define MMEA0_DRAM_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                           0x8
+#define MMEA0_DRAM_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                           0xc
+#define MMEA0_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                   0x10
+#define MMEA0_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                   0x13
+#define MMEA0_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                   0x16
+#define MMEA0_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                   0x19
+#define MMEA0_DRAM_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                           0x1c
+#define MMEA0_DRAM_RD_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                     0x1d
+#define MMEA0_DRAM_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                             0x0000000FL
+#define MMEA0_DRAM_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                             0x000000F0L
+#define MMEA0_DRAM_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                             0x00000F00L
+#define MMEA0_DRAM_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                             0x0000F000L
+#define MMEA0_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                     0x00070000L
+#define MMEA0_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                     0x00380000L
+#define MMEA0_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                     0x01C00000L
+#define MMEA0_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                     0x0E000000L
+#define MMEA0_DRAM_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                             0x10000000L
+#define MMEA0_DRAM_RD_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                       0x20000000L
+//MMEA0_DRAM_WR_CAM_CNTL
+#define MMEA0_DRAM_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                           0x0
+#define MMEA0_DRAM_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                           0x4
+#define MMEA0_DRAM_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                           0x8
+#define MMEA0_DRAM_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                           0xc
+#define MMEA0_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                   0x10
+#define MMEA0_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                   0x13
+#define MMEA0_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                   0x16
+#define MMEA0_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                   0x19
+#define MMEA0_DRAM_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                           0x1c
+#define MMEA0_DRAM_WR_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                     0x1d
+#define MMEA0_DRAM_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                             0x0000000FL
+#define MMEA0_DRAM_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                             0x000000F0L
+#define MMEA0_DRAM_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                             0x00000F00L
+#define MMEA0_DRAM_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                             0x0000F000L
+#define MMEA0_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                     0x00070000L
+#define MMEA0_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                     0x00380000L
+#define MMEA0_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                     0x01C00000L
+#define MMEA0_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                     0x0E000000L
+#define MMEA0_DRAM_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                             0x10000000L
+#define MMEA0_DRAM_WR_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                       0x20000000L
+//MMEA0_DRAM_PAGE_BURST
+#define MMEA0_DRAM_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                             0x0
+#define MMEA0_DRAM_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                             0x8
+#define MMEA0_DRAM_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                             0x10
+#define MMEA0_DRAM_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                             0x18
+#define MMEA0_DRAM_PAGE_BURST__RD_LIMIT_LO_MASK                                                               0x000000FFL
+#define MMEA0_DRAM_PAGE_BURST__RD_LIMIT_HI_MASK                                                               0x0000FF00L
+#define MMEA0_DRAM_PAGE_BURST__WR_LIMIT_LO_MASK                                                               0x00FF0000L
+#define MMEA0_DRAM_PAGE_BURST__WR_LIMIT_HI_MASK                                                               0xFF000000L
+//MMEA0_DRAM_RD_PRI_AGE
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                  0xc
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                  0xf
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                  0x12
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                  0x15
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                    0x00007000L
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                    0x00038000L
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                    0x001C0000L
+#define MMEA0_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                    0x00E00000L
+//MMEA0_DRAM_WR_PRI_AGE
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                  0xc
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                  0xf
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                  0x12
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                  0x15
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                    0x00007000L
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                    0x00038000L
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                    0x001C0000L
+#define MMEA0_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                    0x00E00000L
+//MMEA0_DRAM_RD_PRI_QUEUING
+#define MMEA0_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                          0x0
+#define MMEA0_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                          0x3
+#define MMEA0_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                          0x6
+#define MMEA0_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                          0x9
+#define MMEA0_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA0_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA0_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA0_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                            0x00000E00L
+//MMEA0_DRAM_WR_PRI_QUEUING
+#define MMEA0_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                          0x0
+#define MMEA0_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                          0x3
+#define MMEA0_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                          0x6
+#define MMEA0_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                          0x9
+#define MMEA0_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA0_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA0_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA0_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                            0x00000E00L
+//MMEA0_DRAM_RD_PRI_FIXED
+#define MMEA0_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                              0x0
+#define MMEA0_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                              0x3
+#define MMEA0_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                              0x6
+#define MMEA0_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                              0x9
+#define MMEA0_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                0x00000007L
+#define MMEA0_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                0x00000038L
+#define MMEA0_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                0x000001C0L
+#define MMEA0_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                0x00000E00L
+//MMEA0_DRAM_WR_PRI_FIXED
+#define MMEA0_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                              0x0
+#define MMEA0_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                              0x3
+#define MMEA0_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                              0x6
+#define MMEA0_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                              0x9
+#define MMEA0_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                0x00000007L
+#define MMEA0_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                0x00000038L
+#define MMEA0_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                0x000001C0L
+#define MMEA0_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                0x00000E00L
+//MMEA0_DRAM_RD_PRI_URGENCY
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                          0x0
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                          0x3
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                          0x6
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                          0x9
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                 0xc
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                 0xd
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                 0xe
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                 0xf
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                            0x00000E00L
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                   0x00001000L
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                   0x00002000L
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                   0x00004000L
+#define MMEA0_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                   0x00008000L
+//MMEA0_DRAM_WR_PRI_URGENCY
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                          0x0
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                          0x3
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                          0x6
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                          0x9
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                 0xc
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                 0xd
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                 0xe
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                 0xf
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                            0x00000E00L
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                   0x00001000L
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                   0x00002000L
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                   0x00004000L
+#define MMEA0_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                   0x00008000L
+//MMEA0_DRAM_RD_PRI_QUANT_PRI1
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA0_DRAM_RD_PRI_QUANT_PRI2
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA0_DRAM_RD_PRI_QUANT_PRI3
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA0_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA0_DRAM_WR_PRI_QUANT_PRI1
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA0_DRAM_WR_PRI_QUANT_PRI2
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA0_DRAM_WR_PRI_QUANT_PRI3
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA0_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA0_GMI_RD_CLI2GRP_MAP0
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//MMEA0_GMI_RD_CLI2GRP_MAP1
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define MMEA0_GMI_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//MMEA0_GMI_WR_CLI2GRP_MAP0
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//MMEA0_GMI_WR_CLI2GRP_MAP1
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define MMEA0_GMI_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//MMEA0_GMI_RD_GRP2VC_MAP
+#define MMEA0_GMI_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define MMEA0_GMI_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define MMEA0_GMI_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define MMEA0_GMI_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define MMEA0_GMI_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define MMEA0_GMI_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define MMEA0_GMI_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define MMEA0_GMI_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//MMEA0_GMI_WR_GRP2VC_MAP
+#define MMEA0_GMI_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define MMEA0_GMI_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define MMEA0_GMI_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define MMEA0_GMI_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define MMEA0_GMI_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define MMEA0_GMI_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define MMEA0_GMI_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define MMEA0_GMI_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//MMEA0_GMI_RD_LAZY
+#define MMEA0_GMI_RD_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define MMEA0_GMI_RD_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define MMEA0_GMI_RD_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define MMEA0_GMI_RD_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define MMEA0_GMI_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define MMEA0_GMI_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define MMEA0_GMI_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define MMEA0_GMI_RD_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define MMEA0_GMI_RD_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define MMEA0_GMI_RD_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define MMEA0_GMI_RD_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define MMEA0_GMI_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define MMEA0_GMI_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define MMEA0_GMI_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//MMEA0_GMI_WR_LAZY
+#define MMEA0_GMI_WR_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define MMEA0_GMI_WR_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define MMEA0_GMI_WR_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define MMEA0_GMI_WR_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define MMEA0_GMI_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define MMEA0_GMI_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define MMEA0_GMI_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define MMEA0_GMI_WR_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define MMEA0_GMI_WR_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define MMEA0_GMI_WR_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define MMEA0_GMI_WR_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define MMEA0_GMI_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define MMEA0_GMI_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define MMEA0_GMI_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//MMEA0_GMI_RD_CAM_CNTL
+#define MMEA0_GMI_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define MMEA0_GMI_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define MMEA0_GMI_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define MMEA0_GMI_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define MMEA0_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define MMEA0_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define MMEA0_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define MMEA0_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define MMEA0_GMI_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define MMEA0_GMI_RD_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                      0x1d
+#define MMEA0_GMI_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define MMEA0_GMI_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define MMEA0_GMI_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define MMEA0_GMI_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define MMEA0_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define MMEA0_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define MMEA0_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define MMEA0_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define MMEA0_GMI_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+#define MMEA0_GMI_RD_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                        0x20000000L
+//MMEA0_GMI_WR_CAM_CNTL
+#define MMEA0_GMI_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define MMEA0_GMI_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define MMEA0_GMI_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define MMEA0_GMI_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define MMEA0_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define MMEA0_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define MMEA0_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define MMEA0_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define MMEA0_GMI_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define MMEA0_GMI_WR_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                      0x1d
+#define MMEA0_GMI_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define MMEA0_GMI_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define MMEA0_GMI_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define MMEA0_GMI_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define MMEA0_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define MMEA0_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define MMEA0_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define MMEA0_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define MMEA0_GMI_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+#define MMEA0_GMI_WR_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                        0x20000000L
+//MMEA0_GMI_PAGE_BURST
+#define MMEA0_GMI_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define MMEA0_GMI_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define MMEA0_GMI_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define MMEA0_GMI_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define MMEA0_GMI_PAGE_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define MMEA0_GMI_PAGE_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define MMEA0_GMI_PAGE_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define MMEA0_GMI_PAGE_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//MMEA0_GMI_RD_PRI_AGE
+#define MMEA0_GMI_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define MMEA0_GMI_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define MMEA0_GMI_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define MMEA0_GMI_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define MMEA0_GMI_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define MMEA0_GMI_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define MMEA0_GMI_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define MMEA0_GMI_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define MMEA0_GMI_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define MMEA0_GMI_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define MMEA0_GMI_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define MMEA0_GMI_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define MMEA0_GMI_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define MMEA0_GMI_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define MMEA0_GMI_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define MMEA0_GMI_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//MMEA0_GMI_WR_PRI_AGE
+#define MMEA0_GMI_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define MMEA0_GMI_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define MMEA0_GMI_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define MMEA0_GMI_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define MMEA0_GMI_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define MMEA0_GMI_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define MMEA0_GMI_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define MMEA0_GMI_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define MMEA0_GMI_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define MMEA0_GMI_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define MMEA0_GMI_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define MMEA0_GMI_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define MMEA0_GMI_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define MMEA0_GMI_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define MMEA0_GMI_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define MMEA0_GMI_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//MMEA0_GMI_RD_PRI_QUEUING
+#define MMEA0_GMI_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define MMEA0_GMI_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define MMEA0_GMI_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define MMEA0_GMI_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define MMEA0_GMI_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA0_GMI_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA0_GMI_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA0_GMI_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//MMEA0_GMI_WR_PRI_QUEUING
+#define MMEA0_GMI_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define MMEA0_GMI_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define MMEA0_GMI_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define MMEA0_GMI_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define MMEA0_GMI_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA0_GMI_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA0_GMI_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA0_GMI_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//MMEA0_GMI_RD_PRI_FIXED
+#define MMEA0_GMI_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define MMEA0_GMI_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define MMEA0_GMI_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define MMEA0_GMI_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define MMEA0_GMI_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define MMEA0_GMI_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define MMEA0_GMI_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define MMEA0_GMI_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//MMEA0_GMI_WR_PRI_FIXED
+#define MMEA0_GMI_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define MMEA0_GMI_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define MMEA0_GMI_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define MMEA0_GMI_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define MMEA0_GMI_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define MMEA0_GMI_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define MMEA0_GMI_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define MMEA0_GMI_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//MMEA0_GMI_RD_PRI_URGENCY
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define MMEA0_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//MMEA0_GMI_WR_PRI_URGENCY
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define MMEA0_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//MMEA0_GMI_RD_PRI_URGENCY_MASKING
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                    0x0
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                    0x1
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                    0x2
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                    0x3
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                    0x4
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                    0x5
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                    0x6
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                    0x7
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                    0x8
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                    0x9
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                   0xa
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                   0xb
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                   0xc
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                   0xd
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                   0xe
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                   0xf
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                   0x10
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                   0x11
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                   0x12
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                   0x13
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                   0x14
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                   0x15
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                   0x16
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                   0x17
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                   0x18
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                   0x19
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                   0x1a
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                   0x1b
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                   0x1c
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                   0x1d
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                   0x1e
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                   0x1f
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                      0x00000001L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                      0x00000002L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                      0x00000004L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                      0x00000008L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                      0x00000010L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                      0x00000020L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                      0x00000040L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                      0x00000080L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                      0x00000100L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                      0x00000200L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                     0x00000400L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                     0x00000800L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                     0x00001000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                     0x00002000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                     0x00004000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                     0x00008000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                     0x00010000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                     0x00020000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                     0x00040000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                     0x00080000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                     0x00100000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                     0x00200000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                     0x00400000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                     0x00800000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                     0x01000000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                     0x02000000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                     0x04000000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                     0x08000000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                     0x10000000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                     0x20000000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                     0x40000000L
+#define MMEA0_GMI_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                     0x80000000L
+//MMEA0_GMI_WR_PRI_URGENCY_MASKING
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                    0x0
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                    0x1
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                    0x2
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                    0x3
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                    0x4
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                    0x5
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                    0x6
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                    0x7
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                    0x8
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                    0x9
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                   0xa
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                   0xb
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                   0xc
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                   0xd
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                   0xe
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                   0xf
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                   0x10
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                   0x11
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                   0x12
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                   0x13
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                   0x14
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                   0x15
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                   0x16
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                   0x17
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                   0x18
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                   0x19
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                   0x1a
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                   0x1b
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                   0x1c
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                   0x1d
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                   0x1e
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                   0x1f
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                      0x00000001L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                      0x00000002L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                      0x00000004L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                      0x00000008L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                      0x00000010L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                      0x00000020L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                      0x00000040L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                      0x00000080L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                      0x00000100L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                      0x00000200L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                     0x00000400L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                     0x00000800L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                     0x00001000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                     0x00002000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                     0x00004000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                     0x00008000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                     0x00010000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                     0x00020000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                     0x00040000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                     0x00080000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                     0x00100000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                     0x00200000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                     0x00400000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                     0x00800000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                     0x01000000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                     0x02000000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                     0x04000000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                     0x08000000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                     0x10000000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                     0x20000000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                     0x40000000L
+#define MMEA0_GMI_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                     0x80000000L
+//MMEA0_GMI_RD_PRI_QUANT_PRI1
+#define MMEA0_GMI_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA0_GMI_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA0_GMI_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA0_GMI_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA0_GMI_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA0_GMI_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA0_GMI_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA0_GMI_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA0_GMI_RD_PRI_QUANT_PRI2
+#define MMEA0_GMI_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA0_GMI_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA0_GMI_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA0_GMI_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA0_GMI_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA0_GMI_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA0_GMI_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA0_GMI_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA0_GMI_RD_PRI_QUANT_PRI3
+#define MMEA0_GMI_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA0_GMI_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA0_GMI_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA0_GMI_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA0_GMI_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA0_GMI_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA0_GMI_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA0_GMI_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA0_GMI_WR_PRI_QUANT_PRI1
+#define MMEA0_GMI_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA0_GMI_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA0_GMI_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA0_GMI_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA0_GMI_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA0_GMI_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA0_GMI_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA0_GMI_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA0_GMI_WR_PRI_QUANT_PRI2
+#define MMEA0_GMI_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA0_GMI_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA0_GMI_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA0_GMI_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA0_GMI_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA0_GMI_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA0_GMI_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA0_GMI_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA0_GMI_WR_PRI_QUANT_PRI3
+#define MMEA0_GMI_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA0_GMI_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA0_GMI_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA0_GMI_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA0_GMI_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA0_GMI_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA0_GMI_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA0_GMI_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA0_IO_RD_CLI2GRP_MAP0
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                           0x0
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                           0x2
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                           0x4
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                           0x6
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                           0x8
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                           0xa
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                           0xc
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                           0xe
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                           0x10
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                           0x12
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                          0x14
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                          0x16
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                          0x18
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                          0x1a
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                          0x1c
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                          0x1e
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                             0x00000003L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                             0x0000000CL
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                             0x00000030L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                             0x000000C0L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                             0x00000300L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                             0x00000C00L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                             0x00003000L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                             0x0000C000L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                             0x00030000L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                             0x000C0000L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                            0x00300000L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                            0x00C00000L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                            0x03000000L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                            0x0C000000L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                            0x30000000L
+#define MMEA0_IO_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                            0xC0000000L
+//MMEA0_IO_RD_CLI2GRP_MAP1
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                          0x0
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                          0x2
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                          0x4
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                          0x6
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                          0x8
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                          0xa
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                          0xc
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                          0xe
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                          0x10
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                          0x12
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                          0x14
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                          0x16
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                          0x18
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                          0x1a
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                          0x1c
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                          0x1e
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                            0x00000003L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                            0x0000000CL
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                            0x00000030L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                            0x000000C0L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                            0x00000300L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                            0x00000C00L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                            0x00003000L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                            0x0000C000L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                            0x00030000L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                            0x000C0000L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                            0x00300000L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                            0x00C00000L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                            0x03000000L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                            0x0C000000L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                            0x30000000L
+#define MMEA0_IO_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                            0xC0000000L
+//MMEA0_IO_WR_CLI2GRP_MAP0
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                           0x0
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                           0x2
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                           0x4
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                           0x6
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                           0x8
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                           0xa
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                           0xc
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                           0xe
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                           0x10
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                           0x12
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                          0x14
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                          0x16
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                          0x18
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                          0x1a
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                          0x1c
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                          0x1e
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                             0x00000003L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                             0x0000000CL
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                             0x00000030L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                             0x000000C0L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                             0x00000300L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                             0x00000C00L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                             0x00003000L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                             0x0000C000L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                             0x00030000L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                             0x000C0000L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                            0x00300000L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                            0x00C00000L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                            0x03000000L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                            0x0C000000L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                            0x30000000L
+#define MMEA0_IO_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                            0xC0000000L
+//MMEA0_IO_WR_CLI2GRP_MAP1
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                          0x0
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                          0x2
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                          0x4
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                          0x6
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                          0x8
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                          0xa
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                          0xc
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                          0xe
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                          0x10
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                          0x12
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                          0x14
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                          0x16
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                          0x18
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                          0x1a
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                          0x1c
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                          0x1e
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                            0x00000003L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                            0x0000000CL
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                            0x00000030L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                            0x000000C0L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                            0x00000300L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                            0x00000C00L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                            0x00003000L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                            0x0000C000L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                            0x00030000L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                            0x000C0000L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                            0x00300000L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                            0x00C00000L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                            0x03000000L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                            0x0C000000L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                            0x30000000L
+#define MMEA0_IO_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                            0xC0000000L
+//MMEA0_IO_RD_COMBINE_FLUSH
+#define MMEA0_IO_RD_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                        0x0
+#define MMEA0_IO_RD_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                        0x4
+#define MMEA0_IO_RD_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                        0x8
+#define MMEA0_IO_RD_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                        0xc
+#define MMEA0_IO_RD_COMBINE_FLUSH__COMB_MODE__SHIFT                                                           0x10
+#define MMEA0_IO_RD_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                          0x0000000FL
+#define MMEA0_IO_RD_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                          0x000000F0L
+#define MMEA0_IO_RD_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                          0x00000F00L
+#define MMEA0_IO_RD_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                          0x0000F000L
+#define MMEA0_IO_RD_COMBINE_FLUSH__COMB_MODE_MASK                                                             0x00030000L
+//MMEA0_IO_WR_COMBINE_FLUSH
+#define MMEA0_IO_WR_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                        0x0
+#define MMEA0_IO_WR_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                        0x4
+#define MMEA0_IO_WR_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                        0x8
+#define MMEA0_IO_WR_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                        0xc
+#define MMEA0_IO_WR_COMBINE_FLUSH__COMB_MODE__SHIFT                                                           0x10
+#define MMEA0_IO_WR_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                          0x0000000FL
+#define MMEA0_IO_WR_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                          0x000000F0L
+#define MMEA0_IO_WR_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                          0x00000F00L
+#define MMEA0_IO_WR_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                          0x0000F000L
+#define MMEA0_IO_WR_COMBINE_FLUSH__COMB_MODE_MASK                                                             0x00030000L
+//MMEA0_IO_GROUP_BURST
+#define MMEA0_IO_GROUP_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define MMEA0_IO_GROUP_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define MMEA0_IO_GROUP_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define MMEA0_IO_GROUP_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define MMEA0_IO_GROUP_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define MMEA0_IO_GROUP_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define MMEA0_IO_GROUP_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define MMEA0_IO_GROUP_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//MMEA0_IO_RD_PRI_AGE
+#define MMEA0_IO_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                         0x0
+#define MMEA0_IO_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                         0x3
+#define MMEA0_IO_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                         0x6
+#define MMEA0_IO_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                         0x9
+#define MMEA0_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                    0xc
+#define MMEA0_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                    0xf
+#define MMEA0_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                    0x12
+#define MMEA0_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                    0x15
+#define MMEA0_IO_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                           0x00000007L
+#define MMEA0_IO_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                           0x00000038L
+#define MMEA0_IO_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                           0x000001C0L
+#define MMEA0_IO_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                           0x00000E00L
+#define MMEA0_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                      0x00007000L
+#define MMEA0_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                      0x00038000L
+#define MMEA0_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                      0x001C0000L
+#define MMEA0_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                      0x00E00000L
+//MMEA0_IO_WR_PRI_AGE
+#define MMEA0_IO_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                         0x0
+#define MMEA0_IO_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                         0x3
+#define MMEA0_IO_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                         0x6
+#define MMEA0_IO_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                         0x9
+#define MMEA0_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                    0xc
+#define MMEA0_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                    0xf
+#define MMEA0_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                    0x12
+#define MMEA0_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                    0x15
+#define MMEA0_IO_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                           0x00000007L
+#define MMEA0_IO_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                           0x00000038L
+#define MMEA0_IO_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                           0x000001C0L
+#define MMEA0_IO_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                           0x00000E00L
+#define MMEA0_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                      0x00007000L
+#define MMEA0_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                      0x00038000L
+#define MMEA0_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                      0x001C0000L
+#define MMEA0_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                      0x00E00000L
+//MMEA0_IO_RD_PRI_QUEUING
+#define MMEA0_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                            0x0
+#define MMEA0_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                            0x3
+#define MMEA0_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                            0x6
+#define MMEA0_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                            0x9
+#define MMEA0_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA0_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA0_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA0_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                              0x00000E00L
+//MMEA0_IO_WR_PRI_QUEUING
+#define MMEA0_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                            0x0
+#define MMEA0_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                            0x3
+#define MMEA0_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                            0x6
+#define MMEA0_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                            0x9
+#define MMEA0_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA0_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA0_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA0_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                              0x00000E00L
+//MMEA0_IO_RD_PRI_FIXED
+#define MMEA0_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                0x0
+#define MMEA0_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                0x3
+#define MMEA0_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                0x6
+#define MMEA0_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                0x9
+#define MMEA0_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                  0x00000007L
+#define MMEA0_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                  0x00000038L
+#define MMEA0_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                  0x000001C0L
+#define MMEA0_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                  0x00000E00L
+//MMEA0_IO_WR_PRI_FIXED
+#define MMEA0_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                0x0
+#define MMEA0_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                0x3
+#define MMEA0_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                0x6
+#define MMEA0_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                0x9
+#define MMEA0_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                  0x00000007L
+#define MMEA0_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                  0x00000038L
+#define MMEA0_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                  0x000001C0L
+#define MMEA0_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                  0x00000E00L
+//MMEA0_IO_RD_PRI_URGENCY
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                            0x0
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                            0x3
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                            0x6
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                            0x9
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                   0xc
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                   0xd
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                   0xe
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                   0xf
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                              0x00000E00L
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                     0x00001000L
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                     0x00002000L
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                     0x00004000L
+#define MMEA0_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                     0x00008000L
+//MMEA0_IO_WR_PRI_URGENCY
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                            0x0
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                            0x3
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                            0x6
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                            0x9
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                   0xc
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                   0xd
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                   0xe
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                   0xf
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                              0x00000E00L
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                     0x00001000L
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                     0x00002000L
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                     0x00004000L
+#define MMEA0_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                     0x00008000L
+//MMEA0_IO_RD_PRI_URGENCY_MASKING
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                     0x0
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                     0x1
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                     0x2
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                     0x3
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                     0x4
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                     0x5
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                     0x6
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                     0x7
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                     0x8
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                     0x9
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                    0xa
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                    0xb
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                    0xc
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                    0xd
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                    0xe
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                    0xf
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                    0x10
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                    0x11
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                    0x12
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                    0x13
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                    0x14
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                    0x15
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                    0x16
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                    0x17
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                    0x18
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                    0x19
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                    0x1a
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                    0x1b
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                    0x1c
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                    0x1d
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                    0x1e
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                    0x1f
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                       0x00000001L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                       0x00000002L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                       0x00000004L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                       0x00000008L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                       0x00000010L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                       0x00000020L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                       0x00000040L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                       0x00000080L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                       0x00000100L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                       0x00000200L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                      0x00000400L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                      0x00000800L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                      0x00001000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                      0x00002000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                      0x00004000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                      0x00008000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                      0x00010000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                      0x00020000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                      0x00040000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                      0x00080000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                      0x00100000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                      0x00200000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                      0x00400000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                      0x00800000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                      0x01000000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                      0x02000000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                      0x04000000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                      0x08000000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                      0x10000000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                      0x20000000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                      0x40000000L
+#define MMEA0_IO_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                      0x80000000L
+//MMEA0_IO_WR_PRI_URGENCY_MASKING
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                     0x0
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                     0x1
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                     0x2
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                     0x3
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                     0x4
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                     0x5
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                     0x6
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                     0x7
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                     0x8
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                     0x9
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                    0xa
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                    0xb
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                    0xc
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                    0xd
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                    0xe
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                    0xf
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                    0x10
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                    0x11
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                    0x12
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                    0x13
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                    0x14
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                    0x15
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                    0x16
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                    0x17
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                    0x18
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                    0x19
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                    0x1a
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                    0x1b
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                    0x1c
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                    0x1d
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                    0x1e
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                    0x1f
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                       0x00000001L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                       0x00000002L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                       0x00000004L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                       0x00000008L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                       0x00000010L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                       0x00000020L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                       0x00000040L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                       0x00000080L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                       0x00000100L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                       0x00000200L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                      0x00000400L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                      0x00000800L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                      0x00001000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                      0x00002000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                      0x00004000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                      0x00008000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                      0x00010000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                      0x00020000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                      0x00040000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                      0x00080000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                      0x00100000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                      0x00200000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                      0x00400000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                      0x00800000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                      0x01000000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                      0x02000000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                      0x04000000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                      0x08000000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                      0x10000000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                      0x20000000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                      0x40000000L
+#define MMEA0_IO_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                      0x80000000L
+//MMEA0_IO_RD_PRI_QUANT_PRI1
+#define MMEA0_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA0_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA0_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA0_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA0_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA0_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA0_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA0_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA0_IO_RD_PRI_QUANT_PRI2
+#define MMEA0_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA0_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA0_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA0_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA0_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA0_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA0_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA0_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA0_IO_RD_PRI_QUANT_PRI3
+#define MMEA0_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA0_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA0_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA0_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA0_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA0_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA0_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA0_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA0_IO_WR_PRI_QUANT_PRI1
+#define MMEA0_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA0_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA0_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA0_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA0_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA0_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA0_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA0_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA0_IO_WR_PRI_QUANT_PRI2
+#define MMEA0_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA0_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA0_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA0_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA0_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA0_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA0_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA0_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA0_IO_WR_PRI_QUANT_PRI3
+#define MMEA0_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA0_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA0_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA0_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA0_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA0_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA0_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA0_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA0_SDP_ARB_DRAM
+#define MMEA0_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL__SHIFT                                                      0x0
+#define MMEA0_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA__SHIFT                                                      0x8
+#define MMEA0_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI__SHIFT                                                         0x10
+#define MMEA0_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI__SHIFT                                                         0x11
+#define MMEA0_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES__SHIFT                                                         0x12
+#define MMEA0_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES__SHIFT                                                         0x13
+#define MMEA0_SDP_ARB_DRAM__EOB_ON_EXPIRE__SHIFT                                                              0x14
+#define MMEA0_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                     0x15
+#define MMEA0_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING__SHIFT                                                       0x16
+#define MMEA0_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL_MASK                                                        0x0000007FL
+#define MMEA0_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA_MASK                                                        0x00007F00L
+#define MMEA0_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI_MASK                                                           0x00010000L
+#define MMEA0_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI_MASK                                                           0x00020000L
+#define MMEA0_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES_MASK                                                           0x00040000L
+#define MMEA0_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES_MASK                                                           0x00080000L
+#define MMEA0_SDP_ARB_DRAM__EOB_ON_EXPIRE_MASK                                                                0x00100000L
+#define MMEA0_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE_MASK                                                       0x00200000L
+#define MMEA0_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING_MASK                                                         0x00400000L
+//MMEA0_SDP_ARB_GMI
+#define MMEA0_SDP_ARB_GMI__RDWR_BURST_LIMIT_CYCL__SHIFT                                                       0x0
+#define MMEA0_SDP_ARB_GMI__RDWR_BURST_LIMIT_DATA__SHIFT                                                       0x8
+#define MMEA0_SDP_ARB_GMI__EARLY_SW2RD_ON_PRI__SHIFT                                                          0x10
+#define MMEA0_SDP_ARB_GMI__EARLY_SW2WR_ON_PRI__SHIFT                                                          0x11
+#define MMEA0_SDP_ARB_GMI__EARLY_SW2RD_ON_RES__SHIFT                                                          0x12
+#define MMEA0_SDP_ARB_GMI__EARLY_SW2WR_ON_RES__SHIFT                                                          0x13
+#define MMEA0_SDP_ARB_GMI__EOB_ON_EXPIRE__SHIFT                                                               0x14
+#define MMEA0_SDP_ARB_GMI__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                      0x15
+#define MMEA0_SDP_ARB_GMI__ALLOW_CHAIN_BREAKING__SHIFT                                                        0x16
+#define MMEA0_SDP_ARB_GMI__RDWR_BURST_LIMIT_CYCL_MASK                                                         0x0000007FL
+#define MMEA0_SDP_ARB_GMI__RDWR_BURST_LIMIT_DATA_MASK                                                         0x00007F00L
+#define MMEA0_SDP_ARB_GMI__EARLY_SW2RD_ON_PRI_MASK                                                            0x00010000L
+#define MMEA0_SDP_ARB_GMI__EARLY_SW2WR_ON_PRI_MASK                                                            0x00020000L
+#define MMEA0_SDP_ARB_GMI__EARLY_SW2RD_ON_RES_MASK                                                            0x00040000L
+#define MMEA0_SDP_ARB_GMI__EARLY_SW2WR_ON_RES_MASK                                                            0x00080000L
+#define MMEA0_SDP_ARB_GMI__EOB_ON_EXPIRE_MASK                                                                 0x00100000L
+#define MMEA0_SDP_ARB_GMI__DECOUPLE_RDWR_BNKSTATE_MASK                                                        0x00200000L
+#define MMEA0_SDP_ARB_GMI__ALLOW_CHAIN_BREAKING_MASK                                                          0x00400000L
+//MMEA0_SDP_ARB_FINAL
+#define MMEA0_SDP_ARB_FINAL__DRAM_BURST_LIMIT__SHIFT                                                          0x0
+#define MMEA0_SDP_ARB_FINAL__GMI_BURST_LIMIT__SHIFT                                                           0x5
+#define MMEA0_SDP_ARB_FINAL__IO_BURST_LIMIT__SHIFT                                                            0xa
+#define MMEA0_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER__SHIFT                                                    0xf
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC0__SHIFT                                                                0x11
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC1__SHIFT                                                                0x12
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC2__SHIFT                                                                0x13
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC3__SHIFT                                                                0x14
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC4__SHIFT                                                                0x15
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC5__SHIFT                                                                0x16
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC6__SHIFT                                                                0x17
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC7__SHIFT                                                                0x18
+#define MMEA0_SDP_ARB_FINAL__ERREVENT_ON_ERROR__SHIFT                                                         0x19
+#define MMEA0_SDP_ARB_FINAL__HALTREQ_ON_ERROR__SHIFT                                                          0x1a
+#define MMEA0_SDP_ARB_FINAL__DRAM_BURST_STRETCH__SHIFT                                                        0x1b
+#define MMEA0_SDP_ARB_FINAL__GMI_BURST_STRETCH__SHIFT                                                         0x1c
+#define MMEA0_SDP_ARB_FINAL__DRAM_BURST_LIMIT_MASK                                                            0x0000001FL
+#define MMEA0_SDP_ARB_FINAL__GMI_BURST_LIMIT_MASK                                                             0x000003E0L
+#define MMEA0_SDP_ARB_FINAL__IO_BURST_LIMIT_MASK                                                              0x00007C00L
+#define MMEA0_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER_MASK                                                      0x00018000L
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC0_MASK                                                                  0x00020000L
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC1_MASK                                                                  0x00040000L
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC2_MASK                                                                  0x00080000L
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC3_MASK                                                                  0x00100000L
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC4_MASK                                                                  0x00200000L
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC5_MASK                                                                  0x00400000L
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC6_MASK                                                                  0x00800000L
+#define MMEA0_SDP_ARB_FINAL__RDONLY_VC7_MASK                                                                  0x01000000L
+#define MMEA0_SDP_ARB_FINAL__ERREVENT_ON_ERROR_MASK                                                           0x02000000L
+#define MMEA0_SDP_ARB_FINAL__HALTREQ_ON_ERROR_MASK                                                            0x04000000L
+#define MMEA0_SDP_ARB_FINAL__DRAM_BURST_STRETCH_MASK                                                          0x08000000L
+#define MMEA0_SDP_ARB_FINAL__GMI_BURST_STRETCH_MASK                                                           0x10000000L
+//MMEA0_SDP_DRAM_PRIORITY
+#define MMEA0_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                    0x0
+#define MMEA0_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                    0x4
+#define MMEA0_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                    0x8
+#define MMEA0_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                    0xc
+#define MMEA0_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                    0x10
+#define MMEA0_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                    0x14
+#define MMEA0_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                    0x18
+#define MMEA0_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                    0x1c
+#define MMEA0_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                      0x0000000FL
+#define MMEA0_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                      0x000000F0L
+#define MMEA0_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                      0x00000F00L
+#define MMEA0_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                      0x0000F000L
+#define MMEA0_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                      0x000F0000L
+#define MMEA0_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                      0x00F00000L
+#define MMEA0_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                      0x0F000000L
+#define MMEA0_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                      0xF0000000L
+//MMEA0_SDP_GMI_PRIORITY
+#define MMEA0_SDP_GMI_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                     0x0
+#define MMEA0_SDP_GMI_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                     0x4
+#define MMEA0_SDP_GMI_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                     0x8
+#define MMEA0_SDP_GMI_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                     0xc
+#define MMEA0_SDP_GMI_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                     0x10
+#define MMEA0_SDP_GMI_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                     0x14
+#define MMEA0_SDP_GMI_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                     0x18
+#define MMEA0_SDP_GMI_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                     0x1c
+#define MMEA0_SDP_GMI_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                       0x0000000FL
+#define MMEA0_SDP_GMI_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                       0x000000F0L
+#define MMEA0_SDP_GMI_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                       0x00000F00L
+#define MMEA0_SDP_GMI_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                       0x0000F000L
+#define MMEA0_SDP_GMI_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                       0x000F0000L
+#define MMEA0_SDP_GMI_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                       0x00F00000L
+#define MMEA0_SDP_GMI_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                       0x0F000000L
+#define MMEA0_SDP_GMI_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                       0xF0000000L
+//MMEA0_SDP_IO_PRIORITY
+#define MMEA0_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                      0x0
+#define MMEA0_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                      0x4
+#define MMEA0_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                      0x8
+#define MMEA0_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                      0xc
+#define MMEA0_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                      0x10
+#define MMEA0_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                      0x14
+#define MMEA0_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                      0x18
+#define MMEA0_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                      0x1c
+#define MMEA0_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                        0x0000000FL
+#define MMEA0_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                        0x000000F0L
+#define MMEA0_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                        0x00000F00L
+#define MMEA0_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                        0x0000F000L
+#define MMEA0_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                        0x000F0000L
+#define MMEA0_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                        0x00F00000L
+#define MMEA0_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                        0x0F000000L
+#define MMEA0_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                        0xF0000000L
+//MMEA0_SDP_CREDITS
+#define MMEA0_SDP_CREDITS__TAG_LIMIT__SHIFT                                                                   0x0
+#define MMEA0_SDP_CREDITS__WR_RESP_CREDITS__SHIFT                                                             0x8
+#define MMEA0_SDP_CREDITS__RD_RESP_CREDITS__SHIFT                                                             0x10
+#define MMEA0_SDP_CREDITS__TAG_LIMIT_MASK                                                                     0x000000FFL
+#define MMEA0_SDP_CREDITS__WR_RESP_CREDITS_MASK                                                               0x00007F00L
+#define MMEA0_SDP_CREDITS__RD_RESP_CREDITS_MASK                                                               0x007F0000L
+//MMEA0_SDP_TAG_RESERVE0
+#define MMEA0_SDP_TAG_RESERVE0__VC0__SHIFT                                                                    0x0
+#define MMEA0_SDP_TAG_RESERVE0__VC1__SHIFT                                                                    0x8
+#define MMEA0_SDP_TAG_RESERVE0__VC2__SHIFT                                                                    0x10
+#define MMEA0_SDP_TAG_RESERVE0__VC3__SHIFT                                                                    0x18
+#define MMEA0_SDP_TAG_RESERVE0__VC0_MASK                                                                      0x000000FFL
+#define MMEA0_SDP_TAG_RESERVE0__VC1_MASK                                                                      0x0000FF00L
+#define MMEA0_SDP_TAG_RESERVE0__VC2_MASK                                                                      0x00FF0000L
+#define MMEA0_SDP_TAG_RESERVE0__VC3_MASK                                                                      0xFF000000L
+//MMEA0_SDP_TAG_RESERVE1
+#define MMEA0_SDP_TAG_RESERVE1__VC4__SHIFT                                                                    0x0
+#define MMEA0_SDP_TAG_RESERVE1__VC5__SHIFT                                                                    0x8
+#define MMEA0_SDP_TAG_RESERVE1__VC6__SHIFT                                                                    0x10
+#define MMEA0_SDP_TAG_RESERVE1__VC7__SHIFT                                                                    0x18
+#define MMEA0_SDP_TAG_RESERVE1__VC4_MASK                                                                      0x000000FFL
+#define MMEA0_SDP_TAG_RESERVE1__VC5_MASK                                                                      0x0000FF00L
+#define MMEA0_SDP_TAG_RESERVE1__VC6_MASK                                                                      0x00FF0000L
+#define MMEA0_SDP_TAG_RESERVE1__VC7_MASK                                                                      0xFF000000L
+//MMEA0_SDP_VCC_RESERVE0
+#define MMEA0_SDP_VCC_RESERVE0__VC0_CREDITS__SHIFT                                                            0x0
+#define MMEA0_SDP_VCC_RESERVE0__VC1_CREDITS__SHIFT                                                            0x6
+#define MMEA0_SDP_VCC_RESERVE0__VC2_CREDITS__SHIFT                                                            0xc
+#define MMEA0_SDP_VCC_RESERVE0__VC3_CREDITS__SHIFT                                                            0x12
+#define MMEA0_SDP_VCC_RESERVE0__VC4_CREDITS__SHIFT                                                            0x18
+#define MMEA0_SDP_VCC_RESERVE0__VC0_CREDITS_MASK                                                              0x0000003FL
+#define MMEA0_SDP_VCC_RESERVE0__VC1_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA0_SDP_VCC_RESERVE0__VC2_CREDITS_MASK                                                              0x0003F000L
+#define MMEA0_SDP_VCC_RESERVE0__VC3_CREDITS_MASK                                                              0x00FC0000L
+#define MMEA0_SDP_VCC_RESERVE0__VC4_CREDITS_MASK                                                              0x3F000000L
+//MMEA0_SDP_VCC_RESERVE1
+#define MMEA0_SDP_VCC_RESERVE1__VC5_CREDITS__SHIFT                                                            0x0
+#define MMEA0_SDP_VCC_RESERVE1__VC6_CREDITS__SHIFT                                                            0x6
+#define MMEA0_SDP_VCC_RESERVE1__VC7_CREDITS__SHIFT                                                            0xc
+#define MMEA0_SDP_VCC_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                        0x1f
+#define MMEA0_SDP_VCC_RESERVE1__VC5_CREDITS_MASK                                                              0x0000003FL
+#define MMEA0_SDP_VCC_RESERVE1__VC6_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA0_SDP_VCC_RESERVE1__VC7_CREDITS_MASK                                                              0x0003F000L
+#define MMEA0_SDP_VCC_RESERVE1__DISTRIBUTE_POOL_MASK                                                          0x80000000L
+//MMEA0_SDP_VCD_RESERVE0
+#define MMEA0_SDP_VCD_RESERVE0__VC0_CREDITS__SHIFT                                                            0x0
+#define MMEA0_SDP_VCD_RESERVE0__VC1_CREDITS__SHIFT                                                            0x6
+#define MMEA0_SDP_VCD_RESERVE0__VC2_CREDITS__SHIFT                                                            0xc
+#define MMEA0_SDP_VCD_RESERVE0__VC3_CREDITS__SHIFT                                                            0x12
+#define MMEA0_SDP_VCD_RESERVE0__VC4_CREDITS__SHIFT                                                            0x18
+#define MMEA0_SDP_VCD_RESERVE0__VC0_CREDITS_MASK                                                              0x0000003FL
+#define MMEA0_SDP_VCD_RESERVE0__VC1_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA0_SDP_VCD_RESERVE0__VC2_CREDITS_MASK                                                              0x0003F000L
+#define MMEA0_SDP_VCD_RESERVE0__VC3_CREDITS_MASK                                                              0x00FC0000L
+#define MMEA0_SDP_VCD_RESERVE0__VC4_CREDITS_MASK                                                              0x3F000000L
+//MMEA0_SDP_VCD_RESERVE1
+#define MMEA0_SDP_VCD_RESERVE1__VC5_CREDITS__SHIFT                                                            0x0
+#define MMEA0_SDP_VCD_RESERVE1__VC6_CREDITS__SHIFT                                                            0x6
+#define MMEA0_SDP_VCD_RESERVE1__VC7_CREDITS__SHIFT                                                            0xc
+#define MMEA0_SDP_VCD_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                        0x1f
+#define MMEA0_SDP_VCD_RESERVE1__VC5_CREDITS_MASK                                                              0x0000003FL
+#define MMEA0_SDP_VCD_RESERVE1__VC6_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA0_SDP_VCD_RESERVE1__VC7_CREDITS_MASK                                                              0x0003F000L
+#define MMEA0_SDP_VCD_RESERVE1__DISTRIBUTE_POOL_MASK                                                          0x80000000L
+//MMEA0_SDP_REQ_CNTL
+#define MMEA0_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ__SHIFT                                                  0x0
+#define MMEA0_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE__SHIFT                                                 0x1
+#define MMEA0_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC__SHIFT                                                0x2
+#define MMEA0_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM__SHIFT                                                    0x3
+#define MMEA0_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI__SHIFT                                                     0x4
+#define MMEA0_SDP_REQ_CNTL__INNER_DOMAIN_MODE__SHIFT                                                          0x5
+#define MMEA0_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ__SHIFT                                                       0x6
+#define MMEA0_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE__SHIFT                                                      0x8
+#define MMEA0_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC__SHIFT                                                     0xa
+#define MMEA0_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ_MASK                                                    0x00000001L
+#define MMEA0_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE_MASK                                                   0x00000002L
+#define MMEA0_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC_MASK                                                  0x00000004L
+#define MMEA0_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM_MASK                                                      0x00000008L
+#define MMEA0_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI_MASK                                                       0x00000010L
+#define MMEA0_SDP_REQ_CNTL__INNER_DOMAIN_MODE_MASK                                                            0x00000020L
+#define MMEA0_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ_MASK                                                         0x000000C0L
+#define MMEA0_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE_MASK                                                        0x00000300L
+#define MMEA0_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC_MASK                                                       0x00000C00L
+//MMEA0_MISC
+#define MMEA0_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB__SHIFT                                                        0x0
+#define MMEA0_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB__SHIFT                                                        0x1
+#define MMEA0_MISC__RELATIVE_PRI_IN_GMI_RD_ARB__SHIFT                                                         0x2
+#define MMEA0_MISC__RELATIVE_PRI_IN_GMI_WR_ARB__SHIFT                                                         0x3
+#define MMEA0_MISC__RELATIVE_PRI_IN_IO_RD_ARB__SHIFT                                                          0x4
+#define MMEA0_MISC__RELATIVE_PRI_IN_IO_WR_ARB__SHIFT                                                          0x5
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC0__SHIFT                                                              0x6
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC1__SHIFT                                                              0x7
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC2__SHIFT                                                              0x8
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC3__SHIFT                                                              0x9
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC4__SHIFT                                                              0xa
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC5__SHIFT                                                              0xb
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC6__SHIFT                                                              0xc
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC7__SHIFT                                                              0xd
+#define MMEA0_MISC__EARLY_SDP_ORIGDATA__SHIFT                                                                 0xe
+#define MMEA0_MISC__LINKMGR_DYNAMIC_MODE__SHIFT                                                               0xf
+#define MMEA0_MISC__LINKMGR_HALT_THRESHOLD__SHIFT                                                             0x11
+#define MMEA0_MISC__LINKMGR_RECONNECT_DELAY__SHIFT                                                            0x13
+#define MMEA0_MISC__LINKMGR_IDLE_THRESHOLD__SHIFT                                                             0x15
+#define MMEA0_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB__SHIFT                                                     0x1a
+#define MMEA0_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB__SHIFT                                                      0x1b
+#define MMEA0_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB__SHIFT                                                         0x1c
+#define MMEA0_MISC__FAVOUR_LAST_CS_IN_GMI_ARB__SHIFT                                                          0x1d
+#define MMEA0_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB__SHIFT                                                       0x1e
+#define MMEA0_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB__SHIFT                                                        0x1f
+#define MMEA0_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB_MASK                                                          0x00000001L
+#define MMEA0_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB_MASK                                                          0x00000002L
+#define MMEA0_MISC__RELATIVE_PRI_IN_GMI_RD_ARB_MASK                                                           0x00000004L
+#define MMEA0_MISC__RELATIVE_PRI_IN_GMI_WR_ARB_MASK                                                           0x00000008L
+#define MMEA0_MISC__RELATIVE_PRI_IN_IO_RD_ARB_MASK                                                            0x00000010L
+#define MMEA0_MISC__RELATIVE_PRI_IN_IO_WR_ARB_MASK                                                            0x00000020L
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC0_MASK                                                                0x00000040L
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC1_MASK                                                                0x00000080L
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC2_MASK                                                                0x00000100L
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC3_MASK                                                                0x00000200L
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC4_MASK                                                                0x00000400L
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC5_MASK                                                                0x00000800L
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC6_MASK                                                                0x00001000L
+#define MMEA0_MISC__EARLYWRRET_ENABLE_VC7_MASK                                                                0x00002000L
+#define MMEA0_MISC__EARLY_SDP_ORIGDATA_MASK                                                                   0x00004000L
+#define MMEA0_MISC__LINKMGR_DYNAMIC_MODE_MASK                                                                 0x00018000L
+#define MMEA0_MISC__LINKMGR_HALT_THRESHOLD_MASK                                                               0x00060000L
+#define MMEA0_MISC__LINKMGR_RECONNECT_DELAY_MASK                                                              0x00180000L
+#define MMEA0_MISC__LINKMGR_IDLE_THRESHOLD_MASK                                                               0x03E00000L
+#define MMEA0_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB_MASK                                                       0x04000000L
+#define MMEA0_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB_MASK                                                        0x08000000L
+#define MMEA0_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB_MASK                                                           0x10000000L
+#define MMEA0_MISC__FAVOUR_LAST_CS_IN_GMI_ARB_MASK                                                            0x20000000L
+#define MMEA0_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB_MASK                                                         0x40000000L
+#define MMEA0_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB_MASK                                                          0x80000000L
+//MMEA0_LATENCY_SAMPLING
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_DRAM__SHIFT                                                          0x0
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_DRAM__SHIFT                                                          0x1
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_GMI__SHIFT                                                           0x2
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_GMI__SHIFT                                                           0x3
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_IO__SHIFT                                                            0x4
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_IO__SHIFT                                                            0x5
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_READ__SHIFT                                                          0x6
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_READ__SHIFT                                                          0x7
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_WRITE__SHIFT                                                         0x8
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_WRITE__SHIFT                                                         0x9
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET__SHIFT                                                    0xa
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET__SHIFT                                                    0xb
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET__SHIFT                                                  0xc
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET__SHIFT                                                  0xd
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_VC__SHIFT                                                            0xe
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_VC__SHIFT                                                            0x16
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_DRAM_MASK                                                            0x00000001L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_DRAM_MASK                                                            0x00000002L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_GMI_MASK                                                             0x00000004L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_GMI_MASK                                                             0x00000008L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_IO_MASK                                                              0x00000010L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_IO_MASK                                                              0x00000020L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_READ_MASK                                                            0x00000040L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_READ_MASK                                                            0x00000080L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_WRITE_MASK                                                           0x00000100L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_WRITE_MASK                                                           0x00000200L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET_MASK                                                      0x00000400L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET_MASK                                                      0x00000800L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET_MASK                                                    0x00001000L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET_MASK                                                    0x00002000L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER0_VC_MASK                                                              0x003FC000L
+#define MMEA0_LATENCY_SAMPLING__SAMPLER1_VC_MASK                                                              0x3FC00000L
+//MMEA0_PERFCOUNTER_LO
+#define MMEA0_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define MMEA0_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//MMEA0_PERFCOUNTER_HI
+#define MMEA0_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define MMEA0_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define MMEA0_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define MMEA0_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+//MMEA0_PERFCOUNTER0_CFG
+#define MMEA0_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define MMEA0_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define MMEA0_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define MMEA0_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define MMEA0_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define MMEA0_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define MMEA0_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define MMEA0_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define MMEA0_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define MMEA0_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//MMEA0_PERFCOUNTER1_CFG
+#define MMEA0_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define MMEA0_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define MMEA0_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define MMEA0_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define MMEA0_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define MMEA0_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define MMEA0_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define MMEA0_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define MMEA0_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define MMEA0_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//MMEA0_PERFCOUNTER_RSLT_CNTL
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define MMEA0_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+//MMEA0_DSM_CNTL
+#define MMEA0_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x0
+#define MMEA0_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x2
+#define MMEA0_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x3
+#define MMEA0_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x5
+#define MMEA0_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x6
+#define MMEA0_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x8
+#define MMEA0_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x9
+#define MMEA0_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xb
+#define MMEA0_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0xc
+#define MMEA0_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xe
+#define MMEA0_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0xf
+#define MMEA0_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x11
+#define MMEA0_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x12
+#define MMEA0_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x14
+#define MMEA0_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x15
+#define MMEA0_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x17
+#define MMEA0_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00000003L
+#define MMEA0_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000004L
+#define MMEA0_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00000018L
+#define MMEA0_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000020L
+#define MMEA0_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                0x000000C0L
+#define MMEA0_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000100L
+#define MMEA0_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00000600L
+#define MMEA0_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000800L
+#define MMEA0_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00003000L
+#define MMEA0_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00004000L
+#define MMEA0_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00018000L
+#define MMEA0_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00020000L
+#define MMEA0_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x000C0000L
+#define MMEA0_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00100000L
+#define MMEA0_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00600000L
+#define MMEA0_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00800000L
+//MMEA0_DSM_CNTLA
+#define MMEA0_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                             0x0
+#define MMEA0_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                            0x2
+#define MMEA0_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                             0x3
+#define MMEA0_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                            0x5
+#define MMEA0_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x6
+#define MMEA0_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x8
+#define MMEA0_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x9
+#define MMEA0_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0xb
+#define MMEA0_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0xc
+#define MMEA0_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0xe
+#define MMEA0_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0xf
+#define MMEA0_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x11
+#define MMEA0_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x12
+#define MMEA0_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x14
+#define MMEA0_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                               0x00000003L
+#define MMEA0_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                              0x00000004L
+#define MMEA0_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                               0x00000018L
+#define MMEA0_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                              0x00000020L
+#define MMEA0_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x000000C0L
+#define MMEA0_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000100L
+#define MMEA0_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000600L
+#define MMEA0_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000800L
+#define MMEA0_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00003000L
+#define MMEA0_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00004000L
+#define MMEA0_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00018000L
+#define MMEA0_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00020000L
+#define MMEA0_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x000C0000L
+#define MMEA0_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00100000L
+//MMEA0_DSM_CNTLB
+#define MMEA0_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x0
+#define MMEA0_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x2
+#define MMEA0_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x3
+#define MMEA0_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x5
+#define MMEA0_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x6
+#define MMEA0_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x8
+#define MMEA0_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x9
+#define MMEA0_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xb
+#define MMEA0_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000003L
+#define MMEA0_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000004L
+#define MMEA0_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000018L
+#define MMEA0_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000020L
+#define MMEA0_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA_MASK                                                    0x000000C0L
+#define MMEA0_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000100L
+#define MMEA0_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000600L
+#define MMEA0_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000800L
+//MMEA0_DSM_CNTL2
+#define MMEA0_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x0
+#define MMEA0_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                             0x2
+#define MMEA0_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x3
+#define MMEA0_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                             0x5
+#define MMEA0_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x6
+#define MMEA0_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                            0x8
+#define MMEA0_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x9
+#define MMEA0_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                               0xb
+#define MMEA0_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                               0xc
+#define MMEA0_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                               0xe
+#define MMEA0_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0xf
+#define MMEA0_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x11
+#define MMEA0_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x12
+#define MMEA0_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x14
+#define MMEA0_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x15
+#define MMEA0_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0x17
+#define MMEA0_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                  0x1a
+#define MMEA0_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                               0x00000003L
+#define MMEA0_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                               0x00000004L
+#define MMEA0_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                               0x00000018L
+#define MMEA0_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                               0x00000020L
+#define MMEA0_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                              0x000000C0L
+#define MMEA0_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                              0x00000100L
+#define MMEA0_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00000600L
+#define MMEA0_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000800L
+#define MMEA0_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00003000L
+#define MMEA0_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                 0x00004000L
+#define MMEA0_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00018000L
+#define MMEA0_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00020000L
+#define MMEA0_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x000C0000L
+#define MMEA0_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00100000L
+#define MMEA0_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x00600000L
+#define MMEA0_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00800000L
+#define MMEA0_DSM_CNTL2__INJECT_DELAY_MASK                                                                    0xFC000000L
+//MMEA0_DSM_CNTL2A
+#define MMEA0_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                           0x0
+#define MMEA0_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                           0x2
+#define MMEA0_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                           0x3
+#define MMEA0_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                           0x5
+#define MMEA0_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x6
+#define MMEA0_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x8
+#define MMEA0_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x9
+#define MMEA0_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0xb
+#define MMEA0_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0xc
+#define MMEA0_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0xe
+#define MMEA0_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0xf
+#define MMEA0_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x11
+#define MMEA0_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x12
+#define MMEA0_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x14
+#define MMEA0_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                             0x00000003L
+#define MMEA0_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                             0x00000004L
+#define MMEA0_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                             0x00000018L
+#define MMEA0_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                             0x00000020L
+#define MMEA0_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x000000C0L
+#define MMEA0_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000100L
+#define MMEA0_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000600L
+#define MMEA0_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000800L
+#define MMEA0_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x00003000L
+#define MMEA0_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00004000L
+#define MMEA0_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00018000L
+#define MMEA0_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00020000L
+#define MMEA0_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x000C0000L
+#define MMEA0_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00100000L
+//MMEA0_DSM_CNTL2B
+#define MMEA0_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x0
+#define MMEA0_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY__SHIFT                                                0x2
+#define MMEA0_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x3
+#define MMEA0_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY__SHIFT                                                0x5
+#define MMEA0_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x6
+#define MMEA0_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY__SHIFT                                                0x8
+#define MMEA0_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x9
+#define MMEA0_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY__SHIFT                                                0xb
+#define MMEA0_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000003L
+#define MMEA0_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000004L
+#define MMEA0_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000018L
+#define MMEA0_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000020L
+#define MMEA0_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT_MASK                                                  0x000000C0L
+#define MMEA0_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000100L
+#define MMEA0_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000600L
+#define MMEA0_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000800L
+//MMEA0_CGTT_CLK_CTRL
+#define MMEA0_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define MMEA0_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define MMEA0_CGTT_CLK_CTRL__SPARE0__SHIFT                                                                    0xc
+#define MMEA0_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE__SHIFT                                                 0x14
+#define MMEA0_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ__SHIFT                                                  0x15
+#define MMEA0_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN__SHIFT                                                0x16
+#define MMEA0_CGTT_CLK_CTRL__SPARE1__SHIFT                                                                    0x17
+#define MMEA0_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                               0x1b
+#define MMEA0_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE__SHIFT                                                       0x1c
+#define MMEA0_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ__SHIFT                                                        0x1d
+#define MMEA0_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                      0x1e
+#define MMEA0_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                    0x1f
+#define MMEA0_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define MMEA0_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define MMEA0_CGTT_CLK_CTRL__SPARE0_MASK                                                                      0x000FF000L
+#define MMEA0_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE_MASK                                                   0x00100000L
+#define MMEA0_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ_MASK                                                    0x00200000L
+#define MMEA0_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN_MASK                                                  0x00400000L
+#define MMEA0_CGTT_CLK_CTRL__SPARE1_MASK                                                                      0x07800000L
+#define MMEA0_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                                 0x08000000L
+#define MMEA0_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE_MASK                                                         0x10000000L
+#define MMEA0_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ_MASK                                                          0x20000000L
+#define MMEA0_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                        0x40000000L
+#define MMEA0_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                      0x80000000L
+//MMEA0_EDC_MODE
+#define MMEA0_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                  0x10
+#define MMEA0_EDC_MODE__GATE_FUE__SHIFT                                                                       0x11
+#define MMEA0_EDC_MODE__DED_MODE__SHIFT                                                                       0x14
+#define MMEA0_EDC_MODE__PROP_FED__SHIFT                                                                       0x1d
+#define MMEA0_EDC_MODE__BYPASS__SHIFT                                                                         0x1f
+#define MMEA0_EDC_MODE__COUNT_FED_OUT_MASK                                                                    0x00010000L
+#define MMEA0_EDC_MODE__GATE_FUE_MASK                                                                         0x00020000L
+#define MMEA0_EDC_MODE__DED_MODE_MASK                                                                         0x00300000L
+#define MMEA0_EDC_MODE__PROP_FED_MASK                                                                         0x20000000L
+#define MMEA0_EDC_MODE__BYPASS_MASK                                                                           0x80000000L
+//MMEA0_ERR_STATUS
+#define MMEA0_ERR_STATUS__SDP_RDRSP_STATUS__SHIFT                                                             0x0
+#define MMEA0_ERR_STATUS__SDP_WRRSP_STATUS__SHIFT                                                             0x4
+#define MMEA0_ERR_STATUS__SDP_RDRSP_DATASTATUS__SHIFT                                                         0x8
+#define MMEA0_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR__SHIFT                                                   0xa
+#define MMEA0_ERR_STATUS__CLEAR_ERROR_STATUS__SHIFT                                                           0xb
+#define MMEA0_ERR_STATUS__BUSY_ON_ERROR__SHIFT                                                                0xc
+#define MMEA0_ERR_STATUS__FUE_FLAG__SHIFT                                                                     0xd
+#define MMEA0_ERR_STATUS__IGNORE_RDRSP_FED__SHIFT                                                             0xe
+#define MMEA0_ERR_STATUS__INTERRUPT_ON_FATAL__SHIFT                                                           0xf
+#define MMEA0_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL__SHIFT                                                   0x10
+#define MMEA0_ERR_STATUS__LEVEL_INTERRUPT__SHIFT                                                              0x11
+#define MMEA0_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR__SHIFT                                                     0x12
+#define MMEA0_ERR_STATUS__FUE_FLAG_CLIENT__SHIFT                                                              0x13
+#define MMEA0_ERR_STATUS__SDP_RDRSP_STATUS_MASK                                                               0x0000000FL
+#define MMEA0_ERR_STATUS__SDP_WRRSP_STATUS_MASK                                                               0x000000F0L
+#define MMEA0_ERR_STATUS__SDP_RDRSP_DATASTATUS_MASK                                                           0x00000300L
+#define MMEA0_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR_MASK                                                     0x00000400L
+#define MMEA0_ERR_STATUS__CLEAR_ERROR_STATUS_MASK                                                             0x00000800L
+#define MMEA0_ERR_STATUS__BUSY_ON_ERROR_MASK                                                                  0x00001000L
+#define MMEA0_ERR_STATUS__FUE_FLAG_MASK                                                                       0x00002000L
+#define MMEA0_ERR_STATUS__IGNORE_RDRSP_FED_MASK                                                               0x00004000L
+#define MMEA0_ERR_STATUS__INTERRUPT_ON_FATAL_MASK                                                             0x00008000L
+#define MMEA0_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL_MASK                                                     0x00010000L
+#define MMEA0_ERR_STATUS__LEVEL_INTERRUPT_MASK                                                                0x00020000L
+#define MMEA0_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR_MASK                                                       0x00040000L
+#define MMEA0_ERR_STATUS__FUE_FLAG_CLIENT_MASK                                                                0x00080000L
+//MMEA0_MISC2
+#define MMEA0_MISC2__CSGROUP_SWAP_IN_DRAM_ARB__SHIFT                                                          0x0
+#define MMEA0_MISC2__CSGROUP_SWAP_IN_GMI_ARB__SHIFT                                                           0x1
+#define MMEA0_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM__SHIFT                                                       0x2
+#define MMEA0_MISC2__CSGRP_BURST_LIMIT_DATA_GMI__SHIFT                                                        0x7
+#define MMEA0_MISC2__IO_RDWR_PRIORITY_ENABLE__SHIFT                                                           0xc
+#define MMEA0_MISC2__RRET_SWAP_MODE__SHIFT                                                                    0xd
+#define MMEA0_MISC2__BLOCK_REQUESTS__SHIFT                                                                    0xe
+#define MMEA0_MISC2__REQUESTS_BLOCKED__SHIFT                                                                  0xf
+#define MMEA0_MISC2__DRAM_RD_THROTTLE__SHIFT                                                                  0x10
+#define MMEA0_MISC2__DRAM_WR_THROTTLE__SHIFT                                                                  0x11
+#define MMEA0_MISC2__GMI_RD_THROTTLE__SHIFT                                                                   0x12
+#define MMEA0_MISC2__GMI_WR_THROTTLE__SHIFT                                                                   0x13
+#define MMEA0_MISC2__CSGROUP_SWAP_IN_DRAM_ARB_MASK                                                            0x00000001L
+#define MMEA0_MISC2__CSGROUP_SWAP_IN_GMI_ARB_MASK                                                             0x00000002L
+#define MMEA0_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM_MASK                                                         0x0000007CL
+#define MMEA0_MISC2__CSGRP_BURST_LIMIT_DATA_GMI_MASK                                                          0x00000F80L
+#define MMEA0_MISC2__IO_RDWR_PRIORITY_ENABLE_MASK                                                             0x00001000L
+#define MMEA0_MISC2__RRET_SWAP_MODE_MASK                                                                      0x00002000L
+#define MMEA0_MISC2__BLOCK_REQUESTS_MASK                                                                      0x00004000L
+#define MMEA0_MISC2__REQUESTS_BLOCKED_MASK                                                                    0x00008000L
+#define MMEA0_MISC2__DRAM_RD_THROTTLE_MASK                                                                    0x00010000L
+#define MMEA0_MISC2__DRAM_WR_THROTTLE_MASK                                                                    0x00020000L
+#define MMEA0_MISC2__GMI_RD_THROTTLE_MASK                                                                     0x00040000L
+#define MMEA0_MISC2__GMI_WR_THROTTLE_MASK                                                                     0x00080000L
+//MMEA0_MISC_AON
+#define MMEA0_MISC_AON__LINKMGR_PARTACK_HYSTERESIS__SHIFT                                                     0x0
+#define MMEA0_MISC_AON__LINKMGR_PARTACK_DEASSERT_MODE__SHIFT                                                  0x2
+#define MMEA0_MISC_AON__LINKMGR_PARTACK_HYSTERESIS_MASK                                                       0x00000003L
+#define MMEA0_MISC_AON__LINKMGR_PARTACK_DEASSERT_MODE_MASK                                                    0x00000004L
+
+
+// addressBlock: aid_mmhub_ea_mmeadec1
+//MMEA1_DRAM_RD_CLI2GRP_MAP0
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                         0x0
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                         0x2
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                         0x4
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                         0x6
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                         0x8
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                         0xa
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                         0xc
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                         0xe
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                         0x10
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                         0x12
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                        0x14
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                        0x16
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                        0x18
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                        0x1a
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                        0x1c
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                        0x1e
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                           0x00000003L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                           0x0000000CL
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                           0x00000030L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                           0x000000C0L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                           0x00000300L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                           0x00000C00L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                           0x00003000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                           0x0000C000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                           0x00030000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                           0x000C0000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                          0x00300000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                          0x00C00000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                          0x03000000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                          0x0C000000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                          0x30000000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                          0xC0000000L
+//MMEA1_DRAM_RD_CLI2GRP_MAP1
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                        0x0
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                        0x2
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                        0x4
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                        0x6
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                        0x8
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                        0xa
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                        0xc
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                        0xe
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                        0x10
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                        0x12
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                        0x14
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                        0x16
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                        0x18
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                        0x1a
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                        0x1c
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                        0x1e
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                          0x00000003L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                          0x0000000CL
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                          0x00000030L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                          0x000000C0L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                          0x00000300L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                          0x00000C00L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                          0x00003000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                          0x0000C000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                          0x00030000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                          0x000C0000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                          0x00300000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                          0x00C00000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                          0x03000000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                          0x0C000000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                          0x30000000L
+#define MMEA1_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                          0xC0000000L
+//MMEA1_DRAM_WR_CLI2GRP_MAP0
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                         0x0
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                         0x2
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                         0x4
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                         0x6
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                         0x8
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                         0xa
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                         0xc
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                         0xe
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                         0x10
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                         0x12
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                        0x14
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                        0x16
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                        0x18
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                        0x1a
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                        0x1c
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                        0x1e
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                           0x00000003L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                           0x0000000CL
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                           0x00000030L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                           0x000000C0L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                           0x00000300L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                           0x00000C00L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                           0x00003000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                           0x0000C000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                           0x00030000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                           0x000C0000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                          0x00300000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                          0x00C00000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                          0x03000000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                          0x0C000000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                          0x30000000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                          0xC0000000L
+//MMEA1_DRAM_WR_CLI2GRP_MAP1
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                        0x0
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                        0x2
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                        0x4
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                        0x6
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                        0x8
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                        0xa
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                        0xc
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                        0xe
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                        0x10
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                        0x12
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                        0x14
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                        0x16
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                        0x18
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                        0x1a
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                        0x1c
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                        0x1e
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                          0x00000003L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                          0x0000000CL
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                          0x00000030L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                          0x000000C0L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                          0x00000300L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                          0x00000C00L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                          0x00003000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                          0x0000C000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                          0x00030000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                          0x000C0000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                          0x00300000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                          0x00C00000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                          0x03000000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                          0x0C000000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                          0x30000000L
+#define MMEA1_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                          0xC0000000L
+//MMEA1_DRAM_RD_GRP2VC_MAP
+#define MMEA1_DRAM_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                            0x0
+#define MMEA1_DRAM_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                            0x3
+#define MMEA1_DRAM_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                            0x6
+#define MMEA1_DRAM_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                            0x9
+#define MMEA1_DRAM_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                              0x00000007L
+#define MMEA1_DRAM_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                              0x00000038L
+#define MMEA1_DRAM_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                              0x000001C0L
+#define MMEA1_DRAM_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                              0x00000E00L
+//MMEA1_DRAM_WR_GRP2VC_MAP
+#define MMEA1_DRAM_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                            0x0
+#define MMEA1_DRAM_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                            0x3
+#define MMEA1_DRAM_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                            0x6
+#define MMEA1_DRAM_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                            0x9
+#define MMEA1_DRAM_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                              0x00000007L
+#define MMEA1_DRAM_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                              0x00000038L
+#define MMEA1_DRAM_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                              0x000001C0L
+#define MMEA1_DRAM_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                              0x00000E00L
+//MMEA1_DRAM_RD_LAZY
+#define MMEA1_DRAM_RD_LAZY__GROUP0_DELAY__SHIFT                                                               0x0
+#define MMEA1_DRAM_RD_LAZY__GROUP1_DELAY__SHIFT                                                               0x3
+#define MMEA1_DRAM_RD_LAZY__GROUP2_DELAY__SHIFT                                                               0x6
+#define MMEA1_DRAM_RD_LAZY__GROUP3_DELAY__SHIFT                                                               0x9
+#define MMEA1_DRAM_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                           0xc
+#define MMEA1_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                          0x14
+#define MMEA1_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                          0x1b
+#define MMEA1_DRAM_RD_LAZY__GROUP0_DELAY_MASK                                                                 0x00000007L
+#define MMEA1_DRAM_RD_LAZY__GROUP1_DELAY_MASK                                                                 0x00000038L
+#define MMEA1_DRAM_RD_LAZY__GROUP2_DELAY_MASK                                                                 0x000001C0L
+#define MMEA1_DRAM_RD_LAZY__GROUP3_DELAY_MASK                                                                 0x00000E00L
+#define MMEA1_DRAM_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                             0x0003F000L
+#define MMEA1_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                            0x07F00000L
+#define MMEA1_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                            0x78000000L
+//MMEA1_DRAM_WR_LAZY
+#define MMEA1_DRAM_WR_LAZY__GROUP0_DELAY__SHIFT                                                               0x0
+#define MMEA1_DRAM_WR_LAZY__GROUP1_DELAY__SHIFT                                                               0x3
+#define MMEA1_DRAM_WR_LAZY__GROUP2_DELAY__SHIFT                                                               0x6
+#define MMEA1_DRAM_WR_LAZY__GROUP3_DELAY__SHIFT                                                               0x9
+#define MMEA1_DRAM_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                           0xc
+#define MMEA1_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                          0x14
+#define MMEA1_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                          0x1b
+#define MMEA1_DRAM_WR_LAZY__GROUP0_DELAY_MASK                                                                 0x00000007L
+#define MMEA1_DRAM_WR_LAZY__GROUP1_DELAY_MASK                                                                 0x00000038L
+#define MMEA1_DRAM_WR_LAZY__GROUP2_DELAY_MASK                                                                 0x000001C0L
+#define MMEA1_DRAM_WR_LAZY__GROUP3_DELAY_MASK                                                                 0x00000E00L
+#define MMEA1_DRAM_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                             0x0003F000L
+#define MMEA1_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                            0x07F00000L
+#define MMEA1_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                            0x78000000L
+//MMEA1_DRAM_RD_CAM_CNTL
+#define MMEA1_DRAM_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                           0x0
+#define MMEA1_DRAM_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                           0x4
+#define MMEA1_DRAM_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                           0x8
+#define MMEA1_DRAM_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                           0xc
+#define MMEA1_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                   0x10
+#define MMEA1_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                   0x13
+#define MMEA1_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                   0x16
+#define MMEA1_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                   0x19
+#define MMEA1_DRAM_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                           0x1c
+#define MMEA1_DRAM_RD_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                     0x1d
+#define MMEA1_DRAM_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                             0x0000000FL
+#define MMEA1_DRAM_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                             0x000000F0L
+#define MMEA1_DRAM_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                             0x00000F00L
+#define MMEA1_DRAM_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                             0x0000F000L
+#define MMEA1_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                     0x00070000L
+#define MMEA1_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                     0x00380000L
+#define MMEA1_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                     0x01C00000L
+#define MMEA1_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                     0x0E000000L
+#define MMEA1_DRAM_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                             0x10000000L
+#define MMEA1_DRAM_RD_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                       0x20000000L
+//MMEA1_DRAM_WR_CAM_CNTL
+#define MMEA1_DRAM_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                           0x0
+#define MMEA1_DRAM_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                           0x4
+#define MMEA1_DRAM_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                           0x8
+#define MMEA1_DRAM_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                           0xc
+#define MMEA1_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                   0x10
+#define MMEA1_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                   0x13
+#define MMEA1_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                   0x16
+#define MMEA1_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                   0x19
+#define MMEA1_DRAM_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                           0x1c
+#define MMEA1_DRAM_WR_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                     0x1d
+#define MMEA1_DRAM_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                             0x0000000FL
+#define MMEA1_DRAM_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                             0x000000F0L
+#define MMEA1_DRAM_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                             0x00000F00L
+#define MMEA1_DRAM_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                             0x0000F000L
+#define MMEA1_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                     0x00070000L
+#define MMEA1_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                     0x00380000L
+#define MMEA1_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                     0x01C00000L
+#define MMEA1_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                     0x0E000000L
+#define MMEA1_DRAM_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                             0x10000000L
+#define MMEA1_DRAM_WR_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                       0x20000000L
+//MMEA1_DRAM_PAGE_BURST
+#define MMEA1_DRAM_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                             0x0
+#define MMEA1_DRAM_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                             0x8
+#define MMEA1_DRAM_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                             0x10
+#define MMEA1_DRAM_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                             0x18
+#define MMEA1_DRAM_PAGE_BURST__RD_LIMIT_LO_MASK                                                               0x000000FFL
+#define MMEA1_DRAM_PAGE_BURST__RD_LIMIT_HI_MASK                                                               0x0000FF00L
+#define MMEA1_DRAM_PAGE_BURST__WR_LIMIT_LO_MASK                                                               0x00FF0000L
+#define MMEA1_DRAM_PAGE_BURST__WR_LIMIT_HI_MASK                                                               0xFF000000L
+//MMEA1_DRAM_RD_PRI_AGE
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                  0xc
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                  0xf
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                  0x12
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                  0x15
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                    0x00007000L
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                    0x00038000L
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                    0x001C0000L
+#define MMEA1_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                    0x00E00000L
+//MMEA1_DRAM_WR_PRI_AGE
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                  0xc
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                  0xf
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                  0x12
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                  0x15
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                    0x00007000L
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                    0x00038000L
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                    0x001C0000L
+#define MMEA1_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                    0x00E00000L
+//MMEA1_DRAM_RD_PRI_QUEUING
+#define MMEA1_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                          0x0
+#define MMEA1_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                          0x3
+#define MMEA1_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                          0x6
+#define MMEA1_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                          0x9
+#define MMEA1_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA1_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA1_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA1_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                            0x00000E00L
+//MMEA1_DRAM_WR_PRI_QUEUING
+#define MMEA1_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                          0x0
+#define MMEA1_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                          0x3
+#define MMEA1_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                          0x6
+#define MMEA1_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                          0x9
+#define MMEA1_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA1_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA1_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA1_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                            0x00000E00L
+//MMEA1_DRAM_RD_PRI_FIXED
+#define MMEA1_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                              0x0
+#define MMEA1_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                              0x3
+#define MMEA1_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                              0x6
+#define MMEA1_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                              0x9
+#define MMEA1_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                0x00000007L
+#define MMEA1_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                0x00000038L
+#define MMEA1_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                0x000001C0L
+#define MMEA1_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                0x00000E00L
+//MMEA1_DRAM_WR_PRI_FIXED
+#define MMEA1_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                              0x0
+#define MMEA1_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                              0x3
+#define MMEA1_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                              0x6
+#define MMEA1_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                              0x9
+#define MMEA1_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                0x00000007L
+#define MMEA1_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                0x00000038L
+#define MMEA1_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                0x000001C0L
+#define MMEA1_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                0x00000E00L
+//MMEA1_DRAM_RD_PRI_URGENCY
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                          0x0
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                          0x3
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                          0x6
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                          0x9
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                 0xc
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                 0xd
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                 0xe
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                 0xf
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                            0x00000E00L
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                   0x00001000L
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                   0x00002000L
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                   0x00004000L
+#define MMEA1_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                   0x00008000L
+//MMEA1_DRAM_WR_PRI_URGENCY
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                          0x0
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                          0x3
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                          0x6
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                          0x9
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                 0xc
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                 0xd
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                 0xe
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                 0xf
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                            0x00000E00L
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                   0x00001000L
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                   0x00002000L
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                   0x00004000L
+#define MMEA1_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                   0x00008000L
+//MMEA1_DRAM_RD_PRI_QUANT_PRI1
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA1_DRAM_RD_PRI_QUANT_PRI2
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA1_DRAM_RD_PRI_QUANT_PRI3
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA1_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA1_DRAM_WR_PRI_QUANT_PRI1
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA1_DRAM_WR_PRI_QUANT_PRI2
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA1_DRAM_WR_PRI_QUANT_PRI3
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA1_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA1_GMI_RD_CLI2GRP_MAP0
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//MMEA1_GMI_RD_CLI2GRP_MAP1
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define MMEA1_GMI_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//MMEA1_GMI_WR_CLI2GRP_MAP0
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//MMEA1_GMI_WR_CLI2GRP_MAP1
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define MMEA1_GMI_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//MMEA1_GMI_RD_GRP2VC_MAP
+#define MMEA1_GMI_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define MMEA1_GMI_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define MMEA1_GMI_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define MMEA1_GMI_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define MMEA1_GMI_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define MMEA1_GMI_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define MMEA1_GMI_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define MMEA1_GMI_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//MMEA1_GMI_WR_GRP2VC_MAP
+#define MMEA1_GMI_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define MMEA1_GMI_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define MMEA1_GMI_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define MMEA1_GMI_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define MMEA1_GMI_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define MMEA1_GMI_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define MMEA1_GMI_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define MMEA1_GMI_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//MMEA1_GMI_RD_LAZY
+#define MMEA1_GMI_RD_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define MMEA1_GMI_RD_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define MMEA1_GMI_RD_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define MMEA1_GMI_RD_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define MMEA1_GMI_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define MMEA1_GMI_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define MMEA1_GMI_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define MMEA1_GMI_RD_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define MMEA1_GMI_RD_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define MMEA1_GMI_RD_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define MMEA1_GMI_RD_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define MMEA1_GMI_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define MMEA1_GMI_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define MMEA1_GMI_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//MMEA1_GMI_WR_LAZY
+#define MMEA1_GMI_WR_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define MMEA1_GMI_WR_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define MMEA1_GMI_WR_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define MMEA1_GMI_WR_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define MMEA1_GMI_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define MMEA1_GMI_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define MMEA1_GMI_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define MMEA1_GMI_WR_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define MMEA1_GMI_WR_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define MMEA1_GMI_WR_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define MMEA1_GMI_WR_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define MMEA1_GMI_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define MMEA1_GMI_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define MMEA1_GMI_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//MMEA1_GMI_RD_CAM_CNTL
+#define MMEA1_GMI_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define MMEA1_GMI_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define MMEA1_GMI_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define MMEA1_GMI_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define MMEA1_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define MMEA1_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define MMEA1_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define MMEA1_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define MMEA1_GMI_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define MMEA1_GMI_RD_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                      0x1d
+#define MMEA1_GMI_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define MMEA1_GMI_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define MMEA1_GMI_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define MMEA1_GMI_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define MMEA1_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define MMEA1_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define MMEA1_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define MMEA1_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define MMEA1_GMI_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+#define MMEA1_GMI_RD_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                        0x20000000L
+//MMEA1_GMI_WR_CAM_CNTL
+#define MMEA1_GMI_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define MMEA1_GMI_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define MMEA1_GMI_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define MMEA1_GMI_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define MMEA1_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define MMEA1_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define MMEA1_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define MMEA1_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define MMEA1_GMI_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define MMEA1_GMI_WR_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                      0x1d
+#define MMEA1_GMI_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define MMEA1_GMI_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define MMEA1_GMI_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define MMEA1_GMI_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define MMEA1_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define MMEA1_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define MMEA1_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define MMEA1_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define MMEA1_GMI_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+#define MMEA1_GMI_WR_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                        0x20000000L
+//MMEA1_GMI_PAGE_BURST
+#define MMEA1_GMI_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define MMEA1_GMI_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define MMEA1_GMI_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define MMEA1_GMI_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define MMEA1_GMI_PAGE_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define MMEA1_GMI_PAGE_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define MMEA1_GMI_PAGE_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define MMEA1_GMI_PAGE_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//MMEA1_GMI_RD_PRI_AGE
+#define MMEA1_GMI_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define MMEA1_GMI_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define MMEA1_GMI_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define MMEA1_GMI_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define MMEA1_GMI_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define MMEA1_GMI_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define MMEA1_GMI_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define MMEA1_GMI_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define MMEA1_GMI_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define MMEA1_GMI_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define MMEA1_GMI_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define MMEA1_GMI_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define MMEA1_GMI_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define MMEA1_GMI_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define MMEA1_GMI_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define MMEA1_GMI_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//MMEA1_GMI_WR_PRI_AGE
+#define MMEA1_GMI_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define MMEA1_GMI_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define MMEA1_GMI_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define MMEA1_GMI_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define MMEA1_GMI_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define MMEA1_GMI_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define MMEA1_GMI_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define MMEA1_GMI_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define MMEA1_GMI_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define MMEA1_GMI_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define MMEA1_GMI_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define MMEA1_GMI_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define MMEA1_GMI_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define MMEA1_GMI_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define MMEA1_GMI_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define MMEA1_GMI_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//MMEA1_GMI_RD_PRI_QUEUING
+#define MMEA1_GMI_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define MMEA1_GMI_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define MMEA1_GMI_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define MMEA1_GMI_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define MMEA1_GMI_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA1_GMI_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA1_GMI_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA1_GMI_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//MMEA1_GMI_WR_PRI_QUEUING
+#define MMEA1_GMI_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define MMEA1_GMI_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define MMEA1_GMI_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define MMEA1_GMI_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define MMEA1_GMI_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA1_GMI_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA1_GMI_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA1_GMI_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//MMEA1_GMI_RD_PRI_FIXED
+#define MMEA1_GMI_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define MMEA1_GMI_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define MMEA1_GMI_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define MMEA1_GMI_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define MMEA1_GMI_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define MMEA1_GMI_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define MMEA1_GMI_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define MMEA1_GMI_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//MMEA1_GMI_WR_PRI_FIXED
+#define MMEA1_GMI_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define MMEA1_GMI_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define MMEA1_GMI_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define MMEA1_GMI_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define MMEA1_GMI_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define MMEA1_GMI_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define MMEA1_GMI_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define MMEA1_GMI_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//MMEA1_GMI_RD_PRI_URGENCY
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define MMEA1_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//MMEA1_GMI_WR_PRI_URGENCY
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define MMEA1_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//MMEA1_GMI_RD_PRI_URGENCY_MASKING
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                    0x0
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                    0x1
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                    0x2
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                    0x3
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                    0x4
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                    0x5
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                    0x6
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                    0x7
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                    0x8
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                    0x9
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                   0xa
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                   0xb
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                   0xc
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                   0xd
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                   0xe
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                   0xf
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                   0x10
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                   0x11
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                   0x12
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                   0x13
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                   0x14
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                   0x15
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                   0x16
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                   0x17
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                   0x18
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                   0x19
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                   0x1a
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                   0x1b
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                   0x1c
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                   0x1d
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                   0x1e
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                   0x1f
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                      0x00000001L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                      0x00000002L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                      0x00000004L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                      0x00000008L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                      0x00000010L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                      0x00000020L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                      0x00000040L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                      0x00000080L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                      0x00000100L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                      0x00000200L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                     0x00000400L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                     0x00000800L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                     0x00001000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                     0x00002000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                     0x00004000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                     0x00008000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                     0x00010000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                     0x00020000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                     0x00040000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                     0x00080000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                     0x00100000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                     0x00200000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                     0x00400000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                     0x00800000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                     0x01000000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                     0x02000000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                     0x04000000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                     0x08000000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                     0x10000000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                     0x20000000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                     0x40000000L
+#define MMEA1_GMI_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                     0x80000000L
+//MMEA1_GMI_WR_PRI_URGENCY_MASKING
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                    0x0
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                    0x1
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                    0x2
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                    0x3
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                    0x4
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                    0x5
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                    0x6
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                    0x7
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                    0x8
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                    0x9
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                   0xa
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                   0xb
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                   0xc
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                   0xd
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                   0xe
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                   0xf
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                   0x10
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                   0x11
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                   0x12
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                   0x13
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                   0x14
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                   0x15
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                   0x16
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                   0x17
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                   0x18
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                   0x19
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                   0x1a
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                   0x1b
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                   0x1c
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                   0x1d
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                   0x1e
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                   0x1f
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                      0x00000001L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                      0x00000002L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                      0x00000004L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                      0x00000008L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                      0x00000010L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                      0x00000020L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                      0x00000040L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                      0x00000080L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                      0x00000100L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                      0x00000200L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                     0x00000400L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                     0x00000800L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                     0x00001000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                     0x00002000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                     0x00004000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                     0x00008000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                     0x00010000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                     0x00020000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                     0x00040000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                     0x00080000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                     0x00100000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                     0x00200000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                     0x00400000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                     0x00800000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                     0x01000000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                     0x02000000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                     0x04000000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                     0x08000000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                     0x10000000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                     0x20000000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                     0x40000000L
+#define MMEA1_GMI_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                     0x80000000L
+//MMEA1_GMI_RD_PRI_QUANT_PRI1
+#define MMEA1_GMI_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA1_GMI_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA1_GMI_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA1_GMI_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA1_GMI_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA1_GMI_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA1_GMI_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA1_GMI_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA1_GMI_RD_PRI_QUANT_PRI2
+#define MMEA1_GMI_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA1_GMI_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA1_GMI_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA1_GMI_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA1_GMI_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA1_GMI_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA1_GMI_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA1_GMI_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA1_GMI_RD_PRI_QUANT_PRI3
+#define MMEA1_GMI_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA1_GMI_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA1_GMI_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA1_GMI_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA1_GMI_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA1_GMI_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA1_GMI_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA1_GMI_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA1_GMI_WR_PRI_QUANT_PRI1
+#define MMEA1_GMI_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA1_GMI_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA1_GMI_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA1_GMI_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA1_GMI_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA1_GMI_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA1_GMI_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA1_GMI_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA1_GMI_WR_PRI_QUANT_PRI2
+#define MMEA1_GMI_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA1_GMI_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA1_GMI_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA1_GMI_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA1_GMI_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA1_GMI_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA1_GMI_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA1_GMI_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA1_GMI_WR_PRI_QUANT_PRI3
+#define MMEA1_GMI_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA1_GMI_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA1_GMI_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA1_GMI_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA1_GMI_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA1_GMI_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA1_GMI_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA1_GMI_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA1_IO_RD_CLI2GRP_MAP0
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                           0x0
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                           0x2
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                           0x4
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                           0x6
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                           0x8
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                           0xa
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                           0xc
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                           0xe
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                           0x10
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                           0x12
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                          0x14
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                          0x16
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                          0x18
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                          0x1a
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                          0x1c
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                          0x1e
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                             0x00000003L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                             0x0000000CL
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                             0x00000030L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                             0x000000C0L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                             0x00000300L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                             0x00000C00L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                             0x00003000L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                             0x0000C000L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                             0x00030000L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                             0x000C0000L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                            0x00300000L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                            0x00C00000L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                            0x03000000L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                            0x0C000000L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                            0x30000000L
+#define MMEA1_IO_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                            0xC0000000L
+//MMEA1_IO_RD_CLI2GRP_MAP1
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                          0x0
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                          0x2
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                          0x4
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                          0x6
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                          0x8
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                          0xa
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                          0xc
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                          0xe
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                          0x10
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                          0x12
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                          0x14
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                          0x16
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                          0x18
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                          0x1a
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                          0x1c
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                          0x1e
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                            0x00000003L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                            0x0000000CL
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                            0x00000030L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                            0x000000C0L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                            0x00000300L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                            0x00000C00L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                            0x00003000L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                            0x0000C000L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                            0x00030000L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                            0x000C0000L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                            0x00300000L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                            0x00C00000L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                            0x03000000L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                            0x0C000000L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                            0x30000000L
+#define MMEA1_IO_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                            0xC0000000L
+//MMEA1_IO_WR_CLI2GRP_MAP0
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                           0x0
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                           0x2
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                           0x4
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                           0x6
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                           0x8
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                           0xa
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                           0xc
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                           0xe
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                           0x10
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                           0x12
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                          0x14
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                          0x16
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                          0x18
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                          0x1a
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                          0x1c
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                          0x1e
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                             0x00000003L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                             0x0000000CL
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                             0x00000030L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                             0x000000C0L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                             0x00000300L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                             0x00000C00L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                             0x00003000L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                             0x0000C000L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                             0x00030000L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                             0x000C0000L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                            0x00300000L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                            0x00C00000L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                            0x03000000L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                            0x0C000000L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                            0x30000000L
+#define MMEA1_IO_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                            0xC0000000L
+//MMEA1_IO_WR_CLI2GRP_MAP1
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                          0x0
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                          0x2
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                          0x4
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                          0x6
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                          0x8
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                          0xa
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                          0xc
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                          0xe
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                          0x10
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                          0x12
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                          0x14
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                          0x16
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                          0x18
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                          0x1a
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                          0x1c
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                          0x1e
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                            0x00000003L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                            0x0000000CL
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                            0x00000030L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                            0x000000C0L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                            0x00000300L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                            0x00000C00L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                            0x00003000L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                            0x0000C000L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                            0x00030000L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                            0x000C0000L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                            0x00300000L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                            0x00C00000L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                            0x03000000L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                            0x0C000000L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                            0x30000000L
+#define MMEA1_IO_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                            0xC0000000L
+//MMEA1_IO_RD_COMBINE_FLUSH
+#define MMEA1_IO_RD_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                        0x0
+#define MMEA1_IO_RD_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                        0x4
+#define MMEA1_IO_RD_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                        0x8
+#define MMEA1_IO_RD_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                        0xc
+#define MMEA1_IO_RD_COMBINE_FLUSH__COMB_MODE__SHIFT                                                           0x10
+#define MMEA1_IO_RD_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                          0x0000000FL
+#define MMEA1_IO_RD_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                          0x000000F0L
+#define MMEA1_IO_RD_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                          0x00000F00L
+#define MMEA1_IO_RD_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                          0x0000F000L
+#define MMEA1_IO_RD_COMBINE_FLUSH__COMB_MODE_MASK                                                             0x00030000L
+//MMEA1_IO_WR_COMBINE_FLUSH
+#define MMEA1_IO_WR_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                        0x0
+#define MMEA1_IO_WR_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                        0x4
+#define MMEA1_IO_WR_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                        0x8
+#define MMEA1_IO_WR_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                        0xc
+#define MMEA1_IO_WR_COMBINE_FLUSH__COMB_MODE__SHIFT                                                           0x10
+#define MMEA1_IO_WR_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                          0x0000000FL
+#define MMEA1_IO_WR_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                          0x000000F0L
+#define MMEA1_IO_WR_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                          0x00000F00L
+#define MMEA1_IO_WR_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                          0x0000F000L
+#define MMEA1_IO_WR_COMBINE_FLUSH__COMB_MODE_MASK                                                             0x00030000L
+//MMEA1_IO_GROUP_BURST
+#define MMEA1_IO_GROUP_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define MMEA1_IO_GROUP_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define MMEA1_IO_GROUP_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define MMEA1_IO_GROUP_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define MMEA1_IO_GROUP_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define MMEA1_IO_GROUP_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define MMEA1_IO_GROUP_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define MMEA1_IO_GROUP_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//MMEA1_IO_RD_PRI_AGE
+#define MMEA1_IO_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                         0x0
+#define MMEA1_IO_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                         0x3
+#define MMEA1_IO_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                         0x6
+#define MMEA1_IO_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                         0x9
+#define MMEA1_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                    0xc
+#define MMEA1_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                    0xf
+#define MMEA1_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                    0x12
+#define MMEA1_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                    0x15
+#define MMEA1_IO_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                           0x00000007L
+#define MMEA1_IO_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                           0x00000038L
+#define MMEA1_IO_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                           0x000001C0L
+#define MMEA1_IO_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                           0x00000E00L
+#define MMEA1_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                      0x00007000L
+#define MMEA1_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                      0x00038000L
+#define MMEA1_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                      0x001C0000L
+#define MMEA1_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                      0x00E00000L
+//MMEA1_IO_WR_PRI_AGE
+#define MMEA1_IO_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                         0x0
+#define MMEA1_IO_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                         0x3
+#define MMEA1_IO_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                         0x6
+#define MMEA1_IO_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                         0x9
+#define MMEA1_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                    0xc
+#define MMEA1_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                    0xf
+#define MMEA1_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                    0x12
+#define MMEA1_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                    0x15
+#define MMEA1_IO_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                           0x00000007L
+#define MMEA1_IO_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                           0x00000038L
+#define MMEA1_IO_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                           0x000001C0L
+#define MMEA1_IO_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                           0x00000E00L
+#define MMEA1_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                      0x00007000L
+#define MMEA1_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                      0x00038000L
+#define MMEA1_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                      0x001C0000L
+#define MMEA1_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                      0x00E00000L
+//MMEA1_IO_RD_PRI_QUEUING
+#define MMEA1_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                            0x0
+#define MMEA1_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                            0x3
+#define MMEA1_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                            0x6
+#define MMEA1_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                            0x9
+#define MMEA1_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA1_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA1_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA1_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                              0x00000E00L
+//MMEA1_IO_WR_PRI_QUEUING
+#define MMEA1_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                            0x0
+#define MMEA1_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                            0x3
+#define MMEA1_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                            0x6
+#define MMEA1_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                            0x9
+#define MMEA1_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA1_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA1_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA1_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                              0x00000E00L
+//MMEA1_IO_RD_PRI_FIXED
+#define MMEA1_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                0x0
+#define MMEA1_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                0x3
+#define MMEA1_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                0x6
+#define MMEA1_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                0x9
+#define MMEA1_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                  0x00000007L
+#define MMEA1_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                  0x00000038L
+#define MMEA1_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                  0x000001C0L
+#define MMEA1_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                  0x00000E00L
+//MMEA1_IO_WR_PRI_FIXED
+#define MMEA1_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                0x0
+#define MMEA1_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                0x3
+#define MMEA1_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                0x6
+#define MMEA1_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                0x9
+#define MMEA1_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                  0x00000007L
+#define MMEA1_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                  0x00000038L
+#define MMEA1_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                  0x000001C0L
+#define MMEA1_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                  0x00000E00L
+//MMEA1_IO_RD_PRI_URGENCY
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                            0x0
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                            0x3
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                            0x6
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                            0x9
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                   0xc
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                   0xd
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                   0xe
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                   0xf
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                              0x00000E00L
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                     0x00001000L
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                     0x00002000L
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                     0x00004000L
+#define MMEA1_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                     0x00008000L
+//MMEA1_IO_WR_PRI_URGENCY
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                            0x0
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                            0x3
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                            0x6
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                            0x9
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                   0xc
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                   0xd
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                   0xe
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                   0xf
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                              0x00000E00L
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                     0x00001000L
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                     0x00002000L
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                     0x00004000L
+#define MMEA1_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                     0x00008000L
+//MMEA1_IO_RD_PRI_URGENCY_MASKING
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                     0x0
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                     0x1
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                     0x2
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                     0x3
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                     0x4
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                     0x5
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                     0x6
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                     0x7
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                     0x8
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                     0x9
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                    0xa
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                    0xb
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                    0xc
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                    0xd
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                    0xe
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                    0xf
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                    0x10
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                    0x11
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                    0x12
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                    0x13
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                    0x14
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                    0x15
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                    0x16
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                    0x17
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                    0x18
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                    0x19
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                    0x1a
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                    0x1b
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                    0x1c
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                    0x1d
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                    0x1e
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                    0x1f
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                       0x00000001L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                       0x00000002L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                       0x00000004L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                       0x00000008L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                       0x00000010L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                       0x00000020L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                       0x00000040L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                       0x00000080L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                       0x00000100L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                       0x00000200L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                      0x00000400L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                      0x00000800L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                      0x00001000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                      0x00002000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                      0x00004000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                      0x00008000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                      0x00010000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                      0x00020000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                      0x00040000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                      0x00080000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                      0x00100000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                      0x00200000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                      0x00400000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                      0x00800000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                      0x01000000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                      0x02000000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                      0x04000000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                      0x08000000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                      0x10000000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                      0x20000000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                      0x40000000L
+#define MMEA1_IO_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                      0x80000000L
+//MMEA1_IO_WR_PRI_URGENCY_MASKING
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                     0x0
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                     0x1
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                     0x2
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                     0x3
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                     0x4
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                     0x5
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                     0x6
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                     0x7
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                     0x8
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                     0x9
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                    0xa
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                    0xb
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                    0xc
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                    0xd
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                    0xe
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                    0xf
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                    0x10
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                    0x11
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                    0x12
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                    0x13
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                    0x14
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                    0x15
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                    0x16
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                    0x17
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                    0x18
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                    0x19
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                    0x1a
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                    0x1b
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                    0x1c
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                    0x1d
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                    0x1e
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                    0x1f
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                       0x00000001L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                       0x00000002L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                       0x00000004L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                       0x00000008L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                       0x00000010L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                       0x00000020L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                       0x00000040L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                       0x00000080L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                       0x00000100L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                       0x00000200L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                      0x00000400L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                      0x00000800L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                      0x00001000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                      0x00002000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                      0x00004000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                      0x00008000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                      0x00010000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                      0x00020000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                      0x00040000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                      0x00080000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                      0x00100000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                      0x00200000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                      0x00400000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                      0x00800000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                      0x01000000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                      0x02000000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                      0x04000000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                      0x08000000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                      0x10000000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                      0x20000000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                      0x40000000L
+#define MMEA1_IO_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                      0x80000000L
+//MMEA1_IO_RD_PRI_QUANT_PRI1
+#define MMEA1_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA1_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA1_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA1_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA1_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA1_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA1_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA1_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA1_IO_RD_PRI_QUANT_PRI2
+#define MMEA1_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA1_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA1_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA1_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA1_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA1_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA1_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA1_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA1_IO_RD_PRI_QUANT_PRI3
+#define MMEA1_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA1_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA1_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA1_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA1_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA1_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA1_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA1_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA1_IO_WR_PRI_QUANT_PRI1
+#define MMEA1_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA1_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA1_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA1_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA1_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA1_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA1_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA1_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA1_IO_WR_PRI_QUANT_PRI2
+#define MMEA1_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA1_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA1_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA1_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA1_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA1_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA1_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA1_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA1_IO_WR_PRI_QUANT_PRI3
+#define MMEA1_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA1_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA1_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA1_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA1_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA1_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA1_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA1_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA1_SDP_ARB_DRAM
+#define MMEA1_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL__SHIFT                                                      0x0
+#define MMEA1_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA__SHIFT                                                      0x8
+#define MMEA1_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI__SHIFT                                                         0x10
+#define MMEA1_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI__SHIFT                                                         0x11
+#define MMEA1_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES__SHIFT                                                         0x12
+#define MMEA1_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES__SHIFT                                                         0x13
+#define MMEA1_SDP_ARB_DRAM__EOB_ON_EXPIRE__SHIFT                                                              0x14
+#define MMEA1_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                     0x15
+#define MMEA1_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING__SHIFT                                                       0x16
+#define MMEA1_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL_MASK                                                        0x0000007FL
+#define MMEA1_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA_MASK                                                        0x00007F00L
+#define MMEA1_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI_MASK                                                           0x00010000L
+#define MMEA1_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI_MASK                                                           0x00020000L
+#define MMEA1_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES_MASK                                                           0x00040000L
+#define MMEA1_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES_MASK                                                           0x00080000L
+#define MMEA1_SDP_ARB_DRAM__EOB_ON_EXPIRE_MASK                                                                0x00100000L
+#define MMEA1_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE_MASK                                                       0x00200000L
+#define MMEA1_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING_MASK                                                         0x00400000L
+//MMEA1_SDP_ARB_GMI
+#define MMEA1_SDP_ARB_GMI__RDWR_BURST_LIMIT_CYCL__SHIFT                                                       0x0
+#define MMEA1_SDP_ARB_GMI__RDWR_BURST_LIMIT_DATA__SHIFT                                                       0x8
+#define MMEA1_SDP_ARB_GMI__EARLY_SW2RD_ON_PRI__SHIFT                                                          0x10
+#define MMEA1_SDP_ARB_GMI__EARLY_SW2WR_ON_PRI__SHIFT                                                          0x11
+#define MMEA1_SDP_ARB_GMI__EARLY_SW2RD_ON_RES__SHIFT                                                          0x12
+#define MMEA1_SDP_ARB_GMI__EARLY_SW2WR_ON_RES__SHIFT                                                          0x13
+#define MMEA1_SDP_ARB_GMI__EOB_ON_EXPIRE__SHIFT                                                               0x14
+#define MMEA1_SDP_ARB_GMI__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                      0x15
+#define MMEA1_SDP_ARB_GMI__ALLOW_CHAIN_BREAKING__SHIFT                                                        0x16
+#define MMEA1_SDP_ARB_GMI__RDWR_BURST_LIMIT_CYCL_MASK                                                         0x0000007FL
+#define MMEA1_SDP_ARB_GMI__RDWR_BURST_LIMIT_DATA_MASK                                                         0x00007F00L
+#define MMEA1_SDP_ARB_GMI__EARLY_SW2RD_ON_PRI_MASK                                                            0x00010000L
+#define MMEA1_SDP_ARB_GMI__EARLY_SW2WR_ON_PRI_MASK                                                            0x00020000L
+#define MMEA1_SDP_ARB_GMI__EARLY_SW2RD_ON_RES_MASK                                                            0x00040000L
+#define MMEA1_SDP_ARB_GMI__EARLY_SW2WR_ON_RES_MASK                                                            0x00080000L
+#define MMEA1_SDP_ARB_GMI__EOB_ON_EXPIRE_MASK                                                                 0x00100000L
+#define MMEA1_SDP_ARB_GMI__DECOUPLE_RDWR_BNKSTATE_MASK                                                        0x00200000L
+#define MMEA1_SDP_ARB_GMI__ALLOW_CHAIN_BREAKING_MASK                                                          0x00400000L
+//MMEA1_SDP_ARB_FINAL
+#define MMEA1_SDP_ARB_FINAL__DRAM_BURST_LIMIT__SHIFT                                                          0x0
+#define MMEA1_SDP_ARB_FINAL__GMI_BURST_LIMIT__SHIFT                                                           0x5
+#define MMEA1_SDP_ARB_FINAL__IO_BURST_LIMIT__SHIFT                                                            0xa
+#define MMEA1_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER__SHIFT                                                    0xf
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC0__SHIFT                                                                0x11
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC1__SHIFT                                                                0x12
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC2__SHIFT                                                                0x13
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC3__SHIFT                                                                0x14
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC4__SHIFT                                                                0x15
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC5__SHIFT                                                                0x16
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC6__SHIFT                                                                0x17
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC7__SHIFT                                                                0x18
+#define MMEA1_SDP_ARB_FINAL__ERREVENT_ON_ERROR__SHIFT                                                         0x19
+#define MMEA1_SDP_ARB_FINAL__HALTREQ_ON_ERROR__SHIFT                                                          0x1a
+#define MMEA1_SDP_ARB_FINAL__DRAM_BURST_STRETCH__SHIFT                                                        0x1b
+#define MMEA1_SDP_ARB_FINAL__GMI_BURST_STRETCH__SHIFT                                                         0x1c
+#define MMEA1_SDP_ARB_FINAL__DRAM_BURST_LIMIT_MASK                                                            0x0000001FL
+#define MMEA1_SDP_ARB_FINAL__GMI_BURST_LIMIT_MASK                                                             0x000003E0L
+#define MMEA1_SDP_ARB_FINAL__IO_BURST_LIMIT_MASK                                                              0x00007C00L
+#define MMEA1_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER_MASK                                                      0x00018000L
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC0_MASK                                                                  0x00020000L
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC1_MASK                                                                  0x00040000L
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC2_MASK                                                                  0x00080000L
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC3_MASK                                                                  0x00100000L
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC4_MASK                                                                  0x00200000L
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC5_MASK                                                                  0x00400000L
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC6_MASK                                                                  0x00800000L
+#define MMEA1_SDP_ARB_FINAL__RDONLY_VC7_MASK                                                                  0x01000000L
+#define MMEA1_SDP_ARB_FINAL__ERREVENT_ON_ERROR_MASK                                                           0x02000000L
+#define MMEA1_SDP_ARB_FINAL__HALTREQ_ON_ERROR_MASK                                                            0x04000000L
+#define MMEA1_SDP_ARB_FINAL__DRAM_BURST_STRETCH_MASK                                                          0x08000000L
+#define MMEA1_SDP_ARB_FINAL__GMI_BURST_STRETCH_MASK                                                           0x10000000L
+//MMEA1_SDP_DRAM_PRIORITY
+#define MMEA1_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                    0x0
+#define MMEA1_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                    0x4
+#define MMEA1_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                    0x8
+#define MMEA1_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                    0xc
+#define MMEA1_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                    0x10
+#define MMEA1_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                    0x14
+#define MMEA1_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                    0x18
+#define MMEA1_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                    0x1c
+#define MMEA1_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                      0x0000000FL
+#define MMEA1_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                      0x000000F0L
+#define MMEA1_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                      0x00000F00L
+#define MMEA1_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                      0x0000F000L
+#define MMEA1_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                      0x000F0000L
+#define MMEA1_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                      0x00F00000L
+#define MMEA1_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                      0x0F000000L
+#define MMEA1_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                      0xF0000000L
+//MMEA1_SDP_GMI_PRIORITY
+#define MMEA1_SDP_GMI_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                     0x0
+#define MMEA1_SDP_GMI_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                     0x4
+#define MMEA1_SDP_GMI_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                     0x8
+#define MMEA1_SDP_GMI_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                     0xc
+#define MMEA1_SDP_GMI_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                     0x10
+#define MMEA1_SDP_GMI_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                     0x14
+#define MMEA1_SDP_GMI_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                     0x18
+#define MMEA1_SDP_GMI_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                     0x1c
+#define MMEA1_SDP_GMI_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                       0x0000000FL
+#define MMEA1_SDP_GMI_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                       0x000000F0L
+#define MMEA1_SDP_GMI_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                       0x00000F00L
+#define MMEA1_SDP_GMI_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                       0x0000F000L
+#define MMEA1_SDP_GMI_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                       0x000F0000L
+#define MMEA1_SDP_GMI_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                       0x00F00000L
+#define MMEA1_SDP_GMI_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                       0x0F000000L
+#define MMEA1_SDP_GMI_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                       0xF0000000L
+//MMEA1_SDP_IO_PRIORITY
+#define MMEA1_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                      0x0
+#define MMEA1_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                      0x4
+#define MMEA1_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                      0x8
+#define MMEA1_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                      0xc
+#define MMEA1_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                      0x10
+#define MMEA1_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                      0x14
+#define MMEA1_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                      0x18
+#define MMEA1_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                      0x1c
+#define MMEA1_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                        0x0000000FL
+#define MMEA1_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                        0x000000F0L
+#define MMEA1_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                        0x00000F00L
+#define MMEA1_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                        0x0000F000L
+#define MMEA1_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                        0x000F0000L
+#define MMEA1_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                        0x00F00000L
+#define MMEA1_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                        0x0F000000L
+#define MMEA1_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                        0xF0000000L
+//MMEA1_SDP_CREDITS
+#define MMEA1_SDP_CREDITS__TAG_LIMIT__SHIFT                                                                   0x0
+#define MMEA1_SDP_CREDITS__WR_RESP_CREDITS__SHIFT                                                             0x8
+#define MMEA1_SDP_CREDITS__RD_RESP_CREDITS__SHIFT                                                             0x10
+#define MMEA1_SDP_CREDITS__TAG_LIMIT_MASK                                                                     0x000000FFL
+#define MMEA1_SDP_CREDITS__WR_RESP_CREDITS_MASK                                                               0x00007F00L
+#define MMEA1_SDP_CREDITS__RD_RESP_CREDITS_MASK                                                               0x007F0000L
+//MMEA1_SDP_TAG_RESERVE0
+#define MMEA1_SDP_TAG_RESERVE0__VC0__SHIFT                                                                    0x0
+#define MMEA1_SDP_TAG_RESERVE0__VC1__SHIFT                                                                    0x8
+#define MMEA1_SDP_TAG_RESERVE0__VC2__SHIFT                                                                    0x10
+#define MMEA1_SDP_TAG_RESERVE0__VC3__SHIFT                                                                    0x18
+#define MMEA1_SDP_TAG_RESERVE0__VC0_MASK                                                                      0x000000FFL
+#define MMEA1_SDP_TAG_RESERVE0__VC1_MASK                                                                      0x0000FF00L
+#define MMEA1_SDP_TAG_RESERVE0__VC2_MASK                                                                      0x00FF0000L
+#define MMEA1_SDP_TAG_RESERVE0__VC3_MASK                                                                      0xFF000000L
+//MMEA1_SDP_TAG_RESERVE1
+#define MMEA1_SDP_TAG_RESERVE1__VC4__SHIFT                                                                    0x0
+#define MMEA1_SDP_TAG_RESERVE1__VC5__SHIFT                                                                    0x8
+#define MMEA1_SDP_TAG_RESERVE1__VC6__SHIFT                                                                    0x10
+#define MMEA1_SDP_TAG_RESERVE1__VC7__SHIFT                                                                    0x18
+#define MMEA1_SDP_TAG_RESERVE1__VC4_MASK                                                                      0x000000FFL
+#define MMEA1_SDP_TAG_RESERVE1__VC5_MASK                                                                      0x0000FF00L
+#define MMEA1_SDP_TAG_RESERVE1__VC6_MASK                                                                      0x00FF0000L
+#define MMEA1_SDP_TAG_RESERVE1__VC7_MASK                                                                      0xFF000000L
+//MMEA1_SDP_VCC_RESERVE0
+#define MMEA1_SDP_VCC_RESERVE0__VC0_CREDITS__SHIFT                                                            0x0
+#define MMEA1_SDP_VCC_RESERVE0__VC1_CREDITS__SHIFT                                                            0x6
+#define MMEA1_SDP_VCC_RESERVE0__VC2_CREDITS__SHIFT                                                            0xc
+#define MMEA1_SDP_VCC_RESERVE0__VC3_CREDITS__SHIFT                                                            0x12
+#define MMEA1_SDP_VCC_RESERVE0__VC4_CREDITS__SHIFT                                                            0x18
+#define MMEA1_SDP_VCC_RESERVE0__VC0_CREDITS_MASK                                                              0x0000003FL
+#define MMEA1_SDP_VCC_RESERVE0__VC1_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA1_SDP_VCC_RESERVE0__VC2_CREDITS_MASK                                                              0x0003F000L
+#define MMEA1_SDP_VCC_RESERVE0__VC3_CREDITS_MASK                                                              0x00FC0000L
+#define MMEA1_SDP_VCC_RESERVE0__VC4_CREDITS_MASK                                                              0x3F000000L
+//MMEA1_SDP_VCC_RESERVE1
+#define MMEA1_SDP_VCC_RESERVE1__VC5_CREDITS__SHIFT                                                            0x0
+#define MMEA1_SDP_VCC_RESERVE1__VC6_CREDITS__SHIFT                                                            0x6
+#define MMEA1_SDP_VCC_RESERVE1__VC7_CREDITS__SHIFT                                                            0xc
+#define MMEA1_SDP_VCC_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                        0x1f
+#define MMEA1_SDP_VCC_RESERVE1__VC5_CREDITS_MASK                                                              0x0000003FL
+#define MMEA1_SDP_VCC_RESERVE1__VC6_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA1_SDP_VCC_RESERVE1__VC7_CREDITS_MASK                                                              0x0003F000L
+#define MMEA1_SDP_VCC_RESERVE1__DISTRIBUTE_POOL_MASK                                                          0x80000000L
+//MMEA1_SDP_VCD_RESERVE0
+#define MMEA1_SDP_VCD_RESERVE0__VC0_CREDITS__SHIFT                                                            0x0
+#define MMEA1_SDP_VCD_RESERVE0__VC1_CREDITS__SHIFT                                                            0x6
+#define MMEA1_SDP_VCD_RESERVE0__VC2_CREDITS__SHIFT                                                            0xc
+#define MMEA1_SDP_VCD_RESERVE0__VC3_CREDITS__SHIFT                                                            0x12
+#define MMEA1_SDP_VCD_RESERVE0__VC4_CREDITS__SHIFT                                                            0x18
+#define MMEA1_SDP_VCD_RESERVE0__VC0_CREDITS_MASK                                                              0x0000003FL
+#define MMEA1_SDP_VCD_RESERVE0__VC1_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA1_SDP_VCD_RESERVE0__VC2_CREDITS_MASK                                                              0x0003F000L
+#define MMEA1_SDP_VCD_RESERVE0__VC3_CREDITS_MASK                                                              0x00FC0000L
+#define MMEA1_SDP_VCD_RESERVE0__VC4_CREDITS_MASK                                                              0x3F000000L
+//MMEA1_SDP_VCD_RESERVE1
+#define MMEA1_SDP_VCD_RESERVE1__VC5_CREDITS__SHIFT                                                            0x0
+#define MMEA1_SDP_VCD_RESERVE1__VC6_CREDITS__SHIFT                                                            0x6
+#define MMEA1_SDP_VCD_RESERVE1__VC7_CREDITS__SHIFT                                                            0xc
+#define MMEA1_SDP_VCD_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                        0x1f
+#define MMEA1_SDP_VCD_RESERVE1__VC5_CREDITS_MASK                                                              0x0000003FL
+#define MMEA1_SDP_VCD_RESERVE1__VC6_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA1_SDP_VCD_RESERVE1__VC7_CREDITS_MASK                                                              0x0003F000L
+#define MMEA1_SDP_VCD_RESERVE1__DISTRIBUTE_POOL_MASK                                                          0x80000000L
+//MMEA1_SDP_REQ_CNTL
+#define MMEA1_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ__SHIFT                                                  0x0
+#define MMEA1_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE__SHIFT                                                 0x1
+#define MMEA1_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC__SHIFT                                                0x2
+#define MMEA1_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM__SHIFT                                                    0x3
+#define MMEA1_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI__SHIFT                                                     0x4
+#define MMEA1_SDP_REQ_CNTL__INNER_DOMAIN_MODE__SHIFT                                                          0x5
+#define MMEA1_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ__SHIFT                                                       0x6
+#define MMEA1_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE__SHIFT                                                      0x8
+#define MMEA1_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC__SHIFT                                                     0xa
+#define MMEA1_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ_MASK                                                    0x00000001L
+#define MMEA1_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE_MASK                                                   0x00000002L
+#define MMEA1_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC_MASK                                                  0x00000004L
+#define MMEA1_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM_MASK                                                      0x00000008L
+#define MMEA1_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI_MASK                                                       0x00000010L
+#define MMEA1_SDP_REQ_CNTL__INNER_DOMAIN_MODE_MASK                                                            0x00000020L
+#define MMEA1_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ_MASK                                                         0x000000C0L
+#define MMEA1_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE_MASK                                                        0x00000300L
+#define MMEA1_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC_MASK                                                       0x00000C00L
+//MMEA1_MISC
+#define MMEA1_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB__SHIFT                                                        0x0
+#define MMEA1_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB__SHIFT                                                        0x1
+#define MMEA1_MISC__RELATIVE_PRI_IN_GMI_RD_ARB__SHIFT                                                         0x2
+#define MMEA1_MISC__RELATIVE_PRI_IN_GMI_WR_ARB__SHIFT                                                         0x3
+#define MMEA1_MISC__RELATIVE_PRI_IN_IO_RD_ARB__SHIFT                                                          0x4
+#define MMEA1_MISC__RELATIVE_PRI_IN_IO_WR_ARB__SHIFT                                                          0x5
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC0__SHIFT                                                              0x6
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC1__SHIFT                                                              0x7
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC2__SHIFT                                                              0x8
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC3__SHIFT                                                              0x9
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC4__SHIFT                                                              0xa
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC5__SHIFT                                                              0xb
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC6__SHIFT                                                              0xc
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC7__SHIFT                                                              0xd
+#define MMEA1_MISC__EARLY_SDP_ORIGDATA__SHIFT                                                                 0xe
+#define MMEA1_MISC__LINKMGR_DYNAMIC_MODE__SHIFT                                                               0xf
+#define MMEA1_MISC__LINKMGR_HALT_THRESHOLD__SHIFT                                                             0x11
+#define MMEA1_MISC__LINKMGR_RECONNECT_DELAY__SHIFT                                                            0x13
+#define MMEA1_MISC__LINKMGR_IDLE_THRESHOLD__SHIFT                                                             0x15
+#define MMEA1_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB__SHIFT                                                     0x1a
+#define MMEA1_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB__SHIFT                                                      0x1b
+#define MMEA1_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB__SHIFT                                                         0x1c
+#define MMEA1_MISC__FAVOUR_LAST_CS_IN_GMI_ARB__SHIFT                                                          0x1d
+#define MMEA1_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB__SHIFT                                                       0x1e
+#define MMEA1_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB__SHIFT                                                        0x1f
+#define MMEA1_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB_MASK                                                          0x00000001L
+#define MMEA1_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB_MASK                                                          0x00000002L
+#define MMEA1_MISC__RELATIVE_PRI_IN_GMI_RD_ARB_MASK                                                           0x00000004L
+#define MMEA1_MISC__RELATIVE_PRI_IN_GMI_WR_ARB_MASK                                                           0x00000008L
+#define MMEA1_MISC__RELATIVE_PRI_IN_IO_RD_ARB_MASK                                                            0x00000010L
+#define MMEA1_MISC__RELATIVE_PRI_IN_IO_WR_ARB_MASK                                                            0x00000020L
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC0_MASK                                                                0x00000040L
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC1_MASK                                                                0x00000080L
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC2_MASK                                                                0x00000100L
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC3_MASK                                                                0x00000200L
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC4_MASK                                                                0x00000400L
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC5_MASK                                                                0x00000800L
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC6_MASK                                                                0x00001000L
+#define MMEA1_MISC__EARLYWRRET_ENABLE_VC7_MASK                                                                0x00002000L
+#define MMEA1_MISC__EARLY_SDP_ORIGDATA_MASK                                                                   0x00004000L
+#define MMEA1_MISC__LINKMGR_DYNAMIC_MODE_MASK                                                                 0x00018000L
+#define MMEA1_MISC__LINKMGR_HALT_THRESHOLD_MASK                                                               0x00060000L
+#define MMEA1_MISC__LINKMGR_RECONNECT_DELAY_MASK                                                              0x00180000L
+#define MMEA1_MISC__LINKMGR_IDLE_THRESHOLD_MASK                                                               0x03E00000L
+#define MMEA1_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB_MASK                                                       0x04000000L
+#define MMEA1_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB_MASK                                                        0x08000000L
+#define MMEA1_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB_MASK                                                           0x10000000L
+#define MMEA1_MISC__FAVOUR_LAST_CS_IN_GMI_ARB_MASK                                                            0x20000000L
+#define MMEA1_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB_MASK                                                         0x40000000L
+#define MMEA1_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB_MASK                                                          0x80000000L
+//MMEA1_LATENCY_SAMPLING
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_DRAM__SHIFT                                                          0x0
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_DRAM__SHIFT                                                          0x1
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_GMI__SHIFT                                                           0x2
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_GMI__SHIFT                                                           0x3
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_IO__SHIFT                                                            0x4
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_IO__SHIFT                                                            0x5
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_READ__SHIFT                                                          0x6
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_READ__SHIFT                                                          0x7
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_WRITE__SHIFT                                                         0x8
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_WRITE__SHIFT                                                         0x9
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET__SHIFT                                                    0xa
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET__SHIFT                                                    0xb
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET__SHIFT                                                  0xc
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET__SHIFT                                                  0xd
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_VC__SHIFT                                                            0xe
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_VC__SHIFT                                                            0x16
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_DRAM_MASK                                                            0x00000001L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_DRAM_MASK                                                            0x00000002L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_GMI_MASK                                                             0x00000004L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_GMI_MASK                                                             0x00000008L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_IO_MASK                                                              0x00000010L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_IO_MASK                                                              0x00000020L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_READ_MASK                                                            0x00000040L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_READ_MASK                                                            0x00000080L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_WRITE_MASK                                                           0x00000100L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_WRITE_MASK                                                           0x00000200L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET_MASK                                                      0x00000400L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET_MASK                                                      0x00000800L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET_MASK                                                    0x00001000L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET_MASK                                                    0x00002000L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER0_VC_MASK                                                              0x003FC000L
+#define MMEA1_LATENCY_SAMPLING__SAMPLER1_VC_MASK                                                              0x3FC00000L
+//MMEA1_PERFCOUNTER_LO
+#define MMEA1_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define MMEA1_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//MMEA1_PERFCOUNTER_HI
+#define MMEA1_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define MMEA1_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define MMEA1_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define MMEA1_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+//MMEA1_PERFCOUNTER0_CFG
+#define MMEA1_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define MMEA1_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define MMEA1_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define MMEA1_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define MMEA1_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define MMEA1_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define MMEA1_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define MMEA1_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define MMEA1_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define MMEA1_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//MMEA1_PERFCOUNTER1_CFG
+#define MMEA1_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define MMEA1_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define MMEA1_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define MMEA1_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define MMEA1_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define MMEA1_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define MMEA1_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define MMEA1_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define MMEA1_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define MMEA1_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//MMEA1_PERFCOUNTER_RSLT_CNTL
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define MMEA1_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+//MMEA1_DSM_CNTL
+#define MMEA1_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x0
+#define MMEA1_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x2
+#define MMEA1_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x3
+#define MMEA1_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x5
+#define MMEA1_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x6
+#define MMEA1_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x8
+#define MMEA1_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x9
+#define MMEA1_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xb
+#define MMEA1_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0xc
+#define MMEA1_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xe
+#define MMEA1_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0xf
+#define MMEA1_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x11
+#define MMEA1_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x12
+#define MMEA1_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x14
+#define MMEA1_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x15
+#define MMEA1_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x17
+#define MMEA1_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00000003L
+#define MMEA1_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000004L
+#define MMEA1_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00000018L
+#define MMEA1_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000020L
+#define MMEA1_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                0x000000C0L
+#define MMEA1_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000100L
+#define MMEA1_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00000600L
+#define MMEA1_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000800L
+#define MMEA1_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00003000L
+#define MMEA1_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00004000L
+#define MMEA1_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00018000L
+#define MMEA1_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00020000L
+#define MMEA1_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x000C0000L
+#define MMEA1_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00100000L
+#define MMEA1_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00600000L
+#define MMEA1_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00800000L
+//MMEA1_DSM_CNTLA
+#define MMEA1_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                             0x0
+#define MMEA1_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                            0x2
+#define MMEA1_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                             0x3
+#define MMEA1_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                            0x5
+#define MMEA1_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x6
+#define MMEA1_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x8
+#define MMEA1_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x9
+#define MMEA1_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0xb
+#define MMEA1_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0xc
+#define MMEA1_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0xe
+#define MMEA1_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0xf
+#define MMEA1_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x11
+#define MMEA1_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x12
+#define MMEA1_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x14
+#define MMEA1_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                               0x00000003L
+#define MMEA1_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                              0x00000004L
+#define MMEA1_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                               0x00000018L
+#define MMEA1_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                              0x00000020L
+#define MMEA1_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x000000C0L
+#define MMEA1_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000100L
+#define MMEA1_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000600L
+#define MMEA1_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000800L
+#define MMEA1_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00003000L
+#define MMEA1_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00004000L
+#define MMEA1_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00018000L
+#define MMEA1_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00020000L
+#define MMEA1_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x000C0000L
+#define MMEA1_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00100000L
+//MMEA1_DSM_CNTLB
+#define MMEA1_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x0
+#define MMEA1_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x2
+#define MMEA1_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x3
+#define MMEA1_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x5
+#define MMEA1_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x6
+#define MMEA1_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x8
+#define MMEA1_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x9
+#define MMEA1_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xb
+#define MMEA1_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000003L
+#define MMEA1_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000004L
+#define MMEA1_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000018L
+#define MMEA1_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000020L
+#define MMEA1_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA_MASK                                                    0x000000C0L
+#define MMEA1_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000100L
+#define MMEA1_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000600L
+#define MMEA1_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000800L
+//MMEA1_DSM_CNTL2
+#define MMEA1_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x0
+#define MMEA1_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                             0x2
+#define MMEA1_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x3
+#define MMEA1_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                             0x5
+#define MMEA1_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x6
+#define MMEA1_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                            0x8
+#define MMEA1_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x9
+#define MMEA1_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                               0xb
+#define MMEA1_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                               0xc
+#define MMEA1_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                               0xe
+#define MMEA1_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0xf
+#define MMEA1_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x11
+#define MMEA1_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x12
+#define MMEA1_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x14
+#define MMEA1_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x15
+#define MMEA1_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0x17
+#define MMEA1_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                  0x1a
+#define MMEA1_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                               0x00000003L
+#define MMEA1_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                               0x00000004L
+#define MMEA1_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                               0x00000018L
+#define MMEA1_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                               0x00000020L
+#define MMEA1_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                              0x000000C0L
+#define MMEA1_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                              0x00000100L
+#define MMEA1_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00000600L
+#define MMEA1_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000800L
+#define MMEA1_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00003000L
+#define MMEA1_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                 0x00004000L
+#define MMEA1_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00018000L
+#define MMEA1_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00020000L
+#define MMEA1_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x000C0000L
+#define MMEA1_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00100000L
+#define MMEA1_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x00600000L
+#define MMEA1_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00800000L
+#define MMEA1_DSM_CNTL2__INJECT_DELAY_MASK                                                                    0xFC000000L
+//MMEA1_DSM_CNTL2A
+#define MMEA1_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                           0x0
+#define MMEA1_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                           0x2
+#define MMEA1_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                           0x3
+#define MMEA1_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                           0x5
+#define MMEA1_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x6
+#define MMEA1_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x8
+#define MMEA1_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x9
+#define MMEA1_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0xb
+#define MMEA1_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0xc
+#define MMEA1_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0xe
+#define MMEA1_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0xf
+#define MMEA1_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x11
+#define MMEA1_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x12
+#define MMEA1_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x14
+#define MMEA1_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                             0x00000003L
+#define MMEA1_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                             0x00000004L
+#define MMEA1_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                             0x00000018L
+#define MMEA1_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                             0x00000020L
+#define MMEA1_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x000000C0L
+#define MMEA1_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000100L
+#define MMEA1_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000600L
+#define MMEA1_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000800L
+#define MMEA1_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x00003000L
+#define MMEA1_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00004000L
+#define MMEA1_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00018000L
+#define MMEA1_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00020000L
+#define MMEA1_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x000C0000L
+#define MMEA1_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00100000L
+//MMEA1_DSM_CNTL2B
+#define MMEA1_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x0
+#define MMEA1_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY__SHIFT                                                0x2
+#define MMEA1_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x3
+#define MMEA1_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY__SHIFT                                                0x5
+#define MMEA1_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x6
+#define MMEA1_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY__SHIFT                                                0x8
+#define MMEA1_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x9
+#define MMEA1_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY__SHIFT                                                0xb
+#define MMEA1_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000003L
+#define MMEA1_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000004L
+#define MMEA1_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000018L
+#define MMEA1_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000020L
+#define MMEA1_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT_MASK                                                  0x000000C0L
+#define MMEA1_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000100L
+#define MMEA1_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000600L
+#define MMEA1_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000800L
+//MMEA1_CGTT_CLK_CTRL
+#define MMEA1_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define MMEA1_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define MMEA1_CGTT_CLK_CTRL__SPARE0__SHIFT                                                                    0xc
+#define MMEA1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE__SHIFT                                                 0x14
+#define MMEA1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ__SHIFT                                                  0x15
+#define MMEA1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN__SHIFT                                                0x16
+#define MMEA1_CGTT_CLK_CTRL__SPARE1__SHIFT                                                                    0x17
+#define MMEA1_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                               0x1b
+#define MMEA1_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE__SHIFT                                                       0x1c
+#define MMEA1_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ__SHIFT                                                        0x1d
+#define MMEA1_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                      0x1e
+#define MMEA1_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                    0x1f
+#define MMEA1_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define MMEA1_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define MMEA1_CGTT_CLK_CTRL__SPARE0_MASK                                                                      0x000FF000L
+#define MMEA1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE_MASK                                                   0x00100000L
+#define MMEA1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ_MASK                                                    0x00200000L
+#define MMEA1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN_MASK                                                  0x00400000L
+#define MMEA1_CGTT_CLK_CTRL__SPARE1_MASK                                                                      0x07800000L
+#define MMEA1_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                                 0x08000000L
+#define MMEA1_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE_MASK                                                         0x10000000L
+#define MMEA1_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ_MASK                                                          0x20000000L
+#define MMEA1_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                        0x40000000L
+#define MMEA1_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                      0x80000000L
+//MMEA1_EDC_MODE
+#define MMEA1_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                  0x10
+#define MMEA1_EDC_MODE__GATE_FUE__SHIFT                                                                       0x11
+#define MMEA1_EDC_MODE__DED_MODE__SHIFT                                                                       0x14
+#define MMEA1_EDC_MODE__PROP_FED__SHIFT                                                                       0x1d
+#define MMEA1_EDC_MODE__BYPASS__SHIFT                                                                         0x1f
+#define MMEA1_EDC_MODE__COUNT_FED_OUT_MASK                                                                    0x00010000L
+#define MMEA1_EDC_MODE__GATE_FUE_MASK                                                                         0x00020000L
+#define MMEA1_EDC_MODE__DED_MODE_MASK                                                                         0x00300000L
+#define MMEA1_EDC_MODE__PROP_FED_MASK                                                                         0x20000000L
+#define MMEA1_EDC_MODE__BYPASS_MASK                                                                           0x80000000L
+//MMEA1_ERR_STATUS
+#define MMEA1_ERR_STATUS__SDP_RDRSP_STATUS__SHIFT                                                             0x0
+#define MMEA1_ERR_STATUS__SDP_WRRSP_STATUS__SHIFT                                                             0x4
+#define MMEA1_ERR_STATUS__SDP_RDRSP_DATASTATUS__SHIFT                                                         0x8
+#define MMEA1_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR__SHIFT                                                   0xa
+#define MMEA1_ERR_STATUS__CLEAR_ERROR_STATUS__SHIFT                                                           0xb
+#define MMEA1_ERR_STATUS__BUSY_ON_ERROR__SHIFT                                                                0xc
+#define MMEA1_ERR_STATUS__FUE_FLAG__SHIFT                                                                     0xd
+#define MMEA1_ERR_STATUS__IGNORE_RDRSP_FED__SHIFT                                                             0xe
+#define MMEA1_ERR_STATUS__INTERRUPT_ON_FATAL__SHIFT                                                           0xf
+#define MMEA1_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL__SHIFT                                                   0x10
+#define MMEA1_ERR_STATUS__LEVEL_INTERRUPT__SHIFT                                                              0x11
+#define MMEA1_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR__SHIFT                                                     0x12
+#define MMEA1_ERR_STATUS__FUE_FLAG_CLIENT__SHIFT                                                              0x13
+#define MMEA1_ERR_STATUS__SDP_RDRSP_STATUS_MASK                                                               0x0000000FL
+#define MMEA1_ERR_STATUS__SDP_WRRSP_STATUS_MASK                                                               0x000000F0L
+#define MMEA1_ERR_STATUS__SDP_RDRSP_DATASTATUS_MASK                                                           0x00000300L
+#define MMEA1_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR_MASK                                                     0x00000400L
+#define MMEA1_ERR_STATUS__CLEAR_ERROR_STATUS_MASK                                                             0x00000800L
+#define MMEA1_ERR_STATUS__BUSY_ON_ERROR_MASK                                                                  0x00001000L
+#define MMEA1_ERR_STATUS__FUE_FLAG_MASK                                                                       0x00002000L
+#define MMEA1_ERR_STATUS__IGNORE_RDRSP_FED_MASK                                                               0x00004000L
+#define MMEA1_ERR_STATUS__INTERRUPT_ON_FATAL_MASK                                                             0x00008000L
+#define MMEA1_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL_MASK                                                     0x00010000L
+#define MMEA1_ERR_STATUS__LEVEL_INTERRUPT_MASK                                                                0x00020000L
+#define MMEA1_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR_MASK                                                       0x00040000L
+#define MMEA1_ERR_STATUS__FUE_FLAG_CLIENT_MASK                                                                0x00080000L
+//MMEA1_MISC2
+#define MMEA1_MISC2__CSGROUP_SWAP_IN_DRAM_ARB__SHIFT                                                          0x0
+#define MMEA1_MISC2__CSGROUP_SWAP_IN_GMI_ARB__SHIFT                                                           0x1
+#define MMEA1_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM__SHIFT                                                       0x2
+#define MMEA1_MISC2__CSGRP_BURST_LIMIT_DATA_GMI__SHIFT                                                        0x7
+#define MMEA1_MISC2__IO_RDWR_PRIORITY_ENABLE__SHIFT                                                           0xc
+#define MMEA1_MISC2__RRET_SWAP_MODE__SHIFT                                                                    0xd
+#define MMEA1_MISC2__BLOCK_REQUESTS__SHIFT                                                                    0xe
+#define MMEA1_MISC2__REQUESTS_BLOCKED__SHIFT                                                                  0xf
+#define MMEA1_MISC2__DRAM_RD_THROTTLE__SHIFT                                                                  0x10
+#define MMEA1_MISC2__DRAM_WR_THROTTLE__SHIFT                                                                  0x11
+#define MMEA1_MISC2__GMI_RD_THROTTLE__SHIFT                                                                   0x12
+#define MMEA1_MISC2__GMI_WR_THROTTLE__SHIFT                                                                   0x13
+#define MMEA1_MISC2__CSGROUP_SWAP_IN_DRAM_ARB_MASK                                                            0x00000001L
+#define MMEA1_MISC2__CSGROUP_SWAP_IN_GMI_ARB_MASK                                                             0x00000002L
+#define MMEA1_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM_MASK                                                         0x0000007CL
+#define MMEA1_MISC2__CSGRP_BURST_LIMIT_DATA_GMI_MASK                                                          0x00000F80L
+#define MMEA1_MISC2__IO_RDWR_PRIORITY_ENABLE_MASK                                                             0x00001000L
+#define MMEA1_MISC2__RRET_SWAP_MODE_MASK                                                                      0x00002000L
+#define MMEA1_MISC2__BLOCK_REQUESTS_MASK                                                                      0x00004000L
+#define MMEA1_MISC2__REQUESTS_BLOCKED_MASK                                                                    0x00008000L
+#define MMEA1_MISC2__DRAM_RD_THROTTLE_MASK                                                                    0x00010000L
+#define MMEA1_MISC2__DRAM_WR_THROTTLE_MASK                                                                    0x00020000L
+#define MMEA1_MISC2__GMI_RD_THROTTLE_MASK                                                                     0x00040000L
+#define MMEA1_MISC2__GMI_WR_THROTTLE_MASK                                                                     0x00080000L
+//MMEA1_MISC_AON
+#define MMEA1_MISC_AON__LINKMGR_PARTACK_HYSTERESIS__SHIFT                                                     0x0
+#define MMEA1_MISC_AON__LINKMGR_PARTACK_DEASSERT_MODE__SHIFT                                                  0x2
+#define MMEA1_MISC_AON__LINKMGR_PARTACK_HYSTERESIS_MASK                                                       0x00000003L
+#define MMEA1_MISC_AON__LINKMGR_PARTACK_DEASSERT_MODE_MASK                                                    0x00000004L
+
+
+// addressBlock: aid_mmhub_ea_mmeadec2
+//MMEA2_DRAM_RD_CLI2GRP_MAP0
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                         0x0
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                         0x2
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                         0x4
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                         0x6
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                         0x8
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                         0xa
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                         0xc
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                         0xe
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                         0x10
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                         0x12
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                        0x14
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                        0x16
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                        0x18
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                        0x1a
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                        0x1c
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                        0x1e
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                           0x00000003L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                           0x0000000CL
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                           0x00000030L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                           0x000000C0L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                           0x00000300L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                           0x00000C00L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                           0x00003000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                           0x0000C000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                           0x00030000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                           0x000C0000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                          0x00300000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                          0x00C00000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                          0x03000000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                          0x0C000000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                          0x30000000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                          0xC0000000L
+//MMEA2_DRAM_RD_CLI2GRP_MAP1
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                        0x0
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                        0x2
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                        0x4
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                        0x6
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                        0x8
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                        0xa
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                        0xc
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                        0xe
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                        0x10
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                        0x12
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                        0x14
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                        0x16
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                        0x18
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                        0x1a
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                        0x1c
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                        0x1e
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                          0x00000003L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                          0x0000000CL
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                          0x00000030L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                          0x000000C0L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                          0x00000300L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                          0x00000C00L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                          0x00003000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                          0x0000C000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                          0x00030000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                          0x000C0000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                          0x00300000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                          0x00C00000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                          0x03000000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                          0x0C000000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                          0x30000000L
+#define MMEA2_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                          0xC0000000L
+//MMEA2_DRAM_WR_CLI2GRP_MAP0
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                         0x0
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                         0x2
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                         0x4
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                         0x6
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                         0x8
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                         0xa
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                         0xc
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                         0xe
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                         0x10
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                         0x12
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                        0x14
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                        0x16
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                        0x18
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                        0x1a
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                        0x1c
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                        0x1e
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                           0x00000003L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                           0x0000000CL
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                           0x00000030L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                           0x000000C0L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                           0x00000300L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                           0x00000C00L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                           0x00003000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                           0x0000C000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                           0x00030000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                           0x000C0000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                          0x00300000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                          0x00C00000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                          0x03000000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                          0x0C000000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                          0x30000000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                          0xC0000000L
+//MMEA2_DRAM_WR_CLI2GRP_MAP1
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                        0x0
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                        0x2
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                        0x4
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                        0x6
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                        0x8
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                        0xa
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                        0xc
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                        0xe
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                        0x10
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                        0x12
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                        0x14
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                        0x16
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                        0x18
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                        0x1a
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                        0x1c
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                        0x1e
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                          0x00000003L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                          0x0000000CL
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                          0x00000030L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                          0x000000C0L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                          0x00000300L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                          0x00000C00L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                          0x00003000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                          0x0000C000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                          0x00030000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                          0x000C0000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                          0x00300000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                          0x00C00000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                          0x03000000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                          0x0C000000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                          0x30000000L
+#define MMEA2_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                          0xC0000000L
+//MMEA2_DRAM_RD_GRP2VC_MAP
+#define MMEA2_DRAM_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                            0x0
+#define MMEA2_DRAM_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                            0x3
+#define MMEA2_DRAM_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                            0x6
+#define MMEA2_DRAM_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                            0x9
+#define MMEA2_DRAM_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                              0x00000007L
+#define MMEA2_DRAM_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                              0x00000038L
+#define MMEA2_DRAM_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                              0x000001C0L
+#define MMEA2_DRAM_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                              0x00000E00L
+//MMEA2_DRAM_WR_GRP2VC_MAP
+#define MMEA2_DRAM_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                            0x0
+#define MMEA2_DRAM_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                            0x3
+#define MMEA2_DRAM_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                            0x6
+#define MMEA2_DRAM_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                            0x9
+#define MMEA2_DRAM_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                              0x00000007L
+#define MMEA2_DRAM_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                              0x00000038L
+#define MMEA2_DRAM_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                              0x000001C0L
+#define MMEA2_DRAM_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                              0x00000E00L
+//MMEA2_DRAM_RD_LAZY
+#define MMEA2_DRAM_RD_LAZY__GROUP0_DELAY__SHIFT                                                               0x0
+#define MMEA2_DRAM_RD_LAZY__GROUP1_DELAY__SHIFT                                                               0x3
+#define MMEA2_DRAM_RD_LAZY__GROUP2_DELAY__SHIFT                                                               0x6
+#define MMEA2_DRAM_RD_LAZY__GROUP3_DELAY__SHIFT                                                               0x9
+#define MMEA2_DRAM_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                           0xc
+#define MMEA2_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                          0x14
+#define MMEA2_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                          0x1b
+#define MMEA2_DRAM_RD_LAZY__GROUP0_DELAY_MASK                                                                 0x00000007L
+#define MMEA2_DRAM_RD_LAZY__GROUP1_DELAY_MASK                                                                 0x00000038L
+#define MMEA2_DRAM_RD_LAZY__GROUP2_DELAY_MASK                                                                 0x000001C0L
+#define MMEA2_DRAM_RD_LAZY__GROUP3_DELAY_MASK                                                                 0x00000E00L
+#define MMEA2_DRAM_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                             0x0003F000L
+#define MMEA2_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                            0x07F00000L
+#define MMEA2_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                            0x78000000L
+//MMEA2_DRAM_WR_LAZY
+#define MMEA2_DRAM_WR_LAZY__GROUP0_DELAY__SHIFT                                                               0x0
+#define MMEA2_DRAM_WR_LAZY__GROUP1_DELAY__SHIFT                                                               0x3
+#define MMEA2_DRAM_WR_LAZY__GROUP2_DELAY__SHIFT                                                               0x6
+#define MMEA2_DRAM_WR_LAZY__GROUP3_DELAY__SHIFT                                                               0x9
+#define MMEA2_DRAM_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                           0xc
+#define MMEA2_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                          0x14
+#define MMEA2_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                          0x1b
+#define MMEA2_DRAM_WR_LAZY__GROUP0_DELAY_MASK                                                                 0x00000007L
+#define MMEA2_DRAM_WR_LAZY__GROUP1_DELAY_MASK                                                                 0x00000038L
+#define MMEA2_DRAM_WR_LAZY__GROUP2_DELAY_MASK                                                                 0x000001C0L
+#define MMEA2_DRAM_WR_LAZY__GROUP3_DELAY_MASK                                                                 0x00000E00L
+#define MMEA2_DRAM_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                             0x0003F000L
+#define MMEA2_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                            0x07F00000L
+#define MMEA2_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                            0x78000000L
+//MMEA2_DRAM_RD_CAM_CNTL
+#define MMEA2_DRAM_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                           0x0
+#define MMEA2_DRAM_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                           0x4
+#define MMEA2_DRAM_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                           0x8
+#define MMEA2_DRAM_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                           0xc
+#define MMEA2_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                   0x10
+#define MMEA2_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                   0x13
+#define MMEA2_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                   0x16
+#define MMEA2_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                   0x19
+#define MMEA2_DRAM_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                           0x1c
+#define MMEA2_DRAM_RD_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                     0x1d
+#define MMEA2_DRAM_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                             0x0000000FL
+#define MMEA2_DRAM_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                             0x000000F0L
+#define MMEA2_DRAM_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                             0x00000F00L
+#define MMEA2_DRAM_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                             0x0000F000L
+#define MMEA2_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                     0x00070000L
+#define MMEA2_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                     0x00380000L
+#define MMEA2_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                     0x01C00000L
+#define MMEA2_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                     0x0E000000L
+#define MMEA2_DRAM_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                             0x10000000L
+#define MMEA2_DRAM_RD_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                       0x20000000L
+//MMEA2_DRAM_WR_CAM_CNTL
+#define MMEA2_DRAM_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                           0x0
+#define MMEA2_DRAM_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                           0x4
+#define MMEA2_DRAM_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                           0x8
+#define MMEA2_DRAM_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                           0xc
+#define MMEA2_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                   0x10
+#define MMEA2_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                   0x13
+#define MMEA2_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                   0x16
+#define MMEA2_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                   0x19
+#define MMEA2_DRAM_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                           0x1c
+#define MMEA2_DRAM_WR_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                     0x1d
+#define MMEA2_DRAM_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                             0x0000000FL
+#define MMEA2_DRAM_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                             0x000000F0L
+#define MMEA2_DRAM_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                             0x00000F00L
+#define MMEA2_DRAM_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                             0x0000F000L
+#define MMEA2_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                     0x00070000L
+#define MMEA2_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                     0x00380000L
+#define MMEA2_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                     0x01C00000L
+#define MMEA2_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                     0x0E000000L
+#define MMEA2_DRAM_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                             0x10000000L
+#define MMEA2_DRAM_WR_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                       0x20000000L
+//MMEA2_DRAM_PAGE_BURST
+#define MMEA2_DRAM_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                             0x0
+#define MMEA2_DRAM_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                             0x8
+#define MMEA2_DRAM_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                             0x10
+#define MMEA2_DRAM_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                             0x18
+#define MMEA2_DRAM_PAGE_BURST__RD_LIMIT_LO_MASK                                                               0x000000FFL
+#define MMEA2_DRAM_PAGE_BURST__RD_LIMIT_HI_MASK                                                               0x0000FF00L
+#define MMEA2_DRAM_PAGE_BURST__WR_LIMIT_LO_MASK                                                               0x00FF0000L
+#define MMEA2_DRAM_PAGE_BURST__WR_LIMIT_HI_MASK                                                               0xFF000000L
+//MMEA2_DRAM_RD_PRI_AGE
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                  0xc
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                  0xf
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                  0x12
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                  0x15
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                    0x00007000L
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                    0x00038000L
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                    0x001C0000L
+#define MMEA2_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                    0x00E00000L
+//MMEA2_DRAM_WR_PRI_AGE
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                  0xc
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                  0xf
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                  0x12
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                  0x15
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                    0x00007000L
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                    0x00038000L
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                    0x001C0000L
+#define MMEA2_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                    0x00E00000L
+//MMEA2_DRAM_RD_PRI_QUEUING
+#define MMEA2_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                          0x0
+#define MMEA2_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                          0x3
+#define MMEA2_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                          0x6
+#define MMEA2_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                          0x9
+#define MMEA2_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA2_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA2_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA2_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                            0x00000E00L
+//MMEA2_DRAM_WR_PRI_QUEUING
+#define MMEA2_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                          0x0
+#define MMEA2_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                          0x3
+#define MMEA2_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                          0x6
+#define MMEA2_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                          0x9
+#define MMEA2_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA2_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA2_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA2_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                            0x00000E00L
+//MMEA2_DRAM_RD_PRI_FIXED
+#define MMEA2_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                              0x0
+#define MMEA2_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                              0x3
+#define MMEA2_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                              0x6
+#define MMEA2_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                              0x9
+#define MMEA2_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                0x00000007L
+#define MMEA2_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                0x00000038L
+#define MMEA2_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                0x000001C0L
+#define MMEA2_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                0x00000E00L
+//MMEA2_DRAM_WR_PRI_FIXED
+#define MMEA2_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                              0x0
+#define MMEA2_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                              0x3
+#define MMEA2_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                              0x6
+#define MMEA2_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                              0x9
+#define MMEA2_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                0x00000007L
+#define MMEA2_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                0x00000038L
+#define MMEA2_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                0x000001C0L
+#define MMEA2_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                0x00000E00L
+//MMEA2_DRAM_RD_PRI_URGENCY
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                          0x0
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                          0x3
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                          0x6
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                          0x9
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                 0xc
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                 0xd
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                 0xe
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                 0xf
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                            0x00000E00L
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                   0x00001000L
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                   0x00002000L
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                   0x00004000L
+#define MMEA2_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                   0x00008000L
+//MMEA2_DRAM_WR_PRI_URGENCY
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                          0x0
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                          0x3
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                          0x6
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                          0x9
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                 0xc
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                 0xd
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                 0xe
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                 0xf
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                            0x00000E00L
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                   0x00001000L
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                   0x00002000L
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                   0x00004000L
+#define MMEA2_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                   0x00008000L
+//MMEA2_DRAM_RD_PRI_QUANT_PRI1
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA2_DRAM_RD_PRI_QUANT_PRI2
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA2_DRAM_RD_PRI_QUANT_PRI3
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA2_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA2_DRAM_WR_PRI_QUANT_PRI1
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA2_DRAM_WR_PRI_QUANT_PRI2
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA2_DRAM_WR_PRI_QUANT_PRI3
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA2_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA2_GMI_RD_CLI2GRP_MAP0
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//MMEA2_GMI_RD_CLI2GRP_MAP1
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define MMEA2_GMI_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//MMEA2_GMI_WR_CLI2GRP_MAP0
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//MMEA2_GMI_WR_CLI2GRP_MAP1
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define MMEA2_GMI_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//MMEA2_GMI_RD_GRP2VC_MAP
+#define MMEA2_GMI_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define MMEA2_GMI_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define MMEA2_GMI_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define MMEA2_GMI_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define MMEA2_GMI_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define MMEA2_GMI_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define MMEA2_GMI_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define MMEA2_GMI_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//MMEA2_GMI_WR_GRP2VC_MAP
+#define MMEA2_GMI_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define MMEA2_GMI_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define MMEA2_GMI_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define MMEA2_GMI_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define MMEA2_GMI_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define MMEA2_GMI_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define MMEA2_GMI_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define MMEA2_GMI_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//MMEA2_GMI_RD_LAZY
+#define MMEA2_GMI_RD_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define MMEA2_GMI_RD_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define MMEA2_GMI_RD_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define MMEA2_GMI_RD_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define MMEA2_GMI_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define MMEA2_GMI_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define MMEA2_GMI_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define MMEA2_GMI_RD_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define MMEA2_GMI_RD_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define MMEA2_GMI_RD_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define MMEA2_GMI_RD_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define MMEA2_GMI_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define MMEA2_GMI_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define MMEA2_GMI_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//MMEA2_GMI_WR_LAZY
+#define MMEA2_GMI_WR_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define MMEA2_GMI_WR_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define MMEA2_GMI_WR_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define MMEA2_GMI_WR_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define MMEA2_GMI_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define MMEA2_GMI_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define MMEA2_GMI_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define MMEA2_GMI_WR_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define MMEA2_GMI_WR_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define MMEA2_GMI_WR_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define MMEA2_GMI_WR_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define MMEA2_GMI_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define MMEA2_GMI_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define MMEA2_GMI_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//MMEA2_GMI_RD_CAM_CNTL
+#define MMEA2_GMI_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define MMEA2_GMI_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define MMEA2_GMI_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define MMEA2_GMI_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define MMEA2_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define MMEA2_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define MMEA2_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define MMEA2_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define MMEA2_GMI_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define MMEA2_GMI_RD_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                      0x1d
+#define MMEA2_GMI_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define MMEA2_GMI_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define MMEA2_GMI_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define MMEA2_GMI_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define MMEA2_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define MMEA2_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define MMEA2_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define MMEA2_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define MMEA2_GMI_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+#define MMEA2_GMI_RD_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                        0x20000000L
+//MMEA2_GMI_WR_CAM_CNTL
+#define MMEA2_GMI_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define MMEA2_GMI_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define MMEA2_GMI_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define MMEA2_GMI_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define MMEA2_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define MMEA2_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define MMEA2_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define MMEA2_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define MMEA2_GMI_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define MMEA2_GMI_WR_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                      0x1d
+#define MMEA2_GMI_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define MMEA2_GMI_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define MMEA2_GMI_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define MMEA2_GMI_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define MMEA2_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define MMEA2_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define MMEA2_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define MMEA2_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define MMEA2_GMI_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+#define MMEA2_GMI_WR_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                        0x20000000L
+//MMEA2_GMI_PAGE_BURST
+#define MMEA2_GMI_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define MMEA2_GMI_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define MMEA2_GMI_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define MMEA2_GMI_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define MMEA2_GMI_PAGE_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define MMEA2_GMI_PAGE_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define MMEA2_GMI_PAGE_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define MMEA2_GMI_PAGE_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//MMEA2_GMI_RD_PRI_AGE
+#define MMEA2_GMI_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define MMEA2_GMI_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define MMEA2_GMI_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define MMEA2_GMI_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define MMEA2_GMI_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define MMEA2_GMI_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define MMEA2_GMI_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define MMEA2_GMI_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define MMEA2_GMI_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define MMEA2_GMI_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define MMEA2_GMI_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define MMEA2_GMI_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define MMEA2_GMI_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define MMEA2_GMI_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define MMEA2_GMI_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define MMEA2_GMI_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//MMEA2_GMI_WR_PRI_AGE
+#define MMEA2_GMI_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define MMEA2_GMI_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define MMEA2_GMI_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define MMEA2_GMI_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define MMEA2_GMI_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define MMEA2_GMI_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define MMEA2_GMI_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define MMEA2_GMI_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define MMEA2_GMI_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define MMEA2_GMI_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define MMEA2_GMI_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define MMEA2_GMI_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define MMEA2_GMI_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define MMEA2_GMI_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define MMEA2_GMI_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define MMEA2_GMI_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//MMEA2_GMI_RD_PRI_QUEUING
+#define MMEA2_GMI_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define MMEA2_GMI_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define MMEA2_GMI_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define MMEA2_GMI_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define MMEA2_GMI_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA2_GMI_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA2_GMI_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA2_GMI_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//MMEA2_GMI_WR_PRI_QUEUING
+#define MMEA2_GMI_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define MMEA2_GMI_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define MMEA2_GMI_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define MMEA2_GMI_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define MMEA2_GMI_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA2_GMI_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA2_GMI_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA2_GMI_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//MMEA2_GMI_RD_PRI_FIXED
+#define MMEA2_GMI_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define MMEA2_GMI_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define MMEA2_GMI_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define MMEA2_GMI_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define MMEA2_GMI_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define MMEA2_GMI_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define MMEA2_GMI_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define MMEA2_GMI_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//MMEA2_GMI_WR_PRI_FIXED
+#define MMEA2_GMI_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define MMEA2_GMI_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define MMEA2_GMI_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define MMEA2_GMI_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define MMEA2_GMI_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define MMEA2_GMI_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define MMEA2_GMI_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define MMEA2_GMI_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//MMEA2_GMI_RD_PRI_URGENCY
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define MMEA2_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//MMEA2_GMI_WR_PRI_URGENCY
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define MMEA2_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//MMEA2_GMI_RD_PRI_URGENCY_MASKING
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                    0x0
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                    0x1
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                    0x2
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                    0x3
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                    0x4
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                    0x5
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                    0x6
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                    0x7
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                    0x8
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                    0x9
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                   0xa
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                   0xb
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                   0xc
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                   0xd
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                   0xe
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                   0xf
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                   0x10
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                   0x11
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                   0x12
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                   0x13
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                   0x14
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                   0x15
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                   0x16
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                   0x17
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                   0x18
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                   0x19
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                   0x1a
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                   0x1b
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                   0x1c
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                   0x1d
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                   0x1e
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                   0x1f
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                      0x00000001L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                      0x00000002L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                      0x00000004L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                      0x00000008L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                      0x00000010L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                      0x00000020L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                      0x00000040L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                      0x00000080L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                      0x00000100L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                      0x00000200L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                     0x00000400L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                     0x00000800L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                     0x00001000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                     0x00002000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                     0x00004000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                     0x00008000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                     0x00010000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                     0x00020000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                     0x00040000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                     0x00080000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                     0x00100000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                     0x00200000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                     0x00400000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                     0x00800000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                     0x01000000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                     0x02000000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                     0x04000000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                     0x08000000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                     0x10000000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                     0x20000000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                     0x40000000L
+#define MMEA2_GMI_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                     0x80000000L
+//MMEA2_GMI_WR_PRI_URGENCY_MASKING
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                    0x0
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                    0x1
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                    0x2
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                    0x3
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                    0x4
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                    0x5
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                    0x6
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                    0x7
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                    0x8
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                    0x9
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                   0xa
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                   0xb
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                   0xc
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                   0xd
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                   0xe
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                   0xf
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                   0x10
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                   0x11
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                   0x12
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                   0x13
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                   0x14
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                   0x15
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                   0x16
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                   0x17
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                   0x18
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                   0x19
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                   0x1a
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                   0x1b
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                   0x1c
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                   0x1d
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                   0x1e
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                   0x1f
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                      0x00000001L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                      0x00000002L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                      0x00000004L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                      0x00000008L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                      0x00000010L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                      0x00000020L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                      0x00000040L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                      0x00000080L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                      0x00000100L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                      0x00000200L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                     0x00000400L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                     0x00000800L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                     0x00001000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                     0x00002000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                     0x00004000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                     0x00008000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                     0x00010000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                     0x00020000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                     0x00040000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                     0x00080000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                     0x00100000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                     0x00200000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                     0x00400000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                     0x00800000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                     0x01000000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                     0x02000000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                     0x04000000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                     0x08000000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                     0x10000000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                     0x20000000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                     0x40000000L
+#define MMEA2_GMI_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                     0x80000000L
+//MMEA2_GMI_RD_PRI_QUANT_PRI1
+#define MMEA2_GMI_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA2_GMI_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA2_GMI_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA2_GMI_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA2_GMI_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA2_GMI_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA2_GMI_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA2_GMI_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA2_GMI_RD_PRI_QUANT_PRI2
+#define MMEA2_GMI_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA2_GMI_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA2_GMI_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA2_GMI_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA2_GMI_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA2_GMI_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA2_GMI_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA2_GMI_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA2_GMI_RD_PRI_QUANT_PRI3
+#define MMEA2_GMI_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA2_GMI_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA2_GMI_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA2_GMI_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA2_GMI_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA2_GMI_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA2_GMI_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA2_GMI_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA2_GMI_WR_PRI_QUANT_PRI1
+#define MMEA2_GMI_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA2_GMI_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA2_GMI_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA2_GMI_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA2_GMI_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA2_GMI_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA2_GMI_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA2_GMI_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA2_GMI_WR_PRI_QUANT_PRI2
+#define MMEA2_GMI_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA2_GMI_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA2_GMI_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA2_GMI_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA2_GMI_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA2_GMI_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA2_GMI_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA2_GMI_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA2_GMI_WR_PRI_QUANT_PRI3
+#define MMEA2_GMI_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA2_GMI_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA2_GMI_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA2_GMI_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA2_GMI_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA2_GMI_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA2_GMI_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA2_GMI_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA2_IO_RD_CLI2GRP_MAP0
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                           0x0
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                           0x2
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                           0x4
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                           0x6
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                           0x8
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                           0xa
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                           0xc
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                           0xe
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                           0x10
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                           0x12
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                          0x14
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                          0x16
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                          0x18
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                          0x1a
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                          0x1c
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                          0x1e
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                             0x00000003L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                             0x0000000CL
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                             0x00000030L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                             0x000000C0L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                             0x00000300L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                             0x00000C00L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                             0x00003000L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                             0x0000C000L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                             0x00030000L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                             0x000C0000L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                            0x00300000L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                            0x00C00000L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                            0x03000000L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                            0x0C000000L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                            0x30000000L
+#define MMEA2_IO_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                            0xC0000000L
+//MMEA2_IO_RD_CLI2GRP_MAP1
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                          0x0
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                          0x2
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                          0x4
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                          0x6
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                          0x8
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                          0xa
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                          0xc
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                          0xe
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                          0x10
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                          0x12
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                          0x14
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                          0x16
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                          0x18
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                          0x1a
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                          0x1c
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                          0x1e
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                            0x00000003L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                            0x0000000CL
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                            0x00000030L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                            0x000000C0L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                            0x00000300L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                            0x00000C00L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                            0x00003000L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                            0x0000C000L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                            0x00030000L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                            0x000C0000L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                            0x00300000L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                            0x00C00000L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                            0x03000000L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                            0x0C000000L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                            0x30000000L
+#define MMEA2_IO_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                            0xC0000000L
+//MMEA2_IO_WR_CLI2GRP_MAP0
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                           0x0
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                           0x2
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                           0x4
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                           0x6
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                           0x8
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                           0xa
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                           0xc
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                           0xe
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                           0x10
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                           0x12
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                          0x14
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                          0x16
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                          0x18
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                          0x1a
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                          0x1c
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                          0x1e
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                             0x00000003L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                             0x0000000CL
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                             0x00000030L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                             0x000000C0L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                             0x00000300L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                             0x00000C00L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                             0x00003000L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                             0x0000C000L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                             0x00030000L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                             0x000C0000L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                            0x00300000L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                            0x00C00000L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                            0x03000000L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                            0x0C000000L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                            0x30000000L
+#define MMEA2_IO_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                            0xC0000000L
+//MMEA2_IO_WR_CLI2GRP_MAP1
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                          0x0
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                          0x2
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                          0x4
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                          0x6
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                          0x8
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                          0xa
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                          0xc
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                          0xe
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                          0x10
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                          0x12
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                          0x14
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                          0x16
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                          0x18
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                          0x1a
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                          0x1c
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                          0x1e
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                            0x00000003L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                            0x0000000CL
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                            0x00000030L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                            0x000000C0L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                            0x00000300L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                            0x00000C00L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                            0x00003000L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                            0x0000C000L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                            0x00030000L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                            0x000C0000L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                            0x00300000L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                            0x00C00000L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                            0x03000000L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                            0x0C000000L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                            0x30000000L
+#define MMEA2_IO_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                            0xC0000000L
+//MMEA2_IO_RD_COMBINE_FLUSH
+#define MMEA2_IO_RD_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                        0x0
+#define MMEA2_IO_RD_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                        0x4
+#define MMEA2_IO_RD_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                        0x8
+#define MMEA2_IO_RD_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                        0xc
+#define MMEA2_IO_RD_COMBINE_FLUSH__COMB_MODE__SHIFT                                                           0x10
+#define MMEA2_IO_RD_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                          0x0000000FL
+#define MMEA2_IO_RD_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                          0x000000F0L
+#define MMEA2_IO_RD_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                          0x00000F00L
+#define MMEA2_IO_RD_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                          0x0000F000L
+#define MMEA2_IO_RD_COMBINE_FLUSH__COMB_MODE_MASK                                                             0x00030000L
+//MMEA2_IO_WR_COMBINE_FLUSH
+#define MMEA2_IO_WR_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                        0x0
+#define MMEA2_IO_WR_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                        0x4
+#define MMEA2_IO_WR_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                        0x8
+#define MMEA2_IO_WR_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                        0xc
+#define MMEA2_IO_WR_COMBINE_FLUSH__COMB_MODE__SHIFT                                                           0x10
+#define MMEA2_IO_WR_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                          0x0000000FL
+#define MMEA2_IO_WR_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                          0x000000F0L
+#define MMEA2_IO_WR_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                          0x00000F00L
+#define MMEA2_IO_WR_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                          0x0000F000L
+#define MMEA2_IO_WR_COMBINE_FLUSH__COMB_MODE_MASK                                                             0x00030000L
+//MMEA2_IO_GROUP_BURST
+#define MMEA2_IO_GROUP_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define MMEA2_IO_GROUP_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define MMEA2_IO_GROUP_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define MMEA2_IO_GROUP_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define MMEA2_IO_GROUP_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define MMEA2_IO_GROUP_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define MMEA2_IO_GROUP_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define MMEA2_IO_GROUP_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//MMEA2_IO_RD_PRI_AGE
+#define MMEA2_IO_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                         0x0
+#define MMEA2_IO_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                         0x3
+#define MMEA2_IO_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                         0x6
+#define MMEA2_IO_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                         0x9
+#define MMEA2_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                    0xc
+#define MMEA2_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                    0xf
+#define MMEA2_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                    0x12
+#define MMEA2_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                    0x15
+#define MMEA2_IO_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                           0x00000007L
+#define MMEA2_IO_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                           0x00000038L
+#define MMEA2_IO_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                           0x000001C0L
+#define MMEA2_IO_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                           0x00000E00L
+#define MMEA2_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                      0x00007000L
+#define MMEA2_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                      0x00038000L
+#define MMEA2_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                      0x001C0000L
+#define MMEA2_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                      0x00E00000L
+//MMEA2_IO_WR_PRI_AGE
+#define MMEA2_IO_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                         0x0
+#define MMEA2_IO_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                         0x3
+#define MMEA2_IO_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                         0x6
+#define MMEA2_IO_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                         0x9
+#define MMEA2_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                    0xc
+#define MMEA2_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                    0xf
+#define MMEA2_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                    0x12
+#define MMEA2_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                    0x15
+#define MMEA2_IO_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                           0x00000007L
+#define MMEA2_IO_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                           0x00000038L
+#define MMEA2_IO_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                           0x000001C0L
+#define MMEA2_IO_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                           0x00000E00L
+#define MMEA2_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                      0x00007000L
+#define MMEA2_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                      0x00038000L
+#define MMEA2_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                      0x001C0000L
+#define MMEA2_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                      0x00E00000L
+//MMEA2_IO_RD_PRI_QUEUING
+#define MMEA2_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                            0x0
+#define MMEA2_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                            0x3
+#define MMEA2_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                            0x6
+#define MMEA2_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                            0x9
+#define MMEA2_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA2_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA2_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA2_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                              0x00000E00L
+//MMEA2_IO_WR_PRI_QUEUING
+#define MMEA2_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                            0x0
+#define MMEA2_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                            0x3
+#define MMEA2_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                            0x6
+#define MMEA2_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                            0x9
+#define MMEA2_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA2_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA2_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA2_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                              0x00000E00L
+//MMEA2_IO_RD_PRI_FIXED
+#define MMEA2_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                0x0
+#define MMEA2_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                0x3
+#define MMEA2_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                0x6
+#define MMEA2_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                0x9
+#define MMEA2_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                  0x00000007L
+#define MMEA2_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                  0x00000038L
+#define MMEA2_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                  0x000001C0L
+#define MMEA2_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                  0x00000E00L
+//MMEA2_IO_WR_PRI_FIXED
+#define MMEA2_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                0x0
+#define MMEA2_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                0x3
+#define MMEA2_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                0x6
+#define MMEA2_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                0x9
+#define MMEA2_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                  0x00000007L
+#define MMEA2_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                  0x00000038L
+#define MMEA2_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                  0x000001C0L
+#define MMEA2_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                  0x00000E00L
+//MMEA2_IO_RD_PRI_URGENCY
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                            0x0
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                            0x3
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                            0x6
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                            0x9
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                   0xc
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                   0xd
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                   0xe
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                   0xf
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                              0x00000E00L
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                     0x00001000L
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                     0x00002000L
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                     0x00004000L
+#define MMEA2_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                     0x00008000L
+//MMEA2_IO_WR_PRI_URGENCY
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                            0x0
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                            0x3
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                            0x6
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                            0x9
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                   0xc
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                   0xd
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                   0xe
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                   0xf
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                              0x00000E00L
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                     0x00001000L
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                     0x00002000L
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                     0x00004000L
+#define MMEA2_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                     0x00008000L
+//MMEA2_IO_RD_PRI_URGENCY_MASKING
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                     0x0
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                     0x1
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                     0x2
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                     0x3
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                     0x4
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                     0x5
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                     0x6
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                     0x7
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                     0x8
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                     0x9
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                    0xa
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                    0xb
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                    0xc
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                    0xd
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                    0xe
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                    0xf
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                    0x10
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                    0x11
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                    0x12
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                    0x13
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                    0x14
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                    0x15
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                    0x16
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                    0x17
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                    0x18
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                    0x19
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                    0x1a
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                    0x1b
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                    0x1c
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                    0x1d
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                    0x1e
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                    0x1f
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                       0x00000001L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                       0x00000002L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                       0x00000004L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                       0x00000008L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                       0x00000010L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                       0x00000020L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                       0x00000040L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                       0x00000080L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                       0x00000100L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                       0x00000200L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                      0x00000400L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                      0x00000800L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                      0x00001000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                      0x00002000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                      0x00004000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                      0x00008000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                      0x00010000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                      0x00020000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                      0x00040000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                      0x00080000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                      0x00100000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                      0x00200000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                      0x00400000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                      0x00800000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                      0x01000000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                      0x02000000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                      0x04000000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                      0x08000000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                      0x10000000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                      0x20000000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                      0x40000000L
+#define MMEA2_IO_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                      0x80000000L
+//MMEA2_IO_WR_PRI_URGENCY_MASKING
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                     0x0
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                     0x1
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                     0x2
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                     0x3
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                     0x4
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                     0x5
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                     0x6
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                     0x7
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                     0x8
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                     0x9
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                    0xa
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                    0xb
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                    0xc
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                    0xd
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                    0xe
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                    0xf
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                    0x10
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                    0x11
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                    0x12
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                    0x13
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                    0x14
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                    0x15
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                    0x16
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                    0x17
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                    0x18
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                    0x19
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                    0x1a
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                    0x1b
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                    0x1c
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                    0x1d
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                    0x1e
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                    0x1f
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                       0x00000001L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                       0x00000002L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                       0x00000004L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                       0x00000008L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                       0x00000010L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                       0x00000020L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                       0x00000040L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                       0x00000080L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                       0x00000100L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                       0x00000200L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                      0x00000400L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                      0x00000800L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                      0x00001000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                      0x00002000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                      0x00004000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                      0x00008000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                      0x00010000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                      0x00020000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                      0x00040000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                      0x00080000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                      0x00100000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                      0x00200000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                      0x00400000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                      0x00800000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                      0x01000000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                      0x02000000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                      0x04000000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                      0x08000000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                      0x10000000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                      0x20000000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                      0x40000000L
+#define MMEA2_IO_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                      0x80000000L
+//MMEA2_IO_RD_PRI_QUANT_PRI1
+#define MMEA2_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA2_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA2_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA2_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA2_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA2_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA2_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA2_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA2_IO_RD_PRI_QUANT_PRI2
+#define MMEA2_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA2_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA2_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA2_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA2_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA2_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA2_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA2_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA2_IO_RD_PRI_QUANT_PRI3
+#define MMEA2_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA2_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA2_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA2_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA2_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA2_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA2_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA2_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA2_IO_WR_PRI_QUANT_PRI1
+#define MMEA2_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA2_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA2_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA2_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA2_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA2_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA2_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA2_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA2_IO_WR_PRI_QUANT_PRI2
+#define MMEA2_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA2_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA2_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA2_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA2_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA2_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA2_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA2_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA2_IO_WR_PRI_QUANT_PRI3
+#define MMEA2_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA2_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA2_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA2_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA2_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA2_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA2_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA2_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA2_SDP_ARB_DRAM
+#define MMEA2_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL__SHIFT                                                      0x0
+#define MMEA2_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA__SHIFT                                                      0x8
+#define MMEA2_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI__SHIFT                                                         0x10
+#define MMEA2_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI__SHIFT                                                         0x11
+#define MMEA2_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES__SHIFT                                                         0x12
+#define MMEA2_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES__SHIFT                                                         0x13
+#define MMEA2_SDP_ARB_DRAM__EOB_ON_EXPIRE__SHIFT                                                              0x14
+#define MMEA2_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                     0x15
+#define MMEA2_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING__SHIFT                                                       0x16
+#define MMEA2_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL_MASK                                                        0x0000007FL
+#define MMEA2_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA_MASK                                                        0x00007F00L
+#define MMEA2_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI_MASK                                                           0x00010000L
+#define MMEA2_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI_MASK                                                           0x00020000L
+#define MMEA2_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES_MASK                                                           0x00040000L
+#define MMEA2_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES_MASK                                                           0x00080000L
+#define MMEA2_SDP_ARB_DRAM__EOB_ON_EXPIRE_MASK                                                                0x00100000L
+#define MMEA2_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE_MASK                                                       0x00200000L
+#define MMEA2_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING_MASK                                                         0x00400000L
+//MMEA2_SDP_ARB_GMI
+#define MMEA2_SDP_ARB_GMI__RDWR_BURST_LIMIT_CYCL__SHIFT                                                       0x0
+#define MMEA2_SDP_ARB_GMI__RDWR_BURST_LIMIT_DATA__SHIFT                                                       0x8
+#define MMEA2_SDP_ARB_GMI__EARLY_SW2RD_ON_PRI__SHIFT                                                          0x10
+#define MMEA2_SDP_ARB_GMI__EARLY_SW2WR_ON_PRI__SHIFT                                                          0x11
+#define MMEA2_SDP_ARB_GMI__EARLY_SW2RD_ON_RES__SHIFT                                                          0x12
+#define MMEA2_SDP_ARB_GMI__EARLY_SW2WR_ON_RES__SHIFT                                                          0x13
+#define MMEA2_SDP_ARB_GMI__EOB_ON_EXPIRE__SHIFT                                                               0x14
+#define MMEA2_SDP_ARB_GMI__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                      0x15
+#define MMEA2_SDP_ARB_GMI__ALLOW_CHAIN_BREAKING__SHIFT                                                        0x16
+#define MMEA2_SDP_ARB_GMI__RDWR_BURST_LIMIT_CYCL_MASK                                                         0x0000007FL
+#define MMEA2_SDP_ARB_GMI__RDWR_BURST_LIMIT_DATA_MASK                                                         0x00007F00L
+#define MMEA2_SDP_ARB_GMI__EARLY_SW2RD_ON_PRI_MASK                                                            0x00010000L
+#define MMEA2_SDP_ARB_GMI__EARLY_SW2WR_ON_PRI_MASK                                                            0x00020000L
+#define MMEA2_SDP_ARB_GMI__EARLY_SW2RD_ON_RES_MASK                                                            0x00040000L
+#define MMEA2_SDP_ARB_GMI__EARLY_SW2WR_ON_RES_MASK                                                            0x00080000L
+#define MMEA2_SDP_ARB_GMI__EOB_ON_EXPIRE_MASK                                                                 0x00100000L
+#define MMEA2_SDP_ARB_GMI__DECOUPLE_RDWR_BNKSTATE_MASK                                                        0x00200000L
+#define MMEA2_SDP_ARB_GMI__ALLOW_CHAIN_BREAKING_MASK                                                          0x00400000L
+//MMEA2_SDP_ARB_FINAL
+#define MMEA2_SDP_ARB_FINAL__DRAM_BURST_LIMIT__SHIFT                                                          0x0
+#define MMEA2_SDP_ARB_FINAL__GMI_BURST_LIMIT__SHIFT                                                           0x5
+#define MMEA2_SDP_ARB_FINAL__IO_BURST_LIMIT__SHIFT                                                            0xa
+#define MMEA2_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER__SHIFT                                                    0xf
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC0__SHIFT                                                                0x11
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC1__SHIFT                                                                0x12
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC2__SHIFT                                                                0x13
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC3__SHIFT                                                                0x14
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC4__SHIFT                                                                0x15
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC5__SHIFT                                                                0x16
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC6__SHIFT                                                                0x17
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC7__SHIFT                                                                0x18
+#define MMEA2_SDP_ARB_FINAL__ERREVENT_ON_ERROR__SHIFT                                                         0x19
+#define MMEA2_SDP_ARB_FINAL__HALTREQ_ON_ERROR__SHIFT                                                          0x1a
+#define MMEA2_SDP_ARB_FINAL__DRAM_BURST_STRETCH__SHIFT                                                        0x1b
+#define MMEA2_SDP_ARB_FINAL__GMI_BURST_STRETCH__SHIFT                                                         0x1c
+#define MMEA2_SDP_ARB_FINAL__DRAM_BURST_LIMIT_MASK                                                            0x0000001FL
+#define MMEA2_SDP_ARB_FINAL__GMI_BURST_LIMIT_MASK                                                             0x000003E0L
+#define MMEA2_SDP_ARB_FINAL__IO_BURST_LIMIT_MASK                                                              0x00007C00L
+#define MMEA2_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER_MASK                                                      0x00018000L
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC0_MASK                                                                  0x00020000L
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC1_MASK                                                                  0x00040000L
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC2_MASK                                                                  0x00080000L
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC3_MASK                                                                  0x00100000L
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC4_MASK                                                                  0x00200000L
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC5_MASK                                                                  0x00400000L
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC6_MASK                                                                  0x00800000L
+#define MMEA2_SDP_ARB_FINAL__RDONLY_VC7_MASK                                                                  0x01000000L
+#define MMEA2_SDP_ARB_FINAL__ERREVENT_ON_ERROR_MASK                                                           0x02000000L
+#define MMEA2_SDP_ARB_FINAL__HALTREQ_ON_ERROR_MASK                                                            0x04000000L
+#define MMEA2_SDP_ARB_FINAL__DRAM_BURST_STRETCH_MASK                                                          0x08000000L
+#define MMEA2_SDP_ARB_FINAL__GMI_BURST_STRETCH_MASK                                                           0x10000000L
+//MMEA2_SDP_DRAM_PRIORITY
+#define MMEA2_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                    0x0
+#define MMEA2_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                    0x4
+#define MMEA2_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                    0x8
+#define MMEA2_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                    0xc
+#define MMEA2_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                    0x10
+#define MMEA2_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                    0x14
+#define MMEA2_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                    0x18
+#define MMEA2_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                    0x1c
+#define MMEA2_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                      0x0000000FL
+#define MMEA2_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                      0x000000F0L
+#define MMEA2_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                      0x00000F00L
+#define MMEA2_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                      0x0000F000L
+#define MMEA2_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                      0x000F0000L
+#define MMEA2_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                      0x00F00000L
+#define MMEA2_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                      0x0F000000L
+#define MMEA2_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                      0xF0000000L
+//MMEA2_SDP_GMI_PRIORITY
+#define MMEA2_SDP_GMI_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                     0x0
+#define MMEA2_SDP_GMI_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                     0x4
+#define MMEA2_SDP_GMI_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                     0x8
+#define MMEA2_SDP_GMI_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                     0xc
+#define MMEA2_SDP_GMI_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                     0x10
+#define MMEA2_SDP_GMI_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                     0x14
+#define MMEA2_SDP_GMI_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                     0x18
+#define MMEA2_SDP_GMI_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                     0x1c
+#define MMEA2_SDP_GMI_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                       0x0000000FL
+#define MMEA2_SDP_GMI_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                       0x000000F0L
+#define MMEA2_SDP_GMI_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                       0x00000F00L
+#define MMEA2_SDP_GMI_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                       0x0000F000L
+#define MMEA2_SDP_GMI_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                       0x000F0000L
+#define MMEA2_SDP_GMI_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                       0x00F00000L
+#define MMEA2_SDP_GMI_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                       0x0F000000L
+#define MMEA2_SDP_GMI_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                       0xF0000000L
+//MMEA2_SDP_IO_PRIORITY
+#define MMEA2_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                      0x0
+#define MMEA2_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                      0x4
+#define MMEA2_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                      0x8
+#define MMEA2_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                      0xc
+#define MMEA2_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                      0x10
+#define MMEA2_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                      0x14
+#define MMEA2_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                      0x18
+#define MMEA2_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                      0x1c
+#define MMEA2_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                        0x0000000FL
+#define MMEA2_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                        0x000000F0L
+#define MMEA2_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                        0x00000F00L
+#define MMEA2_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                        0x0000F000L
+#define MMEA2_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                        0x000F0000L
+#define MMEA2_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                        0x00F00000L
+#define MMEA2_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                        0x0F000000L
+#define MMEA2_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                        0xF0000000L
+//MMEA2_SDP_CREDITS
+#define MMEA2_SDP_CREDITS__TAG_LIMIT__SHIFT                                                                   0x0
+#define MMEA2_SDP_CREDITS__WR_RESP_CREDITS__SHIFT                                                             0x8
+#define MMEA2_SDP_CREDITS__RD_RESP_CREDITS__SHIFT                                                             0x10
+#define MMEA2_SDP_CREDITS__TAG_LIMIT_MASK                                                                     0x000000FFL
+#define MMEA2_SDP_CREDITS__WR_RESP_CREDITS_MASK                                                               0x00007F00L
+#define MMEA2_SDP_CREDITS__RD_RESP_CREDITS_MASK                                                               0x007F0000L
+//MMEA2_SDP_TAG_RESERVE0
+#define MMEA2_SDP_TAG_RESERVE0__VC0__SHIFT                                                                    0x0
+#define MMEA2_SDP_TAG_RESERVE0__VC1__SHIFT                                                                    0x8
+#define MMEA2_SDP_TAG_RESERVE0__VC2__SHIFT                                                                    0x10
+#define MMEA2_SDP_TAG_RESERVE0__VC3__SHIFT                                                                    0x18
+#define MMEA2_SDP_TAG_RESERVE0__VC0_MASK                                                                      0x000000FFL
+#define MMEA2_SDP_TAG_RESERVE0__VC1_MASK                                                                      0x0000FF00L
+#define MMEA2_SDP_TAG_RESERVE0__VC2_MASK                                                                      0x00FF0000L
+#define MMEA2_SDP_TAG_RESERVE0__VC3_MASK                                                                      0xFF000000L
+//MMEA2_SDP_TAG_RESERVE1
+#define MMEA2_SDP_TAG_RESERVE1__VC4__SHIFT                                                                    0x0
+#define MMEA2_SDP_TAG_RESERVE1__VC5__SHIFT                                                                    0x8
+#define MMEA2_SDP_TAG_RESERVE1__VC6__SHIFT                                                                    0x10
+#define MMEA2_SDP_TAG_RESERVE1__VC7__SHIFT                                                                    0x18
+#define MMEA2_SDP_TAG_RESERVE1__VC4_MASK                                                                      0x000000FFL
+#define MMEA2_SDP_TAG_RESERVE1__VC5_MASK                                                                      0x0000FF00L
+#define MMEA2_SDP_TAG_RESERVE1__VC6_MASK                                                                      0x00FF0000L
+#define MMEA2_SDP_TAG_RESERVE1__VC7_MASK                                                                      0xFF000000L
+//MMEA2_SDP_VCC_RESERVE0
+#define MMEA2_SDP_VCC_RESERVE0__VC0_CREDITS__SHIFT                                                            0x0
+#define MMEA2_SDP_VCC_RESERVE0__VC1_CREDITS__SHIFT                                                            0x6
+#define MMEA2_SDP_VCC_RESERVE0__VC2_CREDITS__SHIFT                                                            0xc
+#define MMEA2_SDP_VCC_RESERVE0__VC3_CREDITS__SHIFT                                                            0x12
+#define MMEA2_SDP_VCC_RESERVE0__VC4_CREDITS__SHIFT                                                            0x18
+#define MMEA2_SDP_VCC_RESERVE0__VC0_CREDITS_MASK                                                              0x0000003FL
+#define MMEA2_SDP_VCC_RESERVE0__VC1_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA2_SDP_VCC_RESERVE0__VC2_CREDITS_MASK                                                              0x0003F000L
+#define MMEA2_SDP_VCC_RESERVE0__VC3_CREDITS_MASK                                                              0x00FC0000L
+#define MMEA2_SDP_VCC_RESERVE0__VC4_CREDITS_MASK                                                              0x3F000000L
+//MMEA2_SDP_VCC_RESERVE1
+#define MMEA2_SDP_VCC_RESERVE1__VC5_CREDITS__SHIFT                                                            0x0
+#define MMEA2_SDP_VCC_RESERVE1__VC6_CREDITS__SHIFT                                                            0x6
+#define MMEA2_SDP_VCC_RESERVE1__VC7_CREDITS__SHIFT                                                            0xc
+#define MMEA2_SDP_VCC_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                        0x1f
+#define MMEA2_SDP_VCC_RESERVE1__VC5_CREDITS_MASK                                                              0x0000003FL
+#define MMEA2_SDP_VCC_RESERVE1__VC6_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA2_SDP_VCC_RESERVE1__VC7_CREDITS_MASK                                                              0x0003F000L
+#define MMEA2_SDP_VCC_RESERVE1__DISTRIBUTE_POOL_MASK                                                          0x80000000L
+//MMEA2_SDP_VCD_RESERVE0
+#define MMEA2_SDP_VCD_RESERVE0__VC0_CREDITS__SHIFT                                                            0x0
+#define MMEA2_SDP_VCD_RESERVE0__VC1_CREDITS__SHIFT                                                            0x6
+#define MMEA2_SDP_VCD_RESERVE0__VC2_CREDITS__SHIFT                                                            0xc
+#define MMEA2_SDP_VCD_RESERVE0__VC3_CREDITS__SHIFT                                                            0x12
+#define MMEA2_SDP_VCD_RESERVE0__VC4_CREDITS__SHIFT                                                            0x18
+#define MMEA2_SDP_VCD_RESERVE0__VC0_CREDITS_MASK                                                              0x0000003FL
+#define MMEA2_SDP_VCD_RESERVE0__VC1_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA2_SDP_VCD_RESERVE0__VC2_CREDITS_MASK                                                              0x0003F000L
+#define MMEA2_SDP_VCD_RESERVE0__VC3_CREDITS_MASK                                                              0x00FC0000L
+#define MMEA2_SDP_VCD_RESERVE0__VC4_CREDITS_MASK                                                              0x3F000000L
+//MMEA2_SDP_VCD_RESERVE1
+#define MMEA2_SDP_VCD_RESERVE1__VC5_CREDITS__SHIFT                                                            0x0
+#define MMEA2_SDP_VCD_RESERVE1__VC6_CREDITS__SHIFT                                                            0x6
+#define MMEA2_SDP_VCD_RESERVE1__VC7_CREDITS__SHIFT                                                            0xc
+#define MMEA2_SDP_VCD_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                        0x1f
+#define MMEA2_SDP_VCD_RESERVE1__VC5_CREDITS_MASK                                                              0x0000003FL
+#define MMEA2_SDP_VCD_RESERVE1__VC6_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA2_SDP_VCD_RESERVE1__VC7_CREDITS_MASK                                                              0x0003F000L
+#define MMEA2_SDP_VCD_RESERVE1__DISTRIBUTE_POOL_MASK                                                          0x80000000L
+//MMEA2_SDP_REQ_CNTL
+#define MMEA2_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ__SHIFT                                                  0x0
+#define MMEA2_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE__SHIFT                                                 0x1
+#define MMEA2_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC__SHIFT                                                0x2
+#define MMEA2_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM__SHIFT                                                    0x3
+#define MMEA2_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI__SHIFT                                                     0x4
+#define MMEA2_SDP_REQ_CNTL__INNER_DOMAIN_MODE__SHIFT                                                          0x5
+#define MMEA2_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ__SHIFT                                                       0x6
+#define MMEA2_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE__SHIFT                                                      0x8
+#define MMEA2_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC__SHIFT                                                     0xa
+#define MMEA2_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ_MASK                                                    0x00000001L
+#define MMEA2_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE_MASK                                                   0x00000002L
+#define MMEA2_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC_MASK                                                  0x00000004L
+#define MMEA2_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM_MASK                                                      0x00000008L
+#define MMEA2_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI_MASK                                                       0x00000010L
+#define MMEA2_SDP_REQ_CNTL__INNER_DOMAIN_MODE_MASK                                                            0x00000020L
+#define MMEA2_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ_MASK                                                         0x000000C0L
+#define MMEA2_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE_MASK                                                        0x00000300L
+#define MMEA2_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC_MASK                                                       0x00000C00L
+//MMEA2_MISC
+#define MMEA2_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB__SHIFT                                                        0x0
+#define MMEA2_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB__SHIFT                                                        0x1
+#define MMEA2_MISC__RELATIVE_PRI_IN_GMI_RD_ARB__SHIFT                                                         0x2
+#define MMEA2_MISC__RELATIVE_PRI_IN_GMI_WR_ARB__SHIFT                                                         0x3
+#define MMEA2_MISC__RELATIVE_PRI_IN_IO_RD_ARB__SHIFT                                                          0x4
+#define MMEA2_MISC__RELATIVE_PRI_IN_IO_WR_ARB__SHIFT                                                          0x5
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC0__SHIFT                                                              0x6
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC1__SHIFT                                                              0x7
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC2__SHIFT                                                              0x8
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC3__SHIFT                                                              0x9
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC4__SHIFT                                                              0xa
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC5__SHIFT                                                              0xb
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC6__SHIFT                                                              0xc
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC7__SHIFT                                                              0xd
+#define MMEA2_MISC__EARLY_SDP_ORIGDATA__SHIFT                                                                 0xe
+#define MMEA2_MISC__LINKMGR_DYNAMIC_MODE__SHIFT                                                               0xf
+#define MMEA2_MISC__LINKMGR_HALT_THRESHOLD__SHIFT                                                             0x11
+#define MMEA2_MISC__LINKMGR_RECONNECT_DELAY__SHIFT                                                            0x13
+#define MMEA2_MISC__LINKMGR_IDLE_THRESHOLD__SHIFT                                                             0x15
+#define MMEA2_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB__SHIFT                                                     0x1a
+#define MMEA2_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB__SHIFT                                                      0x1b
+#define MMEA2_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB__SHIFT                                                         0x1c
+#define MMEA2_MISC__FAVOUR_LAST_CS_IN_GMI_ARB__SHIFT                                                          0x1d
+#define MMEA2_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB__SHIFT                                                       0x1e
+#define MMEA2_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB__SHIFT                                                        0x1f
+#define MMEA2_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB_MASK                                                          0x00000001L
+#define MMEA2_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB_MASK                                                          0x00000002L
+#define MMEA2_MISC__RELATIVE_PRI_IN_GMI_RD_ARB_MASK                                                           0x00000004L
+#define MMEA2_MISC__RELATIVE_PRI_IN_GMI_WR_ARB_MASK                                                           0x00000008L
+#define MMEA2_MISC__RELATIVE_PRI_IN_IO_RD_ARB_MASK                                                            0x00000010L
+#define MMEA2_MISC__RELATIVE_PRI_IN_IO_WR_ARB_MASK                                                            0x00000020L
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC0_MASK                                                                0x00000040L
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC1_MASK                                                                0x00000080L
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC2_MASK                                                                0x00000100L
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC3_MASK                                                                0x00000200L
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC4_MASK                                                                0x00000400L
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC5_MASK                                                                0x00000800L
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC6_MASK                                                                0x00001000L
+#define MMEA2_MISC__EARLYWRRET_ENABLE_VC7_MASK                                                                0x00002000L
+#define MMEA2_MISC__EARLY_SDP_ORIGDATA_MASK                                                                   0x00004000L
+#define MMEA2_MISC__LINKMGR_DYNAMIC_MODE_MASK                                                                 0x00018000L
+#define MMEA2_MISC__LINKMGR_HALT_THRESHOLD_MASK                                                               0x00060000L
+#define MMEA2_MISC__LINKMGR_RECONNECT_DELAY_MASK                                                              0x00180000L
+#define MMEA2_MISC__LINKMGR_IDLE_THRESHOLD_MASK                                                               0x03E00000L
+#define MMEA2_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB_MASK                                                       0x04000000L
+#define MMEA2_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB_MASK                                                        0x08000000L
+#define MMEA2_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB_MASK                                                           0x10000000L
+#define MMEA2_MISC__FAVOUR_LAST_CS_IN_GMI_ARB_MASK                                                            0x20000000L
+#define MMEA2_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB_MASK                                                         0x40000000L
+#define MMEA2_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB_MASK                                                          0x80000000L
+//MMEA2_LATENCY_SAMPLING
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_DRAM__SHIFT                                                          0x0
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_DRAM__SHIFT                                                          0x1
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_GMI__SHIFT                                                           0x2
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_GMI__SHIFT                                                           0x3
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_IO__SHIFT                                                            0x4
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_IO__SHIFT                                                            0x5
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_READ__SHIFT                                                          0x6
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_READ__SHIFT                                                          0x7
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_WRITE__SHIFT                                                         0x8
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_WRITE__SHIFT                                                         0x9
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET__SHIFT                                                    0xa
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET__SHIFT                                                    0xb
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET__SHIFT                                                  0xc
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET__SHIFT                                                  0xd
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_VC__SHIFT                                                            0xe
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_VC__SHIFT                                                            0x16
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_DRAM_MASK                                                            0x00000001L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_DRAM_MASK                                                            0x00000002L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_GMI_MASK                                                             0x00000004L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_GMI_MASK                                                             0x00000008L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_IO_MASK                                                              0x00000010L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_IO_MASK                                                              0x00000020L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_READ_MASK                                                            0x00000040L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_READ_MASK                                                            0x00000080L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_WRITE_MASK                                                           0x00000100L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_WRITE_MASK                                                           0x00000200L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET_MASK                                                      0x00000400L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET_MASK                                                      0x00000800L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET_MASK                                                    0x00001000L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET_MASK                                                    0x00002000L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER0_VC_MASK                                                              0x003FC000L
+#define MMEA2_LATENCY_SAMPLING__SAMPLER1_VC_MASK                                                              0x3FC00000L
+//MMEA2_PERFCOUNTER_LO
+#define MMEA2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define MMEA2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//MMEA2_PERFCOUNTER_HI
+#define MMEA2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define MMEA2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define MMEA2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define MMEA2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+//MMEA2_PERFCOUNTER0_CFG
+#define MMEA2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define MMEA2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define MMEA2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define MMEA2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define MMEA2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define MMEA2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define MMEA2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define MMEA2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define MMEA2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define MMEA2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//MMEA2_PERFCOUNTER1_CFG
+#define MMEA2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define MMEA2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define MMEA2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define MMEA2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define MMEA2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define MMEA2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define MMEA2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define MMEA2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define MMEA2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define MMEA2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//MMEA2_PERFCOUNTER_RSLT_CNTL
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define MMEA2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+//MMEA2_DSM_CNTL
+#define MMEA2_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x0
+#define MMEA2_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x2
+#define MMEA2_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x3
+#define MMEA2_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x5
+#define MMEA2_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x6
+#define MMEA2_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x8
+#define MMEA2_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x9
+#define MMEA2_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xb
+#define MMEA2_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0xc
+#define MMEA2_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xe
+#define MMEA2_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0xf
+#define MMEA2_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x11
+#define MMEA2_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x12
+#define MMEA2_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x14
+#define MMEA2_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x15
+#define MMEA2_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x17
+#define MMEA2_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00000003L
+#define MMEA2_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000004L
+#define MMEA2_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00000018L
+#define MMEA2_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000020L
+#define MMEA2_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                0x000000C0L
+#define MMEA2_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000100L
+#define MMEA2_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00000600L
+#define MMEA2_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000800L
+#define MMEA2_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00003000L
+#define MMEA2_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00004000L
+#define MMEA2_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00018000L
+#define MMEA2_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00020000L
+#define MMEA2_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x000C0000L
+#define MMEA2_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00100000L
+#define MMEA2_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00600000L
+#define MMEA2_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00800000L
+//MMEA2_DSM_CNTLA
+#define MMEA2_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                             0x0
+#define MMEA2_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                            0x2
+#define MMEA2_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                             0x3
+#define MMEA2_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                            0x5
+#define MMEA2_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x6
+#define MMEA2_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x8
+#define MMEA2_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x9
+#define MMEA2_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0xb
+#define MMEA2_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0xc
+#define MMEA2_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0xe
+#define MMEA2_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0xf
+#define MMEA2_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x11
+#define MMEA2_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x12
+#define MMEA2_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x14
+#define MMEA2_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                               0x00000003L
+#define MMEA2_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                              0x00000004L
+#define MMEA2_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                               0x00000018L
+#define MMEA2_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                              0x00000020L
+#define MMEA2_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x000000C0L
+#define MMEA2_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000100L
+#define MMEA2_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000600L
+#define MMEA2_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000800L
+#define MMEA2_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00003000L
+#define MMEA2_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00004000L
+#define MMEA2_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00018000L
+#define MMEA2_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00020000L
+#define MMEA2_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x000C0000L
+#define MMEA2_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00100000L
+//MMEA2_DSM_CNTLB
+#define MMEA2_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x0
+#define MMEA2_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x2
+#define MMEA2_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x3
+#define MMEA2_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x5
+#define MMEA2_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x6
+#define MMEA2_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x8
+#define MMEA2_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x9
+#define MMEA2_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xb
+#define MMEA2_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000003L
+#define MMEA2_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000004L
+#define MMEA2_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000018L
+#define MMEA2_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000020L
+#define MMEA2_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA_MASK                                                    0x000000C0L
+#define MMEA2_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000100L
+#define MMEA2_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000600L
+#define MMEA2_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000800L
+//MMEA2_DSM_CNTL2
+#define MMEA2_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x0
+#define MMEA2_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                             0x2
+#define MMEA2_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x3
+#define MMEA2_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                             0x5
+#define MMEA2_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x6
+#define MMEA2_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                            0x8
+#define MMEA2_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x9
+#define MMEA2_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                               0xb
+#define MMEA2_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                               0xc
+#define MMEA2_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                               0xe
+#define MMEA2_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0xf
+#define MMEA2_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x11
+#define MMEA2_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x12
+#define MMEA2_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x14
+#define MMEA2_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x15
+#define MMEA2_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0x17
+#define MMEA2_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                  0x1a
+#define MMEA2_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                               0x00000003L
+#define MMEA2_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                               0x00000004L
+#define MMEA2_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                               0x00000018L
+#define MMEA2_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                               0x00000020L
+#define MMEA2_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                              0x000000C0L
+#define MMEA2_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                              0x00000100L
+#define MMEA2_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00000600L
+#define MMEA2_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000800L
+#define MMEA2_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00003000L
+#define MMEA2_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                 0x00004000L
+#define MMEA2_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00018000L
+#define MMEA2_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00020000L
+#define MMEA2_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x000C0000L
+#define MMEA2_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00100000L
+#define MMEA2_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x00600000L
+#define MMEA2_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00800000L
+#define MMEA2_DSM_CNTL2__INJECT_DELAY_MASK                                                                    0xFC000000L
+//MMEA2_DSM_CNTL2A
+#define MMEA2_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                           0x0
+#define MMEA2_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                           0x2
+#define MMEA2_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                           0x3
+#define MMEA2_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                           0x5
+#define MMEA2_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x6
+#define MMEA2_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x8
+#define MMEA2_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x9
+#define MMEA2_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0xb
+#define MMEA2_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0xc
+#define MMEA2_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0xe
+#define MMEA2_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0xf
+#define MMEA2_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x11
+#define MMEA2_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x12
+#define MMEA2_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x14
+#define MMEA2_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                             0x00000003L
+#define MMEA2_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                             0x00000004L
+#define MMEA2_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                             0x00000018L
+#define MMEA2_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                             0x00000020L
+#define MMEA2_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x000000C0L
+#define MMEA2_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000100L
+#define MMEA2_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000600L
+#define MMEA2_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000800L
+#define MMEA2_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x00003000L
+#define MMEA2_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00004000L
+#define MMEA2_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00018000L
+#define MMEA2_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00020000L
+#define MMEA2_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x000C0000L
+#define MMEA2_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00100000L
+//MMEA2_DSM_CNTL2B
+#define MMEA2_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x0
+#define MMEA2_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY__SHIFT                                                0x2
+#define MMEA2_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x3
+#define MMEA2_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY__SHIFT                                                0x5
+#define MMEA2_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x6
+#define MMEA2_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY__SHIFT                                                0x8
+#define MMEA2_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x9
+#define MMEA2_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY__SHIFT                                                0xb
+#define MMEA2_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000003L
+#define MMEA2_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000004L
+#define MMEA2_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000018L
+#define MMEA2_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000020L
+#define MMEA2_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT_MASK                                                  0x000000C0L
+#define MMEA2_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000100L
+#define MMEA2_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000600L
+#define MMEA2_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000800L
+//MMEA2_CGTT_CLK_CTRL
+#define MMEA2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define MMEA2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define MMEA2_CGTT_CLK_CTRL__SPARE0__SHIFT                                                                    0xc
+#define MMEA2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE__SHIFT                                                 0x14
+#define MMEA2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ__SHIFT                                                  0x15
+#define MMEA2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN__SHIFT                                                0x16
+#define MMEA2_CGTT_CLK_CTRL__SPARE1__SHIFT                                                                    0x17
+#define MMEA2_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                               0x1b
+#define MMEA2_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE__SHIFT                                                       0x1c
+#define MMEA2_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ__SHIFT                                                        0x1d
+#define MMEA2_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                      0x1e
+#define MMEA2_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                    0x1f
+#define MMEA2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define MMEA2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define MMEA2_CGTT_CLK_CTRL__SPARE0_MASK                                                                      0x000FF000L
+#define MMEA2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE_MASK                                                   0x00100000L
+#define MMEA2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ_MASK                                                    0x00200000L
+#define MMEA2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN_MASK                                                  0x00400000L
+#define MMEA2_CGTT_CLK_CTRL__SPARE1_MASK                                                                      0x07800000L
+#define MMEA2_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                                 0x08000000L
+#define MMEA2_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE_MASK                                                         0x10000000L
+#define MMEA2_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ_MASK                                                          0x20000000L
+#define MMEA2_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                        0x40000000L
+#define MMEA2_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                      0x80000000L
+//MMEA2_EDC_MODE
+#define MMEA2_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                  0x10
+#define MMEA2_EDC_MODE__GATE_FUE__SHIFT                                                                       0x11
+#define MMEA2_EDC_MODE__DED_MODE__SHIFT                                                                       0x14
+#define MMEA2_EDC_MODE__PROP_FED__SHIFT                                                                       0x1d
+#define MMEA2_EDC_MODE__BYPASS__SHIFT                                                                         0x1f
+#define MMEA2_EDC_MODE__COUNT_FED_OUT_MASK                                                                    0x00010000L
+#define MMEA2_EDC_MODE__GATE_FUE_MASK                                                                         0x00020000L
+#define MMEA2_EDC_MODE__DED_MODE_MASK                                                                         0x00300000L
+#define MMEA2_EDC_MODE__PROP_FED_MASK                                                                         0x20000000L
+#define MMEA2_EDC_MODE__BYPASS_MASK                                                                           0x80000000L
+//MMEA2_ERR_STATUS
+#define MMEA2_ERR_STATUS__SDP_RDRSP_STATUS__SHIFT                                                             0x0
+#define MMEA2_ERR_STATUS__SDP_WRRSP_STATUS__SHIFT                                                             0x4
+#define MMEA2_ERR_STATUS__SDP_RDRSP_DATASTATUS__SHIFT                                                         0x8
+#define MMEA2_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR__SHIFT                                                   0xa
+#define MMEA2_ERR_STATUS__CLEAR_ERROR_STATUS__SHIFT                                                           0xb
+#define MMEA2_ERR_STATUS__BUSY_ON_ERROR__SHIFT                                                                0xc
+#define MMEA2_ERR_STATUS__FUE_FLAG__SHIFT                                                                     0xd
+#define MMEA2_ERR_STATUS__IGNORE_RDRSP_FED__SHIFT                                                             0xe
+#define MMEA2_ERR_STATUS__INTERRUPT_ON_FATAL__SHIFT                                                           0xf
+#define MMEA2_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL__SHIFT                                                   0x10
+#define MMEA2_ERR_STATUS__LEVEL_INTERRUPT__SHIFT                                                              0x11
+#define MMEA2_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR__SHIFT                                                     0x12
+#define MMEA2_ERR_STATUS__FUE_FLAG_CLIENT__SHIFT                                                              0x13
+#define MMEA2_ERR_STATUS__SDP_RDRSP_STATUS_MASK                                                               0x0000000FL
+#define MMEA2_ERR_STATUS__SDP_WRRSP_STATUS_MASK                                                               0x000000F0L
+#define MMEA2_ERR_STATUS__SDP_RDRSP_DATASTATUS_MASK                                                           0x00000300L
+#define MMEA2_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR_MASK                                                     0x00000400L
+#define MMEA2_ERR_STATUS__CLEAR_ERROR_STATUS_MASK                                                             0x00000800L
+#define MMEA2_ERR_STATUS__BUSY_ON_ERROR_MASK                                                                  0x00001000L
+#define MMEA2_ERR_STATUS__FUE_FLAG_MASK                                                                       0x00002000L
+#define MMEA2_ERR_STATUS__IGNORE_RDRSP_FED_MASK                                                               0x00004000L
+#define MMEA2_ERR_STATUS__INTERRUPT_ON_FATAL_MASK                                                             0x00008000L
+#define MMEA2_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL_MASK                                                     0x00010000L
+#define MMEA2_ERR_STATUS__LEVEL_INTERRUPT_MASK                                                                0x00020000L
+#define MMEA2_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR_MASK                                                       0x00040000L
+#define MMEA2_ERR_STATUS__FUE_FLAG_CLIENT_MASK                                                                0x00080000L
+//MMEA2_MISC2
+#define MMEA2_MISC2__CSGROUP_SWAP_IN_DRAM_ARB__SHIFT                                                          0x0
+#define MMEA2_MISC2__CSGROUP_SWAP_IN_GMI_ARB__SHIFT                                                           0x1
+#define MMEA2_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM__SHIFT                                                       0x2
+#define MMEA2_MISC2__CSGRP_BURST_LIMIT_DATA_GMI__SHIFT                                                        0x7
+#define MMEA2_MISC2__IO_RDWR_PRIORITY_ENABLE__SHIFT                                                           0xc
+#define MMEA2_MISC2__RRET_SWAP_MODE__SHIFT                                                                    0xd
+#define MMEA2_MISC2__BLOCK_REQUESTS__SHIFT                                                                    0xe
+#define MMEA2_MISC2__REQUESTS_BLOCKED__SHIFT                                                                  0xf
+#define MMEA2_MISC2__DRAM_RD_THROTTLE__SHIFT                                                                  0x10
+#define MMEA2_MISC2__DRAM_WR_THROTTLE__SHIFT                                                                  0x11
+#define MMEA2_MISC2__GMI_RD_THROTTLE__SHIFT                                                                   0x12
+#define MMEA2_MISC2__GMI_WR_THROTTLE__SHIFT                                                                   0x13
+#define MMEA2_MISC2__CSGROUP_SWAP_IN_DRAM_ARB_MASK                                                            0x00000001L
+#define MMEA2_MISC2__CSGROUP_SWAP_IN_GMI_ARB_MASK                                                             0x00000002L
+#define MMEA2_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM_MASK                                                         0x0000007CL
+#define MMEA2_MISC2__CSGRP_BURST_LIMIT_DATA_GMI_MASK                                                          0x00000F80L
+#define MMEA2_MISC2__IO_RDWR_PRIORITY_ENABLE_MASK                                                             0x00001000L
+#define MMEA2_MISC2__RRET_SWAP_MODE_MASK                                                                      0x00002000L
+#define MMEA2_MISC2__BLOCK_REQUESTS_MASK                                                                      0x00004000L
+#define MMEA2_MISC2__REQUESTS_BLOCKED_MASK                                                                    0x00008000L
+#define MMEA2_MISC2__DRAM_RD_THROTTLE_MASK                                                                    0x00010000L
+#define MMEA2_MISC2__DRAM_WR_THROTTLE_MASK                                                                    0x00020000L
+#define MMEA2_MISC2__GMI_RD_THROTTLE_MASK                                                                     0x00040000L
+#define MMEA2_MISC2__GMI_WR_THROTTLE_MASK                                                                     0x00080000L
+//MMEA2_MISC_AON
+#define MMEA2_MISC_AON__LINKMGR_PARTACK_HYSTERESIS__SHIFT                                                     0x0
+#define MMEA2_MISC_AON__LINKMGR_PARTACK_DEASSERT_MODE__SHIFT                                                  0x2
+#define MMEA2_MISC_AON__LINKMGR_PARTACK_HYSTERESIS_MASK                                                       0x00000003L
+#define MMEA2_MISC_AON__LINKMGR_PARTACK_DEASSERT_MODE_MASK                                                    0x00000004L
+
+
+// addressBlock: aid_mmhub_ea_mmeadec3
+//MMEA3_DRAM_RD_CLI2GRP_MAP0
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                         0x0
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                         0x2
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                         0x4
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                         0x6
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                         0x8
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                         0xa
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                         0xc
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                         0xe
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                         0x10
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                         0x12
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                        0x14
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                        0x16
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                        0x18
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                        0x1a
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                        0x1c
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                        0x1e
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                           0x00000003L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                           0x0000000CL
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                           0x00000030L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                           0x000000C0L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                           0x00000300L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                           0x00000C00L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                           0x00003000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                           0x0000C000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                           0x00030000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                           0x000C0000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                          0x00300000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                          0x00C00000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                          0x03000000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                          0x0C000000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                          0x30000000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                          0xC0000000L
+//MMEA3_DRAM_RD_CLI2GRP_MAP1
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                        0x0
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                        0x2
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                        0x4
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                        0x6
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                        0x8
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                        0xa
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                        0xc
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                        0xe
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                        0x10
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                        0x12
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                        0x14
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                        0x16
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                        0x18
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                        0x1a
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                        0x1c
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                        0x1e
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                          0x00000003L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                          0x0000000CL
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                          0x00000030L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                          0x000000C0L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                          0x00000300L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                          0x00000C00L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                          0x00003000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                          0x0000C000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                          0x00030000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                          0x000C0000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                          0x00300000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                          0x00C00000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                          0x03000000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                          0x0C000000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                          0x30000000L
+#define MMEA3_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                          0xC0000000L
+//MMEA3_DRAM_WR_CLI2GRP_MAP0
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                         0x0
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                         0x2
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                         0x4
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                         0x6
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                         0x8
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                         0xa
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                         0xc
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                         0xe
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                         0x10
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                         0x12
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                        0x14
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                        0x16
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                        0x18
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                        0x1a
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                        0x1c
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                        0x1e
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                           0x00000003L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                           0x0000000CL
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                           0x00000030L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                           0x000000C0L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                           0x00000300L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                           0x00000C00L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                           0x00003000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                           0x0000C000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                           0x00030000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                           0x000C0000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                          0x00300000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                          0x00C00000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                          0x03000000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                          0x0C000000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                          0x30000000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                          0xC0000000L
+//MMEA3_DRAM_WR_CLI2GRP_MAP1
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                        0x0
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                        0x2
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                        0x4
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                        0x6
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                        0x8
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                        0xa
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                        0xc
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                        0xe
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                        0x10
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                        0x12
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                        0x14
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                        0x16
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                        0x18
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                        0x1a
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                        0x1c
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                        0x1e
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                          0x00000003L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                          0x0000000CL
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                          0x00000030L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                          0x000000C0L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                          0x00000300L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                          0x00000C00L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                          0x00003000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                          0x0000C000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                          0x00030000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                          0x000C0000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                          0x00300000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                          0x00C00000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                          0x03000000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                          0x0C000000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                          0x30000000L
+#define MMEA3_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                          0xC0000000L
+//MMEA3_DRAM_RD_GRP2VC_MAP
+#define MMEA3_DRAM_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                            0x0
+#define MMEA3_DRAM_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                            0x3
+#define MMEA3_DRAM_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                            0x6
+#define MMEA3_DRAM_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                            0x9
+#define MMEA3_DRAM_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                              0x00000007L
+#define MMEA3_DRAM_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                              0x00000038L
+#define MMEA3_DRAM_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                              0x000001C0L
+#define MMEA3_DRAM_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                              0x00000E00L
+//MMEA3_DRAM_WR_GRP2VC_MAP
+#define MMEA3_DRAM_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                            0x0
+#define MMEA3_DRAM_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                            0x3
+#define MMEA3_DRAM_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                            0x6
+#define MMEA3_DRAM_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                            0x9
+#define MMEA3_DRAM_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                              0x00000007L
+#define MMEA3_DRAM_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                              0x00000038L
+#define MMEA3_DRAM_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                              0x000001C0L
+#define MMEA3_DRAM_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                              0x00000E00L
+//MMEA3_DRAM_RD_LAZY
+#define MMEA3_DRAM_RD_LAZY__GROUP0_DELAY__SHIFT                                                               0x0
+#define MMEA3_DRAM_RD_LAZY__GROUP1_DELAY__SHIFT                                                               0x3
+#define MMEA3_DRAM_RD_LAZY__GROUP2_DELAY__SHIFT                                                               0x6
+#define MMEA3_DRAM_RD_LAZY__GROUP3_DELAY__SHIFT                                                               0x9
+#define MMEA3_DRAM_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                           0xc
+#define MMEA3_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                          0x14
+#define MMEA3_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                          0x1b
+#define MMEA3_DRAM_RD_LAZY__GROUP0_DELAY_MASK                                                                 0x00000007L
+#define MMEA3_DRAM_RD_LAZY__GROUP1_DELAY_MASK                                                                 0x00000038L
+#define MMEA3_DRAM_RD_LAZY__GROUP2_DELAY_MASK                                                                 0x000001C0L
+#define MMEA3_DRAM_RD_LAZY__GROUP3_DELAY_MASK                                                                 0x00000E00L
+#define MMEA3_DRAM_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                             0x0003F000L
+#define MMEA3_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                            0x07F00000L
+#define MMEA3_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                            0x78000000L
+//MMEA3_DRAM_WR_LAZY
+#define MMEA3_DRAM_WR_LAZY__GROUP0_DELAY__SHIFT                                                               0x0
+#define MMEA3_DRAM_WR_LAZY__GROUP1_DELAY__SHIFT                                                               0x3
+#define MMEA3_DRAM_WR_LAZY__GROUP2_DELAY__SHIFT                                                               0x6
+#define MMEA3_DRAM_WR_LAZY__GROUP3_DELAY__SHIFT                                                               0x9
+#define MMEA3_DRAM_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                           0xc
+#define MMEA3_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                          0x14
+#define MMEA3_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                          0x1b
+#define MMEA3_DRAM_WR_LAZY__GROUP0_DELAY_MASK                                                                 0x00000007L
+#define MMEA3_DRAM_WR_LAZY__GROUP1_DELAY_MASK                                                                 0x00000038L
+#define MMEA3_DRAM_WR_LAZY__GROUP2_DELAY_MASK                                                                 0x000001C0L
+#define MMEA3_DRAM_WR_LAZY__GROUP3_DELAY_MASK                                                                 0x00000E00L
+#define MMEA3_DRAM_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                             0x0003F000L
+#define MMEA3_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                            0x07F00000L
+#define MMEA3_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                            0x78000000L
+//MMEA3_DRAM_RD_CAM_CNTL
+#define MMEA3_DRAM_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                           0x0
+#define MMEA3_DRAM_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                           0x4
+#define MMEA3_DRAM_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                           0x8
+#define MMEA3_DRAM_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                           0xc
+#define MMEA3_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                   0x10
+#define MMEA3_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                   0x13
+#define MMEA3_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                   0x16
+#define MMEA3_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                   0x19
+#define MMEA3_DRAM_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                           0x1c
+#define MMEA3_DRAM_RD_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                     0x1d
+#define MMEA3_DRAM_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                             0x0000000FL
+#define MMEA3_DRAM_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                             0x000000F0L
+#define MMEA3_DRAM_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                             0x00000F00L
+#define MMEA3_DRAM_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                             0x0000F000L
+#define MMEA3_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                     0x00070000L
+#define MMEA3_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                     0x00380000L
+#define MMEA3_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                     0x01C00000L
+#define MMEA3_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                     0x0E000000L
+#define MMEA3_DRAM_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                             0x10000000L
+#define MMEA3_DRAM_RD_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                       0x20000000L
+//MMEA3_DRAM_WR_CAM_CNTL
+#define MMEA3_DRAM_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                           0x0
+#define MMEA3_DRAM_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                           0x4
+#define MMEA3_DRAM_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                           0x8
+#define MMEA3_DRAM_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                           0xc
+#define MMEA3_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                   0x10
+#define MMEA3_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                   0x13
+#define MMEA3_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                   0x16
+#define MMEA3_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                   0x19
+#define MMEA3_DRAM_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                           0x1c
+#define MMEA3_DRAM_WR_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                     0x1d
+#define MMEA3_DRAM_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                             0x0000000FL
+#define MMEA3_DRAM_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                             0x000000F0L
+#define MMEA3_DRAM_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                             0x00000F00L
+#define MMEA3_DRAM_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                             0x0000F000L
+#define MMEA3_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                     0x00070000L
+#define MMEA3_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                     0x00380000L
+#define MMEA3_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                     0x01C00000L
+#define MMEA3_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                     0x0E000000L
+#define MMEA3_DRAM_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                             0x10000000L
+#define MMEA3_DRAM_WR_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                       0x20000000L
+//MMEA3_DRAM_PAGE_BURST
+#define MMEA3_DRAM_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                             0x0
+#define MMEA3_DRAM_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                             0x8
+#define MMEA3_DRAM_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                             0x10
+#define MMEA3_DRAM_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                             0x18
+#define MMEA3_DRAM_PAGE_BURST__RD_LIMIT_LO_MASK                                                               0x000000FFL
+#define MMEA3_DRAM_PAGE_BURST__RD_LIMIT_HI_MASK                                                               0x0000FF00L
+#define MMEA3_DRAM_PAGE_BURST__WR_LIMIT_LO_MASK                                                               0x00FF0000L
+#define MMEA3_DRAM_PAGE_BURST__WR_LIMIT_HI_MASK                                                               0xFF000000L
+//MMEA3_DRAM_RD_PRI_AGE
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                  0xc
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                  0xf
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                  0x12
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                  0x15
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                    0x00007000L
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                    0x00038000L
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                    0x001C0000L
+#define MMEA3_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                    0x00E00000L
+//MMEA3_DRAM_WR_PRI_AGE
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                  0xc
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                  0xf
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                  0x12
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                  0x15
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                    0x00007000L
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                    0x00038000L
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                    0x001C0000L
+#define MMEA3_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                    0x00E00000L
+//MMEA3_DRAM_RD_PRI_QUEUING
+#define MMEA3_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                          0x0
+#define MMEA3_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                          0x3
+#define MMEA3_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                          0x6
+#define MMEA3_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                          0x9
+#define MMEA3_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA3_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA3_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA3_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                            0x00000E00L
+//MMEA3_DRAM_WR_PRI_QUEUING
+#define MMEA3_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                          0x0
+#define MMEA3_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                          0x3
+#define MMEA3_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                          0x6
+#define MMEA3_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                          0x9
+#define MMEA3_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA3_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA3_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA3_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                            0x00000E00L
+//MMEA3_DRAM_RD_PRI_FIXED
+#define MMEA3_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                              0x0
+#define MMEA3_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                              0x3
+#define MMEA3_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                              0x6
+#define MMEA3_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                              0x9
+#define MMEA3_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                0x00000007L
+#define MMEA3_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                0x00000038L
+#define MMEA3_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                0x000001C0L
+#define MMEA3_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                0x00000E00L
+//MMEA3_DRAM_WR_PRI_FIXED
+#define MMEA3_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                              0x0
+#define MMEA3_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                              0x3
+#define MMEA3_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                              0x6
+#define MMEA3_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                              0x9
+#define MMEA3_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                0x00000007L
+#define MMEA3_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                0x00000038L
+#define MMEA3_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                0x000001C0L
+#define MMEA3_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                0x00000E00L
+//MMEA3_DRAM_RD_PRI_URGENCY
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                          0x0
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                          0x3
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                          0x6
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                          0x9
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                 0xc
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                 0xd
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                 0xe
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                 0xf
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                            0x00000E00L
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                   0x00001000L
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                   0x00002000L
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                   0x00004000L
+#define MMEA3_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                   0x00008000L
+//MMEA3_DRAM_WR_PRI_URGENCY
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                          0x0
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                          0x3
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                          0x6
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                          0x9
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                 0xc
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                 0xd
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                 0xe
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                 0xf
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                            0x00000E00L
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                   0x00001000L
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                   0x00002000L
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                   0x00004000L
+#define MMEA3_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                   0x00008000L
+//MMEA3_DRAM_RD_PRI_QUANT_PRI1
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA3_DRAM_RD_PRI_QUANT_PRI2
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA3_DRAM_RD_PRI_QUANT_PRI3
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA3_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA3_DRAM_WR_PRI_QUANT_PRI1
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA3_DRAM_WR_PRI_QUANT_PRI2
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA3_DRAM_WR_PRI_QUANT_PRI3
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA3_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA3_GMI_RD_CLI2GRP_MAP0
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//MMEA3_GMI_RD_CLI2GRP_MAP1
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define MMEA3_GMI_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//MMEA3_GMI_WR_CLI2GRP_MAP0
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//MMEA3_GMI_WR_CLI2GRP_MAP1
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define MMEA3_GMI_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//MMEA3_GMI_RD_GRP2VC_MAP
+#define MMEA3_GMI_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define MMEA3_GMI_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define MMEA3_GMI_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define MMEA3_GMI_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define MMEA3_GMI_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define MMEA3_GMI_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define MMEA3_GMI_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define MMEA3_GMI_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//MMEA3_GMI_WR_GRP2VC_MAP
+#define MMEA3_GMI_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define MMEA3_GMI_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define MMEA3_GMI_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define MMEA3_GMI_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define MMEA3_GMI_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define MMEA3_GMI_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define MMEA3_GMI_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define MMEA3_GMI_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//MMEA3_GMI_RD_LAZY
+#define MMEA3_GMI_RD_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define MMEA3_GMI_RD_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define MMEA3_GMI_RD_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define MMEA3_GMI_RD_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define MMEA3_GMI_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define MMEA3_GMI_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define MMEA3_GMI_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define MMEA3_GMI_RD_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define MMEA3_GMI_RD_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define MMEA3_GMI_RD_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define MMEA3_GMI_RD_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define MMEA3_GMI_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define MMEA3_GMI_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define MMEA3_GMI_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//MMEA3_GMI_WR_LAZY
+#define MMEA3_GMI_WR_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define MMEA3_GMI_WR_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define MMEA3_GMI_WR_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define MMEA3_GMI_WR_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define MMEA3_GMI_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define MMEA3_GMI_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define MMEA3_GMI_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define MMEA3_GMI_WR_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define MMEA3_GMI_WR_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define MMEA3_GMI_WR_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define MMEA3_GMI_WR_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define MMEA3_GMI_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define MMEA3_GMI_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define MMEA3_GMI_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//MMEA3_GMI_RD_CAM_CNTL
+#define MMEA3_GMI_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define MMEA3_GMI_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define MMEA3_GMI_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define MMEA3_GMI_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define MMEA3_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define MMEA3_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define MMEA3_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define MMEA3_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define MMEA3_GMI_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define MMEA3_GMI_RD_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                      0x1d
+#define MMEA3_GMI_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define MMEA3_GMI_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define MMEA3_GMI_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define MMEA3_GMI_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define MMEA3_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define MMEA3_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define MMEA3_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define MMEA3_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define MMEA3_GMI_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+#define MMEA3_GMI_RD_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                        0x20000000L
+//MMEA3_GMI_WR_CAM_CNTL
+#define MMEA3_GMI_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define MMEA3_GMI_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define MMEA3_GMI_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define MMEA3_GMI_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define MMEA3_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define MMEA3_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define MMEA3_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define MMEA3_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define MMEA3_GMI_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define MMEA3_GMI_WR_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                      0x1d
+#define MMEA3_GMI_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define MMEA3_GMI_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define MMEA3_GMI_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define MMEA3_GMI_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define MMEA3_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define MMEA3_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define MMEA3_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define MMEA3_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define MMEA3_GMI_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+#define MMEA3_GMI_WR_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                        0x20000000L
+//MMEA3_GMI_PAGE_BURST
+#define MMEA3_GMI_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define MMEA3_GMI_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define MMEA3_GMI_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define MMEA3_GMI_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define MMEA3_GMI_PAGE_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define MMEA3_GMI_PAGE_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define MMEA3_GMI_PAGE_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define MMEA3_GMI_PAGE_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//MMEA3_GMI_RD_PRI_AGE
+#define MMEA3_GMI_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define MMEA3_GMI_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define MMEA3_GMI_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define MMEA3_GMI_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define MMEA3_GMI_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define MMEA3_GMI_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define MMEA3_GMI_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define MMEA3_GMI_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define MMEA3_GMI_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define MMEA3_GMI_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define MMEA3_GMI_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define MMEA3_GMI_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define MMEA3_GMI_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define MMEA3_GMI_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define MMEA3_GMI_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define MMEA3_GMI_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//MMEA3_GMI_WR_PRI_AGE
+#define MMEA3_GMI_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define MMEA3_GMI_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define MMEA3_GMI_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define MMEA3_GMI_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define MMEA3_GMI_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define MMEA3_GMI_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define MMEA3_GMI_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define MMEA3_GMI_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define MMEA3_GMI_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define MMEA3_GMI_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define MMEA3_GMI_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define MMEA3_GMI_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define MMEA3_GMI_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define MMEA3_GMI_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define MMEA3_GMI_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define MMEA3_GMI_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//MMEA3_GMI_RD_PRI_QUEUING
+#define MMEA3_GMI_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define MMEA3_GMI_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define MMEA3_GMI_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define MMEA3_GMI_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define MMEA3_GMI_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA3_GMI_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA3_GMI_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA3_GMI_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//MMEA3_GMI_WR_PRI_QUEUING
+#define MMEA3_GMI_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define MMEA3_GMI_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define MMEA3_GMI_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define MMEA3_GMI_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define MMEA3_GMI_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA3_GMI_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA3_GMI_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA3_GMI_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//MMEA3_GMI_RD_PRI_FIXED
+#define MMEA3_GMI_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define MMEA3_GMI_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define MMEA3_GMI_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define MMEA3_GMI_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define MMEA3_GMI_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define MMEA3_GMI_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define MMEA3_GMI_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define MMEA3_GMI_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//MMEA3_GMI_WR_PRI_FIXED
+#define MMEA3_GMI_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define MMEA3_GMI_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define MMEA3_GMI_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define MMEA3_GMI_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define MMEA3_GMI_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define MMEA3_GMI_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define MMEA3_GMI_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define MMEA3_GMI_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//MMEA3_GMI_RD_PRI_URGENCY
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define MMEA3_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//MMEA3_GMI_WR_PRI_URGENCY
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define MMEA3_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//MMEA3_GMI_RD_PRI_URGENCY_MASKING
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                    0x0
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                    0x1
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                    0x2
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                    0x3
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                    0x4
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                    0x5
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                    0x6
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                    0x7
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                    0x8
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                    0x9
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                   0xa
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                   0xb
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                   0xc
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                   0xd
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                   0xe
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                   0xf
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                   0x10
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                   0x11
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                   0x12
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                   0x13
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                   0x14
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                   0x15
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                   0x16
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                   0x17
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                   0x18
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                   0x19
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                   0x1a
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                   0x1b
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                   0x1c
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                   0x1d
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                   0x1e
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                   0x1f
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                      0x00000001L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                      0x00000002L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                      0x00000004L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                      0x00000008L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                      0x00000010L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                      0x00000020L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                      0x00000040L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                      0x00000080L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                      0x00000100L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                      0x00000200L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                     0x00000400L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                     0x00000800L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                     0x00001000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                     0x00002000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                     0x00004000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                     0x00008000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                     0x00010000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                     0x00020000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                     0x00040000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                     0x00080000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                     0x00100000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                     0x00200000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                     0x00400000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                     0x00800000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                     0x01000000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                     0x02000000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                     0x04000000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                     0x08000000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                     0x10000000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                     0x20000000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                     0x40000000L
+#define MMEA3_GMI_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                     0x80000000L
+//MMEA3_GMI_WR_PRI_URGENCY_MASKING
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                    0x0
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                    0x1
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                    0x2
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                    0x3
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                    0x4
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                    0x5
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                    0x6
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                    0x7
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                    0x8
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                    0x9
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                   0xa
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                   0xb
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                   0xc
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                   0xd
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                   0xe
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                   0xf
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                   0x10
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                   0x11
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                   0x12
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                   0x13
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                   0x14
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                   0x15
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                   0x16
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                   0x17
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                   0x18
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                   0x19
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                   0x1a
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                   0x1b
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                   0x1c
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                   0x1d
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                   0x1e
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                   0x1f
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                      0x00000001L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                      0x00000002L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                      0x00000004L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                      0x00000008L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                      0x00000010L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                      0x00000020L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                      0x00000040L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                      0x00000080L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                      0x00000100L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                      0x00000200L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                     0x00000400L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                     0x00000800L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                     0x00001000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                     0x00002000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                     0x00004000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                     0x00008000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                     0x00010000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                     0x00020000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                     0x00040000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                     0x00080000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                     0x00100000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                     0x00200000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                     0x00400000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                     0x00800000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                     0x01000000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                     0x02000000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                     0x04000000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                     0x08000000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                     0x10000000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                     0x20000000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                     0x40000000L
+#define MMEA3_GMI_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                     0x80000000L
+//MMEA3_GMI_RD_PRI_QUANT_PRI1
+#define MMEA3_GMI_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA3_GMI_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA3_GMI_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA3_GMI_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA3_GMI_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA3_GMI_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA3_GMI_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA3_GMI_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA3_GMI_RD_PRI_QUANT_PRI2
+#define MMEA3_GMI_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA3_GMI_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA3_GMI_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA3_GMI_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA3_GMI_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA3_GMI_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA3_GMI_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA3_GMI_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA3_GMI_RD_PRI_QUANT_PRI3
+#define MMEA3_GMI_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA3_GMI_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA3_GMI_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA3_GMI_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA3_GMI_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA3_GMI_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA3_GMI_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA3_GMI_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA3_GMI_WR_PRI_QUANT_PRI1
+#define MMEA3_GMI_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA3_GMI_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA3_GMI_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA3_GMI_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA3_GMI_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA3_GMI_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA3_GMI_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA3_GMI_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA3_GMI_WR_PRI_QUANT_PRI2
+#define MMEA3_GMI_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA3_GMI_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA3_GMI_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA3_GMI_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA3_GMI_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA3_GMI_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA3_GMI_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA3_GMI_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA3_GMI_WR_PRI_QUANT_PRI3
+#define MMEA3_GMI_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA3_GMI_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA3_GMI_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA3_GMI_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA3_GMI_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA3_GMI_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA3_GMI_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA3_GMI_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA3_IO_RD_CLI2GRP_MAP0
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                           0x0
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                           0x2
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                           0x4
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                           0x6
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                           0x8
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                           0xa
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                           0xc
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                           0xe
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                           0x10
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                           0x12
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                          0x14
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                          0x16
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                          0x18
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                          0x1a
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                          0x1c
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                          0x1e
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                             0x00000003L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                             0x0000000CL
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                             0x00000030L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                             0x000000C0L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                             0x00000300L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                             0x00000C00L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                             0x00003000L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                             0x0000C000L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                             0x00030000L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                             0x000C0000L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                            0x00300000L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                            0x00C00000L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                            0x03000000L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                            0x0C000000L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                            0x30000000L
+#define MMEA3_IO_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                            0xC0000000L
+//MMEA3_IO_RD_CLI2GRP_MAP1
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                          0x0
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                          0x2
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                          0x4
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                          0x6
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                          0x8
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                          0xa
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                          0xc
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                          0xe
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                          0x10
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                          0x12
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                          0x14
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                          0x16
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                          0x18
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                          0x1a
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                          0x1c
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                          0x1e
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                            0x00000003L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                            0x0000000CL
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                            0x00000030L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                            0x000000C0L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                            0x00000300L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                            0x00000C00L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                            0x00003000L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                            0x0000C000L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                            0x00030000L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                            0x000C0000L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                            0x00300000L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                            0x00C00000L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                            0x03000000L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                            0x0C000000L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                            0x30000000L
+#define MMEA3_IO_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                            0xC0000000L
+//MMEA3_IO_WR_CLI2GRP_MAP0
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                           0x0
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                           0x2
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                           0x4
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                           0x6
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                           0x8
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                           0xa
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                           0xc
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                           0xe
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                           0x10
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                           0x12
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                          0x14
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                          0x16
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                          0x18
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                          0x1a
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                          0x1c
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                          0x1e
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                             0x00000003L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                             0x0000000CL
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                             0x00000030L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                             0x000000C0L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                             0x00000300L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                             0x00000C00L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                             0x00003000L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                             0x0000C000L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                             0x00030000L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                             0x000C0000L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                            0x00300000L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                            0x00C00000L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                            0x03000000L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                            0x0C000000L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                            0x30000000L
+#define MMEA3_IO_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                            0xC0000000L
+//MMEA3_IO_WR_CLI2GRP_MAP1
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                          0x0
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                          0x2
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                          0x4
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                          0x6
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                          0x8
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                          0xa
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                          0xc
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                          0xe
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                          0x10
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                          0x12
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                          0x14
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                          0x16
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                          0x18
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                          0x1a
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                          0x1c
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                          0x1e
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                            0x00000003L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                            0x0000000CL
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                            0x00000030L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                            0x000000C0L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                            0x00000300L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                            0x00000C00L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                            0x00003000L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                            0x0000C000L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                            0x00030000L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                            0x000C0000L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                            0x00300000L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                            0x00C00000L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                            0x03000000L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                            0x0C000000L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                            0x30000000L
+#define MMEA3_IO_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                            0xC0000000L
+//MMEA3_IO_RD_COMBINE_FLUSH
+#define MMEA3_IO_RD_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                        0x0
+#define MMEA3_IO_RD_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                        0x4
+#define MMEA3_IO_RD_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                        0x8
+#define MMEA3_IO_RD_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                        0xc
+#define MMEA3_IO_RD_COMBINE_FLUSH__COMB_MODE__SHIFT                                                           0x10
+#define MMEA3_IO_RD_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                          0x0000000FL
+#define MMEA3_IO_RD_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                          0x000000F0L
+#define MMEA3_IO_RD_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                          0x00000F00L
+#define MMEA3_IO_RD_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                          0x0000F000L
+#define MMEA3_IO_RD_COMBINE_FLUSH__COMB_MODE_MASK                                                             0x00030000L
+//MMEA3_IO_WR_COMBINE_FLUSH
+#define MMEA3_IO_WR_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                        0x0
+#define MMEA3_IO_WR_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                        0x4
+#define MMEA3_IO_WR_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                        0x8
+#define MMEA3_IO_WR_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                        0xc
+#define MMEA3_IO_WR_COMBINE_FLUSH__COMB_MODE__SHIFT                                                           0x10
+#define MMEA3_IO_WR_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                          0x0000000FL
+#define MMEA3_IO_WR_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                          0x000000F0L
+#define MMEA3_IO_WR_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                          0x00000F00L
+#define MMEA3_IO_WR_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                          0x0000F000L
+#define MMEA3_IO_WR_COMBINE_FLUSH__COMB_MODE_MASK                                                             0x00030000L
+//MMEA3_IO_GROUP_BURST
+#define MMEA3_IO_GROUP_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define MMEA3_IO_GROUP_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define MMEA3_IO_GROUP_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define MMEA3_IO_GROUP_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define MMEA3_IO_GROUP_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define MMEA3_IO_GROUP_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define MMEA3_IO_GROUP_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define MMEA3_IO_GROUP_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//MMEA3_IO_RD_PRI_AGE
+#define MMEA3_IO_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                         0x0
+#define MMEA3_IO_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                         0x3
+#define MMEA3_IO_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                         0x6
+#define MMEA3_IO_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                         0x9
+#define MMEA3_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                    0xc
+#define MMEA3_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                    0xf
+#define MMEA3_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                    0x12
+#define MMEA3_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                    0x15
+#define MMEA3_IO_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                           0x00000007L
+#define MMEA3_IO_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                           0x00000038L
+#define MMEA3_IO_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                           0x000001C0L
+#define MMEA3_IO_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                           0x00000E00L
+#define MMEA3_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                      0x00007000L
+#define MMEA3_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                      0x00038000L
+#define MMEA3_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                      0x001C0000L
+#define MMEA3_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                      0x00E00000L
+//MMEA3_IO_WR_PRI_AGE
+#define MMEA3_IO_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                         0x0
+#define MMEA3_IO_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                         0x3
+#define MMEA3_IO_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                         0x6
+#define MMEA3_IO_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                         0x9
+#define MMEA3_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                    0xc
+#define MMEA3_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                    0xf
+#define MMEA3_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                    0x12
+#define MMEA3_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                    0x15
+#define MMEA3_IO_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                           0x00000007L
+#define MMEA3_IO_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                           0x00000038L
+#define MMEA3_IO_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                           0x000001C0L
+#define MMEA3_IO_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                           0x00000E00L
+#define MMEA3_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                      0x00007000L
+#define MMEA3_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                      0x00038000L
+#define MMEA3_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                      0x001C0000L
+#define MMEA3_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                      0x00E00000L
+//MMEA3_IO_RD_PRI_QUEUING
+#define MMEA3_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                            0x0
+#define MMEA3_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                            0x3
+#define MMEA3_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                            0x6
+#define MMEA3_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                            0x9
+#define MMEA3_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA3_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA3_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA3_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                              0x00000E00L
+//MMEA3_IO_WR_PRI_QUEUING
+#define MMEA3_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                            0x0
+#define MMEA3_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                            0x3
+#define MMEA3_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                            0x6
+#define MMEA3_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                            0x9
+#define MMEA3_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA3_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA3_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA3_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                              0x00000E00L
+//MMEA3_IO_RD_PRI_FIXED
+#define MMEA3_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                0x0
+#define MMEA3_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                0x3
+#define MMEA3_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                0x6
+#define MMEA3_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                0x9
+#define MMEA3_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                  0x00000007L
+#define MMEA3_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                  0x00000038L
+#define MMEA3_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                  0x000001C0L
+#define MMEA3_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                  0x00000E00L
+//MMEA3_IO_WR_PRI_FIXED
+#define MMEA3_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                0x0
+#define MMEA3_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                0x3
+#define MMEA3_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                0x6
+#define MMEA3_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                0x9
+#define MMEA3_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                  0x00000007L
+#define MMEA3_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                  0x00000038L
+#define MMEA3_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                  0x000001C0L
+#define MMEA3_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                  0x00000E00L
+//MMEA3_IO_RD_PRI_URGENCY
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                            0x0
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                            0x3
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                            0x6
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                            0x9
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                   0xc
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                   0xd
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                   0xe
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                   0xf
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                              0x00000E00L
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                     0x00001000L
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                     0x00002000L
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                     0x00004000L
+#define MMEA3_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                     0x00008000L
+//MMEA3_IO_WR_PRI_URGENCY
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                            0x0
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                            0x3
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                            0x6
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                            0x9
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                   0xc
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                   0xd
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                   0xe
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                   0xf
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                              0x00000E00L
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                     0x00001000L
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                     0x00002000L
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                     0x00004000L
+#define MMEA3_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                     0x00008000L
+//MMEA3_IO_RD_PRI_URGENCY_MASKING
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                     0x0
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                     0x1
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                     0x2
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                     0x3
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                     0x4
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                     0x5
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                     0x6
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                     0x7
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                     0x8
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                     0x9
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                    0xa
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                    0xb
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                    0xc
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                    0xd
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                    0xe
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                    0xf
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                    0x10
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                    0x11
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                    0x12
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                    0x13
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                    0x14
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                    0x15
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                    0x16
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                    0x17
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                    0x18
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                    0x19
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                    0x1a
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                    0x1b
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                    0x1c
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                    0x1d
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                    0x1e
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                    0x1f
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                       0x00000001L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                       0x00000002L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                       0x00000004L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                       0x00000008L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                       0x00000010L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                       0x00000020L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                       0x00000040L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                       0x00000080L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                       0x00000100L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                       0x00000200L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                      0x00000400L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                      0x00000800L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                      0x00001000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                      0x00002000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                      0x00004000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                      0x00008000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                      0x00010000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                      0x00020000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                      0x00040000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                      0x00080000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                      0x00100000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                      0x00200000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                      0x00400000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                      0x00800000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                      0x01000000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                      0x02000000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                      0x04000000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                      0x08000000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                      0x10000000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                      0x20000000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                      0x40000000L
+#define MMEA3_IO_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                      0x80000000L
+//MMEA3_IO_WR_PRI_URGENCY_MASKING
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                     0x0
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                     0x1
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                     0x2
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                     0x3
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                     0x4
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                     0x5
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                     0x6
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                     0x7
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                     0x8
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                     0x9
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                    0xa
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                    0xb
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                    0xc
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                    0xd
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                    0xe
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                    0xf
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                    0x10
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                    0x11
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                    0x12
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                    0x13
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                    0x14
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                    0x15
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                    0x16
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                    0x17
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                    0x18
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                    0x19
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                    0x1a
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                    0x1b
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                    0x1c
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                    0x1d
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                    0x1e
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                    0x1f
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                       0x00000001L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                       0x00000002L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                       0x00000004L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                       0x00000008L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                       0x00000010L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                       0x00000020L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                       0x00000040L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                       0x00000080L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                       0x00000100L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                       0x00000200L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                      0x00000400L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                      0x00000800L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                      0x00001000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                      0x00002000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                      0x00004000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                      0x00008000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                      0x00010000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                      0x00020000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                      0x00040000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                      0x00080000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                      0x00100000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                      0x00200000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                      0x00400000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                      0x00800000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                      0x01000000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                      0x02000000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                      0x04000000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                      0x08000000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                      0x10000000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                      0x20000000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                      0x40000000L
+#define MMEA3_IO_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                      0x80000000L
+//MMEA3_IO_RD_PRI_QUANT_PRI1
+#define MMEA3_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA3_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA3_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA3_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA3_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA3_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA3_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA3_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA3_IO_RD_PRI_QUANT_PRI2
+#define MMEA3_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA3_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA3_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA3_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA3_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA3_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA3_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA3_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA3_IO_RD_PRI_QUANT_PRI3
+#define MMEA3_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA3_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA3_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA3_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA3_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA3_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA3_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA3_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA3_IO_WR_PRI_QUANT_PRI1
+#define MMEA3_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA3_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA3_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA3_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA3_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA3_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA3_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA3_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA3_IO_WR_PRI_QUANT_PRI2
+#define MMEA3_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA3_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA3_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA3_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA3_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA3_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA3_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA3_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA3_IO_WR_PRI_QUANT_PRI3
+#define MMEA3_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA3_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA3_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA3_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA3_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA3_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA3_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA3_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA3_SDP_ARB_DRAM
+#define MMEA3_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL__SHIFT                                                      0x0
+#define MMEA3_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA__SHIFT                                                      0x8
+#define MMEA3_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI__SHIFT                                                         0x10
+#define MMEA3_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI__SHIFT                                                         0x11
+#define MMEA3_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES__SHIFT                                                         0x12
+#define MMEA3_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES__SHIFT                                                         0x13
+#define MMEA3_SDP_ARB_DRAM__EOB_ON_EXPIRE__SHIFT                                                              0x14
+#define MMEA3_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                     0x15
+#define MMEA3_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING__SHIFT                                                       0x16
+#define MMEA3_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL_MASK                                                        0x0000007FL
+#define MMEA3_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA_MASK                                                        0x00007F00L
+#define MMEA3_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI_MASK                                                           0x00010000L
+#define MMEA3_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI_MASK                                                           0x00020000L
+#define MMEA3_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES_MASK                                                           0x00040000L
+#define MMEA3_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES_MASK                                                           0x00080000L
+#define MMEA3_SDP_ARB_DRAM__EOB_ON_EXPIRE_MASK                                                                0x00100000L
+#define MMEA3_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE_MASK                                                       0x00200000L
+#define MMEA3_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING_MASK                                                         0x00400000L
+//MMEA3_SDP_ARB_GMI
+#define MMEA3_SDP_ARB_GMI__RDWR_BURST_LIMIT_CYCL__SHIFT                                                       0x0
+#define MMEA3_SDP_ARB_GMI__RDWR_BURST_LIMIT_DATA__SHIFT                                                       0x8
+#define MMEA3_SDP_ARB_GMI__EARLY_SW2RD_ON_PRI__SHIFT                                                          0x10
+#define MMEA3_SDP_ARB_GMI__EARLY_SW2WR_ON_PRI__SHIFT                                                          0x11
+#define MMEA3_SDP_ARB_GMI__EARLY_SW2RD_ON_RES__SHIFT                                                          0x12
+#define MMEA3_SDP_ARB_GMI__EARLY_SW2WR_ON_RES__SHIFT                                                          0x13
+#define MMEA3_SDP_ARB_GMI__EOB_ON_EXPIRE__SHIFT                                                               0x14
+#define MMEA3_SDP_ARB_GMI__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                      0x15
+#define MMEA3_SDP_ARB_GMI__ALLOW_CHAIN_BREAKING__SHIFT                                                        0x16
+#define MMEA3_SDP_ARB_GMI__RDWR_BURST_LIMIT_CYCL_MASK                                                         0x0000007FL
+#define MMEA3_SDP_ARB_GMI__RDWR_BURST_LIMIT_DATA_MASK                                                         0x00007F00L
+#define MMEA3_SDP_ARB_GMI__EARLY_SW2RD_ON_PRI_MASK                                                            0x00010000L
+#define MMEA3_SDP_ARB_GMI__EARLY_SW2WR_ON_PRI_MASK                                                            0x00020000L
+#define MMEA3_SDP_ARB_GMI__EARLY_SW2RD_ON_RES_MASK                                                            0x00040000L
+#define MMEA3_SDP_ARB_GMI__EARLY_SW2WR_ON_RES_MASK                                                            0x00080000L
+#define MMEA3_SDP_ARB_GMI__EOB_ON_EXPIRE_MASK                                                                 0x00100000L
+#define MMEA3_SDP_ARB_GMI__DECOUPLE_RDWR_BNKSTATE_MASK                                                        0x00200000L
+#define MMEA3_SDP_ARB_GMI__ALLOW_CHAIN_BREAKING_MASK                                                          0x00400000L
+//MMEA3_SDP_ARB_FINAL
+#define MMEA3_SDP_ARB_FINAL__DRAM_BURST_LIMIT__SHIFT                                                          0x0
+#define MMEA3_SDP_ARB_FINAL__GMI_BURST_LIMIT__SHIFT                                                           0x5
+#define MMEA3_SDP_ARB_FINAL__IO_BURST_LIMIT__SHIFT                                                            0xa
+#define MMEA3_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER__SHIFT                                                    0xf
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC0__SHIFT                                                                0x11
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC1__SHIFT                                                                0x12
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC2__SHIFT                                                                0x13
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC3__SHIFT                                                                0x14
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC4__SHIFT                                                                0x15
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC5__SHIFT                                                                0x16
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC6__SHIFT                                                                0x17
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC7__SHIFT                                                                0x18
+#define MMEA3_SDP_ARB_FINAL__ERREVENT_ON_ERROR__SHIFT                                                         0x19
+#define MMEA3_SDP_ARB_FINAL__HALTREQ_ON_ERROR__SHIFT                                                          0x1a
+#define MMEA3_SDP_ARB_FINAL__DRAM_BURST_STRETCH__SHIFT                                                        0x1b
+#define MMEA3_SDP_ARB_FINAL__GMI_BURST_STRETCH__SHIFT                                                         0x1c
+#define MMEA3_SDP_ARB_FINAL__DRAM_BURST_LIMIT_MASK                                                            0x0000001FL
+#define MMEA3_SDP_ARB_FINAL__GMI_BURST_LIMIT_MASK                                                             0x000003E0L
+#define MMEA3_SDP_ARB_FINAL__IO_BURST_LIMIT_MASK                                                              0x00007C00L
+#define MMEA3_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER_MASK                                                      0x00018000L
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC0_MASK                                                                  0x00020000L
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC1_MASK                                                                  0x00040000L
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC2_MASK                                                                  0x00080000L
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC3_MASK                                                                  0x00100000L
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC4_MASK                                                                  0x00200000L
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC5_MASK                                                                  0x00400000L
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC6_MASK                                                                  0x00800000L
+#define MMEA3_SDP_ARB_FINAL__RDONLY_VC7_MASK                                                                  0x01000000L
+#define MMEA3_SDP_ARB_FINAL__ERREVENT_ON_ERROR_MASK                                                           0x02000000L
+#define MMEA3_SDP_ARB_FINAL__HALTREQ_ON_ERROR_MASK                                                            0x04000000L
+#define MMEA3_SDP_ARB_FINAL__DRAM_BURST_STRETCH_MASK                                                          0x08000000L
+#define MMEA3_SDP_ARB_FINAL__GMI_BURST_STRETCH_MASK                                                           0x10000000L
+//MMEA3_SDP_DRAM_PRIORITY
+#define MMEA3_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                    0x0
+#define MMEA3_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                    0x4
+#define MMEA3_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                    0x8
+#define MMEA3_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                    0xc
+#define MMEA3_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                    0x10
+#define MMEA3_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                    0x14
+#define MMEA3_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                    0x18
+#define MMEA3_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                    0x1c
+#define MMEA3_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                      0x0000000FL
+#define MMEA3_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                      0x000000F0L
+#define MMEA3_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                      0x00000F00L
+#define MMEA3_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                      0x0000F000L
+#define MMEA3_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                      0x000F0000L
+#define MMEA3_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                      0x00F00000L
+#define MMEA3_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                      0x0F000000L
+#define MMEA3_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                      0xF0000000L
+//MMEA3_SDP_GMI_PRIORITY
+#define MMEA3_SDP_GMI_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                     0x0
+#define MMEA3_SDP_GMI_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                     0x4
+#define MMEA3_SDP_GMI_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                     0x8
+#define MMEA3_SDP_GMI_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                     0xc
+#define MMEA3_SDP_GMI_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                     0x10
+#define MMEA3_SDP_GMI_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                     0x14
+#define MMEA3_SDP_GMI_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                     0x18
+#define MMEA3_SDP_GMI_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                     0x1c
+#define MMEA3_SDP_GMI_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                       0x0000000FL
+#define MMEA3_SDP_GMI_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                       0x000000F0L
+#define MMEA3_SDP_GMI_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                       0x00000F00L
+#define MMEA3_SDP_GMI_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                       0x0000F000L
+#define MMEA3_SDP_GMI_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                       0x000F0000L
+#define MMEA3_SDP_GMI_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                       0x00F00000L
+#define MMEA3_SDP_GMI_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                       0x0F000000L
+#define MMEA3_SDP_GMI_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                       0xF0000000L
+//MMEA3_SDP_IO_PRIORITY
+#define MMEA3_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                      0x0
+#define MMEA3_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                      0x4
+#define MMEA3_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                      0x8
+#define MMEA3_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                      0xc
+#define MMEA3_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                      0x10
+#define MMEA3_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                      0x14
+#define MMEA3_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                      0x18
+#define MMEA3_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                      0x1c
+#define MMEA3_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                        0x0000000FL
+#define MMEA3_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                        0x000000F0L
+#define MMEA3_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                        0x00000F00L
+#define MMEA3_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                        0x0000F000L
+#define MMEA3_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                        0x000F0000L
+#define MMEA3_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                        0x00F00000L
+#define MMEA3_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                        0x0F000000L
+#define MMEA3_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                        0xF0000000L
+//MMEA3_SDP_CREDITS
+#define MMEA3_SDP_CREDITS__TAG_LIMIT__SHIFT                                                                   0x0
+#define MMEA3_SDP_CREDITS__WR_RESP_CREDITS__SHIFT                                                             0x8
+#define MMEA3_SDP_CREDITS__RD_RESP_CREDITS__SHIFT                                                             0x10
+#define MMEA3_SDP_CREDITS__TAG_LIMIT_MASK                                                                     0x000000FFL
+#define MMEA3_SDP_CREDITS__WR_RESP_CREDITS_MASK                                                               0x00007F00L
+#define MMEA3_SDP_CREDITS__RD_RESP_CREDITS_MASK                                                               0x007F0000L
+//MMEA3_SDP_TAG_RESERVE0
+#define MMEA3_SDP_TAG_RESERVE0__VC0__SHIFT                                                                    0x0
+#define MMEA3_SDP_TAG_RESERVE0__VC1__SHIFT                                                                    0x8
+#define MMEA3_SDP_TAG_RESERVE0__VC2__SHIFT                                                                    0x10
+#define MMEA3_SDP_TAG_RESERVE0__VC3__SHIFT                                                                    0x18
+#define MMEA3_SDP_TAG_RESERVE0__VC0_MASK                                                                      0x000000FFL
+#define MMEA3_SDP_TAG_RESERVE0__VC1_MASK                                                                      0x0000FF00L
+#define MMEA3_SDP_TAG_RESERVE0__VC2_MASK                                                                      0x00FF0000L
+#define MMEA3_SDP_TAG_RESERVE0__VC3_MASK                                                                      0xFF000000L
+//MMEA3_SDP_TAG_RESERVE1
+#define MMEA3_SDP_TAG_RESERVE1__VC4__SHIFT                                                                    0x0
+#define MMEA3_SDP_TAG_RESERVE1__VC5__SHIFT                                                                    0x8
+#define MMEA3_SDP_TAG_RESERVE1__VC6__SHIFT                                                                    0x10
+#define MMEA3_SDP_TAG_RESERVE1__VC7__SHIFT                                                                    0x18
+#define MMEA3_SDP_TAG_RESERVE1__VC4_MASK                                                                      0x000000FFL
+#define MMEA3_SDP_TAG_RESERVE1__VC5_MASK                                                                      0x0000FF00L
+#define MMEA3_SDP_TAG_RESERVE1__VC6_MASK                                                                      0x00FF0000L
+#define MMEA3_SDP_TAG_RESERVE1__VC7_MASK                                                                      0xFF000000L
+//MMEA3_SDP_VCC_RESERVE0
+#define MMEA3_SDP_VCC_RESERVE0__VC0_CREDITS__SHIFT                                                            0x0
+#define MMEA3_SDP_VCC_RESERVE0__VC1_CREDITS__SHIFT                                                            0x6
+#define MMEA3_SDP_VCC_RESERVE0__VC2_CREDITS__SHIFT                                                            0xc
+#define MMEA3_SDP_VCC_RESERVE0__VC3_CREDITS__SHIFT                                                            0x12
+#define MMEA3_SDP_VCC_RESERVE0__VC4_CREDITS__SHIFT                                                            0x18
+#define MMEA3_SDP_VCC_RESERVE0__VC0_CREDITS_MASK                                                              0x0000003FL
+#define MMEA3_SDP_VCC_RESERVE0__VC1_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA3_SDP_VCC_RESERVE0__VC2_CREDITS_MASK                                                              0x0003F000L
+#define MMEA3_SDP_VCC_RESERVE0__VC3_CREDITS_MASK                                                              0x00FC0000L
+#define MMEA3_SDP_VCC_RESERVE0__VC4_CREDITS_MASK                                                              0x3F000000L
+//MMEA3_SDP_VCC_RESERVE1
+#define MMEA3_SDP_VCC_RESERVE1__VC5_CREDITS__SHIFT                                                            0x0
+#define MMEA3_SDP_VCC_RESERVE1__VC6_CREDITS__SHIFT                                                            0x6
+#define MMEA3_SDP_VCC_RESERVE1__VC7_CREDITS__SHIFT                                                            0xc
+#define MMEA3_SDP_VCC_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                        0x1f
+#define MMEA3_SDP_VCC_RESERVE1__VC5_CREDITS_MASK                                                              0x0000003FL
+#define MMEA3_SDP_VCC_RESERVE1__VC6_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA3_SDP_VCC_RESERVE1__VC7_CREDITS_MASK                                                              0x0003F000L
+#define MMEA3_SDP_VCC_RESERVE1__DISTRIBUTE_POOL_MASK                                                          0x80000000L
+//MMEA3_SDP_VCD_RESERVE0
+#define MMEA3_SDP_VCD_RESERVE0__VC0_CREDITS__SHIFT                                                            0x0
+#define MMEA3_SDP_VCD_RESERVE0__VC1_CREDITS__SHIFT                                                            0x6
+#define MMEA3_SDP_VCD_RESERVE0__VC2_CREDITS__SHIFT                                                            0xc
+#define MMEA3_SDP_VCD_RESERVE0__VC3_CREDITS__SHIFT                                                            0x12
+#define MMEA3_SDP_VCD_RESERVE0__VC4_CREDITS__SHIFT                                                            0x18
+#define MMEA3_SDP_VCD_RESERVE0__VC0_CREDITS_MASK                                                              0x0000003FL
+#define MMEA3_SDP_VCD_RESERVE0__VC1_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA3_SDP_VCD_RESERVE0__VC2_CREDITS_MASK                                                              0x0003F000L
+#define MMEA3_SDP_VCD_RESERVE0__VC3_CREDITS_MASK                                                              0x00FC0000L
+#define MMEA3_SDP_VCD_RESERVE0__VC4_CREDITS_MASK                                                              0x3F000000L
+//MMEA3_SDP_VCD_RESERVE1
+#define MMEA3_SDP_VCD_RESERVE1__VC5_CREDITS__SHIFT                                                            0x0
+#define MMEA3_SDP_VCD_RESERVE1__VC6_CREDITS__SHIFT                                                            0x6
+#define MMEA3_SDP_VCD_RESERVE1__VC7_CREDITS__SHIFT                                                            0xc
+#define MMEA3_SDP_VCD_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                        0x1f
+#define MMEA3_SDP_VCD_RESERVE1__VC5_CREDITS_MASK                                                              0x0000003FL
+#define MMEA3_SDP_VCD_RESERVE1__VC6_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA3_SDP_VCD_RESERVE1__VC7_CREDITS_MASK                                                              0x0003F000L
+#define MMEA3_SDP_VCD_RESERVE1__DISTRIBUTE_POOL_MASK                                                          0x80000000L
+//MMEA3_SDP_REQ_CNTL
+#define MMEA3_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ__SHIFT                                                  0x0
+#define MMEA3_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE__SHIFT                                                 0x1
+#define MMEA3_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC__SHIFT                                                0x2
+#define MMEA3_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM__SHIFT                                                    0x3
+#define MMEA3_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI__SHIFT                                                     0x4
+#define MMEA3_SDP_REQ_CNTL__INNER_DOMAIN_MODE__SHIFT                                                          0x5
+#define MMEA3_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ__SHIFT                                                       0x6
+#define MMEA3_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE__SHIFT                                                      0x8
+#define MMEA3_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC__SHIFT                                                     0xa
+#define MMEA3_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ_MASK                                                    0x00000001L
+#define MMEA3_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE_MASK                                                   0x00000002L
+#define MMEA3_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC_MASK                                                  0x00000004L
+#define MMEA3_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM_MASK                                                      0x00000008L
+#define MMEA3_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI_MASK                                                       0x00000010L
+#define MMEA3_SDP_REQ_CNTL__INNER_DOMAIN_MODE_MASK                                                            0x00000020L
+#define MMEA3_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ_MASK                                                         0x000000C0L
+#define MMEA3_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE_MASK                                                        0x00000300L
+#define MMEA3_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC_MASK                                                       0x00000C00L
+//MMEA3_MISC
+#define MMEA3_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB__SHIFT                                                        0x0
+#define MMEA3_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB__SHIFT                                                        0x1
+#define MMEA3_MISC__RELATIVE_PRI_IN_GMI_RD_ARB__SHIFT                                                         0x2
+#define MMEA3_MISC__RELATIVE_PRI_IN_GMI_WR_ARB__SHIFT                                                         0x3
+#define MMEA3_MISC__RELATIVE_PRI_IN_IO_RD_ARB__SHIFT                                                          0x4
+#define MMEA3_MISC__RELATIVE_PRI_IN_IO_WR_ARB__SHIFT                                                          0x5
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC0__SHIFT                                                              0x6
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC1__SHIFT                                                              0x7
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC2__SHIFT                                                              0x8
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC3__SHIFT                                                              0x9
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC4__SHIFT                                                              0xa
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC5__SHIFT                                                              0xb
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC6__SHIFT                                                              0xc
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC7__SHIFT                                                              0xd
+#define MMEA3_MISC__EARLY_SDP_ORIGDATA__SHIFT                                                                 0xe
+#define MMEA3_MISC__LINKMGR_DYNAMIC_MODE__SHIFT                                                               0xf
+#define MMEA3_MISC__LINKMGR_HALT_THRESHOLD__SHIFT                                                             0x11
+#define MMEA3_MISC__LINKMGR_RECONNECT_DELAY__SHIFT                                                            0x13
+#define MMEA3_MISC__LINKMGR_IDLE_THRESHOLD__SHIFT                                                             0x15
+#define MMEA3_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB__SHIFT                                                     0x1a
+#define MMEA3_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB__SHIFT                                                      0x1b
+#define MMEA3_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB__SHIFT                                                         0x1c
+#define MMEA3_MISC__FAVOUR_LAST_CS_IN_GMI_ARB__SHIFT                                                          0x1d
+#define MMEA3_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB__SHIFT                                                       0x1e
+#define MMEA3_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB__SHIFT                                                        0x1f
+#define MMEA3_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB_MASK                                                          0x00000001L
+#define MMEA3_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB_MASK                                                          0x00000002L
+#define MMEA3_MISC__RELATIVE_PRI_IN_GMI_RD_ARB_MASK                                                           0x00000004L
+#define MMEA3_MISC__RELATIVE_PRI_IN_GMI_WR_ARB_MASK                                                           0x00000008L
+#define MMEA3_MISC__RELATIVE_PRI_IN_IO_RD_ARB_MASK                                                            0x00000010L
+#define MMEA3_MISC__RELATIVE_PRI_IN_IO_WR_ARB_MASK                                                            0x00000020L
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC0_MASK                                                                0x00000040L
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC1_MASK                                                                0x00000080L
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC2_MASK                                                                0x00000100L
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC3_MASK                                                                0x00000200L
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC4_MASK                                                                0x00000400L
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC5_MASK                                                                0x00000800L
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC6_MASK                                                                0x00001000L
+#define MMEA3_MISC__EARLYWRRET_ENABLE_VC7_MASK                                                                0x00002000L
+#define MMEA3_MISC__EARLY_SDP_ORIGDATA_MASK                                                                   0x00004000L
+#define MMEA3_MISC__LINKMGR_DYNAMIC_MODE_MASK                                                                 0x00018000L
+#define MMEA3_MISC__LINKMGR_HALT_THRESHOLD_MASK                                                               0x00060000L
+#define MMEA3_MISC__LINKMGR_RECONNECT_DELAY_MASK                                                              0x00180000L
+#define MMEA3_MISC__LINKMGR_IDLE_THRESHOLD_MASK                                                               0x03E00000L
+#define MMEA3_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB_MASK                                                       0x04000000L
+#define MMEA3_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB_MASK                                                        0x08000000L
+#define MMEA3_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB_MASK                                                           0x10000000L
+#define MMEA3_MISC__FAVOUR_LAST_CS_IN_GMI_ARB_MASK                                                            0x20000000L
+#define MMEA3_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB_MASK                                                         0x40000000L
+#define MMEA3_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB_MASK                                                          0x80000000L
+//MMEA3_LATENCY_SAMPLING
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_DRAM__SHIFT                                                          0x0
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_DRAM__SHIFT                                                          0x1
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_GMI__SHIFT                                                           0x2
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_GMI__SHIFT                                                           0x3
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_IO__SHIFT                                                            0x4
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_IO__SHIFT                                                            0x5
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_READ__SHIFT                                                          0x6
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_READ__SHIFT                                                          0x7
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_WRITE__SHIFT                                                         0x8
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_WRITE__SHIFT                                                         0x9
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET__SHIFT                                                    0xa
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET__SHIFT                                                    0xb
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET__SHIFT                                                  0xc
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET__SHIFT                                                  0xd
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_VC__SHIFT                                                            0xe
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_VC__SHIFT                                                            0x16
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_DRAM_MASK                                                            0x00000001L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_DRAM_MASK                                                            0x00000002L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_GMI_MASK                                                             0x00000004L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_GMI_MASK                                                             0x00000008L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_IO_MASK                                                              0x00000010L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_IO_MASK                                                              0x00000020L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_READ_MASK                                                            0x00000040L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_READ_MASK                                                            0x00000080L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_WRITE_MASK                                                           0x00000100L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_WRITE_MASK                                                           0x00000200L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET_MASK                                                      0x00000400L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET_MASK                                                      0x00000800L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET_MASK                                                    0x00001000L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET_MASK                                                    0x00002000L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER0_VC_MASK                                                              0x003FC000L
+#define MMEA3_LATENCY_SAMPLING__SAMPLER1_VC_MASK                                                              0x3FC00000L
+//MMEA3_PERFCOUNTER_LO
+#define MMEA3_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define MMEA3_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//MMEA3_PERFCOUNTER_HI
+#define MMEA3_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define MMEA3_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define MMEA3_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define MMEA3_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+//MMEA3_PERFCOUNTER0_CFG
+#define MMEA3_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define MMEA3_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define MMEA3_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define MMEA3_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define MMEA3_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define MMEA3_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define MMEA3_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define MMEA3_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define MMEA3_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define MMEA3_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//MMEA3_PERFCOUNTER1_CFG
+#define MMEA3_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define MMEA3_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define MMEA3_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define MMEA3_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define MMEA3_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define MMEA3_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define MMEA3_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define MMEA3_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define MMEA3_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define MMEA3_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//MMEA3_PERFCOUNTER_RSLT_CNTL
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define MMEA3_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+//MMEA3_DSM_CNTL
+#define MMEA3_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x0
+#define MMEA3_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x2
+#define MMEA3_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x3
+#define MMEA3_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x5
+#define MMEA3_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x6
+#define MMEA3_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x8
+#define MMEA3_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x9
+#define MMEA3_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xb
+#define MMEA3_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0xc
+#define MMEA3_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xe
+#define MMEA3_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0xf
+#define MMEA3_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x11
+#define MMEA3_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x12
+#define MMEA3_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x14
+#define MMEA3_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x15
+#define MMEA3_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x17
+#define MMEA3_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00000003L
+#define MMEA3_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000004L
+#define MMEA3_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00000018L
+#define MMEA3_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000020L
+#define MMEA3_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                0x000000C0L
+#define MMEA3_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000100L
+#define MMEA3_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00000600L
+#define MMEA3_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000800L
+#define MMEA3_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00003000L
+#define MMEA3_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00004000L
+#define MMEA3_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00018000L
+#define MMEA3_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00020000L
+#define MMEA3_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x000C0000L
+#define MMEA3_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00100000L
+#define MMEA3_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00600000L
+#define MMEA3_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00800000L
+//MMEA3_DSM_CNTLA
+#define MMEA3_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                             0x0
+#define MMEA3_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                            0x2
+#define MMEA3_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                             0x3
+#define MMEA3_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                            0x5
+#define MMEA3_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x6
+#define MMEA3_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x8
+#define MMEA3_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x9
+#define MMEA3_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0xb
+#define MMEA3_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0xc
+#define MMEA3_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0xe
+#define MMEA3_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0xf
+#define MMEA3_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x11
+#define MMEA3_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x12
+#define MMEA3_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x14
+#define MMEA3_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                               0x00000003L
+#define MMEA3_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                              0x00000004L
+#define MMEA3_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                               0x00000018L
+#define MMEA3_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                              0x00000020L
+#define MMEA3_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x000000C0L
+#define MMEA3_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000100L
+#define MMEA3_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000600L
+#define MMEA3_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000800L
+#define MMEA3_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00003000L
+#define MMEA3_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00004000L
+#define MMEA3_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00018000L
+#define MMEA3_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00020000L
+#define MMEA3_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x000C0000L
+#define MMEA3_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00100000L
+//MMEA3_DSM_CNTLB
+#define MMEA3_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x0
+#define MMEA3_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x2
+#define MMEA3_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x3
+#define MMEA3_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x5
+#define MMEA3_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x6
+#define MMEA3_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x8
+#define MMEA3_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x9
+#define MMEA3_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xb
+#define MMEA3_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000003L
+#define MMEA3_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000004L
+#define MMEA3_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000018L
+#define MMEA3_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000020L
+#define MMEA3_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA_MASK                                                    0x000000C0L
+#define MMEA3_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000100L
+#define MMEA3_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000600L
+#define MMEA3_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000800L
+//MMEA3_DSM_CNTL2
+#define MMEA3_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x0
+#define MMEA3_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                             0x2
+#define MMEA3_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x3
+#define MMEA3_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                             0x5
+#define MMEA3_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x6
+#define MMEA3_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                            0x8
+#define MMEA3_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x9
+#define MMEA3_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                               0xb
+#define MMEA3_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                               0xc
+#define MMEA3_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                               0xe
+#define MMEA3_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0xf
+#define MMEA3_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x11
+#define MMEA3_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x12
+#define MMEA3_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x14
+#define MMEA3_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x15
+#define MMEA3_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0x17
+#define MMEA3_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                  0x1a
+#define MMEA3_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                               0x00000003L
+#define MMEA3_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                               0x00000004L
+#define MMEA3_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                               0x00000018L
+#define MMEA3_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                               0x00000020L
+#define MMEA3_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                              0x000000C0L
+#define MMEA3_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                              0x00000100L
+#define MMEA3_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00000600L
+#define MMEA3_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000800L
+#define MMEA3_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00003000L
+#define MMEA3_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                 0x00004000L
+#define MMEA3_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00018000L
+#define MMEA3_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00020000L
+#define MMEA3_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x000C0000L
+#define MMEA3_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00100000L
+#define MMEA3_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x00600000L
+#define MMEA3_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00800000L
+#define MMEA3_DSM_CNTL2__INJECT_DELAY_MASK                                                                    0xFC000000L
+//MMEA3_DSM_CNTL2A
+#define MMEA3_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                           0x0
+#define MMEA3_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                           0x2
+#define MMEA3_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                           0x3
+#define MMEA3_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                           0x5
+#define MMEA3_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x6
+#define MMEA3_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x8
+#define MMEA3_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x9
+#define MMEA3_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0xb
+#define MMEA3_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0xc
+#define MMEA3_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0xe
+#define MMEA3_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0xf
+#define MMEA3_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x11
+#define MMEA3_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x12
+#define MMEA3_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x14
+#define MMEA3_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                             0x00000003L
+#define MMEA3_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                             0x00000004L
+#define MMEA3_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                             0x00000018L
+#define MMEA3_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                             0x00000020L
+#define MMEA3_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x000000C0L
+#define MMEA3_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000100L
+#define MMEA3_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000600L
+#define MMEA3_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000800L
+#define MMEA3_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x00003000L
+#define MMEA3_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00004000L
+#define MMEA3_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00018000L
+#define MMEA3_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00020000L
+#define MMEA3_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x000C0000L
+#define MMEA3_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00100000L
+//MMEA3_DSM_CNTL2B
+#define MMEA3_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x0
+#define MMEA3_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY__SHIFT                                                0x2
+#define MMEA3_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x3
+#define MMEA3_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY__SHIFT                                                0x5
+#define MMEA3_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x6
+#define MMEA3_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY__SHIFT                                                0x8
+#define MMEA3_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x9
+#define MMEA3_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY__SHIFT                                                0xb
+#define MMEA3_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000003L
+#define MMEA3_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000004L
+#define MMEA3_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000018L
+#define MMEA3_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000020L
+#define MMEA3_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT_MASK                                                  0x000000C0L
+#define MMEA3_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000100L
+#define MMEA3_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000600L
+#define MMEA3_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000800L
+//MMEA3_CGTT_CLK_CTRL
+#define MMEA3_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define MMEA3_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define MMEA3_CGTT_CLK_CTRL__SPARE0__SHIFT                                                                    0xc
+#define MMEA3_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE__SHIFT                                                 0x14
+#define MMEA3_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ__SHIFT                                                  0x15
+#define MMEA3_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN__SHIFT                                                0x16
+#define MMEA3_CGTT_CLK_CTRL__SPARE1__SHIFT                                                                    0x17
+#define MMEA3_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                               0x1b
+#define MMEA3_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE__SHIFT                                                       0x1c
+#define MMEA3_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ__SHIFT                                                        0x1d
+#define MMEA3_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                      0x1e
+#define MMEA3_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                    0x1f
+#define MMEA3_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define MMEA3_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define MMEA3_CGTT_CLK_CTRL__SPARE0_MASK                                                                      0x000FF000L
+#define MMEA3_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE_MASK                                                   0x00100000L
+#define MMEA3_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ_MASK                                                    0x00200000L
+#define MMEA3_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN_MASK                                                  0x00400000L
+#define MMEA3_CGTT_CLK_CTRL__SPARE1_MASK                                                                      0x07800000L
+#define MMEA3_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                                 0x08000000L
+#define MMEA3_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE_MASK                                                         0x10000000L
+#define MMEA3_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ_MASK                                                          0x20000000L
+#define MMEA3_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                        0x40000000L
+#define MMEA3_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                      0x80000000L
+//MMEA3_EDC_MODE
+#define MMEA3_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                  0x10
+#define MMEA3_EDC_MODE__GATE_FUE__SHIFT                                                                       0x11
+#define MMEA3_EDC_MODE__DED_MODE__SHIFT                                                                       0x14
+#define MMEA3_EDC_MODE__PROP_FED__SHIFT                                                                       0x1d
+#define MMEA3_EDC_MODE__BYPASS__SHIFT                                                                         0x1f
+#define MMEA3_EDC_MODE__COUNT_FED_OUT_MASK                                                                    0x00010000L
+#define MMEA3_EDC_MODE__GATE_FUE_MASK                                                                         0x00020000L
+#define MMEA3_EDC_MODE__DED_MODE_MASK                                                                         0x00300000L
+#define MMEA3_EDC_MODE__PROP_FED_MASK                                                                         0x20000000L
+#define MMEA3_EDC_MODE__BYPASS_MASK                                                                           0x80000000L
+//MMEA3_ERR_STATUS
+#define MMEA3_ERR_STATUS__SDP_RDRSP_STATUS__SHIFT                                                             0x0
+#define MMEA3_ERR_STATUS__SDP_WRRSP_STATUS__SHIFT                                                             0x4
+#define MMEA3_ERR_STATUS__SDP_RDRSP_DATASTATUS__SHIFT                                                         0x8
+#define MMEA3_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR__SHIFT                                                   0xa
+#define MMEA3_ERR_STATUS__CLEAR_ERROR_STATUS__SHIFT                                                           0xb
+#define MMEA3_ERR_STATUS__BUSY_ON_ERROR__SHIFT                                                                0xc
+#define MMEA3_ERR_STATUS__FUE_FLAG__SHIFT                                                                     0xd
+#define MMEA3_ERR_STATUS__IGNORE_RDRSP_FED__SHIFT                                                             0xe
+#define MMEA3_ERR_STATUS__INTERRUPT_ON_FATAL__SHIFT                                                           0xf
+#define MMEA3_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL__SHIFT                                                   0x10
+#define MMEA3_ERR_STATUS__LEVEL_INTERRUPT__SHIFT                                                              0x11
+#define MMEA3_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR__SHIFT                                                     0x12
+#define MMEA3_ERR_STATUS__FUE_FLAG_CLIENT__SHIFT                                                              0x13
+#define MMEA3_ERR_STATUS__SDP_RDRSP_STATUS_MASK                                                               0x0000000FL
+#define MMEA3_ERR_STATUS__SDP_WRRSP_STATUS_MASK                                                               0x000000F0L
+#define MMEA3_ERR_STATUS__SDP_RDRSP_DATASTATUS_MASK                                                           0x00000300L
+#define MMEA3_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR_MASK                                                     0x00000400L
+#define MMEA3_ERR_STATUS__CLEAR_ERROR_STATUS_MASK                                                             0x00000800L
+#define MMEA3_ERR_STATUS__BUSY_ON_ERROR_MASK                                                                  0x00001000L
+#define MMEA3_ERR_STATUS__FUE_FLAG_MASK                                                                       0x00002000L
+#define MMEA3_ERR_STATUS__IGNORE_RDRSP_FED_MASK                                                               0x00004000L
+#define MMEA3_ERR_STATUS__INTERRUPT_ON_FATAL_MASK                                                             0x00008000L
+#define MMEA3_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL_MASK                                                     0x00010000L
+#define MMEA3_ERR_STATUS__LEVEL_INTERRUPT_MASK                                                                0x00020000L
+#define MMEA3_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR_MASK                                                       0x00040000L
+#define MMEA3_ERR_STATUS__FUE_FLAG_CLIENT_MASK                                                                0x00080000L
+//MMEA3_MISC2
+#define MMEA3_MISC2__CSGROUP_SWAP_IN_DRAM_ARB__SHIFT                                                          0x0
+#define MMEA3_MISC2__CSGROUP_SWAP_IN_GMI_ARB__SHIFT                                                           0x1
+#define MMEA3_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM__SHIFT                                                       0x2
+#define MMEA3_MISC2__CSGRP_BURST_LIMIT_DATA_GMI__SHIFT                                                        0x7
+#define MMEA3_MISC2__IO_RDWR_PRIORITY_ENABLE__SHIFT                                                           0xc
+#define MMEA3_MISC2__RRET_SWAP_MODE__SHIFT                                                                    0xd
+#define MMEA3_MISC2__BLOCK_REQUESTS__SHIFT                                                                    0xe
+#define MMEA3_MISC2__REQUESTS_BLOCKED__SHIFT                                                                  0xf
+#define MMEA3_MISC2__DRAM_RD_THROTTLE__SHIFT                                                                  0x10
+#define MMEA3_MISC2__DRAM_WR_THROTTLE__SHIFT                                                                  0x11
+#define MMEA3_MISC2__GMI_RD_THROTTLE__SHIFT                                                                   0x12
+#define MMEA3_MISC2__GMI_WR_THROTTLE__SHIFT                                                                   0x13
+#define MMEA3_MISC2__CSGROUP_SWAP_IN_DRAM_ARB_MASK                                                            0x00000001L
+#define MMEA3_MISC2__CSGROUP_SWAP_IN_GMI_ARB_MASK                                                             0x00000002L
+#define MMEA3_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM_MASK                                                         0x0000007CL
+#define MMEA3_MISC2__CSGRP_BURST_LIMIT_DATA_GMI_MASK                                                          0x00000F80L
+#define MMEA3_MISC2__IO_RDWR_PRIORITY_ENABLE_MASK                                                             0x00001000L
+#define MMEA3_MISC2__RRET_SWAP_MODE_MASK                                                                      0x00002000L
+#define MMEA3_MISC2__BLOCK_REQUESTS_MASK                                                                      0x00004000L
+#define MMEA3_MISC2__REQUESTS_BLOCKED_MASK                                                                    0x00008000L
+#define MMEA3_MISC2__DRAM_RD_THROTTLE_MASK                                                                    0x00010000L
+#define MMEA3_MISC2__DRAM_WR_THROTTLE_MASK                                                                    0x00020000L
+#define MMEA3_MISC2__GMI_RD_THROTTLE_MASK                                                                     0x00040000L
+#define MMEA3_MISC2__GMI_WR_THROTTLE_MASK                                                                     0x00080000L
+//MMEA3_MISC_AON
+#define MMEA3_MISC_AON__LINKMGR_PARTACK_HYSTERESIS__SHIFT                                                     0x0
+#define MMEA3_MISC_AON__LINKMGR_PARTACK_DEASSERT_MODE__SHIFT                                                  0x2
+#define MMEA3_MISC_AON__LINKMGR_PARTACK_HYSTERESIS_MASK                                                       0x00000003L
+#define MMEA3_MISC_AON__LINKMGR_PARTACK_DEASSERT_MODE_MASK                                                    0x00000004L
+
+
+// addressBlock: aid_mmhub_ea_mmeadec4
+//MMEA4_DRAM_RD_CLI2GRP_MAP0
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                         0x0
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                         0x2
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                         0x4
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                         0x6
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                         0x8
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                         0xa
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                         0xc
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                         0xe
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                         0x10
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                         0x12
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                        0x14
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                        0x16
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                        0x18
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                        0x1a
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                        0x1c
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                        0x1e
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                           0x00000003L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                           0x0000000CL
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                           0x00000030L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                           0x000000C0L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                           0x00000300L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                           0x00000C00L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                           0x00003000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                           0x0000C000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                           0x00030000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                           0x000C0000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                          0x00300000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                          0x00C00000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                          0x03000000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                          0x0C000000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                          0x30000000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                          0xC0000000L
+//MMEA4_DRAM_RD_CLI2GRP_MAP1
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                        0x0
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                        0x2
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                        0x4
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                        0x6
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                        0x8
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                        0xa
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                        0xc
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                        0xe
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                        0x10
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                        0x12
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                        0x14
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                        0x16
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                        0x18
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                        0x1a
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                        0x1c
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                        0x1e
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                          0x00000003L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                          0x0000000CL
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                          0x00000030L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                          0x000000C0L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                          0x00000300L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                          0x00000C00L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                          0x00003000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                          0x0000C000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                          0x00030000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                          0x000C0000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                          0x00300000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                          0x00C00000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                          0x03000000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                          0x0C000000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                          0x30000000L
+#define MMEA4_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                          0xC0000000L
+//MMEA4_DRAM_WR_CLI2GRP_MAP0
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                         0x0
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                         0x2
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                         0x4
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                         0x6
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                         0x8
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                         0xa
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                         0xc
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                         0xe
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                         0x10
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                         0x12
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                        0x14
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                        0x16
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                        0x18
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                        0x1a
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                        0x1c
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                        0x1e
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                           0x00000003L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                           0x0000000CL
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                           0x00000030L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                           0x000000C0L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                           0x00000300L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                           0x00000C00L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                           0x00003000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                           0x0000C000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                           0x00030000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                           0x000C0000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                          0x00300000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                          0x00C00000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                          0x03000000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                          0x0C000000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                          0x30000000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                          0xC0000000L
+//MMEA4_DRAM_WR_CLI2GRP_MAP1
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                        0x0
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                        0x2
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                        0x4
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                        0x6
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                        0x8
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                        0xa
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                        0xc
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                        0xe
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                        0x10
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                        0x12
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                        0x14
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                        0x16
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                        0x18
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                        0x1a
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                        0x1c
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                        0x1e
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                          0x00000003L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                          0x0000000CL
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                          0x00000030L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                          0x000000C0L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                          0x00000300L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                          0x00000C00L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                          0x00003000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                          0x0000C000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                          0x00030000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                          0x000C0000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                          0x00300000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                          0x00C00000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                          0x03000000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                          0x0C000000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                          0x30000000L
+#define MMEA4_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                          0xC0000000L
+//MMEA4_DRAM_RD_GRP2VC_MAP
+#define MMEA4_DRAM_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                            0x0
+#define MMEA4_DRAM_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                            0x3
+#define MMEA4_DRAM_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                            0x6
+#define MMEA4_DRAM_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                            0x9
+#define MMEA4_DRAM_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                              0x00000007L
+#define MMEA4_DRAM_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                              0x00000038L
+#define MMEA4_DRAM_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                              0x000001C0L
+#define MMEA4_DRAM_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                              0x00000E00L
+//MMEA4_DRAM_WR_GRP2VC_MAP
+#define MMEA4_DRAM_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                            0x0
+#define MMEA4_DRAM_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                            0x3
+#define MMEA4_DRAM_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                            0x6
+#define MMEA4_DRAM_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                            0x9
+#define MMEA4_DRAM_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                              0x00000007L
+#define MMEA4_DRAM_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                              0x00000038L
+#define MMEA4_DRAM_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                              0x000001C0L
+#define MMEA4_DRAM_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                              0x00000E00L
+//MMEA4_DRAM_RD_LAZY
+#define MMEA4_DRAM_RD_LAZY__GROUP0_DELAY__SHIFT                                                               0x0
+#define MMEA4_DRAM_RD_LAZY__GROUP1_DELAY__SHIFT                                                               0x3
+#define MMEA4_DRAM_RD_LAZY__GROUP2_DELAY__SHIFT                                                               0x6
+#define MMEA4_DRAM_RD_LAZY__GROUP3_DELAY__SHIFT                                                               0x9
+#define MMEA4_DRAM_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                           0xc
+#define MMEA4_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                          0x14
+#define MMEA4_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                          0x1b
+#define MMEA4_DRAM_RD_LAZY__GROUP0_DELAY_MASK                                                                 0x00000007L
+#define MMEA4_DRAM_RD_LAZY__GROUP1_DELAY_MASK                                                                 0x00000038L
+#define MMEA4_DRAM_RD_LAZY__GROUP2_DELAY_MASK                                                                 0x000001C0L
+#define MMEA4_DRAM_RD_LAZY__GROUP3_DELAY_MASK                                                                 0x00000E00L
+#define MMEA4_DRAM_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                             0x0003F000L
+#define MMEA4_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                            0x07F00000L
+#define MMEA4_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                            0x78000000L
+//MMEA4_DRAM_WR_LAZY
+#define MMEA4_DRAM_WR_LAZY__GROUP0_DELAY__SHIFT                                                               0x0
+#define MMEA4_DRAM_WR_LAZY__GROUP1_DELAY__SHIFT                                                               0x3
+#define MMEA4_DRAM_WR_LAZY__GROUP2_DELAY__SHIFT                                                               0x6
+#define MMEA4_DRAM_WR_LAZY__GROUP3_DELAY__SHIFT                                                               0x9
+#define MMEA4_DRAM_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                           0xc
+#define MMEA4_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                          0x14
+#define MMEA4_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                          0x1b
+#define MMEA4_DRAM_WR_LAZY__GROUP0_DELAY_MASK                                                                 0x00000007L
+#define MMEA4_DRAM_WR_LAZY__GROUP1_DELAY_MASK                                                                 0x00000038L
+#define MMEA4_DRAM_WR_LAZY__GROUP2_DELAY_MASK                                                                 0x000001C0L
+#define MMEA4_DRAM_WR_LAZY__GROUP3_DELAY_MASK                                                                 0x00000E00L
+#define MMEA4_DRAM_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                             0x0003F000L
+#define MMEA4_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                            0x07F00000L
+#define MMEA4_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                            0x78000000L
+//MMEA4_DRAM_RD_CAM_CNTL
+#define MMEA4_DRAM_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                           0x0
+#define MMEA4_DRAM_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                           0x4
+#define MMEA4_DRAM_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                           0x8
+#define MMEA4_DRAM_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                           0xc
+#define MMEA4_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                   0x10
+#define MMEA4_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                   0x13
+#define MMEA4_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                   0x16
+#define MMEA4_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                   0x19
+#define MMEA4_DRAM_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                           0x1c
+#define MMEA4_DRAM_RD_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                     0x1d
+#define MMEA4_DRAM_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                             0x0000000FL
+#define MMEA4_DRAM_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                             0x000000F0L
+#define MMEA4_DRAM_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                             0x00000F00L
+#define MMEA4_DRAM_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                             0x0000F000L
+#define MMEA4_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                     0x00070000L
+#define MMEA4_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                     0x00380000L
+#define MMEA4_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                     0x01C00000L
+#define MMEA4_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                     0x0E000000L
+#define MMEA4_DRAM_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                             0x10000000L
+#define MMEA4_DRAM_RD_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                       0x20000000L
+//MMEA4_DRAM_WR_CAM_CNTL
+#define MMEA4_DRAM_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                           0x0
+#define MMEA4_DRAM_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                           0x4
+#define MMEA4_DRAM_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                           0x8
+#define MMEA4_DRAM_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                           0xc
+#define MMEA4_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                   0x10
+#define MMEA4_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                   0x13
+#define MMEA4_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                   0x16
+#define MMEA4_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                   0x19
+#define MMEA4_DRAM_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                           0x1c
+#define MMEA4_DRAM_WR_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                     0x1d
+#define MMEA4_DRAM_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                             0x0000000FL
+#define MMEA4_DRAM_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                             0x000000F0L
+#define MMEA4_DRAM_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                             0x00000F00L
+#define MMEA4_DRAM_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                             0x0000F000L
+#define MMEA4_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                     0x00070000L
+#define MMEA4_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                     0x00380000L
+#define MMEA4_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                     0x01C00000L
+#define MMEA4_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                     0x0E000000L
+#define MMEA4_DRAM_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                             0x10000000L
+#define MMEA4_DRAM_WR_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                       0x20000000L
+//MMEA4_DRAM_PAGE_BURST
+#define MMEA4_DRAM_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                             0x0
+#define MMEA4_DRAM_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                             0x8
+#define MMEA4_DRAM_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                             0x10
+#define MMEA4_DRAM_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                             0x18
+#define MMEA4_DRAM_PAGE_BURST__RD_LIMIT_LO_MASK                                                               0x000000FFL
+#define MMEA4_DRAM_PAGE_BURST__RD_LIMIT_HI_MASK                                                               0x0000FF00L
+#define MMEA4_DRAM_PAGE_BURST__WR_LIMIT_LO_MASK                                                               0x00FF0000L
+#define MMEA4_DRAM_PAGE_BURST__WR_LIMIT_HI_MASK                                                               0xFF000000L
+//MMEA4_DRAM_RD_PRI_AGE
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                  0xc
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                  0xf
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                  0x12
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                  0x15
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                    0x00007000L
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                    0x00038000L
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                    0x001C0000L
+#define MMEA4_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                    0x00E00000L
+//MMEA4_DRAM_WR_PRI_AGE
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                  0xc
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                  0xf
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                  0x12
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                  0x15
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                    0x00007000L
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                    0x00038000L
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                    0x001C0000L
+#define MMEA4_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                    0x00E00000L
+//MMEA4_DRAM_RD_PRI_QUEUING
+#define MMEA4_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                          0x0
+#define MMEA4_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                          0x3
+#define MMEA4_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                          0x6
+#define MMEA4_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                          0x9
+#define MMEA4_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA4_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA4_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA4_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                            0x00000E00L
+//MMEA4_DRAM_WR_PRI_QUEUING
+#define MMEA4_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                          0x0
+#define MMEA4_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                          0x3
+#define MMEA4_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                          0x6
+#define MMEA4_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                          0x9
+#define MMEA4_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA4_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA4_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA4_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                            0x00000E00L
+//MMEA4_DRAM_RD_PRI_FIXED
+#define MMEA4_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                              0x0
+#define MMEA4_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                              0x3
+#define MMEA4_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                              0x6
+#define MMEA4_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                              0x9
+#define MMEA4_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                0x00000007L
+#define MMEA4_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                0x00000038L
+#define MMEA4_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                0x000001C0L
+#define MMEA4_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                0x00000E00L
+//MMEA4_DRAM_WR_PRI_FIXED
+#define MMEA4_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                              0x0
+#define MMEA4_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                              0x3
+#define MMEA4_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                              0x6
+#define MMEA4_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                              0x9
+#define MMEA4_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                0x00000007L
+#define MMEA4_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                0x00000038L
+#define MMEA4_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                0x000001C0L
+#define MMEA4_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                0x00000E00L
+//MMEA4_DRAM_RD_PRI_URGENCY
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                          0x0
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                          0x3
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                          0x6
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                          0x9
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                 0xc
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                 0xd
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                 0xe
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                 0xf
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                            0x00000E00L
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                   0x00001000L
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                   0x00002000L
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                   0x00004000L
+#define MMEA4_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                   0x00008000L
+//MMEA4_DRAM_WR_PRI_URGENCY
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                          0x0
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                          0x3
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                          0x6
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                          0x9
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                 0xc
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                 0xd
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                 0xe
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                 0xf
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                            0x00000007L
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                            0x00000038L
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                            0x000001C0L
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                            0x00000E00L
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                   0x00001000L
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                   0x00002000L
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                   0x00004000L
+#define MMEA4_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                   0x00008000L
+//MMEA4_DRAM_RD_PRI_QUANT_PRI1
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA4_DRAM_RD_PRI_QUANT_PRI2
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA4_DRAM_RD_PRI_QUANT_PRI3
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA4_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA4_DRAM_WR_PRI_QUANT_PRI1
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA4_DRAM_WR_PRI_QUANT_PRI2
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA4_DRAM_WR_PRI_QUANT_PRI3
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                 0x0
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                 0x8
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                 0x10
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                 0x18
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                   0x000000FFL
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                   0x0000FF00L
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                   0x00FF0000L
+#define MMEA4_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                   0xFF000000L
+//MMEA4_GMI_RD_CLI2GRP_MAP0
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//MMEA4_GMI_RD_CLI2GRP_MAP1
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define MMEA4_GMI_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//MMEA4_GMI_WR_CLI2GRP_MAP0
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//MMEA4_GMI_WR_CLI2GRP_MAP1
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define MMEA4_GMI_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//MMEA4_GMI_RD_GRP2VC_MAP
+#define MMEA4_GMI_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define MMEA4_GMI_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define MMEA4_GMI_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define MMEA4_GMI_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define MMEA4_GMI_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define MMEA4_GMI_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define MMEA4_GMI_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define MMEA4_GMI_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//MMEA4_GMI_WR_GRP2VC_MAP
+#define MMEA4_GMI_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define MMEA4_GMI_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define MMEA4_GMI_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define MMEA4_GMI_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define MMEA4_GMI_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define MMEA4_GMI_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define MMEA4_GMI_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define MMEA4_GMI_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//MMEA4_GMI_RD_LAZY
+#define MMEA4_GMI_RD_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define MMEA4_GMI_RD_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define MMEA4_GMI_RD_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define MMEA4_GMI_RD_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define MMEA4_GMI_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define MMEA4_GMI_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define MMEA4_GMI_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define MMEA4_GMI_RD_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define MMEA4_GMI_RD_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define MMEA4_GMI_RD_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define MMEA4_GMI_RD_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define MMEA4_GMI_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define MMEA4_GMI_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define MMEA4_GMI_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//MMEA4_GMI_WR_LAZY
+#define MMEA4_GMI_WR_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define MMEA4_GMI_WR_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define MMEA4_GMI_WR_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define MMEA4_GMI_WR_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define MMEA4_GMI_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define MMEA4_GMI_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define MMEA4_GMI_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define MMEA4_GMI_WR_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define MMEA4_GMI_WR_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define MMEA4_GMI_WR_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define MMEA4_GMI_WR_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define MMEA4_GMI_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define MMEA4_GMI_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define MMEA4_GMI_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//MMEA4_GMI_RD_CAM_CNTL
+#define MMEA4_GMI_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define MMEA4_GMI_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define MMEA4_GMI_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define MMEA4_GMI_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define MMEA4_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define MMEA4_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define MMEA4_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define MMEA4_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define MMEA4_GMI_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define MMEA4_GMI_RD_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                      0x1d
+#define MMEA4_GMI_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define MMEA4_GMI_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define MMEA4_GMI_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define MMEA4_GMI_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define MMEA4_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define MMEA4_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define MMEA4_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define MMEA4_GMI_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define MMEA4_GMI_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+#define MMEA4_GMI_RD_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                        0x20000000L
+//MMEA4_GMI_WR_CAM_CNTL
+#define MMEA4_GMI_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define MMEA4_GMI_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define MMEA4_GMI_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define MMEA4_GMI_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define MMEA4_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define MMEA4_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define MMEA4_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define MMEA4_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define MMEA4_GMI_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define MMEA4_GMI_WR_CAM_CNTL__PAGEBASED_CHAINING__SHIFT                                                      0x1d
+#define MMEA4_GMI_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define MMEA4_GMI_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define MMEA4_GMI_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define MMEA4_GMI_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define MMEA4_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define MMEA4_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define MMEA4_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define MMEA4_GMI_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define MMEA4_GMI_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+#define MMEA4_GMI_WR_CAM_CNTL__PAGEBASED_CHAINING_MASK                                                        0x20000000L
+//MMEA4_GMI_PAGE_BURST
+#define MMEA4_GMI_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define MMEA4_GMI_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define MMEA4_GMI_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define MMEA4_GMI_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define MMEA4_GMI_PAGE_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define MMEA4_GMI_PAGE_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define MMEA4_GMI_PAGE_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define MMEA4_GMI_PAGE_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//MMEA4_GMI_RD_PRI_AGE
+#define MMEA4_GMI_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define MMEA4_GMI_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define MMEA4_GMI_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define MMEA4_GMI_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define MMEA4_GMI_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define MMEA4_GMI_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define MMEA4_GMI_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define MMEA4_GMI_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define MMEA4_GMI_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define MMEA4_GMI_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define MMEA4_GMI_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define MMEA4_GMI_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define MMEA4_GMI_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define MMEA4_GMI_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define MMEA4_GMI_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define MMEA4_GMI_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//MMEA4_GMI_WR_PRI_AGE
+#define MMEA4_GMI_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define MMEA4_GMI_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define MMEA4_GMI_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define MMEA4_GMI_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define MMEA4_GMI_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define MMEA4_GMI_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define MMEA4_GMI_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define MMEA4_GMI_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define MMEA4_GMI_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define MMEA4_GMI_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define MMEA4_GMI_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define MMEA4_GMI_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define MMEA4_GMI_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define MMEA4_GMI_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define MMEA4_GMI_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define MMEA4_GMI_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//MMEA4_GMI_RD_PRI_QUEUING
+#define MMEA4_GMI_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define MMEA4_GMI_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define MMEA4_GMI_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define MMEA4_GMI_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define MMEA4_GMI_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA4_GMI_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA4_GMI_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA4_GMI_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//MMEA4_GMI_WR_PRI_QUEUING
+#define MMEA4_GMI_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define MMEA4_GMI_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define MMEA4_GMI_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define MMEA4_GMI_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define MMEA4_GMI_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA4_GMI_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA4_GMI_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA4_GMI_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//MMEA4_GMI_RD_PRI_FIXED
+#define MMEA4_GMI_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define MMEA4_GMI_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define MMEA4_GMI_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define MMEA4_GMI_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define MMEA4_GMI_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define MMEA4_GMI_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define MMEA4_GMI_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define MMEA4_GMI_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//MMEA4_GMI_WR_PRI_FIXED
+#define MMEA4_GMI_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define MMEA4_GMI_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define MMEA4_GMI_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define MMEA4_GMI_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define MMEA4_GMI_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define MMEA4_GMI_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define MMEA4_GMI_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define MMEA4_GMI_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//MMEA4_GMI_RD_PRI_URGENCY
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define MMEA4_GMI_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//MMEA4_GMI_WR_PRI_URGENCY
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define MMEA4_GMI_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//MMEA4_GMI_RD_PRI_URGENCY_MASKING
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                    0x0
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                    0x1
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                    0x2
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                    0x3
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                    0x4
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                    0x5
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                    0x6
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                    0x7
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                    0x8
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                    0x9
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                   0xa
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                   0xb
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                   0xc
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                   0xd
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                   0xe
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                   0xf
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                   0x10
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                   0x11
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                   0x12
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                   0x13
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                   0x14
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                   0x15
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                   0x16
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                   0x17
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                   0x18
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                   0x19
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                   0x1a
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                   0x1b
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                   0x1c
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                   0x1d
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                   0x1e
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                   0x1f
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                      0x00000001L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                      0x00000002L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                      0x00000004L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                      0x00000008L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                      0x00000010L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                      0x00000020L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                      0x00000040L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                      0x00000080L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                      0x00000100L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                      0x00000200L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                     0x00000400L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                     0x00000800L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                     0x00001000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                     0x00002000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                     0x00004000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                     0x00008000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                     0x00010000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                     0x00020000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                     0x00040000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                     0x00080000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                     0x00100000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                     0x00200000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                     0x00400000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                     0x00800000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                     0x01000000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                     0x02000000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                     0x04000000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                     0x08000000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                     0x10000000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                     0x20000000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                     0x40000000L
+#define MMEA4_GMI_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                     0x80000000L
+//MMEA4_GMI_WR_PRI_URGENCY_MASKING
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                    0x0
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                    0x1
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                    0x2
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                    0x3
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                    0x4
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                    0x5
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                    0x6
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                    0x7
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                    0x8
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                    0x9
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                   0xa
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                   0xb
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                   0xc
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                   0xd
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                   0xe
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                   0xf
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                   0x10
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                   0x11
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                   0x12
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                   0x13
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                   0x14
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                   0x15
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                   0x16
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                   0x17
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                   0x18
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                   0x19
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                   0x1a
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                   0x1b
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                   0x1c
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                   0x1d
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                   0x1e
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                   0x1f
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                      0x00000001L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                      0x00000002L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                      0x00000004L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                      0x00000008L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                      0x00000010L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                      0x00000020L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                      0x00000040L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                      0x00000080L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                      0x00000100L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                      0x00000200L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                     0x00000400L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                     0x00000800L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                     0x00001000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                     0x00002000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                     0x00004000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                     0x00008000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                     0x00010000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                     0x00020000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                     0x00040000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                     0x00080000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                     0x00100000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                     0x00200000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                     0x00400000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                     0x00800000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                     0x01000000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                     0x02000000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                     0x04000000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                     0x08000000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                     0x10000000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                     0x20000000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                     0x40000000L
+#define MMEA4_GMI_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                     0x80000000L
+//MMEA4_GMI_RD_PRI_QUANT_PRI1
+#define MMEA4_GMI_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA4_GMI_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA4_GMI_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA4_GMI_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA4_GMI_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA4_GMI_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA4_GMI_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA4_GMI_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA4_GMI_RD_PRI_QUANT_PRI2
+#define MMEA4_GMI_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA4_GMI_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA4_GMI_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA4_GMI_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA4_GMI_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA4_GMI_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA4_GMI_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA4_GMI_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA4_GMI_RD_PRI_QUANT_PRI3
+#define MMEA4_GMI_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA4_GMI_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA4_GMI_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA4_GMI_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA4_GMI_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA4_GMI_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA4_GMI_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA4_GMI_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA4_GMI_WR_PRI_QUANT_PRI1
+#define MMEA4_GMI_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA4_GMI_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA4_GMI_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA4_GMI_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA4_GMI_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA4_GMI_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA4_GMI_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA4_GMI_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA4_GMI_WR_PRI_QUANT_PRI2
+#define MMEA4_GMI_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA4_GMI_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA4_GMI_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA4_GMI_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA4_GMI_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA4_GMI_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA4_GMI_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA4_GMI_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA4_GMI_WR_PRI_QUANT_PRI3
+#define MMEA4_GMI_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define MMEA4_GMI_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define MMEA4_GMI_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define MMEA4_GMI_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define MMEA4_GMI_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define MMEA4_GMI_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define MMEA4_GMI_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define MMEA4_GMI_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//MMEA4_IO_RD_CLI2GRP_MAP0
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                           0x0
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                           0x2
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                           0x4
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                           0x6
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                           0x8
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                           0xa
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                           0xc
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                           0xe
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                           0x10
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                           0x12
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                          0x14
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                          0x16
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                          0x18
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                          0x1a
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                          0x1c
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                          0x1e
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                             0x00000003L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                             0x0000000CL
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                             0x00000030L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                             0x000000C0L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                             0x00000300L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                             0x00000C00L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                             0x00003000L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                             0x0000C000L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                             0x00030000L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                             0x000C0000L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                            0x00300000L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                            0x00C00000L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                            0x03000000L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                            0x0C000000L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                            0x30000000L
+#define MMEA4_IO_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                            0xC0000000L
+//MMEA4_IO_RD_CLI2GRP_MAP1
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                          0x0
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                          0x2
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                          0x4
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                          0x6
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                          0x8
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                          0xa
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                          0xc
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                          0xe
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                          0x10
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                          0x12
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                          0x14
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                          0x16
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                          0x18
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                          0x1a
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                          0x1c
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                          0x1e
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                            0x00000003L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                            0x0000000CL
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                            0x00000030L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                            0x000000C0L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                            0x00000300L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                            0x00000C00L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                            0x00003000L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                            0x0000C000L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                            0x00030000L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                            0x000C0000L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                            0x00300000L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                            0x00C00000L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                            0x03000000L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                            0x0C000000L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                            0x30000000L
+#define MMEA4_IO_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                            0xC0000000L
+//MMEA4_IO_WR_CLI2GRP_MAP0
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                           0x0
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                           0x2
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                           0x4
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                           0x6
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                           0x8
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                           0xa
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                           0xc
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                           0xe
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                           0x10
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                           0x12
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                          0x14
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                          0x16
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                          0x18
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                          0x1a
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                          0x1c
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                          0x1e
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                             0x00000003L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                             0x0000000CL
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                             0x00000030L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                             0x000000C0L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                             0x00000300L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                             0x00000C00L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                             0x00003000L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                             0x0000C000L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                             0x00030000L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                             0x000C0000L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                            0x00300000L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                            0x00C00000L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                            0x03000000L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                            0x0C000000L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                            0x30000000L
+#define MMEA4_IO_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                            0xC0000000L
+//MMEA4_IO_WR_CLI2GRP_MAP1
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                          0x0
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                          0x2
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                          0x4
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                          0x6
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                          0x8
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                          0xa
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                          0xc
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                          0xe
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                          0x10
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                          0x12
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                          0x14
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                          0x16
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                          0x18
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                          0x1a
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                          0x1c
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                          0x1e
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                            0x00000003L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                            0x0000000CL
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                            0x00000030L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                            0x000000C0L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                            0x00000300L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                            0x00000C00L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                            0x00003000L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                            0x0000C000L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                            0x00030000L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                            0x000C0000L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                            0x00300000L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                            0x00C00000L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                            0x03000000L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                            0x0C000000L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                            0x30000000L
+#define MMEA4_IO_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                            0xC0000000L
+//MMEA4_IO_RD_COMBINE_FLUSH
+#define MMEA4_IO_RD_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                        0x0
+#define MMEA4_IO_RD_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                        0x4
+#define MMEA4_IO_RD_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                        0x8
+#define MMEA4_IO_RD_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                        0xc
+#define MMEA4_IO_RD_COMBINE_FLUSH__COMB_MODE__SHIFT                                                           0x10
+#define MMEA4_IO_RD_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                          0x0000000FL
+#define MMEA4_IO_RD_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                          0x000000F0L
+#define MMEA4_IO_RD_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                          0x00000F00L
+#define MMEA4_IO_RD_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                          0x0000F000L
+#define MMEA4_IO_RD_COMBINE_FLUSH__COMB_MODE_MASK                                                             0x00030000L
+//MMEA4_IO_WR_COMBINE_FLUSH
+#define MMEA4_IO_WR_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                        0x0
+#define MMEA4_IO_WR_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                        0x4
+#define MMEA4_IO_WR_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                        0x8
+#define MMEA4_IO_WR_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                        0xc
+#define MMEA4_IO_WR_COMBINE_FLUSH__COMB_MODE__SHIFT                                                           0x10
+#define MMEA4_IO_WR_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                          0x0000000FL
+#define MMEA4_IO_WR_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                          0x000000F0L
+#define MMEA4_IO_WR_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                          0x00000F00L
+#define MMEA4_IO_WR_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                          0x0000F000L
+#define MMEA4_IO_WR_COMBINE_FLUSH__COMB_MODE_MASK                                                             0x00030000L
+//MMEA4_IO_GROUP_BURST
+#define MMEA4_IO_GROUP_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define MMEA4_IO_GROUP_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define MMEA4_IO_GROUP_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define MMEA4_IO_GROUP_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define MMEA4_IO_GROUP_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define MMEA4_IO_GROUP_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define MMEA4_IO_GROUP_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define MMEA4_IO_GROUP_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//MMEA4_IO_RD_PRI_AGE
+#define MMEA4_IO_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                         0x0
+#define MMEA4_IO_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                         0x3
+#define MMEA4_IO_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                         0x6
+#define MMEA4_IO_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                         0x9
+#define MMEA4_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                    0xc
+#define MMEA4_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                    0xf
+#define MMEA4_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                    0x12
+#define MMEA4_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                    0x15
+#define MMEA4_IO_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                           0x00000007L
+#define MMEA4_IO_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                           0x00000038L
+#define MMEA4_IO_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                           0x000001C0L
+#define MMEA4_IO_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                           0x00000E00L
+#define MMEA4_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                      0x00007000L
+#define MMEA4_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                      0x00038000L
+#define MMEA4_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                      0x001C0000L
+#define MMEA4_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                      0x00E00000L
+//MMEA4_IO_WR_PRI_AGE
+#define MMEA4_IO_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                         0x0
+#define MMEA4_IO_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                         0x3
+#define MMEA4_IO_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                         0x6
+#define MMEA4_IO_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                         0x9
+#define MMEA4_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                    0xc
+#define MMEA4_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                    0xf
+#define MMEA4_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                    0x12
+#define MMEA4_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                    0x15
+#define MMEA4_IO_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                           0x00000007L
+#define MMEA4_IO_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                           0x00000038L
+#define MMEA4_IO_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                           0x000001C0L
+#define MMEA4_IO_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                           0x00000E00L
+#define MMEA4_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                      0x00007000L
+#define MMEA4_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                      0x00038000L
+#define MMEA4_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                      0x001C0000L
+#define MMEA4_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                      0x00E00000L
+//MMEA4_IO_RD_PRI_QUEUING
+#define MMEA4_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                            0x0
+#define MMEA4_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                            0x3
+#define MMEA4_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                            0x6
+#define MMEA4_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                            0x9
+#define MMEA4_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA4_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA4_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA4_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                              0x00000E00L
+//MMEA4_IO_WR_PRI_QUEUING
+#define MMEA4_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                            0x0
+#define MMEA4_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                            0x3
+#define MMEA4_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                            0x6
+#define MMEA4_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                            0x9
+#define MMEA4_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA4_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA4_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA4_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                              0x00000E00L
+//MMEA4_IO_RD_PRI_FIXED
+#define MMEA4_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                0x0
+#define MMEA4_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                0x3
+#define MMEA4_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                0x6
+#define MMEA4_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                0x9
+#define MMEA4_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                  0x00000007L
+#define MMEA4_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                  0x00000038L
+#define MMEA4_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                  0x000001C0L
+#define MMEA4_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                  0x00000E00L
+//MMEA4_IO_WR_PRI_FIXED
+#define MMEA4_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                0x0
+#define MMEA4_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                0x3
+#define MMEA4_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                0x6
+#define MMEA4_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                0x9
+#define MMEA4_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                  0x00000007L
+#define MMEA4_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                  0x00000038L
+#define MMEA4_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                  0x000001C0L
+#define MMEA4_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                  0x00000E00L
+//MMEA4_IO_RD_PRI_URGENCY
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                            0x0
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                            0x3
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                            0x6
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                            0x9
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                   0xc
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                   0xd
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                   0xe
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                   0xf
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                              0x00000E00L
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                     0x00001000L
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                     0x00002000L
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                     0x00004000L
+#define MMEA4_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                     0x00008000L
+//MMEA4_IO_WR_PRI_URGENCY
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                            0x0
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                            0x3
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                            0x6
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                            0x9
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                   0xc
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                   0xd
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                   0xe
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                   0xf
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                              0x00000007L
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                              0x00000038L
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                              0x000001C0L
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                              0x00000E00L
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                     0x00001000L
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                     0x00002000L
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                     0x00004000L
+#define MMEA4_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                     0x00008000L
+//MMEA4_IO_RD_PRI_URGENCY_MASKING
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                     0x0
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                     0x1
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                     0x2
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                     0x3
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                     0x4
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                     0x5
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                     0x6
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                     0x7
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                     0x8
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                     0x9
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                    0xa
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                    0xb
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                    0xc
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                    0xd
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                    0xe
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                    0xf
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                    0x10
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                    0x11
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                    0x12
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                    0x13
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                    0x14
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                    0x15
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                    0x16
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                    0x17
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                    0x18
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                    0x19
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                    0x1a
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                    0x1b
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                    0x1c
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                    0x1d
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                    0x1e
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                    0x1f
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                       0x00000001L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                       0x00000002L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                       0x00000004L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                       0x00000008L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                       0x00000010L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                       0x00000020L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                       0x00000040L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                       0x00000080L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                       0x00000100L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                       0x00000200L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                      0x00000400L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                      0x00000800L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                      0x00001000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                      0x00002000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                      0x00004000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                      0x00008000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                      0x00010000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                      0x00020000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                      0x00040000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                      0x00080000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                      0x00100000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                      0x00200000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                      0x00400000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                      0x00800000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                      0x01000000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                      0x02000000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                      0x04000000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                      0x08000000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                      0x10000000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                      0x20000000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                      0x40000000L
+#define MMEA4_IO_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                      0x80000000L
+//MMEA4_IO_WR_PRI_URGENCY_MASKING
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                     0x0
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                     0x1
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                     0x2
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                     0x3
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                     0x4
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                     0x5
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                     0x6
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                     0x7
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                     0x8
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                     0x9
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                    0xa
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                    0xb
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                    0xc
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                    0xd
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                    0xe
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                    0xf
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                    0x10
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                    0x11
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                    0x12
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                    0x13
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                    0x14
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                    0x15
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                    0x16
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                    0x17
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                    0x18
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                    0x19
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                    0x1a
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                    0x1b
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                    0x1c
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                    0x1d
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                    0x1e
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                    0x1f
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                       0x00000001L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                       0x00000002L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                       0x00000004L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                       0x00000008L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                       0x00000010L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                       0x00000020L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                       0x00000040L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                       0x00000080L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                       0x00000100L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                       0x00000200L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                      0x00000400L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                      0x00000800L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                      0x00001000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                      0x00002000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                      0x00004000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                      0x00008000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                      0x00010000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                      0x00020000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                      0x00040000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                      0x00080000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                      0x00100000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                      0x00200000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                      0x00400000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                      0x00800000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                      0x01000000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                      0x02000000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                      0x04000000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                      0x08000000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                      0x10000000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                      0x20000000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                      0x40000000L
+#define MMEA4_IO_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                      0x80000000L
+//MMEA4_IO_RD_PRI_QUANT_PRI1
+#define MMEA4_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA4_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA4_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA4_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA4_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA4_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA4_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA4_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA4_IO_RD_PRI_QUANT_PRI2
+#define MMEA4_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA4_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA4_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA4_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA4_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA4_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA4_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA4_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA4_IO_RD_PRI_QUANT_PRI3
+#define MMEA4_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA4_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA4_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA4_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA4_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA4_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA4_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA4_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA4_IO_WR_PRI_QUANT_PRI1
+#define MMEA4_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA4_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA4_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA4_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA4_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA4_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA4_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA4_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA4_IO_WR_PRI_QUANT_PRI2
+#define MMEA4_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA4_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA4_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA4_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA4_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA4_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA4_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA4_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA4_IO_WR_PRI_QUANT_PRI3
+#define MMEA4_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                   0x0
+#define MMEA4_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                   0x8
+#define MMEA4_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                   0x10
+#define MMEA4_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                   0x18
+#define MMEA4_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                     0x000000FFL
+#define MMEA4_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                     0x0000FF00L
+#define MMEA4_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                     0x00FF0000L
+#define MMEA4_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                     0xFF000000L
+//MMEA4_SDP_ARB_DRAM
+#define MMEA4_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL__SHIFT                                                      0x0
+#define MMEA4_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA__SHIFT                                                      0x8
+#define MMEA4_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI__SHIFT                                                         0x10
+#define MMEA4_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI__SHIFT                                                         0x11
+#define MMEA4_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES__SHIFT                                                         0x12
+#define MMEA4_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES__SHIFT                                                         0x13
+#define MMEA4_SDP_ARB_DRAM__EOB_ON_EXPIRE__SHIFT                                                              0x14
+#define MMEA4_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                     0x15
+#define MMEA4_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING__SHIFT                                                       0x16
+#define MMEA4_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL_MASK                                                        0x0000007FL
+#define MMEA4_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA_MASK                                                        0x00007F00L
+#define MMEA4_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI_MASK                                                           0x00010000L
+#define MMEA4_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI_MASK                                                           0x00020000L
+#define MMEA4_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES_MASK                                                           0x00040000L
+#define MMEA4_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES_MASK                                                           0x00080000L
+#define MMEA4_SDP_ARB_DRAM__EOB_ON_EXPIRE_MASK                                                                0x00100000L
+#define MMEA4_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE_MASK                                                       0x00200000L
+#define MMEA4_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING_MASK                                                         0x00400000L
+//MMEA4_SDP_ARB_GMI
+#define MMEA4_SDP_ARB_GMI__RDWR_BURST_LIMIT_CYCL__SHIFT                                                       0x0
+#define MMEA4_SDP_ARB_GMI__RDWR_BURST_LIMIT_DATA__SHIFT                                                       0x8
+#define MMEA4_SDP_ARB_GMI__EARLY_SW2RD_ON_PRI__SHIFT                                                          0x10
+#define MMEA4_SDP_ARB_GMI__EARLY_SW2WR_ON_PRI__SHIFT                                                          0x11
+#define MMEA4_SDP_ARB_GMI__EARLY_SW2RD_ON_RES__SHIFT                                                          0x12
+#define MMEA4_SDP_ARB_GMI__EARLY_SW2WR_ON_RES__SHIFT                                                          0x13
+#define MMEA4_SDP_ARB_GMI__EOB_ON_EXPIRE__SHIFT                                                               0x14
+#define MMEA4_SDP_ARB_GMI__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                      0x15
+#define MMEA4_SDP_ARB_GMI__ALLOW_CHAIN_BREAKING__SHIFT                                                        0x16
+#define MMEA4_SDP_ARB_GMI__RDWR_BURST_LIMIT_CYCL_MASK                                                         0x0000007FL
+#define MMEA4_SDP_ARB_GMI__RDWR_BURST_LIMIT_DATA_MASK                                                         0x00007F00L
+#define MMEA4_SDP_ARB_GMI__EARLY_SW2RD_ON_PRI_MASK                                                            0x00010000L
+#define MMEA4_SDP_ARB_GMI__EARLY_SW2WR_ON_PRI_MASK                                                            0x00020000L
+#define MMEA4_SDP_ARB_GMI__EARLY_SW2RD_ON_RES_MASK                                                            0x00040000L
+#define MMEA4_SDP_ARB_GMI__EARLY_SW2WR_ON_RES_MASK                                                            0x00080000L
+#define MMEA4_SDP_ARB_GMI__EOB_ON_EXPIRE_MASK                                                                 0x00100000L
+#define MMEA4_SDP_ARB_GMI__DECOUPLE_RDWR_BNKSTATE_MASK                                                        0x00200000L
+#define MMEA4_SDP_ARB_GMI__ALLOW_CHAIN_BREAKING_MASK                                                          0x00400000L
+//MMEA4_SDP_ARB_FINAL
+#define MMEA4_SDP_ARB_FINAL__DRAM_BURST_LIMIT__SHIFT                                                          0x0
+#define MMEA4_SDP_ARB_FINAL__GMI_BURST_LIMIT__SHIFT                                                           0x5
+#define MMEA4_SDP_ARB_FINAL__IO_BURST_LIMIT__SHIFT                                                            0xa
+#define MMEA4_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER__SHIFT                                                    0xf
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC0__SHIFT                                                                0x11
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC1__SHIFT                                                                0x12
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC2__SHIFT                                                                0x13
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC3__SHIFT                                                                0x14
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC4__SHIFT                                                                0x15
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC5__SHIFT                                                                0x16
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC6__SHIFT                                                                0x17
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC7__SHIFT                                                                0x18
+#define MMEA4_SDP_ARB_FINAL__ERREVENT_ON_ERROR__SHIFT                                                         0x19
+#define MMEA4_SDP_ARB_FINAL__HALTREQ_ON_ERROR__SHIFT                                                          0x1a
+#define MMEA4_SDP_ARB_FINAL__DRAM_BURST_STRETCH__SHIFT                                                        0x1b
+#define MMEA4_SDP_ARB_FINAL__GMI_BURST_STRETCH__SHIFT                                                         0x1c
+#define MMEA4_SDP_ARB_FINAL__DRAM_BURST_LIMIT_MASK                                                            0x0000001FL
+#define MMEA4_SDP_ARB_FINAL__GMI_BURST_LIMIT_MASK                                                             0x000003E0L
+#define MMEA4_SDP_ARB_FINAL__IO_BURST_LIMIT_MASK                                                              0x00007C00L
+#define MMEA4_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER_MASK                                                      0x00018000L
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC0_MASK                                                                  0x00020000L
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC1_MASK                                                                  0x00040000L
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC2_MASK                                                                  0x00080000L
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC3_MASK                                                                  0x00100000L
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC4_MASK                                                                  0x00200000L
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC5_MASK                                                                  0x00400000L
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC6_MASK                                                                  0x00800000L
+#define MMEA4_SDP_ARB_FINAL__RDONLY_VC7_MASK                                                                  0x01000000L
+#define MMEA4_SDP_ARB_FINAL__ERREVENT_ON_ERROR_MASK                                                           0x02000000L
+#define MMEA4_SDP_ARB_FINAL__HALTREQ_ON_ERROR_MASK                                                            0x04000000L
+#define MMEA4_SDP_ARB_FINAL__DRAM_BURST_STRETCH_MASK                                                          0x08000000L
+#define MMEA4_SDP_ARB_FINAL__GMI_BURST_STRETCH_MASK                                                           0x10000000L
+//MMEA4_SDP_DRAM_PRIORITY
+#define MMEA4_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                    0x0
+#define MMEA4_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                    0x4
+#define MMEA4_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                    0x8
+#define MMEA4_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                    0xc
+#define MMEA4_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                    0x10
+#define MMEA4_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                    0x14
+#define MMEA4_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                    0x18
+#define MMEA4_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                    0x1c
+#define MMEA4_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                      0x0000000FL
+#define MMEA4_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                      0x000000F0L
+#define MMEA4_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                      0x00000F00L
+#define MMEA4_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                      0x0000F000L
+#define MMEA4_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                      0x000F0000L
+#define MMEA4_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                      0x00F00000L
+#define MMEA4_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                      0x0F000000L
+#define MMEA4_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                      0xF0000000L
+//MMEA4_SDP_GMI_PRIORITY
+#define MMEA4_SDP_GMI_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                     0x0
+#define MMEA4_SDP_GMI_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                     0x4
+#define MMEA4_SDP_GMI_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                     0x8
+#define MMEA4_SDP_GMI_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                     0xc
+#define MMEA4_SDP_GMI_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                     0x10
+#define MMEA4_SDP_GMI_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                     0x14
+#define MMEA4_SDP_GMI_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                     0x18
+#define MMEA4_SDP_GMI_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                     0x1c
+#define MMEA4_SDP_GMI_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                       0x0000000FL
+#define MMEA4_SDP_GMI_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                       0x000000F0L
+#define MMEA4_SDP_GMI_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                       0x00000F00L
+#define MMEA4_SDP_GMI_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                       0x0000F000L
+#define MMEA4_SDP_GMI_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                       0x000F0000L
+#define MMEA4_SDP_GMI_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                       0x00F00000L
+#define MMEA4_SDP_GMI_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                       0x0F000000L
+#define MMEA4_SDP_GMI_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                       0xF0000000L
+//MMEA4_SDP_IO_PRIORITY
+#define MMEA4_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                      0x0
+#define MMEA4_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                      0x4
+#define MMEA4_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                      0x8
+#define MMEA4_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                      0xc
+#define MMEA4_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                      0x10
+#define MMEA4_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                      0x14
+#define MMEA4_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                      0x18
+#define MMEA4_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                      0x1c
+#define MMEA4_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                        0x0000000FL
+#define MMEA4_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                        0x000000F0L
+#define MMEA4_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                        0x00000F00L
+#define MMEA4_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                        0x0000F000L
+#define MMEA4_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                        0x000F0000L
+#define MMEA4_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                        0x00F00000L
+#define MMEA4_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                        0x0F000000L
+#define MMEA4_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                        0xF0000000L
+//MMEA4_SDP_CREDITS
+#define MMEA4_SDP_CREDITS__TAG_LIMIT__SHIFT                                                                   0x0
+#define MMEA4_SDP_CREDITS__WR_RESP_CREDITS__SHIFT                                                             0x8
+#define MMEA4_SDP_CREDITS__RD_RESP_CREDITS__SHIFT                                                             0x10
+#define MMEA4_SDP_CREDITS__TAG_LIMIT_MASK                                                                     0x000000FFL
+#define MMEA4_SDP_CREDITS__WR_RESP_CREDITS_MASK                                                               0x00007F00L
+#define MMEA4_SDP_CREDITS__RD_RESP_CREDITS_MASK                                                               0x007F0000L
+//MMEA4_SDP_TAG_RESERVE0
+#define MMEA4_SDP_TAG_RESERVE0__VC0__SHIFT                                                                    0x0
+#define MMEA4_SDP_TAG_RESERVE0__VC1__SHIFT                                                                    0x8
+#define MMEA4_SDP_TAG_RESERVE0__VC2__SHIFT                                                                    0x10
+#define MMEA4_SDP_TAG_RESERVE0__VC3__SHIFT                                                                    0x18
+#define MMEA4_SDP_TAG_RESERVE0__VC0_MASK                                                                      0x000000FFL
+#define MMEA4_SDP_TAG_RESERVE0__VC1_MASK                                                                      0x0000FF00L
+#define MMEA4_SDP_TAG_RESERVE0__VC2_MASK                                                                      0x00FF0000L
+#define MMEA4_SDP_TAG_RESERVE0__VC3_MASK                                                                      0xFF000000L
+//MMEA4_SDP_TAG_RESERVE1
+#define MMEA4_SDP_TAG_RESERVE1__VC4__SHIFT                                                                    0x0
+#define MMEA4_SDP_TAG_RESERVE1__VC5__SHIFT                                                                    0x8
+#define MMEA4_SDP_TAG_RESERVE1__VC6__SHIFT                                                                    0x10
+#define MMEA4_SDP_TAG_RESERVE1__VC7__SHIFT                                                                    0x18
+#define MMEA4_SDP_TAG_RESERVE1__VC4_MASK                                                                      0x000000FFL
+#define MMEA4_SDP_TAG_RESERVE1__VC5_MASK                                                                      0x0000FF00L
+#define MMEA4_SDP_TAG_RESERVE1__VC6_MASK                                                                      0x00FF0000L
+#define MMEA4_SDP_TAG_RESERVE1__VC7_MASK                                                                      0xFF000000L
+//MMEA4_SDP_VCC_RESERVE0
+#define MMEA4_SDP_VCC_RESERVE0__VC0_CREDITS__SHIFT                                                            0x0
+#define MMEA4_SDP_VCC_RESERVE0__VC1_CREDITS__SHIFT                                                            0x6
+#define MMEA4_SDP_VCC_RESERVE0__VC2_CREDITS__SHIFT                                                            0xc
+#define MMEA4_SDP_VCC_RESERVE0__VC3_CREDITS__SHIFT                                                            0x12
+#define MMEA4_SDP_VCC_RESERVE0__VC4_CREDITS__SHIFT                                                            0x18
+#define MMEA4_SDP_VCC_RESERVE0__VC0_CREDITS_MASK                                                              0x0000003FL
+#define MMEA4_SDP_VCC_RESERVE0__VC1_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA4_SDP_VCC_RESERVE0__VC2_CREDITS_MASK                                                              0x0003F000L
+#define MMEA4_SDP_VCC_RESERVE0__VC3_CREDITS_MASK                                                              0x00FC0000L
+#define MMEA4_SDP_VCC_RESERVE0__VC4_CREDITS_MASK                                                              0x3F000000L
+//MMEA4_SDP_VCC_RESERVE1
+#define MMEA4_SDP_VCC_RESERVE1__VC5_CREDITS__SHIFT                                                            0x0
+#define MMEA4_SDP_VCC_RESERVE1__VC6_CREDITS__SHIFT                                                            0x6
+#define MMEA4_SDP_VCC_RESERVE1__VC7_CREDITS__SHIFT                                                            0xc
+#define MMEA4_SDP_VCC_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                        0x1f
+#define MMEA4_SDP_VCC_RESERVE1__VC5_CREDITS_MASK                                                              0x0000003FL
+#define MMEA4_SDP_VCC_RESERVE1__VC6_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA4_SDP_VCC_RESERVE1__VC7_CREDITS_MASK                                                              0x0003F000L
+#define MMEA4_SDP_VCC_RESERVE1__DISTRIBUTE_POOL_MASK                                                          0x80000000L
+//MMEA4_SDP_VCD_RESERVE0
+#define MMEA4_SDP_VCD_RESERVE0__VC0_CREDITS__SHIFT                                                            0x0
+#define MMEA4_SDP_VCD_RESERVE0__VC1_CREDITS__SHIFT                                                            0x6
+#define MMEA4_SDP_VCD_RESERVE0__VC2_CREDITS__SHIFT                                                            0xc
+#define MMEA4_SDP_VCD_RESERVE0__VC3_CREDITS__SHIFT                                                            0x12
+#define MMEA4_SDP_VCD_RESERVE0__VC4_CREDITS__SHIFT                                                            0x18
+#define MMEA4_SDP_VCD_RESERVE0__VC0_CREDITS_MASK                                                              0x0000003FL
+#define MMEA4_SDP_VCD_RESERVE0__VC1_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA4_SDP_VCD_RESERVE0__VC2_CREDITS_MASK                                                              0x0003F000L
+#define MMEA4_SDP_VCD_RESERVE0__VC3_CREDITS_MASK                                                              0x00FC0000L
+#define MMEA4_SDP_VCD_RESERVE0__VC4_CREDITS_MASK                                                              0x3F000000L
+//MMEA4_SDP_VCD_RESERVE1
+#define MMEA4_SDP_VCD_RESERVE1__VC5_CREDITS__SHIFT                                                            0x0
+#define MMEA4_SDP_VCD_RESERVE1__VC6_CREDITS__SHIFT                                                            0x6
+#define MMEA4_SDP_VCD_RESERVE1__VC7_CREDITS__SHIFT                                                            0xc
+#define MMEA4_SDP_VCD_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                        0x1f
+#define MMEA4_SDP_VCD_RESERVE1__VC5_CREDITS_MASK                                                              0x0000003FL
+#define MMEA4_SDP_VCD_RESERVE1__VC6_CREDITS_MASK                                                              0x00000FC0L
+#define MMEA4_SDP_VCD_RESERVE1__VC7_CREDITS_MASK                                                              0x0003F000L
+#define MMEA4_SDP_VCD_RESERVE1__DISTRIBUTE_POOL_MASK                                                          0x80000000L
+//MMEA4_SDP_REQ_CNTL
+#define MMEA4_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ__SHIFT                                                  0x0
+#define MMEA4_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE__SHIFT                                                 0x1
+#define MMEA4_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC__SHIFT                                                0x2
+#define MMEA4_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM__SHIFT                                                    0x3
+#define MMEA4_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI__SHIFT                                                     0x4
+#define MMEA4_SDP_REQ_CNTL__INNER_DOMAIN_MODE__SHIFT                                                          0x5
+#define MMEA4_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ__SHIFT                                                       0x6
+#define MMEA4_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE__SHIFT                                                      0x8
+#define MMEA4_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC__SHIFT                                                     0xa
+#define MMEA4_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ_MASK                                                    0x00000001L
+#define MMEA4_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE_MASK                                                   0x00000002L
+#define MMEA4_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC_MASK                                                  0x00000004L
+#define MMEA4_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM_MASK                                                      0x00000008L
+#define MMEA4_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI_MASK                                                       0x00000010L
+#define MMEA4_SDP_REQ_CNTL__INNER_DOMAIN_MODE_MASK                                                            0x00000020L
+#define MMEA4_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ_MASK                                                         0x000000C0L
+#define MMEA4_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE_MASK                                                        0x00000300L
+#define MMEA4_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC_MASK                                                       0x00000C00L
+//MMEA4_MISC
+#define MMEA4_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB__SHIFT                                                        0x0
+#define MMEA4_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB__SHIFT                                                        0x1
+#define MMEA4_MISC__RELATIVE_PRI_IN_GMI_RD_ARB__SHIFT                                                         0x2
+#define MMEA4_MISC__RELATIVE_PRI_IN_GMI_WR_ARB__SHIFT                                                         0x3
+#define MMEA4_MISC__RELATIVE_PRI_IN_IO_RD_ARB__SHIFT                                                          0x4
+#define MMEA4_MISC__RELATIVE_PRI_IN_IO_WR_ARB__SHIFT                                                          0x5
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC0__SHIFT                                                              0x6
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC1__SHIFT                                                              0x7
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC2__SHIFT                                                              0x8
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC3__SHIFT                                                              0x9
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC4__SHIFT                                                              0xa
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC5__SHIFT                                                              0xb
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC6__SHIFT                                                              0xc
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC7__SHIFT                                                              0xd
+#define MMEA4_MISC__EARLY_SDP_ORIGDATA__SHIFT                                                                 0xe
+#define MMEA4_MISC__LINKMGR_DYNAMIC_MODE__SHIFT                                                               0xf
+#define MMEA4_MISC__LINKMGR_HALT_THRESHOLD__SHIFT                                                             0x11
+#define MMEA4_MISC__LINKMGR_RECONNECT_DELAY__SHIFT                                                            0x13
+#define MMEA4_MISC__LINKMGR_IDLE_THRESHOLD__SHIFT                                                             0x15
+#define MMEA4_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB__SHIFT                                                     0x1a
+#define MMEA4_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB__SHIFT                                                      0x1b
+#define MMEA4_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB__SHIFT                                                         0x1c
+#define MMEA4_MISC__FAVOUR_LAST_CS_IN_GMI_ARB__SHIFT                                                          0x1d
+#define MMEA4_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB__SHIFT                                                       0x1e
+#define MMEA4_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB__SHIFT                                                        0x1f
+#define MMEA4_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB_MASK                                                          0x00000001L
+#define MMEA4_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB_MASK                                                          0x00000002L
+#define MMEA4_MISC__RELATIVE_PRI_IN_GMI_RD_ARB_MASK                                                           0x00000004L
+#define MMEA4_MISC__RELATIVE_PRI_IN_GMI_WR_ARB_MASK                                                           0x00000008L
+#define MMEA4_MISC__RELATIVE_PRI_IN_IO_RD_ARB_MASK                                                            0x00000010L
+#define MMEA4_MISC__RELATIVE_PRI_IN_IO_WR_ARB_MASK                                                            0x00000020L
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC0_MASK                                                                0x00000040L
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC1_MASK                                                                0x00000080L
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC2_MASK                                                                0x00000100L
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC3_MASK                                                                0x00000200L
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC4_MASK                                                                0x00000400L
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC5_MASK                                                                0x00000800L
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC6_MASK                                                                0x00001000L
+#define MMEA4_MISC__EARLYWRRET_ENABLE_VC7_MASK                                                                0x00002000L
+#define MMEA4_MISC__EARLY_SDP_ORIGDATA_MASK                                                                   0x00004000L
+#define MMEA4_MISC__LINKMGR_DYNAMIC_MODE_MASK                                                                 0x00018000L
+#define MMEA4_MISC__LINKMGR_HALT_THRESHOLD_MASK                                                               0x00060000L
+#define MMEA4_MISC__LINKMGR_RECONNECT_DELAY_MASK                                                              0x00180000L
+#define MMEA4_MISC__LINKMGR_IDLE_THRESHOLD_MASK                                                               0x03E00000L
+#define MMEA4_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB_MASK                                                       0x04000000L
+#define MMEA4_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB_MASK                                                        0x08000000L
+#define MMEA4_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB_MASK                                                           0x10000000L
+#define MMEA4_MISC__FAVOUR_LAST_CS_IN_GMI_ARB_MASK                                                            0x20000000L
+#define MMEA4_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB_MASK                                                         0x40000000L
+#define MMEA4_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB_MASK                                                          0x80000000L
+//MMEA4_LATENCY_SAMPLING
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_DRAM__SHIFT                                                          0x0
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_DRAM__SHIFT                                                          0x1
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_GMI__SHIFT                                                           0x2
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_GMI__SHIFT                                                           0x3
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_IO__SHIFT                                                            0x4
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_IO__SHIFT                                                            0x5
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_READ__SHIFT                                                          0x6
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_READ__SHIFT                                                          0x7
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_WRITE__SHIFT                                                         0x8
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_WRITE__SHIFT                                                         0x9
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET__SHIFT                                                    0xa
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET__SHIFT                                                    0xb
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET__SHIFT                                                  0xc
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET__SHIFT                                                  0xd
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_VC__SHIFT                                                            0xe
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_VC__SHIFT                                                            0x16
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_DRAM_MASK                                                            0x00000001L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_DRAM_MASK                                                            0x00000002L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_GMI_MASK                                                             0x00000004L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_GMI_MASK                                                             0x00000008L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_IO_MASK                                                              0x00000010L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_IO_MASK                                                              0x00000020L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_READ_MASK                                                            0x00000040L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_READ_MASK                                                            0x00000080L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_WRITE_MASK                                                           0x00000100L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_WRITE_MASK                                                           0x00000200L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET_MASK                                                      0x00000400L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET_MASK                                                      0x00000800L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET_MASK                                                    0x00001000L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET_MASK                                                    0x00002000L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER0_VC_MASK                                                              0x003FC000L
+#define MMEA4_LATENCY_SAMPLING__SAMPLER1_VC_MASK                                                              0x3FC00000L
+//MMEA4_PERFCOUNTER_LO
+#define MMEA4_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define MMEA4_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//MMEA4_PERFCOUNTER_HI
+#define MMEA4_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define MMEA4_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define MMEA4_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define MMEA4_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+//MMEA4_PERFCOUNTER0_CFG
+#define MMEA4_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define MMEA4_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define MMEA4_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define MMEA4_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define MMEA4_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define MMEA4_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define MMEA4_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define MMEA4_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define MMEA4_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define MMEA4_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//MMEA4_PERFCOUNTER1_CFG
+#define MMEA4_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define MMEA4_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define MMEA4_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define MMEA4_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define MMEA4_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define MMEA4_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define MMEA4_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define MMEA4_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define MMEA4_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define MMEA4_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//MMEA4_PERFCOUNTER_RSLT_CNTL
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define MMEA4_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+//MMEA4_DSM_CNTL
+#define MMEA4_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x0
+#define MMEA4_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x2
+#define MMEA4_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x3
+#define MMEA4_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x5
+#define MMEA4_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x6
+#define MMEA4_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x8
+#define MMEA4_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x9
+#define MMEA4_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xb
+#define MMEA4_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0xc
+#define MMEA4_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xe
+#define MMEA4_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0xf
+#define MMEA4_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x11
+#define MMEA4_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x12
+#define MMEA4_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x14
+#define MMEA4_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x15
+#define MMEA4_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x17
+#define MMEA4_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00000003L
+#define MMEA4_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000004L
+#define MMEA4_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00000018L
+#define MMEA4_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000020L
+#define MMEA4_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                0x000000C0L
+#define MMEA4_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000100L
+#define MMEA4_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00000600L
+#define MMEA4_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000800L
+#define MMEA4_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00003000L
+#define MMEA4_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00004000L
+#define MMEA4_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00018000L
+#define MMEA4_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00020000L
+#define MMEA4_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x000C0000L
+#define MMEA4_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00100000L
+#define MMEA4_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00600000L
+#define MMEA4_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00800000L
+//MMEA4_DSM_CNTLA
+#define MMEA4_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                             0x0
+#define MMEA4_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                            0x2
+#define MMEA4_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                             0x3
+#define MMEA4_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                            0x5
+#define MMEA4_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x6
+#define MMEA4_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x8
+#define MMEA4_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x9
+#define MMEA4_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0xb
+#define MMEA4_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0xc
+#define MMEA4_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0xe
+#define MMEA4_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0xf
+#define MMEA4_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x11
+#define MMEA4_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x12
+#define MMEA4_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x14
+#define MMEA4_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                               0x00000003L
+#define MMEA4_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                              0x00000004L
+#define MMEA4_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                               0x00000018L
+#define MMEA4_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                              0x00000020L
+#define MMEA4_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x000000C0L
+#define MMEA4_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000100L
+#define MMEA4_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000600L
+#define MMEA4_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000800L
+#define MMEA4_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00003000L
+#define MMEA4_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00004000L
+#define MMEA4_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00018000L
+#define MMEA4_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00020000L
+#define MMEA4_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x000C0000L
+#define MMEA4_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00100000L
+//MMEA4_DSM_CNTLB
+#define MMEA4_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x0
+#define MMEA4_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x2
+#define MMEA4_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x3
+#define MMEA4_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x5
+#define MMEA4_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x6
+#define MMEA4_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0x8
+#define MMEA4_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x9
+#define MMEA4_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xb
+#define MMEA4_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000003L
+#define MMEA4_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000004L
+#define MMEA4_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000018L
+#define MMEA4_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000020L
+#define MMEA4_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA_MASK                                                    0x000000C0L
+#define MMEA4_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000100L
+#define MMEA4_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000600L
+#define MMEA4_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000800L
+//MMEA4_DSM_CNTL2
+#define MMEA4_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x0
+#define MMEA4_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                             0x2
+#define MMEA4_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x3
+#define MMEA4_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                             0x5
+#define MMEA4_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x6
+#define MMEA4_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                            0x8
+#define MMEA4_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x9
+#define MMEA4_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                               0xb
+#define MMEA4_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                               0xc
+#define MMEA4_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                               0xe
+#define MMEA4_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0xf
+#define MMEA4_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x11
+#define MMEA4_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x12
+#define MMEA4_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x14
+#define MMEA4_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x15
+#define MMEA4_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0x17
+#define MMEA4_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                  0x1a
+#define MMEA4_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                               0x00000003L
+#define MMEA4_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                               0x00000004L
+#define MMEA4_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                               0x00000018L
+#define MMEA4_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                               0x00000020L
+#define MMEA4_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                              0x000000C0L
+#define MMEA4_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                              0x00000100L
+#define MMEA4_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00000600L
+#define MMEA4_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000800L
+#define MMEA4_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00003000L
+#define MMEA4_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                 0x00004000L
+#define MMEA4_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00018000L
+#define MMEA4_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00020000L
+#define MMEA4_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x000C0000L
+#define MMEA4_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00100000L
+#define MMEA4_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x00600000L
+#define MMEA4_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00800000L
+#define MMEA4_DSM_CNTL2__INJECT_DELAY_MASK                                                                    0xFC000000L
+//MMEA4_DSM_CNTL2A
+#define MMEA4_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                           0x0
+#define MMEA4_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                           0x2
+#define MMEA4_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                           0x3
+#define MMEA4_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                           0x5
+#define MMEA4_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x6
+#define MMEA4_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x8
+#define MMEA4_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x9
+#define MMEA4_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0xb
+#define MMEA4_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0xc
+#define MMEA4_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0xe
+#define MMEA4_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0xf
+#define MMEA4_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x11
+#define MMEA4_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x12
+#define MMEA4_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x14
+#define MMEA4_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                             0x00000003L
+#define MMEA4_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                             0x00000004L
+#define MMEA4_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                             0x00000018L
+#define MMEA4_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                             0x00000020L
+#define MMEA4_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x000000C0L
+#define MMEA4_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000100L
+#define MMEA4_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000600L
+#define MMEA4_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000800L
+#define MMEA4_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x00003000L
+#define MMEA4_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00004000L
+#define MMEA4_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00018000L
+#define MMEA4_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00020000L
+#define MMEA4_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x000C0000L
+#define MMEA4_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00100000L
+//MMEA4_DSM_CNTL2B
+#define MMEA4_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x0
+#define MMEA4_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY__SHIFT                                                0x2
+#define MMEA4_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x3
+#define MMEA4_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY__SHIFT                                                0x5
+#define MMEA4_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x6
+#define MMEA4_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY__SHIFT                                                0x8
+#define MMEA4_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT__SHIFT                                                0x9
+#define MMEA4_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY__SHIFT                                                0xb
+#define MMEA4_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000003L
+#define MMEA4_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000004L
+#define MMEA4_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000018L
+#define MMEA4_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000020L
+#define MMEA4_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT_MASK                                                  0x000000C0L
+#define MMEA4_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000100L
+#define MMEA4_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000600L
+#define MMEA4_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY_MASK                                                  0x00000800L
+//MMEA4_CGTT_CLK_CTRL
+#define MMEA4_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define MMEA4_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define MMEA4_CGTT_CLK_CTRL__SPARE0__SHIFT                                                                    0xc
+#define MMEA4_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE__SHIFT                                                 0x14
+#define MMEA4_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ__SHIFT                                                  0x15
+#define MMEA4_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN__SHIFT                                                0x16
+#define MMEA4_CGTT_CLK_CTRL__SPARE1__SHIFT                                                                    0x17
+#define MMEA4_CGTT_CLK_CTRL__LS_OVERRIDE__SHIFT                                                               0x1b
+#define MMEA4_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE__SHIFT                                                       0x1c
+#define MMEA4_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ__SHIFT                                                        0x1d
+#define MMEA4_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                      0x1e
+#define MMEA4_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                    0x1f
+#define MMEA4_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define MMEA4_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define MMEA4_CGTT_CLK_CTRL__SPARE0_MASK                                                                      0x000FF000L
+#define MMEA4_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE_MASK                                                   0x00100000L
+#define MMEA4_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ_MASK                                                    0x00200000L
+#define MMEA4_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN_MASK                                                  0x00400000L
+#define MMEA4_CGTT_CLK_CTRL__SPARE1_MASK                                                                      0x07800000L
+#define MMEA4_CGTT_CLK_CTRL__LS_OVERRIDE_MASK                                                                 0x08000000L
+#define MMEA4_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE_MASK                                                         0x10000000L
+#define MMEA4_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ_MASK                                                          0x20000000L
+#define MMEA4_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                        0x40000000L
+#define MMEA4_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                      0x80000000L
+//MMEA4_EDC_MODE
+#define MMEA4_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                  0x10
+#define MMEA4_EDC_MODE__GATE_FUE__SHIFT                                                                       0x11
+#define MMEA4_EDC_MODE__DED_MODE__SHIFT                                                                       0x14
+#define MMEA4_EDC_MODE__PROP_FED__SHIFT                                                                       0x1d
+#define MMEA4_EDC_MODE__BYPASS__SHIFT                                                                         0x1f
+#define MMEA4_EDC_MODE__COUNT_FED_OUT_MASK                                                                    0x00010000L
+#define MMEA4_EDC_MODE__GATE_FUE_MASK                                                                         0x00020000L
+#define MMEA4_EDC_MODE__DED_MODE_MASK                                                                         0x00300000L
+#define MMEA4_EDC_MODE__PROP_FED_MASK                                                                         0x20000000L
+#define MMEA4_EDC_MODE__BYPASS_MASK                                                                           0x80000000L
+//MMEA4_ERR_STATUS
+#define MMEA4_ERR_STATUS__SDP_RDRSP_STATUS__SHIFT                                                             0x0
+#define MMEA4_ERR_STATUS__SDP_WRRSP_STATUS__SHIFT                                                             0x4
+#define MMEA4_ERR_STATUS__SDP_RDRSP_DATASTATUS__SHIFT                                                         0x8
+#define MMEA4_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR__SHIFT                                                   0xa
+#define MMEA4_ERR_STATUS__CLEAR_ERROR_STATUS__SHIFT                                                           0xb
+#define MMEA4_ERR_STATUS__BUSY_ON_ERROR__SHIFT                                                                0xc
+#define MMEA4_ERR_STATUS__FUE_FLAG__SHIFT                                                                     0xd
+#define MMEA4_ERR_STATUS__IGNORE_RDRSP_FED__SHIFT                                                             0xe
+#define MMEA4_ERR_STATUS__INTERRUPT_ON_FATAL__SHIFT                                                           0xf
+#define MMEA4_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL__SHIFT                                                   0x10
+#define MMEA4_ERR_STATUS__LEVEL_INTERRUPT__SHIFT                                                              0x11
+#define MMEA4_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR__SHIFT                                                     0x12
+#define MMEA4_ERR_STATUS__FUE_FLAG_CLIENT__SHIFT                                                              0x13
+#define MMEA4_ERR_STATUS__SDP_RDRSP_STATUS_MASK                                                               0x0000000FL
+#define MMEA4_ERR_STATUS__SDP_WRRSP_STATUS_MASK                                                               0x000000F0L
+#define MMEA4_ERR_STATUS__SDP_RDRSP_DATASTATUS_MASK                                                           0x00000300L
+#define MMEA4_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR_MASK                                                     0x00000400L
+#define MMEA4_ERR_STATUS__CLEAR_ERROR_STATUS_MASK                                                             0x00000800L
+#define MMEA4_ERR_STATUS__BUSY_ON_ERROR_MASK                                                                  0x00001000L
+#define MMEA4_ERR_STATUS__FUE_FLAG_MASK                                                                       0x00002000L
+#define MMEA4_ERR_STATUS__IGNORE_RDRSP_FED_MASK                                                               0x00004000L
+#define MMEA4_ERR_STATUS__INTERRUPT_ON_FATAL_MASK                                                             0x00008000L
+#define MMEA4_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL_MASK                                                     0x00010000L
+#define MMEA4_ERR_STATUS__LEVEL_INTERRUPT_MASK                                                                0x00020000L
+#define MMEA4_ERR_STATUS__BUSY_ON_CMPL_FATAL_ERROR_MASK                                                       0x00040000L
+#define MMEA4_ERR_STATUS__FUE_FLAG_CLIENT_MASK                                                                0x00080000L
+//MMEA4_MISC2
+#define MMEA4_MISC2__CSGROUP_SWAP_IN_DRAM_ARB__SHIFT                                                          0x0
+#define MMEA4_MISC2__CSGROUP_SWAP_IN_GMI_ARB__SHIFT                                                           0x1
+#define MMEA4_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM__SHIFT                                                       0x2
+#define MMEA4_MISC2__CSGRP_BURST_LIMIT_DATA_GMI__SHIFT                                                        0x7
+#define MMEA4_MISC2__IO_RDWR_PRIORITY_ENABLE__SHIFT                                                           0xc
+#define MMEA4_MISC2__RRET_SWAP_MODE__SHIFT                                                                    0xd
+#define MMEA4_MISC2__BLOCK_REQUESTS__SHIFT                                                                    0xe
+#define MMEA4_MISC2__REQUESTS_BLOCKED__SHIFT                                                                  0xf
+#define MMEA4_MISC2__DRAM_RD_THROTTLE__SHIFT                                                                  0x10
+#define MMEA4_MISC2__DRAM_WR_THROTTLE__SHIFT                                                                  0x11
+#define MMEA4_MISC2__GMI_RD_THROTTLE__SHIFT                                                                   0x12
+#define MMEA4_MISC2__GMI_WR_THROTTLE__SHIFT                                                                   0x13
+#define MMEA4_MISC2__CSGROUP_SWAP_IN_DRAM_ARB_MASK                                                            0x00000001L
+#define MMEA4_MISC2__CSGROUP_SWAP_IN_GMI_ARB_MASK                                                             0x00000002L
+#define MMEA4_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM_MASK                                                         0x0000007CL
+#define MMEA4_MISC2__CSGRP_BURST_LIMIT_DATA_GMI_MASK                                                          0x00000F80L
+#define MMEA4_MISC2__IO_RDWR_PRIORITY_ENABLE_MASK                                                             0x00001000L
+#define MMEA4_MISC2__RRET_SWAP_MODE_MASK                                                                      0x00002000L
+#define MMEA4_MISC2__BLOCK_REQUESTS_MASK                                                                      0x00004000L
+#define MMEA4_MISC2__REQUESTS_BLOCKED_MASK                                                                    0x00008000L
+#define MMEA4_MISC2__DRAM_RD_THROTTLE_MASK                                                                    0x00010000L
+#define MMEA4_MISC2__DRAM_WR_THROTTLE_MASK                                                                    0x00020000L
+#define MMEA4_MISC2__GMI_RD_THROTTLE_MASK                                                                     0x00040000L
+#define MMEA4_MISC2__GMI_WR_THROTTLE_MASK                                                                     0x00080000L
+//MMEA4_MISC_AON
+#define MMEA4_MISC_AON__LINKMGR_PARTACK_HYSTERESIS__SHIFT                                                     0x0
+#define MMEA4_MISC_AON__LINKMGR_PARTACK_DEASSERT_MODE__SHIFT                                                  0x2
+#define MMEA4_MISC_AON__LINKMGR_PARTACK_HYSTERESIS_MASK                                                       0x00000003L
+#define MMEA4_MISC_AON__LINKMGR_PARTACK_DEASSERT_MODE_MASK                                                    0x00000004L
+
+
+// addressBlock: aid_mmhub_pctldec0
+//PCTL0_CTRL
+#define PCTL0_CTRL__PG_ENABLE__SHIFT                                                                          0x0
+#define PCTL0_CTRL__ALLOW_DEEP_SLEEP_MODE__SHIFT                                                              0x1
+#define PCTL0_CTRL__STCTRL_RSMU_IDLE_THRESHOLD__SHIFT                                                         0x4
+#define PCTL0_CTRL__STCTRL_DAGB_IDLE_THRESHOLD__SHIFT                                                         0xb
+#define PCTL0_CTRL__STCTRL_IGNORE_PROTECTION_FAULT__SHIFT                                                     0x10
+#define PCTL0_CTRL__OVR_EA0_SDP_PARTACK__SHIFT                                                                0x11
+#define PCTL0_CTRL__OVR_EA1_SDP_PARTACK__SHIFT                                                                0x12
+#define PCTL0_CTRL__OVR_EA2_SDP_PARTACK__SHIFT                                                                0x13
+#define PCTL0_CTRL__OVR_EA3_SDP_PARTACK__SHIFT                                                                0x14
+#define PCTL0_CTRL__OVR_EA4_SDP_PARTACK__SHIFT                                                                0x15
+#define PCTL0_CTRL__OVR_EA5_SDP_PARTACK__SHIFT                                                                0x16
+#define PCTL0_CTRL__OVR_EA0_SDP_FULLACK__SHIFT                                                                0x17
+#define PCTL0_CTRL__OVR_EA1_SDP_FULLACK__SHIFT                                                                0x18
+#define PCTL0_CTRL__OVR_EA2_SDP_FULLACK__SHIFT                                                                0x19
+#define PCTL0_CTRL__OVR_EA3_SDP_FULLACK__SHIFT                                                                0x1a
+#define PCTL0_CTRL__OVR_EA4_SDP_FULLACK__SHIFT                                                                0x1b
+#define PCTL0_CTRL__OVR_EA5_SDP_FULLACK__SHIFT                                                                0x1c
+#define PCTL0_CTRL__RSMU_RDTIMER_ENABLE__SHIFT                                                                0x1d
+#define PCTL0_CTRL__RSMU_RDTIMER_THRESHOLD__SHIFT                                                             0x1e
+#define PCTL0_CTRL__PG_ENABLE_MASK                                                                            0x00000001L
+#define PCTL0_CTRL__ALLOW_DEEP_SLEEP_MODE_MASK                                                                0x0000000EL
+#define PCTL0_CTRL__STCTRL_RSMU_IDLE_THRESHOLD_MASK                                                           0x000007F0L
+#define PCTL0_CTRL__STCTRL_DAGB_IDLE_THRESHOLD_MASK                                                           0x0000F800L
+#define PCTL0_CTRL__STCTRL_IGNORE_PROTECTION_FAULT_MASK                                                       0x00010000L
+#define PCTL0_CTRL__OVR_EA0_SDP_PARTACK_MASK                                                                  0x00020000L
+#define PCTL0_CTRL__OVR_EA1_SDP_PARTACK_MASK                                                                  0x00040000L
+#define PCTL0_CTRL__OVR_EA2_SDP_PARTACK_MASK                                                                  0x00080000L
+#define PCTL0_CTRL__OVR_EA3_SDP_PARTACK_MASK                                                                  0x00100000L
+#define PCTL0_CTRL__OVR_EA4_SDP_PARTACK_MASK                                                                  0x00200000L
+#define PCTL0_CTRL__OVR_EA5_SDP_PARTACK_MASK                                                                  0x00400000L
+#define PCTL0_CTRL__OVR_EA0_SDP_FULLACK_MASK                                                                  0x00800000L
+#define PCTL0_CTRL__OVR_EA1_SDP_FULLACK_MASK                                                                  0x01000000L
+#define PCTL0_CTRL__OVR_EA2_SDP_FULLACK_MASK                                                                  0x02000000L
+#define PCTL0_CTRL__OVR_EA3_SDP_FULLACK_MASK                                                                  0x04000000L
+#define PCTL0_CTRL__OVR_EA4_SDP_FULLACK_MASK                                                                  0x08000000L
+#define PCTL0_CTRL__OVR_EA5_SDP_FULLACK_MASK                                                                  0x10000000L
+#define PCTL0_CTRL__RSMU_RDTIMER_ENABLE_MASK                                                                  0x20000000L
+#define PCTL0_CTRL__RSMU_RDTIMER_THRESHOLD_MASK                                                               0xC0000000L
+//PCTL0_MMHUB_DEEPSLEEP_IB
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS0__SHIFT                                                                  0x0
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS1__SHIFT                                                                  0x1
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS2__SHIFT                                                                  0x2
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS3__SHIFT                                                                  0x3
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS4__SHIFT                                                                  0x4
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS5__SHIFT                                                                  0x5
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS6__SHIFT                                                                  0x6
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS7__SHIFT                                                                  0x7
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS8__SHIFT                                                                  0x8
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS9__SHIFT                                                                  0x9
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS10__SHIFT                                                                 0xa
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS11__SHIFT                                                                 0xb
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS12__SHIFT                                                                 0xc
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS13__SHIFT                                                                 0xd
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS14__SHIFT                                                                 0xe
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS15__SHIFT                                                                 0xf
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS16__SHIFT                                                                 0x10
+#define PCTL0_MMHUB_DEEPSLEEP_IB__SETCLEAR__SHIFT                                                             0x1f
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS0_MASK                                                                    0x00000001L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS1_MASK                                                                    0x00000002L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS2_MASK                                                                    0x00000004L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS3_MASK                                                                    0x00000008L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS4_MASK                                                                    0x00000010L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS5_MASK                                                                    0x00000020L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS6_MASK                                                                    0x00000040L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS7_MASK                                                                    0x00000080L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS8_MASK                                                                    0x00000100L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS9_MASK                                                                    0x00000200L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS10_MASK                                                                   0x00000400L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS11_MASK                                                                   0x00000800L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS12_MASK                                                                   0x00001000L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS13_MASK                                                                   0x00002000L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS14_MASK                                                                   0x00004000L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS15_MASK                                                                   0x00008000L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__DS16_MASK                                                                   0x00010000L
+#define PCTL0_MMHUB_DEEPSLEEP_IB__SETCLEAR_MASK                                                               0x80000000L
+//PCTL0_MMHUB_DEEPSLEEP_OVERRIDE
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS0__SHIFT                                                            0x0
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS1__SHIFT                                                            0x1
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS2__SHIFT                                                            0x2
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS3__SHIFT                                                            0x3
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS4__SHIFT                                                            0x4
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS5__SHIFT                                                            0x5
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS6__SHIFT                                                            0x6
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS7__SHIFT                                                            0x7
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS8__SHIFT                                                            0x8
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS9__SHIFT                                                            0x9
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS10__SHIFT                                                           0xa
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS11__SHIFT                                                           0xb
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS12__SHIFT                                                           0xc
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS13__SHIFT                                                           0xd
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS14__SHIFT                                                           0xe
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS15__SHIFT                                                           0xf
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS16__SHIFT                                                           0x10
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS_ATHUB__SHIFT                                                       0x11
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS_CANE__SHIFT                                                        0x12
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS0_MASK                                                              0x00000001L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS1_MASK                                                              0x00000002L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS2_MASK                                                              0x00000004L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS3_MASK                                                              0x00000008L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS4_MASK                                                              0x00000010L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS5_MASK                                                              0x00000020L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS6_MASK                                                              0x00000040L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS7_MASK                                                              0x00000080L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS8_MASK                                                              0x00000100L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS9_MASK                                                              0x00000200L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS10_MASK                                                             0x00000400L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS11_MASK                                                             0x00000800L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS12_MASK                                                             0x00001000L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS13_MASK                                                             0x00002000L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS14_MASK                                                             0x00004000L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS15_MASK                                                             0x00008000L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS16_MASK                                                             0x00010000L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS_ATHUB_MASK                                                         0x00020000L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE__DS_CANE_MASK                                                          0x00040000L
+//PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS0__SHIFT                                                         0x0
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS1__SHIFT                                                         0x1
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS2__SHIFT                                                         0x2
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS3__SHIFT                                                         0x3
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS4__SHIFT                                                         0x4
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS5__SHIFT                                                         0x5
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS6__SHIFT                                                         0x6
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS7__SHIFT                                                         0x7
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS8__SHIFT                                                         0x8
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS9__SHIFT                                                         0x9
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS10__SHIFT                                                        0xa
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS11__SHIFT                                                        0xb
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS12__SHIFT                                                        0xc
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS13__SHIFT                                                        0xd
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS14__SHIFT                                                        0xe
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS15__SHIFT                                                        0xf
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS16__SHIFT                                                        0x10
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS0_MASK                                                           0x00000001L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS1_MASK                                                           0x00000002L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS2_MASK                                                           0x00000004L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS3_MASK                                                           0x00000008L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS4_MASK                                                           0x00000010L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS5_MASK                                                           0x00000020L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS6_MASK                                                           0x00000040L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS7_MASK                                                           0x00000080L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS8_MASK                                                           0x00000100L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS9_MASK                                                           0x00000200L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS10_MASK                                                          0x00000400L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS11_MASK                                                          0x00000800L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS12_MASK                                                          0x00001000L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS13_MASK                                                          0x00002000L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS14_MASK                                                          0x00004000L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS15_MASK                                                          0x00008000L
+#define PCTL0_MMHUB_DEEPSLEEP_OVERRIDE_IB__DS16_MASK                                                          0x00010000L
+//PCTL0_PG_IGNORE_DEEPSLEEP
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS0__SHIFT                                                                 0x0
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS1__SHIFT                                                                 0x1
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS2__SHIFT                                                                 0x2
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS3__SHIFT                                                                 0x3
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS4__SHIFT                                                                 0x4
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS5__SHIFT                                                                 0x5
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS6__SHIFT                                                                 0x6
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS7__SHIFT                                                                 0x7
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS8__SHIFT                                                                 0x8
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS9__SHIFT                                                                 0x9
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS10__SHIFT                                                                0xa
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS11__SHIFT                                                                0xb
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS12__SHIFT                                                                0xc
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS13__SHIFT                                                                0xd
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS14__SHIFT                                                                0xe
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS15__SHIFT                                                                0xf
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS16__SHIFT                                                                0x10
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS_ATHUB__SHIFT                                                            0x11
+#define PCTL0_PG_IGNORE_DEEPSLEEP__ALLIPS__SHIFT                                                              0x12
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS0_MASK                                                                   0x00000001L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS1_MASK                                                                   0x00000002L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS2_MASK                                                                   0x00000004L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS3_MASK                                                                   0x00000008L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS4_MASK                                                                   0x00000010L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS5_MASK                                                                   0x00000020L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS6_MASK                                                                   0x00000040L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS7_MASK                                                                   0x00000080L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS8_MASK                                                                   0x00000100L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS9_MASK                                                                   0x00000200L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS10_MASK                                                                  0x00000400L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS11_MASK                                                                  0x00000800L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS12_MASK                                                                  0x00001000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS13_MASK                                                                  0x00002000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS14_MASK                                                                  0x00004000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS15_MASK                                                                  0x00008000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS16_MASK                                                                  0x00010000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__DS_ATHUB_MASK                                                              0x00020000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP__ALLIPS_MASK                                                                0x00040000L
+//PCTL0_PG_IGNORE_DEEPSLEEP_IB
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS0__SHIFT                                                              0x0
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS1__SHIFT                                                              0x1
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS2__SHIFT                                                              0x2
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS3__SHIFT                                                              0x3
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS4__SHIFT                                                              0x4
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS5__SHIFT                                                              0x5
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS6__SHIFT                                                              0x6
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS7__SHIFT                                                              0x7
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS8__SHIFT                                                              0x8
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS9__SHIFT                                                              0x9
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS10__SHIFT                                                             0xa
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS11__SHIFT                                                             0xb
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS12__SHIFT                                                             0xc
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS13__SHIFT                                                             0xd
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS14__SHIFT                                                             0xe
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS15__SHIFT                                                             0xf
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS16__SHIFT                                                             0x10
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__ALLIPS__SHIFT                                                           0x11
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS0_MASK                                                                0x00000001L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS1_MASK                                                                0x00000002L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS2_MASK                                                                0x00000004L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS3_MASK                                                                0x00000008L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS4_MASK                                                                0x00000010L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS5_MASK                                                                0x00000020L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS6_MASK                                                                0x00000040L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS7_MASK                                                                0x00000080L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS8_MASK                                                                0x00000100L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS9_MASK                                                                0x00000200L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS10_MASK                                                               0x00000400L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS11_MASK                                                               0x00000800L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS12_MASK                                                               0x00001000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS13_MASK                                                               0x00002000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS14_MASK                                                               0x00004000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS15_MASK                                                               0x00008000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__DS16_MASK                                                               0x00010000L
+#define PCTL0_PG_IGNORE_DEEPSLEEP_IB__ALLIPS_MASK                                                             0x00020000L
+//PCTL0_SLICE0_CFG_DAGB_BUSY
+#define PCTL0_SLICE0_CFG_DAGB_BUSY__DB_LNCFG__SHIFT                                                           0x0
+#define PCTL0_SLICE0_CFG_DAGB_BUSY__DB_LNCFG_MASK                                                             0xFFFFFFFFL
+//PCTL0_SLICE0_CFG_DS_ALLOW
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS0__SHIFT                                                                 0x0
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS1__SHIFT                                                                 0x1
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS2__SHIFT                                                                 0x2
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS3__SHIFT                                                                 0x3
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS4__SHIFT                                                                 0x4
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS5__SHIFT                                                                 0x5
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS6__SHIFT                                                                 0x6
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS7__SHIFT                                                                 0x7
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS8__SHIFT                                                                 0x8
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS9__SHIFT                                                                 0x9
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS10__SHIFT                                                                0xa
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS11__SHIFT                                                                0xb
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS12__SHIFT                                                                0xc
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS13__SHIFT                                                                0xd
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS14__SHIFT                                                                0xe
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS15__SHIFT                                                                0xf
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS16__SHIFT                                                                0x10
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS0_MASK                                                                   0x00000001L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS1_MASK                                                                   0x00000002L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS2_MASK                                                                   0x00000004L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS3_MASK                                                                   0x00000008L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS4_MASK                                                                   0x00000010L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS5_MASK                                                                   0x00000020L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS6_MASK                                                                   0x00000040L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS7_MASK                                                                   0x00000080L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS8_MASK                                                                   0x00000100L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS9_MASK                                                                   0x00000200L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS10_MASK                                                                  0x00000400L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS11_MASK                                                                  0x00000800L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS12_MASK                                                                  0x00001000L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS13_MASK                                                                  0x00002000L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS14_MASK                                                                  0x00004000L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS15_MASK                                                                  0x00008000L
+#define PCTL0_SLICE0_CFG_DS_ALLOW__DS16_MASK                                                                  0x00010000L
+//PCTL0_SLICE0_CFG_DS_ALLOW_IB
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS0__SHIFT                                                              0x0
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS1__SHIFT                                                              0x1
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS2__SHIFT                                                              0x2
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS3__SHIFT                                                              0x3
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS4__SHIFT                                                              0x4
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS5__SHIFT                                                              0x5
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS6__SHIFT                                                              0x6
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS7__SHIFT                                                              0x7
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS8__SHIFT                                                              0x8
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS9__SHIFT                                                              0x9
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS10__SHIFT                                                             0xa
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS11__SHIFT                                                             0xb
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS12__SHIFT                                                             0xc
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS13__SHIFT                                                             0xd
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS14__SHIFT                                                             0xe
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS15__SHIFT                                                             0xf
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS16__SHIFT                                                             0x10
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS0_MASK                                                                0x00000001L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS1_MASK                                                                0x00000002L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS2_MASK                                                                0x00000004L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS3_MASK                                                                0x00000008L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS4_MASK                                                                0x00000010L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS5_MASK                                                                0x00000020L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS6_MASK                                                                0x00000040L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS7_MASK                                                                0x00000080L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS8_MASK                                                                0x00000100L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS9_MASK                                                                0x00000200L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS10_MASK                                                               0x00000400L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS11_MASK                                                               0x00000800L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS12_MASK                                                               0x00001000L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS13_MASK                                                               0x00002000L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS14_MASK                                                               0x00004000L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS15_MASK                                                               0x00008000L
+#define PCTL0_SLICE0_CFG_DS_ALLOW_IB__DS16_MASK                                                               0x00010000L
+//PCTL0_SLICE1_CFG_DAGB_BUSY
+#define PCTL0_SLICE1_CFG_DAGB_BUSY__DB_LNCFG__SHIFT                                                           0x0
+#define PCTL0_SLICE1_CFG_DAGB_BUSY__DB_LNCFG_MASK                                                             0xFFFFFFFFL
+//PCTL0_SLICE1_CFG_DS_ALLOW
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS0__SHIFT                                                                 0x0
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS1__SHIFT                                                                 0x1
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS2__SHIFT                                                                 0x2
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS3__SHIFT                                                                 0x3
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS4__SHIFT                                                                 0x4
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS5__SHIFT                                                                 0x5
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS6__SHIFT                                                                 0x6
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS7__SHIFT                                                                 0x7
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS8__SHIFT                                                                 0x8
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS9__SHIFT                                                                 0x9
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS10__SHIFT                                                                0xa
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS11__SHIFT                                                                0xb
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS12__SHIFT                                                                0xc
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS13__SHIFT                                                                0xd
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS14__SHIFT                                                                0xe
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS15__SHIFT                                                                0xf
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS16__SHIFT                                                                0x10
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS0_MASK                                                                   0x00000001L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS1_MASK                                                                   0x00000002L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS2_MASK                                                                   0x00000004L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS3_MASK                                                                   0x00000008L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS4_MASK                                                                   0x00000010L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS5_MASK                                                                   0x00000020L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS6_MASK                                                                   0x00000040L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS7_MASK                                                                   0x00000080L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS8_MASK                                                                   0x00000100L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS9_MASK                                                                   0x00000200L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS10_MASK                                                                  0x00000400L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS11_MASK                                                                  0x00000800L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS12_MASK                                                                  0x00001000L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS13_MASK                                                                  0x00002000L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS14_MASK                                                                  0x00004000L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS15_MASK                                                                  0x00008000L
+#define PCTL0_SLICE1_CFG_DS_ALLOW__DS16_MASK                                                                  0x00010000L
+//PCTL0_SLICE1_CFG_DS_ALLOW_IB
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS0__SHIFT                                                              0x0
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS1__SHIFT                                                              0x1
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS2__SHIFT                                                              0x2
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS3__SHIFT                                                              0x3
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS4__SHIFT                                                              0x4
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS5__SHIFT                                                              0x5
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS6__SHIFT                                                              0x6
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS7__SHIFT                                                              0x7
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS8__SHIFT                                                              0x8
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS9__SHIFT                                                              0x9
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS10__SHIFT                                                             0xa
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS11__SHIFT                                                             0xb
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS12__SHIFT                                                             0xc
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS13__SHIFT                                                             0xd
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS14__SHIFT                                                             0xe
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS15__SHIFT                                                             0xf
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS16__SHIFT                                                             0x10
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS0_MASK                                                                0x00000001L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS1_MASK                                                                0x00000002L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS2_MASK                                                                0x00000004L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS3_MASK                                                                0x00000008L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS4_MASK                                                                0x00000010L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS5_MASK                                                                0x00000020L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS6_MASK                                                                0x00000040L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS7_MASK                                                                0x00000080L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS8_MASK                                                                0x00000100L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS9_MASK                                                                0x00000200L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS10_MASK                                                               0x00000400L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS11_MASK                                                               0x00000800L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS12_MASK                                                               0x00001000L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS13_MASK                                                               0x00002000L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS14_MASK                                                               0x00004000L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS15_MASK                                                               0x00008000L
+#define PCTL0_SLICE1_CFG_DS_ALLOW_IB__DS16_MASK                                                               0x00010000L
+//PCTL0_SLICE2_CFG_DAGB_BUSY
+#define PCTL0_SLICE2_CFG_DAGB_BUSY__DB_LNCFG__SHIFT                                                           0x0
+#define PCTL0_SLICE2_CFG_DAGB_BUSY__DB_LNCFG_MASK                                                             0xFFFFFFFFL
+//PCTL0_SLICE2_CFG_DS_ALLOW
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS0__SHIFT                                                                 0x0
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS1__SHIFT                                                                 0x1
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS2__SHIFT                                                                 0x2
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS3__SHIFT                                                                 0x3
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS4__SHIFT                                                                 0x4
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS5__SHIFT                                                                 0x5
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS6__SHIFT                                                                 0x6
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS7__SHIFT                                                                 0x7
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS8__SHIFT                                                                 0x8
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS9__SHIFT                                                                 0x9
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS10__SHIFT                                                                0xa
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS11__SHIFT                                                                0xb
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS12__SHIFT                                                                0xc
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS13__SHIFT                                                                0xd
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS14__SHIFT                                                                0xe
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS15__SHIFT                                                                0xf
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS16__SHIFT                                                                0x10
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS0_MASK                                                                   0x00000001L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS1_MASK                                                                   0x00000002L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS2_MASK                                                                   0x00000004L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS3_MASK                                                                   0x00000008L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS4_MASK                                                                   0x00000010L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS5_MASK                                                                   0x00000020L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS6_MASK                                                                   0x00000040L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS7_MASK                                                                   0x00000080L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS8_MASK                                                                   0x00000100L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS9_MASK                                                                   0x00000200L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS10_MASK                                                                  0x00000400L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS11_MASK                                                                  0x00000800L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS12_MASK                                                                  0x00001000L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS13_MASK                                                                  0x00002000L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS14_MASK                                                                  0x00004000L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS15_MASK                                                                  0x00008000L
+#define PCTL0_SLICE2_CFG_DS_ALLOW__DS16_MASK                                                                  0x00010000L
+//PCTL0_SLICE2_CFG_DS_ALLOW_IB
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS0__SHIFT                                                              0x0
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS1__SHIFT                                                              0x1
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS2__SHIFT                                                              0x2
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS3__SHIFT                                                              0x3
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS4__SHIFT                                                              0x4
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS5__SHIFT                                                              0x5
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS6__SHIFT                                                              0x6
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS7__SHIFT                                                              0x7
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS8__SHIFT                                                              0x8
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS9__SHIFT                                                              0x9
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS10__SHIFT                                                             0xa
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS11__SHIFT                                                             0xb
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS12__SHIFT                                                             0xc
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS13__SHIFT                                                             0xd
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS14__SHIFT                                                             0xe
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS15__SHIFT                                                             0xf
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS16__SHIFT                                                             0x10
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS0_MASK                                                                0x00000001L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS1_MASK                                                                0x00000002L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS2_MASK                                                                0x00000004L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS3_MASK                                                                0x00000008L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS4_MASK                                                                0x00000010L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS5_MASK                                                                0x00000020L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS6_MASK                                                                0x00000040L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS7_MASK                                                                0x00000080L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS8_MASK                                                                0x00000100L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS9_MASK                                                                0x00000200L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS10_MASK                                                               0x00000400L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS11_MASK                                                               0x00000800L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS12_MASK                                                               0x00001000L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS13_MASK                                                               0x00002000L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS14_MASK                                                               0x00004000L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS15_MASK                                                               0x00008000L
+#define PCTL0_SLICE2_CFG_DS_ALLOW_IB__DS16_MASK                                                               0x00010000L
+//PCTL0_SLICE3_CFG_DAGB_BUSY
+#define PCTL0_SLICE3_CFG_DAGB_BUSY__DB_LNCFG__SHIFT                                                           0x0
+#define PCTL0_SLICE3_CFG_DAGB_BUSY__DB_LNCFG_MASK                                                             0xFFFFFFFFL
+//PCTL0_SLICE3_CFG_DS_ALLOW
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS0__SHIFT                                                                 0x0
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS1__SHIFT                                                                 0x1
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS2__SHIFT                                                                 0x2
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS3__SHIFT                                                                 0x3
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS4__SHIFT                                                                 0x4
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS5__SHIFT                                                                 0x5
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS6__SHIFT                                                                 0x6
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS7__SHIFT                                                                 0x7
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS8__SHIFT                                                                 0x8
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS9__SHIFT                                                                 0x9
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS10__SHIFT                                                                0xa
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS11__SHIFT                                                                0xb
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS12__SHIFT                                                                0xc
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS13__SHIFT                                                                0xd
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS14__SHIFT                                                                0xe
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS15__SHIFT                                                                0xf
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS16__SHIFT                                                                0x10
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS0_MASK                                                                   0x00000001L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS1_MASK                                                                   0x00000002L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS2_MASK                                                                   0x00000004L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS3_MASK                                                                   0x00000008L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS4_MASK                                                                   0x00000010L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS5_MASK                                                                   0x00000020L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS6_MASK                                                                   0x00000040L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS7_MASK                                                                   0x00000080L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS8_MASK                                                                   0x00000100L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS9_MASK                                                                   0x00000200L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS10_MASK                                                                  0x00000400L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS11_MASK                                                                  0x00000800L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS12_MASK                                                                  0x00001000L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS13_MASK                                                                  0x00002000L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS14_MASK                                                                  0x00004000L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS15_MASK                                                                  0x00008000L
+#define PCTL0_SLICE3_CFG_DS_ALLOW__DS16_MASK                                                                  0x00010000L
+//PCTL0_SLICE3_CFG_DS_ALLOW_IB
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS0__SHIFT                                                              0x0
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS1__SHIFT                                                              0x1
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS2__SHIFT                                                              0x2
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS3__SHIFT                                                              0x3
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS4__SHIFT                                                              0x4
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS5__SHIFT                                                              0x5
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS6__SHIFT                                                              0x6
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS7__SHIFT                                                              0x7
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS8__SHIFT                                                              0x8
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS9__SHIFT                                                              0x9
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS10__SHIFT                                                             0xa
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS11__SHIFT                                                             0xb
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS12__SHIFT                                                             0xc
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS13__SHIFT                                                             0xd
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS14__SHIFT                                                             0xe
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS15__SHIFT                                                             0xf
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS16__SHIFT                                                             0x10
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS0_MASK                                                                0x00000001L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS1_MASK                                                                0x00000002L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS2_MASK                                                                0x00000004L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS3_MASK                                                                0x00000008L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS4_MASK                                                                0x00000010L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS5_MASK                                                                0x00000020L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS6_MASK                                                                0x00000040L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS7_MASK                                                                0x00000080L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS8_MASK                                                                0x00000100L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS9_MASK                                                                0x00000200L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS10_MASK                                                               0x00000400L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS11_MASK                                                               0x00000800L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS12_MASK                                                               0x00001000L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS13_MASK                                                               0x00002000L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS14_MASK                                                               0x00004000L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS15_MASK                                                               0x00008000L
+#define PCTL0_SLICE3_CFG_DS_ALLOW_IB__DS16_MASK                                                               0x00010000L
+//PCTL0_SLICE4_CFG_DAGB_BUSY
+#define PCTL0_SLICE4_CFG_DAGB_BUSY__DB_LNCFG__SHIFT                                                           0x0
+#define PCTL0_SLICE4_CFG_DAGB_BUSY__DB_LNCFG_MASK                                                             0xFFFFFFFFL
+//PCTL0_SLICE4_CFG_DS_ALLOW
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS0__SHIFT                                                                 0x0
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS1__SHIFT                                                                 0x1
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS2__SHIFT                                                                 0x2
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS3__SHIFT                                                                 0x3
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS4__SHIFT                                                                 0x4
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS5__SHIFT                                                                 0x5
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS6__SHIFT                                                                 0x6
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS7__SHIFT                                                                 0x7
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS8__SHIFT                                                                 0x8
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS9__SHIFT                                                                 0x9
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS10__SHIFT                                                                0xa
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS11__SHIFT                                                                0xb
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS12__SHIFT                                                                0xc
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS13__SHIFT                                                                0xd
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS14__SHIFT                                                                0xe
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS15__SHIFT                                                                0xf
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS16__SHIFT                                                                0x10
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS0_MASK                                                                   0x00000001L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS1_MASK                                                                   0x00000002L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS2_MASK                                                                   0x00000004L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS3_MASK                                                                   0x00000008L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS4_MASK                                                                   0x00000010L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS5_MASK                                                                   0x00000020L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS6_MASK                                                                   0x00000040L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS7_MASK                                                                   0x00000080L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS8_MASK                                                                   0x00000100L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS9_MASK                                                                   0x00000200L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS10_MASK                                                                  0x00000400L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS11_MASK                                                                  0x00000800L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS12_MASK                                                                  0x00001000L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS13_MASK                                                                  0x00002000L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS14_MASK                                                                  0x00004000L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS15_MASK                                                                  0x00008000L
+#define PCTL0_SLICE4_CFG_DS_ALLOW__DS16_MASK                                                                  0x00010000L
+//PCTL0_SLICE4_CFG_DS_ALLOW_IB
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS0__SHIFT                                                              0x0
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS1__SHIFT                                                              0x1
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS2__SHIFT                                                              0x2
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS3__SHIFT                                                              0x3
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS4__SHIFT                                                              0x4
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS5__SHIFT                                                              0x5
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS6__SHIFT                                                              0x6
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS7__SHIFT                                                              0x7
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS8__SHIFT                                                              0x8
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS9__SHIFT                                                              0x9
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS10__SHIFT                                                             0xa
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS11__SHIFT                                                             0xb
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS12__SHIFT                                                             0xc
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS13__SHIFT                                                             0xd
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS14__SHIFT                                                             0xe
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS15__SHIFT                                                             0xf
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS16__SHIFT                                                             0x10
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS0_MASK                                                                0x00000001L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS1_MASK                                                                0x00000002L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS2_MASK                                                                0x00000004L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS3_MASK                                                                0x00000008L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS4_MASK                                                                0x00000010L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS5_MASK                                                                0x00000020L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS6_MASK                                                                0x00000040L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS7_MASK                                                                0x00000080L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS8_MASK                                                                0x00000100L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS9_MASK                                                                0x00000200L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS10_MASK                                                               0x00000400L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS11_MASK                                                               0x00000800L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS12_MASK                                                               0x00001000L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS13_MASK                                                               0x00002000L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS14_MASK                                                               0x00004000L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS15_MASK                                                               0x00008000L
+#define PCTL0_SLICE4_CFG_DS_ALLOW_IB__DS16_MASK                                                               0x00010000L
+//PCTL0_UTCL2_MISC
+#define PCTL0_UTCL2_MISC__RENG_EXECUTE_NONSECURE_START_PTR__SHIFT                                             0x0
+#define PCTL0_UTCL2_MISC__CRITICAL_REGS_LOCK__SHIFT                                                           0xb
+#define PCTL0_UTCL2_MISC__TILE_IDLE_THRESHOLD__SHIFT                                                          0xc
+#define PCTL0_UTCL2_MISC__RENG_MEM_LS_ENABLE__SHIFT                                                           0xf
+#define PCTL0_UTCL2_MISC__STCTRL_FORCE_PGFSM_CMD_DONE__SHIFT                                                  0x10
+#define PCTL0_UTCL2_MISC__RENG_EXECUTE_ON_REG_UPDATE__SHIFT                                                   0x11
+#define PCTL0_UTCL2_MISC__RD_TIMER_ENABLE__SHIFT                                                              0x12
+#define PCTL0_UTCL2_MISC__RENG_EXECUTE_NONSECURE_START_PTR_MASK                                               0x000007FFL
+#define PCTL0_UTCL2_MISC__CRITICAL_REGS_LOCK_MASK                                                             0x00000800L
+#define PCTL0_UTCL2_MISC__TILE_IDLE_THRESHOLD_MASK                                                            0x00007000L
+#define PCTL0_UTCL2_MISC__RENG_MEM_LS_ENABLE_MASK                                                             0x00008000L
+#define PCTL0_UTCL2_MISC__STCTRL_FORCE_PGFSM_CMD_DONE_MASK                                                    0x00010000L
+#define PCTL0_UTCL2_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK                                                     0x00020000L
+#define PCTL0_UTCL2_MISC__RD_TIMER_ENABLE_MASK                                                                0x00040000L
+//PCTL0_SLICE0_MISC
+#define PCTL0_SLICE0_MISC__RENG_EXECUTE_NONSECURE_START_PTR__SHIFT                                            0x0
+#define PCTL0_SLICE0_MISC__CRITICAL_REGS_LOCK__SHIFT                                                          0xa
+#define PCTL0_SLICE0_MISC__TILE_IDLE_THRESHOLD__SHIFT                                                         0xb
+#define PCTL0_SLICE0_MISC__RENG_MEM_LS_ENABLE__SHIFT                                                          0xe
+#define PCTL0_SLICE0_MISC__STCTRL_FORCE_PGFSM_CMD_DONE__SHIFT                                                 0xf
+#define PCTL0_SLICE0_MISC__DEEPSLEEP_DISCSDP__SHIFT                                                           0x10
+#define PCTL0_SLICE0_MISC__RENG_EXECUTE_ON_REG_UPDATE__SHIFT                                                  0x11
+#define PCTL0_SLICE0_MISC__RD_TIMER_ENABLE__SHIFT                                                             0x12
+#define PCTL0_SLICE0_MISC__RENG_EXECUTE_NONSECURE_START_PTR_MASK                                              0x000003FFL
+#define PCTL0_SLICE0_MISC__CRITICAL_REGS_LOCK_MASK                                                            0x00000400L
+#define PCTL0_SLICE0_MISC__TILE_IDLE_THRESHOLD_MASK                                                           0x00003800L
+#define PCTL0_SLICE0_MISC__RENG_MEM_LS_ENABLE_MASK                                                            0x00004000L
+#define PCTL0_SLICE0_MISC__STCTRL_FORCE_PGFSM_CMD_DONE_MASK                                                   0x00008000L
+#define PCTL0_SLICE0_MISC__DEEPSLEEP_DISCSDP_MASK                                                             0x00010000L
+#define PCTL0_SLICE0_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK                                                    0x00020000L
+#define PCTL0_SLICE0_MISC__RD_TIMER_ENABLE_MASK                                                               0x00040000L
+//PCTL0_SLICE1_MISC
+#define PCTL0_SLICE1_MISC__RENG_EXECUTE_NONSECURE_START_PTR__SHIFT                                            0x0
+#define PCTL0_SLICE1_MISC__CRITICAL_REGS_LOCK__SHIFT                                                          0xa
+#define PCTL0_SLICE1_MISC__TILE_IDLE_THRESHOLD__SHIFT                                                         0xb
+#define PCTL0_SLICE1_MISC__RENG_MEM_LS_ENABLE__SHIFT                                                          0xe
+#define PCTL0_SLICE1_MISC__STCTRL_FORCE_PGFSM_CMD_DONE__SHIFT                                                 0xf
+#define PCTL0_SLICE1_MISC__DEEPSLEEP_DISCSDP__SHIFT                                                           0x10
+#define PCTL0_SLICE1_MISC__RENG_EXECUTE_ON_REG_UPDATE__SHIFT                                                  0x11
+#define PCTL0_SLICE1_MISC__RD_TIMER_ENABLE__SHIFT                                                             0x12
+#define PCTL0_SLICE1_MISC__RENG_EXECUTE_NONSECURE_START_PTR_MASK                                              0x000003FFL
+#define PCTL0_SLICE1_MISC__CRITICAL_REGS_LOCK_MASK                                                            0x00000400L
+#define PCTL0_SLICE1_MISC__TILE_IDLE_THRESHOLD_MASK                                                           0x00003800L
+#define PCTL0_SLICE1_MISC__RENG_MEM_LS_ENABLE_MASK                                                            0x00004000L
+#define PCTL0_SLICE1_MISC__STCTRL_FORCE_PGFSM_CMD_DONE_MASK                                                   0x00008000L
+#define PCTL0_SLICE1_MISC__DEEPSLEEP_DISCSDP_MASK                                                             0x00010000L
+#define PCTL0_SLICE1_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK                                                    0x00020000L
+#define PCTL0_SLICE1_MISC__RD_TIMER_ENABLE_MASK                                                               0x00040000L
+//PCTL0_SLICE2_MISC
+#define PCTL0_SLICE2_MISC__RENG_EXECUTE_NONSECURE_START_PTR__SHIFT                                            0x0
+#define PCTL0_SLICE2_MISC__CRITICAL_REGS_LOCK__SHIFT                                                          0xa
+#define PCTL0_SLICE2_MISC__TILE_IDLE_THRESHOLD__SHIFT                                                         0xb
+#define PCTL0_SLICE2_MISC__RENG_MEM_LS_ENABLE__SHIFT                                                          0xe
+#define PCTL0_SLICE2_MISC__STCTRL_FORCE_PGFSM_CMD_DONE__SHIFT                                                 0xf
+#define PCTL0_SLICE2_MISC__DEEPSLEEP_DISCSDP__SHIFT                                                           0x10
+#define PCTL0_SLICE2_MISC__RENG_EXECUTE_ON_REG_UPDATE__SHIFT                                                  0x11
+#define PCTL0_SLICE2_MISC__RD_TIMER_ENABLE__SHIFT                                                             0x12
+#define PCTL0_SLICE2_MISC__RENG_EXECUTE_NONSECURE_START_PTR_MASK                                              0x000003FFL
+#define PCTL0_SLICE2_MISC__CRITICAL_REGS_LOCK_MASK                                                            0x00000400L
+#define PCTL0_SLICE2_MISC__TILE_IDLE_THRESHOLD_MASK                                                           0x00003800L
+#define PCTL0_SLICE2_MISC__RENG_MEM_LS_ENABLE_MASK                                                            0x00004000L
+#define PCTL0_SLICE2_MISC__STCTRL_FORCE_PGFSM_CMD_DONE_MASK                                                   0x00008000L
+#define PCTL0_SLICE2_MISC__DEEPSLEEP_DISCSDP_MASK                                                             0x00010000L
+#define PCTL0_SLICE2_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK                                                    0x00020000L
+#define PCTL0_SLICE2_MISC__RD_TIMER_ENABLE_MASK                                                               0x00040000L
+//PCTL0_SLICE3_MISC
+#define PCTL0_SLICE3_MISC__RENG_EXECUTE_NONSECURE_START_PTR__SHIFT                                            0x0
+#define PCTL0_SLICE3_MISC__CRITICAL_REGS_LOCK__SHIFT                                                          0xa
+#define PCTL0_SLICE3_MISC__TILE_IDLE_THRESHOLD__SHIFT                                                         0xb
+#define PCTL0_SLICE3_MISC__RENG_MEM_LS_ENABLE__SHIFT                                                          0xe
+#define PCTL0_SLICE3_MISC__STCTRL_FORCE_PGFSM_CMD_DONE__SHIFT                                                 0xf
+#define PCTL0_SLICE3_MISC__DEEPSLEEP_DISCSDP__SHIFT                                                           0x10
+#define PCTL0_SLICE3_MISC__RENG_EXECUTE_ON_REG_UPDATE__SHIFT                                                  0x11
+#define PCTL0_SLICE3_MISC__RD_TIMER_ENABLE__SHIFT                                                             0x12
+#define PCTL0_SLICE3_MISC__RENG_EXECUTE_NONSECURE_START_PTR_MASK                                              0x000003FFL
+#define PCTL0_SLICE3_MISC__CRITICAL_REGS_LOCK_MASK                                                            0x00000400L
+#define PCTL0_SLICE3_MISC__TILE_IDLE_THRESHOLD_MASK                                                           0x00003800L
+#define PCTL0_SLICE3_MISC__RENG_MEM_LS_ENABLE_MASK                                                            0x00004000L
+#define PCTL0_SLICE3_MISC__STCTRL_FORCE_PGFSM_CMD_DONE_MASK                                                   0x00008000L
+#define PCTL0_SLICE3_MISC__DEEPSLEEP_DISCSDP_MASK                                                             0x00010000L
+#define PCTL0_SLICE3_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK                                                    0x00020000L
+#define PCTL0_SLICE3_MISC__RD_TIMER_ENABLE_MASK                                                               0x00040000L
+//PCTL0_SLICE4_MISC
+#define PCTL0_SLICE4_MISC__RENG_EXECUTE_NONSECURE_START_PTR__SHIFT                                            0x0
+#define PCTL0_SLICE4_MISC__CRITICAL_REGS_LOCK__SHIFT                                                          0xa
+#define PCTL0_SLICE4_MISC__TILE_IDLE_THRESHOLD__SHIFT                                                         0xb
+#define PCTL0_SLICE4_MISC__RENG_MEM_LS_ENABLE__SHIFT                                                          0xe
+#define PCTL0_SLICE4_MISC__STCTRL_FORCE_PGFSM_CMD_DONE__SHIFT                                                 0xf
+#define PCTL0_SLICE4_MISC__DEEPSLEEP_DISCSDP__SHIFT                                                           0x10
+#define PCTL0_SLICE4_MISC__RENG_EXECUTE_ON_REG_UPDATE__SHIFT                                                  0x11
+#define PCTL0_SLICE4_MISC__RD_TIMER_ENABLE__SHIFT                                                             0x12
+#define PCTL0_SLICE4_MISC__RENG_EXECUTE_NONSECURE_START_PTR_MASK                                              0x000003FFL
+#define PCTL0_SLICE4_MISC__CRITICAL_REGS_LOCK_MASK                                                            0x00000400L
+#define PCTL0_SLICE4_MISC__TILE_IDLE_THRESHOLD_MASK                                                           0x00003800L
+#define PCTL0_SLICE4_MISC__RENG_MEM_LS_ENABLE_MASK                                                            0x00004000L
+#define PCTL0_SLICE4_MISC__STCTRL_FORCE_PGFSM_CMD_DONE_MASK                                                   0x00008000L
+#define PCTL0_SLICE4_MISC__DEEPSLEEP_DISCSDP_MASK                                                             0x00010000L
+#define PCTL0_SLICE4_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK                                                    0x00020000L
+#define PCTL0_SLICE4_MISC__RD_TIMER_ENABLE_MASK                                                               0x00040000L
+
+
+// addressBlock: aid_mmhub_l1tlb_vml1dec
+//MC_VM_MX_L1_TLB0_STATUS
+#define MC_VM_MX_L1_TLB0_STATUS__BUSY__SHIFT                                                                  0x0
+#define MC_VM_MX_L1_TLB0_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                   0x1
+#define MC_VM_MX_L1_TLB0_STATUS__BUSY_MASK                                                                    0x00000001L
+#define MC_VM_MX_L1_TLB0_STATUS__FOUND_PARITY_ERRORS_MASK                                                     0x00000002L
+//MC_VM_MX_L1_TLB1_STATUS
+#define MC_VM_MX_L1_TLB1_STATUS__BUSY__SHIFT                                                                  0x0
+#define MC_VM_MX_L1_TLB1_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                   0x1
+#define MC_VM_MX_L1_TLB1_STATUS__BUSY_MASK                                                                    0x00000001L
+#define MC_VM_MX_L1_TLB1_STATUS__FOUND_PARITY_ERRORS_MASK                                                     0x00000002L
+//MC_VM_MX_L1_TLB2_STATUS
+#define MC_VM_MX_L1_TLB2_STATUS__BUSY__SHIFT                                                                  0x0
+#define MC_VM_MX_L1_TLB2_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                   0x1
+#define MC_VM_MX_L1_TLB2_STATUS__BUSY_MASK                                                                    0x00000001L
+#define MC_VM_MX_L1_TLB2_STATUS__FOUND_PARITY_ERRORS_MASK                                                     0x00000002L
+//MC_VM_MX_L1_TLB3_STATUS
+#define MC_VM_MX_L1_TLB3_STATUS__BUSY__SHIFT                                                                  0x0
+#define MC_VM_MX_L1_TLB3_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                   0x1
+#define MC_VM_MX_L1_TLB3_STATUS__BUSY_MASK                                                                    0x00000001L
+#define MC_VM_MX_L1_TLB3_STATUS__FOUND_PARITY_ERRORS_MASK                                                     0x00000002L
+//MC_VM_MX_L1_TLB4_STATUS
+#define MC_VM_MX_L1_TLB4_STATUS__BUSY__SHIFT                                                                  0x0
+#define MC_VM_MX_L1_TLB4_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                   0x1
+#define MC_VM_MX_L1_TLB4_STATUS__BUSY_MASK                                                                    0x00000001L
+#define MC_VM_MX_L1_TLB4_STATUS__FOUND_PARITY_ERRORS_MASK                                                     0x00000002L
+//MC_VM_MX_L1_TLB5_STATUS
+#define MC_VM_MX_L1_TLB5_STATUS__BUSY__SHIFT                                                                  0x0
+#define MC_VM_MX_L1_TLB5_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                   0x1
+#define MC_VM_MX_L1_TLB5_STATUS__BUSY_MASK                                                                    0x00000001L
+#define MC_VM_MX_L1_TLB5_STATUS__FOUND_PARITY_ERRORS_MASK                                                     0x00000002L
+//MC_VM_MX_L1_TLB6_STATUS
+#define MC_VM_MX_L1_TLB6_STATUS__BUSY__SHIFT                                                                  0x0
+#define MC_VM_MX_L1_TLB6_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                   0x1
+#define MC_VM_MX_L1_TLB6_STATUS__BUSY_MASK                                                                    0x00000001L
+#define MC_VM_MX_L1_TLB6_STATUS__FOUND_PARITY_ERRORS_MASK                                                     0x00000002L
+//MC_VM_MX_L1_TLB7_STATUS
+#define MC_VM_MX_L1_TLB7_STATUS__BUSY__SHIFT                                                                  0x0
+#define MC_VM_MX_L1_TLB7_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                   0x1
+#define MC_VM_MX_L1_TLB7_STATUS__BUSY_MASK                                                                    0x00000001L
+#define MC_VM_MX_L1_TLB7_STATUS__FOUND_PARITY_ERRORS_MASK                                                     0x00000002L
+
+
+// addressBlock: aid_mmhub_l1tlb_vml1pldec
+//MC_VM_MX_L1_PERFCOUNTER0_CFG
+#define MC_VM_MX_L1_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                         0x0
+#define MC_VM_MX_L1_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define MC_VM_MX_L1_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                        0x18
+#define MC_VM_MX_L1_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                           0x1c
+#define MC_VM_MX_L1_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                            0x1d
+#define MC_VM_MX_L1_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define MC_VM_MX_L1_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define MC_VM_MX_L1_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define MC_VM_MX_L1_PERFCOUNTER0_CFG__ENABLE_MASK                                                             0x10000000L
+#define MC_VM_MX_L1_PERFCOUNTER0_CFG__CLEAR_MASK                                                              0x20000000L
+//MC_VM_MX_L1_PERFCOUNTER1_CFG
+#define MC_VM_MX_L1_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                         0x0
+#define MC_VM_MX_L1_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define MC_VM_MX_L1_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                        0x18
+#define MC_VM_MX_L1_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                           0x1c
+#define MC_VM_MX_L1_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                            0x1d
+#define MC_VM_MX_L1_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define MC_VM_MX_L1_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define MC_VM_MX_L1_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define MC_VM_MX_L1_PERFCOUNTER1_CFG__ENABLE_MASK                                                             0x10000000L
+#define MC_VM_MX_L1_PERFCOUNTER1_CFG__CLEAR_MASK                                                              0x20000000L
+//MC_VM_MX_L1_PERFCOUNTER2_CFG
+#define MC_VM_MX_L1_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                         0x0
+#define MC_VM_MX_L1_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define MC_VM_MX_L1_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                        0x18
+#define MC_VM_MX_L1_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                           0x1c
+#define MC_VM_MX_L1_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                            0x1d
+#define MC_VM_MX_L1_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define MC_VM_MX_L1_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define MC_VM_MX_L1_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define MC_VM_MX_L1_PERFCOUNTER2_CFG__ENABLE_MASK                                                             0x10000000L
+#define MC_VM_MX_L1_PERFCOUNTER2_CFG__CLEAR_MASK                                                              0x20000000L
+//MC_VM_MX_L1_PERFCOUNTER3_CFG
+#define MC_VM_MX_L1_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                         0x0
+#define MC_VM_MX_L1_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define MC_VM_MX_L1_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                        0x18
+#define MC_VM_MX_L1_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                           0x1c
+#define MC_VM_MX_L1_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                            0x1d
+#define MC_VM_MX_L1_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define MC_VM_MX_L1_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define MC_VM_MX_L1_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define MC_VM_MX_L1_PERFCOUNTER3_CFG__ENABLE_MASK                                                             0x10000000L
+#define MC_VM_MX_L1_PERFCOUNTER3_CFG__CLEAR_MASK                                                              0x20000000L
+//MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                         0x0
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                               0x8
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                0x10
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                  0x18
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                   0x19
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                        0x1a
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                           0x0000000FL
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                 0x0000FF00L
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                  0x00FF0000L
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                    0x01000000L
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                     0x02000000L
+#define MC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                          0x04000000L
+
+
+// addressBlock: aid_mmhub_l1tlb_vml1prdec
+//MC_VM_MX_L1_PERFCOUNTER_LO
+#define MC_VM_MX_L1_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                         0x0
+#define MC_VM_MX_L1_PERFCOUNTER_LO__COUNTER_LO_MASK                                                           0xFFFFFFFFL
+//MC_VM_MX_L1_PERFCOUNTER_HI
+#define MC_VM_MX_L1_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                         0x0
+#define MC_VM_MX_L1_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                      0x10
+#define MC_VM_MX_L1_PERFCOUNTER_HI__COUNTER_HI_MASK                                                           0x0000FFFFL
+#define MC_VM_MX_L1_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                        0xFFFF0000L
+
+
+// addressBlock: aid_mmhub_utcl2_atcl2dec
+//ATC_L2_CNTL
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS__SHIFT                                               0x0
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS__SHIFT                                              0x3
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                                   0x6
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                                  0x7
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READ_REQUESTS__SHIFT                                          0x8
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITE_REQUESTS__SHIFT                                         0xb
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                              0xe
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                             0xf
+#define ATC_L2_CNTL__CACHE_INVALIDATE_MODE__SHIFT                                                             0x10
+#define ATC_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY__SHIFT                                          0x13
+#define ATC_L2_CNTL__FRAG_APT_INTXN_MODE__SHIFT                                                               0x14
+#define ATC_L2_CNTL__CLI_GPA_REQ_FRAG_SIZE__SHIFT                                                             0x16
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS_MASK                                                 0x00000003L
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS_MASK                                                0x00000018L
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                                     0x00000040L
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                                    0x00000080L
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READ_REQUESTS_MASK                                            0x00000300L
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITE_REQUESTS_MASK                                           0x00001800L
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                                0x00004000L
+#define ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                               0x00008000L
+#define ATC_L2_CNTL__CACHE_INVALIDATE_MODE_MASK                                                               0x00070000L
+#define ATC_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY_MASK                                            0x00080000L
+#define ATC_L2_CNTL__FRAG_APT_INTXN_MODE_MASK                                                                 0x00300000L
+#define ATC_L2_CNTL__CLI_GPA_REQ_FRAG_SIZE_MASK                                                               0x0FC00000L
+//ATC_L2_CNTL2
+#define ATC_L2_CNTL2__BANK_SELECT__SHIFT                                                                      0x0
+#define ATC_L2_CNTL2__NUM_BANKS_LOG2__SHIFT                                                                   0x6
+#define ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE__SHIFT                                                             0x9
+#define ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                              0xb
+#define ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS__SHIFT                                                     0xc
+#define ATC_L2_CNTL2__L2_CACHE_VMID_MODE__SHIFT                                                               0xf
+#define ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                         0x12
+#define ATC_L2_CNTL2__BANK_SELECT_MASK                                                                        0x0000003FL
+#define ATC_L2_CNTL2__NUM_BANKS_LOG2_MASK                                                                     0x000001C0L
+#define ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE_MASK                                                               0x00000600L
+#define ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE_MASK                                                0x00000800L
+#define ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS_MASK                                                       0x00007000L
+#define ATC_L2_CNTL2__L2_CACHE_VMID_MODE_MASK                                                                 0x00038000L
+#define ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                           0x00FC0000L
+//ATC_L2_CACHE_DATA0
+#define ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID__SHIFT                                                        0x0
+#define ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID__SHIFT                                                          0x1
+#define ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES__SHIFT                                                          0x2
+#define ATC_L2_CACHE_DATA0__VIRTUAL_PAGE_ADDRESS_HIGH__SHIFT                                                  0x17
+#define ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID_MASK                                                          0x00000001L
+#define ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID_MASK                                                            0x00000002L
+#define ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES_MASK                                                            0x007FFFFCL
+#define ATC_L2_CACHE_DATA0__VIRTUAL_PAGE_ADDRESS_HIGH_MASK                                                    0x07800000L
+//ATC_L2_CACHE_DATA1
+#define ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW__SHIFT                                                   0x0
+#define ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW_MASK                                                     0xFFFFFFFFL
+//ATC_L2_CACHE_DATA2
+#define ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS__SHIFT                                                      0x0
+#define ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS_MASK                                                        0xFFFFFFFFL
+//ATC_L2_CACHE_DATA3
+#define ATC_L2_CACHE_DATA3__PHYSICAL_PAGE_ADDRESS__SHIFT                                                      0x0
+#define ATC_L2_CACHE_DATA3__PHYSICAL_PAGE_ADDRESS_MASK                                                        0xFFFFFFFFL
+//ATC_L2_CNTL3
+#define ATC_L2_CNTL3__L2_SMALLK_FRAGMENT_SIZE__SHIFT                                                          0x0
+#define ATC_L2_CNTL3__L2_MIDK_FRAGMENT_SIZE__SHIFT                                                            0x6
+#define ATC_L2_CNTL3__L2_BIGK_FRAGMENT_SIZE__SHIFT                                                            0xc
+#define ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST__SHIFT                                                  0x12
+#define ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1__SHIFT                                                        0x15
+#define ATC_L2_CNTL3__COMPCLKREQ_OFF_HYSTERESIS__SHIFT                                                        0x1b
+#define ATC_L2_CNTL3__REPEATER_FGCG_OFF__SHIFT                                                                0x1e
+#define ATC_L2_CNTL3__L2_SMALLK_FRAGMENT_SIZE_MASK                                                            0x0000003FL
+#define ATC_L2_CNTL3__L2_MIDK_FRAGMENT_SIZE_MASK                                                              0x00000FC0L
+#define ATC_L2_CNTL3__L2_BIGK_FRAGMENT_SIZE_MASK                                                              0x0003F000L
+#define ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST_MASK                                                    0x001C0000L
+#define ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1_MASK                                                          0x07E00000L
+#define ATC_L2_CNTL3__COMPCLKREQ_OFF_HYSTERESIS_MASK                                                          0x38000000L
+#define ATC_L2_CNTL3__REPEATER_FGCG_OFF_MASK                                                                  0x40000000L
+//ATC_L2_STATUS
+#define ATC_L2_STATUS__BUSY__SHIFT                                                                            0x0
+#define ATC_L2_STATUS__NO_OUTSTANDING_AT_REQUESTS__SHIFT                                                      0x1
+#define ATC_L2_STATUS__BUSY_MASK                                                                              0x00000001L
+#define ATC_L2_STATUS__NO_OUTSTANDING_AT_REQUESTS_MASK                                                        0x00000002L
+//ATC_L2_STATUS2
+#define ATC_L2_STATUS2__UCE_MEM_ADDR__SHIFT                                                                   0x0
+#define ATC_L2_STATUS2__UCE_MEM_INST__SHIFT                                                                   0xc
+#define ATC_L2_STATUS2__UCE_SRT_CACHE__SHIFT                                                                  0x14
+#define ATC_L2_STATUS2__UCE__SHIFT                                                                            0x15
+#define ATC_L2_STATUS2__UCE_MEM_ADDR_MASK                                                                     0x00000FFFL
+#define ATC_L2_STATUS2__UCE_MEM_INST_MASK                                                                     0x000FF000L
+#define ATC_L2_STATUS2__UCE_SRT_CACHE_MASK                                                                    0x00100000L
+#define ATC_L2_STATUS2__UCE_MASK                                                                              0x00200000L
+//ATC_L2_MISC_CG
+#define ATC_L2_MISC_CG__OFFDLY__SHIFT                                                                         0x6
+#define ATC_L2_MISC_CG__ENABLE__SHIFT                                                                         0x12
+#define ATC_L2_MISC_CG__MEM_LS_ENABLE__SHIFT                                                                  0x13
+#define ATC_L2_MISC_CG__OFFDLY_MASK                                                                           0x00000FC0L
+#define ATC_L2_MISC_CG__ENABLE_MASK                                                                           0x00040000L
+#define ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK                                                                    0x00080000L
+//ATC_L2_MEM_POWER_LS
+#define ATC_L2_MEM_POWER_LS__LS_SETUP__SHIFT                                                                  0x0
+#define ATC_L2_MEM_POWER_LS__LS_HOLD__SHIFT                                                                   0x6
+#define ATC_L2_MEM_POWER_LS__LS_SETUP_MASK                                                                    0x0000003FL
+#define ATC_L2_MEM_POWER_LS__LS_HOLD_MASK                                                                     0x00000FC0L
+//ATC_L2_CGTT_CLK_CTRL
+#define ATC_L2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
+#define ATC_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
+#define ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                            0xf
+#define ATC_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE__SHIFT                                                      0x10
+#define ATC_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                            0x18
+#define ATC_L2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
+#define ATC_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
+#define ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK                                                              0x00008000L
+#define ATC_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_MASK                                                        0x00FF0000L
+#define ATC_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                              0xFF000000L
+//ATC_L2_CACHE_4K_DSM_INDEX
+#define ATC_L2_CACHE_4K_DSM_INDEX__INDEX__SHIFT                                                               0x0
+#define ATC_L2_CACHE_4K_DSM_INDEX__INDEX_MASK                                                                 0x000000FFL
+//ATC_L2_CACHE_32K_DSM_INDEX
+#define ATC_L2_CACHE_32K_DSM_INDEX__INDEX__SHIFT                                                              0x0
+#define ATC_L2_CACHE_32K_DSM_INDEX__INDEX_MASK                                                                0x000000FFL
+//ATC_L2_CACHE_2M_DSM_INDEX
+#define ATC_L2_CACHE_2M_DSM_INDEX__INDEX__SHIFT                                                               0x0
+#define ATC_L2_CACHE_2M_DSM_INDEX__INDEX_MASK                                                                 0x000000FFL
+//ATC_L2_CACHE_4K_DSM_CNTL
+#define ATC_L2_CACHE_4K_DSM_CNTL__INJECT_DELAY__SHIFT                                                         0x0
+#define ATC_L2_CACHE_4K_DSM_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                   0x6
+#define ATC_L2_CACHE_4K_DSM_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                  0x8
+#define ATC_L2_CACHE_4K_DSM_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                  0x9
+#define ATC_L2_CACHE_4K_DSM_CNTL__SELECT_INJECT_DELAY__SHIFT                                                  0xb
+#define ATC_L2_CACHE_4K_DSM_CNTL__WRITE_COUNTERS__SHIFT                                                       0xc
+#define ATC_L2_CACHE_4K_DSM_CNTL__SEC_COUNT__SHIFT                                                            0xd
+#define ATC_L2_CACHE_4K_DSM_CNTL__DED_COUNT__SHIFT                                                            0xf
+#define ATC_L2_CACHE_4K_DSM_CNTL__TEST_FUE__SHIFT                                                             0x11
+#define ATC_L2_CACHE_4K_DSM_CNTL__INJECT_DELAY_MASK                                                           0x0000003FL
+#define ATC_L2_CACHE_4K_DSM_CNTL__DSM_IRRITATOR_DATA_MASK                                                     0x000000C0L
+#define ATC_L2_CACHE_4K_DSM_CNTL__ENABLE_SINGLE_WRITE_MASK                                                    0x00000100L
+#define ATC_L2_CACHE_4K_DSM_CNTL__ENABLE_ERROR_INJECT_MASK                                                    0x00000600L
+#define ATC_L2_CACHE_4K_DSM_CNTL__SELECT_INJECT_DELAY_MASK                                                    0x00000800L
+#define ATC_L2_CACHE_4K_DSM_CNTL__WRITE_COUNTERS_MASK                                                         0x00001000L
+#define ATC_L2_CACHE_4K_DSM_CNTL__SEC_COUNT_MASK                                                              0x00006000L
+#define ATC_L2_CACHE_4K_DSM_CNTL__DED_COUNT_MASK                                                              0x00018000L
+#define ATC_L2_CACHE_4K_DSM_CNTL__TEST_FUE_MASK                                                               0x00020000L
+//ATC_L2_CACHE_32K_DSM_CNTL
+#define ATC_L2_CACHE_32K_DSM_CNTL__INJECT_DELAY__SHIFT                                                        0x0
+#define ATC_L2_CACHE_32K_DSM_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                  0x6
+#define ATC_L2_CACHE_32K_DSM_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                 0x8
+#define ATC_L2_CACHE_32K_DSM_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                 0x9
+#define ATC_L2_CACHE_32K_DSM_CNTL__SELECT_INJECT_DELAY__SHIFT                                                 0xb
+#define ATC_L2_CACHE_32K_DSM_CNTL__WRITE_COUNTERS__SHIFT                                                      0xc
+#define ATC_L2_CACHE_32K_DSM_CNTL__SEC_COUNT__SHIFT                                                           0xd
+#define ATC_L2_CACHE_32K_DSM_CNTL__DED_COUNT__SHIFT                                                           0xf
+#define ATC_L2_CACHE_32K_DSM_CNTL__TEST_FUE__SHIFT                                                            0x11
+#define ATC_L2_CACHE_32K_DSM_CNTL__INJECT_DELAY_MASK                                                          0x0000003FL
+#define ATC_L2_CACHE_32K_DSM_CNTL__DSM_IRRITATOR_DATA_MASK                                                    0x000000C0L
+#define ATC_L2_CACHE_32K_DSM_CNTL__ENABLE_SINGLE_WRITE_MASK                                                   0x00000100L
+#define ATC_L2_CACHE_32K_DSM_CNTL__ENABLE_ERROR_INJECT_MASK                                                   0x00000600L
+#define ATC_L2_CACHE_32K_DSM_CNTL__SELECT_INJECT_DELAY_MASK                                                   0x00000800L
+#define ATC_L2_CACHE_32K_DSM_CNTL__WRITE_COUNTERS_MASK                                                        0x00001000L
+#define ATC_L2_CACHE_32K_DSM_CNTL__SEC_COUNT_MASK                                                             0x00006000L
+#define ATC_L2_CACHE_32K_DSM_CNTL__DED_COUNT_MASK                                                             0x00018000L
+#define ATC_L2_CACHE_32K_DSM_CNTL__TEST_FUE_MASK                                                              0x00020000L
+//ATC_L2_CACHE_2M_DSM_CNTL
+#define ATC_L2_CACHE_2M_DSM_CNTL__INJECT_DELAY__SHIFT                                                         0x0
+#define ATC_L2_CACHE_2M_DSM_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                   0x6
+#define ATC_L2_CACHE_2M_DSM_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                  0x8
+#define ATC_L2_CACHE_2M_DSM_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                  0x9
+#define ATC_L2_CACHE_2M_DSM_CNTL__SELECT_INJECT_DELAY__SHIFT                                                  0xb
+#define ATC_L2_CACHE_2M_DSM_CNTL__WRITE_COUNTERS__SHIFT                                                       0xc
+#define ATC_L2_CACHE_2M_DSM_CNTL__SEC_COUNT__SHIFT                                                            0xd
+#define ATC_L2_CACHE_2M_DSM_CNTL__DED_COUNT__SHIFT                                                            0xf
+#define ATC_L2_CACHE_2M_DSM_CNTL__TEST_FUE__SHIFT                                                             0x11
+#define ATC_L2_CACHE_2M_DSM_CNTL__INJECT_DELAY_MASK                                                           0x0000003FL
+#define ATC_L2_CACHE_2M_DSM_CNTL__DSM_IRRITATOR_DATA_MASK                                                     0x000000C0L
+#define ATC_L2_CACHE_2M_DSM_CNTL__ENABLE_SINGLE_WRITE_MASK                                                    0x00000100L
+#define ATC_L2_CACHE_2M_DSM_CNTL__ENABLE_ERROR_INJECT_MASK                                                    0x00000600L
+#define ATC_L2_CACHE_2M_DSM_CNTL__SELECT_INJECT_DELAY_MASK                                                    0x00000800L
+#define ATC_L2_CACHE_2M_DSM_CNTL__WRITE_COUNTERS_MASK                                                         0x00001000L
+#define ATC_L2_CACHE_2M_DSM_CNTL__SEC_COUNT_MASK                                                              0x00006000L
+#define ATC_L2_CACHE_2M_DSM_CNTL__DED_COUNT_MASK                                                              0x00018000L
+#define ATC_L2_CACHE_2M_DSM_CNTL__TEST_FUE_MASK                                                               0x00020000L
+//ATC_L2_CNTL4
+#define ATC_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                          0x0
+#define ATC_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                         0xa
+#define ATC_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                            0x000003FFL
+#define ATC_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                           0x000FFC00L
+//ATC_L2_MM_GROUP_RT_CLASSES
+#define ATC_L2_MM_GROUP_RT_CLASSES__GROUP_RT_CLASS__SHIFT                                                     0x0
+#define ATC_L2_MM_GROUP_RT_CLASSES__GROUP_RT_CLASS_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: aid_mmhub_utcl2_vml2pfdec
+//VM_L2_CNTL
+#define VM_L2_CNTL__ENABLE_L2_CACHE__SHIFT                                                                    0x0
+#define VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING__SHIFT                                                      0x1
+#define VM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE__SHIFT                                                      0x2
+#define VM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE__SHIFT                                                      0x4
+#define VM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE__SHIFT                                                  0x8
+#define VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                            0x9
+#define VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                           0xa
+#define VM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY__SHIFT                                           0xb
+#define VM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE__SHIFT                                                           0xc
+#define VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT                                                            0xf
+#define VM_L2_CNTL__PDE_FAULT_CLASSIFICATION__SHIFT                                                           0x12
+#define VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT                                                      0x13
+#define VM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE__SHIFT                                                        0x15
+#define VM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE__SHIFT                                                             0x1a
+#define VM_L2_CNTL__ENABLE_L2_CACHE_MASK                                                                      0x00000001L
+#define VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK                                                        0x00000002L
+#define VM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE_MASK                                                        0x0000000CL
+#define VM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE_MASK                                                        0x00000030L
+#define VM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE_MASK                                                    0x00000100L
+#define VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK                                              0x00000200L
+#define VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK                                             0x00000400L
+#define VM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY_MASK                                             0x00000800L
+#define VM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE_MASK                                                             0x00007000L
+#define VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE_MASK                                                              0x00038000L
+#define VM_L2_CNTL__PDE_FAULT_CLASSIFICATION_MASK                                                             0x00040000L
+#define VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE_MASK                                                        0x00180000L
+#define VM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE_MASK                                                          0x03E00000L
+#define VM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE_MASK                                                               0x0C000000L
+//VM_L2_CNTL2
+#define VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS__SHIFT                                                            0x0
+#define VM_L2_CNTL2__INVALIDATE_L2_CACHE__SHIFT                                                               0x1
+#define VM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN__SHIFT                                                     0x15
+#define VM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION__SHIFT                                                   0x16
+#define VM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE__SHIFT                                                            0x17
+#define VM_L2_CNTL2__INVALIDATE_CACHE_MODE__SHIFT                                                             0x1a
+#define VM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE__SHIFT                                                          0x1c
+#define VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK                                                              0x00000001L
+#define VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK                                                                 0x00000002L
+#define VM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN_MASK                                                       0x00200000L
+#define VM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION_MASK                                                     0x00400000L
+#define VM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE_MASK                                                              0x03800000L
+#define VM_L2_CNTL2__INVALIDATE_CACHE_MODE_MASK                                                               0x0C000000L
+#define VM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE_MASK                                                            0x70000000L
+//VM_L2_CNTL3
+#define VM_L2_CNTL3__BANK_SELECT__SHIFT                                                                       0x0
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_MODE__SHIFT                                                              0x6
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                          0x8
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                                                       0xf
+#define VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY__SHIFT                                                       0x14
+#define VM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE__SHIFT                                                        0x15
+#define VM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE__SHIFT                                                      0x18
+#define VM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS__SHIFT                                                            0x1c
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS__SHIFT                                                          0x1d
+#define VM_L2_CNTL3__PDE_CACHE_FORCE_MISS__SHIFT                                                              0x1e
+#define VM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY__SHIFT                                                         0x1f
+#define VM_L2_CNTL3__BANK_SELECT_MASK                                                                         0x0000003FL
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_MODE_MASK                                                                0x000000C0L
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                            0x00001F00L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                                                         0x000F8000L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK                                                         0x00100000L
+#define VM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE_MASK                                                          0x00E00000L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE_MASK                                                        0x0F000000L
+#define VM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS_MASK                                                              0x10000000L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS_MASK                                                            0x20000000L
+#define VM_L2_CNTL3__PDE_CACHE_FORCE_MISS_MASK                                                                0x40000000L
+#define VM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY_MASK                                                           0x80000000L
+//VM_L2_STATUS
+#define VM_L2_STATUS__L2_BUSY__SHIFT                                                                          0x0
+#define VM_L2_STATUS__CONTEXT_DOMAIN_BUSY__SHIFT                                                              0x1
+#define VM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS__SHIFT                                                 0x11
+#define VM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS__SHIFT                                               0x12
+#define VM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS__SHIFT                                                   0x13
+#define VM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS__SHIFT                                                   0x14
+#define VM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS__SHIFT                                                   0x15
+#define VM_L2_STATUS__L2_BUSY_MASK                                                                            0x00000001L
+#define VM_L2_STATUS__CONTEXT_DOMAIN_BUSY_MASK                                                                0x0001FFFEL
+#define VM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS_MASK                                                   0x00020000L
+#define VM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS_MASK                                                 0x00040000L
+#define VM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS_MASK                                                     0x00080000L
+#define VM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS_MASK                                                     0x00100000L
+#define VM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS_MASK                                                     0x00200000L
+//VM_DUMMY_PAGE_FAULT_CNTL
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE__SHIFT                                              0x0
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL__SHIFT                                           0x1
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS__SHIFT                                              0x2
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE_MASK                                                0x00000001L
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL_MASK                                             0x00000002L
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS_MASK                                                0x000000FCL
+//VM_DUMMY_PAGE_FAULT_ADDR_LO32
+#define VM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32__SHIFT                                            0x0
+#define VM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32_MASK                                              0xFFFFFFFFL
+//VM_DUMMY_PAGE_FAULT_ADDR_HI32
+#define VM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI4__SHIFT                                             0x0
+#define VM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI4_MASK                                               0x0000000FL
+//VM_L2_PROTECTION_FAULT_CNTL
+#define VM_L2_PROTECTION_FAULT_CNTL__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                0x0
+#define VM_L2_PROTECTION_FAULT_CNTL__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES__SHIFT             0x1
+#define VM_L2_PROTECTION_FAULT_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                             0x2
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x3
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x4
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x5
+#define VM_L2_PROTECTION_FAULT_CNTL__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                 0x6
+#define VM_L2_PROTECTION_FAULT_CNTL__NACK_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x7
+#define VM_L2_PROTECTION_FAULT_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                        0x8
+#define VM_L2_PROTECTION_FAULT_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                             0x9
+#define VM_L2_PROTECTION_FAULT_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0xa
+#define VM_L2_PROTECTION_FAULT_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                             0xb
+#define VM_L2_PROTECTION_FAULT_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                           0xc
+#define VM_L2_PROTECTION_FAULT_CNTL__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                                0xd
+#define VM_L2_PROTECTION_FAULT_CNTL__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                          0x1d
+#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_NO_RETRY_FAULT__SHIFT                                           0x1e
+#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_RETRY_FAULT__SHIFT                                              0x1f
+#define VM_L2_PROTECTION_FAULT_CNTL__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                  0x00000001L
+#define VM_L2_PROTECTION_FAULT_CNTL__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES_MASK               0x00000002L
+#define VM_L2_PROTECTION_FAULT_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                               0x00000004L
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000008L
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000010L
+#define VM_L2_PROTECTION_FAULT_CNTL__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000020L
+#define VM_L2_PROTECTION_FAULT_CNTL__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                   0x00000040L
+#define VM_L2_PROTECTION_FAULT_CNTL__NACK_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000080L
+#define VM_L2_PROTECTION_FAULT_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                          0x00000100L
+#define VM_L2_PROTECTION_FAULT_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                               0x00000200L
+#define VM_L2_PROTECTION_FAULT_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000400L
+#define VM_L2_PROTECTION_FAULT_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                               0x00000800L
+#define VM_L2_PROTECTION_FAULT_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                             0x00001000L
+#define VM_L2_PROTECTION_FAULT_CNTL__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                                  0x1FFFE000L
+#define VM_L2_PROTECTION_FAULT_CNTL__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                            0x20000000L
+#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_NO_RETRY_FAULT_MASK                                             0x40000000L
+#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_RETRY_FAULT_MASK                                                0x80000000L
+//VM_L2_PROTECTION_FAULT_CNTL2
+#define VM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                                    0x0
+#define VM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                              0x10
+#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE__SHIFT                                        0x11
+#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY__SHIFT                             0x12
+#define VM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT__SHIFT                                     0x13
+#define VM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                                      0x0000FFFFL
+#define VM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                                0x00010000L
+#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_MASK                                          0x00020000L
+#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY_MASK                               0x00040000L
+#define VM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT_MASK                                       0x00080000L
+//VM_L2_PROTECTION_FAULT_MM_CNTL3
+#define VM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                  0x0
+#define VM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                    0xFFFFFFFFL
+//VM_L2_PROTECTION_FAULT_MM_CNTL4
+#define VM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                 0x0
+#define VM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                   0xFFFFFFFFL
+//VM_L2_PROTECTION_FAULT_STATUS
+#define VM_L2_PROTECTION_FAULT_STATUS__MORE_FAULTS__SHIFT                                                     0x0
+#define VM_L2_PROTECTION_FAULT_STATUS__WALKER_ERROR__SHIFT                                                    0x1
+#define VM_L2_PROTECTION_FAULT_STATUS__PERMISSION_FAULTS__SHIFT                                               0x4
+#define VM_L2_PROTECTION_FAULT_STATUS__MAPPING_ERROR__SHIFT                                                   0x8
+#define VM_L2_PROTECTION_FAULT_STATUS__CID__SHIFT                                                             0x9
+#define VM_L2_PROTECTION_FAULT_STATUS__RW__SHIFT                                                              0x12
+#define VM_L2_PROTECTION_FAULT_STATUS__ATOMIC__SHIFT                                                          0x13
+#define VM_L2_PROTECTION_FAULT_STATUS__VMID__SHIFT                                                            0x14
+#define VM_L2_PROTECTION_FAULT_STATUS__VF__SHIFT                                                              0x18
+#define VM_L2_PROTECTION_FAULT_STATUS__VFID__SHIFT                                                            0x19
+#define VM_L2_PROTECTION_FAULT_STATUS__UCE__SHIFT                                                             0x1d
+#define VM_L2_PROTECTION_FAULT_STATUS__FED__SHIFT                                                             0x1e
+#define VM_L2_PROTECTION_FAULT_STATUS__MORE_FAULTS_MASK                                                       0x00000001L
+#define VM_L2_PROTECTION_FAULT_STATUS__WALKER_ERROR_MASK                                                      0x0000000EL
+#define VM_L2_PROTECTION_FAULT_STATUS__PERMISSION_FAULTS_MASK                                                 0x000000F0L
+#define VM_L2_PROTECTION_FAULT_STATUS__MAPPING_ERROR_MASK                                                     0x00000100L
+#define VM_L2_PROTECTION_FAULT_STATUS__CID_MASK                                                               0x0003FE00L
+#define VM_L2_PROTECTION_FAULT_STATUS__RW_MASK                                                                0x00040000L
+#define VM_L2_PROTECTION_FAULT_STATUS__ATOMIC_MASK                                                            0x00080000L
+#define VM_L2_PROTECTION_FAULT_STATUS__VMID_MASK                                                              0x00F00000L
+#define VM_L2_PROTECTION_FAULT_STATUS__VF_MASK                                                                0x01000000L
+#define VM_L2_PROTECTION_FAULT_STATUS__VFID_MASK                                                              0x1E000000L
+#define VM_L2_PROTECTION_FAULT_STATUS__UCE_MASK                                                               0x20000000L
+#define VM_L2_PROTECTION_FAULT_STATUS__FED_MASK                                                               0x40000000L
+//VM_L2_PROTECTION_FAULT_ADDR_LO32
+#define VM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32__SHIFT                                       0x0
+#define VM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32_MASK                                         0xFFFFFFFFL
+//VM_L2_PROTECTION_FAULT_ADDR_HI32
+#define VM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI4__SHIFT                                        0x0
+#define VM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI4_MASK                                          0x0000000FL
+//VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32
+#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32__SHIFT                              0x0
+#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32_MASK                                0xFFFFFFFFL
+//VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32
+#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI4__SHIFT                               0x0
+#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI4_MASK                                 0x0000000FL
+//VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                       0x0
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                         0xFFFFFFFFL
+//VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                        0x0
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                          0x0000000FL
+//VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                      0x0
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                        0xFFFFFFFFL
+//VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                       0x0
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                         0x0000000FL
+//VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32
+#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32__SHIFT                         0x0
+#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32_MASK                           0xFFFFFFFFL
+//VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32
+#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI4__SHIFT                          0x0
+#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI4_MASK                            0x0000000FL
+//VM_L2_CNTL4
+#define VM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT__SHIFT                                                       0x0
+#define VM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL__SHIFT                                                      0x6
+#define VM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL__SHIFT                                                      0x7
+#define VM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                           0x8
+#define VM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                          0x12
+#define VM_L2_CNTL4__BPM_CGCGLS_OVERRIDE__SHIFT                                                               0x1c
+#define VM_L2_CNTL4__GC_CH_FGCG_OFF__SHIFT                                                                    0x1d
+#define VM_L2_CNTL4__VFIFO_HEAD_OF_QUEUE__SHIFT                                                               0x1e
+#define VM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT_MASK                                                         0x0000003FL
+#define VM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL_MASK                                                        0x00000040L
+#define VM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL_MASK                                                        0x00000080L
+#define VM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                             0x0003FF00L
+#define VM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                            0x0FFC0000L
+#define VM_L2_CNTL4__BPM_CGCGLS_OVERRIDE_MASK                                                                 0x10000000L
+#define VM_L2_CNTL4__GC_CH_FGCG_OFF_MASK                                                                      0x20000000L
+#define VM_L2_CNTL4__VFIFO_HEAD_OF_QUEUE_MASK                                                                 0x40000000L
+//VM_L2_CNTL5
+#define VM_L2_CNTL5__WALKER_FETCH_PDE_MTYPE_ENABLE__SHIFT                                                     0x0
+#define VM_L2_CNTL5__WALKER_FETCH_PDE_NOALLOC_ENABLE__SHIFT                                                   0x1
+#define VM_L2_CNTL5__WALKER_FETCH_PDE_MTYPE_ENABLE_MASK                                                       0x00000001L
+#define VM_L2_CNTL5__WALKER_FETCH_PDE_NOALLOC_ENABLE_MASK                                                     0x00000002L
+//VM_L2_MM_GROUP_RT_CLASSES
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS__SHIFT                                                    0x0
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS__SHIFT                                                    0x1
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS__SHIFT                                                    0x2
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS__SHIFT                                                    0x3
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS__SHIFT                                                    0x4
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS__SHIFT                                                    0x5
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS__SHIFT                                                    0x6
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS__SHIFT                                                    0x7
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS__SHIFT                                                    0x8
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS__SHIFT                                                    0x9
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS__SHIFT                                                   0xa
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS__SHIFT                                                   0xb
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS__SHIFT                                                   0xc
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS__SHIFT                                                   0xd
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS__SHIFT                                                   0xe
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS__SHIFT                                                   0xf
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS__SHIFT                                                   0x10
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS__SHIFT                                                   0x11
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS__SHIFT                                                   0x12
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS__SHIFT                                                   0x13
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS__SHIFT                                                   0x14
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS__SHIFT                                                   0x15
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS__SHIFT                                                   0x16
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS__SHIFT                                                   0x17
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS__SHIFT                                                   0x18
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS__SHIFT                                                   0x19
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS__SHIFT                                                   0x1a
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS__SHIFT                                                   0x1b
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS__SHIFT                                                   0x1c
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS__SHIFT                                                   0x1d
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS__SHIFT                                                   0x1e
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS__SHIFT                                                   0x1f
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS_MASK                                                      0x00000001L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS_MASK                                                      0x00000002L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS_MASK                                                      0x00000004L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS_MASK                                                      0x00000008L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS_MASK                                                      0x00000010L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS_MASK                                                      0x00000020L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS_MASK                                                      0x00000040L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS_MASK                                                      0x00000080L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS_MASK                                                      0x00000100L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS_MASK                                                      0x00000200L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS_MASK                                                     0x00000400L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS_MASK                                                     0x00000800L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS_MASK                                                     0x00001000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS_MASK                                                     0x00002000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS_MASK                                                     0x00004000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS_MASK                                                     0x00008000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS_MASK                                                     0x00010000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS_MASK                                                     0x00020000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS_MASK                                                     0x00040000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS_MASK                                                     0x00080000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS_MASK                                                     0x00100000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS_MASK                                                     0x00200000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS_MASK                                                     0x00400000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS_MASK                                                     0x00800000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS_MASK                                                     0x01000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS_MASK                                                     0x02000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS_MASK                                                     0x04000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS_MASK                                                     0x08000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS_MASK                                                     0x10000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS_MASK                                                     0x20000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS_MASK                                                     0x40000000L
+#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS_MASK                                                     0x80000000L
+//VM_L2_BANK_SELECT_RESERVED_CID
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID__SHIFT                                        0x0
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID__SHIFT                                       0xa
+#define VM_L2_BANK_SELECT_RESERVED_CID__ENABLE__SHIFT                                                         0x14
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                               0x18
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                            0x19
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID_MASK                                          0x000001FFL
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID_MASK                                         0x0007FC00L
+#define VM_L2_BANK_SELECT_RESERVED_CID__ENABLE_MASK                                                           0x00100000L
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE_MASK                                 0x01000000L
+#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                              0x02000000L
+//VM_L2_BANK_SELECT_RESERVED_CID2
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID__SHIFT                                       0x0
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID__SHIFT                                      0xa
+#define VM_L2_BANK_SELECT_RESERVED_CID2__ENABLE__SHIFT                                                        0x14
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                              0x18
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                           0x19
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID_MASK                                         0x000001FFL
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID_MASK                                        0x0007FC00L
+#define VM_L2_BANK_SELECT_RESERVED_CID2__ENABLE_MASK                                                          0x00100000L
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE_MASK                                0x01000000L
+#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                             0x02000000L
+//VM_L2_CACHE_PARITY_CNTL
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES__SHIFT                                 0x0
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES__SHIFT                               0x1
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES__SHIFT                                    0x2
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE__SHIFT                                 0x3
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE__SHIFT                               0x4
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE__SHIFT                                    0x5
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK__SHIFT                                                      0x6
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER__SHIFT                                                    0x9
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC__SHIFT                                                     0xc
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES_MASK                                   0x00000001L
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES_MASK                                 0x00000002L
+#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES_MASK                                      0x00000004L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE_MASK                                   0x00000008L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE_MASK                                 0x00000010L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE_MASK                                      0x00000020L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK_MASK                                                        0x000001C0L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER_MASK                                                      0x00000E00L
+#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC_MASK                                                       0x0000F000L
+//VM_L2_CGTT_CLK_CTRL
+#define VM_L2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define VM_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define VM_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                             0xf
+#define VM_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE__SHIFT                                                       0x10
+#define VM_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                             0x18
+#define VM_L2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define VM_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define VM_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK                                                               0x00008000L
+#define VM_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_MASK                                                         0x00FF0000L
+#define VM_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                               0xFF000000L
+//VM_L2_CGTT_BUSY_CTRL
+#define VM_L2_CGTT_BUSY_CTRL__READ_DELAY__SHIFT                                                               0x0
+#define VM_L2_CGTT_BUSY_CTRL__ALWAYS_BUSY__SHIFT                                                              0x4
+#define VM_L2_CGTT_BUSY_CTRL__READ_DELAY_MASK                                                                 0x0000000FL
+#define VM_L2_CGTT_BUSY_CTRL__ALWAYS_BUSY_MASK                                                                0x00000010L
+//VML2_MEM_ECC_INDEX
+#define VML2_MEM_ECC_INDEX__INDEX__SHIFT                                                                      0x0
+#define VML2_MEM_ECC_INDEX__INDEX_MASK                                                                        0x000000FFL
+//VML2_WALKER_MEM_ECC_INDEX
+#define VML2_WALKER_MEM_ECC_INDEX__INDEX__SHIFT                                                               0x0
+#define VML2_WALKER_MEM_ECC_INDEX__INDEX_MASK                                                                 0x000000FFL
+//UTCL2_MEM_ECC_INDEX
+#define UTCL2_MEM_ECC_INDEX__INDEX__SHIFT                                                                     0x0
+#define UTCL2_MEM_ECC_INDEX__INDEX_MASK                                                                       0x000000FFL
+//VML2_MEM_ECC_CNTL
+#define VML2_MEM_ECC_CNTL__INJECT_DELAY__SHIFT                                                                0x0
+#define VML2_MEM_ECC_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                          0x6
+#define VML2_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                         0x8
+#define VML2_MEM_ECC_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                         0x9
+#define VML2_MEM_ECC_CNTL__SELECT_INJECT_DELAY__SHIFT                                                         0xb
+#define VML2_MEM_ECC_CNTL__SEC_COUNT__SHIFT                                                                   0xc
+#define VML2_MEM_ECC_CNTL__DED_COUNT__SHIFT                                                                   0xe
+#define VML2_MEM_ECC_CNTL__WRITE_COUNTERS__SHIFT                                                              0x10
+#define VML2_MEM_ECC_CNTL__TEST_FUE__SHIFT                                                                    0x11
+#define VML2_MEM_ECC_CNTL__INJECT_DELAY_MASK                                                                  0x0000003FL
+#define VML2_MEM_ECC_CNTL__DSM_IRRITATOR_DATA_MASK                                                            0x000000C0L
+#define VML2_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE_MASK                                                           0x00000100L
+#define VML2_MEM_ECC_CNTL__ENABLE_ERROR_INJECT_MASK                                                           0x00000600L
+#define VML2_MEM_ECC_CNTL__SELECT_INJECT_DELAY_MASK                                                           0x00000800L
+#define VML2_MEM_ECC_CNTL__SEC_COUNT_MASK                                                                     0x00003000L
+#define VML2_MEM_ECC_CNTL__DED_COUNT_MASK                                                                     0x0000C000L
+#define VML2_MEM_ECC_CNTL__WRITE_COUNTERS_MASK                                                                0x00010000L
+#define VML2_MEM_ECC_CNTL__TEST_FUE_MASK                                                                      0x00020000L
+//VML2_WALKER_MEM_ECC_CNTL
+#define VML2_WALKER_MEM_ECC_CNTL__INJECT_DELAY__SHIFT                                                         0x0
+#define VML2_WALKER_MEM_ECC_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                   0x6
+#define VML2_WALKER_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                  0x8
+#define VML2_WALKER_MEM_ECC_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                  0x9
+#define VML2_WALKER_MEM_ECC_CNTL__SELECT_INJECT_DELAY__SHIFT                                                  0xb
+#define VML2_WALKER_MEM_ECC_CNTL__SEC_COUNT__SHIFT                                                            0xc
+#define VML2_WALKER_MEM_ECC_CNTL__DED_COUNT__SHIFT                                                            0xe
+#define VML2_WALKER_MEM_ECC_CNTL__WRITE_COUNTERS__SHIFT                                                       0x10
+#define VML2_WALKER_MEM_ECC_CNTL__TEST_FUE__SHIFT                                                             0x11
+#define VML2_WALKER_MEM_ECC_CNTL__INJECT_DELAY_MASK                                                           0x0000003FL
+#define VML2_WALKER_MEM_ECC_CNTL__DSM_IRRITATOR_DATA_MASK                                                     0x000000C0L
+#define VML2_WALKER_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE_MASK                                                    0x00000100L
+#define VML2_WALKER_MEM_ECC_CNTL__ENABLE_ERROR_INJECT_MASK                                                    0x00000600L
+#define VML2_WALKER_MEM_ECC_CNTL__SELECT_INJECT_DELAY_MASK                                                    0x00000800L
+#define VML2_WALKER_MEM_ECC_CNTL__SEC_COUNT_MASK                                                              0x00003000L
+#define VML2_WALKER_MEM_ECC_CNTL__DED_COUNT_MASK                                                              0x0000C000L
+#define VML2_WALKER_MEM_ECC_CNTL__WRITE_COUNTERS_MASK                                                         0x00010000L
+#define VML2_WALKER_MEM_ECC_CNTL__TEST_FUE_MASK                                                               0x00020000L
+//UTCL2_MEM_ECC_CNTL
+#define UTCL2_MEM_ECC_CNTL__INJECT_DELAY__SHIFT                                                               0x0
+#define UTCL2_MEM_ECC_CNTL__DSM_IRRITATOR_DATA__SHIFT                                                         0x6
+#define UTCL2_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE__SHIFT                                                        0x8
+#define UTCL2_MEM_ECC_CNTL__ENABLE_ERROR_INJECT__SHIFT                                                        0x9
+#define UTCL2_MEM_ECC_CNTL__SELECT_INJECT_DELAY__SHIFT                                                        0xb
+#define UTCL2_MEM_ECC_CNTL__SEC_COUNT__SHIFT                                                                  0xc
+#define UTCL2_MEM_ECC_CNTL__DED_COUNT__SHIFT                                                                  0xe
+#define UTCL2_MEM_ECC_CNTL__WRITE_COUNTERS__SHIFT                                                             0x10
+#define UTCL2_MEM_ECC_CNTL__TEST_FUE__SHIFT                                                                   0x11
+#define UTCL2_MEM_ECC_CNTL__INJECT_DELAY_MASK                                                                 0x0000003FL
+#define UTCL2_MEM_ECC_CNTL__DSM_IRRITATOR_DATA_MASK                                                           0x000000C0L
+#define UTCL2_MEM_ECC_CNTL__ENABLE_SINGLE_WRITE_MASK                                                          0x00000100L
+#define UTCL2_MEM_ECC_CNTL__ENABLE_ERROR_INJECT_MASK                                                          0x00000600L
+#define UTCL2_MEM_ECC_CNTL__SELECT_INJECT_DELAY_MASK                                                          0x00000800L
+#define UTCL2_MEM_ECC_CNTL__SEC_COUNT_MASK                                                                    0x00003000L
+#define UTCL2_MEM_ECC_CNTL__DED_COUNT_MASK                                                                    0x0000C000L
+#define UTCL2_MEM_ECC_CNTL__WRITE_COUNTERS_MASK                                                               0x00010000L
+#define UTCL2_MEM_ECC_CNTL__TEST_FUE_MASK                                                                     0x00020000L
+//VML2_MEM_ECC_STATUS
+#define VML2_MEM_ECC_STATUS__UCE__SHIFT                                                                       0x0
+#define VML2_MEM_ECC_STATUS__FED__SHIFT                                                                       0x1
+#define VML2_MEM_ECC_STATUS__UCE_MASK                                                                         0x00000001L
+#define VML2_MEM_ECC_STATUS__FED_MASK                                                                         0x00000002L
+//VML2_WALKER_MEM_ECC_STATUS
+#define VML2_WALKER_MEM_ECC_STATUS__UCE__SHIFT                                                                0x0
+#define VML2_WALKER_MEM_ECC_STATUS__FED__SHIFT                                                                0x1
+#define VML2_WALKER_MEM_ECC_STATUS__UCE_MASK                                                                  0x00000001L
+#define VML2_WALKER_MEM_ECC_STATUS__FED_MASK                                                                  0x00000002L
+//UTCL2_MEM_ECC_STATUS
+#define UTCL2_MEM_ECC_STATUS__UCE__SHIFT                                                                      0x0
+#define UTCL2_MEM_ECC_STATUS__FED__SHIFT                                                                      0x1
+#define UTCL2_MEM_ECC_STATUS__UCE_MASK                                                                        0x00000001L
+#define UTCL2_MEM_ECC_STATUS__FED_MASK                                                                        0x00000002L
+//UTCL2_EDC_MODE
+#define UTCL2_EDC_MODE__FORCE_SEC_ON_DED__SHIFT                                                               0xf
+#define UTCL2_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                  0x10
+#define UTCL2_EDC_MODE__GATE_FUE__SHIFT                                                                       0x11
+#define UTCL2_EDC_MODE__DED_MODE__SHIFT                                                                       0x14
+#define UTCL2_EDC_MODE__PROP_FED__SHIFT                                                                       0x1d
+#define UTCL2_EDC_MODE__BYPASS__SHIFT                                                                         0x1f
+#define UTCL2_EDC_MODE__FORCE_SEC_ON_DED_MASK                                                                 0x00008000L
+#define UTCL2_EDC_MODE__COUNT_FED_OUT_MASK                                                                    0x00010000L
+#define UTCL2_EDC_MODE__GATE_FUE_MASK                                                                         0x00020000L
+#define UTCL2_EDC_MODE__DED_MODE_MASK                                                                         0x00300000L
+#define UTCL2_EDC_MODE__PROP_FED_MASK                                                                         0x20000000L
+#define UTCL2_EDC_MODE__BYPASS_MASK                                                                           0x80000000L
+//UTCL2_EDC_CONFIG
+#define UTCL2_EDC_CONFIG__WRITE_DIS__SHIFT                                                                    0x0
+#define UTCL2_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
+#define UTCL2_EDC_CONFIG__WRITE_DIS_MASK                                                                      0x00000001L
+#define UTCL2_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
+
+
+// addressBlock: aid_mmhub_utcl2_vml2vcdec
+//VM_CONTEXT0_CNTL
+#define VM_CONTEXT0_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT0_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT0_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT1_CNTL
+#define VM_CONTEXT1_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT1_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT1_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT2_CNTL
+#define VM_CONTEXT2_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT2_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT2_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT2_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT3_CNTL
+#define VM_CONTEXT3_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT3_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT3_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT3_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT4_CNTL
+#define VM_CONTEXT4_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT4_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT4_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT4_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT5_CNTL
+#define VM_CONTEXT5_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT5_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT5_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT5_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT6_CNTL
+#define VM_CONTEXT6_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT6_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT6_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT6_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT7_CNTL
+#define VM_CONTEXT7_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT7_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT7_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT7_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT8_CNTL
+#define VM_CONTEXT8_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT8_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT8_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT8_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT9_CNTL
+#define VM_CONTEXT9_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
+#define VM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
+#define VM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
+#define VM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
+#define VM_CONTEXT9_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
+#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
+#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
+#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
+#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
+#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
+#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
+#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
+#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
+#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
+#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
+#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
+#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
+#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
+#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
+#define VM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x17
+#define VM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x18
+#define VM_CONTEXT9_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
+#define VM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
+#define VM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
+#define VM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
+#define VM_CONTEXT9_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
+#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
+#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
+#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
+#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
+#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
+#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
+#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
+#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
+#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
+#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
+#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
+#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
+#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
+#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
+#define VM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00800000L
+#define VM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x01000000L
+//VM_CONTEXT10_CNTL
+#define VM_CONTEXT10_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT10_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT10_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT10_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXT11_CNTL
+#define VM_CONTEXT11_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT11_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT11_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT11_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXT12_CNTL
+#define VM_CONTEXT12_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT12_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT12_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT12_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXT13_CNTL
+#define VM_CONTEXT13_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT13_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT13_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT13_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXT14_CNTL
+#define VM_CONTEXT14_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT14_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT14_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT14_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXT15_CNTL
+#define VM_CONTEXT15_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
+#define VM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
+#define VM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
+#define VM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
+#define VM_CONTEXT15_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
+#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
+#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
+#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
+#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
+#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
+#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
+#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
+#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
+#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
+#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
+#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
+#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
+#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
+#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
+#define VM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x17
+#define VM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x18
+#define VM_CONTEXT15_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
+#define VM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
+#define VM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
+#define VM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
+#define VM_CONTEXT15_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
+#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
+#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
+#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
+#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
+#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
+#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
+#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
+#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
+#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
+#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
+#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
+#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
+#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
+#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
+#define VM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00800000L
+#define VM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x01000000L
+//VM_CONTEXTS_DISABLE
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0__SHIFT                                                         0x0
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1__SHIFT                                                         0x1
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2__SHIFT                                                         0x2
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3__SHIFT                                                         0x3
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4__SHIFT                                                         0x4
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5__SHIFT                                                         0x5
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6__SHIFT                                                         0x6
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7__SHIFT                                                         0x7
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8__SHIFT                                                         0x8
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9__SHIFT                                                         0x9
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10__SHIFT                                                        0xa
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11__SHIFT                                                        0xb
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12__SHIFT                                                        0xc
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13__SHIFT                                                        0xd
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14__SHIFT                                                        0xe
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15__SHIFT                                                        0xf
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0_MASK                                                           0x00000001L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1_MASK                                                           0x00000002L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2_MASK                                                           0x00000004L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3_MASK                                                           0x00000008L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4_MASK                                                           0x00000010L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5_MASK                                                           0x00000020L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6_MASK                                                           0x00000040L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7_MASK                                                           0x00000080L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8_MASK                                                           0x00000100L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9_MASK                                                           0x00000200L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10_MASK                                                          0x00000400L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11_MASK                                                          0x00000800L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12_MASK                                                          0x00001000L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13_MASK                                                          0x00002000L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14_MASK                                                          0x00004000L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15_MASK                                                          0x00008000L
+//VM_INVALIDATE_ENG0_SEM
+#define VM_INVALIDATE_ENG0_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG0_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG1_SEM
+#define VM_INVALIDATE_ENG1_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG1_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG2_SEM
+#define VM_INVALIDATE_ENG2_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG2_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG3_SEM
+#define VM_INVALIDATE_ENG3_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG3_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG4_SEM
+#define VM_INVALIDATE_ENG4_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG4_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG5_SEM
+#define VM_INVALIDATE_ENG5_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG5_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG6_SEM
+#define VM_INVALIDATE_ENG6_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG6_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG7_SEM
+#define VM_INVALIDATE_ENG7_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG7_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG8_SEM
+#define VM_INVALIDATE_ENG8_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG8_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG9_SEM
+#define VM_INVALIDATE_ENG9_SEM__SEMAPHORE__SHIFT                                                              0x0
+#define VM_INVALIDATE_ENG9_SEM__SEMAPHORE_MASK                                                                0x00000001L
+//VM_INVALIDATE_ENG10_SEM
+#define VM_INVALIDATE_ENG10_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG10_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG11_SEM
+#define VM_INVALIDATE_ENG11_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG11_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG12_SEM
+#define VM_INVALIDATE_ENG12_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG12_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG13_SEM
+#define VM_INVALIDATE_ENG13_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG13_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG14_SEM
+#define VM_INVALIDATE_ENG14_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG14_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG15_SEM
+#define VM_INVALIDATE_ENG15_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG15_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG16_SEM
+#define VM_INVALIDATE_ENG16_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG16_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG17_SEM
+#define VM_INVALIDATE_ENG17_SEM__SEMAPHORE__SHIFT                                                             0x0
+#define VM_INVALIDATE_ENG17_SEM__SEMAPHORE_MASK                                                               0x00000001L
+//VM_INVALIDATE_ENG0_REQ
+#define VM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG0_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG0_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG0_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG0_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG1_REQ
+#define VM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG1_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG1_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG1_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG1_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG2_REQ
+#define VM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG2_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG2_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG2_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG2_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG3_REQ
+#define VM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG3_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG3_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG3_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG3_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG4_REQ
+#define VM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG4_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG4_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG4_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG4_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG5_REQ
+#define VM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG5_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG5_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG5_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG5_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG6_REQ
+#define VM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG6_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG6_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG6_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG6_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG7_REQ
+#define VM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG7_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG7_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG7_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG7_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG8_REQ
+#define VM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG8_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG8_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG8_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG8_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG9_REQ
+#define VM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG9_REQ__FLUSH_TYPE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
+#define VM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
+#define VM_INVALIDATE_ENG9_REQ__LOG_REQUEST__SHIFT                                                            0x18
+#define VM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG9_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
+#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
+#define VM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
+#define VM_INVALIDATE_ENG9_REQ__LOG_REQUEST_MASK                                                              0x01000000L
+//VM_INVALIDATE_ENG10_REQ
+#define VM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG10_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG10_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG10_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG10_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG11_REQ
+#define VM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG11_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG11_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG11_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG11_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG12_REQ
+#define VM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG12_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG12_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG12_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG12_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG13_REQ
+#define VM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG13_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG13_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG13_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG13_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG14_REQ
+#define VM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG14_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG14_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG14_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG14_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG15_REQ
+#define VM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG15_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG15_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG15_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG15_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG16_REQ
+#define VM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG16_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG16_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG16_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG16_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG17_REQ
+#define VM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG17_REQ__FLUSH_TYPE__SHIFT                                                            0x10
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
+#define VM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
+#define VM_INVALIDATE_ENG17_REQ__LOG_REQUEST__SHIFT                                                           0x18
+#define VM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG17_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
+#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
+#define VM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
+#define VM_INVALIDATE_ENG17_REQ__LOG_REQUEST_MASK                                                             0x01000000L
+//VM_INVALIDATE_ENG0_ACK
+#define VM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG0_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG0_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG1_ACK
+#define VM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG1_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG1_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG2_ACK
+#define VM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG2_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG2_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG3_ACK
+#define VM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG3_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG3_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG4_ACK
+#define VM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG4_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG4_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG5_ACK
+#define VM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG5_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG5_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG6_ACK
+#define VM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG6_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG6_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG7_ACK
+#define VM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG7_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG7_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG8_ACK
+#define VM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG8_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG8_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG9_ACK
+#define VM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
+#define VM_INVALIDATE_ENG9_ACK__SEMAPHORE__SHIFT                                                              0x10
+#define VM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
+#define VM_INVALIDATE_ENG9_ACK__SEMAPHORE_MASK                                                                0x00010000L
+//VM_INVALIDATE_ENG10_ACK
+#define VM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG10_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG10_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG11_ACK
+#define VM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG11_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG11_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG12_ACK
+#define VM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG12_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG12_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG13_ACK
+#define VM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG13_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG13_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG14_ACK
+#define VM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG14_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG14_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG15_ACK
+#define VM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG15_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG15_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG16_ACK
+#define VM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG16_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG16_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG17_ACK
+#define VM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
+#define VM_INVALIDATE_ENG17_ACK__SEMAPHORE__SHIFT                                                             0x10
+#define VM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
+#define VM_INVALIDATE_ENG17_ACK__SEMAPHORE_MASK                                                               0x00010000L
+//VM_INVALIDATE_ENG0_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG0_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG1_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG1_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG2_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG2_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG3_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG3_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG4_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG4_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG5_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG5_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG6_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG6_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG7_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG7_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG8_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG8_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG9_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
+//VM_INVALIDATE_ENG9_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
+#define VM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
+//VM_INVALIDATE_ENG10_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG10_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG11_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG11_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG12_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG12_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG13_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG13_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG14_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG14_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG15_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG15_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG16_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG16_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_INVALIDATE_ENG17_ADDR_RANGE_LO32
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
+//VM_INVALIDATE_ENG17_ADDR_RANGE_HI32
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
+#define VM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
+//VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32
+#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32
+#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
+#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
+//VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
+#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
+//VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+//VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+//VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+//VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+//VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+//VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32
+#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
+#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
+//VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32
+#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
+#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
+
+
+// addressBlock: aid_mmhub_utcl2_vmsharedpfdec
+//MC_VM_NB_MMIOBASE
+#define MC_VM_NB_MMIOBASE__MMIOBASE__SHIFT                                                                    0x0
+#define MC_VM_NB_MMIOBASE__MMIOBASE_MASK                                                                      0xFFFFFFFFL
+//MC_VM_NB_MMIOLIMIT
+#define MC_VM_NB_MMIOLIMIT__MMIOLIMIT__SHIFT                                                                  0x0
+#define MC_VM_NB_MMIOLIMIT__MMIOLIMIT_MASK                                                                    0xFFFFFFFFL
+//MC_VM_NB_PCI_CTRL
+#define MC_VM_NB_PCI_CTRL__MMIOENABLE__SHIFT                                                                  0x17
+#define MC_VM_NB_PCI_CTRL__MMIOENABLE_MASK                                                                    0x00800000L
+//MC_VM_NB_PCI_ARB
+#define MC_VM_NB_PCI_ARB__VGA_HOLE__SHIFT                                                                     0x3
+#define MC_VM_NB_PCI_ARB__VGA_HOLE_MASK                                                                       0x00000008L
+//MC_VM_NB_TOP_OF_DRAM_SLOT1
+#define MC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM__SHIFT                                                        0x17
+#define MC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM_MASK                                                          0xFF800000L
+//MC_VM_NB_LOWER_TOP_OF_DRAM2
+#define MC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE__SHIFT                                                            0x0
+#define MC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2__SHIFT                                                        0x17
+#define MC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE_MASK                                                              0x00000001L
+#define MC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2_MASK                                                          0xFF800000L
+//MC_VM_NB_UPPER_TOP_OF_DRAM2
+#define MC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2__SHIFT                                                        0x0
+#define MC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2_MASK                                                          0x0000FFFFL
+//MC_VM_FB_OFFSET
+#define MC_VM_FB_OFFSET__FB_OFFSET__SHIFT                                                                     0x0
+#define MC_VM_FB_OFFSET__FB_OFFSET_MASK                                                                       0x00FFFFFFL
+//MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB__SHIFT                               0x0
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB_MASK                                 0xFFFFFFFFL
+//MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB__SHIFT                               0x0
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB_MASK                                 0x0000000FL
+//MC_VM_STEERING
+#define MC_VM_STEERING__DEFAULT_STEERING__SHIFT                                                               0x0
+#define MC_VM_STEERING__DEFAULT_STEERING_MASK                                                                 0x00000003L
+//MC_SHARED_VIRT_RESET_REQ
+#define MC_SHARED_VIRT_RESET_REQ__VF__SHIFT                                                                   0x0
+#define MC_SHARED_VIRT_RESET_REQ__PF__SHIFT                                                                   0x1f
+#define MC_SHARED_VIRT_RESET_REQ__VF_MASK                                                                     0x0000FFFFL
+#define MC_SHARED_VIRT_RESET_REQ__PF_MASK                                                                     0x80000000L
+//MC_MEM_POWER_LS
+#define MC_MEM_POWER_LS__LS_SETUP__SHIFT                                                                      0x0
+#define MC_MEM_POWER_LS__LS_HOLD__SHIFT                                                                       0x6
+#define MC_MEM_POWER_LS__LS_SETUP_MASK                                                                        0x0000003FL
+#define MC_MEM_POWER_LS__LS_HOLD_MASK                                                                         0x00000FC0L
+//MC_VM_CACHEABLE_DRAM_ADDRESS_START
+#define MC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS__SHIFT                                                    0x0
+#define MC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS_MASK                                                      0x00FFFFFFL
+//MC_VM_CACHEABLE_DRAM_ADDRESS_END
+#define MC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS__SHIFT                                                      0x0
+#define MC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS_MASK                                                        0x00FFFFFFL
+//MC_VM_APT_CNTL
+#define MC_VM_APT_CNTL__FORCE_MTYPE_UC__SHIFT                                                                 0x0
+#define MC_VM_APT_CNTL__DIRECT_SYSTEM_EN__SHIFT                                                               0x1
+#define MC_VM_APT_CNTL__CHECK_IS_LOCAL__SHIFT                                                                 0x2
+#define MC_VM_APT_CNTL__PERMS_GRANTED__SHIFT                                                                  0x3
+#define MC_VM_APT_CNTL__LOCAL_SYSMEM_APERTURE_CNTL__SHIFT                                                     0x4
+#define MC_VM_APT_CNTL__FORCE_MTYPE_UC_MASK                                                                   0x00000001L
+#define MC_VM_APT_CNTL__DIRECT_SYSTEM_EN_MASK                                                                 0x00000002L
+#define MC_VM_APT_CNTL__CHECK_IS_LOCAL_MASK                                                                   0x00000004L
+#define MC_VM_APT_CNTL__PERMS_GRANTED_MASK                                                                    0x00000008L
+#define MC_VM_APT_CNTL__LOCAL_SYSMEM_APERTURE_CNTL_MASK                                                       0x00000030L
+//MC_VM_LOCAL_HBM_ADDRESS_START
+#define MC_VM_LOCAL_HBM_ADDRESS_START__ADDRESS__SHIFT                                                         0x0
+#define MC_VM_LOCAL_HBM_ADDRESS_START__ADDRESS_MASK                                                           0x00FFFFFFL
+//MC_VM_LOCAL_HBM_ADDRESS_END
+#define MC_VM_LOCAL_HBM_ADDRESS_END__ADDRESS__SHIFT                                                           0x0
+#define MC_VM_LOCAL_HBM_ADDRESS_END__ADDRESS_MASK                                                             0x00FFFFFFL
+//MC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL
+#define MC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL__LOCK__SHIFT                                                        0x0
+#define MC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL__LOCK_MASK                                                          0x00000001L
+//UTCL2_CGTT_CLK_CTRL
+#define UTCL2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define UTCL2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE_EXTRA__SHIFT                                                       0xc
+#define UTCL2_CGTT_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                             0xf
+#define UTCL2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE__SHIFT                                                       0x10
+#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                             0x18
+#define UTCL2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define UTCL2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE_EXTRA_MASK                                                         0x00007000L
+#define UTCL2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK                                                               0x00008000L
+#define UTCL2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_MASK                                                         0x00FF0000L
+#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                               0xFF000000L
+//MC_VM_XGMI_LFB_CNTL
+#define MC_VM_XGMI_LFB_CNTL__PF_LFB_REGION__SHIFT                                                             0x0
+#define MC_VM_XGMI_LFB_CNTL__PF_MAX_REGION__SHIFT                                                             0x4
+#define MC_VM_XGMI_LFB_CNTL__PF_LFB_REGION_MASK                                                               0x0000000FL
+#define MC_VM_XGMI_LFB_CNTL__PF_MAX_REGION_MASK                                                               0x000000F0L
+//MC_VM_XGMI_LFB_SIZE
+#define MC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE__SHIFT                                                               0x0
+#define MC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE_MASK                                                                 0x0001FFFFL
+//MC_VM_CACHEABLE_DRAM_CNTL
+#define MC_VM_CACHEABLE_DRAM_CNTL__ENABLE_CACHEABLE_DRAM_ADDRESS_APERTURE__SHIFT                              0x0
+#define MC_VM_CACHEABLE_DRAM_CNTL__ENABLE_CACHEABLE_DRAM_ADDRESS_APERTURE_MASK                                0x00000001L
+//MC_VM_HOST_MAPPING
+#define MC_VM_HOST_MAPPING__MODE__SHIFT                                                                       0x0
+#define MC_VM_HOST_MAPPING__MODE_MASK                                                                         0x00000001L
+
+
+// addressBlock: aid_mmhub_utcl2_vmsharedvcdec
+//MC_VM_FB_LOCATION_BASE
+#define MC_VM_FB_LOCATION_BASE__FB_BASE__SHIFT                                                                0x0
+#define MC_VM_FB_LOCATION_BASE__FB_BASE_MASK                                                                  0x00FFFFFFL
+//MC_VM_FB_LOCATION_TOP
+#define MC_VM_FB_LOCATION_TOP__FB_TOP__SHIFT                                                                  0x0
+#define MC_VM_FB_LOCATION_TOP__FB_TOP_MASK                                                                    0x00FFFFFFL
+//MC_VM_AGP_TOP
+#define MC_VM_AGP_TOP__AGP_TOP__SHIFT                                                                         0x0
+#define MC_VM_AGP_TOP__AGP_TOP_MASK                                                                           0x00FFFFFFL
+//MC_VM_AGP_BOT
+#define MC_VM_AGP_BOT__AGP_BOT__SHIFT                                                                         0x0
+#define MC_VM_AGP_BOT__AGP_BOT_MASK                                                                           0x00FFFFFFL
+//MC_VM_AGP_BASE
+#define MC_VM_AGP_BASE__AGP_BASE__SHIFT                                                                       0x0
+#define MC_VM_AGP_BASE__AGP_BASE_MASK                                                                         0x00FFFFFFL
+//MC_VM_SYSTEM_APERTURE_LOW_ADDR
+#define MC_VM_SYSTEM_APERTURE_LOW_ADDR__LOGICAL_ADDR__SHIFT                                                   0x0
+#define MC_VM_SYSTEM_APERTURE_LOW_ADDR__LOGICAL_ADDR_MASK                                                     0x3FFFFFFFL
+//MC_VM_SYSTEM_APERTURE_HIGH_ADDR
+#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR__LOGICAL_ADDR__SHIFT                                                  0x0
+#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR__LOGICAL_ADDR_MASK                                                    0x3FFFFFFFL
+//MC_VM_MX_L1_TLB_CNTL
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT                                                            0x0
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT                                                       0x3
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT                                          0x5
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT                                             0x6
+#define MC_VM_MX_L1_TLB_CNTL__ECO_BITS__SHIFT                                                                 0x7
+#define MC_VM_MX_L1_TLB_CNTL__MTYPE__SHIFT                                                                    0xb
+#define MC_VM_MX_L1_TLB_CNTL__ATC_EN__SHIFT                                                                   0xd
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK                                                              0x00000001L
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK                                                         0x00000018L
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK                                            0x00000020L
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK                                               0x00000040L
+#define MC_VM_MX_L1_TLB_CNTL__ECO_BITS_MASK                                                                   0x00000780L
+#define MC_VM_MX_L1_TLB_CNTL__MTYPE_MASK                                                                      0x00001800L
+#define MC_VM_MX_L1_TLB_CNTL__ATC_EN_MASK                                                                     0x00002000L
+
+
+// addressBlock: aid_mmhub_utcl2_vmsharedhvdec
+//MC_VM_FB_SIZE_OFFSET_VF0
+#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF1
+#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF2
+#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF3
+#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF4
+#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF5
+#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF6
+#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF7
+#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF8
+#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF9
+#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_SIZE__SHIFT                                                           0x0
+#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_OFFSET__SHIFT                                                         0x10
+#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_SIZE_MASK                                                             0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF10
+#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF11
+#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF12
+#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF13
+#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF14
+#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//MC_VM_FB_SIZE_OFFSET_VF15
+#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_SIZE__SHIFT                                                          0x0
+#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_OFFSET__SHIFT                                                        0x10
+#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_SIZE_MASK                                                            0x0000FFFFL
+#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
+//VM_IOMMU_MMIO_CNTRL_1
+#define VM_IOMMU_MMIO_CNTRL_1__MARC_EN__SHIFT                                                                 0x8
+#define VM_IOMMU_MMIO_CNTRL_1__MARC_EN_MASK                                                                   0x00000100L
+//MC_VM_MARC_BASE_LO_0
+#define MC_VM_MARC_BASE_LO_0__MARC_BASE_LO_0__SHIFT                                                           0xc
+#define MC_VM_MARC_BASE_LO_0__MARC_BASE_LO_0_MASK                                                             0xFFFFF000L
+//MC_VM_MARC_BASE_LO_1
+#define MC_VM_MARC_BASE_LO_1__MARC_BASE_LO_1__SHIFT                                                           0xc
+#define MC_VM_MARC_BASE_LO_1__MARC_BASE_LO_1_MASK                                                             0xFFFFF000L
+//MC_VM_MARC_BASE_LO_2
+#define MC_VM_MARC_BASE_LO_2__MARC_BASE_LO_2__SHIFT                                                           0xc
+#define MC_VM_MARC_BASE_LO_2__MARC_BASE_LO_2_MASK                                                             0xFFFFF000L
+//MC_VM_MARC_BASE_LO_3
+#define MC_VM_MARC_BASE_LO_3__MARC_BASE_LO_3__SHIFT                                                           0xc
+#define MC_VM_MARC_BASE_LO_3__MARC_BASE_LO_3_MASK                                                             0xFFFFF000L
+//MC_VM_MARC_BASE_HI_0
+#define MC_VM_MARC_BASE_HI_0__MARC_BASE_HI_0__SHIFT                                                           0x0
+#define MC_VM_MARC_BASE_HI_0__MARC_BASE_HI_0_MASK                                                             0x000FFFFFL
+//MC_VM_MARC_BASE_HI_1
+#define MC_VM_MARC_BASE_HI_1__MARC_BASE_HI_1__SHIFT                                                           0x0
+#define MC_VM_MARC_BASE_HI_1__MARC_BASE_HI_1_MASK                                                             0x000FFFFFL
+//MC_VM_MARC_BASE_HI_2
+#define MC_VM_MARC_BASE_HI_2__MARC_BASE_HI_2__SHIFT                                                           0x0
+#define MC_VM_MARC_BASE_HI_2__MARC_BASE_HI_2_MASK                                                             0x000FFFFFL
+//MC_VM_MARC_BASE_HI_3
+#define MC_VM_MARC_BASE_HI_3__MARC_BASE_HI_3__SHIFT                                                           0x0
+#define MC_VM_MARC_BASE_HI_3__MARC_BASE_HI_3_MASK                                                             0x000FFFFFL
+//MC_VM_MARC_RELOC_LO_0
+#define MC_VM_MARC_RELOC_LO_0__MARC_ENABLE_0__SHIFT                                                           0x0
+#define MC_VM_MARC_RELOC_LO_0__MARC_READONLY_0__SHIFT                                                         0x1
+#define MC_VM_MARC_RELOC_LO_0__MARC_RELOC_LO_0__SHIFT                                                         0xc
+#define MC_VM_MARC_RELOC_LO_0__MARC_ENABLE_0_MASK                                                             0x00000001L
+#define MC_VM_MARC_RELOC_LO_0__MARC_READONLY_0_MASK                                                           0x00000002L
+#define MC_VM_MARC_RELOC_LO_0__MARC_RELOC_LO_0_MASK                                                           0xFFFFF000L
+//MC_VM_MARC_RELOC_LO_1
+#define MC_VM_MARC_RELOC_LO_1__MARC_ENABLE_1__SHIFT                                                           0x0
+#define MC_VM_MARC_RELOC_LO_1__MARC_READONLY_1__SHIFT                                                         0x1
+#define MC_VM_MARC_RELOC_LO_1__MARC_RELOC_LO_1__SHIFT                                                         0xc
+#define MC_VM_MARC_RELOC_LO_1__MARC_ENABLE_1_MASK                                                             0x00000001L
+#define MC_VM_MARC_RELOC_LO_1__MARC_READONLY_1_MASK                                                           0x00000002L
+#define MC_VM_MARC_RELOC_LO_1__MARC_RELOC_LO_1_MASK                                                           0xFFFFF000L
+//MC_VM_MARC_RELOC_LO_2
+#define MC_VM_MARC_RELOC_LO_2__MARC_ENABLE_2__SHIFT                                                           0x0
+#define MC_VM_MARC_RELOC_LO_2__MARC_READONLY_2__SHIFT                                                         0x1
+#define MC_VM_MARC_RELOC_LO_2__MARC_RELOC_LO_2__SHIFT                                                         0xc
+#define MC_VM_MARC_RELOC_LO_2__MARC_ENABLE_2_MASK                                                             0x00000001L
+#define MC_VM_MARC_RELOC_LO_2__MARC_READONLY_2_MASK                                                           0x00000002L
+#define MC_VM_MARC_RELOC_LO_2__MARC_RELOC_LO_2_MASK                                                           0xFFFFF000L
+//MC_VM_MARC_RELOC_LO_3
+#define MC_VM_MARC_RELOC_LO_3__MARC_ENABLE_3__SHIFT                                                           0x0
+#define MC_VM_MARC_RELOC_LO_3__MARC_READONLY_3__SHIFT                                                         0x1
+#define MC_VM_MARC_RELOC_LO_3__MARC_RELOC_LO_3__SHIFT                                                         0xc
+#define MC_VM_MARC_RELOC_LO_3__MARC_ENABLE_3_MASK                                                             0x00000001L
+#define MC_VM_MARC_RELOC_LO_3__MARC_READONLY_3_MASK                                                           0x00000002L
+#define MC_VM_MARC_RELOC_LO_3__MARC_RELOC_LO_3_MASK                                                           0xFFFFF000L
+//MC_VM_MARC_RELOC_HI_0
+#define MC_VM_MARC_RELOC_HI_0__MARC_RELOC_HI_0__SHIFT                                                         0x0
+#define MC_VM_MARC_RELOC_HI_0__MARC_RELOC_HI_0_MASK                                                           0x000FFFFFL
+//MC_VM_MARC_RELOC_HI_1
+#define MC_VM_MARC_RELOC_HI_1__MARC_RELOC_HI_1__SHIFT                                                         0x0
+#define MC_VM_MARC_RELOC_HI_1__MARC_RELOC_HI_1_MASK                                                           0x000FFFFFL
+//MC_VM_MARC_RELOC_HI_2
+#define MC_VM_MARC_RELOC_HI_2__MARC_RELOC_HI_2__SHIFT                                                         0x0
+#define MC_VM_MARC_RELOC_HI_2__MARC_RELOC_HI_2_MASK                                                           0x000FFFFFL
+//MC_VM_MARC_RELOC_HI_3
+#define MC_VM_MARC_RELOC_HI_3__MARC_RELOC_HI_3__SHIFT                                                         0x0
+#define MC_VM_MARC_RELOC_HI_3__MARC_RELOC_HI_3_MASK                                                           0x000FFFFFL
+//MC_VM_MARC_LEN_LO_0
+#define MC_VM_MARC_LEN_LO_0__MARC_LEN_LO_0__SHIFT                                                             0xc
+#define MC_VM_MARC_LEN_LO_0__MARC_LEN_LO_0_MASK                                                               0xFFFFF000L
+//MC_VM_MARC_LEN_LO_1
+#define MC_VM_MARC_LEN_LO_1__MARC_LEN_LO_1__SHIFT                                                             0xc
+#define MC_VM_MARC_LEN_LO_1__MARC_LEN_LO_1_MASK                                                               0xFFFFF000L
+//MC_VM_MARC_LEN_LO_2
+#define MC_VM_MARC_LEN_LO_2__MARC_LEN_LO_2__SHIFT                                                             0xc
+#define MC_VM_MARC_LEN_LO_2__MARC_LEN_LO_2_MASK                                                               0xFFFFF000L
+//MC_VM_MARC_LEN_LO_3
+#define MC_VM_MARC_LEN_LO_3__MARC_LEN_LO_3__SHIFT                                                             0xc
+#define MC_VM_MARC_LEN_LO_3__MARC_LEN_LO_3_MASK                                                               0xFFFFF000L
+//MC_VM_MARC_LEN_HI_0
+#define MC_VM_MARC_LEN_HI_0__MARC_LEN_HI_0__SHIFT                                                             0x0
+#define MC_VM_MARC_LEN_HI_0__MARC_LEN_HI_0_MASK                                                               0x000FFFFFL
+//MC_VM_MARC_LEN_HI_1
+#define MC_VM_MARC_LEN_HI_1__MARC_LEN_HI_1__SHIFT                                                             0x0
+#define MC_VM_MARC_LEN_HI_1__MARC_LEN_HI_1_MASK                                                               0x000FFFFFL
+//MC_VM_MARC_LEN_HI_2
+#define MC_VM_MARC_LEN_HI_2__MARC_LEN_HI_2__SHIFT                                                             0x0
+#define MC_VM_MARC_LEN_HI_2__MARC_LEN_HI_2_MASK                                                               0x000FFFFFL
+//MC_VM_MARC_LEN_HI_3
+#define MC_VM_MARC_LEN_HI_3__MARC_LEN_HI_3__SHIFT                                                             0x0
+#define MC_VM_MARC_LEN_HI_3__MARC_LEN_HI_3_MASK                                                               0x000FFFFFL
+//VM_IOMMU_CONTROL_REGISTER
+#define VM_IOMMU_CONTROL_REGISTER__IOMMUEN__SHIFT                                                             0x0
+#define VM_IOMMU_CONTROL_REGISTER__IOMMUEN_MASK                                                               0x00000001L
+//VM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER
+#define VM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN__SHIFT                                  0xd
+#define VM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN_MASK                                    0x00002000L
+//VM_PCIE_ATS_CNTL
+#define VM_PCIE_ATS_CNTL__STU__SHIFT                                                                          0x10
+#define VM_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                                   0x1f
+#define VM_PCIE_ATS_CNTL__STU_MASK                                                                            0x001F0000L
+#define VM_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                                     0x80000000L
+//VM_PCIE_ATS_CNTL_VF_0
+#define VM_PCIE_ATS_CNTL_VF_0__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_0__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_1
+#define VM_PCIE_ATS_CNTL_VF_1__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_1__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_2
+#define VM_PCIE_ATS_CNTL_VF_2__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_2__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_3
+#define VM_PCIE_ATS_CNTL_VF_3__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_3__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_4
+#define VM_PCIE_ATS_CNTL_VF_4__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_4__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_5
+#define VM_PCIE_ATS_CNTL_VF_5__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_5__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_6
+#define VM_PCIE_ATS_CNTL_VF_6__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_6__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_7
+#define VM_PCIE_ATS_CNTL_VF_7__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_7__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_8
+#define VM_PCIE_ATS_CNTL_VF_8__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_8__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_9
+#define VM_PCIE_ATS_CNTL_VF_9__ATC_ENABLE__SHIFT                                                              0x1f
+#define VM_PCIE_ATS_CNTL_VF_9__ATC_ENABLE_MASK                                                                0x80000000L
+//VM_PCIE_ATS_CNTL_VF_10
+#define VM_PCIE_ATS_CNTL_VF_10__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_10__ATC_ENABLE_MASK                                                               0x80000000L
+//VM_PCIE_ATS_CNTL_VF_11
+#define VM_PCIE_ATS_CNTL_VF_11__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_11__ATC_ENABLE_MASK                                                               0x80000000L
+//VM_PCIE_ATS_CNTL_VF_12
+#define VM_PCIE_ATS_CNTL_VF_12__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_12__ATC_ENABLE_MASK                                                               0x80000000L
+//VM_PCIE_ATS_CNTL_VF_13
+#define VM_PCIE_ATS_CNTL_VF_13__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_13__ATC_ENABLE_MASK                                                               0x80000000L
+//VM_PCIE_ATS_CNTL_VF_14
+#define VM_PCIE_ATS_CNTL_VF_14__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_14__ATC_ENABLE_MASK                                                               0x80000000L
+//VM_PCIE_ATS_CNTL_VF_15
+#define VM_PCIE_ATS_CNTL_VF_15__ATC_ENABLE__SHIFT                                                             0x1f
+#define VM_PCIE_ATS_CNTL_VF_15__ATC_ENABLE_MASK                                                               0x80000000L
+//MC_SHARED_ACTIVE_FCN_ID
+#define MC_SHARED_ACTIVE_FCN_ID__VFID__SHIFT                                                                  0x0
+#define MC_SHARED_ACTIVE_FCN_ID__VF__SHIFT                                                                    0x1f
+#define MC_SHARED_ACTIVE_FCN_ID__VFID_MASK                                                                    0x0000000FL
+#define MC_SHARED_ACTIVE_FCN_ID__VF_MASK                                                                      0x80000000L
+//MC_VM_XGMI_GPUIOV_ENABLE
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF0__SHIFT                                                           0x0
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF1__SHIFT                                                           0x1
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF2__SHIFT                                                           0x2
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF3__SHIFT                                                           0x3
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF4__SHIFT                                                           0x4
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF5__SHIFT                                                           0x5
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF6__SHIFT                                                           0x6
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF7__SHIFT                                                           0x7
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF8__SHIFT                                                           0x8
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF9__SHIFT                                                           0x9
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF10__SHIFT                                                          0xa
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF11__SHIFT                                                          0xb
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF12__SHIFT                                                          0xc
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF13__SHIFT                                                          0xd
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF14__SHIFT                                                          0xe
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF15__SHIFT                                                          0xf
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_PF__SHIFT                                                            0x1f
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF0_MASK                                                             0x00000001L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF1_MASK                                                             0x00000002L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF2_MASK                                                             0x00000004L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF3_MASK                                                             0x00000008L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF4_MASK                                                             0x00000010L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF5_MASK                                                             0x00000020L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF6_MASK                                                             0x00000040L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF7_MASK                                                             0x00000080L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF8_MASK                                                             0x00000100L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF9_MASK                                                             0x00000200L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF10_MASK                                                            0x00000400L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF11_MASK                                                            0x00000800L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF12_MASK                                                            0x00001000L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF13_MASK                                                            0x00002000L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF14_MASK                                                            0x00004000L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF15_MASK                                                            0x00008000L
+#define MC_VM_XGMI_GPUIOV_ENABLE__ENABLE_PF_MASK                                                              0x80000000L
+
+
+// addressBlock: aid_mmhub_utcl2_atcl2pfcntrdec
+//ATC_L2_PERFCOUNTER_LO
+#define ATC_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                              0x0
+#define ATC_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                0xFFFFFFFFL
+//ATC_L2_PERFCOUNTER_HI
+#define ATC_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                              0x0
+#define ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                           0x10
+#define ATC_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                0x0000FFFFL
+#define ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                             0xFFFF0000L
+
+
+// addressBlock: aid_mmhub_utcl2_atcl2pfcntldec
+//ATC_L2_PERFCOUNTER0_CFG
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                              0x0
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                          0x8
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                             0x18
+#define ATC_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                0x1c
+#define ATC_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                 0x1d
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                0x000000FFL
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                            0x0000FF00L
+#define ATC_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                               0x0F000000L
+#define ATC_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                  0x10000000L
+#define ATC_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                   0x20000000L
+//ATC_L2_PERFCOUNTER1_CFG
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                              0x0
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                          0x8
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                             0x18
+#define ATC_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                0x1c
+#define ATC_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                 0x1d
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                0x000000FFL
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                            0x0000FF00L
+#define ATC_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                               0x0F000000L
+#define ATC_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                  0x10000000L
+#define ATC_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                   0x20000000L
+//ATC_L2_PERFCOUNTER_RSLT_CNTL
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                              0x0
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                    0x8
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                     0x10
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                       0x18
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                        0x19
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                             0x1a
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                0x0000000FL
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                      0x0000FF00L
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                       0x00FF0000L
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                         0x01000000L
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                          0x02000000L
+#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                               0x04000000L
+
+
+// addressBlock: aid_mmhub_utcl2_vml2pldec
+//MC_VM_L2_PERFCOUNTER0_CFG
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER1_CFG
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER2_CFG
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER2_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER2_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER3_CFG
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER3_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER3_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER4_CFG
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER4_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER4_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER4_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER4_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER5_CFG
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER5_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER5_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER5_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER5_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER6_CFG
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER6_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER6_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER6_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER6_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER7_CFG
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE__SHIFT                                                           0x18
+#define MC_VM_L2_PERFCOUNTER7_CFG__ENABLE__SHIFT                                                              0x1c
+#define MC_VM_L2_PERFCOUNTER7_CFG__CLEAR__SHIFT                                                               0x1d
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define MC_VM_L2_PERFCOUNTER7_CFG__ENABLE_MASK                                                                0x10000000L
+#define MC_VM_L2_PERFCOUNTER7_CFG__CLEAR_MASK                                                                 0x20000000L
+//MC_VM_L2_PERFCOUNTER_RSLT_CNTL
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                            0x0
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                  0x8
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                   0x10
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                     0x18
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                      0x19
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                           0x1a
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                              0x0000000FL
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                    0x0000FF00L
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                     0x00FF0000L
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                       0x01000000L
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                        0x02000000L
+#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                             0x04000000L
+
+
+// addressBlock: aid_mmhub_utcl2_vml2prdec
+//MC_VM_L2_PERFCOUNTER_LO
+#define MC_VM_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                              0xFFFFFFFFL
+//MC_VM_L2_PERFCOUNTER_HI
+#define MC_VM_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                            0x0
+#define MC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                         0x10
+#define MC_VM_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                              0x0000FFFFL
+#define MC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                           0xFFFF0000L
+
+
+// addressBlock: aid_mmhub_utcl2_l2tlbdec
+//L2TLB_TLB0_STATUS
+#define L2TLB_TLB0_STATUS__BUSY__SHIFT                                                                        0x0
+#define L2TLB_TLB0_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                         0x1
+#define L2TLB_TLB0_STATUS__BUSY_MASK                                                                          0x00000001L
+#define L2TLB_TLB0_STATUS__FOUND_PARITY_ERRORS_MASK                                                           0x00000002L
+//UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO__ADDR__SHIFT                                             0x0
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO__ADDR_MASK                                               0xFFFFFFFFL
+//UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__ADDR__SHIFT                                             0x0
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VMID__SHIFT                                             0x4
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VFID__SHIFT                                             0x9
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VF__SHIFT                                               0xd
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__GPA__SHIFT                                              0xe
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__RD_PERM__SHIFT                                          0x10
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__WR_PERM__SHIFT                                          0x11
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__EX_PERM__SHIFT                                          0x12
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__CLIENT_ID__SHIFT                                        0x13
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__REQ__SHIFT                                              0x1f
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__ADDR_MASK                                               0x0000000FL
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VMID_MASK                                               0x000000F0L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VFID_MASK                                               0x00001E00L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VF_MASK                                                 0x00002000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__GPA_MASK                                                0x0000C000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__RD_PERM_MASK                                            0x00010000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__WR_PERM_MASK                                            0x00020000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__EX_PERM_MASK                                            0x00040000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__CLIENT_ID_MASK                                          0x0FF80000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__REQ_MASK                                                0x80000000L
+//UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO__ADDR__SHIFT                                            0x0
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO__ADDR_MASK                                              0xFFFFFFFFL
+//UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ADDR__SHIFT                                            0x0
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PERMS__SHIFT                                           0x4
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__FRAGMENT_SIZE__SHIFT                                   0x7
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SNOOP__SHIFT                                           0xd
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SPA__SHIFT                                             0xe
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__IO__SHIFT                                              0xf
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PTE_TMZ__SHIFT                                         0x10
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NO_PTE__SHIFT                                          0x11
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MTYPE__SHIFT                                           0x12
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MEMLOG__SHIFT                                          0x14
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__LLC_NOALLOC__SHIFT                                     0x15
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NACK__SHIFT                                            0x16
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ACK__SHIFT                                             0x1f
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ADDR_MASK                                              0x0000000FL
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PERMS_MASK                                             0x00000070L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__FRAGMENT_SIZE_MASK                                     0x00001F80L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SNOOP_MASK                                             0x00002000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SPA_MASK                                               0x00004000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__IO_MASK                                                0x00008000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PTE_TMZ_MASK                                           0x00010000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NO_PTE_MASK                                            0x00020000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MTYPE_MASK                                             0x000C0000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MEMLOG_MASK                                            0x00100000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__LLC_NOALLOC_MASK                                       0x00200000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NACK_MASK                                              0x00C00000L
+#define UTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ACK_MASK                                               0x80000000L
+
+
+// addressBlock: aid_mmhub_utcl2_l2tlbpldec
+//L2TLB_PERFCOUNTER0_CFG
+#define L2TLB_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define L2TLB_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                              0x18
+#define L2TLB_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                 0x1c
+#define L2TLB_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                  0x1d
+#define L2TLB_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define L2TLB_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define L2TLB_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define L2TLB_PERFCOUNTER0_CFG__ENABLE_MASK                                                                   0x10000000L
+#define L2TLB_PERFCOUNTER0_CFG__CLEAR_MASK                                                                    0x20000000L
+//L2TLB_PERFCOUNTER1_CFG
+#define L2TLB_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define L2TLB_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                              0x18
+#define L2TLB_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                 0x1c
+#define L2TLB_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                  0x1d
+#define L2TLB_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define L2TLB_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define L2TLB_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define L2TLB_PERFCOUNTER1_CFG__ENABLE_MASK                                                                   0x10000000L
+#define L2TLB_PERFCOUNTER1_CFG__CLEAR_MASK                                                                    0x20000000L
+//L2TLB_PERFCOUNTER2_CFG
+#define L2TLB_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define L2TLB_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                              0x18
+#define L2TLB_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                 0x1c
+#define L2TLB_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                  0x1d
+#define L2TLB_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define L2TLB_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define L2TLB_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define L2TLB_PERFCOUNTER2_CFG__ENABLE_MASK                                                                   0x10000000L
+#define L2TLB_PERFCOUNTER2_CFG__CLEAR_MASK                                                                    0x20000000L
+//L2TLB_PERFCOUNTER3_CFG
+#define L2TLB_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                           0x8
+#define L2TLB_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                              0x18
+#define L2TLB_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                                 0x1c
+#define L2TLB_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                                  0x1d
+#define L2TLB_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                                 0x000000FFL
+#define L2TLB_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                             0x0000FF00L
+#define L2TLB_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                                0x0F000000L
+#define L2TLB_PERFCOUNTER3_CFG__ENABLE_MASK                                                                   0x10000000L
+#define L2TLB_PERFCOUNTER3_CFG__CLEAR_MASK                                                                    0x20000000L
+//L2TLB_PERFCOUNTER_RSLT_CNTL
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                               0x0
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                     0x8
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                      0x10
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                        0x18
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                         0x19
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                              0x1a
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                 0x0000000FL
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                       0x0000FF00L
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                        0x00FF0000L
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                          0x01000000L
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                           0x02000000L
+#define L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                0x04000000L
+
+
+// addressBlock: aid_mmhub_utcl2_l2tlbprdec
+//L2TLB_PERFCOUNTER_LO
+#define L2TLB_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                 0xFFFFFFFFL
+//L2TLB_PERFCOUNTER_HI
+#define L2TLB_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                               0x0
+#define L2TLB_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                            0x10
+#define L2TLB_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                 0x0000FFFFL
+#define L2TLB_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                              0xFFFF0000L
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mp/mp_13_0_6_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_13_0_6_offset.h
new file mode 100644
index 000000000000..f04fa95a770c
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_13_0_6_offset.h
@@ -0,0 +1,456 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _mp_13_0_6_OFFSET_HEADER
+#define _mp_13_0_6_OFFSET_HEADER
+
+
+
+// addressBlock: aid_mp_SmuMp0_SmnDec
+// base address: 0x0
+#define regMP0_SMN_C2PMSG_32                                                                            0x0060
+#define regMP0_SMN_C2PMSG_32_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_33                                                                            0x0061
+#define regMP0_SMN_C2PMSG_33_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_34                                                                            0x0062
+#define regMP0_SMN_C2PMSG_34_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_35                                                                            0x0063
+#define regMP0_SMN_C2PMSG_35_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_36                                                                            0x0064
+#define regMP0_SMN_C2PMSG_36_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_37                                                                            0x0065
+#define regMP0_SMN_C2PMSG_37_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_38                                                                            0x0066
+#define regMP0_SMN_C2PMSG_38_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_39                                                                            0x0067
+#define regMP0_SMN_C2PMSG_39_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_40                                                                            0x0068
+#define regMP0_SMN_C2PMSG_40_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_41                                                                            0x0069
+#define regMP0_SMN_C2PMSG_41_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_42                                                                            0x006a
+#define regMP0_SMN_C2PMSG_42_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_43                                                                            0x006b
+#define regMP0_SMN_C2PMSG_43_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_44                                                                            0x006c
+#define regMP0_SMN_C2PMSG_44_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_45                                                                            0x006d
+#define regMP0_SMN_C2PMSG_45_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_46                                                                            0x006e
+#define regMP0_SMN_C2PMSG_46_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_47                                                                            0x006f
+#define regMP0_SMN_C2PMSG_47_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_48                                                                            0x0070
+#define regMP0_SMN_C2PMSG_48_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_49                                                                            0x0071
+#define regMP0_SMN_C2PMSG_49_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_50                                                                            0x0072
+#define regMP0_SMN_C2PMSG_50_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_51                                                                            0x0073
+#define regMP0_SMN_C2PMSG_51_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_52                                                                            0x0074
+#define regMP0_SMN_C2PMSG_52_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_53                                                                            0x0075
+#define regMP0_SMN_C2PMSG_53_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_54                                                                            0x0076
+#define regMP0_SMN_C2PMSG_54_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_55                                                                            0x0077
+#define regMP0_SMN_C2PMSG_55_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_56                                                                            0x0078
+#define regMP0_SMN_C2PMSG_56_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_57                                                                            0x0079
+#define regMP0_SMN_C2PMSG_57_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_58                                                                            0x007a
+#define regMP0_SMN_C2PMSG_58_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_59                                                                            0x007b
+#define regMP0_SMN_C2PMSG_59_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_60                                                                            0x007c
+#define regMP0_SMN_C2PMSG_60_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_61                                                                            0x007d
+#define regMP0_SMN_C2PMSG_61_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_62                                                                            0x007e
+#define regMP0_SMN_C2PMSG_62_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_63                                                                            0x007f
+#define regMP0_SMN_C2PMSG_63_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_64                                                                            0x0080
+#define regMP0_SMN_C2PMSG_64_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_65                                                                            0x0081
+#define regMP0_SMN_C2PMSG_65_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_66                                                                            0x0082
+#define regMP0_SMN_C2PMSG_66_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_67                                                                            0x0083
+#define regMP0_SMN_C2PMSG_67_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_68                                                                            0x0084
+#define regMP0_SMN_C2PMSG_68_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_69                                                                            0x0085
+#define regMP0_SMN_C2PMSG_69_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_70                                                                            0x0086
+#define regMP0_SMN_C2PMSG_70_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_71                                                                            0x0087
+#define regMP0_SMN_C2PMSG_71_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_72                                                                            0x0088
+#define regMP0_SMN_C2PMSG_72_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_73                                                                            0x0089
+#define regMP0_SMN_C2PMSG_73_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_74                                                                            0x008a
+#define regMP0_SMN_C2PMSG_74_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_75                                                                            0x008b
+#define regMP0_SMN_C2PMSG_75_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_76                                                                            0x008c
+#define regMP0_SMN_C2PMSG_76_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_77                                                                            0x008d
+#define regMP0_SMN_C2PMSG_77_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_78                                                                            0x008e
+#define regMP0_SMN_C2PMSG_78_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_79                                                                            0x008f
+#define regMP0_SMN_C2PMSG_79_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_80                                                                            0x0090
+#define regMP0_SMN_C2PMSG_80_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_81                                                                            0x0091
+#define regMP0_SMN_C2PMSG_81_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_82                                                                            0x0092
+#define regMP0_SMN_C2PMSG_82_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_83                                                                            0x0093
+#define regMP0_SMN_C2PMSG_83_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_84                                                                            0x0094
+#define regMP0_SMN_C2PMSG_84_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_85                                                                            0x0095
+#define regMP0_SMN_C2PMSG_85_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_86                                                                            0x0096
+#define regMP0_SMN_C2PMSG_86_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_87                                                                            0x0097
+#define regMP0_SMN_C2PMSG_87_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_88                                                                            0x0098
+#define regMP0_SMN_C2PMSG_88_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_89                                                                            0x0099
+#define regMP0_SMN_C2PMSG_89_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_90                                                                            0x009a
+#define regMP0_SMN_C2PMSG_90_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_91                                                                            0x009b
+#define regMP0_SMN_C2PMSG_91_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_92                                                                            0x009c
+#define regMP0_SMN_C2PMSG_92_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_93                                                                            0x009d
+#define regMP0_SMN_C2PMSG_93_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_94                                                                            0x009e
+#define regMP0_SMN_C2PMSG_94_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_95                                                                            0x009f
+#define regMP0_SMN_C2PMSG_95_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_96                                                                            0x00a0
+#define regMP0_SMN_C2PMSG_96_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_97                                                                            0x00a1
+#define regMP0_SMN_C2PMSG_97_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_98                                                                            0x00a2
+#define regMP0_SMN_C2PMSG_98_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_99                                                                            0x00a3
+#define regMP0_SMN_C2PMSG_99_BASE_IDX                                                                   0
+#define regMP0_SMN_C2PMSG_100                                                                           0x00a4
+#define regMP0_SMN_C2PMSG_100_BASE_IDX                                                                  0
+#define regMP0_SMN_C2PMSG_101                                                                           0x00a5
+#define regMP0_SMN_C2PMSG_101_BASE_IDX                                                                  0
+#define regMP0_SMN_C2PMSG_102                                                                           0x00a6
+#define regMP0_SMN_C2PMSG_102_BASE_IDX                                                                  0
+#define regMP0_SMN_C2PMSG_103                                                                           0x00a7
+#define regMP0_SMN_C2PMSG_103_BASE_IDX                                                                  0
+#define regMP0_SMN_IH_CREDIT                                                                            0x00c1
+#define regMP0_SMN_IH_CREDIT_BASE_IDX                                                                   0
+#define regMP0_SMN_IH_SW_INT                                                                            0x00c2
+#define regMP0_SMN_IH_SW_INT_BASE_IDX                                                                   0
+#define regMP0_SMN_IH_SW_INT_CTRL                                                                       0x00c3
+#define regMP0_SMN_IH_SW_INT_CTRL_BASE_IDX                                                              0
+
+
+// addressBlock: aid_mp_SmuMp1_SmnDec
+// base address: 0x0
+#define regMP1_SMN_C2PMSG_32                                                                            0x0260
+#define regMP1_SMN_C2PMSG_32_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_33                                                                            0x0261
+#define regMP1_SMN_C2PMSG_33_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_34                                                                            0x0262
+#define regMP1_SMN_C2PMSG_34_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_35                                                                            0x0263
+#define regMP1_SMN_C2PMSG_35_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_36                                                                            0x0264
+#define regMP1_SMN_C2PMSG_36_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_37                                                                            0x0265
+#define regMP1_SMN_C2PMSG_37_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_38                                                                            0x0266
+#define regMP1_SMN_C2PMSG_38_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_39                                                                            0x0267
+#define regMP1_SMN_C2PMSG_39_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_40                                                                            0x0268
+#define regMP1_SMN_C2PMSG_40_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_41                                                                            0x0269
+#define regMP1_SMN_C2PMSG_41_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_42                                                                            0x026a
+#define regMP1_SMN_C2PMSG_42_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_43                                                                            0x026b
+#define regMP1_SMN_C2PMSG_43_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_44                                                                            0x026c
+#define regMP1_SMN_C2PMSG_44_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_45                                                                            0x026d
+#define regMP1_SMN_C2PMSG_45_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_46                                                                            0x026e
+#define regMP1_SMN_C2PMSG_46_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_47                                                                            0x026f
+#define regMP1_SMN_C2PMSG_47_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_48                                                                            0x0270
+#define regMP1_SMN_C2PMSG_48_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_49                                                                            0x0271
+#define regMP1_SMN_C2PMSG_49_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_50                                                                            0x0272
+#define regMP1_SMN_C2PMSG_50_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_51                                                                            0x0273
+#define regMP1_SMN_C2PMSG_51_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_52                                                                            0x0274
+#define regMP1_SMN_C2PMSG_52_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_53                                                                            0x0275
+#define regMP1_SMN_C2PMSG_53_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_54                                                                            0x0276
+#define regMP1_SMN_C2PMSG_54_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_55                                                                            0x0277
+#define regMP1_SMN_C2PMSG_55_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_56                                                                            0x0278
+#define regMP1_SMN_C2PMSG_56_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_57                                                                            0x0279
+#define regMP1_SMN_C2PMSG_57_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_58                                                                            0x027a
+#define regMP1_SMN_C2PMSG_58_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_59                                                                            0x027b
+#define regMP1_SMN_C2PMSG_59_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_60                                                                            0x027c
+#define regMP1_SMN_C2PMSG_60_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_61                                                                            0x027d
+#define regMP1_SMN_C2PMSG_61_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_62                                                                            0x027e
+#define regMP1_SMN_C2PMSG_62_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_63                                                                            0x027f
+#define regMP1_SMN_C2PMSG_63_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_64                                                                            0x0280
+#define regMP1_SMN_C2PMSG_64_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_65                                                                            0x0281
+#define regMP1_SMN_C2PMSG_65_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_66                                                                            0x0282
+#define regMP1_SMN_C2PMSG_66_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_67                                                                            0x0283
+#define regMP1_SMN_C2PMSG_67_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_68                                                                            0x0284
+#define regMP1_SMN_C2PMSG_68_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_69                                                                            0x0285
+#define regMP1_SMN_C2PMSG_69_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_70                                                                            0x0286
+#define regMP1_SMN_C2PMSG_70_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_71                                                                            0x0287
+#define regMP1_SMN_C2PMSG_71_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_72                                                                            0x0288
+#define regMP1_SMN_C2PMSG_72_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_73                                                                            0x0289
+#define regMP1_SMN_C2PMSG_73_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_74                                                                            0x028a
+#define regMP1_SMN_C2PMSG_74_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_75                                                                            0x028b
+#define regMP1_SMN_C2PMSG_75_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_76                                                                            0x028c
+#define regMP1_SMN_C2PMSG_76_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_77                                                                            0x028d
+#define regMP1_SMN_C2PMSG_77_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_78                                                                            0x028e
+#define regMP1_SMN_C2PMSG_78_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_79                                                                            0x028f
+#define regMP1_SMN_C2PMSG_79_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_80                                                                            0x0290
+#define regMP1_SMN_C2PMSG_80_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_81                                                                            0x0291
+#define regMP1_SMN_C2PMSG_81_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_82                                                                            0x0292
+#define regMP1_SMN_C2PMSG_82_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_83                                                                            0x0293
+#define regMP1_SMN_C2PMSG_83_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_84                                                                            0x0294
+#define regMP1_SMN_C2PMSG_84_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_85                                                                            0x0295
+#define regMP1_SMN_C2PMSG_85_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_86                                                                            0x0296
+#define regMP1_SMN_C2PMSG_86_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_87                                                                            0x0297
+#define regMP1_SMN_C2PMSG_87_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_88                                                                            0x0298
+#define regMP1_SMN_C2PMSG_88_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_89                                                                            0x0299
+#define regMP1_SMN_C2PMSG_89_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_90                                                                            0x029a
+#define regMP1_SMN_C2PMSG_90_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_91                                                                            0x029b
+#define regMP1_SMN_C2PMSG_91_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_92                                                                            0x029c
+#define regMP1_SMN_C2PMSG_92_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_93                                                                            0x029d
+#define regMP1_SMN_C2PMSG_93_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_94                                                                            0x029e
+#define regMP1_SMN_C2PMSG_94_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_95                                                                            0x029f
+#define regMP1_SMN_C2PMSG_95_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_96                                                                            0x02a0
+#define regMP1_SMN_C2PMSG_96_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_97                                                                            0x02a1
+#define regMP1_SMN_C2PMSG_97_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_98                                                                            0x02a2
+#define regMP1_SMN_C2PMSG_98_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_99                                                                            0x02a3
+#define regMP1_SMN_C2PMSG_99_BASE_IDX                                                                   0
+#define regMP1_SMN_C2PMSG_100                                                                           0x02a4
+#define regMP1_SMN_C2PMSG_100_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_101                                                                           0x02a5
+#define regMP1_SMN_C2PMSG_101_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_102                                                                           0x02a6
+#define regMP1_SMN_C2PMSG_102_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_103                                                                           0x02a7
+#define regMP1_SMN_C2PMSG_103_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_104                                                                           0x02a8
+#define regMP1_SMN_C2PMSG_104_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_105                                                                           0x02a9
+#define regMP1_SMN_C2PMSG_105_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_106                                                                           0x02aa
+#define regMP1_SMN_C2PMSG_106_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_107                                                                           0x02ab
+#define regMP1_SMN_C2PMSG_107_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_108                                                                           0x02ac
+#define regMP1_SMN_C2PMSG_108_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_109                                                                           0x02ad
+#define regMP1_SMN_C2PMSG_109_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_110                                                                           0x02ae
+#define regMP1_SMN_C2PMSG_110_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_111                                                                           0x02af
+#define regMP1_SMN_C2PMSG_111_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_112                                                                           0x02b0
+#define regMP1_SMN_C2PMSG_112_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_113                                                                           0x02b1
+#define regMP1_SMN_C2PMSG_113_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_114                                                                           0x02b2
+#define regMP1_SMN_C2PMSG_114_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_115                                                                           0x02b3
+#define regMP1_SMN_C2PMSG_115_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_116                                                                           0x02b4
+#define regMP1_SMN_C2PMSG_116_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_117                                                                           0x02b5
+#define regMP1_SMN_C2PMSG_117_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_118                                                                           0x02b6
+#define regMP1_SMN_C2PMSG_118_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_119                                                                           0x02b7
+#define regMP1_SMN_C2PMSG_119_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_120                                                                           0x02b8
+#define regMP1_SMN_C2PMSG_120_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_121                                                                           0x02b9
+#define regMP1_SMN_C2PMSG_121_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_122                                                                           0x02ba
+#define regMP1_SMN_C2PMSG_122_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_123                                                                           0x02bb
+#define regMP1_SMN_C2PMSG_123_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_124                                                                           0x02bc
+#define regMP1_SMN_C2PMSG_124_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_125                                                                           0x02bd
+#define regMP1_SMN_C2PMSG_125_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_126                                                                           0x02be
+#define regMP1_SMN_C2PMSG_126_BASE_IDX                                                                  0
+#define regMP1_SMN_C2PMSG_127                                                                           0x02bf
+#define regMP1_SMN_C2PMSG_127_BASE_IDX                                                                  0
+#define regMP1_SMN_IH_CREDIT                                                                            0x02c1
+#define regMP1_SMN_IH_CREDIT_BASE_IDX                                                                   0
+#define regMP1_SMN_IH_SW_INT                                                                            0x02c2
+#define regMP1_SMN_IH_SW_INT_BASE_IDX                                                                   0
+#define regMP1_SMN_IH_SW_INT_CTRL                                                                       0x02c3
+#define regMP1_SMN_IH_SW_INT_CTRL_BASE_IDX                                                              0
+#define regMP1_SMN_FPS_CNT                                                                              0x02c4
+#define regMP1_SMN_FPS_CNT_BASE_IDX                                                                     0
+#define regMP1_SMN_PUB_CTRL                                                                             0x02c5
+#define regMP1_SMN_PUB_CTRL_BASE_IDX                                                                    0
+#define regMP1_SMN_EXT_SCRATCH0                                                                         0x0340
+#define regMP1_SMN_EXT_SCRATCH0_BASE_IDX                                                                0
+#define regMP1_SMN_EXT_SCRATCH1                                                                         0x0341
+#define regMP1_SMN_EXT_SCRATCH1_BASE_IDX                                                                0
+#define regMP1_SMN_EXT_SCRATCH2                                                                         0x0342
+#define regMP1_SMN_EXT_SCRATCH2_BASE_IDX                                                                0
+#define regMP1_SMN_EXT_SCRATCH3                                                                         0x0343
+#define regMP1_SMN_EXT_SCRATCH3_BASE_IDX                                                                0
+#define regMP1_SMN_EXT_SCRATCH4                                                                         0x0344
+#define regMP1_SMN_EXT_SCRATCH4_BASE_IDX                                                                0
+#define regMP1_SMN_EXT_SCRATCH5                                                                         0x0345
+#define regMP1_SMN_EXT_SCRATCH5_BASE_IDX                                                                0
+#define regMP1_SMN_EXT_SCRATCH6                                                                         0x0346
+#define regMP1_SMN_EXT_SCRATCH6_BASE_IDX                                                                0
+#define regMP1_SMN_EXT_SCRATCH7                                                                         0x0347
+#define regMP1_SMN_EXT_SCRATCH7_BASE_IDX                                                                0
+#define regMP1_SMN_EXT_SCRATCH8                                                                         0x0348
+#define regMP1_SMN_EXT_SCRATCH8_BASE_IDX                                                                0
+#define regMP1_SMN_EXT_SCRATCH10                                                                        0x034a
+#define regMP1_SMN_EXT_SCRATCH10_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH11                                                                        0x034b
+#define regMP1_SMN_EXT_SCRATCH11_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH12                                                                        0x034c
+#define regMP1_SMN_EXT_SCRATCH12_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH13                                                                        0x034d
+#define regMP1_SMN_EXT_SCRATCH13_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH14                                                                        0x034e
+#define regMP1_SMN_EXT_SCRATCH14_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH15                                                                        0x034f
+#define regMP1_SMN_EXT_SCRATCH15_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH16                                                                        0x0350
+#define regMP1_SMN_EXT_SCRATCH16_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH17                                                                        0x0351
+#define regMP1_SMN_EXT_SCRATCH17_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH18                                                                        0x0352
+#define regMP1_SMN_EXT_SCRATCH18_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH19                                                                        0x0353
+#define regMP1_SMN_EXT_SCRATCH19_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH20                                                                        0x0354
+#define regMP1_SMN_EXT_SCRATCH20_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH21                                                                        0x0355
+#define regMP1_SMN_EXT_SCRATCH21_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH22                                                                        0x0356
+#define regMP1_SMN_EXT_SCRATCH22_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH23                                                                        0x0357
+#define regMP1_SMN_EXT_SCRATCH23_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH24                                                                        0x0358
+#define regMP1_SMN_EXT_SCRATCH24_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH25                                                                        0x0359
+#define regMP1_SMN_EXT_SCRATCH25_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH26                                                                        0x035a
+#define regMP1_SMN_EXT_SCRATCH26_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH27                                                                        0x035b
+#define regMP1_SMN_EXT_SCRATCH27_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH28                                                                        0x035c
+#define regMP1_SMN_EXT_SCRATCH28_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH29                                                                        0x035d
+#define regMP1_SMN_EXT_SCRATCH29_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH30                                                                        0x035e
+#define regMP1_SMN_EXT_SCRATCH30_BASE_IDX                                                               0
+#define regMP1_SMN_EXT_SCRATCH31                                                                        0x035f
+#define regMP1_SMN_EXT_SCRATCH31_BASE_IDX                                                               0
+
+
+// addressBlock: aid_mp_SmuMp1Pub_CruDec
+// base address: 0x0
+#define regMP1_FIRMWARE_FLAGS                                                                           0xbee00a
+#define regMP1_FIRMWARE_FLAGS_BASE_IDX                                                                  0
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mp/mp_13_0_6_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_13_0_6_sh_mask.h
new file mode 100644
index 000000000000..780d9824d5ed
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_13_0_6_sh_mask.h
@@ -0,0 +1,674 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _mp_13_0_6_SH_MASK_HEADER
+#define _mp_13_0_6_SH_MASK_HEADER
+
+
+// addressBlock: aid_mp_SmuMp0_SmnDec
+//MP0_SMN_C2PMSG_32
+#define MP0_SMN_C2PMSG_32__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_32__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_33
+#define MP0_SMN_C2PMSG_33__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_33__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_34
+#define MP0_SMN_C2PMSG_34__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_34__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_35
+#define MP0_SMN_C2PMSG_35__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_35__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_36
+#define MP0_SMN_C2PMSG_36__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_36__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_37
+#define MP0_SMN_C2PMSG_37__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_37__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_38
+#define MP0_SMN_C2PMSG_38__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_38__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_39
+#define MP0_SMN_C2PMSG_39__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_39__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_40
+#define MP0_SMN_C2PMSG_40__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_40__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_41
+#define MP0_SMN_C2PMSG_41__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_41__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_42
+#define MP0_SMN_C2PMSG_42__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_42__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_43
+#define MP0_SMN_C2PMSG_43__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_43__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_44
+#define MP0_SMN_C2PMSG_44__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_44__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_45
+#define MP0_SMN_C2PMSG_45__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_45__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_46
+#define MP0_SMN_C2PMSG_46__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_46__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_47
+#define MP0_SMN_C2PMSG_47__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_47__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_48
+#define MP0_SMN_C2PMSG_48__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_48__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_49
+#define MP0_SMN_C2PMSG_49__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_49__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_50
+#define MP0_SMN_C2PMSG_50__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_50__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_51
+#define MP0_SMN_C2PMSG_51__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_51__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_52
+#define MP0_SMN_C2PMSG_52__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_52__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_53
+#define MP0_SMN_C2PMSG_53__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_53__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_54
+#define MP0_SMN_C2PMSG_54__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_54__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_55
+#define MP0_SMN_C2PMSG_55__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_55__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_56
+#define MP0_SMN_C2PMSG_56__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_56__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_57
+#define MP0_SMN_C2PMSG_57__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_57__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_58
+#define MP0_SMN_C2PMSG_58__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_58__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_59
+#define MP0_SMN_C2PMSG_59__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_59__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_60
+#define MP0_SMN_C2PMSG_60__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_60__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_61
+#define MP0_SMN_C2PMSG_61__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_61__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_62
+#define MP0_SMN_C2PMSG_62__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_62__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_63
+#define MP0_SMN_C2PMSG_63__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_63__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_64
+#define MP0_SMN_C2PMSG_64__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_64__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_65
+#define MP0_SMN_C2PMSG_65__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_65__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_66
+#define MP0_SMN_C2PMSG_66__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_66__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_67
+#define MP0_SMN_C2PMSG_67__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_67__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_68
+#define MP0_SMN_C2PMSG_68__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_68__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_69
+#define MP0_SMN_C2PMSG_69__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_69__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_70
+#define MP0_SMN_C2PMSG_70__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_70__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_71
+#define MP0_SMN_C2PMSG_71__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_71__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_72
+#define MP0_SMN_C2PMSG_72__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_72__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_73
+#define MP0_SMN_C2PMSG_73__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_73__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_74
+#define MP0_SMN_C2PMSG_74__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_74__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_75
+#define MP0_SMN_C2PMSG_75__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_75__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_76
+#define MP0_SMN_C2PMSG_76__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_76__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_77
+#define MP0_SMN_C2PMSG_77__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_77__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_78
+#define MP0_SMN_C2PMSG_78__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_78__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_79
+#define MP0_SMN_C2PMSG_79__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_79__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_80
+#define MP0_SMN_C2PMSG_80__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_80__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_81
+#define MP0_SMN_C2PMSG_81__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_81__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_82
+#define MP0_SMN_C2PMSG_82__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_82__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_83
+#define MP0_SMN_C2PMSG_83__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_83__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_84
+#define MP0_SMN_C2PMSG_84__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_84__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_85
+#define MP0_SMN_C2PMSG_85__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_85__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_86
+#define MP0_SMN_C2PMSG_86__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_86__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_87
+#define MP0_SMN_C2PMSG_87__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_87__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_88
+#define MP0_SMN_C2PMSG_88__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_88__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_89
+#define MP0_SMN_C2PMSG_89__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_89__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_90
+#define MP0_SMN_C2PMSG_90__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_90__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_91
+#define MP0_SMN_C2PMSG_91__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_91__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_92
+#define MP0_SMN_C2PMSG_92__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_92__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_93
+#define MP0_SMN_C2PMSG_93__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_93__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_94
+#define MP0_SMN_C2PMSG_94__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_94__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_95
+#define MP0_SMN_C2PMSG_95__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_95__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_96
+#define MP0_SMN_C2PMSG_96__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_96__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_97
+#define MP0_SMN_C2PMSG_97__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_97__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_98
+#define MP0_SMN_C2PMSG_98__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_98__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_99
+#define MP0_SMN_C2PMSG_99__CONTENT__SHIFT                                                                     0x0
+#define MP0_SMN_C2PMSG_99__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP0_SMN_C2PMSG_100
+#define MP0_SMN_C2PMSG_100__CONTENT__SHIFT                                                                    0x0
+#define MP0_SMN_C2PMSG_100__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP0_SMN_C2PMSG_101
+#define MP0_SMN_C2PMSG_101__CONTENT__SHIFT                                                                    0x0
+#define MP0_SMN_C2PMSG_101__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP0_SMN_C2PMSG_102
+#define MP0_SMN_C2PMSG_102__CONTENT__SHIFT                                                                    0x0
+#define MP0_SMN_C2PMSG_102__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP0_SMN_C2PMSG_103
+#define MP0_SMN_C2PMSG_103__CONTENT__SHIFT                                                                    0x0
+#define MP0_SMN_C2PMSG_103__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP0_SMN_IH_CREDIT
+#define MP0_SMN_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                0x0
+#define MP0_SMN_IH_CREDIT__CLIENT_ID__SHIFT                                                                   0x10
+#define MP0_SMN_IH_CREDIT__CREDIT_VALUE_MASK                                                                  0x00000003L
+#define MP0_SMN_IH_CREDIT__CLIENT_ID_MASK                                                                     0x00FF0000L
+//MP0_SMN_IH_SW_INT
+#define MP0_SMN_IH_SW_INT__ID__SHIFT                                                                          0x0
+#define MP0_SMN_IH_SW_INT__VALID__SHIFT                                                                       0x8
+#define MP0_SMN_IH_SW_INT__ID_MASK                                                                            0x000000FFL
+#define MP0_SMN_IH_SW_INT__VALID_MASK                                                                         0x00000100L
+//MP0_SMN_IH_SW_INT_CTRL
+#define MP0_SMN_IH_SW_INT_CTRL__INT_MASK__SHIFT                                                               0x0
+#define MP0_SMN_IH_SW_INT_CTRL__INT_ACK__SHIFT                                                                0x8
+#define MP0_SMN_IH_SW_INT_CTRL__INT_MASK_MASK                                                                 0x00000001L
+#define MP0_SMN_IH_SW_INT_CTRL__INT_ACK_MASK                                                                  0x00000100L
+
+
+// addressBlock: aid_mp_SmuMp1_SmnDec
+//MP1_SMN_C2PMSG_32
+#define MP1_SMN_C2PMSG_32__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_32__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_33
+#define MP1_SMN_C2PMSG_33__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_33__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_34
+#define MP1_SMN_C2PMSG_34__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_34__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_35
+#define MP1_SMN_C2PMSG_35__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_35__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_36
+#define MP1_SMN_C2PMSG_36__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_36__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_37
+#define MP1_SMN_C2PMSG_37__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_37__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_38
+#define MP1_SMN_C2PMSG_38__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_38__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_39
+#define MP1_SMN_C2PMSG_39__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_39__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_40
+#define MP1_SMN_C2PMSG_40__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_40__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_41
+#define MP1_SMN_C2PMSG_41__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_41__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_42
+#define MP1_SMN_C2PMSG_42__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_42__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_43
+#define MP1_SMN_C2PMSG_43__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_43__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_44
+#define MP1_SMN_C2PMSG_44__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_44__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_45
+#define MP1_SMN_C2PMSG_45__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_45__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_46
+#define MP1_SMN_C2PMSG_46__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_46__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_47
+#define MP1_SMN_C2PMSG_47__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_47__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_48
+#define MP1_SMN_C2PMSG_48__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_48__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_49
+#define MP1_SMN_C2PMSG_49__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_49__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_50
+#define MP1_SMN_C2PMSG_50__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_50__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_51
+#define MP1_SMN_C2PMSG_51__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_51__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_52
+#define MP1_SMN_C2PMSG_52__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_52__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_53
+#define MP1_SMN_C2PMSG_53__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_53__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_54
+#define MP1_SMN_C2PMSG_54__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_54__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_55
+#define MP1_SMN_C2PMSG_55__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_55__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_56
+#define MP1_SMN_C2PMSG_56__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_56__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_57
+#define MP1_SMN_C2PMSG_57__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_57__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_58
+#define MP1_SMN_C2PMSG_58__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_58__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_59
+#define MP1_SMN_C2PMSG_59__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_59__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_60
+#define MP1_SMN_C2PMSG_60__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_60__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_61
+#define MP1_SMN_C2PMSG_61__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_61__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_62
+#define MP1_SMN_C2PMSG_62__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_62__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_63
+#define MP1_SMN_C2PMSG_63__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_63__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_64
+#define MP1_SMN_C2PMSG_64__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_64__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_65
+#define MP1_SMN_C2PMSG_65__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_65__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_66
+#define MP1_SMN_C2PMSG_66__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_66__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_67
+#define MP1_SMN_C2PMSG_67__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_67__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_68
+#define MP1_SMN_C2PMSG_68__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_68__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_69
+#define MP1_SMN_C2PMSG_69__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_69__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_70
+#define MP1_SMN_C2PMSG_70__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_70__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_71
+#define MP1_SMN_C2PMSG_71__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_71__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_72
+#define MP1_SMN_C2PMSG_72__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_72__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_73
+#define MP1_SMN_C2PMSG_73__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_73__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_74
+#define MP1_SMN_C2PMSG_74__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_74__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_75
+#define MP1_SMN_C2PMSG_75__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_75__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_76
+#define MP1_SMN_C2PMSG_76__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_76__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_77
+#define MP1_SMN_C2PMSG_77__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_77__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_78
+#define MP1_SMN_C2PMSG_78__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_78__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_79
+#define MP1_SMN_C2PMSG_79__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_79__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_80
+#define MP1_SMN_C2PMSG_80__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_80__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_81
+#define MP1_SMN_C2PMSG_81__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_81__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_82
+#define MP1_SMN_C2PMSG_82__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_82__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_83
+#define MP1_SMN_C2PMSG_83__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_83__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_84
+#define MP1_SMN_C2PMSG_84__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_84__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_85
+#define MP1_SMN_C2PMSG_85__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_85__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_86
+#define MP1_SMN_C2PMSG_86__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_86__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_87
+#define MP1_SMN_C2PMSG_87__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_87__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_88
+#define MP1_SMN_C2PMSG_88__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_88__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_89
+#define MP1_SMN_C2PMSG_89__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_89__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_90
+#define MP1_SMN_C2PMSG_90__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_90__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_91
+#define MP1_SMN_C2PMSG_91__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_91__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_92
+#define MP1_SMN_C2PMSG_92__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_92__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_93
+#define MP1_SMN_C2PMSG_93__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_93__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_94
+#define MP1_SMN_C2PMSG_94__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_94__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_95
+#define MP1_SMN_C2PMSG_95__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_95__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_96
+#define MP1_SMN_C2PMSG_96__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_96__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_97
+#define MP1_SMN_C2PMSG_97__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_97__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_98
+#define MP1_SMN_C2PMSG_98__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_98__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_99
+#define MP1_SMN_C2PMSG_99__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_99__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_100
+#define MP1_SMN_C2PMSG_100__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_100__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_101
+#define MP1_SMN_C2PMSG_101__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_101__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_102
+#define MP1_SMN_C2PMSG_102__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_102__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_103
+#define MP1_SMN_C2PMSG_103__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_103__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_104
+#define MP1_SMN_C2PMSG_104__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_104__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_105
+#define MP1_SMN_C2PMSG_105__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_105__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_106
+#define MP1_SMN_C2PMSG_106__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_106__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_107
+#define MP1_SMN_C2PMSG_107__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_107__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_108
+#define MP1_SMN_C2PMSG_108__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_108__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_109
+#define MP1_SMN_C2PMSG_109__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_109__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_110
+#define MP1_SMN_C2PMSG_110__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_110__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_111
+#define MP1_SMN_C2PMSG_111__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_111__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_112
+#define MP1_SMN_C2PMSG_112__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_112__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_113
+#define MP1_SMN_C2PMSG_113__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_113__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_114
+#define MP1_SMN_C2PMSG_114__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_114__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_115
+#define MP1_SMN_C2PMSG_115__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_115__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_116
+#define MP1_SMN_C2PMSG_116__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_116__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_117
+#define MP1_SMN_C2PMSG_117__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_117__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_118
+#define MP1_SMN_C2PMSG_118__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_118__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_119
+#define MP1_SMN_C2PMSG_119__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_119__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_120
+#define MP1_SMN_C2PMSG_120__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_120__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_121
+#define MP1_SMN_C2PMSG_121__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_121__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_122
+#define MP1_SMN_C2PMSG_122__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_122__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_123
+#define MP1_SMN_C2PMSG_123__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_123__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_124
+#define MP1_SMN_C2PMSG_124__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_124__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_125
+#define MP1_SMN_C2PMSG_125__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_125__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_126
+#define MP1_SMN_C2PMSG_126__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_126__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_127
+#define MP1_SMN_C2PMSG_127__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_127__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_IH_CREDIT
+#define MP1_SMN_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                0x0
+#define MP1_SMN_IH_CREDIT__CLIENT_ID__SHIFT                                                                   0x10
+#define MP1_SMN_IH_CREDIT__CREDIT_VALUE_MASK                                                                  0x00000003L
+#define MP1_SMN_IH_CREDIT__CLIENT_ID_MASK                                                                     0x00FF0000L
+//MP1_SMN_IH_SW_INT
+#define MP1_SMN_IH_SW_INT__ID__SHIFT                                                                          0x0
+#define MP1_SMN_IH_SW_INT__VALID__SHIFT                                                                       0x8
+#define MP1_SMN_IH_SW_INT__ID_MASK                                                                            0x000000FFL
+#define MP1_SMN_IH_SW_INT__VALID_MASK                                                                         0x00000100L
+//MP1_SMN_IH_SW_INT_CTRL
+#define MP1_SMN_IH_SW_INT_CTRL__INT_MASK__SHIFT                                                               0x0
+#define MP1_SMN_IH_SW_INT_CTRL__INT_ACK__SHIFT                                                                0x8
+#define MP1_SMN_IH_SW_INT_CTRL__INT_MASK_MASK                                                                 0x00000001L
+#define MP1_SMN_IH_SW_INT_CTRL__INT_ACK_MASK                                                                  0x00000100L
+//MP1_SMN_FPS_CNT
+#define MP1_SMN_FPS_CNT__COUNT__SHIFT                                                                         0x0
+#define MP1_SMN_FPS_CNT__COUNT_MASK                                                                           0xFFFFFFFFL
+//MP1_SMN_PUB_CTRL
+#define MP1_SMN_PUB_CTRL__LX3_RESET__SHIFT                                                                    0x0
+#define MP1_SMN_PUB_CTRL__LX3_RESET_MASK                                                                      0x00000001L
+//MP1_SMN_EXT_SCRATCH0
+#define MP1_SMN_EXT_SCRATCH0__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH0__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH1
+#define MP1_SMN_EXT_SCRATCH1__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH1__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH2
+#define MP1_SMN_EXT_SCRATCH2__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH2__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH3
+#define MP1_SMN_EXT_SCRATCH3__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH3__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH4
+#define MP1_SMN_EXT_SCRATCH4__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH4__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH5
+#define MP1_SMN_EXT_SCRATCH5__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH5__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH6
+#define MP1_SMN_EXT_SCRATCH6__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH6__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH7
+#define MP1_SMN_EXT_SCRATCH7__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH7__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH8
+#define MP1_SMN_EXT_SCRATCH8__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH8__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH10
+#define MP1_SMN_EXT_SCRATCH10__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH10__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH11
+#define MP1_SMN_EXT_SCRATCH11__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH11__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH12
+#define MP1_SMN_EXT_SCRATCH12__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH12__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH13
+#define MP1_SMN_EXT_SCRATCH13__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH13__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH14
+#define MP1_SMN_EXT_SCRATCH14__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH14__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH15
+#define MP1_SMN_EXT_SCRATCH15__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH15__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH16
+#define MP1_SMN_EXT_SCRATCH16__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH16__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH17
+#define MP1_SMN_EXT_SCRATCH17__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH17__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH18
+#define MP1_SMN_EXT_SCRATCH18__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH18__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH19
+#define MP1_SMN_EXT_SCRATCH19__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH19__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH20
+#define MP1_SMN_EXT_SCRATCH20__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH20__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH21
+#define MP1_SMN_EXT_SCRATCH21__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH21__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH22
+#define MP1_SMN_EXT_SCRATCH22__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH22__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH23
+#define MP1_SMN_EXT_SCRATCH23__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH23__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH24
+#define MP1_SMN_EXT_SCRATCH24__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH24__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH25
+#define MP1_SMN_EXT_SCRATCH25__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH25__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH26
+#define MP1_SMN_EXT_SCRATCH26__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH26__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH27
+#define MP1_SMN_EXT_SCRATCH27__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH27__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH28
+#define MP1_SMN_EXT_SCRATCH28__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH28__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH29
+#define MP1_SMN_EXT_SCRATCH29__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH29__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH30
+#define MP1_SMN_EXT_SCRATCH30__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH30__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH31
+#define MP1_SMN_EXT_SCRATCH31__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH31__DATA_MASK                                                                      0xFFFFFFFFL
+
+
+// addressBlock: aid_mp_SmuMp1Pub_CruDec
+//MP1_FIRMWARE_FLAGS
+#define MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT                                                         0x0
+#define MP1_FIRMWARE_FLAGS__RESERVED__SHIFT                                                                   0x1
+#define MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK                                                           0x00000001L
+#define MP1_FIRMWARE_FLAGS__RESERVED_MASK                                                                     0xFFFFFFFEL
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_9_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_9_0_offset.h
new file mode 100644
index 000000000000..033f2796c1e3
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_9_0_offset.h
@@ -0,0 +1,10002 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _nbio_7_9_0_OFFSET_HEADER
+#define _nbio_7_9_0_OFFSET_HEADER
+
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_SYSDEC
+// base address: 0x0
+#define regBIF_BX0_PCIE_INDEX                                                                           0x000c
+#define regBIF_BX0_PCIE_INDEX_BASE_IDX                                                                  0
+#define regBIF_BX0_PCIE_DATA                                                                            0x000d
+#define regBIF_BX0_PCIE_DATA_BASE_IDX                                                                   0
+#define regBIF_BX0_PCIE_INDEX2                                                                          0x000e
+#define regBIF_BX0_PCIE_INDEX2_BASE_IDX                                                                 0
+#define regBIF_BX0_PCIE_DATA2                                                                           0x000f
+#define regBIF_BX0_PCIE_DATA2_BASE_IDX                                                                  0
+#define regBIF_BX0_PCIE_INDEX_HI                                                                        0x0010
+#define regBIF_BX0_PCIE_INDEX_HI_BASE_IDX                                                               0
+#define regBIF_BX0_PCIE_INDEX2_HI                                                                       0x0011
+#define regBIF_BX0_PCIE_INDEX2_HI_BASE_IDX                                                              0
+#define regBIF_BX0_SBIOS_SCRATCH_0                                                                      0x0034
+#define regBIF_BX0_SBIOS_SCRATCH_0_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_1                                                                      0x0035
+#define regBIF_BX0_SBIOS_SCRATCH_1_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_2                                                                      0x0036
+#define regBIF_BX0_SBIOS_SCRATCH_2_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_3                                                                      0x0037
+#define regBIF_BX0_SBIOS_SCRATCH_3_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_0                                                                       0x0038
+#define regBIF_BX0_BIOS_SCRATCH_0_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_1                                                                       0x0039
+#define regBIF_BX0_BIOS_SCRATCH_1_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_2                                                                       0x003a
+#define regBIF_BX0_BIOS_SCRATCH_2_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_3                                                                       0x003b
+#define regBIF_BX0_BIOS_SCRATCH_3_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_4                                                                       0x003c
+#define regBIF_BX0_BIOS_SCRATCH_4_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_5                                                                       0x003d
+#define regBIF_BX0_BIOS_SCRATCH_5_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_6                                                                       0x003e
+#define regBIF_BX0_BIOS_SCRATCH_6_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_7                                                                       0x003f
+#define regBIF_BX0_BIOS_SCRATCH_7_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_8                                                                       0x0040
+#define regBIF_BX0_BIOS_SCRATCH_8_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_9                                                                       0x0041
+#define regBIF_BX0_BIOS_SCRATCH_9_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_10                                                                      0x0042
+#define regBIF_BX0_BIOS_SCRATCH_10_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_11                                                                      0x0043
+#define regBIF_BX0_BIOS_SCRATCH_11_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_12                                                                      0x0044
+#define regBIF_BX0_BIOS_SCRATCH_12_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_13                                                                      0x0045
+#define regBIF_BX0_BIOS_SCRATCH_13_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_14                                                                      0x0046
+#define regBIF_BX0_BIOS_SCRATCH_14_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_15                                                                      0x0047
+#define regBIF_BX0_BIOS_SCRATCH_15_BASE_IDX                                                             1
+#define regBIF_BX0_BIF_RLC_INTR_CNTL                                                                    0x004c
+#define regBIF_BX0_BIF_RLC_INTR_CNTL_BASE_IDX                                                           1
+#define regBIF_BX0_BIF_VCE_INTR_CNTL                                                                    0x004d
+#define regBIF_BX0_BIF_VCE_INTR_CNTL_BASE_IDX                                                           1
+#define regBIF_BX0_BIF_UVD_INTR_CNTL                                                                    0x004e
+#define regBIF_BX0_BIF_UVD_INTR_CNTL_BASE_IDX                                                           1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR0                                                                0x006c
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR0_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0                                                          0x006d
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR1                                                                0x006e
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR1_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1                                                          0x006f
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR2                                                                0x0070
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR2_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2                                                          0x0071
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR3                                                                0x0072
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR3_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3                                                          0x0073
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR4                                                                0x0074
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR4_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4                                                          0x0075
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR5                                                                0x0076
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR5_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5                                                          0x0077
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR6                                                                0x0078
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR6_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6                                                          0x0079
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR7                                                                0x007a
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR7_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7                                                          0x007b
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_CNTL                                                                 0x007c
+#define regBIF_BX0_GFX_MMIOREG_CAM_CNTL_BASE_IDX                                                        1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL                                                             0x007d
+#define regBIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL_BASE_IDX                                                    1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ONE_CPL                                                              0x007e
+#define regBIF_BX0_GFX_MMIOREG_CAM_ONE_CPL_BASE_IDX                                                     1
+#define regBIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL                                                     0x007f
+#define regBIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL_BASE_IDX                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_0                                                                     0x0080
+#define regBIF_BX0_DRIVER_SCRATCH_0_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_1                                                                     0x0081
+#define regBIF_BX0_DRIVER_SCRATCH_1_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_2                                                                     0x0082
+#define regBIF_BX0_DRIVER_SCRATCH_2_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_3                                                                     0x0083
+#define regBIF_BX0_DRIVER_SCRATCH_3_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_4                                                                     0x0084
+#define regBIF_BX0_DRIVER_SCRATCH_4_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_5                                                                     0x0085
+#define regBIF_BX0_DRIVER_SCRATCH_5_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_6                                                                     0x0086
+#define regBIF_BX0_DRIVER_SCRATCH_6_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_7                                                                     0x0087
+#define regBIF_BX0_DRIVER_SCRATCH_7_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_8                                                                     0x0088
+#define regBIF_BX0_DRIVER_SCRATCH_8_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_9                                                                     0x0089
+#define regBIF_BX0_DRIVER_SCRATCH_9_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_10                                                                    0x008a
+#define regBIF_BX0_DRIVER_SCRATCH_10_BASE_IDX                                                           1
+#define regBIF_BX0_DRIVER_SCRATCH_11                                                                    0x008b
+#define regBIF_BX0_DRIVER_SCRATCH_11_BASE_IDX                                                           1
+#define regBIF_BX0_DRIVER_SCRATCH_12                                                                    0x008c
+#define regBIF_BX0_DRIVER_SCRATCH_12_BASE_IDX                                                           1
+#define regBIF_BX0_DRIVER_SCRATCH_13                                                                    0x008d
+#define regBIF_BX0_DRIVER_SCRATCH_13_BASE_IDX                                                           1
+#define regBIF_BX0_DRIVER_SCRATCH_14                                                                    0x008e
+#define regBIF_BX0_DRIVER_SCRATCH_14_BASE_IDX                                                           1
+#define regBIF_BX0_DRIVER_SCRATCH_15                                                                    0x008f
+#define regBIF_BX0_DRIVER_SCRATCH_15_BASE_IDX                                                           1
+#define regBIF_BX0_FW_SCRATCH_0                                                                         0x0090
+#define regBIF_BX0_FW_SCRATCH_0_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_1                                                                         0x0091
+#define regBIF_BX0_FW_SCRATCH_1_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_2                                                                         0x0092
+#define regBIF_BX0_FW_SCRATCH_2_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_3                                                                         0x0093
+#define regBIF_BX0_FW_SCRATCH_3_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_4                                                                         0x0094
+#define regBIF_BX0_FW_SCRATCH_4_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_5                                                                         0x0095
+#define regBIF_BX0_FW_SCRATCH_5_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_6                                                                         0x0096
+#define regBIF_BX0_FW_SCRATCH_6_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_7                                                                         0x0097
+#define regBIF_BX0_FW_SCRATCH_7_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_8                                                                         0x0098
+#define regBIF_BX0_FW_SCRATCH_8_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_9                                                                         0x0099
+#define regBIF_BX0_FW_SCRATCH_9_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_10                                                                        0x009a
+#define regBIF_BX0_FW_SCRATCH_10_BASE_IDX                                                               1
+#define regBIF_BX0_FW_SCRATCH_11                                                                        0x009b
+#define regBIF_BX0_FW_SCRATCH_11_BASE_IDX                                                               1
+#define regBIF_BX0_FW_SCRATCH_12                                                                        0x009c
+#define regBIF_BX0_FW_SCRATCH_12_BASE_IDX                                                               1
+#define regBIF_BX0_FW_SCRATCH_13                                                                        0x009d
+#define regBIF_BX0_FW_SCRATCH_13_BASE_IDX                                                               1
+#define regBIF_BX0_FW_SCRATCH_14                                                                        0x009e
+#define regBIF_BX0_FW_SCRATCH_14_BASE_IDX                                                               1
+#define regBIF_BX0_FW_SCRATCH_15                                                                        0x009f
+#define regBIF_BX0_FW_SCRATCH_15_BASE_IDX                                                               1
+#define regBIF_BX0_SBIOS_SCRATCH_4                                                                      0x00a0
+#define regBIF_BX0_SBIOS_SCRATCH_4_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_5                                                                      0x00a1
+#define regBIF_BX0_SBIOS_SCRATCH_5_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_6                                                                      0x00a2
+#define regBIF_BX0_SBIOS_SCRATCH_6_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_7                                                                      0x00a3
+#define regBIF_BX0_SBIOS_SCRATCH_7_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_8                                                                      0x00a4
+#define regBIF_BX0_SBIOS_SCRATCH_8_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_9                                                                      0x00a5
+#define regBIF_BX0_SBIOS_SCRATCH_9_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_10                                                                     0x00a6
+#define regBIF_BX0_SBIOS_SCRATCH_10_BASE_IDX                                                            1
+#define regBIF_BX0_SBIOS_SCRATCH_11                                                                     0x00a7
+#define regBIF_BX0_SBIOS_SCRATCH_11_BASE_IDX                                                            1
+#define regBIF_BX0_SBIOS_SCRATCH_12                                                                     0x00a8
+#define regBIF_BX0_SBIOS_SCRATCH_12_BASE_IDX                                                            1
+#define regBIF_BX0_SBIOS_SCRATCH_13                                                                     0x00a9
+#define regBIF_BX0_SBIOS_SCRATCH_13_BASE_IDX                                                            1
+#define regBIF_BX0_SBIOS_SCRATCH_14                                                                     0x00aa
+#define regBIF_BX0_SBIOS_SCRATCH_14_BASE_IDX                                                            1
+#define regBIF_BX0_SBIOS_SCRATCH_15                                                                     0x00ab
+#define regBIF_BX0_SBIOS_SCRATCH_15_BASE_IDX                                                            1
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwn_dev0_BIFDEC1
+// base address: 0x0
+#define regRCC_DWN_DEV0_0_DN_PCIE_RESERVED                                                              0x0060
+#define regRCC_DWN_DEV0_0_DN_PCIE_RESERVED_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_0_DN_PCIE_SCRATCH                                                               0x0061
+#define regRCC_DWN_DEV0_0_DN_PCIE_SCRATCH_BASE_IDX                                                      2
+#define regRCC_DWN_DEV0_0_DN_PCIE_CNTL                                                                  0x0063
+#define regRCC_DWN_DEV0_0_DN_PCIE_CNTL_BASE_IDX                                                         2
+#define regRCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL                                                           0x0064
+#define regRCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL_BASE_IDX                                                  2
+#define regRCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2                                                              0x0065
+#define regRCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL                                                              0x0066
+#define regRCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL                                                              0x0067
+#define regRCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_F0                                                              0x0068
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_F0_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC                                                            0x0069
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC_BASE_IDX                                                   2
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2                                                           0x006a
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2_BASE_IDX                                                  2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwnp_dev0_BIFDEC1
+// base address: 0x0
+#define regRCC_DWNP_DEV0_0_PCIE_ERR_CNTL                                                                0x006c
+#define regRCC_DWNP_DEV0_0_PCIE_ERR_CNTL_BASE_IDX                                                       2
+#define regRCC_DWNP_DEV0_0_PCIE_RX_CNTL                                                                 0x006d
+#define regRCC_DWNP_DEV0_0_PCIE_RX_CNTL_BASE_IDX                                                        2
+#define regRCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL                                                           0x006e
+#define regRCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL_BASE_IDX                                                  2
+#define regRCC_DWNP_DEV0_0_PCIE_LC_CNTL2                                                                0x006f
+#define regRCC_DWNP_DEV0_0_PCIE_LC_CNTL2_BASE_IDX                                                       2
+#define regRCC_DWNP_DEV0_0_PCIEP_STRAP_MISC                                                             0x0070
+#define regRCC_DWNP_DEV0_0_PCIEP_STRAP_MISC_BASE_IDX                                                    2
+#define regRCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP                                                         0x0071
+#define regRCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP_BASE_IDX                                                2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_ep_dev0_BIFDEC1
+// base address: 0x0
+#define regRCC_EP_DEV0_0_EP_PCIE_SCRATCH                                                                0x0040
+#define regRCC_EP_DEV0_0_EP_PCIE_SCRATCH_BASE_IDX                                                       2
+#define regRCC_EP_DEV0_0_EP_PCIE_CNTL                                                                   0x0042
+#define regRCC_EP_DEV0_0_EP_PCIE_CNTL_BASE_IDX                                                          2
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_CNTL                                                               0x0043
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_STATUS                                                             0x0044
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_STATUS_BASE_IDX                                                    2
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL2                                                               0x0045
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL2_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_BUS_CNTL                                                               0x0046
+#define regRCC_EP_DEV0_0_EP_PCIE_BUS_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_CFG_CNTL                                                               0x0047
+#define regRCC_EP_DEV0_0_EP_PCIE_CFG_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL                                                            0x0049
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0                                               0x004a
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1                                               0x004a
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2                                               0x004a
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3                                               0x004a
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4                                               0x004b
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5                                               0x004b
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6                                               0x004b
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7                                               0x004b
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC                                                             0x004c
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC_BASE_IDX                                                    2
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2                                                            0x004d
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP                                                             0x004f
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP_BASE_IDX                                                    2
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                               0x0050
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL                                                            0x0050
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                               0x0050
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                               0x0051
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                               0x0051
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                               0x0051
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                               0x0051
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                               0x0052
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                               0x0052
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                               0x0052
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_PME_CONTROL                                                            0x0052
+#define regRCC_EP_DEV0_0_EP_PCIE_PME_CONTROL_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_0_EP_PCIEP_RESERVED                                                              0x0053
+#define regRCC_EP_DEV0_0_EP_PCIEP_RESERVED_BASE_IDX                                                     2
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_CNTL                                                                0x0055
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_CNTL_BASE_IDX                                                       2
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID                                                        0x0056
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID_BASE_IDX                                               2
+#define regRCC_EP_DEV0_0_EP_PCIE_ERR_CNTL                                                               0x0057
+#define regRCC_EP_DEV0_0_EP_PCIE_ERR_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL                                                                0x0058
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL_BASE_IDX                                                       2
+#define regRCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL                                                          0x0059
+#define regRCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL_BASE_IDX                                                 2
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_pf_SYSPFVFDEC
+// base address: 0x0
+#define regBIF_BX_PF0_MM_INDEX                                                                          0x0000
+#define regBIF_BX_PF0_MM_INDEX_BASE_IDX                                                                 0
+#define regBIF_BX_PF0_MM_DATA                                                                           0x0001
+#define regBIF_BX_PF0_MM_DATA_BASE_IDX                                                                  0
+#define regBIF_BX_PF0_MM_INDEX_HI                                                                       0x0006
+#define regBIF_BX_PF0_MM_INDEX_HI_BASE_IDX                                                              0
+#define regBIF_BX_PF0_RSMU_INDEX                                                                        0x0000
+#define regBIF_BX_PF0_RSMU_INDEX_BASE_IDX                                                               1
+#define regBIF_BX_PF0_RSMU_DATA                                                                         0x0001
+#define regBIF_BX_PF0_RSMU_DATA_BASE_IDX                                                                1
+#define regBIF_BX_PF0_RSMU_INDEX_HI                                                                     0x0002
+#define regBIF_BX_PF0_RSMU_INDEX_HI_BASE_IDX                                                            1
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_BIFDEC1
+// base address: 0x0
+#define regBIF_BX0_CC_BIF_BX_STRAP0                                                                     0x00e2
+#define regBIF_BX0_CC_BIF_BX_STRAP0_BASE_IDX                                                            2
+#define regBIF_BX0_CC_BIF_BX_PINSTRAP0                                                                  0x00e4
+#define regBIF_BX0_CC_BIF_BX_PINSTRAP0_BASE_IDX                                                         2
+#define regBIF_BX0_BIF_MM_INDACCESS_CNTL                                                                0x00e6
+#define regBIF_BX0_BIF_MM_INDACCESS_CNTL_BASE_IDX                                                       2
+#define regBIF_BX0_BUS_CNTL                                                                             0x00e7
+#define regBIF_BX0_BUS_CNTL_BASE_IDX                                                                    2
+#define regBIF_BX0_BIF_SCRATCH0                                                                         0x00e8
+#define regBIF_BX0_BIF_SCRATCH0_BASE_IDX                                                                2
+#define regBIF_BX0_BIF_SCRATCH1                                                                         0x00e9
+#define regBIF_BX0_BIF_SCRATCH1_BASE_IDX                                                                2
+#define regBIF_BX0_BX_RESET_EN                                                                          0x00ed
+#define regBIF_BX0_BX_RESET_EN_BASE_IDX                                                                 2
+#define regBIF_BX0_MM_CFGREGS_CNTL                                                                      0x00ee
+#define regBIF_BX0_MM_CFGREGS_CNTL_BASE_IDX                                                             2
+#define regBIF_BX0_BX_RESET_CNTL                                                                        0x00f0
+#define regBIF_BX0_BX_RESET_CNTL_BASE_IDX                                                               2
+#define regBIF_BX0_INTERRUPT_CNTL                                                                       0x00f1
+#define regBIF_BX0_INTERRUPT_CNTL_BASE_IDX                                                              2
+#define regBIF_BX0_INTERRUPT_CNTL2                                                                      0x00f2
+#define regBIF_BX0_INTERRUPT_CNTL2_BASE_IDX                                                             2
+#define regBIF_BX0_CLKREQB_PAD_CNTL                                                                     0x00f8
+#define regBIF_BX0_CLKREQB_PAD_CNTL_BASE_IDX                                                            2
+#define regBIF_BX0_BIF_FEATURES_CONTROL_MISC                                                            0x00fb
+#define regBIF_BX0_BIF_FEATURES_CONTROL_MISC_BASE_IDX                                                   2
+#define regBIF_BX0_HDP_ATOMIC_CONTROL_MISC                                                              0x00fc
+#define regBIF_BX0_HDP_ATOMIC_CONTROL_MISC_BASE_IDX                                                     2
+#define regBIF_BX0_BIF_DOORBELL_CNTL                                                                    0x00fd
+#define regBIF_BX0_BIF_DOORBELL_CNTL_BASE_IDX                                                           2
+#define regBIF_BX0_BIF_DOORBELL_INT_CNTL                                                                0x00fe
+#define regBIF_BX0_BIF_DOORBELL_INT_CNTL_BASE_IDX                                                       2
+#define regBIF_BX0_BIF_FB_EN                                                                            0x0100
+#define regBIF_BX0_BIF_FB_EN_BASE_IDX                                                                   2
+#define regBIF_BX0_BIF_INTR_CNTL                                                                        0x0101
+#define regBIF_BX0_BIF_INTR_CNTL_BASE_IDX                                                               2
+#define regBIF_BX0_BIF_MST_TRANS_PENDING_VF                                                             0x0109
+#define regBIF_BX0_BIF_MST_TRANS_PENDING_VF_BASE_IDX                                                    2
+#define regBIF_BX0_BIF_SLV_TRANS_PENDING_VF                                                             0x010a
+#define regBIF_BX0_BIF_SLV_TRANS_PENDING_VF_BASE_IDX                                                    2
+#define regBIF_BX0_BACO_CNTL                                                                            0x010b
+#define regBIF_BX0_BACO_CNTL_BASE_IDX                                                                   2
+#define regBIF_BX0_BIF_BACO_EXIT_TIME0                                                                  0x010c
+#define regBIF_BX0_BIF_BACO_EXIT_TIME0_BASE_IDX                                                         2
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER1                                                                 0x010d
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER1_BASE_IDX                                                        2
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER2                                                                 0x010e
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER2_BASE_IDX                                                        2
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER3                                                                 0x010f
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER3_BASE_IDX                                                        2
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER4                                                                 0x0110
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER4_BASE_IDX                                                        2
+#define regBIF_BX0_MEM_TYPE_CNTL                                                                        0x0111
+#define regBIF_BX0_MEM_TYPE_CNTL_BASE_IDX                                                               2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_CNTL                                                               0x0113
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_CNTL_BASE_IDX                                                      2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_0                                                                  0x0114
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_0_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_1                                                                  0x0115
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_1_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_2                                                                  0x0116
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_2_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_3                                                                  0x0117
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_3_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_4                                                                  0x0118
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_4_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_5                                                                  0x0119
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_5_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_6                                                                  0x011a
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_6_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_7                                                                  0x011b
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_7_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_8                                                                  0x011c
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_8_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_9                                                                  0x011d
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_9_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_10                                                                 0x011e
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_10_BASE_IDX                                                        2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_11                                                                 0x011f
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_11_BASE_IDX                                                        2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_12                                                                 0x0120
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_12_BASE_IDX                                                        2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_13                                                                 0x0121
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_13_BASE_IDX                                                        2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_14                                                                 0x0122
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_14_BASE_IDX                                                        2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_15                                                                 0x0123
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_15_BASE_IDX                                                        2
+#define regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL                                                             0x012d
+#define regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL_BASE_IDX                                                    2
+#define regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL                                                             0x012e
+#define regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL_BASE_IDX                                                    2
+#define regBIF_BX0_BIF_RB_CNTL                                                                          0x012f
+#define regBIF_BX0_BIF_RB_CNTL_BASE_IDX                                                                 2
+#define regBIF_BX0_BIF_RB_BASE                                                                          0x0130
+#define regBIF_BX0_BIF_RB_BASE_BASE_IDX                                                                 2
+#define regBIF_BX0_BIF_RB_RPTR                                                                          0x0131
+#define regBIF_BX0_BIF_RB_RPTR_BASE_IDX                                                                 2
+#define regBIF_BX0_BIF_RB_WPTR                                                                          0x0132
+#define regBIF_BX0_BIF_RB_WPTR_BASE_IDX                                                                 2
+#define regBIF_BX0_BIF_RB_WPTR_ADDR_HI                                                                  0x0133
+#define regBIF_BX0_BIF_RB_WPTR_ADDR_HI_BASE_IDX                                                         2
+#define regBIF_BX0_BIF_RB_WPTR_ADDR_LO                                                                  0x0134
+#define regBIF_BX0_BIF_RB_WPTR_ADDR_LO_BASE_IDX                                                         2
+#define regBIF_BX0_MAILBOX_INDEX                                                                        0x0135
+#define regBIF_BX0_MAILBOX_INDEX_BASE_IDX                                                               2
+#define regBIF_BX0_BIF_MP1_INTR_CTRL                                                                    0x0142
+#define regBIF_BX0_BIF_MP1_INTR_CTRL_BASE_IDX                                                           2
+#define regBIF_BX0_BIF_PERSTB_PAD_CNTL                                                                  0x0146
+#define regBIF_BX0_BIF_PERSTB_PAD_CNTL_BASE_IDX                                                         2
+#define regBIF_BX0_BIF_PX_EN_PAD_CNTL                                                                   0x0147
+#define regBIF_BX0_BIF_PX_EN_PAD_CNTL_BASE_IDX                                                          2
+#define regBIF_BX0_BIF_REFPADKIN_PAD_CNTL                                                               0x0148
+#define regBIF_BX0_BIF_REFPADKIN_PAD_CNTL_BASE_IDX                                                      2
+#define regBIF_BX0_BIF_CLKREQB_PAD_CNTL                                                                 0x0149
+#define regBIF_BX0_BIF_CLKREQB_PAD_CNTL_BASE_IDX                                                        2
+#define regBIF_BX0_BIF_PWRBRK_PAD_CNTL                                                                  0x014a
+#define regBIF_BX0_BIF_PWRBRK_PAD_CNTL_BASE_IDX                                                         2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_BIFDEC1
+// base address: 0x0
+#define regRCC_DEV0_0_RCC_ERR_INT_CNTL                                                                  0x0086
+#define regRCC_DEV0_0_RCC_ERR_INT_CNTL_BASE_IDX                                                         2
+#define regRCC_DEV0_0_RCC_BACO_CNTL_MISC                                                                0x0087
+#define regRCC_DEV0_0_RCC_BACO_CNTL_MISC_BASE_IDX                                                       2
+#define regRCC_DEV0_0_RCC_RESET_EN                                                                      0x0088
+#define regRCC_DEV0_0_RCC_RESET_EN_BASE_IDX                                                             2
+#define regRCC_DEV0_0_RCC_VDM_SUPPORT                                                                   0x0089
+#define regRCC_DEV0_0_RCC_VDM_SUPPORT_BASE_IDX                                                          2
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0                                                            0x008a
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1                                                            0x008b
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_GPUIOV_REGION                                                                 0x008c
+#define regRCC_DEV0_0_RCC_GPUIOV_REGION_BASE_IDX                                                        2
+#define regRCC_DEV0_0_RCC_GPU_HOSTVM_EN                                                                 0x008d
+#define regRCC_DEV0_0_RCC_GPU_HOSTVM_EN_BASE_IDX                                                        2
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL                                                         0x008e
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL_BASE_IDX                                                2
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET                                                   0x008f
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET_BASE_IDX                                          2
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE                                                         0x008f
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE_BASE_IDX                                                2
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE0                                                               0x00be
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE0_BASE_IDX                                                      2
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE1                                                               0x00bf
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE1_BASE_IDX                                                      2
+#define regRCC_DEV0_0_RCC_BUS_CNTL                                                                      0x00c1
+#define regRCC_DEV0_0_RCC_BUS_CNTL_BASE_IDX                                                             2
+#define regRCC_DEV0_0_RCC_CONFIG_CNTL                                                                   0x00c2
+#define regRCC_DEV0_0_RCC_CONFIG_CNTL_BASE_IDX                                                          2
+#define regRCC_DEV0_0_RCC_CONFIG_F0_BASE                                                                0x00c6
+#define regRCC_DEV0_0_RCC_CONFIG_F0_BASE_BASE_IDX                                                       2
+#define regRCC_DEV0_0_RCC_CONFIG_APER_SIZE                                                              0x00c7
+#define regRCC_DEV0_0_RCC_CONFIG_APER_SIZE_BASE_IDX                                                     2
+#define regRCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE                                                          0x00c8
+#define regRCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE_BASE_IDX                                                 2
+#define regRCC_DEV0_0_RCC_XDMA_LO                                                                       0x00c9
+#define regRCC_DEV0_0_RCC_XDMA_LO_BASE_IDX                                                              2
+#define regRCC_DEV0_0_RCC_XDMA_HI                                                                       0x00ca
+#define regRCC_DEV0_0_RCC_XDMA_HI_BASE_IDX                                                              2
+#define regRCC_DEV0_0_RCC_FEATURES_CONTROL_MISC                                                         0x00cb
+#define regRCC_DEV0_0_RCC_FEATURES_CONTROL_MISC_BASE_IDX                                                2
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL1                                                                  0x00cc
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL1_BASE_IDX                                                         2
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST0                                                                  0x00cd
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST0_BASE_IDX                                                         2
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST1                                                                  0x00ce
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST1_BASE_IDX                                                         2
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL2                                                                  0x00cf
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL2_BASE_IDX                                                         2
+#define regRCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM                                                           0x00d0
+#define regRCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM_BASE_IDX                                                  2
+#define regRCC_DEV0_0_RCC_HOST_BUSNUM                                                                   0x00d1
+#define regRCC_DEV0_0_RCC_HOST_BUSNUM_BASE_IDX                                                          2
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI                                                            0x00d2
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO                                                            0x00d3
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI                                                            0x00d4
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO                                                            0x00d5
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI                                                            0x00d6
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO                                                            0x00d7
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI                                                            0x00d8
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO                                                            0x00d9
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST0                                                              0x00da
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST0_BASE_IDX                                                     2
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST1                                                              0x00db
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST1_BASE_IDX                                                     2
+#define regRCC_DEV0_0_RCC_DEV0_LINK_CNTL                                                                0x00dd
+#define regRCC_DEV0_0_RCC_DEV0_LINK_CNTL_BASE_IDX                                                       2
+#define regRCC_DEV0_0_RCC_CMN_LINK_CNTL                                                                 0x00de
+#define regRCC_DEV0_0_RCC_CMN_LINK_CNTL_BASE_IDX                                                        2
+#define regRCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE                                                        0x00df
+#define regRCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE_BASE_IDX                                               2
+#define regRCC_DEV0_0_RCC_LTR_LSWITCH_CNTL                                                              0x00e0
+#define regRCC_DEV0_0_RCC_LTR_LSWITCH_CNTL_BASE_IDX                                                     2
+#define regRCC_DEV0_0_RCC_MH_ARB_CNTL                                                                   0x00e1
+#define regRCC_DEV0_0_RCC_MH_ARB_CNTL_BASE_IDX                                                          2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_LO                                                          0x0400
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_HI                                                          0x0401
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_MSG_DATA                                                         0x0402
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_CONTROL                                                          0x0403
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_LO                                                          0x0404
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_HI                                                          0x0405
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_MSG_DATA                                                         0x0406
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_CONTROL                                                          0x0407
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_LO                                                          0x0408
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_HI                                                          0x0409
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_MSG_DATA                                                         0x040a
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_CONTROL                                                          0x040b
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_LO                                                          0x040c
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_HI                                                          0x040d
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_MSG_DATA                                                         0x040e
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_CONTROL                                                          0x040f
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_GFXMSIX_PBA                                                                    0x0800
+#define regRCC_DEV0_EPF0_GFXMSIX_PBA_BASE_IDX 4
+
+
+// addressBlock: aid_nbio_nbif0_rcc_strap_BIFDEC1
+// base address: 0x0
+#define regRCC_STRAP0_RCC_BIF_STRAP0                                                                    0x0000
+#define regRCC_STRAP0_RCC_BIF_STRAP0_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP1                                                                    0x0001
+#define regRCC_STRAP0_RCC_BIF_STRAP1_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP2                                                                    0x0002
+#define regRCC_STRAP0_RCC_BIF_STRAP2_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP3                                                                    0x0003
+#define regRCC_STRAP0_RCC_BIF_STRAP3_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP4                                                                    0x0004
+#define regRCC_STRAP0_RCC_BIF_STRAP4_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP5                                                                    0x0005
+#define regRCC_STRAP0_RCC_BIF_STRAP5_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP6                                                                    0x0006
+#define regRCC_STRAP0_RCC_BIF_STRAP6_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP0                                                              0x0007
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP0_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP1                                                              0x0008
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP1_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP10                                                             0x0009
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP10_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP11                                                             0x000a
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP11_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP12                                                             0x000b
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP12_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP13                                                             0x000c
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP13_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP14                                                             0x000d
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP14_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP2                                                              0x000e
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP2_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP3                                                              0x000f
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP3_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP4                                                              0x0010
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP4_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP5                                                              0x0011
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP5_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP6                                                              0x0012
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP6_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP7                                                              0x0013
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP7_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP8                                                              0x0014
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP8_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP9                                                              0x0015
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP9_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0                                                              0x0016
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP1                                                              0x0017
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP1_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP13                                                             0x0018
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP13_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP14                                                             0x0019
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP14_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP15                                                             0x001a
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP15_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP16                                                             0x001b
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP16_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP17                                                             0x001c
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP17_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP18                                                             0x001d
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP18_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP2                                                              0x001e
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP2_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP26                                                             0x001f
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP26_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP3                                                              0x0020
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP3_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP4                                                              0x0021
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP4_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP5                                                              0x0022
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP5_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP8                                                              0x0024
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP8_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP9                                                              0x0025
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP9_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP0                                                              0x0026
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP0_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP2                                                              0x0032
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP2_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP20                                                             0x0033
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP20_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP21                                                             0x0034
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP21_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP22                                                             0x0035
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP22_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP23                                                             0x0036
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP23_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP24                                                             0x0037
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP24_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP25                                                             0x0038
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP25_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP3                                                              0x0039
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP3_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP4                                                              0x003a
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP4_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP5                                                              0x003b
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP5_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP6                                                              0x003c
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP6_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP7                                                              0x003d
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP7_BASE_IDX                                                     2
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_pf_BIFPFVFDEC1
+// base address: 0x0
+#define regBIF_BX_PF0_BIF_BME_STATUS                                                                    0x00eb
+#define regBIF_BX_PF0_BIF_BME_STATUS_BASE_IDX                                                           2
+#define regBIF_BX_PF0_BIF_ATOMIC_ERR_LOG                                                                0x00ec
+#define regBIF_BX_PF0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                                       2
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                              0x00f3
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                                     2
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                               0x00f4
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                                      2
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL                                                   0x00f5
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                          2
+#define regBIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL                                                      0x00f6
+#define regBIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                             2
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL                                                      0x00f7
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                             2
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                                 0x00f9
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                                        2
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                            0x00fa
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                                   2
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_REQ                                                                 0x0106
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_REQ_BASE_IDX                                                        2
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_DONE                                                                0x0107
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_DONE_BASE_IDX                                                       2
+#define regBIF_BX_PF0_BIF_TRANS_PENDING                                                                 0x0108
+#define regBIF_BX_PF0_BIF_TRANS_PENDING_BASE_IDX                                                        2
+#define regBIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS                                                          0x0112
+#define regBIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                                 2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0                                                            0x0136
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1                                                            0x0137
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2                                                            0x0138
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3                                                            0x0139
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0                                                            0x013a
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1                                                            0x013b
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2                                                            0x013c
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW3                                                            0x013d
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_CONTROL                                                                   0x013e
+#define regBIF_BX_PF0_MAILBOX_CONTROL_BASE_IDX                                                          2
+#define regBIF_BX_PF0_MAILBOX_INT_CNTL                                                                  0x013f
+#define regBIF_BX_PF0_MAILBOX_INT_CNTL_BASE_IDX                                                         2
+#define regBIF_BX_PF0_BIF_VMHV_MAILBOX                                                                  0x0140
+#define regBIF_BX_PF0_BIF_VMHV_MAILBOX_BASE_IDX                                                         2
+#define regBIF_BX_PF0_PARTITION_COMPUTE_CAP                                                             0x0161
+#define regBIF_BX_PF0_PARTITION_COMPUTE_CAP_BASE_IDX                                                    2
+#define regBIF_BX_PF0_PARTITION_MEM_CAP                                                                 0x0162
+#define regBIF_BX_PF0_PARTITION_MEM_CAP_BASE_IDX                                                        2
+#define regBIF_BX_PF0_PARTITION_COMPUTE_STATUS                                                          0x0163
+#define regBIF_BX_PF0_PARTITION_COMPUTE_STATUS_BASE_IDX                                                 2
+#define regBIF_BX_PF0_PARTITION_MEM_STATUS                                                              0x0164
+#define regBIF_BX_PF0_PARTITION_MEM_STATUS_BASE_IDX                                                     2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_BIFPFVFDEC1[13440..14975]
+// base address: 0x3480
+#define regRCC_DEV0_EPF0_RCC_ERR_LOG                                                                    0x0085
+#define regRCC_DEV0_EPF0_RCC_ERR_LOG_BASE_IDX                                                           2
+#define regRCC_DEV0_EPF0_RCC_DOORBELL_APER_EN                                                           0x00c0
+#define regRCC_DEV0_EPF0_RCC_DOORBELL_APER_EN_BASE_IDX                                                  2
+#define regRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE                                                             0x00c3
+#define regRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE_BASE_IDX                                                    2
+#define regRCC_DEV0_EPF0_RCC_CONFIG_RESERVED                                                            0x00c4
+#define regRCC_DEV0_EPF0_RCC_CONFIG_RESERVED_BASE_IDX                                                   2
+#define regRCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER                                                        0x00c5
+#define regRCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                               2
+
+
+// addressBlock: aid_nbio_nbif0_gdc_GDCDEC
+// base address: 0x0
+#define regGDC0_A2S_CNTL_CL0 0x0000
+#define regGDC0_A2S_CNTL_CL0_BASE_IDX 3
+#define regGDC0_A2S_CNTL_CL1 0x0001
+#define regGDC0_A2S_CNTL_CL1_BASE_IDX 3
+#define regGDC0_A2S_CNTL3_CL0 0x0018
+#define regGDC0_A2S_CNTL3_CL0_BASE_IDX 3
+#define regGDC0_A2S_CNTL3_CL1 0x0019
+#define regGDC0_A2S_CNTL3_CL1_BASE_IDX 3
+#define regGDC0_A2S_CNTL_SW0 0x0030
+#define regGDC0_A2S_CNTL_SW0_BASE_IDX 3
+#define regGDC0_A2S_CNTL_SW1 0x0031
+#define regGDC0_A2S_CNTL_SW1_BASE_IDX 3
+#define regGDC0_A2S_CNTL_SW2 0x0032
+#define regGDC0_A2S_CNTL_SW2_BASE_IDX 3
+#define regGDC0_A2S_TAG_ALLOC_0 0x003d
+#define regGDC0_A2S_TAG_ALLOC_0_BASE_IDX 3
+#define regGDC0_A2S_TAG_ALLOC_1 0x003e
+#define regGDC0_A2S_TAG_ALLOC_1_BASE_IDX 3
+#define regGDC0_A2S_MISC_CNTL 0x0041
+#define regGDC0_A2S_MISC_CNTL_BASE_IDX 3
+#define regGDC0_SHUB_REGS_IF_CTL 0x0043
+#define regGDC0_SHUB_REGS_IF_CTL_BASE_IDX 3
+#define regGDC0_NGDC_MGCG_CTRL 0x004a
+#define regGDC0_NGDC_MGCG_CTRL_BASE_IDX 3
+#define regGDC0_NGDC_RESERVED_0 0x004b
+#define regGDC0_NGDC_RESERVED_0_BASE_IDX 3
+#define regGDC0_NGDC_RESERVED_1 0x004c
+#define regGDC0_NGDC_RESERVED_1_BASE_IDX 3
+#define regGDC0_NBIF_GFX_DOORBELL_STATUS 0x004f
+#define regGDC0_NBIF_GFX_DOORBELL_STATUS_BASE_IDX 3
+#define regGDC0_ATDMA_MISC_CNTL 0x005d
+#define regGDC0_ATDMA_MISC_CNTL_BASE_IDX 3
+#define regGDC0_S2A_MISC_CNTL 0x005f
+#define regGDC0_S2A_MISC_CNTL_BASE_IDX 3
+#define regGDC0_NGDC_PG_MISC_CTRL 0x0078
+#define regGDC0_NGDC_PG_MISC_CTRL_BASE_IDX 3
+#define regGDC0_NGDC_PGMST_CTRL 0x0079
+#define regGDC0_NGDC_PGMST_CTRL_BASE_IDX 3
+#define regGDC0_NGDC_PGSLV_CTRL 0x007a
+#define regGDC0_NGDC_PGSLV_CTRL_BASE_IDX 3
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VENDOR_ID                                                                  0x0000
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_ID                                                                  0x0002
+#define cfgBIF_CFG_DEV0_EPF0_COMMAND                                                                    0x0004
+#define cfgBIF_CFG_DEV0_EPF0_STATUS                                                                     0x0006
+#define cfgBIF_CFG_DEV0_EPF0_REVISION_ID                                                                0x0008
+#define cfgBIF_CFG_DEV0_EPF0_PROG_INTERFACE                                                             0x0009
+#define cfgBIF_CFG_DEV0_EPF0_SUB_CLASS                                                                  0x000a
+#define cfgBIF_CFG_DEV0_EPF0_BASE_CLASS                                                                 0x000b
+#define cfgBIF_CFG_DEV0_EPF0_CACHE_LINE                                                                 0x000c
+#define cfgBIF_CFG_DEV0_EPF0_LATENCY                                                                    0x000d
+#define cfgBIF_CFG_DEV0_EPF0_HEADER                                                                     0x000e
+#define cfgBIF_CFG_DEV0_EPF0_BIST                                                                       0x000f
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_1                                                                0x0010
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_2                                                                0x0014
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_3                                                                0x0018
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_4                                                                0x001c
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_5                                                                0x0020
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_6                                                                0x0024
+#define cfgBIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR                                                            0x0028
+#define cfgBIF_CFG_DEV0_EPF0_ADAPTER_ID                                                                 0x002c
+#define cfgBIF_CFG_DEV0_EPF0_ROM_BASE_ADDR                                                              0x0030
+#define cfgBIF_CFG_DEV0_EPF0_CAP_PTR                                                                    0x0034
+#define cfgBIF_CFG_DEV0_EPF0_INTERRUPT_LINE                                                             0x003c
+#define cfgBIF_CFG_DEV0_EPF0_INTERRUPT_PIN                                                              0x003d
+#define cfgBIF_CFG_DEV0_EPF0_MIN_GRANT                                                                  0x003e
+#define cfgBIF_CFG_DEV0_EPF0_MAX_LATENCY                                                                0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST                                                            0x0048
+#define cfgBIF_CFG_DEV0_EPF0_ADAPTER_ID_W                                                               0x004c
+#define cfgBIF_CFG_DEV0_EPF0_PMI_CAP_LIST                                                               0x0050
+#define cfgBIF_CFG_DEV0_EPF0_PMI_CAP                                                                    0x0052
+#define cfgBIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL                                                            0x0054
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CAP_LIST                                                              0x0064
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CAP                                                                   0x0066
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CAP                                                                 0x0068
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CNTL                                                                0x006c
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_STATUS                                                              0x006e
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP                                                                   0x0070
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL                                                                  0x0074
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS                                                                0x0076
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CAP2                                                                0x0088
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CNTL2                                                               0x008c
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_STATUS2                                                             0x008e
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP2                                                                  0x0090
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL2                                                                 0x0094
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS2                                                               0x0096
+#define cfgBIF_CFG_DEV0_EPF0_MSI_CAP_LIST                                                               0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_CNTL                                                               0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO                                                            0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI                                                            0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_DATA                                                               0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA                                                           0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MASK                                                                   0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64                                                            0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64                                                        0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MASK_64                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_MSI_PENDING                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_MSI_PENDING_64                                                             0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_CAP_LIST                                                              0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL                                                              0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_TABLE                                                                 0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_PBA                                                                   0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                          0x0100
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR                                                   0x0104
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1                                                      0x0108
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2                                                      0x010c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST                                                       0x0110
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1                                                      0x0114
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2                                                      0x0118
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL                                                          0x011c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS                                                        0x011e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP                                                      0x0120
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL                                                     0x0124
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS                                                   0x012a
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP                                                      0x012c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL                                                     0x0130
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS                                                   0x0136
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                           0x0140
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1                                                    0x0144
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2                                                    0x0148
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                              0x0150
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS                                                     0x0154
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK                                                       0x0158
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY                                                   0x015c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS                                                       0x0160
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK                                                         0x0164
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL                                                      0x0168
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0                                                              0x016c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1                                                              0x0170
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2                                                              0x0174
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3                                                              0x0178
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0                                                       0x0188
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1                                                       0x018c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2                                                       0x0190
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3                                                       0x0194
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST                                                      0x0200
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR1_CAP                                                              0x0204
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL                                                             0x0208
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR2_CAP                                                              0x020c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL                                                             0x0210
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR3_CAP                                                              0x0214
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL                                                             0x0218
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR4_CAP                                                              0x021c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL                                                             0x0220
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR5_CAP                                                              0x0224
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL                                                             0x0228
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR6_CAP                                                              0x022c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL                                                             0x0230
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                               0x0240
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA_SELECT                                                0x0244
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA                                                       0x0248
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_CAP                                                        0x024c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST                                                      0x0250
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_CAP                                                               0x0254
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_LATENCY_INDICATOR                                                 0x0258
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS                                                            0x025c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_CNTL                                                              0x025e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                              0x0260
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                              0x0261
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                              0x0262
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                              0x0263
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                              0x0264
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                              0x0265
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                              0x0266
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                              0x0267
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST                                                0x0270
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3                                                            0x0274
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS                                                     0x0278
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL                                              0x027c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL                                              0x027e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL                                              0x0280
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL                                              0x0282
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL                                              0x0284
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL                                              0x0286
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL                                              0x0288
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL                                              0x028a
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL                                              0x028c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL                                              0x028e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL                                             0x0290
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL                                             0x0292
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL                                             0x0294
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL                                             0x0296
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL                                             0x0298
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL                                             0x029a
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST                                                      0x02a0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ACS_CAP                                                               0x02a4
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL                                                              0x02a6
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST                                                      0x02b0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ATS_CAP                                                               0x02b4
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL                                                              0x02b6
+#define cfgPCIE_PAGE_REQ_ENH_CAP_LIST                                                                   0x02c0
+#define cfgPCIE_PAGE_REQ_CNTL                                                                           0x02c4
+#define cfgPCIE_PAGE_REQ_STATUS                                                                         0x02c6
+#define cfgPCIE_OUTSTAND_PAGE_REQ_CAPACITY                                                              0x02c8
+#define cfgPCIE_OUTSTAND_PAGE_REQ_ALLOC                                                                 0x02cc
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST                                                    0x02d0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PASID_CAP                                                             0x02d4
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL                                                            0x02d6
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_ENH_CAP_LIST                                                       0x02f0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_CAP                                                                0x02f4
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_CNTL                                                               0x02f6
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_ADDR0                                                              0x02f8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_ADDR1                                                              0x02fc
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_RCV0                                                               0x0300
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_RCV1                                                               0x0304
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_ALL0                                                         0x0308
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_ALL1                                                         0x030c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_UNTRANSLATED_0                                               0x0310
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_UNTRANSLATED_1                                               0x0314
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST                                                      0x0320
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LTR_CAP                                                               0x0324
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST                                                      0x0328
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ARI_CAP                                                               0x032c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL                                                              0x032e
+#define cfgPCIE_SRIOV_ENH_CAP_LIST                                                                      0x0330
+#define cfgPCIE_SRIOV_CAP                                                                               0x0334
+#define cfgPCIE_SRIOV_CONTROL                                                                           0x0338
+#define cfgPCIE_SRIOV_STATUS                                                                            0x033a
+#define cfgPCIE_SRIOV_INITIAL_VFS                                                                       0x033c
+#define cfgPCIE_SRIOV_TOTAL_VFS                                                                         0x033e
+#define cfgPCIE_SRIOV_NUM_VFS                                                                           0x0340
+#define cfgPCIE_SRIOV_FUNC_DEP_LINK                                                                     0x0342
+#define cfgPCIE_SRIOV_FIRST_VF_OFFSET                                                                   0x0344
+#define cfgPCIE_SRIOV_VF_STRIDE                                                                         0x0346
+#define cfgPCIE_SRIOV_VF_DEVICE_ID                                                                      0x034a
+#define cfgPCIE_SRIOV_SUPPORTED_PAGE_SIZE                                                               0x034c
+#define cfgPCIE_SRIOV_SYSTEM_PAGE_SIZE                                                                  0x0350
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_0                                                                    0x0354
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_1                                                                    0x0358
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_2                                                                    0x035c
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_3                                                                    0x0360
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_4                                                                    0x0364
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_5                                                                    0x0368
+#define cfgPCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET                                                   0x036c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST                                                      0x0400
+#define cfgBIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP                                                      0x0404
+#define cfgBIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS                                                   0x0408
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST                                                 0x0410
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP_16GT                                                              0x0414
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL_16GT                                                             0x0418
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS_16GT                                                           0x041c
+#define cfgBIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                          0x0420
+#define cfgBIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT                                           0x0424
+#define cfgBIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT                                           0x0428
+#define cfgBIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT                                              0x0430
+#define cfgBIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT                                              0x0431
+#define cfgBIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT                                              0x0432
+#define cfgBIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT                                              0x0433
+#define cfgBIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT                                              0x0434
+#define cfgBIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT                                              0x0435
+#define cfgBIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT                                              0x0436
+#define cfgBIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT                                              0x0437
+#define cfgBIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT                                              0x0438
+#define cfgBIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT                                              0x0439
+#define cfgBIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT                                             0x043a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT                                             0x043b
+#define cfgBIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT                                             0x043c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT                                             0x043d
+#define cfgBIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT                                             0x043e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT                                             0x043f
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST                                                0x0450
+#define cfgBIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP                                                         0x0454
+#define cfgBIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS                                                      0x0456
+#define cfgBIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL                                                 0x0458
+#define cfgBIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS                                               0x045a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL                                                 0x045c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS                                               0x045e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL                                                 0x0460
+#define cfgBIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS                                               0x0462
+#define cfgBIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL                                                 0x0464
+#define cfgBIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS                                               0x0466
+#define cfgBIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL                                                 0x0468
+#define cfgBIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS                                               0x046a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL                                                 0x046c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS                                               0x046e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL                                                 0x0470
+#define cfgBIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS                                               0x0472
+#define cfgBIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL                                                 0x0474
+#define cfgBIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS                                               0x0476
+#define cfgBIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL                                                 0x0478
+#define cfgBIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS                                               0x047a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL                                                 0x047c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS                                               0x047e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL                                                0x0480
+#define cfgBIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS                                              0x0482
+#define cfgBIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL                                                0x0484
+#define cfgBIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS                                              0x0486
+#define cfgBIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL                                                0x0488
+#define cfgBIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS                                              0x048a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL                                                0x048c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS                                              0x048e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL                                                0x0490
+#define cfgBIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS                                              0x0492
+#define cfgBIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL                                                0x0494
+#define cfgBIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS                                              0x0496
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP_32GT                                                              0x0504
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL_32GT                                                             0x0508
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS_32GT                                                           0x050c
+#define cfgPCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV                                                     0x0700
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV                                                              0x0704
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW                                                 0x0708
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE                                                  0x070c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS                                                  0x0710
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL                                                0x0714
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0                                                0x0718
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1                                                0x071c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2                                                0x0720
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT                                                      0x0724
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB                                                     0x0728
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION                                                       0x0730
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE                                         0x0734
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB                                                       0x0738
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB                                                       0x073c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB                                                       0x0740
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB                                                       0x0744
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB                                                       0x0748
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB                                                       0x074c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB                                                       0x0750
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB                                                       0x0754
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB                                                       0x0758
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB                                                       0x075c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB                                                      0x0760
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB                                                      0x0764
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB                                                      0x0768
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB                                                      0x076c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB                                                      0x0770
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB                                                      0x0774
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB                                                      0x0778
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB                                                      0x077c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB                                                      0x0780
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB                                                      0x0784
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB                                                      0x0788
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB                                                      0x078c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB                                                      0x0790
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB                                                      0x0794
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB                                                      0x0798
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB                                                      0x079c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB                                                      0x07a0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB                                                      0x07a4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB                                                      0x07a8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB                                                      0x07ac
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB                                                      0x07b0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS                                                      0x07c0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1                                                     0x07c4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2                                                     0x07c8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3                                                     0x07cc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4                                                     0x07d0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW0                                                  0x07f0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW1                                                  0x07f4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW2                                                  0x07f8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW3                                                  0x07fc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW4                                                  0x0800
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW5                                                  0x0804
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW6                                                  0x0808
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW7                                                  0x080c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW8                                                  0x0810
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW0                                                  0x0820
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW1                                                  0x0824
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW2                                                  0x0828
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW3                                                  0x082c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW4                                                  0x0830
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW5                                                  0x0834
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW6                                                  0x0838
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW7                                                  0x083c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW8                                                  0x0840
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW0                                                  0x0850
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW1                                                  0x0854
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW2                                                  0x0858
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW3                                                  0x085c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW4                                                  0x0860
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW5                                                  0x0864
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW6                                                  0x0868
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW7                                                  0x086c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW8                                                  0x0870
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW0                                                  0x0880
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW1                                                  0x0884
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW2                                                  0x0888
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW3                                                  0x088c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW4                                                  0x0890
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW5                                                  0x0894
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW6                                                  0x0898
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW7                                                  0x089c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW8                                                  0x08a0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW0                                                  0x08b0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW1                                                  0x08b4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW2                                                  0x08b8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW3                                                  0x08bc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW4                                                  0x08c0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW5                                                  0x08c4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW6                                                  0x08c8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW7                                                  0x08cc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW8                                                  0x08d0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW0                                                  0x08e0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW1                                                  0x08e4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW2                                                  0x08e8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW3                                                  0x08ec
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW4                                                  0x08f0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW5                                                  0x08f4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW6                                                  0x08f8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW7                                                  0x08fc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW8                                                  0x0900
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW0                                                  0x0910
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW1                                                  0x0914
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW2                                                  0x0918
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW3                                                  0x091c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW4                                                  0x0920
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW5                                                  0x0924
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW6                                                  0x0928
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW7                                                  0x092c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW8                                                  0x0930
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW0                                                  0x0940
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW1                                                  0x0944
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW2                                                  0x0948
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW3                                                  0x094c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW4                                                  0x0950
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW5                                                  0x0954
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW6                                                  0x0958
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW7                                                  0x095c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW8                                                  0x0960
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW0                                                  0x0970
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW1                                                  0x0974
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW2                                                  0x0978
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW3                                                  0x097c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW4                                                  0x0980
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW5                                                  0x0984
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW6                                                  0x0988
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW7                                                  0x098c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW8                                                  0x0990
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW0                                                  0x09a0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW1                                                  0x09a4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW2                                                  0x09a8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW3                                                  0x09ac
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW4                                                  0x09b0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW5                                                  0x09b4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW6                                                  0x09b8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW7                                                  0x09bc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW8                                                  0x09c0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW0                                                 0x09d0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW1                                                 0x09d4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW2                                                 0x09d8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW3                                                 0x09dc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW4                                                 0x09e0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW5                                                 0x09e4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW6                                                 0x09e8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW7                                                 0x09ec
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW8                                                 0x09f0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW0                                                 0x0a00
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW1                                                 0x0a04
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW2                                                 0x0a08
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW3                                                 0x0a0c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW4                                                 0x0a10
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW5                                                 0x0a14
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW6                                                 0x0a18
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW7                                                 0x0a1c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW8                                                 0x0a20
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW0                                                  0x0a30
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW1                                                  0x0a34
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW2                                                  0x0a38
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW3                                                  0x0a3c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW4                                                  0x0a40
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW5                                                  0x0a44
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW6                                                  0x0a48
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW7                                                  0x0a4c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW8                                                  0x0a50
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW0                                                  0x0a60
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW1                                                  0x0a64
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW2                                                  0x0a68
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW3                                                  0x0a6c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW4                                                  0x0a70
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW5                                                  0x0a74
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW6                                                  0x0a78
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW7                                                  0x0a7c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW8                                                  0x0a80
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW0                                                  0x0a90
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW1                                                  0x0a94
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW2                                                  0x0a98
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW3                                                  0x0a9c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW4                                                  0x0aa0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW5                                                  0x0aa4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW6                                                  0x0aa8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW7                                                  0x0aac
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW8                                                  0x0ab0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW0                                                  0x0ac0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW1                                                  0x0ac4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW2                                                  0x0ac8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW3                                                  0x0acc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW4                                                  0x0ad0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW5                                                  0x0ad4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW6                                                  0x0ad8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW7                                                  0x0adc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW8                                                  0x0ae0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW0                                                  0x0af0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW1                                                  0x0af4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW2                                                  0x0af8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW3                                                  0x0afc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW4                                                  0x0b00
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW5                                                  0x0b04
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW6                                                  0x0b08
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW7                                                  0x0b0c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW8                                                  0x0b10
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW0                                                  0x0b20
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW1                                                  0x0b24
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW2                                                  0x0b28
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW3                                                  0x0b2c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW4                                                  0x0b30
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW5                                                  0x0b34
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW6                                                  0x0b38
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW7                                                  0x0b3c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW8                                                  0x0b40
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW0                                                  0x0b50
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW1                                                  0x0b54
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW2                                                  0x0b58
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW3                                                  0x0b5c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW4                                                  0x0b60
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW5                                                  0x0b64
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW6                                                  0x0b68
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW7                                                  0x0b6c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW8                                                  0x0b70
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW0                                                  0x0b80
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW1                                                  0x0b84
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW2                                                  0x0b88
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW3                                                  0x0b8c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW4                                                  0x0b90
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW5                                                  0x0b94
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW6                                                  0x0b98
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW7                                                  0x0b9c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW8                                                  0x0ba0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE                                        0x0c00
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A8_15_INTR_ENABLE                                       0x0c04
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE                                        0x0c08
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE                                       0x0c0c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS                                        0x0c10
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A8_15_INTR_STATUS                                       0x0c14
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS                                        0x0c18
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS                                       0x0c1c
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf1_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF1_VENDOR_ID                                                                  0x0000
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_ID                                                                  0x0002
+#define cfgBIF_CFG_DEV0_EPF1_COMMAND                                                                    0x0004
+#define cfgBIF_CFG_DEV0_EPF1_STATUS                                                                     0x0006
+#define cfgBIF_CFG_DEV0_EPF1_REVISION_ID                                                                0x0008
+#define cfgBIF_CFG_DEV0_EPF1_PROG_INTERFACE                                                             0x0009
+#define cfgBIF_CFG_DEV0_EPF1_SUB_CLASS                                                                  0x000a
+#define cfgBIF_CFG_DEV0_EPF1_BASE_CLASS                                                                 0x000b
+#define cfgBIF_CFG_DEV0_EPF1_CACHE_LINE                                                                 0x000c
+#define cfgBIF_CFG_DEV0_EPF1_LATENCY                                                                    0x000d
+#define cfgBIF_CFG_DEV0_EPF1_HEADER                                                                     0x000e
+#define cfgBIF_CFG_DEV0_EPF1_BIST                                                                       0x000f
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_1                                                                0x0010
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_2                                                                0x0014
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_3                                                                0x0018
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_4                                                                0x001c
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_5                                                                0x0020
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_6                                                                0x0024
+#define cfgBIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR                                                            0x0028
+#define cfgBIF_CFG_DEV0_EPF1_ADAPTER_ID                                                                 0x002c
+#define cfgBIF_CFG_DEV0_EPF1_ROM_BASE_ADDR                                                              0x0030
+#define cfgBIF_CFG_DEV0_EPF1_CAP_PTR                                                                    0x0034
+#define cfgBIF_CFG_DEV0_EPF1_INTERRUPT_LINE                                                             0x003c
+#define cfgBIF_CFG_DEV0_EPF1_INTERRUPT_PIN                                                              0x003d
+#define cfgBIF_CFG_DEV0_EPF1_MIN_GRANT                                                                  0x003e
+#define cfgBIF_CFG_DEV0_EPF1_MAX_LATENCY                                                                0x003f
+#define cfgBIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST                                                            0x0048
+#define cfgBIF_CFG_DEV0_EPF1_ADAPTER_ID_W                                                               0x004c
+#define cfgBIF_CFG_DEV0_EPF1_PMI_CAP_LIST                                                               0x0050
+#define cfgBIF_CFG_DEV0_EPF1_PMI_CAP                                                                    0x0052
+#define cfgBIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL                                                            0x0054
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_CAP_LIST                                                              0x0064
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_CAP                                                                   0x0066
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_CAP                                                                 0x0068
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_CNTL                                                                0x006c
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_STATUS                                                              0x006e
+#define cfgBIF_CFG_DEV0_EPF1_LINK_CAP                                                                   0x0070
+#define cfgBIF_CFG_DEV0_EPF1_LINK_CNTL                                                                  0x0074
+#define cfgBIF_CFG_DEV0_EPF1_LINK_STATUS                                                                0x0076
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_CAP2                                                                0x0088
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_CNTL2                                                               0x008c
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_STATUS2                                                             0x008e
+#define cfgBIF_CFG_DEV0_EPF1_LINK_CAP2                                                                  0x0090
+#define cfgBIF_CFG_DEV0_EPF1_LINK_CNTL2                                                                 0x0094
+#define cfgBIF_CFG_DEV0_EPF1_LINK_STATUS2                                                               0x0096
+#define cfgBIF_CFG_DEV0_EPF1_MSI_CAP_LIST                                                               0x00a0
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MSG_CNTL                                                               0x00a2
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO                                                            0x00a4
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI                                                            0x00a8
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MSG_DATA                                                               0x00a8
+#define cfgBIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA                                                           0x00aa
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MASK                                                                   0x00ac
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64                                                            0x00ac
+#define cfgBIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64                                                        0x00ae
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MASK_64                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF1_MSI_PENDING                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF1_MSI_PENDING_64                                                             0x00b4
+#define cfgBIF_CFG_DEV0_EPF1_MSIX_CAP_LIST                                                              0x00c0
+#define cfgBIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL                                                              0x00c2
+#define cfgBIF_CFG_DEV0_EPF1_MSIX_TABLE                                                                 0x00c4
+#define cfgBIF_CFG_DEV0_EPF1_MSIX_PBA                                                                   0x00c8
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                          0x0100
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR                                                   0x0104
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1                                                      0x0108
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2                                                      0x010c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                              0x0150
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS                                                     0x0154
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK                                                       0x0158
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY                                                   0x015c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS                                                       0x0160
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK                                                         0x0164
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL                                                      0x0168
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0                                                              0x016c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1                                                              0x0170
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2                                                              0x0174
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3                                                              0x0178
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0                                                       0x0188
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1                                                       0x018c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2                                                       0x0190
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3                                                       0x0194
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST                                                      0x0200
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP                                                              0x0204
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL                                                             0x0208
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP                                                              0x020c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL                                                             0x0210
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP                                                              0x0214
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL                                                             0x0218
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP                                                              0x021c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL                                                             0x0220
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP                                                              0x0224
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL                                                             0x0228
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP                                                              0x022c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL                                                             0x0230
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST                                               0x0240
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT                                                0x0244
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA                                                       0x0248
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP                                                        0x024c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST                                                      0x0250
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_CAP                                                               0x0254
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR                                                 0x0258
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS                                                            0x025c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL                                                              0x025e
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                              0x0260
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                              0x0261
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                              0x0262
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                              0x0263
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                              0x0264
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                              0x0265
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                              0x0266
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                              0x0267
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST                                                      0x02a0
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ACS_CAP                                                               0x02a4
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL                                                              0x02a6
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST                                                    0x02d0
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PASID_CAP                                                             0x02d4
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL                                                            0x02d6
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST                                                      0x0328
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ARI_CAP                                                               0x032c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL                                                              0x032e
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_rc_bifcfgdecp
+// base address: 0x10100000
+#define regBIF_CFG_DEV0_RC0_VENDOR_ID                                                                   0x0000
+#define regBIF_CFG_DEV0_RC0_VENDOR_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_DEVICE_ID                                                                   0x0000
+#define regBIF_CFG_DEV0_RC0_DEVICE_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_COMMAND                                                                     0x0001
+#define regBIF_CFG_DEV0_RC0_COMMAND_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_STATUS                                                                      0x0001
+#define regBIF_CFG_DEV0_RC0_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_REVISION_ID                                                                 0x0002
+#define regBIF_CFG_DEV0_RC0_REVISION_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PROG_INTERFACE                                                              0x0002
+#define regBIF_CFG_DEV0_RC0_PROG_INTERFACE_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SUB_CLASS                                                                   0x0002
+#define regBIF_CFG_DEV0_RC0_SUB_CLASS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_BASE_CLASS                                                                  0x0002
+#define regBIF_CFG_DEV0_RC0_BASE_CLASS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_CACHE_LINE                                                                  0x0003
+#define regBIF_CFG_DEV0_RC0_CACHE_LINE_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LATENCY                                                                     0x0003
+#define regBIF_CFG_DEV0_RC0_LATENCY_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_HEADER                                                                      0x0003
+#define regBIF_CFG_DEV0_RC0_HEADER_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_BIST                                                                        0x0003
+#define regBIF_CFG_DEV0_RC0_BIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_1                                                                 0x0004
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_1_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_2                                                                 0x0005
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY                                                      0x0006
+#define regBIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT                                                               0x0007
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SECONDARY_STATUS                                                            0x0007
+#define regBIF_CFG_DEV0_RC0_SECONDARY_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MEM_BASE_LIMIT                                                              0x0008
+#define regBIF_CFG_DEV0_RC0_MEM_BASE_LIMIT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_LIMIT                                                             0x0009
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_LIMIT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_UPPER                                                             0x000a
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_UPPER_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER                                                            0x000b
+#define regBIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI                                                            0x000c
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_CAP_PTR                                                                     0x000d
+#define regBIF_CFG_DEV0_RC0_CAP_PTR_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_ROM_BASE_ADDR                                                               0x000e
+#define regBIF_CFG_DEV0_RC0_ROM_BASE_ADDR_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_LINE                                                              0x000f
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_LINE_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_PIN                                                               0x000f
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_PIN_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL                                                             0x000f
+#define regBIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL                                                             0x0010
+#define regBIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PMI_CAP_LIST                                                                0x0014
+#define regBIF_CFG_DEV0_RC0_PMI_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PMI_CAP                                                                     0x0014
+#define regBIF_CFG_DEV0_RC0_PMI_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PMI_STATUS_CNTL                                                             0x0015
+#define regBIF_CFG_DEV0_RC0_PMI_STATUS_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP_LIST                                                               0x0016
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP                                                                    0x0016
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP                                                                  0x0017
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL                                                                 0x0018
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS                                                               0x0018
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_CAP                                                                    0x0019
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL                                                                   0x001a
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS                                                                 0x001a
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP                                                                    0x001b
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL                                                                   0x001c
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS                                                                 0x001c
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_ROOT_CNTL                                                                   0x001d
+#define regBIF_CFG_DEV0_RC0_ROOT_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_ROOT_CAP                                                                    0x001d
+#define regBIF_CFG_DEV0_RC0_ROOT_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_ROOT_STATUS                                                                 0x001e
+#define regBIF_CFG_DEV0_RC0_ROOT_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP2                                                                 0x001f
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL2                                                                0x0020
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS2                                                              0x0020
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_CAP2                                                                   0x0021
+#define regBIF_CFG_DEV0_RC0_LINK_CAP2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL2                                                                  0x0022
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS2                                                                0x0022
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP2                                                                   0x0023
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL2                                                                  0x0024
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS2                                                                0x0024
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MSI_CAP_LIST                                                                0x0028
+#define regBIF_CFG_DEV0_RC0_MSI_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_CNTL                                                                0x0028
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO                                                             0x0029
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI                                                             0x002a
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA                                                                0x002a
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA                                                            0x002a
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA_64                                                             0x002b
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA_64_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64                                                         0x002b
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SSID_CAP_LIST                                                               0x0030
+#define regBIF_CFG_DEV0_RC0_SSID_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_SSID_CAP                                                                    0x0031
+#define regBIF_CFG_DEV0_RC0_SSID_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                           0x0040
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR                                                    0x0041
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1                                                       0x0042
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2                                                       0x0043
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST                                                        0x0044
+#define regBIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1                                                       0x0045
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2                                                       0x0046
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL                                                           0x0047
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS                                                         0x0047
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP                                                       0x0048
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL                                                      0x0049
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS                                                    0x004a
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP                                                       0x004b
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL                                                      0x004c
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS                                                    0x004d
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                            0x0050
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1                                                     0x0051
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2                                                     0x0052
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                               0x0054
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS                                                      0x0055
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK                                                        0x0056
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY                                                    0x0057
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS                                                        0x0058
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK                                                          0x0059
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL                                                       0x005a
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG0                                                               0x005b
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG0_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG1                                                               0x005c
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG1_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG2                                                               0x005d
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG3                                                               0x005e
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG3_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD                                                           0x005f
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS                                                        0x0060
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID                                                             0x0061
+#define regBIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0                                                        0x0062
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1                                                        0x0063
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2                                                        0x0064
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3                                                        0x0065
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST                                                 0x009c
+#define regBIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3                                                             0x009d
+#define regBIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS                                                      0x009e
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL                                               0x009f
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL                                               0x009f
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL                                               0x00a0
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL                                               0x00a0
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL                                               0x00a1
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL                                               0x00a1
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL                                               0x00a2
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL                                               0x00a2
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL                                               0x00a3
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL                                               0x00a3
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL                                              0x00a4
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL                                              0x00a4
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL                                              0x00a5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL                                              0x00a5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL                                              0x00a6
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL                                              0x00a6
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST                                                       0x00a8
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CAP                                                                0x00a9
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CNTL                                                               0x00a9
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST                                                       0x0100
+#define regBIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP                                                       0x0101
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS                                                    0x0102
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST                                                  0x0104
+#define regBIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_16GT                                                               0x0105
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_16GT                                                              0x0106
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_16GT                                                            0x0107
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                           0x0108
+#define regBIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT                                            0x0109
+#define regBIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT                                            0x010a
+#define regBIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT                                               0x010c
+#define regBIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT                                               0x010c
+#define regBIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT                                               0x010c
+#define regBIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT                                               0x010c
+#define regBIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT                                               0x010d
+#define regBIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT                                               0x010d
+#define regBIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT                                               0x010d
+#define regBIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT                                               0x010d
+#define regBIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT                                               0x010e
+#define regBIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT                                               0x010e
+#define regBIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT                                              0x010e
+#define regBIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT                                              0x010e
+#define regBIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT                                              0x010f
+#define regBIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT                                              0x010f
+#define regBIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT                                              0x010f
+#define regBIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT                                              0x010f
+#define regBIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST                                                 0x0114
+#define regBIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_CAP                                                          0x0115
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS                                                       0x0115
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL                                                  0x0116
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS                                                0x0116
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL                                                  0x0117
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS                                                0x0117
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL                                                  0x0118
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS                                                0x0118
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL                                                  0x0119
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS                                                0x0119
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL                                                  0x011a
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS                                                0x011a
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL                                                  0x011b
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS                                                0x011b
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL                                                  0x011c
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS                                                0x011c
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL                                                  0x011d
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS                                                0x011d
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL                                                  0x011e
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS                                                0x011e
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL                                                  0x011f
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS                                                0x011f
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL                                                 0x0120
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS                                               0x0120
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL                                                 0x0121
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS                                               0x0121
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL                                                 0x0122
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS                                               0x0122
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL                                                 0x0123
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS                                               0x0123
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL                                                 0x0124
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS                                               0x0124
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL                                                 0x0125
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS                                               0x0125
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_32GT                                                               0x0141
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_32GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_32GT                                                              0x0142
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_32GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_32GT                                                            0x0143
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_32GT_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_ID                                                                0x10000
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_ID                                                                0x10000
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_COMMAND                                                                  0x10001
+#define regBIF_CFG_DEV0_EPF0_0_COMMAND_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_STATUS                                                                   0x10001
+#define regBIF_CFG_DEV0_EPF0_0_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_REVISION_ID                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_0_REVISION_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PROG_INTERFACE                                                           0x10002
+#define regBIF_CFG_DEV0_EPF0_0_PROG_INTERFACE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_SUB_CLASS                                                                0x10002
+#define regBIF_CFG_DEV0_EPF0_0_SUB_CLASS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_BASE_CLASS                                                               0x10002
+#define regBIF_CFG_DEV0_EPF0_0_BASE_CLASS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_CACHE_LINE                                                               0x10003
+#define regBIF_CFG_DEV0_EPF0_0_CACHE_LINE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LATENCY                                                                  0x10003
+#define regBIF_CFG_DEV0_EPF0_0_LATENCY_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_HEADER                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_0_HEADER_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_BIST                                                                     0x10003
+#define regBIF_CFG_DEV0_EPF0_0_BIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_1                                                              0x10004
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_2                                                              0x10005
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_3                                                              0x10006
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_4                                                              0x10007
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_5                                                              0x10008
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_6                                                              0x10009
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR                                                          0x1000a
+#define regBIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID                                                               0x1000b
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR                                                            0x1000c
+#define regBIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_CAP_PTR                                                                  0x1000d
+#define regBIF_CFG_DEV0_EPF0_0_CAP_PTR_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE                                                           0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MIN_GRANT                                                                0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_MIN_GRANT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MAX_LATENCY                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_MAX_LATENCY_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST                                                          0x10012
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W                                                             0x10013
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST                                                             0x10014
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP                                                                  0x10014
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL                                                          0x10015
+#define regBIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST                                                            0x10019
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP                                                                 0x10019
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP                                                               0x1001a
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL                                                              0x1001b
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP                                                                 0x1001c
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL                                                                0x1001d
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP2                                                              0x10022
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2                                                             0x10023
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP2                                                                0x10024
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL2                                                               0x10025
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST                                                             0x10028
+#define regBIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL                                                             0x10028
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO                                                          0x10029
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI                                                          0x1002a
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA                                                             0x1002a
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA                                                         0x1002a
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK                                                                 0x1002b
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64                                                          0x1002b
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64                                                      0x1002b
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK_64                                                              0x1002c
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK_64_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING                                                              0x1002c
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING_64                                                           0x1002d
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING_64_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST                                                            0x10030
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL                                                            0x10030
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_TABLE                                                               0x10031
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_TABLE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_PBA                                                                 0x10032
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_PBA_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x10040
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x10041
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1                                                    0x10042
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2                                                    0x10043
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST                                                     0x10044
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1                                                    0x10045
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2                                                    0x10046
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL                                                        0x10047
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS                                                      0x10047
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP                                                    0x10048
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL                                                   0x10049
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS                                                 0x1004a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP                                                    0x1004b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL                                                   0x1004c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS                                                 0x1004d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                         0x10050
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1                                                  0x10051
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2                                                  0x10052
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x10054
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS                                                   0x10055
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK                                                     0x10056
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x10057
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS                                                     0x10058
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK                                                       0x10059
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x1005a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0                                                            0x1005b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1                                                            0x1005c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2                                                            0x1005d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3                                                            0x1005e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0                                                     0x10062
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1                                                     0x10063
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2                                                     0x10064
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3                                                     0x10065
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST                                                    0x10080
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP                                                            0x10081
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL                                                           0x10082
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP                                                            0x10083
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL                                                           0x10084
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP                                                            0x10085
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL                                                           0x10086
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP                                                            0x10087
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL                                                           0x10088
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP                                                            0x10089
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL                                                           0x1008a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP                                                            0x1008b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL                                                           0x1008c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x10090
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x10091
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA                                                     0x10092
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP                                                      0x10093
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST                                                    0x10094
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP                                                             0x10095
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR                                               0x10096
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS                                                          0x10097
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL                                                            0x10097
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x10098
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x10098
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x10098
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x10098
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x10099
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x10099
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x10099
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x10099
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST                                              0x1009c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3                                                          0x1009d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS                                                   0x1009e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL                                            0x1009f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL                                            0x1009f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL                                            0x100a0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL                                            0x100a0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL                                            0x100a1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL                                            0x100a1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL                                            0x100a2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL                                            0x100a2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL                                            0x100a3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL                                            0x100a3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL                                           0x100a4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL                                           0x100a4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL                                           0x100a5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL                                           0x100a5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL                                           0x100a6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL                                           0x100a6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST                                                    0x100a8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP                                                             0x100a9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL                                                            0x100a9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST                                                    0x100ac
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP                                                             0x100ad
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL                                                            0x100ad
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST                                               0x100b0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL                                                       0x100b1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS                                                     0x100b1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY                                          0x100b2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC                                             0x100b3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST                                                  0x100b4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP                                                           0x100b5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL                                                          0x100b5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_ENH_CAP_LIST                                                     0x100bc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_CAP                                                              0x100bd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_CNTL                                                             0x100bd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR0                                                            0x100be
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR1                                                            0x100bf
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_RCV0                                                             0x100c0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_RCV0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_RCV1                                                             0x100c1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_RCV1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_ALL0                                                       0x100c2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_ALL0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_ALL1                                                       0x100c3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_ALL1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_UNTRANSLATED_0                                             0x100c4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_UNTRANSLATED_0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_UNTRANSLATED_1                                             0x100c5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_UNTRANSLATED_1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST                                                    0x100c8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP                                                             0x100c9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST                                                    0x100ca
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP                                                             0x100cb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL                                                            0x100cb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST                                                  0x100cc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP                                                           0x100cd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL                                                       0x100ce
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS                                                        0x100ce
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS                                                   0x100cf
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS                                                     0x100cf
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS                                                       0x100d0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK                                                 0x100d0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET                                               0x100d1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE                                                     0x100d1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID                                                  0x100d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE                                           0x100d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE                                              0x100d4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0                                                0x100d5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1                                                0x100d6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2                                                0x100d7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3                                                0x100d8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4                                                0x100d9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5                                                0x100da
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET                               0x100db
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST                                                    0x10100
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP                                                    0x10101
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS                                                 0x10102
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST                                               0x10104
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT                                                            0x10105
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT                                                           0x10106
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT                                                         0x10107
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                        0x10108
+#define regBIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT                                         0x10109
+#define regBIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT                                         0x1010a
+#define regBIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT                                            0x1010c
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT                                            0x1010c
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT                                            0x1010c
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT                                            0x1010c
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT                                            0x1010d
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT                                            0x1010d
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT                                            0x1010d
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT                                            0x1010d
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT                                            0x1010e
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT                                            0x1010e
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT                                           0x1010e
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT                                           0x1010e
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT                                           0x1010f
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT                                           0x1010f
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT                                           0x1010f
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT                                           0x1010f
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST                                              0x10114
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP                                                       0x10115
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS                                                    0x10115
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL                                               0x10116
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS                                             0x10116
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL                                               0x10117
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS                                             0x10117
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL                                               0x10118
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS                                             0x10118
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL                                               0x10119
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS                                             0x10119
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL                                               0x1011a
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS                                             0x1011a
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL                                               0x1011b
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS                                             0x1011b
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL                                               0x1011c
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS                                             0x1011c
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL                                               0x1011d
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS                                             0x1011d
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL                                               0x1011e
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS                                             0x1011e
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL                                               0x1011f
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS                                             0x1011f
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL                                              0x10120
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS                                            0x10120
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL                                              0x10121
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS                                            0x10121
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL                                              0x10122
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS                                            0x10122
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL                                              0x10123
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS                                            0x10123
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL                                              0x10124
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS                                            0x10124
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL                                              0x10125
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS                                            0x10125
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT                                                            0x10141
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT                                                           0x10142
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT                                                         0x10143
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV                                 0x101c0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV                                          0x101c1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW                             0x101c2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE                              0x101c3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS                              0x101c4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL                            0x101c5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0                            0x101c6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1                            0x101c7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2                            0x101c8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT                                  0x101c9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB                                 0x101ca
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION                                   0x101cc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE                     0x101cd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB                                   0x101ce
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB                                   0x101cf
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB                                   0x101d0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB                                   0x101d1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB                                   0x101d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB                                   0x101d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB                                   0x101d4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB                                   0x101d5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB                                   0x101d6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB                                   0x101d7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB                                  0x101d8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB                                  0x101d9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB                                  0x101da
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB                                  0x101db
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB                                  0x101dc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB                                  0x101dd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB                                  0x101de
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB                                  0x101df
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB                                  0x101e0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB                                  0x101e1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB                                  0x101e2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB                                  0x101e3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB                                  0x101e4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB                                  0x101e5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB                                  0x101e6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB                                  0x101e7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB                                  0x101e8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB                                  0x101e9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB                                  0x101ea
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB                                  0x101eb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB                                  0x101ec
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS                                  0x101f0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1                                 0x101f1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2                                 0x101f2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3                                 0x101f3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4                                 0x101f4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW0                              0x101fc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW1                              0x101fd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW2                              0x101fe
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW3                              0x101ff
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW4                              0x10200
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW5                              0x10201
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW6                              0x10202
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW7                              0x10203
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW8                              0x10204
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW0                              0x10208
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW1                              0x10209
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW2                              0x1020a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW3                              0x1020b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW4                              0x1020c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW5                              0x1020d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW6                              0x1020e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW7                              0x1020f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW8                              0x10210
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW0                              0x10214
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW1                              0x10215
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW2                              0x10216
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW3                              0x10217
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW4                              0x10218
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW5                              0x10219
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW6                              0x1021a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW7                              0x1021b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW8                              0x1021c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW0                              0x10220
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW1                              0x10221
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW2                              0x10222
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW3                              0x10223
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW4                              0x10224
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW5                              0x10225
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW6                              0x10226
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW7                              0x10227
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW8                              0x10228
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW0                              0x1022c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW1                              0x1022d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW2                              0x1022e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW3                              0x1022f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW4                              0x10230
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW5                              0x10231
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW6                              0x10232
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW7                              0x10233
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW8                              0x10234
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW0                              0x10238
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW1                              0x10239
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW2                              0x1023a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW3                              0x1023b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW4                              0x1023c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW5                              0x1023d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW6                              0x1023e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW7                              0x1023f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW8                              0x10240
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW0                              0x10244
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW1                              0x10245
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW2                              0x10246
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW3                              0x10247
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW4                              0x10248
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW5                              0x10249
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW6                              0x1024a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW7                              0x1024b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW8                              0x1024c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW0                              0x10250
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW1                              0x10251
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW2                              0x10252
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW3                              0x10253
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW4                              0x10254
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW5                              0x10255
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW6                              0x10256
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW7                              0x10257
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW8                              0x10258
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW0                              0x1025c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW1                              0x1025d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW2                              0x1025e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW3                              0x1025f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW4                              0x10260
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW5                              0x10261
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW6                              0x10262
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW7                              0x10263
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW8                              0x10264
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW0                              0x10268
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW1                              0x10269
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW2                              0x1026a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW3                              0x1026b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW4                              0x1026c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW5                              0x1026d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW6                              0x1026e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW7                              0x1026f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW8                              0x10270
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW0                             0x10274
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW1                             0x10275
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW2                             0x10276
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW3                             0x10277
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW4                             0x10278
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW5                             0x10279
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW6                             0x1027a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW7                             0x1027b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW8                             0x1027c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW0                             0x10280
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW1                             0x10281
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW2                             0x10282
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW3                             0x10283
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW4                             0x10284
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW5                             0x10285
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW6                             0x10286
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW7                             0x10287
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW8                             0x10288
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW0                              0x1028c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW1                              0x1028d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW2                              0x1028e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW3                              0x1028f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW4                              0x10290
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW5                              0x10291
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW6                              0x10292
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW7                              0x10293
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW8                              0x10294
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW0                              0x10298
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW1                              0x10299
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW2                              0x1029a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW3                              0x1029b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW4                              0x1029c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW5                              0x1029d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW6                              0x1029e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW7                              0x1029f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW8                              0x102a0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW0                              0x102a4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW1                              0x102a5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW2                              0x102a6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW3                              0x102a7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW4                              0x102a8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW5                              0x102a9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW6                              0x102aa
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW7                              0x102ab
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW8                              0x102ac
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW0                              0x102b0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW1                              0x102b1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW2                              0x102b2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW3                              0x102b3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW4                              0x102b4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW5                              0x102b5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW6                              0x102b6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW7                              0x102b7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW8                              0x102b8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW0                              0x102bc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW1                              0x102bd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW2                              0x102be
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW3                              0x102bf
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW4                              0x102c0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW5                              0x102c1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW6                              0x102c2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW7                              0x102c3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW8                              0x102c4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW0                              0x102c8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW1                              0x102c9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW2                              0x102ca
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW3                              0x102cb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW4                              0x102cc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW5                              0x102cd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW6                              0x102ce
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW7                              0x102cf
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW8                              0x102d0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW0                              0x102d4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW1                              0x102d5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW2                              0x102d6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW3                              0x102d7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW4                              0x102d8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW5                              0x102d9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW6                              0x102da
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW7                              0x102db
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW8                              0x102dc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW0                              0x102e0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW1                              0x102e1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW2                              0x102e2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW3                              0x102e3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW4                              0x102e4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW5                              0x102e5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW6                              0x102e6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW7                              0x102e7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW8                              0x102e8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW8_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE                    0x10300
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A8_15_INTR_ENABLE                   0x10301
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A8_15_INTR_ENABLE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE                    0x10302
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE                   0x10303
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS                    0x10304
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A8_15_INTR_STATUS                   0x10305
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A8_15_INTR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS                    0x10306
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS                   0x10307
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf1_bifcfgdecp
+// base address: 0x10141000
+#define regBIF_CFG_DEV0_EPF1_0_VENDOR_ID                                                                0x10400
+#define regBIF_CFG_DEV0_EPF1_0_VENDOR_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_ID                                                                0x10400
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_COMMAND                                                                  0x10401
+#define regBIF_CFG_DEV0_EPF1_0_COMMAND_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_STATUS                                                                   0x10401
+#define regBIF_CFG_DEV0_EPF1_0_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_REVISION_ID                                                              0x10402
+#define regBIF_CFG_DEV0_EPF1_0_REVISION_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PROG_INTERFACE                                                           0x10402
+#define regBIF_CFG_DEV0_EPF1_0_PROG_INTERFACE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_SUB_CLASS                                                                0x10402
+#define regBIF_CFG_DEV0_EPF1_0_SUB_CLASS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_BASE_CLASS                                                               0x10402
+#define regBIF_CFG_DEV0_EPF1_0_BASE_CLASS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_CACHE_LINE                                                               0x10403
+#define regBIF_CFG_DEV0_EPF1_0_CACHE_LINE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_LATENCY                                                                  0x10403
+#define regBIF_CFG_DEV0_EPF1_0_LATENCY_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_HEADER                                                                   0x10403
+#define regBIF_CFG_DEV0_EPF1_0_HEADER_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_BIST                                                                     0x10403
+#define regBIF_CFG_DEV0_EPF1_0_BIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_1                                                              0x10404
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_2                                                              0x10405
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_3                                                              0x10406
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_4                                                              0x10407
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_5                                                              0x10408
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_6                                                              0x10409
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_CARDBUS_CIS_PTR                                                          0x1040a
+#define regBIF_CFG_DEV0_EPF1_0_CARDBUS_CIS_PTR_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_ADAPTER_ID                                                               0x1040b
+#define regBIF_CFG_DEV0_EPF1_0_ADAPTER_ID_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR                                                            0x1040c
+#define regBIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_CAP_PTR                                                                  0x1040d
+#define regBIF_CFG_DEV0_EPF1_0_CAP_PTR_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_INTERRUPT_LINE                                                           0x1040f
+#define regBIF_CFG_DEV0_EPF1_0_INTERRUPT_LINE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_INTERRUPT_PIN                                                            0x1040f
+#define regBIF_CFG_DEV0_EPF1_0_INTERRUPT_PIN_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MIN_GRANT                                                                0x1040f
+#define regBIF_CFG_DEV0_EPF1_0_MIN_GRANT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MAX_LATENCY                                                              0x1040f
+#define regBIF_CFG_DEV0_EPF1_0_MAX_LATENCY_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST                                                          0x10412
+#define regBIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W                                                             0x10413
+#define regBIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST                                                             0x10414
+#define regBIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PMI_CAP                                                                  0x10414
+#define regBIF_CFG_DEV0_EPF1_0_PMI_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL                                                          0x10415
+#define regBIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST                                                            0x10419
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CAP                                                                 0x10419
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CAP                                                               0x1041a
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CNTL                                                              0x1041b
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_STATUS                                                            0x1041b
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CAP                                                                 0x1041c
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CNTL                                                                0x1041d
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_LINK_STATUS                                                              0x1041d
+#define regBIF_CFG_DEV0_EPF1_0_LINK_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CAP2                                                              0x10422
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CAP2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2                                                             0x10423
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_STATUS2                                                           0x10423
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_STATUS2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CAP2                                                                0x10424
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CAP2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CNTL2                                                               0x10425
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CNTL2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_LINK_STATUS2                                                             0x10425
+#define regBIF_CFG_DEV0_EPF1_0_LINK_STATUS2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST                                                             0x10428
+#define regBIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL                                                             0x10428
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_LO                                                          0x10429
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_LO_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_HI                                                          0x1042a
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_HI_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA                                                             0x1042a
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA                                                         0x1042a
+#define regBIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MASK                                                                 0x1042b
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MASK_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_64                                                          0x1042b
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_64_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_64                                                      0x1042b
+#define regBIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_64_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MASK_64                                                              0x1042c
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MASK_64_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_PENDING                                                              0x1042c
+#define regBIF_CFG_DEV0_EPF1_0_MSI_PENDING_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSI_PENDING_64                                                           0x1042d
+#define regBIF_CFG_DEV0_EPF1_0_MSI_PENDING_64_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST                                                            0x10430
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL                                                            0x10430
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_TABLE                                                               0x10431
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_TABLE_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_PBA                                                                 0x10432
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_PBA_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x10440
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x10441
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC1                                                    0x10442
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC2                                                    0x10443
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x10454
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS                                                   0x10455
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK                                                     0x10456
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x10457
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS                                                     0x10458
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK                                                       0x10459
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x1045a
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG0                                                            0x1045b
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG1                                                            0x1045c
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG2                                                            0x1045d
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG3                                                            0x1045e
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG0                                                     0x10462
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG1                                                     0x10463
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG2                                                     0x10464
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG3                                                     0x10465
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST                                                    0x10480
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CAP                                                            0x10481
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL                                                           0x10482
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CAP                                                            0x10483
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL                                                           0x10484
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CAP                                                            0x10485
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL                                                           0x10486
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CAP                                                            0x10487
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL                                                           0x10488
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CAP                                                            0x10489
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL                                                           0x1048a
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CAP                                                            0x1048b
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL                                                           0x1048c
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x10490
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x10491
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA                                                     0x10492
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_CAP                                                      0x10493
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST                                                    0x10494
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP                                                             0x10495
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_LATENCY_INDICATOR                                               0x10496
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS                                                          0x10497
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_CNTL                                                            0x10497
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x10498
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x10498
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x10498
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x10498
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x10499
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x10499
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x10499
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x10499
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST                                                    0x104a8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP                                                             0x104a9
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL                                                            0x104a9
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST                                                  0x104b4
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP                                                           0x104b5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL                                                          0x104b5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST                                                    0x104ca
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP                                                             0x104cb
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP_BASE_IDX 8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL                                                            0x104cb
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_DEV0_1_RCC_VDM_SUPPORT                                                                   0xc440
+#define regRCC_DEV0_1_RCC_VDM_SUPPORT_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_BUS_CNTL                                                                      0xc441
+#define regRCC_DEV0_1_RCC_BUS_CNTL_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_FEATURES_CONTROL_MISC                                                         0xc442
+#define regRCC_DEV0_1_RCC_FEATURES_CONTROL_MISC_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_DEV0_LINK_CNTL                                                                0xc443
+#define regRCC_DEV0_1_RCC_DEV0_LINK_CNTL_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_CMN_LINK_CNTL                                                                 0xc444
+#define regRCC_DEV0_1_RCC_CMN_LINK_CNTL_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE                                                        0xc445
+#define regRCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_LTR_LSWITCH_CNTL                                                              0xc446
+#define regRCC_DEV0_1_RCC_LTR_LSWITCH_CNTL_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_MH_ARB_CNTL                                                                   0xc447
+#define regRCC_DEV0_1_RCC_MH_ARB_CNTL_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0                                                            0xc448
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1                                                            0xc449
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_ep_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_EP_DEV0_1_EP_PCIE_SCRATCH                                                                0xc44c
+#define regRCC_EP_DEV0_1_EP_PCIE_SCRATCH_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_CNTL                                                                   0xc44e
+#define regRCC_EP_DEV0_1_EP_PCIE_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_CNTL                                                               0xc44f
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_STATUS                                                             0xc450
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_STATUS_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL2                                                               0xc451
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL2_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_BUS_CNTL                                                               0xc452
+#define regRCC_EP_DEV0_1_EP_PCIE_BUS_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_CFG_CNTL                                                               0xc453
+#define regRCC_EP_DEV0_1_EP_PCIE_CFG_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL                                                            0xc454
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC                                                             0xc455
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2                                                            0xc456
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP                                                             0xc457
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                               0xc458
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL                                                            0xc458
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                               0xc458
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                               0xc459
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                               0xc459
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                               0xc459
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                               0xc459
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                               0xc45a
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                               0xc45a
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                               0xc45a
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_PME_CONTROL                                                            0xc45c
+#define regRCC_EP_DEV0_1_EP_PCIE_PME_CONTROL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIEP_RESERVED                                                              0xc45d
+#define regRCC_EP_DEV0_1_EP_PCIEP_RESERVED_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_CNTL                                                                0xc45f
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID                                                        0xc460
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_ERR_CNTL                                                               0xc461
+#define regRCC_EP_DEV0_1_EP_PCIE_ERR_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL                                                                0xc462
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL                                                          0xc463
+#define regRCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwn_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_DWN_DEV0_1_DN_PCIE_RESERVED                                                              0xc468
+#define regRCC_DWN_DEV0_1_DN_PCIE_RESERVED_BASE_IDX 8
+#define regRCC_DWN_DEV0_1_DN_PCIE_SCRATCH                                                               0xc469
+#define regRCC_DWN_DEV0_1_DN_PCIE_SCRATCH_BASE_IDX 8
+#define regRCC_DWN_DEV0_1_DN_PCIE_CNTL                                                                  0xc46b
+#define regRCC_DWN_DEV0_1_DN_PCIE_CNTL_BASE_IDX 8
+#define regRCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL                                                           0xc46c
+#define regRCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL_BASE_IDX 8
+#define regRCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2                                                              0xc46d
+#define regRCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2_BASE_IDX 8
+#define regRCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL                                                              0xc46e
+#define regRCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL_BASE_IDX 8
+#define regRCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL                                                              0xc46f
+#define regRCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL_BASE_IDX 8
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_F0                                                              0xc470
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_F0_BASE_IDX 8
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC                                                            0xc471
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC_BASE_IDX 8
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2                                                           0xc472
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwnp_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_DWNP_DEV0_1_PCIE_ERR_CNTL                                                                0xc475
+#define regRCC_DWNP_DEV0_1_PCIE_ERR_CNTL_BASE_IDX 8
+#define regRCC_DWNP_DEV0_1_PCIE_RX_CNTL                                                                 0xc476
+#define regRCC_DWNP_DEV0_1_PCIE_RX_CNTL_BASE_IDX 8
+#define regRCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL                                                           0xc477
+#define regRCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL_BASE_IDX 8
+#define regRCC_DWNP_DEV0_1_PCIE_LC_CNTL2                                                                0xc478
+#define regRCC_DWNP_DEV0_1_PCIE_LC_CNTL2_BASE_IDX 8
+#define regRCC_DWNP_DEV0_1_PCIEP_STRAP_MISC                                                             0xc479
+#define regRCC_DWNP_DEV0_1_PCIEP_STRAP_MISC_BASE_IDX 8
+#define regRCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP                                                         0xc47a
+#define regRCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_pfc_amdgfx_RCCPFCDEC
+// base address: 0x10134000
+#define regRCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL                                                              0xd040
+#define regRCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL_BASE_IDX 8
+#define regRCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE                                                           0xd041
+#define regRCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE_BASE_IDX 8
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0                                                      0xd042
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0_BASE_IDX 8
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1                                                      0xd043
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1_BASE_IDX 8
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2                                                      0xd044
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2_BASE_IDX 8
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3                                                      0xd045
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3_BASE_IDX 8
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4                                                      0xd046
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4_BASE_IDX 8
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5                                                      0xd047
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5_BASE_IDX 8
+#define regRCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL                                                           0xd048
+#define regRCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_pciemsix_0_usb_MSIXTDEC
+// base address: 0x10168000
+#define regPCIEMSIX_VECT0_ADDR_LO                                                                       0x1a000
+#define regPCIEMSIX_VECT0_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT0_ADDR_HI                                                                       0x1a001
+#define regPCIEMSIX_VECT0_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT0_MSG_DATA                                                                      0x1a002
+#define regPCIEMSIX_VECT0_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT0_CONTROL                                                                       0x1a003
+#define regPCIEMSIX_VECT0_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT1_ADDR_LO                                                                       0x1a004
+#define regPCIEMSIX_VECT1_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT1_ADDR_HI                                                                       0x1a005
+#define regPCIEMSIX_VECT1_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT1_MSG_DATA                                                                      0x1a006
+#define regPCIEMSIX_VECT1_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT1_CONTROL                                                                       0x1a007
+#define regPCIEMSIX_VECT1_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT2_ADDR_LO                                                                       0x1a008
+#define regPCIEMSIX_VECT2_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT2_ADDR_HI                                                                       0x1a009
+#define regPCIEMSIX_VECT2_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT2_MSG_DATA                                                                      0x1a00a
+#define regPCIEMSIX_VECT2_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT2_CONTROL                                                                       0x1a00b
+#define regPCIEMSIX_VECT2_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT3_ADDR_LO                                                                       0x1a00c
+#define regPCIEMSIX_VECT3_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT3_ADDR_HI                                                                       0x1a00d
+#define regPCIEMSIX_VECT3_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT3_MSG_DATA                                                                      0x1a00e
+#define regPCIEMSIX_VECT3_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT3_CONTROL                                                                       0x1a00f
+#define regPCIEMSIX_VECT3_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT4_ADDR_LO                                                                       0x1a010
+#define regPCIEMSIX_VECT4_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT4_ADDR_HI                                                                       0x1a011
+#define regPCIEMSIX_VECT4_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT4_MSG_DATA                                                                      0x1a012
+#define regPCIEMSIX_VECT4_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT4_CONTROL                                                                       0x1a013
+#define regPCIEMSIX_VECT4_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT5_ADDR_LO                                                                       0x1a014
+#define regPCIEMSIX_VECT5_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT5_ADDR_HI                                                                       0x1a015
+#define regPCIEMSIX_VECT5_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT5_MSG_DATA                                                                      0x1a016
+#define regPCIEMSIX_VECT5_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT5_CONTROL                                                                       0x1a017
+#define regPCIEMSIX_VECT5_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT6_ADDR_LO                                                                       0x1a018
+#define regPCIEMSIX_VECT6_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT6_ADDR_HI                                                                       0x1a019
+#define regPCIEMSIX_VECT6_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT6_MSG_DATA                                                                      0x1a01a
+#define regPCIEMSIX_VECT6_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT6_CONTROL                                                                       0x1a01b
+#define regPCIEMSIX_VECT6_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT7_ADDR_LO                                                                       0x1a01c
+#define regPCIEMSIX_VECT7_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT7_ADDR_HI                                                                       0x1a01d
+#define regPCIEMSIX_VECT7_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT7_MSG_DATA                                                                      0x1a01e
+#define regPCIEMSIX_VECT7_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT7_CONTROL                                                                       0x1a01f
+#define regPCIEMSIX_VECT7_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT8_ADDR_LO                                                                       0x1a020
+#define regPCIEMSIX_VECT8_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT8_ADDR_HI                                                                       0x1a021
+#define regPCIEMSIX_VECT8_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT8_MSG_DATA                                                                      0x1a022
+#define regPCIEMSIX_VECT8_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT8_CONTROL                                                                       0x1a023
+#define regPCIEMSIX_VECT8_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT9_ADDR_LO                                                                       0x1a024
+#define regPCIEMSIX_VECT9_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT9_ADDR_HI                                                                       0x1a025
+#define regPCIEMSIX_VECT9_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT9_MSG_DATA                                                                      0x1a026
+#define regPCIEMSIX_VECT9_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT9_CONTROL                                                                       0x1a027
+#define regPCIEMSIX_VECT9_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT10_ADDR_LO                                                                      0x1a028
+#define regPCIEMSIX_VECT10_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT10_ADDR_HI                                                                      0x1a029
+#define regPCIEMSIX_VECT10_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT10_MSG_DATA                                                                     0x1a02a
+#define regPCIEMSIX_VECT10_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT10_CONTROL                                                                      0x1a02b
+#define regPCIEMSIX_VECT10_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT11_ADDR_LO                                                                      0x1a02c
+#define regPCIEMSIX_VECT11_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT11_ADDR_HI                                                                      0x1a02d
+#define regPCIEMSIX_VECT11_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT11_MSG_DATA                                                                     0x1a02e
+#define regPCIEMSIX_VECT11_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT11_CONTROL                                                                      0x1a02f
+#define regPCIEMSIX_VECT11_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT12_ADDR_LO                                                                      0x1a030
+#define regPCIEMSIX_VECT12_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT12_ADDR_HI                                                                      0x1a031
+#define regPCIEMSIX_VECT12_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT12_MSG_DATA                                                                     0x1a032
+#define regPCIEMSIX_VECT12_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT12_CONTROL                                                                      0x1a033
+#define regPCIEMSIX_VECT12_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT13_ADDR_LO                                                                      0x1a034
+#define regPCIEMSIX_VECT13_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT13_ADDR_HI                                                                      0x1a035
+#define regPCIEMSIX_VECT13_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT13_MSG_DATA                                                                     0x1a036
+#define regPCIEMSIX_VECT13_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT13_CONTROL                                                                      0x1a037
+#define regPCIEMSIX_VECT13_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT14_ADDR_LO                                                                      0x1a038
+#define regPCIEMSIX_VECT14_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT14_ADDR_HI                                                                      0x1a039
+#define regPCIEMSIX_VECT14_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT14_MSG_DATA                                                                     0x1a03a
+#define regPCIEMSIX_VECT14_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT14_CONTROL                                                                      0x1a03b
+#define regPCIEMSIX_VECT14_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT15_ADDR_LO                                                                      0x1a03c
+#define regPCIEMSIX_VECT15_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT15_ADDR_HI                                                                      0x1a03d
+#define regPCIEMSIX_VECT15_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT15_MSG_DATA                                                                     0x1a03e
+#define regPCIEMSIX_VECT15_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT15_CONTROL                                                                      0x1a03f
+#define regPCIEMSIX_VECT15_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT16_ADDR_LO                                                                      0x1a040
+#define regPCIEMSIX_VECT16_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT16_ADDR_HI                                                                      0x1a041
+#define regPCIEMSIX_VECT16_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT16_MSG_DATA                                                                     0x1a042
+#define regPCIEMSIX_VECT16_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT16_CONTROL                                                                      0x1a043
+#define regPCIEMSIX_VECT16_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT17_ADDR_LO                                                                      0x1a044
+#define regPCIEMSIX_VECT17_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT17_ADDR_HI                                                                      0x1a045
+#define regPCIEMSIX_VECT17_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT17_MSG_DATA                                                                     0x1a046
+#define regPCIEMSIX_VECT17_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT17_CONTROL                                                                      0x1a047
+#define regPCIEMSIX_VECT17_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT18_ADDR_LO                                                                      0x1a048
+#define regPCIEMSIX_VECT18_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT18_ADDR_HI                                                                      0x1a049
+#define regPCIEMSIX_VECT18_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT18_MSG_DATA                                                                     0x1a04a
+#define regPCIEMSIX_VECT18_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT18_CONTROL                                                                      0x1a04b
+#define regPCIEMSIX_VECT18_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT19_ADDR_LO                                                                      0x1a04c
+#define regPCIEMSIX_VECT19_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT19_ADDR_HI                                                                      0x1a04d
+#define regPCIEMSIX_VECT19_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT19_MSG_DATA                                                                     0x1a04e
+#define regPCIEMSIX_VECT19_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT19_CONTROL                                                                      0x1a04f
+#define regPCIEMSIX_VECT19_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT20_ADDR_LO                                                                      0x1a050
+#define regPCIEMSIX_VECT20_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT20_ADDR_HI                                                                      0x1a051
+#define regPCIEMSIX_VECT20_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT20_MSG_DATA                                                                     0x1a052
+#define regPCIEMSIX_VECT20_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT20_CONTROL                                                                      0x1a053
+#define regPCIEMSIX_VECT20_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT21_ADDR_LO                                                                      0x1a054
+#define regPCIEMSIX_VECT21_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT21_ADDR_HI                                                                      0x1a055
+#define regPCIEMSIX_VECT21_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT21_MSG_DATA                                                                     0x1a056
+#define regPCIEMSIX_VECT21_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT21_CONTROL                                                                      0x1a057
+#define regPCIEMSIX_VECT21_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT22_ADDR_LO                                                                      0x1a058
+#define regPCIEMSIX_VECT22_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT22_ADDR_HI                                                                      0x1a059
+#define regPCIEMSIX_VECT22_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT22_MSG_DATA                                                                     0x1a05a
+#define regPCIEMSIX_VECT22_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT22_CONTROL                                                                      0x1a05b
+#define regPCIEMSIX_VECT22_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT23_ADDR_LO                                                                      0x1a05c
+#define regPCIEMSIX_VECT23_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT23_ADDR_HI                                                                      0x1a05d
+#define regPCIEMSIX_VECT23_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT23_MSG_DATA                                                                     0x1a05e
+#define regPCIEMSIX_VECT23_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT23_CONTROL                                                                      0x1a05f
+#define regPCIEMSIX_VECT23_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT24_ADDR_LO                                                                      0x1a060
+#define regPCIEMSIX_VECT24_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT24_ADDR_HI                                                                      0x1a061
+#define regPCIEMSIX_VECT24_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT24_MSG_DATA                                                                     0x1a062
+#define regPCIEMSIX_VECT24_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT24_CONTROL                                                                      0x1a063
+#define regPCIEMSIX_VECT24_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT25_ADDR_LO                                                                      0x1a064
+#define regPCIEMSIX_VECT25_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT25_ADDR_HI                                                                      0x1a065
+#define regPCIEMSIX_VECT25_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT25_MSG_DATA                                                                     0x1a066
+#define regPCIEMSIX_VECT25_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT25_CONTROL                                                                      0x1a067
+#define regPCIEMSIX_VECT25_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT26_ADDR_LO                                                                      0x1a068
+#define regPCIEMSIX_VECT26_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT26_ADDR_HI                                                                      0x1a069
+#define regPCIEMSIX_VECT26_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT26_MSG_DATA                                                                     0x1a06a
+#define regPCIEMSIX_VECT26_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT26_CONTROL                                                                      0x1a06b
+#define regPCIEMSIX_VECT26_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT27_ADDR_LO                                                                      0x1a06c
+#define regPCIEMSIX_VECT27_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT27_ADDR_HI                                                                      0x1a06d
+#define regPCIEMSIX_VECT27_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT27_MSG_DATA                                                                     0x1a06e
+#define regPCIEMSIX_VECT27_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT27_CONTROL                                                                      0x1a06f
+#define regPCIEMSIX_VECT27_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT28_ADDR_LO                                                                      0x1a070
+#define regPCIEMSIX_VECT28_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT28_ADDR_HI                                                                      0x1a071
+#define regPCIEMSIX_VECT28_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT28_MSG_DATA                                                                     0x1a072
+#define regPCIEMSIX_VECT28_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT28_CONTROL                                                                      0x1a073
+#define regPCIEMSIX_VECT28_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT29_ADDR_LO                                                                      0x1a074
+#define regPCIEMSIX_VECT29_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT29_ADDR_HI                                                                      0x1a075
+#define regPCIEMSIX_VECT29_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT29_MSG_DATA                                                                     0x1a076
+#define regPCIEMSIX_VECT29_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT29_CONTROL                                                                      0x1a077
+#define regPCIEMSIX_VECT29_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT30_ADDR_LO                                                                      0x1a078
+#define regPCIEMSIX_VECT30_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT30_ADDR_HI                                                                      0x1a079
+#define regPCIEMSIX_VECT30_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT30_MSG_DATA                                                                     0x1a07a
+#define regPCIEMSIX_VECT30_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT30_CONTROL                                                                      0x1a07b
+#define regPCIEMSIX_VECT30_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT31_ADDR_LO                                                                      0x1a07c
+#define regPCIEMSIX_VECT31_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT31_ADDR_HI                                                                      0x1a07d
+#define regPCIEMSIX_VECT31_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT31_MSG_DATA                                                                     0x1a07e
+#define regPCIEMSIX_VECT31_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT31_CONTROL                                                                      0x1a07f
+#define regPCIEMSIX_VECT31_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT32_ADDR_LO                                                                      0x1a080
+#define regPCIEMSIX_VECT32_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT32_ADDR_HI                                                                      0x1a081
+#define regPCIEMSIX_VECT32_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT32_MSG_DATA                                                                     0x1a082
+#define regPCIEMSIX_VECT32_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT32_CONTROL                                                                      0x1a083
+#define regPCIEMSIX_VECT32_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT33_ADDR_LO                                                                      0x1a084
+#define regPCIEMSIX_VECT33_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT33_ADDR_HI                                                                      0x1a085
+#define regPCIEMSIX_VECT33_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT33_MSG_DATA                                                                     0x1a086
+#define regPCIEMSIX_VECT33_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT33_CONTROL                                                                      0x1a087
+#define regPCIEMSIX_VECT33_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT34_ADDR_LO                                                                      0x1a088
+#define regPCIEMSIX_VECT34_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT34_ADDR_HI                                                                      0x1a089
+#define regPCIEMSIX_VECT34_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT34_MSG_DATA                                                                     0x1a08a
+#define regPCIEMSIX_VECT34_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT34_CONTROL                                                                      0x1a08b
+#define regPCIEMSIX_VECT34_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT35_ADDR_LO                                                                      0x1a08c
+#define regPCIEMSIX_VECT35_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT35_ADDR_HI                                                                      0x1a08d
+#define regPCIEMSIX_VECT35_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT35_MSG_DATA                                                                     0x1a08e
+#define regPCIEMSIX_VECT35_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT35_CONTROL                                                                      0x1a08f
+#define regPCIEMSIX_VECT35_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT36_ADDR_LO                                                                      0x1a090
+#define regPCIEMSIX_VECT36_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT36_ADDR_HI                                                                      0x1a091
+#define regPCIEMSIX_VECT36_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT36_MSG_DATA                                                                     0x1a092
+#define regPCIEMSIX_VECT36_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT36_CONTROL                                                                      0x1a093
+#define regPCIEMSIX_VECT36_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT37_ADDR_LO                                                                      0x1a094
+#define regPCIEMSIX_VECT37_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT37_ADDR_HI                                                                      0x1a095
+#define regPCIEMSIX_VECT37_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT37_MSG_DATA                                                                     0x1a096
+#define regPCIEMSIX_VECT37_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT37_CONTROL                                                                      0x1a097
+#define regPCIEMSIX_VECT37_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT38_ADDR_LO                                                                      0x1a098
+#define regPCIEMSIX_VECT38_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT38_ADDR_HI                                                                      0x1a099
+#define regPCIEMSIX_VECT38_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT38_MSG_DATA                                                                     0x1a09a
+#define regPCIEMSIX_VECT38_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT38_CONTROL                                                                      0x1a09b
+#define regPCIEMSIX_VECT38_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT39_ADDR_LO                                                                      0x1a09c
+#define regPCIEMSIX_VECT39_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT39_ADDR_HI                                                                      0x1a09d
+#define regPCIEMSIX_VECT39_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT39_MSG_DATA                                                                     0x1a09e
+#define regPCIEMSIX_VECT39_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT39_CONTROL                                                                      0x1a09f
+#define regPCIEMSIX_VECT39_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT40_ADDR_LO                                                                      0x1a0a0
+#define regPCIEMSIX_VECT40_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT40_ADDR_HI                                                                      0x1a0a1
+#define regPCIEMSIX_VECT40_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT40_MSG_DATA                                                                     0x1a0a2
+#define regPCIEMSIX_VECT40_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT40_CONTROL                                                                      0x1a0a3
+#define regPCIEMSIX_VECT40_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT41_ADDR_LO                                                                      0x1a0a4
+#define regPCIEMSIX_VECT41_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT41_ADDR_HI                                                                      0x1a0a5
+#define regPCIEMSIX_VECT41_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT41_MSG_DATA                                                                     0x1a0a6
+#define regPCIEMSIX_VECT41_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT41_CONTROL                                                                      0x1a0a7
+#define regPCIEMSIX_VECT41_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT42_ADDR_LO                                                                      0x1a0a8
+#define regPCIEMSIX_VECT42_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT42_ADDR_HI                                                                      0x1a0a9
+#define regPCIEMSIX_VECT42_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT42_MSG_DATA                                                                     0x1a0aa
+#define regPCIEMSIX_VECT42_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT42_CONTROL                                                                      0x1a0ab
+#define regPCIEMSIX_VECT42_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT43_ADDR_LO                                                                      0x1a0ac
+#define regPCIEMSIX_VECT43_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT43_ADDR_HI                                                                      0x1a0ad
+#define regPCIEMSIX_VECT43_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT43_MSG_DATA                                                                     0x1a0ae
+#define regPCIEMSIX_VECT43_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT43_CONTROL                                                                      0x1a0af
+#define regPCIEMSIX_VECT43_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT44_ADDR_LO                                                                      0x1a0b0
+#define regPCIEMSIX_VECT44_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT44_ADDR_HI                                                                      0x1a0b1
+#define regPCIEMSIX_VECT44_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT44_MSG_DATA                                                                     0x1a0b2
+#define regPCIEMSIX_VECT44_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT44_CONTROL                                                                      0x1a0b3
+#define regPCIEMSIX_VECT44_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT45_ADDR_LO                                                                      0x1a0b4
+#define regPCIEMSIX_VECT45_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT45_ADDR_HI                                                                      0x1a0b5
+#define regPCIEMSIX_VECT45_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT45_MSG_DATA                                                                     0x1a0b6
+#define regPCIEMSIX_VECT45_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT45_CONTROL                                                                      0x1a0b7
+#define regPCIEMSIX_VECT45_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT46_ADDR_LO                                                                      0x1a0b8
+#define regPCIEMSIX_VECT46_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT46_ADDR_HI                                                                      0x1a0b9
+#define regPCIEMSIX_VECT46_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT46_MSG_DATA                                                                     0x1a0ba
+#define regPCIEMSIX_VECT46_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT46_CONTROL                                                                      0x1a0bb
+#define regPCIEMSIX_VECT46_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT47_ADDR_LO                                                                      0x1a0bc
+#define regPCIEMSIX_VECT47_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT47_ADDR_HI                                                                      0x1a0bd
+#define regPCIEMSIX_VECT47_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT47_MSG_DATA                                                                     0x1a0be
+#define regPCIEMSIX_VECT47_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT47_CONTROL                                                                      0x1a0bf
+#define regPCIEMSIX_VECT47_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT48_ADDR_LO                                                                      0x1a0c0
+#define regPCIEMSIX_VECT48_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT48_ADDR_HI                                                                      0x1a0c1
+#define regPCIEMSIX_VECT48_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT48_MSG_DATA                                                                     0x1a0c2
+#define regPCIEMSIX_VECT48_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT48_CONTROL                                                                      0x1a0c3
+#define regPCIEMSIX_VECT48_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT49_ADDR_LO                                                                      0x1a0c4
+#define regPCIEMSIX_VECT49_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT49_ADDR_HI                                                                      0x1a0c5
+#define regPCIEMSIX_VECT49_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT49_MSG_DATA                                                                     0x1a0c6
+#define regPCIEMSIX_VECT49_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT49_CONTROL                                                                      0x1a0c7
+#define regPCIEMSIX_VECT49_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT50_ADDR_LO                                                                      0x1a0c8
+#define regPCIEMSIX_VECT50_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT50_ADDR_HI                                                                      0x1a0c9
+#define regPCIEMSIX_VECT50_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT50_MSG_DATA                                                                     0x1a0ca
+#define regPCIEMSIX_VECT50_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT50_CONTROL                                                                      0x1a0cb
+#define regPCIEMSIX_VECT50_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT51_ADDR_LO                                                                      0x1a0cc
+#define regPCIEMSIX_VECT51_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT51_ADDR_HI                                                                      0x1a0cd
+#define regPCIEMSIX_VECT51_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT51_MSG_DATA                                                                     0x1a0ce
+#define regPCIEMSIX_VECT51_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT51_CONTROL                                                                      0x1a0cf
+#define regPCIEMSIX_VECT51_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT52_ADDR_LO                                                                      0x1a0d0
+#define regPCIEMSIX_VECT52_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT52_ADDR_HI                                                                      0x1a0d1
+#define regPCIEMSIX_VECT52_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT52_MSG_DATA                                                                     0x1a0d2
+#define regPCIEMSIX_VECT52_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT52_CONTROL                                                                      0x1a0d3
+#define regPCIEMSIX_VECT52_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT53_ADDR_LO                                                                      0x1a0d4
+#define regPCIEMSIX_VECT53_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT53_ADDR_HI                                                                      0x1a0d5
+#define regPCIEMSIX_VECT53_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT53_MSG_DATA                                                                     0x1a0d6
+#define regPCIEMSIX_VECT53_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT53_CONTROL                                                                      0x1a0d7
+#define regPCIEMSIX_VECT53_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT54_ADDR_LO                                                                      0x1a0d8
+#define regPCIEMSIX_VECT54_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT54_ADDR_HI                                                                      0x1a0d9
+#define regPCIEMSIX_VECT54_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT54_MSG_DATA                                                                     0x1a0da
+#define regPCIEMSIX_VECT54_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT54_CONTROL                                                                      0x1a0db
+#define regPCIEMSIX_VECT54_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT55_ADDR_LO                                                                      0x1a0dc
+#define regPCIEMSIX_VECT55_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT55_ADDR_HI                                                                      0x1a0dd
+#define regPCIEMSIX_VECT55_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT55_MSG_DATA                                                                     0x1a0de
+#define regPCIEMSIX_VECT55_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT55_CONTROL                                                                      0x1a0df
+#define regPCIEMSIX_VECT55_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT56_ADDR_LO                                                                      0x1a0e0
+#define regPCIEMSIX_VECT56_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT56_ADDR_HI                                                                      0x1a0e1
+#define regPCIEMSIX_VECT56_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT56_MSG_DATA                                                                     0x1a0e2
+#define regPCIEMSIX_VECT56_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT56_CONTROL                                                                      0x1a0e3
+#define regPCIEMSIX_VECT56_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT57_ADDR_LO                                                                      0x1a0e4
+#define regPCIEMSIX_VECT57_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT57_ADDR_HI                                                                      0x1a0e5
+#define regPCIEMSIX_VECT57_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT57_MSG_DATA                                                                     0x1a0e6
+#define regPCIEMSIX_VECT57_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT57_CONTROL                                                                      0x1a0e7
+#define regPCIEMSIX_VECT57_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT58_ADDR_LO                                                                      0x1a0e8
+#define regPCIEMSIX_VECT58_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT58_ADDR_HI                                                                      0x1a0e9
+#define regPCIEMSIX_VECT58_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT58_MSG_DATA                                                                     0x1a0ea
+#define regPCIEMSIX_VECT58_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT58_CONTROL                                                                      0x1a0eb
+#define regPCIEMSIX_VECT58_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT59_ADDR_LO                                                                      0x1a0ec
+#define regPCIEMSIX_VECT59_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT59_ADDR_HI                                                                      0x1a0ed
+#define regPCIEMSIX_VECT59_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT59_MSG_DATA                                                                     0x1a0ee
+#define regPCIEMSIX_VECT59_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT59_CONTROL                                                                      0x1a0ef
+#define regPCIEMSIX_VECT59_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT60_ADDR_LO                                                                      0x1a0f0
+#define regPCIEMSIX_VECT60_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT60_ADDR_HI                                                                      0x1a0f1
+#define regPCIEMSIX_VECT60_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT60_MSG_DATA                                                                     0x1a0f2
+#define regPCIEMSIX_VECT60_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT60_CONTROL                                                                      0x1a0f3
+#define regPCIEMSIX_VECT60_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT61_ADDR_LO                                                                      0x1a0f4
+#define regPCIEMSIX_VECT61_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT61_ADDR_HI                                                                      0x1a0f5
+#define regPCIEMSIX_VECT61_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT61_MSG_DATA                                                                     0x1a0f6
+#define regPCIEMSIX_VECT61_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT61_CONTROL                                                                      0x1a0f7
+#define regPCIEMSIX_VECT61_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT62_ADDR_LO                                                                      0x1a0f8
+#define regPCIEMSIX_VECT62_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT62_ADDR_HI                                                                      0x1a0f9
+#define regPCIEMSIX_VECT62_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT62_MSG_DATA                                                                     0x1a0fa
+#define regPCIEMSIX_VECT62_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT62_CONTROL                                                                      0x1a0fb
+#define regPCIEMSIX_VECT62_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT63_ADDR_LO                                                                      0x1a0fc
+#define regPCIEMSIX_VECT63_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT63_ADDR_HI                                                                      0x1a0fd
+#define regPCIEMSIX_VECT63_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT63_MSG_DATA                                                                     0x1a0fe
+#define regPCIEMSIX_VECT63_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT63_CONTROL                                                                      0x1a0ff
+#define regPCIEMSIX_VECT63_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT64_ADDR_LO                                                                      0x1a100
+#define regPCIEMSIX_VECT64_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT64_ADDR_HI                                                                      0x1a101
+#define regPCIEMSIX_VECT64_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT64_MSG_DATA                                                                     0x1a102
+#define regPCIEMSIX_VECT64_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT64_CONTROL                                                                      0x1a103
+#define regPCIEMSIX_VECT64_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT65_ADDR_LO                                                                      0x1a104
+#define regPCIEMSIX_VECT65_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT65_ADDR_HI                                                                      0x1a105
+#define regPCIEMSIX_VECT65_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT65_MSG_DATA                                                                     0x1a106
+#define regPCIEMSIX_VECT65_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT65_CONTROL                                                                      0x1a107
+#define regPCIEMSIX_VECT65_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT66_ADDR_LO                                                                      0x1a108
+#define regPCIEMSIX_VECT66_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT66_ADDR_HI                                                                      0x1a109
+#define regPCIEMSIX_VECT66_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT66_MSG_DATA                                                                     0x1a10a
+#define regPCIEMSIX_VECT66_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT66_CONTROL                                                                      0x1a10b
+#define regPCIEMSIX_VECT66_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT67_ADDR_LO                                                                      0x1a10c
+#define regPCIEMSIX_VECT67_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT67_ADDR_HI                                                                      0x1a10d
+#define regPCIEMSIX_VECT67_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT67_MSG_DATA                                                                     0x1a10e
+#define regPCIEMSIX_VECT67_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT67_CONTROL                                                                      0x1a10f
+#define regPCIEMSIX_VECT67_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT68_ADDR_LO                                                                      0x1a110
+#define regPCIEMSIX_VECT68_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT68_ADDR_HI                                                                      0x1a111
+#define regPCIEMSIX_VECT68_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT68_MSG_DATA                                                                     0x1a112
+#define regPCIEMSIX_VECT68_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT68_CONTROL                                                                      0x1a113
+#define regPCIEMSIX_VECT68_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT69_ADDR_LO                                                                      0x1a114
+#define regPCIEMSIX_VECT69_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT69_ADDR_HI                                                                      0x1a115
+#define regPCIEMSIX_VECT69_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT69_MSG_DATA                                                                     0x1a116
+#define regPCIEMSIX_VECT69_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT69_CONTROL                                                                      0x1a117
+#define regPCIEMSIX_VECT69_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT70_ADDR_LO                                                                      0x1a118
+#define regPCIEMSIX_VECT70_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT70_ADDR_HI                                                                      0x1a119
+#define regPCIEMSIX_VECT70_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT70_MSG_DATA                                                                     0x1a11a
+#define regPCIEMSIX_VECT70_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT70_CONTROL                                                                      0x1a11b
+#define regPCIEMSIX_VECT70_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT71_ADDR_LO                                                                      0x1a11c
+#define regPCIEMSIX_VECT71_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT71_ADDR_HI                                                                      0x1a11d
+#define regPCIEMSIX_VECT71_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT71_MSG_DATA                                                                     0x1a11e
+#define regPCIEMSIX_VECT71_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT71_CONTROL                                                                      0x1a11f
+#define regPCIEMSIX_VECT71_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT72_ADDR_LO                                                                      0x1a120
+#define regPCIEMSIX_VECT72_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT72_ADDR_HI                                                                      0x1a121
+#define regPCIEMSIX_VECT72_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT72_MSG_DATA                                                                     0x1a122
+#define regPCIEMSIX_VECT72_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT72_CONTROL                                                                      0x1a123
+#define regPCIEMSIX_VECT72_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT73_ADDR_LO                                                                      0x1a124
+#define regPCIEMSIX_VECT73_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT73_ADDR_HI                                                                      0x1a125
+#define regPCIEMSIX_VECT73_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT73_MSG_DATA                                                                     0x1a126
+#define regPCIEMSIX_VECT73_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT73_CONTROL                                                                      0x1a127
+#define regPCIEMSIX_VECT73_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT74_ADDR_LO                                                                      0x1a128
+#define regPCIEMSIX_VECT74_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT74_ADDR_HI                                                                      0x1a129
+#define regPCIEMSIX_VECT74_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT74_MSG_DATA                                                                     0x1a12a
+#define regPCIEMSIX_VECT74_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT74_CONTROL                                                                      0x1a12b
+#define regPCIEMSIX_VECT74_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT75_ADDR_LO                                                                      0x1a12c
+#define regPCIEMSIX_VECT75_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT75_ADDR_HI                                                                      0x1a12d
+#define regPCIEMSIX_VECT75_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT75_MSG_DATA                                                                     0x1a12e
+#define regPCIEMSIX_VECT75_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT75_CONTROL                                                                      0x1a12f
+#define regPCIEMSIX_VECT75_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT76_ADDR_LO                                                                      0x1a130
+#define regPCIEMSIX_VECT76_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT76_ADDR_HI                                                                      0x1a131
+#define regPCIEMSIX_VECT76_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT76_MSG_DATA                                                                     0x1a132
+#define regPCIEMSIX_VECT76_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT76_CONTROL                                                                      0x1a133
+#define regPCIEMSIX_VECT76_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT77_ADDR_LO                                                                      0x1a134
+#define regPCIEMSIX_VECT77_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT77_ADDR_HI                                                                      0x1a135
+#define regPCIEMSIX_VECT77_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT77_MSG_DATA                                                                     0x1a136
+#define regPCIEMSIX_VECT77_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT77_CONTROL                                                                      0x1a137
+#define regPCIEMSIX_VECT77_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT78_ADDR_LO                                                                      0x1a138
+#define regPCIEMSIX_VECT78_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT78_ADDR_HI                                                                      0x1a139
+#define regPCIEMSIX_VECT78_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT78_MSG_DATA                                                                     0x1a13a
+#define regPCIEMSIX_VECT78_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT78_CONTROL                                                                      0x1a13b
+#define regPCIEMSIX_VECT78_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT79_ADDR_LO                                                                      0x1a13c
+#define regPCIEMSIX_VECT79_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT79_ADDR_HI                                                                      0x1a13d
+#define regPCIEMSIX_VECT79_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT79_MSG_DATA                                                                     0x1a13e
+#define regPCIEMSIX_VECT79_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT79_CONTROL                                                                      0x1a13f
+#define regPCIEMSIX_VECT79_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT80_ADDR_LO                                                                      0x1a140
+#define regPCIEMSIX_VECT80_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT80_ADDR_HI                                                                      0x1a141
+#define regPCIEMSIX_VECT80_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT80_MSG_DATA                                                                     0x1a142
+#define regPCIEMSIX_VECT80_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT80_CONTROL                                                                      0x1a143
+#define regPCIEMSIX_VECT80_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT81_ADDR_LO                                                                      0x1a144
+#define regPCIEMSIX_VECT81_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT81_ADDR_HI                                                                      0x1a145
+#define regPCIEMSIX_VECT81_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT81_MSG_DATA                                                                     0x1a146
+#define regPCIEMSIX_VECT81_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT81_CONTROL                                                                      0x1a147
+#define regPCIEMSIX_VECT81_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT82_ADDR_LO                                                                      0x1a148
+#define regPCIEMSIX_VECT82_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT82_ADDR_HI                                                                      0x1a149
+#define regPCIEMSIX_VECT82_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT82_MSG_DATA                                                                     0x1a14a
+#define regPCIEMSIX_VECT82_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT82_CONTROL                                                                      0x1a14b
+#define regPCIEMSIX_VECT82_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT83_ADDR_LO                                                                      0x1a14c
+#define regPCIEMSIX_VECT83_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT83_ADDR_HI                                                                      0x1a14d
+#define regPCIEMSIX_VECT83_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT83_MSG_DATA                                                                     0x1a14e
+#define regPCIEMSIX_VECT83_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT83_CONTROL                                                                      0x1a14f
+#define regPCIEMSIX_VECT83_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT84_ADDR_LO                                                                      0x1a150
+#define regPCIEMSIX_VECT84_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT84_ADDR_HI                                                                      0x1a151
+#define regPCIEMSIX_VECT84_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT84_MSG_DATA                                                                     0x1a152
+#define regPCIEMSIX_VECT84_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT84_CONTROL                                                                      0x1a153
+#define regPCIEMSIX_VECT84_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT85_ADDR_LO                                                                      0x1a154
+#define regPCIEMSIX_VECT85_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT85_ADDR_HI                                                                      0x1a155
+#define regPCIEMSIX_VECT85_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT85_MSG_DATA                                                                     0x1a156
+#define regPCIEMSIX_VECT85_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT85_CONTROL                                                                      0x1a157
+#define regPCIEMSIX_VECT85_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT86_ADDR_LO                                                                      0x1a158
+#define regPCIEMSIX_VECT86_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT86_ADDR_HI                                                                      0x1a159
+#define regPCIEMSIX_VECT86_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT86_MSG_DATA                                                                     0x1a15a
+#define regPCIEMSIX_VECT86_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT86_CONTROL                                                                      0x1a15b
+#define regPCIEMSIX_VECT86_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT87_ADDR_LO                                                                      0x1a15c
+#define regPCIEMSIX_VECT87_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT87_ADDR_HI                                                                      0x1a15d
+#define regPCIEMSIX_VECT87_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT87_MSG_DATA                                                                     0x1a15e
+#define regPCIEMSIX_VECT87_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT87_CONTROL                                                                      0x1a15f
+#define regPCIEMSIX_VECT87_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT88_ADDR_LO                                                                      0x1a160
+#define regPCIEMSIX_VECT88_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT88_ADDR_HI                                                                      0x1a161
+#define regPCIEMSIX_VECT88_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT88_MSG_DATA                                                                     0x1a162
+#define regPCIEMSIX_VECT88_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT88_CONTROL                                                                      0x1a163
+#define regPCIEMSIX_VECT88_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT89_ADDR_LO                                                                      0x1a164
+#define regPCIEMSIX_VECT89_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT89_ADDR_HI                                                                      0x1a165
+#define regPCIEMSIX_VECT89_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT89_MSG_DATA                                                                     0x1a166
+#define regPCIEMSIX_VECT89_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT89_CONTROL                                                                      0x1a167
+#define regPCIEMSIX_VECT89_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT90_ADDR_LO                                                                      0x1a168
+#define regPCIEMSIX_VECT90_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT90_ADDR_HI                                                                      0x1a169
+#define regPCIEMSIX_VECT90_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT90_MSG_DATA                                                                     0x1a16a
+#define regPCIEMSIX_VECT90_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT90_CONTROL                                                                      0x1a16b
+#define regPCIEMSIX_VECT90_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT91_ADDR_LO                                                                      0x1a16c
+#define regPCIEMSIX_VECT91_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT91_ADDR_HI                                                                      0x1a16d
+#define regPCIEMSIX_VECT91_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT91_MSG_DATA                                                                     0x1a16e
+#define regPCIEMSIX_VECT91_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT91_CONTROL                                                                      0x1a16f
+#define regPCIEMSIX_VECT91_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT92_ADDR_LO                                                                      0x1a170
+#define regPCIEMSIX_VECT92_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT92_ADDR_HI                                                                      0x1a171
+#define regPCIEMSIX_VECT92_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT92_MSG_DATA                                                                     0x1a172
+#define regPCIEMSIX_VECT92_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT92_CONTROL                                                                      0x1a173
+#define regPCIEMSIX_VECT92_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT93_ADDR_LO                                                                      0x1a174
+#define regPCIEMSIX_VECT93_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT93_ADDR_HI                                                                      0x1a175
+#define regPCIEMSIX_VECT93_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT93_MSG_DATA                                                                     0x1a176
+#define regPCIEMSIX_VECT93_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT93_CONTROL                                                                      0x1a177
+#define regPCIEMSIX_VECT93_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT94_ADDR_LO                                                                      0x1a178
+#define regPCIEMSIX_VECT94_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT94_ADDR_HI                                                                      0x1a179
+#define regPCIEMSIX_VECT94_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT94_MSG_DATA                                                                     0x1a17a
+#define regPCIEMSIX_VECT94_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT94_CONTROL                                                                      0x1a17b
+#define regPCIEMSIX_VECT94_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT95_ADDR_LO                                                                      0x1a17c
+#define regPCIEMSIX_VECT95_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT95_ADDR_HI                                                                      0x1a17d
+#define regPCIEMSIX_VECT95_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT95_MSG_DATA                                                                     0x1a17e
+#define regPCIEMSIX_VECT95_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT95_CONTROL                                                                      0x1a17f
+#define regPCIEMSIX_VECT95_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT96_ADDR_LO                                                                      0x1a180
+#define regPCIEMSIX_VECT96_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT96_ADDR_HI                                                                      0x1a181
+#define regPCIEMSIX_VECT96_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT96_MSG_DATA                                                                     0x1a182
+#define regPCIEMSIX_VECT96_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT96_CONTROL                                                                      0x1a183
+#define regPCIEMSIX_VECT96_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT97_ADDR_LO                                                                      0x1a184
+#define regPCIEMSIX_VECT97_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT97_ADDR_HI                                                                      0x1a185
+#define regPCIEMSIX_VECT97_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT97_MSG_DATA                                                                     0x1a186
+#define regPCIEMSIX_VECT97_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT97_CONTROL                                                                      0x1a187
+#define regPCIEMSIX_VECT97_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT98_ADDR_LO                                                                      0x1a188
+#define regPCIEMSIX_VECT98_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT98_ADDR_HI                                                                      0x1a189
+#define regPCIEMSIX_VECT98_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT98_MSG_DATA                                                                     0x1a18a
+#define regPCIEMSIX_VECT98_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT98_CONTROL                                                                      0x1a18b
+#define regPCIEMSIX_VECT98_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT99_ADDR_LO                                                                      0x1a18c
+#define regPCIEMSIX_VECT99_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT99_ADDR_HI                                                                      0x1a18d
+#define regPCIEMSIX_VECT99_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT99_MSG_DATA                                                                     0x1a18e
+#define regPCIEMSIX_VECT99_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT99_CONTROL                                                                      0x1a18f
+#define regPCIEMSIX_VECT99_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT100_ADDR_LO                                                                     0x1a190
+#define regPCIEMSIX_VECT100_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT100_ADDR_HI                                                                     0x1a191
+#define regPCIEMSIX_VECT100_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT100_MSG_DATA                                                                    0x1a192
+#define regPCIEMSIX_VECT100_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT100_CONTROL                                                                     0x1a193
+#define regPCIEMSIX_VECT100_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT101_ADDR_LO                                                                     0x1a194
+#define regPCIEMSIX_VECT101_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT101_ADDR_HI                                                                     0x1a195
+#define regPCIEMSIX_VECT101_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT101_MSG_DATA                                                                    0x1a196
+#define regPCIEMSIX_VECT101_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT101_CONTROL                                                                     0x1a197
+#define regPCIEMSIX_VECT101_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT102_ADDR_LO                                                                     0x1a198
+#define regPCIEMSIX_VECT102_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT102_ADDR_HI                                                                     0x1a199
+#define regPCIEMSIX_VECT102_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT102_MSG_DATA                                                                    0x1a19a
+#define regPCIEMSIX_VECT102_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT102_CONTROL                                                                     0x1a19b
+#define regPCIEMSIX_VECT102_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT103_ADDR_LO                                                                     0x1a19c
+#define regPCIEMSIX_VECT103_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT103_ADDR_HI                                                                     0x1a19d
+#define regPCIEMSIX_VECT103_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT103_MSG_DATA                                                                    0x1a19e
+#define regPCIEMSIX_VECT103_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT103_CONTROL                                                                     0x1a19f
+#define regPCIEMSIX_VECT103_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT104_ADDR_LO                                                                     0x1a1a0
+#define regPCIEMSIX_VECT104_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT104_ADDR_HI                                                                     0x1a1a1
+#define regPCIEMSIX_VECT104_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT104_MSG_DATA                                                                    0x1a1a2
+#define regPCIEMSIX_VECT104_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT104_CONTROL                                                                     0x1a1a3
+#define regPCIEMSIX_VECT104_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT105_ADDR_LO                                                                     0x1a1a4
+#define regPCIEMSIX_VECT105_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT105_ADDR_HI                                                                     0x1a1a5
+#define regPCIEMSIX_VECT105_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT105_MSG_DATA                                                                    0x1a1a6
+#define regPCIEMSIX_VECT105_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT105_CONTROL                                                                     0x1a1a7
+#define regPCIEMSIX_VECT105_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT106_ADDR_LO                                                                     0x1a1a8
+#define regPCIEMSIX_VECT106_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT106_ADDR_HI                                                                     0x1a1a9
+#define regPCIEMSIX_VECT106_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT106_MSG_DATA                                                                    0x1a1aa
+#define regPCIEMSIX_VECT106_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT106_CONTROL                                                                     0x1a1ab
+#define regPCIEMSIX_VECT106_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT107_ADDR_LO                                                                     0x1a1ac
+#define regPCIEMSIX_VECT107_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT107_ADDR_HI                                                                     0x1a1ad
+#define regPCIEMSIX_VECT107_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT107_MSG_DATA                                                                    0x1a1ae
+#define regPCIEMSIX_VECT107_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT107_CONTROL                                                                     0x1a1af
+#define regPCIEMSIX_VECT107_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT108_ADDR_LO                                                                     0x1a1b0
+#define regPCIEMSIX_VECT108_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT108_ADDR_HI                                                                     0x1a1b1
+#define regPCIEMSIX_VECT108_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT108_MSG_DATA                                                                    0x1a1b2
+#define regPCIEMSIX_VECT108_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT108_CONTROL                                                                     0x1a1b3
+#define regPCIEMSIX_VECT108_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT109_ADDR_LO                                                                     0x1a1b4
+#define regPCIEMSIX_VECT109_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT109_ADDR_HI                                                                     0x1a1b5
+#define regPCIEMSIX_VECT109_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT109_MSG_DATA                                                                    0x1a1b6
+#define regPCIEMSIX_VECT109_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT109_CONTROL                                                                     0x1a1b7
+#define regPCIEMSIX_VECT109_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT110_ADDR_LO                                                                     0x1a1b8
+#define regPCIEMSIX_VECT110_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT110_ADDR_HI                                                                     0x1a1b9
+#define regPCIEMSIX_VECT110_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT110_MSG_DATA                                                                    0x1a1ba
+#define regPCIEMSIX_VECT110_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT110_CONTROL                                                                     0x1a1bb
+#define regPCIEMSIX_VECT110_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT111_ADDR_LO                                                                     0x1a1bc
+#define regPCIEMSIX_VECT111_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT111_ADDR_HI                                                                     0x1a1bd
+#define regPCIEMSIX_VECT111_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT111_MSG_DATA                                                                    0x1a1be
+#define regPCIEMSIX_VECT111_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT111_CONTROL                                                                     0x1a1bf
+#define regPCIEMSIX_VECT111_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT112_ADDR_LO                                                                     0x1a1c0
+#define regPCIEMSIX_VECT112_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT112_ADDR_HI                                                                     0x1a1c1
+#define regPCIEMSIX_VECT112_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT112_MSG_DATA                                                                    0x1a1c2
+#define regPCIEMSIX_VECT112_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT112_CONTROL                                                                     0x1a1c3
+#define regPCIEMSIX_VECT112_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT113_ADDR_LO                                                                     0x1a1c4
+#define regPCIEMSIX_VECT113_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT113_ADDR_HI                                                                     0x1a1c5
+#define regPCIEMSIX_VECT113_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT113_MSG_DATA                                                                    0x1a1c6
+#define regPCIEMSIX_VECT113_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT113_CONTROL                                                                     0x1a1c7
+#define regPCIEMSIX_VECT113_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT114_ADDR_LO                                                                     0x1a1c8
+#define regPCIEMSIX_VECT114_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT114_ADDR_HI                                                                     0x1a1c9
+#define regPCIEMSIX_VECT114_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT114_MSG_DATA                                                                    0x1a1ca
+#define regPCIEMSIX_VECT114_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT114_CONTROL                                                                     0x1a1cb
+#define regPCIEMSIX_VECT114_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT115_ADDR_LO                                                                     0x1a1cc
+#define regPCIEMSIX_VECT115_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT115_ADDR_HI                                                                     0x1a1cd
+#define regPCIEMSIX_VECT115_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT115_MSG_DATA                                                                    0x1a1ce
+#define regPCIEMSIX_VECT115_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT115_CONTROL                                                                     0x1a1cf
+#define regPCIEMSIX_VECT115_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT116_ADDR_LO                                                                     0x1a1d0
+#define regPCIEMSIX_VECT116_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT116_ADDR_HI                                                                     0x1a1d1
+#define regPCIEMSIX_VECT116_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT116_MSG_DATA                                                                    0x1a1d2
+#define regPCIEMSIX_VECT116_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT116_CONTROL                                                                     0x1a1d3
+#define regPCIEMSIX_VECT116_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT117_ADDR_LO                                                                     0x1a1d4
+#define regPCIEMSIX_VECT117_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT117_ADDR_HI                                                                     0x1a1d5
+#define regPCIEMSIX_VECT117_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT117_MSG_DATA                                                                    0x1a1d6
+#define regPCIEMSIX_VECT117_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT117_CONTROL                                                                     0x1a1d7
+#define regPCIEMSIX_VECT117_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT118_ADDR_LO                                                                     0x1a1d8
+#define regPCIEMSIX_VECT118_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT118_ADDR_HI                                                                     0x1a1d9
+#define regPCIEMSIX_VECT118_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT118_MSG_DATA                                                                    0x1a1da
+#define regPCIEMSIX_VECT118_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT118_CONTROL                                                                     0x1a1db
+#define regPCIEMSIX_VECT118_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT119_ADDR_LO                                                                     0x1a1dc
+#define regPCIEMSIX_VECT119_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT119_ADDR_HI                                                                     0x1a1dd
+#define regPCIEMSIX_VECT119_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT119_MSG_DATA                                                                    0x1a1de
+#define regPCIEMSIX_VECT119_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT119_CONTROL                                                                     0x1a1df
+#define regPCIEMSIX_VECT119_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT120_ADDR_LO                                                                     0x1a1e0
+#define regPCIEMSIX_VECT120_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT120_ADDR_HI                                                                     0x1a1e1
+#define regPCIEMSIX_VECT120_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT120_MSG_DATA                                                                    0x1a1e2
+#define regPCIEMSIX_VECT120_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT120_CONTROL                                                                     0x1a1e3
+#define regPCIEMSIX_VECT120_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT121_ADDR_LO                                                                     0x1a1e4
+#define regPCIEMSIX_VECT121_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT121_ADDR_HI                                                                     0x1a1e5
+#define regPCIEMSIX_VECT121_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT121_MSG_DATA                                                                    0x1a1e6
+#define regPCIEMSIX_VECT121_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT121_CONTROL                                                                     0x1a1e7
+#define regPCIEMSIX_VECT121_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT122_ADDR_LO                                                                     0x1a1e8
+#define regPCIEMSIX_VECT122_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT122_ADDR_HI                                                                     0x1a1e9
+#define regPCIEMSIX_VECT122_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT122_MSG_DATA                                                                    0x1a1ea
+#define regPCIEMSIX_VECT122_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT122_CONTROL                                                                     0x1a1eb
+#define regPCIEMSIX_VECT122_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT123_ADDR_LO                                                                     0x1a1ec
+#define regPCIEMSIX_VECT123_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT123_ADDR_HI                                                                     0x1a1ed
+#define regPCIEMSIX_VECT123_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT123_MSG_DATA                                                                    0x1a1ee
+#define regPCIEMSIX_VECT123_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT123_CONTROL                                                                     0x1a1ef
+#define regPCIEMSIX_VECT123_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT124_ADDR_LO                                                                     0x1a1f0
+#define regPCIEMSIX_VECT124_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT124_ADDR_HI                                                                     0x1a1f1
+#define regPCIEMSIX_VECT124_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT124_MSG_DATA                                                                    0x1a1f2
+#define regPCIEMSIX_VECT124_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT124_CONTROL                                                                     0x1a1f3
+#define regPCIEMSIX_VECT124_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT125_ADDR_LO                                                                     0x1a1f4
+#define regPCIEMSIX_VECT125_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT125_ADDR_HI                                                                     0x1a1f5
+#define regPCIEMSIX_VECT125_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT125_MSG_DATA                                                                    0x1a1f6
+#define regPCIEMSIX_VECT125_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT125_CONTROL                                                                     0x1a1f7
+#define regPCIEMSIX_VECT125_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT126_ADDR_LO                                                                     0x1a1f8
+#define regPCIEMSIX_VECT126_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT126_ADDR_HI                                                                     0x1a1f9
+#define regPCIEMSIX_VECT126_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT126_MSG_DATA                                                                    0x1a1fa
+#define regPCIEMSIX_VECT126_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT126_CONTROL                                                                     0x1a1fb
+#define regPCIEMSIX_VECT126_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT127_ADDR_LO                                                                     0x1a1fc
+#define regPCIEMSIX_VECT127_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT127_ADDR_HI                                                                     0x1a1fd
+#define regPCIEMSIX_VECT127_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT127_MSG_DATA                                                                    0x1a1fe
+#define regPCIEMSIX_VECT127_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT127_CONTROL                                                                     0x1a1ff
+#define regPCIEMSIX_VECT127_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT128_ADDR_LO                                                                     0x1a200
+#define regPCIEMSIX_VECT128_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT128_ADDR_HI                                                                     0x1a201
+#define regPCIEMSIX_VECT128_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT128_MSG_DATA                                                                    0x1a202
+#define regPCIEMSIX_VECT128_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT128_CONTROL                                                                     0x1a203
+#define regPCIEMSIX_VECT128_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT129_ADDR_LO                                                                     0x1a204
+#define regPCIEMSIX_VECT129_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT129_ADDR_HI                                                                     0x1a205
+#define regPCIEMSIX_VECT129_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT129_MSG_DATA                                                                    0x1a206
+#define regPCIEMSIX_VECT129_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT129_CONTROL                                                                     0x1a207
+#define regPCIEMSIX_VECT129_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT130_ADDR_LO                                                                     0x1a208
+#define regPCIEMSIX_VECT130_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT130_ADDR_HI                                                                     0x1a209
+#define regPCIEMSIX_VECT130_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT130_MSG_DATA                                                                    0x1a20a
+#define regPCIEMSIX_VECT130_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT130_CONTROL                                                                     0x1a20b
+#define regPCIEMSIX_VECT130_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT131_ADDR_LO                                                                     0x1a20c
+#define regPCIEMSIX_VECT131_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT131_ADDR_HI                                                                     0x1a20d
+#define regPCIEMSIX_VECT131_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT131_MSG_DATA                                                                    0x1a20e
+#define regPCIEMSIX_VECT131_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT131_CONTROL                                                                     0x1a20f
+#define regPCIEMSIX_VECT131_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT132_ADDR_LO                                                                     0x1a210
+#define regPCIEMSIX_VECT132_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT132_ADDR_HI                                                                     0x1a211
+#define regPCIEMSIX_VECT132_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT132_MSG_DATA                                                                    0x1a212
+#define regPCIEMSIX_VECT132_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT132_CONTROL                                                                     0x1a213
+#define regPCIEMSIX_VECT132_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT133_ADDR_LO                                                                     0x1a214
+#define regPCIEMSIX_VECT133_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT133_ADDR_HI                                                                     0x1a215
+#define regPCIEMSIX_VECT133_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT133_MSG_DATA                                                                    0x1a216
+#define regPCIEMSIX_VECT133_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT133_CONTROL                                                                     0x1a217
+#define regPCIEMSIX_VECT133_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT134_ADDR_LO                                                                     0x1a218
+#define regPCIEMSIX_VECT134_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT134_ADDR_HI                                                                     0x1a219
+#define regPCIEMSIX_VECT134_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT134_MSG_DATA                                                                    0x1a21a
+#define regPCIEMSIX_VECT134_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT134_CONTROL                                                                     0x1a21b
+#define regPCIEMSIX_VECT134_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT135_ADDR_LO                                                                     0x1a21c
+#define regPCIEMSIX_VECT135_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT135_ADDR_HI                                                                     0x1a21d
+#define regPCIEMSIX_VECT135_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT135_MSG_DATA                                                                    0x1a21e
+#define regPCIEMSIX_VECT135_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT135_CONTROL                                                                     0x1a21f
+#define regPCIEMSIX_VECT135_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT136_ADDR_LO                                                                     0x1a220
+#define regPCIEMSIX_VECT136_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT136_ADDR_HI                                                                     0x1a221
+#define regPCIEMSIX_VECT136_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT136_MSG_DATA                                                                    0x1a222
+#define regPCIEMSIX_VECT136_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT136_CONTROL                                                                     0x1a223
+#define regPCIEMSIX_VECT136_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT137_ADDR_LO                                                                     0x1a224
+#define regPCIEMSIX_VECT137_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT137_ADDR_HI                                                                     0x1a225
+#define regPCIEMSIX_VECT137_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT137_MSG_DATA                                                                    0x1a226
+#define regPCIEMSIX_VECT137_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT137_CONTROL                                                                     0x1a227
+#define regPCIEMSIX_VECT137_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT138_ADDR_LO                                                                     0x1a228
+#define regPCIEMSIX_VECT138_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT138_ADDR_HI                                                                     0x1a229
+#define regPCIEMSIX_VECT138_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT138_MSG_DATA                                                                    0x1a22a
+#define regPCIEMSIX_VECT138_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT138_CONTROL                                                                     0x1a22b
+#define regPCIEMSIX_VECT138_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT139_ADDR_LO                                                                     0x1a22c
+#define regPCIEMSIX_VECT139_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT139_ADDR_HI                                                                     0x1a22d
+#define regPCIEMSIX_VECT139_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT139_MSG_DATA                                                                    0x1a22e
+#define regPCIEMSIX_VECT139_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT139_CONTROL                                                                     0x1a22f
+#define regPCIEMSIX_VECT139_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT140_ADDR_LO                                                                     0x1a230
+#define regPCIEMSIX_VECT140_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT140_ADDR_HI                                                                     0x1a231
+#define regPCIEMSIX_VECT140_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT140_MSG_DATA                                                                    0x1a232
+#define regPCIEMSIX_VECT140_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT140_CONTROL                                                                     0x1a233
+#define regPCIEMSIX_VECT140_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT141_ADDR_LO                                                                     0x1a234
+#define regPCIEMSIX_VECT141_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT141_ADDR_HI                                                                     0x1a235
+#define regPCIEMSIX_VECT141_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT141_MSG_DATA                                                                    0x1a236
+#define regPCIEMSIX_VECT141_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT141_CONTROL                                                                     0x1a237
+#define regPCIEMSIX_VECT141_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT142_ADDR_LO                                                                     0x1a238
+#define regPCIEMSIX_VECT142_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT142_ADDR_HI                                                                     0x1a239
+#define regPCIEMSIX_VECT142_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT142_MSG_DATA                                                                    0x1a23a
+#define regPCIEMSIX_VECT142_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT142_CONTROL                                                                     0x1a23b
+#define regPCIEMSIX_VECT142_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT143_ADDR_LO                                                                     0x1a23c
+#define regPCIEMSIX_VECT143_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT143_ADDR_HI                                                                     0x1a23d
+#define regPCIEMSIX_VECT143_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT143_MSG_DATA                                                                    0x1a23e
+#define regPCIEMSIX_VECT143_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT143_CONTROL                                                                     0x1a23f
+#define regPCIEMSIX_VECT143_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT144_ADDR_LO                                                                     0x1a240
+#define regPCIEMSIX_VECT144_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT144_ADDR_HI                                                                     0x1a241
+#define regPCIEMSIX_VECT144_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT144_MSG_DATA                                                                    0x1a242
+#define regPCIEMSIX_VECT144_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT144_CONTROL                                                                     0x1a243
+#define regPCIEMSIX_VECT144_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT145_ADDR_LO                                                                     0x1a244
+#define regPCIEMSIX_VECT145_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT145_ADDR_HI                                                                     0x1a245
+#define regPCIEMSIX_VECT145_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT145_MSG_DATA                                                                    0x1a246
+#define regPCIEMSIX_VECT145_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT145_CONTROL                                                                     0x1a247
+#define regPCIEMSIX_VECT145_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT146_ADDR_LO                                                                     0x1a248
+#define regPCIEMSIX_VECT146_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT146_ADDR_HI                                                                     0x1a249
+#define regPCIEMSIX_VECT146_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT146_MSG_DATA                                                                    0x1a24a
+#define regPCIEMSIX_VECT146_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT146_CONTROL                                                                     0x1a24b
+#define regPCIEMSIX_VECT146_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT147_ADDR_LO                                                                     0x1a24c
+#define regPCIEMSIX_VECT147_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT147_ADDR_HI                                                                     0x1a24d
+#define regPCIEMSIX_VECT147_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT147_MSG_DATA                                                                    0x1a24e
+#define regPCIEMSIX_VECT147_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT147_CONTROL                                                                     0x1a24f
+#define regPCIEMSIX_VECT147_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT148_ADDR_LO                                                                     0x1a250
+#define regPCIEMSIX_VECT148_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT148_ADDR_HI                                                                     0x1a251
+#define regPCIEMSIX_VECT148_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT148_MSG_DATA                                                                    0x1a252
+#define regPCIEMSIX_VECT148_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT148_CONTROL                                                                     0x1a253
+#define regPCIEMSIX_VECT148_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT149_ADDR_LO                                                                     0x1a254
+#define regPCIEMSIX_VECT149_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT149_ADDR_HI                                                                     0x1a255
+#define regPCIEMSIX_VECT149_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT149_MSG_DATA                                                                    0x1a256
+#define regPCIEMSIX_VECT149_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT149_CONTROL                                                                     0x1a257
+#define regPCIEMSIX_VECT149_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT150_ADDR_LO                                                                     0x1a258
+#define regPCIEMSIX_VECT150_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT150_ADDR_HI                                                                     0x1a259
+#define regPCIEMSIX_VECT150_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT150_MSG_DATA                                                                    0x1a25a
+#define regPCIEMSIX_VECT150_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT150_CONTROL                                                                     0x1a25b
+#define regPCIEMSIX_VECT150_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT151_ADDR_LO                                                                     0x1a25c
+#define regPCIEMSIX_VECT151_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT151_ADDR_HI                                                                     0x1a25d
+#define regPCIEMSIX_VECT151_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT151_MSG_DATA                                                                    0x1a25e
+#define regPCIEMSIX_VECT151_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT151_CONTROL                                                                     0x1a25f
+#define regPCIEMSIX_VECT151_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT152_ADDR_LO                                                                     0x1a260
+#define regPCIEMSIX_VECT152_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT152_ADDR_HI                                                                     0x1a261
+#define regPCIEMSIX_VECT152_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT152_MSG_DATA                                                                    0x1a262
+#define regPCIEMSIX_VECT152_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT152_CONTROL                                                                     0x1a263
+#define regPCIEMSIX_VECT152_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT153_ADDR_LO                                                                     0x1a264
+#define regPCIEMSIX_VECT153_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT153_ADDR_HI                                                                     0x1a265
+#define regPCIEMSIX_VECT153_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT153_MSG_DATA                                                                    0x1a266
+#define regPCIEMSIX_VECT153_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT153_CONTROL                                                                     0x1a267
+#define regPCIEMSIX_VECT153_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT154_ADDR_LO                                                                     0x1a268
+#define regPCIEMSIX_VECT154_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT154_ADDR_HI                                                                     0x1a269
+#define regPCIEMSIX_VECT154_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT154_MSG_DATA                                                                    0x1a26a
+#define regPCIEMSIX_VECT154_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT154_CONTROL                                                                     0x1a26b
+#define regPCIEMSIX_VECT154_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT155_ADDR_LO                                                                     0x1a26c
+#define regPCIEMSIX_VECT155_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT155_ADDR_HI                                                                     0x1a26d
+#define regPCIEMSIX_VECT155_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT155_MSG_DATA                                                                    0x1a26e
+#define regPCIEMSIX_VECT155_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT155_CONTROL                                                                     0x1a26f
+#define regPCIEMSIX_VECT155_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT156_ADDR_LO                                                                     0x1a270
+#define regPCIEMSIX_VECT156_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT156_ADDR_HI                                                                     0x1a271
+#define regPCIEMSIX_VECT156_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT156_MSG_DATA                                                                    0x1a272
+#define regPCIEMSIX_VECT156_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT156_CONTROL                                                                     0x1a273
+#define regPCIEMSIX_VECT156_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT157_ADDR_LO                                                                     0x1a274
+#define regPCIEMSIX_VECT157_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT157_ADDR_HI                                                                     0x1a275
+#define regPCIEMSIX_VECT157_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT157_MSG_DATA                                                                    0x1a276
+#define regPCIEMSIX_VECT157_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT157_CONTROL                                                                     0x1a277
+#define regPCIEMSIX_VECT157_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT158_ADDR_LO                                                                     0x1a278
+#define regPCIEMSIX_VECT158_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT158_ADDR_HI                                                                     0x1a279
+#define regPCIEMSIX_VECT158_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT158_MSG_DATA                                                                    0x1a27a
+#define regPCIEMSIX_VECT158_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT158_CONTROL                                                                     0x1a27b
+#define regPCIEMSIX_VECT158_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT159_ADDR_LO                                                                     0x1a27c
+#define regPCIEMSIX_VECT159_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT159_ADDR_HI                                                                     0x1a27d
+#define regPCIEMSIX_VECT159_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT159_MSG_DATA                                                                    0x1a27e
+#define regPCIEMSIX_VECT159_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT159_CONTROL                                                                     0x1a27f
+#define regPCIEMSIX_VECT159_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT160_ADDR_LO                                                                     0x1a280
+#define regPCIEMSIX_VECT160_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT160_ADDR_HI                                                                     0x1a281
+#define regPCIEMSIX_VECT160_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT160_MSG_DATA                                                                    0x1a282
+#define regPCIEMSIX_VECT160_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT160_CONTROL                                                                     0x1a283
+#define regPCIEMSIX_VECT160_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT161_ADDR_LO                                                                     0x1a284
+#define regPCIEMSIX_VECT161_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT161_ADDR_HI                                                                     0x1a285
+#define regPCIEMSIX_VECT161_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT161_MSG_DATA                                                                    0x1a286
+#define regPCIEMSIX_VECT161_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT161_CONTROL                                                                     0x1a287
+#define regPCIEMSIX_VECT161_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT162_ADDR_LO                                                                     0x1a288
+#define regPCIEMSIX_VECT162_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT162_ADDR_HI                                                                     0x1a289
+#define regPCIEMSIX_VECT162_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT162_MSG_DATA                                                                    0x1a28a
+#define regPCIEMSIX_VECT162_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT162_CONTROL                                                                     0x1a28b
+#define regPCIEMSIX_VECT162_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT163_ADDR_LO                                                                     0x1a28c
+#define regPCIEMSIX_VECT163_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT163_ADDR_HI                                                                     0x1a28d
+#define regPCIEMSIX_VECT163_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT163_MSG_DATA                                                                    0x1a28e
+#define regPCIEMSIX_VECT163_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT163_CONTROL                                                                     0x1a28f
+#define regPCIEMSIX_VECT163_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT164_ADDR_LO                                                                     0x1a290
+#define regPCIEMSIX_VECT164_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT164_ADDR_HI                                                                     0x1a291
+#define regPCIEMSIX_VECT164_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT164_MSG_DATA                                                                    0x1a292
+#define regPCIEMSIX_VECT164_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT164_CONTROL                                                                     0x1a293
+#define regPCIEMSIX_VECT164_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT165_ADDR_LO                                                                     0x1a294
+#define regPCIEMSIX_VECT165_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT165_ADDR_HI                                                                     0x1a295
+#define regPCIEMSIX_VECT165_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT165_MSG_DATA                                                                    0x1a296
+#define regPCIEMSIX_VECT165_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT165_CONTROL                                                                     0x1a297
+#define regPCIEMSIX_VECT165_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT166_ADDR_LO                                                                     0x1a298
+#define regPCIEMSIX_VECT166_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT166_ADDR_HI                                                                     0x1a299
+#define regPCIEMSIX_VECT166_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT166_MSG_DATA                                                                    0x1a29a
+#define regPCIEMSIX_VECT166_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT166_CONTROL                                                                     0x1a29b
+#define regPCIEMSIX_VECT166_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT167_ADDR_LO                                                                     0x1a29c
+#define regPCIEMSIX_VECT167_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT167_ADDR_HI                                                                     0x1a29d
+#define regPCIEMSIX_VECT167_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT167_MSG_DATA                                                                    0x1a29e
+#define regPCIEMSIX_VECT167_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT167_CONTROL                                                                     0x1a29f
+#define regPCIEMSIX_VECT167_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT168_ADDR_LO                                                                     0x1a2a0
+#define regPCIEMSIX_VECT168_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT168_ADDR_HI                                                                     0x1a2a1
+#define regPCIEMSIX_VECT168_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT168_MSG_DATA                                                                    0x1a2a2
+#define regPCIEMSIX_VECT168_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT168_CONTROL                                                                     0x1a2a3
+#define regPCIEMSIX_VECT168_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT169_ADDR_LO                                                                     0x1a2a4
+#define regPCIEMSIX_VECT169_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT169_ADDR_HI                                                                     0x1a2a5
+#define regPCIEMSIX_VECT169_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT169_MSG_DATA                                                                    0x1a2a6
+#define regPCIEMSIX_VECT169_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT169_CONTROL                                                                     0x1a2a7
+#define regPCIEMSIX_VECT169_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT170_ADDR_LO                                                                     0x1a2a8
+#define regPCIEMSIX_VECT170_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT170_ADDR_HI                                                                     0x1a2a9
+#define regPCIEMSIX_VECT170_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT170_MSG_DATA                                                                    0x1a2aa
+#define regPCIEMSIX_VECT170_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT170_CONTROL                                                                     0x1a2ab
+#define regPCIEMSIX_VECT170_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT171_ADDR_LO                                                                     0x1a2ac
+#define regPCIEMSIX_VECT171_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT171_ADDR_HI                                                                     0x1a2ad
+#define regPCIEMSIX_VECT171_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT171_MSG_DATA                                                                    0x1a2ae
+#define regPCIEMSIX_VECT171_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT171_CONTROL                                                                     0x1a2af
+#define regPCIEMSIX_VECT171_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT172_ADDR_LO                                                                     0x1a2b0
+#define regPCIEMSIX_VECT172_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT172_ADDR_HI                                                                     0x1a2b1
+#define regPCIEMSIX_VECT172_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT172_MSG_DATA                                                                    0x1a2b2
+#define regPCIEMSIX_VECT172_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT172_CONTROL                                                                     0x1a2b3
+#define regPCIEMSIX_VECT172_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT173_ADDR_LO                                                                     0x1a2b4
+#define regPCIEMSIX_VECT173_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT173_ADDR_HI                                                                     0x1a2b5
+#define regPCIEMSIX_VECT173_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT173_MSG_DATA                                                                    0x1a2b6
+#define regPCIEMSIX_VECT173_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT173_CONTROL                                                                     0x1a2b7
+#define regPCIEMSIX_VECT173_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT174_ADDR_LO                                                                     0x1a2b8
+#define regPCIEMSIX_VECT174_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT174_ADDR_HI                                                                     0x1a2b9
+#define regPCIEMSIX_VECT174_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT174_MSG_DATA                                                                    0x1a2ba
+#define regPCIEMSIX_VECT174_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT174_CONTROL                                                                     0x1a2bb
+#define regPCIEMSIX_VECT174_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT175_ADDR_LO                                                                     0x1a2bc
+#define regPCIEMSIX_VECT175_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT175_ADDR_HI                                                                     0x1a2bd
+#define regPCIEMSIX_VECT175_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT175_MSG_DATA                                                                    0x1a2be
+#define regPCIEMSIX_VECT175_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT175_CONTROL                                                                     0x1a2bf
+#define regPCIEMSIX_VECT175_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT176_ADDR_LO                                                                     0x1a2c0
+#define regPCIEMSIX_VECT176_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT176_ADDR_HI                                                                     0x1a2c1
+#define regPCIEMSIX_VECT176_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT176_MSG_DATA                                                                    0x1a2c2
+#define regPCIEMSIX_VECT176_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT176_CONTROL                                                                     0x1a2c3
+#define regPCIEMSIX_VECT176_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT177_ADDR_LO                                                                     0x1a2c4
+#define regPCIEMSIX_VECT177_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT177_ADDR_HI                                                                     0x1a2c5
+#define regPCIEMSIX_VECT177_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT177_MSG_DATA                                                                    0x1a2c6
+#define regPCIEMSIX_VECT177_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT177_CONTROL                                                                     0x1a2c7
+#define regPCIEMSIX_VECT177_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT178_ADDR_LO                                                                     0x1a2c8
+#define regPCIEMSIX_VECT178_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT178_ADDR_HI                                                                     0x1a2c9
+#define regPCIEMSIX_VECT178_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT178_MSG_DATA                                                                    0x1a2ca
+#define regPCIEMSIX_VECT178_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT178_CONTROL                                                                     0x1a2cb
+#define regPCIEMSIX_VECT178_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT179_ADDR_LO                                                                     0x1a2cc
+#define regPCIEMSIX_VECT179_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT179_ADDR_HI                                                                     0x1a2cd
+#define regPCIEMSIX_VECT179_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT179_MSG_DATA                                                                    0x1a2ce
+#define regPCIEMSIX_VECT179_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT179_CONTROL                                                                     0x1a2cf
+#define regPCIEMSIX_VECT179_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT180_ADDR_LO                                                                     0x1a2d0
+#define regPCIEMSIX_VECT180_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT180_ADDR_HI                                                                     0x1a2d1
+#define regPCIEMSIX_VECT180_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT180_MSG_DATA                                                                    0x1a2d2
+#define regPCIEMSIX_VECT180_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT180_CONTROL                                                                     0x1a2d3
+#define regPCIEMSIX_VECT180_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT181_ADDR_LO                                                                     0x1a2d4
+#define regPCIEMSIX_VECT181_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT181_ADDR_HI                                                                     0x1a2d5
+#define regPCIEMSIX_VECT181_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT181_MSG_DATA                                                                    0x1a2d6
+#define regPCIEMSIX_VECT181_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT181_CONTROL                                                                     0x1a2d7
+#define regPCIEMSIX_VECT181_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT182_ADDR_LO                                                                     0x1a2d8
+#define regPCIEMSIX_VECT182_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT182_ADDR_HI                                                                     0x1a2d9
+#define regPCIEMSIX_VECT182_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT182_MSG_DATA                                                                    0x1a2da
+#define regPCIEMSIX_VECT182_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT182_CONTROL                                                                     0x1a2db
+#define regPCIEMSIX_VECT182_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT183_ADDR_LO                                                                     0x1a2dc
+#define regPCIEMSIX_VECT183_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT183_ADDR_HI                                                                     0x1a2dd
+#define regPCIEMSIX_VECT183_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT183_MSG_DATA                                                                    0x1a2de
+#define regPCIEMSIX_VECT183_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT183_CONTROL                                                                     0x1a2df
+#define regPCIEMSIX_VECT183_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT184_ADDR_LO                                                                     0x1a2e0
+#define regPCIEMSIX_VECT184_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT184_ADDR_HI                                                                     0x1a2e1
+#define regPCIEMSIX_VECT184_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT184_MSG_DATA                                                                    0x1a2e2
+#define regPCIEMSIX_VECT184_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT184_CONTROL                                                                     0x1a2e3
+#define regPCIEMSIX_VECT184_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT185_ADDR_LO                                                                     0x1a2e4
+#define regPCIEMSIX_VECT185_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT185_ADDR_HI                                                                     0x1a2e5
+#define regPCIEMSIX_VECT185_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT185_MSG_DATA                                                                    0x1a2e6
+#define regPCIEMSIX_VECT185_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT185_CONTROL                                                                     0x1a2e7
+#define regPCIEMSIX_VECT185_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT186_ADDR_LO                                                                     0x1a2e8
+#define regPCIEMSIX_VECT186_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT186_ADDR_HI                                                                     0x1a2e9
+#define regPCIEMSIX_VECT186_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT186_MSG_DATA                                                                    0x1a2ea
+#define regPCIEMSIX_VECT186_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT186_CONTROL                                                                     0x1a2eb
+#define regPCIEMSIX_VECT186_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT187_ADDR_LO                                                                     0x1a2ec
+#define regPCIEMSIX_VECT187_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT187_ADDR_HI                                                                     0x1a2ed
+#define regPCIEMSIX_VECT187_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT187_MSG_DATA                                                                    0x1a2ee
+#define regPCIEMSIX_VECT187_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT187_CONTROL                                                                     0x1a2ef
+#define regPCIEMSIX_VECT187_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT188_ADDR_LO                                                                     0x1a2f0
+#define regPCIEMSIX_VECT188_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT188_ADDR_HI                                                                     0x1a2f1
+#define regPCIEMSIX_VECT188_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT188_MSG_DATA                                                                    0x1a2f2
+#define regPCIEMSIX_VECT188_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT188_CONTROL                                                                     0x1a2f3
+#define regPCIEMSIX_VECT188_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT189_ADDR_LO                                                                     0x1a2f4
+#define regPCIEMSIX_VECT189_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT189_ADDR_HI                                                                     0x1a2f5
+#define regPCIEMSIX_VECT189_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT189_MSG_DATA                                                                    0x1a2f6
+#define regPCIEMSIX_VECT189_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT189_CONTROL                                                                     0x1a2f7
+#define regPCIEMSIX_VECT189_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT190_ADDR_LO                                                                     0x1a2f8
+#define regPCIEMSIX_VECT190_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT190_ADDR_HI                                                                     0x1a2f9
+#define regPCIEMSIX_VECT190_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT190_MSG_DATA                                                                    0x1a2fa
+#define regPCIEMSIX_VECT190_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT190_CONTROL                                                                     0x1a2fb
+#define regPCIEMSIX_VECT190_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT191_ADDR_LO                                                                     0x1a2fc
+#define regPCIEMSIX_VECT191_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT191_ADDR_HI                                                                     0x1a2fd
+#define regPCIEMSIX_VECT191_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT191_MSG_DATA                                                                    0x1a2fe
+#define regPCIEMSIX_VECT191_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT191_CONTROL                                                                     0x1a2ff
+#define regPCIEMSIX_VECT191_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT192_ADDR_LO                                                                     0x1a300
+#define regPCIEMSIX_VECT192_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT192_ADDR_HI                                                                     0x1a301
+#define regPCIEMSIX_VECT192_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT192_MSG_DATA                                                                    0x1a302
+#define regPCIEMSIX_VECT192_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT192_CONTROL                                                                     0x1a303
+#define regPCIEMSIX_VECT192_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT193_ADDR_LO                                                                     0x1a304
+#define regPCIEMSIX_VECT193_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT193_ADDR_HI                                                                     0x1a305
+#define regPCIEMSIX_VECT193_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT193_MSG_DATA                                                                    0x1a306
+#define regPCIEMSIX_VECT193_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT193_CONTROL                                                                     0x1a307
+#define regPCIEMSIX_VECT193_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT194_ADDR_LO                                                                     0x1a308
+#define regPCIEMSIX_VECT194_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT194_ADDR_HI                                                                     0x1a309
+#define regPCIEMSIX_VECT194_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT194_MSG_DATA                                                                    0x1a30a
+#define regPCIEMSIX_VECT194_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT194_CONTROL                                                                     0x1a30b
+#define regPCIEMSIX_VECT194_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT195_ADDR_LO                                                                     0x1a30c
+#define regPCIEMSIX_VECT195_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT195_ADDR_HI                                                                     0x1a30d
+#define regPCIEMSIX_VECT195_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT195_MSG_DATA                                                                    0x1a30e
+#define regPCIEMSIX_VECT195_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT195_CONTROL                                                                     0x1a30f
+#define regPCIEMSIX_VECT195_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT196_ADDR_LO                                                                     0x1a310
+#define regPCIEMSIX_VECT196_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT196_ADDR_HI                                                                     0x1a311
+#define regPCIEMSIX_VECT196_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT196_MSG_DATA                                                                    0x1a312
+#define regPCIEMSIX_VECT196_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT196_CONTROL                                                                     0x1a313
+#define regPCIEMSIX_VECT196_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT197_ADDR_LO                                                                     0x1a314
+#define regPCIEMSIX_VECT197_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT197_ADDR_HI                                                                     0x1a315
+#define regPCIEMSIX_VECT197_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT197_MSG_DATA                                                                    0x1a316
+#define regPCIEMSIX_VECT197_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT197_CONTROL                                                                     0x1a317
+#define regPCIEMSIX_VECT197_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT198_ADDR_LO                                                                     0x1a318
+#define regPCIEMSIX_VECT198_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT198_ADDR_HI                                                                     0x1a319
+#define regPCIEMSIX_VECT198_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT198_MSG_DATA                                                                    0x1a31a
+#define regPCIEMSIX_VECT198_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT198_CONTROL                                                                     0x1a31b
+#define regPCIEMSIX_VECT198_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT199_ADDR_LO                                                                     0x1a31c
+#define regPCIEMSIX_VECT199_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT199_ADDR_HI                                                                     0x1a31d
+#define regPCIEMSIX_VECT199_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT199_MSG_DATA                                                                    0x1a31e
+#define regPCIEMSIX_VECT199_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT199_CONTROL                                                                     0x1a31f
+#define regPCIEMSIX_VECT199_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT200_ADDR_LO                                                                     0x1a320
+#define regPCIEMSIX_VECT200_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT200_ADDR_HI                                                                     0x1a321
+#define regPCIEMSIX_VECT200_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT200_MSG_DATA                                                                    0x1a322
+#define regPCIEMSIX_VECT200_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT200_CONTROL                                                                     0x1a323
+#define regPCIEMSIX_VECT200_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT201_ADDR_LO                                                                     0x1a324
+#define regPCIEMSIX_VECT201_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT201_ADDR_HI                                                                     0x1a325
+#define regPCIEMSIX_VECT201_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT201_MSG_DATA                                                                    0x1a326
+#define regPCIEMSIX_VECT201_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT201_CONTROL                                                                     0x1a327
+#define regPCIEMSIX_VECT201_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT202_ADDR_LO                                                                     0x1a328
+#define regPCIEMSIX_VECT202_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT202_ADDR_HI                                                                     0x1a329
+#define regPCIEMSIX_VECT202_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT202_MSG_DATA                                                                    0x1a32a
+#define regPCIEMSIX_VECT202_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT202_CONTROL                                                                     0x1a32b
+#define regPCIEMSIX_VECT202_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT203_ADDR_LO                                                                     0x1a32c
+#define regPCIEMSIX_VECT203_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT203_ADDR_HI                                                                     0x1a32d
+#define regPCIEMSIX_VECT203_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT203_MSG_DATA                                                                    0x1a32e
+#define regPCIEMSIX_VECT203_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT203_CONTROL                                                                     0x1a32f
+#define regPCIEMSIX_VECT203_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT204_ADDR_LO                                                                     0x1a330
+#define regPCIEMSIX_VECT204_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT204_ADDR_HI                                                                     0x1a331
+#define regPCIEMSIX_VECT204_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT204_MSG_DATA                                                                    0x1a332
+#define regPCIEMSIX_VECT204_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT204_CONTROL                                                                     0x1a333
+#define regPCIEMSIX_VECT204_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT205_ADDR_LO                                                                     0x1a334
+#define regPCIEMSIX_VECT205_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT205_ADDR_HI                                                                     0x1a335
+#define regPCIEMSIX_VECT205_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT205_MSG_DATA                                                                    0x1a336
+#define regPCIEMSIX_VECT205_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT205_CONTROL                                                                     0x1a337
+#define regPCIEMSIX_VECT205_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT206_ADDR_LO                                                                     0x1a338
+#define regPCIEMSIX_VECT206_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT206_ADDR_HI                                                                     0x1a339
+#define regPCIEMSIX_VECT206_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT206_MSG_DATA                                                                    0x1a33a
+#define regPCIEMSIX_VECT206_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT206_CONTROL                                                                     0x1a33b
+#define regPCIEMSIX_VECT206_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT207_ADDR_LO                                                                     0x1a33c
+#define regPCIEMSIX_VECT207_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT207_ADDR_HI                                                                     0x1a33d
+#define regPCIEMSIX_VECT207_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT207_MSG_DATA                                                                    0x1a33e
+#define regPCIEMSIX_VECT207_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT207_CONTROL                                                                     0x1a33f
+#define regPCIEMSIX_VECT207_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT208_ADDR_LO                                                                     0x1a340
+#define regPCIEMSIX_VECT208_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT208_ADDR_HI                                                                     0x1a341
+#define regPCIEMSIX_VECT208_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT208_MSG_DATA                                                                    0x1a342
+#define regPCIEMSIX_VECT208_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT208_CONTROL                                                                     0x1a343
+#define regPCIEMSIX_VECT208_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT209_ADDR_LO                                                                     0x1a344
+#define regPCIEMSIX_VECT209_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT209_ADDR_HI                                                                     0x1a345
+#define regPCIEMSIX_VECT209_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT209_MSG_DATA                                                                    0x1a346
+#define regPCIEMSIX_VECT209_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT209_CONTROL                                                                     0x1a347
+#define regPCIEMSIX_VECT209_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT210_ADDR_LO                                                                     0x1a348
+#define regPCIEMSIX_VECT210_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT210_ADDR_HI                                                                     0x1a349
+#define regPCIEMSIX_VECT210_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT210_MSG_DATA                                                                    0x1a34a
+#define regPCIEMSIX_VECT210_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT210_CONTROL                                                                     0x1a34b
+#define regPCIEMSIX_VECT210_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT211_ADDR_LO                                                                     0x1a34c
+#define regPCIEMSIX_VECT211_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT211_ADDR_HI                                                                     0x1a34d
+#define regPCIEMSIX_VECT211_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT211_MSG_DATA                                                                    0x1a34e
+#define regPCIEMSIX_VECT211_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT211_CONTROL                                                                     0x1a34f
+#define regPCIEMSIX_VECT211_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT212_ADDR_LO                                                                     0x1a350
+#define regPCIEMSIX_VECT212_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT212_ADDR_HI                                                                     0x1a351
+#define regPCIEMSIX_VECT212_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT212_MSG_DATA                                                                    0x1a352
+#define regPCIEMSIX_VECT212_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT212_CONTROL                                                                     0x1a353
+#define regPCIEMSIX_VECT212_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT213_ADDR_LO                                                                     0x1a354
+#define regPCIEMSIX_VECT213_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT213_ADDR_HI                                                                     0x1a355
+#define regPCIEMSIX_VECT213_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT213_MSG_DATA                                                                    0x1a356
+#define regPCIEMSIX_VECT213_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT213_CONTROL                                                                     0x1a357
+#define regPCIEMSIX_VECT213_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT214_ADDR_LO                                                                     0x1a358
+#define regPCIEMSIX_VECT214_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT214_ADDR_HI                                                                     0x1a359
+#define regPCIEMSIX_VECT214_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT214_MSG_DATA                                                                    0x1a35a
+#define regPCIEMSIX_VECT214_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT214_CONTROL                                                                     0x1a35b
+#define regPCIEMSIX_VECT214_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT215_ADDR_LO                                                                     0x1a35c
+#define regPCIEMSIX_VECT215_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT215_ADDR_HI                                                                     0x1a35d
+#define regPCIEMSIX_VECT215_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT215_MSG_DATA                                                                    0x1a35e
+#define regPCIEMSIX_VECT215_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT215_CONTROL                                                                     0x1a35f
+#define regPCIEMSIX_VECT215_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT216_ADDR_LO                                                                     0x1a360
+#define regPCIEMSIX_VECT216_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT216_ADDR_HI                                                                     0x1a361
+#define regPCIEMSIX_VECT216_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT216_MSG_DATA                                                                    0x1a362
+#define regPCIEMSIX_VECT216_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT216_CONTROL                                                                     0x1a363
+#define regPCIEMSIX_VECT216_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT217_ADDR_LO                                                                     0x1a364
+#define regPCIEMSIX_VECT217_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT217_ADDR_HI                                                                     0x1a365
+#define regPCIEMSIX_VECT217_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT217_MSG_DATA                                                                    0x1a366
+#define regPCIEMSIX_VECT217_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT217_CONTROL                                                                     0x1a367
+#define regPCIEMSIX_VECT217_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT218_ADDR_LO                                                                     0x1a368
+#define regPCIEMSIX_VECT218_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT218_ADDR_HI                                                                     0x1a369
+#define regPCIEMSIX_VECT218_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT218_MSG_DATA                                                                    0x1a36a
+#define regPCIEMSIX_VECT218_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT218_CONTROL                                                                     0x1a36b
+#define regPCIEMSIX_VECT218_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT219_ADDR_LO                                                                     0x1a36c
+#define regPCIEMSIX_VECT219_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT219_ADDR_HI                                                                     0x1a36d
+#define regPCIEMSIX_VECT219_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT219_MSG_DATA                                                                    0x1a36e
+#define regPCIEMSIX_VECT219_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT219_CONTROL                                                                     0x1a36f
+#define regPCIEMSIX_VECT219_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT220_ADDR_LO                                                                     0x1a370
+#define regPCIEMSIX_VECT220_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT220_ADDR_HI                                                                     0x1a371
+#define regPCIEMSIX_VECT220_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT220_MSG_DATA                                                                    0x1a372
+#define regPCIEMSIX_VECT220_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT220_CONTROL                                                                     0x1a373
+#define regPCIEMSIX_VECT220_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT221_ADDR_LO                                                                     0x1a374
+#define regPCIEMSIX_VECT221_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT221_ADDR_HI                                                                     0x1a375
+#define regPCIEMSIX_VECT221_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT221_MSG_DATA                                                                    0x1a376
+#define regPCIEMSIX_VECT221_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT221_CONTROL                                                                     0x1a377
+#define regPCIEMSIX_VECT221_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT222_ADDR_LO                                                                     0x1a378
+#define regPCIEMSIX_VECT222_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT222_ADDR_HI                                                                     0x1a379
+#define regPCIEMSIX_VECT222_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT222_MSG_DATA                                                                    0x1a37a
+#define regPCIEMSIX_VECT222_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT222_CONTROL                                                                     0x1a37b
+#define regPCIEMSIX_VECT222_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT223_ADDR_LO                                                                     0x1a37c
+#define regPCIEMSIX_VECT223_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT223_ADDR_HI                                                                     0x1a37d
+#define regPCIEMSIX_VECT223_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT223_MSG_DATA                                                                    0x1a37e
+#define regPCIEMSIX_VECT223_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT223_CONTROL                                                                     0x1a37f
+#define regPCIEMSIX_VECT223_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT224_ADDR_LO                                                                     0x1a380
+#define regPCIEMSIX_VECT224_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT224_ADDR_HI                                                                     0x1a381
+#define regPCIEMSIX_VECT224_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT224_MSG_DATA                                                                    0x1a382
+#define regPCIEMSIX_VECT224_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT224_CONTROL                                                                     0x1a383
+#define regPCIEMSIX_VECT224_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT225_ADDR_LO                                                                     0x1a384
+#define regPCIEMSIX_VECT225_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT225_ADDR_HI                                                                     0x1a385
+#define regPCIEMSIX_VECT225_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT225_MSG_DATA                                                                    0x1a386
+#define regPCIEMSIX_VECT225_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT225_CONTROL                                                                     0x1a387
+#define regPCIEMSIX_VECT225_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT226_ADDR_LO                                                                     0x1a388
+#define regPCIEMSIX_VECT226_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT226_ADDR_HI                                                                     0x1a389
+#define regPCIEMSIX_VECT226_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT226_MSG_DATA                                                                    0x1a38a
+#define regPCIEMSIX_VECT226_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT226_CONTROL                                                                     0x1a38b
+#define regPCIEMSIX_VECT226_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT227_ADDR_LO                                                                     0x1a38c
+#define regPCIEMSIX_VECT227_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT227_ADDR_HI                                                                     0x1a38d
+#define regPCIEMSIX_VECT227_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT227_MSG_DATA                                                                    0x1a38e
+#define regPCIEMSIX_VECT227_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT227_CONTROL                                                                     0x1a38f
+#define regPCIEMSIX_VECT227_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT228_ADDR_LO                                                                     0x1a390
+#define regPCIEMSIX_VECT228_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT228_ADDR_HI                                                                     0x1a391
+#define regPCIEMSIX_VECT228_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT228_MSG_DATA                                                                    0x1a392
+#define regPCIEMSIX_VECT228_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT228_CONTROL                                                                     0x1a393
+#define regPCIEMSIX_VECT228_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT229_ADDR_LO                                                                     0x1a394
+#define regPCIEMSIX_VECT229_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT229_ADDR_HI                                                                     0x1a395
+#define regPCIEMSIX_VECT229_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT229_MSG_DATA                                                                    0x1a396
+#define regPCIEMSIX_VECT229_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT229_CONTROL                                                                     0x1a397
+#define regPCIEMSIX_VECT229_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT230_ADDR_LO                                                                     0x1a398
+#define regPCIEMSIX_VECT230_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT230_ADDR_HI                                                                     0x1a399
+#define regPCIEMSIX_VECT230_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT230_MSG_DATA                                                                    0x1a39a
+#define regPCIEMSIX_VECT230_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT230_CONTROL                                                                     0x1a39b
+#define regPCIEMSIX_VECT230_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT231_ADDR_LO                                                                     0x1a39c
+#define regPCIEMSIX_VECT231_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT231_ADDR_HI                                                                     0x1a39d
+#define regPCIEMSIX_VECT231_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT231_MSG_DATA                                                                    0x1a39e
+#define regPCIEMSIX_VECT231_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT231_CONTROL                                                                     0x1a39f
+#define regPCIEMSIX_VECT231_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT232_ADDR_LO                                                                     0x1a3a0
+#define regPCIEMSIX_VECT232_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT232_ADDR_HI                                                                     0x1a3a1
+#define regPCIEMSIX_VECT232_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT232_MSG_DATA                                                                    0x1a3a2
+#define regPCIEMSIX_VECT232_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT232_CONTROL                                                                     0x1a3a3
+#define regPCIEMSIX_VECT232_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT233_ADDR_LO                                                                     0x1a3a4
+#define regPCIEMSIX_VECT233_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT233_ADDR_HI                                                                     0x1a3a5
+#define regPCIEMSIX_VECT233_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT233_MSG_DATA                                                                    0x1a3a6
+#define regPCIEMSIX_VECT233_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT233_CONTROL                                                                     0x1a3a7
+#define regPCIEMSIX_VECT233_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT234_ADDR_LO                                                                     0x1a3a8
+#define regPCIEMSIX_VECT234_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT234_ADDR_HI                                                                     0x1a3a9
+#define regPCIEMSIX_VECT234_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT234_MSG_DATA                                                                    0x1a3aa
+#define regPCIEMSIX_VECT234_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT234_CONTROL                                                                     0x1a3ab
+#define regPCIEMSIX_VECT234_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT235_ADDR_LO                                                                     0x1a3ac
+#define regPCIEMSIX_VECT235_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT235_ADDR_HI                                                                     0x1a3ad
+#define regPCIEMSIX_VECT235_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT235_MSG_DATA                                                                    0x1a3ae
+#define regPCIEMSIX_VECT235_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT235_CONTROL                                                                     0x1a3af
+#define regPCIEMSIX_VECT235_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT236_ADDR_LO                                                                     0x1a3b0
+#define regPCIEMSIX_VECT236_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT236_ADDR_HI                                                                     0x1a3b1
+#define regPCIEMSIX_VECT236_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT236_MSG_DATA                                                                    0x1a3b2
+#define regPCIEMSIX_VECT236_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT236_CONTROL                                                                     0x1a3b3
+#define regPCIEMSIX_VECT236_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT237_ADDR_LO                                                                     0x1a3b4
+#define regPCIEMSIX_VECT237_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT237_ADDR_HI                                                                     0x1a3b5
+#define regPCIEMSIX_VECT237_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT237_MSG_DATA                                                                    0x1a3b6
+#define regPCIEMSIX_VECT237_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT237_CONTROL                                                                     0x1a3b7
+#define regPCIEMSIX_VECT237_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT238_ADDR_LO                                                                     0x1a3b8
+#define regPCIEMSIX_VECT238_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT238_ADDR_HI                                                                     0x1a3b9
+#define regPCIEMSIX_VECT238_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT238_MSG_DATA                                                                    0x1a3ba
+#define regPCIEMSIX_VECT238_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT238_CONTROL                                                                     0x1a3bb
+#define regPCIEMSIX_VECT238_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT239_ADDR_LO                                                                     0x1a3bc
+#define regPCIEMSIX_VECT239_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT239_ADDR_HI                                                                     0x1a3bd
+#define regPCIEMSIX_VECT239_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT239_MSG_DATA                                                                    0x1a3be
+#define regPCIEMSIX_VECT239_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT239_CONTROL                                                                     0x1a3bf
+#define regPCIEMSIX_VECT239_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT240_ADDR_LO                                                                     0x1a3c0
+#define regPCIEMSIX_VECT240_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT240_ADDR_HI                                                                     0x1a3c1
+#define regPCIEMSIX_VECT240_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT240_MSG_DATA                                                                    0x1a3c2
+#define regPCIEMSIX_VECT240_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT240_CONTROL                                                                     0x1a3c3
+#define regPCIEMSIX_VECT240_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT241_ADDR_LO                                                                     0x1a3c4
+#define regPCIEMSIX_VECT241_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT241_ADDR_HI                                                                     0x1a3c5
+#define regPCIEMSIX_VECT241_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT241_MSG_DATA                                                                    0x1a3c6
+#define regPCIEMSIX_VECT241_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT241_CONTROL                                                                     0x1a3c7
+#define regPCIEMSIX_VECT241_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT242_ADDR_LO                                                                     0x1a3c8
+#define regPCIEMSIX_VECT242_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT242_ADDR_HI                                                                     0x1a3c9
+#define regPCIEMSIX_VECT242_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT242_MSG_DATA                                                                    0x1a3ca
+#define regPCIEMSIX_VECT242_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT242_CONTROL                                                                     0x1a3cb
+#define regPCIEMSIX_VECT242_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT243_ADDR_LO                                                                     0x1a3cc
+#define regPCIEMSIX_VECT243_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT243_ADDR_HI                                                                     0x1a3cd
+#define regPCIEMSIX_VECT243_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT243_MSG_DATA                                                                    0x1a3ce
+#define regPCIEMSIX_VECT243_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT243_CONTROL                                                                     0x1a3cf
+#define regPCIEMSIX_VECT243_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT244_ADDR_LO                                                                     0x1a3d0
+#define regPCIEMSIX_VECT244_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT244_ADDR_HI                                                                     0x1a3d1
+#define regPCIEMSIX_VECT244_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT244_MSG_DATA                                                                    0x1a3d2
+#define regPCIEMSIX_VECT244_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT244_CONTROL                                                                     0x1a3d3
+#define regPCIEMSIX_VECT244_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT245_ADDR_LO                                                                     0x1a3d4
+#define regPCIEMSIX_VECT245_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT245_ADDR_HI                                                                     0x1a3d5
+#define regPCIEMSIX_VECT245_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT245_MSG_DATA                                                                    0x1a3d6
+#define regPCIEMSIX_VECT245_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT245_CONTROL                                                                     0x1a3d7
+#define regPCIEMSIX_VECT245_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT246_ADDR_LO                                                                     0x1a3d8
+#define regPCIEMSIX_VECT246_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT246_ADDR_HI                                                                     0x1a3d9
+#define regPCIEMSIX_VECT246_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT246_MSG_DATA                                                                    0x1a3da
+#define regPCIEMSIX_VECT246_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT246_CONTROL                                                                     0x1a3db
+#define regPCIEMSIX_VECT246_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT247_ADDR_LO                                                                     0x1a3dc
+#define regPCIEMSIX_VECT247_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT247_ADDR_HI                                                                     0x1a3dd
+#define regPCIEMSIX_VECT247_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT247_MSG_DATA                                                                    0x1a3de
+#define regPCIEMSIX_VECT247_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT247_CONTROL                                                                     0x1a3df
+#define regPCIEMSIX_VECT247_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT248_ADDR_LO                                                                     0x1a3e0
+#define regPCIEMSIX_VECT248_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT248_ADDR_HI                                                                     0x1a3e1
+#define regPCIEMSIX_VECT248_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT248_MSG_DATA                                                                    0x1a3e2
+#define regPCIEMSIX_VECT248_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT248_CONTROL                                                                     0x1a3e3
+#define regPCIEMSIX_VECT248_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT249_ADDR_LO                                                                     0x1a3e4
+#define regPCIEMSIX_VECT249_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT249_ADDR_HI                                                                     0x1a3e5
+#define regPCIEMSIX_VECT249_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT249_MSG_DATA                                                                    0x1a3e6
+#define regPCIEMSIX_VECT249_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT249_CONTROL                                                                     0x1a3e7
+#define regPCIEMSIX_VECT249_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT250_ADDR_LO                                                                     0x1a3e8
+#define regPCIEMSIX_VECT250_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT250_ADDR_HI                                                                     0x1a3e9
+#define regPCIEMSIX_VECT250_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT250_MSG_DATA                                                                    0x1a3ea
+#define regPCIEMSIX_VECT250_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT250_CONTROL                                                                     0x1a3eb
+#define regPCIEMSIX_VECT250_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT251_ADDR_LO                                                                     0x1a3ec
+#define regPCIEMSIX_VECT251_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT251_ADDR_HI                                                                     0x1a3ed
+#define regPCIEMSIX_VECT251_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT251_MSG_DATA                                                                    0x1a3ee
+#define regPCIEMSIX_VECT251_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT251_CONTROL                                                                     0x1a3ef
+#define regPCIEMSIX_VECT251_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT252_ADDR_LO                                                                     0x1a3f0
+#define regPCIEMSIX_VECT252_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT252_ADDR_HI                                                                     0x1a3f1
+#define regPCIEMSIX_VECT252_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT252_MSG_DATA                                                                    0x1a3f2
+#define regPCIEMSIX_VECT252_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT252_CONTROL                                                                     0x1a3f3
+#define regPCIEMSIX_VECT252_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT253_ADDR_LO                                                                     0x1a3f4
+#define regPCIEMSIX_VECT253_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT253_ADDR_HI                                                                     0x1a3f5
+#define regPCIEMSIX_VECT253_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT253_MSG_DATA                                                                    0x1a3f6
+#define regPCIEMSIX_VECT253_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT253_CONTROL                                                                     0x1a3f7
+#define regPCIEMSIX_VECT253_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT254_ADDR_LO                                                                     0x1a3f8
+#define regPCIEMSIX_VECT254_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT254_ADDR_HI                                                                     0x1a3f9
+#define regPCIEMSIX_VECT254_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT254_MSG_DATA                                                                    0x1a3fa
+#define regPCIEMSIX_VECT254_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT254_CONTROL                                                                     0x1a3fb
+#define regPCIEMSIX_VECT254_CONTROL_BASE_IDX 8
+#define regPCIEMSIX_VECT255_ADDR_LO                                                                     0x1a3fc
+#define regPCIEMSIX_VECT255_ADDR_LO_BASE_IDX 8
+#define regPCIEMSIX_VECT255_ADDR_HI                                                                     0x1a3fd
+#define regPCIEMSIX_VECT255_ADDR_HI_BASE_IDX 8
+#define regPCIEMSIX_VECT255_MSG_DATA                                                                    0x1a3fe
+#define regPCIEMSIX_VECT255_MSG_DATA_BASE_IDX 8
+#define regPCIEMSIX_VECT255_CONTROL                                                                     0x1a3ff
+#define regPCIEMSIX_VECT255_CONTROL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_pciemsix_0_usb_MSIXPDEC
+// base address: 0x10169000
+#define regPCIEMSIX_PBA_0                                                                               0x1a400
+#define regPCIEMSIX_PBA_0_BASE_IDX 8
+#define regPCIEMSIX_PBA_1                                                                               0x1a401
+#define regPCIEMSIX_PBA_1_BASE_IDX 8
+#define regPCIEMSIX_PBA_2                                                                               0x1a402
+#define regPCIEMSIX_PBA_2_BASE_IDX 8
+#define regPCIEMSIX_PBA_3                                                                               0x1a403
+#define regPCIEMSIX_PBA_3_BASE_IDX 8
+#define regPCIEMSIX_PBA_4                                                                               0x1a404
+#define regPCIEMSIX_PBA_4_BASE_IDX 8
+#define regPCIEMSIX_PBA_5                                                                               0x1a405
+#define regPCIEMSIX_PBA_5_BASE_IDX 8
+#define regPCIEMSIX_PBA_6                                                                               0x1a406
+#define regPCIEMSIX_PBA_6_BASE_IDX 8
+#define regPCIEMSIX_PBA_7                                                                               0x1a407
+#define regPCIEMSIX_PBA_7_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_bif_swus_SUMDEC
+// base address: 0x1013b000
+#define regSUM_INDEX                                                                                    0xec38
+#define regSUM_INDEX_BASE_IDX 8
+#define regSUM_DATA                                                                                     0xec39
+#define regSUM_DATA_BASE_IDX 8
+#define regSUM_INDEX_HI                                                                                 0xec3b
+#define regSUM_INDEX_HI_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_strap_rcc_strap_internal
+// base address: 0x10100000
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP0                                                              0xc400
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP0_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP1                                                              0xc401
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP1_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP2                                                              0xc402
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP2_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP3                                                              0xc403
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP3_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP4                                                              0xc404
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP4_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP5                                                              0xc405
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP5_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP6                                                              0xc406
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP6_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP7                                                              0xc407
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP7_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP8                                                              0xc408
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP8_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP9                                                              0xc409
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP9_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP10                                                             0xc40a
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP10_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP11                                                             0xc40b
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP11_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP12                                                             0xc40c
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP12_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP13                                                             0xc40d
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP13_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP14                                                             0xc40e
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP14_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP0                                                                         0xc480
+#define regRCC_DEV1_PORT_STRAP0_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP1                                                                         0xc481
+#define regRCC_DEV1_PORT_STRAP1_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP2                                                                         0xc482
+#define regRCC_DEV1_PORT_STRAP2_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP3                                                                         0xc483
+#define regRCC_DEV1_PORT_STRAP3_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP4                                                                         0xc484
+#define regRCC_DEV1_PORT_STRAP4_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP5                                                                         0xc485
+#define regRCC_DEV1_PORT_STRAP5_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP6                                                                         0xc486
+#define regRCC_DEV1_PORT_STRAP6_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP7                                                                         0xc487
+#define regRCC_DEV1_PORT_STRAP7_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP8                                                                         0xc488
+#define regRCC_DEV1_PORT_STRAP8_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP9                                                                         0xc489
+#define regRCC_DEV1_PORT_STRAP9_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP10                                                                        0xc48a
+#define regRCC_DEV1_PORT_STRAP10_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP11                                                                        0xc48b
+#define regRCC_DEV1_PORT_STRAP11_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP12                                                                        0xc48c
+#define regRCC_DEV1_PORT_STRAP12_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP13                                                                        0xc48d
+#define regRCC_DEV1_PORT_STRAP13_BASE_IDX 8
+#define regRCC_DEV1_PORT_STRAP14                                                                        0xc48e
+#define regRCC_DEV1_PORT_STRAP14_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP0                                                                         0xc500
+#define regRCC_DEV2_PORT_STRAP0_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP1                                                                         0xc501
+#define regRCC_DEV2_PORT_STRAP1_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP2                                                                         0xc502
+#define regRCC_DEV2_PORT_STRAP2_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP3                                                                         0xc503
+#define regRCC_DEV2_PORT_STRAP3_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP4                                                                         0xc504
+#define regRCC_DEV2_PORT_STRAP4_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP5                                                                         0xc505
+#define regRCC_DEV2_PORT_STRAP5_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP6                                                                         0xc506
+#define regRCC_DEV2_PORT_STRAP6_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP7                                                                         0xc507
+#define regRCC_DEV2_PORT_STRAP7_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP8                                                                         0xc508
+#define regRCC_DEV2_PORT_STRAP8_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP9                                                                         0xc509
+#define regRCC_DEV2_PORT_STRAP9_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP10                                                                        0xc50a
+#define regRCC_DEV2_PORT_STRAP10_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP11                                                                        0xc50b
+#define regRCC_DEV2_PORT_STRAP11_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP12                                                                        0xc50c
+#define regRCC_DEV2_PORT_STRAP12_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP13                                                                        0xc50d
+#define regRCC_DEV2_PORT_STRAP13_BASE_IDX 8
+#define regRCC_DEV2_PORT_STRAP14                                                                        0xc50e
+#define regRCC_DEV2_PORT_STRAP14_BASE_IDX 8
+#define regRCC_STRAP1_RCC_BIF_STRAP0                                                                    0xc600
+#define regRCC_STRAP1_RCC_BIF_STRAP0_BASE_IDX 8
+#define regRCC_STRAP1_RCC_BIF_STRAP1                                                                    0xc601
+#define regRCC_STRAP1_RCC_BIF_STRAP1_BASE_IDX 8
+#define regRCC_STRAP1_RCC_BIF_STRAP2                                                                    0xc602
+#define regRCC_STRAP1_RCC_BIF_STRAP2_BASE_IDX 8
+#define regRCC_STRAP1_RCC_BIF_STRAP3                                                                    0xc603
+#define regRCC_STRAP1_RCC_BIF_STRAP3_BASE_IDX 8
+#define regRCC_STRAP1_RCC_BIF_STRAP4                                                                    0xc604
+#define regRCC_STRAP1_RCC_BIF_STRAP4_BASE_IDX 8
+#define regRCC_STRAP1_RCC_BIF_STRAP5                                                                    0xc605
+#define regRCC_STRAP1_RCC_BIF_STRAP5_BASE_IDX 8
+#define regRCC_STRAP1_RCC_BIF_STRAP6                                                                    0xc606
+#define regRCC_STRAP1_RCC_BIF_STRAP6_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP0                                                              0xd000
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP0_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP1                                                              0xd001
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP1_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP2                                                              0xd002
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP2_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP3                                                              0xd003
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP3_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP4                                                              0xd004
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP4_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP5                                                              0xd005
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP5_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP8                                                              0xd008
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP8_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP9                                                              0xd009
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP9_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP13                                                             0xd00d
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP13_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP14                                                             0xd00e
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP14_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP15                                                             0xd00f
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP15_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP16                                                             0xd010
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP16_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP17                                                             0xd011
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP17_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP18                                                             0xd012
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP18_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP26                                                             0xd01a
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP26_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP0                                                              0xd080
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP0_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP2                                                              0xd082
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP2_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP3                                                              0xd083
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP3_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP4                                                              0xd084
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP4_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP5                                                              0xd085
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP5_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP6                                                              0xd086
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP6_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP7                                                              0xd087
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP7_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP20                                                             0xd094
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP20_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP21                                                             0xd095
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP21_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP22                                                             0xd096
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP22_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP23                                                             0xd097
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP23_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP24                                                             0xd098
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP24_BASE_IDX 8
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP25                                                             0xd099
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP25_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP0                                                                         0xd100
+#define regRCC_DEV0_EPF2_STRAP0_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP2                                                                         0xd102
+#define regRCC_DEV0_EPF2_STRAP2_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP3                                                                         0xd103
+#define regRCC_DEV0_EPF2_STRAP3_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP4                                                                         0xd104
+#define regRCC_DEV0_EPF2_STRAP4_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP5                                                                         0xd105
+#define regRCC_DEV0_EPF2_STRAP5_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP6                                                                         0xd106
+#define regRCC_DEV0_EPF2_STRAP6_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP7                                                                         0xd107
+#define regRCC_DEV0_EPF2_STRAP7_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP10                                                                        0xd10a
+#define regRCC_DEV0_EPF2_STRAP10_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP11                                                                        0xd10b
+#define regRCC_DEV0_EPF2_STRAP11_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP12                                                                        0xd10c
+#define regRCC_DEV0_EPF2_STRAP12_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP13                                                                        0xd10d
+#define regRCC_DEV0_EPF2_STRAP13_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP14                                                                        0xd10e
+#define regRCC_DEV0_EPF2_STRAP14_BASE_IDX 8
+#define regRCC_DEV0_EPF2_STRAP20                                                                        0xd114
+#define regRCC_DEV0_EPF2_STRAP20_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP0                                                                         0xd180
+#define regRCC_DEV0_EPF3_STRAP0_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP2                                                                         0xd182
+#define regRCC_DEV0_EPF3_STRAP2_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP3                                                                         0xd183
+#define regRCC_DEV0_EPF3_STRAP3_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP4                                                                         0xd184
+#define regRCC_DEV0_EPF3_STRAP4_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP5                                                                         0xd185
+#define regRCC_DEV0_EPF3_STRAP5_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP6                                                                         0xd186
+#define regRCC_DEV0_EPF3_STRAP6_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP7                                                                         0xd187
+#define regRCC_DEV0_EPF3_STRAP7_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP10                                                                        0xd18a
+#define regRCC_DEV0_EPF3_STRAP10_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP11                                                                        0xd18b
+#define regRCC_DEV0_EPF3_STRAP11_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP12                                                                        0xd18c
+#define regRCC_DEV0_EPF3_STRAP12_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP13                                                                        0xd18d
+#define regRCC_DEV0_EPF3_STRAP13_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP14                                                                        0xd18e
+#define regRCC_DEV0_EPF3_STRAP14_BASE_IDX 8
+#define regRCC_DEV0_EPF3_STRAP20                                                                        0xd194
+#define regRCC_DEV0_EPF3_STRAP20_BASE_IDX 8
+#define regRCC_DEV0_EPF4_STRAP0                                                                         0xd200
+#define regRCC_DEV0_EPF4_STRAP0_BASE_IDX 8
+#define regRCC_DEV0_EPF4_STRAP2                                                                         0xd202
+#define regRCC_DEV0_EPF4_STRAP2_BASE_IDX 8
+#define regRCC_DEV0_EPF4_STRAP3                                                                         0xd203
+#define regRCC_DEV0_EPF4_STRAP3_BASE_IDX 8
+#define regRCC_DEV0_EPF4_STRAP4                                                                         0xd204
+#define regRCC_DEV0_EPF4_STRAP4_BASE_IDX 8
+#define regRCC_DEV0_EPF4_STRAP5                                                                         0xd205
+#define regRCC_DEV0_EPF4_STRAP5_BASE_IDX 8
+#define regRCC_DEV0_EPF4_STRAP6                                                                         0xd206
+#define regRCC_DEV0_EPF4_STRAP6_BASE_IDX 8
+#define regRCC_DEV0_EPF4_STRAP7                                                                         0xd207
+#define regRCC_DEV0_EPF4_STRAP7_BASE_IDX 8
+#define regRCC_DEV0_EPF4_STRAP13                                                                        0xd20d
+#define regRCC_DEV0_EPF4_STRAP13_BASE_IDX 8
+#define regRCC_DEV0_EPF4_STRAP14                                                                        0xd20e
+#define regRCC_DEV0_EPF4_STRAP14_BASE_IDX 8
+#define regRCC_DEV0_EPF5_STRAP0                                                                         0xd280
+#define regRCC_DEV0_EPF5_STRAP0_BASE_IDX 8
+#define regRCC_DEV0_EPF5_STRAP2                                                                         0xd282
+#define regRCC_DEV0_EPF5_STRAP2_BASE_IDX 8
+#define regRCC_DEV0_EPF5_STRAP3                                                                         0xd283
+#define regRCC_DEV0_EPF5_STRAP3_BASE_IDX 8
+#define regRCC_DEV0_EPF5_STRAP4                                                                         0xd284
+#define regRCC_DEV0_EPF5_STRAP4_BASE_IDX 8
+#define regRCC_DEV0_EPF5_STRAP5                                                                         0xd285
+#define regRCC_DEV0_EPF5_STRAP5_BASE_IDX 8
+#define regRCC_DEV0_EPF5_STRAP6                                                                         0xd286
+#define regRCC_DEV0_EPF5_STRAP6_BASE_IDX 8
+#define regRCC_DEV0_EPF5_STRAP7                                                                         0xd287
+#define regRCC_DEV0_EPF5_STRAP7_BASE_IDX 8
+#define regRCC_DEV0_EPF5_STRAP13                                                                        0xd28d
+#define regRCC_DEV0_EPF5_STRAP13_BASE_IDX 8
+#define regRCC_DEV0_EPF5_STRAP14                                                                        0xd28e
+#define regRCC_DEV0_EPF5_STRAP14_BASE_IDX 8
+#define regRCC_DEV0_EPF6_STRAP0                                                                         0xd300
+#define regRCC_DEV0_EPF6_STRAP0_BASE_IDX 8
+#define regRCC_DEV0_EPF6_STRAP2                                                                         0xd302
+#define regRCC_DEV0_EPF6_STRAP2_BASE_IDX 8
+#define regRCC_DEV0_EPF6_STRAP3                                                                         0xd303
+#define regRCC_DEV0_EPF6_STRAP3_BASE_IDX 8
+#define regRCC_DEV0_EPF6_STRAP4                                                                         0xd304
+#define regRCC_DEV0_EPF6_STRAP4_BASE_IDX 8
+#define regRCC_DEV0_EPF6_STRAP5                                                                         0xd305
+#define regRCC_DEV0_EPF6_STRAP5_BASE_IDX 8
+#define regRCC_DEV0_EPF6_STRAP6                                                                         0xd306
+#define regRCC_DEV0_EPF6_STRAP6_BASE_IDX 8
+#define regRCC_DEV0_EPF6_STRAP13                                                                        0xd30d
+#define regRCC_DEV0_EPF6_STRAP13_BASE_IDX 8
+#define regRCC_DEV0_EPF6_STRAP14                                                                        0xd30e
+#define regRCC_DEV0_EPF6_STRAP14_BASE_IDX 8
+#define regRCC_DEV0_EPF7_STRAP0                                                                         0xd380
+#define regRCC_DEV0_EPF7_STRAP0_BASE_IDX 8
+#define regRCC_DEV0_EPF7_STRAP2                                                                         0xd382
+#define regRCC_DEV0_EPF7_STRAP2_BASE_IDX 8
+#define regRCC_DEV0_EPF7_STRAP3                                                                         0xd383
+#define regRCC_DEV0_EPF7_STRAP3_BASE_IDX 8
+#define regRCC_DEV0_EPF7_STRAP4                                                                         0xd384
+#define regRCC_DEV0_EPF7_STRAP4_BASE_IDX 8
+#define regRCC_DEV0_EPF7_STRAP5                                                                         0xd385
+#define regRCC_DEV0_EPF7_STRAP5_BASE_IDX 8
+#define regRCC_DEV0_EPF7_STRAP6                                                                         0xd386
+#define regRCC_DEV0_EPF7_STRAP6_BASE_IDX 8
+#define regRCC_DEV0_EPF7_STRAP7                                                                         0xd387
+#define regRCC_DEV0_EPF7_STRAP7_BASE_IDX 8
+#define regRCC_DEV0_EPF7_STRAP13                                                                        0xd38d
+#define regRCC_DEV0_EPF7_STRAP13_BASE_IDX 8
+#define regRCC_DEV0_EPF7_STRAP14                                                                        0xd38e
+#define regRCC_DEV0_EPF7_STRAP14_BASE_IDX 8
+#define regRCC_DEV1_EPF0_STRAP0                                                                         0xd400
+#define regRCC_DEV1_EPF0_STRAP0_BASE_IDX 8
+#define regRCC_DEV1_EPF0_STRAP2                                                                         0xd402
+#define regRCC_DEV1_EPF0_STRAP2_BASE_IDX 8
+#define regRCC_DEV1_EPF0_STRAP3                                                                         0xd403
+#define regRCC_DEV1_EPF0_STRAP3_BASE_IDX 8
+#define regRCC_DEV1_EPF0_STRAP4                                                                         0xd404
+#define regRCC_DEV1_EPF0_STRAP4_BASE_IDX 8
+#define regRCC_DEV1_EPF0_STRAP5                                                                         0xd405
+#define regRCC_DEV1_EPF0_STRAP5_BASE_IDX 8
+#define regRCC_DEV1_EPF0_STRAP6                                                                         0xd406
+#define regRCC_DEV1_EPF0_STRAP6_BASE_IDX 8
+#define regRCC_DEV1_EPF0_STRAP7                                                                         0xd407
+#define regRCC_DEV1_EPF0_STRAP7_BASE_IDX 8
+#define regRCC_DEV1_EPF0_STRAP13                                                                        0xd40d
+#define regRCC_DEV1_EPF0_STRAP13_BASE_IDX 8
+#define regRCC_DEV1_EPF0_STRAP14                                                                        0xd40e
+#define regRCC_DEV1_EPF0_STRAP14_BASE_IDX 8
+#define regRCC_DEV1_EPF1_STRAP0                                                                         0xd480
+#define regRCC_DEV1_EPF1_STRAP0_BASE_IDX 8
+#define regRCC_DEV1_EPF1_STRAP2                                                                         0xd482
+#define regRCC_DEV1_EPF1_STRAP2_BASE_IDX 8
+#define regRCC_DEV1_EPF1_STRAP3                                                                         0xd483
+#define regRCC_DEV1_EPF1_STRAP3_BASE_IDX 8
+#define regRCC_DEV1_EPF1_STRAP4                                                                         0xd484
+#define regRCC_DEV1_EPF1_STRAP4_BASE_IDX 8
+#define regRCC_DEV1_EPF1_STRAP5                                                                         0xd485
+#define regRCC_DEV1_EPF1_STRAP5_BASE_IDX 8
+#define regRCC_DEV1_EPF1_STRAP6                                                                         0xd486
+#define regRCC_DEV1_EPF1_STRAP6_BASE_IDX 8
+#define regRCC_DEV1_EPF1_STRAP7                                                                         0xd487
+#define regRCC_DEV1_EPF1_STRAP7_BASE_IDX 8
+#define regRCC_DEV1_EPF1_STRAP13                                                                        0xd48d
+#define regRCC_DEV1_EPF1_STRAP13_BASE_IDX 8
+#define regRCC_DEV1_EPF1_STRAP14                                                                        0xd48e
+#define regRCC_DEV1_EPF1_STRAP14_BASE_IDX 8
+#define regRCC_DEV2_EPF0_STRAP0                                                                         0xd800
+#define regRCC_DEV2_EPF0_STRAP0_BASE_IDX 8
+#define regRCC_DEV2_EPF0_STRAP2                                                                         0xd802
+#define regRCC_DEV2_EPF0_STRAP2_BASE_IDX 8
+#define regRCC_DEV2_EPF0_STRAP3                                                                         0xd803
+#define regRCC_DEV2_EPF0_STRAP3_BASE_IDX 8
+#define regRCC_DEV2_EPF0_STRAP4                                                                         0xd804
+#define regRCC_DEV2_EPF0_STRAP4_BASE_IDX 8
+#define regRCC_DEV2_EPF0_STRAP5                                                                         0xd805
+#define regRCC_DEV2_EPF0_STRAP5_BASE_IDX 8
+#define regRCC_DEV2_EPF0_STRAP6                                                                         0xd806
+#define regRCC_DEV2_EPF0_STRAP6_BASE_IDX 8
+#define regRCC_DEV2_EPF0_STRAP7                                                                         0xd807
+#define regRCC_DEV2_EPF0_STRAP7_BASE_IDX 8
+#define regRCC_DEV2_EPF0_STRAP13                                                                        0xd80d
+#define regRCC_DEV2_EPF0_STRAP13_BASE_IDX 8
+#define regRCC_DEV2_EPF0_STRAP14                                                                        0xd80e
+#define regRCC_DEV2_EPF0_STRAP14_BASE_IDX 8
+#define regRCC_DEV2_EPF1_STRAP0                                                                         0xd880
+#define regRCC_DEV2_EPF1_STRAP0_BASE_IDX 8
+#define regRCC_DEV2_EPF1_STRAP2                                                                         0xd882
+#define regRCC_DEV2_EPF1_STRAP2_BASE_IDX 8
+#define regRCC_DEV2_EPF1_STRAP3                                                                         0xd883
+#define regRCC_DEV2_EPF1_STRAP3_BASE_IDX 8
+#define regRCC_DEV2_EPF1_STRAP4                                                                         0xd884
+#define regRCC_DEV2_EPF1_STRAP4_BASE_IDX 8
+#define regRCC_DEV2_EPF1_STRAP5                                                                         0xd885
+#define regRCC_DEV2_EPF1_STRAP5_BASE_IDX 8
+#define regRCC_DEV2_EPF1_STRAP6                                                                         0xd886
+#define regRCC_DEV2_EPF1_STRAP6_BASE_IDX 8
+#define regRCC_DEV2_EPF1_STRAP13                                                                        0xd88d
+#define regRCC_DEV2_EPF1_STRAP13_BASE_IDX 8
+#define regRCC_DEV2_EPF1_STRAP14                                                                        0xd88e
+#define regRCC_DEV2_EPF1_STRAP14_BASE_IDX 8
+#define regRCC_DEV2_EPF2_STRAP0                                                                         0xd900
+#define regRCC_DEV2_EPF2_STRAP0_BASE_IDX 8
+#define regRCC_DEV2_EPF2_STRAP2                                                                         0xd902
+#define regRCC_DEV2_EPF2_STRAP2_BASE_IDX 8
+#define regRCC_DEV2_EPF2_STRAP3                                                                         0xd903
+#define regRCC_DEV2_EPF2_STRAP3_BASE_IDX 8
+#define regRCC_DEV2_EPF2_STRAP4                                                                         0xd904
+#define regRCC_DEV2_EPF2_STRAP4_BASE_IDX 8
+#define regRCC_DEV2_EPF2_STRAP5                                                                         0xd905
+#define regRCC_DEV2_EPF2_STRAP5_BASE_IDX 8
+#define regRCC_DEV2_EPF2_STRAP6                                                                         0xd906
+#define regRCC_DEV2_EPF2_STRAP6_BASE_IDX 8
+#define regRCC_DEV2_EPF2_STRAP13                                                                        0xd90d
+#define regRCC_DEV2_EPF2_STRAP13_BASE_IDX 8
+#define regRCC_DEV2_EPF2_STRAP14                                                                        0xd90e
+#define regRCC_DEV2_EPF2_STRAP14_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_bif_rst_bif_rst_regblk
+// base address: 0x10100000
+#define regHARD_RST_CTRL                                                                                0xe000
+#define regHARD_RST_CTRL_BASE_IDX 8
+#define regSELF_SOFT_RST                                                                                0xe002
+#define regSELF_SOFT_RST_BASE_IDX 8
+#define regBIF_GFX_DRV_VPU_RST                                                                          0xe003
+#define regBIF_GFX_DRV_VPU_RST_BASE_IDX 8
+#define regBIF_RST_MISC_CTRL                                                                            0xe004
+#define regBIF_RST_MISC_CTRL_BASE_IDX 8
+#define regBIF_RST_MISC_CTRL2                                                                           0xe005
+#define regBIF_RST_MISC_CTRL2_BASE_IDX 8
+#define regBIF_RST_MISC_CTRL3                                                                           0xe006
+#define regBIF_RST_MISC_CTRL3_BASE_IDX 8
+#define regDEV0_PF0_FLR_RST_CTRL                                                                        0xe008
+#define regDEV0_PF0_FLR_RST_CTRL_BASE_IDX 8
+#define regDEV0_PF1_FLR_RST_CTRL                                                                        0xe009
+#define regDEV0_PF1_FLR_RST_CTRL_BASE_IDX 8
+#define regBIF_INST_RESET_INTR_STS                                                                      0xe010
+#define regBIF_INST_RESET_INTR_STS_BASE_IDX 8
+#define regBIF_PF_FLR_INTR_STS                                                                          0xe011
+#define regBIF_PF_FLR_INTR_STS_BASE_IDX 8
+#define regBIF_D3HOTD0_INTR_STS                                                                         0xe012
+#define regBIF_D3HOTD0_INTR_STS_BASE_IDX 8
+#define regBIF_POWER_INTR_STS                                                                           0xe014
+#define regBIF_POWER_INTR_STS_BASE_IDX 8
+#define regBIF_PF_DSTATE_INTR_STS                                                                       0xe015
+#define regBIF_PF_DSTATE_INTR_STS_BASE_IDX 8
+#define regSELF_SOFT_RST_2                                                                              0xe016
+#define regSELF_SOFT_RST_2_BASE_IDX 8
+#define regBIF_INST_RESET_INTR_MASK                                                                     0xe020
+#define regBIF_INST_RESET_INTR_MASK_BASE_IDX 8
+#define regBIF_PF_FLR_INTR_MASK                                                                         0xe021
+#define regBIF_PF_FLR_INTR_MASK_BASE_IDX 8
+#define regBIF_D3HOTD0_INTR_MASK                                                                        0xe022
+#define regBIF_D3HOTD0_INTR_MASK_BASE_IDX 8
+#define regBIF_POWER_INTR_MASK                                                                          0xe024
+#define regBIF_POWER_INTR_MASK_BASE_IDX 8
+#define regBIF_PF_DSTATE_INTR_MASK                                                                      0xe025
+#define regBIF_PF_DSTATE_INTR_MASK_BASE_IDX 8
+#define regBIF_PF_FLR_RST                                                                               0xe040
+#define regBIF_PF_FLR_RST_BASE_IDX 8
+#define regBIF_DEV0_PF0_DSTATE_VALUE                                                                    0xe050
+#define regBIF_DEV0_PF0_DSTATE_VALUE_BASE_IDX 8
+#define regBIF_DEV0_PF1_DSTATE_VALUE                                                                    0xe051
+#define regBIF_DEV0_PF1_DSTATE_VALUE_BASE_IDX 8
+#define regDEV0_PF0_D3HOTD0_RST_CTRL                                                                    0xe078
+#define regDEV0_PF0_D3HOTD0_RST_CTRL_BASE_IDX 8
+#define regDEV0_PF1_D3HOTD0_RST_CTRL                                                                    0xe079
+#define regDEV0_PF1_D3HOTD0_RST_CTRL_BASE_IDX 8
+#define regBIF_PORT0_DSTATE_VALUE                                                                       0xe230
+#define regBIF_PORT0_DSTATE_VALUE_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_bif_misc_bif_misc_regblk
+// base address: 0x10100000
+#define regREGS_ROM_OFFSET_CTRL                                                                         0xcc23
+#define regREGS_ROM_OFFSET_CTRL_BASE_IDX 8
+#define regNBIF_STRAP_BIOS_CNTL                                                                         0xcc81
+#define regNBIF_STRAP_BIOS_CNTL_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_0                                                                       0xcd00
+#define regDOORBELL0_CTRL_ENTRY_0_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_1                                                                       0xcd01
+#define regDOORBELL0_CTRL_ENTRY_1_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_2                                                                       0xcd02
+#define regDOORBELL0_CTRL_ENTRY_2_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_3                                                                       0xcd03
+#define regDOORBELL0_CTRL_ENTRY_3_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_4                                                                       0xcd04
+#define regDOORBELL0_CTRL_ENTRY_4_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_5                                                                       0xcd05
+#define regDOORBELL0_CTRL_ENTRY_5_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_6                                                                       0xcd06
+#define regDOORBELL0_CTRL_ENTRY_6_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_7                                                                       0xcd07
+#define regDOORBELL0_CTRL_ENTRY_7_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_8                                                                       0xcd08
+#define regDOORBELL0_CTRL_ENTRY_8_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_9                                                                       0xcd09
+#define regDOORBELL0_CTRL_ENTRY_9_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_10                                                                      0xcd0a
+#define regDOORBELL0_CTRL_ENTRY_10_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_11                                                                      0xcd0b
+#define regDOORBELL0_CTRL_ENTRY_11_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_12                                                                      0xcd0c
+#define regDOORBELL0_CTRL_ENTRY_12_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_13                                                                      0xcd0d
+#define regDOORBELL0_CTRL_ENTRY_13_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_14                                                                      0xcd0e
+#define regDOORBELL0_CTRL_ENTRY_14_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_15                                                                      0xcd0f
+#define regDOORBELL0_CTRL_ENTRY_15_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_16                                                                      0xcd10
+#define regDOORBELL0_CTRL_ENTRY_16_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_17                                                                      0xcd11
+#define regDOORBELL0_CTRL_ENTRY_17_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_18                                                                      0xcd12
+#define regDOORBELL0_CTRL_ENTRY_18_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_19                                                                      0xcd13
+#define regDOORBELL0_CTRL_ENTRY_19_BASE_IDX 8
+#define regDOORBELL0_CTRL_ENTRY_20                                                                      0xcd14
+#define regDOORBELL0_CTRL_ENTRY_20_BASE_IDX 8
+#define regAID0_VF0_BASE_ADDR                                                                           0xcd40
+#define regAID0_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID1_VF0_BASE_ADDR                                                                           0xcd41
+#define regAID1_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID2_VF0_BASE_ADDR                                                                           0xcd42
+#define regAID2_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID3_VF0_BASE_ADDR                                                                           0xcd43
+#define regAID3_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC0_VF0_BASE_ADDR                                                                      0xcd44
+#define regAID0_XCC0_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC1_VF0_BASE_ADDR                                                                      0xcd45
+#define regAID0_XCC1_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC0_VF0_BASE_ADDR                                                                      0xcd46
+#define regAID1_XCC0_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC1_VF0_BASE_ADDR                                                                      0xcd47
+#define regAID1_XCC1_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC0_VF0_BASE_ADDR                                                                      0xcd48
+#define regAID2_XCC0_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC1_VF0_BASE_ADDR                                                                      0xcd49
+#define regAID2_XCC1_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC0_VF0_BASE_ADDR                                                                      0xcd4a
+#define regAID3_XCC0_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC1_VF0_BASE_ADDR                                                                      0xcd4b
+#define regAID3_XCC1_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID0_NBIF_VF0_BASE_ADDR                                                                      0xcd4c
+#define regAID0_NBIF_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID0_ATHUB_VF0_BASE_ADDR                                                                     0xcd4d
+#define regAID0_ATHUB_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID0_IH_VF0_BASE_ADDR                                                                        0xcd4e
+#define regAID0_IH_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID0_HDP_VF0_BASE_ADDR                                                                       0xcd4f
+#define regAID0_HDP_VF0_BASE_ADDR_BASE_IDX 8
+#define regAID0_VF1_BASE_ADDR                                                                           0xcd50
+#define regAID0_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID1_VF1_BASE_ADDR                                                                           0xcd51
+#define regAID1_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID2_VF1_BASE_ADDR                                                                           0xcd52
+#define regAID2_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID3_VF1_BASE_ADDR                                                                           0xcd53
+#define regAID3_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC0_VF1_BASE_ADDR                                                                      0xcd54
+#define regAID0_XCC0_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC1_VF1_BASE_ADDR                                                                      0xcd55
+#define regAID0_XCC1_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC0_VF1_BASE_ADDR                                                                      0xcd56
+#define regAID1_XCC0_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC1_VF1_BASE_ADDR                                                                      0xcd57
+#define regAID1_XCC1_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC0_VF1_BASE_ADDR                                                                      0xcd58
+#define regAID2_XCC0_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC1_VF1_BASE_ADDR                                                                      0xcd59
+#define regAID2_XCC1_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC0_VF1_BASE_ADDR                                                                      0xcd5a
+#define regAID3_XCC0_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC1_VF1_BASE_ADDR                                                                      0xcd5b
+#define regAID3_XCC1_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID0_NBIF_VF1_BASE_ADDR                                                                      0xcd5c
+#define regAID0_NBIF_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID0_ATHUB_VF1_BASE_ADDR                                                                     0xcd5d
+#define regAID0_ATHUB_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID0_IH_VF1_BASE_ADDR                                                                        0xcd5e
+#define regAID0_IH_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID0_HDP_VF1_BASE_ADDR                                                                       0xcd5f
+#define regAID0_HDP_VF1_BASE_ADDR_BASE_IDX 8
+#define regAID0_VF2_BASE_ADDR                                                                           0xcd60
+#define regAID0_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID1_VF2_BASE_ADDR                                                                           0xcd61
+#define regAID1_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID2_VF2_BASE_ADDR                                                                           0xcd62
+#define regAID2_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID3_VF2_BASE_ADDR                                                                           0xcd63
+#define regAID3_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC0_VF2_BASE_ADDR                                                                      0xcd64
+#define regAID0_XCC0_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC1_VF2_BASE_ADDR                                                                      0xcd65
+#define regAID0_XCC1_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC0_VF2_BASE_ADDR                                                                      0xcd66
+#define regAID1_XCC0_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC1_VF2_BASE_ADDR                                                                      0xcd67
+#define regAID1_XCC1_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC0_VF2_BASE_ADDR                                                                      0xcd68
+#define regAID2_XCC0_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC1_VF2_BASE_ADDR                                                                      0xcd69
+#define regAID2_XCC1_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC0_VF2_BASE_ADDR                                                                      0xcd6a
+#define regAID3_XCC0_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC1_VF2_BASE_ADDR                                                                      0xcd6b
+#define regAID3_XCC1_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID0_NBIF_VF2_BASE_ADDR                                                                      0xcd6c
+#define regAID0_NBIF_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID0_ATHUB_VF2_BASE_ADDR                                                                     0xcd6d
+#define regAID0_ATHUB_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID0_IH_VF2_BASE_ADDR                                                                        0xcd6e
+#define regAID0_IH_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID0_HDP_VF2_BASE_ADDR                                                                       0xcd6f
+#define regAID0_HDP_VF2_BASE_ADDR_BASE_IDX 8
+#define regAID0_VF3_BASE_ADDR                                                                           0xcd70
+#define regAID0_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID1_VF3_BASE_ADDR                                                                           0xcd71
+#define regAID1_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID2_VF3_BASE_ADDR                                                                           0xcd72
+#define regAID2_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID3_VF3_BASE_ADDR                                                                           0xcd73
+#define regAID3_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC0_VF3_BASE_ADDR                                                                      0xcd74
+#define regAID0_XCC0_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC1_VF3_BASE_ADDR                                                                      0xcd75
+#define regAID0_XCC1_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC0_VF3_BASE_ADDR                                                                      0xcd76
+#define regAID1_XCC0_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC1_VF3_BASE_ADDR                                                                      0xcd77
+#define regAID1_XCC1_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC0_VF3_BASE_ADDR                                                                      0xcd78
+#define regAID2_XCC0_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC1_VF3_BASE_ADDR                                                                      0xcd79
+#define regAID2_XCC1_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC0_VF3_BASE_ADDR                                                                      0xcd7a
+#define regAID3_XCC0_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC1_VF3_BASE_ADDR                                                                      0xcd7b
+#define regAID3_XCC1_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID0_NBIF_VF3_BASE_ADDR                                                                      0xcd7c
+#define regAID0_NBIF_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID0_ATHUB_VF3_BASE_ADDR                                                                     0xcd7d
+#define regAID0_ATHUB_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID0_IH_VF3_BASE_ADDR                                                                        0xcd7e
+#define regAID0_IH_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID0_HDP_VF3_BASE_ADDR                                                                       0xcd7f
+#define regAID0_HDP_VF3_BASE_ADDR_BASE_IDX 8
+#define regAID0_VF4_BASE_ADDR                                                                           0xcd80
+#define regAID0_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID1_VF4_BASE_ADDR                                                                           0xcd81
+#define regAID1_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID2_VF4_BASE_ADDR                                                                           0xcd82
+#define regAID2_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID3_VF4_BASE_ADDR                                                                           0xcd83
+#define regAID3_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC0_VF4_BASE_ADDR                                                                      0xcd84
+#define regAID0_XCC0_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC1_VF4_BASE_ADDR                                                                      0xcd85
+#define regAID0_XCC1_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC0_VF4_BASE_ADDR                                                                      0xcd86
+#define regAID1_XCC0_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC1_VF4_BASE_ADDR                                                                      0xcd87
+#define regAID1_XCC1_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC0_VF4_BASE_ADDR                                                                      0xcd88
+#define regAID2_XCC0_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC1_VF4_BASE_ADDR                                                                      0xcd89
+#define regAID2_XCC1_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC0_VF4_BASE_ADDR                                                                      0xcd8a
+#define regAID3_XCC0_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC1_VF4_BASE_ADDR                                                                      0xcd8b
+#define regAID3_XCC1_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID0_NBIF_VF4_BASE_ADDR                                                                      0xcd8c
+#define regAID0_NBIF_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID0_ATHUB_VF4_BASE_ADDR                                                                     0xcd8d
+#define regAID0_ATHUB_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID0_IH_VF4_BASE_ADDR                                                                        0xcd8e
+#define regAID0_IH_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID0_HDP_VF4_BASE_ADDR                                                                       0xcd8f
+#define regAID0_HDP_VF4_BASE_ADDR_BASE_IDX 8
+#define regAID0_VF5_BASE_ADDR                                                                           0xcd90
+#define regAID0_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID1_VF5_BASE_ADDR                                                                           0xcd91
+#define regAID1_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID2_VF5_BASE_ADDR                                                                           0xcd92
+#define regAID2_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID3_VF5_BASE_ADDR                                                                           0xcd93
+#define regAID3_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC0_VF5_BASE_ADDR                                                                      0xcd94
+#define regAID0_XCC0_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC1_VF5_BASE_ADDR                                                                      0xcd95
+#define regAID0_XCC1_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC0_VF5_BASE_ADDR                                                                      0xcd96
+#define regAID1_XCC0_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC1_VF5_BASE_ADDR                                                                      0xcd97
+#define regAID1_XCC1_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC0_VF5_BASE_ADDR                                                                      0xcd98
+#define regAID2_XCC0_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC1_VF5_BASE_ADDR                                                                      0xcd99
+#define regAID2_XCC1_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC0_VF5_BASE_ADDR                                                                      0xcd9a
+#define regAID3_XCC0_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC1_VF5_BASE_ADDR                                                                      0xcd9b
+#define regAID3_XCC1_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID0_NBIF_VF5_BASE_ADDR                                                                      0xcd9c
+#define regAID0_NBIF_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID0_ATHUB_VF5_BASE_ADDR                                                                     0xcd9d
+#define regAID0_ATHUB_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID0_IH_VF5_BASE_ADDR                                                                        0xcd9e
+#define regAID0_IH_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID0_HDP_VF5_BASE_ADDR                                                                       0xcd9f
+#define regAID0_HDP_VF5_BASE_ADDR_BASE_IDX 8
+#define regAID0_VF6_BASE_ADDR                                                                           0xcda0
+#define regAID0_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID1_VF6_BASE_ADDR                                                                           0xcda1
+#define regAID1_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID2_VF6_BASE_ADDR                                                                           0xcda2
+#define regAID2_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID3_VF6_BASE_ADDR                                                                           0xcda3
+#define regAID3_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC0_VF6_BASE_ADDR                                                                      0xcda4
+#define regAID0_XCC0_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC1_VF6_BASE_ADDR                                                                      0xcda5
+#define regAID0_XCC1_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC0_VF6_BASE_ADDR                                                                      0xcda6
+#define regAID1_XCC0_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC1_VF6_BASE_ADDR                                                                      0xcda7
+#define regAID1_XCC1_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC0_VF6_BASE_ADDR                                                                      0xcda8
+#define regAID2_XCC0_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC1_VF6_BASE_ADDR                                                                      0xcda9
+#define regAID2_XCC1_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC0_VF6_BASE_ADDR                                                                      0xcdaa
+#define regAID3_XCC0_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC1_VF6_BASE_ADDR                                                                      0xcdab
+#define regAID3_XCC1_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID0_NBIF_VF6_BASE_ADDR                                                                      0xcdac
+#define regAID0_NBIF_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID0_ATHUB_VF6_BASE_ADDR                                                                     0xcdad
+#define regAID0_ATHUB_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID0_IH_VF6_BASE_ADDR                                                                        0xcdae
+#define regAID0_IH_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID0_HDP_VF6_BASE_ADDR                                                                       0xcdaf
+#define regAID0_HDP_VF6_BASE_ADDR_BASE_IDX 8
+#define regAID0_VF7_BASE_ADDR                                                                           0xcdb0
+#define regAID0_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID1_VF7_BASE_ADDR                                                                           0xcdb1
+#define regAID1_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID2_VF7_BASE_ADDR                                                                           0xcdb2
+#define regAID2_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID3_VF7_BASE_ADDR                                                                           0xcdb3
+#define regAID3_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC0_VF7_BASE_ADDR                                                                      0xcdb4
+#define regAID0_XCC0_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC1_VF7_BASE_ADDR                                                                      0xcdb5
+#define regAID0_XCC1_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC0_VF7_BASE_ADDR                                                                      0xcdb6
+#define regAID1_XCC0_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC1_VF7_BASE_ADDR                                                                      0xcdb7
+#define regAID1_XCC1_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC0_VF7_BASE_ADDR                                                                      0xcdb8
+#define regAID2_XCC0_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC1_VF7_BASE_ADDR                                                                      0xcdb9
+#define regAID2_XCC1_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC0_VF7_BASE_ADDR                                                                      0xcdba
+#define regAID3_XCC0_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC1_VF7_BASE_ADDR                                                                      0xcdbb
+#define regAID3_XCC1_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID0_NBIF_VF7_BASE_ADDR                                                                      0xcdbc
+#define regAID0_NBIF_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID0_ATHUB_VF7_BASE_ADDR                                                                     0xcdbd
+#define regAID0_ATHUB_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID0_IH_VF7_BASE_ADDR                                                                        0xcdbe
+#define regAID0_IH_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID0_HDP_VF7_BASE_ADDR                                                                       0xcdbf
+#define regAID0_HDP_VF7_BASE_ADDR_BASE_IDX 8
+#define regAID0_PF_BASE_ADDR                                                                            0xcdc0
+#define regAID0_PF_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC0_PF_BASE_ADDR                                                                       0xcdc1
+#define regAID0_XCC0_PF_BASE_ADDR_BASE_IDX 8
+#define regAID0_XCC1_PF_BASE_ADDR                                                                       0xcdc2
+#define regAID0_XCC1_PF_BASE_ADDR_BASE_IDX 8
+#define regAID1_PF_BASE_ADDR                                                                            0xcdc3
+#define regAID1_PF_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC0_PF_BASE_ADDR                                                                       0xcdc4
+#define regAID1_XCC0_PF_BASE_ADDR_BASE_IDX 8
+#define regAID1_XCC1_PF_BASE_ADDR                                                                       0xcdc5
+#define regAID1_XCC1_PF_BASE_ADDR_BASE_IDX 8
+#define regAID2_PF_BASE_ADDR                                                                            0xcdc6
+#define regAID2_PF_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC0_PF_BASE_ADDR                                                                       0xcdc7
+#define regAID2_XCC0_PF_BASE_ADDR_BASE_IDX 8
+#define regAID2_XCC1_PF_BASE_ADDR                                                                       0xcdc8
+#define regAID2_XCC1_PF_BASE_ADDR_BASE_IDX 8
+#define regAID3_PF_BASE_ADDR                                                                            0xcdc9
+#define regAID3_PF_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC0_PF_BASE_ADDR                                                                       0xcdca
+#define regAID3_XCC0_PF_BASE_ADDR_BASE_IDX 8
+#define regAID3_XCC1_PF_BASE_ADDR                                                                       0xcdcb
+#define regAID3_XCC1_PF_BASE_ADDR_BASE_IDX 8
+#define regNBIF_RRMT_CNTL                                                                               0xcddc
+#define regNBIF_RRMT_CNTL_BASE_IDX 8
+#define regBIFC_DOORBELL_ACCESS_EN_PF                                                                   0xcf6e
+#define regBIFC_DOORBELL_ACCESS_EN_PF_BASE_IDX 8
+#define regBIFC_DOORBELL_ACCESS_EN_VF0                                                                  0xcf6f
+#define regBIFC_DOORBELL_ACCESS_EN_VF0_BASE_IDX 8
+#define regBIFC_DOORBELL_ACCESS_EN_VF1                                                                  0xcf70
+#define regBIFC_DOORBELL_ACCESS_EN_VF1_BASE_IDX 8
+#define regBIFC_DOORBELL_ACCESS_EN_VF2                                                                  0xcf71
+#define regBIFC_DOORBELL_ACCESS_EN_VF2_BASE_IDX 8
+#define regBIFC_DOORBELL_ACCESS_EN_VF3                                                                  0xcf72
+#define regBIFC_DOORBELL_ACCESS_EN_VF3_BASE_IDX 8
+#define regBIFC_DOORBELL_ACCESS_EN_VF4                                                                  0xcf73
+#define regBIFC_DOORBELL_ACCESS_EN_VF4_BASE_IDX 8
+#define regBIFC_DOORBELL_ACCESS_EN_VF5                                                                  0xcf74
+#define regBIFC_DOORBELL_ACCESS_EN_VF5_BASE_IDX 8
+#define regBIFC_DOORBELL_ACCESS_EN_VF6                                                                  0xcf75
+#define regBIFC_DOORBELL_ACCESS_EN_VF6_BASE_IDX 8
+#define regBIFC_DOORBELL_ACCESS_EN_VF7                                                                  0xcf76
+#define regBIFC_DOORBELL_ACCESS_EN_VF7_BASE_IDX 8
+#define regMISC_SCRATCH                                                                                 0xe800
+#define regMISC_SCRATCH_BASE_IDX 8
+#define regINTR_LINE_POLARITY                                                                           0xe801
+#define regINTR_LINE_POLARITY_BASE_IDX 8
+#define regINTR_LINE_ENABLE                                                                             0xe802
+#define regINTR_LINE_ENABLE_BASE_IDX 8
+#define regOUTSTANDING_VC_ALLOC                                                                         0xe803
+#define regOUTSTANDING_VC_ALLOC_BASE_IDX 8
+#define regBIFC_MISC_CTRL0                                                                              0xe804
+#define regBIFC_MISC_CTRL0_BASE_IDX 8
+#define regBIFC_MISC_CTRL1                                                                              0xe805
+#define regBIFC_MISC_CTRL1_BASE_IDX 8
+#define regBIFC_BME_ERR_LOG_LB                                                                          0xe806
+#define regBIFC_BME_ERR_LOG_LB_BASE_IDX 8
+#define regBIFC_LC_TIMER_CTRL                                                                           0xe807
+#define regBIFC_LC_TIMER_CTRL_BASE_IDX 8
+#define regBIFC_RCCBIH_BME_ERR_LOG0                                                                     0xe808
+#define regBIFC_RCCBIH_BME_ERR_LOG0_BASE_IDX 8
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1                                                            0xe80a
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1_BASE_IDX 8
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3                                                            0xe80b
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3_BASE_IDX 8
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5                                                            0xe80c
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5_BASE_IDX 8
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7                                                            0xe80d
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7_BASE_IDX 8
+#define regBIFC_DMA_ATTR_CNTL2_DEV0                                                                     0xe81a
+#define regBIFC_DMA_ATTR_CNTL2_DEV0_BASE_IDX 8
+#define regBME_DUMMY_CNTL_0                                                                             0xe825
+#define regBME_DUMMY_CNTL_0_BASE_IDX 8
+#define regBIFC_THT_CNTL                                                                                0xe827
+#define regBIFC_THT_CNTL_BASE_IDX 8
+#define regBIFC_HSTARB_CNTL                                                                             0xe828
+#define regBIFC_HSTARB_CNTL_BASE_IDX 8
+#define regBIFC_GSI_CNTL                                                                                0xe829
+#define regBIFC_GSI_CNTL_BASE_IDX 8
+#define regBIFC_PCIEFUNC_CNTL                                                                           0xe82a
+#define regBIFC_PCIEFUNC_CNTL_BASE_IDX 8
+#define regBIFC_PASID_CHECK_DIS                                                                         0xe82b
+#define regBIFC_PASID_CHECK_DIS_BASE_IDX 8
+#define regBIFC_SDP_CNTL_0                                                                              0xe82c
+#define regBIFC_SDP_CNTL_0_BASE_IDX 8
+#define regBIFC_SDP_CNTL_1                                                                              0xe82d
+#define regBIFC_SDP_CNTL_1_BASE_IDX 8
+#define regBIFC_PASID_STS                                                                               0xe82e
+#define regBIFC_PASID_STS_BASE_IDX 8
+#define regBIFC_ATHUB_ACT_CNTL                                                                          0xe82f
+#define regBIFC_ATHUB_ACT_CNTL_BASE_IDX 8
+#define regBIFC_PERF_CNTL_0                                                                             0xe830
+#define regBIFC_PERF_CNTL_0_BASE_IDX 8
+#define regBIFC_PERF_CNTL_1                                                                             0xe831
+#define regBIFC_PERF_CNTL_1_BASE_IDX 8
+#define regBIFC_PERF_CNT_MMIO_RD_L32BIT                                                                 0xe832
+#define regBIFC_PERF_CNT_MMIO_RD_L32BIT_BASE_IDX 8
+#define regBIFC_PERF_CNT_MMIO_WR_L32BIT                                                                 0xe833
+#define regBIFC_PERF_CNT_MMIO_WR_L32BIT_BASE_IDX 8
+#define regBIFC_PERF_CNT_DMA_RD_L32BIT                                                                  0xe834
+#define regBIFC_PERF_CNT_DMA_RD_L32BIT_BASE_IDX 8
+#define regBIFC_PERF_CNT_DMA_WR_L32BIT                                                                  0xe835
+#define regBIFC_PERF_CNT_DMA_WR_L32BIT_BASE_IDX 8
+#define regNBIF_REGIF_ERRSET_CTRL                                                                       0xe836
+#define regNBIF_REGIF_ERRSET_CTRL_BASE_IDX 8
+#define regBIFC_SDP_CNTL_2                                                                              0xe837
+#define regBIFC_SDP_CNTL_2_BASE_IDX 8
+#define regNBIF_PGMST_CTRL                                                                              0xe838
+#define regNBIF_PGMST_CTRL_BASE_IDX 8
+#define regNBIF_PGSLV_CTRL                                                                              0xe839
+#define regNBIF_PGSLV_CTRL_BASE_IDX 8
+#define regNBIF_PG_MISC_CTRL                                                                            0xe83a
+#define regNBIF_PG_MISC_CTRL_BASE_IDX 8
+#define regSMN_MST_EP_CNTL3                                                                             0xe83c
+#define regSMN_MST_EP_CNTL3_BASE_IDX 8
+#define regSMN_MST_EP_CNTL4                                                                             0xe83d
+#define regSMN_MST_EP_CNTL4_BASE_IDX 8
+#define regSMN_MST_CNTL1                                                                                0xe83e
+#define regSMN_MST_CNTL1_BASE_IDX 8
+#define regSMN_MST_EP_CNTL5                                                                             0xe83f
+#define regSMN_MST_EP_CNTL5_BASE_IDX 8
+#define regBIF_SELFRING_BUFFER_VID                                                                      0xe840
+#define regBIF_SELFRING_BUFFER_VID_BASE_IDX 8
+#define regBIF_SELFRING_VECTOR_CNTL                                                                     0xe841
+#define regBIF_SELFRING_VECTOR_CNTL_BASE_IDX 8
+#define regNBIF_STRAP_WRITE_CTRL                                                                        0xe845
+#define regNBIF_STRAP_WRITE_CTRL_BASE_IDX 8
+#define regNBIF_INTX_DSTATE_MISC_CNTL                                                                   0xe846
+#define regNBIF_INTX_DSTATE_MISC_CNTL_BASE_IDX 8
+#define regNBIF_PENDING_MISC_CNTL                                                                       0xe847
+#define regNBIF_PENDING_MISC_CNTL_BASE_IDX 8
+#define regBIF_GMI_WRR_WEIGHT                                                                           0xe848
+#define regBIF_GMI_WRR_WEIGHT_BASE_IDX 8
+#define regBIF_GMI_WRR_WEIGHT2                                                                          0xe849
+#define regBIF_GMI_WRR_WEIGHT2_BASE_IDX 8
+#define regBIF_GMI_WRR_WEIGHT3                                                                          0xe84a
+#define regBIF_GMI_WRR_WEIGHT3_BASE_IDX 8
+#define regNBIF_PWRBRK_REQUEST                                                                          0xe84c
+#define regNBIF_PWRBRK_REQUEST_BASE_IDX 8
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F0                                                                   0xe850
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F0_BASE_IDX 8
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F1                                                                   0xe851
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F1_BASE_IDX 8
+#define regBIF_DMA_MP4_ERR_LOG                                                                          0xe870
+#define regBIF_DMA_MP4_ERR_LOG_BASE_IDX 8
+#define regBIF_PASID_ERR_LOG                                                                            0xe871
+#define regBIF_PASID_ERR_LOG_BASE_IDX 8
+#define regBIF_PASID_ERR_CLR                                                                            0xe872
+#define regBIF_PASID_ERR_CLR_BASE_IDX 8
+#define regNBIF_VWIRE_CTRL                                                                              0xe880
+#define regNBIF_VWIRE_CTRL_BASE_IDX 8
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL                                                                   0xe881
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL_BASE_IDX 8
+#define regNBIF_SMN_VWR_VCHG_RST_CTRL0                                                                  0xe882
+#define regNBIF_SMN_VWR_VCHG_RST_CTRL0_BASE_IDX 8
+#define regNBIF_SMN_VWR_VCHG_TRIG                                                                       0xe884
+#define regNBIF_SMN_VWR_VCHG_TRIG_BASE_IDX 8
+#define regNBIF_SMN_VWR_WTRIG_CNTL                                                                      0xe885
+#define regNBIF_SMN_VWR_WTRIG_CNTL_BASE_IDX 8
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL_1                                                                 0xe886
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL_1_BASE_IDX 8
+#define regNBIF_MGCG_CTRL_LCLK                                                                          0xe887
+#define regNBIF_MGCG_CTRL_LCLK_BASE_IDX 8
+#define regNBIF_DS_CTRL_LCLK                                                                            0xe888
+#define regNBIF_DS_CTRL_LCLK_BASE_IDX 8
+#define regSMN_MST_CNTL0                                                                                0xe889
+#define regSMN_MST_CNTL0_BASE_IDX 8
+#define regSMN_MST_EP_CNTL1                                                                             0xe88a
+#define regSMN_MST_EP_CNTL1_BASE_IDX 8
+#define regSMN_MST_EP_CNTL2                                                                             0xe88b
+#define regSMN_MST_EP_CNTL2_BASE_IDX 8
+#define regNBIF_SDP_VWR_VCHG_DIS_CTRL                                                                   0xe88c
+#define regNBIF_SDP_VWR_VCHG_DIS_CTRL_BASE_IDX 8
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL0                                                                  0xe88d
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL0_BASE_IDX 8
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL1                                                                  0xe88e
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL1_BASE_IDX 8
+#define regNBIF_SDP_VWR_VCHG_TRIG                                                                       0xe88f
+#define regNBIF_SDP_VWR_VCHG_TRIG_BASE_IDX 8
+#define regNBIF_SHUB_TODET_CTRL                                                                         0xe898
+#define regNBIF_SHUB_TODET_CTRL_BASE_IDX 8
+#define regNBIF_SHUB_TODET_CLIENT_CTRL                                                                  0xe899
+#define regNBIF_SHUB_TODET_CLIENT_CTRL_BASE_IDX 8
+#define regNBIF_SHUB_TODET_CLIENT_STATUS                                                                0xe89a
+#define regNBIF_SHUB_TODET_CLIENT_STATUS_BASE_IDX 8
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL                                                               0xe89b
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL_BASE_IDX 8
+#define regNBIF_SHUB_TODET_CLIENT_CTRL2                                                                 0xe89c
+#define regNBIF_SHUB_TODET_CLIENT_CTRL2_BASE_IDX 8
+#define regNBIF_SHUB_TODET_CLIENT_STATUS2                                                               0xe89d
+#define regNBIF_SHUB_TODET_CLIENT_STATUS2_BASE_IDX 8
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL2                                                              0xe89e
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL2_BASE_IDX 8
+#define regBIFC_BME_ERR_LOG_HB                                                                          0xe8ab
+#define regBIFC_BME_ERR_LOG_HB_BASE_IDX 8
+#define regBIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC                                                            0xe8c0
+#define regBIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC_BASE_IDX 8
+#define regBIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC                                                            0xe8c1
+#define regBIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC_BASE_IDX 8
+#define regBIFC_GMI_SDP_REQ_POOLCRED_ALLOC                                                              0xe8c2
+#define regBIFC_GMI_SDP_REQ_POOLCRED_ALLOC_BASE_IDX 8
+#define regBIFC_GMI_SDP_DAT_POOLCRED_ALLOC                                                              0xe8c3
+#define regBIFC_GMI_SDP_DAT_POOLCRED_ALLOC_BASE_IDX 8
+#define regDISCON_HYSTERESIS_HEAD_CTRL                                                                  0xe8c6
+#define regDISCON_HYSTERESIS_HEAD_CTRL_BASE_IDX 8
+#define regBIFC_EARLY_WAKEUP_CNTL                                                                       0xe8d2
+#define regBIFC_EARLY_WAKEUP_CNTL_BASE_IDX 8
+#define regBIFC_PERF_CNT_MMIO_RD_H16BIT                                                                 0xe8f0
+#define regBIFC_PERF_CNT_MMIO_RD_H16BIT_BASE_IDX 8
+#define regBIFC_PERF_CNT_MMIO_WR_H16BIT                                                                 0xe8f1
+#define regBIFC_PERF_CNT_MMIO_WR_H16BIT_BASE_IDX 8
+#define regBIFC_PERF_CNT_DMA_RD_H16BIT                                                                  0xe8f2
+#define regBIFC_PERF_CNT_DMA_RD_H16BIT_BASE_IDX 8
+#define regBIFC_PERF_CNT_DMA_WR_H16BIT                                                                  0xe8f3
+#define regBIFC_PERF_CNT_DMA_WR_H16BIT_BASE_IDX 8
+#define regBIFC_A2S_SDP_PORT_CTRL                                                                       0xeb00
+#define regBIFC_A2S_SDP_PORT_CTRL_BASE_IDX 8
+#define regBIFC_A2S_CNTL_SW0                                                                            0xeb01
+#define regBIFC_A2S_CNTL_SW0_BASE_IDX 8
+#define regBIFC_A2S_MISC_CNTL                                                                           0xeb02
+#define regBIFC_A2S_MISC_CNTL_BASE_IDX 8
+#define regBIFC_A2S_TAG_ALLOC_0                                                                         0xeb03
+#define regBIFC_A2S_TAG_ALLOC_0_BASE_IDX 8
+#define regBIFC_A2S_TAG_ALLOC_1                                                                         0xeb04
+#define regBIFC_A2S_TAG_ALLOC_1_BASE_IDX 8
+#define regBIFC_A2S_CNTL_CL0                                                                            0xeb05
+#define regBIFC_A2S_CNTL_CL0_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwn_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_DWN_DEV0_2_DN_PCIE_RESERVED                                                              0x8d80
+#define regRCC_DWN_DEV0_2_DN_PCIE_RESERVED_BASE_IDX 8
+#define regRCC_DWN_DEV0_2_DN_PCIE_SCRATCH                                                               0x8d81
+#define regRCC_DWN_DEV0_2_DN_PCIE_SCRATCH_BASE_IDX 8
+#define regRCC_DWN_DEV0_2_DN_PCIE_CNTL                                                                  0x8d83
+#define regRCC_DWN_DEV0_2_DN_PCIE_CNTL_BASE_IDX 8
+#define regRCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL                                                           0x8d84
+#define regRCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL_BASE_IDX 8
+#define regRCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2                                                              0x8d85
+#define regRCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2_BASE_IDX 8
+#define regRCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL                                                              0x8d86
+#define regRCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL_BASE_IDX 8
+#define regRCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL                                                              0x8d87
+#define regRCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL_BASE_IDX 8
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_F0                                                              0x8d88
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_F0_BASE_IDX 8
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC                                                            0x8d89
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC_BASE_IDX 8
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2                                                           0x8d8a
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwnp_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_DWNP_DEV0_2_PCIE_ERR_CNTL                                                                0x8d8c
+#define regRCC_DWNP_DEV0_2_PCIE_ERR_CNTL_BASE_IDX 8
+#define regRCC_DWNP_DEV0_2_PCIE_RX_CNTL                                                                 0x8d8d
+#define regRCC_DWNP_DEV0_2_PCIE_RX_CNTL_BASE_IDX 8
+#define regRCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL                                                           0x8d8e
+#define regRCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL_BASE_IDX 8
+#define regRCC_DWNP_DEV0_2_PCIE_LC_CNTL2                                                                0x8d8f
+#define regRCC_DWNP_DEV0_2_PCIE_LC_CNTL2_BASE_IDX 8
+#define regRCC_DWNP_DEV0_2_PCIEP_STRAP_MISC                                                             0x8d90
+#define regRCC_DWNP_DEV0_2_PCIEP_STRAP_MISC_BASE_IDX 8
+#define regRCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP                                                         0x8d91
+#define regRCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_ep_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_EP_DEV0_2_EP_PCIE_SCRATCH                                                                0x8d60
+#define regRCC_EP_DEV0_2_EP_PCIE_SCRATCH_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_CNTL                                                                   0x8d62
+#define regRCC_EP_DEV0_2_EP_PCIE_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_CNTL                                                               0x8d63
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_STATUS                                                             0x8d64
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_STATUS_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL2                                                               0x8d65
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL2_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_BUS_CNTL                                                               0x8d66
+#define regRCC_EP_DEV0_2_EP_PCIE_BUS_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_CFG_CNTL                                                               0x8d67
+#define regRCC_EP_DEV0_2_EP_PCIE_CFG_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL                                                            0x8d69
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0                                               0x8d6a
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1                                               0x8d6a
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2                                               0x8d6a
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3                                               0x8d6a
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4                                               0x8d6b
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5                                               0x8d6b
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6                                               0x8d6b
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX 8
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7                                               0x8d6b
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC                                                             0x8d6c
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2                                                            0x8d6d
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP                                                             0x8d6f
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                               0x8d70
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL                                                            0x8d70
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                               0x8d70
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX 8
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                               0x8d71
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX 8
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                               0x8d71
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX 8
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                               0x8d71
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX 8
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                               0x8d71
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX 8
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                               0x8d72
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX 8
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                               0x8d72
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX 8
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                               0x8d72
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_PME_CONTROL                                                            0x8d72
+#define regRCC_EP_DEV0_2_EP_PCIE_PME_CONTROL_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIEP_RESERVED                                                              0x8d73
+#define regRCC_EP_DEV0_2_EP_PCIEP_RESERVED_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_CNTL                                                                0x8d75
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID                                                        0x8d76
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_ERR_CNTL                                                               0x8d77
+#define regRCC_EP_DEV0_2_EP_PCIE_ERR_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL                                                                0x8d78
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL_BASE_IDX 8
+#define regRCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL                                                          0x8d79
+#define regRCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_DEV0_1_RCC_ERR_INT_CNTL                                                                  0x8da6
+#define regRCC_DEV0_1_RCC_ERR_INT_CNTL_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_BACO_CNTL_MISC                                                                0x8da7
+#define regRCC_DEV0_1_RCC_BACO_CNTL_MISC_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_RESET_EN                                                                      0x8da8
+#define regRCC_DEV0_1_RCC_RESET_EN_BASE_IDX 8
+#define regRCC_DEV0_2_RCC_VDM_SUPPORT                                                                   0x8da9
+#define regRCC_DEV0_2_RCC_VDM_SUPPORT_BASE_IDX 8
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0                                                            0x8daa
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0_BASE_IDX 8
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1                                                            0x8dab
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_GPUIOV_REGION                                                                 0x8dac
+#define regRCC_DEV0_1_RCC_GPUIOV_REGION_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_GPU_HOSTVM_EN                                                                 0x8dad
+#define regRCC_DEV0_1_RCC_GPU_HOSTVM_EN_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL                                                         0x8dae
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET                                                   0x8daf
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE                                                         0x8daf
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE0                                                               0x8dde
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE0_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE1                                                               0x8ddf
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE1_BASE_IDX 8
+#define regRCC_DEV0_2_RCC_BUS_CNTL                                                                      0x8de1
+#define regRCC_DEV0_2_RCC_BUS_CNTL_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_CONFIG_CNTL                                                                   0x8de2
+#define regRCC_DEV0_1_RCC_CONFIG_CNTL_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_CONFIG_F0_BASE                                                                0x8de6
+#define regRCC_DEV0_1_RCC_CONFIG_F0_BASE_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_CONFIG_APER_SIZE                                                              0x8de7
+#define regRCC_DEV0_1_RCC_CONFIG_APER_SIZE_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE                                                          0x8de8
+#define regRCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_XDMA_LO                                                                       0x8de9
+#define regRCC_DEV0_1_RCC_XDMA_LO_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_XDMA_HI                                                                       0x8dea
+#define regRCC_DEV0_1_RCC_XDMA_HI_BASE_IDX 8
+#define regRCC_DEV0_2_RCC_FEATURES_CONTROL_MISC                                                         0x8deb
+#define regRCC_DEV0_2_RCC_FEATURES_CONTROL_MISC_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL1                                                                  0x8dec
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL1_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST0                                                                  0x8ded
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST0_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST1                                                                  0x8dee
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST1_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL2                                                                  0x8def
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL2_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM                                                           0x8df0
+#define regRCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_HOST_BUSNUM                                                                   0x8df1
+#define regRCC_DEV0_1_RCC_HOST_BUSNUM_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI                                                            0x8df2
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO                                                            0x8df3
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI                                                            0x8df4
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO                                                            0x8df5
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI                                                            0x8df6
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO                                                            0x8df7
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI                                                            0x8df8
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO                                                            0x8df9
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST0                                                              0x8dfa
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST0_BASE_IDX 8
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST1                                                              0x8dfb
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST1_BASE_IDX 8
+#define regRCC_DEV0_2_RCC_DEV0_LINK_CNTL                                                                0x8dfd
+#define regRCC_DEV0_2_RCC_DEV0_LINK_CNTL_BASE_IDX 8
+#define regRCC_DEV0_2_RCC_CMN_LINK_CNTL                                                                 0x8dfe
+#define regRCC_DEV0_2_RCC_CMN_LINK_CNTL_BASE_IDX 8
+#define regRCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE                                                        0x8dff
+#define regRCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE_BASE_IDX 8
+#define regRCC_DEV0_2_RCC_LTR_LSWITCH_CNTL                                                              0x8e00
+#define regRCC_DEV0_2_RCC_LTR_LSWITCH_CNTL_BASE_IDX 8
+#define regRCC_DEV0_2_RCC_MH_ARB_CNTL                                                                   0x8e01
+#define regRCC_DEV0_2_RCC_MH_ARB_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_SYSDEC
+// base address: 0x10120000
+#define regBIF_BX1_PCIE_INDEX                                                                           0x800c
+#define regBIF_BX1_PCIE_INDEX_BASE_IDX 8
+#define regBIF_BX1_PCIE_DATA                                                                            0x800d
+#define regBIF_BX1_PCIE_DATA_BASE_IDX 8
+#define regBIF_BX1_PCIE_INDEX2                                                                          0x800e
+#define regBIF_BX1_PCIE_INDEX2_BASE_IDX 8
+#define regBIF_BX1_PCIE_DATA2                                                                           0x800f
+#define regBIF_BX1_PCIE_DATA2_BASE_IDX 8
+#define regBIF_BX1_PCIE_INDEX_HI                                                                        0x8010
+#define regBIF_BX1_PCIE_INDEX_HI_BASE_IDX 8
+#define regBIF_BX1_PCIE_INDEX2_HI                                                                       0x8011
+#define regBIF_BX1_PCIE_INDEX2_HI_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_0                                                                      0x8048
+#define regBIF_BX1_SBIOS_SCRATCH_0_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_1                                                                      0x8049
+#define regBIF_BX1_SBIOS_SCRATCH_1_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_2                                                                      0x804a
+#define regBIF_BX1_SBIOS_SCRATCH_2_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_3                                                                      0x804b
+#define regBIF_BX1_SBIOS_SCRATCH_3_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_0                                                                       0x804c
+#define regBIF_BX1_BIOS_SCRATCH_0_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_1                                                                       0x804d
+#define regBIF_BX1_BIOS_SCRATCH_1_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_2                                                                       0x804e
+#define regBIF_BX1_BIOS_SCRATCH_2_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_3                                                                       0x804f
+#define regBIF_BX1_BIOS_SCRATCH_3_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_4                                                                       0x8050
+#define regBIF_BX1_BIOS_SCRATCH_4_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_5                                                                       0x8051
+#define regBIF_BX1_BIOS_SCRATCH_5_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_6                                                                       0x8052
+#define regBIF_BX1_BIOS_SCRATCH_6_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_7                                                                       0x8053
+#define regBIF_BX1_BIOS_SCRATCH_7_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_8                                                                       0x8054
+#define regBIF_BX1_BIOS_SCRATCH_8_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_9                                                                       0x8055
+#define regBIF_BX1_BIOS_SCRATCH_9_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_10                                                                      0x8056
+#define regBIF_BX1_BIOS_SCRATCH_10_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_11                                                                      0x8057
+#define regBIF_BX1_BIOS_SCRATCH_11_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_12                                                                      0x8058
+#define regBIF_BX1_BIOS_SCRATCH_12_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_13                                                                      0x8059
+#define regBIF_BX1_BIOS_SCRATCH_13_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_14                                                                      0x805a
+#define regBIF_BX1_BIOS_SCRATCH_14_BASE_IDX 8
+#define regBIF_BX1_BIOS_SCRATCH_15                                                                      0x805b
+#define regBIF_BX1_BIOS_SCRATCH_15_BASE_IDX 8
+#define regBIF_BX1_BIF_RLC_INTR_CNTL                                                                    0x8060
+#define regBIF_BX1_BIF_RLC_INTR_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_VCE_INTR_CNTL                                                                    0x8061
+#define regBIF_BX1_BIF_VCE_INTR_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_UVD_INTR_CNTL                                                                    0x8062
+#define regBIF_BX1_BIF_UVD_INTR_CNTL_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR0                                                                0x8080
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR0_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0                                                          0x8081
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR1                                                                0x8082
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR1_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1                                                          0x8083
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR2                                                                0x8084
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR2_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2                                                          0x8085
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR3                                                                0x8086
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR3_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3                                                          0x8087
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR4                                                                0x8088
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR4_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4                                                          0x8089
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR5                                                                0x808a
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR5_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5                                                          0x808b
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR6                                                                0x808c
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR6_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6                                                          0x808d
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR7                                                                0x808e
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR7_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7                                                          0x808f
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_CNTL                                                                 0x8090
+#define regBIF_BX1_GFX_MMIOREG_CAM_CNTL_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL                                                             0x8091
+#define regBIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_ONE_CPL                                                              0x8092
+#define regBIF_BX1_GFX_MMIOREG_CAM_ONE_CPL_BASE_IDX 8
+#define regBIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL                                                     0x8093
+#define regBIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_0                                                                     0x8094
+#define regBIF_BX1_DRIVER_SCRATCH_0_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_1                                                                     0x8095
+#define regBIF_BX1_DRIVER_SCRATCH_1_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_2                                                                     0x8096
+#define regBIF_BX1_DRIVER_SCRATCH_2_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_3                                                                     0x8097
+#define regBIF_BX1_DRIVER_SCRATCH_3_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_4                                                                     0x8098
+#define regBIF_BX1_DRIVER_SCRATCH_4_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_5                                                                     0x8099
+#define regBIF_BX1_DRIVER_SCRATCH_5_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_6                                                                     0x809a
+#define regBIF_BX1_DRIVER_SCRATCH_6_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_7                                                                     0x809b
+#define regBIF_BX1_DRIVER_SCRATCH_7_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_8                                                                     0x809c
+#define regBIF_BX1_DRIVER_SCRATCH_8_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_9                                                                     0x809d
+#define regBIF_BX1_DRIVER_SCRATCH_9_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_10                                                                    0x809e
+#define regBIF_BX1_DRIVER_SCRATCH_10_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_11                                                                    0x809f
+#define regBIF_BX1_DRIVER_SCRATCH_11_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_12                                                                    0x80a0
+#define regBIF_BX1_DRIVER_SCRATCH_12_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_13                                                                    0x80a1
+#define regBIF_BX1_DRIVER_SCRATCH_13_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_14                                                                    0x80a2
+#define regBIF_BX1_DRIVER_SCRATCH_14_BASE_IDX 8
+#define regBIF_BX1_DRIVER_SCRATCH_15                                                                    0x80a3
+#define regBIF_BX1_DRIVER_SCRATCH_15_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_0                                                                         0x80a4
+#define regBIF_BX1_FW_SCRATCH_0_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_1                                                                         0x80a5
+#define regBIF_BX1_FW_SCRATCH_1_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_2                                                                         0x80a6
+#define regBIF_BX1_FW_SCRATCH_2_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_3                                                                         0x80a7
+#define regBIF_BX1_FW_SCRATCH_3_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_4                                                                         0x80a8
+#define regBIF_BX1_FW_SCRATCH_4_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_5                                                                         0x80a9
+#define regBIF_BX1_FW_SCRATCH_5_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_6                                                                         0x80aa
+#define regBIF_BX1_FW_SCRATCH_6_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_7                                                                         0x80ab
+#define regBIF_BX1_FW_SCRATCH_7_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_8                                                                         0x80ac
+#define regBIF_BX1_FW_SCRATCH_8_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_9                                                                         0x80ad
+#define regBIF_BX1_FW_SCRATCH_9_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_10                                                                        0x80ae
+#define regBIF_BX1_FW_SCRATCH_10_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_11                                                                        0x80af
+#define regBIF_BX1_FW_SCRATCH_11_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_12                                                                        0x80b0
+#define regBIF_BX1_FW_SCRATCH_12_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_13                                                                        0x80b1
+#define regBIF_BX1_FW_SCRATCH_13_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_14                                                                        0x80b2
+#define regBIF_BX1_FW_SCRATCH_14_BASE_IDX 8
+#define regBIF_BX1_FW_SCRATCH_15                                                                        0x80b3
+#define regBIF_BX1_FW_SCRATCH_15_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_4                                                                      0x80b4
+#define regBIF_BX1_SBIOS_SCRATCH_4_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_5                                                                      0x80b5
+#define regBIF_BX1_SBIOS_SCRATCH_5_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_6                                                                      0x80b6
+#define regBIF_BX1_SBIOS_SCRATCH_6_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_7                                                                      0x80b7
+#define regBIF_BX1_SBIOS_SCRATCH_7_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_8                                                                      0x80b8
+#define regBIF_BX1_SBIOS_SCRATCH_8_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_9                                                                      0x80b9
+#define regBIF_BX1_SBIOS_SCRATCH_9_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_10                                                                     0x80ba
+#define regBIF_BX1_SBIOS_SCRATCH_10_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_11                                                                     0x80bb
+#define regBIF_BX1_SBIOS_SCRATCH_11_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_12                                                                     0x80bc
+#define regBIF_BX1_SBIOS_SCRATCH_12_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_13                                                                     0x80bd
+#define regBIF_BX1_SBIOS_SCRATCH_13_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_14                                                                     0x80be
+#define regBIF_BX1_SBIOS_SCRATCH_14_BASE_IDX 8
+#define regBIF_BX1_SBIOS_SCRATCH_15                                                                     0x80bf
+#define regBIF_BX1_SBIOS_SCRATCH_15_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_pf_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_PF1_MM_INDEX                                                                          0x8000
+#define regBIF_BX_PF1_MM_INDEX_BASE_IDX 8
+#define regBIF_BX_PF1_MM_DATA                                                                           0x8001
+#define regBIF_BX_PF1_MM_DATA_BASE_IDX 8
+#define regBIF_BX_PF1_MM_INDEX_HI                                                                       0x8006
+#define regBIF_BX_PF1_MM_INDEX_HI_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_BIFDEC1
+// base address: 0x10120000
+#define regBIF_BX1_CC_BIF_BX_STRAP0                                                                     0x8e02
+#define regBIF_BX1_CC_BIF_BX_STRAP0_BASE_IDX 8
+#define regBIF_BX1_CC_BIF_BX_PINSTRAP0                                                                  0x8e04
+#define regBIF_BX1_CC_BIF_BX_PINSTRAP0_BASE_IDX 8
+#define regBIF_BX1_BIF_MM_INDACCESS_CNTL                                                                0x8e06
+#define regBIF_BX1_BIF_MM_INDACCESS_CNTL_BASE_IDX 8
+#define regBIF_BX1_BUS_CNTL                                                                             0x8e07
+#define regBIF_BX1_BUS_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_SCRATCH0                                                                         0x8e08
+#define regBIF_BX1_BIF_SCRATCH0_BASE_IDX 8
+#define regBIF_BX1_BIF_SCRATCH1                                                                         0x8e09
+#define regBIF_BX1_BIF_SCRATCH1_BASE_IDX 8
+#define regBIF_BX1_BX_RESET_EN                                                                          0x8e0d
+#define regBIF_BX1_BX_RESET_EN_BASE_IDX 8
+#define regBIF_BX1_MM_CFGREGS_CNTL                                                                      0x8e0e
+#define regBIF_BX1_MM_CFGREGS_CNTL_BASE_IDX 8
+#define regBIF_BX1_BX_RESET_CNTL                                                                        0x8e10
+#define regBIF_BX1_BX_RESET_CNTL_BASE_IDX 8
+#define regBIF_BX1_INTERRUPT_CNTL                                                                       0x8e11
+#define regBIF_BX1_INTERRUPT_CNTL_BASE_IDX 8
+#define regBIF_BX1_INTERRUPT_CNTL2                                                                      0x8e12
+#define regBIF_BX1_INTERRUPT_CNTL2_BASE_IDX 8
+#define regBIF_BX1_CLKREQB_PAD_CNTL                                                                     0x8e18
+#define regBIF_BX1_CLKREQB_PAD_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_FEATURES_CONTROL_MISC                                                            0x8e1b
+#define regBIF_BX1_BIF_FEATURES_CONTROL_MISC_BASE_IDX 8
+#define regBIF_BX1_HDP_ATOMIC_CONTROL_MISC                                                              0x8e1c
+#define regBIF_BX1_HDP_ATOMIC_CONTROL_MISC_BASE_IDX 8
+#define regBIF_BX1_BIF_DOORBELL_CNTL                                                                    0x8e1d
+#define regBIF_BX1_BIF_DOORBELL_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_DOORBELL_INT_CNTL                                                                0x8e1e
+#define regBIF_BX1_BIF_DOORBELL_INT_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_FB_EN                                                                            0x8e20
+#define regBIF_BX1_BIF_FB_EN_BASE_IDX 8
+#define regBIF_BX1_BIF_INTR_CNTL                                                                        0x8e21
+#define regBIF_BX1_BIF_INTR_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_MST_TRANS_PENDING_VF                                                             0x8e29
+#define regBIF_BX1_BIF_MST_TRANS_PENDING_VF_BASE_IDX 8
+#define regBIF_BX1_BIF_SLV_TRANS_PENDING_VF                                                             0x8e2a
+#define regBIF_BX1_BIF_SLV_TRANS_PENDING_VF_BASE_IDX 8
+#define regBIF_BX1_BACO_CNTL                                                                            0x8e2b
+#define regBIF_BX1_BACO_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_BACO_EXIT_TIME0                                                                  0x8e2c
+#define regBIF_BX1_BIF_BACO_EXIT_TIME0_BASE_IDX 8
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER1                                                                 0x8e2d
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER1_BASE_IDX 8
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER2                                                                 0x8e2e
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER2_BASE_IDX 8
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER3                                                                 0x8e2f
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER3_BASE_IDX 8
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER4                                                                 0x8e30
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER4_BASE_IDX 8
+#define regBIF_BX1_MEM_TYPE_CNTL                                                                        0x8e31
+#define regBIF_BX1_MEM_TYPE_CNTL_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_CNTL                                                               0x8e33
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_CNTL_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_0                                                                  0x8e34
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_0_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_1                                                                  0x8e35
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_1_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_2                                                                  0x8e36
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_2_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_3                                                                  0x8e37
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_3_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_4                                                                  0x8e38
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_4_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_5                                                                  0x8e39
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_5_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_6                                                                  0x8e3a
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_6_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_7                                                                  0x8e3b
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_7_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_8                                                                  0x8e3c
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_8_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_9                                                                  0x8e3d
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_9_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_10                                                                 0x8e3e
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_10_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_11                                                                 0x8e3f
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_11_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_12                                                                 0x8e40
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_12_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_13                                                                 0x8e41
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_13_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_14                                                                 0x8e42
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_14_BASE_IDX 8
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_15                                                                 0x8e43
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_15_BASE_IDX 8
+#define regBIF_BX1_VF_REGWR_EN                                                                          0x8e44
+#define regBIF_BX1_VF_REGWR_EN_BASE_IDX 8
+#define regBIF_BX1_VF_DOORBELL_EN                                                                       0x8e45
+#define regBIF_BX1_VF_DOORBELL_EN_BASE_IDX 8
+#define regBIF_BX1_VF_FB_EN                                                                             0x8e46
+#define regBIF_BX1_VF_FB_EN_BASE_IDX 8
+#define regBIF_BX1_VF_REGWR_STATUS                                                                      0x8e47
+#define regBIF_BX1_VF_REGWR_STATUS_BASE_IDX 8
+#define regBIF_BX1_VF_DOORBELL_STATUS                                                                   0x8e48
+#define regBIF_BX1_VF_DOORBELL_STATUS_BASE_IDX 8
+#define regBIF_BX1_VF_FB_STATUS                                                                         0x8e49
+#define regBIF_BX1_VF_FB_STATUS_BASE_IDX 8
+#define regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL                                                             0x8e4d
+#define regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL_BASE_IDX 8
+#define regBIF_BX1_REMAP_HDP_REG_FLUSH_CNTL                                                             0x8e4e
+#define regBIF_BX1_REMAP_HDP_REG_FLUSH_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_RB_CNTL                                                                          0x8e4f
+#define regBIF_BX1_BIF_RB_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_RB_BASE                                                                          0x8e50
+#define regBIF_BX1_BIF_RB_BASE_BASE_IDX 8
+#define regBIF_BX1_BIF_RB_RPTR                                                                          0x8e51
+#define regBIF_BX1_BIF_RB_RPTR_BASE_IDX 8
+#define regBIF_BX1_BIF_RB_WPTR                                                                          0x8e52
+#define regBIF_BX1_BIF_RB_WPTR_BASE_IDX 8
+#define regBIF_BX1_BIF_RB_WPTR_ADDR_HI                                                                  0x8e53
+#define regBIF_BX1_BIF_RB_WPTR_ADDR_HI_BASE_IDX 8
+#define regBIF_BX1_BIF_RB_WPTR_ADDR_LO                                                                  0x8e54
+#define regBIF_BX1_BIF_RB_WPTR_ADDR_LO_BASE_IDX 8
+#define regBIF_BX1_MAILBOX_INDEX                                                                        0x8e55
+#define regBIF_BX1_MAILBOX_INDEX_BASE_IDX 8
+#define regBIF_BX1_BIF_MP1_INTR_CTRL                                                                    0x8e62
+#define regBIF_BX1_BIF_MP1_INTR_CTRL_BASE_IDX 8
+#define regBIF_BX1_BIF_PERSTB_PAD_CNTL                                                                  0x8e66
+#define regBIF_BX1_BIF_PERSTB_PAD_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_PX_EN_PAD_CNTL                                                                   0x8e67
+#define regBIF_BX1_BIF_PX_EN_PAD_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_REFPADKIN_PAD_CNTL                                                               0x8e68
+#define regBIF_BX1_BIF_REFPADKIN_PAD_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_CLKREQB_PAD_CNTL                                                                 0x8e69
+#define regBIF_BX1_BIF_CLKREQB_PAD_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_PWRBRK_PAD_CNTL                                                                  0x8e6a
+#define regBIF_BX1_BIF_PWRBRK_PAD_CNTL_BASE_IDX 8
+#define regBIF_BX1_BIF_VCN0_GPUIOV_CFG_SIZE                                                             0x8e6b
+#define regBIF_BX1_BIF_VCN0_GPUIOV_CFG_SIZE_BASE_IDX 8
+#define regBIF_BX1_BIF_VCN1_GPUIOV_CFG_SIZE                                                             0x8e6c
+#define regBIF_BX1_BIF_VCN1_GPUIOV_CFG_SIZE_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_pf_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_PF1_BIF_BME_STATUS                                                                    0x8e0b
+#define regBIF_BX_PF1_BIF_BME_STATUS_BASE_IDX 8
+#define regBIF_BX_PF1_BIF_ATOMIC_ERR_LOG                                                                0x8e0c
+#define regBIF_BX_PF1_BIF_ATOMIC_ERR_LOG_BASE_IDX 8
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                              0x8e13
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX 8
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                               0x8e14
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX 8
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL                                                   0x8e15
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX 8
+#define regBIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL                                                      0x8e16
+#define regBIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX 8
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL                                                      0x8e17
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX 8
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                                 0x8e19
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX 8
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                            0x8e1a
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX 8
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_REQ                                                                 0x8e26
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_REQ_BASE_IDX 8
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_DONE                                                                0x8e27
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_DONE_BASE_IDX 8
+#define regBIF_BX_PF1_BIF_TRANS_PENDING                                                                 0x8e28
+#define regBIF_BX_PF1_BIF_TRANS_PENDING_BASE_IDX 8
+#define regBIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS                                                          0x8e32
+#define regBIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX 8
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW0                                                            0x8e56
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX 8
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW1                                                            0x8e57
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX 8
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW2                                                            0x8e58
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX 8
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW3                                                            0x8e59
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX 8
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW0                                                            0x8e5a
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX 8
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW1                                                            0x8e5b
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX 8
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW2                                                            0x8e5c
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX 8
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW3                                                            0x8e5d
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX 8
+#define regBIF_BX_PF1_MAILBOX_CONTROL                                                                   0x8e5e
+#define regBIF_BX_PF1_MAILBOX_CONTROL_BASE_IDX 8
+#define regBIF_BX_PF1_MAILBOX_INT_CNTL                                                                  0x8e5f
+#define regBIF_BX_PF1_MAILBOX_INT_CNTL_BASE_IDX 8
+#define regBIF_BX_PF1_BIF_VMHV_MAILBOX                                                                  0x8e60
+#define regBIF_BX_PF1_BIF_VMHV_MAILBOX_BASE_IDX 8
+#define regBIF_BX_PF1_PARTITION_COMPUTE_CAP                                                             0x8e81
+#define regBIF_BX_PF1_PARTITION_COMPUTE_CAP_BASE_IDX 8
+#define regBIF_BX_PF1_PARTITION_MEM_CAP                                                                 0x8e82
+#define regBIF_BX_PF1_PARTITION_MEM_CAP_BASE_IDX 8
+#define regBIF_BX_PF1_PARTITION_COMPUTE_STATUS                                                          0x8e83
+#define regBIF_BX_PF1_PARTITION_COMPUTE_STATUS_BASE_IDX 8
+#define regBIF_BX_PF1_PARTITION_MEM_STATUS                                                              0x8e84
+#define regBIF_BX_PF1_PARTITION_MEM_STATUS_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_rcc_strap_BIFDEC1:1
+// base address: 0x10120000
+#define regRCC_STRAP2_RCC_BIF_STRAP0                                                                    0x8d20
+#define regRCC_STRAP2_RCC_BIF_STRAP0_BASE_IDX 8
+#define regRCC_STRAP2_RCC_BIF_STRAP1                                                                    0x8d21
+#define regRCC_STRAP2_RCC_BIF_STRAP1_BASE_IDX 8
+#define regRCC_STRAP2_RCC_BIF_STRAP2                                                                    0x8d22
+#define regRCC_STRAP2_RCC_BIF_STRAP2_BASE_IDX 8
+#define regRCC_STRAP2_RCC_BIF_STRAP3                                                                    0x8d23
+#define regRCC_STRAP2_RCC_BIF_STRAP3_BASE_IDX 8
+#define regRCC_STRAP2_RCC_BIF_STRAP4                                                                    0x8d24
+#define regRCC_STRAP2_RCC_BIF_STRAP4_BASE_IDX 8
+#define regRCC_STRAP2_RCC_BIF_STRAP5                                                                    0x8d25
+#define regRCC_STRAP2_RCC_BIF_STRAP5_BASE_IDX 8
+#define regRCC_STRAP2_RCC_BIF_STRAP6                                                                    0x8d26
+#define regRCC_STRAP2_RCC_BIF_STRAP6_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP0                                                              0x8d27
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP0_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP1                                                              0x8d28
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP1_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP10                                                             0x8d29
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP10_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP11                                                             0x8d2a
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP11_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP12                                                             0x8d2b
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP12_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP13                                                             0x8d2c
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP13_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP14                                                             0x8d2d
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP14_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP2                                                              0x8d2e
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP2_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP3                                                              0x8d2f
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP3_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP4                                                              0x8d30
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP4_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP5                                                              0x8d31
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP5_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP6                                                              0x8d32
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP6_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP7                                                              0x8d33
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP7_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP8                                                              0x8d34
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP8_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP9                                                              0x8d35
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP9_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP0                                                              0x8d36
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP0_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP1                                                              0x8d37
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP1_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP13                                                             0x8d38
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP13_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP14                                                             0x8d39
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP14_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP15                                                             0x8d3a
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP15_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP16                                                             0x8d3b
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP16_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP17                                                             0x8d3c
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP17_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP18                                                             0x8d3d
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP18_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP2                                                              0x8d3e
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP2_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP26                                                             0x8d3f
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP26_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP3                                                              0x8d40
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP3_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP4                                                              0x8d41
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP4_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP5                                                              0x8d42
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP5_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP8                                                              0x8d44
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP8_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP9                                                              0x8d45
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP9_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP0                                                              0x8d46
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP0_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP2                                                              0x8d52
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP2_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP20                                                             0x8d53
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP20_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP21                                                             0x8d54
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP21_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP22                                                             0x8d55
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP22_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP23                                                             0x8d56
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP23_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP24                                                             0x8d57
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP24_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP25                                                             0x8d58
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP25_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP3                                                              0x8d59
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP3_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP4                                                              0x8d5a
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP4_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP5                                                              0x8d5b
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP5_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP6                                                              0x8d5c
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP6_BASE_IDX 8
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP7                                                              0x8d5d
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP7_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_gdc_hst_sion_SIONDEC
+// base address: 0x1400000
+#define regS2A_DOORBELL_ENTRY_0_CTRL 0x7a80
+#define regS2A_DOORBELL_ENTRY_0_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_1_CTRL 0x7a81
+#define regS2A_DOORBELL_ENTRY_1_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_2_CTRL 0x7a82
+#define regS2A_DOORBELL_ENTRY_2_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_3_CTRL 0x7a83
+#define regS2A_DOORBELL_ENTRY_3_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_4_CTRL 0x7a84
+#define regS2A_DOORBELL_ENTRY_4_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_5_CTRL 0x7a85
+#define regS2A_DOORBELL_ENTRY_5_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_6_CTRL 0x7a86
+#define regS2A_DOORBELL_ENTRY_6_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_7_CTRL 0x7a87
+#define regS2A_DOORBELL_ENTRY_7_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_8_CTRL 0x7a88
+#define regS2A_DOORBELL_ENTRY_8_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_9_CTRL 0x7a89
+#define regS2A_DOORBELL_ENTRY_9_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_10_CTRL 0x7a8a
+#define regS2A_DOORBELL_ENTRY_10_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_11_CTRL 0x7a8b
+#define regS2A_DOORBELL_ENTRY_11_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_12_CTRL 0x7a8c
+#define regS2A_DOORBELL_ENTRY_12_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_13_CTRL 0x7a8d
+#define regS2A_DOORBELL_ENTRY_13_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_14_CTRL 0x7a8e
+#define regS2A_DOORBELL_ENTRY_14_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_ENTRY_15_CTRL 0x7a8f
+#define regS2A_DOORBELL_ENTRY_15_CTRL_BASE_IDX 5
+#define regS2A_DOORBELL_COMMON_CTRL_REG 0x7a90
+#define regS2A_DOORBELL_COMMON_CTRL_REG_BASE_IDX 5
+
+
+// addressBlock: aid_nbio_nbif0_gdc_GDCDEC
+// base address: 0x1400000
+#define regGDC1_A2S_CNTL_CL0 0x0ea0
+#define regGDC1_A2S_CNTL_CL0_BASE_IDX 5
+#define regGDC1_A2S_CNTL_CL1 0x0ea1
+#define regGDC1_A2S_CNTL_CL1_BASE_IDX 5
+#define regGDC1_A2S_CNTL3_CL0 0x0eb8
+#define regGDC1_A2S_CNTL3_CL0_BASE_IDX 5
+#define regGDC1_A2S_CNTL3_CL1 0x0eb9
+#define regGDC1_A2S_CNTL3_CL1_BASE_IDX 5
+#define regGDC1_A2S_CNTL_SW0 0x0ed0
+#define regGDC1_A2S_CNTL_SW0_BASE_IDX 5
+#define regGDC1_A2S_CNTL_SW1 0x0ed1
+#define regGDC1_A2S_CNTL_SW1_BASE_IDX 5
+#define regGDC1_A2S_CNTL_SW2 0x0ed2
+#define regGDC1_A2S_CNTL_SW2_BASE_IDX 5
+#define regGDC1_A2S_TAG_ALLOC_0 0x0edd
+#define regGDC1_A2S_TAG_ALLOC_0_BASE_IDX 5
+#define regGDC1_A2S_TAG_ALLOC_1 0x0ede
+#define regGDC1_A2S_TAG_ALLOC_1_BASE_IDX 5
+#define regGDC1_A2S_MISC_CNTL 0x0ee1
+#define regGDC1_A2S_MISC_CNTL_BASE_IDX 5
+#define regGDC1_SHUB_REGS_IF_CTL 0x0ee3
+#define regGDC1_SHUB_REGS_IF_CTL_BASE_IDX 5
+#define regGDC1_NGDC_MGCG_CTRL 0x0eea
+#define regGDC1_NGDC_MGCG_CTRL_BASE_IDX 5
+#define regGDC1_NGDC_RESERVED_0 0x0eeb
+#define regGDC1_NGDC_RESERVED_0_BASE_IDX 5
+#define regGDC1_NGDC_RESERVED_1 0x0eec
+#define regGDC1_NGDC_RESERVED_1_BASE_IDX 5
+#define regGDC1_NBIF_GFX_DOORBELL_STATUS 0x0eef
+#define regGDC1_NBIF_GFX_DOORBELL_STATUS_BASE_IDX 5
+#define regGDC1_ATDMA_MISC_CNTL 0x0efd
+#define regGDC1_ATDMA_MISC_CNTL_BASE_IDX 5
+#define regGDC1_S2A_MISC_CNTL 0x0eff
+#define regGDC1_S2A_MISC_CNTL_BASE_IDX 5
+#define regGDC1_NGDC_EARLY_WAKEUP_CTRL 0x0f01
+#define regGDC1_NGDC_EARLY_WAKEUP_CTRL_BASE_IDX 5
+#define regGDC1_NGDC_PG_MISC_CTRL 0x0f18
+#define regGDC1_NGDC_PG_MISC_CTRL_BASE_IDX 5
+#define regGDC1_NGDC_PGMST_CTRL 0x0f19
+#define regGDC1_NGDC_PGMST_CTRL_BASE_IDX 5
+#define regGDC1_NGDC_PGSLV_CTRL 0x0f1a
+#define regGDC1_NGDC_PGSLV_CTRL_BASE_IDX 5
+
+
+// addressBlock: aid_nbio_nbif0_gdc_sec_GDCSEC_DEC
+// base address: 0x1400000
+#define regXCC_DOORBELL_FENCE 0x740c
+#define regXCC_DOORBELL_FENCE_BASE_IDX 5
+
+
+// addressBlock: aid_nbio_nbif0_gdc_rst_GDCRST_DEC
+// base address: 0x1400000
+#define regSHUB_PF_FLR_RST 0x7c00
+#define regSHUB_PF_FLR_RST_BASE_IDX 5
+#define regSHUB_GFX_DRV_VPU_RST 0x7c01
+#define regSHUB_GFX_DRV_VPU_RST_BASE_IDX 5
+#define regSHUB_LINK_RESET 0x7c02
+#define regSHUB_LINK_RESET_BASE_IDX 5
+#define regSHUB_HARD_RST_CTRL 0x7c10
+#define regSHUB_HARD_RST_CTRL_BASE_IDX 5
+#define regSHUB_SOFT_RST_CTRL 0x7c11
+#define regSHUB_SOFT_RST_CTRL_BASE_IDX 5
+#define regSHUB_SDP_PORT_RST 0x7c12
+#define regSHUB_SDP_PORT_RST_BASE_IDX 5
+#define regSHUB_RST_MISC_TRL 0x7c13
+#define regSHUB_RST_MISC_TRL_BASE_IDX 5
+
+
+// addressBlock: aid_nbio_nbif0_syshub_mmreg_syshubdirect
+// base address: 0x1400000
+#define regHST_CLK0_SW0_CL0_CNTL 0x4140
+#define regHST_CLK0_SW0_CL0_CNTL_BASE_IDX 5
+#define regHST_CLK0_SW1_CL0_CNTL 0x4160
+#define regHST_CLK0_SW1_CL0_CNTL_BASE_IDX 5
+#define regHST_CLK0_SW1_CL1_CNTL 0x4161
+#define regHST_CLK0_SW1_CL1_CNTL_BASE_IDX 5
+#define regHST_CLK0_SW1_CL2_CNTL 0x4162
+#define regHST_CLK0_SW1_CL2_CNTL_BASE_IDX 5
+#define regDMA_CLK0_SW0_CL0_CNTL 0x4240
+#define regDMA_CLK0_SW0_CL0_CNTL_BASE_IDX 5
+#define regDMA_CLK0_SW0_CL1_CNTL 0x4241
+#define regDMA_CLK0_SW0_CL1_CNTL_BASE_IDX 5
+#define regNIC400_1_ASIB_0_FN_MOD 0xc042
+#define regNIC400_1_ASIB_0_FN_MOD_BASE_IDX 5
+#define regNIC400_1_IB_0_FN_MOD 0xfc42
+#define regNIC400_1_IB_0_FN_MOD_BASE_IDX 5
+#define regNIC400_2_ASIB_0_FN_MOD 0x10c42
+#define regNIC400_2_ASIB_0_FN_MOD_BASE_IDX 5
+#define regNIC400_2_ASIB_0_QOS_CNTL 0x10c43
+#define regNIC400_2_ASIB_0_QOS_CNTL_BASE_IDX 5
+#define regNIC400_2_ASIB_0_MAX_OT 0x10c44
+#define regNIC400_2_ASIB_0_MAX_OT_BASE_IDX 5
+#define regNIC400_2_ASIB_0_MAX_COMB_OT 0x10c45
+#define regNIC400_2_ASIB_0_MAX_COMB_OT_BASE_IDX 5
+#define regNIC400_2_ASIB_0_AW_P 0x10c46
+#define regNIC400_2_ASIB_0_AW_P_BASE_IDX 5
+#define regNIC400_2_ASIB_0_AW_B 0x10c47
+#define regNIC400_2_ASIB_0_AW_B_BASE_IDX 5
+#define regNIC400_2_ASIB_0_AW_R 0x10c48
+#define regNIC400_2_ASIB_0_AW_R_BASE_IDX 5
+#define regNIC400_2_ASIB_0_AR_P 0x10c49
+#define regNIC400_2_ASIB_0_AR_P_BASE_IDX 5
+#define regNIC400_2_ASIB_0_AR_B 0x10c4a
+#define regNIC400_2_ASIB_0_AR_B_BASE_IDX 5
+#define regNIC400_2_ASIB_0_AR_R 0x10c4b
+#define regNIC400_2_ASIB_0_AR_R_BASE_IDX 5
+#define regNIC400_2_ASIB_0_TARGET_FC 0x10c4c
+#define regNIC400_2_ASIB_0_TARGET_FC_BASE_IDX 5
+#define regNIC400_2_ASIB_0_KI_FC 0x10c4d
+#define regNIC400_2_ASIB_0_KI_FC_BASE_IDX 5
+#define regNIC400_2_ASIB_0_QOS_RANGE 0x10c4e
+#define regNIC400_2_ASIB_0_QOS_RANGE_BASE_IDX 5
+#define regNIC400_2_ASIB_1_FN_MOD 0x11042
+#define regNIC400_2_ASIB_1_FN_MOD_BASE_IDX 5
+#define regNIC400_2_ASIB_1_QOS_CNTL 0x11043
+#define regNIC400_2_ASIB_1_QOS_CNTL_BASE_IDX 5
+#define regNIC400_2_ASIB_1_MAX_OT 0x11044
+#define regNIC400_2_ASIB_1_MAX_OT_BASE_IDX 5
+#define regNIC400_2_ASIB_1_MAX_COMB_OT 0x11045
+#define regNIC400_2_ASIB_1_MAX_COMB_OT_BASE_IDX 5
+#define regNIC400_2_ASIB_1_AW_P 0x11046
+#define regNIC400_2_ASIB_1_AW_P_BASE_IDX 5
+#define regNIC400_2_ASIB_1_AW_B 0x11047
+#define regNIC400_2_ASIB_1_AW_B_BASE_IDX 5
+#define regNIC400_2_ASIB_1_AW_R 0x11048
+#define regNIC400_2_ASIB_1_AW_R_BASE_IDX 5
+#define regNIC400_2_ASIB_1_AR_P 0x11049
+#define regNIC400_2_ASIB_1_AR_P_BASE_IDX 5
+#define regNIC400_2_ASIB_1_AR_B 0x1104a
+#define regNIC400_2_ASIB_1_AR_B_BASE_IDX 5
+#define regNIC400_2_ASIB_1_AR_R 0x1104b
+#define regNIC400_2_ASIB_1_AR_R_BASE_IDX 5
+#define regNIC400_2_ASIB_1_TARGET_FC 0x1104c
+#define regNIC400_2_ASIB_1_TARGET_FC_BASE_IDX 5
+#define regNIC400_2_ASIB_1_KI_FC 0x1104d
+#define regNIC400_2_ASIB_1_KI_FC_BASE_IDX 5
+#define regNIC400_2_ASIB_1_QOS_RANGE 0x1104e
+#define regNIC400_2_ASIB_1_QOS_RANGE_BASE_IDX 5
+#define regNIC400_2_IB_0_FN_MOD 0x13c42
+#define regNIC400_2_IB_0_FN_MOD_BASE_IDX 5
+
+
+// addressBlock: aid_nbio_iohub_nb_nbcfg_nb_cfgdec
+// base address: 0x13b00000
+#define regNB_NBCFG0_NBCFG_SCRATCH_4                                                                    0xe8001e
+#define regNB_NBCFG0_NBCFG_SCRATCH_4_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_misc_misc_cfgdec
+// base address: 0x13b10000
+#define regNB_CNTL                                                                                      0xe84000
+#define regNB_CNTL_BASE_IDX 8
+#define regNB_SPARE1                                                                                    0xe84003
+#define regNB_SPARE1_BASE_IDX 8
+#define regNB_SPARE2                                                                                    0xe84004
+#define regNB_SPARE2_BASE_IDX 8
+#define regNB_REVID                                                                                     0xe84005
+#define regNB_REVID_BASE_IDX 8
+#define regNBIO_LCLK_DS_MASK                                                                            0xe84009
+#define regNBIO_LCLK_DS_MASK_BASE_IDX 8
+#define regNB_BUS_NUM_CNTL                                                                              0xe84011
+#define regNB_BUS_NUM_CNTL_BASE_IDX 8
+#define regNB_MMIOBASE                                                                                  0xe84017
+#define regNB_MMIOBASE_BASE_IDX 8
+#define regNB_MMIOLIMIT                                                                                 0xe84018
+#define regNB_MMIOLIMIT_BASE_IDX 8
+#define regNB_LOWER_TOP_OF_DRAM2                                                                        0xe84019
+#define regNB_LOWER_TOP_OF_DRAM2_BASE_IDX 8
+#define regNB_UPPER_TOP_OF_DRAM2                                                                        0xe8401a
+#define regNB_UPPER_TOP_OF_DRAM2_BASE_IDX 8
+#define regNB_LOWER_DRAM2_BASE                                                                          0xe8401b
+#define regNB_LOWER_DRAM2_BASE_BASE_IDX 8
+#define regNB_UPPER_DRAM2_BASE                                                                          0xe8401c
+#define regNB_UPPER_DRAM2_BASE_BASE_IDX 8
+#define regSB_LOCATION                                                                                  0xe8401f
+#define regSB_LOCATION_BASE_IDX 8
+#define regSW_US_LOCATION                                                                               0xe84020
+#define regSW_US_LOCATION_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr0                                                                    0xe8402e
+#define regNB_PROG_DEVICE_REMAP_PBr0_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr1                                                                    0xe8402f
+#define regNB_PROG_DEVICE_REMAP_PBr1_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr2                                                                    0xe84030
+#define regNB_PROG_DEVICE_REMAP_PBr2_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr3                                                                    0xe84031
+#define regNB_PROG_DEVICE_REMAP_PBr3_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr4                                                                    0xe84032
+#define regNB_PROG_DEVICE_REMAP_PBr4_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr5                                                                    0xe84033
+#define regNB_PROG_DEVICE_REMAP_PBr5_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr6                                                                    0xe84034
+#define regNB_PROG_DEVICE_REMAP_PBr6_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr7                                                                    0xe84035
+#define regNB_PROG_DEVICE_REMAP_PBr7_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr8                                                                    0xe84036
+#define regNB_PROG_DEVICE_REMAP_PBr8_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr10                                                                   0xe84038
+#define regNB_PROG_DEVICE_REMAP_PBr10_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr11                                                                   0xe84039
+#define regNB_PROG_DEVICE_REMAP_PBr11_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr12                                                                   0xe8403a
+#define regNB_PROG_DEVICE_REMAP_PBr12_BASE_IDX 8
+#define regNB_PROG_DEVICE_REMAP_PBr13                                                                   0xe8403b
+#define regNB_PROG_DEVICE_REMAP_PBr13_BASE_IDX 8
+#define regSW_NMI_CNTL                                                                                  0xe84042
+#define regSW_NMI_CNTL_BASE_IDX 8
+#define regSW_SMI_CNTL                                                                                  0xe84043
+#define regSW_SMI_CNTL_BASE_IDX 8
+#define regSW_SCI_CNTL                                                                                  0xe84044
+#define regSW_SCI_CNTL_BASE_IDX 8
+#define regAPML_SW_STATUS                                                                               0xe84045
+#define regAPML_SW_STATUS_BASE_IDX 8
+#define regSW_GIC_SPI_CNTL                                                                              0xe84047
+#define regSW_GIC_SPI_CNTL_BASE_IDX 8
+#define regSW_SYNCFLOOD_CNTL                                                                            0xe84049
+#define regSW_SYNCFLOOD_CNTL_BASE_IDX 8
+#define regNB_TOP_OF_DRAM3                                                                              0xe8404e
+#define regNB_TOP_OF_DRAM3_BASE_IDX 8
+#define regCAM_CONTROL                                                                                  0xe84052
+#define regCAM_CONTROL_BASE_IDX 8
+#define regCAM_TARGET_INDEX_ADDR_BOTTOM                                                                 0xe84053
+#define regCAM_TARGET_INDEX_ADDR_BOTTOM_BASE_IDX 8
+#define regCAM_TARGET_INDEX_ADDR_TOP                                                                    0xe84054
+#define regCAM_TARGET_INDEX_ADDR_TOP_BASE_IDX 8
+#define regCAM_TARGET_INDEX_DATA                                                                        0xe84055
+#define regCAM_TARGET_INDEX_DATA_BASE_IDX 8
+#define regCAM_TARGET_INDEX_DATA_MASK                                                                   0xe84056
+#define regCAM_TARGET_INDEX_DATA_MASK_BASE_IDX 8
+#define regCAM_TARGET_DATA_ADDR_BOTTOM                                                                  0xe84057
+#define regCAM_TARGET_DATA_ADDR_BOTTOM_BASE_IDX 8
+#define regCAM_TARGET_DATA_ADDR_TOP                                                                     0xe84059
+#define regCAM_TARGET_DATA_ADDR_TOP_BASE_IDX 8
+#define regCAM_TARGET_DATA                                                                              0xe8405a
+#define regCAM_TARGET_DATA_BASE_IDX 8
+#define regCAM_TARGET_DATA_MASK                                                                         0xe8405b
+#define regCAM_TARGET_DATA_MASK_BASE_IDX 8
+#define regP_DMA_DROPPED_LOG_LOWER                                                                      0xe84060
+#define regP_DMA_DROPPED_LOG_LOWER_BASE_IDX 8
+#define regP_DMA_DROPPED_LOG_UPPER                                                                      0xe84061
+#define regP_DMA_DROPPED_LOG_UPPER_BASE_IDX 8
+#define regNP_DMA_DROPPED_LOG_LOWER                                                                     0xe84062
+#define regNP_DMA_DROPPED_LOG_LOWER_BASE_IDX 8
+#define regNP_DMA_DROPPED_LOG_UPPER                                                                     0xe84063
+#define regNP_DMA_DROPPED_LOG_UPPER_BASE_IDX 8
+#define regPCIE_VDM_NODE0_CTRL4                                                                         0xe84064
+#define regPCIE_VDM_NODE0_CTRL4_BASE_IDX 8
+#define regPCIE_VDM_CNTL2                                                                               0xe8408c
+#define regPCIE_VDM_CNTL2_BASE_IDX 8
+#define regPCIE_VDM_CNTL3                                                                               0xe8408d
+#define regPCIE_VDM_CNTL3_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT0_0                                                                    0xe84090
+#define regSTALL_CONTROL_XBARPORT0_0_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT0_1                                                                    0xe84091
+#define regSTALL_CONTROL_XBARPORT0_1_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT1_0                                                                    0xe84093
+#define regSTALL_CONTROL_XBARPORT1_0_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT1_1                                                                    0xe84094
+#define regSTALL_CONTROL_XBARPORT1_1_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT2_0                                                                    0xe84096
+#define regSTALL_CONTROL_XBARPORT2_0_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT2_1                                                                    0xe84097
+#define regSTALL_CONTROL_XBARPORT2_1_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT3_0                                                                    0xe84099
+#define regSTALL_CONTROL_XBARPORT3_0_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT3_1                                                                    0xe8409a
+#define regSTALL_CONTROL_XBARPORT3_1_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT4_0                                                                    0xe8409c
+#define regSTALL_CONTROL_XBARPORT4_0_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT4_1                                                                    0xe8409d
+#define regSTALL_CONTROL_XBARPORT4_1_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT5_0                                                                    0xe8409f
+#define regSTALL_CONTROL_XBARPORT5_0_BASE_IDX 8
+#define regSTALL_CONTROL_XBARPORT5_1                                                                    0xe840a0
+#define regSTALL_CONTROL_XBARPORT5_1_BASE_IDX 8
+#define regNB_DRAM3_BASE                                                                                0xe840b1
+#define regNB_DRAM3_BASE_BASE_IDX 8
+#define regPSP_BASE_ADDR_LO                                                                             0xe840b8
+#define regPSP_BASE_ADDR_LO_BASE_IDX 8
+#define regPSP_BASE_ADDR_HI                                                                             0xe840b9
+#define regPSP_BASE_ADDR_HI_BASE_IDX 8
+#define regSMU_BASE_ADDR_LO                                                                             0xe840ba
+#define regSMU_BASE_ADDR_LO_BASE_IDX 8
+#define regSMU_BASE_ADDR_HI                                                                             0xe840bb
+#define regSMU_BASE_ADDR_HI_BASE_IDX 8
+#define regSCRATCH_4                                                                                    0xe840fc
+#define regSCRATCH_4_BASE_IDX 8
+#define regSCRATCH_5                                                                                    0xe840fd
+#define regSCRATCH_5_BASE_IDX 8
+#define regSMU_BLOCK_CPU                                                                                0xe840fe
+#define regSMU_BLOCK_CPU_BASE_IDX 8
+#define regSMU_BLOCK_CPU_STATUS                                                                         0xe840ff
+#define regSMU_BLOCK_CPU_STATUS_BASE_IDX 8
+#define regTRAP_STATUS                                                                                  0xe84100
+#define regTRAP_STATUS_BASE_IDX 8
+#define regTRAP_REQUEST0                                                                                0xe84101
+#define regTRAP_REQUEST0_BASE_IDX 8
+#define regTRAP_REQUEST1                                                                                0xe84102
+#define regTRAP_REQUEST1_BASE_IDX 8
+#define regTRAP_REQUEST2                                                                                0xe84103
+#define regTRAP_REQUEST2_BASE_IDX 8
+#define regTRAP_REQUEST3                                                                                0xe84104
+#define regTRAP_REQUEST3_BASE_IDX 8
+#define regTRAP_REQUEST4                                                                                0xe84105
+#define regTRAP_REQUEST4_BASE_IDX 8
+#define regTRAP_REQUEST5                                                                                0xe84106
+#define regTRAP_REQUEST5_BASE_IDX 8
+#define regTRAP_REQUEST_DATASTRB0                                                                       0xe84108
+#define regTRAP_REQUEST_DATASTRB0_BASE_IDX 8
+#define regTRAP_REQUEST_DATASTRB1                                                                       0xe84109
+#define regTRAP_REQUEST_DATASTRB1_BASE_IDX 8
+#define regTRAP_REQUEST_DATA0                                                                           0xe84110
+#define regTRAP_REQUEST_DATA0_BASE_IDX 8
+#define regTRAP_REQUEST_DATA1                                                                           0xe84111
+#define regTRAP_REQUEST_DATA1_BASE_IDX 8
+#define regTRAP_REQUEST_DATA2                                                                           0xe84112
+#define regTRAP_REQUEST_DATA2_BASE_IDX 8
+#define regTRAP_REQUEST_DATA3                                                                           0xe84113
+#define regTRAP_REQUEST_DATA3_BASE_IDX 8
+#define regTRAP_REQUEST_DATA4                                                                           0xe84114
+#define regTRAP_REQUEST_DATA4_BASE_IDX 8
+#define regTRAP_REQUEST_DATA5                                                                           0xe84115
+#define regTRAP_REQUEST_DATA5_BASE_IDX 8
+#define regTRAP_REQUEST_DATA6                                                                           0xe84116
+#define regTRAP_REQUEST_DATA6_BASE_IDX 8
+#define regTRAP_REQUEST_DATA7                                                                           0xe84117
+#define regTRAP_REQUEST_DATA7_BASE_IDX 8
+#define regTRAP_REQUEST_DATA8                                                                           0xe84118
+#define regTRAP_REQUEST_DATA8_BASE_IDX 8
+#define regTRAP_REQUEST_DATA9                                                                           0xe84119
+#define regTRAP_REQUEST_DATA9_BASE_IDX 8
+#define regTRAP_REQUEST_DATA10                                                                          0xe8411a
+#define regTRAP_REQUEST_DATA10_BASE_IDX 8
+#define regTRAP_REQUEST_DATA11                                                                          0xe8411b
+#define regTRAP_REQUEST_DATA11_BASE_IDX 8
+#define regTRAP_REQUEST_DATA12                                                                          0xe8411c
+#define regTRAP_REQUEST_DATA12_BASE_IDX 8
+#define regTRAP_REQUEST_DATA13                                                                          0xe8411d
+#define regTRAP_REQUEST_DATA13_BASE_IDX 8
+#define regTRAP_REQUEST_DATA14                                                                          0xe8411e
+#define regTRAP_REQUEST_DATA14_BASE_IDX 8
+#define regTRAP_REQUEST_DATA15                                                                          0xe8411f
+#define regTRAP_REQUEST_DATA15_BASE_IDX 8
+#define regTRAP_RESPONSE_CONTROL                                                                        0xe84130
+#define regTRAP_RESPONSE_CONTROL_BASE_IDX 8
+#define regTRAP_RESPONSE0                                                                               0xe84131
+#define regTRAP_RESPONSE0_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA0                                                                          0xe84140
+#define regTRAP_RESPONSE_DATA0_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA1                                                                          0xe84141
+#define regTRAP_RESPONSE_DATA1_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA2                                                                          0xe84142
+#define regTRAP_RESPONSE_DATA2_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA3                                                                          0xe84143
+#define regTRAP_RESPONSE_DATA3_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA4                                                                          0xe84144
+#define regTRAP_RESPONSE_DATA4_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA5                                                                          0xe84145
+#define regTRAP_RESPONSE_DATA5_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA6                                                                          0xe84146
+#define regTRAP_RESPONSE_DATA6_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA7                                                                          0xe84147
+#define regTRAP_RESPONSE_DATA7_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA8                                                                          0xe84148
+#define regTRAP_RESPONSE_DATA8_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA9                                                                          0xe84149
+#define regTRAP_RESPONSE_DATA9_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA10                                                                         0xe8414a
+#define regTRAP_RESPONSE_DATA10_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA11                                                                         0xe8414b
+#define regTRAP_RESPONSE_DATA11_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA12                                                                         0xe8414c
+#define regTRAP_RESPONSE_DATA12_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA13                                                                         0xe8414d
+#define regTRAP_RESPONSE_DATA13_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA14                                                                         0xe8414e
+#define regTRAP_RESPONSE_DATA14_BASE_IDX 8
+#define regTRAP_RESPONSE_DATA15                                                                         0xe8414f
+#define regTRAP_RESPONSE_DATA15_BASE_IDX 8
+#define regTRAP0_CONTROL0                                                                               0xe84200
+#define regTRAP0_CONTROL0_BASE_IDX 8
+#define regTRAP0_ADDRESS_LO                                                                             0xe84202
+#define regTRAP0_ADDRESS_LO_BASE_IDX 8
+#define regTRAP0_ADDRESS_HI                                                                             0xe84203
+#define regTRAP0_ADDRESS_HI_BASE_IDX 8
+#define regTRAP0_COMMAND                                                                                0xe84204
+#define regTRAP0_COMMAND_BASE_IDX 8
+#define regTRAP0_ADDRESS_LO_MASK                                                                        0xe84206
+#define regTRAP0_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP0_ADDRESS_HI_MASK                                                                        0xe84207
+#define regTRAP0_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP0_COMMAND_MASK                                                                           0xe84208
+#define regTRAP0_COMMAND_MASK_BASE_IDX 8
+#define regTRAP1_CONTROL0                                                                               0xe84210
+#define regTRAP1_CONTROL0_BASE_IDX 8
+#define regTRAP1_ADDRESS_LO                                                                             0xe84212
+#define regTRAP1_ADDRESS_LO_BASE_IDX 8
+#define regTRAP1_ADDRESS_HI                                                                             0xe84213
+#define regTRAP1_ADDRESS_HI_BASE_IDX 8
+#define regTRAP1_COMMAND                                                                                0xe84214
+#define regTRAP1_COMMAND_BASE_IDX 8
+#define regTRAP1_ADDRESS_LO_MASK                                                                        0xe84216
+#define regTRAP1_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP1_ADDRESS_HI_MASK                                                                        0xe84217
+#define regTRAP1_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP1_COMMAND_MASK                                                                           0xe84218
+#define regTRAP1_COMMAND_MASK_BASE_IDX 8
+#define regTRAP2_CONTROL0                                                                               0xe84220
+#define regTRAP2_CONTROL0_BASE_IDX 8
+#define regTRAP2_ADDRESS_LO                                                                             0xe84222
+#define regTRAP2_ADDRESS_LO_BASE_IDX 8
+#define regTRAP2_ADDRESS_HI                                                                             0xe84223
+#define regTRAP2_ADDRESS_HI_BASE_IDX 8
+#define regTRAP2_COMMAND                                                                                0xe84224
+#define regTRAP2_COMMAND_BASE_IDX 8
+#define regTRAP2_ADDRESS_LO_MASK                                                                        0xe84226
+#define regTRAP2_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP2_ADDRESS_HI_MASK                                                                        0xe84227
+#define regTRAP2_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP2_COMMAND_MASK                                                                           0xe84228
+#define regTRAP2_COMMAND_MASK_BASE_IDX 8
+#define regTRAP3_CONTROL0                                                                               0xe84230
+#define regTRAP3_CONTROL0_BASE_IDX 8
+#define regTRAP3_ADDRESS_LO                                                                             0xe84232
+#define regTRAP3_ADDRESS_LO_BASE_IDX 8
+#define regTRAP3_ADDRESS_HI                                                                             0xe84233
+#define regTRAP3_ADDRESS_HI_BASE_IDX 8
+#define regTRAP3_COMMAND                                                                                0xe84234
+#define regTRAP3_COMMAND_BASE_IDX 8
+#define regTRAP3_ADDRESS_LO_MASK                                                                        0xe84236
+#define regTRAP3_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP3_ADDRESS_HI_MASK                                                                        0xe84237
+#define regTRAP3_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP3_COMMAND_MASK                                                                           0xe84238
+#define regTRAP3_COMMAND_MASK_BASE_IDX 8
+#define regTRAP4_CONTROL0                                                                               0xe84240
+#define regTRAP4_CONTROL0_BASE_IDX 8
+#define regTRAP4_ADDRESS_LO                                                                             0xe84242
+#define regTRAP4_ADDRESS_LO_BASE_IDX 8
+#define regTRAP4_ADDRESS_HI                                                                             0xe84243
+#define regTRAP4_ADDRESS_HI_BASE_IDX 8
+#define regTRAP4_COMMAND                                                                                0xe84244
+#define regTRAP4_COMMAND_BASE_IDX 8
+#define regTRAP4_ADDRESS_LO_MASK                                                                        0xe84246
+#define regTRAP4_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP4_ADDRESS_HI_MASK                                                                        0xe84247
+#define regTRAP4_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP4_COMMAND_MASK                                                                           0xe84248
+#define regTRAP4_COMMAND_MASK_BASE_IDX 8
+#define regTRAP5_CONTROL0                                                                               0xe84250
+#define regTRAP5_CONTROL0_BASE_IDX 8
+#define regTRAP5_ADDRESS_LO                                                                             0xe84252
+#define regTRAP5_ADDRESS_LO_BASE_IDX 8
+#define regTRAP5_ADDRESS_HI                                                                             0xe84253
+#define regTRAP5_ADDRESS_HI_BASE_IDX 8
+#define regTRAP5_COMMAND                                                                                0xe84254
+#define regTRAP5_COMMAND_BASE_IDX 8
+#define regTRAP5_ADDRESS_LO_MASK                                                                        0xe84256
+#define regTRAP5_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP5_ADDRESS_HI_MASK                                                                        0xe84257
+#define regTRAP5_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP5_COMMAND_MASK                                                                           0xe84258
+#define regTRAP5_COMMAND_MASK_BASE_IDX 8
+#define regTRAP6_CONTROL0                                                                               0xe84260
+#define regTRAP6_CONTROL0_BASE_IDX 8
+#define regTRAP6_ADDRESS_LO                                                                             0xe84262
+#define regTRAP6_ADDRESS_LO_BASE_IDX 8
+#define regTRAP6_ADDRESS_HI                                                                             0xe84263
+#define regTRAP6_ADDRESS_HI_BASE_IDX 8
+#define regTRAP6_COMMAND                                                                                0xe84264
+#define regTRAP6_COMMAND_BASE_IDX 8
+#define regTRAP6_ADDRESS_LO_MASK                                                                        0xe84266
+#define regTRAP6_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP6_ADDRESS_HI_MASK                                                                        0xe84267
+#define regTRAP6_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP6_COMMAND_MASK                                                                           0xe84268
+#define regTRAP6_COMMAND_MASK_BASE_IDX 8
+#define regTRAP7_CONTROL0                                                                               0xe84270
+#define regTRAP7_CONTROL0_BASE_IDX 8
+#define regTRAP7_ADDRESS_LO                                                                             0xe84272
+#define regTRAP7_ADDRESS_LO_BASE_IDX 8
+#define regTRAP7_ADDRESS_HI                                                                             0xe84273
+#define regTRAP7_ADDRESS_HI_BASE_IDX 8
+#define regTRAP7_COMMAND                                                                                0xe84274
+#define regTRAP7_COMMAND_BASE_IDX 8
+#define regTRAP7_ADDRESS_LO_MASK                                                                        0xe84276
+#define regTRAP7_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP7_ADDRESS_HI_MASK                                                                        0xe84277
+#define regTRAP7_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP7_COMMAND_MASK                                                                           0xe84278
+#define regTRAP7_COMMAND_MASK_BASE_IDX 8
+#define regTRAP8_CONTROL0                                                                               0xe84280
+#define regTRAP8_CONTROL0_BASE_IDX 8
+#define regTRAP8_ADDRESS_LO                                                                             0xe84282
+#define regTRAP8_ADDRESS_LO_BASE_IDX 8
+#define regTRAP8_ADDRESS_HI                                                                             0xe84283
+#define regTRAP8_ADDRESS_HI_BASE_IDX 8
+#define regTRAP8_COMMAND                                                                                0xe84284
+#define regTRAP8_COMMAND_BASE_IDX 8
+#define regTRAP8_ADDRESS_LO_MASK                                                                        0xe84286
+#define regTRAP8_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP8_ADDRESS_HI_MASK                                                                        0xe84287
+#define regTRAP8_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP8_COMMAND_MASK                                                                           0xe84288
+#define regTRAP8_COMMAND_MASK_BASE_IDX 8
+#define regTRAP9_CONTROL0                                                                               0xe84290
+#define regTRAP9_CONTROL0_BASE_IDX 8
+#define regTRAP9_ADDRESS_LO                                                                             0xe84292
+#define regTRAP9_ADDRESS_LO_BASE_IDX 8
+#define regTRAP9_ADDRESS_HI                                                                             0xe84293
+#define regTRAP9_ADDRESS_HI_BASE_IDX 8
+#define regTRAP9_COMMAND                                                                                0xe84294
+#define regTRAP9_COMMAND_BASE_IDX 8
+#define regTRAP9_ADDRESS_LO_MASK                                                                        0xe84296
+#define regTRAP9_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP9_ADDRESS_HI_MASK                                                                        0xe84297
+#define regTRAP9_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP9_COMMAND_MASK                                                                           0xe84298
+#define regTRAP9_COMMAND_MASK_BASE_IDX 8
+#define regTRAP10_CONTROL0                                                                              0xe842a0
+#define regTRAP10_CONTROL0_BASE_IDX 8
+#define regTRAP10_ADDRESS_LO                                                                            0xe842a2
+#define regTRAP10_ADDRESS_LO_BASE_IDX 8
+#define regTRAP10_ADDRESS_HI                                                                            0xe842a3
+#define regTRAP10_ADDRESS_HI_BASE_IDX 8
+#define regTRAP10_COMMAND                                                                               0xe842a4
+#define regTRAP10_COMMAND_BASE_IDX 8
+#define regTRAP10_ADDRESS_LO_MASK                                                                       0xe842a6
+#define regTRAP10_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP10_ADDRESS_HI_MASK                                                                       0xe842a7
+#define regTRAP10_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP10_COMMAND_MASK                                                                          0xe842a8
+#define regTRAP10_COMMAND_MASK_BASE_IDX 8
+#define regTRAP11_CONTROL0                                                                              0xe842b0
+#define regTRAP11_CONTROL0_BASE_IDX 8
+#define regTRAP11_ADDRESS_LO                                                                            0xe842b2
+#define regTRAP11_ADDRESS_LO_BASE_IDX 8
+#define regTRAP11_ADDRESS_HI                                                                            0xe842b3
+#define regTRAP11_ADDRESS_HI_BASE_IDX 8
+#define regTRAP11_COMMAND                                                                               0xe842b4
+#define regTRAP11_COMMAND_BASE_IDX 8
+#define regTRAP11_ADDRESS_LO_MASK                                                                       0xe842b6
+#define regTRAP11_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP11_ADDRESS_HI_MASK                                                                       0xe842b7
+#define regTRAP11_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP11_COMMAND_MASK                                                                          0xe842b8
+#define regTRAP11_COMMAND_MASK_BASE_IDX 8
+#define regTRAP12_CONTROL0                                                                              0xe842c0
+#define regTRAP12_CONTROL0_BASE_IDX 8
+#define regTRAP12_ADDRESS_LO                                                                            0xe842c2
+#define regTRAP12_ADDRESS_LO_BASE_IDX 8
+#define regTRAP12_ADDRESS_HI                                                                            0xe842c3
+#define regTRAP12_ADDRESS_HI_BASE_IDX 8
+#define regTRAP12_COMMAND                                                                               0xe842c4
+#define regTRAP12_COMMAND_BASE_IDX 8
+#define regTRAP12_ADDRESS_LO_MASK                                                                       0xe842c6
+#define regTRAP12_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP12_ADDRESS_HI_MASK                                                                       0xe842c7
+#define regTRAP12_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP12_COMMAND_MASK                                                                          0xe842c8
+#define regTRAP12_COMMAND_MASK_BASE_IDX 8
+#define regTRAP13_CONTROL0                                                                              0xe842d0
+#define regTRAP13_CONTROL0_BASE_IDX 8
+#define regTRAP13_ADDRESS_LO                                                                            0xe842d2
+#define regTRAP13_ADDRESS_LO_BASE_IDX 8
+#define regTRAP13_ADDRESS_HI                                                                            0xe842d3
+#define regTRAP13_ADDRESS_HI_BASE_IDX 8
+#define regTRAP13_COMMAND                                                                               0xe842d4
+#define regTRAP13_COMMAND_BASE_IDX 8
+#define regTRAP13_ADDRESS_LO_MASK                                                                       0xe842d6
+#define regTRAP13_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP13_ADDRESS_HI_MASK                                                                       0xe842d7
+#define regTRAP13_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP13_COMMAND_MASK                                                                          0xe842d8
+#define regTRAP13_COMMAND_MASK_BASE_IDX 8
+#define regTRAP14_CONTROL0                                                                              0xe842e0
+#define regTRAP14_CONTROL0_BASE_IDX 8
+#define regTRAP14_ADDRESS_LO                                                                            0xe842e2
+#define regTRAP14_ADDRESS_LO_BASE_IDX 8
+#define regTRAP14_ADDRESS_HI                                                                            0xe842e3
+#define regTRAP14_ADDRESS_HI_BASE_IDX 8
+#define regTRAP14_COMMAND                                                                               0xe842e4
+#define regTRAP14_COMMAND_BASE_IDX 8
+#define regTRAP14_ADDRESS_LO_MASK                                                                       0xe842e6
+#define regTRAP14_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP14_ADDRESS_HI_MASK                                                                       0xe842e7
+#define regTRAP14_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP14_COMMAND_MASK                                                                          0xe842e8
+#define regTRAP14_COMMAND_MASK_BASE_IDX 8
+#define regTRAP15_CONTROL0                                                                              0xe842f0
+#define regTRAP15_CONTROL0_BASE_IDX 8
+#define regTRAP15_ADDRESS_LO                                                                            0xe842f2
+#define regTRAP15_ADDRESS_LO_BASE_IDX 8
+#define regTRAP15_ADDRESS_HI                                                                            0xe842f3
+#define regTRAP15_ADDRESS_HI_BASE_IDX 8
+#define regTRAP15_COMMAND                                                                               0xe842f4
+#define regTRAP15_COMMAND_BASE_IDX 8
+#define regTRAP15_ADDRESS_LO_MASK                                                                       0xe842f6
+#define regTRAP15_ADDRESS_LO_MASK_BASE_IDX 8
+#define regTRAP15_ADDRESS_HI_MASK                                                                       0xe842f7
+#define regTRAP15_ADDRESS_HI_MASK_BASE_IDX 8
+#define regTRAP15_COMMAND_MASK                                                                          0xe842f8
+#define regTRAP15_COMMAND_MASK_BASE_IDX 8
+#define regSB_COMMAND                                                                                   0xe85000
+#define regSB_COMMAND_BASE_IDX 8
+#define regSB_SUB_BUS_NUMBER_LATENCY                                                                    0xe85001
+#define regSB_SUB_BUS_NUMBER_LATENCY_BASE_IDX 8
+#define regSB_IO_BASE_LIMIT                                                                             0xe85002
+#define regSB_IO_BASE_LIMIT_BASE_IDX 8
+#define regSB_MEM_BASE_LIMIT                                                                            0xe85003
+#define regSB_MEM_BASE_LIMIT_BASE_IDX 8
+#define regSB_PREF_BASE_LIMIT                                                                           0xe85004
+#define regSB_PREF_BASE_LIMIT_BASE_IDX 8
+#define regSB_PREF_BASE_UPPER                                                                           0xe85005
+#define regSB_PREF_BASE_UPPER_BASE_IDX 8
+#define regSB_PREF_LIMIT_UPPER                                                                          0xe85006
+#define regSB_PREF_LIMIT_UPPER_BASE_IDX 8
+#define regSB_IO_BASE_LIMIT_HI                                                                          0xe85007
+#define regSB_IO_BASE_LIMIT_HI_BASE_IDX 8
+#define regSB_IRQ_BRIDGE_CNTL                                                                           0xe85008
+#define regSB_IRQ_BRIDGE_CNTL_BASE_IDX 8
+#define regSB_EXT_BRIDGE_CNTL                                                                           0xe85009
+#define regSB_EXT_BRIDGE_CNTL_BASE_IDX 8
+#define regSB_PMI_STATUS_CNTL                                                                           0xe8500a
+#define regSB_PMI_STATUS_CNTL_BASE_IDX 8
+#define regSB_SLOT_CAP                                                                                  0xe8500b
+#define regSB_SLOT_CAP_BASE_IDX 8
+#define regSB_ROOT_CNTL                                                                                 0xe8500c
+#define regSB_ROOT_CNTL_BASE_IDX 8
+#define regSB_DEVICE_CNTL2                                                                              0xe8500d
+#define regSB_DEVICE_CNTL2_BASE_IDX 8
+#define regMCA_SMN_INT_REQ_ADDR                                                                         0xe85020
+#define regMCA_SMN_INT_REQ_ADDR_BASE_IDX 8
+#define regMCA_SMN_INT_MCM_ADDR                                                                         0xe85021
+#define regMCA_SMN_INT_MCM_ADDR_BASE_IDX 8
+#define regMCA_SMN_INT_APERTUREID                                                                       0xe85022
+#define regMCA_SMN_INT_APERTUREID_BASE_IDX 8
+#define regMCA_SMN_INT_CONTROL                                                                          0xe85023
+#define regMCA_SMN_INT_CONTROL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_rascfg_ras_cfgdec
+// base address: 0x13b20000
+#define regPARITY_CONTROL_0                                                                             0xe88000
+#define regPARITY_CONTROL_0_BASE_IDX 8
+#define regPARITY_CONTROL_1                                                                             0xe88001
+#define regPARITY_CONTROL_1_BASE_IDX 8
+#define regPARITY_SEVERITY_CONTROL_UNCORR_0                                                             0xe88002
+#define regPARITY_SEVERITY_CONTROL_UNCORR_0_BASE_IDX 8
+#define regPARITY_SEVERITY_CONTROL_CORR_0                                                               0xe88004
+#define regPARITY_SEVERITY_CONTROL_CORR_0_BASE_IDX 8
+#define regPARITY_SEVERITY_CONTROL_UCP_0                                                                0xe88006
+#define regPARITY_SEVERITY_CONTROL_UCP_0_BASE_IDX 8
+#define regRAS_GLOBAL_STATUS_LO                                                                         0xe88008
+#define regRAS_GLOBAL_STATUS_LO_BASE_IDX 8
+#define regRAS_GLOBAL_STATUS_HI                                                                         0xe88009
+#define regRAS_GLOBAL_STATUS_HI_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP0                                                              0xe8800a
+#define regPARITY_ERROR_STATUS_UNCORR_GRP0_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP1                                                              0xe8800b
+#define regPARITY_ERROR_STATUS_UNCORR_GRP1_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP2                                                              0xe8800c
+#define regPARITY_ERROR_STATUS_UNCORR_GRP2_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP3                                                              0xe8800d
+#define regPARITY_ERROR_STATUS_UNCORR_GRP3_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP4                                                              0xe8800e
+#define regPARITY_ERROR_STATUS_UNCORR_GRP4_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP5                                                              0xe8800f
+#define regPARITY_ERROR_STATUS_UNCORR_GRP5_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP6                                                              0xe88010
+#define regPARITY_ERROR_STATUS_UNCORR_GRP6_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP7                                                              0xe88011
+#define regPARITY_ERROR_STATUS_UNCORR_GRP7_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP10                                                             0xe88014
+#define regPARITY_ERROR_STATUS_UNCORR_GRP10_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP11                                                             0xe88015
+#define regPARITY_ERROR_STATUS_UNCORR_GRP11_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP12                                                             0xe88016
+#define regPARITY_ERROR_STATUS_UNCORR_GRP12_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP13                                                             0xe88017
+#define regPARITY_ERROR_STATUS_UNCORR_GRP13_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP14                                                             0xe88018
+#define regPARITY_ERROR_STATUS_UNCORR_GRP14_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP15                                                             0xe88019
+#define regPARITY_ERROR_STATUS_UNCORR_GRP15_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UNCORR_GRP16                                                             0xe8801a
+#define regPARITY_ERROR_STATUS_UNCORR_GRP16_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP0                                                                0xe8801b
+#define regPARITY_ERROR_STATUS_CORR_GRP0_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP1                                                                0xe8801c
+#define regPARITY_ERROR_STATUS_CORR_GRP1_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP2                                                                0xe8801d
+#define regPARITY_ERROR_STATUS_CORR_GRP2_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP3                                                                0xe8801e
+#define regPARITY_ERROR_STATUS_CORR_GRP3_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP4                                                                0xe8801f
+#define regPARITY_ERROR_STATUS_CORR_GRP4_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP5                                                                0xe88020
+#define regPARITY_ERROR_STATUS_CORR_GRP5_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP6                                                                0xe88021
+#define regPARITY_ERROR_STATUS_CORR_GRP6_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP7                                                                0xe88022
+#define regPARITY_ERROR_STATUS_CORR_GRP7_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP10                                                               0xe88025
+#define regPARITY_ERROR_STATUS_CORR_GRP10_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP11                                                               0xe88026
+#define regPARITY_ERROR_STATUS_CORR_GRP11_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP12                                                               0xe88027
+#define regPARITY_ERROR_STATUS_CORR_GRP12_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP13                                                               0xe88028
+#define regPARITY_ERROR_STATUS_CORR_GRP13_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP14                                                               0xe88029
+#define regPARITY_ERROR_STATUS_CORR_GRP14_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP15                                                               0xe8802a
+#define regPARITY_ERROR_STATUS_CORR_GRP15_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP16                                                               0xe8802b
+#define regPARITY_ERROR_STATUS_CORR_GRP16_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_CORR_GRP17                                                               0xe8802c
+#define regPARITY_ERROR_STATUS_CORR_GRP17_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP0                                                                     0xe8802d
+#define regPARITY_COUNTER_CORR_GRP0_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP1                                                                     0xe8802e
+#define regPARITY_COUNTER_CORR_GRP1_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP2                                                                     0xe8802f
+#define regPARITY_COUNTER_CORR_GRP2_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP3                                                                     0xe88030
+#define regPARITY_COUNTER_CORR_GRP3_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP4                                                                     0xe88031
+#define regPARITY_COUNTER_CORR_GRP4_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP5                                                                     0xe88032
+#define regPARITY_COUNTER_CORR_GRP5_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP6                                                                     0xe88033
+#define regPARITY_COUNTER_CORR_GRP6_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP7                                                                     0xe88034
+#define regPARITY_COUNTER_CORR_GRP7_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP10                                                                    0xe88037
+#define regPARITY_COUNTER_CORR_GRP10_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP11                                                                    0xe88038
+#define regPARITY_COUNTER_CORR_GRP11_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP12                                                                    0xe88039
+#define regPARITY_COUNTER_CORR_GRP12_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP13                                                                    0xe8803a
+#define regPARITY_COUNTER_CORR_GRP13_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP14                                                                    0xe8803b
+#define regPARITY_COUNTER_CORR_GRP14_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP15                                                                    0xe8803c
+#define regPARITY_COUNTER_CORR_GRP15_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP16                                                                    0xe8803d
+#define regPARITY_COUNTER_CORR_GRP16_BASE_IDX 8
+#define regPARITY_COUNTER_CORR_GRP17                                                                    0xe8803e
+#define regPARITY_COUNTER_CORR_GRP17_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP0                                                                 0xe8803f
+#define regPARITY_ERROR_STATUS_UCP_GRP0_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP1                                                                 0xe88040
+#define regPARITY_ERROR_STATUS_UCP_GRP1_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP2                                                                 0xe88041
+#define regPARITY_ERROR_STATUS_UCP_GRP2_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP3                                                                 0xe88042
+#define regPARITY_ERROR_STATUS_UCP_GRP3_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP4                                                                 0xe88043
+#define regPARITY_ERROR_STATUS_UCP_GRP4_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP5                                                                 0xe88044
+#define regPARITY_ERROR_STATUS_UCP_GRP5_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP6                                                                 0xe88045
+#define regPARITY_ERROR_STATUS_UCP_GRP6_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP7                                                                 0xe88046
+#define regPARITY_ERROR_STATUS_UCP_GRP7_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP10                                                                0xe88049
+#define regPARITY_ERROR_STATUS_UCP_GRP10_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP11                                                                0xe8804a
+#define regPARITY_ERROR_STATUS_UCP_GRP11_BASE_IDX 8
+#define regPARITY_ERROR_STATUS_UCP_GRP12                                                                0xe8804b
+#define regPARITY_ERROR_STATUS_UCP_GRP12_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP0                                                                      0xe8804c
+#define regPARITY_COUNTER_UCP_GRP0_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP1                                                                      0xe8804d
+#define regPARITY_COUNTER_UCP_GRP1_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP2                                                                      0xe8804e
+#define regPARITY_COUNTER_UCP_GRP2_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP3                                                                      0xe8804f
+#define regPARITY_COUNTER_UCP_GRP3_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP4                                                                      0xe88050
+#define regPARITY_COUNTER_UCP_GRP4_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP5                                                                      0xe88051
+#define regPARITY_COUNTER_UCP_GRP5_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP6                                                                      0xe88052
+#define regPARITY_COUNTER_UCP_GRP6_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP7                                                                      0xe88053
+#define regPARITY_COUNTER_UCP_GRP7_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP10                                                                     0xe88056
+#define regPARITY_COUNTER_UCP_GRP10_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP11                                                                     0xe88057
+#define regPARITY_COUNTER_UCP_GRP11_BASE_IDX 8
+#define regPARITY_COUNTER_UCP_GRP12                                                                     0xe88058
+#define regPARITY_COUNTER_UCP_GRP12_BASE_IDX 8
+#define regMISC_SEVERITY_CONTROL                                                                        0xe88059
+#define regMISC_SEVERITY_CONTROL_BASE_IDX 8
+#define regMISC_RAS_CONTROL                                                                             0xe8805a
+#define regMISC_RAS_CONTROL_BASE_IDX 8
+#define regRAS_SCRATCH_0                                                                                0xe8805b
+#define regRAS_SCRATCH_0_BASE_IDX 8
+#define regRAS_SCRATCH_1                                                                                0xe8805c
+#define regRAS_SCRATCH_1_BASE_IDX 8
+#define regErrEvent_ACTION_CONTROL                                                                      0xe8805d
+#define regErrEvent_ACTION_CONTROL_BASE_IDX 8
+#define regParitySerr_ACTION_CONTROL                                                                    0xe8805e
+#define regParitySerr_ACTION_CONTROL_BASE_IDX 8
+#define regParityFatal_ACTION_CONTROL                                                                   0xe8805f
+#define regParityFatal_ACTION_CONTROL_BASE_IDX 8
+#define regParityNonFatal_ACTION_CONTROL                                                                0xe88060
+#define regParityNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regParityCorr_ACTION_CONTROL                                                                    0xe88061
+#define regParityCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortASerr_ACTION_CONTROL                                                                0xe88062
+#define regPCIE0PortASerr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortAIntFatal_ACTION_CONTROL                                                            0xe88063
+#define regPCIE0PortAIntFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortAIntNonFatal_ACTION_CONTROL                                                         0xe88064
+#define regPCIE0PortAIntNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortAIntCorr_ACTION_CONTROL                                                             0xe88065
+#define regPCIE0PortAIntCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortAExtFatal_ACTION_CONTROL                                                            0xe88066
+#define regPCIE0PortAExtFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortAExtNonFatal_ACTION_CONTROL                                                         0xe88067
+#define regPCIE0PortAExtNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortAExtCorr_ACTION_CONTROL                                                             0xe88068
+#define regPCIE0PortAExtCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortAParityErr_ACTION_CONTROL                                                           0xe88069
+#define regPCIE0PortAParityErr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortBSerr_ACTION_CONTROL                                                                0xe8806a
+#define regPCIE0PortBSerr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortBIntFatal_ACTION_CONTROL                                                            0xe8806b
+#define regPCIE0PortBIntFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortBIntNonFatal_ACTION_CONTROL                                                         0xe8806c
+#define regPCIE0PortBIntNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortBIntCorr_ACTION_CONTROL                                                             0xe8806d
+#define regPCIE0PortBIntCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortBExtFatal_ACTION_CONTROL                                                            0xe8806e
+#define regPCIE0PortBExtFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortBExtNonFatal_ACTION_CONTROL                                                         0xe8806f
+#define regPCIE0PortBExtNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortBExtCorr_ACTION_CONTROL                                                             0xe88070
+#define regPCIE0PortBExtCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortBParityErr_ACTION_CONTROL                                                           0xe88071
+#define regPCIE0PortBParityErr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortCSerr_ACTION_CONTROL                                                                0xe88072
+#define regPCIE0PortCSerr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortCIntFatal_ACTION_CONTROL                                                            0xe88073
+#define regPCIE0PortCIntFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortCIntNonFatal_ACTION_CONTROL                                                         0xe88074
+#define regPCIE0PortCIntNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortCIntCorr_ACTION_CONTROL                                                             0xe88075
+#define regPCIE0PortCIntCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortCExtFatal_ACTION_CONTROL                                                            0xe88076
+#define regPCIE0PortCExtFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortCExtNonFatal_ACTION_CONTROL                                                         0xe88077
+#define regPCIE0PortCExtNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortCExtCorr_ACTION_CONTROL                                                             0xe88078
+#define regPCIE0PortCExtCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortCParityErr_ACTION_CONTROL                                                           0xe88079
+#define regPCIE0PortCParityErr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortDSerr_ACTION_CONTROL                                                                0xe8807a
+#define regPCIE0PortDSerr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortDIntFatal_ACTION_CONTROL                                                            0xe8807b
+#define regPCIE0PortDIntFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortDIntNonFatal_ACTION_CONTROL                                                         0xe8807c
+#define regPCIE0PortDIntNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortDIntCorr_ACTION_CONTROL                                                             0xe8807d
+#define regPCIE0PortDIntCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortDExtFatal_ACTION_CONTROL                                                            0xe8807e
+#define regPCIE0PortDExtFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortDExtNonFatal_ACTION_CONTROL                                                         0xe8807f
+#define regPCIE0PortDExtNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortDExtCorr_ACTION_CONTROL                                                             0xe88080
+#define regPCIE0PortDExtCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortDParityErr_ACTION_CONTROL                                                           0xe88081
+#define regPCIE0PortDParityErr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortESerr_ACTION_CONTROL                                                                0xe88082
+#define regPCIE0PortESerr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortEIntFatal_ACTION_CONTROL                                                            0xe88083
+#define regPCIE0PortEIntFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortEIntNonFatal_ACTION_CONTROL                                                         0xe88084
+#define regPCIE0PortEIntNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortEIntCorr_ACTION_CONTROL                                                             0xe88085
+#define regPCIE0PortEIntCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortEExtFatal_ACTION_CONTROL                                                            0xe88086
+#define regPCIE0PortEExtFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortEExtNonFatal_ACTION_CONTROL                                                         0xe88087
+#define regPCIE0PortEExtNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortEExtCorr_ACTION_CONTROL                                                             0xe88088
+#define regPCIE0PortEExtCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortEParityErr_ACTION_CONTROL                                                           0xe88089
+#define regPCIE0PortEParityErr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortFSerr_ACTION_CONTROL                                                                0xe8808a
+#define regPCIE0PortFSerr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortFIntFatal_ACTION_CONTROL                                                            0xe8808b
+#define regPCIE0PortFIntFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortFIntNonFatal_ACTION_CONTROL                                                         0xe8808c
+#define regPCIE0PortFIntNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortFIntCorr_ACTION_CONTROL                                                             0xe8808d
+#define regPCIE0PortFIntCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortFExtFatal_ACTION_CONTROL                                                            0xe8808e
+#define regPCIE0PortFExtFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortFExtNonFatal_ACTION_CONTROL                                                         0xe8808f
+#define regPCIE0PortFExtNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortFExtCorr_ACTION_CONTROL                                                             0xe88090
+#define regPCIE0PortFExtCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortFParityErr_ACTION_CONTROL                                                           0xe88091
+#define regPCIE0PortFParityErr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortGSerr_ACTION_CONTROL                                                                0xe88092
+#define regPCIE0PortGSerr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortGIntFatal_ACTION_CONTROL                                                            0xe88093
+#define regPCIE0PortGIntFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortGIntNonFatal_ACTION_CONTROL                                                         0xe88094
+#define regPCIE0PortGIntNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortGIntCorr_ACTION_CONTROL                                                             0xe88095
+#define regPCIE0PortGIntCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortGExtFatal_ACTION_CONTROL                                                            0xe88096
+#define regPCIE0PortGExtFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortGExtNonFatal_ACTION_CONTROL                                                         0xe88097
+#define regPCIE0PortGExtNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortGExtCorr_ACTION_CONTROL                                                             0xe88098
+#define regPCIE0PortGExtCorr_ACTION_CONTROL_BASE_IDX 8
+#define regPCIE0PortGParityErr_ACTION_CONTROL                                                           0xe88099
+#define regPCIE0PortGParityErr_ACTION_CONTROL_BASE_IDX 8
+#define regNBIF1PortASerr_ACTION_CONTROL                                                                0xe880ca
+#define regNBIF1PortASerr_ACTION_CONTROL_BASE_IDX 8
+#define regNBIF1PortAIntFatal_ACTION_CONTROL                                                            0xe880cb
+#define regNBIF1PortAIntFatal_ACTION_CONTROL_BASE_IDX 8
+#define regNBIF1PortAIntNonFatal_ACTION_CONTROL                                                         0xe880cc
+#define regNBIF1PortAIntNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regNBIF1PortAIntCorr_ACTION_CONTROL                                                             0xe880cd
+#define regNBIF1PortAIntCorr_ACTION_CONTROL_BASE_IDX 8
+#define regNBIF1PortAExtFatal_ACTION_CONTROL                                                            0xe880ce
+#define regNBIF1PortAExtFatal_ACTION_CONTROL_BASE_IDX 8
+#define regNBIF1PortAExtNonFatal_ACTION_CONTROL                                                         0xe880cf
+#define regNBIF1PortAExtNonFatal_ACTION_CONTROL_BASE_IDX 8
+#define regNBIF1PortAExtCorr_ACTION_CONTROL                                                             0xe880d0
+#define regNBIF1PortAExtCorr_ACTION_CONTROL_BASE_IDX 8
+#define regNBIF1PortAParityErr_ACTION_CONTROL                                                           0xe880d1
+#define regNBIF1PortAParityErr_ACTION_CONTROL_BASE_IDX 8
+#define regSYNCFLOOD_STATUS                                                                             0xe88200
+#define regSYNCFLOOD_STATUS_BASE_IDX 8
+#define regNMI_STATUS                                                                                   0xe88201
+#define regNMI_STATUS_BASE_IDX 8
+#define regPOISON_ACTION_CONTROL                                                                        0xe88205
+#define regPOISON_ACTION_CONTROL_BASE_IDX 8
+#define regINTERNAL_POISON_STATUS                                                                       0xe88206
+#define regINTERNAL_POISON_STATUS_BASE_IDX 8
+#define regINTERNAL_POISON_MASK                                                                         0xe88207
+#define regINTERNAL_POISON_MASK_BASE_IDX 8
+#define regEGRESS_POISON_STATUS_LO                                                                      0xe88208
+#define regEGRESS_POISON_STATUS_LO_BASE_IDX 8
+#define regEGRESS_POISON_STATUS_HI                                                                      0xe88209
+#define regEGRESS_POISON_STATUS_HI_BASE_IDX 8
+#define regEGRESS_POISON_MASK_LO                                                                        0xe8820a
+#define regEGRESS_POISON_MASK_LO_BASE_IDX 8
+#define regEGRESS_POISON_MASK_HI                                                                        0xe8820b
+#define regEGRESS_POISON_MASK_HI_BASE_IDX 8
+#define regEGRESS_POISON_SEVERITY_DOWN                                                                  0xe8820c
+#define regEGRESS_POISON_SEVERITY_DOWN_BASE_IDX 8
+#define regEGRESS_POISON_SEVERITY_UPPER                                                                 0xe8820d
+#define regEGRESS_POISON_SEVERITY_UPPER_BASE_IDX 8
+#define regAPML_STATUS                                                                                  0xe88370
+#define regAPML_STATUS_BASE_IDX 8
+#define regAPML_CONTROL                                                                                 0xe88371
+#define regAPML_CONTROL_BASE_IDX 8
+#define regAPML_TRIGGER                                                                                 0xe88372
+#define regAPML_TRIGGER_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg0_devind_cfgdecp
+// base address: 0x13b31000
+#define regNB_PCIE0DEVINDCFG0_STEERING_CNTL                                                             0xe8c403
+#define regNB_PCIE0DEVINDCFG0_STEERING_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg1_devind_cfgdecp
+// base address: 0x13b31400
+#define regNB_PCIE0DEVINDCFG1_STEERING_CNTL                                                             0xe8c503
+#define regNB_PCIE0DEVINDCFG1_STEERING_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg2_devind_cfgdecp
+// base address: 0x13b31800
+#define regNB_PCIE0DEVINDCFG2_STEERING_CNTL                                                             0xe8c603
+#define regNB_PCIE0DEVINDCFG2_STEERING_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg3_devind_cfgdecp
+// base address: 0x13b31c00
+#define regNB_PCIE0DEVINDCFG3_STEERING_CNTL                                                             0xe8c703
+#define regNB_PCIE0DEVINDCFG3_STEERING_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg4_devind_cfgdecp
+// base address: 0x13b32000
+#define regNB_PCIE0DEVINDCFG4_STEERING_CNTL                                                             0xe8c803
+#define regNB_PCIE0DEVINDCFG4_STEERING_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg5_devind_cfgdecp
+// base address: 0x13b32400
+#define regNB_PCIE0DEVINDCFG5_STEERING_CNTL                                                             0xe8c903
+#define regNB_PCIE0DEVINDCFG5_STEERING_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg6_devind_cfgdecp
+// base address: 0x13b32800
+#define regNB_PCIE0DEVINDCFG6_STEERING_CNTL                                                             0xe8ca03
+#define regNB_PCIE0DEVINDCFG6_STEERING_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_NBIF1devindcfg0_devind_cfgdecp
+// base address: 0x13b38000
+#define regNB_NBIF1DEVINDCFG0_STEERING_CNTL                                                             0xe8e003
+#define regNB_NBIF1DEVINDCFG0_STEERING_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_intSBdevindcfg0_devind_cfgdecp
+// base address: 0x13b3c000
+#define regNB_INTSBDEVINDCFG0_STEERING_CNTL                                                             0xe8f003
+#define regNB_INTSBDEVINDCFG0_STEERING_CNTL_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg0_pciercbdgind_cfgdec
+// base address: 0x13b7d600
+#define regNB_PCIE0RCBDG_INDCFG0_RC_SMN_INDEX_EXTENSION                                                 0xe9f5b7
+#define regNB_PCIE0RCBDG_INDCFG0_RC_SMN_INDEX_EXTENSION_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG0_RC_SMN_INDEX                                                           0xe9f5b8
+#define regNB_PCIE0RCBDG_INDCFG0_RC_SMN_INDEX_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG0_RC_SMN_DATA                                                            0xe9f5b9
+#define regNB_PCIE0RCBDG_INDCFG0_RC_SMN_DATA_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg1_pciercbdgind_cfgdec
+// base address: 0x13b7d700
+#define regNB_PCIE0RCBDG_INDCFG1_RC_SMN_INDEX_EXTENSION                                                 0xe9f5f7
+#define regNB_PCIE0RCBDG_INDCFG1_RC_SMN_INDEX_EXTENSION_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG1_RC_SMN_INDEX                                                           0xe9f5f8
+#define regNB_PCIE0RCBDG_INDCFG1_RC_SMN_INDEX_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG1_RC_SMN_DATA                                                            0xe9f5f9
+#define regNB_PCIE0RCBDG_INDCFG1_RC_SMN_DATA_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg2_pciercbdgind_cfgdec
+// base address: 0x13b7d800
+#define regNB_PCIE0RCBDG_INDCFG2_RC_SMN_INDEX_EXTENSION                                                 0xe9f637
+#define regNB_PCIE0RCBDG_INDCFG2_RC_SMN_INDEX_EXTENSION_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG2_RC_SMN_INDEX                                                           0xe9f638
+#define regNB_PCIE0RCBDG_INDCFG2_RC_SMN_INDEX_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG2_RC_SMN_DATA                                                            0xe9f639
+#define regNB_PCIE0RCBDG_INDCFG2_RC_SMN_DATA_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg3_pciercbdgind_cfgdec
+// base address: 0x13b7d900
+#define regNB_PCIE0RCBDG_INDCFG3_RC_SMN_INDEX_EXTENSION                                                 0xe9f677
+#define regNB_PCIE0RCBDG_INDCFG3_RC_SMN_INDEX_EXTENSION_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG3_RC_SMN_INDEX                                                           0xe9f678
+#define regNB_PCIE0RCBDG_INDCFG3_RC_SMN_INDEX_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG3_RC_SMN_DATA                                                            0xe9f679
+#define regNB_PCIE0RCBDG_INDCFG3_RC_SMN_DATA_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg4_pciercbdgind_cfgdec
+// base address: 0x13b7da00
+#define regNB_PCIE0RCBDG_INDCFG4_RC_SMN_INDEX_EXTENSION                                                 0xe9f6b7
+#define regNB_PCIE0RCBDG_INDCFG4_RC_SMN_INDEX_EXTENSION_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG4_RC_SMN_INDEX                                                           0xe9f6b8
+#define regNB_PCIE0RCBDG_INDCFG4_RC_SMN_INDEX_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG4_RC_SMN_DATA                                                            0xe9f6b9
+#define regNB_PCIE0RCBDG_INDCFG4_RC_SMN_DATA_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg5_pciercbdgind_cfgdec
+// base address: 0x13b7db00
+#define regNB_PCIE0RCBDG_INDCFG5_RC_SMN_INDEX_EXTENSION                                                 0xe9f6f7
+#define regNB_PCIE0RCBDG_INDCFG5_RC_SMN_INDEX_EXTENSION_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG5_RC_SMN_INDEX                                                           0xe9f6f8
+#define regNB_PCIE0RCBDG_INDCFG5_RC_SMN_INDEX_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG5_RC_SMN_DATA                                                            0xe9f6f9
+#define regNB_PCIE0RCBDG_INDCFG5_RC_SMN_DATA_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg6_pciercbdgind_cfgdec
+// base address: 0x13b7dc00
+#define regNB_PCIE0RCBDG_INDCFG6_RC_SMN_INDEX_EXTENSION                                                 0xe9f737
+#define regNB_PCIE0RCBDG_INDCFG6_RC_SMN_INDEX_EXTENSION_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG6_RC_SMN_INDEX                                                           0xe9f738
+#define regNB_PCIE0RCBDG_INDCFG6_RC_SMN_INDEX_BASE_IDX 8
+#define regNB_PCIE0RCBDG_INDCFG6_RC_SMN_DATA                                                            0xe9f739
+#define regNB_PCIE0RCBDG_INDCFG6_RC_SMN_DATA_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_NBIF1rcbdg_indcfg0_pciercbdgind_cfgdec
+// base address: 0x13b7f200
+#define regNB_NBIF1RCBDG_INDCFG0_RC_SMN_INDEX_EXTENSION                                                 0xe9fcb7
+#define regNB_NBIF1RCBDG_INDCFG0_RC_SMN_INDEX_EXTENSION_BASE_IDX 8
+#define regNB_NBIF1RCBDG_INDCFG0_RC_SMN_INDEX                                                           0xe9fcb8
+#define regNB_NBIF1RCBDG_INDCFG0_RC_SMN_INDEX_BASE_IDX 8
+#define regNB_NBIF1RCBDG_INDCFG0_RC_SMN_DATA                                                            0xe9fcb9
+#define regNB_NBIF1RCBDG_INDCFG0_RC_SMN_DATA_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_iommu_l2a_l2acfg
+// base address: 0x15700000
+#define regL2_PERF_CNTL_0                                                                               0x1580000
+#define regL2_PERF_CNTL_0_BASE_IDX 8
+#define regL2_PERF_COUNT_0                                                                              0x1580001
+#define regL2_PERF_COUNT_0_BASE_IDX 8
+#define regL2_PERF_COUNT_1                                                                              0x1580002
+#define regL2_PERF_COUNT_1_BASE_IDX 8
+#define regL2_PERF_CNTL_1                                                                               0x1580003
+#define regL2_PERF_CNTL_1_BASE_IDX 8
+#define regL2_PERF_COUNT_2                                                                              0x1580004
+#define regL2_PERF_COUNT_2_BASE_IDX 8
+#define regL2_PERF_COUNT_3                                                                              0x1580005
+#define regL2_PERF_COUNT_3_BASE_IDX 8
+#define regL2_STATUS_0                                                                                  0x1580008
+#define regL2_STATUS_0_BASE_IDX 8
+#define regL2_CONTROL_0                                                                                 0x158000c
+#define regL2_CONTROL_0_BASE_IDX 8
+#define regL2_CONTROL_1                                                                                 0x158000d
+#define regL2_CONTROL_1_BASE_IDX 8
+#define regL2_DTC_CONTROL                                                                               0x1580010
+#define regL2_DTC_CONTROL_BASE_IDX 8
+#define regL2_DTC_HASH_CONTROL                                                                          0x1580011
+#define regL2_DTC_HASH_CONTROL_BASE_IDX 8
+#define regL2_DTC_WAY_CONTROL                                                                           0x1580012
+#define regL2_DTC_WAY_CONTROL_BASE_IDX 8
+#define regL2_ITC_CONTROL                                                                               0x1580014
+#define regL2_ITC_CONTROL_BASE_IDX 8
+#define regL2_ITC_HASH_CONTROL                                                                          0x1580015
+#define regL2_ITC_HASH_CONTROL_BASE_IDX 8
+#define regL2_ITC_WAY_CONTROL                                                                           0x1580016
+#define regL2_ITC_WAY_CONTROL_BASE_IDX 8
+#define regL2_PTC_A_CONTROL                                                                             0x1580018
+#define regL2_PTC_A_CONTROL_BASE_IDX 8
+#define regL2_PTC_A_HASH_CONTROL                                                                        0x1580019
+#define regL2_PTC_A_HASH_CONTROL_BASE_IDX 8
+#define regL2_PTC_A_WAY_CONTROL                                                                         0x158001a
+#define regL2_PTC_A_WAY_CONTROL_BASE_IDX 8
+#define regL2A_UPDATE_FILTER_CNTL                                                                       0x1580022
+#define regL2A_UPDATE_FILTER_CNTL_BASE_IDX 8
+#define regL2_ERR_RULE_CONTROL_3                                                                        0x1580030
+#define regL2_ERR_RULE_CONTROL_3_BASE_IDX 8
+#define regL2_ERR_RULE_CONTROL_4                                                                        0x1580031
+#define regL2_ERR_RULE_CONTROL_4_BASE_IDX 8
+#define regL2_ERR_RULE_CONTROL_5                                                                        0x1580032
+#define regL2_ERR_RULE_CONTROL_5_BASE_IDX 8
+#define regL2_L2A_CK_GATE_CONTROL                                                                       0x1580033
+#define regL2_L2A_CK_GATE_CONTROL_BASE_IDX 8
+#define regL2_L2A_PGSIZE_CONTROL                                                                        0x1580034
+#define regL2_L2A_PGSIZE_CONTROL_BASE_IDX 8
+#define regL2_PWRGATE_CNTRL_REG_0                                                                       0x158003e
+#define regL2_PWRGATE_CNTRL_REG_0_BASE_IDX 8
+#define regL2_PWRGATE_CNTRL_REG_3                                                                       0x1580041
+#define regL2_PWRGATE_CNTRL_REG_3_BASE_IDX 8
+#define regL2_ECO_CNTRL_0                                                                               0x1580042
+#define regL2_ECO_CNTRL_0_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_iommu_l2indx_l2indxcfg
+// base address: 0x13f01000
+#define regL2_STATUS_1                                                                                  0xf80448
+#define regL2_STATUS_1_BASE_IDX 8
+#define regL2_SB_LOCATION                                                                               0xf8044b
+#define regL2_SB_LOCATION_BASE_IDX 8
+#define regL2_CONTROL_5                                                                                 0xf8044c
+#define regL2_CONTROL_5_BASE_IDX 8
+#define regL2_CONTROL_6                                                                                 0xf8044f
+#define regL2_CONTROL_6_BASE_IDX 8
+#define regL2_PDC_CONTROL                                                                               0xf80450
+#define regL2_PDC_CONTROL_BASE_IDX 8
+#define regL2_PDC_HASH_CONTROL                                                                          0xf80451
+#define regL2_PDC_HASH_CONTROL_BASE_IDX 8
+#define regL2_PDC_WAY_CONTROL                                                                           0xf80452
+#define regL2_PDC_WAY_CONTROL_BASE_IDX 8
+#define regL2B_UPDATE_FILTER_CNTL                                                                       0xf80453
+#define regL2B_UPDATE_FILTER_CNTL_BASE_IDX 8
+#define regL2_TW_CONTROL                                                                                0xf80454
+#define regL2_TW_CONTROL_BASE_IDX 8
+#define regL2_CP_CONTROL                                                                                0xf80456
+#define regL2_CP_CONTROL_BASE_IDX 8
+#define regL2_CP_CONTROL_1                                                                              0xf80457
+#define regL2_CP_CONTROL_1_BASE_IDX 8
+#define regL2_TW_CONTROL_1                                                                              0xf8045a
+#define regL2_TW_CONTROL_1_BASE_IDX 8
+#define regL2_TW_CONTROL_2                                                                              0xf80461
+#define regL2_TW_CONTROL_2_BASE_IDX 8
+#define regL2_TW_CONTROL_3                                                                              0xf80462
+#define regL2_TW_CONTROL_3_BASE_IDX 8
+#define regL2_CREDIT_CONTROL_0                                                                          0xf80470
+#define regL2_CREDIT_CONTROL_0_BASE_IDX 8
+#define regL2_CREDIT_CONTROL_1                                                                          0xf80471
+#define regL2_CREDIT_CONTROL_1_BASE_IDX 8
+#define regL2_ERR_RULE_CONTROL_0                                                                        0xf80480
+#define regL2_ERR_RULE_CONTROL_0_BASE_IDX 8
+#define regL2_ERR_RULE_CONTROL_1                                                                        0xf80481
+#define regL2_ERR_RULE_CONTROL_1_BASE_IDX 8
+#define regL2_ERR_RULE_CONTROL_2                                                                        0xf80482
+#define regL2_ERR_RULE_CONTROL_2_BASE_IDX 8
+#define regL2_L2B_CK_GATE_CONTROL                                                                       0xf80490
+#define regL2_L2B_CK_GATE_CONTROL_BASE_IDX 8
+#define regPPR_CONTROL                                                                                  0xf80492
+#define regPPR_CONTROL_BASE_IDX 8
+#define regL2_L2B_PGSIZE_CONTROL                                                                        0xf80494
+#define regL2_L2B_PGSIZE_CONTROL_BASE_IDX 8
+#define regL2_PERF_CNTL_2                                                                               0xf80499
+#define regL2_PERF_CNTL_2_BASE_IDX 8
+#define regL2_PERF_COUNT_4                                                                              0xf8049a
+#define regL2_PERF_COUNT_4_BASE_IDX 8
+#define regL2_PERF_COUNT_5                                                                              0xf8049b
+#define regL2_PERF_COUNT_5_BASE_IDX 8
+#define regL2_PERF_CNTL_3                                                                               0xf8049c
+#define regL2_PERF_CNTL_3_BASE_IDX 8
+#define regL2_PERF_COUNT_6                                                                              0xf8049d
+#define regL2_PERF_COUNT_6_BASE_IDX 8
+#define regL2_PERF_COUNT_7                                                                              0xf8049e
+#define regL2_PERF_COUNT_7_BASE_IDX 8
+#define regL2B_SDP_PARITY_ERROR_EN                                                                      0xf804a2
+#define regL2B_SDP_PARITY_ERROR_EN_BASE_IDX 8
+#define regL2_ECO_CNTRL_1                                                                               0xf804a3
+#define regL2_ECO_CNTRL_1_BASE_IDX 8
+#define regL2_CP_CONTROL_2                                                                              0xf804bf
+#define regL2_CP_CONTROL_2_BASE_IDX 8
+#define regL2_CP_CONTROL_3                                                                              0xf804c0
+#define regL2_CP_CONTROL_3_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_iohub_nb_ioapiccfg_ioapic_cfgdec
+// base address: 0x14300000
+#define regFEATURES_ENABLE                                                                              0x1080000
+#define regFEATURES_ENABLE_BASE_IDX 8
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_rc_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_RC_VENDOR_ID                                                                    0x0000
+#define cfgBIF_CFG_DEV0_RC_DEVICE_ID                                                                    0x0002
+#define cfgBIF_CFG_DEV0_RC_COMMAND                                                                      0x0004
+#define cfgBIF_CFG_DEV0_RC_STATUS                                                                       0x0006
+#define cfgBIF_CFG_DEV0_RC_REVISION_ID                                                                  0x0008
+#define cfgBIF_CFG_DEV0_RC_PROG_INTERFACE                                                               0x0009
+#define cfgBIF_CFG_DEV0_RC_SUB_CLASS                                                                    0x000a
+#define cfgBIF_CFG_DEV0_RC_BASE_CLASS                                                                   0x000b
+#define cfgBIF_CFG_DEV0_RC_CACHE_LINE                                                                   0x000c
+#define cfgBIF_CFG_DEV0_RC_LATENCY                                                                      0x000d
+#define cfgBIF_CFG_DEV0_RC_HEADER                                                                       0x000e
+#define cfgBIF_CFG_DEV0_RC_BIST                                                                         0x000f
+#define cfgBIF_CFG_DEV0_RC_BASE_ADDR_1                                                                  0x0010
+#define cfgBIF_CFG_DEV0_RC_BASE_ADDR_2                                                                  0x0014
+#define cfgBIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY                                                       0x0018
+#define cfgBIF_CFG_DEV0_RC_IO_BASE_LIMIT                                                                0x001c
+#define cfgBIF_CFG_DEV0_RC_SECONDARY_STATUS                                                             0x001e
+#define cfgBIF_CFG_DEV0_RC_MEM_BASE_LIMIT                                                               0x0020
+#define cfgBIF_CFG_DEV0_RC_PREF_BASE_LIMIT                                                              0x0024
+#define cfgBIF_CFG_DEV0_RC_PREF_BASE_UPPER                                                              0x0028
+#define cfgBIF_CFG_DEV0_RC_PREF_LIMIT_UPPER                                                             0x002c
+#define cfgBIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI                                                             0x0030
+#define cfgBIF_CFG_DEV0_RC_CAP_PTR                                                                      0x0034
+#define cfgBIF_CFG_DEV0_RC_ROM_BASE_ADDR                                                                0x0038
+#define cfgBIF_CFG_DEV0_RC_INTERRUPT_LINE                                                               0x003c
+#define cfgBIF_CFG_DEV0_RC_INTERRUPT_PIN                                                                0x003d
+#define cfgIRQ_BRIDGE_CNTL                                                                              0x003e
+#define cfgBIF_CFG_DEV0_RC_EXT_BRIDGE_CNTL                                                              0x0040
+#define cfgBIF_CFG_DEV0_RC_PMI_CAP_LIST                                                                 0x0050
+#define cfgBIF_CFG_DEV0_RC_PMI_CAP                                                                      0x0052
+#define cfgBIF_CFG_DEV0_RC_PMI_STATUS_CNTL                                                              0x0054
+#define cfgBIF_CFG_DEV0_RC_PCIE_CAP_LIST                                                                0x0058
+#define cfgBIF_CFG_DEV0_RC_PCIE_CAP                                                                     0x005a
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CAP                                                                   0x005c
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CNTL                                                                  0x0060
+#define cfgBIF_CFG_DEV0_RC_DEVICE_STATUS                                                                0x0062
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP                                                                     0x0064
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL                                                                    0x0068
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS                                                                  0x006a
+#define cfgBIF_CFG_DEV0_RC_SLOT_CAP                                                                     0x006c
+#define cfgBIF_CFG_DEV0_RC_SLOT_CNTL                                                                    0x0070
+#define cfgBIF_CFG_DEV0_RC_SLOT_STATUS                                                                  0x0072
+#define cfgBIF_CFG_DEV0_RC_ROOT_CNTL                                                                    0x0074
+#define cfgBIF_CFG_DEV0_RC_ROOT_CAP                                                                     0x0076
+#define cfgBIF_CFG_DEV0_RC_ROOT_STATUS                                                                  0x0078
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CAP2                                                                  0x007c
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CNTL2                                                                 0x0080
+#define cfgBIF_CFG_DEV0_RC_DEVICE_STATUS2                                                               0x0082
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP2                                                                    0x0084
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL2                                                                   0x0088
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS2                                                                 0x008a
+#define cfgBIF_CFG_DEV0_RC_SLOT_CAP2                                                                    0x008c
+#define cfgBIF_CFG_DEV0_RC_SLOT_CNTL2                                                                   0x0090
+#define cfgBIF_CFG_DEV0_RC_SLOT_STATUS2                                                                 0x0092
+#define cfgBIF_CFG_DEV0_RC_MSI_CAP_LIST                                                                 0x00a0
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_CNTL                                                                 0x00a2
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO                                                              0x00a4
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI                                                              0x00a8
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_DATA                                                                 0x00a8
+#define cfgBIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA                                                             0x00aa
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_DATA_64                                                              0x00ac
+#define cfgBIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64                                                          0x00ae
+#define cfgBIF_CFG_DEV0_RC_SSID_CAP_LIST                                                                0x00c0
+#define cfgBIF_CFG_DEV0_RC_SSID_CAP                                                                     0x00c4
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                            0x0100
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR                                                     0x0104
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1                                                        0x0108
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2                                                        0x010c
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST                                                         0x0110
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1                                                        0x0114
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2                                                        0x0118
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL                                                            0x011c
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS                                                          0x011e
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP                                                        0x0120
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL                                                       0x0124
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS                                                     0x012a
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP                                                        0x012c
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL                                                       0x0130
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS                                                     0x0136
+#define cfgBIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                             0x0140
+#define cfgBIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1                                                      0x0144
+#define cfgBIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2                                                      0x0148
+#define cfgBIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                                0x0150
+#define cfgBIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS                                                       0x0154
+#define cfgBIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK                                                         0x0158
+#define cfgBIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY                                                     0x015c
+#define cfgBIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS                                                         0x0160
+#define cfgBIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK                                                           0x0164
+#define cfgBIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL                                                        0x0168
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG0                                                                0x016c
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG1                                                                0x0170
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG2                                                                0x0174
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG3                                                                0x0178
+#define cfgBIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD                                                            0x017c
+#define cfgBIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS                                                         0x0180
+#define cfgBIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID                                                              0x0184
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0                                                         0x0188
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1                                                         0x018c
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2                                                         0x0190
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3                                                         0x0194
+#define cfgBIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST                                                  0x0270
+#define cfgBIF_CFG_DEV0_RC_PCIE_LINK_CNTL3                                                              0x0274
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS                                                       0x0278
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL                                                0x027c
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL                                                0x027e
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL                                                0x0280
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL                                                0x0282
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL                                                0x0284
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL                                                0x0286
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL                                                0x0288
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL                                                0x028a
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL                                                0x028c
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL                                                0x028e
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL                                               0x0290
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL                                               0x0292
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL                                               0x0294
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL                                               0x0296
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL                                               0x0298
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL                                               0x029a
+#define cfgBIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST                                                        0x02a0
+#define cfgBIF_CFG_DEV0_RC_PCIE_ACS_CAP                                                                 0x02a4
+#define cfgBIF_CFG_DEV0_RC_PCIE_ACS_CNTL                                                                0x02a6
+#define cfgBIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST                                                        0x0400
+#define cfgBIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP                                                        0x0404
+#define cfgBIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS                                                     0x0408
+#define cfgBIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST                                                   0x0410
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP_16GT                                                                0x0414
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL_16GT                                                               0x0418
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS_16GT                                                             0x041c
+#define cfgBIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT                                            0x0420
+#define cfgBIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT                                             0x0424
+#define cfgBIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT                                             0x0428
+#define cfgBIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT                                                0x0430
+#define cfgBIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT                                                0x0431
+#define cfgBIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT                                                0x0432
+#define cfgBIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT                                                0x0433
+#define cfgBIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT                                                0x0434
+#define cfgBIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT                                                0x0435
+#define cfgBIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT                                                0x0436
+#define cfgBIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT                                                0x0437
+#define cfgBIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT                                                0x0438
+#define cfgBIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT                                                0x0439
+#define cfgBIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT                                               0x043a
+#define cfgBIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT                                               0x043b
+#define cfgBIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT                                               0x043c
+#define cfgBIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT                                               0x043d
+#define cfgBIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT                                               0x043e
+#define cfgBIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT                                               0x043f
+#define cfgBIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST                                                  0x0450
+#define cfgBIF_CFG_DEV0_RC_MARGINING_PORT_CAP                                                           0x0454
+#define cfgBIF_CFG_DEV0_RC_MARGINING_PORT_STATUS                                                        0x0456
+#define cfgBIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL                                                   0x0458
+#define cfgBIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS                                                 0x045a
+#define cfgBIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL                                                   0x045c
+#define cfgBIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS                                                 0x045e
+#define cfgBIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL                                                   0x0460
+#define cfgBIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS                                                 0x0462
+#define cfgBIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL                                                   0x0464
+#define cfgBIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS                                                 0x0466
+#define cfgBIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL                                                   0x0468
+#define cfgBIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS                                                 0x046a
+#define cfgBIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL                                                   0x046c
+#define cfgBIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS                                                 0x046e
+#define cfgBIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL                                                   0x0470
+#define cfgBIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS                                                 0x0472
+#define cfgBIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL                                                   0x0474
+#define cfgBIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS                                                 0x0476
+#define cfgBIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL                                                   0x0478
+#define cfgBIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS                                                 0x047a
+#define cfgBIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL                                                   0x047c
+#define cfgBIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS                                                 0x047e
+#define cfgBIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL                                                  0x0480
+#define cfgBIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS                                                0x0482
+#define cfgBIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL                                                  0x0484
+#define cfgBIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS                                                0x0486
+#define cfgBIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL                                                  0x0488
+#define cfgBIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS                                                0x048a
+#define cfgBIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL                                                  0x048c
+#define cfgBIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS                                                0x048e
+#define cfgBIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL                                                  0x0490
+#define cfgBIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS                                                0x0492
+#define cfgBIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL                                                  0x0494
+#define cfgBIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS                                                0x0496
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP_32GT                                                                0x0504
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL_32GT                                                               0x0508
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS_32GT                                                             0x050c
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf0_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF0_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF0_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF0_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF0_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF0_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF0_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF0_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF0_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF0_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0150
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS                                                 0x0154
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK                                                   0x0158
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY                                               0x015c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS                                                   0x0160
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK                                                     0x0164
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL                                                  0x0168
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG0                                                          0x016c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG1                                                          0x0170
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG2                                                          0x0174
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG3                                                          0x0178
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG0                                                   0x0188
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG1                                                   0x018c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG2                                                   0x0190
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG3                                                   0x0194
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST                                                  0x02b0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP                                                           0x02b4
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL                                                          0x02b6
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST                                                  0x0328
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP                                                           0x032c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL                                                          0x032e
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf1_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF1_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF1_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF1_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF1_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF1_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF1_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF1_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF1_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF1_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0150
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS                                                 0x0154
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK                                                   0x0158
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY                                               0x015c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS                                                   0x0160
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK                                                     0x0164
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL                                                  0x0168
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG0                                                          0x016c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG1                                                          0x0170
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG2                                                          0x0174
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG3                                                          0x0178
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG0                                                   0x0188
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG1                                                   0x018c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG2                                                   0x0190
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG3                                                   0x0194
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST                                                  0x02b0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP                                                           0x02b4
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL                                                          0x02b6
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST                                                  0x0328
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP                                                           0x032c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL                                                          0x032e
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf2_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF2_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF2_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF2_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF2_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF2_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF2_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF2_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF2_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF2_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0150
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS                                                 0x0154
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK                                                   0x0158
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY                                               0x015c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS                                                   0x0160
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK                                                     0x0164
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL                                                  0x0168
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG0                                                          0x016c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG1                                                          0x0170
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG2                                                          0x0174
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG3                                                          0x0178
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG0                                                   0x0188
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG1                                                   0x018c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG2                                                   0x0190
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG3                                                   0x0194
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST                                                  0x02b0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP                                                           0x02b4
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL                                                          0x02b6
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST                                                  0x0328
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP                                                           0x032c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL                                                          0x032e
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf3_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF3_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF3_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF3_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF3_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF3_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF3_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF3_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF3_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF3_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0150
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS                                                 0x0154
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK                                                   0x0158
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY                                               0x015c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS                                                   0x0160
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK                                                     0x0164
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL                                                  0x0168
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG0                                                          0x016c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG1                                                          0x0170
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG2                                                          0x0174
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG3                                                          0x0178
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG0                                                   0x0188
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG1                                                   0x018c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG2                                                   0x0190
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG3                                                   0x0194
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST                                                  0x02b0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP                                                           0x02b4
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL                                                          0x02b6
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST                                                  0x0328
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP                                                           0x032c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL                                                          0x032e
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf4_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF4_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF4_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF4_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF4_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF4_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF4_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF4_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF4_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF4_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0150
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS                                                 0x0154
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK                                                   0x0158
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY                                               0x015c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS                                                   0x0160
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK                                                     0x0164
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL                                                  0x0168
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG0                                                          0x016c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG1                                                          0x0170
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG2                                                          0x0174
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG3                                                          0x0178
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG0                                                   0x0188
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG1                                                   0x018c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG2                                                   0x0190
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG3                                                   0x0194
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST                                                  0x02b0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP                                                           0x02b4
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL                                                          0x02b6
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST                                                  0x0328
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP                                                           0x032c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL                                                          0x032e
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf5_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF5_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF5_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF5_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF5_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF5_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF5_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF5_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF5_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF5_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0150
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS                                                 0x0154
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK                                                   0x0158
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY                                               0x015c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS                                                   0x0160
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK                                                     0x0164
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL                                                  0x0168
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG0                                                          0x016c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG1                                                          0x0170
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG2                                                          0x0174
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG3                                                          0x0178
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG0                                                   0x0188
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG1                                                   0x018c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG2                                                   0x0190
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG3                                                   0x0194
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST                                                  0x02b0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP                                                           0x02b4
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL                                                          0x02b6
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST                                                  0x0328
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP                                                           0x032c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL                                                          0x032e
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf6_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF6_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF6_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF6_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF6_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF6_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF6_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF6_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF6_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF6_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0150
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS                                                 0x0154
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK                                                   0x0158
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY                                               0x015c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS                                                   0x0160
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK                                                     0x0164
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL                                                  0x0168
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG0                                                          0x016c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG1                                                          0x0170
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG2                                                          0x0174
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG3                                                          0x0178
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG0                                                   0x0188
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG1                                                   0x018c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG2                                                   0x0190
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG3                                                   0x0194
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST                                                  0x02b0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP                                                           0x02b4
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL                                                          0x02b6
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST                                                  0x0328
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP                                                           0x032c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL                                                          0x032e
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf7_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF7_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF7_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF7_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF7_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF7_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF7_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF7_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF7_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF7_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0150
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS                                                 0x0154
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK                                                   0x0158
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY                                               0x015c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS                                                   0x0160
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK                                                     0x0164
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL                                                  0x0168
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG0                                                          0x016c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG1                                                          0x0170
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG2                                                          0x0174
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG3                                                          0x0178
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG0                                                   0x0188
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG1                                                   0x018c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG2                                                   0x0190
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG3                                                   0x0194
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST                                                  0x02b0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP                                                           0x02b4
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL                                                          0x02b6
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST                                                  0x0328
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP                                                           0x032c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL                                                          0x032e
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf0_BIFPFVFDEC1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF0_BIF_BME_STATUS                                                          0x00eb
+#define regBIF_BX_DEV0_EPF0_VF0_BIF_BME_STATUS_BASE_IDX                                                 2
+#define regBIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG                                                      0x00ec
+#define regBIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                    0x00f3
+#define regBIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                           2
+#define regBIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                     0x00f4
+#define regBIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_CNTL                                         0x00f5
+#define regBIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                2
+#define regBIF_BX_DEV0_EPF0_VF0_HDP_REG_COHERENCY_FLUSH_CNTL                                            0x00f6
+#define regBIF_BX_DEV0_EPF0_VF0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL                                            0x00f7
+#define regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                       0x00f9
+#define regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                  0x00fa
+#define regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ                                                       0x0106
+#define regBIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE                                                      0x0107
+#define regBIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF0_BIF_TRANS_PENDING                                                       0x0108
+#define regBIF_BX_DEV0_EPF0_VF0_BIF_TRANS_PENDING_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF0_NBIF_GFX_ADDR_LUT_BYPASS                                                0x0112
+#define regBIF_BX_DEV0_EPF0_VF0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW0                                                  0x0136
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW1                                                  0x0137
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW2                                                  0x0138
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW3                                                  0x0139
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW0                                                  0x013a
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW1                                                  0x013b
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW2                                                  0x013c
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW3                                                  0x013d
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL                                                         0x013e
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL_BASE_IDX                                                2
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_INT_CNTL                                                        0x013f
+#define regBIF_BX_DEV0_EPF0_VF0_MAILBOX_INT_CNTL_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX                                                        0x0140
+#define regBIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX_BASE_IDX                                               2
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf0_SYSPFVFDEC
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF0_MM_INDEX                                                                0x0000
+#define regBIF_BX_DEV0_EPF0_VF0_MM_INDEX_BASE_IDX                                                       0
+#define regBIF_BX_DEV0_EPF0_VF0_MM_DATA                                                                 0x0001
+#define regBIF_BX_DEV0_EPF0_VF0_MM_DATA_BASE_IDX                                                        0
+#define regBIF_BX_DEV0_EPF0_VF0_MM_INDEX_HI                                                             0x0006
+#define regBIF_BX_DEV0_EPF0_VF0_MM_INDEX_HI_BASE_IDX                                                    0
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf0_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF0_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF0_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF0_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF0_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF0_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF0_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF0_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF0_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf0_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_PBA_BASE_IDX 4
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf1_BIFPFVFDEC1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF1_BIF_BME_STATUS                                                          0x00eb
+#define regBIF_BX_DEV0_EPF0_VF1_BIF_BME_STATUS_BASE_IDX                                                 2
+#define regBIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG                                                      0x00ec
+#define regBIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                    0x00f3
+#define regBIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                           2
+#define regBIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                     0x00f4
+#define regBIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_CNTL                                         0x00f5
+#define regBIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                2
+#define regBIF_BX_DEV0_EPF0_VF1_HDP_REG_COHERENCY_FLUSH_CNTL                                            0x00f6
+#define regBIF_BX_DEV0_EPF0_VF1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_FLUSH_CNTL                                            0x00f7
+#define regBIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                       0x00f9
+#define regBIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                  0x00fa
+#define regBIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ                                                       0x0106
+#define regBIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE                                                      0x0107
+#define regBIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF1_BIF_TRANS_PENDING                                                       0x0108
+#define regBIF_BX_DEV0_EPF0_VF1_BIF_TRANS_PENDING_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF1_NBIF_GFX_ADDR_LUT_BYPASS                                                0x0112
+#define regBIF_BX_DEV0_EPF0_VF1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW0                                                  0x0136
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW1                                                  0x0137
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW2                                                  0x0138
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW3                                                  0x0139
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW0                                                  0x013a
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW1                                                  0x013b
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW2                                                  0x013c
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW3                                                  0x013d
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL                                                         0x013e
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL_BASE_IDX                                                2
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_INT_CNTL                                                        0x013f
+#define regBIF_BX_DEV0_EPF0_VF1_MAILBOX_INT_CNTL_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX                                                        0x0140
+#define regBIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX_BASE_IDX                                               2
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf1_SYSPFVFDEC
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF1_MM_INDEX                                                                0x0000
+#define regBIF_BX_DEV0_EPF0_VF1_MM_INDEX_BASE_IDX                                                       0
+#define regBIF_BX_DEV0_EPF0_VF1_MM_DATA                                                                 0x0001
+#define regBIF_BX_DEV0_EPF0_VF1_MM_DATA_BASE_IDX                                                        0
+#define regBIF_BX_DEV0_EPF0_VF1_MM_INDEX_HI                                                             0x0006
+#define regBIF_BX_DEV0_EPF0_VF1_MM_INDEX_HI_BASE_IDX                                                    0
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf1_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF1_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF1_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF1_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF1_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF1_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF1_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF1_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF1_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf1_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_PBA_BASE_IDX 4
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf2_BIFPFVFDEC1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF2_BIF_BME_STATUS                                                          0x00eb
+#define regBIF_BX_DEV0_EPF0_VF2_BIF_BME_STATUS_BASE_IDX                                                 2
+#define regBIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG                                                      0x00ec
+#define regBIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                    0x00f3
+#define regBIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                           2
+#define regBIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                     0x00f4
+#define regBIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_CNTL                                         0x00f5
+#define regBIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                2
+#define regBIF_BX_DEV0_EPF0_VF2_HDP_REG_COHERENCY_FLUSH_CNTL                                            0x00f6
+#define regBIF_BX_DEV0_EPF0_VF2_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_FLUSH_CNTL                                            0x00f7
+#define regBIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                       0x00f9
+#define regBIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                  0x00fa
+#define regBIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ                                                       0x0106
+#define regBIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE                                                      0x0107
+#define regBIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF2_BIF_TRANS_PENDING                                                       0x0108
+#define regBIF_BX_DEV0_EPF0_VF2_BIF_TRANS_PENDING_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF2_NBIF_GFX_ADDR_LUT_BYPASS                                                0x0112
+#define regBIF_BX_DEV0_EPF0_VF2_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW0                                                  0x0136
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW1                                                  0x0137
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW2                                                  0x0138
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW3                                                  0x0139
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW0                                                  0x013a
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW1                                                  0x013b
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW2                                                  0x013c
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW3                                                  0x013d
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL                                                         0x013e
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL_BASE_IDX                                                2
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_INT_CNTL                                                        0x013f
+#define regBIF_BX_DEV0_EPF0_VF2_MAILBOX_INT_CNTL_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX                                                        0x0140
+#define regBIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX_BASE_IDX                                               2
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf2_SYSPFVFDEC
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF2_MM_INDEX                                                                0x0000
+#define regBIF_BX_DEV0_EPF0_VF2_MM_INDEX_BASE_IDX                                                       0
+#define regBIF_BX_DEV0_EPF0_VF2_MM_DATA                                                                 0x0001
+#define regBIF_BX_DEV0_EPF0_VF2_MM_DATA_BASE_IDX                                                        0
+#define regBIF_BX_DEV0_EPF0_VF2_MM_INDEX_HI                                                             0x0006
+#define regBIF_BX_DEV0_EPF0_VF2_MM_INDEX_HI_BASE_IDX                                                    0
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf2_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF2_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF2_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF2_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF2_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF2_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF2_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF2_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF2_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf2_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_PBA_BASE_IDX 4
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf3_BIFPFVFDEC1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF3_BIF_BME_STATUS                                                          0x00eb
+#define regBIF_BX_DEV0_EPF0_VF3_BIF_BME_STATUS_BASE_IDX                                                 2
+#define regBIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG                                                      0x00ec
+#define regBIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                    0x00f3
+#define regBIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                           2
+#define regBIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                     0x00f4
+#define regBIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_CNTL                                         0x00f5
+#define regBIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                2
+#define regBIF_BX_DEV0_EPF0_VF3_HDP_REG_COHERENCY_FLUSH_CNTL                                            0x00f6
+#define regBIF_BX_DEV0_EPF0_VF3_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_FLUSH_CNTL                                            0x00f7
+#define regBIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                       0x00f9
+#define regBIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                  0x00fa
+#define regBIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ                                                       0x0106
+#define regBIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE                                                      0x0107
+#define regBIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF3_BIF_TRANS_PENDING                                                       0x0108
+#define regBIF_BX_DEV0_EPF0_VF3_BIF_TRANS_PENDING_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF3_NBIF_GFX_ADDR_LUT_BYPASS                                                0x0112
+#define regBIF_BX_DEV0_EPF0_VF3_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW0                                                  0x0136
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW1                                                  0x0137
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW2                                                  0x0138
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW3                                                  0x0139
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW0                                                  0x013a
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW1                                                  0x013b
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW2                                                  0x013c
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW3                                                  0x013d
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL                                                         0x013e
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL_BASE_IDX                                                2
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_INT_CNTL                                                        0x013f
+#define regBIF_BX_DEV0_EPF0_VF3_MAILBOX_INT_CNTL_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX                                                        0x0140
+#define regBIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX_BASE_IDX                                               2
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf3_SYSPFVFDEC
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF3_MM_INDEX                                                                0x0000
+#define regBIF_BX_DEV0_EPF0_VF3_MM_INDEX_BASE_IDX                                                       0
+#define regBIF_BX_DEV0_EPF0_VF3_MM_DATA                                                                 0x0001
+#define regBIF_BX_DEV0_EPF0_VF3_MM_DATA_BASE_IDX                                                        0
+#define regBIF_BX_DEV0_EPF0_VF3_MM_INDEX_HI                                                             0x0006
+#define regBIF_BX_DEV0_EPF0_VF3_MM_INDEX_HI_BASE_IDX                                                    0
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf3_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF3_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF3_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF3_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF3_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF3_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF3_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF3_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF3_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf3_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_PBA_BASE_IDX 4
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf4_BIFPFVFDEC1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF4_BIF_BME_STATUS                                                          0x00eb
+#define regBIF_BX_DEV0_EPF0_VF4_BIF_BME_STATUS_BASE_IDX                                                 2
+#define regBIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG                                                      0x00ec
+#define regBIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                    0x00f3
+#define regBIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                           2
+#define regBIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                     0x00f4
+#define regBIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_CNTL                                         0x00f5
+#define regBIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                2
+#define regBIF_BX_DEV0_EPF0_VF4_HDP_REG_COHERENCY_FLUSH_CNTL                                            0x00f6
+#define regBIF_BX_DEV0_EPF0_VF4_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_FLUSH_CNTL                                            0x00f7
+#define regBIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                       0x00f9
+#define regBIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                  0x00fa
+#define regBIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ                                                       0x0106
+#define regBIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE                                                      0x0107
+#define regBIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF4_BIF_TRANS_PENDING                                                       0x0108
+#define regBIF_BX_DEV0_EPF0_VF4_BIF_TRANS_PENDING_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF4_NBIF_GFX_ADDR_LUT_BYPASS                                                0x0112
+#define regBIF_BX_DEV0_EPF0_VF4_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW0                                                  0x0136
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW1                                                  0x0137
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW2                                                  0x0138
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW3                                                  0x0139
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW0                                                  0x013a
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW1                                                  0x013b
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW2                                                  0x013c
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW3                                                  0x013d
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL                                                         0x013e
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL_BASE_IDX                                                2
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_INT_CNTL                                                        0x013f
+#define regBIF_BX_DEV0_EPF0_VF4_MAILBOX_INT_CNTL_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX                                                        0x0140
+#define regBIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX_BASE_IDX                                               2
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf4_SYSPFVFDEC
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF4_MM_INDEX                                                                0x0000
+#define regBIF_BX_DEV0_EPF0_VF4_MM_INDEX_BASE_IDX                                                       0
+#define regBIF_BX_DEV0_EPF0_VF4_MM_DATA                                                                 0x0001
+#define regBIF_BX_DEV0_EPF0_VF4_MM_DATA_BASE_IDX                                                        0
+#define regBIF_BX_DEV0_EPF0_VF4_MM_INDEX_HI                                                             0x0006
+#define regBIF_BX_DEV0_EPF0_VF4_MM_INDEX_HI_BASE_IDX                                                    0
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf4_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF4_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF4_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF4_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF4_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF4_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF4_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF4_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF4_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf4_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_PBA_BASE_IDX 4
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf5_BIFPFVFDEC1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF5_BIF_BME_STATUS                                                          0x00eb
+#define regBIF_BX_DEV0_EPF0_VF5_BIF_BME_STATUS_BASE_IDX                                                 2
+#define regBIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG                                                      0x00ec
+#define regBIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                    0x00f3
+#define regBIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                           2
+#define regBIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                     0x00f4
+#define regBIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_CNTL                                         0x00f5
+#define regBIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                2
+#define regBIF_BX_DEV0_EPF0_VF5_HDP_REG_COHERENCY_FLUSH_CNTL                                            0x00f6
+#define regBIF_BX_DEV0_EPF0_VF5_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_FLUSH_CNTL                                            0x00f7
+#define regBIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                       0x00f9
+#define regBIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                  0x00fa
+#define regBIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ                                                       0x0106
+#define regBIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE                                                      0x0107
+#define regBIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF5_BIF_TRANS_PENDING                                                       0x0108
+#define regBIF_BX_DEV0_EPF0_VF5_BIF_TRANS_PENDING_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF5_NBIF_GFX_ADDR_LUT_BYPASS                                                0x0112
+#define regBIF_BX_DEV0_EPF0_VF5_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW0                                                  0x0136
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW1                                                  0x0137
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW2                                                  0x0138
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW3                                                  0x0139
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW0                                                  0x013a
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW1                                                  0x013b
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW2                                                  0x013c
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW3                                                  0x013d
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL                                                         0x013e
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL_BASE_IDX                                                2
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_INT_CNTL                                                        0x013f
+#define regBIF_BX_DEV0_EPF0_VF5_MAILBOX_INT_CNTL_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX                                                        0x0140
+#define regBIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX_BASE_IDX                                               2
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf5_SYSPFVFDEC
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF5_MM_INDEX                                                                0x0000
+#define regBIF_BX_DEV0_EPF0_VF5_MM_INDEX_BASE_IDX                                                       0
+#define regBIF_BX_DEV0_EPF0_VF5_MM_DATA                                                                 0x0001
+#define regBIF_BX_DEV0_EPF0_VF5_MM_DATA_BASE_IDX                                                        0
+#define regBIF_BX_DEV0_EPF0_VF5_MM_INDEX_HI                                                             0x0006
+#define regBIF_BX_DEV0_EPF0_VF5_MM_INDEX_HI_BASE_IDX                                                    0
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf5_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF5_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF5_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF5_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF5_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF5_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF5_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF5_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF5_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf5_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_PBA_BASE_IDX 4
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf6_BIFPFVFDEC1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF6_BIF_BME_STATUS                                                          0x00eb
+#define regBIF_BX_DEV0_EPF0_VF6_BIF_BME_STATUS_BASE_IDX                                                 2
+#define regBIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG                                                      0x00ec
+#define regBIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                    0x00f3
+#define regBIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                           2
+#define regBIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                     0x00f4
+#define regBIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_CNTL                                         0x00f5
+#define regBIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                2
+#define regBIF_BX_DEV0_EPF0_VF6_HDP_REG_COHERENCY_FLUSH_CNTL                                            0x00f6
+#define regBIF_BX_DEV0_EPF0_VF6_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_FLUSH_CNTL                                            0x00f7
+#define regBIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                       0x00f9
+#define regBIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                  0x00fa
+#define regBIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ                                                       0x0106
+#define regBIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE                                                      0x0107
+#define regBIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF6_BIF_TRANS_PENDING                                                       0x0108
+#define regBIF_BX_DEV0_EPF0_VF6_BIF_TRANS_PENDING_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF6_NBIF_GFX_ADDR_LUT_BYPASS                                                0x0112
+#define regBIF_BX_DEV0_EPF0_VF6_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW0                                                  0x0136
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW1                                                  0x0137
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW2                                                  0x0138
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW3                                                  0x0139
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW0                                                  0x013a
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW1                                                  0x013b
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW2                                                  0x013c
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW3                                                  0x013d
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL                                                         0x013e
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL_BASE_IDX                                                2
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_INT_CNTL                                                        0x013f
+#define regBIF_BX_DEV0_EPF0_VF6_MAILBOX_INT_CNTL_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX                                                        0x0140
+#define regBIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX_BASE_IDX                                               2
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf6_SYSPFVFDEC
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF6_MM_INDEX                                                                0x0000
+#define regBIF_BX_DEV0_EPF0_VF6_MM_INDEX_BASE_IDX                                                       0
+#define regBIF_BX_DEV0_EPF0_VF6_MM_DATA                                                                 0x0001
+#define regBIF_BX_DEV0_EPF0_VF6_MM_DATA_BASE_IDX                                                        0
+#define regBIF_BX_DEV0_EPF0_VF6_MM_INDEX_HI                                                             0x0006
+#define regBIF_BX_DEV0_EPF0_VF6_MM_INDEX_HI_BASE_IDX                                                    0
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf6_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF6_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF6_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF6_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF6_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF6_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF6_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF6_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF6_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf6_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_PBA_BASE_IDX 4
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf7_BIFPFVFDEC1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF7_BIF_BME_STATUS                                                          0x00eb
+#define regBIF_BX_DEV0_EPF0_VF7_BIF_BME_STATUS_BASE_IDX                                                 2
+#define regBIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG                                                      0x00ec
+#define regBIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                    0x00f3
+#define regBIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                           2
+#define regBIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                     0x00f4
+#define regBIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_CNTL                                         0x00f5
+#define regBIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                2
+#define regBIF_BX_DEV0_EPF0_VF7_HDP_REG_COHERENCY_FLUSH_CNTL                                            0x00f6
+#define regBIF_BX_DEV0_EPF0_VF7_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_FLUSH_CNTL                                            0x00f7
+#define regBIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                   2
+#define regBIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                       0x00f9
+#define regBIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                  0x00fa
+#define regBIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ                                                       0x0106
+#define regBIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE                                                      0x0107
+#define regBIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF7_BIF_TRANS_PENDING                                                       0x0108
+#define regBIF_BX_DEV0_EPF0_VF7_BIF_TRANS_PENDING_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF7_NBIF_GFX_ADDR_LUT_BYPASS                                                0x0112
+#define regBIF_BX_DEV0_EPF0_VF7_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW0                                                  0x0136
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW1                                                  0x0137
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW2                                                  0x0138
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW3                                                  0x0139
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW0                                                  0x013a
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW1                                                  0x013b
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW2                                                  0x013c
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW3                                                  0x013d
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                         2
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL                                                         0x013e
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL_BASE_IDX                                                2
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_INT_CNTL                                                        0x013f
+#define regBIF_BX_DEV0_EPF0_VF7_MAILBOX_INT_CNTL_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX                                                        0x0140
+#define regBIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX_BASE_IDX                                               2
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf7_SYSPFVFDEC
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF7_MM_INDEX                                                                0x0000
+#define regBIF_BX_DEV0_EPF0_VF7_MM_INDEX_BASE_IDX                                                       0
+#define regBIF_BX_DEV0_EPF0_VF7_MM_DATA                                                                 0x0001
+#define regBIF_BX_DEV0_EPF0_VF7_MM_DATA_BASE_IDX                                                        0
+#define regBIF_BX_DEV0_EPF0_VF7_MM_INDEX_HI                                                             0x0006
+#define regBIF_BX_DEV0_EPF0_VF7_MM_INDEX_HI_BASE_IDX                                                    0
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf7_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF7_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF7_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF7_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF7_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF7_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF7_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF7_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF7_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf7_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_LO_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_HI_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_MSG_DATA_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_CONTROL_BASE_IDX 4
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_PBA_BASE_IDX 4
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_9_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_9_0_sh_mask.h
new file mode 100644
index 000000000000..a22481e7bcdb
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_9_0_sh_mask.h
@@ -0,0 +1,38900 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _nbio_7_9_0_SH_MASK_HEADER
+#define _nbio_7_9_0_SH_MASK_HEADER
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_SYSDEC
+//BIF_BX0_PCIE_INDEX
+#define BIF_BX0_PCIE_INDEX__PCIE_INDEX__SHIFT                                                                 0x0
+#define BIF_BX0_PCIE_INDEX__PCIE_INDEX_MASK                                                                   0xFFFFFFFFL
+//BIF_BX0_PCIE_DATA
+#define BIF_BX0_PCIE_DATA__PCIE_DATA__SHIFT                                                                   0x0
+#define BIF_BX0_PCIE_DATA__PCIE_DATA_MASK                                                                     0xFFFFFFFFL
+//BIF_BX0_PCIE_INDEX2
+#define BIF_BX0_PCIE_INDEX2__PCIE_INDEX2__SHIFT                                                               0x0
+#define BIF_BX0_PCIE_INDEX2__PCIE_INDEX2_MASK                                                                 0xFFFFFFFFL
+//BIF_BX0_PCIE_DATA2
+#define BIF_BX0_PCIE_DATA2__PCIE_DATA2__SHIFT                                                                 0x0
+#define BIF_BX0_PCIE_DATA2__PCIE_DATA2_MASK                                                                   0xFFFFFFFFL
+//BIF_BX0_PCIE_INDEX_HI
+#define BIF_BX0_PCIE_INDEX_HI__PCIE_INDEX_HI__SHIFT                                                           0x0
+#define BIF_BX0_PCIE_INDEX_HI__PCIE_INDEX_HI_MASK                                                             0x000000FFL
+//BIF_BX0_PCIE_INDEX2_HI
+#define BIF_BX0_PCIE_INDEX2_HI__PCIE_INDEX2_HI__SHIFT                                                         0x0
+#define BIF_BX0_PCIE_INDEX2_HI__PCIE_INDEX2_HI_MASK                                                           0x000000FFL
+//BIF_BX0_SBIOS_SCRATCH_0
+#define BIF_BX0_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_1
+#define BIF_BX0_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_2
+#define BIF_BX0_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_3
+#define BIF_BX0_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_0
+#define BIF_BX0_BIOS_SCRATCH_0__BIOS_SCRATCH_0__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_0__BIOS_SCRATCH_0_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_1
+#define BIF_BX0_BIOS_SCRATCH_1__BIOS_SCRATCH_1__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_1__BIOS_SCRATCH_1_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_2
+#define BIF_BX0_BIOS_SCRATCH_2__BIOS_SCRATCH_2__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_2__BIOS_SCRATCH_2_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_3
+#define BIF_BX0_BIOS_SCRATCH_3__BIOS_SCRATCH_3__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_3__BIOS_SCRATCH_3_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_4
+#define BIF_BX0_BIOS_SCRATCH_4__BIOS_SCRATCH_4__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_4__BIOS_SCRATCH_4_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_5
+#define BIF_BX0_BIOS_SCRATCH_5__BIOS_SCRATCH_5__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_5__BIOS_SCRATCH_5_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_6
+#define BIF_BX0_BIOS_SCRATCH_6__BIOS_SCRATCH_6__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_6__BIOS_SCRATCH_6_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_7
+#define BIF_BX0_BIOS_SCRATCH_7__BIOS_SCRATCH_7__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_7__BIOS_SCRATCH_7_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_8
+#define BIF_BX0_BIOS_SCRATCH_8__BIOS_SCRATCH_8__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_8__BIOS_SCRATCH_8_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_9
+#define BIF_BX0_BIOS_SCRATCH_9__BIOS_SCRATCH_9__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_9__BIOS_SCRATCH_9_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_10
+#define BIF_BX0_BIOS_SCRATCH_10__BIOS_SCRATCH_10__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_10__BIOS_SCRATCH_10_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_11
+#define BIF_BX0_BIOS_SCRATCH_11__BIOS_SCRATCH_11__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_11__BIOS_SCRATCH_11_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_12
+#define BIF_BX0_BIOS_SCRATCH_12__BIOS_SCRATCH_12__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_12__BIOS_SCRATCH_12_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_13
+#define BIF_BX0_BIOS_SCRATCH_13__BIOS_SCRATCH_13__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_13__BIOS_SCRATCH_13_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_14
+#define BIF_BX0_BIOS_SCRATCH_14__BIOS_SCRATCH_14__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_14__BIOS_SCRATCH_14_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_15
+#define BIF_BX0_BIOS_SCRATCH_15__BIOS_SCRATCH_15__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_15__BIOS_SCRATCH_15_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIF_RLC_INTR_CNTL
+//BIF_BX0_BIF_VCE_INTR_CNTL
+//BIF_BX0_BIF_UVD_INTR_CNTL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR1
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR2
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR3
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR4
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR5
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR6
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR7
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_CNTL
+#define BIF_BX0_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE_MASK                                                         0x000000FFL
+//BIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL
+#define BIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL__SHIFT                                                 0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL_MASK                                                   0xFFFFFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ONE_CPL
+#define BIF_BX0_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL__SHIFT                                                   0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL
+#define BIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL__SHIFT                                 0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL_MASK                                   0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_0
+#define BIF_BX0_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_1
+#define BIF_BX0_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_2
+#define BIF_BX0_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_3
+#define BIF_BX0_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_4
+#define BIF_BX0_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_5
+#define BIF_BX0_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_6
+#define BIF_BX0_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_7
+#define BIF_BX0_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_8
+#define BIF_BX0_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_9
+#define BIF_BX0_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_10
+#define BIF_BX0_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_11
+#define BIF_BX0_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_12
+#define BIF_BX0_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_13
+#define BIF_BX0_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_14
+#define BIF_BX0_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_15
+#define BIF_BX0_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_0
+#define BIF_BX0_FW_SCRATCH_0__FW_SCRATCH_0__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_0__FW_SCRATCH_0_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_1
+#define BIF_BX0_FW_SCRATCH_1__FW_SCRATCH_1__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_1__FW_SCRATCH_1_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_2
+#define BIF_BX0_FW_SCRATCH_2__FW_SCRATCH_2__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_2__FW_SCRATCH_2_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_3
+#define BIF_BX0_FW_SCRATCH_3__FW_SCRATCH_3__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_3__FW_SCRATCH_3_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_4
+#define BIF_BX0_FW_SCRATCH_4__FW_SCRATCH_4__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_4__FW_SCRATCH_4_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_5
+#define BIF_BX0_FW_SCRATCH_5__FW_SCRATCH_5__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_5__FW_SCRATCH_5_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_6
+#define BIF_BX0_FW_SCRATCH_6__FW_SCRATCH_6__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_6__FW_SCRATCH_6_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_7
+#define BIF_BX0_FW_SCRATCH_7__FW_SCRATCH_7__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_7__FW_SCRATCH_7_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_8
+#define BIF_BX0_FW_SCRATCH_8__FW_SCRATCH_8__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_8__FW_SCRATCH_8_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_9
+#define BIF_BX0_FW_SCRATCH_9__FW_SCRATCH_9__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_9__FW_SCRATCH_9_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_10
+#define BIF_BX0_FW_SCRATCH_10__FW_SCRATCH_10__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_10__FW_SCRATCH_10_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_11
+#define BIF_BX0_FW_SCRATCH_11__FW_SCRATCH_11__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_11__FW_SCRATCH_11_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_12
+#define BIF_BX0_FW_SCRATCH_12__FW_SCRATCH_12__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_12__FW_SCRATCH_12_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_13
+#define BIF_BX0_FW_SCRATCH_13__FW_SCRATCH_13__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_13__FW_SCRATCH_13_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_14
+#define BIF_BX0_FW_SCRATCH_14__FW_SCRATCH_14__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_14__FW_SCRATCH_14_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_15
+#define BIF_BX0_FW_SCRATCH_15__FW_SCRATCH_15__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_15__FW_SCRATCH_15_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_4
+#define BIF_BX0_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_5
+#define BIF_BX0_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_6
+#define BIF_BX0_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_7
+#define BIF_BX0_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_8
+#define BIF_BX0_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_9
+#define BIF_BX0_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_10
+#define BIF_BX0_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_11
+#define BIF_BX0_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_12
+#define BIF_BX0_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_13
+#define BIF_BX0_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_14
+#define BIF_BX0_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_15
+#define BIF_BX0_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwn_dev0_BIFDEC1
+//RCC_DWN_DEV0_0_DN_PCIE_RESERVED
+#define RCC_DWN_DEV0_0_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                 0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                   0xFFFFFFFFL
+//RCC_DWN_DEV0_0_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV0_0_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV0_0_DN_PCIE_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                    0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                              0x7
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                              0x1e
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                      0x00000001L
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                0x00000080L
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                0x40000000L
+//RCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                0x19
+#define RCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                  0x06000000L
+//RCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                               0x1c
+#define RCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                 0x70000000L
+//RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                             0x7
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                   0x8
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                               0x00000080L
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                     0x00000100L
+//RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                      0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                 0x1
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                 0x2
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                 0x3
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                 0x4
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                        0x00000001L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                   0x00000002L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                   0x00000004L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                   0x00000008L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                   0x00000010L
+//RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                0x11
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                        0x15
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                     0x00000001L
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                  0x00020000L
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                          0x00E00000L
+//RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                             0x18
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                          0x1d
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                               0x01000000L
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                            0x20000000L
+//RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                    0x2
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                      0x00000004L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwnp_dev0_BIFDEC1
+//RCC_DWNP_DEV0_0_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                               0x0
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                             0x8
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                    0xb
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                        0x11
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                               0x12
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                           0x13
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                              0x14
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                 0x00000001L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                               0x00000700L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                      0x00000800L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                          0x00020000L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                 0x00040000L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                             0x00080000L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                0x00100000L
+//RCC_DWNP_DEV0_0_PCIE_RX_CNTL
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                        0x8
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                              0x9
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                          0x14
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                     0x15
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                           0x1b
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                          0x00000100L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                0x00000200L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                            0x00100000L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                       0x00200000L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                             0x08000000L
+//RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                           0x0
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                           0x1
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                           0x2
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                           0x3
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                             0x00000001L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                             0x00000002L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                             0x00000004L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                             0x00000008L
+//RCC_DWNP_DEV0_0_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                               0x0
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                     0x1b
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                 0x00000001L
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                       0x08000000L
+//RCC_DWNP_DEV0_0_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV0_0_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                          0xa
+#define RCC_DWNP_DEV0_0_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                            0x00000400L
+//RCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                     0x0
+#define RCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                       0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_ep_dev0_BIFDEC1
+//RCC_EP_DEV0_0_EP_PCIE_SCRATCH
+#define RCC_EP_DEV0_0_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                    0x0
+#define RCC_EP_DEV0_0_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                      0xFFFFFFFFL
+//RCC_EP_DEV0_0_EP_PCIE_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                  0x7
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                            0x8
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                               0x1e
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                    0x00000080L
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                              0x00000100L
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                 0x40000000L
+//RCC_EP_DEV0_0_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                0x0
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                           0x1
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                               0x2
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                            0x3
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                0x4
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                         0x6
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                  0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                             0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                 0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                              0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                  0x00000010L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                           0x00000040L
+//RCC_EP_DEV0_0_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                          0x0
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                     0x1
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                         0x2
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                      0x3
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                          0x4
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                   0x6
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                0x7
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                            0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                       0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                           0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                        0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                            0x00000010L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                     0x00000040L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                  0x00000080L
+//RCC_EP_DEV0_0_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                   0x0
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                     0x00000001L
+//RCC_EP_DEV0_0_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                              0x7
+#define RCC_EP_DEV0_0_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                0x00000080L
+//RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                       0x0
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                  0x1
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                  0x2
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                  0x3
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                  0x4
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                         0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                    0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                    0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                    0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                    0x00000010L
+//RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                           0x1d
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                             0x20000000L
+//RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                         0x4
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                           0x00000010L
+//RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//RCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0xFFL
+//RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                             0x0
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                         0x8
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                               0x001FL
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                           0x0100L
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV0_0_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                           0x0
+#define RCC_EP_DEV0_0_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                             0x1FL
+//RCC_EP_DEV0_0_EP_PCIEP_RESERVED
+#define RCC_EP_DEV0_0_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                0x0
+#define RCC_EP_DEV0_0_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                  0xFFFFFFFFL
+//RCC_EP_DEV0_0_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                 0xa
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                  0xc
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                   0x18
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                   0x19
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                   0x1a
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                   0x00000C00L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                    0x00003000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                     0x01000000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                     0x02000000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                     0x04000000L
+//RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                0x0
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                  0x3
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                     0x8
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                  0x00000007L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                    0x000000F8L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                       0x0000FF00L
+//RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                              0x0
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                            0x8
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                       0x11
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                               0x12
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                   0x18
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                   0x19
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                   0x1a
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                   0x1b
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                   0x1c
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                   0x1d
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                   0x1e
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                   0x1f
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                              0x00000700L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                         0x00020000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                 0x00040000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                     0x01000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                     0x02000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                     0x04000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                     0x08000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                     0x10000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                     0x20000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                     0x40000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                     0x80000000L
+//RCC_EP_DEV0_0_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                       0x8
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                0x9
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                         0x14
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                       0x15
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                         0x16
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                      0x18
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                          0x19
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                      0x1a
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                         0x00000100L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                  0x00000200L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                           0x00100000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                         0x00200000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                           0x00400000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                        0x01000000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                            0x02000000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                        0x04000000L
+//RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                          0x0
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                          0x1
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                          0x2
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                          0x3
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                            0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                            0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                            0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                            0x00000008L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_pf_SYSPFVFDEC
+//BIF_BX_PF0_MM_INDEX
+#define BIF_BX_PF0_MM_INDEX__MM_OFFSET__SHIFT                                                                 0x0
+#define BIF_BX_PF0_MM_INDEX__MM_APER__SHIFT                                                                   0x1f
+#define BIF_BX_PF0_MM_INDEX__MM_OFFSET_MASK                                                                   0x7FFFFFFFL
+#define BIF_BX_PF0_MM_INDEX__MM_APER_MASK                                                                     0x80000000L
+//BIF_BX_PF0_MM_DATA
+#define BIF_BX_PF0_MM_DATA__MM_DATA__SHIFT                                                                    0x0
+#define BIF_BX_PF0_MM_DATA__MM_DATA_MASK                                                                      0xFFFFFFFFL
+//BIF_BX_PF0_MM_INDEX_HI
+#define BIF_BX_PF0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                           0x0
+#define BIF_BX_PF0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                             0xFFFFFFFFL
+//BIF_BX_PF0_RSMU_INDEX
+#define BIF_BX_PF0_RSMU_INDEX__RSMU_INDEX__SHIFT                                                              0x0
+#define BIF_BX_PF0_RSMU_INDEX__RSMU_INDEX_MASK                                                                0xFFFFFFFFL
+//BIF_BX_PF0_RSMU_DATA
+#define BIF_BX_PF0_RSMU_DATA__RSMU_DATA__SHIFT                                                                0x0
+#define BIF_BX_PF0_RSMU_DATA__RSMU_DATA_MASK                                                                  0xFFFFFFFFL
+//BIF_BX_PF0_RSMU_INDEX_HI
+#define BIF_BX_PF0_RSMU_INDEX_HI__RSMU_INDEX_HI__SHIFT                                                        0x0
+#define BIF_BX_PF0_RSMU_INDEX_HI__RSMU_INDEX_HI_MASK                                                          0x000000FFL
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_BIFDEC1
+//BIF_BX0_CC_BIF_BX_STRAP0
+#define BIF_BX0_CC_BIF_BX_STRAP0__STRAP_RESERVED__SHIFT                                                       0x19
+#define BIF_BX0_CC_BIF_BX_STRAP0__STRAP_RESERVED_MASK                                                         0xFE000000L
+//BIF_BX0_CC_BIF_BX_PINSTRAP0
+//BIF_BX0_BIF_MM_INDACCESS_CNTL
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__WRITE_DIS__SHIFT                                                       0x0
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS__SHIFT                                                0x1
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__WRITE_DIS_MASK                                                         0x00000001L
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS_MASK                                                  0x00000002L
+//BIF_BX0_BUS_CNTL
+#define BIF_BX0_BUS_CNTL__VGA_REG_COHERENCY_DIS__SHIFT                                                        0x6
+#define BIF_BX0_BUS_CNTL__VGA_MEM_COHERENCY_DIS__SHIFT                                                        0x7
+#define BIF_BX0_BUS_CNTL__SET_AZ_TC__SHIFT                                                                    0xa
+#define BIF_BX0_BUS_CNTL__SET_MC_TC__SHIFT                                                                    0xd
+#define BIF_BX0_BUS_CNTL__ZERO_BE_WR_EN__SHIFT                                                                0x10
+#define BIF_BX0_BUS_CNTL__ZERO_BE_RD_EN__SHIFT                                                                0x11
+#define BIF_BX0_BUS_CNTL__RD_STALL_IO_WR__SHIFT                                                               0x12
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS__SHIFT                                              0x18
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS__SHIFT                                          0x19
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS__SHIFT                                         0x1a
+#define BIF_BX0_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS__SHIFT                                        0x1b
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS__SHIFT                                          0x1c
+#define BIF_BX0_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN__SHIFT                                                     0x1d
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_WR_EN__SHIFT                                                          0x1e
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_RD_EN__SHIFT                                                          0x1f
+#define BIF_BX0_BUS_CNTL__VGA_REG_COHERENCY_DIS_MASK                                                          0x00000040L
+#define BIF_BX0_BUS_CNTL__VGA_MEM_COHERENCY_DIS_MASK                                                          0x00000080L
+#define BIF_BX0_BUS_CNTL__SET_AZ_TC_MASK                                                                      0x00001C00L
+#define BIF_BX0_BUS_CNTL__SET_MC_TC_MASK                                                                      0x0000E000L
+#define BIF_BX0_BUS_CNTL__ZERO_BE_WR_EN_MASK                                                                  0x00010000L
+#define BIF_BX0_BUS_CNTL__ZERO_BE_RD_EN_MASK                                                                  0x00020000L
+#define BIF_BX0_BUS_CNTL__RD_STALL_IO_WR_MASK                                                                 0x00040000L
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS_MASK                                                0x01000000L
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS_MASK                                            0x02000000L
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS_MASK                                           0x04000000L
+#define BIF_BX0_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS_MASK                                          0x08000000L
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS_MASK                                            0x10000000L
+#define BIF_BX0_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN_MASK                                                       0x20000000L
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_WR_EN_MASK                                                            0x40000000L
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_RD_EN_MASK                                                            0x80000000L
+//BIF_BX0_BIF_SCRATCH0
+#define BIF_BX0_BIF_SCRATCH0__BIF_SCRATCH0__SHIFT                                                             0x0
+#define BIF_BX0_BIF_SCRATCH0__BIF_SCRATCH0_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_BIF_SCRATCH1
+#define BIF_BX0_BIF_SCRATCH1__BIF_SCRATCH1__SHIFT                                                             0x0
+#define BIF_BX0_BIF_SCRATCH1__BIF_SCRATCH1_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_BX_RESET_EN
+#define BIF_BX0_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN__SHIFT                                                  0x10
+#define BIF_BX0_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN_MASK                                                    0x00010000L
+//BIF_BX0_MM_CFGREGS_CNTL
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL__SHIFT                                                       0x0
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL__SHIFT                                                        0x6
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN__SHIFT                                                       0x1f
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL_MASK                                                         0x00000007L
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL_MASK                                                          0x000000C0L
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN_MASK                                                         0x80000000L
+//BIF_BX0_BX_RESET_CNTL
+#define BIF_BX0_BX_RESET_CNTL__LINK_TRAIN_EN__SHIFT                                                           0x0
+#define BIF_BX0_BX_RESET_CNTL__LINK_TRAIN_EN_MASK                                                             0x00000001L
+//BIF_BX0_INTERRUPT_CNTL
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE__SHIFT                                                   0x0
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_EN__SHIFT                                                         0x1
+#define BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN__SHIFT                                                     0x3
+#define BIF_BX0_INTERRUPT_CNTL__IH_INTR_DLY_CNTR__SHIFT                                                       0x4
+#define BIF_BX0_INTERRUPT_CNTL__GEN_IH_INT_EN__SHIFT                                                          0x8
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_NONSNOOP_EN__SHIFT                                                 0xf
+#define BIF_BX0_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN__SHIFT                                               0x10
+#define BIF_BX0_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS__SHIFT                                   0x11
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_RELAX_ORDER_EN__SHIFT                                              0x12
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK                                                     0x00000001L
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_EN_MASK                                                           0x00000002L
+#define BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK                                                       0x00000008L
+#define BIF_BX0_INTERRUPT_CNTL__IH_INTR_DLY_CNTR_MASK                                                         0x000000F0L
+#define BIF_BX0_INTERRUPT_CNTL__GEN_IH_INT_EN_MASK                                                            0x00000100L
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_NONSNOOP_EN_MASK                                                   0x00008000L
+#define BIF_BX0_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN_MASK                                                 0x00010000L
+#define BIF_BX0_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS_MASK                                     0x00020000L
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_RELAX_ORDER_EN_MASK                                                0x00040000L
+//BIF_BX0_INTERRUPT_CNTL2
+#define BIF_BX0_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR__SHIFT                                                      0x0
+#define BIF_BX0_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR_MASK                                                        0xFFFFFFFFL
+//BIF_BX0_CLKREQB_PAD_CNTL
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_A__SHIFT                                                        0x0
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL__SHIFT                                                      0x1
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE__SHIFT                                                     0x2
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE__SHIFT                                                    0x3
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0__SHIFT                                                      0x5
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1__SHIFT                                                      0x6
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2__SHIFT                                                      0x7
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3__SHIFT                                                      0x8
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN__SHIFT                                                    0x9
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE__SHIFT                                                     0xa
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN__SHIFT                                                   0xb
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN__SHIFT                                                  0xc
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y__SHIFT                                                        0xd
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_A_MASK                                                          0x00000001L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL_MASK                                                        0x00000002L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE_MASK                                                       0x00000004L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE_MASK                                                      0x00000018L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0_MASK                                                        0x00000020L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1_MASK                                                        0x00000040L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2_MASK                                                        0x00000080L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3_MASK                                                        0x00000100L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN_MASK                                                      0x00000200L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE_MASK                                                       0x00000400L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN_MASK                                                     0x00000800L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN_MASK                                                    0x00001000L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y_MASK                                                          0x00002000L
+//BIF_BX0_BIF_FEATURES_CONTROL_MISC
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS__SHIFT                                          0x0
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS__SHIFT                                          0x1
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS__SHIFT                                          0x2
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS__SHIFT                                          0x3
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_RB_MSI_VEC_NOT_ENABLED_MODE__SHIFT                             0xb
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_RB_SET_OVERFLOW_EN__SHIFT                                      0xc
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS__SHIFT                                          0xd
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS__SHIFT                                       0xe
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BME_HDL_NONVIR_EN__SHIFT                                           0xf
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT__SHIFT                                           0x10
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR__SHIFT                   0x19
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS_MASK                                            0x00000001L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS_MASK                                            0x00000002L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS_MASK                                            0x00000004L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS_MASK                                            0x00000008L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_RB_MSI_VEC_NOT_ENABLED_MODE_MASK                               0x00000800L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_RB_SET_OVERFLOW_EN_MASK                                        0x00001000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS_MASK                                            0x00002000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS_MASK                                         0x00004000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BME_HDL_NONVIR_EN_MASK                                             0x00008000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT_MASK                                             0x01FF0000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR_MASK                     0x02000000L
+//BIF_BX0_HDP_ATOMIC_CONTROL_MISC
+#define BIF_BX0_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT__SHIFT                                      0x0
+#define BIF_BX0_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT_MASK                                        0x000000FFL
+//BIF_BX0_BIF_DOORBELL_CNTL
+#define BIF_BX0_BIF_DOORBELL_CNTL__SELF_RING_DIS__SHIFT                                                       0x0
+#define BIF_BX0_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS__SHIFT                                                     0x1
+#define BIF_BX0_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN__SHIFT                                                    0x2
+#define BIF_BX0_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS__SHIFT                                         0x3
+#define BIF_BX0_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN__SHIFT                                                 0x4
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS__SHIFT                                                  0x18
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0__SHIFT                                               0x19
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1__SHIFT                                               0x1a
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2__SHIFT                                               0x1b
+#define BIF_BX0_BIF_DOORBELL_CNTL__SELF_RING_DIS_MASK                                                         0x00000001L
+#define BIF_BX0_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS_MASK                                                       0x00000002L
+#define BIF_BX0_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN_MASK                                                      0x00000004L
+#define BIF_BX0_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS_MASK                                           0x00000008L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN_MASK                                                   0x00000010L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS_MASK                                                    0x01000000L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0_MASK                                                 0x02000000L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1_MASK                                                 0x04000000L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2_MASK                                                 0x08000000L
+//BIF_BX0_BIF_DOORBELL_INT_CNTL
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS__SHIFT                                       0x0
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS__SHIFT                                      0x1
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS__SHIFT                            0x2
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR__SHIFT                                        0x10
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR__SHIFT                                       0x11
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR__SHIFT                             0x12
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                            0x17
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE__SHIFT                                      0x18
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE__SHIFT                                     0x19
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE__SHIFT                           0x1a
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                               0x1c
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                          0x1d
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                          0x1e
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                              0x1f
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS_MASK                                         0x00000001L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS_MASK                                        0x00000002L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS_MASK                              0x00000004L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR_MASK                                          0x00010000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR_MASK                                         0x00020000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR_MASK                               0x00040000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE_MASK                              0x00800000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE_MASK                                        0x01000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE_MASK                                       0x02000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE_MASK                             0x04000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE_MASK                                 0x10000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                            0x20000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                            0x40000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE_MASK                                0x80000000L
+//BIF_BX0_BIF_FB_EN
+#define BIF_BX0_BIF_FB_EN__FB_READ_EN__SHIFT                                                                  0x0
+#define BIF_BX0_BIF_FB_EN__FB_WRITE_EN__SHIFT                                                                 0x1
+#define BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK                                                                    0x00000001L
+#define BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK                                                                   0x00000002L
+//BIF_BX0_BIF_INTR_CNTL
+#define BIF_BX0_BIF_INTR_CNTL__RAS_INTR_VEC_SEL__SHIFT                                                        0x0
+#define BIF_BX0_BIF_INTR_CNTL__RAS_INTR_VEC_SEL_MASK                                                          0x00000001L
+//BIF_BX0_BIF_MST_TRANS_PENDING_VF
+#define BIF_BX0_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX0_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX0_BIF_SLV_TRANS_PENDING_VF
+#define BIF_BX0_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX0_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX0_BACO_CNTL
+#define BIF_BX0_BACO_CNTL__BACO_EN__SHIFT                                                                     0x0
+#define BIF_BX0_BACO_CNTL__BACO_DUMMY_EN__SHIFT                                                               0x2
+#define BIF_BX0_BACO_CNTL__BACO_POWER_OFF__SHIFT                                                              0x3
+#define BIF_BX0_BACO_CNTL__BACO_DSTATE_BYPASS__SHIFT                                                          0x5
+#define BIF_BX0_BACO_CNTL__BACO_RST_INTR_MASK__SHIFT                                                          0x6
+#define BIF_BX0_BACO_CNTL__BACO_MODE__SHIFT                                                                   0x8
+#define BIF_BX0_BACO_CNTL__RCU_BIF_CONFIG_DONE__SHIFT                                                         0x9
+#define BIF_BX0_BACO_CNTL__PWRGOOD_VDDSOC__SHIFT                                                              0x10
+#define BIF_BX0_BACO_CNTL__BACO_AUTO_EXIT__SHIFT                                                              0x1f
+#define BIF_BX0_BACO_CNTL__BACO_EN_MASK                                                                       0x00000001L
+#define BIF_BX0_BACO_CNTL__BACO_DUMMY_EN_MASK                                                                 0x00000004L
+#define BIF_BX0_BACO_CNTL__BACO_POWER_OFF_MASK                                                                0x00000008L
+#define BIF_BX0_BACO_CNTL__BACO_DSTATE_BYPASS_MASK                                                            0x00000020L
+#define BIF_BX0_BACO_CNTL__BACO_RST_INTR_MASK_MASK                                                            0x00000040L
+#define BIF_BX0_BACO_CNTL__BACO_MODE_MASK                                                                     0x00000100L
+#define BIF_BX0_BACO_CNTL__RCU_BIF_CONFIG_DONE_MASK                                                           0x00000200L
+#define BIF_BX0_BACO_CNTL__PWRGOOD_VDDSOC_MASK                                                                0x00010000L
+#define BIF_BX0_BACO_CNTL__BACO_AUTO_EXIT_MASK                                                                0x80000000L
+//BIF_BX0_BIF_BACO_EXIT_TIME0
+#define BIF_BX0_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER__SHIFT                                          0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER_MASK                                            0x000FFFFFL
+//BIF_BX0_BIF_BACO_EXIT_TIMER1
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER__SHIFT                                         0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN__SHIFT                                            0x18
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS__SHIFT                                                 0x1a
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH__SHIFT                                           0x1b
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW__SHIFT                                            0x1c
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL__SHIFT                                                    0x1d
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS__SHIFT                                     0x1f
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER_MASK                                           0x000FFFFFL
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN_MASK                                              0x01000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS_MASK                                                   0x04000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH_MASK                                             0x08000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW_MASK                                              0x10000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL_MASK                                                      0x60000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS_MASK                                       0x80000000L
+//BIF_BX0_BIF_BACO_EXIT_TIMER2
+#define BIF_BX0_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER__SHIFT                                         0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER_MASK                                           0x000FFFFFL
+//BIF_BX0_BIF_BACO_EXIT_TIMER3
+#define BIF_BX0_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER__SHIFT                                     0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER_MASK                                       0x000FFFFFL
+//BIF_BX0_BIF_BACO_EXIT_TIMER4
+#define BIF_BX0_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER__SHIFT                                      0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER_MASK                                        0x000FFFFFL
+//BIF_BX0_MEM_TYPE_CNTL
+#define BIF_BX0_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3__SHIFT                                                        0x0
+#define BIF_BX0_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3_MASK                                                          0x00000001L
+//BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE__SHIFT                                                     0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE__SHIFT                                                  0x1
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE__SHIFT                                                    0x8
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE_MASK                                                       0x00000001L
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE_MASK                                                    0x00000002L
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE_MASK                                                      0x00000100L
+//BIF_BX0_NBIF_GFX_ADDR_LUT_0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_0__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_0__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_1
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_1__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_1__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_2
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_2__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_2__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_3
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_3__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_3__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_4
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_4__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_4__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_5
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_5__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_5__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_6
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_6__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_6__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_7
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_7__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_7__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_8
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_8__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_8__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_9
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_9__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_9__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_10
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_10__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_10__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_11
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_11__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_11__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_12
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_12__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_12__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_13
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_13__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_13__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_14
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_14__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_14__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_15
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_15__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_15__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL
+#define BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL
+#define BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX0_BIF_RB_CNTL
+#define BIF_BX0_BIF_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define BIF_BX0_BIF_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define BIF_BX0_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT                                                     0x8
+#define BIF_BX0_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER__SHIFT                                                      0x9
+#define BIF_BX0_BIF_RB_CNTL__BIF_RB_TRAN__SHIFT                                                               0x11
+#define BIF_BX0_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL__SHIFT                                                  0x19
+#define BIF_BX0_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY__SHIFT                                                      0x1a
+#define BIF_BX0_BIF_RB_CNTL__RB_INTR_ARB_MODE__SHIFT                                                          0x1d
+#define BIF_BX0_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE__SHIFT                                                     0x1e
+#define BIF_BX0_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT                                                       0x1f
+#define BIF_BX0_BIF_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define BIF_BX0_BIF_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define BIF_BX0_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK                                                       0x00000100L
+#define BIF_BX0_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER_MASK                                                        0x00003E00L
+#define BIF_BX0_BIF_RB_CNTL__BIF_RB_TRAN_MASK                                                                 0x00020000L
+#define BIF_BX0_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL_MASK                                                    0x02000000L
+#define BIF_BX0_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY_MASK                                                        0x1C000000L
+#define BIF_BX0_BIF_RB_CNTL__RB_INTR_ARB_MODE_MASK                                                            0x20000000L
+#define BIF_BX0_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE_MASK                                                       0x40000000L
+#define BIF_BX0_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK                                                         0x80000000L
+//BIF_BX0_BIF_RB_BASE
+#define BIF_BX0_BIF_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define BIF_BX0_BIF_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//BIF_BX0_BIF_RB_RPTR
+#define BIF_BX0_BIF_RB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define BIF_BX0_BIF_RB_RPTR__OFFSET_MASK                                                                      0x0003FFFCL
+//BIF_BX0_BIF_RB_WPTR
+#define BIF_BX0_BIF_RB_WPTR__BIF_RB_OVERFLOW__SHIFT                                                           0x0
+#define BIF_BX0_BIF_RB_WPTR__OFFSET__SHIFT                                                                    0x2
+#define BIF_BX0_BIF_RB_WPTR__BIF_RB_OVERFLOW_MASK                                                             0x00000001L
+#define BIF_BX0_BIF_RB_WPTR__OFFSET_MASK                                                                      0x0003FFFCL
+//BIF_BX0_BIF_RB_WPTR_ADDR_HI
+#define BIF_BX0_BIF_RB_WPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_BIF_RB_WPTR_ADDR_HI__ADDR_MASK                                                                0x000000FFL
+//BIF_BX0_BIF_RB_WPTR_ADDR_LO
+#define BIF_BX0_BIF_RB_WPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define BIF_BX0_BIF_RB_WPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//BIF_BX0_MAILBOX_INDEX
+#define BIF_BX0_MAILBOX_INDEX__MAILBOX_INDEX__SHIFT                                                           0x0
+#define BIF_BX0_MAILBOX_INDEX__MAILBOX_INDEX_MASK                                                             0x0000001FL
+//BIF_BX0_BIF_MP1_INTR_CTRL
+#define BIF_BX0_BIF_MP1_INTR_CTRL__BACO_EXIT_DONE__SHIFT                                                      0x0
+#define BIF_BX0_BIF_MP1_INTR_CTRL__BACO_EXIT_DONE_MASK                                                        0x00000001L
+//BIF_BX0_BIF_PERSTB_PAD_CNTL
+#define BIF_BX0_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX0_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL_MASK                                                     0x0000FFFFL
+//BIF_BX0_BIF_PX_EN_PAD_CNTL
+#define BIF_BX0_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL__SHIFT                                                     0x0
+#define BIF_BX0_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL_MASK                                                       0x00000FFFL
+//BIF_BX0_BIF_REFPADKIN_PAD_CNTL
+#define BIF_BX0_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL__SHIFT                                             0x0
+#define BIF_BX0_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL_MASK                                               0x000000FFL
+//BIF_BX0_BIF_CLKREQB_PAD_CNTL
+#define BIF_BX0_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL__SHIFT                                                 0x0
+#define BIF_BX0_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_MASK                                                   0x7FFFFFFFL
+//BIF_BX0_BIF_PWRBRK_PAD_CNTL
+#define BIF_BX0_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX0_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL_MASK                                                     0x000000FFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_BIFDEC1
+//RCC_DEV0_0_RCC_ERR_INT_CNTL
+#define RCC_DEV0_0_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN__SHIFT                                0x0
+#define RCC_DEV0_0_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN_MASK                                  0x00000001L
+//RCC_DEV0_0_RCC_BACO_CNTL_MISC
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS__SHIFT                                                 0x0
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS__SHIFT                                                  0x1
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS_MASK                                                   0x00000001L
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS_MASK                                                    0x00000002L
+//RCC_DEV0_0_RCC_RESET_EN
+#define RCC_DEV0_0_RCC_RESET_EN__DB_APER_RESET_EN__SHIFT                                                      0xf
+#define RCC_DEV0_0_RCC_RESET_EN__DB_APER_RESET_EN_MASK                                                        0x00008000L
+//RCC_DEV0_0_RCC_VDM_SUPPORT
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                       0x0
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                      0x1
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                  0x2
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                        0x3
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                    0x4
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                         0x00000001L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                        0x00000002L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                    0x00000004L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                          0x00000008L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                      0x00000010L
+//RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                 0x0
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                              0x1
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                0x2
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                 0x3
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                           0x4
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                  0x5
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                 0xb
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                 0x12
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                0x19
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                   0x00000001L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                0x00000002L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                  0x00000004L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                   0x00000008L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                             0x00000010L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                    0x000007E0L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                   0x0003F800L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                   0x01FC0000L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                  0xFE000000L
+//RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                             0x0
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                              0x6
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                         0xc
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                      0x11
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                               0x0000003FL
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                0x00000FC0L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                           0x0001F000L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                        0x00FE0000L
+//RCC_DEV0_0_RCC_GPUIOV_REGION
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__LFB_REGION__SHIFT                                                       0x0
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__MAX_REGION__SHIFT                                                       0x4
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__LFB_REGION_MASK                                                         0x0000000FL
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__MAX_REGION_MASK                                                         0x000000F0L
+//RCC_DEV0_0_RCC_GPU_HOSTVM_EN
+#define RCC_DEV0_0_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN__SHIFT                                                    0x0
+#define RCC_DEV0_0_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN_MASK                                                      0x00000001L
+//RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE__SHIFT                              0x0
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN__SHIFT                                    0x1
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE_MASK                                0x00000001L
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN_MASK                                      0x00000002L
+//RCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET__SHIFT                        0x0
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET_MASK                          0xFFFFL
+//RCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE__SHIFT                                    0x0
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE_MASK                                      0xFFFFL
+//RCC_DEV0_0_RCC_PEER_REG_RANGE0
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_0_RCC_PEER_REG_RANGE1
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_0_RCC_BUS_CNTL
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                            0x2
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                           0x3
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                            0x4
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                         0x5
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                        0x6
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                         0x7
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                        0x8
+#define RCC_DEV0_0_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                 0xc
+#define RCC_DEV0_0_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                           0xd
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x10
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x11
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x12
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x13
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x14
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x15
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                 0x18
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                 0x19
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                            0x1c
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                            0x1d
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                              0x00000004L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                             0x00000008L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                              0x00000010L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                           0x00000020L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                          0x00000040L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                           0x00000080L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                          0x00000100L
+#define RCC_DEV0_0_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                   0x00001000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                             0x00002000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00010000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00020000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00040000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00080000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00100000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00200000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                   0x01000000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                   0x0E000000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                              0x10000000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                              0xE0000000L
+//RCC_DEV0_0_RCC_CONFIG_CNTL
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B__SHIFT                                               0x2
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GRPH_ADRSEL__SHIFT                                                        0x3
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN_MASK                                                       0x00000001L
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B_MASK                                                 0x00000004L
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GRPH_ADRSEL_MASK                                                          0x00000018L
+//RCC_DEV0_0_RCC_CONFIG_F0_BASE
+#define RCC_DEV0_0_RCC_CONFIG_F0_BASE__F0_BASE__SHIFT                                                         0x0
+#define RCC_DEV0_0_RCC_CONFIG_F0_BASE__F0_BASE_MASK                                                           0xFFFFFFFFL
+//RCC_DEV0_0_RCC_CONFIG_APER_SIZE
+#define RCC_DEV0_0_RCC_CONFIG_APER_SIZE__APER_SIZE__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_CONFIG_APER_SIZE__APER_SIZE_MASK                                                       0xFFFFFFFFL
+//RCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE
+#define RCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE__SHIFT                                             0x0
+#define RCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE_MASK                                               0x07FFFFFFL
+//RCC_DEV0_0_RCC_XDMA_LO
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_APER_EN__SHIFT                                                       0x1f
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND_MASK                                                     0x7FFFFFFFL
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_APER_EN_MASK                                                         0x80000000L
+//RCC_DEV0_0_RCC_XDMA_HI
+#define RCC_DEV0_0_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND_MASK                                                     0x7FFFFFFFL
+//RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS__SHIFT                                   0x7
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                 0x8
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                    0x9
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                    0xa
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                 0xb
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                  0xc
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                      0xd
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                      0xe
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                         0xf
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                 0x10
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                           0x11
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                               0x12
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                     0x13
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS_MASK                                     0x00000080L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                   0x00000100L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                      0x00000200L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                      0x00000400L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                   0x00000800L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                    0x00001000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                        0x00002000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                        0x00004000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                           0x00008000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                   0x00010000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                             0x00020000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                 0x00040000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                       0x00080000L
+//RCC_DEV0_0_RCC_BUSNUM_CNTL1
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL1__ID_MASK__SHIFT                                                           0x0
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL1__ID_MASK_MASK                                                             0x000000FFL
+//RCC_DEV0_0_RCC_BUSNUM_LIST0
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID0__SHIFT                                                               0x0
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID1__SHIFT                                                               0x8
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID2__SHIFT                                                               0x10
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID3__SHIFT                                                               0x18
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID0_MASK                                                                 0x000000FFL
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID1_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID2_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID3_MASK                                                                 0xFF000000L
+//RCC_DEV0_0_RCC_BUSNUM_LIST1
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID4__SHIFT                                                               0x0
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID5__SHIFT                                                               0x8
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID6__SHIFT                                                               0x10
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID7__SHIFT                                                               0x18
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID4_MASK                                                                 0x000000FFL
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID5_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID6_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID7_MASK                                                                 0xFF000000L
+//RCC_DEV0_0_RCC_BUSNUM_CNTL2
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL__SHIFT                                                    0x0
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN__SHIFT                                                     0x8
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__HDPREG_CNTL__SHIFT                                                       0x10
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH__SHIFT                                           0x11
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL_MASK                                                      0x000000FFL
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN_MASK                                                       0x00000100L
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__HDPREG_CNTL_MASK                                                         0x00010000L
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH_MASK                                             0x00020000L
+//RCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM
+#define RCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN_MASK                                                     0x00000001L
+//RCC_DEV0_0_RCC_HOST_BUSNUM
+#define RCC_DEV0_0_RCC_HOST_BUSNUM__HOST_ID__SHIFT                                                            0x0
+#define RCC_DEV0_0_RCC_HOST_BUSNUM__HOST_ID_MASK                                                              0x0000FFFFL
+//RCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1__SHIFT                                                   0x8
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2__SHIFT                                                   0x10
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3__SHIFT                                                   0x18
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0_MASK                                                     0x000000FFL
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1_MASK                                                     0x0000FF00L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2_MASK                                                     0x00FF0000L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3_MASK                                                     0xFF000000L
+//RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5__SHIFT                                                   0x8
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6__SHIFT                                                   0x10
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7__SHIFT                                                   0x18
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4_MASK                                                     0x000000FFL
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5_MASK                                                     0x0000FF00L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6_MASK                                                     0x00FF0000L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7_MASK                                                     0xFF000000L
+//RCC_DEV0_0_RCC_DEV0_LINK_CNTL
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                  0x0
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                 0x8
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                            0x10
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                            0x11
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                    0x00000001L
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                   0x00000100L
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                              0x00010000L
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                              0x00020000L
+//RCC_DEV0_0_RCC_CMN_LINK_CNTL
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                             0x0
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                              0x1
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                             0x2
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                          0x3
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                             0x10
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                               0x00000001L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                0x00000002L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                               0x00000004L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                            0x00000008L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                               0xFFFF0000L
+//RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                            0x0
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                            0x8
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                              0x000000FFL
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                              0x00001F00L
+//RCC_DEV0_0_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV0_0_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_DEV0_0_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                           0x000003FFL
+//RCC_DEV0_0_RCC_MH_ARB_CNTL
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                        0x0
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                0x1
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                          0x00000001L
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                  0x00007FFEL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_BIFDEC2
+//RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                                 0x1
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_2__SHIFT                                                 0x2
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_3__SHIFT                                                 0x3
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                                   0x00000001L
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                                   0x00000002L
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_2_MASK                                                   0x00000004L
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_3_MASK                                                   0x00000008L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_strap_BIFDEC1
+//RCC_STRAP0_RCC_BIF_STRAP0
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN4_DIS__SHIFT                                                      0x0
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT__SHIFT                              0x1
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_VGA_DIS_PIN__SHIFT                                                   0x2
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_MEM_AP_SIZE_PIN__SHIFT                                               0x3
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN__SHIFT                                               0x6
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PX_CAPABLE__SHIFT                                                    0x7
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3__SHIFT                                                 0x8
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN__SHIFT                                  0x9
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR__SHIFT                                         0xa
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN__SHIFT                                     0xb
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR__SHIFT                                              0xc
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR__SHIFT                                             0xd
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                     0xe
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                  0xf
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR__SHIFT                                              0x10
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR_DN__SHIFT                                           0x11
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_AUD_PIN__SHIFT                                                       0x12
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN3_DIS__SHIFT                                                      0x18
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4__SHIFT                                                 0x19
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_QUICKSIM_START__SHIFT                                                0x1a
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS__SHIFT                                   0x1c
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN__SHIFT                                      0x1d
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE__SHIFT                                                   0x1e
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN__SHIFT                                            0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN4_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT_MASK                                0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_VGA_DIS_PIN_MASK                                                     0x00000004L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_MEM_AP_SIZE_PIN_MASK                                                 0x00000038L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN_MASK                                                 0x00000040L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK                                                      0x00000080L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3_MASK                                                   0x00000100L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN_MASK                                    0x00000200L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR_MASK                                           0x00000400L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN_MASK                                       0x00000800L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR_MASK                                                0x00001000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR_MASK                                               0x00002000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                       0x00004000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                    0x00008000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR_MASK                                                0x00010000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR_DN_MASK                                             0x00020000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_AUD_PIN_MASK                                                         0x000C0000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN3_DIS_MASK                                                        0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4_MASK                                                   0x02000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_QUICKSIM_START_MASK                                                  0x04000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING_MASK                                       0x08000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS_MASK                                     0x10000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN_MASK                                        0x20000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE_MASK                                                     0x40000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN_MASK                                              0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP1
+#define RCC_STRAP0_RCC_BIF_STRAP1__FUSESTRAP_VALID__SHIFT                                                     0x0
+#define RCC_STRAP0_RCC_BIF_STRAP1__ROMSTRAP_VALID__SHIFT                                                      0x1
+#define RCC_STRAP0_RCC_BIF_STRAP1__WRITE_DISABLE__SHIFT                                                       0x2
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN__SHIFT                                      0x3
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS__SHIFT                                     0x5
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE__SHIFT                                       0x6
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_READY__SHIFT                                               0x7
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN__SHIFT                                                  0x8
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN__SHIFT                                                 0x9
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE__SHIFT                                                  0xa
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE__SHIFT                                        0xc
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_HWREV_LSB2__SHIFT                                                    0xd
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWREV_LSB2__SHIFT                                                    0xf
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY__SHIFT                                             0x11
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS__SHIFT                                             0x12
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN__SHIFT                                                        0x13
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN__SHIFT                                                   0x14
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGIN_EN__SHIFT                                                     0x15
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN__SHIFT                                             0x16
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN__SHIFT                                     0x17
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_S5_REGS_ACCESS_DIS__SHIFT                                            0x18
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_S5_MMREG_WR_POSTED_EN__SHIFT                                         0x19
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE__SHIFT                                             0x1a
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN__SHIFT                                       0x1b
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN_EP__SHIFT                                                     0x1d
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN__SHIFT                                                         0x1e
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN_DN__SHIFT                                                      0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP1__FUSESTRAP_VALID_MASK                                                       0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP1__ROMSTRAP_VALID_MASK                                                        0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP1__WRITE_DISABLE_MASK                                                         0x00000004L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN_MASK                                        0x00000008L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS_MASK                                       0x00000020L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE_MASK                                         0x00000040L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_READY_MASK                                                 0x00000080L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN_MASK                                                    0x00000100L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN_MASK                                                   0x00000200L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE_MASK                                                    0x00000C00L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE_MASK                                          0x00001000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_HWREV_LSB2_MASK                                                      0x00006000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWREV_LSB2_MASK                                                      0x00018000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY_MASK                                               0x00020000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS_MASK                                               0x00040000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN_MASK                                                          0x00080000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN_MASK                                                     0x00100000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGIN_EN_MASK                                                       0x00200000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN_MASK                                               0x00400000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN_MASK                                       0x00800000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_S5_REGS_ACCESS_DIS_MASK                                              0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_S5_MMREG_WR_POSTED_EN_MASK                                           0x02000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE_MASK                                               0x04000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN_MASK                                         0x18000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN_EP_MASK                                                       0x20000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN_MASK                                                           0x40000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN_DN_MASK                                                        0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP2
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS__SHIFT                                            0x3
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS__SHIFT                                            0x4
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN__SHIFT                                          0x7
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP0_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS_MASK                                              0x00000008L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS_MASK                                              0x00000010L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN_MASK                                            0x00000080L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP0_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP3
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP0_RCC_BIF_STRAP4
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER__SHIFT                                          0x0
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER__SHIFT                                           0x10
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER_MASK                                            0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER_MASK                                             0xFFFF0000L
+//RCC_STRAP0_RCC_BIF_STRAP5
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+//RCC_STRAP0_RCC_BIF_STRAP6
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_GEN5_DIS__SHIFT                                                      0x0
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5__SHIFT                                                 0x1
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN__SHIFT                                                   0x2
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_GEN5_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5_MASK                                                   0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN_MASK                                                     0x00000004L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0__SHIFT                                0x13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0__SHIFT                                   0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0__SHIFT                            0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0__SHIFT                             0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0__SHIFT                             0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0__SHIFT                                      0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0_MASK                                            0x00010000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0_MASK                                  0x00080000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0_MASK                                     0x00E00000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0_MASK                              0x01000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0_MASK                               0x0E000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0_MASK                               0x70000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0_MASK                                        0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0__SHIFT                                   0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0_MASK                                     0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0__SHIFT                           0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0__SHIFT                  0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0__SHIFT                         0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0__SHIFT                       0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0__SHIFT                                0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0_MASK                                0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0_MASK                             0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0_MASK                    0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0_MASK                           0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0_MASK                         0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0_MASK                                  0x0007FFC0L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0__SHIFT                             0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0__SHIFT                                 0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0__SHIFT                                      0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0__SHIFT                                         0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0__SHIFT                               0x1e
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0_MASK                               0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0_MASK                                   0x0FFF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0_MASK                                        0x10000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0_MASK                                           0x20000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0_MASK                                 0x40000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0__SHIFT                                0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0_MASK                                  0x00FFFFFFL
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0__SHIFT                          0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0__SHIFT     0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0__SHIFT                        0x9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0__SHIFT                               0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0_MASK                            0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0_MASK       0x00000100L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0_MASK                          0x000FFE00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0_MASK                                 0xFFF00000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0__SHIFT                               0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0__SHIFT                             0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0__SHIFT                                 0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0__SHIFT                          0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0__SHIFT                                         0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0_MASK                                 0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0_MASK                               0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0_MASK                                   0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0_MASK                            0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0_MASK                                           0x00000010L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0__SHIFT                                          0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0__SHIFT                                      0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0__SHIFT                                          0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0__SHIFT                                      0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0__SHIFT                                        0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ERR_REPORTING_DIS_DEV0__SHIFT                                  0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0__SHIFT                             0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0__SHIFT                                0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0__SHIFT                                    0x9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0__SHIFT                              0xc
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0__SHIFT                      0xd
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0__SHIFT                                    0xe
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0__SHIFT                                            0xf
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0__SHIFT                                    0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0__SHIFT                                    0x11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0__SHIFT                             0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0__SHIFT                                   0x17
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0__SHIFT                              0x1a
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0__SHIFT                                    0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0_MASK                                   0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0_MASK                                            0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0_MASK                                        0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0_MASK                                            0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0_MASK                                        0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0_MASK                                          0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ERR_REPORTING_DIS_DEV0_MASK                                    0x00000040L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0_MASK                               0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0_MASK                                  0x00000100L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0_MASK                                      0x00000E00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0_MASK                                0x00001000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0_MASK                        0x00002000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0_MASK                                      0x00004000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0_MASK                                              0x00008000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0_MASK                                      0x00010000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0_MASK                                      0x00020000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0_MASK                               0x00700000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0_MASK                                     0x03800000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0_MASK                                0x1C000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0_MASK                                      0xE0000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0__SHIFT                     0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0__SHIFT                                             0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0__SHIFT                                          0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0__SHIFT                                0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0__SHIFT                                          0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0__SHIFT                                  0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0__SHIFT                                     0x9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0xb
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0xe
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0__SHIFT                                         0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0__SHIFT                                      0x1b
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0__SHIFT                                       0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0__SHIFT                                         0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0_MASK                       0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0_MASK                                               0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0_MASK                                            0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0_MASK                                  0x00000038L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0_MASK                                            0x00000040L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0_MASK                                    0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0_MASK                                       0x00000600L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x00003800L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0_MASK  0x0003C000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x001C0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0_MASK  0x01E00000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0_MASK                                           0x06000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0_MASK                                        0x18000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0_MASK                                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0_MASK                                           0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0__SHIFT                              0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0__SHIFT                              0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0_MASK                                0xFF000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0__SHIFT                        0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0__SHIFT                                 0x11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0__SHIFT                                  0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0__SHIFT                                           0x13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0__SHIFT                                           0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0__SHIFT                                        0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0__SHIFT                                0x16
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0__SHIFT                           0x17
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0__SHIFT                        0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0__SHIFT                        0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0__SHIFT                     0x1a
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0__SHIFT                         0x1b
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0__SHIFT                          0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0__SHIFT                       0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0__SHIFT                                            0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0_MASK                          0x00010000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0_MASK                                   0x00020000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0_MASK                                    0x00040000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0_MASK                                             0x00080000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0_MASK                                             0x00100000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0_MASK                                          0x00200000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0_MASK                                  0x00400000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0_MASK                             0x00800000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0_MASK                          0x01000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0_MASK                          0x02000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0_MASK                       0x04000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0_MASK                           0x08000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0_MASK                            0x10000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0_MASK                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0_MASK                                              0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0__SHIFT                         0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0__SHIFT                                    0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                      0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                      0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                   0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0xc
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0__SHIFT                              0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0__SHIFT                            0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0__SHIFT                     0x13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0__SHIFT                                  0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0_MASK                                           0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0_MASK                           0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0_MASK                                      0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                        0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                        0x00000040L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                     0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x00000F00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0x0000F000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0_MASK                                0x00030000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0_MASK                              0x00040000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0_MASK                       0x00080000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0_MASK                                      0x00100000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0_MASK                                    0x00E00000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x0F000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0xF0000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0__SHIFT                                        0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0__SHIFT                                    0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0__SHIFT                                    0xc
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0__SHIFT                                          0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0__SHIFT                                          0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0_MASK                                          0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0_MASK                                      0x00000F00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0_MASK                                      0x0000F000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0_MASK                                            0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0_MASK                                            0x1F000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0_MASK                                            0xE0000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0__SHIFT                              0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0__SHIFT                              0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0_MASK                                0xFF000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0__SHIFT                                       0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0_MASK                                         0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP1
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0__SHIFT                       0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0_MASK                                0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0_MASK                         0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP13
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0__SHIFT                                 0x8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0__SHIFT                                0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0_MASK                                   0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0_MASK                                   0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0_MASK                                  0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0_MASK                                  0xFF000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0__SHIFT                                      0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0_MASK                                        0x0000FFFFL
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0__SHIFT                                  0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0__SHIFT                     0x19
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0__SHIFT                       0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0_MASK                                   0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0_MASK                                    0x00FFF000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0_MASK                                        0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0_MASK                       0x3E000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0_MASK                         0x40000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0__SHIFT                               0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0_MASK                                     0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0_MASK                                 0x00FFF000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0__SHIFT                                   0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0__SHIFT                                0xd
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0_MASK                                0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0_MASK                                     0x00001000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0_MASK                                  0x01FFE000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0__SHIFT                            0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0_MASK                              0x00000FFFL
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP2
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0__SHIFT                                        0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0__SHIFT                                       0x6
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0__SHIFT                                   0x7
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0__SHIFT                                 0x9
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0__SHIFT                          0xe
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0__SHIFT                                          0xf
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0__SHIFT                                0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0__SHIFT                                          0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0__SHIFT                                           0x17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0__SHIFT                                   0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0__SHIFT                                        0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0__SHIFT                  0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0__SHIFT               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0__SHIFT                       0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0_MASK                                          0x00000001L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0_MASK                                         0x00000040L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0_MASK                                     0x00000080L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0_MASK                                   0x00003E00L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0_MASK                            0x00004000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0_MASK                                            0x00008000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0_MASK                                            0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0_MASK                                  0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0_MASK                                            0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0_MASK                                             0x00800000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0_MASK                                     0x07000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0_MASK                                       0x08000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0_MASK                                          0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0_MASK                    0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0_MASK                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0_MASK                         0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP26
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP26__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0__SHIFT                             0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP26__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0_MASK                               0x00000FFFL
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP3
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0__SHIFT                      0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0__SHIFT                              0x13
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0__SHIFT                                         0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0__SHIFT                                  0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0__SHIFT                                         0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0__SHIFT                        0x1a
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0__SHIFT                       0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0__SHIFT                                0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0__SHIFT                                       0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0__SHIFT                               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0__SHIFT                                          0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0_MASK                        0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0_MASK                                0x00080000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0_MASK                                           0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0_MASK                                    0x00E00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0_MASK                                           0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0_MASK                          0x04000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0_MASK                         0x08000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0_MASK                                  0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0_MASK                                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0_MASK                                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0_MASK                                            0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP4
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0__SHIFT                                          0xa
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0__SHIFT                                       0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0__SHIFT                                     0x17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0__SHIFT                                   0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0_MASK                                   0x000003FFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0_MASK                                            0x00000400L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0_MASK                                         0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0_MASK                                       0x0F800000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0_MASK                                     0x70000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP5
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0__SHIFT                            0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0_MASK                                     0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0_MASK                              0x40000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0__SHIFT                                0x3
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0__SHIFT                                     0x4
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0__SHIFT                                      0x7
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_MEM_AP_SIZE_DEV0_F0__SHIFT                                     0x9
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0__SHIFT                                     0xd
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                           0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MEM_AP_SIZE_DEV0_F0__SHIFT                                  0x13
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0__SHIFT                                  0x17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VGA_DIS_DEV0_F0__SHIFT                                         0x1a
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0__SHIFT                                0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0__SHIFT                           0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0_MASK                                0x00000007L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0_MASK                                  0x00000008L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0_MASK                                       0x00000070L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0_MASK                                        0x00000080L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_MEM_AP_SIZE_DEV0_F0_MASK                                       0x00001E00L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0_MASK                                       0x0000E000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0_MASK                             0x00070000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MEM_AP_SIZE_DEV0_F0_MASK                                    0x00780000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0_MASK                                    0x03800000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VGA_DIS_DEV0_F0_MASK                                           0x04000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0_MASK                                  0x38000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0_MASK                             0xC0000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP9
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0__SHIFT                           0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0__SHIFT                               0x12
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0__SHIFT                        0x13
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0__SHIFT                                    0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0__SHIFT                                 0x16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0__SHIFT                                 0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0_MASK                             0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0_MASK                                 0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0_MASK                          0x00080000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0_MASK                                      0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0_MASK                                   0x00C00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0_MASK                                   0x0F000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1__SHIFT                                    0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1__SHIFT                                         0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1__SHIFT                           0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1__SHIFT                                      0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1__SHIFT                                      0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1_MASK                                      0x000F0000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1_MASK                                      0x00F00000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1_MASK                                           0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1_MASK                             0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1_MASK                                        0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1_MASK                                        0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP2
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1__SHIFT                                   0x7
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1__SHIFT                                 0x9
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1__SHIFT                          0xe
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1__SHIFT                                0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1__SHIFT                                          0x15
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_VC_EN_DEV0_F1__SHIFT                                           0x17
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1__SHIFT                                   0x18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1__SHIFT                                        0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1__SHIFT                  0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1__SHIFT               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1__SHIFT                       0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1_MASK                                     0x00000080L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1_MASK                                   0x00003E00L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1_MASK                            0x00004000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1_MASK                                            0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1_MASK                                  0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1_MASK                                            0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_VC_EN_DEV0_F1_MASK                                             0x00800000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1_MASK                                     0x07000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1_MASK                                          0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1_MASK                    0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1_MASK                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1_MASK                         0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP20
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP21
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP22
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP23
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP24
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP25
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP3
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1__SHIFT                      0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1__SHIFT                              0x13
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1__SHIFT                                         0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1__SHIFT                                         0x18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1__SHIFT                        0x1a
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1__SHIFT                       0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1__SHIFT                                       0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1__SHIFT                               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1__SHIFT                                          0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1_MASK                        0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1_MASK                                0x00080000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1_MASK                                           0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1_MASK                                           0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1_MASK                          0x04000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1_MASK                         0x08000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1_MASK                                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1_MASK                                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1_MASK                                            0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP4
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1__SHIFT                                 0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1__SHIFT                                       0x15
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1__SHIFT                                     0x17
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1__SHIFT                                   0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1__SHIFT                                  0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1_MASK                                   0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1_MASK                                         0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1_MASK                                       0x0F800000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1_MASK                                     0x70000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1_MASK                                    0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP5
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1__SHIFT                            0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1_MASK                                     0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1_MASK                                       0x38000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1_MASK                              0x40000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP6
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1_MASK                                          0x00000001L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP7
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_pf_BIFPFVFDEC1
+//BIF_BX_PF0_BIF_BME_STATUS
+#define BIF_BX_PF0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                                      0x0
+#define BIF_BX_PF0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                                0x10
+#define BIF_BX_PF0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                                        0x00000001L
+#define BIF_BX_PF0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                                  0x00010000L
+//BIF_BX_PF0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                                0x0
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                             0x1
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                                0x2
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                                    0x3
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                          0x10
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                                       0x11
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                          0x12
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                              0x13
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                                  0x00000001L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                               0x00000002L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                                  0x00000004L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                                      0x00000008L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                            0x00010000L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                                         0x00020000L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                            0x00040000L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                                0x00080000L
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT          0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK            0xFFFFFFFFL
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT            0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK              0xFFFFFFFFL
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT                      0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT                    0x1
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT                    0x8
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK                        0x00000001L
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK                      0x00000002L
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK                      0x000FFF00L
+//BIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                          0x0
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                            0x00000001L
+//BIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_PF0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                              0x0
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                              0x1
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                              0x2
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                              0x3
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                              0x4
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                              0x5
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                              0x6
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                              0x7
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                              0x8
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                              0x9
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                            0xa
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                            0xb
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                                        0xc
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                                        0xd
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                                        0xe
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                                        0xf
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                                        0x10
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                                        0x11
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                                        0x12
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                                        0x13
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                                        0x14
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                                        0x15
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                                       0x16
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                                       0x17
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                                       0x18
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                                       0x19
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                                       0x1a
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                                       0x1b
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                                       0x1c
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                                       0x1d
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                                       0x1e
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                                       0x1f
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                                0x00000001L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                                0x00000002L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                                0x00000004L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                                0x00000008L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                                0x00000010L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                                0x00000020L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                                0x00000040L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                                0x00000080L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                                0x00000100L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                                0x00000200L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                              0x00000400L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                              0x00000800L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                          0x00001000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                          0x00002000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                          0x00004000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                          0x00008000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                          0x00010000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                          0x00020000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                          0x00040000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                          0x00080000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                          0x00100000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                          0x00200000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                                         0x00400000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                                         0x00800000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                                         0x01000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                                         0x02000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                                         0x04000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                                         0x08000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                                         0x10000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                                         0x20000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                                         0x40000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                                         0x80000000L
+//BIF_BX_PF0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                             0x0
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                             0x1
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                             0x2
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                             0x3
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                             0x4
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                             0x5
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                             0x6
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                             0x7
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                             0x8
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                             0x9
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                           0xa
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                           0xb
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                                       0xc
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                                       0xd
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                                       0xe
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                                       0xf
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                                       0x10
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                                       0x11
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                                       0x12
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                                       0x13
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                                       0x14
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                                       0x15
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                                      0x16
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                                      0x17
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                                      0x18
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                                      0x19
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                                      0x1a
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                                      0x1b
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                                      0x1c
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                                      0x1d
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                                      0x1e
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                                      0x1f
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                               0x00000001L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                               0x00000002L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                               0x00000004L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                               0x00000008L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                               0x00000010L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                               0x00000020L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                               0x00000040L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                               0x00000080L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                               0x00000100L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                               0x00000200L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                             0x00000400L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                             0x00000800L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                                         0x00001000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                                         0x00002000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                                         0x00004000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                                         0x00008000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                                         0x00010000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                                         0x00020000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                                         0x00040000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                                         0x00080000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                                         0x00100000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                                         0x00200000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                                        0x00400000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                                        0x00800000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                                        0x01000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                                        0x02000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                                        0x04000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                                        0x08000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                                        0x10000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                                        0x20000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                                        0x40000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                                        0x80000000L
+//BIF_BX_PF0_BIF_TRANS_PENDING
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                            0x0
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                            0x1
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                              0x00000001L
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                              0x00000002L
+//BIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                                0x0
+#define BIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                                  0x00000001L
+//BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_CONTROL
+#define BIF_BX_PF0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                                      0x0
+#define BIF_BX_PF0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                                        0x1
+#define BIF_BX_PF0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                                      0x8
+#define BIF_BX_PF0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                                        0x9
+#define BIF_BX_PF0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                                        0x00000001L
+#define BIF_BX_PF0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                          0x00000002L
+#define BIF_BX_PF0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                                        0x00000100L
+#define BIF_BX_PF0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                          0x00000200L
+//BIF_BX_PF0_MAILBOX_INT_CNTL
+#define BIF_BX_PF0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                                      0x0
+#define BIF_BX_PF0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                                        0x1
+#define BIF_BX_PF0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                                        0x00000001L
+#define BIF_BX_PF0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                          0x00000002L
+//BIF_BX_PF0_BIF_VMHV_MAILBOX
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                                      0x0
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                                    0x1
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                                         0x8
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                                        0xf
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                                         0x10
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                                        0x17
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                          0x18
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                          0x19
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                                        0x00000001L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                                      0x00000002L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                           0x00000F00L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                          0x00008000L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                           0x000F0000L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                          0x00800000L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                            0x01000000L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                            0x02000000L
+//BIF_BX_PF0_PARTITION_COMPUTE_CAP
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                                  0x0
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                                  0x1
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                                  0x2
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                                  0x3
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                                  0x4
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                             0xa
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                                    0x00000001L
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                                    0x00000002L
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                                    0x00000004L
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                                    0x00000008L
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                                    0x00000010L
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                               0x0003FC00L
+//BIF_BX_PF0_PARTITION_MEM_CAP
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                                     0x0
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                                     0x1
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                                     0x2
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                                     0x3
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                                     0x5
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                                     0x7
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                                       0x00000001L
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                                       0x00000002L
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                                       0x00000004L
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                                       0x00000008L
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                                       0x00000020L
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                                       0x00000080L
+//BIF_BX_PF0_PARTITION_COMPUTE_STATUS
+#define BIF_BX_PF0_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                            0x4
+#define BIF_BX_PF0_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                              0x000000F0L
+//BIF_BX_PF0_PARTITION_MEM_STATUS
+#define BIF_BX_PF0_PARTITION_MEM_STATUS__CHANGE_STATUE__SHIFT                                                 0x0
+#define BIF_BX_PF0_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                                      0x4
+#define BIF_BX_PF0_PARTITION_MEM_STATUS__CHANGE_STATUE_MASK                                                   0x0000000FL
+#define BIF_BX_PF0_PARTITION_MEM_STATUS__NPS_MODE_MASK                                                        0x00000FF0L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_BIFPFVFDEC1[13440..14975]
+//RCC_DEV0_EPF0_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                                  0x0
+#define RCC_DEV0_EPF0_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                         0x1
+#define RCC_DEV0_EPF0_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                    0x00000001L
+#define RCC_DEV0_EPF0_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                           0x00000002L
+//RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                       0x0
+#define RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                         0x00000001L
+//RCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                              0x1f
+#define RCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                           0x00000001L
+#define RCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                                0x80000000L
+
+
+// addressBlock: aid_nbio_nbif0_gdc_GDCDEC
+//GDC0_A2S_CNTL_CL0
+#define GDC0_A2S_CNTL_CL0__NSNOOP_MAP__SHIFT                                                                  0x0
+#define GDC0_A2S_CNTL_CL0__REQPASSPW_VC0_MAP__SHIFT                                                           0x2
+#define GDC0_A2S_CNTL_CL0__REQPASSPW_NVC0_MAP__SHIFT                                                          0x4
+#define GDC0_A2S_CNTL_CL0__REQRSPPASSPW_VC0_MAP__SHIFT                                                        0x6
+#define GDC0_A2S_CNTL_CL0__REQRSPPASSPW_NVC0_MAP__SHIFT                                                       0x8
+#define GDC0_A2S_CNTL_CL0__BLKLVL_MAP__SHIFT                                                                  0xa
+#define GDC0_A2S_CNTL_CL0__DATERR_MAP__SHIFT                                                                  0xc
+#define GDC0_A2S_CNTL_CL0__EXOKAY_WR_MAP__SHIFT                                                               0xe
+#define GDC0_A2S_CNTL_CL0__EXOKAY_RD_MAP__SHIFT                                                               0x10
+#define GDC0_A2S_CNTL_CL0__RESP_WR_MAP__SHIFT                                                                 0x12
+#define GDC0_A2S_CNTL_CL0__RESP_RD_MAP__SHIFT                                                                 0x14
+#define GDC0_A2S_CNTL_CL0__RDRSP_ERRMAP__SHIFT                                                                0x16
+#define GDC0_A2S_CNTL_CL0__RDRSP_SEL_MODE__SHIFT                                                              0x18
+#define GDC0_A2S_CNTL_CL0__STATIC_VC_ENABLE__SHIFT                                                            0x1b
+#define GDC0_A2S_CNTL_CL0__STATIC_VC_VALUE__SHIFT                                                             0x1c
+#define GDC0_A2S_CNTL_CL0__NSNOOP_MAP_MASK                                                                    0x00000003L
+#define GDC0_A2S_CNTL_CL0__REQPASSPW_VC0_MAP_MASK                                                             0x0000000CL
+#define GDC0_A2S_CNTL_CL0__REQPASSPW_NVC0_MAP_MASK                                                            0x00000030L
+#define GDC0_A2S_CNTL_CL0__REQRSPPASSPW_VC0_MAP_MASK                                                          0x000000C0L
+#define GDC0_A2S_CNTL_CL0__REQRSPPASSPW_NVC0_MAP_MASK                                                         0x00000300L
+#define GDC0_A2S_CNTL_CL0__BLKLVL_MAP_MASK                                                                    0x00000C00L
+#define GDC0_A2S_CNTL_CL0__DATERR_MAP_MASK                                                                    0x00003000L
+#define GDC0_A2S_CNTL_CL0__EXOKAY_WR_MAP_MASK                                                                 0x0000C000L
+#define GDC0_A2S_CNTL_CL0__EXOKAY_RD_MAP_MASK                                                                 0x00030000L
+#define GDC0_A2S_CNTL_CL0__RESP_WR_MAP_MASK                                                                   0x000C0000L
+#define GDC0_A2S_CNTL_CL0__RESP_RD_MAP_MASK                                                                   0x00300000L
+#define GDC0_A2S_CNTL_CL0__RDRSP_ERRMAP_MASK                                                                  0x00C00000L
+#define GDC0_A2S_CNTL_CL0__RDRSP_SEL_MODE_MASK                                                                0x07000000L
+#define GDC0_A2S_CNTL_CL0__STATIC_VC_ENABLE_MASK                                                              0x08000000L
+#define GDC0_A2S_CNTL_CL0__STATIC_VC_VALUE_MASK                                                               0x70000000L
+//GDC0_A2S_CNTL_CL1
+#define GDC0_A2S_CNTL_CL1__NSNOOP_MAP__SHIFT                                                                  0x0
+#define GDC0_A2S_CNTL_CL1__REQPASSPW_VC0_MAP__SHIFT                                                           0x2
+#define GDC0_A2S_CNTL_CL1__REQPASSPW_NVC0_MAP__SHIFT                                                          0x4
+#define GDC0_A2S_CNTL_CL1__REQRSPPASSPW_VC0_MAP__SHIFT                                                        0x6
+#define GDC0_A2S_CNTL_CL1__REQRSPPASSPW_NVC0_MAP__SHIFT                                                       0x8
+#define GDC0_A2S_CNTL_CL1__BLKLVL_MAP__SHIFT                                                                  0xa
+#define GDC0_A2S_CNTL_CL1__DATERR_MAP__SHIFT                                                                  0xc
+#define GDC0_A2S_CNTL_CL1__EXOKAY_WR_MAP__SHIFT                                                               0xe
+#define GDC0_A2S_CNTL_CL1__EXOKAY_RD_MAP__SHIFT                                                               0x10
+#define GDC0_A2S_CNTL_CL1__RESP_WR_MAP__SHIFT                                                                 0x12
+#define GDC0_A2S_CNTL_CL1__RESP_RD_MAP__SHIFT                                                                 0x14
+#define GDC0_A2S_CNTL_CL1__RDRSP_ERRMAP__SHIFT                                                                0x16
+#define GDC0_A2S_CNTL_CL1__RDRSP_SEL_MODE__SHIFT                                                              0x18
+#define GDC0_A2S_CNTL_CL1__STATIC_VC_ENABLE__SHIFT                                                            0x1b
+#define GDC0_A2S_CNTL_CL1__STATIC_VC_VALUE__SHIFT                                                             0x1c
+#define GDC0_A2S_CNTL_CL1__NSNOOP_MAP_MASK                                                                    0x00000003L
+#define GDC0_A2S_CNTL_CL1__REQPASSPW_VC0_MAP_MASK                                                             0x0000000CL
+#define GDC0_A2S_CNTL_CL1__REQPASSPW_NVC0_MAP_MASK                                                            0x00000030L
+#define GDC0_A2S_CNTL_CL1__REQRSPPASSPW_VC0_MAP_MASK                                                          0x000000C0L
+#define GDC0_A2S_CNTL_CL1__REQRSPPASSPW_NVC0_MAP_MASK                                                         0x00000300L
+#define GDC0_A2S_CNTL_CL1__BLKLVL_MAP_MASK                                                                    0x00000C00L
+#define GDC0_A2S_CNTL_CL1__DATERR_MAP_MASK                                                                    0x00003000L
+#define GDC0_A2S_CNTL_CL1__EXOKAY_WR_MAP_MASK                                                                 0x0000C000L
+#define GDC0_A2S_CNTL_CL1__EXOKAY_RD_MAP_MASK                                                                 0x00030000L
+#define GDC0_A2S_CNTL_CL1__RESP_WR_MAP_MASK                                                                   0x000C0000L
+#define GDC0_A2S_CNTL_CL1__RESP_RD_MAP_MASK                                                                   0x00300000L
+#define GDC0_A2S_CNTL_CL1__RDRSP_ERRMAP_MASK                                                                  0x00C00000L
+#define GDC0_A2S_CNTL_CL1__RDRSP_SEL_MODE_MASK                                                                0x07000000L
+#define GDC0_A2S_CNTL_CL1__STATIC_VC_ENABLE_MASK                                                              0x08000000L
+#define GDC0_A2S_CNTL_CL1__STATIC_VC_VALUE_MASK                                                               0x70000000L
+//GDC0_A2S_CNTL3_CL0
+#define GDC0_A2S_CNTL3_CL0__FORCE_WR_PH__SHIFT                                                                0x0
+#define GDC0_A2S_CNTL3_CL0__FORCE_WR_STEERING__SHIFT                                                          0x2
+#define GDC0_A2S_CNTL3_CL0__WR_ST_TAG_MODE__SHIFT                                                             0x3
+#define GDC0_A2S_CNTL3_CL0__FORCE_WR_ST_ENTRY__SHIFT                                                          0x4
+#define GDC0_A2S_CNTL3_CL0__FORCE_WR_PH_MASK                                                                  0x00000003L
+#define GDC0_A2S_CNTL3_CL0__FORCE_WR_STEERING_MASK                                                            0x00000004L
+#define GDC0_A2S_CNTL3_CL0__WR_ST_TAG_MODE_MASK                                                               0x00000008L
+#define GDC0_A2S_CNTL3_CL0__FORCE_WR_ST_ENTRY_MASK                                                            0x000003F0L
+//GDC0_A2S_CNTL3_CL1
+#define GDC0_A2S_CNTL3_CL1__FORCE_WR_PH__SHIFT                                                                0x0
+#define GDC0_A2S_CNTL3_CL1__FORCE_WR_STEERING__SHIFT                                                          0x2
+#define GDC0_A2S_CNTL3_CL1__WR_ST_TAG_MODE__SHIFT                                                             0x3
+#define GDC0_A2S_CNTL3_CL1__FORCE_WR_ST_ENTRY__SHIFT                                                          0x4
+#define GDC0_A2S_CNTL3_CL1__FORCE_WR_PH_MASK                                                                  0x00000003L
+#define GDC0_A2S_CNTL3_CL1__FORCE_WR_STEERING_MASK                                                            0x00000004L
+#define GDC0_A2S_CNTL3_CL1__WR_ST_TAG_MODE_MASK                                                               0x00000008L
+#define GDC0_A2S_CNTL3_CL1__FORCE_WR_ST_ENTRY_MASK                                                            0x000003F0L
+//GDC0_A2S_CNTL_SW0
+#define GDC0_A2S_CNTL_SW0__STATIC_VC_ENABLE__SHIFT                                                            0x0
+#define GDC0_A2S_CNTL_SW0__STATIC_VC_VALUE__SHIFT                                                             0x1
+#define GDC0_A2S_CNTL_SW0__FORCE_RSP_REORDER_EN__SHIFT                                                        0x6
+#define GDC0_A2S_CNTL_SW0__SDP_WR_CHAIN_DIS__SHIFT                                                            0x9
+#define GDC0_A2S_CNTL_SW0__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                     0xa
+#define GDC0_A2S_CNTL_SW0__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                  0xb
+#define GDC0_A2S_CNTL_SW0__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                 0xc
+#define GDC0_A2S_CNTL_SW0__WRR_RD_WEIGHT__SHIFT                                                               0x10
+#define GDC0_A2S_CNTL_SW0__WRR_WR_WEIGHT__SHIFT                                                               0x18
+#define GDC0_A2S_CNTL_SW0__STATIC_VC_ENABLE_MASK                                                              0x00000001L
+#define GDC0_A2S_CNTL_SW0__STATIC_VC_VALUE_MASK                                                               0x0000000EL
+#define GDC0_A2S_CNTL_SW0__FORCE_RSP_REORDER_EN_MASK                                                          0x00000040L
+#define GDC0_A2S_CNTL_SW0__SDP_WR_CHAIN_DIS_MASK                                                              0x00000200L
+#define GDC0_A2S_CNTL_SW0__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                       0x00000400L
+#define GDC0_A2S_CNTL_SW0__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                    0x00000800L
+#define GDC0_A2S_CNTL_SW0__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                   0x00001000L
+#define GDC0_A2S_CNTL_SW0__WRR_RD_WEIGHT_MASK                                                                 0x00FF0000L
+#define GDC0_A2S_CNTL_SW0__WRR_WR_WEIGHT_MASK                                                                 0xFF000000L
+//GDC0_A2S_CNTL_SW1
+#define GDC0_A2S_CNTL_SW1__STATIC_VC_ENABLE__SHIFT                                                            0x0
+#define GDC0_A2S_CNTL_SW1__STATIC_VC_VALUE__SHIFT                                                             0x1
+#define GDC0_A2S_CNTL_SW1__FORCE_RSP_REORDER_EN__SHIFT                                                        0x6
+#define GDC0_A2S_CNTL_SW1__SDP_WR_CHAIN_DIS__SHIFT                                                            0x9
+#define GDC0_A2S_CNTL_SW1__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                     0xa
+#define GDC0_A2S_CNTL_SW1__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                  0xb
+#define GDC0_A2S_CNTL_SW1__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                 0xc
+#define GDC0_A2S_CNTL_SW1__WRR_RD_WEIGHT__SHIFT                                                               0x10
+#define GDC0_A2S_CNTL_SW1__WRR_WR_WEIGHT__SHIFT                                                               0x18
+#define GDC0_A2S_CNTL_SW1__STATIC_VC_ENABLE_MASK                                                              0x00000001L
+#define GDC0_A2S_CNTL_SW1__STATIC_VC_VALUE_MASK                                                               0x0000000EL
+#define GDC0_A2S_CNTL_SW1__FORCE_RSP_REORDER_EN_MASK                                                          0x00000040L
+#define GDC0_A2S_CNTL_SW1__SDP_WR_CHAIN_DIS_MASK                                                              0x00000200L
+#define GDC0_A2S_CNTL_SW1__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                       0x00000400L
+#define GDC0_A2S_CNTL_SW1__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                    0x00000800L
+#define GDC0_A2S_CNTL_SW1__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                   0x00001000L
+#define GDC0_A2S_CNTL_SW1__WRR_RD_WEIGHT_MASK                                                                 0x00FF0000L
+#define GDC0_A2S_CNTL_SW1__WRR_WR_WEIGHT_MASK                                                                 0xFF000000L
+//GDC0_A2S_CNTL_SW2
+#define GDC0_A2S_CNTL_SW2__STATIC_VC_ENABLE__SHIFT                                                            0x0
+#define GDC0_A2S_CNTL_SW2__STATIC_VC_VALUE__SHIFT                                                             0x1
+#define GDC0_A2S_CNTL_SW2__FORCE_RSP_REORDER_EN__SHIFT                                                        0x6
+#define GDC0_A2S_CNTL_SW2__SDP_WR_CHAIN_DIS__SHIFT                                                            0x9
+#define GDC0_A2S_CNTL_SW2__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                     0xa
+#define GDC0_A2S_CNTL_SW2__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                  0xb
+#define GDC0_A2S_CNTL_SW2__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                 0xc
+#define GDC0_A2S_CNTL_SW2__WRR_RD_WEIGHT__SHIFT                                                               0x10
+#define GDC0_A2S_CNTL_SW2__WRR_WR_WEIGHT__SHIFT                                                               0x18
+#define GDC0_A2S_CNTL_SW2__STATIC_VC_ENABLE_MASK                                                              0x00000001L
+#define GDC0_A2S_CNTL_SW2__STATIC_VC_VALUE_MASK                                                               0x0000000EL
+#define GDC0_A2S_CNTL_SW2__FORCE_RSP_REORDER_EN_MASK                                                          0x00000040L
+#define GDC0_A2S_CNTL_SW2__SDP_WR_CHAIN_DIS_MASK                                                              0x00000200L
+#define GDC0_A2S_CNTL_SW2__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                       0x00000400L
+#define GDC0_A2S_CNTL_SW2__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                    0x00000800L
+#define GDC0_A2S_CNTL_SW2__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                   0x00001000L
+#define GDC0_A2S_CNTL_SW2__WRR_RD_WEIGHT_MASK                                                                 0x00FF0000L
+#define GDC0_A2S_CNTL_SW2__WRR_WR_WEIGHT_MASK                                                                 0xFF000000L
+//GDC0_A2S_TAG_ALLOC_0
+#define GDC0_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_WR__SHIFT                                                     0x0
+#define GDC0_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_RD__SHIFT                                                     0x8
+#define GDC0_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC1_WR__SHIFT                                                     0x10
+#define GDC0_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_WR_MASK                                                       0x000000FFL
+#define GDC0_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_RD_MASK                                                       0x0000FF00L
+#define GDC0_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC1_WR_MASK                                                       0x00FF0000L
+//GDC0_A2S_TAG_ALLOC_1
+#define GDC0_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC3_WR__SHIFT                                                     0x0
+#define GDC0_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_WR__SHIFT                                                     0x10
+#define GDC0_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_RD__SHIFT                                                     0x18
+#define GDC0_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC3_WR_MASK                                                       0x000000FFL
+#define GDC0_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_WR_MASK                                                       0x00FF0000L
+#define GDC0_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_RD_MASK                                                       0xFF000000L
+//GDC0_A2S_MISC_CNTL
+#define GDC0_A2S_MISC_CNTL__BLKLVL_FOR_MSG__SHIFT                                                             0x0
+#define GDC0_A2S_MISC_CNTL__RESERVE_2_CRED_FOR_NPWR_REQ_DIS__SHIFT                                            0x2
+#define GDC0_A2S_MISC_CNTL__WRR_LRG_SIZE_MODE__SHIFT                                                          0x3
+#define GDC0_A2S_MISC_CNTL__FORCE_RSP_REORDER_EN__SHIFT                                                       0x4
+#define GDC0_A2S_MISC_CNTL__RSP_REORDER_DIS__SHIFT                                                            0x5
+#define GDC0_A2S_MISC_CNTL__WRRSP_ACCUM_SEL__SHIFT                                                            0x6
+#define GDC0_A2S_MISC_CNTL__WRRSP_TAGFIFO_CONT_RD_DIS__SHIFT                                                  0x7
+#define GDC0_A2S_MISC_CNTL__RDRSP_TAGFIFO_CONT_RD_DIS__SHIFT                                                  0x8
+#define GDC0_A2S_MISC_CNTL__RDRSP_STS_DATSTS_PRIORITY__SHIFT                                                  0x9
+#define GDC0_A2S_MISC_CNTL__INSERT_RD_ON_2ND_WDAT_EN__SHIFT                                                   0xa
+#define GDC0_A2S_MISC_CNTL__WR_TAG_SET_MIN__SHIFT                                                             0x10
+#define GDC0_A2S_MISC_CNTL__RD_TAG_SET_MIN__SHIFT                                                             0x15
+#define GDC0_A2S_MISC_CNTL__WRR_LRG_MODE__SHIFT                                                               0x1a
+#define GDC0_A2S_MISC_CNTL__WRR_LRG_COUNTER_MODE__SHIFT                                                       0x1b
+#define GDC0_A2S_MISC_CNTL__BLKLVL_FOR_MSG_MASK                                                               0x00000003L
+#define GDC0_A2S_MISC_CNTL__RESERVE_2_CRED_FOR_NPWR_REQ_DIS_MASK                                              0x00000004L
+#define GDC0_A2S_MISC_CNTL__WRR_LRG_SIZE_MODE_MASK                                                            0x00000008L
+#define GDC0_A2S_MISC_CNTL__FORCE_RSP_REORDER_EN_MASK                                                         0x00000010L
+#define GDC0_A2S_MISC_CNTL__RSP_REORDER_DIS_MASK                                                              0x00000020L
+#define GDC0_A2S_MISC_CNTL__WRRSP_ACCUM_SEL_MASK                                                              0x00000040L
+#define GDC0_A2S_MISC_CNTL__WRRSP_TAGFIFO_CONT_RD_DIS_MASK                                                    0x00000080L
+#define GDC0_A2S_MISC_CNTL__RDRSP_TAGFIFO_CONT_RD_DIS_MASK                                                    0x00000100L
+#define GDC0_A2S_MISC_CNTL__RDRSP_STS_DATSTS_PRIORITY_MASK                                                    0x00000200L
+#define GDC0_A2S_MISC_CNTL__INSERT_RD_ON_2ND_WDAT_EN_MASK                                                     0x00000400L
+#define GDC0_A2S_MISC_CNTL__WR_TAG_SET_MIN_MASK                                                               0x001F0000L
+#define GDC0_A2S_MISC_CNTL__RD_TAG_SET_MIN_MASK                                                               0x03E00000L
+#define GDC0_A2S_MISC_CNTL__WRR_LRG_MODE_MASK                                                                 0x04000000L
+#define GDC0_A2S_MISC_CNTL__WRR_LRG_COUNTER_MODE_MASK                                                         0x08000000L
+//GDC0_SHUB_REGS_IF_CTL
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS__SHIFT                                0x0
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS__SHIFT                                             0x1
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS_MASK                                  0x00000001L
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS_MASK                                               0x00000002L
+//GDC0_NGDC_MGCG_CTRL
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_EN__SHIFT                                                              0x0
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_MODE__SHIFT                                                            0x1
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS__SHIFT                                                      0x2
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS__SHIFT                                                         0xa
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS__SHIFT                                                         0xb
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS__SHIFT                                                         0xc
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS__SHIFT                                                         0xd
+#define GDC0_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN__SHIFT                                                         0xe
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_EN_MASK                                                                0x00000001L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_MODE_MASK                                                              0x00000002L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS_MASK                                                        0x000003FCL
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS_MASK                                                           0x00000400L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS_MASK                                                           0x00000800L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS_MASK                                                           0x00001000L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS_MASK                                                           0x00002000L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN_MASK                                                           0x00004000L
+//GDC0_NGDC_RESERVED_0
+#define GDC0_NGDC_RESERVED_0__RESERVED__SHIFT                                                                 0x0
+#define GDC0_NGDC_RESERVED_0__RESERVED_MASK                                                                   0xFFFFFFFFL
+//GDC0_NGDC_RESERVED_1
+#define GDC0_NGDC_RESERVED_1__RESERVED__SHIFT                                                                 0x0
+#define GDC0_NGDC_RESERVED_1__RESERVED_MASK                                                                   0xFFFFFFFFL
+//GDC0_NBIF_GFX_DOORBELL_STATUS
+#define GDC0_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT__SHIFT                                          0x0
+#define GDC0_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT_MASK                                            0x0000FFFFL
+//GDC0_ATDMA_MISC_CNTL
+#define GDC0_ATDMA_MISC_CNTL__WRR_ARB_MODE__SHIFT                                                             0x0
+#define GDC0_ATDMA_MISC_CNTL__INSERT_RD_ON_2ND_WDAT_EN__SHIFT                                                 0x1
+#define GDC0_ATDMA_MISC_CNTL__RDRSP_ARB_MODE__SHIFT                                                           0x2
+#define GDC0_ATDMA_MISC_CNTL__WRR_VC6_WEIGHT__SHIFT                                                           0x8
+#define GDC0_ATDMA_MISC_CNTL__WRR_VC0_WEIGHT__SHIFT                                                           0x10
+#define GDC0_ATDMA_MISC_CNTL__WRR_VC1_WEIGHT__SHIFT                                                           0x18
+#define GDC0_ATDMA_MISC_CNTL__WRR_ARB_MODE_MASK                                                               0x00000001L
+#define GDC0_ATDMA_MISC_CNTL__INSERT_RD_ON_2ND_WDAT_EN_MASK                                                   0x00000002L
+#define GDC0_ATDMA_MISC_CNTL__RDRSP_ARB_MODE_MASK                                                             0x0000000CL
+#define GDC0_ATDMA_MISC_CNTL__WRR_VC6_WEIGHT_MASK                                                             0x0000FF00L
+#define GDC0_ATDMA_MISC_CNTL__WRR_VC0_WEIGHT_MASK                                                             0x00FF0000L
+#define GDC0_ATDMA_MISC_CNTL__WRR_VC1_WEIGHT_MASK                                                             0xFF000000L
+//GDC0_S2A_MISC_CNTL
+#define GDC0_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS__SHIFT                                                         0x3
+#define GDC0_S2A_MISC_CNTL__ATM_ARB_MODE__SHIFT                                                               0x8
+#define GDC0_S2A_MISC_CNTL__RB_ARB_MODE__SHIFT                                                                0xa
+#define GDC0_S2A_MISC_CNTL__HSTR_ARB_MODE__SHIFT                                                              0xc
+#define GDC0_S2A_MISC_CNTL__HDP_PERF_ENH_DIS__SHIFT                                                           0xf
+#define GDC0_S2A_MISC_CNTL__WRSP_ARB_MODE__SHIFT                                                              0x10
+#define GDC0_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS_MASK                                                           0x00000008L
+#define GDC0_S2A_MISC_CNTL__ATM_ARB_MODE_MASK                                                                 0x00000300L
+#define GDC0_S2A_MISC_CNTL__RB_ARB_MODE_MASK                                                                  0x00000C00L
+#define GDC0_S2A_MISC_CNTL__HSTR_ARB_MODE_MASK                                                                0x00003000L
+#define GDC0_S2A_MISC_CNTL__HDP_PERF_ENH_DIS_MASK                                                             0x00008000L
+#define GDC0_S2A_MISC_CNTL__WRSP_ARB_MODE_MASK                                                                0x000F0000L
+//GDC0_NGDC_PG_MISC_CTRL
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_ENDP_D3_ONLY__SHIFT                                                   0xa
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM1__SHIFT                                                      0xd
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_DS_ALLOW_DIS__SHIFT                                                   0xe
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM2__SHIFT                                                      0x10
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_CFG_REFCLK_CYCLE_FOR_200NS__SHIFT                                        0x18
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_CFG_PG_EXIT_OVERRIDE__SHIFT                                              0x1f
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_ENDP_D3_ONLY_MASK                                                     0x00000400L
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM1_MASK                                                        0x00002000L
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_DS_ALLOW_DIS_MASK                                                     0x00004000L
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM2_MASK                                                        0x00010000L
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_CFG_REFCLK_CYCLE_FOR_200NS_MASK                                          0x3F000000L
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_CFG_PG_EXIT_OVERRIDE_MASK                                                0x80000000L
+//GDC0_NGDC_PGMST_CTRL
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_PG_HYSTERESIS__SHIFT                                                   0x0
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_PG_EN__SHIFT                                                           0x8
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_IDLENESS_COUNT_EN__SHIFT                                               0xa
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_FW_PG_EXIT_EN__SHIFT                                                   0xe
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_PG_HYSTERESIS_MASK                                                     0x000000FFL
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_PG_EN_MASK                                                             0x00000100L
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_IDLENESS_COUNT_EN_MASK                                                 0x00003C00L
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_FW_PG_EXIT_EN_MASK                                                     0x0000C000L
+//GDC0_NGDC_PGSLV_CTRL
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_0_IDLE_HYSTERESIS__SHIFT                                       0x0
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_1_IDLE_HYSTERESIS__SHIFT                                       0x5
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_GDCCLK_IDLE_HYSTERESIS__SHIFT                                          0xa
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_0_IDLE_HYSTERESIS_MASK                                         0x0000001FL
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_1_IDLE_HYSTERESIS_MASK                                         0x000003E0L
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_GDCCLK_IDLE_HYSTERESIS_MASK                                            0x00007C00L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF0_COMMAND
+#define BIF_CFG_DEV0_EPF0_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF0_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF0_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF0_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF0_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF0_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF0_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF0_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF0_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF0_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF0_STATUS
+#define BIF_CFG_DEV0_EPF0_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF0_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF0_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF0_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF0_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF0_LATENCY
+#define BIF_CFG_DEV0_EPF0_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_HEADER
+#define BIF_CFG_DEV0_EPF0_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF0_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF0_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF0_BIST
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF0_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV0_EPF0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF0_PMI_CAP
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV0_EPF0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV0_EPF0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF0_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                             0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                               0x00000070L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                               0x00000C00L
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                               0x000EL
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                       0x0001L
+//BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                      0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                            0x1f
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                        0x000000FEL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                        0x000E0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                  0x07000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                              0x80000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                               0x0002L
+//BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                      0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                          0x003F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                            0x1f
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                        0x000000FEL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                        0x000E0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                  0x07000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                              0x80000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                               0x0002L
+//BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                    0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                   0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                       0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                      0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                     0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                  0x00000002L
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                       0x0000FE00L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                 0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                       0x0080L
+//BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL__STU__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL__STU_MASK                                                             0x001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                      0x8000L
+//PCIE_PAGE_REQ_ENH_CAP_LIST
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID__SHIFT                                                             0x0
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER__SHIFT                                                            0x10
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                           0x14
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID_MASK                                                               0x0000FFFFL
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER_MASK                                                              0x000F0000L
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR_MASK                                                             0xFFF00000L
+//PCIE_PAGE_REQ_CNTL
+#define PCIE_PAGE_REQ_CNTL__PRI_ENABLE__SHIFT                                                                 0x0
+#define PCIE_PAGE_REQ_CNTL__PRI_RESET__SHIFT                                                                  0x1
+#define PCIE_PAGE_REQ_CNTL__PRI_ENABLE_MASK                                                                   0x0001L
+#define PCIE_PAGE_REQ_CNTL__PRI_RESET_MASK                                                                    0x0002L
+//PCIE_PAGE_REQ_STATUS
+#define PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE__SHIFT                                                         0x0
+#define PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX__SHIFT                                            0x1
+#define PCIE_PAGE_REQ_STATUS__STOPPED__SHIFT                                                                  0x8
+#define PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED__SHIFT                                              0xf
+#define PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE_MASK                                                           0x0001L
+#define PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX_MASK                                              0x0002L
+#define PCIE_PAGE_REQ_STATUS__STOPPED_MASK                                                                    0x0100L
+#define PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED_MASK                                                0x8000L
+//PCIE_OUTSTAND_PAGE_REQ_CAPACITY
+#define PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY__SHIFT                                    0x0
+#define PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY_MASK                                      0xFFFFFFFFL
+//PCIE_OUTSTAND_PAGE_REQ_ALLOC
+#define PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC__SHIFT                                          0x0
+#define PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC_MASK                                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+//BIF_CFG_DEV0_EPF0_PCIE_MC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_MC_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_CAP__MC_MAX_GROUP__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_CAP__MC_WIN_SIZE_REQ__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_CAP__MC_ECRC_REGEN_SUPP__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_CAP__MC_MAX_GROUP_MASK                                                      0x003FL
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_CAP__MC_WIN_SIZE_REQ_MASK                                                   0x3F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_CAP__MC_ECRC_REGEN_SUPP_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_PCIE_MC_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_CNTL__MC_NUM_GROUP__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_CNTL__MC_ENABLE__SHIFT                                                      0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_CNTL__MC_NUM_GROUP_MASK                                                     0x003FL
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_CNTL__MC_ENABLE_MASK                                                        0x8000L
+//BIF_CFG_DEV0_EPF0_PCIE_MC_ADDR0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ADDR0__MC_INDEX_POS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ADDR0__MC_BASE_ADDR_0__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ADDR0__MC_INDEX_POS_MASK                                                    0x0000003FL
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ADDR0__MC_BASE_ADDR_0_MASK                                                  0xFFFFF000L
+//BIF_CFG_DEV0_EPF0_PCIE_MC_ADDR1
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ADDR1__MC_BASE_ADDR_1__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_ADDR1__MC_BASE_ADDR_1_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_MC_RCV0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_RCV0__MC_RECEIVE_0__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_RCV0__MC_RECEIVE_0_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_MC_RCV1
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_RCV1__MC_RECEIVE_1__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_RCV1__MC_RECEIVE_1_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_ALL0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_ALL0__MC_BLOCK_ALL_0__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_ALL0__MC_BLOCK_ALL_0_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_ALL1
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_ALL1__MC_BLOCK_ALL_1__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_ALL1__MC_BLOCK_ALL_1_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_UNTRANSLATED_0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_UNTRANSLATED_0__MC_BLOCK_UNTRANSLATED_0__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_UNTRANSLATED_0__MC_BLOCK_UNTRANSLATED_0_MASK                          0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_UNTRANSLATED_1
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_UNTRANSLATED_1__MC_BLOCK_UNTRANSLATED_1__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MC_BLOCK_UNTRANSLATED_1__MC_BLOCK_UNTRANSLATED_1_MASK                          0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE__SHIFT                                       0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE_MASK                                          0x000003FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE_MASK                                          0x00001C00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE_MASK                                         0x03FF0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE_MASK                                         0x1C000000L
+//BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//PCIE_SRIOV_ENH_CAP_LIST
+#define PCIE_SRIOV_ENH_CAP_LIST__CAP_ID__SHIFT                                                                0x0
+#define PCIE_SRIOV_ENH_CAP_LIST__CAP_VER__SHIFT                                                               0x10
+#define PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                              0x14
+#define PCIE_SRIOV_ENH_CAP_LIST__CAP_ID_MASK                                                                  0x0000FFFFL
+#define PCIE_SRIOV_ENH_CAP_LIST__CAP_VER_MASK                                                                 0x000F0000L
+#define PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR_MASK                                                                0xFFF00000L
+//PCIE_SRIOV_CAP
+#define PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP__SHIFT                                                         0x0
+#define PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED__SHIFT                                              0x1
+#define PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                       0x2
+#define PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM__SHIFT                                                0x15
+#define PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP_MASK                                                           0x00000001L
+#define PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED_MASK                                                0x00000002L
+#define PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                         0x00000004L
+#define PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM_MASK                                                  0xFFE00000L
+//PCIE_SRIOV_CONTROL
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE__SHIFT                                                            0x0
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE__SHIFT                                                  0x1
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE__SHIFT                                             0x2
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MSE__SHIFT                                                               0x3
+#define PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY__SHIFT                                                    0x4
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                      0x5
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE_MASK                                                              0x0001L
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE_MASK                                                    0x0002L
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE_MASK                                               0x0004L
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MSE_MASK                                                                 0x0008L
+#define PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY_MASK                                                      0x0010L
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                        0x0020L
+//PCIE_SRIOV_STATUS
+#define PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS__SHIFT                                                   0x0
+#define PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS_MASK                                                     0x0001L
+//PCIE_SRIOV_INITIAL_VFS
+#define PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS__SHIFT                                                      0x0
+#define PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS_MASK                                                        0xFFFFL
+//PCIE_SRIOV_TOTAL_VFS
+#define PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS__SHIFT                                                          0x0
+#define PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS_MASK                                                            0xFFFFL
+//PCIE_SRIOV_NUM_VFS
+#define PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS__SHIFT                                                              0x0
+#define PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS_MASK                                                                0xFFFFL
+//PCIE_SRIOV_FUNC_DEP_LINK
+#define PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK__SHIFT                                                  0x0
+#define PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK_MASK                                                    0xFFL
+//PCIE_SRIOV_FIRST_VF_OFFSET
+#define PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET__SHIFT                                              0x0
+#define PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET_MASK                                                0xFFFFL
+//PCIE_SRIOV_VF_STRIDE
+#define PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE__SHIFT                                                          0x0
+#define PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE_MASK                                                            0xFFFFL
+//PCIE_SRIOV_VF_DEVICE_ID
+#define PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID__SHIFT                                                    0x0
+#define PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID_MASK                                                      0xFFFFL
+//PCIE_SRIOV_SUPPORTED_PAGE_SIZE
+#define PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE__SHIFT                                      0x0
+#define PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE_MASK                                        0xFFFFFFFFL
+//PCIE_SRIOV_SYSTEM_PAGE_SIZE
+#define PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE__SHIFT                                            0x0
+#define PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE_MASK                                              0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_0
+#define PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_1
+#define PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_2
+#define PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_3
+#define PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_4
+#define PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_5
+#define PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET
+#define PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR__SHIFT                       0x0
+#define PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SHIFT              0x3
+#define PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR_MASK                         0x00000007L
+#define PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_MASK                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                   0x1f
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                     0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                     0x80000000L
+//BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                 0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_EPF0_LINK_CAP_16GT
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_16GT__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_16GT__RESERVED_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_16GT__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_16GT__RESERVED_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                           0x2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                             0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                             0x00000004L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                             0x00000008L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                               0x00000010L
+//BIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                    0x0001L
+//BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                              0x1
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                0x0002L
+//BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LINK_CAP_32GT
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                             0x9
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                             0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                             0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                               0x00000200L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                               0x00000400L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                0x0000F800L
+//BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                               0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                     0x00000700L
+//BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                           0x2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                      0xa
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                             0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                             0x00000004L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                             0x00000008L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                               0x00000010L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                         0x00000020L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                        0x00000400L
+//PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER__SHIFT                                              0x10
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR__SHIFT                                             0x14
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID_MASK                                                 0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER_MASK                                                0x000F0000L
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR_MASK                                               0xFFF00000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID__SHIFT                                                       0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV__SHIFT                                                      0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH__SHIFT                                                   0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID_MASK                                                         0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV_MASK                                                        0x000F0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH_MASK                                                     0xFFF00000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN__SHIFT                      0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN__SHIFT                    0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN_MASK                        0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN_MASK                      0x02000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS__SHIFT                  0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS__SHIFT                0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS_MASK                    0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS_MASK                  0x02000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR__SHIFT                                     0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR_MASK                                       0x0001L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX__SHIFT                                        0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA__SHIFT                                    0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID__SHIFT                                   0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA__SHIFT                                    0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK__SHIFT                                     0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX_MASK                                          0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA_MASK                                      0x00000F00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID_MASK                                     0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA_MASK                                      0x000F0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK_MASK                                       0x01000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK__SHIFT                                     0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID__SHIFT                                   0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK__SHIFT                                     0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID__SHIFT                                   0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK__SHIFT                                     0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID__SHIFT                                   0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK__SHIFT                                     0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID__SHIFT                                   0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK__SHIFT                                     0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID__SHIFT                                   0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK__SHIFT                                     0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID__SHIFT                                   0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK__SHIFT                                     0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID__SHIFT                                   0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK__SHIFT                                     0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID__SHIFT                                   0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK__SHIFT                                     0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID__SHIFT                                   0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK__SHIFT                                     0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID__SHIFT                                   0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK__SHIFT                                    0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID__SHIFT                                  0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK__SHIFT                                    0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID__SHIFT                                  0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK__SHIFT                                    0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID__SHIFT                                  0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK__SHIFT                                    0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID__SHIFT                                  0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK__SHIFT                                    0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID__SHIFT                                  0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK__SHIFT                                    0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID__SHIFT                                  0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK_MASK                                       0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID_MASK                                     0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK_MASK                                       0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID_MASK                                     0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK_MASK                                       0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID_MASK                                     0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK_MASK                                       0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID_MASK                                     0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK_MASK                                       0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID_MASK                                     0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK_MASK                                       0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID_MASK                                     0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK_MASK                                       0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID_MASK                                     0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK_MASK                                       0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID_MASK                                     0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK_MASK                                       0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID_MASK                                     0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK_MASK                                       0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID_MASK                                     0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK_MASK                                      0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID_MASK                                    0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK_MASK                                      0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID_MASK                                    0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK_MASK                                      0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID_MASK                                    0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK_MASK                                      0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID_MASK                                    0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK_MASK                                      0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID_MASK                                    0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK_MASK                                      0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID_MASK                                    0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK__SHIFT                                    0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID__SHIFT                                  0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK__SHIFT                                    0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID__SHIFT                                  0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK__SHIFT                                    0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID__SHIFT                                  0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK__SHIFT                                    0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID__SHIFT                                  0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK__SHIFT                                    0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID__SHIFT                                  0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK__SHIFT                                    0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID__SHIFT                                  0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK__SHIFT                                    0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID__SHIFT                                  0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK__SHIFT                                    0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID__SHIFT                                  0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK__SHIFT                                    0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID__SHIFT                                  0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK__SHIFT                                    0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID__SHIFT                                  0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK__SHIFT                                    0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID__SHIFT                                  0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK__SHIFT                                    0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID__SHIFT                                  0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK__SHIFT                                    0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID__SHIFT                                  0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK__SHIFT                                    0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID__SHIFT                                  0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK__SHIFT                                    0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID__SHIFT                                  0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK__SHIFT                                      0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID__SHIFT                                    0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK_MASK                                      0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID_MASK                                    0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK_MASK                                      0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID_MASK                                    0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK_MASK                                      0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID_MASK                                    0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK_MASK                                      0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID_MASK                                    0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK_MASK                                      0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID_MASK                                    0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK_MASK                                      0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID_MASK                                    0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK_MASK                                      0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID_MASK                                    0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK_MASK                                      0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID_MASK                                    0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK_MASK                                      0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID_MASK                                    0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK_MASK                                      0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID_MASK                                    0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK_MASK                                      0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID_MASK                                    0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK_MASK                                      0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID_MASK                                    0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK_MASK                                      0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID_MASK                                    0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK_MASK                                      0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID_MASK                                    0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK_MASK                                      0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID_MASK                                    0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK_MASK                                        0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID_MASK                                      0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__CONTEXT_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__LOC__SHIFT                                                   0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__CONTEXT_OFFSET__SHIFT                                        0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__CONTEXT_SIZE_MASK                                            0x0000007FL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__LOC_MASK                                                     0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__CONTEXT_OFFSET_MASK                                          0xFFFFFC00L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_AVAILABLE__SHIFT                                   0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_CONSUMED__SHIFT                                    0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_AVAILABLE_MASK                                     0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_CONSUMED_MASK                                      0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__LFB_REGION__SHIFT                                             0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__MAX_REGION__SHIFT                                             0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__LFB_REGION_MASK                                               0x0000000FL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__MAX_REGION_MASK                                               0x000000F0L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_VF__SHIFT                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_PF__SHIFT                  0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_VF_MASK                    0x7FFFFFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_PF_MASK                    0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB__VF0_FB_SIZE__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB__VF0_FB_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB__VF0_FB_SIZE_MASK                                              0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB__VF0_FB_OFFSET_MASK                                            0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB__VF1_FB_SIZE__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB__VF1_FB_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB__VF1_FB_SIZE_MASK                                              0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB__VF1_FB_OFFSET_MASK                                            0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB__VF2_FB_SIZE__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB__VF2_FB_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB__VF2_FB_SIZE_MASK                                              0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB__VF2_FB_OFFSET_MASK                                            0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB__VF3_FB_SIZE__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB__VF3_FB_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB__VF3_FB_SIZE_MASK                                              0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB__VF3_FB_OFFSET_MASK                                            0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB__VF4_FB_SIZE__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB__VF4_FB_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB__VF4_FB_SIZE_MASK                                              0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB__VF4_FB_OFFSET_MASK                                            0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB__VF5_FB_SIZE__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB__VF5_FB_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB__VF5_FB_SIZE_MASK                                              0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB__VF5_FB_OFFSET_MASK                                            0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB__VF6_FB_SIZE__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB__VF6_FB_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB__VF6_FB_SIZE_MASK                                              0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB__VF6_FB_OFFSET_MASK                                            0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB__VF7_FB_SIZE__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB__VF7_FB_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB__VF7_FB_SIZE_MASK                                              0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB__VF7_FB_OFFSET_MASK                                            0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB__VF8_FB_SIZE__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB__VF8_FB_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB__VF8_FB_SIZE_MASK                                              0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB__VF8_FB_OFFSET_MASK                                            0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB__VF9_FB_SIZE__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB__VF9_FB_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB__VF9_FB_SIZE_MASK                                              0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB__VF9_FB_OFFSET_MASK                                            0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB__VF10_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB__VF10_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB__VF10_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB__VF10_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB__VF11_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB__VF11_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB__VF11_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB__VF11_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB__VF12_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB__VF12_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB__VF12_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB__VF12_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB__VF13_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB__VF13_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB__VF13_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB__VF13_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB__VF14_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB__VF14_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB__VF14_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB__VF14_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB__VF15_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB__VF15_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB__VF15_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB__VF15_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB__VF16_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB__VF16_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB__VF16_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB__VF16_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB__VF17_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB__VF17_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB__VF17_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB__VF17_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB__VF18_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB__VF18_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB__VF18_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB__VF18_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB__VF19_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB__VF19_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB__VF19_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB__VF19_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB__VF20_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB__VF20_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB__VF20_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB__VF20_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB__VF21_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB__VF21_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB__VF21_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB__VF21_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB__VF22_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB__VF22_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB__VF22_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB__VF22_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB__VF23_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB__VF23_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB__VF23_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB__VF23_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB__VF24_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB__VF24_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB__VF24_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB__VF24_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB__VF25_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB__VF25_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB__VF25_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB__VF25_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB__VF26_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB__VF26_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB__VF26_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB__VF26_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB__VF27_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB__VF27_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB__VF27_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB__VF27_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB__VF28_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB__VF28_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB__VF28_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB__VF28_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB__VF29_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB__VF29_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB__VF29_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB__VF29_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB__VF30_FB_SIZE__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB__VF30_FB_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB__VF30_FB_SIZE_MASK                                            0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB__VF30_FB_OFFSET_MASK                                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN0SCH_OFFSET__SHIFT                                        0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN1SCH_OFFSET__SHIFT                                        0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN2SCH_OFFSET__SHIFT                                        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN3SCH_OFFSET__SHIFT                                        0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN0SCH_OFFSET_MASK                                          0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN1SCH_OFFSET_MASK                                          0x0000FF00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN2SCH_OFFSET_MASK                                          0x00FF0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN3SCH_OFFSET_MASK                                          0xFF000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN4SCH_OFFSET__SHIFT                                       0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN5SCH_OFFSET__SHIFT                                       0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN6SCH_OFFSET__SHIFT                                       0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN7SCH_OFFSET__SHIFT                                       0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN4SCH_OFFSET_MASK                                         0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN5SCH_OFFSET_MASK                                         0x0000FF00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN6SCH_OFFSET_MASK                                         0x00FF0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN7SCH_OFFSET_MASK                                         0xFF000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN8SCH_OFFSET__SHIFT                                       0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN9SCH_OFFSET__SHIFT                                       0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN10SCH_OFFSET__SHIFT                                      0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN11SCH_OFFSET__SHIFT                                      0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN8SCH_OFFSET_MASK                                         0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN9SCH_OFFSET_MASK                                         0x0000FF00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN10SCH_OFFSET_MASK                                        0x00FF0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN11SCH_OFFSET_MASK                                        0xFF000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX0SCH_OFFSET__SHIFT                                       0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX1SCH_OFFSET__SHIFT                                       0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX2SCH_OFFSET__SHIFT                                       0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX3SCH_OFFSET__SHIFT                                       0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX0SCH_OFFSET_MASK                                         0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX1SCH_OFFSET_MASK                                         0x0000FF00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX2SCH_OFFSET_MASK                                         0x00FF0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX3SCH_OFFSET_MASK                                         0xFF000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX4SCH_OFFSET__SHIFT                                       0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX5SCH_OFFSET__SHIFT                                       0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX6SCH_OFFSET__SHIFT                                       0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX7SCH_OFFSET__SHIFT                                       0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX4SCH_OFFSET_MASK                                         0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX5SCH_OFFSET_MASK                                         0x0000FF00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX6SCH_OFFSET_MASK                                         0x00FF0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX7SCH_OFFSET_MASK                                         0xFF000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW0__DW0__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW0__DW0_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW1__DW1__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW1__DW1_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW2__DW2__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW2__DW2_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW3__DW3__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW3__DW3_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW4__DW4__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW4__DW4_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW5__DW5__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW5__DW5_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW6__DW6__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW6__DW6_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW7__DW7__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW7__DW7_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW8__DW8__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW8__DW8_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW0__DW0__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW0__DW0_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW1__DW1__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW1__DW1_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW2__DW2__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW2__DW2_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW3__DW3__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW3__DW3_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW4__DW4__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW4__DW4_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW5__DW5__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW5__DW5_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW6__DW6__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW6__DW6_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW7__DW7__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW7__DW7_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW8__DW8__SHIFT                                              0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW8__DW8_MASK                                                0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW0__DW0__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW0__DW0_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW1__DW1__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW1__DW1_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW2__DW2__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW2__DW2_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW3__DW3__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW3__DW3_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW4__DW4__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW4__DW4_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW5__DW5__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW5__DW5_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW6__DW6__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW6__DW6_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW7__DW7__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW7__DW7_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW8__DW8__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW8__DW8_MASK                                                 0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_CMD_COMPLETE_INTR_EN__SHIFT            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_HANG_NEED_FLR_INTR_EN__SHIFT           0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_CMD_COMPLETE_INTR_EN__SHIFT            0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_HANG_NEED_FLR_INTR_EN__SHIFT           0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_CMD_COMPLETE_INTR_EN__SHIFT            0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_HANG_NEED_FLR_INTR_EN__SHIFT           0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_CMD_COMPLETE_INTR_EN__SHIFT            0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_HANG_NEED_FLR_INTR_EN__SHIFT           0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_CMD_COMPLETE_INTR_EN__SHIFT            0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_HANG_NEED_FLR_INTR_EN__SHIFT           0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_CMD_COMPLETE_INTR_EN__SHIFT            0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_HANG_NEED_FLR_INTR_EN__SHIFT           0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_CMD_COMPLETE_INTR_EN__SHIFT            0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_HANG_NEED_FLR_INTR_EN__SHIFT           0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_CMD_COMPLETE_INTR_EN__SHIFT            0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_HANG_NEED_FLR_INTR_EN__SHIFT           0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_CMD_COMPLETE_INTR_EN_MASK              0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_HANG_NEED_FLR_INTR_EN_MASK             0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_CMD_COMPLETE_INTR_EN_MASK              0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_HANG_NEED_FLR_INTR_EN_MASK             0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_CMD_COMPLETE_INTR_EN_MASK              0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_HANG_NEED_FLR_INTR_EN_MASK             0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_CMD_COMPLETE_INTR_EN_MASK              0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_HANG_NEED_FLR_INTR_EN_MASK             0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_CMD_COMPLETE_INTR_EN_MASK              0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_HANG_NEED_FLR_INTR_EN_MASK             0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_CMD_COMPLETE_INTR_EN_MASK              0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_HANG_NEED_FLR_INTR_EN_MASK             0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_CMD_COMPLETE_INTR_EN_MASK              0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_HANG_SELF_RECOVERED_INTR_EN_MASK       0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_HANG_NEED_FLR_INTR_EN_MASK             0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_VM_BUSY_TRANSITION_INTR_EN_MASK        0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_CMD_COMPLETE_INTR_EN_MASK              0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_HANG_SELF_RECOVERED_INTR_EN_MASK       0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_HANG_NEED_FLR_INTR_EN_MASK             0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_VM_BUSY_TRANSITION_INTR_EN_MASK        0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A8_15_INTR_ENABLE
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_CMD_COMPLETE_INTR_EN__SHIFT            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_HANG_NEED_FLR_INTR_EN__SHIFT           0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_CMD_COMPLETE_INTR_EN__SHIFT            0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_HANG_NEED_FLR_INTR_EN__SHIFT           0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_CMD_COMPLETE_INTR_EN__SHIFT            0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_HANG_NEED_FLR_INTR_EN__SHIFT           0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_CMD_COMPLETE_INTR_EN__SHIFT            0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_HANG_NEED_FLR_INTR_EN__SHIFT           0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_CMD_COMPLETE_INTR_EN__SHIFT            0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_HANG_NEED_FLR_INTR_EN__SHIFT           0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_CMD_COMPLETE_INTR_EN__SHIFT            0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_HANG_NEED_FLR_INTR_EN__SHIFT           0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_CMD_COMPLETE_INTR_EN__SHIFT            0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_HANG_NEED_FLR_INTR_EN__SHIFT           0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_CMD_COMPLETE_INTR_EN__SHIFT            0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_HANG_SELF_RECOVERED_INTR_EN__SHIFT     0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_HANG_NEED_FLR_INTR_EN__SHIFT           0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_VM_BUSY_TRANSITION_INTR_EN__SHIFT      0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_CMD_COMPLETE_INTR_EN_MASK              0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_HANG_NEED_FLR_INTR_EN_MASK             0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_CMD_COMPLETE_INTR_EN_MASK              0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_HANG_NEED_FLR_INTR_EN_MASK             0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_CMD_COMPLETE_INTR_EN_MASK              0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_HANG_NEED_FLR_INTR_EN_MASK             0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_CMD_COMPLETE_INTR_EN_MASK              0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_HANG_NEED_FLR_INTR_EN_MASK             0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_CMD_COMPLETE_INTR_EN_MASK              0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_HANG_NEED_FLR_INTR_EN_MASK             0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_CMD_COMPLETE_INTR_EN_MASK              0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_HANG_SELF_RECOVERED_INTR_EN_MASK       0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_HANG_NEED_FLR_INTR_EN_MASK             0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_VM_BUSY_TRANSITION_INTR_EN_MASK        0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_CMD_COMPLETE_INTR_EN_MASK              0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_HANG_SELF_RECOVERED_INTR_EN_MASK       0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_HANG_NEED_FLR_INTR_EN_MASK             0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_VM_BUSY_TRANSITION_INTR_EN_MASK        0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_CMD_COMPLETE_INTR_EN_MASK              0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_HANG_SELF_RECOVERED_INTR_EN_MASK       0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_HANG_NEED_FLR_INTR_EN_MASK             0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_VM_BUSY_TRANSITION_INTR_EN_MASK        0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_CMD_COMPLETE_INTR_EN__SHIFT           0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_HANG_SELF_RECOVERED_INTR_EN__SHIFT    0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_HANG_NEED_FLR_INTR_EN__SHIFT          0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_VM_BUSY_TRANSITION_INTR_EN__SHIFT     0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_CMD_COMPLETE_INTR_EN__SHIFT           0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_HANG_SELF_RECOVERED_INTR_EN__SHIFT    0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_HANG_NEED_FLR_INTR_EN__SHIFT          0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_VM_BUSY_TRANSITION_INTR_EN__SHIFT     0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_CMD_COMPLETE_INTR_EN__SHIFT          0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_HANG_SELF_RECOVERED_INTR_EN__SHIFT   0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_HANG_NEED_FLR_INTR_EN__SHIFT         0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_VM_BUSY_TRANSITION_INTR_EN__SHIFT    0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_CMD_COMPLETE_INTR_EN__SHIFT          0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_HANG_SELF_RECOVERED_INTR_EN__SHIFT   0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_HANG_NEED_FLR_INTR_EN__SHIFT         0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_VM_BUSY_TRANSITION_INTR_EN__SHIFT    0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_CMD_COMPLETE_INTR_EN_MASK             0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_HANG_SELF_RECOVERED_INTR_EN_MASK      0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_HANG_NEED_FLR_INTR_EN_MASK            0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_VM_BUSY_TRANSITION_INTR_EN_MASK       0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_CMD_COMPLETE_INTR_EN_MASK             0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_HANG_SELF_RECOVERED_INTR_EN_MASK      0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_HANG_NEED_FLR_INTR_EN_MASK            0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_VM_BUSY_TRANSITION_INTR_EN_MASK       0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_CMD_COMPLETE_INTR_EN_MASK            0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_HANG_SELF_RECOVERED_INTR_EN_MASK     0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_HANG_NEED_FLR_INTR_EN_MASK           0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_VM_BUSY_TRANSITION_INTR_EN_MASK      0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_CMD_COMPLETE_INTR_EN_MASK            0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_HANG_SELF_RECOVERED_INTR_EN_MASK     0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_HANG_NEED_FLR_INTR_EN_MASK           0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_VM_BUSY_TRANSITION_INTR_EN_MASK      0x00008000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_CMD_COMPLETE_INTR_STATUS__SHIFT        0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_CMD_COMPLETE_INTR_STATUS__SHIFT        0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_CMD_COMPLETE_INTR_STATUS__SHIFT        0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_HANG_NEED_FLR_INTR_STATUS__SHIFT       0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_CMD_COMPLETE_INTR_STATUS__SHIFT        0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_HANG_NEED_FLR_INTR_STATUS__SHIFT       0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_CMD_COMPLETE_INTR_STATUS__SHIFT        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_CMD_COMPLETE_INTR_STATUS__SHIFT        0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_CMD_COMPLETE_INTR_STATUS__SHIFT        0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_CMD_COMPLETE_INTR_STATUS__SHIFT        0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_CMD_COMPLETE_INTR_STATUS_MASK          0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_HANG_NEED_FLR_INTR_STATUS_MASK         0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_CMD_COMPLETE_INTR_STATUS_MASK          0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_HANG_NEED_FLR_INTR_STATUS_MASK         0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_CMD_COMPLETE_INTR_STATUS_MASK          0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_HANG_NEED_FLR_INTR_STATUS_MASK         0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_CMD_COMPLETE_INTR_STATUS_MASK          0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_HANG_NEED_FLR_INTR_STATUS_MASK         0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_CMD_COMPLETE_INTR_STATUS_MASK          0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_HANG_NEED_FLR_INTR_STATUS_MASK         0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_CMD_COMPLETE_INTR_STATUS_MASK          0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_HANG_NEED_FLR_INTR_STATUS_MASK         0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_CMD_COMPLETE_INTR_STATUS_MASK          0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_HANG_NEED_FLR_INTR_STATUS_MASK         0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_CMD_COMPLETE_INTR_STATUS_MASK          0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_HANG_NEED_FLR_INTR_STATUS_MASK         0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A8_15_INTR_STATUS
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_CMD_COMPLETE_INTR_STATUS__SHIFT        0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_CMD_COMPLETE_INTR_STATUS__SHIFT        0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_CMD_COMPLETE_INTR_STATUS__SHIFT        0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_HANG_NEED_FLR_INTR_STATUS__SHIFT       0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_CMD_COMPLETE_INTR_STATUS__SHIFT        0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_HANG_NEED_FLR_INTR_STATUS__SHIFT       0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_CMD_COMPLETE_INTR_STATUS__SHIFT        0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_CMD_COMPLETE_INTR_STATUS__SHIFT        0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_CMD_COMPLETE_INTR_STATUS__SHIFT        0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_CMD_COMPLETE_INTR_STATUS__SHIFT        0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_HANG_NEED_FLR_INTR_STATUS__SHIFT       0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_CMD_COMPLETE_INTR_STATUS_MASK          0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_HANG_NEED_FLR_INTR_STATUS_MASK         0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_CMD_COMPLETE_INTR_STATUS_MASK          0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_HANG_NEED_FLR_INTR_STATUS_MASK         0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_CMD_COMPLETE_INTR_STATUS_MASK          0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_HANG_NEED_FLR_INTR_STATUS_MASK         0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_CMD_COMPLETE_INTR_STATUS_MASK          0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_HANG_NEED_FLR_INTR_STATUS_MASK         0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_CMD_COMPLETE_INTR_STATUS_MASK          0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_HANG_NEED_FLR_INTR_STATUS_MASK         0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_CMD_COMPLETE_INTR_STATUS_MASK          0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_HANG_NEED_FLR_INTR_STATUS_MASK         0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_CMD_COMPLETE_INTR_STATUS_MASK          0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_HANG_NEED_FLR_INTR_STATUS_MASK         0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_CMD_COMPLETE_INTR_STATUS_MASK          0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_HANG_SELF_RECOVERED_INTR_STATUS_MASK   0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_HANG_NEED_FLR_INTR_STATUS_MASK         0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_VM_BUSY_TRANSITION_INTR_STATUS_MASK    0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_CMD_COMPLETE_INTR_STATUS__SHIFT       0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_HANG_NEED_FLR_INTR_STATUS__SHIFT      0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_CMD_COMPLETE_INTR_STATUS__SHIFT       0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_HANG_NEED_FLR_INTR_STATUS__SHIFT      0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_CMD_COMPLETE_INTR_STATUS__SHIFT      0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_HANG_NEED_FLR_INTR_STATUS__SHIFT     0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_CMD_COMPLETE_INTR_STATUS__SHIFT      0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_HANG_NEED_FLR_INTR_STATUS__SHIFT     0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_CMD_COMPLETE_INTR_STATUS_MASK         0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_HANG_NEED_FLR_INTR_STATUS_MASK        0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_VM_BUSY_TRANSITION_INTR_STATUS_MASK   0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_CMD_COMPLETE_INTR_STATUS_MASK         0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_HANG_NEED_FLR_INTR_STATUS_MASK        0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_VM_BUSY_TRANSITION_INTR_STATUS_MASK   0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_CMD_COMPLETE_INTR_STATUS_MASK        0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_HANG_NEED_FLR_INTR_STATUS_MASK       0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_CMD_COMPLETE_INTR_STATUS_MASK        0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_HANG_NEED_FLR_INTR_STATUS_MASK       0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00008000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf1_bifcfgdecp
+//BIF_CFG_DEV0_EPF1_VENDOR_ID
+#define BIF_CFG_DEV0_EPF1_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF1_DEVICE_ID
+#define BIF_CFG_DEV0_EPF1_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF1_COMMAND
+#define BIF_CFG_DEV0_EPF1_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF1_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF1_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF1_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF1_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF1_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF1_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF1_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF1_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF1_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF1_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF1_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF1_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF1_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF1_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF1_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF1_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF1_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF1_STATUS
+#define BIF_CFG_DEV0_EPF1_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF1_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF1_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF1_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF1_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF1_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF1_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF1_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF1_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF1_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF1_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF1_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF1_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF1_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF1_REVISION_ID
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF1_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF1_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF1_SUB_CLASS
+#define BIF_CFG_DEV0_EPF1_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF1_BASE_CLASS
+#define BIF_CFG_DEV0_EPF1_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_CACHE_LINE
+#define BIF_CFG_DEV0_EPF1_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF1_LATENCY
+#define BIF_CFG_DEV0_EPF1_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_HEADER
+#define BIF_CFG_DEV0_EPF1_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF1_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF1_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF1_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF1_BIST
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF1_CAP_PTR
+#define BIF_CFG_DEV0_EPF1_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF1_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF1_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF1_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF1_MIN_GRANT
+#define BIF_CFG_DEV0_EPF1_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF1_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF1_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF1_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV0_EPF1_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF1_PMI_CAP
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF1_PCIE_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV0_EPF1_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF1_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF1_LINK_CAP
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF1_LINK_CNTL
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF1_LINK_STATUS
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF1_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF1_LINK_CAP2
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF1_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF1_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF1_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MASK
+#define BIF_CFG_DEV0_EPF1_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF1_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_PENDING
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF1_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_MSIX_PBA
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+//BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_rc_bifcfgdecp
+//BIF_CFG_DEV0_RC0_VENDOR_ID
+#define BIF_CFG_DEV0_RC0_VENDOR_ID__VENDOR_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_VENDOR_ID__VENDOR_ID_MASK                                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_DEVICE_ID
+#define BIF_CFG_DEV0_RC0_DEVICE_ID__DEVICE_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_ID__DEVICE_ID_MASK                                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_COMMAND
+#define BIF_CFG_DEV0_RC0_COMMAND__IOEN_DN__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC0_COMMAND__MEMEN_DN__SHIFT                                                             0x1
+#define BIF_CFG_DEV0_RC0_COMMAND__BUS_MASTER_EN__SHIFT                                                        0x2
+#define BIF_CFG_DEV0_RC0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_RC0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_RC0_COMMAND__PAL_SNOOP_EN__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_RC0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                                0x6
+#define BIF_CFG_DEV0_RC0_COMMAND__AD_STEPPING__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_RC0_COMMAND__SERR_EN__SHIFT                                                              0x8
+#define BIF_CFG_DEV0_RC0_COMMAND__FAST_B2B_EN__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_RC0_COMMAND__INT_DIS__SHIFT                                                              0xa
+#define BIF_CFG_DEV0_RC0_COMMAND__IOEN_DN_MASK                                                                0x0001L
+#define BIF_CFG_DEV0_RC0_COMMAND__MEMEN_DN_MASK                                                               0x0002L
+#define BIF_CFG_DEV0_RC0_COMMAND__BUS_MASTER_EN_MASK                                                          0x0004L
+#define BIF_CFG_DEV0_RC0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV0_RC0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_RC0_COMMAND__PAL_SNOOP_EN_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_RC0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_RC0_COMMAND__AD_STEPPING_MASK                                                            0x0080L
+#define BIF_CFG_DEV0_RC0_COMMAND__SERR_EN_MASK                                                                0x0100L
+#define BIF_CFG_DEV0_RC0_COMMAND__FAST_B2B_EN_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_RC0_COMMAND__INT_DIS_MASK                                                                0x0400L
+//BIF_CFG_DEV0_RC0_STATUS
+#define BIF_CFG_DEV0_RC0_STATUS__IMMEDIATE_READINESS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC0_STATUS__INT_STATUS__SHIFT                                                            0x3
+#define BIF_CFG_DEV0_RC0_STATUS__CAP_LIST__SHIFT                                                              0x4
+#define BIF_CFG_DEV0_RC0_STATUS__PCI_66_CAP__SHIFT                                                            0x5
+#define BIF_CFG_DEV0_RC0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_RC0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                              0x8
+#define BIF_CFG_DEV0_RC0_STATUS__DEVSEL_TIMING__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_RC0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_RC0_STATUS__IMMEDIATE_READINESS_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_RC0_STATUS__INT_STATUS_MASK                                                              0x0008L
+#define BIF_CFG_DEV0_RC0_STATUS__CAP_LIST_MASK                                                                0x0010L
+#define BIF_CFG_DEV0_RC0_STATUS__PCI_66_CAP_MASK                                                              0x0020L
+#define BIF_CFG_DEV0_RC0_STATUS__FAST_BACK_CAPABLE_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_RC0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                0x0100L
+#define BIF_CFG_DEV0_RC0_STATUS__DEVSEL_TIMING_MASK                                                           0x0600L
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                   0x4000L
+#define BIF_CFG_DEV0_RC0_STATUS__PARITY_ERROR_DETECTED_MASK                                                   0x8000L
+//BIF_CFG_DEV0_RC0_REVISION_ID
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MINOR_REV_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MINOR_REV_ID_MASK                                                       0x0FL
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MAJOR_REV_ID_MASK                                                       0xF0L
+//BIF_CFG_DEV0_RC0_PROG_INTERFACE
+#define BIF_CFG_DEV0_RC0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_PROG_INTERFACE__PROG_INTERFACE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_RC0_SUB_CLASS
+#define BIF_CFG_DEV0_RC0_SUB_CLASS__SUB_CLASS__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_SUB_CLASS__SUB_CLASS_MASK                                                            0xFFL
+//BIF_CFG_DEV0_RC0_BASE_CLASS
+#define BIF_CFG_DEV0_RC0_BASE_CLASS__BASE_CLASS__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_BASE_CLASS__BASE_CLASS_MASK                                                          0xFFL
+//BIF_CFG_DEV0_RC0_CACHE_LINE
+#define BIF_CFG_DEV0_RC0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                     0xFFL
+//BIF_CFG_DEV0_RC0_LATENCY
+#define BIF_CFG_DEV0_RC0_LATENCY__LATENCY_TIMER__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_LATENCY__LATENCY_TIMER_MASK                                                          0xFFL
+//BIF_CFG_DEV0_RC0_HEADER
+#define BIF_CFG_DEV0_RC0_HEADER__HEADER_TYPE__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC0_HEADER__DEVICE_TYPE__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_RC0_HEADER__HEADER_TYPE_MASK                                                             0x7FL
+#define BIF_CFG_DEV0_RC0_HEADER__DEVICE_TYPE_MASK                                                             0x80L
+//BIF_CFG_DEV0_RC0_BIST
+#define BIF_CFG_DEV0_RC0_BIST__BIST_COMP__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC0_BIST__BIST_STRT__SHIFT                                                               0x6
+#define BIF_CFG_DEV0_RC0_BIST__BIST_CAP__SHIFT                                                                0x7
+#define BIF_CFG_DEV0_RC0_BIST__BIST_COMP_MASK                                                                 0x0FL
+#define BIF_CFG_DEV0_RC0_BIST__BIST_STRT_MASK                                                                 0x40L
+#define BIF_CFG_DEV0_RC0_BIST__BIST_CAP_MASK                                                                  0x80L
+//BIF_CFG_DEV0_RC0_BASE_ADDR_1
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_1__BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_BASE_ADDR_2
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_2__BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER__SHIFT                               0x18
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                           0x0000FF00L
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                             0x00FF0000L
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER_MASK                                 0xFF000000L
+//BIF_CFG_DEV0_RC0_IO_BASE_LIMIT
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE_TYPE__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT_TYPE__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE_TYPE_MASK                                                     0x000FL
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT_TYPE_MASK                                                    0x0F00L
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT_MASK                                                         0xF000L
+//BIF_CFG_DEV0_RC0_SECONDARY_STATUS
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PCI_66_CAP__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__FAST_BACK_CAPABLE__SHIFT                                           0x7
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                    0x8
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__DEVSEL_TIMING__SHIFT                                               0x9
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                         0xb
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                       0xc
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                       0xd
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR__SHIFT                                       0xe
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PARITY_ERROR_DETECTED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PCI_66_CAP_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__FAST_BACK_CAPABLE_MASK                                             0x0080L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                      0x0100L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__DEVSEL_TIMING_MASK                                                 0x0600L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__SIGNAL_TARGET_ABORT_MASK                                           0x0800L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_TARGET_ABORT_MASK                                         0x1000L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_MASTER_ABORT_MASK                                         0x2000L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR_MASK                                         0x4000L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PARITY_ERROR_DETECTED_MASK                                         0x8000L
+//BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                0x4
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                   0x0000000FL
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                  0x0000FFF0L
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                             0x0000000FL
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                            0x0000FFF0L
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_PREF_BASE_UPPER
+#define BIF_CFG_DEV0_RC0_PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER
+#define BIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                              0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                              0x10
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                0xFFFF0000L
+//BIF_CFG_DEV0_RC0_CAP_PTR
+#define BIF_CFG_DEV0_RC0_CAP_PTR__CAP_PTR__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC0_CAP_PTR__CAP_PTR_MASK                                                                0xFFL
+//BIF_CFG_DEV0_RC0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                          0x1
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                      0xb
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                       0x00000001L
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                            0x0000000EL
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                           0x000000F0L
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                        0xFFFFF800L
+//BIF_CFG_DEV0_RC0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_RC0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_RC0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_RC0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                    0xFFL
+//BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SERR_EN__SHIFT                                                      0x1
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__FAST_B2B_EN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER__SHIFT                                        0x8
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER__SHIFT                                      0x9
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS__SHIFT                                         0xa
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE__SHIFT                                    0xb
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN_MASK                                             0x0001L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SERR_EN_MASK                                                        0x0002L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE_MASK                                              0x0020L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__FAST_B2B_EN_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER_MASK                                          0x0100L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER_MASK                                        0x0200L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS_MASK                                           0x0400L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE_MASK                                      0x0800L
+//BIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL
+#define BIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                  0x01L
+//BIF_CFG_DEV0_RC0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC0_PMI_CAP
+#define BIF_CFG_DEV0_RC0_PMI_CAP__VERSION__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_CLOCK__SHIFT                                                            0x3
+#define BIF_CFG_DEV0_RC0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                  0x4
+#define BIF_CFG_DEV0_RC0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_RC0_PMI_CAP__AUX_CURRENT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D1_SUPPORT__SHIFT                                                           0x9
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D2_SUPPORT__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_SUPPORT__SHIFT                                                          0xb
+#define BIF_CFG_DEV0_RC0_PMI_CAP__VERSION_MASK                                                                0x0007L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_CLOCK_MASK                                                              0x0008L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                    0x0010L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__AUX_CURRENT_MASK                                                            0x01C0L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D1_SUPPORT_MASK                                                             0x0200L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D2_SUPPORT_MASK                                                             0x0400L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_SUPPORT_MASK                                                            0xF800L
+//BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                                0x16
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                   0x17
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                     0x18
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                    0x00000003L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                  0x00000008L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_EN_MASK                                                         0x00000100L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                    0x00001E00L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                     0x00006000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                     0x00008000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                  0x00400000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                     0x00800000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                       0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_RC0_PCIE_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__VERSION_MASK                                                               0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__DEVICE_TYPE_MASK                                                           0x00F0L
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                      0x0100L
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                       0x3E00L
+//BIF_CFG_DEV0_RC0_DEVICE_CAP
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                      0x3
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                             0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                          0xf
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                         0x1a
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                       0x1c
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                        0x00000018L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__EXTENDED_TAG_MASK                                                        0x00000020L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                              0x000001C0L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                               0x00000E00L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                            0x00008000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                            0x00010000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                           0x03FC0000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                           0x0C000000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__FLR_CAPABLE_MASK                                                         0x10000000L
+//BIF_CFG_DEV0_RC0_DEVICE_CNTL
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                  0xa
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                      0xb
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                            0xc
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN__SHIFT                                              0xf
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                   0x00E0L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                    0x0100L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                    0x0200L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                    0x0400L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                        0x0800L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                              0x7000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN_MASK                                                0x8000L
+//BIF_CFG_DEV0_RC0_DEVICE_STATUS
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__CORR_ERR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                      0x2
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__AUX_PWR__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                              0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__CORR_ERR_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                    0x0002L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__FATAL_ERR_MASK                                                        0x0004L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__USR_DETECTED_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__AUX_PWR_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                                0x0020L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                    0x0040L
+//BIF_CFG_DEV0_RC0_LINK_CAP
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_SPEED__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_WIDTH__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PM_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_RC0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                              0x12
+#define BIF_CFG_DEV0_RC0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                         0x13
+#define BIF_CFG_DEV0_RC0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                            0x15
+#define BIF_CFG_DEV0_RC0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                         0x16
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PORT_NUMBER__SHIFT                                                         0x18
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_SPEED_MASK                                                            0x0000000FL
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_WIDTH_MASK                                                            0x000003F0L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PM_SUPPORT_MASK                                                            0x00000C00L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                      0x00007000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                       0x00038000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                                0x00040000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                           0x00080000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                           0x00100000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                              0x00200000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                           0x00400000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PORT_NUMBER_MASK                                                           0xFF000000L
+//BIF_CFG_DEV0_RC0_LINK_CNTL
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PM_CONTROL__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                       0x2
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_DIS__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                   0x6
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                          0xb
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                              0xe
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PM_CONTROL_MASK                                                           0x0003L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                         0x0004L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_DIS_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__RETRAIN_LINK_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                     0x0040L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__EXTENDED_SYNC_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                            0x0100L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                            0x0400L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                            0x0800L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                                0xC000L
+//BIF_CFG_DEV0_RC0_LINK_STATUS
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                            0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_TRAINING__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__DL_ACTIVE__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                        0xe
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                        0xf
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                 0x000FL
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                              0x03F0L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_TRAINING_MASK                                                      0x0800L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__DL_ACTIVE_MASK                                                          0x2000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                          0x4000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                          0x8000L
+//BIF_CFG_DEV0_RC0_SLOT_CAP
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_BUTTON_PRESENT__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_CONTROLLER_PRESENT__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__MRL_SENSOR_PRESENT__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_INDICATOR_PRESENT__SHIFT                                              0x3
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_INDICATOR_PRESENT__SHIFT                                               0x4
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_SURPRISE__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_CAPABLE__SHIFT                                                     0x6
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED__SHIFT                                      0x12
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PHYSICAL_SLOT_NUM__SHIFT                                                   0x13
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_BUTTON_PRESENT_MASK                                                   0x00000001L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_CONTROLLER_PRESENT_MASK                                                0x00000002L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__MRL_SENSOR_PRESENT_MASK                                                    0x00000004L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_INDICATOR_PRESENT_MASK                                                0x00000008L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_INDICATOR_PRESENT_MASK                                                 0x00000010L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_SURPRISE_MASK                                                      0x00000020L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_CAPABLE_MASK                                                       0x00000040L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                  0x00007F80L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                  0x00018000L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT_MASK                                         0x00020000L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED_MASK                                        0x00040000L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PHYSICAL_SLOT_NUM_MASK                                                     0xFFF80000L
+//BIF_CFG_DEV0_RC0_SLOT_CNTL
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_FAULT_DETECTED_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__MRL_SENSOR_CHANGED_EN__SHIFT                                              0x2
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN__SHIFT                                         0x3
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__HOTPLUG_INTR_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_INDICATOR_CNTL__SHIFT                                                0x6
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_INDICATOR_CNTL__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_CONTROLLER_CNTL__SHIFT                                                0xa
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL__SHIFT                                         0xb
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__DL_STATE_CHANGED_EN__SHIFT                                                0xc
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE__SHIFT                                        0xd
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__INBAND_PD_DISABLE__SHIFT                                                  0xe
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN_MASK                                               0x0001L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_FAULT_DETECTED_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__MRL_SENSOR_CHANGED_EN_MASK                                                0x0004L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN_MASK                                           0x0008L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN_MASK                                            0x0010L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__HOTPLUG_INTR_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_INDICATOR_CNTL_MASK                                                  0x00C0L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_INDICATOR_CNTL_MASK                                                   0x0300L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_CONTROLLER_CNTL_MASK                                                  0x0400L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL_MASK                                           0x0800L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__DL_STATE_CHANGED_EN_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE_MASK                                          0x2000L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__INBAND_PD_DISABLE_MASK                                                    0x4000L
+//BIF_CFG_DEV0_RC0_SLOT_STATUS
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ATTN_BUTTON_PRESSED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PWR_FAULT_DETECTED__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_CHANGED__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_CHANGED__SHIFT                                          0x3
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__COMMAND_COMPLETED__SHIFT                                                0x4
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_STATE__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_STATE__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS__SHIFT                                     0x7
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__DL_STATE_CHANGED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ATTN_BUTTON_PRESSED_MASK                                                0x0001L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PWR_FAULT_DETECTED_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_CHANGED_MASK                                                 0x0004L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_CHANGED_MASK                                            0x0008L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__COMMAND_COMPLETED_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_STATE_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_STATE_MASK                                              0x0040L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS_MASK                                       0x0080L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__DL_STATE_CHANGED_MASK                                                   0x0100L
+//BIF_CFG_DEV0_RC0_ROOT_CNTL
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_CORR_ERR_EN__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_FATAL_ERR_EN__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__PM_INTERRUPT_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_CORR_ERR_EN_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_FATAL_ERR_EN_MASK                                                 0x0004L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__PM_INTERRUPT_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                           0x0010L
+//BIF_CFG_DEV0_RC0_ROOT_CAP
+#define BIF_CFG_DEV0_RC0_ROOT_CAP__CRS_SOFTWARE_VISIBILITY__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_ROOT_CAP__CRS_SOFTWARE_VISIBILITY_MASK                                               0x0001L
+//BIF_CFG_DEV0_RC0_ROOT_STATUS
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_REQUESTOR_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_STATUS__SHIFT                                                       0x10
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_PENDING__SHIFT                                                      0x11
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_REQUESTOR_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_STATUS_MASK                                                         0x00010000L
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_PENDING_MASK                                                        0x00020000L
+//BIF_CFG_DEV0_RC0_DEVICE_CAP2
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                        0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                         0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                       0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                       0x7
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                           0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                        0xa
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                               0xc
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                    0xe
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                  0x10
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                  0x11
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                   0x12
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                     0x14
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                     0x15
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                         0x16
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                   0x18
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                    0x1f
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                        0x0000000FL
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                          0x00000010L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                           0x00000020L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                         0x00000080L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                             0x00000200L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                          0x00000400L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                      0x00000800L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                 0x00003000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                      0x0000C000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                     0x000C0000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                       0x00100000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                           0x00C00000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                     0x03000000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                      0x04000000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                      0x80000000L
+//BIF_CFG_DEV0_RC0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                               0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                        0x7
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                              0x8
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                           0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__LTR_EN__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                    0xb
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                    0xc
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                         0xd
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                 0x000FL
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                 0x0020L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                               0x0040L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                          0x0080L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                                0x0100L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                             0x0200L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__LTR_EN_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                      0x0800L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                      0x1000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__OBFF_EN_MASK                                                           0x6000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                       0x8000L
+//BIF_CFG_DEV0_RC0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS2__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS2__RESERVED_MASK                                                        0xFFFFL
+//BIF_CFG_DEV0_RC0_LINK_CAP2
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                                0x8
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                           0x9
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                          0x17
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                          0x18
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                      0x1f
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                 0x000000FEL
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                  0x00000100L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                             0x0000FE00L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                             0x007F0000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                            0x00800000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                            0x01000000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__DRS_SUPPORTED_MASK                                                        0x80000000L
+//BIF_CFG_DEV0_RC0_LINK_CNTL2
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                       0x5
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                              0xa
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                             0xc
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                   0x000FL
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                         0x0020L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                               0x0040L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__XMIT_MARGIN_MASK                                                         0x0380L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                                0x0400L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                      0x0800L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                               0xF000L
+//BIF_CFG_DEV0_RC0_LINK_STATUS2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                 0x2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                 0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                   0x5
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                               0x6
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                               0x7
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                            0x8
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                              0x0001L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                         0x0002L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                   0x0004L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                   0x0008L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                   0x0010L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                     0x0020L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                 0x0080L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                              0x0300L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                     0x7000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                              0x8000L
+//BIF_CFG_DEV0_RC0_SLOT_CAP2
+#define BIF_CFG_DEV0_RC0_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED_MASK                                          0x00000001L
+//BIF_CFG_DEV0_RC0_SLOT_CNTL2
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL2__RESERVED__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL2__RESERVED_MASK                                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_SLOT_STATUS2
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS2__RESERVED__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS2__RESERVED_MASK                                                          0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EN_MASK                                                            0x0001L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                     0x000EL
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                      0x0070L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                         0x0100L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                              0x0200L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                               0x0400L
+//BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                              0x2
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                                0xFFFFFFFCL
+//BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA__MSI_DATA_MASK                                                          0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                  0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                    0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_SSID_CAP_LIST
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_RC0_SSID_CAP
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_VENDOR_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_ID__SHIFT                                                        0x10
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_VENDOR_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_ID_MASK                                                          0xFFFF0000L
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                   0x14
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                       0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                      0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                     0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                              0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                              0x4
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                                0x8
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                              0xa
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                             0x00000007L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                                0x00000070L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                  0x00000300L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                                0x00000C00L
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                    0x18
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                      0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                            0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                                0x000EL
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                        0x0001L
+//BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                         0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                  0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                           0x007F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                    0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                 0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                             0x1f
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                         0x000000FEL
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                         0x000E0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                   0x07000000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                               0x80000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                              0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                 0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                0x0002L
+//BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                         0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                  0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                           0x003F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                    0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                 0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                             0x1f
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                         0x000000FEL
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                         0x000E0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                   0x07000000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                               0x80000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                              0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                 0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                0x0002L
+//BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                     0x5
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                         0xd
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                      0x10
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                        0x12
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                       0x13
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                  0x15
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                 0x16
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                 0x17
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                        0x18
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                         0x19
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                    0x1a
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                          0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                       0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                          0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                           0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                    0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                        0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                         0x00020000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                          0x00040000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                         0x00080000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                   0x00100000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                   0x00400000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                   0x00800000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                          0x01000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                           0x02000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                      0x04000000L
+//BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                         0x5
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                            0xc
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                             0xd
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                           0x11
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                            0x12
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                           0x13
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                     0x14
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                      0x15
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                     0x16
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                     0x17
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                            0x18
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                             0x19
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                        0x1a
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                              0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                           0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                              0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                               0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                        0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                            0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                             0x00020000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                              0x00040000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                             0x00080000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                       0x00100000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                        0x00200000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                       0x00400000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                              0x01000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                               0x02000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                          0x04000000L
+//BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                    0x4
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                 0x5
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                    0xc
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                     0xd
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                              0xf
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                  0x10
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                   0x11
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                    0x12
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                   0x13
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                             0x14
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                              0x15
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                             0x16
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                             0x17
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                    0x18
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                     0x19
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT                0x1a
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                      0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                   0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                      0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                       0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                                0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                     0x00020000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                      0x00040000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                     0x00080000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                               0x00100000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                                0x00200000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                               0x00400000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                               0x00800000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                      0x01000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                       0x02000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                  0x04000000L
+//BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                             0xc
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                            0xd
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                            0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                           0x00000080L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                                0x00000100L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                               0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                              0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                       0x00008000L
+//BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                              0x6
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                             0x7
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                 0xc
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                                0xd
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                         0xe
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                         0xf
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                                0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                               0x00000080L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                           0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                           0x00008000L
+//BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                           0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                     0x9
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                      0xa
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                 0xb
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                         0xc
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                            0x0000001FL
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                             0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                              0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                           0x00000080L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                            0x00000100L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                       0x00000200L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                        0x00000400L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                   0x00000800L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                           0x00001000L
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN__SHIFT                                           0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN_MASK                                             0x00000004L
+//BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD__SHIFT                                      0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD__SHIFT                                 0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD__SHIFT                            0x3
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD__SHIFT                                 0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD__SHIFT                                    0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM__SHIFT                                     0x1b
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD_MASK                                        0x00000002L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD_MASK                                   0x00000004L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD_MASK                              0x00000008L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD_MASK                                   0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS_MASK                                          0x00000180L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM_MASK                                       0xF8000000L
+//BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID_MASK                                      0xFFFF0000L
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                 0x1
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                      0x9
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                   0x00000002L
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                        0x0000FE00L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                  0x0000FFFFL
+//BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                            0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                            0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                         0x3
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                              0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                           0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                             0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                 0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                              0x0002L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                              0x0004L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                           0x0008L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                               0x0010L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                                0x0020L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                             0x0040L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                               0x0080L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                        0xFF00L
+//BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                        0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                     0x3
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                  0xa
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                                0xc
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                             0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                          0x0004L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                       0x0008L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                         0x0040L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                           0x0080L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                    0x0300L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                    0x0C00L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                  0x1000L
+//BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                    0x0
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                    0x1f
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                      0x007FFFFFL
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                      0x80000000L
+//BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                  0x007FFFFFL
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC0_LINK_CAP_16GT
+#define BIF_CFG_DEV0_RC0_LINK_CAP_16GT__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC0_LINK_CAP_16GT__RESERVED_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_16GT__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_16GT__RESERVED_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                            0x1
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                    0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                              0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                              0x00000004L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                              0x00000008L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                                0x00000010L
+//BIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT          0x0
+#define BIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK            0x0000FFFFL
+//BIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT            0x0
+#define BIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK              0x0000FFFFL
+//BIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT            0x0
+#define BIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK              0x0000FFFFL
+//BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                   0x0
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                     0x0001L
+//BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                        0x0
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                               0x1
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                          0x0001L
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                 0x0002L
+//BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LINK_CAP_32GT
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                              0x9
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                              0xa
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                               0xb
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                              0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                                0x00000100L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                                0x00000200L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                                0x00000400L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                 0x0000F800L
+//BIF_CFG_DEV0_RC0_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                    0x8
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                                0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                      0x00000700L
+//BIF_CFG_DEV0_RC0_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                            0x1
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                    0x8
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                 0x9
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                       0xa
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                    0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                              0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                              0x00000004L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                              0x00000008L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                                0x00000010L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                          0x000000C0L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                   0x00000200L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                         0x00000400L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_COMMAND
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF0_0_STATUS
+#define BIF_CFG_DEV0_EPF0_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF0_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF0_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF0_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF0_0_LATENCY
+#define BIF_CFG_DEV0_EPF0_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_0_HEADER
+#define BIF_CFG_DEV0_EPF0_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF0_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF0_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF0_0_BIST
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF0_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF0_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                             0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                           0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                           0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                          0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                             0x00000070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                             0x00000C00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                             0x000EL
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                     0x0001L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                               0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                          0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                    0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                 0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                          0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                        0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                      0x000000FEL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                      0x000E0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                0x07000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                            0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                              0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                             0x0002L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                               0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                          0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                    0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                        0x003F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                 0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                          0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                        0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                      0x000000FEL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                      0x000E0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                0x07000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                            0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                              0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                             0x0002L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                 0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                     0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                    0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                        0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                     0x0000FE00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                               0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                     0x0080L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__STU__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__STU_MASK                                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_ENABLE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_RESET__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_ENABLE_MASK                                               0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_RESET_MASK                                                0x0002L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX__SHIFT                        0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__STOPPED__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED__SHIFT                          0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX_MASK                          0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__STOPPED_MASK                                                0x0100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED_MASK                            0x8000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY_MASK                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC_MASK                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ENH_CAP_LIST__CAP_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ENH_CAP_LIST__CAP_VER__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                             0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ENH_CAP_LIST__CAP_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ENH_CAP_LIST__CAP_VER_MASK                                                0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ENH_CAP_LIST__NEXT_PTR_MASK                                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_CAP__MC_MAX_GROUP__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_CAP__MC_WIN_SIZE_REQ__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_CAP__MC_ECRC_REGEN_SUPP__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_CAP__MC_MAX_GROUP_MASK                                                    0x003FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_CAP__MC_WIN_SIZE_REQ_MASK                                                 0x3F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_CAP__MC_ECRC_REGEN_SUPP_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_CNTL__MC_NUM_GROUP__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_CNTL__MC_ENABLE__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_CNTL__MC_NUM_GROUP_MASK                                                   0x003FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_CNTL__MC_ENABLE_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR0__MC_INDEX_POS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR0__MC_BASE_ADDR_0__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR0__MC_INDEX_POS_MASK                                                  0x0000003FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR0__MC_BASE_ADDR_0_MASK                                                0xFFFFF000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR1__MC_BASE_ADDR_1__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_ADDR1__MC_BASE_ADDR_1_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_RCV0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_RCV0__MC_RECEIVE_0__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_RCV0__MC_RECEIVE_0_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_RCV1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_RCV1__MC_RECEIVE_1__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_RCV1__MC_RECEIVE_1_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_ALL0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_ALL0__MC_BLOCK_ALL_0__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_ALL0__MC_BLOCK_ALL_0_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_ALL1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_ALL1__MC_BLOCK_ALL_1__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_ALL1__MC_BLOCK_ALL_1_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_UNTRANSLATED_0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_UNTRANSLATED_0__MC_BLOCK_UNTRANSLATED_0__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_UNTRANSLATED_0__MC_BLOCK_UNTRANSLATED_0_MASK                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_UNTRANSLATED_1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_UNTRANSLATED_1__MC_BLOCK_UNTRANSLATED_1__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MC_BLOCK_UNTRANSLATED_1__MC_BLOCK_UNTRANSLATED_1_MASK                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE__SHIFT                                      0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE__SHIFT                                     0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE_MASK                                        0x000003FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE_MASK                                        0x00001C00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE_MASK                                       0x03FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE_MASK                                       0x1C000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED__SHIFT                          0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM__SHIFT                            0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED_MASK                            0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM_MASK                              0xFFE00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE__SHIFT                              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE__SHIFT                                           0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE_MASK                                0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE_MASK                           0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE_MASK                                             0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE_MASK                    0x0020L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS_MASK                                 0x0001L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS_MASK                                    0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS_MASK                                        0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS_MASK                                            0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK_MASK                                0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET_MASK                            0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE_MASK                                        0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID_MASK                                  0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE_MASK                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE_MASK                          0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR__SHIFT   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR_MASK     0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_MASK  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                 0x1f
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                   0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                               0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT__RESERVED__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT__RESERVED_MASK                                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT__RESERVED_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                         0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                         0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                           0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                 0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                           0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                           0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                             0x00000010L
+//BIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT       0x0
+#define BIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK         0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                  0x0001L
+//BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                            0x1
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                              0x0002L
+//BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                           0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                           0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                        0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                             0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                             0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                              0x0000F800L
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                             0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                   0x00000700L
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                         0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                         0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                           0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                     0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                     0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                              0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                    0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                 0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                           0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                           0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                             0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                       0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                       0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                      0x00000400L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER__SHIFT                          0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR__SHIFT                         0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID_MASK                             0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER_MASK                            0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR_MASK                           0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID_MASK                                     0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV_MASK                                    0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN_MASK    0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN_MASK  0x02000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS_MASK  0x02000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR_MASK                   0x0001L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID__SHIFT               0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA__SHIFT                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX_MASK                      0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA_MASK                  0x00000F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID_MASK                 0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA_MASK                  0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK_MASK                   0x01000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID__SHIFT               0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK__SHIFT                 0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID__SHIFT               0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID__SHIFT               0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK__SHIFT                 0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID__SHIFT               0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID__SHIFT               0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK__SHIFT                 0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID__SHIFT               0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK__SHIFT                 0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID__SHIFT               0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK__SHIFT                 0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID__SHIFT               0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK__SHIFT                 0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID__SHIFT               0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK__SHIFT                 0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID__SHIFT               0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK__SHIFT                0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID__SHIFT              0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK__SHIFT                0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID__SHIFT              0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK__SHIFT                0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID__SHIFT              0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK__SHIFT                0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID__SHIFT              0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK__SHIFT                0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID__SHIFT              0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK__SHIFT                0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID__SHIFT              0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK_MASK                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID_MASK                 0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK_MASK                   0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID_MASK                 0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK_MASK                   0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID_MASK                 0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK_MASK                   0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID_MASK                 0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK_MASK                   0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID_MASK                 0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK_MASK                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID_MASK                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK_MASK                   0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID_MASK                 0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK_MASK                   0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID_MASK                 0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK_MASK                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID_MASK                 0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK_MASK                   0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID_MASK                 0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK_MASK                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID_MASK                0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK_MASK                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID_MASK                0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK_MASK                  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID_MASK                0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK_MASK                  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID_MASK                0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK_MASK                  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID_MASK                0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK_MASK                  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID_MASK                0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID__SHIFT              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK__SHIFT                0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID__SHIFT              0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK__SHIFT                0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID__SHIFT              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK__SHIFT                0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID__SHIFT              0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID__SHIFT              0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK__SHIFT                0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID__SHIFT              0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK__SHIFT                0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID__SHIFT              0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK__SHIFT                0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID__SHIFT              0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK__SHIFT                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID__SHIFT              0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK__SHIFT                0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID__SHIFT              0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK__SHIFT                0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID__SHIFT              0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK__SHIFT                0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID__SHIFT              0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK__SHIFT                0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID__SHIFT              0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK__SHIFT                0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID__SHIFT              0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK__SHIFT                0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID__SHIFT              0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK__SHIFT                  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID__SHIFT                0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK_MASK                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID_MASK                0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK_MASK                  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID_MASK                0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK_MASK                  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID_MASK                0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK_MASK                  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID_MASK                0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK_MASK                  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID_MASK                0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK_MASK                  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID_MASK                0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK_MASK                  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID_MASK                0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK_MASK                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID_MASK                0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK_MASK                  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID_MASK                0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK_MASK                  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID_MASK                0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK_MASK                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID_MASK                0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK_MASK                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID_MASK                0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK_MASK                  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID_MASK                0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK_MASK                  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID_MASK                0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK_MASK                  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID_MASK                0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK_MASK                    0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID_MASK                  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__CONTEXT_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__LOC__SHIFT                               0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__CONTEXT_OFFSET__SHIFT                    0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__CONTEXT_SIZE_MASK                        0x0000007FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__LOC_MASK                                 0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT__CONTEXT_OFFSET_MASK                      0xFFFFFC00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_AVAILABLE__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_CONSUMED__SHIFT                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_AVAILABLE_MASK                 0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_CONSUMED_MASK                  0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__LFB_REGION__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__MAX_REGION__SHIFT                         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__LFB_REGION_MASK                           0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__MAX_REGION_MASK                           0x000000F0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_VF__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_PF__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_VF_MASK  0x7FFFFFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_PF_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB__VF0_FB_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB__VF0_FB_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB__VF0_FB_SIZE_MASK                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB__VF0_FB_OFFSET_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB__VF1_FB_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB__VF1_FB_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB__VF1_FB_SIZE_MASK                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB__VF1_FB_OFFSET_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB__VF2_FB_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB__VF2_FB_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB__VF2_FB_SIZE_MASK                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB__VF2_FB_OFFSET_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB__VF3_FB_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB__VF3_FB_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB__VF3_FB_SIZE_MASK                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB__VF3_FB_OFFSET_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB__VF4_FB_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB__VF4_FB_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB__VF4_FB_SIZE_MASK                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB__VF4_FB_OFFSET_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB__VF5_FB_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB__VF5_FB_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB__VF5_FB_SIZE_MASK                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB__VF5_FB_OFFSET_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB__VF6_FB_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB__VF6_FB_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB__VF6_FB_SIZE_MASK                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB__VF6_FB_OFFSET_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB__VF7_FB_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB__VF7_FB_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB__VF7_FB_SIZE_MASK                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB__VF7_FB_OFFSET_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB__VF8_FB_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB__VF8_FB_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB__VF8_FB_SIZE_MASK                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB__VF8_FB_OFFSET_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB__VF9_FB_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB__VF9_FB_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB__VF9_FB_SIZE_MASK                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB__VF9_FB_OFFSET_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB__VF10_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB__VF10_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB__VF10_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB__VF10_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB__VF11_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB__VF11_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB__VF11_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB__VF11_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB__VF12_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB__VF12_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB__VF12_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB__VF12_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB__VF13_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB__VF13_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB__VF13_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB__VF13_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB__VF14_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB__VF14_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB__VF14_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB__VF14_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB__VF15_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB__VF15_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB__VF15_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB__VF15_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB__VF16_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB__VF16_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB__VF16_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF16_FB__VF16_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB__VF17_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB__VF17_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB__VF17_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF17_FB__VF17_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB__VF18_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB__VF18_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB__VF18_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF18_FB__VF18_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB__VF19_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB__VF19_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB__VF19_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF19_FB__VF19_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB__VF20_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB__VF20_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB__VF20_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF20_FB__VF20_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB__VF21_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB__VF21_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB__VF21_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF21_FB__VF21_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB__VF22_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB__VF22_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB__VF22_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF22_FB__VF22_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB__VF23_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB__VF23_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB__VF23_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF23_FB__VF23_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB__VF24_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB__VF24_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB__VF24_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF24_FB__VF24_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB__VF25_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB__VF25_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB__VF25_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF25_FB__VF25_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB__VF26_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB__VF26_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB__VF26_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF26_FB__VF26_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB__VF27_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB__VF27_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB__VF27_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF27_FB__VF27_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB__VF28_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB__VF28_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB__VF28_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF28_FB__VF28_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB__VF29_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB__VF29_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB__VF29_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF29_FB__VF29_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB__VF30_FB_SIZE__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB__VF30_FB_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB__VF30_FB_SIZE_MASK                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF30_FB__VF30_FB_OFFSET_MASK                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN0SCH_OFFSET__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN1SCH_OFFSET__SHIFT                    0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN2SCH_OFFSET__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN3SCH_OFFSET__SHIFT                    0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN0SCH_OFFSET_MASK                      0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN1SCH_OFFSET_MASK                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN2SCH_OFFSET_MASK                      0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS__VCN3SCH_OFFSET_MASK                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN4SCH_OFFSET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN5SCH_OFFSET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN6SCH_OFFSET__SHIFT                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN7SCH_OFFSET__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN4SCH_OFFSET_MASK                     0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN5SCH_OFFSET_MASK                     0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN6SCH_OFFSET_MASK                     0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__VCN7SCH_OFFSET_MASK                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN8SCH_OFFSET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN9SCH_OFFSET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN10SCH_OFFSET__SHIFT                  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN11SCH_OFFSET__SHIFT                  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN8SCH_OFFSET_MASK                     0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN9SCH_OFFSET_MASK                     0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN10SCH_OFFSET_MASK                    0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__VCN11SCH_OFFSET_MASK                    0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX0SCH_OFFSET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX1SCH_OFFSET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX2SCH_OFFSET__SHIFT                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX3SCH_OFFSET__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX0SCH_OFFSET_MASK                     0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX1SCH_OFFSET_MASK                     0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX2SCH_OFFSET_MASK                     0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__GFX3SCH_OFFSET_MASK                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX4SCH_OFFSET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX5SCH_OFFSET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX6SCH_OFFSET__SHIFT                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX7SCH_OFFSET__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX4SCH_OFFSET_MASK                     0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX5SCH_OFFSET_MASK                     0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX6SCH_OFFSET_MASK                     0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__GFX7SCH_OFFSET_MASK                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD0SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD1SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD2SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD3SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD4SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD5SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD6SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD7SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD8SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD9SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW0__DW0__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW0__DW0_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW1__DW1__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW1__DW1_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW2__DW2__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW2__DW2_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW3__DW3__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW3__DW3_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW4__DW4__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW4__DW4_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW5__DW5__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW5__DW5_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW6__DW6__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW6__DW6_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW7__DW7__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW7__DW7_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW8__DW8__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD10SCH_DW8__DW8_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW0__DW0__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW0__DW0_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW1__DW1__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW1__DW1_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW2__DW2__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW2__DW2_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW3__DW3__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW3__DW3_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW4__DW4__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW4__DW4_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW5__DW5__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW5__DW5_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW6__DW6__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW6__DW6_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW7__DW7__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW7__DW7_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW8__DW8__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVD11SCH_DW8__DW8_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX0SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX1SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX2SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX3SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX4SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX5SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX6SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW0__DW0__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW0__DW0_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW1__DW1__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW1__DW1_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW2__DW2__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW2__DW2_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW3__DW3__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW3__DW3_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW4__DW4__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW4__DW4_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW5__DW5__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW5__DW5_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW6__DW6__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW6__DW6_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW7__DW7__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW7__DW7_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW8__DW8__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFX7SCH_DW8__DW8_MASK                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_CMD_COMPLETE_INTR_EN__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_HANG_NEED_FLR_INTR_EN__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_CMD_COMPLETE_INTR_EN__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_HANG_NEED_FLR_INTR_EN__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_CMD_COMPLETE_INTR_EN__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_HANG_NEED_FLR_INTR_EN__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_CMD_COMPLETE_INTR_EN__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_HANG_NEED_FLR_INTR_EN__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_CMD_COMPLETE_INTR_EN__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_HANG_NEED_FLR_INTR_EN__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_CMD_COMPLETE_INTR_EN__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_HANG_NEED_FLR_INTR_EN__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_CMD_COMPLETE_INTR_EN__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_HANG_NEED_FLR_INTR_EN__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_CMD_COMPLETE_INTR_EN__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_HANG_NEED_FLR_INTR_EN__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_CMD_COMPLETE_INTR_EN_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_HANG_NEED_FLR_INTR_EN_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A0_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_CMD_COMPLETE_INTR_EN_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_HANG_NEED_FLR_INTR_EN_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A1_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_CMD_COMPLETE_INTR_EN_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_HANG_NEED_FLR_INTR_EN_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A2_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_CMD_COMPLETE_INTR_EN_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_HANG_NEED_FLR_INTR_EN_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A3_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_CMD_COMPLETE_INTR_EN_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_HANG_NEED_FLR_INTR_EN_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A4_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_CMD_COMPLETE_INTR_EN_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_HANG_NEED_FLR_INTR_EN_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A5_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_CMD_COMPLETE_INTR_EN_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_HANG_SELF_RECOVERED_INTR_EN_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_HANG_NEED_FLR_INTR_EN_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A6_VM_BUSY_TRANSITION_INTR_EN_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_CMD_COMPLETE_INTR_EN_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_HANG_SELF_RECOVERED_INTR_EN_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_HANG_NEED_FLR_INTR_EN_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_ENABLE__ENGA_A7_VM_BUSY_TRANSITION_INTR_EN_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A8_15_INTR_ENABLE
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_CMD_COMPLETE_INTR_EN__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_HANG_NEED_FLR_INTR_EN__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_CMD_COMPLETE_INTR_EN__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_HANG_NEED_FLR_INTR_EN__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_CMD_COMPLETE_INTR_EN__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_HANG_NEED_FLR_INTR_EN__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_CMD_COMPLETE_INTR_EN__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_HANG_NEED_FLR_INTR_EN__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_CMD_COMPLETE_INTR_EN__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_HANG_NEED_FLR_INTR_EN__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_CMD_COMPLETE_INTR_EN__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_HANG_NEED_FLR_INTR_EN__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_CMD_COMPLETE_INTR_EN__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_HANG_NEED_FLR_INTR_EN__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_CMD_COMPLETE_INTR_EN__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_HANG_NEED_FLR_INTR_EN__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_CMD_COMPLETE_INTR_EN_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_HANG_NEED_FLR_INTR_EN_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B0_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_CMD_COMPLETE_INTR_EN_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_HANG_NEED_FLR_INTR_EN_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B1_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_CMD_COMPLETE_INTR_EN_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_HANG_NEED_FLR_INTR_EN_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B2_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_CMD_COMPLETE_INTR_EN_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_HANG_NEED_FLR_INTR_EN_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B3_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_CMD_COMPLETE_INTR_EN_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_HANG_NEED_FLR_INTR_EN_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B4_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_CMD_COMPLETE_INTR_EN_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_HANG_NEED_FLR_INTR_EN_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B5_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_CMD_COMPLETE_INTR_EN_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_HANG_SELF_RECOVERED_INTR_EN_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_HANG_NEED_FLR_INTR_EN_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B6_VM_BUSY_TRANSITION_INTR_EN_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_CMD_COMPLETE_INTR_EN_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_HANG_SELF_RECOVERED_INTR_EN_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_HANG_NEED_FLR_INTR_EN_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_ENABLE__ENGB_B7_VM_BUSY_TRANSITION_INTR_EN_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_CMD_COMPLETE_INTR_EN__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_HANG_NEED_FLR_INTR_EN__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_CMD_COMPLETE_INTR_EN__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_HANG_NEED_FLR_INTR_EN__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_CMD_COMPLETE_INTR_EN__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_HANG_NEED_FLR_INTR_EN__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_CMD_COMPLETE_INTR_EN__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_HANG_NEED_FLR_INTR_EN__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_CMD_COMPLETE_INTR_EN_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_HANG_NEED_FLR_INTR_EN_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B8_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_CMD_COMPLETE_INTR_EN_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_HANG_NEED_FLR_INTR_EN_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B9_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_CMD_COMPLETE_INTR_EN_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_HANG_NEED_FLR_INTR_EN_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B10_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_CMD_COMPLETE_INTR_EN_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_HANG_NEED_FLR_INTR_EN_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_ENABLE__ENGB_B11_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00008000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_CMD_COMPLETE_INTR_STATUS__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_CMD_COMPLETE_INTR_STATUS__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_CMD_COMPLETE_INTR_STATUS__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_CMD_COMPLETE_INTR_STATUS__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_CMD_COMPLETE_INTR_STATUS__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_CMD_COMPLETE_INTR_STATUS__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_CMD_COMPLETE_INTR_STATUS__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_CMD_COMPLETE_INTR_STATUS__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_CMD_COMPLETE_INTR_STATUS_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A0_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_CMD_COMPLETE_INTR_STATUS_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A1_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_CMD_COMPLETE_INTR_STATUS_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A2_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_CMD_COMPLETE_INTR_STATUS_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_HANG_NEED_FLR_INTR_STATUS_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A3_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_CMD_COMPLETE_INTR_STATUS_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_HANG_NEED_FLR_INTR_STATUS_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A4_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_CMD_COMPLETE_INTR_STATUS_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_HANG_NEED_FLR_INTR_STATUS_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A5_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_CMD_COMPLETE_INTR_STATUS_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_HANG_NEED_FLR_INTR_STATUS_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A6_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_CMD_COMPLETE_INTR_STATUS_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_HANG_NEED_FLR_INTR_STATUS_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A0_7_INTR_STATUS__ENGA_A7_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGA_A8_15_INTR_STATUS
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_CMD_COMPLETE_INTR_STATUS__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_CMD_COMPLETE_INTR_STATUS__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_CMD_COMPLETE_INTR_STATUS__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_CMD_COMPLETE_INTR_STATUS__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_CMD_COMPLETE_INTR_STATUS__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_CMD_COMPLETE_INTR_STATUS__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_CMD_COMPLETE_INTR_STATUS__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_CMD_COMPLETE_INTR_STATUS__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_CMD_COMPLETE_INTR_STATUS_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B0_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_CMD_COMPLETE_INTR_STATUS_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B1_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_CMD_COMPLETE_INTR_STATUS_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B2_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_CMD_COMPLETE_INTR_STATUS_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_HANG_NEED_FLR_INTR_STATUS_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B3_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_CMD_COMPLETE_INTR_STATUS_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_HANG_NEED_FLR_INTR_STATUS_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B4_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_CMD_COMPLETE_INTR_STATUS_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_HANG_NEED_FLR_INTR_STATUS_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B5_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_CMD_COMPLETE_INTR_STATUS_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_HANG_NEED_FLR_INTR_STATUS_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B6_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_CMD_COMPLETE_INTR_STATUS_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_HANG_NEED_FLR_INTR_STATUS_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B0_7_INTR_STATUS__ENGB_B7_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_CMD_COMPLETE_INTR_STATUS__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_CMD_COMPLETE_INTR_STATUS__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_CMD_COMPLETE_INTR_STATUS__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_CMD_COMPLETE_INTR_STATUS__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_CMD_COMPLETE_INTR_STATUS_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B8_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_CMD_COMPLETE_INTR_STATUS_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B9_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_CMD_COMPLETE_INTR_STATUS_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B10_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_CMD_COMPLETE_INTR_STATUS_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_HANG_NEED_FLR_INTR_STATUS_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENGB_B8_15_INTR_STATUS__ENGB_B11_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00008000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf1_bifcfgdecp
+//BIF_CFG_DEV0_EPF1_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_COMMAND
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF1_0_STATUS
+#define BIF_CFG_DEV0_EPF1_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF1_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF1_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF1_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF1_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF1_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF1_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF1_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF1_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF1_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF1_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF1_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF1_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF1_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF1_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF1_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF1_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF1_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF1_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF1_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF1_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF1_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF1_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF1_0_LATENCY
+#define BIF_CFG_DEV0_EPF1_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF1_0_HEADER
+#define BIF_CFG_DEV0_EPF1_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF1_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF1_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF1_0_BIST
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF1_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF1_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF1_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF1_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF1_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF1_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF1_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF1_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF1_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF1_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF1_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF1_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF1_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF1_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF1_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF1_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF1_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF1_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF1_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF1_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF1_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF1_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF1_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF1_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF1_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF1_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF1_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_RCCPORTDEC
+//RCC_DEV0_1_RCC_VDM_SUPPORT
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                       0x0
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                      0x1
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                  0x2
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                        0x3
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                    0x4
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                         0x00000001L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                        0x00000002L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                    0x00000004L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                          0x00000008L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                      0x00000010L
+//RCC_DEV0_1_RCC_BUS_CNTL
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                            0x2
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                           0x3
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                            0x4
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                         0x5
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                        0x6
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                         0x7
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                        0x8
+#define RCC_DEV0_1_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                 0xc
+#define RCC_DEV0_1_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                           0xd
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x10
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x11
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x12
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x13
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x14
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x15
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                 0x18
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                 0x19
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                            0x1c
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                            0x1d
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                              0x00000004L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                             0x00000008L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                              0x00000010L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                           0x00000020L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                          0x00000040L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                           0x00000080L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                          0x00000100L
+#define RCC_DEV0_1_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                   0x00001000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                             0x00002000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00010000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00020000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00040000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00080000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00100000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00200000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                   0x01000000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                   0x0E000000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                              0x10000000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                              0xE0000000L
+//RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS__SHIFT                                   0x7
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                 0x8
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                    0x9
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                    0xa
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                 0xb
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                  0xc
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                      0xd
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                      0xe
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                         0xf
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                 0x10
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                           0x11
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                               0x12
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                     0x13
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS_MASK                                     0x00000080L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                   0x00000100L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                      0x00000200L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                      0x00000400L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                   0x00000800L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                    0x00001000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                        0x00002000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                        0x00004000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                           0x00008000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                   0x00010000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                             0x00020000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                 0x00040000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                       0x00080000L
+//RCC_DEV0_1_RCC_DEV0_LINK_CNTL
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                  0x0
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                 0x8
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                            0x10
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                            0x11
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                    0x00000001L
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                   0x00000100L
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                              0x00010000L
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                              0x00020000L
+//RCC_DEV0_1_RCC_CMN_LINK_CNTL
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                             0x0
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                              0x1
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                             0x2
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                          0x3
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                             0x10
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                               0x00000001L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                0x00000002L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                               0x00000004L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                            0x00000008L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                               0xFFFF0000L
+//RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                            0x0
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                            0x8
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                              0x000000FFL
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                              0x00001F00L
+//RCC_DEV0_1_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV0_1_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_DEV0_1_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                           0x000003FFL
+//RCC_DEV0_1_RCC_MH_ARB_CNTL
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                        0x0
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                0x1
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                          0x00000001L
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                  0x00007FFEL
+//RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                 0x0
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                              0x1
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                0x2
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                 0x3
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                           0x4
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                  0x5
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                 0xb
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                 0x12
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                0x19
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                   0x00000001L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                0x00000002L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                  0x00000004L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                   0x00000008L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                             0x00000010L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                    0x000007E0L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                   0x0003F800L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                   0x01FC0000L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                  0xFE000000L
+//RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                             0x0
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                              0x6
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                         0xc
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                      0x11
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                               0x0000003FL
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                0x00000FC0L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                           0x0001F000L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                        0x00FE0000L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_ep_dev0_RCCPORTDEC
+//RCC_EP_DEV0_1_EP_PCIE_SCRATCH
+#define RCC_EP_DEV0_1_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                    0x0
+#define RCC_EP_DEV0_1_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                      0xFFFFFFFFL
+//RCC_EP_DEV0_1_EP_PCIE_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                  0x7
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                            0x8
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                               0x1e
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                    0x00000080L
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                              0x00000100L
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                 0x40000000L
+//RCC_EP_DEV0_1_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                0x0
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                           0x1
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                               0x2
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                            0x3
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                0x4
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                         0x6
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                  0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                             0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                 0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                              0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                  0x00000010L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                           0x00000040L
+//RCC_EP_DEV0_1_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                          0x0
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                     0x1
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                         0x2
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                      0x3
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                          0x4
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                   0x6
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                0x7
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                            0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                       0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                           0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                        0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                            0x00000010L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                     0x00000040L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                  0x00000080L
+//RCC_EP_DEV0_1_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                   0x0
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                     0x00000001L
+//RCC_EP_DEV0_1_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                              0x7
+#define RCC_EP_DEV0_1_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                0x00000080L
+//RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                       0x0
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                  0x1
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                  0x2
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                  0x3
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                  0x4
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                         0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                    0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                    0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                    0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                    0x00000010L
+//RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                           0x1d
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                             0x20000000L
+//RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                         0x4
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                           0x00000010L
+//RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//RCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0xFFL
+//RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                             0x0
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                         0x8
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                               0x001FL
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                           0x0100L
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV0_1_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                           0x0
+#define RCC_EP_DEV0_1_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                             0x1FL
+//RCC_EP_DEV0_1_EP_PCIEP_RESERVED
+#define RCC_EP_DEV0_1_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                0x0
+#define RCC_EP_DEV0_1_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                  0xFFFFFFFFL
+//RCC_EP_DEV0_1_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                 0xa
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                  0xc
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                   0x18
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                   0x19
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                   0x1a
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                   0x00000C00L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                    0x00003000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                     0x01000000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                     0x02000000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                     0x04000000L
+//RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                0x0
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                  0x3
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                     0x8
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                  0x00000007L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                    0x000000F8L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                       0x0000FF00L
+//RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                              0x0
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                            0x8
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                       0x11
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                               0x12
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                   0x18
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                   0x19
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                   0x1a
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                   0x1b
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                   0x1c
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                   0x1d
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                   0x1e
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                   0x1f
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                              0x00000700L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                         0x00020000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                 0x00040000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                     0x01000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                     0x02000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                     0x04000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                     0x08000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                     0x10000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                     0x20000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                     0x40000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                     0x80000000L
+//RCC_EP_DEV0_1_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                       0x8
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                0x9
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                         0x14
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                       0x15
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                         0x16
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                      0x18
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                          0x19
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                      0x1a
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                         0x00000100L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                  0x00000200L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                           0x00100000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                         0x00200000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                           0x00400000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                        0x01000000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                            0x02000000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                        0x04000000L
+//RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                          0x0
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                          0x1
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                          0x2
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                          0x3
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                            0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                            0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                            0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                            0x00000008L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwn_dev0_RCCPORTDEC
+//RCC_DWN_DEV0_1_DN_PCIE_RESERVED
+#define RCC_DWN_DEV0_1_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                 0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                   0xFFFFFFFFL
+//RCC_DWN_DEV0_1_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV0_1_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV0_1_DN_PCIE_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                    0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                              0x7
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                              0x1e
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                      0x00000001L
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                0x00000080L
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                0x40000000L
+//RCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                0x19
+#define RCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                  0x06000000L
+//RCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                               0x1c
+#define RCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                 0x70000000L
+//RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                             0x7
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                   0x8
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                               0x00000080L
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                     0x00000100L
+//RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                      0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                 0x1
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                 0x2
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                 0x3
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                 0x4
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                        0x00000001L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                   0x00000002L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                   0x00000004L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                   0x00000008L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                   0x00000010L
+//RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                0x11
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                        0x15
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                     0x00000001L
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                  0x00020000L
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                          0x00E00000L
+//RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                             0x18
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                          0x1d
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                               0x01000000L
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                            0x20000000L
+//RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                    0x2
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                      0x00000004L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwnp_dev0_RCCPORTDEC
+//RCC_DWNP_DEV0_1_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                               0x0
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                             0x8
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                    0xb
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                        0x11
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                               0x12
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                           0x13
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                              0x14
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                 0x00000001L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                               0x00000700L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                      0x00000800L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                          0x00020000L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                 0x00040000L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                             0x00080000L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                0x00100000L
+//RCC_DWNP_DEV0_1_PCIE_RX_CNTL
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                        0x8
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                              0x9
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                          0x14
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                     0x15
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                           0x1b
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                          0x00000100L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                0x00000200L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                            0x00100000L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                       0x00200000L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                             0x08000000L
+//RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                           0x0
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                           0x1
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                           0x2
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                           0x3
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                             0x00000001L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                             0x00000002L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                             0x00000004L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                             0x00000008L
+//RCC_DWNP_DEV0_1_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                               0x0
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                     0x1b
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                 0x00000001L
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                       0x08000000L
+//RCC_DWNP_DEV0_1_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV0_1_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                          0xa
+#define RCC_DWNP_DEV0_1_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                            0x00000400L
+//RCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                     0x0
+#define RCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                       0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_pfc_amdgfx_RCCPFCDEC
+//RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE__SHIFT                                           0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE__SHIFT                                           0xa
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT__SHIFT                                             0xf
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE__SHIFT                                        0x10
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE__SHIFT                                        0x1a
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT__SHIFT                                          0x1f
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE_MASK                                             0x000003FFL
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE_MASK                                             0x00001C00L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT_MASK                                               0x00008000L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE_MASK                                          0x03FF0000L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE_MASK                                          0x1C000000L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT_MASK                                            0x80000000L
+//RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN__SHIFT                                         0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS__SHIFT                                     0x8
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN_MASK                                           0x00000001L
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS_MASK                                       0x00000100L
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS__SHIFT                                0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS__SHIFT                            0x1
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS__SHIFT                          0x2
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS__SHIFT                              0x3
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS__SHIFT                                0x4
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS__SHIFT                               0x5
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS__SHIFT                         0x6
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS__SHIFT                  0x7
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS_MASK                                  0x00000001L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS_MASK                              0x00000002L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS_MASK                            0x00000004L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS_MASK                                0x00000008L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS_MASK                                  0x00000010L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS_MASK                                 0x00000020L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS_MASK                           0x00000040L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS_MASK                    0x00000080L
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX__SHIFT                                    0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX_MASK                                      0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE__SHIFT                                       0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE__SHIFT                                0x3
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE_MASK                                         0x00000007L
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE_MASK                                  0x00000008L
+
+
+// addressBlock: aid_nbio_nbif0_pciemsix_0_usb_MSIXTDEC
+//PCIEMSIX_VECT0_ADDR_LO
+#define PCIEMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                                            0x2
+#define PCIEMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                                              0xFFFFFFFCL
+//PCIEMSIX_VECT0_ADDR_HI
+#define PCIEMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                                            0x0
+#define PCIEMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT0_MSG_DATA
+#define PCIEMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                                              0x0
+#define PCIEMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_VECT0_CONTROL
+#define PCIEMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                                               0x0
+#define PCIEMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                                 0x00000001L
+//PCIEMSIX_VECT1_ADDR_LO
+#define PCIEMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                                            0x2
+#define PCIEMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                                              0xFFFFFFFCL
+//PCIEMSIX_VECT1_ADDR_HI
+#define PCIEMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                                            0x0
+#define PCIEMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT1_MSG_DATA
+#define PCIEMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                                              0x0
+#define PCIEMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_VECT1_CONTROL
+#define PCIEMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                                               0x0
+#define PCIEMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                                 0x00000001L
+//PCIEMSIX_VECT2_ADDR_LO
+#define PCIEMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                                            0x2
+#define PCIEMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                                              0xFFFFFFFCL
+//PCIEMSIX_VECT2_ADDR_HI
+#define PCIEMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                                            0x0
+#define PCIEMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT2_MSG_DATA
+#define PCIEMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                                              0x0
+#define PCIEMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_VECT2_CONTROL
+#define PCIEMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                                               0x0
+#define PCIEMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                                 0x00000001L
+//PCIEMSIX_VECT3_ADDR_LO
+#define PCIEMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                                            0x2
+#define PCIEMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                                              0xFFFFFFFCL
+//PCIEMSIX_VECT3_ADDR_HI
+#define PCIEMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                                            0x0
+#define PCIEMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT3_MSG_DATA
+#define PCIEMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                                              0x0
+#define PCIEMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_VECT3_CONTROL
+#define PCIEMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                                               0x0
+#define PCIEMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                                 0x00000001L
+//PCIEMSIX_VECT4_ADDR_LO
+#define PCIEMSIX_VECT4_ADDR_LO__MSG_ADDR_LO__SHIFT                                                            0x2
+#define PCIEMSIX_VECT4_ADDR_LO__MSG_ADDR_LO_MASK                                                              0xFFFFFFFCL
+//PCIEMSIX_VECT4_ADDR_HI
+#define PCIEMSIX_VECT4_ADDR_HI__MSG_ADDR_HI__SHIFT                                                            0x0
+#define PCIEMSIX_VECT4_ADDR_HI__MSG_ADDR_HI_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT4_MSG_DATA
+#define PCIEMSIX_VECT4_MSG_DATA__MSG_DATA__SHIFT                                                              0x0
+#define PCIEMSIX_VECT4_MSG_DATA__MSG_DATA_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_VECT4_CONTROL
+#define PCIEMSIX_VECT4_CONTROL__MASK_BIT__SHIFT                                                               0x0
+#define PCIEMSIX_VECT4_CONTROL__MASK_BIT_MASK                                                                 0x00000001L
+//PCIEMSIX_VECT5_ADDR_LO
+#define PCIEMSIX_VECT5_ADDR_LO__MSG_ADDR_LO__SHIFT                                                            0x2
+#define PCIEMSIX_VECT5_ADDR_LO__MSG_ADDR_LO_MASK                                                              0xFFFFFFFCL
+//PCIEMSIX_VECT5_ADDR_HI
+#define PCIEMSIX_VECT5_ADDR_HI__MSG_ADDR_HI__SHIFT                                                            0x0
+#define PCIEMSIX_VECT5_ADDR_HI__MSG_ADDR_HI_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT5_MSG_DATA
+#define PCIEMSIX_VECT5_MSG_DATA__MSG_DATA__SHIFT                                                              0x0
+#define PCIEMSIX_VECT5_MSG_DATA__MSG_DATA_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_VECT5_CONTROL
+#define PCIEMSIX_VECT5_CONTROL__MASK_BIT__SHIFT                                                               0x0
+#define PCIEMSIX_VECT5_CONTROL__MASK_BIT_MASK                                                                 0x00000001L
+//PCIEMSIX_VECT6_ADDR_LO
+#define PCIEMSIX_VECT6_ADDR_LO__MSG_ADDR_LO__SHIFT                                                            0x2
+#define PCIEMSIX_VECT6_ADDR_LO__MSG_ADDR_LO_MASK                                                              0xFFFFFFFCL
+//PCIEMSIX_VECT6_ADDR_HI
+#define PCIEMSIX_VECT6_ADDR_HI__MSG_ADDR_HI__SHIFT                                                            0x0
+#define PCIEMSIX_VECT6_ADDR_HI__MSG_ADDR_HI_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT6_MSG_DATA
+#define PCIEMSIX_VECT6_MSG_DATA__MSG_DATA__SHIFT                                                              0x0
+#define PCIEMSIX_VECT6_MSG_DATA__MSG_DATA_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_VECT6_CONTROL
+#define PCIEMSIX_VECT6_CONTROL__MASK_BIT__SHIFT                                                               0x0
+#define PCIEMSIX_VECT6_CONTROL__MASK_BIT_MASK                                                                 0x00000001L
+//PCIEMSIX_VECT7_ADDR_LO
+#define PCIEMSIX_VECT7_ADDR_LO__MSG_ADDR_LO__SHIFT                                                            0x2
+#define PCIEMSIX_VECT7_ADDR_LO__MSG_ADDR_LO_MASK                                                              0xFFFFFFFCL
+//PCIEMSIX_VECT7_ADDR_HI
+#define PCIEMSIX_VECT7_ADDR_HI__MSG_ADDR_HI__SHIFT                                                            0x0
+#define PCIEMSIX_VECT7_ADDR_HI__MSG_ADDR_HI_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT7_MSG_DATA
+#define PCIEMSIX_VECT7_MSG_DATA__MSG_DATA__SHIFT                                                              0x0
+#define PCIEMSIX_VECT7_MSG_DATA__MSG_DATA_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_VECT7_CONTROL
+#define PCIEMSIX_VECT7_CONTROL__MASK_BIT__SHIFT                                                               0x0
+#define PCIEMSIX_VECT7_CONTROL__MASK_BIT_MASK                                                                 0x00000001L
+//PCIEMSIX_VECT8_ADDR_LO
+#define PCIEMSIX_VECT8_ADDR_LO__MSG_ADDR_LO__SHIFT                                                            0x2
+#define PCIEMSIX_VECT8_ADDR_LO__MSG_ADDR_LO_MASK                                                              0xFFFFFFFCL
+//PCIEMSIX_VECT8_ADDR_HI
+#define PCIEMSIX_VECT8_ADDR_HI__MSG_ADDR_HI__SHIFT                                                            0x0
+#define PCIEMSIX_VECT8_ADDR_HI__MSG_ADDR_HI_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT8_MSG_DATA
+#define PCIEMSIX_VECT8_MSG_DATA__MSG_DATA__SHIFT                                                              0x0
+#define PCIEMSIX_VECT8_MSG_DATA__MSG_DATA_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_VECT8_CONTROL
+#define PCIEMSIX_VECT8_CONTROL__MASK_BIT__SHIFT                                                               0x0
+#define PCIEMSIX_VECT8_CONTROL__MASK_BIT_MASK                                                                 0x00000001L
+//PCIEMSIX_VECT9_ADDR_LO
+#define PCIEMSIX_VECT9_ADDR_LO__MSG_ADDR_LO__SHIFT                                                            0x2
+#define PCIEMSIX_VECT9_ADDR_LO__MSG_ADDR_LO_MASK                                                              0xFFFFFFFCL
+//PCIEMSIX_VECT9_ADDR_HI
+#define PCIEMSIX_VECT9_ADDR_HI__MSG_ADDR_HI__SHIFT                                                            0x0
+#define PCIEMSIX_VECT9_ADDR_HI__MSG_ADDR_HI_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT9_MSG_DATA
+#define PCIEMSIX_VECT9_MSG_DATA__MSG_DATA__SHIFT                                                              0x0
+#define PCIEMSIX_VECT9_MSG_DATA__MSG_DATA_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_VECT9_CONTROL
+#define PCIEMSIX_VECT9_CONTROL__MASK_BIT__SHIFT                                                               0x0
+#define PCIEMSIX_VECT9_CONTROL__MASK_BIT_MASK                                                                 0x00000001L
+//PCIEMSIX_VECT10_ADDR_LO
+#define PCIEMSIX_VECT10_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT10_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT10_ADDR_HI
+#define PCIEMSIX_VECT10_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT10_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT10_MSG_DATA
+#define PCIEMSIX_VECT10_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT10_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT10_CONTROL
+#define PCIEMSIX_VECT10_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT10_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT11_ADDR_LO
+#define PCIEMSIX_VECT11_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT11_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT11_ADDR_HI
+#define PCIEMSIX_VECT11_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT11_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT11_MSG_DATA
+#define PCIEMSIX_VECT11_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT11_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT11_CONTROL
+#define PCIEMSIX_VECT11_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT11_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT12_ADDR_LO
+#define PCIEMSIX_VECT12_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT12_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT12_ADDR_HI
+#define PCIEMSIX_VECT12_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT12_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT12_MSG_DATA
+#define PCIEMSIX_VECT12_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT12_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT12_CONTROL
+#define PCIEMSIX_VECT12_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT12_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT13_ADDR_LO
+#define PCIEMSIX_VECT13_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT13_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT13_ADDR_HI
+#define PCIEMSIX_VECT13_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT13_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT13_MSG_DATA
+#define PCIEMSIX_VECT13_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT13_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT13_CONTROL
+#define PCIEMSIX_VECT13_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT13_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT14_ADDR_LO
+#define PCIEMSIX_VECT14_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT14_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT14_ADDR_HI
+#define PCIEMSIX_VECT14_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT14_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT14_MSG_DATA
+#define PCIEMSIX_VECT14_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT14_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT14_CONTROL
+#define PCIEMSIX_VECT14_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT14_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT15_ADDR_LO
+#define PCIEMSIX_VECT15_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT15_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT15_ADDR_HI
+#define PCIEMSIX_VECT15_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT15_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT15_MSG_DATA
+#define PCIEMSIX_VECT15_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT15_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT15_CONTROL
+#define PCIEMSIX_VECT15_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT15_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT16_ADDR_LO
+#define PCIEMSIX_VECT16_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT16_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT16_ADDR_HI
+#define PCIEMSIX_VECT16_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT16_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT16_MSG_DATA
+#define PCIEMSIX_VECT16_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT16_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT16_CONTROL
+#define PCIEMSIX_VECT16_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT16_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT17_ADDR_LO
+#define PCIEMSIX_VECT17_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT17_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT17_ADDR_HI
+#define PCIEMSIX_VECT17_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT17_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT17_MSG_DATA
+#define PCIEMSIX_VECT17_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT17_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT17_CONTROL
+#define PCIEMSIX_VECT17_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT17_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT18_ADDR_LO
+#define PCIEMSIX_VECT18_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT18_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT18_ADDR_HI
+#define PCIEMSIX_VECT18_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT18_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT18_MSG_DATA
+#define PCIEMSIX_VECT18_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT18_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT18_CONTROL
+#define PCIEMSIX_VECT18_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT18_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT19_ADDR_LO
+#define PCIEMSIX_VECT19_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT19_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT19_ADDR_HI
+#define PCIEMSIX_VECT19_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT19_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT19_MSG_DATA
+#define PCIEMSIX_VECT19_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT19_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT19_CONTROL
+#define PCIEMSIX_VECT19_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT19_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT20_ADDR_LO
+#define PCIEMSIX_VECT20_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT20_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT20_ADDR_HI
+#define PCIEMSIX_VECT20_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT20_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT20_MSG_DATA
+#define PCIEMSIX_VECT20_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT20_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT20_CONTROL
+#define PCIEMSIX_VECT20_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT20_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT21_ADDR_LO
+#define PCIEMSIX_VECT21_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT21_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT21_ADDR_HI
+#define PCIEMSIX_VECT21_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT21_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT21_MSG_DATA
+#define PCIEMSIX_VECT21_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT21_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT21_CONTROL
+#define PCIEMSIX_VECT21_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT21_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT22_ADDR_LO
+#define PCIEMSIX_VECT22_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT22_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT22_ADDR_HI
+#define PCIEMSIX_VECT22_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT22_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT22_MSG_DATA
+#define PCIEMSIX_VECT22_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT22_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT22_CONTROL
+#define PCIEMSIX_VECT22_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT22_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT23_ADDR_LO
+#define PCIEMSIX_VECT23_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT23_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT23_ADDR_HI
+#define PCIEMSIX_VECT23_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT23_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT23_MSG_DATA
+#define PCIEMSIX_VECT23_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT23_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT23_CONTROL
+#define PCIEMSIX_VECT23_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT23_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT24_ADDR_LO
+#define PCIEMSIX_VECT24_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT24_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT24_ADDR_HI
+#define PCIEMSIX_VECT24_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT24_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT24_MSG_DATA
+#define PCIEMSIX_VECT24_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT24_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT24_CONTROL
+#define PCIEMSIX_VECT24_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT24_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT25_ADDR_LO
+#define PCIEMSIX_VECT25_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT25_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT25_ADDR_HI
+#define PCIEMSIX_VECT25_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT25_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT25_MSG_DATA
+#define PCIEMSIX_VECT25_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT25_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT25_CONTROL
+#define PCIEMSIX_VECT25_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT25_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT26_ADDR_LO
+#define PCIEMSIX_VECT26_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT26_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT26_ADDR_HI
+#define PCIEMSIX_VECT26_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT26_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT26_MSG_DATA
+#define PCIEMSIX_VECT26_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT26_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT26_CONTROL
+#define PCIEMSIX_VECT26_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT26_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT27_ADDR_LO
+#define PCIEMSIX_VECT27_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT27_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT27_ADDR_HI
+#define PCIEMSIX_VECT27_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT27_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT27_MSG_DATA
+#define PCIEMSIX_VECT27_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT27_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT27_CONTROL
+#define PCIEMSIX_VECT27_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT27_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT28_ADDR_LO
+#define PCIEMSIX_VECT28_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT28_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT28_ADDR_HI
+#define PCIEMSIX_VECT28_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT28_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT28_MSG_DATA
+#define PCIEMSIX_VECT28_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT28_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT28_CONTROL
+#define PCIEMSIX_VECT28_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT28_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT29_ADDR_LO
+#define PCIEMSIX_VECT29_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT29_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT29_ADDR_HI
+#define PCIEMSIX_VECT29_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT29_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT29_MSG_DATA
+#define PCIEMSIX_VECT29_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT29_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT29_CONTROL
+#define PCIEMSIX_VECT29_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT29_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT30_ADDR_LO
+#define PCIEMSIX_VECT30_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT30_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT30_ADDR_HI
+#define PCIEMSIX_VECT30_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT30_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT30_MSG_DATA
+#define PCIEMSIX_VECT30_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT30_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT30_CONTROL
+#define PCIEMSIX_VECT30_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT30_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT31_ADDR_LO
+#define PCIEMSIX_VECT31_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT31_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT31_ADDR_HI
+#define PCIEMSIX_VECT31_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT31_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT31_MSG_DATA
+#define PCIEMSIX_VECT31_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT31_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT31_CONTROL
+#define PCIEMSIX_VECT31_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT31_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT32_ADDR_LO
+#define PCIEMSIX_VECT32_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT32_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT32_ADDR_HI
+#define PCIEMSIX_VECT32_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT32_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT32_MSG_DATA
+#define PCIEMSIX_VECT32_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT32_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT32_CONTROL
+#define PCIEMSIX_VECT32_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT32_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT33_ADDR_LO
+#define PCIEMSIX_VECT33_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT33_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT33_ADDR_HI
+#define PCIEMSIX_VECT33_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT33_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT33_MSG_DATA
+#define PCIEMSIX_VECT33_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT33_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT33_CONTROL
+#define PCIEMSIX_VECT33_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT33_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT34_ADDR_LO
+#define PCIEMSIX_VECT34_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT34_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT34_ADDR_HI
+#define PCIEMSIX_VECT34_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT34_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT34_MSG_DATA
+#define PCIEMSIX_VECT34_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT34_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT34_CONTROL
+#define PCIEMSIX_VECT34_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT34_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT35_ADDR_LO
+#define PCIEMSIX_VECT35_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT35_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT35_ADDR_HI
+#define PCIEMSIX_VECT35_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT35_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT35_MSG_DATA
+#define PCIEMSIX_VECT35_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT35_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT35_CONTROL
+#define PCIEMSIX_VECT35_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT35_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT36_ADDR_LO
+#define PCIEMSIX_VECT36_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT36_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT36_ADDR_HI
+#define PCIEMSIX_VECT36_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT36_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT36_MSG_DATA
+#define PCIEMSIX_VECT36_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT36_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT36_CONTROL
+#define PCIEMSIX_VECT36_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT36_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT37_ADDR_LO
+#define PCIEMSIX_VECT37_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT37_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT37_ADDR_HI
+#define PCIEMSIX_VECT37_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT37_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT37_MSG_DATA
+#define PCIEMSIX_VECT37_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT37_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT37_CONTROL
+#define PCIEMSIX_VECT37_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT37_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT38_ADDR_LO
+#define PCIEMSIX_VECT38_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT38_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT38_ADDR_HI
+#define PCIEMSIX_VECT38_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT38_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT38_MSG_DATA
+#define PCIEMSIX_VECT38_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT38_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT38_CONTROL
+#define PCIEMSIX_VECT38_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT38_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT39_ADDR_LO
+#define PCIEMSIX_VECT39_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT39_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT39_ADDR_HI
+#define PCIEMSIX_VECT39_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT39_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT39_MSG_DATA
+#define PCIEMSIX_VECT39_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT39_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT39_CONTROL
+#define PCIEMSIX_VECT39_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT39_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT40_ADDR_LO
+#define PCIEMSIX_VECT40_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT40_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT40_ADDR_HI
+#define PCIEMSIX_VECT40_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT40_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT40_MSG_DATA
+#define PCIEMSIX_VECT40_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT40_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT40_CONTROL
+#define PCIEMSIX_VECT40_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT40_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT41_ADDR_LO
+#define PCIEMSIX_VECT41_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT41_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT41_ADDR_HI
+#define PCIEMSIX_VECT41_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT41_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT41_MSG_DATA
+#define PCIEMSIX_VECT41_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT41_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT41_CONTROL
+#define PCIEMSIX_VECT41_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT41_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT42_ADDR_LO
+#define PCIEMSIX_VECT42_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT42_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT42_ADDR_HI
+#define PCIEMSIX_VECT42_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT42_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT42_MSG_DATA
+#define PCIEMSIX_VECT42_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT42_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT42_CONTROL
+#define PCIEMSIX_VECT42_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT42_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT43_ADDR_LO
+#define PCIEMSIX_VECT43_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT43_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT43_ADDR_HI
+#define PCIEMSIX_VECT43_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT43_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT43_MSG_DATA
+#define PCIEMSIX_VECT43_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT43_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT43_CONTROL
+#define PCIEMSIX_VECT43_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT43_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT44_ADDR_LO
+#define PCIEMSIX_VECT44_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT44_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT44_ADDR_HI
+#define PCIEMSIX_VECT44_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT44_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT44_MSG_DATA
+#define PCIEMSIX_VECT44_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT44_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT44_CONTROL
+#define PCIEMSIX_VECT44_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT44_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT45_ADDR_LO
+#define PCIEMSIX_VECT45_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT45_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT45_ADDR_HI
+#define PCIEMSIX_VECT45_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT45_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT45_MSG_DATA
+#define PCIEMSIX_VECT45_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT45_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT45_CONTROL
+#define PCIEMSIX_VECT45_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT45_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT46_ADDR_LO
+#define PCIEMSIX_VECT46_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT46_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT46_ADDR_HI
+#define PCIEMSIX_VECT46_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT46_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT46_MSG_DATA
+#define PCIEMSIX_VECT46_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT46_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT46_CONTROL
+#define PCIEMSIX_VECT46_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT46_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT47_ADDR_LO
+#define PCIEMSIX_VECT47_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT47_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT47_ADDR_HI
+#define PCIEMSIX_VECT47_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT47_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT47_MSG_DATA
+#define PCIEMSIX_VECT47_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT47_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT47_CONTROL
+#define PCIEMSIX_VECT47_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT47_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT48_ADDR_LO
+#define PCIEMSIX_VECT48_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT48_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT48_ADDR_HI
+#define PCIEMSIX_VECT48_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT48_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT48_MSG_DATA
+#define PCIEMSIX_VECT48_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT48_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT48_CONTROL
+#define PCIEMSIX_VECT48_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT48_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT49_ADDR_LO
+#define PCIEMSIX_VECT49_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT49_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT49_ADDR_HI
+#define PCIEMSIX_VECT49_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT49_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT49_MSG_DATA
+#define PCIEMSIX_VECT49_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT49_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT49_CONTROL
+#define PCIEMSIX_VECT49_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT49_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT50_ADDR_LO
+#define PCIEMSIX_VECT50_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT50_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT50_ADDR_HI
+#define PCIEMSIX_VECT50_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT50_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT50_MSG_DATA
+#define PCIEMSIX_VECT50_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT50_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT50_CONTROL
+#define PCIEMSIX_VECT50_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT50_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT51_ADDR_LO
+#define PCIEMSIX_VECT51_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT51_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT51_ADDR_HI
+#define PCIEMSIX_VECT51_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT51_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT51_MSG_DATA
+#define PCIEMSIX_VECT51_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT51_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT51_CONTROL
+#define PCIEMSIX_VECT51_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT51_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT52_ADDR_LO
+#define PCIEMSIX_VECT52_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT52_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT52_ADDR_HI
+#define PCIEMSIX_VECT52_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT52_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT52_MSG_DATA
+#define PCIEMSIX_VECT52_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT52_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT52_CONTROL
+#define PCIEMSIX_VECT52_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT52_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT53_ADDR_LO
+#define PCIEMSIX_VECT53_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT53_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT53_ADDR_HI
+#define PCIEMSIX_VECT53_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT53_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT53_MSG_DATA
+#define PCIEMSIX_VECT53_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT53_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT53_CONTROL
+#define PCIEMSIX_VECT53_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT53_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT54_ADDR_LO
+#define PCIEMSIX_VECT54_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT54_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT54_ADDR_HI
+#define PCIEMSIX_VECT54_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT54_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT54_MSG_DATA
+#define PCIEMSIX_VECT54_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT54_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT54_CONTROL
+#define PCIEMSIX_VECT54_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT54_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT55_ADDR_LO
+#define PCIEMSIX_VECT55_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT55_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT55_ADDR_HI
+#define PCIEMSIX_VECT55_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT55_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT55_MSG_DATA
+#define PCIEMSIX_VECT55_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT55_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT55_CONTROL
+#define PCIEMSIX_VECT55_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT55_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT56_ADDR_LO
+#define PCIEMSIX_VECT56_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT56_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT56_ADDR_HI
+#define PCIEMSIX_VECT56_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT56_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT56_MSG_DATA
+#define PCIEMSIX_VECT56_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT56_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT56_CONTROL
+#define PCIEMSIX_VECT56_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT56_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT57_ADDR_LO
+#define PCIEMSIX_VECT57_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT57_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT57_ADDR_HI
+#define PCIEMSIX_VECT57_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT57_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT57_MSG_DATA
+#define PCIEMSIX_VECT57_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT57_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT57_CONTROL
+#define PCIEMSIX_VECT57_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT57_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT58_ADDR_LO
+#define PCIEMSIX_VECT58_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT58_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT58_ADDR_HI
+#define PCIEMSIX_VECT58_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT58_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT58_MSG_DATA
+#define PCIEMSIX_VECT58_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT58_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT58_CONTROL
+#define PCIEMSIX_VECT58_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT58_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT59_ADDR_LO
+#define PCIEMSIX_VECT59_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT59_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT59_ADDR_HI
+#define PCIEMSIX_VECT59_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT59_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT59_MSG_DATA
+#define PCIEMSIX_VECT59_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT59_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT59_CONTROL
+#define PCIEMSIX_VECT59_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT59_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT60_ADDR_LO
+#define PCIEMSIX_VECT60_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT60_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT60_ADDR_HI
+#define PCIEMSIX_VECT60_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT60_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT60_MSG_DATA
+#define PCIEMSIX_VECT60_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT60_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT60_CONTROL
+#define PCIEMSIX_VECT60_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT60_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT61_ADDR_LO
+#define PCIEMSIX_VECT61_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT61_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT61_ADDR_HI
+#define PCIEMSIX_VECT61_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT61_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT61_MSG_DATA
+#define PCIEMSIX_VECT61_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT61_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT61_CONTROL
+#define PCIEMSIX_VECT61_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT61_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT62_ADDR_LO
+#define PCIEMSIX_VECT62_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT62_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT62_ADDR_HI
+#define PCIEMSIX_VECT62_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT62_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT62_MSG_DATA
+#define PCIEMSIX_VECT62_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT62_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT62_CONTROL
+#define PCIEMSIX_VECT62_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT62_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT63_ADDR_LO
+#define PCIEMSIX_VECT63_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT63_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT63_ADDR_HI
+#define PCIEMSIX_VECT63_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT63_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT63_MSG_DATA
+#define PCIEMSIX_VECT63_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT63_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT63_CONTROL
+#define PCIEMSIX_VECT63_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT63_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT64_ADDR_LO
+#define PCIEMSIX_VECT64_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT64_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT64_ADDR_HI
+#define PCIEMSIX_VECT64_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT64_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT64_MSG_DATA
+#define PCIEMSIX_VECT64_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT64_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT64_CONTROL
+#define PCIEMSIX_VECT64_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT64_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT65_ADDR_LO
+#define PCIEMSIX_VECT65_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT65_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT65_ADDR_HI
+#define PCIEMSIX_VECT65_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT65_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT65_MSG_DATA
+#define PCIEMSIX_VECT65_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT65_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT65_CONTROL
+#define PCIEMSIX_VECT65_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT65_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT66_ADDR_LO
+#define PCIEMSIX_VECT66_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT66_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT66_ADDR_HI
+#define PCIEMSIX_VECT66_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT66_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT66_MSG_DATA
+#define PCIEMSIX_VECT66_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT66_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT66_CONTROL
+#define PCIEMSIX_VECT66_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT66_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT67_ADDR_LO
+#define PCIEMSIX_VECT67_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT67_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT67_ADDR_HI
+#define PCIEMSIX_VECT67_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT67_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT67_MSG_DATA
+#define PCIEMSIX_VECT67_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT67_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT67_CONTROL
+#define PCIEMSIX_VECT67_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT67_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT68_ADDR_LO
+#define PCIEMSIX_VECT68_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT68_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT68_ADDR_HI
+#define PCIEMSIX_VECT68_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT68_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT68_MSG_DATA
+#define PCIEMSIX_VECT68_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT68_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT68_CONTROL
+#define PCIEMSIX_VECT68_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT68_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT69_ADDR_LO
+#define PCIEMSIX_VECT69_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT69_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT69_ADDR_HI
+#define PCIEMSIX_VECT69_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT69_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT69_MSG_DATA
+#define PCIEMSIX_VECT69_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT69_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT69_CONTROL
+#define PCIEMSIX_VECT69_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT69_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT70_ADDR_LO
+#define PCIEMSIX_VECT70_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT70_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT70_ADDR_HI
+#define PCIEMSIX_VECT70_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT70_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT70_MSG_DATA
+#define PCIEMSIX_VECT70_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT70_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT70_CONTROL
+#define PCIEMSIX_VECT70_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT70_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT71_ADDR_LO
+#define PCIEMSIX_VECT71_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT71_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT71_ADDR_HI
+#define PCIEMSIX_VECT71_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT71_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT71_MSG_DATA
+#define PCIEMSIX_VECT71_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT71_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT71_CONTROL
+#define PCIEMSIX_VECT71_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT71_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT72_ADDR_LO
+#define PCIEMSIX_VECT72_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT72_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT72_ADDR_HI
+#define PCIEMSIX_VECT72_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT72_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT72_MSG_DATA
+#define PCIEMSIX_VECT72_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT72_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT72_CONTROL
+#define PCIEMSIX_VECT72_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT72_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT73_ADDR_LO
+#define PCIEMSIX_VECT73_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT73_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT73_ADDR_HI
+#define PCIEMSIX_VECT73_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT73_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT73_MSG_DATA
+#define PCIEMSIX_VECT73_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT73_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT73_CONTROL
+#define PCIEMSIX_VECT73_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT73_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT74_ADDR_LO
+#define PCIEMSIX_VECT74_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT74_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT74_ADDR_HI
+#define PCIEMSIX_VECT74_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT74_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT74_MSG_DATA
+#define PCIEMSIX_VECT74_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT74_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT74_CONTROL
+#define PCIEMSIX_VECT74_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT74_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT75_ADDR_LO
+#define PCIEMSIX_VECT75_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT75_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT75_ADDR_HI
+#define PCIEMSIX_VECT75_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT75_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT75_MSG_DATA
+#define PCIEMSIX_VECT75_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT75_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT75_CONTROL
+#define PCIEMSIX_VECT75_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT75_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT76_ADDR_LO
+#define PCIEMSIX_VECT76_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT76_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT76_ADDR_HI
+#define PCIEMSIX_VECT76_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT76_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT76_MSG_DATA
+#define PCIEMSIX_VECT76_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT76_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT76_CONTROL
+#define PCIEMSIX_VECT76_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT76_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT77_ADDR_LO
+#define PCIEMSIX_VECT77_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT77_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT77_ADDR_HI
+#define PCIEMSIX_VECT77_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT77_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT77_MSG_DATA
+#define PCIEMSIX_VECT77_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT77_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT77_CONTROL
+#define PCIEMSIX_VECT77_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT77_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT78_ADDR_LO
+#define PCIEMSIX_VECT78_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT78_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT78_ADDR_HI
+#define PCIEMSIX_VECT78_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT78_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT78_MSG_DATA
+#define PCIEMSIX_VECT78_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT78_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT78_CONTROL
+#define PCIEMSIX_VECT78_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT78_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT79_ADDR_LO
+#define PCIEMSIX_VECT79_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT79_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT79_ADDR_HI
+#define PCIEMSIX_VECT79_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT79_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT79_MSG_DATA
+#define PCIEMSIX_VECT79_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT79_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT79_CONTROL
+#define PCIEMSIX_VECT79_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT79_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT80_ADDR_LO
+#define PCIEMSIX_VECT80_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT80_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT80_ADDR_HI
+#define PCIEMSIX_VECT80_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT80_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT80_MSG_DATA
+#define PCIEMSIX_VECT80_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT80_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT80_CONTROL
+#define PCIEMSIX_VECT80_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT80_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT81_ADDR_LO
+#define PCIEMSIX_VECT81_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT81_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT81_ADDR_HI
+#define PCIEMSIX_VECT81_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT81_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT81_MSG_DATA
+#define PCIEMSIX_VECT81_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT81_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT81_CONTROL
+#define PCIEMSIX_VECT81_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT81_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT82_ADDR_LO
+#define PCIEMSIX_VECT82_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT82_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT82_ADDR_HI
+#define PCIEMSIX_VECT82_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT82_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT82_MSG_DATA
+#define PCIEMSIX_VECT82_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT82_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT82_CONTROL
+#define PCIEMSIX_VECT82_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT82_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT83_ADDR_LO
+#define PCIEMSIX_VECT83_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT83_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT83_ADDR_HI
+#define PCIEMSIX_VECT83_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT83_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT83_MSG_DATA
+#define PCIEMSIX_VECT83_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT83_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT83_CONTROL
+#define PCIEMSIX_VECT83_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT83_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT84_ADDR_LO
+#define PCIEMSIX_VECT84_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT84_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT84_ADDR_HI
+#define PCIEMSIX_VECT84_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT84_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT84_MSG_DATA
+#define PCIEMSIX_VECT84_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT84_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT84_CONTROL
+#define PCIEMSIX_VECT84_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT84_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT85_ADDR_LO
+#define PCIEMSIX_VECT85_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT85_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT85_ADDR_HI
+#define PCIEMSIX_VECT85_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT85_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT85_MSG_DATA
+#define PCIEMSIX_VECT85_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT85_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT85_CONTROL
+#define PCIEMSIX_VECT85_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT85_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT86_ADDR_LO
+#define PCIEMSIX_VECT86_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT86_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT86_ADDR_HI
+#define PCIEMSIX_VECT86_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT86_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT86_MSG_DATA
+#define PCIEMSIX_VECT86_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT86_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT86_CONTROL
+#define PCIEMSIX_VECT86_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT86_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT87_ADDR_LO
+#define PCIEMSIX_VECT87_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT87_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT87_ADDR_HI
+#define PCIEMSIX_VECT87_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT87_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT87_MSG_DATA
+#define PCIEMSIX_VECT87_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT87_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT87_CONTROL
+#define PCIEMSIX_VECT87_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT87_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT88_ADDR_LO
+#define PCIEMSIX_VECT88_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT88_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT88_ADDR_HI
+#define PCIEMSIX_VECT88_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT88_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT88_MSG_DATA
+#define PCIEMSIX_VECT88_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT88_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT88_CONTROL
+#define PCIEMSIX_VECT88_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT88_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT89_ADDR_LO
+#define PCIEMSIX_VECT89_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT89_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT89_ADDR_HI
+#define PCIEMSIX_VECT89_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT89_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT89_MSG_DATA
+#define PCIEMSIX_VECT89_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT89_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT89_CONTROL
+#define PCIEMSIX_VECT89_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT89_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT90_ADDR_LO
+#define PCIEMSIX_VECT90_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT90_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT90_ADDR_HI
+#define PCIEMSIX_VECT90_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT90_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT90_MSG_DATA
+#define PCIEMSIX_VECT90_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT90_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT90_CONTROL
+#define PCIEMSIX_VECT90_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT90_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT91_ADDR_LO
+#define PCIEMSIX_VECT91_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT91_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT91_ADDR_HI
+#define PCIEMSIX_VECT91_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT91_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT91_MSG_DATA
+#define PCIEMSIX_VECT91_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT91_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT91_CONTROL
+#define PCIEMSIX_VECT91_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT91_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT92_ADDR_LO
+#define PCIEMSIX_VECT92_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT92_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT92_ADDR_HI
+#define PCIEMSIX_VECT92_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT92_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT92_MSG_DATA
+#define PCIEMSIX_VECT92_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT92_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT92_CONTROL
+#define PCIEMSIX_VECT92_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT92_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT93_ADDR_LO
+#define PCIEMSIX_VECT93_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT93_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT93_ADDR_HI
+#define PCIEMSIX_VECT93_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT93_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT93_MSG_DATA
+#define PCIEMSIX_VECT93_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT93_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT93_CONTROL
+#define PCIEMSIX_VECT93_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT93_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT94_ADDR_LO
+#define PCIEMSIX_VECT94_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT94_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT94_ADDR_HI
+#define PCIEMSIX_VECT94_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT94_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT94_MSG_DATA
+#define PCIEMSIX_VECT94_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT94_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT94_CONTROL
+#define PCIEMSIX_VECT94_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT94_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT95_ADDR_LO
+#define PCIEMSIX_VECT95_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT95_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT95_ADDR_HI
+#define PCIEMSIX_VECT95_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT95_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT95_MSG_DATA
+#define PCIEMSIX_VECT95_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT95_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT95_CONTROL
+#define PCIEMSIX_VECT95_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT95_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT96_ADDR_LO
+#define PCIEMSIX_VECT96_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT96_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT96_ADDR_HI
+#define PCIEMSIX_VECT96_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT96_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT96_MSG_DATA
+#define PCIEMSIX_VECT96_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT96_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT96_CONTROL
+#define PCIEMSIX_VECT96_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT96_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT97_ADDR_LO
+#define PCIEMSIX_VECT97_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT97_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT97_ADDR_HI
+#define PCIEMSIX_VECT97_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT97_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT97_MSG_DATA
+#define PCIEMSIX_VECT97_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT97_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT97_CONTROL
+#define PCIEMSIX_VECT97_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT97_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT98_ADDR_LO
+#define PCIEMSIX_VECT98_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT98_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT98_ADDR_HI
+#define PCIEMSIX_VECT98_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT98_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT98_MSG_DATA
+#define PCIEMSIX_VECT98_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT98_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT98_CONTROL
+#define PCIEMSIX_VECT98_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT98_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT99_ADDR_LO
+#define PCIEMSIX_VECT99_ADDR_LO__MSG_ADDR_LO__SHIFT                                                           0x2
+#define PCIEMSIX_VECT99_ADDR_LO__MSG_ADDR_LO_MASK                                                             0xFFFFFFFCL
+//PCIEMSIX_VECT99_ADDR_HI
+#define PCIEMSIX_VECT99_ADDR_HI__MSG_ADDR_HI__SHIFT                                                           0x0
+#define PCIEMSIX_VECT99_ADDR_HI__MSG_ADDR_HI_MASK                                                             0xFFFFFFFFL
+//PCIEMSIX_VECT99_MSG_DATA
+#define PCIEMSIX_VECT99_MSG_DATA__MSG_DATA__SHIFT                                                             0x0
+#define PCIEMSIX_VECT99_MSG_DATA__MSG_DATA_MASK                                                               0xFFFFFFFFL
+//PCIEMSIX_VECT99_CONTROL
+#define PCIEMSIX_VECT99_CONTROL__MASK_BIT__SHIFT                                                              0x0
+#define PCIEMSIX_VECT99_CONTROL__MASK_BIT_MASK                                                                0x00000001L
+//PCIEMSIX_VECT100_ADDR_LO
+#define PCIEMSIX_VECT100_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT100_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT100_ADDR_HI
+#define PCIEMSIX_VECT100_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT100_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT100_MSG_DATA
+#define PCIEMSIX_VECT100_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT100_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT100_CONTROL
+#define PCIEMSIX_VECT100_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT100_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT101_ADDR_LO
+#define PCIEMSIX_VECT101_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT101_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT101_ADDR_HI
+#define PCIEMSIX_VECT101_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT101_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT101_MSG_DATA
+#define PCIEMSIX_VECT101_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT101_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT101_CONTROL
+#define PCIEMSIX_VECT101_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT101_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT102_ADDR_LO
+#define PCIEMSIX_VECT102_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT102_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT102_ADDR_HI
+#define PCIEMSIX_VECT102_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT102_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT102_MSG_DATA
+#define PCIEMSIX_VECT102_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT102_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT102_CONTROL
+#define PCIEMSIX_VECT102_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT102_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT103_ADDR_LO
+#define PCIEMSIX_VECT103_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT103_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT103_ADDR_HI
+#define PCIEMSIX_VECT103_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT103_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT103_MSG_DATA
+#define PCIEMSIX_VECT103_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT103_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT103_CONTROL
+#define PCIEMSIX_VECT103_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT103_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT104_ADDR_LO
+#define PCIEMSIX_VECT104_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT104_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT104_ADDR_HI
+#define PCIEMSIX_VECT104_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT104_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT104_MSG_DATA
+#define PCIEMSIX_VECT104_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT104_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT104_CONTROL
+#define PCIEMSIX_VECT104_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT104_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT105_ADDR_LO
+#define PCIEMSIX_VECT105_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT105_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT105_ADDR_HI
+#define PCIEMSIX_VECT105_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT105_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT105_MSG_DATA
+#define PCIEMSIX_VECT105_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT105_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT105_CONTROL
+#define PCIEMSIX_VECT105_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT105_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT106_ADDR_LO
+#define PCIEMSIX_VECT106_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT106_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT106_ADDR_HI
+#define PCIEMSIX_VECT106_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT106_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT106_MSG_DATA
+#define PCIEMSIX_VECT106_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT106_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT106_CONTROL
+#define PCIEMSIX_VECT106_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT106_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT107_ADDR_LO
+#define PCIEMSIX_VECT107_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT107_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT107_ADDR_HI
+#define PCIEMSIX_VECT107_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT107_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT107_MSG_DATA
+#define PCIEMSIX_VECT107_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT107_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT107_CONTROL
+#define PCIEMSIX_VECT107_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT107_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT108_ADDR_LO
+#define PCIEMSIX_VECT108_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT108_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT108_ADDR_HI
+#define PCIEMSIX_VECT108_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT108_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT108_MSG_DATA
+#define PCIEMSIX_VECT108_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT108_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT108_CONTROL
+#define PCIEMSIX_VECT108_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT108_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT109_ADDR_LO
+#define PCIEMSIX_VECT109_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT109_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT109_ADDR_HI
+#define PCIEMSIX_VECT109_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT109_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT109_MSG_DATA
+#define PCIEMSIX_VECT109_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT109_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT109_CONTROL
+#define PCIEMSIX_VECT109_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT109_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT110_ADDR_LO
+#define PCIEMSIX_VECT110_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT110_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT110_ADDR_HI
+#define PCIEMSIX_VECT110_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT110_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT110_MSG_DATA
+#define PCIEMSIX_VECT110_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT110_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT110_CONTROL
+#define PCIEMSIX_VECT110_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT110_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT111_ADDR_LO
+#define PCIEMSIX_VECT111_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT111_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT111_ADDR_HI
+#define PCIEMSIX_VECT111_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT111_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT111_MSG_DATA
+#define PCIEMSIX_VECT111_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT111_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT111_CONTROL
+#define PCIEMSIX_VECT111_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT111_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT112_ADDR_LO
+#define PCIEMSIX_VECT112_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT112_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT112_ADDR_HI
+#define PCIEMSIX_VECT112_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT112_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT112_MSG_DATA
+#define PCIEMSIX_VECT112_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT112_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT112_CONTROL
+#define PCIEMSIX_VECT112_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT112_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT113_ADDR_LO
+#define PCIEMSIX_VECT113_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT113_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT113_ADDR_HI
+#define PCIEMSIX_VECT113_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT113_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT113_MSG_DATA
+#define PCIEMSIX_VECT113_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT113_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT113_CONTROL
+#define PCIEMSIX_VECT113_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT113_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT114_ADDR_LO
+#define PCIEMSIX_VECT114_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT114_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT114_ADDR_HI
+#define PCIEMSIX_VECT114_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT114_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT114_MSG_DATA
+#define PCIEMSIX_VECT114_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT114_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT114_CONTROL
+#define PCIEMSIX_VECT114_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT114_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT115_ADDR_LO
+#define PCIEMSIX_VECT115_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT115_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT115_ADDR_HI
+#define PCIEMSIX_VECT115_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT115_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT115_MSG_DATA
+#define PCIEMSIX_VECT115_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT115_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT115_CONTROL
+#define PCIEMSIX_VECT115_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT115_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT116_ADDR_LO
+#define PCIEMSIX_VECT116_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT116_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT116_ADDR_HI
+#define PCIEMSIX_VECT116_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT116_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT116_MSG_DATA
+#define PCIEMSIX_VECT116_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT116_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT116_CONTROL
+#define PCIEMSIX_VECT116_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT116_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT117_ADDR_LO
+#define PCIEMSIX_VECT117_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT117_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT117_ADDR_HI
+#define PCIEMSIX_VECT117_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT117_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT117_MSG_DATA
+#define PCIEMSIX_VECT117_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT117_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT117_CONTROL
+#define PCIEMSIX_VECT117_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT117_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT118_ADDR_LO
+#define PCIEMSIX_VECT118_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT118_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT118_ADDR_HI
+#define PCIEMSIX_VECT118_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT118_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT118_MSG_DATA
+#define PCIEMSIX_VECT118_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT118_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT118_CONTROL
+#define PCIEMSIX_VECT118_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT118_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT119_ADDR_LO
+#define PCIEMSIX_VECT119_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT119_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT119_ADDR_HI
+#define PCIEMSIX_VECT119_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT119_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT119_MSG_DATA
+#define PCIEMSIX_VECT119_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT119_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT119_CONTROL
+#define PCIEMSIX_VECT119_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT119_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT120_ADDR_LO
+#define PCIEMSIX_VECT120_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT120_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT120_ADDR_HI
+#define PCIEMSIX_VECT120_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT120_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT120_MSG_DATA
+#define PCIEMSIX_VECT120_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT120_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT120_CONTROL
+#define PCIEMSIX_VECT120_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT120_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT121_ADDR_LO
+#define PCIEMSIX_VECT121_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT121_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT121_ADDR_HI
+#define PCIEMSIX_VECT121_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT121_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT121_MSG_DATA
+#define PCIEMSIX_VECT121_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT121_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT121_CONTROL
+#define PCIEMSIX_VECT121_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT121_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT122_ADDR_LO
+#define PCIEMSIX_VECT122_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT122_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT122_ADDR_HI
+#define PCIEMSIX_VECT122_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT122_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT122_MSG_DATA
+#define PCIEMSIX_VECT122_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT122_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT122_CONTROL
+#define PCIEMSIX_VECT122_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT122_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT123_ADDR_LO
+#define PCIEMSIX_VECT123_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT123_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT123_ADDR_HI
+#define PCIEMSIX_VECT123_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT123_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT123_MSG_DATA
+#define PCIEMSIX_VECT123_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT123_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT123_CONTROL
+#define PCIEMSIX_VECT123_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT123_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT124_ADDR_LO
+#define PCIEMSIX_VECT124_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT124_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT124_ADDR_HI
+#define PCIEMSIX_VECT124_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT124_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT124_MSG_DATA
+#define PCIEMSIX_VECT124_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT124_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT124_CONTROL
+#define PCIEMSIX_VECT124_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT124_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT125_ADDR_LO
+#define PCIEMSIX_VECT125_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT125_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT125_ADDR_HI
+#define PCIEMSIX_VECT125_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT125_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT125_MSG_DATA
+#define PCIEMSIX_VECT125_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT125_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT125_CONTROL
+#define PCIEMSIX_VECT125_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT125_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT126_ADDR_LO
+#define PCIEMSIX_VECT126_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT126_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT126_ADDR_HI
+#define PCIEMSIX_VECT126_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT126_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT126_MSG_DATA
+#define PCIEMSIX_VECT126_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT126_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT126_CONTROL
+#define PCIEMSIX_VECT126_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT126_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT127_ADDR_LO
+#define PCIEMSIX_VECT127_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT127_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT127_ADDR_HI
+#define PCIEMSIX_VECT127_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT127_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT127_MSG_DATA
+#define PCIEMSIX_VECT127_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT127_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT127_CONTROL
+#define PCIEMSIX_VECT127_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT127_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT128_ADDR_LO
+#define PCIEMSIX_VECT128_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT128_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT128_ADDR_HI
+#define PCIEMSIX_VECT128_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT128_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT128_MSG_DATA
+#define PCIEMSIX_VECT128_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT128_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT128_CONTROL
+#define PCIEMSIX_VECT128_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT128_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT129_ADDR_LO
+#define PCIEMSIX_VECT129_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT129_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT129_ADDR_HI
+#define PCIEMSIX_VECT129_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT129_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT129_MSG_DATA
+#define PCIEMSIX_VECT129_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT129_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT129_CONTROL
+#define PCIEMSIX_VECT129_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT129_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT130_ADDR_LO
+#define PCIEMSIX_VECT130_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT130_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT130_ADDR_HI
+#define PCIEMSIX_VECT130_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT130_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT130_MSG_DATA
+#define PCIEMSIX_VECT130_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT130_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT130_CONTROL
+#define PCIEMSIX_VECT130_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT130_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT131_ADDR_LO
+#define PCIEMSIX_VECT131_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT131_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT131_ADDR_HI
+#define PCIEMSIX_VECT131_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT131_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT131_MSG_DATA
+#define PCIEMSIX_VECT131_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT131_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT131_CONTROL
+#define PCIEMSIX_VECT131_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT131_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT132_ADDR_LO
+#define PCIEMSIX_VECT132_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT132_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT132_ADDR_HI
+#define PCIEMSIX_VECT132_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT132_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT132_MSG_DATA
+#define PCIEMSIX_VECT132_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT132_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT132_CONTROL
+#define PCIEMSIX_VECT132_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT132_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT133_ADDR_LO
+#define PCIEMSIX_VECT133_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT133_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT133_ADDR_HI
+#define PCIEMSIX_VECT133_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT133_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT133_MSG_DATA
+#define PCIEMSIX_VECT133_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT133_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT133_CONTROL
+#define PCIEMSIX_VECT133_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT133_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT134_ADDR_LO
+#define PCIEMSIX_VECT134_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT134_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT134_ADDR_HI
+#define PCIEMSIX_VECT134_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT134_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT134_MSG_DATA
+#define PCIEMSIX_VECT134_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT134_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT134_CONTROL
+#define PCIEMSIX_VECT134_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT134_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT135_ADDR_LO
+#define PCIEMSIX_VECT135_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT135_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT135_ADDR_HI
+#define PCIEMSIX_VECT135_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT135_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT135_MSG_DATA
+#define PCIEMSIX_VECT135_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT135_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT135_CONTROL
+#define PCIEMSIX_VECT135_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT135_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT136_ADDR_LO
+#define PCIEMSIX_VECT136_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT136_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT136_ADDR_HI
+#define PCIEMSIX_VECT136_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT136_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT136_MSG_DATA
+#define PCIEMSIX_VECT136_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT136_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT136_CONTROL
+#define PCIEMSIX_VECT136_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT136_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT137_ADDR_LO
+#define PCIEMSIX_VECT137_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT137_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT137_ADDR_HI
+#define PCIEMSIX_VECT137_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT137_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT137_MSG_DATA
+#define PCIEMSIX_VECT137_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT137_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT137_CONTROL
+#define PCIEMSIX_VECT137_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT137_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT138_ADDR_LO
+#define PCIEMSIX_VECT138_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT138_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT138_ADDR_HI
+#define PCIEMSIX_VECT138_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT138_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT138_MSG_DATA
+#define PCIEMSIX_VECT138_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT138_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT138_CONTROL
+#define PCIEMSIX_VECT138_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT138_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT139_ADDR_LO
+#define PCIEMSIX_VECT139_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT139_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT139_ADDR_HI
+#define PCIEMSIX_VECT139_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT139_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT139_MSG_DATA
+#define PCIEMSIX_VECT139_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT139_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT139_CONTROL
+#define PCIEMSIX_VECT139_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT139_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT140_ADDR_LO
+#define PCIEMSIX_VECT140_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT140_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT140_ADDR_HI
+#define PCIEMSIX_VECT140_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT140_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT140_MSG_DATA
+#define PCIEMSIX_VECT140_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT140_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT140_CONTROL
+#define PCIEMSIX_VECT140_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT140_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT141_ADDR_LO
+#define PCIEMSIX_VECT141_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT141_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT141_ADDR_HI
+#define PCIEMSIX_VECT141_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT141_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT141_MSG_DATA
+#define PCIEMSIX_VECT141_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT141_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT141_CONTROL
+#define PCIEMSIX_VECT141_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT141_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT142_ADDR_LO
+#define PCIEMSIX_VECT142_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT142_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT142_ADDR_HI
+#define PCIEMSIX_VECT142_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT142_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT142_MSG_DATA
+#define PCIEMSIX_VECT142_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT142_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT142_CONTROL
+#define PCIEMSIX_VECT142_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT142_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT143_ADDR_LO
+#define PCIEMSIX_VECT143_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT143_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT143_ADDR_HI
+#define PCIEMSIX_VECT143_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT143_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT143_MSG_DATA
+#define PCIEMSIX_VECT143_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT143_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT143_CONTROL
+#define PCIEMSIX_VECT143_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT143_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT144_ADDR_LO
+#define PCIEMSIX_VECT144_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT144_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT144_ADDR_HI
+#define PCIEMSIX_VECT144_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT144_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT144_MSG_DATA
+#define PCIEMSIX_VECT144_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT144_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT144_CONTROL
+#define PCIEMSIX_VECT144_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT144_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT145_ADDR_LO
+#define PCIEMSIX_VECT145_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT145_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT145_ADDR_HI
+#define PCIEMSIX_VECT145_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT145_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT145_MSG_DATA
+#define PCIEMSIX_VECT145_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT145_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT145_CONTROL
+#define PCIEMSIX_VECT145_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT145_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT146_ADDR_LO
+#define PCIEMSIX_VECT146_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT146_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT146_ADDR_HI
+#define PCIEMSIX_VECT146_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT146_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT146_MSG_DATA
+#define PCIEMSIX_VECT146_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT146_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT146_CONTROL
+#define PCIEMSIX_VECT146_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT146_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT147_ADDR_LO
+#define PCIEMSIX_VECT147_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT147_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT147_ADDR_HI
+#define PCIEMSIX_VECT147_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT147_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT147_MSG_DATA
+#define PCIEMSIX_VECT147_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT147_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT147_CONTROL
+#define PCIEMSIX_VECT147_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT147_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT148_ADDR_LO
+#define PCIEMSIX_VECT148_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT148_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT148_ADDR_HI
+#define PCIEMSIX_VECT148_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT148_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT148_MSG_DATA
+#define PCIEMSIX_VECT148_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT148_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT148_CONTROL
+#define PCIEMSIX_VECT148_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT148_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT149_ADDR_LO
+#define PCIEMSIX_VECT149_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT149_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT149_ADDR_HI
+#define PCIEMSIX_VECT149_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT149_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT149_MSG_DATA
+#define PCIEMSIX_VECT149_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT149_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT149_CONTROL
+#define PCIEMSIX_VECT149_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT149_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT150_ADDR_LO
+#define PCIEMSIX_VECT150_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT150_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT150_ADDR_HI
+#define PCIEMSIX_VECT150_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT150_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT150_MSG_DATA
+#define PCIEMSIX_VECT150_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT150_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT150_CONTROL
+#define PCIEMSIX_VECT150_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT150_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT151_ADDR_LO
+#define PCIEMSIX_VECT151_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT151_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT151_ADDR_HI
+#define PCIEMSIX_VECT151_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT151_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT151_MSG_DATA
+#define PCIEMSIX_VECT151_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT151_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT151_CONTROL
+#define PCIEMSIX_VECT151_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT151_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT152_ADDR_LO
+#define PCIEMSIX_VECT152_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT152_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT152_ADDR_HI
+#define PCIEMSIX_VECT152_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT152_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT152_MSG_DATA
+#define PCIEMSIX_VECT152_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT152_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT152_CONTROL
+#define PCIEMSIX_VECT152_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT152_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT153_ADDR_LO
+#define PCIEMSIX_VECT153_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT153_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT153_ADDR_HI
+#define PCIEMSIX_VECT153_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT153_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT153_MSG_DATA
+#define PCIEMSIX_VECT153_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT153_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT153_CONTROL
+#define PCIEMSIX_VECT153_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT153_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT154_ADDR_LO
+#define PCIEMSIX_VECT154_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT154_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT154_ADDR_HI
+#define PCIEMSIX_VECT154_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT154_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT154_MSG_DATA
+#define PCIEMSIX_VECT154_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT154_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT154_CONTROL
+#define PCIEMSIX_VECT154_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT154_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT155_ADDR_LO
+#define PCIEMSIX_VECT155_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT155_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT155_ADDR_HI
+#define PCIEMSIX_VECT155_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT155_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT155_MSG_DATA
+#define PCIEMSIX_VECT155_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT155_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT155_CONTROL
+#define PCIEMSIX_VECT155_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT155_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT156_ADDR_LO
+#define PCIEMSIX_VECT156_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT156_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT156_ADDR_HI
+#define PCIEMSIX_VECT156_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT156_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT156_MSG_DATA
+#define PCIEMSIX_VECT156_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT156_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT156_CONTROL
+#define PCIEMSIX_VECT156_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT156_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT157_ADDR_LO
+#define PCIEMSIX_VECT157_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT157_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT157_ADDR_HI
+#define PCIEMSIX_VECT157_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT157_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT157_MSG_DATA
+#define PCIEMSIX_VECT157_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT157_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT157_CONTROL
+#define PCIEMSIX_VECT157_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT157_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT158_ADDR_LO
+#define PCIEMSIX_VECT158_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT158_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT158_ADDR_HI
+#define PCIEMSIX_VECT158_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT158_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT158_MSG_DATA
+#define PCIEMSIX_VECT158_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT158_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT158_CONTROL
+#define PCIEMSIX_VECT158_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT158_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT159_ADDR_LO
+#define PCIEMSIX_VECT159_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT159_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT159_ADDR_HI
+#define PCIEMSIX_VECT159_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT159_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT159_MSG_DATA
+#define PCIEMSIX_VECT159_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT159_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT159_CONTROL
+#define PCIEMSIX_VECT159_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT159_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT160_ADDR_LO
+#define PCIEMSIX_VECT160_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT160_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT160_ADDR_HI
+#define PCIEMSIX_VECT160_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT160_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT160_MSG_DATA
+#define PCIEMSIX_VECT160_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT160_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT160_CONTROL
+#define PCIEMSIX_VECT160_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT160_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT161_ADDR_LO
+#define PCIEMSIX_VECT161_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT161_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT161_ADDR_HI
+#define PCIEMSIX_VECT161_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT161_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT161_MSG_DATA
+#define PCIEMSIX_VECT161_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT161_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT161_CONTROL
+#define PCIEMSIX_VECT161_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT161_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT162_ADDR_LO
+#define PCIEMSIX_VECT162_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT162_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT162_ADDR_HI
+#define PCIEMSIX_VECT162_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT162_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT162_MSG_DATA
+#define PCIEMSIX_VECT162_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT162_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT162_CONTROL
+#define PCIEMSIX_VECT162_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT162_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT163_ADDR_LO
+#define PCIEMSIX_VECT163_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT163_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT163_ADDR_HI
+#define PCIEMSIX_VECT163_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT163_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT163_MSG_DATA
+#define PCIEMSIX_VECT163_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT163_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT163_CONTROL
+#define PCIEMSIX_VECT163_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT163_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT164_ADDR_LO
+#define PCIEMSIX_VECT164_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT164_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT164_ADDR_HI
+#define PCIEMSIX_VECT164_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT164_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT164_MSG_DATA
+#define PCIEMSIX_VECT164_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT164_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT164_CONTROL
+#define PCIEMSIX_VECT164_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT164_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT165_ADDR_LO
+#define PCIEMSIX_VECT165_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT165_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT165_ADDR_HI
+#define PCIEMSIX_VECT165_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT165_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT165_MSG_DATA
+#define PCIEMSIX_VECT165_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT165_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT165_CONTROL
+#define PCIEMSIX_VECT165_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT165_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT166_ADDR_LO
+#define PCIEMSIX_VECT166_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT166_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT166_ADDR_HI
+#define PCIEMSIX_VECT166_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT166_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT166_MSG_DATA
+#define PCIEMSIX_VECT166_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT166_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT166_CONTROL
+#define PCIEMSIX_VECT166_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT166_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT167_ADDR_LO
+#define PCIEMSIX_VECT167_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT167_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT167_ADDR_HI
+#define PCIEMSIX_VECT167_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT167_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT167_MSG_DATA
+#define PCIEMSIX_VECT167_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT167_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT167_CONTROL
+#define PCIEMSIX_VECT167_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT167_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT168_ADDR_LO
+#define PCIEMSIX_VECT168_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT168_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT168_ADDR_HI
+#define PCIEMSIX_VECT168_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT168_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT168_MSG_DATA
+#define PCIEMSIX_VECT168_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT168_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT168_CONTROL
+#define PCIEMSIX_VECT168_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT168_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT169_ADDR_LO
+#define PCIEMSIX_VECT169_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT169_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT169_ADDR_HI
+#define PCIEMSIX_VECT169_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT169_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT169_MSG_DATA
+#define PCIEMSIX_VECT169_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT169_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT169_CONTROL
+#define PCIEMSIX_VECT169_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT169_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT170_ADDR_LO
+#define PCIEMSIX_VECT170_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT170_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT170_ADDR_HI
+#define PCIEMSIX_VECT170_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT170_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT170_MSG_DATA
+#define PCIEMSIX_VECT170_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT170_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT170_CONTROL
+#define PCIEMSIX_VECT170_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT170_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT171_ADDR_LO
+#define PCIEMSIX_VECT171_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT171_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT171_ADDR_HI
+#define PCIEMSIX_VECT171_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT171_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT171_MSG_DATA
+#define PCIEMSIX_VECT171_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT171_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT171_CONTROL
+#define PCIEMSIX_VECT171_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT171_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT172_ADDR_LO
+#define PCIEMSIX_VECT172_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT172_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT172_ADDR_HI
+#define PCIEMSIX_VECT172_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT172_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT172_MSG_DATA
+#define PCIEMSIX_VECT172_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT172_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT172_CONTROL
+#define PCIEMSIX_VECT172_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT172_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT173_ADDR_LO
+#define PCIEMSIX_VECT173_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT173_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT173_ADDR_HI
+#define PCIEMSIX_VECT173_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT173_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT173_MSG_DATA
+#define PCIEMSIX_VECT173_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT173_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT173_CONTROL
+#define PCIEMSIX_VECT173_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT173_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT174_ADDR_LO
+#define PCIEMSIX_VECT174_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT174_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT174_ADDR_HI
+#define PCIEMSIX_VECT174_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT174_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT174_MSG_DATA
+#define PCIEMSIX_VECT174_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT174_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT174_CONTROL
+#define PCIEMSIX_VECT174_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT174_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT175_ADDR_LO
+#define PCIEMSIX_VECT175_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT175_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT175_ADDR_HI
+#define PCIEMSIX_VECT175_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT175_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT175_MSG_DATA
+#define PCIEMSIX_VECT175_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT175_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT175_CONTROL
+#define PCIEMSIX_VECT175_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT175_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT176_ADDR_LO
+#define PCIEMSIX_VECT176_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT176_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT176_ADDR_HI
+#define PCIEMSIX_VECT176_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT176_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT176_MSG_DATA
+#define PCIEMSIX_VECT176_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT176_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT176_CONTROL
+#define PCIEMSIX_VECT176_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT176_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT177_ADDR_LO
+#define PCIEMSIX_VECT177_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT177_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT177_ADDR_HI
+#define PCIEMSIX_VECT177_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT177_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT177_MSG_DATA
+#define PCIEMSIX_VECT177_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT177_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT177_CONTROL
+#define PCIEMSIX_VECT177_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT177_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT178_ADDR_LO
+#define PCIEMSIX_VECT178_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT178_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT178_ADDR_HI
+#define PCIEMSIX_VECT178_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT178_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT178_MSG_DATA
+#define PCIEMSIX_VECT178_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT178_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT178_CONTROL
+#define PCIEMSIX_VECT178_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT178_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT179_ADDR_LO
+#define PCIEMSIX_VECT179_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT179_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT179_ADDR_HI
+#define PCIEMSIX_VECT179_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT179_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT179_MSG_DATA
+#define PCIEMSIX_VECT179_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT179_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT179_CONTROL
+#define PCIEMSIX_VECT179_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT179_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT180_ADDR_LO
+#define PCIEMSIX_VECT180_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT180_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT180_ADDR_HI
+#define PCIEMSIX_VECT180_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT180_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT180_MSG_DATA
+#define PCIEMSIX_VECT180_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT180_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT180_CONTROL
+#define PCIEMSIX_VECT180_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT180_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT181_ADDR_LO
+#define PCIEMSIX_VECT181_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT181_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT181_ADDR_HI
+#define PCIEMSIX_VECT181_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT181_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT181_MSG_DATA
+#define PCIEMSIX_VECT181_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT181_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT181_CONTROL
+#define PCIEMSIX_VECT181_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT181_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT182_ADDR_LO
+#define PCIEMSIX_VECT182_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT182_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT182_ADDR_HI
+#define PCIEMSIX_VECT182_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT182_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT182_MSG_DATA
+#define PCIEMSIX_VECT182_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT182_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT182_CONTROL
+#define PCIEMSIX_VECT182_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT182_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT183_ADDR_LO
+#define PCIEMSIX_VECT183_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT183_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT183_ADDR_HI
+#define PCIEMSIX_VECT183_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT183_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT183_MSG_DATA
+#define PCIEMSIX_VECT183_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT183_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT183_CONTROL
+#define PCIEMSIX_VECT183_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT183_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT184_ADDR_LO
+#define PCIEMSIX_VECT184_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT184_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT184_ADDR_HI
+#define PCIEMSIX_VECT184_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT184_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT184_MSG_DATA
+#define PCIEMSIX_VECT184_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT184_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT184_CONTROL
+#define PCIEMSIX_VECT184_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT184_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT185_ADDR_LO
+#define PCIEMSIX_VECT185_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT185_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT185_ADDR_HI
+#define PCIEMSIX_VECT185_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT185_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT185_MSG_DATA
+#define PCIEMSIX_VECT185_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT185_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT185_CONTROL
+#define PCIEMSIX_VECT185_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT185_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT186_ADDR_LO
+#define PCIEMSIX_VECT186_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT186_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT186_ADDR_HI
+#define PCIEMSIX_VECT186_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT186_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT186_MSG_DATA
+#define PCIEMSIX_VECT186_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT186_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT186_CONTROL
+#define PCIEMSIX_VECT186_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT186_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT187_ADDR_LO
+#define PCIEMSIX_VECT187_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT187_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT187_ADDR_HI
+#define PCIEMSIX_VECT187_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT187_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT187_MSG_DATA
+#define PCIEMSIX_VECT187_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT187_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT187_CONTROL
+#define PCIEMSIX_VECT187_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT187_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT188_ADDR_LO
+#define PCIEMSIX_VECT188_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT188_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT188_ADDR_HI
+#define PCIEMSIX_VECT188_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT188_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT188_MSG_DATA
+#define PCIEMSIX_VECT188_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT188_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT188_CONTROL
+#define PCIEMSIX_VECT188_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT188_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT189_ADDR_LO
+#define PCIEMSIX_VECT189_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT189_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT189_ADDR_HI
+#define PCIEMSIX_VECT189_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT189_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT189_MSG_DATA
+#define PCIEMSIX_VECT189_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT189_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT189_CONTROL
+#define PCIEMSIX_VECT189_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT189_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT190_ADDR_LO
+#define PCIEMSIX_VECT190_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT190_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT190_ADDR_HI
+#define PCIEMSIX_VECT190_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT190_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT190_MSG_DATA
+#define PCIEMSIX_VECT190_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT190_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT190_CONTROL
+#define PCIEMSIX_VECT190_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT190_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT191_ADDR_LO
+#define PCIEMSIX_VECT191_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT191_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT191_ADDR_HI
+#define PCIEMSIX_VECT191_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT191_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT191_MSG_DATA
+#define PCIEMSIX_VECT191_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT191_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT191_CONTROL
+#define PCIEMSIX_VECT191_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT191_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT192_ADDR_LO
+#define PCIEMSIX_VECT192_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT192_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT192_ADDR_HI
+#define PCIEMSIX_VECT192_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT192_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT192_MSG_DATA
+#define PCIEMSIX_VECT192_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT192_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT192_CONTROL
+#define PCIEMSIX_VECT192_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT192_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT193_ADDR_LO
+#define PCIEMSIX_VECT193_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT193_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT193_ADDR_HI
+#define PCIEMSIX_VECT193_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT193_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT193_MSG_DATA
+#define PCIEMSIX_VECT193_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT193_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT193_CONTROL
+#define PCIEMSIX_VECT193_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT193_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT194_ADDR_LO
+#define PCIEMSIX_VECT194_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT194_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT194_ADDR_HI
+#define PCIEMSIX_VECT194_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT194_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT194_MSG_DATA
+#define PCIEMSIX_VECT194_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT194_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT194_CONTROL
+#define PCIEMSIX_VECT194_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT194_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT195_ADDR_LO
+#define PCIEMSIX_VECT195_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT195_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT195_ADDR_HI
+#define PCIEMSIX_VECT195_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT195_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT195_MSG_DATA
+#define PCIEMSIX_VECT195_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT195_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT195_CONTROL
+#define PCIEMSIX_VECT195_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT195_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT196_ADDR_LO
+#define PCIEMSIX_VECT196_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT196_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT196_ADDR_HI
+#define PCIEMSIX_VECT196_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT196_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT196_MSG_DATA
+#define PCIEMSIX_VECT196_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT196_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT196_CONTROL
+#define PCIEMSIX_VECT196_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT196_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT197_ADDR_LO
+#define PCIEMSIX_VECT197_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT197_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT197_ADDR_HI
+#define PCIEMSIX_VECT197_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT197_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT197_MSG_DATA
+#define PCIEMSIX_VECT197_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT197_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT197_CONTROL
+#define PCIEMSIX_VECT197_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT197_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT198_ADDR_LO
+#define PCIEMSIX_VECT198_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT198_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT198_ADDR_HI
+#define PCIEMSIX_VECT198_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT198_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT198_MSG_DATA
+#define PCIEMSIX_VECT198_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT198_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT198_CONTROL
+#define PCIEMSIX_VECT198_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT198_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT199_ADDR_LO
+#define PCIEMSIX_VECT199_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT199_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT199_ADDR_HI
+#define PCIEMSIX_VECT199_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT199_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT199_MSG_DATA
+#define PCIEMSIX_VECT199_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT199_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT199_CONTROL
+#define PCIEMSIX_VECT199_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT199_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT200_ADDR_LO
+#define PCIEMSIX_VECT200_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT200_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT200_ADDR_HI
+#define PCIEMSIX_VECT200_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT200_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT200_MSG_DATA
+#define PCIEMSIX_VECT200_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT200_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT200_CONTROL
+#define PCIEMSIX_VECT200_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT200_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT201_ADDR_LO
+#define PCIEMSIX_VECT201_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT201_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT201_ADDR_HI
+#define PCIEMSIX_VECT201_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT201_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT201_MSG_DATA
+#define PCIEMSIX_VECT201_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT201_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT201_CONTROL
+#define PCIEMSIX_VECT201_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT201_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT202_ADDR_LO
+#define PCIEMSIX_VECT202_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT202_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT202_ADDR_HI
+#define PCIEMSIX_VECT202_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT202_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT202_MSG_DATA
+#define PCIEMSIX_VECT202_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT202_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT202_CONTROL
+#define PCIEMSIX_VECT202_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT202_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT203_ADDR_LO
+#define PCIEMSIX_VECT203_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT203_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT203_ADDR_HI
+#define PCIEMSIX_VECT203_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT203_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT203_MSG_DATA
+#define PCIEMSIX_VECT203_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT203_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT203_CONTROL
+#define PCIEMSIX_VECT203_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT203_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT204_ADDR_LO
+#define PCIEMSIX_VECT204_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT204_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT204_ADDR_HI
+#define PCIEMSIX_VECT204_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT204_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT204_MSG_DATA
+#define PCIEMSIX_VECT204_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT204_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT204_CONTROL
+#define PCIEMSIX_VECT204_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT204_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT205_ADDR_LO
+#define PCIEMSIX_VECT205_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT205_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT205_ADDR_HI
+#define PCIEMSIX_VECT205_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT205_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT205_MSG_DATA
+#define PCIEMSIX_VECT205_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT205_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT205_CONTROL
+#define PCIEMSIX_VECT205_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT205_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT206_ADDR_LO
+#define PCIEMSIX_VECT206_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT206_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT206_ADDR_HI
+#define PCIEMSIX_VECT206_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT206_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT206_MSG_DATA
+#define PCIEMSIX_VECT206_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT206_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT206_CONTROL
+#define PCIEMSIX_VECT206_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT206_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT207_ADDR_LO
+#define PCIEMSIX_VECT207_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT207_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT207_ADDR_HI
+#define PCIEMSIX_VECT207_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT207_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT207_MSG_DATA
+#define PCIEMSIX_VECT207_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT207_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT207_CONTROL
+#define PCIEMSIX_VECT207_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT207_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT208_ADDR_LO
+#define PCIEMSIX_VECT208_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT208_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT208_ADDR_HI
+#define PCIEMSIX_VECT208_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT208_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT208_MSG_DATA
+#define PCIEMSIX_VECT208_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT208_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT208_CONTROL
+#define PCIEMSIX_VECT208_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT208_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT209_ADDR_LO
+#define PCIEMSIX_VECT209_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT209_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT209_ADDR_HI
+#define PCIEMSIX_VECT209_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT209_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT209_MSG_DATA
+#define PCIEMSIX_VECT209_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT209_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT209_CONTROL
+#define PCIEMSIX_VECT209_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT209_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT210_ADDR_LO
+#define PCIEMSIX_VECT210_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT210_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT210_ADDR_HI
+#define PCIEMSIX_VECT210_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT210_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT210_MSG_DATA
+#define PCIEMSIX_VECT210_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT210_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT210_CONTROL
+#define PCIEMSIX_VECT210_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT210_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT211_ADDR_LO
+#define PCIEMSIX_VECT211_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT211_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT211_ADDR_HI
+#define PCIEMSIX_VECT211_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT211_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT211_MSG_DATA
+#define PCIEMSIX_VECT211_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT211_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT211_CONTROL
+#define PCIEMSIX_VECT211_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT211_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT212_ADDR_LO
+#define PCIEMSIX_VECT212_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT212_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT212_ADDR_HI
+#define PCIEMSIX_VECT212_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT212_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT212_MSG_DATA
+#define PCIEMSIX_VECT212_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT212_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT212_CONTROL
+#define PCIEMSIX_VECT212_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT212_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT213_ADDR_LO
+#define PCIEMSIX_VECT213_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT213_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT213_ADDR_HI
+#define PCIEMSIX_VECT213_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT213_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT213_MSG_DATA
+#define PCIEMSIX_VECT213_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT213_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT213_CONTROL
+#define PCIEMSIX_VECT213_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT213_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT214_ADDR_LO
+#define PCIEMSIX_VECT214_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT214_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT214_ADDR_HI
+#define PCIEMSIX_VECT214_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT214_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT214_MSG_DATA
+#define PCIEMSIX_VECT214_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT214_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT214_CONTROL
+#define PCIEMSIX_VECT214_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT214_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT215_ADDR_LO
+#define PCIEMSIX_VECT215_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT215_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT215_ADDR_HI
+#define PCIEMSIX_VECT215_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT215_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT215_MSG_DATA
+#define PCIEMSIX_VECT215_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT215_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT215_CONTROL
+#define PCIEMSIX_VECT215_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT215_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT216_ADDR_LO
+#define PCIEMSIX_VECT216_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT216_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT216_ADDR_HI
+#define PCIEMSIX_VECT216_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT216_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT216_MSG_DATA
+#define PCIEMSIX_VECT216_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT216_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT216_CONTROL
+#define PCIEMSIX_VECT216_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT216_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT217_ADDR_LO
+#define PCIEMSIX_VECT217_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT217_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT217_ADDR_HI
+#define PCIEMSIX_VECT217_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT217_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT217_MSG_DATA
+#define PCIEMSIX_VECT217_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT217_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT217_CONTROL
+#define PCIEMSIX_VECT217_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT217_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT218_ADDR_LO
+#define PCIEMSIX_VECT218_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT218_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT218_ADDR_HI
+#define PCIEMSIX_VECT218_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT218_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT218_MSG_DATA
+#define PCIEMSIX_VECT218_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT218_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT218_CONTROL
+#define PCIEMSIX_VECT218_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT218_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT219_ADDR_LO
+#define PCIEMSIX_VECT219_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT219_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT219_ADDR_HI
+#define PCIEMSIX_VECT219_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT219_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT219_MSG_DATA
+#define PCIEMSIX_VECT219_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT219_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT219_CONTROL
+#define PCIEMSIX_VECT219_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT219_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT220_ADDR_LO
+#define PCIEMSIX_VECT220_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT220_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT220_ADDR_HI
+#define PCIEMSIX_VECT220_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT220_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT220_MSG_DATA
+#define PCIEMSIX_VECT220_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT220_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT220_CONTROL
+#define PCIEMSIX_VECT220_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT220_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT221_ADDR_LO
+#define PCIEMSIX_VECT221_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT221_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT221_ADDR_HI
+#define PCIEMSIX_VECT221_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT221_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT221_MSG_DATA
+#define PCIEMSIX_VECT221_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT221_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT221_CONTROL
+#define PCIEMSIX_VECT221_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT221_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT222_ADDR_LO
+#define PCIEMSIX_VECT222_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT222_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT222_ADDR_HI
+#define PCIEMSIX_VECT222_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT222_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT222_MSG_DATA
+#define PCIEMSIX_VECT222_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT222_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT222_CONTROL
+#define PCIEMSIX_VECT222_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT222_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT223_ADDR_LO
+#define PCIEMSIX_VECT223_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT223_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT223_ADDR_HI
+#define PCIEMSIX_VECT223_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT223_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT223_MSG_DATA
+#define PCIEMSIX_VECT223_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT223_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT223_CONTROL
+#define PCIEMSIX_VECT223_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT223_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT224_ADDR_LO
+#define PCIEMSIX_VECT224_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT224_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT224_ADDR_HI
+#define PCIEMSIX_VECT224_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT224_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT224_MSG_DATA
+#define PCIEMSIX_VECT224_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT224_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT224_CONTROL
+#define PCIEMSIX_VECT224_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT224_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT225_ADDR_LO
+#define PCIEMSIX_VECT225_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT225_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT225_ADDR_HI
+#define PCIEMSIX_VECT225_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT225_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT225_MSG_DATA
+#define PCIEMSIX_VECT225_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT225_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT225_CONTROL
+#define PCIEMSIX_VECT225_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT225_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT226_ADDR_LO
+#define PCIEMSIX_VECT226_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT226_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT226_ADDR_HI
+#define PCIEMSIX_VECT226_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT226_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT226_MSG_DATA
+#define PCIEMSIX_VECT226_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT226_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT226_CONTROL
+#define PCIEMSIX_VECT226_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT226_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT227_ADDR_LO
+#define PCIEMSIX_VECT227_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT227_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT227_ADDR_HI
+#define PCIEMSIX_VECT227_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT227_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT227_MSG_DATA
+#define PCIEMSIX_VECT227_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT227_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT227_CONTROL
+#define PCIEMSIX_VECT227_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT227_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT228_ADDR_LO
+#define PCIEMSIX_VECT228_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT228_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT228_ADDR_HI
+#define PCIEMSIX_VECT228_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT228_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT228_MSG_DATA
+#define PCIEMSIX_VECT228_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT228_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT228_CONTROL
+#define PCIEMSIX_VECT228_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT228_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT229_ADDR_LO
+#define PCIEMSIX_VECT229_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT229_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT229_ADDR_HI
+#define PCIEMSIX_VECT229_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT229_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT229_MSG_DATA
+#define PCIEMSIX_VECT229_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT229_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT229_CONTROL
+#define PCIEMSIX_VECT229_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT229_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT230_ADDR_LO
+#define PCIEMSIX_VECT230_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT230_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT230_ADDR_HI
+#define PCIEMSIX_VECT230_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT230_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT230_MSG_DATA
+#define PCIEMSIX_VECT230_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT230_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT230_CONTROL
+#define PCIEMSIX_VECT230_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT230_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT231_ADDR_LO
+#define PCIEMSIX_VECT231_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT231_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT231_ADDR_HI
+#define PCIEMSIX_VECT231_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT231_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT231_MSG_DATA
+#define PCIEMSIX_VECT231_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT231_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT231_CONTROL
+#define PCIEMSIX_VECT231_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT231_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT232_ADDR_LO
+#define PCIEMSIX_VECT232_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT232_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT232_ADDR_HI
+#define PCIEMSIX_VECT232_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT232_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT232_MSG_DATA
+#define PCIEMSIX_VECT232_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT232_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT232_CONTROL
+#define PCIEMSIX_VECT232_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT232_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT233_ADDR_LO
+#define PCIEMSIX_VECT233_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT233_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT233_ADDR_HI
+#define PCIEMSIX_VECT233_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT233_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT233_MSG_DATA
+#define PCIEMSIX_VECT233_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT233_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT233_CONTROL
+#define PCIEMSIX_VECT233_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT233_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT234_ADDR_LO
+#define PCIEMSIX_VECT234_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT234_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT234_ADDR_HI
+#define PCIEMSIX_VECT234_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT234_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT234_MSG_DATA
+#define PCIEMSIX_VECT234_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT234_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT234_CONTROL
+#define PCIEMSIX_VECT234_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT234_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT235_ADDR_LO
+#define PCIEMSIX_VECT235_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT235_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT235_ADDR_HI
+#define PCIEMSIX_VECT235_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT235_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT235_MSG_DATA
+#define PCIEMSIX_VECT235_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT235_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT235_CONTROL
+#define PCIEMSIX_VECT235_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT235_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT236_ADDR_LO
+#define PCIEMSIX_VECT236_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT236_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT236_ADDR_HI
+#define PCIEMSIX_VECT236_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT236_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT236_MSG_DATA
+#define PCIEMSIX_VECT236_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT236_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT236_CONTROL
+#define PCIEMSIX_VECT236_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT236_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT237_ADDR_LO
+#define PCIEMSIX_VECT237_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT237_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT237_ADDR_HI
+#define PCIEMSIX_VECT237_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT237_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT237_MSG_DATA
+#define PCIEMSIX_VECT237_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT237_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT237_CONTROL
+#define PCIEMSIX_VECT237_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT237_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT238_ADDR_LO
+#define PCIEMSIX_VECT238_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT238_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT238_ADDR_HI
+#define PCIEMSIX_VECT238_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT238_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT238_MSG_DATA
+#define PCIEMSIX_VECT238_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT238_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT238_CONTROL
+#define PCIEMSIX_VECT238_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT238_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT239_ADDR_LO
+#define PCIEMSIX_VECT239_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT239_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT239_ADDR_HI
+#define PCIEMSIX_VECT239_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT239_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT239_MSG_DATA
+#define PCIEMSIX_VECT239_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT239_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT239_CONTROL
+#define PCIEMSIX_VECT239_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT239_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT240_ADDR_LO
+#define PCIEMSIX_VECT240_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT240_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT240_ADDR_HI
+#define PCIEMSIX_VECT240_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT240_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT240_MSG_DATA
+#define PCIEMSIX_VECT240_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT240_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT240_CONTROL
+#define PCIEMSIX_VECT240_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT240_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT241_ADDR_LO
+#define PCIEMSIX_VECT241_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT241_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT241_ADDR_HI
+#define PCIEMSIX_VECT241_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT241_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT241_MSG_DATA
+#define PCIEMSIX_VECT241_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT241_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT241_CONTROL
+#define PCIEMSIX_VECT241_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT241_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT242_ADDR_LO
+#define PCIEMSIX_VECT242_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT242_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT242_ADDR_HI
+#define PCIEMSIX_VECT242_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT242_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT242_MSG_DATA
+#define PCIEMSIX_VECT242_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT242_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT242_CONTROL
+#define PCIEMSIX_VECT242_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT242_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT243_ADDR_LO
+#define PCIEMSIX_VECT243_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT243_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT243_ADDR_HI
+#define PCIEMSIX_VECT243_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT243_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT243_MSG_DATA
+#define PCIEMSIX_VECT243_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT243_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT243_CONTROL
+#define PCIEMSIX_VECT243_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT243_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT244_ADDR_LO
+#define PCIEMSIX_VECT244_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT244_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT244_ADDR_HI
+#define PCIEMSIX_VECT244_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT244_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT244_MSG_DATA
+#define PCIEMSIX_VECT244_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT244_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT244_CONTROL
+#define PCIEMSIX_VECT244_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT244_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT245_ADDR_LO
+#define PCIEMSIX_VECT245_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT245_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT245_ADDR_HI
+#define PCIEMSIX_VECT245_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT245_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT245_MSG_DATA
+#define PCIEMSIX_VECT245_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT245_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT245_CONTROL
+#define PCIEMSIX_VECT245_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT245_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT246_ADDR_LO
+#define PCIEMSIX_VECT246_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT246_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT246_ADDR_HI
+#define PCIEMSIX_VECT246_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT246_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT246_MSG_DATA
+#define PCIEMSIX_VECT246_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT246_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT246_CONTROL
+#define PCIEMSIX_VECT246_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT246_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT247_ADDR_LO
+#define PCIEMSIX_VECT247_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT247_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT247_ADDR_HI
+#define PCIEMSIX_VECT247_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT247_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT247_MSG_DATA
+#define PCIEMSIX_VECT247_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT247_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT247_CONTROL
+#define PCIEMSIX_VECT247_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT247_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT248_ADDR_LO
+#define PCIEMSIX_VECT248_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT248_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT248_ADDR_HI
+#define PCIEMSIX_VECT248_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT248_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT248_MSG_DATA
+#define PCIEMSIX_VECT248_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT248_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT248_CONTROL
+#define PCIEMSIX_VECT248_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT248_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT249_ADDR_LO
+#define PCIEMSIX_VECT249_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT249_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT249_ADDR_HI
+#define PCIEMSIX_VECT249_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT249_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT249_MSG_DATA
+#define PCIEMSIX_VECT249_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT249_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT249_CONTROL
+#define PCIEMSIX_VECT249_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT249_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT250_ADDR_LO
+#define PCIEMSIX_VECT250_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT250_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT250_ADDR_HI
+#define PCIEMSIX_VECT250_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT250_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT250_MSG_DATA
+#define PCIEMSIX_VECT250_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT250_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT250_CONTROL
+#define PCIEMSIX_VECT250_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT250_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT251_ADDR_LO
+#define PCIEMSIX_VECT251_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT251_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT251_ADDR_HI
+#define PCIEMSIX_VECT251_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT251_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT251_MSG_DATA
+#define PCIEMSIX_VECT251_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT251_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT251_CONTROL
+#define PCIEMSIX_VECT251_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT251_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT252_ADDR_LO
+#define PCIEMSIX_VECT252_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT252_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT252_ADDR_HI
+#define PCIEMSIX_VECT252_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT252_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT252_MSG_DATA
+#define PCIEMSIX_VECT252_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT252_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT252_CONTROL
+#define PCIEMSIX_VECT252_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT252_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT253_ADDR_LO
+#define PCIEMSIX_VECT253_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT253_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT253_ADDR_HI
+#define PCIEMSIX_VECT253_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT253_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT253_MSG_DATA
+#define PCIEMSIX_VECT253_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT253_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT253_CONTROL
+#define PCIEMSIX_VECT253_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT253_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT254_ADDR_LO
+#define PCIEMSIX_VECT254_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT254_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT254_ADDR_HI
+#define PCIEMSIX_VECT254_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT254_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT254_MSG_DATA
+#define PCIEMSIX_VECT254_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT254_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT254_CONTROL
+#define PCIEMSIX_VECT254_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT254_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+//PCIEMSIX_VECT255_ADDR_LO
+#define PCIEMSIX_VECT255_ADDR_LO__MSG_ADDR_LO__SHIFT                                                          0x2
+#define PCIEMSIX_VECT255_ADDR_LO__MSG_ADDR_LO_MASK                                                            0xFFFFFFFCL
+//PCIEMSIX_VECT255_ADDR_HI
+#define PCIEMSIX_VECT255_ADDR_HI__MSG_ADDR_HI__SHIFT                                                          0x0
+#define PCIEMSIX_VECT255_ADDR_HI__MSG_ADDR_HI_MASK                                                            0xFFFFFFFFL
+//PCIEMSIX_VECT255_MSG_DATA
+#define PCIEMSIX_VECT255_MSG_DATA__MSG_DATA__SHIFT                                                            0x0
+#define PCIEMSIX_VECT255_MSG_DATA__MSG_DATA_MASK                                                              0xFFFFFFFFL
+//PCIEMSIX_VECT255_CONTROL
+#define PCIEMSIX_VECT255_CONTROL__MASK_BIT__SHIFT                                                             0x0
+#define PCIEMSIX_VECT255_CONTROL__MASK_BIT_MASK                                                               0x00000001L
+
+
+// addressBlock: aid_nbio_nbif0_pciemsix_0_usb_MSIXPDEC
+//PCIEMSIX_PBA_0
+#define PCIEMSIX_PBA_0__MSIX_PENDING_BITS__SHIFT                                                              0x0
+#define PCIEMSIX_PBA_0__MSIX_PENDING_BITS_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_PBA_1
+#define PCIEMSIX_PBA_1__MSIX_PENDING_BITS__SHIFT                                                              0x0
+#define PCIEMSIX_PBA_1__MSIX_PENDING_BITS_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_PBA_2
+#define PCIEMSIX_PBA_2__MSIX_PENDING_BITS__SHIFT                                                              0x0
+#define PCIEMSIX_PBA_2__MSIX_PENDING_BITS_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_PBA_3
+#define PCIEMSIX_PBA_3__MSIX_PENDING_BITS__SHIFT                                                              0x0
+#define PCIEMSIX_PBA_3__MSIX_PENDING_BITS_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_PBA_4
+#define PCIEMSIX_PBA_4__MSIX_PENDING_BITS__SHIFT                                                              0x0
+#define PCIEMSIX_PBA_4__MSIX_PENDING_BITS_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_PBA_5
+#define PCIEMSIX_PBA_5__MSIX_PENDING_BITS__SHIFT                                                              0x0
+#define PCIEMSIX_PBA_5__MSIX_PENDING_BITS_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_PBA_6
+#define PCIEMSIX_PBA_6__MSIX_PENDING_BITS__SHIFT                                                              0x0
+#define PCIEMSIX_PBA_6__MSIX_PENDING_BITS_MASK                                                                0xFFFFFFFFL
+//PCIEMSIX_PBA_7
+#define PCIEMSIX_PBA_7__MSIX_PENDING_BITS__SHIFT                                                              0x0
+#define PCIEMSIX_PBA_7__MSIX_PENDING_BITS_MASK                                                                0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_bif_swus_SUMDEC
+//SUM_INDEX
+#define SUM_INDEX__SUM_INDEX__SHIFT                                                                           0x0
+#define SUM_INDEX__SUM_INDEX_MASK                                                                             0xFFFFFFFFL
+//SUM_DATA
+#define SUM_DATA__SUM_DATA__SHIFT                                                                             0x0
+#define SUM_DATA__SUM_DATA_MASK                                                                               0xFFFFFFFFL
+//SUM_INDEX_HI
+#define SUM_INDEX_HI__SUM_INDEX_HI__SHIFT                                                                     0x0
+#define SUM_INDEX_HI__SUM_INDEX_HI_MASK                                                                       0x000000FFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_strap_rcc_strap_internal
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0__SHIFT                                0x13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0__SHIFT                                   0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0__SHIFT                            0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0__SHIFT                             0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0__SHIFT                             0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0__SHIFT                                      0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0_MASK                                            0x00010000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0_MASK                                  0x00080000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0_MASK                                     0x00E00000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0_MASK                              0x01000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0_MASK                               0x0E000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0_MASK                               0x70000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0_MASK                                        0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0__SHIFT                                   0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0_MASK                                     0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0__SHIFT                                          0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0__SHIFT                                      0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0__SHIFT                                          0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0__SHIFT                                      0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0__SHIFT                                        0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ERR_REPORTING_DIS_DEV0__SHIFT                                  0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0__SHIFT                             0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0__SHIFT                                0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0__SHIFT                                    0x9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0__SHIFT                              0xc
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0__SHIFT                      0xd
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0__SHIFT                                    0xe
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0__SHIFT                                            0xf
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0__SHIFT                                    0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0__SHIFT                                    0x11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0__SHIFT                             0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0__SHIFT                                   0x17
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0__SHIFT                              0x1a
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0__SHIFT                                    0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0_MASK                                   0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0_MASK                                            0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0_MASK                                        0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0_MASK                                            0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0_MASK                                        0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0_MASK                                          0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ERR_REPORTING_DIS_DEV0_MASK                                    0x00000040L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0_MASK                               0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0_MASK                                  0x00000100L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0_MASK                                      0x00000E00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0_MASK                                0x00001000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0_MASK                        0x00002000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0_MASK                                      0x00004000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0_MASK                                              0x00008000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0_MASK                                      0x00010000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0_MASK                                      0x00020000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0_MASK                               0x00700000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0_MASK                                     0x03800000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0_MASK                                0x1C000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0_MASK                                      0xE0000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0__SHIFT                     0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0__SHIFT                                             0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0__SHIFT                                          0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0__SHIFT                                0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0__SHIFT                                          0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0__SHIFT                                  0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0__SHIFT                                     0x9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0xb
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0xe
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0__SHIFT                                         0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0__SHIFT                                      0x1b
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0__SHIFT                                       0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0__SHIFT                                         0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0_MASK                       0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0_MASK                                               0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0_MASK                                            0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0_MASK                                  0x00000038L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0_MASK                                            0x00000040L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0_MASK                                    0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0_MASK                                       0x00000600L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x00003800L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0_MASK  0x0003C000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x001C0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0_MASK  0x01E00000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0_MASK                                           0x06000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0_MASK                                        0x18000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0_MASK                                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0_MASK                                           0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0__SHIFT                              0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0__SHIFT                              0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0_MASK                                0xFF000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0__SHIFT                        0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0__SHIFT                                 0x11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0__SHIFT                                  0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0__SHIFT                                           0x13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0__SHIFT                                           0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0__SHIFT                                        0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0__SHIFT                                0x16
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0__SHIFT                           0x17
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0__SHIFT                        0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0__SHIFT                        0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0__SHIFT                     0x1a
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0__SHIFT                         0x1b
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0__SHIFT                          0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0__SHIFT                       0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0__SHIFT                                            0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0_MASK                          0x00010000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0_MASK                                   0x00020000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0_MASK                                    0x00040000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0_MASK                                             0x00080000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0_MASK                                             0x00100000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0_MASK                                          0x00200000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0_MASK                                  0x00400000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0_MASK                             0x00800000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0_MASK                          0x01000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0_MASK                          0x02000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0_MASK                       0x04000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0_MASK                           0x08000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0_MASK                            0x10000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0_MASK                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0_MASK                                              0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0__SHIFT                                         0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0__SHIFT                         0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0__SHIFT                                    0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                      0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                      0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                   0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0xc
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0__SHIFT                              0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0__SHIFT                            0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0__SHIFT                     0x13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0__SHIFT                                  0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0_MASK                                           0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0_MASK                           0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0_MASK                                      0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                        0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                        0x00000040L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                     0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x00000F00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0x0000F000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0_MASK                                0x00030000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0_MASK                              0x00040000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0_MASK                       0x00080000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0_MASK                                      0x00100000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0_MASK                                    0x00E00000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x0F000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0xF0000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0__SHIFT                                        0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0__SHIFT                                    0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0__SHIFT                                    0xc
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0__SHIFT                                          0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0__SHIFT                                          0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0_MASK                                          0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0_MASK                                      0x00000F00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0_MASK                                      0x0000F000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0_MASK                                            0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0_MASK                                            0x1F000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0_MASK                                            0xE0000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0__SHIFT                              0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0__SHIFT                              0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0_MASK                                0xFF000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0__SHIFT                                       0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0_MASK                                         0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0__SHIFT                           0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0__SHIFT                  0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0__SHIFT                         0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0__SHIFT                       0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0__SHIFT                                0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0_MASK                                0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0_MASK                             0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0_MASK                    0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0_MASK                           0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0_MASK                         0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0_MASK                                  0x0007FFC0L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0__SHIFT                             0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0__SHIFT                                 0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0__SHIFT                                      0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0__SHIFT                                         0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0__SHIFT                               0x1e
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0_MASK                               0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0_MASK                                   0x0FFF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0_MASK                                        0x10000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0_MASK                                           0x20000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0_MASK                                 0x40000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0__SHIFT                                0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0_MASK                                  0x00FFFFFFL
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0__SHIFT                          0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0__SHIFT     0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0__SHIFT                        0x9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0__SHIFT                               0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0_MASK                            0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0_MASK       0x00000100L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0_MASK                          0x000FFE00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0_MASK                                 0xFFF00000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0__SHIFT                               0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0__SHIFT                             0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0__SHIFT                                 0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0__SHIFT                          0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0__SHIFT                                         0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0_MASK                                 0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0_MASK                               0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0_MASK                                   0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0_MASK                            0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0_MASK                                           0x00000010L
+//RCC_DEV1_PORT_STRAP0
+//RCC_DEV1_PORT_STRAP1
+//RCC_DEV1_PORT_STRAP2
+//RCC_DEV1_PORT_STRAP3
+//RCC_DEV1_PORT_STRAP4
+//RCC_DEV1_PORT_STRAP5
+//RCC_DEV1_PORT_STRAP6
+//RCC_DEV1_PORT_STRAP7
+//RCC_DEV1_PORT_STRAP8
+//RCC_DEV1_PORT_STRAP9
+//RCC_DEV1_PORT_STRAP10
+//RCC_DEV1_PORT_STRAP11
+//RCC_DEV1_PORT_STRAP12
+//RCC_DEV1_PORT_STRAP13
+//RCC_DEV1_PORT_STRAP14
+//RCC_DEV2_PORT_STRAP0
+//RCC_DEV2_PORT_STRAP1
+//RCC_DEV2_PORT_STRAP2
+//RCC_DEV2_PORT_STRAP3
+//RCC_DEV2_PORT_STRAP4
+//RCC_DEV2_PORT_STRAP5
+//RCC_DEV2_PORT_STRAP6
+//RCC_DEV2_PORT_STRAP7
+//RCC_DEV2_PORT_STRAP8
+//RCC_DEV2_PORT_STRAP9
+//RCC_DEV2_PORT_STRAP10
+//RCC_DEV2_PORT_STRAP11
+//RCC_DEV2_PORT_STRAP12
+//RCC_DEV2_PORT_STRAP13
+//RCC_DEV2_PORT_STRAP14
+//RCC_STRAP1_RCC_BIF_STRAP0
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN4_DIS__SHIFT                                                      0x0
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT__SHIFT                              0x1
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_VGA_DIS_PIN__SHIFT                                                   0x2
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_MEM_AP_SIZE_PIN__SHIFT                                               0x3
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN__SHIFT                                               0x6
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PX_CAPABLE__SHIFT                                                    0x7
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3__SHIFT                                                 0x8
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN__SHIFT                                  0x9
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR__SHIFT                                         0xa
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN__SHIFT                                     0xb
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR__SHIFT                                              0xc
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR__SHIFT                                             0xd
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                     0xe
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                  0xf
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR__SHIFT                                              0x10
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR_DN__SHIFT                                           0x11
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_AUD_PIN__SHIFT                                                       0x12
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN3_DIS__SHIFT                                                      0x18
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4__SHIFT                                                 0x19
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_QUICKSIM_START__SHIFT                                                0x1a
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS__SHIFT                                   0x1c
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN__SHIFT                                      0x1d
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE__SHIFT                                                   0x1e
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN__SHIFT                                            0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN4_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT_MASK                                0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_VGA_DIS_PIN_MASK                                                     0x00000004L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_MEM_AP_SIZE_PIN_MASK                                                 0x00000038L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN_MASK                                                 0x00000040L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK                                                      0x00000080L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3_MASK                                                   0x00000100L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN_MASK                                    0x00000200L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR_MASK                                           0x00000400L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN_MASK                                       0x00000800L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR_MASK                                                0x00001000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR_MASK                                               0x00002000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                       0x00004000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                    0x00008000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR_MASK                                                0x00010000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR_DN_MASK                                             0x00020000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_AUD_PIN_MASK                                                         0x000C0000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN3_DIS_MASK                                                        0x01000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4_MASK                                                   0x02000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_QUICKSIM_START_MASK                                                  0x04000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING_MASK                                       0x08000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS_MASK                                     0x10000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN_MASK                                        0x20000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE_MASK                                                     0x40000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN_MASK                                              0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP1
+#define RCC_STRAP1_RCC_BIF_STRAP1__FUSESTRAP_VALID__SHIFT                                                     0x0
+#define RCC_STRAP1_RCC_BIF_STRAP1__ROMSTRAP_VALID__SHIFT                                                      0x1
+#define RCC_STRAP1_RCC_BIF_STRAP1__WRITE_DISABLE__SHIFT                                                       0x2
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN__SHIFT                                      0x3
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS__SHIFT                                     0x5
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE__SHIFT                                       0x6
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_READY__SHIFT                                               0x7
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN__SHIFT                                                  0x8
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN__SHIFT                                                 0x9
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE__SHIFT                                                  0xa
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE__SHIFT                                        0xc
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_HWREV_LSB2__SHIFT                                                    0xd
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWREV_LSB2__SHIFT                                                    0xf
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY__SHIFT                                             0x11
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS__SHIFT                                             0x12
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN__SHIFT                                                        0x13
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN__SHIFT                                                   0x14
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGIN_EN__SHIFT                                                     0x15
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN__SHIFT                                             0x16
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN__SHIFT                                     0x17
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_S5_REGS_ACCESS_DIS__SHIFT                                            0x18
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_S5_MMREG_WR_POSTED_EN__SHIFT                                         0x19
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE__SHIFT                                             0x1a
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN__SHIFT                                       0x1b
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN_EP__SHIFT                                                     0x1d
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN__SHIFT                                                         0x1e
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN_DN__SHIFT                                                      0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP1__FUSESTRAP_VALID_MASK                                                       0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP1__ROMSTRAP_VALID_MASK                                                        0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP1__WRITE_DISABLE_MASK                                                         0x00000004L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN_MASK                                        0x00000008L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS_MASK                                       0x00000020L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE_MASK                                         0x00000040L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_READY_MASK                                                 0x00000080L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN_MASK                                                    0x00000100L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN_MASK                                                   0x00000200L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE_MASK                                                    0x00000C00L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE_MASK                                          0x00001000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_HWREV_LSB2_MASK                                                      0x00006000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWREV_LSB2_MASK                                                      0x00018000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY_MASK                                               0x00020000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS_MASK                                               0x00040000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN_MASK                                                          0x00080000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN_MASK                                                     0x00100000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGIN_EN_MASK                                                       0x00200000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN_MASK                                               0x00400000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN_MASK                                       0x00800000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_S5_REGS_ACCESS_DIS_MASK                                              0x01000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_S5_MMREG_WR_POSTED_EN_MASK                                           0x02000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE_MASK                                               0x04000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN_MASK                                         0x18000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN_EP_MASK                                                       0x20000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN_MASK                                                           0x40000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN_DN_MASK                                                        0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP2
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS__SHIFT                                            0x3
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS__SHIFT                                            0x4
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN__SHIFT                                          0x7
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP1_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS_MASK                                              0x00000008L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS_MASK                                              0x00000010L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN_MASK                                            0x00000080L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP1_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP3
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP1_RCC_BIF_STRAP4
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER__SHIFT                                          0x0
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER__SHIFT                                           0x10
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER_MASK                                            0x0000FFFFL
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER_MASK                                             0xFFFF0000L
+//RCC_STRAP1_RCC_BIF_STRAP5
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+//RCC_STRAP1_RCC_BIF_STRAP6
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_GEN5_DIS__SHIFT                                                      0x0
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5__SHIFT                                                 0x1
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN__SHIFT                                                   0x2
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_GEN5_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5_MASK                                                   0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN_MASK                                                     0x00000004L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP1
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0__SHIFT                       0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0_MASK                                0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0_MASK                         0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP2
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0__SHIFT                                        0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0__SHIFT                                       0x6
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0__SHIFT                                   0x7
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0__SHIFT                                 0x9
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0__SHIFT                          0xe
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0__SHIFT                                          0xf
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0__SHIFT                                0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0__SHIFT                                          0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0__SHIFT                                           0x17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0__SHIFT                                   0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0__SHIFT                                        0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0__SHIFT                  0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0__SHIFT               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0__SHIFT                       0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0_MASK                                          0x00000001L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0_MASK                                         0x00000040L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0_MASK                                     0x00000080L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0_MASK                                   0x00003E00L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0_MASK                            0x00004000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0_MASK                                            0x00008000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0_MASK                                            0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0_MASK                                  0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0_MASK                                            0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0_MASK                                             0x00800000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0_MASK                                     0x07000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0_MASK                                       0x08000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0_MASK                                          0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0_MASK                    0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0_MASK                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0_MASK                         0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP3
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0__SHIFT                      0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0__SHIFT                              0x13
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0__SHIFT                                         0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0__SHIFT                                  0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0__SHIFT                                         0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0__SHIFT                        0x1a
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0__SHIFT                       0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0__SHIFT                                0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0__SHIFT                                       0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0__SHIFT                               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0__SHIFT                                          0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0_MASK                        0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0_MASK                                0x00080000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0_MASK                                           0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0_MASK                                    0x00E00000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0_MASK                                           0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0_MASK                          0x04000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0_MASK                         0x08000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0_MASK                                  0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0_MASK                                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0_MASK                                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0_MASK                                            0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP4
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0__SHIFT                                          0xa
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0__SHIFT                                       0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0__SHIFT                                     0x17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0__SHIFT                                   0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0_MASK                                   0x000003FFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0_MASK                                            0x00000400L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0_MASK                                         0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0_MASK                                       0x0F800000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0_MASK                                     0x70000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP5
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0__SHIFT                            0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0_MASK                                     0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0_MASK                              0x40000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0__SHIFT                                0x3
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0__SHIFT                                     0x4
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0__SHIFT                                      0x7
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_MEM_AP_SIZE_DEV0_F0__SHIFT                                     0x9
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0__SHIFT                                     0xd
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                           0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MEM_AP_SIZE_DEV0_F0__SHIFT                                  0x13
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0__SHIFT                                  0x17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VGA_DIS_DEV0_F0__SHIFT                                         0x1a
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0__SHIFT                                0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0__SHIFT                           0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0_MASK                                0x00000007L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0_MASK                                  0x00000008L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0_MASK                                       0x00000070L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0_MASK                                        0x00000080L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_MEM_AP_SIZE_DEV0_F0_MASK                                       0x00001E00L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0_MASK                                       0x0000E000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0_MASK                             0x00070000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MEM_AP_SIZE_DEV0_F0_MASK                                    0x00780000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0_MASK                                    0x03800000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VGA_DIS_DEV0_F0_MASK                                           0x04000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0_MASK                                  0x38000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0_MASK                             0xC0000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP9
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0__SHIFT                           0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0__SHIFT                               0x12
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0__SHIFT                        0x13
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0__SHIFT                                    0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0__SHIFT                                 0x16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0__SHIFT                                 0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0_MASK                             0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0_MASK                                 0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0_MASK                          0x00080000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0_MASK                                      0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0_MASK                                   0x00C00000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0_MASK                                   0x0F000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP13
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0__SHIFT                                 0x8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0__SHIFT                                0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0_MASK                                   0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0_MASK                                   0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0_MASK                                  0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0_MASK                                  0xFF000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0__SHIFT                                      0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0_MASK                                        0x0000FFFFL
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0__SHIFT                                  0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0__SHIFT                     0x19
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0__SHIFT                       0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0_MASK                                   0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0_MASK                                    0x00FFF000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0_MASK                                        0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0_MASK                       0x3E000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0_MASK                         0x40000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0__SHIFT                               0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0_MASK                                     0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0_MASK                                 0x00FFF000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0__SHIFT                                   0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0__SHIFT                                0xd
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0_MASK                                0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0_MASK                                     0x00001000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0_MASK                                  0x01FFE000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0__SHIFT                            0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0_MASK                              0x00000FFFL
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP26
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP26__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0__SHIFT                             0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP26__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0_MASK                               0x00000FFFL
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1__SHIFT                                    0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1__SHIFT                                         0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1__SHIFT                           0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1__SHIFT                                      0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1__SHIFT                                      0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1_MASK                                      0x000F0000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1_MASK                                      0x00F00000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1_MASK                                           0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1_MASK                             0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1_MASK                                        0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1_MASK                                        0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP2
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1__SHIFT                                   0x7
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1__SHIFT                                 0x9
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1__SHIFT                          0xe
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1__SHIFT                                0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1__SHIFT                                          0x15
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_VC_EN_DEV0_F1__SHIFT                                           0x17
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1__SHIFT                                   0x18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1__SHIFT                                        0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1__SHIFT                  0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1__SHIFT               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1__SHIFT                       0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1_MASK                                     0x00000080L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1_MASK                                   0x00003E00L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1_MASK                            0x00004000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1_MASK                                            0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1_MASK                                  0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1_MASK                                            0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_VC_EN_DEV0_F1_MASK                                             0x00800000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1_MASK                                     0x07000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1_MASK                                          0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1_MASK                    0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1_MASK                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1_MASK                         0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP3
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1__SHIFT                      0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1__SHIFT                              0x13
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1__SHIFT                                         0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1__SHIFT                                         0x18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1__SHIFT                        0x1a
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1__SHIFT                       0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1__SHIFT                                       0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1__SHIFT                               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1__SHIFT                                          0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1_MASK                        0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1_MASK                                0x00080000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1_MASK                                           0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1_MASK                                           0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1_MASK                          0x04000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1_MASK                         0x08000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1_MASK                                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1_MASK                                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1_MASK                                            0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP4
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1__SHIFT                                 0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1__SHIFT                                       0x15
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1__SHIFT                                     0x17
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1__SHIFT                                   0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1__SHIFT                                  0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1_MASK                                   0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1_MASK                                         0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1_MASK                                       0x0F800000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1_MASK                                     0x70000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1_MASK                                    0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP5
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1__SHIFT                            0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1_MASK                                     0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1_MASK                                       0x38000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1_MASK                              0x40000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP6
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1_MASK                                          0x00000001L
+
+
+// addressBlock: aid_nbio_nbif0_bif_rst_bif_rst_regblk
+//HARD_RST_CTRL
+#define HARD_RST_CTRL__DSPT_CFG_RST_EN__SHIFT                                                                 0x0
+#define HARD_RST_CTRL__DSPT_CFG_STICKY_RST_EN__SHIFT                                                          0x1
+#define HARD_RST_CTRL__DSPT_PRV_RST_EN__SHIFT                                                                 0x2
+#define HARD_RST_CTRL__DSPT_PRV_STICKY_RST_EN__SHIFT                                                          0x3
+#define HARD_RST_CTRL__EP_CFG_RST_EN__SHIFT                                                                   0x4
+#define HARD_RST_CTRL__EP_CFG_STICKY_RST_EN__SHIFT                                                            0x5
+#define HARD_RST_CTRL__EP_PRV_RST_EN__SHIFT                                                                   0x6
+#define HARD_RST_CTRL__EP_PRV_STICKY_RST_EN__SHIFT                                                            0x7
+#define HARD_RST_CTRL__SDP_PORT_RESET_EN__SHIFT                                                               0x9
+#define HARD_RST_CTRL__SION_AON_RESET_EN__SHIFT                                                               0xa
+#define HARD_RST_CTRL__STRAP_RST_EN__SHIFT                                                                    0x17
+#define HARD_RST_CTRL__CORE_STICKY_RST_EN__SHIFT                                                              0x1d
+#define HARD_RST_CTRL__RELOAD_STRAP_EN__SHIFT                                                                 0x1e
+#define HARD_RST_CTRL__CORE_RST_EN__SHIFT                                                                     0x1f
+#define HARD_RST_CTRL__DSPT_CFG_RST_EN_MASK                                                                   0x00000001L
+#define HARD_RST_CTRL__DSPT_CFG_STICKY_RST_EN_MASK                                                            0x00000002L
+#define HARD_RST_CTRL__DSPT_PRV_RST_EN_MASK                                                                   0x00000004L
+#define HARD_RST_CTRL__DSPT_PRV_STICKY_RST_EN_MASK                                                            0x00000008L
+#define HARD_RST_CTRL__EP_CFG_RST_EN_MASK                                                                     0x00000010L
+#define HARD_RST_CTRL__EP_CFG_STICKY_RST_EN_MASK                                                              0x00000020L
+#define HARD_RST_CTRL__EP_PRV_RST_EN_MASK                                                                     0x00000040L
+#define HARD_RST_CTRL__EP_PRV_STICKY_RST_EN_MASK                                                              0x00000080L
+#define HARD_RST_CTRL__SDP_PORT_RESET_EN_MASK                                                                 0x00000200L
+#define HARD_RST_CTRL__SION_AON_RESET_EN_MASK                                                                 0x00000400L
+#define HARD_RST_CTRL__STRAP_RST_EN_MASK                                                                      0x00800000L
+#define HARD_RST_CTRL__CORE_STICKY_RST_EN_MASK                                                                0x20000000L
+#define HARD_RST_CTRL__RELOAD_STRAP_EN_MASK                                                                   0x40000000L
+#define HARD_RST_CTRL__CORE_RST_EN_MASK                                                                       0x80000000L
+//SELF_SOFT_RST
+#define SELF_SOFT_RST__DSPT0_CFG_RST__SHIFT                                                                   0x0
+#define SELF_SOFT_RST__DSPT0_CFG_STICKY_RST__SHIFT                                                            0x1
+#define SELF_SOFT_RST__DSPT0_PRV_RST__SHIFT                                                                   0x2
+#define SELF_SOFT_RST__DSPT0_PRV_STICKY_RST__SHIFT                                                            0x3
+#define SELF_SOFT_RST__EP0_CFG_RST__SHIFT                                                                     0x4
+#define SELF_SOFT_RST__EP0_CFG_STICKY_RST__SHIFT                                                              0x5
+#define SELF_SOFT_RST__EP0_PRV_RST__SHIFT                                                                     0x6
+#define SELF_SOFT_RST__EP0_PRV_STICKY_RST__SHIFT                                                              0x7
+#define SELF_SOFT_RST__HRPU_SDP_PORT_RST__SHIFT                                                               0x18
+#define SELF_SOFT_RST__GSID_SDP_PORT_RST__SHIFT                                                               0x19
+#define SELF_SOFT_RST__GMIU_SDP_PORT_RST__SHIFT                                                               0x1a
+#define SELF_SOFT_RST__GMID_SDP_PORT_RST__SHIFT                                                               0x1b
+#define SELF_SOFT_RST__CORE_STICKY_RST__SHIFT                                                                 0x1d
+#define SELF_SOFT_RST__RELOAD_STRAP__SHIFT                                                                    0x1e
+#define SELF_SOFT_RST__CORE_RST__SHIFT                                                                        0x1f
+#define SELF_SOFT_RST__DSPT0_CFG_RST_MASK                                                                     0x00000001L
+#define SELF_SOFT_RST__DSPT0_CFG_STICKY_RST_MASK                                                              0x00000002L
+#define SELF_SOFT_RST__DSPT0_PRV_RST_MASK                                                                     0x00000004L
+#define SELF_SOFT_RST__DSPT0_PRV_STICKY_RST_MASK                                                              0x00000008L
+#define SELF_SOFT_RST__EP0_CFG_RST_MASK                                                                       0x00000010L
+#define SELF_SOFT_RST__EP0_CFG_STICKY_RST_MASK                                                                0x00000020L
+#define SELF_SOFT_RST__EP0_PRV_RST_MASK                                                                       0x00000040L
+#define SELF_SOFT_RST__EP0_PRV_STICKY_RST_MASK                                                                0x00000080L
+#define SELF_SOFT_RST__HRPU_SDP_PORT_RST_MASK                                                                 0x01000000L
+#define SELF_SOFT_RST__GSID_SDP_PORT_RST_MASK                                                                 0x02000000L
+#define SELF_SOFT_RST__GMIU_SDP_PORT_RST_MASK                                                                 0x04000000L
+#define SELF_SOFT_RST__GMID_SDP_PORT_RST_MASK                                                                 0x08000000L
+#define SELF_SOFT_RST__CORE_STICKY_RST_MASK                                                                   0x20000000L
+#define SELF_SOFT_RST__RELOAD_STRAP_MASK                                                                      0x40000000L
+#define SELF_SOFT_RST__CORE_RST_MASK                                                                          0x80000000L
+//BIF_GFX_DRV_VPU_RST
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_RST__SHIFT                                                      0x0
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_FLR_EXC_RST__SHIFT                                              0x1
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_STICKY_RST__SHIFT                                               0x2
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_RST__SHIFT                                                      0x3
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_STICKY_RST__SHIFT                                               0x4
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_RST__SHIFT                                                      0x5
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_STICKY_RST__SHIFT                                               0x6
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_PRV_RST__SHIFT                                                      0x7
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_RST_MASK                                                        0x00000001L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_FLR_EXC_RST_MASK                                                0x00000002L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_STICKY_RST_MASK                                                 0x00000004L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_RST_MASK                                                        0x00000008L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_STICKY_RST_MASK                                                 0x00000010L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_RST_MASK                                                        0x00000020L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_STICKY_RST_MASK                                                 0x00000040L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_PRV_RST_MASK                                                        0x00000080L
+//BIF_RST_MISC_CTRL
+#define BIF_RST_MISC_CTRL__ERRSTATUS_KEPT_IN_PERSTB__SHIFT                                                    0x0
+#define BIF_RST_MISC_CTRL__DRV_RST_MODE__SHIFT                                                                0x2
+#define BIF_RST_MISC_CTRL__DRV_RST_CFG_MASK__SHIFT                                                            0x4
+#define BIF_RST_MISC_CTRL__DRV_RST_BITS_AUTO_CLEAR__SHIFT                                                     0x5
+#define BIF_RST_MISC_CTRL__FLR_RST_BIT_AUTO_CLEAR__SHIFT                                                      0x6
+#define BIF_RST_MISC_CTRL__STRAP_EP_LNK_RST_IOV_EN__SHIFT                                                     0x8
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_MODE__SHIFT                                                          0x9
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_TIMEOUT__SHIFT                                                       0xa
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER_SEL__SHIFT                                                           0xd
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER2_SEL__SHIFT                                                          0xf
+#define BIF_RST_MISC_CTRL__SRIOV_SAVE_VFS_ON_VFENABLE_CLR__SHIFT                                              0x11
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_DIS__SHIFT                                                       0x17
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_RSPSTS__SHIFT                                                    0x18
+#define BIF_RST_MISC_CTRL__ERRSTATUS_KEPT_IN_PERSTB_MASK                                                      0x00000001L
+#define BIF_RST_MISC_CTRL__DRV_RST_MODE_MASK                                                                  0x0000000CL
+#define BIF_RST_MISC_CTRL__DRV_RST_CFG_MASK_MASK                                                              0x00000010L
+#define BIF_RST_MISC_CTRL__DRV_RST_BITS_AUTO_CLEAR_MASK                                                       0x00000020L
+#define BIF_RST_MISC_CTRL__FLR_RST_BIT_AUTO_CLEAR_MASK                                                        0x00000040L
+#define BIF_RST_MISC_CTRL__STRAP_EP_LNK_RST_IOV_EN_MASK                                                       0x00000100L
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_MODE_MASK                                                            0x00000200L
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_TIMEOUT_MASK                                                         0x00001C00L
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER_SEL_MASK                                                             0x00006000L
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER2_SEL_MASK                                                            0x00018000L
+#define BIF_RST_MISC_CTRL__SRIOV_SAVE_VFS_ON_VFENABLE_CLR_MASK                                                0x000E0000L
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_DIS_MASK                                                         0x00800000L
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_RSPSTS_MASK                                                      0x03000000L
+//BIF_RST_MISC_CTRL2
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_PROTECT__SHIFT                                                       0x0
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_PROTECT__SHIFT                                                       0x1
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_PROTECT__SHIFT                                                      0x2
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT__SHIFT                                                            0xf
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_TRANS_IDLE__SHIFT                                                    0x10
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_TRANS_IDLE__SHIFT                                                    0x11
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_TRANS_IDLE__SHIFT                                                   0x12
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT_DIS__SHIFT                                                        0x1e
+#define BIF_RST_MISC_CTRL2__ALL_RST_TRANS_IDLE__SHIFT                                                         0x1f
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_PROTECT_MASK                                                         0x00000001L
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_PROTECT_MASK                                                         0x00000002L
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_PROTECT_MASK                                                        0x00000004L
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT_MASK                                                              0x00008000L
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_TRANS_IDLE_MASK                                                      0x00010000L
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_TRANS_IDLE_MASK                                                      0x00020000L
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_TRANS_IDLE_MASK                                                     0x00040000L
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT_DIS_MASK                                                          0x40000000L
+#define BIF_RST_MISC_CTRL2__ALL_RST_TRANS_IDLE_MASK                                                           0x80000000L
+//BIF_RST_MISC_CTRL3
+#define BIF_RST_MISC_CTRL3__TIMER_SCALE__SHIFT                                                                0x0
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_TIMEOUT__SHIFT                                                        0x4
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_MODE__SHIFT                                                           0x6
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_HARD__SHIFT                                                    0x7
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SOFT__SHIFT                                                    0xa
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SELF__SHIFT                                                    0xd
+#define BIF_RST_MISC_CTRL3__TIMER_SCALE_MASK                                                                  0x0000000FL
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_TIMEOUT_MASK                                                          0x00000030L
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_MODE_MASK                                                             0x00000040L
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_HARD_MASK                                                      0x00000380L
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SOFT_MASK                                                      0x00001C00L
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SELF_MASK                                                      0x0000E000L
+//DEV0_PF0_FLR_RST_CTRL
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF0_FLR_RST_CTRL__VF_CFG_EN__SHIFT                                                               0x5
+#define DEV0_PF0_FLR_RST_CTRL__VF_CFG_STICKY_EN__SHIFT                                                        0x6
+#define DEV0_PF0_FLR_RST_CTRL__VF_PRV_EN__SHIFT                                                               0x7
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_EN__SHIFT                                                          0x8
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_FLR_EXC_EN__SHIFT                                                  0x9
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_STICKY_EN__SHIFT                                                   0xa
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PRV_EN__SHIFT                                                          0xb
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PRV_STICKY_EN__SHIFT                                                   0xc
+#define DEV0_PF0_FLR_RST_CTRL__VF_VF_CFG_EN__SHIFT                                                            0xd
+#define DEV0_PF0_FLR_RST_CTRL__VF_VF_CFG_STICKY_EN__SHIFT                                                     0xe
+#define DEV0_PF0_FLR_RST_CTRL__VF_VF_PRV_EN__SHIFT                                                            0xf
+#define DEV0_PF0_FLR_RST_CTRL__FLR_TWICE_EN__SHIFT                                                            0x10
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF0_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF0_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PFCOPY_PRV_EN__SHIFT                                                   0x1f
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF0_FLR_RST_CTRL__VF_CFG_EN_MASK                                                                 0x00000020L
+#define DEV0_PF0_FLR_RST_CTRL__VF_CFG_STICKY_EN_MASK                                                          0x00000040L
+#define DEV0_PF0_FLR_RST_CTRL__VF_PRV_EN_MASK                                                                 0x00000080L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_EN_MASK                                                            0x00000100L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_FLR_EXC_EN_MASK                                                    0x00000200L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_STICKY_EN_MASK                                                     0x00000400L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PRV_EN_MASK                                                            0x00000800L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PRV_STICKY_EN_MASK                                                     0x00001000L
+#define DEV0_PF0_FLR_RST_CTRL__VF_VF_CFG_EN_MASK                                                              0x00002000L
+#define DEV0_PF0_FLR_RST_CTRL__VF_VF_CFG_STICKY_EN_MASK                                                       0x00004000L
+#define DEV0_PF0_FLR_RST_CTRL__VF_VF_PRV_EN_MASK                                                              0x00008000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_TWICE_EN_MASK                                                              0x00010000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PFCOPY_PRV_EN_MASK                                                     0x80000000L
+//DEV0_PF1_FLR_RST_CTRL
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF1_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF1_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//BIF_INST_RESET_INTR_STS
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_INTR_STS__SHIFT                                               0x0
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_CFG_ONLY_INTR_STS__SHIFT                                      0x1
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M0_INTR_STS__SHIFT                                                 0x2
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M1_INTR_STS__SHIFT                                                 0x3
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M2_INTR_STS__SHIFT                                                 0x4
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_INTR_STS_MASK                                                 0x00000001L
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_CFG_ONLY_INTR_STS_MASK                                        0x00000002L
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M0_INTR_STS_MASK                                                   0x00000004L
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M1_INTR_STS_MASK                                                   0x00000008L
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M2_INTR_STS_MASK                                                   0x00000010L
+//BIF_PF_FLR_INTR_STS
+#define BIF_PF_FLR_INTR_STS__DEV0_PF0_FLR_INTR_STS__SHIFT                                                     0x0
+#define BIF_PF_FLR_INTR_STS__DEV0_PF1_FLR_INTR_STS__SHIFT                                                     0x1
+#define BIF_PF_FLR_INTR_STS__DEV0_PF0_FLR_INTR_STS_MASK                                                       0x00000001L
+#define BIF_PF_FLR_INTR_STS__DEV0_PF1_FLR_INTR_STS_MASK                                                       0x00000002L
+//BIF_D3HOTD0_INTR_STS
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF0_D3HOTD0_INTR_STS__SHIFT                                                0x0
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF1_D3HOTD0_INTR_STS__SHIFT                                                0x1
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF0_D3HOTD0_INTR_STS_MASK                                                  0x00000001L
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF1_D3HOTD0_INTR_STS_MASK                                                  0x00000002L
+//BIF_POWER_INTR_STS
+#define BIF_POWER_INTR_STS__DEV0_PME_TURN_OFF_INTR_STS__SHIFT                                                 0x0
+#define BIF_POWER_INTR_STS__PORT0_DSTATE_INTR_STS__SHIFT                                                      0x10
+#define BIF_POWER_INTR_STS__DEV0_PME_TURN_OFF_INTR_STS_MASK                                                   0x00000001L
+#define BIF_POWER_INTR_STS__PORT0_DSTATE_INTR_STS_MASK                                                        0x00010000L
+//BIF_PF_DSTATE_INTR_STS
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF0_DSTATE_INTR_STS__SHIFT                                               0x0
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF1_DSTATE_INTR_STS__SHIFT                                               0x1
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF2_DSTATE_INTR_STS__SHIFT                                               0x2
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF3_DSTATE_INTR_STS__SHIFT                                               0x3
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF4_DSTATE_INTR_STS__SHIFT                                               0x4
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF5_DSTATE_INTR_STS__SHIFT                                               0x5
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF6_DSTATE_INTR_STS__SHIFT                                               0x6
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF7_DSTATE_INTR_STS__SHIFT                                               0x7
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF0_DSTATE_INTR_STS_MASK                                                 0x00000001L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF1_DSTATE_INTR_STS_MASK                                                 0x00000002L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF2_DSTATE_INTR_STS_MASK                                                 0x00000004L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF3_DSTATE_INTR_STS_MASK                                                 0x00000008L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF4_DSTATE_INTR_STS_MASK                                                 0x00000010L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF5_DSTATE_INTR_STS_MASK                                                 0x00000020L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF6_DSTATE_INTR_STS_MASK                                                 0x00000040L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF7_DSTATE_INTR_STS_MASK                                                 0x00000080L
+//SELF_SOFT_RST_2
+#define SELF_SOFT_RST_2__DSPT3_CFG_RST__SHIFT                                                                 0x0
+#define SELF_SOFT_RST_2__DSPT3_CFG_STICKY_RST__SHIFT                                                          0x1
+#define SELF_SOFT_RST_2__DSPT3_PRV_RST__SHIFT                                                                 0x2
+#define SELF_SOFT_RST_2__DSPT3_PRV_STICKY_RST__SHIFT                                                          0x3
+#define SELF_SOFT_RST_2__EP3_CFG_RST__SHIFT                                                                   0x4
+#define SELF_SOFT_RST_2__EP3_CFG_STICKY_RST__SHIFT                                                            0x5
+#define SELF_SOFT_RST_2__EP3_PRV_RST__SHIFT                                                                   0x6
+#define SELF_SOFT_RST_2__EP3_PRV_STICKY_RST__SHIFT                                                            0x7
+#define SELF_SOFT_RST_2__GMISP0_SDP_PORT_RST__SHIFT                                                           0x18
+#define SELF_SOFT_RST_2__STRAP_RST__SHIFT                                                                     0x19
+#define SELF_SOFT_RST_2__DSPT3_CFG_RST_MASK                                                                   0x00000001L
+#define SELF_SOFT_RST_2__DSPT3_CFG_STICKY_RST_MASK                                                            0x00000002L
+#define SELF_SOFT_RST_2__DSPT3_PRV_RST_MASK                                                                   0x00000004L
+#define SELF_SOFT_RST_2__DSPT3_PRV_STICKY_RST_MASK                                                            0x00000008L
+#define SELF_SOFT_RST_2__EP3_CFG_RST_MASK                                                                     0x00000010L
+#define SELF_SOFT_RST_2__EP3_CFG_STICKY_RST_MASK                                                              0x00000020L
+#define SELF_SOFT_RST_2__EP3_PRV_RST_MASK                                                                     0x00000040L
+#define SELF_SOFT_RST_2__EP3_PRV_STICKY_RST_MASK                                                              0x00000080L
+#define SELF_SOFT_RST_2__GMISP0_SDP_PORT_RST_MASK                                                             0x01000000L
+#define SELF_SOFT_RST_2__STRAP_RST_MASK                                                                       0x02000000L
+//BIF_INST_RESET_INTR_MASK
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_INTR_MASK__SHIFT                                             0x0
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_CFG_ONLY_INTR_MASK__SHIFT                                    0x1
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M0_INTR_MASK__SHIFT                                               0x2
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M1_INTR_MASK__SHIFT                                               0x3
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M2_INTR_MASK__SHIFT                                               0x4
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_INTR_MASK_MASK                                               0x00000001L
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_CFG_ONLY_INTR_MASK_MASK                                      0x00000002L
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M0_INTR_MASK_MASK                                                 0x00000004L
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M1_INTR_MASK_MASK                                                 0x00000008L
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M2_INTR_MASK_MASK                                                 0x00000010L
+//BIF_PF_FLR_INTR_MASK
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF0_FLR_INTR_MASK__SHIFT                                                   0x0
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF1_FLR_INTR_MASK__SHIFT                                                   0x1
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF0_FLR_INTR_MASK_MASK                                                     0x00000001L
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF1_FLR_INTR_MASK_MASK                                                     0x00000002L
+//BIF_D3HOTD0_INTR_MASK
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF0_D3HOTD0_INTR_MASK__SHIFT                                              0x0
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF1_D3HOTD0_INTR_MASK__SHIFT                                              0x1
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF0_D3HOTD0_INTR_MASK_MASK                                                0x00000001L
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF1_D3HOTD0_INTR_MASK_MASK                                                0x00000002L
+//BIF_POWER_INTR_MASK
+#define BIF_POWER_INTR_MASK__DEV0_PME_TURN_OFF_INTR_MASK__SHIFT                                               0x0
+#define BIF_POWER_INTR_MASK__PORT0_DSTATE_INTR_MASK__SHIFT                                                    0x10
+#define BIF_POWER_INTR_MASK__DEV0_PME_TURN_OFF_INTR_MASK_MASK                                                 0x00000001L
+#define BIF_POWER_INTR_MASK__PORT0_DSTATE_INTR_MASK_MASK                                                      0x00010000L
+//BIF_PF_DSTATE_INTR_MASK
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF0_DSTATE_INTR_MASK__SHIFT                                             0x0
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF1_DSTATE_INTR_MASK__SHIFT                                             0x1
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF2_DSTATE_INTR_MASK__SHIFT                                             0x2
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF3_DSTATE_INTR_MASK__SHIFT                                             0x3
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF4_DSTATE_INTR_MASK__SHIFT                                             0x4
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF5_DSTATE_INTR_MASK__SHIFT                                             0x5
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF6_DSTATE_INTR_MASK__SHIFT                                             0x6
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF7_DSTATE_INTR_MASK__SHIFT                                             0x7
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF0_DSTATE_INTR_MASK_MASK                                               0x00000001L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF1_DSTATE_INTR_MASK_MASK                                               0x00000002L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF2_DSTATE_INTR_MASK_MASK                                               0x00000004L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF3_DSTATE_INTR_MASK_MASK                                               0x00000008L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF4_DSTATE_INTR_MASK_MASK                                               0x00000010L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF5_DSTATE_INTR_MASK_MASK                                               0x00000020L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF6_DSTATE_INTR_MASK_MASK                                               0x00000040L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF7_DSTATE_INTR_MASK_MASK                                               0x00000080L
+//BIF_PF_FLR_RST
+#define BIF_PF_FLR_RST__DEV0_PF0_FLR_RST__SHIFT                                                               0x0
+#define BIF_PF_FLR_RST__DEV0_PF1_FLR_RST__SHIFT                                                               0x1
+#define BIF_PF_FLR_RST__DEV0_PF0_FLR_RST_MASK                                                                 0x00000001L
+#define BIF_PF_FLR_RST__DEV0_PF1_FLR_RST_MASK                                                                 0x00000002L
+//BIF_DEV0_PF0_DSTATE_VALUE
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV0_PF1_DSTATE_VALUE
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//DEV0_PF0_D3HOTD0_RST_CTRL
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV0_PF1_D3HOTD0_RST_CTRL
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//BIF_PORT0_DSTATE_VALUE
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_TGT_VALUE__SHIFT                                                 0x0
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_ACK_VALUE__SHIFT                                                 0x10
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_TGT_VALUE_MASK                                                   0x00000003L
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_ACK_VALUE_MASK                                                   0x00030000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_misc_bif_misc_regblk
+//REGS_ROM_OFFSET_CTRL
+#define REGS_ROM_OFFSET_CTRL__ROM_OFFSET__SHIFT                                                               0x0
+#define REGS_ROM_OFFSET_CTRL__ROM_OFFSET_MASK                                                                 0x7FL
+//NBIF_STRAP_BIOS_CNTL
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_BIOS_EN__SHIFT                                                       0x0
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_PCIE_ID_BIOS_EN__SHIFT                                               0x1
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_BIOS_EN_MASK                                                         0x00000001L
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_PCIE_ID_BIOS_EN_MASK                                                 0x00000002L
+//DOORBELL0_CTRL_ENTRY_0
+#define DOORBELL0_CTRL_ENTRY_0__BIF_DOORBELL0_RANGE_OFFSET_ENTRY__SHIFT                                       0x0
+#define DOORBELL0_CTRL_ENTRY_0__BIF_DOORBELL0_RANGE_SIZE_ENTRY__SHIFT                                         0xa
+#define DOORBELL0_CTRL_ENTRY_0__DOORBELL0_FENCE_ENABLE_ENTRY__SHIFT                                           0x10
+#define DOORBELL0_CTRL_ENTRY_0__BIF_DOORBELL0_RANGE_OFFSET_ENTRY_MASK                                         0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_0__BIF_DOORBELL0_RANGE_SIZE_ENTRY_MASK                                           0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_0__DOORBELL0_FENCE_ENABLE_ENTRY_MASK                                             0x001F0000L
+//DOORBELL0_CTRL_ENTRY_1
+#define DOORBELL0_CTRL_ENTRY_1__BIF_DOORBELL1_RANGE_OFFSET_ENTRY__SHIFT                                       0x0
+#define DOORBELL0_CTRL_ENTRY_1__BIF_DOORBELL1_RANGE_SIZE_ENTRY__SHIFT                                         0xa
+#define DOORBELL0_CTRL_ENTRY_1__DOORBELL1_FENCE_ENABLE_ENTRY__SHIFT                                           0x10
+#define DOORBELL0_CTRL_ENTRY_1__BIF_DOORBELL1_RANGE_OFFSET_ENTRY_MASK                                         0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_1__BIF_DOORBELL1_RANGE_SIZE_ENTRY_MASK                                           0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_1__DOORBELL1_FENCE_ENABLE_ENTRY_MASK                                             0x001F0000L
+//DOORBELL0_CTRL_ENTRY_2
+#define DOORBELL0_CTRL_ENTRY_2__BIF_DOORBELL2_RANGE_OFFSET_ENTRY__SHIFT                                       0x0
+#define DOORBELL0_CTRL_ENTRY_2__BIF_DOORBELL2_RANGE_SIZE_ENTRY__SHIFT                                         0xa
+#define DOORBELL0_CTRL_ENTRY_2__DOORBELL2_FENCE_ENABLE_ENTRY__SHIFT                                           0x10
+#define DOORBELL0_CTRL_ENTRY_2__BIF_DOORBELL2_RANGE_OFFSET_ENTRY_MASK                                         0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_2__BIF_DOORBELL2_RANGE_SIZE_ENTRY_MASK                                           0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_2__DOORBELL2_FENCE_ENABLE_ENTRY_MASK                                             0x001F0000L
+//DOORBELL0_CTRL_ENTRY_3
+#define DOORBELL0_CTRL_ENTRY_3__BIF_DOORBELL3_RANGE_OFFSET_ENTRY__SHIFT                                       0x0
+#define DOORBELL0_CTRL_ENTRY_3__BIF_DOORBELL3_RANGE_SIZE_ENTRY__SHIFT                                         0xa
+#define DOORBELL0_CTRL_ENTRY_3__DOORBELL3_FENCE_ENABLE_ENTRY__SHIFT                                           0x10
+#define DOORBELL0_CTRL_ENTRY_3__BIF_DOORBELL3_RANGE_OFFSET_ENTRY_MASK                                         0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_3__BIF_DOORBELL3_RANGE_SIZE_ENTRY_MASK                                           0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_3__DOORBELL3_FENCE_ENABLE_ENTRY_MASK                                             0x001F0000L
+//DOORBELL0_CTRL_ENTRY_4
+#define DOORBELL0_CTRL_ENTRY_4__BIF_DOORBELL4_RANGE_OFFSET_ENTRY__SHIFT                                       0x0
+#define DOORBELL0_CTRL_ENTRY_4__BIF_DOORBELL4_RANGE_SIZE_ENTRY__SHIFT                                         0xa
+#define DOORBELL0_CTRL_ENTRY_4__DOORBELL4_FENCE_ENABLE_ENTRY__SHIFT                                           0x10
+#define DOORBELL0_CTRL_ENTRY_4__BIF_DOORBELL4_RANGE_OFFSET_ENTRY_MASK                                         0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_4__BIF_DOORBELL4_RANGE_SIZE_ENTRY_MASK                                           0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_4__DOORBELL4_FENCE_ENABLE_ENTRY_MASK                                             0x001F0000L
+//DOORBELL0_CTRL_ENTRY_5
+#define DOORBELL0_CTRL_ENTRY_5__BIF_DOORBELL5_RANGE_OFFSET_ENTRY__SHIFT                                       0x0
+#define DOORBELL0_CTRL_ENTRY_5__BIF_DOORBELL5_RANGE_SIZE_ENTRY__SHIFT                                         0xa
+#define DOORBELL0_CTRL_ENTRY_5__DOORBELL5_FENCE_ENABLE_ENTRY__SHIFT                                           0x10
+#define DOORBELL0_CTRL_ENTRY_5__BIF_DOORBELL5_RANGE_OFFSET_ENTRY_MASK                                         0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_5__BIF_DOORBELL5_RANGE_SIZE_ENTRY_MASK                                           0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_5__DOORBELL5_FENCE_ENABLE_ENTRY_MASK                                             0x001F0000L
+//DOORBELL0_CTRL_ENTRY_6
+#define DOORBELL0_CTRL_ENTRY_6__BIF_DOORBELL6_RANGE_OFFSET_ENTRY__SHIFT                                       0x0
+#define DOORBELL0_CTRL_ENTRY_6__BIF_DOORBELL6_RANGE_SIZE_ENTRY__SHIFT                                         0xa
+#define DOORBELL0_CTRL_ENTRY_6__DOORBELL6_FENCE_ENABLE_ENTRY__SHIFT                                           0x10
+#define DOORBELL0_CTRL_ENTRY_6__BIF_DOORBELL6_RANGE_OFFSET_ENTRY_MASK                                         0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_6__BIF_DOORBELL6_RANGE_SIZE_ENTRY_MASK                                           0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_6__DOORBELL6_FENCE_ENABLE_ENTRY_MASK                                             0x001F0000L
+//DOORBELL0_CTRL_ENTRY_7
+#define DOORBELL0_CTRL_ENTRY_7__BIF_DOORBELL7_RANGE_OFFSET_ENTRY__SHIFT                                       0x0
+#define DOORBELL0_CTRL_ENTRY_7__BIF_DOORBELL7_RANGE_SIZE_ENTRY__SHIFT                                         0xa
+#define DOORBELL0_CTRL_ENTRY_7__DOORBELL7_FENCE_ENABLE_ENTRY__SHIFT                                           0x10
+#define DOORBELL0_CTRL_ENTRY_7__BIF_DOORBELL7_RANGE_OFFSET_ENTRY_MASK                                         0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_7__BIF_DOORBELL7_RANGE_SIZE_ENTRY_MASK                                           0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_7__DOORBELL7_FENCE_ENABLE_ENTRY_MASK                                             0x001F0000L
+//DOORBELL0_CTRL_ENTRY_8
+#define DOORBELL0_CTRL_ENTRY_8__BIF_DOORBELL8_RANGE_OFFSET_ENTRY__SHIFT                                       0x0
+#define DOORBELL0_CTRL_ENTRY_8__BIF_DOORBELL8_RANGE_SIZE_ENTRY__SHIFT                                         0xa
+#define DOORBELL0_CTRL_ENTRY_8__DOORBELL8_FENCE_ENABLE_ENTRY__SHIFT                                           0x10
+#define DOORBELL0_CTRL_ENTRY_8__BIF_DOORBELL8_RANGE_OFFSET_ENTRY_MASK                                         0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_8__BIF_DOORBELL8_RANGE_SIZE_ENTRY_MASK                                           0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_8__DOORBELL8_FENCE_ENABLE_ENTRY_MASK                                             0x001F0000L
+//DOORBELL0_CTRL_ENTRY_9
+#define DOORBELL0_CTRL_ENTRY_9__BIF_DOORBELL9_RANGE_OFFSET_ENTRY__SHIFT                                       0x0
+#define DOORBELL0_CTRL_ENTRY_9__BIF_DOORBELL9_RANGE_SIZE_ENTRY__SHIFT                                         0xa
+#define DOORBELL0_CTRL_ENTRY_9__DOORBELL9_FENCE_ENABLE_ENTRY__SHIFT                                           0x10
+#define DOORBELL0_CTRL_ENTRY_9__BIF_DOORBELL9_RANGE_OFFSET_ENTRY_MASK                                         0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_9__BIF_DOORBELL9_RANGE_SIZE_ENTRY_MASK                                           0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_9__DOORBELL9_FENCE_ENABLE_ENTRY_MASK                                             0x001F0000L
+//DOORBELL0_CTRL_ENTRY_10
+#define DOORBELL0_CTRL_ENTRY_10__BIF_DOORBELL10_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_10__BIF_DOORBELL10_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_10__DOORBELL10_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_10__BIF_DOORBELL10_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_10__BIF_DOORBELL10_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_10__DOORBELL10_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//DOORBELL0_CTRL_ENTRY_11
+#define DOORBELL0_CTRL_ENTRY_11__BIF_DOORBELL11_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_11__BIF_DOORBELL11_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_11__DOORBELL11_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_11__BIF_DOORBELL11_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_11__BIF_DOORBELL11_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_11__DOORBELL11_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//DOORBELL0_CTRL_ENTRY_12
+#define DOORBELL0_CTRL_ENTRY_12__BIF_DOORBELL12_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_12__BIF_DOORBELL12_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_12__DOORBELL12_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_12__BIF_DOORBELL12_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_12__BIF_DOORBELL12_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_12__DOORBELL12_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//DOORBELL0_CTRL_ENTRY_13
+#define DOORBELL0_CTRL_ENTRY_13__BIF_DOORBELL13_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_13__BIF_DOORBELL13_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_13__DOORBELL13_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_13__BIF_DOORBELL13_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_13__BIF_DOORBELL13_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_13__DOORBELL13_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//DOORBELL0_CTRL_ENTRY_14
+#define DOORBELL0_CTRL_ENTRY_14__BIF_DOORBELL14_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_14__BIF_DOORBELL14_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_14__DOORBELL14_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_14__BIF_DOORBELL14_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_14__BIF_DOORBELL14_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_14__DOORBELL14_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//DOORBELL0_CTRL_ENTRY_15
+#define DOORBELL0_CTRL_ENTRY_15__BIF_DOORBELL15_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_15__BIF_DOORBELL15_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_15__DOORBELL15_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_15__BIF_DOORBELL15_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_15__BIF_DOORBELL15_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_15__DOORBELL15_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//DOORBELL0_CTRL_ENTRY_16
+#define DOORBELL0_CTRL_ENTRY_16__BIF_DOORBELL16_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_16__BIF_DOORBELL16_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_16__DOORBELL16_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_16__BIF_DOORBELL16_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_16__BIF_DOORBELL16_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_16__DOORBELL16_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//DOORBELL0_CTRL_ENTRY_17
+#define DOORBELL0_CTRL_ENTRY_17__BIF_DOORBELL17_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_17__BIF_DOORBELL17_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_17__DOORBELL17_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_17__BIF_DOORBELL17_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_17__BIF_DOORBELL17_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_17__DOORBELL17_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//DOORBELL0_CTRL_ENTRY_18
+#define DOORBELL0_CTRL_ENTRY_18__BIF_DOORBELL18_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_18__BIF_DOORBELL18_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_18__DOORBELL18_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_18__BIF_DOORBELL18_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_18__BIF_DOORBELL18_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_18__DOORBELL18_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//DOORBELL0_CTRL_ENTRY_19
+#define DOORBELL0_CTRL_ENTRY_19__BIF_DOORBELL19_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_19__BIF_DOORBELL19_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_19__DOORBELL19_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_19__BIF_DOORBELL19_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_19__BIF_DOORBELL19_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_19__DOORBELL19_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//DOORBELL0_CTRL_ENTRY_20
+#define DOORBELL0_CTRL_ENTRY_20__BIF_DOORBELL20_RANGE_OFFSET_ENTRY__SHIFT                                     0x0
+#define DOORBELL0_CTRL_ENTRY_20__BIF_DOORBELL20_RANGE_SIZE_ENTRY__SHIFT                                       0xa
+#define DOORBELL0_CTRL_ENTRY_20__DOORBELL20_FENCE_ENABLE_ENTRY__SHIFT                                         0x10
+#define DOORBELL0_CTRL_ENTRY_20__BIF_DOORBELL20_RANGE_OFFSET_ENTRY_MASK                                       0x000003FFL
+#define DOORBELL0_CTRL_ENTRY_20__BIF_DOORBELL20_RANGE_SIZE_ENTRY_MASK                                         0x00007C00L
+#define DOORBELL0_CTRL_ENTRY_20__DOORBELL20_FENCE_ENABLE_ENTRY_MASK                                           0x001F0000L
+//AID0_VF0_BASE_ADDR
+#define AID0_VF0_BASE_ADDR__AID0_VF0_BASE_ADDR__SHIFT                                                         0x0
+#define AID0_VF0_BASE_ADDR__AID0_VF0_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID1_VF0_BASE_ADDR
+#define AID1_VF0_BASE_ADDR__AID1_VF0_BASE_ADDR__SHIFT                                                         0x0
+#define AID1_VF0_BASE_ADDR__AID1_VF0_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID2_VF0_BASE_ADDR
+#define AID2_VF0_BASE_ADDR__AID2_VF0_BASE_ADDR__SHIFT                                                         0x0
+#define AID2_VF0_BASE_ADDR__AID2_VF0_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID3_VF0_BASE_ADDR
+#define AID3_VF0_BASE_ADDR__AID3_VF0_BASE_ADDR__SHIFT                                                         0x0
+#define AID3_VF0_BASE_ADDR__AID3_VF0_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID0_XCC0_VF0_BASE_ADDR
+#define AID0_XCC0_VF0_BASE_ADDR__AID0_XCC0_VF0_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC0_VF0_BASE_ADDR__AID0_XCC0_VF0_BASE_ADDR_MASK                                                 0x0001FFFFL
+//AID0_XCC1_VF0_BASE_ADDR
+#define AID0_XCC1_VF0_BASE_ADDR__AID0_XCC1_VF0_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC1_VF0_BASE_ADDR__AID0_XCC1_VF0_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC0_VF0_BASE_ADDR
+#define AID1_XCC0_VF0_BASE_ADDR__AID1_XCC0_VF0_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC0_VF0_BASE_ADDR__AID1_XCC0_VF0_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC1_VF0_BASE_ADDR
+#define AID1_XCC1_VF0_BASE_ADDR__AID1_XCC1_VF0_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC1_VF0_BASE_ADDR__AID1_XCC1_VF0_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC0_VF0_BASE_ADDR
+#define AID2_XCC0_VF0_BASE_ADDR__AID2_XCC0_VF0_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC0_VF0_BASE_ADDR__AID2_XCC0_VF0_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC1_VF0_BASE_ADDR
+#define AID2_XCC1_VF0_BASE_ADDR__AID2_XCC1_VF0_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC1_VF0_BASE_ADDR__AID2_XCC1_VF0_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC0_VF0_BASE_ADDR
+#define AID3_XCC0_VF0_BASE_ADDR__AID3_XCC0_VF0_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC0_VF0_BASE_ADDR__AID3_XCC0_VF0_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC1_VF0_BASE_ADDR
+#define AID3_XCC1_VF0_BASE_ADDR__AID3_XCC1_VF0_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC1_VF0_BASE_ADDR__AID3_XCC1_VF0_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_NBIF_VF0_BASE_ADDR
+#define AID0_NBIF_VF0_BASE_ADDR__AID0_NBIF_VF0_BASE_ADDR__SHIFT                                               0x0
+#define AID0_NBIF_VF0_BASE_ADDR__AID0_NBIF_VF0_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_ATHUB_VF0_BASE_ADDR
+#define AID0_ATHUB_VF0_BASE_ADDR__AID0_ATHUB_VF0_BASE_ADDR__SHIFT                                             0x0
+#define AID0_ATHUB_VF0_BASE_ADDR__AID0_ATHUB_VF0_BASE_ADDR_MASK                                               0x0000FFFFL
+//AID0_IH_VF0_BASE_ADDR
+#define AID0_IH_VF0_BASE_ADDR__AID0_IH_VF0_BASE_ADDR__SHIFT                                                   0x0
+#define AID0_IH_VF0_BASE_ADDR__AID0_IH_VF0_BASE_ADDR_MASK                                                     0x0000FFFFL
+//AID0_HDP_VF0_BASE_ADDR
+#define AID0_HDP_VF0_BASE_ADDR__AID0_HDP_VF0_BASE_ADDR__SHIFT                                                 0x0
+#define AID0_HDP_VF0_BASE_ADDR__AID0_HDP_VF0_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID0_VF1_BASE_ADDR
+#define AID0_VF1_BASE_ADDR__AID0_VF1_BASE_ADDR__SHIFT                                                         0x0
+#define AID0_VF1_BASE_ADDR__AID0_VF1_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID1_VF1_BASE_ADDR
+#define AID1_VF1_BASE_ADDR__AID1_VF1_BASE_ADDR__SHIFT                                                         0x0
+#define AID1_VF1_BASE_ADDR__AID1_VF1_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID2_VF1_BASE_ADDR
+#define AID2_VF1_BASE_ADDR__AID2_VF1_BASE_ADDR__SHIFT                                                         0x0
+#define AID2_VF1_BASE_ADDR__AID2_VF1_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID3_VF1_BASE_ADDR
+#define AID3_VF1_BASE_ADDR__AID3_VF1_BASE_ADDR__SHIFT                                                         0x0
+#define AID3_VF1_BASE_ADDR__AID3_VF1_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID0_XCC0_VF1_BASE_ADDR
+#define AID0_XCC0_VF1_BASE_ADDR__AID0_XCC0_VF1_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC0_VF1_BASE_ADDR__AID0_XCC0_VF1_BASE_ADDR_MASK                                                 0x0001FFFFL
+//AID0_XCC1_VF1_BASE_ADDR
+#define AID0_XCC1_VF1_BASE_ADDR__AID0_XCC1_VF1_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC1_VF1_BASE_ADDR__AID0_XCC1_VF1_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC0_VF1_BASE_ADDR
+#define AID1_XCC0_VF1_BASE_ADDR__AID1_XCC0_VF1_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC0_VF1_BASE_ADDR__AID1_XCC0_VF1_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC1_VF1_BASE_ADDR
+#define AID1_XCC1_VF1_BASE_ADDR__AID1_XCC1_VF1_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC1_VF1_BASE_ADDR__AID1_XCC1_VF1_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC0_VF1_BASE_ADDR
+#define AID2_XCC0_VF1_BASE_ADDR__AID2_XCC0_VF1_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC0_VF1_BASE_ADDR__AID2_XCC0_VF1_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC1_VF1_BASE_ADDR
+#define AID2_XCC1_VF1_BASE_ADDR__AID2_XCC1_VF1_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC1_VF1_BASE_ADDR__AID2_XCC1_VF1_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC0_VF1_BASE_ADDR
+#define AID3_XCC0_VF1_BASE_ADDR__AID3_XCC0_VF1_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC0_VF1_BASE_ADDR__AID3_XCC0_VF1_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC1_VF1_BASE_ADDR
+#define AID3_XCC1_VF1_BASE_ADDR__AID3_XCC1_VF1_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC1_VF1_BASE_ADDR__AID3_XCC1_VF1_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_NBIF_VF1_BASE_ADDR
+#define AID0_NBIF_VF1_BASE_ADDR__AID0_NBIF_VF1_BASE_ADDR__SHIFT                                               0x0
+#define AID0_NBIF_VF1_BASE_ADDR__AID0_NBIF_VF1_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_ATHUB_VF1_BASE_ADDR
+#define AID0_ATHUB_VF1_BASE_ADDR__AID0_ATHUB_VF1_BASE_ADDR__SHIFT                                             0x0
+#define AID0_ATHUB_VF1_BASE_ADDR__AID0_ATHUB_VF1_BASE_ADDR_MASK                                               0x0000FFFFL
+//AID0_IH_VF1_BASE_ADDR
+#define AID0_IH_VF1_BASE_ADDR__AID0_IH_VF1_BASE_ADDR__SHIFT                                                   0x0
+#define AID0_IH_VF1_BASE_ADDR__AID0_IH_VF1_BASE_ADDR_MASK                                                     0x0000FFFFL
+//AID0_HDP_VF1_BASE_ADDR
+#define AID0_HDP_VF1_BASE_ADDR__AID0_HDP_VF1_BASE_ADDR__SHIFT                                                 0x0
+#define AID0_HDP_VF1_BASE_ADDR__AID0_HDP_VF1_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID0_VF2_BASE_ADDR
+#define AID0_VF2_BASE_ADDR__AID0_VF2_BASE_ADDR__SHIFT                                                         0x0
+#define AID0_VF2_BASE_ADDR__AID0_VF2_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID1_VF2_BASE_ADDR
+#define AID1_VF2_BASE_ADDR__AID1_VF2_BASE_ADDR__SHIFT                                                         0x0
+#define AID1_VF2_BASE_ADDR__AID1_VF2_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID2_VF2_BASE_ADDR
+#define AID2_VF2_BASE_ADDR__AID2_VF2_BASE_ADDR__SHIFT                                                         0x0
+#define AID2_VF2_BASE_ADDR__AID2_VF2_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID3_VF2_BASE_ADDR
+#define AID3_VF2_BASE_ADDR__AID3_VF2_BASE_ADDR__SHIFT                                                         0x0
+#define AID3_VF2_BASE_ADDR__AID3_VF2_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID0_XCC0_VF2_BASE_ADDR
+#define AID0_XCC0_VF2_BASE_ADDR__AID0_XCC0_VF2_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC0_VF2_BASE_ADDR__AID0_XCC0_VF2_BASE_ADDR_MASK                                                 0x0001FFFFL
+//AID0_XCC1_VF2_BASE_ADDR
+#define AID0_XCC1_VF2_BASE_ADDR__AID0_XCC1_VF2_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC1_VF2_BASE_ADDR__AID0_XCC1_VF2_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC0_VF2_BASE_ADDR
+#define AID1_XCC0_VF2_BASE_ADDR__AID1_XCC0_VF2_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC0_VF2_BASE_ADDR__AID1_XCC0_VF2_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC1_VF2_BASE_ADDR
+#define AID1_XCC1_VF2_BASE_ADDR__AID1_XCC1_VF2_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC1_VF2_BASE_ADDR__AID1_XCC1_VF2_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC0_VF2_BASE_ADDR
+#define AID2_XCC0_VF2_BASE_ADDR__AID2_XCC0_VF2_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC0_VF2_BASE_ADDR__AID2_XCC0_VF2_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC1_VF2_BASE_ADDR
+#define AID2_XCC1_VF2_BASE_ADDR__AID2_XCC1_VF2_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC1_VF2_BASE_ADDR__AID2_XCC1_VF2_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC0_VF2_BASE_ADDR
+#define AID3_XCC0_VF2_BASE_ADDR__AID3_XCC0_VF2_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC0_VF2_BASE_ADDR__AID3_XCC0_VF2_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC1_VF2_BASE_ADDR
+#define AID3_XCC1_VF2_BASE_ADDR__AID3_XCC1_VF2_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC1_VF2_BASE_ADDR__AID3_XCC1_VF2_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_NBIF_VF2_BASE_ADDR
+#define AID0_NBIF_VF2_BASE_ADDR__AID0_NBIF_VF2_BASE_ADDR__SHIFT                                               0x0
+#define AID0_NBIF_VF2_BASE_ADDR__AID0_NBIF_VF2_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_ATHUB_VF2_BASE_ADDR
+#define AID0_ATHUB_VF2_BASE_ADDR__AID0_ATHUB_VF2_BASE_ADDR__SHIFT                                             0x0
+#define AID0_ATHUB_VF2_BASE_ADDR__AID0_ATHUB_VF2_BASE_ADDR_MASK                                               0x0000FFFFL
+//AID0_IH_VF2_BASE_ADDR
+#define AID0_IH_VF2_BASE_ADDR__AID0_IH_VF2_BASE_ADDR__SHIFT                                                   0x0
+#define AID0_IH_VF2_BASE_ADDR__AID0_IH_VF2_BASE_ADDR_MASK                                                     0x0000FFFFL
+//AID0_HDP_VF2_BASE_ADDR
+#define AID0_HDP_VF2_BASE_ADDR__AID0_HDP_VF2_BASE_ADDR__SHIFT                                                 0x0
+#define AID0_HDP_VF2_BASE_ADDR__AID0_HDP_VF2_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID0_VF3_BASE_ADDR
+#define AID0_VF3_BASE_ADDR__AID0_VF3_BASE_ADDR__SHIFT                                                         0x0
+#define AID0_VF3_BASE_ADDR__AID0_VF3_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID1_VF3_BASE_ADDR
+#define AID1_VF3_BASE_ADDR__AID1_VF3_BASE_ADDR__SHIFT                                                         0x0
+#define AID1_VF3_BASE_ADDR__AID1_VF3_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID2_VF3_BASE_ADDR
+#define AID2_VF3_BASE_ADDR__AID2_VF3_BASE_ADDR__SHIFT                                                         0x0
+#define AID2_VF3_BASE_ADDR__AID2_VF3_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID3_VF3_BASE_ADDR
+#define AID3_VF3_BASE_ADDR__AID3_VF3_BASE_ADDR__SHIFT                                                         0x0
+#define AID3_VF3_BASE_ADDR__AID3_VF3_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID0_XCC0_VF3_BASE_ADDR
+#define AID0_XCC0_VF3_BASE_ADDR__AID0_XCC0_VF3_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC0_VF3_BASE_ADDR__AID0_XCC0_VF3_BASE_ADDR_MASK                                                 0x0001FFFFL
+//AID0_XCC1_VF3_BASE_ADDR
+#define AID0_XCC1_VF3_BASE_ADDR__AID0_XCC1_VF3_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC1_VF3_BASE_ADDR__AID0_XCC1_VF3_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC0_VF3_BASE_ADDR
+#define AID1_XCC0_VF3_BASE_ADDR__AID1_XCC0_VF3_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC0_VF3_BASE_ADDR__AID1_XCC0_VF3_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC1_VF3_BASE_ADDR
+#define AID1_XCC1_VF3_BASE_ADDR__AID1_XCC1_VF3_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC1_VF3_BASE_ADDR__AID1_XCC1_VF3_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC0_VF3_BASE_ADDR
+#define AID2_XCC0_VF3_BASE_ADDR__AID2_XCC0_VF3_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC0_VF3_BASE_ADDR__AID2_XCC0_VF3_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC1_VF3_BASE_ADDR
+#define AID2_XCC1_VF3_BASE_ADDR__AID2_XCC1_VF3_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC1_VF3_BASE_ADDR__AID2_XCC1_VF3_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC0_VF3_BASE_ADDR
+#define AID3_XCC0_VF3_BASE_ADDR__AID3_XCC0_VF3_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC0_VF3_BASE_ADDR__AID3_XCC0_VF3_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC1_VF3_BASE_ADDR
+#define AID3_XCC1_VF3_BASE_ADDR__AID3_XCC1_VF3_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC1_VF3_BASE_ADDR__AID3_XCC1_VF3_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_NBIF_VF3_BASE_ADDR
+#define AID0_NBIF_VF3_BASE_ADDR__AID0_NBIF_VF3_BASE_ADDR__SHIFT                                               0x0
+#define AID0_NBIF_VF3_BASE_ADDR__AID0_NBIF_VF3_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_ATHUB_VF3_BASE_ADDR
+#define AID0_ATHUB_VF3_BASE_ADDR__AID0_ATHUB_VF3_BASE_ADDR__SHIFT                                             0x0
+#define AID0_ATHUB_VF3_BASE_ADDR__AID0_ATHUB_VF3_BASE_ADDR_MASK                                               0x0000FFFFL
+//AID0_IH_VF3_BASE_ADDR
+#define AID0_IH_VF3_BASE_ADDR__AID0_IH_VF3_BASE_ADDR__SHIFT                                                   0x0
+#define AID0_IH_VF3_BASE_ADDR__AID0_IH_VF3_BASE_ADDR_MASK                                                     0x0000FFFFL
+//AID0_HDP_VF3_BASE_ADDR
+#define AID0_HDP_VF3_BASE_ADDR__AID0_HDP_VF3_BASE_ADDR__SHIFT                                                 0x0
+#define AID0_HDP_VF3_BASE_ADDR__AID0_HDP_VF3_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID0_VF4_BASE_ADDR
+#define AID0_VF4_BASE_ADDR__AID0_VF4_BASE_ADDR__SHIFT                                                         0x0
+#define AID0_VF4_BASE_ADDR__AID0_VF4_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID1_VF4_BASE_ADDR
+#define AID1_VF4_BASE_ADDR__AID1_VF4_BASE_ADDR__SHIFT                                                         0x0
+#define AID1_VF4_BASE_ADDR__AID1_VF4_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID2_VF4_BASE_ADDR
+#define AID2_VF4_BASE_ADDR__AID2_VF4_BASE_ADDR__SHIFT                                                         0x0
+#define AID2_VF4_BASE_ADDR__AID2_VF4_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID3_VF4_BASE_ADDR
+#define AID3_VF4_BASE_ADDR__AID3_VF4_BASE_ADDR__SHIFT                                                         0x0
+#define AID3_VF4_BASE_ADDR__AID3_VF4_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID0_XCC0_VF4_BASE_ADDR
+#define AID0_XCC0_VF4_BASE_ADDR__AID0_XCC0_VF4_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC0_VF4_BASE_ADDR__AID0_XCC0_VF4_BASE_ADDR_MASK                                                 0x0001FFFFL
+//AID0_XCC1_VF4_BASE_ADDR
+#define AID0_XCC1_VF4_BASE_ADDR__AID0_XCC1_VF4_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC1_VF4_BASE_ADDR__AID0_XCC1_VF4_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC0_VF4_BASE_ADDR
+#define AID1_XCC0_VF4_BASE_ADDR__AID1_XCC0_VF4_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC0_VF4_BASE_ADDR__AID1_XCC0_VF4_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC1_VF4_BASE_ADDR
+#define AID1_XCC1_VF4_BASE_ADDR__AID1_XCC1_VF4_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC1_VF4_BASE_ADDR__AID1_XCC1_VF4_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC0_VF4_BASE_ADDR
+#define AID2_XCC0_VF4_BASE_ADDR__AID2_XCC0_VF4_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC0_VF4_BASE_ADDR__AID2_XCC0_VF4_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC1_VF4_BASE_ADDR
+#define AID2_XCC1_VF4_BASE_ADDR__AID2_XCC1_VF4_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC1_VF4_BASE_ADDR__AID2_XCC1_VF4_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC0_VF4_BASE_ADDR
+#define AID3_XCC0_VF4_BASE_ADDR__AID3_XCC0_VF4_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC0_VF4_BASE_ADDR__AID3_XCC0_VF4_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC1_VF4_BASE_ADDR
+#define AID3_XCC1_VF4_BASE_ADDR__AID3_XCC1_VF4_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC1_VF4_BASE_ADDR__AID3_XCC1_VF4_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_NBIF_VF4_BASE_ADDR
+#define AID0_NBIF_VF4_BASE_ADDR__AID0_NBIF_VF4_BASE_ADDR__SHIFT                                               0x0
+#define AID0_NBIF_VF4_BASE_ADDR__AID0_NBIF_VF4_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_ATHUB_VF4_BASE_ADDR
+#define AID0_ATHUB_VF4_BASE_ADDR__AID0_ATHUB_VF4_BASE_ADDR__SHIFT                                             0x0
+#define AID0_ATHUB_VF4_BASE_ADDR__AID0_ATHUB_VF4_BASE_ADDR_MASK                                               0x0000FFFFL
+//AID0_IH_VF4_BASE_ADDR
+#define AID0_IH_VF4_BASE_ADDR__AID0_IH_VF4_BASE_ADDR__SHIFT                                                   0x0
+#define AID0_IH_VF4_BASE_ADDR__AID0_IH_VF4_BASE_ADDR_MASK                                                     0x0000FFFFL
+//AID0_HDP_VF4_BASE_ADDR
+#define AID0_HDP_VF4_BASE_ADDR__AID0_HDP_VF4_BASE_ADDR__SHIFT                                                 0x0
+#define AID0_HDP_VF4_BASE_ADDR__AID0_HDP_VF4_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID0_VF5_BASE_ADDR
+#define AID0_VF5_BASE_ADDR__AID0_VF5_BASE_ADDR__SHIFT                                                         0x0
+#define AID0_VF5_BASE_ADDR__AID0_VF5_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID1_VF5_BASE_ADDR
+#define AID1_VF5_BASE_ADDR__AID1_VF5_BASE_ADDR__SHIFT                                                         0x0
+#define AID1_VF5_BASE_ADDR__AID1_VF5_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID2_VF5_BASE_ADDR
+#define AID2_VF5_BASE_ADDR__AID2_VF5_BASE_ADDR__SHIFT                                                         0x0
+#define AID2_VF5_BASE_ADDR__AID2_VF5_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID3_VF5_BASE_ADDR
+#define AID3_VF5_BASE_ADDR__AID3_VF5_BASE_ADDR__SHIFT                                                         0x0
+#define AID3_VF5_BASE_ADDR__AID3_VF5_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID0_XCC0_VF5_BASE_ADDR
+#define AID0_XCC0_VF5_BASE_ADDR__AID0_XCC0_VF5_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC0_VF5_BASE_ADDR__AID0_XCC0_VF5_BASE_ADDR_MASK                                                 0x0001FFFFL
+//AID0_XCC1_VF5_BASE_ADDR
+#define AID0_XCC1_VF5_BASE_ADDR__AID0_XCC1_VF5_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC1_VF5_BASE_ADDR__AID0_XCC1_VF5_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC0_VF5_BASE_ADDR
+#define AID1_XCC0_VF5_BASE_ADDR__AID1_XCC0_VF5_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC0_VF5_BASE_ADDR__AID1_XCC0_VF5_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC1_VF5_BASE_ADDR
+#define AID1_XCC1_VF5_BASE_ADDR__AID1_XCC1_VF5_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC1_VF5_BASE_ADDR__AID1_XCC1_VF5_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC0_VF5_BASE_ADDR
+#define AID2_XCC0_VF5_BASE_ADDR__AID2_XCC0_VF5_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC0_VF5_BASE_ADDR__AID2_XCC0_VF5_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC1_VF5_BASE_ADDR
+#define AID2_XCC1_VF5_BASE_ADDR__AID2_XCC1_VF5_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC1_VF5_BASE_ADDR__AID2_XCC1_VF5_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC0_VF5_BASE_ADDR
+#define AID3_XCC0_VF5_BASE_ADDR__AID3_XCC0_VF5_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC0_VF5_BASE_ADDR__AID3_XCC0_VF5_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC1_VF5_BASE_ADDR
+#define AID3_XCC1_VF5_BASE_ADDR__AID3_XCC1_VF5_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC1_VF5_BASE_ADDR__AID3_XCC1_VF5_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_NBIF_VF5_BASE_ADDR
+#define AID0_NBIF_VF5_BASE_ADDR__AID0_NBIF_VF5_BASE_ADDR__SHIFT                                               0x0
+#define AID0_NBIF_VF5_BASE_ADDR__AID0_NBIF_VF5_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_ATHUB_VF5_BASE_ADDR
+#define AID0_ATHUB_VF5_BASE_ADDR__AID0_ATHUB_VF5_BASE_ADDR__SHIFT                                             0x0
+#define AID0_ATHUB_VF5_BASE_ADDR__AID0_ATHUB_VF5_BASE_ADDR_MASK                                               0x0000FFFFL
+//AID0_IH_VF5_BASE_ADDR
+#define AID0_IH_VF5_BASE_ADDR__AID0_IH_VF5_BASE_ADDR__SHIFT                                                   0x0
+#define AID0_IH_VF5_BASE_ADDR__AID0_IH_VF5_BASE_ADDR_MASK                                                     0x0000FFFFL
+//AID0_HDP_VF5_BASE_ADDR
+#define AID0_HDP_VF5_BASE_ADDR__AID0_HDP_VF5_BASE_ADDR__SHIFT                                                 0x0
+#define AID0_HDP_VF5_BASE_ADDR__AID0_HDP_VF5_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID0_VF6_BASE_ADDR
+#define AID0_VF6_BASE_ADDR__AID0_VF6_BASE_ADDR__SHIFT                                                         0x0
+#define AID0_VF6_BASE_ADDR__AID0_VF6_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID1_VF6_BASE_ADDR
+#define AID1_VF6_BASE_ADDR__AID1_VF6_BASE_ADDR__SHIFT                                                         0x0
+#define AID1_VF6_BASE_ADDR__AID1_VF6_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID2_VF6_BASE_ADDR
+#define AID2_VF6_BASE_ADDR__AID2_VF6_BASE_ADDR__SHIFT                                                         0x0
+#define AID2_VF6_BASE_ADDR__AID2_VF6_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID3_VF6_BASE_ADDR
+#define AID3_VF6_BASE_ADDR__AID3_VF6_BASE_ADDR__SHIFT                                                         0x0
+#define AID3_VF6_BASE_ADDR__AID3_VF6_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID0_XCC0_VF6_BASE_ADDR
+#define AID0_XCC0_VF6_BASE_ADDR__AID0_XCC0_VF6_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC0_VF6_BASE_ADDR__AID0_XCC0_VF6_BASE_ADDR_MASK                                                 0x0001FFFFL
+//AID0_XCC1_VF6_BASE_ADDR
+#define AID0_XCC1_VF6_BASE_ADDR__AID0_XCC1_VF6_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC1_VF6_BASE_ADDR__AID0_XCC1_VF6_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC0_VF6_BASE_ADDR
+#define AID1_XCC0_VF6_BASE_ADDR__AID1_XCC0_VF6_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC0_VF6_BASE_ADDR__AID1_XCC0_VF6_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC1_VF6_BASE_ADDR
+#define AID1_XCC1_VF6_BASE_ADDR__AID1_XCC1_VF6_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC1_VF6_BASE_ADDR__AID1_XCC1_VF6_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC0_VF6_BASE_ADDR
+#define AID2_XCC0_VF6_BASE_ADDR__AID2_XCC0_VF6_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC0_VF6_BASE_ADDR__AID2_XCC0_VF6_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC1_VF6_BASE_ADDR
+#define AID2_XCC1_VF6_BASE_ADDR__AID2_XCC1_VF6_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC1_VF6_BASE_ADDR__AID2_XCC1_VF6_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC0_VF6_BASE_ADDR
+#define AID3_XCC0_VF6_BASE_ADDR__AID3_XCC0_VF6_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC0_VF6_BASE_ADDR__AID3_XCC0_VF6_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC1_VF6_BASE_ADDR
+#define AID3_XCC1_VF6_BASE_ADDR__AID3_XCC1_VF6_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC1_VF6_BASE_ADDR__AID3_XCC1_VF6_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_NBIF_VF6_BASE_ADDR
+#define AID0_NBIF_VF6_BASE_ADDR__AID0_NBIF_VF6_BASE_ADDR__SHIFT                                               0x0
+#define AID0_NBIF_VF6_BASE_ADDR__AID0_NBIF_VF6_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_ATHUB_VF6_BASE_ADDR
+#define AID0_ATHUB_VF6_BASE_ADDR__AID0_ATHUB_VF6_BASE_ADDR__SHIFT                                             0x0
+#define AID0_ATHUB_VF6_BASE_ADDR__AID0_ATHUB_VF6_BASE_ADDR_MASK                                               0x0000FFFFL
+//AID0_IH_VF6_BASE_ADDR
+#define AID0_IH_VF6_BASE_ADDR__AID0_IH_VF6_BASE_ADDR__SHIFT                                                   0x0
+#define AID0_IH_VF6_BASE_ADDR__AID0_IH_VF6_BASE_ADDR_MASK                                                     0x0000FFFFL
+//AID0_HDP_VF6_BASE_ADDR
+#define AID0_HDP_VF6_BASE_ADDR__AID0_HDP_VF6_BASE_ADDR__SHIFT                                                 0x0
+#define AID0_HDP_VF6_BASE_ADDR__AID0_HDP_VF6_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID0_VF7_BASE_ADDR
+#define AID0_VF7_BASE_ADDR__AID0_VF7_BASE_ADDR__SHIFT                                                         0x0
+#define AID0_VF7_BASE_ADDR__AID0_VF7_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID1_VF7_BASE_ADDR
+#define AID1_VF7_BASE_ADDR__AID1_VF7_BASE_ADDR__SHIFT                                                         0x0
+#define AID1_VF7_BASE_ADDR__AID1_VF7_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID2_VF7_BASE_ADDR
+#define AID2_VF7_BASE_ADDR__AID2_VF7_BASE_ADDR__SHIFT                                                         0x0
+#define AID2_VF7_BASE_ADDR__AID2_VF7_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID3_VF7_BASE_ADDR
+#define AID3_VF7_BASE_ADDR__AID3_VF7_BASE_ADDR__SHIFT                                                         0x0
+#define AID3_VF7_BASE_ADDR__AID3_VF7_BASE_ADDR_MASK                                                           0x0000FFFFL
+//AID0_XCC0_VF7_BASE_ADDR
+#define AID0_XCC0_VF7_BASE_ADDR__AID0_XCC0_VF7_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC0_VF7_BASE_ADDR__AID0_XCC0_VF7_BASE_ADDR_MASK                                                 0x0001FFFFL
+//AID0_XCC1_VF7_BASE_ADDR
+#define AID0_XCC1_VF7_BASE_ADDR__AID0_XCC1_VF7_BASE_ADDR__SHIFT                                               0x0
+#define AID0_XCC1_VF7_BASE_ADDR__AID0_XCC1_VF7_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC0_VF7_BASE_ADDR
+#define AID1_XCC0_VF7_BASE_ADDR__AID1_XCC0_VF7_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC0_VF7_BASE_ADDR__AID1_XCC0_VF7_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID1_XCC1_VF7_BASE_ADDR
+#define AID1_XCC1_VF7_BASE_ADDR__AID1_XCC1_VF7_BASE_ADDR__SHIFT                                               0x0
+#define AID1_XCC1_VF7_BASE_ADDR__AID1_XCC1_VF7_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC0_VF7_BASE_ADDR
+#define AID2_XCC0_VF7_BASE_ADDR__AID2_XCC0_VF7_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC0_VF7_BASE_ADDR__AID2_XCC0_VF7_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID2_XCC1_VF7_BASE_ADDR
+#define AID2_XCC1_VF7_BASE_ADDR__AID2_XCC1_VF7_BASE_ADDR__SHIFT                                               0x0
+#define AID2_XCC1_VF7_BASE_ADDR__AID2_XCC1_VF7_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC0_VF7_BASE_ADDR
+#define AID3_XCC0_VF7_BASE_ADDR__AID3_XCC0_VF7_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC0_VF7_BASE_ADDR__AID3_XCC0_VF7_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID3_XCC1_VF7_BASE_ADDR
+#define AID3_XCC1_VF7_BASE_ADDR__AID3_XCC1_VF7_BASE_ADDR__SHIFT                                               0x0
+#define AID3_XCC1_VF7_BASE_ADDR__AID3_XCC1_VF7_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_NBIF_VF7_BASE_ADDR
+#define AID0_NBIF_VF7_BASE_ADDR__AID0_NBIF_VF7_BASE_ADDR__SHIFT                                               0x0
+#define AID0_NBIF_VF7_BASE_ADDR__AID0_NBIF_VF7_BASE_ADDR_MASK                                                 0x0000FFFFL
+//AID0_ATHUB_VF7_BASE_ADDR
+#define AID0_ATHUB_VF7_BASE_ADDR__AID0_ATHUB_VF7_BASE_ADDR__SHIFT                                             0x0
+#define AID0_ATHUB_VF7_BASE_ADDR__AID0_ATHUB_VF7_BASE_ADDR_MASK                                               0x0000FFFFL
+//AID0_IH_VF7_BASE_ADDR
+#define AID0_IH_VF7_BASE_ADDR__AID0_IH_VF7_BASE_ADDR__SHIFT                                                   0x0
+#define AID0_IH_VF7_BASE_ADDR__AID0_IH_VF7_BASE_ADDR_MASK                                                     0x0000FFFFL
+//AID0_HDP_VF7_BASE_ADDR
+#define AID0_HDP_VF7_BASE_ADDR__AID0_HDP_VF7_BASE_ADDR__SHIFT                                                 0x0
+#define AID0_HDP_VF7_BASE_ADDR__AID0_HDP_VF7_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID0_PF_BASE_ADDR
+#define AID0_PF_BASE_ADDR__AID0_PF_BASE_ADDR__SHIFT                                                           0x0
+#define AID0_PF_BASE_ADDR__AID0_PF_BASE_ADDR_MASK                                                             0x0000FFFFL
+//AID0_XCC0_PF_BASE_ADDR
+#define AID0_XCC0_PF_BASE_ADDR__AID0_XCC0_PF_BASE_ADDR__SHIFT                                                 0x0
+#define AID0_XCC0_PF_BASE_ADDR__AID0_XCC0_PF_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID0_XCC1_PF_BASE_ADDR
+#define AID0_XCC1_PF_BASE_ADDR__AID0_XCC1_PF_BASE_ADDR__SHIFT                                                 0x0
+#define AID0_XCC1_PF_BASE_ADDR__AID0_XCC1_PF_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID1_PF_BASE_ADDR
+#define AID1_PF_BASE_ADDR__AID1_PF_BASE_ADDR__SHIFT                                                           0x0
+#define AID1_PF_BASE_ADDR__AID1_PF_BASE_ADDR_MASK                                                             0x0000FFFFL
+//AID1_XCC0_PF_BASE_ADDR
+#define AID1_XCC0_PF_BASE_ADDR__AID1_XCC0_PF_BASE_ADDR__SHIFT                                                 0x0
+#define AID1_XCC0_PF_BASE_ADDR__AID1_XCC0_PF_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID1_XCC1_PF_BASE_ADDR
+#define AID1_XCC1_PF_BASE_ADDR__AID1_XCC1_PF_BASE_ADDR__SHIFT                                                 0x0
+#define AID1_XCC1_PF_BASE_ADDR__AID1_XCC1_PF_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID2_PF_BASE_ADDR
+#define AID2_PF_BASE_ADDR__AID2_PF_BASE_ADDR__SHIFT                                                           0x0
+#define AID2_PF_BASE_ADDR__AID2_PF_BASE_ADDR_MASK                                                             0x0000FFFFL
+//AID2_XCC0_PF_BASE_ADDR
+#define AID2_XCC0_PF_BASE_ADDR__AID2_XCC0_PF_BASE_ADDR__SHIFT                                                 0x0
+#define AID2_XCC0_PF_BASE_ADDR__AID2_XCC0_PF_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID2_XCC1_PF_BASE_ADDR
+#define AID2_XCC1_PF_BASE_ADDR__AID2_XCC1_PF_BASE_ADDR__SHIFT                                                 0x0
+#define AID2_XCC1_PF_BASE_ADDR__AID2_XCC1_PF_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID3_PF_BASE_ADDR
+#define AID3_PF_BASE_ADDR__AID3_PF_BASE_ADDR__SHIFT                                                           0x0
+#define AID3_PF_BASE_ADDR__AID3_PF_BASE_ADDR_MASK                                                             0x0000FFFFL
+//AID3_XCC0_PF_BASE_ADDR
+#define AID3_XCC0_PF_BASE_ADDR__AID3_XCC0_PF_BASE_ADDR__SHIFT                                                 0x0
+#define AID3_XCC0_PF_BASE_ADDR__AID3_XCC0_PF_BASE_ADDR_MASK                                                   0x0000FFFFL
+//AID3_XCC1_PF_BASE_ADDR
+#define AID3_XCC1_PF_BASE_ADDR__AID3_XCC1_PF_BASE_ADDR__SHIFT                                                 0x0
+#define AID3_XCC1_PF_BASE_ADDR__AID3_XCC1_PF_BASE_ADDR_MASK                                                   0x0000FFFFL
+//NBIF_RRMT_CNTL
+#define NBIF_RRMT_CNTL__PARTITION_MODE__SHIFT                                                                 0x0
+#define NBIF_RRMT_CNTL__AID_DIE_ID__SHIFT                                                                     0x4
+#define NBIF_RRMT_CNTL__RRMT_ENABLE__SHIFT                                                                    0x8
+#define NBIF_RRMT_CNTL__RRMT_Invalid_Address_H__SHIFT                                                         0x18
+#define NBIF_RRMT_CNTL__PARTITION_MODE_MASK                                                                   0x00000007L
+#define NBIF_RRMT_CNTL__AID_DIE_ID_MASK                                                                       0x00000030L
+#define NBIF_RRMT_CNTL__RRMT_ENABLE_MASK                                                                      0x00000100L
+#define NBIF_RRMT_CNTL__RRMT_Invalid_Address_H_MASK                                                           0xFF000000L
+//BIFC_DOORBELL_ACCESS_EN_PF
+#define BIFC_DOORBELL_ACCESS_EN_PF__BIFC_DOORBELL_ACCESS_EN_PF__SHIFT                                         0x0
+#define BIFC_DOORBELL_ACCESS_EN_PF__BIFC_DOORBELL_ACCESS_EN_PF_MASK                                           0x000FFFFFL
+//BIFC_DOORBELL_ACCESS_EN_VF0
+#define BIFC_DOORBELL_ACCESS_EN_VF0__BIFC_DOORBELL_ACCESS_EN_VF0__SHIFT                                       0x0
+#define BIFC_DOORBELL_ACCESS_EN_VF0__BIFC_DOORBELL_ACCESS_EN_VF0_MASK                                         0x000FFFFFL
+//BIFC_DOORBELL_ACCESS_EN_VF1
+#define BIFC_DOORBELL_ACCESS_EN_VF1__BIFC_DOORBELL_ACCESS_EN_VF1__SHIFT                                       0x0
+#define BIFC_DOORBELL_ACCESS_EN_VF1__BIFC_DOORBELL_ACCESS_EN_VF1_MASK                                         0x000FFFFFL
+//BIFC_DOORBELL_ACCESS_EN_VF2
+#define BIFC_DOORBELL_ACCESS_EN_VF2__BIFC_DOORBELL_ACCESS_EN_VF2__SHIFT                                       0x0
+#define BIFC_DOORBELL_ACCESS_EN_VF2__BIFC_DOORBELL_ACCESS_EN_VF2_MASK                                         0x000FFFFFL
+//BIFC_DOORBELL_ACCESS_EN_VF3
+#define BIFC_DOORBELL_ACCESS_EN_VF3__BIFC_DOORBELL_ACCESS_EN_VF3__SHIFT                                       0x0
+#define BIFC_DOORBELL_ACCESS_EN_VF3__BIFC_DOORBELL_ACCESS_EN_VF3_MASK                                         0x000FFFFFL
+//BIFC_DOORBELL_ACCESS_EN_VF4
+#define BIFC_DOORBELL_ACCESS_EN_VF4__BIFC_DOORBELL_ACCESS_EN_VF4__SHIFT                                       0x0
+#define BIFC_DOORBELL_ACCESS_EN_VF4__BIFC_DOORBELL_ACCESS_EN_VF4_MASK                                         0x000FFFFFL
+//BIFC_DOORBELL_ACCESS_EN_VF5
+#define BIFC_DOORBELL_ACCESS_EN_VF5__BIFC_DOORBELL_ACCESS_EN_VF5__SHIFT                                       0x0
+#define BIFC_DOORBELL_ACCESS_EN_VF5__BIFC_DOORBELL_ACCESS_EN_VF5_MASK                                         0x000FFFFFL
+//BIFC_DOORBELL_ACCESS_EN_VF6
+#define BIFC_DOORBELL_ACCESS_EN_VF6__BIFC_DOORBELL_ACCESS_EN_VF6__SHIFT                                       0x0
+#define BIFC_DOORBELL_ACCESS_EN_VF6__BIFC_DOORBELL_ACCESS_EN_VF6_MASK                                         0x000FFFFFL
+//BIFC_DOORBELL_ACCESS_EN_VF7
+#define BIFC_DOORBELL_ACCESS_EN_VF7__BIFC_DOORBELL_ACCESS_EN_VF7__SHIFT                                       0x0
+#define BIFC_DOORBELL_ACCESS_EN_VF7__BIFC_DOORBELL_ACCESS_EN_VF7_MASK                                         0x000FFFFFL
+//MISC_SCRATCH
+#define MISC_SCRATCH__MISC_SCRATCH0__SHIFT                                                                    0x0
+#define MISC_SCRATCH__MISC_SCRATCH0_MASK                                                                      0xFFFFFFFFL
+//INTR_LINE_POLARITY
+#define INTR_LINE_POLARITY__INTR_LINE_POLARITY_DEV0__SHIFT                                                    0x0
+#define INTR_LINE_POLARITY__INTR_LINE_POLARITY_DEV0_MASK                                                      0x000000FFL
+//INTR_LINE_ENABLE
+#define INTR_LINE_ENABLE__INTR_LINE_ENABLE_DEV0__SHIFT                                                        0x0
+#define INTR_LINE_ENABLE__INTR_LINE_ENABLE_DEV0_MASK                                                          0x000000FFL
+//OUTSTANDING_VC_ALLOC
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC0_ALLOC__SHIFT                                                0x0
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC1_ALLOC__SHIFT                                                0x2
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC2_ALLOC__SHIFT                                                0x4
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC3_ALLOC__SHIFT                                                0x6
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC4_ALLOC__SHIFT                                                0x8
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC5_ALLOC__SHIFT                                                0xa
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC6_ALLOC__SHIFT                                                0xc
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC7_ALLOC__SHIFT                                                0xe
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_THRD__SHIFT                                                     0x10
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC0_ALLOC__SHIFT                                                0x18
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC1_ALLOC__SHIFT                                                0x1a
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_THRD__SHIFT                                                     0x1c
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC0_ALLOC_MASK                                                  0x00000003L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC1_ALLOC_MASK                                                  0x0000000CL
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC2_ALLOC_MASK                                                  0x00000030L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC3_ALLOC_MASK                                                  0x000000C0L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC4_ALLOC_MASK                                                  0x00000300L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC5_ALLOC_MASK                                                  0x00000C00L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC6_ALLOC_MASK                                                  0x00003000L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC7_ALLOC_MASK                                                  0x0000C000L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_THRD_MASK                                                       0x000F0000L
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC0_ALLOC_MASK                                                  0x03000000L
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC1_ALLOC_MASK                                                  0x0C000000L
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_THRD_MASK                                                       0xF0000000L
+//BIFC_MISC_CTRL0
+#define BIFC_MISC_CTRL0__VWIRE_TARG_UNITID_CHECK_EN__SHIFT                                                    0x0
+#define BIFC_MISC_CTRL0__VWIRE_SRC_UNITID_CHECK_EN__SHIFT                                                     0x1
+#define BIFC_MISC_CTRL0__DMA_VC4_NON_DVM_STS__SHIFT                                                           0x4
+#define BIFC_MISC_CTRL0__DMA_CHAIN_BREAK_IN_RCMODE__SHIFT                                                     0x8
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_P__SHIFT                                                          0x9
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_FLUSH_EN__SHIFT                                                   0xb
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_NPWR_DIS__SHIFT                                                   0xc
+#define BIFC_MISC_CTRL0__GSI_SET_PRECEEDINGWR_DIS__SHIFT                                                      0xd
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_NP__SHIFT                                                         0xe
+#define BIFC_MISC_CTRL0__HRP_CHAIN_DISABLE__SHIFT                                                             0xf
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_LENGTH_CHK_DIS__SHIFT                                                     0x10
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_FAILED_STS_SEL__SHIFT                                                     0x11
+#define BIFC_MISC_CTRL0__DMA_FORCE_VF_AS_PF_SRIOIVEN_LOW__SHIFT                                               0x12
+#define BIFC_MISC_CTRL0__DMA_ADDR_KEEP_PH__SHIFT                                                              0x13
+#define BIFC_MISC_CTRL0__RCC_GMI_TD_FORCE_ZERO__SHIFT                                                         0x14
+#define BIFC_MISC_CTRL0__HST_FLUSH_DEFER_EN__SHIFT                                                            0x15
+#define BIFC_MISC_CTRL0__HST_FLUSH_CLR_LOCK_EN__SHIFT                                                         0x16
+#define BIFC_MISC_CTRL0__STFETCH_BLOCK_IN_RST__SHIFT                                                          0x17
+#define BIFC_MISC_CTRL0__PCIE_CAPABILITY_PROT_DIS__SHIFT                                                      0x18
+#define BIFC_MISC_CTRL0__ATS_MSG_BLOCK_IN_RST__SHIFT                                                          0x19
+#define BIFC_MISC_CTRL0__DMA_2ND_REQ_DIS__SHIFT                                                               0x1a
+#define BIFC_MISC_CTRL0__PORT_DSTATE_BYPASS_MODE__SHIFT                                                       0x1b
+#define BIFC_MISC_CTRL0__PME_TURNOFF_MODE__SHIFT                                                              0x1c
+#define BIFC_MISC_CTRL0__DMA_ALL_RST_PROTECT_STS_SEL__SHIFT                                                   0x1d
+#define BIFC_MISC_CTRL0__HDP_P2P_DIRECT_ADD_ADJUST__SHIFT                                                     0x1e
+#define BIFC_MISC_CTRL0__PCIESWUS_SELECTION__SHIFT                                                            0x1f
+#define BIFC_MISC_CTRL0__VWIRE_TARG_UNITID_CHECK_EN_MASK                                                      0x00000001L
+#define BIFC_MISC_CTRL0__VWIRE_SRC_UNITID_CHECK_EN_MASK                                                       0x00000006L
+#define BIFC_MISC_CTRL0__DMA_VC4_NON_DVM_STS_MASK                                                             0x000000F0L
+#define BIFC_MISC_CTRL0__DMA_CHAIN_BREAK_IN_RCMODE_MASK                                                       0x00000100L
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_P_MASK                                                            0x00000200L
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_FLUSH_EN_MASK                                                     0x00000800L
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_NPWR_DIS_MASK                                                     0x00001000L
+#define BIFC_MISC_CTRL0__GSI_SET_PRECEEDINGWR_DIS_MASK                                                        0x00002000L
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_NP_MASK                                                           0x00004000L
+#define BIFC_MISC_CTRL0__HRP_CHAIN_DISABLE_MASK                                                               0x00008000L
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_LENGTH_CHK_DIS_MASK                                                       0x00010000L
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_FAILED_STS_SEL_MASK                                                       0x00020000L
+#define BIFC_MISC_CTRL0__DMA_FORCE_VF_AS_PF_SRIOIVEN_LOW_MASK                                                 0x00040000L
+#define BIFC_MISC_CTRL0__DMA_ADDR_KEEP_PH_MASK                                                                0x00080000L
+#define BIFC_MISC_CTRL0__RCC_GMI_TD_FORCE_ZERO_MASK                                                           0x00100000L
+#define BIFC_MISC_CTRL0__HST_FLUSH_DEFER_EN_MASK                                                              0x00200000L
+#define BIFC_MISC_CTRL0__HST_FLUSH_CLR_LOCK_EN_MASK                                                           0x00400000L
+#define BIFC_MISC_CTRL0__STFETCH_BLOCK_IN_RST_MASK                                                            0x00800000L
+#define BIFC_MISC_CTRL0__PCIE_CAPABILITY_PROT_DIS_MASK                                                        0x01000000L
+#define BIFC_MISC_CTRL0__ATS_MSG_BLOCK_IN_RST_MASK                                                            0x02000000L
+#define BIFC_MISC_CTRL0__DMA_2ND_REQ_DIS_MASK                                                                 0x04000000L
+#define BIFC_MISC_CTRL0__PORT_DSTATE_BYPASS_MODE_MASK                                                         0x08000000L
+#define BIFC_MISC_CTRL0__PME_TURNOFF_MODE_MASK                                                                0x10000000L
+#define BIFC_MISC_CTRL0__DMA_ALL_RST_PROTECT_STS_SEL_MASK                                                     0x20000000L
+#define BIFC_MISC_CTRL0__HDP_P2P_DIRECT_ADD_ADJUST_MASK                                                       0x40000000L
+#define BIFC_MISC_CTRL0__PCIESWUS_SELECTION_MASK                                                              0x80000000L
+//BIFC_MISC_CTRL1
+#define BIFC_MISC_CTRL1__THT_HST_CPLD_POISON_REPORT__SHIFT                                                    0x0
+#define BIFC_MISC_CTRL1__DMA_REQ_POISON_REPORT__SHIFT                                                         0x1
+#define BIFC_MISC_CTRL1__DMA_REQ_ACSVIO_REPORT__SHIFT                                                         0x2
+#define BIFC_MISC_CTRL1__DMA_RSP_POISON_CPLD_REPORT__SHIFT                                                    0x3
+#define BIFC_MISC_CTRL1__GSI_SMN_WORST_ERR_STSTUS__SHIFT                                                      0x4
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE1_FOR_ERROR__SHIFT                                           0x5
+#define BIFC_MISC_CTRL1__GSI_RDWR_BALANCE_DIS__SHIFT                                                          0x6
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_DROP__SHIFT                                                        0x7
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_STS__SHIFT                                                      0x8
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_DATASTS__SHIFT                                                  0xa
+#define BIFC_MISC_CTRL1__DROP_OTHER_HT_ADDR_REQ__SHIFT                                                        0xc
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE__SHIFT                                                 0xd
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE_VALUE__SHIFT                                           0xe
+#define BIFC_MISC_CTRL1__UPS_SDP_RDY_TIE1__SHIFT                                                              0xf
+#define BIFC_MISC_CTRL1__GMI_RCC_DN_BME_DROP_DIS__SHIFT                                                       0x10
+#define BIFC_MISC_CTRL1__GMI_RCC_EP_BME_DROP_DIS__SHIFT                                                       0x11
+#define BIFC_MISC_CTRL1__GMI_BIH_DN_BME_DROP_DIS__SHIFT                                                       0x12
+#define BIFC_MISC_CTRL1__GMI_BIH_EP_BME_DROP_DIS__SHIFT                                                       0x13
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE0_FOR_ERROR__SHIFT                                           0x14
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE__SHIFT                                                 0x15
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE_VALUE__SHIFT                                           0x16
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_FOR_AERLOG__SHIFT                                                  0x17
+#define BIFC_MISC_CTRL1__GMI_RDSIZED_REQATTR_MASK__SHIFT                                                      0x18
+#define BIFC_MISC_CTRL1__GMI_RDSIZEDDW_REQATTR_MASK__SHIFT                                                    0x19
+#define BIFC_MISC_CTRL1__GMI_WRSIZED_REQATTR_MASK__SHIFT                                                      0x1a
+#define BIFC_MISC_CTRL1__GMI_WRSIZEDFL_REQATTR_MASK__SHIFT                                                    0x1b
+#define BIFC_MISC_CTRL1__GMI_FORCE_NOT_SEND_NON_BASEVC_RSPCREDIT__SHIFT                                       0x1c
+#define BIFC_MISC_CTRL1__GMI_CPLBUF_EN__SHIFT                                                                 0x1d
+#define BIFC_MISC_CTRL1__GMI_MSG_BLOCKLVL_SEL__SHIFT                                                          0x1e
+#define BIFC_MISC_CTRL1__THT_HST_CPLD_POISON_REPORT_MASK                                                      0x00000001L
+#define BIFC_MISC_CTRL1__DMA_REQ_POISON_REPORT_MASK                                                           0x00000002L
+#define BIFC_MISC_CTRL1__DMA_REQ_ACSVIO_REPORT_MASK                                                           0x00000004L
+#define BIFC_MISC_CTRL1__DMA_RSP_POISON_CPLD_REPORT_MASK                                                      0x00000008L
+#define BIFC_MISC_CTRL1__GSI_SMN_WORST_ERR_STSTUS_MASK                                                        0x00000010L
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE1_FOR_ERROR_MASK                                             0x00000020L
+#define BIFC_MISC_CTRL1__GSI_RDWR_BALANCE_DIS_MASK                                                            0x00000040L
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_DROP_MASK                                                          0x00000080L
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_STS_MASK                                                        0x00000300L
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_DATASTS_MASK                                                    0x00000C00L
+#define BIFC_MISC_CTRL1__DROP_OTHER_HT_ADDR_REQ_MASK                                                          0x00001000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE_MASK                                                   0x00002000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE_VALUE_MASK                                             0x00004000L
+#define BIFC_MISC_CTRL1__UPS_SDP_RDY_TIE1_MASK                                                                0x00008000L
+#define BIFC_MISC_CTRL1__GMI_RCC_DN_BME_DROP_DIS_MASK                                                         0x00010000L
+#define BIFC_MISC_CTRL1__GMI_RCC_EP_BME_DROP_DIS_MASK                                                         0x00020000L
+#define BIFC_MISC_CTRL1__GMI_BIH_DN_BME_DROP_DIS_MASK                                                         0x00040000L
+#define BIFC_MISC_CTRL1__GMI_BIH_EP_BME_DROP_DIS_MASK                                                         0x00080000L
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE0_FOR_ERROR_MASK                                             0x00100000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE_MASK                                                   0x00200000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE_VALUE_MASK                                             0x00400000L
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_FOR_AERLOG_MASK                                                    0x00800000L
+#define BIFC_MISC_CTRL1__GMI_RDSIZED_REQATTR_MASK_MASK                                                        0x01000000L
+#define BIFC_MISC_CTRL1__GMI_RDSIZEDDW_REQATTR_MASK_MASK                                                      0x02000000L
+#define BIFC_MISC_CTRL1__GMI_WRSIZED_REQATTR_MASK_MASK                                                        0x04000000L
+#define BIFC_MISC_CTRL1__GMI_WRSIZEDFL_REQATTR_MASK_MASK                                                      0x08000000L
+#define BIFC_MISC_CTRL1__GMI_FORCE_NOT_SEND_NON_BASEVC_RSPCREDIT_MASK                                         0x10000000L
+#define BIFC_MISC_CTRL1__GMI_CPLBUF_EN_MASK                                                                   0x20000000L
+#define BIFC_MISC_CTRL1__GMI_MSG_BLOCKLVL_SEL_MASK                                                            0xC0000000L
+//BIFC_BME_ERR_LOG_LB
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F0__SHIFT                                                    0x0
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F1__SHIFT                                                    0x1
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F0__SHIFT                                              0x10
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F1__SHIFT                                              0x11
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F0_MASK                                                      0x00000001L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F1_MASK                                                      0x00000002L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F0_MASK                                                0x00010000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F1_MASK                                                0x00020000L
+//BIFC_LC_TIMER_CTRL
+#define BIFC_LC_TIMER_CTRL__ASPM_IDLE_TIMER_SCALE__SHIFT                                                      0x0
+#define BIFC_LC_TIMER_CTRL__L1_EXIT_TIMER_SCALE__SHIFT                                                        0x10
+#define BIFC_LC_TIMER_CTRL__ASPM_IDLE_TIMER_SCALE_MASK                                                        0x0000FFFFL
+#define BIFC_LC_TIMER_CTRL__L1_EXIT_TIMER_SCALE_MASK                                                          0xFFFF0000L
+//BIFC_RCCBIH_BME_ERR_LOG0
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F0__SHIFT                                            0x0
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F1__SHIFT                                            0x1
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F0__SHIFT                                      0x10
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F1__SHIFT                                      0x11
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F0_MASK                                              0x00000001L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F1_MASK                                              0x00000002L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F0_MASK                                        0x00010000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F1_MASK                                        0x00020000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F0__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F0__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_IDO_DEV0_F0__SHIFT                                      0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F0__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F0__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F0__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F0__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_NONIDO_DEV0_F0__SHIFT                                   0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F1__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F1__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_IDO_DEV0_F1__SHIFT                                      0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F1__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F1__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F1__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F1__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_NONIDO_DEV0_F1__SHIFT                                   0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F0_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F0_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_IDO_DEV0_F0_MASK                                        0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F0_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F0_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F0_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F0_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_NONIDO_DEV0_F0_MASK                                     0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F1_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F1_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_IDO_DEV0_F1_MASK                                        0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F1_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F1_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F1_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F1_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_NONIDO_DEV0_F1_MASK                                     0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F2__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F2__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_IDO_DEV0_F2__SHIFT                                      0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F2__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F2__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F2__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F2__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_NONIDO_DEV0_F2__SHIFT                                   0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F3__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F3__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_IDO_DEV0_F3__SHIFT                                      0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F3__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F3__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F3__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F3__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_NONIDO_DEV0_F3__SHIFT                                   0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F2_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F2_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_IDO_DEV0_F2_MASK                                        0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F2_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F2_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F2_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F2_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_NONIDO_DEV0_F2_MASK                                     0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F3_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F3_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_IDO_DEV0_F3_MASK                                        0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F3_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F3_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F3_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F3_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_NONIDO_DEV0_F3_MASK                                     0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F4__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F4__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_IDO_DEV0_F4__SHIFT                                      0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F4__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F4__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F4__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F4__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_NONIDO_DEV0_F4__SHIFT                                   0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F5__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F5__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_IDO_DEV0_F5__SHIFT                                      0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F5__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F5__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F5__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F5__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_NONIDO_DEV0_F5__SHIFT                                   0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F4_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F4_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_IDO_DEV0_F4_MASK                                        0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F4_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F4_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F4_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F4_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_NONIDO_DEV0_F4_MASK                                     0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F5_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F5_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_IDO_DEV0_F5_MASK                                        0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F5_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F5_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F5_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F5_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_NONIDO_DEV0_F5_MASK                                     0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F6__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F6__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_IDO_DEV0_F6__SHIFT                                      0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F6__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F6__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F6__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F6__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_NONIDO_DEV0_F6__SHIFT                                   0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F7__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F7__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_IDO_DEV0_F7__SHIFT                                      0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F7__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F7__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F7__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F7__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_NONIDO_DEV0_F7__SHIFT                                   0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F6_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F6_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_IDO_DEV0_F6_MASK                                        0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F6_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F6_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F6_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F6_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_NONIDO_DEV0_F6_MASK                                     0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F7_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F7_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_IDO_DEV0_F7_MASK                                        0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F7_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F7_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F7_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F7_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_NONIDO_DEV0_F7_MASK                                     0xC0000000L
+//BIFC_DMA_ATTR_CNTL2_DEV0
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F0__SHIFT                               0x0
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F1__SHIFT                               0x4
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F2__SHIFT                               0x8
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F3__SHIFT                               0xc
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F4__SHIFT                               0x10
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F5__SHIFT                               0x14
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F6__SHIFT                               0x18
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F7__SHIFT                               0x1c
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F0_MASK                                 0x00000001L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F1_MASK                                 0x00000010L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F2_MASK                                 0x00000100L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F3_MASK                                 0x00001000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F4_MASK                                 0x00010000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F5_MASK                                 0x00100000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F6_MASK                                 0x01000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F7_MASK                                 0x10000000L
+//BME_DUMMY_CNTL_0
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F0__SHIFT                                                     0x0
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F1__SHIFT                                                     0x2
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F2__SHIFT                                                     0x4
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F3__SHIFT                                                     0x6
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F4__SHIFT                                                     0x8
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F5__SHIFT                                                     0xa
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F6__SHIFT                                                     0xc
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F7__SHIFT                                                     0xe
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F0_MASK                                                       0x00000003L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F1_MASK                                                       0x0000000CL
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F2_MASK                                                       0x00000030L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F3_MASK                                                       0x000000C0L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F4_MASK                                                       0x00000300L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F5_MASK                                                       0x00000C00L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F6_MASK                                                       0x00003000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F7_MASK                                                       0x0000C000L
+//BIFC_THT_CNTL
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_RD_VC0__SHIFT                                                         0x0
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC0__SHIFT                                                         0x4
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC1__SHIFT                                                         0x8
+#define BIFC_THT_CNTL__UR_OVRD_FOR_ECRC_EN__SHIFT                                                             0x10
+#define BIFC_THT_CNTL__THT_NTB_VC0_APER0_ADSC_PUSH_DIS__SHIFT                                                 0x18
+#define BIFC_THT_CNTL__THT_NTB_VC0_OTHAPER_ADSC_PUSH_DIS__SHIFT                                               0x19
+#define BIFC_THT_CNTL__THT_NTB_VC1_APER0_ADSC_PUSH_DIS__SHIFT                                                 0x1a
+#define BIFC_THT_CNTL__THT_NTB_VC1_OTHAPER_ADSC_PUSH_DIS__SHIFT                                               0x1b
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_RD_VC0_MASK                                                           0x0000000FL
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC0_MASK                                                           0x000000F0L
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC1_MASK                                                           0x00000F00L
+#define BIFC_THT_CNTL__UR_OVRD_FOR_ECRC_EN_MASK                                                               0x00010000L
+#define BIFC_THT_CNTL__THT_NTB_VC0_APER0_ADSC_PUSH_DIS_MASK                                                   0x01000000L
+#define BIFC_THT_CNTL__THT_NTB_VC0_OTHAPER_ADSC_PUSH_DIS_MASK                                                 0x02000000L
+#define BIFC_THT_CNTL__THT_NTB_VC1_APER0_ADSC_PUSH_DIS_MASK                                                   0x04000000L
+#define BIFC_THT_CNTL__THT_NTB_VC1_OTHAPER_ADSC_PUSH_DIS_MASK                                                 0x08000000L
+//BIFC_HSTARB_CNTL
+#define BIFC_HSTARB_CNTL__SLVARB_MODE__SHIFT                                                                  0x0
+#define BIFC_HSTARB_CNTL__CFG_BLOCK_P_EN__SHIFT                                                               0x8
+#define BIFC_HSTARB_CNTL__SLVARB_MODE_MASK                                                                    0x00000003L
+#define BIFC_HSTARB_CNTL__CFG_BLOCK_P_EN_MASK                                                                 0x00000100L
+//BIFC_GSI_CNTL
+#define BIFC_GSI_CNTL__GSI_SDP_RSP_ARB_MODE__SHIFT                                                            0x0
+#define BIFC_GSI_CNTL__GSI_CPL_RSP_ARB_MODE__SHIFT                                                            0x2
+#define BIFC_GSI_CNTL__GSI_CPL_INTERLEAVING_EN__SHIFT                                                         0x5
+#define BIFC_GSI_CNTL__GSI_CPL_PCR_EP_CAUSE_UR_EN__SHIFT                                                      0x6
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_P_EP_CAUSE_UR_EN__SHIFT                                                    0x7
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_NP_EP_CAUSE_UR_EN__SHIFT                                                   0x8
+#define BIFC_GSI_CNTL__GSI_SDP_REQ_ARB_MODE__SHIFT                                                            0xa
+#define BIFC_GSI_CNTL__GSI_SMN_REQ_ARB_MODE__SHIFT                                                            0xc
+#define BIFC_GSI_CNTL__GSI_SMN_PARITY_CHK_BE_MSK__SHIFT                                                       0xf
+#define BIFC_GSI_CNTL__GSI_SMN_BURST_EN__SHIFT                                                                0x10
+#define BIFC_GSI_CNTL__GSI_SMN_256B_SPLIT_64B_EN__SHIFT                                                       0x11
+#define BIFC_GSI_CNTL__SMN_PP_PIPE_ENABLE__SHIFT                                                              0x1b
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_FBFLUSH__SHIFT                                                    0x1c
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPFLUSH__SHIFT                                                   0x1d
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPRD__SHIFT                                                      0x1e
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_ALL__SHIFT                                                        0x1f
+#define BIFC_GSI_CNTL__GSI_SDP_RSP_ARB_MODE_MASK                                                              0x00000003L
+#define BIFC_GSI_CNTL__GSI_CPL_RSP_ARB_MODE_MASK                                                              0x0000001CL
+#define BIFC_GSI_CNTL__GSI_CPL_INTERLEAVING_EN_MASK                                                           0x00000020L
+#define BIFC_GSI_CNTL__GSI_CPL_PCR_EP_CAUSE_UR_EN_MASK                                                        0x00000040L
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_P_EP_CAUSE_UR_EN_MASK                                                      0x00000080L
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_NP_EP_CAUSE_UR_EN_MASK                                                     0x00000100L
+#define BIFC_GSI_CNTL__GSI_SDP_REQ_ARB_MODE_MASK                                                              0x00000C00L
+#define BIFC_GSI_CNTL__GSI_SMN_REQ_ARB_MODE_MASK                                                              0x00003000L
+#define BIFC_GSI_CNTL__GSI_SMN_PARITY_CHK_BE_MSK_MASK                                                         0x00008000L
+#define BIFC_GSI_CNTL__GSI_SMN_BURST_EN_MASK                                                                  0x00010000L
+#define BIFC_GSI_CNTL__GSI_SMN_256B_SPLIT_64B_EN_MASK                                                         0x00020000L
+#define BIFC_GSI_CNTL__SMN_PP_PIPE_ENABLE_MASK                                                                0x08000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_FBFLUSH_MASK                                                      0x10000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPFLUSH_MASK                                                     0x20000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPRD_MASK                                                        0x40000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_ALL_MASK                                                          0x80000000L
+//BIFC_PCIEFUNC_CNTL
+#define BIFC_PCIEFUNC_CNTL__DMA_NON_PCIEFUNC_BUSDEVFUNC__SHIFT                                                0x0
+#define BIFC_PCIEFUNC_CNTL__DMA_NON_PCIEFUNC_BUSDEVFUNC_MASK                                                  0x0000FFFFL
+//BIFC_PASID_CHECK_DIS
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F0__SHIFT                                                  0x0
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F1__SHIFT                                                  0x1
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F0_MASK                                                    0x00000001L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F1_MASK                                                    0x00000002L
+//BIFC_SDP_CNTL_0
+#define BIFC_SDP_CNTL_0__HRP_SDP_DISCON_HYSTERESIS__SHIFT                                                     0x0
+#define BIFC_SDP_CNTL_0__GSI_SDP_DISCON_HYSTERESIS__SHIFT                                                     0x8
+#define BIFC_SDP_CNTL_0__GMI_DNS_SDP_DISCON_HYSTERESIS__SHIFT                                                 0x10
+#define BIFC_SDP_CNTL_0__GMI_UPS_SDP_DISCON_HYSTERESIS__SHIFT                                                 0x18
+#define BIFC_SDP_CNTL_0__HRP_SDP_DISCON_HYSTERESIS_MASK                                                       0x000000FFL
+#define BIFC_SDP_CNTL_0__GSI_SDP_DISCON_HYSTERESIS_MASK                                                       0x0000FF00L
+#define BIFC_SDP_CNTL_0__GMI_DNS_SDP_DISCON_HYSTERESIS_MASK                                                   0x00FF0000L
+#define BIFC_SDP_CNTL_0__GMI_UPS_SDP_DISCON_HYSTERESIS_MASK                                                   0xFF000000L
+//BIFC_SDP_CNTL_1
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_DIS__SHIFT                                                            0x0
+#define BIFC_SDP_CNTL_1__GSI_SDP_DISCON_DIS__SHIFT                                                            0x1
+#define BIFC_SDP_CNTL_1__GMI_DNS_SDP_DISCON_DIS__SHIFT                                                        0x2
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_DIS__SHIFT                                                        0x3
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_VLINK_NONL0_ONLY__SHIFT                                               0x4
+#define BIFC_SDP_CNTL_1__NP_KEEP_GOING_STALL_P__SHIFT                                                         0x5
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_VLINK_NONL0_ONLY__SHIFT                                           0x7
+#define BIFC_SDP_CNTL_1__ATOMIC_STALL_BY_RDWR_EN__SHIFT                                                       0x8
+#define BIFC_SDP_CNTL_1__POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT                                            0x9
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_DIS_MASK                                                              0x00000001L
+#define BIFC_SDP_CNTL_1__GSI_SDP_DISCON_DIS_MASK                                                              0x00000002L
+#define BIFC_SDP_CNTL_1__GMI_DNS_SDP_DISCON_DIS_MASK                                                          0x00000004L
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_DIS_MASK                                                          0x00000008L
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_VLINK_NONL0_ONLY_MASK                                                 0x00000010L
+#define BIFC_SDP_CNTL_1__NP_KEEP_GOING_STALL_P_MASK                                                           0x00000020L
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_VLINK_NONL0_ONLY_MASK                                             0x00000080L
+#define BIFC_SDP_CNTL_1__ATOMIC_STALL_BY_RDWR_EN_MASK                                                         0x00000100L
+#define BIFC_SDP_CNTL_1__POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK                                              0x00000200L
+//BIFC_PASID_STS
+#define BIFC_PASID_STS__PASID_STS__SHIFT                                                                      0x0
+#define BIFC_PASID_STS__PASID_STS_MASK                                                                        0x0000000FL
+//BIFC_ATHUB_ACT_CNTL
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_RSP_STS_TYPE__SHIFT                                                0x0
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_SLFR_DATAERR_RSP_STS_TYPE__SHIFT                                           0x3
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_REQ_DROP_DIS__SHIFT                                                0x8
+#define BIFC_ATHUB_ACT_CNTL__GSI_ATHUB_ACT_FLUSH_TRIGGER__SHIFT                                               0x9
+#define BIFC_ATHUB_ACT_CNTL__GMI_ATHUB_ACT_FLUSH_TRIGGER__SHIFT                                               0xa
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_RSP_STS_TYPE_MASK                                                  0x00000007L
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_SLFR_DATAERR_RSP_STS_TYPE_MASK                                             0x00000038L
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_REQ_DROP_DIS_MASK                                                  0x00000100L
+#define BIFC_ATHUB_ACT_CNTL__GSI_ATHUB_ACT_FLUSH_TRIGGER_MASK                                                 0x00000200L
+#define BIFC_ATHUB_ACT_CNTL__GMI_ATHUB_ACT_FLUSH_TRIGGER_MASK                                                 0x00000400L
+//BIFC_PERF_CNTL_0
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_EN__SHIFT                                                          0x0
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_EN__SHIFT                                                          0x1
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_RESET__SHIFT                                                       0x8
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_RESET__SHIFT                                                       0x9
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_SEL__SHIFT                                                         0x10
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_SEL__SHIFT                                                         0x18
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_EN_MASK                                                            0x00000001L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_EN_MASK                                                            0x00000002L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_RESET_MASK                                                         0x00000100L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_RESET_MASK                                                         0x00000200L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_SEL_MASK                                                           0x007F0000L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_SEL_MASK                                                           0x7F000000L
+//BIFC_PERF_CNTL_1
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_EN__SHIFT                                                           0x0
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_EN__SHIFT                                                           0x1
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_RESET__SHIFT                                                        0x4
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_RESET__SHIFT                                                        0x5
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_SEL__SHIFT                                                          0x8
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_SEL__SHIFT                                                          0x10
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_EN_MASK                                                             0x00000001L
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_EN_MASK                                                             0x00000002L
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_RESET_MASK                                                          0x00000010L
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_RESET_MASK                                                          0x00000020L
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_SEL_MASK                                                            0x0000FF00L
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_SEL_MASK                                                            0x01FF0000L
+//BIFC_PERF_CNT_MMIO_RD_L32BIT
+#define BIFC_PERF_CNT_MMIO_RD_L32BIT__PERF_CNT_MMIO_RD_VALUE_L32BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_RD_L32BIT__PERF_CNT_MMIO_RD_VALUE_L32BIT_MASK                                      0xFFFFFFFFL
+//BIFC_PERF_CNT_MMIO_WR_L32BIT
+#define BIFC_PERF_CNT_MMIO_WR_L32BIT__PERF_CNT_MMIO_WR_VALUE_L32BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_WR_L32BIT__PERF_CNT_MMIO_WR_VALUE_L32BIT_MASK                                      0xFFFFFFFFL
+//BIFC_PERF_CNT_DMA_RD_L32BIT
+#define BIFC_PERF_CNT_DMA_RD_L32BIT__PERF_CNT_DMA_RD_VALUE_L32BIT__SHIFT                                      0x0
+#define BIFC_PERF_CNT_DMA_RD_L32BIT__PERF_CNT_DMA_RD_VALUE_L32BIT_MASK                                        0xFFFFFFFFL
+//BIFC_PERF_CNT_DMA_WR_L32BIT
+#define BIFC_PERF_CNT_DMA_WR_L32BIT__PERF_CNT_DMA_WR_VALUE_L32BIT__SHIFT                                      0x0
+#define BIFC_PERF_CNT_DMA_WR_L32BIT__PERF_CNT_DMA_WR_VALUE_L32BIT_MASK                                        0xFFFFFFFFL
+//NBIF_REGIF_ERRSET_CTRL
+#define NBIF_REGIF_ERRSET_CTRL__DROP_NONPF_MMREGREQ_SETERR_DIS__SHIFT                                         0x0
+#define NBIF_REGIF_ERRSET_CTRL__DROP_NONPF_MMREGREQ_SETERR_DIS_MASK                                           0x00000001L
+//BIFC_SDP_CNTL_2
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS__SHIFT                                                    0x0
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS_H__SHIFT                                                  0x8
+#define BIFC_SDP_CNTL_2__HRP_SDP_DISCON_HYSTERESIS_H__SHIFT                                                   0x10
+#define BIFC_SDP_CNTL_2__GSI_SDP_DISCON_HYSTERESIS_H__SHIFT                                                   0x18
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS_MASK                                                      0x000000FFL
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS_H_MASK                                                    0x00000F00L
+#define BIFC_SDP_CNTL_2__HRP_SDP_DISCON_HYSTERESIS_H_MASK                                                     0x000F0000L
+#define BIFC_SDP_CNTL_2__GSI_SDP_DISCON_HYSTERESIS_H_MASK                                                     0x0F000000L
+//NBIF_PGMST_CTRL
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_HYSTERESIS__SHIFT                                                        0x0
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_EN__SHIFT                                                                0x8
+#define NBIF_PGMST_CTRL__NBIF_CFG_IDLENESS_COUNT_EN__SHIFT                                                    0xa
+#define NBIF_PGMST_CTRL__NBIF_CFG_FW_PG_EXIT_EN__SHIFT                                                        0xe
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_HYSTERESIS_MASK                                                          0x000000FFL
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_EN_MASK                                                                  0x00000100L
+#define NBIF_PGMST_CTRL__NBIF_CFG_IDLENESS_COUNT_EN_MASK                                                      0x00003C00L
+#define NBIF_PGMST_CTRL__NBIF_CFG_FW_PG_EXIT_EN_MASK                                                          0x0000C000L
+//NBIF_PGSLV_CTRL
+#define NBIF_PGSLV_CTRL__NBIF_CFG_IDLE_HYSTERESIS__SHIFT                                                      0x0
+#define NBIF_PGSLV_CTRL__NBIF_CFG_IDLE_HYSTERESIS_MASK                                                        0x0000001FL
+//NBIF_PG_MISC_CTRL
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_0_IDLE_HYSTERESIS__SHIFT                                          0x0
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_1_IDLE_HYSTERESIS__SHIFT                                          0x5
+#define NBIF_PG_MISC_CTRL__NBIF_PG_ENDP_D3_ONLY__SHIFT                                                        0xa
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM1__SHIFT                                                           0xd
+#define NBIF_PG_MISC_CTRL__NBIF_PG_DS_ALLOW_DIS__SHIFT                                                        0xe
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM2__SHIFT                                                           0x10
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_REFCLK_CYCLE_FOR_200NS__SHIFT                                             0x18
+#define NBIF_PG_MISC_CTRL__NBIF_PG_PCIE_NBIF_LD_MASK__SHIFT                                                   0x1e
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_PG_EXIT_OVERRIDE__SHIFT                                                   0x1f
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_0_IDLE_HYSTERESIS_MASK                                            0x0000001FL
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_1_IDLE_HYSTERESIS_MASK                                            0x000003E0L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_ENDP_D3_ONLY_MASK                                                          0x00000400L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM1_MASK                                                             0x00002000L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_DS_ALLOW_DIS_MASK                                                          0x00004000L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM2_MASK                                                             0x00010000L
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_REFCLK_CYCLE_FOR_200NS_MASK                                               0x3F000000L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_PCIE_NBIF_LD_MASK_MASK                                                     0x40000000L
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_PG_EXIT_OVERRIDE_MASK                                                     0x80000000L
+//SMN_MST_EP_CNTL3
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF0__SHIFT                                                0x0
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF1__SHIFT                                                0x1
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF2__SHIFT                                                0x2
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF3__SHIFT                                                0x3
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF4__SHIFT                                                0x4
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF5__SHIFT                                                0x5
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF6__SHIFT                                                0x6
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF7__SHIFT                                                0x7
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF0_MASK                                                  0x00000001L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF1_MASK                                                  0x00000002L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF2_MASK                                                  0x00000004L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF3_MASK                                                  0x00000008L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF4_MASK                                                  0x00000010L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF5_MASK                                                  0x00000020L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF6_MASK                                                  0x00000040L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF7_MASK                                                  0x00000080L
+//SMN_MST_EP_CNTL4
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF0__SHIFT                                                0x0
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF1__SHIFT                                                0x1
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF2__SHIFT                                                0x2
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF3__SHIFT                                                0x3
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF4__SHIFT                                                0x4
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF5__SHIFT                                                0x5
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF6__SHIFT                                                0x6
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF7__SHIFT                                                0x7
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF0_MASK                                                  0x00000001L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF1_MASK                                                  0x00000002L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF2_MASK                                                  0x00000004L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF3_MASK                                                  0x00000008L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF4_MASK                                                  0x00000010L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF5_MASK                                                  0x00000020L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF6_MASK                                                  0x00000040L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF7_MASK                                                  0x00000080L
+//SMN_MST_CNTL1
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_UPS__SHIFT                                                    0x0
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_DNS_DEV0__SHIFT                                               0x10
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_UPS_MASK                                                      0x00000001L
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_DNS_DEV0_MASK                                                 0x00010000L
+//SMN_MST_EP_CNTL5
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF0__SHIFT                                         0x0
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF1__SHIFT                                         0x1
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF2__SHIFT                                         0x2
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF3__SHIFT                                         0x3
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF4__SHIFT                                         0x4
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF5__SHIFT                                         0x5
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF6__SHIFT                                         0x6
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF7__SHIFT                                         0x7
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF0_MASK                                           0x00000001L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF1_MASK                                           0x00000002L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF2_MASK                                           0x00000004L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF3_MASK                                           0x00000008L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF4_MASK                                           0x00000010L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF5_MASK                                           0x00000020L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF6_MASK                                           0x00000040L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF7_MASK                                           0x00000080L
+//BIF_SELFRING_BUFFER_VID
+#define BIF_SELFRING_BUFFER_VID__DOORBELL_MONITOR_CID__SHIFT                                                  0x0
+#define BIF_SELFRING_BUFFER_VID__RAS_CNTLR_INTR_CID__SHIFT                                                    0x8
+#define BIF_SELFRING_BUFFER_VID__RAS_ATHUB_ERR_EVENT_INTR_CID__SHIFT                                          0x10
+#define BIF_SELFRING_BUFFER_VID__DOORBELL_MONITOR_CID_MASK                                                    0x000000FFL
+#define BIF_SELFRING_BUFFER_VID__RAS_CNTLR_INTR_CID_MASK                                                      0x0000FF00L
+#define BIF_SELFRING_BUFFER_VID__RAS_ATHUB_ERR_EVENT_INTR_CID_MASK                                            0x00FF0000L
+//BIF_SELFRING_VECTOR_CNTL
+#define BIF_SELFRING_VECTOR_CNTL__MISC_DB_MNTR_INTR_DIS__SHIFT                                                0x0
+#define BIF_SELFRING_VECTOR_CNTL__DB_MNTR_TS_FROM__SHIFT                                                      0x1
+#define BIF_SELFRING_VECTOR_CNTL__MISC_DB_MNTR_INTR_DIS_MASK                                                  0x00000001L
+#define BIF_SELFRING_VECTOR_CNTL__DB_MNTR_TS_FROM_MASK                                                        0x00000002L
+//NBIF_STRAP_WRITE_CTRL
+#define NBIF_STRAP_WRITE_CTRL__NBIF_STRAP_WRITE_ONCE_ENABLE__SHIFT                                            0x0
+#define NBIF_STRAP_WRITE_CTRL__NBIF_STRAP_WRITE_ONCE_ENABLE_MASK                                              0x00000001L
+//NBIF_INTX_DSTATE_MISC_CNTL
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_EP__SHIFT                                      0x0
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_DN__SHIFT                                      0x1
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_SWUS__SHIFT                                    0x2
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_EP__SHIFT                                         0x3
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_DN__SHIFT                                         0x4
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_EP__SHIFT                                                     0x5
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_DN__SHIFT                                                     0x6
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_SWUS__SHIFT                                                   0x7
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_EP_MASK                                        0x00000001L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_DN_MASK                                        0x00000002L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_SWUS_MASK                                      0x00000004L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_EP_MASK                                           0x00000008L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_DN_MASK                                           0x00000010L
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_EP_MASK                                                       0x00000020L
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_DN_MASK                                                       0x00000040L
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_SWUS_MASK                                                     0x00000080L
+//NBIF_PENDING_MISC_CNTL
+#define NBIF_PENDING_MISC_CNTL__FLR_MST_PEND_CHK_DIS__SHIFT                                                   0x0
+#define NBIF_PENDING_MISC_CNTL__FLR_SLV_PEND_CHK_DIS__SHIFT                                                   0x1
+#define NBIF_PENDING_MISC_CNTL__FLR_MST_PEND_CHK_DIS_MASK                                                     0x00000001L
+#define NBIF_PENDING_MISC_CNTL__FLR_SLV_PEND_CHK_DIS_MASK                                                     0x00000002L
+//BIF_GMI_WRR_WEIGHT
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_COUNTER_MODE__SHIFT                                               0x1d
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_MODE__SHIFT                                                       0x1e
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_SIZE_MODE__SHIFT                                                  0x1f
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_COUNTER_MODE_MASK                                                 0x20000000L
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_MODE_MASK                                                         0x40000000L
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_SIZE_MODE_MASK                                                    0x80000000L
+//BIF_GMI_WRR_WEIGHT2
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY0_WEIGHT__SHIFT                                                     0x0
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY1_WEIGHT__SHIFT                                                     0x8
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY2_WEIGHT__SHIFT                                                     0x10
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY3_WEIGHT__SHIFT                                                     0x18
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY0_WEIGHT_MASK                                                       0x000000FFL
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY1_WEIGHT_MASK                                                       0x0000FF00L
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY2_WEIGHT_MASK                                                       0x00FF0000L
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY3_WEIGHT_MASK                                                       0xFF000000L
+//BIF_GMI_WRR_WEIGHT3
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY4_WEIGHT__SHIFT                                                     0x0
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY5_WEIGHT__SHIFT                                                     0x8
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY6_WEIGHT__SHIFT                                                     0x10
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY7_WEIGHT__SHIFT                                                     0x18
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY4_WEIGHT_MASK                                                       0x000000FFL
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY5_WEIGHT_MASK                                                       0x0000FF00L
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY6_WEIGHT_MASK                                                       0x00FF0000L
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY7_WEIGHT_MASK                                                       0xFF000000L
+//NBIF_PWRBRK_REQUEST
+#define NBIF_PWRBRK_REQUEST__NBIF_PWRBRK_REQUEST__SHIFT                                                       0x0
+#define NBIF_PWRBRK_REQUEST__NBIF_PWRBRK_REQUEST_MASK                                                         0x00000001L
+//BIF_ATOMIC_ERR_LOG_DEV0_F0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_OPCODE_DEV0_F0__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_REQEN_LOW_DEV0_F0__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_LENGTH_DEV0_F0__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_NR_DEV0_F0__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_NR_DEV0_F0__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_OPCODE_DEV0_F0_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_REQEN_LOW_DEV0_F0_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_LENGTH_DEV0_F0_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_NR_DEV0_F0_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_NR_DEV0_F0_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV0_F1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_OPCODE_DEV0_F1__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_REQEN_LOW_DEV0_F1__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_LENGTH_DEV0_F1__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_NR_DEV0_F1__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_OPCODE_DEV0_F1__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F1__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_LENGTH_DEV0_F1__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_NR_DEV0_F1__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_OPCODE_DEV0_F1_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_REQEN_LOW_DEV0_F1_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_LENGTH_DEV0_F1_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_NR_DEV0_F1_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_OPCODE_DEV0_F1_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F1_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_LENGTH_DEV0_F1_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_NR_DEV0_F1_MASK                                           0x00080000L
+//BIF_DMA_MP4_ERR_LOG
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_VC4_NON_DVM_ERR__SHIFT                                                    0x0
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_ATOMIC_REQEN_LOW_ERR__SHIFT                                               0x1
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_VC4_NON_DVM_ERR__SHIFT                                              0x10
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_ATOMIC_REQEN_LOW_ERR__SHIFT                                         0x11
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_VC4_NON_DVM_ERR_MASK                                                      0x00000001L
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_ATOMIC_REQEN_LOW_ERR_MASK                                                 0x00000002L
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_VC4_NON_DVM_ERR_MASK                                                0x00010000L
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_ATOMIC_REQEN_LOW_ERR_MASK                                           0x00020000L
+//BIF_PASID_ERR_LOG
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F0__SHIFT                                                           0x0
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F1__SHIFT                                                           0x1
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F0_MASK                                                             0x00000001L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F1_MASK                                                             0x00000002L
+//BIF_PASID_ERR_CLR
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F0__SHIFT                                                       0x0
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F1__SHIFT                                                       0x1
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F0_MASK                                                         0x00000001L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F1_MASK                                                         0x00000002L
+//NBIF_VWIRE_CTRL
+#define NBIF_VWIRE_CTRL__NBIF_SMN_VWR_DIS__SHIFT                                                              0x0
+#define NBIF_VWIRE_CTRL__SMN_VWR_RESET_DELAY_CNT__SHIFT                                                       0x4
+#define NBIF_VWIRE_CTRL__SMN_VWR_POSTED__SHIFT                                                                0x8
+#define NBIF_VWIRE_CTRL__NBIF_SDP_UPS_VWR_DIS__SHIFT                                                          0x10
+#define NBIF_VWIRE_CTRL__SDP_VWR_RESET_DELAY_CNT__SHIFT                                                       0x14
+#define NBIF_VWIRE_CTRL__SDP_VWR_BLOCKLVL__SHIFT                                                              0x1a
+#define NBIF_VWIRE_CTRL__NBIF_SMN_VWR_DIS_MASK                                                                0x00000001L
+#define NBIF_VWIRE_CTRL__SMN_VWR_RESET_DELAY_CNT_MASK                                                         0x000000F0L
+#define NBIF_VWIRE_CTRL__SMN_VWR_POSTED_MASK                                                                  0x00000100L
+#define NBIF_VWIRE_CTRL__NBIF_SDP_UPS_VWR_DIS_MASK                                                            0x00010000L
+#define NBIF_VWIRE_CTRL__SDP_VWR_RESET_DELAY_CNT_MASK                                                         0x00F00000L
+#define NBIF_VWIRE_CTRL__SDP_VWR_BLOCKLVL_MASK                                                                0x0C000000L
+//NBIF_SMN_VWR_VCHG_DIS_CTRL
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET0_DIS__SHIFT                                              0x0
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET1_DIS__SHIFT                                              0x1
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET2_DIS__SHIFT                                              0x2
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET3_DIS__SHIFT                                              0x3
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET4_DIS__SHIFT                                              0x4
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET5_DIS__SHIFT                                              0x5
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET6_DIS__SHIFT                                              0x6
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET0_DIS_MASK                                                0x00000001L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET1_DIS_MASK                                                0x00000002L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET2_DIS_MASK                                                0x00000004L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET3_DIS_MASK                                                0x00000008L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET4_DIS_MASK                                                0x00000010L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET5_DIS_MASK                                                0x00000020L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET6_DIS_MASK                                                0x00000040L
+//NBIF_SMN_VWR_VCHG_RST_CTRL0
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET0_RST_DEF_REV__SHIFT                                     0x0
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET1_RST_DEF_REV__SHIFT                                     0x1
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET2_RST_DEF_REV__SHIFT                                     0x2
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET3_RST_DEF_REV__SHIFT                                     0x3
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET4_RST_DEF_REV__SHIFT                                     0x4
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET5_RST_DEF_REV__SHIFT                                     0x5
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET6_RST_DEF_REV__SHIFT                                     0x6
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET0_RST_DEF_REV_MASK                                       0x00000001L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET1_RST_DEF_REV_MASK                                       0x00000002L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET2_RST_DEF_REV_MASK                                       0x00000004L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET3_RST_DEF_REV_MASK                                       0x00000008L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET4_RST_DEF_REV_MASK                                       0x00000010L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET5_RST_DEF_REV_MASK                                       0x00000020L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET6_RST_DEF_REV_MASK                                       0x00000040L
+//NBIF_SMN_VWR_VCHG_TRIG
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET0_TRIG__SHIFT                                                 0x0
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET1_TRIG__SHIFT                                                 0x1
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET2_TRIG__SHIFT                                                 0x2
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET3_TRIG__SHIFT                                                 0x3
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET4_TRIG__SHIFT                                                 0x4
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET5_TRIG__SHIFT                                                 0x5
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET6_TRIG__SHIFT                                                 0x6
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET0_TRIG_MASK                                                   0x00000001L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET1_TRIG_MASK                                                   0x00000002L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET2_TRIG_MASK                                                   0x00000004L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET3_TRIG_MASK                                                   0x00000008L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET4_TRIG_MASK                                                   0x00000010L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET5_TRIG_MASK                                                   0x00000020L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET6_TRIG_MASK                                                   0x00000040L
+//NBIF_SMN_VWR_WTRIG_CNTL
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET0_DIS__SHIFT                                                0x0
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET1_DIS__SHIFT                                                0x1
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET2_DIS__SHIFT                                                0x2
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET3_DIS__SHIFT                                                0x3
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET4_DIS__SHIFT                                                0x4
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET5_DIS__SHIFT                                                0x5
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET6_DIS__SHIFT                                                0x6
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET0_DIS_MASK                                                  0x00000001L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET1_DIS_MASK                                                  0x00000002L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET2_DIS_MASK                                                  0x00000004L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET3_DIS_MASK                                                  0x00000008L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET4_DIS_MASK                                                  0x00000010L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET5_DIS_MASK                                                  0x00000020L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET6_DIS_MASK                                                  0x00000040L
+//NBIF_SMN_VWR_VCHG_DIS_CTRL_1
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET0_DIFFDET_DEF_REV__SHIFT                                0x0
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET1_DIFFDET_DEF_REV__SHIFT                                0x1
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET2_DIFFDET_DEF_REV__SHIFT                                0x2
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET3_DIFFDET_DEF_REV__SHIFT                                0x3
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET4_DIFFDET_DEF_REV__SHIFT                                0x4
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET5_DIFFDET_DEF_REV__SHIFT                                0x5
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET6_DIFFDET_DEF_REV__SHIFT                                0x6
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET0_DIFFDET_DEF_REV_MASK                                  0x00000001L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET1_DIFFDET_DEF_REV_MASK                                  0x00000002L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET2_DIFFDET_DEF_REV_MASK                                  0x00000004L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET3_DIFFDET_DEF_REV_MASK                                  0x00000008L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET4_DIFFDET_DEF_REV_MASK                                  0x00000010L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET5_DIFFDET_DEF_REV_MASK                                  0x00000020L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET6_DIFFDET_DEF_REV_MASK                                  0x00000040L
+//NBIF_MGCG_CTRL_LCLK
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_EN_LCLK__SHIFT                                                         0x0
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_MODE_LCLK__SHIFT                                                       0x1
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HYSTERESIS_LCLK__SHIFT                                                 0x2
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HST_DIS_LCLK__SHIFT                                                    0xa
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_DMA_DIS_LCLK__SHIFT                                                    0xb
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_REG_DIS_LCLK__SHIFT                                                    0xc
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_AER_DIS_LCLK__SHIFT                                                    0xd
+#define NBIF_MGCG_CTRL_LCLK__NBIF_SRAM_FGCG_EN_LCLK__SHIFT                                                    0xe
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_EN_LCLK_MASK                                                           0x00000001L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_MODE_LCLK_MASK                                                         0x00000002L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HYSTERESIS_LCLK_MASK                                                   0x000003FCL
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HST_DIS_LCLK_MASK                                                      0x00000400L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_DMA_DIS_LCLK_MASK                                                      0x00000800L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_REG_DIS_LCLK_MASK                                                      0x00001000L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_AER_DIS_LCLK_MASK                                                      0x00002000L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_SRAM_FGCG_EN_LCLK_MASK                                                      0x00004000L
+//NBIF_DS_CTRL_LCLK
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_EN__SHIFT                                                             0x0
+#define NBIF_DS_CTRL_LCLK__ATHUB_LCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                           0x1
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_TIMER__SHIFT                                                          0x10
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_EN_MASK                                                               0x00000001L
+#define NBIF_DS_CTRL_LCLK__ATHUB_LCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                             0x00000002L
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_TIMER_MASK                                                            0xFFFF0000L
+//SMN_MST_CNTL0
+#define SMN_MST_CNTL0__SMN_ARB_MODE__SHIFT                                                                    0x0
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_UPS__SHIFT                                                           0x8
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_UPS__SHIFT                                                           0x9
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_UPS__SHIFT                                                            0xa
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_UPS__SHIFT                                                      0xb
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_DNS_DEV0__SHIFT                                                      0x10
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_DNS_DEV0__SHIFT                                                      0x14
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_DNS_DEV0__SHIFT                                                       0x18
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_DNS_DEV0__SHIFT                                                 0x1c
+#define SMN_MST_CNTL0__SMN_ARB_MODE_MASK                                                                      0x00000003L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_UPS_MASK                                                             0x00000100L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_UPS_MASK                                                             0x00000200L
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_UPS_MASK                                                              0x00000400L
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_UPS_MASK                                                        0x00000800L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_DNS_DEV0_MASK                                                        0x00010000L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_DNS_DEV0_MASK                                                        0x00100000L
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_DNS_DEV0_MASK                                                         0x01000000L
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_DNS_DEV0_MASK                                                   0x10000000L
+//SMN_MST_EP_CNTL1
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF0__SHIFT                                                 0x0
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF1__SHIFT                                                 0x1
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF2__SHIFT                                                 0x2
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF3__SHIFT                                                 0x3
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF4__SHIFT                                                 0x4
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF5__SHIFT                                                 0x5
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF6__SHIFT                                                 0x6
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF7__SHIFT                                                 0x7
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF0_MASK                                                   0x00000001L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF1_MASK                                                   0x00000002L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF2_MASK                                                   0x00000004L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF3_MASK                                                   0x00000008L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF4_MASK                                                   0x00000010L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF5_MASK                                                   0x00000020L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF6_MASK                                                   0x00000040L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF7_MASK                                                   0x00000080L
+//SMN_MST_EP_CNTL2
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF0__SHIFT                                           0x0
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF1__SHIFT                                           0x1
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF2__SHIFT                                           0x2
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF3__SHIFT                                           0x3
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF4__SHIFT                                           0x4
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF5__SHIFT                                           0x5
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF6__SHIFT                                           0x6
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF7__SHIFT                                           0x7
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF0_MASK                                             0x00000001L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF1_MASK                                             0x00000002L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF2_MASK                                             0x00000004L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF3_MASK                                             0x00000008L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF4_MASK                                             0x00000010L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF5_MASK                                             0x00000020L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF6_MASK                                             0x00000040L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF7_MASK                                             0x00000080L
+//NBIF_SDP_VWR_VCHG_DIS_CTRL
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F0_DIS__SHIFT                                           0x0
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F1_DIS__SHIFT                                           0x1
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F2_DIS__SHIFT                                           0x2
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F3_DIS__SHIFT                                           0x3
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F4_DIS__SHIFT                                           0x4
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F5_DIS__SHIFT                                           0x5
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F6_DIS__SHIFT                                           0x6
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F7_DIS__SHIFT                                           0x7
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_SWDS_P0_DIS__SHIFT                                           0x18
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F0_DIS_MASK                                             0x00000001L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F1_DIS_MASK                                             0x00000002L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F2_DIS_MASK                                             0x00000004L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F3_DIS_MASK                                             0x00000008L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F4_DIS_MASK                                             0x00000010L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F5_DIS_MASK                                             0x00000020L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F6_DIS_MASK                                             0x00000040L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F7_DIS_MASK                                             0x00000080L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_SWDS_P0_DIS_MASK                                             0x01000000L
+//NBIF_SDP_VWR_VCHG_RST_CTRL0
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_EN__SHIFT                                  0x0
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_EN__SHIFT                                  0x1
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_EN__SHIFT                                  0x2
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_EN__SHIFT                                  0x3
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_EN__SHIFT                                  0x4
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_EN__SHIFT                                  0x5
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_EN__SHIFT                                  0x6
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_EN__SHIFT                                  0x7
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_EN__SHIFT                                  0x18
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_EN_MASK                                    0x00000001L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_EN_MASK                                    0x00000002L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_EN_MASK                                    0x00000004L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_EN_MASK                                    0x00000008L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_EN_MASK                                    0x00000010L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_EN_MASK                                    0x00000020L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_EN_MASK                                    0x00000040L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_EN_MASK                                    0x00000080L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_EN_MASK                                    0x01000000L
+//NBIF_SDP_VWR_VCHG_RST_CTRL1
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_VAL__SHIFT                                 0x0
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_VAL__SHIFT                                 0x1
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_VAL__SHIFT                                 0x2
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_VAL__SHIFT                                 0x3
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_VAL__SHIFT                                 0x4
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_VAL__SHIFT                                 0x5
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_VAL__SHIFT                                 0x6
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_VAL__SHIFT                                 0x7
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_VAL__SHIFT                                 0x18
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_VAL_MASK                                   0x00000001L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_VAL_MASK                                   0x00000002L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_VAL_MASK                                   0x00000004L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_VAL_MASK                                   0x00000008L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_VAL_MASK                                   0x00000010L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_VAL_MASK                                   0x00000020L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_VAL_MASK                                   0x00000040L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_VAL_MASK                                   0x00000080L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_VAL_MASK                                   0x01000000L
+//NBIF_SDP_VWR_VCHG_TRIG
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F0_TRIG__SHIFT                                              0x0
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F1_TRIG__SHIFT                                              0x1
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F2_TRIG__SHIFT                                              0x2
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F3_TRIG__SHIFT                                              0x3
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F4_TRIG__SHIFT                                              0x4
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F5_TRIG__SHIFT                                              0x5
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F6_TRIG__SHIFT                                              0x6
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F7_TRIG__SHIFT                                              0x7
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_SWDS_P0_TRIG__SHIFT                                              0x18
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F0_TRIG_MASK                                                0x00000001L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F1_TRIG_MASK                                                0x00000002L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F2_TRIG_MASK                                                0x00000004L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F3_TRIG_MASK                                                0x00000008L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F4_TRIG_MASK                                                0x00000010L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F5_TRIG_MASK                                                0x00000020L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F6_TRIG_MASK                                                0x00000040L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F7_TRIG_MASK                                                0x00000080L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_SWDS_P0_TRIG_MASK                                                0x01000000L
+//NBIF_SHUB_TODET_CTRL
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_EN__SHIFT                                                       0x0
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_AER_LOG_EN__SHIFT                                               0x1
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_TIMER_UNIT__SHIFT                                               0x8
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TIMEOUT_COUNT__SHIFT                                                  0x10
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_EN_MASK                                                         0x00000001L
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_AER_LOG_EN_MASK                                                 0x00000002L
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_TIMER_UNIT_MASK                                                 0x00000700L
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TIMEOUT_COUNT_MASK                                                    0xFFFF0000L
+//NBIF_SHUB_TODET_CLIENT_CTRL
+#define NBIF_SHUB_TODET_CLIENT_CTRL__NBIF_SHUB_TODET_SLVERR_EN__SHIFT                                         0x0
+#define NBIF_SHUB_TODET_CLIENT_CTRL__NBIF_SHUB_TODET_SLVERR_EN_MASK                                           0xFFFFFFFFL
+//NBIF_SHUB_TODET_CLIENT_STATUS
+#define NBIF_SHUB_TODET_CLIENT_STATUS__NBIF_SHUB_TODET_CLIENT_STATUS__SHIFT                                   0x0
+#define NBIF_SHUB_TODET_CLIENT_STATUS__NBIF_SHUB_TODET_CLIENT_STATUS_MASK                                     0xFFFFFFFFL
+//NBIF_SHUB_TODET_SYNCFLOOD_CTRL
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL__NBIF_SHUB_TODET_SYNCFLOOD_EN__SHIFT                                   0x0
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL__NBIF_SHUB_TODET_SYNCFLOOD_EN_MASK                                     0xFFFFFFFFL
+//NBIF_SHUB_TODET_CLIENT_CTRL2
+#define NBIF_SHUB_TODET_CLIENT_CTRL2__NBIF_SHUB_TODET_SLVERR_EN2__SHIFT                                       0x0
+#define NBIF_SHUB_TODET_CLIENT_CTRL2__NBIF_SHUB_TODET_SLVERR_EN2_MASK                                         0xFFFFFFFFL
+//NBIF_SHUB_TODET_CLIENT_STATUS2
+#define NBIF_SHUB_TODET_CLIENT_STATUS2__NBIF_SHUB_TODET_CLIENT_STATUS2__SHIFT                                 0x0
+#define NBIF_SHUB_TODET_CLIENT_STATUS2__NBIF_SHUB_TODET_CLIENT_STATUS2_MASK                                   0xFFFFFFFFL
+//NBIF_SHUB_TODET_SYNCFLOOD_CTRL2
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL2__NBIF_SHUB_TODET_SYNCFLOOD_EN2__SHIFT                                 0x0
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL2__NBIF_SHUB_TODET_SYNCFLOOD_EN2_MASK                                   0xFFFFFFFFL
+//BIFC_BME_ERR_LOG_HB
+//BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                   0x0
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                   0x4
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                     0x0000000FL
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                     0x000000F0L
+//BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                   0x0
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                   0x4
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                     0x0000000FL
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                     0x000000F0L
+//BIFC_GMI_SDP_REQ_POOLCRED_ALLOC
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                     0x0
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                     0x4
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                     0x8
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                     0xc
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                     0x10
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                     0x14
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                     0x18
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                     0x1c
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                       0x0000000FL
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                       0x000000F0L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                       0x00000F00L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                       0x0000F000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                       0x000F0000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                       0x00F00000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                       0x0F000000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                       0xF0000000L
+//BIFC_GMI_SDP_DAT_POOLCRED_ALLOC
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                     0x0
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                     0x4
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                     0x8
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                     0xc
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                     0x10
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                     0x14
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                     0x18
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                     0x1c
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                       0x0000000FL
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                       0x000000F0L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                       0x00000F00L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                       0x0000F000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                       0x000F0000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                       0x00F00000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                       0x0F000000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                       0xF0000000L
+//DISCON_HYSTERESIS_HEAD_CTRL
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_DNS_SDP_DISCON_HYSTERESIS_H__SHIFT                                   0x0
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_UPS_SDP_DISCON_HYSTERESIS_H__SHIFT                                   0x8
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_DNS_SDP_DISCON_HYSTERESIS_H_MASK                                     0x0000000FL
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_UPS_SDP_DISCON_HYSTERESIS_H_MASK                                     0x00000F00L
+//BIFC_EARLY_WAKEUP_CNTL
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_ACTIVE__SHIFT                                     0x0
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_DS_EXIT__SHIFT                                    0x1
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_ALLOW_AER_ACTIVE__SHIFT                                     0x2
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_ACTIVE_MASK                                       0x00000001L
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_DS_EXIT_MASK                                      0x00000002L
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_ALLOW_AER_ACTIVE_MASK                                       0x00000004L
+//BIFC_PERF_CNT_MMIO_RD_H16BIT
+#define BIFC_PERF_CNT_MMIO_RD_H16BIT__PERF_CNT_MMIO_RD_VALUE_H16BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_RD_H16BIT__PERF_CNT_MMIO_RD_VALUE_H16BIT_MASK                                      0x0000FFFFL
+//BIFC_PERF_CNT_MMIO_WR_H16BIT
+#define BIFC_PERF_CNT_MMIO_WR_H16BIT__PERF_CNT_MMIO_WR_VALUE_H16BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_WR_H16BIT__PERF_CNT_MMIO_WR_VALUE_H16BIT_MASK                                      0x0000FFFFL
+//BIFC_PERF_CNT_DMA_RD_H16BIT
+#define BIFC_PERF_CNT_DMA_RD_H16BIT__PERF_CNT_DMA_RD_VALUE_H16BIT__SHIFT                                      0x0
+#define BIFC_PERF_CNT_DMA_RD_H16BIT__PERF_CNT_DMA_RD_VALUE_H16BIT_MASK                                        0x0000FFFFL
+//BIFC_PERF_CNT_DMA_WR_H16BIT
+#define BIFC_PERF_CNT_DMA_WR_H16BIT__PERF_CNT_DMA_WR_VALUE_H16BIT__SHIFT                                      0x0
+#define BIFC_PERF_CNT_DMA_WR_H16BIT__PERF_CNT_DMA_WR_VALUE_H16BIT_MASK                                        0x0000FFFFL
+//BIFC_A2S_SDP_PORT_CTRL
+#define BIFC_A2S_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS__SHIFT                                                  0x0
+#define BIFC_A2S_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_H__SHIFT                                                0x8
+#define BIFC_A2S_SDP_PORT_CTRL__SDP_DISCON_DIS__SHIFT                                                         0xc
+#define BIFC_A2S_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_MASK                                                    0x000000FFL
+#define BIFC_A2S_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_H_MASK                                                  0x00000F00L
+#define BIFC_A2S_SDP_PORT_CTRL__SDP_DISCON_DIS_MASK                                                           0x00001000L
+//BIFC_A2S_CNTL_SW0
+#define BIFC_A2S_CNTL_SW0__RDRSP_ERRMAP__SHIFT                                                                0x0
+#define BIFC_A2S_CNTL_SW0__RDRSP_SEL_MODE__SHIFT                                                              0x2
+#define BIFC_A2S_CNTL_SW0__STATIC_VC_ENABLE__SHIFT                                                            0x5
+#define BIFC_A2S_CNTL_SW0__STATIC_VC_VALUE__SHIFT                                                             0x6
+#define BIFC_A2S_CNTL_SW0__SDP_WR_CHAIN_DIS__SHIFT                                                            0x9
+#define BIFC_A2S_CNTL_SW0__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                     0xa
+#define BIFC_A2S_CNTL_SW0__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                  0xb
+#define BIFC_A2S_CNTL_SW0__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                 0xc
+#define BIFC_A2S_CNTL_SW0__WRR_RD_WEIGHT__SHIFT                                                               0x10
+#define BIFC_A2S_CNTL_SW0__WRR_WR_WEIGHT__SHIFT                                                               0x18
+#define BIFC_A2S_CNTL_SW0__RDRSP_ERRMAP_MASK                                                                  0x00000003L
+#define BIFC_A2S_CNTL_SW0__RDRSP_SEL_MODE_MASK                                                                0x0000001CL
+#define BIFC_A2S_CNTL_SW0__STATIC_VC_ENABLE_MASK                                                              0x00000020L
+#define BIFC_A2S_CNTL_SW0__STATIC_VC_VALUE_MASK                                                               0x000001C0L
+#define BIFC_A2S_CNTL_SW0__SDP_WR_CHAIN_DIS_MASK                                                              0x00000200L
+#define BIFC_A2S_CNTL_SW0__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                       0x00000400L
+#define BIFC_A2S_CNTL_SW0__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                    0x00000800L
+#define BIFC_A2S_CNTL_SW0__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                   0x00001000L
+#define BIFC_A2S_CNTL_SW0__WRR_RD_WEIGHT_MASK                                                                 0x00FF0000L
+#define BIFC_A2S_CNTL_SW0__WRR_WR_WEIGHT_MASK                                                                 0xFF000000L
+//BIFC_A2S_MISC_CNTL
+#define BIFC_A2S_MISC_CNTL__BLKLVL_FOR_MSG__SHIFT                                                             0x0
+#define BIFC_A2S_MISC_CNTL__RESERVE_2_CRED_FOR_NPWR_REQ_DIS__SHIFT                                            0x2
+#define BIFC_A2S_MISC_CNTL__WRR_LRG_SIZE_MODE__SHIFT                                                          0x3
+#define BIFC_A2S_MISC_CNTL__FORCE_RSP_REORDER_EN__SHIFT                                                       0x4
+#define BIFC_A2S_MISC_CNTL__RSP_REORDER_DIS__SHIFT                                                            0x5
+#define BIFC_A2S_MISC_CNTL__WRRSP_ACCUM_SEL__SHIFT                                                            0x6
+#define BIFC_A2S_MISC_CNTL__WRRSP_TAGFIFO_CONT_RD_DIS__SHIFT                                                  0x7
+#define BIFC_A2S_MISC_CNTL__RDRSP_TAGFIFO_CONT_RD_DIS__SHIFT                                                  0x8
+#define BIFC_A2S_MISC_CNTL__RDRSP_STS_DATSTS_PRIORITY__SHIFT                                                  0x9
+#define BIFC_A2S_MISC_CNTL__INSERT_RD_ON_2ND_WDAT_EN__SHIFT                                                   0xa
+#define BIFC_A2S_MISC_CNTL__WR_TAG_SET_MIN__SHIFT                                                             0x10
+#define BIFC_A2S_MISC_CNTL__RD_TAG_SET_MIN__SHIFT                                                             0x15
+#define BIFC_A2S_MISC_CNTL__WRR_LRG_MODE__SHIFT                                                               0x1a
+#define BIFC_A2S_MISC_CNTL__WRR_LRG_COUNTER_MODE__SHIFT                                                       0x1b
+#define BIFC_A2S_MISC_CNTL__BYPASS_OVERRIDE_EN__SHIFT                                                         0x1c
+#define BIFC_A2S_MISC_CNTL__BYPASS_OVERRIDE_VALUE__SHIFT                                                      0x1d
+#define BIFC_A2S_MISC_CNTL__BLKLVL_FOR_MSG_MASK                                                               0x00000003L
+#define BIFC_A2S_MISC_CNTL__RESERVE_2_CRED_FOR_NPWR_REQ_DIS_MASK                                              0x00000004L
+#define BIFC_A2S_MISC_CNTL__WRR_LRG_SIZE_MODE_MASK                                                            0x00000008L
+#define BIFC_A2S_MISC_CNTL__FORCE_RSP_REORDER_EN_MASK                                                         0x00000010L
+#define BIFC_A2S_MISC_CNTL__RSP_REORDER_DIS_MASK                                                              0x00000020L
+#define BIFC_A2S_MISC_CNTL__WRRSP_ACCUM_SEL_MASK                                                              0x00000040L
+#define BIFC_A2S_MISC_CNTL__WRRSP_TAGFIFO_CONT_RD_DIS_MASK                                                    0x00000080L
+#define BIFC_A2S_MISC_CNTL__RDRSP_TAGFIFO_CONT_RD_DIS_MASK                                                    0x00000100L
+#define BIFC_A2S_MISC_CNTL__RDRSP_STS_DATSTS_PRIORITY_MASK                                                    0x00000200L
+#define BIFC_A2S_MISC_CNTL__INSERT_RD_ON_2ND_WDAT_EN_MASK                                                     0x00000400L
+#define BIFC_A2S_MISC_CNTL__WR_TAG_SET_MIN_MASK                                                               0x001F0000L
+#define BIFC_A2S_MISC_CNTL__RD_TAG_SET_MIN_MASK                                                               0x03E00000L
+#define BIFC_A2S_MISC_CNTL__WRR_LRG_MODE_MASK                                                                 0x04000000L
+#define BIFC_A2S_MISC_CNTL__WRR_LRG_COUNTER_MODE_MASK                                                         0x08000000L
+#define BIFC_A2S_MISC_CNTL__BYPASS_OVERRIDE_EN_MASK                                                           0x10000000L
+#define BIFC_A2S_MISC_CNTL__BYPASS_OVERRIDE_VALUE_MASK                                                        0x20000000L
+//BIFC_A2S_TAG_ALLOC_0
+#define BIFC_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_WR__SHIFT                                                     0x0
+#define BIFC_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_RD__SHIFT                                                     0x8
+#define BIFC_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC1_WR__SHIFT                                                     0x10
+#define BIFC_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_WR_MASK                                                       0x000000FFL
+#define BIFC_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_RD_MASK                                                       0x0000FF00L
+#define BIFC_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC1_WR_MASK                                                       0x00FF0000L
+//BIFC_A2S_TAG_ALLOC_1
+#define BIFC_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC3_WR__SHIFT                                                     0x0
+#define BIFC_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_WR__SHIFT                                                     0x10
+#define BIFC_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_RD__SHIFT                                                     0x18
+#define BIFC_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC3_WR_MASK                                                       0x000000FFL
+#define BIFC_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_WR_MASK                                                       0x00FF0000L
+#define BIFC_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_RD_MASK                                                       0xFF000000L
+//BIFC_A2S_CNTL_CL0
+#define BIFC_A2S_CNTL_CL0__NSNOOP_MAP__SHIFT                                                                  0x0
+#define BIFC_A2S_CNTL_CL0__REQPASSPW_VC0_MAP__SHIFT                                                           0x2
+#define BIFC_A2S_CNTL_CL0__REQPASSPW_NVC0_MAP__SHIFT                                                          0x4
+#define BIFC_A2S_CNTL_CL0__REQRSPPASSPW_VC0_MAP__SHIFT                                                        0x6
+#define BIFC_A2S_CNTL_CL0__REQRSPPASSPW_NVC0_MAP__SHIFT                                                       0x8
+#define BIFC_A2S_CNTL_CL0__BLKLVL_MAP__SHIFT                                                                  0xa
+#define BIFC_A2S_CNTL_CL0__DATERR_MAP__SHIFT                                                                  0xc
+#define BIFC_A2S_CNTL_CL0__EXOKAY_WR_MAP__SHIFT                                                               0xe
+#define BIFC_A2S_CNTL_CL0__EXOKAY_RD_MAP__SHIFT                                                               0x10
+#define BIFC_A2S_CNTL_CL0__RESP_WR_MAP__SHIFT                                                                 0x12
+#define BIFC_A2S_CNTL_CL0__RESP_RD_MAP__SHIFT                                                                 0x14
+#define BIFC_A2S_CNTL_CL0__NSNOOP_MAP_MASK                                                                    0x00000003L
+#define BIFC_A2S_CNTL_CL0__REQPASSPW_VC0_MAP_MASK                                                             0x0000000CL
+#define BIFC_A2S_CNTL_CL0__REQPASSPW_NVC0_MAP_MASK                                                            0x00000030L
+#define BIFC_A2S_CNTL_CL0__REQRSPPASSPW_VC0_MAP_MASK                                                          0x000000C0L
+#define BIFC_A2S_CNTL_CL0__REQRSPPASSPW_NVC0_MAP_MASK                                                         0x00000300L
+#define BIFC_A2S_CNTL_CL0__BLKLVL_MAP_MASK                                                                    0x00000C00L
+#define BIFC_A2S_CNTL_CL0__DATERR_MAP_MASK                                                                    0x00003000L
+#define BIFC_A2S_CNTL_CL0__EXOKAY_WR_MAP_MASK                                                                 0x0000C000L
+#define BIFC_A2S_CNTL_CL0__EXOKAY_RD_MAP_MASK                                                                 0x00030000L
+#define BIFC_A2S_CNTL_CL0__RESP_WR_MAP_MASK                                                                   0x000C0000L
+#define BIFC_A2S_CNTL_CL0__RESP_RD_MAP_MASK                                                                   0x00300000L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwn_dev0_BIFDEC1
+//RCC_DWN_DEV0_2_DN_PCIE_RESERVED
+#define RCC_DWN_DEV0_2_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                 0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                   0xFFFFFFFFL
+//RCC_DWN_DEV0_2_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV0_2_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV0_2_DN_PCIE_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                    0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                              0x7
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                              0x1e
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                      0x00000001L
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                0x00000080L
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                0x40000000L
+//RCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                0x19
+#define RCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                  0x06000000L
+//RCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                               0x1c
+#define RCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                 0x70000000L
+//RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                             0x7
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                   0x8
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                               0x00000080L
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                     0x00000100L
+//RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                      0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                 0x1
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                 0x2
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                 0x3
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                 0x4
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                        0x00000001L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                   0x00000002L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                   0x00000004L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                   0x00000008L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                   0x00000010L
+//RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                0x11
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                        0x15
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                     0x00000001L
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                  0x00020000L
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                          0x00E00000L
+//RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                             0x18
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                          0x1d
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                               0x01000000L
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                            0x20000000L
+//RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                    0x2
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                      0x00000004L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dwnp_dev0_BIFDEC1
+//RCC_DWNP_DEV0_2_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                               0x0
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                             0x8
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                    0xb
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                        0x11
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                               0x12
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                           0x13
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                              0x14
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                 0x00000001L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                               0x00000700L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                      0x00000800L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                          0x00020000L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                 0x00040000L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                             0x00080000L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                0x00100000L
+//RCC_DWNP_DEV0_2_PCIE_RX_CNTL
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                        0x8
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                              0x9
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                          0x14
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                     0x15
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                           0x1b
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                          0x00000100L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                0x00000200L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                            0x00100000L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                       0x00200000L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                             0x08000000L
+//RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                           0x0
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                           0x1
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                           0x2
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                           0x3
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                             0x00000001L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                             0x00000002L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                             0x00000004L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                             0x00000008L
+//RCC_DWNP_DEV0_2_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                               0x0
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                     0x1b
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                 0x00000001L
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                       0x08000000L
+//RCC_DWNP_DEV0_2_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV0_2_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                          0xa
+#define RCC_DWNP_DEV0_2_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                            0x00000400L
+//RCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                     0x0
+#define RCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                       0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_ep_dev0_BIFDEC1
+//RCC_EP_DEV0_2_EP_PCIE_SCRATCH
+#define RCC_EP_DEV0_2_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                    0x0
+#define RCC_EP_DEV0_2_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                      0xFFFFFFFFL
+//RCC_EP_DEV0_2_EP_PCIE_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                  0x7
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                            0x8
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                               0x1e
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                    0x00000080L
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                              0x00000100L
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                 0x40000000L
+//RCC_EP_DEV0_2_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                0x0
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                           0x1
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                               0x2
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                            0x3
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                0x4
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                         0x6
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                  0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                             0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                 0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                              0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                  0x00000010L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                           0x00000040L
+//RCC_EP_DEV0_2_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                          0x0
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                     0x1
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                         0x2
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                      0x3
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                          0x4
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                   0x6
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                0x7
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                            0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                       0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                           0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                        0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                            0x00000010L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                     0x00000040L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                  0x00000080L
+//RCC_EP_DEV0_2_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                   0x0
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                     0x00000001L
+//RCC_EP_DEV0_2_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                              0x7
+#define RCC_EP_DEV0_2_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                0x00000080L
+//RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                       0x0
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                  0x1
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                  0x2
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                  0x3
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                  0x4
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                         0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                    0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                    0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                    0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                    0x00000010L
+//RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                           0x1d
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                             0x20000000L
+//RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                         0x4
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                           0x00000010L
+//RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//RCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0xFFL
+//RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                             0x0
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                         0x8
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                               0x001FL
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                           0x0100L
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV0_2_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                           0x0
+#define RCC_EP_DEV0_2_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                             0x1FL
+//RCC_EP_DEV0_2_EP_PCIEP_RESERVED
+#define RCC_EP_DEV0_2_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                0x0
+#define RCC_EP_DEV0_2_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                  0xFFFFFFFFL
+//RCC_EP_DEV0_2_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                 0xa
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                  0xc
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                   0x18
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                   0x19
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                   0x1a
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                   0x00000C00L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                    0x00003000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                     0x01000000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                     0x02000000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                     0x04000000L
+//RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                0x0
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                  0x3
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                     0x8
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                  0x00000007L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                    0x000000F8L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                       0x0000FF00L
+//RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                              0x0
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                            0x8
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                       0x11
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                               0x12
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                   0x18
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                   0x19
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                   0x1a
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                   0x1b
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                   0x1c
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                   0x1d
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                   0x1e
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                   0x1f
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                              0x00000700L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                         0x00020000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                 0x00040000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                     0x01000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                     0x02000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                     0x04000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                     0x08000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                     0x10000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                     0x20000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                     0x40000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                     0x80000000L
+//RCC_EP_DEV0_2_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                       0x8
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                0x9
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                         0x14
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                       0x15
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                         0x16
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                      0x18
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                          0x19
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                      0x1a
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                         0x00000100L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                  0x00000200L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                           0x00100000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                         0x00200000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                           0x00400000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                        0x01000000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                            0x02000000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                        0x04000000L
+//RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                          0x0
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                          0x1
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                          0x2
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                          0x3
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                            0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                            0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                            0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                            0x00000008L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_BIFDEC1
+//RCC_DEV0_1_RCC_ERR_INT_CNTL
+#define RCC_DEV0_1_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN__SHIFT                                0x0
+#define RCC_DEV0_1_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN_MASK                                  0x00000001L
+//RCC_DEV0_1_RCC_BACO_CNTL_MISC
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS__SHIFT                                                 0x0
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS__SHIFT                                                  0x1
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS_MASK                                                   0x00000001L
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS_MASK                                                    0x00000002L
+//RCC_DEV0_1_RCC_RESET_EN
+#define RCC_DEV0_1_RCC_RESET_EN__DB_APER_RESET_EN__SHIFT                                                      0xf
+#define RCC_DEV0_1_RCC_RESET_EN__DB_APER_RESET_EN_MASK                                                        0x00008000L
+//RCC_DEV0_2_RCC_VDM_SUPPORT
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                       0x0
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                      0x1
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                  0x2
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                        0x3
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                    0x4
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                         0x00000001L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                        0x00000002L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                    0x00000004L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                          0x00000008L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                      0x00000010L
+//RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                 0x0
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                              0x1
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                0x2
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                 0x3
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                           0x4
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                  0x5
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                 0xb
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                 0x12
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                0x19
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                   0x00000001L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                0x00000002L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                  0x00000004L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                   0x00000008L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                             0x00000010L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                    0x000007E0L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                   0x0003F800L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                   0x01FC0000L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                  0xFE000000L
+//RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                             0x0
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                              0x6
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                         0xc
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                      0x11
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                               0x0000003FL
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                0x00000FC0L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                           0x0001F000L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                        0x00FE0000L
+//RCC_DEV0_1_RCC_GPUIOV_REGION
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__LFB_REGION__SHIFT                                                       0x0
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__MAX_REGION__SHIFT                                                       0x4
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__LFB_REGION_MASK                                                         0x0000000FL
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__MAX_REGION_MASK                                                         0x000000F0L
+//RCC_DEV0_1_RCC_GPU_HOSTVM_EN
+#define RCC_DEV0_1_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN__SHIFT                                                    0x0
+#define RCC_DEV0_1_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN_MASK                                                      0x00000001L
+//RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE__SHIFT                              0x0
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN__SHIFT                                    0x1
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE_MASK                                0x00000001L
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN_MASK                                      0x00000002L
+//RCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET__SHIFT                        0x0
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET_MASK                          0xFFFFL
+//RCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE__SHIFT                                    0x0
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE_MASK                                      0xFFFFL
+//RCC_DEV0_1_RCC_PEER_REG_RANGE0
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_1_RCC_PEER_REG_RANGE1
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_2_RCC_BUS_CNTL
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                            0x2
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                           0x3
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                            0x4
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                         0x5
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                        0x6
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                         0x7
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                        0x8
+#define RCC_DEV0_2_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                 0xc
+#define RCC_DEV0_2_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                           0xd
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x10
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x11
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x12
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x13
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x14
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x15
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                 0x18
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                 0x19
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                            0x1c
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                            0x1d
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                              0x00000004L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                             0x00000008L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                              0x00000010L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                           0x00000020L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                          0x00000040L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                           0x00000080L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                          0x00000100L
+#define RCC_DEV0_2_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                   0x00001000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                             0x00002000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00010000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00020000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00040000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00080000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00100000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00200000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                   0x01000000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                   0x0E000000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                              0x10000000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                              0xE0000000L
+//RCC_DEV0_1_RCC_CONFIG_CNTL
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B__SHIFT                                               0x2
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GRPH_ADRSEL__SHIFT                                                        0x3
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN_MASK                                                       0x00000001L
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B_MASK                                                 0x00000004L
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GRPH_ADRSEL_MASK                                                          0x00000018L
+//RCC_DEV0_1_RCC_CONFIG_F0_BASE
+#define RCC_DEV0_1_RCC_CONFIG_F0_BASE__F0_BASE__SHIFT                                                         0x0
+#define RCC_DEV0_1_RCC_CONFIG_F0_BASE__F0_BASE_MASK                                                           0xFFFFFFFFL
+//RCC_DEV0_1_RCC_CONFIG_APER_SIZE
+#define RCC_DEV0_1_RCC_CONFIG_APER_SIZE__APER_SIZE__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_CONFIG_APER_SIZE__APER_SIZE_MASK                                                       0xFFFFFFFFL
+//RCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE
+#define RCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE__SHIFT                                             0x0
+#define RCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE_MASK                                               0x07FFFFFFL
+//RCC_DEV0_1_RCC_XDMA_LO
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_APER_EN__SHIFT                                                       0x1f
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND_MASK                                                     0x7FFFFFFFL
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_APER_EN_MASK                                                         0x80000000L
+//RCC_DEV0_1_RCC_XDMA_HI
+#define RCC_DEV0_1_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND_MASK                                                     0x7FFFFFFFL
+//RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS__SHIFT                                   0x7
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                 0x8
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                    0x9
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                    0xa
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                 0xb
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                  0xc
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                      0xd
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                      0xe
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                         0xf
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                 0x10
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                           0x11
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                               0x12
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                     0x13
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS_MASK                                     0x00000080L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                   0x00000100L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                      0x00000200L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                      0x00000400L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                   0x00000800L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                    0x00001000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                        0x00002000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                        0x00004000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                           0x00008000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                   0x00010000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                             0x00020000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                 0x00040000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                       0x00080000L
+//RCC_DEV0_1_RCC_BUSNUM_CNTL1
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL1__ID_MASK__SHIFT                                                           0x0
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL1__ID_MASK_MASK                                                             0x000000FFL
+//RCC_DEV0_1_RCC_BUSNUM_LIST0
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID0__SHIFT                                                               0x0
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID1__SHIFT                                                               0x8
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID2__SHIFT                                                               0x10
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID3__SHIFT                                                               0x18
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID0_MASK                                                                 0x000000FFL
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID1_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID2_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID3_MASK                                                                 0xFF000000L
+//RCC_DEV0_1_RCC_BUSNUM_LIST1
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID4__SHIFT                                                               0x0
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID5__SHIFT                                                               0x8
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID6__SHIFT                                                               0x10
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID7__SHIFT                                                               0x18
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID4_MASK                                                                 0x000000FFL
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID5_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID6_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID7_MASK                                                                 0xFF000000L
+//RCC_DEV0_1_RCC_BUSNUM_CNTL2
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL__SHIFT                                                    0x0
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN__SHIFT                                                     0x8
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__HDPREG_CNTL__SHIFT                                                       0x10
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH__SHIFT                                           0x11
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL_MASK                                                      0x000000FFL
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN_MASK                                                       0x00000100L
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__HDPREG_CNTL_MASK                                                         0x00010000L
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH_MASK                                             0x00020000L
+//RCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM
+#define RCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN_MASK                                                     0x00000001L
+//RCC_DEV0_1_RCC_HOST_BUSNUM
+#define RCC_DEV0_1_RCC_HOST_BUSNUM__HOST_ID__SHIFT                                                            0x0
+#define RCC_DEV0_1_RCC_HOST_BUSNUM__HOST_ID_MASK                                                              0x0000FFFFL
+//RCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1__SHIFT                                                   0x8
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2__SHIFT                                                   0x10
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3__SHIFT                                                   0x18
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0_MASK                                                     0x000000FFL
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1_MASK                                                     0x0000FF00L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2_MASK                                                     0x00FF0000L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3_MASK                                                     0xFF000000L
+//RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5__SHIFT                                                   0x8
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6__SHIFT                                                   0x10
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7__SHIFT                                                   0x18
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4_MASK                                                     0x000000FFL
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5_MASK                                                     0x0000FF00L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6_MASK                                                     0x00FF0000L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7_MASK                                                     0xFF000000L
+//RCC_DEV0_2_RCC_DEV0_LINK_CNTL
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                  0x0
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                 0x8
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                            0x10
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                            0x11
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                    0x00000001L
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                   0x00000100L
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                              0x00010000L
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                              0x00020000L
+//RCC_DEV0_2_RCC_CMN_LINK_CNTL
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                             0x0
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                              0x1
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                             0x2
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                          0x3
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                             0x10
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                               0x00000001L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                0x00000002L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                               0x00000004L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                            0x00000008L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                               0xFFFF0000L
+//RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                            0x0
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                            0x8
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                              0x000000FFL
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                              0x00001F00L
+//RCC_DEV0_2_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV0_2_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_DEV0_2_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                           0x000003FFL
+//RCC_DEV0_2_RCC_MH_ARB_CNTL
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                        0x0
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                0x1
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                          0x00000001L
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                  0x00007FFEL
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_SYSDEC
+//BIF_BX1_PCIE_INDEX
+#define BIF_BX1_PCIE_INDEX__PCIE_INDEX__SHIFT                                                                 0x0
+#define BIF_BX1_PCIE_INDEX__PCIE_INDEX_MASK                                                                   0xFFFFFFFFL
+//BIF_BX1_PCIE_DATA
+#define BIF_BX1_PCIE_DATA__PCIE_DATA__SHIFT                                                                   0x0
+#define BIF_BX1_PCIE_DATA__PCIE_DATA_MASK                                                                     0xFFFFFFFFL
+//BIF_BX1_PCIE_INDEX2
+#define BIF_BX1_PCIE_INDEX2__PCIE_INDEX2__SHIFT                                                               0x0
+#define BIF_BX1_PCIE_INDEX2__PCIE_INDEX2_MASK                                                                 0xFFFFFFFFL
+//BIF_BX1_PCIE_DATA2
+#define BIF_BX1_PCIE_DATA2__PCIE_DATA2__SHIFT                                                                 0x0
+#define BIF_BX1_PCIE_DATA2__PCIE_DATA2_MASK                                                                   0xFFFFFFFFL
+//BIF_BX1_PCIE_INDEX_HI
+#define BIF_BX1_PCIE_INDEX_HI__PCIE_INDEX_HI__SHIFT                                                           0x0
+#define BIF_BX1_PCIE_INDEX_HI__PCIE_INDEX_HI_MASK                                                             0x000000FFL
+//BIF_BX1_PCIE_INDEX2_HI
+#define BIF_BX1_PCIE_INDEX2_HI__PCIE_INDEX2_HI__SHIFT                                                         0x0
+#define BIF_BX1_PCIE_INDEX2_HI__PCIE_INDEX2_HI_MASK                                                           0x000000FFL
+//BIF_BX1_SBIOS_SCRATCH_0
+#define BIF_BX1_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_1
+#define BIF_BX1_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_2
+#define BIF_BX1_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_3
+#define BIF_BX1_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_0
+#define BIF_BX1_BIOS_SCRATCH_0__BIOS_SCRATCH_0__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_0__BIOS_SCRATCH_0_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_1
+#define BIF_BX1_BIOS_SCRATCH_1__BIOS_SCRATCH_1__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_1__BIOS_SCRATCH_1_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_2
+#define BIF_BX1_BIOS_SCRATCH_2__BIOS_SCRATCH_2__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_2__BIOS_SCRATCH_2_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_3
+#define BIF_BX1_BIOS_SCRATCH_3__BIOS_SCRATCH_3__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_3__BIOS_SCRATCH_3_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_4
+#define BIF_BX1_BIOS_SCRATCH_4__BIOS_SCRATCH_4__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_4__BIOS_SCRATCH_4_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_5
+#define BIF_BX1_BIOS_SCRATCH_5__BIOS_SCRATCH_5__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_5__BIOS_SCRATCH_5_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_6
+#define BIF_BX1_BIOS_SCRATCH_6__BIOS_SCRATCH_6__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_6__BIOS_SCRATCH_6_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_7
+#define BIF_BX1_BIOS_SCRATCH_7__BIOS_SCRATCH_7__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_7__BIOS_SCRATCH_7_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_8
+#define BIF_BX1_BIOS_SCRATCH_8__BIOS_SCRATCH_8__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_8__BIOS_SCRATCH_8_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_9
+#define BIF_BX1_BIOS_SCRATCH_9__BIOS_SCRATCH_9__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_9__BIOS_SCRATCH_9_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_10
+#define BIF_BX1_BIOS_SCRATCH_10__BIOS_SCRATCH_10__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_10__BIOS_SCRATCH_10_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_11
+#define BIF_BX1_BIOS_SCRATCH_11__BIOS_SCRATCH_11__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_11__BIOS_SCRATCH_11_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_12
+#define BIF_BX1_BIOS_SCRATCH_12__BIOS_SCRATCH_12__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_12__BIOS_SCRATCH_12_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_13
+#define BIF_BX1_BIOS_SCRATCH_13__BIOS_SCRATCH_13__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_13__BIOS_SCRATCH_13_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_14
+#define BIF_BX1_BIOS_SCRATCH_14__BIOS_SCRATCH_14__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_14__BIOS_SCRATCH_14_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_15
+#define BIF_BX1_BIOS_SCRATCH_15__BIOS_SCRATCH_15__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_15__BIOS_SCRATCH_15_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIF_RLC_INTR_CNTL
+//BIF_BX1_BIF_VCE_INTR_CNTL
+//BIF_BX1_BIF_UVD_INTR_CNTL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR1
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR2
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR3
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR4
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR5
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR6
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR7
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_CNTL
+#define BIF_BX1_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE_MASK                                                         0x000000FFL
+//BIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL
+#define BIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL__SHIFT                                                 0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL_MASK                                                   0xFFFFFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ONE_CPL
+#define BIF_BX1_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL__SHIFT                                                   0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL
+#define BIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL__SHIFT                                 0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL_MASK                                   0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_0
+#define BIF_BX1_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_1
+#define BIF_BX1_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_2
+#define BIF_BX1_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_3
+#define BIF_BX1_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_4
+#define BIF_BX1_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_5
+#define BIF_BX1_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_6
+#define BIF_BX1_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_7
+#define BIF_BX1_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_8
+#define BIF_BX1_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_9
+#define BIF_BX1_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_10
+#define BIF_BX1_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_11
+#define BIF_BX1_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_12
+#define BIF_BX1_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_13
+#define BIF_BX1_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_14
+#define BIF_BX1_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_15
+#define BIF_BX1_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_0
+#define BIF_BX1_FW_SCRATCH_0__FW_SCRATCH_0__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_0__FW_SCRATCH_0_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_1
+#define BIF_BX1_FW_SCRATCH_1__FW_SCRATCH_1__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_1__FW_SCRATCH_1_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_2
+#define BIF_BX1_FW_SCRATCH_2__FW_SCRATCH_2__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_2__FW_SCRATCH_2_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_3
+#define BIF_BX1_FW_SCRATCH_3__FW_SCRATCH_3__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_3__FW_SCRATCH_3_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_4
+#define BIF_BX1_FW_SCRATCH_4__FW_SCRATCH_4__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_4__FW_SCRATCH_4_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_5
+#define BIF_BX1_FW_SCRATCH_5__FW_SCRATCH_5__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_5__FW_SCRATCH_5_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_6
+#define BIF_BX1_FW_SCRATCH_6__FW_SCRATCH_6__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_6__FW_SCRATCH_6_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_7
+#define BIF_BX1_FW_SCRATCH_7__FW_SCRATCH_7__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_7__FW_SCRATCH_7_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_8
+#define BIF_BX1_FW_SCRATCH_8__FW_SCRATCH_8__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_8__FW_SCRATCH_8_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_9
+#define BIF_BX1_FW_SCRATCH_9__FW_SCRATCH_9__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_9__FW_SCRATCH_9_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_10
+#define BIF_BX1_FW_SCRATCH_10__FW_SCRATCH_10__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_10__FW_SCRATCH_10_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_11
+#define BIF_BX1_FW_SCRATCH_11__FW_SCRATCH_11__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_11__FW_SCRATCH_11_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_12
+#define BIF_BX1_FW_SCRATCH_12__FW_SCRATCH_12__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_12__FW_SCRATCH_12_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_13
+#define BIF_BX1_FW_SCRATCH_13__FW_SCRATCH_13__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_13__FW_SCRATCH_13_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_14
+#define BIF_BX1_FW_SCRATCH_14__FW_SCRATCH_14__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_14__FW_SCRATCH_14_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_15
+#define BIF_BX1_FW_SCRATCH_15__FW_SCRATCH_15__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_15__FW_SCRATCH_15_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_4
+#define BIF_BX1_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_5
+#define BIF_BX1_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_6
+#define BIF_BX1_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_7
+#define BIF_BX1_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_8
+#define BIF_BX1_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_9
+#define BIF_BX1_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_10
+#define BIF_BX1_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_11
+#define BIF_BX1_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_12
+#define BIF_BX1_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_13
+#define BIF_BX1_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_14
+#define BIF_BX1_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_15
+#define BIF_BX1_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_pf_SYSPFVFDEC
+//BIF_BX_PF1_MM_INDEX
+#define BIF_BX_PF1_MM_INDEX__MM_OFFSET__SHIFT                                                                 0x0
+#define BIF_BX_PF1_MM_INDEX__MM_APER__SHIFT                                                                   0x1f
+#define BIF_BX_PF1_MM_INDEX__MM_OFFSET_MASK                                                                   0x7FFFFFFFL
+#define BIF_BX_PF1_MM_INDEX__MM_APER_MASK                                                                     0x80000000L
+//BIF_BX_PF1_MM_DATA
+#define BIF_BX_PF1_MM_DATA__MM_DATA__SHIFT                                                                    0x0
+#define BIF_BX_PF1_MM_DATA__MM_DATA_MASK                                                                      0xFFFFFFFFL
+//BIF_BX_PF1_MM_INDEX_HI
+#define BIF_BX_PF1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                           0x0
+#define BIF_BX_PF1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_BIFDEC1
+//BIF_BX1_CC_BIF_BX_STRAP0
+#define BIF_BX1_CC_BIF_BX_STRAP0__STRAP_RESERVED__SHIFT                                                       0x19
+#define BIF_BX1_CC_BIF_BX_STRAP0__STRAP_RESERVED_MASK                                                         0xFE000000L
+//BIF_BX1_CC_BIF_BX_PINSTRAP0
+//BIF_BX1_BIF_MM_INDACCESS_CNTL
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__WRITE_DIS__SHIFT                                                       0x0
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS__SHIFT                                                0x1
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__WRITE_DIS_MASK                                                         0x00000001L
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS_MASK                                                  0x00000002L
+//BIF_BX1_BUS_CNTL
+#define BIF_BX1_BUS_CNTL__VGA_REG_COHERENCY_DIS__SHIFT                                                        0x6
+#define BIF_BX1_BUS_CNTL__VGA_MEM_COHERENCY_DIS__SHIFT                                                        0x7
+#define BIF_BX1_BUS_CNTL__SET_AZ_TC__SHIFT                                                                    0xa
+#define BIF_BX1_BUS_CNTL__SET_MC_TC__SHIFT                                                                    0xd
+#define BIF_BX1_BUS_CNTL__ZERO_BE_WR_EN__SHIFT                                                                0x10
+#define BIF_BX1_BUS_CNTL__ZERO_BE_RD_EN__SHIFT                                                                0x11
+#define BIF_BX1_BUS_CNTL__RD_STALL_IO_WR__SHIFT                                                               0x12
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS__SHIFT                                              0x18
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS__SHIFT                                          0x19
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS__SHIFT                                         0x1a
+#define BIF_BX1_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS__SHIFT                                        0x1b
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS__SHIFT                                          0x1c
+#define BIF_BX1_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN__SHIFT                                                     0x1d
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_WR_EN__SHIFT                                                          0x1e
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_RD_EN__SHIFT                                                          0x1f
+#define BIF_BX1_BUS_CNTL__VGA_REG_COHERENCY_DIS_MASK                                                          0x00000040L
+#define BIF_BX1_BUS_CNTL__VGA_MEM_COHERENCY_DIS_MASK                                                          0x00000080L
+#define BIF_BX1_BUS_CNTL__SET_AZ_TC_MASK                                                                      0x00001C00L
+#define BIF_BX1_BUS_CNTL__SET_MC_TC_MASK                                                                      0x0000E000L
+#define BIF_BX1_BUS_CNTL__ZERO_BE_WR_EN_MASK                                                                  0x00010000L
+#define BIF_BX1_BUS_CNTL__ZERO_BE_RD_EN_MASK                                                                  0x00020000L
+#define BIF_BX1_BUS_CNTL__RD_STALL_IO_WR_MASK                                                                 0x00040000L
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS_MASK                                                0x01000000L
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS_MASK                                            0x02000000L
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS_MASK                                           0x04000000L
+#define BIF_BX1_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS_MASK                                          0x08000000L
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS_MASK                                            0x10000000L
+#define BIF_BX1_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN_MASK                                                       0x20000000L
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_WR_EN_MASK                                                            0x40000000L
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_RD_EN_MASK                                                            0x80000000L
+//BIF_BX1_BIF_SCRATCH0
+#define BIF_BX1_BIF_SCRATCH0__BIF_SCRATCH0__SHIFT                                                             0x0
+#define BIF_BX1_BIF_SCRATCH0__BIF_SCRATCH0_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_BIF_SCRATCH1
+#define BIF_BX1_BIF_SCRATCH1__BIF_SCRATCH1__SHIFT                                                             0x0
+#define BIF_BX1_BIF_SCRATCH1__BIF_SCRATCH1_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_BX_RESET_EN
+#define BIF_BX1_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN__SHIFT                                                  0x10
+#define BIF_BX1_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN_MASK                                                    0x00010000L
+//BIF_BX1_MM_CFGREGS_CNTL
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL__SHIFT                                                       0x0
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL__SHIFT                                                        0x6
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN__SHIFT                                                       0x1f
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL_MASK                                                         0x00000007L
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL_MASK                                                          0x000000C0L
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN_MASK                                                         0x80000000L
+//BIF_BX1_BX_RESET_CNTL
+#define BIF_BX1_BX_RESET_CNTL__LINK_TRAIN_EN__SHIFT                                                           0x0
+#define BIF_BX1_BX_RESET_CNTL__LINK_TRAIN_EN_MASK                                                             0x00000001L
+//BIF_BX1_INTERRUPT_CNTL
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE__SHIFT                                                   0x0
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_EN__SHIFT                                                         0x1
+#define BIF_BX1_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN__SHIFT                                                     0x3
+#define BIF_BX1_INTERRUPT_CNTL__IH_INTR_DLY_CNTR__SHIFT                                                       0x4
+#define BIF_BX1_INTERRUPT_CNTL__GEN_IH_INT_EN__SHIFT                                                          0x8
+#define BIF_BX1_INTERRUPT_CNTL__BIF_RB_REQ_NONSNOOP_EN__SHIFT                                                 0xf
+#define BIF_BX1_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN__SHIFT                                               0x10
+#define BIF_BX1_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS__SHIFT                                   0x11
+#define BIF_BX1_INTERRUPT_CNTL__BIF_RB_REQ_RELAX_ORDER_EN__SHIFT                                              0x12
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK                                                     0x00000001L
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_EN_MASK                                                           0x00000002L
+#define BIF_BX1_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK                                                       0x00000008L
+#define BIF_BX1_INTERRUPT_CNTL__IH_INTR_DLY_CNTR_MASK                                                         0x000000F0L
+#define BIF_BX1_INTERRUPT_CNTL__GEN_IH_INT_EN_MASK                                                            0x00000100L
+#define BIF_BX1_INTERRUPT_CNTL__BIF_RB_REQ_NONSNOOP_EN_MASK                                                   0x00008000L
+#define BIF_BX1_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN_MASK                                                 0x00010000L
+#define BIF_BX1_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS_MASK                                     0x00020000L
+#define BIF_BX1_INTERRUPT_CNTL__BIF_RB_REQ_RELAX_ORDER_EN_MASK                                                0x00040000L
+//BIF_BX1_INTERRUPT_CNTL2
+#define BIF_BX1_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR__SHIFT                                                      0x0
+#define BIF_BX1_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR_MASK                                                        0xFFFFFFFFL
+//BIF_BX1_CLKREQB_PAD_CNTL
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_A__SHIFT                                                        0x0
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL__SHIFT                                                      0x1
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE__SHIFT                                                     0x2
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE__SHIFT                                                    0x3
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0__SHIFT                                                      0x5
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1__SHIFT                                                      0x6
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2__SHIFT                                                      0x7
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3__SHIFT                                                      0x8
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN__SHIFT                                                    0x9
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE__SHIFT                                                     0xa
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN__SHIFT                                                   0xb
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN__SHIFT                                                  0xc
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y__SHIFT                                                        0xd
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_A_MASK                                                          0x00000001L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL_MASK                                                        0x00000002L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE_MASK                                                       0x00000004L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE_MASK                                                      0x00000018L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0_MASK                                                        0x00000020L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1_MASK                                                        0x00000040L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2_MASK                                                        0x00000080L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3_MASK                                                        0x00000100L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN_MASK                                                      0x00000200L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE_MASK                                                       0x00000400L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN_MASK                                                     0x00000800L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN_MASK                                                    0x00001000L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y_MASK                                                          0x00002000L
+//BIF_BX1_BIF_FEATURES_CONTROL_MISC
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS__SHIFT                                          0x0
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS__SHIFT                                          0x1
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS__SHIFT                                          0x2
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS__SHIFT                                          0x3
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_RB_MSI_VEC_NOT_ENABLED_MODE__SHIFT                             0xb
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_RB_SET_OVERFLOW_EN__SHIFT                                      0xc
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS__SHIFT                                          0xd
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS__SHIFT                                       0xe
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BME_HDL_NONVIR_EN__SHIFT                                           0xf
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT__SHIFT                                           0x10
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR__SHIFT                   0x19
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS_MASK                                            0x00000001L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS_MASK                                            0x00000002L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS_MASK                                            0x00000004L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS_MASK                                            0x00000008L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_RB_MSI_VEC_NOT_ENABLED_MODE_MASK                               0x00000800L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_RB_SET_OVERFLOW_EN_MASK                                        0x00001000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS_MASK                                            0x00002000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS_MASK                                         0x00004000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BME_HDL_NONVIR_EN_MASK                                             0x00008000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT_MASK                                             0x01FF0000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR_MASK                     0x02000000L
+//BIF_BX1_HDP_ATOMIC_CONTROL_MISC
+#define BIF_BX1_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT__SHIFT                                      0x0
+#define BIF_BX1_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT_MASK                                        0x000000FFL
+//BIF_BX1_BIF_DOORBELL_CNTL
+#define BIF_BX1_BIF_DOORBELL_CNTL__SELF_RING_DIS__SHIFT                                                       0x0
+#define BIF_BX1_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS__SHIFT                                                     0x1
+#define BIF_BX1_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN__SHIFT                                                    0x2
+#define BIF_BX1_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS__SHIFT                                         0x3
+#define BIF_BX1_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN__SHIFT                                                 0x4
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS__SHIFT                                                  0x18
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0__SHIFT                                               0x19
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1__SHIFT                                               0x1a
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2__SHIFT                                               0x1b
+#define BIF_BX1_BIF_DOORBELL_CNTL__SELF_RING_DIS_MASK                                                         0x00000001L
+#define BIF_BX1_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS_MASK                                                       0x00000002L
+#define BIF_BX1_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN_MASK                                                      0x00000004L
+#define BIF_BX1_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS_MASK                                           0x00000008L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN_MASK                                                   0x00000010L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS_MASK                                                    0x01000000L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0_MASK                                                 0x02000000L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1_MASK                                                 0x04000000L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2_MASK                                                 0x08000000L
+//BIF_BX1_BIF_DOORBELL_INT_CNTL
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS__SHIFT                                       0x0
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS__SHIFT                                      0x1
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS__SHIFT                            0x2
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR__SHIFT                                        0x10
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR__SHIFT                                       0x11
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR__SHIFT                             0x12
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                            0x17
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE__SHIFT                                      0x18
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE__SHIFT                                     0x19
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE__SHIFT                           0x1a
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                               0x1c
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                          0x1d
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                          0x1e
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                              0x1f
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS_MASK                                         0x00000001L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS_MASK                                        0x00000002L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS_MASK                              0x00000004L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR_MASK                                          0x00010000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR_MASK                                         0x00020000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR_MASK                               0x00040000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE_MASK                              0x00800000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE_MASK                                        0x01000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE_MASK                                       0x02000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE_MASK                             0x04000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE_MASK                                 0x10000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                            0x20000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                            0x40000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE_MASK                                0x80000000L
+//BIF_BX1_BIF_FB_EN
+#define BIF_BX1_BIF_FB_EN__FB_READ_EN__SHIFT                                                                  0x0
+#define BIF_BX1_BIF_FB_EN__FB_WRITE_EN__SHIFT                                                                 0x1
+#define BIF_BX1_BIF_FB_EN__FB_READ_EN_MASK                                                                    0x00000001L
+#define BIF_BX1_BIF_FB_EN__FB_WRITE_EN_MASK                                                                   0x00000002L
+//BIF_BX1_BIF_INTR_CNTL
+#define BIF_BX1_BIF_INTR_CNTL__RAS_INTR_VEC_SEL__SHIFT                                                        0x0
+#define BIF_BX1_BIF_INTR_CNTL__RAS_INTR_VEC_SEL_MASK                                                          0x00000001L
+//BIF_BX1_BIF_MST_TRANS_PENDING_VF
+#define BIF_BX1_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX1_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX1_BIF_SLV_TRANS_PENDING_VF
+#define BIF_BX1_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX1_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX1_BACO_CNTL
+#define BIF_BX1_BACO_CNTL__BACO_EN__SHIFT                                                                     0x0
+#define BIF_BX1_BACO_CNTL__BACO_DUMMY_EN__SHIFT                                                               0x2
+#define BIF_BX1_BACO_CNTL__BACO_POWER_OFF__SHIFT                                                              0x3
+#define BIF_BX1_BACO_CNTL__BACO_DSTATE_BYPASS__SHIFT                                                          0x5
+#define BIF_BX1_BACO_CNTL__BACO_RST_INTR_MASK__SHIFT                                                          0x6
+#define BIF_BX1_BACO_CNTL__BACO_MODE__SHIFT                                                                   0x8
+#define BIF_BX1_BACO_CNTL__RCU_BIF_CONFIG_DONE__SHIFT                                                         0x9
+#define BIF_BX1_BACO_CNTL__PWRGOOD_VDDSOC__SHIFT                                                              0x10
+#define BIF_BX1_BACO_CNTL__BACO_AUTO_EXIT__SHIFT                                                              0x1f
+#define BIF_BX1_BACO_CNTL__BACO_EN_MASK                                                                       0x00000001L
+#define BIF_BX1_BACO_CNTL__BACO_DUMMY_EN_MASK                                                                 0x00000004L
+#define BIF_BX1_BACO_CNTL__BACO_POWER_OFF_MASK                                                                0x00000008L
+#define BIF_BX1_BACO_CNTL__BACO_DSTATE_BYPASS_MASK                                                            0x00000020L
+#define BIF_BX1_BACO_CNTL__BACO_RST_INTR_MASK_MASK                                                            0x00000040L
+#define BIF_BX1_BACO_CNTL__BACO_MODE_MASK                                                                     0x00000100L
+#define BIF_BX1_BACO_CNTL__RCU_BIF_CONFIG_DONE_MASK                                                           0x00000200L
+#define BIF_BX1_BACO_CNTL__PWRGOOD_VDDSOC_MASK                                                                0x00010000L
+#define BIF_BX1_BACO_CNTL__BACO_AUTO_EXIT_MASK                                                                0x80000000L
+//BIF_BX1_BIF_BACO_EXIT_TIME0
+#define BIF_BX1_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER__SHIFT                                          0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER_MASK                                            0x000FFFFFL
+//BIF_BX1_BIF_BACO_EXIT_TIMER1
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER__SHIFT                                         0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN__SHIFT                                            0x18
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS__SHIFT                                                 0x1a
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH__SHIFT                                           0x1b
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW__SHIFT                                            0x1c
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL__SHIFT                                                    0x1d
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS__SHIFT                                     0x1f
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER_MASK                                           0x000FFFFFL
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN_MASK                                              0x01000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS_MASK                                                   0x04000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH_MASK                                             0x08000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW_MASK                                              0x10000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL_MASK                                                      0x60000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS_MASK                                       0x80000000L
+//BIF_BX1_BIF_BACO_EXIT_TIMER2
+#define BIF_BX1_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER__SHIFT                                         0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER_MASK                                           0x000FFFFFL
+//BIF_BX1_BIF_BACO_EXIT_TIMER3
+#define BIF_BX1_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER__SHIFT                                     0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER_MASK                                       0x000FFFFFL
+//BIF_BX1_BIF_BACO_EXIT_TIMER4
+#define BIF_BX1_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER__SHIFT                                      0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER_MASK                                        0x000FFFFFL
+//BIF_BX1_MEM_TYPE_CNTL
+#define BIF_BX1_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3__SHIFT                                                        0x0
+#define BIF_BX1_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3_MASK                                                          0x00000001L
+//BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE__SHIFT                                                     0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE__SHIFT                                                  0x1
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE__SHIFT                                                    0x8
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE_MASK                                                       0x00000001L
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE_MASK                                                    0x00000002L
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE_MASK                                                      0x00000100L
+//BIF_BX1_NBIF_GFX_ADDR_LUT_0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_0__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_0__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_1
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_1__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_1__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_2
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_2__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_2__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_3
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_3__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_3__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_4
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_4__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_4__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_5
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_5__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_5__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_6
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_6__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_6__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_7
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_7__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_7__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_8
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_8__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_8__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_9
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_9__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_9__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_10
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_10__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_10__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_11
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_11__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_11__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_12
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_12__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_12__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_13
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_13__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_13__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_14
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_14__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_14__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_15
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_15__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_15__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_VF_REGWR_EN
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF0__SHIFT                                                           0x0
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF1__SHIFT                                                           0x1
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF2__SHIFT                                                           0x2
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF3__SHIFT                                                           0x3
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF4__SHIFT                                                           0x4
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF5__SHIFT                                                           0x5
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF6__SHIFT                                                           0x6
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF7__SHIFT                                                           0x7
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF8__SHIFT                                                           0x8
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF9__SHIFT                                                           0x9
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF10__SHIFT                                                          0xa
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF11__SHIFT                                                          0xb
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF12__SHIFT                                                          0xc
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF13__SHIFT                                                          0xd
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF14__SHIFT                                                          0xe
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF15__SHIFT                                                          0xf
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF16__SHIFT                                                          0x10
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF17__SHIFT                                                          0x11
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF18__SHIFT                                                          0x12
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF19__SHIFT                                                          0x13
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF20__SHIFT                                                          0x14
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF21__SHIFT                                                          0x15
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF22__SHIFT                                                          0x16
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF23__SHIFT                                                          0x17
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF24__SHIFT                                                          0x18
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF25__SHIFT                                                          0x19
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF26__SHIFT                                                          0x1a
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF27__SHIFT                                                          0x1b
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF28__SHIFT                                                          0x1c
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF29__SHIFT                                                          0x1d
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF30__SHIFT                                                          0x1e
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF0_MASK                                                             0x00000001L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF1_MASK                                                             0x00000002L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF2_MASK                                                             0x00000004L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF3_MASK                                                             0x00000008L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF4_MASK                                                             0x00000010L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF5_MASK                                                             0x00000020L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF6_MASK                                                             0x00000040L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF7_MASK                                                             0x00000080L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF8_MASK                                                             0x00000100L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF9_MASK                                                             0x00000200L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF10_MASK                                                            0x00000400L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF11_MASK                                                            0x00000800L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF12_MASK                                                            0x00001000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF13_MASK                                                            0x00002000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF14_MASK                                                            0x00004000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF15_MASK                                                            0x00008000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF16_MASK                                                            0x00010000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF17_MASK                                                            0x00020000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF18_MASK                                                            0x00040000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF19_MASK                                                            0x00080000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF20_MASK                                                            0x00100000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF21_MASK                                                            0x00200000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF22_MASK                                                            0x00400000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF23_MASK                                                            0x00800000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF24_MASK                                                            0x01000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF25_MASK                                                            0x02000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF26_MASK                                                            0x04000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF27_MASK                                                            0x08000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF28_MASK                                                            0x10000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF29_MASK                                                            0x20000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF30_MASK                                                            0x40000000L
+//BIF_BX1_VF_DOORBELL_EN
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF0__SHIFT                                                     0x0
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF1__SHIFT                                                     0x1
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF2__SHIFT                                                     0x2
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF3__SHIFT                                                     0x3
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF4__SHIFT                                                     0x4
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF5__SHIFT                                                     0x5
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF6__SHIFT                                                     0x6
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF7__SHIFT                                                     0x7
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF8__SHIFT                                                     0x8
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF9__SHIFT                                                     0x9
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF10__SHIFT                                                    0xa
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF11__SHIFT                                                    0xb
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF12__SHIFT                                                    0xc
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF13__SHIFT                                                    0xd
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF14__SHIFT                                                    0xe
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF15__SHIFT                                                    0xf
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF16__SHIFT                                                    0x10
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF17__SHIFT                                                    0x11
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF18__SHIFT                                                    0x12
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF19__SHIFT                                                    0x13
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF20__SHIFT                                                    0x14
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF21__SHIFT                                                    0x15
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF22__SHIFT                                                    0x16
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF23__SHIFT                                                    0x17
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF24__SHIFT                                                    0x18
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF25__SHIFT                                                    0x19
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF26__SHIFT                                                    0x1a
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF27__SHIFT                                                    0x1b
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF28__SHIFT                                                    0x1c
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF29__SHIFT                                                    0x1d
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF30__SHIFT                                                    0x1e
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_RD_LOG_DIS__SHIFT                                                 0x1f
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF0_MASK                                                       0x00000001L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF1_MASK                                                       0x00000002L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF2_MASK                                                       0x00000004L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF3_MASK                                                       0x00000008L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF4_MASK                                                       0x00000010L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF5_MASK                                                       0x00000020L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF6_MASK                                                       0x00000040L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF7_MASK                                                       0x00000080L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF8_MASK                                                       0x00000100L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF9_MASK                                                       0x00000200L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF10_MASK                                                      0x00000400L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF11_MASK                                                      0x00000800L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF12_MASK                                                      0x00001000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF13_MASK                                                      0x00002000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF14_MASK                                                      0x00004000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF15_MASK                                                      0x00008000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF16_MASK                                                      0x00010000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF17_MASK                                                      0x00020000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF18_MASK                                                      0x00040000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF19_MASK                                                      0x00080000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF20_MASK                                                      0x00100000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF21_MASK                                                      0x00200000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF22_MASK                                                      0x00400000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF23_MASK                                                      0x00800000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF24_MASK                                                      0x01000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF25_MASK                                                      0x02000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF26_MASK                                                      0x04000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF27_MASK                                                      0x08000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF28_MASK                                                      0x10000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF29_MASK                                                      0x20000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF30_MASK                                                      0x40000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_RD_LOG_DIS_MASK                                                   0x80000000L
+//BIF_BX1_VF_FB_EN
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF0__SHIFT                                                                 0x0
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF1__SHIFT                                                                 0x1
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF2__SHIFT                                                                 0x2
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF3__SHIFT                                                                 0x3
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF4__SHIFT                                                                 0x4
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF5__SHIFT                                                                 0x5
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF6__SHIFT                                                                 0x6
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF7__SHIFT                                                                 0x7
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF8__SHIFT                                                                 0x8
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF9__SHIFT                                                                 0x9
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF10__SHIFT                                                                0xa
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF11__SHIFT                                                                0xb
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF12__SHIFT                                                                0xc
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF13__SHIFT                                                                0xd
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF14__SHIFT                                                                0xe
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF15__SHIFT                                                                0xf
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF16__SHIFT                                                                0x10
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF17__SHIFT                                                                0x11
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF18__SHIFT                                                                0x12
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF19__SHIFT                                                                0x13
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF20__SHIFT                                                                0x14
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF21__SHIFT                                                                0x15
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF22__SHIFT                                                                0x16
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF23__SHIFT                                                                0x17
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF24__SHIFT                                                                0x18
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF25__SHIFT                                                                0x19
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF26__SHIFT                                                                0x1a
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF27__SHIFT                                                                0x1b
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF28__SHIFT                                                                0x1c
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF29__SHIFT                                                                0x1d
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF30__SHIFT                                                                0x1e
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF0_MASK                                                                   0x00000001L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF1_MASK                                                                   0x00000002L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF2_MASK                                                                   0x00000004L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF3_MASK                                                                   0x00000008L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF4_MASK                                                                   0x00000010L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF5_MASK                                                                   0x00000020L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF6_MASK                                                                   0x00000040L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF7_MASK                                                                   0x00000080L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF8_MASK                                                                   0x00000100L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF9_MASK                                                                   0x00000200L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF10_MASK                                                                  0x00000400L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF11_MASK                                                                  0x00000800L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF12_MASK                                                                  0x00001000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF13_MASK                                                                  0x00002000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF14_MASK                                                                  0x00004000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF15_MASK                                                                  0x00008000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF16_MASK                                                                  0x00010000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF17_MASK                                                                  0x00020000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF18_MASK                                                                  0x00040000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF19_MASK                                                                  0x00080000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF20_MASK                                                                  0x00100000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF21_MASK                                                                  0x00200000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF22_MASK                                                                  0x00400000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF23_MASK                                                                  0x00800000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF24_MASK                                                                  0x01000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF25_MASK                                                                  0x02000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF26_MASK                                                                  0x04000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF27_MASK                                                                  0x08000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF28_MASK                                                                  0x10000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF29_MASK                                                                  0x20000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF30_MASK                                                                  0x40000000L
+//BIF_BX1_VF_REGWR_STATUS
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF0__SHIFT                                                   0x0
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF1__SHIFT                                                   0x1
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF2__SHIFT                                                   0x2
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF3__SHIFT                                                   0x3
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF4__SHIFT                                                   0x4
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF5__SHIFT                                                   0x5
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF6__SHIFT                                                   0x6
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF7__SHIFT                                                   0x7
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF8__SHIFT                                                   0x8
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF9__SHIFT                                                   0x9
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF10__SHIFT                                                  0xa
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF11__SHIFT                                                  0xb
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF12__SHIFT                                                  0xc
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF13__SHIFT                                                  0xd
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF14__SHIFT                                                  0xe
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF15__SHIFT                                                  0xf
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF16__SHIFT                                                  0x10
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF17__SHIFT                                                  0x11
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF18__SHIFT                                                  0x12
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF19__SHIFT                                                  0x13
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF20__SHIFT                                                  0x14
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF21__SHIFT                                                  0x15
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF22__SHIFT                                                  0x16
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF23__SHIFT                                                  0x17
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF24__SHIFT                                                  0x18
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF25__SHIFT                                                  0x19
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF26__SHIFT                                                  0x1a
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF27__SHIFT                                                  0x1b
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF28__SHIFT                                                  0x1c
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF29__SHIFT                                                  0x1d
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF30__SHIFT                                                  0x1e
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF0_MASK                                                     0x00000001L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF1_MASK                                                     0x00000002L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF2_MASK                                                     0x00000004L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF3_MASK                                                     0x00000008L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF4_MASK                                                     0x00000010L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF5_MASK                                                     0x00000020L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF6_MASK                                                     0x00000040L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF7_MASK                                                     0x00000080L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF8_MASK                                                     0x00000100L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF9_MASK                                                     0x00000200L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF10_MASK                                                    0x00000400L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF11_MASK                                                    0x00000800L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF12_MASK                                                    0x00001000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF13_MASK                                                    0x00002000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF14_MASK                                                    0x00004000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF15_MASK                                                    0x00008000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF16_MASK                                                    0x00010000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF17_MASK                                                    0x00020000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF18_MASK                                                    0x00040000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF19_MASK                                                    0x00080000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF20_MASK                                                    0x00100000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF21_MASK                                                    0x00200000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF22_MASK                                                    0x00400000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF23_MASK                                                    0x00800000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF24_MASK                                                    0x01000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF25_MASK                                                    0x02000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF26_MASK                                                    0x04000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF27_MASK                                                    0x08000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF28_MASK                                                    0x10000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF29_MASK                                                    0x20000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF30_MASK                                                    0x40000000L
+//BIF_BX1_VF_DOORBELL_STATUS
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF0__SHIFT                                             0x0
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF1__SHIFT                                             0x1
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF2__SHIFT                                             0x2
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF3__SHIFT                                             0x3
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF4__SHIFT                                             0x4
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF5__SHIFT                                             0x5
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF6__SHIFT                                             0x6
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF7__SHIFT                                             0x7
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF8__SHIFT                                             0x8
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF9__SHIFT                                             0x9
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF10__SHIFT                                            0xa
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF11__SHIFT                                            0xb
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF12__SHIFT                                            0xc
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF13__SHIFT                                            0xd
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF14__SHIFT                                            0xe
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF15__SHIFT                                            0xf
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF16__SHIFT                                            0x10
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF17__SHIFT                                            0x11
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF18__SHIFT                                            0x12
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF19__SHIFT                                            0x13
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF20__SHIFT                                            0x14
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF21__SHIFT                                            0x15
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF22__SHIFT                                            0x16
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF23__SHIFT                                            0x17
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF24__SHIFT                                            0x18
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF25__SHIFT                                            0x19
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF26__SHIFT                                            0x1a
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF27__SHIFT                                            0x1b
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF28__SHIFT                                            0x1c
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF29__SHIFT                                            0x1d
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF30__SHIFT                                            0x1e
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF0_MASK                                               0x00000001L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF1_MASK                                               0x00000002L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF2_MASK                                               0x00000004L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF3_MASK                                               0x00000008L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF4_MASK                                               0x00000010L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF5_MASK                                               0x00000020L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF6_MASK                                               0x00000040L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF7_MASK                                               0x00000080L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF8_MASK                                               0x00000100L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF9_MASK                                               0x00000200L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF10_MASK                                              0x00000400L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF11_MASK                                              0x00000800L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF12_MASK                                              0x00001000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF13_MASK                                              0x00002000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF14_MASK                                              0x00004000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF15_MASK                                              0x00008000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF16_MASK                                              0x00010000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF17_MASK                                              0x00020000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF18_MASK                                              0x00040000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF19_MASK                                              0x00080000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF20_MASK                                              0x00100000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF21_MASK                                              0x00200000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF22_MASK                                              0x00400000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF23_MASK                                              0x00800000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF24_MASK                                              0x01000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF25_MASK                                              0x02000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF26_MASK                                              0x04000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF27_MASK                                              0x08000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF28_MASK                                              0x10000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF29_MASK                                              0x20000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF30_MASK                                              0x40000000L
+//BIF_BX1_VF_FB_STATUS
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF0__SHIFT                                                         0x0
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF1__SHIFT                                                         0x1
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF2__SHIFT                                                         0x2
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF3__SHIFT                                                         0x3
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF4__SHIFT                                                         0x4
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF5__SHIFT                                                         0x5
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF6__SHIFT                                                         0x6
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF7__SHIFT                                                         0x7
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF8__SHIFT                                                         0x8
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF9__SHIFT                                                         0x9
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF10__SHIFT                                                        0xa
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF11__SHIFT                                                        0xb
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF12__SHIFT                                                        0xc
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF13__SHIFT                                                        0xd
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF14__SHIFT                                                        0xe
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF15__SHIFT                                                        0xf
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF16__SHIFT                                                        0x10
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF17__SHIFT                                                        0x11
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF18__SHIFT                                                        0x12
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF19__SHIFT                                                        0x13
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF20__SHIFT                                                        0x14
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF21__SHIFT                                                        0x15
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF22__SHIFT                                                        0x16
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF23__SHIFT                                                        0x17
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF24__SHIFT                                                        0x18
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF25__SHIFT                                                        0x19
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF26__SHIFT                                                        0x1a
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF27__SHIFT                                                        0x1b
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF28__SHIFT                                                        0x1c
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF29__SHIFT                                                        0x1d
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF30__SHIFT                                                        0x1e
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF0_MASK                                                           0x00000001L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF1_MASK                                                           0x00000002L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF2_MASK                                                           0x00000004L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF3_MASK                                                           0x00000008L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF4_MASK                                                           0x00000010L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF5_MASK                                                           0x00000020L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF6_MASK                                                           0x00000040L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF7_MASK                                                           0x00000080L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF8_MASK                                                           0x00000100L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF9_MASK                                                           0x00000200L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF10_MASK                                                          0x00000400L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF11_MASK                                                          0x00000800L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF12_MASK                                                          0x00001000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF13_MASK                                                          0x00002000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF14_MASK                                                          0x00004000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF15_MASK                                                          0x00008000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF16_MASK                                                          0x00010000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF17_MASK                                                          0x00020000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF18_MASK                                                          0x00040000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF19_MASK                                                          0x00080000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF20_MASK                                                          0x00100000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF21_MASK                                                          0x00200000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF22_MASK                                                          0x00400000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF23_MASK                                                          0x00800000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF24_MASK                                                          0x01000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF25_MASK                                                          0x02000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF26_MASK                                                          0x04000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF27_MASK                                                          0x08000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF28_MASK                                                          0x10000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF29_MASK                                                          0x20000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF30_MASK                                                          0x40000000L
+//BIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL
+#define BIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX1_REMAP_HDP_REG_FLUSH_CNTL
+#define BIF_BX1_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX1_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX1_BIF_RB_CNTL
+#define BIF_BX1_BIF_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define BIF_BX1_BIF_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define BIF_BX1_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT                                                     0x8
+#define BIF_BX1_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER__SHIFT                                                      0x9
+#define BIF_BX1_BIF_RB_CNTL__BIF_RB_TRAN__SHIFT                                                               0x11
+#define BIF_BX1_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL__SHIFT                                                  0x19
+#define BIF_BX1_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY__SHIFT                                                      0x1a
+#define BIF_BX1_BIF_RB_CNTL__RB_INTR_ARB_MODE__SHIFT                                                          0x1d
+#define BIF_BX1_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE__SHIFT                                                     0x1e
+#define BIF_BX1_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT                                                       0x1f
+#define BIF_BX1_BIF_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define BIF_BX1_BIF_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define BIF_BX1_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK                                                       0x00000100L
+#define BIF_BX1_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER_MASK                                                        0x00003E00L
+#define BIF_BX1_BIF_RB_CNTL__BIF_RB_TRAN_MASK                                                                 0x00020000L
+#define BIF_BX1_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL_MASK                                                    0x02000000L
+#define BIF_BX1_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY_MASK                                                        0x1C000000L
+#define BIF_BX1_BIF_RB_CNTL__RB_INTR_ARB_MODE_MASK                                                            0x20000000L
+#define BIF_BX1_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE_MASK                                                       0x40000000L
+#define BIF_BX1_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK                                                         0x80000000L
+//BIF_BX1_BIF_RB_BASE
+#define BIF_BX1_BIF_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define BIF_BX1_BIF_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//BIF_BX1_BIF_RB_RPTR
+#define BIF_BX1_BIF_RB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define BIF_BX1_BIF_RB_RPTR__OFFSET_MASK                                                                      0x0003FFFCL
+//BIF_BX1_BIF_RB_WPTR
+#define BIF_BX1_BIF_RB_WPTR__BIF_RB_OVERFLOW__SHIFT                                                           0x0
+#define BIF_BX1_BIF_RB_WPTR__OFFSET__SHIFT                                                                    0x2
+#define BIF_BX1_BIF_RB_WPTR__BIF_RB_OVERFLOW_MASK                                                             0x00000001L
+#define BIF_BX1_BIF_RB_WPTR__OFFSET_MASK                                                                      0x0003FFFCL
+//BIF_BX1_BIF_RB_WPTR_ADDR_HI
+#define BIF_BX1_BIF_RB_WPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_BIF_RB_WPTR_ADDR_HI__ADDR_MASK                                                                0x000000FFL
+//BIF_BX1_BIF_RB_WPTR_ADDR_LO
+#define BIF_BX1_BIF_RB_WPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define BIF_BX1_BIF_RB_WPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//BIF_BX1_MAILBOX_INDEX
+#define BIF_BX1_MAILBOX_INDEX__MAILBOX_INDEX__SHIFT                                                           0x0
+#define BIF_BX1_MAILBOX_INDEX__MAILBOX_INDEX_MASK                                                             0x0000001FL
+//BIF_BX1_BIF_MP1_INTR_CTRL
+#define BIF_BX1_BIF_MP1_INTR_CTRL__BACO_EXIT_DONE__SHIFT                                                      0x0
+#define BIF_BX1_BIF_MP1_INTR_CTRL__BACO_EXIT_DONE_MASK                                                        0x00000001L
+//BIF_BX1_BIF_PERSTB_PAD_CNTL
+#define BIF_BX1_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX1_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL_MASK                                                     0x0000FFFFL
+//BIF_BX1_BIF_PX_EN_PAD_CNTL
+#define BIF_BX1_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL__SHIFT                                                     0x0
+#define BIF_BX1_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL_MASK                                                       0x00000FFFL
+//BIF_BX1_BIF_REFPADKIN_PAD_CNTL
+#define BIF_BX1_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL__SHIFT                                             0x0
+#define BIF_BX1_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL_MASK                                               0x000000FFL
+//BIF_BX1_BIF_CLKREQB_PAD_CNTL
+#define BIF_BX1_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL__SHIFT                                                 0x0
+#define BIF_BX1_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_MASK                                                   0x7FFFFFFFL
+//BIF_BX1_BIF_PWRBRK_PAD_CNTL
+#define BIF_BX1_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX1_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL_MASK                                                     0x000000FFL
+//BIF_BX1_BIF_VCN0_GPUIOV_CFG_SIZE
+#define BIF_BX1_BIF_VCN0_GPUIOV_CFG_SIZE__VCN0_GPUIOV_CFG_SIZE__SHIFT                                         0x0
+#define BIF_BX1_BIF_VCN0_GPUIOV_CFG_SIZE__VCN0_GPUIOV_CFG_SIZE_MASK                                           0x0000000FL
+//BIF_BX1_BIF_VCN1_GPUIOV_CFG_SIZE
+#define BIF_BX1_BIF_VCN1_GPUIOV_CFG_SIZE__VCN1_GPUIOV_CFG_SIZE__SHIFT                                         0x0
+#define BIF_BX1_BIF_VCN1_GPUIOV_CFG_SIZE__VCN1_GPUIOV_CFG_SIZE_MASK                                           0x0000000FL
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_pf_BIFPFVFDEC1
+//BIF_BX_PF1_BIF_BME_STATUS
+#define BIF_BX_PF1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                                      0x0
+#define BIF_BX_PF1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                                0x10
+#define BIF_BX_PF1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                                        0x00000001L
+#define BIF_BX_PF1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                                  0x00010000L
+//BIF_BX_PF1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                                0x0
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                             0x1
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                                0x2
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                                    0x3
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                          0x10
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                                       0x11
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                          0x12
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                              0x13
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                                  0x00000001L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                               0x00000002L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                                  0x00000004L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                                      0x00000008L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                            0x00010000L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                                         0x00020000L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                            0x00040000L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                                0x00080000L
+//BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT          0x0
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK            0xFFFFFFFFL
+//BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT            0x0
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK              0xFFFFFFFFL
+//BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT                      0x0
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT                    0x1
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT                    0x8
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK                        0x00000001L
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK                      0x00000002L
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK                      0x000FFF00L
+//BIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                          0x0
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                            0x00000001L
+//BIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_PF1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                              0x0
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                              0x1
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                              0x2
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                              0x3
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                              0x4
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                              0x5
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                              0x6
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                              0x7
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                              0x8
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                              0x9
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                            0xa
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                            0xb
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                                        0xc
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                                        0xd
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                                        0xe
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                                        0xf
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                                        0x10
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                                        0x11
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                                        0x12
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                                        0x13
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                                        0x14
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                                        0x15
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                                       0x16
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                                       0x17
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                                       0x18
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                                       0x19
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                                       0x1a
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                                       0x1b
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                                       0x1c
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                                       0x1d
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                                       0x1e
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                                       0x1f
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                                0x00000001L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                                0x00000002L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                                0x00000004L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                                0x00000008L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                                0x00000010L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                                0x00000020L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                                0x00000040L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                                0x00000080L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                                0x00000100L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                                0x00000200L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                              0x00000400L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                              0x00000800L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                          0x00001000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                          0x00002000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                          0x00004000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                          0x00008000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                          0x00010000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                          0x00020000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                          0x00040000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                          0x00080000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                          0x00100000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                          0x00200000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                                         0x00400000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                                         0x00800000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                                         0x01000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                                         0x02000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                                         0x04000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                                         0x08000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                                         0x10000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                                         0x20000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                                         0x40000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                                         0x80000000L
+//BIF_BX_PF1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                             0x0
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                             0x1
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                             0x2
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                             0x3
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                             0x4
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                             0x5
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                             0x6
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                             0x7
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                             0x8
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                             0x9
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                           0xa
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                           0xb
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                                       0xc
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                                       0xd
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                                       0xe
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                                       0xf
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                                       0x10
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                                       0x11
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                                       0x12
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                                       0x13
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                                       0x14
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                                       0x15
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                                      0x16
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                                      0x17
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                                      0x18
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                                      0x19
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                                      0x1a
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                                      0x1b
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                                      0x1c
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                                      0x1d
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                                      0x1e
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                                      0x1f
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                               0x00000001L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                               0x00000002L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                               0x00000004L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                               0x00000008L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                               0x00000010L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                               0x00000020L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                               0x00000040L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                               0x00000080L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                               0x00000100L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                               0x00000200L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                             0x00000400L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                             0x00000800L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                                         0x00001000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                                         0x00002000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                                         0x00004000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                                         0x00008000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                                         0x00010000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                                         0x00020000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                                         0x00040000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                                         0x00080000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                                         0x00100000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                                         0x00200000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                                        0x00400000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                                        0x00800000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                                        0x01000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                                        0x02000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                                        0x04000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                                        0x08000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                                        0x10000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                                        0x20000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                                        0x40000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                                        0x80000000L
+//BIF_BX_PF1_BIF_TRANS_PENDING
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                            0x0
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                            0x1
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                              0x00000001L
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                              0x00000002L
+//BIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                                0x0
+#define BIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                                  0x00000001L
+//BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_CONTROL
+#define BIF_BX_PF1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                                      0x0
+#define BIF_BX_PF1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                                        0x1
+#define BIF_BX_PF1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                                      0x8
+#define BIF_BX_PF1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                                        0x9
+#define BIF_BX_PF1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                                        0x00000001L
+#define BIF_BX_PF1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                          0x00000002L
+#define BIF_BX_PF1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                                        0x00000100L
+#define BIF_BX_PF1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                          0x00000200L
+//BIF_BX_PF1_MAILBOX_INT_CNTL
+#define BIF_BX_PF1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                                      0x0
+#define BIF_BX_PF1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                                        0x1
+#define BIF_BX_PF1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                                        0x00000001L
+#define BIF_BX_PF1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                          0x00000002L
+//BIF_BX_PF1_BIF_VMHV_MAILBOX
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                                      0x0
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                                    0x1
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                                         0x8
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                                        0xf
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                                         0x10
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                                        0x17
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                          0x18
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                          0x19
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                                        0x00000001L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                                      0x00000002L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                           0x00000F00L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                          0x00008000L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                           0x000F0000L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                          0x00800000L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                            0x01000000L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                            0x02000000L
+//BIF_BX_PF1_PARTITION_COMPUTE_CAP
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                                  0x0
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                                  0x1
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                                  0x2
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                                  0x3
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                                  0x4
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                             0xa
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                                    0x00000001L
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                                    0x00000002L
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                                    0x00000004L
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                                    0x00000008L
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                                    0x00000010L
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                               0x0003FC00L
+//BIF_BX_PF1_PARTITION_MEM_CAP
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                                     0x0
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                                     0x1
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                                     0x2
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                                     0x3
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                                     0x5
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                                     0x7
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                                       0x00000001L
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                                       0x00000002L
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                                       0x00000004L
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                                       0x00000008L
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                                       0x00000020L
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                                       0x00000080L
+//BIF_BX_PF1_PARTITION_COMPUTE_STATUS
+#define BIF_BX_PF1_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                            0x4
+#define BIF_BX_PF1_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                              0x000000F0L
+//BIF_BX_PF1_PARTITION_MEM_STATUS
+#define BIF_BX_PF1_PARTITION_MEM_STATUS__CHANGE_STATUE__SHIFT                                                 0x0
+#define BIF_BX_PF1_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                                      0x4
+#define BIF_BX_PF1_PARTITION_MEM_STATUS__CHANGE_STATUE_MASK                                                   0x0000000FL
+#define BIF_BX_PF1_PARTITION_MEM_STATUS__NPS_MODE_MASK                                                        0x00000FF0L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_strap_BIFDEC1:1
+//RCC_STRAP2_RCC_BIF_STRAP0
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN4_DIS__SHIFT                                                      0x0
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT__SHIFT                              0x1
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_VGA_DIS_PIN__SHIFT                                                   0x2
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_MEM_AP_SIZE_PIN__SHIFT                                               0x3
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN__SHIFT                                               0x6
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PX_CAPABLE__SHIFT                                                    0x7
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3__SHIFT                                                 0x8
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN__SHIFT                                  0x9
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR__SHIFT                                         0xa
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN__SHIFT                                     0xb
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR__SHIFT                                              0xc
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR__SHIFT                                             0xd
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                     0xe
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                  0xf
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR__SHIFT                                              0x10
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR_DN__SHIFT                                           0x11
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_AUD_PIN__SHIFT                                                       0x12
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN3_DIS__SHIFT                                                      0x18
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4__SHIFT                                                 0x19
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_QUICKSIM_START__SHIFT                                                0x1a
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS__SHIFT                                   0x1c
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN__SHIFT                                      0x1d
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE__SHIFT                                                   0x1e
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN__SHIFT                                            0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN4_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT_MASK                                0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_VGA_DIS_PIN_MASK                                                     0x00000004L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_MEM_AP_SIZE_PIN_MASK                                                 0x00000038L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN_MASK                                                 0x00000040L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK                                                      0x00000080L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3_MASK                                                   0x00000100L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN_MASK                                    0x00000200L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR_MASK                                           0x00000400L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN_MASK                                       0x00000800L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR_MASK                                                0x00001000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR_MASK                                               0x00002000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                       0x00004000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                    0x00008000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR_MASK                                                0x00010000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_TC_ERR_DN_MASK                                             0x00020000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_AUD_PIN_MASK                                                         0x000C0000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN3_DIS_MASK                                                        0x01000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4_MASK                                                   0x02000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_QUICKSIM_START_MASK                                                  0x04000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING_MASK                                       0x08000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS_MASK                                     0x10000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN_MASK                                        0x20000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE_MASK                                                     0x40000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN_MASK                                              0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP1
+#define RCC_STRAP2_RCC_BIF_STRAP1__FUSESTRAP_VALID__SHIFT                                                     0x0
+#define RCC_STRAP2_RCC_BIF_STRAP1__ROMSTRAP_VALID__SHIFT                                                      0x1
+#define RCC_STRAP2_RCC_BIF_STRAP1__WRITE_DISABLE__SHIFT                                                       0x2
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN__SHIFT                                      0x3
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS__SHIFT                                     0x5
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE__SHIFT                                       0x6
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_READY__SHIFT                                               0x7
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN__SHIFT                                                  0x8
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN__SHIFT                                                 0x9
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE__SHIFT                                                  0xa
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE__SHIFT                                        0xc
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_HWREV_LSB2__SHIFT                                                    0xd
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWREV_LSB2__SHIFT                                                    0xf
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY__SHIFT                                             0x11
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS__SHIFT                                             0x12
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN__SHIFT                                                        0x13
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN__SHIFT                                                   0x14
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGIN_EN__SHIFT                                                     0x15
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN__SHIFT                                             0x16
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN__SHIFT                                     0x17
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_S5_REGS_ACCESS_DIS__SHIFT                                            0x18
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_S5_MMREG_WR_POSTED_EN__SHIFT                                         0x19
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE__SHIFT                                             0x1a
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN__SHIFT                                       0x1b
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN_EP__SHIFT                                                     0x1d
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN__SHIFT                                                         0x1e
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN_DN__SHIFT                                                      0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP1__FUSESTRAP_VALID_MASK                                                       0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP1__ROMSTRAP_VALID_MASK                                                        0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP1__WRITE_DISABLE_MASK                                                         0x00000004L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN_MASK                                        0x00000008L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS_MASK                                       0x00000020L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE_MASK                                         0x00000040L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_READY_MASK                                                 0x00000080L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN_MASK                                                    0x00000100L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN_MASK                                                   0x00000200L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE_MASK                                                    0x00000C00L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE_MASK                                          0x00001000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_HWREV_LSB2_MASK                                                      0x00006000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWREV_LSB2_MASK                                                      0x00018000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY_MASK                                               0x00020000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS_MASK                                               0x00040000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN_MASK                                                          0x00080000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN_MASK                                                     0x00100000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGIN_EN_MASK                                                       0x00200000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN_MASK                                               0x00400000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN_MASK                                       0x00800000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_S5_REGS_ACCESS_DIS_MASK                                              0x01000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_S5_MMREG_WR_POSTED_EN_MASK                                           0x02000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE_MASK                                               0x04000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN_MASK                                         0x18000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN_EP_MASK                                                       0x20000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN_MASK                                                           0x40000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN_DN_MASK                                                        0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP2
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS__SHIFT                                            0x3
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS__SHIFT                                            0x4
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN__SHIFT                                          0x7
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP2_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS_MASK                                              0x00000008L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS_MASK                                              0x00000010L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN_MASK                                            0x00000080L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP2_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP3
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP2_RCC_BIF_STRAP4
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER__SHIFT                                          0x0
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER__SHIFT                                           0x10
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER_MASK                                            0x0000FFFFL
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER_MASK                                             0xFFFF0000L
+//RCC_STRAP2_RCC_BIF_STRAP5
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+//RCC_STRAP2_RCC_BIF_STRAP6
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_GEN5_DIS__SHIFT                                                      0x0
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5__SHIFT                                                 0x1
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN__SHIFT                                                   0x2
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_GEN5_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5_MASK                                                   0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN_MASK                                                     0x00000004L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0__SHIFT                                0x13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0__SHIFT                                   0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0__SHIFT                            0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0__SHIFT                             0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0__SHIFT                             0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0__SHIFT                                      0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0_MASK                                            0x00010000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0_MASK                                  0x00080000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0_MASK                                     0x00E00000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0_MASK                              0x01000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0_MASK                               0x0E000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0_MASK                               0x70000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0_MASK                                        0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0__SHIFT                                   0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0_MASK                                     0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0__SHIFT                           0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0__SHIFT                  0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0__SHIFT                         0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0__SHIFT                       0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0__SHIFT                                0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0_MASK                                0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0_MASK                             0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0_MASK                    0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0_MASK                           0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0_MASK                         0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0_MASK                                  0x0007FFC0L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0__SHIFT                             0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0__SHIFT                                 0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0__SHIFT                                      0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0__SHIFT                                         0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0__SHIFT                               0x1e
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0_MASK                               0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0_MASK                                   0x0FFF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0_MASK                                        0x10000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0_MASK                                           0x20000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0_MASK                                 0x40000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0__SHIFT                                0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0_MASK                                  0x00FFFFFFL
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0__SHIFT                          0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0__SHIFT     0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0__SHIFT                        0x9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0__SHIFT                               0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0_MASK                            0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0_MASK       0x00000100L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0_MASK                          0x000FFE00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0_MASK                                 0xFFF00000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0__SHIFT                               0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0__SHIFT                             0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0__SHIFT                                 0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0__SHIFT                          0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0__SHIFT                                         0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0_MASK                                 0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0_MASK                               0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0_MASK                                   0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0_MASK                            0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0_MASK                                           0x00000010L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0__SHIFT                                          0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0__SHIFT                                      0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0__SHIFT                                          0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0__SHIFT                                      0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0__SHIFT                                        0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ERR_REPORTING_DIS_DEV0__SHIFT                                  0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0__SHIFT                             0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0__SHIFT                                0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0__SHIFT                                    0x9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0__SHIFT                              0xc
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0__SHIFT                      0xd
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0__SHIFT                                    0xe
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0__SHIFT                                            0xf
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0__SHIFT                                    0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0__SHIFT                                    0x11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0__SHIFT                             0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0__SHIFT                                   0x17
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0__SHIFT                              0x1a
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0__SHIFT                                    0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0_MASK                                   0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0_MASK                                            0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0_MASK                                        0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0_MASK                                            0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0_MASK                                        0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0_MASK                                          0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ERR_REPORTING_DIS_DEV0_MASK                                    0x00000040L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0_MASK                               0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0_MASK                                  0x00000100L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0_MASK                                      0x00000E00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0_MASK                                0x00001000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0_MASK                        0x00002000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0_MASK                                      0x00004000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0_MASK                                              0x00008000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0_MASK                                      0x00010000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0_MASK                                      0x00020000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0_MASK                               0x00700000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0_MASK                                     0x03800000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0_MASK                                0x1C000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0_MASK                                      0xE0000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0__SHIFT                     0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0__SHIFT                                             0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0__SHIFT                                          0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0__SHIFT                                0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0__SHIFT                                          0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0__SHIFT                                  0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0__SHIFT                                     0x9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0xb
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0xe
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0__SHIFT                                         0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0__SHIFT                                      0x1b
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0__SHIFT                                       0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0__SHIFT                                         0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0_MASK                       0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0_MASK                                               0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0_MASK                                            0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0_MASK                                  0x00000038L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0_MASK                                            0x00000040L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0_MASK                                    0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0_MASK                                       0x00000600L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x00003800L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0_MASK  0x0003C000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x001C0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0_MASK  0x01E00000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0_MASK                                           0x06000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0_MASK                                        0x18000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0_MASK                                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0_MASK                                           0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0__SHIFT                              0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0__SHIFT                              0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0_MASK                                0xFF000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0__SHIFT                        0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0__SHIFT                                 0x11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0__SHIFT                                  0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0__SHIFT                                           0x13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0__SHIFT                                           0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0__SHIFT                                        0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0__SHIFT                                0x16
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0__SHIFT                           0x17
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0__SHIFT                        0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0__SHIFT                        0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0__SHIFT                     0x1a
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0__SHIFT                         0x1b
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0__SHIFT                          0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0__SHIFT                       0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0__SHIFT                                            0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0_MASK                          0x00010000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0_MASK                                   0x00020000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0_MASK                                    0x00040000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0_MASK                                             0x00080000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0_MASK                                             0x00100000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0_MASK                                          0x00200000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0_MASK                                  0x00400000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0_MASK                             0x00800000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0_MASK                          0x01000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0_MASK                          0x02000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0_MASK                       0x04000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0_MASK                           0x08000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0_MASK                            0x10000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0_MASK                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0_MASK                                              0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0__SHIFT                                         0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0__SHIFT                         0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0__SHIFT                                    0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                      0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                      0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                   0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0xc
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0__SHIFT                              0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0__SHIFT                            0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0__SHIFT                     0x13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0__SHIFT                                  0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0_MASK                                           0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0_MASK                           0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0_MASK                                      0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                        0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                        0x00000040L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                     0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x00000F00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0x0000F000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0_MASK                                0x00030000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0_MASK                              0x00040000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0_MASK                       0x00080000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0_MASK                                      0x00100000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0_MASK                                    0x00E00000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x0F000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0xF0000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0__SHIFT                                        0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0__SHIFT                                    0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0__SHIFT                                    0xc
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0__SHIFT                                          0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0__SHIFT                                          0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0_MASK                                          0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0_MASK                                      0x00000F00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0_MASK                                      0x0000F000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0_MASK                                            0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0_MASK                                            0x1F000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0_MASK                                            0xE0000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0__SHIFT                              0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0__SHIFT                              0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0_MASK                                0xFF000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0__SHIFT                                       0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0_MASK                                         0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP1
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0__SHIFT                       0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0_MASK                                0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0_MASK                         0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP13
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0__SHIFT                                 0x8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0__SHIFT                                0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0_MASK                                   0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0_MASK                                   0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0_MASK                                  0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0_MASK                                  0xFF000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0__SHIFT                                      0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0_MASK                                        0x0000FFFFL
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0__SHIFT                                  0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0__SHIFT                     0x19
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0__SHIFT                       0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0_MASK                                   0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0_MASK                                    0x00FFF000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0_MASK                                        0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0_MASK                       0x3E000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0_MASK                         0x40000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0__SHIFT                               0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0_MASK                                     0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0_MASK                                 0x00FFF000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0__SHIFT                                   0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0__SHIFT                                0xd
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0_MASK                                0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0_MASK                                     0x00001000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0_MASK                                  0x01FFE000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0__SHIFT                            0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0_MASK                              0x00000FFFL
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP2
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0__SHIFT                                        0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0__SHIFT                                       0x6
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0__SHIFT                                   0x7
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0__SHIFT                                 0x9
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0__SHIFT                          0xe
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0__SHIFT                                          0xf
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0__SHIFT                                0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0__SHIFT                                          0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0__SHIFT                                           0x17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0__SHIFT                                   0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0__SHIFT                                        0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0__SHIFT                  0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0__SHIFT               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0__SHIFT                       0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0_MASK                                          0x00000001L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0_MASK                                         0x00000040L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0_MASK                                     0x00000080L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0_MASK                                   0x00003E00L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0_MASK                            0x00004000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0_MASK                                            0x00008000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0_MASK                                            0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0_MASK                                  0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0_MASK                                            0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0_MASK                                             0x00800000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0_MASK                                     0x07000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0_MASK                                       0x08000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0_MASK                                          0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0_MASK                    0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0_MASK                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0_MASK                         0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP26
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP26__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0__SHIFT                             0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP26__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0_MASK                               0x00000FFFL
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP3
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0__SHIFT                      0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0__SHIFT                              0x13
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0__SHIFT                                         0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0__SHIFT                                  0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0__SHIFT                                         0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0__SHIFT                        0x1a
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0__SHIFT                       0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0__SHIFT                                0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0__SHIFT                                       0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0__SHIFT                               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0__SHIFT                                          0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0_MASK                        0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0_MASK                                0x00080000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0_MASK                                           0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0_MASK                                    0x00E00000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0_MASK                                           0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0_MASK                          0x04000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0_MASK                         0x08000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0_MASK                                  0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0_MASK                                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0_MASK                                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0_MASK                                            0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP4
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0__SHIFT                                          0xa
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0__SHIFT                                       0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0__SHIFT                                     0x17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0__SHIFT                                   0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0_MASK                                   0x000003FFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0_MASK                                            0x00000400L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0_MASK                                         0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0_MASK                                       0x0F800000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0_MASK                                     0x70000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP5
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0__SHIFT                            0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0_MASK                                     0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0_MASK                              0x40000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0__SHIFT                                0x3
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0__SHIFT                                     0x4
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0__SHIFT                                      0x7
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_MEM_AP_SIZE_DEV0_F0__SHIFT                                     0x9
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0__SHIFT                                     0xd
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                           0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MEM_AP_SIZE_DEV0_F0__SHIFT                                  0x13
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0__SHIFT                                  0x17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VGA_DIS_DEV0_F0__SHIFT                                         0x1a
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0__SHIFT                                0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0__SHIFT                           0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0_MASK                                0x00000007L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0_MASK                                  0x00000008L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0_MASK                                       0x00000070L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0_MASK                                        0x00000080L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_MEM_AP_SIZE_DEV0_F0_MASK                                       0x00001E00L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0_MASK                                       0x0000E000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0_MASK                             0x00070000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MEM_AP_SIZE_DEV0_F0_MASK                                    0x00780000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0_MASK                                    0x03800000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VGA_DIS_DEV0_F0_MASK                                           0x04000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0_MASK                                  0x38000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0_MASK                             0xC0000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP9
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0__SHIFT                           0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0__SHIFT                               0x12
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0__SHIFT                        0x13
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0__SHIFT                                    0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0__SHIFT                                 0x16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0__SHIFT                                 0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0_MASK                             0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0_MASK                                 0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0_MASK                          0x00080000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0_MASK                                      0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0_MASK                                   0x00C00000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0_MASK                                   0x0F000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1__SHIFT                                    0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1__SHIFT                                         0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1__SHIFT                           0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1__SHIFT                                      0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1__SHIFT                                      0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1_MASK                                      0x000F0000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1_MASK                                      0x00F00000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1_MASK                                           0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1_MASK                             0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1_MASK                                        0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1_MASK                                        0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP2
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1__SHIFT                                   0x7
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1__SHIFT                                 0x9
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1__SHIFT                          0xe
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1__SHIFT                                0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1__SHIFT                                          0x15
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_VC_EN_DEV0_F1__SHIFT                                           0x17
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1__SHIFT                                   0x18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1__SHIFT                                        0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1__SHIFT                  0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1__SHIFT               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1__SHIFT                       0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1_MASK                                     0x00000080L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1_MASK                                   0x00003E00L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1_MASK                            0x00004000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1_MASK                                            0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1_MASK                                  0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1_MASK                                            0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_VC_EN_DEV0_F1_MASK                                             0x00800000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1_MASK                                     0x07000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1_MASK                                          0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1_MASK                    0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1_MASK                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1_MASK                         0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP20
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP21
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP22
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP23
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP24
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP25
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP3
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1__SHIFT                      0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1__SHIFT                              0x13
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1__SHIFT                                         0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1__SHIFT                                         0x18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1__SHIFT                        0x1a
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1__SHIFT                       0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1__SHIFT                                       0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1__SHIFT                               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1__SHIFT                                          0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1_MASK                        0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1_MASK                                0x00080000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1_MASK                                           0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1_MASK                                           0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1_MASK                          0x04000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1_MASK                         0x08000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1_MASK                                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1_MASK                                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1_MASK                                            0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP4
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1__SHIFT                                 0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1__SHIFT                                       0x15
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1__SHIFT                                     0x17
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1__SHIFT                                   0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1__SHIFT                                  0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1_MASK                                   0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1_MASK                                         0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1_MASK                                       0x0F800000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1_MASK                                     0x70000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1_MASK                                    0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP5
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1__SHIFT                            0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1_MASK                                     0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1_MASK                                       0x38000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1_MASK                              0x40000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP6
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1_MASK                                          0x00000001L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP7
+
+
+// addressBlock: aid_nbio_nbif0_gdc_hst_sion_SIONDEC
+//S2A_DOORBELL_ENTRY_0_CTRL
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE__SHIFT                                           0x0
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID__SHIFT                                             0x1
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE__SHIFT                                     0x6
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET__SHIFT                                     0x7
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE__SHIFT                                       0x11
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS__SHIFT                                0x19
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET__SHIFT                         0x1a
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE__SHIFT                               0x1c
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE_MASK                                             0x00000001L
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID_MASK                                               0x0000003EL
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE_MASK                                       0x00000040L
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET_MASK                                       0x0001FF80L
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE_MASK                                         0x01FE0000L
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS_MASK                                  0x02000000L
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET_MASK                           0x04000000L
+#define S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE_MASK                                 0xF0000000L
+//S2A_DOORBELL_ENTRY_1_CTRL
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE__SHIFT                                           0x0
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID__SHIFT                                             0x1
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE__SHIFT                                     0x6
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET__SHIFT                                     0x7
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE__SHIFT                                       0x11
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS__SHIFT                                0x19
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET__SHIFT                         0x1a
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE__SHIFT                               0x1c
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE_MASK                                             0x00000001L
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID_MASK                                               0x0000003EL
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE_MASK                                       0x00000040L
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET_MASK                                       0x0001FF80L
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE_MASK                                         0x01FE0000L
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS_MASK                                  0x02000000L
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET_MASK                           0x04000000L
+#define S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE_MASK                                 0xF0000000L
+//S2A_DOORBELL_ENTRY_2_CTRL
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE__SHIFT                                           0x0
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID__SHIFT                                             0x1
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE__SHIFT                                     0x6
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET__SHIFT                                     0x7
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE__SHIFT                                       0x11
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS__SHIFT                                0x19
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET__SHIFT                         0x1a
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE__SHIFT                               0x1c
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE_MASK                                             0x00000001L
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID_MASK                                               0x0000003EL
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE_MASK                                       0x00000040L
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET_MASK                                       0x0001FF80L
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE_MASK                                         0x01FE0000L
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS_MASK                                  0x02000000L
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET_MASK                           0x04000000L
+#define S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE_MASK                                 0xF0000000L
+//S2A_DOORBELL_ENTRY_3_CTRL
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE__SHIFT                                           0x0
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID__SHIFT                                             0x1
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE__SHIFT                                     0x6
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET__SHIFT                                     0x7
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE__SHIFT                                       0x11
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS__SHIFT                                0x19
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET__SHIFT                         0x1a
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE__SHIFT                               0x1c
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE_MASK                                             0x00000001L
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID_MASK                                               0x0000003EL
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE_MASK                                       0x00000040L
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET_MASK                                       0x0001FF80L
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE_MASK                                         0x01FE0000L
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS_MASK                                  0x02000000L
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET_MASK                           0x04000000L
+#define S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE_MASK                                 0xF0000000L
+//S2A_DOORBELL_ENTRY_4_CTRL
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE__SHIFT                                           0x0
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID__SHIFT                                             0x1
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE__SHIFT                                     0x6
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET__SHIFT                                     0x7
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE__SHIFT                                       0x11
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS__SHIFT                                0x19
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET__SHIFT                         0x1a
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE__SHIFT                               0x1c
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE_MASK                                             0x00000001L
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID_MASK                                               0x0000003EL
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE_MASK                                       0x00000040L
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET_MASK                                       0x0001FF80L
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE_MASK                                         0x01FE0000L
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS_MASK                                  0x02000000L
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET_MASK                           0x04000000L
+#define S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE_MASK                                 0xF0000000L
+//S2A_DOORBELL_ENTRY_5_CTRL
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE__SHIFT                                           0x0
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID__SHIFT                                             0x1
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE__SHIFT                                     0x6
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET__SHIFT                                     0x7
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE__SHIFT                                       0x11
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS__SHIFT                                0x19
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET__SHIFT                         0x1a
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE__SHIFT                               0x1c
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE_MASK                                             0x00000001L
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID_MASK                                               0x0000003EL
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE_MASK                                       0x00000040L
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET_MASK                                       0x0001FF80L
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE_MASK                                         0x01FE0000L
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS_MASK                                  0x02000000L
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET_MASK                           0x04000000L
+#define S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE_MASK                                 0xF0000000L
+//S2A_DOORBELL_ENTRY_6_CTRL
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_ENABLE__SHIFT                                           0x0
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWID__SHIFT                                             0x1
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_FENCE_ENABLE__SHIFT                                     0x6
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_OFFSET__SHIFT                                     0x7
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_SIZE__SHIFT                                       0x11
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_64BIT_SUPPORT_DIS__SHIFT                                0x19
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_NEED_DEDUCT_RANGE_OFFSET__SHIFT                         0x1a
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE__SHIFT                               0x1c
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_ENABLE_MASK                                             0x00000001L
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWID_MASK                                               0x0000003EL
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_FENCE_ENABLE_MASK                                       0x00000040L
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_OFFSET_MASK                                       0x0001FF80L
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_SIZE_MASK                                         0x01FE0000L
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_64BIT_SUPPORT_DIS_MASK                                  0x02000000L
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_NEED_DEDUCT_RANGE_OFFSET_MASK                           0x04000000L
+#define S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE_MASK                                 0xF0000000L
+//S2A_DOORBELL_ENTRY_7_CTRL
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_ENABLE__SHIFT                                           0x0
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWID__SHIFT                                             0x1
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_FENCE_ENABLE__SHIFT                                     0x6
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_OFFSET__SHIFT                                     0x7
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_SIZE__SHIFT                                       0x11
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_64BIT_SUPPORT_DIS__SHIFT                                0x19
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_NEED_DEDUCT_RANGE_OFFSET__SHIFT                         0x1a
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWADDR_31_28_VALUE__SHIFT                               0x1c
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_ENABLE_MASK                                             0x00000001L
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWID_MASK                                               0x0000003EL
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_FENCE_ENABLE_MASK                                       0x00000040L
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_OFFSET_MASK                                       0x0001FF80L
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_SIZE_MASK                                         0x01FE0000L
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_64BIT_SUPPORT_DIS_MASK                                  0x02000000L
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_NEED_DEDUCT_RANGE_OFFSET_MASK                           0x04000000L
+#define S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWADDR_31_28_VALUE_MASK                                 0xF0000000L
+//S2A_DOORBELL_ENTRY_8_CTRL
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_ENABLE__SHIFT                                           0x0
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWID__SHIFT                                             0x1
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_FENCE_ENABLE__SHIFT                                     0x6
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_OFFSET__SHIFT                                     0x7
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_SIZE__SHIFT                                       0x11
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_64BIT_SUPPORT_DIS__SHIFT                                0x19
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_NEED_DEDUCT_RANGE_OFFSET__SHIFT                         0x1a
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWADDR_31_28_VALUE__SHIFT                               0x1c
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_ENABLE_MASK                                             0x00000001L
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWID_MASK                                               0x0000003EL
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_FENCE_ENABLE_MASK                                       0x00000040L
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_OFFSET_MASK                                       0x0001FF80L
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_SIZE_MASK                                         0x01FE0000L
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_64BIT_SUPPORT_DIS_MASK                                  0x02000000L
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_NEED_DEDUCT_RANGE_OFFSET_MASK                           0x04000000L
+#define S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWADDR_31_28_VALUE_MASK                                 0xF0000000L
+//S2A_DOORBELL_ENTRY_9_CTRL
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_ENABLE__SHIFT                                           0x0
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWID__SHIFT                                             0x1
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_FENCE_ENABLE__SHIFT                                     0x6
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_OFFSET__SHIFT                                     0x7
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_SIZE__SHIFT                                       0x11
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_64BIT_SUPPORT_DIS__SHIFT                                0x19
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_NEED_DEDUCT_RANGE_OFFSET__SHIFT                         0x1a
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWADDR_31_28_VALUE__SHIFT                               0x1c
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_ENABLE_MASK                                             0x00000001L
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWID_MASK                                               0x0000003EL
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_FENCE_ENABLE_MASK                                       0x00000040L
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_OFFSET_MASK                                       0x0001FF80L
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_SIZE_MASK                                         0x01FE0000L
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_64BIT_SUPPORT_DIS_MASK                                  0x02000000L
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_NEED_DEDUCT_RANGE_OFFSET_MASK                           0x04000000L
+#define S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWADDR_31_28_VALUE_MASK                                 0xF0000000L
+//S2A_DOORBELL_ENTRY_10_CTRL
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_ENABLE__SHIFT                                         0x0
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWID__SHIFT                                           0x1
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_FENCE_ENABLE__SHIFT                                   0x6
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_OFFSET__SHIFT                                   0x7
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_SIZE__SHIFT                                     0x11
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_64BIT_SUPPORT_DIS__SHIFT                              0x19
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_NEED_DEDUCT_RANGE_OFFSET__SHIFT                       0x1a
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWADDR_31_28_VALUE__SHIFT                             0x1c
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_ENABLE_MASK                                           0x00000001L
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWID_MASK                                             0x0000003EL
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_FENCE_ENABLE_MASK                                     0x00000040L
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_OFFSET_MASK                                     0x0001FF80L
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_SIZE_MASK                                       0x01FE0000L
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_64BIT_SUPPORT_DIS_MASK                                0x02000000L
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_NEED_DEDUCT_RANGE_OFFSET_MASK                         0x04000000L
+#define S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWADDR_31_28_VALUE_MASK                               0xF0000000L
+//S2A_DOORBELL_ENTRY_11_CTRL
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_ENABLE__SHIFT                                         0x0
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWID__SHIFT                                           0x1
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_FENCE_ENABLE__SHIFT                                   0x6
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_OFFSET__SHIFT                                   0x7
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_SIZE__SHIFT                                     0x11
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_64BIT_SUPPORT_DIS__SHIFT                              0x19
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_NEED_DEDUCT_RANGE_OFFSET__SHIFT                       0x1a
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWADDR_31_28_VALUE__SHIFT                             0x1c
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_ENABLE_MASK                                           0x00000001L
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWID_MASK                                             0x0000003EL
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_FENCE_ENABLE_MASK                                     0x00000040L
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_OFFSET_MASK                                     0x0001FF80L
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_SIZE_MASK                                       0x01FE0000L
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_64BIT_SUPPORT_DIS_MASK                                0x02000000L
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_NEED_DEDUCT_RANGE_OFFSET_MASK                         0x04000000L
+#define S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWADDR_31_28_VALUE_MASK                               0xF0000000L
+//S2A_DOORBELL_ENTRY_12_CTRL
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_ENABLE__SHIFT                                         0x0
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWID__SHIFT                                           0x1
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_FENCE_ENABLE__SHIFT                                   0x6
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_OFFSET__SHIFT                                   0x7
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_SIZE__SHIFT                                     0x11
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_64BIT_SUPPORT_DIS__SHIFT                              0x19
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_NEED_DEDUCT_RANGE_OFFSET__SHIFT                       0x1a
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWADDR_31_28_VALUE__SHIFT                             0x1c
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_ENABLE_MASK                                           0x00000001L
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWID_MASK                                             0x0000003EL
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_FENCE_ENABLE_MASK                                     0x00000040L
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_OFFSET_MASK                                     0x0001FF80L
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_SIZE_MASK                                       0x01FE0000L
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_64BIT_SUPPORT_DIS_MASK                                0x02000000L
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_NEED_DEDUCT_RANGE_OFFSET_MASK                         0x04000000L
+#define S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWADDR_31_28_VALUE_MASK                               0xF0000000L
+//S2A_DOORBELL_ENTRY_13_CTRL
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_ENABLE__SHIFT                                         0x0
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWID__SHIFT                                           0x1
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_FENCE_ENABLE__SHIFT                                   0x6
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_OFFSET__SHIFT                                   0x7
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_SIZE__SHIFT                                     0x11
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_64BIT_SUPPORT_DIS__SHIFT                              0x19
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_NEED_DEDUCT_RANGE_OFFSET__SHIFT                       0x1a
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWADDR_31_28_VALUE__SHIFT                             0x1c
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_ENABLE_MASK                                           0x00000001L
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWID_MASK                                             0x0000003EL
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_FENCE_ENABLE_MASK                                     0x00000040L
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_OFFSET_MASK                                     0x0001FF80L
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_SIZE_MASK                                       0x01FE0000L
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_64BIT_SUPPORT_DIS_MASK                                0x02000000L
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_NEED_DEDUCT_RANGE_OFFSET_MASK                         0x04000000L
+#define S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWADDR_31_28_VALUE_MASK                               0xF0000000L
+//S2A_DOORBELL_ENTRY_14_CTRL
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_ENABLE__SHIFT                                         0x0
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWID__SHIFT                                           0x1
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_FENCE_ENABLE__SHIFT                                   0x6
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_OFFSET__SHIFT                                   0x7
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_SIZE__SHIFT                                     0x11
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_64BIT_SUPPORT_DIS__SHIFT                              0x19
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_NEED_DEDUCT_RANGE_OFFSET__SHIFT                       0x1a
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWADDR_31_28_VALUE__SHIFT                             0x1c
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_ENABLE_MASK                                           0x00000001L
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWID_MASK                                             0x0000003EL
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_FENCE_ENABLE_MASK                                     0x00000040L
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_OFFSET_MASK                                     0x0001FF80L
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_SIZE_MASK                                       0x01FE0000L
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_64BIT_SUPPORT_DIS_MASK                                0x02000000L
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_NEED_DEDUCT_RANGE_OFFSET_MASK                         0x04000000L
+#define S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWADDR_31_28_VALUE_MASK                               0xF0000000L
+//S2A_DOORBELL_ENTRY_15_CTRL
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_ENABLE__SHIFT                                         0x0
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWID__SHIFT                                           0x1
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_FENCE_ENABLE__SHIFT                                   0x6
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_OFFSET__SHIFT                                   0x7
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_SIZE__SHIFT                                     0x11
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_64BIT_SUPPORT_DIS__SHIFT                              0x19
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_NEED_DEDUCT_RANGE_OFFSET__SHIFT                       0x1a
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWADDR_31_28_VALUE__SHIFT                             0x1c
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_ENABLE_MASK                                           0x00000001L
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWID_MASK                                             0x0000003EL
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_FENCE_ENABLE_MASK                                     0x00000040L
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_OFFSET_MASK                                     0x0001FF80L
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_SIZE_MASK                                       0x01FE0000L
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_64BIT_SUPPORT_DIS_MASK                                0x02000000L
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_NEED_DEDUCT_RANGE_OFFSET_MASK                         0x04000000L
+#define S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWADDR_31_28_VALUE_MASK                               0xF0000000L
+//S2A_DOORBELL_COMMON_CTRL_REG
+#define S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBELL_FENCE_ONCE_TRIGGER_DIS__SHIFT                              0x0
+#define S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBELL_FENCE_ONCE_TRIGGER_DIS_MASK                                0x00000001L
+
+
+// addressBlock: aid_nbio_nbif0_gdc_GDCDEC
+//GDC1_A2S_CNTL_CL0
+#define GDC1_A2S_CNTL_CL0__NSNOOP_MAP__SHIFT                                                                  0x0
+#define GDC1_A2S_CNTL_CL0__REQPASSPW_VC0_MAP__SHIFT                                                           0x2
+#define GDC1_A2S_CNTL_CL0__REQPASSPW_NVC0_MAP__SHIFT                                                          0x4
+#define GDC1_A2S_CNTL_CL0__REQRSPPASSPW_VC0_MAP__SHIFT                                                        0x6
+#define GDC1_A2S_CNTL_CL0__REQRSPPASSPW_NVC0_MAP__SHIFT                                                       0x8
+#define GDC1_A2S_CNTL_CL0__BLKLVL_MAP__SHIFT                                                                  0xa
+#define GDC1_A2S_CNTL_CL0__DATERR_MAP__SHIFT                                                                  0xc
+#define GDC1_A2S_CNTL_CL0__EXOKAY_WR_MAP__SHIFT                                                               0xe
+#define GDC1_A2S_CNTL_CL0__EXOKAY_RD_MAP__SHIFT                                                               0x10
+#define GDC1_A2S_CNTL_CL0__RESP_WR_MAP__SHIFT                                                                 0x12
+#define GDC1_A2S_CNTL_CL0__RESP_RD_MAP__SHIFT                                                                 0x14
+#define GDC1_A2S_CNTL_CL0__RDRSP_ERRMAP__SHIFT                                                                0x16
+#define GDC1_A2S_CNTL_CL0__RDRSP_SEL_MODE__SHIFT                                                              0x18
+#define GDC1_A2S_CNTL_CL0__STATIC_VC_ENABLE__SHIFT                                                            0x1b
+#define GDC1_A2S_CNTL_CL0__STATIC_VC_VALUE__SHIFT                                                             0x1c
+#define GDC1_A2S_CNTL_CL0__NSNOOP_MAP_MASK                                                                    0x00000003L
+#define GDC1_A2S_CNTL_CL0__REQPASSPW_VC0_MAP_MASK                                                             0x0000000CL
+#define GDC1_A2S_CNTL_CL0__REQPASSPW_NVC0_MAP_MASK                                                            0x00000030L
+#define GDC1_A2S_CNTL_CL0__REQRSPPASSPW_VC0_MAP_MASK                                                          0x000000C0L
+#define GDC1_A2S_CNTL_CL0__REQRSPPASSPW_NVC0_MAP_MASK                                                         0x00000300L
+#define GDC1_A2S_CNTL_CL0__BLKLVL_MAP_MASK                                                                    0x00000C00L
+#define GDC1_A2S_CNTL_CL0__DATERR_MAP_MASK                                                                    0x00003000L
+#define GDC1_A2S_CNTL_CL0__EXOKAY_WR_MAP_MASK                                                                 0x0000C000L
+#define GDC1_A2S_CNTL_CL0__EXOKAY_RD_MAP_MASK                                                                 0x00030000L
+#define GDC1_A2S_CNTL_CL0__RESP_WR_MAP_MASK                                                                   0x000C0000L
+#define GDC1_A2S_CNTL_CL0__RESP_RD_MAP_MASK                                                                   0x00300000L
+#define GDC1_A2S_CNTL_CL0__RDRSP_ERRMAP_MASK                                                                  0x00C00000L
+#define GDC1_A2S_CNTL_CL0__RDRSP_SEL_MODE_MASK                                                                0x07000000L
+#define GDC1_A2S_CNTL_CL0__STATIC_VC_ENABLE_MASK                                                              0x08000000L
+#define GDC1_A2S_CNTL_CL0__STATIC_VC_VALUE_MASK                                                               0x70000000L
+//GDC1_A2S_CNTL_CL1
+#define GDC1_A2S_CNTL_CL1__NSNOOP_MAP__SHIFT                                                                  0x0
+#define GDC1_A2S_CNTL_CL1__REQPASSPW_VC0_MAP__SHIFT                                                           0x2
+#define GDC1_A2S_CNTL_CL1__REQPASSPW_NVC0_MAP__SHIFT                                                          0x4
+#define GDC1_A2S_CNTL_CL1__REQRSPPASSPW_VC0_MAP__SHIFT                                                        0x6
+#define GDC1_A2S_CNTL_CL1__REQRSPPASSPW_NVC0_MAP__SHIFT                                                       0x8
+#define GDC1_A2S_CNTL_CL1__BLKLVL_MAP__SHIFT                                                                  0xa
+#define GDC1_A2S_CNTL_CL1__DATERR_MAP__SHIFT                                                                  0xc
+#define GDC1_A2S_CNTL_CL1__EXOKAY_WR_MAP__SHIFT                                                               0xe
+#define GDC1_A2S_CNTL_CL1__EXOKAY_RD_MAP__SHIFT                                                               0x10
+#define GDC1_A2S_CNTL_CL1__RESP_WR_MAP__SHIFT                                                                 0x12
+#define GDC1_A2S_CNTL_CL1__RESP_RD_MAP__SHIFT                                                                 0x14
+#define GDC1_A2S_CNTL_CL1__RDRSP_ERRMAP__SHIFT                                                                0x16
+#define GDC1_A2S_CNTL_CL1__RDRSP_SEL_MODE__SHIFT                                                              0x18
+#define GDC1_A2S_CNTL_CL1__STATIC_VC_ENABLE__SHIFT                                                            0x1b
+#define GDC1_A2S_CNTL_CL1__STATIC_VC_VALUE__SHIFT                                                             0x1c
+#define GDC1_A2S_CNTL_CL1__NSNOOP_MAP_MASK                                                                    0x00000003L
+#define GDC1_A2S_CNTL_CL1__REQPASSPW_VC0_MAP_MASK                                                             0x0000000CL
+#define GDC1_A2S_CNTL_CL1__REQPASSPW_NVC0_MAP_MASK                                                            0x00000030L
+#define GDC1_A2S_CNTL_CL1__REQRSPPASSPW_VC0_MAP_MASK                                                          0x000000C0L
+#define GDC1_A2S_CNTL_CL1__REQRSPPASSPW_NVC0_MAP_MASK                                                         0x00000300L
+#define GDC1_A2S_CNTL_CL1__BLKLVL_MAP_MASK                                                                    0x00000C00L
+#define GDC1_A2S_CNTL_CL1__DATERR_MAP_MASK                                                                    0x00003000L
+#define GDC1_A2S_CNTL_CL1__EXOKAY_WR_MAP_MASK                                                                 0x0000C000L
+#define GDC1_A2S_CNTL_CL1__EXOKAY_RD_MAP_MASK                                                                 0x00030000L
+#define GDC1_A2S_CNTL_CL1__RESP_WR_MAP_MASK                                                                   0x000C0000L
+#define GDC1_A2S_CNTL_CL1__RESP_RD_MAP_MASK                                                                   0x00300000L
+#define GDC1_A2S_CNTL_CL1__RDRSP_ERRMAP_MASK                                                                  0x00C00000L
+#define GDC1_A2S_CNTL_CL1__RDRSP_SEL_MODE_MASK                                                                0x07000000L
+#define GDC1_A2S_CNTL_CL1__STATIC_VC_ENABLE_MASK                                                              0x08000000L
+#define GDC1_A2S_CNTL_CL1__STATIC_VC_VALUE_MASK                                                               0x70000000L
+//GDC1_A2S_CNTL3_CL0
+#define GDC1_A2S_CNTL3_CL0__FORCE_WR_PH__SHIFT                                                                0x0
+#define GDC1_A2S_CNTL3_CL0__FORCE_WR_STEERING__SHIFT                                                          0x2
+#define GDC1_A2S_CNTL3_CL0__WR_ST_TAG_MODE__SHIFT                                                             0x3
+#define GDC1_A2S_CNTL3_CL0__FORCE_WR_ST_ENTRY__SHIFT                                                          0x4
+#define GDC1_A2S_CNTL3_CL0__FORCE_WR_PH_MASK                                                                  0x00000003L
+#define GDC1_A2S_CNTL3_CL0__FORCE_WR_STEERING_MASK                                                            0x00000004L
+#define GDC1_A2S_CNTL3_CL0__WR_ST_TAG_MODE_MASK                                                               0x00000008L
+#define GDC1_A2S_CNTL3_CL0__FORCE_WR_ST_ENTRY_MASK                                                            0x000003F0L
+//GDC1_A2S_CNTL3_CL1
+#define GDC1_A2S_CNTL3_CL1__FORCE_WR_PH__SHIFT                                                                0x0
+#define GDC1_A2S_CNTL3_CL1__FORCE_WR_STEERING__SHIFT                                                          0x2
+#define GDC1_A2S_CNTL3_CL1__WR_ST_TAG_MODE__SHIFT                                                             0x3
+#define GDC1_A2S_CNTL3_CL1__FORCE_WR_ST_ENTRY__SHIFT                                                          0x4
+#define GDC1_A2S_CNTL3_CL1__FORCE_WR_PH_MASK                                                                  0x00000003L
+#define GDC1_A2S_CNTL3_CL1__FORCE_WR_STEERING_MASK                                                            0x00000004L
+#define GDC1_A2S_CNTL3_CL1__WR_ST_TAG_MODE_MASK                                                               0x00000008L
+#define GDC1_A2S_CNTL3_CL1__FORCE_WR_ST_ENTRY_MASK                                                            0x000003F0L
+//GDC1_A2S_CNTL_SW0
+#define GDC1_A2S_CNTL_SW0__STATIC_VC_ENABLE__SHIFT                                                            0x0
+#define GDC1_A2S_CNTL_SW0__STATIC_VC_VALUE__SHIFT                                                             0x1
+#define GDC1_A2S_CNTL_SW0__FORCE_RSP_REORDER_EN__SHIFT                                                        0x6
+#define GDC1_A2S_CNTL_SW0__SDP_WR_CHAIN_DIS__SHIFT                                                            0x9
+#define GDC1_A2S_CNTL_SW0__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                     0xa
+#define GDC1_A2S_CNTL_SW0__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                  0xb
+#define GDC1_A2S_CNTL_SW0__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                 0xc
+#define GDC1_A2S_CNTL_SW0__WRR_RD_WEIGHT__SHIFT                                                               0x10
+#define GDC1_A2S_CNTL_SW0__WRR_WR_WEIGHT__SHIFT                                                               0x18
+#define GDC1_A2S_CNTL_SW0__STATIC_VC_ENABLE_MASK                                                              0x00000001L
+#define GDC1_A2S_CNTL_SW0__STATIC_VC_VALUE_MASK                                                               0x0000000EL
+#define GDC1_A2S_CNTL_SW0__FORCE_RSP_REORDER_EN_MASK                                                          0x00000040L
+#define GDC1_A2S_CNTL_SW0__SDP_WR_CHAIN_DIS_MASK                                                              0x00000200L
+#define GDC1_A2S_CNTL_SW0__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                       0x00000400L
+#define GDC1_A2S_CNTL_SW0__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                    0x00000800L
+#define GDC1_A2S_CNTL_SW0__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                   0x00001000L
+#define GDC1_A2S_CNTL_SW0__WRR_RD_WEIGHT_MASK                                                                 0x00FF0000L
+#define GDC1_A2S_CNTL_SW0__WRR_WR_WEIGHT_MASK                                                                 0xFF000000L
+//GDC1_A2S_CNTL_SW1
+#define GDC1_A2S_CNTL_SW1__STATIC_VC_ENABLE__SHIFT                                                            0x0
+#define GDC1_A2S_CNTL_SW1__STATIC_VC_VALUE__SHIFT                                                             0x1
+#define GDC1_A2S_CNTL_SW1__FORCE_RSP_REORDER_EN__SHIFT                                                        0x6
+#define GDC1_A2S_CNTL_SW1__SDP_WR_CHAIN_DIS__SHIFT                                                            0x9
+#define GDC1_A2S_CNTL_SW1__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                     0xa
+#define GDC1_A2S_CNTL_SW1__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                  0xb
+#define GDC1_A2S_CNTL_SW1__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                 0xc
+#define GDC1_A2S_CNTL_SW1__WRR_RD_WEIGHT__SHIFT                                                               0x10
+#define GDC1_A2S_CNTL_SW1__WRR_WR_WEIGHT__SHIFT                                                               0x18
+#define GDC1_A2S_CNTL_SW1__STATIC_VC_ENABLE_MASK                                                              0x00000001L
+#define GDC1_A2S_CNTL_SW1__STATIC_VC_VALUE_MASK                                                               0x0000000EL
+#define GDC1_A2S_CNTL_SW1__FORCE_RSP_REORDER_EN_MASK                                                          0x00000040L
+#define GDC1_A2S_CNTL_SW1__SDP_WR_CHAIN_DIS_MASK                                                              0x00000200L
+#define GDC1_A2S_CNTL_SW1__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                       0x00000400L
+#define GDC1_A2S_CNTL_SW1__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                    0x00000800L
+#define GDC1_A2S_CNTL_SW1__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                   0x00001000L
+#define GDC1_A2S_CNTL_SW1__WRR_RD_WEIGHT_MASK                                                                 0x00FF0000L
+#define GDC1_A2S_CNTL_SW1__WRR_WR_WEIGHT_MASK                                                                 0xFF000000L
+//GDC1_A2S_CNTL_SW2
+#define GDC1_A2S_CNTL_SW2__STATIC_VC_ENABLE__SHIFT                                                            0x0
+#define GDC1_A2S_CNTL_SW2__STATIC_VC_VALUE__SHIFT                                                             0x1
+#define GDC1_A2S_CNTL_SW2__FORCE_RSP_REORDER_EN__SHIFT                                                        0x6
+#define GDC1_A2S_CNTL_SW2__SDP_WR_CHAIN_DIS__SHIFT                                                            0x9
+#define GDC1_A2S_CNTL_SW2__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                     0xa
+#define GDC1_A2S_CNTL_SW2__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                  0xb
+#define GDC1_A2S_CNTL_SW2__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                 0xc
+#define GDC1_A2S_CNTL_SW2__WRR_RD_WEIGHT__SHIFT                                                               0x10
+#define GDC1_A2S_CNTL_SW2__WRR_WR_WEIGHT__SHIFT                                                               0x18
+#define GDC1_A2S_CNTL_SW2__STATIC_VC_ENABLE_MASK                                                              0x00000001L
+#define GDC1_A2S_CNTL_SW2__STATIC_VC_VALUE_MASK                                                               0x0000000EL
+#define GDC1_A2S_CNTL_SW2__FORCE_RSP_REORDER_EN_MASK                                                          0x00000040L
+#define GDC1_A2S_CNTL_SW2__SDP_WR_CHAIN_DIS_MASK                                                              0x00000200L
+#define GDC1_A2S_CNTL_SW2__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                       0x00000400L
+#define GDC1_A2S_CNTL_SW2__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                    0x00000800L
+#define GDC1_A2S_CNTL_SW2__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                   0x00001000L
+#define GDC1_A2S_CNTL_SW2__WRR_RD_WEIGHT_MASK                                                                 0x00FF0000L
+#define GDC1_A2S_CNTL_SW2__WRR_WR_WEIGHT_MASK                                                                 0xFF000000L
+//GDC1_A2S_TAG_ALLOC_0
+#define GDC1_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_WR__SHIFT                                                     0x0
+#define GDC1_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_RD__SHIFT                                                     0x8
+#define GDC1_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC1_WR__SHIFT                                                     0x10
+#define GDC1_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_WR_MASK                                                       0x000000FFL
+#define GDC1_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_RD_MASK                                                       0x0000FF00L
+#define GDC1_A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC1_WR_MASK                                                       0x00FF0000L
+//GDC1_A2S_TAG_ALLOC_1
+#define GDC1_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC3_WR__SHIFT                                                     0x0
+#define GDC1_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_WR__SHIFT                                                     0x10
+#define GDC1_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_RD__SHIFT                                                     0x18
+#define GDC1_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC3_WR_MASK                                                       0x000000FFL
+#define GDC1_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_WR_MASK                                                       0x00FF0000L
+#define GDC1_A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_RD_MASK                                                       0xFF000000L
+//GDC1_A2S_MISC_CNTL
+#define GDC1_A2S_MISC_CNTL__BLKLVL_FOR_MSG__SHIFT                                                             0x0
+#define GDC1_A2S_MISC_CNTL__RESERVE_2_CRED_FOR_NPWR_REQ_DIS__SHIFT                                            0x2
+#define GDC1_A2S_MISC_CNTL__WRR_LRG_SIZE_MODE__SHIFT                                                          0x3
+#define GDC1_A2S_MISC_CNTL__FORCE_RSP_REORDER_EN__SHIFT                                                       0x4
+#define GDC1_A2S_MISC_CNTL__RSP_REORDER_DIS__SHIFT                                                            0x5
+#define GDC1_A2S_MISC_CNTL__WRRSP_ACCUM_SEL__SHIFT                                                            0x6
+#define GDC1_A2S_MISC_CNTL__WRRSP_TAGFIFO_CONT_RD_DIS__SHIFT                                                  0x7
+#define GDC1_A2S_MISC_CNTL__RDRSP_TAGFIFO_CONT_RD_DIS__SHIFT                                                  0x8
+#define GDC1_A2S_MISC_CNTL__RDRSP_STS_DATSTS_PRIORITY__SHIFT                                                  0x9
+#define GDC1_A2S_MISC_CNTL__INSERT_RD_ON_2ND_WDAT_EN__SHIFT                                                   0xa
+#define GDC1_A2S_MISC_CNTL__WR_TAG_SET_MIN__SHIFT                                                             0x10
+#define GDC1_A2S_MISC_CNTL__RD_TAG_SET_MIN__SHIFT                                                             0x15
+#define GDC1_A2S_MISC_CNTL__WRR_LRG_MODE__SHIFT                                                               0x1a
+#define GDC1_A2S_MISC_CNTL__WRR_LRG_COUNTER_MODE__SHIFT                                                       0x1b
+#define GDC1_A2S_MISC_CNTL__BLKLVL_FOR_MSG_MASK                                                               0x00000003L
+#define GDC1_A2S_MISC_CNTL__RESERVE_2_CRED_FOR_NPWR_REQ_DIS_MASK                                              0x00000004L
+#define GDC1_A2S_MISC_CNTL__WRR_LRG_SIZE_MODE_MASK                                                            0x00000008L
+#define GDC1_A2S_MISC_CNTL__FORCE_RSP_REORDER_EN_MASK                                                         0x00000010L
+#define GDC1_A2S_MISC_CNTL__RSP_REORDER_DIS_MASK                                                              0x00000020L
+#define GDC1_A2S_MISC_CNTL__WRRSP_ACCUM_SEL_MASK                                                              0x00000040L
+#define GDC1_A2S_MISC_CNTL__WRRSP_TAGFIFO_CONT_RD_DIS_MASK                                                    0x00000080L
+#define GDC1_A2S_MISC_CNTL__RDRSP_TAGFIFO_CONT_RD_DIS_MASK                                                    0x00000100L
+#define GDC1_A2S_MISC_CNTL__RDRSP_STS_DATSTS_PRIORITY_MASK                                                    0x00000200L
+#define GDC1_A2S_MISC_CNTL__INSERT_RD_ON_2ND_WDAT_EN_MASK                                                     0x00000400L
+#define GDC1_A2S_MISC_CNTL__WR_TAG_SET_MIN_MASK                                                               0x001F0000L
+#define GDC1_A2S_MISC_CNTL__RD_TAG_SET_MIN_MASK                                                               0x03E00000L
+#define GDC1_A2S_MISC_CNTL__WRR_LRG_MODE_MASK                                                                 0x04000000L
+#define GDC1_A2S_MISC_CNTL__WRR_LRG_COUNTER_MODE_MASK                                                         0x08000000L
+//GDC1_SHUB_REGS_IF_CTL
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS__SHIFT                                0x0
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS__SHIFT                                             0x1
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS_MASK                                  0x00000001L
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS_MASK                                               0x00000002L
+//GDC1_NGDC_MGCG_CTRL
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_EN__SHIFT                                                              0x0
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_MODE__SHIFT                                                            0x1
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS__SHIFT                                                      0x2
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS__SHIFT                                                         0xa
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS__SHIFT                                                         0xb
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS__SHIFT                                                         0xc
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS__SHIFT                                                         0xd
+#define GDC1_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN__SHIFT                                                         0xe
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_EN_MASK                                                                0x00000001L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_MODE_MASK                                                              0x00000002L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS_MASK                                                        0x000003FCL
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS_MASK                                                           0x00000400L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS_MASK                                                           0x00000800L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS_MASK                                                           0x00001000L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS_MASK                                                           0x00002000L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN_MASK                                                           0x00004000L
+//GDC1_NGDC_RESERVED_0
+#define GDC1_NGDC_RESERVED_0__RESERVED__SHIFT                                                                 0x0
+#define GDC1_NGDC_RESERVED_0__RESERVED_MASK                                                                   0xFFFFFFFFL
+//GDC1_NGDC_RESERVED_1
+#define GDC1_NGDC_RESERVED_1__RESERVED__SHIFT                                                                 0x0
+#define GDC1_NGDC_RESERVED_1__RESERVED_MASK                                                                   0xFFFFFFFFL
+//GDC1_NBIF_GFX_DOORBELL_STATUS
+#define GDC1_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT__SHIFT                                          0x0
+#define GDC1_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT_MASK                                            0x0000FFFFL
+//GDC1_ATDMA_MISC_CNTL
+#define GDC1_ATDMA_MISC_CNTL__WRR_ARB_MODE__SHIFT                                                             0x0
+#define GDC1_ATDMA_MISC_CNTL__INSERT_RD_ON_2ND_WDAT_EN__SHIFT                                                 0x1
+#define GDC1_ATDMA_MISC_CNTL__RDRSP_ARB_MODE__SHIFT                                                           0x2
+#define GDC1_ATDMA_MISC_CNTL__WRR_VC6_WEIGHT__SHIFT                                                           0x8
+#define GDC1_ATDMA_MISC_CNTL__WRR_VC0_WEIGHT__SHIFT                                                           0x10
+#define GDC1_ATDMA_MISC_CNTL__WRR_VC1_WEIGHT__SHIFT                                                           0x18
+#define GDC1_ATDMA_MISC_CNTL__WRR_ARB_MODE_MASK                                                               0x00000001L
+#define GDC1_ATDMA_MISC_CNTL__INSERT_RD_ON_2ND_WDAT_EN_MASK                                                   0x00000002L
+#define GDC1_ATDMA_MISC_CNTL__RDRSP_ARB_MODE_MASK                                                             0x0000000CL
+#define GDC1_ATDMA_MISC_CNTL__WRR_VC6_WEIGHT_MASK                                                             0x0000FF00L
+#define GDC1_ATDMA_MISC_CNTL__WRR_VC0_WEIGHT_MASK                                                             0x00FF0000L
+#define GDC1_ATDMA_MISC_CNTL__WRR_VC1_WEIGHT_MASK                                                             0xFF000000L
+//GDC1_S2A_MISC_CNTL
+#define GDC1_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS__SHIFT                                                         0x3
+#define GDC1_S2A_MISC_CNTL__ATM_ARB_MODE__SHIFT                                                               0x8
+#define GDC1_S2A_MISC_CNTL__RB_ARB_MODE__SHIFT                                                                0xa
+#define GDC1_S2A_MISC_CNTL__HSTR_ARB_MODE__SHIFT                                                              0xc
+#define GDC1_S2A_MISC_CNTL__HDP_PERF_ENH_DIS__SHIFT                                                           0xf
+#define GDC1_S2A_MISC_CNTL__WRSP_ARB_MODE__SHIFT                                                              0x10
+#define GDC1_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS_MASK                                                           0x00000008L
+#define GDC1_S2A_MISC_CNTL__ATM_ARB_MODE_MASK                                                                 0x00000300L
+#define GDC1_S2A_MISC_CNTL__RB_ARB_MODE_MASK                                                                  0x00000C00L
+#define GDC1_S2A_MISC_CNTL__HSTR_ARB_MODE_MASK                                                                0x00003000L
+#define GDC1_S2A_MISC_CNTL__HDP_PERF_ENH_DIS_MASK                                                             0x00008000L
+#define GDC1_S2A_MISC_CNTL__WRSP_ARB_MODE_MASK                                                                0x000F0000L
+//GDC1_NGDC_EARLY_WAKEUP_CTRL
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_ACTIVE__SHIFT                                0x0
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_DS_EXIT__SHIFT                               0x1
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_ALLOW_AER_ACTIVE__SHIFT                                0x2
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_ACTIVE_MASK                                  0x00000001L
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_DS_EXIT_MASK                                 0x00000002L
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_ALLOW_AER_ACTIVE_MASK                                  0x00000004L
+//GDC1_NGDC_PG_MISC_CTRL
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_ENDP_D3_ONLY__SHIFT                                                   0xa
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM1__SHIFT                                                      0xd
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_DS_ALLOW_DIS__SHIFT                                                   0xe
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM2__SHIFT                                                      0x10
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_CFG_REFCLK_CYCLE_FOR_200NS__SHIFT                                        0x18
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_CFG_PG_EXIT_OVERRIDE__SHIFT                                              0x1f
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_ENDP_D3_ONLY_MASK                                                     0x00000400L
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM1_MASK                                                        0x00002000L
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_DS_ALLOW_DIS_MASK                                                     0x00004000L
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM2_MASK                                                        0x00010000L
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_CFG_REFCLK_CYCLE_FOR_200NS_MASK                                          0x3F000000L
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_CFG_PG_EXIT_OVERRIDE_MASK                                                0x80000000L
+//GDC1_NGDC_PGMST_CTRL
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_PG_HYSTERESIS__SHIFT                                                   0x0
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_PG_EN__SHIFT                                                           0x8
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_IDLENESS_COUNT_EN__SHIFT                                               0xa
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_FW_PG_EXIT_EN__SHIFT                                                   0xe
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_PG_HYSTERESIS_MASK                                                     0x000000FFL
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_PG_EN_MASK                                                             0x00000100L
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_IDLENESS_COUNT_EN_MASK                                                 0x00003C00L
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_FW_PG_EXIT_EN_MASK                                                     0x0000C000L
+//GDC1_NGDC_PGSLV_CTRL
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_0_IDLE_HYSTERESIS__SHIFT                                       0x0
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_1_IDLE_HYSTERESIS__SHIFT                                       0x5
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_GDCCLK_IDLE_HYSTERESIS__SHIFT                                          0xa
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_0_IDLE_HYSTERESIS_MASK                                         0x0000001FL
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_1_IDLE_HYSTERESIS_MASK                                         0x000003E0L
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_GDCCLK_IDLE_HYSTERESIS_MASK                                            0x00007C00L
+
+
+// addressBlock: aid_nbio_nbif0_gdc_sec_GDCSEC_DEC
+//XCC_DOORBELL_FENCE
+#define XCC_DOORBELL_FENCE__XCC_0_DOORBELL_FENCE__SHIFT                                                       0x0
+#define XCC_DOORBELL_FENCE__XCC_1_DOORBELL_FENCE__SHIFT                                                       0x1
+#define XCC_DOORBELL_FENCE__XCC_2_DOORBELL_FENCE__SHIFT                                                       0x2
+#define XCC_DOORBELL_FENCE__XCC_3_DOORBELL_FENCE__SHIFT                                                       0x3
+#define XCC_DOORBELL_FENCE__XCC_4_DOORBELL_FENCE__SHIFT                                                       0x4
+#define XCC_DOORBELL_FENCE__XCC_5_DOORBELL_FENCE__SHIFT                                                       0x5
+#define XCC_DOORBELL_FENCE__XCC_6_DOORBELL_FENCE__SHIFT                                                       0x6
+#define XCC_DOORBELL_FENCE__XCC_7_DOORBELL_FENCE__SHIFT                                                       0x7
+#define XCC_DOORBELL_FENCE__SHUB_SLV_MODE__SHIFT                                                              0x10
+#define XCC_DOORBELL_FENCE__RMOTE_CP_SENT__SHIFT                                                              0x11
+#define XCC_DOORBELL_FENCE__CP_0_SENT__SHIFT                                                                  0x12
+#define XCC_DOORBELL_FENCE__CP_1_SENT__SHIFT                                                                  0x13
+#define XCC_DOORBELL_FENCE__CP_2_SENT__SHIFT                                                                  0x14
+#define XCC_DOORBELL_FENCE__CP_3_SENT__SHIFT                                                                  0x15
+#define XCC_DOORBELL_FENCE__CP_4_SENT__SHIFT                                                                  0x16
+#define XCC_DOORBELL_FENCE__CP_5_SENT__SHIFT                                                                  0x17
+#define XCC_DOORBELL_FENCE__CP_6_SENT__SHIFT                                                                  0x18
+#define XCC_DOORBELL_FENCE__CP_7_SENT__SHIFT                                                                  0x19
+#define XCC_DOORBELL_FENCE__REMOTE_CLIENT_SENT__SHIFT                                                         0x1a
+#define XCC_DOORBELL_FENCE__REMOTE_CLIENT_CLR_PENDING__SHIFT                                                  0x1b
+#define XCC_DOORBELL_FENCE__XCC_0_DOORBELL_FENCE_MASK                                                         0x00000001L
+#define XCC_DOORBELL_FENCE__XCC_1_DOORBELL_FENCE_MASK                                                         0x00000002L
+#define XCC_DOORBELL_FENCE__XCC_2_DOORBELL_FENCE_MASK                                                         0x00000004L
+#define XCC_DOORBELL_FENCE__XCC_3_DOORBELL_FENCE_MASK                                                         0x00000008L
+#define XCC_DOORBELL_FENCE__XCC_4_DOORBELL_FENCE_MASK                                                         0x00000010L
+#define XCC_DOORBELL_FENCE__XCC_5_DOORBELL_FENCE_MASK                                                         0x00000020L
+#define XCC_DOORBELL_FENCE__XCC_6_DOORBELL_FENCE_MASK                                                         0x00000040L
+#define XCC_DOORBELL_FENCE__XCC_7_DOORBELL_FENCE_MASK                                                         0x00000080L
+#define XCC_DOORBELL_FENCE__SHUB_SLV_MODE_MASK                                                                0x00010000L
+#define XCC_DOORBELL_FENCE__RMOTE_CP_SENT_MASK                                                                0x00020000L
+#define XCC_DOORBELL_FENCE__CP_0_SENT_MASK                                                                    0x00040000L
+#define XCC_DOORBELL_FENCE__CP_1_SENT_MASK                                                                    0x00080000L
+#define XCC_DOORBELL_FENCE__CP_2_SENT_MASK                                                                    0x00100000L
+#define XCC_DOORBELL_FENCE__CP_3_SENT_MASK                                                                    0x00200000L
+#define XCC_DOORBELL_FENCE__CP_4_SENT_MASK                                                                    0x00400000L
+#define XCC_DOORBELL_FENCE__CP_5_SENT_MASK                                                                    0x00800000L
+#define XCC_DOORBELL_FENCE__CP_6_SENT_MASK                                                                    0x01000000L
+#define XCC_DOORBELL_FENCE__CP_7_SENT_MASK                                                                    0x02000000L
+#define XCC_DOORBELL_FENCE__REMOTE_CLIENT_SENT_MASK                                                           0x04000000L
+#define XCC_DOORBELL_FENCE__REMOTE_CLIENT_CLR_PENDING_MASK                                                    0x08000000L
+
+
+// addressBlock: aid_nbio_nbif0_gdc_rst_GDCRST_DEC
+//SHUB_PF_FLR_RST
+#define SHUB_PF_FLR_RST__DEV0_PF0_FLR_RST__SHIFT                                                              0x0
+#define SHUB_PF_FLR_RST__DEV0_PF1_FLR_RST__SHIFT                                                              0x1
+#define SHUB_PF_FLR_RST__DEV0_PF0_FLR_RST_MASK                                                                0x00000001L
+#define SHUB_PF_FLR_RST__DEV0_PF1_FLR_RST_MASK                                                                0x00000002L
+//SHUB_GFX_DRV_VPU_RST
+#define SHUB_GFX_DRV_VPU_RST__GFX_DRV_MODE1_RST__SHIFT                                                        0x0
+#define SHUB_GFX_DRV_VPU_RST__GFX_DRV_MODE1_RST_MASK                                                          0x00000001L
+//SHUB_LINK_RESET
+#define SHUB_LINK_RESET__LINK_P0_RESET__SHIFT                                                                 0x0
+#define SHUB_LINK_RESET__LINK_P1_RESET__SHIFT                                                                 0x1
+#define SHUB_LINK_RESET__LINK_P2_RESET__SHIFT                                                                 0x2
+#define SHUB_LINK_RESET__LINK_P3_RESET__SHIFT                                                                 0x3
+#define SHUB_LINK_RESET__LINK_P0_RESET_MASK                                                                   0x00000001L
+#define SHUB_LINK_RESET__LINK_P1_RESET_MASK                                                                   0x00000002L
+#define SHUB_LINK_RESET__LINK_P2_RESET_MASK                                                                   0x00000004L
+#define SHUB_LINK_RESET__LINK_P3_RESET_MASK                                                                   0x00000008L
+//SHUB_HARD_RST_CTRL
+#define SHUB_HARD_RST_CTRL__COR_RESET_EN__SHIFT                                                               0x0
+#define SHUB_HARD_RST_CTRL__REG_RESET_EN__SHIFT                                                               0x1
+#define SHUB_HARD_RST_CTRL__STY_RESET_EN__SHIFT                                                               0x2
+#define SHUB_HARD_RST_CTRL__NIC400_RESET_EN__SHIFT                                                            0x3
+#define SHUB_HARD_RST_CTRL__SDP_PORT_RESET_EN__SHIFT                                                          0x4
+#define SHUB_HARD_RST_CTRL__SION_AON_RESET_EN__SHIFT                                                          0x5
+#define SHUB_HARD_RST_CTRL__COR_RESET_EN_MASK                                                                 0x00000001L
+#define SHUB_HARD_RST_CTRL__REG_RESET_EN_MASK                                                                 0x00000002L
+#define SHUB_HARD_RST_CTRL__STY_RESET_EN_MASK                                                                 0x00000004L
+#define SHUB_HARD_RST_CTRL__NIC400_RESET_EN_MASK                                                              0x00000008L
+#define SHUB_HARD_RST_CTRL__SDP_PORT_RESET_EN_MASK                                                            0x00000010L
+#define SHUB_HARD_RST_CTRL__SION_AON_RESET_EN_MASK                                                            0x00000020L
+//SHUB_SOFT_RST_CTRL
+#define SHUB_SOFT_RST_CTRL__COR_RESET_EN__SHIFT                                                               0x0
+#define SHUB_SOFT_RST_CTRL__REG_RESET_EN__SHIFT                                                               0x1
+#define SHUB_SOFT_RST_CTRL__STY_RESET_EN__SHIFT                                                               0x2
+#define SHUB_SOFT_RST_CTRL__NIC400_RESET_EN__SHIFT                                                            0x3
+#define SHUB_SOFT_RST_CTRL__SDP_PORT_RESET_EN__SHIFT                                                          0x4
+#define SHUB_SOFT_RST_CTRL__SION_AON_RESET_EN__SHIFT                                                          0x5
+#define SHUB_SOFT_RST_CTRL__COR_RESET_EN_MASK                                                                 0x00000001L
+#define SHUB_SOFT_RST_CTRL__REG_RESET_EN_MASK                                                                 0x00000002L
+#define SHUB_SOFT_RST_CTRL__STY_RESET_EN_MASK                                                                 0x00000004L
+#define SHUB_SOFT_RST_CTRL__NIC400_RESET_EN_MASK                                                              0x00000008L
+#define SHUB_SOFT_RST_CTRL__SDP_PORT_RESET_EN_MASK                                                            0x00000010L
+#define SHUB_SOFT_RST_CTRL__SION_AON_RESET_EN_MASK                                                            0x00000020L
+//SHUB_SDP_PORT_RST
+#define SHUB_SDP_PORT_RST__A2S_SDP_PORT_RST__SHIFT                                                            0x0
+#define SHUB_SDP_PORT_RST__NBIFSION_BIF_SDP_PORT_RST__SHIFT                                                   0x1
+#define SHUB_SDP_PORT_RST__ATHUB_HST_SDP_PORT_RST__SHIFT                                                      0x2
+#define SHUB_SDP_PORT_RST__ATHUB_DMA_SDP_PORT_RST__SHIFT                                                      0x3
+#define SHUB_SDP_PORT_RST__ATDMA_NBIFSOIN_SDP_PORT_RST__SHIFT                                                 0x4
+#define SHUB_SDP_PORT_RST__MP4SDP_SDP_PORT_RST__SHIFT                                                         0x6
+#define SHUB_SDP_PORT_RST__NTB_HST_SDP_PORT_RST__SHIFT                                                        0x8
+#define SHUB_SDP_PORT_RST__NTB_DMA_SDP_PORT_RST__SHIFT                                                        0x9
+#define SHUB_SDP_PORT_RST__MPDMATF_DMA_SDP_PORT_RST__SHIFT                                                    0xa
+#define SHUB_SDP_PORT_RST__MPDMAPM_DMA_SDP_PORT_RST__SHIFT                                                    0xb
+#define SHUB_SDP_PORT_RST__MPDMATF_HST_SDP_PORT_RST__SHIFT                                                    0xc
+#define SHUB_SDP_PORT_RST__SHUB_CNDI_HST_SDP_PORT_RST__SHIFT                                                  0xd
+#define SHUB_SDP_PORT_RST__SION_AON_RST__SHIFT                                                                0x18
+#define SHUB_SDP_PORT_RST__A2S_SDP_PORT_RST_MASK                                                              0x00000001L
+#define SHUB_SDP_PORT_RST__NBIFSION_BIF_SDP_PORT_RST_MASK                                                     0x00000002L
+#define SHUB_SDP_PORT_RST__ATHUB_HST_SDP_PORT_RST_MASK                                                        0x00000004L
+#define SHUB_SDP_PORT_RST__ATHUB_DMA_SDP_PORT_RST_MASK                                                        0x00000008L
+#define SHUB_SDP_PORT_RST__ATDMA_NBIFSOIN_SDP_PORT_RST_MASK                                                   0x00000010L
+#define SHUB_SDP_PORT_RST__MP4SDP_SDP_PORT_RST_MASK                                                           0x00000040L
+#define SHUB_SDP_PORT_RST__NTB_HST_SDP_PORT_RST_MASK                                                          0x00000100L
+#define SHUB_SDP_PORT_RST__NTB_DMA_SDP_PORT_RST_MASK                                                          0x00000200L
+#define SHUB_SDP_PORT_RST__MPDMATF_DMA_SDP_PORT_RST_MASK                                                      0x00000400L
+#define SHUB_SDP_PORT_RST__MPDMAPM_DMA_SDP_PORT_RST_MASK                                                      0x00000800L
+#define SHUB_SDP_PORT_RST__MPDMATF_HST_SDP_PORT_RST_MASK                                                      0x00001000L
+#define SHUB_SDP_PORT_RST__SHUB_CNDI_HST_SDP_PORT_RST_MASK                                                    0x00002000L
+#define SHUB_SDP_PORT_RST__SION_AON_RST_MASK                                                                  0x01000000L
+
+
+// addressBlock: aid_nbio_nbif0_syshub_mmreg_syshubdirect
+//HST_CLK0_SW0_CL0_CNTL
+#define HST_CLK0_SW0_CL0_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define HST_CLK0_SW0_CL0_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define HST_CLK0_SW0_CL0_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define HST_CLK0_SW0_CL0_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+//HST_CLK0_SW1_CL0_CNTL
+#define HST_CLK0_SW1_CL0_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define HST_CLK0_SW1_CL0_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define HST_CLK0_SW1_CL0_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define HST_CLK0_SW1_CL0_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+//HST_CLK0_SW1_CL1_CNTL
+#define HST_CLK0_SW1_CL1_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define HST_CLK0_SW1_CL1_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define HST_CLK0_SW1_CL1_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define HST_CLK0_SW1_CL1_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+//HST_CLK0_SW1_CL2_CNTL
+#define HST_CLK0_SW1_CL2_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define HST_CLK0_SW1_CL2_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define HST_CLK0_SW1_CL2_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define HST_CLK0_SW1_CL2_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+//DMA_CLK0_SW0_CL0_CNTL
+#define DMA_CLK0_SW0_CL0_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define DMA_CLK0_SW0_CL0_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define DMA_CLK0_SW0_CL0_CNTL__QOS_STATIC_OVERRIDE_EN__SHIFT                                                  0x8
+#define DMA_CLK0_SW0_CL0_CNTL__QOS_STATIC_OVERRIDE_VALUE__SHIFT                                               0x9
+#define DMA_CLK0_SW0_CL0_CNTL__READ_WRR_WEIGHT__SHIFT                                                         0x10
+#define DMA_CLK0_SW0_CL0_CNTL__WRITE_WRR_WEIGHT__SHIFT                                                        0x18
+#define DMA_CLK0_SW0_CL0_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define DMA_CLK0_SW0_CL0_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+#define DMA_CLK0_SW0_CL0_CNTL__QOS_STATIC_OVERRIDE_EN_MASK                                                    0x00000100L
+#define DMA_CLK0_SW0_CL0_CNTL__QOS_STATIC_OVERRIDE_VALUE_MASK                                                 0x00001E00L
+#define DMA_CLK0_SW0_CL0_CNTL__READ_WRR_WEIGHT_MASK                                                           0x00FF0000L
+#define DMA_CLK0_SW0_CL0_CNTL__WRITE_WRR_WEIGHT_MASK                                                          0xFF000000L
+//DMA_CLK0_SW0_CL1_CNTL
+#define DMA_CLK0_SW0_CL1_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define DMA_CLK0_SW0_CL1_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define DMA_CLK0_SW0_CL1_CNTL__QOS_STATIC_OVERRIDE_EN__SHIFT                                                  0x8
+#define DMA_CLK0_SW0_CL1_CNTL__QOS_STATIC_OVERRIDE_VALUE__SHIFT                                               0x9
+#define DMA_CLK0_SW0_CL1_CNTL__READ_WRR_WEIGHT__SHIFT                                                         0x10
+#define DMA_CLK0_SW0_CL1_CNTL__WRITE_WRR_WEIGHT__SHIFT                                                        0x18
+#define DMA_CLK0_SW0_CL1_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define DMA_CLK0_SW0_CL1_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+#define DMA_CLK0_SW0_CL1_CNTL__QOS_STATIC_OVERRIDE_EN_MASK                                                    0x00000100L
+#define DMA_CLK0_SW0_CL1_CNTL__QOS_STATIC_OVERRIDE_VALUE_MASK                                                 0x00001E00L
+#define DMA_CLK0_SW0_CL1_CNTL__READ_WRR_WEIGHT_MASK                                                           0x00FF0000L
+#define DMA_CLK0_SW0_CL1_CNTL__WRITE_WRR_WEIGHT_MASK                                                          0xFF000000L
+//NIC400_1_ASIB_0_FN_MOD
+#define NIC400_1_ASIB_0_FN_MOD__read_iss_override__SHIFT                                                      0x0
+#define NIC400_1_ASIB_0_FN_MOD__write_iss_override__SHIFT                                                     0x1
+#define NIC400_1_ASIB_0_FN_MOD__read_iss_override_MASK                                                        0x00000001L
+#define NIC400_1_ASIB_0_FN_MOD__write_iss_override_MASK                                                       0x00000002L
+//NIC400_1_IB_0_FN_MOD
+#define NIC400_1_IB_0_FN_MOD__read_iss_override__SHIFT                                                        0x0
+#define NIC400_1_IB_0_FN_MOD__write_iss_override__SHIFT                                                       0x1
+#define NIC400_1_IB_0_FN_MOD__read_iss_override_MASK                                                          0x00000001L
+#define NIC400_1_IB_0_FN_MOD__write_iss_override_MASK                                                         0x00000002L
+//NIC400_2_ASIB_0_FN_MOD
+#define NIC400_2_ASIB_0_FN_MOD__read_iss_override__SHIFT                                                      0x0
+#define NIC400_2_ASIB_0_FN_MOD__write_iss_override__SHIFT                                                     0x1
+#define NIC400_2_ASIB_0_FN_MOD__read_iss_override_MASK                                                        0x00000001L
+#define NIC400_2_ASIB_0_FN_MOD__write_iss_override_MASK                                                       0x00000002L
+//NIC400_2_ASIB_0_QOS_CNTL
+#define NIC400_2_ASIB_0_QOS_CNTL__en_aw_rate__SHIFT                                                           0x0
+#define NIC400_2_ASIB_0_QOS_CNTL__en_ar_rate__SHIFT                                                           0x1
+#define NIC400_2_ASIB_0_QOS_CNTL__en_awar_rate__SHIFT                                                         0x2
+#define NIC400_2_ASIB_0_QOS_CNTL__en_aw_fc__SHIFT                                                             0x3
+#define NIC400_2_ASIB_0_QOS_CNTL__en_ar_fc__SHIFT                                                             0x4
+#define NIC400_2_ASIB_0_QOS_CNTL__en_aw_ot__SHIFT                                                             0x5
+#define NIC400_2_ASIB_0_QOS_CNTL__en_ar_ot__SHIFT                                                             0x6
+#define NIC400_2_ASIB_0_QOS_CNTL__en_awar_ot__SHIFT                                                           0x7
+#define NIC400_2_ASIB_0_QOS_CNTL__mode_aw_fc__SHIFT                                                           0x10
+#define NIC400_2_ASIB_0_QOS_CNTL__mode_ar_fc__SHIFT                                                           0x14
+#define NIC400_2_ASIB_0_QOS_CNTL__en_aw_rate_MASK                                                             0x00000001L
+#define NIC400_2_ASIB_0_QOS_CNTL__en_ar_rate_MASK                                                             0x00000002L
+#define NIC400_2_ASIB_0_QOS_CNTL__en_awar_rate_MASK                                                           0x00000004L
+#define NIC400_2_ASIB_0_QOS_CNTL__en_aw_fc_MASK                                                               0x00000008L
+#define NIC400_2_ASIB_0_QOS_CNTL__en_ar_fc_MASK                                                               0x00000010L
+#define NIC400_2_ASIB_0_QOS_CNTL__en_aw_ot_MASK                                                               0x00000020L
+#define NIC400_2_ASIB_0_QOS_CNTL__en_ar_ot_MASK                                                               0x00000040L
+#define NIC400_2_ASIB_0_QOS_CNTL__en_awar_ot_MASK                                                             0x00000080L
+#define NIC400_2_ASIB_0_QOS_CNTL__mode_aw_fc_MASK                                                             0x00010000L
+#define NIC400_2_ASIB_0_QOS_CNTL__mode_ar_fc_MASK                                                             0x00100000L
+//NIC400_2_ASIB_0_MAX_OT
+#define NIC400_2_ASIB_0_MAX_OT__aw_max_otf__SHIFT                                                             0x0
+#define NIC400_2_ASIB_0_MAX_OT__aw_max_oti__SHIFT                                                             0x8
+#define NIC400_2_ASIB_0_MAX_OT__ar_max_otf__SHIFT                                                             0x10
+#define NIC400_2_ASIB_0_MAX_OT__ar_max_oti__SHIFT                                                             0x18
+#define NIC400_2_ASIB_0_MAX_OT__aw_max_otf_MASK                                                               0x000000FFL
+#define NIC400_2_ASIB_0_MAX_OT__aw_max_oti_MASK                                                               0x00003F00L
+#define NIC400_2_ASIB_0_MAX_OT__ar_max_otf_MASK                                                               0x00FF0000L
+#define NIC400_2_ASIB_0_MAX_OT__ar_max_oti_MASK                                                               0x3F000000L
+//NIC400_2_ASIB_0_MAX_COMB_OT
+#define NIC400_2_ASIB_0_MAX_COMB_OT__awar_max_otf__SHIFT                                                      0x0
+#define NIC400_2_ASIB_0_MAX_COMB_OT__awar_max_oti__SHIFT                                                      0x8
+#define NIC400_2_ASIB_0_MAX_COMB_OT__awar_max_otf_MASK                                                        0x000000FFL
+#define NIC400_2_ASIB_0_MAX_COMB_OT__awar_max_oti_MASK                                                        0x00007F00L
+//NIC400_2_ASIB_0_AW_P
+#define NIC400_2_ASIB_0_AW_P__aw_p__SHIFT                                                                     0x18
+#define NIC400_2_ASIB_0_AW_P__aw_p_MASK                                                                       0xFF000000L
+//NIC400_2_ASIB_0_AW_B
+#define NIC400_2_ASIB_0_AW_B__aw_b__SHIFT                                                                     0x0
+#define NIC400_2_ASIB_0_AW_B__aw_b_MASK                                                                       0x0000FFFFL
+//NIC400_2_ASIB_0_AW_R
+#define NIC400_2_ASIB_0_AW_R__aw_r__SHIFT                                                                     0x14
+#define NIC400_2_ASIB_0_AW_R__aw_r_MASK                                                                       0xFFF00000L
+//NIC400_2_ASIB_0_AR_P
+#define NIC400_2_ASIB_0_AR_P__ar_p__SHIFT                                                                     0x18
+#define NIC400_2_ASIB_0_AR_P__ar_p_MASK                                                                       0xFF000000L
+//NIC400_2_ASIB_0_AR_B
+#define NIC400_2_ASIB_0_AR_B__ar_b__SHIFT                                                                     0x0
+#define NIC400_2_ASIB_0_AR_B__ar_b_MASK                                                                       0x0000FFFFL
+//NIC400_2_ASIB_0_AR_R
+#define NIC400_2_ASIB_0_AR_R__ar_r__SHIFT                                                                     0x14
+#define NIC400_2_ASIB_0_AR_R__ar_r_MASK                                                                       0xFFF00000L
+//NIC400_2_ASIB_0_TARGET_FC
+#define NIC400_2_ASIB_0_TARGET_FC__aw_tgt_latency__SHIFT                                                      0x0
+#define NIC400_2_ASIB_0_TARGET_FC__ar_tgt_latency__SHIFT                                                      0x10
+#define NIC400_2_ASIB_0_TARGET_FC__aw_tgt_latency_MASK                                                        0x00000FFFL
+#define NIC400_2_ASIB_0_TARGET_FC__ar_tgt_latency_MASK                                                        0x0FFF0000L
+//NIC400_2_ASIB_0_KI_FC
+#define NIC400_2_ASIB_0_KI_FC__aw_tgt_latency__SHIFT                                                          0x0
+#define NIC400_2_ASIB_0_KI_FC__ar_tgt_latency__SHIFT                                                          0x8
+#define NIC400_2_ASIB_0_KI_FC__aw_tgt_latency_MASK                                                            0x00000007L
+#define NIC400_2_ASIB_0_KI_FC__ar_tgt_latency_MASK                                                            0x00000700L
+//NIC400_2_ASIB_0_QOS_RANGE
+#define NIC400_2_ASIB_0_QOS_RANGE__aw_min_qos__SHIFT                                                          0x0
+#define NIC400_2_ASIB_0_QOS_RANGE__aw_max_qos__SHIFT                                                          0x8
+#define NIC400_2_ASIB_0_QOS_RANGE__ar_min_qos__SHIFT                                                          0x10
+#define NIC400_2_ASIB_0_QOS_RANGE__ar_max_qos__SHIFT                                                          0x18
+#define NIC400_2_ASIB_0_QOS_RANGE__aw_min_qos_MASK                                                            0x0000000FL
+#define NIC400_2_ASIB_0_QOS_RANGE__aw_max_qos_MASK                                                            0x00000F00L
+#define NIC400_2_ASIB_0_QOS_RANGE__ar_min_qos_MASK                                                            0x000F0000L
+#define NIC400_2_ASIB_0_QOS_RANGE__ar_max_qos_MASK                                                            0x0F000000L
+//NIC400_2_ASIB_1_FN_MOD
+#define NIC400_2_ASIB_1_FN_MOD__read_iss_override__SHIFT                                                      0x0
+#define NIC400_2_ASIB_1_FN_MOD__write_iss_override__SHIFT                                                     0x1
+#define NIC400_2_ASIB_1_FN_MOD__read_iss_override_MASK                                                        0x00000001L
+#define NIC400_2_ASIB_1_FN_MOD__write_iss_override_MASK                                                       0x00000002L
+//NIC400_2_ASIB_1_QOS_CNTL
+#define NIC400_2_ASIB_1_QOS_CNTL__en_aw_rate__SHIFT                                                           0x0
+#define NIC400_2_ASIB_1_QOS_CNTL__en_ar_rate__SHIFT                                                           0x1
+#define NIC400_2_ASIB_1_QOS_CNTL__en_awar_rate__SHIFT                                                         0x2
+#define NIC400_2_ASIB_1_QOS_CNTL__en_aw_fc__SHIFT                                                             0x3
+#define NIC400_2_ASIB_1_QOS_CNTL__en_ar_fc__SHIFT                                                             0x4
+#define NIC400_2_ASIB_1_QOS_CNTL__en_aw_ot__SHIFT                                                             0x5
+#define NIC400_2_ASIB_1_QOS_CNTL__en_ar_ot__SHIFT                                                             0x6
+#define NIC400_2_ASIB_1_QOS_CNTL__en_awar_ot__SHIFT                                                           0x7
+#define NIC400_2_ASIB_1_QOS_CNTL__mode_aw_fc__SHIFT                                                           0x10
+#define NIC400_2_ASIB_1_QOS_CNTL__mode_ar_fc__SHIFT                                                           0x14
+#define NIC400_2_ASIB_1_QOS_CNTL__en_aw_rate_MASK                                                             0x00000001L
+#define NIC400_2_ASIB_1_QOS_CNTL__en_ar_rate_MASK                                                             0x00000002L
+#define NIC400_2_ASIB_1_QOS_CNTL__en_awar_rate_MASK                                                           0x00000004L
+#define NIC400_2_ASIB_1_QOS_CNTL__en_aw_fc_MASK                                                               0x00000008L
+#define NIC400_2_ASIB_1_QOS_CNTL__en_ar_fc_MASK                                                               0x00000010L
+#define NIC400_2_ASIB_1_QOS_CNTL__en_aw_ot_MASK                                                               0x00000020L
+#define NIC400_2_ASIB_1_QOS_CNTL__en_ar_ot_MASK                                                               0x00000040L
+#define NIC400_2_ASIB_1_QOS_CNTL__en_awar_ot_MASK                                                             0x00000080L
+#define NIC400_2_ASIB_1_QOS_CNTL__mode_aw_fc_MASK                                                             0x00010000L
+#define NIC400_2_ASIB_1_QOS_CNTL__mode_ar_fc_MASK                                                             0x00100000L
+//NIC400_2_ASIB_1_MAX_OT
+#define NIC400_2_ASIB_1_MAX_OT__aw_max_otf__SHIFT                                                             0x0
+#define NIC400_2_ASIB_1_MAX_OT__aw_max_oti__SHIFT                                                             0x8
+#define NIC400_2_ASIB_1_MAX_OT__ar_max_otf__SHIFT                                                             0x10
+#define NIC400_2_ASIB_1_MAX_OT__ar_max_oti__SHIFT                                                             0x18
+#define NIC400_2_ASIB_1_MAX_OT__aw_max_otf_MASK                                                               0x000000FFL
+#define NIC400_2_ASIB_1_MAX_OT__aw_max_oti_MASK                                                               0x00003F00L
+#define NIC400_2_ASIB_1_MAX_OT__ar_max_otf_MASK                                                               0x00FF0000L
+#define NIC400_2_ASIB_1_MAX_OT__ar_max_oti_MASK                                                               0x3F000000L
+//NIC400_2_ASIB_1_MAX_COMB_OT
+#define NIC400_2_ASIB_1_MAX_COMB_OT__awar_max_otf__SHIFT                                                      0x0
+#define NIC400_2_ASIB_1_MAX_COMB_OT__awar_max_oti__SHIFT                                                      0x8
+#define NIC400_2_ASIB_1_MAX_COMB_OT__awar_max_otf_MASK                                                        0x000000FFL
+#define NIC400_2_ASIB_1_MAX_COMB_OT__awar_max_oti_MASK                                                        0x00007F00L
+//NIC400_2_ASIB_1_AW_P
+#define NIC400_2_ASIB_1_AW_P__aw_p__SHIFT                                                                     0x18
+#define NIC400_2_ASIB_1_AW_P__aw_p_MASK                                                                       0xFF000000L
+//NIC400_2_ASIB_1_AW_B
+#define NIC400_2_ASIB_1_AW_B__aw_b__SHIFT                                                                     0x0
+#define NIC400_2_ASIB_1_AW_B__aw_b_MASK                                                                       0x0000FFFFL
+//NIC400_2_ASIB_1_AW_R
+#define NIC400_2_ASIB_1_AW_R__aw_r__SHIFT                                                                     0x14
+#define NIC400_2_ASIB_1_AW_R__aw_r_MASK                                                                       0xFFF00000L
+//NIC400_2_ASIB_1_AR_P
+#define NIC400_2_ASIB_1_AR_P__ar_p__SHIFT                                                                     0x18
+#define NIC400_2_ASIB_1_AR_P__ar_p_MASK                                                                       0xFF000000L
+//NIC400_2_ASIB_1_AR_B
+#define NIC400_2_ASIB_1_AR_B__ar_b__SHIFT                                                                     0x0
+#define NIC400_2_ASIB_1_AR_B__ar_b_MASK                                                                       0x0000FFFFL
+//NIC400_2_ASIB_1_AR_R
+#define NIC400_2_ASIB_1_AR_R__ar_r__SHIFT                                                                     0x14
+#define NIC400_2_ASIB_1_AR_R__ar_r_MASK                                                                       0xFFF00000L
+//NIC400_2_ASIB_1_TARGET_FC
+#define NIC400_2_ASIB_1_TARGET_FC__aw_tgt_latency__SHIFT                                                      0x0
+#define NIC400_2_ASIB_1_TARGET_FC__ar_tgt_latency__SHIFT                                                      0x10
+#define NIC400_2_ASIB_1_TARGET_FC__aw_tgt_latency_MASK                                                        0x00000FFFL
+#define NIC400_2_ASIB_1_TARGET_FC__ar_tgt_latency_MASK                                                        0x0FFF0000L
+//NIC400_2_ASIB_1_KI_FC
+#define NIC400_2_ASIB_1_KI_FC__aw_tgt_latency__SHIFT                                                          0x0
+#define NIC400_2_ASIB_1_KI_FC__ar_tgt_latency__SHIFT                                                          0x8
+#define NIC400_2_ASIB_1_KI_FC__aw_tgt_latency_MASK                                                            0x00000007L
+#define NIC400_2_ASIB_1_KI_FC__ar_tgt_latency_MASK                                                            0x00000700L
+//NIC400_2_ASIB_1_QOS_RANGE
+#define NIC400_2_ASIB_1_QOS_RANGE__aw_min_qos__SHIFT                                                          0x0
+#define NIC400_2_ASIB_1_QOS_RANGE__aw_max_qos__SHIFT                                                          0x8
+#define NIC400_2_ASIB_1_QOS_RANGE__ar_min_qos__SHIFT                                                          0x10
+#define NIC400_2_ASIB_1_QOS_RANGE__ar_max_qos__SHIFT                                                          0x18
+#define NIC400_2_ASIB_1_QOS_RANGE__aw_min_qos_MASK                                                            0x0000000FL
+#define NIC400_2_ASIB_1_QOS_RANGE__aw_max_qos_MASK                                                            0x00000F00L
+#define NIC400_2_ASIB_1_QOS_RANGE__ar_min_qos_MASK                                                            0x000F0000L
+#define NIC400_2_ASIB_1_QOS_RANGE__ar_max_qos_MASK                                                            0x0F000000L
+//NIC400_2_IB_0_FN_MOD
+#define NIC400_2_IB_0_FN_MOD__read_iss_override__SHIFT                                                        0x0
+#define NIC400_2_IB_0_FN_MOD__write_iss_override__SHIFT                                                       0x1
+#define NIC400_2_IB_0_FN_MOD__read_iss_override_MASK                                                          0x00000001L
+#define NIC400_2_IB_0_FN_MOD__write_iss_override_MASK                                                         0x00000002L
+
+
+// addressBlock: aid_nbio_iohub_nb_nbcfg_nb_cfgdec
+//NB_NBCFG0_NBCFG_SCRATCH_4
+#define NB_NBCFG0_NBCFG_SCRATCH_4__NBCFG_SCRATCH_4__SHIFT                                                     0x0
+#define NB_NBCFG0_NBCFG_SCRATCH_4__NBCFG_SCRATCH_4_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_iohub_nb_misc_misc_cfgdec
+//NB_CNTL
+#define NB_CNTL__HWINIT_WR_LOCK__SHIFT                                                                        0x7
+#define NB_CNTL__HWINIT_WR_LOCK_MASK                                                                          0x00000080L
+//NB_SPARE1
+#define NB_SPARE1__NB_SPARE1_RW__SHIFT                                                                        0x0
+#define NB_SPARE1__NB_SPARE1_RW_MASK                                                                          0xFFFFFFFFL
+//NB_SPARE2
+#define NB_SPARE2__NB_SPARE2_RW1C_0__SHIFT                                                                    0x0
+#define NB_SPARE2__NB_SPARE2_RW1C_1__SHIFT                                                                    0x1
+#define NB_SPARE2__NB_SPARE2_RW1C_2__SHIFT                                                                    0x2
+#define NB_SPARE2__NB_SPARE2_RW1C_3__SHIFT                                                                    0x3
+#define NB_SPARE2__NB_SPARE2_RW1C_4__SHIFT                                                                    0x4
+#define NB_SPARE2__NB_SPARE2_RW1C_5__SHIFT                                                                    0x5
+#define NB_SPARE2__NB_SPARE2_RW1C_6__SHIFT                                                                    0x6
+#define NB_SPARE2__NB_SPARE2_RW1C_7__SHIFT                                                                    0x7
+#define NB_SPARE2__NB_SPARE2_RW1C_8__SHIFT                                                                    0x8
+#define NB_SPARE2__NB_SPARE2_RW1C_9__SHIFT                                                                    0x9
+#define NB_SPARE2__NB_SPARE2_RW1C_10__SHIFT                                                                   0xa
+#define NB_SPARE2__NB_SPARE2_RW1C_11__SHIFT                                                                   0xb
+#define NB_SPARE2__NB_SPARE2_RW1C_12__SHIFT                                                                   0xc
+#define NB_SPARE2__NB_SPARE2_RW1C_13__SHIFT                                                                   0xd
+#define NB_SPARE2__NB_SPARE2_RW1C_14__SHIFT                                                                   0xe
+#define NB_SPARE2__NB_SPARE2_RW1C_15__SHIFT                                                                   0xf
+#define NB_SPARE2__NB_SPARE2_RW1C_16__SHIFT                                                                   0x10
+#define NB_SPARE2__NB_SPARE2_RW1C_17__SHIFT                                                                   0x11
+#define NB_SPARE2__NB_SPARE2_RW1C_18__SHIFT                                                                   0x12
+#define NB_SPARE2__NB_SPARE2_RW1C_19__SHIFT                                                                   0x13
+#define NB_SPARE2__NB_SPARE2_RW1C_20__SHIFT                                                                   0x14
+#define NB_SPARE2__NB_SPARE2_RW1C_21__SHIFT                                                                   0x15
+#define NB_SPARE2__NB_SPARE2_RW1C_22__SHIFT                                                                   0x16
+#define NB_SPARE2__NB_SPARE2_RW1C_23__SHIFT                                                                   0x17
+#define NB_SPARE2__NB_SPARE2_RW1C_24__SHIFT                                                                   0x18
+#define NB_SPARE2__NB_SPARE2_RW1C_25__SHIFT                                                                   0x19
+#define NB_SPARE2__NB_SPARE2_RW1C_26__SHIFT                                                                   0x1a
+#define NB_SPARE2__NB_SPARE2_RW1C_27__SHIFT                                                                   0x1b
+#define NB_SPARE2__NB_SPARE2_RW1C_28__SHIFT                                                                   0x1c
+#define NB_SPARE2__NB_SPARE2_RW1C_29__SHIFT                                                                   0x1d
+#define NB_SPARE2__NB_SPARE2_RW1C_30__SHIFT                                                                   0x1e
+#define NB_SPARE2__NB_SPARE2_RW1C_31__SHIFT                                                                   0x1f
+#define NB_SPARE2__NB_SPARE2_RW1C_0_MASK                                                                      0x00000001L
+#define NB_SPARE2__NB_SPARE2_RW1C_1_MASK                                                                      0x00000002L
+#define NB_SPARE2__NB_SPARE2_RW1C_2_MASK                                                                      0x00000004L
+#define NB_SPARE2__NB_SPARE2_RW1C_3_MASK                                                                      0x00000008L
+#define NB_SPARE2__NB_SPARE2_RW1C_4_MASK                                                                      0x00000010L
+#define NB_SPARE2__NB_SPARE2_RW1C_5_MASK                                                                      0x00000020L
+#define NB_SPARE2__NB_SPARE2_RW1C_6_MASK                                                                      0x00000040L
+#define NB_SPARE2__NB_SPARE2_RW1C_7_MASK                                                                      0x00000080L
+#define NB_SPARE2__NB_SPARE2_RW1C_8_MASK                                                                      0x00000100L
+#define NB_SPARE2__NB_SPARE2_RW1C_9_MASK                                                                      0x00000200L
+#define NB_SPARE2__NB_SPARE2_RW1C_10_MASK                                                                     0x00000400L
+#define NB_SPARE2__NB_SPARE2_RW1C_11_MASK                                                                     0x00000800L
+#define NB_SPARE2__NB_SPARE2_RW1C_12_MASK                                                                     0x00001000L
+#define NB_SPARE2__NB_SPARE2_RW1C_13_MASK                                                                     0x00002000L
+#define NB_SPARE2__NB_SPARE2_RW1C_14_MASK                                                                     0x00004000L
+#define NB_SPARE2__NB_SPARE2_RW1C_15_MASK                                                                     0x00008000L
+#define NB_SPARE2__NB_SPARE2_RW1C_16_MASK                                                                     0x00010000L
+#define NB_SPARE2__NB_SPARE2_RW1C_17_MASK                                                                     0x00020000L
+#define NB_SPARE2__NB_SPARE2_RW1C_18_MASK                                                                     0x00040000L
+#define NB_SPARE2__NB_SPARE2_RW1C_19_MASK                                                                     0x00080000L
+#define NB_SPARE2__NB_SPARE2_RW1C_20_MASK                                                                     0x00100000L
+#define NB_SPARE2__NB_SPARE2_RW1C_21_MASK                                                                     0x00200000L
+#define NB_SPARE2__NB_SPARE2_RW1C_22_MASK                                                                     0x00400000L
+#define NB_SPARE2__NB_SPARE2_RW1C_23_MASK                                                                     0x00800000L
+#define NB_SPARE2__NB_SPARE2_RW1C_24_MASK                                                                     0x01000000L
+#define NB_SPARE2__NB_SPARE2_RW1C_25_MASK                                                                     0x02000000L
+#define NB_SPARE2__NB_SPARE2_RW1C_26_MASK                                                                     0x04000000L
+#define NB_SPARE2__NB_SPARE2_RW1C_27_MASK                                                                     0x08000000L
+#define NB_SPARE2__NB_SPARE2_RW1C_28_MASK                                                                     0x10000000L
+#define NB_SPARE2__NB_SPARE2_RW1C_29_MASK                                                                     0x20000000L
+#define NB_SPARE2__NB_SPARE2_RW1C_30_MASK                                                                     0x40000000L
+#define NB_SPARE2__NB_SPARE2_RW1C_31_MASK                                                                     0x80000000L
+//NB_REVID
+#define NB_REVID__REVISION_ID__SHIFT                                                                          0x0
+#define NB_REVID__REVISION_ID_MASK                                                                            0x000003FFL
+//NBIO_LCLK_DS_MASK
+#define NBIO_LCLK_DS_MASK__LCLK_DS_MASK__SHIFT                                                                0x0
+#define NBIO_LCLK_DS_MASK__LCLK_DS_MASK_MASK                                                                  0xFFFFFFFFL
+//NB_BUS_NUM_CNTL
+#define NB_BUS_NUM_CNTL__NB_BUS_NUM__SHIFT                                                                    0x0
+#define NB_BUS_NUM_CNTL__NB_BUS_LAT_Mode__SHIFT                                                               0x8
+#define NB_BUS_NUM_CNTL__NB_SEGMENT__SHIFT                                                                    0x10
+#define NB_BUS_NUM_CNTL__NB_BUS_NUM_MASK                                                                      0x000000FFL
+#define NB_BUS_NUM_CNTL__NB_BUS_LAT_Mode_MASK                                                                 0x00000100L
+#define NB_BUS_NUM_CNTL__NB_SEGMENT_MASK                                                                      0x00FF0000L
+//NB_MMIOBASE
+#define NB_MMIOBASE__MMIOBASE__SHIFT                                                                          0x0
+#define NB_MMIOBASE__MMIOBASE_MASK                                                                            0xFFFFFFFFL
+//NB_MMIOLIMIT
+#define NB_MMIOLIMIT__MMIOLIMIT__SHIFT                                                                        0x0
+#define NB_MMIOLIMIT__MMIOLIMIT_MASK                                                                          0xFFFFFFFFL
+//NB_LOWER_TOP_OF_DRAM2
+#define NB_LOWER_TOP_OF_DRAM2__ENABLE__SHIFT                                                                  0x0
+#define NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2__SHIFT                                                              0x17
+#define NB_LOWER_TOP_OF_DRAM2__ENABLE_MASK                                                                    0x00000001L
+#define NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2_MASK                                                                0xFF800000L
+//NB_UPPER_TOP_OF_DRAM2
+#define NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2__SHIFT                                                              0x0
+#define NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2_MASK                                                                0x000001FFL
+//NB_LOWER_DRAM2_BASE
+#define NB_LOWER_DRAM2_BASE__LOWER_DRAM2_BASE__SHIFT                                                          0x17
+#define NB_LOWER_DRAM2_BASE__LOWER_DRAM2_BASE_MASK                                                            0xFF800000L
+//NB_UPPER_DRAM2_BASE
+#define NB_UPPER_DRAM2_BASE__UPPER_DRAM2_BASE__SHIFT                                                          0x0
+#define NB_UPPER_DRAM2_BASE__UPPER_DRAM2_BASE_MASK                                                            0x000001FFL
+//SB_LOCATION
+#define SB_LOCATION__SBlocated_Port__SHIFT                                                                    0x0
+#define SB_LOCATION__SBlocated_Core__SHIFT                                                                    0x10
+#define SB_LOCATION__SBlocated_Port_MASK                                                                      0x0000FFFFL
+#define SB_LOCATION__SBlocated_Core_MASK                                                                      0xFFFF0000L
+//SW_US_LOCATION
+#define SW_US_LOCATION__SW_USlocated_Port__SHIFT                                                              0x0
+#define SW_US_LOCATION__SW_USlocated_Core__SHIFT                                                              0x10
+#define SW_US_LOCATION__SW_USlocated_Port_MASK                                                                0x0000FFFFL
+#define SW_US_LOCATION__SW_USlocated_Core_MASK                                                                0xFFFF0000L
+//NB_PROG_DEVICE_REMAP_PBr0
+#define NB_PROG_DEVICE_REMAP_PBr0__PBr0_DevFnMap__SHIFT                                                       0x0
+#define NB_PROG_DEVICE_REMAP_PBr0__PBr0_DevFnMap_MASK                                                         0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr1
+#define NB_PROG_DEVICE_REMAP_PBr1__PBr1_DevFnMap__SHIFT                                                       0x0
+#define NB_PROG_DEVICE_REMAP_PBr1__PBr1_DevFnMap_MASK                                                         0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr2
+#define NB_PROG_DEVICE_REMAP_PBr2__PBr2_DevFnMap__SHIFT                                                       0x0
+#define NB_PROG_DEVICE_REMAP_PBr2__PBr2_DevFnMap_MASK                                                         0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr3
+#define NB_PROG_DEVICE_REMAP_PBr3__PBr3_DevFnMap__SHIFT                                                       0x0
+#define NB_PROG_DEVICE_REMAP_PBr3__PBr3_DevFnMap_MASK                                                         0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr4
+#define NB_PROG_DEVICE_REMAP_PBr4__PBr4_DevFnMap__SHIFT                                                       0x0
+#define NB_PROG_DEVICE_REMAP_PBr4__PBr4_DevFnMap_MASK                                                         0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr5
+#define NB_PROG_DEVICE_REMAP_PBr5__PBr5_DevFnMap__SHIFT                                                       0x0
+#define NB_PROG_DEVICE_REMAP_PBr5__PBr5_DevFnMap_MASK                                                         0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr6
+#define NB_PROG_DEVICE_REMAP_PBr6__PBr6_DevFnMap__SHIFT                                                       0x0
+#define NB_PROG_DEVICE_REMAP_PBr6__PBr6_DevFnMap_MASK                                                         0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr7
+#define NB_PROG_DEVICE_REMAP_PBr7__PBr7_DevFnMap__SHIFT                                                       0x0
+#define NB_PROG_DEVICE_REMAP_PBr7__PBr7_DevFnMap_MASK                                                         0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr8
+#define NB_PROG_DEVICE_REMAP_PBr8__PBr8_DevFnMap__SHIFT                                                       0x0
+#define NB_PROG_DEVICE_REMAP_PBr8__PBr8_DevFnMap_MASK                                                         0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr10
+#define NB_PROG_DEVICE_REMAP_PBr10__PBr10_DevFnMap__SHIFT                                                     0x0
+#define NB_PROG_DEVICE_REMAP_PBr10__PBr10_DevFnMap_MASK                                                       0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr11
+#define NB_PROG_DEVICE_REMAP_PBr11__PBr11_DevFnMap__SHIFT                                                     0x0
+#define NB_PROG_DEVICE_REMAP_PBr11__PBr11_DevFnMap_MASK                                                       0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr12
+#define NB_PROG_DEVICE_REMAP_PBr12__PBr12_DevFnMap__SHIFT                                                     0x0
+#define NB_PROG_DEVICE_REMAP_PBr12__PBr12_DevFnMap_MASK                                                       0x000000FFL
+//NB_PROG_DEVICE_REMAP_PBr13
+#define NB_PROG_DEVICE_REMAP_PBr13__PBr13_DevFnMap__SHIFT                                                     0x0
+#define NB_PROG_DEVICE_REMAP_PBr13__PBr13_DevFnMap_MASK                                                       0x000000FFL
+//SW_NMI_CNTL
+#define SW_NMI_CNTL__SW_NMI_Status__SHIFT                                                                     0x0
+#define SW_NMI_CNTL__SW_NMI_Status_MASK                                                                       0xFFFFFFFFL
+//SW_SMI_CNTL
+#define SW_SMI_CNTL__SW_SMI_Status__SHIFT                                                                     0x0
+#define SW_SMI_CNTL__SW_SMI_Status_MASK                                                                       0xFFFFFFFFL
+//SW_SCI_CNTL
+#define SW_SCI_CNTL__SW_SCI_Status__SHIFT                                                                     0x0
+#define SW_SCI_CNTL__SW_SCI_Status_MASK                                                                       0xFFFFFFFFL
+//APML_SW_STATUS
+#define APML_SW_STATUS__APML_NMI_STATUS__SHIFT                                                                0x0
+#define APML_SW_STATUS__APML_NMI_STATUS_MASK                                                                  0x00000001L
+//SW_GIC_SPI_CNTL
+#define SW_GIC_SPI_CNTL__SW_NMI_GIC_SPI_Vector__SHIFT                                                         0x0
+#define SW_GIC_SPI_CNTL__SW_SMI_GIC_SPI_Vector__SHIFT                                                         0x8
+#define SW_GIC_SPI_CNTL__SW_SCI_GIC_SPI_Vector__SHIFT                                                         0x10
+#define SW_GIC_SPI_CNTL__SW_NMI_GIC_SPI_Vector_MASK                                                           0x000000FFL
+#define SW_GIC_SPI_CNTL__SW_SMI_GIC_SPI_Vector_MASK                                                           0x0000FF00L
+#define SW_GIC_SPI_CNTL__SW_SCI_GIC_SPI_Vector_MASK                                                           0x00FF0000L
+//SW_SYNCFLOOD_CNTL
+#define SW_SYNCFLOOD_CNTL__SW_SYNCFLOOD_PRIVATE__SHIFT                                                        0x0
+#define SW_SYNCFLOOD_CNTL__SW_SYNCFLOOD_APML__SHIFT                                                           0x1
+#define SW_SYNCFLOOD_CNTL__SW_SYNCFLOOD_PRIVATE_MASK                                                          0x00000001L
+#define SW_SYNCFLOOD_CNTL__SW_SYNCFLOOD_APML_MASK                                                             0x00000002L
+//NB_TOP_OF_DRAM3
+#define NB_TOP_OF_DRAM3__TOM3_LIMIT__SHIFT                                                                    0x0
+#define NB_TOP_OF_DRAM3__TOM3_ENABLE__SHIFT                                                                   0x1f
+#define NB_TOP_OF_DRAM3__TOM3_LIMIT_MASK                                                                      0x3FFFFFFFL
+#define NB_TOP_OF_DRAM3__TOM3_ENABLE_MASK                                                                     0x80000000L
+//CAM_CONTROL
+#define CAM_CONTROL__CAM_En__SHIFT                                                                            0x0
+#define CAM_CONTROL__Op__SHIFT                                                                                0x1
+#define CAM_CONTROL__AccessType__SHIFT                                                                        0x2
+#define CAM_CONTROL__DataMatchEn__SHIFT                                                                       0x3
+#define CAM_CONTROL__VC__SHIFT                                                                                0x4
+#define CAM_CONTROL__CrossTrigger__SHIFT                                                                      0x8
+#define CAM_CONTROL__CAM_En_MASK                                                                              0x00000001L
+#define CAM_CONTROL__Op_MASK                                                                                  0x00000002L
+#define CAM_CONTROL__AccessType_MASK                                                                          0x00000004L
+#define CAM_CONTROL__DataMatchEn_MASK                                                                         0x00000008L
+#define CAM_CONTROL__VC_MASK                                                                                  0x00000070L
+#define CAM_CONTROL__CrossTrigger_MASK                                                                        0x00000F00L
+//CAM_TARGET_INDEX_ADDR_BOTTOM
+#define CAM_TARGET_INDEX_ADDR_BOTTOM__IndexAddrBottom__SHIFT                                                  0x0
+#define CAM_TARGET_INDEX_ADDR_BOTTOM__IndexAddrBottom_MASK                                                    0xFFFFFFFFL
+//CAM_TARGET_INDEX_ADDR_TOP
+#define CAM_TARGET_INDEX_ADDR_TOP__IndexAddrTop__SHIFT                                                        0x0
+#define CAM_TARGET_INDEX_ADDR_TOP__IndexAddrTop_MASK                                                          0xFFFFFFFFL
+//CAM_TARGET_INDEX_DATA
+#define CAM_TARGET_INDEX_DATA__IndexData__SHIFT                                                               0x0
+#define CAM_TARGET_INDEX_DATA__IndexData_MASK                                                                 0xFFFFFFFFL
+//CAM_TARGET_INDEX_DATA_MASK
+#define CAM_TARGET_INDEX_DATA_MASK__IndexDataMask__SHIFT                                                      0x0
+#define CAM_TARGET_INDEX_DATA_MASK__IndexDataMask_MASK                                                        0xFFFFFFFFL
+//CAM_TARGET_DATA_ADDR_BOTTOM
+#define CAM_TARGET_DATA_ADDR_BOTTOM__DataAddrBottom__SHIFT                                                    0x0
+#define CAM_TARGET_DATA_ADDR_BOTTOM__DataAddrBottom_MASK                                                      0xFFFFFFFFL
+//CAM_TARGET_DATA_ADDR_TOP
+#define CAM_TARGET_DATA_ADDR_TOP__DataAddrTop__SHIFT                                                          0x0
+#define CAM_TARGET_DATA_ADDR_TOP__DataAddrTop_MASK                                                            0xFFFFFFFFL
+//CAM_TARGET_DATA
+#define CAM_TARGET_DATA__Data__SHIFT                                                                          0x0
+#define CAM_TARGET_DATA__Data_MASK                                                                            0xFFFFFFFFL
+//CAM_TARGET_DATA_MASK
+#define CAM_TARGET_DATA_MASK__DataMask__SHIFT                                                                 0x0
+#define CAM_TARGET_DATA_MASK__DataMask_MASK                                                                   0xFFFFFFFFL
+//P_DMA_DROPPED_LOG_LOWER
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_0__SHIFT                                             0x0
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_1__SHIFT                                             0x1
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_2__SHIFT                                             0x2
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_3__SHIFT                                             0x3
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_4__SHIFT                                             0x4
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_5__SHIFT                                             0x5
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_6__SHIFT                                             0x6
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_7__SHIFT                                             0x7
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_8__SHIFT                                             0x8
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_9__SHIFT                                             0x9
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_10__SHIFT                                            0xa
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_11__SHIFT                                            0xb
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_12__SHIFT                                            0xc
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_13__SHIFT                                            0xd
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_14__SHIFT                                            0xe
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_15__SHIFT                                            0xf
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_16__SHIFT                                            0x10
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_17__SHIFT                                            0x11
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_18__SHIFT                                            0x12
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_19__SHIFT                                            0x13
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_20__SHIFT                                            0x14
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_21__SHIFT                                            0x15
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_22__SHIFT                                            0x16
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_23__SHIFT                                            0x17
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_24__SHIFT                                            0x18
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_25__SHIFT                                            0x19
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_26__SHIFT                                            0x1a
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_27__SHIFT                                            0x1b
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_28__SHIFT                                            0x1c
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_29__SHIFT                                            0x1d
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_30__SHIFT                                            0x1e
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_31__SHIFT                                            0x1f
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_0_MASK                                               0x00000001L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_1_MASK                                               0x00000002L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_2_MASK                                               0x00000004L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_3_MASK                                               0x00000008L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_4_MASK                                               0x00000010L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_5_MASK                                               0x00000020L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_6_MASK                                               0x00000040L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_7_MASK                                               0x00000080L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_8_MASK                                               0x00000100L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_9_MASK                                               0x00000200L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_10_MASK                                              0x00000400L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_11_MASK                                              0x00000800L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_12_MASK                                              0x00001000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_13_MASK                                              0x00002000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_14_MASK                                              0x00004000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_15_MASK                                              0x00008000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_16_MASK                                              0x00010000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_17_MASK                                              0x00020000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_18_MASK                                              0x00040000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_19_MASK                                              0x00080000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_20_MASK                                              0x00100000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_21_MASK                                              0x00200000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_22_MASK                                              0x00400000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_23_MASK                                              0x00800000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_24_MASK                                              0x01000000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_25_MASK                                              0x02000000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_26_MASK                                              0x04000000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_27_MASK                                              0x08000000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_28_MASK                                              0x10000000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_29_MASK                                              0x20000000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_30_MASK                                              0x40000000L
+#define P_DMA_DROPPED_LOG_LOWER__P_DMA_DROPPED_LOG_LOWER_31_MASK                                              0x80000000L
+//P_DMA_DROPPED_LOG_UPPER
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_0__SHIFT                                             0x0
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_1__SHIFT                                             0x1
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_2__SHIFT                                             0x2
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_3__SHIFT                                             0x3
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_4__SHIFT                                             0x4
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_5__SHIFT                                             0x5
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_6__SHIFT                                             0x6
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_7__SHIFT                                             0x7
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_8__SHIFT                                             0x8
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_9__SHIFT                                             0x9
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_10__SHIFT                                            0xa
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_11__SHIFT                                            0xb
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_12__SHIFT                                            0xc
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_13__SHIFT                                            0xd
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_14__SHIFT                                            0xe
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_15__SHIFT                                            0xf
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_16__SHIFT                                            0x10
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_17__SHIFT                                            0x11
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_18__SHIFT                                            0x12
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_19__SHIFT                                            0x13
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_20__SHIFT                                            0x14
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_21__SHIFT                                            0x15
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_22__SHIFT                                            0x16
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_23__SHIFT                                            0x17
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_24__SHIFT                                            0x18
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_25__SHIFT                                            0x19
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_26__SHIFT                                            0x1a
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_27__SHIFT                                            0x1b
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_28__SHIFT                                            0x1c
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_29__SHIFT                                            0x1d
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_30__SHIFT                                            0x1e
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_31__SHIFT                                            0x1f
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_0_MASK                                               0x00000001L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_1_MASK                                               0x00000002L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_2_MASK                                               0x00000004L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_3_MASK                                               0x00000008L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_4_MASK                                               0x00000010L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_5_MASK                                               0x00000020L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_6_MASK                                               0x00000040L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_7_MASK                                               0x00000080L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_8_MASK                                               0x00000100L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_9_MASK                                               0x00000200L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_10_MASK                                              0x00000400L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_11_MASK                                              0x00000800L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_12_MASK                                              0x00001000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_13_MASK                                              0x00002000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_14_MASK                                              0x00004000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_15_MASK                                              0x00008000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_16_MASK                                              0x00010000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_17_MASK                                              0x00020000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_18_MASK                                              0x00040000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_19_MASK                                              0x00080000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_20_MASK                                              0x00100000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_21_MASK                                              0x00200000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_22_MASK                                              0x00400000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_23_MASK                                              0x00800000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_24_MASK                                              0x01000000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_25_MASK                                              0x02000000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_26_MASK                                              0x04000000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_27_MASK                                              0x08000000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_28_MASK                                              0x10000000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_29_MASK                                              0x20000000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_30_MASK                                              0x40000000L
+#define P_DMA_DROPPED_LOG_UPPER__P_DMA_DROPPED_LOG_UPPER_31_MASK                                              0x80000000L
+//NP_DMA_DROPPED_LOG_LOWER
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_0__SHIFT                                           0x0
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_1__SHIFT                                           0x1
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_2__SHIFT                                           0x2
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_3__SHIFT                                           0x3
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_4__SHIFT                                           0x4
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_5__SHIFT                                           0x5
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_6__SHIFT                                           0x6
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_7__SHIFT                                           0x7
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_8__SHIFT                                           0x8
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_9__SHIFT                                           0x9
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_10__SHIFT                                          0xa
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_11__SHIFT                                          0xb
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_12__SHIFT                                          0xc
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_13__SHIFT                                          0xd
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_14__SHIFT                                          0xe
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_15__SHIFT                                          0xf
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_16__SHIFT                                          0x10
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_17__SHIFT                                          0x11
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_18__SHIFT                                          0x12
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_19__SHIFT                                          0x13
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_20__SHIFT                                          0x14
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_21__SHIFT                                          0x15
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_22__SHIFT                                          0x16
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_23__SHIFT                                          0x17
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_24__SHIFT                                          0x18
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_25__SHIFT                                          0x19
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_26__SHIFT                                          0x1a
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_27__SHIFT                                          0x1b
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_28__SHIFT                                          0x1c
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_29__SHIFT                                          0x1d
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_30__SHIFT                                          0x1e
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_31__SHIFT                                          0x1f
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_0_MASK                                             0x00000001L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_1_MASK                                             0x00000002L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_2_MASK                                             0x00000004L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_3_MASK                                             0x00000008L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_4_MASK                                             0x00000010L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_5_MASK                                             0x00000020L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_6_MASK                                             0x00000040L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_7_MASK                                             0x00000080L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_8_MASK                                             0x00000100L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_9_MASK                                             0x00000200L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_10_MASK                                            0x00000400L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_11_MASK                                            0x00000800L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_12_MASK                                            0x00001000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_13_MASK                                            0x00002000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_14_MASK                                            0x00004000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_15_MASK                                            0x00008000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_16_MASK                                            0x00010000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_17_MASK                                            0x00020000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_18_MASK                                            0x00040000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_19_MASK                                            0x00080000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_20_MASK                                            0x00100000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_21_MASK                                            0x00200000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_22_MASK                                            0x00400000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_23_MASK                                            0x00800000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_24_MASK                                            0x01000000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_25_MASK                                            0x02000000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_26_MASK                                            0x04000000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_27_MASK                                            0x08000000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_28_MASK                                            0x10000000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_29_MASK                                            0x20000000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_30_MASK                                            0x40000000L
+#define NP_DMA_DROPPED_LOG_LOWER__NP_DMA_DROPPED_LOG_LOWER_31_MASK                                            0x80000000L
+//NP_DMA_DROPPED_LOG_UPPER
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_0__SHIFT                                           0x0
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_1__SHIFT                                           0x1
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_2__SHIFT                                           0x2
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_3__SHIFT                                           0x3
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_4__SHIFT                                           0x4
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_5__SHIFT                                           0x5
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_6__SHIFT                                           0x6
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_7__SHIFT                                           0x7
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_8__SHIFT                                           0x8
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_9__SHIFT                                           0x9
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_10__SHIFT                                          0xa
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_11__SHIFT                                          0xb
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_12__SHIFT                                          0xc
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_13__SHIFT                                          0xd
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_14__SHIFT                                          0xe
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_15__SHIFT                                          0xf
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_16__SHIFT                                          0x10
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_17__SHIFT                                          0x11
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_18__SHIFT                                          0x12
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_19__SHIFT                                          0x13
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_20__SHIFT                                          0x14
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_21__SHIFT                                          0x15
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_22__SHIFT                                          0x16
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_23__SHIFT                                          0x17
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_24__SHIFT                                          0x18
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_25__SHIFT                                          0x19
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_26__SHIFT                                          0x1a
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_27__SHIFT                                          0x1b
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_28__SHIFT                                          0x1c
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_29__SHIFT                                          0x1d
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_30__SHIFT                                          0x1e
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_31__SHIFT                                          0x1f
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_0_MASK                                             0x00000001L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_1_MASK                                             0x00000002L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_2_MASK                                             0x00000004L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_3_MASK                                             0x00000008L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_4_MASK                                             0x00000010L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_5_MASK                                             0x00000020L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_6_MASK                                             0x00000040L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_7_MASK                                             0x00000080L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_8_MASK                                             0x00000100L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_9_MASK                                             0x00000200L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_10_MASK                                            0x00000400L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_11_MASK                                            0x00000800L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_12_MASK                                            0x00001000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_13_MASK                                            0x00002000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_14_MASK                                            0x00004000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_15_MASK                                            0x00008000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_16_MASK                                            0x00010000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_17_MASK                                            0x00020000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_18_MASK                                            0x00040000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_19_MASK                                            0x00080000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_20_MASK                                            0x00100000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_21_MASK                                            0x00200000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_22_MASK                                            0x00400000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_23_MASK                                            0x00800000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_24_MASK                                            0x01000000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_25_MASK                                            0x02000000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_26_MASK                                            0x04000000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_27_MASK                                            0x08000000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_28_MASK                                            0x10000000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_29_MASK                                            0x20000000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_30_MASK                                            0x40000000L
+#define NP_DMA_DROPPED_LOG_UPPER__NP_DMA_DROPPED_LOG_UPPER_31_MASK                                            0x80000000L
+//PCIE_VDM_NODE0_CTRL4
+#define PCIE_VDM_NODE0_CTRL4__BUS_RANGE_BASE__SHIFT                                                           0x0
+#define PCIE_VDM_NODE0_CTRL4__BUS_RANGE_LIMIT__SHIFT                                                          0x8
+#define PCIE_VDM_NODE0_CTRL4__NODE0_PRESENT__SHIFT                                                            0x1f
+#define PCIE_VDM_NODE0_CTRL4__BUS_RANGE_BASE_MASK                                                             0x000000FFL
+#define PCIE_VDM_NODE0_CTRL4__BUS_RANGE_LIMIT_MASK                                                            0x0000FF00L
+#define PCIE_VDM_NODE0_CTRL4__NODE0_PRESENT_MASK                                                              0x80000000L
+//PCIE_VDM_CNTL2
+#define PCIE_VDM_CNTL2__VdmP2pMode__SHIFT                                                                     0x0
+#define PCIE_VDM_CNTL2__MCTPEndpointEn__SHIFT                                                                 0x2
+#define PCIE_VDM_CNTL2__MCTPMultiSegEn__SHIFT                                                                 0x3
+#define PCIE_VDM_CNTL2__MCTPT2SMUEn__SHIFT                                                                    0x4
+#define PCIE_VDM_CNTL2__AMDVDM2SMUEn__SHIFT                                                                   0x5
+#define PCIE_VDM_CNTL2__OtherVDM2SMUEn__SHIFT                                                                 0x6
+#define PCIE_VDM_CNTL2__RouteAllToMCTPMaster__SHIFT                                                           0x7
+#define PCIE_VDM_CNTL2__MCTPMasterSeg__SHIFT                                                                  0x8
+#define PCIE_VDM_CNTL2__MCTPMasterID__SHIFT                                                                   0x10
+#define PCIE_VDM_CNTL2__VdmP2pMode_MASK                                                                       0x00000003L
+#define PCIE_VDM_CNTL2__MCTPEndpointEn_MASK                                                                   0x00000004L
+#define PCIE_VDM_CNTL2__MCTPMultiSegEn_MASK                                                                   0x00000008L
+#define PCIE_VDM_CNTL2__MCTPT2SMUEn_MASK                                                                      0x00000010L
+#define PCIE_VDM_CNTL2__AMDVDM2SMUEn_MASK                                                                     0x00000020L
+#define PCIE_VDM_CNTL2__OtherVDM2SMUEn_MASK                                                                   0x00000040L
+#define PCIE_VDM_CNTL2__RouteAllToMCTPMaster_MASK                                                             0x00000080L
+#define PCIE_VDM_CNTL2__MCTPMasterSeg_MASK                                                                    0x0000FF00L
+#define PCIE_VDM_CNTL2__MCTPMasterID_MASK                                                                     0xFFFF0000L
+//PCIE_VDM_CNTL3
+#define PCIE_VDM_CNTL3__APMTPMasterValid__SHIFT                                                               0xf
+#define PCIE_VDM_CNTL3__APMTPMasterID__SHIFT                                                                  0x10
+#define PCIE_VDM_CNTL3__APMTPMasterValid_MASK                                                                 0x00008000L
+#define PCIE_VDM_CNTL3__APMTPMasterID_MASK                                                                    0xFFFF0000L
+//STALL_CONTROL_XBARPORT0_0
+#define STALL_CONTROL_XBARPORT0_0__StallVC0ReqEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT0_0__StallVC1ReqEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT0_0__StallVC2ReqEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT0_0__StallVC3ReqEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT0_0__StallVC4ReqEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT0_0__StallVC5ReqEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT0_0__StallVC7ReqEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT0_0__StallVC0ReqEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT0_0__StallVC1ReqEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT0_0__StallVC2ReqEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT0_0__StallVC3ReqEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT0_0__StallVC4ReqEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT0_0__StallVC5ReqEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT0_0__StallVC7ReqEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT0_1
+#define STALL_CONTROL_XBARPORT0_1__StallVC0RspEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT0_1__StallVC1RspEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT0_1__StallVC2RspEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT0_1__StallVC3RspEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT0_1__StallVC4RspEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT0_1__StallVC5RspEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT0_1__StallVC7RspEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT0_1__StallVC0RspEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT0_1__StallVC1RspEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT0_1__StallVC2RspEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT0_1__StallVC3RspEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT0_1__StallVC4RspEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT0_1__StallVC5RspEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT0_1__StallVC7RspEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT1_0
+#define STALL_CONTROL_XBARPORT1_0__StallVC0ReqEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT1_0__StallVC1ReqEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT1_0__StallVC2ReqEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT1_0__StallVC3ReqEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT1_0__StallVC4ReqEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT1_0__StallVC5ReqEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT1_0__StallVC7ReqEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT1_0__StallVC0ReqEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT1_0__StallVC1ReqEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT1_0__StallVC2ReqEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT1_0__StallVC3ReqEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT1_0__StallVC4ReqEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT1_0__StallVC5ReqEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT1_0__StallVC7ReqEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT1_1
+#define STALL_CONTROL_XBARPORT1_1__StallVC0RspEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT1_1__StallVC1RspEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT1_1__StallVC2RspEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT1_1__StallVC3RspEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT1_1__StallVC4RspEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT1_1__StallVC5RspEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT1_1__StallVC7RspEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT1_1__StallVC0RspEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT1_1__StallVC1RspEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT1_1__StallVC2RspEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT1_1__StallVC3RspEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT1_1__StallVC4RspEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT1_1__StallVC5RspEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT1_1__StallVC7RspEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT2_0
+#define STALL_CONTROL_XBARPORT2_0__StallVC0ReqEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT2_0__StallVC1ReqEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT2_0__StallVC2ReqEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT2_0__StallVC3ReqEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT2_0__StallVC4ReqEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT2_0__StallVC5ReqEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT2_0__StallVC7ReqEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT2_0__StallVC0ReqEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT2_0__StallVC1ReqEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT2_0__StallVC2ReqEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT2_0__StallVC3ReqEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT2_0__StallVC4ReqEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT2_0__StallVC5ReqEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT2_0__StallVC7ReqEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT2_1
+#define STALL_CONTROL_XBARPORT2_1__StallVC0RspEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT2_1__StallVC1RspEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT2_1__StallVC2RspEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT2_1__StallVC3RspEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT2_1__StallVC4RspEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT2_1__StallVC5RspEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT2_1__StallVC7RspEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT2_1__StallVC0RspEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT2_1__StallVC1RspEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT2_1__StallVC2RspEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT2_1__StallVC3RspEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT2_1__StallVC4RspEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT2_1__StallVC5RspEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT2_1__StallVC7RspEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT3_0
+#define STALL_CONTROL_XBARPORT3_0__StallVC0ReqEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT3_0__StallVC1ReqEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT3_0__StallVC2ReqEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT3_0__StallVC3ReqEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT3_0__StallVC4ReqEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT3_0__StallVC5ReqEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT3_0__StallVC7ReqEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT3_0__StallVC0ReqEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT3_0__StallVC1ReqEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT3_0__StallVC2ReqEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT3_0__StallVC3ReqEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT3_0__StallVC4ReqEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT3_0__StallVC5ReqEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT3_0__StallVC7ReqEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT3_1
+#define STALL_CONTROL_XBARPORT3_1__StallVC0RspEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT3_1__StallVC1RspEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT3_1__StallVC2RspEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT3_1__StallVC3RspEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT3_1__StallVC4RspEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT3_1__StallVC5RspEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT3_1__StallVC7RspEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT3_1__StallVC0RspEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT3_1__StallVC1RspEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT3_1__StallVC2RspEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT3_1__StallVC3RspEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT3_1__StallVC4RspEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT3_1__StallVC5RspEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT3_1__StallVC7RspEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT4_0
+#define STALL_CONTROL_XBARPORT4_0__StallVC0ReqEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT4_0__StallVC1ReqEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT4_0__StallVC2ReqEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT4_0__StallVC3ReqEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT4_0__StallVC4ReqEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT4_0__StallVC5ReqEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT4_0__StallVC7ReqEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT4_0__StallVC0ReqEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT4_0__StallVC1ReqEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT4_0__StallVC2ReqEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT4_0__StallVC3ReqEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT4_0__StallVC4ReqEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT4_0__StallVC5ReqEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT4_0__StallVC7ReqEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT4_1
+#define STALL_CONTROL_XBARPORT4_1__StallVC0RspEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT4_1__StallVC1RspEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT4_1__StallVC2RspEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT4_1__StallVC3RspEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT4_1__StallVC4RspEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT4_1__StallVC5RspEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT4_1__StallVC7RspEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT4_1__StallVC0RspEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT4_1__StallVC1RspEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT4_1__StallVC2RspEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT4_1__StallVC3RspEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT4_1__StallVC4RspEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT4_1__StallVC5RspEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT4_1__StallVC7RspEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT5_0
+#define STALL_CONTROL_XBARPORT5_0__StallVC0ReqEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT5_0__StallVC1ReqEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT5_0__StallVC2ReqEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT5_0__StallVC3ReqEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT5_0__StallVC4ReqEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT5_0__StallVC5ReqEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT5_0__StallVC7ReqEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT5_0__StallVC0ReqEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT5_0__StallVC1ReqEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT5_0__StallVC2ReqEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT5_0__StallVC3ReqEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT5_0__StallVC4ReqEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT5_0__StallVC5ReqEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT5_0__StallVC7ReqEn_MASK                                                         0x30000000L
+//STALL_CONTROL_XBARPORT5_1
+#define STALL_CONTROL_XBARPORT5_1__StallVC0RspEn__SHIFT                                                       0x0
+#define STALL_CONTROL_XBARPORT5_1__StallVC1RspEn__SHIFT                                                       0x4
+#define STALL_CONTROL_XBARPORT5_1__StallVC2RspEn__SHIFT                                                       0x8
+#define STALL_CONTROL_XBARPORT5_1__StallVC3RspEn__SHIFT                                                       0xc
+#define STALL_CONTROL_XBARPORT5_1__StallVC4RspEn__SHIFT                                                       0x10
+#define STALL_CONTROL_XBARPORT5_1__StallVC5RspEn__SHIFT                                                       0x14
+#define STALL_CONTROL_XBARPORT5_1__StallVC7RspEn__SHIFT                                                       0x1c
+#define STALL_CONTROL_XBARPORT5_1__StallVC0RspEn_MASK                                                         0x00000003L
+#define STALL_CONTROL_XBARPORT5_1__StallVC1RspEn_MASK                                                         0x00000030L
+#define STALL_CONTROL_XBARPORT5_1__StallVC2RspEn_MASK                                                         0x00000300L
+#define STALL_CONTROL_XBARPORT5_1__StallVC3RspEn_MASK                                                         0x00003000L
+#define STALL_CONTROL_XBARPORT5_1__StallVC4RspEn_MASK                                                         0x00030000L
+#define STALL_CONTROL_XBARPORT5_1__StallVC5RspEn_MASK                                                         0x00300000L
+#define STALL_CONTROL_XBARPORT5_1__StallVC7RspEn_MASK                                                         0x30000000L
+//NB_DRAM3_BASE
+#define NB_DRAM3_BASE__DRAM3_BASE__SHIFT                                                                      0x0
+#define NB_DRAM3_BASE__DRAM3_BASE_MASK                                                                        0x3FFFFFFFL
+//PSP_BASE_ADDR_LO
+#define PSP_BASE_ADDR_LO__PSP_MMIO_EN__SHIFT                                                                  0x0
+#define PSP_BASE_ADDR_LO__PSP_MMIO_LOCK__SHIFT                                                                0x8
+#define PSP_BASE_ADDR_LO__PSP_BASE_ADDR_LO__SHIFT                                                             0x14
+#define PSP_BASE_ADDR_LO__PSP_MMIO_EN_MASK                                                                    0x00000001L
+#define PSP_BASE_ADDR_LO__PSP_MMIO_LOCK_MASK                                                                  0x00000100L
+#define PSP_BASE_ADDR_LO__PSP_BASE_ADDR_LO_MASK                                                               0xFFF00000L
+//PSP_BASE_ADDR_HI
+#define PSP_BASE_ADDR_HI__PSP_BASE_ADDR_HI__SHIFT                                                             0x0
+#define PSP_BASE_ADDR_HI__PSP_BASE_ADDR_HI_MASK                                                               0x0000FFFFL
+//SMU_BASE_ADDR_LO
+#define SMU_BASE_ADDR_LO__SMU_MMIO_EN__SHIFT                                                                  0x0
+#define SMU_BASE_ADDR_LO__SMU_MMIO_LOCK__SHIFT                                                                0x1
+#define SMU_BASE_ADDR_LO__SMU_BASE_ADDR_LO__SHIFT                                                             0x14
+#define SMU_BASE_ADDR_LO__SMU_MMIO_EN_MASK                                                                    0x00000001L
+#define SMU_BASE_ADDR_LO__SMU_MMIO_LOCK_MASK                                                                  0x00000002L
+#define SMU_BASE_ADDR_LO__SMU_BASE_ADDR_LO_MASK                                                               0xFFF00000L
+//SMU_BASE_ADDR_HI
+#define SMU_BASE_ADDR_HI__SMU_BASE_ADDR_HI__SHIFT                                                             0x0
+#define SMU_BASE_ADDR_HI__SMU_BASE_ADDR_HI_MASK                                                               0x0000FFFFL
+//SCRATCH_4
+#define SCRATCH_4__SCRATCH_4__SHIFT                                                                           0x0
+#define SCRATCH_4__SCRATCH_4_MASK                                                                             0xFFFFFFFFL
+//SCRATCH_5
+#define SCRATCH_5__SCRATCH_5__SHIFT                                                                           0x0
+#define SCRATCH_5__SCRATCH_5_MASK                                                                             0xFFFFFFFFL
+//SMU_BLOCK_CPU
+#define SMU_BLOCK_CPU__SMUBlockCPU_Valid__SHIFT                                                               0x0
+#define SMU_BLOCK_CPU__SMUBlockCPU_Valid_MASK                                                                 0x00000001L
+//SMU_BLOCK_CPU_STATUS
+#define SMU_BLOCK_CPU_STATUS__SMUBlockCPU_Status__SHIFT                                                       0x0
+#define SMU_BLOCK_CPU_STATUS__SMUBlockCPU_Status_MASK                                                         0x00000001L
+//TRAP_STATUS
+#define TRAP_STATUS__TrapReqValid__SHIFT                                                                      0x0
+#define TRAP_STATUS__TrapNumber__SHIFT                                                                        0x8
+#define TRAP_STATUS__TrapS2Vld__SHIFT                                                                         0xc
+#define TRAP_STATUS__TrapS2Number__SHIFT                                                                      0x10
+#define TRAP_STATUS__TrapReqValid_MASK                                                                        0x00000001L
+#define TRAP_STATUS__TrapNumber_MASK                                                                          0x00000F00L
+#define TRAP_STATUS__TrapS2Vld_MASK                                                                           0x00001000L
+#define TRAP_STATUS__TrapS2Number_MASK                                                                        0x03FF0000L
+//TRAP_REQUEST0
+#define TRAP_REQUEST0__TrapReqAddrLo__SHIFT                                                                   0x2
+#define TRAP_REQUEST0__TrapReqAddrLo_MASK                                                                     0xFFFFFFFCL
+//TRAP_REQUEST1
+#define TRAP_REQUEST1__TrapReqAddrHi__SHIFT                                                                   0x0
+#define TRAP_REQUEST1__TrapReqAddrHi_MASK                                                                     0xFFFFFFFFL
+//TRAP_REQUEST2
+#define TRAP_REQUEST2__TrapReqCmd__SHIFT                                                                      0x0
+#define TRAP_REQUEST2__TrapAttr__SHIFT                                                                        0x8
+#define TRAP_REQUEST2__TrapReqLen__SHIFT                                                                      0x10
+#define TRAP_REQUEST2__TrapReqCmd_MASK                                                                        0x0000003FL
+#define TRAP_REQUEST2__TrapAttr_MASK                                                                          0x0000FF00L
+#define TRAP_REQUEST2__TrapReqLen_MASK                                                                        0x003F0000L
+//TRAP_REQUEST3
+#define TRAP_REQUEST3__TrapReqVC__SHIFT                                                                       0x0
+#define TRAP_REQUEST3__TrapReqBlockLevel__SHIFT                                                               0x4
+#define TRAP_REQUEST3__TrapReqChain__SHIFT                                                                    0x6
+#define TRAP_REQUEST3__TrapReqIO__SHIFT                                                                       0x7
+#define TRAP_REQUEST3__TrapReqPassPW__SHIFT                                                                   0x8
+#define TRAP_REQUEST3__TrapReqRspPassPW__SHIFT                                                                0x9
+#define TRAP_REQUEST3__TrapReqUnitID__SHIFT                                                                   0x10
+#define TRAP_REQUEST3__TrapReqVC_MASK                                                                         0x00000007L
+#define TRAP_REQUEST3__TrapReqBlockLevel_MASK                                                                 0x00000030L
+#define TRAP_REQUEST3__TrapReqChain_MASK                                                                      0x00000040L
+#define TRAP_REQUEST3__TrapReqIO_MASK                                                                         0x00000080L
+#define TRAP_REQUEST3__TrapReqPassPW_MASK                                                                     0x00000100L
+#define TRAP_REQUEST3__TrapReqRspPassPW_MASK                                                                  0x00000200L
+#define TRAP_REQUEST3__TrapReqUnitID_MASK                                                                     0x003F0000L
+//TRAP_REQUEST4
+#define TRAP_REQUEST4__TrapReqSecLevel__SHIFT                                                                 0x0
+#define TRAP_REQUEST4__TrapReqSecLevel_MASK                                                                   0x0000000FL
+//TRAP_REQUEST5
+#define TRAP_REQUEST5__TrapReqDataVC__SHIFT                                                                   0x0
+#define TRAP_REQUEST5__TrapReqDataErr__SHIFT                                                                  0x4
+#define TRAP_REQUEST5__TrapReqDataParity__SHIFT                                                               0x8
+#define TRAP_REQUEST5__TrapReqDataVC_MASK                                                                     0x00000007L
+#define TRAP_REQUEST5__TrapReqDataErr_MASK                                                                    0x00000010L
+#define TRAP_REQUEST5__TrapReqDataParity_MASK                                                                 0x0000FF00L
+//TRAP_REQUEST_DATASTRB0
+#define TRAP_REQUEST_DATASTRB0__TrapReqDataBytEn0__SHIFT                                                      0x0
+#define TRAP_REQUEST_DATASTRB0__TrapReqDataBytEn0_MASK                                                        0xFFFFFFFFL
+//TRAP_REQUEST_DATASTRB1
+#define TRAP_REQUEST_DATASTRB1__TrapReqDataBytEn1__SHIFT                                                      0x0
+#define TRAP_REQUEST_DATASTRB1__TrapReqDataBytEn1_MASK                                                        0xFFFFFFFFL
+//TRAP_REQUEST_DATA0
+#define TRAP_REQUEST_DATA0__TrapReqData0__SHIFT                                                               0x0
+#define TRAP_REQUEST_DATA0__TrapReqData0_MASK                                                                 0xFFFFFFFFL
+//TRAP_REQUEST_DATA1
+#define TRAP_REQUEST_DATA1__TrapReqData1__SHIFT                                                               0x0
+#define TRAP_REQUEST_DATA1__TrapReqData1_MASK                                                                 0xFFFFFFFFL
+//TRAP_REQUEST_DATA2
+#define TRAP_REQUEST_DATA2__TrapReqData2__SHIFT                                                               0x0
+#define TRAP_REQUEST_DATA2__TrapReqData2_MASK                                                                 0xFFFFFFFFL
+//TRAP_REQUEST_DATA3
+#define TRAP_REQUEST_DATA3__TrapReqData3__SHIFT                                                               0x0
+#define TRAP_REQUEST_DATA3__TrapReqData3_MASK                                                                 0xFFFFFFFFL
+//TRAP_REQUEST_DATA4
+#define TRAP_REQUEST_DATA4__TrapReqData4__SHIFT                                                               0x0
+#define TRAP_REQUEST_DATA4__TrapReqData4_MASK                                                                 0xFFFFFFFFL
+//TRAP_REQUEST_DATA5
+#define TRAP_REQUEST_DATA5__TrapReqData5__SHIFT                                                               0x0
+#define TRAP_REQUEST_DATA5__TrapReqData5_MASK                                                                 0xFFFFFFFFL
+//TRAP_REQUEST_DATA6
+#define TRAP_REQUEST_DATA6__TrapReqData6__SHIFT                                                               0x0
+#define TRAP_REQUEST_DATA6__TrapReqData6_MASK                                                                 0xFFFFFFFFL
+//TRAP_REQUEST_DATA7
+#define TRAP_REQUEST_DATA7__TrapReqData7__SHIFT                                                               0x0
+#define TRAP_REQUEST_DATA7__TrapReqData7_MASK                                                                 0xFFFFFFFFL
+//TRAP_REQUEST_DATA8
+#define TRAP_REQUEST_DATA8__TrapReqData8__SHIFT                                                               0x0
+#define TRAP_REQUEST_DATA8__TrapReqData8_MASK                                                                 0xFFFFFFFFL
+//TRAP_REQUEST_DATA9
+#define TRAP_REQUEST_DATA9__TrapReqData9__SHIFT                                                               0x0
+#define TRAP_REQUEST_DATA9__TrapReqData9_MASK                                                                 0xFFFFFFFFL
+//TRAP_REQUEST_DATA10
+#define TRAP_REQUEST_DATA10__TrapReqData10__SHIFT                                                             0x0
+#define TRAP_REQUEST_DATA10__TrapReqData10_MASK                                                               0xFFFFFFFFL
+//TRAP_REQUEST_DATA11
+#define TRAP_REQUEST_DATA11__TrapReqData11__SHIFT                                                             0x0
+#define TRAP_REQUEST_DATA11__TrapReqData11_MASK                                                               0xFFFFFFFFL
+//TRAP_REQUEST_DATA12
+#define TRAP_REQUEST_DATA12__TrapReqData12__SHIFT                                                             0x0
+#define TRAP_REQUEST_DATA12__TrapReqData12_MASK                                                               0xFFFFFFFFL
+//TRAP_REQUEST_DATA13
+#define TRAP_REQUEST_DATA13__TrapReqData13__SHIFT                                                             0x0
+#define TRAP_REQUEST_DATA13__TrapReqData13_MASK                                                               0xFFFFFFFFL
+//TRAP_REQUEST_DATA14
+#define TRAP_REQUEST_DATA14__TrapReqData14__SHIFT                                                             0x0
+#define TRAP_REQUEST_DATA14__TrapReqData14_MASK                                                               0xFFFFFFFFL
+//TRAP_REQUEST_DATA15
+#define TRAP_REQUEST_DATA15__TrapReqData15__SHIFT                                                             0x0
+#define TRAP_REQUEST_DATA15__TrapReqData15_MASK                                                               0xFFFFFFFFL
+//TRAP_RESPONSE_CONTROL
+#define TRAP_RESPONSE_CONTROL__TrapRspTrigger__SHIFT                                                          0x0
+#define TRAP_RESPONSE_CONTROL__TrapRspReqPassthru__SHIFT                                                      0x1
+#define TRAP_RESPONSE_CONTROL__TrapRspTrigger_MASK                                                            0x00000001L
+#define TRAP_RESPONSE_CONTROL__TrapRspReqPassthru_MASK                                                        0x00000002L
+//TRAP_RESPONSE0
+#define TRAP_RESPONSE0__TrapRspPassPW__SHIFT                                                                  0x0
+#define TRAP_RESPONSE0__TrapRspStatus__SHIFT                                                                  0x4
+#define TRAP_RESPONSE0__TrapRspDataStatus__SHIFT                                                              0x10
+#define TRAP_RESPONSE0__TrapRspPassPW_MASK                                                                    0x00000001L
+#define TRAP_RESPONSE0__TrapRspStatus_MASK                                                                    0x000000F0L
+#define TRAP_RESPONSE0__TrapRspDataStatus_MASK                                                                0x00FF0000L
+//TRAP_RESPONSE_DATA0
+#define TRAP_RESPONSE_DATA0__TrapRdRspData0__SHIFT                                                            0x0
+#define TRAP_RESPONSE_DATA0__TrapRdRspData0_MASK                                                              0xFFFFFFFFL
+//TRAP_RESPONSE_DATA1
+#define TRAP_RESPONSE_DATA1__TrapRdRspData1__SHIFT                                                            0x0
+#define TRAP_RESPONSE_DATA1__TrapRdRspData1_MASK                                                              0xFFFFFFFFL
+//TRAP_RESPONSE_DATA2
+#define TRAP_RESPONSE_DATA2__TrapRdRspData2__SHIFT                                                            0x0
+#define TRAP_RESPONSE_DATA2__TrapRdRspData2_MASK                                                              0xFFFFFFFFL
+//TRAP_RESPONSE_DATA3
+#define TRAP_RESPONSE_DATA3__TrapRdRspData3__SHIFT                                                            0x0
+#define TRAP_RESPONSE_DATA3__TrapRdRspData3_MASK                                                              0xFFFFFFFFL
+//TRAP_RESPONSE_DATA4
+#define TRAP_RESPONSE_DATA4__TrapRdRspData4__SHIFT                                                            0x0
+#define TRAP_RESPONSE_DATA4__TrapRdRspData4_MASK                                                              0xFFFFFFFFL
+//TRAP_RESPONSE_DATA5
+#define TRAP_RESPONSE_DATA5__TrapRdRspData5__SHIFT                                                            0x0
+#define TRAP_RESPONSE_DATA5__TrapRdRspData5_MASK                                                              0xFFFFFFFFL
+//TRAP_RESPONSE_DATA6
+#define TRAP_RESPONSE_DATA6__TrapRdRspData6__SHIFT                                                            0x0
+#define TRAP_RESPONSE_DATA6__TrapRdRspData6_MASK                                                              0xFFFFFFFFL
+//TRAP_RESPONSE_DATA7
+#define TRAP_RESPONSE_DATA7__TrapRdRspData7__SHIFT                                                            0x0
+#define TRAP_RESPONSE_DATA7__TrapRdRspData7_MASK                                                              0xFFFFFFFFL
+//TRAP_RESPONSE_DATA8
+#define TRAP_RESPONSE_DATA8__TrapRdRspData8__SHIFT                                                            0x0
+#define TRAP_RESPONSE_DATA8__TrapRdRspData8_MASK                                                              0xFFFFFFFFL
+//TRAP_RESPONSE_DATA9
+#define TRAP_RESPONSE_DATA9__TrapRdRspData9__SHIFT                                                            0x0
+#define TRAP_RESPONSE_DATA9__TrapRdRspData9_MASK                                                              0xFFFFFFFFL
+//TRAP_RESPONSE_DATA10
+#define TRAP_RESPONSE_DATA10__TrapRdRspData10__SHIFT                                                          0x0
+#define TRAP_RESPONSE_DATA10__TrapRdRspData10_MASK                                                            0xFFFFFFFFL
+//TRAP_RESPONSE_DATA11
+#define TRAP_RESPONSE_DATA11__TrapRdRspData11__SHIFT                                                          0x0
+#define TRAP_RESPONSE_DATA11__TrapRdRspData11_MASK                                                            0xFFFFFFFFL
+//TRAP_RESPONSE_DATA12
+#define TRAP_RESPONSE_DATA12__TrapRdRspData12__SHIFT                                                          0x0
+#define TRAP_RESPONSE_DATA12__TrapRdRspData12_MASK                                                            0xFFFFFFFFL
+//TRAP_RESPONSE_DATA13
+#define TRAP_RESPONSE_DATA13__TrapRdRspData13__SHIFT                                                          0x0
+#define TRAP_RESPONSE_DATA13__TrapRdRspData13_MASK                                                            0xFFFFFFFFL
+//TRAP_RESPONSE_DATA14
+#define TRAP_RESPONSE_DATA14__TrapRdRspData14__SHIFT                                                          0x0
+#define TRAP_RESPONSE_DATA14__TrapRdRspData14_MASK                                                            0xFFFFFFFFL
+//TRAP_RESPONSE_DATA15
+#define TRAP_RESPONSE_DATA15__TrapRdRspData15__SHIFT                                                          0x0
+#define TRAP_RESPONSE_DATA15__TrapRdRspData15_MASK                                                            0xFFFFFFFFL
+//TRAP0_CONTROL0
+#define TRAP0_CONTROL0__Trap0En__SHIFT                                                                        0x0
+#define TRAP0_CONTROL0__Trap0SMUIntr__SHIFT                                                                   0x3
+#define TRAP0_CONTROL0__Trap0Stage2Ptr__SHIFT                                                                 0xe
+#define TRAP0_CONTROL0__Trap0CrossTrigger__SHIFT                                                              0x18
+#define TRAP0_CONTROL0__Trap0Stage2En__SHIFT                                                                  0x1f
+#define TRAP0_CONTROL0__Trap0En_MASK                                                                          0x00000001L
+#define TRAP0_CONTROL0__Trap0SMUIntr_MASK                                                                     0x00000008L
+#define TRAP0_CONTROL0__Trap0Stage2Ptr_MASK                                                                   0x00FFC000L
+#define TRAP0_CONTROL0__Trap0CrossTrigger_MASK                                                                0x0F000000L
+#define TRAP0_CONTROL0__Trap0Stage2En_MASK                                                                    0x80000000L
+//TRAP0_ADDRESS_LO
+#define TRAP0_ADDRESS_LO__Trap0AddrLo__SHIFT                                                                  0x2
+#define TRAP0_ADDRESS_LO__Trap0AddrLo_MASK                                                                    0xFFFFFFFCL
+//TRAP0_ADDRESS_HI
+#define TRAP0_ADDRESS_HI__Trap0AddrHi__SHIFT                                                                  0x0
+#define TRAP0_ADDRESS_HI__Trap0AddrHi_MASK                                                                    0xFFFFFFFFL
+//TRAP0_COMMAND
+#define TRAP0_COMMAND__Trap0Cmd0__SHIFT                                                                       0x0
+#define TRAP0_COMMAND__Trap0Cmd1__SHIFT                                                                       0x8
+#define TRAP0_COMMAND__Trap0Cmd0_MASK                                                                         0x0000003FL
+#define TRAP0_COMMAND__Trap0Cmd1_MASK                                                                         0x00003F00L
+//TRAP0_ADDRESS_LO_MASK
+#define TRAP0_ADDRESS_LO_MASK__Trap0AddrLoMask__SHIFT                                                         0x2
+#define TRAP0_ADDRESS_LO_MASK__Trap0AddrLoMask_MASK                                                           0xFFFFFFFCL
+//TRAP0_ADDRESS_HI_MASK
+#define TRAP0_ADDRESS_HI_MASK__Trap0AddrHiMask__SHIFT                                                         0x0
+#define TRAP0_ADDRESS_HI_MASK__Trap0AddrHiMask_MASK                                                           0xFFFFFFFFL
+//TRAP0_COMMAND_MASK
+#define TRAP0_COMMAND_MASK__Trap0Cmd0Mask__SHIFT                                                              0x0
+#define TRAP0_COMMAND_MASK__Trap0Cmd1Mask__SHIFT                                                              0x8
+#define TRAP0_COMMAND_MASK__Trap0Cmd0Mask_MASK                                                                0x0000003FL
+#define TRAP0_COMMAND_MASK__Trap0Cmd1Mask_MASK                                                                0x00003F00L
+//TRAP1_CONTROL0
+#define TRAP1_CONTROL0__Trap1En__SHIFT                                                                        0x0
+#define TRAP1_CONTROL0__Trap1SMUIntr__SHIFT                                                                   0x3
+#define TRAP1_CONTROL0__Trap1Stage2Ptr__SHIFT                                                                 0xe
+#define TRAP1_CONTROL0__Trap1CrossTrigger__SHIFT                                                              0x18
+#define TRAP1_CONTROL0__Trap1Stage2En__SHIFT                                                                  0x1f
+#define TRAP1_CONTROL0__Trap1En_MASK                                                                          0x00000001L
+#define TRAP1_CONTROL0__Trap1SMUIntr_MASK                                                                     0x00000008L
+#define TRAP1_CONTROL0__Trap1Stage2Ptr_MASK                                                                   0x00FFC000L
+#define TRAP1_CONTROL0__Trap1CrossTrigger_MASK                                                                0x0F000000L
+#define TRAP1_CONTROL0__Trap1Stage2En_MASK                                                                    0x80000000L
+//TRAP1_ADDRESS_LO
+#define TRAP1_ADDRESS_LO__Trap1AddrLo__SHIFT                                                                  0x2
+#define TRAP1_ADDRESS_LO__Trap1AddrLo_MASK                                                                    0xFFFFFFFCL
+//TRAP1_ADDRESS_HI
+#define TRAP1_ADDRESS_HI__Trap1AddrHi__SHIFT                                                                  0x0
+#define TRAP1_ADDRESS_HI__Trap1AddrHi_MASK                                                                    0xFFFFFFFFL
+//TRAP1_COMMAND
+#define TRAP1_COMMAND__Trap1Cmd0__SHIFT                                                                       0x0
+#define TRAP1_COMMAND__Trap1Cmd1__SHIFT                                                                       0x8
+#define TRAP1_COMMAND__Trap1Cmd0_MASK                                                                         0x0000003FL
+#define TRAP1_COMMAND__Trap1Cmd1_MASK                                                                         0x00003F00L
+//TRAP1_ADDRESS_LO_MASK
+#define TRAP1_ADDRESS_LO_MASK__Trap1AddrLoMask__SHIFT                                                         0x2
+#define TRAP1_ADDRESS_LO_MASK__Trap1AddrLoMask_MASK                                                           0xFFFFFFFCL
+//TRAP1_ADDRESS_HI_MASK
+#define TRAP1_ADDRESS_HI_MASK__Trap1AddrHiMask__SHIFT                                                         0x0
+#define TRAP1_ADDRESS_HI_MASK__Trap1AddrHiMask_MASK                                                           0xFFFFFFFFL
+//TRAP1_COMMAND_MASK
+#define TRAP1_COMMAND_MASK__Trap1Cmd0Mask__SHIFT                                                              0x0
+#define TRAP1_COMMAND_MASK__Trap1Cmd1Mask__SHIFT                                                              0x8
+#define TRAP1_COMMAND_MASK__Trap1Cmd0Mask_MASK                                                                0x0000003FL
+#define TRAP1_COMMAND_MASK__Trap1Cmd1Mask_MASK                                                                0x00003F00L
+//TRAP2_CONTROL0
+#define TRAP2_CONTROL0__Trap2En__SHIFT                                                                        0x0
+#define TRAP2_CONTROL0__Trap2SMUIntr__SHIFT                                                                   0x3
+#define TRAP2_CONTROL0__Trap2Stage2Ptr__SHIFT                                                                 0xe
+#define TRAP2_CONTROL0__Trap2CrossTrigger__SHIFT                                                              0x18
+#define TRAP2_CONTROL0__Trap2Stage2En__SHIFT                                                                  0x1f
+#define TRAP2_CONTROL0__Trap2En_MASK                                                                          0x00000001L
+#define TRAP2_CONTROL0__Trap2SMUIntr_MASK                                                                     0x00000008L
+#define TRAP2_CONTROL0__Trap2Stage2Ptr_MASK                                                                   0x00FFC000L
+#define TRAP2_CONTROL0__Trap2CrossTrigger_MASK                                                                0x0F000000L
+#define TRAP2_CONTROL0__Trap2Stage2En_MASK                                                                    0x80000000L
+//TRAP2_ADDRESS_LO
+#define TRAP2_ADDRESS_LO__Trap2AddrLo__SHIFT                                                                  0x2
+#define TRAP2_ADDRESS_LO__Trap2AddrLo_MASK                                                                    0xFFFFFFFCL
+//TRAP2_ADDRESS_HI
+#define TRAP2_ADDRESS_HI__Trap2AddrHi__SHIFT                                                                  0x0
+#define TRAP2_ADDRESS_HI__Trap2AddrHi_MASK                                                                    0xFFFFFFFFL
+//TRAP2_COMMAND
+#define TRAP2_COMMAND__Trap2Cmd0__SHIFT                                                                       0x0
+#define TRAP2_COMMAND__Trap2Cmd1__SHIFT                                                                       0x8
+#define TRAP2_COMMAND__Trap2Cmd0_MASK                                                                         0x0000003FL
+#define TRAP2_COMMAND__Trap2Cmd1_MASK                                                                         0x00003F00L
+//TRAP2_ADDRESS_LO_MASK
+#define TRAP2_ADDRESS_LO_MASK__Trap2AddrLoMask__SHIFT                                                         0x2
+#define TRAP2_ADDRESS_LO_MASK__Trap2AddrLoMask_MASK                                                           0xFFFFFFFCL
+//TRAP2_ADDRESS_HI_MASK
+#define TRAP2_ADDRESS_HI_MASK__Trap2AddrHiMask__SHIFT                                                         0x0
+#define TRAP2_ADDRESS_HI_MASK__Trap2AddrHiMask_MASK                                                           0xFFFFFFFFL
+//TRAP2_COMMAND_MASK
+#define TRAP2_COMMAND_MASK__Trap2Cmd0Mask__SHIFT                                                              0x0
+#define TRAP2_COMMAND_MASK__Trap2Cmd1Mask__SHIFT                                                              0x8
+#define TRAP2_COMMAND_MASK__Trap2Cmd0Mask_MASK                                                                0x0000003FL
+#define TRAP2_COMMAND_MASK__Trap2Cmd1Mask_MASK                                                                0x00003F00L
+//TRAP3_CONTROL0
+#define TRAP3_CONTROL0__Trap3En__SHIFT                                                                        0x0
+#define TRAP3_CONTROL0__Trap3SMUIntr__SHIFT                                                                   0x3
+#define TRAP3_CONTROL0__Trap3Stage2Ptr__SHIFT                                                                 0xe
+#define TRAP3_CONTROL0__Trap3CrossTrigger__SHIFT                                                              0x18
+#define TRAP3_CONTROL0__Trap3Stage2En__SHIFT                                                                  0x1f
+#define TRAP3_CONTROL0__Trap3En_MASK                                                                          0x00000001L
+#define TRAP3_CONTROL0__Trap3SMUIntr_MASK                                                                     0x00000008L
+#define TRAP3_CONTROL0__Trap3Stage2Ptr_MASK                                                                   0x00FFC000L
+#define TRAP3_CONTROL0__Trap3CrossTrigger_MASK                                                                0x0F000000L
+#define TRAP3_CONTROL0__Trap3Stage2En_MASK                                                                    0x80000000L
+//TRAP3_ADDRESS_LO
+#define TRAP3_ADDRESS_LO__Trap3AddrLo__SHIFT                                                                  0x2
+#define TRAP3_ADDRESS_LO__Trap3AddrLo_MASK                                                                    0xFFFFFFFCL
+//TRAP3_ADDRESS_HI
+#define TRAP3_ADDRESS_HI__Trap3AddrHi__SHIFT                                                                  0x0
+#define TRAP3_ADDRESS_HI__Trap3AddrHi_MASK                                                                    0xFFFFFFFFL
+//TRAP3_COMMAND
+#define TRAP3_COMMAND__Trap3Cmd0__SHIFT                                                                       0x0
+#define TRAP3_COMMAND__Trap3Cmd1__SHIFT                                                                       0x8
+#define TRAP3_COMMAND__Trap3Cmd0_MASK                                                                         0x0000003FL
+#define TRAP3_COMMAND__Trap3Cmd1_MASK                                                                         0x00003F00L
+//TRAP3_ADDRESS_LO_MASK
+#define TRAP3_ADDRESS_LO_MASK__Trap3AddrLoMask__SHIFT                                                         0x2
+#define TRAP3_ADDRESS_LO_MASK__Trap3AddrLoMask_MASK                                                           0xFFFFFFFCL
+//TRAP3_ADDRESS_HI_MASK
+#define TRAP3_ADDRESS_HI_MASK__Trap3AddrHiMask__SHIFT                                                         0x0
+#define TRAP3_ADDRESS_HI_MASK__Trap3AddrHiMask_MASK                                                           0xFFFFFFFFL
+//TRAP3_COMMAND_MASK
+#define TRAP3_COMMAND_MASK__Trap3Cmd0Mask__SHIFT                                                              0x0
+#define TRAP3_COMMAND_MASK__Trap3Cmd1Mask__SHIFT                                                              0x8
+#define TRAP3_COMMAND_MASK__Trap3Cmd0Mask_MASK                                                                0x0000003FL
+#define TRAP3_COMMAND_MASK__Trap3Cmd1Mask_MASK                                                                0x00003F00L
+//TRAP4_CONTROL0
+#define TRAP4_CONTROL0__Trap4En__SHIFT                                                                        0x0
+#define TRAP4_CONTROL0__Trap4SMUIntr__SHIFT                                                                   0x3
+#define TRAP4_CONTROL0__Trap4Stage2Ptr__SHIFT                                                                 0xe
+#define TRAP4_CONTROL0__Trap4CrossTrigger__SHIFT                                                              0x18
+#define TRAP4_CONTROL0__Trap4Stage2En__SHIFT                                                                  0x1f
+#define TRAP4_CONTROL0__Trap4En_MASK                                                                          0x00000001L
+#define TRAP4_CONTROL0__Trap4SMUIntr_MASK                                                                     0x00000008L
+#define TRAP4_CONTROL0__Trap4Stage2Ptr_MASK                                                                   0x00FFC000L
+#define TRAP4_CONTROL0__Trap4CrossTrigger_MASK                                                                0x0F000000L
+#define TRAP4_CONTROL0__Trap4Stage2En_MASK                                                                    0x80000000L
+//TRAP4_ADDRESS_LO
+#define TRAP4_ADDRESS_LO__Trap4AddrLo__SHIFT                                                                  0x2
+#define TRAP4_ADDRESS_LO__Trap4AddrLo_MASK                                                                    0xFFFFFFFCL
+//TRAP4_ADDRESS_HI
+#define TRAP4_ADDRESS_HI__Trap4AddrHi__SHIFT                                                                  0x0
+#define TRAP4_ADDRESS_HI__Trap4AddrHi_MASK                                                                    0xFFFFFFFFL
+//TRAP4_COMMAND
+#define TRAP4_COMMAND__Trap4Cmd0__SHIFT                                                                       0x0
+#define TRAP4_COMMAND__Trap4Cmd1__SHIFT                                                                       0x8
+#define TRAP4_COMMAND__Trap4Cmd0_MASK                                                                         0x0000003FL
+#define TRAP4_COMMAND__Trap4Cmd1_MASK                                                                         0x00003F00L
+//TRAP4_ADDRESS_LO_MASK
+#define TRAP4_ADDRESS_LO_MASK__Trap4AddrLoMask__SHIFT                                                         0x2
+#define TRAP4_ADDRESS_LO_MASK__Trap4AddrLoMask_MASK                                                           0xFFFFFFFCL
+//TRAP4_ADDRESS_HI_MASK
+#define TRAP4_ADDRESS_HI_MASK__Trap4AddrHiMask__SHIFT                                                         0x0
+#define TRAP4_ADDRESS_HI_MASK__Trap4AddrHiMask_MASK                                                           0xFFFFFFFFL
+//TRAP4_COMMAND_MASK
+#define TRAP4_COMMAND_MASK__Trap4Cmd0Mask__SHIFT                                                              0x0
+#define TRAP4_COMMAND_MASK__Trap4Cmd1Mask__SHIFT                                                              0x8
+#define TRAP4_COMMAND_MASK__Trap4Cmd0Mask_MASK                                                                0x0000003FL
+#define TRAP4_COMMAND_MASK__Trap4Cmd1Mask_MASK                                                                0x00003F00L
+//TRAP5_CONTROL0
+#define TRAP5_CONTROL0__Trap5En__SHIFT                                                                        0x0
+#define TRAP5_CONTROL0__Trap5SMUIntr__SHIFT                                                                   0x3
+#define TRAP5_CONTROL0__Trap5Stage2Ptr__SHIFT                                                                 0xe
+#define TRAP5_CONTROL0__Trap5CrossTrigger__SHIFT                                                              0x18
+#define TRAP5_CONTROL0__Trap5Stage2En__SHIFT                                                                  0x1f
+#define TRAP5_CONTROL0__Trap5En_MASK                                                                          0x00000001L
+#define TRAP5_CONTROL0__Trap5SMUIntr_MASK                                                                     0x00000008L
+#define TRAP5_CONTROL0__Trap5Stage2Ptr_MASK                                                                   0x00FFC000L
+#define TRAP5_CONTROL0__Trap5CrossTrigger_MASK                                                                0x0F000000L
+#define TRAP5_CONTROL0__Trap5Stage2En_MASK                                                                    0x80000000L
+//TRAP5_ADDRESS_LO
+#define TRAP5_ADDRESS_LO__Trap5AddrLo__SHIFT                                                                  0x2
+#define TRAP5_ADDRESS_LO__Trap5AddrLo_MASK                                                                    0xFFFFFFFCL
+//TRAP5_ADDRESS_HI
+#define TRAP5_ADDRESS_HI__Trap5AddrHi__SHIFT                                                                  0x0
+#define TRAP5_ADDRESS_HI__Trap5AddrHi_MASK                                                                    0xFFFFFFFFL
+//TRAP5_COMMAND
+#define TRAP5_COMMAND__Trap5Cmd0__SHIFT                                                                       0x0
+#define TRAP5_COMMAND__Trap5Cmd1__SHIFT                                                                       0x8
+#define TRAP5_COMMAND__Trap5Cmd0_MASK                                                                         0x0000003FL
+#define TRAP5_COMMAND__Trap5Cmd1_MASK                                                                         0x00003F00L
+//TRAP5_ADDRESS_LO_MASK
+#define TRAP5_ADDRESS_LO_MASK__Trap5AddrLoMask__SHIFT                                                         0x2
+#define TRAP5_ADDRESS_LO_MASK__Trap5AddrLoMask_MASK                                                           0xFFFFFFFCL
+//TRAP5_ADDRESS_HI_MASK
+#define TRAP5_ADDRESS_HI_MASK__Trap5AddrHiMask__SHIFT                                                         0x0
+#define TRAP5_ADDRESS_HI_MASK__Trap5AddrHiMask_MASK                                                           0xFFFFFFFFL
+//TRAP5_COMMAND_MASK
+#define TRAP5_COMMAND_MASK__Trap5Cmd0Mask__SHIFT                                                              0x0
+#define TRAP5_COMMAND_MASK__Trap5Cmd1Mask__SHIFT                                                              0x8
+#define TRAP5_COMMAND_MASK__Trap5Cmd0Mask_MASK                                                                0x0000003FL
+#define TRAP5_COMMAND_MASK__Trap5Cmd1Mask_MASK                                                                0x00003F00L
+//TRAP6_CONTROL0
+#define TRAP6_CONTROL0__Trap6En__SHIFT                                                                        0x0
+#define TRAP6_CONTROL0__Trap6SMUIntr__SHIFT                                                                   0x3
+#define TRAP6_CONTROL0__Trap6Stage2Ptr__SHIFT                                                                 0xe
+#define TRAP6_CONTROL0__Trap6CrossTrigger__SHIFT                                                              0x18
+#define TRAP6_CONTROL0__Trap6Stage2En__SHIFT                                                                  0x1f
+#define TRAP6_CONTROL0__Trap6En_MASK                                                                          0x00000001L
+#define TRAP6_CONTROL0__Trap6SMUIntr_MASK                                                                     0x00000008L
+#define TRAP6_CONTROL0__Trap6Stage2Ptr_MASK                                                                   0x00FFC000L
+#define TRAP6_CONTROL0__Trap6CrossTrigger_MASK                                                                0x0F000000L
+#define TRAP6_CONTROL0__Trap6Stage2En_MASK                                                                    0x80000000L
+//TRAP6_ADDRESS_LO
+#define TRAP6_ADDRESS_LO__Trap6AddrLo__SHIFT                                                                  0x2
+#define TRAP6_ADDRESS_LO__Trap6AddrLo_MASK                                                                    0xFFFFFFFCL
+//TRAP6_ADDRESS_HI
+#define TRAP6_ADDRESS_HI__Trap6AddrHi__SHIFT                                                                  0x0
+#define TRAP6_ADDRESS_HI__Trap6AddrHi_MASK                                                                    0xFFFFFFFFL
+//TRAP6_COMMAND
+#define TRAP6_COMMAND__Trap6Cmd0__SHIFT                                                                       0x0
+#define TRAP6_COMMAND__Trap6Cmd1__SHIFT                                                                       0x8
+#define TRAP6_COMMAND__Trap6Cmd0_MASK                                                                         0x0000003FL
+#define TRAP6_COMMAND__Trap6Cmd1_MASK                                                                         0x00003F00L
+//TRAP6_ADDRESS_LO_MASK
+#define TRAP6_ADDRESS_LO_MASK__Trap6AddrLoMask__SHIFT                                                         0x2
+#define TRAP6_ADDRESS_LO_MASK__Trap6AddrLoMask_MASK                                                           0xFFFFFFFCL
+//TRAP6_ADDRESS_HI_MASK
+#define TRAP6_ADDRESS_HI_MASK__Trap6AddrHiMask__SHIFT                                                         0x0
+#define TRAP6_ADDRESS_HI_MASK__Trap6AddrHiMask_MASK                                                           0xFFFFFFFFL
+//TRAP6_COMMAND_MASK
+#define TRAP6_COMMAND_MASK__Trap6Cmd0Mask__SHIFT                                                              0x0
+#define TRAP6_COMMAND_MASK__Trap6Cmd1Mask__SHIFT                                                              0x8
+#define TRAP6_COMMAND_MASK__Trap6Cmd0Mask_MASK                                                                0x0000003FL
+#define TRAP6_COMMAND_MASK__Trap6Cmd1Mask_MASK                                                                0x00003F00L
+//TRAP7_CONTROL0
+#define TRAP7_CONTROL0__Trap7En__SHIFT                                                                        0x0
+#define TRAP7_CONTROL0__Trap7SMUIntr__SHIFT                                                                   0x3
+#define TRAP7_CONTROL0__Trap7Stage2Ptr__SHIFT                                                                 0xe
+#define TRAP7_CONTROL0__Trap7CrossTrigger__SHIFT                                                              0x18
+#define TRAP7_CONTROL0__Trap7Stage2En__SHIFT                                                                  0x1f
+#define TRAP7_CONTROL0__Trap7En_MASK                                                                          0x00000001L
+#define TRAP7_CONTROL0__Trap7SMUIntr_MASK                                                                     0x00000008L
+#define TRAP7_CONTROL0__Trap7Stage2Ptr_MASK                                                                   0x00FFC000L
+#define TRAP7_CONTROL0__Trap7CrossTrigger_MASK                                                                0x0F000000L
+#define TRAP7_CONTROL0__Trap7Stage2En_MASK                                                                    0x80000000L
+//TRAP7_ADDRESS_LO
+#define TRAP7_ADDRESS_LO__Trap7AddrLo__SHIFT                                                                  0x2
+#define TRAP7_ADDRESS_LO__Trap7AddrLo_MASK                                                                    0xFFFFFFFCL
+//TRAP7_ADDRESS_HI
+#define TRAP7_ADDRESS_HI__Trap7AddrHi__SHIFT                                                                  0x0
+#define TRAP7_ADDRESS_HI__Trap7AddrHi_MASK                                                                    0xFFFFFFFFL
+//TRAP7_COMMAND
+#define TRAP7_COMMAND__Trap7Cmd0__SHIFT                                                                       0x0
+#define TRAP7_COMMAND__Trap7Cmd1__SHIFT                                                                       0x8
+#define TRAP7_COMMAND__Trap7Cmd0_MASK                                                                         0x0000003FL
+#define TRAP7_COMMAND__Trap7Cmd1_MASK                                                                         0x00003F00L
+//TRAP7_ADDRESS_LO_MASK
+#define TRAP7_ADDRESS_LO_MASK__Trap7AddrLoMask__SHIFT                                                         0x2
+#define TRAP7_ADDRESS_LO_MASK__Trap7AddrLoMask_MASK                                                           0xFFFFFFFCL
+//TRAP7_ADDRESS_HI_MASK
+#define TRAP7_ADDRESS_HI_MASK__Trap7AddrHiMask__SHIFT                                                         0x0
+#define TRAP7_ADDRESS_HI_MASK__Trap7AddrHiMask_MASK                                                           0xFFFFFFFFL
+//TRAP7_COMMAND_MASK
+#define TRAP7_COMMAND_MASK__Trap7Cmd0Mask__SHIFT                                                              0x0
+#define TRAP7_COMMAND_MASK__Trap7Cmd1Mask__SHIFT                                                              0x8
+#define TRAP7_COMMAND_MASK__Trap7Cmd0Mask_MASK                                                                0x0000003FL
+#define TRAP7_COMMAND_MASK__Trap7Cmd1Mask_MASK                                                                0x00003F00L
+//TRAP8_CONTROL0
+#define TRAP8_CONTROL0__Trap8En__SHIFT                                                                        0x0
+#define TRAP8_CONTROL0__Trap8SMUIntr__SHIFT                                                                   0x3
+#define TRAP8_CONTROL0__Trap8Stage2Ptr__SHIFT                                                                 0xe
+#define TRAP8_CONTROL0__Trap8CrossTrigger__SHIFT                                                              0x18
+#define TRAP8_CONTROL0__Trap8Stage2En__SHIFT                                                                  0x1f
+#define TRAP8_CONTROL0__Trap8En_MASK                                                                          0x00000001L
+#define TRAP8_CONTROL0__Trap8SMUIntr_MASK                                                                     0x00000008L
+#define TRAP8_CONTROL0__Trap8Stage2Ptr_MASK                                                                   0x00FFC000L
+#define TRAP8_CONTROL0__Trap8CrossTrigger_MASK                                                                0x0F000000L
+#define TRAP8_CONTROL0__Trap8Stage2En_MASK                                                                    0x80000000L
+//TRAP8_ADDRESS_LO
+#define TRAP8_ADDRESS_LO__Trap8AddrLo__SHIFT                                                                  0x2
+#define TRAP8_ADDRESS_LO__Trap8AddrLo_MASK                                                                    0xFFFFFFFCL
+//TRAP8_ADDRESS_HI
+#define TRAP8_ADDRESS_HI__Trap8AddrHi__SHIFT                                                                  0x0
+#define TRAP8_ADDRESS_HI__Trap8AddrHi_MASK                                                                    0xFFFFFFFFL
+//TRAP8_COMMAND
+#define TRAP8_COMMAND__Trap8Cmd0__SHIFT                                                                       0x0
+#define TRAP8_COMMAND__Trap8Cmd1__SHIFT                                                                       0x8
+#define TRAP8_COMMAND__Trap8Cmd0_MASK                                                                         0x0000003FL
+#define TRAP8_COMMAND__Trap8Cmd1_MASK                                                                         0x00003F00L
+//TRAP8_ADDRESS_LO_MASK
+#define TRAP8_ADDRESS_LO_MASK__Trap8AddrLoMask__SHIFT                                                         0x2
+#define TRAP8_ADDRESS_LO_MASK__Trap8AddrLoMask_MASK                                                           0xFFFFFFFCL
+//TRAP8_ADDRESS_HI_MASK
+#define TRAP8_ADDRESS_HI_MASK__Trap8AddrHiMask__SHIFT                                                         0x0
+#define TRAP8_ADDRESS_HI_MASK__Trap8AddrHiMask_MASK                                                           0xFFFFFFFFL
+//TRAP8_COMMAND_MASK
+#define TRAP8_COMMAND_MASK__Trap8Cmd0Mask__SHIFT                                                              0x0
+#define TRAP8_COMMAND_MASK__Trap8Cmd1Mask__SHIFT                                                              0x8
+#define TRAP8_COMMAND_MASK__Trap8Cmd0Mask_MASK                                                                0x0000003FL
+#define TRAP8_COMMAND_MASK__Trap8Cmd1Mask_MASK                                                                0x00003F00L
+//TRAP9_CONTROL0
+#define TRAP9_CONTROL0__Trap9En__SHIFT                                                                        0x0
+#define TRAP9_CONTROL0__Trap9SMUIntr__SHIFT                                                                   0x3
+#define TRAP9_CONTROL0__Trap9Stage2Ptr__SHIFT                                                                 0xe
+#define TRAP9_CONTROL0__Trap9CrossTrigger__SHIFT                                                              0x18
+#define TRAP9_CONTROL0__Trap9Stage2En__SHIFT                                                                  0x1f
+#define TRAP9_CONTROL0__Trap9En_MASK                                                                          0x00000001L
+#define TRAP9_CONTROL0__Trap9SMUIntr_MASK                                                                     0x00000008L
+#define TRAP9_CONTROL0__Trap9Stage2Ptr_MASK                                                                   0x00FFC000L
+#define TRAP9_CONTROL0__Trap9CrossTrigger_MASK                                                                0x0F000000L
+#define TRAP9_CONTROL0__Trap9Stage2En_MASK                                                                    0x80000000L
+//TRAP9_ADDRESS_LO
+#define TRAP9_ADDRESS_LO__Trap9AddrLo__SHIFT                                                                  0x2
+#define TRAP9_ADDRESS_LO__Trap9AddrLo_MASK                                                                    0xFFFFFFFCL
+//TRAP9_ADDRESS_HI
+#define TRAP9_ADDRESS_HI__Trap9AddrHi__SHIFT                                                                  0x0
+#define TRAP9_ADDRESS_HI__Trap9AddrHi_MASK                                                                    0xFFFFFFFFL
+//TRAP9_COMMAND
+#define TRAP9_COMMAND__Trap9Cmd0__SHIFT                                                                       0x0
+#define TRAP9_COMMAND__Trap9Cmd1__SHIFT                                                                       0x8
+#define TRAP9_COMMAND__Trap9Cmd0_MASK                                                                         0x0000003FL
+#define TRAP9_COMMAND__Trap9Cmd1_MASK                                                                         0x00003F00L
+//TRAP9_ADDRESS_LO_MASK
+#define TRAP9_ADDRESS_LO_MASK__Trap9AddrLoMask__SHIFT                                                         0x2
+#define TRAP9_ADDRESS_LO_MASK__Trap9AddrLoMask_MASK                                                           0xFFFFFFFCL
+//TRAP9_ADDRESS_HI_MASK
+#define TRAP9_ADDRESS_HI_MASK__Trap9AddrHiMask__SHIFT                                                         0x0
+#define TRAP9_ADDRESS_HI_MASK__Trap9AddrHiMask_MASK                                                           0xFFFFFFFFL
+//TRAP9_COMMAND_MASK
+#define TRAP9_COMMAND_MASK__Trap9Cmd0Mask__SHIFT                                                              0x0
+#define TRAP9_COMMAND_MASK__Trap9Cmd1Mask__SHIFT                                                              0x8
+#define TRAP9_COMMAND_MASK__Trap9Cmd0Mask_MASK                                                                0x0000003FL
+#define TRAP9_COMMAND_MASK__Trap9Cmd1Mask_MASK                                                                0x00003F00L
+//TRAP10_CONTROL0
+#define TRAP10_CONTROL0__Trap10En__SHIFT                                                                      0x0
+#define TRAP10_CONTROL0__Trap10SMUIntr__SHIFT                                                                 0x3
+#define TRAP10_CONTROL0__Trap10Stage2Ptr__SHIFT                                                               0xe
+#define TRAP10_CONTROL0__Trap10CrossTrigger__SHIFT                                                            0x18
+#define TRAP10_CONTROL0__Trap10Stage2En__SHIFT                                                                0x1f
+#define TRAP10_CONTROL0__Trap10En_MASK                                                                        0x00000001L
+#define TRAP10_CONTROL0__Trap10SMUIntr_MASK                                                                   0x00000008L
+#define TRAP10_CONTROL0__Trap10Stage2Ptr_MASK                                                                 0x00FFC000L
+#define TRAP10_CONTROL0__Trap10CrossTrigger_MASK                                                              0x0F000000L
+#define TRAP10_CONTROL0__Trap10Stage2En_MASK                                                                  0x80000000L
+//TRAP10_ADDRESS_LO
+#define TRAP10_ADDRESS_LO__Trap10AddrLo__SHIFT                                                                0x2
+#define TRAP10_ADDRESS_LO__Trap10AddrLo_MASK                                                                  0xFFFFFFFCL
+//TRAP10_ADDRESS_HI
+#define TRAP10_ADDRESS_HI__Trap10AddrHi__SHIFT                                                                0x0
+#define TRAP10_ADDRESS_HI__Trap10AddrHi_MASK                                                                  0xFFFFFFFFL
+//TRAP10_COMMAND
+#define TRAP10_COMMAND__Trap10Cmd0__SHIFT                                                                     0x0
+#define TRAP10_COMMAND__Trap10Cmd1__SHIFT                                                                     0x8
+#define TRAP10_COMMAND__Trap10Cmd0_MASK                                                                       0x0000003FL
+#define TRAP10_COMMAND__Trap10Cmd1_MASK                                                                       0x00003F00L
+//TRAP10_ADDRESS_LO_MASK
+#define TRAP10_ADDRESS_LO_MASK__Trap10AddrLoMask__SHIFT                                                       0x2
+#define TRAP10_ADDRESS_LO_MASK__Trap10AddrLoMask_MASK                                                         0xFFFFFFFCL
+//TRAP10_ADDRESS_HI_MASK
+#define TRAP10_ADDRESS_HI_MASK__Trap10AddrHiMask__SHIFT                                                       0x0
+#define TRAP10_ADDRESS_HI_MASK__Trap10AddrHiMask_MASK                                                         0xFFFFFFFFL
+//TRAP10_COMMAND_MASK
+#define TRAP10_COMMAND_MASK__Trap10Cmd0Mask__SHIFT                                                            0x0
+#define TRAP10_COMMAND_MASK__Trap10Cmd1Mask__SHIFT                                                            0x8
+#define TRAP10_COMMAND_MASK__Trap10Cmd0Mask_MASK                                                              0x0000003FL
+#define TRAP10_COMMAND_MASK__Trap10Cmd1Mask_MASK                                                              0x00003F00L
+//TRAP11_CONTROL0
+#define TRAP11_CONTROL0__Trap11En__SHIFT                                                                      0x0
+#define TRAP11_CONTROL0__Trap11SMUIntr__SHIFT                                                                 0x3
+#define TRAP11_CONTROL0__Trap11Stage2Ptr__SHIFT                                                               0xe
+#define TRAP11_CONTROL0__Trap11CrossTrigger__SHIFT                                                            0x18
+#define TRAP11_CONTROL0__Trap11Stage2En__SHIFT                                                                0x1f
+#define TRAP11_CONTROL0__Trap11En_MASK                                                                        0x00000001L
+#define TRAP11_CONTROL0__Trap11SMUIntr_MASK                                                                   0x00000008L
+#define TRAP11_CONTROL0__Trap11Stage2Ptr_MASK                                                                 0x00FFC000L
+#define TRAP11_CONTROL0__Trap11CrossTrigger_MASK                                                              0x0F000000L
+#define TRAP11_CONTROL0__Trap11Stage2En_MASK                                                                  0x80000000L
+//TRAP11_ADDRESS_LO
+#define TRAP11_ADDRESS_LO__Trap11AddrLo__SHIFT                                                                0x2
+#define TRAP11_ADDRESS_LO__Trap11AddrLo_MASK                                                                  0xFFFFFFFCL
+//TRAP11_ADDRESS_HI
+#define TRAP11_ADDRESS_HI__Trap11AddrHi__SHIFT                                                                0x0
+#define TRAP11_ADDRESS_HI__Trap11AddrHi_MASK                                                                  0xFFFFFFFFL
+//TRAP11_COMMAND
+#define TRAP11_COMMAND__Trap11Cmd0__SHIFT                                                                     0x0
+#define TRAP11_COMMAND__Trap11Cmd1__SHIFT                                                                     0x8
+#define TRAP11_COMMAND__Trap11Cmd0_MASK                                                                       0x0000003FL
+#define TRAP11_COMMAND__Trap11Cmd1_MASK                                                                       0x00003F00L
+//TRAP11_ADDRESS_LO_MASK
+#define TRAP11_ADDRESS_LO_MASK__Trap11AddrLoMask__SHIFT                                                       0x2
+#define TRAP11_ADDRESS_LO_MASK__Trap11AddrLoMask_MASK                                                         0xFFFFFFFCL
+//TRAP11_ADDRESS_HI_MASK
+#define TRAP11_ADDRESS_HI_MASK__Trap11AddrHiMask__SHIFT                                                       0x0
+#define TRAP11_ADDRESS_HI_MASK__Trap11AddrHiMask_MASK                                                         0xFFFFFFFFL
+//TRAP11_COMMAND_MASK
+#define TRAP11_COMMAND_MASK__Trap11Cmd0Mask__SHIFT                                                            0x0
+#define TRAP11_COMMAND_MASK__Trap11Cmd1Mask__SHIFT                                                            0x8
+#define TRAP11_COMMAND_MASK__Trap11Cmd0Mask_MASK                                                              0x0000003FL
+#define TRAP11_COMMAND_MASK__Trap11Cmd1Mask_MASK                                                              0x00003F00L
+//TRAP12_CONTROL0
+#define TRAP12_CONTROL0__Trap12En__SHIFT                                                                      0x0
+#define TRAP12_CONTROL0__Trap12SMUIntr__SHIFT                                                                 0x3
+#define TRAP12_CONTROL0__Trap12Stage2Ptr__SHIFT                                                               0xe
+#define TRAP12_CONTROL0__Trap12CrossTrigger__SHIFT                                                            0x18
+#define TRAP12_CONTROL0__Trap12Stage2En__SHIFT                                                                0x1f
+#define TRAP12_CONTROL0__Trap12En_MASK                                                                        0x00000001L
+#define TRAP12_CONTROL0__Trap12SMUIntr_MASK                                                                   0x00000008L
+#define TRAP12_CONTROL0__Trap12Stage2Ptr_MASK                                                                 0x00FFC000L
+#define TRAP12_CONTROL0__Trap12CrossTrigger_MASK                                                              0x0F000000L
+#define TRAP12_CONTROL0__Trap12Stage2En_MASK                                                                  0x80000000L
+//TRAP12_ADDRESS_LO
+#define TRAP12_ADDRESS_LO__Trap12AddrLo__SHIFT                                                                0x2
+#define TRAP12_ADDRESS_LO__Trap12AddrLo_MASK                                                                  0xFFFFFFFCL
+//TRAP12_ADDRESS_HI
+#define TRAP12_ADDRESS_HI__Trap12AddrHi__SHIFT                                                                0x0
+#define TRAP12_ADDRESS_HI__Trap12AddrHi_MASK                                                                  0xFFFFFFFFL
+//TRAP12_COMMAND
+#define TRAP12_COMMAND__Trap12Cmd0__SHIFT                                                                     0x0
+#define TRAP12_COMMAND__Trap12Cmd1__SHIFT                                                                     0x8
+#define TRAP12_COMMAND__Trap12Cmd0_MASK                                                                       0x0000003FL
+#define TRAP12_COMMAND__Trap12Cmd1_MASK                                                                       0x00003F00L
+//TRAP12_ADDRESS_LO_MASK
+#define TRAP12_ADDRESS_LO_MASK__Trap12AddrLoMask__SHIFT                                                       0x2
+#define TRAP12_ADDRESS_LO_MASK__Trap12AddrLoMask_MASK                                                         0xFFFFFFFCL
+//TRAP12_ADDRESS_HI_MASK
+#define TRAP12_ADDRESS_HI_MASK__Trap12AddrHiMask__SHIFT                                                       0x0
+#define TRAP12_ADDRESS_HI_MASK__Trap12AddrHiMask_MASK                                                         0xFFFFFFFFL
+//TRAP12_COMMAND_MASK
+#define TRAP12_COMMAND_MASK__Trap12Cmd0Mask__SHIFT                                                            0x0
+#define TRAP12_COMMAND_MASK__Trap12Cmd1Mask__SHIFT                                                            0x8
+#define TRAP12_COMMAND_MASK__Trap12Cmd0Mask_MASK                                                              0x0000003FL
+#define TRAP12_COMMAND_MASK__Trap12Cmd1Mask_MASK                                                              0x00003F00L
+//TRAP13_CONTROL0
+#define TRAP13_CONTROL0__Trap13En__SHIFT                                                                      0x0
+#define TRAP13_CONTROL0__Trap13SMUIntr__SHIFT                                                                 0x3
+#define TRAP13_CONTROL0__Trap13Stage2Ptr__SHIFT                                                               0xe
+#define TRAP13_CONTROL0__Trap13CrossTrigger__SHIFT                                                            0x18
+#define TRAP13_CONTROL0__Trap13Stage2En__SHIFT                                                                0x1f
+#define TRAP13_CONTROL0__Trap13En_MASK                                                                        0x00000001L
+#define TRAP13_CONTROL0__Trap13SMUIntr_MASK                                                                   0x00000008L
+#define TRAP13_CONTROL0__Trap13Stage2Ptr_MASK                                                                 0x00FFC000L
+#define TRAP13_CONTROL0__Trap13CrossTrigger_MASK                                                              0x0F000000L
+#define TRAP13_CONTROL0__Trap13Stage2En_MASK                                                                  0x80000000L
+//TRAP13_ADDRESS_LO
+#define TRAP13_ADDRESS_LO__Trap13AddrLo__SHIFT                                                                0x2
+#define TRAP13_ADDRESS_LO__Trap13AddrLo_MASK                                                                  0xFFFFFFFCL
+//TRAP13_ADDRESS_HI
+#define TRAP13_ADDRESS_HI__Trap13AddrHi__SHIFT                                                                0x0
+#define TRAP13_ADDRESS_HI__Trap13AddrHi_MASK                                                                  0xFFFFFFFFL
+//TRAP13_COMMAND
+#define TRAP13_COMMAND__Trap13Cmd0__SHIFT                                                                     0x0
+#define TRAP13_COMMAND__Trap13Cmd1__SHIFT                                                                     0x8
+#define TRAP13_COMMAND__Trap13Cmd0_MASK                                                                       0x0000003FL
+#define TRAP13_COMMAND__Trap13Cmd1_MASK                                                                       0x00003F00L
+//TRAP13_ADDRESS_LO_MASK
+#define TRAP13_ADDRESS_LO_MASK__Trap13AddrLoMask__SHIFT                                                       0x2
+#define TRAP13_ADDRESS_LO_MASK__Trap13AddrLoMask_MASK                                                         0xFFFFFFFCL
+//TRAP13_ADDRESS_HI_MASK
+#define TRAP13_ADDRESS_HI_MASK__Trap13AddrHiMask__SHIFT                                                       0x0
+#define TRAP13_ADDRESS_HI_MASK__Trap13AddrHiMask_MASK                                                         0xFFFFFFFFL
+//TRAP13_COMMAND_MASK
+#define TRAP13_COMMAND_MASK__Trap13Cmd0Mask__SHIFT                                                            0x0
+#define TRAP13_COMMAND_MASK__Trap13Cmd1Mask__SHIFT                                                            0x8
+#define TRAP13_COMMAND_MASK__Trap13Cmd0Mask_MASK                                                              0x0000003FL
+#define TRAP13_COMMAND_MASK__Trap13Cmd1Mask_MASK                                                              0x00003F00L
+//TRAP14_CONTROL0
+#define TRAP14_CONTROL0__Trap14En__SHIFT                                                                      0x0
+#define TRAP14_CONTROL0__Trap14SMUIntr__SHIFT                                                                 0x3
+#define TRAP14_CONTROL0__Trap14Stage2Ptr__SHIFT                                                               0xe
+#define TRAP14_CONTROL0__Trap14CrossTrigger__SHIFT                                                            0x18
+#define TRAP14_CONTROL0__Trap14Stage2En__SHIFT                                                                0x1f
+#define TRAP14_CONTROL0__Trap14En_MASK                                                                        0x00000001L
+#define TRAP14_CONTROL0__Trap14SMUIntr_MASK                                                                   0x00000008L
+#define TRAP14_CONTROL0__Trap14Stage2Ptr_MASK                                                                 0x00FFC000L
+#define TRAP14_CONTROL0__Trap14CrossTrigger_MASK                                                              0x0F000000L
+#define TRAP14_CONTROL0__Trap14Stage2En_MASK                                                                  0x80000000L
+//TRAP14_ADDRESS_LO
+#define TRAP14_ADDRESS_LO__Trap14AddrLo__SHIFT                                                                0x2
+#define TRAP14_ADDRESS_LO__Trap14AddrLo_MASK                                                                  0xFFFFFFFCL
+//TRAP14_ADDRESS_HI
+#define TRAP14_ADDRESS_HI__Trap14AddrHi__SHIFT                                                                0x0
+#define TRAP14_ADDRESS_HI__Trap14AddrHi_MASK                                                                  0xFFFFFFFFL
+//TRAP14_COMMAND
+#define TRAP14_COMMAND__Trap14Cmd0__SHIFT                                                                     0x0
+#define TRAP14_COMMAND__Trap14Cmd1__SHIFT                                                                     0x8
+#define TRAP14_COMMAND__Trap14Cmd0_MASK                                                                       0x0000003FL
+#define TRAP14_COMMAND__Trap14Cmd1_MASK                                                                       0x00003F00L
+//TRAP14_ADDRESS_LO_MASK
+#define TRAP14_ADDRESS_LO_MASK__Trap14AddrLoMask__SHIFT                                                       0x2
+#define TRAP14_ADDRESS_LO_MASK__Trap14AddrLoMask_MASK                                                         0xFFFFFFFCL
+//TRAP14_ADDRESS_HI_MASK
+#define TRAP14_ADDRESS_HI_MASK__Trap14AddrHiMask__SHIFT                                                       0x0
+#define TRAP14_ADDRESS_HI_MASK__Trap14AddrHiMask_MASK                                                         0xFFFFFFFFL
+//TRAP14_COMMAND_MASK
+#define TRAP14_COMMAND_MASK__Trap14Cmd0Mask__SHIFT                                                            0x0
+#define TRAP14_COMMAND_MASK__Trap14Cmd1Mask__SHIFT                                                            0x8
+#define TRAP14_COMMAND_MASK__Trap14Cmd0Mask_MASK                                                              0x0000003FL
+#define TRAP14_COMMAND_MASK__Trap14Cmd1Mask_MASK                                                              0x00003F00L
+//TRAP15_CONTROL0
+#define TRAP15_CONTROL0__Trap15En__SHIFT                                                                      0x0
+#define TRAP15_CONTROL0__Trap15SMUIntr__SHIFT                                                                 0x3
+#define TRAP15_CONTROL0__Trap15Stage2Ptr__SHIFT                                                               0xe
+#define TRAP15_CONTROL0__Trap15CrossTrigger__SHIFT                                                            0x18
+#define TRAP15_CONTROL0__Trap15Stage2En__SHIFT                                                                0x1f
+#define TRAP15_CONTROL0__Trap15En_MASK                                                                        0x00000001L
+#define TRAP15_CONTROL0__Trap15SMUIntr_MASK                                                                   0x00000008L
+#define TRAP15_CONTROL0__Trap15Stage2Ptr_MASK                                                                 0x00FFC000L
+#define TRAP15_CONTROL0__Trap15CrossTrigger_MASK                                                              0x0F000000L
+#define TRAP15_CONTROL0__Trap15Stage2En_MASK                                                                  0x80000000L
+//TRAP15_ADDRESS_LO
+#define TRAP15_ADDRESS_LO__Trap15AddrLo__SHIFT                                                                0x2
+#define TRAP15_ADDRESS_LO__Trap15AddrLo_MASK                                                                  0xFFFFFFFCL
+//TRAP15_ADDRESS_HI
+#define TRAP15_ADDRESS_HI__Trap15AddrHi__SHIFT                                                                0x0
+#define TRAP15_ADDRESS_HI__Trap15AddrHi_MASK                                                                  0xFFFFFFFFL
+//TRAP15_COMMAND
+#define TRAP15_COMMAND__Trap15Cmd0__SHIFT                                                                     0x0
+#define TRAP15_COMMAND__Trap15Cmd1__SHIFT                                                                     0x8
+#define TRAP15_COMMAND__Trap15Cmd0_MASK                                                                       0x0000003FL
+#define TRAP15_COMMAND__Trap15Cmd1_MASK                                                                       0x00003F00L
+//TRAP15_ADDRESS_LO_MASK
+#define TRAP15_ADDRESS_LO_MASK__Trap15AddrLoMask__SHIFT                                                       0x2
+#define TRAP15_ADDRESS_LO_MASK__Trap15AddrLoMask_MASK                                                         0xFFFFFFFCL
+//TRAP15_ADDRESS_HI_MASK
+#define TRAP15_ADDRESS_HI_MASK__Trap15AddrHiMask__SHIFT                                                       0x0
+#define TRAP15_ADDRESS_HI_MASK__Trap15AddrHiMask_MASK                                                         0xFFFFFFFFL
+//TRAP15_COMMAND_MASK
+#define TRAP15_COMMAND_MASK__Trap15Cmd0Mask__SHIFT                                                            0x0
+#define TRAP15_COMMAND_MASK__Trap15Cmd1Mask__SHIFT                                                            0x8
+#define TRAP15_COMMAND_MASK__Trap15Cmd0Mask_MASK                                                              0x0000003FL
+#define TRAP15_COMMAND_MASK__Trap15Cmd1Mask_MASK                                                              0x00003F00L
+//SB_COMMAND
+#define SB_COMMAND__IO_ACCESS_EN__SHIFT                                                                       0x0
+#define SB_COMMAND__MEM_ACCESS_EN__SHIFT                                                                      0x1
+#define SB_COMMAND__BUS_MASTER_EN__SHIFT                                                                      0x2
+#define SB_COMMAND__IO_ACCESS_EN_MASK                                                                         0x0001L
+#define SB_COMMAND__MEM_ACCESS_EN_MASK                                                                        0x0002L
+#define SB_COMMAND__BUS_MASTER_EN_MASK                                                                        0x0004L
+//SB_SUB_BUS_NUMBER_LATENCY
+#define SB_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                                       0x8
+#define SB_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                                         0x10
+#define SB_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                                         0x0000FF00L
+#define SB_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                                           0x00FF0000L
+//SB_IO_BASE_LIMIT
+#define SB_IO_BASE_LIMIT__IO_BASE__SHIFT                                                                      0x4
+#define SB_IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                                     0xc
+#define SB_IO_BASE_LIMIT__IO_BASE_MASK                                                                        0x00F0L
+#define SB_IO_BASE_LIMIT__IO_LIMIT_MASK                                                                       0xF000L
+//SB_MEM_BASE_LIMIT
+#define SB_MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                              0x4
+#define SB_MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                                             0x14
+#define SB_MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                                0x0000FFF0L
+#define SB_MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                               0xFFF00000L
+//SB_PREF_BASE_LIMIT
+#define SB_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                                        0x4
+#define SB_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                                       0x14
+#define SB_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                                          0x0000FFF0L
+#define SB_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                                         0xFFF00000L
+//SB_PREF_BASE_UPPER
+#define SB_PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                                            0x0
+#define SB_PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                              0xFFFFFFFFL
+//SB_PREF_LIMIT_UPPER
+#define SB_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                                          0x0
+#define SB_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                                            0xFFFFFFFFL
+//SB_IO_BASE_LIMIT_HI
+#define SB_IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                                             0x0
+#define SB_IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                                            0x10
+#define SB_IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                               0x0000FFFFL
+#define SB_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                              0xFFFF0000L
+//SB_IRQ_BRIDGE_CNTL
+#define SB_IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                                     0x2
+#define SB_IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                                     0x3
+#define SB_IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                                    0x4
+#define SB_IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                                       0x0004L
+#define SB_IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                                       0x0008L
+#define SB_IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                                      0x0010L
+//SB_EXT_BRIDGE_CNTL
+#define SB_EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                              0x0
+#define SB_EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                                0x01L
+//SB_PMI_STATUS_CNTL
+#define SB_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                                0x0
+#define SB_PMI_STATUS_CNTL__POWER_STATE_MASK                                                                  0x03L
+//SB_SLOT_CAP
+#define SB_SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                              0x7
+#define SB_SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                              0xf
+#define SB_SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                                0x00007F80L
+#define SB_SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                                0x00018000L
+//SB_ROOT_CNTL
+#define SB_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                                       0x4
+#define SB_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                                         0x0010L
+//SB_DEVICE_CNTL2
+#define SB_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                                             0x5
+#define SB_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                               0x0020L
+//MCA_SMN_INT_REQ_ADDR
+#define MCA_SMN_INT_REQ_ADDR__SMN_INT_REQ_ADDR__SHIFT                                                         0x0
+#define MCA_SMN_INT_REQ_ADDR__SMN_INT_REQ_ADDR_MASK                                                           0x000FFFFFL
+//MCA_SMN_INT_MCM_ADDR
+#define MCA_SMN_INT_MCM_ADDR__SMN_INT_MCM_ADDR__SHIFT                                                         0x0
+#define MCA_SMN_INT_MCM_ADDR__SMN_INT_MCM_ADDR_MASK                                                           0x000000FFL
+//MCA_SMN_INT_APERTUREID
+#define MCA_SMN_INT_APERTUREID__SMN_INT_APERTUREID__SHIFT                                                     0x0
+#define MCA_SMN_INT_APERTUREID__SMN_INT_APERTUREID_MASK                                                       0x00000FFFL
+//MCA_SMN_INT_CONTROL
+#define MCA_SMN_INT_CONTROL__MCACrossTrigger__SHIFT                                                           0x0
+#define MCA_SMN_INT_CONTROL__MCACrossTrigger_MASK                                                             0x0000000FL
+
+
+// addressBlock: aid_nbio_iohub_nb_rascfg_ras_cfgdec
+//PARITY_CONTROL_0
+#define PARITY_CONTROL_0__ParityCorrThreshold__SHIFT                                                          0x0
+#define PARITY_CONTROL_0__ParityUCPThreshold__SHIFT                                                           0x10
+#define PARITY_CONTROL_0__ParityCorrThreshold_MASK                                                            0x0000FFFFL
+#define PARITY_CONTROL_0__ParityUCPThreshold_MASK                                                             0xFFFF0000L
+//PARITY_CONTROL_1
+#define PARITY_CONTROL_1__ParityErrGenGroupSel__SHIFT                                                         0x0
+#define PARITY_CONTROL_1__ParityErrGenGroupTypeSel__SHIFT                                                     0x8
+#define PARITY_CONTROL_1__ParityErrGenIdSel__SHIFT                                                            0xb
+#define PARITY_CONTROL_1__ParityErrGenCmd__SHIFT                                                              0x10
+#define PARITY_CONTROL_1__ParityErrGenTrigger__SHIFT                                                          0x1e
+#define PARITY_CONTROL_1__ParityErrGenInjectAllow__SHIFT                                                      0x1f
+#define PARITY_CONTROL_1__ParityErrGenGroupSel_MASK                                                           0x000000FFL
+#define PARITY_CONTROL_1__ParityErrGenGroupTypeSel_MASK                                                       0x00000100L
+#define PARITY_CONTROL_1__ParityErrGenIdSel_MASK                                                              0x0000F800L
+#define PARITY_CONTROL_1__ParityErrGenCmd_MASK                                                                0x000F0000L
+#define PARITY_CONTROL_1__ParityErrGenTrigger_MASK                                                            0x40000000L
+#define PARITY_CONTROL_1__ParityErrGenInjectAllow_MASK                                                        0x80000000L
+//PARITY_SEVERITY_CONTROL_UNCORR_0
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp0__SHIFT                                       0x0
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp1__SHIFT                                       0x2
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp2__SHIFT                                       0x4
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp3__SHIFT                                       0x6
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp4__SHIFT                                       0x8
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp5__SHIFT                                       0xa
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp6__SHIFT                                       0xc
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp7__SHIFT                                       0xe
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp8__SHIFT                                       0x10
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp9__SHIFT                                       0x12
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp10__SHIFT                                      0x14
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp11__SHIFT                                      0x16
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp12__SHIFT                                      0x18
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp13__SHIFT                                      0x1a
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp14__SHIFT                                      0x1c
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp15__SHIFT                                      0x1e
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp0_MASK                                         0x00000003L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp1_MASK                                         0x0000000CL
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp2_MASK                                         0x00000030L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp3_MASK                                         0x000000C0L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp4_MASK                                         0x00000300L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp5_MASK                                         0x00000C00L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp6_MASK                                         0x00003000L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp7_MASK                                         0x0000C000L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp8_MASK                                         0x00030000L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp9_MASK                                         0x000C0000L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp10_MASK                                        0x00300000L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp11_MASK                                        0x00C00000L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp12_MASK                                        0x03000000L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp13_MASK                                        0x0C000000L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp14_MASK                                        0x30000000L
+#define PARITY_SEVERITY_CONTROL_UNCORR_0__ParityErrSevUnCorrGrp15_MASK                                        0xC0000000L
+//PARITY_SEVERITY_CONTROL_CORR_0
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp0__SHIFT                                           0x0
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp1__SHIFT                                           0x2
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp2__SHIFT                                           0x4
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp3__SHIFT                                           0x6
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp4__SHIFT                                           0x8
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp5__SHIFT                                           0xa
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp6__SHIFT                                           0xc
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp7__SHIFT                                           0xe
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp8__SHIFT                                           0x10
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp9__SHIFT                                           0x12
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp10__SHIFT                                          0x14
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp11__SHIFT                                          0x16
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp12__SHIFT                                          0x18
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp13__SHIFT                                          0x1a
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp14__SHIFT                                          0x1c
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp15__SHIFT                                          0x1e
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp0_MASK                                             0x00000003L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp1_MASK                                             0x0000000CL
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp2_MASK                                             0x00000030L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp3_MASK                                             0x000000C0L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp4_MASK                                             0x00000300L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp5_MASK                                             0x00000C00L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp6_MASK                                             0x00003000L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp7_MASK                                             0x0000C000L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp8_MASK                                             0x00030000L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp9_MASK                                             0x000C0000L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp10_MASK                                            0x00300000L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp11_MASK                                            0x00C00000L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp12_MASK                                            0x03000000L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp13_MASK                                            0x0C000000L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp14_MASK                                            0x30000000L
+#define PARITY_SEVERITY_CONTROL_CORR_0__ParityErrSevCorrGrp15_MASK                                            0xC0000000L
+//PARITY_SEVERITY_CONTROL_UCP_0
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp0__SHIFT                                             0x0
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp1__SHIFT                                             0x2
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp2__SHIFT                                             0x4
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp3__SHIFT                                             0x6
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp4__SHIFT                                             0x8
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp5__SHIFT                                             0xa
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp6__SHIFT                                             0xc
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp7__SHIFT                                             0xe
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp8__SHIFT                                             0x10
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp9__SHIFT                                             0x12
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp10__SHIFT                                            0x14
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp11__SHIFT                                            0x16
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp12__SHIFT                                            0x18
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp0_MASK                                               0x00000003L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp1_MASK                                               0x0000000CL
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp2_MASK                                               0x00000030L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp3_MASK                                               0x000000C0L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp4_MASK                                               0x00000300L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp5_MASK                                               0x00000C00L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp6_MASK                                               0x00003000L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp7_MASK                                               0x0000C000L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp8_MASK                                               0x00030000L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp9_MASK                                               0x000C0000L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp10_MASK                                              0x00300000L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp11_MASK                                              0x00C00000L
+#define PARITY_SEVERITY_CONTROL_UCP_0__ParityErrSevUCPGrp12_MASK                                              0x03000000L
+//RAS_GLOBAL_STATUS_LO
+#define RAS_GLOBAL_STATUS_LO__ParityErrCorr__SHIFT                                                            0x0
+#define RAS_GLOBAL_STATUS_LO__ParityErrNonFatal__SHIFT                                                        0x1
+#define RAS_GLOBAL_STATUS_LO__ParityErrFatal__SHIFT                                                           0x2
+#define RAS_GLOBAL_STATUS_LO__ParityErrSerr__SHIFT                                                            0x3
+#define RAS_GLOBAL_STATUS_LO__HPLGWA_NMI__SHIFT                                                               0x6
+#define RAS_GLOBAL_STATUS_LO__HPLGWA_SCI__SHIFT                                                               0x7
+#define RAS_GLOBAL_STATUS_LO__HPLGWA_SMI__SHIFT                                                               0x8
+#define RAS_GLOBAL_STATUS_LO__SW_SMI__SHIFT                                                                   0x9
+#define RAS_GLOBAL_STATUS_LO__SW_SCI__SHIFT                                                                   0xa
+#define RAS_GLOBAL_STATUS_LO__SW_NMI__SHIFT                                                                   0xb
+#define RAS_GLOBAL_STATUS_LO__APML_NMI__SHIFT                                                                 0xc
+#define RAS_GLOBAL_STATUS_LO__APML_SyncFld__SHIFT                                                             0xd
+#define RAS_GLOBAL_STATUS_LO__PIN_SyncFld_NMI__SHIFT                                                          0xe
+#define RAS_GLOBAL_STATUS_LO__APML_SyncFld_Private__SHIFT                                                     0xf
+#define RAS_GLOBAL_STATUS_LO__ParityErrCorr_MASK                                                              0x00000001L
+#define RAS_GLOBAL_STATUS_LO__ParityErrNonFatal_MASK                                                          0x00000002L
+#define RAS_GLOBAL_STATUS_LO__ParityErrFatal_MASK                                                             0x00000004L
+#define RAS_GLOBAL_STATUS_LO__ParityErrSerr_MASK                                                              0x00000008L
+#define RAS_GLOBAL_STATUS_LO__HPLGWA_NMI_MASK                                                                 0x00000040L
+#define RAS_GLOBAL_STATUS_LO__HPLGWA_SCI_MASK                                                                 0x00000080L
+#define RAS_GLOBAL_STATUS_LO__HPLGWA_SMI_MASK                                                                 0x00000100L
+#define RAS_GLOBAL_STATUS_LO__SW_SMI_MASK                                                                     0x00000200L
+#define RAS_GLOBAL_STATUS_LO__SW_SCI_MASK                                                                     0x00000400L
+#define RAS_GLOBAL_STATUS_LO__SW_NMI_MASK                                                                     0x00000800L
+#define RAS_GLOBAL_STATUS_LO__APML_NMI_MASK                                                                   0x00001000L
+#define RAS_GLOBAL_STATUS_LO__APML_SyncFld_MASK                                                               0x00002000L
+#define RAS_GLOBAL_STATUS_LO__PIN_SyncFld_NMI_MASK                                                            0x00004000L
+#define RAS_GLOBAL_STATUS_LO__APML_SyncFld_Private_MASK                                                       0x00008000L
+//RAS_GLOBAL_STATUS_HI
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortAErr__SHIFT                                                            0x0
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortBErr__SHIFT                                                            0x1
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortCErr__SHIFT                                                            0x2
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortDErr__SHIFT                                                            0x3
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortEErr__SHIFT                                                            0x4
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortFErr__SHIFT                                                            0x5
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortGErr__SHIFT                                                            0x6
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortHErr__SHIFT                                                            0x7
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortIErr__SHIFT                                                            0x8
+#define RAS_GLOBAL_STATUS_HI__PCIE1PortAErr__SHIFT                                                            0x9
+#define RAS_GLOBAL_STATUS_HI__PCIE1PortBErr__SHIFT                                                            0xa
+#define RAS_GLOBAL_STATUS_HI__PCIE1PortCErr__SHIFT                                                            0xb
+#define RAS_GLOBAL_STATUS_HI__PCIE1PortDErr__SHIFT                                                            0xc
+#define RAS_GLOBAL_STATUS_HI__NBIF1PortAErr__SHIFT                                                            0xd
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortAErr_MASK                                                              0x00000001L
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortBErr_MASK                                                              0x00000002L
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortCErr_MASK                                                              0x00000004L
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortDErr_MASK                                                              0x00000008L
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortEErr_MASK                                                              0x00000010L
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortFErr_MASK                                                              0x00000020L
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortGErr_MASK                                                              0x00000040L
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortHErr_MASK                                                              0x00000080L
+#define RAS_GLOBAL_STATUS_HI__PCIE0PortIErr_MASK                                                              0x00000100L
+#define RAS_GLOBAL_STATUS_HI__PCIE1PortAErr_MASK                                                              0x00000200L
+#define RAS_GLOBAL_STATUS_HI__PCIE1PortBErr_MASK                                                              0x00000400L
+#define RAS_GLOBAL_STATUS_HI__PCIE1PortCErr_MASK                                                              0x00000800L
+#define RAS_GLOBAL_STATUS_HI__PCIE1PortDErr_MASK                                                              0x00001000L
+#define RAS_GLOBAL_STATUS_HI__NBIF1PortAErr_MASK                                                              0x00002000L
+//PARITY_ERROR_STATUS_UNCORR_GRP0
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id0__SHIFT                                         0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id1__SHIFT                                         0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id2__SHIFT                                         0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id3__SHIFT                                         0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id4__SHIFT                                         0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id5__SHIFT                                         0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id6__SHIFT                                         0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id7__SHIFT                                         0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id8__SHIFT                                         0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id9__SHIFT                                         0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id10__SHIFT                                        0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id11__SHIFT                                        0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id12__SHIFT                                        0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id13__SHIFT                                        0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id14__SHIFT                                        0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id15__SHIFT                                        0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id16__SHIFT                                        0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id17__SHIFT                                        0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id18__SHIFT                                        0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id19__SHIFT                                        0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id20__SHIFT                                        0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id21__SHIFT                                        0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id22__SHIFT                                        0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id23__SHIFT                                        0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id24__SHIFT                                        0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id25__SHIFT                                        0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id26__SHIFT                                        0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id27__SHIFT                                        0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id28__SHIFT                                        0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id29__SHIFT                                        0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id30__SHIFT                                        0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id31__SHIFT                                        0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id0_MASK                                           0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id1_MASK                                           0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id2_MASK                                           0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id3_MASK                                           0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id4_MASK                                           0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id5_MASK                                           0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id6_MASK                                           0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id7_MASK                                           0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id8_MASK                                           0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id9_MASK                                           0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id10_MASK                                          0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id11_MASK                                          0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id12_MASK                                          0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id13_MASK                                          0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id14_MASK                                          0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id15_MASK                                          0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id16_MASK                                          0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id17_MASK                                          0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id18_MASK                                          0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id19_MASK                                          0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id20_MASK                                          0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id21_MASK                                          0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id22_MASK                                          0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id23_MASK                                          0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id24_MASK                                          0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id25_MASK                                          0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id26_MASK                                          0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id27_MASK                                          0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id28_MASK                                          0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id29_MASK                                          0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id30_MASK                                          0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP0__ParityErrDetected_Id31_MASK                                          0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP1
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id0__SHIFT                                         0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id1__SHIFT                                         0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id2__SHIFT                                         0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id3__SHIFT                                         0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id4__SHIFT                                         0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id5__SHIFT                                         0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id6__SHIFT                                         0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id7__SHIFT                                         0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id8__SHIFT                                         0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id9__SHIFT                                         0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id10__SHIFT                                        0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id11__SHIFT                                        0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id12__SHIFT                                        0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id13__SHIFT                                        0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id14__SHIFT                                        0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id15__SHIFT                                        0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id16__SHIFT                                        0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id17__SHIFT                                        0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id18__SHIFT                                        0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id19__SHIFT                                        0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id20__SHIFT                                        0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id21__SHIFT                                        0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id22__SHIFT                                        0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id23__SHIFT                                        0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id24__SHIFT                                        0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id25__SHIFT                                        0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id26__SHIFT                                        0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id27__SHIFT                                        0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id28__SHIFT                                        0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id29__SHIFT                                        0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id30__SHIFT                                        0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id31__SHIFT                                        0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id0_MASK                                           0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id1_MASK                                           0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id2_MASK                                           0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id3_MASK                                           0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id4_MASK                                           0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id5_MASK                                           0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id6_MASK                                           0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id7_MASK                                           0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id8_MASK                                           0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id9_MASK                                           0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id10_MASK                                          0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id11_MASK                                          0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id12_MASK                                          0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id13_MASK                                          0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id14_MASK                                          0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id15_MASK                                          0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id16_MASK                                          0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id17_MASK                                          0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id18_MASK                                          0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id19_MASK                                          0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id20_MASK                                          0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id21_MASK                                          0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id22_MASK                                          0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id23_MASK                                          0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id24_MASK                                          0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id25_MASK                                          0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id26_MASK                                          0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id27_MASK                                          0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id28_MASK                                          0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id29_MASK                                          0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id30_MASK                                          0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP1__ParityErrDetected_Id31_MASK                                          0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP2
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id0__SHIFT                                         0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id1__SHIFT                                         0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id2__SHIFT                                         0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id3__SHIFT                                         0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id4__SHIFT                                         0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id5__SHIFT                                         0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id6__SHIFT                                         0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id7__SHIFT                                         0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id8__SHIFT                                         0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id9__SHIFT                                         0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id10__SHIFT                                        0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id11__SHIFT                                        0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id12__SHIFT                                        0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id13__SHIFT                                        0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id14__SHIFT                                        0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id15__SHIFT                                        0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id16__SHIFT                                        0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id17__SHIFT                                        0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id18__SHIFT                                        0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id19__SHIFT                                        0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id20__SHIFT                                        0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id21__SHIFT                                        0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id22__SHIFT                                        0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id23__SHIFT                                        0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id24__SHIFT                                        0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id25__SHIFT                                        0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id26__SHIFT                                        0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id27__SHIFT                                        0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id28__SHIFT                                        0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id29__SHIFT                                        0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id30__SHIFT                                        0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id31__SHIFT                                        0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id0_MASK                                           0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id1_MASK                                           0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id2_MASK                                           0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id3_MASK                                           0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id4_MASK                                           0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id5_MASK                                           0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id6_MASK                                           0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id7_MASK                                           0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id8_MASK                                           0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id9_MASK                                           0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id10_MASK                                          0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id11_MASK                                          0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id12_MASK                                          0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id13_MASK                                          0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id14_MASK                                          0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id15_MASK                                          0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id16_MASK                                          0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id17_MASK                                          0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id18_MASK                                          0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id19_MASK                                          0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id20_MASK                                          0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id21_MASK                                          0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id22_MASK                                          0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id23_MASK                                          0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id24_MASK                                          0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id25_MASK                                          0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id26_MASK                                          0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id27_MASK                                          0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id28_MASK                                          0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id29_MASK                                          0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id30_MASK                                          0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP2__ParityErrDetected_Id31_MASK                                          0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP3
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id0__SHIFT                                         0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id1__SHIFT                                         0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id2__SHIFT                                         0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id3__SHIFT                                         0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id4__SHIFT                                         0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id5__SHIFT                                         0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id6__SHIFT                                         0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id7__SHIFT                                         0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id8__SHIFT                                         0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id9__SHIFT                                         0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id10__SHIFT                                        0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id11__SHIFT                                        0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id12__SHIFT                                        0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id13__SHIFT                                        0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id14__SHIFT                                        0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id15__SHIFT                                        0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id16__SHIFT                                        0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id17__SHIFT                                        0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id18__SHIFT                                        0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id19__SHIFT                                        0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id20__SHIFT                                        0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id21__SHIFT                                        0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id22__SHIFT                                        0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id23__SHIFT                                        0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id24__SHIFT                                        0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id25__SHIFT                                        0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id26__SHIFT                                        0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id27__SHIFT                                        0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id28__SHIFT                                        0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id29__SHIFT                                        0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id30__SHIFT                                        0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id31__SHIFT                                        0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id0_MASK                                           0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id1_MASK                                           0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id2_MASK                                           0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id3_MASK                                           0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id4_MASK                                           0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id5_MASK                                           0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id6_MASK                                           0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id7_MASK                                           0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id8_MASK                                           0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id9_MASK                                           0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id10_MASK                                          0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id11_MASK                                          0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id12_MASK                                          0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id13_MASK                                          0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id14_MASK                                          0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id15_MASK                                          0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id16_MASK                                          0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id17_MASK                                          0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id18_MASK                                          0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id19_MASK                                          0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id20_MASK                                          0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id21_MASK                                          0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id22_MASK                                          0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id23_MASK                                          0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id24_MASK                                          0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id25_MASK                                          0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id26_MASK                                          0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id27_MASK                                          0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id28_MASK                                          0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id29_MASK                                          0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id30_MASK                                          0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP3__ParityErrDetected_Id31_MASK                                          0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP4
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id0__SHIFT                                         0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id1__SHIFT                                         0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id2__SHIFT                                         0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id3__SHIFT                                         0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id4__SHIFT                                         0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id5__SHIFT                                         0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id6__SHIFT                                         0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id7__SHIFT                                         0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id8__SHIFT                                         0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id9__SHIFT                                         0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id10__SHIFT                                        0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id11__SHIFT                                        0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id12__SHIFT                                        0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id13__SHIFT                                        0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id14__SHIFT                                        0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id15__SHIFT                                        0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id16__SHIFT                                        0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id17__SHIFT                                        0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id18__SHIFT                                        0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id19__SHIFT                                        0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id20__SHIFT                                        0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id21__SHIFT                                        0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id22__SHIFT                                        0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id23__SHIFT                                        0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id24__SHIFT                                        0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id25__SHIFT                                        0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id26__SHIFT                                        0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id27__SHIFT                                        0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id28__SHIFT                                        0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id29__SHIFT                                        0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id30__SHIFT                                        0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id31__SHIFT                                        0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id0_MASK                                           0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id1_MASK                                           0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id2_MASK                                           0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id3_MASK                                           0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id4_MASK                                           0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id5_MASK                                           0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id6_MASK                                           0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id7_MASK                                           0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id8_MASK                                           0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id9_MASK                                           0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id10_MASK                                          0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id11_MASK                                          0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id12_MASK                                          0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id13_MASK                                          0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id14_MASK                                          0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id15_MASK                                          0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id16_MASK                                          0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id17_MASK                                          0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id18_MASK                                          0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id19_MASK                                          0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id20_MASK                                          0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id21_MASK                                          0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id22_MASK                                          0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id23_MASK                                          0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id24_MASK                                          0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id25_MASK                                          0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id26_MASK                                          0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id27_MASK                                          0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id28_MASK                                          0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id29_MASK                                          0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id30_MASK                                          0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP4__ParityErrDetected_Id31_MASK                                          0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP5
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id0__SHIFT                                         0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id1__SHIFT                                         0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id2__SHIFT                                         0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id3__SHIFT                                         0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id4__SHIFT                                         0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id5__SHIFT                                         0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id6__SHIFT                                         0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id7__SHIFT                                         0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id8__SHIFT                                         0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id9__SHIFT                                         0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id10__SHIFT                                        0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id11__SHIFT                                        0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id12__SHIFT                                        0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id13__SHIFT                                        0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id14__SHIFT                                        0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id15__SHIFT                                        0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id16__SHIFT                                        0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id17__SHIFT                                        0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id18__SHIFT                                        0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id19__SHIFT                                        0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id20__SHIFT                                        0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id21__SHIFT                                        0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id22__SHIFT                                        0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id23__SHIFT                                        0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id24__SHIFT                                        0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id25__SHIFT                                        0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id26__SHIFT                                        0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id27__SHIFT                                        0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id28__SHIFT                                        0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id29__SHIFT                                        0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id30__SHIFT                                        0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id31__SHIFT                                        0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id0_MASK                                           0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id1_MASK                                           0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id2_MASK                                           0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id3_MASK                                           0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id4_MASK                                           0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id5_MASK                                           0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id6_MASK                                           0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id7_MASK                                           0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id8_MASK                                           0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id9_MASK                                           0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id10_MASK                                          0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id11_MASK                                          0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id12_MASK                                          0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id13_MASK                                          0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id14_MASK                                          0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id15_MASK                                          0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id16_MASK                                          0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id17_MASK                                          0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id18_MASK                                          0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id19_MASK                                          0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id20_MASK                                          0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id21_MASK                                          0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id22_MASK                                          0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id23_MASK                                          0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id24_MASK                                          0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id25_MASK                                          0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id26_MASK                                          0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id27_MASK                                          0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id28_MASK                                          0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id29_MASK                                          0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id30_MASK                                          0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP5__ParityErrDetected_Id31_MASK                                          0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP6
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id0__SHIFT                                         0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id1__SHIFT                                         0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id2__SHIFT                                         0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id3__SHIFT                                         0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id4__SHIFT                                         0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id5__SHIFT                                         0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id6__SHIFT                                         0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id7__SHIFT                                         0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id8__SHIFT                                         0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id9__SHIFT                                         0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id10__SHIFT                                        0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id11__SHIFT                                        0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id12__SHIFT                                        0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id13__SHIFT                                        0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id14__SHIFT                                        0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id15__SHIFT                                        0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id16__SHIFT                                        0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id17__SHIFT                                        0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id18__SHIFT                                        0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id19__SHIFT                                        0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id20__SHIFT                                        0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id21__SHIFT                                        0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id22__SHIFT                                        0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id23__SHIFT                                        0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id24__SHIFT                                        0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id25__SHIFT                                        0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id26__SHIFT                                        0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id27__SHIFT                                        0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id28__SHIFT                                        0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id29__SHIFT                                        0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id30__SHIFT                                        0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id31__SHIFT                                        0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id0_MASK                                           0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id1_MASK                                           0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id2_MASK                                           0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id3_MASK                                           0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id4_MASK                                           0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id5_MASK                                           0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id6_MASK                                           0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id7_MASK                                           0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id8_MASK                                           0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id9_MASK                                           0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id10_MASK                                          0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id11_MASK                                          0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id12_MASK                                          0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id13_MASK                                          0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id14_MASK                                          0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id15_MASK                                          0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id16_MASK                                          0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id17_MASK                                          0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id18_MASK                                          0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id19_MASK                                          0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id20_MASK                                          0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id21_MASK                                          0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id22_MASK                                          0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id23_MASK                                          0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id24_MASK                                          0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id25_MASK                                          0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id26_MASK                                          0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id27_MASK                                          0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id28_MASK                                          0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id29_MASK                                          0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id30_MASK                                          0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP6__ParityErrDetected_Id31_MASK                                          0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP7
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id0__SHIFT                                         0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id1__SHIFT                                         0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id2__SHIFT                                         0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id3__SHIFT                                         0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id4__SHIFT                                         0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id5__SHIFT                                         0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id6__SHIFT                                         0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id7__SHIFT                                         0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id8__SHIFT                                         0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id9__SHIFT                                         0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id10__SHIFT                                        0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id11__SHIFT                                        0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id12__SHIFT                                        0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id13__SHIFT                                        0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id14__SHIFT                                        0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id15__SHIFT                                        0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id16__SHIFT                                        0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id17__SHIFT                                        0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id18__SHIFT                                        0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id19__SHIFT                                        0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id20__SHIFT                                        0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id21__SHIFT                                        0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id22__SHIFT                                        0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id23__SHIFT                                        0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id24__SHIFT                                        0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id25__SHIFT                                        0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id26__SHIFT                                        0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id27__SHIFT                                        0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id28__SHIFT                                        0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id29__SHIFT                                        0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id30__SHIFT                                        0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id31__SHIFT                                        0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id0_MASK                                           0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id1_MASK                                           0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id2_MASK                                           0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id3_MASK                                           0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id4_MASK                                           0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id5_MASK                                           0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id6_MASK                                           0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id7_MASK                                           0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id8_MASK                                           0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id9_MASK                                           0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id10_MASK                                          0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id11_MASK                                          0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id12_MASK                                          0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id13_MASK                                          0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id14_MASK                                          0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id15_MASK                                          0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id16_MASK                                          0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id17_MASK                                          0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id18_MASK                                          0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id19_MASK                                          0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id20_MASK                                          0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id21_MASK                                          0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id22_MASK                                          0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id23_MASK                                          0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id24_MASK                                          0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id25_MASK                                          0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id26_MASK                                          0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id27_MASK                                          0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id28_MASK                                          0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id29_MASK                                          0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id30_MASK                                          0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP7__ParityErrDetected_Id31_MASK                                          0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP10
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id0__SHIFT                                        0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id1__SHIFT                                        0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id2__SHIFT                                        0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id3__SHIFT                                        0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id4__SHIFT                                        0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id5__SHIFT                                        0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id6__SHIFT                                        0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id7__SHIFT                                        0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id8__SHIFT                                        0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id9__SHIFT                                        0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id10__SHIFT                                       0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id11__SHIFT                                       0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id12__SHIFT                                       0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id13__SHIFT                                       0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id14__SHIFT                                       0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id15__SHIFT                                       0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id16__SHIFT                                       0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id17__SHIFT                                       0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id18__SHIFT                                       0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id19__SHIFT                                       0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id20__SHIFT                                       0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id21__SHIFT                                       0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id22__SHIFT                                       0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id23__SHIFT                                       0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id24__SHIFT                                       0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id25__SHIFT                                       0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id26__SHIFT                                       0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id27__SHIFT                                       0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id28__SHIFT                                       0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id29__SHIFT                                       0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id30__SHIFT                                       0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id31__SHIFT                                       0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id0_MASK                                          0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id1_MASK                                          0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id2_MASK                                          0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id3_MASK                                          0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id4_MASK                                          0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id5_MASK                                          0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id6_MASK                                          0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id7_MASK                                          0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id8_MASK                                          0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id9_MASK                                          0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id10_MASK                                         0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id11_MASK                                         0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id12_MASK                                         0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id13_MASK                                         0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id14_MASK                                         0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id15_MASK                                         0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id16_MASK                                         0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id17_MASK                                         0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id18_MASK                                         0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id19_MASK                                         0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id20_MASK                                         0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id21_MASK                                         0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id22_MASK                                         0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id23_MASK                                         0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id24_MASK                                         0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id25_MASK                                         0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id26_MASK                                         0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id27_MASK                                         0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id28_MASK                                         0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id29_MASK                                         0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id30_MASK                                         0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP10__ParityErrDetected_Id31_MASK                                         0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP11
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id0__SHIFT                                        0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id1__SHIFT                                        0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id2__SHIFT                                        0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id3__SHIFT                                        0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id4__SHIFT                                        0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id5__SHIFT                                        0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id6__SHIFT                                        0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id7__SHIFT                                        0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id8__SHIFT                                        0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id9__SHIFT                                        0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id10__SHIFT                                       0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id11__SHIFT                                       0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id12__SHIFT                                       0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id13__SHIFT                                       0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id14__SHIFT                                       0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id15__SHIFT                                       0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id16__SHIFT                                       0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id17__SHIFT                                       0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id18__SHIFT                                       0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id19__SHIFT                                       0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id20__SHIFT                                       0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id21__SHIFT                                       0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id22__SHIFT                                       0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id23__SHIFT                                       0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id24__SHIFT                                       0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id25__SHIFT                                       0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id26__SHIFT                                       0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id27__SHIFT                                       0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id28__SHIFT                                       0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id29__SHIFT                                       0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id30__SHIFT                                       0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id31__SHIFT                                       0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id0_MASK                                          0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id1_MASK                                          0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id2_MASK                                          0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id3_MASK                                          0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id4_MASK                                          0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id5_MASK                                          0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id6_MASK                                          0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id7_MASK                                          0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id8_MASK                                          0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id9_MASK                                          0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id10_MASK                                         0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id11_MASK                                         0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id12_MASK                                         0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id13_MASK                                         0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id14_MASK                                         0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id15_MASK                                         0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id16_MASK                                         0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id17_MASK                                         0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id18_MASK                                         0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id19_MASK                                         0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id20_MASK                                         0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id21_MASK                                         0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id22_MASK                                         0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id23_MASK                                         0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id24_MASK                                         0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id25_MASK                                         0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id26_MASK                                         0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id27_MASK                                         0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id28_MASK                                         0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id29_MASK                                         0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id30_MASK                                         0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP11__ParityErrDetected_Id31_MASK                                         0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP12
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id0__SHIFT                                        0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id1__SHIFT                                        0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id2__SHIFT                                        0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id3__SHIFT                                        0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id4__SHIFT                                        0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id5__SHIFT                                        0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id6__SHIFT                                        0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id7__SHIFT                                        0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id8__SHIFT                                        0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id9__SHIFT                                        0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id10__SHIFT                                       0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id11__SHIFT                                       0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id12__SHIFT                                       0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id13__SHIFT                                       0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id14__SHIFT                                       0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id15__SHIFT                                       0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id16__SHIFT                                       0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id17__SHIFT                                       0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id18__SHIFT                                       0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id19__SHIFT                                       0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id20__SHIFT                                       0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id21__SHIFT                                       0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id22__SHIFT                                       0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id23__SHIFT                                       0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id24__SHIFT                                       0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id25__SHIFT                                       0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id26__SHIFT                                       0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id27__SHIFT                                       0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id28__SHIFT                                       0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id29__SHIFT                                       0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id30__SHIFT                                       0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id31__SHIFT                                       0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id0_MASK                                          0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id1_MASK                                          0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id2_MASK                                          0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id3_MASK                                          0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id4_MASK                                          0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id5_MASK                                          0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id6_MASK                                          0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id7_MASK                                          0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id8_MASK                                          0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id9_MASK                                          0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id10_MASK                                         0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id11_MASK                                         0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id12_MASK                                         0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id13_MASK                                         0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id14_MASK                                         0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id15_MASK                                         0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id16_MASK                                         0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id17_MASK                                         0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id18_MASK                                         0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id19_MASK                                         0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id20_MASK                                         0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id21_MASK                                         0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id22_MASK                                         0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id23_MASK                                         0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id24_MASK                                         0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id25_MASK                                         0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id26_MASK                                         0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id27_MASK                                         0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id28_MASK                                         0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id29_MASK                                         0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id30_MASK                                         0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP12__ParityErrDetected_Id31_MASK                                         0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP13
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id0__SHIFT                                        0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id1__SHIFT                                        0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id2__SHIFT                                        0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id3__SHIFT                                        0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id4__SHIFT                                        0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id5__SHIFT                                        0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id6__SHIFT                                        0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id7__SHIFT                                        0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id8__SHIFT                                        0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id9__SHIFT                                        0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id10__SHIFT                                       0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id11__SHIFT                                       0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id12__SHIFT                                       0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id13__SHIFT                                       0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id14__SHIFT                                       0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id15__SHIFT                                       0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id16__SHIFT                                       0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id17__SHIFT                                       0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id18__SHIFT                                       0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id19__SHIFT                                       0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id20__SHIFT                                       0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id21__SHIFT                                       0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id22__SHIFT                                       0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id23__SHIFT                                       0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id24__SHIFT                                       0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id25__SHIFT                                       0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id26__SHIFT                                       0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id27__SHIFT                                       0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id28__SHIFT                                       0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id29__SHIFT                                       0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id30__SHIFT                                       0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id31__SHIFT                                       0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id0_MASK                                          0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id1_MASK                                          0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id2_MASK                                          0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id3_MASK                                          0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id4_MASK                                          0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id5_MASK                                          0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id6_MASK                                          0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id7_MASK                                          0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id8_MASK                                          0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id9_MASK                                          0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id10_MASK                                         0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id11_MASK                                         0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id12_MASK                                         0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id13_MASK                                         0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id14_MASK                                         0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id15_MASK                                         0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id16_MASK                                         0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id17_MASK                                         0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id18_MASK                                         0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id19_MASK                                         0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id20_MASK                                         0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id21_MASK                                         0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id22_MASK                                         0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id23_MASK                                         0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id24_MASK                                         0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id25_MASK                                         0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id26_MASK                                         0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id27_MASK                                         0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id28_MASK                                         0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id29_MASK                                         0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id30_MASK                                         0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP13__ParityErrDetected_Id31_MASK                                         0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP14
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id0__SHIFT                                        0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id1__SHIFT                                        0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id2__SHIFT                                        0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id3__SHIFT                                        0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id4__SHIFT                                        0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id5__SHIFT                                        0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id6__SHIFT                                        0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id7__SHIFT                                        0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id8__SHIFT                                        0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id9__SHIFT                                        0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id10__SHIFT                                       0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id11__SHIFT                                       0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id12__SHIFT                                       0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id13__SHIFT                                       0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id14__SHIFT                                       0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id15__SHIFT                                       0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id16__SHIFT                                       0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id17__SHIFT                                       0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id18__SHIFT                                       0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id19__SHIFT                                       0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id20__SHIFT                                       0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id21__SHIFT                                       0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id22__SHIFT                                       0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id23__SHIFT                                       0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id24__SHIFT                                       0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id25__SHIFT                                       0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id26__SHIFT                                       0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id27__SHIFT                                       0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id28__SHIFT                                       0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id29__SHIFT                                       0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id30__SHIFT                                       0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id31__SHIFT                                       0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id0_MASK                                          0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id1_MASK                                          0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id2_MASK                                          0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id3_MASK                                          0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id4_MASK                                          0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id5_MASK                                          0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id6_MASK                                          0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id7_MASK                                          0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id8_MASK                                          0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id9_MASK                                          0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id10_MASK                                         0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id11_MASK                                         0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id12_MASK                                         0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id13_MASK                                         0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id14_MASK                                         0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id15_MASK                                         0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id16_MASK                                         0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id17_MASK                                         0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id18_MASK                                         0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id19_MASK                                         0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id20_MASK                                         0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id21_MASK                                         0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id22_MASK                                         0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id23_MASK                                         0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id24_MASK                                         0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id25_MASK                                         0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id26_MASK                                         0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id27_MASK                                         0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id28_MASK                                         0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id29_MASK                                         0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id30_MASK                                         0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP14__ParityErrDetected_Id31_MASK                                         0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP15
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id0__SHIFT                                        0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id1__SHIFT                                        0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id2__SHIFT                                        0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id3__SHIFT                                        0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id4__SHIFT                                        0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id5__SHIFT                                        0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id6__SHIFT                                        0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id7__SHIFT                                        0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id8__SHIFT                                        0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id9__SHIFT                                        0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id10__SHIFT                                       0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id11__SHIFT                                       0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id12__SHIFT                                       0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id13__SHIFT                                       0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id14__SHIFT                                       0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id15__SHIFT                                       0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id16__SHIFT                                       0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id17__SHIFT                                       0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id18__SHIFT                                       0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id19__SHIFT                                       0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id20__SHIFT                                       0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id21__SHIFT                                       0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id22__SHIFT                                       0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id23__SHIFT                                       0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id24__SHIFT                                       0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id25__SHIFT                                       0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id26__SHIFT                                       0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id27__SHIFT                                       0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id28__SHIFT                                       0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id29__SHIFT                                       0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id30__SHIFT                                       0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id31__SHIFT                                       0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id0_MASK                                          0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id1_MASK                                          0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id2_MASK                                          0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id3_MASK                                          0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id4_MASK                                          0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id5_MASK                                          0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id6_MASK                                          0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id7_MASK                                          0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id8_MASK                                          0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id9_MASK                                          0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id10_MASK                                         0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id11_MASK                                         0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id12_MASK                                         0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id13_MASK                                         0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id14_MASK                                         0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id15_MASK                                         0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id16_MASK                                         0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id17_MASK                                         0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id18_MASK                                         0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id19_MASK                                         0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id20_MASK                                         0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id21_MASK                                         0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id22_MASK                                         0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id23_MASK                                         0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id24_MASK                                         0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id25_MASK                                         0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id26_MASK                                         0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id27_MASK                                         0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id28_MASK                                         0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id29_MASK                                         0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id30_MASK                                         0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP15__ParityErrDetected_Id31_MASK                                         0x80000000L
+//PARITY_ERROR_STATUS_UNCORR_GRP16
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id0__SHIFT                                        0x0
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id1__SHIFT                                        0x1
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id2__SHIFT                                        0x2
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id3__SHIFT                                        0x3
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id4__SHIFT                                        0x4
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id5__SHIFT                                        0x5
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id6__SHIFT                                        0x6
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id7__SHIFT                                        0x7
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id8__SHIFT                                        0x8
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id9__SHIFT                                        0x9
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id10__SHIFT                                       0xa
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id11__SHIFT                                       0xb
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id12__SHIFT                                       0xc
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id13__SHIFT                                       0xd
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id14__SHIFT                                       0xe
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id15__SHIFT                                       0xf
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id16__SHIFT                                       0x10
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id17__SHIFT                                       0x11
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id18__SHIFT                                       0x12
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id19__SHIFT                                       0x13
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id20__SHIFT                                       0x14
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id21__SHIFT                                       0x15
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id22__SHIFT                                       0x16
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id23__SHIFT                                       0x17
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id24__SHIFT                                       0x18
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id25__SHIFT                                       0x19
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id26__SHIFT                                       0x1a
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id27__SHIFT                                       0x1b
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id28__SHIFT                                       0x1c
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id29__SHIFT                                       0x1d
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id30__SHIFT                                       0x1e
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id31__SHIFT                                       0x1f
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id0_MASK                                          0x00000001L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id1_MASK                                          0x00000002L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id2_MASK                                          0x00000004L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id3_MASK                                          0x00000008L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id4_MASK                                          0x00000010L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id5_MASK                                          0x00000020L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id6_MASK                                          0x00000040L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id7_MASK                                          0x00000080L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id8_MASK                                          0x00000100L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id9_MASK                                          0x00000200L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id10_MASK                                         0x00000400L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id11_MASK                                         0x00000800L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id12_MASK                                         0x00001000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id13_MASK                                         0x00002000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id14_MASK                                         0x00004000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id15_MASK                                         0x00008000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id16_MASK                                         0x00010000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id17_MASK                                         0x00020000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id18_MASK                                         0x00040000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id19_MASK                                         0x00080000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id20_MASK                                         0x00100000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id21_MASK                                         0x00200000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id22_MASK                                         0x00400000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id23_MASK                                         0x00800000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id24_MASK                                         0x01000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id25_MASK                                         0x02000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id26_MASK                                         0x04000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id27_MASK                                         0x08000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id28_MASK                                         0x10000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id29_MASK                                         0x20000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id30_MASK                                         0x40000000L
+#define PARITY_ERROR_STATUS_UNCORR_GRP16__ParityErrDetected_Id31_MASK                                         0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP0
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP0__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP1
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP1__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP2
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP2__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP3
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP3__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP4
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP4__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP5
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP5__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP6
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP6__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP7
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP7__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP10
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id0__SHIFT                                          0x0
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id1__SHIFT                                          0x1
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id2__SHIFT                                          0x2
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id3__SHIFT                                          0x3
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id4__SHIFT                                          0x4
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id5__SHIFT                                          0x5
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id6__SHIFT                                          0x6
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id7__SHIFT                                          0x7
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id8__SHIFT                                          0x8
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id9__SHIFT                                          0x9
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id10__SHIFT                                         0xa
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id11__SHIFT                                         0xb
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id12__SHIFT                                         0xc
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id13__SHIFT                                         0xd
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id14__SHIFT                                         0xe
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id15__SHIFT                                         0xf
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id16__SHIFT                                         0x10
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id17__SHIFT                                         0x11
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id18__SHIFT                                         0x12
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id19__SHIFT                                         0x13
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id20__SHIFT                                         0x14
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id21__SHIFT                                         0x15
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id22__SHIFT                                         0x16
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id23__SHIFT                                         0x17
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id24__SHIFT                                         0x18
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id25__SHIFT                                         0x19
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id26__SHIFT                                         0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id27__SHIFT                                         0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id28__SHIFT                                         0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id29__SHIFT                                         0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id30__SHIFT                                         0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id31__SHIFT                                         0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id0_MASK                                            0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id1_MASK                                            0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id2_MASK                                            0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id3_MASK                                            0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id4_MASK                                            0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id5_MASK                                            0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id6_MASK                                            0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id7_MASK                                            0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id8_MASK                                            0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id9_MASK                                            0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id10_MASK                                           0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id11_MASK                                           0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id12_MASK                                           0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id13_MASK                                           0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id14_MASK                                           0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id15_MASK                                           0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id16_MASK                                           0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id17_MASK                                           0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id18_MASK                                           0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id19_MASK                                           0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id20_MASK                                           0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id21_MASK                                           0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id22_MASK                                           0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id23_MASK                                           0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id24_MASK                                           0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id25_MASK                                           0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id26_MASK                                           0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id27_MASK                                           0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id28_MASK                                           0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id29_MASK                                           0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id30_MASK                                           0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP10__ParityErrDetected_Id31_MASK                                           0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP11
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id0__SHIFT                                          0x0
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id1__SHIFT                                          0x1
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id2__SHIFT                                          0x2
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id3__SHIFT                                          0x3
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id4__SHIFT                                          0x4
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id5__SHIFT                                          0x5
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id6__SHIFT                                          0x6
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id7__SHIFT                                          0x7
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id8__SHIFT                                          0x8
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id9__SHIFT                                          0x9
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id10__SHIFT                                         0xa
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id11__SHIFT                                         0xb
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id12__SHIFT                                         0xc
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id13__SHIFT                                         0xd
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id14__SHIFT                                         0xe
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id15__SHIFT                                         0xf
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id16__SHIFT                                         0x10
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id17__SHIFT                                         0x11
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id18__SHIFT                                         0x12
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id19__SHIFT                                         0x13
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id20__SHIFT                                         0x14
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id21__SHIFT                                         0x15
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id22__SHIFT                                         0x16
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id23__SHIFT                                         0x17
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id24__SHIFT                                         0x18
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id25__SHIFT                                         0x19
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id26__SHIFT                                         0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id27__SHIFT                                         0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id28__SHIFT                                         0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id29__SHIFT                                         0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id30__SHIFT                                         0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id31__SHIFT                                         0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id0_MASK                                            0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id1_MASK                                            0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id2_MASK                                            0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id3_MASK                                            0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id4_MASK                                            0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id5_MASK                                            0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id6_MASK                                            0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id7_MASK                                            0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id8_MASK                                            0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id9_MASK                                            0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id10_MASK                                           0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id11_MASK                                           0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id12_MASK                                           0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id13_MASK                                           0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id14_MASK                                           0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id15_MASK                                           0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id16_MASK                                           0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id17_MASK                                           0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id18_MASK                                           0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id19_MASK                                           0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id20_MASK                                           0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id21_MASK                                           0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id22_MASK                                           0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id23_MASK                                           0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id24_MASK                                           0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id25_MASK                                           0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id26_MASK                                           0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id27_MASK                                           0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id28_MASK                                           0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id29_MASK                                           0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id30_MASK                                           0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP11__ParityErrDetected_Id31_MASK                                           0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP12
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id0__SHIFT                                          0x0
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id1__SHIFT                                          0x1
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id2__SHIFT                                          0x2
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id3__SHIFT                                          0x3
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id4__SHIFT                                          0x4
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id5__SHIFT                                          0x5
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id6__SHIFT                                          0x6
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id7__SHIFT                                          0x7
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id8__SHIFT                                          0x8
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id9__SHIFT                                          0x9
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id10__SHIFT                                         0xa
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id11__SHIFT                                         0xb
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id12__SHIFT                                         0xc
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id13__SHIFT                                         0xd
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id14__SHIFT                                         0xe
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id15__SHIFT                                         0xf
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id16__SHIFT                                         0x10
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id17__SHIFT                                         0x11
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id18__SHIFT                                         0x12
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id19__SHIFT                                         0x13
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id20__SHIFT                                         0x14
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id21__SHIFT                                         0x15
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id22__SHIFT                                         0x16
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id23__SHIFT                                         0x17
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id24__SHIFT                                         0x18
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id25__SHIFT                                         0x19
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id26__SHIFT                                         0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id27__SHIFT                                         0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id28__SHIFT                                         0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id29__SHIFT                                         0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id30__SHIFT                                         0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id31__SHIFT                                         0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id0_MASK                                            0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id1_MASK                                            0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id2_MASK                                            0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id3_MASK                                            0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id4_MASK                                            0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id5_MASK                                            0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id6_MASK                                            0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id7_MASK                                            0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id8_MASK                                            0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id9_MASK                                            0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id10_MASK                                           0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id11_MASK                                           0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id12_MASK                                           0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id13_MASK                                           0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id14_MASK                                           0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id15_MASK                                           0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id16_MASK                                           0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id17_MASK                                           0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id18_MASK                                           0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id19_MASK                                           0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id20_MASK                                           0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id21_MASK                                           0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id22_MASK                                           0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id23_MASK                                           0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id24_MASK                                           0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id25_MASK                                           0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id26_MASK                                           0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id27_MASK                                           0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id28_MASK                                           0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id29_MASK                                           0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id30_MASK                                           0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP12__ParityErrDetected_Id31_MASK                                           0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP13
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id0__SHIFT                                          0x0
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id1__SHIFT                                          0x1
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id2__SHIFT                                          0x2
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id3__SHIFT                                          0x3
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id4__SHIFT                                          0x4
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id5__SHIFT                                          0x5
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id6__SHIFT                                          0x6
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id7__SHIFT                                          0x7
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id8__SHIFT                                          0x8
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id9__SHIFT                                          0x9
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id10__SHIFT                                         0xa
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id11__SHIFT                                         0xb
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id12__SHIFT                                         0xc
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id13__SHIFT                                         0xd
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id14__SHIFT                                         0xe
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id15__SHIFT                                         0xf
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id16__SHIFT                                         0x10
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id17__SHIFT                                         0x11
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id18__SHIFT                                         0x12
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id19__SHIFT                                         0x13
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id20__SHIFT                                         0x14
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id21__SHIFT                                         0x15
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id22__SHIFT                                         0x16
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id23__SHIFT                                         0x17
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id24__SHIFT                                         0x18
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id25__SHIFT                                         0x19
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id26__SHIFT                                         0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id27__SHIFT                                         0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id28__SHIFT                                         0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id29__SHIFT                                         0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id30__SHIFT                                         0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id31__SHIFT                                         0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id0_MASK                                            0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id1_MASK                                            0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id2_MASK                                            0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id3_MASK                                            0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id4_MASK                                            0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id5_MASK                                            0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id6_MASK                                            0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id7_MASK                                            0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id8_MASK                                            0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id9_MASK                                            0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id10_MASK                                           0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id11_MASK                                           0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id12_MASK                                           0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id13_MASK                                           0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id14_MASK                                           0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id15_MASK                                           0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id16_MASK                                           0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id17_MASK                                           0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id18_MASK                                           0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id19_MASK                                           0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id20_MASK                                           0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id21_MASK                                           0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id22_MASK                                           0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id23_MASK                                           0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id24_MASK                                           0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id25_MASK                                           0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id26_MASK                                           0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id27_MASK                                           0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id28_MASK                                           0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id29_MASK                                           0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id30_MASK                                           0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP13__ParityErrDetected_Id31_MASK                                           0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP14
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id0__SHIFT                                          0x0
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id1__SHIFT                                          0x1
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id2__SHIFT                                          0x2
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id3__SHIFT                                          0x3
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id4__SHIFT                                          0x4
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id5__SHIFT                                          0x5
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id6__SHIFT                                          0x6
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id7__SHIFT                                          0x7
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id8__SHIFT                                          0x8
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id9__SHIFT                                          0x9
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id10__SHIFT                                         0xa
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id11__SHIFT                                         0xb
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id12__SHIFT                                         0xc
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id13__SHIFT                                         0xd
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id14__SHIFT                                         0xe
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id15__SHIFT                                         0xf
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id16__SHIFT                                         0x10
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id17__SHIFT                                         0x11
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id18__SHIFT                                         0x12
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id19__SHIFT                                         0x13
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id20__SHIFT                                         0x14
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id21__SHIFT                                         0x15
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id22__SHIFT                                         0x16
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id23__SHIFT                                         0x17
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id24__SHIFT                                         0x18
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id25__SHIFT                                         0x19
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id26__SHIFT                                         0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id27__SHIFT                                         0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id28__SHIFT                                         0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id29__SHIFT                                         0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id30__SHIFT                                         0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id31__SHIFT                                         0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id0_MASK                                            0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id1_MASK                                            0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id2_MASK                                            0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id3_MASK                                            0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id4_MASK                                            0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id5_MASK                                            0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id6_MASK                                            0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id7_MASK                                            0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id8_MASK                                            0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id9_MASK                                            0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id10_MASK                                           0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id11_MASK                                           0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id12_MASK                                           0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id13_MASK                                           0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id14_MASK                                           0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id15_MASK                                           0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id16_MASK                                           0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id17_MASK                                           0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id18_MASK                                           0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id19_MASK                                           0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id20_MASK                                           0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id21_MASK                                           0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id22_MASK                                           0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id23_MASK                                           0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id24_MASK                                           0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id25_MASK                                           0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id26_MASK                                           0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id27_MASK                                           0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id28_MASK                                           0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id29_MASK                                           0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id30_MASK                                           0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP14__ParityErrDetected_Id31_MASK                                           0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP15
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id0__SHIFT                                          0x0
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id1__SHIFT                                          0x1
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id2__SHIFT                                          0x2
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id3__SHIFT                                          0x3
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id4__SHIFT                                          0x4
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id5__SHIFT                                          0x5
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id6__SHIFT                                          0x6
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id7__SHIFT                                          0x7
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id8__SHIFT                                          0x8
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id9__SHIFT                                          0x9
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id10__SHIFT                                         0xa
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id11__SHIFT                                         0xb
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id12__SHIFT                                         0xc
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id13__SHIFT                                         0xd
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id14__SHIFT                                         0xe
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id15__SHIFT                                         0xf
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id16__SHIFT                                         0x10
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id17__SHIFT                                         0x11
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id18__SHIFT                                         0x12
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id19__SHIFT                                         0x13
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id20__SHIFT                                         0x14
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id21__SHIFT                                         0x15
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id22__SHIFT                                         0x16
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id23__SHIFT                                         0x17
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id24__SHIFT                                         0x18
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id25__SHIFT                                         0x19
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id26__SHIFT                                         0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id27__SHIFT                                         0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id28__SHIFT                                         0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id29__SHIFT                                         0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id30__SHIFT                                         0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id31__SHIFT                                         0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id0_MASK                                            0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id1_MASK                                            0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id2_MASK                                            0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id3_MASK                                            0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id4_MASK                                            0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id5_MASK                                            0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id6_MASK                                            0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id7_MASK                                            0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id8_MASK                                            0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id9_MASK                                            0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id10_MASK                                           0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id11_MASK                                           0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id12_MASK                                           0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id13_MASK                                           0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id14_MASK                                           0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id15_MASK                                           0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id16_MASK                                           0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id17_MASK                                           0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id18_MASK                                           0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id19_MASK                                           0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id20_MASK                                           0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id21_MASK                                           0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id22_MASK                                           0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id23_MASK                                           0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id24_MASK                                           0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id25_MASK                                           0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id26_MASK                                           0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id27_MASK                                           0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id28_MASK                                           0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id29_MASK                                           0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id30_MASK                                           0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP15__ParityErrDetected_Id31_MASK                                           0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP16
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id0__SHIFT                                          0x0
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id1__SHIFT                                          0x1
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id2__SHIFT                                          0x2
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id3__SHIFT                                          0x3
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id4__SHIFT                                          0x4
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id5__SHIFT                                          0x5
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id6__SHIFT                                          0x6
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id7__SHIFT                                          0x7
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id8__SHIFT                                          0x8
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id9__SHIFT                                          0x9
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id10__SHIFT                                         0xa
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id11__SHIFT                                         0xb
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id12__SHIFT                                         0xc
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id13__SHIFT                                         0xd
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id14__SHIFT                                         0xe
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id15__SHIFT                                         0xf
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id16__SHIFT                                         0x10
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id17__SHIFT                                         0x11
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id18__SHIFT                                         0x12
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id19__SHIFT                                         0x13
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id20__SHIFT                                         0x14
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id21__SHIFT                                         0x15
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id22__SHIFT                                         0x16
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id23__SHIFT                                         0x17
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id24__SHIFT                                         0x18
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id25__SHIFT                                         0x19
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id26__SHIFT                                         0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id27__SHIFT                                         0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id28__SHIFT                                         0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id29__SHIFT                                         0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id30__SHIFT                                         0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id31__SHIFT                                         0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id0_MASK                                            0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id1_MASK                                            0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id2_MASK                                            0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id3_MASK                                            0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id4_MASK                                            0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id5_MASK                                            0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id6_MASK                                            0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id7_MASK                                            0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id8_MASK                                            0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id9_MASK                                            0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id10_MASK                                           0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id11_MASK                                           0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id12_MASK                                           0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id13_MASK                                           0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id14_MASK                                           0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id15_MASK                                           0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id16_MASK                                           0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id17_MASK                                           0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id18_MASK                                           0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id19_MASK                                           0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id20_MASK                                           0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id21_MASK                                           0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id22_MASK                                           0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id23_MASK                                           0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id24_MASK                                           0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id25_MASK                                           0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id26_MASK                                           0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id27_MASK                                           0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id28_MASK                                           0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id29_MASK                                           0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id30_MASK                                           0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP16__ParityErrDetected_Id31_MASK                                           0x80000000L
+//PARITY_ERROR_STATUS_CORR_GRP17
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id0__SHIFT                                          0x0
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id1__SHIFT                                          0x1
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id2__SHIFT                                          0x2
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id3__SHIFT                                          0x3
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id4__SHIFT                                          0x4
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id5__SHIFT                                          0x5
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id6__SHIFT                                          0x6
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id7__SHIFT                                          0x7
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id8__SHIFT                                          0x8
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id9__SHIFT                                          0x9
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id10__SHIFT                                         0xa
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id11__SHIFT                                         0xb
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id12__SHIFT                                         0xc
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id13__SHIFT                                         0xd
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id14__SHIFT                                         0xe
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id15__SHIFT                                         0xf
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id16__SHIFT                                         0x10
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id17__SHIFT                                         0x11
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id18__SHIFT                                         0x12
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id19__SHIFT                                         0x13
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id20__SHIFT                                         0x14
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id21__SHIFT                                         0x15
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id22__SHIFT                                         0x16
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id23__SHIFT                                         0x17
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id24__SHIFT                                         0x18
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id25__SHIFT                                         0x19
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id26__SHIFT                                         0x1a
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id27__SHIFT                                         0x1b
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id28__SHIFT                                         0x1c
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id29__SHIFT                                         0x1d
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id30__SHIFT                                         0x1e
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id31__SHIFT                                         0x1f
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id0_MASK                                            0x00000001L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id1_MASK                                            0x00000002L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id2_MASK                                            0x00000004L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id3_MASK                                            0x00000008L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id4_MASK                                            0x00000010L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id5_MASK                                            0x00000020L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id6_MASK                                            0x00000040L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id7_MASK                                            0x00000080L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id8_MASK                                            0x00000100L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id9_MASK                                            0x00000200L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id10_MASK                                           0x00000400L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id11_MASK                                           0x00000800L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id12_MASK                                           0x00001000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id13_MASK                                           0x00002000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id14_MASK                                           0x00004000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id15_MASK                                           0x00008000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id16_MASK                                           0x00010000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id17_MASK                                           0x00020000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id18_MASK                                           0x00040000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id19_MASK                                           0x00080000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id20_MASK                                           0x00100000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id21_MASK                                           0x00200000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id22_MASK                                           0x00400000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id23_MASK                                           0x00800000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id24_MASK                                           0x01000000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id25_MASK                                           0x02000000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id26_MASK                                           0x04000000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id27_MASK                                           0x08000000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id28_MASK                                           0x10000000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id29_MASK                                           0x20000000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id30_MASK                                           0x40000000L
+#define PARITY_ERROR_STATUS_CORR_GRP17__ParityErrDetected_Id31_MASK                                           0x80000000L
+//PARITY_COUNTER_CORR_GRP0
+#define PARITY_COUNTER_CORR_GRP0__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_CORR_GRP0__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_CORR_GRP0__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP0__ResetEn_MASK                                                                0x80000000L
+//PARITY_COUNTER_CORR_GRP1
+#define PARITY_COUNTER_CORR_GRP1__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_CORR_GRP1__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_CORR_GRP1__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP1__ResetEn_MASK                                                                0x80000000L
+//PARITY_COUNTER_CORR_GRP2
+#define PARITY_COUNTER_CORR_GRP2__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_CORR_GRP2__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_CORR_GRP2__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP2__ResetEn_MASK                                                                0x80000000L
+//PARITY_COUNTER_CORR_GRP3
+#define PARITY_COUNTER_CORR_GRP3__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_CORR_GRP3__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_CORR_GRP3__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP3__ResetEn_MASK                                                                0x80000000L
+//PARITY_COUNTER_CORR_GRP4
+#define PARITY_COUNTER_CORR_GRP4__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_CORR_GRP4__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_CORR_GRP4__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP4__ResetEn_MASK                                                                0x80000000L
+//PARITY_COUNTER_CORR_GRP5
+#define PARITY_COUNTER_CORR_GRP5__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_CORR_GRP5__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_CORR_GRP5__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP5__ResetEn_MASK                                                                0x80000000L
+//PARITY_COUNTER_CORR_GRP6
+#define PARITY_COUNTER_CORR_GRP6__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_CORR_GRP6__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_CORR_GRP6__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP6__ResetEn_MASK                                                                0x80000000L
+//PARITY_COUNTER_CORR_GRP7
+#define PARITY_COUNTER_CORR_GRP7__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_CORR_GRP7__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_CORR_GRP7__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP7__ResetEn_MASK                                                                0x80000000L
+//PARITY_COUNTER_CORR_GRP10
+#define PARITY_COUNTER_CORR_GRP10__ThresholdCounter__SHIFT                                                    0x0
+#define PARITY_COUNTER_CORR_GRP10__ResetEn__SHIFT                                                             0x1f
+#define PARITY_COUNTER_CORR_GRP10__ThresholdCounter_MASK                                                      0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP10__ResetEn_MASK                                                               0x80000000L
+//PARITY_COUNTER_CORR_GRP11
+#define PARITY_COUNTER_CORR_GRP11__ThresholdCounter__SHIFT                                                    0x0
+#define PARITY_COUNTER_CORR_GRP11__ResetEn__SHIFT                                                             0x1f
+#define PARITY_COUNTER_CORR_GRP11__ThresholdCounter_MASK                                                      0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP11__ResetEn_MASK                                                               0x80000000L
+//PARITY_COUNTER_CORR_GRP12
+#define PARITY_COUNTER_CORR_GRP12__ThresholdCounter__SHIFT                                                    0x0
+#define PARITY_COUNTER_CORR_GRP12__ResetEn__SHIFT                                                             0x1f
+#define PARITY_COUNTER_CORR_GRP12__ThresholdCounter_MASK                                                      0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP12__ResetEn_MASK                                                               0x80000000L
+//PARITY_COUNTER_CORR_GRP13
+#define PARITY_COUNTER_CORR_GRP13__ThresholdCounter__SHIFT                                                    0x0
+#define PARITY_COUNTER_CORR_GRP13__ResetEn__SHIFT                                                             0x1f
+#define PARITY_COUNTER_CORR_GRP13__ThresholdCounter_MASK                                                      0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP13__ResetEn_MASK                                                               0x80000000L
+//PARITY_COUNTER_CORR_GRP14
+#define PARITY_COUNTER_CORR_GRP14__ThresholdCounter__SHIFT                                                    0x0
+#define PARITY_COUNTER_CORR_GRP14__ResetEn__SHIFT                                                             0x1f
+#define PARITY_COUNTER_CORR_GRP14__ThresholdCounter_MASK                                                      0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP14__ResetEn_MASK                                                               0x80000000L
+//PARITY_COUNTER_CORR_GRP15
+#define PARITY_COUNTER_CORR_GRP15__ThresholdCounter__SHIFT                                                    0x0
+#define PARITY_COUNTER_CORR_GRP15__ResetEn__SHIFT                                                             0x1f
+#define PARITY_COUNTER_CORR_GRP15__ThresholdCounter_MASK                                                      0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP15__ResetEn_MASK                                                               0x80000000L
+//PARITY_COUNTER_CORR_GRP16
+#define PARITY_COUNTER_CORR_GRP16__ThresholdCounter__SHIFT                                                    0x0
+#define PARITY_COUNTER_CORR_GRP16__ResetEn__SHIFT                                                             0x1f
+#define PARITY_COUNTER_CORR_GRP16__ThresholdCounter_MASK                                                      0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP16__ResetEn_MASK                                                               0x80000000L
+//PARITY_COUNTER_CORR_GRP17
+#define PARITY_COUNTER_CORR_GRP17__ThresholdCounter__SHIFT                                                    0x0
+#define PARITY_COUNTER_CORR_GRP17__ResetEn__SHIFT                                                             0x1f
+#define PARITY_COUNTER_CORR_GRP17__ThresholdCounter_MASK                                                      0x0000FFFFL
+#define PARITY_COUNTER_CORR_GRP17__ResetEn_MASK                                                               0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP0
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id0__SHIFT                                            0x0
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id1__SHIFT                                            0x1
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id2__SHIFT                                            0x2
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id3__SHIFT                                            0x3
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id4__SHIFT                                            0x4
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id5__SHIFT                                            0x5
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id6__SHIFT                                            0x6
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id7__SHIFT                                            0x7
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id8__SHIFT                                            0x8
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id9__SHIFT                                            0x9
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id10__SHIFT                                           0xa
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id11__SHIFT                                           0xb
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id12__SHIFT                                           0xc
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id13__SHIFT                                           0xd
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id14__SHIFT                                           0xe
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id15__SHIFT                                           0xf
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id16__SHIFT                                           0x10
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id17__SHIFT                                           0x11
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id18__SHIFT                                           0x12
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id19__SHIFT                                           0x13
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id20__SHIFT                                           0x14
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id21__SHIFT                                           0x15
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id22__SHIFT                                           0x16
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id23__SHIFT                                           0x17
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id24__SHIFT                                           0x18
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id25__SHIFT                                           0x19
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id26__SHIFT                                           0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id27__SHIFT                                           0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id28__SHIFT                                           0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id29__SHIFT                                           0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id30__SHIFT                                           0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id31__SHIFT                                           0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id0_MASK                                              0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id1_MASK                                              0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id2_MASK                                              0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id3_MASK                                              0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id4_MASK                                              0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id5_MASK                                              0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id6_MASK                                              0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id7_MASK                                              0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id8_MASK                                              0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id9_MASK                                              0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id10_MASK                                             0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id11_MASK                                             0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id12_MASK                                             0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id13_MASK                                             0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id14_MASK                                             0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id15_MASK                                             0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id16_MASK                                             0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id17_MASK                                             0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id18_MASK                                             0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id19_MASK                                             0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id20_MASK                                             0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id21_MASK                                             0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id22_MASK                                             0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id23_MASK                                             0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id24_MASK                                             0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id25_MASK                                             0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id26_MASK                                             0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id27_MASK                                             0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id28_MASK                                             0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id29_MASK                                             0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id30_MASK                                             0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP0__ParityErrDetected_Id31_MASK                                             0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP1
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id0__SHIFT                                            0x0
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id1__SHIFT                                            0x1
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id2__SHIFT                                            0x2
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id3__SHIFT                                            0x3
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id4__SHIFT                                            0x4
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id5__SHIFT                                            0x5
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id6__SHIFT                                            0x6
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id7__SHIFT                                            0x7
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id8__SHIFT                                            0x8
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id9__SHIFT                                            0x9
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id10__SHIFT                                           0xa
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id11__SHIFT                                           0xb
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id12__SHIFT                                           0xc
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id13__SHIFT                                           0xd
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id14__SHIFT                                           0xe
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id15__SHIFT                                           0xf
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id16__SHIFT                                           0x10
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id17__SHIFT                                           0x11
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id18__SHIFT                                           0x12
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id19__SHIFT                                           0x13
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id20__SHIFT                                           0x14
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id21__SHIFT                                           0x15
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id22__SHIFT                                           0x16
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id23__SHIFT                                           0x17
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id24__SHIFT                                           0x18
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id25__SHIFT                                           0x19
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id26__SHIFT                                           0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id27__SHIFT                                           0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id28__SHIFT                                           0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id29__SHIFT                                           0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id30__SHIFT                                           0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id31__SHIFT                                           0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id0_MASK                                              0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id1_MASK                                              0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id2_MASK                                              0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id3_MASK                                              0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id4_MASK                                              0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id5_MASK                                              0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id6_MASK                                              0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id7_MASK                                              0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id8_MASK                                              0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id9_MASK                                              0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id10_MASK                                             0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id11_MASK                                             0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id12_MASK                                             0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id13_MASK                                             0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id14_MASK                                             0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id15_MASK                                             0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id16_MASK                                             0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id17_MASK                                             0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id18_MASK                                             0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id19_MASK                                             0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id20_MASK                                             0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id21_MASK                                             0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id22_MASK                                             0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id23_MASK                                             0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id24_MASK                                             0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id25_MASK                                             0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id26_MASK                                             0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id27_MASK                                             0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id28_MASK                                             0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id29_MASK                                             0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id30_MASK                                             0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP1__ParityErrDetected_Id31_MASK                                             0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP2
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id0__SHIFT                                            0x0
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id1__SHIFT                                            0x1
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id2__SHIFT                                            0x2
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id3__SHIFT                                            0x3
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id4__SHIFT                                            0x4
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id5__SHIFT                                            0x5
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id6__SHIFT                                            0x6
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id7__SHIFT                                            0x7
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id8__SHIFT                                            0x8
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id9__SHIFT                                            0x9
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id10__SHIFT                                           0xa
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id11__SHIFT                                           0xb
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id12__SHIFT                                           0xc
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id13__SHIFT                                           0xd
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id14__SHIFT                                           0xe
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id15__SHIFT                                           0xf
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id16__SHIFT                                           0x10
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id17__SHIFT                                           0x11
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id18__SHIFT                                           0x12
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id19__SHIFT                                           0x13
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id20__SHIFT                                           0x14
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id21__SHIFT                                           0x15
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id22__SHIFT                                           0x16
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id23__SHIFT                                           0x17
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id24__SHIFT                                           0x18
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id25__SHIFT                                           0x19
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id26__SHIFT                                           0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id27__SHIFT                                           0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id28__SHIFT                                           0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id29__SHIFT                                           0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id30__SHIFT                                           0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id31__SHIFT                                           0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id0_MASK                                              0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id1_MASK                                              0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id2_MASK                                              0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id3_MASK                                              0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id4_MASK                                              0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id5_MASK                                              0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id6_MASK                                              0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id7_MASK                                              0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id8_MASK                                              0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id9_MASK                                              0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id10_MASK                                             0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id11_MASK                                             0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id12_MASK                                             0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id13_MASK                                             0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id14_MASK                                             0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id15_MASK                                             0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id16_MASK                                             0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id17_MASK                                             0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id18_MASK                                             0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id19_MASK                                             0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id20_MASK                                             0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id21_MASK                                             0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id22_MASK                                             0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id23_MASK                                             0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id24_MASK                                             0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id25_MASK                                             0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id26_MASK                                             0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id27_MASK                                             0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id28_MASK                                             0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id29_MASK                                             0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id30_MASK                                             0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP2__ParityErrDetected_Id31_MASK                                             0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP3
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id0__SHIFT                                            0x0
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id1__SHIFT                                            0x1
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id2__SHIFT                                            0x2
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id3__SHIFT                                            0x3
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id4__SHIFT                                            0x4
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id5__SHIFT                                            0x5
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id6__SHIFT                                            0x6
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id7__SHIFT                                            0x7
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id8__SHIFT                                            0x8
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id9__SHIFT                                            0x9
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id10__SHIFT                                           0xa
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id11__SHIFT                                           0xb
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id12__SHIFT                                           0xc
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id13__SHIFT                                           0xd
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id14__SHIFT                                           0xe
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id15__SHIFT                                           0xf
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id16__SHIFT                                           0x10
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id17__SHIFT                                           0x11
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id18__SHIFT                                           0x12
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id19__SHIFT                                           0x13
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id20__SHIFT                                           0x14
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id21__SHIFT                                           0x15
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id22__SHIFT                                           0x16
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id23__SHIFT                                           0x17
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id24__SHIFT                                           0x18
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id25__SHIFT                                           0x19
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id26__SHIFT                                           0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id27__SHIFT                                           0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id28__SHIFT                                           0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id29__SHIFT                                           0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id30__SHIFT                                           0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id31__SHIFT                                           0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id0_MASK                                              0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id1_MASK                                              0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id2_MASK                                              0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id3_MASK                                              0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id4_MASK                                              0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id5_MASK                                              0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id6_MASK                                              0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id7_MASK                                              0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id8_MASK                                              0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id9_MASK                                              0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id10_MASK                                             0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id11_MASK                                             0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id12_MASK                                             0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id13_MASK                                             0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id14_MASK                                             0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id15_MASK                                             0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id16_MASK                                             0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id17_MASK                                             0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id18_MASK                                             0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id19_MASK                                             0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id20_MASK                                             0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id21_MASK                                             0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id22_MASK                                             0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id23_MASK                                             0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id24_MASK                                             0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id25_MASK                                             0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id26_MASK                                             0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id27_MASK                                             0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id28_MASK                                             0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id29_MASK                                             0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id30_MASK                                             0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP3__ParityErrDetected_Id31_MASK                                             0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP4
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id0__SHIFT                                            0x0
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id1__SHIFT                                            0x1
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id2__SHIFT                                            0x2
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id3__SHIFT                                            0x3
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id4__SHIFT                                            0x4
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id5__SHIFT                                            0x5
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id6__SHIFT                                            0x6
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id7__SHIFT                                            0x7
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id8__SHIFT                                            0x8
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id9__SHIFT                                            0x9
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id10__SHIFT                                           0xa
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id11__SHIFT                                           0xb
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id12__SHIFT                                           0xc
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id13__SHIFT                                           0xd
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id14__SHIFT                                           0xe
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id15__SHIFT                                           0xf
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id16__SHIFT                                           0x10
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id17__SHIFT                                           0x11
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id18__SHIFT                                           0x12
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id19__SHIFT                                           0x13
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id20__SHIFT                                           0x14
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id21__SHIFT                                           0x15
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id22__SHIFT                                           0x16
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id23__SHIFT                                           0x17
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id24__SHIFT                                           0x18
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id25__SHIFT                                           0x19
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id26__SHIFT                                           0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id27__SHIFT                                           0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id28__SHIFT                                           0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id29__SHIFT                                           0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id30__SHIFT                                           0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id31__SHIFT                                           0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id0_MASK                                              0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id1_MASK                                              0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id2_MASK                                              0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id3_MASK                                              0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id4_MASK                                              0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id5_MASK                                              0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id6_MASK                                              0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id7_MASK                                              0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id8_MASK                                              0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id9_MASK                                              0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id10_MASK                                             0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id11_MASK                                             0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id12_MASK                                             0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id13_MASK                                             0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id14_MASK                                             0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id15_MASK                                             0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id16_MASK                                             0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id17_MASK                                             0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id18_MASK                                             0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id19_MASK                                             0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id20_MASK                                             0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id21_MASK                                             0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id22_MASK                                             0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id23_MASK                                             0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id24_MASK                                             0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id25_MASK                                             0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id26_MASK                                             0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id27_MASK                                             0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id28_MASK                                             0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id29_MASK                                             0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id30_MASK                                             0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP4__ParityErrDetected_Id31_MASK                                             0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP5
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id0__SHIFT                                            0x0
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id1__SHIFT                                            0x1
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id2__SHIFT                                            0x2
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id3__SHIFT                                            0x3
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id4__SHIFT                                            0x4
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id5__SHIFT                                            0x5
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id6__SHIFT                                            0x6
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id7__SHIFT                                            0x7
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id8__SHIFT                                            0x8
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id9__SHIFT                                            0x9
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id10__SHIFT                                           0xa
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id11__SHIFT                                           0xb
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id12__SHIFT                                           0xc
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id13__SHIFT                                           0xd
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id14__SHIFT                                           0xe
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id15__SHIFT                                           0xf
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id16__SHIFT                                           0x10
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id17__SHIFT                                           0x11
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id18__SHIFT                                           0x12
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id19__SHIFT                                           0x13
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id20__SHIFT                                           0x14
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id21__SHIFT                                           0x15
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id22__SHIFT                                           0x16
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id23__SHIFT                                           0x17
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id24__SHIFT                                           0x18
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id25__SHIFT                                           0x19
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id26__SHIFT                                           0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id27__SHIFT                                           0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id28__SHIFT                                           0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id29__SHIFT                                           0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id30__SHIFT                                           0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id31__SHIFT                                           0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id0_MASK                                              0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id1_MASK                                              0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id2_MASK                                              0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id3_MASK                                              0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id4_MASK                                              0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id5_MASK                                              0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id6_MASK                                              0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id7_MASK                                              0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id8_MASK                                              0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id9_MASK                                              0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id10_MASK                                             0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id11_MASK                                             0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id12_MASK                                             0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id13_MASK                                             0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id14_MASK                                             0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id15_MASK                                             0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id16_MASK                                             0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id17_MASK                                             0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id18_MASK                                             0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id19_MASK                                             0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id20_MASK                                             0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id21_MASK                                             0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id22_MASK                                             0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id23_MASK                                             0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id24_MASK                                             0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id25_MASK                                             0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id26_MASK                                             0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id27_MASK                                             0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id28_MASK                                             0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id29_MASK                                             0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id30_MASK                                             0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP5__ParityErrDetected_Id31_MASK                                             0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP6
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id0__SHIFT                                            0x0
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id1__SHIFT                                            0x1
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id2__SHIFT                                            0x2
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id3__SHIFT                                            0x3
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id4__SHIFT                                            0x4
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id5__SHIFT                                            0x5
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id6__SHIFT                                            0x6
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id7__SHIFT                                            0x7
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id8__SHIFT                                            0x8
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id9__SHIFT                                            0x9
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id10__SHIFT                                           0xa
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id11__SHIFT                                           0xb
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id12__SHIFT                                           0xc
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id13__SHIFT                                           0xd
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id14__SHIFT                                           0xe
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id15__SHIFT                                           0xf
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id16__SHIFT                                           0x10
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id17__SHIFT                                           0x11
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id18__SHIFT                                           0x12
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id19__SHIFT                                           0x13
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id20__SHIFT                                           0x14
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id21__SHIFT                                           0x15
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id22__SHIFT                                           0x16
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id23__SHIFT                                           0x17
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id24__SHIFT                                           0x18
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id25__SHIFT                                           0x19
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id26__SHIFT                                           0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id27__SHIFT                                           0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id28__SHIFT                                           0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id29__SHIFT                                           0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id30__SHIFT                                           0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id31__SHIFT                                           0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id0_MASK                                              0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id1_MASK                                              0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id2_MASK                                              0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id3_MASK                                              0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id4_MASK                                              0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id5_MASK                                              0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id6_MASK                                              0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id7_MASK                                              0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id8_MASK                                              0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id9_MASK                                              0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id10_MASK                                             0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id11_MASK                                             0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id12_MASK                                             0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id13_MASK                                             0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id14_MASK                                             0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id15_MASK                                             0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id16_MASK                                             0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id17_MASK                                             0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id18_MASK                                             0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id19_MASK                                             0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id20_MASK                                             0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id21_MASK                                             0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id22_MASK                                             0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id23_MASK                                             0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id24_MASK                                             0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id25_MASK                                             0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id26_MASK                                             0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id27_MASK                                             0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id28_MASK                                             0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id29_MASK                                             0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id30_MASK                                             0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP6__ParityErrDetected_Id31_MASK                                             0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP7
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id0__SHIFT                                            0x0
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id1__SHIFT                                            0x1
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id2__SHIFT                                            0x2
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id3__SHIFT                                            0x3
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id4__SHIFT                                            0x4
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id5__SHIFT                                            0x5
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id6__SHIFT                                            0x6
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id7__SHIFT                                            0x7
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id8__SHIFT                                            0x8
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id9__SHIFT                                            0x9
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id10__SHIFT                                           0xa
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id11__SHIFT                                           0xb
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id12__SHIFT                                           0xc
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id13__SHIFT                                           0xd
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id14__SHIFT                                           0xe
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id15__SHIFT                                           0xf
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id16__SHIFT                                           0x10
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id17__SHIFT                                           0x11
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id18__SHIFT                                           0x12
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id19__SHIFT                                           0x13
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id20__SHIFT                                           0x14
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id21__SHIFT                                           0x15
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id22__SHIFT                                           0x16
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id23__SHIFT                                           0x17
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id24__SHIFT                                           0x18
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id25__SHIFT                                           0x19
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id26__SHIFT                                           0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id27__SHIFT                                           0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id28__SHIFT                                           0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id29__SHIFT                                           0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id30__SHIFT                                           0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id31__SHIFT                                           0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id0_MASK                                              0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id1_MASK                                              0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id2_MASK                                              0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id3_MASK                                              0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id4_MASK                                              0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id5_MASK                                              0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id6_MASK                                              0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id7_MASK                                              0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id8_MASK                                              0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id9_MASK                                              0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id10_MASK                                             0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id11_MASK                                             0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id12_MASK                                             0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id13_MASK                                             0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id14_MASK                                             0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id15_MASK                                             0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id16_MASK                                             0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id17_MASK                                             0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id18_MASK                                             0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id19_MASK                                             0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id20_MASK                                             0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id21_MASK                                             0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id22_MASK                                             0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id23_MASK                                             0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id24_MASK                                             0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id25_MASK                                             0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id26_MASK                                             0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id27_MASK                                             0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id28_MASK                                             0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id29_MASK                                             0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id30_MASK                                             0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP7__ParityErrDetected_Id31_MASK                                             0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP10
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP10__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP11
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP11__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_ERROR_STATUS_UCP_GRP12
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id0__SHIFT                                           0x0
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id1__SHIFT                                           0x1
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id2__SHIFT                                           0x2
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id3__SHIFT                                           0x3
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id4__SHIFT                                           0x4
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id5__SHIFT                                           0x5
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id6__SHIFT                                           0x6
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id7__SHIFT                                           0x7
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id8__SHIFT                                           0x8
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id9__SHIFT                                           0x9
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id10__SHIFT                                          0xa
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id11__SHIFT                                          0xb
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id12__SHIFT                                          0xc
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id13__SHIFT                                          0xd
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id14__SHIFT                                          0xe
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id15__SHIFT                                          0xf
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id16__SHIFT                                          0x10
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id17__SHIFT                                          0x11
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id18__SHIFT                                          0x12
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id19__SHIFT                                          0x13
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id20__SHIFT                                          0x14
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id21__SHIFT                                          0x15
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id22__SHIFT                                          0x16
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id23__SHIFT                                          0x17
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id24__SHIFT                                          0x18
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id25__SHIFT                                          0x19
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id26__SHIFT                                          0x1a
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id27__SHIFT                                          0x1b
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id28__SHIFT                                          0x1c
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id29__SHIFT                                          0x1d
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id30__SHIFT                                          0x1e
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id31__SHIFT                                          0x1f
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id0_MASK                                             0x00000001L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id1_MASK                                             0x00000002L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id2_MASK                                             0x00000004L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id3_MASK                                             0x00000008L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id4_MASK                                             0x00000010L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id5_MASK                                             0x00000020L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id6_MASK                                             0x00000040L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id7_MASK                                             0x00000080L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id8_MASK                                             0x00000100L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id9_MASK                                             0x00000200L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id10_MASK                                            0x00000400L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id11_MASK                                            0x00000800L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id12_MASK                                            0x00001000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id13_MASK                                            0x00002000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id14_MASK                                            0x00004000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id15_MASK                                            0x00008000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id16_MASK                                            0x00010000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id17_MASK                                            0x00020000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id18_MASK                                            0x00040000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id19_MASK                                            0x00080000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id20_MASK                                            0x00100000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id21_MASK                                            0x00200000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id22_MASK                                            0x00400000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id23_MASK                                            0x00800000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id24_MASK                                            0x01000000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id25_MASK                                            0x02000000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id26_MASK                                            0x04000000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id27_MASK                                            0x08000000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id28_MASK                                            0x10000000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id29_MASK                                            0x20000000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id30_MASK                                            0x40000000L
+#define PARITY_ERROR_STATUS_UCP_GRP12__ParityErrDetected_Id31_MASK                                            0x80000000L
+//PARITY_COUNTER_UCP_GRP0
+#define PARITY_COUNTER_UCP_GRP0__ThresholdCounter__SHIFT                                                      0x0
+#define PARITY_COUNTER_UCP_GRP0__ResetEn__SHIFT                                                               0x1f
+#define PARITY_COUNTER_UCP_GRP0__ThresholdCounter_MASK                                                        0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP0__ResetEn_MASK                                                                 0x80000000L
+//PARITY_COUNTER_UCP_GRP1
+#define PARITY_COUNTER_UCP_GRP1__ThresholdCounter__SHIFT                                                      0x0
+#define PARITY_COUNTER_UCP_GRP1__ResetEn__SHIFT                                                               0x1f
+#define PARITY_COUNTER_UCP_GRP1__ThresholdCounter_MASK                                                        0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP1__ResetEn_MASK                                                                 0x80000000L
+//PARITY_COUNTER_UCP_GRP2
+#define PARITY_COUNTER_UCP_GRP2__ThresholdCounter__SHIFT                                                      0x0
+#define PARITY_COUNTER_UCP_GRP2__ResetEn__SHIFT                                                               0x1f
+#define PARITY_COUNTER_UCP_GRP2__ThresholdCounter_MASK                                                        0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP2__ResetEn_MASK                                                                 0x80000000L
+//PARITY_COUNTER_UCP_GRP3
+#define PARITY_COUNTER_UCP_GRP3__ThresholdCounter__SHIFT                                                      0x0
+#define PARITY_COUNTER_UCP_GRP3__ResetEn__SHIFT                                                               0x1f
+#define PARITY_COUNTER_UCP_GRP3__ThresholdCounter_MASK                                                        0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP3__ResetEn_MASK                                                                 0x80000000L
+//PARITY_COUNTER_UCP_GRP4
+#define PARITY_COUNTER_UCP_GRP4__ThresholdCounter__SHIFT                                                      0x0
+#define PARITY_COUNTER_UCP_GRP4__ResetEn__SHIFT                                                               0x1f
+#define PARITY_COUNTER_UCP_GRP4__ThresholdCounter_MASK                                                        0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP4__ResetEn_MASK                                                                 0x80000000L
+//PARITY_COUNTER_UCP_GRP5
+#define PARITY_COUNTER_UCP_GRP5__ThresholdCounter__SHIFT                                                      0x0
+#define PARITY_COUNTER_UCP_GRP5__ResetEn__SHIFT                                                               0x1f
+#define PARITY_COUNTER_UCP_GRP5__ThresholdCounter_MASK                                                        0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP5__ResetEn_MASK                                                                 0x80000000L
+//PARITY_COUNTER_UCP_GRP6
+#define PARITY_COUNTER_UCP_GRP6__ThresholdCounter__SHIFT                                                      0x0
+#define PARITY_COUNTER_UCP_GRP6__ResetEn__SHIFT                                                               0x1f
+#define PARITY_COUNTER_UCP_GRP6__ThresholdCounter_MASK                                                        0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP6__ResetEn_MASK                                                                 0x80000000L
+//PARITY_COUNTER_UCP_GRP7
+#define PARITY_COUNTER_UCP_GRP7__ThresholdCounter__SHIFT                                                      0x0
+#define PARITY_COUNTER_UCP_GRP7__ResetEn__SHIFT                                                               0x1f
+#define PARITY_COUNTER_UCP_GRP7__ThresholdCounter_MASK                                                        0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP7__ResetEn_MASK                                                                 0x80000000L
+//PARITY_COUNTER_UCP_GRP10
+#define PARITY_COUNTER_UCP_GRP10__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_UCP_GRP10__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_UCP_GRP10__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP10__ResetEn_MASK                                                                0x80000000L
+//PARITY_COUNTER_UCP_GRP11
+#define PARITY_COUNTER_UCP_GRP11__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_UCP_GRP11__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_UCP_GRP11__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP11__ResetEn_MASK                                                                0x80000000L
+//PARITY_COUNTER_UCP_GRP12
+#define PARITY_COUNTER_UCP_GRP12__ThresholdCounter__SHIFT                                                     0x0
+#define PARITY_COUNTER_UCP_GRP12__ResetEn__SHIFT                                                              0x1f
+#define PARITY_COUNTER_UCP_GRP12__ThresholdCounter_MASK                                                       0x0000FFFFL
+#define PARITY_COUNTER_UCP_GRP12__ResetEn_MASK                                                                0x80000000L
+//MISC_SEVERITY_CONTROL
+#define MISC_SEVERITY_CONTROL__ErrEventErrSev__SHIFT                                                          0x4
+#define MISC_SEVERITY_CONTROL__PcieParityErrSev__SHIFT                                                        0x6
+#define MISC_SEVERITY_CONTROL__ErrEventErrSev_MASK                                                            0x00000030L
+#define MISC_SEVERITY_CONTROL__PcieParityErrSev_MASK                                                          0x000000C0L
+//MISC_RAS_CONTROL
+#define MISC_RAS_CONTROL__PIN_NMI_SyncFlood_En__SHIFT                                                         0x2
+#define MISC_RAS_CONTROL__GNB_SB_LinkNeverDis__SHIFT                                                          0x3
+#define MISC_RAS_CONTROL__InterruptOutputDis__SHIFT                                                           0x9
+#define MISC_RAS_CONTROL__LinkDisOutputDis__SHIFT                                                             0xa
+#define MISC_RAS_CONTROL__SyncFldOutputDis__SHIFT                                                             0xb
+#define MISC_RAS_CONTROL__PCIe_NMI_En__SHIFT                                                                  0xc
+#define MISC_RAS_CONTROL__PCIe_SCI_En__SHIFT                                                                  0xd
+#define MISC_RAS_CONTROL__PCIe_SMI_En__SHIFT                                                                  0xe
+#define MISC_RAS_CONTROL__SW_SCI_En__SHIFT                                                                    0xf
+#define MISC_RAS_CONTROL__SW_SMI_En__SHIFT                                                                    0x10
+#define MISC_RAS_CONTROL__SW_NMI_En__SHIFT                                                                    0x11
+#define MISC_RAS_CONTROL__PIN_NMI_SyncFlood_En_MASK                                                           0x00000004L
+#define MISC_RAS_CONTROL__GNB_SB_LinkNeverDis_MASK                                                            0x00000008L
+#define MISC_RAS_CONTROL__InterruptOutputDis_MASK                                                             0x00000200L
+#define MISC_RAS_CONTROL__LinkDisOutputDis_MASK                                                               0x00000400L
+#define MISC_RAS_CONTROL__SyncFldOutputDis_MASK                                                               0x00000800L
+#define MISC_RAS_CONTROL__PCIe_NMI_En_MASK                                                                    0x00001000L
+#define MISC_RAS_CONTROL__PCIe_SCI_En_MASK                                                                    0x00002000L
+#define MISC_RAS_CONTROL__PCIe_SMI_En_MASK                                                                    0x00004000L
+#define MISC_RAS_CONTROL__SW_SCI_En_MASK                                                                      0x00008000L
+#define MISC_RAS_CONTROL__SW_SMI_En_MASK                                                                      0x00010000L
+#define MISC_RAS_CONTROL__SW_NMI_En_MASK                                                                      0x00020000L
+//RAS_SCRATCH_0
+#define RAS_SCRATCH_0__SCRATCH_0__SHIFT                                                                       0x0
+#define RAS_SCRATCH_0__SCRATCH_0_MASK                                                                         0xFFFFFFFFL
+//RAS_SCRATCH_1
+#define RAS_SCRATCH_1__SCRATCH_1__SHIFT                                                                       0x0
+#define RAS_SCRATCH_1__SCRATCH_1_MASK                                                                         0xFFFFFFFFL
+//ErrEvent_ACTION_CONTROL
+#define ErrEvent_ACTION_CONTROL__APML_ERR_En__SHIFT                                                           0x0
+#define ErrEvent_ACTION_CONTROL__IntrGenSel__SHIFT                                                            0x1
+#define ErrEvent_ACTION_CONTROL__LinkDis_En__SHIFT                                                            0x3
+#define ErrEvent_ACTION_CONTROL__SyncFlood_En__SHIFT                                                          0x4
+#define ErrEvent_ACTION_CONTROL__APML_ERR_En_MASK                                                             0x00000001L
+#define ErrEvent_ACTION_CONTROL__IntrGenSel_MASK                                                              0x00000006L
+#define ErrEvent_ACTION_CONTROL__LinkDis_En_MASK                                                              0x00000008L
+#define ErrEvent_ACTION_CONTROL__SyncFlood_En_MASK                                                            0x00000010L
+//ParitySerr_ACTION_CONTROL
+#define ParitySerr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                         0x0
+#define ParitySerr_ACTION_CONTROL__IntrGenSel__SHIFT                                                          0x1
+#define ParitySerr_ACTION_CONTROL__LinkDis_En__SHIFT                                                          0x3
+#define ParitySerr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                        0x4
+#define ParitySerr_ACTION_CONTROL__APML_ERR_En_MASK                                                           0x00000001L
+#define ParitySerr_ACTION_CONTROL__IntrGenSel_MASK                                                            0x00000006L
+#define ParitySerr_ACTION_CONTROL__LinkDis_En_MASK                                                            0x00000008L
+#define ParitySerr_ACTION_CONTROL__SyncFlood_En_MASK                                                          0x00000010L
+//ParityFatal_ACTION_CONTROL
+#define ParityFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                        0x0
+#define ParityFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                         0x1
+#define ParityFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                         0x3
+#define ParityFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                       0x4
+#define ParityFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                          0x00000001L
+#define ParityFatal_ACTION_CONTROL__IntrGenSel_MASK                                                           0x00000006L
+#define ParityFatal_ACTION_CONTROL__LinkDis_En_MASK                                                           0x00000008L
+#define ParityFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                         0x00000010L
+//ParityNonFatal_ACTION_CONTROL
+#define ParityNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                     0x0
+#define ParityNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                      0x1
+#define ParityNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                      0x3
+#define ParityNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                    0x4
+#define ParityNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                       0x00000001L
+#define ParityNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                        0x00000006L
+#define ParityNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                        0x00000008L
+#define ParityNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                      0x00000010L
+//ParityCorr_ACTION_CONTROL
+#define ParityCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                         0x0
+#define ParityCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                          0x1
+#define ParityCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                          0x3
+#define ParityCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                        0x4
+#define ParityCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                           0x00000001L
+#define ParityCorr_ACTION_CONTROL__IntrGenSel_MASK                                                            0x00000006L
+#define ParityCorr_ACTION_CONTROL__LinkDis_En_MASK                                                            0x00000008L
+#define ParityCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                          0x00000010L
+//PCIE0PortASerr_ACTION_CONTROL
+#define PCIE0PortASerr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                     0x0
+#define PCIE0PortASerr_ACTION_CONTROL__IntrGenSel__SHIFT                                                      0x1
+#define PCIE0PortASerr_ACTION_CONTROL__LinkDis_En__SHIFT                                                      0x3
+#define PCIE0PortASerr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                    0x4
+#define PCIE0PortASerr_ACTION_CONTROL__APML_ERR_En_MASK                                                       0x00000001L
+#define PCIE0PortASerr_ACTION_CONTROL__IntrGenSel_MASK                                                        0x00000006L
+#define PCIE0PortASerr_ACTION_CONTROL__LinkDis_En_MASK                                                        0x00000008L
+#define PCIE0PortASerr_ACTION_CONTROL__SyncFlood_En_MASK                                                      0x00000010L
+//PCIE0PortAIntFatal_ACTION_CONTROL
+#define PCIE0PortAIntFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortAIntFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortAIntFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortAIntFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortAIntFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortAIntFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortAIntFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortAIntFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortAIntNonFatal_ACTION_CONTROL
+#define PCIE0PortAIntNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortAIntNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortAIntNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortAIntNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortAIntNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortAIntNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortAIntNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortAIntNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortAIntCorr_ACTION_CONTROL
+#define PCIE0PortAIntCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortAIntCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortAIntCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortAIntCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortAIntCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortAIntCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortAIntCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortAIntCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortAExtFatal_ACTION_CONTROL
+#define PCIE0PortAExtFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortAExtFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortAExtFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortAExtFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortAExtFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortAExtFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortAExtFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortAExtFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortAExtNonFatal_ACTION_CONTROL
+#define PCIE0PortAExtNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortAExtNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortAExtNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortAExtNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortAExtNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortAExtNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortAExtNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortAExtNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortAExtCorr_ACTION_CONTROL
+#define PCIE0PortAExtCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortAExtCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortAExtCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortAExtCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortAExtCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortAExtCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortAExtCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortAExtCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortAParityErr_ACTION_CONTROL
+#define PCIE0PortAParityErr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                0x0
+#define PCIE0PortAParityErr_ACTION_CONTROL__IntrGenSel__SHIFT                                                 0x1
+#define PCIE0PortAParityErr_ACTION_CONTROL__LinkDis_En__SHIFT                                                 0x3
+#define PCIE0PortAParityErr_ACTION_CONTROL__SyncFlood_En__SHIFT                                               0x4
+#define PCIE0PortAParityErr_ACTION_CONTROL__APML_ERR_En_MASK                                                  0x00000001L
+#define PCIE0PortAParityErr_ACTION_CONTROL__IntrGenSel_MASK                                                   0x00000006L
+#define PCIE0PortAParityErr_ACTION_CONTROL__LinkDis_En_MASK                                                   0x00000008L
+#define PCIE0PortAParityErr_ACTION_CONTROL__SyncFlood_En_MASK                                                 0x00000010L
+//PCIE0PortBSerr_ACTION_CONTROL
+#define PCIE0PortBSerr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                     0x0
+#define PCIE0PortBSerr_ACTION_CONTROL__IntrGenSel__SHIFT                                                      0x1
+#define PCIE0PortBSerr_ACTION_CONTROL__LinkDis_En__SHIFT                                                      0x3
+#define PCIE0PortBSerr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                    0x4
+#define PCIE0PortBSerr_ACTION_CONTROL__APML_ERR_En_MASK                                                       0x00000001L
+#define PCIE0PortBSerr_ACTION_CONTROL__IntrGenSel_MASK                                                        0x00000006L
+#define PCIE0PortBSerr_ACTION_CONTROL__LinkDis_En_MASK                                                        0x00000008L
+#define PCIE0PortBSerr_ACTION_CONTROL__SyncFlood_En_MASK                                                      0x00000010L
+//PCIE0PortBIntFatal_ACTION_CONTROL
+#define PCIE0PortBIntFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortBIntFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortBIntFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortBIntFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortBIntFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortBIntFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortBIntFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortBIntFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortBIntNonFatal_ACTION_CONTROL
+#define PCIE0PortBIntNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortBIntNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortBIntNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortBIntNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortBIntNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortBIntNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortBIntNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortBIntNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortBIntCorr_ACTION_CONTROL
+#define PCIE0PortBIntCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortBIntCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortBIntCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortBIntCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortBIntCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortBIntCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortBIntCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortBIntCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortBExtFatal_ACTION_CONTROL
+#define PCIE0PortBExtFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortBExtFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortBExtFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortBExtFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortBExtFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortBExtFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortBExtFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortBExtFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortBExtNonFatal_ACTION_CONTROL
+#define PCIE0PortBExtNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortBExtNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortBExtNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortBExtNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortBExtNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortBExtNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortBExtNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortBExtNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortBExtCorr_ACTION_CONTROL
+#define PCIE0PortBExtCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortBExtCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortBExtCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortBExtCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortBExtCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortBExtCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortBExtCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortBExtCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortBParityErr_ACTION_CONTROL
+#define PCIE0PortBParityErr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                0x0
+#define PCIE0PortBParityErr_ACTION_CONTROL__IntrGenSel__SHIFT                                                 0x1
+#define PCIE0PortBParityErr_ACTION_CONTROL__LinkDis_En__SHIFT                                                 0x3
+#define PCIE0PortBParityErr_ACTION_CONTROL__SyncFlood_En__SHIFT                                               0x4
+#define PCIE0PortBParityErr_ACTION_CONTROL__APML_ERR_En_MASK                                                  0x00000001L
+#define PCIE0PortBParityErr_ACTION_CONTROL__IntrGenSel_MASK                                                   0x00000006L
+#define PCIE0PortBParityErr_ACTION_CONTROL__LinkDis_En_MASK                                                   0x00000008L
+#define PCIE0PortBParityErr_ACTION_CONTROL__SyncFlood_En_MASK                                                 0x00000010L
+//PCIE0PortCSerr_ACTION_CONTROL
+#define PCIE0PortCSerr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                     0x0
+#define PCIE0PortCSerr_ACTION_CONTROL__IntrGenSel__SHIFT                                                      0x1
+#define PCIE0PortCSerr_ACTION_CONTROL__LinkDis_En__SHIFT                                                      0x3
+#define PCIE0PortCSerr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                    0x4
+#define PCIE0PortCSerr_ACTION_CONTROL__APML_ERR_En_MASK                                                       0x00000001L
+#define PCIE0PortCSerr_ACTION_CONTROL__IntrGenSel_MASK                                                        0x00000006L
+#define PCIE0PortCSerr_ACTION_CONTROL__LinkDis_En_MASK                                                        0x00000008L
+#define PCIE0PortCSerr_ACTION_CONTROL__SyncFlood_En_MASK                                                      0x00000010L
+//PCIE0PortCIntFatal_ACTION_CONTROL
+#define PCIE0PortCIntFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortCIntFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortCIntFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortCIntFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortCIntFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortCIntFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortCIntFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortCIntFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortCIntNonFatal_ACTION_CONTROL
+#define PCIE0PortCIntNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortCIntNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortCIntNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortCIntNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortCIntNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortCIntNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortCIntNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortCIntNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortCIntCorr_ACTION_CONTROL
+#define PCIE0PortCIntCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortCIntCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortCIntCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortCIntCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortCIntCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortCIntCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortCIntCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortCIntCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortCExtFatal_ACTION_CONTROL
+#define PCIE0PortCExtFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortCExtFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortCExtFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortCExtFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortCExtFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortCExtFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortCExtFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortCExtFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortCExtNonFatal_ACTION_CONTROL
+#define PCIE0PortCExtNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortCExtNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortCExtNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortCExtNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortCExtNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortCExtNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortCExtNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortCExtNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortCExtCorr_ACTION_CONTROL
+#define PCIE0PortCExtCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortCExtCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortCExtCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortCExtCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortCExtCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortCExtCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortCExtCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortCExtCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortCParityErr_ACTION_CONTROL
+#define PCIE0PortCParityErr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                0x0
+#define PCIE0PortCParityErr_ACTION_CONTROL__IntrGenSel__SHIFT                                                 0x1
+#define PCIE0PortCParityErr_ACTION_CONTROL__LinkDis_En__SHIFT                                                 0x3
+#define PCIE0PortCParityErr_ACTION_CONTROL__SyncFlood_En__SHIFT                                               0x4
+#define PCIE0PortCParityErr_ACTION_CONTROL__APML_ERR_En_MASK                                                  0x00000001L
+#define PCIE0PortCParityErr_ACTION_CONTROL__IntrGenSel_MASK                                                   0x00000006L
+#define PCIE0PortCParityErr_ACTION_CONTROL__LinkDis_En_MASK                                                   0x00000008L
+#define PCIE0PortCParityErr_ACTION_CONTROL__SyncFlood_En_MASK                                                 0x00000010L
+//PCIE0PortDSerr_ACTION_CONTROL
+#define PCIE0PortDSerr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                     0x0
+#define PCIE0PortDSerr_ACTION_CONTROL__IntrGenSel__SHIFT                                                      0x1
+#define PCIE0PortDSerr_ACTION_CONTROL__LinkDis_En__SHIFT                                                      0x3
+#define PCIE0PortDSerr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                    0x4
+#define PCIE0PortDSerr_ACTION_CONTROL__APML_ERR_En_MASK                                                       0x00000001L
+#define PCIE0PortDSerr_ACTION_CONTROL__IntrGenSel_MASK                                                        0x00000006L
+#define PCIE0PortDSerr_ACTION_CONTROL__LinkDis_En_MASK                                                        0x00000008L
+#define PCIE0PortDSerr_ACTION_CONTROL__SyncFlood_En_MASK                                                      0x00000010L
+//PCIE0PortDIntFatal_ACTION_CONTROL
+#define PCIE0PortDIntFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortDIntFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortDIntFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortDIntFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortDIntFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortDIntFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortDIntFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortDIntFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortDIntNonFatal_ACTION_CONTROL
+#define PCIE0PortDIntNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortDIntNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortDIntNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortDIntNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortDIntNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortDIntNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortDIntNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortDIntNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortDIntCorr_ACTION_CONTROL
+#define PCIE0PortDIntCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortDIntCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortDIntCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortDIntCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortDIntCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortDIntCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortDIntCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortDIntCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortDExtFatal_ACTION_CONTROL
+#define PCIE0PortDExtFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortDExtFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortDExtFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortDExtFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortDExtFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortDExtFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortDExtFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortDExtFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortDExtNonFatal_ACTION_CONTROL
+#define PCIE0PortDExtNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortDExtNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortDExtNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortDExtNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortDExtNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortDExtNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortDExtNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortDExtNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortDExtCorr_ACTION_CONTROL
+#define PCIE0PortDExtCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortDExtCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortDExtCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortDExtCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortDExtCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortDExtCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortDExtCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortDExtCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortDParityErr_ACTION_CONTROL
+#define PCIE0PortDParityErr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                0x0
+#define PCIE0PortDParityErr_ACTION_CONTROL__IntrGenSel__SHIFT                                                 0x1
+#define PCIE0PortDParityErr_ACTION_CONTROL__LinkDis_En__SHIFT                                                 0x3
+#define PCIE0PortDParityErr_ACTION_CONTROL__SyncFlood_En__SHIFT                                               0x4
+#define PCIE0PortDParityErr_ACTION_CONTROL__APML_ERR_En_MASK                                                  0x00000001L
+#define PCIE0PortDParityErr_ACTION_CONTROL__IntrGenSel_MASK                                                   0x00000006L
+#define PCIE0PortDParityErr_ACTION_CONTROL__LinkDis_En_MASK                                                   0x00000008L
+#define PCIE0PortDParityErr_ACTION_CONTROL__SyncFlood_En_MASK                                                 0x00000010L
+//PCIE0PortESerr_ACTION_CONTROL
+#define PCIE0PortESerr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                     0x0
+#define PCIE0PortESerr_ACTION_CONTROL__IntrGenSel__SHIFT                                                      0x1
+#define PCIE0PortESerr_ACTION_CONTROL__LinkDis_En__SHIFT                                                      0x3
+#define PCIE0PortESerr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                    0x4
+#define PCIE0PortESerr_ACTION_CONTROL__APML_ERR_En_MASK                                                       0x00000001L
+#define PCIE0PortESerr_ACTION_CONTROL__IntrGenSel_MASK                                                        0x00000006L
+#define PCIE0PortESerr_ACTION_CONTROL__LinkDis_En_MASK                                                        0x00000008L
+#define PCIE0PortESerr_ACTION_CONTROL__SyncFlood_En_MASK                                                      0x00000010L
+//PCIE0PortEIntFatal_ACTION_CONTROL
+#define PCIE0PortEIntFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortEIntFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortEIntFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortEIntFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortEIntFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortEIntFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortEIntFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortEIntFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortEIntNonFatal_ACTION_CONTROL
+#define PCIE0PortEIntNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortEIntNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortEIntNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortEIntNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortEIntNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortEIntNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortEIntNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortEIntNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortEIntCorr_ACTION_CONTROL
+#define PCIE0PortEIntCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortEIntCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortEIntCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortEIntCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortEIntCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortEIntCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortEIntCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortEIntCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortEExtFatal_ACTION_CONTROL
+#define PCIE0PortEExtFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortEExtFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortEExtFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortEExtFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortEExtFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortEExtFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortEExtFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortEExtFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortEExtNonFatal_ACTION_CONTROL
+#define PCIE0PortEExtNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortEExtNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortEExtNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortEExtNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortEExtNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortEExtNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortEExtNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortEExtNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortEExtCorr_ACTION_CONTROL
+#define PCIE0PortEExtCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortEExtCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortEExtCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortEExtCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortEExtCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortEExtCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortEExtCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortEExtCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortEParityErr_ACTION_CONTROL
+#define PCIE0PortEParityErr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                0x0
+#define PCIE0PortEParityErr_ACTION_CONTROL__IntrGenSel__SHIFT                                                 0x1
+#define PCIE0PortEParityErr_ACTION_CONTROL__LinkDis_En__SHIFT                                                 0x3
+#define PCIE0PortEParityErr_ACTION_CONTROL__SyncFlood_En__SHIFT                                               0x4
+#define PCIE0PortEParityErr_ACTION_CONTROL__APML_ERR_En_MASK                                                  0x00000001L
+#define PCIE0PortEParityErr_ACTION_CONTROL__IntrGenSel_MASK                                                   0x00000006L
+#define PCIE0PortEParityErr_ACTION_CONTROL__LinkDis_En_MASK                                                   0x00000008L
+#define PCIE0PortEParityErr_ACTION_CONTROL__SyncFlood_En_MASK                                                 0x00000010L
+//PCIE0PortFSerr_ACTION_CONTROL
+#define PCIE0PortFSerr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                     0x0
+#define PCIE0PortFSerr_ACTION_CONTROL__IntrGenSel__SHIFT                                                      0x1
+#define PCIE0PortFSerr_ACTION_CONTROL__LinkDis_En__SHIFT                                                      0x3
+#define PCIE0PortFSerr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                    0x4
+#define PCIE0PortFSerr_ACTION_CONTROL__APML_ERR_En_MASK                                                       0x00000001L
+#define PCIE0PortFSerr_ACTION_CONTROL__IntrGenSel_MASK                                                        0x00000006L
+#define PCIE0PortFSerr_ACTION_CONTROL__LinkDis_En_MASK                                                        0x00000008L
+#define PCIE0PortFSerr_ACTION_CONTROL__SyncFlood_En_MASK                                                      0x00000010L
+//PCIE0PortFIntFatal_ACTION_CONTROL
+#define PCIE0PortFIntFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortFIntFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortFIntFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortFIntFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortFIntFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortFIntFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortFIntFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortFIntFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortFIntNonFatal_ACTION_CONTROL
+#define PCIE0PortFIntNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortFIntNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortFIntNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortFIntNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortFIntNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortFIntNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortFIntNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortFIntNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortFIntCorr_ACTION_CONTROL
+#define PCIE0PortFIntCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortFIntCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortFIntCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortFIntCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortFIntCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortFIntCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortFIntCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortFIntCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortFExtFatal_ACTION_CONTROL
+#define PCIE0PortFExtFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortFExtFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortFExtFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortFExtFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortFExtFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortFExtFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortFExtFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortFExtFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortFExtNonFatal_ACTION_CONTROL
+#define PCIE0PortFExtNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortFExtNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortFExtNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortFExtNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortFExtNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortFExtNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortFExtNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortFExtNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortFExtCorr_ACTION_CONTROL
+#define PCIE0PortFExtCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortFExtCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortFExtCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortFExtCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortFExtCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortFExtCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortFExtCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortFExtCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortFParityErr_ACTION_CONTROL
+#define PCIE0PortFParityErr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                0x0
+#define PCIE0PortFParityErr_ACTION_CONTROL__IntrGenSel__SHIFT                                                 0x1
+#define PCIE0PortFParityErr_ACTION_CONTROL__LinkDis_En__SHIFT                                                 0x3
+#define PCIE0PortFParityErr_ACTION_CONTROL__SyncFlood_En__SHIFT                                               0x4
+#define PCIE0PortFParityErr_ACTION_CONTROL__APML_ERR_En_MASK                                                  0x00000001L
+#define PCIE0PortFParityErr_ACTION_CONTROL__IntrGenSel_MASK                                                   0x00000006L
+#define PCIE0PortFParityErr_ACTION_CONTROL__LinkDis_En_MASK                                                   0x00000008L
+#define PCIE0PortFParityErr_ACTION_CONTROL__SyncFlood_En_MASK                                                 0x00000010L
+//PCIE0PortGSerr_ACTION_CONTROL
+#define PCIE0PortGSerr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                     0x0
+#define PCIE0PortGSerr_ACTION_CONTROL__IntrGenSel__SHIFT                                                      0x1
+#define PCIE0PortGSerr_ACTION_CONTROL__LinkDis_En__SHIFT                                                      0x3
+#define PCIE0PortGSerr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                    0x4
+#define PCIE0PortGSerr_ACTION_CONTROL__APML_ERR_En_MASK                                                       0x00000001L
+#define PCIE0PortGSerr_ACTION_CONTROL__IntrGenSel_MASK                                                        0x00000006L
+#define PCIE0PortGSerr_ACTION_CONTROL__LinkDis_En_MASK                                                        0x00000008L
+#define PCIE0PortGSerr_ACTION_CONTROL__SyncFlood_En_MASK                                                      0x00000010L
+//PCIE0PortGIntFatal_ACTION_CONTROL
+#define PCIE0PortGIntFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortGIntFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortGIntFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortGIntFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortGIntFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortGIntFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortGIntFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortGIntFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortGIntNonFatal_ACTION_CONTROL
+#define PCIE0PortGIntNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortGIntNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortGIntNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortGIntNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortGIntNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortGIntNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortGIntNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortGIntNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortGIntCorr_ACTION_CONTROL
+#define PCIE0PortGIntCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortGIntCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortGIntCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortGIntCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortGIntCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortGIntCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortGIntCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortGIntCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortGExtFatal_ACTION_CONTROL
+#define PCIE0PortGExtFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define PCIE0PortGExtFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define PCIE0PortGExtFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define PCIE0PortGExtFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define PCIE0PortGExtFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define PCIE0PortGExtFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define PCIE0PortGExtFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define PCIE0PortGExtFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//PCIE0PortGExtNonFatal_ACTION_CONTROL
+#define PCIE0PortGExtNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define PCIE0PortGExtNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define PCIE0PortGExtNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define PCIE0PortGExtNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define PCIE0PortGExtNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define PCIE0PortGExtNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define PCIE0PortGExtNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define PCIE0PortGExtNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//PCIE0PortGExtCorr_ACTION_CONTROL
+#define PCIE0PortGExtCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define PCIE0PortGExtCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define PCIE0PortGExtCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define PCIE0PortGExtCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define PCIE0PortGExtCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define PCIE0PortGExtCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define PCIE0PortGExtCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define PCIE0PortGExtCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//PCIE0PortGParityErr_ACTION_CONTROL
+#define PCIE0PortGParityErr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                0x0
+#define PCIE0PortGParityErr_ACTION_CONTROL__IntrGenSel__SHIFT                                                 0x1
+#define PCIE0PortGParityErr_ACTION_CONTROL__LinkDis_En__SHIFT                                                 0x3
+#define PCIE0PortGParityErr_ACTION_CONTROL__SyncFlood_En__SHIFT                                               0x4
+#define PCIE0PortGParityErr_ACTION_CONTROL__APML_ERR_En_MASK                                                  0x00000001L
+#define PCIE0PortGParityErr_ACTION_CONTROL__IntrGenSel_MASK                                                   0x00000006L
+#define PCIE0PortGParityErr_ACTION_CONTROL__LinkDis_En_MASK                                                   0x00000008L
+#define PCIE0PortGParityErr_ACTION_CONTROL__SyncFlood_En_MASK                                                 0x00000010L
+//NBIF1PortASerr_ACTION_CONTROL
+#define NBIF1PortASerr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                     0x0
+#define NBIF1PortASerr_ACTION_CONTROL__IntrGenSel__SHIFT                                                      0x1
+#define NBIF1PortASerr_ACTION_CONTROL__LinkDis_En__SHIFT                                                      0x3
+#define NBIF1PortASerr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                    0x4
+#define NBIF1PortASerr_ACTION_CONTROL__APML_ERR_En_MASK                                                       0x00000001L
+#define NBIF1PortASerr_ACTION_CONTROL__IntrGenSel_MASK                                                        0x00000006L
+#define NBIF1PortASerr_ACTION_CONTROL__LinkDis_En_MASK                                                        0x00000008L
+#define NBIF1PortASerr_ACTION_CONTROL__SyncFlood_En_MASK                                                      0x00000010L
+//NBIF1PortAIntFatal_ACTION_CONTROL
+#define NBIF1PortAIntFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define NBIF1PortAIntFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define NBIF1PortAIntFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define NBIF1PortAIntFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define NBIF1PortAIntFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define NBIF1PortAIntFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define NBIF1PortAIntFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define NBIF1PortAIntFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//NBIF1PortAIntNonFatal_ACTION_CONTROL
+#define NBIF1PortAIntNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define NBIF1PortAIntNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define NBIF1PortAIntNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define NBIF1PortAIntNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define NBIF1PortAIntNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define NBIF1PortAIntNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define NBIF1PortAIntNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define NBIF1PortAIntNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//NBIF1PortAIntCorr_ACTION_CONTROL
+#define NBIF1PortAIntCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define NBIF1PortAIntCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define NBIF1PortAIntCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define NBIF1PortAIntCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define NBIF1PortAIntCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define NBIF1PortAIntCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define NBIF1PortAIntCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define NBIF1PortAIntCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//NBIF1PortAExtFatal_ACTION_CONTROL
+#define NBIF1PortAExtFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                                 0x0
+#define NBIF1PortAExtFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                                  0x1
+#define NBIF1PortAExtFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                                  0x3
+#define NBIF1PortAExtFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                                0x4
+#define NBIF1PortAExtFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                   0x00000001L
+#define NBIF1PortAExtFatal_ACTION_CONTROL__IntrGenSel_MASK                                                    0x00000006L
+#define NBIF1PortAExtFatal_ACTION_CONTROL__LinkDis_En_MASK                                                    0x00000008L
+#define NBIF1PortAExtFatal_ACTION_CONTROL__SyncFlood_En_MASK                                                  0x00000010L
+//NBIF1PortAExtNonFatal_ACTION_CONTROL
+#define NBIF1PortAExtNonFatal_ACTION_CONTROL__APML_ERR_En__SHIFT                                              0x0
+#define NBIF1PortAExtNonFatal_ACTION_CONTROL__IntrGenSel__SHIFT                                               0x1
+#define NBIF1PortAExtNonFatal_ACTION_CONTROL__LinkDis_En__SHIFT                                               0x3
+#define NBIF1PortAExtNonFatal_ACTION_CONTROL__SyncFlood_En__SHIFT                                             0x4
+#define NBIF1PortAExtNonFatal_ACTION_CONTROL__APML_ERR_En_MASK                                                0x00000001L
+#define NBIF1PortAExtNonFatal_ACTION_CONTROL__IntrGenSel_MASK                                                 0x00000006L
+#define NBIF1PortAExtNonFatal_ACTION_CONTROL__LinkDis_En_MASK                                                 0x00000008L
+#define NBIF1PortAExtNonFatal_ACTION_CONTROL__SyncFlood_En_MASK                                               0x00000010L
+//NBIF1PortAExtCorr_ACTION_CONTROL
+#define NBIF1PortAExtCorr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                  0x0
+#define NBIF1PortAExtCorr_ACTION_CONTROL__IntrGenSel__SHIFT                                                   0x1
+#define NBIF1PortAExtCorr_ACTION_CONTROL__LinkDis_En__SHIFT                                                   0x3
+#define NBIF1PortAExtCorr_ACTION_CONTROL__SyncFlood_En__SHIFT                                                 0x4
+#define NBIF1PortAExtCorr_ACTION_CONTROL__APML_ERR_En_MASK                                                    0x00000001L
+#define NBIF1PortAExtCorr_ACTION_CONTROL__IntrGenSel_MASK                                                     0x00000006L
+#define NBIF1PortAExtCorr_ACTION_CONTROL__LinkDis_En_MASK                                                     0x00000008L
+#define NBIF1PortAExtCorr_ACTION_CONTROL__SyncFlood_En_MASK                                                   0x00000010L
+//NBIF1PortAParityErr_ACTION_CONTROL
+#define NBIF1PortAParityErr_ACTION_CONTROL__APML_ERR_En__SHIFT                                                0x0
+#define NBIF1PortAParityErr_ACTION_CONTROL__IntrGenSel__SHIFT                                                 0x1
+#define NBIF1PortAParityErr_ACTION_CONTROL__LinkDis_En__SHIFT                                                 0x3
+#define NBIF1PortAParityErr_ACTION_CONTROL__SyncFlood_En__SHIFT                                               0x4
+#define NBIF1PortAParityErr_ACTION_CONTROL__APML_ERR_En_MASK                                                  0x00000001L
+#define NBIF1PortAParityErr_ACTION_CONTROL__IntrGenSel_MASK                                                   0x00000006L
+#define NBIF1PortAParityErr_ACTION_CONTROL__LinkDis_En_MASK                                                   0x00000008L
+#define NBIF1PortAParityErr_ACTION_CONTROL__SyncFlood_En_MASK                                                 0x00000010L
+//SYNCFLOOD_STATUS
+#define SYNCFLOOD_STATUS__SyncfloodFromRASCntl__SHIFT                                                         0x0
+#define SYNCFLOOD_STATUS__SyncfloodFromAPML__SHIFT                                                            0x1
+#define SYNCFLOOD_STATUS__SyncfloodFromPin__SHIFT                                                             0x2
+#define SYNCFLOOD_STATUS__SyncfloodFromPrivate__SHIFT                                                         0x4
+#define SYNCFLOOD_STATUS__SyncfloodFromMCA__SHIFT                                                             0x5
+#define SYNCFLOOD_STATUS__SyncfloodFromRASCntl_MASK                                                           0x00000001L
+#define SYNCFLOOD_STATUS__SyncfloodFromAPML_MASK                                                              0x00000002L
+#define SYNCFLOOD_STATUS__SyncfloodFromPin_MASK                                                               0x00000004L
+#define SYNCFLOOD_STATUS__SyncfloodFromPrivate_MASK                                                           0x00000010L
+#define SYNCFLOOD_STATUS__SyncfloodFromMCA_MASK                                                               0x00000020L
+//NMI_STATUS
+#define NMI_STATUS__NMIFromPin__SHIFT                                                                         0x0
+#define NMI_STATUS__NMIFromPin_MASK                                                                           0x00000001L
+//POISON_ACTION_CONTROL
+#define POISON_ACTION_CONTROL__IntPoisonAPMLErrEn__SHIFT                                                      0x0
+#define POISON_ACTION_CONTROL__IntPoisonIntrGenSel__SHIFT                                                     0x1
+#define POISON_ACTION_CONTROL__IntPoisonLinkDisEn__SHIFT                                                      0x3
+#define POISON_ACTION_CONTROL__IntPoisonSyncFloodEn__SHIFT                                                    0x4
+#define POISON_ACTION_CONTROL__EgressPoisonLSAPMLErrEn__SHIFT                                                 0x8
+#define POISON_ACTION_CONTROL__EgressPoisonLSIntrGenSel__SHIFT                                                0x9
+#define POISON_ACTION_CONTROL__EgressPoisonLSLinkDisEn__SHIFT                                                 0xb
+#define POISON_ACTION_CONTROL__EgressPoisonLSSyncFloodEn__SHIFT                                               0xc
+#define POISON_ACTION_CONTROL__EgressPoisonHSAPMLErrEn__SHIFT                                                 0x10
+#define POISON_ACTION_CONTROL__EgressPoisonHSIntrGenSel__SHIFT                                                0x11
+#define POISON_ACTION_CONTROL__EgressPoisonHSLinkDisEn__SHIFT                                                 0x13
+#define POISON_ACTION_CONTROL__EgressPoisonHSSyncFloodEn__SHIFT                                               0x14
+#define POISON_ACTION_CONTROL__IntPoisonAPMLErrEn_MASK                                                        0x00000001L
+#define POISON_ACTION_CONTROL__IntPoisonIntrGenSel_MASK                                                       0x00000006L
+#define POISON_ACTION_CONTROL__IntPoisonLinkDisEn_MASK                                                        0x00000008L
+#define POISON_ACTION_CONTROL__IntPoisonSyncFloodEn_MASK                                                      0x00000010L
+#define POISON_ACTION_CONTROL__EgressPoisonLSAPMLErrEn_MASK                                                   0x00000100L
+#define POISON_ACTION_CONTROL__EgressPoisonLSIntrGenSel_MASK                                                  0x00000600L
+#define POISON_ACTION_CONTROL__EgressPoisonLSLinkDisEn_MASK                                                   0x00000800L
+#define POISON_ACTION_CONTROL__EgressPoisonLSSyncFloodEn_MASK                                                 0x00001000L
+#define POISON_ACTION_CONTROL__EgressPoisonHSAPMLErrEn_MASK                                                   0x00010000L
+#define POISON_ACTION_CONTROL__EgressPoisonHSIntrGenSel_MASK                                                  0x00060000L
+#define POISON_ACTION_CONTROL__EgressPoisonHSLinkDisEn_MASK                                                   0x00080000L
+#define POISON_ACTION_CONTROL__EgressPoisonHSSyncFloodEn_MASK                                                 0x00100000L
+//INTERNAL_POISON_STATUS
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_0__SHIFT                                                      0x0
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_1__SHIFT                                                      0x1
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_2__SHIFT                                                      0x2
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_3__SHIFT                                                      0x3
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_4__SHIFT                                                      0x4
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_5__SHIFT                                                      0x5
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_6__SHIFT                                                      0x6
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_7__SHIFT                                                      0x7
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_0_MASK                                                        0x00000001L
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_1_MASK                                                        0x00000002L
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_2_MASK                                                        0x00000004L
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_3_MASK                                                        0x00000008L
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_4_MASK                                                        0x00000010L
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_5_MASK                                                        0x00000020L
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_6_MASK                                                        0x00000040L
+#define INTERNAL_POISON_STATUS__IntPoisonStatus_7_MASK                                                        0x00000080L
+//INTERNAL_POISON_MASK
+#define INTERNAL_POISON_MASK__IntPoisonMask__SHIFT                                                            0x0
+#define INTERNAL_POISON_MASK__IntPoisonMask_MASK                                                              0x000000FFL
+//EGRESS_POISON_STATUS_LO
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_0__SHIFT                                                0x0
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_1__SHIFT                                                0x1
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_2__SHIFT                                                0x2
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_3__SHIFT                                                0x3
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_4__SHIFT                                                0x4
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_5__SHIFT                                                0x5
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_6__SHIFT                                                0x6
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_7__SHIFT                                                0x7
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_8__SHIFT                                                0x8
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_9__SHIFT                                                0x9
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_10__SHIFT                                               0xa
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_11__SHIFT                                               0xb
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_12__SHIFT                                               0xc
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_13__SHIFT                                               0xd
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_14__SHIFT                                               0xe
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_15__SHIFT                                               0xf
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_16__SHIFT                                               0x10
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_17__SHIFT                                               0x11
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_18__SHIFT                                               0x12
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_19__SHIFT                                               0x13
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_20__SHIFT                                               0x14
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_21__SHIFT                                               0x15
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_22__SHIFT                                               0x16
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_23__SHIFT                                               0x17
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_24__SHIFT                                               0x18
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_25__SHIFT                                               0x19
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_26__SHIFT                                               0x1a
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_27__SHIFT                                               0x1b
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_28__SHIFT                                               0x1c
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_29__SHIFT                                               0x1d
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_30__SHIFT                                               0x1e
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_31__SHIFT                                               0x1f
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_0_MASK                                                  0x00000001L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_1_MASK                                                  0x00000002L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_2_MASK                                                  0x00000004L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_3_MASK                                                  0x00000008L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_4_MASK                                                  0x00000010L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_5_MASK                                                  0x00000020L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_6_MASK                                                  0x00000040L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_7_MASK                                                  0x00000080L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_8_MASK                                                  0x00000100L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_9_MASK                                                  0x00000200L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_10_MASK                                                 0x00000400L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_11_MASK                                                 0x00000800L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_12_MASK                                                 0x00001000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_13_MASK                                                 0x00002000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_14_MASK                                                 0x00004000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_15_MASK                                                 0x00008000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_16_MASK                                                 0x00010000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_17_MASK                                                 0x00020000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_18_MASK                                                 0x00040000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_19_MASK                                                 0x00080000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_20_MASK                                                 0x00100000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_21_MASK                                                 0x00200000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_22_MASK                                                 0x00400000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_23_MASK                                                 0x00800000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_24_MASK                                                 0x01000000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_25_MASK                                                 0x02000000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_26_MASK                                                 0x04000000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_27_MASK                                                 0x08000000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_28_MASK                                                 0x10000000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_29_MASK                                                 0x20000000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_30_MASK                                                 0x40000000L
+#define EGRESS_POISON_STATUS_LO__EgressPoisonStatusLo_31_MASK                                                 0x80000000L
+//EGRESS_POISON_STATUS_HI
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_0__SHIFT                                                0x0
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_1__SHIFT                                                0x1
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_2__SHIFT                                                0x2
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_3__SHIFT                                                0x3
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_4__SHIFT                                                0x4
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_5__SHIFT                                                0x5
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_6__SHIFT                                                0x6
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_7__SHIFT                                                0x7
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_8__SHIFT                                                0x8
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_9__SHIFT                                                0x9
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_10__SHIFT                                               0xa
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_11__SHIFT                                               0xb
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_12__SHIFT                                               0xc
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_13__SHIFT                                               0xd
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_14__SHIFT                                               0xe
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_15__SHIFT                                               0xf
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_16__SHIFT                                               0x10
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_17__SHIFT                                               0x11
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_18__SHIFT                                               0x12
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_19__SHIFT                                               0x13
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_20__SHIFT                                               0x14
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_21__SHIFT                                               0x15
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_22__SHIFT                                               0x16
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_23__SHIFT                                               0x17
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_24__SHIFT                                               0x18
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_25__SHIFT                                               0x19
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_26__SHIFT                                               0x1a
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_27__SHIFT                                               0x1b
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_28__SHIFT                                               0x1c
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_29__SHIFT                                               0x1d
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_30__SHIFT                                               0x1e
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_31__SHIFT                                               0x1f
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_0_MASK                                                  0x00000001L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_1_MASK                                                  0x00000002L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_2_MASK                                                  0x00000004L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_3_MASK                                                  0x00000008L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_4_MASK                                                  0x00000010L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_5_MASK                                                  0x00000020L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_6_MASK                                                  0x00000040L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_7_MASK                                                  0x00000080L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_8_MASK                                                  0x00000100L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_9_MASK                                                  0x00000200L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_10_MASK                                                 0x00000400L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_11_MASK                                                 0x00000800L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_12_MASK                                                 0x00001000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_13_MASK                                                 0x00002000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_14_MASK                                                 0x00004000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_15_MASK                                                 0x00008000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_16_MASK                                                 0x00010000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_17_MASK                                                 0x00020000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_18_MASK                                                 0x00040000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_19_MASK                                                 0x00080000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_20_MASK                                                 0x00100000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_21_MASK                                                 0x00200000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_22_MASK                                                 0x00400000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_23_MASK                                                 0x00800000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_24_MASK                                                 0x01000000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_25_MASK                                                 0x02000000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_26_MASK                                                 0x04000000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_27_MASK                                                 0x08000000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_28_MASK                                                 0x10000000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_29_MASK                                                 0x20000000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_30_MASK                                                 0x40000000L
+#define EGRESS_POISON_STATUS_HI__EgressPoisonStatusHi_31_MASK                                                 0x80000000L
+//EGRESS_POISON_MASK_LO
+#define EGRESS_POISON_MASK_LO__EgressPoisonMaskLo__SHIFT                                                      0x0
+#define EGRESS_POISON_MASK_LO__EgressPoisonMaskLo_MASK                                                        0xFFFFFFFFL
+//EGRESS_POISON_MASK_HI
+#define EGRESS_POISON_MASK_HI__EgressPoisonMaskHi__SHIFT                                                      0x0
+#define EGRESS_POISON_MASK_HI__EgressPoisonMaskHi_MASK                                                        0xFFFFFFFFL
+//EGRESS_POISON_SEVERITY_DOWN
+#define EGRESS_POISON_SEVERITY_DOWN__EgressPoisonSeverityDown__SHIFT                                          0x0
+#define EGRESS_POISON_SEVERITY_DOWN__EgressPoisonSeverityDown_MASK                                            0xFFFFFFFFL
+//EGRESS_POISON_SEVERITY_UPPER
+#define EGRESS_POISON_SEVERITY_UPPER__EgressPoisonSeverityUpper__SHIFT                                        0x0
+#define EGRESS_POISON_SEVERITY_UPPER__EgressPoisonSeverityUpper_MASK                                          0xFFFFFFFFL
+//APML_STATUS
+#define APML_STATUS__APML_Corr__SHIFT                                                                         0x0
+#define APML_STATUS__APML_NonFatal__SHIFT                                                                     0x1
+#define APML_STATUS__APML_Fatal__SHIFT                                                                        0x2
+#define APML_STATUS__APML_Serr__SHIFT                                                                         0x3
+#define APML_STATUS__APML_IntPoisonErr__SHIFT                                                                 0x4
+#define APML_STATUS__APML_EgressPoisonErrLo__SHIFT                                                            0x5
+#define APML_STATUS__APML_EgressPoisonErrHi__SHIFT                                                            0x6
+#define APML_STATUS__APML_Corr_MASK                                                                           0x00000001L
+#define APML_STATUS__APML_NonFatal_MASK                                                                       0x00000002L
+#define APML_STATUS__APML_Fatal_MASK                                                                          0x00000004L
+#define APML_STATUS__APML_Serr_MASK                                                                           0x00000008L
+#define APML_STATUS__APML_IntPoisonErr_MASK                                                                   0x00000010L
+#define APML_STATUS__APML_EgressPoisonErrLo_MASK                                                              0x00000020L
+#define APML_STATUS__APML_EgressPoisonErrHi_MASK                                                              0x00000040L
+//APML_CONTROL
+#define APML_CONTROL__APML_NMI_En__SHIFT                                                                      0x0
+#define APML_CONTROL__APML_SyncFlood_En__SHIFT                                                                0x1
+#define APML_CONTROL__APML_OutputDis__SHIFT                                                                   0x8
+#define APML_CONTROL__APML_NMI_En_MASK                                                                        0x00000001L
+#define APML_CONTROL__APML_SyncFlood_En_MASK                                                                  0x00000002L
+#define APML_CONTROL__APML_OutputDis_MASK                                                                     0x00000100L
+//APML_TRIGGER
+#define APML_TRIGGER__APML_NMI_TRIGGER__SHIFT                                                                 0x0
+#define APML_TRIGGER__APML_NMI_TRIGGER_MASK                                                                   0x00000001L
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg0_devind_cfgdecp
+//NB_PCIE0DEVINDCFG0_STEERING_CNTL
+#define NB_PCIE0DEVINDCFG0_STEERING_CNTL__ForceSteering__SHIFT                                                0x0
+#define NB_PCIE0DEVINDCFG0_STEERING_CNTL__SteeringValue__SHIFT                                                0x8
+#define NB_PCIE0DEVINDCFG0_STEERING_CNTL__ForceSteering_MASK                                                  0x00000001L
+#define NB_PCIE0DEVINDCFG0_STEERING_CNTL__SteeringValue_MASK                                                  0x0000FF00L
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg1_devind_cfgdecp
+//NB_PCIE0DEVINDCFG1_STEERING_CNTL
+#define NB_PCIE0DEVINDCFG1_STEERING_CNTL__ForceSteering__SHIFT                                                0x0
+#define NB_PCIE0DEVINDCFG1_STEERING_CNTL__SteeringValue__SHIFT                                                0x8
+#define NB_PCIE0DEVINDCFG1_STEERING_CNTL__ForceSteering_MASK                                                  0x00000001L
+#define NB_PCIE0DEVINDCFG1_STEERING_CNTL__SteeringValue_MASK                                                  0x0000FF00L
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg2_devind_cfgdecp
+//NB_PCIE0DEVINDCFG2_STEERING_CNTL
+#define NB_PCIE0DEVINDCFG2_STEERING_CNTL__ForceSteering__SHIFT                                                0x0
+#define NB_PCIE0DEVINDCFG2_STEERING_CNTL__SteeringValue__SHIFT                                                0x8
+#define NB_PCIE0DEVINDCFG2_STEERING_CNTL__ForceSteering_MASK                                                  0x00000001L
+#define NB_PCIE0DEVINDCFG2_STEERING_CNTL__SteeringValue_MASK                                                  0x0000FF00L
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg3_devind_cfgdecp
+//NB_PCIE0DEVINDCFG3_STEERING_CNTL
+#define NB_PCIE0DEVINDCFG3_STEERING_CNTL__ForceSteering__SHIFT                                                0x0
+#define NB_PCIE0DEVINDCFG3_STEERING_CNTL__SteeringValue__SHIFT                                                0x8
+#define NB_PCIE0DEVINDCFG3_STEERING_CNTL__ForceSteering_MASK                                                  0x00000001L
+#define NB_PCIE0DEVINDCFG3_STEERING_CNTL__SteeringValue_MASK                                                  0x0000FF00L
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg4_devind_cfgdecp
+//NB_PCIE0DEVINDCFG4_STEERING_CNTL
+#define NB_PCIE0DEVINDCFG4_STEERING_CNTL__ForceSteering__SHIFT                                                0x0
+#define NB_PCIE0DEVINDCFG4_STEERING_CNTL__SteeringValue__SHIFT                                                0x8
+#define NB_PCIE0DEVINDCFG4_STEERING_CNTL__ForceSteering_MASK                                                  0x00000001L
+#define NB_PCIE0DEVINDCFG4_STEERING_CNTL__SteeringValue_MASK                                                  0x0000FF00L
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg5_devind_cfgdecp
+//NB_PCIE0DEVINDCFG5_STEERING_CNTL
+#define NB_PCIE0DEVINDCFG5_STEERING_CNTL__ForceSteering__SHIFT                                                0x0
+#define NB_PCIE0DEVINDCFG5_STEERING_CNTL__SteeringValue__SHIFT                                                0x8
+#define NB_PCIE0DEVINDCFG5_STEERING_CNTL__ForceSteering_MASK                                                  0x00000001L
+#define NB_PCIE0DEVINDCFG5_STEERING_CNTL__SteeringValue_MASK                                                  0x0000FF00L
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0devindcfg6_devind_cfgdecp
+//NB_PCIE0DEVINDCFG6_STEERING_CNTL
+#define NB_PCIE0DEVINDCFG6_STEERING_CNTL__ForceSteering__SHIFT                                                0x0
+#define NB_PCIE0DEVINDCFG6_STEERING_CNTL__SteeringValue__SHIFT                                                0x8
+#define NB_PCIE0DEVINDCFG6_STEERING_CNTL__ForceSteering_MASK                                                  0x00000001L
+#define NB_PCIE0DEVINDCFG6_STEERING_CNTL__SteeringValue_MASK                                                  0x0000FF00L
+
+
+// addressBlock: aid_nbio_iohub_nb_NBIF1devindcfg0_devind_cfgdecp
+//NB_NBIF1DEVINDCFG0_STEERING_CNTL
+#define NB_NBIF1DEVINDCFG0_STEERING_CNTL__ForceSteering__SHIFT                                                0x0
+#define NB_NBIF1DEVINDCFG0_STEERING_CNTL__SteeringValue__SHIFT                                                0x8
+#define NB_NBIF1DEVINDCFG0_STEERING_CNTL__ForceSteering_MASK                                                  0x00000001L
+#define NB_NBIF1DEVINDCFG0_STEERING_CNTL__SteeringValue_MASK                                                  0x0000FF00L
+
+
+// addressBlock: aid_nbio_iohub_nb_intSBdevindcfg0_devind_cfgdecp
+//NB_INTSBDEVINDCFG0_STEERING_CNTL
+#define NB_INTSBDEVINDCFG0_STEERING_CNTL__ForceSteering__SHIFT                                                0x0
+#define NB_INTSBDEVINDCFG0_STEERING_CNTL__SteeringValue__SHIFT                                                0x8
+#define NB_INTSBDEVINDCFG0_STEERING_CNTL__ForceSteering_MASK                                                  0x00000001L
+#define NB_INTSBDEVINDCFG0_STEERING_CNTL__SteeringValue_MASK                                                  0x0000FF00L
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg0_pciercbdgind_cfgdec
+//NB_PCIE0RCBDG_INDCFG0_RC_SMN_INDEX_EXTENSION
+#define NB_PCIE0RCBDG_INDCFG0_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION__SHIFT                           0x0
+#define NB_PCIE0RCBDG_INDCFG0_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION_MASK                             0x000000FFL
+//NB_PCIE0RCBDG_INDCFG0_RC_SMN_INDEX
+#define NB_PCIE0RCBDG_INDCFG0_RC_SMN_INDEX__RC_SMN_INDEX__SHIFT                                               0x0
+#define NB_PCIE0RCBDG_INDCFG0_RC_SMN_INDEX__RC_SMN_INDEX_MASK                                                 0xFFFFFFFFL
+//NB_PCIE0RCBDG_INDCFG0_RC_SMN_DATA
+#define NB_PCIE0RCBDG_INDCFG0_RC_SMN_DATA__RC_SMN_DATA__SHIFT                                                 0x0
+#define NB_PCIE0RCBDG_INDCFG0_RC_SMN_DATA__RC_SMN_DATA_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg1_pciercbdgind_cfgdec
+//NB_PCIE0RCBDG_INDCFG1_RC_SMN_INDEX_EXTENSION
+#define NB_PCIE0RCBDG_INDCFG1_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION__SHIFT                           0x0
+#define NB_PCIE0RCBDG_INDCFG1_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION_MASK                             0x000000FFL
+//NB_PCIE0RCBDG_INDCFG1_RC_SMN_INDEX
+#define NB_PCIE0RCBDG_INDCFG1_RC_SMN_INDEX__RC_SMN_INDEX__SHIFT                                               0x0
+#define NB_PCIE0RCBDG_INDCFG1_RC_SMN_INDEX__RC_SMN_INDEX_MASK                                                 0xFFFFFFFFL
+//NB_PCIE0RCBDG_INDCFG1_RC_SMN_DATA
+#define NB_PCIE0RCBDG_INDCFG1_RC_SMN_DATA__RC_SMN_DATA__SHIFT                                                 0x0
+#define NB_PCIE0RCBDG_INDCFG1_RC_SMN_DATA__RC_SMN_DATA_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg2_pciercbdgind_cfgdec
+//NB_PCIE0RCBDG_INDCFG2_RC_SMN_INDEX_EXTENSION
+#define NB_PCIE0RCBDG_INDCFG2_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION__SHIFT                           0x0
+#define NB_PCIE0RCBDG_INDCFG2_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION_MASK                             0x000000FFL
+//NB_PCIE0RCBDG_INDCFG2_RC_SMN_INDEX
+#define NB_PCIE0RCBDG_INDCFG2_RC_SMN_INDEX__RC_SMN_INDEX__SHIFT                                               0x0
+#define NB_PCIE0RCBDG_INDCFG2_RC_SMN_INDEX__RC_SMN_INDEX_MASK                                                 0xFFFFFFFFL
+//NB_PCIE0RCBDG_INDCFG2_RC_SMN_DATA
+#define NB_PCIE0RCBDG_INDCFG2_RC_SMN_DATA__RC_SMN_DATA__SHIFT                                                 0x0
+#define NB_PCIE0RCBDG_INDCFG2_RC_SMN_DATA__RC_SMN_DATA_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg3_pciercbdgind_cfgdec
+//NB_PCIE0RCBDG_INDCFG3_RC_SMN_INDEX_EXTENSION
+#define NB_PCIE0RCBDG_INDCFG3_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION__SHIFT                           0x0
+#define NB_PCIE0RCBDG_INDCFG3_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION_MASK                             0x000000FFL
+//NB_PCIE0RCBDG_INDCFG3_RC_SMN_INDEX
+#define NB_PCIE0RCBDG_INDCFG3_RC_SMN_INDEX__RC_SMN_INDEX__SHIFT                                               0x0
+#define NB_PCIE0RCBDG_INDCFG3_RC_SMN_INDEX__RC_SMN_INDEX_MASK                                                 0xFFFFFFFFL
+//NB_PCIE0RCBDG_INDCFG3_RC_SMN_DATA
+#define NB_PCIE0RCBDG_INDCFG3_RC_SMN_DATA__RC_SMN_DATA__SHIFT                                                 0x0
+#define NB_PCIE0RCBDG_INDCFG3_RC_SMN_DATA__RC_SMN_DATA_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg4_pciercbdgind_cfgdec
+//NB_PCIE0RCBDG_INDCFG4_RC_SMN_INDEX_EXTENSION
+#define NB_PCIE0RCBDG_INDCFG4_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION__SHIFT                           0x0
+#define NB_PCIE0RCBDG_INDCFG4_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION_MASK                             0x000000FFL
+//NB_PCIE0RCBDG_INDCFG4_RC_SMN_INDEX
+#define NB_PCIE0RCBDG_INDCFG4_RC_SMN_INDEX__RC_SMN_INDEX__SHIFT                                               0x0
+#define NB_PCIE0RCBDG_INDCFG4_RC_SMN_INDEX__RC_SMN_INDEX_MASK                                                 0xFFFFFFFFL
+//NB_PCIE0RCBDG_INDCFG4_RC_SMN_DATA
+#define NB_PCIE0RCBDG_INDCFG4_RC_SMN_DATA__RC_SMN_DATA__SHIFT                                                 0x0
+#define NB_PCIE0RCBDG_INDCFG4_RC_SMN_DATA__RC_SMN_DATA_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg5_pciercbdgind_cfgdec
+//NB_PCIE0RCBDG_INDCFG5_RC_SMN_INDEX_EXTENSION
+#define NB_PCIE0RCBDG_INDCFG5_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION__SHIFT                           0x0
+#define NB_PCIE0RCBDG_INDCFG5_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION_MASK                             0x000000FFL
+//NB_PCIE0RCBDG_INDCFG5_RC_SMN_INDEX
+#define NB_PCIE0RCBDG_INDCFG5_RC_SMN_INDEX__RC_SMN_INDEX__SHIFT                                               0x0
+#define NB_PCIE0RCBDG_INDCFG5_RC_SMN_INDEX__RC_SMN_INDEX_MASK                                                 0xFFFFFFFFL
+//NB_PCIE0RCBDG_INDCFG5_RC_SMN_DATA
+#define NB_PCIE0RCBDG_INDCFG5_RC_SMN_DATA__RC_SMN_DATA__SHIFT                                                 0x0
+#define NB_PCIE0RCBDG_INDCFG5_RC_SMN_DATA__RC_SMN_DATA_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_iohub_nb_PCIE0rcbdg_indcfg6_pciercbdgind_cfgdec
+//NB_PCIE0RCBDG_INDCFG6_RC_SMN_INDEX_EXTENSION
+#define NB_PCIE0RCBDG_INDCFG6_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION__SHIFT                           0x0
+#define NB_PCIE0RCBDG_INDCFG6_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION_MASK                             0x000000FFL
+//NB_PCIE0RCBDG_INDCFG6_RC_SMN_INDEX
+#define NB_PCIE0RCBDG_INDCFG6_RC_SMN_INDEX__RC_SMN_INDEX__SHIFT                                               0x0
+#define NB_PCIE0RCBDG_INDCFG6_RC_SMN_INDEX__RC_SMN_INDEX_MASK                                                 0xFFFFFFFFL
+//NB_PCIE0RCBDG_INDCFG6_RC_SMN_DATA
+#define NB_PCIE0RCBDG_INDCFG6_RC_SMN_DATA__RC_SMN_DATA__SHIFT                                                 0x0
+#define NB_PCIE0RCBDG_INDCFG6_RC_SMN_DATA__RC_SMN_DATA_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_iohub_nb_NBIF1rcbdg_indcfg0_pciercbdgind_cfgdec
+//NB_NBIF1RCBDG_INDCFG0_RC_SMN_INDEX_EXTENSION
+#define NB_NBIF1RCBDG_INDCFG0_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION__SHIFT                           0x0
+#define NB_NBIF1RCBDG_INDCFG0_RC_SMN_INDEX_EXTENSION__RC_SMN_INDEX_EXTENSION_MASK                             0x000000FFL
+//NB_NBIF1RCBDG_INDCFG0_RC_SMN_INDEX
+#define NB_NBIF1RCBDG_INDCFG0_RC_SMN_INDEX__RC_SMN_INDEX__SHIFT                                               0x0
+#define NB_NBIF1RCBDG_INDCFG0_RC_SMN_INDEX__RC_SMN_INDEX_MASK                                                 0xFFFFFFFFL
+//NB_NBIF1RCBDG_INDCFG0_RC_SMN_DATA
+#define NB_NBIF1RCBDG_INDCFG0_RC_SMN_DATA__RC_SMN_DATA__SHIFT                                                 0x0
+#define NB_NBIF1RCBDG_INDCFG0_RC_SMN_DATA__RC_SMN_DATA_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_iohub_iommu_l2a_l2acfg
+//L2_PERF_CNTL_0
+#define L2_PERF_CNTL_0__L2PerfEvent0__SHIFT                                                                   0x0
+#define L2_PERF_CNTL_0__L2PerfEvent1__SHIFT                                                                   0x8
+#define L2_PERF_CNTL_0__L2PerfCountUpper0__SHIFT                                                              0x10
+#define L2_PERF_CNTL_0__L2PerfCountUpper1__SHIFT                                                              0x18
+#define L2_PERF_CNTL_0__L2PerfEvent0_MASK                                                                     0x000000FFL
+#define L2_PERF_CNTL_0__L2PerfEvent1_MASK                                                                     0x0000FF00L
+#define L2_PERF_CNTL_0__L2PerfCountUpper0_MASK                                                                0x00FF0000L
+#define L2_PERF_CNTL_0__L2PerfCountUpper1_MASK                                                                0xFF000000L
+//L2_PERF_COUNT_0
+#define L2_PERF_COUNT_0__L2PerfCount0__SHIFT                                                                  0x0
+#define L2_PERF_COUNT_0__L2PerfCount0_MASK                                                                    0xFFFFFFFFL
+//L2_PERF_COUNT_1
+#define L2_PERF_COUNT_1__L2PerfCount1__SHIFT                                                                  0x0
+#define L2_PERF_COUNT_1__L2PerfCount1_MASK                                                                    0xFFFFFFFFL
+//L2_PERF_CNTL_1
+#define L2_PERF_CNTL_1__L2PerfEvent2__SHIFT                                                                   0x0
+#define L2_PERF_CNTL_1__L2PerfEvent3__SHIFT                                                                   0x8
+#define L2_PERF_CNTL_1__L2PerfCountUpper2__SHIFT                                                              0x10
+#define L2_PERF_CNTL_1__L2PerfCountUpper3__SHIFT                                                              0x18
+#define L2_PERF_CNTL_1__L2PerfEvent2_MASK                                                                     0x000000FFL
+#define L2_PERF_CNTL_1__L2PerfEvent3_MASK                                                                     0x0000FF00L
+#define L2_PERF_CNTL_1__L2PerfCountUpper2_MASK                                                                0x00FF0000L
+#define L2_PERF_CNTL_1__L2PerfCountUpper3_MASK                                                                0xFF000000L
+//L2_PERF_COUNT_2
+#define L2_PERF_COUNT_2__L2PerfCount2__SHIFT                                                                  0x0
+#define L2_PERF_COUNT_2__L2PerfCount2_MASK                                                                    0xFFFFFFFFL
+//L2_PERF_COUNT_3
+#define L2_PERF_COUNT_3__L2PerfCount3__SHIFT                                                                  0x0
+#define L2_PERF_COUNT_3__L2PerfCount3_MASK                                                                    0xFFFFFFFFL
+//L2_STATUS_0
+#define L2_STATUS_0__L2STATUS0__SHIFT                                                                         0x0
+#define L2_STATUS_0__L2STATUS0_MASK                                                                           0xFFFFFFFFL
+//L2_CONTROL_0
+#define L2_CONTROL_0__AllowL1CacheVZero__SHIFT                                                                0x1
+#define L2_CONTROL_0__AllowL1CacheATSRsp__SHIFT                                                               0x2
+#define L2_CONTROL_0__DTCHitVZeroOrIVZero__SHIFT                                                              0x3
+#define L2_CONTROL_0__SIDEPTEOnUntransExcl__SHIFT                                                             0xa
+#define L2_CONTROL_0__SIDEPTEOnAddrTransExcl__SHIFT                                                           0xb
+#define L2_CONTROL_0__AllowL1CacheLargePagemode0__SHIFT                                                       0x13
+#define L2_CONTROL_0__IFifoBurstLength__SHIFT                                                                 0x14
+#define L2_CONTROL_0__IFifoClientPriority__SHIFT                                                              0x18
+#define L2_CONTROL_0__AllowL1CacheVZero_MASK                                                                  0x00000002L
+#define L2_CONTROL_0__AllowL1CacheATSRsp_MASK                                                                 0x00000004L
+#define L2_CONTROL_0__DTCHitVZeroOrIVZero_MASK                                                                0x00000008L
+#define L2_CONTROL_0__SIDEPTEOnUntransExcl_MASK                                                               0x00000400L
+#define L2_CONTROL_0__SIDEPTEOnAddrTransExcl_MASK                                                             0x00000800L
+#define L2_CONTROL_0__AllowL1CacheLargePagemode0_MASK                                                         0x00080000L
+#define L2_CONTROL_0__IFifoBurstLength_MASK                                                                   0x00F00000L
+#define L2_CONTROL_0__IFifoClientPriority_MASK                                                                0xFF000000L
+//L2_CONTROL_1
+#define L2_CONTROL_1__SeqInvBurstLimitInv__SHIFT                                                              0x0
+#define L2_CONTROL_1__SeqInvBurstLimitL2Req__SHIFT                                                            0x8
+#define L2_CONTROL_1__SeqInvBurstLimitEn__SHIFT                                                               0x10
+#define L2_CONTROL_1__DBUSDis__SHIFT                                                                          0x11
+#define L2_CONTROL_1__PerfThreshold__SHIFT                                                                    0x18
+#define L2_CONTROL_1__SeqInvBurstLimitInv_MASK                                                                0x000000FFL
+#define L2_CONTROL_1__SeqInvBurstLimitL2Req_MASK                                                              0x0000FF00L
+#define L2_CONTROL_1__SeqInvBurstLimitEn_MASK                                                                 0x00010000L
+#define L2_CONTROL_1__DBUSDis_MASK                                                                            0x00020000L
+#define L2_CONTROL_1__PerfThreshold_MASK                                                                      0xFF000000L
+//L2_DTC_CONTROL
+#define L2_DTC_CONTROL__DTCLRUUpdatePri__SHIFT                                                                0x3
+#define L2_DTC_CONTROL__DTCParityEn__SHIFT                                                                    0x4
+#define L2_DTC_CONTROL__DTCInvalidationSel__SHIFT                                                             0x8
+#define L2_DTC_CONTROL__DTCSoftInvalidate__SHIFT                                                              0xa
+#define L2_DTC_CONTROL__DTCBypass__SHIFT                                                                      0xd
+#define L2_DTC_CONTROL__DTCParitySupport__SHIFT                                                               0xf
+#define L2_DTC_CONTROL__DTCWays__SHIFT                                                                        0x10
+#define L2_DTC_CONTROL__DTCEntries__SHIFT                                                                     0x1c
+#define L2_DTC_CONTROL__DTCLRUUpdatePri_MASK                                                                  0x00000008L
+#define L2_DTC_CONTROL__DTCParityEn_MASK                                                                      0x00000010L
+#define L2_DTC_CONTROL__DTCInvalidationSel_MASK                                                               0x00000300L
+#define L2_DTC_CONTROL__DTCSoftInvalidate_MASK                                                                0x00000400L
+#define L2_DTC_CONTROL__DTCBypass_MASK                                                                        0x00002000L
+#define L2_DTC_CONTROL__DTCParitySupport_MASK                                                                 0x00008000L
+#define L2_DTC_CONTROL__DTCWays_MASK                                                                          0x00FF0000L
+#define L2_DTC_CONTROL__DTCEntries_MASK                                                                       0xF0000000L
+//L2_DTC_HASH_CONTROL
+#define L2_DTC_HASH_CONTROL__DTCAddressMask__SHIFT                                                            0x10
+#define L2_DTC_HASH_CONTROL__DTCAddressMask_MASK                                                              0xFFFF0000L
+//L2_DTC_WAY_CONTROL
+#define L2_DTC_WAY_CONTROL__DTCWayDisable__SHIFT                                                              0x0
+#define L2_DTC_WAY_CONTROL__DTCWayAccessDisable__SHIFT                                                        0x10
+#define L2_DTC_WAY_CONTROL__DTCWayDisable_MASK                                                                0x0000FFFFL
+#define L2_DTC_WAY_CONTROL__DTCWayAccessDisable_MASK                                                          0xFFFF0000L
+//L2_ITC_CONTROL
+#define L2_ITC_CONTROL__ITCLRUUpdatePri__SHIFT                                                                0x3
+#define L2_ITC_CONTROL__ITCParityEn__SHIFT                                                                    0x4
+#define L2_ITC_CONTROL__ITCInvalidationSel__SHIFT                                                             0x8
+#define L2_ITC_CONTROL__ITCSoftInvalidate__SHIFT                                                              0xa
+#define L2_ITC_CONTROL__ITCBypass__SHIFT                                                                      0xd
+#define L2_ITC_CONTROL__ITCParitySupport__SHIFT                                                               0xf
+#define L2_ITC_CONTROL__ITCWays__SHIFT                                                                        0x10
+#define L2_ITC_CONTROL__ITCEntries__SHIFT                                                                     0x1c
+#define L2_ITC_CONTROL__ITCLRUUpdatePri_MASK                                                                  0x00000008L
+#define L2_ITC_CONTROL__ITCParityEn_MASK                                                                      0x00000010L
+#define L2_ITC_CONTROL__ITCInvalidationSel_MASK                                                               0x00000300L
+#define L2_ITC_CONTROL__ITCSoftInvalidate_MASK                                                                0x00000400L
+#define L2_ITC_CONTROL__ITCBypass_MASK                                                                        0x00002000L
+#define L2_ITC_CONTROL__ITCParitySupport_MASK                                                                 0x00008000L
+#define L2_ITC_CONTROL__ITCWays_MASK                                                                          0x00FF0000L
+#define L2_ITC_CONTROL__ITCEntries_MASK                                                                       0xF0000000L
+//L2_ITC_HASH_CONTROL
+#define L2_ITC_HASH_CONTROL__ITCAddressMask__SHIFT                                                            0x10
+#define L2_ITC_HASH_CONTROL__ITCAddressMask_MASK                                                              0xFFFF0000L
+//L2_ITC_WAY_CONTROL
+#define L2_ITC_WAY_CONTROL__ITCWayDisable__SHIFT                                                              0x0
+#define L2_ITC_WAY_CONTROL__ITCWayAccessDisable__SHIFT                                                        0x10
+#define L2_ITC_WAY_CONTROL__ITCWayDisable_MASK                                                                0x0000FFFFL
+#define L2_ITC_WAY_CONTROL__ITCWayAccessDisable_MASK                                                          0xFFFF0000L
+//L2_PTC_A_CONTROL
+#define L2_PTC_A_CONTROL__PTCAStoreFinalATSeperate__SHIFT                                                     0x1
+#define L2_PTC_A_CONTROL__PTCAStorePartialATSeperate__SHIFT                                                   0x2
+#define L2_PTC_A_CONTROL__PTCALRUUpdatePri__SHIFT                                                             0x3
+#define L2_PTC_A_CONTROL__PTCAParityEn__SHIFT                                                                 0x4
+#define L2_PTC_A_CONTROL__PTCAInvalidationSel__SHIFT                                                          0x8
+#define L2_PTC_A_CONTROL__PTCASoftInvalidate__SHIFT                                                           0xa
+#define L2_PTC_A_CONTROL__PTCA2MMode__SHIFT                                                                   0xb
+#define L2_PTC_A_CONTROL__PCTA_Inv_Overlapping_Pages__SHIFT                                                   0xc
+#define L2_PTC_A_CONTROL__PTCABypass__SHIFT                                                                   0xd
+#define L2_PTC_A_CONTROL__PTCAFastInvalidateGuest__SHIFT                                                      0xe
+#define L2_PTC_A_CONTROL__PTCAParitySupport__SHIFT                                                            0xf
+#define L2_PTC_A_CONTROL__PTCAWays__SHIFT                                                                     0x10
+#define L2_PTC_A_CONTROL__PTCAEntries__SHIFT                                                                  0x1c
+#define L2_PTC_A_CONTROL__PTCAStoreFinalATSeperate_MASK                                                       0x00000002L
+#define L2_PTC_A_CONTROL__PTCAStorePartialATSeperate_MASK                                                     0x00000004L
+#define L2_PTC_A_CONTROL__PTCALRUUpdatePri_MASK                                                               0x00000008L
+#define L2_PTC_A_CONTROL__PTCAParityEn_MASK                                                                   0x00000010L
+#define L2_PTC_A_CONTROL__PTCAInvalidationSel_MASK                                                            0x00000300L
+#define L2_PTC_A_CONTROL__PTCASoftInvalidate_MASK                                                             0x00000400L
+#define L2_PTC_A_CONTROL__PTCA2MMode_MASK                                                                     0x00000800L
+#define L2_PTC_A_CONTROL__PCTA_Inv_Overlapping_Pages_MASK                                                     0x00001000L
+#define L2_PTC_A_CONTROL__PTCABypass_MASK                                                                     0x00002000L
+#define L2_PTC_A_CONTROL__PTCAFastInvalidateGuest_MASK                                                        0x00004000L
+#define L2_PTC_A_CONTROL__PTCAParitySupport_MASK                                                              0x00008000L
+#define L2_PTC_A_CONTROL__PTCAWays_MASK                                                                       0x00FF0000L
+#define L2_PTC_A_CONTROL__PTCAEntries_MASK                                                                    0xF0000000L
+//L2_PTC_A_HASH_CONTROL
+#define L2_PTC_A_HASH_CONTROL__PTCAAddressMask__SHIFT                                                         0x10
+#define L2_PTC_A_HASH_CONTROL__PTCAAddressMask_MASK                                                           0xFFFF0000L
+//L2_PTC_A_WAY_CONTROL
+#define L2_PTC_A_WAY_CONTROL__PTCAWayDisable__SHIFT                                                           0x0
+#define L2_PTC_A_WAY_CONTROL__PTCAWayAccessDisable__SHIFT                                                     0x10
+#define L2_PTC_A_WAY_CONTROL__PTCAWayDisable_MASK                                                             0x0000FFFFL
+#define L2_PTC_A_WAY_CONTROL__PTCAWayAccessDisable_MASK                                                       0xFFFF0000L
+//L2A_UPDATE_FILTER_CNTL
+#define L2A_UPDATE_FILTER_CNTL__L2a_Update_Filter_Bypass__SHIFT                                               0x0
+#define L2A_UPDATE_FILTER_CNTL__L2a_Update_Filter_RdLatency__SHIFT                                            0x1
+#define L2A_UPDATE_FILTER_CNTL__L2a_Update_Filter_Bypass_MASK                                                 0x00000001L
+#define L2A_UPDATE_FILTER_CNTL__L2a_Update_Filter_RdLatency_MASK                                              0x0000001EL
+//L2_ERR_RULE_CONTROL_3
+#define L2_ERR_RULE_CONTROL_3__ERRRuleLock1__SHIFT                                                            0x0
+#define L2_ERR_RULE_CONTROL_3__ERRRuleDisable3__SHIFT                                                         0x4
+#define L2_ERR_RULE_CONTROL_3__ERRRuleLock1_MASK                                                              0x00000001L
+#define L2_ERR_RULE_CONTROL_3__ERRRuleDisable3_MASK                                                           0xFFFFFFF0L
+//L2_ERR_RULE_CONTROL_4
+#define L2_ERR_RULE_CONTROL_4__ERRRuleDisable4__SHIFT                                                         0x0
+#define L2_ERR_RULE_CONTROL_4__ERRRuleDisable4_MASK                                                           0xFFFFFFFFL
+//L2_ERR_RULE_CONTROL_5
+#define L2_ERR_RULE_CONTROL_5__ERRRuleDisable5__SHIFT                                                         0x0
+#define L2_ERR_RULE_CONTROL_5__ERRRuleDisable5_MASK                                                           0xFFFFFFFFL
+//L2_L2A_CK_GATE_CONTROL
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2ARegsDisable__SHIFT                                                   0x0
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2ADynamicDisable__SHIFT                                                0x1
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2ACacheDisable__SHIFT                                                  0x2
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2APerfDisable__SHIFT                                                   0x3
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2ALength__SHIFT                                                        0x10
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2AStop__SHIFT                                                          0x12
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2APortClkDis__SHIFT                                                    0x14
+#define L2_L2A_CK_GATE_CONTROL__Reserved__SHIFT                                                               0x15
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2ARegsDisable_MASK                                                     0x00000001L
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2ADynamicDisable_MASK                                                  0x00000002L
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2ACacheDisable_MASK                                                    0x00000004L
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2APerfDisable_MASK                                                     0x00000008L
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2ALength_MASK                                                          0x00030000L
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2AStop_MASK                                                            0x000C0000L
+#define L2_L2A_CK_GATE_CONTROL__CKGateL2APortClkDis_MASK                                                      0x00100000L
+#define L2_L2A_CK_GATE_CONTROL__Reserved_MASK                                                                 0xFFE00000L
+//L2_L2A_PGSIZE_CONTROL
+#define L2_L2A_PGSIZE_CONTROL__L2AREG_GST_PGSIZE__SHIFT                                                       0x0
+#define L2_L2A_PGSIZE_CONTROL__L2AREG_HOST_PGSIZE__SHIFT                                                      0x8
+#define L2_L2A_PGSIZE_CONTROL__L2AREG_PTCSCAN_MODE__SHIFT                                                     0x11
+#define L2_L2A_PGSIZE_CONTROL__L2AREG_GST_PGSIZE_MASK                                                         0x0000007FL
+#define L2_L2A_PGSIZE_CONTROL__L2AREG_HOST_PGSIZE_MASK                                                        0x00007F00L
+#define L2_L2A_PGSIZE_CONTROL__L2AREG_PTCSCAN_MODE_MASK                                                       0x000E0000L
+//L2_PWRGATE_CNTRL_REG_0
+#define L2_PWRGATE_CNTRL_REG_0__IP_PG_thres__SHIFT                                                            0x0
+#define L2_PWRGATE_CNTRL_REG_0__IP_PG_thres_MASK                                                              0xFFFFFFFFL
+//L2_PWRGATE_CNTRL_REG_3
+#define L2_PWRGATE_CNTRL_REG_3__IP_PG_en__SHIFT                                                               0x0
+#define L2_PWRGATE_CNTRL_REG_3__IP_PG_busy__SHIFT                                                             0x1
+#define L2_PWRGATE_CNTRL_REG_3__L2_PG_STATUS__SHIFT                                                           0x2
+#define L2_PWRGATE_CNTRL_REG_3__CFG_FW_PG_EXIT_EN__SHIFT                                                      0x3
+#define L2_PWRGATE_CNTRL_REG_3__IP_PG_en_MASK                                                                 0x00000001L
+#define L2_PWRGATE_CNTRL_REG_3__IP_PG_busy_MASK                                                               0x00000002L
+#define L2_PWRGATE_CNTRL_REG_3__L2_PG_STATUS_MASK                                                             0x00000004L
+#define L2_PWRGATE_CNTRL_REG_3__CFG_FW_PG_EXIT_EN_MASK                                                        0x00000018L
+//L2_ECO_CNTRL_0
+#define L2_ECO_CNTRL_0__L2_ECO_0__SHIFT                                                                       0x0
+#define L2_ECO_CNTRL_0__L2_ECO_0_MASK                                                                         0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_iohub_iommu_l2indx_l2indxcfg
+//L2_STATUS_1
+#define L2_STATUS_1__L2STATUS1__SHIFT                                                                         0x0
+#define L2_STATUS_1__L2STATUS1_MASK                                                                           0xFFFFFFFFL
+//L2_SB_LOCATION
+#define L2_SB_LOCATION__SBlocated_Port__SHIFT                                                                 0x0
+#define L2_SB_LOCATION__SBlocated_Core__SHIFT                                                                 0x10
+#define L2_SB_LOCATION__SBlocated_Port_MASK                                                                   0x0000FFFFL
+#define L2_SB_LOCATION__SBlocated_Core_MASK                                                                   0xFFFF0000L
+//L2_CONTROL_5
+#define L2_CONTROL_5__QueueArbFBPri__SHIFT                                                                    0x0
+#define L2_CONTROL_5__FC1Dis__SHIFT                                                                           0x2
+#define L2_CONTROL_5__DTCUpdateVOneIVZero__SHIFT                                                              0x3
+#define L2_CONTROL_5__DTCUpdateVZeroIVOne__SHIFT                                                              0x4
+#define L2_CONTROL_5__FC3Dis__SHIFT                                                                           0x6
+#define L2_CONTROL_5__ForceTWonVCQoS__SHIFT                                                                   0xb
+#define L2_CONTROL_5__GST_partial_ptc_cntrl__SHIFT                                                            0xc
+#define L2_CONTROL_5__STORE_PDPE_QOS_PTC__SHIFT                                                               0x14
+#define L2_CONTROL_5__DTCUpdatePri__SHIFT                                                                     0x19
+#define L2_CONTROL_5__L2B_L2A_v1_trans_credits__SHIFT                                                         0x1a
+#define L2_CONTROL_5__QueueArbFBPri_MASK                                                                      0x00000001L
+#define L2_CONTROL_5__FC1Dis_MASK                                                                             0x00000004L
+#define L2_CONTROL_5__DTCUpdateVOneIVZero_MASK                                                                0x00000008L
+#define L2_CONTROL_5__DTCUpdateVZeroIVOne_MASK                                                                0x00000010L
+#define L2_CONTROL_5__FC3Dis_MASK                                                                             0x00000040L
+#define L2_CONTROL_5__ForceTWonVCQoS_MASK                                                                     0x00000800L
+#define L2_CONTROL_5__GST_partial_ptc_cntrl_MASK                                                              0x0007F000L
+#define L2_CONTROL_5__STORE_PDPE_QOS_PTC_MASK                                                                 0x00100000L
+#define L2_CONTROL_5__DTCUpdatePri_MASK                                                                       0x02000000L
+#define L2_CONTROL_5__L2B_L2A_v1_trans_credits_MASK                                                           0xFC000000L
+//L2_CONTROL_6
+#define L2_CONTROL_6__SeqInvBurstLimitInv__SHIFT                                                              0x0
+#define L2_CONTROL_6__SeqInvBurstLimitPDCReq__SHIFT                                                           0x8
+#define L2_CONTROL_6__SeqInvBurstLimitEn__SHIFT                                                               0x10
+#define L2_CONTROL_6__Perf2Threshold__SHIFT                                                                   0x18
+#define L2_CONTROL_6__SeqInvBurstLimitInv_MASK                                                                0x000000FFL
+#define L2_CONTROL_6__SeqInvBurstLimitPDCReq_MASK                                                             0x0000FF00L
+#define L2_CONTROL_6__SeqInvBurstLimitEn_MASK                                                                 0x00010000L
+#define L2_CONTROL_6__Perf2Threshold_MASK                                                                     0xFF000000L
+//L2_PDC_CONTROL
+#define L2_PDC_CONTROL__PDCLRUUpdatePri__SHIFT                                                                0x3
+#define L2_PDC_CONTROL__PDCParityEn__SHIFT                                                                    0x4
+#define L2_PDC_CONTROL__PDCInvalidationSel__SHIFT                                                             0x8
+#define L2_PDC_CONTROL__PDCSoftInvalidate__SHIFT                                                              0xa
+#define L2_PDC_CONTROL__PDC_Inv_Overlapping_Pages__SHIFT                                                      0xb
+#define L2_PDC_CONTROL__PDCSearchDirection__SHIFT                                                             0xc
+#define L2_PDC_CONTROL__PDCBypass__SHIFT                                                                      0xd
+#define L2_PDC_CONTROL__PDCModeLookupFix__SHIFT                                                               0xe
+#define L2_PDC_CONTROL__PDCParitySupport__SHIFT                                                               0xf
+#define L2_PDC_CONTROL__PDCWays__SHIFT                                                                        0x10
+#define L2_PDC_CONTROL__PDCEntries__SHIFT                                                                     0x1c
+#define L2_PDC_CONTROL__PDCLRUUpdatePri_MASK                                                                  0x00000008L
+#define L2_PDC_CONTROL__PDCParityEn_MASK                                                                      0x00000010L
+#define L2_PDC_CONTROL__PDCInvalidationSel_MASK                                                               0x00000300L
+#define L2_PDC_CONTROL__PDCSoftInvalidate_MASK                                                                0x00000400L
+#define L2_PDC_CONTROL__PDC_Inv_Overlapping_Pages_MASK                                                        0x00000800L
+#define L2_PDC_CONTROL__PDCSearchDirection_MASK                                                               0x00001000L
+#define L2_PDC_CONTROL__PDCBypass_MASK                                                                        0x00002000L
+#define L2_PDC_CONTROL__PDCModeLookupFix_MASK                                                                 0x00004000L
+#define L2_PDC_CONTROL__PDCParitySupport_MASK                                                                 0x00008000L
+#define L2_PDC_CONTROL__PDCWays_MASK                                                                          0x00FF0000L
+#define L2_PDC_CONTROL__PDCEntries_MASK                                                                       0xF0000000L
+//L2_PDC_HASH_CONTROL
+#define L2_PDC_HASH_CONTROL__PDCAddressMask__SHIFT                                                            0x10
+#define L2_PDC_HASH_CONTROL__PDCAddressMask_MASK                                                              0xFFFF0000L
+//L2_PDC_WAY_CONTROL
+#define L2_PDC_WAY_CONTROL__PDCWayDisable__SHIFT                                                              0x0
+#define L2_PDC_WAY_CONTROL__PDCWayAccessDisable__SHIFT                                                        0x10
+#define L2_PDC_WAY_CONTROL__PDCWayDisable_MASK                                                                0x0000FFFFL
+#define L2_PDC_WAY_CONTROL__PDCWayAccessDisable_MASK                                                          0xFFFF0000L
+//L2B_UPDATE_FILTER_CNTL
+#define L2B_UPDATE_FILTER_CNTL__L2b_Update_Filter_Bypass__SHIFT                                               0x0
+#define L2B_UPDATE_FILTER_CNTL__L2b_Update_Filter_RdLatency__SHIFT                                            0x1
+#define L2B_UPDATE_FILTER_CNTL__L2b_Update_Filter_Bypass_MASK                                                 0x00000001L
+#define L2B_UPDATE_FILTER_CNTL__L2b_Update_Filter_RdLatency_MASK                                              0x0000001EL
+//L2_TW_CONTROL
+#define L2_TW_CONTROL__RESERVED__SHIFT                                                                        0x0
+#define L2_TW_CONTROL__TWForceCoherent__SHIFT                                                                 0x6
+#define L2_TW_CONTROL__TWPrefetchEn__SHIFT                                                                    0x8
+#define L2_TW_CONTROL__TWPrefetchOnly4KDis__SHIFT                                                             0x9
+#define L2_TW_CONTROL__TWPTEOnUntransExcl__SHIFT                                                              0xa
+#define L2_TW_CONTROL__TWPTEOnAddrTransExcl__SHIFT                                                            0xb
+#define L2_TW_CONTROL__TWPrefetchRange__SHIFT                                                                 0xc
+#define L2_TW_CONTROL__TWFilter_Dis__SHIFT                                                                    0x10
+#define L2_TW_CONTROL__TWFilter_64B_Dis__SHIFT                                                                0x11
+#define L2_TW_CONTROL__TWContWalkOnPErrDis__SHIFT                                                             0x12
+#define L2_TW_CONTROL__TWSetAccessBit_Dis__SHIFT                                                              0x13
+#define L2_TW_CONTROL__TWClearAPBit_Dis__SHIFT                                                                0x14
+#define L2_TW_CONTROL__TWCacheNestedPTE__SHIFT                                                                0x19
+#define L2_TW_CONTROL__RESERVED_MASK                                                                          0x0000003FL
+#define L2_TW_CONTROL__TWForceCoherent_MASK                                                                   0x00000040L
+#define L2_TW_CONTROL__TWPrefetchEn_MASK                                                                      0x00000100L
+#define L2_TW_CONTROL__TWPrefetchOnly4KDis_MASK                                                               0x00000200L
+#define L2_TW_CONTROL__TWPTEOnUntransExcl_MASK                                                                0x00000400L
+#define L2_TW_CONTROL__TWPTEOnAddrTransExcl_MASK                                                              0x00000800L
+#define L2_TW_CONTROL__TWPrefetchRange_MASK                                                                   0x00007000L
+#define L2_TW_CONTROL__TWFilter_Dis_MASK                                                                      0x00010000L
+#define L2_TW_CONTROL__TWFilter_64B_Dis_MASK                                                                  0x00020000L
+#define L2_TW_CONTROL__TWContWalkOnPErrDis_MASK                                                               0x00040000L
+#define L2_TW_CONTROL__TWSetAccessBit_Dis_MASK                                                                0x00080000L
+#define L2_TW_CONTROL__TWClearAPBit_Dis_MASK                                                                  0x00100000L
+#define L2_TW_CONTROL__TWCacheNestedPTE_MASK                                                                  0x02000000L
+//L2_CP_CONTROL
+#define L2_CP_CONTROL__CPPrefetchDis__SHIFT                                                                   0x0
+#define L2_CP_CONTROL__CPFlushOnWait__SHIFT                                                                   0x1
+#define L2_CP_CONTROL__CPFlushOnInv__SHIFT                                                                    0x2
+#define L2_CP_CONTROL__CPStallCmdErrForIOTLBCmpl__SHIFT                                                       0x3
+#define L2_CP_CONTROL__CPForceReqPassPW__SHIFT                                                                0x4
+#define L2_CP_CONTROL__CPForceOneOutstandingCommand__SHIFT                                                    0x5
+#define L2_CP_CONTROL__CPRdDelay__SHIFT                                                                       0x10
+#define L2_CP_CONTROL__CPPrefetchDis_MASK                                                                     0x00000001L
+#define L2_CP_CONTROL__CPFlushOnWait_MASK                                                                     0x00000002L
+#define L2_CP_CONTROL__CPFlushOnInv_MASK                                                                      0x00000004L
+#define L2_CP_CONTROL__CPStallCmdErrForIOTLBCmpl_MASK                                                         0x00000008L
+#define L2_CP_CONTROL__CPForceReqPassPW_MASK                                                                  0x00000010L
+#define L2_CP_CONTROL__CPForceOneOutstandingCommand_MASK                                                      0x00000020L
+#define L2_CP_CONTROL__CPRdDelay_MASK                                                                         0xFFFF0000L
+//L2_CP_CONTROL_1
+#define L2_CP_CONTROL_1__CPL1Off__SHIFT                                                                       0x0
+#define L2_CP_CONTROL_1__Reserved__SHIFT                                                                      0x10
+#define L2_CP_CONTROL_1__CPL1Off_MASK                                                                         0x0000FFFFL
+#define L2_CP_CONTROL_1__Reserved_MASK                                                                        0xFFFF0000L
+//L2_TW_CONTROL_1
+#define L2_TW_CONTROL_1__TWTraceEn__SHIFT                                                                     0x0
+#define L2_TW_CONTROL_1__TWTraceNoWrap__SHIFT                                                                 0x1
+#define L2_TW_CONTROL_1__TWTraceForceDisable__SHIFT                                                           0x2
+#define L2_TW_CONTROL_1__TWTraceMask__SHIFT                                                                   0xf
+#define L2_TW_CONTROL_1__TWTraceEn_MASK                                                                       0x00000001L
+#define L2_TW_CONTROL_1__TWTraceNoWrap_MASK                                                                   0x00000002L
+#define L2_TW_CONTROL_1__TWTraceForceDisable_MASK                                                             0x00000004L
+#define L2_TW_CONTROL_1__TWTraceMask_MASK                                                                     0xFFFF8000L
+//L2_TW_CONTROL_2
+#define L2_TW_CONTROL_2__TWTraceAddrLo__SHIFT                                                                 0xc
+#define L2_TW_CONTROL_2__TWTraceAddrLo_MASK                                                                   0xFFFFF000L
+//L2_TW_CONTROL_3
+#define L2_TW_CONTROL_3__TWTraceAddrHi__SHIFT                                                                 0x0
+#define L2_TW_CONTROL_3__TWTraceAddrHi_MASK                                                                   0xFFFFFFFFL
+//L2_CREDIT_CONTROL_0
+#define L2_CREDIT_CONTROL_0__FC1Credits__SHIFT                                                                0x0
+#define L2_CREDIT_CONTROL_0__FC1Override__SHIFT                                                               0x7
+#define L2_CREDIT_CONTROL_0__FC3Credits__SHIFT                                                                0xf
+#define L2_CREDIT_CONTROL_0__FC3Override__SHIFT                                                               0x15
+#define L2_CREDIT_CONTROL_0__FC1Credits_MASK                                                                  0x0000007FL
+#define L2_CREDIT_CONTROL_0__FC1Override_MASK                                                                 0x00000080L
+#define L2_CREDIT_CONTROL_0__FC3Credits_MASK                                                                  0x001F8000L
+#define L2_CREDIT_CONTROL_0__FC3Override_MASK                                                                 0x00200000L
+//L2_CREDIT_CONTROL_1
+#define L2_CREDIT_CONTROL_1__CP_PREFETCH_credits__SHIFT                                                       0x10
+#define L2_CREDIT_CONTROL_1__PPR_MCIF_credits__SHIFT                                                          0x14
+#define L2_CREDIT_CONTROL_1__CP_PREFETCH_credits_MASK                                                         0x000F0000L
+#define L2_CREDIT_CONTROL_1__PPR_MCIF_credits_MASK                                                            0x00F00000L
+//L2_ERR_RULE_CONTROL_0
+#define L2_ERR_RULE_CONTROL_0__ERRRuleLock0__SHIFT                                                            0x0
+#define L2_ERR_RULE_CONTROL_0__ERRRuleDisable0__SHIFT                                                         0x1
+#define L2_ERR_RULE_CONTROL_0__ERRRuleLock0_MASK                                                              0x00000001L
+#define L2_ERR_RULE_CONTROL_0__ERRRuleDisable0_MASK                                                           0xFFFFFFFEL
+//L2_ERR_RULE_CONTROL_1
+#define L2_ERR_RULE_CONTROL_1__ERRRuleDisable1__SHIFT                                                         0x0
+#define L2_ERR_RULE_CONTROL_1__ERRRuleDisable1_MASK                                                           0xFFFFFFFFL
+//L2_ERR_RULE_CONTROL_2
+#define L2_ERR_RULE_CONTROL_2__ERRRuleDisable2__SHIFT                                                         0x0
+#define L2_ERR_RULE_CONTROL_2__ERRRuleDisable2_MASK                                                           0xFFFFFFFFL
+//L2_L2B_CK_GATE_CONTROL
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BRegsDisable__SHIFT                                                   0x0
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BDynamicDisable__SHIFT                                                0x1
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BMiscDisable__SHIFT                                                   0x2
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BCacheDisable__SHIFT                                                  0x3
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BAPCDisable__SHIFT                                                    0x4
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BPerfDisable__SHIFT                                                   0x5
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BPortClkDis__SHIFT                                                    0x6
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BLength__SHIFT                                                        0x10
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BStop__SHIFT                                                          0x12
+#define L2_L2B_CK_GATE_CONTROL__Reserved__SHIFT                                                               0x14
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BRegsDisable_MASK                                                     0x00000001L
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BDynamicDisable_MASK                                                  0x00000002L
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BMiscDisable_MASK                                                     0x00000004L
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BCacheDisable_MASK                                                    0x00000008L
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BAPCDisable_MASK                                                      0x00000010L
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BPerfDisable_MASK                                                     0x00000020L
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BPortClkDis_MASK                                                      0x00000040L
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BLength_MASK                                                          0x00030000L
+#define L2_L2B_CK_GATE_CONTROL__CKGateL2BStop_MASK                                                            0x000C0000L
+#define L2_L2B_CK_GATE_CONTROL__Reserved_MASK                                                                 0xFFF00000L
+//PPR_CONTROL
+#define PPR_CONTROL__PPR_IntTimeDelay__SHIFT                                                                  0x0
+#define PPR_CONTROL__PPR_IntReqDelay__SHIFT                                                                   0x8
+#define PPR_CONTROL__PPR_IntCoallesce_En__SHIFT                                                               0x10
+#define PPR_CONTROL__PPR_IntTimeDelay_MASK                                                                    0x000000FFL
+#define PPR_CONTROL__PPR_IntReqDelay_MASK                                                                     0x0000FF00L
+#define PPR_CONTROL__PPR_IntCoallesce_En_MASK                                                                 0x00010000L
+//L2_L2B_PGSIZE_CONTROL
+#define L2_L2B_PGSIZE_CONTROL__L2BREG_GST_PGSIZE__SHIFT                                                       0x0
+#define L2_L2B_PGSIZE_CONTROL__L2BREG_HOST_PGSIZE__SHIFT                                                      0x8
+#define L2_L2B_PGSIZE_CONTROL__L2BREG_GST_PGSIZE_MASK                                                         0x0000007FL
+#define L2_L2B_PGSIZE_CONTROL__L2BREG_HOST_PGSIZE_MASK                                                        0x00007F00L
+//L2_PERF_CNTL_2
+#define L2_PERF_CNTL_2__L2PerfEvent4__SHIFT                                                                   0x0
+#define L2_PERF_CNTL_2__L2PerfEvent5__SHIFT                                                                   0x8
+#define L2_PERF_CNTL_2__L2PerfCountUpper4__SHIFT                                                              0x10
+#define L2_PERF_CNTL_2__L2PerfCountUpper5__SHIFT                                                              0x18
+#define L2_PERF_CNTL_2__L2PerfEvent4_MASK                                                                     0x000000FFL
+#define L2_PERF_CNTL_2__L2PerfEvent5_MASK                                                                     0x0000FF00L
+#define L2_PERF_CNTL_2__L2PerfCountUpper4_MASK                                                                0x00FF0000L
+#define L2_PERF_CNTL_2__L2PerfCountUpper5_MASK                                                                0xFF000000L
+//L2_PERF_COUNT_4
+#define L2_PERF_COUNT_4__L2PerfCount4__SHIFT                                                                  0x0
+#define L2_PERF_COUNT_4__L2PerfCount4_MASK                                                                    0xFFFFFFFFL
+//L2_PERF_COUNT_5
+#define L2_PERF_COUNT_5__L2PerfCount5__SHIFT                                                                  0x0
+#define L2_PERF_COUNT_5__L2PerfCount5_MASK                                                                    0xFFFFFFFFL
+//L2_PERF_CNTL_3
+#define L2_PERF_CNTL_3__L2PerfEvent6__SHIFT                                                                   0x0
+#define L2_PERF_CNTL_3__L2PerfEvent7__SHIFT                                                                   0x8
+#define L2_PERF_CNTL_3__L2PerfCountUpper6__SHIFT                                                              0x10
+#define L2_PERF_CNTL_3__L2PerfCountUpper7__SHIFT                                                              0x18
+#define L2_PERF_CNTL_3__L2PerfEvent6_MASK                                                                     0x000000FFL
+#define L2_PERF_CNTL_3__L2PerfEvent7_MASK                                                                     0x0000FF00L
+#define L2_PERF_CNTL_3__L2PerfCountUpper6_MASK                                                                0x00FF0000L
+#define L2_PERF_CNTL_3__L2PerfCountUpper7_MASK                                                                0xFF000000L
+//L2_PERF_COUNT_6
+#define L2_PERF_COUNT_6__L2PerfCount6__SHIFT                                                                  0x0
+#define L2_PERF_COUNT_6__L2PerfCount6_MASK                                                                    0xFFFFFFFFL
+//L2_PERF_COUNT_7
+#define L2_PERF_COUNT_7__L2PerfCount7__SHIFT                                                                  0x0
+#define L2_PERF_COUNT_7__L2PerfCount7_MASK                                                                    0xFFFFFFFFL
+//L2B_SDP_PARITY_ERROR_EN
+#define L2B_SDP_PARITY_ERROR_EN__DVM_PARITY_ERROR_EN__SHIFT                                                   0x0
+#define L2B_SDP_PARITY_ERROR_EN__CP_PARITY_ERROR_EN__SHIFT                                                    0x1
+#define L2B_SDP_PARITY_ERROR_EN__TWW_PARITY_ERROR_EN__SHIFT                                                   0x2
+#define L2B_SDP_PARITY_ERROR_EN__VFMMIO_PARITY_ERROR_EN__SHIFT                                                0x3
+#define L2B_SDP_PARITY_ERROR_EN__MSG_PARITY_ERROR_EN__SHIFT                                                   0x4
+#define L2B_SDP_PARITY_ERROR_EN__DVM_PARITY_ERROR_EN_MASK                                                     0x00000001L
+#define L2B_SDP_PARITY_ERROR_EN__CP_PARITY_ERROR_EN_MASK                                                      0x00000002L
+#define L2B_SDP_PARITY_ERROR_EN__TWW_PARITY_ERROR_EN_MASK                                                     0x00000004L
+#define L2B_SDP_PARITY_ERROR_EN__VFMMIO_PARITY_ERROR_EN_MASK                                                  0x00000008L
+#define L2B_SDP_PARITY_ERROR_EN__MSG_PARITY_ERROR_EN_MASK                                                     0x00000010L
+//L2_ECO_CNTRL_1
+#define L2_ECO_CNTRL_1__L2_ECO_1__SHIFT                                                                       0x0
+#define L2_ECO_CNTRL_1__L2_ECO_1_MASK                                                                         0xFFFFFFFFL
+//L2_CP_CONTROL_2
+#define L2_CP_CONTROL_2__OVERCONSTRAIN_CMD_WQ_ON_INV__SHIFT                                                   0x0
+#define L2_CP_CONTROL_2__LEGACY_CWWB_PATH__SHIFT                                                              0x1
+#define L2_CP_CONTROL_2__gstcp_always_refetch_v1__SHIFT                                                       0x2
+#define L2_CP_CONTROL_2__inv_waitcmpl_mode__SHIFT                                                             0x16
+#define L2_CP_CONTROL_2__inv_dvmsync_mode__SHIFT                                                              0x18
+#define L2_CP_CONTROL_2__inv_pspflush_mode__SHIFT                                                             0x1a
+#define L2_CP_CONTROL_2__wqmask_propagation_latency__SHIFT                                                    0x1c
+#define L2_CP_CONTROL_2__OVERCONSTRAIN_CMD_WQ_ON_INV_MASK                                                     0x00000001L
+#define L2_CP_CONTROL_2__LEGACY_CWWB_PATH_MASK                                                                0x00000002L
+#define L2_CP_CONTROL_2__gstcp_always_refetch_v1_MASK                                                         0x00000004L
+#define L2_CP_CONTROL_2__inv_waitcmpl_mode_MASK                                                               0x00C00000L
+#define L2_CP_CONTROL_2__inv_dvmsync_mode_MASK                                                                0x03000000L
+#define L2_CP_CONTROL_2__inv_pspflush_mode_MASK                                                               0x0C000000L
+#define L2_CP_CONTROL_2__wqmask_propagation_latency_MASK                                                      0xF0000000L
+//L2_CP_CONTROL_3
+#define L2_CP_CONTROL_3__INV_CMD_PRIORITY__SHIFT                                                              0x0
+#define L2_CP_CONTROL_3__WAIT_CMD_PRIORITY__SHIFT                                                             0x4
+#define L2_CP_CONTROL_3__SYNC_CMD_PRIORITY__SHIFT                                                             0x8
+#define L2_CP_CONTROL_3__PSP_CMD_PRIORITY__SHIFT                                                              0xc
+#define L2_CP_CONTROL_3__INV_CMD_PRIORITY_MASK                                                                0x0000000FL
+#define L2_CP_CONTROL_3__WAIT_CMD_PRIORITY_MASK                                                               0x000000F0L
+#define L2_CP_CONTROL_3__SYNC_CMD_PRIORITY_MASK                                                               0x00000F00L
+#define L2_CP_CONTROL_3__PSP_CMD_PRIORITY_MASK                                                                0x0000F000L
+
+
+// addressBlock: aid_nbio_iohub_nb_ioapiccfg_ioapic_cfgdec
+//FEATURES_ENABLE
+#define FEATURES_ENABLE__Ioapic_id_ext_en__SHIFT                                                              0x2
+#define FEATURES_ENABLE__Ioapic_sb_feature_en__SHIFT                                                          0x4
+#define FEATURES_ENABLE__Ioapic_secondary_en__SHIFT                                                           0x5
+#define FEATURES_ENABLE__Ioapic_processor_mode__SHIFT                                                         0x8
+#define FEATURES_ENABLE__INTx_LevelOnlyMode__SHIFT                                                            0x9
+#define FEATURES_ENABLE__Ioapic_id_ext_en_MASK                                                                0x00000004L
+#define FEATURES_ENABLE__Ioapic_sb_feature_en_MASK                                                            0x00000010L
+#define FEATURES_ENABLE__Ioapic_secondary_en_MASK                                                             0x00000020L
+#define FEATURES_ENABLE__Ioapic_processor_mode_MASK                                                           0x00000100L
+#define FEATURES_ENABLE__INTx_LevelOnlyMode_MASK                                                              0x00000200L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_rc_bifcfgdecp
+//BIF_CFG_DEV0_RC_VENDOR_ID
+#define BIF_CFG_DEV0_RC_VENDOR_ID__VENDOR_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_VENDOR_ID__VENDOR_ID_MASK                                                             0xFFFFL
+//BIF_CFG_DEV0_RC_DEVICE_ID
+#define BIF_CFG_DEV0_RC_DEVICE_ID__DEVICE_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_DEVICE_ID__DEVICE_ID_MASK                                                             0xFFFFL
+//BIF_CFG_DEV0_RC_COMMAND
+#define BIF_CFG_DEV0_RC_COMMAND__IOEN_DN__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_COMMAND__MEMEN_DN__SHIFT                                                              0x1
+#define BIF_CFG_DEV0_RC_COMMAND__BUS_MASTER_EN__SHIFT                                                         0x2
+#define BIF_CFG_DEV0_RC_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                      0x3
+#define BIF_CFG_DEV0_RC_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_RC_COMMAND__PAL_SNOOP_EN__SHIFT                                                          0x5
+#define BIF_CFG_DEV0_RC_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                                 0x6
+#define BIF_CFG_DEV0_RC_COMMAND__AD_STEPPING__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_RC_COMMAND__SERR_EN__SHIFT                                                               0x8
+#define BIF_CFG_DEV0_RC_COMMAND__FAST_B2B_EN__SHIFT                                                           0x9
+#define BIF_CFG_DEV0_RC_COMMAND__INT_DIS__SHIFT                                                               0xa
+#define BIF_CFG_DEV0_RC_COMMAND__IOEN_DN_MASK                                                                 0x0001L
+#define BIF_CFG_DEV0_RC_COMMAND__MEMEN_DN_MASK                                                                0x0002L
+#define BIF_CFG_DEV0_RC_COMMAND__BUS_MASTER_EN_MASK                                                           0x0004L
+#define BIF_CFG_DEV0_RC_COMMAND__SPECIAL_CYCLE_EN_MASK                                                        0x0008L
+#define BIF_CFG_DEV0_RC_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_RC_COMMAND__PAL_SNOOP_EN_MASK                                                            0x0020L
+#define BIF_CFG_DEV0_RC_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                   0x0040L
+#define BIF_CFG_DEV0_RC_COMMAND__AD_STEPPING_MASK                                                             0x0080L
+#define BIF_CFG_DEV0_RC_COMMAND__SERR_EN_MASK                                                                 0x0100L
+#define BIF_CFG_DEV0_RC_COMMAND__FAST_B2B_EN_MASK                                                             0x0200L
+#define BIF_CFG_DEV0_RC_COMMAND__INT_DIS_MASK                                                                 0x0400L
+//BIF_CFG_DEV0_RC_STATUS
+#define BIF_CFG_DEV0_RC_STATUS__IMMEDIATE_READINESS__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_RC_STATUS__INT_STATUS__SHIFT                                                             0x3
+#define BIF_CFG_DEV0_RC_STATUS__CAP_LIST__SHIFT                                                               0x4
+#define BIF_CFG_DEV0_RC_STATUS__PCI_66_CAP__SHIFT                                                             0x5
+#define BIF_CFG_DEV0_RC_STATUS__FAST_BACK_CAPABLE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_RC_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                               0x8
+#define BIF_CFG_DEV0_RC_STATUS__DEVSEL_TIMING__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_RC_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_RC_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                  0xe
+#define BIF_CFG_DEV0_RC_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_RC_STATUS__IMMEDIATE_READINESS_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_RC_STATUS__INT_STATUS_MASK                                                               0x0008L
+#define BIF_CFG_DEV0_RC_STATUS__CAP_LIST_MASK                                                                 0x0010L
+#define BIF_CFG_DEV0_RC_STATUS__PCI_66_CAP_MASK                                                               0x0020L
+#define BIF_CFG_DEV0_RC_STATUS__FAST_BACK_CAPABLE_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_RC_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_RC_STATUS__DEVSEL_TIMING_MASK                                                            0x0600L
+#define BIF_CFG_DEV0_RC_STATUS__SIGNAL_TARGET_ABORT_MASK                                                      0x0800L
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_TARGET_ABORT_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_MASTER_ABORT_MASK                                                    0x2000L
+#define BIF_CFG_DEV0_RC_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                    0x4000L
+#define BIF_CFG_DEV0_RC_STATUS__PARITY_ERROR_DETECTED_MASK                                                    0x8000L
+//BIF_CFG_DEV0_RC_REVISION_ID
+#define BIF_CFG_DEV0_RC_REVISION_ID__MINOR_REV_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC_REVISION_ID__MAJOR_REV_ID__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_RC_REVISION_ID__MINOR_REV_ID_MASK                                                        0x0FL
+#define BIF_CFG_DEV0_RC_REVISION_ID__MAJOR_REV_ID_MASK                                                        0xF0L
+//BIF_CFG_DEV0_RC_PROG_INTERFACE
+#define BIF_CFG_DEV0_RC_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_PROG_INTERFACE__PROG_INTERFACE_MASK                                                   0xFFL
+//BIF_CFG_DEV0_RC_SUB_CLASS
+#define BIF_CFG_DEV0_RC_SUB_CLASS__SUB_CLASS__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_SUB_CLASS__SUB_CLASS_MASK                                                             0xFFL
+//BIF_CFG_DEV0_RC_BASE_CLASS
+#define BIF_CFG_DEV0_RC_BASE_CLASS__BASE_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_BASE_CLASS__BASE_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_RC_CACHE_LINE
+#define BIF_CFG_DEV0_RC_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_RC_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                      0xFFL
+//BIF_CFG_DEV0_RC_LATENCY
+#define BIF_CFG_DEV0_RC_LATENCY__LATENCY_TIMER__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_LATENCY__LATENCY_TIMER_MASK                                                           0xFFL
+//BIF_CFG_DEV0_RC_HEADER
+#define BIF_CFG_DEV0_RC_HEADER__HEADER_TYPE__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_RC_HEADER__DEVICE_TYPE__SHIFT                                                            0x7
+#define BIF_CFG_DEV0_RC_HEADER__HEADER_TYPE_MASK                                                              0x7FL
+#define BIF_CFG_DEV0_RC_HEADER__DEVICE_TYPE_MASK                                                              0x80L
+//BIF_CFG_DEV0_RC_BIST
+#define BIF_CFG_DEV0_RC_BIST__BIST_COMP__SHIFT                                                                0x0
+#define BIF_CFG_DEV0_RC_BIST__BIST_STRT__SHIFT                                                                0x6
+#define BIF_CFG_DEV0_RC_BIST__BIST_CAP__SHIFT                                                                 0x7
+#define BIF_CFG_DEV0_RC_BIST__BIST_COMP_MASK                                                                  0x0FL
+#define BIF_CFG_DEV0_RC_BIST__BIST_STRT_MASK                                                                  0x40L
+#define BIF_CFG_DEV0_RC_BIST__BIST_CAP_MASK                                                                   0x80L
+//BIF_CFG_DEV0_RC_BASE_ADDR_1
+#define BIF_CFG_DEV0_RC_BASE_ADDR_1__BASE_ADDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_BASE_ADDR_1__BASE_ADDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_BASE_ADDR_2
+#define BIF_CFG_DEV0_RC_BASE_ADDR_2__BASE_ADDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_BASE_ADDR_2__BASE_ADDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER__SHIFT                                0x18
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                            0x0000FF00L
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                              0x00FF0000L
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER_MASK                                  0xFF000000L
+//BIF_CFG_DEV0_RC_IO_BASE_LIMIT
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_BASE_TYPE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_BASE__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_LIMIT_TYPE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                        0xc
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_BASE_TYPE_MASK                                                      0x000FL
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_BASE_MASK                                                           0x00F0L
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_LIMIT_TYPE_MASK                                                     0x0F00L
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_LIMIT_MASK                                                          0xF000L
+//BIF_CFG_DEV0_RC_SECONDARY_STATUS
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__PCI_66_CAP__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__FAST_BACK_CAPABLE__SHIFT                                            0x7
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                     0x8
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__DEVSEL_TIMING__SHIFT                                                0x9
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                          0xb
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                        0xc
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                        0xd
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR__SHIFT                                        0xe
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__PARITY_ERROR_DETECTED__SHIFT                                        0xf
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__PCI_66_CAP_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__FAST_BACK_CAPABLE_MASK                                              0x0080L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                       0x0100L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__DEVSEL_TIMING_MASK                                                  0x0600L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__SIGNAL_TARGET_ABORT_MASK                                            0x0800L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_TARGET_ABORT_MASK                                          0x1000L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_MASTER_ABORT_MASK                                          0x2000L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR_MASK                                          0x4000L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__PARITY_ERROR_DETECTED_MASK                                          0x8000L
+//BIF_CFG_DEV0_RC_MEM_BASE_LIMIT
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                    0x0000000FL
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                   0x0000FFF0L
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC_PREF_BASE_LIMIT
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                           0x4
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                              0x0000000FL
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                             0x0000FFF0L
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_PREF_BASE_UPPER
+#define BIF_CFG_DEV0_RC_PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PREF_LIMIT_UPPER
+#define BIF_CFG_DEV0_RC_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                               0x10
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                 0xFFFF0000L
+//BIF_CFG_DEV0_RC_CAP_PTR
+#define BIF_CFG_DEV0_RC_CAP_PTR__CAP_PTR__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_CAP_PTR__CAP_PTR_MASK                                                                 0xFFL
+//BIF_CFG_DEV0_RC_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                           0x1
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                        0x00000001L
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                             0x0000000EL
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                            0x000000F0L
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__BASE_ADDR_MASK                                                         0xFFFFF800L
+//BIF_CFG_DEV0_RC_INTERRUPT_LINE
+#define BIF_CFG_DEV0_RC_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                   0xFFL
+//BIF_CFG_DEV0_RC_INTERRUPT_PIN
+#define BIF_CFG_DEV0_RC_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                     0xFFL
+//IRQ_BRIDGE_CNTL
+#define IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN__SHIFT                                                            0x0
+#define IRQ_BRIDGE_CNTL__SERR_EN__SHIFT                                                                       0x1
+#define IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                                        0x2
+#define IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                                        0x3
+#define IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                                       0x4
+#define IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE__SHIFT                                                             0x5
+#define IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET__SHIFT                                                           0x6
+#define IRQ_BRIDGE_CNTL__FAST_B2B_EN__SHIFT                                                                   0x7
+#define IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER__SHIFT                                                         0x8
+#define IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER__SHIFT                                                       0x9
+#define IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS__SHIFT                                                          0xa
+#define IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE__SHIFT                                                     0xb
+#define IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN_MASK                                                              0x0001L
+#define IRQ_BRIDGE_CNTL__SERR_EN_MASK                                                                         0x0002L
+#define IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                                          0x0004L
+#define IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                                          0x0008L
+#define IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                                         0x0010L
+#define IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE_MASK                                                               0x0020L
+#define IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_MASK                                                             0x0040L
+#define IRQ_BRIDGE_CNTL__FAST_B2B_EN_MASK                                                                     0x0080L
+#define IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER_MASK                                                           0x0100L
+#define IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER_MASK                                                         0x0200L
+#define IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS_MASK                                                            0x0400L
+#define IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE_MASK                                                       0x0800L
+//BIF_CFG_DEV0_RC_EXT_BRIDGE_CNTL
+#define BIF_CFG_DEV0_RC_EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                   0x01L
+//BIF_CFG_DEV0_RC_PMI_CAP_LIST
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__CAP_ID_MASK                                                             0x00FFL
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__NEXT_PTR_MASK                                                           0xFF00L
+//BIF_CFG_DEV0_RC_PMI_CAP
+#define BIF_CFG_DEV0_RC_PMI_CAP__VERSION__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_CLOCK__SHIFT                                                             0x3
+#define BIF_CFG_DEV0_RC_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                   0x4
+#define BIF_CFG_DEV0_RC_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_RC_PMI_CAP__AUX_CURRENT__SHIFT                                                           0x6
+#define BIF_CFG_DEV0_RC_PMI_CAP__D1_SUPPORT__SHIFT                                                            0x9
+#define BIF_CFG_DEV0_RC_PMI_CAP__D2_SUPPORT__SHIFT                                                            0xa
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_SUPPORT__SHIFT                                                           0xb
+#define BIF_CFG_DEV0_RC_PMI_CAP__VERSION_MASK                                                                 0x0007L
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_CLOCK_MASK                                                               0x0008L
+#define BIF_CFG_DEV0_RC_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                     0x0010L
+#define BIF_CFG_DEV0_RC_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                       0x0020L
+#define BIF_CFG_DEV0_RC_PMI_CAP__AUX_CURRENT_MASK                                                             0x01C0L
+#define BIF_CFG_DEV0_RC_PMI_CAP__D1_SUPPORT_MASK                                                              0x0200L
+#define BIF_CFG_DEV0_RC_PMI_CAP__D2_SUPPORT_MASK                                                              0x0400L
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_SUPPORT_MASK                                                             0xF800L
+//BIF_CFG_DEV0_RC_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_EN__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                   0x9
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                                 0x16
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                    0x17
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__POWER_STATE_MASK                                                     0x00000003L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_EN_MASK                                                          0x00000100L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                     0x00001E00L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                      0x00006000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_STATUS_MASK                                                      0x00008000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                   0x00400000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                      0x00800000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PMI_DATA_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC_PCIE_CAP
+#define BIF_CFG_DEV0_RC_PCIE_CAP__VERSION__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC_PCIE_CAP__DEVICE_TYPE__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_RC_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_RC_PCIE_CAP__VERSION_MASK                                                                0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_CAP__DEVICE_TYPE_MASK                                                            0x00F0L
+#define BIF_CFG_DEV0_RC_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                       0x0100L
+#define BIF_CFG_DEV0_RC_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                        0x3E00L
+//BIF_CFG_DEV0_RC_DEVICE_CAP
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                              0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                           0xf
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                          0x12
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                          0x1a
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                        0x1c
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                  0x00000007L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__PHANTOM_FUNC_MASK                                                         0x00000018L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__EXTENDED_TAG_MASK                                                         0x00000020L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                               0x000001C0L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                                0x00000E00L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                             0x00008000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                             0x00010000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                            0x03FC0000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                            0x0C000000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__FLR_CAPABLE_MASK                                                          0x10000000L
+//BIF_CFG_DEV0_RC_DEVICE_CNTL
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                      0x2
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                   0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                   0xa
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                             0xc
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN__SHIFT                                               0xf
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__CORR_ERR_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                    0x0002L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                        0x0004L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__USR_REPORT_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                      0x0010L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                    0x00E0L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                     0x0200L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                     0x0400L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                         0x0800L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                               0x7000L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN_MASK                                                 0x8000L
+//BIF_CFG_DEV0_RC_DEVICE_STATUS
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__CORR_ERR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__FATAL_ERR__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__USR_DETECTED__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__AUX_PWR__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                               0x5
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                   0x6
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__CORR_ERR_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__FATAL_ERR_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__USR_DETECTED_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__AUX_PWR_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                                 0x0020L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                     0x0040L
+//BIF_CFG_DEV0_RC_LINK_CAP
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_SPEED__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_WIDTH__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_RC_LINK_CAP__PM_SUPPORT__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_RC_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_RC_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                      0xf
+#define BIF_CFG_DEV0_RC_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                               0x12
+#define BIF_CFG_DEV0_RC_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                          0x13
+#define BIF_CFG_DEV0_RC_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                             0x15
+#define BIF_CFG_DEV0_RC_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                          0x16
+#define BIF_CFG_DEV0_RC_LINK_CAP__PORT_NUMBER__SHIFT                                                          0x18
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_SPEED_MASK                                                             0x0000000FL
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_WIDTH_MASK                                                             0x000003F0L
+#define BIF_CFG_DEV0_RC_LINK_CAP__PM_SUPPORT_MASK                                                             0x00000C00L
+#define BIF_CFG_DEV0_RC_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                       0x00007000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__L1_EXIT_LATENCY_MASK                                                        0x00038000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                                 0x00040000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                            0x00100000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                               0x00200000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                            0x00400000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__PORT_NUMBER_MASK                                                            0xFF000000L
+//BIF_CFG_DEV0_RC_LINK_CNTL
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PM_CONTROL__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                        0x2
+#define BIF_CFG_DEV0_RC_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_DIS__SHIFT                                                            0x4
+#define BIF_CFG_DEV0_RC_LINK_CNTL__RETRAIN_LINK__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_RC_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                    0x6
+#define BIF_CFG_DEV0_RC_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_RC_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                         0x9
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                           0xa
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                           0xb
+#define BIF_CFG_DEV0_RC_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                               0xe
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PM_CONTROL_MASK                                                            0x0003L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                          0x0004L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_DIS_MASK                                                              0x0010L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__RETRAIN_LINK_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                      0x0040L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__EXTENDED_SYNC_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                             0x0100L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                           0x0200L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                             0x0400L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                             0x0800L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                                 0xC000L
+//BIF_CFG_DEV0_RC_LINK_STATUS
+#define BIF_CFG_DEV0_RC_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                             0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_TRAINING__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_RC_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_RC_LINK_STATUS__DL_ACTIVE__SHIFT                                                         0xd
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                         0xe
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                         0xf
+#define BIF_CFG_DEV0_RC_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_RC_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                               0x03F0L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_TRAINING_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                      0x1000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__DL_ACTIVE_MASK                                                           0x2000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                           0x4000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                           0x8000L
+//BIF_CFG_DEV0_RC_SLOT_CAP
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ATTN_BUTTON_PRESENT__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PWR_CONTROLLER_PRESENT__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC_SLOT_CAP__MRL_SENSOR_PRESENT__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ATTN_INDICATOR_PRESENT__SHIFT                                               0x3
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PWR_INDICATOR_PRESENT__SHIFT                                                0x4
+#define BIF_CFG_DEV0_RC_SLOT_CAP__HOTPLUG_SURPRISE__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_RC_SLOT_CAP__HOTPLUG_CAPABLE__SHIFT                                                      0x6
+#define BIF_CFG_DEV0_RC_SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_RC_SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED__SHIFT                                       0x12
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PHYSICAL_SLOT_NUM__SHIFT                                                    0x13
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ATTN_BUTTON_PRESENT_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PWR_CONTROLLER_PRESENT_MASK                                                 0x00000002L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__MRL_SENSOR_PRESENT_MASK                                                     0x00000004L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ATTN_INDICATOR_PRESENT_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PWR_INDICATOR_PRESENT_MASK                                                  0x00000010L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__HOTPLUG_SURPRISE_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__HOTPLUG_CAPABLE_MASK                                                        0x00000040L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                   0x00007F80L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                   0x00018000L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PHYSICAL_SLOT_NUM_MASK                                                      0xFFF80000L
+//BIF_CFG_DEV0_RC_SLOT_CNTL
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_FAULT_DETECTED_EN__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__MRL_SENSOR_CHANGED_EN__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN__SHIFT                                          0x3
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__HOTPLUG_INTR_EN__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ATTN_INDICATOR_CNTL__SHIFT                                                 0x6
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_INDICATOR_CNTL__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_CONTROLLER_CNTL__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL__SHIFT                                          0xb
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__DL_STATE_CHANGED_EN__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE__SHIFT                                         0xd
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__INBAND_PD_DISABLE__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN_MASK                                                0x0001L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_FAULT_DETECTED_EN_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__MRL_SENSOR_CHANGED_EN_MASK                                                 0x0004L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN_MASK                                            0x0008L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__HOTPLUG_INTR_EN_MASK                                                       0x0020L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ATTN_INDICATOR_CNTL_MASK                                                   0x00C0L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_INDICATOR_CNTL_MASK                                                    0x0300L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_CONTROLLER_CNTL_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL_MASK                                            0x0800L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__DL_STATE_CHANGED_EN_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE_MASK                                           0x2000L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__INBAND_PD_DISABLE_MASK                                                     0x4000L
+//BIF_CFG_DEV0_RC_SLOT_STATUS
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__ATTN_BUTTON_PRESSED__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PWR_FAULT_DETECTED__SHIFT                                                0x1
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__MRL_SENSOR_CHANGED__SHIFT                                                0x2
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PRESENCE_DETECT_CHANGED__SHIFT                                           0x3
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__COMMAND_COMPLETED__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__MRL_SENSOR_STATE__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PRESENCE_DETECT_STATE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__DL_STATE_CHANGED__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__ATTN_BUTTON_PRESSED_MASK                                                 0x0001L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PWR_FAULT_DETECTED_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__MRL_SENSOR_CHANGED_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PRESENCE_DETECT_CHANGED_MASK                                             0x0008L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__COMMAND_COMPLETED_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__MRL_SENSOR_STATE_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PRESENCE_DETECT_STATE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS_MASK                                        0x0080L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__DL_STATE_CHANGED_MASK                                                    0x0100L
+//BIF_CFG_DEV0_RC_ROOT_CNTL
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN__SHIFT                                             0x1
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__PM_INTERRUPT_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN_MASK                                               0x0002L
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__PM_INTERRUPT_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                            0x0010L
+//BIF_CFG_DEV0_RC_ROOT_CAP
+#define BIF_CFG_DEV0_RC_ROOT_CAP__CRS_SOFTWARE_VISIBILITY__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_ROOT_CAP__CRS_SOFTWARE_VISIBILITY_MASK                                                0x0001L
+//BIF_CFG_DEV0_RC_ROOT_STATUS
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_REQUESTOR_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_STATUS__SHIFT                                                        0x10
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_PENDING__SHIFT                                                       0x11
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_REQUESTOR_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_STATUS_MASK                                                          0x00010000L
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_PENDING_MASK                                                         0x00020000L
+//BIF_CFG_DEV0_RC_DEVICE_CAP2
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                       0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                          0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                        0x6
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                        0x7
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                        0x8
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                         0xa
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                                0xc
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                     0xe
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                   0x11
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                    0x12
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                      0x14
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                      0x15
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                          0x16
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                    0x18
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                     0x1a
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                         0x0000000FL
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                          0x00000040L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                          0x00000100L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                              0x00000200L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                           0x00000400L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                       0x00000800L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                       0x0000C000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                     0x00020000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                      0x000C0000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                        0x00200000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                            0x00C00000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                      0x03000000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                       0x04000000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_RC_DEVICE_CNTL2
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                                0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                              0x6
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                               0x8
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__LTR_EN__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                     0xb
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                     0xc
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__OBFF_EN__SHIFT                                                          0xd
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                  0x0020L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                                0x0040L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                           0x0080L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                              0x0200L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__LTR_EN_MASK                                                             0x0400L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                       0x0800L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                       0x1000L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__OBFF_EN_MASK                                                            0x6000L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                        0x8000L
+//BIF_CFG_DEV0_RC_DEVICE_STATUS2
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS2__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS2__RESERVED_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_RC_LINK_CAP2
+#define BIF_CFG_DEV0_RC_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                                0x1
+#define BIF_CFG_DEV0_RC_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                           0x17
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                           0x18
+#define BIF_CFG_DEV0_RC_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_RC_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                  0x000000FEL
+#define BIF_CFG_DEV0_RC_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                   0x00000100L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                              0x0000FE00L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                              0x007F0000L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                             0x00800000L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                             0x01000000L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__DRS_SUPPORTED_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_RC_LINK_CNTL2
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                        0x5
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                              0x6
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                               0xa
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                              0xc
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                    0x000FL
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                          0x0020L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                                0x0040L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__XMIT_MARGIN_MASK                                                          0x0380L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                                0xF000L
+//BIF_CFG_DEV0_RC_LINK_STATUS2
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                  0x2
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                  0x3
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                  0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                    0x5
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                                0x6
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                                0x7
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                             0x8
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                    0xc
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                             0xf
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                               0x0001L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                          0x0002L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                    0x0004L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                    0x0008L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                    0x0010L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                      0x0020L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                               0x0300L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                      0x7000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                               0x8000L
+//BIF_CFG_DEV0_RC_SLOT_CAP2
+#define BIF_CFG_DEV0_RC_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED_MASK                                           0x00000001L
+//BIF_CFG_DEV0_RC_SLOT_CNTL2
+#define BIF_CFG_DEV0_RC_SLOT_CNTL2__RESERVED__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_SLOT_CNTL2__RESERVED_MASK                                                             0xFFFFL
+//BIF_CFG_DEV0_RC_SLOT_STATUS2
+#define BIF_CFG_DEV0_RC_SLOT_STATUS2__RESERVED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_SLOT_STATUS2__RESERVED_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_CAP_LIST
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__CAP_ID_MASK                                                             0x00FFL
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__NEXT_PTR_MASK                                                           0xFF00L
+//BIF_CFG_DEV0_RC_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EN__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                    0x1
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                        0x8
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                             0x9
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                              0xa
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EN_MASK                                                             0x0001L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                      0x000EL
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                       0x0070L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_64BIT_MASK                                                          0x0080L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                          0x0100L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                               0x0200L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                                0x0400L
+//BIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//BIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_MSI_MSG_DATA
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA__MSI_DATA__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA__MSI_DATA_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_RC_SSID_CAP_LIST
+#define BIF_CFG_DEV0_RC_SSID_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_SSID_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC_SSID_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC_SSID_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC_SSID_CAP
+#define BIF_CFG_DEV0_RC_SSID_CAP__SUBSYSTEM_VENDOR_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_SSID_CAP__SUBSYSTEM_ID__SHIFT                                                         0x10
+#define BIF_CFG_DEV0_RC_SSID_CAP__SUBSYSTEM_VENDOR_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_SSID_CAP__SUBSYSTEM_ID_MASK                                                           0xFFFF0000L
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                             0x10
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                  0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                 0x14
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                     0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                    0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                               0xa
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                                 0x00000070L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                                 0x00000C00L
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                     0x18
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                                0x000000FFL
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                             0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                                 0x000EL
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                         0x0001L
+//BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                        0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                              0x1f
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                          0x000000FEL
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                      0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                          0x000E0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                    0x07000000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                                0x80000000L
+//BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                               0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                  0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                 0x0002L
+//BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                        0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                            0x003F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                              0x1f
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                          0x000000FEL
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                      0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                          0x000E0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                    0x07000000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                                0x80000000L
+//BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                               0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                  0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                 0x0002L
+//BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                      0x5
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                         0xc
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                          0xd
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                       0x10
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                         0x12
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                        0x13
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                  0x14
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                   0x15
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                  0x16
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                  0x17
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                         0x18
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                          0x19
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                     0x1a
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                           0x01000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                            0x02000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                       0x04000000L
+//BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                             0x4
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                          0x5
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                             0xc
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                              0xd
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                         0xe
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                       0xf
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                            0x11
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                             0x12
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                            0x13
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                      0x14
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                       0x15
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                      0x16
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                      0x17
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                             0x18
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                              0x19
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                         0x1a
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                               0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                               0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                                0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                           0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                             0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                              0x00020000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                              0x00080000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                        0x00800000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                               0x01000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                                0x02000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                           0x04000000L
+//BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                     0x4
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                  0x5
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                     0xc
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                      0xd
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                                 0xe
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                               0xf
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                    0x11
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                     0x12
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                    0x13
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                              0x14
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                               0x15
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                              0x16
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                              0x17
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                     0x18
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                      0x19
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x1a
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                                0x00100000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                                0x00400000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                                0x00800000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                       0x01000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                        0x02000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                   0x04000000L
+//BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                           0x6
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                          0x7
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                               0x8
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                              0xc
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                             0xd
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                      0xe
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                                0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                               0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                        0x00008000L
+//BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                               0x6
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                              0x7
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                   0x8
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                  0xc
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                                 0xd
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                          0xe
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                          0xf
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                                 0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                                 0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                                0x00000080L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                    0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                   0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                            0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                            0x00008000L
+//BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                            0x5
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                          0x7
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                      0x9
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                  0xb
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                          0xc
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                             0x0000001FL
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                              0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                    0x00000800L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                            0x00001000L
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG0__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG1__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG2__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG3__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN__SHIFT                                            0x2
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN_MASK                                              0x00000004L
+//BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD__SHIFT                                  0x2
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD__SHIFT                             0x3
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL__SHIFT                                0x4
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD__SHIFT                                  0x5
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD__SHIFT                                     0x6
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM__SHIFT                                      0x1b
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD_MASK                                    0x00000004L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD_MASK                               0x00000008L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL_MASK                                  0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS_MASK                                           0x00000180L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM_MASK                                        0xF8000000L
+//BIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID
+#define BIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID__SHIFT                                     0x10
+#define BIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                  0x1
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                       0x9
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                    0x00000002L
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                         0x0000FE00L
+//BIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                   0x0000FFFFL
+//BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                             0x1
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                             0x2
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                          0x3
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                              0x4
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                               0x5
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                              0x7
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                               0x0002L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                               0x0004L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                            0x0008L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                                0x0010L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                                 0x0020L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                              0x0040L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                                0x0080L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                         0xFF00L
+//BIF_CFG_DEV0_RC_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                         0x2
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                      0x3
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                           0x5
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                          0x7
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                   0x8
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                   0xa
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                                 0xc
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                              0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                           0x0002L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                           0x0004L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                        0x0008L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                            0x0010L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                             0x0020L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                            0x0080L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                     0x0300L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                     0x0C00L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                   0x1000L
+//BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                     0x1f
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                       0x007FFFFFL
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                       0x80000000L
+//BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                   0x007FFFFFL
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                           0x14
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                              0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                             0xFFF00000L
+//BIF_CFG_DEV0_RC_LINK_CAP_16GT
+#define BIF_CFG_DEV0_RC_LINK_CAP_16GT__RESERVED__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC_LINK_CAP_16GT__RESERVED_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LINK_CNTL_16GT__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL_16GT__RESERVED_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                             0x1
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                             0x2
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                             0x3
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                     0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                               0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                               0x00000004L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                               0x00000008L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                                 0x00000010L
+//BIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT             0x0
+#define BIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK               0x0000FFFFL
+//BIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT             0x0
+#define BIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK               0x0000FFFFL
+//BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                    0x0
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                      0x0001L
+//BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                                0x1
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                           0x0001L
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                  0x0002L
+//BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LINK_CAP_32GT
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                               0x8
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                               0xa
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                                0xb
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                               0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                            0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                                 0x00000200L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                                 0x00000400L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                  0x0000F800L
+//BIF_CFG_DEV0_RC_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                     0x8
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                    0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                                 0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                       0x00000700L
+//BIF_CFG_DEV0_RC_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                             0x1
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                             0x2
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                             0x3
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                         0x5
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                         0x6
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                     0x8
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                  0x9
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                        0xa
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                     0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                               0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                               0x00000004L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                               0x00000008L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                           0x00000020L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                          0x00000400L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf0_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF0_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF0_STATUS
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF0_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF0_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF0_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF0_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF0_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_HEADER
+#define BIF_CFG_DEV0_EPF0_VF0_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF0_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF0_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF0_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF0_BIST
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF0_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF0_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF0_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF0_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF0_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf1_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF1_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF1_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF1_STATUS
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF1_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF1_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF1_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF1_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF1_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF1_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF1_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF1_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF1_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF1_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_HEADER
+#define BIF_CFG_DEV0_EPF0_VF1_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF1_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF1_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF1_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF1_BIST
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF1_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF1_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF1_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF1_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF1_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF1_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf2_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF2_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF2_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF2_STATUS
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF2_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF2_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF2_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF2_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF2_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF2_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF2_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF2_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF2_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF2_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_HEADER
+#define BIF_CFG_DEV0_EPF0_VF2_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF2_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF2_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF2_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF2_BIST
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF2_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF2_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF2_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF2_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF2_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF2_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF2_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf3_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF3_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF3_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF3_STATUS
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF3_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF3_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF3_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF3_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF3_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF3_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF3_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF3_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF3_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF3_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_HEADER
+#define BIF_CFG_DEV0_EPF0_VF3_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF3_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF3_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF3_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF3_BIST
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF3_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF3_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF3_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF3_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF3_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF3_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF3_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf4_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF4_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF4_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF4_STATUS
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF4_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF4_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF4_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF4_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF4_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF4_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF4_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF4_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF4_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF4_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_HEADER
+#define BIF_CFG_DEV0_EPF0_VF4_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF4_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF4_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF4_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF4_BIST
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF4_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF4_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF4_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF4_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF4_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF4_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF4_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf5_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF5_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF5_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF5_STATUS
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF5_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF5_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF5_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF5_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF5_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF5_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF5_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF5_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF5_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF5_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_HEADER
+#define BIF_CFG_DEV0_EPF0_VF5_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF5_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF5_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF5_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF5_BIST
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF5_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF5_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF5_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF5_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF5_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF5_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF5_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf6_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF6_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF6_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF6_STATUS
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF6_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF6_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF6_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF6_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF6_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF6_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF6_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF6_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF6_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF6_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_HEADER
+#define BIF_CFG_DEV0_EPF0_VF6_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF6_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF6_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF6_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF6_BIST
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF6_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF6_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF6_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF6_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF6_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF6_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF6_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+
+
+// addressBlock: aid_nbio_nbif0_bif_cfg_dev0_epf0_vf7_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF7_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF7_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF7_STATUS
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF7_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF7_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF7_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF7_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF7_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF7_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF7_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF7_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF7_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF7_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_HEADER
+#define BIF_CFG_DEV0_EPF0_VF7_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF7_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF7_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF7_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF7_BIST
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF7_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF7_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF7_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF7_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF7_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF7_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF7_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf0_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF0_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                      0x10
+#define BIF_BX_DEV0_EPF0_VF0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                        0x00010000L
+//BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                   0x1
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                          0x3
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                0x10
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                             0x11
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                0x12
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                    0x13
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                            0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                  0x00010000L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                  0x00040000L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                      0x00080000L
+//BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK    0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT            0x0
+#define BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT          0x1
+#define BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT          0x8
+#define BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK            0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT      0x0
+#define BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK        0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                    0x0
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                    0x1
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                    0x2
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                    0x3
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                    0x4
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                    0x5
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                    0x6
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                    0x7
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                    0x8
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                    0x9
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                  0xa
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                  0xb
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                              0xc
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                              0xd
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                              0xe
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                              0xf
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                              0x10
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                              0x11
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                              0x12
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                              0x13
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                              0x14
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                              0x15
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                             0x16
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                             0x17
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                             0x18
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                             0x19
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                             0x1a
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                             0x1b
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                             0x1c
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                             0x1d
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                             0x1e
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                             0x1f
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                      0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                      0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                      0x00000010L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                      0x00000020L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                      0x00000040L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                      0x00000080L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                      0x00000100L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                      0x00000200L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                    0x00000400L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                    0x00000800L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                0x00001000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                0x00002000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                0x00004000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                0x00020000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                0x00080000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                0x00100000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                0x00200000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                               0x00400000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                               0x00800000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                               0x01000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                               0x02000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                               0x04000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                               0x08000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                               0x10000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                               0x20000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                               0x40000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                               0x80000000L
+//BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                   0x0
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                   0x1
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                   0x2
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                   0x3
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                   0x4
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                   0x5
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                   0x6
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                   0x7
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                   0x8
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                   0x9
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                 0xa
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                 0xb
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                             0xc
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                             0xd
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                             0xe
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                             0xf
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                             0x10
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                             0x11
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                             0x12
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                             0x13
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                             0x14
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                             0x15
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                            0x16
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                            0x17
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                            0x18
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                            0x19
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                            0x1a
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                            0x1b
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                            0x1c
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                            0x1d
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                            0x1e
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                            0x1f
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                     0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                     0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                     0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                     0x00000010L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                     0x00000020L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                     0x00000040L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                     0x00000080L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                     0x00000100L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                     0x00000200L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                   0x00000400L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                   0x00000800L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                               0x00001000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                               0x00002000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                               0x00004000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                               0x00008000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                               0x00010000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                               0x00040000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                               0x00080000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                               0x00100000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                               0x00200000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                              0x00400000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                              0x01000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                              0x02000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                              0x04000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                              0x08000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                              0x10000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                              0x20000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                              0x40000000L
+#define BIF_BX_DEV0_EPF0_VF0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                              0x80000000L
+//BIF_BX_DEV0_EPF0_VF0_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                  0x0
+#define BIF_BX_DEV0_EPF0_VF0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                  0x1
+#define BIF_BX_DEV0_EPF0_VF0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                    0x00000002L
+//BIF_BX_DEV0_EPF0_VF0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                        0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                            0x8
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                              0x9
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                              0x00000100L
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                0x00000200L
+//BIF_BX_DEV0_EPF0_VF0_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                0x00000002L
+//BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                            0x0
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                          0x1
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                               0x8
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                              0xf
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                               0x10
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                              0x17
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                0x18
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                0x19
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                 0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                 0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                0x00800000L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                  0x01000000L
+#define BIF_BX_DEV0_EPF0_VF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                  0x02000000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf0_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF0_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF0_MM_INDEX__MM_OFFSET__SHIFT                                                       0x0
+#define BIF_BX_DEV0_EPF0_VF0_MM_INDEX__MM_APER__SHIFT                                                         0x1f
+#define BIF_BX_DEV0_EPF0_VF0_MM_INDEX__MM_OFFSET_MASK                                                         0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF0_MM_INDEX__MM_APER_MASK                                                           0x80000000L
+//BIF_BX_DEV0_EPF0_VF0_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF0_MM_DATA__MM_DATA__SHIFT                                                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_MM_DATA__MM_DATA_MASK                                                            0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf0_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF0_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF0_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF0_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF0_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF0_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF0_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF0_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF0_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF0_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf0_BIFDEC2
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf1_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF1_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                      0x10
+#define BIF_BX_DEV0_EPF0_VF1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                        0x00010000L
+//BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                   0x1
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                          0x3
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                0x10
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                             0x11
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                0x12
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                    0x13
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                            0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                  0x00010000L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                  0x00040000L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                      0x00080000L
+//BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK    0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT            0x0
+#define BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT          0x1
+#define BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT          0x8
+#define BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK            0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT      0x0
+#define BIF_BX_DEV0_EPF0_VF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK        0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                    0x0
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                    0x1
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                    0x2
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                    0x3
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                    0x4
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                    0x5
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                    0x6
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                    0x7
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                    0x8
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                    0x9
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                  0xa
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                  0xb
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                              0xc
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                              0xd
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                              0xe
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                              0xf
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                              0x10
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                              0x11
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                              0x12
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                              0x13
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                              0x14
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                              0x15
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                             0x16
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                             0x17
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                             0x18
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                             0x19
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                             0x1a
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                             0x1b
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                             0x1c
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                             0x1d
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                             0x1e
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                             0x1f
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                      0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                      0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                      0x00000010L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                      0x00000020L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                      0x00000040L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                      0x00000080L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                      0x00000100L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                      0x00000200L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                    0x00000400L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                    0x00000800L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                0x00001000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                0x00002000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                0x00004000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                0x00020000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                0x00080000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                0x00100000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                0x00200000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                               0x00400000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                               0x00800000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                               0x01000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                               0x02000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                               0x04000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                               0x08000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                               0x10000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                               0x20000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                               0x40000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                               0x80000000L
+//BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                   0x0
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                   0x1
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                   0x2
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                   0x3
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                   0x4
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                   0x5
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                   0x6
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                   0x7
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                   0x8
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                   0x9
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                 0xa
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                 0xb
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                             0xc
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                             0xd
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                             0xe
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                             0xf
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                             0x10
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                             0x11
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                             0x12
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                             0x13
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                             0x14
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                             0x15
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                            0x16
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                            0x17
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                            0x18
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                            0x19
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                            0x1a
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                            0x1b
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                            0x1c
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                            0x1d
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                            0x1e
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                            0x1f
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                     0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                     0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                     0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                     0x00000010L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                     0x00000020L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                     0x00000040L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                     0x00000080L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                     0x00000100L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                     0x00000200L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                   0x00000400L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                   0x00000800L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                               0x00001000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                               0x00002000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                               0x00004000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                               0x00008000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                               0x00010000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                               0x00040000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                               0x00080000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                               0x00100000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                               0x00200000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                              0x00400000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                              0x01000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                              0x02000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                              0x04000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                              0x08000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                              0x10000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                              0x20000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                              0x40000000L
+#define BIF_BX_DEV0_EPF0_VF1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                              0x80000000L
+//BIF_BX_DEV0_EPF0_VF1_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                  0x0
+#define BIF_BX_DEV0_EPF0_VF1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                  0x1
+#define BIF_BX_DEV0_EPF0_VF1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                    0x00000002L
+//BIF_BX_DEV0_EPF0_VF1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                        0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                            0x8
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                              0x9
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                              0x00000100L
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                0x00000200L
+//BIF_BX_DEV0_EPF0_VF1_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                0x00000002L
+//BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                            0x0
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                          0x1
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                               0x8
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                              0xf
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                               0x10
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                              0x17
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                0x18
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                0x19
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                 0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                 0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                0x00800000L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                  0x01000000L
+#define BIF_BX_DEV0_EPF0_VF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                  0x02000000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf1_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF1_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF1_MM_INDEX__MM_OFFSET__SHIFT                                                       0x0
+#define BIF_BX_DEV0_EPF0_VF1_MM_INDEX__MM_APER__SHIFT                                                         0x1f
+#define BIF_BX_DEV0_EPF0_VF1_MM_INDEX__MM_OFFSET_MASK                                                         0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF1_MM_INDEX__MM_APER_MASK                                                           0x80000000L
+//BIF_BX_DEV0_EPF0_VF1_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF1_MM_DATA__MM_DATA__SHIFT                                                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_MM_DATA__MM_DATA_MASK                                                            0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf1_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF1_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF1_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF1_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF1_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF1_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF1_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF1_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF1_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF1_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF1_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF1_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF1_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf1_BIFDEC2
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf2_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF2_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF2_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF2_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                      0x10
+#define BIF_BX_DEV0_EPF0_VF2_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                        0x00010000L
+//BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                   0x1
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                          0x3
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                0x10
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                             0x11
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                0x12
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                    0x13
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                            0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                  0x00010000L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                  0x00040000L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                      0x00080000L
+//BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK    0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT            0x0
+#define BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT          0x1
+#define BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT          0x8
+#define BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK            0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF2_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT      0x0
+#define BIF_BX_DEV0_EPF0_VF2_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK        0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                    0x0
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                    0x1
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                    0x2
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                    0x3
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                    0x4
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                    0x5
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                    0x6
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                    0x7
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                    0x8
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                    0x9
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                  0xa
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                  0xb
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                              0xc
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                              0xd
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                              0xe
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                              0xf
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                              0x10
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                              0x11
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                              0x12
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                              0x13
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                              0x14
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                              0x15
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                             0x16
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                             0x17
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                             0x18
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                             0x19
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                             0x1a
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                             0x1b
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                             0x1c
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                             0x1d
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                             0x1e
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                             0x1f
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP0_MASK                                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP1_MASK                                                      0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP2_MASK                                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP3_MASK                                                      0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP4_MASK                                                      0x00000010L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP5_MASK                                                      0x00000020L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP6_MASK                                                      0x00000040L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP7_MASK                                                      0x00000080L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP8_MASK                                                      0x00000100L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__CP9_MASK                                                      0x00000200L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                    0x00000400L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                    0x00000800L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                0x00001000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                0x00002000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                0x00004000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                0x00020000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                0x00080000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                0x00100000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                0x00200000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                               0x00400000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                               0x00800000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                               0x01000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                               0x02000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                               0x04000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                               0x08000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                               0x10000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                               0x20000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                               0x40000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                               0x80000000L
+//BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                   0x0
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                   0x1
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                   0x2
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                   0x3
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                   0x4
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                   0x5
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                   0x6
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                   0x7
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                   0x8
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                   0x9
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                 0xa
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                 0xb
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                             0xc
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                             0xd
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                             0xe
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                             0xf
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                             0x10
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                             0x11
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                             0x12
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                             0x13
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                             0x14
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                             0x15
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                            0x16
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                            0x17
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                            0x18
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                            0x19
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                            0x1a
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                            0x1b
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                            0x1c
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                            0x1d
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                            0x1e
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                            0x1f
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP0_MASK                                                     0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP1_MASK                                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP2_MASK                                                     0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP3_MASK                                                     0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP4_MASK                                                     0x00000010L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP5_MASK                                                     0x00000020L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP6_MASK                                                     0x00000040L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP7_MASK                                                     0x00000080L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP8_MASK                                                     0x00000100L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__CP9_MASK                                                     0x00000200L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                   0x00000400L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                   0x00000800L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                               0x00001000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                               0x00002000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                               0x00004000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                               0x00008000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                               0x00010000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                               0x00040000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                               0x00080000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                               0x00100000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                               0x00200000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                              0x00400000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                              0x01000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                              0x02000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                              0x04000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                              0x08000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                              0x10000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                              0x20000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                              0x40000000L
+#define BIF_BX_DEV0_EPF0_VF2_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                              0x80000000L
+//BIF_BX_DEV0_EPF0_VF2_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF2_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                  0x0
+#define BIF_BX_DEV0_EPF0_VF2_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                  0x1
+#define BIF_BX_DEV0_EPF0_VF2_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                    0x00000002L
+//BIF_BX_DEV0_EPF0_VF2_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF2_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF2_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                        0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                            0x8
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                              0x9
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                              0x00000100L
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                0x00000200L
+//BIF_BX_DEV0_EPF0_VF2_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                0x00000002L
+//BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                            0x0
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                          0x1
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                               0x8
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                              0xf
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                               0x10
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                              0x17
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                0x18
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                0x19
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                 0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                 0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                0x00800000L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                  0x01000000L
+#define BIF_BX_DEV0_EPF0_VF2_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                  0x02000000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf2_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF2_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF2_MM_INDEX__MM_OFFSET__SHIFT                                                       0x0
+#define BIF_BX_DEV0_EPF0_VF2_MM_INDEX__MM_APER__SHIFT                                                         0x1f
+#define BIF_BX_DEV0_EPF0_VF2_MM_INDEX__MM_OFFSET_MASK                                                         0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF2_MM_INDEX__MM_APER_MASK                                                           0x80000000L
+//BIF_BX_DEV0_EPF0_VF2_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF2_MM_DATA__MM_DATA__SHIFT                                                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_MM_DATA__MM_DATA_MASK                                                            0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF2_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF2_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf2_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF2_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF2_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF2_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF2_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF2_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF2_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF2_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF2_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF2_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF2_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF2_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF2_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf2_BIFDEC2
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf3_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF3_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF3_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF3_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                      0x10
+#define BIF_BX_DEV0_EPF0_VF3_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                        0x00010000L
+//BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                   0x1
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                          0x3
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                0x10
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                             0x11
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                0x12
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                    0x13
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                            0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                  0x00010000L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                  0x00040000L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                      0x00080000L
+//BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK    0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT            0x0
+#define BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT          0x1
+#define BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT          0x8
+#define BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK            0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF3_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT      0x0
+#define BIF_BX_DEV0_EPF0_VF3_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK        0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                    0x0
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                    0x1
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                    0x2
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                    0x3
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                    0x4
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                    0x5
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                    0x6
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                    0x7
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                    0x8
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                    0x9
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                  0xa
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                  0xb
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                              0xc
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                              0xd
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                              0xe
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                              0xf
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                              0x10
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                              0x11
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                              0x12
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                              0x13
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                              0x14
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                              0x15
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                             0x16
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                             0x17
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                             0x18
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                             0x19
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                             0x1a
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                             0x1b
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                             0x1c
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                             0x1d
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                             0x1e
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                             0x1f
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP0_MASK                                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP1_MASK                                                      0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP2_MASK                                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP3_MASK                                                      0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP4_MASK                                                      0x00000010L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP5_MASK                                                      0x00000020L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP6_MASK                                                      0x00000040L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP7_MASK                                                      0x00000080L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP8_MASK                                                      0x00000100L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__CP9_MASK                                                      0x00000200L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                    0x00000400L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                    0x00000800L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                0x00001000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                0x00002000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                0x00004000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                0x00020000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                0x00080000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                0x00100000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                0x00200000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                               0x00400000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                               0x00800000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                               0x01000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                               0x02000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                               0x04000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                               0x08000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                               0x10000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                               0x20000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                               0x40000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                               0x80000000L
+//BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                   0x0
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                   0x1
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                   0x2
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                   0x3
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                   0x4
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                   0x5
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                   0x6
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                   0x7
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                   0x8
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                   0x9
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                 0xa
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                 0xb
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                             0xc
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                             0xd
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                             0xe
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                             0xf
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                             0x10
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                             0x11
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                             0x12
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                             0x13
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                             0x14
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                             0x15
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                            0x16
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                            0x17
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                            0x18
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                            0x19
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                            0x1a
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                            0x1b
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                            0x1c
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                            0x1d
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                            0x1e
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                            0x1f
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP0_MASK                                                     0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP1_MASK                                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP2_MASK                                                     0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP3_MASK                                                     0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP4_MASK                                                     0x00000010L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP5_MASK                                                     0x00000020L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP6_MASK                                                     0x00000040L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP7_MASK                                                     0x00000080L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP8_MASK                                                     0x00000100L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__CP9_MASK                                                     0x00000200L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                   0x00000400L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                   0x00000800L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                               0x00001000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                               0x00002000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                               0x00004000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                               0x00008000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                               0x00010000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                               0x00040000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                               0x00080000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                               0x00100000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                               0x00200000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                              0x00400000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                              0x01000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                              0x02000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                              0x04000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                              0x08000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                              0x10000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                              0x20000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                              0x40000000L
+#define BIF_BX_DEV0_EPF0_VF3_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                              0x80000000L
+//BIF_BX_DEV0_EPF0_VF3_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF3_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                  0x0
+#define BIF_BX_DEV0_EPF0_VF3_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                  0x1
+#define BIF_BX_DEV0_EPF0_VF3_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                    0x00000002L
+//BIF_BX_DEV0_EPF0_VF3_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF3_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF3_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                        0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                            0x8
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                              0x9
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                              0x00000100L
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                0x00000200L
+//BIF_BX_DEV0_EPF0_VF3_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                0x00000002L
+//BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                            0x0
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                          0x1
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                               0x8
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                              0xf
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                               0x10
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                              0x17
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                0x18
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                0x19
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                 0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                 0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                0x00800000L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                  0x01000000L
+#define BIF_BX_DEV0_EPF0_VF3_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                  0x02000000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf3_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF3_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF3_MM_INDEX__MM_OFFSET__SHIFT                                                       0x0
+#define BIF_BX_DEV0_EPF0_VF3_MM_INDEX__MM_APER__SHIFT                                                         0x1f
+#define BIF_BX_DEV0_EPF0_VF3_MM_INDEX__MM_OFFSET_MASK                                                         0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF3_MM_INDEX__MM_APER_MASK                                                           0x80000000L
+//BIF_BX_DEV0_EPF0_VF3_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF3_MM_DATA__MM_DATA__SHIFT                                                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_MM_DATA__MM_DATA_MASK                                                            0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF3_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF3_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf3_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF3_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF3_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF3_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF3_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF3_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF3_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF3_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF3_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF3_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF3_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF3_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF3_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf3_BIFDEC2
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf4_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF4_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF4_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF4_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                      0x10
+#define BIF_BX_DEV0_EPF0_VF4_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                        0x00010000L
+//BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                   0x1
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                          0x3
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                0x10
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                             0x11
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                0x12
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                    0x13
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                            0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                  0x00010000L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                  0x00040000L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                      0x00080000L
+//BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK    0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT            0x0
+#define BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT          0x1
+#define BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT          0x8
+#define BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK            0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF4_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT      0x0
+#define BIF_BX_DEV0_EPF0_VF4_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK        0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                    0x0
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                    0x1
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                    0x2
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                    0x3
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                    0x4
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                    0x5
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                    0x6
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                    0x7
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                    0x8
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                    0x9
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                  0xa
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                  0xb
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                              0xc
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                              0xd
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                              0xe
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                              0xf
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                              0x10
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                              0x11
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                              0x12
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                              0x13
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                              0x14
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                              0x15
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                             0x16
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                             0x17
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                             0x18
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                             0x19
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                             0x1a
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                             0x1b
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                             0x1c
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                             0x1d
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                             0x1e
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                             0x1f
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP0_MASK                                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP1_MASK                                                      0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP2_MASK                                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP3_MASK                                                      0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP4_MASK                                                      0x00000010L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP5_MASK                                                      0x00000020L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP6_MASK                                                      0x00000040L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP7_MASK                                                      0x00000080L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP8_MASK                                                      0x00000100L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__CP9_MASK                                                      0x00000200L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                    0x00000400L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                    0x00000800L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                0x00001000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                0x00002000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                0x00004000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                0x00020000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                0x00080000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                0x00100000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                0x00200000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                               0x00400000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                               0x00800000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                               0x01000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                               0x02000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                               0x04000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                               0x08000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                               0x10000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                               0x20000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                               0x40000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                               0x80000000L
+//BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                   0x0
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                   0x1
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                   0x2
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                   0x3
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                   0x4
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                   0x5
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                   0x6
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                   0x7
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                   0x8
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                   0x9
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                 0xa
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                 0xb
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                             0xc
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                             0xd
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                             0xe
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                             0xf
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                             0x10
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                             0x11
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                             0x12
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                             0x13
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                             0x14
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                             0x15
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                            0x16
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                            0x17
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                            0x18
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                            0x19
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                            0x1a
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                            0x1b
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                            0x1c
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                            0x1d
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                            0x1e
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                            0x1f
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP0_MASK                                                     0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP1_MASK                                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP2_MASK                                                     0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP3_MASK                                                     0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP4_MASK                                                     0x00000010L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP5_MASK                                                     0x00000020L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP6_MASK                                                     0x00000040L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP7_MASK                                                     0x00000080L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP8_MASK                                                     0x00000100L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__CP9_MASK                                                     0x00000200L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                   0x00000400L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                   0x00000800L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                               0x00001000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                               0x00002000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                               0x00004000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                               0x00008000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                               0x00010000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                               0x00040000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                               0x00080000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                               0x00100000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                               0x00200000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                              0x00400000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                              0x01000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                              0x02000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                              0x04000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                              0x08000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                              0x10000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                              0x20000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                              0x40000000L
+#define BIF_BX_DEV0_EPF0_VF4_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                              0x80000000L
+//BIF_BX_DEV0_EPF0_VF4_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF4_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                  0x0
+#define BIF_BX_DEV0_EPF0_VF4_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                  0x1
+#define BIF_BX_DEV0_EPF0_VF4_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                    0x00000002L
+//BIF_BX_DEV0_EPF0_VF4_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF4_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF4_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                        0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                            0x8
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                              0x9
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                              0x00000100L
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                0x00000200L
+//BIF_BX_DEV0_EPF0_VF4_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                0x00000002L
+//BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                            0x0
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                          0x1
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                               0x8
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                              0xf
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                               0x10
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                              0x17
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                0x18
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                0x19
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                 0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                 0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                0x00800000L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                  0x01000000L
+#define BIF_BX_DEV0_EPF0_VF4_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                  0x02000000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf4_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF4_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF4_MM_INDEX__MM_OFFSET__SHIFT                                                       0x0
+#define BIF_BX_DEV0_EPF0_VF4_MM_INDEX__MM_APER__SHIFT                                                         0x1f
+#define BIF_BX_DEV0_EPF0_VF4_MM_INDEX__MM_OFFSET_MASK                                                         0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF4_MM_INDEX__MM_APER_MASK                                                           0x80000000L
+//BIF_BX_DEV0_EPF0_VF4_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF4_MM_DATA__MM_DATA__SHIFT                                                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_MM_DATA__MM_DATA_MASK                                                            0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF4_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF4_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf4_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF4_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF4_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF4_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF4_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF4_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF4_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF4_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF4_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF4_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF4_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF4_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF4_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf4_BIFDEC2
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf5_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF5_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF5_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF5_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                      0x10
+#define BIF_BX_DEV0_EPF0_VF5_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                        0x00010000L
+//BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                   0x1
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                          0x3
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                0x10
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                             0x11
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                0x12
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                    0x13
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                            0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                  0x00010000L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                  0x00040000L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                      0x00080000L
+//BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK    0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT            0x0
+#define BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT          0x1
+#define BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT          0x8
+#define BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK            0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF5_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT      0x0
+#define BIF_BX_DEV0_EPF0_VF5_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK        0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                    0x0
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                    0x1
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                    0x2
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                    0x3
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                    0x4
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                    0x5
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                    0x6
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                    0x7
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                    0x8
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                    0x9
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                  0xa
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                  0xb
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                              0xc
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                              0xd
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                              0xe
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                              0xf
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                              0x10
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                              0x11
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                              0x12
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                              0x13
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                              0x14
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                              0x15
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                             0x16
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                             0x17
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                             0x18
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                             0x19
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                             0x1a
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                             0x1b
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                             0x1c
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                             0x1d
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                             0x1e
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                             0x1f
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP0_MASK                                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP1_MASK                                                      0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP2_MASK                                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP3_MASK                                                      0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP4_MASK                                                      0x00000010L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP5_MASK                                                      0x00000020L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP6_MASK                                                      0x00000040L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP7_MASK                                                      0x00000080L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP8_MASK                                                      0x00000100L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__CP9_MASK                                                      0x00000200L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                    0x00000400L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                    0x00000800L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                0x00001000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                0x00002000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                0x00004000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                0x00020000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                0x00080000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                0x00100000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                0x00200000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                               0x00400000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                               0x00800000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                               0x01000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                               0x02000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                               0x04000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                               0x08000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                               0x10000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                               0x20000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                               0x40000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                               0x80000000L
+//BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                   0x0
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                   0x1
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                   0x2
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                   0x3
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                   0x4
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                   0x5
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                   0x6
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                   0x7
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                   0x8
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                   0x9
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                 0xa
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                 0xb
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                             0xc
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                             0xd
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                             0xe
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                             0xf
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                             0x10
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                             0x11
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                             0x12
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                             0x13
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                             0x14
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                             0x15
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                            0x16
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                            0x17
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                            0x18
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                            0x19
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                            0x1a
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                            0x1b
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                            0x1c
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                            0x1d
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                            0x1e
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                            0x1f
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP0_MASK                                                     0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP1_MASK                                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP2_MASK                                                     0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP3_MASK                                                     0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP4_MASK                                                     0x00000010L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP5_MASK                                                     0x00000020L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP6_MASK                                                     0x00000040L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP7_MASK                                                     0x00000080L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP8_MASK                                                     0x00000100L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__CP9_MASK                                                     0x00000200L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                   0x00000400L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                   0x00000800L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                               0x00001000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                               0x00002000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                               0x00004000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                               0x00008000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                               0x00010000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                               0x00040000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                               0x00080000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                               0x00100000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                               0x00200000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                              0x00400000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                              0x01000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                              0x02000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                              0x04000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                              0x08000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                              0x10000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                              0x20000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                              0x40000000L
+#define BIF_BX_DEV0_EPF0_VF5_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                              0x80000000L
+//BIF_BX_DEV0_EPF0_VF5_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF5_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                  0x0
+#define BIF_BX_DEV0_EPF0_VF5_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                  0x1
+#define BIF_BX_DEV0_EPF0_VF5_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                    0x00000002L
+//BIF_BX_DEV0_EPF0_VF5_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF5_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF5_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                        0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                            0x8
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                              0x9
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                              0x00000100L
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                0x00000200L
+//BIF_BX_DEV0_EPF0_VF5_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                0x00000002L
+//BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                            0x0
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                          0x1
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                               0x8
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                              0xf
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                               0x10
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                              0x17
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                0x18
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                0x19
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                 0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                 0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                0x00800000L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                  0x01000000L
+#define BIF_BX_DEV0_EPF0_VF5_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                  0x02000000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf5_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF5_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF5_MM_INDEX__MM_OFFSET__SHIFT                                                       0x0
+#define BIF_BX_DEV0_EPF0_VF5_MM_INDEX__MM_APER__SHIFT                                                         0x1f
+#define BIF_BX_DEV0_EPF0_VF5_MM_INDEX__MM_OFFSET_MASK                                                         0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF5_MM_INDEX__MM_APER_MASK                                                           0x80000000L
+//BIF_BX_DEV0_EPF0_VF5_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF5_MM_DATA__MM_DATA__SHIFT                                                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_MM_DATA__MM_DATA_MASK                                                            0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF5_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF5_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf5_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF5_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF5_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF5_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF5_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF5_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF5_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF5_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF5_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF5_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF5_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF5_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF5_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf5_BIFDEC2
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf6_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF6_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF6_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF6_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                      0x10
+#define BIF_BX_DEV0_EPF0_VF6_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                        0x00010000L
+//BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                   0x1
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                          0x3
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                0x10
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                             0x11
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                0x12
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                    0x13
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                            0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                  0x00010000L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                  0x00040000L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                      0x00080000L
+//BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK    0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT            0x0
+#define BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT          0x1
+#define BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT          0x8
+#define BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK            0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF6_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT      0x0
+#define BIF_BX_DEV0_EPF0_VF6_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK        0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                    0x0
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                    0x1
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                    0x2
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                    0x3
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                    0x4
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                    0x5
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                    0x6
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                    0x7
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                    0x8
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                    0x9
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                  0xa
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                  0xb
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                              0xc
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                              0xd
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                              0xe
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                              0xf
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                              0x10
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                              0x11
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                              0x12
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                              0x13
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                              0x14
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                              0x15
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                             0x16
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                             0x17
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                             0x18
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                             0x19
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                             0x1a
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                             0x1b
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                             0x1c
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                             0x1d
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                             0x1e
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                             0x1f
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP0_MASK                                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP1_MASK                                                      0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP2_MASK                                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP3_MASK                                                      0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP4_MASK                                                      0x00000010L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP5_MASK                                                      0x00000020L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP6_MASK                                                      0x00000040L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP7_MASK                                                      0x00000080L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP8_MASK                                                      0x00000100L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__CP9_MASK                                                      0x00000200L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                    0x00000400L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                    0x00000800L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                0x00001000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                0x00002000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                0x00004000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                0x00020000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                0x00080000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                0x00100000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                0x00200000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                               0x00400000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                               0x00800000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                               0x01000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                               0x02000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                               0x04000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                               0x08000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                               0x10000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                               0x20000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                               0x40000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                               0x80000000L
+//BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                   0x0
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                   0x1
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                   0x2
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                   0x3
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                   0x4
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                   0x5
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                   0x6
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                   0x7
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                   0x8
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                   0x9
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                 0xa
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                 0xb
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                             0xc
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                             0xd
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                             0xe
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                             0xf
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                             0x10
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                             0x11
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                             0x12
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                             0x13
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                             0x14
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                             0x15
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                            0x16
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                            0x17
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                            0x18
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                            0x19
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                            0x1a
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                            0x1b
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                            0x1c
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                            0x1d
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                            0x1e
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                            0x1f
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP0_MASK                                                     0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP1_MASK                                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP2_MASK                                                     0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP3_MASK                                                     0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP4_MASK                                                     0x00000010L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP5_MASK                                                     0x00000020L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP6_MASK                                                     0x00000040L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP7_MASK                                                     0x00000080L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP8_MASK                                                     0x00000100L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__CP9_MASK                                                     0x00000200L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                   0x00000400L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                   0x00000800L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                               0x00001000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                               0x00002000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                               0x00004000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                               0x00008000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                               0x00010000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                               0x00040000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                               0x00080000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                               0x00100000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                               0x00200000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                              0x00400000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                              0x01000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                              0x02000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                              0x04000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                              0x08000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                              0x10000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                              0x20000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                              0x40000000L
+#define BIF_BX_DEV0_EPF0_VF6_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                              0x80000000L
+//BIF_BX_DEV0_EPF0_VF6_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF6_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                  0x0
+#define BIF_BX_DEV0_EPF0_VF6_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                  0x1
+#define BIF_BX_DEV0_EPF0_VF6_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                    0x00000002L
+//BIF_BX_DEV0_EPF0_VF6_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF6_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF6_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                        0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                            0x8
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                              0x9
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                              0x00000100L
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                0x00000200L
+//BIF_BX_DEV0_EPF0_VF6_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                0x00000002L
+//BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                            0x0
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                          0x1
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                               0x8
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                              0xf
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                               0x10
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                              0x17
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                0x18
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                0x19
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                 0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                 0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                0x00800000L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                  0x01000000L
+#define BIF_BX_DEV0_EPF0_VF6_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                  0x02000000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf6_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF6_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF6_MM_INDEX__MM_OFFSET__SHIFT                                                       0x0
+#define BIF_BX_DEV0_EPF0_VF6_MM_INDEX__MM_APER__SHIFT                                                         0x1f
+#define BIF_BX_DEV0_EPF0_VF6_MM_INDEX__MM_OFFSET_MASK                                                         0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF6_MM_INDEX__MM_APER_MASK                                                           0x80000000L
+//BIF_BX_DEV0_EPF0_VF6_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF6_MM_DATA__MM_DATA__SHIFT                                                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_MM_DATA__MM_DATA_MASK                                                            0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF6_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF6_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf6_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF6_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF6_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF6_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF6_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF6_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF6_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF6_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF6_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF6_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF6_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF6_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF6_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf6_BIFDEC2
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf7_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF7_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF7_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF7_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                      0x10
+#define BIF_BX_DEV0_EPF0_VF7_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                        0x00010000L
+//BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                   0x1
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                          0x3
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                0x10
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                             0x11
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                0x12
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                    0x13
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                            0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                  0x00010000L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                  0x00040000L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                      0x00080000L
+//BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK    0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT            0x0
+#define BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT          0x1
+#define BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT          0x8
+#define BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK            0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF7_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                            0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT      0x0
+#define BIF_BX_DEV0_EPF0_VF7_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK        0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                    0x0
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                    0x1
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                    0x2
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                    0x3
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                    0x4
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                    0x5
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                    0x6
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                    0x7
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                    0x8
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                    0x9
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                  0xa
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                  0xb
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                              0xc
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                              0xd
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                              0xe
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                              0xf
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                              0x10
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                              0x11
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                              0x12
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                              0x13
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                              0x14
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                              0x15
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                             0x16
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                             0x17
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                             0x18
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                             0x19
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                             0x1a
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                             0x1b
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                             0x1c
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                             0x1d
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                             0x1e
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                             0x1f
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP0_MASK                                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP1_MASK                                                      0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP2_MASK                                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP3_MASK                                                      0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP4_MASK                                                      0x00000010L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP5_MASK                                                      0x00000020L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP6_MASK                                                      0x00000040L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP7_MASK                                                      0x00000080L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP8_MASK                                                      0x00000100L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__CP9_MASK                                                      0x00000200L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                    0x00000400L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                    0x00000800L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                0x00001000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                0x00002000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                0x00004000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                0x00020000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                0x00080000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                0x00100000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                0x00200000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                               0x00400000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                               0x00800000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                               0x01000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                               0x02000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                               0x04000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                               0x08000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                               0x10000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                               0x20000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                               0x40000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                               0x80000000L
+//BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                   0x0
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                   0x1
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                   0x2
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                   0x3
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                   0x4
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                   0x5
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                   0x6
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                   0x7
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                   0x8
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                   0x9
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                 0xa
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                 0xb
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                             0xc
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                             0xd
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                             0xe
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                             0xf
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                             0x10
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                             0x11
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                             0x12
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                             0x13
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                             0x14
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                             0x15
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                            0x16
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                            0x17
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                            0x18
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                            0x19
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                            0x1a
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                            0x1b
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                            0x1c
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                            0x1d
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                            0x1e
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                            0x1f
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP0_MASK                                                     0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP1_MASK                                                     0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP2_MASK                                                     0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP3_MASK                                                     0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP4_MASK                                                     0x00000010L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP5_MASK                                                     0x00000020L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP6_MASK                                                     0x00000040L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP7_MASK                                                     0x00000080L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP8_MASK                                                     0x00000100L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__CP9_MASK                                                     0x00000200L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                   0x00000400L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                   0x00000800L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                               0x00001000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                               0x00002000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                               0x00004000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                               0x00008000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                               0x00010000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                               0x00020000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                               0x00040000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                               0x00080000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                               0x00100000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                               0x00200000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                              0x00400000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                              0x01000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                              0x02000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                              0x04000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                              0x08000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                              0x10000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                              0x20000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                              0x40000000L
+#define BIF_BX_DEV0_EPF0_VF7_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                              0x80000000L
+//BIF_BX_DEV0_EPF0_VF7_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF7_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                  0x0
+#define BIF_BX_DEV0_EPF0_VF7_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                  0x1
+#define BIF_BX_DEV0_EPF0_VF7_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                    0x00000002L
+//BIF_BX_DEV0_EPF0_VF7_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF7_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF7_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                        0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                       0x0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                         0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                            0x8
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                              0x9
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                              0x00000100L
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                0x00000200L
+//BIF_BX_DEV0_EPF0_VF7_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                            0x0
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                              0x1
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                0x00000002L
+//BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                            0x0
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                          0x1
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                               0x8
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                              0xf
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                               0x10
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                              0x17
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                0x18
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                0x19
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                              0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                            0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                 0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                0x00008000L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                 0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                0x00800000L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                  0x01000000L
+#define BIF_BX_DEV0_EPF0_VF7_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                  0x02000000L
+
+
+// addressBlock: aid_nbio_nbif0_bif_bx_dev0_epf0_vf7_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF7_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF7_MM_INDEX__MM_OFFSET__SHIFT                                                       0x0
+#define BIF_BX_DEV0_EPF0_VF7_MM_INDEX__MM_APER__SHIFT                                                         0x1f
+#define BIF_BX_DEV0_EPF0_VF7_MM_INDEX__MM_OFFSET_MASK                                                         0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF7_MM_INDEX__MM_APER_MASK                                                           0x80000000L
+//BIF_BX_DEV0_EPF0_VF7_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF7_MM_DATA__MM_DATA__SHIFT                                                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_MM_DATA__MM_DATA_MASK                                                            0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF7_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF7_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf7_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF7_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF7_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF7_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF7_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF7_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF7_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF7_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF7_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF7_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF7_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF7_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF7_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: aid_nbio_nbif0_rcc_dev0_epf0_vf7_BIFDEC2
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_offset.h
index bd129266ebfd..a84a7cfaf71e 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_offset.h
@@ -135,6 +135,8 @@
 #define mmIH_RB_WPTR_ADDR_LO_BASE_IDX                                                                  0
 #define mmIH_DOORBELL_RPTR                                                                             0x0087
 #define mmIH_DOORBELL_RPTR_BASE_IDX                                                                    0
+#define mmIH_DOORBELL_RETRY_CAM                                                                        0x0088
+#define mmIH_DOORBELL_RETRY_CAM_BASE_IDX                                                               0
 #define mmIH_RB_CNTL_RING1                                                                             0x008c
 #define mmIH_RB_CNTL_RING1_BASE_IDX                                                                    0
 #define mmIH_RB_BASE_RING1                                                                             0x008d
@@ -159,6 +161,8 @@
 #define mmIH_RB_WPTR_RING2_BASE_IDX                                                                    0
 #define mmIH_DOORBELL_RPTR_RING2                                                                       0x009f
 #define mmIH_DOORBELL_RPTR_RING2_BASE_IDX                                                              0
+#define mmIH_RETRY_CAM_ACK                                                                             0x00a4
+#define mmIH_RETRY_CAM_ACK_BASE_IDX                                                                    0
 #define mmIH_VERSION                                                                                   0x00a5
 #define mmIH_VERSION_BASE_IDX                                                                          0
 #define mmIH_CNTL                                                                                      0x00c0
@@ -235,6 +239,8 @@
 #define mmIH_MMHUB_ERROR_BASE_IDX                                                                      0
 #define mmIH_MEM_POWER_CTRL                                                                            0x00e8
 #define mmIH_MEM_POWER_CTRL_BASE_IDX                                                                   0
+#define mmIH_RETRY_INT_CAM_CNTL                                                                        0x00e9
+#define mmIH_RETRY_INT_CAM_CNTL_BASE_IDX                                                               0
 #define mmIH_REGISTER_LAST_PART2                                                                       0x00ff
 #define mmIH_REGISTER_LAST_PART2_BASE_IDX                                                              0
 #define mmSEM_CLK_CTRL                                                                                 0x0100
diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_sh_mask.h
index 3ea83ea9ce3a..75c04fc275a0 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_sh_mask.h
@@ -349,6 +349,17 @@
 #define IH_DOORBELL_RPTR_RING2__ENABLE__SHIFT                                                                 0x1c
 #define IH_DOORBELL_RPTR_RING2__OFFSET_MASK                                                                   0x03FFFFFFL
 #define IH_DOORBELL_RPTR_RING2__ENABLE_MASK                                                                   0x10000000L
+//IH_RETRY_INT_CAM_CNTL
+#define IH_RETRY_INT_CAM_CNTL__CAM_SIZE__SHIFT                                                                0x0
+#define IH_RETRY_INT_CAM_CNTL__BACK_PRESSURE_SKID_VALUE__SHIFT                                                0x8
+#define IH_RETRY_INT_CAM_CNTL__ENABLE__SHIFT                                                                  0x10
+#define IH_RETRY_INT_CAM_CNTL__BACK_PRESSURE_ENABLE__SHIFT                                                    0x11
+#define IH_RETRY_INT_CAM_CNTL__PER_VF_ENTRY_SIZE__SHIFT                                                       0x14
+#define IH_RETRY_INT_CAM_CNTL__CAM_SIZE_MASK                                                                  0x0000001FL
+#define IH_RETRY_INT_CAM_CNTL__BACK_PRESSURE_SKID_VALUE_MASK                                                  0x00003F00L
+#define IH_RETRY_INT_CAM_CNTL__ENABLE_MASK                                                                    0x00010000L
+#define IH_RETRY_INT_CAM_CNTL__BACK_PRESSURE_ENABLE_MASK                                                      0x00020000L
+#define IH_RETRY_INT_CAM_CNTL__PER_VF_ENTRY_SIZE_MASK                                                         0x00300000L
 //IH_VERSION
 #define IH_VERSION__MINVER__SHIFT                                                                             0x0
 #define IH_VERSION__MAJVER__SHIFT                                                                             0x8
diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_4_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_4_2_offset.h
new file mode 100644
index 000000000000..bd30bd818a58
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_4_2_offset.h
@@ -0,0 +1,263 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _osssys_4_4_2_OFFSET_HEADER
+#define _osssys_4_4_2_OFFSET_HEADER
+
+
+
+// addressBlock: aid_osssys_osssysdec
+// base address: 0x4280
+#define regIH_VMID_0_LUT                                                                                0x0000
+#define regIH_VMID_0_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_1_LUT                                                                                0x0001
+#define regIH_VMID_1_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_2_LUT                                                                                0x0002
+#define regIH_VMID_2_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_3_LUT                                                                                0x0003
+#define regIH_VMID_3_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_4_LUT                                                                                0x0004
+#define regIH_VMID_4_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_5_LUT                                                                                0x0005
+#define regIH_VMID_5_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_6_LUT                                                                                0x0006
+#define regIH_VMID_6_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_7_LUT                                                                                0x0007
+#define regIH_VMID_7_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_8_LUT                                                                                0x0008
+#define regIH_VMID_8_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_9_LUT                                                                                0x0009
+#define regIH_VMID_9_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_10_LUT                                                                               0x000a
+#define regIH_VMID_10_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_11_LUT                                                                               0x000b
+#define regIH_VMID_11_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_12_LUT                                                                               0x000c
+#define regIH_VMID_12_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_13_LUT                                                                               0x000d
+#define regIH_VMID_13_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_14_LUT                                                                               0x000e
+#define regIH_VMID_14_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_15_LUT                                                                               0x000f
+#define regIH_VMID_15_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_0_LUT_MM                                                                             0x0010
+#define regIH_VMID_0_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_1_LUT_MM                                                                             0x0011
+#define regIH_VMID_1_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_2_LUT_MM                                                                             0x0012
+#define regIH_VMID_2_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_3_LUT_MM                                                                             0x0013
+#define regIH_VMID_3_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_4_LUT_MM                                                                             0x0014
+#define regIH_VMID_4_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_5_LUT_MM                                                                             0x0015
+#define regIH_VMID_5_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_6_LUT_MM                                                                             0x0016
+#define regIH_VMID_6_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_7_LUT_MM                                                                             0x0017
+#define regIH_VMID_7_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_8_LUT_MM                                                                             0x0018
+#define regIH_VMID_8_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_9_LUT_MM                                                                             0x0019
+#define regIH_VMID_9_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_10_LUT_MM                                                                            0x001a
+#define regIH_VMID_10_LUT_MM_BASE_IDX                                                                   0
+#define regIH_VMID_11_LUT_MM                                                                            0x001b
+#define regIH_VMID_11_LUT_MM_BASE_IDX                                                                   0
+#define regIH_VMID_12_LUT_MM                                                                            0x001c
+#define regIH_VMID_12_LUT_MM_BASE_IDX                                                                   0
+#define regIH_VMID_13_LUT_MM                                                                            0x001d
+#define regIH_VMID_13_LUT_MM_BASE_IDX                                                                   0
+#define regIH_VMID_14_LUT_MM                                                                            0x001e
+#define regIH_VMID_14_LUT_MM_BASE_IDX                                                                   0
+#define regIH_VMID_15_LUT_MM                                                                            0x001f
+#define regIH_VMID_15_LUT_MM_BASE_IDX                                                                   0
+#define regIH_COOKIE_0                                                                                  0x0020
+#define regIH_COOKIE_0_BASE_IDX                                                                         0
+#define regIH_COOKIE_1                                                                                  0x0021
+#define regIH_COOKIE_1_BASE_IDX                                                                         0
+#define regIH_COOKIE_2                                                                                  0x0022
+#define regIH_COOKIE_2_BASE_IDX                                                                         0
+#define regIH_COOKIE_3                                                                                  0x0023
+#define regIH_COOKIE_3_BASE_IDX                                                                         0
+#define regIH_COOKIE_4                                                                                  0x0024
+#define regIH_COOKIE_4_BASE_IDX                                                                         0
+#define regIH_COOKIE_5                                                                                  0x0025
+#define regIH_COOKIE_5_BASE_IDX                                                                         0
+#define regIH_COOKIE_6                                                                                  0x0026
+#define regIH_COOKIE_6_BASE_IDX                                                                         0
+#define regIH_COOKIE_7                                                                                  0x0027
+#define regIH_COOKIE_7_BASE_IDX                                                                         0
+#define regIH_REGISTER_LAST_PART0                                                                       0x003f
+#define regIH_REGISTER_LAST_PART0_BASE_IDX                                                              0
+#define regIH_RB_CNTL                                                                                   0x0080
+#define regIH_RB_CNTL_BASE_IDX                                                                          0
+#define regIH_RB_BASE                                                                                   0x0081
+#define regIH_RB_BASE_BASE_IDX                                                                          0
+#define regIH_RB_BASE_HI                                                                                0x0082
+#define regIH_RB_BASE_HI_BASE_IDX                                                                       0
+#define regIH_RB_RPTR                                                                                   0x0083
+#define regIH_RB_RPTR_BASE_IDX                                                                          0
+#define regIH_RB_WPTR                                                                                   0x0084
+#define regIH_RB_WPTR_BASE_IDX                                                                          0
+#define regIH_RB_WPTR_ADDR_HI                                                                           0x0085
+#define regIH_RB_WPTR_ADDR_HI_BASE_IDX                                                                  0
+#define regIH_RB_WPTR_ADDR_LO                                                                           0x0086
+#define regIH_RB_WPTR_ADDR_LO_BASE_IDX                                                                  0
+#define regIH_DOORBELL_RPTR                                                                             0x0087
+#define regIH_DOORBELL_RPTR_BASE_IDX                                                                    0
+#define regIH_DOORBELL_RETRY_CAM                                                                        0x0088
+#define regIH_DOORBELL_RETRY_CAM_BASE_IDX                                                               0
+#define regIH_RB_CNTL_RING1                                                                             0x008c
+#define regIH_RB_CNTL_RING1_BASE_IDX                                                                    0
+#define regIH_RB_BASE_RING1                                                                             0x008d
+#define regIH_RB_BASE_RING1_BASE_IDX                                                                    0
+#define regIH_RB_BASE_HI_RING1                                                                          0x008e
+#define regIH_RB_BASE_HI_RING1_BASE_IDX                                                                 0
+#define regIH_RB_RPTR_RING1                                                                             0x008f
+#define regIH_RB_RPTR_RING1_BASE_IDX                                                                    0
+#define regIH_RB_WPTR_RING1                                                                             0x0090
+#define regIH_RB_WPTR_RING1_BASE_IDX                                                                    0
+#define regIH_DOORBELL_RPTR_RING1                                                                       0x0093
+#define regIH_DOORBELL_RPTR_RING1_BASE_IDX                                                              0
+#define regIH_RETRY_CAM_ACK                                                                             0x00a4
+#define regIH_RETRY_CAM_ACK_BASE_IDX                                                                    0
+#define regIH_VERSION                                                                                   0x00a5
+#define regIH_VERSION_BASE_IDX                                                                          0
+#define regIH_CNTL                                                                                      0x00c0
+#define regIH_CNTL_BASE_IDX                                                                             0
+#define regIH_CNTL2                                                                                     0x00c1
+#define regIH_CNTL2_BASE_IDX                                                                            0
+#define regIH_STATUS                                                                                    0x00c2
+#define regIH_STATUS_BASE_IDX                                                                           0
+#define regIH_PERFMON_CNTL                                                                              0x00c3
+#define regIH_PERFMON_CNTL_BASE_IDX                                                                     0
+#define regIH_PERFCOUNTER0_RESULT                                                                       0x00c4
+#define regIH_PERFCOUNTER0_RESULT_BASE_IDX                                                              0
+#define regIH_PERFCOUNTER1_RESULT                                                                       0x00c5
+#define regIH_PERFCOUNTER1_RESULT_BASE_IDX                                                              0
+#define regIH_DSM_MATCH_VALUE_BIT_31_0                                                                  0x00c7
+#define regIH_DSM_MATCH_VALUE_BIT_31_0_BASE_IDX                                                         0
+#define regIH_DSM_MATCH_VALUE_BIT_63_32                                                                 0x00c8
+#define regIH_DSM_MATCH_VALUE_BIT_63_32_BASE_IDX                                                        0
+#define regIH_DSM_MATCH_VALUE_BIT_95_64                                                                 0x00c9
+#define regIH_DSM_MATCH_VALUE_BIT_95_64_BASE_IDX                                                        0
+#define regIH_DSM_MATCH_FIELD_CONTROL                                                                   0x00ca
+#define regIH_DSM_MATCH_FIELD_CONTROL_BASE_IDX                                                          0
+#define regIH_DSM_MATCH_DATA_CONTROL                                                                    0x00cb
+#define regIH_DSM_MATCH_DATA_CONTROL_BASE_IDX                                                           0
+#define regIH_DSM_MATCH_FCN_ID                                                                          0x00cc
+#define regIH_DSM_MATCH_FCN_ID_BASE_IDX                                                                 0
+#define regIH_LIMIT_INT_RATE_CNTL                                                                       0x00cd
+#define regIH_LIMIT_INT_RATE_CNTL_BASE_IDX                                                              0
+#define regIH_VF_RB_STATUS                                                                              0x00ce
+#define regIH_VF_RB_STATUS_BASE_IDX                                                                     0
+#define regIH_VF_RB_STATUS2                                                                             0x00cf
+#define regIH_VF_RB_STATUS2_BASE_IDX                                                                    0
+#define regIH_VF_RB1_STATUS                                                                             0x00d0
+#define regIH_VF_RB1_STATUS_BASE_IDX                                                                    0
+#define regIH_VF_RB1_STATUS2                                                                            0x00d1
+#define regIH_VF_RB1_STATUS2_BASE_IDX                                                                   0
+#define regIH_INT_FLOOD_CNTL                                                                            0x00d5
+#define regIH_INT_FLOOD_CNTL_BASE_IDX                                                                   0
+#define regIH_RB0_INT_FLOOD_STATUS                                                                      0x00d6
+#define regIH_RB0_INT_FLOOD_STATUS_BASE_IDX                                                             0
+#define regIH_RB1_INT_FLOOD_STATUS                                                                      0x00d7
+#define regIH_RB1_INT_FLOOD_STATUS_BASE_IDX                                                             0
+#define regIH_INT_FLOOD_STATUS                                                                          0x00d9
+#define regIH_INT_FLOOD_STATUS_BASE_IDX                                                                 0
+#define regIH_STORM_CLIENT_LIST_CNTL                                                                    0x00da
+#define regIH_STORM_CLIENT_LIST_CNTL_BASE_IDX                                                           0
+#define regIH_CLK_CTRL                                                                                  0x00db
+#define regIH_CLK_CTRL_BASE_IDX                                                                         0
+#define regIH_INT_FLAGS                                                                                 0x00dc
+#define regIH_INT_FLAGS_BASE_IDX                                                                        0
+#define regIH_LAST_INT_INFO0                                                                            0x00dd
+#define regIH_LAST_INT_INFO0_BASE_IDX                                                                   0
+#define regIH_LAST_INT_INFO1                                                                            0x00de
+#define regIH_LAST_INT_INFO1_BASE_IDX                                                                   0
+#define regIH_LAST_INT_INFO2                                                                            0x00df
+#define regIH_LAST_INT_INFO2_BASE_IDX                                                                   0
+#define regIH_SCRATCH                                                                                   0x00e0
+#define regIH_SCRATCH_BASE_IDX                                                                          0
+#define regIH_CLIENT_CREDIT_ERROR                                                                       0x00e1
+#define regIH_CLIENT_CREDIT_ERROR_BASE_IDX                                                              0
+#define regIH_GPU_IOV_VIOLATION_LOG                                                                     0x00e2
+#define regIH_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                            0
+#define regIH_GPU_IOV_VIOLATION_LOG2                                                                    0x00e3
+#define regIH_GPU_IOV_VIOLATION_LOG2_BASE_IDX                                                           0
+#define regIH_COOKIE_REC_VIOLATION_LOG                                                                  0x00e4
+#define regIH_COOKIE_REC_VIOLATION_LOG_BASE_IDX                                                         0
+#define regIH_CREDIT_STATUS                                                                             0x00e5
+#define regIH_CREDIT_STATUS_BASE_IDX                                                                    0
+#define regIH_MMHUB_ERROR                                                                               0x00e6
+#define regIH_MMHUB_ERROR_BASE_IDX                                                                      0
+#define regIH_MEM_POWER_CTRL                                                                            0x00e9
+#define regIH_MEM_POWER_CTRL_BASE_IDX                                                                   0
+#define regIH_RETRY_INT_CAM_CNTL                                                                        0x00ea
+#define regIH_RETRY_INT_CAM_CNTL_BASE_IDX                                                               0
+#define regIH_VMID_LUT_INDEX                                                                            0x00ec
+#define regIH_VMID_LUT_INDEX_BASE_IDX                                                                   0
+#define regIH_MEM_POWER_CTRL2                                                                           0x00f1
+#define regIH_MEM_POWER_CTRL2_BASE_IDX                                                                  0
+#define regIH_REGISTER_LAST_PART2                                                                       0x00ff
+#define regIH_REGISTER_LAST_PART2_BASE_IDX                                                              0
+#define regSEM_MAILBOX                                                                                  0x010a
+#define regSEM_MAILBOX_BASE_IDX                                                                         0
+#define regSEM_MAILBOX_CLEAR                                                                            0x010b
+#define regSEM_MAILBOX_CLEAR_BASE_IDX                                                                   0
+#define regSEM_REGISTER_LAST_PART2                                                                      0x017f
+#define regSEM_REGISTER_LAST_PART2_BASE_IDX                                                             0
+#define regIH_ACTIVE_FCN_ID                                                                             0x0180
+#define regIH_ACTIVE_FCN_ID_BASE_IDX                                                                    0
+#define regIH_VIRT_RESET_REQ                                                                            0x0181
+#define regIH_VIRT_RESET_REQ_BASE_IDX                                                                   0
+#define regIH_CLIENT_CFG                                                                                0x0184
+#define regIH_CLIENT_CFG_BASE_IDX                                                                       0
+#define regIH_CLIENT_CFG_INDEX                                                                          0x0188
+#define regIH_CLIENT_CFG_INDEX_BASE_IDX                                                                 0
+#define regIH_CLIENT_CFG_DATA                                                                           0x0189
+#define regIH_CLIENT_CFG_DATA_BASE_IDX                                                                  0
+#define regIH_CLIENT_CFG_DATA2                                                                          0x018a
+#define regIH_CLIENT_CFG_DATA2_BASE_IDX                                                                 0
+#define regIH_CID_REMAP_INDEX                                                                           0x018b
+#define regIH_CID_REMAP_INDEX_BASE_IDX                                                                  0
+#define regIH_CID_REMAP_DATA                                                                            0x018c
+#define regIH_CID_REMAP_DATA_BASE_IDX                                                                   0
+#define regIH_CHICKEN                                                                                   0x018d
+#define regIH_CHICKEN_BASE_IDX                                                                          0
+#define regIH_INT_DROP_CNTL                                                                             0x018f
+#define regIH_INT_DROP_CNTL_BASE_IDX                                                                    0
+#define regIH_INT_DROP_MATCH_VALUE0                                                                     0x0190
+#define regIH_INT_DROP_MATCH_VALUE0_BASE_IDX                                                            0
+#define regIH_INT_DROP_MATCH_VALUE1                                                                     0x0191
+#define regIH_INT_DROP_MATCH_VALUE1_BASE_IDX                                                            0
+#define regIH_INT_DROP_MATCH_MASK0                                                                      0x0192
+#define regIH_INT_DROP_MATCH_MASK0_BASE_IDX                                                             0
+#define regIH_INT_DROP_MATCH_MASK1                                                                      0x0193
+#define regIH_INT_DROP_MATCH_MASK1_BASE_IDX                                                             0
+#define regIH_MMHUB_CNTL                                                                                0x019e
+#define regIH_MMHUB_CNTL_BASE_IDX                                                                       0
+#define regIH_REGISTER_LAST_PART1                                                                       0x019f
+#define regIH_REGISTER_LAST_PART1_BASE_IDX                                                              0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_4_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_4_2_sh_mask.h
new file mode 100644
index 000000000000..7d1cf3bac801
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_4_2_sh_mask.h
@@ -0,0 +1,995 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _osssys_4_4_2_SH_MASK_HEADER
+#define _osssys_4_4_2_SH_MASK_HEADER
+
+
+// addressBlock: aid_osssys_osssysdec
+//IH_VMID_0_LUT
+#define IH_VMID_0_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_0_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_1_LUT
+#define IH_VMID_1_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_1_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_2_LUT
+#define IH_VMID_2_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_2_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_3_LUT
+#define IH_VMID_3_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_3_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_4_LUT
+#define IH_VMID_4_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_4_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_5_LUT
+#define IH_VMID_5_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_5_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_6_LUT
+#define IH_VMID_6_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_6_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_7_LUT
+#define IH_VMID_7_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_7_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_8_LUT
+#define IH_VMID_8_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_8_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_9_LUT
+#define IH_VMID_9_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_9_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_10_LUT
+#define IH_VMID_10_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_10_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_11_LUT
+#define IH_VMID_11_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_11_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_12_LUT
+#define IH_VMID_12_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_12_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_13_LUT
+#define IH_VMID_13_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_13_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_14_LUT
+#define IH_VMID_14_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_14_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_15_LUT
+#define IH_VMID_15_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_15_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_0_LUT_MM
+#define IH_VMID_0_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_0_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_1_LUT_MM
+#define IH_VMID_1_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_1_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_2_LUT_MM
+#define IH_VMID_2_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_2_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_3_LUT_MM
+#define IH_VMID_3_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_3_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_4_LUT_MM
+#define IH_VMID_4_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_4_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_5_LUT_MM
+#define IH_VMID_5_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_5_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_6_LUT_MM
+#define IH_VMID_6_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_6_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_7_LUT_MM
+#define IH_VMID_7_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_7_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_8_LUT_MM
+#define IH_VMID_8_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_8_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_9_LUT_MM
+#define IH_VMID_9_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_9_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_10_LUT_MM
+#define IH_VMID_10_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_10_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_VMID_11_LUT_MM
+#define IH_VMID_11_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_11_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_VMID_12_LUT_MM
+#define IH_VMID_12_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_12_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_VMID_13_LUT_MM
+#define IH_VMID_13_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_13_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_VMID_14_LUT_MM
+#define IH_VMID_14_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_14_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_VMID_15_LUT_MM
+#define IH_VMID_15_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_15_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_COOKIE_0
+#define IH_COOKIE_0__CLIENT_ID__SHIFT                                                                         0x0
+#define IH_COOKIE_0__SOURCE_ID__SHIFT                                                                         0x8
+#define IH_COOKIE_0__RING_ID__SHIFT                                                                           0x10
+#define IH_COOKIE_0__VM_ID__SHIFT                                                                             0x18
+#define IH_COOKIE_0__RESERVED__SHIFT                                                                          0x1c
+#define IH_COOKIE_0__VMID_TYPE__SHIFT                                                                         0x1f
+#define IH_COOKIE_0__CLIENT_ID_MASK                                                                           0x000000FFL
+#define IH_COOKIE_0__SOURCE_ID_MASK                                                                           0x0000FF00L
+#define IH_COOKIE_0__RING_ID_MASK                                                                             0x00FF0000L
+#define IH_COOKIE_0__VM_ID_MASK                                                                               0x0F000000L
+#define IH_COOKIE_0__RESERVED_MASK                                                                            0x70000000L
+#define IH_COOKIE_0__VMID_TYPE_MASK                                                                           0x80000000L
+//IH_COOKIE_1
+#define IH_COOKIE_1__TIMESTAMP_31_0__SHIFT                                                                    0x0
+#define IH_COOKIE_1__TIMESTAMP_31_0_MASK                                                                      0xFFFFFFFFL
+//IH_COOKIE_2
+#define IH_COOKIE_2__TIMESTAMP_47_32__SHIFT                                                                   0x0
+#define IH_COOKIE_2__RESERVED__SHIFT                                                                          0x10
+#define IH_COOKIE_2__TIMESTAMP_SRC__SHIFT                                                                     0x1f
+#define IH_COOKIE_2__TIMESTAMP_47_32_MASK                                                                     0x0000FFFFL
+#define IH_COOKIE_2__RESERVED_MASK                                                                            0x7FFF0000L
+#define IH_COOKIE_2__TIMESTAMP_SRC_MASK                                                                       0x80000000L
+//IH_COOKIE_3
+#define IH_COOKIE_3__PAS_ID__SHIFT                                                                            0x0
+#define IH_COOKIE_3__RESERVED__SHIFT                                                                          0x10
+#define IH_COOKIE_3__PASID_SRC__SHIFT                                                                         0x1f
+#define IH_COOKIE_3__PAS_ID_MASK                                                                              0x0000FFFFL
+#define IH_COOKIE_3__RESERVED_MASK                                                                            0x7FFF0000L
+#define IH_COOKIE_3__PASID_SRC_MASK                                                                           0x80000000L
+//IH_COOKIE_4
+#define IH_COOKIE_4__CONTEXT_ID_31_0__SHIFT                                                                   0x0
+#define IH_COOKIE_4__CONTEXT_ID_31_0_MASK                                                                     0xFFFFFFFFL
+//IH_COOKIE_5
+#define IH_COOKIE_5__CONTEXT_ID_63_32__SHIFT                                                                  0x0
+#define IH_COOKIE_5__CONTEXT_ID_63_32_MASK                                                                    0xFFFFFFFFL
+//IH_COOKIE_6
+#define IH_COOKIE_6__CONTEXT_ID_95_64__SHIFT                                                                  0x0
+#define IH_COOKIE_6__CONTEXT_ID_95_64_MASK                                                                    0xFFFFFFFFL
+//IH_COOKIE_7
+#define IH_COOKIE_7__CONTEXT_ID_128_96__SHIFT                                                                 0x0
+#define IH_COOKIE_7__CONTEXT_ID_128_96_MASK                                                                   0xFFFFFFFFL
+//IH_REGISTER_LAST_PART0
+#define IH_REGISTER_LAST_PART0__RESERVED__SHIFT                                                               0x0
+#define IH_REGISTER_LAST_PART0__RESERVED_MASK                                                                 0xFFFFFFFFL
+//IH_RB_CNTL
+#define IH_RB_CNTL__RB_ENABLE__SHIFT                                                                          0x0
+#define IH_RB_CNTL__RB_SIZE__SHIFT                                                                            0x1
+#define IH_RB_CNTL__RB_GPU_TS_ENABLE__SHIFT                                                                   0x7
+#define IH_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT                                                              0x8
+#define IH_RB_CNTL__RB_FULL_DRAIN_ENABLE__SHIFT                                                               0x9
+#define IH_RB_CNTL__FULL_DRAIN_CLEAR__SHIFT                                                                   0xa
+#define IH_RB_CNTL__PAGE_RB_CLEAR__SHIFT                                                                      0xb
+#define IH_RB_CNTL__RB_USED_INT_THRESHOLD__SHIFT                                                              0xc
+#define IH_RB_CNTL__WPTR_OVERFLOW_ENABLE__SHIFT                                                               0x10
+#define IH_RB_CNTL__ENABLE_INTR__SHIFT                                                                        0x11
+#define IH_RB_CNTL__MC_SWAP__SHIFT                                                                            0x12
+#define IH_RB_CNTL__MC_SNOOP__SHIFT                                                                           0x14
+#define IH_RB_CNTL__RPTR_REARM__SHIFT                                                                         0x15
+#define IH_RB_CNTL__MC_RO__SHIFT                                                                              0x16
+#define IH_RB_CNTL__MC_VMID__SHIFT                                                                            0x18
+#define IH_RB_CNTL__MC_SPACE__SHIFT                                                                           0x1c
+#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT                                                                0x1f
+#define IH_RB_CNTL__RB_ENABLE_MASK                                                                            0x00000001L
+#define IH_RB_CNTL__RB_SIZE_MASK                                                                              0x0000003EL
+#define IH_RB_CNTL__RB_GPU_TS_ENABLE_MASK                                                                     0x00000080L
+#define IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK                                                                0x00000100L
+#define IH_RB_CNTL__RB_FULL_DRAIN_ENABLE_MASK                                                                 0x00000200L
+#define IH_RB_CNTL__FULL_DRAIN_CLEAR_MASK                                                                     0x00000400L
+#define IH_RB_CNTL__PAGE_RB_CLEAR_MASK                                                                        0x00000800L
+#define IH_RB_CNTL__RB_USED_INT_THRESHOLD_MASK                                                                0x0000F000L
+#define IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK                                                                 0x00010000L
+#define IH_RB_CNTL__ENABLE_INTR_MASK                                                                          0x00020000L
+#define IH_RB_CNTL__MC_SWAP_MASK                                                                              0x000C0000L
+#define IH_RB_CNTL__MC_SNOOP_MASK                                                                             0x00100000L
+#define IH_RB_CNTL__RPTR_REARM_MASK                                                                           0x00200000L
+#define IH_RB_CNTL__MC_RO_MASK                                                                                0x00400000L
+#define IH_RB_CNTL__MC_VMID_MASK                                                                              0x0F000000L
+#define IH_RB_CNTL__MC_SPACE_MASK                                                                             0x70000000L
+#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK                                                                  0x80000000L
+//IH_RB_BASE
+#define IH_RB_BASE__ADDR__SHIFT                                                                               0x0
+#define IH_RB_BASE__ADDR_MASK                                                                                 0xFFFFFFFFL
+//IH_RB_BASE_HI
+#define IH_RB_BASE_HI__ADDR__SHIFT                                                                            0x0
+#define IH_RB_BASE_HI__ADDR_MASK                                                                              0x000000FFL
+//IH_RB_RPTR
+#define IH_RB_RPTR__OFFSET__SHIFT                                                                             0x2
+#define IH_RB_RPTR__OFFSET_MASK                                                                               0x0003FFFCL
+//IH_RB_WPTR
+#define IH_RB_WPTR__RB_OVERFLOW__SHIFT                                                                        0x0
+#define IH_RB_WPTR__OFFSET__SHIFT                                                                             0x2
+#define IH_RB_WPTR__RB_LEFT_NONE__SHIFT                                                                       0x12
+#define IH_RB_WPTR__RB_MAY_OVERFLOW__SHIFT                                                                    0x13
+#define IH_RB_WPTR__RB_OVERFLOW_MASK                                                                          0x00000001L
+#define IH_RB_WPTR__OFFSET_MASK                                                                               0x0003FFFCL
+#define IH_RB_WPTR__RB_LEFT_NONE_MASK                                                                         0x00040000L
+#define IH_RB_WPTR__RB_MAY_OVERFLOW_MASK                                                                      0x00080000L
+//IH_RB_WPTR_ADDR_HI
+#define IH_RB_WPTR_ADDR_HI__ADDR__SHIFT                                                                       0x0
+#define IH_RB_WPTR_ADDR_HI__ADDR_MASK                                                                         0x0000FFFFL
+//IH_RB_WPTR_ADDR_LO
+#define IH_RB_WPTR_ADDR_LO__ADDR__SHIFT                                                                       0x2
+#define IH_RB_WPTR_ADDR_LO__ADDR_MASK                                                                         0xFFFFFFFCL
+//IH_DOORBELL_RPTR
+#define IH_DOORBELL_RPTR__OFFSET__SHIFT                                                                       0x0
+#define IH_DOORBELL_RPTR__ENABLE__SHIFT                                                                       0x1c
+#define IH_DOORBELL_RPTR__OFFSET_MASK                                                                         0x03FFFFFFL
+#define IH_DOORBELL_RPTR__ENABLE_MASK                                                                         0x10000000L
+//IH_DOORBELL_RETRY_CAM
+#define IH_DOORBELL_RETRY_CAM__OFFSET__SHIFT                                                                  0x0
+#define IH_DOORBELL_RETRY_CAM__ENABLE__SHIFT                                                                  0x1c
+#define IH_DOORBELL_RETRY_CAM__OFFSET_MASK                                                                    0x03FFFFFFL
+#define IH_DOORBELL_RETRY_CAM__ENABLE_MASK                                                                    0x10000000L
+//IH_RB_CNTL_RING1
+#define IH_RB_CNTL_RING1__RB_ENABLE__SHIFT                                                                    0x0
+#define IH_RB_CNTL_RING1__RB_SIZE__SHIFT                                                                      0x1
+#define IH_RB_CNTL_RING1__RB_GPU_TS_ENABLE__SHIFT                                                             0x7
+#define IH_RB_CNTL_RING1__RB_FULL_DRAIN_ENABLE__SHIFT                                                         0x9
+#define IH_RB_CNTL_RING1__FULL_DRAIN_CLEAR__SHIFT                                                             0xa
+#define IH_RB_CNTL_RING1__PAGE_RB_CLEAR__SHIFT                                                                0xb
+#define IH_RB_CNTL_RING1__RB_USED_INT_THRESHOLD__SHIFT                                                        0xc
+#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_ENABLE__SHIFT                                                         0x10
+#define IH_RB_CNTL_RING1__MC_SWAP__SHIFT                                                                      0x12
+#define IH_RB_CNTL_RING1__MC_SNOOP__SHIFT                                                                     0x14
+#define IH_RB_CNTL_RING1__MC_RO__SHIFT                                                                        0x16
+#define IH_RB_CNTL_RING1__MC_VMID__SHIFT                                                                      0x18
+#define IH_RB_CNTL_RING1__MC_SPACE__SHIFT                                                                     0x1c
+#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_CLEAR__SHIFT                                                          0x1f
+#define IH_RB_CNTL_RING1__RB_ENABLE_MASK                                                                      0x00000001L
+#define IH_RB_CNTL_RING1__RB_SIZE_MASK                                                                        0x0000003EL
+#define IH_RB_CNTL_RING1__RB_GPU_TS_ENABLE_MASK                                                               0x00000080L
+#define IH_RB_CNTL_RING1__RB_FULL_DRAIN_ENABLE_MASK                                                           0x00000200L
+#define IH_RB_CNTL_RING1__FULL_DRAIN_CLEAR_MASK                                                               0x00000400L
+#define IH_RB_CNTL_RING1__PAGE_RB_CLEAR_MASK                                                                  0x00000800L
+#define IH_RB_CNTL_RING1__RB_USED_INT_THRESHOLD_MASK                                                          0x0000F000L
+#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_ENABLE_MASK                                                           0x00010000L
+#define IH_RB_CNTL_RING1__MC_SWAP_MASK                                                                        0x000C0000L
+#define IH_RB_CNTL_RING1__MC_SNOOP_MASK                                                                       0x00100000L
+#define IH_RB_CNTL_RING1__MC_RO_MASK                                                                          0x00400000L
+#define IH_RB_CNTL_RING1__MC_VMID_MASK                                                                        0x0F000000L
+#define IH_RB_CNTL_RING1__MC_SPACE_MASK                                                                       0x70000000L
+#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_CLEAR_MASK                                                            0x80000000L
+//IH_RB_BASE_RING1
+#define IH_RB_BASE_RING1__ADDR__SHIFT                                                                         0x0
+#define IH_RB_BASE_RING1__ADDR_MASK                                                                           0xFFFFFFFFL
+//IH_RB_BASE_HI_RING1
+#define IH_RB_BASE_HI_RING1__ADDR__SHIFT                                                                      0x0
+#define IH_RB_BASE_HI_RING1__ADDR_MASK                                                                        0x000000FFL
+//IH_RB_RPTR_RING1
+#define IH_RB_RPTR_RING1__OFFSET__SHIFT                                                                       0x2
+#define IH_RB_RPTR_RING1__OFFSET_MASK                                                                         0x0003FFFCL
+//IH_RB_WPTR_RING1
+#define IH_RB_WPTR_RING1__RB_OVERFLOW__SHIFT                                                                  0x0
+#define IH_RB_WPTR_RING1__OFFSET__SHIFT                                                                       0x2
+#define IH_RB_WPTR_RING1__RB_LEFT_NONE__SHIFT                                                                 0x12
+#define IH_RB_WPTR_RING1__RB_MAY_OVERFLOW__SHIFT                                                              0x13
+#define IH_RB_WPTR_RING1__RB_OVERFLOW_MASK                                                                    0x00000001L
+#define IH_RB_WPTR_RING1__OFFSET_MASK                                                                         0x0003FFFCL
+#define IH_RB_WPTR_RING1__RB_LEFT_NONE_MASK                                                                   0x00040000L
+#define IH_RB_WPTR_RING1__RB_MAY_OVERFLOW_MASK                                                                0x00080000L
+//IH_DOORBELL_RPTR_RING1
+#define IH_DOORBELL_RPTR_RING1__OFFSET__SHIFT                                                                 0x0
+#define IH_DOORBELL_RPTR_RING1__ENABLE__SHIFT                                                                 0x1c
+#define IH_DOORBELL_RPTR_RING1__OFFSET_MASK                                                                   0x03FFFFFFL
+#define IH_DOORBELL_RPTR_RING1__ENABLE_MASK                                                                   0x10000000L
+//IH_RETRY_CAM_ACK
+#define IH_RETRY_CAM_ACK__INDEX__SHIFT                                                                        0x0
+#define IH_RETRY_CAM_ACK__INDEX_MASK                                                                          0x000003FFL
+//IH_VERSION
+#define IH_VERSION__MINVER__SHIFT                                                                             0x0
+#define IH_VERSION__MAJVER__SHIFT                                                                             0x8
+#define IH_VERSION__REV__SHIFT                                                                                0x10
+#define IH_VERSION__MINVER_MASK                                                                               0x0000007FL
+#define IH_VERSION__MAJVER_MASK                                                                               0x00007F00L
+#define IH_VERSION__REV_MASK                                                                                  0x003F0000L
+//IH_CNTL
+#define IH_CNTL__WPTR_WRITEBACK_TIMER__SHIFT                                                                  0x0
+#define IH_CNTL__IH_IDLE_HYSTERESIS_CNTL__SHIFT                                                               0x6
+#define IH_CNTL__IH_FIFO_HIGHWATER__SHIFT                                                                     0x8
+#define IH_CNTL__MC_WR_CLEAN_CNT__SHIFT                                                                       0x14
+#define IH_CNTL__SRAM_ECC_ENABLE__SHIFT                                                                       0x19
+#define IH_CNTL__FED_ENABLE__SHIFT                                                                            0x1a
+#define IH_CNTL__WPTR_WRITEBACK_TIMER_MASK                                                                    0x0000001FL
+#define IH_CNTL__IH_IDLE_HYSTERESIS_CNTL_MASK                                                                 0x000000C0L
+#define IH_CNTL__IH_FIFO_HIGHWATER_MASK                                                                       0x00007F00L
+#define IH_CNTL__MC_WR_CLEAN_CNT_MASK                                                                         0x01F00000L
+#define IH_CNTL__SRAM_ECC_ENABLE_MASK                                                                         0x02000000L
+#define IH_CNTL__FED_ENABLE_MASK                                                                              0x04000000L
+//IH_CNTL2
+#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_TIMEOUT__SHIFT                                                    0x0
+#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_ENABLE__SHIFT                                                     0x8
+#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_TIMEOUT_MASK                                                      0x0000001FL
+#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_ENABLE_MASK                                                       0x00000100L
+//IH_STATUS
+#define IH_STATUS__IDLE__SHIFT                                                                                0x0
+#define IH_STATUS__INPUT_IDLE__SHIFT                                                                          0x1
+#define IH_STATUS__BUFFER_IDLE__SHIFT                                                                         0x2
+#define IH_STATUS__RB_FULL__SHIFT                                                                             0x3
+#define IH_STATUS__RB_FULL_DRAIN__SHIFT                                                                       0x4
+#define IH_STATUS__RB_OVERFLOW__SHIFT                                                                         0x5
+#define IH_STATUS__MC_WR_IDLE__SHIFT                                                                          0x6
+#define IH_STATUS__MC_WR_STALL__SHIFT                                                                         0x7
+#define IH_STATUS__MC_WR_CLEAN_PENDING__SHIFT                                                                 0x8
+#define IH_STATUS__MC_WR_CLEAN_STALL__SHIFT                                                                   0x9
+#define IH_STATUS__BIF_INTERRUPT_LINE__SHIFT                                                                  0xa
+#define IH_STATUS__SWITCH_READY__SHIFT                                                                        0xb
+#define IH_STATUS__RB1_FULL__SHIFT                                                                            0xc
+#define IH_STATUS__RB1_FULL_DRAIN__SHIFT                                                                      0xd
+#define IH_STATUS__RB1_OVERFLOW__SHIFT                                                                        0xe
+#define IH_STATUS__SELF_INT_GEN_IDLE__SHIFT                                                                   0x12
+#define IH_STATUS__RETRY_INT_CAM_IDLE__SHIFT                                                                  0x13
+#define IH_STATUS__IH_BUFFER_MEM_POWER_GATED__SHIFT                                                           0x14
+#define IH_STATUS__IH_RETRY_INT_CAM_MEM_POWER_GATED__SHIFT                                                    0x15
+#define IH_STATUS__IH_PASID_LUT_MEM_POWER_GATED__SHIFT                                                        0x16
+#define IH_STATUS__IDLE_MASK                                                                                  0x00000001L
+#define IH_STATUS__INPUT_IDLE_MASK                                                                            0x00000002L
+#define IH_STATUS__BUFFER_IDLE_MASK                                                                           0x00000004L
+#define IH_STATUS__RB_FULL_MASK                                                                               0x00000008L
+#define IH_STATUS__RB_FULL_DRAIN_MASK                                                                         0x00000010L
+#define IH_STATUS__RB_OVERFLOW_MASK                                                                           0x00000020L
+#define IH_STATUS__MC_WR_IDLE_MASK                                                                            0x00000040L
+#define IH_STATUS__MC_WR_STALL_MASK                                                                           0x00000080L
+#define IH_STATUS__MC_WR_CLEAN_PENDING_MASK                                                                   0x00000100L
+#define IH_STATUS__MC_WR_CLEAN_STALL_MASK                                                                     0x00000200L
+#define IH_STATUS__BIF_INTERRUPT_LINE_MASK                                                                    0x00000400L
+#define IH_STATUS__SWITCH_READY_MASK                                                                          0x00000800L
+#define IH_STATUS__RB1_FULL_MASK                                                                              0x00001000L
+#define IH_STATUS__RB1_FULL_DRAIN_MASK                                                                        0x00002000L
+#define IH_STATUS__RB1_OVERFLOW_MASK                                                                          0x00004000L
+#define IH_STATUS__SELF_INT_GEN_IDLE_MASK                                                                     0x00040000L
+#define IH_STATUS__RETRY_INT_CAM_IDLE_MASK                                                                    0x00080000L
+#define IH_STATUS__IH_BUFFER_MEM_POWER_GATED_MASK                                                             0x00100000L
+#define IH_STATUS__IH_RETRY_INT_CAM_MEM_POWER_GATED_MASK                                                      0x00200000L
+#define IH_STATUS__IH_PASID_LUT_MEM_POWER_GATED_MASK                                                          0x00400000L
+//IH_PERFMON_CNTL
+#define IH_PERFMON_CNTL__ENABLE0__SHIFT                                                                       0x0
+#define IH_PERFMON_CNTL__CLEAR0__SHIFT                                                                        0x1
+#define IH_PERFMON_CNTL__PERF_SEL0__SHIFT                                                                     0x2
+#define IH_PERFMON_CNTL__ENABLE1__SHIFT                                                                       0x10
+#define IH_PERFMON_CNTL__CLEAR1__SHIFT                                                                        0x11
+#define IH_PERFMON_CNTL__PERF_SEL1__SHIFT                                                                     0x12
+#define IH_PERFMON_CNTL__ENABLE0_MASK                                                                         0x00000001L
+#define IH_PERFMON_CNTL__CLEAR0_MASK                                                                          0x00000002L
+#define IH_PERFMON_CNTL__PERF_SEL0_MASK                                                                       0x00000FFCL
+#define IH_PERFMON_CNTL__ENABLE1_MASK                                                                         0x00010000L
+#define IH_PERFMON_CNTL__CLEAR1_MASK                                                                          0x00020000L
+#define IH_PERFMON_CNTL__PERF_SEL1_MASK                                                                       0x0FFC0000L
+//IH_PERFCOUNTER0_RESULT
+#define IH_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT                                                             0x0
+#define IH_PERFCOUNTER0_RESULT__PERF_COUNT_MASK                                                               0xFFFFFFFFL
+//IH_PERFCOUNTER1_RESULT
+#define IH_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT                                                             0x0
+#define IH_PERFCOUNTER1_RESULT__PERF_COUNT_MASK                                                               0xFFFFFFFFL
+//IH_DSM_MATCH_VALUE_BIT_31_0
+#define IH_DSM_MATCH_VALUE_BIT_31_0__VALUE__SHIFT                                                             0x0
+#define IH_DSM_MATCH_VALUE_BIT_31_0__VALUE_MASK                                                               0xFFFFFFFFL
+//IH_DSM_MATCH_VALUE_BIT_63_32
+#define IH_DSM_MATCH_VALUE_BIT_63_32__VALUE__SHIFT                                                            0x0
+#define IH_DSM_MATCH_VALUE_BIT_63_32__VALUE_MASK                                                              0xFFFFFFFFL
+//IH_DSM_MATCH_VALUE_BIT_95_64
+#define IH_DSM_MATCH_VALUE_BIT_95_64__VALUE__SHIFT                                                            0x0
+#define IH_DSM_MATCH_VALUE_BIT_95_64__VALUE_MASK                                                              0xFFFFFFFFL
+//IH_DSM_MATCH_FIELD_CONTROL
+#define IH_DSM_MATCH_FIELD_CONTROL__SRC_EN__SHIFT                                                             0x0
+#define IH_DSM_MATCH_FIELD_CONTROL__FCNID_EN__SHIFT                                                           0x1
+#define IH_DSM_MATCH_FIELD_CONTROL__TIMESTAMP_EN__SHIFT                                                       0x2
+#define IH_DSM_MATCH_FIELD_CONTROL__RINGID_EN__SHIFT                                                          0x3
+#define IH_DSM_MATCH_FIELD_CONTROL__VMID_EN__SHIFT                                                            0x4
+#define IH_DSM_MATCH_FIELD_CONTROL__PASID_EN__SHIFT                                                           0x5
+#define IH_DSM_MATCH_FIELD_CONTROL__CLIENT_ID_EN__SHIFT                                                       0x6
+#define IH_DSM_MATCH_FIELD_CONTROL__DIEID_EN__SHIFT                                                           0x7
+#define IH_DSM_MATCH_FIELD_CONTROL__SRC_EN_MASK                                                               0x00000001L
+#define IH_DSM_MATCH_FIELD_CONTROL__FCNID_EN_MASK                                                             0x00000002L
+#define IH_DSM_MATCH_FIELD_CONTROL__TIMESTAMP_EN_MASK                                                         0x00000004L
+#define IH_DSM_MATCH_FIELD_CONTROL__RINGID_EN_MASK                                                            0x00000008L
+#define IH_DSM_MATCH_FIELD_CONTROL__VMID_EN_MASK                                                              0x00000010L
+#define IH_DSM_MATCH_FIELD_CONTROL__PASID_EN_MASK                                                             0x00000020L
+#define IH_DSM_MATCH_FIELD_CONTROL__CLIENT_ID_EN_MASK                                                         0x00000040L
+#define IH_DSM_MATCH_FIELD_CONTROL__DIEID_EN_MASK                                                             0x00000080L
+//IH_DSM_MATCH_DATA_CONTROL
+#define IH_DSM_MATCH_DATA_CONTROL__VALUE__SHIFT                                                               0x0
+#define IH_DSM_MATCH_DATA_CONTROL__VALUE_MASK                                                                 0x0FFFFFFFL
+//IH_DSM_MATCH_FCN_ID
+#define IH_DSM_MATCH_FCN_ID__PF_VF__SHIFT                                                                     0x0
+#define IH_DSM_MATCH_FCN_ID__VF_ID__SHIFT                                                                     0x1
+#define IH_DSM_MATCH_FCN_ID__PF_VF_MASK                                                                       0x00000001L
+#define IH_DSM_MATCH_FCN_ID__VF_ID_MASK                                                                       0x0000000EL
+//IH_LIMIT_INT_RATE_CNTL
+#define IH_LIMIT_INT_RATE_CNTL__LIMIT_ENABLE__SHIFT                                                           0x0
+#define IH_LIMIT_INT_RATE_CNTL__PERF_INTERVAL__SHIFT                                                          0x1
+#define IH_LIMIT_INT_RATE_CNTL__PERF_THRESHOLD__SHIFT                                                         0x5
+#define IH_LIMIT_INT_RATE_CNTL__RETURN_DELAY__SHIFT                                                           0x11
+#define IH_LIMIT_INT_RATE_CNTL__PERF_RESULT__SHIFT                                                            0x15
+#define IH_LIMIT_INT_RATE_CNTL__LIMIT_ENABLE_MASK                                                             0x00000001L
+#define IH_LIMIT_INT_RATE_CNTL__PERF_INTERVAL_MASK                                                            0x0000001EL
+#define IH_LIMIT_INT_RATE_CNTL__PERF_THRESHOLD_MASK                                                           0x0000FFE0L
+#define IH_LIMIT_INT_RATE_CNTL__RETURN_DELAY_MASK                                                             0x001E0000L
+#define IH_LIMIT_INT_RATE_CNTL__PERF_RESULT_MASK                                                              0xFFE00000L
+//IH_VF_RB_STATUS
+#define IH_VF_RB_STATUS__RB_FULL_DRAIN_VF__SHIFT                                                              0x0
+#define IH_VF_RB_STATUS__RB_OVERFLOW_VF__SHIFT                                                                0x10
+#define IH_VF_RB_STATUS__RB_FULL_DRAIN_VF_MASK                                                                0x000000FFL
+#define IH_VF_RB_STATUS__RB_OVERFLOW_VF_MASK                                                                  0x00FF0000L
+//IH_VF_RB_STATUS2
+#define IH_VF_RB_STATUS2__RB_FULL_VF__SHIFT                                                                   0x0
+#define IH_VF_RB_STATUS2__BIF_INTERRUPT_LINE_VF__SHIFT                                                        0x10
+#define IH_VF_RB_STATUS2__RB_FULL_VF_MASK                                                                     0x000000FFL
+#define IH_VF_RB_STATUS2__BIF_INTERRUPT_LINE_VF_MASK                                                          0x00FF0000L
+//IH_VF_RB1_STATUS
+#define IH_VF_RB1_STATUS__RB_FULL_DRAIN_VF__SHIFT                                                             0x0
+#define IH_VF_RB1_STATUS__RB_OVERFLOW_VF__SHIFT                                                               0x10
+#define IH_VF_RB1_STATUS__RB_FULL_DRAIN_VF_MASK                                                               0x000000FFL
+#define IH_VF_RB1_STATUS__RB_OVERFLOW_VF_MASK                                                                 0x00FF0000L
+//IH_VF_RB1_STATUS2
+#define IH_VF_RB1_STATUS2__RB_FULL_VF__SHIFT                                                                  0x0
+#define IH_VF_RB1_STATUS2__RB_FULL_VF_MASK                                                                    0x000000FFL
+//IH_INT_FLOOD_CNTL
+#define IH_INT_FLOOD_CNTL__HIGHWATER__SHIFT                                                                   0x0
+#define IH_INT_FLOOD_CNTL__FLOOD_CNTL_ENABLE__SHIFT                                                           0x3
+#define IH_INT_FLOOD_CNTL__CLEAR_INT_FLOOD_STATUS__SHIFT                                                      0x4
+#define IH_INT_FLOOD_CNTL__HIGHWATER_MASK                                                                     0x00000007L
+#define IH_INT_FLOOD_CNTL__FLOOD_CNTL_ENABLE_MASK                                                             0x00000008L
+#define IH_INT_FLOOD_CNTL__CLEAR_INT_FLOOD_STATUS_MASK                                                        0x00000010L
+//IH_RB0_INT_FLOOD_STATUS
+#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_VF__SHIFT                                                     0x0
+#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED__SHIFT                                                        0x1f
+#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_VF_MASK                                                       0x000000FFL
+#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_MASK                                                          0x80000000L
+//IH_RB1_INT_FLOOD_STATUS
+#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_VF__SHIFT                                                     0x0
+#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED__SHIFT                                                        0x1f
+#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_VF_MASK                                                       0x000000FFL
+#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_MASK                                                          0x80000000L
+//IH_INT_FLOOD_STATUS
+#define IH_INT_FLOOD_STATUS__INT_DROP_CNT__SHIFT                                                              0x0
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_CLIENT_ID__SHIFT                                                  0x8
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_SOURCE_ID__SHIFT                                                  0x10
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_ID__SHIFT                                                      0x18
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF__SHIFT                                                         0x1c
+#define IH_INT_FLOOD_STATUS__INT_DROPPED__SHIFT                                                               0x1e
+#define IH_INT_FLOOD_STATUS__INT_DROP_CNT_MASK                                                                0x000000FFL
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_CLIENT_ID_MASK                                                    0x0000FF00L
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_SOURCE_ID_MASK                                                    0x00FF0000L
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_ID_MASK                                                        0x07000000L
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_MASK                                                           0x10000000L
+#define IH_INT_FLOOD_STATUS__INT_DROPPED_MASK                                                                 0x40000000L
+//IH_STORM_CLIENT_LIST_CNTL
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT1_IS_STORM_CLIENT__SHIFT                                             0x1
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT2_IS_STORM_CLIENT__SHIFT                                             0x2
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT3_IS_STORM_CLIENT__SHIFT                                             0x3
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT4_IS_STORM_CLIENT__SHIFT                                             0x4
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT5_IS_STORM_CLIENT__SHIFT                                             0x5
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT6_IS_STORM_CLIENT__SHIFT                                             0x6
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT7_IS_STORM_CLIENT__SHIFT                                             0x7
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT8_IS_STORM_CLIENT__SHIFT                                             0x8
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT9_IS_STORM_CLIENT__SHIFT                                             0x9
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT10_IS_STORM_CLIENT__SHIFT                                            0xa
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT11_IS_STORM_CLIENT__SHIFT                                            0xb
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT12_IS_STORM_CLIENT__SHIFT                                            0xc
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT13_IS_STORM_CLIENT__SHIFT                                            0xd
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT14_IS_STORM_CLIENT__SHIFT                                            0xe
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT15_IS_STORM_CLIENT__SHIFT                                            0xf
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT16_IS_STORM_CLIENT__SHIFT                                            0x10
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT17_IS_STORM_CLIENT__SHIFT                                            0x11
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT18_IS_STORM_CLIENT__SHIFT                                            0x12
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT19_IS_STORM_CLIENT__SHIFT                                            0x13
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT20_IS_STORM_CLIENT__SHIFT                                            0x14
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT21_IS_STORM_CLIENT__SHIFT                                            0x15
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT22_IS_STORM_CLIENT__SHIFT                                            0x16
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT23_IS_STORM_CLIENT__SHIFT                                            0x17
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT24_IS_STORM_CLIENT__SHIFT                                            0x18
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT25_IS_STORM_CLIENT__SHIFT                                            0x19
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT26_IS_STORM_CLIENT__SHIFT                                            0x1a
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT27_IS_STORM_CLIENT__SHIFT                                            0x1b
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT28_IS_STORM_CLIENT__SHIFT                                            0x1c
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT29_IS_STORM_CLIENT__SHIFT                                            0x1d
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT30_IS_STORM_CLIENT__SHIFT                                            0x1e
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT31_IS_STORM_CLIENT__SHIFT                                            0x1f
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT1_IS_STORM_CLIENT_MASK                                               0x00000002L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT2_IS_STORM_CLIENT_MASK                                               0x00000004L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT3_IS_STORM_CLIENT_MASK                                               0x00000008L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT4_IS_STORM_CLIENT_MASK                                               0x00000010L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT5_IS_STORM_CLIENT_MASK                                               0x00000020L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT6_IS_STORM_CLIENT_MASK                                               0x00000040L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT7_IS_STORM_CLIENT_MASK                                               0x00000080L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT8_IS_STORM_CLIENT_MASK                                               0x00000100L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT9_IS_STORM_CLIENT_MASK                                               0x00000200L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT10_IS_STORM_CLIENT_MASK                                              0x00000400L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT11_IS_STORM_CLIENT_MASK                                              0x00000800L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT12_IS_STORM_CLIENT_MASK                                              0x00001000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT13_IS_STORM_CLIENT_MASK                                              0x00002000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT14_IS_STORM_CLIENT_MASK                                              0x00004000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT15_IS_STORM_CLIENT_MASK                                              0x00008000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT16_IS_STORM_CLIENT_MASK                                              0x00010000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT17_IS_STORM_CLIENT_MASK                                              0x00020000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT18_IS_STORM_CLIENT_MASK                                              0x00040000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT19_IS_STORM_CLIENT_MASK                                              0x00080000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT20_IS_STORM_CLIENT_MASK                                              0x00100000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT21_IS_STORM_CLIENT_MASK                                              0x00200000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT22_IS_STORM_CLIENT_MASK                                              0x00400000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT23_IS_STORM_CLIENT_MASK                                              0x00800000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT24_IS_STORM_CLIENT_MASK                                              0x01000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT25_IS_STORM_CLIENT_MASK                                              0x02000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT26_IS_STORM_CLIENT_MASK                                              0x04000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT27_IS_STORM_CLIENT_MASK                                              0x08000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT28_IS_STORM_CLIENT_MASK                                              0x10000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT29_IS_STORM_CLIENT_MASK                                              0x20000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT30_IS_STORM_CLIENT_MASK                                              0x40000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT31_IS_STORM_CLIENT_MASK                                              0x80000000L
+//IH_CLK_CTRL
+#define IH_CLK_CTRL__IH_PASID_LUT_MEM_CLK_SOFT_OVERRIDE__SHIFT                                                0x17
+#define IH_CLK_CTRL__IH_RAS_CLK_SOFT_OVERRIDE__SHIFT                                                          0x18
+#define IH_CLK_CTRL__IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE__SHIFT                                            0x19
+#define IH_CLK_CTRL__IH_BUFFER_MEM_CLK_SOFT_OVERRIDE__SHIFT                                                   0x1a
+#define IH_CLK_CTRL__DBUS_MUX_CLK_SOFT_OVERRIDE__SHIFT                                                        0x1b
+#define IH_CLK_CTRL__OSSSYS_SHARE_CLK_SOFT_OVERRIDE__SHIFT                                                    0x1c
+#define IH_CLK_CTRL__LIMIT_SMN_CLK_SOFT_OVERRIDE__SHIFT                                                       0x1d
+#define IH_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE__SHIFT                                                             0x1e
+#define IH_CLK_CTRL__REG_CLK_SOFT_OVERRIDE__SHIFT                                                             0x1f
+#define IH_CLK_CTRL__IH_PASID_LUT_MEM_CLK_SOFT_OVERRIDE_MASK                                                  0x00800000L
+#define IH_CLK_CTRL__IH_RAS_CLK_SOFT_OVERRIDE_MASK                                                            0x01000000L
+#define IH_CLK_CTRL__IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE_MASK                                              0x02000000L
+#define IH_CLK_CTRL__IH_BUFFER_MEM_CLK_SOFT_OVERRIDE_MASK                                                     0x04000000L
+#define IH_CLK_CTRL__DBUS_MUX_CLK_SOFT_OVERRIDE_MASK                                                          0x08000000L
+#define IH_CLK_CTRL__OSSSYS_SHARE_CLK_SOFT_OVERRIDE_MASK                                                      0x10000000L
+#define IH_CLK_CTRL__LIMIT_SMN_CLK_SOFT_OVERRIDE_MASK                                                         0x20000000L
+#define IH_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE_MASK                                                               0x40000000L
+#define IH_CLK_CTRL__REG_CLK_SOFT_OVERRIDE_MASK                                                               0x80000000L
+//IH_INT_FLAGS
+#define IH_INT_FLAGS__CLIENT_0_FLAG__SHIFT                                                                    0x0
+#define IH_INT_FLAGS__CLIENT_1_FLAG__SHIFT                                                                    0x1
+#define IH_INT_FLAGS__CLIENT_2_FLAG__SHIFT                                                                    0x2
+#define IH_INT_FLAGS__CLIENT_3_FLAG__SHIFT                                                                    0x3
+#define IH_INT_FLAGS__CLIENT_4_FLAG__SHIFT                                                                    0x4
+#define IH_INT_FLAGS__CLIENT_5_FLAG__SHIFT                                                                    0x5
+#define IH_INT_FLAGS__CLIENT_6_FLAG__SHIFT                                                                    0x6
+#define IH_INT_FLAGS__CLIENT_7_FLAG__SHIFT                                                                    0x7
+#define IH_INT_FLAGS__CLIENT_8_FLAG__SHIFT                                                                    0x8
+#define IH_INT_FLAGS__CLIENT_9_FLAG__SHIFT                                                                    0x9
+#define IH_INT_FLAGS__CLIENT_10_FLAG__SHIFT                                                                   0xa
+#define IH_INT_FLAGS__CLIENT_11_FLAG__SHIFT                                                                   0xb
+#define IH_INT_FLAGS__CLIENT_12_FLAG__SHIFT                                                                   0xc
+#define IH_INT_FLAGS__CLIENT_13_FLAG__SHIFT                                                                   0xd
+#define IH_INT_FLAGS__CLIENT_14_FLAG__SHIFT                                                                   0xe
+#define IH_INT_FLAGS__CLIENT_15_FLAG__SHIFT                                                                   0xf
+#define IH_INT_FLAGS__CLIENT_16_FLAG__SHIFT                                                                   0x10
+#define IH_INT_FLAGS__CLIENT_17_FLAG__SHIFT                                                                   0x11
+#define IH_INT_FLAGS__CLIENT_18_FLAG__SHIFT                                                                   0x12
+#define IH_INT_FLAGS__CLIENT_19_FLAG__SHIFT                                                                   0x13
+#define IH_INT_FLAGS__CLIENT_20_FLAG__SHIFT                                                                   0x14
+#define IH_INT_FLAGS__CLIENT_21_FLAG__SHIFT                                                                   0x15
+#define IH_INT_FLAGS__CLIENT_22_FLAG__SHIFT                                                                   0x16
+#define IH_INT_FLAGS__CLIENT_23_FLAG__SHIFT                                                                   0x17
+#define IH_INT_FLAGS__CLIENT_24_FLAG__SHIFT                                                                   0x18
+#define IH_INT_FLAGS__CLIENT_25_FLAG__SHIFT                                                                   0x19
+#define IH_INT_FLAGS__CLIENT_26_FLAG__SHIFT                                                                   0x1a
+#define IH_INT_FLAGS__CLIENT_27_FLAG__SHIFT                                                                   0x1b
+#define IH_INT_FLAGS__CLIENT_28_FLAG__SHIFT                                                                   0x1c
+#define IH_INT_FLAGS__CLIENT_29_FLAG__SHIFT                                                                   0x1d
+#define IH_INT_FLAGS__CLIENT_30_FLAG__SHIFT                                                                   0x1e
+#define IH_INT_FLAGS__CLIENT_31_FLAG__SHIFT                                                                   0x1f
+#define IH_INT_FLAGS__CLIENT_0_FLAG_MASK                                                                      0x00000001L
+#define IH_INT_FLAGS__CLIENT_1_FLAG_MASK                                                                      0x00000002L
+#define IH_INT_FLAGS__CLIENT_2_FLAG_MASK                                                                      0x00000004L
+#define IH_INT_FLAGS__CLIENT_3_FLAG_MASK                                                                      0x00000008L
+#define IH_INT_FLAGS__CLIENT_4_FLAG_MASK                                                                      0x00000010L
+#define IH_INT_FLAGS__CLIENT_5_FLAG_MASK                                                                      0x00000020L
+#define IH_INT_FLAGS__CLIENT_6_FLAG_MASK                                                                      0x00000040L
+#define IH_INT_FLAGS__CLIENT_7_FLAG_MASK                                                                      0x00000080L
+#define IH_INT_FLAGS__CLIENT_8_FLAG_MASK                                                                      0x00000100L
+#define IH_INT_FLAGS__CLIENT_9_FLAG_MASK                                                                      0x00000200L
+#define IH_INT_FLAGS__CLIENT_10_FLAG_MASK                                                                     0x00000400L
+#define IH_INT_FLAGS__CLIENT_11_FLAG_MASK                                                                     0x00000800L
+#define IH_INT_FLAGS__CLIENT_12_FLAG_MASK                                                                     0x00001000L
+#define IH_INT_FLAGS__CLIENT_13_FLAG_MASK                                                                     0x00002000L
+#define IH_INT_FLAGS__CLIENT_14_FLAG_MASK                                                                     0x00004000L
+#define IH_INT_FLAGS__CLIENT_15_FLAG_MASK                                                                     0x00008000L
+#define IH_INT_FLAGS__CLIENT_16_FLAG_MASK                                                                     0x00010000L
+#define IH_INT_FLAGS__CLIENT_17_FLAG_MASK                                                                     0x00020000L
+#define IH_INT_FLAGS__CLIENT_18_FLAG_MASK                                                                     0x00040000L
+#define IH_INT_FLAGS__CLIENT_19_FLAG_MASK                                                                     0x00080000L
+#define IH_INT_FLAGS__CLIENT_20_FLAG_MASK                                                                     0x00100000L
+#define IH_INT_FLAGS__CLIENT_21_FLAG_MASK                                                                     0x00200000L
+#define IH_INT_FLAGS__CLIENT_22_FLAG_MASK                                                                     0x00400000L
+#define IH_INT_FLAGS__CLIENT_23_FLAG_MASK                                                                     0x00800000L
+#define IH_INT_FLAGS__CLIENT_24_FLAG_MASK                                                                     0x01000000L
+#define IH_INT_FLAGS__CLIENT_25_FLAG_MASK                                                                     0x02000000L
+#define IH_INT_FLAGS__CLIENT_26_FLAG_MASK                                                                     0x04000000L
+#define IH_INT_FLAGS__CLIENT_27_FLAG_MASK                                                                     0x08000000L
+#define IH_INT_FLAGS__CLIENT_28_FLAG_MASK                                                                     0x10000000L
+#define IH_INT_FLAGS__CLIENT_29_FLAG_MASK                                                                     0x20000000L
+#define IH_INT_FLAGS__CLIENT_30_FLAG_MASK                                                                     0x40000000L
+#define IH_INT_FLAGS__CLIENT_31_FLAG_MASK                                                                     0x80000000L
+//IH_LAST_INT_INFO0
+#define IH_LAST_INT_INFO0__CLIENT_ID__SHIFT                                                                   0x0
+#define IH_LAST_INT_INFO0__SOURCE_ID__SHIFT                                                                   0x8
+#define IH_LAST_INT_INFO0__RING_ID__SHIFT                                                                     0x10
+#define IH_LAST_INT_INFO0__VM_ID__SHIFT                                                                       0x18
+#define IH_LAST_INT_INFO0__VMID_TYPE__SHIFT                                                                   0x1f
+#define IH_LAST_INT_INFO0__CLIENT_ID_MASK                                                                     0x000000FFL
+#define IH_LAST_INT_INFO0__SOURCE_ID_MASK                                                                     0x0000FF00L
+#define IH_LAST_INT_INFO0__RING_ID_MASK                                                                       0x00FF0000L
+#define IH_LAST_INT_INFO0__VM_ID_MASK                                                                         0x0F000000L
+#define IH_LAST_INT_INFO0__VMID_TYPE_MASK                                                                     0x80000000L
+//IH_LAST_INT_INFO1
+#define IH_LAST_INT_INFO1__CONTEXT_ID__SHIFT                                                                  0x0
+#define IH_LAST_INT_INFO1__CONTEXT_ID_MASK                                                                    0xFFFFFFFFL
+//IH_LAST_INT_INFO2
+#define IH_LAST_INT_INFO2__PAS_ID__SHIFT                                                                      0x0
+#define IH_LAST_INT_INFO2__VF_ID__SHIFT                                                                       0x10
+#define IH_LAST_INT_INFO2__VF__SHIFT                                                                          0x14
+#define IH_LAST_INT_INFO2__PAS_ID_MASK                                                                        0x0000FFFFL
+#define IH_LAST_INT_INFO2__VF_ID_MASK                                                                         0x00070000L
+#define IH_LAST_INT_INFO2__VF_MASK                                                                            0x00100000L
+//IH_SCRATCH
+#define IH_SCRATCH__DATA__SHIFT                                                                               0x0
+#define IH_SCRATCH__DATA_MASK                                                                                 0xFFFFFFFFL
+//IH_CLIENT_CREDIT_ERROR
+#define IH_CLIENT_CREDIT_ERROR__CLEAR__SHIFT                                                                  0x0
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_1_ERROR__SHIFT                                                         0x1
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_2_ERROR__SHIFT                                                         0x2
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_3_ERROR__SHIFT                                                         0x3
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_4_ERROR__SHIFT                                                         0x4
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_5_ERROR__SHIFT                                                         0x5
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_6_ERROR__SHIFT                                                         0x6
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_7_ERROR__SHIFT                                                         0x7
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_8_ERROR__SHIFT                                                         0x8
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_9_ERROR__SHIFT                                                         0x9
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_10_ERROR__SHIFT                                                        0xa
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_11_ERROR__SHIFT                                                        0xb
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_12_ERROR__SHIFT                                                        0xc
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_13_ERROR__SHIFT                                                        0xd
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_14_ERROR__SHIFT                                                        0xe
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_15_ERROR__SHIFT                                                        0xf
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_16_ERROR__SHIFT                                                        0x10
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_17_ERROR__SHIFT                                                        0x11
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_18_ERROR__SHIFT                                                        0x12
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_19_ERROR__SHIFT                                                        0x13
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_20_ERROR__SHIFT                                                        0x14
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_21_ERROR__SHIFT                                                        0x15
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_22_ERROR__SHIFT                                                        0x16
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_23_ERROR__SHIFT                                                        0x17
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_24_ERROR__SHIFT                                                        0x18
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_25_ERROR__SHIFT                                                        0x19
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_26_ERROR__SHIFT                                                        0x1a
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_27_ERROR__SHIFT                                                        0x1b
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_28_ERROR__SHIFT                                                        0x1c
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_29_ERROR__SHIFT                                                        0x1d
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_30_ERROR__SHIFT                                                        0x1e
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_31_ERROR__SHIFT                                                        0x1f
+#define IH_CLIENT_CREDIT_ERROR__CLEAR_MASK                                                                    0x00000001L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_1_ERROR_MASK                                                           0x00000002L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_2_ERROR_MASK                                                           0x00000004L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_3_ERROR_MASK                                                           0x00000008L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_4_ERROR_MASK                                                           0x00000010L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_5_ERROR_MASK                                                           0x00000020L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_6_ERROR_MASK                                                           0x00000040L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_7_ERROR_MASK                                                           0x00000080L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_8_ERROR_MASK                                                           0x00000100L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_9_ERROR_MASK                                                           0x00000200L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_10_ERROR_MASK                                                          0x00000400L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_11_ERROR_MASK                                                          0x00000800L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_12_ERROR_MASK                                                          0x00001000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_13_ERROR_MASK                                                          0x00002000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_14_ERROR_MASK                                                          0x00004000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_15_ERROR_MASK                                                          0x00008000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_16_ERROR_MASK                                                          0x00010000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_17_ERROR_MASK                                                          0x00020000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_18_ERROR_MASK                                                          0x00040000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_19_ERROR_MASK                                                          0x00080000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_20_ERROR_MASK                                                          0x00100000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_21_ERROR_MASK                                                          0x00200000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_22_ERROR_MASK                                                          0x00400000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_23_ERROR_MASK                                                          0x00800000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_24_ERROR_MASK                                                          0x01000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_25_ERROR_MASK                                                          0x02000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_26_ERROR_MASK                                                          0x04000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_27_ERROR_MASK                                                          0x08000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_28_ERROR_MASK                                                          0x10000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_29_ERROR_MASK                                                          0x20000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_30_ERROR_MASK                                                          0x40000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_31_ERROR_MASK                                                          0x80000000L
+//IH_GPU_IOV_VIOLATION_LOG
+#define IH_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                     0x0
+#define IH_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                            0x1
+#define IH_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                              0x2
+#define IH_GPU_IOV_VIOLATION_LOG__OPCODE__SHIFT                                                               0x12
+#define IH_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                                   0x13
+#define IH_GPU_IOV_VIOLATION_LOG__VF_ID__SHIFT                                                                0x14
+#define IH_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                       0x00000001L
+#define IH_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                              0x00000002L
+#define IH_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                                0x0003FFFCL
+#define IH_GPU_IOV_VIOLATION_LOG__OPCODE_MASK                                                                 0x00040000L
+#define IH_GPU_IOV_VIOLATION_LOG__VF_MASK                                                                     0x00080000L
+#define IH_GPU_IOV_VIOLATION_LOG__VF_ID_MASK                                                                  0x00700000L
+//IH_GPU_IOV_VIOLATION_LOG2
+#define IH_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID__SHIFT                                                        0x0
+#define IH_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID_MASK                                                          0x000003FFL
+//IH_COOKIE_REC_VIOLATION_LOG
+#define IH_COOKIE_REC_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                  0x0
+#define IH_COOKIE_REC_VIOLATION_LOG__CLIENT_ID__SHIFT                                                         0x8
+#define IH_COOKIE_REC_VIOLATION_LOG__INITIATOR_ID__SHIFT                                                      0x10
+#define IH_COOKIE_REC_VIOLATION_LOG__DIE_ID__SHIFT                                                            0x1a
+#define IH_COOKIE_REC_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                    0x00000001L
+#define IH_COOKIE_REC_VIOLATION_LOG__CLIENT_ID_MASK                                                           0x0000FF00L
+#define IH_COOKIE_REC_VIOLATION_LOG__INITIATOR_ID_MASK                                                        0x03FF0000L
+#define IH_COOKIE_REC_VIOLATION_LOG__DIE_ID_MASK                                                              0x3C000000L
+//IH_CREDIT_STATUS
+#define IH_CREDIT_STATUS__CLIENT_1_CREDIT_RETURNED__SHIFT                                                     0x1
+#define IH_CREDIT_STATUS__CLIENT_2_CREDIT_RETURNED__SHIFT                                                     0x2
+#define IH_CREDIT_STATUS__CLIENT_3_CREDIT_RETURNED__SHIFT                                                     0x3
+#define IH_CREDIT_STATUS__CLIENT_4_CREDIT_RETURNED__SHIFT                                                     0x4
+#define IH_CREDIT_STATUS__CLIENT_5_CREDIT_RETURNED__SHIFT                                                     0x5
+#define IH_CREDIT_STATUS__CLIENT_6_CREDIT_RETURNED__SHIFT                                                     0x6
+#define IH_CREDIT_STATUS__CLIENT_7_CREDIT_RETURNED__SHIFT                                                     0x7
+#define IH_CREDIT_STATUS__CLIENT_8_CREDIT_RETURNED__SHIFT                                                     0x8
+#define IH_CREDIT_STATUS__CLIENT_9_CREDIT_RETURNED__SHIFT                                                     0x9
+#define IH_CREDIT_STATUS__CLIENT_10_CREDIT_RETURNED__SHIFT                                                    0xa
+#define IH_CREDIT_STATUS__CLIENT_11_CREDIT_RETURNED__SHIFT                                                    0xb
+#define IH_CREDIT_STATUS__CLIENT_12_CREDIT_RETURNED__SHIFT                                                    0xc
+#define IH_CREDIT_STATUS__CLIENT_13_CREDIT_RETURNED__SHIFT                                                    0xd
+#define IH_CREDIT_STATUS__CLIENT_14_CREDIT_RETURNED__SHIFT                                                    0xe
+#define IH_CREDIT_STATUS__CLIENT_15_CREDIT_RETURNED__SHIFT                                                    0xf
+#define IH_CREDIT_STATUS__CLIENT_16_CREDIT_RETURNED__SHIFT                                                    0x10
+#define IH_CREDIT_STATUS__CLIENT_17_CREDIT_RETURNED__SHIFT                                                    0x11
+#define IH_CREDIT_STATUS__CLIENT_18_CREDIT_RETURNED__SHIFT                                                    0x12
+#define IH_CREDIT_STATUS__CLIENT_19_CREDIT_RETURNED__SHIFT                                                    0x13
+#define IH_CREDIT_STATUS__CLIENT_20_CREDIT_RETURNED__SHIFT                                                    0x14
+#define IH_CREDIT_STATUS__CLIENT_21_CREDIT_RETURNED__SHIFT                                                    0x15
+#define IH_CREDIT_STATUS__CLIENT_22_CREDIT_RETURNED__SHIFT                                                    0x16
+#define IH_CREDIT_STATUS__CLIENT_23_CREDIT_RETURNED__SHIFT                                                    0x17
+#define IH_CREDIT_STATUS__CLIENT_24_CREDIT_RETURNED__SHIFT                                                    0x18
+#define IH_CREDIT_STATUS__CLIENT_25_CREDIT_RETURNED__SHIFT                                                    0x19
+#define IH_CREDIT_STATUS__CLIENT_26_CREDIT_RETURNED__SHIFT                                                    0x1a
+#define IH_CREDIT_STATUS__CLIENT_27_CREDIT_RETURNED__SHIFT                                                    0x1b
+#define IH_CREDIT_STATUS__CLIENT_28_CREDIT_RETURNED__SHIFT                                                    0x1c
+#define IH_CREDIT_STATUS__CLIENT_29_CREDIT_RETURNED__SHIFT                                                    0x1d
+#define IH_CREDIT_STATUS__CLIENT_30_CREDIT_RETURNED__SHIFT                                                    0x1e
+#define IH_CREDIT_STATUS__CLIENT_31_CREDIT_RETURNED__SHIFT                                                    0x1f
+#define IH_CREDIT_STATUS__CLIENT_1_CREDIT_RETURNED_MASK                                                       0x00000002L
+#define IH_CREDIT_STATUS__CLIENT_2_CREDIT_RETURNED_MASK                                                       0x00000004L
+#define IH_CREDIT_STATUS__CLIENT_3_CREDIT_RETURNED_MASK                                                       0x00000008L
+#define IH_CREDIT_STATUS__CLIENT_4_CREDIT_RETURNED_MASK                                                       0x00000010L
+#define IH_CREDIT_STATUS__CLIENT_5_CREDIT_RETURNED_MASK                                                       0x00000020L
+#define IH_CREDIT_STATUS__CLIENT_6_CREDIT_RETURNED_MASK                                                       0x00000040L
+#define IH_CREDIT_STATUS__CLIENT_7_CREDIT_RETURNED_MASK                                                       0x00000080L
+#define IH_CREDIT_STATUS__CLIENT_8_CREDIT_RETURNED_MASK                                                       0x00000100L
+#define IH_CREDIT_STATUS__CLIENT_9_CREDIT_RETURNED_MASK                                                       0x00000200L
+#define IH_CREDIT_STATUS__CLIENT_10_CREDIT_RETURNED_MASK                                                      0x00000400L
+#define IH_CREDIT_STATUS__CLIENT_11_CREDIT_RETURNED_MASK                                                      0x00000800L
+#define IH_CREDIT_STATUS__CLIENT_12_CREDIT_RETURNED_MASK                                                      0x00001000L
+#define IH_CREDIT_STATUS__CLIENT_13_CREDIT_RETURNED_MASK                                                      0x00002000L
+#define IH_CREDIT_STATUS__CLIENT_14_CREDIT_RETURNED_MASK                                                      0x00004000L
+#define IH_CREDIT_STATUS__CLIENT_15_CREDIT_RETURNED_MASK                                                      0x00008000L
+#define IH_CREDIT_STATUS__CLIENT_16_CREDIT_RETURNED_MASK                                                      0x00010000L
+#define IH_CREDIT_STATUS__CLIENT_17_CREDIT_RETURNED_MASK                                                      0x00020000L
+#define IH_CREDIT_STATUS__CLIENT_18_CREDIT_RETURNED_MASK                                                      0x00040000L
+#define IH_CREDIT_STATUS__CLIENT_19_CREDIT_RETURNED_MASK                                                      0x00080000L
+#define IH_CREDIT_STATUS__CLIENT_20_CREDIT_RETURNED_MASK                                                      0x00100000L
+#define IH_CREDIT_STATUS__CLIENT_21_CREDIT_RETURNED_MASK                                                      0x00200000L
+#define IH_CREDIT_STATUS__CLIENT_22_CREDIT_RETURNED_MASK                                                      0x00400000L
+#define IH_CREDIT_STATUS__CLIENT_23_CREDIT_RETURNED_MASK                                                      0x00800000L
+#define IH_CREDIT_STATUS__CLIENT_24_CREDIT_RETURNED_MASK                                                      0x01000000L
+#define IH_CREDIT_STATUS__CLIENT_25_CREDIT_RETURNED_MASK                                                      0x02000000L
+#define IH_CREDIT_STATUS__CLIENT_26_CREDIT_RETURNED_MASK                                                      0x04000000L
+#define IH_CREDIT_STATUS__CLIENT_27_CREDIT_RETURNED_MASK                                                      0x08000000L
+#define IH_CREDIT_STATUS__CLIENT_28_CREDIT_RETURNED_MASK                                                      0x10000000L
+#define IH_CREDIT_STATUS__CLIENT_29_CREDIT_RETURNED_MASK                                                      0x20000000L
+#define IH_CREDIT_STATUS__CLIENT_30_CREDIT_RETURNED_MASK                                                      0x40000000L
+#define IH_CREDIT_STATUS__CLIENT_31_CREDIT_RETURNED_MASK                                                      0x80000000L
+//IH_MMHUB_ERROR
+#define IH_MMHUB_ERROR__IH_BRESP_01__SHIFT                                                                    0x1
+#define IH_MMHUB_ERROR__IH_BRESP_10__SHIFT                                                                    0x2
+#define IH_MMHUB_ERROR__IH_BRESP_11__SHIFT                                                                    0x3
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_01__SHIFT                                                               0x5
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_10__SHIFT                                                               0x6
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_11__SHIFT                                                               0x7
+#define IH_MMHUB_ERROR__IH_BUSER_FED__SHIFT                                                                   0x8
+#define IH_MMHUB_ERROR__IH_BRESP_01_MASK                                                                      0x00000002L
+#define IH_MMHUB_ERROR__IH_BRESP_10_MASK                                                                      0x00000004L
+#define IH_MMHUB_ERROR__IH_BRESP_11_MASK                                                                      0x00000008L
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_01_MASK                                                                 0x00000020L
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_10_MASK                                                                 0x00000040L
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_11_MASK                                                                 0x00000080L
+#define IH_MMHUB_ERROR__IH_BUSER_FED_MASK                                                                     0x00000100L
+//IH_MEM_POWER_CTRL
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_CTRL_EN__SHIFT                                                 0x0
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_LS_EN__SHIFT                                                   0x1
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DS_EN__SHIFT                                                   0x2
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_SD_EN__SHIFT                                                   0x3
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_IDLE_HYSTERESIS__SHIFT                                               0x4
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_UP_RECOVER_DELAY__SHIFT                                        0x8
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DOWN_LS_ENTER_DELAY__SHIFT                                     0xe
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_CTRL_EN__SHIFT                                          0x10
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_LS_EN__SHIFT                                            0x11
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_DS_EN__SHIFT                                            0x12
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_SD_EN__SHIFT                                            0x13
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_IDLE_HYSTERESIS__SHIFT                                        0x14
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_UP_RECOVER_DELAY__SHIFT                                 0x18
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_DOWN_LS_ENTER_DELAY__SHIFT                              0x1e
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_CTRL_EN_MASK                                                   0x00000001L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_LS_EN_MASK                                                     0x00000002L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DS_EN_MASK                                                     0x00000004L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_SD_EN_MASK                                                     0x00000008L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_IDLE_HYSTERESIS_MASK                                                 0x00000070L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_UP_RECOVER_DELAY_MASK                                          0x00003F00L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DOWN_LS_ENTER_DELAY_MASK                                       0x0000C000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_CTRL_EN_MASK                                            0x00010000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_LS_EN_MASK                                              0x00020000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_DS_EN_MASK                                              0x00040000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_SD_EN_MASK                                              0x00080000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_IDLE_HYSTERESIS_MASK                                          0x00700000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_UP_RECOVER_DELAY_MASK                                   0x3F000000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_DOWN_LS_ENTER_DELAY_MASK                                0xC0000000L
+//IH_RETRY_INT_CAM_CNTL
+#define IH_RETRY_INT_CAM_CNTL__CAM_SIZE__SHIFT                                                                0x0
+#define IH_RETRY_INT_CAM_CNTL__BACK_PRESSURE_SKID_VALUE__SHIFT                                                0x8
+#define IH_RETRY_INT_CAM_CNTL__ENABLE__SHIFT                                                                  0x10
+#define IH_RETRY_INT_CAM_CNTL__BACK_PRESSURE_ENABLE__SHIFT                                                    0x11
+#define IH_RETRY_INT_CAM_CNTL__PER_VF_ENTRY_SIZE__SHIFT                                                       0x14
+#define IH_RETRY_INT_CAM_CNTL__CAM_SIZE_MASK                                                                  0x0000001FL
+#define IH_RETRY_INT_CAM_CNTL__BACK_PRESSURE_SKID_VALUE_MASK                                                  0x00007F00L
+#define IH_RETRY_INT_CAM_CNTL__ENABLE_MASK                                                                    0x00010000L
+#define IH_RETRY_INT_CAM_CNTL__BACK_PRESSURE_ENABLE_MASK                                                      0x00020000L
+#define IH_RETRY_INT_CAM_CNTL__PER_VF_ENTRY_SIZE_MASK                                                         0x00300000L
+//IH_VMID_LUT_INDEX
+#define IH_VMID_LUT_INDEX__INDEX__SHIFT                                                                       0x0
+#define IH_VMID_LUT_INDEX__INDEX_MASK                                                                         0x0000000FL
+//IH_MEM_POWER_CTRL2
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_CTRL_EN__SHIFT                                             0x0
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_LS_EN__SHIFT                                               0x1
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_DS_EN__SHIFT                                               0x2
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_SD_EN__SHIFT                                               0x3
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_IDLE_HYSTERESIS__SHIFT                                           0x4
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_UP_RECOVER_DELAY__SHIFT                                    0x8
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_DOWN_LS_ENTER_DELAY__SHIFT                                 0xe
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_CTRL_EN_MASK                                               0x00000001L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_LS_EN_MASK                                                 0x00000002L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_DS_EN_MASK                                                 0x00000004L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_SD_EN_MASK                                                 0x00000008L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_IDLE_HYSTERESIS_MASK                                             0x00000070L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_UP_RECOVER_DELAY_MASK                                      0x00003F00L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_DOWN_LS_ENTER_DELAY_MASK                                   0x0000C000L
+//IH_REGISTER_LAST_PART2
+#define IH_REGISTER_LAST_PART2__RESERVED__SHIFT                                                               0x0
+#define IH_REGISTER_LAST_PART2__RESERVED_MASK                                                                 0xFFFFFFFFL
+//SEM_MAILBOX
+#define SEM_MAILBOX__HOSTPORT__SHIFT                                                                          0x0
+#define SEM_MAILBOX__RESERVED__SHIFT                                                                          0x10
+#define SEM_MAILBOX__HOSTPORT_MASK                                                                            0x0000FFFFL
+#define SEM_MAILBOX__RESERVED_MASK                                                                            0xFFFF0000L
+//SEM_MAILBOX_CLEAR
+#define SEM_MAILBOX_CLEAR__CLEAR__SHIFT                                                                       0x0
+#define SEM_MAILBOX_CLEAR__RESERVED__SHIFT                                                                    0x10
+#define SEM_MAILBOX_CLEAR__CLEAR_MASK                                                                         0x0000FFFFL
+#define SEM_MAILBOX_CLEAR__RESERVED_MASK                                                                      0xFFFF0000L
+//SEM_REGISTER_LAST_PART2
+#define SEM_REGISTER_LAST_PART2__RESERVED__SHIFT                                                              0x0
+#define SEM_REGISTER_LAST_PART2__RESERVED_MASK                                                                0xFFFFFFFFL
+//IH_ACTIVE_FCN_ID
+#define IH_ACTIVE_FCN_ID__VF_ID__SHIFT                                                                        0x0
+#define IH_ACTIVE_FCN_ID__RESERVED__SHIFT                                                                     0x3
+#define IH_ACTIVE_FCN_ID__PF_VF__SHIFT                                                                        0x1f
+#define IH_ACTIVE_FCN_ID__VF_ID_MASK                                                                          0x00000007L
+#define IH_ACTIVE_FCN_ID__RESERVED_MASK                                                                       0x7FFFFFF8L
+#define IH_ACTIVE_FCN_ID__PF_VF_MASK                                                                          0x80000000L
+//IH_VIRT_RESET_REQ
+#define IH_VIRT_RESET_REQ__VF__SHIFT                                                                          0x0
+#define IH_VIRT_RESET_REQ__PF__SHIFT                                                                          0x1f
+#define IH_VIRT_RESET_REQ__VF_MASK                                                                            0x000000FFL
+#define IH_VIRT_RESET_REQ__PF_MASK                                                                            0x80000000L
+//IH_CLIENT_CFG
+#define IH_CLIENT_CFG__TOTAL_CLIENT_NUM__SHIFT                                                                0x0
+#define IH_CLIENT_CFG__TOTAL_CLIENT_NUM_MASK                                                                  0x0000007FL
+//IH_CLIENT_CFG_INDEX
+#define IH_CLIENT_CFG_INDEX__INDEX__SHIFT                                                                     0x0
+#define IH_CLIENT_CFG_INDEX__INDEX_MASK                                                                       0x0000001FL
+//IH_CLIENT_CFG_DATA
+#define IH_CLIENT_CFG_DATA__CLIENT_TYPE__SHIFT                                                                0x13
+#define IH_CLIENT_CFG_DATA__RING_ID__SHIFT                                                                    0x14
+#define IH_CLIENT_CFG_DATA__VF_RB_SELECT__SHIFT                                                               0x16
+#define IH_CLIENT_CFG_DATA__OVERWRITE_RING_ID_WITH_ACTIVE_FCN_ID__SHIFT                                       0x18
+#define IH_CLIENT_CFG_DATA__INTERFACE_TYPE__SHIFT                                                             0x19
+#define IH_CLIENT_CFG_DATA__DIE_TYPE__SHIFT                                                                   0x1a
+#define IH_CLIENT_CFG_DATA__CLIENT_TYPE_MASK                                                                  0x00080000L
+#define IH_CLIENT_CFG_DATA__RING_ID_MASK                                                                      0x00300000L
+#define IH_CLIENT_CFG_DATA__VF_RB_SELECT_MASK                                                                 0x00C00000L
+#define IH_CLIENT_CFG_DATA__OVERWRITE_RING_ID_WITH_ACTIVE_FCN_ID_MASK                                         0x01000000L
+#define IH_CLIENT_CFG_DATA__INTERFACE_TYPE_MASK                                                               0x02000000L
+#define IH_CLIENT_CFG_DATA__DIE_TYPE_MASK                                                                     0x04000000L
+//IH_CLIENT_CFG_DATA2
+#define IH_CLIENT_CFG_DATA2__CREDIT_RETURN_ADDR__SHIFT                                                        0x0
+#define IH_CLIENT_CFG_DATA2__CREDIT_RETURN_ADDR_MASK                                                          0xFFFFFFFFL
+//IH_CID_REMAP_INDEX
+#define IH_CID_REMAP_INDEX__INDEX__SHIFT                                                                      0x0
+#define IH_CID_REMAP_INDEX__INDEX_MASK                                                                        0x00000007L
+//IH_CID_REMAP_DATA
+#define IH_CID_REMAP_DATA__CLIENT_ID__SHIFT                                                                   0x0
+#define IH_CID_REMAP_DATA__INITIATOR_ID__SHIFT                                                                0x8
+#define IH_CID_REMAP_DATA__CLIENT_ID_REMAP__SHIFT                                                             0x18
+#define IH_CID_REMAP_DATA__CLIENT_ID_MASK                                                                     0x000000FFL
+#define IH_CID_REMAP_DATA__INITIATOR_ID_MASK                                                                  0x0003FF00L
+#define IH_CID_REMAP_DATA__CLIENT_ID_REMAP_MASK                                                               0xFF000000L
+//IH_CHICKEN
+#define IH_CHICKEN__ACTIVE_FCN_ID_PROT_ENABLE__SHIFT                                                          0x0
+#define IH_CHICKEN__MC_SPACE_FBPA_ENABLE__SHIFT                                                               0x3
+#define IH_CHICKEN__MC_SPACE_GPA_ENABLE__SHIFT                                                                0x4
+#define IH_CHICKEN__REG_FIREWALL_ENABLE__SHIFT                                                                0x5
+#define IH_CHICKEN__ACTIVE_FCN_ID_PROT_ENABLE_MASK                                                            0x00000001L
+#define IH_CHICKEN__MC_SPACE_FBPA_ENABLE_MASK                                                                 0x00000008L
+#define IH_CHICKEN__MC_SPACE_GPA_ENABLE_MASK                                                                  0x00000010L
+#define IH_CHICKEN__REG_FIREWALL_ENABLE_MASK                                                                  0x00000020L
+//IH_INT_DROP_CNTL
+#define IH_INT_DROP_CNTL__INT_DROP_EN__SHIFT                                                                  0x0
+#define IH_INT_DROP_CNTL__CLIENT_ID_MATCH_EN__SHIFT                                                           0x1
+#define IH_INT_DROP_CNTL__SOURCE_ID_MATCH_EN__SHIFT                                                           0x2
+#define IH_INT_DROP_CNTL__VF_ID_MATCH_EN__SHIFT                                                               0x3
+#define IH_INT_DROP_CNTL__VF_MATCH_EN__SHIFT                                                                  0x4
+#define IH_INT_DROP_CNTL__CONTEXT_ID_MATCH_EN__SHIFT                                                          0x5
+#define IH_INT_DROP_CNTL__INT_DROP_MODE__SHIFT                                                                0x6
+#define IH_INT_DROP_CNTL__UTCL2_RETRY_INT_DROP_EN__SHIFT                                                      0x8
+#define IH_INT_DROP_CNTL__INT_DROPPED__SHIFT                                                                  0x10
+#define IH_INT_DROP_CNTL__INT_DROP_EN_MASK                                                                    0x00000001L
+#define IH_INT_DROP_CNTL__CLIENT_ID_MATCH_EN_MASK                                                             0x00000002L
+#define IH_INT_DROP_CNTL__SOURCE_ID_MATCH_EN_MASK                                                             0x00000004L
+#define IH_INT_DROP_CNTL__VF_ID_MATCH_EN_MASK                                                                 0x00000008L
+#define IH_INT_DROP_CNTL__VF_MATCH_EN_MASK                                                                    0x00000010L
+#define IH_INT_DROP_CNTL__CONTEXT_ID_MATCH_EN_MASK                                                            0x00000020L
+#define IH_INT_DROP_CNTL__INT_DROP_MODE_MASK                                                                  0x000000C0L
+#define IH_INT_DROP_CNTL__UTCL2_RETRY_INT_DROP_EN_MASK                                                        0x00000100L
+#define IH_INT_DROP_CNTL__INT_DROPPED_MASK                                                                    0x00010000L
+//IH_INT_DROP_MATCH_VALUE0
+#define IH_INT_DROP_MATCH_VALUE0__CLIENT_ID_MATCH_VALUE__SHIFT                                                0x0
+#define IH_INT_DROP_MATCH_VALUE0__SOURCE_ID_MATCH_VALUE__SHIFT                                                0x8
+#define IH_INT_DROP_MATCH_VALUE0__VF_ID_MATCH_VALUE__SHIFT                                                    0x10
+#define IH_INT_DROP_MATCH_VALUE0__VF_MATCH_VALUE__SHIFT                                                       0x17
+#define IH_INT_DROP_MATCH_VALUE0__CONTEXT_ID_39_32_MATCH_VALUE__SHIFT                                         0x18
+#define IH_INT_DROP_MATCH_VALUE0__CLIENT_ID_MATCH_VALUE_MASK                                                  0x000000FFL
+#define IH_INT_DROP_MATCH_VALUE0__SOURCE_ID_MATCH_VALUE_MASK                                                  0x0000FF00L
+#define IH_INT_DROP_MATCH_VALUE0__VF_ID_MATCH_VALUE_MASK                                                      0x000F0000L
+#define IH_INT_DROP_MATCH_VALUE0__VF_MATCH_VALUE_MASK                                                         0x00800000L
+#define IH_INT_DROP_MATCH_VALUE0__CONTEXT_ID_39_32_MATCH_VALUE_MASK                                           0xFF000000L
+//IH_INT_DROP_MATCH_VALUE1
+#define IH_INT_DROP_MATCH_VALUE1__CONTEXT_ID_31_0_MATCH_VALUE__SHIFT                                          0x0
+#define IH_INT_DROP_MATCH_VALUE1__CONTEXT_ID_31_0_MATCH_VALUE_MASK                                            0xFFFFFFFFL
+//IH_INT_DROP_MATCH_MASK0
+#define IH_INT_DROP_MATCH_MASK0__CLIENT_ID_MATCH_MASK__SHIFT                                                  0x0
+#define IH_INT_DROP_MATCH_MASK0__SOURCE_ID_MATCH_MASK__SHIFT                                                  0x8
+#define IH_INT_DROP_MATCH_MASK0__VF_ID_MATCH_MASK__SHIFT                                                      0x10
+#define IH_INT_DROP_MATCH_MASK0__VF_MATCH_MASK__SHIFT                                                         0x17
+#define IH_INT_DROP_MATCH_MASK0__CONTEXT_ID_39_32_MATCH_MASK__SHIFT                                           0x18
+#define IH_INT_DROP_MATCH_MASK0__CLIENT_ID_MATCH_MASK_MASK                                                    0x000000FFL
+#define IH_INT_DROP_MATCH_MASK0__SOURCE_ID_MATCH_MASK_MASK                                                    0x0000FF00L
+#define IH_INT_DROP_MATCH_MASK0__VF_ID_MATCH_MASK_MASK                                                        0x000F0000L
+#define IH_INT_DROP_MATCH_MASK0__VF_MATCH_MASK_MASK                                                           0x00800000L
+#define IH_INT_DROP_MATCH_MASK0__CONTEXT_ID_39_32_MATCH_MASK_MASK                                             0xFF000000L
+//IH_INT_DROP_MATCH_MASK1
+#define IH_INT_DROP_MATCH_MASK1__CONTEXT_ID_31_0_MATCH_MASK__SHIFT                                            0x0
+#define IH_INT_DROP_MATCH_MASK1__CONTEXT_ID_31_0_MATCH_MASK_MASK                                              0xFFFFFFFFL
+//IH_MMHUB_CNTL
+#define IH_MMHUB_CNTL__UNITID__SHIFT                                                                          0x0
+#define IH_MMHUB_CNTL__IV_TLVL__SHIFT                                                                         0x8
+#define IH_MMHUB_CNTL__WPTR_WB_TLVL__SHIFT                                                                    0xc
+#define IH_MMHUB_CNTL__UNITID_MASK                                                                            0x0000003FL
+#define IH_MMHUB_CNTL__IV_TLVL_MASK                                                                           0x00000F00L
+#define IH_MMHUB_CNTL__WPTR_WB_TLVL_MASK                                                                      0x0000F000L
+//IH_REGISTER_LAST_PART1
+#define IH_REGISTER_LAST_PART1__RESERVED__SHIFT                                                               0x0
+#define IH_REGISTER_LAST_PART1__RESERVED_MASK                                                                 0xFFFFFFFFL
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/sdma/sdma_4_4_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/sdma/sdma_4_4_2_offset.h
new file mode 100644
index 000000000000..31bef0776ded
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/sdma/sdma_4_4_2_offset.h
@@ -0,0 +1,1109 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _sdma_4_4_2_OFFSET_HEADER
+#define _sdma_4_4_2_OFFSET_HEADER
+
+
+
+// addressBlock: aid_sdma_insts_sdma0_sdmadec
+// base address: 0x4980
+#define regSDMA_UCODE_ADDR                                                                              0x0000
+#define regSDMA_UCODE_ADDR_BASE_IDX                                                                     0
+#define regSDMA_UCODE_DATA                                                                              0x0001
+#define regSDMA_UCODE_DATA_BASE_IDX                                                                     0
+#define regSDMA_F32_CNTL                                                                                0x0002
+#define regSDMA_F32_CNTL_BASE_IDX                                                                       0
+#define regSDMA_MMHUB_CNTL                                                                              0x0005
+#define regSDMA_MMHUB_CNTL_BASE_IDX                                                                     0
+#define regSDMA_MMHUB_TRUSTLVL                                                                          0x0006
+#define regSDMA_MMHUB_TRUSTLVL_BASE_IDX                                                                 0
+#define regSDMA_VM_CNTL                                                                                 0x0010
+#define regSDMA_VM_CNTL_BASE_IDX                                                                        0
+#define regSDMA_VM_CTX_LO                                                                               0x0011
+#define regSDMA_VM_CTX_LO_BASE_IDX                                                                      0
+#define regSDMA_VM_CTX_HI                                                                               0x0012
+#define regSDMA_VM_CTX_HI_BASE_IDX                                                                      0
+#define regSDMA_ACTIVE_FCN_ID                                                                           0x0013
+#define regSDMA_ACTIVE_FCN_ID_BASE_IDX                                                                  0
+#define regSDMA_VM_CTX_CNTL                                                                             0x0014
+#define regSDMA_VM_CTX_CNTL_BASE_IDX                                                                    0
+#define regSDMA_VIRT_RESET_REQ                                                                          0x0015
+#define regSDMA_VIRT_RESET_REQ_BASE_IDX                                                                 0
+#define regSDMA_VF_ENABLE                                                                               0x0016
+#define regSDMA_VF_ENABLE_BASE_IDX                                                                      0
+#define regSDMA_CONTEXT_REG_TYPE0                                                                       0x0017
+#define regSDMA_CONTEXT_REG_TYPE0_BASE_IDX                                                              0
+#define regSDMA_CONTEXT_REG_TYPE1                                                                       0x0018
+#define regSDMA_CONTEXT_REG_TYPE1_BASE_IDX                                                              0
+#define regSDMA_CONTEXT_REG_TYPE2                                                                       0x0019
+#define regSDMA_CONTEXT_REG_TYPE2_BASE_IDX                                                              0
+#define regSDMA_CONTEXT_REG_TYPE3                                                                       0x001a
+#define regSDMA_CONTEXT_REG_TYPE3_BASE_IDX                                                              0
+#define regSDMA_PUB_REG_TYPE0                                                                           0x001b
+#define regSDMA_PUB_REG_TYPE0_BASE_IDX                                                                  0
+#define regSDMA_PUB_REG_TYPE1                                                                           0x001c
+#define regSDMA_PUB_REG_TYPE1_BASE_IDX                                                                  0
+#define regSDMA_PUB_REG_TYPE2                                                                           0x001d
+#define regSDMA_PUB_REG_TYPE2_BASE_IDX                                                                  0
+#define regSDMA_PUB_REG_TYPE3                                                                           0x001e
+#define regSDMA_PUB_REG_TYPE3_BASE_IDX                                                                  0
+#define regSDMA_CONTEXT_GROUP_BOUNDARY                                                                  0x001f
+#define regSDMA_CONTEXT_GROUP_BOUNDARY_BASE_IDX                                                         0
+#define regSDMA_RB_RPTR_FETCH_HI                                                                        0x0020
+#define regSDMA_RB_RPTR_FETCH_HI_BASE_IDX                                                               0
+#define regSDMA_SEM_WAIT_FAIL_TIMER_CNTL                                                                0x0021
+#define regSDMA_SEM_WAIT_FAIL_TIMER_CNTL_BASE_IDX                                                       0
+#define regSDMA_RB_RPTR_FETCH                                                                           0x0022
+#define regSDMA_RB_RPTR_FETCH_BASE_IDX                                                                  0
+#define regSDMA_IB_OFFSET_FETCH                                                                         0x0023
+#define regSDMA_IB_OFFSET_FETCH_BASE_IDX                                                                0
+#define regSDMA_PROGRAM                                                                                 0x0024
+#define regSDMA_PROGRAM_BASE_IDX                                                                        0
+#define regSDMA_STATUS_REG                                                                              0x0025
+#define regSDMA_STATUS_REG_BASE_IDX                                                                     0
+#define regSDMA_STATUS1_REG                                                                             0x0026
+#define regSDMA_STATUS1_REG_BASE_IDX                                                                    0
+#define regSDMA_RD_BURST_CNTL                                                                           0x0027
+#define regSDMA_RD_BURST_CNTL_BASE_IDX                                                                  0
+#define regSDMA_HBM_PAGE_CONFIG                                                                         0x0028
+#define regSDMA_HBM_PAGE_CONFIG_BASE_IDX                                                                0
+#define regSDMA_UCODE_CHECKSUM                                                                          0x0029
+#define regSDMA_UCODE_CHECKSUM_BASE_IDX                                                                 0
+#define regSDMA_FREEZE                                                                                  0x002b
+#define regSDMA_FREEZE_BASE_IDX                                                                         0
+#define regSDMA_PHASE0_QUANTUM                                                                          0x002c
+#define regSDMA_PHASE0_QUANTUM_BASE_IDX                                                                 0
+#define regSDMA_PHASE1_QUANTUM                                                                          0x002d
+#define regSDMA_PHASE1_QUANTUM_BASE_IDX                                                                 0
+#define regSDMA_POWER_GATING                                                                            0x002e
+#define regSDMA_POWER_GATING_BASE_IDX                                                                   0
+#define regSDMA_PGFSM_CONFIG                                                                            0x002f
+#define regSDMA_PGFSM_CONFIG_BASE_IDX                                                                   0
+#define regSDMA_PGFSM_WRITE                                                                             0x0030
+#define regSDMA_PGFSM_WRITE_BASE_IDX                                                                    0
+#define regSDMA_PGFSM_READ                                                                              0x0031
+#define regSDMA_PGFSM_READ_BASE_IDX                                                                     0
+#define regCC_SDMA_EDC_CONFIG                                                                           0x0032
+#define regCC_SDMA_EDC_CONFIG_BASE_IDX                                                                  0
+#define regSDMA_BA_THRESHOLD                                                                            0x0033
+#define regSDMA_BA_THRESHOLD_BASE_IDX                                                                   0
+#define regSDMA_ID                                                                                      0x0034
+#define regSDMA_ID_BASE_IDX                                                                             0
+#define regSDMA_VERSION                                                                                 0x0035
+#define regSDMA_VERSION_BASE_IDX                                                                        0
+#define regSDMA_EDC_COUNTER                                                                             0x0036
+#define regSDMA_EDC_COUNTER_BASE_IDX                                                                    0
+#define regSDMA_EDC_COUNTER2                                                                            0x0037
+#define regSDMA_EDC_COUNTER2_BASE_IDX                                                                   0
+#define regSDMA_STATUS2_REG                                                                             0x0038
+#define regSDMA_STATUS2_REG_BASE_IDX                                                                    0
+#define regSDMA_ATOMIC_CNTL                                                                             0x0039
+#define regSDMA_ATOMIC_CNTL_BASE_IDX                                                                    0
+#define regSDMA_ATOMIC_PREOP_LO                                                                         0x003a
+#define regSDMA_ATOMIC_PREOP_LO_BASE_IDX                                                                0
+#define regSDMA_ATOMIC_PREOP_HI                                                                         0x003b
+#define regSDMA_ATOMIC_PREOP_HI_BASE_IDX                                                                0
+#define regSDMA_UTCL1_CNTL                                                                              0x003c
+#define regSDMA_UTCL1_CNTL_BASE_IDX                                                                     0
+#define regSDMA_UTCL1_WATERMK                                                                           0x003d
+#define regSDMA_UTCL1_WATERMK_BASE_IDX                                                                  0
+#define regSDMA_UTCL1_RD_STATUS                                                                         0x003e
+#define regSDMA_UTCL1_RD_STATUS_BASE_IDX                                                                0
+#define regSDMA_UTCL1_WR_STATUS                                                                         0x003f
+#define regSDMA_UTCL1_WR_STATUS_BASE_IDX                                                                0
+#define regSDMA_UTCL1_INV0                                                                              0x0040
+#define regSDMA_UTCL1_INV0_BASE_IDX                                                                     0
+#define regSDMA_UTCL1_INV1                                                                              0x0041
+#define regSDMA_UTCL1_INV1_BASE_IDX                                                                     0
+#define regSDMA_UTCL1_INV2                                                                              0x0042
+#define regSDMA_UTCL1_INV2_BASE_IDX                                                                     0
+#define regSDMA_UTCL1_RD_XNACK0                                                                         0x0043
+#define regSDMA_UTCL1_RD_XNACK0_BASE_IDX                                                                0
+#define regSDMA_UTCL1_RD_XNACK1                                                                         0x0044
+#define regSDMA_UTCL1_RD_XNACK1_BASE_IDX                                                                0
+#define regSDMA_UTCL1_WR_XNACK0                                                                         0x0045
+#define regSDMA_UTCL1_WR_XNACK0_BASE_IDX                                                                0
+#define regSDMA_UTCL1_WR_XNACK1                                                                         0x0046
+#define regSDMA_UTCL1_WR_XNACK1_BASE_IDX                                                                0
+#define regSDMA_UTCL1_TIMEOUT                                                                           0x0047
+#define regSDMA_UTCL1_TIMEOUT_BASE_IDX                                                                  0
+#define regSDMA_UTCL1_PAGE                                                                              0x0048
+#define regSDMA_UTCL1_PAGE_BASE_IDX                                                                     0
+#define regSDMA_POWER_CNTL_IDLE                                                                         0x0049
+#define regSDMA_POWER_CNTL_IDLE_BASE_IDX                                                                0
+#define regSDMA_RELAX_ORDERING_LUT                                                                      0x004a
+#define regSDMA_RELAX_ORDERING_LUT_BASE_IDX                                                             0
+#define regSDMA_CHICKEN_BITS_2                                                                          0x004b
+#define regSDMA_CHICKEN_BITS_2_BASE_IDX                                                                 0
+#define regSDMA_STATUS3_REG                                                                             0x004c
+#define regSDMA_STATUS3_REG_BASE_IDX                                                                    0
+#define regSDMA_PHYSICAL_ADDR_LO                                                                        0x004d
+#define regSDMA_PHYSICAL_ADDR_LO_BASE_IDX                                                               0
+#define regSDMA_PHYSICAL_ADDR_HI                                                                        0x004e
+#define regSDMA_PHYSICAL_ADDR_HI_BASE_IDX                                                               0
+#define regSDMA_PHASE2_QUANTUM                                                                          0x004f
+#define regSDMA_PHASE2_QUANTUM_BASE_IDX                                                                 0
+#define regSDMA_ERROR_LOG                                                                               0x0050
+#define regSDMA_ERROR_LOG_BASE_IDX                                                                      0
+#define regSDMA_PUB_DUMMY_REG0                                                                          0x0051
+#define regSDMA_PUB_DUMMY_REG0_BASE_IDX                                                                 0
+#define regSDMA_PUB_DUMMY_REG1                                                                          0x0052
+#define regSDMA_PUB_DUMMY_REG1_BASE_IDX                                                                 0
+#define regSDMA_PUB_DUMMY_REG2                                                                          0x0053
+#define regSDMA_PUB_DUMMY_REG2_BASE_IDX                                                                 0
+#define regSDMA_PUB_DUMMY_REG3                                                                          0x0054
+#define regSDMA_PUB_DUMMY_REG3_BASE_IDX                                                                 0
+#define regSDMA_F32_COUNTER                                                                             0x0055
+#define regSDMA_F32_COUNTER_BASE_IDX                                                                    0
+#define regSDMA_PERFCNT_PERFCOUNTER0_CFG                                                                0x0057
+#define regSDMA_PERFCNT_PERFCOUNTER0_CFG_BASE_IDX                                                       0
+#define regSDMA_PERFCNT_PERFCOUNTER1_CFG                                                                0x0058
+#define regSDMA_PERFCNT_PERFCOUNTER1_CFG_BASE_IDX                                                       0
+#define regSDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL                                                           0x0059
+#define regSDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                  0
+#define regSDMA_PERFCNT_MISC_CNTL                                                                       0x005a
+#define regSDMA_PERFCNT_MISC_CNTL_BASE_IDX                                                              0
+#define regSDMA_PERFCNT_PERFCOUNTER_LO                                                                  0x005b
+#define regSDMA_PERFCNT_PERFCOUNTER_LO_BASE_IDX                                                         0
+#define regSDMA_PERFCNT_PERFCOUNTER_HI                                                                  0x005c
+#define regSDMA_PERFCNT_PERFCOUNTER_HI_BASE_IDX                                                         0
+#define regSDMA_CRD_CNTL                                                                                0x005d
+#define regSDMA_CRD_CNTL_BASE_IDX                                                                       0
+#define regSDMA_GPU_IOV_VIOLATION_LOG                                                                   0x005e
+#define regSDMA_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                          0
+#define regSDMA_ULV_CNTL                                                                                0x005f
+#define regSDMA_ULV_CNTL_BASE_IDX                                                                       0
+#define regSDMA_EA_DBIT_ADDR_DATA                                                                       0x0060
+#define regSDMA_EA_DBIT_ADDR_DATA_BASE_IDX                                                              0
+#define regSDMA_EA_DBIT_ADDR_INDEX                                                                      0x0061
+#define regSDMA_EA_DBIT_ADDR_INDEX_BASE_IDX                                                             0
+#define regSDMA_GPU_IOV_VIOLATION_LOG2                                                                  0x0062
+#define regSDMA_GPU_IOV_VIOLATION_LOG2_BASE_IDX                                                         0
+#define regSDMA_STATUS4_REG                                                                             0x0063
+#define regSDMA_STATUS4_REG_BASE_IDX                                                                    0
+#define regSDMA_SCRATCH_RAM_DATA                                                                        0x0064
+#define regSDMA_SCRATCH_RAM_DATA_BASE_IDX                                                               0
+#define regSDMA_SCRATCH_RAM_ADDR                                                                        0x0065
+#define regSDMA_SCRATCH_RAM_ADDR_BASE_IDX                                                               0
+#define regSDMA_CE_CTRL                                                                                 0x0066
+#define regSDMA_CE_CTRL_BASE_IDX                                                                        0
+#define regSDMA_RAS_STATUS                                                                              0x0067
+#define regSDMA_RAS_STATUS_BASE_IDX                                                                     0
+#define regSDMA_CLK_STATUS                                                                              0x0068
+#define regSDMA_CLK_STATUS_BASE_IDX                                                                     0
+#define regSDMA_POWER_CNTL                                                                              0x006b
+#define regSDMA_POWER_CNTL_BASE_IDX                                                                     0
+#define regSDMA_CLK_CTRL                                                                                0x006c
+#define regSDMA_CLK_CTRL_BASE_IDX                                                                       0
+#define regSDMA_CNTL                                                                                    0x006d
+#define regSDMA_CNTL_BASE_IDX                                                                           0
+#define regSDMA_CHICKEN_BITS                                                                            0x006e
+#define regSDMA_CHICKEN_BITS_BASE_IDX                                                                   0
+#define regSDMA_GB_ADDR_CONFIG                                                                          0x006f
+#define regSDMA_GB_ADDR_CONFIG_BASE_IDX                                                                 0
+#define regSDMA_GB_ADDR_CONFIG_READ                                                                     0x0070
+#define regSDMA_GB_ADDR_CONFIG_READ_BASE_IDX                                                            0
+#define regSDMA_GFX_RB_CNTL                                                                             0x0080
+#define regSDMA_GFX_RB_CNTL_BASE_IDX                                                                    0
+#define regSDMA_GFX_RB_BASE                                                                             0x0081
+#define regSDMA_GFX_RB_BASE_BASE_IDX                                                                    0
+#define regSDMA_GFX_RB_BASE_HI                                                                          0x0082
+#define regSDMA_GFX_RB_BASE_HI_BASE_IDX                                                                 0
+#define regSDMA_GFX_RB_RPTR                                                                             0x0083
+#define regSDMA_GFX_RB_RPTR_BASE_IDX                                                                    0
+#define regSDMA_GFX_RB_RPTR_HI                                                                          0x0084
+#define regSDMA_GFX_RB_RPTR_HI_BASE_IDX                                                                 0
+#define regSDMA_GFX_RB_WPTR                                                                             0x0085
+#define regSDMA_GFX_RB_WPTR_BASE_IDX                                                                    0
+#define regSDMA_GFX_RB_WPTR_HI                                                                          0x0086
+#define regSDMA_GFX_RB_WPTR_HI_BASE_IDX                                                                 0
+#define regSDMA_GFX_RB_WPTR_POLL_CNTL                                                                   0x0087
+#define regSDMA_GFX_RB_WPTR_POLL_CNTL_BASE_IDX                                                          0
+#define regSDMA_GFX_RB_RPTR_ADDR_HI                                                                     0x0088
+#define regSDMA_GFX_RB_RPTR_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA_GFX_RB_RPTR_ADDR_LO                                                                     0x0089
+#define regSDMA_GFX_RB_RPTR_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA_GFX_IB_CNTL                                                                             0x008a
+#define regSDMA_GFX_IB_CNTL_BASE_IDX                                                                    0
+#define regSDMA_GFX_IB_RPTR                                                                             0x008b
+#define regSDMA_GFX_IB_RPTR_BASE_IDX                                                                    0
+#define regSDMA_GFX_IB_OFFSET                                                                           0x008c
+#define regSDMA_GFX_IB_OFFSET_BASE_IDX                                                                  0
+#define regSDMA_GFX_IB_BASE_LO                                                                          0x008d
+#define regSDMA_GFX_IB_BASE_LO_BASE_IDX                                                                 0
+#define regSDMA_GFX_IB_BASE_HI                                                                          0x008e
+#define regSDMA_GFX_IB_BASE_HI_BASE_IDX                                                                 0
+#define regSDMA_GFX_IB_SIZE                                                                             0x008f
+#define regSDMA_GFX_IB_SIZE_BASE_IDX                                                                    0
+#define regSDMA_GFX_SKIP_CNTL                                                                           0x0090
+#define regSDMA_GFX_SKIP_CNTL_BASE_IDX                                                                  0
+#define regSDMA_GFX_CONTEXT_STATUS                                                                      0x0091
+#define regSDMA_GFX_CONTEXT_STATUS_BASE_IDX                                                             0
+#define regSDMA_GFX_DOORBELL                                                                            0x0092
+#define regSDMA_GFX_DOORBELL_BASE_IDX                                                                   0
+#define regSDMA_GFX_CONTEXT_CNTL                                                                        0x0093
+#define regSDMA_GFX_CONTEXT_CNTL_BASE_IDX                                                               0
+#define regSDMA_GFX_STATUS                                                                              0x00a8
+#define regSDMA_GFX_STATUS_BASE_IDX                                                                     0
+#define regSDMA_GFX_DOORBELL_LOG                                                                        0x00a9
+#define regSDMA_GFX_DOORBELL_LOG_BASE_IDX                                                               0
+#define regSDMA_GFX_WATERMARK                                                                           0x00aa
+#define regSDMA_GFX_WATERMARK_BASE_IDX                                                                  0
+#define regSDMA_GFX_DOORBELL_OFFSET                                                                     0x00ab
+#define regSDMA_GFX_DOORBELL_OFFSET_BASE_IDX                                                            0
+#define regSDMA_GFX_CSA_ADDR_LO                                                                         0x00ac
+#define regSDMA_GFX_CSA_ADDR_LO_BASE_IDX                                                                0
+#define regSDMA_GFX_CSA_ADDR_HI                                                                         0x00ad
+#define regSDMA_GFX_CSA_ADDR_HI_BASE_IDX                                                                0
+#define regSDMA_GFX_IB_SUB_REMAIN                                                                       0x00af
+#define regSDMA_GFX_IB_SUB_REMAIN_BASE_IDX                                                              0
+#define regSDMA_GFX_PREEMPT                                                                             0x00b0
+#define regSDMA_GFX_PREEMPT_BASE_IDX                                                                    0
+#define regSDMA_GFX_DUMMY_REG                                                                           0x00b1
+#define regSDMA_GFX_DUMMY_REG_BASE_IDX                                                                  0
+#define regSDMA_GFX_RB_WPTR_POLL_ADDR_HI                                                                0x00b2
+#define regSDMA_GFX_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA_GFX_RB_WPTR_POLL_ADDR_LO                                                                0x00b3
+#define regSDMA_GFX_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA_GFX_RB_AQL_CNTL                                                                         0x00b4
+#define regSDMA_GFX_RB_AQL_CNTL_BASE_IDX                                                                0
+#define regSDMA_GFX_MINOR_PTR_UPDATE                                                                    0x00b5
+#define regSDMA_GFX_MINOR_PTR_UPDATE_BASE_IDX                                                           0
+#define regSDMA_GFX_MIDCMD_DATA0                                                                        0x00c0
+#define regSDMA_GFX_MIDCMD_DATA0_BASE_IDX                                                               0
+#define regSDMA_GFX_MIDCMD_DATA1                                                                        0x00c1
+#define regSDMA_GFX_MIDCMD_DATA1_BASE_IDX                                                               0
+#define regSDMA_GFX_MIDCMD_DATA2                                                                        0x00c2
+#define regSDMA_GFX_MIDCMD_DATA2_BASE_IDX                                                               0
+#define regSDMA_GFX_MIDCMD_DATA3                                                                        0x00c3
+#define regSDMA_GFX_MIDCMD_DATA3_BASE_IDX                                                               0
+#define regSDMA_GFX_MIDCMD_DATA4                                                                        0x00c4
+#define regSDMA_GFX_MIDCMD_DATA4_BASE_IDX                                                               0
+#define regSDMA_GFX_MIDCMD_DATA5                                                                        0x00c5
+#define regSDMA_GFX_MIDCMD_DATA5_BASE_IDX                                                               0
+#define regSDMA_GFX_MIDCMD_DATA6                                                                        0x00c6
+#define regSDMA_GFX_MIDCMD_DATA6_BASE_IDX                                                               0
+#define regSDMA_GFX_MIDCMD_DATA7                                                                        0x00c7
+#define regSDMA_GFX_MIDCMD_DATA7_BASE_IDX                                                               0
+#define regSDMA_GFX_MIDCMD_DATA8                                                                        0x00c8
+#define regSDMA_GFX_MIDCMD_DATA8_BASE_IDX                                                               0
+#define regSDMA_GFX_MIDCMD_DATA9                                                                        0x00c9
+#define regSDMA_GFX_MIDCMD_DATA9_BASE_IDX                                                               0
+#define regSDMA_GFX_MIDCMD_DATA10                                                                       0x00ca
+#define regSDMA_GFX_MIDCMD_DATA10_BASE_IDX                                                              0
+#define regSDMA_GFX_MIDCMD_CNTL                                                                         0x00cb
+#define regSDMA_GFX_MIDCMD_CNTL_BASE_IDX                                                                0
+#define regSDMA_PAGE_RB_CNTL                                                                            0x00d8
+#define regSDMA_PAGE_RB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_PAGE_RB_BASE                                                                            0x00d9
+#define regSDMA_PAGE_RB_BASE_BASE_IDX                                                                   0
+#define regSDMA_PAGE_RB_BASE_HI                                                                         0x00da
+#define regSDMA_PAGE_RB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_PAGE_RB_RPTR                                                                            0x00db
+#define regSDMA_PAGE_RB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_PAGE_RB_RPTR_HI                                                                         0x00dc
+#define regSDMA_PAGE_RB_RPTR_HI_BASE_IDX                                                                0
+#define regSDMA_PAGE_RB_WPTR                                                                            0x00dd
+#define regSDMA_PAGE_RB_WPTR_BASE_IDX                                                                   0
+#define regSDMA_PAGE_RB_WPTR_HI                                                                         0x00de
+#define regSDMA_PAGE_RB_WPTR_HI_BASE_IDX                                                                0
+#define regSDMA_PAGE_RB_WPTR_POLL_CNTL                                                                  0x00df
+#define regSDMA_PAGE_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
+#define regSDMA_PAGE_RB_RPTR_ADDR_HI                                                                    0x00e0
+#define regSDMA_PAGE_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
+#define regSDMA_PAGE_RB_RPTR_ADDR_LO                                                                    0x00e1
+#define regSDMA_PAGE_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
+#define regSDMA_PAGE_IB_CNTL                                                                            0x00e2
+#define regSDMA_PAGE_IB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_PAGE_IB_RPTR                                                                            0x00e3
+#define regSDMA_PAGE_IB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_PAGE_IB_OFFSET                                                                          0x00e4
+#define regSDMA_PAGE_IB_OFFSET_BASE_IDX                                                                 0
+#define regSDMA_PAGE_IB_BASE_LO                                                                         0x00e5
+#define regSDMA_PAGE_IB_BASE_LO_BASE_IDX                                                                0
+#define regSDMA_PAGE_IB_BASE_HI                                                                         0x00e6
+#define regSDMA_PAGE_IB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_PAGE_IB_SIZE                                                                            0x00e7
+#define regSDMA_PAGE_IB_SIZE_BASE_IDX                                                                   0
+#define regSDMA_PAGE_SKIP_CNTL                                                                          0x00e8
+#define regSDMA_PAGE_SKIP_CNTL_BASE_IDX                                                                 0
+#define regSDMA_PAGE_CONTEXT_STATUS                                                                     0x00e9
+#define regSDMA_PAGE_CONTEXT_STATUS_BASE_IDX                                                            0
+#define regSDMA_PAGE_DOORBELL                                                                           0x00ea
+#define regSDMA_PAGE_DOORBELL_BASE_IDX                                                                  0
+#define regSDMA_PAGE_STATUS                                                                             0x0100
+#define regSDMA_PAGE_STATUS_BASE_IDX                                                                    0
+#define regSDMA_PAGE_DOORBELL_LOG                                                                       0x0101
+#define regSDMA_PAGE_DOORBELL_LOG_BASE_IDX                                                              0
+#define regSDMA_PAGE_WATERMARK                                                                          0x0102
+#define regSDMA_PAGE_WATERMARK_BASE_IDX                                                                 0
+#define regSDMA_PAGE_DOORBELL_OFFSET                                                                    0x0103
+#define regSDMA_PAGE_DOORBELL_OFFSET_BASE_IDX                                                           0
+#define regSDMA_PAGE_CSA_ADDR_LO                                                                        0x0104
+#define regSDMA_PAGE_CSA_ADDR_LO_BASE_IDX                                                               0
+#define regSDMA_PAGE_CSA_ADDR_HI                                                                        0x0105
+#define regSDMA_PAGE_CSA_ADDR_HI_BASE_IDX                                                               0
+#define regSDMA_PAGE_IB_SUB_REMAIN                                                                      0x0107
+#define regSDMA_PAGE_IB_SUB_REMAIN_BASE_IDX                                                             0
+#define regSDMA_PAGE_PREEMPT                                                                            0x0108
+#define regSDMA_PAGE_PREEMPT_BASE_IDX                                                                   0
+#define regSDMA_PAGE_DUMMY_REG                                                                          0x0109
+#define regSDMA_PAGE_DUMMY_REG_BASE_IDX                                                                 0
+#define regSDMA_PAGE_RB_WPTR_POLL_ADDR_HI                                                               0x010a
+#define regSDMA_PAGE_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
+#define regSDMA_PAGE_RB_WPTR_POLL_ADDR_LO                                                               0x010b
+#define regSDMA_PAGE_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
+#define regSDMA_PAGE_RB_AQL_CNTL                                                                        0x010c
+#define regSDMA_PAGE_RB_AQL_CNTL_BASE_IDX                                                               0
+#define regSDMA_PAGE_MINOR_PTR_UPDATE                                                                   0x010d
+#define regSDMA_PAGE_MINOR_PTR_UPDATE_BASE_IDX                                                          0
+#define regSDMA_PAGE_MIDCMD_DATA0                                                                       0x0118
+#define regSDMA_PAGE_MIDCMD_DATA0_BASE_IDX                                                              0
+#define regSDMA_PAGE_MIDCMD_DATA1                                                                       0x0119
+#define regSDMA_PAGE_MIDCMD_DATA1_BASE_IDX                                                              0
+#define regSDMA_PAGE_MIDCMD_DATA2                                                                       0x011a
+#define regSDMA_PAGE_MIDCMD_DATA2_BASE_IDX                                                              0
+#define regSDMA_PAGE_MIDCMD_DATA3                                                                       0x011b
+#define regSDMA_PAGE_MIDCMD_DATA3_BASE_IDX                                                              0
+#define regSDMA_PAGE_MIDCMD_DATA4                                                                       0x011c
+#define regSDMA_PAGE_MIDCMD_DATA4_BASE_IDX                                                              0
+#define regSDMA_PAGE_MIDCMD_DATA5                                                                       0x011d
+#define regSDMA_PAGE_MIDCMD_DATA5_BASE_IDX                                                              0
+#define regSDMA_PAGE_MIDCMD_DATA6                                                                       0x011e
+#define regSDMA_PAGE_MIDCMD_DATA6_BASE_IDX                                                              0
+#define regSDMA_PAGE_MIDCMD_DATA7                                                                       0x011f
+#define regSDMA_PAGE_MIDCMD_DATA7_BASE_IDX                                                              0
+#define regSDMA_PAGE_MIDCMD_DATA8                                                                       0x0120
+#define regSDMA_PAGE_MIDCMD_DATA8_BASE_IDX                                                              0
+#define regSDMA_PAGE_MIDCMD_DATA9                                                                       0x0121
+#define regSDMA_PAGE_MIDCMD_DATA9_BASE_IDX                                                              0
+#define regSDMA_PAGE_MIDCMD_DATA10                                                                      0x0122
+#define regSDMA_PAGE_MIDCMD_DATA10_BASE_IDX                                                             0
+#define regSDMA_PAGE_MIDCMD_CNTL                                                                        0x0123
+#define regSDMA_PAGE_MIDCMD_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC0_RB_CNTL                                                                            0x0130
+#define regSDMA_RLC0_RB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC0_RB_BASE                                                                            0x0131
+#define regSDMA_RLC0_RB_BASE_BASE_IDX                                                                   0
+#define regSDMA_RLC0_RB_BASE_HI                                                                         0x0132
+#define regSDMA_RLC0_RB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC0_RB_RPTR                                                                            0x0133
+#define regSDMA_RLC0_RB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC0_RB_RPTR_HI                                                                         0x0134
+#define regSDMA_RLC0_RB_RPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC0_RB_WPTR                                                                            0x0135
+#define regSDMA_RLC0_RB_WPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC0_RB_WPTR_HI                                                                         0x0136
+#define regSDMA_RLC0_RB_WPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC0_RB_WPTR_POLL_CNTL                                                                  0x0137
+#define regSDMA_RLC0_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
+#define regSDMA_RLC0_RB_RPTR_ADDR_HI                                                                    0x0138
+#define regSDMA_RLC0_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
+#define regSDMA_RLC0_RB_RPTR_ADDR_LO                                                                    0x0139
+#define regSDMA_RLC0_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
+#define regSDMA_RLC0_IB_CNTL                                                                            0x013a
+#define regSDMA_RLC0_IB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC0_IB_RPTR                                                                            0x013b
+#define regSDMA_RLC0_IB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC0_IB_OFFSET                                                                          0x013c
+#define regSDMA_RLC0_IB_OFFSET_BASE_IDX                                                                 0
+#define regSDMA_RLC0_IB_BASE_LO                                                                         0x013d
+#define regSDMA_RLC0_IB_BASE_LO_BASE_IDX                                                                0
+#define regSDMA_RLC0_IB_BASE_HI                                                                         0x013e
+#define regSDMA_RLC0_IB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC0_IB_SIZE                                                                            0x013f
+#define regSDMA_RLC0_IB_SIZE_BASE_IDX                                                                   0
+#define regSDMA_RLC0_SKIP_CNTL                                                                          0x0140
+#define regSDMA_RLC0_SKIP_CNTL_BASE_IDX                                                                 0
+#define regSDMA_RLC0_CONTEXT_STATUS                                                                     0x0141
+#define regSDMA_RLC0_CONTEXT_STATUS_BASE_IDX                                                            0
+#define regSDMA_RLC0_DOORBELL                                                                           0x0142
+#define regSDMA_RLC0_DOORBELL_BASE_IDX                                                                  0
+#define regSDMA_RLC0_STATUS                                                                             0x0158
+#define regSDMA_RLC0_STATUS_BASE_IDX                                                                    0
+#define regSDMA_RLC0_DOORBELL_LOG                                                                       0x0159
+#define regSDMA_RLC0_DOORBELL_LOG_BASE_IDX                                                              0
+#define regSDMA_RLC0_WATERMARK                                                                          0x015a
+#define regSDMA_RLC0_WATERMARK_BASE_IDX                                                                 0
+#define regSDMA_RLC0_DOORBELL_OFFSET                                                                    0x015b
+#define regSDMA_RLC0_DOORBELL_OFFSET_BASE_IDX                                                           0
+#define regSDMA_RLC0_CSA_ADDR_LO                                                                        0x015c
+#define regSDMA_RLC0_CSA_ADDR_LO_BASE_IDX                                                               0
+#define regSDMA_RLC0_CSA_ADDR_HI                                                                        0x015d
+#define regSDMA_RLC0_CSA_ADDR_HI_BASE_IDX                                                               0
+#define regSDMA_RLC0_IB_SUB_REMAIN                                                                      0x015f
+#define regSDMA_RLC0_IB_SUB_REMAIN_BASE_IDX                                                             0
+#define regSDMA_RLC0_PREEMPT                                                                            0x0160
+#define regSDMA_RLC0_PREEMPT_BASE_IDX                                                                   0
+#define regSDMA_RLC0_DUMMY_REG                                                                          0x0161
+#define regSDMA_RLC0_DUMMY_REG_BASE_IDX                                                                 0
+#define regSDMA_RLC0_RB_WPTR_POLL_ADDR_HI                                                               0x0162
+#define regSDMA_RLC0_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
+#define regSDMA_RLC0_RB_WPTR_POLL_ADDR_LO                                                               0x0163
+#define regSDMA_RLC0_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
+#define regSDMA_RLC0_RB_AQL_CNTL                                                                        0x0164
+#define regSDMA_RLC0_RB_AQL_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC0_MINOR_PTR_UPDATE                                                                   0x0165
+#define regSDMA_RLC0_MINOR_PTR_UPDATE_BASE_IDX                                                          0
+#define regSDMA_RLC0_MIDCMD_DATA0                                                                       0x0170
+#define regSDMA_RLC0_MIDCMD_DATA0_BASE_IDX                                                              0
+#define regSDMA_RLC0_MIDCMD_DATA1                                                                       0x0171
+#define regSDMA_RLC0_MIDCMD_DATA1_BASE_IDX                                                              0
+#define regSDMA_RLC0_MIDCMD_DATA2                                                                       0x0172
+#define regSDMA_RLC0_MIDCMD_DATA2_BASE_IDX                                                              0
+#define regSDMA_RLC0_MIDCMD_DATA3                                                                       0x0173
+#define regSDMA_RLC0_MIDCMD_DATA3_BASE_IDX                                                              0
+#define regSDMA_RLC0_MIDCMD_DATA4                                                                       0x0174
+#define regSDMA_RLC0_MIDCMD_DATA4_BASE_IDX                                                              0
+#define regSDMA_RLC0_MIDCMD_DATA5                                                                       0x0175
+#define regSDMA_RLC0_MIDCMD_DATA5_BASE_IDX                                                              0
+#define regSDMA_RLC0_MIDCMD_DATA6                                                                       0x0176
+#define regSDMA_RLC0_MIDCMD_DATA6_BASE_IDX                                                              0
+#define regSDMA_RLC0_MIDCMD_DATA7                                                                       0x0177
+#define regSDMA_RLC0_MIDCMD_DATA7_BASE_IDX                                                              0
+#define regSDMA_RLC0_MIDCMD_DATA8                                                                       0x0178
+#define regSDMA_RLC0_MIDCMD_DATA8_BASE_IDX                                                              0
+#define regSDMA_RLC0_MIDCMD_DATA9                                                                       0x0179
+#define regSDMA_RLC0_MIDCMD_DATA9_BASE_IDX                                                              0
+#define regSDMA_RLC0_MIDCMD_DATA10                                                                      0x017a
+#define regSDMA_RLC0_MIDCMD_DATA10_BASE_IDX                                                             0
+#define regSDMA_RLC0_MIDCMD_CNTL                                                                        0x017b
+#define regSDMA_RLC0_MIDCMD_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC1_RB_CNTL                                                                            0x0188
+#define regSDMA_RLC1_RB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC1_RB_BASE                                                                            0x0189
+#define regSDMA_RLC1_RB_BASE_BASE_IDX                                                                   0
+#define regSDMA_RLC1_RB_BASE_HI                                                                         0x018a
+#define regSDMA_RLC1_RB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC1_RB_RPTR                                                                            0x018b
+#define regSDMA_RLC1_RB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC1_RB_RPTR_HI                                                                         0x018c
+#define regSDMA_RLC1_RB_RPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC1_RB_WPTR                                                                            0x018d
+#define regSDMA_RLC1_RB_WPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC1_RB_WPTR_HI                                                                         0x018e
+#define regSDMA_RLC1_RB_WPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC1_RB_WPTR_POLL_CNTL                                                                  0x018f
+#define regSDMA_RLC1_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
+#define regSDMA_RLC1_RB_RPTR_ADDR_HI                                                                    0x0190
+#define regSDMA_RLC1_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
+#define regSDMA_RLC1_RB_RPTR_ADDR_LO                                                                    0x0191
+#define regSDMA_RLC1_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
+#define regSDMA_RLC1_IB_CNTL                                                                            0x0192
+#define regSDMA_RLC1_IB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC1_IB_RPTR                                                                            0x0193
+#define regSDMA_RLC1_IB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC1_IB_OFFSET                                                                          0x0194
+#define regSDMA_RLC1_IB_OFFSET_BASE_IDX                                                                 0
+#define regSDMA_RLC1_IB_BASE_LO                                                                         0x0195
+#define regSDMA_RLC1_IB_BASE_LO_BASE_IDX                                                                0
+#define regSDMA_RLC1_IB_BASE_HI                                                                         0x0196
+#define regSDMA_RLC1_IB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC1_IB_SIZE                                                                            0x0197
+#define regSDMA_RLC1_IB_SIZE_BASE_IDX                                                                   0
+#define regSDMA_RLC1_SKIP_CNTL                                                                          0x0198
+#define regSDMA_RLC1_SKIP_CNTL_BASE_IDX                                                                 0
+#define regSDMA_RLC1_CONTEXT_STATUS                                                                     0x0199
+#define regSDMA_RLC1_CONTEXT_STATUS_BASE_IDX                                                            0
+#define regSDMA_RLC1_DOORBELL                                                                           0x019a
+#define regSDMA_RLC1_DOORBELL_BASE_IDX                                                                  0
+#define regSDMA_RLC1_STATUS                                                                             0x01b0
+#define regSDMA_RLC1_STATUS_BASE_IDX                                                                    0
+#define regSDMA_RLC1_DOORBELL_LOG                                                                       0x01b1
+#define regSDMA_RLC1_DOORBELL_LOG_BASE_IDX                                                              0
+#define regSDMA_RLC1_WATERMARK                                                                          0x01b2
+#define regSDMA_RLC1_WATERMARK_BASE_IDX                                                                 0
+#define regSDMA_RLC1_DOORBELL_OFFSET                                                                    0x01b3
+#define regSDMA_RLC1_DOORBELL_OFFSET_BASE_IDX                                                           0
+#define regSDMA_RLC1_CSA_ADDR_LO                                                                        0x01b4
+#define regSDMA_RLC1_CSA_ADDR_LO_BASE_IDX                                                               0
+#define regSDMA_RLC1_CSA_ADDR_HI                                                                        0x01b5
+#define regSDMA_RLC1_CSA_ADDR_HI_BASE_IDX                                                               0
+#define regSDMA_RLC1_IB_SUB_REMAIN                                                                      0x01b7
+#define regSDMA_RLC1_IB_SUB_REMAIN_BASE_IDX                                                             0
+#define regSDMA_RLC1_PREEMPT                                                                            0x01b8
+#define regSDMA_RLC1_PREEMPT_BASE_IDX                                                                   0
+#define regSDMA_RLC1_DUMMY_REG                                                                          0x01b9
+#define regSDMA_RLC1_DUMMY_REG_BASE_IDX                                                                 0
+#define regSDMA_RLC1_RB_WPTR_POLL_ADDR_HI                                                               0x01ba
+#define regSDMA_RLC1_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
+#define regSDMA_RLC1_RB_WPTR_POLL_ADDR_LO                                                               0x01bb
+#define regSDMA_RLC1_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
+#define regSDMA_RLC1_RB_AQL_CNTL                                                                        0x01bc
+#define regSDMA_RLC1_RB_AQL_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC1_MINOR_PTR_UPDATE                                                                   0x01bd
+#define regSDMA_RLC1_MINOR_PTR_UPDATE_BASE_IDX                                                          0
+#define regSDMA_RLC1_MIDCMD_DATA0                                                                       0x01c8
+#define regSDMA_RLC1_MIDCMD_DATA0_BASE_IDX                                                              0
+#define regSDMA_RLC1_MIDCMD_DATA1                                                                       0x01c9
+#define regSDMA_RLC1_MIDCMD_DATA1_BASE_IDX                                                              0
+#define regSDMA_RLC1_MIDCMD_DATA2                                                                       0x01ca
+#define regSDMA_RLC1_MIDCMD_DATA2_BASE_IDX                                                              0
+#define regSDMA_RLC1_MIDCMD_DATA3                                                                       0x01cb
+#define regSDMA_RLC1_MIDCMD_DATA3_BASE_IDX                                                              0
+#define regSDMA_RLC1_MIDCMD_DATA4                                                                       0x01cc
+#define regSDMA_RLC1_MIDCMD_DATA4_BASE_IDX                                                              0
+#define regSDMA_RLC1_MIDCMD_DATA5                                                                       0x01cd
+#define regSDMA_RLC1_MIDCMD_DATA5_BASE_IDX                                                              0
+#define regSDMA_RLC1_MIDCMD_DATA6                                                                       0x01ce
+#define regSDMA_RLC1_MIDCMD_DATA6_BASE_IDX                                                              0
+#define regSDMA_RLC1_MIDCMD_DATA7                                                                       0x01cf
+#define regSDMA_RLC1_MIDCMD_DATA7_BASE_IDX                                                              0
+#define regSDMA_RLC1_MIDCMD_DATA8                                                                       0x01d0
+#define regSDMA_RLC1_MIDCMD_DATA8_BASE_IDX                                                              0
+#define regSDMA_RLC1_MIDCMD_DATA9                                                                       0x01d1
+#define regSDMA_RLC1_MIDCMD_DATA9_BASE_IDX                                                              0
+#define regSDMA_RLC1_MIDCMD_DATA10                                                                      0x01d2
+#define regSDMA_RLC1_MIDCMD_DATA10_BASE_IDX                                                             0
+#define regSDMA_RLC1_MIDCMD_CNTL                                                                        0x01d3
+#define regSDMA_RLC1_MIDCMD_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC2_RB_CNTL                                                                            0x01e0
+#define regSDMA_RLC2_RB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC2_RB_BASE                                                                            0x01e1
+#define regSDMA_RLC2_RB_BASE_BASE_IDX                                                                   0
+#define regSDMA_RLC2_RB_BASE_HI                                                                         0x01e2
+#define regSDMA_RLC2_RB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC2_RB_RPTR                                                                            0x01e3
+#define regSDMA_RLC2_RB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC2_RB_RPTR_HI                                                                         0x01e4
+#define regSDMA_RLC2_RB_RPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC2_RB_WPTR                                                                            0x01e5
+#define regSDMA_RLC2_RB_WPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC2_RB_WPTR_HI                                                                         0x01e6
+#define regSDMA_RLC2_RB_WPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC2_RB_WPTR_POLL_CNTL                                                                  0x01e7
+#define regSDMA_RLC2_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
+#define regSDMA_RLC2_RB_RPTR_ADDR_HI                                                                    0x01e8
+#define regSDMA_RLC2_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
+#define regSDMA_RLC2_RB_RPTR_ADDR_LO                                                                    0x01e9
+#define regSDMA_RLC2_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
+#define regSDMA_RLC2_IB_CNTL                                                                            0x01ea
+#define regSDMA_RLC2_IB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC2_IB_RPTR                                                                            0x01eb
+#define regSDMA_RLC2_IB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC2_IB_OFFSET                                                                          0x01ec
+#define regSDMA_RLC2_IB_OFFSET_BASE_IDX                                                                 0
+#define regSDMA_RLC2_IB_BASE_LO                                                                         0x01ed
+#define regSDMA_RLC2_IB_BASE_LO_BASE_IDX                                                                0
+#define regSDMA_RLC2_IB_BASE_HI                                                                         0x01ee
+#define regSDMA_RLC2_IB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC2_IB_SIZE                                                                            0x01ef
+#define regSDMA_RLC2_IB_SIZE_BASE_IDX                                                                   0
+#define regSDMA_RLC2_SKIP_CNTL                                                                          0x01f0
+#define regSDMA_RLC2_SKIP_CNTL_BASE_IDX                                                                 0
+#define regSDMA_RLC2_CONTEXT_STATUS                                                                     0x01f1
+#define regSDMA_RLC2_CONTEXT_STATUS_BASE_IDX                                                            0
+#define regSDMA_RLC2_DOORBELL                                                                           0x01f2
+#define regSDMA_RLC2_DOORBELL_BASE_IDX                                                                  0
+#define regSDMA_RLC2_STATUS                                                                             0x0208
+#define regSDMA_RLC2_STATUS_BASE_IDX                                                                    0
+#define regSDMA_RLC2_DOORBELL_LOG                                                                       0x0209
+#define regSDMA_RLC2_DOORBELL_LOG_BASE_IDX                                                              0
+#define regSDMA_RLC2_WATERMARK                                                                          0x020a
+#define regSDMA_RLC2_WATERMARK_BASE_IDX                                                                 0
+#define regSDMA_RLC2_DOORBELL_OFFSET                                                                    0x020b
+#define regSDMA_RLC2_DOORBELL_OFFSET_BASE_IDX                                                           0
+#define regSDMA_RLC2_CSA_ADDR_LO                                                                        0x020c
+#define regSDMA_RLC2_CSA_ADDR_LO_BASE_IDX                                                               0
+#define regSDMA_RLC2_CSA_ADDR_HI                                                                        0x020d
+#define regSDMA_RLC2_CSA_ADDR_HI_BASE_IDX                                                               0
+#define regSDMA_RLC2_IB_SUB_REMAIN                                                                      0x020f
+#define regSDMA_RLC2_IB_SUB_REMAIN_BASE_IDX                                                             0
+#define regSDMA_RLC2_PREEMPT                                                                            0x0210
+#define regSDMA_RLC2_PREEMPT_BASE_IDX                                                                   0
+#define regSDMA_RLC2_DUMMY_REG                                                                          0x0211
+#define regSDMA_RLC2_DUMMY_REG_BASE_IDX                                                                 0
+#define regSDMA_RLC2_RB_WPTR_POLL_ADDR_HI                                                               0x0212
+#define regSDMA_RLC2_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
+#define regSDMA_RLC2_RB_WPTR_POLL_ADDR_LO                                                               0x0213
+#define regSDMA_RLC2_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
+#define regSDMA_RLC2_RB_AQL_CNTL                                                                        0x0214
+#define regSDMA_RLC2_RB_AQL_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC2_MINOR_PTR_UPDATE                                                                   0x0215
+#define regSDMA_RLC2_MINOR_PTR_UPDATE_BASE_IDX                                                          0
+#define regSDMA_RLC2_MIDCMD_DATA0                                                                       0x0220
+#define regSDMA_RLC2_MIDCMD_DATA0_BASE_IDX                                                              0
+#define regSDMA_RLC2_MIDCMD_DATA1                                                                       0x0221
+#define regSDMA_RLC2_MIDCMD_DATA1_BASE_IDX                                                              0
+#define regSDMA_RLC2_MIDCMD_DATA2                                                                       0x0222
+#define regSDMA_RLC2_MIDCMD_DATA2_BASE_IDX                                                              0
+#define regSDMA_RLC2_MIDCMD_DATA3                                                                       0x0223
+#define regSDMA_RLC2_MIDCMD_DATA3_BASE_IDX                                                              0
+#define regSDMA_RLC2_MIDCMD_DATA4                                                                       0x0224
+#define regSDMA_RLC2_MIDCMD_DATA4_BASE_IDX                                                              0
+#define regSDMA_RLC2_MIDCMD_DATA5                                                                       0x0225
+#define regSDMA_RLC2_MIDCMD_DATA5_BASE_IDX                                                              0
+#define regSDMA_RLC2_MIDCMD_DATA6                                                                       0x0226
+#define regSDMA_RLC2_MIDCMD_DATA6_BASE_IDX                                                              0
+#define regSDMA_RLC2_MIDCMD_DATA7                                                                       0x0227
+#define regSDMA_RLC2_MIDCMD_DATA7_BASE_IDX                                                              0
+#define regSDMA_RLC2_MIDCMD_DATA8                                                                       0x0228
+#define regSDMA_RLC2_MIDCMD_DATA8_BASE_IDX                                                              0
+#define regSDMA_RLC2_MIDCMD_DATA9                                                                       0x0229
+#define regSDMA_RLC2_MIDCMD_DATA9_BASE_IDX                                                              0
+#define regSDMA_RLC2_MIDCMD_DATA10                                                                      0x022a
+#define regSDMA_RLC2_MIDCMD_DATA10_BASE_IDX                                                             0
+#define regSDMA_RLC2_MIDCMD_CNTL                                                                        0x022b
+#define regSDMA_RLC2_MIDCMD_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC3_RB_CNTL                                                                            0x0238
+#define regSDMA_RLC3_RB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC3_RB_BASE                                                                            0x0239
+#define regSDMA_RLC3_RB_BASE_BASE_IDX                                                                   0
+#define regSDMA_RLC3_RB_BASE_HI                                                                         0x023a
+#define regSDMA_RLC3_RB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC3_RB_RPTR                                                                            0x023b
+#define regSDMA_RLC3_RB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC3_RB_RPTR_HI                                                                         0x023c
+#define regSDMA_RLC3_RB_RPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC3_RB_WPTR                                                                            0x023d
+#define regSDMA_RLC3_RB_WPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC3_RB_WPTR_HI                                                                         0x023e
+#define regSDMA_RLC3_RB_WPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC3_RB_WPTR_POLL_CNTL                                                                  0x023f
+#define regSDMA_RLC3_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
+#define regSDMA_RLC3_RB_RPTR_ADDR_HI                                                                    0x0240
+#define regSDMA_RLC3_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
+#define regSDMA_RLC3_RB_RPTR_ADDR_LO                                                                    0x0241
+#define regSDMA_RLC3_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
+#define regSDMA_RLC3_IB_CNTL                                                                            0x0242
+#define regSDMA_RLC3_IB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC3_IB_RPTR                                                                            0x0243
+#define regSDMA_RLC3_IB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC3_IB_OFFSET                                                                          0x0244
+#define regSDMA_RLC3_IB_OFFSET_BASE_IDX                                                                 0
+#define regSDMA_RLC3_IB_BASE_LO                                                                         0x0245
+#define regSDMA_RLC3_IB_BASE_LO_BASE_IDX                                                                0
+#define regSDMA_RLC3_IB_BASE_HI                                                                         0x0246
+#define regSDMA_RLC3_IB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC3_IB_SIZE                                                                            0x0247
+#define regSDMA_RLC3_IB_SIZE_BASE_IDX                                                                   0
+#define regSDMA_RLC3_SKIP_CNTL                                                                          0x0248
+#define regSDMA_RLC3_SKIP_CNTL_BASE_IDX                                                                 0
+#define regSDMA_RLC3_CONTEXT_STATUS                                                                     0x0249
+#define regSDMA_RLC3_CONTEXT_STATUS_BASE_IDX                                                            0
+#define regSDMA_RLC3_DOORBELL                                                                           0x024a
+#define regSDMA_RLC3_DOORBELL_BASE_IDX                                                                  0
+#define regSDMA_RLC3_STATUS                                                                             0x0260
+#define regSDMA_RLC3_STATUS_BASE_IDX                                                                    0
+#define regSDMA_RLC3_DOORBELL_LOG                                                                       0x0261
+#define regSDMA_RLC3_DOORBELL_LOG_BASE_IDX                                                              0
+#define regSDMA_RLC3_WATERMARK                                                                          0x0262
+#define regSDMA_RLC3_WATERMARK_BASE_IDX                                                                 0
+#define regSDMA_RLC3_DOORBELL_OFFSET                                                                    0x0263
+#define regSDMA_RLC3_DOORBELL_OFFSET_BASE_IDX                                                           0
+#define regSDMA_RLC3_CSA_ADDR_LO                                                                        0x0264
+#define regSDMA_RLC3_CSA_ADDR_LO_BASE_IDX                                                               0
+#define regSDMA_RLC3_CSA_ADDR_HI                                                                        0x0265
+#define regSDMA_RLC3_CSA_ADDR_HI_BASE_IDX                                                               0
+#define regSDMA_RLC3_IB_SUB_REMAIN                                                                      0x0267
+#define regSDMA_RLC3_IB_SUB_REMAIN_BASE_IDX                                                             0
+#define regSDMA_RLC3_PREEMPT                                                                            0x0268
+#define regSDMA_RLC3_PREEMPT_BASE_IDX                                                                   0
+#define regSDMA_RLC3_DUMMY_REG                                                                          0x0269
+#define regSDMA_RLC3_DUMMY_REG_BASE_IDX                                                                 0
+#define regSDMA_RLC3_RB_WPTR_POLL_ADDR_HI                                                               0x026a
+#define regSDMA_RLC3_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
+#define regSDMA_RLC3_RB_WPTR_POLL_ADDR_LO                                                               0x026b
+#define regSDMA_RLC3_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
+#define regSDMA_RLC3_RB_AQL_CNTL                                                                        0x026c
+#define regSDMA_RLC3_RB_AQL_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC3_MINOR_PTR_UPDATE                                                                   0x026d
+#define regSDMA_RLC3_MINOR_PTR_UPDATE_BASE_IDX                                                          0
+#define regSDMA_RLC3_MIDCMD_DATA0                                                                       0x0278
+#define regSDMA_RLC3_MIDCMD_DATA0_BASE_IDX                                                              0
+#define regSDMA_RLC3_MIDCMD_DATA1                                                                       0x0279
+#define regSDMA_RLC3_MIDCMD_DATA1_BASE_IDX                                                              0
+#define regSDMA_RLC3_MIDCMD_DATA2                                                                       0x027a
+#define regSDMA_RLC3_MIDCMD_DATA2_BASE_IDX                                                              0
+#define regSDMA_RLC3_MIDCMD_DATA3                                                                       0x027b
+#define regSDMA_RLC3_MIDCMD_DATA3_BASE_IDX                                                              0
+#define regSDMA_RLC3_MIDCMD_DATA4                                                                       0x027c
+#define regSDMA_RLC3_MIDCMD_DATA4_BASE_IDX                                                              0
+#define regSDMA_RLC3_MIDCMD_DATA5                                                                       0x027d
+#define regSDMA_RLC3_MIDCMD_DATA5_BASE_IDX                                                              0
+#define regSDMA_RLC3_MIDCMD_DATA6                                                                       0x027e
+#define regSDMA_RLC3_MIDCMD_DATA6_BASE_IDX                                                              0
+#define regSDMA_RLC3_MIDCMD_DATA7                                                                       0x027f
+#define regSDMA_RLC3_MIDCMD_DATA7_BASE_IDX                                                              0
+#define regSDMA_RLC3_MIDCMD_DATA8                                                                       0x0280
+#define regSDMA_RLC3_MIDCMD_DATA8_BASE_IDX                                                              0
+#define regSDMA_RLC3_MIDCMD_DATA9                                                                       0x0281
+#define regSDMA_RLC3_MIDCMD_DATA9_BASE_IDX                                                              0
+#define regSDMA_RLC3_MIDCMD_DATA10                                                                      0x0282
+#define regSDMA_RLC3_MIDCMD_DATA10_BASE_IDX                                                             0
+#define regSDMA_RLC3_MIDCMD_CNTL                                                                        0x0283
+#define regSDMA_RLC3_MIDCMD_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC4_RB_CNTL                                                                            0x0290
+#define regSDMA_RLC4_RB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC4_RB_BASE                                                                            0x0291
+#define regSDMA_RLC4_RB_BASE_BASE_IDX                                                                   0
+#define regSDMA_RLC4_RB_BASE_HI                                                                         0x0292
+#define regSDMA_RLC4_RB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC4_RB_RPTR                                                                            0x0293
+#define regSDMA_RLC4_RB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC4_RB_RPTR_HI                                                                         0x0294
+#define regSDMA_RLC4_RB_RPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC4_RB_WPTR                                                                            0x0295
+#define regSDMA_RLC4_RB_WPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC4_RB_WPTR_HI                                                                         0x0296
+#define regSDMA_RLC4_RB_WPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC4_RB_WPTR_POLL_CNTL                                                                  0x0297
+#define regSDMA_RLC4_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
+#define regSDMA_RLC4_RB_RPTR_ADDR_HI                                                                    0x0298
+#define regSDMA_RLC4_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
+#define regSDMA_RLC4_RB_RPTR_ADDR_LO                                                                    0x0299
+#define regSDMA_RLC4_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
+#define regSDMA_RLC4_IB_CNTL                                                                            0x029a
+#define regSDMA_RLC4_IB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC4_IB_RPTR                                                                            0x029b
+#define regSDMA_RLC4_IB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC4_IB_OFFSET                                                                          0x029c
+#define regSDMA_RLC4_IB_OFFSET_BASE_IDX                                                                 0
+#define regSDMA_RLC4_IB_BASE_LO                                                                         0x029d
+#define regSDMA_RLC4_IB_BASE_LO_BASE_IDX                                                                0
+#define regSDMA_RLC4_IB_BASE_HI                                                                         0x029e
+#define regSDMA_RLC4_IB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC4_IB_SIZE                                                                            0x029f
+#define regSDMA_RLC4_IB_SIZE_BASE_IDX                                                                   0
+#define regSDMA_RLC4_SKIP_CNTL                                                                          0x02a0
+#define regSDMA_RLC4_SKIP_CNTL_BASE_IDX                                                                 0
+#define regSDMA_RLC4_CONTEXT_STATUS                                                                     0x02a1
+#define regSDMA_RLC4_CONTEXT_STATUS_BASE_IDX                                                            0
+#define regSDMA_RLC4_DOORBELL                                                                           0x02a2
+#define regSDMA_RLC4_DOORBELL_BASE_IDX                                                                  0
+#define regSDMA_RLC4_STATUS                                                                             0x02b8
+#define regSDMA_RLC4_STATUS_BASE_IDX                                                                    0
+#define regSDMA_RLC4_DOORBELL_LOG                                                                       0x02b9
+#define regSDMA_RLC4_DOORBELL_LOG_BASE_IDX                                                              0
+#define regSDMA_RLC4_WATERMARK                                                                          0x02ba
+#define regSDMA_RLC4_WATERMARK_BASE_IDX                                                                 0
+#define regSDMA_RLC4_DOORBELL_OFFSET                                                                    0x02bb
+#define regSDMA_RLC4_DOORBELL_OFFSET_BASE_IDX                                                           0
+#define regSDMA_RLC4_CSA_ADDR_LO                                                                        0x02bc
+#define regSDMA_RLC4_CSA_ADDR_LO_BASE_IDX                                                               0
+#define regSDMA_RLC4_CSA_ADDR_HI                                                                        0x02bd
+#define regSDMA_RLC4_CSA_ADDR_HI_BASE_IDX                                                               0
+#define regSDMA_RLC4_IB_SUB_REMAIN                                                                      0x02bf
+#define regSDMA_RLC4_IB_SUB_REMAIN_BASE_IDX                                                             0
+#define regSDMA_RLC4_PREEMPT                                                                            0x02c0
+#define regSDMA_RLC4_PREEMPT_BASE_IDX                                                                   0
+#define regSDMA_RLC4_DUMMY_REG                                                                          0x02c1
+#define regSDMA_RLC4_DUMMY_REG_BASE_IDX                                                                 0
+#define regSDMA_RLC4_RB_WPTR_POLL_ADDR_HI                                                               0x02c2
+#define regSDMA_RLC4_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
+#define regSDMA_RLC4_RB_WPTR_POLL_ADDR_LO                                                               0x02c3
+#define regSDMA_RLC4_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
+#define regSDMA_RLC4_RB_AQL_CNTL                                                                        0x02c4
+#define regSDMA_RLC4_RB_AQL_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC4_MINOR_PTR_UPDATE                                                                   0x02c5
+#define regSDMA_RLC4_MINOR_PTR_UPDATE_BASE_IDX                                                          0
+#define regSDMA_RLC4_MIDCMD_DATA0                                                                       0x02d0
+#define regSDMA_RLC4_MIDCMD_DATA0_BASE_IDX                                                              0
+#define regSDMA_RLC4_MIDCMD_DATA1                                                                       0x02d1
+#define regSDMA_RLC4_MIDCMD_DATA1_BASE_IDX                                                              0
+#define regSDMA_RLC4_MIDCMD_DATA2                                                                       0x02d2
+#define regSDMA_RLC4_MIDCMD_DATA2_BASE_IDX                                                              0
+#define regSDMA_RLC4_MIDCMD_DATA3                                                                       0x02d3
+#define regSDMA_RLC4_MIDCMD_DATA3_BASE_IDX                                                              0
+#define regSDMA_RLC4_MIDCMD_DATA4                                                                       0x02d4
+#define regSDMA_RLC4_MIDCMD_DATA4_BASE_IDX                                                              0
+#define regSDMA_RLC4_MIDCMD_DATA5                                                                       0x02d5
+#define regSDMA_RLC4_MIDCMD_DATA5_BASE_IDX                                                              0
+#define regSDMA_RLC4_MIDCMD_DATA6                                                                       0x02d6
+#define regSDMA_RLC4_MIDCMD_DATA6_BASE_IDX                                                              0
+#define regSDMA_RLC4_MIDCMD_DATA7                                                                       0x02d7
+#define regSDMA_RLC4_MIDCMD_DATA7_BASE_IDX                                                              0
+#define regSDMA_RLC4_MIDCMD_DATA8                                                                       0x02d8
+#define regSDMA_RLC4_MIDCMD_DATA8_BASE_IDX                                                              0
+#define regSDMA_RLC4_MIDCMD_DATA9                                                                       0x02d9
+#define regSDMA_RLC4_MIDCMD_DATA9_BASE_IDX                                                              0
+#define regSDMA_RLC4_MIDCMD_DATA10                                                                      0x02da
+#define regSDMA_RLC4_MIDCMD_DATA10_BASE_IDX                                                             0
+#define regSDMA_RLC4_MIDCMD_CNTL                                                                        0x02db
+#define regSDMA_RLC4_MIDCMD_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC5_RB_CNTL                                                                            0x02e8
+#define regSDMA_RLC5_RB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC5_RB_BASE                                                                            0x02e9
+#define regSDMA_RLC5_RB_BASE_BASE_IDX                                                                   0
+#define regSDMA_RLC5_RB_BASE_HI                                                                         0x02ea
+#define regSDMA_RLC5_RB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC5_RB_RPTR                                                                            0x02eb
+#define regSDMA_RLC5_RB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC5_RB_RPTR_HI                                                                         0x02ec
+#define regSDMA_RLC5_RB_RPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC5_RB_WPTR                                                                            0x02ed
+#define regSDMA_RLC5_RB_WPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC5_RB_WPTR_HI                                                                         0x02ee
+#define regSDMA_RLC5_RB_WPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC5_RB_WPTR_POLL_CNTL                                                                  0x02ef
+#define regSDMA_RLC5_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
+#define regSDMA_RLC5_RB_RPTR_ADDR_HI                                                                    0x02f0
+#define regSDMA_RLC5_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
+#define regSDMA_RLC5_RB_RPTR_ADDR_LO                                                                    0x02f1
+#define regSDMA_RLC5_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
+#define regSDMA_RLC5_IB_CNTL                                                                            0x02f2
+#define regSDMA_RLC5_IB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC5_IB_RPTR                                                                            0x02f3
+#define regSDMA_RLC5_IB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC5_IB_OFFSET                                                                          0x02f4
+#define regSDMA_RLC5_IB_OFFSET_BASE_IDX                                                                 0
+#define regSDMA_RLC5_IB_BASE_LO                                                                         0x02f5
+#define regSDMA_RLC5_IB_BASE_LO_BASE_IDX                                                                0
+#define regSDMA_RLC5_IB_BASE_HI                                                                         0x02f6
+#define regSDMA_RLC5_IB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC5_IB_SIZE                                                                            0x02f7
+#define regSDMA_RLC5_IB_SIZE_BASE_IDX                                                                   0
+#define regSDMA_RLC5_SKIP_CNTL                                                                          0x02f8
+#define regSDMA_RLC5_SKIP_CNTL_BASE_IDX                                                                 0
+#define regSDMA_RLC5_CONTEXT_STATUS                                                                     0x02f9
+#define regSDMA_RLC5_CONTEXT_STATUS_BASE_IDX                                                            0
+#define regSDMA_RLC5_DOORBELL                                                                           0x02fa
+#define regSDMA_RLC5_DOORBELL_BASE_IDX                                                                  0
+#define regSDMA_RLC5_STATUS                                                                             0x0310
+#define regSDMA_RLC5_STATUS_BASE_IDX                                                                    0
+#define regSDMA_RLC5_DOORBELL_LOG                                                                       0x0311
+#define regSDMA_RLC5_DOORBELL_LOG_BASE_IDX                                                              0
+#define regSDMA_RLC5_WATERMARK                                                                          0x0312
+#define regSDMA_RLC5_WATERMARK_BASE_IDX                                                                 0
+#define regSDMA_RLC5_DOORBELL_OFFSET                                                                    0x0313
+#define regSDMA_RLC5_DOORBELL_OFFSET_BASE_IDX                                                           0
+#define regSDMA_RLC5_CSA_ADDR_LO                                                                        0x0314
+#define regSDMA_RLC5_CSA_ADDR_LO_BASE_IDX                                                               0
+#define regSDMA_RLC5_CSA_ADDR_HI                                                                        0x0315
+#define regSDMA_RLC5_CSA_ADDR_HI_BASE_IDX                                                               0
+#define regSDMA_RLC5_IB_SUB_REMAIN                                                                      0x0317
+#define regSDMA_RLC5_IB_SUB_REMAIN_BASE_IDX                                                             0
+#define regSDMA_RLC5_PREEMPT                                                                            0x0318
+#define regSDMA_RLC5_PREEMPT_BASE_IDX                                                                   0
+#define regSDMA_RLC5_DUMMY_REG                                                                          0x0319
+#define regSDMA_RLC5_DUMMY_REG_BASE_IDX                                                                 0
+#define regSDMA_RLC5_RB_WPTR_POLL_ADDR_HI                                                               0x031a
+#define regSDMA_RLC5_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
+#define regSDMA_RLC5_RB_WPTR_POLL_ADDR_LO                                                               0x031b
+#define regSDMA_RLC5_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
+#define regSDMA_RLC5_RB_AQL_CNTL                                                                        0x031c
+#define regSDMA_RLC5_RB_AQL_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC5_MINOR_PTR_UPDATE                                                                   0x031d
+#define regSDMA_RLC5_MINOR_PTR_UPDATE_BASE_IDX                                                          0
+#define regSDMA_RLC5_MIDCMD_DATA0                                                                       0x0328
+#define regSDMA_RLC5_MIDCMD_DATA0_BASE_IDX                                                              0
+#define regSDMA_RLC5_MIDCMD_DATA1                                                                       0x0329
+#define regSDMA_RLC5_MIDCMD_DATA1_BASE_IDX                                                              0
+#define regSDMA_RLC5_MIDCMD_DATA2                                                                       0x032a
+#define regSDMA_RLC5_MIDCMD_DATA2_BASE_IDX                                                              0
+#define regSDMA_RLC5_MIDCMD_DATA3                                                                       0x032b
+#define regSDMA_RLC5_MIDCMD_DATA3_BASE_IDX                                                              0
+#define regSDMA_RLC5_MIDCMD_DATA4                                                                       0x032c
+#define regSDMA_RLC5_MIDCMD_DATA4_BASE_IDX                                                              0
+#define regSDMA_RLC5_MIDCMD_DATA5                                                                       0x032d
+#define regSDMA_RLC5_MIDCMD_DATA5_BASE_IDX                                                              0
+#define regSDMA_RLC5_MIDCMD_DATA6                                                                       0x032e
+#define regSDMA_RLC5_MIDCMD_DATA6_BASE_IDX                                                              0
+#define regSDMA_RLC5_MIDCMD_DATA7                                                                       0x032f
+#define regSDMA_RLC5_MIDCMD_DATA7_BASE_IDX                                                              0
+#define regSDMA_RLC5_MIDCMD_DATA8                                                                       0x0330
+#define regSDMA_RLC5_MIDCMD_DATA8_BASE_IDX                                                              0
+#define regSDMA_RLC5_MIDCMD_DATA9                                                                       0x0331
+#define regSDMA_RLC5_MIDCMD_DATA9_BASE_IDX                                                              0
+#define regSDMA_RLC5_MIDCMD_DATA10                                                                      0x0332
+#define regSDMA_RLC5_MIDCMD_DATA10_BASE_IDX                                                             0
+#define regSDMA_RLC5_MIDCMD_CNTL                                                                        0x0333
+#define regSDMA_RLC5_MIDCMD_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC6_RB_CNTL                                                                            0x0340
+#define regSDMA_RLC6_RB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC6_RB_BASE                                                                            0x0341
+#define regSDMA_RLC6_RB_BASE_BASE_IDX                                                                   0
+#define regSDMA_RLC6_RB_BASE_HI                                                                         0x0342
+#define regSDMA_RLC6_RB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC6_RB_RPTR                                                                            0x0343
+#define regSDMA_RLC6_RB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC6_RB_RPTR_HI                                                                         0x0344
+#define regSDMA_RLC6_RB_RPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC6_RB_WPTR                                                                            0x0345
+#define regSDMA_RLC6_RB_WPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC6_RB_WPTR_HI                                                                         0x0346
+#define regSDMA_RLC6_RB_WPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC6_RB_WPTR_POLL_CNTL                                                                  0x0347
+#define regSDMA_RLC6_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
+#define regSDMA_RLC6_RB_RPTR_ADDR_HI                                                                    0x0348
+#define regSDMA_RLC6_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
+#define regSDMA_RLC6_RB_RPTR_ADDR_LO                                                                    0x0349
+#define regSDMA_RLC6_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
+#define regSDMA_RLC6_IB_CNTL                                                                            0x034a
+#define regSDMA_RLC6_IB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC6_IB_RPTR                                                                            0x034b
+#define regSDMA_RLC6_IB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC6_IB_OFFSET                                                                          0x034c
+#define regSDMA_RLC6_IB_OFFSET_BASE_IDX                                                                 0
+#define regSDMA_RLC6_IB_BASE_LO                                                                         0x034d
+#define regSDMA_RLC6_IB_BASE_LO_BASE_IDX                                                                0
+#define regSDMA_RLC6_IB_BASE_HI                                                                         0x034e
+#define regSDMA_RLC6_IB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC6_IB_SIZE                                                                            0x034f
+#define regSDMA_RLC6_IB_SIZE_BASE_IDX                                                                   0
+#define regSDMA_RLC6_SKIP_CNTL                                                                          0x0350
+#define regSDMA_RLC6_SKIP_CNTL_BASE_IDX                                                                 0
+#define regSDMA_RLC6_CONTEXT_STATUS                                                                     0x0351
+#define regSDMA_RLC6_CONTEXT_STATUS_BASE_IDX                                                            0
+#define regSDMA_RLC6_DOORBELL                                                                           0x0352
+#define regSDMA_RLC6_DOORBELL_BASE_IDX                                                                  0
+#define regSDMA_RLC6_STATUS                                                                             0x0368
+#define regSDMA_RLC6_STATUS_BASE_IDX                                                                    0
+#define regSDMA_RLC6_DOORBELL_LOG                                                                       0x0369
+#define regSDMA_RLC6_DOORBELL_LOG_BASE_IDX                                                              0
+#define regSDMA_RLC6_WATERMARK                                                                          0x036a
+#define regSDMA_RLC6_WATERMARK_BASE_IDX                                                                 0
+#define regSDMA_RLC6_DOORBELL_OFFSET                                                                    0x036b
+#define regSDMA_RLC6_DOORBELL_OFFSET_BASE_IDX                                                           0
+#define regSDMA_RLC6_CSA_ADDR_LO                                                                        0x036c
+#define regSDMA_RLC6_CSA_ADDR_LO_BASE_IDX                                                               0
+#define regSDMA_RLC6_CSA_ADDR_HI                                                                        0x036d
+#define regSDMA_RLC6_CSA_ADDR_HI_BASE_IDX                                                               0
+#define regSDMA_RLC6_IB_SUB_REMAIN                                                                      0x036f
+#define regSDMA_RLC6_IB_SUB_REMAIN_BASE_IDX                                                             0
+#define regSDMA_RLC6_PREEMPT                                                                            0x0370
+#define regSDMA_RLC6_PREEMPT_BASE_IDX                                                                   0
+#define regSDMA_RLC6_DUMMY_REG                                                                          0x0371
+#define regSDMA_RLC6_DUMMY_REG_BASE_IDX                                                                 0
+#define regSDMA_RLC6_RB_WPTR_POLL_ADDR_HI                                                               0x0372
+#define regSDMA_RLC6_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
+#define regSDMA_RLC6_RB_WPTR_POLL_ADDR_LO                                                               0x0373
+#define regSDMA_RLC6_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
+#define regSDMA_RLC6_RB_AQL_CNTL                                                                        0x0374
+#define regSDMA_RLC6_RB_AQL_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC6_MINOR_PTR_UPDATE                                                                   0x0375
+#define regSDMA_RLC6_MINOR_PTR_UPDATE_BASE_IDX                                                          0
+#define regSDMA_RLC6_MIDCMD_DATA0                                                                       0x0380
+#define regSDMA_RLC6_MIDCMD_DATA0_BASE_IDX                                                              0
+#define regSDMA_RLC6_MIDCMD_DATA1                                                                       0x0381
+#define regSDMA_RLC6_MIDCMD_DATA1_BASE_IDX                                                              0
+#define regSDMA_RLC6_MIDCMD_DATA2                                                                       0x0382
+#define regSDMA_RLC6_MIDCMD_DATA2_BASE_IDX                                                              0
+#define regSDMA_RLC6_MIDCMD_DATA3                                                                       0x0383
+#define regSDMA_RLC6_MIDCMD_DATA3_BASE_IDX                                                              0
+#define regSDMA_RLC6_MIDCMD_DATA4                                                                       0x0384
+#define regSDMA_RLC6_MIDCMD_DATA4_BASE_IDX                                                              0
+#define regSDMA_RLC6_MIDCMD_DATA5                                                                       0x0385
+#define regSDMA_RLC6_MIDCMD_DATA5_BASE_IDX                                                              0
+#define regSDMA_RLC6_MIDCMD_DATA6                                                                       0x0386
+#define regSDMA_RLC6_MIDCMD_DATA6_BASE_IDX                                                              0
+#define regSDMA_RLC6_MIDCMD_DATA7                                                                       0x0387
+#define regSDMA_RLC6_MIDCMD_DATA7_BASE_IDX                                                              0
+#define regSDMA_RLC6_MIDCMD_DATA8                                                                       0x0388
+#define regSDMA_RLC6_MIDCMD_DATA8_BASE_IDX                                                              0
+#define regSDMA_RLC6_MIDCMD_DATA9                                                                       0x0389
+#define regSDMA_RLC6_MIDCMD_DATA9_BASE_IDX                                                              0
+#define regSDMA_RLC6_MIDCMD_DATA10                                                                      0x038a
+#define regSDMA_RLC6_MIDCMD_DATA10_BASE_IDX                                                             0
+#define regSDMA_RLC6_MIDCMD_CNTL                                                                        0x038b
+#define regSDMA_RLC6_MIDCMD_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC7_RB_CNTL                                                                            0x0398
+#define regSDMA_RLC7_RB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC7_RB_BASE                                                                            0x0399
+#define regSDMA_RLC7_RB_BASE_BASE_IDX                                                                   0
+#define regSDMA_RLC7_RB_BASE_HI                                                                         0x039a
+#define regSDMA_RLC7_RB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC7_RB_RPTR                                                                            0x039b
+#define regSDMA_RLC7_RB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC7_RB_RPTR_HI                                                                         0x039c
+#define regSDMA_RLC7_RB_RPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC7_RB_WPTR                                                                            0x039d
+#define regSDMA_RLC7_RB_WPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC7_RB_WPTR_HI                                                                         0x039e
+#define regSDMA_RLC7_RB_WPTR_HI_BASE_IDX                                                                0
+#define regSDMA_RLC7_RB_WPTR_POLL_CNTL                                                                  0x039f
+#define regSDMA_RLC7_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
+#define regSDMA_RLC7_RB_RPTR_ADDR_HI                                                                    0x03a0
+#define regSDMA_RLC7_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
+#define regSDMA_RLC7_RB_RPTR_ADDR_LO                                                                    0x03a1
+#define regSDMA_RLC7_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
+#define regSDMA_RLC7_IB_CNTL                                                                            0x03a2
+#define regSDMA_RLC7_IB_CNTL_BASE_IDX                                                                   0
+#define regSDMA_RLC7_IB_RPTR                                                                            0x03a3
+#define regSDMA_RLC7_IB_RPTR_BASE_IDX                                                                   0
+#define regSDMA_RLC7_IB_OFFSET                                                                          0x03a4
+#define regSDMA_RLC7_IB_OFFSET_BASE_IDX                                                                 0
+#define regSDMA_RLC7_IB_BASE_LO                                                                         0x03a5
+#define regSDMA_RLC7_IB_BASE_LO_BASE_IDX                                                                0
+#define regSDMA_RLC7_IB_BASE_HI                                                                         0x03a6
+#define regSDMA_RLC7_IB_BASE_HI_BASE_IDX                                                                0
+#define regSDMA_RLC7_IB_SIZE                                                                            0x03a7
+#define regSDMA_RLC7_IB_SIZE_BASE_IDX                                                                   0
+#define regSDMA_RLC7_SKIP_CNTL                                                                          0x03a8
+#define regSDMA_RLC7_SKIP_CNTL_BASE_IDX                                                                 0
+#define regSDMA_RLC7_CONTEXT_STATUS                                                                     0x03a9
+#define regSDMA_RLC7_CONTEXT_STATUS_BASE_IDX                                                            0
+#define regSDMA_RLC7_DOORBELL                                                                           0x03aa
+#define regSDMA_RLC7_DOORBELL_BASE_IDX                                                                  0
+#define regSDMA_RLC7_STATUS                                                                             0x03c0
+#define regSDMA_RLC7_STATUS_BASE_IDX                                                                    0
+#define regSDMA_RLC7_DOORBELL_LOG                                                                       0x03c1
+#define regSDMA_RLC7_DOORBELL_LOG_BASE_IDX                                                              0
+#define regSDMA_RLC7_WATERMARK                                                                          0x03c2
+#define regSDMA_RLC7_WATERMARK_BASE_IDX                                                                 0
+#define regSDMA_RLC7_DOORBELL_OFFSET                                                                    0x03c3
+#define regSDMA_RLC7_DOORBELL_OFFSET_BASE_IDX                                                           0
+#define regSDMA_RLC7_CSA_ADDR_LO                                                                        0x03c4
+#define regSDMA_RLC7_CSA_ADDR_LO_BASE_IDX                                                               0
+#define regSDMA_RLC7_CSA_ADDR_HI                                                                        0x03c5
+#define regSDMA_RLC7_CSA_ADDR_HI_BASE_IDX                                                               0
+#define regSDMA_RLC7_IB_SUB_REMAIN                                                                      0x03c7
+#define regSDMA_RLC7_IB_SUB_REMAIN_BASE_IDX                                                             0
+#define regSDMA_RLC7_PREEMPT                                                                            0x03c8
+#define regSDMA_RLC7_PREEMPT_BASE_IDX                                                                   0
+#define regSDMA_RLC7_DUMMY_REG                                                                          0x03c9
+#define regSDMA_RLC7_DUMMY_REG_BASE_IDX                                                                 0
+#define regSDMA_RLC7_RB_WPTR_POLL_ADDR_HI                                                               0x03ca
+#define regSDMA_RLC7_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
+#define regSDMA_RLC7_RB_WPTR_POLL_ADDR_LO                                                               0x03cb
+#define regSDMA_RLC7_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
+#define regSDMA_RLC7_RB_AQL_CNTL                                                                        0x03cc
+#define regSDMA_RLC7_RB_AQL_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC7_MINOR_PTR_UPDATE                                                                   0x03cd
+#define regSDMA_RLC7_MINOR_PTR_UPDATE_BASE_IDX                                                          0
+#define regSDMA_RLC7_MIDCMD_DATA0                                                                       0x03d8
+#define regSDMA_RLC7_MIDCMD_DATA0_BASE_IDX                                                              0
+#define regSDMA_RLC7_MIDCMD_DATA1                                                                       0x03d9
+#define regSDMA_RLC7_MIDCMD_DATA1_BASE_IDX                                                              0
+#define regSDMA_RLC7_MIDCMD_DATA2                                                                       0x03da
+#define regSDMA_RLC7_MIDCMD_DATA2_BASE_IDX                                                              0
+#define regSDMA_RLC7_MIDCMD_DATA3                                                                       0x03db
+#define regSDMA_RLC7_MIDCMD_DATA3_BASE_IDX                                                              0
+#define regSDMA_RLC7_MIDCMD_DATA4                                                                       0x03dc
+#define regSDMA_RLC7_MIDCMD_DATA4_BASE_IDX                                                              0
+#define regSDMA_RLC7_MIDCMD_DATA5                                                                       0x03dd
+#define regSDMA_RLC7_MIDCMD_DATA5_BASE_IDX                                                              0
+#define regSDMA_RLC7_MIDCMD_DATA6                                                                       0x03de
+#define regSDMA_RLC7_MIDCMD_DATA6_BASE_IDX                                                              0
+#define regSDMA_RLC7_MIDCMD_DATA7                                                                       0x03df
+#define regSDMA_RLC7_MIDCMD_DATA7_BASE_IDX                                                              0
+#define regSDMA_RLC7_MIDCMD_DATA8                                                                       0x03e0
+#define regSDMA_RLC7_MIDCMD_DATA8_BASE_IDX                                                              0
+#define regSDMA_RLC7_MIDCMD_DATA9                                                                       0x03e1
+#define regSDMA_RLC7_MIDCMD_DATA9_BASE_IDX                                                              0
+#define regSDMA_RLC7_MIDCMD_DATA10                                                                      0x03e2
+#define regSDMA_RLC7_MIDCMD_DATA10_BASE_IDX                                                             0
+#define regSDMA_RLC7_MIDCMD_CNTL                                                                        0x03e3
+#define regSDMA_RLC7_MIDCMD_CNTL_BASE_IDX                                                               0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/sdma/sdma_4_4_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/sdma/sdma_4_4_2_sh_mask.h
new file mode 100644
index 000000000000..e46cb3339355
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/sdma/sdma_4_4_2_sh_mask.h
@@ -0,0 +1,3276 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _sdma_4_4_2_SH_MASK_HEADER
+#define _sdma_4_4_2_SH_MASK_HEADER
+
+
+// addressBlock: aid_sdma_insts_sdma0_sdmadec
+//SDMA_UCODE_ADDR
+#define SDMA_UCODE_ADDR__VALUE__SHIFT                                                                         0x0
+#define SDMA_UCODE_ADDR__VALUE_MASK                                                                           0x00003FFFL
+//SDMA_UCODE_DATA
+#define SDMA_UCODE_DATA__VALUE__SHIFT                                                                         0x0
+#define SDMA_UCODE_DATA__VALUE_MASK                                                                           0xFFFFFFFFL
+//SDMA_F32_CNTL
+#define SDMA_F32_CNTL__HALT__SHIFT                                                                            0x0
+#define SDMA_F32_CNTL__STEP__SHIFT                                                                            0x1
+#define SDMA_F32_CNTL__DBG_SELECT_BITS__SHIFT                                                                 0x2
+#define SDMA_F32_CNTL__RESET__SHIFT                                                                           0x8
+#define SDMA_F32_CNTL__CHECKSUM_CLR__SHIFT                                                                    0x9
+#define SDMA_F32_CNTL__HALT_MASK                                                                              0x00000001L
+#define SDMA_F32_CNTL__STEP_MASK                                                                              0x00000002L
+#define SDMA_F32_CNTL__DBG_SELECT_BITS_MASK                                                                   0x000000FCL
+#define SDMA_F32_CNTL__RESET_MASK                                                                             0x00000100L
+#define SDMA_F32_CNTL__CHECKSUM_CLR_MASK                                                                      0x00000200L
+//SDMA_MMHUB_CNTL
+#define SDMA_MMHUB_CNTL__UNIT_ID__SHIFT                                                                       0x0
+#define SDMA_MMHUB_CNTL__UNIT_ID_MASK                                                                         0x0000003FL
+//SDMA_MMHUB_TRUSTLVL
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG0__SHIFT                                                                  0x0
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG1__SHIFT                                                                  0x4
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG2__SHIFT                                                                  0x8
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG3__SHIFT                                                                  0xc
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG4__SHIFT                                                                  0x10
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG5__SHIFT                                                                  0x14
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG6__SHIFT                                                                  0x18
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG7__SHIFT                                                                  0x1c
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG0_MASK                                                                    0x0000000FL
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG1_MASK                                                                    0x000000F0L
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG2_MASK                                                                    0x00000F00L
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG3_MASK                                                                    0x0000F000L
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG4_MASK                                                                    0x000F0000L
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG5_MASK                                                                    0x00F00000L
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG6_MASK                                                                    0x0F000000L
+#define SDMA_MMHUB_TRUSTLVL__SECFLAG7_MASK                                                                    0xF0000000L
+//SDMA_VM_CNTL
+#define SDMA_VM_CNTL__CMD__SHIFT                                                                              0x0
+#define SDMA_VM_CNTL__CMD_MASK                                                                                0x0000000FL
+//SDMA_VM_CTX_LO
+#define SDMA_VM_CTX_LO__ADDR__SHIFT                                                                           0x2
+#define SDMA_VM_CTX_LO__ADDR_MASK                                                                             0xFFFFFFFCL
+//SDMA_VM_CTX_HI
+#define SDMA_VM_CTX_HI__ADDR__SHIFT                                                                           0x0
+#define SDMA_VM_CTX_HI__ADDR_MASK                                                                             0xFFFFFFFFL
+//SDMA_ACTIVE_FCN_ID
+#define SDMA_ACTIVE_FCN_ID__VFID__SHIFT                                                                       0x0
+#define SDMA_ACTIVE_FCN_ID__RESERVED__SHIFT                                                                   0x4
+#define SDMA_ACTIVE_FCN_ID__VF__SHIFT                                                                         0x1f
+#define SDMA_ACTIVE_FCN_ID__VFID_MASK                                                                         0x0000000FL
+#define SDMA_ACTIVE_FCN_ID__RESERVED_MASK                                                                     0x7FFFFFF0L
+#define SDMA_ACTIVE_FCN_ID__VF_MASK                                                                           0x80000000L
+//SDMA_VM_CTX_CNTL
+#define SDMA_VM_CTX_CNTL__PRIV__SHIFT                                                                         0x0
+#define SDMA_VM_CTX_CNTL__VMID__SHIFT                                                                         0x4
+#define SDMA_VM_CTX_CNTL__PRIV_MASK                                                                           0x00000001L
+#define SDMA_VM_CTX_CNTL__VMID_MASK                                                                           0x000000F0L
+//SDMA_VIRT_RESET_REQ
+#define SDMA_VIRT_RESET_REQ__VF__SHIFT                                                                        0x0
+#define SDMA_VIRT_RESET_REQ__PF__SHIFT                                                                        0x1f
+#define SDMA_VIRT_RESET_REQ__VF_MASK                                                                          0x0000FFFFL
+#define SDMA_VIRT_RESET_REQ__PF_MASK                                                                          0x80000000L
+//SDMA_VF_ENABLE
+#define SDMA_VF_ENABLE__VF_ENABLE__SHIFT                                                                      0x0
+#define SDMA_VF_ENABLE__VF_ENABLE_MASK                                                                        0x00000001L
+//SDMA_CONTEXT_REG_TYPE0
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_CNTL__SHIFT                                                       0x0
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_BASE__SHIFT                                                       0x1
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_BASE_HI__SHIFT                                                    0x2
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_RPTR__SHIFT                                                       0x3
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_RPTR_HI__SHIFT                                                    0x4
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_WPTR__SHIFT                                                       0x5
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_WPTR_HI__SHIFT                                                    0x6
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_WPTR_POLL_CNTL__SHIFT                                             0x7
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_RPTR_ADDR_HI__SHIFT                                               0x8
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_RPTR_ADDR_LO__SHIFT                                               0x9
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_CNTL__SHIFT                                                       0xa
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_RPTR__SHIFT                                                       0xb
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_OFFSET__SHIFT                                                     0xc
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_BASE_LO__SHIFT                                                    0xd
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_BASE_HI__SHIFT                                                    0xe
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_SIZE__SHIFT                                                       0xf
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_SKIP_CNTL__SHIFT                                                     0x10
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_CONTEXT_STATUS__SHIFT                                                0x11
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_DOORBELL__SHIFT                                                      0x12
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_CONTEXT_CNTL__SHIFT                                                  0x13
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_CNTL_MASK                                                         0x00000001L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_BASE_MASK                                                         0x00000002L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_BASE_HI_MASK                                                      0x00000004L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_RPTR_MASK                                                         0x00000008L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_RPTR_HI_MASK                                                      0x00000010L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_WPTR_MASK                                                         0x00000020L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_WPTR_HI_MASK                                                      0x00000040L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_WPTR_POLL_CNTL_MASK                                               0x00000080L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_RPTR_ADDR_HI_MASK                                                 0x00000100L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_RB_RPTR_ADDR_LO_MASK                                                 0x00000200L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_CNTL_MASK                                                         0x00000400L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_RPTR_MASK                                                         0x00000800L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_OFFSET_MASK                                                       0x00001000L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_BASE_LO_MASK                                                      0x00002000L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_BASE_HI_MASK                                                      0x00004000L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_IB_SIZE_MASK                                                         0x00008000L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_SKIP_CNTL_MASK                                                       0x00010000L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_CONTEXT_STATUS_MASK                                                  0x00020000L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_DOORBELL_MASK                                                        0x00040000L
+#define SDMA_CONTEXT_REG_TYPE0__SDMA_GFX_CONTEXT_CNTL_MASK                                                    0x00080000L
+//SDMA_CONTEXT_REG_TYPE1
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_STATUS__SHIFT                                                        0x8
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_DOORBELL_LOG__SHIFT                                                  0x9
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_WATERMARK__SHIFT                                                     0xa
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_DOORBELL_OFFSET__SHIFT                                               0xb
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_CSA_ADDR_LO__SHIFT                                                   0xc
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_CSA_ADDR_HI__SHIFT                                                   0xd
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_IB_SUB_REMAIN__SHIFT                                                 0xf
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_PREEMPT__SHIFT                                                       0x10
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_DUMMY_REG__SHIFT                                                     0x11
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_RB_WPTR_POLL_ADDR_HI__SHIFT                                          0x12
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_RB_WPTR_POLL_ADDR_LO__SHIFT                                          0x13
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_RB_AQL_CNTL__SHIFT                                                   0x14
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_MINOR_PTR_UPDATE__SHIFT                                              0x15
+#define SDMA_CONTEXT_REG_TYPE1__RESERVED__SHIFT                                                               0x16
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_STATUS_MASK                                                          0x00000100L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_DOORBELL_LOG_MASK                                                    0x00000200L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_WATERMARK_MASK                                                       0x00000400L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_DOORBELL_OFFSET_MASK                                                 0x00000800L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_CSA_ADDR_LO_MASK                                                     0x00001000L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_CSA_ADDR_HI_MASK                                                     0x00002000L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_IB_SUB_REMAIN_MASK                                                   0x00008000L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_PREEMPT_MASK                                                         0x00010000L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_DUMMY_REG_MASK                                                       0x00020000L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_RB_WPTR_POLL_ADDR_HI_MASK                                            0x00040000L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_RB_WPTR_POLL_ADDR_LO_MASK                                            0x00080000L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_RB_AQL_CNTL_MASK                                                     0x00100000L
+#define SDMA_CONTEXT_REG_TYPE1__SDMA_GFX_MINOR_PTR_UPDATE_MASK                                                0x00200000L
+#define SDMA_CONTEXT_REG_TYPE1__RESERVED_MASK                                                                 0xFFC00000L
+//SDMA_CONTEXT_REG_TYPE2
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA0__SHIFT                                                  0x0
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA1__SHIFT                                                  0x1
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA2__SHIFT                                                  0x2
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA3__SHIFT                                                  0x3
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA4__SHIFT                                                  0x4
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA5__SHIFT                                                  0x5
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA6__SHIFT                                                  0x6
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA7__SHIFT                                                  0x7
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA8__SHIFT                                                  0x8
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA9__SHIFT                                                  0x9
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA10__SHIFT                                                 0xa
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_CNTL__SHIFT                                                   0xb
+#define SDMA_CONTEXT_REG_TYPE2__RESERVED__SHIFT                                                               0xe
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA0_MASK                                                    0x00000001L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA1_MASK                                                    0x00000002L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA2_MASK                                                    0x00000004L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA3_MASK                                                    0x00000008L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA4_MASK                                                    0x00000010L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA5_MASK                                                    0x00000020L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA6_MASK                                                    0x00000040L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA7_MASK                                                    0x00000080L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA8_MASK                                                    0x00000100L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA9_MASK                                                    0x00000200L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_DATA10_MASK                                                   0x00000400L
+#define SDMA_CONTEXT_REG_TYPE2__SDMA_GFX_MIDCMD_CNTL_MASK                                                     0x00000800L
+#define SDMA_CONTEXT_REG_TYPE2__RESERVED_MASK                                                                 0xFFFFC000L
+//SDMA_CONTEXT_REG_TYPE3
+#define SDMA_CONTEXT_REG_TYPE3__RESERVED__SHIFT                                                               0x0
+#define SDMA_CONTEXT_REG_TYPE3__RESERVED_MASK                                                                 0xFFFFFFFFL
+//SDMA_PUB_REG_TYPE0
+#define SDMA_PUB_REG_TYPE0__SDMA_UCODE_ADDR__SHIFT                                                            0x0
+#define SDMA_PUB_REG_TYPE0__SDMA_UCODE_DATA__SHIFT                                                            0x1
+#define SDMA_PUB_REG_TYPE0__SDMA_F32_CNTL__SHIFT                                                              0x2
+#define SDMA_PUB_REG_TYPE0__SDMA_MMHUB_CNTL__SHIFT                                                            0x5
+#define SDMA_PUB_REG_TYPE0__SDMA_MMHUB_TRUSTLVL__SHIFT                                                        0x6
+#define SDMA_PUB_REG_TYPE0__RESERVED_14_10__SHIFT                                                             0xa
+#define SDMA_PUB_REG_TYPE0__SDMA_VM_CNTL__SHIFT                                                               0x10
+#define SDMA_PUB_REG_TYPE0__SDMA_VM_CTX_LO__SHIFT                                                             0x11
+#define SDMA_PUB_REG_TYPE0__SDMA_VM_CTX_HI__SHIFT                                                             0x12
+#define SDMA_PUB_REG_TYPE0__SDMA_ACTIVE_FCN_ID__SHIFT                                                         0x13
+#define SDMA_PUB_REG_TYPE0__SDMA_VM_CTX_CNTL__SHIFT                                                           0x14
+#define SDMA_PUB_REG_TYPE0__SDMA_VIRT_RESET_REQ__SHIFT                                                        0x15
+#define SDMA_PUB_REG_TYPE0__SDMA_VF_ENABLE__SHIFT                                                             0x16
+#define SDMA_PUB_REG_TYPE0__SDMA_CONTEXT_REG_TYPE0__SHIFT                                                     0x17
+#define SDMA_PUB_REG_TYPE0__SDMA_CONTEXT_REG_TYPE1__SHIFT                                                     0x18
+#define SDMA_PUB_REG_TYPE0__SDMA_CONTEXT_REG_TYPE2__SHIFT                                                     0x19
+#define SDMA_PUB_REG_TYPE0__SDMA_CONTEXT_REG_TYPE3__SHIFT                                                     0x1a
+#define SDMA_PUB_REG_TYPE0__SDMA_PUB_REG_TYPE0__SHIFT                                                         0x1b
+#define SDMA_PUB_REG_TYPE0__SDMA_PUB_REG_TYPE1__SHIFT                                                         0x1c
+#define SDMA_PUB_REG_TYPE0__SDMA_PUB_REG_TYPE2__SHIFT                                                         0x1d
+#define SDMA_PUB_REG_TYPE0__SDMA_PUB_REG_TYPE3__SHIFT                                                         0x1e
+#define SDMA_PUB_REG_TYPE0__SDMA_CONTEXT_GROUP_BOUNDARY__SHIFT                                                0x1f
+#define SDMA_PUB_REG_TYPE0__SDMA_UCODE_ADDR_MASK                                                              0x00000001L
+#define SDMA_PUB_REG_TYPE0__SDMA_UCODE_DATA_MASK                                                              0x00000002L
+#define SDMA_PUB_REG_TYPE0__SDMA_F32_CNTL_MASK                                                                0x00000004L
+#define SDMA_PUB_REG_TYPE0__SDMA_MMHUB_CNTL_MASK                                                              0x00000020L
+#define SDMA_PUB_REG_TYPE0__SDMA_MMHUB_TRUSTLVL_MASK                                                          0x00000040L
+#define SDMA_PUB_REG_TYPE0__RESERVED_14_10_MASK                                                               0x00007C00L
+#define SDMA_PUB_REG_TYPE0__SDMA_VM_CNTL_MASK                                                                 0x00010000L
+#define SDMA_PUB_REG_TYPE0__SDMA_VM_CTX_LO_MASK                                                               0x00020000L
+#define SDMA_PUB_REG_TYPE0__SDMA_VM_CTX_HI_MASK                                                               0x00040000L
+#define SDMA_PUB_REG_TYPE0__SDMA_ACTIVE_FCN_ID_MASK                                                           0x00080000L
+#define SDMA_PUB_REG_TYPE0__SDMA_VM_CTX_CNTL_MASK                                                             0x00100000L
+#define SDMA_PUB_REG_TYPE0__SDMA_VIRT_RESET_REQ_MASK                                                          0x00200000L
+#define SDMA_PUB_REG_TYPE0__SDMA_VF_ENABLE_MASK                                                               0x00400000L
+#define SDMA_PUB_REG_TYPE0__SDMA_CONTEXT_REG_TYPE0_MASK                                                       0x00800000L
+#define SDMA_PUB_REG_TYPE0__SDMA_CONTEXT_REG_TYPE1_MASK                                                       0x01000000L
+#define SDMA_PUB_REG_TYPE0__SDMA_CONTEXT_REG_TYPE2_MASK                                                       0x02000000L
+#define SDMA_PUB_REG_TYPE0__SDMA_CONTEXT_REG_TYPE3_MASK                                                       0x04000000L
+#define SDMA_PUB_REG_TYPE0__SDMA_PUB_REG_TYPE0_MASK                                                           0x08000000L
+#define SDMA_PUB_REG_TYPE0__SDMA_PUB_REG_TYPE1_MASK                                                           0x10000000L
+#define SDMA_PUB_REG_TYPE0__SDMA_PUB_REG_TYPE2_MASK                                                           0x20000000L
+#define SDMA_PUB_REG_TYPE0__SDMA_PUB_REG_TYPE3_MASK                                                           0x40000000L
+#define SDMA_PUB_REG_TYPE0__SDMA_CONTEXT_GROUP_BOUNDARY_MASK                                                  0x80000000L
+//SDMA_PUB_REG_TYPE1
+#define SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH_HI__SHIFT                                                      0x0
+#define SDMA_PUB_REG_TYPE1__SDMA_SEM_WAIT_FAIL_TIMER_CNTL__SHIFT                                              0x1
+#define SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH__SHIFT                                                         0x2
+#define SDMA_PUB_REG_TYPE1__SDMA_IB_OFFSET_FETCH__SHIFT                                                       0x3
+#define SDMA_PUB_REG_TYPE1__SDMA_PROGRAM__SHIFT                                                               0x4
+#define SDMA_PUB_REG_TYPE1__SDMA_STATUS_REG__SHIFT                                                            0x5
+#define SDMA_PUB_REG_TYPE1__SDMA_STATUS1_REG__SHIFT                                                           0x6
+#define SDMA_PUB_REG_TYPE1__SDMA_RD_BURST_CNTL__SHIFT                                                         0x7
+#define SDMA_PUB_REG_TYPE1__SDMA_HBM_PAGE_CONFIG__SHIFT                                                       0x8
+#define SDMA_PUB_REG_TYPE1__SDMA_UCODE_CHECKSUM__SHIFT                                                        0x9
+#define SDMA_PUB_REG_TYPE1__RESERVED_10_10__SHIFT                                                             0xa
+#define SDMA_PUB_REG_TYPE1__SDMA_FREEZE__SHIFT                                                                0xb
+#define SDMA_PUB_REG_TYPE1__SDMA_PHASE0_QUANTUM__SHIFT                                                        0xc
+#define SDMA_PUB_REG_TYPE1__SDMA_PHASE1_QUANTUM__SHIFT                                                        0xd
+#define SDMA_PUB_REG_TYPE1__SDMA_POWER_GATING__SHIFT                                                          0xe
+#define SDMA_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG__SHIFT                                                          0xf
+#define SDMA_PUB_REG_TYPE1__SDMA_PGFSM_WRITE__SHIFT                                                           0x10
+#define SDMA_PUB_REG_TYPE1__SDMA_PGFSM_READ__SHIFT                                                            0x11
+#define SDMA_PUB_REG_TYPE1__CC_SDMA_EDC_CONFIG__SHIFT                                                         0x12
+#define SDMA_PUB_REG_TYPE1__SDMA_BA_THRESHOLD__SHIFT                                                          0x13
+#define SDMA_PUB_REG_TYPE1__SDMA_ID__SHIFT                                                                    0x14
+#define SDMA_PUB_REG_TYPE1__SDMA_VERSION__SHIFT                                                               0x15
+#define SDMA_PUB_REG_TYPE1__SDMA_EDC_COUNTER__SHIFT                                                           0x16
+#define SDMA_PUB_REG_TYPE1__SDMA_EDC_COUNTER2__SHIFT                                                          0x17
+#define SDMA_PUB_REG_TYPE1__SDMA_STATUS2_REG__SHIFT                                                           0x18
+#define SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_CNTL__SHIFT                                                           0x19
+#define SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_LO__SHIFT                                                       0x1a
+#define SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_HI__SHIFT                                                       0x1b
+#define SDMA_PUB_REG_TYPE1__SDMA_UTCL1_CNTL__SHIFT                                                            0x1c
+#define SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WATERMK__SHIFT                                                         0x1d
+#define SDMA_PUB_REG_TYPE1__SDMA_UTCL1_RD_STATUS__SHIFT                                                       0x1e
+#define SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WR_STATUS__SHIFT                                                       0x1f
+#define SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH_HI_MASK                                                        0x00000001L
+#define SDMA_PUB_REG_TYPE1__SDMA_SEM_WAIT_FAIL_TIMER_CNTL_MASK                                                0x00000002L
+#define SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH_MASK                                                           0x00000004L
+#define SDMA_PUB_REG_TYPE1__SDMA_IB_OFFSET_FETCH_MASK                                                         0x00000008L
+#define SDMA_PUB_REG_TYPE1__SDMA_PROGRAM_MASK                                                                 0x00000010L
+#define SDMA_PUB_REG_TYPE1__SDMA_STATUS_REG_MASK                                                              0x00000020L
+#define SDMA_PUB_REG_TYPE1__SDMA_STATUS1_REG_MASK                                                             0x00000040L
+#define SDMA_PUB_REG_TYPE1__SDMA_RD_BURST_CNTL_MASK                                                           0x00000080L
+#define SDMA_PUB_REG_TYPE1__SDMA_HBM_PAGE_CONFIG_MASK                                                         0x00000100L
+#define SDMA_PUB_REG_TYPE1__SDMA_UCODE_CHECKSUM_MASK                                                          0x00000200L
+#define SDMA_PUB_REG_TYPE1__RESERVED_10_10_MASK                                                               0x00000400L
+#define SDMA_PUB_REG_TYPE1__SDMA_FREEZE_MASK                                                                  0x00000800L
+#define SDMA_PUB_REG_TYPE1__SDMA_PHASE0_QUANTUM_MASK                                                          0x00001000L
+#define SDMA_PUB_REG_TYPE1__SDMA_PHASE1_QUANTUM_MASK                                                          0x00002000L
+#define SDMA_PUB_REG_TYPE1__SDMA_POWER_GATING_MASK                                                            0x00004000L
+#define SDMA_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG_MASK                                                            0x00008000L
+#define SDMA_PUB_REG_TYPE1__SDMA_PGFSM_WRITE_MASK                                                             0x00010000L
+#define SDMA_PUB_REG_TYPE1__SDMA_PGFSM_READ_MASK                                                              0x00020000L
+#define SDMA_PUB_REG_TYPE1__CC_SDMA_EDC_CONFIG_MASK                                                           0x00040000L
+#define SDMA_PUB_REG_TYPE1__SDMA_BA_THRESHOLD_MASK                                                            0x00080000L
+#define SDMA_PUB_REG_TYPE1__SDMA_ID_MASK                                                                      0x00100000L
+#define SDMA_PUB_REG_TYPE1__SDMA_VERSION_MASK                                                                 0x00200000L
+#define SDMA_PUB_REG_TYPE1__SDMA_EDC_COUNTER_MASK                                                             0x00400000L
+#define SDMA_PUB_REG_TYPE1__SDMA_EDC_COUNTER2_MASK                                                            0x00800000L
+#define SDMA_PUB_REG_TYPE1__SDMA_STATUS2_REG_MASK                                                             0x01000000L
+#define SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_CNTL_MASK                                                             0x02000000L
+#define SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_LO_MASK                                                         0x04000000L
+#define SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_HI_MASK                                                         0x08000000L
+#define SDMA_PUB_REG_TYPE1__SDMA_UTCL1_CNTL_MASK                                                              0x10000000L
+#define SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WATERMK_MASK                                                           0x20000000L
+#define SDMA_PUB_REG_TYPE1__SDMA_UTCL1_RD_STATUS_MASK                                                         0x40000000L
+#define SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WR_STATUS_MASK                                                         0x80000000L
+//SDMA_PUB_REG_TYPE2
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV0__SHIFT                                                            0x0
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV1__SHIFT                                                            0x1
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV2__SHIFT                                                            0x2
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK0__SHIFT                                                       0x3
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK1__SHIFT                                                       0x4
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK0__SHIFT                                                       0x5
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK1__SHIFT                                                       0x6
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_TIMEOUT__SHIFT                                                         0x7
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_PAGE__SHIFT                                                            0x8
+#define SDMA_PUB_REG_TYPE2__SDMA_POWER_CNTL_IDLE__SHIFT                                                       0x9
+#define SDMA_PUB_REG_TYPE2__SDMA_RELAX_ORDERING_LUT__SHIFT                                                    0xa
+#define SDMA_PUB_REG_TYPE2__SDMA_CHICKEN_BITS_2__SHIFT                                                        0xb
+#define SDMA_PUB_REG_TYPE2__SDMA_STATUS3_REG__SHIFT                                                           0xc
+#define SDMA_PUB_REG_TYPE2__SDMA_PHYSICAL_ADDR_LO__SHIFT                                                      0xd
+#define SDMA_PUB_REG_TYPE2__SDMA_PHYSICAL_ADDR_HI__SHIFT                                                      0xe
+#define SDMA_PUB_REG_TYPE2__SDMA_PHASE2_QUANTUM__SHIFT                                                        0xf
+#define SDMA_PUB_REG_TYPE2__SDMA_ERROR_LOG__SHIFT                                                             0x10
+#define SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG0__SHIFT                                                        0x11
+#define SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG1__SHIFT                                                        0x12
+#define SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG2__SHIFT                                                        0x13
+#define SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG3__SHIFT                                                        0x14
+#define SDMA_PUB_REG_TYPE2__SDMA_F32_COUNTER__SHIFT                                                           0x15
+#define SDMA_PUB_REG_TYPE2__RESERVED_22_22__SHIFT                                                             0x16
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_PERFCOUNTER0_CFG__SHIFT                                              0x17
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_PERFCOUNTER1_CFG__SHIFT                                              0x18
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__SHIFT                                         0x19
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_MISC_CNTL__SHIFT                                                     0x1a
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_PERFCOUNTER_LO__SHIFT                                                0x1b
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_PERFCOUNTER_HI__SHIFT                                                0x1c
+#define SDMA_PUB_REG_TYPE2__SDMA_CRD_CNTL__SHIFT                                                              0x1d
+#define SDMA_PUB_REG_TYPE2__SDMA_GPU_IOV_VIOLATION_LOG__SHIFT                                                 0x1e
+#define SDMA_PUB_REG_TYPE2__SDMA_ULV_CNTL__SHIFT                                                              0x1f
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV0_MASK                                                              0x00000001L
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV1_MASK                                                              0x00000002L
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV2_MASK                                                              0x00000004L
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK0_MASK                                                         0x00000008L
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK1_MASK                                                         0x00000010L
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK0_MASK                                                         0x00000020L
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK1_MASK                                                         0x00000040L
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_TIMEOUT_MASK                                                           0x00000080L
+#define SDMA_PUB_REG_TYPE2__SDMA_UTCL1_PAGE_MASK                                                              0x00000100L
+#define SDMA_PUB_REG_TYPE2__SDMA_POWER_CNTL_IDLE_MASK                                                         0x00000200L
+#define SDMA_PUB_REG_TYPE2__SDMA_RELAX_ORDERING_LUT_MASK                                                      0x00000400L
+#define SDMA_PUB_REG_TYPE2__SDMA_CHICKEN_BITS_2_MASK                                                          0x00000800L
+#define SDMA_PUB_REG_TYPE2__SDMA_STATUS3_REG_MASK                                                             0x00001000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PHYSICAL_ADDR_LO_MASK                                                        0x00002000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PHYSICAL_ADDR_HI_MASK                                                        0x00004000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PHASE2_QUANTUM_MASK                                                          0x00008000L
+#define SDMA_PUB_REG_TYPE2__SDMA_ERROR_LOG_MASK                                                               0x00010000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG0_MASK                                                          0x00020000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG1_MASK                                                          0x00040000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG2_MASK                                                          0x00080000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG3_MASK                                                          0x00100000L
+#define SDMA_PUB_REG_TYPE2__SDMA_F32_COUNTER_MASK                                                             0x00200000L
+#define SDMA_PUB_REG_TYPE2__RESERVED_22_22_MASK                                                               0x00400000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_PERFCOUNTER0_CFG_MASK                                                0x00800000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_PERFCOUNTER1_CFG_MASK                                                0x01000000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL_MASK                                           0x02000000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_MISC_CNTL_MASK                                                       0x04000000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_PERFCOUNTER_LO_MASK                                                  0x08000000L
+#define SDMA_PUB_REG_TYPE2__SDMA_PERFCNT_PERFCOUNTER_HI_MASK                                                  0x10000000L
+#define SDMA_PUB_REG_TYPE2__SDMA_CRD_CNTL_MASK                                                                0x20000000L
+#define SDMA_PUB_REG_TYPE2__SDMA_GPU_IOV_VIOLATION_LOG_MASK                                                   0x40000000L
+#define SDMA_PUB_REG_TYPE2__SDMA_ULV_CNTL_MASK                                                                0x80000000L
+//SDMA_PUB_REG_TYPE3
+#define SDMA_PUB_REG_TYPE3__SDMA_EA_DBIT_ADDR_DATA__SHIFT                                                     0x0
+#define SDMA_PUB_REG_TYPE3__SDMA_EA_DBIT_ADDR_INDEX__SHIFT                                                    0x1
+#define SDMA_PUB_REG_TYPE3__SDMA_GPU_IOV_VIOLATION_LOG2__SHIFT                                                0x2
+#define SDMA_PUB_REG_TYPE3__SDMA_STATUS4_REG__SHIFT                                                           0x3
+#define SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_DATA__SHIFT                                                      0x4
+#define SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_ADDR__SHIFT                                                      0x5
+#define SDMA_PUB_REG_TYPE3__SDMA_CE_CTRL__SHIFT                                                               0x6
+#define SDMA_PUB_REG_TYPE3__SDMA_RAS_STATUS__SHIFT                                                            0x7
+#define SDMA_PUB_REG_TYPE3__SDMA_CLK_STATUS__SHIFT                                                            0x8
+#define SDMA_PUB_REG_TYPE3__SDMA_POWER_CNTL__SHIFT                                                            0xb
+#define SDMA_PUB_REG_TYPE3__SDMA_CLK_CTRL__SHIFT                                                              0xc
+#define SDMA_PUB_REG_TYPE3__SDMA_CNTL__SHIFT                                                                  0xd
+#define SDMA_PUB_REG_TYPE3__SDMA_CHICKEN_BITS__SHIFT                                                          0xe
+#define SDMA_PUB_REG_TYPE3__SDMA_GB_ADDR_CONFIG__SHIFT                                                        0xf
+#define SDMA_PUB_REG_TYPE3__SDMA_GB_ADDR_CONFIG_READ__SHIFT                                                   0x10
+#define SDMA_PUB_REG_TYPE3__RESERVED__SHIFT                                                                   0x13
+#define SDMA_PUB_REG_TYPE3__SDMA_EA_DBIT_ADDR_DATA_MASK                                                       0x00000001L
+#define SDMA_PUB_REG_TYPE3__SDMA_EA_DBIT_ADDR_INDEX_MASK                                                      0x00000002L
+#define SDMA_PUB_REG_TYPE3__SDMA_GPU_IOV_VIOLATION_LOG2_MASK                                                  0x00000004L
+#define SDMA_PUB_REG_TYPE3__SDMA_STATUS4_REG_MASK                                                             0x00000008L
+#define SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_DATA_MASK                                                        0x00000010L
+#define SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_ADDR_MASK                                                        0x00000020L
+#define SDMA_PUB_REG_TYPE3__SDMA_CE_CTRL_MASK                                                                 0x00000040L
+#define SDMA_PUB_REG_TYPE3__SDMA_RAS_STATUS_MASK                                                              0x00000080L
+#define SDMA_PUB_REG_TYPE3__SDMA_CLK_STATUS_MASK                                                              0x00000100L
+#define SDMA_PUB_REG_TYPE3__SDMA_POWER_CNTL_MASK                                                              0x00000800L
+#define SDMA_PUB_REG_TYPE3__SDMA_CLK_CTRL_MASK                                                                0x00001000L
+#define SDMA_PUB_REG_TYPE3__SDMA_CNTL_MASK                                                                    0x00002000L
+#define SDMA_PUB_REG_TYPE3__SDMA_CHICKEN_BITS_MASK                                                            0x00004000L
+#define SDMA_PUB_REG_TYPE3__SDMA_GB_ADDR_CONFIG_MASK                                                          0x00008000L
+#define SDMA_PUB_REG_TYPE3__SDMA_GB_ADDR_CONFIG_READ_MASK                                                     0x00010000L
+#define SDMA_PUB_REG_TYPE3__RESERVED_MASK                                                                     0xFFF80000L
+//SDMA_CONTEXT_GROUP_BOUNDARY
+#define SDMA_CONTEXT_GROUP_BOUNDARY__RESERVED__SHIFT                                                          0x0
+#define SDMA_CONTEXT_GROUP_BOUNDARY__RESERVED_MASK                                                            0xFFFFFFFFL
+//SDMA_RB_RPTR_FETCH_HI
+#define SDMA_RB_RPTR_FETCH_HI__OFFSET__SHIFT                                                                  0x0
+#define SDMA_RB_RPTR_FETCH_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
+//SDMA_SEM_WAIT_FAIL_TIMER_CNTL
+#define SDMA_SEM_WAIT_FAIL_TIMER_CNTL__TIMER__SHIFT                                                           0x0
+#define SDMA_SEM_WAIT_FAIL_TIMER_CNTL__TIMER_MASK                                                             0xFFFFFFFFL
+//SDMA_RB_RPTR_FETCH
+#define SDMA_RB_RPTR_FETCH__OFFSET__SHIFT                                                                     0x2
+#define SDMA_RB_RPTR_FETCH__OFFSET_MASK                                                                       0xFFFFFFFCL
+//SDMA_IB_OFFSET_FETCH
+#define SDMA_IB_OFFSET_FETCH__OFFSET__SHIFT                                                                   0x2
+#define SDMA_IB_OFFSET_FETCH__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA_PROGRAM
+#define SDMA_PROGRAM__STREAM__SHIFT                                                                           0x0
+#define SDMA_PROGRAM__STREAM_MASK                                                                             0xFFFFFFFFL
+//SDMA_STATUS_REG
+#define SDMA_STATUS_REG__IDLE__SHIFT                                                                          0x0
+#define SDMA_STATUS_REG__REG_IDLE__SHIFT                                                                      0x1
+#define SDMA_STATUS_REG__RB_EMPTY__SHIFT                                                                      0x2
+#define SDMA_STATUS_REG__RB_FULL__SHIFT                                                                       0x3
+#define SDMA_STATUS_REG__RB_CMD_IDLE__SHIFT                                                                   0x4
+#define SDMA_STATUS_REG__RB_CMD_FULL__SHIFT                                                                   0x5
+#define SDMA_STATUS_REG__IB_CMD_IDLE__SHIFT                                                                   0x6
+#define SDMA_STATUS_REG__IB_CMD_FULL__SHIFT                                                                   0x7
+#define SDMA_STATUS_REG__BLOCK_IDLE__SHIFT                                                                    0x8
+#define SDMA_STATUS_REG__INSIDE_IB__SHIFT                                                                     0x9
+#define SDMA_STATUS_REG__EX_IDLE__SHIFT                                                                       0xa
+#define SDMA_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE__SHIFT                                                     0xb
+#define SDMA_STATUS_REG__PACKET_READY__SHIFT                                                                  0xc
+#define SDMA_STATUS_REG__MC_WR_IDLE__SHIFT                                                                    0xd
+#define SDMA_STATUS_REG__SRBM_IDLE__SHIFT                                                                     0xe
+#define SDMA_STATUS_REG__CONTEXT_EMPTY__SHIFT                                                                 0xf
+#define SDMA_STATUS_REG__DELTA_RPTR_FULL__SHIFT                                                               0x10
+#define SDMA_STATUS_REG__RB_MC_RREQ_IDLE__SHIFT                                                               0x11
+#define SDMA_STATUS_REG__IB_MC_RREQ_IDLE__SHIFT                                                               0x12
+#define SDMA_STATUS_REG__MC_RD_IDLE__SHIFT                                                                    0x13
+#define SDMA_STATUS_REG__DELTA_RPTR_EMPTY__SHIFT                                                              0x14
+#define SDMA_STATUS_REG__MC_RD_RET_STALL__SHIFT                                                               0x15
+#define SDMA_STATUS_REG__MC_RD_NO_POLL_IDLE__SHIFT                                                            0x16
+#define SDMA_STATUS_REG__DRM_IDLE__SHIFT                                                                      0x17
+#define SDMA_STATUS_REG__DRM_MASK_FULL__SHIFT                                                                 0x18
+#define SDMA_STATUS_REG__PREV_CMD_IDLE__SHIFT                                                                 0x19
+#define SDMA_STATUS_REG__SEM_IDLE__SHIFT                                                                      0x1a
+#define SDMA_STATUS_REG__SEM_REQ_STALL__SHIFT                                                                 0x1b
+#define SDMA_STATUS_REG__SEM_RESP_STATE__SHIFT                                                                0x1c
+#define SDMA_STATUS_REG__INT_IDLE__SHIFT                                                                      0x1e
+#define SDMA_STATUS_REG__INT_REQ_STALL__SHIFT                                                                 0x1f
+#define SDMA_STATUS_REG__IDLE_MASK                                                                            0x00000001L
+#define SDMA_STATUS_REG__REG_IDLE_MASK                                                                        0x00000002L
+#define SDMA_STATUS_REG__RB_EMPTY_MASK                                                                        0x00000004L
+#define SDMA_STATUS_REG__RB_FULL_MASK                                                                         0x00000008L
+#define SDMA_STATUS_REG__RB_CMD_IDLE_MASK                                                                     0x00000010L
+#define SDMA_STATUS_REG__RB_CMD_FULL_MASK                                                                     0x00000020L
+#define SDMA_STATUS_REG__IB_CMD_IDLE_MASK                                                                     0x00000040L
+#define SDMA_STATUS_REG__IB_CMD_FULL_MASK                                                                     0x00000080L
+#define SDMA_STATUS_REG__BLOCK_IDLE_MASK                                                                      0x00000100L
+#define SDMA_STATUS_REG__INSIDE_IB_MASK                                                                       0x00000200L
+#define SDMA_STATUS_REG__EX_IDLE_MASK                                                                         0x00000400L
+#define SDMA_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE_MASK                                                       0x00000800L
+#define SDMA_STATUS_REG__PACKET_READY_MASK                                                                    0x00001000L
+#define SDMA_STATUS_REG__MC_WR_IDLE_MASK                                                                      0x00002000L
+#define SDMA_STATUS_REG__SRBM_IDLE_MASK                                                                       0x00004000L
+#define SDMA_STATUS_REG__CONTEXT_EMPTY_MASK                                                                   0x00008000L
+#define SDMA_STATUS_REG__DELTA_RPTR_FULL_MASK                                                                 0x00010000L
+#define SDMA_STATUS_REG__RB_MC_RREQ_IDLE_MASK                                                                 0x00020000L
+#define SDMA_STATUS_REG__IB_MC_RREQ_IDLE_MASK                                                                 0x00040000L
+#define SDMA_STATUS_REG__MC_RD_IDLE_MASK                                                                      0x00080000L
+#define SDMA_STATUS_REG__DELTA_RPTR_EMPTY_MASK                                                                0x00100000L
+#define SDMA_STATUS_REG__MC_RD_RET_STALL_MASK                                                                 0x00200000L
+#define SDMA_STATUS_REG__MC_RD_NO_POLL_IDLE_MASK                                                              0x00400000L
+#define SDMA_STATUS_REG__DRM_IDLE_MASK                                                                        0x00800000L
+#define SDMA_STATUS_REG__DRM_MASK_FULL_MASK                                                                   0x01000000L
+#define SDMA_STATUS_REG__PREV_CMD_IDLE_MASK                                                                   0x02000000L
+#define SDMA_STATUS_REG__SEM_IDLE_MASK                                                                        0x04000000L
+#define SDMA_STATUS_REG__SEM_REQ_STALL_MASK                                                                   0x08000000L
+#define SDMA_STATUS_REG__SEM_RESP_STATE_MASK                                                                  0x30000000L
+#define SDMA_STATUS_REG__INT_IDLE_MASK                                                                        0x40000000L
+#define SDMA_STATUS_REG__INT_REQ_STALL_MASK                                                                   0x80000000L
+//SDMA_STATUS1_REG
+#define SDMA_STATUS1_REG__CE_WREQ_IDLE__SHIFT                                                                 0x0
+#define SDMA_STATUS1_REG__CE_WR_IDLE__SHIFT                                                                   0x1
+#define SDMA_STATUS1_REG__CE_SPLIT_IDLE__SHIFT                                                                0x2
+#define SDMA_STATUS1_REG__CE_RREQ_IDLE__SHIFT                                                                 0x3
+#define SDMA_STATUS1_REG__CE_OUT_IDLE__SHIFT                                                                  0x4
+#define SDMA_STATUS1_REG__CE_IN_IDLE__SHIFT                                                                   0x5
+#define SDMA_STATUS1_REG__CE_DST_IDLE__SHIFT                                                                  0x6
+#define SDMA_STATUS1_REG__CE_DRM_IDLE__SHIFT                                                                  0x7
+#define SDMA_STATUS1_REG__CE_DRM1_IDLE__SHIFT                                                                 0x8
+#define SDMA_STATUS1_REG__CE_CMD_IDLE__SHIFT                                                                  0x9
+#define SDMA_STATUS1_REG__CE_AFIFO_FULL__SHIFT                                                                0xa
+#define SDMA_STATUS1_REG__CE_DRM_FULL__SHIFT                                                                  0xb
+#define SDMA_STATUS1_REG__CE_DRM1_FULL__SHIFT                                                                 0xc
+#define SDMA_STATUS1_REG__CE_INFO_FULL__SHIFT                                                                 0xd
+#define SDMA_STATUS1_REG__CE_INFO1_FULL__SHIFT                                                                0xe
+#define SDMA_STATUS1_REG__EX_START__SHIFT                                                                     0xf
+#define SDMA_STATUS1_REG__DRM_CTX_RESTORE__SHIFT                                                              0x10
+#define SDMA_STATUS1_REG__CE_RD_STALL__SHIFT                                                                  0x11
+#define SDMA_STATUS1_REG__CE_WR_STALL__SHIFT                                                                  0x12
+#define SDMA_STATUS1_REG__CE_WREQ_IDLE_MASK                                                                   0x00000001L
+#define SDMA_STATUS1_REG__CE_WR_IDLE_MASK                                                                     0x00000002L
+#define SDMA_STATUS1_REG__CE_SPLIT_IDLE_MASK                                                                  0x00000004L
+#define SDMA_STATUS1_REG__CE_RREQ_IDLE_MASK                                                                   0x00000008L
+#define SDMA_STATUS1_REG__CE_OUT_IDLE_MASK                                                                    0x00000010L
+#define SDMA_STATUS1_REG__CE_IN_IDLE_MASK                                                                     0x00000020L
+#define SDMA_STATUS1_REG__CE_DST_IDLE_MASK                                                                    0x00000040L
+#define SDMA_STATUS1_REG__CE_DRM_IDLE_MASK                                                                    0x00000080L
+#define SDMA_STATUS1_REG__CE_DRM1_IDLE_MASK                                                                   0x00000100L
+#define SDMA_STATUS1_REG__CE_CMD_IDLE_MASK                                                                    0x00000200L
+#define SDMA_STATUS1_REG__CE_AFIFO_FULL_MASK                                                                  0x00000400L
+#define SDMA_STATUS1_REG__CE_DRM_FULL_MASK                                                                    0x00000800L
+#define SDMA_STATUS1_REG__CE_DRM1_FULL_MASK                                                                   0x00001000L
+#define SDMA_STATUS1_REG__CE_INFO_FULL_MASK                                                                   0x00002000L
+#define SDMA_STATUS1_REG__CE_INFO1_FULL_MASK                                                                  0x00004000L
+#define SDMA_STATUS1_REG__EX_START_MASK                                                                       0x00008000L
+#define SDMA_STATUS1_REG__DRM_CTX_RESTORE_MASK                                                                0x00010000L
+#define SDMA_STATUS1_REG__CE_RD_STALL_MASK                                                                    0x00020000L
+#define SDMA_STATUS1_REG__CE_WR_STALL_MASK                                                                    0x00040000L
+//SDMA_RD_BURST_CNTL
+#define SDMA_RD_BURST_CNTL__RD_BURST__SHIFT                                                                   0x0
+#define SDMA_RD_BURST_CNTL__CMD_BUFFER_RD_BURST__SHIFT                                                        0x2
+#define SDMA_RD_BURST_CNTL__RD_BURST_MASK                                                                     0x00000003L
+#define SDMA_RD_BURST_CNTL__CMD_BUFFER_RD_BURST_MASK                                                          0x0000000CL
+//SDMA_HBM_PAGE_CONFIG
+#define SDMA_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT__SHIFT                                                       0x0
+#define SDMA_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT_MASK                                                         0x00000003L
+//SDMA_UCODE_CHECKSUM
+#define SDMA_UCODE_CHECKSUM__DATA__SHIFT                                                                      0x0
+#define SDMA_UCODE_CHECKSUM__DATA_MASK                                                                        0xFFFFFFFFL
+//SDMA_FREEZE
+#define SDMA_FREEZE__PREEMPT__SHIFT                                                                           0x0
+#define SDMA_FREEZE__FREEZE__SHIFT                                                                            0x4
+#define SDMA_FREEZE__FROZEN__SHIFT                                                                            0x5
+#define SDMA_FREEZE__F32_FREEZE__SHIFT                                                                        0x6
+#define SDMA_FREEZE__PREEMPT_MASK                                                                             0x00000001L
+#define SDMA_FREEZE__FREEZE_MASK                                                                              0x00000010L
+#define SDMA_FREEZE__FROZEN_MASK                                                                              0x00000020L
+#define SDMA_FREEZE__F32_FREEZE_MASK                                                                          0x00000040L
+//SDMA_PHASE0_QUANTUM
+#define SDMA_PHASE0_QUANTUM__UNIT__SHIFT                                                                      0x0
+#define SDMA_PHASE0_QUANTUM__VALUE__SHIFT                                                                     0x8
+#define SDMA_PHASE0_QUANTUM__PREFER__SHIFT                                                                    0x1e
+#define SDMA_PHASE0_QUANTUM__UNIT_MASK                                                                        0x0000000FL
+#define SDMA_PHASE0_QUANTUM__VALUE_MASK                                                                       0x00FFFF00L
+#define SDMA_PHASE0_QUANTUM__PREFER_MASK                                                                      0x40000000L
+//SDMA_PHASE1_QUANTUM
+#define SDMA_PHASE1_QUANTUM__UNIT__SHIFT                                                                      0x0
+#define SDMA_PHASE1_QUANTUM__VALUE__SHIFT                                                                     0x8
+#define SDMA_PHASE1_QUANTUM__PREFER__SHIFT                                                                    0x1e
+#define SDMA_PHASE1_QUANTUM__UNIT_MASK                                                                        0x0000000FL
+#define SDMA_PHASE1_QUANTUM__VALUE_MASK                                                                       0x00FFFF00L
+#define SDMA_PHASE1_QUANTUM__PREFER_MASK                                                                      0x40000000L
+//SDMA_POWER_GATING
+#define SDMA_POWER_GATING__SDMA_POWER_OFF_CONDITION__SHIFT                                                    0x0
+#define SDMA_POWER_GATING__SDMA_POWER_ON_CONDITION__SHIFT                                                     0x1
+#define SDMA_POWER_GATING__SDMA_POWER_OFF_REQ__SHIFT                                                          0x2
+#define SDMA_POWER_GATING__SDMA_POWER_ON_REQ__SHIFT                                                           0x3
+#define SDMA_POWER_GATING__PG_CNTL_STATUS__SHIFT                                                              0x4
+#define SDMA_POWER_GATING__SDMA_POWER_OFF_CONDITION_MASK                                                      0x00000001L
+#define SDMA_POWER_GATING__SDMA_POWER_ON_CONDITION_MASK                                                       0x00000002L
+#define SDMA_POWER_GATING__SDMA_POWER_OFF_REQ_MASK                                                            0x00000004L
+#define SDMA_POWER_GATING__SDMA_POWER_ON_REQ_MASK                                                             0x00000008L
+#define SDMA_POWER_GATING__PG_CNTL_STATUS_MASK                                                                0x00000030L
+//SDMA_PGFSM_CONFIG
+#define SDMA_PGFSM_CONFIG__FSM_ADDR__SHIFT                                                                    0x0
+#define SDMA_PGFSM_CONFIG__POWER_DOWN__SHIFT                                                                  0x8
+#define SDMA_PGFSM_CONFIG__POWER_UP__SHIFT                                                                    0x9
+#define SDMA_PGFSM_CONFIG__P1_SELECT__SHIFT                                                                   0xa
+#define SDMA_PGFSM_CONFIG__P2_SELECT__SHIFT                                                                   0xb
+#define SDMA_PGFSM_CONFIG__WRITE__SHIFT                                                                       0xc
+#define SDMA_PGFSM_CONFIG__READ__SHIFT                                                                        0xd
+#define SDMA_PGFSM_CONFIG__SRBM_OVERRIDE__SHIFT                                                               0x1b
+#define SDMA_PGFSM_CONFIG__REG_ADDR__SHIFT                                                                    0x1c
+#define SDMA_PGFSM_CONFIG__FSM_ADDR_MASK                                                                      0x000000FFL
+#define SDMA_PGFSM_CONFIG__POWER_DOWN_MASK                                                                    0x00000100L
+#define SDMA_PGFSM_CONFIG__POWER_UP_MASK                                                                      0x00000200L
+#define SDMA_PGFSM_CONFIG__P1_SELECT_MASK                                                                     0x00000400L
+#define SDMA_PGFSM_CONFIG__P2_SELECT_MASK                                                                     0x00000800L
+#define SDMA_PGFSM_CONFIG__WRITE_MASK                                                                         0x00001000L
+#define SDMA_PGFSM_CONFIG__READ_MASK                                                                          0x00002000L
+#define SDMA_PGFSM_CONFIG__SRBM_OVERRIDE_MASK                                                                 0x08000000L
+#define SDMA_PGFSM_CONFIG__REG_ADDR_MASK                                                                      0xF0000000L
+//SDMA_PGFSM_WRITE
+#define SDMA_PGFSM_WRITE__VALUE__SHIFT                                                                        0x0
+#define SDMA_PGFSM_WRITE__VALUE_MASK                                                                          0xFFFFFFFFL
+//SDMA_PGFSM_READ
+#define SDMA_PGFSM_READ__VALUE__SHIFT                                                                         0x0
+#define SDMA_PGFSM_READ__VALUE_MASK                                                                           0x00FFFFFFL
+//CC_SDMA_EDC_CONFIG
+#define CC_SDMA_EDC_CONFIG__WRITE_DIS__SHIFT                                                                  0x0
+#define CC_SDMA_EDC_CONFIG__DIS_EDC__SHIFT                                                                    0x1
+#define CC_SDMA_EDC_CONFIG__WRITE_DIS_MASK                                                                    0x00000001L
+#define CC_SDMA_EDC_CONFIG__DIS_EDC_MASK                                                                      0x00000002L
+//SDMA_BA_THRESHOLD
+#define SDMA_BA_THRESHOLD__READ_THRES__SHIFT                                                                  0x0
+#define SDMA_BA_THRESHOLD__WRITE_THRES__SHIFT                                                                 0x10
+#define SDMA_BA_THRESHOLD__READ_THRES_MASK                                                                    0x000003FFL
+#define SDMA_BA_THRESHOLD__WRITE_THRES_MASK                                                                   0x03FF0000L
+//SDMA_ID
+#define SDMA_ID__DEVICE_ID__SHIFT                                                                             0x0
+#define SDMA_ID__DEVICE_ID_MASK                                                                               0x000000FFL
+//SDMA_VERSION
+#define SDMA_VERSION__MINVER__SHIFT                                                                           0x0
+#define SDMA_VERSION__MAJVER__SHIFT                                                                           0x8
+#define SDMA_VERSION__REV__SHIFT                                                                              0x10
+#define SDMA_VERSION__MINVER_MASK                                                                             0x0000007FL
+#define SDMA_VERSION__MAJVER_MASK                                                                             0x00007F00L
+#define SDMA_VERSION__REV_MASK                                                                                0x003F0000L
+//SDMA_EDC_COUNTER
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED__SHIFT                                                     0x0
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED__SHIFT                                                     0x2
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED__SHIFT                                                     0x4
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED__SHIFT                                                     0x6
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED__SHIFT                                                     0x8
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED__SHIFT                                                     0xa
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED__SHIFT                                                     0xc
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED__SHIFT                                                     0xe
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF8_SED__SHIFT                                                     0x10
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF9_SED__SHIFT                                                     0x12
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF10_SED__SHIFT                                                    0x14
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF11_SED__SHIFT                                                    0x16
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF12_SED__SHIFT                                                    0x18
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF13_SED__SHIFT                                                    0x1a
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF14_SED__SHIFT                                                    0x1c
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF15_SED__SHIFT                                                    0x1e
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED_MASK                                                       0x00000003L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED_MASK                                                       0x0000000CL
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED_MASK                                                       0x00000030L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED_MASK                                                       0x000000C0L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED_MASK                                                       0x00000300L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED_MASK                                                       0x00000C00L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED_MASK                                                       0x00003000L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED_MASK                                                       0x0000C000L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF8_SED_MASK                                                       0x00030000L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF9_SED_MASK                                                       0x000C0000L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF10_SED_MASK                                                      0x00300000L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF11_SED_MASK                                                      0x00C00000L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF12_SED_MASK                                                      0x03000000L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF13_SED_MASK                                                      0x0C000000L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF14_SED_MASK                                                      0x30000000L
+#define SDMA_EDC_COUNTER__SDMA_MBANK_DATA_BUF15_SED_MASK                                                      0xC0000000L
+//SDMA_EDC_COUNTER2
+#define SDMA_EDC_COUNTER2__SDMA_UCODE_BUF_SED__SHIFT                                                          0x0
+#define SDMA_EDC_COUNTER2__SDMA_RB_CMD_BUF_SED__SHIFT                                                         0x2
+#define SDMA_EDC_COUNTER2__SDMA_IB_CMD_BUF_SED__SHIFT                                                         0x4
+#define SDMA_EDC_COUNTER2__SDMA_UTCL1_RD_FIFO_SED__SHIFT                                                      0x6
+#define SDMA_EDC_COUNTER2__SDMA_UTCL1_RDBST_FIFO_SED__SHIFT                                                   0x8
+#define SDMA_EDC_COUNTER2__SDMA_UTCL1_WR_FIFO_SED__SHIFT                                                      0xa
+#define SDMA_EDC_COUNTER2__SDMA_DATA_LUT_FIFO_SED__SHIFT                                                      0xc
+#define SDMA_EDC_COUNTER2__SDMA_SPLIT_DATA_BUF_SED__SHIFT                                                     0xe
+#define SDMA_EDC_COUNTER2__SDMA_MC_WR_ADDR_FIFO_SED__SHIFT                                                    0x10
+#define SDMA_EDC_COUNTER2__SDMA_MC_RDRET_BUF_SED__SHIFT                                                       0x12
+#define SDMA_EDC_COUNTER2__SDMA_UCODE_BUF_SED_MASK                                                            0x00000003L
+#define SDMA_EDC_COUNTER2__SDMA_RB_CMD_BUF_SED_MASK                                                           0x0000000CL
+#define SDMA_EDC_COUNTER2__SDMA_IB_CMD_BUF_SED_MASK                                                           0x00000030L
+#define SDMA_EDC_COUNTER2__SDMA_UTCL1_RD_FIFO_SED_MASK                                                        0x000000C0L
+#define SDMA_EDC_COUNTER2__SDMA_UTCL1_RDBST_FIFO_SED_MASK                                                     0x00000300L
+#define SDMA_EDC_COUNTER2__SDMA_UTCL1_WR_FIFO_SED_MASK                                                        0x00000C00L
+#define SDMA_EDC_COUNTER2__SDMA_DATA_LUT_FIFO_SED_MASK                                                        0x00003000L
+#define SDMA_EDC_COUNTER2__SDMA_SPLIT_DATA_BUF_SED_MASK                                                       0x0000C000L
+#define SDMA_EDC_COUNTER2__SDMA_MC_WR_ADDR_FIFO_SED_MASK                                                      0x00030000L
+#define SDMA_EDC_COUNTER2__SDMA_MC_RDRET_BUF_SED_MASK                                                         0x000C0000L
+//SDMA_STATUS2_REG
+#define SDMA_STATUS2_REG__ID__SHIFT                                                                           0x0
+#define SDMA_STATUS2_REG__F32_INSTR_PTR__SHIFT                                                                0x3
+#define SDMA_STATUS2_REG__CMD_OP__SHIFT                                                                       0x10
+#define SDMA_STATUS2_REG__ID_MASK                                                                             0x00000007L
+#define SDMA_STATUS2_REG__F32_INSTR_PTR_MASK                                                                  0x0000FFF8L
+#define SDMA_STATUS2_REG__CMD_OP_MASK                                                                         0xFFFF0000L
+//SDMA_ATOMIC_CNTL
+#define SDMA_ATOMIC_CNTL__LOOP_TIMER__SHIFT                                                                   0x0
+#define SDMA_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT                                                        0x1f
+#define SDMA_ATOMIC_CNTL__LOOP_TIMER_MASK                                                                     0x7FFFFFFFL
+#define SDMA_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK                                                          0x80000000L
+//SDMA_ATOMIC_PREOP_LO
+#define SDMA_ATOMIC_PREOP_LO__DATA__SHIFT                                                                     0x0
+#define SDMA_ATOMIC_PREOP_LO__DATA_MASK                                                                       0xFFFFFFFFL
+//SDMA_ATOMIC_PREOP_HI
+#define SDMA_ATOMIC_PREOP_HI__DATA__SHIFT                                                                     0x0
+#define SDMA_ATOMIC_PREOP_HI__DATA_MASK                                                                       0xFFFFFFFFL
+//SDMA_UTCL1_CNTL
+#define SDMA_UTCL1_CNTL__REDO_ENABLE__SHIFT                                                                   0x0
+#define SDMA_UTCL1_CNTL__REDO_DELAY__SHIFT                                                                    0x1
+#define SDMA_UTCL1_CNTL__REDO_WATERMK__SHIFT                                                                  0xb
+#define SDMA_UTCL1_CNTL__INVACK_DELAY__SHIFT                                                                  0xe
+#define SDMA_UTCL1_CNTL__REQL2_CREDIT__SHIFT                                                                  0x18
+#define SDMA_UTCL1_CNTL__VADDR_WATERMK__SHIFT                                                                 0x1d
+#define SDMA_UTCL1_CNTL__REDO_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_UTCL1_CNTL__REDO_DELAY_MASK                                                                      0x000007FEL
+#define SDMA_UTCL1_CNTL__REDO_WATERMK_MASK                                                                    0x00003800L
+#define SDMA_UTCL1_CNTL__INVACK_DELAY_MASK                                                                    0x00FFC000L
+#define SDMA_UTCL1_CNTL__REQL2_CREDIT_MASK                                                                    0x1F000000L
+#define SDMA_UTCL1_CNTL__VADDR_WATERMK_MASK                                                                   0xE0000000L
+//SDMA_UTCL1_WATERMK
+#define SDMA_UTCL1_WATERMK__REQ_WATERMK__SHIFT                                                                0x0
+#define SDMA_UTCL1_WATERMK__REQ_DEPTH__SHIFT                                                                  0x3
+#define SDMA_UTCL1_WATERMK__PAGE_WATERMK__SHIFT                                                               0x5
+#define SDMA_UTCL1_WATERMK__INVREQ_WATERMK__SHIFT                                                             0x8
+#define SDMA_UTCL1_WATERMK__RESERVED__SHIFT                                                                   0x10
+#define SDMA_UTCL1_WATERMK__REQ_WATERMK_MASK                                                                  0x00000007L
+#define SDMA_UTCL1_WATERMK__REQ_DEPTH_MASK                                                                    0x00000018L
+#define SDMA_UTCL1_WATERMK__PAGE_WATERMK_MASK                                                                 0x000000E0L
+#define SDMA_UTCL1_WATERMK__INVREQ_WATERMK_MASK                                                               0x0000FF00L
+#define SDMA_UTCL1_WATERMK__RESERVED_MASK                                                                     0xFFFF0000L
+//SDMA_UTCL1_RD_STATUS
+#define SDMA_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT                                                 0x0
+#define SDMA_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT                                                      0x1
+#define SDMA_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY__SHIFT                                                       0x2
+#define SDMA_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT                                                    0x3
+#define SDMA_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT                                                0x4
+#define SDMA_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT                                                     0x5
+#define SDMA_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT                                                  0x6
+#define SDMA_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT                                                    0x7
+#define SDMA_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT                                                   0x8
+#define SDMA_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT                                                  0x9
+#define SDMA_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL__SHIFT                                                       0xa
+#define SDMA_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL__SHIFT                                                        0xb
+#define SDMA_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT                                                     0xc
+#define SDMA_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT                                                 0xd
+#define SDMA_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL__SHIFT                                                      0xe
+#define SDMA_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT                                                   0xf
+#define SDMA_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT                                                     0x10
+#define SDMA_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT                                                    0x11
+#define SDMA_UTCL1_RD_STATUS__PAGE_FAULT__SHIFT                                                               0x12
+#define SDMA_UTCL1_RD_STATUS__PAGE_NULL__SHIFT                                                                0x13
+#define SDMA_UTCL1_RD_STATUS__REQL2_IDLE__SHIFT                                                               0x14
+#define SDMA_UTCL1_RD_STATUS__CE_L1_STALL__SHIFT                                                              0x15
+#define SDMA_UTCL1_RD_STATUS__NEXT_RD_VECTOR__SHIFT                                                           0x16
+#define SDMA_UTCL1_RD_STATUS__MERGE_STATE__SHIFT                                                              0x1a
+#define SDMA_UTCL1_RD_STATUS__ADDR_RD_RTR__SHIFT                                                              0x1d
+#define SDMA_UTCL1_RD_STATUS__WPTR_POLLING__SHIFT                                                             0x1e
+#define SDMA_UTCL1_RD_STATUS__INVREQ_SIZE__SHIFT                                                              0x1f
+#define SDMA_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK                                                   0x00000001L
+#define SDMA_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY_MASK                                                        0x00000002L
+#define SDMA_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY_MASK                                                         0x00000004L
+#define SDMA_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK                                                      0x00000008L
+#define SDMA_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK                                                  0x00000010L
+#define SDMA_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY_MASK                                                       0x00000020L
+#define SDMA_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK                                                    0x00000040L
+#define SDMA_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK                                                      0x00000080L
+#define SDMA_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK                                                     0x00000100L
+#define SDMA_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK                                                    0x00000200L
+#define SDMA_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL_MASK                                                         0x00000400L
+#define SDMA_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL_MASK                                                          0x00000800L
+#define SDMA_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL_MASK                                                       0x00001000L
+#define SDMA_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK                                                   0x00002000L
+#define SDMA_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL_MASK                                                        0x00004000L
+#define SDMA_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK                                                     0x00008000L
+#define SDMA_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL_MASK                                                       0x00010000L
+#define SDMA_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL_MASK                                                      0x00020000L
+#define SDMA_UTCL1_RD_STATUS__PAGE_FAULT_MASK                                                                 0x00040000L
+#define SDMA_UTCL1_RD_STATUS__PAGE_NULL_MASK                                                                  0x00080000L
+#define SDMA_UTCL1_RD_STATUS__REQL2_IDLE_MASK                                                                 0x00100000L
+#define SDMA_UTCL1_RD_STATUS__CE_L1_STALL_MASK                                                                0x00200000L
+#define SDMA_UTCL1_RD_STATUS__NEXT_RD_VECTOR_MASK                                                             0x03C00000L
+#define SDMA_UTCL1_RD_STATUS__MERGE_STATE_MASK                                                                0x1C000000L
+#define SDMA_UTCL1_RD_STATUS__ADDR_RD_RTR_MASK                                                                0x20000000L
+#define SDMA_UTCL1_RD_STATUS__WPTR_POLLING_MASK                                                               0x40000000L
+#define SDMA_UTCL1_RD_STATUS__INVREQ_SIZE_MASK                                                                0x80000000L
+//SDMA_UTCL1_WR_STATUS
+#define SDMA_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT                                                 0x0
+#define SDMA_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT                                                      0x1
+#define SDMA_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY__SHIFT                                                       0x2
+#define SDMA_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT                                                    0x3
+#define SDMA_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT                                                0x4
+#define SDMA_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT                                                     0x5
+#define SDMA_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT                                                  0x6
+#define SDMA_UTCL1_WR_STATUS__REDO_ARR_EMPTY__SHIFT                                                           0x7
+#define SDMA_UTCL1_WR_STATUS__RESERVED_8__SHIFT                                                               0x8
+#define SDMA_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT                                                  0x9
+#define SDMA_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL__SHIFT                                                       0xa
+#define SDMA_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL__SHIFT                                                        0xb
+#define SDMA_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT                                                     0xc
+#define SDMA_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT                                                 0xd
+#define SDMA_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL__SHIFT                                                      0xe
+#define SDMA_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT                                                   0xf
+#define SDMA_UTCL1_WR_STATUS__REDO_ARR_FULL__SHIFT                                                            0x10
+#define SDMA_UTCL1_WR_STATUS__RESERVED_17__SHIFT                                                              0x11
+#define SDMA_UTCL1_WR_STATUS__PAGE_FAULT__SHIFT                                                               0x12
+#define SDMA_UTCL1_WR_STATUS__PAGE_NULL__SHIFT                                                                0x13
+#define SDMA_UTCL1_WR_STATUS__REQL2_IDLE__SHIFT                                                               0x14
+#define SDMA_UTCL1_WR_STATUS__F32_WR_RTR__SHIFT                                                               0x15
+#define SDMA_UTCL1_WR_STATUS__NEXT_WR_VECTOR__SHIFT                                                           0x16
+#define SDMA_UTCL1_WR_STATUS__MERGE_STATE__SHIFT                                                              0x19
+#define SDMA_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY__SHIFT                                                     0x1c
+#define SDMA_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL__SHIFT                                                      0x1d
+#define SDMA_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY__SHIFT                                                    0x1e
+#define SDMA_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL__SHIFT                                                     0x1f
+#define SDMA_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK                                                   0x00000001L
+#define SDMA_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY_MASK                                                        0x00000002L
+#define SDMA_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY_MASK                                                         0x00000004L
+#define SDMA_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK                                                      0x00000008L
+#define SDMA_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK                                                  0x00000010L
+#define SDMA_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY_MASK                                                       0x00000020L
+#define SDMA_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK                                                    0x00000040L
+#define SDMA_UTCL1_WR_STATUS__REDO_ARR_EMPTY_MASK                                                             0x00000080L
+#define SDMA_UTCL1_WR_STATUS__RESERVED_8_MASK                                                                 0x00000100L
+#define SDMA_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK                                                    0x00000200L
+#define SDMA_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL_MASK                                                         0x00000400L
+#define SDMA_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL_MASK                                                          0x00000800L
+#define SDMA_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL_MASK                                                       0x00001000L
+#define SDMA_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK                                                   0x00002000L
+#define SDMA_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL_MASK                                                        0x00004000L
+#define SDMA_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK                                                     0x00008000L
+#define SDMA_UTCL1_WR_STATUS__REDO_ARR_FULL_MASK                                                              0x00010000L
+#define SDMA_UTCL1_WR_STATUS__RESERVED_17_MASK                                                                0x00020000L
+#define SDMA_UTCL1_WR_STATUS__PAGE_FAULT_MASK                                                                 0x00040000L
+#define SDMA_UTCL1_WR_STATUS__PAGE_NULL_MASK                                                                  0x00080000L
+#define SDMA_UTCL1_WR_STATUS__REQL2_IDLE_MASK                                                                 0x00100000L
+#define SDMA_UTCL1_WR_STATUS__F32_WR_RTR_MASK                                                                 0x00200000L
+#define SDMA_UTCL1_WR_STATUS__NEXT_WR_VECTOR_MASK                                                             0x01C00000L
+#define SDMA_UTCL1_WR_STATUS__MERGE_STATE_MASK                                                                0x0E000000L
+#define SDMA_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY_MASK                                                       0x10000000L
+#define SDMA_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL_MASK                                                        0x20000000L
+#define SDMA_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY_MASK                                                      0x40000000L
+#define SDMA_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL_MASK                                                       0x80000000L
+//SDMA_UTCL1_INV0
+#define SDMA_UTCL1_INV0__INV_MIDDLE__SHIFT                                                                    0x0
+#define SDMA_UTCL1_INV0__RD_TIMEOUT__SHIFT                                                                    0x1
+#define SDMA_UTCL1_INV0__WR_TIMEOUT__SHIFT                                                                    0x2
+#define SDMA_UTCL1_INV0__RD_IN_INVADR__SHIFT                                                                  0x3
+#define SDMA_UTCL1_INV0__WR_IN_INVADR__SHIFT                                                                  0x4
+#define SDMA_UTCL1_INV0__PAGE_NULL_SW__SHIFT                                                                  0x5
+#define SDMA_UTCL1_INV0__XNACK_IS_INVADR__SHIFT                                                               0x6
+#define SDMA_UTCL1_INV0__INVREQ_ENABLE__SHIFT                                                                 0x7
+#define SDMA_UTCL1_INV0__NACK_TIMEOUT_SW__SHIFT                                                               0x8
+#define SDMA_UTCL1_INV0__NFLUSH_INV_IDLE__SHIFT                                                               0x9
+#define SDMA_UTCL1_INV0__FLUSH_INV_IDLE__SHIFT                                                                0xa
+#define SDMA_UTCL1_INV0__INV_FLUSHTYPE__SHIFT                                                                 0xb
+#define SDMA_UTCL1_INV0__INV_VMID_VEC__SHIFT                                                                  0xc
+#define SDMA_UTCL1_INV0__INV_ADDR_HI__SHIFT                                                                   0x1c
+#define SDMA_UTCL1_INV0__INV_MIDDLE_MASK                                                                      0x00000001L
+#define SDMA_UTCL1_INV0__RD_TIMEOUT_MASK                                                                      0x00000002L
+#define SDMA_UTCL1_INV0__WR_TIMEOUT_MASK                                                                      0x00000004L
+#define SDMA_UTCL1_INV0__RD_IN_INVADR_MASK                                                                    0x00000008L
+#define SDMA_UTCL1_INV0__WR_IN_INVADR_MASK                                                                    0x00000010L
+#define SDMA_UTCL1_INV0__PAGE_NULL_SW_MASK                                                                    0x00000020L
+#define SDMA_UTCL1_INV0__XNACK_IS_INVADR_MASK                                                                 0x00000040L
+#define SDMA_UTCL1_INV0__INVREQ_ENABLE_MASK                                                                   0x00000080L
+#define SDMA_UTCL1_INV0__NACK_TIMEOUT_SW_MASK                                                                 0x00000100L
+#define SDMA_UTCL1_INV0__NFLUSH_INV_IDLE_MASK                                                                 0x00000200L
+#define SDMA_UTCL1_INV0__FLUSH_INV_IDLE_MASK                                                                  0x00000400L
+#define SDMA_UTCL1_INV0__INV_FLUSHTYPE_MASK                                                                   0x00000800L
+#define SDMA_UTCL1_INV0__INV_VMID_VEC_MASK                                                                    0x0FFFF000L
+#define SDMA_UTCL1_INV0__INV_ADDR_HI_MASK                                                                     0xF0000000L
+//SDMA_UTCL1_INV1
+#define SDMA_UTCL1_INV1__INV_ADDR_LO__SHIFT                                                                   0x0
+#define SDMA_UTCL1_INV1__INV_ADDR_LO_MASK                                                                     0xFFFFFFFFL
+//SDMA_UTCL1_INV2
+#define SDMA_UTCL1_INV2__INV_NFLUSH_VMID_VEC__SHIFT                                                           0x0
+#define SDMA_UTCL1_INV2__INV_NFLUSH_VMID_VEC_MASK                                                             0xFFFFFFFFL
+//SDMA_UTCL1_RD_XNACK0
+#define SDMA_UTCL1_RD_XNACK0__XNACK_ADDR_LO__SHIFT                                                            0x0
+#define SDMA_UTCL1_RD_XNACK0__XNACK_ADDR_LO_MASK                                                              0xFFFFFFFFL
+//SDMA_UTCL1_RD_XNACK1
+#define SDMA_UTCL1_RD_XNACK1__XNACK_ADDR_HI__SHIFT                                                            0x0
+#define SDMA_UTCL1_RD_XNACK1__XNACK_VMID__SHIFT                                                               0x4
+#define SDMA_UTCL1_RD_XNACK1__XNACK_VECTOR__SHIFT                                                             0x8
+#define SDMA_UTCL1_RD_XNACK1__IS_XNACK__SHIFT                                                                 0x1a
+#define SDMA_UTCL1_RD_XNACK1__XNACK_ADDR_HI_MASK                                                              0x0000000FL
+#define SDMA_UTCL1_RD_XNACK1__XNACK_VMID_MASK                                                                 0x000000F0L
+#define SDMA_UTCL1_RD_XNACK1__XNACK_VECTOR_MASK                                                               0x03FFFF00L
+#define SDMA_UTCL1_RD_XNACK1__IS_XNACK_MASK                                                                   0x0C000000L
+//SDMA_UTCL1_WR_XNACK0
+#define SDMA_UTCL1_WR_XNACK0__XNACK_ADDR_LO__SHIFT                                                            0x0
+#define SDMA_UTCL1_WR_XNACK0__XNACK_ADDR_LO_MASK                                                              0xFFFFFFFFL
+//SDMA_UTCL1_WR_XNACK1
+#define SDMA_UTCL1_WR_XNACK1__XNACK_ADDR_HI__SHIFT                                                            0x0
+#define SDMA_UTCL1_WR_XNACK1__XNACK_VMID__SHIFT                                                               0x4
+#define SDMA_UTCL1_WR_XNACK1__XNACK_VECTOR__SHIFT                                                             0x8
+#define SDMA_UTCL1_WR_XNACK1__IS_XNACK__SHIFT                                                                 0x1a
+#define SDMA_UTCL1_WR_XNACK1__XNACK_ADDR_HI_MASK                                                              0x0000000FL
+#define SDMA_UTCL1_WR_XNACK1__XNACK_VMID_MASK                                                                 0x000000F0L
+#define SDMA_UTCL1_WR_XNACK1__XNACK_VECTOR_MASK                                                               0x03FFFF00L
+#define SDMA_UTCL1_WR_XNACK1__IS_XNACK_MASK                                                                   0x0C000000L
+//SDMA_UTCL1_TIMEOUT
+#define SDMA_UTCL1_TIMEOUT__RD_XNACK_LIMIT__SHIFT                                                             0x0
+#define SDMA_UTCL1_TIMEOUT__WR_XNACK_LIMIT__SHIFT                                                             0x10
+#define SDMA_UTCL1_TIMEOUT__RD_XNACK_LIMIT_MASK                                                               0x0000FFFFL
+#define SDMA_UTCL1_TIMEOUT__WR_XNACK_LIMIT_MASK                                                               0xFFFF0000L
+//SDMA_UTCL1_PAGE
+#define SDMA_UTCL1_PAGE__VM_HOLE__SHIFT                                                                       0x0
+#define SDMA_UTCL1_PAGE__REQ_TYPE__SHIFT                                                                      0x1
+#define SDMA_UTCL1_PAGE__TMZ_ENABLE__SHIFT                                                                    0x5
+#define SDMA_UTCL1_PAGE__USE_MTYPE__SHIFT                                                                     0x6
+#define SDMA_UTCL1_PAGE__USE_PT_SNOOP__SHIFT                                                                  0x9
+#define SDMA_UTCL1_PAGE__LLC_NOALLOC__SHIFT                                                                   0xa
+#define SDMA_UTCL1_PAGE__VM_HOLE_MASK                                                                         0x00000001L
+#define SDMA_UTCL1_PAGE__REQ_TYPE_MASK                                                                        0x0000001EL
+#define SDMA_UTCL1_PAGE__TMZ_ENABLE_MASK                                                                      0x00000020L
+#define SDMA_UTCL1_PAGE__USE_MTYPE_MASK                                                                       0x000001C0L
+#define SDMA_UTCL1_PAGE__USE_PT_SNOOP_MASK                                                                    0x00000200L
+#define SDMA_UTCL1_PAGE__LLC_NOALLOC_MASK                                                                     0x00000400L
+//SDMA_POWER_CNTL_IDLE
+#define SDMA_POWER_CNTL_IDLE__DELAY0__SHIFT                                                                   0x0
+#define SDMA_POWER_CNTL_IDLE__DELAY1__SHIFT                                                                   0x10
+#define SDMA_POWER_CNTL_IDLE__DELAY2__SHIFT                                                                   0x18
+#define SDMA_POWER_CNTL_IDLE__DELAY0_MASK                                                                     0x0000FFFFL
+#define SDMA_POWER_CNTL_IDLE__DELAY1_MASK                                                                     0x00FF0000L
+#define SDMA_POWER_CNTL_IDLE__DELAY2_MASK                                                                     0xFF000000L
+//SDMA_RELAX_ORDERING_LUT
+#define SDMA_RELAX_ORDERING_LUT__RESERVED0__SHIFT                                                             0x0
+#define SDMA_RELAX_ORDERING_LUT__COPY__SHIFT                                                                  0x1
+#define SDMA_RELAX_ORDERING_LUT__WRITE__SHIFT                                                                 0x2
+#define SDMA_RELAX_ORDERING_LUT__RESERVED3__SHIFT                                                             0x3
+#define SDMA_RELAX_ORDERING_LUT__RESERVED4__SHIFT                                                             0x4
+#define SDMA_RELAX_ORDERING_LUT__FENCE__SHIFT                                                                 0x5
+#define SDMA_RELAX_ORDERING_LUT__RESERVED76__SHIFT                                                            0x6
+#define SDMA_RELAX_ORDERING_LUT__POLL_MEM__SHIFT                                                              0x8
+#define SDMA_RELAX_ORDERING_LUT__COND_EXE__SHIFT                                                              0x9
+#define SDMA_RELAX_ORDERING_LUT__ATOMIC__SHIFT                                                                0xa
+#define SDMA_RELAX_ORDERING_LUT__CONST_FILL__SHIFT                                                            0xb
+#define SDMA_RELAX_ORDERING_LUT__PTEPDE__SHIFT                                                                0xc
+#define SDMA_RELAX_ORDERING_LUT__TIMESTAMP__SHIFT                                                             0xd
+#define SDMA_RELAX_ORDERING_LUT__RESERVED__SHIFT                                                              0xe
+#define SDMA_RELAX_ORDERING_LUT__WORLD_SWITCH__SHIFT                                                          0x1b
+#define SDMA_RELAX_ORDERING_LUT__RPTR_WRB__SHIFT                                                              0x1c
+#define SDMA_RELAX_ORDERING_LUT__WPTR_POLL__SHIFT                                                             0x1d
+#define SDMA_RELAX_ORDERING_LUT__IB_FETCH__SHIFT                                                              0x1e
+#define SDMA_RELAX_ORDERING_LUT__RB_FETCH__SHIFT                                                              0x1f
+#define SDMA_RELAX_ORDERING_LUT__RESERVED0_MASK                                                               0x00000001L
+#define SDMA_RELAX_ORDERING_LUT__COPY_MASK                                                                    0x00000002L
+#define SDMA_RELAX_ORDERING_LUT__WRITE_MASK                                                                   0x00000004L
+#define SDMA_RELAX_ORDERING_LUT__RESERVED3_MASK                                                               0x00000008L
+#define SDMA_RELAX_ORDERING_LUT__RESERVED4_MASK                                                               0x00000010L
+#define SDMA_RELAX_ORDERING_LUT__FENCE_MASK                                                                   0x00000020L
+#define SDMA_RELAX_ORDERING_LUT__RESERVED76_MASK                                                              0x000000C0L
+#define SDMA_RELAX_ORDERING_LUT__POLL_MEM_MASK                                                                0x00000100L
+#define SDMA_RELAX_ORDERING_LUT__COND_EXE_MASK                                                                0x00000200L
+#define SDMA_RELAX_ORDERING_LUT__ATOMIC_MASK                                                                  0x00000400L
+#define SDMA_RELAX_ORDERING_LUT__CONST_FILL_MASK                                                              0x00000800L
+#define SDMA_RELAX_ORDERING_LUT__PTEPDE_MASK                                                                  0x00001000L
+#define SDMA_RELAX_ORDERING_LUT__TIMESTAMP_MASK                                                               0x00002000L
+#define SDMA_RELAX_ORDERING_LUT__RESERVED_MASK                                                                0x07FFC000L
+#define SDMA_RELAX_ORDERING_LUT__WORLD_SWITCH_MASK                                                            0x08000000L
+#define SDMA_RELAX_ORDERING_LUT__RPTR_WRB_MASK                                                                0x10000000L
+#define SDMA_RELAX_ORDERING_LUT__WPTR_POLL_MASK                                                               0x20000000L
+#define SDMA_RELAX_ORDERING_LUT__IB_FETCH_MASK                                                                0x40000000L
+#define SDMA_RELAX_ORDERING_LUT__RB_FETCH_MASK                                                                0x80000000L
+//SDMA_CHICKEN_BITS_2
+#define SDMA_CHICKEN_BITS_2__F32_CMD_PROC_DELAY__SHIFT                                                        0x0
+#define SDMA_CHICKEN_BITS_2__F32_SEND_POSTCODE_EN__SHIFT                                                      0x4
+#define SDMA_CHICKEN_BITS_2__F32_CMD_PROC_DELAY_MASK                                                          0x0000000FL
+#define SDMA_CHICKEN_BITS_2__F32_SEND_POSTCODE_EN_MASK                                                        0x00000010L
+//SDMA_STATUS3_REG
+#define SDMA_STATUS3_REG__CMD_OP_STATUS__SHIFT                                                                0x0
+#define SDMA_STATUS3_REG__PREV_VM_CMD__SHIFT                                                                  0x10
+#define SDMA_STATUS3_REG__EXCEPTION_IDLE__SHIFT                                                               0x14
+#define SDMA_STATUS3_REG__QUEUE_ID_MATCH__SHIFT                                                               0x15
+#define SDMA_STATUS3_REG__INT_QUEUE_ID__SHIFT                                                                 0x16
+#define SDMA_STATUS3_REG__CMD_OP_STATUS_MASK                                                                  0x0000FFFFL
+#define SDMA_STATUS3_REG__PREV_VM_CMD_MASK                                                                    0x000F0000L
+#define SDMA_STATUS3_REG__EXCEPTION_IDLE_MASK                                                                 0x00100000L
+#define SDMA_STATUS3_REG__QUEUE_ID_MATCH_MASK                                                                 0x00200000L
+#define SDMA_STATUS3_REG__INT_QUEUE_ID_MASK                                                                   0x03C00000L
+//SDMA_PHYSICAL_ADDR_LO
+#define SDMA_PHYSICAL_ADDR_LO__D_VALID__SHIFT                                                                 0x0
+#define SDMA_PHYSICAL_ADDR_LO__DIRTY__SHIFT                                                                   0x1
+#define SDMA_PHYSICAL_ADDR_LO__PHY_VALID__SHIFT                                                               0x2
+#define SDMA_PHYSICAL_ADDR_LO__ADDR__SHIFT                                                                    0xc
+#define SDMA_PHYSICAL_ADDR_LO__D_VALID_MASK                                                                   0x00000001L
+#define SDMA_PHYSICAL_ADDR_LO__DIRTY_MASK                                                                     0x00000002L
+#define SDMA_PHYSICAL_ADDR_LO__PHY_VALID_MASK                                                                 0x00000004L
+#define SDMA_PHYSICAL_ADDR_LO__ADDR_MASK                                                                      0xFFFFF000L
+//SDMA_PHYSICAL_ADDR_HI
+#define SDMA_PHYSICAL_ADDR_HI__ADDR__SHIFT                                                                    0x0
+#define SDMA_PHYSICAL_ADDR_HI__ADDR_MASK                                                                      0x0000FFFFL
+//SDMA_PHASE2_QUANTUM
+#define SDMA_PHASE2_QUANTUM__UNIT__SHIFT                                                                      0x0
+#define SDMA_PHASE2_QUANTUM__VALUE__SHIFT                                                                     0x8
+#define SDMA_PHASE2_QUANTUM__PREFER__SHIFT                                                                    0x1e
+#define SDMA_PHASE2_QUANTUM__UNIT_MASK                                                                        0x0000000FL
+#define SDMA_PHASE2_QUANTUM__VALUE_MASK                                                                       0x00FFFF00L
+#define SDMA_PHASE2_QUANTUM__PREFER_MASK                                                                      0x40000000L
+//SDMA_ERROR_LOG
+#define SDMA_ERROR_LOG__OVERRIDE__SHIFT                                                                       0x0
+#define SDMA_ERROR_LOG__STATUS__SHIFT                                                                         0x10
+#define SDMA_ERROR_LOG__OVERRIDE_MASK                                                                         0x0000FFFFL
+#define SDMA_ERROR_LOG__STATUS_MASK                                                                           0xFFFF0000L
+//SDMA_PUB_DUMMY_REG0
+#define SDMA_PUB_DUMMY_REG0__VALUE__SHIFT                                                                     0x0
+#define SDMA_PUB_DUMMY_REG0__VALUE_MASK                                                                       0xFFFFFFFFL
+//SDMA_PUB_DUMMY_REG1
+#define SDMA_PUB_DUMMY_REG1__VALUE__SHIFT                                                                     0x0
+#define SDMA_PUB_DUMMY_REG1__VALUE_MASK                                                                       0xFFFFFFFFL
+//SDMA_PUB_DUMMY_REG2
+#define SDMA_PUB_DUMMY_REG2__VALUE__SHIFT                                                                     0x0
+#define SDMA_PUB_DUMMY_REG2__VALUE_MASK                                                                       0xFFFFFFFFL
+//SDMA_PUB_DUMMY_REG3
+#define SDMA_PUB_DUMMY_REG3__VALUE__SHIFT                                                                     0x0
+#define SDMA_PUB_DUMMY_REG3__VALUE_MASK                                                                       0xFFFFFFFFL
+//SDMA_F32_COUNTER
+#define SDMA_F32_COUNTER__VALUE__SHIFT                                                                        0x0
+#define SDMA_F32_COUNTER__VALUE_MASK                                                                          0xFFFFFFFFL
+//SDMA_PERFCNT_PERFCOUNTER0_CFG
+#define SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                        0x0
+#define SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                       0x18
+#define SDMA_PERFCNT_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                          0x1c
+#define SDMA_PERFCNT_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                           0x1d
+#define SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define SDMA_PERFCNT_PERFCOUNTER0_CFG__ENABLE_MASK                                                            0x10000000L
+#define SDMA_PERFCNT_PERFCOUNTER0_CFG__CLEAR_MASK                                                             0x20000000L
+//SDMA_PERFCNT_PERFCOUNTER1_CFG
+#define SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                        0x0
+#define SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                       0x18
+#define SDMA_PERFCNT_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                          0x1c
+#define SDMA_PERFCNT_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                           0x1d
+#define SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define SDMA_PERFCNT_PERFCOUNTER1_CFG__ENABLE_MASK                                                            0x10000000L
+#define SDMA_PERFCNT_PERFCOUNTER1_CFG__CLEAR_MASK                                                             0x20000000L
+//SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                        0x0
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                              0x8
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                               0x10
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                 0x18
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                  0x19
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                       0x1a
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                          0x0000000FL
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                0x0000FF00L
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                 0x00FF0000L
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                   0x01000000L
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                    0x02000000L
+#define SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                         0x04000000L
+//SDMA_PERFCNT_MISC_CNTL
+#define SDMA_PERFCNT_MISC_CNTL__CMD_OP__SHIFT                                                                 0x0
+#define SDMA_PERFCNT_MISC_CNTL__MMHUB_REQ_EVENT_SELECT__SHIFT                                                 0x10
+#define SDMA_PERFCNT_MISC_CNTL__CMD_OP_MASK                                                                   0x0000FFFFL
+#define SDMA_PERFCNT_MISC_CNTL__MMHUB_REQ_EVENT_SELECT_MASK                                                   0x00010000L
+//SDMA_PERFCNT_PERFCOUNTER_LO
+#define SDMA_PERFCNT_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                        0x0
+#define SDMA_PERFCNT_PERFCOUNTER_LO__COUNTER_LO_MASK                                                          0xFFFFFFFFL
+//SDMA_PERFCNT_PERFCOUNTER_HI
+#define SDMA_PERFCNT_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                        0x0
+#define SDMA_PERFCNT_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                     0x10
+#define SDMA_PERFCNT_PERFCOUNTER_HI__COUNTER_HI_MASK                                                          0x0000FFFFL
+#define SDMA_PERFCNT_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                       0xFFFF0000L
+//SDMA_CRD_CNTL
+#define SDMA_CRD_CNTL__DRM_CREDIT__SHIFT                                                                      0x0
+#define SDMA_CRD_CNTL__MC_WRREQ_CREDIT__SHIFT                                                                 0x7
+#define SDMA_CRD_CNTL__MC_RDREQ_CREDIT__SHIFT                                                                 0xd
+#define SDMA_CRD_CNTL__DRM_CREDIT_MASK                                                                        0x0000007FL
+#define SDMA_CRD_CNTL__MC_WRREQ_CREDIT_MASK                                                                   0x00001F80L
+#define SDMA_CRD_CNTL__MC_RDREQ_CREDIT_MASK                                                                   0x0007E000L
+//SDMA_GPU_IOV_VIOLATION_LOG
+#define SDMA_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                   0x0
+#define SDMA_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                          0x1
+#define SDMA_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                            0x2
+#define SDMA_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION__SHIFT                                                    0x14
+#define SDMA_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                                 0x15
+#define SDMA_GPU_IOV_VIOLATION_LOG__VFID__SHIFT                                                               0x16
+#define SDMA_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                     0x00000001L
+#define SDMA_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                            0x00000002L
+#define SDMA_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                              0x000FFFFCL
+#define SDMA_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION_MASK                                                      0x00100000L
+#define SDMA_GPU_IOV_VIOLATION_LOG__VF_MASK                                                                   0x00200000L
+#define SDMA_GPU_IOV_VIOLATION_LOG__VFID_MASK                                                                 0x03C00000L
+//SDMA_ULV_CNTL
+#define SDMA_ULV_CNTL__HYSTERESIS__SHIFT                                                                      0x0
+#define SDMA_ULV_CNTL__ENTER_ULV_INT_CLR__SHIFT                                                               0x1b
+#define SDMA_ULV_CNTL__EXIT_ULV_INT_CLR__SHIFT                                                                0x1c
+#define SDMA_ULV_CNTL__ENTER_ULV_INT__SHIFT                                                                   0x1d
+#define SDMA_ULV_CNTL__EXIT_ULV_INT__SHIFT                                                                    0x1e
+#define SDMA_ULV_CNTL__ULV_STATUS__SHIFT                                                                      0x1f
+#define SDMA_ULV_CNTL__HYSTERESIS_MASK                                                                        0x0000001FL
+#define SDMA_ULV_CNTL__ENTER_ULV_INT_CLR_MASK                                                                 0x08000000L
+#define SDMA_ULV_CNTL__EXIT_ULV_INT_CLR_MASK                                                                  0x10000000L
+#define SDMA_ULV_CNTL__ENTER_ULV_INT_MASK                                                                     0x20000000L
+#define SDMA_ULV_CNTL__EXIT_ULV_INT_MASK                                                                      0x40000000L
+#define SDMA_ULV_CNTL__ULV_STATUS_MASK                                                                        0x80000000L
+//SDMA_EA_DBIT_ADDR_DATA
+#define SDMA_EA_DBIT_ADDR_DATA__VALUE__SHIFT                                                                  0x0
+#define SDMA_EA_DBIT_ADDR_DATA__VALUE_MASK                                                                    0xFFFFFFFFL
+//SDMA_EA_DBIT_ADDR_INDEX
+#define SDMA_EA_DBIT_ADDR_INDEX__VALUE__SHIFT                                                                 0x0
+#define SDMA_EA_DBIT_ADDR_INDEX__VALUE_MASK                                                                   0x00000007L
+//SDMA_GPU_IOV_VIOLATION_LOG2
+#define SDMA_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID__SHIFT                                                      0x0
+#define SDMA_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID_MASK                                                        0x000003FFL
+//SDMA_STATUS4_REG
+#define SDMA_STATUS4_REG__IDLE__SHIFT                                                                         0x0
+#define SDMA_STATUS4_REG__IH_OUTSTANDING__SHIFT                                                               0x2
+#define SDMA_STATUS4_REG__SEM_OUTSTANDING__SHIFT                                                              0x3
+#define SDMA_STATUS4_REG__MMHUB_RD_OUTSTANDING__SHIFT                                                         0x4
+#define SDMA_STATUS4_REG__MMHUB_WR_OUTSTANDING__SHIFT                                                         0x5
+#define SDMA_STATUS4_REG__UTCL2_RD_OUTSTANDING__SHIFT                                                         0x6
+#define SDMA_STATUS4_REG__UTCL2_WR_OUTSTANDING__SHIFT                                                         0x7
+#define SDMA_STATUS4_REG__REG_POLLING__SHIFT                                                                  0x8
+#define SDMA_STATUS4_REG__MEM_POLLING__SHIFT                                                                  0x9
+#define SDMA_STATUS4_REG__UTCL2_RD_XNACK__SHIFT                                                               0xa
+#define SDMA_STATUS4_REG__UTCL2_WR_XNACK__SHIFT                                                               0xc
+#define SDMA_STATUS4_REG__ACTIVE_QUEUE_ID__SHIFT                                                              0xe
+#define SDMA_STATUS4_REG__SRIOV_WATING_RLCV_CMD__SHIFT                                                        0x12
+#define SDMA_STATUS4_REG__SRIOV_SDMA_EXECUTING_CMD__SHIFT                                                     0x13
+#define SDMA_STATUS4_REG__VM_HOLE_STATUS__SHIFT                                                               0x14
+#define SDMA_STATUS4_REG__IDLE_MASK                                                                           0x00000001L
+#define SDMA_STATUS4_REG__IH_OUTSTANDING_MASK                                                                 0x00000004L
+#define SDMA_STATUS4_REG__SEM_OUTSTANDING_MASK                                                                0x00000008L
+#define SDMA_STATUS4_REG__MMHUB_RD_OUTSTANDING_MASK                                                           0x00000010L
+#define SDMA_STATUS4_REG__MMHUB_WR_OUTSTANDING_MASK                                                           0x00000020L
+#define SDMA_STATUS4_REG__UTCL2_RD_OUTSTANDING_MASK                                                           0x00000040L
+#define SDMA_STATUS4_REG__UTCL2_WR_OUTSTANDING_MASK                                                           0x00000080L
+#define SDMA_STATUS4_REG__REG_POLLING_MASK                                                                    0x00000100L
+#define SDMA_STATUS4_REG__MEM_POLLING_MASK                                                                    0x00000200L
+#define SDMA_STATUS4_REG__UTCL2_RD_XNACK_MASK                                                                 0x00000C00L
+#define SDMA_STATUS4_REG__UTCL2_WR_XNACK_MASK                                                                 0x00003000L
+#define SDMA_STATUS4_REG__ACTIVE_QUEUE_ID_MASK                                                                0x0003C000L
+#define SDMA_STATUS4_REG__SRIOV_WATING_RLCV_CMD_MASK                                                          0x00040000L
+#define SDMA_STATUS4_REG__SRIOV_SDMA_EXECUTING_CMD_MASK                                                       0x00080000L
+#define SDMA_STATUS4_REG__VM_HOLE_STATUS_MASK                                                                 0x00100000L
+//SDMA_SCRATCH_RAM_DATA
+#define SDMA_SCRATCH_RAM_DATA__DATA__SHIFT                                                                    0x0
+#define SDMA_SCRATCH_RAM_DATA__DATA_MASK                                                                      0xFFFFFFFFL
+//SDMA_SCRATCH_RAM_ADDR
+#define SDMA_SCRATCH_RAM_ADDR__ADDR__SHIFT                                                                    0x0
+#define SDMA_SCRATCH_RAM_ADDR__ADDR_MASK                                                                      0x0000007FL
+//SDMA_CE_CTRL
+#define SDMA_CE_CTRL__RD_LUT_WATERMARK__SHIFT                                                                 0x0
+#define SDMA_CE_CTRL__RD_LUT_DEPTH__SHIFT                                                                     0x3
+#define SDMA_CE_CTRL__WR_AFIFO_WATERMARK__SHIFT                                                               0x5
+#define SDMA_CE_CTRL__RESERVED__SHIFT                                                                         0x8
+#define SDMA_CE_CTRL__RD_LUT_WATERMARK_MASK                                                                   0x00000007L
+#define SDMA_CE_CTRL__RD_LUT_DEPTH_MASK                                                                       0x00000018L
+#define SDMA_CE_CTRL__WR_AFIFO_WATERMARK_MASK                                                                 0x000000E0L
+#define SDMA_CE_CTRL__RESERVED_MASK                                                                           0xFFFFFF00L
+//SDMA_RAS_STATUS
+#define SDMA_RAS_STATUS__RB_FETCH_ECC__SHIFT                                                                  0x0
+#define SDMA_RAS_STATUS__IB_FETCH_ECC__SHIFT                                                                  0x1
+#define SDMA_RAS_STATUS__F32_DATA_ECC__SHIFT                                                                  0x2
+#define SDMA_RAS_STATUS__WPTR_ATOMIC_ECC__SHIFT                                                               0x3
+#define SDMA_RAS_STATUS__COPY_DATA_ECC__SHIFT                                                                 0x4
+#define SDMA_RAS_STATUS__SRAM_ECC__SHIFT                                                                      0x5
+#define SDMA_RAS_STATUS__RB_FETCH_NACK_GEN_ERR__SHIFT                                                         0x8
+#define SDMA_RAS_STATUS__IB_FETCH_NACK_GEN_ERR__SHIFT                                                         0x9
+#define SDMA_RAS_STATUS__F32_DATA_NACK_GEN_ERR__SHIFT                                                         0xa
+#define SDMA_RAS_STATUS__COPY_DATA_NACK_GEN_ERR__SHIFT                                                        0xb
+#define SDMA_RAS_STATUS__WRRET_DATA_NACK_GEN_ERR__SHIFT                                                       0xc
+#define SDMA_RAS_STATUS__WPTR_RPTR_ATOMIC_NACK_GEN_ERR__SHIFT                                                 0xd
+#define SDMA_RAS_STATUS__ECC_PWRMGT_INT_BUSY__SHIFT                                                           0xe
+#define SDMA_RAS_STATUS__RB_FETCH_ECC_MASK                                                                    0x00000001L
+#define SDMA_RAS_STATUS__IB_FETCH_ECC_MASK                                                                    0x00000002L
+#define SDMA_RAS_STATUS__F32_DATA_ECC_MASK                                                                    0x00000004L
+#define SDMA_RAS_STATUS__WPTR_ATOMIC_ECC_MASK                                                                 0x00000008L
+#define SDMA_RAS_STATUS__COPY_DATA_ECC_MASK                                                                   0x00000010L
+#define SDMA_RAS_STATUS__SRAM_ECC_MASK                                                                        0x00000020L
+#define SDMA_RAS_STATUS__RB_FETCH_NACK_GEN_ERR_MASK                                                           0x00000100L
+#define SDMA_RAS_STATUS__IB_FETCH_NACK_GEN_ERR_MASK                                                           0x00000200L
+#define SDMA_RAS_STATUS__F32_DATA_NACK_GEN_ERR_MASK                                                           0x00000400L
+#define SDMA_RAS_STATUS__COPY_DATA_NACK_GEN_ERR_MASK                                                          0x00000800L
+#define SDMA_RAS_STATUS__WRRET_DATA_NACK_GEN_ERR_MASK                                                         0x00001000L
+#define SDMA_RAS_STATUS__WPTR_RPTR_ATOMIC_NACK_GEN_ERR_MASK                                                   0x00002000L
+#define SDMA_RAS_STATUS__ECC_PWRMGT_INT_BUSY_MASK                                                             0x00004000L
+//SDMA_CLK_STATUS
+#define SDMA_CLK_STATUS__DYN_CLK__SHIFT                                                                       0x0
+#define SDMA_CLK_STATUS__PTR_CLK__SHIFT                                                                       0x1
+#define SDMA_CLK_STATUS__REG_CLK__SHIFT                                                                       0x2
+#define SDMA_CLK_STATUS__F32_CLK__SHIFT                                                                       0x3
+#define SDMA_CLK_STATUS__CE_CLK__SHIFT                                                                        0x4
+#define SDMA_CLK_STATUS__PERF_CLK__SHIFT                                                                      0x5
+#define SDMA_CLK_STATUS__DYN_CLK_MASK                                                                         0x00000001L
+#define SDMA_CLK_STATUS__PTR_CLK_MASK                                                                         0x00000002L
+#define SDMA_CLK_STATUS__REG_CLK_MASK                                                                         0x00000004L
+#define SDMA_CLK_STATUS__F32_CLK_MASK                                                                         0x00000008L
+#define SDMA_CLK_STATUS__CE_CLK_MASK                                                                          0x00000010L
+#define SDMA_CLK_STATUS__PERF_CLK_MASK                                                                        0x00000020L
+//SDMA_POWER_CNTL
+#define SDMA_POWER_CNTL__PG_CNTL_ENABLE__SHIFT                                                                0x0
+#define SDMA_POWER_CNTL__EXT_PG_POWER_ON_REQ__SHIFT                                                           0x1
+#define SDMA_POWER_CNTL__EXT_PG_POWER_OFF_REQ__SHIFT                                                          0x2
+#define SDMA_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME__SHIFT                                                    0x3
+#define SDMA_POWER_CNTL__MEM_POWER_OVERRIDE__SHIFT                                                            0x8
+#define SDMA_POWER_CNTL__MEM_POWER_LS_EN__SHIFT                                                               0x9
+#define SDMA_POWER_CNTL__MEM_POWER_DS_EN__SHIFT                                                               0xa
+#define SDMA_POWER_CNTL__MEM_POWER_SD_EN__SHIFT                                                               0xb
+#define SDMA_POWER_CNTL__MEM_POWER_DELAY__SHIFT                                                               0xc
+#define SDMA_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME__SHIFT                                                   0x1a
+#define SDMA_POWER_CNTL__PG_CNTL_ENABLE_MASK                                                                  0x00000001L
+#define SDMA_POWER_CNTL__EXT_PG_POWER_ON_REQ_MASK                                                             0x00000002L
+#define SDMA_POWER_CNTL__EXT_PG_POWER_OFF_REQ_MASK                                                            0x00000004L
+#define SDMA_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK                                                      0x000000F8L
+#define SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK                                                              0x00000100L
+#define SDMA_POWER_CNTL__MEM_POWER_LS_EN_MASK                                                                 0x00000200L
+#define SDMA_POWER_CNTL__MEM_POWER_DS_EN_MASK                                                                 0x00000400L
+#define SDMA_POWER_CNTL__MEM_POWER_SD_EN_MASK                                                                 0x00000800L
+#define SDMA_POWER_CNTL__MEM_POWER_DELAY_MASK                                                                 0x003FF000L
+#define SDMA_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK                                                     0xFC000000L
+//SDMA_CLK_CTRL
+#define SDMA_CLK_CTRL__ON_DELAY__SHIFT                                                                        0x0
+#define SDMA_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                                  0x4
+#define SDMA_CLK_CTRL__RESERVED__SHIFT                                                                        0xc
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                                  0x18
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                                  0x19
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                                  0x1a
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                                  0x1b
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                                  0x1c
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                                  0x1d
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                                  0x1e
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                                  0x1f
+#define SDMA_CLK_CTRL__ON_DELAY_MASK                                                                          0x0000000FL
+#define SDMA_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                    0x00000FF0L
+#define SDMA_CLK_CTRL__RESERVED_MASK                                                                          0x00FFF000L
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                    0x01000000L
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                    0x02000000L
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                    0x04000000L
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                    0x08000000L
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                    0x10000000L
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                                    0x20000000L
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                                    0x40000000L
+#define SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                                    0x80000000L
+//SDMA_CNTL
+#define SDMA_CNTL__TRAP_ENABLE__SHIFT                                                                         0x0
+#define SDMA_CNTL__UTC_L1_ENABLE__SHIFT                                                                       0x1
+#define SDMA_CNTL__SEM_WAIT_INT_ENABLE__SHIFT                                                                 0x2
+#define SDMA_CNTL__DATA_SWAP_ENABLE__SHIFT                                                                    0x3
+#define SDMA_CNTL__FENCE_SWAP_ENABLE__SHIFT                                                                   0x4
+#define SDMA_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                               0x5
+#define SDMA_CNTL__MIDCMD_EXPIRE_ENABLE__SHIFT                                                                0x6
+#define SDMA_CNTL__REG_WRITE_PROTECT_INT_ENABLE__SHIFT                                                        0x7
+#define SDMA_CNTL__INVALID_DOORBELL_INT_ENABLE__SHIFT                                                         0x8
+#define SDMA_CNTL__VM_HOLE_INT_ENABLE__SHIFT                                                                  0x9
+#define SDMA_CNTL__DRAM_ECC_INT_ENABLE__SHIFT                                                                 0xa
+#define SDMA_CNTL__PAGE_RETRY_TIMEOUT_INT_ENABLE__SHIFT                                                       0xb
+#define SDMA_CNTL__PAGE_NULL_INT_ENABLE__SHIFT                                                                0xc
+#define SDMA_CNTL__PAGE_FAULT_INT_ENABLE__SHIFT                                                               0xd
+#define SDMA_CNTL__NACK_GEN_ERR_INT_ENABLE__SHIFT                                                             0xe
+#define SDMA_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT                                                           0x11
+#define SDMA_CNTL__AUTO_CTXSW_ENABLE__SHIFT                                                                   0x12
+#define SDMA_CNTL__DRM_RESTORE_ENABLE__SHIFT                                                                  0x13
+#define SDMA_CNTL__CTXEMPTY_INT_ENABLE__SHIFT                                                                 0x1c
+#define SDMA_CNTL__FROZEN_INT_ENABLE__SHIFT                                                                   0x1d
+#define SDMA_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT                                                               0x1e
+#define SDMA_CNTL__RB_PREEMPT_INT_ENABLE__SHIFT                                                               0x1f
+#define SDMA_CNTL__TRAP_ENABLE_MASK                                                                           0x00000001L
+#define SDMA_CNTL__UTC_L1_ENABLE_MASK                                                                         0x00000002L
+#define SDMA_CNTL__SEM_WAIT_INT_ENABLE_MASK                                                                   0x00000004L
+#define SDMA_CNTL__DATA_SWAP_ENABLE_MASK                                                                      0x00000008L
+#define SDMA_CNTL__FENCE_SWAP_ENABLE_MASK                                                                     0x00000010L
+#define SDMA_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                                 0x00000020L
+#define SDMA_CNTL__MIDCMD_EXPIRE_ENABLE_MASK                                                                  0x00000040L
+#define SDMA_CNTL__REG_WRITE_PROTECT_INT_ENABLE_MASK                                                          0x00000080L
+#define SDMA_CNTL__INVALID_DOORBELL_INT_ENABLE_MASK                                                           0x00000100L
+#define SDMA_CNTL__VM_HOLE_INT_ENABLE_MASK                                                                    0x00000200L
+#define SDMA_CNTL__DRAM_ECC_INT_ENABLE_MASK                                                                   0x00000400L
+#define SDMA_CNTL__PAGE_RETRY_TIMEOUT_INT_ENABLE_MASK                                                         0x00000800L
+#define SDMA_CNTL__PAGE_NULL_INT_ENABLE_MASK                                                                  0x00001000L
+#define SDMA_CNTL__PAGE_FAULT_INT_ENABLE_MASK                                                                 0x00002000L
+#define SDMA_CNTL__NACK_GEN_ERR_INT_ENABLE_MASK                                                               0x00004000L
+#define SDMA_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK                                                             0x00020000L
+#define SDMA_CNTL__AUTO_CTXSW_ENABLE_MASK                                                                     0x00040000L
+#define SDMA_CNTL__DRM_RESTORE_ENABLE_MASK                                                                    0x00080000L
+#define SDMA_CNTL__CTXEMPTY_INT_ENABLE_MASK                                                                   0x10000000L
+#define SDMA_CNTL__FROZEN_INT_ENABLE_MASK                                                                     0x20000000L
+#define SDMA_CNTL__IB_PREEMPT_INT_ENABLE_MASK                                                                 0x40000000L
+#define SDMA_CNTL__RB_PREEMPT_INT_ENABLE_MASK                                                                 0x80000000L
+//SDMA_CHICKEN_BITS
+#define SDMA_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE__SHIFT                                                      0x0
+#define SDMA_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE__SHIFT                                                  0x1
+#define SDMA_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE__SHIFT                                         0x2
+#define SDMA_CHICKEN_BITS__F32_MGCG_ENABLE__SHIFT                                                             0x3
+#define SDMA_CHICKEN_BITS__WRITE_BURST_LENGTH__SHIFT                                                          0x8
+#define SDMA_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE__SHIFT                                                      0xa
+#define SDMA_CHICKEN_BITS__COPY_OVERLAP_ENABLE__SHIFT                                                         0x10
+#define SDMA_CHICKEN_BITS__RAW_CHECK_ENABLE__SHIFT                                                            0x11
+#define SDMA_CHICKEN_BITS__SRBM_POLL_RETRYING__SHIFT                                                          0x14
+#define SDMA_CHICKEN_BITS__CG_STATUS_OUTPUT__SHIFT                                                            0x17
+#define SDMA_CHICKEN_BITS__SRAM_FGCG_ENABLE__SHIFT                                                            0x1a
+#define SDMA_CHICKEN_BITS__RESERVED__SHIFT                                                                    0x1b
+#define SDMA_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE_MASK                                                        0x00000001L
+#define SDMA_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE_MASK                                                    0x00000002L
+#define SDMA_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE_MASK                                           0x00000004L
+#define SDMA_CHICKEN_BITS__F32_MGCG_ENABLE_MASK                                                               0x00000008L
+#define SDMA_CHICKEN_BITS__WRITE_BURST_LENGTH_MASK                                                            0x00000300L
+#define SDMA_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE_MASK                                                        0x00001C00L
+#define SDMA_CHICKEN_BITS__COPY_OVERLAP_ENABLE_MASK                                                           0x00010000L
+#define SDMA_CHICKEN_BITS__RAW_CHECK_ENABLE_MASK                                                              0x00020000L
+#define SDMA_CHICKEN_BITS__SRBM_POLL_RETRYING_MASK                                                            0x00100000L
+#define SDMA_CHICKEN_BITS__CG_STATUS_OUTPUT_MASK                                                              0x00800000L
+#define SDMA_CHICKEN_BITS__SRAM_FGCG_ENABLE_MASK                                                              0x04000000L
+#define SDMA_CHICKEN_BITS__RESERVED_MASK                                                                      0xF8000000L
+//SDMA_GB_ADDR_CONFIG
+#define SDMA_GB_ADDR_CONFIG__NUM_PIPES__SHIFT                                                                 0x0
+#define SDMA_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                      0x3
+#define SDMA_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT                                                      0x8
+#define SDMA_GB_ADDR_CONFIG__NUM_BANKS__SHIFT                                                                 0xc
+#define SDMA_GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT                                                        0x13
+#define SDMA_GB_ADDR_CONFIG__NUM_PIPES_MASK                                                                   0x00000007L
+#define SDMA_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                        0x00000038L
+#define SDMA_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK                                                        0x00000700L
+#define SDMA_GB_ADDR_CONFIG__NUM_BANKS_MASK                                                                   0x00007000L
+#define SDMA_GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK                                                          0x00180000L
+//SDMA_GB_ADDR_CONFIG_READ
+#define SDMA_GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                            0x0
+#define SDMA_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                 0x3
+#define SDMA_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE__SHIFT                                                 0x8
+#define SDMA_GB_ADDR_CONFIG_READ__NUM_BANKS__SHIFT                                                            0xc
+#define SDMA_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                   0x13
+#define SDMA_GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                              0x00000007L
+#define SDMA_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                   0x00000038L
+#define SDMA_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE_MASK                                                   0x00000700L
+#define SDMA_GB_ADDR_CONFIG_READ__NUM_BANKS_MASK                                                              0x00007000L
+#define SDMA_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                     0x00180000L
+//SDMA_GFX_RB_CNTL
+#define SDMA_GFX_RB_CNTL__RB_ENABLE__SHIFT                                                                    0x0
+#define SDMA_GFX_RB_CNTL__RB_SIZE__SHIFT                                                                      0x1
+#define SDMA_GFX_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                               0x9
+#define SDMA_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                        0xc
+#define SDMA_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                   0xd
+#define SDMA_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                         0x10
+#define SDMA_GFX_RB_CNTL__RB_PRIV__SHIFT                                                                      0x17
+#define SDMA_GFX_RB_CNTL__RB_VMID__SHIFT                                                                      0x18
+#define SDMA_GFX_RB_CNTL__RB_ENABLE_MASK                                                                      0x00000001L
+#define SDMA_GFX_RB_CNTL__RB_SIZE_MASK                                                                        0x0000003EL
+#define SDMA_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                 0x00000200L
+#define SDMA_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                          0x00001000L
+#define SDMA_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                     0x00002000L
+#define SDMA_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                           0x001F0000L
+#define SDMA_GFX_RB_CNTL__RB_PRIV_MASK                                                                        0x00800000L
+#define SDMA_GFX_RB_CNTL__RB_VMID_MASK                                                                        0x0F000000L
+//SDMA_GFX_RB_BASE
+#define SDMA_GFX_RB_BASE__ADDR__SHIFT                                                                         0x0
+#define SDMA_GFX_RB_BASE__ADDR_MASK                                                                           0xFFFFFFFFL
+//SDMA_GFX_RB_BASE_HI
+#define SDMA_GFX_RB_BASE_HI__ADDR__SHIFT                                                                      0x0
+#define SDMA_GFX_RB_BASE_HI__ADDR_MASK                                                                        0x00FFFFFFL
+//SDMA_GFX_RB_RPTR
+#define SDMA_GFX_RB_RPTR__OFFSET__SHIFT                                                                       0x0
+#define SDMA_GFX_RB_RPTR__OFFSET_MASK                                                                         0xFFFFFFFFL
+//SDMA_GFX_RB_RPTR_HI
+#define SDMA_GFX_RB_RPTR_HI__OFFSET__SHIFT                                                                    0x0
+#define SDMA_GFX_RB_RPTR_HI__OFFSET_MASK                                                                      0xFFFFFFFFL
+//SDMA_GFX_RB_WPTR
+#define SDMA_GFX_RB_WPTR__OFFSET__SHIFT                                                                       0x0
+#define SDMA_GFX_RB_WPTR__OFFSET_MASK                                                                         0xFFFFFFFFL
+//SDMA_GFX_RB_WPTR_HI
+#define SDMA_GFX_RB_WPTR_HI__OFFSET__SHIFT                                                                    0x0
+#define SDMA_GFX_RB_WPTR_HI__OFFSET_MASK                                                                      0xFFFFFFFFL
+//SDMA_GFX_RB_WPTR_POLL_CNTL
+#define SDMA_GFX_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                             0x0
+#define SDMA_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                        0x1
+#define SDMA_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                    0x2
+#define SDMA_GFX_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                          0x4
+#define SDMA_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                    0x10
+#define SDMA_GFX_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                               0x00000001L
+#define SDMA_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                          0x00000002L
+#define SDMA_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                      0x00000004L
+#define SDMA_GFX_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                            0x0000FFF0L
+#define SDMA_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                      0xFFFF0000L
+//SDMA_GFX_RB_RPTR_ADDR_HI
+#define SDMA_GFX_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA_GFX_RB_RPTR_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA_GFX_RB_RPTR_ADDR_LO
+#define SDMA_GFX_RB_RPTR_ADDR_LO__RPTR_WB_IDLE__SHIFT                                                         0x0
+#define SDMA_GFX_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA_GFX_RB_RPTR_ADDR_LO__RPTR_WB_IDLE_MASK                                                           0x00000001L
+#define SDMA_GFX_RB_RPTR_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA_GFX_IB_CNTL
+#define SDMA_GFX_IB_CNTL__IB_ENABLE__SHIFT                                                                    0x0
+#define SDMA_GFX_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                               0x4
+#define SDMA_GFX_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                             0x8
+#define SDMA_GFX_IB_CNTL__CMD_VMID__SHIFT                                                                     0x10
+#define SDMA_GFX_IB_CNTL__IB_PRIV__SHIFT                                                                      0x1f
+#define SDMA_GFX_IB_CNTL__IB_ENABLE_MASK                                                                      0x00000001L
+#define SDMA_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                 0x00000010L
+#define SDMA_GFX_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                               0x00000100L
+#define SDMA_GFX_IB_CNTL__CMD_VMID_MASK                                                                       0x000F0000L
+#define SDMA_GFX_IB_CNTL__IB_PRIV_MASK                                                                        0x80000000L
+//SDMA_GFX_IB_RPTR
+#define SDMA_GFX_IB_RPTR__OFFSET__SHIFT                                                                       0x2
+#define SDMA_GFX_IB_RPTR__OFFSET_MASK                                                                         0x003FFFFCL
+//SDMA_GFX_IB_OFFSET
+#define SDMA_GFX_IB_OFFSET__OFFSET__SHIFT                                                                     0x2
+#define SDMA_GFX_IB_OFFSET__OFFSET_MASK                                                                       0x003FFFFCL
+//SDMA_GFX_IB_BASE_LO
+#define SDMA_GFX_IB_BASE_LO__ADDR__SHIFT                                                                      0x5
+#define SDMA_GFX_IB_BASE_LO__ADDR_MASK                                                                        0xFFFFFFE0L
+//SDMA_GFX_IB_BASE_HI
+#define SDMA_GFX_IB_BASE_HI__ADDR__SHIFT                                                                      0x0
+#define SDMA_GFX_IB_BASE_HI__ADDR_MASK                                                                        0xFFFFFFFFL
+//SDMA_GFX_IB_SIZE
+#define SDMA_GFX_IB_SIZE__SIZE__SHIFT                                                                         0x0
+#define SDMA_GFX_IB_SIZE__SIZE_MASK                                                                           0x000FFFFFL
+//SDMA_GFX_SKIP_CNTL
+#define SDMA_GFX_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                 0x0
+#define SDMA_GFX_SKIP_CNTL__SKIP_COUNT_MASK                                                                   0x000FFFFFL
+//SDMA_GFX_CONTEXT_STATUS
+#define SDMA_GFX_CONTEXT_STATUS__SELECTED__SHIFT                                                              0x0
+#define SDMA_GFX_CONTEXT_STATUS__IDLE__SHIFT                                                                  0x2
+#define SDMA_GFX_CONTEXT_STATUS__EXPIRED__SHIFT                                                               0x3
+#define SDMA_GFX_CONTEXT_STATUS__EXCEPTION__SHIFT                                                             0x4
+#define SDMA_GFX_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                            0x7
+#define SDMA_GFX_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                           0x8
+#define SDMA_GFX_CONTEXT_STATUS__PREEMPTED__SHIFT                                                             0x9
+#define SDMA_GFX_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                       0xa
+#define SDMA_GFX_CONTEXT_STATUS__SELECTED_MASK                                                                0x00000001L
+#define SDMA_GFX_CONTEXT_STATUS__IDLE_MASK                                                                    0x00000004L
+#define SDMA_GFX_CONTEXT_STATUS__EXPIRED_MASK                                                                 0x00000008L
+#define SDMA_GFX_CONTEXT_STATUS__EXCEPTION_MASK                                                               0x00000070L
+#define SDMA_GFX_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                              0x00000080L
+#define SDMA_GFX_CONTEXT_STATUS__CTXSW_READY_MASK                                                             0x00000100L
+#define SDMA_GFX_CONTEXT_STATUS__PREEMPTED_MASK                                                               0x00000200L
+#define SDMA_GFX_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                         0x00000400L
+//SDMA_GFX_DOORBELL
+#define SDMA_GFX_DOORBELL__ENABLE__SHIFT                                                                      0x1c
+#define SDMA_GFX_DOORBELL__CAPTURED__SHIFT                                                                    0x1e
+#define SDMA_GFX_DOORBELL__ENABLE_MASK                                                                        0x10000000L
+#define SDMA_GFX_DOORBELL__CAPTURED_MASK                                                                      0x40000000L
+//SDMA_GFX_CONTEXT_CNTL
+#define SDMA_GFX_CONTEXT_CNTL__RESUME_CTX__SHIFT                                                              0x10
+#define SDMA_GFX_CONTEXT_CNTL__SESSION_SEL__SHIFT                                                             0x18
+#define SDMA_GFX_CONTEXT_CNTL__RESUME_CTX_MASK                                                                0x00010000L
+#define SDMA_GFX_CONTEXT_CNTL__SESSION_SEL_MASK                                                               0x0F000000L
+//SDMA_GFX_STATUS
+#define SDMA_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                        0x0
+#define SDMA_GFX_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                           0x8
+#define SDMA_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                          0x000000FFL
+#define SDMA_GFX_STATUS__WPTR_UPDATE_PENDING_MASK                                                             0x00000100L
+//SDMA_GFX_DOORBELL_LOG
+#define SDMA_GFX_DOORBELL_LOG__BE_ERROR__SHIFT                                                                0x0
+#define SDMA_GFX_DOORBELL_LOG__DATA__SHIFT                                                                    0x2
+#define SDMA_GFX_DOORBELL_LOG__BE_ERROR_MASK                                                                  0x00000001L
+#define SDMA_GFX_DOORBELL_LOG__DATA_MASK                                                                      0xFFFFFFFCL
+//SDMA_GFX_WATERMARK
+#define SDMA_GFX_WATERMARK__RD_OUTSTANDING__SHIFT                                                             0x0
+#define SDMA_GFX_WATERMARK__WR_OUTSTANDING__SHIFT                                                             0x10
+#define SDMA_GFX_WATERMARK__RD_OUTSTANDING_MASK                                                               0x00000FFFL
+#define SDMA_GFX_WATERMARK__WR_OUTSTANDING_MASK                                                               0x03FF0000L
+//SDMA_GFX_DOORBELL_OFFSET
+#define SDMA_GFX_DOORBELL_OFFSET__OFFSET__SHIFT                                                               0x2
+#define SDMA_GFX_DOORBELL_OFFSET__OFFSET_MASK                                                                 0x0FFFFFFCL
+//SDMA_GFX_CSA_ADDR_LO
+#define SDMA_GFX_CSA_ADDR_LO__ADDR__SHIFT                                                                     0x2
+#define SDMA_GFX_CSA_ADDR_LO__ADDR_MASK                                                                       0xFFFFFFFCL
+//SDMA_GFX_CSA_ADDR_HI
+#define SDMA_GFX_CSA_ADDR_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_GFX_CSA_ADDR_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA_GFX_IB_SUB_REMAIN
+#define SDMA_GFX_IB_SUB_REMAIN__SIZE__SHIFT                                                                   0x0
+#define SDMA_GFX_IB_SUB_REMAIN__SIZE_MASK                                                                     0x000FFFFFL
+//SDMA_GFX_PREEMPT
+#define SDMA_GFX_PREEMPT__IB_PREEMPT__SHIFT                                                                   0x0
+#define SDMA_GFX_PREEMPT__IB_PREEMPT_MASK                                                                     0x00000001L
+//SDMA_GFX_DUMMY_REG
+#define SDMA_GFX_DUMMY_REG__DUMMY__SHIFT                                                                      0x0
+#define SDMA_GFX_DUMMY_REG__DUMMY_MASK                                                                        0xFFFFFFFFL
+//SDMA_GFX_RB_WPTR_POLL_ADDR_HI
+#define SDMA_GFX_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA_GFX_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA_GFX_RB_WPTR_POLL_ADDR_LO
+#define SDMA_GFX_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA_GFX_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA_GFX_RB_AQL_CNTL
+#define SDMA_GFX_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                               0x0
+#define SDMA_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                          0x1
+#define SDMA_GFX_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                              0x8
+#define SDMA_GFX_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                 0x00000001L
+#define SDMA_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                            0x000000FEL
+#define SDMA_GFX_RB_AQL_CNTL__PACKET_STEP_MASK                                                                0x0000FF00L
+//SDMA_GFX_MINOR_PTR_UPDATE
+#define SDMA_GFX_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                              0x0
+#define SDMA_GFX_MINOR_PTR_UPDATE__ENABLE_MASK                                                                0x00000001L
+//SDMA_GFX_MIDCMD_DATA0
+#define SDMA_GFX_MIDCMD_DATA0__DATA0__SHIFT                                                                   0x0
+#define SDMA_GFX_MIDCMD_DATA0__DATA0_MASK                                                                     0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_DATA1
+#define SDMA_GFX_MIDCMD_DATA1__DATA1__SHIFT                                                                   0x0
+#define SDMA_GFX_MIDCMD_DATA1__DATA1_MASK                                                                     0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_DATA2
+#define SDMA_GFX_MIDCMD_DATA2__DATA2__SHIFT                                                                   0x0
+#define SDMA_GFX_MIDCMD_DATA2__DATA2_MASK                                                                     0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_DATA3
+#define SDMA_GFX_MIDCMD_DATA3__DATA3__SHIFT                                                                   0x0
+#define SDMA_GFX_MIDCMD_DATA3__DATA3_MASK                                                                     0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_DATA4
+#define SDMA_GFX_MIDCMD_DATA4__DATA4__SHIFT                                                                   0x0
+#define SDMA_GFX_MIDCMD_DATA4__DATA4_MASK                                                                     0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_DATA5
+#define SDMA_GFX_MIDCMD_DATA5__DATA5__SHIFT                                                                   0x0
+#define SDMA_GFX_MIDCMD_DATA5__DATA5_MASK                                                                     0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_DATA6
+#define SDMA_GFX_MIDCMD_DATA6__DATA6__SHIFT                                                                   0x0
+#define SDMA_GFX_MIDCMD_DATA6__DATA6_MASK                                                                     0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_DATA7
+#define SDMA_GFX_MIDCMD_DATA7__DATA7__SHIFT                                                                   0x0
+#define SDMA_GFX_MIDCMD_DATA7__DATA7_MASK                                                                     0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_DATA8
+#define SDMA_GFX_MIDCMD_DATA8__DATA8__SHIFT                                                                   0x0
+#define SDMA_GFX_MIDCMD_DATA8__DATA8_MASK                                                                     0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_DATA9
+#define SDMA_GFX_MIDCMD_DATA9__DATA9__SHIFT                                                                   0x0
+#define SDMA_GFX_MIDCMD_DATA9__DATA9_MASK                                                                     0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_DATA10
+#define SDMA_GFX_MIDCMD_DATA10__DATA10__SHIFT                                                                 0x0
+#define SDMA_GFX_MIDCMD_DATA10__DATA10_MASK                                                                   0xFFFFFFFFL
+//SDMA_GFX_MIDCMD_CNTL
+#define SDMA_GFX_MIDCMD_CNTL__DATA_VALID__SHIFT                                                               0x0
+#define SDMA_GFX_MIDCMD_CNTL__COPY_MODE__SHIFT                                                                0x1
+#define SDMA_GFX_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                              0x4
+#define SDMA_GFX_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                            0x8
+#define SDMA_GFX_MIDCMD_CNTL__DATA_VALID_MASK                                                                 0x00000001L
+#define SDMA_GFX_MIDCMD_CNTL__COPY_MODE_MASK                                                                  0x00000002L
+#define SDMA_GFX_MIDCMD_CNTL__SPLIT_STATE_MASK                                                                0x000000F0L
+#define SDMA_GFX_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                              0x00000100L
+//SDMA_PAGE_RB_CNTL
+#define SDMA_PAGE_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_PAGE_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
+#define SDMA_PAGE_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
+#define SDMA_PAGE_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
+#define SDMA_PAGE_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
+#define SDMA_PAGE_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
+#define SDMA_PAGE_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
+#define SDMA_PAGE_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
+#define SDMA_PAGE_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_PAGE_RB_CNTL__RB_SIZE_MASK                                                                       0x0000003EL
+#define SDMA_PAGE_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
+#define SDMA_PAGE_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
+#define SDMA_PAGE_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
+#define SDMA_PAGE_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
+#define SDMA_PAGE_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
+#define SDMA_PAGE_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
+//SDMA_PAGE_RB_BASE
+#define SDMA_PAGE_RB_BASE__ADDR__SHIFT                                                                        0x0
+#define SDMA_PAGE_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
+//SDMA_PAGE_RB_BASE_HI
+#define SDMA_PAGE_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_PAGE_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
+//SDMA_PAGE_RB_RPTR
+#define SDMA_PAGE_RB_RPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_PAGE_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_PAGE_RB_RPTR_HI
+#define SDMA_PAGE_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_PAGE_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_PAGE_RB_WPTR
+#define SDMA_PAGE_RB_WPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_PAGE_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_PAGE_RB_WPTR_HI
+#define SDMA_PAGE_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_PAGE_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_PAGE_RB_WPTR_POLL_CNTL
+#define SDMA_PAGE_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
+#define SDMA_PAGE_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
+#define SDMA_PAGE_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
+#define SDMA_PAGE_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
+#define SDMA_PAGE_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
+#define SDMA_PAGE_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
+#define SDMA_PAGE_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
+#define SDMA_PAGE_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
+#define SDMA_PAGE_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
+#define SDMA_PAGE_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
+//SDMA_PAGE_RB_RPTR_ADDR_HI
+#define SDMA_PAGE_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
+#define SDMA_PAGE_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA_PAGE_RB_RPTR_ADDR_LO
+#define SDMA_PAGE_RB_RPTR_ADDR_LO__RPTR_WB_IDLE__SHIFT                                                        0x0
+#define SDMA_PAGE_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
+#define SDMA_PAGE_RB_RPTR_ADDR_LO__RPTR_WB_IDLE_MASK                                                          0x00000001L
+#define SDMA_PAGE_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
+//SDMA_PAGE_IB_CNTL
+#define SDMA_PAGE_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_PAGE_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
+#define SDMA_PAGE_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
+#define SDMA_PAGE_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
+#define SDMA_PAGE_IB_CNTL__IB_PRIV__SHIFT                                                                     0x1f
+#define SDMA_PAGE_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_PAGE_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
+#define SDMA_PAGE_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
+#define SDMA_PAGE_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
+#define SDMA_PAGE_IB_CNTL__IB_PRIV_MASK                                                                       0x80000000L
+//SDMA_PAGE_IB_RPTR
+#define SDMA_PAGE_IB_RPTR__OFFSET__SHIFT                                                                      0x2
+#define SDMA_PAGE_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
+//SDMA_PAGE_IB_OFFSET
+#define SDMA_PAGE_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
+#define SDMA_PAGE_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
+//SDMA_PAGE_IB_BASE_LO
+#define SDMA_PAGE_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
+#define SDMA_PAGE_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
+//SDMA_PAGE_IB_BASE_HI
+#define SDMA_PAGE_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_PAGE_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA_PAGE_IB_SIZE
+#define SDMA_PAGE_IB_SIZE__SIZE__SHIFT                                                                        0x0
+#define SDMA_PAGE_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
+//SDMA_PAGE_SKIP_CNTL
+#define SDMA_PAGE_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
+#define SDMA_PAGE_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x000FFFFFL
+//SDMA_PAGE_CONTEXT_STATUS
+#define SDMA_PAGE_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
+#define SDMA_PAGE_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
+#define SDMA_PAGE_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
+#define SDMA_PAGE_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
+#define SDMA_PAGE_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
+#define SDMA_PAGE_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
+#define SDMA_PAGE_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
+#define SDMA_PAGE_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
+#define SDMA_PAGE_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
+#define SDMA_PAGE_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
+#define SDMA_PAGE_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
+#define SDMA_PAGE_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
+#define SDMA_PAGE_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
+#define SDMA_PAGE_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
+#define SDMA_PAGE_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
+#define SDMA_PAGE_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
+//SDMA_PAGE_DOORBELL
+#define SDMA_PAGE_DOORBELL__ENABLE__SHIFT                                                                     0x1c
+#define SDMA_PAGE_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
+#define SDMA_PAGE_DOORBELL__ENABLE_MASK                                                                       0x10000000L
+#define SDMA_PAGE_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
+//SDMA_PAGE_STATUS
+#define SDMA_PAGE_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
+#define SDMA_PAGE_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
+#define SDMA_PAGE_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
+#define SDMA_PAGE_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
+//SDMA_PAGE_DOORBELL_LOG
+#define SDMA_PAGE_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
+#define SDMA_PAGE_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
+#define SDMA_PAGE_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
+#define SDMA_PAGE_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
+//SDMA_PAGE_WATERMARK
+#define SDMA_PAGE_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
+#define SDMA_PAGE_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
+#define SDMA_PAGE_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
+#define SDMA_PAGE_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
+//SDMA_PAGE_DOORBELL_OFFSET
+#define SDMA_PAGE_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
+#define SDMA_PAGE_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
+//SDMA_PAGE_CSA_ADDR_LO
+#define SDMA_PAGE_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
+#define SDMA_PAGE_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
+//SDMA_PAGE_CSA_ADDR_HI
+#define SDMA_PAGE_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
+#define SDMA_PAGE_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
+//SDMA_PAGE_IB_SUB_REMAIN
+#define SDMA_PAGE_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
+#define SDMA_PAGE_IB_SUB_REMAIN__SIZE_MASK                                                                    0x000FFFFFL
+//SDMA_PAGE_PREEMPT
+#define SDMA_PAGE_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
+#define SDMA_PAGE_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
+//SDMA_PAGE_DUMMY_REG
+#define SDMA_PAGE_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
+#define SDMA_PAGE_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
+//SDMA_PAGE_RB_WPTR_POLL_ADDR_HI
+#define SDMA_PAGE_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define SDMA_PAGE_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
+//SDMA_PAGE_RB_WPTR_POLL_ADDR_LO
+#define SDMA_PAGE_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define SDMA_PAGE_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+//SDMA_PAGE_RB_AQL_CNTL
+#define SDMA_PAGE_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
+#define SDMA_PAGE_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
+#define SDMA_PAGE_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
+#define SDMA_PAGE_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
+#define SDMA_PAGE_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
+#define SDMA_PAGE_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
+//SDMA_PAGE_MINOR_PTR_UPDATE
+#define SDMA_PAGE_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
+#define SDMA_PAGE_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
+//SDMA_PAGE_MIDCMD_DATA0
+#define SDMA_PAGE_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
+#define SDMA_PAGE_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_DATA1
+#define SDMA_PAGE_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
+#define SDMA_PAGE_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_DATA2
+#define SDMA_PAGE_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
+#define SDMA_PAGE_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_DATA3
+#define SDMA_PAGE_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
+#define SDMA_PAGE_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_DATA4
+#define SDMA_PAGE_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
+#define SDMA_PAGE_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_DATA5
+#define SDMA_PAGE_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
+#define SDMA_PAGE_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_DATA6
+#define SDMA_PAGE_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
+#define SDMA_PAGE_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_DATA7
+#define SDMA_PAGE_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
+#define SDMA_PAGE_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_DATA8
+#define SDMA_PAGE_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
+#define SDMA_PAGE_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_DATA9
+#define SDMA_PAGE_MIDCMD_DATA9__DATA9__SHIFT                                                                  0x0
+#define SDMA_PAGE_MIDCMD_DATA9__DATA9_MASK                                                                    0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_DATA10
+#define SDMA_PAGE_MIDCMD_DATA10__DATA10__SHIFT                                                                0x0
+#define SDMA_PAGE_MIDCMD_DATA10__DATA10_MASK                                                                  0xFFFFFFFFL
+//SDMA_PAGE_MIDCMD_CNTL
+#define SDMA_PAGE_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
+#define SDMA_PAGE_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
+#define SDMA_PAGE_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
+#define SDMA_PAGE_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
+#define SDMA_PAGE_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
+#define SDMA_PAGE_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
+#define SDMA_PAGE_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
+#define SDMA_PAGE_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
+//SDMA_RLC0_RB_CNTL
+#define SDMA_RLC0_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC0_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
+#define SDMA_RLC0_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
+#define SDMA_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
+#define SDMA_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
+#define SDMA_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
+#define SDMA_RLC0_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
+#define SDMA_RLC0_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
+#define SDMA_RLC0_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC0_RB_CNTL__RB_SIZE_MASK                                                                       0x0000003EL
+#define SDMA_RLC0_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
+#define SDMA_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
+#define SDMA_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
+#define SDMA_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
+#define SDMA_RLC0_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
+#define SDMA_RLC0_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
+//SDMA_RLC0_RB_BASE
+#define SDMA_RLC0_RB_BASE__ADDR__SHIFT                                                                        0x0
+#define SDMA_RLC0_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
+//SDMA_RLC0_RB_BASE_HI
+#define SDMA_RLC0_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC0_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
+//SDMA_RLC0_RB_RPTR
+#define SDMA_RLC0_RB_RPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC0_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC0_RB_RPTR_HI
+#define SDMA_RLC0_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC0_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC0_RB_WPTR
+#define SDMA_RLC0_RB_WPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC0_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC0_RB_WPTR_HI
+#define SDMA_RLC0_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC0_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC0_RB_WPTR_POLL_CNTL
+#define SDMA_RLC0_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
+#define SDMA_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
+#define SDMA_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
+#define SDMA_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
+#define SDMA_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
+#define SDMA_RLC0_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
+#define SDMA_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
+#define SDMA_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
+#define SDMA_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
+#define SDMA_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
+//SDMA_RLC0_RB_RPTR_ADDR_HI
+#define SDMA_RLC0_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
+#define SDMA_RLC0_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC0_RB_RPTR_ADDR_LO
+#define SDMA_RLC0_RB_RPTR_ADDR_LO__RPTR_WB_IDLE__SHIFT                                                        0x0
+#define SDMA_RLC0_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
+#define SDMA_RLC0_RB_RPTR_ADDR_LO__RPTR_WB_IDLE_MASK                                                          0x00000001L
+#define SDMA_RLC0_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
+//SDMA_RLC0_IB_CNTL
+#define SDMA_RLC0_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC0_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
+#define SDMA_RLC0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
+#define SDMA_RLC0_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
+#define SDMA_RLC0_IB_CNTL__IB_PRIV__SHIFT                                                                     0x1f
+#define SDMA_RLC0_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC0_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
+#define SDMA_RLC0_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
+#define SDMA_RLC0_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
+#define SDMA_RLC0_IB_CNTL__IB_PRIV_MASK                                                                       0x80000000L
+//SDMA_RLC0_IB_RPTR
+#define SDMA_RLC0_IB_RPTR__OFFSET__SHIFT                                                                      0x2
+#define SDMA_RLC0_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
+//SDMA_RLC0_IB_OFFSET
+#define SDMA_RLC0_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
+#define SDMA_RLC0_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
+//SDMA_RLC0_IB_BASE_LO
+#define SDMA_RLC0_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
+#define SDMA_RLC0_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
+//SDMA_RLC0_IB_BASE_HI
+#define SDMA_RLC0_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC0_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC0_IB_SIZE
+#define SDMA_RLC0_IB_SIZE__SIZE__SHIFT                                                                        0x0
+#define SDMA_RLC0_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
+//SDMA_RLC0_SKIP_CNTL
+#define SDMA_RLC0_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
+#define SDMA_RLC0_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x000FFFFFL
+//SDMA_RLC0_CONTEXT_STATUS
+#define SDMA_RLC0_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
+#define SDMA_RLC0_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
+#define SDMA_RLC0_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
+#define SDMA_RLC0_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
+#define SDMA_RLC0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
+#define SDMA_RLC0_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
+#define SDMA_RLC0_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
+#define SDMA_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
+#define SDMA_RLC0_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
+#define SDMA_RLC0_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
+#define SDMA_RLC0_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
+#define SDMA_RLC0_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
+#define SDMA_RLC0_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
+#define SDMA_RLC0_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
+#define SDMA_RLC0_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
+#define SDMA_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
+//SDMA_RLC0_DOORBELL
+#define SDMA_RLC0_DOORBELL__ENABLE__SHIFT                                                                     0x1c
+#define SDMA_RLC0_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
+#define SDMA_RLC0_DOORBELL__ENABLE_MASK                                                                       0x10000000L
+#define SDMA_RLC0_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
+//SDMA_RLC0_STATUS
+#define SDMA_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
+#define SDMA_RLC0_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
+#define SDMA_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
+#define SDMA_RLC0_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
+//SDMA_RLC0_DOORBELL_LOG
+#define SDMA_RLC0_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
+#define SDMA_RLC0_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
+#define SDMA_RLC0_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
+#define SDMA_RLC0_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
+//SDMA_RLC0_WATERMARK
+#define SDMA_RLC0_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
+#define SDMA_RLC0_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
+#define SDMA_RLC0_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
+#define SDMA_RLC0_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
+//SDMA_RLC0_DOORBELL_OFFSET
+#define SDMA_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
+#define SDMA_RLC0_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
+//SDMA_RLC0_CSA_ADDR_LO
+#define SDMA_RLC0_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
+#define SDMA_RLC0_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
+//SDMA_RLC0_CSA_ADDR_HI
+#define SDMA_RLC0_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
+#define SDMA_RLC0_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
+//SDMA_RLC0_IB_SUB_REMAIN
+#define SDMA_RLC0_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
+#define SDMA_RLC0_IB_SUB_REMAIN__SIZE_MASK                                                                    0x000FFFFFL
+//SDMA_RLC0_PREEMPT
+#define SDMA_RLC0_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
+#define SDMA_RLC0_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
+//SDMA_RLC0_DUMMY_REG
+#define SDMA_RLC0_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
+#define SDMA_RLC0_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC0_RB_WPTR_POLL_ADDR_HI
+#define SDMA_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define SDMA_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
+//SDMA_RLC0_RB_WPTR_POLL_ADDR_LO
+#define SDMA_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define SDMA_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+//SDMA_RLC0_RB_AQL_CNTL
+#define SDMA_RLC0_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
+#define SDMA_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
+#define SDMA_RLC0_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
+#define SDMA_RLC0_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
+#define SDMA_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
+#define SDMA_RLC0_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
+//SDMA_RLC0_MINOR_PTR_UPDATE
+#define SDMA_RLC0_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
+#define SDMA_RLC0_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
+//SDMA_RLC0_MIDCMD_DATA0
+#define SDMA_RLC0_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
+#define SDMA_RLC0_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_DATA1
+#define SDMA_RLC0_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
+#define SDMA_RLC0_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_DATA2
+#define SDMA_RLC0_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
+#define SDMA_RLC0_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_DATA3
+#define SDMA_RLC0_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
+#define SDMA_RLC0_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_DATA4
+#define SDMA_RLC0_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
+#define SDMA_RLC0_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_DATA5
+#define SDMA_RLC0_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
+#define SDMA_RLC0_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_DATA6
+#define SDMA_RLC0_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
+#define SDMA_RLC0_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_DATA7
+#define SDMA_RLC0_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
+#define SDMA_RLC0_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_DATA8
+#define SDMA_RLC0_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
+#define SDMA_RLC0_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_DATA9
+#define SDMA_RLC0_MIDCMD_DATA9__DATA9__SHIFT                                                                  0x0
+#define SDMA_RLC0_MIDCMD_DATA9__DATA9_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_DATA10
+#define SDMA_RLC0_MIDCMD_DATA10__DATA10__SHIFT                                                                0x0
+#define SDMA_RLC0_MIDCMD_DATA10__DATA10_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC0_MIDCMD_CNTL
+#define SDMA_RLC0_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
+#define SDMA_RLC0_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
+#define SDMA_RLC0_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
+#define SDMA_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
+#define SDMA_RLC0_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
+#define SDMA_RLC0_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
+#define SDMA_RLC0_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
+#define SDMA_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
+//SDMA_RLC1_RB_CNTL
+#define SDMA_RLC1_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC1_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
+#define SDMA_RLC1_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
+#define SDMA_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
+#define SDMA_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
+#define SDMA_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
+#define SDMA_RLC1_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
+#define SDMA_RLC1_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
+#define SDMA_RLC1_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC1_RB_CNTL__RB_SIZE_MASK                                                                       0x0000003EL
+#define SDMA_RLC1_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
+#define SDMA_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
+#define SDMA_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
+#define SDMA_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
+#define SDMA_RLC1_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
+#define SDMA_RLC1_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
+//SDMA_RLC1_RB_BASE
+#define SDMA_RLC1_RB_BASE__ADDR__SHIFT                                                                        0x0
+#define SDMA_RLC1_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
+//SDMA_RLC1_RB_BASE_HI
+#define SDMA_RLC1_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC1_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
+//SDMA_RLC1_RB_RPTR
+#define SDMA_RLC1_RB_RPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC1_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC1_RB_RPTR_HI
+#define SDMA_RLC1_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC1_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC1_RB_WPTR
+#define SDMA_RLC1_RB_WPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC1_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC1_RB_WPTR_HI
+#define SDMA_RLC1_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC1_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC1_RB_WPTR_POLL_CNTL
+#define SDMA_RLC1_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
+#define SDMA_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
+#define SDMA_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
+#define SDMA_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
+#define SDMA_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
+#define SDMA_RLC1_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
+#define SDMA_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
+#define SDMA_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
+#define SDMA_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
+#define SDMA_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
+//SDMA_RLC1_RB_RPTR_ADDR_HI
+#define SDMA_RLC1_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
+#define SDMA_RLC1_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC1_RB_RPTR_ADDR_LO
+#define SDMA_RLC1_RB_RPTR_ADDR_LO__RPTR_WB_IDLE__SHIFT                                                        0x0
+#define SDMA_RLC1_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
+#define SDMA_RLC1_RB_RPTR_ADDR_LO__RPTR_WB_IDLE_MASK                                                          0x00000001L
+#define SDMA_RLC1_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
+//SDMA_RLC1_IB_CNTL
+#define SDMA_RLC1_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC1_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
+#define SDMA_RLC1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
+#define SDMA_RLC1_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
+#define SDMA_RLC1_IB_CNTL__IB_PRIV__SHIFT                                                                     0x1f
+#define SDMA_RLC1_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC1_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
+#define SDMA_RLC1_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
+#define SDMA_RLC1_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
+#define SDMA_RLC1_IB_CNTL__IB_PRIV_MASK                                                                       0x80000000L
+//SDMA_RLC1_IB_RPTR
+#define SDMA_RLC1_IB_RPTR__OFFSET__SHIFT                                                                      0x2
+#define SDMA_RLC1_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
+//SDMA_RLC1_IB_OFFSET
+#define SDMA_RLC1_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
+#define SDMA_RLC1_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
+//SDMA_RLC1_IB_BASE_LO
+#define SDMA_RLC1_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
+#define SDMA_RLC1_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
+//SDMA_RLC1_IB_BASE_HI
+#define SDMA_RLC1_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC1_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC1_IB_SIZE
+#define SDMA_RLC1_IB_SIZE__SIZE__SHIFT                                                                        0x0
+#define SDMA_RLC1_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
+//SDMA_RLC1_SKIP_CNTL
+#define SDMA_RLC1_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
+#define SDMA_RLC1_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x000FFFFFL
+//SDMA_RLC1_CONTEXT_STATUS
+#define SDMA_RLC1_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
+#define SDMA_RLC1_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
+#define SDMA_RLC1_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
+#define SDMA_RLC1_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
+#define SDMA_RLC1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
+#define SDMA_RLC1_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
+#define SDMA_RLC1_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
+#define SDMA_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
+#define SDMA_RLC1_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
+#define SDMA_RLC1_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
+#define SDMA_RLC1_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
+#define SDMA_RLC1_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
+#define SDMA_RLC1_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
+#define SDMA_RLC1_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
+#define SDMA_RLC1_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
+#define SDMA_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
+//SDMA_RLC1_DOORBELL
+#define SDMA_RLC1_DOORBELL__ENABLE__SHIFT                                                                     0x1c
+#define SDMA_RLC1_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
+#define SDMA_RLC1_DOORBELL__ENABLE_MASK                                                                       0x10000000L
+#define SDMA_RLC1_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
+//SDMA_RLC1_STATUS
+#define SDMA_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
+#define SDMA_RLC1_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
+#define SDMA_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
+#define SDMA_RLC1_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
+//SDMA_RLC1_DOORBELL_LOG
+#define SDMA_RLC1_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
+#define SDMA_RLC1_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
+#define SDMA_RLC1_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
+#define SDMA_RLC1_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
+//SDMA_RLC1_WATERMARK
+#define SDMA_RLC1_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
+#define SDMA_RLC1_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
+#define SDMA_RLC1_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
+#define SDMA_RLC1_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
+//SDMA_RLC1_DOORBELL_OFFSET
+#define SDMA_RLC1_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
+#define SDMA_RLC1_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
+//SDMA_RLC1_CSA_ADDR_LO
+#define SDMA_RLC1_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
+#define SDMA_RLC1_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
+//SDMA_RLC1_CSA_ADDR_HI
+#define SDMA_RLC1_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
+#define SDMA_RLC1_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
+//SDMA_RLC1_IB_SUB_REMAIN
+#define SDMA_RLC1_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
+#define SDMA_RLC1_IB_SUB_REMAIN__SIZE_MASK                                                                    0x000FFFFFL
+//SDMA_RLC1_PREEMPT
+#define SDMA_RLC1_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
+#define SDMA_RLC1_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
+//SDMA_RLC1_DUMMY_REG
+#define SDMA_RLC1_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
+#define SDMA_RLC1_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC1_RB_WPTR_POLL_ADDR_HI
+#define SDMA_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define SDMA_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
+//SDMA_RLC1_RB_WPTR_POLL_ADDR_LO
+#define SDMA_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define SDMA_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+//SDMA_RLC1_RB_AQL_CNTL
+#define SDMA_RLC1_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
+#define SDMA_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
+#define SDMA_RLC1_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
+#define SDMA_RLC1_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
+#define SDMA_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
+#define SDMA_RLC1_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
+//SDMA_RLC1_MINOR_PTR_UPDATE
+#define SDMA_RLC1_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
+#define SDMA_RLC1_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
+//SDMA_RLC1_MIDCMD_DATA0
+#define SDMA_RLC1_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
+#define SDMA_RLC1_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_DATA1
+#define SDMA_RLC1_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
+#define SDMA_RLC1_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_DATA2
+#define SDMA_RLC1_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
+#define SDMA_RLC1_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_DATA3
+#define SDMA_RLC1_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
+#define SDMA_RLC1_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_DATA4
+#define SDMA_RLC1_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
+#define SDMA_RLC1_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_DATA5
+#define SDMA_RLC1_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
+#define SDMA_RLC1_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_DATA6
+#define SDMA_RLC1_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
+#define SDMA_RLC1_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_DATA7
+#define SDMA_RLC1_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
+#define SDMA_RLC1_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_DATA8
+#define SDMA_RLC1_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
+#define SDMA_RLC1_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_DATA9
+#define SDMA_RLC1_MIDCMD_DATA9__DATA9__SHIFT                                                                  0x0
+#define SDMA_RLC1_MIDCMD_DATA9__DATA9_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_DATA10
+#define SDMA_RLC1_MIDCMD_DATA10__DATA10__SHIFT                                                                0x0
+#define SDMA_RLC1_MIDCMD_DATA10__DATA10_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC1_MIDCMD_CNTL
+#define SDMA_RLC1_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
+#define SDMA_RLC1_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
+#define SDMA_RLC1_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
+#define SDMA_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
+#define SDMA_RLC1_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
+#define SDMA_RLC1_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
+#define SDMA_RLC1_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
+#define SDMA_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
+//SDMA_RLC2_RB_CNTL
+#define SDMA_RLC2_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC2_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
+#define SDMA_RLC2_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
+#define SDMA_RLC2_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
+#define SDMA_RLC2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
+#define SDMA_RLC2_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
+#define SDMA_RLC2_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
+#define SDMA_RLC2_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
+#define SDMA_RLC2_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC2_RB_CNTL__RB_SIZE_MASK                                                                       0x0000003EL
+#define SDMA_RLC2_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
+#define SDMA_RLC2_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
+#define SDMA_RLC2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
+#define SDMA_RLC2_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
+#define SDMA_RLC2_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
+#define SDMA_RLC2_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
+//SDMA_RLC2_RB_BASE
+#define SDMA_RLC2_RB_BASE__ADDR__SHIFT                                                                        0x0
+#define SDMA_RLC2_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
+//SDMA_RLC2_RB_BASE_HI
+#define SDMA_RLC2_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC2_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
+//SDMA_RLC2_RB_RPTR
+#define SDMA_RLC2_RB_RPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC2_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC2_RB_RPTR_HI
+#define SDMA_RLC2_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC2_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC2_RB_WPTR
+#define SDMA_RLC2_RB_WPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC2_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC2_RB_WPTR_HI
+#define SDMA_RLC2_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC2_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC2_RB_WPTR_POLL_CNTL
+#define SDMA_RLC2_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
+#define SDMA_RLC2_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
+#define SDMA_RLC2_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
+#define SDMA_RLC2_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
+#define SDMA_RLC2_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
+#define SDMA_RLC2_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
+#define SDMA_RLC2_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
+#define SDMA_RLC2_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
+#define SDMA_RLC2_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
+#define SDMA_RLC2_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
+//SDMA_RLC2_RB_RPTR_ADDR_HI
+#define SDMA_RLC2_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
+#define SDMA_RLC2_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC2_RB_RPTR_ADDR_LO
+#define SDMA_RLC2_RB_RPTR_ADDR_LO__RPTR_WB_IDLE__SHIFT                                                        0x0
+#define SDMA_RLC2_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
+#define SDMA_RLC2_RB_RPTR_ADDR_LO__RPTR_WB_IDLE_MASK                                                          0x00000001L
+#define SDMA_RLC2_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
+//SDMA_RLC2_IB_CNTL
+#define SDMA_RLC2_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC2_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
+#define SDMA_RLC2_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
+#define SDMA_RLC2_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
+#define SDMA_RLC2_IB_CNTL__IB_PRIV__SHIFT                                                                     0x1f
+#define SDMA_RLC2_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC2_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
+#define SDMA_RLC2_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
+#define SDMA_RLC2_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
+#define SDMA_RLC2_IB_CNTL__IB_PRIV_MASK                                                                       0x80000000L
+//SDMA_RLC2_IB_RPTR
+#define SDMA_RLC2_IB_RPTR__OFFSET__SHIFT                                                                      0x2
+#define SDMA_RLC2_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
+//SDMA_RLC2_IB_OFFSET
+#define SDMA_RLC2_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
+#define SDMA_RLC2_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
+//SDMA_RLC2_IB_BASE_LO
+#define SDMA_RLC2_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
+#define SDMA_RLC2_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
+//SDMA_RLC2_IB_BASE_HI
+#define SDMA_RLC2_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC2_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC2_IB_SIZE
+#define SDMA_RLC2_IB_SIZE__SIZE__SHIFT                                                                        0x0
+#define SDMA_RLC2_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
+//SDMA_RLC2_SKIP_CNTL
+#define SDMA_RLC2_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
+#define SDMA_RLC2_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x000FFFFFL
+//SDMA_RLC2_CONTEXT_STATUS
+#define SDMA_RLC2_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
+#define SDMA_RLC2_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
+#define SDMA_RLC2_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
+#define SDMA_RLC2_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
+#define SDMA_RLC2_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
+#define SDMA_RLC2_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
+#define SDMA_RLC2_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
+#define SDMA_RLC2_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
+#define SDMA_RLC2_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
+#define SDMA_RLC2_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
+#define SDMA_RLC2_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
+#define SDMA_RLC2_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
+#define SDMA_RLC2_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
+#define SDMA_RLC2_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
+#define SDMA_RLC2_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
+#define SDMA_RLC2_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
+//SDMA_RLC2_DOORBELL
+#define SDMA_RLC2_DOORBELL__ENABLE__SHIFT                                                                     0x1c
+#define SDMA_RLC2_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
+#define SDMA_RLC2_DOORBELL__ENABLE_MASK                                                                       0x10000000L
+#define SDMA_RLC2_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
+//SDMA_RLC2_STATUS
+#define SDMA_RLC2_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
+#define SDMA_RLC2_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
+#define SDMA_RLC2_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
+#define SDMA_RLC2_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
+//SDMA_RLC2_DOORBELL_LOG
+#define SDMA_RLC2_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
+#define SDMA_RLC2_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
+#define SDMA_RLC2_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
+#define SDMA_RLC2_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
+//SDMA_RLC2_WATERMARK
+#define SDMA_RLC2_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
+#define SDMA_RLC2_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
+#define SDMA_RLC2_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
+#define SDMA_RLC2_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
+//SDMA_RLC2_DOORBELL_OFFSET
+#define SDMA_RLC2_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
+#define SDMA_RLC2_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
+//SDMA_RLC2_CSA_ADDR_LO
+#define SDMA_RLC2_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
+#define SDMA_RLC2_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
+//SDMA_RLC2_CSA_ADDR_HI
+#define SDMA_RLC2_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
+#define SDMA_RLC2_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
+//SDMA_RLC2_IB_SUB_REMAIN
+#define SDMA_RLC2_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
+#define SDMA_RLC2_IB_SUB_REMAIN__SIZE_MASK                                                                    0x000FFFFFL
+//SDMA_RLC2_PREEMPT
+#define SDMA_RLC2_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
+#define SDMA_RLC2_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
+//SDMA_RLC2_DUMMY_REG
+#define SDMA_RLC2_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
+#define SDMA_RLC2_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC2_RB_WPTR_POLL_ADDR_HI
+#define SDMA_RLC2_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define SDMA_RLC2_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
+//SDMA_RLC2_RB_WPTR_POLL_ADDR_LO
+#define SDMA_RLC2_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define SDMA_RLC2_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+//SDMA_RLC2_RB_AQL_CNTL
+#define SDMA_RLC2_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
+#define SDMA_RLC2_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
+#define SDMA_RLC2_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
+#define SDMA_RLC2_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
+#define SDMA_RLC2_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
+#define SDMA_RLC2_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
+//SDMA_RLC2_MINOR_PTR_UPDATE
+#define SDMA_RLC2_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
+#define SDMA_RLC2_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
+//SDMA_RLC2_MIDCMD_DATA0
+#define SDMA_RLC2_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
+#define SDMA_RLC2_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_DATA1
+#define SDMA_RLC2_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
+#define SDMA_RLC2_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_DATA2
+#define SDMA_RLC2_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
+#define SDMA_RLC2_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_DATA3
+#define SDMA_RLC2_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
+#define SDMA_RLC2_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_DATA4
+#define SDMA_RLC2_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
+#define SDMA_RLC2_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_DATA5
+#define SDMA_RLC2_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
+#define SDMA_RLC2_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_DATA6
+#define SDMA_RLC2_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
+#define SDMA_RLC2_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_DATA7
+#define SDMA_RLC2_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
+#define SDMA_RLC2_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_DATA8
+#define SDMA_RLC2_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
+#define SDMA_RLC2_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_DATA9
+#define SDMA_RLC2_MIDCMD_DATA9__DATA9__SHIFT                                                                  0x0
+#define SDMA_RLC2_MIDCMD_DATA9__DATA9_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_DATA10
+#define SDMA_RLC2_MIDCMD_DATA10__DATA10__SHIFT                                                                0x0
+#define SDMA_RLC2_MIDCMD_DATA10__DATA10_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC2_MIDCMD_CNTL
+#define SDMA_RLC2_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
+#define SDMA_RLC2_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
+#define SDMA_RLC2_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
+#define SDMA_RLC2_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
+#define SDMA_RLC2_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
+#define SDMA_RLC2_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
+#define SDMA_RLC2_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
+#define SDMA_RLC2_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
+//SDMA_RLC3_RB_CNTL
+#define SDMA_RLC3_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC3_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
+#define SDMA_RLC3_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
+#define SDMA_RLC3_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
+#define SDMA_RLC3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
+#define SDMA_RLC3_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
+#define SDMA_RLC3_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
+#define SDMA_RLC3_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
+#define SDMA_RLC3_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC3_RB_CNTL__RB_SIZE_MASK                                                                       0x0000003EL
+#define SDMA_RLC3_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
+#define SDMA_RLC3_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
+#define SDMA_RLC3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
+#define SDMA_RLC3_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
+#define SDMA_RLC3_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
+#define SDMA_RLC3_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
+//SDMA_RLC3_RB_BASE
+#define SDMA_RLC3_RB_BASE__ADDR__SHIFT                                                                        0x0
+#define SDMA_RLC3_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
+//SDMA_RLC3_RB_BASE_HI
+#define SDMA_RLC3_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC3_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
+//SDMA_RLC3_RB_RPTR
+#define SDMA_RLC3_RB_RPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC3_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC3_RB_RPTR_HI
+#define SDMA_RLC3_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC3_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC3_RB_WPTR
+#define SDMA_RLC3_RB_WPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC3_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC3_RB_WPTR_HI
+#define SDMA_RLC3_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC3_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC3_RB_WPTR_POLL_CNTL
+#define SDMA_RLC3_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
+#define SDMA_RLC3_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
+#define SDMA_RLC3_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
+#define SDMA_RLC3_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
+#define SDMA_RLC3_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
+#define SDMA_RLC3_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
+#define SDMA_RLC3_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
+#define SDMA_RLC3_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
+#define SDMA_RLC3_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
+#define SDMA_RLC3_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
+//SDMA_RLC3_RB_RPTR_ADDR_HI
+#define SDMA_RLC3_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
+#define SDMA_RLC3_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC3_RB_RPTR_ADDR_LO
+#define SDMA_RLC3_RB_RPTR_ADDR_LO__RPTR_WB_IDLE__SHIFT                                                        0x0
+#define SDMA_RLC3_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
+#define SDMA_RLC3_RB_RPTR_ADDR_LO__RPTR_WB_IDLE_MASK                                                          0x00000001L
+#define SDMA_RLC3_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
+//SDMA_RLC3_IB_CNTL
+#define SDMA_RLC3_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC3_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
+#define SDMA_RLC3_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
+#define SDMA_RLC3_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
+#define SDMA_RLC3_IB_CNTL__IB_PRIV__SHIFT                                                                     0x1f
+#define SDMA_RLC3_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC3_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
+#define SDMA_RLC3_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
+#define SDMA_RLC3_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
+#define SDMA_RLC3_IB_CNTL__IB_PRIV_MASK                                                                       0x80000000L
+//SDMA_RLC3_IB_RPTR
+#define SDMA_RLC3_IB_RPTR__OFFSET__SHIFT                                                                      0x2
+#define SDMA_RLC3_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
+//SDMA_RLC3_IB_OFFSET
+#define SDMA_RLC3_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
+#define SDMA_RLC3_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
+//SDMA_RLC3_IB_BASE_LO
+#define SDMA_RLC3_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
+#define SDMA_RLC3_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
+//SDMA_RLC3_IB_BASE_HI
+#define SDMA_RLC3_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC3_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC3_IB_SIZE
+#define SDMA_RLC3_IB_SIZE__SIZE__SHIFT                                                                        0x0
+#define SDMA_RLC3_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
+//SDMA_RLC3_SKIP_CNTL
+#define SDMA_RLC3_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
+#define SDMA_RLC3_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x000FFFFFL
+//SDMA_RLC3_CONTEXT_STATUS
+#define SDMA_RLC3_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
+#define SDMA_RLC3_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
+#define SDMA_RLC3_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
+#define SDMA_RLC3_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
+#define SDMA_RLC3_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
+#define SDMA_RLC3_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
+#define SDMA_RLC3_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
+#define SDMA_RLC3_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
+#define SDMA_RLC3_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
+#define SDMA_RLC3_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
+#define SDMA_RLC3_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
+#define SDMA_RLC3_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
+#define SDMA_RLC3_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
+#define SDMA_RLC3_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
+#define SDMA_RLC3_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
+#define SDMA_RLC3_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
+//SDMA_RLC3_DOORBELL
+#define SDMA_RLC3_DOORBELL__ENABLE__SHIFT                                                                     0x1c
+#define SDMA_RLC3_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
+#define SDMA_RLC3_DOORBELL__ENABLE_MASK                                                                       0x10000000L
+#define SDMA_RLC3_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
+//SDMA_RLC3_STATUS
+#define SDMA_RLC3_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
+#define SDMA_RLC3_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
+#define SDMA_RLC3_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
+#define SDMA_RLC3_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
+//SDMA_RLC3_DOORBELL_LOG
+#define SDMA_RLC3_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
+#define SDMA_RLC3_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
+#define SDMA_RLC3_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
+#define SDMA_RLC3_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
+//SDMA_RLC3_WATERMARK
+#define SDMA_RLC3_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
+#define SDMA_RLC3_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
+#define SDMA_RLC3_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
+#define SDMA_RLC3_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
+//SDMA_RLC3_DOORBELL_OFFSET
+#define SDMA_RLC3_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
+#define SDMA_RLC3_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
+//SDMA_RLC3_CSA_ADDR_LO
+#define SDMA_RLC3_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
+#define SDMA_RLC3_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
+//SDMA_RLC3_CSA_ADDR_HI
+#define SDMA_RLC3_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
+#define SDMA_RLC3_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
+//SDMA_RLC3_IB_SUB_REMAIN
+#define SDMA_RLC3_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
+#define SDMA_RLC3_IB_SUB_REMAIN__SIZE_MASK                                                                    0x000FFFFFL
+//SDMA_RLC3_PREEMPT
+#define SDMA_RLC3_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
+#define SDMA_RLC3_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
+//SDMA_RLC3_DUMMY_REG
+#define SDMA_RLC3_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
+#define SDMA_RLC3_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC3_RB_WPTR_POLL_ADDR_HI
+#define SDMA_RLC3_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define SDMA_RLC3_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
+//SDMA_RLC3_RB_WPTR_POLL_ADDR_LO
+#define SDMA_RLC3_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define SDMA_RLC3_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+//SDMA_RLC3_RB_AQL_CNTL
+#define SDMA_RLC3_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
+#define SDMA_RLC3_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
+#define SDMA_RLC3_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
+#define SDMA_RLC3_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
+#define SDMA_RLC3_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
+#define SDMA_RLC3_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
+//SDMA_RLC3_MINOR_PTR_UPDATE
+#define SDMA_RLC3_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
+#define SDMA_RLC3_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
+//SDMA_RLC3_MIDCMD_DATA0
+#define SDMA_RLC3_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
+#define SDMA_RLC3_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_DATA1
+#define SDMA_RLC3_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
+#define SDMA_RLC3_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_DATA2
+#define SDMA_RLC3_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
+#define SDMA_RLC3_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_DATA3
+#define SDMA_RLC3_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
+#define SDMA_RLC3_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_DATA4
+#define SDMA_RLC3_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
+#define SDMA_RLC3_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_DATA5
+#define SDMA_RLC3_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
+#define SDMA_RLC3_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_DATA6
+#define SDMA_RLC3_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
+#define SDMA_RLC3_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_DATA7
+#define SDMA_RLC3_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
+#define SDMA_RLC3_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_DATA8
+#define SDMA_RLC3_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
+#define SDMA_RLC3_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_DATA9
+#define SDMA_RLC3_MIDCMD_DATA9__DATA9__SHIFT                                                                  0x0
+#define SDMA_RLC3_MIDCMD_DATA9__DATA9_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_DATA10
+#define SDMA_RLC3_MIDCMD_DATA10__DATA10__SHIFT                                                                0x0
+#define SDMA_RLC3_MIDCMD_DATA10__DATA10_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC3_MIDCMD_CNTL
+#define SDMA_RLC3_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
+#define SDMA_RLC3_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
+#define SDMA_RLC3_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
+#define SDMA_RLC3_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
+#define SDMA_RLC3_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
+#define SDMA_RLC3_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
+#define SDMA_RLC3_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
+#define SDMA_RLC3_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
+//SDMA_RLC4_RB_CNTL
+#define SDMA_RLC4_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC4_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
+#define SDMA_RLC4_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
+#define SDMA_RLC4_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
+#define SDMA_RLC4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
+#define SDMA_RLC4_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
+#define SDMA_RLC4_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
+#define SDMA_RLC4_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
+#define SDMA_RLC4_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC4_RB_CNTL__RB_SIZE_MASK                                                                       0x0000003EL
+#define SDMA_RLC4_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
+#define SDMA_RLC4_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
+#define SDMA_RLC4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
+#define SDMA_RLC4_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
+#define SDMA_RLC4_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
+#define SDMA_RLC4_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
+//SDMA_RLC4_RB_BASE
+#define SDMA_RLC4_RB_BASE__ADDR__SHIFT                                                                        0x0
+#define SDMA_RLC4_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
+//SDMA_RLC4_RB_BASE_HI
+#define SDMA_RLC4_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC4_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
+//SDMA_RLC4_RB_RPTR
+#define SDMA_RLC4_RB_RPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC4_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC4_RB_RPTR_HI
+#define SDMA_RLC4_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC4_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC4_RB_WPTR
+#define SDMA_RLC4_RB_WPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC4_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC4_RB_WPTR_HI
+#define SDMA_RLC4_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC4_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC4_RB_WPTR_POLL_CNTL
+#define SDMA_RLC4_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
+#define SDMA_RLC4_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
+#define SDMA_RLC4_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
+#define SDMA_RLC4_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
+#define SDMA_RLC4_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
+#define SDMA_RLC4_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
+#define SDMA_RLC4_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
+#define SDMA_RLC4_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
+#define SDMA_RLC4_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
+#define SDMA_RLC4_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
+//SDMA_RLC4_RB_RPTR_ADDR_HI
+#define SDMA_RLC4_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
+#define SDMA_RLC4_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC4_RB_RPTR_ADDR_LO
+#define SDMA_RLC4_RB_RPTR_ADDR_LO__RPTR_WB_IDLE__SHIFT                                                        0x0
+#define SDMA_RLC4_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
+#define SDMA_RLC4_RB_RPTR_ADDR_LO__RPTR_WB_IDLE_MASK                                                          0x00000001L
+#define SDMA_RLC4_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
+//SDMA_RLC4_IB_CNTL
+#define SDMA_RLC4_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC4_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
+#define SDMA_RLC4_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
+#define SDMA_RLC4_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
+#define SDMA_RLC4_IB_CNTL__IB_PRIV__SHIFT                                                                     0x1f
+#define SDMA_RLC4_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC4_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
+#define SDMA_RLC4_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
+#define SDMA_RLC4_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
+#define SDMA_RLC4_IB_CNTL__IB_PRIV_MASK                                                                       0x80000000L
+//SDMA_RLC4_IB_RPTR
+#define SDMA_RLC4_IB_RPTR__OFFSET__SHIFT                                                                      0x2
+#define SDMA_RLC4_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
+//SDMA_RLC4_IB_OFFSET
+#define SDMA_RLC4_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
+#define SDMA_RLC4_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
+//SDMA_RLC4_IB_BASE_LO
+#define SDMA_RLC4_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
+#define SDMA_RLC4_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
+//SDMA_RLC4_IB_BASE_HI
+#define SDMA_RLC4_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC4_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC4_IB_SIZE
+#define SDMA_RLC4_IB_SIZE__SIZE__SHIFT                                                                        0x0
+#define SDMA_RLC4_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
+//SDMA_RLC4_SKIP_CNTL
+#define SDMA_RLC4_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
+#define SDMA_RLC4_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x000FFFFFL
+//SDMA_RLC4_CONTEXT_STATUS
+#define SDMA_RLC4_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
+#define SDMA_RLC4_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
+#define SDMA_RLC4_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
+#define SDMA_RLC4_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
+#define SDMA_RLC4_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
+#define SDMA_RLC4_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
+#define SDMA_RLC4_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
+#define SDMA_RLC4_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
+#define SDMA_RLC4_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
+#define SDMA_RLC4_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
+#define SDMA_RLC4_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
+#define SDMA_RLC4_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
+#define SDMA_RLC4_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
+#define SDMA_RLC4_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
+#define SDMA_RLC4_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
+#define SDMA_RLC4_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
+//SDMA_RLC4_DOORBELL
+#define SDMA_RLC4_DOORBELL__ENABLE__SHIFT                                                                     0x1c
+#define SDMA_RLC4_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
+#define SDMA_RLC4_DOORBELL__ENABLE_MASK                                                                       0x10000000L
+#define SDMA_RLC4_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
+//SDMA_RLC4_STATUS
+#define SDMA_RLC4_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
+#define SDMA_RLC4_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
+#define SDMA_RLC4_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
+#define SDMA_RLC4_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
+//SDMA_RLC4_DOORBELL_LOG
+#define SDMA_RLC4_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
+#define SDMA_RLC4_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
+#define SDMA_RLC4_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
+#define SDMA_RLC4_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
+//SDMA_RLC4_WATERMARK
+#define SDMA_RLC4_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
+#define SDMA_RLC4_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
+#define SDMA_RLC4_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
+#define SDMA_RLC4_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
+//SDMA_RLC4_DOORBELL_OFFSET
+#define SDMA_RLC4_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
+#define SDMA_RLC4_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
+//SDMA_RLC4_CSA_ADDR_LO
+#define SDMA_RLC4_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
+#define SDMA_RLC4_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
+//SDMA_RLC4_CSA_ADDR_HI
+#define SDMA_RLC4_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
+#define SDMA_RLC4_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
+//SDMA_RLC4_IB_SUB_REMAIN
+#define SDMA_RLC4_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
+#define SDMA_RLC4_IB_SUB_REMAIN__SIZE_MASK                                                                    0x000FFFFFL
+//SDMA_RLC4_PREEMPT
+#define SDMA_RLC4_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
+#define SDMA_RLC4_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
+//SDMA_RLC4_DUMMY_REG
+#define SDMA_RLC4_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
+#define SDMA_RLC4_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC4_RB_WPTR_POLL_ADDR_HI
+#define SDMA_RLC4_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define SDMA_RLC4_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
+//SDMA_RLC4_RB_WPTR_POLL_ADDR_LO
+#define SDMA_RLC4_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define SDMA_RLC4_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+//SDMA_RLC4_RB_AQL_CNTL
+#define SDMA_RLC4_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
+#define SDMA_RLC4_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
+#define SDMA_RLC4_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
+#define SDMA_RLC4_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
+#define SDMA_RLC4_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
+#define SDMA_RLC4_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
+//SDMA_RLC4_MINOR_PTR_UPDATE
+#define SDMA_RLC4_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
+#define SDMA_RLC4_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
+//SDMA_RLC4_MIDCMD_DATA0
+#define SDMA_RLC4_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
+#define SDMA_RLC4_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_DATA1
+#define SDMA_RLC4_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
+#define SDMA_RLC4_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_DATA2
+#define SDMA_RLC4_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
+#define SDMA_RLC4_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_DATA3
+#define SDMA_RLC4_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
+#define SDMA_RLC4_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_DATA4
+#define SDMA_RLC4_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
+#define SDMA_RLC4_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_DATA5
+#define SDMA_RLC4_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
+#define SDMA_RLC4_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_DATA6
+#define SDMA_RLC4_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
+#define SDMA_RLC4_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_DATA7
+#define SDMA_RLC4_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
+#define SDMA_RLC4_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_DATA8
+#define SDMA_RLC4_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
+#define SDMA_RLC4_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_DATA9
+#define SDMA_RLC4_MIDCMD_DATA9__DATA9__SHIFT                                                                  0x0
+#define SDMA_RLC4_MIDCMD_DATA9__DATA9_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_DATA10
+#define SDMA_RLC4_MIDCMD_DATA10__DATA10__SHIFT                                                                0x0
+#define SDMA_RLC4_MIDCMD_DATA10__DATA10_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC4_MIDCMD_CNTL
+#define SDMA_RLC4_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
+#define SDMA_RLC4_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
+#define SDMA_RLC4_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
+#define SDMA_RLC4_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
+#define SDMA_RLC4_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
+#define SDMA_RLC4_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
+#define SDMA_RLC4_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
+#define SDMA_RLC4_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
+//SDMA_RLC5_RB_CNTL
+#define SDMA_RLC5_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC5_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
+#define SDMA_RLC5_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
+#define SDMA_RLC5_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
+#define SDMA_RLC5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
+#define SDMA_RLC5_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
+#define SDMA_RLC5_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
+#define SDMA_RLC5_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
+#define SDMA_RLC5_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC5_RB_CNTL__RB_SIZE_MASK                                                                       0x0000003EL
+#define SDMA_RLC5_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
+#define SDMA_RLC5_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
+#define SDMA_RLC5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
+#define SDMA_RLC5_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
+#define SDMA_RLC5_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
+#define SDMA_RLC5_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
+//SDMA_RLC5_RB_BASE
+#define SDMA_RLC5_RB_BASE__ADDR__SHIFT                                                                        0x0
+#define SDMA_RLC5_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
+//SDMA_RLC5_RB_BASE_HI
+#define SDMA_RLC5_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC5_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
+//SDMA_RLC5_RB_RPTR
+#define SDMA_RLC5_RB_RPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC5_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC5_RB_RPTR_HI
+#define SDMA_RLC5_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC5_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC5_RB_WPTR
+#define SDMA_RLC5_RB_WPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC5_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC5_RB_WPTR_HI
+#define SDMA_RLC5_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC5_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC5_RB_WPTR_POLL_CNTL
+#define SDMA_RLC5_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
+#define SDMA_RLC5_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
+#define SDMA_RLC5_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
+#define SDMA_RLC5_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
+#define SDMA_RLC5_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
+#define SDMA_RLC5_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
+#define SDMA_RLC5_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
+#define SDMA_RLC5_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
+#define SDMA_RLC5_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
+#define SDMA_RLC5_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
+//SDMA_RLC5_RB_RPTR_ADDR_HI
+#define SDMA_RLC5_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
+#define SDMA_RLC5_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC5_RB_RPTR_ADDR_LO
+#define SDMA_RLC5_RB_RPTR_ADDR_LO__RPTR_WB_IDLE__SHIFT                                                        0x0
+#define SDMA_RLC5_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
+#define SDMA_RLC5_RB_RPTR_ADDR_LO__RPTR_WB_IDLE_MASK                                                          0x00000001L
+#define SDMA_RLC5_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
+//SDMA_RLC5_IB_CNTL
+#define SDMA_RLC5_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC5_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
+#define SDMA_RLC5_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
+#define SDMA_RLC5_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
+#define SDMA_RLC5_IB_CNTL__IB_PRIV__SHIFT                                                                     0x1f
+#define SDMA_RLC5_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC5_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
+#define SDMA_RLC5_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
+#define SDMA_RLC5_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
+#define SDMA_RLC5_IB_CNTL__IB_PRIV_MASK                                                                       0x80000000L
+//SDMA_RLC5_IB_RPTR
+#define SDMA_RLC5_IB_RPTR__OFFSET__SHIFT                                                                      0x2
+#define SDMA_RLC5_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
+//SDMA_RLC5_IB_OFFSET
+#define SDMA_RLC5_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
+#define SDMA_RLC5_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
+//SDMA_RLC5_IB_BASE_LO
+#define SDMA_RLC5_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
+#define SDMA_RLC5_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
+//SDMA_RLC5_IB_BASE_HI
+#define SDMA_RLC5_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC5_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC5_IB_SIZE
+#define SDMA_RLC5_IB_SIZE__SIZE__SHIFT                                                                        0x0
+#define SDMA_RLC5_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
+//SDMA_RLC5_SKIP_CNTL
+#define SDMA_RLC5_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
+#define SDMA_RLC5_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x000FFFFFL
+//SDMA_RLC5_CONTEXT_STATUS
+#define SDMA_RLC5_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
+#define SDMA_RLC5_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
+#define SDMA_RLC5_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
+#define SDMA_RLC5_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
+#define SDMA_RLC5_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
+#define SDMA_RLC5_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
+#define SDMA_RLC5_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
+#define SDMA_RLC5_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
+#define SDMA_RLC5_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
+#define SDMA_RLC5_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
+#define SDMA_RLC5_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
+#define SDMA_RLC5_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
+#define SDMA_RLC5_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
+#define SDMA_RLC5_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
+#define SDMA_RLC5_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
+#define SDMA_RLC5_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
+//SDMA_RLC5_DOORBELL
+#define SDMA_RLC5_DOORBELL__ENABLE__SHIFT                                                                     0x1c
+#define SDMA_RLC5_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
+#define SDMA_RLC5_DOORBELL__ENABLE_MASK                                                                       0x10000000L
+#define SDMA_RLC5_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
+//SDMA_RLC5_STATUS
+#define SDMA_RLC5_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
+#define SDMA_RLC5_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
+#define SDMA_RLC5_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
+#define SDMA_RLC5_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
+//SDMA_RLC5_DOORBELL_LOG
+#define SDMA_RLC5_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
+#define SDMA_RLC5_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
+#define SDMA_RLC5_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
+#define SDMA_RLC5_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
+//SDMA_RLC5_WATERMARK
+#define SDMA_RLC5_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
+#define SDMA_RLC5_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
+#define SDMA_RLC5_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
+#define SDMA_RLC5_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
+//SDMA_RLC5_DOORBELL_OFFSET
+#define SDMA_RLC5_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
+#define SDMA_RLC5_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
+//SDMA_RLC5_CSA_ADDR_LO
+#define SDMA_RLC5_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
+#define SDMA_RLC5_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
+//SDMA_RLC5_CSA_ADDR_HI
+#define SDMA_RLC5_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
+#define SDMA_RLC5_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
+//SDMA_RLC5_IB_SUB_REMAIN
+#define SDMA_RLC5_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
+#define SDMA_RLC5_IB_SUB_REMAIN__SIZE_MASK                                                                    0x000FFFFFL
+//SDMA_RLC5_PREEMPT
+#define SDMA_RLC5_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
+#define SDMA_RLC5_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
+//SDMA_RLC5_DUMMY_REG
+#define SDMA_RLC5_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
+#define SDMA_RLC5_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC5_RB_WPTR_POLL_ADDR_HI
+#define SDMA_RLC5_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define SDMA_RLC5_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
+//SDMA_RLC5_RB_WPTR_POLL_ADDR_LO
+#define SDMA_RLC5_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define SDMA_RLC5_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+//SDMA_RLC5_RB_AQL_CNTL
+#define SDMA_RLC5_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
+#define SDMA_RLC5_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
+#define SDMA_RLC5_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
+#define SDMA_RLC5_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
+#define SDMA_RLC5_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
+#define SDMA_RLC5_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
+//SDMA_RLC5_MINOR_PTR_UPDATE
+#define SDMA_RLC5_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
+#define SDMA_RLC5_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
+//SDMA_RLC5_MIDCMD_DATA0
+#define SDMA_RLC5_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
+#define SDMA_RLC5_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_DATA1
+#define SDMA_RLC5_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
+#define SDMA_RLC5_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_DATA2
+#define SDMA_RLC5_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
+#define SDMA_RLC5_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_DATA3
+#define SDMA_RLC5_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
+#define SDMA_RLC5_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_DATA4
+#define SDMA_RLC5_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
+#define SDMA_RLC5_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_DATA5
+#define SDMA_RLC5_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
+#define SDMA_RLC5_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_DATA6
+#define SDMA_RLC5_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
+#define SDMA_RLC5_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_DATA7
+#define SDMA_RLC5_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
+#define SDMA_RLC5_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_DATA8
+#define SDMA_RLC5_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
+#define SDMA_RLC5_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_DATA9
+#define SDMA_RLC5_MIDCMD_DATA9__DATA9__SHIFT                                                                  0x0
+#define SDMA_RLC5_MIDCMD_DATA9__DATA9_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_DATA10
+#define SDMA_RLC5_MIDCMD_DATA10__DATA10__SHIFT                                                                0x0
+#define SDMA_RLC5_MIDCMD_DATA10__DATA10_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC5_MIDCMD_CNTL
+#define SDMA_RLC5_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
+#define SDMA_RLC5_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
+#define SDMA_RLC5_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
+#define SDMA_RLC5_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
+#define SDMA_RLC5_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
+#define SDMA_RLC5_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
+#define SDMA_RLC5_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
+#define SDMA_RLC5_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
+//SDMA_RLC6_RB_CNTL
+#define SDMA_RLC6_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC6_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
+#define SDMA_RLC6_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
+#define SDMA_RLC6_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
+#define SDMA_RLC6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
+#define SDMA_RLC6_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
+#define SDMA_RLC6_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
+#define SDMA_RLC6_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
+#define SDMA_RLC6_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC6_RB_CNTL__RB_SIZE_MASK                                                                       0x0000003EL
+#define SDMA_RLC6_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
+#define SDMA_RLC6_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
+#define SDMA_RLC6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
+#define SDMA_RLC6_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
+#define SDMA_RLC6_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
+#define SDMA_RLC6_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
+//SDMA_RLC6_RB_BASE
+#define SDMA_RLC6_RB_BASE__ADDR__SHIFT                                                                        0x0
+#define SDMA_RLC6_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
+//SDMA_RLC6_RB_BASE_HI
+#define SDMA_RLC6_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC6_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
+//SDMA_RLC6_RB_RPTR
+#define SDMA_RLC6_RB_RPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC6_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC6_RB_RPTR_HI
+#define SDMA_RLC6_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC6_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC6_RB_WPTR
+#define SDMA_RLC6_RB_WPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC6_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC6_RB_WPTR_HI
+#define SDMA_RLC6_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC6_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC6_RB_WPTR_POLL_CNTL
+#define SDMA_RLC6_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
+#define SDMA_RLC6_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
+#define SDMA_RLC6_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
+#define SDMA_RLC6_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
+#define SDMA_RLC6_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
+#define SDMA_RLC6_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
+#define SDMA_RLC6_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
+#define SDMA_RLC6_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
+#define SDMA_RLC6_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
+#define SDMA_RLC6_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
+//SDMA_RLC6_RB_RPTR_ADDR_HI
+#define SDMA_RLC6_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
+#define SDMA_RLC6_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC6_RB_RPTR_ADDR_LO
+#define SDMA_RLC6_RB_RPTR_ADDR_LO__RPTR_WB_IDLE__SHIFT                                                        0x0
+#define SDMA_RLC6_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
+#define SDMA_RLC6_RB_RPTR_ADDR_LO__RPTR_WB_IDLE_MASK                                                          0x00000001L
+#define SDMA_RLC6_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
+//SDMA_RLC6_IB_CNTL
+#define SDMA_RLC6_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC6_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
+#define SDMA_RLC6_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
+#define SDMA_RLC6_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
+#define SDMA_RLC6_IB_CNTL__IB_PRIV__SHIFT                                                                     0x1f
+#define SDMA_RLC6_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC6_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
+#define SDMA_RLC6_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
+#define SDMA_RLC6_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
+#define SDMA_RLC6_IB_CNTL__IB_PRIV_MASK                                                                       0x80000000L
+//SDMA_RLC6_IB_RPTR
+#define SDMA_RLC6_IB_RPTR__OFFSET__SHIFT                                                                      0x2
+#define SDMA_RLC6_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
+//SDMA_RLC6_IB_OFFSET
+#define SDMA_RLC6_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
+#define SDMA_RLC6_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
+//SDMA_RLC6_IB_BASE_LO
+#define SDMA_RLC6_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
+#define SDMA_RLC6_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
+//SDMA_RLC6_IB_BASE_HI
+#define SDMA_RLC6_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC6_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC6_IB_SIZE
+#define SDMA_RLC6_IB_SIZE__SIZE__SHIFT                                                                        0x0
+#define SDMA_RLC6_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
+//SDMA_RLC6_SKIP_CNTL
+#define SDMA_RLC6_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
+#define SDMA_RLC6_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x000FFFFFL
+//SDMA_RLC6_CONTEXT_STATUS
+#define SDMA_RLC6_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
+#define SDMA_RLC6_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
+#define SDMA_RLC6_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
+#define SDMA_RLC6_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
+#define SDMA_RLC6_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
+#define SDMA_RLC6_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
+#define SDMA_RLC6_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
+#define SDMA_RLC6_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
+#define SDMA_RLC6_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
+#define SDMA_RLC6_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
+#define SDMA_RLC6_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
+#define SDMA_RLC6_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
+#define SDMA_RLC6_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
+#define SDMA_RLC6_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
+#define SDMA_RLC6_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
+#define SDMA_RLC6_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
+//SDMA_RLC6_DOORBELL
+#define SDMA_RLC6_DOORBELL__ENABLE__SHIFT                                                                     0x1c
+#define SDMA_RLC6_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
+#define SDMA_RLC6_DOORBELL__ENABLE_MASK                                                                       0x10000000L
+#define SDMA_RLC6_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
+//SDMA_RLC6_STATUS
+#define SDMA_RLC6_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
+#define SDMA_RLC6_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
+#define SDMA_RLC6_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
+#define SDMA_RLC6_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
+//SDMA_RLC6_DOORBELL_LOG
+#define SDMA_RLC6_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
+#define SDMA_RLC6_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
+#define SDMA_RLC6_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
+#define SDMA_RLC6_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
+//SDMA_RLC6_WATERMARK
+#define SDMA_RLC6_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
+#define SDMA_RLC6_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
+#define SDMA_RLC6_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
+#define SDMA_RLC6_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
+//SDMA_RLC6_DOORBELL_OFFSET
+#define SDMA_RLC6_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
+#define SDMA_RLC6_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
+//SDMA_RLC6_CSA_ADDR_LO
+#define SDMA_RLC6_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
+#define SDMA_RLC6_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
+//SDMA_RLC6_CSA_ADDR_HI
+#define SDMA_RLC6_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
+#define SDMA_RLC6_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
+//SDMA_RLC6_IB_SUB_REMAIN
+#define SDMA_RLC6_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
+#define SDMA_RLC6_IB_SUB_REMAIN__SIZE_MASK                                                                    0x000FFFFFL
+//SDMA_RLC6_PREEMPT
+#define SDMA_RLC6_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
+#define SDMA_RLC6_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
+//SDMA_RLC6_DUMMY_REG
+#define SDMA_RLC6_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
+#define SDMA_RLC6_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC6_RB_WPTR_POLL_ADDR_HI
+#define SDMA_RLC6_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define SDMA_RLC6_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
+//SDMA_RLC6_RB_WPTR_POLL_ADDR_LO
+#define SDMA_RLC6_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define SDMA_RLC6_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+//SDMA_RLC6_RB_AQL_CNTL
+#define SDMA_RLC6_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
+#define SDMA_RLC6_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
+#define SDMA_RLC6_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
+#define SDMA_RLC6_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
+#define SDMA_RLC6_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
+#define SDMA_RLC6_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
+//SDMA_RLC6_MINOR_PTR_UPDATE
+#define SDMA_RLC6_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
+#define SDMA_RLC6_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
+//SDMA_RLC6_MIDCMD_DATA0
+#define SDMA_RLC6_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
+#define SDMA_RLC6_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_DATA1
+#define SDMA_RLC6_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
+#define SDMA_RLC6_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_DATA2
+#define SDMA_RLC6_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
+#define SDMA_RLC6_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_DATA3
+#define SDMA_RLC6_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
+#define SDMA_RLC6_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_DATA4
+#define SDMA_RLC6_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
+#define SDMA_RLC6_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_DATA5
+#define SDMA_RLC6_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
+#define SDMA_RLC6_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_DATA6
+#define SDMA_RLC6_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
+#define SDMA_RLC6_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_DATA7
+#define SDMA_RLC6_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
+#define SDMA_RLC6_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_DATA8
+#define SDMA_RLC6_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
+#define SDMA_RLC6_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_DATA9
+#define SDMA_RLC6_MIDCMD_DATA9__DATA9__SHIFT                                                                  0x0
+#define SDMA_RLC6_MIDCMD_DATA9__DATA9_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_DATA10
+#define SDMA_RLC6_MIDCMD_DATA10__DATA10__SHIFT                                                                0x0
+#define SDMA_RLC6_MIDCMD_DATA10__DATA10_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC6_MIDCMD_CNTL
+#define SDMA_RLC6_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
+#define SDMA_RLC6_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
+#define SDMA_RLC6_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
+#define SDMA_RLC6_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
+#define SDMA_RLC6_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
+#define SDMA_RLC6_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
+#define SDMA_RLC6_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
+#define SDMA_RLC6_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
+//SDMA_RLC7_RB_CNTL
+#define SDMA_RLC7_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC7_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
+#define SDMA_RLC7_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
+#define SDMA_RLC7_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
+#define SDMA_RLC7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
+#define SDMA_RLC7_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
+#define SDMA_RLC7_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
+#define SDMA_RLC7_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
+#define SDMA_RLC7_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC7_RB_CNTL__RB_SIZE_MASK                                                                       0x0000003EL
+#define SDMA_RLC7_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
+#define SDMA_RLC7_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
+#define SDMA_RLC7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
+#define SDMA_RLC7_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
+#define SDMA_RLC7_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
+#define SDMA_RLC7_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
+//SDMA_RLC7_RB_BASE
+#define SDMA_RLC7_RB_BASE__ADDR__SHIFT                                                                        0x0
+#define SDMA_RLC7_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
+//SDMA_RLC7_RB_BASE_HI
+#define SDMA_RLC7_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC7_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
+//SDMA_RLC7_RB_RPTR
+#define SDMA_RLC7_RB_RPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC7_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC7_RB_RPTR_HI
+#define SDMA_RLC7_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC7_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC7_RB_WPTR
+#define SDMA_RLC7_RB_WPTR__OFFSET__SHIFT                                                                      0x0
+#define SDMA_RLC7_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
+//SDMA_RLC7_RB_WPTR_HI
+#define SDMA_RLC7_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
+#define SDMA_RLC7_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA_RLC7_RB_WPTR_POLL_CNTL
+#define SDMA_RLC7_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
+#define SDMA_RLC7_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
+#define SDMA_RLC7_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
+#define SDMA_RLC7_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
+#define SDMA_RLC7_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
+#define SDMA_RLC7_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
+#define SDMA_RLC7_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
+#define SDMA_RLC7_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
+#define SDMA_RLC7_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
+#define SDMA_RLC7_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
+//SDMA_RLC7_RB_RPTR_ADDR_HI
+#define SDMA_RLC7_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
+#define SDMA_RLC7_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC7_RB_RPTR_ADDR_LO
+#define SDMA_RLC7_RB_RPTR_ADDR_LO__RPTR_WB_IDLE__SHIFT                                                        0x0
+#define SDMA_RLC7_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
+#define SDMA_RLC7_RB_RPTR_ADDR_LO__RPTR_WB_IDLE_MASK                                                          0x00000001L
+#define SDMA_RLC7_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
+//SDMA_RLC7_IB_CNTL
+#define SDMA_RLC7_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
+#define SDMA_RLC7_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
+#define SDMA_RLC7_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
+#define SDMA_RLC7_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
+#define SDMA_RLC7_IB_CNTL__IB_PRIV__SHIFT                                                                     0x1f
+#define SDMA_RLC7_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
+#define SDMA_RLC7_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
+#define SDMA_RLC7_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
+#define SDMA_RLC7_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
+#define SDMA_RLC7_IB_CNTL__IB_PRIV_MASK                                                                       0x80000000L
+//SDMA_RLC7_IB_RPTR
+#define SDMA_RLC7_IB_RPTR__OFFSET__SHIFT                                                                      0x2
+#define SDMA_RLC7_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
+//SDMA_RLC7_IB_OFFSET
+#define SDMA_RLC7_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
+#define SDMA_RLC7_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
+//SDMA_RLC7_IB_BASE_LO
+#define SDMA_RLC7_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
+#define SDMA_RLC7_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
+//SDMA_RLC7_IB_BASE_HI
+#define SDMA_RLC7_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA_RLC7_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC7_IB_SIZE
+#define SDMA_RLC7_IB_SIZE__SIZE__SHIFT                                                                        0x0
+#define SDMA_RLC7_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
+//SDMA_RLC7_SKIP_CNTL
+#define SDMA_RLC7_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
+#define SDMA_RLC7_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x000FFFFFL
+//SDMA_RLC7_CONTEXT_STATUS
+#define SDMA_RLC7_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
+#define SDMA_RLC7_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
+#define SDMA_RLC7_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
+#define SDMA_RLC7_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
+#define SDMA_RLC7_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
+#define SDMA_RLC7_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
+#define SDMA_RLC7_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
+#define SDMA_RLC7_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
+#define SDMA_RLC7_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
+#define SDMA_RLC7_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
+#define SDMA_RLC7_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
+#define SDMA_RLC7_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
+#define SDMA_RLC7_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
+#define SDMA_RLC7_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
+#define SDMA_RLC7_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
+#define SDMA_RLC7_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
+//SDMA_RLC7_DOORBELL
+#define SDMA_RLC7_DOORBELL__ENABLE__SHIFT                                                                     0x1c
+#define SDMA_RLC7_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
+#define SDMA_RLC7_DOORBELL__ENABLE_MASK                                                                       0x10000000L
+#define SDMA_RLC7_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
+//SDMA_RLC7_STATUS
+#define SDMA_RLC7_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
+#define SDMA_RLC7_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
+#define SDMA_RLC7_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
+#define SDMA_RLC7_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
+//SDMA_RLC7_DOORBELL_LOG
+#define SDMA_RLC7_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
+#define SDMA_RLC7_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
+#define SDMA_RLC7_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
+#define SDMA_RLC7_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
+//SDMA_RLC7_WATERMARK
+#define SDMA_RLC7_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
+#define SDMA_RLC7_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
+#define SDMA_RLC7_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
+#define SDMA_RLC7_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
+//SDMA_RLC7_DOORBELL_OFFSET
+#define SDMA_RLC7_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
+#define SDMA_RLC7_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
+//SDMA_RLC7_CSA_ADDR_LO
+#define SDMA_RLC7_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
+#define SDMA_RLC7_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
+//SDMA_RLC7_CSA_ADDR_HI
+#define SDMA_RLC7_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
+#define SDMA_RLC7_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
+//SDMA_RLC7_IB_SUB_REMAIN
+#define SDMA_RLC7_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
+#define SDMA_RLC7_IB_SUB_REMAIN__SIZE_MASK                                                                    0x000FFFFFL
+//SDMA_RLC7_PREEMPT
+#define SDMA_RLC7_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
+#define SDMA_RLC7_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
+//SDMA_RLC7_DUMMY_REG
+#define SDMA_RLC7_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
+#define SDMA_RLC7_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
+//SDMA_RLC7_RB_WPTR_POLL_ADDR_HI
+#define SDMA_RLC7_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define SDMA_RLC7_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
+//SDMA_RLC7_RB_WPTR_POLL_ADDR_LO
+#define SDMA_RLC7_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define SDMA_RLC7_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+//SDMA_RLC7_RB_AQL_CNTL
+#define SDMA_RLC7_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
+#define SDMA_RLC7_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
+#define SDMA_RLC7_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
+#define SDMA_RLC7_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
+#define SDMA_RLC7_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
+#define SDMA_RLC7_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
+//SDMA_RLC7_MINOR_PTR_UPDATE
+#define SDMA_RLC7_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
+#define SDMA_RLC7_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
+//SDMA_RLC7_MIDCMD_DATA0
+#define SDMA_RLC7_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
+#define SDMA_RLC7_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_DATA1
+#define SDMA_RLC7_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
+#define SDMA_RLC7_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_DATA2
+#define SDMA_RLC7_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
+#define SDMA_RLC7_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_DATA3
+#define SDMA_RLC7_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
+#define SDMA_RLC7_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_DATA4
+#define SDMA_RLC7_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
+#define SDMA_RLC7_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_DATA5
+#define SDMA_RLC7_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
+#define SDMA_RLC7_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_DATA6
+#define SDMA_RLC7_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
+#define SDMA_RLC7_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_DATA7
+#define SDMA_RLC7_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
+#define SDMA_RLC7_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_DATA8
+#define SDMA_RLC7_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
+#define SDMA_RLC7_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_DATA9
+#define SDMA_RLC7_MIDCMD_DATA9__DATA9__SHIFT                                                                  0x0
+#define SDMA_RLC7_MIDCMD_DATA9__DATA9_MASK                                                                    0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_DATA10
+#define SDMA_RLC7_MIDCMD_DATA10__DATA10__SHIFT                                                                0x0
+#define SDMA_RLC7_MIDCMD_DATA10__DATA10_MASK                                                                  0xFFFFFFFFL
+//SDMA_RLC7_MIDCMD_CNTL
+#define SDMA_RLC7_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
+#define SDMA_RLC7_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
+#define SDMA_RLC7_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
+#define SDMA_RLC7_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
+#define SDMA_RLC7_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
+#define SDMA_RLC7_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
+#define SDMA_RLC7_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
+#define SDMA_RLC7_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
index 75f18791cdb9..9f542f6e19ed 100644
--- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -104,7 +104,9 @@ enum pp_clock_type {
 	PP_FCLK,
 	PP_DCEFCLK,
 	PP_VCLK,
+	PP_VCLK1,
 	PP_DCLK,
+	PP_DCLK1,
 	OD_SCLK,
 	OD_MCLK,
 	OD_VDDC_CURVE,
@@ -160,6 +162,8 @@ enum PP_SMC_POWER_PROFILE {
 	PP_SMC_POWER_PROFILE_COMPUTE      = 0x5,
 	PP_SMC_POWER_PROFILE_CUSTOM       = 0x6,
 	PP_SMC_POWER_PROFILE_WINDOW3D     = 0x7,
+	PP_SMC_POWER_PROFILE_CAPPED	  = 0x8,
+	PP_SMC_POWER_PROFILE_UNCAPPED	  = 0x9,
 	PP_SMC_POWER_PROFILE_COUNT,
 };
 
@@ -331,6 +335,8 @@ struct amd_pm_funcs {
 	int (*get_mclk_od)(void *handle);
 	int (*set_mclk_od)(void *handle, uint32_t value);
 	int (*read_sensor)(void *handle, int idx, void *value, int *size);
+	int (*get_apu_thermal_limit)(void *handle, uint32_t *limit);
+	int (*set_apu_thermal_limit)(void *handle, uint32_t limit);
 	enum amd_dpm_forced_level (*get_performance_level)(void *handle);
 	enum amd_pm_state_type (*get_current_power_state)(void *handle);
 	int (*get_fan_speed_rpm)(void *handle, uint32_t *rpm);
diff --git a/drivers/gpu/drm/amd/include/v11_structs.h b/drivers/gpu/drm/amd/include/v11_structs.h
index b8ff7456ae0b..f8008270f813 100644
--- a/drivers/gpu/drm/amd/include/v11_structs.h
+++ b/drivers/gpu/drm/amd/include/v11_structs.h
@@ -25,14 +25,14 @@
 #define V11_STRUCTS_H_
 
 struct v11_gfx_mqd {
-	uint32_t reserved_0; // offset: 0  (0x0)
-	uint32_t reserved_1; // offset: 1  (0x1)
-	uint32_t reserved_2; // offset: 2  (0x2)
-	uint32_t reserved_3; // offset: 3  (0x3)
-	uint32_t reserved_4; // offset: 4  (0x4)
-	uint32_t reserved_5; // offset: 5  (0x5)
-	uint32_t reserved_6; // offset: 6  (0x6)
-	uint32_t reserved_7; // offset: 7  (0x7)
+	uint32_t shadow_base_lo; // offset: 0  (0x0)
+	uint32_t shadow_base_hi; // offset: 1  (0x1)
+	uint32_t gds_bkup_base_lo; // offset: 2  (0x2)
+	uint32_t gds_bkup_base_hi; // offset: 3  (0x3)
+	uint32_t fw_work_area_base_lo; // offset: 4  (0x4)
+	uint32_t fw_work_area_base_hi; // offset: 5  (0x5)
+	uint32_t shadow_initialized; // offset: 6  (0x6)
+	uint32_t ib_vmid; // offset: 7  (0x7)
 	uint32_t reserved_8; // offset: 8  (0x8)
 	uint32_t reserved_9; // offset: 9  (0x9)
 	uint32_t reserved_10; // offset: 10  (0xA)
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
index 6e79d3352d0b..300e156b924f 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
@@ -456,6 +456,34 @@ int amdgpu_dpm_read_sensor(struct amdgpu_device *adev, enum amd_pp_sensors senso
 	return ret;
 }
 
+int amdgpu_dpm_get_apu_thermal_limit(struct amdgpu_device *adev, uint32_t *limit)
+{
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+	int ret = -EINVAL;
+
+	if (pp_funcs && pp_funcs->get_apu_thermal_limit) {
+		mutex_lock(&adev->pm.mutex);
+		ret = pp_funcs->get_apu_thermal_limit(adev->powerplay.pp_handle, limit);
+		mutex_unlock(&adev->pm.mutex);
+	}
+
+	return ret;
+}
+
+int amdgpu_dpm_set_apu_thermal_limit(struct amdgpu_device *adev, uint32_t limit)
+{
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+	int ret = -EINVAL;
+
+	if (pp_funcs && pp_funcs->set_apu_thermal_limit) {
+		mutex_lock(&adev->pm.mutex);
+		ret = pp_funcs->set_apu_thermal_limit(adev->powerplay.pp_handle, limit);
+		mutex_unlock(&adev->pm.mutex);
+	}
+
+	return ret;
+}
+
 void amdgpu_dpm_compute_clocks(struct amdgpu_device *adev)
 {
 	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
index bf6d63673b5a..58c2246918fd 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
@@ -91,6 +91,8 @@ const char * const amdgpu_pp_profile_name[] = {
 	"COMPUTE",
 	"CUSTOM",
 	"WINDOW_3D",
+	"CAPPED",
+	"UNCAPPED",
 };
 
 /**
@@ -1178,6 +1180,21 @@ static ssize_t amdgpu_set_pp_dpm_vclk(struct device *dev,
 	return amdgpu_set_pp_dpm_clock(dev, PP_VCLK, buf, count);
 }
 
+static ssize_t amdgpu_get_pp_dpm_vclk1(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	return amdgpu_get_pp_dpm_clock(dev, PP_VCLK1, buf);
+}
+
+static ssize_t amdgpu_set_pp_dpm_vclk1(struct device *dev,
+		struct device_attribute *attr,
+		const char *buf,
+		size_t count)
+{
+	return amdgpu_set_pp_dpm_clock(dev, PP_VCLK1, buf, count);
+}
+
 static ssize_t amdgpu_get_pp_dpm_dclk(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
@@ -1193,6 +1210,21 @@ static ssize_t amdgpu_set_pp_dpm_dclk(struct device *dev,
 	return amdgpu_set_pp_dpm_clock(dev, PP_DCLK, buf, count);
 }
 
+static ssize_t amdgpu_get_pp_dpm_dclk1(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	return amdgpu_get_pp_dpm_clock(dev, PP_DCLK1, buf);
+}
+
+static ssize_t amdgpu_set_pp_dpm_dclk1(struct device *dev,
+		struct device_attribute *attr,
+		const char *buf,
+		size_t count)
+{
+	return amdgpu_set_pp_dpm_clock(dev, PP_DCLK1, buf, count);
+}
+
 static ssize_t amdgpu_get_pp_dpm_dcefclk(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
@@ -1686,6 +1718,82 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,
 }
 
 /**
+ * DOC: apu_thermal_cap
+ *
+ * The amdgpu driver provides a sysfs API for retrieving/updating thermal
+ * limit temperature in millidegrees Celsius
+ *
+ * Reading back the file shows you core limit value
+ *
+ * Writing an integer to the file, sets a new thermal limit. The value
+ * should be between 0 and 100. If the value is less than 0 or greater
+ * than 100, then the write request will be ignored.
+ */
+static ssize_t amdgpu_get_apu_thermal_cap(struct device *dev,
+					 struct device_attribute *attr,
+					 char *buf)
+{
+	int ret, size;
+	u32 limit;
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+
+	ret = pm_runtime_get_sync(ddev->dev);
+	if (ret < 0) {
+		pm_runtime_put_autosuspend(ddev->dev);
+		return ret;
+	}
+
+	ret = amdgpu_dpm_get_apu_thermal_limit(adev, &limit);
+	if (!ret)
+		size = sysfs_emit(buf, "%u\n", limit);
+	else
+		size = sysfs_emit(buf, "failed to get thermal limit\n");
+
+	pm_runtime_mark_last_busy(ddev->dev);
+	pm_runtime_put_autosuspend(ddev->dev);
+
+	return size;
+}
+
+static ssize_t amdgpu_set_apu_thermal_cap(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf,
+					 size_t count)
+{
+	int ret;
+	u32 value;
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+
+	ret = kstrtou32(buf, 10, &value);
+	if (ret)
+		return ret;
+
+	if (value > 100) {
+		dev_err(dev, "Invalid argument !\n");
+		return -EINVAL;
+	}
+
+	ret = pm_runtime_get_sync(ddev->dev);
+	if (ret < 0) {
+		pm_runtime_put_autosuspend(ddev->dev);
+		return ret;
+	}
+
+	ret = amdgpu_dpm_set_apu_thermal_limit(adev, value);
+	if (ret) {
+		dev_err(dev, "failed to update thermal limit\n");
+		return ret;
+	}
+
+	pm_runtime_mark_last_busy(ddev->dev);
+	pm_runtime_put_autosuspend(ddev->dev);
+
+	return count;
+}
+
+/**
  * DOC: gpu_metrics
  *
  * The amdgpu driver provides a sysfs API for retrieving current gpu
@@ -1924,7 +2032,9 @@ static struct amdgpu_device_attr amdgpu_device_attrs[] = {
 	AMDGPU_DEVICE_ATTR_RW(pp_dpm_socclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RW(pp_dpm_fclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RW(pp_dpm_vclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_vclk1,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RW(pp_dpm_dclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_dclk1,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RW(pp_dpm_dcefclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RW(pp_dpm_pcie,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RW(pp_sclk_od,				ATTR_FLAG_BASIC),
@@ -1937,6 +2047,7 @@ static struct amdgpu_device_attr amdgpu_device_attrs[] = {
 	AMDGPU_DEVICE_ATTR_RW(pp_features,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RO(unique_id,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RW(thermal_throttling_logging,		ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
+	AMDGPU_DEVICE_ATTR_RW(apu_thermal_cap,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RO(gpu_metrics,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RO(smartshift_apu_power,			ATTR_FLAG_BASIC,
 			      .attr_update = ss_power_attr_update),
@@ -2012,6 +2123,12 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_
 		      gc_ver == IP_VERSION(11, 0, 2) ||
 		      gc_ver == IP_VERSION(11, 0, 3)))
 			*states = ATTR_STATE_UNSUPPORTED;
+	} else if (DEVICE_ATTR_IS(pp_dpm_vclk1)) {
+		if (!((gc_ver == IP_VERSION(10, 3, 1) ||
+			   gc_ver == IP_VERSION(10, 3, 0) ||
+			   gc_ver == IP_VERSION(11, 0, 2) ||
+			   gc_ver == IP_VERSION(11, 0, 3)) && adev->vcn.num_vcn_inst >= 2))
+			*states = ATTR_STATE_UNSUPPORTED;
 	} else if (DEVICE_ATTR_IS(pp_dpm_dclk)) {
 		if (!(gc_ver == IP_VERSION(10, 3, 1) ||
 		      gc_ver == IP_VERSION(10, 3, 0) ||
@@ -2020,6 +2137,12 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_
 		      gc_ver == IP_VERSION(11, 0, 2) ||
 		      gc_ver == IP_VERSION(11, 0, 3)))
 			*states = ATTR_STATE_UNSUPPORTED;
+	} else if (DEVICE_ATTR_IS(pp_dpm_dclk1)) {
+		if (!((gc_ver == IP_VERSION(10, 3, 1) ||
+			   gc_ver == IP_VERSION(10, 3, 0) ||
+			   gc_ver == IP_VERSION(11, 0, 2) ||
+			   gc_ver == IP_VERSION(11, 0, 3)) && adev->vcn.num_vcn_inst >= 2))
+			*states = ATTR_STATE_UNSUPPORTED;
 	} else if (DEVICE_ATTR_IS(pp_power_profile_mode)) {
 		if (amdgpu_dpm_get_power_profile_mode(adev, NULL) == -EOPNOTSUPP)
 			*states = ATTR_STATE_UNSUPPORTED;
@@ -3272,7 +3395,7 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
 	      attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't manage state */
 		effective_mode &= ~S_IWUSR;
 
-	/* not implemented yet for GC 10.3.1 APUs */
+	/* In the case of APUs, this is only implemented on Vangogh */
 	if (((adev->family == AMDGPU_FAMILY_SI) ||
 	     ((adev->flags & AMD_IS_APU) && (gc_ver != IP_VERSION(10, 3, 1)))) &&
 	    (attr == &sensor_dev_attr_power1_cap_max.dev_attr.attr ||
@@ -3281,7 +3404,7 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
 	     attr == &sensor_dev_attr_power1_cap_default.dev_attr.attr))
 		return 0;
 
-	/* not implemented yet for APUs having <= GC 9.3.0 */
+	/* not implemented yet for APUs having < GC 9.3.0 (Renoir) */
 	if (((adev->family == AMDGPU_FAMILY_SI) ||
 	     ((adev->flags & AMD_IS_APU) && (gc_ver < IP_VERSION(9, 3, 0)))) &&
 	    (attr == &sensor_dev_attr_power1_average.dev_attr.attr))
diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
index 16addceca68f..d178f3f44081 100644
--- a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
@@ -369,6 +369,9 @@ struct amdgpu_pm {
 int amdgpu_dpm_read_sensor(struct amdgpu_device *adev, enum amd_pp_sensors sensor,
 			   void *data, uint32_t *size);
 
+int amdgpu_dpm_get_apu_thermal_limit(struct amdgpu_device *adev, uint32_t *limit);
+int amdgpu_dpm_set_apu_thermal_limit(struct amdgpu_device *adev, uint32_t limit);
+
 int amdgpu_dpm_set_powergating_by_smu(struct amdgpu_device *adev,
 				      uint32_t block_type, bool gate);
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index 0652b001ad54..5633c5797e85 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
@@ -40,6 +40,7 @@
 #include "smu_v13_0_0_ppt.h"
 #include "smu_v13_0_4_ppt.h"
 #include "smu_v13_0_5_ppt.h"
+#include "smu_v13_0_6_ppt.h"
 #include "smu_v13_0_7_ppt.h"
 #include "amd_pcie.h"
 
@@ -161,10 +162,15 @@ int smu_get_dpm_freq_range(struct smu_context *smu,
 
 int smu_set_gfx_power_up_by_imu(struct smu_context *smu)
 {
-	if (!smu->ppt_funcs || !smu->ppt_funcs->set_gfx_power_up_by_imu)
-		return -EOPNOTSUPP;
+	int ret = 0;
+	struct amdgpu_device *adev = smu->adev;
 
-	return smu->ppt_funcs->set_gfx_power_up_by_imu(smu);
+	if (smu->ppt_funcs->set_gfx_power_up_by_imu) {
+		ret = smu->ppt_funcs->set_gfx_power_up_by_imu(smu);
+		if (ret)
+			dev_err(adev->dev, "Failed to enable gfx imu!\n");
+	}
+	return ret;
 }
 
 static u32 smu_get_mclk(void *handle, bool low)
@@ -195,6 +201,19 @@ static u32 smu_get_sclk(void *handle, bool low)
 	return clk_freq * 100;
 }
 
+static int smu_set_gfx_imu_enable(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+
+	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
+		return 0;
+
+	if (amdgpu_in_reset(smu->adev) || adev->in_s0ix)
+		return 0;
+
+	return smu_set_gfx_power_up_by_imu(smu);
+}
+
 static int smu_dpm_set_vcn_enable(struct smu_context *smu,
 				  bool enable)
 {
@@ -609,6 +628,11 @@ static int smu_set_funcs(struct amdgpu_device *adev)
 	case IP_VERSION(13, 0, 10):
 		smu_v13_0_0_set_ppt_funcs(smu);
 		break;
+	case IP_VERSION(13, 0, 6):
+		smu_v13_0_6_set_ppt_funcs(smu);
+		/* Enable pp_od_clk_voltage node */
+		smu->od_enabled = true;
+		break;
 	case IP_VERSION(13, 0, 7):
 		smu_v13_0_7_set_ppt_funcs(smu);
 		break;
@@ -1390,15 +1414,9 @@ static int smu_hw_init(void *handle)
 	}
 
 	if (smu->is_apu) {
-		if ((smu->ppt_funcs->set_gfx_power_up_by_imu) &&
-				likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
-			ret = smu->ppt_funcs->set_gfx_power_up_by_imu(smu);
-			if (ret) {
-				dev_err(adev->dev, "Failed to Enable gfx imu!\n");
-				return ret;
-			}
-		}
-
+		ret = smu_set_gfx_imu_enable(smu);
+		if (ret)
+			return ret;
 		smu_dpm_set_vcn_enable(smu, true);
 		smu_dpm_set_jpeg_enable(smu, true);
 		smu_set_gfx_cgpg(smu, true);
@@ -1675,6 +1693,10 @@ static int smu_resume(void *handle)
 		return ret;
 	}
 
+	ret = smu_set_gfx_imu_enable(smu);
+	if (ret)
+		return ret;
+
 	smu_set_gfx_cgpg(smu, true);
 
 	smu->disable_uclk_switch = 0;
@@ -1690,8 +1712,6 @@ static int smu_display_configuration_change(void *handle,
 					    const struct amd_pp_display_configuration *display_config)
 {
 	struct smu_context *smu = handle;
-	int index = 0;
-	int num_of_active_display = 0;
 
 	if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled)
 		return -EOPNOTSUPP;
@@ -1702,11 +1722,6 @@ static int smu_display_configuration_change(void *handle,
 	smu_set_min_dcef_deep_sleep(smu,
 				    display_config->min_dcef_deep_sleep_set_clk / 100);
 
-	for (index = 0; index < display_config->num_path_including_non_display; index++) {
-		if (display_config->displays[index].controller_id != 0)
-			num_of_active_display++;
-	}
-
 	return 0;
 }
 
@@ -2000,8 +2015,12 @@ static int smu_force_ppclk_levels(void *handle,
 		clk_type = SMU_DCEFCLK; break;
 	case PP_VCLK:
 		clk_type = SMU_VCLK; break;
+	case PP_VCLK1:
+		clk_type = SMU_VCLK1; break;
 	case PP_DCLK:
 		clk_type = SMU_DCLK; break;
+	case PP_DCLK1:
+		clk_type = SMU_DCLK1; break;
 	case OD_SCLK:
 		clk_type = SMU_OD_SCLK; break;
 	case OD_MCLK:
@@ -2387,8 +2406,12 @@ static enum smu_clk_type smu_convert_to_smuclk(enum pp_clock_type type)
 		clk_type = SMU_DCEFCLK; break;
 	case PP_VCLK:
 		clk_type = SMU_VCLK; break;
+	case PP_VCLK1:
+		clk_type = SMU_VCLK1; break;
 	case PP_DCLK:
 		clk_type = SMU_DCLK; break;
+	case PP_DCLK1:
+		clk_type = SMU_DCLK1; break;
 	case OD_SCLK:
 		clk_type = SMU_OD_SCLK; break;
 	case OD_MCLK:
@@ -2532,6 +2555,28 @@ unlock:
 	return ret;
 }
 
+static int smu_get_apu_thermal_limit(void *handle, uint32_t *limit)
+{
+	int ret = -EINVAL;
+	struct smu_context *smu = handle;
+
+	if (smu->ppt_funcs && smu->ppt_funcs->get_apu_thermal_limit)
+		ret = smu->ppt_funcs->get_apu_thermal_limit(smu, limit);
+
+	return ret;
+}
+
+static int smu_set_apu_thermal_limit(void *handle, uint32_t limit)
+{
+	int ret = -EINVAL;
+	struct smu_context *smu = handle;
+
+	if (smu->ppt_funcs && smu->ppt_funcs->set_apu_thermal_limit)
+		ret = smu->ppt_funcs->set_apu_thermal_limit(smu, limit);
+
+	return ret;
+}
+
 static int smu_get_power_profile_mode(void *handle, char *buf)
 {
 	struct smu_context *smu = handle;
@@ -3033,6 +3078,8 @@ static const struct amd_pm_funcs swsmu_pm_funcs = {
 	.emit_clock_levels       = smu_emit_ppclk_levels,
 	.force_performance_level = smu_force_performance_level,
 	.read_sensor             = smu_read_sensor,
+	.get_apu_thermal_limit       = smu_get_apu_thermal_limit,
+	.set_apu_thermal_limit       = smu_set_apu_thermal_limit,
 	.get_performance_level   = smu_get_performance_level,
 	.get_current_power_state = smu_get_current_power_state,
 	.get_fan_speed_rpm       = smu_get_fan_speed_rpm,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
index 2a03d85bf4e2..09469c750a96 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
@@ -722,6 +722,18 @@ struct pptable_funcs {
 			   void *data, uint32_t *size);
 
 	/**
+	 * @get_apu_thermal_limit: get apu core limit from smu
+	 * &limit: current limit temperature in millidegrees Celsius
+	 */
+	int (*get_apu_thermal_limit)(struct smu_context *smu, uint32_t *limit);
+
+	/**
+	 * @set_apu_thermal_limit: update all controllers with new limit
+	 * &limit: limit temperature to be setted, in millidegrees Celsius
+	 */
+	int (*set_apu_thermal_limit)(struct smu_context *smu, uint32_t limit);
+
+	/**
 	 * @pre_display_config_changed: Prepare GPU for a display configuration
 	 *                              change.
 	 *
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu11_driver_if_vangogh.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu11_driver_if_vangogh.h
index 8361ebd8d876..21e6028a49e6 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu11_driver_if_vangogh.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu11_driver_if_vangogh.h
@@ -238,7 +238,9 @@ typedef struct {
 #define WORKLOAD_PPLIB_VR_BIT 3
 #define WORKLOAD_PPLIB_COMPUTE_BIT 4
 #define WORKLOAD_PPLIB_CUSTOM_BIT 5
-#define WORKLOAD_PPLIB_COUNT 6
+#define WORKLOAD_PPLIB_CAPPED_BIT 6
+#define WORKLOAD_PPLIB_UNCAPPED_BIT 7
+#define WORKLOAD_PPLIB_COUNT 8
 
 #define TABLE_BIOS_IF            0 // Called by BIOS
 #define TABLE_WATERMARKS         1 // Called by DAL through VBIOS
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_6.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_6.h
new file mode 100644
index 000000000000..be596777cd2c
--- /dev/null
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_6.h
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef SMU_13_0_6_DRIVER_IF_H
+#define SMU_13_0_6_DRIVER_IF_H
+
+// *** IMPORTANT ***
+// PMFW TEAM: Always increment the interface version if
+// anything is changed in this file
+#define SMU13_0_6_DRIVER_IF_VERSION 0x08042022
+
+//I2C Interface
+#define NUM_I2C_CONTROLLERS                8
+#define I2C_CONTROLLER_ENABLED             1
+#define I2C_CONTROLLER_DISABLED            0
+
+#define MAX_SW_I2C_COMMANDS                24
+
+typedef enum {
+  I2C_CONTROLLER_PORT_0, //CKSVII2C0
+  I2C_CONTROLLER_PORT_1, //CKSVII2C1
+  I2C_CONTROLLER_PORT_COUNT,
+} I2cControllerPort_e;
+
+typedef enum {
+  UNSUPPORTED_1,              //50  Kbits/s not supported anymore!
+  I2C_SPEED_STANDARD_100K,    //100 Kbits/s
+  I2C_SPEED_FAST_400K,        //400 Kbits/s
+  I2C_SPEED_FAST_PLUS_1M,     //1   Mbits/s (in fast mode)
+  UNSUPPORTED_2,              //1   Mbits/s (in high speed mode)  not supported anymore!
+  UNSUPPORTED_3,              //2.3 Mbits/s  not supported anymore!
+  I2C_SPEED_COUNT,
+} I2cSpeed_e;
+
+typedef enum {
+  I2C_CMD_READ,
+  I2C_CMD_WRITE,
+  I2C_CMD_COUNT,
+} I2cCmdType_e;
+
+#define CMDCONFIG_STOP_BIT             0
+#define CMDCONFIG_RESTART_BIT          1
+#define CMDCONFIG_READWRITE_BIT        2 //bit should be 0 for read, 1 for write
+
+#define CMDCONFIG_STOP_MASK           (1 << CMDCONFIG_STOP_BIT)
+#define CMDCONFIG_RESTART_MASK        (1 << CMDCONFIG_RESTART_BIT)
+#define CMDCONFIG_READWRITE_MASK      (1 << CMDCONFIG_READWRITE_BIT)
+
+typedef struct {
+  uint8_t ReadWriteData;  //Return data for read. Data to send for write
+  uint8_t CmdConfig; //Includes whether associated command should have a stop or restart command, and is a read or write
+} SwI2cCmd_t; //SW I2C Command Table
+
+typedef struct {
+  uint8_t    I2CcontrollerPort; //CKSVII2C0(0) or //CKSVII2C1(1)
+  uint8_t    I2CSpeed;          //Use I2cSpeed_e to indicate speed to select
+  uint8_t    SlaveAddress;      //Slave address of device
+  uint8_t    NumCmds;           //Number of commands
+  SwI2cCmd_t SwI2cCmds[MAX_SW_I2C_COMMANDS];
+} SwI2cRequest_t; // SW I2C Request Table
+
+typedef struct {
+  SwI2cRequest_t SwI2cRequest;
+  uint32_t       Spare[8];
+  uint32_t       MmHubPadding[8]; // SMU internal use
+} SwI2cRequestExternal_t;
+
+typedef enum {
+  PPCLK_VCLK,
+  PPCLK_DCLK,
+  PPCLK_SOCCLK,
+  PPCLK_UCLK,
+  PPCLK_FCLK,
+  PPCLK_LCLK,
+  PPCLK_COUNT,
+} PPCLK_e;
+
+typedef enum {
+  GPIO_INT_POLARITY_ACTIVE_LOW,
+  GPIO_INT_POLARITY_ACTIVE_HIGH,
+} GpioIntPolarity_e;
+
+//TODO confirm if this is used in SMU_13_0_6 PPSMC_MSG_SetUclkDpmMode
+typedef enum {
+  UCLK_DPM_MODE_BANDWIDTH,
+  UCLK_DPM_MODE_LATENCY,
+} UCLK_DPM_MODE_e;
+
+typedef struct {
+  //0-26 SOC, 27-29 SOCIO
+  uint16_t avgPsmCount[30];
+  uint16_t minPsmCount[30];
+  float    avgPsmVoltage[30];
+  float    minPsmVoltage[30];
+} AvfsDebugTableAid_t;
+
+typedef struct {
+  //0-27 GFX, 28-29 SOC
+  uint16_t avgPsmCount[30];
+  uint16_t minPsmCount[30];
+  float    avgPsmVoltage[30];
+  float    minPsmVoltage[30];
+} AvfsDebugTableXcd_t;
+
+// These defines are used with the following messages:
+// SMC_MSG_TransferTableDram2Smu
+// SMC_MSG_TransferTableSmu2Dram
+// #define TABLE_PPTABLE                 0
+// #define TABLE_AVFS_PSM_DEBUG          1
+// #define TABLE_AVFS_FUSE_OVERRIDE      2
+// #define TABLE_PMSTATUSLOG             3
+// #define TABLE_SMU_METRICS             4
+// #define TABLE_DRIVER_SMU_CONFIG       5
+// #define TABLE_I2C_COMMANDS            6
+// #define TABLE_COUNT                   7
+
+// // Table transfer status
+// #define TABLE_TRANSFER_OK         0x0
+// #define TABLE_TRANSFER_FAILED     0xFF
+// #define TABLE_TRANSFER_PENDING    0xAB
+
+#endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_6_pmfw.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_6_pmfw.h
new file mode 100644
index 000000000000..bdccbb4a6276
--- /dev/null
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_6_pmfw.h
@@ -0,0 +1,212 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef SMU_13_0_6_PMFW_H
+#define SMU_13_0_6_PMFW_H
+
+#define NUM_VCLK_DPM_LEVELS   4
+#define NUM_DCLK_DPM_LEVELS   4
+#define NUM_SOCCLK_DPM_LEVELS 4
+#define NUM_LCLK_DPM_LEVELS   4
+#define NUM_UCLK_DPM_LEVELS   4
+#define NUM_FCLK_DPM_LEVELS   4
+#define NUM_XGMI_DPM_LEVELS   2
+#define NUM_CXL_BITRATES      4
+#define NUM_PCIE_BITRATES     4
+#define NUM_XGMI_BITRATES     4
+#define NUM_XGMI_WIDTHS       3
+
+typedef enum {
+/*0*/   FEATURE_DATA_CALCULATION            = 0,
+/*1*/   FEATURE_DPM_CCLK                    = 1,
+/*2*/   FEATURE_DPM_FCLK                    = 2,
+/*3*/   FEATURE_DPM_GFXCLK                  = 3,
+/*4*/   FEATURE_DPM_LCLK                    = 4,
+/*5*/   FEATURE_DPM_SOCCLK                  = 5,
+/*6*/   FEATURE_DPM_UCLK                    = 6,
+/*7*/   FEATURE_DPM_VCN                     = 7,
+/*8*/   FEATURE_DPM_XGMI                    = 8,
+/*9*/   FEATURE_DS_FCLK                     = 9,
+/*10*/  FEATURE_DS_GFXCLK                   = 10,
+/*11*/  FEATURE_DS_LCLK                     = 11,
+/*12*/  FEATURE_DS_MP0CLK                   = 12,
+/*13*/  FEATURE_DS_MP1CLK                   = 13,
+/*14*/  FEATURE_DS_MPIOCLK                  = 14,
+/*15*/  FEATURE_DS_SOCCLK                   = 15,
+/*16*/  FEATURE_DS_VCN                      = 16,
+/*17*/  FEATURE_APCC_DFLL                   = 17,
+/*18*/  FEATURE_APCC_PLUS                   = 18,
+/*19*/  FEATURE_DF_CSTATE                   = 19,
+/*20*/  FEATURE_CC6                         = 20,
+/*21*/  FEATURE_PC6                         = 21,
+/*22*/  FEATURE_CPPC                        = 22,
+/*23*/  FEATURE_PPT                         = 23,
+/*24*/  FEATURE_TDC                         = 24,
+/*25*/  FEATURE_THERMAL                     = 25,
+/*26*/  FEATURE_SOC_PCC                     = 26,
+/*27*/  FEATURE_CCD_PCC                     = 27,
+/*28*/  FEATURE_CCD_EDC                     = 28,
+/*29*/  FEATURE_PROCHOT                     = 29,
+/*30*/  FEATURE_DVO_CCLK                    = 30,
+/*31*/  FEATURE_FDD_AID_HBM                 = 31,
+/*32*/  FEATURE_FDD_AID_SOC                 = 32,
+/*33*/  FEATURE_FDD_XCD_EDC                 = 33,
+/*34*/  FEATURE_FDD_XCD_XVMIN               = 34,
+/*35*/  FEATURE_FW_CTF                      = 35,
+/*36*/  FEATURE_GFXOFF                      = 36,
+/*37*/  FEATURE_SMU_CG                      = 37,
+/*38*/  FEATURE_PSI7                        = 38,
+/*39*/  FEATURE_CSTATE_BOOST                = 39,
+/*40*/  FEATURE_XGMI_PER_LINK_PWR_DOWN      = 40,
+/*41*/  FEATURE_CXL_QOS                     = 41,
+/*42*/  FEATURE_SOC_DC_RTC                  = 42,
+/*43*/  FEATURE_GFX_DC_RTC                  = 43,
+
+/*44*/  NUM_FEATURES                        = 44
+} FEATURE_LIST_e;
+
+//enum for MPIO PCIe gen speed msgs
+typedef enum {
+  PCIE_LINK_SPEED_INDEX_TABLE_GEN1,
+  PCIE_LINK_SPEED_INDEX_TABLE_GEN2,
+  PCIE_LINK_SPEED_INDEX_TABLE_GEN3,
+  PCIE_LINK_SPEED_INDEX_TABLE_GEN4,
+  PCIE_LINK_SPEED_INDEX_TABLE_GEN4_ESM,
+  PCIE_LINK_SPEED_INDEX_TABLE_GEN5,
+  PCIE_LINK_SPEED_INDEX_TABLE_COUNT
+} PCIE_LINK_SPEED_INDEX_TABLE_e;
+
+typedef enum {
+  VOLTAGE_COLD_0,
+  VOLTAGE_COLD_1,
+  VOLTAGE_COLD_2,
+  VOLTAGE_COLD_3,
+  VOLTAGE_COLD_4,
+  VOLTAGE_COLD_5,
+  VOLTAGE_COLD_6,
+  VOLTAGE_COLD_7,
+  VOLTAGE_MID_0,
+  VOLTAGE_MID_1,
+  VOLTAGE_MID_2,
+  VOLTAGE_MID_3,
+  VOLTAGE_MID_4,
+  VOLTAGE_MID_5,
+  VOLTAGE_MID_6,
+  VOLTAGE_MID_7,
+  VOLTAGE_HOT_0,
+  VOLTAGE_HOT_1,
+  VOLTAGE_HOT_2,
+  VOLTAGE_HOT_3,
+  VOLTAGE_HOT_4,
+  VOLTAGE_HOT_5,
+  VOLTAGE_HOT_6,
+  VOLTAGE_HOT_7,
+  VOLTAGE_GUARDBAND_COUNT
+} GFX_GUARDBAND_e;
+
+#define SMU_METRICS_TABLE_VERSION 0x1
+
+typedef struct {
+  uint32_t AccumulationCounter;
+
+  //TEMPERATURE
+  uint32_t MaxSocketTemperature;
+  uint32_t MaxVrTemperature;
+  uint32_t MaxHbmTemperature;
+  uint64_t MaxSocketTemperatureAcc;
+  uint64_t MaxVrTemperatureAcc;
+  uint64_t MaxHbmTemperatureAcc;
+
+  //POWER
+  uint32_t SocketPowerLimit;
+  uint32_t MaxSocketPowerLimit;
+  uint32_t SocketPower;
+
+  //ENERGY
+  uint64_t Timestamp;
+  uint64_t SocketEnergyAcc;
+  uint64_t CcdEnergyAcc;
+  uint64_t XcdEnergyAcc;
+  uint64_t AidEnergyAcc;
+  uint64_t HbmEnergyAcc;
+
+  //FREQUENCY
+  uint32_t CclkFrequencyLimit;
+  uint32_t GfxclkFrequencyLimit;
+  uint32_t FclkFrequency;
+  uint32_t UclkFrequency;
+  uint32_t SocclkFrequency[4];
+  uint32_t VclkFrequency[4];
+  uint32_t DclkFrequency[4];
+  uint32_t LclkFrequency[4];
+  uint64_t GfxclkFrequencyAcc[8];
+  uint64_t CclkFrequencyAcc[96];
+
+  //FREQUENCY RANGE
+  uint32_t MaxCclkFrequency;
+  uint32_t MinCclkFrequency;
+  uint32_t MaxGfxclkFrequency;
+  uint32_t MinGfxclkFrequency;
+  uint32_t FclkFrequencyTable[4];
+  uint32_t UclkFrequencyTable[4];
+  uint32_t SocclkFrequencyTable[4];
+  uint32_t VclkFrequencyTable[4];
+  uint32_t DclkFrequencyTable[4];
+  uint32_t LclkFrequencyTable[4];
+  uint32_t MaxLclkDpmRange;
+  uint32_t MinLclkDpmRange;
+
+  //XGMI
+  uint32_t XgmiWidth;
+  uint32_t XgmiBitrate;
+  uint64_t XgmiReadBandwidthAcc[8];
+  uint64_t XgmiWriteBandwidthAcc[8];
+
+  //ACTIVITY
+  uint32_t SocketC0Residency;
+  uint32_t SocketGfxBusy;
+  uint32_t DramBandwidthUtilization;
+  uint64_t SocketC0ResidencyAcc;
+  uint64_t SocketGfxBusyAcc;
+  uint64_t DramBandwidthAcc;
+  uint32_t MaxDramBandwidth;
+  uint64_t DramBandwidthUtilizationAcc;
+  uint64_t PcieBandwidthAcc[4];
+
+  //THROTTLERS
+  uint32_t ProchotResidencyAcc;
+  uint32_t PptResidencyAcc;
+  uint32_t SocketThmResidencyAcc;
+  uint32_t VrThmResidencyAcc;
+  uint32_t HbmThmResidencyAcc;
+} MetricsTable_t;
+
+#define SMU_VF_METRICS_TABLE_VERSION 0x1
+
+typedef struct {
+  uint32_t AccumulationCounter;
+  uint32_t InstGfxclk_TargFreq;
+  uint64_t AccGfxclk_TargFreq;
+  uint64_t AccGfxRsmuDpm_Busy;
+} VfMetricsTable_t;
+
+#endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_6_ppsmc.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_6_ppsmc.h
new file mode 100644
index 000000000000..b838e8db395a
--- /dev/null
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_6_ppsmc.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef SMU_13_0_6_PPSMC_H
+#define SMU_13_0_6_PPSMC_H
+
+// SMU Response Codes:
+#define PPSMC_Result_OK                             0x1
+#define PPSMC_Result_Failed                         0xFF
+#define PPSMC_Result_UnknownCmd                     0xFE
+#define PPSMC_Result_CmdRejectedPrereq              0xFD
+#define PPSMC_Result_CmdRejectedBusy                0xFC
+
+// Message Definitions:
+#define PPSMC_MSG_TestMessage                       0x1
+#define PPSMC_MSG_GetSmuVersion                     0x2
+#define PPSMC_MSG_GfxDriverReset                    0x3
+#define PPSMC_MSG_GetDriverIfVersion                0x4
+#define PPSMC_MSG_EnableAllSmuFeatures              0x5
+#define PPSMC_MSG_DisableAllSmuFeatures             0x6
+#define PPSMC_MSG_RequestI2cTransaction             0x7
+#define PPSMC_MSG_GetMetricsVersion                 0x8
+#define PPSMC_MSG_GetMetricsTable                   0x9
+#define PPSMC_MSG_GetEccInfoTable                   0xA
+#define PPSMC_MSG_GetEnabledSmuFeaturesLow          0xB
+#define PPSMC_MSG_GetEnabledSmuFeaturesHigh         0xC
+#define PPSMC_MSG_SetDriverDramAddrHigh             0xD
+#define PPSMC_MSG_SetDriverDramAddrLow              0xE
+#define PPSMC_MSG_SetToolsDramAddrHigh              0xF
+#define PPSMC_MSG_SetToolsDramAddrLow               0x10
+#define PPSMC_MSG_SetSystemVirtualDramAddrHigh      0x11
+#define PPSMC_MSG_SetSystemVirtualDramAddrLow       0x12
+#define PPSMC_MSG_SetSoftMinByFreq                  0x13
+#define PPSMC_MSG_SetSoftMaxByFreq                  0x14
+#define PPSMC_MSG_GetMinDpmFreq                     0x15
+#define PPSMC_MSG_GetMaxDpmFreq                     0x16
+#define PPSMC_MSG_GetDpmFreqByIndex                 0x17
+#define PPSMC_MSG_SetPptLimit                       0x18
+#define PPSMC_MSG_GetPptLimit                       0x19
+#define PPSMC_MSG_DramLogSetDramAddrHigh            0x1A
+#define PPSMC_MSG_DramLogSetDramAddrLow             0x1B
+#define PPSMC_MSG_DramLogSetDramSize                0x1C
+#define PPSMC_MSG_GetDebugData                      0x1D
+#define PPSMC_MSG_HeavySBR                          0x1E
+#define PPSMC_MSG_SetNumBadHbmPagesRetired          0x1F
+#define PPSMC_MSG_DFCstateControl                   0x20
+#define PPSMC_MSG_GetGmiPwrDnHyst                   0x21
+#define PPSMC_MSG_SetGmiPwrDnHyst                   0x22
+#define PPSMC_MSG_GmiPwrDnControl                   0x23
+#define PPSMC_MSG_EnterGfxoff                       0x24
+#define PPSMC_MSG_ExitGfxoff                        0x25
+#define PPSMC_MSG_EnableDeterminism                 0x26
+#define PPSMC_MSG_DisableDeterminism                0x27
+#define PPSMC_MSG_DumpSTBtoDram                     0x28
+#define PPSMC_MSG_STBtoDramLogSetDramAddrHigh       0x29
+#define PPSMC_MSG_STBtoDramLogSetDramAddrLow        0x2A
+#define PPSMC_MSG_STBtoDramLogSetDramSize           0x2B
+#define PPSMC_MSG_SetSystemVirtualSTBtoDramAddrHigh 0x2C
+#define PPSMC_MSG_SetSystemVirtualSTBtoDramAddrLow  0x2D
+#define PPSMC_MSG_GfxDriverResetRecovery            0x2E
+#define PPSMC_MSG_TriggerVFFLR                      0x2F
+#define PPSMC_MSG_SetSoftMinGfxClk                  0x30
+#define PPSMC_MSG_SetSoftMaxGfxClk                  0x31
+#define PPSMC_MSG_GetMinGfxDpmFreq                  0x32
+#define PPSMC_MSG_GetMaxGfxDpmFreq                  0x33
+#define PPSMC_Message_Count                         0x34
+
+//PPSMC Reset Types for driver msg argument
+#define PPSMC_RESET_TYPE_DRIVER_MODE_1_RESET        0x1
+#define PPSMC_RESET_TYPE_DRIVER_MODE_2_RESET	      0x2
+#define PPSMC_RESET_TYPE_DRIVER_MODE_3_RESET        0x3
+
+typedef uint32_t PPSMC_Result;
+typedef uint32_t PPSMC_MSG;
+
+#endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
index 96f6c2db955b..297b70b9388f 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
@@ -243,7 +243,9 @@
 	__SMU_DUMMY_MAP(SetNumBadMemoryPagesRetired),		\
 	__SMU_DUMMY_MAP(SetBadMemoryPagesRetiredFlagsPerChannel), \
 	__SMU_DUMMY_MAP(AllowGpo),	\
-	__SMU_DUMMY_MAP(Mode2Reset),
+	__SMU_DUMMY_MAP(Mode2Reset),	\
+	__SMU_DUMMY_MAP(RequestI2cTransaction), \
+	__SMU_DUMMY_MAP(GetMetricsTable),
 
 #undef __SMU_DUMMY_MAP
 #define __SMU_DUMMY_MAP(type)	SMU_MSG_##type
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
index 85a090b9e3d9..7944ce80e5c3 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
@@ -34,6 +34,7 @@
 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_0_10 0x32
 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_7 0x37
 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_10 0x1D
+#define SMU13_DRIVER_IF_VERSION_SMU_V13_0_6 0x0
 
 #define SMU13_MODE1_RESET_WAIT_TIME_IN_MS 500  //500ms
 
@@ -250,6 +251,10 @@ int smu_v13_0_set_single_dpm_table(struct smu_context *smu,
 				   enum smu_clk_type clk_type,
 				   struct smu_13_0_dpm_table *single_dpm_table);
 
+int smu_v13_0_get_dpm_freq_by_index(struct smu_context *smu,
+				    enum smu_clk_type clk_type, uint16_t level,
+				    uint32_t *value);
+
 int smu_v13_0_get_current_pcie_link_width_level(struct smu_context *smu);
 
 int smu_v13_0_get_current_pcie_link_width(struct smu_context *smu);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
index 95da6dd1cc65..c4000518dc56 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
@@ -304,7 +304,8 @@ navi10_get_allowed_feature_mask(struct smu_context *smu,
 				| FEATURE_MASK(FEATURE_GFX_SS_BIT)
 				| FEATURE_MASK(FEATURE_APCC_DFLL_BIT)
 				| FEATURE_MASK(FEATURE_FW_CTF_BIT)
-				| FEATURE_MASK(FEATURE_OUT_OF_BAND_MONITOR_BIT);
+				| FEATURE_MASK(FEATURE_OUT_OF_BAND_MONITOR_BIT)
+				| FEATURE_MASK(FEATURE_TEMP_DEPENDENT_VMIN_BIT);
 
 	if (adev->pm.pp_feature & PP_SCLK_DPM_MASK)
 		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
index cb10c7e31264..7433dcaa16e0 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
@@ -203,6 +203,8 @@ static struct cmn2asic_mapping vangogh_workload_map[PP_SMC_POWER_PROFILE_COUNT]
 	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VR,			WORKLOAD_PPLIB_VR_BIT),
 	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_COMPUTE,		WORKLOAD_PPLIB_COMPUTE_BIT),
 	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_CUSTOM,		WORKLOAD_PPLIB_CUSTOM_BIT),
+	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_CAPPED,		WORKLOAD_PPLIB_CAPPED_BIT),
+	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_UNCAPPED,		WORKLOAD_PPLIB_UNCAPPED_BIT),
 };
 
 static const uint8_t vangogh_throttler_map[] = {
@@ -1046,7 +1048,7 @@ static int vangogh_get_power_profile_mode(struct smu_context *smu,
 	if (!buf)
 		return -EINVAL;
 
-	for (i = 0; i <= PP_SMC_POWER_PROFILE_CUSTOM; i++) {
+	for (i = 0; i < PP_SMC_POWER_PROFILE_COUNT; i++) {
 		/*
 		 * Conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT
 		 * Not all profile modes are supported on vangogh.
@@ -1070,7 +1072,7 @@ static int vangogh_set_power_profile_mode(struct smu_context *smu, long *input,
 	int workload_type, ret;
 	uint32_t profile_mode = input[size];
 
-	if (profile_mode > PP_SMC_POWER_PROFILE_CUSTOM) {
+	if (profile_mode >= PP_SMC_POWER_PROFILE_COUNT) {
 		dev_err(smu->adev->dev, "Invalid power profile mode %d\n", profile_mode);
 		return -EINVAL;
 	}
@@ -1590,6 +1592,21 @@ static int vangogh_read_sensor(struct smu_context *smu,
 	return ret;
 }
 
+static int vangogh_get_apu_thermal_limit(struct smu_context *smu, uint32_t *limit)
+{
+	return smu_cmn_send_smc_msg_with_param(smu,
+					      SMU_MSG_GetThermalLimit,
+					      0, limit);
+}
+
+static int vangogh_set_apu_thermal_limit(struct smu_context *smu, uint32_t limit)
+{
+	return smu_cmn_send_smc_msg_with_param(smu,
+					      SMU_MSG_SetReducedThermalLimit,
+					      limit, NULL);
+}
+
+
 static int vangogh_set_watermarks_table(struct smu_context *smu,
 				       struct pp_smu_wm_range_sets *clock_ranges)
 {
@@ -2372,6 +2389,7 @@ static u32 vangogh_set_gfxoff_residency(struct smu_context *smu, bool start)
  * vangogh_get_gfxoff_residency
  *
  * @smu: amdgpu_device pointer
+ * @residency: placeholder for return value
  *
  * This function will be used to get gfxoff residency.
  *
@@ -2390,6 +2408,7 @@ static u32 vangogh_get_gfxoff_residency(struct smu_context *smu, uint32_t *resid
  * vangogh_get_gfxoff_entrycount - get gfxoff entry count
  *
  * @smu: amdgpu_device pointer
+ * @entrycount: placeholder for return value
  *
  * This function will be used to get gfxoff entry count
  *
@@ -2425,6 +2444,8 @@ static const struct pptable_funcs vangogh_ppt_funcs = {
 	.dpm_set_jpeg_enable = vangogh_dpm_set_jpeg_enable,
 	.is_dpm_running = vangogh_is_dpm_running,
 	.read_sensor = vangogh_read_sensor,
+	.get_apu_thermal_limit = vangogh_get_apu_thermal_limit,
+	.set_apu_thermal_limit = vangogh_set_apu_thermal_limit,
 	.get_enabled_mask = smu_cmn_get_enabled_mask,
 	.get_pp_feature_mask = smu_cmn_get_pp_feature_mask,
 	.set_watermarks_table = vangogh_set_watermarks_table,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/Makefile b/drivers/gpu/drm/amd/pm/swsmu/smu13/Makefile
index 9043f6ef1aee..7f3493b6c53c 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/Makefile
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/Makefile
@@ -24,7 +24,7 @@
 # It provides the smu management services for the driver.
 
 SMU13_MGR = smu_v13_0.o aldebaran_ppt.o yellow_carp_ppt.o smu_v13_0_0_ppt.o smu_v13_0_4_ppt.o \
-	    smu_v13_0_5_ppt.o smu_v13_0_7_ppt.o
+	    smu_v13_0_5_ppt.o smu_v13_0_7_ppt.o smu_v13_0_6_ppt.o
 
 AMD_SWSMU_SMU13MGR = $(addprefix $(AMD_SWSMU_PATH)/smu13/,$(SMU13_MGR))
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
index a52ed0580fd7..73175c993da9 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
@@ -294,6 +294,10 @@ int smu_v13_0_check_fw_version(struct smu_context *smu)
 	case IP_VERSION(13, 0, 5):
 		smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_5;
 		break;
+	case IP_VERSION(13, 0, 6):
+		smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_6;
+		adev->pm.fw_version = smu_version;
+		break;
 	default:
 		dev_err(adev->dev, "smu unsupported IP version: 0x%x.\n",
 			adev->ip_versions[MP1_HWIP][0]);
@@ -1914,10 +1918,9 @@ int smu_v13_0_set_power_source(struct smu_context *smu,
 					       NULL);
 }
 
-static int smu_v13_0_get_dpm_freq_by_index(struct smu_context *smu,
-					   enum smu_clk_type clk_type,
-					   uint16_t level,
-					   uint32_t *value)
+int smu_v13_0_get_dpm_freq_by_index(struct smu_context *smu,
+				    enum smu_clk_type clk_type, uint16_t level,
+				    uint32_t *value)
 {
 	int ret = 0, clk_id = 0;
 	uint32_t param;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
index a5c97d61e92a..09405ef1e3c8 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
@@ -1587,7 +1587,9 @@ static int smu_v13_0_0_get_power_profile_mode(struct smu_context *smu,
 		workload_type = smu_cmn_to_asic_specific_index(smu,
 							       CMN2ASIC_MAPPING_WORKLOAD,
 							       i);
-		if (workload_type < 0)
+		if (workload_type == -ENOTSUPP)
+			continue;
+		else if (workload_type < 0)
 			return -EINVAL;
 
 		result = smu_cmn_update_table(smu,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
new file mode 100644
index 000000000000..ea8f3d6fb98b
--- /dev/null
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
@@ -0,0 +1,2069 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#define SWSMU_CODE_LAYER_L2
+
+#include <linux/firmware.h>
+#include "amdgpu.h"
+#include "amdgpu_smu.h"
+#include "atomfirmware.h"
+#include "amdgpu_atomfirmware.h"
+#include "amdgpu_atombios.h"
+#include "smu_v13_0_6_pmfw.h"
+#include "smu13_driver_if_v13_0_6.h"
+#include "smu_v13_0_6_ppsmc.h"
+#include "soc15_common.h"
+#include "atom.h"
+#include "power_state.h"
+#include "smu_v13_0.h"
+#include "smu_v13_0_6_ppt.h"
+#include "nbio/nbio_7_4_offset.h"
+#include "nbio/nbio_7_4_sh_mask.h"
+#include "thm/thm_11_0_2_offset.h"
+#include "thm/thm_11_0_2_sh_mask.h"
+#include "amdgpu_xgmi.h"
+#include <linux/pci.h>
+#include "amdgpu_ras.h"
+#include "smu_cmn.h"
+#include "mp/mp_13_0_6_offset.h"
+#include "mp/mp_13_0_6_sh_mask.h"
+
+#undef MP1_Public
+#undef smnMP1_FIRMWARE_FLAGS
+
+/* TODO: Check final register offsets */
+#define MP1_Public 0x03b00000
+#define smnMP1_FIRMWARE_FLAGS 0x3010028
+/*
+ * DO NOT use these for err/warn/info/debug messages.
+ * Use dev_err, dev_warn, dev_info and dev_dbg instead.
+ * They are more MGPU friendly.
+ */
+#undef pr_err
+#undef pr_warn
+#undef pr_info
+#undef pr_debug
+
+#define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c))
+
+#define SMU_13_0_6_FEA_MAP(smu_feature, smu_13_0_6_feature)                    \
+	[smu_feature] = { 1, (smu_13_0_6_feature) }
+
+#define FEATURE_MASK(feature) (1ULL << feature)
+#define SMC_DPM_FEATURE                                                        \
+	(FEATURE_MASK(FEATURE_DATA_CALCULATION) |                              \
+	 FEATURE_MASK(FEATURE_DPM_GFXCLK) | FEATURE_MASK(FEATURE_DPM_UCLK) |   \
+	 FEATURE_MASK(FEATURE_DPM_SOCCLK) | FEATURE_MASK(FEATURE_DPM_FCLK) |   \
+	 FEATURE_MASK(FEATURE_DPM_LCLK) | FEATURE_MASK(FEATURE_DPM_XGMI) |     \
+	 FEATURE_MASK(FEATURE_DPM_VCN))
+
+/* possible frequency drift (1Mhz) */
+#define EPSILON 1
+
+#define smnPCIE_ESM_CTRL 0x111003D0
+
+static const struct cmn2asic_msg_mapping smu_v13_0_6_message_map[SMU_MSG_MAX_COUNT] = {
+	MSG_MAP(TestMessage,			     PPSMC_MSG_TestMessage,			0),
+	MSG_MAP(GetSmuVersion,			     PPSMC_MSG_GetSmuVersion,			1),
+	MSG_MAP(GetDriverIfVersion,		     PPSMC_MSG_GetDriverIfVersion,		1),
+	MSG_MAP(EnableAllSmuFeatures,		     PPSMC_MSG_EnableAllSmuFeatures,		1),
+	MSG_MAP(DisableAllSmuFeatures,		     PPSMC_MSG_DisableAllSmuFeatures,		1),
+	MSG_MAP(RequestI2cTransaction,		     PPSMC_MSG_RequestI2cTransaction,		0),
+	MSG_MAP(GetMetricsTable,		     PPSMC_MSG_GetMetricsTable,			1),
+	MSG_MAP(GetEnabledSmuFeaturesHigh,	     PPSMC_MSG_GetEnabledSmuFeaturesHigh,	1),
+	MSG_MAP(GetEnabledSmuFeaturesLow,	     PPSMC_MSG_GetEnabledSmuFeaturesLow,	1),
+	MSG_MAP(SetDriverDramAddrHigh,		     PPSMC_MSG_SetDriverDramAddrHigh,		1),
+	MSG_MAP(SetDriverDramAddrLow,		     PPSMC_MSG_SetDriverDramAddrLow,		1),
+	MSG_MAP(SetToolsDramAddrHigh,		     PPSMC_MSG_SetToolsDramAddrHigh,		0),
+	MSG_MAP(SetToolsDramAddrLow,		     PPSMC_MSG_SetToolsDramAddrLow,		0),
+	MSG_MAP(SetSoftMinByFreq,		     PPSMC_MSG_SetSoftMinByFreq,		0),
+	MSG_MAP(SetSoftMaxByFreq,		     PPSMC_MSG_SetSoftMaxByFreq,		0),
+	MSG_MAP(GetMinDpmFreq,			     PPSMC_MSG_GetMinDpmFreq,			0),
+	MSG_MAP(GetMaxDpmFreq,			     PPSMC_MSG_GetMaxDpmFreq,			0),
+	MSG_MAP(GetDpmFreqByIndex,		     PPSMC_MSG_GetDpmFreqByIndex,		1),
+	MSG_MAP(SetPptLimit,			     PPSMC_MSG_SetPptLimit,			0),
+	MSG_MAP(GetPptLimit,			     PPSMC_MSG_GetPptLimit,			1),
+	MSG_MAP(GfxDeviceDriverReset,		     PPSMC_MSG_GfxDriverReset,			0),
+	MSG_MAP(DramLogSetDramAddrHigh,		     PPSMC_MSG_DramLogSetDramAddrHigh,		0),
+	MSG_MAP(DramLogSetDramAddrLow,		     PPSMC_MSG_DramLogSetDramAddrLow,		0),
+	MSG_MAP(DramLogSetDramSize,		     PPSMC_MSG_DramLogSetDramSize,		0),
+	MSG_MAP(GetDebugData,			     PPSMC_MSG_GetDebugData,			0),
+	MSG_MAP(SetNumBadHbmPagesRetired,	     PPSMC_MSG_SetNumBadHbmPagesRetired,	0),
+	MSG_MAP(DFCstateControl,		     PPSMC_MSG_DFCstateControl,			0),
+	MSG_MAP(GetGmiPwrDnHyst,		     PPSMC_MSG_GetGmiPwrDnHyst,			0),
+	MSG_MAP(SetGmiPwrDnHyst,		     PPSMC_MSG_SetGmiPwrDnHyst,			0),
+	MSG_MAP(GmiPwrDnControl,		     PPSMC_MSG_GmiPwrDnControl,			0),
+	MSG_MAP(EnterGfxoff,			     PPSMC_MSG_EnterGfxoff,			0),
+	MSG_MAP(ExitGfxoff,			     PPSMC_MSG_ExitGfxoff,			0),
+	MSG_MAP(EnableDeterminism,		     PPSMC_MSG_EnableDeterminism,		0),
+	MSG_MAP(DisableDeterminism,		     PPSMC_MSG_DisableDeterminism,		0),
+	MSG_MAP(GfxDriverResetRecovery,		     PPSMC_MSG_GfxDriverResetRecovery,		0),
+	MSG_MAP(GetMinGfxclkFrequency,               PPSMC_MSG_GetMinGfxDpmFreq,                0),
+	MSG_MAP(GetMaxGfxclkFrequency,               PPSMC_MSG_GetMaxGfxDpmFreq,                0),
+	MSG_MAP(SetSoftMinGfxclk,                    PPSMC_MSG_SetSoftMinGfxClk,                0),
+	MSG_MAP(SetSoftMaxGfxClk,                    PPSMC_MSG_SetSoftMaxGfxClk,                0),
+};
+
+static const struct cmn2asic_mapping smu_v13_0_6_clk_map[SMU_CLK_COUNT] = {
+	CLK_MAP(SOCCLK, PPCLK_SOCCLK),
+	CLK_MAP(FCLK, PPCLK_FCLK),
+	CLK_MAP(UCLK, PPCLK_UCLK),
+	CLK_MAP(MCLK, PPCLK_UCLK),
+	CLK_MAP(DCLK, PPCLK_DCLK),
+	CLK_MAP(VCLK, PPCLK_VCLK),
+	CLK_MAP(LCLK, PPCLK_LCLK),
+};
+
+static const struct cmn2asic_mapping smu_v13_0_6_feature_mask_map[SMU_FEATURE_COUNT] = {
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DATA_CALCULATIONS_BIT, 		FEATURE_DATA_CALCULATION),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DPM_GFXCLK_BIT, 			FEATURE_DPM_GFXCLK),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DPM_UCLK_BIT, 			FEATURE_DPM_UCLK),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DPM_SOCCLK_BIT, 			FEATURE_DPM_SOCCLK),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DPM_FCLK_BIT, 			FEATURE_DPM_FCLK),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DPM_LCLK_BIT, 			FEATURE_DPM_LCLK),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DPM_VCLK_BIT,			FEATURE_DPM_VCN),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DPM_DCLK_BIT,			FEATURE_DPM_VCN),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DPM_XGMI_BIT, 			FEATURE_DPM_XGMI),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DS_GFXCLK_BIT, 			FEATURE_DS_GFXCLK),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DS_SOCCLK_BIT, 			FEATURE_DS_SOCCLK),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DS_LCLK_BIT, 			FEATURE_DS_LCLK),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DS_FCLK_BIT, 			FEATURE_DS_FCLK),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_VCN_DPM_BIT, 			FEATURE_DPM_VCN),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_PPT_BIT, 			FEATURE_PPT),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_TDC_BIT, 			FEATURE_TDC),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_APCC_DFLL_BIT, 			FEATURE_APCC_DFLL),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_MP1_CG_BIT, 			FEATURE_SMU_CG),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_GFXOFF_BIT, 			FEATURE_GFXOFF),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_FW_CTF_BIT, 			FEATURE_FW_CTF),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_THERMAL_BIT, 			FEATURE_THERMAL),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_XGMI_PER_LINK_PWR_DWN_BIT,	FEATURE_XGMI_PER_LINK_PWR_DOWN),
+	SMU_13_0_6_FEA_MAP(SMU_FEATURE_DF_CSTATE_BIT, 			FEATURE_DF_CSTATE),
+};
+
+#define TABLE_PMSTATUSLOG             0
+#define TABLE_SMU_METRICS             1
+#define TABLE_I2C_COMMANDS            2
+#define TABLE_COUNT                   3
+
+static const struct cmn2asic_mapping smu_v13_0_6_table_map[SMU_TABLE_COUNT] = {
+	TAB_MAP(PMSTATUSLOG),
+	TAB_MAP(SMU_METRICS),
+	TAB_MAP(I2C_COMMANDS),
+};
+
+#define THROTTLER_PROCHOT_GFX_BIT  0
+#define THROTTLER_PPT_BIT 1
+#define THROTTLER_TEMP_SOC_BIT 2
+#define THROTTLER_TEMP_VR_GFX_BIT 3
+#define THROTTLER_TEMP_HBM_BIT 4
+
+static const uint8_t smu_v13_0_6_throttler_map[] = {
+	[THROTTLER_PPT_BIT]		= (SMU_THROTTLER_PPT0_BIT),
+	[THROTTLER_TEMP_SOC_BIT]	= (SMU_THROTTLER_TEMP_GPU_BIT),
+	[THROTTLER_TEMP_HBM_BIT]	= (SMU_THROTTLER_TEMP_MEM_BIT),
+	[THROTTLER_TEMP_VR_GFX_BIT]	= (SMU_THROTTLER_TEMP_VR_GFX_BIT),
+	[THROTTLER_PROCHOT_GFX_BIT]	= (SMU_THROTTLER_PROCHOT_GFX_BIT),
+};
+
+struct PPTable_t {
+	uint32_t MaxSocketPowerLimit;
+	uint32_t MaxGfxclkFrequency;
+	uint32_t MinGfxclkFrequency;
+	uint32_t FclkFrequencyTable[4];
+	uint32_t UclkFrequencyTable[4];
+	uint32_t SocclkFrequencyTable[4];
+	uint32_t VclkFrequencyTable[4];
+	uint32_t DclkFrequencyTable[4];
+	uint32_t LclkFrequencyTable[4];
+	uint32_t MaxLclkDpmRange;
+	uint32_t MinLclkDpmRange;
+	bool Init;
+};
+
+#define SMUQ10_TO_UINT(x) ((x) >> 10)
+
+struct smu_v13_0_6_dpm_map {
+	enum smu_clk_type clk_type;
+	uint32_t feature_num;
+	struct smu_13_0_dpm_table *dpm_table;
+	uint32_t *freq_table;
+};
+
+static int smu_v13_0_6_tables_init(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *tables = smu_table->tables;
+	struct amdgpu_device *adev = smu->adev;
+
+	if (!(adev->flags & AMD_IS_APU))
+		SMU_TABLE_INIT(tables, SMU_TABLE_PMSTATUSLOG, SMU13_TOOL_SIZE,
+			       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+
+	SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, sizeof(MetricsTable_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+
+	SMU_TABLE_INIT(tables, SMU_TABLE_I2C_COMMANDS, sizeof(SwI2cRequest_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+
+	smu_table->metrics_table = kzalloc(sizeof(MetricsTable_t), GFP_KERNEL);
+	if (!smu_table->metrics_table)
+		return -ENOMEM;
+	smu_table->metrics_time = 0;
+
+	smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_3);
+	smu_table->gpu_metrics_table =
+		kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL);
+	if (!smu_table->gpu_metrics_table) {
+		kfree(smu_table->metrics_table);
+		return -ENOMEM;
+	}
+
+	smu_table->driver_pptable =
+		kzalloc(sizeof(struct PPTable_t), GFP_KERNEL);
+	if (!smu_table->driver_pptable) {
+		kfree(smu_table->metrics_table);
+		kfree(smu_table->gpu_metrics_table);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int smu_v13_0_6_allocate_dpm_context(struct smu_context *smu)
+{
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+
+	smu_dpm->dpm_context =
+		kzalloc(sizeof(struct smu_13_0_dpm_context), GFP_KERNEL);
+	if (!smu_dpm->dpm_context)
+		return -ENOMEM;
+	smu_dpm->dpm_context_size = sizeof(struct smu_13_0_dpm_context);
+
+	return 0;
+}
+
+static int smu_v13_0_6_init_smc_tables(struct smu_context *smu)
+{
+	int ret = 0;
+
+	ret = smu_v13_0_6_tables_init(smu);
+	if (ret)
+		return ret;
+
+	ret = smu_v13_0_6_allocate_dpm_context(smu);
+
+	return ret;
+}
+
+static int smu_v13_0_6_get_allowed_feature_mask(struct smu_context *smu,
+						uint32_t *feature_mask,
+						uint32_t num)
+{
+	if (num > 2)
+		return -EINVAL;
+
+	/* pptable will handle the features to enable */
+	memset(feature_mask, 0xFF, sizeof(uint32_t) * num);
+
+	return 0;
+}
+
+static int smu_v13_0_6_get_metrics_table(struct smu_context *smu,
+					 void *metrics_table, bool bypass_cache)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	uint32_t table_size = smu_table->tables[SMU_TABLE_SMU_METRICS].size;
+	struct smu_table *table = &smu_table->driver_table;
+	int ret;
+
+	if (bypass_cache || !smu_table->metrics_time ||
+	    time_after(jiffies,
+		       smu_table->metrics_time + msecs_to_jiffies(1))) {
+		ret = smu_cmn_send_smc_msg(smu, SMU_MSG_GetMetricsTable, NULL);
+		if (ret) {
+			dev_info(smu->adev->dev,
+				 "Failed to export SMU metrics table!\n");
+			return ret;
+		}
+
+		amdgpu_asic_invalidate_hdp(smu->adev, NULL);
+		memcpy(smu_table->metrics_table, table->cpu_addr, table_size);
+
+		smu_table->metrics_time = jiffies;
+	}
+
+	if (metrics_table)
+		memcpy(metrics_table, smu_table->metrics_table, table_size);
+
+	return 0;
+}
+
+static int smu_v13_0_6_setup_driver_pptable(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	MetricsTable_t *metrics = (MetricsTable_t *)smu_table->metrics_table;
+	struct PPTable_t *pptable =
+		(struct PPTable_t *)smu_table->driver_pptable;
+	int ret;
+	int i;
+
+	/* Store one-time values in driver PPTable */
+	if (!pptable->Init) {
+		ret = smu_v13_0_6_get_metrics_table(smu, NULL, false);
+		if (ret)
+			return ret;
+
+		pptable->MaxSocketPowerLimit =
+			SMUQ10_TO_UINT(metrics->MaxSocketPowerLimit);
+		pptable->MaxGfxclkFrequency =
+			SMUQ10_TO_UINT(metrics->MaxGfxclkFrequency);
+		pptable->MinGfxclkFrequency =
+			SMUQ10_TO_UINT(metrics->MinGfxclkFrequency);
+
+		for (i = 0; i < 4; ++i) {
+			pptable->FclkFrequencyTable[i] =
+				SMUQ10_TO_UINT(metrics->FclkFrequencyTable[i]);
+			pptable->UclkFrequencyTable[i] =
+				SMUQ10_TO_UINT(metrics->UclkFrequencyTable[i]);
+			pptable->SocclkFrequencyTable[i] = SMUQ10_TO_UINT(
+				metrics->SocclkFrequencyTable[i]);
+			pptable->VclkFrequencyTable[i] =
+				SMUQ10_TO_UINT(metrics->VclkFrequencyTable[i]);
+			pptable->DclkFrequencyTable[i] =
+				SMUQ10_TO_UINT(metrics->DclkFrequencyTable[i]);
+			pptable->LclkFrequencyTable[i] =
+				SMUQ10_TO_UINT(metrics->LclkFrequencyTable[i]);
+		}
+
+		pptable->Init = true;
+	}
+
+	return 0;
+}
+
+static int smu_v13_0_6_get_dpm_ultimate_freq(struct smu_context *smu,
+					     enum smu_clk_type clk_type,
+					     uint32_t *min, uint32_t *max)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct PPTable_t *pptable =
+		(struct PPTable_t *)smu_table->driver_pptable;
+	uint32_t clock_limit = 0, param;
+	int ret = 0, clk_id = 0;
+
+	if (!smu_cmn_clk_dpm_is_enabled(smu, clk_type)) {
+		switch (clk_type) {
+		case SMU_MCLK:
+		case SMU_UCLK:
+			if (pptable->Init)
+				clock_limit = pptable->UclkFrequencyTable[0];
+			break;
+		case SMU_GFXCLK:
+		case SMU_SCLK:
+			if (pptable->Init)
+				clock_limit = pptable->MinGfxclkFrequency;
+			break;
+		case SMU_SOCCLK:
+			if (pptable->Init)
+				clock_limit = pptable->UclkFrequencyTable[0];
+			break;
+		case SMU_FCLK:
+			if (pptable->Init)
+				clock_limit = pptable->FclkFrequencyTable[0];
+			break;
+		case SMU_VCLK:
+			if (pptable->Init)
+				clock_limit = pptable->VclkFrequencyTable[0];
+			break;
+		case SMU_DCLK:
+			if (pptable->Init)
+				clock_limit = pptable->DclkFrequencyTable[0];
+			break;
+		default:
+			break;
+		}
+
+		if (min)
+			*min = clock_limit;
+
+		if (max)
+			*max = clock_limit;
+
+		return 0;
+	}
+
+	if (!(clk_type == SMU_GFXCLK || clk_type == SMU_SCLK)) {
+		clk_id = smu_cmn_to_asic_specific_index(
+			smu, CMN2ASIC_MAPPING_CLK, clk_type);
+		if (clk_id < 0) {
+			ret = -EINVAL;
+			goto failed;
+		}
+		param = (clk_id & 0xffff) << 16;
+	}
+
+	if (max) {
+		if (clk_type == SMU_GFXCLK || clk_type == SMU_SCLK)
+			ret = smu_cmn_send_smc_msg(
+				smu, SMU_MSG_GetMaxGfxclkFrequency, max);
+		else
+			ret = smu_cmn_send_smc_msg_with_param(
+				smu, SMU_MSG_GetMaxDpmFreq, param, max);
+		if (ret)
+			goto failed;
+	}
+
+	if (min) {
+		if (clk_type == SMU_GFXCLK || clk_type == SMU_SCLK)
+			ret = smu_cmn_send_smc_msg(
+				smu, SMU_MSG_GetMinGfxclkFrequency, min);
+		else
+			ret = smu_cmn_send_smc_msg_with_param(
+				smu, SMU_MSG_GetMinDpmFreq, param, min);
+	}
+
+failed:
+	return ret;
+}
+
+static int smu_v13_0_6_get_dpm_level_count(struct smu_context *smu,
+					  enum smu_clk_type clk_type,
+					  uint32_t *levels)
+{
+	int ret;
+
+	ret = smu_v13_0_get_dpm_freq_by_index(smu, clk_type, 0xff, levels);
+	if (!ret)
+		++(*levels);
+
+	return ret;
+}
+
+static int smu_v13_0_6_set_default_dpm_table(struct smu_context *smu)
+{
+	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_13_0_dpm_table *dpm_table = NULL;
+	struct PPTable_t *pptable =
+		(struct PPTable_t *)smu_table->driver_pptable;
+	uint32_t gfxclkmin, gfxclkmax, levels;
+	int ret = 0, i, j;
+	struct smu_v13_0_6_dpm_map dpm_map[] = {
+		{ SMU_SOCCLK, SMU_FEATURE_DPM_SOCCLK_BIT,
+		  &dpm_context->dpm_tables.soc_table,
+		  pptable->SocclkFrequencyTable },
+		{ SMU_UCLK, SMU_FEATURE_DPM_UCLK_BIT,
+		  &dpm_context->dpm_tables.uclk_table,
+		  pptable->UclkFrequencyTable },
+		{ SMU_FCLK, SMU_FEATURE_DPM_FCLK_BIT,
+		  &dpm_context->dpm_tables.fclk_table,
+		  pptable->FclkFrequencyTable },
+		{ SMU_VCLK, SMU_FEATURE_DPM_VCLK_BIT,
+		  &dpm_context->dpm_tables.vclk_table,
+		  pptable->VclkFrequencyTable },
+		{ SMU_DCLK, SMU_FEATURE_DPM_DCLK_BIT,
+		  &dpm_context->dpm_tables.dclk_table,
+		  pptable->DclkFrequencyTable },
+	};
+
+	smu_v13_0_6_setup_driver_pptable(smu);
+
+	/* gfxclk dpm table setup */
+	dpm_table = &dpm_context->dpm_tables.gfx_table;
+	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
+		/* In the case of gfxclk, only fine-grained dpm is honored.
+		 * Get min/max values from FW.
+		 */
+		ret = smu_v13_0_6_get_dpm_ultimate_freq(smu, SMU_GFXCLK,
+							&gfxclkmin, &gfxclkmax);
+		if (ret)
+			return ret;
+
+		dpm_table->count = 2;
+		dpm_table->dpm_levels[0].value = gfxclkmin;
+		dpm_table->dpm_levels[0].enabled = true;
+		dpm_table->dpm_levels[1].value = gfxclkmax;
+		dpm_table->dpm_levels[1].enabled = true;
+		dpm_table->min = dpm_table->dpm_levels[0].value;
+		dpm_table->max = dpm_table->dpm_levels[1].value;
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = pptable->MinGfxclkFrequency;
+		dpm_table->dpm_levels[0].enabled = true;
+		dpm_table->min = dpm_table->dpm_levels[0].value;
+		dpm_table->max = dpm_table->dpm_levels[0].value;
+	}
+
+	for (j = 0; j < ARRAY_SIZE(dpm_map); j++) {
+		dpm_table = dpm_map[j].dpm_table;
+		levels = 1;
+		if (smu_cmn_feature_is_enabled(smu, dpm_map[j].feature_num)) {
+			ret = smu_v13_0_6_get_dpm_level_count(
+				smu, dpm_map[j].clk_type, &levels);
+			if (ret)
+				return ret;
+		}
+		dpm_table->count = levels;
+		for (i = 0; i < dpm_table->count; ++i) {
+			dpm_table->dpm_levels[i].value =
+				dpm_map[j].freq_table[i];
+			dpm_table->dpm_levels[i].enabled = true;
+
+		}
+		dpm_table->min = dpm_table->dpm_levels[0].value;
+		dpm_table->max = dpm_table->dpm_levels[levels - 1].value;
+
+	}
+
+	return 0;
+}
+
+static int smu_v13_0_6_setup_pptable(struct smu_context *smu)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+
+	/* TODO: PPTable is not available.
+	 * 1) Find an alternate way to get 'PPTable values' here.
+	 * 2) Check if there is SW CTF
+	 */
+	table_context->thermal_controller_type = 0;
+
+	return 0;
+}
+
+static int smu_v13_0_6_check_fw_status(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t mp1_fw_flags;
+
+	mp1_fw_flags =
+		RREG32_PCIE(MP1_Public | (smnMP1_FIRMWARE_FLAGS & 0xffffffff));
+
+	if ((mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) >>
+	    MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT)
+		return 0;
+
+	return -EIO;
+}
+
+static int smu_v13_0_6_populate_umd_state_clk(struct smu_context *smu)
+{
+	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
+	struct smu_13_0_dpm_table *gfx_table =
+		&dpm_context->dpm_tables.gfx_table;
+	struct smu_13_0_dpm_table *mem_table =
+		&dpm_context->dpm_tables.uclk_table;
+	struct smu_13_0_dpm_table *soc_table =
+		&dpm_context->dpm_tables.soc_table;
+	struct smu_umd_pstate_table *pstate_table = &smu->pstate_table;
+
+	pstate_table->gfxclk_pstate.min = gfx_table->min;
+	pstate_table->gfxclk_pstate.peak = gfx_table->max;
+	pstate_table->gfxclk_pstate.curr.min = gfx_table->min;
+	pstate_table->gfxclk_pstate.curr.max = gfx_table->max;
+
+	pstate_table->uclk_pstate.min = mem_table->min;
+	pstate_table->uclk_pstate.peak = mem_table->max;
+	pstate_table->uclk_pstate.curr.min = mem_table->min;
+	pstate_table->uclk_pstate.curr.max = mem_table->max;
+
+	pstate_table->socclk_pstate.min = soc_table->min;
+	pstate_table->socclk_pstate.peak = soc_table->max;
+	pstate_table->socclk_pstate.curr.min = soc_table->min;
+	pstate_table->socclk_pstate.curr.max = soc_table->max;
+
+	if (gfx_table->count > SMU_13_0_6_UMD_PSTATE_GFXCLK_LEVEL &&
+	    mem_table->count > SMU_13_0_6_UMD_PSTATE_MCLK_LEVEL &&
+	    soc_table->count > SMU_13_0_6_UMD_PSTATE_SOCCLK_LEVEL) {
+		pstate_table->gfxclk_pstate.standard =
+			gfx_table->dpm_levels[SMU_13_0_6_UMD_PSTATE_GFXCLK_LEVEL].value;
+		pstate_table->uclk_pstate.standard =
+			mem_table->dpm_levels[SMU_13_0_6_UMD_PSTATE_MCLK_LEVEL].value;
+		pstate_table->socclk_pstate.standard =
+			soc_table->dpm_levels[SMU_13_0_6_UMD_PSTATE_SOCCLK_LEVEL].value;
+	} else {
+		pstate_table->gfxclk_pstate.standard =
+			pstate_table->gfxclk_pstate.min;
+		pstate_table->uclk_pstate.standard =
+			pstate_table->uclk_pstate.min;
+		pstate_table->socclk_pstate.standard =
+			pstate_table->socclk_pstate.min;
+	}
+
+	return 0;
+}
+
+static int smu_v13_0_6_get_clk_table(struct smu_context *smu,
+				     struct pp_clock_levels_with_latency *clocks,
+				     struct smu_13_0_dpm_table *dpm_table)
+{
+	int i, count;
+
+	count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS :
+						      dpm_table->count;
+	clocks->num_levels = count;
+
+	for (i = 0; i < count; i++) {
+		clocks->data[i].clocks_in_khz =
+			dpm_table->dpm_levels[i].value * 1000;
+		clocks->data[i].latency_in_us = 0;
+	}
+
+	return 0;
+}
+
+static int smu_v13_0_6_freqs_in_same_level(int32_t frequency1,
+					   int32_t frequency2)
+{
+	return (abs(frequency1 - frequency2) <= EPSILON);
+}
+
+static uint32_t smu_v13_0_6_get_throttler_status(struct smu_context *smu,
+						 MetricsTable_t *metrics)
+{
+	uint32_t  throttler_status = 0;
+
+	throttler_status |= metrics->ProchotResidencyAcc > 0 ? 1U << THROTTLER_PROCHOT_GFX_BIT : 0;
+	throttler_status |= metrics->PptResidencyAcc > 0 ? 1U << THROTTLER_PPT_BIT : 0;
+	throttler_status |= metrics->SocketThmResidencyAcc > 0 ?  1U << THROTTLER_TEMP_SOC_BIT : 0;
+	throttler_status |= metrics->VrThmResidencyAcc > 0 ? 1U << THROTTLER_TEMP_VR_GFX_BIT : 0;
+	throttler_status |= metrics->HbmThmResidencyAcc > 0 ? 1U << THROTTLER_TEMP_HBM_BIT : 0;
+
+	return throttler_status;
+}
+
+static int smu_v13_0_6_get_smu_metrics_data(struct smu_context *smu,
+					    MetricsMember_t member,
+					    uint32_t *value)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	MetricsTable_t *metrics = (MetricsTable_t *)smu_table->metrics_table;
+	int ret = 0;
+
+	ret = smu_v13_0_6_get_metrics_table(smu, NULL, false);
+	if (ret)
+		return ret;
+
+	/* For clocks with multiple instances, only report the first one */
+	switch (member) {
+	case METRICS_CURR_GFXCLK:
+	case METRICS_AVERAGE_GFXCLK:
+		*value = 0;
+		break;
+	case METRICS_CURR_SOCCLK:
+	case METRICS_AVERAGE_SOCCLK:
+		*value = SMUQ10_TO_UINT(metrics->SocclkFrequency[0]);
+		break;
+	case METRICS_CURR_UCLK:
+	case METRICS_AVERAGE_UCLK:
+		*value = SMUQ10_TO_UINT(metrics->UclkFrequency);
+		break;
+	case METRICS_CURR_VCLK:
+		*value = SMUQ10_TO_UINT(metrics->VclkFrequency[0]);
+		break;
+	case METRICS_CURR_DCLK:
+		*value = SMUQ10_TO_UINT(metrics->DclkFrequency[0]);
+		break;
+	case METRICS_CURR_FCLK:
+		*value = SMUQ10_TO_UINT(metrics->FclkFrequency);
+		break;
+	case METRICS_AVERAGE_GFXACTIVITY:
+		*value = SMUQ10_TO_UINT(metrics->SocketGfxBusy);
+		break;
+	case METRICS_AVERAGE_MEMACTIVITY:
+		*value = SMUQ10_TO_UINT(metrics->DramBandwidthUtilization);
+		break;
+	case METRICS_AVERAGE_SOCKETPOWER:
+		*value = SMUQ10_TO_UINT(metrics->SocketPower) << 8;
+		break;
+	case METRICS_TEMPERATURE_HOTSPOT:
+		*value = SMUQ10_TO_UINT(metrics->MaxSocketTemperature);
+		break;
+	case METRICS_TEMPERATURE_MEM:
+		*value = SMUQ10_TO_UINT(metrics->MaxHbmTemperature);
+		break;
+	/* This is the max of all VRs and not just SOC VR.
+	 * No need to define another data type for the same.
+	 */
+	case METRICS_TEMPERATURE_VRSOC:
+		*value = SMUQ10_TO_UINT(metrics->MaxVrTemperature);
+		break;
+	case METRICS_THROTTLER_STATUS:
+		*value = smu_v13_0_6_get_throttler_status(smu, metrics);
+		break;
+	default:
+		*value = UINT_MAX;
+		break;
+	}
+
+	return ret;
+}
+
+static int smu_v13_0_6_get_current_clk_freq_by_table(struct smu_context *smu,
+						     enum smu_clk_type clk_type,
+						     uint32_t *value)
+{
+	MetricsMember_t member_type;
+
+	if (!value)
+		return -EINVAL;
+
+	switch (clk_type) {
+	case SMU_GFXCLK:
+		member_type = METRICS_CURR_GFXCLK;
+		break;
+	case SMU_UCLK:
+		member_type = METRICS_CURR_UCLK;
+		break;
+	case SMU_SOCCLK:
+		member_type = METRICS_CURR_SOCCLK;
+		break;
+	case SMU_VCLK:
+		member_type = METRICS_CURR_VCLK;
+		break;
+	case SMU_DCLK:
+		member_type = METRICS_CURR_DCLK;
+		break;
+	case SMU_FCLK:
+		member_type = METRICS_CURR_FCLK;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return smu_v13_0_6_get_smu_metrics_data(smu, member_type, value);
+}
+
+static int smu_v13_0_6_print_clk_levels(struct smu_context *smu,
+					enum smu_clk_type type, char *buf)
+{
+	int i, now, size = 0;
+	int ret = 0;
+	struct smu_umd_pstate_table *pstate_table = &smu->pstate_table;
+	struct pp_clock_levels_with_latency clocks;
+	struct smu_13_0_dpm_table *single_dpm_table;
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	struct smu_13_0_dpm_context *dpm_context = NULL;
+	uint32_t display_levels;
+	uint32_t freq_values[3] = { 0 };
+	uint32_t min_clk, max_clk;
+
+	smu_cmn_get_sysfs_buf(&buf, &size);
+
+	if (amdgpu_ras_intr_triggered()) {
+		size += sysfs_emit_at(buf, size, "unavailable\n");
+		return size;
+	}
+
+	dpm_context = smu_dpm->dpm_context;
+
+	switch (type) {
+	case SMU_OD_SCLK:
+		size += sysfs_emit_at(buf, size, "%s:\n", "GFXCLK");
+		fallthrough;
+	case SMU_SCLK:
+		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_GFXCLK,
+								&now);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get current gfx clk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
+		ret = smu_v13_0_6_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get gfx clk levels Failed!");
+			return ret;
+		}
+
+		display_levels = clocks.num_levels;
+
+		min_clk = pstate_table->gfxclk_pstate.curr.min;
+		max_clk = pstate_table->gfxclk_pstate.curr.max;
+
+		freq_values[0] = min_clk;
+		freq_values[1] = max_clk;
+
+		/* fine-grained dpm has only 2 levels */
+		if (now > min_clk && now < max_clk) {
+			display_levels = clocks.num_levels + 1;
+			freq_values[2] = max_clk;
+			freq_values[1] = now;
+		}
+
+		/*
+		 * For DPM disabled case, there will be only one clock level.
+		 * And it's safe to assume that is always the current clock.
+		 */
+		if (display_levels == clocks.num_levels) {
+			for (i = 0; i < clocks.num_levels; i++)
+				size += sysfs_emit_at(
+					buf, size, "%d: %uMhz %s\n", i,
+					freq_values[i],
+					(clocks.num_levels == 1) ?
+						"*" :
+						(smu_v13_0_6_freqs_in_same_level(
+							 freq_values[i], now) ?
+							 "*" :
+							 ""));
+		} else {
+			for (i = 0; i < display_levels; i++)
+				size += sysfs_emit_at(buf, size,
+						      "%d: %uMhz %s\n", i,
+						      freq_values[i],
+						      i == 1 ? "*" : "");
+		}
+
+		break;
+
+	case SMU_OD_MCLK:
+		size += sysfs_emit_at(buf, size, "%s:\n", "MCLK");
+		fallthrough;
+	case SMU_MCLK:
+		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_UCLK,
+								&now);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get current mclk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_context->dpm_tables.uclk_table);
+		ret = smu_v13_0_6_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get memory clk levels Failed!");
+			return ret;
+		}
+
+		for (i = 0; i < clocks.num_levels; i++)
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				clocks.data[i].clocks_in_khz / 1000,
+				(clocks.num_levels == 1) ?
+					"*" :
+					(smu_v13_0_6_freqs_in_same_level(
+						 clocks.data[i].clocks_in_khz /
+							 1000,
+						 now) ?
+						 "*" :
+						 ""));
+		break;
+
+	case SMU_SOCCLK:
+		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_SOCCLK,
+								&now);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get current socclk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_context->dpm_tables.soc_table);
+		ret = smu_v13_0_6_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get socclk levels Failed!");
+			return ret;
+		}
+
+		for (i = 0; i < clocks.num_levels; i++)
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				clocks.data[i].clocks_in_khz / 1000,
+				(clocks.num_levels == 1) ?
+					"*" :
+					(smu_v13_0_6_freqs_in_same_level(
+						 clocks.data[i].clocks_in_khz /
+							 1000,
+						 now) ?
+						 "*" :
+						 ""));
+		break;
+
+	case SMU_FCLK:
+		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_FCLK,
+								&now);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get current fclk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_context->dpm_tables.fclk_table);
+		ret = smu_v13_0_6_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get fclk levels Failed!");
+			return ret;
+		}
+
+		for (i = 0; i < single_dpm_table->count; i++)
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				single_dpm_table->dpm_levels[i].value,
+				(clocks.num_levels == 1) ?
+					"*" :
+					(smu_v13_0_6_freqs_in_same_level(
+						 clocks.data[i].clocks_in_khz /
+							 1000,
+						 now) ?
+						 "*" :
+						 ""));
+		break;
+
+	case SMU_VCLK:
+		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_VCLK,
+								&now);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get current vclk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_context->dpm_tables.vclk_table);
+		ret = smu_v13_0_6_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get vclk levels Failed!");
+			return ret;
+		}
+
+		for (i = 0; i < single_dpm_table->count; i++)
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				single_dpm_table->dpm_levels[i].value,
+				(clocks.num_levels == 1) ?
+					"*" :
+					(smu_v13_0_6_freqs_in_same_level(
+						 clocks.data[i].clocks_in_khz /
+							 1000,
+						 now) ?
+						 "*" :
+						 ""));
+		break;
+
+	case SMU_DCLK:
+		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_DCLK,
+							       &now);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get current dclk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_context->dpm_tables.dclk_table);
+		ret = smu_v13_0_6_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get dclk levels Failed!");
+			return ret;
+		}
+
+		for (i = 0; i < single_dpm_table->count; i++)
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				single_dpm_table->dpm_levels[i].value,
+				(clocks.num_levels == 1) ?
+					"*" :
+					(smu_v13_0_6_freqs_in_same_level(
+						 clocks.data[i].clocks_in_khz /
+							 1000,
+						 now) ?
+						 "*" :
+						 ""));
+		break;
+
+	default:
+		break;
+	}
+
+	return size;
+}
+
+static int smu_v13_0_6_upload_dpm_level(struct smu_context *smu, bool max,
+					uint32_t feature_mask, uint32_t level)
+{
+	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
+	uint32_t freq;
+	int ret = 0;
+
+	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT) &&
+	    (feature_mask & FEATURE_MASK(FEATURE_DPM_GFXCLK))) {
+		freq = dpm_context->dpm_tables.gfx_table.dpm_levels[level].value;
+		ret = smu_cmn_send_smc_msg_with_param(
+			smu,
+			(max ? SMU_MSG_SetSoftMaxGfxClk :
+			       SMU_MSG_SetSoftMinGfxclk),
+			freq & 0xffff, NULL);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Failed to set soft %s gfxclk !\n",
+				max ? "max" : "min");
+			return ret;
+		}
+	}
+
+	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT) &&
+	    (feature_mask & FEATURE_MASK(FEATURE_DPM_UCLK))) {
+		freq = dpm_context->dpm_tables.uclk_table.dpm_levels[level]
+			       .value;
+		ret = smu_cmn_send_smc_msg_with_param(
+			smu,
+			(max ? SMU_MSG_SetSoftMaxByFreq :
+			       SMU_MSG_SetSoftMinByFreq),
+			(PPCLK_UCLK << 16) | (freq & 0xffff), NULL);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Failed to set soft %s memclk !\n",
+				max ? "max" : "min");
+			return ret;
+		}
+	}
+
+	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT) &&
+	    (feature_mask & FEATURE_MASK(FEATURE_DPM_SOCCLK))) {
+		freq = dpm_context->dpm_tables.soc_table.dpm_levels[level].value;
+		ret = smu_cmn_send_smc_msg_with_param(
+			smu,
+			(max ? SMU_MSG_SetSoftMaxByFreq :
+			       SMU_MSG_SetSoftMinByFreq),
+			(PPCLK_SOCCLK << 16) | (freq & 0xffff), NULL);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Failed to set soft %s socclk !\n",
+				max ? "max" : "min");
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+static int smu_v13_0_6_force_clk_levels(struct smu_context *smu,
+					enum smu_clk_type type, uint32_t mask)
+{
+	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
+	struct smu_13_0_dpm_table *single_dpm_table = NULL;
+	uint32_t soft_min_level, soft_max_level;
+	int ret = 0;
+
+	soft_min_level = mask ? (ffs(mask) - 1) : 0;
+	soft_max_level = mask ? (fls(mask) - 1) : 0;
+
+	switch (type) {
+	case SMU_SCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
+		if (soft_max_level >= single_dpm_table->count) {
+			dev_err(smu->adev->dev,
+				"Clock level specified %d is over max allowed %d\n",
+				soft_max_level, single_dpm_table->count - 1);
+			ret = -EINVAL;
+			break;
+		}
+
+		ret = smu_v13_0_6_upload_dpm_level(
+			smu, false, FEATURE_MASK(FEATURE_DPM_GFXCLK),
+			soft_min_level);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Failed to upload boot level to lowest!\n");
+			break;
+		}
+
+		ret = smu_v13_0_6_upload_dpm_level(
+			smu, true, FEATURE_MASK(FEATURE_DPM_GFXCLK),
+			soft_max_level);
+		if (ret)
+			dev_err(smu->adev->dev,
+				"Failed to upload dpm max level to highest!\n");
+
+		break;
+
+	case SMU_MCLK:
+	case SMU_SOCCLK:
+	case SMU_FCLK:
+		/*
+		 * Should not arrive here since smu_13_0_6 does not
+		 * support mclk/socclk/fclk softmin/softmax settings
+		 */
+		ret = -EINVAL;
+		break;
+
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static int smu_v13_0_6_get_current_activity_percent(struct smu_context *smu,
+						    enum amd_pp_sensors sensor,
+						    uint32_t *value)
+{
+	int ret = 0;
+
+	if (!value)
+		return -EINVAL;
+
+	switch (sensor) {
+	case AMDGPU_PP_SENSOR_GPU_LOAD:
+		ret = smu_v13_0_6_get_smu_metrics_data(
+			smu, METRICS_AVERAGE_GFXACTIVITY, value);
+		break;
+	case AMDGPU_PP_SENSOR_MEM_LOAD:
+		ret = smu_v13_0_6_get_smu_metrics_data(
+			smu, METRICS_AVERAGE_MEMACTIVITY, value);
+		break;
+	default:
+		dev_err(smu->adev->dev,
+			"Invalid sensor for retrieving clock activity\n");
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static int smu_v13_0_6_get_gpu_power(struct smu_context *smu, uint32_t *value)
+{
+	if (!value)
+		return -EINVAL;
+
+	return smu_v13_0_6_get_smu_metrics_data(smu, METRICS_AVERAGE_SOCKETPOWER,
+					       value);
+}
+
+static int smu_v13_0_6_thermal_get_temperature(struct smu_context *smu,
+					       enum amd_pp_sensors sensor,
+					       uint32_t *value)
+{
+	int ret = 0;
+
+	if (!value)
+		return -EINVAL;
+
+	switch (sensor) {
+	case AMDGPU_PP_SENSOR_HOTSPOT_TEMP:
+		ret = smu_v13_0_6_get_smu_metrics_data(
+			smu, METRICS_TEMPERATURE_HOTSPOT, value);
+		break;
+	case AMDGPU_PP_SENSOR_MEM_TEMP:
+		ret = smu_v13_0_6_get_smu_metrics_data(
+			smu, METRICS_TEMPERATURE_MEM, value);
+		break;
+	default:
+		dev_err(smu->adev->dev, "Invalid sensor for retrieving temp\n");
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static int smu_v13_0_6_read_sensor(struct smu_context *smu,
+				   enum amd_pp_sensors sensor, void *data,
+				   uint32_t *size)
+{
+	int ret = 0;
+
+	if (amdgpu_ras_intr_triggered())
+		return 0;
+
+	if (!data || !size)
+		return -EINVAL;
+
+	switch (sensor) {
+	case AMDGPU_PP_SENSOR_MEM_LOAD:
+	case AMDGPU_PP_SENSOR_GPU_LOAD:
+		ret = smu_v13_0_6_get_current_activity_percent(smu, sensor,
+							       (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GPU_POWER:
+		ret = smu_v13_0_6_get_gpu_power(smu, (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_HOTSPOT_TEMP:
+	case AMDGPU_PP_SENSOR_MEM_TEMP:
+		ret = smu_v13_0_6_thermal_get_temperature(smu, sensor,
+							  (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GFX_MCLK:
+		ret = smu_v13_0_6_get_current_clk_freq_by_table(
+			smu, SMU_UCLK, (uint32_t *)data);
+		/* the output clock frequency in 10K unit */
+		*(uint32_t *)data *= 100;
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GFX_SCLK:
+		ret = smu_v13_0_6_get_current_clk_freq_by_table(
+			smu, SMU_GFXCLK, (uint32_t *)data);
+		*(uint32_t *)data *= 100;
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_VDDGFX:
+		ret = smu_v13_0_get_gfx_vdd(smu, (uint32_t *)data);
+		*size = 4;
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
+
+	return ret;
+}
+
+static int smu_v13_0_6_get_power_limit(struct smu_context *smu,
+				       uint32_t *current_power_limit,
+				       uint32_t *default_power_limit,
+				       uint32_t *max_power_limit)
+{
+        struct smu_table_context *smu_table = &smu->smu_table;
+        struct PPTable_t *pptable =
+                (struct PPTable_t *)smu_table->driver_pptable;
+	uint32_t power_limit = 0;
+	int ret;
+
+	if (!smu_cmn_feature_is_enabled(smu, SMU_FEATURE_PPT_BIT)) {
+		if (current_power_limit)
+			*current_power_limit = 0;
+		if (default_power_limit)
+			*default_power_limit = 0;
+		if (max_power_limit)
+			*max_power_limit = 0;
+
+		dev_warn(
+			smu->adev->dev,
+			"PPT feature is not enabled, power values can't be fetched.");
+
+		return 0;
+	}
+
+	ret = smu_cmn_send_smc_msg(smu, SMU_MSG_GetPptLimit, &power_limit);
+
+	if (ret) {
+		dev_err(smu->adev->dev, "Couldn't get PPT limit");
+		return -EINVAL;
+	}
+
+	if (current_power_limit)
+		*current_power_limit = power_limit;
+	if (default_power_limit)
+		*default_power_limit = power_limit;
+
+	if (max_power_limit) {
+		*max_power_limit = pptable->MaxSocketPowerLimit;
+	}
+
+	return 0;
+}
+
+static int smu_v13_0_6_set_power_limit(struct smu_context *smu,
+				       enum smu_ppt_limit_type limit_type,
+				       uint32_t limit)
+{
+	return smu_v13_0_set_power_limit(smu, limit_type, limit);
+}
+
+static int smu_v13_0_6_system_features_control(struct smu_context *smu,
+					       bool enable)
+{
+	int ret;
+
+	/* Nothing to be done for APU */
+	if (smu->adev->flags & AMD_IS_APU)
+		return 0;
+
+	ret = smu_v13_0_system_features_control(smu, enable);
+
+	return ret;
+}
+
+static int smu_v13_0_6_set_gfx_soft_freq_limited_range(struct smu_context *smu,
+						       uint32_t min,
+						       uint32_t max)
+{
+	int ret;
+
+	ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxGfxClk,
+					      max & 0xffff, NULL);
+	if (ret)
+		return ret;
+
+	ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMinGfxclk,
+					      min & 0xffff, NULL);
+
+	return ret;
+}
+
+static int smu_v13_0_6_set_performance_level(struct smu_context *smu,
+					     enum amd_dpm_forced_level level)
+{
+	struct smu_dpm_context *smu_dpm = &(smu->smu_dpm);
+	struct smu_13_0_dpm_context *dpm_context = smu_dpm->dpm_context;
+	struct smu_13_0_dpm_table *gfx_table =
+		&dpm_context->dpm_tables.gfx_table;
+	struct smu_umd_pstate_table *pstate_table = &smu->pstate_table;
+	int ret;
+
+	/* Disable determinism if switching to another mode */
+	if ((smu_dpm->dpm_level == AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM) &&
+	    (level != AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM)) {
+		smu_cmn_send_smc_msg(smu, SMU_MSG_DisableDeterminism, NULL);
+		pstate_table->gfxclk_pstate.curr.max = gfx_table->max;
+	}
+
+	switch (level) {
+	case AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM:
+		return 0;
+
+	case AMD_DPM_FORCED_LEVEL_AUTO:
+		if ((gfx_table->min == pstate_table->gfxclk_pstate.curr.min) &&
+		    (gfx_table->max == pstate_table->gfxclk_pstate.curr.max))
+			return 0;
+
+		ret = smu_v13_0_6_set_gfx_soft_freq_limited_range(
+			smu, gfx_table->min, gfx_table->max);
+		if (ret)
+			return ret;
+
+		pstate_table->gfxclk_pstate.curr.min = gfx_table->min;
+		pstate_table->gfxclk_pstate.curr.max = gfx_table->max;
+		return 0;
+	case AMD_DPM_FORCED_LEVEL_MANUAL:
+		return 0;
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+static int smu_v13_0_6_set_soft_freq_limited_range(struct smu_context *smu,
+						   enum smu_clk_type clk_type,
+						   uint32_t min, uint32_t max)
+{
+	struct smu_dpm_context *smu_dpm = &(smu->smu_dpm);
+	struct smu_13_0_dpm_context *dpm_context = smu_dpm->dpm_context;
+	struct smu_umd_pstate_table *pstate_table = &smu->pstate_table;
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t min_clk;
+	uint32_t max_clk;
+	int ret = 0;
+
+	if (clk_type != SMU_GFXCLK && clk_type != SMU_SCLK)
+		return -EINVAL;
+
+	if ((smu_dpm->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL) &&
+	    (smu_dpm->dpm_level != AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM))
+		return -EINVAL;
+
+	if (smu_dpm->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) {
+		if (min >= max) {
+			dev_err(smu->adev->dev,
+				"Minimum GFX clk should be less than the maximum allowed clock\n");
+			return -EINVAL;
+		}
+
+		if ((min == pstate_table->gfxclk_pstate.curr.min) &&
+		    (max == pstate_table->gfxclk_pstate.curr.max))
+			return 0;
+
+		ret = smu_v13_0_6_set_gfx_soft_freq_limited_range(smu, min, max);
+		if (!ret) {
+			pstate_table->gfxclk_pstate.curr.min = min;
+			pstate_table->gfxclk_pstate.curr.max = max;
+		}
+
+		return ret;
+	}
+
+	if (smu_dpm->dpm_level == AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM) {
+		if (!max || (max < dpm_context->dpm_tables.gfx_table.min) ||
+		    (max > dpm_context->dpm_tables.gfx_table.max)) {
+			dev_warn(
+				adev->dev,
+				"Invalid max frequency %d MHz specified for determinism\n",
+				max);
+			return -EINVAL;
+		}
+
+		/* Restore default min/max clocks and enable determinism */
+		min_clk = dpm_context->dpm_tables.gfx_table.min;
+		max_clk = dpm_context->dpm_tables.gfx_table.max;
+		ret = smu_v13_0_6_set_gfx_soft_freq_limited_range(smu, min_clk,
+								 max_clk);
+		if (!ret) {
+			usleep_range(500, 1000);
+			ret = smu_cmn_send_smc_msg_with_param(
+				smu, SMU_MSG_EnableDeterminism, max, NULL);
+			if (ret) {
+				dev_err(adev->dev,
+					"Failed to enable determinism at GFX clock %d MHz\n",
+					max);
+			} else {
+				pstate_table->gfxclk_pstate.curr.min = min_clk;
+				pstate_table->gfxclk_pstate.curr.max = max;
+			}
+		}
+	}
+
+	return ret;
+}
+
+static int smu_v13_0_6_usr_edit_dpm_table(struct smu_context *smu,
+					  enum PP_OD_DPM_TABLE_COMMAND type,
+					  long input[], uint32_t size)
+{
+	struct smu_dpm_context *smu_dpm = &(smu->smu_dpm);
+	struct smu_13_0_dpm_context *dpm_context = smu_dpm->dpm_context;
+	struct smu_umd_pstate_table *pstate_table = &smu->pstate_table;
+	uint32_t min_clk;
+	uint32_t max_clk;
+	int ret = 0;
+
+	/* Only allowed in manual or determinism mode */
+	if ((smu_dpm->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL) &&
+	    (smu_dpm->dpm_level != AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM))
+		return -EINVAL;
+
+	switch (type) {
+	case PP_OD_EDIT_SCLK_VDDC_TABLE:
+		if (size != 2) {
+			dev_err(smu->adev->dev,
+				"Input parameter number not correct\n");
+			return -EINVAL;
+		}
+
+		if (input[0] == 0) {
+			if (input[1] < dpm_context->dpm_tables.gfx_table.min) {
+				dev_warn(
+					smu->adev->dev,
+					"Minimum GFX clk (%ld) MHz specified is less than the minimum allowed (%d) MHz\n",
+					input[1],
+					dpm_context->dpm_tables.gfx_table.min);
+				pstate_table->gfxclk_pstate.custom.min =
+					pstate_table->gfxclk_pstate.curr.min;
+				return -EINVAL;
+			}
+
+			pstate_table->gfxclk_pstate.custom.min = input[1];
+		} else if (input[0] == 1) {
+			if (input[1] > dpm_context->dpm_tables.gfx_table.max) {
+				dev_warn(
+					smu->adev->dev,
+					"Maximum GFX clk (%ld) MHz specified is greater than the maximum allowed (%d) MHz\n",
+					input[1],
+					dpm_context->dpm_tables.gfx_table.max);
+				pstate_table->gfxclk_pstate.custom.max =
+					pstate_table->gfxclk_pstate.curr.max;
+				return -EINVAL;
+			}
+
+			pstate_table->gfxclk_pstate.custom.max = input[1];
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case PP_OD_RESTORE_DEFAULT_TABLE:
+		if (size != 0) {
+			dev_err(smu->adev->dev,
+				"Input parameter number not correct\n");
+			return -EINVAL;
+		} else {
+			/* Use the default frequencies for manual and determinism mode */
+			min_clk = dpm_context->dpm_tables.gfx_table.min;
+			max_clk = dpm_context->dpm_tables.gfx_table.max;
+
+			return smu_v13_0_6_set_soft_freq_limited_range(
+				smu, SMU_GFXCLK, min_clk, max_clk);
+		}
+		break;
+	case PP_OD_COMMIT_DPM_TABLE:
+		if (size != 0) {
+			dev_err(smu->adev->dev,
+				"Input parameter number not correct\n");
+			return -EINVAL;
+		} else {
+			if (!pstate_table->gfxclk_pstate.custom.min)
+				pstate_table->gfxclk_pstate.custom.min =
+					pstate_table->gfxclk_pstate.curr.min;
+
+			if (!pstate_table->gfxclk_pstate.custom.max)
+				pstate_table->gfxclk_pstate.custom.max =
+					pstate_table->gfxclk_pstate.curr.max;
+
+			min_clk = pstate_table->gfxclk_pstate.custom.min;
+			max_clk = pstate_table->gfxclk_pstate.custom.max;
+
+			return smu_v13_0_6_set_soft_freq_limited_range(
+				smu, SMU_GFXCLK, min_clk, max_clk);
+		}
+		break;
+	default:
+		return -ENOSYS;
+	}
+
+	return ret;
+}
+
+static int smu_v13_0_6_get_enabled_mask(struct smu_context *smu,
+					uint64_t *feature_mask)
+{
+	uint32_t smu_version;
+	int ret;
+
+	smu_cmn_get_smc_version(smu, NULL, &smu_version);
+	ret = smu_cmn_get_enabled_mask(smu, feature_mask);
+
+	if (ret == -EIO && smu_version < 0x552F00) {
+		*feature_mask = 0;
+		ret = 0;
+	}
+
+	return ret;
+}
+
+static bool smu_v13_0_6_is_dpm_running(struct smu_context *smu)
+{
+	int ret;
+	uint64_t feature_enabled;
+
+	ret = smu_v13_0_6_get_enabled_mask(smu, &feature_enabled);
+
+	if (ret)
+		return false;
+
+	return !!(feature_enabled & SMC_DPM_FEATURE);
+}
+
+static int smu_v13_0_6_request_i2c_xfer(struct smu_context *smu,
+					void *table_data)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *table = &smu_table->driver_table;
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t table_size;
+	int ret = 0;
+
+	if (!table_data)
+		return -EINVAL;
+
+	table_size = smu_table->tables[SMU_TABLE_I2C_COMMANDS].size;
+
+	memcpy(table->cpu_addr, table_data, table_size);
+	/* Flush hdp cache */
+	amdgpu_asic_flush_hdp(adev, NULL);
+	ret = smu_cmn_send_smc_msg(smu, SMU_MSG_RequestI2cTransaction,
+					  NULL);
+
+	return ret;
+}
+
+static int smu_v13_0_6_i2c_xfer(struct i2c_adapter *i2c_adap,
+				struct i2c_msg *msg, int num_msgs)
+{
+	struct amdgpu_smu_i2c_bus *smu_i2c = i2c_get_adapdata(i2c_adap);
+	struct amdgpu_device *adev = smu_i2c->adev;
+	struct smu_context *smu = adev->powerplay.pp_handle;
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *table = &smu_table->driver_table;
+	SwI2cRequest_t *req, *res = (SwI2cRequest_t *)table->cpu_addr;
+	int i, j, r, c;
+	u16 dir;
+
+	if (!adev->pm.dpm_enabled)
+		return -EBUSY;
+
+	req = kzalloc(sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	req->I2CcontrollerPort = smu_i2c->port;
+	req->I2CSpeed = I2C_SPEED_FAST_400K;
+	req->SlaveAddress = msg[0].addr << 1; /* wants an 8-bit address */
+	dir = msg[0].flags & I2C_M_RD;
+
+	for (c = i = 0; i < num_msgs; i++) {
+		for (j = 0; j < msg[i].len; j++, c++) {
+			SwI2cCmd_t *cmd = &req->SwI2cCmds[c];
+
+			if (!(msg[i].flags & I2C_M_RD)) {
+				/* write */
+				cmd->CmdConfig |= CMDCONFIG_READWRITE_MASK;
+				cmd->ReadWriteData = msg[i].buf[j];
+			}
+
+			if ((dir ^ msg[i].flags) & I2C_M_RD) {
+				/* The direction changes.
+				 */
+				dir = msg[i].flags & I2C_M_RD;
+				cmd->CmdConfig |= CMDCONFIG_RESTART_MASK;
+			}
+
+			req->NumCmds++;
+
+			/*
+			 * Insert STOP if we are at the last byte of either last
+			 * message for the transaction or the client explicitly
+			 * requires a STOP at this particular message.
+			 */
+			if ((j == msg[i].len - 1) &&
+			    ((i == num_msgs - 1) || (msg[i].flags & I2C_M_STOP))) {
+				cmd->CmdConfig &= ~CMDCONFIG_RESTART_MASK;
+				cmd->CmdConfig |= CMDCONFIG_STOP_MASK;
+			}
+		}
+	}
+	mutex_lock(&adev->pm.mutex);
+	r = smu_v13_0_6_request_i2c_xfer(smu, req);
+	mutex_unlock(&adev->pm.mutex);
+	if (r)
+		goto fail;
+
+	for (c = i = 0; i < num_msgs; i++) {
+		if (!(msg[i].flags & I2C_M_RD)) {
+			c += msg[i].len;
+			continue;
+		}
+		for (j = 0; j < msg[i].len; j++, c++) {
+			SwI2cCmd_t *cmd = &res->SwI2cCmds[c];
+
+			msg[i].buf[j] = cmd->ReadWriteData;
+		}
+	}
+	r = num_msgs;
+fail:
+	kfree(req);
+	return r;
+}
+
+static u32 smu_v13_0_6_i2c_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm smu_v13_0_6_i2c_algo = {
+	.master_xfer = smu_v13_0_6_i2c_xfer,
+	.functionality = smu_v13_0_6_i2c_func,
+};
+
+static const struct i2c_adapter_quirks smu_v13_0_6_i2c_control_quirks = {
+	.flags = I2C_AQ_COMB | I2C_AQ_COMB_SAME_ADDR | I2C_AQ_NO_ZERO_LEN,
+	.max_read_len = MAX_SW_I2C_COMMANDS,
+	.max_write_len = MAX_SW_I2C_COMMANDS,
+	.max_comb_1st_msg_len = 2,
+	.max_comb_2nd_msg_len = MAX_SW_I2C_COMMANDS - 2,
+};
+
+static int smu_v13_0_6_i2c_control_init(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	int res, i;
+
+	for (i = 0; i < MAX_SMU_I2C_BUSES; i++) {
+		struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i];
+		struct i2c_adapter *control = &smu_i2c->adapter;
+
+		smu_i2c->adev = adev;
+		smu_i2c->port = i;
+		mutex_init(&smu_i2c->mutex);
+		control->owner = THIS_MODULE;
+		control->class = I2C_CLASS_SPD;
+		control->dev.parent = &adev->pdev->dev;
+		control->algo = &smu_v13_0_6_i2c_algo;
+		snprintf(control->name, sizeof(control->name), "AMDGPU SMU %d", i);
+		control->quirks = &smu_v13_0_6_i2c_control_quirks;
+		i2c_set_adapdata(control, smu_i2c);
+
+		res = i2c_add_adapter(control);
+		if (res) {
+			DRM_ERROR("Failed to register hw i2c, err: %d\n", res);
+			goto Out_err;
+		}
+	}
+
+	adev->pm.ras_eeprom_i2c_bus = &adev->pm.smu_i2c[0].adapter;
+	adev->pm.fru_eeprom_i2c_bus = &adev->pm.smu_i2c[0].adapter;
+
+	return 0;
+Out_err:
+	for ( ; i >= 0; i--) {
+		struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i];
+		struct i2c_adapter *control = &smu_i2c->adapter;
+
+		i2c_del_adapter(control);
+	}
+	return res;
+}
+
+static void smu_v13_0_6_i2c_control_fini(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	int i;
+
+	for (i = 0; i < MAX_SMU_I2C_BUSES; i++) {
+		struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i];
+		struct i2c_adapter *control = &smu_i2c->adapter;
+
+		i2c_del_adapter(control);
+	}
+	adev->pm.ras_eeprom_i2c_bus = NULL;
+	adev->pm.fru_eeprom_i2c_bus = NULL;
+}
+
+static void smu_v13_0_6_get_unique_id(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	//SmuMetrics_t *metrics = smu->smu_table.metrics_table;
+	uint32_t upper32 = 0, lower32 = 0;
+	int ret;
+
+	ret = smu_cmn_get_metrics_table(smu, NULL, false);
+	if (ret)
+		goto out;
+
+	//upper32 = metrics->PublicSerialNumUpper32;
+	//lower32 = metrics->PublicSerialNumLower32;
+
+out:
+	adev->unique_id = ((uint64_t)upper32 << 32) | lower32;
+	if (adev->serial[0] == '\0')
+		sprintf(adev->serial, "%016llx", adev->unique_id);
+}
+
+static bool smu_v13_0_6_is_baco_supported(struct smu_context *smu)
+{
+	/* smu_13_0_6 does not support baco */
+
+	return false;
+}
+
+static int smu_v13_0_6_set_df_cstate(struct smu_context *smu,
+				     enum pp_df_cstate state)
+{
+	return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_DFCstateControl,
+					       state, NULL);
+}
+
+static int smu_v13_0_6_allow_xgmi_power_down(struct smu_context *smu, bool en)
+{
+	return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_GmiPwrDnControl,
+					       en ? 0 : 1, NULL);
+}
+
+static const struct throttling_logging_label {
+	uint32_t feature_mask;
+	const char *label;
+} logging_label[] = {
+	{ (1U << THROTTLER_TEMP_HBM_BIT), "HBM" },
+	{ (1U << THROTTLER_TEMP_SOC_BIT), "SOC" },
+	{ (1U << THROTTLER_TEMP_VR_GFX_BIT), "VR limit" },
+};
+static void smu_v13_0_6_log_thermal_throttling_event(struct smu_context *smu)
+{
+	int ret;
+	int throttler_idx, throtting_events = 0, buf_idx = 0;
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t throttler_status;
+	char log_buf[256];
+
+	ret = smu_v13_0_6_get_smu_metrics_data(smu, METRICS_THROTTLER_STATUS,
+					      &throttler_status);
+	if (ret)
+		return;
+
+	memset(log_buf, 0, sizeof(log_buf));
+	for (throttler_idx = 0; throttler_idx < ARRAY_SIZE(logging_label);
+	     throttler_idx++) {
+		if (throttler_status &
+		    logging_label[throttler_idx].feature_mask) {
+			throtting_events++;
+			buf_idx += snprintf(log_buf + buf_idx,
+					    sizeof(log_buf) - buf_idx, "%s%s",
+					    throtting_events > 1 ? " and " : "",
+					    logging_label[throttler_idx].label);
+			if (buf_idx >= sizeof(log_buf)) {
+				dev_err(adev->dev, "buffer overflow!\n");
+				log_buf[sizeof(log_buf) - 1] = '\0';
+				break;
+			}
+		}
+	}
+
+	dev_warn(
+		adev->dev,
+		"WARN: GPU thermal throttling temperature reached, expect performance decrease. %s.\n",
+		log_buf);
+	kgd2kfd_smi_event_throttle(
+		smu->adev->kfd.dev,
+		smu_cmn_get_indep_throttler_status(throttler_status,
+						   smu_v13_0_6_throttler_map));
+}
+
+static int smu_v13_0_6_get_current_pcie_link_speed(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t esm_ctrl;
+
+	/* TODO: confirm this on real target */
+	esm_ctrl = RREG32_PCIE(smnPCIE_ESM_CTRL);
+	if ((esm_ctrl >> 15) & 0x1FFFF)
+		return (((esm_ctrl >> 8) & 0x3F) + 128);
+
+	return smu_v13_0_get_current_pcie_link_speed(smu);
+}
+
+static ssize_t smu_v13_0_6_get_gpu_metrics(struct smu_context *smu, void **table)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct gpu_metrics_v1_3 *gpu_metrics =
+		(struct gpu_metrics_v1_3 *)smu_table->gpu_metrics_table;
+	MetricsTable_t *metrics;
+	int i, ret = 0;
+
+	metrics = kzalloc(sizeof(MetricsTable_t), GFP_KERNEL);
+	ret = smu_v13_0_6_get_metrics_table(smu, metrics, true);
+	if (ret)
+		return ret;
+
+	smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 3);
+
+	/* TODO: Decide on how to fill in zero value fields */
+	gpu_metrics->temperature_edge = 0;
+	gpu_metrics->temperature_hotspot = 0;
+	gpu_metrics->temperature_mem = 0;
+	gpu_metrics->temperature_vrgfx = 0;
+	gpu_metrics->temperature_vrsoc = 0;
+	gpu_metrics->temperature_vrmem = 0;
+
+	gpu_metrics->average_gfx_activity = 0;
+	gpu_metrics->average_umc_activity = 0;
+	gpu_metrics->average_mm_activity = 0;
+
+	gpu_metrics->average_socket_power = 0;
+	gpu_metrics->energy_accumulator = 0;
+
+	gpu_metrics->average_gfxclk_frequency = 0;
+	gpu_metrics->average_socclk_frequency = 0;
+	gpu_metrics->average_uclk_frequency = 0;
+	gpu_metrics->average_vclk0_frequency = 0;
+	gpu_metrics->average_dclk0_frequency = 0;
+
+	gpu_metrics->current_gfxclk = 0;
+	gpu_metrics->current_socclk = 0;
+	gpu_metrics->current_uclk = 0;
+	gpu_metrics->current_vclk0 = 0;
+	gpu_metrics->current_dclk0 = 0;
+
+	gpu_metrics->throttle_status = 0;
+	gpu_metrics->indep_throttle_status = smu_cmn_get_indep_throttler_status(
+		gpu_metrics->throttle_status, smu_v13_0_6_throttler_map);
+
+	gpu_metrics->current_fan_speed = 0;
+
+	gpu_metrics->pcie_link_width = 0;
+	gpu_metrics->pcie_link_speed = smu_v13_0_6_get_current_pcie_link_speed(smu);
+
+	gpu_metrics->system_clock_counter = ktime_get_boottime_ns();
+
+	gpu_metrics->gfx_activity_acc = 0;
+	gpu_metrics->mem_activity_acc = 0;
+
+	for (i = 0; i < NUM_HBM_INSTANCES; i++)
+		gpu_metrics->temperature_hbm[i] = 0;
+
+	gpu_metrics->firmware_timestamp = 0;
+
+	*table = (void *)gpu_metrics;
+	kfree(metrics);
+
+	return sizeof(struct gpu_metrics_v1_3);
+}
+
+static int smu_v13_0_6_mode2_reset(struct smu_context *smu)
+{
+	u32 smu_version;
+	int ret = 0, index;
+	struct amdgpu_device *adev = smu->adev;
+	int timeout = 10;
+
+	smu_cmn_get_smc_version(smu, NULL, &smu_version);
+
+	index = smu_cmn_to_asic_specific_index(smu, CMN2ASIC_MAPPING_MSG,
+					       SMU_MSG_GfxDeviceDriverReset);
+
+	mutex_lock(&smu->message_lock);
+	ret = smu_cmn_send_msg_without_waiting(smu, (uint16_t)index,
+					       SMU_RESET_MODE_2);
+	/* This is similar to FLR, wait till max FLR timeout */
+	msleep(100);
+	dev_dbg(smu->adev->dev, "restore config space...\n");
+	/* Restore the config space saved during init */
+	amdgpu_device_load_pci_state(adev->pdev);
+
+	dev_dbg(smu->adev->dev, "wait for reset ack\n");
+	while (ret == -ETIME && timeout) {
+		ret = smu_cmn_wait_for_response(smu);
+		/* Wait a bit more time for getting ACK */
+		if (ret == -ETIME) {
+			--timeout;
+			usleep_range(500, 1000);
+			continue;
+		}
+
+		if (ret != 1) {
+			dev_err(adev->dev,
+				"failed to send mode2 message \tparam: 0x%08x response %#x\n",
+				SMU_RESET_MODE_2, ret);
+			goto out;
+		}
+	}
+
+	if (ret == 1)
+		ret = 0;
+out:
+	mutex_unlock(&smu->message_lock);
+
+	return ret;
+}
+
+static int smu_v13_0_6_mode1_reset(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	struct amdgpu_ras *ras;
+	u32 fatal_err, param;
+	int ret = 0;
+
+	ras = amdgpu_ras_get_context(adev);
+	fatal_err = 0;
+	param = SMU_RESET_MODE_1;
+
+	/* fatal error triggered by ras, PMFW supports the flag */
+	if (ras && atomic_read(&ras->in_recovery))
+		fatal_err = 1;
+
+	param |= (fatal_err << 16);
+	ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_GfxDeviceDriverReset,
+					      param, NULL);
+
+	if (!ret)
+		msleep(SMU13_MODE1_RESET_WAIT_TIME_IN_MS);
+
+	return ret;
+}
+
+static bool smu_v13_0_6_is_mode1_reset_supported(struct smu_context *smu)
+{
+	/* TODO: Enable this when FW support is added */
+	return false;
+}
+
+static bool smu_v13_0_6_is_mode2_reset_supported(struct smu_context *smu)
+{
+	return true;
+}
+
+static int smu_v13_0_6_smu_send_hbm_bad_page_num(struct smu_context *smu,
+						 uint32_t size)
+{
+	int ret = 0;
+
+	/* message SMU to update the bad page number on SMUBUS */
+	ret = smu_cmn_send_smc_msg_with_param(
+		smu, SMU_MSG_SetNumBadHbmPagesRetired, size, NULL);
+	if (ret)
+		dev_err(smu->adev->dev,
+			"[%s] failed to message SMU to update HBM bad pages number\n",
+			__func__);
+
+	return ret;
+}
+
+static const struct pptable_funcs smu_v13_0_6_ppt_funcs = {
+	/* init dpm */
+	.get_allowed_feature_mask = smu_v13_0_6_get_allowed_feature_mask,
+	/* dpm/clk tables */
+	.set_default_dpm_table = smu_v13_0_6_set_default_dpm_table,
+	.populate_umd_state_clk = smu_v13_0_6_populate_umd_state_clk,
+	.print_clk_levels = smu_v13_0_6_print_clk_levels,
+	.force_clk_levels = smu_v13_0_6_force_clk_levels,
+	.read_sensor = smu_v13_0_6_read_sensor,
+	.set_performance_level = smu_v13_0_6_set_performance_level,
+	.get_power_limit = smu_v13_0_6_get_power_limit,
+	.is_dpm_running = smu_v13_0_6_is_dpm_running,
+	.get_unique_id = smu_v13_0_6_get_unique_id,
+	.init_smc_tables = smu_v13_0_6_init_smc_tables,
+	.fini_smc_tables = smu_v13_0_fini_smc_tables,
+	.init_power = smu_v13_0_init_power,
+	.fini_power = smu_v13_0_fini_power,
+	.check_fw_status = smu_v13_0_6_check_fw_status,
+	/* pptable related */
+	.check_fw_version = smu_v13_0_check_fw_version,
+	.set_driver_table_location = smu_v13_0_set_driver_table_location,
+	.set_tool_table_location = smu_v13_0_set_tool_table_location,
+	.notify_memory_pool_location = smu_v13_0_notify_memory_pool_location,
+	.system_features_control = smu_v13_0_6_system_features_control,
+	.send_smc_msg_with_param = smu_cmn_send_smc_msg_with_param,
+	.send_smc_msg = smu_cmn_send_smc_msg,
+	.get_enabled_mask = smu_v13_0_6_get_enabled_mask,
+	.feature_is_enabled = smu_cmn_feature_is_enabled,
+	.set_power_limit = smu_v13_0_6_set_power_limit,
+	.set_xgmi_pstate = smu_v13_0_set_xgmi_pstate,
+	/* TODO: Thermal limits unknown, skip these for now
+	.register_irq_handler = smu_v13_0_register_irq_handler,
+	.enable_thermal_alert = smu_v13_0_enable_thermal_alert,
+	.disable_thermal_alert = smu_v13_0_disable_thermal_alert,
+	*/
+	.setup_pptable = smu_v13_0_6_setup_pptable,
+	.baco_is_support = smu_v13_0_6_is_baco_supported,
+	.get_dpm_ultimate_freq = smu_v13_0_6_get_dpm_ultimate_freq,
+	.set_soft_freq_limited_range = smu_v13_0_6_set_soft_freq_limited_range,
+	.od_edit_dpm_table = smu_v13_0_6_usr_edit_dpm_table,
+	.set_df_cstate = smu_v13_0_6_set_df_cstate,
+	.allow_xgmi_power_down = smu_v13_0_6_allow_xgmi_power_down,
+	.log_thermal_throttling_event = smu_v13_0_6_log_thermal_throttling_event,
+	.get_pp_feature_mask = smu_cmn_get_pp_feature_mask,
+	.set_pp_feature_mask = smu_cmn_set_pp_feature_mask,
+	.get_gpu_metrics = smu_v13_0_6_get_gpu_metrics,
+	.mode1_reset_is_support = smu_v13_0_6_is_mode1_reset_supported,
+	.mode2_reset_is_support = smu_v13_0_6_is_mode2_reset_supported,
+	.mode1_reset = smu_v13_0_6_mode1_reset,
+	.mode2_reset = smu_v13_0_6_mode2_reset,
+	.wait_for_event = smu_v13_0_wait_for_event,
+	.i2c_init = smu_v13_0_6_i2c_control_init,
+	.i2c_fini = smu_v13_0_6_i2c_control_fini,
+	.send_hbm_bad_pages_num = smu_v13_0_6_smu_send_hbm_bad_page_num,
+};
+
+void smu_v13_0_6_set_ppt_funcs(struct smu_context *smu)
+{
+	smu->ppt_funcs = &smu_v13_0_6_ppt_funcs;
+	smu->message_map = smu_v13_0_6_message_map;
+	smu->clock_map = smu_v13_0_6_clk_map;
+	smu->feature_map = smu_v13_0_6_feature_mask_map;
+	smu->table_map = smu_v13_0_6_table_map;
+	smu_v13_0_set_smu_mailbox_registers(smu);
+}
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
new file mode 100644
index 000000000000..f0fa42a645c0
--- /dev/null
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SMU_13_0_6_PPT_H__
+#define __SMU_13_0_6_PPT_H__
+
+#define SMU_13_0_6_UMD_PSTATE_GFXCLK_LEVEL 0x2
+#define SMU_13_0_6_UMD_PSTATE_SOCCLK_LEVEL 0x4
+#define SMU_13_0_6_UMD_PSTATE_MCLK_LEVEL 0x2
+
+extern void smu_v13_0_6_set_ppt_funcs(struct smu_context *smu);
+
+#endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
index 4399416dd9b8..3d9ff46706fb 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
@@ -1546,7 +1546,9 @@ static int smu_v13_0_7_get_power_profile_mode(struct smu_context *smu, char *buf
 		workload_type = smu_cmn_to_asic_specific_index(smu,
 							       CMN2ASIC_MAPPING_WORKLOAD,
 							       i);
-		if (workload_type < 0) {
+		if (workload_type == -ENOTSUPP)
+			continue;
+		else if (workload_type < 0) {
 			result = -EINVAL;
 			goto out;
 		}
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
index d5abafc5a682..3ecb900e6ecd 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
@@ -478,13 +478,13 @@ int smu_cmn_to_asic_specific_index(struct smu_context *smu,
 		return mapping.map_to;
 
 	case CMN2ASIC_MAPPING_WORKLOAD:
-		if (index > PP_SMC_POWER_PROFILE_WINDOW3D ||
+		if (index >= PP_SMC_POWER_PROFILE_COUNT ||
 		    !smu->workload_map)
 			return -EINVAL;
 
 		mapping = smu->workload_map[index];
 		if (!mapping.valid_mapping)
-			return -EINVAL;
+			return -ENOTSUPP;
 
 		return mapping.map_to;
 
diff --git a/drivers/gpu/drm/arm/hdlcd_drv.c b/drivers/gpu/drm/arm/hdlcd_drv.c
index e3507dd6f82a..9020bf820bc8 100644
--- a/drivers/gpu/drm/arm/hdlcd_drv.c
+++ b/drivers/gpu/drm/arm/hdlcd_drv.c
@@ -26,7 +26,7 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_debugfs.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper.h>
@@ -100,7 +100,6 @@ static int hdlcd_load(struct drm_device *drm, unsigned long flags)
 {
 	struct hdlcd_drm_private *hdlcd = drm_to_hdlcd_priv(drm);
 	struct platform_device *pdev = to_platform_device(drm->dev);
-	struct resource *res;
 	u32 version;
 	int ret;
 
@@ -115,8 +114,7 @@ static int hdlcd_load(struct drm_device *drm, unsigned long flags)
 	atomic_set(&hdlcd->dma_end_count, 0);
 #endif
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	hdlcd->mmio = devm_ioremap_resource(drm->dev, res);
+	hdlcd->mmio = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(hdlcd->mmio)) {
 		DRM_ERROR("failed to map control registers area\n");
 		ret = PTR_ERR(hdlcd->mmio);
@@ -301,7 +299,7 @@ static int hdlcd_drm_bind(struct device *dev)
 	if (ret)
 		goto err_register;
 
-	drm_fbdev_generic_setup(drm, 32);
+	drm_fbdev_dma_setup(drm, 32);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c
index 589c1c66a6dc..c03cfd57b752 100644
--- a/drivers/gpu/drm/arm/malidp_drv.c
+++ b/drivers/gpu/drm/arm/malidp_drv.c
@@ -19,7 +19,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -649,7 +649,7 @@ static ssize_t core_id_show(struct device *dev, struct device_attribute *attr,
 	struct drm_device *drm = dev_get_drvdata(dev);
 	struct malidp_drm *malidp = drm_to_malidp(drm);
 
-	return snprintf(buf, PAGE_SIZE, "%08x\n", malidp->core_id);
+	return sysfs_emit(buf, "%08x\n", malidp->core_id);
 }
 
 static DEVICE_ATTR_RO(core_id);
@@ -724,8 +724,7 @@ static int malidp_bind(struct device *dev)
 	hwdev->hw = (struct malidp_hw *)of_device_get_match_data(dev);
 	malidp->dev = hwdev;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	hwdev->regs = devm_ioremap_resource(dev, res);
+	hwdev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(hwdev->regs))
 		return PTR_ERR(hwdev->regs);
 
@@ -852,7 +851,7 @@ static int malidp_bind(struct device *dev)
 	if (ret)
 		goto register_fail;
 
-	drm_fbdev_generic_setup(drm, 32);
+	drm_fbdev_dma_setup(drm, 32);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
index ecfb060d2557..c8c7f8215155 100644
--- a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
+++ b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
@@ -15,7 +15,7 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
@@ -341,7 +341,7 @@ static int aspeed_gfx_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_unload;
 
-	drm_fbdev_generic_setup(&priv->drm, 32);
+	drm_fbdev_dma_setup(&priv->drm, 32);
 	return 0;
 
 err_unload:
diff --git a/drivers/gpu/drm/ast/ast_dp.c b/drivers/gpu/drm/ast/ast_dp.c
index 56483860306b..fbb070f63e36 100644
--- a/drivers/gpu/drm/ast/ast_dp.c
+++ b/drivers/gpu/drm/ast/ast_dp.c
@@ -9,7 +9,7 @@
 
 int ast_astdp_read_edid(struct drm_device *dev, u8 *ediddata)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u8 i = 0, j = 0;
 
 	/*
@@ -125,7 +125,7 @@ void ast_dp_launch(struct drm_device *dev, u8 bPower)
 	u8 bDPTX = 0;
 	u8 bDPExecute = 1;
 
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	// S3 come back, need more time to wait BMC ready.
 	if (bPower)
 		WaitCount = 300;
@@ -172,7 +172,7 @@ void ast_dp_launch(struct drm_device *dev, u8 bPower)
 
 void ast_dp_power_on_off(struct drm_device *dev, bool on)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	// Read and Turn off DP PHY sleep
 	u8 bE3 = ast_get_index_reg_mask(ast, AST_IO_CRTC_PORT, 0xE3, AST_DP_VIDEO_ENABLE);
 
@@ -188,7 +188,7 @@ void ast_dp_power_on_off(struct drm_device *dev, bool on)
 
 void ast_dp_set_on_off(struct drm_device *dev, bool on)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u8 video_on_off = on;
 
 	// Video On/Off
@@ -208,7 +208,7 @@ void ast_dp_set_on_off(struct drm_device *dev, bool on)
 
 void ast_dp_set_mode(struct drm_crtc *crtc, struct ast_vbios_mode_info *vbios_mode)
 {
-	struct ast_private *ast = to_ast_private(crtc->dev);
+	struct ast_device *ast = to_ast_device(crtc->dev);
 
 	u32 ulRefreshRateIndex;
 	u8 ModeIdx;
diff --git a/drivers/gpu/drm/ast/ast_dp501.c b/drivers/gpu/drm/ast/ast_dp501.c
index 4f75a9efb610..1bc35a992369 100644
--- a/drivers/gpu/drm/ast/ast_dp501.c
+++ b/drivers/gpu/drm/ast/ast_dp501.c
@@ -10,7 +10,7 @@ MODULE_FIRMWARE("ast_dp501_fw.bin");
 
 static void ast_release_firmware(void *data)
 {
-	struct ast_private *ast = data;
+	struct ast_device *ast = data;
 
 	release_firmware(ast->dp501_fw);
 	ast->dp501_fw = NULL;
@@ -18,7 +18,7 @@ static void ast_release_firmware(void *data)
 
 static int ast_load_dp501_microcode(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	int ret;
 
 	ret = request_firmware(&ast->dp501_fw, "ast_dp501_fw.bin", dev->dev);
@@ -28,7 +28,7 @@ static int ast_load_dp501_microcode(struct drm_device *dev)
 	return devm_add_action_or_reset(dev->dev, ast_release_firmware, ast);
 }
 
-static void send_ack(struct ast_private *ast)
+static void send_ack(struct ast_device *ast)
 {
 	u8 sendack;
 	sendack = ast_get_index_reg_mask(ast, AST_IO_CRTC_PORT, 0x9b, 0xff);
@@ -36,7 +36,7 @@ static void send_ack(struct ast_private *ast)
 	ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0x9b, 0x00, sendack);
 }
 
-static void send_nack(struct ast_private *ast)
+static void send_nack(struct ast_device *ast)
 {
 	u8 sendack;
 	sendack = ast_get_index_reg_mask(ast, AST_IO_CRTC_PORT, 0x9b, 0xff);
@@ -44,7 +44,7 @@ static void send_nack(struct ast_private *ast)
 	ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0x9b, 0x00, sendack);
 }
 
-static bool wait_ack(struct ast_private *ast)
+static bool wait_ack(struct ast_device *ast)
 {
 	u8 waitack;
 	u32 retry = 0;
@@ -60,7 +60,7 @@ static bool wait_ack(struct ast_private *ast)
 		return false;
 }
 
-static bool wait_nack(struct ast_private *ast)
+static bool wait_nack(struct ast_device *ast)
 {
 	u8 waitack;
 	u32 retry = 0;
@@ -76,18 +76,18 @@ static bool wait_nack(struct ast_private *ast)
 		return false;
 }
 
-static void set_cmd_trigger(struct ast_private *ast)
+static void set_cmd_trigger(struct ast_device *ast)
 {
 	ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0x9b, ~0x40, 0x40);
 }
 
-static void clear_cmd_trigger(struct ast_private *ast)
+static void clear_cmd_trigger(struct ast_device *ast)
 {
 	ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0x9b, ~0x40, 0x00);
 }
 
 #if 0
-static bool wait_fw_ready(struct ast_private *ast)
+static bool wait_fw_ready(struct ast_device *ast)
 {
 	u8 waitready;
 	u32 retry = 0;
@@ -106,7 +106,7 @@ static bool wait_fw_ready(struct ast_private *ast)
 
 static bool ast_write_cmd(struct drm_device *dev, u8 data)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	int retry = 0;
 	if (wait_nack(ast)) {
 		send_nack(ast);
@@ -128,7 +128,7 @@ static bool ast_write_cmd(struct drm_device *dev, u8 data)
 
 static bool ast_write_data(struct drm_device *dev, u8 data)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 
 	if (wait_nack(ast)) {
 		send_nack(ast);
@@ -146,7 +146,7 @@ static bool ast_write_data(struct drm_device *dev, u8 data)
 #if 0
 static bool ast_read_data(struct drm_device *dev, u8 *data)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u8 tmp;
 
 	*data = 0;
@@ -163,7 +163,7 @@ static bool ast_read_data(struct drm_device *dev, u8 *data)
 	return true;
 }
 
-static void clear_cmd(struct ast_private *ast)
+static void clear_cmd(struct ast_device *ast)
 {
 	send_nack(ast);
 	ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0x9a, 0x00, 0x00);
@@ -178,14 +178,14 @@ void ast_set_dp501_video_output(struct drm_device *dev, u8 mode)
 	msleep(10);
 }
 
-static u32 get_fw_base(struct ast_private *ast)
+static u32 get_fw_base(struct ast_device *ast)
 {
 	return ast_mindwm(ast, 0x1e6e2104) & 0x7fffffff;
 }
 
 bool ast_backup_fw(struct drm_device *dev, u8 *addr, u32 size)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u32 i, data;
 	u32 boot_address;
 
@@ -204,7 +204,7 @@ bool ast_backup_fw(struct drm_device *dev, u8 *addr, u32 size)
 
 static bool ast_launch_m68k(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u32 i, data, len = 0;
 	u32 boot_address;
 	u8 *fw_addr = NULL;
@@ -274,7 +274,7 @@ static bool ast_launch_m68k(struct drm_device *dev)
 
 bool ast_dp501_read_edid(struct drm_device *dev, u8 *ediddata)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u32 i, boot_address, offset, data;
 	u32 *pEDIDidx;
 
@@ -334,7 +334,7 @@ bool ast_dp501_read_edid(struct drm_device *dev, u8 *ediddata)
 
 static bool ast_init_dvo(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u8 jreg;
 	u32 data;
 	ast_write32(ast, 0xf004, 0x1e6e0000);
@@ -407,7 +407,7 @@ static bool ast_init_dvo(struct drm_device *dev)
 
 static void ast_init_analog(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u32 data;
 
 	/*
@@ -434,7 +434,7 @@ static void ast_init_analog(struct drm_device *dev)
 
 void ast_init_3rdtx(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u8 jreg;
 
 	if (ast->chip == AST2300 || ast->chip == AST2400) {
diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c
index d78852c7cf5b..e1224ef4ad83 100644
--- a/drivers/gpu/drm/ast/ast_drv.c
+++ b/drivers/gpu/drm/ast/ast_drv.c
@@ -89,27 +89,13 @@ static const struct pci_device_id ast_pciidlist[] = {
 
 MODULE_DEVICE_TABLE(pci, ast_pciidlist);
 
-static int ast_remove_conflicting_framebuffers(struct pci_dev *pdev)
-{
-	bool primary = false;
-	resource_size_t base, size;
-
-	base = pci_resource_start(pdev, 0);
-	size = pci_resource_len(pdev, 0);
-#ifdef CONFIG_X86
-	primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
-#endif
-
-	return drm_aperture_remove_conflicting_framebuffers(base, size, primary, &ast_driver);
-}
-
 static int ast_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
-	struct ast_private *ast;
+	struct ast_device *ast;
 	struct drm_device *dev;
 	int ret;
 
-	ret = ast_remove_conflicting_framebuffers(pdev);
+	ret = drm_aperture_remove_conflicting_pci_framebuffers(pdev, &ast_driver);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h
index d51b81fea9c8..a501169cddad 100644
--- a/drivers/gpu/drm/ast/ast_drv.h
+++ b/drivers/gpu/drm/ast/ast_drv.h
@@ -157,7 +157,7 @@ to_ast_sil164_connector(struct drm_connector *connector)
  * Device
  */
 
-struct ast_private {
+struct ast_device {
 	struct drm_device base;
 
 	struct mutex ioregs_lock; /* Protects access to I/O registers in ioregs */
@@ -210,14 +210,14 @@ struct ast_private {
 	const struct firmware *dp501_fw;	/* dp501 fw */
 };
 
-static inline struct ast_private *to_ast_private(struct drm_device *dev)
+static inline struct ast_device *to_ast_device(struct drm_device *dev)
 {
-	return container_of(dev, struct ast_private, base);
+	return container_of(dev, struct ast_device, base);
 }
 
-struct ast_private *ast_device_create(const struct drm_driver *drv,
-				      struct pci_dev *pdev,
-				      unsigned long flags);
+struct ast_device *ast_device_create(const struct drm_driver *drv,
+				     struct pci_dev *pdev,
+				     unsigned long flags);
 
 #define AST_IO_AR_PORT_WRITE		(0x40)
 #define AST_IO_MISC_PORT_WRITE		(0x42)
@@ -238,62 +238,44 @@ struct ast_private *ast_device_create(const struct drm_driver *drv,
 #define AST_IO_VGACRCB_HWC_ENABLED     BIT(1)
 #define AST_IO_VGACRCB_HWC_16BPP       BIT(0) /* set: ARGB4444, cleared: 2bpp palette */
 
-#define __ast_read(x) \
-static inline u##x ast_read##x(struct ast_private *ast, u32 reg) { \
-u##x val = 0;\
-val = ioread##x(ast->regs + reg); \
-return val;\
+static inline u32 ast_read32(struct ast_device *ast, u32 reg)
+{
+	return ioread32(ast->regs + reg);
 }
 
-__ast_read(8);
-__ast_read(16);
-__ast_read(32)
-
-#define __ast_io_read(x) \
-static inline u##x ast_io_read##x(struct ast_private *ast, u32 reg) { \
-u##x val = 0;\
-val = ioread##x(ast->ioregs + reg); \
-return val;\
+static inline void ast_write32(struct ast_device *ast, u32 reg, u32 val)
+{
+	iowrite32(val, ast->regs + reg);
 }
 
-__ast_io_read(8);
-__ast_io_read(16);
-__ast_io_read(32);
-
-#define __ast_write(x) \
-static inline void ast_write##x(struct ast_private *ast, u32 reg, u##x val) {\
-	iowrite##x(val, ast->regs + reg);\
-	}
-
-__ast_write(8);
-__ast_write(16);
-__ast_write(32);
-
-#define __ast_io_write(x) \
-static inline void ast_io_write##x(struct ast_private *ast, u32 reg, u##x val) {\
-	iowrite##x(val, ast->ioregs + reg);\
-	}
+static inline u8 ast_io_read8(struct ast_device *ast, u32 reg)
+{
+	return ioread8(ast->ioregs + reg);
+}
 
-__ast_io_write(8);
-__ast_io_write(16);
-#undef __ast_io_write
+static inline void ast_io_write8(struct ast_device *ast, u32 reg, u8 val)
+{
+	iowrite8(val, ast->ioregs + reg);
+}
 
-static inline void ast_set_index_reg(struct ast_private *ast,
+static inline void ast_set_index_reg(struct ast_device *ast,
 				     uint32_t base, uint8_t index,
 				     uint8_t val)
 {
-	ast_io_write16(ast, base, ((u16)val << 8) | index);
+	ast_io_write8(ast, base, index);
+	++base;
+	ast_io_write8(ast, base, val);
 }
 
-void ast_set_index_reg_mask(struct ast_private *ast,
+void ast_set_index_reg_mask(struct ast_device *ast,
 			    uint32_t base, uint8_t index,
 			    uint8_t mask, uint8_t val);
-uint8_t ast_get_index_reg(struct ast_private *ast,
+uint8_t ast_get_index_reg(struct ast_device *ast,
 			  uint32_t base, uint8_t index);
-uint8_t ast_get_index_reg_mask(struct ast_private *ast,
+uint8_t ast_get_index_reg_mask(struct ast_device *ast,
 			       uint32_t base, uint8_t index, uint8_t mask);
 
-static inline void ast_open_key(struct ast_private *ast)
+static inline void ast_open_key(struct ast_device *ast)
 {
 	ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0x80, 0xA8);
 }
@@ -352,7 +334,7 @@ struct ast_crtc_state {
 
 #define to_ast_crtc_state(state) container_of(state, struct ast_crtc_state, base)
 
-int ast_mode_config_init(struct ast_private *ast);
+int ast_mode_config_init(struct ast_device *ast);
 
 #define AST_MM_ALIGN_SHIFT 4
 #define AST_MM_ALIGN_MASK ((1 << AST_MM_ALIGN_SHIFT) - 1)
@@ -476,16 +458,16 @@ int ast_mode_config_init(struct ast_private *ast);
 #define ASTDP_1366x768_60		0x1E
 #define ASTDP_1152x864_75		0x1F
 
-int ast_mm_init(struct ast_private *ast);
+int ast_mm_init(struct ast_device *ast);
 
 /* ast post */
 void ast_enable_vga(struct drm_device *dev);
 void ast_enable_mmio(struct drm_device *dev);
 bool ast_is_vga_enabled(struct drm_device *dev);
 void ast_post_gpu(struct drm_device *dev);
-u32 ast_mindwm(struct ast_private *ast, u32 r);
-void ast_moutdwm(struct ast_private *ast, u32 r, u32 v);
-void ast_patch_ahb_2500(struct ast_private *ast);
+u32 ast_mindwm(struct ast_device *ast, u32 r);
+void ast_moutdwm(struct ast_device *ast, u32 r, u32 v);
+void ast_patch_ahb_2500(struct ast_device *ast);
 /* ast dp501 */
 void ast_set_dp501_video_output(struct drm_device *dev, u8 mode);
 bool ast_backup_fw(struct drm_device *dev, u8 *addr, u32 size);
diff --git a/drivers/gpu/drm/ast/ast_i2c.c b/drivers/gpu/drm/ast/ast_i2c.c
index 93e91c36d649..d64045c0b849 100644
--- a/drivers/gpu/drm/ast/ast_i2c.c
+++ b/drivers/gpu/drm/ast/ast_i2c.c
@@ -29,7 +29,7 @@
 static void ast_i2c_setsda(void *i2c_priv, int data)
 {
 	struct ast_i2c_chan *i2c = i2c_priv;
-	struct ast_private *ast = to_ast_private(i2c->dev);
+	struct ast_device *ast = to_ast_device(i2c->dev);
 	int i;
 	u8 ujcrb7, jtemp;
 
@@ -45,7 +45,7 @@ static void ast_i2c_setsda(void *i2c_priv, int data)
 static void ast_i2c_setscl(void *i2c_priv, int clock)
 {
 	struct ast_i2c_chan *i2c = i2c_priv;
-	struct ast_private *ast = to_ast_private(i2c->dev);
+	struct ast_device *ast = to_ast_device(i2c->dev);
 	int i;
 	u8 ujcrb7, jtemp;
 
@@ -61,7 +61,7 @@ static void ast_i2c_setscl(void *i2c_priv, int clock)
 static int ast_i2c_getsda(void *i2c_priv)
 {
 	struct ast_i2c_chan *i2c = i2c_priv;
-	struct ast_private *ast = to_ast_private(i2c->dev);
+	struct ast_device *ast = to_ast_device(i2c->dev);
 	uint32_t val, val2, count, pass;
 
 	count = 0;
@@ -83,7 +83,7 @@ static int ast_i2c_getsda(void *i2c_priv)
 static int ast_i2c_getscl(void *i2c_priv)
 {
 	struct ast_i2c_chan *i2c = i2c_priv;
-	struct ast_private *ast = to_ast_private(i2c->dev);
+	struct ast_device *ast = to_ast_device(i2c->dev);
 	uint32_t val, val2, count, pass;
 
 	count = 0;
diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c
index f83ce77127cb..794ffd4a29c5 100644
--- a/drivers/gpu/drm/ast/ast_main.c
+++ b/drivers/gpu/drm/ast/ast_main.c
@@ -35,7 +35,7 @@
 
 #include "ast_drv.h"
 
-void ast_set_index_reg_mask(struct ast_private *ast,
+void ast_set_index_reg_mask(struct ast_device *ast,
 			    uint32_t base, uint8_t index,
 			    uint8_t mask, uint8_t val)
 {
@@ -45,7 +45,7 @@ void ast_set_index_reg_mask(struct ast_private *ast,
 	ast_set_index_reg(ast, base, index, tmp);
 }
 
-uint8_t ast_get_index_reg(struct ast_private *ast,
+uint8_t ast_get_index_reg(struct ast_device *ast,
 			  uint32_t base, uint8_t index)
 {
 	uint8_t ret;
@@ -54,7 +54,7 @@ uint8_t ast_get_index_reg(struct ast_private *ast,
 	return ret;
 }
 
-uint8_t ast_get_index_reg_mask(struct ast_private *ast,
+uint8_t ast_get_index_reg_mask(struct ast_device *ast,
 			       uint32_t base, uint8_t index, uint8_t mask)
 {
 	uint8_t ret;
@@ -66,7 +66,7 @@ uint8_t ast_get_index_reg_mask(struct ast_private *ast,
 static void ast_detect_config_mode(struct drm_device *dev, u32 *scu_rev)
 {
 	struct device_node *np = dev->dev->of_node;
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
 	uint32_t data, jregd0, jregd1;
 
@@ -122,7 +122,7 @@ static void ast_detect_config_mode(struct drm_device *dev, u32 *scu_rev)
 
 static int ast_detect_chip(struct drm_device *dev, bool *need_post)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
 	uint32_t jreg, scu_rev;
 
@@ -271,7 +271,7 @@ static int ast_detect_chip(struct drm_device *dev, bool *need_post)
 static int ast_get_dram_info(struct drm_device *dev)
 {
 	struct device_node *np = dev->dev->of_node;
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	uint32_t mcr_cfg, mcr_scu_mpll, mcr_scu_strap;
 	uint32_t denum, num, div, ref_pll, dsel;
 
@@ -394,22 +394,22 @@ static int ast_get_dram_info(struct drm_device *dev)
  */
 static void ast_device_release(void *data)
 {
-	struct ast_private *ast = data;
+	struct ast_device *ast = data;
 
 	/* enable standard VGA decode */
 	ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0xa1, 0x04);
 }
 
-struct ast_private *ast_device_create(const struct drm_driver *drv,
-				      struct pci_dev *pdev,
-				      unsigned long flags)
+struct ast_device *ast_device_create(const struct drm_driver *drv,
+				     struct pci_dev *pdev,
+				     unsigned long flags)
 {
 	struct drm_device *dev;
-	struct ast_private *ast;
+	struct ast_device *ast;
 	bool need_post;
 	int ret = 0;
 
-	ast = devm_drm_dev_alloc(&pdev->dev, drv, struct ast_private, base);
+	ast = devm_drm_dev_alloc(&pdev->dev, drv, struct ast_device, base);
 	if (IS_ERR(ast))
 		return ast;
 	dev = &ast->base;
diff --git a/drivers/gpu/drm/ast/ast_mm.c b/drivers/gpu/drm/ast/ast_mm.c
index 248284a4b3ff..e16af60deef9 100644
--- a/drivers/gpu/drm/ast/ast_mm.c
+++ b/drivers/gpu/drm/ast/ast_mm.c
@@ -33,7 +33,7 @@
 
 #include "ast_drv.h"
 
-static u32 ast_get_vram_size(struct ast_private *ast)
+static u32 ast_get_vram_size(struct ast_device *ast)
 {
 	u8 jreg;
 	u32 vram_size;
@@ -73,7 +73,7 @@ static u32 ast_get_vram_size(struct ast_private *ast)
 	return vram_size;
 }
 
-int ast_mm_init(struct ast_private *ast)
+int ast_mm_init(struct ast_device *ast)
 {
 	struct drm_device *dev = &ast->base;
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index 984ec590a7e7..36374828f6c8 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -51,7 +51,7 @@
 
 #define AST_LUT_SIZE 256
 
-static inline void ast_load_palette_index(struct ast_private *ast,
+static inline void ast_load_palette_index(struct ast_device *ast,
 				     u8 index, u8 red, u8 green,
 				     u8 blue)
 {
@@ -65,7 +65,7 @@ static inline void ast_load_palette_index(struct ast_private *ast,
 	ast_io_read8(ast, AST_IO_SEQ_PORT);
 }
 
-static void ast_crtc_set_gamma_linear(struct ast_private *ast,
+static void ast_crtc_set_gamma_linear(struct ast_device *ast,
 				      const struct drm_format_info *format)
 {
 	int i;
@@ -84,7 +84,7 @@ static void ast_crtc_set_gamma_linear(struct ast_private *ast,
 	}
 }
 
-static void ast_crtc_set_gamma(struct ast_private *ast,
+static void ast_crtc_set_gamma(struct ast_device *ast,
 			       const struct drm_format_info *format,
 			       struct drm_color_lut *lut)
 {
@@ -232,7 +232,7 @@ static bool ast_get_vbios_mode_info(const struct drm_format_info *format,
 	return true;
 }
 
-static void ast_set_vbios_color_reg(struct ast_private *ast,
+static void ast_set_vbios_color_reg(struct ast_device *ast,
 				    const struct drm_format_info *format,
 				    const struct ast_vbios_mode_info *vbios_mode)
 {
@@ -263,7 +263,7 @@ static void ast_set_vbios_color_reg(struct ast_private *ast,
 	}
 }
 
-static void ast_set_vbios_mode_reg(struct ast_private *ast,
+static void ast_set_vbios_mode_reg(struct ast_device *ast,
 				   const struct drm_display_mode *adjusted_mode,
 				   const struct ast_vbios_mode_info *vbios_mode)
 {
@@ -287,7 +287,7 @@ static void ast_set_vbios_mode_reg(struct ast_private *ast,
 	}
 }
 
-static void ast_set_std_reg(struct ast_private *ast,
+static void ast_set_std_reg(struct ast_device *ast,
 			    struct drm_display_mode *mode,
 			    struct ast_vbios_mode_info *vbios_mode)
 {
@@ -335,7 +335,7 @@ static void ast_set_std_reg(struct ast_private *ast,
 		ast_set_index_reg(ast, AST_IO_GR_PORT, i, stdtable->gr[i]);
 }
 
-static void ast_set_crtc_reg(struct ast_private *ast,
+static void ast_set_crtc_reg(struct ast_device *ast,
 			     struct drm_display_mode *mode,
 			     struct ast_vbios_mode_info *vbios_mode)
 {
@@ -450,7 +450,7 @@ static void ast_set_crtc_reg(struct ast_private *ast,
 	ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0x11, 0x7f, 0x80);
 }
 
-static void ast_set_offset_reg(struct ast_private *ast,
+static void ast_set_offset_reg(struct ast_device *ast,
 			       struct drm_framebuffer *fb)
 {
 	u16 offset;
@@ -460,7 +460,7 @@ static void ast_set_offset_reg(struct ast_private *ast,
 	ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0xb0, (offset >> 8) & 0x3f);
 }
 
-static void ast_set_dclk_reg(struct ast_private *ast,
+static void ast_set_dclk_reg(struct ast_device *ast,
 			     struct drm_display_mode *mode,
 			     struct ast_vbios_mode_info *vbios_mode)
 {
@@ -478,7 +478,7 @@ static void ast_set_dclk_reg(struct ast_private *ast,
 			       ((clk_info->param3 & 0x3) << 4));
 }
 
-static void ast_set_color_reg(struct ast_private *ast,
+static void ast_set_color_reg(struct ast_device *ast,
 			      const struct drm_format_info *format)
 {
 	u8 jregA0 = 0, jregA3 = 0, jregA8 = 0;
@@ -507,7 +507,7 @@ static void ast_set_color_reg(struct ast_private *ast,
 	ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0xa8, 0xfd, jregA8);
 }
 
-static void ast_set_crtthd_reg(struct ast_private *ast)
+static void ast_set_crtthd_reg(struct ast_device *ast)
 {
 	/* Set Threshold */
 	if (ast->chip == AST2600) {
@@ -529,7 +529,7 @@ static void ast_set_crtthd_reg(struct ast_private *ast)
 	}
 }
 
-static void ast_set_sync_reg(struct ast_private *ast,
+static void ast_set_sync_reg(struct ast_device *ast,
 			     struct drm_display_mode *mode,
 			     struct ast_vbios_mode_info *vbios_mode)
 {
@@ -544,7 +544,7 @@ static void ast_set_sync_reg(struct ast_private *ast,
 	ast_io_write8(ast, AST_IO_MISC_PORT_WRITE, jreg);
 }
 
-static void ast_set_start_address_crt1(struct ast_private *ast,
+static void ast_set_start_address_crt1(struct ast_device *ast,
 				       unsigned int offset)
 {
 	u32 addr;
@@ -556,7 +556,7 @@ static void ast_set_start_address_crt1(struct ast_private *ast,
 
 }
 
-static void ast_wait_for_vretrace(struct ast_private *ast)
+static void ast_wait_for_vretrace(struct ast_device *ast)
 {
 	unsigned long timeout = jiffies + HZ;
 	u8 vgair1;
@@ -645,7 +645,7 @@ static void ast_primary_plane_helper_atomic_update(struct drm_plane *plane,
 						   struct drm_atomic_state *state)
 {
 	struct drm_device *dev = plane->dev;
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
 	struct drm_framebuffer *fb = plane_state->fb;
@@ -672,23 +672,34 @@ static void ast_primary_plane_helper_atomic_update(struct drm_plane *plane,
 
 	/*
 	 * Some BMCs stop scanning out the video signal after the driver
-	 * reprogrammed the offset or scanout address. This stalls display
-	 * output for several seconds and makes the display unusable.
-	 * Therefore only update the offset if it changes and reprogram the
-	 * address after enabling the plane.
+	 * reprogrammed the offset. This stalls display output for several
+	 * seconds and makes the display unusable. Therefore only update
+	 * the offset if it changes.
 	 */
 	if (!old_fb || old_fb->pitches[0] != fb->pitches[0])
 		ast_set_offset_reg(ast, fb);
-	if (!old_fb) {
-		ast_set_start_address_crt1(ast, (u32)ast_plane->offset);
-		ast_set_index_reg_mask(ast, AST_IO_SEQ_PORT, 0x1, 0xdf, 0x00);
-	}
+}
+
+static void ast_primary_plane_helper_atomic_enable(struct drm_plane *plane,
+						   struct drm_atomic_state *state)
+{
+	struct ast_device *ast = to_ast_device(plane->dev);
+	struct ast_plane *ast_plane = to_ast_plane(plane);
+
+	/*
+	 * Some BMCs stop scanning out the video signal after the driver
+	 * reprogrammed the scanout address. This stalls display
+	 * output for several seconds and makes the display unusable.
+	 * Therefore only reprogram the address after enabling the plane.
+	 */
+	ast_set_start_address_crt1(ast, (u32)ast_plane->offset);
+	ast_set_index_reg_mask(ast, AST_IO_SEQ_PORT, 0x1, 0xdf, 0x00);
 }
 
 static void ast_primary_plane_helper_atomic_disable(struct drm_plane *plane,
 						    struct drm_atomic_state *state)
 {
-	struct ast_private *ast = to_ast_private(plane->dev);
+	struct ast_device *ast = to_ast_device(plane->dev);
 
 	ast_set_index_reg_mask(ast, AST_IO_SEQ_PORT, 0x1, 0xdf, 0x20);
 }
@@ -697,6 +708,7 @@ static const struct drm_plane_helper_funcs ast_primary_plane_helper_funcs = {
 	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
 	.atomic_check = ast_primary_plane_helper_atomic_check,
 	.atomic_update = ast_primary_plane_helper_atomic_update,
+	.atomic_enable = ast_primary_plane_helper_atomic_enable,
 	.atomic_disable = ast_primary_plane_helper_atomic_disable,
 };
 
@@ -707,7 +719,7 @@ static const struct drm_plane_funcs ast_primary_plane_funcs = {
 	DRM_GEM_SHADOW_PLANE_FUNCS,
 };
 
-static int ast_primary_plane_init(struct ast_private *ast)
+static int ast_primary_plane_init(struct ast_device *ast)
 {
 	struct drm_device *dev = &ast->base;
 	struct ast_plane *ast_primary_plane = &ast->primary_plane;
@@ -800,7 +812,7 @@ static void ast_update_cursor_image(u8 __iomem *dst, const u8 *src, int width, i
 	writel(0, dst + AST_HWC_SIGNATURE_HOTSPOTY);
 }
 
-static void ast_set_cursor_base(struct ast_private *ast, u64 address)
+static void ast_set_cursor_base(struct ast_device *ast, u64 address)
 {
 	u8 addr0 = (address >> 3) & 0xff;
 	u8 addr1 = (address >> 11) & 0xff;
@@ -811,7 +823,7 @@ static void ast_set_cursor_base(struct ast_private *ast, u64 address)
 	ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0xca, addr2);
 }
 
-static void ast_set_cursor_location(struct ast_private *ast, u16 x, u16 y,
+static void ast_set_cursor_location(struct ast_device *ast, u16 x, u16 y,
 				    u8 x_offset, u8 y_offset)
 {
 	u8 x0 = (x & 0x00ff);
@@ -827,7 +839,7 @@ static void ast_set_cursor_location(struct ast_private *ast, u16 x, u16 y,
 	ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0xc7, y1);
 }
 
-static void ast_set_cursor_enabled(struct ast_private *ast, bool enabled)
+static void ast_set_cursor_enabled(struct ast_device *ast, bool enabled)
 {
 	static const u8 mask = (u8)~(AST_IO_VGACRCB_HWC_16BPP |
 				     AST_IO_VGACRCB_HWC_ENABLED);
@@ -876,7 +888,7 @@ static void ast_cursor_plane_helper_atomic_update(struct drm_plane *plane,
 	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
 	struct drm_framebuffer *fb = plane_state->fb;
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
-	struct ast_private *ast = to_ast_private(plane->dev);
+	struct ast_device *ast = to_ast_device(plane->dev);
 	struct iosys_map src_map = shadow_plane_state->data[0];
 	struct drm_rect damage;
 	const u8 *src = src_map.vaddr; /* TODO: Use mapping abstraction properly */
@@ -931,7 +943,7 @@ static void ast_cursor_plane_helper_atomic_update(struct drm_plane *plane,
 static void ast_cursor_plane_helper_atomic_disable(struct drm_plane *plane,
 						   struct drm_atomic_state *state)
 {
-	struct ast_private *ast = to_ast_private(plane->dev);
+	struct ast_device *ast = to_ast_device(plane->dev);
 
 	ast_set_cursor_enabled(ast, false);
 }
@@ -950,7 +962,7 @@ static const struct drm_plane_funcs ast_cursor_plane_funcs = {
 	DRM_GEM_SHADOW_PLANE_FUNCS,
 };
 
-static int ast_cursor_plane_init(struct ast_private *ast)
+static int ast_cursor_plane_init(struct ast_device *ast)
 {
 	struct drm_device *dev = &ast->base;
 	struct ast_plane *ast_cursor_plane = &ast->cursor_plane;
@@ -995,7 +1007,7 @@ static int ast_cursor_plane_init(struct ast_private *ast)
 
 static void ast_crtc_dpms(struct drm_crtc *crtc, int mode)
 {
-	struct ast_private *ast = to_ast_private(crtc->dev);
+	struct ast_device *ast = to_ast_device(crtc->dev);
 	u8 ch = AST_DPMS_VSYNC_OFF | AST_DPMS_HSYNC_OFF;
 	struct ast_crtc_state *ast_state;
 	const struct drm_format_info *format;
@@ -1052,7 +1064,7 @@ static void ast_crtc_dpms(struct drm_crtc *crtc, int mode)
 static enum drm_mode_status
 ast_crtc_helper_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode *mode)
 {
-	struct ast_private *ast = to_ast_private(crtc->dev);
+	struct ast_device *ast = to_ast_device(crtc->dev);
 	enum drm_mode_status status;
 	uint32_t jtemp;
 
@@ -1177,7 +1189,7 @@ ast_crtc_helper_atomic_flush(struct drm_crtc *crtc,
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
 	struct drm_device *dev = crtc->dev;
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct ast_crtc_state *ast_crtc_state = to_ast_crtc_state(crtc_state);
 	struct ast_vbios_mode_info *vbios_mode_info = &ast_crtc_state->vbios_mode_info;
 
@@ -1202,7 +1214,7 @@ ast_crtc_helper_atomic_flush(struct drm_crtc *crtc,
 static void ast_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state)
 {
 	struct drm_device *dev = crtc->dev;
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 	struct ast_crtc_state *ast_crtc_state = to_ast_crtc_state(crtc_state);
 	struct ast_vbios_mode_info *vbios_mode_info =
@@ -1224,7 +1236,7 @@ static void ast_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_ato
 {
 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc);
 	struct drm_device *dev = crtc->dev;
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 
 	ast_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
 
@@ -1312,7 +1324,7 @@ static const struct drm_crtc_funcs ast_crtc_funcs = {
 
 static int ast_crtc_init(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct drm_crtc *crtc = &ast->crtc;
 	int ret;
 
@@ -1338,7 +1350,7 @@ static int ast_vga_connector_helper_get_modes(struct drm_connector *connector)
 {
 	struct ast_vga_connector *ast_vga_connector = to_ast_vga_connector(connector);
 	struct drm_device *dev = connector->dev;
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct edid *edid;
 	int count;
 
@@ -1411,7 +1423,7 @@ static int ast_vga_connector_init(struct drm_device *dev,
 	return 0;
 }
 
-static int ast_vga_output_init(struct ast_private *ast)
+static int ast_vga_output_init(struct ast_device *ast)
 {
 	struct drm_device *dev = &ast->base;
 	struct drm_crtc *crtc = &ast->crtc;
@@ -1444,7 +1456,7 @@ static int ast_sil164_connector_helper_get_modes(struct drm_connector *connector
 {
 	struct ast_sil164_connector *ast_sil164_connector = to_ast_sil164_connector(connector);
 	struct drm_device *dev = connector->dev;
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct edid *edid;
 	int count;
 
@@ -1517,7 +1529,7 @@ static int ast_sil164_connector_init(struct drm_device *dev,
 	return 0;
 }
 
-static int ast_sil164_output_init(struct ast_private *ast)
+static int ast_sil164_output_init(struct ast_device *ast)
 {
 	struct drm_device *dev = &ast->base;
 	struct drm_crtc *crtc = &ast->crtc;
@@ -1604,7 +1616,7 @@ static int ast_dp501_connector_init(struct drm_device *dev, struct drm_connector
 	return 0;
 }
 
-static int ast_dp501_output_init(struct ast_private *ast)
+static int ast_dp501_output_init(struct ast_device *ast)
 {
 	struct drm_device *dev = &ast->base;
 	struct drm_crtc *crtc = &ast->crtc;
@@ -1691,7 +1703,7 @@ static int ast_astdp_connector_init(struct drm_device *dev, struct drm_connector
 	return 0;
 }
 
-static int ast_astdp_output_init(struct ast_private *ast)
+static int ast_astdp_output_init(struct ast_device *ast)
 {
 	struct drm_device *dev = &ast->base;
 	struct drm_crtc *crtc = &ast->crtc;
@@ -1721,7 +1733,7 @@ static int ast_astdp_output_init(struct ast_private *ast)
 
 static void ast_mode_config_helper_atomic_commit_tail(struct drm_atomic_state *state)
 {
-	struct ast_private *ast = to_ast_private(state->dev);
+	struct ast_device *ast = to_ast_device(state->dev);
 
 	/*
 	 * Concurrent operations could possibly trigger a call to
@@ -1742,7 +1754,7 @@ static enum drm_mode_status ast_mode_config_mode_valid(struct drm_device *dev,
 						       const struct drm_display_mode *mode)
 {
 	static const unsigned long max_bpp = 4; /* DRM_FORMAT_XRGB8888 */
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	unsigned long fbsize, fbpages, max_fbpages;
 
 	max_fbpages = (ast->vram_fb_available) >> PAGE_SHIFT;
@@ -1763,7 +1775,7 @@ static const struct drm_mode_config_funcs ast_mode_config_funcs = {
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-int ast_mode_config_init(struct ast_private *ast)
+int ast_mode_config_init(struct ast_device *ast)
 {
 	struct drm_device *dev = &ast->base;
 	int ret;
diff --git a/drivers/gpu/drm/ast/ast_post.c b/drivers/gpu/drm/ast/ast_post.c
index 82fd3c8adee1..71bb36b865fd 100644
--- a/drivers/gpu/drm/ast/ast_post.c
+++ b/drivers/gpu/drm/ast/ast_post.c
@@ -39,7 +39,7 @@ static void ast_post_chip_2500(struct drm_device *dev);
 
 void ast_enable_vga(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 
 	ast_io_write8(ast, AST_IO_VGA_ENABLE_PORT, 0x01);
 	ast_io_write8(ast, AST_IO_MISC_PORT_WRITE, 0x01);
@@ -47,7 +47,7 @@ void ast_enable_vga(struct drm_device *dev)
 
 void ast_enable_mmio(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 
 	ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0xa1, 0x06);
 }
@@ -55,7 +55,7 @@ void ast_enable_mmio(struct drm_device *dev)
 
 bool ast_is_vga_enabled(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u8 ch;
 
 	ch = ast_io_read8(ast, AST_IO_VGA_ENABLE_PORT);
@@ -70,7 +70,7 @@ static const u8 extreginfo_ast2300[] = { 0x0f, 0x04, 0x1f, 0xff };
 static void
 ast_set_def_ext_reg(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
 	u8 i, index, reg;
 	const u8 *ext_reg_info;
@@ -110,7 +110,7 @@ ast_set_def_ext_reg(struct drm_device *dev)
 	ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0xb6, 0xff, reg);
 }
 
-u32 ast_mindwm(struct ast_private *ast, u32 r)
+u32 ast_mindwm(struct ast_device *ast, u32 r)
 {
 	uint32_t data;
 
@@ -123,7 +123,7 @@ u32 ast_mindwm(struct ast_private *ast, u32 r)
 	return ast_read32(ast, 0x10000 + (r & 0x0000ffff));
 }
 
-void ast_moutdwm(struct ast_private *ast, u32 r, u32 v)
+void ast_moutdwm(struct ast_device *ast, u32 r, u32 v)
 {
 	uint32_t data;
 	ast_write32(ast, 0xf004, r & 0xffff0000);
@@ -162,7 +162,7 @@ static const u32 pattern_AST2150[14] = {
 	0x20F050E0
 };
 
-static u32 mmctestburst2_ast2150(struct ast_private *ast, u32 datagen)
+static u32 mmctestburst2_ast2150(struct ast_device *ast, u32 datagen)
 {
 	u32 data, timeout;
 
@@ -192,7 +192,7 @@ static u32 mmctestburst2_ast2150(struct ast_private *ast, u32 datagen)
 }
 
 #if 0 /* unused in DDX driver - here for completeness */
-static u32 mmctestsingle2_ast2150(struct ast_private *ast, u32 datagen)
+static u32 mmctestsingle2_ast2150(struct ast_device *ast, u32 datagen)
 {
 	u32 data, timeout;
 
@@ -212,7 +212,7 @@ static u32 mmctestsingle2_ast2150(struct ast_private *ast, u32 datagen)
 }
 #endif
 
-static int cbrtest_ast2150(struct ast_private *ast)
+static int cbrtest_ast2150(struct ast_device *ast)
 {
 	int i;
 
@@ -222,7 +222,7 @@ static int cbrtest_ast2150(struct ast_private *ast)
 	return 1;
 }
 
-static int cbrscan_ast2150(struct ast_private *ast, int busw)
+static int cbrscan_ast2150(struct ast_device *ast, int busw)
 {
 	u32 patcnt, loop;
 
@@ -239,7 +239,7 @@ static int cbrscan_ast2150(struct ast_private *ast, int busw)
 }
 
 
-static void cbrdlli_ast2150(struct ast_private *ast, int busw)
+static void cbrdlli_ast2150(struct ast_device *ast, int busw)
 {
 	u32 dll_min[4], dll_max[4], dlli, data, passcnt;
 
@@ -273,7 +273,7 @@ cbr_start:
 
 static void ast_init_dram_reg(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u8 j;
 	u32 data, temp, i;
 	const struct ast_dramstruct *dram_reg_info;
@@ -366,7 +366,7 @@ static void ast_init_dram_reg(struct drm_device *dev)
 
 void ast_post_gpu(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
 	u32 reg;
 
@@ -449,7 +449,7 @@ static const u32 pattern[8] = {
 	0x7C61D253
 };
 
-static bool mmc_test(struct ast_private *ast, u32 datagen, u8 test_ctl)
+static bool mmc_test(struct ast_device *ast, u32 datagen, u8 test_ctl)
 {
 	u32 data, timeout;
 
@@ -469,7 +469,7 @@ static bool mmc_test(struct ast_private *ast, u32 datagen, u8 test_ctl)
 	return true;
 }
 
-static u32 mmc_test2(struct ast_private *ast, u32 datagen, u8 test_ctl)
+static u32 mmc_test2(struct ast_device *ast, u32 datagen, u8 test_ctl)
 {
 	u32 data, timeout;
 
@@ -490,32 +490,32 @@ static u32 mmc_test2(struct ast_private *ast, u32 datagen, u8 test_ctl)
 }
 
 
-static bool mmc_test_burst(struct ast_private *ast, u32 datagen)
+static bool mmc_test_burst(struct ast_device *ast, u32 datagen)
 {
 	return mmc_test(ast, datagen, 0xc1);
 }
 
-static u32 mmc_test_burst2(struct ast_private *ast, u32 datagen)
+static u32 mmc_test_burst2(struct ast_device *ast, u32 datagen)
 {
 	return mmc_test2(ast, datagen, 0x41);
 }
 
-static bool mmc_test_single(struct ast_private *ast, u32 datagen)
+static bool mmc_test_single(struct ast_device *ast, u32 datagen)
 {
 	return mmc_test(ast, datagen, 0xc5);
 }
 
-static u32 mmc_test_single2(struct ast_private *ast, u32 datagen)
+static u32 mmc_test_single2(struct ast_device *ast, u32 datagen)
 {
 	return mmc_test2(ast, datagen, 0x05);
 }
 
-static bool mmc_test_single_2500(struct ast_private *ast, u32 datagen)
+static bool mmc_test_single_2500(struct ast_device *ast, u32 datagen)
 {
 	return mmc_test(ast, datagen, 0x85);
 }
 
-static int cbr_test(struct ast_private *ast)
+static int cbr_test(struct ast_device *ast)
 {
 	u32 data;
 	int i;
@@ -534,7 +534,7 @@ static int cbr_test(struct ast_private *ast)
 	return 1;
 }
 
-static int cbr_scan(struct ast_private *ast)
+static int cbr_scan(struct ast_device *ast)
 {
 	u32 data, data2, patcnt, loop;
 
@@ -555,7 +555,7 @@ static int cbr_scan(struct ast_private *ast)
 	return data2;
 }
 
-static u32 cbr_test2(struct ast_private *ast)
+static u32 cbr_test2(struct ast_device *ast)
 {
 	u32 data;
 
@@ -569,7 +569,7 @@ static u32 cbr_test2(struct ast_private *ast)
 	return ~data & 0xffff;
 }
 
-static u32 cbr_scan2(struct ast_private *ast)
+static u32 cbr_scan2(struct ast_device *ast)
 {
 	u32 data, data2, patcnt, loop;
 
@@ -590,7 +590,7 @@ static u32 cbr_scan2(struct ast_private *ast)
 	return data2;
 }
 
-static bool cbr_test3(struct ast_private *ast)
+static bool cbr_test3(struct ast_device *ast)
 {
 	if (!mmc_test_burst(ast, 0))
 		return false;
@@ -599,7 +599,7 @@ static bool cbr_test3(struct ast_private *ast)
 	return true;
 }
 
-static bool cbr_scan3(struct ast_private *ast)
+static bool cbr_scan3(struct ast_device *ast)
 {
 	u32 patcnt, loop;
 
@@ -615,7 +615,7 @@ static bool cbr_scan3(struct ast_private *ast)
 	return true;
 }
 
-static bool finetuneDQI_L(struct ast_private *ast, struct ast2300_dram_param *param)
+static bool finetuneDQI_L(struct ast_device *ast, struct ast2300_dram_param *param)
 {
 	u32 gold_sadj[2], dllmin[16], dllmax[16], dlli, data, cnt, mask, passcnt, retry = 0;
 	bool status = false;
@@ -714,7 +714,7 @@ FINETUNE_DONE:
 	return status;
 } /* finetuneDQI_L */
 
-static void finetuneDQSI(struct ast_private *ast)
+static void finetuneDQSI(struct ast_device *ast)
 {
 	u32 dlli, dqsip, dqidly;
 	u32 reg_mcr18, reg_mcr0c, passcnt[2], diff;
@@ -804,7 +804,7 @@ static void finetuneDQSI(struct ast_private *ast)
 	ast_moutdwm(ast, 0x1E6E0018, reg_mcr18);
 
 }
-static bool cbr_dll2(struct ast_private *ast, struct ast2300_dram_param *param)
+static bool cbr_dll2(struct ast_device *ast, struct ast2300_dram_param *param)
 {
 	u32 dllmin[2], dllmax[2], dlli, data, passcnt, retry = 0;
 	bool status = false;
@@ -860,7 +860,7 @@ CBR_DONE2:
 	return status;
 } /* CBRDLL2 */
 
-static void get_ddr3_info(struct ast_private *ast, struct ast2300_dram_param *param)
+static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *param)
 {
 	u32 trap, trap_AC2, trap_MRS;
 
@@ -1102,7 +1102,7 @@ static void get_ddr3_info(struct ast_private *ast, struct ast2300_dram_param *pa
 
 }
 
-static void ddr3_init(struct ast_private *ast, struct ast2300_dram_param *param)
+static void ddr3_init(struct ast_device *ast, struct ast2300_dram_param *param)
 {
 	u32 data, data2, retry = 0;
 
@@ -1225,7 +1225,7 @@ ddr3_init_start:
 
 }
 
-static void get_ddr2_info(struct ast_private *ast, struct ast2300_dram_param *param)
+static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *param)
 {
 	u32 trap, trap_AC2, trap_MRS;
 
@@ -1472,7 +1472,7 @@ static void get_ddr2_info(struct ast_private *ast, struct ast2300_dram_param *pa
 	}
 }
 
-static void ddr2_init(struct ast_private *ast, struct ast2300_dram_param *param)
+static void ddr2_init(struct ast_device *ast, struct ast2300_dram_param *param)
 {
 	u32 data, data2, retry = 0;
 
@@ -1600,7 +1600,7 @@ ddr2_init_start:
 
 static void ast_post_chip_2300(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	struct ast2300_dram_param param;
 	u32 temp;
 	u8 reg;
@@ -1681,7 +1681,7 @@ static void ast_post_chip_2300(struct drm_device *dev)
 	} while ((reg & 0x40) == 0);
 }
 
-static bool cbr_test_2500(struct ast_private *ast)
+static bool cbr_test_2500(struct ast_device *ast)
 {
 	ast_moutdwm(ast, 0x1E6E0074, 0x0000FFFF);
 	ast_moutdwm(ast, 0x1E6E007C, 0xFF00FF00);
@@ -1692,7 +1692,7 @@ static bool cbr_test_2500(struct ast_private *ast)
 	return true;
 }
 
-static bool ddr_test_2500(struct ast_private *ast)
+static bool ddr_test_2500(struct ast_device *ast)
 {
 	ast_moutdwm(ast, 0x1E6E0074, 0x0000FFFF);
 	ast_moutdwm(ast, 0x1E6E007C, 0xFF00FF00);
@@ -1709,7 +1709,7 @@ static bool ddr_test_2500(struct ast_private *ast)
 	return true;
 }
 
-static void ddr_init_common_2500(struct ast_private *ast)
+static void ddr_init_common_2500(struct ast_device *ast)
 {
 	ast_moutdwm(ast, 0x1E6E0034, 0x00020080);
 	ast_moutdwm(ast, 0x1E6E0008, 0x2003000F);
@@ -1732,7 +1732,7 @@ static void ddr_init_common_2500(struct ast_private *ast)
 	ast_moutdwm(ast, 0x1E6E024C, 0x80808080);
 }
 
-static void ddr_phy_init_2500(struct ast_private *ast)
+static void ddr_phy_init_2500(struct ast_device *ast)
 {
 	u32 data, pass, timecnt;
 
@@ -1766,7 +1766,7 @@ static void ddr_phy_init_2500(struct ast_private *ast)
  * 4Gb : 0x80000000 ~ 0x9FFFFFFF
  * 8Gb : 0x80000000 ~ 0xBFFFFFFF
  */
-static void check_dram_size_2500(struct ast_private *ast, u32 tRFC)
+static void check_dram_size_2500(struct ast_device *ast, u32 tRFC)
 {
 	u32 reg_04, reg_14;
 
@@ -1797,7 +1797,7 @@ static void check_dram_size_2500(struct ast_private *ast, u32 tRFC)
 	ast_moutdwm(ast, 0x1E6E0014, reg_14);
 }
 
-static void enable_cache_2500(struct ast_private *ast)
+static void enable_cache_2500(struct ast_device *ast)
 {
 	u32 reg_04, data;
 
@@ -1810,7 +1810,7 @@ static void enable_cache_2500(struct ast_private *ast)
 	ast_moutdwm(ast, 0x1E6E0004, reg_04 | 0x400);
 }
 
-static void set_mpll_2500(struct ast_private *ast)
+static void set_mpll_2500(struct ast_device *ast)
 {
 	u32 addr, data, param;
 
@@ -1837,7 +1837,7 @@ static void set_mpll_2500(struct ast_private *ast)
 	udelay(100);
 }
 
-static void reset_mmc_2500(struct ast_private *ast)
+static void reset_mmc_2500(struct ast_device *ast)
 {
 	ast_moutdwm(ast, 0x1E78505C, 0x00000004);
 	ast_moutdwm(ast, 0x1E785044, 0x00000001);
@@ -1848,7 +1848,7 @@ static void reset_mmc_2500(struct ast_private *ast)
 	ast_moutdwm(ast, 0x1E6E0000, 0xFC600309);
 }
 
-static void ddr3_init_2500(struct ast_private *ast, const u32 *ddr_table)
+static void ddr3_init_2500(struct ast_device *ast, const u32 *ddr_table)
 {
 
 	ast_moutdwm(ast, 0x1E6E0004, 0x00000303);
@@ -1892,7 +1892,7 @@ static void ddr3_init_2500(struct ast_private *ast, const u32 *ddr_table)
 	ast_moutdwm(ast, 0x1E6E0038, 0xFFFFFF00);
 }
 
-static void ddr4_init_2500(struct ast_private *ast, const u32 *ddr_table)
+static void ddr4_init_2500(struct ast_device *ast, const u32 *ddr_table)
 {
 	u32 data, data2, pass, retrycnt;
 	u32 ddr_vref, phy_vref;
@@ -2002,7 +2002,7 @@ static void ddr4_init_2500(struct ast_private *ast, const u32 *ddr_table)
 	ast_moutdwm(ast, 0x1E6E0038, 0xFFFFFF00);
 }
 
-static bool ast_dram_init_2500(struct ast_private *ast)
+static bool ast_dram_init_2500(struct ast_device *ast)
 {
 	u32 data;
 	u32 max_tries = 5;
@@ -2030,7 +2030,7 @@ static bool ast_dram_init_2500(struct ast_private *ast)
 	return true;
 }
 
-void ast_patch_ahb_2500(struct ast_private *ast)
+void ast_patch_ahb_2500(struct ast_device *ast)
 {
 	u32	data;
 
@@ -2066,7 +2066,7 @@ void ast_patch_ahb_2500(struct ast_private *ast)
 
 void ast_post_chip_2500(struct drm_device *dev)
 {
-	struct ast_private *ast = to_ast_private(dev);
+	struct ast_device *ast = to_ast_device(dev);
 	u32 temp;
 	u8 reg;
 
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
index 4e806b06d35d..29603561d501 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
@@ -19,7 +19,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
@@ -760,7 +760,7 @@ static int atmel_hlcdc_dc_drm_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_unload;
 
-	drm_fbdev_generic_setup(ddev, 24);
+	drm_fbdev_dma_setup(ddev, 24);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index 8b2226f72b24..f076a09afac0 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -220,6 +220,18 @@ config DRM_PARADE_PS8640
 	  The PS8640 is a high-performance and low-power
 	  MIPI DSI to eDP converter
 
+config DRM_SAMSUNG_DSIM
+	tristate "Samsung MIPI DSIM bridge driver"
+	depends on COMMON_CLK
+	depends on OF && HAS_IOMEM
+	select DRM_KMS_HELPER
+	select DRM_MIPI_DSI
+	select DRM_PANEL_BRIDGE
+	help
+	  The Samsung MIPI DSIM bridge controller driver.
+	  This MIPI DSIM bridge can be found it on Exynos SoCs and
+	  NXP's i.MX8M Mini/Nano.
+
 config DRM_SIL_SII8620
 	tristate "Silicon Image SII8620 HDMI/MHL bridge"
 	depends on OF
@@ -326,7 +338,7 @@ config DRM_TI_DLPC3433
 	  input that produces a DMD output in RGB565, RGB666, RGB888
 	  formats.
 
-	  It supports upto 720p resolution with 60 and 120 Hz refresh
+	  It supports up to 720p resolution with 60 and 120 Hz refresh
 	  rates.
 
 config DRM_TI_TFP410
diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
index 52f6e8b4a821..2b892b7ed59e 100644
--- a/drivers/gpu/drm/bridge/Makefile
+++ b/drivers/gpu/drm/bridge/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW) += megachips-stdpxxxx-ge-b850v
 obj-$(CONFIG_DRM_NXP_PTN3460) += nxp-ptn3460.o
 obj-$(CONFIG_DRM_PARADE_PS8622) += parade-ps8622.o
 obj-$(CONFIG_DRM_PARADE_PS8640) += parade-ps8640.o
+obj-$(CONFIG_DRM_SAMSUNG_DSIM) += samsung-dsim.o
 obj-$(CONFIG_DRM_SIL_SII8620) += sil-sii8620.o
 obj-$(CONFIG_DRM_SII902X) += sii902x.o
 obj-$(CONFIG_DRM_SII9234) += sii9234.o
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7533.c b/drivers/gpu/drm/bridge/adv7511/adv7533.c
index fdfeadcefe80..7e3e56441aed 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7533.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7533.c
@@ -103,22 +103,19 @@ void adv7533_dsi_power_off(struct adv7511 *adv)
 enum drm_mode_status adv7533_mode_valid(struct adv7511 *adv,
 					const struct drm_display_mode *mode)
 {
-	int lanes;
+	unsigned long max_lane_freq;
 	struct mipi_dsi_device *dsi = adv->dsi;
+	u8 bpp = mipi_dsi_pixel_format_to_bpp(dsi->format);
 
-	if (mode->clock > 80000)
-		lanes = 4;
-	else
-		lanes = 3;
-
-	/*
-	 * TODO: add support for dynamic switching of lanes
-	 * by using the bridge pre_enable() op . Till then filter
-	 * out the modes which shall need different number of lanes
-	 * than what was configured in the device tree.
-	 */
-	if (lanes != dsi->lanes)
-		return MODE_BAD;
+	/* Check max clock for either 7533 or 7535 */
+	if (mode->clock > (adv->type == ADV7533 ? 80000 : 148500))
+		return MODE_CLOCK_HIGH;
+
+	/* Check max clock for each lane */
+	max_lane_freq = (adv->type == ADV7533 ? 800000 : 891000);
+
+	if (mode->clock * bpp > max_lane_freq * adv->num_dsi_lanes)
+		return MODE_CLOCK_HIGH;
 
 	return MODE_OK;
 }
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
index 5dbfc7226b31..f50d65f54314 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
@@ -1278,7 +1278,7 @@ err_disable_pclk:
 	return ret;
 }
 
-static int cdns_dsi_drm_remove(struct platform_device *pdev)
+static void cdns_dsi_drm_remove(struct platform_device *pdev)
 {
 	struct cdns_dsi *dsi = platform_get_drvdata(pdev);
 
@@ -1288,8 +1288,6 @@ static int cdns_dsi_drm_remove(struct platform_device *pdev)
 		dsi->platform_ops->deinit(dsi);
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static const struct of_device_id cdns_dsi_of_match[] = {
@@ -1303,7 +1301,7 @@ MODULE_DEVICE_TABLE(of, cdns_dsi_of_match);
 
 static struct platform_driver cdns_dsi_platform_driver = {
 	.probe  = cdns_dsi_drm_probe,
-	.remove = cdns_dsi_drm_remove,
+	.remove_new = cdns_dsi_drm_remove,
 	.driver = {
 		.name   = "cdns-dsi",
 		.of_match_table = cdns_dsi_of_match,
diff --git a/drivers/gpu/drm/bridge/display-connector.c b/drivers/gpu/drm/bridge/display-connector.c
index 9a12449ad7b8..56ae511367b1 100644
--- a/drivers/gpu/drm/bridge/display-connector.c
+++ b/drivers/gpu/drm/bridge/display-connector.c
@@ -271,12 +271,9 @@ static int display_connector_probe(struct platform_device *pdev)
 	    type == DRM_MODE_CONNECTOR_DisplayPort) {
 		conn->hpd_gpio = devm_gpiod_get_optional(&pdev->dev, "hpd",
 							 GPIOD_IN);
-		if (IS_ERR(conn->hpd_gpio)) {
-			if (PTR_ERR(conn->hpd_gpio) != -EPROBE_DEFER)
-				dev_err(&pdev->dev,
-					"Unable to retrieve HPD GPIO\n");
-			return PTR_ERR(conn->hpd_gpio);
-		}
+		if (IS_ERR(conn->hpd_gpio))
+			return dev_err_probe(&pdev->dev, PTR_ERR(conn->hpd_gpio),
+					     "Unable to retrieve HPD GPIO\n");
 
 		conn->hpd_irq = gpiod_to_irq(conn->hpd_gpio);
 	} else {
@@ -382,7 +379,7 @@ static int display_connector_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int display_connector_remove(struct platform_device *pdev)
+static void display_connector_remove(struct platform_device *pdev)
 {
 	struct display_connector *conn = platform_get_drvdata(pdev);
 
@@ -396,8 +393,6 @@ static int display_connector_remove(struct platform_device *pdev)
 
 	if (!IS_ERR(conn->bridge.ddc))
 		i2c_put_adapter(conn->bridge.ddc);
-
-	return 0;
 }
 
 static const struct of_device_id display_connector_match[] = {
@@ -426,7 +421,7 @@ MODULE_DEVICE_TABLE(of, display_connector_match);
 
 static struct platform_driver display_connector_driver = {
 	.probe	= display_connector_probe,
-	.remove	= display_connector_remove,
+	.remove_new = display_connector_remove,
 	.driver		= {
 		.name		= "display-connector",
 		.of_match_table	= display_connector_match,
diff --git a/drivers/gpu/drm/bridge/fsl-ldb.c b/drivers/gpu/drm/bridge/fsl-ldb.c
index 6bac160b395b..682623369498 100644
--- a/drivers/gpu/drm/bridge/fsl-ldb.c
+++ b/drivers/gpu/drm/bridge/fsl-ldb.c
@@ -84,10 +84,16 @@ struct fsl_ldb {
 	struct drm_bridge *panel_bridge;
 	struct clk *clk;
 	struct regmap *regmap;
-	bool lvds_dual_link;
 	const struct fsl_ldb_devdata *devdata;
+	bool ch0_enabled;
+	bool ch1_enabled;
 };
 
+static bool fsl_ldb_is_dual(const struct fsl_ldb *fsl_ldb)
+{
+	return (fsl_ldb->ch0_enabled && fsl_ldb->ch1_enabled);
+}
+
 static inline struct fsl_ldb *to_fsl_ldb(struct drm_bridge *bridge)
 {
 	return container_of(bridge, struct fsl_ldb, bridge);
@@ -95,7 +101,7 @@ static inline struct fsl_ldb *to_fsl_ldb(struct drm_bridge *bridge)
 
 static unsigned long fsl_ldb_link_frequency(struct fsl_ldb *fsl_ldb, int clock)
 {
-	if (fsl_ldb->lvds_dual_link)
+	if (fsl_ldb_is_dual(fsl_ldb))
 		return clock * 3500;
 	else
 		return clock * 7000;
@@ -170,35 +176,28 @@ static void fsl_ldb_atomic_enable(struct drm_bridge *bridge,
 
 	configured_link_freq = clk_get_rate(fsl_ldb->clk);
 	if (configured_link_freq != requested_link_freq)
-		dev_warn(fsl_ldb->dev, "Configured LDB clock (%lu Hz) does not match requested LVDS clock: %lu Hz",
+		dev_warn(fsl_ldb->dev, "Configured LDB clock (%lu Hz) does not match requested LVDS clock: %lu Hz\n",
 			 configured_link_freq,
 			 requested_link_freq);
 
 	clk_prepare_enable(fsl_ldb->clk);
 
 	/* Program LDB_CTRL */
-	reg = LDB_CTRL_CH0_ENABLE;
-
-	if (fsl_ldb->lvds_dual_link)
-		reg |= LDB_CTRL_CH1_ENABLE | LDB_CTRL_SPLIT_MODE;
+	reg =	(fsl_ldb->ch0_enabled ? LDB_CTRL_CH0_ENABLE : 0) |
+		(fsl_ldb->ch1_enabled ? LDB_CTRL_CH1_ENABLE : 0) |
+		(fsl_ldb_is_dual(fsl_ldb) ? LDB_CTRL_SPLIT_MODE : 0);
 
-	if (lvds_format_24bpp) {
-		reg |= LDB_CTRL_CH0_DATA_WIDTH;
-		if (fsl_ldb->lvds_dual_link)
-			reg |= LDB_CTRL_CH1_DATA_WIDTH;
-	}
+	if (lvds_format_24bpp)
+		reg |=	(fsl_ldb->ch0_enabled ? LDB_CTRL_CH0_DATA_WIDTH : 0) |
+			(fsl_ldb->ch1_enabled ? LDB_CTRL_CH1_DATA_WIDTH : 0);
 
-	if (lvds_format_jeida) {
-		reg |= LDB_CTRL_CH0_BIT_MAPPING;
-		if (fsl_ldb->lvds_dual_link)
-			reg |= LDB_CTRL_CH1_BIT_MAPPING;
-	}
+	if (lvds_format_jeida)
+		reg |=	(fsl_ldb->ch0_enabled ? LDB_CTRL_CH0_BIT_MAPPING : 0) |
+			(fsl_ldb->ch1_enabled ? LDB_CTRL_CH1_BIT_MAPPING : 0);
 
-	if (mode->flags & DRM_MODE_FLAG_PVSYNC) {
-		reg |= LDB_CTRL_DI0_VSYNC_POLARITY;
-		if (fsl_ldb->lvds_dual_link)
-			reg |= LDB_CTRL_DI1_VSYNC_POLARITY;
-	}
+	if (mode->flags & DRM_MODE_FLAG_PVSYNC)
+		reg |=	(fsl_ldb->ch0_enabled ? LDB_CTRL_DI0_VSYNC_POLARITY : 0) |
+			(fsl_ldb->ch1_enabled ? LDB_CTRL_DI1_VSYNC_POLARITY : 0);
 
 	regmap_write(fsl_ldb->regmap, fsl_ldb->devdata->ldb_ctrl, reg);
 
@@ -210,9 +209,8 @@ static void fsl_ldb_atomic_enable(struct drm_bridge *bridge,
 	/* Wait for VBG to stabilize. */
 	usleep_range(15, 20);
 
-	reg |= LVDS_CTRL_CH0_EN;
-	if (fsl_ldb->lvds_dual_link)
-		reg |= LVDS_CTRL_CH1_EN;
+	reg |=	(fsl_ldb->ch0_enabled ? LVDS_CTRL_CH0_EN : 0) |
+		(fsl_ldb->ch1_enabled ? LVDS_CTRL_CH1_EN : 0);
 
 	regmap_write(fsl_ldb->regmap, fsl_ldb->devdata->lvds_ctrl, reg);
 }
@@ -265,7 +263,7 @@ fsl_ldb_mode_valid(struct drm_bridge *bridge,
 {
 	struct fsl_ldb *fsl_ldb = to_fsl_ldb(bridge);
 
-	if (mode->clock > (fsl_ldb->lvds_dual_link ? 160000 : 80000))
+	if (mode->clock > (fsl_ldb_is_dual(fsl_ldb) ? 160000 : 80000))
 		return MODE_CLOCK_HIGH;
 
 	return MODE_OK;
@@ -286,7 +284,7 @@ static int fsl_ldb_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *panel_node;
-	struct device_node *port1, *port2;
+	struct device_node *remote1, *remote2;
 	struct drm_panel *panel;
 	struct fsl_ldb *fsl_ldb;
 	int dual_link;
@@ -311,10 +309,23 @@ static int fsl_ldb_probe(struct platform_device *pdev)
 	if (IS_ERR(fsl_ldb->regmap))
 		return PTR_ERR(fsl_ldb->regmap);
 
-	/* Locate the panel DT node. */
-	panel_node = of_graph_get_remote_node(dev->of_node, 1, 0);
-	if (!panel_node)
-		return -ENXIO;
+	/* Locate the remote ports and the panel node */
+	remote1 = of_graph_get_remote_node(dev->of_node, 1, 0);
+	remote2 = of_graph_get_remote_node(dev->of_node, 2, 0);
+	fsl_ldb->ch0_enabled = (remote1 != NULL);
+	fsl_ldb->ch1_enabled = (remote2 != NULL);
+	panel_node = of_node_get(remote1 ? remote1 : remote2);
+	of_node_put(remote1);
+	of_node_put(remote2);
+
+	if (!fsl_ldb->ch0_enabled && !fsl_ldb->ch1_enabled) {
+		of_node_put(panel_node);
+		return dev_err_probe(dev, -ENXIO, "No panel node found");
+	}
+
+	dev_dbg(dev, "Using %s\n",
+		fsl_ldb_is_dual(fsl_ldb) ? "dual-link mode" :
+		fsl_ldb->ch0_enabled ? "channel 0" : "channel 1");
 
 	panel = of_drm_find_panel(panel_node);
 	of_node_put(panel_node);
@@ -325,20 +336,26 @@ static int fsl_ldb_probe(struct platform_device *pdev)
 	if (IS_ERR(fsl_ldb->panel_bridge))
 		return PTR_ERR(fsl_ldb->panel_bridge);
 
-	/* Determine whether this is dual-link configuration */
-	port1 = of_graph_get_port_by_id(dev->of_node, 1);
-	port2 = of_graph_get_port_by_id(dev->of_node, 2);
-	dual_link = drm_of_lvds_get_dual_link_pixel_order(port1, port2);
-	of_node_put(port1);
-	of_node_put(port2);
 
-	if (dual_link == DRM_LVDS_DUAL_LINK_EVEN_ODD_PIXELS) {
-		dev_err(dev, "LVDS channel pixel swap not supported.\n");
-		return -EINVAL;
-	}
+	if (fsl_ldb_is_dual(fsl_ldb)) {
+		struct device_node *port1, *port2;
+
+		port1 = of_graph_get_port_by_id(dev->of_node, 1);
+		port2 = of_graph_get_port_by_id(dev->of_node, 2);
+		dual_link = drm_of_lvds_get_dual_link_pixel_order(port1, port2);
+		of_node_put(port1);
+		of_node_put(port2);
 
-	if (dual_link == DRM_LVDS_DUAL_LINK_ODD_EVEN_PIXELS)
-		fsl_ldb->lvds_dual_link = true;
+		if (dual_link < 0)
+			return dev_err_probe(dev, dual_link,
+					     "Error getting dual link configuration\n");
+
+		/* Only DRM_LVDS_DUAL_LINK_ODD_EVEN_PIXELS is supported */
+		if (dual_link == DRM_LVDS_DUAL_LINK_EVEN_ODD_PIXELS) {
+			dev_err(dev, "LVDS channel pixel swap not supported.\n");
+			return -EINVAL;
+		}
+	}
 
 	platform_set_drvdata(pdev, fsl_ldb);
 
@@ -347,13 +364,11 @@ static int fsl_ldb_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int fsl_ldb_remove(struct platform_device *pdev)
+static void fsl_ldb_remove(struct platform_device *pdev)
 {
 	struct fsl_ldb *fsl_ldb = platform_get_drvdata(pdev);
 
 	drm_bridge_remove(&fsl_ldb->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id fsl_ldb_match[] = {
@@ -367,7 +382,7 @@ MODULE_DEVICE_TABLE(of, fsl_ldb_match);
 
 static struct platform_driver fsl_ldb_driver = {
 	.probe	= fsl_ldb_probe,
-	.remove	= fsl_ldb_remove,
+	.remove_new = fsl_ldb_remove,
 	.driver		= {
 		.name		= "fsl-ldb",
 		.of_match_table	= fsl_ldb_match,
diff --git a/drivers/gpu/drm/bridge/imx/imx8qm-ldb-drv.c b/drivers/gpu/drm/bridge/imx/imx8qm-ldb-drv.c
index 178af8d2d80b..386032a02599 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qm-ldb-drv.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qm-ldb-drv.c
@@ -532,7 +532,7 @@ static int imx8qm_ldb_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int imx8qm_ldb_remove(struct platform_device *pdev)
+static void imx8qm_ldb_remove(struct platform_device *pdev)
 {
 	struct imx8qm_ldb *imx8qm_ldb = platform_get_drvdata(pdev);
 	struct ldb *ldb = &imx8qm_ldb->base;
@@ -540,8 +540,6 @@ static int imx8qm_ldb_remove(struct platform_device *pdev)
 	ldb_remove_bridge_helper(ldb);
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static int __maybe_unused imx8qm_ldb_runtime_suspend(struct device *dev)
@@ -573,7 +571,7 @@ MODULE_DEVICE_TABLE(of, imx8qm_ldb_dt_ids);
 
 static struct platform_driver imx8qm_ldb_driver = {
 	.probe	= imx8qm_ldb_probe,
-	.remove = imx8qm_ldb_remove,
+	.remove_new = imx8qm_ldb_remove,
 	.driver	= {
 		.pm = &imx8qm_ldb_pm_ops,
 		.name = DRIVER_NAME,
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-ldb-drv.c b/drivers/gpu/drm/bridge/imx/imx8qxp-ldb-drv.c
index 63948d5d20fd..c806576b1e22 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-ldb-drv.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-ldb-drv.c
@@ -667,7 +667,7 @@ static int imx8qxp_ldb_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int imx8qxp_ldb_remove(struct platform_device *pdev)
+static void imx8qxp_ldb_remove(struct platform_device *pdev)
 {
 	struct imx8qxp_ldb *imx8qxp_ldb = platform_get_drvdata(pdev);
 	struct ldb *ldb = &imx8qxp_ldb->base;
@@ -675,8 +675,6 @@ static int imx8qxp_ldb_remove(struct platform_device *pdev)
 	ldb_remove_bridge_helper(ldb);
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static int __maybe_unused imx8qxp_ldb_runtime_suspend(struct device *dev)
@@ -708,7 +706,7 @@ MODULE_DEVICE_TABLE(of, imx8qxp_ldb_dt_ids);
 
 static struct platform_driver imx8qxp_ldb_driver = {
 	.probe	= imx8qxp_ldb_probe,
-	.remove = imx8qxp_ldb_remove,
+	.remove_new = imx8qxp_ldb_remove,
 	.driver	= {
 		.pm = &imx8qxp_ldb_pm_ops,
 		.name = DRIVER_NAME,
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c
index 503bd8db8afe..d0868a6ac6c9 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c
@@ -357,7 +357,7 @@ free_child:
 	return ret;
 }
 
-static int imx8qxp_pc_bridge_remove(struct platform_device *pdev)
+static void imx8qxp_pc_bridge_remove(struct platform_device *pdev)
 {
 	struct imx8qxp_pc *pc = platform_get_drvdata(pdev);
 	struct imx8qxp_pc_channel *ch;
@@ -374,8 +374,6 @@ static int imx8qxp_pc_bridge_remove(struct platform_device *pdev)
 	}
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static int __maybe_unused imx8qxp_pc_runtime_suspend(struct device *dev)
@@ -435,7 +433,7 @@ MODULE_DEVICE_TABLE(of, imx8qxp_pc_dt_ids);
 
 static struct platform_driver imx8qxp_pc_bridge_driver = {
 	.probe	= imx8qxp_pc_bridge_probe,
-	.remove = imx8qxp_pc_bridge_remove,
+	.remove_new = imx8qxp_pc_bridge_remove,
 	.driver	= {
 		.pm = &imx8qxp_pc_pm_ops,
 		.name = DRIVER_NAME,
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
index 9e5f2b4dc2e5..ed8b7a4e0e11 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
@@ -313,7 +313,7 @@ imx8qxp_pixel_link_find_next_bridge(struct imx8qxp_pixel_link *pl)
 		}
 
 		/* specially select the next bridge with companion PXL2DPI */
-		if (of_find_property(remote, "fsl,companion-pxl2dpi", NULL))
+		if (of_property_present(remote, "fsl,companion-pxl2dpi"))
 			bridge_sel = ep_cnt;
 
 		ep_cnt++;
@@ -398,13 +398,11 @@ static int imx8qxp_pixel_link_bridge_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int imx8qxp_pixel_link_bridge_remove(struct platform_device *pdev)
+static void imx8qxp_pixel_link_bridge_remove(struct platform_device *pdev)
 {
 	struct imx8qxp_pixel_link *pl = platform_get_drvdata(pdev);
 
 	drm_bridge_remove(&pl->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id imx8qxp_pixel_link_dt_ids[] = {
@@ -416,7 +414,7 @@ MODULE_DEVICE_TABLE(of, imx8qxp_pixel_link_dt_ids);
 
 static struct platform_driver imx8qxp_pixel_link_bridge_driver = {
 	.probe	= imx8qxp_pixel_link_bridge_probe,
-	.remove = imx8qxp_pixel_link_bridge_remove,
+	.remove_new = imx8qxp_pixel_link_bridge_remove,
 	.driver	= {
 		.of_match_table = imx8qxp_pixel_link_dt_ids,
 		.name = DRIVER_NAME,
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
index d0fec82f0cf8..4a886cb808ca 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
@@ -455,15 +455,13 @@ static int imx8qxp_pxl2dpi_bridge_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int imx8qxp_pxl2dpi_bridge_remove(struct platform_device *pdev)
+static void imx8qxp_pxl2dpi_bridge_remove(struct platform_device *pdev)
 {
 	struct imx8qxp_pxl2dpi *p2d = platform_get_drvdata(pdev);
 
 	drm_bridge_remove(&p2d->bridge);
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static const struct of_device_id imx8qxp_pxl2dpi_dt_ids[] = {
@@ -474,7 +472,7 @@ MODULE_DEVICE_TABLE(of, imx8qxp_pxl2dpi_dt_ids);
 
 static struct platform_driver imx8qxp_pxl2dpi_bridge_driver = {
 	.probe	= imx8qxp_pxl2dpi_bridge_probe,
-	.remove = imx8qxp_pxl2dpi_bridge_remove,
+	.remove_new = imx8qxp_pxl2dpi_bridge_remove,
 	.driver	= {
 		.of_match_table = imx8qxp_pxl2dpi_dt_ids,
 		.name = DRIVER_NAME,
diff --git a/drivers/gpu/drm/bridge/ite-it6505.c b/drivers/gpu/drm/bridge/ite-it6505.c
index bc451b2a77c2..abaf6e23775e 100644
--- a/drivers/gpu/drm/bridge/ite-it6505.c
+++ b/drivers/gpu/drm/bridge/ite-it6505.c
@@ -258,12 +258,12 @@
 #define REG_AUD_INFOFRAM_SUM 0xFB
 
 /* the following six registers are in bank1 */
-#define REG_DRV_0_DB_800_MV 0x7E
-#define REG_PRE_0_DB_800_MV 0x7F
-#define REG_PRE_3P5_DB_800_MV 0x81
-#define REG_SSC_CTRL0 0x88
-#define REG_SSC_CTRL1 0x89
-#define REG_SSC_CTRL2 0x8A
+#define REG_DRV_0_DB_800_MV 0x17E
+#define REG_PRE_0_DB_800_MV 0x17F
+#define REG_PRE_3P5_DB_800_MV 0x181
+#define REG_SSC_CTRL0 0x188
+#define REG_SSC_CTRL1 0x189
+#define REG_SSC_CTRL2 0x18A
 
 #define RBR DP_LINK_BW_1_62
 #define HBR DP_LINK_BW_2_7
@@ -489,7 +489,7 @@ static const struct it6505_audio_sample_rate_map audio_sample_rate_map[] = {
 };
 
 static const struct regmap_range it6505_bridge_volatile_ranges[] = {
-	{ .range_min = 0, .range_max = 0xFF },
+	{ .range_min = 0, .range_max = 0x1FF },
 };
 
 static const struct regmap_access_table it6505_bridge_volatile_table = {
@@ -497,11 +497,27 @@ static const struct regmap_access_table it6505_bridge_volatile_table = {
 	.n_yes_ranges = ARRAY_SIZE(it6505_bridge_volatile_ranges),
 };
 
+static const struct regmap_range_cfg it6505_regmap_banks[] = {
+	{
+		.name = "it6505",
+		.range_min = 0x00,
+		.range_max = 0x1FF,
+		.selector_reg = REG_BANK_SEL,
+		.selector_mask = 0x1,
+		.selector_shift = 0,
+		.window_start = 0x00,
+		.window_len = 0x100,
+	},
+};
+
 static const struct regmap_config it6505_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
 	.volatile_table = &it6505_bridge_volatile_table,
 	.cache_type = REGCACHE_NONE,
+	.ranges = it6505_regmap_banks,
+	.num_ranges = ARRAY_SIZE(it6505_regmap_banks),
+	.max_register = 0x1FF,
 };
 
 static int it6505_read(struct it6505 *it6505, unsigned int reg_addr)
@@ -1267,7 +1283,6 @@ static void it6505_init(struct it6505 *it6505)
 	it6505_write(it6505, REG_TIME_STMP_CTRL,
 		     EN_SSC_GAT | EN_ENHANCE_VID_STMP | EN_ENHANCE_AUD_STMP);
 	it6505_write(it6505, REG_INFOFRAME_CTRL, 0x00);
-	it6505_write(it6505, REG_BANK_SEL, 0x01);
 	it6505_write(it6505, REG_DRV_0_DB_800_MV,
 		     afe_setting_table[it6505->afe_setting][0]);
 	it6505_write(it6505, REG_PRE_0_DB_800_MV,
@@ -1277,7 +1292,6 @@ static void it6505_init(struct it6505 *it6505)
 	it6505_write(it6505, REG_SSC_CTRL0, 0x9E);
 	it6505_write(it6505, REG_SSC_CTRL1, 0x1C);
 	it6505_write(it6505, REG_SSC_CTRL2, 0x42);
-	it6505_write(it6505, REG_BANK_SEL, 0x00);
 }
 
 static void it6505_video_disable(struct it6505 *it6505)
@@ -1506,11 +1520,9 @@ static void it6505_setup_ssc(struct it6505 *it6505)
 	it6505_set_bits(it6505, REG_TRAIN_CTRL0, SPREAD_AMP_5,
 			it6505->enable_ssc ? SPREAD_AMP_5 : 0x00);
 	if (it6505->enable_ssc) {
-		it6505_write(it6505, REG_BANK_SEL, 0x01);
 		it6505_write(it6505, REG_SSC_CTRL0, 0x9E);
 		it6505_write(it6505, REG_SSC_CTRL1, 0x1C);
 		it6505_write(it6505, REG_SSC_CTRL2, 0x42);
-		it6505_write(it6505, REG_BANK_SEL, 0x00);
 		it6505_write(it6505, REG_SP_CTRL0, 0x07);
 		it6505_write(it6505, REG_IP_CTRL1, 0x29);
 		it6505_write(it6505, REG_IP_CTRL2, 0x03);
diff --git a/drivers/gpu/drm/bridge/lontium-lt8912b.c b/drivers/gpu/drm/bridge/lontium-lt8912b.c
index b40baced1331..13c131ade268 100644
--- a/drivers/gpu/drm/bridge/lontium-lt8912b.c
+++ b/drivers/gpu/drm/bridge/lontium-lt8912b.c
@@ -504,7 +504,6 @@ static int lt8912_attach_dsi(struct lt8912 *lt)
 	dsi->format = MIPI_DSI_FMT_RGB888;
 
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO |
-			  MIPI_DSI_MODE_VIDEO_BURST |
 			  MIPI_DSI_MODE_LPM |
 			  MIPI_DSI_MODE_NO_EOT_PACKET;
 
diff --git a/drivers/gpu/drm/bridge/lvds-codec.c b/drivers/gpu/drm/bridge/lvds-codec.c
index 39e7004de720..67368f23d4aa 100644
--- a/drivers/gpu/drm/bridge/lvds-codec.c
+++ b/drivers/gpu/drm/bridge/lvds-codec.c
@@ -215,13 +215,11 @@ static int lvds_codec_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int lvds_codec_remove(struct platform_device *pdev)
+static void lvds_codec_remove(struct platform_device *pdev)
 {
 	struct lvds_codec *lvds_codec = platform_get_drvdata(pdev);
 
 	drm_bridge_remove(&lvds_codec->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id lvds_codec_match[] = {
@@ -243,7 +241,7 @@ MODULE_DEVICE_TABLE(of, lvds_codec_match);
 
 static struct platform_driver lvds_codec_driver = {
 	.probe	= lvds_codec_probe,
-	.remove	= lvds_codec_remove,
+	.remove_new = lvds_codec_remove,
 	.driver		= {
 		.name		= "lvds-codec",
 		.of_match_table	= lvds_codec_match,
diff --git a/drivers/gpu/drm/bridge/nwl-dsi.c b/drivers/gpu/drm/bridge/nwl-dsi.c
index 6dc2a4e191d7..4a5f5c4f5dcc 100644
--- a/drivers/gpu/drm/bridge/nwl-dsi.c
+++ b/drivers/gpu/drm/bridge/nwl-dsi.c
@@ -1199,7 +1199,7 @@ static int nwl_dsi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int nwl_dsi_remove(struct platform_device *pdev)
+static void nwl_dsi_remove(struct platform_device *pdev)
 {
 	struct nwl_dsi *dsi = platform_get_drvdata(pdev);
 
@@ -1207,12 +1207,11 @@ static int nwl_dsi_remove(struct platform_device *pdev)
 	mipi_dsi_host_unregister(&dsi->dsi_host);
 	drm_bridge_remove(&dsi->bridge);
 	pm_runtime_disable(&pdev->dev);
-	return 0;
 }
 
 static struct platform_driver nwl_dsi_driver = {
 	.probe		= nwl_dsi_probe,
-	.remove		= nwl_dsi_remove,
+	.remove_new	= nwl_dsi_remove,
 	.driver		= {
 		.of_match_table = nwl_dsi_dt_ids,
 		.name	= DRV_NAME,
diff --git a/drivers/gpu/drm/bridge/panel.c b/drivers/gpu/drm/bridge/panel.c
index e8aae3cdc73d..9316384b4474 100644
--- a/drivers/gpu/drm/bridge/panel.c
+++ b/drivers/gpu/drm/bridge/panel.c
@@ -81,6 +81,8 @@ static int panel_bridge_attach(struct drm_bridge *bridge,
 		return ret;
 	}
 
+	drm_panel_bridge_set_orientation(connector, bridge);
+
 	drm_connector_attach_encoder(&panel_bridge->connector,
 					  bridge->encoder);
 
@@ -109,30 +111,82 @@ static void panel_bridge_detach(struct drm_bridge *bridge)
 		drm_connector_cleanup(connector);
 }
 
-static void panel_bridge_pre_enable(struct drm_bridge *bridge)
+static void panel_bridge_atomic_pre_enable(struct drm_bridge *bridge,
+				struct drm_bridge_state *old_bridge_state)
 {
 	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
+	struct drm_atomic_state *atomic_state = old_bridge_state->base.state;
+	struct drm_encoder *encoder = bridge->encoder;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state;
+
+	crtc = drm_atomic_get_new_crtc_for_encoder(atomic_state, encoder);
+	if (!crtc)
+		return;
+
+	old_crtc_state = drm_atomic_get_old_crtc_state(atomic_state, crtc);
+	if (old_crtc_state && old_crtc_state->self_refresh_active)
+		return;
 
 	drm_panel_prepare(panel_bridge->panel);
 }
 
-static void panel_bridge_enable(struct drm_bridge *bridge)
+static void panel_bridge_atomic_enable(struct drm_bridge *bridge,
+				struct drm_bridge_state *old_bridge_state)
 {
 	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
+	struct drm_atomic_state *atomic_state = old_bridge_state->base.state;
+	struct drm_encoder *encoder = bridge->encoder;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state;
+
+	crtc = drm_atomic_get_new_crtc_for_encoder(atomic_state, encoder);
+	if (!crtc)
+		return;
+
+	old_crtc_state = drm_atomic_get_old_crtc_state(atomic_state, crtc);
+	if (old_crtc_state && old_crtc_state->self_refresh_active)
+		return;
 
 	drm_panel_enable(panel_bridge->panel);
 }
 
-static void panel_bridge_disable(struct drm_bridge *bridge)
+static void panel_bridge_atomic_disable(struct drm_bridge *bridge,
+				struct drm_bridge_state *old_bridge_state)
 {
 	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
+	struct drm_atomic_state *atomic_state = old_bridge_state->base.state;
+	struct drm_encoder *encoder = bridge->encoder;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+
+	crtc = drm_atomic_get_old_crtc_for_encoder(atomic_state, encoder);
+	if (!crtc)
+		return;
+
+	new_crtc_state = drm_atomic_get_new_crtc_state(atomic_state, crtc);
+	if (new_crtc_state && new_crtc_state->self_refresh_active)
+		return;
 
 	drm_panel_disable(panel_bridge->panel);
 }
 
-static void panel_bridge_post_disable(struct drm_bridge *bridge)
+static void panel_bridge_atomic_post_disable(struct drm_bridge *bridge,
+				struct drm_bridge_state *old_bridge_state)
 {
 	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
+	struct drm_atomic_state *atomic_state = old_bridge_state->base.state;
+	struct drm_encoder *encoder = bridge->encoder;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+
+	crtc = drm_atomic_get_old_crtc_for_encoder(atomic_state, encoder);
+	if (!crtc)
+		return;
+
+	new_crtc_state = drm_atomic_get_new_crtc_state(atomic_state, crtc);
+	if (new_crtc_state && new_crtc_state->self_refresh_active)
+		return;
 
 	drm_panel_unprepare(panel_bridge->panel);
 }
@@ -159,10 +213,10 @@ static void panel_bridge_debugfs_init(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs panel_bridge_bridge_funcs = {
 	.attach = panel_bridge_attach,
 	.detach = panel_bridge_detach,
-	.pre_enable = panel_bridge_pre_enable,
-	.enable = panel_bridge_enable,
-	.disable = panel_bridge_disable,
-	.post_disable = panel_bridge_post_disable,
+	.atomic_pre_enable = panel_bridge_atomic_pre_enable,
+	.atomic_enable = panel_bridge_atomic_enable,
+	.atomic_disable = panel_bridge_atomic_disable,
+	.atomic_post_disable = panel_bridge_atomic_post_disable,
 	.get_modes = panel_bridge_get_modes,
 	.atomic_reset = drm_atomic_helper_bridge_reset,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
diff --git a/drivers/gpu/drm/bridge/parade-ps8622.c b/drivers/gpu/drm/bridge/parade-ps8622.c
index 530ee6a19e7e..efa80e309b98 100644
--- a/drivers/gpu/drm/bridge/parade-ps8622.c
+++ b/drivers/gpu/drm/bridge/parade-ps8622.c
@@ -496,7 +496,7 @@ static int ps8622_probe(struct i2c_client *client)
 		ps8622->lane_count = ps8622->max_lane_count;
 	}
 
-	if (!of_find_property(dev->of_node, "use-external-pwm", NULL)) {
+	if (!of_property_read_bool(dev->of_node, "use-external-pwm")) {
 		ps8622->bl = backlight_device_register("ps8622-backlight",
 				dev, ps8622, &ps8622_backlight_ops,
 				NULL);
diff --git a/drivers/gpu/drm/bridge/parade-ps8640.c b/drivers/gpu/drm/bridge/parade-ps8640.c
index 4b361d7d5e44..c3eb45179405 100644
--- a/drivers/gpu/drm/bridge/parade-ps8640.c
+++ b/drivers/gpu/drm/bridge/parade-ps8640.c
@@ -105,6 +105,7 @@ struct ps8640 {
 	struct gpio_desc *gpio_reset;
 	struct gpio_desc *gpio_powerdown;
 	struct device_link *link;
+	struct edid *edid;
 	bool pre_enabled;
 	bool need_post_hpd_delay;
 };
@@ -183,7 +184,7 @@ static int _ps8640_wait_hpd_asserted(struct ps8640 *ps_bridge, unsigned long wai
 	 * actually connected to GPIO9).
 	 */
 	ret = regmap_read_poll_timeout(map, PAGE2_GPIO_H, status,
-				       status & PS_GPIO9, wait_us / 10, wait_us);
+				       status & PS_GPIO9, 20000, wait_us);
 
 	/*
 	 * The first time we see HPD go high after a reset we delay an extra
@@ -542,34 +543,44 @@ static struct edid *ps8640_bridge_get_edid(struct drm_bridge *bridge,
 					   struct drm_connector *connector)
 {
 	struct ps8640 *ps_bridge = bridge_to_ps8640(bridge);
+	struct device *dev = &ps_bridge->page[PAGE0_DP_CNTL]->dev;
 	bool poweroff = !ps_bridge->pre_enabled;
-	struct edid *edid;
 
-	/*
-	 * When we end calling get_edid() triggered by an ioctl, i.e
-	 *
-	 *   drm_mode_getconnector (ioctl)
-	 *     -> drm_helper_probe_single_connector_modes
-	 *        -> drm_bridge_connector_get_modes
-	 *           -> ps8640_bridge_get_edid
-	 *
-	 * We need to make sure that what we need is enabled before reading
-	 * EDID, for this chip, we need to do a full poweron, otherwise it will
-	 * fail.
-	 */
-	drm_atomic_bridge_chain_pre_enable(bridge, connector->state->state);
+	if (!ps_bridge->edid) {
+		/*
+		 * When we end calling get_edid() triggered by an ioctl, i.e
+		 *
+		 *   drm_mode_getconnector (ioctl)
+		 *     -> drm_helper_probe_single_connector_modes
+		 *        -> drm_bridge_connector_get_modes
+		 *           -> ps8640_bridge_get_edid
+		 *
+		 * We need to make sure that what we need is enabled before
+		 * reading EDID, for this chip, we need to do a full poweron,
+		 * otherwise it will fail.
+		 */
+		if (poweroff)
+			drm_atomic_bridge_chain_pre_enable(bridge,
+							   connector->state->state);
 
-	edid = drm_get_edid(connector,
-			    ps_bridge->page[PAGE0_DP_CNTL]->adapter);
+		ps_bridge->edid = drm_get_edid(connector,
+					       ps_bridge->page[PAGE0_DP_CNTL]->adapter);
 
-	/*
-	 * If we call the get_edid() function without having enabled the chip
-	 * before, return the chip to its original power state.
-	 */
-	if (poweroff)
-		drm_atomic_bridge_chain_post_disable(bridge, connector->state->state);
+		/*
+		 * If we call the get_edid() function without having enabled the
+		 * chip before, return the chip to its original power state.
+		 */
+		if (poweroff)
+			drm_atomic_bridge_chain_post_disable(bridge,
+							     connector->state->state);
+	}
+
+	if (!ps_bridge->edid) {
+		dev_err(dev, "Failed to get EDID\n");
+		return NULL;
+	}
 
-	return edid;
+	return drm_edid_duplicate(ps_bridge->edid);
 }
 
 static void ps8640_runtime_disable(void *data)
@@ -766,6 +777,13 @@ static int ps8640_probe(struct i2c_client *client)
 	return ret;
 }
 
+static void ps8640_remove(struct i2c_client *client)
+{
+	struct ps8640 *ps_bridge = i2c_get_clientdata(client);
+
+	kfree(ps_bridge->edid);
+}
+
 static const struct of_device_id ps8640_match[] = {
 	{ .compatible = "parade,ps8640" },
 	{ }
@@ -774,6 +792,7 @@ MODULE_DEVICE_TABLE(of, ps8640_match);
 
 static struct i2c_driver ps8640_driver = {
 	.probe_new = ps8640_probe,
+	.remove = ps8640_remove,
 	.driver = {
 		.name = "ps8640",
 		.of_match_table = ps8640_match,
diff --git a/drivers/gpu/drm/bridge/samsung-dsim.c b/drivers/gpu/drm/bridge/samsung-dsim.c
new file mode 100644
index 000000000000..e0a402a85787
--- /dev/null
+++ b/drivers/gpu/drm/bridge/samsung-dsim.c
@@ -0,0 +1,1967 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Samsung MIPI DSIM bridge driver.
+ *
+ * Copyright (C) 2021 Amarula Solutions(India)
+ * Copyright (c) 2014 Samsung Electronics Co., Ltd
+ * Author: Jagan Teki <jagan@amarulasolutions.com>
+ *
+ * Based on exynos_drm_dsi from
+ * Tomasz Figa <t.figa@samsung.com>
+ */
+
+#include <asm/unaligned.h>
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/irq.h>
+#include <linux/media-bus-format.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/bridge/samsung-dsim.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_print.h>
+
+/* returns true iff both arguments logically differs */
+#define NEQV(a, b) (!(a) ^ !(b))
+
+/* DSIM_STATUS */
+#define DSIM_STOP_STATE_DAT(x)		(((x) & 0xf) << 0)
+#define DSIM_STOP_STATE_CLK		BIT(8)
+#define DSIM_TX_READY_HS_CLK		BIT(10)
+#define DSIM_PLL_STABLE			BIT(31)
+
+/* DSIM_SWRST */
+#define DSIM_FUNCRST			BIT(16)
+#define DSIM_SWRST			BIT(0)
+
+/* DSIM_TIMEOUT */
+#define DSIM_LPDR_TIMEOUT(x)		((x) << 0)
+#define DSIM_BTA_TIMEOUT(x)		((x) << 16)
+
+/* DSIM_CLKCTRL */
+#define DSIM_ESC_PRESCALER(x)		(((x) & 0xffff) << 0)
+#define DSIM_ESC_PRESCALER_MASK		(0xffff << 0)
+#define DSIM_LANE_ESC_CLK_EN_CLK	BIT(19)
+#define DSIM_LANE_ESC_CLK_EN_DATA(x)	(((x) & 0xf) << 20)
+#define DSIM_LANE_ESC_CLK_EN_DATA_MASK	(0xf << 20)
+#define DSIM_BYTE_CLKEN			BIT(24)
+#define DSIM_BYTE_CLK_SRC(x)		(((x) & 0x3) << 25)
+#define DSIM_BYTE_CLK_SRC_MASK		(0x3 << 25)
+#define DSIM_PLL_BYPASS			BIT(27)
+#define DSIM_ESC_CLKEN			BIT(28)
+#define DSIM_TX_REQUEST_HSCLK		BIT(31)
+
+/* DSIM_CONFIG */
+#define DSIM_LANE_EN_CLK		BIT(0)
+#define DSIM_LANE_EN(x)			(((x) & 0xf) << 1)
+#define DSIM_NUM_OF_DATA_LANE(x)	(((x) & 0x3) << 5)
+#define DSIM_SUB_PIX_FORMAT(x)		(((x) & 0x7) << 8)
+#define DSIM_MAIN_PIX_FORMAT_MASK	(0x7 << 12)
+#define DSIM_MAIN_PIX_FORMAT_RGB888	(0x7 << 12)
+#define DSIM_MAIN_PIX_FORMAT_RGB666	(0x6 << 12)
+#define DSIM_MAIN_PIX_FORMAT_RGB666_P	(0x5 << 12)
+#define DSIM_MAIN_PIX_FORMAT_RGB565	(0x4 << 12)
+#define DSIM_SUB_VC			(((x) & 0x3) << 16)
+#define DSIM_MAIN_VC			(((x) & 0x3) << 18)
+#define DSIM_HSA_DISABLE_MODE		BIT(20)
+#define DSIM_HBP_DISABLE_MODE		BIT(21)
+#define DSIM_HFP_DISABLE_MODE		BIT(22)
+/*
+ * The i.MX 8M Mini Applications Processor Reference Manual,
+ * Rev. 3, 11/2020 Page 4091
+ * The i.MX 8M Nano Applications Processor Reference Manual,
+ * Rev. 2, 07/2022 Page 3058
+ * The i.MX 8M Plus Applications Processor Reference Manual,
+ * Rev. 1, 06/2021 Page 5436
+ * all claims this bit is 'HseDisableMode' with the definition
+ * 0 = Disables transfer
+ * 1 = Enables transfer
+ *
+ * This clearly states that HSE is not a disabled bit.
+ *
+ * The naming convention follows as per the manual and the
+ * driver logic is based on the MIPI_DSI_MODE_VIDEO_HSE flag.
+ */
+#define DSIM_HSE_DISABLE_MODE		BIT(23)
+#define DSIM_AUTO_MODE			BIT(24)
+#define DSIM_VIDEO_MODE			BIT(25)
+#define DSIM_BURST_MODE			BIT(26)
+#define DSIM_SYNC_INFORM		BIT(27)
+#define DSIM_EOT_DISABLE		BIT(28)
+#define DSIM_MFLUSH_VS			BIT(29)
+/* This flag is valid only for exynos3250/3472/5260/5430 */
+#define DSIM_CLKLANE_STOP		BIT(30)
+
+/* DSIM_ESCMODE */
+#define DSIM_TX_TRIGGER_RST		BIT(4)
+#define DSIM_TX_LPDT_LP			BIT(6)
+#define DSIM_CMD_LPDT_LP		BIT(7)
+#define DSIM_FORCE_BTA			BIT(16)
+#define DSIM_FORCE_STOP_STATE		BIT(20)
+#define DSIM_STOP_STATE_CNT(x)		(((x) & 0x7ff) << 21)
+#define DSIM_STOP_STATE_CNT_MASK	(0x7ff << 21)
+
+/* DSIM_MDRESOL */
+#define DSIM_MAIN_STAND_BY		BIT(31)
+#define DSIM_MAIN_VRESOL(x, num_bits)	(((x) & ((1 << (num_bits)) - 1)) << 16)
+#define DSIM_MAIN_HRESOL(x, num_bits)	(((x) & ((1 << (num_bits)) - 1)) << 0)
+
+/* DSIM_MVPORCH */
+#define DSIM_CMD_ALLOW(x)		((x) << 28)
+#define DSIM_STABLE_VFP(x)		((x) << 16)
+#define DSIM_MAIN_VBP(x)		((x) << 0)
+#define DSIM_CMD_ALLOW_MASK		(0xf << 28)
+#define DSIM_STABLE_VFP_MASK		(0x7ff << 16)
+#define DSIM_MAIN_VBP_MASK		(0x7ff << 0)
+
+/* DSIM_MHPORCH */
+#define DSIM_MAIN_HFP(x)		((x) << 16)
+#define DSIM_MAIN_HBP(x)		((x) << 0)
+#define DSIM_MAIN_HFP_MASK		((0xffff) << 16)
+#define DSIM_MAIN_HBP_MASK		((0xffff) << 0)
+
+/* DSIM_MSYNC */
+#define DSIM_MAIN_VSA(x)		((x) << 22)
+#define DSIM_MAIN_HSA(x)		((x) << 0)
+#define DSIM_MAIN_VSA_MASK		((0x3ff) << 22)
+#define DSIM_MAIN_HSA_MASK		((0xffff) << 0)
+
+/* DSIM_SDRESOL */
+#define DSIM_SUB_STANDY(x)		((x) << 31)
+#define DSIM_SUB_VRESOL(x)		((x) << 16)
+#define DSIM_SUB_HRESOL(x)		((x) << 0)
+#define DSIM_SUB_STANDY_MASK		((0x1) << 31)
+#define DSIM_SUB_VRESOL_MASK		((0x7ff) << 16)
+#define DSIM_SUB_HRESOL_MASK		((0x7ff) << 0)
+
+/* DSIM_INTSRC */
+#define DSIM_INT_PLL_STABLE		BIT(31)
+#define DSIM_INT_SW_RST_RELEASE		BIT(30)
+#define DSIM_INT_SFR_FIFO_EMPTY		BIT(29)
+#define DSIM_INT_SFR_HDR_FIFO_EMPTY	BIT(28)
+#define DSIM_INT_BTA			BIT(25)
+#define DSIM_INT_FRAME_DONE		BIT(24)
+#define DSIM_INT_RX_TIMEOUT		BIT(21)
+#define DSIM_INT_BTA_TIMEOUT		BIT(20)
+#define DSIM_INT_RX_DONE		BIT(18)
+#define DSIM_INT_RX_TE			BIT(17)
+#define DSIM_INT_RX_ACK			BIT(16)
+#define DSIM_INT_RX_ECC_ERR		BIT(15)
+#define DSIM_INT_RX_CRC_ERR		BIT(14)
+
+/* DSIM_FIFOCTRL */
+#define DSIM_RX_DATA_FULL		BIT(25)
+#define DSIM_RX_DATA_EMPTY		BIT(24)
+#define DSIM_SFR_HEADER_FULL		BIT(23)
+#define DSIM_SFR_HEADER_EMPTY		BIT(22)
+#define DSIM_SFR_PAYLOAD_FULL		BIT(21)
+#define DSIM_SFR_PAYLOAD_EMPTY		BIT(20)
+#define DSIM_I80_HEADER_FULL		BIT(19)
+#define DSIM_I80_HEADER_EMPTY		BIT(18)
+#define DSIM_I80_PAYLOAD_FULL		BIT(17)
+#define DSIM_I80_PAYLOAD_EMPTY		BIT(16)
+#define DSIM_SD_HEADER_FULL		BIT(15)
+#define DSIM_SD_HEADER_EMPTY		BIT(14)
+#define DSIM_SD_PAYLOAD_FULL		BIT(13)
+#define DSIM_SD_PAYLOAD_EMPTY		BIT(12)
+#define DSIM_MD_HEADER_FULL		BIT(11)
+#define DSIM_MD_HEADER_EMPTY		BIT(10)
+#define DSIM_MD_PAYLOAD_FULL		BIT(9)
+#define DSIM_MD_PAYLOAD_EMPTY		BIT(8)
+#define DSIM_RX_FIFO			BIT(4)
+#define DSIM_SFR_FIFO			BIT(3)
+#define DSIM_I80_FIFO			BIT(2)
+#define DSIM_SD_FIFO			BIT(1)
+#define DSIM_MD_FIFO			BIT(0)
+
+/* DSIM_PHYACCHR */
+#define DSIM_AFC_EN			BIT(14)
+#define DSIM_AFC_CTL(x)			(((x) & 0x7) << 5)
+
+/* DSIM_PLLCTRL */
+#define DSIM_FREQ_BAND(x)		((x) << 24)
+#define DSIM_PLL_EN			BIT(23)
+#define DSIM_PLL_P(x, offset)		((x) << (offset))
+#define DSIM_PLL_M(x)			((x) << 4)
+#define DSIM_PLL_S(x)			((x) << 1)
+
+/* DSIM_PHYCTRL */
+#define DSIM_PHYCTRL_ULPS_EXIT(x)	(((x) & 0x1ff) << 0)
+#define DSIM_PHYCTRL_B_DPHYCTL_VREG_LP	BIT(30)
+#define DSIM_PHYCTRL_B_DPHYCTL_SLEW_UP	BIT(14)
+
+/* DSIM_PHYTIMING */
+#define DSIM_PHYTIMING_LPX(x)		((x) << 8)
+#define DSIM_PHYTIMING_HS_EXIT(x)	((x) << 0)
+
+/* DSIM_PHYTIMING1 */
+#define DSIM_PHYTIMING1_CLK_PREPARE(x)	((x) << 24)
+#define DSIM_PHYTIMING1_CLK_ZERO(x)	((x) << 16)
+#define DSIM_PHYTIMING1_CLK_POST(x)	((x) << 8)
+#define DSIM_PHYTIMING1_CLK_TRAIL(x)	((x) << 0)
+
+/* DSIM_PHYTIMING2 */
+#define DSIM_PHYTIMING2_HS_PREPARE(x)	((x) << 16)
+#define DSIM_PHYTIMING2_HS_ZERO(x)	((x) << 8)
+#define DSIM_PHYTIMING2_HS_TRAIL(x)	((x) << 0)
+
+#define DSI_MAX_BUS_WIDTH		4
+#define DSI_NUM_VIRTUAL_CHANNELS	4
+#define DSI_TX_FIFO_SIZE		2048
+#define DSI_RX_FIFO_SIZE		256
+#define DSI_XFER_TIMEOUT_MS		100
+#define DSI_RX_FIFO_EMPTY		0x30800002
+
+#define OLD_SCLK_MIPI_CLK_NAME		"pll_clk"
+
+static const char *const clk_names[5] = {
+	"bus_clk",
+	"sclk_mipi",
+	"phyclk_mipidphy0_bitclkdiv8",
+	"phyclk_mipidphy0_rxclkesc0",
+	"sclk_rgb_vclk_to_dsim0"
+};
+
+enum samsung_dsim_transfer_type {
+	EXYNOS_DSI_TX,
+	EXYNOS_DSI_RX,
+};
+
+enum reg_idx {
+	DSIM_STATUS_REG,	/* Status register */
+	DSIM_SWRST_REG,		/* Software reset register */
+	DSIM_CLKCTRL_REG,	/* Clock control register */
+	DSIM_TIMEOUT_REG,	/* Time out register */
+	DSIM_CONFIG_REG,	/* Configuration register */
+	DSIM_ESCMODE_REG,	/* Escape mode register */
+	DSIM_MDRESOL_REG,
+	DSIM_MVPORCH_REG,	/* Main display Vporch register */
+	DSIM_MHPORCH_REG,	/* Main display Hporch register */
+	DSIM_MSYNC_REG,		/* Main display sync area register */
+	DSIM_INTSRC_REG,	/* Interrupt source register */
+	DSIM_INTMSK_REG,	/* Interrupt mask register */
+	DSIM_PKTHDR_REG,	/* Packet Header FIFO register */
+	DSIM_PAYLOAD_REG,	/* Payload FIFO register */
+	DSIM_RXFIFO_REG,	/* Read FIFO register */
+	DSIM_FIFOCTRL_REG,	/* FIFO status and control register */
+	DSIM_PLLCTRL_REG,	/* PLL control register */
+	DSIM_PHYCTRL_REG,
+	DSIM_PHYTIMING_REG,
+	DSIM_PHYTIMING1_REG,
+	DSIM_PHYTIMING2_REG,
+	NUM_REGS
+};
+
+static const unsigned int exynos_reg_ofs[] = {
+	[DSIM_STATUS_REG] =  0x00,
+	[DSIM_SWRST_REG] =  0x04,
+	[DSIM_CLKCTRL_REG] =  0x08,
+	[DSIM_TIMEOUT_REG] =  0x0c,
+	[DSIM_CONFIG_REG] =  0x10,
+	[DSIM_ESCMODE_REG] =  0x14,
+	[DSIM_MDRESOL_REG] =  0x18,
+	[DSIM_MVPORCH_REG] =  0x1c,
+	[DSIM_MHPORCH_REG] =  0x20,
+	[DSIM_MSYNC_REG] =  0x24,
+	[DSIM_INTSRC_REG] =  0x2c,
+	[DSIM_INTMSK_REG] =  0x30,
+	[DSIM_PKTHDR_REG] =  0x34,
+	[DSIM_PAYLOAD_REG] =  0x38,
+	[DSIM_RXFIFO_REG] =  0x3c,
+	[DSIM_FIFOCTRL_REG] =  0x44,
+	[DSIM_PLLCTRL_REG] =  0x4c,
+	[DSIM_PHYCTRL_REG] =  0x5c,
+	[DSIM_PHYTIMING_REG] =  0x64,
+	[DSIM_PHYTIMING1_REG] =  0x68,
+	[DSIM_PHYTIMING2_REG] =  0x6c,
+};
+
+static const unsigned int exynos5433_reg_ofs[] = {
+	[DSIM_STATUS_REG] = 0x04,
+	[DSIM_SWRST_REG] = 0x0C,
+	[DSIM_CLKCTRL_REG] = 0x10,
+	[DSIM_TIMEOUT_REG] = 0x14,
+	[DSIM_CONFIG_REG] = 0x18,
+	[DSIM_ESCMODE_REG] = 0x1C,
+	[DSIM_MDRESOL_REG] = 0x20,
+	[DSIM_MVPORCH_REG] = 0x24,
+	[DSIM_MHPORCH_REG] = 0x28,
+	[DSIM_MSYNC_REG] = 0x2C,
+	[DSIM_INTSRC_REG] = 0x34,
+	[DSIM_INTMSK_REG] = 0x38,
+	[DSIM_PKTHDR_REG] = 0x3C,
+	[DSIM_PAYLOAD_REG] = 0x40,
+	[DSIM_RXFIFO_REG] = 0x44,
+	[DSIM_FIFOCTRL_REG] = 0x4C,
+	[DSIM_PLLCTRL_REG] = 0x94,
+	[DSIM_PHYCTRL_REG] = 0xA4,
+	[DSIM_PHYTIMING_REG] = 0xB4,
+	[DSIM_PHYTIMING1_REG] = 0xB8,
+	[DSIM_PHYTIMING2_REG] = 0xBC,
+};
+
+enum reg_value_idx {
+	RESET_TYPE,
+	PLL_TIMER,
+	STOP_STATE_CNT,
+	PHYCTRL_ULPS_EXIT,
+	PHYCTRL_VREG_LP,
+	PHYCTRL_SLEW_UP,
+	PHYTIMING_LPX,
+	PHYTIMING_HS_EXIT,
+	PHYTIMING_CLK_PREPARE,
+	PHYTIMING_CLK_ZERO,
+	PHYTIMING_CLK_POST,
+	PHYTIMING_CLK_TRAIL,
+	PHYTIMING_HS_PREPARE,
+	PHYTIMING_HS_ZERO,
+	PHYTIMING_HS_TRAIL
+};
+
+static const unsigned int reg_values[] = {
+	[RESET_TYPE] = DSIM_SWRST,
+	[PLL_TIMER] = 500,
+	[STOP_STATE_CNT] = 0xf,
+	[PHYCTRL_ULPS_EXIT] = DSIM_PHYCTRL_ULPS_EXIT(0x0af),
+	[PHYCTRL_VREG_LP] = 0,
+	[PHYCTRL_SLEW_UP] = 0,
+	[PHYTIMING_LPX] = DSIM_PHYTIMING_LPX(0x06),
+	[PHYTIMING_HS_EXIT] = DSIM_PHYTIMING_HS_EXIT(0x0b),
+	[PHYTIMING_CLK_PREPARE] = DSIM_PHYTIMING1_CLK_PREPARE(0x07),
+	[PHYTIMING_CLK_ZERO] = DSIM_PHYTIMING1_CLK_ZERO(0x27),
+	[PHYTIMING_CLK_POST] = DSIM_PHYTIMING1_CLK_POST(0x0d),
+	[PHYTIMING_CLK_TRAIL] = DSIM_PHYTIMING1_CLK_TRAIL(0x08),
+	[PHYTIMING_HS_PREPARE] = DSIM_PHYTIMING2_HS_PREPARE(0x09),
+	[PHYTIMING_HS_ZERO] = DSIM_PHYTIMING2_HS_ZERO(0x0d),
+	[PHYTIMING_HS_TRAIL] = DSIM_PHYTIMING2_HS_TRAIL(0x0b),
+};
+
+static const unsigned int exynos5422_reg_values[] = {
+	[RESET_TYPE] = DSIM_SWRST,
+	[PLL_TIMER] = 500,
+	[STOP_STATE_CNT] = 0xf,
+	[PHYCTRL_ULPS_EXIT] = DSIM_PHYCTRL_ULPS_EXIT(0xaf),
+	[PHYCTRL_VREG_LP] = 0,
+	[PHYCTRL_SLEW_UP] = 0,
+	[PHYTIMING_LPX] = DSIM_PHYTIMING_LPX(0x08),
+	[PHYTIMING_HS_EXIT] = DSIM_PHYTIMING_HS_EXIT(0x0d),
+	[PHYTIMING_CLK_PREPARE] = DSIM_PHYTIMING1_CLK_PREPARE(0x09),
+	[PHYTIMING_CLK_ZERO] = DSIM_PHYTIMING1_CLK_ZERO(0x30),
+	[PHYTIMING_CLK_POST] = DSIM_PHYTIMING1_CLK_POST(0x0e),
+	[PHYTIMING_CLK_TRAIL] = DSIM_PHYTIMING1_CLK_TRAIL(0x0a),
+	[PHYTIMING_HS_PREPARE] = DSIM_PHYTIMING2_HS_PREPARE(0x0c),
+	[PHYTIMING_HS_ZERO] = DSIM_PHYTIMING2_HS_ZERO(0x11),
+	[PHYTIMING_HS_TRAIL] = DSIM_PHYTIMING2_HS_TRAIL(0x0d),
+};
+
+static const unsigned int exynos5433_reg_values[] = {
+	[RESET_TYPE] = DSIM_FUNCRST,
+	[PLL_TIMER] = 22200,
+	[STOP_STATE_CNT] = 0xa,
+	[PHYCTRL_ULPS_EXIT] = DSIM_PHYCTRL_ULPS_EXIT(0x190),
+	[PHYCTRL_VREG_LP] = DSIM_PHYCTRL_B_DPHYCTL_VREG_LP,
+	[PHYCTRL_SLEW_UP] = DSIM_PHYCTRL_B_DPHYCTL_SLEW_UP,
+	[PHYTIMING_LPX] = DSIM_PHYTIMING_LPX(0x07),
+	[PHYTIMING_HS_EXIT] = DSIM_PHYTIMING_HS_EXIT(0x0c),
+	[PHYTIMING_CLK_PREPARE] = DSIM_PHYTIMING1_CLK_PREPARE(0x09),
+	[PHYTIMING_CLK_ZERO] = DSIM_PHYTIMING1_CLK_ZERO(0x2d),
+	[PHYTIMING_CLK_POST] = DSIM_PHYTIMING1_CLK_POST(0x0e),
+	[PHYTIMING_CLK_TRAIL] = DSIM_PHYTIMING1_CLK_TRAIL(0x09),
+	[PHYTIMING_HS_PREPARE] = DSIM_PHYTIMING2_HS_PREPARE(0x0b),
+	[PHYTIMING_HS_ZERO] = DSIM_PHYTIMING2_HS_ZERO(0x10),
+	[PHYTIMING_HS_TRAIL] = DSIM_PHYTIMING2_HS_TRAIL(0x0c),
+};
+
+static const unsigned int imx8mm_dsim_reg_values[] = {
+	[RESET_TYPE] = DSIM_SWRST,
+	[PLL_TIMER] = 500,
+	[STOP_STATE_CNT] = 0xf,
+	[PHYCTRL_ULPS_EXIT] = 0,
+	[PHYCTRL_VREG_LP] = 0,
+	[PHYCTRL_SLEW_UP] = 0,
+	[PHYTIMING_LPX] = DSIM_PHYTIMING_LPX(0x06),
+	[PHYTIMING_HS_EXIT] = DSIM_PHYTIMING_HS_EXIT(0x0b),
+	[PHYTIMING_CLK_PREPARE] = DSIM_PHYTIMING1_CLK_PREPARE(0x07),
+	[PHYTIMING_CLK_ZERO] = DSIM_PHYTIMING1_CLK_ZERO(0x26),
+	[PHYTIMING_CLK_POST] = DSIM_PHYTIMING1_CLK_POST(0x0d),
+	[PHYTIMING_CLK_TRAIL] = DSIM_PHYTIMING1_CLK_TRAIL(0x08),
+	[PHYTIMING_HS_PREPARE] = DSIM_PHYTIMING2_HS_PREPARE(0x08),
+	[PHYTIMING_HS_ZERO] = DSIM_PHYTIMING2_HS_ZERO(0x0d),
+	[PHYTIMING_HS_TRAIL] = DSIM_PHYTIMING2_HS_TRAIL(0x0b),
+};
+
+static const struct samsung_dsim_driver_data exynos3_dsi_driver_data = {
+	.reg_ofs = exynos_reg_ofs,
+	.plltmr_reg = 0x50,
+	.has_freqband = 1,
+	.has_clklane_stop = 1,
+	.num_clks = 2,
+	.max_freq = 1000,
+	.wait_for_reset = 1,
+	.num_bits_resol = 11,
+	.pll_p_offset = 13,
+	.reg_values = reg_values,
+};
+
+static const struct samsung_dsim_driver_data exynos4_dsi_driver_data = {
+	.reg_ofs = exynos_reg_ofs,
+	.plltmr_reg = 0x50,
+	.has_freqband = 1,
+	.has_clklane_stop = 1,
+	.num_clks = 2,
+	.max_freq = 1000,
+	.wait_for_reset = 1,
+	.num_bits_resol = 11,
+	.pll_p_offset = 13,
+	.reg_values = reg_values,
+};
+
+static const struct samsung_dsim_driver_data exynos5_dsi_driver_data = {
+	.reg_ofs = exynos_reg_ofs,
+	.plltmr_reg = 0x58,
+	.num_clks = 2,
+	.max_freq = 1000,
+	.wait_for_reset = 1,
+	.num_bits_resol = 11,
+	.pll_p_offset = 13,
+	.reg_values = reg_values,
+};
+
+static const struct samsung_dsim_driver_data exynos5433_dsi_driver_data = {
+	.reg_ofs = exynos5433_reg_ofs,
+	.plltmr_reg = 0xa0,
+	.has_clklane_stop = 1,
+	.num_clks = 5,
+	.max_freq = 1500,
+	.wait_for_reset = 0,
+	.num_bits_resol = 12,
+	.pll_p_offset = 13,
+	.reg_values = exynos5433_reg_values,
+};
+
+static const struct samsung_dsim_driver_data exynos5422_dsi_driver_data = {
+	.reg_ofs = exynos5433_reg_ofs,
+	.plltmr_reg = 0xa0,
+	.has_clklane_stop = 1,
+	.num_clks = 2,
+	.max_freq = 1500,
+	.wait_for_reset = 1,
+	.num_bits_resol = 12,
+	.pll_p_offset = 13,
+	.reg_values = exynos5422_reg_values,
+};
+
+static const struct samsung_dsim_driver_data imx8mm_dsi_driver_data = {
+	.reg_ofs = exynos5433_reg_ofs,
+	.plltmr_reg = 0xa0,
+	.has_clklane_stop = 1,
+	.num_clks = 2,
+	.max_freq = 2100,
+	.wait_for_reset = 0,
+	.num_bits_resol = 12,
+	/*
+	 * Unlike Exynos, PLL_P(PMS_P) offset 14 is used in i.MX8M Mini/Nano/Plus
+	 * downstream driver - drivers/gpu/drm/bridge/sec-dsim.c
+	 */
+	.pll_p_offset = 14,
+	.reg_values = imx8mm_dsim_reg_values,
+};
+
+static const struct samsung_dsim_driver_data *
+samsung_dsim_types[DSIM_TYPE_COUNT] = {
+	[DSIM_TYPE_EXYNOS3250] = &exynos3_dsi_driver_data,
+	[DSIM_TYPE_EXYNOS4210] = &exynos4_dsi_driver_data,
+	[DSIM_TYPE_EXYNOS5410] = &exynos5_dsi_driver_data,
+	[DSIM_TYPE_EXYNOS5422] = &exynos5422_dsi_driver_data,
+	[DSIM_TYPE_EXYNOS5433] = &exynos5433_dsi_driver_data,
+	[DSIM_TYPE_IMX8MM] = &imx8mm_dsi_driver_data,
+	[DSIM_TYPE_IMX8MP] = &imx8mm_dsi_driver_data,
+};
+
+static inline struct samsung_dsim *host_to_dsi(struct mipi_dsi_host *h)
+{
+	return container_of(h, struct samsung_dsim, dsi_host);
+}
+
+static inline struct samsung_dsim *bridge_to_dsi(struct drm_bridge *b)
+{
+	return container_of(b, struct samsung_dsim, bridge);
+}
+
+static inline void samsung_dsim_write(struct samsung_dsim *dsi,
+				      enum reg_idx idx, u32 val)
+{
+	writel(val, dsi->reg_base + dsi->driver_data->reg_ofs[idx]);
+}
+
+static inline u32 samsung_dsim_read(struct samsung_dsim *dsi, enum reg_idx idx)
+{
+	return readl(dsi->reg_base + dsi->driver_data->reg_ofs[idx]);
+}
+
+static void samsung_dsim_wait_for_reset(struct samsung_dsim *dsi)
+{
+	if (wait_for_completion_timeout(&dsi->completed, msecs_to_jiffies(300)))
+		return;
+
+	dev_err(dsi->dev, "timeout waiting for reset\n");
+}
+
+static void samsung_dsim_reset(struct samsung_dsim *dsi)
+{
+	u32 reset_val = dsi->driver_data->reg_values[RESET_TYPE];
+
+	reinit_completion(&dsi->completed);
+	samsung_dsim_write(dsi, DSIM_SWRST_REG, reset_val);
+}
+
+#ifndef MHZ
+#define MHZ	(1000 * 1000)
+#endif
+
+static unsigned long samsung_dsim_pll_find_pms(struct samsung_dsim *dsi,
+					       unsigned long fin,
+					       unsigned long fout,
+					       u8 *p, u16 *m, u8 *s)
+{
+	const struct samsung_dsim_driver_data *driver_data = dsi->driver_data;
+	unsigned long best_freq = 0;
+	u32 min_delta = 0xffffffff;
+	u8 p_min, p_max;
+	u8 _p, best_p;
+	u16 _m, best_m;
+	u8 _s, best_s;
+
+	p_min = DIV_ROUND_UP(fin, (12 * MHZ));
+	p_max = fin / (6 * MHZ);
+
+	for (_p = p_min; _p <= p_max; ++_p) {
+		for (_s = 0; _s <= 5; ++_s) {
+			u64 tmp;
+			u32 delta;
+
+			tmp = (u64)fout * (_p << _s);
+			do_div(tmp, fin);
+			_m = tmp;
+			if (_m < 41 || _m > 125)
+				continue;
+
+			tmp = (u64)_m * fin;
+			do_div(tmp, _p);
+			if (tmp < 500 * MHZ ||
+			    tmp > driver_data->max_freq * MHZ)
+				continue;
+
+			tmp = (u64)_m * fin;
+			do_div(tmp, _p << _s);
+
+			delta = abs(fout - tmp);
+			if (delta < min_delta) {
+				best_p = _p;
+				best_m = _m;
+				best_s = _s;
+				min_delta = delta;
+				best_freq = tmp;
+			}
+		}
+	}
+
+	if (best_freq) {
+		*p = best_p;
+		*m = best_m;
+		*s = best_s;
+	}
+
+	return best_freq;
+}
+
+static unsigned long samsung_dsim_set_pll(struct samsung_dsim *dsi,
+					  unsigned long freq)
+{
+	const struct samsung_dsim_driver_data *driver_data = dsi->driver_data;
+	unsigned long fin, fout;
+	int timeout;
+	u8 p, s;
+	u16 m;
+	u32 reg;
+
+	fin = dsi->pll_clk_rate;
+	fout = samsung_dsim_pll_find_pms(dsi, fin, freq, &p, &m, &s);
+	if (!fout) {
+		dev_err(dsi->dev,
+			"failed to find PLL PMS for requested frequency\n");
+		return 0;
+	}
+	dev_dbg(dsi->dev, "PLL freq %lu, (p %d, m %d, s %d)\n", fout, p, m, s);
+
+	writel(driver_data->reg_values[PLL_TIMER],
+	       dsi->reg_base + driver_data->plltmr_reg);
+
+	reg = DSIM_PLL_EN | DSIM_PLL_P(p, driver_data->pll_p_offset) |
+	      DSIM_PLL_M(m) | DSIM_PLL_S(s);
+
+	if (driver_data->has_freqband) {
+		static const unsigned long freq_bands[] = {
+			100 * MHZ, 120 * MHZ, 160 * MHZ, 200 * MHZ,
+			270 * MHZ, 320 * MHZ, 390 * MHZ, 450 * MHZ,
+			510 * MHZ, 560 * MHZ, 640 * MHZ, 690 * MHZ,
+			770 * MHZ, 870 * MHZ, 950 * MHZ,
+		};
+		int band;
+
+		for (band = 0; band < ARRAY_SIZE(freq_bands); ++band)
+			if (fout < freq_bands[band])
+				break;
+
+		dev_dbg(dsi->dev, "band %d\n", band);
+
+		reg |= DSIM_FREQ_BAND(band);
+	}
+
+	samsung_dsim_write(dsi, DSIM_PLLCTRL_REG, reg);
+
+	timeout = 1000;
+	do {
+		if (timeout-- == 0) {
+			dev_err(dsi->dev, "PLL failed to stabilize\n");
+			return 0;
+		}
+		reg = samsung_dsim_read(dsi, DSIM_STATUS_REG);
+	} while ((reg & DSIM_PLL_STABLE) == 0);
+
+	return fout;
+}
+
+static int samsung_dsim_enable_clock(struct samsung_dsim *dsi)
+{
+	unsigned long hs_clk, byte_clk, esc_clk;
+	unsigned long esc_div;
+	u32 reg;
+
+	hs_clk = samsung_dsim_set_pll(dsi, dsi->burst_clk_rate);
+	if (!hs_clk) {
+		dev_err(dsi->dev, "failed to configure DSI PLL\n");
+		return -EFAULT;
+	}
+
+	byte_clk = hs_clk / 8;
+	esc_div = DIV_ROUND_UP(byte_clk, dsi->esc_clk_rate);
+	esc_clk = byte_clk / esc_div;
+
+	if (esc_clk > 20 * MHZ) {
+		++esc_div;
+		esc_clk = byte_clk / esc_div;
+	}
+
+	dev_dbg(dsi->dev, "hs_clk = %lu, byte_clk = %lu, esc_clk = %lu\n",
+		hs_clk, byte_clk, esc_clk);
+
+	reg = samsung_dsim_read(dsi, DSIM_CLKCTRL_REG);
+	reg &= ~(DSIM_ESC_PRESCALER_MASK | DSIM_LANE_ESC_CLK_EN_CLK
+			| DSIM_LANE_ESC_CLK_EN_DATA_MASK | DSIM_PLL_BYPASS
+			| DSIM_BYTE_CLK_SRC_MASK);
+	reg |= DSIM_ESC_CLKEN | DSIM_BYTE_CLKEN
+			| DSIM_ESC_PRESCALER(esc_div)
+			| DSIM_LANE_ESC_CLK_EN_CLK
+			| DSIM_LANE_ESC_CLK_EN_DATA(BIT(dsi->lanes) - 1)
+			| DSIM_BYTE_CLK_SRC(0)
+			| DSIM_TX_REQUEST_HSCLK;
+	samsung_dsim_write(dsi, DSIM_CLKCTRL_REG, reg);
+
+	return 0;
+}
+
+static void samsung_dsim_set_phy_ctrl(struct samsung_dsim *dsi)
+{
+	const struct samsung_dsim_driver_data *driver_data = dsi->driver_data;
+	const unsigned int *reg_values = driver_data->reg_values;
+	u32 reg;
+
+	if (driver_data->has_freqband)
+		return;
+
+	/* B D-PHY: D-PHY Master & Slave Analog Block control */
+	reg = reg_values[PHYCTRL_ULPS_EXIT] | reg_values[PHYCTRL_VREG_LP] |
+		reg_values[PHYCTRL_SLEW_UP];
+	samsung_dsim_write(dsi, DSIM_PHYCTRL_REG, reg);
+
+	/*
+	 * T LPX: Transmitted length of any Low-Power state period
+	 * T HS-EXIT: Time that the transmitter drives LP-11 following a HS
+	 *	burst
+	 */
+	reg = reg_values[PHYTIMING_LPX] | reg_values[PHYTIMING_HS_EXIT];
+	samsung_dsim_write(dsi, DSIM_PHYTIMING_REG, reg);
+
+	/*
+	 * T CLK-PREPARE: Time that the transmitter drives the Clock Lane LP-00
+	 *	Line state immediately before the HS-0 Line state starting the
+	 *	HS transmission
+	 * T CLK-ZERO: Time that the transmitter drives the HS-0 state prior to
+	 *	transmitting the Clock.
+	 * T CLK_POST: Time that the transmitter continues to send HS clock
+	 *	after the last associated Data Lane has transitioned to LP Mode
+	 *	Interval is defined as the period from the end of T HS-TRAIL to
+	 *	the beginning of T CLK-TRAIL
+	 * T CLK-TRAIL: Time that the transmitter drives the HS-0 state after
+	 *	the last payload clock bit of a HS transmission burst
+	 */
+	reg = reg_values[PHYTIMING_CLK_PREPARE] |
+		reg_values[PHYTIMING_CLK_ZERO] |
+		reg_values[PHYTIMING_CLK_POST] |
+		reg_values[PHYTIMING_CLK_TRAIL];
+
+	samsung_dsim_write(dsi, DSIM_PHYTIMING1_REG, reg);
+
+	/*
+	 * T HS-PREPARE: Time that the transmitter drives the Data Lane LP-00
+	 *	Line state immediately before the HS-0 Line state starting the
+	 *	HS transmission
+	 * T HS-ZERO: Time that the transmitter drives the HS-0 state prior to
+	 *	transmitting the Sync sequence.
+	 * T HS-TRAIL: Time that the transmitter drives the flipped differential
+	 *	state after last payload data bit of a HS transmission burst
+	 */
+	reg = reg_values[PHYTIMING_HS_PREPARE] | reg_values[PHYTIMING_HS_ZERO] |
+		reg_values[PHYTIMING_HS_TRAIL];
+	samsung_dsim_write(dsi, DSIM_PHYTIMING2_REG, reg);
+}
+
+static void samsung_dsim_disable_clock(struct samsung_dsim *dsi)
+{
+	u32 reg;
+
+	reg = samsung_dsim_read(dsi, DSIM_CLKCTRL_REG);
+	reg &= ~(DSIM_LANE_ESC_CLK_EN_CLK | DSIM_LANE_ESC_CLK_EN_DATA_MASK
+			| DSIM_ESC_CLKEN | DSIM_BYTE_CLKEN);
+	samsung_dsim_write(dsi, DSIM_CLKCTRL_REG, reg);
+
+	reg = samsung_dsim_read(dsi, DSIM_PLLCTRL_REG);
+	reg &= ~DSIM_PLL_EN;
+	samsung_dsim_write(dsi, DSIM_PLLCTRL_REG, reg);
+}
+
+static void samsung_dsim_enable_lane(struct samsung_dsim *dsi, u32 lane)
+{
+	u32 reg = samsung_dsim_read(dsi, DSIM_CONFIG_REG);
+
+	reg |= (DSIM_NUM_OF_DATA_LANE(dsi->lanes - 1) | DSIM_LANE_EN_CLK |
+			DSIM_LANE_EN(lane));
+	samsung_dsim_write(dsi, DSIM_CONFIG_REG, reg);
+}
+
+static int samsung_dsim_init_link(struct samsung_dsim *dsi)
+{
+	const struct samsung_dsim_driver_data *driver_data = dsi->driver_data;
+	int timeout;
+	u32 reg;
+	u32 lanes_mask;
+
+	/* Initialize FIFO pointers */
+	reg = samsung_dsim_read(dsi, DSIM_FIFOCTRL_REG);
+	reg &= ~0x1f;
+	samsung_dsim_write(dsi, DSIM_FIFOCTRL_REG, reg);
+
+	usleep_range(9000, 11000);
+
+	reg |= 0x1f;
+	samsung_dsim_write(dsi, DSIM_FIFOCTRL_REG, reg);
+	usleep_range(9000, 11000);
+
+	/* DSI configuration */
+	reg = 0;
+
+	/*
+	 * The first bit of mode_flags specifies display configuration.
+	 * If this bit is set[= MIPI_DSI_MODE_VIDEO], dsi will support video
+	 * mode, otherwise it will support command mode.
+	 */
+	if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO) {
+		reg |= DSIM_VIDEO_MODE;
+
+		/*
+		 * The user manual describes that following bits are ignored in
+		 * command mode.
+		 */
+		if (!(dsi->mode_flags & MIPI_DSI_MODE_VSYNC_FLUSH))
+			reg |= DSIM_MFLUSH_VS;
+		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE)
+			reg |= DSIM_SYNC_INFORM;
+		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_BURST)
+			reg |= DSIM_BURST_MODE;
+		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_AUTO_VERT)
+			reg |= DSIM_AUTO_MODE;
+		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_HSE)
+			reg |= DSIM_HSE_DISABLE_MODE;
+		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_NO_HFP)
+			reg |= DSIM_HFP_DISABLE_MODE;
+		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_NO_HBP)
+			reg |= DSIM_HBP_DISABLE_MODE;
+		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_NO_HSA)
+			reg |= DSIM_HSA_DISABLE_MODE;
+	}
+
+	if (dsi->mode_flags & MIPI_DSI_MODE_NO_EOT_PACKET)
+		reg |= DSIM_EOT_DISABLE;
+
+	switch (dsi->format) {
+	case MIPI_DSI_FMT_RGB888:
+		reg |= DSIM_MAIN_PIX_FORMAT_RGB888;
+		break;
+	case MIPI_DSI_FMT_RGB666:
+		reg |= DSIM_MAIN_PIX_FORMAT_RGB666;
+		break;
+	case MIPI_DSI_FMT_RGB666_PACKED:
+		reg |= DSIM_MAIN_PIX_FORMAT_RGB666_P;
+		break;
+	case MIPI_DSI_FMT_RGB565:
+		reg |= DSIM_MAIN_PIX_FORMAT_RGB565;
+		break;
+	default:
+		dev_err(dsi->dev, "invalid pixel format\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Use non-continuous clock mode if the periparal wants and
+	 * host controller supports
+	 *
+	 * In non-continous clock mode, host controller will turn off
+	 * the HS clock between high-speed transmissions to reduce
+	 * power consumption.
+	 */
+	if (driver_data->has_clklane_stop &&
+	    dsi->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS)
+		reg |= DSIM_CLKLANE_STOP;
+	samsung_dsim_write(dsi, DSIM_CONFIG_REG, reg);
+
+	lanes_mask = BIT(dsi->lanes) - 1;
+	samsung_dsim_enable_lane(dsi, lanes_mask);
+
+	/* Check clock and data lane state are stop state */
+	timeout = 100;
+	do {
+		if (timeout-- == 0) {
+			dev_err(dsi->dev, "waiting for bus lanes timed out\n");
+			return -EFAULT;
+		}
+
+		reg = samsung_dsim_read(dsi, DSIM_STATUS_REG);
+		if ((reg & DSIM_STOP_STATE_DAT(lanes_mask))
+		    != DSIM_STOP_STATE_DAT(lanes_mask))
+			continue;
+	} while (!(reg & (DSIM_STOP_STATE_CLK | DSIM_TX_READY_HS_CLK)));
+
+	reg = samsung_dsim_read(dsi, DSIM_ESCMODE_REG);
+	reg &= ~DSIM_STOP_STATE_CNT_MASK;
+	reg |= DSIM_STOP_STATE_CNT(driver_data->reg_values[STOP_STATE_CNT]);
+	samsung_dsim_write(dsi, DSIM_ESCMODE_REG, reg);
+
+	reg = DSIM_BTA_TIMEOUT(0xff) | DSIM_LPDR_TIMEOUT(0xffff);
+	samsung_dsim_write(dsi, DSIM_TIMEOUT_REG, reg);
+
+	return 0;
+}
+
+static void samsung_dsim_set_display_mode(struct samsung_dsim *dsi)
+{
+	struct drm_display_mode *m = &dsi->mode;
+	unsigned int num_bits_resol = dsi->driver_data->num_bits_resol;
+	u32 reg;
+
+	if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO) {
+		reg = DSIM_CMD_ALLOW(0xf)
+			| DSIM_STABLE_VFP(m->vsync_start - m->vdisplay)
+			| DSIM_MAIN_VBP(m->vtotal - m->vsync_end);
+		samsung_dsim_write(dsi, DSIM_MVPORCH_REG, reg);
+
+		reg = DSIM_MAIN_HFP(m->hsync_start - m->hdisplay)
+			| DSIM_MAIN_HBP(m->htotal - m->hsync_end);
+		samsung_dsim_write(dsi, DSIM_MHPORCH_REG, reg);
+
+		reg = DSIM_MAIN_VSA(m->vsync_end - m->vsync_start)
+			| DSIM_MAIN_HSA(m->hsync_end - m->hsync_start);
+		samsung_dsim_write(dsi, DSIM_MSYNC_REG, reg);
+	}
+	reg =  DSIM_MAIN_HRESOL(m->hdisplay, num_bits_resol) |
+		DSIM_MAIN_VRESOL(m->vdisplay, num_bits_resol);
+
+	samsung_dsim_write(dsi, DSIM_MDRESOL_REG, reg);
+
+	dev_dbg(dsi->dev, "LCD size = %dx%d\n", m->hdisplay, m->vdisplay);
+}
+
+static void samsung_dsim_set_display_enable(struct samsung_dsim *dsi, bool enable)
+{
+	u32 reg;
+
+	reg = samsung_dsim_read(dsi, DSIM_MDRESOL_REG);
+	if (enable)
+		reg |= DSIM_MAIN_STAND_BY;
+	else
+		reg &= ~DSIM_MAIN_STAND_BY;
+	samsung_dsim_write(dsi, DSIM_MDRESOL_REG, reg);
+}
+
+static int samsung_dsim_wait_for_hdr_fifo(struct samsung_dsim *dsi)
+{
+	int timeout = 2000;
+
+	do {
+		u32 reg = samsung_dsim_read(dsi, DSIM_FIFOCTRL_REG);
+
+		if (!(reg & DSIM_SFR_HEADER_FULL))
+			return 0;
+
+		if (!cond_resched())
+			usleep_range(950, 1050);
+	} while (--timeout);
+
+	return -ETIMEDOUT;
+}
+
+static void samsung_dsim_set_cmd_lpm(struct samsung_dsim *dsi, bool lpm)
+{
+	u32 v = samsung_dsim_read(dsi, DSIM_ESCMODE_REG);
+
+	if (lpm)
+		v |= DSIM_CMD_LPDT_LP;
+	else
+		v &= ~DSIM_CMD_LPDT_LP;
+
+	samsung_dsim_write(dsi, DSIM_ESCMODE_REG, v);
+}
+
+static void samsung_dsim_force_bta(struct samsung_dsim *dsi)
+{
+	u32 v = samsung_dsim_read(dsi, DSIM_ESCMODE_REG);
+
+	v |= DSIM_FORCE_BTA;
+	samsung_dsim_write(dsi, DSIM_ESCMODE_REG, v);
+}
+
+static void samsung_dsim_send_to_fifo(struct samsung_dsim *dsi,
+				      struct samsung_dsim_transfer *xfer)
+{
+	struct device *dev = dsi->dev;
+	struct mipi_dsi_packet *pkt = &xfer->packet;
+	const u8 *payload = pkt->payload + xfer->tx_done;
+	u16 length = pkt->payload_length - xfer->tx_done;
+	bool first = !xfer->tx_done;
+	u32 reg;
+
+	dev_dbg(dev, "< xfer %pK: tx len %u, done %u, rx len %u, done %u\n",
+		xfer, length, xfer->tx_done, xfer->rx_len, xfer->rx_done);
+
+	if (length > DSI_TX_FIFO_SIZE)
+		length = DSI_TX_FIFO_SIZE;
+
+	xfer->tx_done += length;
+
+	/* Send payload */
+	while (length >= 4) {
+		reg = get_unaligned_le32(payload);
+		samsung_dsim_write(dsi, DSIM_PAYLOAD_REG, reg);
+		payload += 4;
+		length -= 4;
+	}
+
+	reg = 0;
+	switch (length) {
+	case 3:
+		reg |= payload[2] << 16;
+		fallthrough;
+	case 2:
+		reg |= payload[1] << 8;
+		fallthrough;
+	case 1:
+		reg |= payload[0];
+		samsung_dsim_write(dsi, DSIM_PAYLOAD_REG, reg);
+		break;
+	}
+
+	/* Send packet header */
+	if (!first)
+		return;
+
+	reg = get_unaligned_le32(pkt->header);
+	if (samsung_dsim_wait_for_hdr_fifo(dsi)) {
+		dev_err(dev, "waiting for header FIFO timed out\n");
+		return;
+	}
+
+	if (NEQV(xfer->flags & MIPI_DSI_MSG_USE_LPM,
+		 dsi->state & DSIM_STATE_CMD_LPM)) {
+		samsung_dsim_set_cmd_lpm(dsi, xfer->flags & MIPI_DSI_MSG_USE_LPM);
+		dsi->state ^= DSIM_STATE_CMD_LPM;
+	}
+
+	samsung_dsim_write(dsi, DSIM_PKTHDR_REG, reg);
+
+	if (xfer->flags & MIPI_DSI_MSG_REQ_ACK)
+		samsung_dsim_force_bta(dsi);
+}
+
+static void samsung_dsim_read_from_fifo(struct samsung_dsim *dsi,
+					struct samsung_dsim_transfer *xfer)
+{
+	u8 *payload = xfer->rx_payload + xfer->rx_done;
+	bool first = !xfer->rx_done;
+	struct device *dev = dsi->dev;
+	u16 length;
+	u32 reg;
+
+	if (first) {
+		reg = samsung_dsim_read(dsi, DSIM_RXFIFO_REG);
+
+		switch (reg & 0x3f) {
+		case MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_2BYTE:
+		case MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_2BYTE:
+			if (xfer->rx_len >= 2) {
+				payload[1] = reg >> 16;
+				++xfer->rx_done;
+			}
+			fallthrough;
+		case MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_1BYTE:
+		case MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_1BYTE:
+			payload[0] = reg >> 8;
+			++xfer->rx_done;
+			xfer->rx_len = xfer->rx_done;
+			xfer->result = 0;
+			goto clear_fifo;
+		case MIPI_DSI_RX_ACKNOWLEDGE_AND_ERROR_REPORT:
+			dev_err(dev, "DSI Error Report: 0x%04x\n", (reg >> 8) & 0xffff);
+			xfer->result = 0;
+			goto clear_fifo;
+		}
+
+		length = (reg >> 8) & 0xffff;
+		if (length > xfer->rx_len) {
+			dev_err(dev,
+				"response too long (%u > %u bytes), stripping\n",
+				xfer->rx_len, length);
+			length = xfer->rx_len;
+		} else if (length < xfer->rx_len) {
+			xfer->rx_len = length;
+		}
+	}
+
+	length = xfer->rx_len - xfer->rx_done;
+	xfer->rx_done += length;
+
+	/* Receive payload */
+	while (length >= 4) {
+		reg = samsung_dsim_read(dsi, DSIM_RXFIFO_REG);
+		payload[0] = (reg >>  0) & 0xff;
+		payload[1] = (reg >>  8) & 0xff;
+		payload[2] = (reg >> 16) & 0xff;
+		payload[3] = (reg >> 24) & 0xff;
+		payload += 4;
+		length -= 4;
+	}
+
+	if (length) {
+		reg = samsung_dsim_read(dsi, DSIM_RXFIFO_REG);
+		switch (length) {
+		case 3:
+			payload[2] = (reg >> 16) & 0xff;
+			fallthrough;
+		case 2:
+			payload[1] = (reg >> 8) & 0xff;
+			fallthrough;
+		case 1:
+			payload[0] = reg & 0xff;
+		}
+	}
+
+	if (xfer->rx_done == xfer->rx_len)
+		xfer->result = 0;
+
+clear_fifo:
+	length = DSI_RX_FIFO_SIZE / 4;
+	do {
+		reg = samsung_dsim_read(dsi, DSIM_RXFIFO_REG);
+		if (reg == DSI_RX_FIFO_EMPTY)
+			break;
+	} while (--length);
+}
+
+static void samsung_dsim_transfer_start(struct samsung_dsim *dsi)
+{
+	unsigned long flags;
+	struct samsung_dsim_transfer *xfer;
+	bool start = false;
+
+again:
+	spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+	if (list_empty(&dsi->transfer_list)) {
+		spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+		return;
+	}
+
+	xfer = list_first_entry(&dsi->transfer_list,
+				struct samsung_dsim_transfer, list);
+
+	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+
+	if (xfer->packet.payload_length &&
+	    xfer->tx_done == xfer->packet.payload_length)
+		/* waiting for RX */
+		return;
+
+	samsung_dsim_send_to_fifo(dsi, xfer);
+
+	if (xfer->packet.payload_length || xfer->rx_len)
+		return;
+
+	xfer->result = 0;
+	complete(&xfer->completed);
+
+	spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+	list_del_init(&xfer->list);
+	start = !list_empty(&dsi->transfer_list);
+
+	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+
+	if (start)
+		goto again;
+}
+
+static bool samsung_dsim_transfer_finish(struct samsung_dsim *dsi)
+{
+	struct samsung_dsim_transfer *xfer;
+	unsigned long flags;
+	bool start = true;
+
+	spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+	if (list_empty(&dsi->transfer_list)) {
+		spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+		return false;
+	}
+
+	xfer = list_first_entry(&dsi->transfer_list,
+				struct samsung_dsim_transfer, list);
+
+	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+
+	dev_dbg(dsi->dev,
+		"> xfer %pK, tx_len %zu, tx_done %u, rx_len %u, rx_done %u\n",
+		xfer, xfer->packet.payload_length, xfer->tx_done, xfer->rx_len,
+		xfer->rx_done);
+
+	if (xfer->tx_done != xfer->packet.payload_length)
+		return true;
+
+	if (xfer->rx_done != xfer->rx_len)
+		samsung_dsim_read_from_fifo(dsi, xfer);
+
+	if (xfer->rx_done != xfer->rx_len)
+		return true;
+
+	spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+	list_del_init(&xfer->list);
+	start = !list_empty(&dsi->transfer_list);
+
+	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+
+	if (!xfer->rx_len)
+		xfer->result = 0;
+	complete(&xfer->completed);
+
+	return start;
+}
+
+static void samsung_dsim_remove_transfer(struct samsung_dsim *dsi,
+					 struct samsung_dsim_transfer *xfer)
+{
+	unsigned long flags;
+	bool start;
+
+	spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+	if (!list_empty(&dsi->transfer_list) &&
+	    xfer == list_first_entry(&dsi->transfer_list,
+				     struct samsung_dsim_transfer, list)) {
+		list_del_init(&xfer->list);
+		start = !list_empty(&dsi->transfer_list);
+		spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+		if (start)
+			samsung_dsim_transfer_start(dsi);
+		return;
+	}
+
+	list_del_init(&xfer->list);
+
+	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+}
+
+static int samsung_dsim_transfer(struct samsung_dsim *dsi,
+				 struct samsung_dsim_transfer *xfer)
+{
+	unsigned long flags;
+	bool stopped;
+
+	xfer->tx_done = 0;
+	xfer->rx_done = 0;
+	xfer->result = -ETIMEDOUT;
+	init_completion(&xfer->completed);
+
+	spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+	stopped = list_empty(&dsi->transfer_list);
+	list_add_tail(&xfer->list, &dsi->transfer_list);
+
+	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+
+	if (stopped)
+		samsung_dsim_transfer_start(dsi);
+
+	wait_for_completion_timeout(&xfer->completed,
+				    msecs_to_jiffies(DSI_XFER_TIMEOUT_MS));
+	if (xfer->result == -ETIMEDOUT) {
+		struct mipi_dsi_packet *pkt = &xfer->packet;
+
+		samsung_dsim_remove_transfer(dsi, xfer);
+		dev_err(dsi->dev, "xfer timed out: %*ph %*ph\n", 4, pkt->header,
+			(int)pkt->payload_length, pkt->payload);
+		return -ETIMEDOUT;
+	}
+
+	/* Also covers hardware timeout condition */
+	return xfer->result;
+}
+
+static irqreturn_t samsung_dsim_irq(int irq, void *dev_id)
+{
+	struct samsung_dsim *dsi = dev_id;
+	u32 status;
+
+	status = samsung_dsim_read(dsi, DSIM_INTSRC_REG);
+	if (!status) {
+		static unsigned long j;
+
+		if (printk_timed_ratelimit(&j, 500))
+			dev_warn(dsi->dev, "spurious interrupt\n");
+		return IRQ_HANDLED;
+	}
+	samsung_dsim_write(dsi, DSIM_INTSRC_REG, status);
+
+	if (status & DSIM_INT_SW_RST_RELEASE) {
+		unsigned long mask = ~(DSIM_INT_RX_DONE |
+				       DSIM_INT_SFR_FIFO_EMPTY |
+				       DSIM_INT_SFR_HDR_FIFO_EMPTY |
+				       DSIM_INT_RX_ECC_ERR |
+				       DSIM_INT_SW_RST_RELEASE);
+		samsung_dsim_write(dsi, DSIM_INTMSK_REG, mask);
+		complete(&dsi->completed);
+		return IRQ_HANDLED;
+	}
+
+	if (!(status & (DSIM_INT_RX_DONE | DSIM_INT_SFR_FIFO_EMPTY |
+			DSIM_INT_PLL_STABLE)))
+		return IRQ_HANDLED;
+
+	if (samsung_dsim_transfer_finish(dsi))
+		samsung_dsim_transfer_start(dsi);
+
+	return IRQ_HANDLED;
+}
+
+static void samsung_dsim_enable_irq(struct samsung_dsim *dsi)
+{
+	enable_irq(dsi->irq);
+
+	if (dsi->te_gpio)
+		enable_irq(gpiod_to_irq(dsi->te_gpio));
+}
+
+static void samsung_dsim_disable_irq(struct samsung_dsim *dsi)
+{
+	if (dsi->te_gpio)
+		disable_irq(gpiod_to_irq(dsi->te_gpio));
+
+	disable_irq(dsi->irq);
+}
+
+static int samsung_dsim_init(struct samsung_dsim *dsi)
+{
+	const struct samsung_dsim_driver_data *driver_data = dsi->driver_data;
+
+	if (dsi->state & DSIM_STATE_INITIALIZED)
+		return 0;
+
+	samsung_dsim_reset(dsi);
+	samsung_dsim_enable_irq(dsi);
+
+	if (driver_data->reg_values[RESET_TYPE] == DSIM_FUNCRST)
+		samsung_dsim_enable_lane(dsi, BIT(dsi->lanes) - 1);
+
+	samsung_dsim_enable_clock(dsi);
+	if (driver_data->wait_for_reset)
+		samsung_dsim_wait_for_reset(dsi);
+	samsung_dsim_set_phy_ctrl(dsi);
+	samsung_dsim_init_link(dsi);
+
+	dsi->state |= DSIM_STATE_INITIALIZED;
+
+	return 0;
+}
+
+static void samsung_dsim_atomic_pre_enable(struct drm_bridge *bridge,
+					   struct drm_bridge_state *old_bridge_state)
+{
+	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
+	int ret;
+
+	if (dsi->state & DSIM_STATE_ENABLED)
+		return;
+
+	ret = pm_runtime_resume_and_get(dsi->dev);
+	if (ret < 0) {
+		dev_err(dsi->dev, "failed to enable DSI device.\n");
+		return;
+	}
+
+	dsi->state |= DSIM_STATE_ENABLED;
+
+	/*
+	 * For Exynos-DSIM the downstream bridge, or panel are expecting
+	 * the host initialization during DSI transfer.
+	 */
+	if (!samsung_dsim_hw_is_exynos(dsi->plat_data->hw_type)) {
+		ret = samsung_dsim_init(dsi);
+		if (ret)
+			return;
+	}
+}
+
+static void samsung_dsim_atomic_enable(struct drm_bridge *bridge,
+				       struct drm_bridge_state *old_bridge_state)
+{
+	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
+
+	samsung_dsim_set_display_mode(dsi);
+	samsung_dsim_set_display_enable(dsi, true);
+
+	dsi->state |= DSIM_STATE_VIDOUT_AVAILABLE;
+}
+
+static void samsung_dsim_atomic_disable(struct drm_bridge *bridge,
+					struct drm_bridge_state *old_bridge_state)
+{
+	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
+
+	if (!(dsi->state & DSIM_STATE_ENABLED))
+		return;
+
+	dsi->state &= ~DSIM_STATE_VIDOUT_AVAILABLE;
+}
+
+static void samsung_dsim_atomic_post_disable(struct drm_bridge *bridge,
+					     struct drm_bridge_state *old_bridge_state)
+{
+	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
+
+	samsung_dsim_set_display_enable(dsi, false);
+
+	dsi->state &= ~DSIM_STATE_ENABLED;
+	pm_runtime_put_sync(dsi->dev);
+}
+
+/*
+ * This pixel output formats list referenced from,
+ * AN13573 i.MX 8/RT MIPI DSI/CSI-2, Rev. 0, 21 March 2022
+ * 3.7.4 Pixel formats
+ * Table 14. DSI pixel packing formats
+ */
+static const u32 samsung_dsim_pixel_output_fmts[] = {
+	MEDIA_BUS_FMT_YUYV10_1X20,
+	MEDIA_BUS_FMT_YUYV12_1X24,
+	MEDIA_BUS_FMT_UYVY8_1X16,
+	MEDIA_BUS_FMT_RGB101010_1X30,
+	MEDIA_BUS_FMT_RGB121212_1X36,
+	MEDIA_BUS_FMT_RGB565_1X16,
+	MEDIA_BUS_FMT_RGB666_1X18,
+	MEDIA_BUS_FMT_RGB888_1X24,
+};
+
+static bool samsung_dsim_pixel_output_fmt_supported(u32 fmt)
+{
+	int i;
+
+	if (fmt == MEDIA_BUS_FMT_FIXED)
+		return false;
+
+	for (i = 0; i < ARRAY_SIZE(samsung_dsim_pixel_output_fmts); i++) {
+		if (samsung_dsim_pixel_output_fmts[i] == fmt)
+			return true;
+	}
+
+	return false;
+}
+
+static u32 *
+samsung_dsim_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
+				       struct drm_bridge_state *bridge_state,
+				       struct drm_crtc_state *crtc_state,
+				       struct drm_connector_state *conn_state,
+				       u32 output_fmt,
+				       unsigned int *num_input_fmts)
+{
+	u32 *input_fmts;
+
+	input_fmts = kmalloc(sizeof(*input_fmts), GFP_KERNEL);
+	if (!input_fmts)
+		return NULL;
+
+	if (!samsung_dsim_pixel_output_fmt_supported(output_fmt))
+		/*
+		 * Some bridge/display drivers are still not able to pass the
+		 * correct format, so handle those pipelines by falling back
+		 * to the default format till the supported formats finalized.
+		 */
+		output_fmt = MEDIA_BUS_FMT_RGB888_1X24;
+
+	input_fmts[0] = output_fmt;
+	*num_input_fmts = 1;
+
+	return input_fmts;
+}
+
+static int samsung_dsim_atomic_check(struct drm_bridge *bridge,
+				     struct drm_bridge_state *bridge_state,
+				     struct drm_crtc_state *crtc_state,
+				     struct drm_connector_state *conn_state)
+{
+	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
+	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
+
+	/*
+	 * The i.MX8M Mini/Nano glue logic between LCDIF and DSIM
+	 * inverts HS/VS/DE sync signals polarity, therefore, while
+	 * i.MX 8M Mini Applications Processor Reference Manual Rev. 3, 11/2020
+	 * 13.6.3.5.2 RGB interface
+	 * i.MX 8M Nano Applications Processor Reference Manual Rev. 2, 07/2022
+	 * 13.6.2.7.2 RGB interface
+	 * both claim "Vsync, Hsync, and VDEN are active high signals.", the
+	 * LCDIF must generate inverted HS/VS/DE signals, i.e. active LOW.
+	 *
+	 * The i.MX8M Plus glue logic between LCDIFv3 and DSIM does not
+	 * implement the same behavior, therefore LCDIFv3 must generate
+	 * HS/VS/DE signals active HIGH.
+	 */
+	if (dsi->plat_data->hw_type == DSIM_TYPE_IMX8MM) {
+		adjusted_mode->flags |= (DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC);
+		adjusted_mode->flags &= ~(DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC);
+	} else if (dsi->plat_data->hw_type == DSIM_TYPE_IMX8MP) {
+		adjusted_mode->flags &= ~(DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC);
+		adjusted_mode->flags |= (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC);
+	}
+
+	return 0;
+}
+
+static void samsung_dsim_mode_set(struct drm_bridge *bridge,
+				  const struct drm_display_mode *mode,
+				  const struct drm_display_mode *adjusted_mode)
+{
+	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
+
+	drm_mode_copy(&dsi->mode, adjusted_mode);
+}
+
+static int samsung_dsim_attach(struct drm_bridge *bridge,
+			       enum drm_bridge_attach_flags flags)
+{
+	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
+
+	return drm_bridge_attach(bridge->encoder, dsi->out_bridge, bridge,
+				 flags);
+}
+
+static const struct drm_bridge_funcs samsung_dsim_bridge_funcs = {
+	.atomic_duplicate_state		= drm_atomic_helper_bridge_duplicate_state,
+	.atomic_destroy_state		= drm_atomic_helper_bridge_destroy_state,
+	.atomic_reset			= drm_atomic_helper_bridge_reset,
+	.atomic_get_input_bus_fmts	= samsung_dsim_atomic_get_input_bus_fmts,
+	.atomic_check			= samsung_dsim_atomic_check,
+	.atomic_pre_enable		= samsung_dsim_atomic_pre_enable,
+	.atomic_enable			= samsung_dsim_atomic_enable,
+	.atomic_disable			= samsung_dsim_atomic_disable,
+	.atomic_post_disable		= samsung_dsim_atomic_post_disable,
+	.mode_set			= samsung_dsim_mode_set,
+	.attach				= samsung_dsim_attach,
+};
+
+static irqreturn_t samsung_dsim_te_irq_handler(int irq, void *dev_id)
+{
+	struct samsung_dsim *dsi = (struct samsung_dsim *)dev_id;
+	const struct samsung_dsim_plat_data *pdata = dsi->plat_data;
+
+	if (pdata->host_ops && pdata->host_ops->te_irq_handler)
+		return pdata->host_ops->te_irq_handler(dsi);
+
+	return IRQ_HANDLED;
+}
+
+static int samsung_dsim_register_te_irq(struct samsung_dsim *dsi, struct device *dev)
+{
+	int te_gpio_irq;
+	int ret;
+
+	dsi->te_gpio = devm_gpiod_get_optional(dev, "te", GPIOD_IN);
+	if (!dsi->te_gpio)
+		return 0;
+	else if (IS_ERR(dsi->te_gpio))
+		return dev_err_probe(dev, PTR_ERR(dsi->te_gpio), "failed to get te GPIO\n");
+
+	te_gpio_irq = gpiod_to_irq(dsi->te_gpio);
+
+	ret = request_threaded_irq(te_gpio_irq, samsung_dsim_te_irq_handler, NULL,
+				   IRQF_TRIGGER_RISING | IRQF_NO_AUTOEN, "TE", dsi);
+	if (ret) {
+		dev_err(dsi->dev, "request interrupt failed with %d\n", ret);
+		gpiod_put(dsi->te_gpio);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int samsung_dsim_host_attach(struct mipi_dsi_host *host,
+				    struct mipi_dsi_device *device)
+{
+	struct samsung_dsim *dsi = host_to_dsi(host);
+	const struct samsung_dsim_plat_data *pdata = dsi->plat_data;
+	struct device *dev = dsi->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node *remote;
+	struct drm_panel *panel;
+	int ret;
+
+	/*
+	 * Devices can also be child nodes when we also control that device
+	 * through the upstream device (ie, MIPI-DCS for a MIPI-DSI device).
+	 *
+	 * Lookup for a child node of the given parent that isn't either port
+	 * or ports.
+	 */
+	for_each_available_child_of_node(np, remote) {
+		if (of_node_name_eq(remote, "port") ||
+		    of_node_name_eq(remote, "ports"))
+			continue;
+
+		goto of_find_panel_or_bridge;
+	}
+
+	/*
+	 * of_graph_get_remote_node() produces a noisy error message if port
+	 * node isn't found and the absence of the port is a legit case here,
+	 * so at first we silently check whether graph presents in the
+	 * device-tree node.
+	 */
+	if (!of_graph_is_present(np))
+		return -ENODEV;
+
+	remote = of_graph_get_remote_node(np, 1, 0);
+
+of_find_panel_or_bridge:
+	if (!remote)
+		return -ENODEV;
+
+	panel = of_drm_find_panel(remote);
+	if (!IS_ERR(panel)) {
+		dsi->out_bridge = devm_drm_panel_bridge_add(dev, panel);
+	} else {
+		dsi->out_bridge = of_drm_find_bridge(remote);
+		if (!dsi->out_bridge)
+			dsi->out_bridge = ERR_PTR(-EINVAL);
+	}
+
+	of_node_put(remote);
+
+	if (IS_ERR(dsi->out_bridge)) {
+		ret = PTR_ERR(dsi->out_bridge);
+		DRM_DEV_ERROR(dev, "failed to find the bridge: %d\n", ret);
+		return ret;
+	}
+
+	DRM_DEV_INFO(dev, "Attached %s device\n", device->name);
+
+	drm_bridge_add(&dsi->bridge);
+
+	/*
+	 * This is a temporary solution and should be made by more generic way.
+	 *
+	 * If attached panel device is for command mode one, dsi should register
+	 * TE interrupt handler.
+	 */
+	if (!(device->mode_flags & MIPI_DSI_MODE_VIDEO)) {
+		ret = samsung_dsim_register_te_irq(dsi, &device->dev);
+		if (ret)
+			return ret;
+	}
+
+	if (pdata->host_ops && pdata->host_ops->attach) {
+		ret = pdata->host_ops->attach(dsi, device);
+		if (ret)
+			return ret;
+	}
+
+	dsi->lanes = device->lanes;
+	dsi->format = device->format;
+	dsi->mode_flags = device->mode_flags;
+
+	return 0;
+}
+
+static void samsung_dsim_unregister_te_irq(struct samsung_dsim *dsi)
+{
+	if (dsi->te_gpio) {
+		free_irq(gpiod_to_irq(dsi->te_gpio), dsi);
+		gpiod_put(dsi->te_gpio);
+	}
+}
+
+static int samsung_dsim_host_detach(struct mipi_dsi_host *host,
+				    struct mipi_dsi_device *device)
+{
+	struct samsung_dsim *dsi = host_to_dsi(host);
+	const struct samsung_dsim_plat_data *pdata = dsi->plat_data;
+
+	dsi->out_bridge = NULL;
+
+	if (pdata->host_ops && pdata->host_ops->detach)
+		pdata->host_ops->detach(dsi, device);
+
+	samsung_dsim_unregister_te_irq(dsi);
+
+	drm_bridge_remove(&dsi->bridge);
+
+	return 0;
+}
+
+static ssize_t samsung_dsim_host_transfer(struct mipi_dsi_host *host,
+					  const struct mipi_dsi_msg *msg)
+{
+	struct samsung_dsim *dsi = host_to_dsi(host);
+	struct samsung_dsim_transfer xfer;
+	int ret;
+
+	if (!(dsi->state & DSIM_STATE_ENABLED))
+		return -EINVAL;
+
+	ret = samsung_dsim_init(dsi);
+	if (ret)
+		return ret;
+
+	ret = mipi_dsi_create_packet(&xfer.packet, msg);
+	if (ret < 0)
+		return ret;
+
+	xfer.rx_len = msg->rx_len;
+	xfer.rx_payload = msg->rx_buf;
+	xfer.flags = msg->flags;
+
+	ret = samsung_dsim_transfer(dsi, &xfer);
+	return (ret < 0) ? ret : xfer.rx_done;
+}
+
+static const struct mipi_dsi_host_ops samsung_dsim_ops = {
+	.attach = samsung_dsim_host_attach,
+	.detach = samsung_dsim_host_detach,
+	.transfer = samsung_dsim_host_transfer,
+};
+
+static int samsung_dsim_of_read_u32(const struct device_node *np,
+				    const char *propname, u32 *out_value)
+{
+	int ret = of_property_read_u32(np, propname, out_value);
+
+	if (ret < 0)
+		pr_err("%pOF: failed to get '%s' property\n", np, propname);
+
+	return ret;
+}
+
+static int samsung_dsim_parse_dt(struct samsung_dsim *dsi)
+{
+	struct device *dev = dsi->dev;
+	struct device_node *node = dev->of_node;
+	int ret;
+
+	ret = samsung_dsim_of_read_u32(node, "samsung,pll-clock-frequency",
+				       &dsi->pll_clk_rate);
+	if (ret < 0)
+		return ret;
+
+	ret = samsung_dsim_of_read_u32(node, "samsung,burst-clock-frequency",
+				       &dsi->burst_clk_rate);
+	if (ret < 0)
+		return ret;
+
+	ret = samsung_dsim_of_read_u32(node, "samsung,esc-clock-frequency",
+				       &dsi->esc_clk_rate);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int generic_dsim_register_host(struct samsung_dsim *dsi)
+{
+	return mipi_dsi_host_register(&dsi->dsi_host);
+}
+
+static void generic_dsim_unregister_host(struct samsung_dsim *dsi)
+{
+	mipi_dsi_host_unregister(&dsi->dsi_host);
+}
+
+static const struct samsung_dsim_host_ops generic_dsim_host_ops = {
+	.register_host = generic_dsim_register_host,
+	.unregister_host = generic_dsim_unregister_host,
+};
+
+static const struct drm_bridge_timings samsung_dsim_bridge_timings_de_high = {
+	.input_bus_flags = DRM_BUS_FLAG_DE_HIGH,
+};
+
+static const struct drm_bridge_timings samsung_dsim_bridge_timings_de_low = {
+	.input_bus_flags = DRM_BUS_FLAG_DE_LOW,
+};
+
+int samsung_dsim_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct samsung_dsim *dsi;
+	int ret, i;
+
+	dsi = devm_kzalloc(dev, sizeof(*dsi), GFP_KERNEL);
+	if (!dsi)
+		return -ENOMEM;
+
+	init_completion(&dsi->completed);
+	spin_lock_init(&dsi->transfer_lock);
+	INIT_LIST_HEAD(&dsi->transfer_list);
+
+	dsi->dsi_host.ops = &samsung_dsim_ops;
+	dsi->dsi_host.dev = dev;
+
+	dsi->dev = dev;
+	dsi->plat_data = of_device_get_match_data(dev);
+	dsi->driver_data = samsung_dsim_types[dsi->plat_data->hw_type];
+
+	dsi->supplies[0].supply = "vddcore";
+	dsi->supplies[1].supply = "vddio";
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(dsi->supplies),
+				      dsi->supplies);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get regulators\n");
+
+	dsi->clks = devm_kcalloc(dev, dsi->driver_data->num_clks,
+				 sizeof(*dsi->clks), GFP_KERNEL);
+	if (!dsi->clks)
+		return -ENOMEM;
+
+	for (i = 0; i < dsi->driver_data->num_clks; i++) {
+		dsi->clks[i] = devm_clk_get(dev, clk_names[i]);
+		if (IS_ERR(dsi->clks[i])) {
+			if (strcmp(clk_names[i], "sclk_mipi") == 0) {
+				dsi->clks[i] = devm_clk_get(dev, OLD_SCLK_MIPI_CLK_NAME);
+				if (!IS_ERR(dsi->clks[i]))
+					continue;
+			}
+
+			dev_info(dev, "failed to get the clock: %s\n", clk_names[i]);
+			return PTR_ERR(dsi->clks[i]);
+		}
+	}
+
+	dsi->reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(dsi->reg_base))
+		return PTR_ERR(dsi->reg_base);
+
+	dsi->phy = devm_phy_optional_get(dev, "dsim");
+	if (IS_ERR(dsi->phy)) {
+		dev_info(dev, "failed to get dsim phy\n");
+		return PTR_ERR(dsi->phy);
+	}
+
+	dsi->irq = platform_get_irq(pdev, 0);
+	if (dsi->irq < 0)
+		return dsi->irq;
+
+	ret = devm_request_threaded_irq(dev, dsi->irq, NULL,
+					samsung_dsim_irq,
+					IRQF_ONESHOT | IRQF_NO_AUTOEN,
+					dev_name(dev), dsi);
+	if (ret) {
+		dev_err(dev, "failed to request dsi irq\n");
+		return ret;
+	}
+
+	ret = samsung_dsim_parse_dt(dsi);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, dsi);
+
+	pm_runtime_enable(dev);
+
+	dsi->bridge.funcs = &samsung_dsim_bridge_funcs;
+	dsi->bridge.of_node = dev->of_node;
+	dsi->bridge.type = DRM_MODE_CONNECTOR_DSI;
+
+	/* DE_LOW: i.MX8M Mini/Nano LCDIF-DSIM glue logic inverts HS/VS/DE */
+	if (dsi->plat_data->hw_type == DSIM_TYPE_IMX8MM)
+		dsi->bridge.timings = &samsung_dsim_bridge_timings_de_low;
+	else
+		dsi->bridge.timings = &samsung_dsim_bridge_timings_de_high;
+
+	if (dsi->plat_data->host_ops && dsi->plat_data->host_ops->register_host)
+		ret = dsi->plat_data->host_ops->register_host(dsi);
+
+	if (ret)
+		goto err_disable_runtime;
+
+	return 0;
+
+err_disable_runtime:
+	pm_runtime_disable(dev);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(samsung_dsim_probe);
+
+int samsung_dsim_remove(struct platform_device *pdev)
+{
+	struct samsung_dsim *dsi = platform_get_drvdata(pdev);
+
+	pm_runtime_disable(&pdev->dev);
+
+	if (dsi->plat_data->host_ops && dsi->plat_data->host_ops->unregister_host)
+		dsi->plat_data->host_ops->unregister_host(dsi);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(samsung_dsim_remove);
+
+static int __maybe_unused samsung_dsim_suspend(struct device *dev)
+{
+	struct samsung_dsim *dsi = dev_get_drvdata(dev);
+	const struct samsung_dsim_driver_data *driver_data = dsi->driver_data;
+	int ret, i;
+
+	usleep_range(10000, 20000);
+
+	if (dsi->state & DSIM_STATE_INITIALIZED) {
+		dsi->state &= ~DSIM_STATE_INITIALIZED;
+
+		samsung_dsim_disable_clock(dsi);
+
+		samsung_dsim_disable_irq(dsi);
+	}
+
+	dsi->state &= ~DSIM_STATE_CMD_LPM;
+
+	phy_power_off(dsi->phy);
+
+	for (i = driver_data->num_clks - 1; i > -1; i--)
+		clk_disable_unprepare(dsi->clks[i]);
+
+	ret = regulator_bulk_disable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
+	if (ret < 0)
+		dev_err(dsi->dev, "cannot disable regulators %d\n", ret);
+
+	return 0;
+}
+
+static int __maybe_unused samsung_dsim_resume(struct device *dev)
+{
+	struct samsung_dsim *dsi = dev_get_drvdata(dev);
+	const struct samsung_dsim_driver_data *driver_data = dsi->driver_data;
+	int ret, i;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
+	if (ret < 0) {
+		dev_err(dsi->dev, "cannot enable regulators %d\n", ret);
+		return ret;
+	}
+
+	for (i = 0; i < driver_data->num_clks; i++) {
+		ret = clk_prepare_enable(dsi->clks[i]);
+		if (ret < 0)
+			goto err_clk;
+	}
+
+	ret = phy_power_on(dsi->phy);
+	if (ret < 0) {
+		dev_err(dsi->dev, "cannot enable phy %d\n", ret);
+		goto err_clk;
+	}
+
+	return 0;
+
+err_clk:
+	while (--i > -1)
+		clk_disable_unprepare(dsi->clks[i]);
+	regulator_bulk_disable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
+
+	return ret;
+}
+
+const struct dev_pm_ops samsung_dsim_pm_ops = {
+	SET_RUNTIME_PM_OPS(samsung_dsim_suspend, samsung_dsim_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+};
+EXPORT_SYMBOL_GPL(samsung_dsim_pm_ops);
+
+static const struct samsung_dsim_plat_data samsung_dsim_imx8mm_pdata = {
+	.hw_type = DSIM_TYPE_IMX8MM,
+	.host_ops = &generic_dsim_host_ops,
+};
+
+static const struct samsung_dsim_plat_data samsung_dsim_imx8mp_pdata = {
+	.hw_type = DSIM_TYPE_IMX8MP,
+	.host_ops = &generic_dsim_host_ops,
+};
+
+static const struct of_device_id samsung_dsim_of_match[] = {
+	{
+		.compatible = "fsl,imx8mm-mipi-dsim",
+		.data = &samsung_dsim_imx8mm_pdata,
+	},
+	{
+		.compatible = "fsl,imx8mp-mipi-dsim",
+		.data = &samsung_dsim_imx8mp_pdata,
+	},
+	{ /* sentinel. */ }
+};
+MODULE_DEVICE_TABLE(of, samsung_dsim_of_match);
+
+static struct platform_driver samsung_dsim_driver = {
+	.probe = samsung_dsim_probe,
+	.remove = samsung_dsim_remove,
+	.driver = {
+		   .name = "samsung-dsim",
+		   .owner = THIS_MODULE,
+		   .pm = &samsung_dsim_pm_ops,
+		   .of_match_table = samsung_dsim_of_match,
+	},
+};
+
+module_platform_driver(samsung_dsim_driver);
+
+MODULE_AUTHOR("Jagan Teki <jagan@amarulasolutions.com>");
+MODULE_DESCRIPTION("Samsung MIPI DSIM controller bridge");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/bridge/sii9234.c b/drivers/gpu/drm/bridge/sii9234.c
index 099b510ff285..2d17f227867b 100644
--- a/drivers/gpu/drm/bridge/sii9234.c
+++ b/drivers/gpu/drm/bridge/sii9234.c
@@ -867,11 +867,6 @@ static int sii9234_init_resources(struct sii9234 *ctx,
 	return 0;
 }
 
-static inline struct sii9234 *bridge_to_sii9234(struct drm_bridge *bridge)
-{
-	return container_of(bridge, struct sii9234, bridge);
-}
-
 static enum drm_mode_status sii9234_mode_valid(struct drm_bridge *bridge,
 					 const struct drm_display_info *info,
 					 const struct drm_display_mode *mode)
diff --git a/drivers/gpu/drm/bridge/simple-bridge.c b/drivers/gpu/drm/bridge/simple-bridge.c
index 2c5c5211bdab..d85d9ee463b8 100644
--- a/drivers/gpu/drm/bridge/simple-bridge.c
+++ b/drivers/gpu/drm/bridge/simple-bridge.c
@@ -202,11 +202,9 @@ static int simple_bridge_probe(struct platform_device *pdev)
 
 	sbridge->enable = devm_gpiod_get_optional(&pdev->dev, "enable",
 						  GPIOD_OUT_LOW);
-	if (IS_ERR(sbridge->enable)) {
-		if (PTR_ERR(sbridge->enable) != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "Unable to retrieve enable GPIO\n");
-		return PTR_ERR(sbridge->enable);
-	}
+	if (IS_ERR(sbridge->enable))
+		return dev_err_probe(&pdev->dev, PTR_ERR(sbridge->enable),
+				     "Unable to retrieve enable GPIO\n");
 
 	/* Register the bridge. */
 	sbridge->bridge.funcs = &simple_bridge_bridge_funcs;
@@ -218,13 +216,11 @@ static int simple_bridge_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int simple_bridge_remove(struct platform_device *pdev)
+static void simple_bridge_remove(struct platform_device *pdev)
 {
 	struct simple_bridge *sbridge = platform_get_drvdata(pdev);
 
 	drm_bridge_remove(&sbridge->bridge);
-
-	return 0;
 }
 
 /*
@@ -301,7 +297,7 @@ MODULE_DEVICE_TABLE(of, simple_bridge_match);
 
 static struct platform_driver simple_bridge_driver = {
 	.probe	= simple_bridge_probe,
-	.remove	= simple_bridge_remove,
+	.remove_new = simple_bridge_remove,
 	.driver		= {
 		.name		= "simple-bridge",
 		.of_match_table	= simple_bridge_match,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
index 4efb62bcdb63..67b8d17a722a 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
@@ -584,13 +584,11 @@ err:
 	return ret;
 }
 
-static int snd_dw_hdmi_remove(struct platform_device *pdev)
+static void snd_dw_hdmi_remove(struct platform_device *pdev)
 {
 	struct snd_dw_hdmi *dw = platform_get_drvdata(pdev);
 
 	snd_card_free(dw->card);
-
-	return 0;
 }
 
 #if defined(CONFIG_PM_SLEEP) && defined(IS_NOT_BROKEN)
@@ -625,7 +623,7 @@ static SIMPLE_DEV_PM_OPS(snd_dw_hdmi_pm, snd_dw_hdmi_suspend,
 
 static struct platform_driver snd_dw_hdmi_driver = {
 	.probe	= snd_dw_hdmi_probe,
-	.remove	= snd_dw_hdmi_remove,
+	.remove_new = snd_dw_hdmi_remove,
 	.driver	= {
 		.name = DRIVER_NAME,
 		.pm = PM_OPS,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c
index c8f44bcb298a..9389ce526eb1 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c
@@ -296,19 +296,17 @@ static int dw_hdmi_cec_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int dw_hdmi_cec_remove(struct platform_device *pdev)
+static void dw_hdmi_cec_remove(struct platform_device *pdev)
 {
 	struct dw_hdmi_cec *cec = platform_get_drvdata(pdev);
 
 	cec_notifier_cec_adap_unregister(cec->notify, cec->adap);
 	cec_unregister_adapter(cec->adap);
-
-	return 0;
 }
 
 static struct platform_driver dw_hdmi_cec_driver = {
 	.probe	= dw_hdmi_cec_probe,
-	.remove	= dw_hdmi_cec_remove,
+	.remove_new = dw_hdmi_cec_remove,
 	.driver = {
 		.name = "dw-hdmi-cec",
 	},
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-gp-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-gp-audio.c
index 557966239677..423762da2ab4 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-gp-audio.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-gp-audio.c
@@ -172,18 +172,16 @@ static int snd_dw_hdmi_probe(struct platform_device *pdev)
 	return PTR_ERR_OR_ZERO(dw->audio_pdev);
 }
 
-static int snd_dw_hdmi_remove(struct platform_device *pdev)
+static void snd_dw_hdmi_remove(struct platform_device *pdev)
 {
 	struct snd_dw_hdmi *dw = platform_get_drvdata(pdev);
 
 	platform_device_unregister(dw->audio_pdev);
-
-	return 0;
 }
 
 static struct platform_driver snd_dw_hdmi_driver = {
 	.probe	= snd_dw_hdmi_probe,
-	.remove	= snd_dw_hdmi_remove,
+	.remove_new = snd_dw_hdmi_remove,
 	.driver	= {
 		.name = DRIVER_NAME,
 	},
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
index d751820c6da6..26c187d20d97 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
@@ -216,18 +216,16 @@ static int snd_dw_hdmi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int snd_dw_hdmi_remove(struct platform_device *pdev)
+static void snd_dw_hdmi_remove(struct platform_device *pdev)
 {
 	struct platform_device *platform = dev_get_drvdata(&pdev->dev);
 
 	platform_device_unregister(platform);
-
-	return 0;
 }
 
 static struct platform_driver snd_dw_hdmi_driver = {
 	.probe	= snd_dw_hdmi_probe,
-	.remove	= snd_dw_hdmi_remove,
+	.remove_new = snd_dw_hdmi_remove,
 	.driver	= {
 		.name = DRIVER_NAME,
 	},
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
index aa51c61a78c7..603bb3c51027 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
@@ -1426,9 +1426,9 @@ void dw_hdmi_set_high_tmds_clock_ratio(struct dw_hdmi *hdmi,
 	/* Control for TMDS Bit Period/TMDS Clock-Period Ratio */
 	if (dw_hdmi_support_scdc(hdmi, display)) {
 		if (mtmdsclock > HDMI14_MAX_TMDSCLK)
-			drm_scdc_set_high_tmds_clock_ratio(hdmi->ddc, 1);
+			drm_scdc_set_high_tmds_clock_ratio(&hdmi->connector, 1);
 		else
-			drm_scdc_set_high_tmds_clock_ratio(hdmi->ddc, 0);
+			drm_scdc_set_high_tmds_clock_ratio(&hdmi->connector, 0);
 	}
 }
 EXPORT_SYMBOL_GPL(dw_hdmi_set_high_tmds_clock_ratio);
@@ -2116,7 +2116,7 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi,
 				min_t(u8, bytes, SCDC_MIN_SOURCE_VERSION));
 
 			/* Enabled Scrambling in the Sink */
-			drm_scdc_set_scrambling(hdmi->ddc, 1);
+			drm_scdc_set_scrambling(&hdmi->connector, 1);
 
 			/*
 			 * To activate the scrambler feature, you must ensure
@@ -2132,7 +2132,7 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi,
 			hdmi_writeb(hdmi, 0, HDMI_FC_SCRAMBLER_CTRL);
 			hdmi_writeb(hdmi, (u8)~HDMI_MC_SWRSTZ_TMDSSWRST_REQ,
 				    HDMI_MC_SWRSTZ);
-			drm_scdc_set_scrambling(hdmi->ddc, 0);
+			drm_scdc_set_scrambling(&hdmi->connector, 0);
 		}
 	}
 
diff --git a/drivers/gpu/drm/bridge/tc358762.c b/drivers/gpu/drm/bridge/tc358762.c
index 0b6a28436885..77f7f7f54757 100644
--- a/drivers/gpu/drm/bridge/tc358762.c
+++ b/drivers/gpu/drm/bridge/tc358762.c
@@ -229,6 +229,7 @@ static int tc358762_probe(struct mipi_dsi_device *dsi)
 	ctx->bridge.funcs = &tc358762_bridge_funcs;
 	ctx->bridge.type = DRM_MODE_CONNECTOR_DPI;
 	ctx->bridge.of_node = dev->of_node;
+	ctx->bridge.pre_enable_prev_first = true;
 
 	drm_bridge_add(&ctx->bridge);
 
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index 6d16ec45ea61..91f7cb56a654 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -1896,10 +1896,10 @@ static int tc_mipi_dsi_host_attach(struct tc_data *tc)
 				     "failed to create dsi device\n");
 
 	tc->dsi = dsi;
-
 	dsi->lanes = dsi_lanes;
 	dsi->format = MIPI_DSI_FMT_RGB888;
-	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+			  MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS;
 
 	ret = mipi_dsi_attach(dsi);
 	if (ret < 0) {
diff --git a/drivers/gpu/drm/bridge/thc63lvd1024.c b/drivers/gpu/drm/bridge/thc63lvd1024.c
index e21078b2f8b5..d4c1a601bbb5 100644
--- a/drivers/gpu/drm/bridge/thc63lvd1024.c
+++ b/drivers/gpu/drm/bridge/thc63lvd1024.c
@@ -230,13 +230,11 @@ static int thc63_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int thc63_remove(struct platform_device *pdev)
+static void thc63_remove(struct platform_device *pdev)
 {
 	struct thc63_dev *thc63 = platform_get_drvdata(pdev);
 
 	drm_bridge_remove(&thc63->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id thc63_match[] = {
@@ -247,7 +245,7 @@ MODULE_DEVICE_TABLE(of, thc63_match);
 
 static struct platform_driver thc63_driver = {
 	.probe	= thc63_probe,
-	.remove	= thc63_remove,
+	.remove_new = thc63_remove,
 	.driver	= {
 		.name		= "thc63lvd1024",
 		.of_match_table	= thc63_match,
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
index 91ecfbe45bf9..75286c9afbb9 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
@@ -642,7 +642,9 @@ static int sn65dsi83_host_attach(struct sn65dsi83 *ctx)
 
 	dsi->lanes = dsi_lanes;
 	dsi->format = MIPI_DSI_FMT_RGB888;
-	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+			  MIPI_DSI_MODE_VIDEO_NO_HFP | MIPI_DSI_MODE_VIDEO_NO_HBP |
+			  MIPI_DSI_MODE_VIDEO_NO_HSA | MIPI_DSI_MODE_NO_EOT_PACKET;
 
 	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
@@ -698,8 +700,10 @@ static int sn65dsi83_probe(struct i2c_client *client)
 	drm_bridge_add(&ctx->bridge);
 
 	ret = sn65dsi83_host_attach(ctx);
-	if (ret)
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to attach DSI host\n");
 		goto err_remove_bridge;
+	}
 
 	return 0;
 
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c
index 1e26fa63845a..7a748785c545 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c
@@ -363,7 +363,7 @@ static int __maybe_unused ti_sn65dsi86_resume(struct device *dev)
 	/* td2: min 100 us after regulators before enabling the GPIO */
 	usleep_range(100, 110);
 
-	gpiod_set_value(pdata->enable_gpio, 1);
+	gpiod_set_value_cansleep(pdata->enable_gpio, 1);
 
 	/*
 	 * If we have a reference clock we can enable communication w/ the
@@ -386,7 +386,7 @@ static int __maybe_unused ti_sn65dsi86_suspend(struct device *dev)
 	if (pdata->refclk)
 		ti_sn65dsi86_disable_comms(pdata);
 
-	gpiod_set_value(pdata->enable_gpio, 0);
+	gpiod_set_value_cansleep(pdata->enable_gpio, 0);
 
 	ret = regulator_bulk_disable(SN_REGULATOR_SUPPLY_NUM, pdata->supplies);
 	if (ret)
diff --git a/drivers/gpu/drm/bridge/ti-tfp410.c b/drivers/gpu/drm/bridge/ti-tfp410.c
index 6db69df0e18b..ab63225cd635 100644
--- a/drivers/gpu/drm/bridge/ti-tfp410.c
+++ b/drivers/gpu/drm/bridge/ti-tfp410.c
@@ -355,11 +355,9 @@ static int tfp410_probe(struct platform_device *pdev)
 	return tfp410_init(&pdev->dev, false);
 }
 
-static int tfp410_remove(struct platform_device *pdev)
+static void tfp410_remove(struct platform_device *pdev)
 {
 	tfp410_fini(&pdev->dev);
-
-	return 0;
 }
 
 static const struct of_device_id tfp410_match[] = {
@@ -370,7 +368,7 @@ MODULE_DEVICE_TABLE(of, tfp410_match);
 
 static struct platform_driver tfp410_platform_driver = {
 	.probe	= tfp410_probe,
-	.remove	= tfp410_remove,
+	.remove_new = tfp410_remove,
 	.driver	= {
 		.name		= "tfp410-bridge",
 		.of_match_table	= tfp410_match,
diff --git a/drivers/gpu/drm/display/drm_dp_aux_dev.c b/drivers/gpu/drm/display/drm_dp_aux_dev.c
index 098e482e65a2..29555b9f03c8 100644
--- a/drivers/gpu/drm/display/drm_dp_aux_dev.c
+++ b/drivers/gpu/drm/display/drm_dp_aux_dev.c
@@ -330,7 +330,7 @@ int drm_dp_aux_dev_init(void)
 {
 	int res;
 
-	drm_dp_aux_dev_class = class_create(THIS_MODULE, "drm_dp_aux_dev");
+	drm_dp_aux_dev_class = class_create("drm_dp_aux_dev");
 	if (IS_ERR(drm_dp_aux_dev_class)) {
 		return PTR_ERR(drm_dp_aux_dev_class);
 	}
diff --git a/drivers/gpu/drm/display/drm_scdc_helper.c b/drivers/gpu/drm/display/drm_scdc_helper.c
index c3ad4ab2b456..6d2f244e5830 100644
--- a/drivers/gpu/drm/display/drm_scdc_helper.c
+++ b/drivers/gpu/drm/display/drm_scdc_helper.c
@@ -26,6 +26,8 @@
 #include <linux/delay.h>
 
 #include <drm/display/drm_scdc_helper.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_device.h>
 #include <drm/drm_print.h>
 
 /**
@@ -140,7 +142,7 @@ EXPORT_SYMBOL(drm_scdc_write);
 
 /**
  * drm_scdc_get_scrambling_status - what is status of scrambling?
- * @adapter: I2C adapter for DDC channel
+ * @connector: connector
  *
  * Reads the scrambler status over SCDC, and checks the
  * scrambling status.
@@ -148,14 +150,16 @@ EXPORT_SYMBOL(drm_scdc_write);
  * Returns:
  * True if the scrambling is enabled, false otherwise.
  */
-bool drm_scdc_get_scrambling_status(struct i2c_adapter *adapter)
+bool drm_scdc_get_scrambling_status(struct drm_connector *connector)
 {
 	u8 status;
 	int ret;
 
-	ret = drm_scdc_readb(adapter, SCDC_SCRAMBLER_STATUS, &status);
+	ret = drm_scdc_readb(connector->ddc, SCDC_SCRAMBLER_STATUS, &status);
 	if (ret < 0) {
-		DRM_DEBUG_KMS("Failed to read scrambling status: %d\n", ret);
+		drm_dbg_kms(connector->dev,
+			    "[CONNECTOR:%d:%s] Failed to read scrambling status: %d\n",
+			    connector->base.id, connector->name, ret);
 		return false;
 	}
 
@@ -165,7 +169,7 @@ EXPORT_SYMBOL(drm_scdc_get_scrambling_status);
 
 /**
  * drm_scdc_set_scrambling - enable scrambling
- * @adapter: I2C adapter for DDC channel
+ * @connector: connector
  * @enable: bool to indicate if scrambling is to be enabled/disabled
  *
  * Writes the TMDS config register over SCDC channel, and:
@@ -175,14 +179,17 @@ EXPORT_SYMBOL(drm_scdc_get_scrambling_status);
  * Returns:
  * True if scrambling is set/reset successfully, false otherwise.
  */
-bool drm_scdc_set_scrambling(struct i2c_adapter *adapter, bool enable)
+bool drm_scdc_set_scrambling(struct drm_connector *connector,
+			     bool enable)
 {
 	u8 config;
 	int ret;
 
-	ret = drm_scdc_readb(adapter, SCDC_TMDS_CONFIG, &config);
+	ret = drm_scdc_readb(connector->ddc, SCDC_TMDS_CONFIG, &config);
 	if (ret < 0) {
-		DRM_DEBUG_KMS("Failed to read TMDS config: %d\n", ret);
+		drm_dbg_kms(connector->dev,
+			    "[CONNECTOR:%d:%s] Failed to read TMDS config: %d\n",
+			    connector->base.id, connector->name, ret);
 		return false;
 	}
 
@@ -191,9 +198,11 @@ bool drm_scdc_set_scrambling(struct i2c_adapter *adapter, bool enable)
 	else
 		config &= ~SCDC_SCRAMBLING_ENABLE;
 
-	ret = drm_scdc_writeb(adapter, SCDC_TMDS_CONFIG, config);
+	ret = drm_scdc_writeb(connector->ddc, SCDC_TMDS_CONFIG, config);
 	if (ret < 0) {
-		DRM_DEBUG_KMS("Failed to enable scrambling: %d\n", ret);
+		drm_dbg_kms(connector->dev,
+			    "[CONNECTOR:%d:%s] Failed to enable scrambling: %d\n",
+			    connector->base.id, connector->name, ret);
 		return false;
 	}
 
@@ -203,7 +212,7 @@ EXPORT_SYMBOL(drm_scdc_set_scrambling);
 
 /**
  * drm_scdc_set_high_tmds_clock_ratio - set TMDS clock ratio
- * @adapter: I2C adapter for DDC channel
+ * @connector: connector
  * @set: ret or reset the high clock ratio
  *
  *
@@ -230,14 +239,17 @@ EXPORT_SYMBOL(drm_scdc_set_scrambling);
  * Returns:
  * True if write is successful, false otherwise.
  */
-bool drm_scdc_set_high_tmds_clock_ratio(struct i2c_adapter *adapter, bool set)
+bool drm_scdc_set_high_tmds_clock_ratio(struct drm_connector *connector,
+					bool set)
 {
 	u8 config;
 	int ret;
 
-	ret = drm_scdc_readb(adapter, SCDC_TMDS_CONFIG, &config);
+	ret = drm_scdc_readb(connector->ddc, SCDC_TMDS_CONFIG, &config);
 	if (ret < 0) {
-		DRM_DEBUG_KMS("Failed to read TMDS config: %d\n", ret);
+		drm_dbg_kms(connector->dev,
+			    "[CONNECTOR:%d:%s] Failed to read TMDS config: %d\n",
+			    connector->base.id, connector->name, ret);
 		return false;
 	}
 
@@ -246,9 +258,11 @@ bool drm_scdc_set_high_tmds_clock_ratio(struct i2c_adapter *adapter, bool set)
 	else
 		config &= ~SCDC_TMDS_BIT_CLOCK_RATIO_BY_40;
 
-	ret = drm_scdc_writeb(adapter, SCDC_TMDS_CONFIG, config);
+	ret = drm_scdc_writeb(connector->ddc, SCDC_TMDS_CONFIG, config);
 	if (ret < 0) {
-		DRM_DEBUG_KMS("Failed to set TMDS clock ratio: %d\n", ret);
+		drm_dbg_kms(connector->dev,
+			    "[CONNECTOR:%d:%s] Failed to set TMDS clock ratio: %d\n",
+			    connector->base.id, connector->name, ret);
 		return false;
 	}
 
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index fed41800fea7..b4c6ffc438da 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -985,6 +985,66 @@ drm_atomic_get_new_connector_for_encoder(const struct drm_atomic_state *state,
 EXPORT_SYMBOL(drm_atomic_get_new_connector_for_encoder);
 
 /**
+ * drm_atomic_get_old_crtc_for_encoder - Get old crtc for an encoder
+ * @state: Atomic state
+ * @encoder: The encoder to fetch the crtc state for
+ *
+ * This function finds and returns the crtc that was connected to @encoder
+ * as specified by the @state.
+ *
+ * Returns: The old crtc connected to @encoder, or NULL if the encoder is
+ * not connected.
+ */
+struct drm_crtc *
+drm_atomic_get_old_crtc_for_encoder(struct drm_atomic_state *state,
+				    struct drm_encoder *encoder)
+{
+	struct drm_connector *connector;
+	struct drm_connector_state *conn_state;
+
+	connector = drm_atomic_get_old_connector_for_encoder(state, encoder);
+	if (!connector)
+		return NULL;
+
+	conn_state = drm_atomic_get_old_connector_state(state, connector);
+	if (!conn_state)
+		return NULL;
+
+	return conn_state->crtc;
+}
+EXPORT_SYMBOL(drm_atomic_get_old_crtc_for_encoder);
+
+/**
+ * drm_atomic_get_new_crtc_for_encoder - Get new crtc for an encoder
+ * @state: Atomic state
+ * @encoder: The encoder to fetch the crtc state for
+ *
+ * This function finds and returns the crtc that will be connected to @encoder
+ * as specified by the @state.
+ *
+ * Returns: The new crtc connected to @encoder, or NULL if the encoder is
+ * not connected.
+ */
+struct drm_crtc *
+drm_atomic_get_new_crtc_for_encoder(struct drm_atomic_state *state,
+				    struct drm_encoder *encoder)
+{
+	struct drm_connector *connector;
+	struct drm_connector_state *conn_state;
+
+	connector = drm_atomic_get_new_connector_for_encoder(state, encoder);
+	if (!connector)
+		return NULL;
+
+	conn_state = drm_atomic_get_new_connector_state(state, connector);
+	if (!conn_state)
+		return NULL;
+
+	return conn_state->crtc;
+}
+EXPORT_SYMBOL(drm_atomic_get_new_crtc_for_encoder);
+
+/**
  * drm_atomic_get_connector_state - get connector state
  * @state: global atomic state object
  * @connector: connector to get state object for
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index d579fd8f7cb8..2c2c9caf0be5 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -1511,6 +1511,47 @@ void drm_atomic_helper_commit_modeset_enables(struct drm_device *dev,
 }
 EXPORT_SYMBOL(drm_atomic_helper_commit_modeset_enables);
 
+/*
+ * For atomic updates which touch just a single CRTC, calculate the time of the
+ * next vblank, and inform all the fences of the deadline.
+ */
+static void set_fence_deadline(struct drm_device *dev,
+			       struct drm_atomic_state *state)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_plane *plane;
+	struct drm_plane_state *new_plane_state;
+	ktime_t vbltime = 0;
+	int i;
+
+	for_each_new_crtc_in_state (state, crtc, new_crtc_state, i) {
+		ktime_t v;
+
+		if (drm_atomic_crtc_needs_modeset(new_crtc_state))
+			continue;
+
+		if (!new_crtc_state->active)
+			continue;
+
+		if (drm_crtc_next_vblank_start(crtc, &v))
+			continue;
+
+		if (!vbltime || ktime_before(v, vbltime))
+			vbltime = v;
+	}
+
+	/* If no CRTCs updated, then nothing to do: */
+	if (!vbltime)
+		return;
+
+	for_each_new_plane_in_state (state, plane, new_plane_state, i) {
+		if (!new_plane_state->fence)
+			continue;
+		dma_fence_set_deadline(new_plane_state->fence, vbltime);
+	}
+}
+
 /**
  * drm_atomic_helper_wait_for_fences - wait for fences stashed in plane state
  * @dev: DRM device
@@ -1540,6 +1581,8 @@ int drm_atomic_helper_wait_for_fences(struct drm_device *dev,
 	struct drm_plane_state *new_plane_state;
 	int i, ret;
 
+	set_fence_deadline(dev, state);
+
 	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
 		if (!new_plane_state->fence)
 			continue;
@@ -2702,6 +2745,11 @@ void drm_atomic_helper_commit_planes(struct drm_device *dev,
 			funcs->atomic_disable(plane, old_state);
 		} else if (new_plane_state->crtc || disabling) {
 			funcs->atomic_update(plane, old_state);
+
+			if (!disabling && funcs->atomic_enable) {
+				if (drm_atomic_plane_enabling(old_plane_state, new_plane_state))
+					funcs->atomic_enable(plane, old_state);
+			}
 		}
 	}
 
@@ -2762,6 +2810,7 @@ drm_atomic_helper_commit_planes_on_crtc(struct drm_crtc_state *old_crtc_state)
 		struct drm_plane_state *new_plane_state =
 			drm_atomic_get_new_plane_state(old_state, plane);
 		const struct drm_plane_helper_funcs *plane_funcs;
+		bool disabling;
 
 		plane_funcs = plane->helper_private;
 
@@ -2771,12 +2820,18 @@ drm_atomic_helper_commit_planes_on_crtc(struct drm_crtc_state *old_crtc_state)
 		WARN_ON(new_plane_state->crtc &&
 			new_plane_state->crtc != crtc);
 
-		if (drm_atomic_plane_disabling(old_plane_state, new_plane_state) &&
-		    plane_funcs->atomic_disable)
+		disabling = drm_atomic_plane_disabling(old_plane_state, new_plane_state);
+
+		if (disabling && plane_funcs->atomic_disable) {
 			plane_funcs->atomic_disable(plane, old_state);
-		else if (new_plane_state->crtc ||
-			 drm_atomic_plane_disabling(old_plane_state, new_plane_state))
+		} else if (new_plane_state->crtc || disabling) {
 			plane_funcs->atomic_update(plane, old_state);
+
+			if (!disabling && plane_funcs->atomic_enable) {
+				if (drm_atomic_plane_enabling(old_plane_state, new_plane_state))
+					plane_funcs->atomic_enable(plane, old_state);
+			}
+		}
 	}
 
 	if (crtc_funcs && crtc_funcs->atomic_flush)
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 9d0250c28e9b..48df7a5ea503 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -33,9 +33,11 @@
 #include <drm/drm_sysfs.h>
 #include <drm/drm_utils.h>
 
-#include <linux/fb.h>
+#include <linux/property.h>
 #include <linux/uaccess.h>
 
+#include <video/cmdline.h>
+
 #include "drm_crtc_internal.h"
 #include "drm_internal.h"
 
@@ -154,9 +156,10 @@ EXPORT_SYMBOL(drm_get_connector_type_name);
 static void drm_connector_get_cmdline_mode(struct drm_connector *connector)
 {
 	struct drm_cmdline_mode *mode = &connector->cmdline_mode;
-	char *option = NULL;
+	const char *option;
 
-	if (fb_get_options(connector->name, &option))
+	option = video_get_options(connector->name);
+	if (!option)
 		return;
 
 	if (!drm_mode_parse_command_line_for_connector(option,
@@ -1446,6 +1449,20 @@ static const struct drm_prop_enum_list dp_colorspaces[] = {
  *	a firmware handled hotkey. Therefor userspace must not include the
  *	privacy-screen sw-state in an atomic commit unless it wants to change
  *	its value.
+ *
+ * left margin, right margin, top margin, bottom margin:
+ *	Add margins to the connector's viewport. This is typically used to
+ *	mitigate overscan on TVs.
+ *
+ *	The value is the size in pixels of the black border which will be
+ *	added. The attached CRTC's content will be scaled to fill the whole
+ *	area inside the margin.
+ *
+ *	The margins configuration might be sent to the sink, e.g. via HDMI AVI
+ *	InfoFrames.
+ *
+ *	Drivers can set up these properties by calling
+ *	drm_mode_create_tv_margin_properties().
  */
 
 int drm_connector_create_standard_properties(struct drm_device *dev)
@@ -1590,10 +1607,6 @@ EXPORT_SYMBOL(drm_connector_attach_dp_subconnector_property);
 
 /*
  * TODO: Document the properties:
- *   - left margin
- *   - right margin
- *   - top margin
- *   - bottom margin
  *   - brightness
  *   - contrast
  *   - flicker reduction
@@ -1602,7 +1615,6 @@ EXPORT_SYMBOL(drm_connector_attach_dp_subconnector_property);
  *   - overscan
  *   - saturation
  *   - select subconnector
- *   - subconnector
  */
 /**
  * DOC: Analog TV Connector Properties
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index 4f643a490dc3..4855230ba2c6 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -80,7 +80,7 @@ static int drm_clients_info(struct seq_file *m, void *data)
 	seq_printf(m,
 		   "%20s %5s %3s master a %5s %10s\n",
 		   "command",
-		   "pid",
+		   "tgid",
 		   "dev",
 		   "uid",
 		   "magic");
@@ -94,7 +94,7 @@ static int drm_clients_info(struct seq_file *m, void *data)
 		bool is_current_master = drm_is_current_master(priv);
 
 		rcu_read_lock(); /* locks pid_task()->comm */
-		task = pid_task(priv->pid, PIDTYPE_PID);
+		task = pid_task(priv->pid, PIDTYPE_TGID);
 		uid = task ? __task_cred(task)->euid : GLOBAL_ROOT_UID;
 		seq_printf(m, "%20s %5d %3d   %c    %c %5d %10u\n",
 			   task ? task->comm : "<unknown>",
diff --git a/drivers/gpu/drm/drm_displayid.c b/drivers/gpu/drm/drm_displayid.c
index 38ea8203df45..9edc111be7ee 100644
--- a/drivers/gpu/drm/drm_displayid.c
+++ b/drivers/gpu/drm/drm_displayid.c
@@ -7,13 +7,29 @@
 #include <drm/drm_edid.h>
 #include <drm/drm_print.h>
 
-static int validate_displayid(const u8 *displayid, int length, int idx)
+static const struct displayid_header *
+displayid_get_header(const u8 *displayid, int length, int index)
+{
+	const struct displayid_header *base;
+
+	if (sizeof(*base) > length - index)
+		return ERR_PTR(-EINVAL);
+
+	base = (const struct displayid_header *)&displayid[index];
+
+	return base;
+}
+
+static const struct displayid_header *
+validate_displayid(const u8 *displayid, int length, int idx)
 {
 	int i, dispid_length;
 	u8 csum = 0;
 	const struct displayid_header *base;
 
-	base = (const struct displayid_header *)&displayid[idx];
+	base = displayid_get_header(displayid, length, idx);
+	if (IS_ERR(base))
+		return base;
 
 	DRM_DEBUG_KMS("base revision 0x%x, length %d, %d %d\n",
 		      base->rev, base->bytes, base->prod_id, base->ext_count);
@@ -21,16 +37,16 @@ static int validate_displayid(const u8 *displayid, int length, int idx)
 	/* +1 for DispID checksum */
 	dispid_length = sizeof(*base) + base->bytes + 1;
 	if (dispid_length > length - idx)
-		return -EINVAL;
+		return ERR_PTR(-EINVAL);
 
 	for (i = 0; i < dispid_length; i++)
 		csum += displayid[idx + i];
 	if (csum) {
 		DRM_NOTE("DisplayID checksum invalid, remainder is %d\n", csum);
-		return -EINVAL;
+		return ERR_PTR(-EINVAL);
 	}
 
-	return 0;
+	return base;
 }
 
 static const u8 *drm_find_displayid_extension(const struct drm_edid *drm_edid,
@@ -39,7 +55,6 @@ static const u8 *drm_find_displayid_extension(const struct drm_edid *drm_edid,
 {
 	const u8 *displayid = drm_find_edid_extension(drm_edid, DISPLAYID_EXT, ext_index);
 	const struct displayid_header *base;
-	int ret;
 
 	if (!displayid)
 		return NULL;
@@ -48,11 +63,10 @@ static const u8 *drm_find_displayid_extension(const struct drm_edid *drm_edid,
 	*length = EDID_LENGTH - 1;
 	*idx = 1;
 
-	ret = validate_displayid(displayid, *length, *idx);
-	if (ret)
+	base = validate_displayid(displayid, *length, *idx);
+	if (IS_ERR(base))
 		return NULL;
 
-	base = (const struct displayid_header *)&displayid[*idx];
 	*length = *idx + sizeof(*base) + base->bytes;
 
 	return displayid;
@@ -109,6 +123,9 @@ __displayid_iter_next(struct displayid_iter *iter)
 	}
 
 	for (;;) {
+		/* The first section we encounter is the base section */
+		bool base_section = !iter->section;
+
 		iter->section = drm_find_displayid_extension(iter->drm_edid,
 							     &iter->length,
 							     &iter->idx,
@@ -118,6 +135,18 @@ __displayid_iter_next(struct displayid_iter *iter)
 			return NULL;
 		}
 
+		/* Save the structure version and primary use case. */
+		if (base_section) {
+			const struct displayid_header *base;
+
+			base = displayid_get_header(iter->section, iter->length,
+						    iter->idx);
+			if (!IS_ERR(base)) {
+				iter->version = base->rev;
+				iter->primary_use = base->prod_id;
+			}
+		}
+
 		iter->idx += sizeof(struct displayid_header);
 
 		block = displayid_iter_block(iter);
@@ -130,3 +159,18 @@ void displayid_iter_end(struct displayid_iter *iter)
 {
 	memset(iter, 0, sizeof(*iter));
 }
+
+/* DisplayID Structure Version/Revision from the Base Section. */
+u8 displayid_version(const struct displayid_iter *iter)
+{
+	return iter->version;
+}
+
+/*
+ * DisplayID Primary Use Case (2.0+) or Product Type Identifier (1.0-1.3) from
+ * the Base Section.
+ */
+u8 displayid_primary_use(const struct displayid_iter *iter)
+{
+	return iter->primary_use;
+}
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index c6eb8972451a..cee0cc522ed9 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -691,9 +691,11 @@ static int drm_dev_init(struct drm_device *dev,
 		}
 	}
 
-	ret = drm_dev_set_unique(dev, dev_name(parent));
-	if (ret)
+	dev->unique = drmm_kstrdup(dev, dev_name(parent), GFP_KERNEL);
+	if (!dev->unique) {
+		ret = -ENOMEM;
 		goto err;
+	}
 
 	return 0;
 
@@ -1000,26 +1002,6 @@ void drm_dev_unregister(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_dev_unregister);
 
-/**
- * drm_dev_set_unique - Set the unique name of a DRM device
- * @dev: device of which to set the unique name
- * @name: unique name
- *
- * Sets the unique name of a DRM device using the specified string. This is
- * already done by drm_dev_init(), drivers should only override the default
- * unique name for backwards compatibility reasons.
- *
- * Return: 0 on success or a negative error code on failure.
- */
-int drm_dev_set_unique(struct drm_device *dev, const char *name)
-{
-	drmm_kfree(dev, dev->unique);
-	dev->unique = drmm_kstrdup(dev, name, GFP_KERNEL);
-
-	return dev->unique ? 0 : -ENOMEM;
-}
-EXPORT_SYMBOL(drm_dev_set_unique);
-
 /*
  * DRM Core
  * The DRM core module initializes all global DRM objects and makes them
diff --git a/drivers/gpu/drm/drm_dumb_buffers.c b/drivers/gpu/drm/drm_dumb_buffers.c
index ad17fa21cebb..70032bba1c97 100644
--- a/drivers/gpu/drm/drm_dumb_buffers.c
+++ b/drivers/gpu/drm/drm_dumb_buffers.c
@@ -139,10 +139,7 @@ int drm_mode_destroy_dumb(struct drm_device *dev, u32 handle,
 	if (!dev->driver->dumb_create)
 		return -ENOSYS;
 
-	if (dev->driver->dumb_destroy)
-		return dev->driver->dumb_destroy(file_priv, dev, handle);
-	else
-		return drm_gem_dumb_destroy(file_priv, dev, handle);
+	return drm_gem_handle_delete(file_priv, handle);
 }
 
 int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 261a62e15934..0454da505687 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -3424,10 +3424,6 @@ static struct drm_display_mode *drm_mode_detailed(struct drm_connector *connecto
 			    connector->base.id, connector->name);
 		return NULL;
 	}
-	if (!(pt->misc & DRM_EDID_PT_SEPARATE_SYNC)) {
-		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Composite sync not supported\n",
-			    connector->base.id, connector->name);
-	}
 
 	/* it is incorrect if hsync/vsync width is zero */
 	if (!hsync_pulse_width || !vsync_pulse_width) {
@@ -3474,10 +3470,27 @@ static struct drm_display_mode *drm_mode_detailed(struct drm_connector *connecto
 	if (info->quirks & EDID_QUIRK_DETAILED_SYNC_PP) {
 		mode->flags |= DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC;
 	} else {
-		mode->flags |= (pt->misc & DRM_EDID_PT_HSYNC_POSITIVE) ?
-			DRM_MODE_FLAG_PHSYNC : DRM_MODE_FLAG_NHSYNC;
-		mode->flags |= (pt->misc & DRM_EDID_PT_VSYNC_POSITIVE) ?
-			DRM_MODE_FLAG_PVSYNC : DRM_MODE_FLAG_NVSYNC;
+		switch (pt->misc & DRM_EDID_PT_SYNC_MASK) {
+		case DRM_EDID_PT_ANALOG_CSYNC:
+		case DRM_EDID_PT_BIPOLAR_ANALOG_CSYNC:
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Analog composite sync!\n",
+				    connector->base.id, connector->name);
+			mode->flags |= DRM_MODE_FLAG_CSYNC | DRM_MODE_FLAG_NCSYNC;
+			break;
+		case DRM_EDID_PT_DIGITAL_CSYNC:
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Digital composite sync!\n",
+				    connector->base.id, connector->name);
+			mode->flags |= DRM_MODE_FLAG_CSYNC;
+			mode->flags |= (pt->misc & DRM_EDID_PT_HSYNC_POSITIVE) ?
+				DRM_MODE_FLAG_PCSYNC : DRM_MODE_FLAG_NCSYNC;
+			break;
+		case DRM_EDID_PT_DIGITAL_SEPARATE_SYNC:
+			mode->flags |= (pt->misc & DRM_EDID_PT_HSYNC_POSITIVE) ?
+				DRM_MODE_FLAG_PHSYNC : DRM_MODE_FLAG_NHSYNC;
+			mode->flags |= (pt->misc & DRM_EDID_PT_VSYNC_POSITIVE) ?
+				DRM_MODE_FLAG_PVSYNC : DRM_MODE_FLAG_NVSYNC;
+			break;
+		}
 	}
 
 set_size:
@@ -6433,6 +6446,29 @@ static void drm_reset_display_info(struct drm_connector *connector)
 	info->quirks = 0;
 }
 
+static void update_displayid_info(struct drm_connector *connector,
+				  const struct drm_edid *drm_edid)
+{
+	struct drm_display_info *info = &connector->display_info;
+	const struct displayid_block *block;
+	struct displayid_iter iter;
+
+	displayid_iter_edid_begin(drm_edid, &iter);
+	displayid_iter_for_each(block, &iter) {
+		if (displayid_version(&iter) == DISPLAY_ID_STRUCTURE_VER_20 &&
+		    (displayid_primary_use(&iter) == PRIMARY_USE_HEAD_MOUNTED_VR ||
+		     displayid_primary_use(&iter) == PRIMARY_USE_HEAD_MOUNTED_AR))
+			info->non_desktop = true;
+
+		/*
+		 * We're only interested in the base section here, no need to
+		 * iterate further.
+		 */
+		break;
+	}
+	displayid_iter_end(&iter);
+}
+
 static void update_display_info(struct drm_connector *connector,
 				const struct drm_edid *drm_edid)
 {
@@ -6463,6 +6499,8 @@ static void update_display_info(struct drm_connector *connector,
 	info->color_formats |= DRM_COLOR_FORMAT_RGB444;
 	drm_parse_cea_ext(connector, drm_edid);
 
+	update_displayid_info(connector, drm_edid);
+
 	/*
 	 * Digital sink with "DFP 1.x compliant TMDS" according to EDID 1.3?
 	 *
@@ -7242,6 +7280,15 @@ static void drm_parse_tiled_block(struct drm_connector *connector,
 	}
 }
 
+static bool displayid_is_tiled_block(const struct displayid_iter *iter,
+				     const struct displayid_block *block)
+{
+	return (displayid_version(iter) == DISPLAY_ID_STRUCTURE_VER_12 &&
+		block->tag == DATA_BLOCK_TILED_DISPLAY) ||
+		(displayid_version(iter) == DISPLAY_ID_STRUCTURE_VER_20 &&
+		 block->tag == DATA_BLOCK_2_TILED_DISPLAY_TOPOLOGY);
+}
+
 static void _drm_update_tile_info(struct drm_connector *connector,
 				  const struct drm_edid *drm_edid)
 {
@@ -7252,7 +7299,7 @@ static void _drm_update_tile_info(struct drm_connector *connector,
 
 	displayid_iter_edid_begin(drm_edid, &iter);
 	displayid_iter_for_each(block, &iter) {
-		if (block->tag == DATA_BLOCK_TILED_DISPLAY)
+		if (displayid_is_tiled_block(&iter, block))
 			drm_parse_tiled_block(connector, block);
 	}
 	displayid_iter_end(&iter);
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index a39998047f8a..64458982be40 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -60,16 +60,17 @@ MODULE_PARM_DESC(drm_fbdev_overalloc,
  * In order to keep user-space compatibility, we want in certain use-cases
  * to keep leaking the fbdev physical address to the user-space program
  * handling the fbdev buffer.
- * This is a bad habit essentially kept into closed source opengl driver
- * that should really be moved into open-source upstream projects instead
- * of using legacy physical addresses in user space to communicate with
- * other out-of-tree kernel modules.
+ *
+ * This is a bad habit, essentially kept to support closed-source OpenGL
+ * drivers that should really be moved into open-source upstream projects
+ * instead of using legacy physical addresses in user space to communicate
+ * with other out-of-tree kernel modules.
  *
  * This module_param *should* be removed as soon as possible and be
  * considered as a broken and legacy behaviour from a modern fbdev device.
  */
-#if IS_ENABLED(CONFIG_DRM_FBDEV_LEAK_PHYS_SMEM)
 static bool drm_leak_fbdev_smem;
+#if IS_ENABLED(CONFIG_DRM_FBDEV_LEAK_PHYS_SMEM)
 module_param_unsafe(drm_leak_fbdev_smem, bool, 0600);
 MODULE_PARM_DESC(drm_leak_fbdev_smem,
 		 "Allow unsafe leaking fbdev physical smem address [default=false]");
@@ -539,6 +540,29 @@ err_release:
 EXPORT_SYMBOL(drm_fb_helper_alloc_info);
 
 /**
+ * drm_fb_helper_release_info - release fb_info and its members
+ * @fb_helper: driver-allocated fbdev helper
+ *
+ * A helper to release fb_info and the member cmap.  Drivers do not
+ * need to release the allocated fb_info structure themselves, this is
+ * automatically done when calling drm_fb_helper_fini().
+ */
+void drm_fb_helper_release_info(struct drm_fb_helper *fb_helper)
+{
+	struct fb_info *info = fb_helper->info;
+
+	if (!info)
+		return;
+
+	fb_helper->info = NULL;
+
+	if (info->cmap.len)
+		fb_dealloc_cmap(&info->cmap);
+	framebuffer_release(info);
+}
+EXPORT_SYMBOL(drm_fb_helper_release_info);
+
+/**
  * drm_fb_helper_unregister_info - unregister fb_info framebuffer device
  * @fb_helper: driver-allocated fbdev helper, can be NULL
  *
@@ -561,8 +585,6 @@ EXPORT_SYMBOL(drm_fb_helper_unregister_info);
  */
 void drm_fb_helper_fini(struct drm_fb_helper *fb_helper)
 {
-	struct fb_info *info;
-
 	if (!fb_helper)
 		return;
 
@@ -574,13 +596,7 @@ void drm_fb_helper_fini(struct drm_fb_helper *fb_helper)
 	cancel_work_sync(&fb_helper->resume_work);
 	cancel_work_sync(&fb_helper->damage_work);
 
-	info = fb_helper->info;
-	if (info) {
-		if (info->cmap.len)
-			fb_dealloc_cmap(&info->cmap);
-		framebuffer_release(info);
-	}
-	fb_helper->info = NULL;
+	drm_fb_helper_release_info(fb_helper);
 
 	mutex_lock(&kernel_fb_helper_lock);
 	if (!list_empty(&fb_helper->kernel_fb_list)) {
@@ -657,7 +673,7 @@ static void drm_fb_helper_memory_range_to_clip(struct fb_info *info, off_t off,
 void drm_fb_helper_deferred_io(struct fb_info *info, struct list_head *pagereflist)
 {
 	struct drm_fb_helper *helper = info->par;
-	unsigned long start, end, min_off, max_off;
+	unsigned long start, end, min_off, max_off, total_size;
 	struct fb_deferred_io_pageref *pageref;
 	struct drm_rect damage_area;
 
@@ -675,7 +691,11 @@ void drm_fb_helper_deferred_io(struct fb_info *info, struct list_head *pagerefli
 	 * of the screen and account for non-existing scanlines. Hence,
 	 * keep the covered memory area within the screen buffer.
 	 */
-	max_off = min(max_off, info->screen_size);
+	if (info->screen_size)
+		total_size = info->screen_size;
+	else
+		total_size = info->fix.smem_len;
+	max_off = min(max_off, total_size);
 
 	if (min_off < max_off) {
 		drm_fb_helper_memory_range_to_clip(info, min_off, max_off - min_off, &damage_area);
@@ -1517,6 +1537,27 @@ static void drm_fb_helper_fill_pixel_fmt(struct fb_var_screeninfo *var,
 	}
 }
 
+static void __fill_var(struct fb_var_screeninfo *var,
+		       struct drm_framebuffer *fb)
+{
+	int i;
+
+	var->xres_virtual = fb->width;
+	var->yres_virtual = fb->height;
+	var->accel_flags = FB_ACCELF_TEXT;
+	var->bits_per_pixel = drm_format_info_bpp(fb->format, 0);
+
+	var->height = var->width = 0;
+	var->left_margin = var->right_margin = 0;
+	var->upper_margin = var->lower_margin = 0;
+	var->hsync_len = var->vsync_len = 0;
+	var->sync = var->vmode = 0;
+	var->rotate = 0;
+	var->colorspace = 0;
+	for (i = 0; i < 4; i++)
+		var->reserved[i] = 0;
+}
+
 /**
  * drm_fb_helper_check_var - implementation for &fb_ops.fb_check_var
  * @var: screeninfo to check
@@ -1569,6 +1610,23 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var,
 		return -EINVAL;
 	}
 
+	__fill_var(var, fb);
+
+	/*
+	 * fb_pan_display() validates this, but fb_set_par() doesn't and just
+	 * falls over. Note that __fill_var above adjusts y/res_virtual.
+	 */
+	if (var->yoffset > var->yres_virtual - var->yres ||
+	    var->xoffset > var->xres_virtual - var->xres)
+		return -EINVAL;
+
+	/* We neither support grayscale nor FOURCC (also stored in here). */
+	if (var->grayscale > 0)
+		return -EINVAL;
+
+	if (var->nonstd)
+		return -EINVAL;
+
 	/*
 	 * Workaround for SDL 1.2, which is known to be setting all pixel format
 	 * fields values to zero in some cases. We treat this situation as a
@@ -1584,11 +1642,6 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var,
 	}
 
 	/*
-	 * Likewise, bits_per_pixel should be rounded up to a supported value.
-	 */
-	var->bits_per_pixel = bpp;
-
-	/*
 	 * drm fbdev emulation doesn't support changing the pixel format at all,
 	 * so reject all pixel format changing requests.
 	 */
@@ -1618,11 +1671,6 @@ int drm_fb_helper_set_par(struct fb_info *info)
 	if (oops_in_progress)
 		return -EBUSY;
 
-	if (var->pixclock != 0) {
-		drm_err(fb_helper->dev, "PIXEL CLOCK SET\n");
-		return -EINVAL;
-	}
-
 	/*
 	 * Normally we want to make sure that a kms master takes precedence over
 	 * fbdev, to avoid fbdev flickering and occasionally stealing the
@@ -1964,10 +2012,6 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper)
 		return ret;
 	}
 
-#if IS_ENABLED(CONFIG_DRM_FBDEV_LEAK_PHYS_SMEM)
-	fb_helper->hint_leak_smem_start = drm_leak_fbdev_smem;
-#endif
-
 	/* push down into drivers */
 	ret = (*fb_helper->funcs->fb_probe)(fb_helper, &sizes);
 	if (ret < 0)
@@ -2020,12 +2064,9 @@ static void drm_fb_helper_fill_var(struct fb_info *info,
 	}
 
 	info->pseudo_palette = fb_helper->pseudo_palette;
-	info->var.xres_virtual = fb->width;
-	info->var.yres_virtual = fb->height;
-	info->var.bits_per_pixel = drm_format_info_bpp(format, 0);
-	info->var.accel_flags = FB_ACCELF_TEXT;
 	info->var.xoffset = 0;
 	info->var.yoffset = 0;
+	__fill_var(&info->var, fb);
 	info->var.activate = FB_ACTIVATE_NOW;
 
 	drm_fb_helper_fill_pixel_fmt(&info->var, format);
@@ -2166,11 +2207,8 @@ __drm_fb_helper_initial_config_and_unlock(struct drm_fb_helper *fb_helper)
 
 	info = fb_helper->info;
 	info->var.pixclock = 0;
-	/* Shamelessly allow physical address leaking to userspace */
-#if IS_ENABLED(CONFIG_DRM_FBDEV_LEAK_PHYS_SMEM)
-	if (!fb_helper->hint_leak_smem_start)
-#endif
-		/* don't leak any physical addresses to userspace */
+
+	if (!drm_leak_fbdev_smem)
 		info->flags |= FBINFO_HIDE_SMEM_START;
 
 	/* Need to drop locks to avoid recursive deadlock in
diff --git a/drivers/gpu/drm/drm_fbdev_dma.c b/drivers/gpu/drm/drm_fbdev_dma.c
new file mode 100644
index 000000000000..728deffcc0d9
--- /dev/null
+++ b/drivers/gpu/drm/drm_fbdev_dma.c
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: MIT
+
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_dma_helper.h>
+
+#include <drm/drm_fbdev_dma.h>
+
+/*
+ * struct fb_ops
+ */
+
+static int drm_fbdev_dma_fb_open(struct fb_info *info, int user)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+
+	/* No need to take a ref for fbcon because it unbinds on unregister */
+	if (user && !try_module_get(fb_helper->dev->driver->fops->owner))
+		return -ENODEV;
+
+	return 0;
+}
+
+static int drm_fbdev_dma_fb_release(struct fb_info *info, int user)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+
+	if (user)
+		module_put(fb_helper->dev->driver->fops->owner);
+
+	return 0;
+}
+
+static void drm_fbdev_dma_fb_destroy(struct fb_info *info)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+
+	if (!fb_helper->dev)
+		return;
+
+	drm_fb_helper_fini(fb_helper);
+
+	drm_client_buffer_vunmap(fb_helper->buffer);
+	drm_client_framebuffer_delete(fb_helper->buffer);
+	drm_client_release(&fb_helper->client);
+	drm_fb_helper_unprepare(fb_helper);
+	kfree(fb_helper);
+}
+
+static int drm_fbdev_dma_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+	struct drm_device *dev = fb_helper->dev;
+
+	if (drm_WARN_ON_ONCE(dev, !fb_helper->dev->driver->gem_prime_mmap))
+		return -ENODEV;
+
+	return fb_helper->dev->driver->gem_prime_mmap(fb_helper->buffer->gem, vma);
+}
+
+static const struct fb_ops drm_fbdev_dma_fb_ops = {
+	.owner = THIS_MODULE,
+	.fb_open = drm_fbdev_dma_fb_open,
+	.fb_release = drm_fbdev_dma_fb_release,
+	.fb_read = drm_fb_helper_sys_read,
+	.fb_write = drm_fb_helper_sys_write,
+	DRM_FB_HELPER_DEFAULT_OPS,
+	.fb_fillrect = drm_fb_helper_sys_fillrect,
+	.fb_copyarea = drm_fb_helper_sys_copyarea,
+	.fb_imageblit = drm_fb_helper_sys_imageblit,
+	.fb_destroy = drm_fbdev_dma_fb_destroy,
+	.fb_mmap = drm_fbdev_dma_fb_mmap,
+};
+
+/*
+ * struct drm_fb_helper
+ */
+
+static int drm_fbdev_dma_helper_fb_probe(struct drm_fb_helper *fb_helper,
+					 struct drm_fb_helper_surface_size *sizes)
+{
+	struct drm_client_dev *client = &fb_helper->client;
+	struct drm_device *dev = fb_helper->dev;
+	struct drm_client_buffer *buffer;
+	struct drm_gem_dma_object *dma_obj;
+	struct drm_framebuffer *fb;
+	struct fb_info *info;
+	u32 format;
+	struct iosys_map map;
+	int ret;
+
+	drm_dbg_kms(dev, "surface width(%d), height(%d) and bpp(%d)\n",
+		    sizes->surface_width, sizes->surface_height,
+		    sizes->surface_bpp);
+
+	format = drm_mode_legacy_fb_format(sizes->surface_bpp, sizes->surface_depth);
+	buffer = drm_client_framebuffer_create(client, sizes->surface_width,
+					       sizes->surface_height, format);
+	if (IS_ERR(buffer))
+		return PTR_ERR(buffer);
+	dma_obj = to_drm_gem_dma_obj(buffer->gem);
+
+	fb = buffer->fb;
+	if (drm_WARN_ON(dev, fb->funcs->dirty)) {
+		ret = -ENODEV; /* damage handling not supported; use generic emulation */
+		goto err_drm_client_buffer_delete;
+	}
+
+	ret = drm_client_buffer_vmap(buffer, &map);
+	if (ret) {
+		goto err_drm_client_buffer_delete;
+	} else if (drm_WARN_ON(dev, map.is_iomem)) {
+		ret = -ENODEV; /* I/O memory not supported; use generic emulation */
+		goto err_drm_client_buffer_delete;
+	}
+
+	fb_helper->buffer = buffer;
+	fb_helper->fb = buffer->fb;
+
+	info = drm_fb_helper_alloc_info(fb_helper);
+	if (IS_ERR(info)) {
+		ret = PTR_ERR(info);
+		goto err_drm_client_buffer_vunmap;
+	}
+
+	drm_fb_helper_fill_info(info, fb_helper, sizes);
+
+	info->fbops = &drm_fbdev_dma_fb_ops;
+	info->flags = FBINFO_DEFAULT;
+
+	/* screen */
+	info->flags |= FBINFO_VIRTFB; /* system memory */
+	if (dma_obj->map_noncoherent)
+		info->flags |= FBINFO_READS_FAST; /* signal caching */
+	info->screen_size = sizes->surface_height * fb->pitches[0];
+	info->screen_buffer = map.vaddr;
+	info->fix.smem_start = page_to_phys(virt_to_page(info->screen_buffer));
+	info->fix.smem_len = info->screen_size;
+
+	return 0;
+
+err_drm_client_buffer_vunmap:
+	fb_helper->fb = NULL;
+	fb_helper->buffer = NULL;
+	drm_client_buffer_vunmap(buffer);
+err_drm_client_buffer_delete:
+	drm_client_framebuffer_delete(buffer);
+	return ret;
+}
+
+static const struct drm_fb_helper_funcs drm_fbdev_dma_helper_funcs = {
+	.fb_probe = drm_fbdev_dma_helper_fb_probe,
+};
+
+/*
+ * struct drm_client_funcs
+ */
+
+static void drm_fbdev_dma_client_unregister(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+
+	if (fb_helper->info) {
+		drm_fb_helper_unregister_info(fb_helper);
+	} else {
+		drm_client_release(&fb_helper->client);
+		drm_fb_helper_unprepare(fb_helper);
+		kfree(fb_helper);
+	}
+}
+
+static int drm_fbdev_dma_client_restore(struct drm_client_dev *client)
+{
+	drm_fb_helper_lastclose(client->dev);
+
+	return 0;
+}
+
+static int drm_fbdev_dma_client_hotplug(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+	struct drm_device *dev = client->dev;
+	int ret;
+
+	if (dev->fb_helper)
+		return drm_fb_helper_hotplug_event(dev->fb_helper);
+
+	ret = drm_fb_helper_init(dev, fb_helper);
+	if (ret)
+		goto err_drm_err;
+
+	if (!drm_drv_uses_atomic_modeset(dev))
+		drm_helper_disable_unused_functions(dev);
+
+	ret = drm_fb_helper_initial_config(fb_helper);
+	if (ret)
+		goto err_drm_fb_helper_fini;
+
+	return 0;
+
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(fb_helper);
+err_drm_err:
+	drm_err(dev, "fbdev-dma: Failed to setup generic emulation (ret=%d)\n", ret);
+	return ret;
+}
+
+static const struct drm_client_funcs drm_fbdev_dma_client_funcs = {
+	.owner		= THIS_MODULE,
+	.unregister	= drm_fbdev_dma_client_unregister,
+	.restore	= drm_fbdev_dma_client_restore,
+	.hotplug	= drm_fbdev_dma_client_hotplug,
+};
+
+/**
+ * drm_fbdev_dma_setup() - Setup fbdev emulation for GEM DMA helpers
+ * @dev: DRM device
+ * @preferred_bpp: Preferred bits per pixel for the device.
+ *                 @dev->mode_config.preferred_depth is used if this is zero.
+ *
+ * This function sets up fbdev emulation for GEM DMA drivers that support
+ * dumb buffers with a virtual address and that can be mmap'ed.
+ * drm_fbdev_dma_setup() shall be called after the DRM driver registered
+ * the new DRM device with drm_dev_register().
+ *
+ * Restore, hotplug events and teardown are all taken care of. Drivers that do
+ * suspend/resume need to call drm_fb_helper_set_suspend_unlocked() themselves.
+ * Simple drivers might use drm_mode_config_helper_suspend().
+ *
+ * This function is safe to call even when there are no connectors present.
+ * Setup will be retried on the next hotplug event.
+ *
+ * The fbdev is destroyed by drm_dev_unregister().
+ */
+void drm_fbdev_dma_setup(struct drm_device *dev, unsigned int preferred_bpp)
+{
+	struct drm_fb_helper *fb_helper;
+	int ret;
+
+	drm_WARN(dev, !dev->registered, "Device has not been registered.\n");
+	drm_WARN(dev, dev->fb_helper, "fb_helper is already set!\n");
+
+	fb_helper = kzalloc(sizeof(*fb_helper), GFP_KERNEL);
+	if (!fb_helper)
+		return;
+	drm_fb_helper_prepare(dev, fb_helper, preferred_bpp, &drm_fbdev_dma_helper_funcs);
+
+	ret = drm_client_init(dev, &fb_helper->client, "fbdev", &drm_fbdev_dma_client_funcs);
+	if (ret) {
+		drm_err(dev, "Failed to register client: %d\n", ret);
+		goto err_drm_client_init;
+	}
+
+	ret = drm_fbdev_dma_client_hotplug(&fb_helper->client);
+	if (ret)
+		drm_dbg_kms(dev, "client hotplug ret=%d\n", ret);
+
+	drm_client_register(&fb_helper->client);
+
+	return;
+
+err_drm_client_init:
+	drm_fb_helper_unprepare(fb_helper);
+	kfree(fb_helper);
+}
+EXPORT_SYMBOL(drm_fbdev_dma_setup);
diff --git a/drivers/gpu/drm/drm_fbdev_generic.c b/drivers/gpu/drm/drm_fbdev_generic.c
index 4d6325e91565..8e5148bf40bb 100644
--- a/drivers/gpu/drm/drm_fbdev_generic.c
+++ b/drivers/gpu/drm/drm_fbdev_generic.c
@@ -7,22 +7,13 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_framebuffer.h>
+#include <drm/drm_gem.h>
 #include <drm/drm_print.h>
 
 #include <drm/drm_fbdev_generic.h>
 
-static bool drm_fbdev_use_shadow_fb(struct drm_fb_helper *fb_helper)
-{
-	struct drm_device *dev = fb_helper->dev;
-	struct drm_framebuffer *fb = fb_helper->fb;
-
-	return dev->mode_config.prefer_shadow_fbdev ||
-	       dev->mode_config.prefer_shadow ||
-	       fb->funcs->dirty;
-}
-
 /* @user: 1=userspace, 0=fbcon */
-static int drm_fbdev_fb_open(struct fb_info *info, int user)
+static int drm_fbdev_generic_fb_open(struct fb_info *info, int user)
 {
 	struct drm_fb_helper *fb_helper = info->par;
 
@@ -33,7 +24,7 @@ static int drm_fbdev_fb_open(struct fb_info *info, int user)
 	return 0;
 }
 
-static int drm_fbdev_fb_release(struct fb_info *info, int user)
+static int drm_fbdev_generic_fb_release(struct fb_info *info, int user)
 {
 	struct drm_fb_helper *fb_helper = info->par;
 
@@ -43,133 +34,51 @@ static int drm_fbdev_fb_release(struct fb_info *info, int user)
 	return 0;
 }
 
-static void drm_fbdev_fb_destroy(struct fb_info *info)
+static void drm_fbdev_generic_fb_destroy(struct fb_info *info)
 {
 	struct drm_fb_helper *fb_helper = info->par;
-	void *shadow = NULL;
+	void *shadow = info->screen_buffer;
 
 	if (!fb_helper->dev)
 		return;
 
-	if (info->fbdefio)
-		fb_deferred_io_cleanup(info);
-	if (drm_fbdev_use_shadow_fb(fb_helper))
-		shadow = info->screen_buffer;
-
+	fb_deferred_io_cleanup(info);
 	drm_fb_helper_fini(fb_helper);
-
-	if (shadow)
-		vfree(shadow);
-	else if (fb_helper->buffer)
-		drm_client_buffer_vunmap(fb_helper->buffer);
-
+	vfree(shadow);
 	drm_client_framebuffer_delete(fb_helper->buffer);
-	drm_client_release(&fb_helper->client);
 
+	drm_client_release(&fb_helper->client);
 	drm_fb_helper_unprepare(fb_helper);
 	kfree(fb_helper);
 }
 
-static int drm_fbdev_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
-{
-	struct drm_fb_helper *fb_helper = info->par;
-
-	if (drm_fbdev_use_shadow_fb(fb_helper))
-		return fb_deferred_io_mmap(info, vma);
-	else if (fb_helper->dev->driver->gem_prime_mmap)
-		return fb_helper->dev->driver->gem_prime_mmap(fb_helper->buffer->gem, vma);
-	else
-		return -ENODEV;
-}
-
-static bool drm_fbdev_use_iomem(struct fb_info *info)
-{
-	struct drm_fb_helper *fb_helper = info->par;
-	struct drm_client_buffer *buffer = fb_helper->buffer;
-
-	return !drm_fbdev_use_shadow_fb(fb_helper) && buffer->map.is_iomem;
-}
-
-static ssize_t drm_fbdev_fb_read(struct fb_info *info, char __user *buf,
-				 size_t count, loff_t *ppos)
-{
-	ssize_t ret;
-
-	if (drm_fbdev_use_iomem(info))
-		ret = drm_fb_helper_cfb_read(info, buf, count, ppos);
-	else
-		ret = drm_fb_helper_sys_read(info, buf, count, ppos);
-
-	return ret;
-}
-
-static ssize_t drm_fbdev_fb_write(struct fb_info *info, const char __user *buf,
-				  size_t count, loff_t *ppos)
-{
-	ssize_t ret;
-
-	if (drm_fbdev_use_iomem(info))
-		ret = drm_fb_helper_cfb_write(info, buf, count, ppos);
-	else
-		ret = drm_fb_helper_sys_write(info, buf, count, ppos);
-
-	return ret;
-}
-
-static void drm_fbdev_fb_fillrect(struct fb_info *info,
-				  const struct fb_fillrect *rect)
-{
-	if (drm_fbdev_use_iomem(info))
-		drm_fb_helper_cfb_fillrect(info, rect);
-	else
-		drm_fb_helper_sys_fillrect(info, rect);
-}
-
-static void drm_fbdev_fb_copyarea(struct fb_info *info,
-				  const struct fb_copyarea *area)
-{
-	if (drm_fbdev_use_iomem(info))
-		drm_fb_helper_cfb_copyarea(info, area);
-	else
-		drm_fb_helper_sys_copyarea(info, area);
-}
-
-static void drm_fbdev_fb_imageblit(struct fb_info *info,
-				   const struct fb_image *image)
-{
-	if (drm_fbdev_use_iomem(info))
-		drm_fb_helper_cfb_imageblit(info, image);
-	else
-		drm_fb_helper_sys_imageblit(info, image);
-}
-
-static const struct fb_ops drm_fbdev_fb_ops = {
+static const struct fb_ops drm_fbdev_generic_fb_ops = {
 	.owner		= THIS_MODULE,
+	.fb_open	= drm_fbdev_generic_fb_open,
+	.fb_release	= drm_fbdev_generic_fb_release,
+	.fb_read	= drm_fb_helper_sys_read,
+	.fb_write	= drm_fb_helper_sys_write,
 	DRM_FB_HELPER_DEFAULT_OPS,
-	.fb_open	= drm_fbdev_fb_open,
-	.fb_release	= drm_fbdev_fb_release,
-	.fb_destroy	= drm_fbdev_fb_destroy,
-	.fb_mmap	= drm_fbdev_fb_mmap,
-	.fb_read	= drm_fbdev_fb_read,
-	.fb_write	= drm_fbdev_fb_write,
-	.fb_fillrect	= drm_fbdev_fb_fillrect,
-	.fb_copyarea	= drm_fbdev_fb_copyarea,
-	.fb_imageblit	= drm_fbdev_fb_imageblit,
+	.fb_fillrect	= drm_fb_helper_sys_fillrect,
+	.fb_copyarea	= drm_fb_helper_sys_copyarea,
+	.fb_imageblit	= drm_fb_helper_sys_imageblit,
+	.fb_mmap	= fb_deferred_io_mmap,
+	.fb_destroy	= drm_fbdev_generic_fb_destroy,
 };
 
 /*
  * This function uses the client API to create a framebuffer backed by a dumb buffer.
  */
-static int drm_fbdev_fb_probe(struct drm_fb_helper *fb_helper,
-			      struct drm_fb_helper_surface_size *sizes)
+static int drm_fbdev_generic_helper_fb_probe(struct drm_fb_helper *fb_helper,
+					     struct drm_fb_helper_surface_size *sizes)
 {
 	struct drm_client_dev *client = &fb_helper->client;
 	struct drm_device *dev = fb_helper->dev;
 	struct drm_client_buffer *buffer;
-	struct drm_framebuffer *fb;
 	struct fb_info *info;
+	size_t screen_size;
+	void *screen_buffer;
 	u32 format;
-	struct iosys_map map;
 	int ret;
 
 	drm_dbg_kms(dev, "surface width(%d), height(%d) and bpp(%d)\n",
@@ -184,64 +93,56 @@ static int drm_fbdev_fb_probe(struct drm_fb_helper *fb_helper,
 
 	fb_helper->buffer = buffer;
 	fb_helper->fb = buffer->fb;
-	fb = buffer->fb;
 
-	info = drm_fb_helper_alloc_info(fb_helper);
-	if (IS_ERR(info))
-		return PTR_ERR(info);
+	screen_size = buffer->gem->size;
+	screen_buffer = vzalloc(screen_size);
+	if (!screen_buffer) {
+		ret = -ENOMEM;
+		goto err_drm_client_framebuffer_delete;
+	}
 
-	info->fbops = &drm_fbdev_fb_ops;
-	info->screen_size = sizes->surface_height * fb->pitches[0];
-	info->fix.smem_len = info->screen_size;
-	info->flags = FBINFO_DEFAULT;
+	info = drm_fb_helper_alloc_info(fb_helper);
+	if (IS_ERR(info)) {
+		ret = PTR_ERR(info);
+		goto err_vfree;
+	}
 
 	drm_fb_helper_fill_info(info, fb_helper, sizes);
 
-	if (drm_fbdev_use_shadow_fb(fb_helper)) {
-		info->screen_buffer = vzalloc(info->screen_size);
-		if (!info->screen_buffer)
-			return -ENOMEM;
-		info->flags |= FBINFO_VIRTFB | FBINFO_READS_FAST;
+	info->fbops = &drm_fbdev_generic_fb_ops;
+	info->flags = FBINFO_DEFAULT;
 
-		/* Set a default deferred I/O handler */
-		fb_helper->fbdefio.delay = HZ / 20;
-		fb_helper->fbdefio.deferred_io = drm_fb_helper_deferred_io;
+	/* screen */
+	info->flags |= FBINFO_VIRTFB | FBINFO_READS_FAST;
+	info->screen_buffer = screen_buffer;
+	info->fix.smem_start = page_to_phys(vmalloc_to_page(info->screen_buffer));
+	info->fix.smem_len = screen_size;
 
-		info->fbdefio = &fb_helper->fbdefio;
-		ret = fb_deferred_io_init(info);
-		if (ret)
-			return ret;
-	} else {
-		/* buffer is mapped for HW framebuffer */
-		ret = drm_client_buffer_vmap(fb_helper->buffer, &map);
-		if (ret)
-			return ret;
-		if (map.is_iomem) {
-			info->screen_base = map.vaddr_iomem;
-		} else {
-			info->screen_buffer = map.vaddr;
-			info->flags |= FBINFO_VIRTFB;
-		}
-
-		/*
-		 * Shamelessly leak the physical address to user-space. As
-		 * page_to_phys() is undefined for I/O memory, warn in this
-		 * case.
-		 */
-#if IS_ENABLED(CONFIG_DRM_FBDEV_LEAK_PHYS_SMEM)
-		if (fb_helper->hint_leak_smem_start && info->fix.smem_start == 0 &&
-		    !drm_WARN_ON_ONCE(dev, map.is_iomem))
-			info->fix.smem_start =
-				page_to_phys(virt_to_page(info->screen_buffer));
-#endif
-	}
+	/* deferred I/O */
+	fb_helper->fbdefio.delay = HZ / 20;
+	fb_helper->fbdefio.deferred_io = drm_fb_helper_deferred_io;
+
+	info->fbdefio = &fb_helper->fbdefio;
+	ret = fb_deferred_io_init(info);
+	if (ret)
+		goto err_drm_fb_helper_release_info;
 
 	return 0;
+
+err_drm_fb_helper_release_info:
+	drm_fb_helper_release_info(fb_helper);
+err_vfree:
+	vfree(screen_buffer);
+err_drm_client_framebuffer_delete:
+	fb_helper->fb = NULL;
+	fb_helper->buffer = NULL;
+	drm_client_framebuffer_delete(buffer);
+	return ret;
 }
 
-static void drm_fbdev_damage_blit_real(struct drm_fb_helper *fb_helper,
-				       struct drm_clip_rect *clip,
-				       struct iosys_map *dst)
+static void drm_fbdev_generic_damage_blit_real(struct drm_fb_helper *fb_helper,
+					       struct drm_clip_rect *clip,
+					       struct iosys_map *dst)
 {
 	struct drm_framebuffer *fb = fb_helper->fb;
 	size_t offset = clip->y1 * fb->pitches[0];
@@ -278,8 +179,8 @@ static void drm_fbdev_damage_blit_real(struct drm_fb_helper *fb_helper,
 	}
 }
 
-static int drm_fbdev_damage_blit(struct drm_fb_helper *fb_helper,
-				 struct drm_clip_rect *clip)
+static int drm_fbdev_generic_damage_blit(struct drm_fb_helper *fb_helper,
+					 struct drm_clip_rect *clip)
 {
 	struct drm_client_buffer *buffer = fb_helper->buffer;
 	struct iosys_map map, dst;
@@ -303,7 +204,7 @@ static int drm_fbdev_damage_blit(struct drm_fb_helper *fb_helper,
 		goto out;
 
 	dst = map;
-	drm_fbdev_damage_blit_real(fb_helper, clip, &dst);
+	drm_fbdev_generic_damage_blit_real(fb_helper, clip, &dst);
 
 	drm_client_buffer_vunmap(buffer);
 
@@ -313,23 +214,19 @@ out:
 	return ret;
 }
 
-static int drm_fbdev_fb_dirty(struct drm_fb_helper *helper, struct drm_clip_rect *clip)
+static int drm_fbdev_generic_helper_fb_dirty(struct drm_fb_helper *helper,
+					     struct drm_clip_rect *clip)
 {
 	struct drm_device *dev = helper->dev;
 	int ret;
 
-	if (!drm_fbdev_use_shadow_fb(helper))
-		return 0;
-
 	/* Call damage handlers only if necessary */
 	if (!(clip->x1 < clip->x2 && clip->y1 < clip->y2))
 		return 0;
 
-	if (helper->buffer) {
-		ret = drm_fbdev_damage_blit(helper, clip);
-		if (drm_WARN_ONCE(dev, ret, "Damage blitter failed: ret=%d\n", ret))
-			return ret;
-	}
+	ret = drm_fbdev_generic_damage_blit(helper, clip);
+	if (drm_WARN_ONCE(dev, ret, "Damage blitter failed: ret=%d\n", ret))
+		return ret;
 
 	if (helper->fb->funcs->dirty) {
 		ret = helper->fb->funcs->dirty(helper->fb, NULL, 0, 0, clip, 1);
@@ -340,12 +237,12 @@ static int drm_fbdev_fb_dirty(struct drm_fb_helper *helper, struct drm_clip_rect
 	return 0;
 }
 
-static const struct drm_fb_helper_funcs drm_fb_helper_generic_funcs = {
-	.fb_probe = drm_fbdev_fb_probe,
-	.fb_dirty = drm_fbdev_fb_dirty,
+static const struct drm_fb_helper_funcs drm_fbdev_generic_helper_funcs = {
+	.fb_probe = drm_fbdev_generic_helper_fb_probe,
+	.fb_dirty = drm_fbdev_generic_helper_fb_dirty,
 };
 
-static void drm_fbdev_client_unregister(struct drm_client_dev *client)
+static void drm_fbdev_generic_client_unregister(struct drm_client_dev *client)
 {
 	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
 
@@ -358,14 +255,14 @@ static void drm_fbdev_client_unregister(struct drm_client_dev *client)
 	}
 }
 
-static int drm_fbdev_client_restore(struct drm_client_dev *client)
+static int drm_fbdev_generic_client_restore(struct drm_client_dev *client)
 {
 	drm_fb_helper_lastclose(client->dev);
 
 	return 0;
 }
 
-static int drm_fbdev_client_hotplug(struct drm_client_dev *client)
+static int drm_fbdev_generic_client_hotplug(struct drm_client_dev *client)
 {
 	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
 	struct drm_device *dev = client->dev;
@@ -394,11 +291,11 @@ err_drm_err:
 	return ret;
 }
 
-static const struct drm_client_funcs drm_fbdev_client_funcs = {
+static const struct drm_client_funcs drm_fbdev_generic_client_funcs = {
 	.owner		= THIS_MODULE,
-	.unregister	= drm_fbdev_client_unregister,
-	.restore	= drm_fbdev_client_restore,
-	.hotplug	= drm_fbdev_client_hotplug,
+	.unregister	= drm_fbdev_generic_client_unregister,
+	.restore	= drm_fbdev_generic_client_restore,
+	.hotplug	= drm_fbdev_generic_client_hotplug,
 };
 
 /**
@@ -415,20 +312,16 @@ static const struct drm_client_funcs drm_fbdev_client_funcs = {
  * suspend/resume need to call drm_fb_helper_set_suspend_unlocked() themselves.
  * Simple drivers might use drm_mode_config_helper_suspend().
  *
- * Drivers that set the dirty callback on their framebuffer will get a shadow
- * fbdev buffer that is blitted onto the real buffer. This is done in order to
- * make deferred I/O work with all kinds of buffers. A shadow buffer can be
- * requested explicitly by setting struct drm_mode_config.prefer_shadow or
- * struct drm_mode_config.prefer_shadow_fbdev to true beforehand. This is
- * required to use generic fbdev emulation with SHMEM helpers.
+ * In order to provide fixed mmap-able memory ranges, generic fbdev emulation
+ * uses a shadow buffer in system memory. The implementation blits the shadow
+ * fbdev buffer onto the real buffer in regular intervals.
  *
  * This function is safe to call even when there are no connectors present.
  * Setup will be retried on the next hotplug event.
  *
  * The fbdev is destroyed by drm_dev_unregister().
  */
-void drm_fbdev_generic_setup(struct drm_device *dev,
-			     unsigned int preferred_bpp)
+void drm_fbdev_generic_setup(struct drm_device *dev, unsigned int preferred_bpp)
 {
 	struct drm_fb_helper *fb_helper;
 	int ret;
@@ -439,15 +332,15 @@ void drm_fbdev_generic_setup(struct drm_device *dev,
 	fb_helper = kzalloc(sizeof(*fb_helper), GFP_KERNEL);
 	if (!fb_helper)
 		return;
-	drm_fb_helper_prepare(dev, fb_helper, preferred_bpp, &drm_fb_helper_generic_funcs);
+	drm_fb_helper_prepare(dev, fb_helper, preferred_bpp, &drm_fbdev_generic_helper_funcs);
 
-	ret = drm_client_init(dev, &fb_helper->client, "fbdev", &drm_fbdev_client_funcs);
+	ret = drm_client_init(dev, &fb_helper->client, "fbdev", &drm_fbdev_generic_client_funcs);
 	if (ret) {
 		drm_err(dev, "Failed to register client: %d\n", ret);
 		goto err_drm_client_init;
 	}
 
-	ret = drm_fbdev_client_hotplug(&fb_helper->client);
+	ret = drm_fbdev_generic_client_hotplug(&fb_helper->client);
 	if (ret)
 		drm_dbg_kms(dev, "client hotplug ret=%d\n", ret);
 
diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c
index a51ff8cee049..c1018c470047 100644
--- a/drivers/gpu/drm/drm_file.c
+++ b/drivers/gpu/drm/drm_file.c
@@ -156,7 +156,7 @@ struct drm_file *drm_file_alloc(struct drm_minor *minor)
 	if (!file)
 		return ERR_PTR(-ENOMEM);
 
-	file->pid = get_pid(task_pid(current));
+	file->pid = get_pid(task_tgid(current));
 	file->minor = minor;
 
 	/* for compatibility root is always authenticated */
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index a5d392f7e11f..1a5a2cd0d4ec 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -336,13 +336,6 @@ out:
 }
 EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset);
 
-int drm_gem_dumb_destroy(struct drm_file *file,
-			 struct drm_device *dev,
-			 u32 handle)
-{
-	return drm_gem_handle_delete(file, handle);
-}
-
 /**
  * drm_gem_handle_create_tail - internal functions to create a handle
  * @file_priv: drm file-private structure to register the handle for
@@ -1344,7 +1337,15 @@ drm_gem_lru_remove(struct drm_gem_object *obj)
 }
 EXPORT_SYMBOL(drm_gem_lru_remove);
 
-static void
+/**
+ * drm_gem_lru_move_tail_locked - move the object to the tail of the LRU
+ *
+ * Like &drm_gem_lru_move_tail but lru lock must be held
+ *
+ * @lru: The LRU to move the object into.
+ * @obj: The GEM object to move into this LRU
+ */
+void
 drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj)
 {
 	lockdep_assert_held_once(lru->lock);
@@ -1356,6 +1357,7 @@ drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj
 	list_add_tail(&obj->lru_node, &lru->list);
 	obj->lru = lru;
 }
+EXPORT_SYMBOL(drm_gem_lru_move_tail_locked);
 
 /**
  * drm_gem_lru_move_tail - move the object to the tail of the LRU
@@ -1471,3 +1473,21 @@ tail:
 	return freed;
 }
 EXPORT_SYMBOL(drm_gem_lru_scan);
+
+/**
+ * drm_gem_evict - helper to evict backing pages for a GEM object
+ * @obj: obj in question
+ */
+int drm_gem_evict(struct drm_gem_object *obj)
+{
+	dma_resv_assert_held(obj->resv);
+
+	if (!dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ))
+		return -EBUSY;
+
+	if (obj->funcs->evict)
+		return obj->funcs->evict(obj);
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_gem_evict);
diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
index 2b2163c8138e..4ea6507a77e5 100644
--- a/drivers/gpu/drm/drm_gem_shmem_helper.c
+++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
@@ -141,7 +141,7 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
 {
 	struct drm_gem_object *obj = &shmem->base;
 
-	WARN_ON(shmem->vmap_use_count);
+	drm_WARN_ON(obj->dev, shmem->vmap_use_count);
 
 	if (obj->import_attach) {
 		drm_prime_gem_destroy(obj, shmem->sgt);
@@ -156,7 +156,7 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)
 			drm_gem_shmem_put_pages(shmem);
 	}
 
-	WARN_ON(shmem->pages_use_count);
+	drm_WARN_ON(obj->dev, shmem->pages_use_count);
 
 	drm_gem_object_release(obj);
 	mutex_destroy(&shmem->pages_lock);
@@ -175,7 +175,8 @@ static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem)
 
 	pages = drm_gem_get_pages(obj);
 	if (IS_ERR(pages)) {
-		DRM_DEBUG_KMS("Failed to get pages (%ld)\n", PTR_ERR(pages));
+		drm_dbg_kms(obj->dev, "Failed to get pages (%ld)\n",
+			    PTR_ERR(pages));
 		shmem->pages_use_count = 0;
 		return PTR_ERR(pages);
 	}
@@ -207,9 +208,10 @@ static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem)
  */
 int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)
 {
+	struct drm_gem_object *obj = &shmem->base;
 	int ret;
 
-	WARN_ON(shmem->base.import_attach);
+	drm_WARN_ON(obj->dev, obj->import_attach);
 
 	ret = mutex_lock_interruptible(&shmem->pages_lock);
 	if (ret)
@@ -225,7 +227,7 @@ static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem)
 {
 	struct drm_gem_object *obj = &shmem->base;
 
-	if (WARN_ON_ONCE(!shmem->pages_use_count))
+	if (drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count))
 		return;
 
 	if (--shmem->pages_use_count > 0)
@@ -268,7 +270,9 @@ EXPORT_SYMBOL(drm_gem_shmem_put_pages);
  */
 int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem)
 {
-	WARN_ON(shmem->base.import_attach);
+	struct drm_gem_object *obj = &shmem->base;
+
+	drm_WARN_ON(obj->dev, obj->import_attach);
 
 	return drm_gem_shmem_get_pages(shmem);
 }
@@ -283,7 +287,9 @@ EXPORT_SYMBOL(drm_gem_shmem_pin);
  */
 void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem)
 {
-	WARN_ON(shmem->base.import_attach);
+	struct drm_gem_object *obj = &shmem->base;
+
+	drm_WARN_ON(obj->dev, obj->import_attach);
 
 	drm_gem_shmem_put_pages(shmem);
 }
@@ -295,24 +301,22 @@ static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem,
 	struct drm_gem_object *obj = &shmem->base;
 	int ret = 0;
 
-	if (shmem->vmap_use_count++ > 0) {
-		iosys_map_set_vaddr(map, shmem->vaddr);
-		return 0;
-	}
-
 	if (obj->import_attach) {
 		ret = dma_buf_vmap(obj->import_attach->dmabuf, map);
 		if (!ret) {
-			if (WARN_ON(map->is_iomem)) {
+			if (drm_WARN_ON(obj->dev, map->is_iomem)) {
 				dma_buf_vunmap(obj->import_attach->dmabuf, map);
-				ret = -EIO;
-				goto err_put_pages;
+				return -EIO;
 			}
-			shmem->vaddr = map->vaddr;
 		}
 	} else {
 		pgprot_t prot = PAGE_KERNEL;
 
+		if (shmem->vmap_use_count++ > 0) {
+			iosys_map_set_vaddr(map, shmem->vaddr);
+			return 0;
+		}
+
 		ret = drm_gem_shmem_get_pages(shmem);
 		if (ret)
 			goto err_zero_use;
@@ -328,7 +332,7 @@ static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem,
 	}
 
 	if (ret) {
-		DRM_DEBUG_KMS("Failed to vmap pages, error %d\n", ret);
+		drm_dbg_kms(obj->dev, "Failed to vmap pages, error %d\n", ret);
 		goto err_put_pages;
 	}
 
@@ -378,15 +382,15 @@ static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem,
 {
 	struct drm_gem_object *obj = &shmem->base;
 
-	if (WARN_ON_ONCE(!shmem->vmap_use_count))
-		return;
-
-	if (--shmem->vmap_use_count > 0)
-		return;
-
 	if (obj->import_attach) {
 		dma_buf_vunmap(obj->import_attach->dmabuf, map);
 	} else {
+		if (drm_WARN_ON_ONCE(obj->dev, !shmem->vmap_use_count))
+			return;
+
+		if (--shmem->vmap_use_count > 0)
+			return;
+
 		vunmap(shmem->vaddr);
 		drm_gem_shmem_put_pages(shmem);
 	}
@@ -461,7 +465,7 @@ void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem)
 	struct drm_gem_object *obj = &shmem->base;
 	struct drm_device *dev = obj->dev;
 
-	WARN_ON(!drm_gem_shmem_is_purgeable(shmem));
+	drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem));
 
 	dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0);
 	sg_free_table(shmem->sgt);
@@ -550,7 +554,7 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf)
 	mutex_lock(&shmem->pages_lock);
 
 	if (page_offset >= num_pages ||
-	    WARN_ON_ONCE(!shmem->pages) ||
+	    drm_WARN_ON_ONCE(obj->dev, !shmem->pages) ||
 	    shmem->madv < 0) {
 		ret = VM_FAULT_SIGBUS;
 	} else {
@@ -569,7 +573,7 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma)
 	struct drm_gem_object *obj = vma->vm_private_data;
 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
 
-	WARN_ON(shmem->base.import_attach);
+	drm_WARN_ON(obj->dev, obj->import_attach);
 
 	mutex_lock(&shmem->pages_lock);
 
@@ -578,7 +582,7 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma)
 	 * mmap'd, vm_open() just grabs an additional reference for the new
 	 * mm the vma is getting copied into (ie. on fork()).
 	 */
-	if (!WARN_ON_ONCE(!shmem->pages_use_count))
+	if (!drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count))
 		shmem->pages_use_count++;
 
 	mutex_unlock(&shmem->pages_lock);
@@ -651,6 +655,9 @@ EXPORT_SYMBOL_GPL(drm_gem_shmem_mmap);
 void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem,
 			      struct drm_printer *p, unsigned int indent)
 {
+	if (shmem->base.import_attach)
+		return;
+
 	drm_printf_indent(p, indent, "pages_use_count=%u\n", shmem->pages_use_count);
 	drm_printf_indent(p, indent, "vmap_use_count=%u\n", shmem->vmap_use_count);
 	drm_printf_indent(p, indent, "vaddr=%p\n", shmem->vaddr);
@@ -675,7 +682,7 @@ struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem)
 {
 	struct drm_gem_object *obj = &shmem->base;
 
-	WARN_ON(shmem->base.import_attach);
+	drm_WARN_ON(obj->dev, obj->import_attach);
 
 	return drm_prime_pages_to_sg(obj->dev, shmem->pages, obj->size >> PAGE_SHIFT);
 }
@@ -690,7 +697,7 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_
 	if (shmem->sgt)
 		return shmem->sgt;
 
-	WARN_ON(obj->import_attach);
+	drm_WARN_ON(obj->dev, obj->import_attach);
 
 	ret = drm_gem_shmem_get_pages_locked(shmem);
 	if (ret)
diff --git a/drivers/gpu/drm/drm_gem_vram_helper.c b/drivers/gpu/drm/drm_gem_vram_helper.c
index d40b3edb52d0..0bea3df2a16d 100644
--- a/drivers/gpu/drm/drm_gem_vram_helper.c
+++ b/drivers/gpu/drm/drm_gem_vram_helper.c
@@ -916,6 +916,17 @@ static int bo_driver_move(struct ttm_buffer_object *bo,
 {
 	struct drm_gem_vram_object *gbo;
 
+	if (!bo->resource) {
+		if (new_mem->mem_type != TTM_PL_SYSTEM) {
+			hop->mem_type = TTM_PL_SYSTEM;
+			hop->flags = TTM_PL_FLAG_TEMPORARY;
+			return -EMULTIHOP;
+		}
+
+		ttm_bo_move_null(bo, new_mem);
+		return 0;
+	}
+
 	gbo = drm_gem_vram_of_bo(bo);
 
 	return drm_gem_vram_bo_driver_move(gbo, evict, ctx, new_mem);
diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h
index ed2103ee272c..d7e023bbb0d5 100644
--- a/drivers/gpu/drm/drm_internal.h
+++ b/drivers/gpu/drm/drm_internal.h
@@ -178,9 +178,6 @@ void drm_gem_unpin(struct drm_gem_object *obj);
 int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map);
 void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map);
 
-int drm_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev,
-			 u32 handle);
-
 /* drm_debugfs.c drm_debugfs_crc.c */
 #if defined(CONFIG_DEBUG_FS)
 int drm_debugfs_init(struct drm_minor *minor, int minor_id,
diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c
index b41aaf2bb9f1..295382cd09b0 100644
--- a/drivers/gpu/drm/drm_mipi_dsi.c
+++ b/drivers/gpu/drm/drm_mipi_dsi.c
@@ -160,7 +160,7 @@ of_mipi_dsi_device_add(struct mipi_dsi_host *host, struct device_node *node)
 	int ret;
 	u32 reg;
 
-	if (of_modalias_node(node, info.type, sizeof(info.type)) < 0) {
+	if (of_alias_from_compatible(node, info.type, sizeof(info.type)) < 0) {
 		drm_err(host, "modalias failure on %pOF\n", node);
 		return ERR_PTR(-EINVAL);
 	}
@@ -329,7 +329,7 @@ int mipi_dsi_host_register(struct mipi_dsi_host *host)
 
 	for_each_available_child_of_node(host->dev->of_node, node) {
 		/* skip nodes without reg property */
-		if (!of_find_property(node, "reg", NULL))
+		if (!of_property_present(node, "reg"))
 			continue;
 		of_mipi_dsi_device_add(host, node);
 	}
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index 40d482a01178..ac9a406250c5 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -2339,8 +2339,7 @@ static int drm_mode_parse_cmdline_named_mode(const char *name,
  * @mode: preallocated drm_cmdline_mode structure to fill out
  *
  * This parses @mode_option command line modeline for modes and options to
- * configure the connector. If @mode_option is NULL the default command line
- * modeline in fb_mode_option will be parsed instead.
+ * configure the connector.
  *
  * This uses the same parameters as the fb modedb.c, except for an extra
  * force-enable, force-enable-digital and force-disable bit at the end::
diff --git a/drivers/gpu/drm/drm_of.c b/drivers/gpu/drm/drm_of.c
index 7bbcb999bb75..177b600895d3 100644
--- a/drivers/gpu/drm/drm_of.c
+++ b/drivers/gpu/drm/drm_of.c
@@ -10,6 +10,7 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_device.h>
 #include <drm/drm_encoder.h>
+#include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
@@ -493,3 +494,53 @@ int drm_of_get_data_lanes_count_ep(const struct device_node *port,
 	return ret;
 }
 EXPORT_SYMBOL_GPL(drm_of_get_data_lanes_count_ep);
+
+#if IS_ENABLED(CONFIG_DRM_MIPI_DSI)
+
+/**
+ * drm_of_get_dsi_bus - find the DSI bus for a given device
+ * @dev: parent device of display (SPI, I2C)
+ *
+ * Gets parent DSI bus for a DSI device controlled through a bus other
+ * than MIPI-DCS (SPI, I2C, etc.) using the Device Tree.
+ *
+ * Returns pointer to mipi_dsi_host if successful, -EINVAL if the
+ * request is unsupported, -EPROBE_DEFER if the DSI host is found but
+ * not available, or -ENODEV otherwise.
+ */
+struct mipi_dsi_host *drm_of_get_dsi_bus(struct device *dev)
+{
+	struct mipi_dsi_host *dsi_host;
+	struct device_node *endpoint, *dsi_host_node;
+
+	/*
+	 * Get first endpoint child from device.
+	 */
+	endpoint = of_graph_get_next_endpoint(dev->of_node, NULL);
+	if (!endpoint)
+		return ERR_PTR(-ENODEV);
+
+	/*
+	 * Follow the first endpoint to get the DSI host node and then
+	 * release the endpoint since we no longer need it.
+	 */
+	dsi_host_node = of_graph_get_remote_port_parent(endpoint);
+	of_node_put(endpoint);
+	if (!dsi_host_node)
+		return ERR_PTR(-ENODEV);
+
+	/*
+	 * Get the DSI host from the DSI host node. If we get an error
+	 * or the return is null assume we're not ready to probe just
+	 * yet. Release the DSI host node since we're done with it.
+	 */
+	dsi_host = of_find_mipi_dsi_host_by_node(dsi_host_node);
+	of_node_put(dsi_host_node);
+	if (IS_ERR_OR_NULL(dsi_host))
+		return ERR_PTR(-EPROBE_DEFER);
+
+	return dsi_host;
+}
+EXPORT_SYMBOL_GPL(drm_of_get_dsi_bus);
+
+#endif /* CONFIG_DRM_MIPI_DSI */
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index f924b8b4ab6b..d29dafce9bb0 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -285,7 +285,7 @@ EXPORT_SYMBOL(drm_gem_dmabuf_release);
 
 /**
  * drm_gem_prime_fd_to_handle - PRIME import function for GEM drivers
- * @dev: dev to export the buffer from
+ * @dev: drm_device to import into
  * @file_priv: drm file-private structure
  * @prime_fd: fd id of the dma-buf which should be imported
  * @handle: pointer to storage for the handle of the imported buffer object
@@ -544,7 +544,8 @@ int drm_prime_handle_to_fd_ioctl(struct drm_device *dev, void *data,
  * Optional pinning of buffers is handled at dma-buf attach and detach time in
  * drm_gem_map_attach() and drm_gem_map_detach(). Backing storage itself is
  * handled by drm_gem_map_dma_buf() and drm_gem_unmap_dma_buf(), which relies on
- * &drm_gem_object_funcs.get_sg_table.
+ * &drm_gem_object_funcs.get_sg_table. If &drm_gem_object_funcs.get_sg_table is
+ * unimplemented, exports into another device are rejected.
  *
  * For kernel-internal access there's drm_gem_dmabuf_vmap() and
  * drm_gem_dmabuf_vunmap(). Userspace mmap support is provided by
@@ -583,6 +584,9 @@ int drm_gem_map_attach(struct dma_buf *dma_buf,
 {
 	struct drm_gem_object *obj = dma_buf->priv;
 
+	if (!obj->funcs->get_sg_table)
+		return -ENOSYS;
+
 	return drm_gem_pin(obj);
 }
 EXPORT_SYMBOL(drm_gem_map_attach);
@@ -925,7 +929,7 @@ struct drm_gem_object *drm_gem_prime_import_dev(struct drm_device *dev,
 		obj = dma_buf->priv;
 		if (obj->dev == dev) {
 			/*
-			 * Importing dmabuf exported from out own gem increases
+			 * Importing dmabuf exported from our own gem increases
 			 * refcount on gem itself instead of f_count of dmabuf.
 			 */
 			drm_gem_object_get(obj);
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index 8127be134c39..2fb9bf901a2c 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -590,8 +590,9 @@ retry:
 		 */
 		dev->mode_config.delayed_event = true;
 		if (dev->mode_config.poll_enabled)
-			schedule_delayed_work(&dev->mode_config.output_poll_work,
-					      0);
+			mod_delayed_work(system_wq,
+					 &dev->mode_config.output_poll_work,
+					 0);
 	}
 
 	/* Re-enable polling in case the global poll config changed. */
diff --git a/drivers/gpu/drm/drm_suballoc.c b/drivers/gpu/drm/drm_suballoc.c
new file mode 100644
index 000000000000..38cc7a123819
--- /dev/null
+++ b/drivers/gpu/drm/drm_suballoc.c
@@ -0,0 +1,457 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright 2011 Red Hat Inc.
+ * Copyright 2023 Intel Corporation.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ */
+/* Algorithm:
+ *
+ * We store the last allocated bo in "hole", we always try to allocate
+ * after the last allocated bo. Principle is that in a linear GPU ring
+ * progression was is after last is the oldest bo we allocated and thus
+ * the first one that should no longer be in use by the GPU.
+ *
+ * If it's not the case we skip over the bo after last to the closest
+ * done bo if such one exist. If none exist and we are not asked to
+ * block we report failure to allocate.
+ *
+ * If we are asked to block we wait on all the oldest fence of all
+ * rings. We just wait for any of those fence to complete.
+ */
+
+#include <drm/drm_suballoc.h>
+#include <drm/drm_print.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/dma-fence.h>
+
+static void drm_suballoc_remove_locked(struct drm_suballoc *sa);
+static void drm_suballoc_try_free(struct drm_suballoc_manager *sa_manager);
+
+/**
+ * drm_suballoc_manager_init() - Initialise the drm_suballoc_manager
+ * @sa_manager: pointer to the sa_manager
+ * @size: number of bytes we want to suballocate
+ * @align: alignment for each suballocated chunk
+ *
+ * Prepares the suballocation manager for suballocations.
+ */
+void drm_suballoc_manager_init(struct drm_suballoc_manager *sa_manager,
+			       size_t size, size_t align)
+{
+	unsigned int i;
+
+	BUILD_BUG_ON(!is_power_of_2(DRM_SUBALLOC_MAX_QUEUES));
+
+	if (!align)
+		align = 1;
+
+	/* alignment must be a power of 2 */
+	if (WARN_ON_ONCE(align & (align - 1)))
+		align = roundup_pow_of_two(align);
+
+	init_waitqueue_head(&sa_manager->wq);
+	sa_manager->size = size;
+	sa_manager->align = align;
+	sa_manager->hole = &sa_manager->olist;
+	INIT_LIST_HEAD(&sa_manager->olist);
+	for (i = 0; i < DRM_SUBALLOC_MAX_QUEUES; ++i)
+		INIT_LIST_HEAD(&sa_manager->flist[i]);
+}
+EXPORT_SYMBOL(drm_suballoc_manager_init);
+
+/**
+ * drm_suballoc_manager_fini() - Destroy the drm_suballoc_manager
+ * @sa_manager: pointer to the sa_manager
+ *
+ * Cleans up the suballocation manager after use. All fences added
+ * with drm_suballoc_free() must be signaled, or we cannot clean up
+ * the entire manager.
+ */
+void drm_suballoc_manager_fini(struct drm_suballoc_manager *sa_manager)
+{
+	struct drm_suballoc *sa, *tmp;
+
+	if (!sa_manager->size)
+		return;
+
+	if (!list_empty(&sa_manager->olist)) {
+		sa_manager->hole = &sa_manager->olist;
+		drm_suballoc_try_free(sa_manager);
+		if (!list_empty(&sa_manager->olist))
+			DRM_ERROR("sa_manager is not empty, clearing anyway\n");
+	}
+	list_for_each_entry_safe(sa, tmp, &sa_manager->olist, olist) {
+		drm_suballoc_remove_locked(sa);
+	}
+
+	sa_manager->size = 0;
+}
+EXPORT_SYMBOL(drm_suballoc_manager_fini);
+
+static void drm_suballoc_remove_locked(struct drm_suballoc *sa)
+{
+	struct drm_suballoc_manager *sa_manager = sa->manager;
+
+	if (sa_manager->hole == &sa->olist)
+		sa_manager->hole = sa->olist.prev;
+
+	list_del_init(&sa->olist);
+	list_del_init(&sa->flist);
+	dma_fence_put(sa->fence);
+	kfree(sa);
+}
+
+static void drm_suballoc_try_free(struct drm_suballoc_manager *sa_manager)
+{
+	struct drm_suballoc *sa, *tmp;
+
+	if (sa_manager->hole->next == &sa_manager->olist)
+		return;
+
+	sa = list_entry(sa_manager->hole->next, struct drm_suballoc, olist);
+	list_for_each_entry_safe_from(sa, tmp, &sa_manager->olist, olist) {
+		if (!sa->fence || !dma_fence_is_signaled(sa->fence))
+			return;
+
+		drm_suballoc_remove_locked(sa);
+	}
+}
+
+static size_t drm_suballoc_hole_soffset(struct drm_suballoc_manager *sa_manager)
+{
+	struct list_head *hole = sa_manager->hole;
+
+	if (hole != &sa_manager->olist)
+		return list_entry(hole, struct drm_suballoc, olist)->eoffset;
+
+	return 0;
+}
+
+static size_t drm_suballoc_hole_eoffset(struct drm_suballoc_manager *sa_manager)
+{
+	struct list_head *hole = sa_manager->hole;
+
+	if (hole->next != &sa_manager->olist)
+		return list_entry(hole->next, struct drm_suballoc, olist)->soffset;
+	return sa_manager->size;
+}
+
+static bool drm_suballoc_try_alloc(struct drm_suballoc_manager *sa_manager,
+				   struct drm_suballoc *sa,
+				   size_t size, size_t align)
+{
+	size_t soffset, eoffset, wasted;
+
+	soffset = drm_suballoc_hole_soffset(sa_manager);
+	eoffset = drm_suballoc_hole_eoffset(sa_manager);
+	wasted = round_up(soffset, align) - soffset;
+
+	if ((eoffset - soffset) >= (size + wasted)) {
+		soffset += wasted;
+
+		sa->manager = sa_manager;
+		sa->soffset = soffset;
+		sa->eoffset = soffset + size;
+		list_add(&sa->olist, sa_manager->hole);
+		INIT_LIST_HEAD(&sa->flist);
+		sa_manager->hole = &sa->olist;
+		return true;
+	}
+	return false;
+}
+
+static bool __drm_suballoc_event(struct drm_suballoc_manager *sa_manager,
+				 size_t size, size_t align)
+{
+	size_t soffset, eoffset, wasted;
+	unsigned int i;
+
+	for (i = 0; i < DRM_SUBALLOC_MAX_QUEUES; ++i)
+		if (!list_empty(&sa_manager->flist[i]))
+			return true;
+
+	soffset = drm_suballoc_hole_soffset(sa_manager);
+	eoffset = drm_suballoc_hole_eoffset(sa_manager);
+	wasted = round_up(soffset, align) - soffset;
+
+	return ((eoffset - soffset) >= (size + wasted));
+}
+
+/**
+ * drm_suballoc_event() - Check if we can stop waiting
+ * @sa_manager: pointer to the sa_manager
+ * @size: number of bytes we want to allocate
+ * @align: alignment we need to match
+ *
+ * Return: true if either there is a fence we can wait for or
+ * enough free memory to satisfy the allocation directly.
+ * false otherwise.
+ */
+static bool drm_suballoc_event(struct drm_suballoc_manager *sa_manager,
+			       size_t size, size_t align)
+{
+	bool ret;
+
+	spin_lock(&sa_manager->wq.lock);
+	ret = __drm_suballoc_event(sa_manager, size, align);
+	spin_unlock(&sa_manager->wq.lock);
+	return ret;
+}
+
+static bool drm_suballoc_next_hole(struct drm_suballoc_manager *sa_manager,
+				   struct dma_fence **fences,
+				   unsigned int *tries)
+{
+	struct drm_suballoc *best_bo = NULL;
+	unsigned int i, best_idx;
+	size_t soffset, best, tmp;
+
+	/* if hole points to the end of the buffer */
+	if (sa_manager->hole->next == &sa_manager->olist) {
+		/* try again with its beginning */
+		sa_manager->hole = &sa_manager->olist;
+		return true;
+	}
+
+	soffset = drm_suballoc_hole_soffset(sa_manager);
+	/* to handle wrap around we add sa_manager->size */
+	best = sa_manager->size * 2;
+	/* go over all fence list and try to find the closest sa
+	 * of the current last
+	 */
+	for (i = 0; i < DRM_SUBALLOC_MAX_QUEUES; ++i) {
+		struct drm_suballoc *sa;
+
+		fences[i] = NULL;
+
+		if (list_empty(&sa_manager->flist[i]))
+			continue;
+
+		sa = list_first_entry(&sa_manager->flist[i],
+				      struct drm_suballoc, flist);
+
+		if (!dma_fence_is_signaled(sa->fence)) {
+			fences[i] = sa->fence;
+			continue;
+		}
+
+		/* limit the number of tries each freelist gets */
+		if (tries[i] > 2)
+			continue;
+
+		tmp = sa->soffset;
+		if (tmp < soffset) {
+			/* wrap around, pretend it's after */
+			tmp += sa_manager->size;
+		}
+		tmp -= soffset;
+		if (tmp < best) {
+			/* this sa bo is the closest one */
+			best = tmp;
+			best_idx = i;
+			best_bo = sa;
+		}
+	}
+
+	if (best_bo) {
+		++tries[best_idx];
+		sa_manager->hole = best_bo->olist.prev;
+
+		/*
+		 * We know that this one is signaled,
+		 * so it's safe to remove it.
+		 */
+		drm_suballoc_remove_locked(best_bo);
+		return true;
+	}
+	return false;
+}
+
+/**
+ * drm_suballoc_new() - Make a suballocation.
+ * @sa_manager: pointer to the sa_manager
+ * @size: number of bytes we want to suballocate.
+ * @gfp: gfp flags used for memory allocation. Typically GFP_KERNEL but
+ *       the argument is provided for suballocations from reclaim context or
+ *       where the caller wants to avoid pipelining rather than wait for
+ *       reclaim.
+ * @intr: Whether to perform waits interruptible. This should typically
+ *        always be true, unless the caller needs to propagate a
+ *        non-interruptible context from above layers.
+ * @align: Alignment. Must not exceed the default manager alignment.
+ *         If @align is zero, then the manager alignment is used.
+ *
+ * Try to make a suballocation of size @size, which will be rounded
+ * up to the alignment specified in specified in drm_suballoc_manager_init().
+ *
+ * Return: a new suballocated bo, or an ERR_PTR.
+ */
+struct drm_suballoc *
+drm_suballoc_new(struct drm_suballoc_manager *sa_manager, size_t size,
+		 gfp_t gfp, bool intr, size_t align)
+{
+	struct dma_fence *fences[DRM_SUBALLOC_MAX_QUEUES];
+	unsigned int tries[DRM_SUBALLOC_MAX_QUEUES];
+	unsigned int count;
+	int i, r;
+	struct drm_suballoc *sa;
+
+	if (WARN_ON_ONCE(align > sa_manager->align))
+		return ERR_PTR(-EINVAL);
+	if (WARN_ON_ONCE(size > sa_manager->size || !size))
+		return ERR_PTR(-EINVAL);
+
+	if (!align)
+		align = sa_manager->align;
+
+	sa = kmalloc(sizeof(*sa), gfp);
+	if (!sa)
+		return ERR_PTR(-ENOMEM);
+	sa->manager = sa_manager;
+	sa->fence = NULL;
+	INIT_LIST_HEAD(&sa->olist);
+	INIT_LIST_HEAD(&sa->flist);
+
+	spin_lock(&sa_manager->wq.lock);
+	do {
+		for (i = 0; i < DRM_SUBALLOC_MAX_QUEUES; ++i)
+			tries[i] = 0;
+
+		do {
+			drm_suballoc_try_free(sa_manager);
+
+			if (drm_suballoc_try_alloc(sa_manager, sa,
+						   size, align)) {
+				spin_unlock(&sa_manager->wq.lock);
+				return sa;
+			}
+
+			/* see if we can skip over some allocations */
+		} while (drm_suballoc_next_hole(sa_manager, fences, tries));
+
+		for (i = 0, count = 0; i < DRM_SUBALLOC_MAX_QUEUES; ++i)
+			if (fences[i])
+				fences[count++] = dma_fence_get(fences[i]);
+
+		if (count) {
+			long t;
+
+			spin_unlock(&sa_manager->wq.lock);
+			t = dma_fence_wait_any_timeout(fences, count, intr,
+						       MAX_SCHEDULE_TIMEOUT,
+						       NULL);
+			for (i = 0; i < count; ++i)
+				dma_fence_put(fences[i]);
+
+			r = (t > 0) ? 0 : t;
+			spin_lock(&sa_manager->wq.lock);
+		} else if (intr) {
+			/* if we have nothing to wait for block */
+			r = wait_event_interruptible_locked
+				(sa_manager->wq,
+				 __drm_suballoc_event(sa_manager, size, align));
+		} else {
+			spin_unlock(&sa_manager->wq.lock);
+			wait_event(sa_manager->wq,
+				   drm_suballoc_event(sa_manager, size, align));
+			r = 0;
+			spin_lock(&sa_manager->wq.lock);
+		}
+	} while (!r);
+
+	spin_unlock(&sa_manager->wq.lock);
+	kfree(sa);
+	return ERR_PTR(r);
+}
+EXPORT_SYMBOL(drm_suballoc_new);
+
+/**
+ * drm_suballoc_free - Free a suballocation
+ * @suballoc: pointer to the suballocation
+ * @fence: fence that signals when suballocation is idle
+ *
+ * Free the suballocation. The suballocation can be re-used after @fence signals.
+ */
+void drm_suballoc_free(struct drm_suballoc *suballoc,
+		       struct dma_fence *fence)
+{
+	struct drm_suballoc_manager *sa_manager;
+
+	if (!suballoc)
+		return;
+
+	sa_manager = suballoc->manager;
+
+	spin_lock(&sa_manager->wq.lock);
+	if (fence && !dma_fence_is_signaled(fence)) {
+		u32 idx;
+
+		suballoc->fence = dma_fence_get(fence);
+		idx = fence->context & (DRM_SUBALLOC_MAX_QUEUES - 1);
+		list_add_tail(&suballoc->flist, &sa_manager->flist[idx]);
+	} else {
+		drm_suballoc_remove_locked(suballoc);
+	}
+	wake_up_all_locked(&sa_manager->wq);
+	spin_unlock(&sa_manager->wq.lock);
+}
+EXPORT_SYMBOL(drm_suballoc_free);
+
+#ifdef CONFIG_DEBUG_FS
+void drm_suballoc_dump_debug_info(struct drm_suballoc_manager *sa_manager,
+				  struct drm_printer *p,
+				  unsigned long long suballoc_base)
+{
+	struct drm_suballoc *i;
+
+	spin_lock(&sa_manager->wq.lock);
+	list_for_each_entry(i, &sa_manager->olist, olist) {
+		unsigned long long soffset = i->soffset;
+		unsigned long long eoffset = i->eoffset;
+
+		if (&i->olist == sa_manager->hole)
+			drm_puts(p, ">");
+		else
+			drm_puts(p, " ");
+
+		drm_printf(p, "[0x%010llx 0x%010llx] size %8lld",
+			   suballoc_base + soffset, suballoc_base + eoffset,
+			   eoffset - soffset);
+
+		if (i->fence)
+			drm_printf(p, " protected by 0x%016llx on context %llu",
+				   (unsigned long long)i->fence->seqno,
+				   (unsigned long long)i->fence->context);
+
+		drm_puts(p, "\n");
+	}
+	spin_unlock(&sa_manager->wq.lock);
+}
+EXPORT_SYMBOL(drm_suballoc_dump_debug_info);
+#endif
+MODULE_AUTHOR("Multiple");
+MODULE_DESCRIPTION("Range suballocator helper");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index 183130355997..3c22a803201d 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -112,7 +112,7 @@ int drm_sysfs_init(void)
 {
 	int err;
 
-	drm_class = class_create(THIS_MODULE, "drm");
+	drm_class = class_create("drm");
 	if (IS_ERR(drm_class))
 		return PTR_ERR(drm_class);
 
diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c
index 2ff31717a3de..877e2067534f 100644
--- a/drivers/gpu/drm/drm_vblank.c
+++ b/drivers/gpu/drm/drm_vblank.c
@@ -844,10 +844,9 @@ bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
 EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
 
 /**
- * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
- *                             vblank interval
- * @dev: DRM device
- * @pipe: index of CRTC whose vblank timestamp to retrieve
+ * drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most
+ *                                  recent vblank interval
+ * @crtc: CRTC whose vblank timestamp to retrieve
  * @tvblank: Pointer to target time which should receive the timestamp
  * @in_vblank_irq:
  *     True when called from drm_crtc_handle_vblank().  Some drivers
@@ -865,10 +864,9 @@ EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
  * True if timestamp is considered to be very precise, false otherwise.
  */
 static bool
-drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
-			  ktime_t *tvblank, bool in_vblank_irq)
+drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank,
+			       bool in_vblank_irq)
 {
-	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 	bool ret = false;
 
 	/* Define requested maximum error on timestamps (nanoseconds). */
@@ -876,8 +874,6 @@ drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
 
 	/* Query driver if possible and precision timestamping enabled. */
 	if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
-		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
-
 		ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
 							tvblank, in_vblank_irq);
 	}
@@ -891,6 +887,15 @@ drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
 	return ret;
 }
 
+static bool
+drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
+			  ktime_t *tvblank, bool in_vblank_irq)
+{
+	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
+
+	return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq);
+}
+
 /**
  * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
  * @crtc: which counter to retrieve
@@ -980,6 +985,42 @@ u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
 }
 EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
 
+/**
+ * drm_crtc_next_vblank_start - calculate the time of the next vblank
+ * @crtc: the crtc for which to calculate next vblank time
+ * @vblanktime: pointer to time to receive the next vblank timestamp.
+ *
+ * Calculate the expected time of the start of the next vblank period,
+ * based on time of previous vblank and frame duration
+ */
+int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime)
+{
+	unsigned int pipe = drm_crtc_index(crtc);
+	struct drm_vblank_crtc *vblank;
+	struct drm_display_mode *mode;
+	u64 vblank_start;
+
+	if (!drm_dev_has_vblank(crtc->dev))
+		return -EINVAL;
+
+	vblank = &crtc->dev->vblank[pipe];
+	mode = &vblank->hwmode;
+
+	if (!vblank->framedur_ns || !vblank->linedur_ns)
+		return -EINVAL;
+
+	if (!drm_crtc_get_last_vbltimestamp(crtc, vblanktime, false))
+		return -EINVAL;
+
+	vblank_start = DIV_ROUND_DOWN_ULL(
+			(u64)vblank->framedur_ns * mode->crtc_vblank_start,
+			mode->crtc_vtotal);
+	*vblanktime  = ktime_add(*vblanktime, ns_to_ktime(vblank_start));
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_crtc_next_vblank_start);
+
 static void send_vblank_event(struct drm_device *dev,
 		struct drm_pending_vblank_event *e,
 		u64 seq, ktime_t now)
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig
index 3d2f025d4fd4..0cb92d651ff1 100644
--- a/drivers/gpu/drm/exynos/Kconfig
+++ b/drivers/gpu/drm/exynos/Kconfig
@@ -2,7 +2,7 @@
 config DRM_EXYNOS
 	tristate "DRM Support for Samsung SoC Exynos Series"
 	depends on OF && DRM && COMMON_CLK
-	depends on ARCH_S3C64XX || ARCH_S5PV210 || ARCH_EXYNOS || ARCH_MULTIPLATFORM || COMPILE_TEST
+	depends on ARCH_S3C64XX || ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST
 	depends on MMU
 	select DRM_DISPLAY_HELPER if DRM_EXYNOS_DP
 	select DRM_KMS_HELPER
@@ -59,6 +59,7 @@ config DRM_EXYNOS_DSI
 	depends on DRM_EXYNOS_FIMD || DRM_EXYNOS5433_DECON || DRM_EXYNOS7_DECON
 	select DRM_MIPI_DSI
 	select DRM_PANEL
+	select DRM_SAMSUNG_DSIM
 	default n
 	help
 	  This enables support for Exynos MIPI-DSI device.
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index 16c539657f73..6b73fb7a83c3 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -16,7 +16,6 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_helper.h>
 #include <drm/drm_file.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_ioctl.h>
@@ -108,7 +107,6 @@ static const struct drm_driver exynos_drm_driver = {
 	.driver_features	= DRIVER_MODESET | DRIVER_GEM
 				  | DRIVER_ATOMIC | DRIVER_RENDER,
 	.open			= exynos_drm_open,
-	.lastclose		= drm_fb_helper_lastclose,
 	.postclose		= exynos_drm_postclose,
 	.dumb_create		= exynos_drm_gem_dumb_create,
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
@@ -288,19 +286,15 @@ static int exynos_drm_bind(struct device *dev)
 	/* init kms poll for handling hpd */
 	drm_kms_helper_poll_init(drm);
 
-	ret = exynos_drm_fbdev_init(drm);
-	if (ret)
-		goto err_cleanup_poll;
-
 	/* register the DRM device */
 	ret = drm_dev_register(drm, 0);
 	if (ret < 0)
-		goto err_cleanup_fbdev;
+		goto err_cleanup_poll;
+
+	exynos_drm_fbdev_setup(drm);
 
 	return 0;
 
-err_cleanup_fbdev:
-	exynos_drm_fbdev_fini(drm);
 err_cleanup_poll:
 	drm_kms_helper_poll_fini(drm);
 err_unbind_all:
@@ -321,7 +315,6 @@ static void exynos_drm_unbind(struct device *dev)
 
 	drm_dev_unregister(drm);
 
-	exynos_drm_fbdev_fini(drm);
 	drm_kms_helper_poll_fini(drm);
 
 	component_unbind_all(drm->dev, drm);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index 6ae9056e7a18..81d501efd013 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -197,8 +197,6 @@ struct drm_exynos_file_private {
  * @wait: wait an atomic commit to finish
  */
 struct exynos_drm_private {
-	struct drm_fb_helper *fb_helper;
-
 	struct device *g2d_dev;
 	struct device *dma_dev;
 	void *mapping;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
index 06d6513ddaae..fc81f728e6ba 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dsi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
@@ -1,1521 +1,56 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Samsung SoC MIPI DSI Master driver.
+ * Samsung MIPI DSIM glue for Exynos SoCs.
  *
  * Copyright (c) 2014 Samsung Electronics Co., Ltd
  *
  * Contacts: Tomasz Figa <t.figa@samsung.com>
-*/
+ */
 
-#include <linux/clk.h>
-#include <linux/delay.h>
 #include <linux/component.h>
-#include <linux/gpio/consumer.h>
-#include <linux/irq.h>
 #include <linux/of_device.h>
-#include <linux/of_graph.h>
-#include <linux/phy/phy.h>
-#include <linux/regulator/consumer.h>
-
-#include <asm/unaligned.h>
 
-#include <video/mipi_display.h>
-#include <video/videomode.h>
-
-#include <drm/drm_atomic_helper.h>
-#include <drm/drm_bridge.h>
-#include <drm/drm_mipi_dsi.h>
-#include <drm/drm_panel.h>
-#include <drm/drm_print.h>
+#include <drm/bridge/samsung-dsim.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
 
 #include "exynos_drm_crtc.h"
 #include "exynos_drm_drv.h"
 
-/* returns true iff both arguments logically differs */
-#define NEQV(a, b) (!(a) ^ !(b))
-
-/* DSIM_STATUS */
-#define DSIM_STOP_STATE_DAT(x)		(((x) & 0xf) << 0)
-#define DSIM_STOP_STATE_CLK		(1 << 8)
-#define DSIM_TX_READY_HS_CLK		(1 << 10)
-#define DSIM_PLL_STABLE			(1 << 31)
-
-/* DSIM_SWRST */
-#define DSIM_FUNCRST			(1 << 16)
-#define DSIM_SWRST			(1 << 0)
-
-/* DSIM_TIMEOUT */
-#define DSIM_LPDR_TIMEOUT(x)		((x) << 0)
-#define DSIM_BTA_TIMEOUT(x)		((x) << 16)
-
-/* DSIM_CLKCTRL */
-#define DSIM_ESC_PRESCALER(x)		(((x) & 0xffff) << 0)
-#define DSIM_ESC_PRESCALER_MASK		(0xffff << 0)
-#define DSIM_LANE_ESC_CLK_EN_CLK	(1 << 19)
-#define DSIM_LANE_ESC_CLK_EN_DATA(x)	(((x) & 0xf) << 20)
-#define DSIM_LANE_ESC_CLK_EN_DATA_MASK	(0xf << 20)
-#define DSIM_BYTE_CLKEN			(1 << 24)
-#define DSIM_BYTE_CLK_SRC(x)		(((x) & 0x3) << 25)
-#define DSIM_BYTE_CLK_SRC_MASK		(0x3 << 25)
-#define DSIM_PLL_BYPASS			(1 << 27)
-#define DSIM_ESC_CLKEN			(1 << 28)
-#define DSIM_TX_REQUEST_HSCLK		(1 << 31)
-
-/* DSIM_CONFIG */
-#define DSIM_LANE_EN_CLK		(1 << 0)
-#define DSIM_LANE_EN(x)			(((x) & 0xf) << 1)
-#define DSIM_NUM_OF_DATA_LANE(x)	(((x) & 0x3) << 5)
-#define DSIM_SUB_PIX_FORMAT(x)		(((x) & 0x7) << 8)
-#define DSIM_MAIN_PIX_FORMAT_MASK	(0x7 << 12)
-#define DSIM_MAIN_PIX_FORMAT_RGB888	(0x7 << 12)
-#define DSIM_MAIN_PIX_FORMAT_RGB666	(0x6 << 12)
-#define DSIM_MAIN_PIX_FORMAT_RGB666_P	(0x5 << 12)
-#define DSIM_MAIN_PIX_FORMAT_RGB565	(0x4 << 12)
-#define DSIM_SUB_VC			(((x) & 0x3) << 16)
-#define DSIM_MAIN_VC			(((x) & 0x3) << 18)
-#define DSIM_HSA_DISABLE_MODE		(1 << 20)
-#define DSIM_HBP_DISABLE_MODE		(1 << 21)
-#define DSIM_HFP_DISABLE_MODE		(1 << 22)
-/*
- * The i.MX 8M Mini Applications Processor Reference Manual,
- * Rev. 3, 11/2020 Page 4091
- * The i.MX 8M Nano Applications Processor Reference Manual,
- * Rev. 2, 07/2022 Page 3058
- * The i.MX 8M Plus Applications Processor Reference Manual,
- * Rev. 1, 06/2021 Page 5436
- * named this bit as 'HseDisableMode' but the bit definition
- * is quite opposite like
- * 0 = Disables transfer
- * 1 = Enables transfer
- * which clearly states that HSE is not a disable bit.
- *
- * This bit is named as per the manual even though it is not
- * a disable bit however the driver logic for handling HSE
- * is based on the MIPI_DSI_MODE_VIDEO_HSE flag itself.
- */
-#define DSIM_HSE_DISABLE_MODE		(1 << 23)
-#define DSIM_AUTO_MODE			(1 << 24)
-#define DSIM_VIDEO_MODE			(1 << 25)
-#define DSIM_BURST_MODE			(1 << 26)
-#define DSIM_SYNC_INFORM		(1 << 27)
-#define DSIM_EOT_DISABLE		(1 << 28)
-#define DSIM_MFLUSH_VS			(1 << 29)
-/* This flag is valid only for exynos3250/3472/5260/5430 */
-#define DSIM_CLKLANE_STOP		(1 << 30)
-
-/* DSIM_ESCMODE */
-#define DSIM_TX_TRIGGER_RST		(1 << 4)
-#define DSIM_TX_LPDT_LP			(1 << 6)
-#define DSIM_CMD_LPDT_LP		(1 << 7)
-#define DSIM_FORCE_BTA			(1 << 16)
-#define DSIM_FORCE_STOP_STATE		(1 << 20)
-#define DSIM_STOP_STATE_CNT(x)		(((x) & 0x7ff) << 21)
-#define DSIM_STOP_STATE_CNT_MASK	(0x7ff << 21)
-
-/* DSIM_MDRESOL */
-#define DSIM_MAIN_STAND_BY		(1 << 31)
-#define DSIM_MAIN_VRESOL(x, num_bits)	(((x) & ((1 << (num_bits)) - 1)) << 16)
-#define DSIM_MAIN_HRESOL(x, num_bits)	(((x) & ((1 << (num_bits)) - 1)) << 0)
-
-/* DSIM_MVPORCH */
-#define DSIM_CMD_ALLOW(x)		((x) << 28)
-#define DSIM_STABLE_VFP(x)		((x) << 16)
-#define DSIM_MAIN_VBP(x)		((x) << 0)
-#define DSIM_CMD_ALLOW_MASK		(0xf << 28)
-#define DSIM_STABLE_VFP_MASK		(0x7ff << 16)
-#define DSIM_MAIN_VBP_MASK		(0x7ff << 0)
-
-/* DSIM_MHPORCH */
-#define DSIM_MAIN_HFP(x)		((x) << 16)
-#define DSIM_MAIN_HBP(x)		((x) << 0)
-#define DSIM_MAIN_HFP_MASK		((0xffff) << 16)
-#define DSIM_MAIN_HBP_MASK		((0xffff) << 0)
-
-/* DSIM_MSYNC */
-#define DSIM_MAIN_VSA(x)		((x) << 22)
-#define DSIM_MAIN_HSA(x)		((x) << 0)
-#define DSIM_MAIN_VSA_MASK		((0x3ff) << 22)
-#define DSIM_MAIN_HSA_MASK		((0xffff) << 0)
-
-/* DSIM_SDRESOL */
-#define DSIM_SUB_STANDY(x)		((x) << 31)
-#define DSIM_SUB_VRESOL(x)		((x) << 16)
-#define DSIM_SUB_HRESOL(x)		((x) << 0)
-#define DSIM_SUB_STANDY_MASK		((0x1) << 31)
-#define DSIM_SUB_VRESOL_MASK		((0x7ff) << 16)
-#define DSIM_SUB_HRESOL_MASK		((0x7ff) << 0)
-
-/* DSIM_INTSRC */
-#define DSIM_INT_PLL_STABLE		(1 << 31)
-#define DSIM_INT_SW_RST_RELEASE		(1 << 30)
-#define DSIM_INT_SFR_FIFO_EMPTY		(1 << 29)
-#define DSIM_INT_SFR_HDR_FIFO_EMPTY	(1 << 28)
-#define DSIM_INT_BTA			(1 << 25)
-#define DSIM_INT_FRAME_DONE		(1 << 24)
-#define DSIM_INT_RX_TIMEOUT		(1 << 21)
-#define DSIM_INT_BTA_TIMEOUT		(1 << 20)
-#define DSIM_INT_RX_DONE		(1 << 18)
-#define DSIM_INT_RX_TE			(1 << 17)
-#define DSIM_INT_RX_ACK			(1 << 16)
-#define DSIM_INT_RX_ECC_ERR		(1 << 15)
-#define DSIM_INT_RX_CRC_ERR		(1 << 14)
-
-/* DSIM_FIFOCTRL */
-#define DSIM_RX_DATA_FULL		(1 << 25)
-#define DSIM_RX_DATA_EMPTY		(1 << 24)
-#define DSIM_SFR_HEADER_FULL		(1 << 23)
-#define DSIM_SFR_HEADER_EMPTY		(1 << 22)
-#define DSIM_SFR_PAYLOAD_FULL		(1 << 21)
-#define DSIM_SFR_PAYLOAD_EMPTY		(1 << 20)
-#define DSIM_I80_HEADER_FULL		(1 << 19)
-#define DSIM_I80_HEADER_EMPTY		(1 << 18)
-#define DSIM_I80_PAYLOAD_FULL		(1 << 17)
-#define DSIM_I80_PAYLOAD_EMPTY		(1 << 16)
-#define DSIM_SD_HEADER_FULL		(1 << 15)
-#define DSIM_SD_HEADER_EMPTY		(1 << 14)
-#define DSIM_SD_PAYLOAD_FULL		(1 << 13)
-#define DSIM_SD_PAYLOAD_EMPTY		(1 << 12)
-#define DSIM_MD_HEADER_FULL		(1 << 11)
-#define DSIM_MD_HEADER_EMPTY		(1 << 10)
-#define DSIM_MD_PAYLOAD_FULL		(1 << 9)
-#define DSIM_MD_PAYLOAD_EMPTY		(1 << 8)
-#define DSIM_RX_FIFO			(1 << 4)
-#define DSIM_SFR_FIFO			(1 << 3)
-#define DSIM_I80_FIFO			(1 << 2)
-#define DSIM_SD_FIFO			(1 << 1)
-#define DSIM_MD_FIFO			(1 << 0)
-
-/* DSIM_PHYACCHR */
-#define DSIM_AFC_EN			(1 << 14)
-#define DSIM_AFC_CTL(x)			(((x) & 0x7) << 5)
-
-/* DSIM_PLLCTRL */
-#define DSIM_FREQ_BAND(x)		((x) << 24)
-#define DSIM_PLL_EN			(1 << 23)
-#define DSIM_PLL_P(x)			((x) << 13)
-#define DSIM_PLL_M(x)			((x) << 4)
-#define DSIM_PLL_S(x)			((x) << 1)
-
-/* DSIM_PHYCTRL */
-#define DSIM_PHYCTRL_ULPS_EXIT(x)	(((x) & 0x1ff) << 0)
-#define DSIM_PHYCTRL_B_DPHYCTL_VREG_LP	(1 << 30)
-#define DSIM_PHYCTRL_B_DPHYCTL_SLEW_UP	(1 << 14)
-
-/* DSIM_PHYTIMING */
-#define DSIM_PHYTIMING_LPX(x)		((x) << 8)
-#define DSIM_PHYTIMING_HS_EXIT(x)	((x) << 0)
-
-/* DSIM_PHYTIMING1 */
-#define DSIM_PHYTIMING1_CLK_PREPARE(x)	((x) << 24)
-#define DSIM_PHYTIMING1_CLK_ZERO(x)	((x) << 16)
-#define DSIM_PHYTIMING1_CLK_POST(x)	((x) << 8)
-#define DSIM_PHYTIMING1_CLK_TRAIL(x)	((x) << 0)
-
-/* DSIM_PHYTIMING2 */
-#define DSIM_PHYTIMING2_HS_PREPARE(x)	((x) << 16)
-#define DSIM_PHYTIMING2_HS_ZERO(x)	((x) << 8)
-#define DSIM_PHYTIMING2_HS_TRAIL(x)	((x) << 0)
-
-#define DSI_MAX_BUS_WIDTH		4
-#define DSI_NUM_VIRTUAL_CHANNELS	4
-#define DSI_TX_FIFO_SIZE		2048
-#define DSI_RX_FIFO_SIZE		256
-#define DSI_XFER_TIMEOUT_MS		100
-#define DSI_RX_FIFO_EMPTY		0x30800002
-
-#define OLD_SCLK_MIPI_CLK_NAME "pll_clk"
-
-static const char *const clk_names[5] = { "bus_clk", "sclk_mipi",
-	"phyclk_mipidphy0_bitclkdiv8", "phyclk_mipidphy0_rxclkesc0",
-	"sclk_rgb_vclk_to_dsim0" };
-
-enum exynos_dsi_transfer_type {
-	EXYNOS_DSI_TX,
-	EXYNOS_DSI_RX,
-};
-
-struct exynos_dsi_transfer {
-	struct list_head list;
-	struct completion completed;
-	int result;
-	struct mipi_dsi_packet packet;
-	u16 flags;
-	u16 tx_done;
-
-	u8 *rx_payload;
-	u16 rx_len;
-	u16 rx_done;
-};
-
-#define DSIM_STATE_ENABLED		BIT(0)
-#define DSIM_STATE_INITIALIZED		BIT(1)
-#define DSIM_STATE_CMD_LPM		BIT(2)
-#define DSIM_STATE_VIDOUT_AVAILABLE	BIT(3)
-
-struct exynos_dsi_driver_data {
-	const unsigned int *reg_ofs;
-	unsigned int plltmr_reg;
-	unsigned int has_freqband:1;
-	unsigned int has_clklane_stop:1;
-	unsigned int num_clks;
-	unsigned int max_freq;
-	unsigned int wait_for_reset;
-	unsigned int num_bits_resol;
-	const unsigned int *reg_values;
-};
-
 struct exynos_dsi {
 	struct drm_encoder encoder;
-	struct mipi_dsi_host dsi_host;
-	struct drm_bridge bridge;
-	struct drm_bridge *out_bridge;
-	struct device *dev;
-	struct drm_display_mode mode;
-
-	void __iomem *reg_base;
-	struct phy *phy;
-	struct clk **clks;
-	struct regulator_bulk_data supplies[2];
-	int irq;
-	struct gpio_desc *te_gpio;
-
-	u32 pll_clk_rate;
-	u32 burst_clk_rate;
-	u32 esc_clk_rate;
-	u32 lanes;
-	u32 mode_flags;
-	u32 format;
-
-	int state;
-	struct drm_property *brightness;
-	struct completion completed;
-
-	spinlock_t transfer_lock; /* protects transfer_list */
-	struct list_head transfer_list;
-
-	const struct exynos_dsi_driver_data *driver_data;
-};
-
-#define host_to_dsi(host) container_of(host, struct exynos_dsi, dsi_host)
-
-static inline struct exynos_dsi *bridge_to_dsi(struct drm_bridge *b)
-{
-	return container_of(b, struct exynos_dsi, bridge);
-}
-
-enum reg_idx {
-	DSIM_STATUS_REG,	/* Status register */
-	DSIM_SWRST_REG,		/* Software reset register */
-	DSIM_CLKCTRL_REG,	/* Clock control register */
-	DSIM_TIMEOUT_REG,	/* Time out register */
-	DSIM_CONFIG_REG,	/* Configuration register */
-	DSIM_ESCMODE_REG,	/* Escape mode register */
-	DSIM_MDRESOL_REG,
-	DSIM_MVPORCH_REG,	/* Main display Vporch register */
-	DSIM_MHPORCH_REG,	/* Main display Hporch register */
-	DSIM_MSYNC_REG,		/* Main display sync area register */
-	DSIM_INTSRC_REG,	/* Interrupt source register */
-	DSIM_INTMSK_REG,	/* Interrupt mask register */
-	DSIM_PKTHDR_REG,	/* Packet Header FIFO register */
-	DSIM_PAYLOAD_REG,	/* Payload FIFO register */
-	DSIM_RXFIFO_REG,	/* Read FIFO register */
-	DSIM_FIFOCTRL_REG,	/* FIFO status and control register */
-	DSIM_PLLCTRL_REG,	/* PLL control register */
-	DSIM_PHYCTRL_REG,
-	DSIM_PHYTIMING_REG,
-	DSIM_PHYTIMING1_REG,
-	DSIM_PHYTIMING2_REG,
-	NUM_REGS
-};
-
-static inline void exynos_dsi_write(struct exynos_dsi *dsi, enum reg_idx idx,
-				    u32 val)
-{
-
-	writel(val, dsi->reg_base + dsi->driver_data->reg_ofs[idx]);
-}
-
-static inline u32 exynos_dsi_read(struct exynos_dsi *dsi, enum reg_idx idx)
-{
-	return readl(dsi->reg_base + dsi->driver_data->reg_ofs[idx]);
-}
-
-static const unsigned int exynos_reg_ofs[] = {
-	[DSIM_STATUS_REG] =  0x00,
-	[DSIM_SWRST_REG] =  0x04,
-	[DSIM_CLKCTRL_REG] =  0x08,
-	[DSIM_TIMEOUT_REG] =  0x0c,
-	[DSIM_CONFIG_REG] =  0x10,
-	[DSIM_ESCMODE_REG] =  0x14,
-	[DSIM_MDRESOL_REG] =  0x18,
-	[DSIM_MVPORCH_REG] =  0x1c,
-	[DSIM_MHPORCH_REG] =  0x20,
-	[DSIM_MSYNC_REG] =  0x24,
-	[DSIM_INTSRC_REG] =  0x2c,
-	[DSIM_INTMSK_REG] =  0x30,
-	[DSIM_PKTHDR_REG] =  0x34,
-	[DSIM_PAYLOAD_REG] =  0x38,
-	[DSIM_RXFIFO_REG] =  0x3c,
-	[DSIM_FIFOCTRL_REG] =  0x44,
-	[DSIM_PLLCTRL_REG] =  0x4c,
-	[DSIM_PHYCTRL_REG] =  0x5c,
-	[DSIM_PHYTIMING_REG] =  0x64,
-	[DSIM_PHYTIMING1_REG] =  0x68,
-	[DSIM_PHYTIMING2_REG] =  0x6c,
-};
-
-static const unsigned int exynos5433_reg_ofs[] = {
-	[DSIM_STATUS_REG] = 0x04,
-	[DSIM_SWRST_REG] = 0x0C,
-	[DSIM_CLKCTRL_REG] = 0x10,
-	[DSIM_TIMEOUT_REG] = 0x14,
-	[DSIM_CONFIG_REG] = 0x18,
-	[DSIM_ESCMODE_REG] = 0x1C,
-	[DSIM_MDRESOL_REG] = 0x20,
-	[DSIM_MVPORCH_REG] = 0x24,
-	[DSIM_MHPORCH_REG] = 0x28,
-	[DSIM_MSYNC_REG] = 0x2C,
-	[DSIM_INTSRC_REG] = 0x34,
-	[DSIM_INTMSK_REG] = 0x38,
-	[DSIM_PKTHDR_REG] = 0x3C,
-	[DSIM_PAYLOAD_REG] = 0x40,
-	[DSIM_RXFIFO_REG] = 0x44,
-	[DSIM_FIFOCTRL_REG] = 0x4C,
-	[DSIM_PLLCTRL_REG] = 0x94,
-	[DSIM_PHYCTRL_REG] = 0xA4,
-	[DSIM_PHYTIMING_REG] = 0xB4,
-	[DSIM_PHYTIMING1_REG] = 0xB8,
-	[DSIM_PHYTIMING2_REG] = 0xBC,
-};
-
-enum reg_value_idx {
-	RESET_TYPE,
-	PLL_TIMER,
-	STOP_STATE_CNT,
-	PHYCTRL_ULPS_EXIT,
-	PHYCTRL_VREG_LP,
-	PHYCTRL_SLEW_UP,
-	PHYTIMING_LPX,
-	PHYTIMING_HS_EXIT,
-	PHYTIMING_CLK_PREPARE,
-	PHYTIMING_CLK_ZERO,
-	PHYTIMING_CLK_POST,
-	PHYTIMING_CLK_TRAIL,
-	PHYTIMING_HS_PREPARE,
-	PHYTIMING_HS_ZERO,
-	PHYTIMING_HS_TRAIL
-};
-
-static const unsigned int reg_values[] = {
-	[RESET_TYPE] = DSIM_SWRST,
-	[PLL_TIMER] = 500,
-	[STOP_STATE_CNT] = 0xf,
-	[PHYCTRL_ULPS_EXIT] = DSIM_PHYCTRL_ULPS_EXIT(0x0af),
-	[PHYCTRL_VREG_LP] = 0,
-	[PHYCTRL_SLEW_UP] = 0,
-	[PHYTIMING_LPX] = DSIM_PHYTIMING_LPX(0x06),
-	[PHYTIMING_HS_EXIT] = DSIM_PHYTIMING_HS_EXIT(0x0b),
-	[PHYTIMING_CLK_PREPARE] = DSIM_PHYTIMING1_CLK_PREPARE(0x07),
-	[PHYTIMING_CLK_ZERO] = DSIM_PHYTIMING1_CLK_ZERO(0x27),
-	[PHYTIMING_CLK_POST] = DSIM_PHYTIMING1_CLK_POST(0x0d),
-	[PHYTIMING_CLK_TRAIL] = DSIM_PHYTIMING1_CLK_TRAIL(0x08),
-	[PHYTIMING_HS_PREPARE] = DSIM_PHYTIMING2_HS_PREPARE(0x09),
-	[PHYTIMING_HS_ZERO] = DSIM_PHYTIMING2_HS_ZERO(0x0d),
-	[PHYTIMING_HS_TRAIL] = DSIM_PHYTIMING2_HS_TRAIL(0x0b),
-};
-
-static const unsigned int exynos5422_reg_values[] = {
-	[RESET_TYPE] = DSIM_SWRST,
-	[PLL_TIMER] = 500,
-	[STOP_STATE_CNT] = 0xf,
-	[PHYCTRL_ULPS_EXIT] = DSIM_PHYCTRL_ULPS_EXIT(0xaf),
-	[PHYCTRL_VREG_LP] = 0,
-	[PHYCTRL_SLEW_UP] = 0,
-	[PHYTIMING_LPX] = DSIM_PHYTIMING_LPX(0x08),
-	[PHYTIMING_HS_EXIT] = DSIM_PHYTIMING_HS_EXIT(0x0d),
-	[PHYTIMING_CLK_PREPARE] = DSIM_PHYTIMING1_CLK_PREPARE(0x09),
-	[PHYTIMING_CLK_ZERO] = DSIM_PHYTIMING1_CLK_ZERO(0x30),
-	[PHYTIMING_CLK_POST] = DSIM_PHYTIMING1_CLK_POST(0x0e),
-	[PHYTIMING_CLK_TRAIL] = DSIM_PHYTIMING1_CLK_TRAIL(0x0a),
-	[PHYTIMING_HS_PREPARE] = DSIM_PHYTIMING2_HS_PREPARE(0x0c),
-	[PHYTIMING_HS_ZERO] = DSIM_PHYTIMING2_HS_ZERO(0x11),
-	[PHYTIMING_HS_TRAIL] = DSIM_PHYTIMING2_HS_TRAIL(0x0d),
-};
-
-static const unsigned int exynos5433_reg_values[] = {
-	[RESET_TYPE] = DSIM_FUNCRST,
-	[PLL_TIMER] = 22200,
-	[STOP_STATE_CNT] = 0xa,
-	[PHYCTRL_ULPS_EXIT] = DSIM_PHYCTRL_ULPS_EXIT(0x190),
-	[PHYCTRL_VREG_LP] = DSIM_PHYCTRL_B_DPHYCTL_VREG_LP,
-	[PHYCTRL_SLEW_UP] = DSIM_PHYCTRL_B_DPHYCTL_SLEW_UP,
-	[PHYTIMING_LPX] = DSIM_PHYTIMING_LPX(0x07),
-	[PHYTIMING_HS_EXIT] = DSIM_PHYTIMING_HS_EXIT(0x0c),
-	[PHYTIMING_CLK_PREPARE] = DSIM_PHYTIMING1_CLK_PREPARE(0x09),
-	[PHYTIMING_CLK_ZERO] = DSIM_PHYTIMING1_CLK_ZERO(0x2d),
-	[PHYTIMING_CLK_POST] = DSIM_PHYTIMING1_CLK_POST(0x0e),
-	[PHYTIMING_CLK_TRAIL] = DSIM_PHYTIMING1_CLK_TRAIL(0x09),
-	[PHYTIMING_HS_PREPARE] = DSIM_PHYTIMING2_HS_PREPARE(0x0b),
-	[PHYTIMING_HS_ZERO] = DSIM_PHYTIMING2_HS_ZERO(0x10),
-	[PHYTIMING_HS_TRAIL] = DSIM_PHYTIMING2_HS_TRAIL(0x0c),
-};
-
-static const struct exynos_dsi_driver_data exynos3_dsi_driver_data = {
-	.reg_ofs = exynos_reg_ofs,
-	.plltmr_reg = 0x50,
-	.has_freqband = 1,
-	.has_clklane_stop = 1,
-	.num_clks = 2,
-	.max_freq = 1000,
-	.wait_for_reset = 1,
-	.num_bits_resol = 11,
-	.reg_values = reg_values,
 };
 
-static const struct exynos_dsi_driver_data exynos4_dsi_driver_data = {
-	.reg_ofs = exynos_reg_ofs,
-	.plltmr_reg = 0x50,
-	.has_freqband = 1,
-	.has_clklane_stop = 1,
-	.num_clks = 2,
-	.max_freq = 1000,
-	.wait_for_reset = 1,
-	.num_bits_resol = 11,
-	.reg_values = reg_values,
-};
-
-static const struct exynos_dsi_driver_data exynos5_dsi_driver_data = {
-	.reg_ofs = exynos_reg_ofs,
-	.plltmr_reg = 0x58,
-	.num_clks = 2,
-	.max_freq = 1000,
-	.wait_for_reset = 1,
-	.num_bits_resol = 11,
-	.reg_values = reg_values,
-};
-
-static const struct exynos_dsi_driver_data exynos5433_dsi_driver_data = {
-	.reg_ofs = exynos5433_reg_ofs,
-	.plltmr_reg = 0xa0,
-	.has_clklane_stop = 1,
-	.num_clks = 5,
-	.max_freq = 1500,
-	.wait_for_reset = 0,
-	.num_bits_resol = 12,
-	.reg_values = exynos5433_reg_values,
-};
-
-static const struct exynos_dsi_driver_data exynos5422_dsi_driver_data = {
-	.reg_ofs = exynos5433_reg_ofs,
-	.plltmr_reg = 0xa0,
-	.has_clklane_stop = 1,
-	.num_clks = 2,
-	.max_freq = 1500,
-	.wait_for_reset = 1,
-	.num_bits_resol = 12,
-	.reg_values = exynos5422_reg_values,
-};
-
-static const struct of_device_id exynos_dsi_of_match[] = {
-	{ .compatible = "samsung,exynos3250-mipi-dsi",
-	  .data = &exynos3_dsi_driver_data },
-	{ .compatible = "samsung,exynos4210-mipi-dsi",
-	  .data = &exynos4_dsi_driver_data },
-	{ .compatible = "samsung,exynos5410-mipi-dsi",
-	  .data = &exynos5_dsi_driver_data },
-	{ .compatible = "samsung,exynos5422-mipi-dsi",
-	  .data = &exynos5422_dsi_driver_data },
-	{ .compatible = "samsung,exynos5433-mipi-dsi",
-	  .data = &exynos5433_dsi_driver_data },
-	{ }
-};
-
-static void exynos_dsi_wait_for_reset(struct exynos_dsi *dsi)
-{
-	if (wait_for_completion_timeout(&dsi->completed, msecs_to_jiffies(300)))
-		return;
-
-	dev_err(dsi->dev, "timeout waiting for reset\n");
-}
-
-static void exynos_dsi_reset(struct exynos_dsi *dsi)
-{
-	u32 reset_val = dsi->driver_data->reg_values[RESET_TYPE];
-
-	reinit_completion(&dsi->completed);
-	exynos_dsi_write(dsi, DSIM_SWRST_REG, reset_val);
-}
-
-#ifndef MHZ
-#define MHZ	(1000*1000)
-#endif
-
-static unsigned long exynos_dsi_pll_find_pms(struct exynos_dsi *dsi,
-		unsigned long fin, unsigned long fout, u8 *p, u16 *m, u8 *s)
-{
-	const struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
-	unsigned long best_freq = 0;
-	u32 min_delta = 0xffffffff;
-	u8 p_min, p_max;
-	u8 _p, best_p;
-	u16 _m, best_m;
-	u8 _s, best_s;
-
-	p_min = DIV_ROUND_UP(fin, (12 * MHZ));
-	p_max = fin / (6 * MHZ);
-
-	for (_p = p_min; _p <= p_max; ++_p) {
-		for (_s = 0; _s <= 5; ++_s) {
-			u64 tmp;
-			u32 delta;
-
-			tmp = (u64)fout * (_p << _s);
-			do_div(tmp, fin);
-			_m = tmp;
-			if (_m < 41 || _m > 125)
-				continue;
-
-			tmp = (u64)_m * fin;
-			do_div(tmp, _p);
-			if (tmp < 500 * MHZ ||
-					tmp > driver_data->max_freq * MHZ)
-				continue;
-
-			tmp = (u64)_m * fin;
-			do_div(tmp, _p << _s);
-
-			delta = abs(fout - tmp);
-			if (delta < min_delta) {
-				best_p = _p;
-				best_m = _m;
-				best_s = _s;
-				min_delta = delta;
-				best_freq = tmp;
-			}
-		}
-	}
-
-	if (best_freq) {
-		*p = best_p;
-		*m = best_m;
-		*s = best_s;
-	}
-
-	return best_freq;
-}
-
-static unsigned long exynos_dsi_set_pll(struct exynos_dsi *dsi,
-					unsigned long freq)
-{
-	const struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
-	unsigned long fin, fout;
-	int timeout;
-	u8 p, s;
-	u16 m;
-	u32 reg;
-
-	fin = dsi->pll_clk_rate;
-	fout = exynos_dsi_pll_find_pms(dsi, fin, freq, &p, &m, &s);
-	if (!fout) {
-		dev_err(dsi->dev,
-			"failed to find PLL PMS for requested frequency\n");
-		return 0;
-	}
-	dev_dbg(dsi->dev, "PLL freq %lu, (p %d, m %d, s %d)\n", fout, p, m, s);
-
-	writel(driver_data->reg_values[PLL_TIMER],
-			dsi->reg_base + driver_data->plltmr_reg);
-
-	reg = DSIM_PLL_EN | DSIM_PLL_P(p) | DSIM_PLL_M(m) | DSIM_PLL_S(s);
-
-	if (driver_data->has_freqband) {
-		static const unsigned long freq_bands[] = {
-			100 * MHZ, 120 * MHZ, 160 * MHZ, 200 * MHZ,
-			270 * MHZ, 320 * MHZ, 390 * MHZ, 450 * MHZ,
-			510 * MHZ, 560 * MHZ, 640 * MHZ, 690 * MHZ,
-			770 * MHZ, 870 * MHZ, 950 * MHZ,
-		};
-		int band;
-
-		for (band = 0; band < ARRAY_SIZE(freq_bands); ++band)
-			if (fout < freq_bands[band])
-				break;
-
-		dev_dbg(dsi->dev, "band %d\n", band);
-
-		reg |= DSIM_FREQ_BAND(band);
-	}
-
-	exynos_dsi_write(dsi, DSIM_PLLCTRL_REG, reg);
-
-	timeout = 1000;
-	do {
-		if (timeout-- == 0) {
-			dev_err(dsi->dev, "PLL failed to stabilize\n");
-			return 0;
-		}
-		reg = exynos_dsi_read(dsi, DSIM_STATUS_REG);
-	} while ((reg & DSIM_PLL_STABLE) == 0);
-
-	return fout;
-}
-
-static int exynos_dsi_enable_clock(struct exynos_dsi *dsi)
-{
-	unsigned long hs_clk, byte_clk, esc_clk;
-	unsigned long esc_div;
-	u32 reg;
-
-	hs_clk = exynos_dsi_set_pll(dsi, dsi->burst_clk_rate);
-	if (!hs_clk) {
-		dev_err(dsi->dev, "failed to configure DSI PLL\n");
-		return -EFAULT;
-	}
-
-	byte_clk = hs_clk / 8;
-	esc_div = DIV_ROUND_UP(byte_clk, dsi->esc_clk_rate);
-	esc_clk = byte_clk / esc_div;
-
-	if (esc_clk > 20 * MHZ) {
-		++esc_div;
-		esc_clk = byte_clk / esc_div;
-	}
-
-	dev_dbg(dsi->dev, "hs_clk = %lu, byte_clk = %lu, esc_clk = %lu\n",
-		hs_clk, byte_clk, esc_clk);
-
-	reg = exynos_dsi_read(dsi, DSIM_CLKCTRL_REG);
-	reg &= ~(DSIM_ESC_PRESCALER_MASK | DSIM_LANE_ESC_CLK_EN_CLK
-			| DSIM_LANE_ESC_CLK_EN_DATA_MASK | DSIM_PLL_BYPASS
-			| DSIM_BYTE_CLK_SRC_MASK);
-	reg |= DSIM_ESC_CLKEN | DSIM_BYTE_CLKEN
-			| DSIM_ESC_PRESCALER(esc_div)
-			| DSIM_LANE_ESC_CLK_EN_CLK
-			| DSIM_LANE_ESC_CLK_EN_DATA(BIT(dsi->lanes) - 1)
-			| DSIM_BYTE_CLK_SRC(0)
-			| DSIM_TX_REQUEST_HSCLK;
-	exynos_dsi_write(dsi, DSIM_CLKCTRL_REG, reg);
-
-	return 0;
-}
-
-static void exynos_dsi_set_phy_ctrl(struct exynos_dsi *dsi)
-{
-	const struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
-	const unsigned int *reg_values = driver_data->reg_values;
-	u32 reg;
-
-	if (driver_data->has_freqband)
-		return;
-
-	/* B D-PHY: D-PHY Master & Slave Analog Block control */
-	reg = reg_values[PHYCTRL_ULPS_EXIT] | reg_values[PHYCTRL_VREG_LP] |
-		reg_values[PHYCTRL_SLEW_UP];
-	exynos_dsi_write(dsi, DSIM_PHYCTRL_REG, reg);
-
-	/*
-	 * T LPX: Transmitted length of any Low-Power state period
-	 * T HS-EXIT: Time that the transmitter drives LP-11 following a HS
-	 *	burst
-	 */
-	reg = reg_values[PHYTIMING_LPX] | reg_values[PHYTIMING_HS_EXIT];
-	exynos_dsi_write(dsi, DSIM_PHYTIMING_REG, reg);
-
-	/*
-	 * T CLK-PREPARE: Time that the transmitter drives the Clock Lane LP-00
-	 *	Line state immediately before the HS-0 Line state starting the
-	 *	HS transmission
-	 * T CLK-ZERO: Time that the transmitter drives the HS-0 state prior to
-	 *	transmitting the Clock.
-	 * T CLK_POST: Time that the transmitter continues to send HS clock
-	 *	after the last associated Data Lane has transitioned to LP Mode
-	 *	Interval is defined as the period from the end of T HS-TRAIL to
-	 *	the beginning of T CLK-TRAIL
-	 * T CLK-TRAIL: Time that the transmitter drives the HS-0 state after
-	 *	the last payload clock bit of a HS transmission burst
-	 */
-	reg = reg_values[PHYTIMING_CLK_PREPARE] |
-		reg_values[PHYTIMING_CLK_ZERO] |
-		reg_values[PHYTIMING_CLK_POST] |
-		reg_values[PHYTIMING_CLK_TRAIL];
-
-	exynos_dsi_write(dsi, DSIM_PHYTIMING1_REG, reg);
-
-	/*
-	 * T HS-PREPARE: Time that the transmitter drives the Data Lane LP-00
-	 *	Line state immediately before the HS-0 Line state starting the
-	 *	HS transmission
-	 * T HS-ZERO: Time that the transmitter drives the HS-0 state prior to
-	 *	transmitting the Sync sequence.
-	 * T HS-TRAIL: Time that the transmitter drives the flipped differential
-	 *	state after last payload data bit of a HS transmission burst
-	 */
-	reg = reg_values[PHYTIMING_HS_PREPARE] | reg_values[PHYTIMING_HS_ZERO] |
-		reg_values[PHYTIMING_HS_TRAIL];
-	exynos_dsi_write(dsi, DSIM_PHYTIMING2_REG, reg);
-}
-
-static void exynos_dsi_disable_clock(struct exynos_dsi *dsi)
-{
-	u32 reg;
-
-	reg = exynos_dsi_read(dsi, DSIM_CLKCTRL_REG);
-	reg &= ~(DSIM_LANE_ESC_CLK_EN_CLK | DSIM_LANE_ESC_CLK_EN_DATA_MASK
-			| DSIM_ESC_CLKEN | DSIM_BYTE_CLKEN);
-	exynos_dsi_write(dsi, DSIM_CLKCTRL_REG, reg);
-
-	reg = exynos_dsi_read(dsi, DSIM_PLLCTRL_REG);
-	reg &= ~DSIM_PLL_EN;
-	exynos_dsi_write(dsi, DSIM_PLLCTRL_REG, reg);
-}
-
-static void exynos_dsi_enable_lane(struct exynos_dsi *dsi, u32 lane)
-{
-	u32 reg = exynos_dsi_read(dsi, DSIM_CONFIG_REG);
-	reg |= (DSIM_NUM_OF_DATA_LANE(dsi->lanes - 1) | DSIM_LANE_EN_CLK |
-			DSIM_LANE_EN(lane));
-	exynos_dsi_write(dsi, DSIM_CONFIG_REG, reg);
-}
-
-static int exynos_dsi_init_link(struct exynos_dsi *dsi)
-{
-	const struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
-	int timeout;
-	u32 reg;
-	u32 lanes_mask;
-
-	/* Initialize FIFO pointers */
-	reg = exynos_dsi_read(dsi, DSIM_FIFOCTRL_REG);
-	reg &= ~0x1f;
-	exynos_dsi_write(dsi, DSIM_FIFOCTRL_REG, reg);
-
-	usleep_range(9000, 11000);
-
-	reg |= 0x1f;
-	exynos_dsi_write(dsi, DSIM_FIFOCTRL_REG, reg);
-	usleep_range(9000, 11000);
-
-	/* DSI configuration */
-	reg = 0;
-
-	/*
-	 * The first bit of mode_flags specifies display configuration.
-	 * If this bit is set[= MIPI_DSI_MODE_VIDEO], dsi will support video
-	 * mode, otherwise it will support command mode.
-	 */
-	if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO) {
-		reg |= DSIM_VIDEO_MODE;
-
-		/*
-		 * The user manual describes that following bits are ignored in
-		 * command mode.
-		 */
-		if (!(dsi->mode_flags & MIPI_DSI_MODE_VSYNC_FLUSH))
-			reg |= DSIM_MFLUSH_VS;
-		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE)
-			reg |= DSIM_SYNC_INFORM;
-		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_BURST)
-			reg |= DSIM_BURST_MODE;
-		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_AUTO_VERT)
-			reg |= DSIM_AUTO_MODE;
-		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_HSE)
-			reg |= DSIM_HSE_DISABLE_MODE;
-		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_NO_HFP)
-			reg |= DSIM_HFP_DISABLE_MODE;
-		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_NO_HBP)
-			reg |= DSIM_HBP_DISABLE_MODE;
-		if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_NO_HSA)
-			reg |= DSIM_HSA_DISABLE_MODE;
-	}
-
-	if (dsi->mode_flags & MIPI_DSI_MODE_NO_EOT_PACKET)
-		reg |= DSIM_EOT_DISABLE;
-
-	switch (dsi->format) {
-	case MIPI_DSI_FMT_RGB888:
-		reg |= DSIM_MAIN_PIX_FORMAT_RGB888;
-		break;
-	case MIPI_DSI_FMT_RGB666:
-		reg |= DSIM_MAIN_PIX_FORMAT_RGB666;
-		break;
-	case MIPI_DSI_FMT_RGB666_PACKED:
-		reg |= DSIM_MAIN_PIX_FORMAT_RGB666_P;
-		break;
-	case MIPI_DSI_FMT_RGB565:
-		reg |= DSIM_MAIN_PIX_FORMAT_RGB565;
-		break;
-	default:
-		dev_err(dsi->dev, "invalid pixel format\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * Use non-continuous clock mode if the periparal wants and
-	 * host controller supports
-	 *
-	 * In non-continous clock mode, host controller will turn off
-	 * the HS clock between high-speed transmissions to reduce
-	 * power consumption.
-	 */
-	if (driver_data->has_clklane_stop &&
-			dsi->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS) {
-		reg |= DSIM_CLKLANE_STOP;
-	}
-	exynos_dsi_write(dsi, DSIM_CONFIG_REG, reg);
-
-	lanes_mask = BIT(dsi->lanes) - 1;
-	exynos_dsi_enable_lane(dsi, lanes_mask);
-
-	/* Check clock and data lane state are stop state */
-	timeout = 100;
-	do {
-		if (timeout-- == 0) {
-			dev_err(dsi->dev, "waiting for bus lanes timed out\n");
-			return -EFAULT;
-		}
-
-		reg = exynos_dsi_read(dsi, DSIM_STATUS_REG);
-		if ((reg & DSIM_STOP_STATE_DAT(lanes_mask))
-		    != DSIM_STOP_STATE_DAT(lanes_mask))
-			continue;
-	} while (!(reg & (DSIM_STOP_STATE_CLK | DSIM_TX_READY_HS_CLK)));
-
-	reg = exynos_dsi_read(dsi, DSIM_ESCMODE_REG);
-	reg &= ~DSIM_STOP_STATE_CNT_MASK;
-	reg |= DSIM_STOP_STATE_CNT(driver_data->reg_values[STOP_STATE_CNT]);
-	exynos_dsi_write(dsi, DSIM_ESCMODE_REG, reg);
-
-	reg = DSIM_BTA_TIMEOUT(0xff) | DSIM_LPDR_TIMEOUT(0xffff);
-	exynos_dsi_write(dsi, DSIM_TIMEOUT_REG, reg);
-
-	return 0;
-}
-
-static void exynos_dsi_set_display_mode(struct exynos_dsi *dsi)
-{
-	struct drm_display_mode *m = &dsi->mode;
-	unsigned int num_bits_resol = dsi->driver_data->num_bits_resol;
-	u32 reg;
-
-	if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO) {
-		reg = DSIM_CMD_ALLOW(0xf)
-			| DSIM_STABLE_VFP(m->vsync_start - m->vdisplay)
-			| DSIM_MAIN_VBP(m->vtotal - m->vsync_end);
-		exynos_dsi_write(dsi, DSIM_MVPORCH_REG, reg);
-
-		reg = DSIM_MAIN_HFP(m->hsync_start - m->hdisplay)
-			| DSIM_MAIN_HBP(m->htotal - m->hsync_end);
-		exynos_dsi_write(dsi, DSIM_MHPORCH_REG, reg);
-
-		reg = DSIM_MAIN_VSA(m->vsync_end - m->vsync_start)
-			| DSIM_MAIN_HSA(m->hsync_end - m->hsync_start);
-		exynos_dsi_write(dsi, DSIM_MSYNC_REG, reg);
-	}
-	reg =  DSIM_MAIN_HRESOL(m->hdisplay, num_bits_resol) |
-		DSIM_MAIN_VRESOL(m->vdisplay, num_bits_resol);
-
-	exynos_dsi_write(dsi, DSIM_MDRESOL_REG, reg);
-
-	dev_dbg(dsi->dev, "LCD size = %dx%d\n", m->hdisplay, m->vdisplay);
-}
-
-static void exynos_dsi_set_display_enable(struct exynos_dsi *dsi, bool enable)
-{
-	u32 reg;
-
-	reg = exynos_dsi_read(dsi, DSIM_MDRESOL_REG);
-	if (enable)
-		reg |= DSIM_MAIN_STAND_BY;
-	else
-		reg &= ~DSIM_MAIN_STAND_BY;
-	exynos_dsi_write(dsi, DSIM_MDRESOL_REG, reg);
-}
-
-static int exynos_dsi_wait_for_hdr_fifo(struct exynos_dsi *dsi)
-{
-	int timeout = 2000;
-
-	do {
-		u32 reg = exynos_dsi_read(dsi, DSIM_FIFOCTRL_REG);
-
-		if (!(reg & DSIM_SFR_HEADER_FULL))
-			return 0;
-
-		if (!cond_resched())
-			usleep_range(950, 1050);
-	} while (--timeout);
-
-	return -ETIMEDOUT;
-}
-
-static void exynos_dsi_set_cmd_lpm(struct exynos_dsi *dsi, bool lpm)
-{
-	u32 v = exynos_dsi_read(dsi, DSIM_ESCMODE_REG);
-
-	if (lpm)
-		v |= DSIM_CMD_LPDT_LP;
-	else
-		v &= ~DSIM_CMD_LPDT_LP;
-
-	exynos_dsi_write(dsi, DSIM_ESCMODE_REG, v);
-}
-
-static void exynos_dsi_force_bta(struct exynos_dsi *dsi)
-{
-	u32 v = exynos_dsi_read(dsi, DSIM_ESCMODE_REG);
-	v |= DSIM_FORCE_BTA;
-	exynos_dsi_write(dsi, DSIM_ESCMODE_REG, v);
-}
-
-static void exynos_dsi_send_to_fifo(struct exynos_dsi *dsi,
-					struct exynos_dsi_transfer *xfer)
-{
-	struct device *dev = dsi->dev;
-	struct mipi_dsi_packet *pkt = &xfer->packet;
-	const u8 *payload = pkt->payload + xfer->tx_done;
-	u16 length = pkt->payload_length - xfer->tx_done;
-	bool first = !xfer->tx_done;
-	u32 reg;
-
-	dev_dbg(dev, "< xfer %pK: tx len %u, done %u, rx len %u, done %u\n",
-		xfer, length, xfer->tx_done, xfer->rx_len, xfer->rx_done);
-
-	if (length > DSI_TX_FIFO_SIZE)
-		length = DSI_TX_FIFO_SIZE;
-
-	xfer->tx_done += length;
-
-	/* Send payload */
-	while (length >= 4) {
-		reg = get_unaligned_le32(payload);
-		exynos_dsi_write(dsi, DSIM_PAYLOAD_REG, reg);
-		payload += 4;
-		length -= 4;
-	}
-
-	reg = 0;
-	switch (length) {
-	case 3:
-		reg |= payload[2] << 16;
-		fallthrough;
-	case 2:
-		reg |= payload[1] << 8;
-		fallthrough;
-	case 1:
-		reg |= payload[0];
-		exynos_dsi_write(dsi, DSIM_PAYLOAD_REG, reg);
-		break;
-	}
-
-	/* Send packet header */
-	if (!first)
-		return;
-
-	reg = get_unaligned_le32(pkt->header);
-	if (exynos_dsi_wait_for_hdr_fifo(dsi)) {
-		dev_err(dev, "waiting for header FIFO timed out\n");
-		return;
-	}
-
-	if (NEQV(xfer->flags & MIPI_DSI_MSG_USE_LPM,
-		 dsi->state & DSIM_STATE_CMD_LPM)) {
-		exynos_dsi_set_cmd_lpm(dsi, xfer->flags & MIPI_DSI_MSG_USE_LPM);
-		dsi->state ^= DSIM_STATE_CMD_LPM;
-	}
-
-	exynos_dsi_write(dsi, DSIM_PKTHDR_REG, reg);
-
-	if (xfer->flags & MIPI_DSI_MSG_REQ_ACK)
-		exynos_dsi_force_bta(dsi);
-}
-
-static void exynos_dsi_read_from_fifo(struct exynos_dsi *dsi,
-					struct exynos_dsi_transfer *xfer)
-{
-	u8 *payload = xfer->rx_payload + xfer->rx_done;
-	bool first = !xfer->rx_done;
-	struct device *dev = dsi->dev;
-	u16 length;
-	u32 reg;
-
-	if (first) {
-		reg = exynos_dsi_read(dsi, DSIM_RXFIFO_REG);
-
-		switch (reg & 0x3f) {
-		case MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_2BYTE:
-		case MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_2BYTE:
-			if (xfer->rx_len >= 2) {
-				payload[1] = reg >> 16;
-				++xfer->rx_done;
-			}
-			fallthrough;
-		case MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_1BYTE:
-		case MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_1BYTE:
-			payload[0] = reg >> 8;
-			++xfer->rx_done;
-			xfer->rx_len = xfer->rx_done;
-			xfer->result = 0;
-			goto clear_fifo;
-		case MIPI_DSI_RX_ACKNOWLEDGE_AND_ERROR_REPORT:
-			dev_err(dev, "DSI Error Report: 0x%04x\n",
-				(reg >> 8) & 0xffff);
-			xfer->result = 0;
-			goto clear_fifo;
-		}
-
-		length = (reg >> 8) & 0xffff;
-		if (length > xfer->rx_len) {
-			dev_err(dev,
-				"response too long (%u > %u bytes), stripping\n",
-				xfer->rx_len, length);
-			length = xfer->rx_len;
-		} else if (length < xfer->rx_len)
-			xfer->rx_len = length;
-	}
-
-	length = xfer->rx_len - xfer->rx_done;
-	xfer->rx_done += length;
-
-	/* Receive payload */
-	while (length >= 4) {
-		reg = exynos_dsi_read(dsi, DSIM_RXFIFO_REG);
-		payload[0] = (reg >>  0) & 0xff;
-		payload[1] = (reg >>  8) & 0xff;
-		payload[2] = (reg >> 16) & 0xff;
-		payload[3] = (reg >> 24) & 0xff;
-		payload += 4;
-		length -= 4;
-	}
-
-	if (length) {
-		reg = exynos_dsi_read(dsi, DSIM_RXFIFO_REG);
-		switch (length) {
-		case 3:
-			payload[2] = (reg >> 16) & 0xff;
-			fallthrough;
-		case 2:
-			payload[1] = (reg >> 8) & 0xff;
-			fallthrough;
-		case 1:
-			payload[0] = reg & 0xff;
-		}
-	}
-
-	if (xfer->rx_done == xfer->rx_len)
-		xfer->result = 0;
-
-clear_fifo:
-	length = DSI_RX_FIFO_SIZE / 4;
-	do {
-		reg = exynos_dsi_read(dsi, DSIM_RXFIFO_REG);
-		if (reg == DSI_RX_FIFO_EMPTY)
-			break;
-	} while (--length);
-}
-
-static void exynos_dsi_transfer_start(struct exynos_dsi *dsi)
-{
-	unsigned long flags;
-	struct exynos_dsi_transfer *xfer;
-	bool start = false;
-
-again:
-	spin_lock_irqsave(&dsi->transfer_lock, flags);
-
-	if (list_empty(&dsi->transfer_list)) {
-		spin_unlock_irqrestore(&dsi->transfer_lock, flags);
-		return;
-	}
-
-	xfer = list_first_entry(&dsi->transfer_list,
-					struct exynos_dsi_transfer, list);
-
-	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
-
-	if (xfer->packet.payload_length &&
-	    xfer->tx_done == xfer->packet.payload_length)
-		/* waiting for RX */
-		return;
-
-	exynos_dsi_send_to_fifo(dsi, xfer);
-
-	if (xfer->packet.payload_length || xfer->rx_len)
-		return;
-
-	xfer->result = 0;
-	complete(&xfer->completed);
-
-	spin_lock_irqsave(&dsi->transfer_lock, flags);
-
-	list_del_init(&xfer->list);
-	start = !list_empty(&dsi->transfer_list);
-
-	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
-
-	if (start)
-		goto again;
-}
-
-static bool exynos_dsi_transfer_finish(struct exynos_dsi *dsi)
-{
-	struct exynos_dsi_transfer *xfer;
-	unsigned long flags;
-	bool start = true;
-
-	spin_lock_irqsave(&dsi->transfer_lock, flags);
-
-	if (list_empty(&dsi->transfer_list)) {
-		spin_unlock_irqrestore(&dsi->transfer_lock, flags);
-		return false;
-	}
-
-	xfer = list_first_entry(&dsi->transfer_list,
-					struct exynos_dsi_transfer, list);
-
-	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
-
-	dev_dbg(dsi->dev,
-		"> xfer %pK, tx_len %zu, tx_done %u, rx_len %u, rx_done %u\n",
-		xfer, xfer->packet.payload_length, xfer->tx_done, xfer->rx_len,
-		xfer->rx_done);
-
-	if (xfer->tx_done != xfer->packet.payload_length)
-		return true;
-
-	if (xfer->rx_done != xfer->rx_len)
-		exynos_dsi_read_from_fifo(dsi, xfer);
-
-	if (xfer->rx_done != xfer->rx_len)
-		return true;
-
-	spin_lock_irqsave(&dsi->transfer_lock, flags);
-
-	list_del_init(&xfer->list);
-	start = !list_empty(&dsi->transfer_list);
-
-	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
-
-	if (!xfer->rx_len)
-		xfer->result = 0;
-	complete(&xfer->completed);
-
-	return start;
-}
-
-static void exynos_dsi_remove_transfer(struct exynos_dsi *dsi,
-					struct exynos_dsi_transfer *xfer)
+static irqreturn_t exynos_dsi_te_irq_handler(struct samsung_dsim *dsim)
 {
-	unsigned long flags;
-	bool start;
-
-	spin_lock_irqsave(&dsi->transfer_lock, flags);
-
-	if (!list_empty(&dsi->transfer_list) &&
-	    xfer == list_first_entry(&dsi->transfer_list,
-				     struct exynos_dsi_transfer, list)) {
-		list_del_init(&xfer->list);
-		start = !list_empty(&dsi->transfer_list);
-		spin_unlock_irqrestore(&dsi->transfer_lock, flags);
-		if (start)
-			exynos_dsi_transfer_start(dsi);
-		return;
-	}
-
-	list_del_init(&xfer->list);
-
-	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
-}
-
-static int exynos_dsi_transfer(struct exynos_dsi *dsi,
-					struct exynos_dsi_transfer *xfer)
-{
-	unsigned long flags;
-	bool stopped;
-
-	xfer->tx_done = 0;
-	xfer->rx_done = 0;
-	xfer->result = -ETIMEDOUT;
-	init_completion(&xfer->completed);
-
-	spin_lock_irqsave(&dsi->transfer_lock, flags);
-
-	stopped = list_empty(&dsi->transfer_list);
-	list_add_tail(&xfer->list, &dsi->transfer_list);
-
-	spin_unlock_irqrestore(&dsi->transfer_lock, flags);
-
-	if (stopped)
-		exynos_dsi_transfer_start(dsi);
-
-	wait_for_completion_timeout(&xfer->completed,
-				    msecs_to_jiffies(DSI_XFER_TIMEOUT_MS));
-	if (xfer->result == -ETIMEDOUT) {
-		struct mipi_dsi_packet *pkt = &xfer->packet;
-		exynos_dsi_remove_transfer(dsi, xfer);
-		dev_err(dsi->dev, "xfer timed out: %*ph %*ph\n", 4, pkt->header,
-			(int)pkt->payload_length, pkt->payload);
-		return -ETIMEDOUT;
-	}
-
-	/* Also covers hardware timeout condition */
-	return xfer->result;
-}
-
-static irqreturn_t exynos_dsi_irq(int irq, void *dev_id)
-{
-	struct exynos_dsi *dsi = dev_id;
-	u32 status;
-
-	status = exynos_dsi_read(dsi, DSIM_INTSRC_REG);
-	if (!status) {
-		static unsigned long int j;
-		if (printk_timed_ratelimit(&j, 500))
-			dev_warn(dsi->dev, "spurious interrupt\n");
-		return IRQ_HANDLED;
-	}
-	exynos_dsi_write(dsi, DSIM_INTSRC_REG, status);
-
-	if (status & DSIM_INT_SW_RST_RELEASE) {
-		u32 mask = ~(DSIM_INT_RX_DONE | DSIM_INT_SFR_FIFO_EMPTY |
-			DSIM_INT_SFR_HDR_FIFO_EMPTY | DSIM_INT_RX_ECC_ERR |
-			DSIM_INT_SW_RST_RELEASE);
-		exynos_dsi_write(dsi, DSIM_INTMSK_REG, mask);
-		complete(&dsi->completed);
-		return IRQ_HANDLED;
-	}
-
-	if (!(status & (DSIM_INT_RX_DONE | DSIM_INT_SFR_FIFO_EMPTY |
-			DSIM_INT_PLL_STABLE)))
-		return IRQ_HANDLED;
-
-	if (exynos_dsi_transfer_finish(dsi))
-		exynos_dsi_transfer_start(dsi);
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t exynos_dsi_te_irq_handler(int irq, void *dev_id)
-{
-	struct exynos_dsi *dsi = (struct exynos_dsi *)dev_id;
+	struct exynos_dsi *dsi = dsim->priv;
 	struct drm_encoder *encoder = &dsi->encoder;
 
-	if (dsi->state & DSIM_STATE_VIDOUT_AVAILABLE)
+	if (dsim->state & DSIM_STATE_VIDOUT_AVAILABLE)
 		exynos_drm_crtc_te_handler(encoder->crtc);
 
 	return IRQ_HANDLED;
 }
 
-static void exynos_dsi_enable_irq(struct exynos_dsi *dsi)
-{
-	enable_irq(dsi->irq);
-
-	if (dsi->te_gpio)
-		enable_irq(gpiod_to_irq(dsi->te_gpio));
-}
-
-static void exynos_dsi_disable_irq(struct exynos_dsi *dsi)
-{
-	if (dsi->te_gpio)
-		disable_irq(gpiod_to_irq(dsi->te_gpio));
-
-	disable_irq(dsi->irq);
-}
-
-static int exynos_dsi_init(struct exynos_dsi *dsi)
-{
-	const struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
-
-	exynos_dsi_reset(dsi);
-	exynos_dsi_enable_irq(dsi);
-
-	if (driver_data->reg_values[RESET_TYPE] == DSIM_FUNCRST)
-		exynos_dsi_enable_lane(dsi, BIT(dsi->lanes) - 1);
-
-	exynos_dsi_enable_clock(dsi);
-	if (driver_data->wait_for_reset)
-		exynos_dsi_wait_for_reset(dsi);
-	exynos_dsi_set_phy_ctrl(dsi);
-	exynos_dsi_init_link(dsi);
-
-	return 0;
-}
-
-static int exynos_dsi_register_te_irq(struct exynos_dsi *dsi,
-				      struct device *panel)
-{
-	int ret;
-	int te_gpio_irq;
-
-	dsi->te_gpio = gpiod_get_optional(panel, "te", GPIOD_IN);
-	if (!dsi->te_gpio) {
-		return 0;
-	} else if (IS_ERR(dsi->te_gpio)) {
-		dev_err(dsi->dev, "gpio request failed with %ld\n",
-				PTR_ERR(dsi->te_gpio));
-		return PTR_ERR(dsi->te_gpio);
-	}
-
-	te_gpio_irq = gpiod_to_irq(dsi->te_gpio);
-
-	ret = request_threaded_irq(te_gpio_irq, exynos_dsi_te_irq_handler, NULL,
-				   IRQF_TRIGGER_RISING | IRQF_NO_AUTOEN, "TE", dsi);
-	if (ret) {
-		dev_err(dsi->dev, "request interrupt failed with %d\n", ret);
-		gpiod_put(dsi->te_gpio);
-		return ret;
-	}
-
-	return 0;
-}
-
-static void exynos_dsi_unregister_te_irq(struct exynos_dsi *dsi)
-{
-	if (dsi->te_gpio) {
-		free_irq(gpiod_to_irq(dsi->te_gpio), dsi);
-		gpiod_put(dsi->te_gpio);
-	}
-}
-
-static void exynos_dsi_atomic_pre_enable(struct drm_bridge *bridge,
-					 struct drm_bridge_state *old_bridge_state)
-{
-	struct exynos_dsi *dsi = bridge_to_dsi(bridge);
-	int ret;
-
-	if (dsi->state & DSIM_STATE_ENABLED)
-		return;
-
-	ret = pm_runtime_resume_and_get(dsi->dev);
-	if (ret < 0) {
-		dev_err(dsi->dev, "failed to enable DSI device.\n");
-		return;
-	}
-
-	dsi->state |= DSIM_STATE_ENABLED;
-}
-
-static void exynos_dsi_atomic_enable(struct drm_bridge *bridge,
-				     struct drm_bridge_state *old_bridge_state)
-{
-	struct exynos_dsi *dsi = bridge_to_dsi(bridge);
-
-	exynos_dsi_set_display_mode(dsi);
-	exynos_dsi_set_display_enable(dsi, true);
-
-	dsi->state |= DSIM_STATE_VIDOUT_AVAILABLE;
-
-	return;
-}
-
-static void exynos_dsi_atomic_disable(struct drm_bridge *bridge,
-				      struct drm_bridge_state *old_bridge_state)
-{
-	struct exynos_dsi *dsi = bridge_to_dsi(bridge);
-
-	if (!(dsi->state & DSIM_STATE_ENABLED))
-		return;
-
-	dsi->state &= ~DSIM_STATE_VIDOUT_AVAILABLE;
-}
-
-static void exynos_dsi_atomic_post_disable(struct drm_bridge *bridge,
-					   struct drm_bridge_state *old_bridge_state)
-{
-	struct exynos_dsi *dsi = bridge_to_dsi(bridge);
-
-	exynos_dsi_set_display_enable(dsi, false);
-
-	dsi->state &= ~DSIM_STATE_ENABLED;
-	pm_runtime_put_sync(dsi->dev);
-}
-
-static void exynos_dsi_mode_set(struct drm_bridge *bridge,
-				const struct drm_display_mode *mode,
-				const struct drm_display_mode *adjusted_mode)
-{
-	struct exynos_dsi *dsi = bridge_to_dsi(bridge);
-
-	drm_mode_copy(&dsi->mode, adjusted_mode);
-}
-
-static int exynos_dsi_attach(struct drm_bridge *bridge,
-			     enum drm_bridge_attach_flags flags)
-{
-	struct exynos_dsi *dsi = bridge_to_dsi(bridge);
-
-	return drm_bridge_attach(bridge->encoder, dsi->out_bridge, bridge,
-				 flags);
-}
-
-static const struct drm_bridge_funcs exynos_dsi_bridge_funcs = {
-	.atomic_duplicate_state		= drm_atomic_helper_bridge_duplicate_state,
-	.atomic_destroy_state		= drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset			= drm_atomic_helper_bridge_reset,
-	.atomic_pre_enable		= exynos_dsi_atomic_pre_enable,
-	.atomic_enable			= exynos_dsi_atomic_enable,
-	.atomic_disable			= exynos_dsi_atomic_disable,
-	.atomic_post_disable		= exynos_dsi_atomic_post_disable,
-	.mode_set			= exynos_dsi_mode_set,
-	.attach				= exynos_dsi_attach,
-};
-
-MODULE_DEVICE_TABLE(of, exynos_dsi_of_match);
-
-static int exynos_dsi_host_attach(struct mipi_dsi_host *host,
+static int exynos_dsi_host_attach(struct samsung_dsim *dsim,
 				  struct mipi_dsi_device *device)
 {
-	struct exynos_dsi *dsi = host_to_dsi(host);
-	struct device *dev = dsi->dev;
+	struct exynos_dsi *dsi = dsim->priv;
 	struct drm_encoder *encoder = &dsi->encoder;
 	struct drm_device *drm = encoder->dev;
-	struct drm_panel *panel;
-	int ret;
-
-	panel = of_drm_find_panel(device->dev.of_node);
-	if (!IS_ERR(panel)) {
-		dsi->out_bridge = devm_drm_panel_bridge_add(dev, panel);
-	} else {
-		dsi->out_bridge = of_drm_find_bridge(device->dev.of_node);
-		if (!dsi->out_bridge)
-			dsi->out_bridge = ERR_PTR(-EINVAL);
-	}
-
-	if (IS_ERR(dsi->out_bridge)) {
-		ret = PTR_ERR(dsi->out_bridge);
-		DRM_DEV_ERROR(dev, "failed to find the bridge: %d\n", ret);
-		return ret;
-	}
-
-	DRM_DEV_INFO(dev, "Attached %s device\n", device->name);
 
-	drm_bridge_add(&dsi->bridge);
-
-	drm_bridge_attach(encoder, &dsi->bridge,
+	drm_bridge_attach(encoder, &dsim->bridge,
 			  list_first_entry_or_null(&encoder->bridge_chain,
 						   struct drm_bridge,
 						   chain_node), 0);
 
-	/*
-	 * This is a temporary solution and should be made by more generic way.
-	 *
-	 * If attached panel device is for command mode one, dsi should register
-	 * TE interrupt handler.
-	 */
-	if (!(device->mode_flags & MIPI_DSI_MODE_VIDEO)) {
-		ret = exynos_dsi_register_te_irq(dsi, &device->dev);
-		if (ret)
-			return ret;
-	}
-
 	mutex_lock(&drm->mode_config.mutex);
 
-	dsi->lanes = device->lanes;
-	dsi->format = device->format;
-	dsi->mode_flags = device->mode_flags;
+	dsim->lanes = device->lanes;
+	dsim->format = device->format;
+	dsim->mode_flags = device->mode_flags;
 	exynos_drm_crtc_get_by_type(drm, EXYNOS_DISPLAY_TYPE_LCD)->i80_mode =
-			!(dsi->mode_flags & MIPI_DSI_MODE_VIDEO);
+			!(dsim->mode_flags & MIPI_DSI_MODE_VIDEO);
 
 	mutex_unlock(&drm->mode_config.mutex);
 
@@ -1525,100 +60,20 @@ static int exynos_dsi_host_attach(struct mipi_dsi_host *host,
 	return 0;
 }
 
-static int exynos_dsi_host_detach(struct mipi_dsi_host *host,
-				  struct mipi_dsi_device *device)
+static void exynos_dsi_host_detach(struct samsung_dsim *dsim,
+				   struct mipi_dsi_device *device)
 {
-	struct exynos_dsi *dsi = host_to_dsi(host);
+	struct exynos_dsi *dsi = dsim->priv;
 	struct drm_device *drm = dsi->encoder.dev;
 
-	if (dsi->out_bridge->funcs->detach)
-		dsi->out_bridge->funcs->detach(dsi->out_bridge);
-	dsi->out_bridge = NULL;
-
 	if (drm->mode_config.poll_enabled)
 		drm_kms_helper_hotplug_event(drm);
-
-	exynos_dsi_unregister_te_irq(dsi);
-
-	drm_bridge_remove(&dsi->bridge);
-
-	return 0;
 }
 
-static ssize_t exynos_dsi_host_transfer(struct mipi_dsi_host *host,
-					 const struct mipi_dsi_msg *msg)
+static int exynos_dsi_bind(struct device *dev, struct device *master, void *data)
 {
-	struct exynos_dsi *dsi = host_to_dsi(host);
-	struct exynos_dsi_transfer xfer;
-	int ret;
-
-	if (!(dsi->state & DSIM_STATE_ENABLED))
-		return -EINVAL;
-
-	if (!(dsi->state & DSIM_STATE_INITIALIZED)) {
-		ret = exynos_dsi_init(dsi);
-		if (ret)
-			return ret;
-		dsi->state |= DSIM_STATE_INITIALIZED;
-	}
-
-	ret = mipi_dsi_create_packet(&xfer.packet, msg);
-	if (ret < 0)
-		return ret;
-
-	xfer.rx_len = msg->rx_len;
-	xfer.rx_payload = msg->rx_buf;
-	xfer.flags = msg->flags;
-
-	ret = exynos_dsi_transfer(dsi, &xfer);
-	return (ret < 0) ? ret : xfer.rx_done;
-}
-
-static const struct mipi_dsi_host_ops exynos_dsi_ops = {
-	.attach = exynos_dsi_host_attach,
-	.detach = exynos_dsi_host_detach,
-	.transfer = exynos_dsi_host_transfer,
-};
-
-static int exynos_dsi_of_read_u32(const struct device_node *np,
-				  const char *propname, u32 *out_value)
-{
-	int ret = of_property_read_u32(np, propname, out_value);
-
-	if (ret < 0)
-		pr_err("%pOF: failed to get '%s' property\n", np, propname);
-
-	return ret;
-}
-
-static int exynos_dsi_parse_dt(struct exynos_dsi *dsi)
-{
-	struct device *dev = dsi->dev;
-	struct device_node *node = dev->of_node;
-	int ret;
-
-	ret = exynos_dsi_of_read_u32(node, "samsung,pll-clock-frequency",
-				     &dsi->pll_clk_rate);
-	if (ret < 0)
-		return ret;
-
-	ret = exynos_dsi_of_read_u32(node, "samsung,burst-clock-frequency",
-				     &dsi->burst_clk_rate);
-	if (ret < 0)
-		return ret;
-
-	ret = exynos_dsi_of_read_u32(node, "samsung,esc-clock-frequency",
-				     &dsi->esc_clk_rate);
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
-static int exynos_dsi_bind(struct device *dev, struct device *master,
-				void *data)
-{
-	struct exynos_dsi *dsi = dev_get_drvdata(dev);
+	struct samsung_dsim *dsim = dev_get_drvdata(dev);
+	struct exynos_dsi *dsi = dsim->priv;
 	struct drm_encoder *encoder = &dsi->encoder;
 	struct drm_device *drm_dev = data;
 	int ret;
@@ -1629,17 +84,16 @@ static int exynos_dsi_bind(struct device *dev, struct device *master,
 	if (ret < 0)
 		return ret;
 
-	return mipi_dsi_host_register(&dsi->dsi_host);
+	return mipi_dsi_host_register(&dsim->dsi_host);
 }
 
-static void exynos_dsi_unbind(struct device *dev, struct device *master,
-				void *data)
+static void exynos_dsi_unbind(struct device *dev, struct device *master, void *data)
 {
-	struct exynos_dsi *dsi = dev_get_drvdata(dev);
+	struct samsung_dsim *dsim = dev_get_drvdata(dev);
 
-	exynos_dsi_atomic_disable(&dsi->bridge, NULL);
+	dsim->bridge.funcs->atomic_disable(&dsim->bridge, NULL);
 
-	mipi_dsi_host_unregister(&dsi->dsi_host);
+	mipi_dsi_host_unregister(&dsim->dsi_host);
 }
 
 static const struct component_ops exynos_dsi_component_ops = {
@@ -1647,189 +101,90 @@ static const struct component_ops exynos_dsi_component_ops = {
 	.unbind	= exynos_dsi_unbind,
 };
 
-static int exynos_dsi_probe(struct platform_device *pdev)
+static int exynos_dsi_register_host(struct samsung_dsim *dsim)
 {
-	struct device *dev = &pdev->dev;
 	struct exynos_dsi *dsi;
-	int ret, i;
 
-	dsi = devm_kzalloc(dev, sizeof(*dsi), GFP_KERNEL);
+	dsi = devm_kzalloc(dsim->dev, sizeof(*dsi), GFP_KERNEL);
 	if (!dsi)
 		return -ENOMEM;
 
-	init_completion(&dsi->completed);
-	spin_lock_init(&dsi->transfer_lock);
-	INIT_LIST_HEAD(&dsi->transfer_list);
-
-	dsi->dsi_host.ops = &exynos_dsi_ops;
-	dsi->dsi_host.dev = dev;
+	dsim->priv = dsi;
+	dsim->bridge.pre_enable_prev_first = true;
 
-	dsi->dev = dev;
-	dsi->driver_data = of_device_get_match_data(dev);
-
-	dsi->supplies[0].supply = "vddcore";
-	dsi->supplies[1].supply = "vddio";
-	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(dsi->supplies),
-				      dsi->supplies);
-	if (ret)
-		return dev_err_probe(dev, ret, "failed to get regulators\n");
-
-	dsi->clks = devm_kcalloc(dev,
-			dsi->driver_data->num_clks, sizeof(*dsi->clks),
-			GFP_KERNEL);
-	if (!dsi->clks)
-		return -ENOMEM;
-
-	for (i = 0; i < dsi->driver_data->num_clks; i++) {
-		dsi->clks[i] = devm_clk_get(dev, clk_names[i]);
-		if (IS_ERR(dsi->clks[i])) {
-			if (strcmp(clk_names[i], "sclk_mipi") == 0) {
-				dsi->clks[i] = devm_clk_get(dev,
-							OLD_SCLK_MIPI_CLK_NAME);
-				if (!IS_ERR(dsi->clks[i]))
-					continue;
-			}
-
-			dev_info(dev, "failed to get the clock: %s\n",
-					clk_names[i]);
-			return PTR_ERR(dsi->clks[i]);
-		}
-	}
-
-	dsi->reg_base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(dsi->reg_base))
-		return PTR_ERR(dsi->reg_base);
-
-	dsi->phy = devm_phy_get(dev, "dsim");
-	if (IS_ERR(dsi->phy)) {
-		dev_info(dev, "failed to get dsim phy\n");
-		return PTR_ERR(dsi->phy);
-	}
-
-	dsi->irq = platform_get_irq(pdev, 0);
-	if (dsi->irq < 0)
-		return dsi->irq;
-
-	ret = devm_request_threaded_irq(dev, dsi->irq, NULL,
-					exynos_dsi_irq,
-					IRQF_ONESHOT | IRQF_NO_AUTOEN,
-					dev_name(dev), dsi);
-	if (ret) {
-		dev_err(dev, "failed to request dsi irq\n");
-		return ret;
-	}
-
-	ret = exynos_dsi_parse_dt(dsi);
-	if (ret)
-		return ret;
-
-	platform_set_drvdata(pdev, dsi);
-
-	pm_runtime_enable(dev);
-
-	dsi->bridge.funcs = &exynos_dsi_bridge_funcs;
-	dsi->bridge.of_node = dev->of_node;
-	dsi->bridge.type = DRM_MODE_CONNECTOR_DSI;
-	dsi->bridge.pre_enable_prev_first = true;
-
-	ret = component_add(dev, &exynos_dsi_component_ops);
-	if (ret)
-		goto err_disable_runtime;
-
-	return 0;
-
-err_disable_runtime:
-	pm_runtime_disable(dev);
-
-	return ret;
+	return component_add(dsim->dev, &exynos_dsi_component_ops);
 }
 
-static int exynos_dsi_remove(struct platform_device *pdev)
+static void exynos_dsi_unregister_host(struct samsung_dsim *dsim)
 {
-	pm_runtime_disable(&pdev->dev);
-
-	component_del(&pdev->dev, &exynos_dsi_component_ops);
-
-	return 0;
-}
-
-static int __maybe_unused exynos_dsi_suspend(struct device *dev)
-{
-	struct exynos_dsi *dsi = dev_get_drvdata(dev);
-	const struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
-	int ret, i;
-
-	usleep_range(10000, 20000);
-
-	if (dsi->state & DSIM_STATE_INITIALIZED) {
-		dsi->state &= ~DSIM_STATE_INITIALIZED;
-
-		exynos_dsi_disable_clock(dsi);
-
-		exynos_dsi_disable_irq(dsi);
-	}
-
-	dsi->state &= ~DSIM_STATE_CMD_LPM;
-
-	phy_power_off(dsi->phy);
-
-	for (i = driver_data->num_clks - 1; i > -1; i--)
-		clk_disable_unprepare(dsi->clks[i]);
-
-	ret = regulator_bulk_disable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
-	if (ret < 0)
-		dev_err(dsi->dev, "cannot disable regulators %d\n", ret);
-
-	return 0;
+	component_del(dsim->dev, &exynos_dsi_component_ops);
 }
 
-static int __maybe_unused exynos_dsi_resume(struct device *dev)
-{
-	struct exynos_dsi *dsi = dev_get_drvdata(dev);
-	const struct exynos_dsi_driver_data *driver_data = dsi->driver_data;
-	int ret, i;
-
-	ret = regulator_bulk_enable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
-	if (ret < 0) {
-		dev_err(dsi->dev, "cannot enable regulators %d\n", ret);
-		return ret;
-	}
+static const struct samsung_dsim_host_ops exynos_dsi_exynos_host_ops = {
+	.register_host = exynos_dsi_register_host,
+	.unregister_host = exynos_dsi_unregister_host,
+	.attach = exynos_dsi_host_attach,
+	.detach = exynos_dsi_host_detach,
+	.te_irq_handler = exynos_dsi_te_irq_handler,
+};
 
-	for (i = 0; i < driver_data->num_clks; i++) {
-		ret = clk_prepare_enable(dsi->clks[i]);
-		if (ret < 0)
-			goto err_clk;
-	}
+static const struct samsung_dsim_plat_data exynos3250_dsi_pdata = {
+	.hw_type = DSIM_TYPE_EXYNOS3250,
+	.host_ops = &exynos_dsi_exynos_host_ops,
+};
 
-	ret = phy_power_on(dsi->phy);
-	if (ret < 0) {
-		dev_err(dsi->dev, "cannot enable phy %d\n", ret);
-		goto err_clk;
-	}
+static const struct samsung_dsim_plat_data exynos4210_dsi_pdata = {
+	.hw_type = DSIM_TYPE_EXYNOS4210,
+	.host_ops = &exynos_dsi_exynos_host_ops,
+};
 
-	return 0;
+static const struct samsung_dsim_plat_data exynos5410_dsi_pdata = {
+	.hw_type = DSIM_TYPE_EXYNOS5410,
+	.host_ops = &exynos_dsi_exynos_host_ops,
+};
 
-err_clk:
-	while (--i > -1)
-		clk_disable_unprepare(dsi->clks[i]);
-	regulator_bulk_disable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
+static const struct samsung_dsim_plat_data exynos5422_dsi_pdata = {
+	.hw_type = DSIM_TYPE_EXYNOS5422,
+	.host_ops = &exynos_dsi_exynos_host_ops,
+};
 
-	return ret;
-}
+static const struct samsung_dsim_plat_data exynos5433_dsi_pdata = {
+	.hw_type = DSIM_TYPE_EXYNOS5433,
+	.host_ops = &exynos_dsi_exynos_host_ops,
+};
 
-static const struct dev_pm_ops exynos_dsi_pm_ops = {
-	SET_RUNTIME_PM_OPS(exynos_dsi_suspend, exynos_dsi_resume, NULL)
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
-				pm_runtime_force_resume)
+static const struct of_device_id exynos_dsi_of_match[] = {
+	{
+		.compatible = "samsung,exynos3250-mipi-dsi",
+		.data = &exynos3250_dsi_pdata,
+	},
+	{
+		.compatible = "samsung,exynos4210-mipi-dsi",
+		.data = &exynos4210_dsi_pdata,
+	},
+	{
+		.compatible = "samsung,exynos5410-mipi-dsi",
+		.data = &exynos5410_dsi_pdata,
+	},
+	{
+		.compatible = "samsung,exynos5422-mipi-dsi",
+		.data = &exynos5422_dsi_pdata,
+	},
+	{
+		.compatible = "samsung,exynos5433-mipi-dsi",
+		.data = &exynos5433_dsi_pdata,
+	},
+	{ /* sentinel. */ }
 };
+MODULE_DEVICE_TABLE(of, exynos_dsi_of_match);
 
 struct platform_driver dsi_driver = {
-	.probe = exynos_dsi_probe,
-	.remove = exynos_dsi_remove,
+	.probe = samsung_dsim_probe,
+	.remove = samsung_dsim_remove,
 	.driver = {
 		   .name = "exynos-dsi",
 		   .owner = THIS_MODULE,
-		   .pm = &exynos_dsi_pm_ops,
+		   .pm = &samsung_dsim_pm_ops,
 		   .of_match_table = exynos_dsi_of_match,
 	},
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index 97f2dee2db29..fc1c5608db96 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -11,7 +11,6 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
-#include <drm/drm_fb_helper.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -157,7 +156,6 @@ static struct drm_mode_config_helper_funcs exynos_drm_mode_config_helpers = {
 
 static const struct drm_mode_config_funcs exynos_drm_mode_config_funcs = {
 	.fb_create = exynos_user_fb_create,
-	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
index 4929ffe5a09a..ea4b3d248aac 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
@@ -8,16 +8,12 @@
  *	Seung-Woo Kim <sw0312.kim@samsung.com>
  */
 
-#include <linux/console.h>
-#include <linux/dma-mapping.h>
-#include <linux/vmalloc.h>
-
-#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_prime.h>
-#include <drm/drm_probe_helper.h>
 #include <drm/exynos_drm.h>
 
 #include "exynos_drm_drv.h"
@@ -27,22 +23,26 @@
 #define MAX_CONNECTOR		4
 #define PREFERRED_BPP		32
 
-#define to_exynos_fbdev(x)	container_of(x, struct exynos_drm_fbdev,\
-				drm_fb_helper)
+static int exynos_drm_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
+{
+	struct drm_fb_helper *helper = info->par;
+	struct drm_gem_object *obj = drm_gem_fb_get_obj(helper->fb, 0);
 
-struct exynos_drm_fbdev {
-	struct drm_fb_helper	drm_fb_helper;
-	struct exynos_drm_gem	*exynos_gem;
-};
+	return drm_gem_prime_mmap(obj, vma);
+}
 
-static int exynos_drm_fb_mmap(struct fb_info *info,
-			struct vm_area_struct *vma)
+static void exynos_drm_fb_destroy(struct fb_info *info)
 {
-	struct drm_fb_helper *helper = info->par;
-	struct exynos_drm_fbdev *exynos_fbd = to_exynos_fbdev(helper);
-	struct exynos_drm_gem *exynos_gem = exynos_fbd->exynos_gem;
+	struct drm_fb_helper *fb_helper = info->par;
+	struct drm_framebuffer *fb = fb_helper->fb;
+
+	drm_fb_helper_fini(fb_helper);
 
-	return drm_gem_prime_mmap(&exynos_gem->base, vma);
+	drm_framebuffer_remove(fb);
+
+	drm_client_release(&fb_helper->client);
+	drm_fb_helper_unprepare(fb_helper);
+	kfree(fb_helper);
 }
 
 static const struct fb_ops exynos_drm_fb_ops = {
@@ -54,6 +54,7 @@ static const struct fb_ops exynos_drm_fb_ops = {
 	.fb_fillrect	= drm_fb_helper_cfb_fillrect,
 	.fb_copyarea	= drm_fb_helper_cfb_copyarea,
 	.fb_imageblit	= drm_fb_helper_cfb_imageblit,
+	.fb_destroy	= exynos_drm_fb_destroy,
 };
 
 static int exynos_drm_fbdev_update(struct drm_fb_helper *helper,
@@ -89,7 +90,6 @@ static int exynos_drm_fbdev_update(struct drm_fb_helper *helper,
 static int exynos_drm_fbdev_create(struct drm_fb_helper *helper,
 				    struct drm_fb_helper_surface_size *sizes)
 {
-	struct exynos_drm_fbdev *exynos_fbdev = to_exynos_fbdev(helper);
 	struct exynos_drm_gem *exynos_gem;
 	struct drm_device *dev = helper->dev;
 	struct drm_mode_fb_cmd2 mode_cmd = { 0 };
@@ -113,8 +113,6 @@ static int exynos_drm_fbdev_create(struct drm_fb_helper *helper,
 	if (IS_ERR(exynos_gem))
 		return PTR_ERR(exynos_gem);
 
-	exynos_fbdev->exynos_gem = exynos_gem;
-
 	helper->fb =
 		exynos_drm_framebuffer_init(dev, &mode_cmd, &exynos_gem, 1);
 	if (IS_ERR(helper->fb)) {
@@ -127,19 +125,13 @@ static int exynos_drm_fbdev_create(struct drm_fb_helper *helper,
 	if (ret < 0)
 		goto err_destroy_framebuffer;
 
-	return ret;
+	return 0;
 
 err_destroy_framebuffer:
 	drm_framebuffer_cleanup(helper->fb);
+	helper->fb = NULL;
 err_destroy_gem:
 	exynos_drm_gem_destroy(exynos_gem);
-
-	/*
-	 * if failed, all resources allocated above would be released by
-	 * drm_mode_config_cleanup() when drm_load() had been called prior
-	 * to any specific driver such as fimd or hdmi driver.
-	 */
-
 	return ret;
 }
 
@@ -147,80 +139,92 @@ static const struct drm_fb_helper_funcs exynos_drm_fb_helper_funcs = {
 	.fb_probe =	exynos_drm_fbdev_create,
 };
 
-int exynos_drm_fbdev_init(struct drm_device *dev)
+/*
+ * struct drm_client
+ */
+
+static void exynos_drm_fbdev_client_unregister(struct drm_client_dev *client)
 {
-	struct exynos_drm_fbdev *fbdev;
-	struct exynos_drm_private *private = dev->dev_private;
-	struct drm_fb_helper *helper;
-	int ret;
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+
+	if (fb_helper->info) {
+		drm_fb_helper_unregister_info(fb_helper);
+	} else {
+		drm_client_release(&fb_helper->client);
+		drm_fb_helper_unprepare(fb_helper);
+		kfree(fb_helper);
+	}
+}
+
+static int exynos_drm_fbdev_client_restore(struct drm_client_dev *client)
+{
+	drm_fb_helper_lastclose(client->dev);
 
-	if (!dev->mode_config.num_crtc)
-		return 0;
+	return 0;
+}
 
-	fbdev = kzalloc(sizeof(*fbdev), GFP_KERNEL);
-	if (!fbdev)
-		return -ENOMEM;
+static int exynos_drm_fbdev_client_hotplug(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+	struct drm_device *dev = client->dev;
+	int ret;
 
-	private->fb_helper = helper = &fbdev->drm_fb_helper;
+	if (dev->fb_helper)
+		return drm_fb_helper_hotplug_event(dev->fb_helper);
 
-	drm_fb_helper_prepare(dev, helper, PREFERRED_BPP, &exynos_drm_fb_helper_funcs);
+	ret = drm_fb_helper_init(dev, fb_helper);
+	if (ret)
+		goto err_drm_err;
 
-	ret = drm_fb_helper_init(dev, helper);
-	if (ret < 0) {
-		DRM_DEV_ERROR(dev->dev,
-			      "failed to initialize drm fb helper.\n");
-		goto err_init;
-	}
+	if (!drm_drv_uses_atomic_modeset(dev))
+		drm_helper_disable_unused_functions(dev);
 
-	ret = drm_fb_helper_initial_config(helper);
-	if (ret < 0) {
-		DRM_DEV_ERROR(dev->dev,
-			      "failed to set up hw configuration.\n");
-		goto err_setup;
-	}
+	ret = drm_fb_helper_initial_config(fb_helper);
+	if (ret)
+		goto err_drm_fb_helper_fini;
 
 	return 0;
 
-err_setup:
-	drm_fb_helper_fini(helper);
-err_init:
-	drm_fb_helper_unprepare(helper);
-	private->fb_helper = NULL;
-	kfree(fbdev);
-
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(fb_helper);
+err_drm_err:
+	drm_err(dev, "Failed to setup fbdev emulation (ret=%d)\n", ret);
 	return ret;
 }
 
-static void exynos_drm_fbdev_destroy(struct drm_device *dev,
-				      struct drm_fb_helper *fb_helper)
+static const struct drm_client_funcs exynos_drm_fbdev_client_funcs = {
+	.owner		= THIS_MODULE,
+	.unregister	= exynos_drm_fbdev_client_unregister,
+	.restore	= exynos_drm_fbdev_client_restore,
+	.hotplug	= exynos_drm_fbdev_client_hotplug,
+};
+
+void exynos_drm_fbdev_setup(struct drm_device *dev)
 {
-	struct drm_framebuffer *fb;
+	struct drm_fb_helper *fb_helper;
+	int ret;
 
-	/* release drm framebuffer and real buffer */
-	if (fb_helper->fb && fb_helper->fb->funcs) {
-		fb = fb_helper->fb;
-		if (fb)
-			drm_framebuffer_remove(fb);
-	}
+	drm_WARN(dev, !dev->registered, "Device has not been registered.\n");
+	drm_WARN(dev, dev->fb_helper, "fb_helper is already set!\n");
 
-	drm_fb_helper_unregister_info(fb_helper);
+	fb_helper = kzalloc(sizeof(*fb_helper), GFP_KERNEL);
+	if (!fb_helper)
+		return;
+	drm_fb_helper_prepare(dev, fb_helper, PREFERRED_BPP, &exynos_drm_fb_helper_funcs);
 
-	drm_fb_helper_fini(fb_helper);
-}
+	ret = drm_client_init(dev, &fb_helper->client, "fbdev", &exynos_drm_fbdev_client_funcs);
+	if (ret)
+		goto err_drm_client_init;
 
-void exynos_drm_fbdev_fini(struct drm_device *dev)
-{
-	struct exynos_drm_private *private = dev->dev_private;
-	struct exynos_drm_fbdev *fbdev;
+	ret = exynos_drm_fbdev_client_hotplug(&fb_helper->client);
+	if (ret)
+		drm_dbg_kms(dev, "client hotplug ret=%d\n", ret);
 
-	if (!private || !private->fb_helper)
-		return;
+	drm_client_register(&fb_helper->client);
 
-	fbdev = to_exynos_fbdev(private->fb_helper);
+	return;
 
-	exynos_drm_fbdev_destroy(dev, private->fb_helper);
-	drm_fb_helper_unprepare(private->fb_helper);
-	kfree(fbdev);
-	private->fb_helper = NULL;
+err_drm_client_init:
+	drm_fb_helper_unprepare(fb_helper);
+	kfree(fb_helper);
 }
-
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.h b/drivers/gpu/drm/exynos/exynos_drm_fbdev.h
index 3b1e98e84580..1e1dea627cd9 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.h
@@ -12,27 +12,11 @@
 #define _EXYNOS_DRM_FBDEV_H_
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
-
-int exynos_drm_fbdev_init(struct drm_device *dev);
-void exynos_drm_fbdev_fini(struct drm_device *dev);
-
+void exynos_drm_fbdev_setup(struct drm_device *dev);
 #else
-
-static inline int exynos_drm_fbdev_init(struct drm_device *dev)
-{
-	return 0;
-}
-
-static inline void exynos_drm_fbdev_fini(struct drm_device *dev)
+static inline void exynos_drm_fbdev_setup(struct drm_device *dev)
 {
 }
-
-static inline void exynos_drm_fbdev_restore_mode(struct drm_device *dev)
-{
-}
-
-#define exynos_drm_output_poll_changed (NULL)
-
 #endif
 
 #endif
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
index 8579c7629f5e..c09ba019ba5e 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
@@ -20,7 +20,7 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_modeset_helper.h>
 #include <drm/drm_module.h>
@@ -333,7 +333,7 @@ static int fsl_dcu_drm_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto put;
 
-	drm_fbdev_generic_setup(drm, legacyfb_depth);
+	drm_fbdev_dma_setup(drm, legacyfb_depth);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/gma500/Makefile b/drivers/gpu/drm/gma500/Makefile
index 63012bf2485a..4f302cd5e1a6 100644
--- a/drivers/gpu/drm/gma500/Makefile
+++ b/drivers/gpu/drm/gma500/Makefile
@@ -38,5 +38,6 @@ gma500_gfx-y += \
 	  psb_irq.o
 
 gma500_gfx-$(CONFIG_ACPI) +=  opregion.o
+gma500_gfx-$(CONFIG_DRM_FBDEV_EMULATION) +=  fbdev.o
 
 obj-$(CONFIG_DRM_GMA500) += gma500_gfx.o
diff --git a/drivers/gpu/drm/gma500/fbdev.c b/drivers/gpu/drm/gma500/fbdev.c
new file mode 100644
index 000000000000..62287407e717
--- /dev/null
+++ b/drivers/gpu/drm/gma500/fbdev.c
@@ -0,0 +1,344 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/**************************************************************************
+ * Copyright (c) 2007-2011, Intel Corporation.
+ * All Rights Reserved.
+ *
+ **************************************************************************/
+
+#include <linux/pfn_t.h>
+
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_framebuffer.h>
+
+#include "gem.h"
+#include "psb_drv.h"
+
+/*
+ * VM area struct
+ */
+
+static vm_fault_t psb_fbdev_vm_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct fb_info *info = vma->vm_private_data;
+	unsigned long address = vmf->address - (vmf->pgoff << PAGE_SHIFT);
+	unsigned long pfn = info->fix.smem_start >> PAGE_SHIFT;
+	vm_fault_t err = VM_FAULT_SIGBUS;
+	unsigned long page_num = vma_pages(vma);
+	unsigned long i;
+
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+	for (i = 0; i < page_num; ++i) {
+		err = vmf_insert_mixed(vma, address, __pfn_to_pfn_t(pfn, PFN_DEV));
+		if (unlikely(err & VM_FAULT_ERROR))
+			break;
+		address += PAGE_SIZE;
+		++pfn;
+	}
+
+	return err;
+}
+
+static const struct vm_operations_struct psb_fbdev_vm_ops = {
+	.fault	= psb_fbdev_vm_fault,
+};
+
+/*
+ * struct fb_ops
+ */
+
+#define CMAP_TOHW(_val, _width) ((((_val) << (_width)) + 0x7FFF - (_val)) >> 16)
+
+static int psb_fbdev_fb_setcolreg(unsigned int regno,
+				  unsigned int red, unsigned int green,
+				  unsigned int blue, unsigned int transp,
+				  struct fb_info *info)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+	struct drm_framebuffer *fb = fb_helper->fb;
+	uint32_t v;
+
+	if (!fb)
+		return -ENOMEM;
+
+	if (regno > 255)
+		return 1;
+
+	red = CMAP_TOHW(red, info->var.red.length);
+	blue = CMAP_TOHW(blue, info->var.blue.length);
+	green = CMAP_TOHW(green, info->var.green.length);
+	transp = CMAP_TOHW(transp, info->var.transp.length);
+
+	v = (red << info->var.red.offset) |
+	    (green << info->var.green.offset) |
+	    (blue << info->var.blue.offset) |
+	    (transp << info->var.transp.offset);
+
+	if (regno < 16) {
+		switch (fb->format->cpp[0] * 8) {
+		case 16:
+			((uint32_t *) info->pseudo_palette)[regno] = v;
+			break;
+		case 24:
+		case 32:
+			((uint32_t *) info->pseudo_palette)[regno] = v;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int psb_fbdev_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
+{
+	if (vma->vm_pgoff != 0)
+		return -EINVAL;
+	if (vma->vm_pgoff > (~0UL >> PAGE_SHIFT))
+		return -EINVAL;
+
+	/*
+	 * If this is a GEM object then info->screen_base is the virtual
+	 * kernel remapping of the object. FIXME: Review if this is
+	 * suitable for our mmap work
+	 */
+	vma->vm_ops = &psb_fbdev_vm_ops;
+	vma->vm_private_data = info;
+	vm_flags_set(vma, VM_IO | VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP);
+
+	return 0;
+}
+
+static void psb_fbdev_fb_destroy(struct fb_info *info)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+	struct drm_framebuffer *fb = fb_helper->fb;
+	struct drm_gem_object *obj = fb->obj[0];
+
+	drm_fb_helper_fini(fb_helper);
+
+	drm_framebuffer_unregister_private(fb);
+	fb->obj[0] = NULL;
+	drm_framebuffer_cleanup(fb);
+	kfree(fb);
+
+	drm_gem_object_put(obj);
+
+	drm_client_release(&fb_helper->client);
+
+	drm_fb_helper_unprepare(fb_helper);
+	kfree(fb_helper);
+}
+
+static const struct fb_ops psb_fbdev_fb_ops = {
+	.owner = THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
+	.fb_setcolreg = psb_fbdev_fb_setcolreg,
+	.fb_read = drm_fb_helper_cfb_read,
+	.fb_write = drm_fb_helper_cfb_write,
+	.fb_fillrect = drm_fb_helper_cfb_fillrect,
+	.fb_copyarea = drm_fb_helper_cfb_copyarea,
+	.fb_imageblit = drm_fb_helper_cfb_imageblit,
+	.fb_mmap = psb_fbdev_fb_mmap,
+	.fb_destroy = psb_fbdev_fb_destroy,
+};
+
+/*
+ * struct drm_fb_helper_funcs
+ */
+
+static int psb_fbdev_fb_probe(struct drm_fb_helper *fb_helper,
+			      struct drm_fb_helper_surface_size *sizes)
+{
+	struct drm_device *dev = fb_helper->dev;
+	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+	struct fb_info *info;
+	struct drm_framebuffer *fb;
+	struct drm_mode_fb_cmd2 mode_cmd = { };
+	int size;
+	int ret;
+	struct psb_gem_object *backing;
+	struct drm_gem_object *obj;
+	u32 bpp, depth;
+
+	/* No 24-bit packed mode */
+	if (sizes->surface_bpp == 24) {
+		sizes->surface_bpp = 32;
+		sizes->surface_depth = 24;
+	}
+	bpp = sizes->surface_bpp;
+	depth = sizes->surface_depth;
+
+	/*
+	 * If the mode does not fit in 32 bit then switch to 16 bit to get
+	 * a console on full resolution. The X mode setting server will
+	 * allocate its own 32-bit GEM framebuffer.
+	 */
+	size = ALIGN(sizes->surface_width * DIV_ROUND_UP(bpp, 8), 64) *
+		     sizes->surface_height;
+	size = ALIGN(size, PAGE_SIZE);
+
+	if (size > dev_priv->vram_stolen_size) {
+		sizes->surface_bpp = 16;
+		sizes->surface_depth = 16;
+	}
+	bpp = sizes->surface_bpp;
+	depth = sizes->surface_depth;
+
+	mode_cmd.width = sizes->surface_width;
+	mode_cmd.height = sizes->surface_height;
+	mode_cmd.pitches[0] = ALIGN(mode_cmd.width * DIV_ROUND_UP(bpp, 8), 64);
+	mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
+
+	size = mode_cmd.pitches[0] * mode_cmd.height;
+	size = ALIGN(size, PAGE_SIZE);
+
+	/* Allocate the framebuffer in the GTT with stolen page backing */
+	backing = psb_gem_create(dev, size, "fb", true, PAGE_SIZE);
+	if (IS_ERR(backing))
+		return PTR_ERR(backing);
+	obj = &backing->base;
+
+	fb = psb_framebuffer_create(dev, &mode_cmd, obj);
+	if (IS_ERR(fb)) {
+		ret = PTR_ERR(fb);
+		goto err_drm_gem_object_put;
+	}
+
+	fb_helper->fb = fb;
+
+	info = drm_fb_helper_alloc_info(fb_helper);
+	if (IS_ERR(info)) {
+		ret = PTR_ERR(info);
+		goto err_drm_framebuffer_unregister_private;
+	}
+
+	info->fbops = &psb_fbdev_fb_ops;
+	info->flags = FBINFO_DEFAULT;
+	/* Accessed stolen memory directly */
+	info->screen_base = dev_priv->vram_addr + backing->offset;
+	info->screen_size = size;
+
+	drm_fb_helper_fill_info(info, fb_helper, sizes);
+
+	info->fix.smem_start = dev_priv->stolen_base + backing->offset;
+	info->fix.smem_len = size;
+	info->fix.ywrapstep = 0;
+	info->fix.ypanstep = 0;
+	info->fix.mmio_start = pci_resource_start(pdev, 0);
+	info->fix.mmio_len = pci_resource_len(pdev, 0);
+
+	memset(info->screen_base, 0, info->screen_size);
+
+	/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
+
+	dev_dbg(dev->dev, "allocated %dx%d fb\n", fb->width, fb->height);
+
+	return 0;
+
+err_drm_framebuffer_unregister_private:
+	drm_framebuffer_unregister_private(fb);
+	fb->obj[0] = NULL;
+	drm_framebuffer_cleanup(fb);
+	kfree(fb);
+err_drm_gem_object_put:
+	drm_gem_object_put(obj);
+	return ret;
+}
+
+static const struct drm_fb_helper_funcs psb_fbdev_fb_helper_funcs = {
+	.fb_probe = psb_fbdev_fb_probe,
+};
+
+/*
+ * struct drm_client_funcs and setup code
+ */
+
+static void psb_fbdev_client_unregister(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+
+	if (fb_helper->info) {
+		drm_fb_helper_unregister_info(fb_helper);
+	} else {
+		drm_fb_helper_unprepare(fb_helper);
+		drm_client_release(&fb_helper->client);
+		kfree(fb_helper);
+	}
+}
+
+static int psb_fbdev_client_restore(struct drm_client_dev *client)
+{
+	drm_fb_helper_lastclose(client->dev);
+
+	return 0;
+}
+
+static int psb_fbdev_client_hotplug(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+	struct drm_device *dev = client->dev;
+	int ret;
+
+	if (dev->fb_helper)
+		return drm_fb_helper_hotplug_event(dev->fb_helper);
+
+	ret = drm_fb_helper_init(dev, fb_helper);
+	if (ret)
+		goto err_drm_err;
+
+	if (!drm_drv_uses_atomic_modeset(dev))
+		drm_helper_disable_unused_functions(dev);
+
+	ret = drm_fb_helper_initial_config(fb_helper);
+	if (ret)
+		goto err_drm_fb_helper_fini;
+
+	return 0;
+
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(fb_helper);
+err_drm_err:
+	drm_err(dev, "Failed to setup gma500 fbdev emulation (ret=%d)\n", ret);
+	return ret;
+}
+
+static const struct drm_client_funcs psb_fbdev_client_funcs = {
+	.owner		= THIS_MODULE,
+	.unregister	= psb_fbdev_client_unregister,
+	.restore	= psb_fbdev_client_restore,
+	.hotplug	= psb_fbdev_client_hotplug,
+};
+
+void psb_fbdev_setup(struct drm_psb_private *dev_priv)
+{
+	struct drm_device *dev = &dev_priv->dev;
+	struct drm_fb_helper *fb_helper;
+	int ret;
+
+	fb_helper = kzalloc(sizeof(*fb_helper), GFP_KERNEL);
+	if (!fb_helper)
+		return;
+	drm_fb_helper_prepare(dev, fb_helper, 32, &psb_fbdev_fb_helper_funcs);
+
+	ret = drm_client_init(dev, &fb_helper->client, "fbdev-gma500", &psb_fbdev_client_funcs);
+	if (ret) {
+		drm_err(dev, "Failed to register client: %d\n", ret);
+		goto err_drm_fb_helper_unprepare;
+	}
+
+	ret = psb_fbdev_client_hotplug(&fb_helper->client);
+	if (ret)
+		drm_dbg_kms(dev, "client hotplug ret=%d\n", ret);
+
+	drm_client_register(&fb_helper->client);
+
+	return;
+
+err_drm_fb_helper_unprepare:
+	drm_fb_helper_unprepare(fb_helper);
+	kfree(fb_helper);
+}
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 50611eb7f134..1a374702b696 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -5,158 +5,18 @@
  *
  **************************************************************************/
 
-#include <linux/console.h>
-#include <linux/delay.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/module.h>
-#include <linux/pfn_t.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-#include <linux/tty.h>
-
-#include <drm/drm.h>
-#include <drm/drm_crtc.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_fb_helper.h>
-#include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper.h>
 
 #include "framebuffer.h"
-#include "gem.h"
 #include "psb_drv.h"
-#include "psb_intel_drv.h"
-#include "psb_intel_reg.h"
 
 static const struct drm_framebuffer_funcs psb_fb_funcs = {
 	.destroy = drm_gem_fb_destroy,
 	.create_handle = drm_gem_fb_create_handle,
 };
 
-#define CMAP_TOHW(_val, _width) ((((_val) << (_width)) + 0x7FFF - (_val)) >> 16)
-
-static int psbfb_setcolreg(unsigned regno, unsigned red, unsigned green,
-			   unsigned blue, unsigned transp,
-			   struct fb_info *info)
-{
-	struct drm_fb_helper *fb_helper = info->par;
-	struct drm_framebuffer *fb = fb_helper->fb;
-	uint32_t v;
-
-	if (!fb)
-		return -ENOMEM;
-
-	if (regno > 255)
-		return 1;
-
-	red = CMAP_TOHW(red, info->var.red.length);
-	blue = CMAP_TOHW(blue, info->var.blue.length);
-	green = CMAP_TOHW(green, info->var.green.length);
-	transp = CMAP_TOHW(transp, info->var.transp.length);
-
-	v = (red << info->var.red.offset) |
-	    (green << info->var.green.offset) |
-	    (blue << info->var.blue.offset) |
-	    (transp << info->var.transp.offset);
-
-	if (regno < 16) {
-		switch (fb->format->cpp[0] * 8) {
-		case 16:
-			((uint32_t *) info->pseudo_palette)[regno] = v;
-			break;
-		case 24:
-		case 32:
-			((uint32_t *) info->pseudo_palette)[regno] = v;
-			break;
-		}
-	}
-
-	return 0;
-}
-
-static vm_fault_t psbfb_vm_fault(struct vm_fault *vmf)
-{
-	struct vm_area_struct *vma = vmf->vma;
-	struct drm_framebuffer *fb = vma->vm_private_data;
-	struct drm_device *dev = fb->dev;
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	struct psb_gem_object *pobj = to_psb_gem_object(fb->obj[0]);
-	int page_num;
-	int i;
-	unsigned long address;
-	vm_fault_t ret = VM_FAULT_SIGBUS;
-	unsigned long pfn;
-	unsigned long phys_addr = (unsigned long)dev_priv->stolen_base + pobj->offset;
-
-	page_num = vma_pages(vma);
-	address = vmf->address - (vmf->pgoff << PAGE_SHIFT);
-
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-
-	for (i = 0; i < page_num; i++) {
-		pfn = (phys_addr >> PAGE_SHIFT);
-
-		ret = vmf_insert_mixed(vma, address,
-				__pfn_to_pfn_t(pfn, PFN_DEV));
-		if (unlikely(ret & VM_FAULT_ERROR))
-			break;
-		address += PAGE_SIZE;
-		phys_addr += PAGE_SIZE;
-	}
-	return ret;
-}
-
-static void psbfb_vm_open(struct vm_area_struct *vma)
-{
-}
-
-static void psbfb_vm_close(struct vm_area_struct *vma)
-{
-}
-
-static const struct vm_operations_struct psbfb_vm_ops = {
-	.fault	= psbfb_vm_fault,
-	.open	= psbfb_vm_open,
-	.close	= psbfb_vm_close
-};
-
-static int psbfb_mmap(struct fb_info *info, struct vm_area_struct *vma)
-{
-	struct drm_fb_helper *fb_helper = info->par;
-	struct drm_framebuffer *fb = fb_helper->fb;
-
-	if (vma->vm_pgoff != 0)
-		return -EINVAL;
-	if (vma->vm_pgoff > (~0UL >> PAGE_SHIFT))
-		return -EINVAL;
-
-	/*
-	 * If this is a GEM object then info->screen_base is the virtual
-	 * kernel remapping of the object. FIXME: Review if this is
-	 * suitable for our mmap work
-	 */
-	vma->vm_ops = &psbfb_vm_ops;
-	vma->vm_private_data = (void *)fb;
-	vm_flags_set(vma, VM_IO | VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP);
-	return 0;
-}
-
-static const struct fb_ops psbfb_unaccel_ops = {
-	.owner = THIS_MODULE,
-	DRM_FB_HELPER_DEFAULT_OPS,
-	.fb_setcolreg = psbfb_setcolreg,
-	.fb_read = drm_fb_helper_cfb_read,
-	.fb_write = drm_fb_helper_cfb_write,
-	.fb_fillrect = drm_fb_helper_cfb_fillrect,
-	.fb_copyarea = drm_fb_helper_cfb_copyarea,
-	.fb_imageblit = drm_fb_helper_cfb_imageblit,
-	.fb_mmap = psbfb_mmap,
-};
-
 /**
  *	psb_framebuffer_init	-	initialize a framebuffer
  *	@dev: our DRM device
@@ -207,11 +67,9 @@ static int psb_framebuffer_init(struct drm_device *dev,
  *
  *	TODO: review object references
  */
-
-static struct drm_framebuffer *psb_framebuffer_create
-			(struct drm_device *dev,
-			 const struct drm_mode_fb_cmd2 *mode_cmd,
-			 struct drm_gem_object *obj)
+struct drm_framebuffer *psb_framebuffer_create(struct drm_device *dev,
+					       const struct drm_mode_fb_cmd2 *mode_cmd,
+					       struct drm_gem_object *obj)
 {
 	struct drm_framebuffer *fb;
 	int ret;
@@ -229,93 +87,6 @@ static struct drm_framebuffer *psb_framebuffer_create
 }
 
 /**
- *	psbfb_create		-	create a framebuffer
- *	@fb_helper: the framebuffer helper
- *	@sizes: specification of the layout
- *
- *	Create a framebuffer to the specifications provided
- */
-static int psbfb_create(struct drm_fb_helper *fb_helper,
-				struct drm_fb_helper_surface_size *sizes)
-{
-	struct drm_device *dev = fb_helper->dev;
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	struct pci_dev *pdev = to_pci_dev(dev->dev);
-	struct fb_info *info;
-	struct drm_framebuffer *fb;
-	struct drm_mode_fb_cmd2 mode_cmd;
-	int size;
-	int ret;
-	struct psb_gem_object *backing;
-	struct drm_gem_object *obj;
-	u32 bpp, depth;
-
-	mode_cmd.width = sizes->surface_width;
-	mode_cmd.height = sizes->surface_height;
-	bpp = sizes->surface_bpp;
-	depth = sizes->surface_depth;
-
-	/* No 24bit packed */
-	if (bpp == 24)
-		bpp = 32;
-
-	mode_cmd.pitches[0] = ALIGN(mode_cmd.width * DIV_ROUND_UP(bpp, 8), 64);
-
-	size = mode_cmd.pitches[0] * mode_cmd.height;
-	size = ALIGN(size, PAGE_SIZE);
-
-	/* Allocate the framebuffer in the GTT with stolen page backing */
-	backing = psb_gem_create(dev, size, "fb", true, PAGE_SIZE);
-	if (IS_ERR(backing))
-		return PTR_ERR(backing);
-	obj = &backing->base;
-
-	memset(dev_priv->vram_addr + backing->offset, 0, size);
-
-	info = drm_fb_helper_alloc_info(fb_helper);
-	if (IS_ERR(info)) {
-		ret = PTR_ERR(info);
-		goto err_drm_gem_object_put;
-	}
-
-	mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
-
-	fb = psb_framebuffer_create(dev, &mode_cmd, obj);
-	if (IS_ERR(fb)) {
-		ret = PTR_ERR(fb);
-		goto err_drm_gem_object_put;
-	}
-
-	fb_helper->fb = fb;
-
-	info->fbops = &psbfb_unaccel_ops;
-
-	info->fix.smem_start = dev_priv->fb_base;
-	info->fix.smem_len = size;
-	info->fix.ywrapstep = 0;
-	info->fix.ypanstep = 0;
-
-	/* Accessed stolen memory directly */
-	info->screen_base = dev_priv->vram_addr + backing->offset;
-	info->screen_size = size;
-
-	drm_fb_helper_fill_info(info, fb_helper, sizes);
-
-	info->fix.mmio_start = pci_resource_start(pdev, 0);
-	info->fix.mmio_len = pci_resource_len(pdev, 0);
-
-	/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
-
-	dev_dbg(dev->dev, "allocated %dx%d fb\n", fb->width, fb->height);
-
-	return 0;
-
-err_drm_gem_object_put:
-	drm_gem_object_put(obj);
-	return ret;
-}
-
-/**
  *	psb_user_framebuffer_create	-	create framebuffer
  *	@dev: our DRM device
  *	@filp: client file
@@ -346,108 +117,8 @@ static struct drm_framebuffer *psb_user_framebuffer_create
 	return fb;
 }
 
-static int psbfb_probe(struct drm_fb_helper *fb_helper,
-				struct drm_fb_helper_surface_size *sizes)
-{
-	struct drm_device *dev = fb_helper->dev;
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	unsigned int fb_size;
-	int bytespp;
-
-	bytespp = sizes->surface_bpp / 8;
-	if (bytespp == 3)	/* no 24bit packed */
-		bytespp = 4;
-
-	/* If the mode will not fit in 32bit then switch to 16bit to get
-	   a console on full resolution. The X mode setting server will
-	   allocate its own 32bit GEM framebuffer */
-	fb_size = ALIGN(sizes->surface_width * bytespp, 64) *
-		  sizes->surface_height;
-	fb_size = ALIGN(fb_size, PAGE_SIZE);
-
-	if (fb_size > dev_priv->vram_stolen_size) {
-                sizes->surface_bpp = 16;
-                sizes->surface_depth = 16;
-        }
-
-	return psbfb_create(fb_helper, sizes);
-}
-
-static const struct drm_fb_helper_funcs psb_fb_helper_funcs = {
-	.fb_probe = psbfb_probe,
-};
-
-static int psb_fbdev_destroy(struct drm_device *dev,
-			     struct drm_fb_helper *fb_helper)
-{
-	struct drm_framebuffer *fb = fb_helper->fb;
-
-	drm_fb_helper_unregister_info(fb_helper);
-
-	drm_fb_helper_fini(fb_helper);
-	drm_framebuffer_unregister_private(fb);
-	drm_framebuffer_cleanup(fb);
-
-	if (fb->obj[0])
-		drm_gem_object_put(fb->obj[0]);
-	kfree(fb);
-
-	return 0;
-}
-
-int psb_fbdev_init(struct drm_device *dev)
-{
-	struct drm_fb_helper *fb_helper;
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	int ret;
-
-	fb_helper = kzalloc(sizeof(*fb_helper), GFP_KERNEL);
-	if (!fb_helper) {
-		dev_err(dev->dev, "no memory\n");
-		return -ENOMEM;
-	}
-
-	dev_priv->fb_helper = fb_helper;
-
-	drm_fb_helper_prepare(dev, fb_helper, 32, &psb_fb_helper_funcs);
-
-	ret = drm_fb_helper_init(dev, fb_helper);
-	if (ret)
-		goto free;
-
-	/* disable all the possible outputs/crtcs before entering KMS mode */
-	drm_helper_disable_unused_functions(dev);
-
-	ret = drm_fb_helper_initial_config(fb_helper);
-	if (ret)
-		goto fini;
-
-	return 0;
-
-fini:
-	drm_fb_helper_fini(fb_helper);
-free:
-	drm_fb_helper_unprepare(fb_helper);
-	kfree(fb_helper);
-	return ret;
-}
-
-static void psb_fbdev_fini(struct drm_device *dev)
-{
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-
-	if (!dev_priv->fb_helper)
-		return;
-
-	psb_fbdev_destroy(dev, dev_priv->fb_helper);
-	drm_fb_helper_unprepare(dev_priv->fb_helper);
-	kfree(dev_priv->fb_helper);
-	dev_priv->fb_helper = NULL;
-}
-
 static const struct drm_mode_config_funcs psb_mode_funcs = {
 	.fb_create = psb_user_framebuffer_create,
-	.output_poll_changed = drm_fb_helper_output_poll_changed,
 };
 
 static void psb_setup_outputs(struct drm_device *dev)
@@ -515,7 +186,6 @@ void psb_modeset_init(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
 	struct psb_intel_mode_device *mode_dev = &dev_priv->mode_dev;
-	struct pci_dev *pdev = to_pci_dev(dev->dev);
 	int i;
 
 	if (drmm_mode_config_init(dev))
@@ -526,10 +196,6 @@ void psb_modeset_init(struct drm_device *dev)
 
 	dev->mode_config.funcs = &psb_mode_funcs;
 
-	/* set memory base */
-	/* Oaktrail and Poulsbo should use BAR 2*/
-	pci_read_config_dword(pdev, PSB_BSM, (u32 *)&(dev_priv->fb_base));
-
 	/* num pipes is 2 for PSB but 1 for Mrst */
 	for (i = 0; i < dev_priv->num_pipe; i++)
 		psb_intel_crtc_init(dev, i, mode_dev);
@@ -550,6 +216,5 @@ void psb_modeset_cleanup(struct drm_device *dev)
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
 	if (dev_priv->modeset) {
 		drm_kms_helper_poll_fini(dev);
-		psb_fbdev_fini(dev);
 	}
 }
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index cd9c73f5a64a..2ce96b1b9c74 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -21,7 +21,6 @@
 #include <drm/drm.h>
 #include <drm/drm_aperture.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_helper.h>
 #include <drm/drm_file.h>
 #include <drm/drm_ioctl.h>
 #include <drm/drm_pciids.h>
@@ -387,7 +386,6 @@ static int psb_driver_load(struct drm_device *dev, unsigned long flags)
 	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
 
 	psb_modeset_init(dev);
-	psb_fbdev_init(dev);
 	drm_kms_helper_poll_init(dev);
 
 	/* Only add backlight support if we have LVDS or MIPI output */
@@ -452,6 +450,8 @@ static int psb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (ret)
 		return ret;
 
+	psb_fbdev_setup(dev_priv);
+
 	return 0;
 }
 
@@ -477,7 +477,6 @@ static const struct file_operations psb_gem_fops = {
 
 static const struct drm_driver driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM,
-	.lastclose = drm_fb_helper_lastclose,
 
 	.num_ioctls = ARRAY_SIZE(psb_ioctls),
 
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index a5df6d2f2cab..f7f709df99b4 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -193,8 +193,6 @@
 #define KSEL_BYPASS_25 6
 #define KSEL_BYPASS_83_100 7
 
-struct drm_fb_helper;
-
 struct opregion_header;
 struct opregion_acpi;
 struct opregion_swsci;
@@ -522,9 +520,6 @@ struct drm_psb_private {
 	uint32_t blc_adj1;
 	uint32_t blc_adj2;
 
-	struct drm_fb_helper *fb_helper;
-	resource_size_t fb_base;
-
 	bool dsr_enable;
 	u32 dsr_fb_update;
 	bool dpi_panel_on[3];
@@ -610,7 +605,19 @@ extern void psb_lid_timer_takedown(struct drm_psb_private *dev_priv);
 /* modesetting */
 extern void psb_modeset_init(struct drm_device *dev);
 extern void psb_modeset_cleanup(struct drm_device *dev);
-extern int psb_fbdev_init(struct drm_device *dev);
+
+/* framebuffer */
+struct drm_framebuffer *psb_framebuffer_create(struct drm_device *dev,
+					       const struct drm_mode_fb_cmd2 *mode_cmd,
+					       struct drm_gem_object *obj);
+
+/* fbdev */
+#if defined(CONFIG_DRM_FBDEV_EMULATION)
+void psb_fbdev_setup(struct drm_psb_private *dev_priv);
+#else
+static inline void psb_fbdev_setup(struct drm_psb_private *dev_priv)
+{ }
+#endif
 
 /* backlight.c */
 int gma_backlight_init(struct drm_device *dev);
diff --git a/drivers/gpu/drm/gma500/psb_irq.c b/drivers/gpu/drm/gma500/psb_irq.c
index d421031462df..343c51250207 100644
--- a/drivers/gpu/drm/gma500/psb_irq.c
+++ b/drivers/gpu/drm/gma500/psb_irq.c
@@ -32,17 +32,6 @@ static inline u32 gma_pipestat(int pipe)
 	BUG();
 }
 
-static inline u32 gma_pipe_event(int pipe)
-{
-	if (pipe == 0)
-		return _PSB_PIPEA_EVENT_FLAG;
-	if (pipe == 1)
-		return _MDFLD_PIPEB_EVENT_FLAG;
-	if (pipe == 2)
-		return _MDFLD_PIPEC_EVENT_FLAG;
-	BUG();
-}
-
 static inline u32 gma_pipeconf(int pipe)
 {
 	if (pipe == 0)
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
index 98f4e44976e0..06a0ca157e89 100644
--- a/drivers/gpu/drm/i915/Kconfig
+++ b/drivers/gpu/drm/i915/Kconfig
@@ -164,11 +164,5 @@ menu "drm/i915 Profile Guided Optimisation"
 	source "drivers/gpu/drm/i915/Kconfig.profile"
 endmenu
 
-menu "drm/i915 Unstable Evolution"
-	visible if EXPERT && STAGING && BROKEN
-	depends on DRM_I915
-	source "drivers/gpu/drm/i915/Kconfig.unstable"
-endmenu
-
 config DRM_I915_GVT
 	bool
diff --git a/drivers/gpu/drm/i915/Kconfig.unstable b/drivers/gpu/drm/i915/Kconfig.unstable
deleted file mode 100644
index cf151a297ed7..000000000000
--- a/drivers/gpu/drm/i915/Kconfig.unstable
+++ /dev/null
@@ -1,21 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config DRM_I915_UNSTABLE
-	bool "Enable unstable API for early prototype development"
-	depends on EXPERT
-	depends on STAGING
-	depends on BROKEN # should never be enabled by distros!
-	# We use the dependency on !COMPILE_TEST to not be enabled in
-	# allmodconfig or allyesconfig configurations
-	depends on !COMPILE_TEST
-	default n
-	help
-	  Enable prototype uAPI under general discussion before they are
-	  finalized. Such prototypes may be withdrawn or substantially
-	  changed before release. They are only enabled here so that a wide
-	  number of interested parties (userspace driver developers) can
-	  verify that the uAPI meet their expectations. These uAPI should
-	  never be used in production.
-
-	  Recommended for driver developers _only_.
-
-	  If in the slightest bit of doubt, say "N".
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 918470a04591..97b0d4ae221a 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -47,10 +47,10 @@ i915-y += i915_driver.o \
 	  i915_switcheroo.o \
 	  i915_sysfs.o \
 	  i915_utils.o \
+	  intel_clock_gating.o \
 	  intel_device_info.o \
 	  intel_memory_region.o \
 	  intel_pcode.o \
-	  intel_pm.o \
 	  intel_region_ttm.o \
 	  intel_runtime_pm.o \
 	  intel_sbi.o \
@@ -195,6 +195,7 @@ i915-y += \
 i915-y += \
 	  gt/uc/intel_gsc_fw.o \
 	  gt/uc/intel_gsc_uc.o \
+	  gt/uc/intel_gsc_uc_heci_cmd_submit.o\
 	  gt/uc/intel_guc.o \
 	  gt/uc/intel_guc_ads.o \
 	  gt/uc/intel_guc_capture.o \
@@ -239,6 +240,7 @@ i915-y += \
 	display/intel_display_power.o \
 	display/intel_display_power_map.o \
 	display/intel_display_power_well.o \
+	display/intel_display_rps.o \
 	display/intel_dmc.o \
 	display/intel_dpio_phy.o \
 	display/intel_dpll.o \
@@ -254,6 +256,7 @@ i915-y += \
 	display/intel_frontbuffer.o \
 	display/intel_global_state.o \
 	display/intel_hdcp.o \
+	display/intel_hdcp_gsc.o \
 	display/intel_hotplug.o \
 	display/intel_hti.o \
 	display/intel_lpe_audio.o \
@@ -266,10 +269,13 @@ i915-y += \
 	display/intel_psr.o \
 	display/intel_quirks.o \
 	display/intel_sprite.o \
+	display/intel_sprite_uapi.o \
 	display/intel_tc.o \
 	display/intel_vblank.o \
 	display/intel_vga.o \
+	display/intel_wm.o \
 	display/i9xx_plane.o \
+	display/i9xx_wm.o \
 	display/skl_scaler.o \
 	display/skl_universal_plane.o \
 	display/skl_watermark.o
@@ -363,6 +369,13 @@ include $(src)/gvt/Makefile
 obj-$(CONFIG_DRM_I915) += i915.o
 obj-$(CONFIG_DRM_I915_GVT_KVMGT) += kvmgt.o
 
+# kernel-doc test
+#
+# Enable locally for CONFIG_DRM_I915_WERROR=y. See also scripts/Makefile.build
+ifdef CONFIG_DRM_I915_WERROR
+    cmd_checkdoc = $(srctree)/scripts/kernel-doc -none $<
+endif
+
 # header test
 
 # exclude some broken headers from the test coverage
@@ -374,7 +387,8 @@ always-$(CONFIG_DRM_I915_WERROR) += \
 		$(shell cd $(srctree)/$(src) && find * -name '*.h')))
 
 quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@)
-      cmd_hdrtest = $(CC) $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; touch $@
+      cmd_hdrtest = $(CC) $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; \
+		$(srctree)/scripts/kernel-doc -none $<; touch $@
 
 $(obj)/%.hdrtest: $(src)/%.h FORCE
 	$(call if_changed_dep,hdrtest)
diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c
index fa754038d669..920d570f7594 100644
--- a/drivers/gpu/drm/i915/display/g4x_dp.c
+++ b/drivers/gpu/drm/i915/display/g4x_dp.c
@@ -17,6 +17,7 @@
 #include "intel_display_power.h"
 #include "intel_display_types.h"
 #include "intel_dp.h"
+#include "intel_dp_aux.h"
 #include "intel_dp_link_training.h"
 #include "intel_dpio_phy.h"
 #include "intel_fifo_underrun.h"
@@ -136,16 +137,12 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
 
 		intel_dp->DP |= DP_PIPE_SEL_IVB(crtc->pipe);
 	} else if (HAS_PCH_CPT(dev_priv) && port != PORT_A) {
-		u32 trans_dp;
-
 		intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT;
 
-		trans_dp = intel_de_read(dev_priv, TRANS_DP_CTL(crtc->pipe));
-		if (drm_dp_enhanced_frame_cap(intel_dp->dpcd))
-			trans_dp |= TRANS_DP_ENH_FRAMING;
-		else
-			trans_dp &= ~TRANS_DP_ENH_FRAMING;
-		intel_de_write(dev_priv, TRANS_DP_CTL(crtc->pipe), trans_dp);
+		intel_de_rmw(dev_priv, TRANS_DP_CTL(crtc->pipe),
+			     TRANS_DP_ENH_FRAMING,
+			     drm_dp_enhanced_frame_cap(intel_dp->dpcd) ?
+			     TRANS_DP_ENH_FRAMING : 0);
 	} else {
 		if (IS_G4X(dev_priv) && pipe_config->limited_color_range)
 			intel_dp->DP |= DP_COLOR_RANGE_16_235;
@@ -1200,29 +1197,6 @@ static bool g4x_digital_port_connected(struct intel_encoder *encoder)
 	return intel_de_read(dev_priv, PORT_HOTPLUG_STAT) & bit;
 }
 
-static bool gm45_digital_port_connected(struct intel_encoder *encoder)
-{
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	u32 bit;
-
-	switch (encoder->hpd_pin) {
-	case HPD_PORT_B:
-		bit = PORTB_HOTPLUG_LIVE_STATUS_GM45;
-		break;
-	case HPD_PORT_C:
-		bit = PORTC_HOTPLUG_LIVE_STATUS_GM45;
-		break;
-	case HPD_PORT_D:
-		bit = PORTD_HOTPLUG_LIVE_STATUS_GM45;
-		break;
-	default:
-		MISSING_CASE(encoder->hpd_pin);
-		return false;
-	}
-
-	return intel_de_read(dev_priv, PORT_HOTPLUG_STAT) & bit;
-}
-
 static bool ilk_digital_port_connected(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
@@ -1279,11 +1253,19 @@ static const struct drm_encoder_funcs intel_dp_enc_funcs = {
 bool g4x_dp_init(struct drm_i915_private *dev_priv,
 		 i915_reg_t output_reg, enum port port)
 {
+	const struct intel_bios_encoder_data *devdata;
 	struct intel_digital_port *dig_port;
 	struct intel_encoder *intel_encoder;
 	struct drm_encoder *encoder;
 	struct intel_connector *intel_connector;
 
+	devdata = intel_bios_encoder_data_lookup(dev_priv, port);
+
+	/* FIXME bail? */
+	if (!devdata)
+		drm_dbg_kms(&dev_priv->drm, "No VBT child device for DP-%c\n",
+			    port_name(port));
+
 	dig_port = kzalloc(sizeof(*dig_port), GFP_KERNEL);
 	if (!dig_port)
 		return false;
@@ -1295,6 +1277,8 @@ bool g4x_dp_init(struct drm_i915_private *dev_priv,
 	intel_encoder = &dig_port->base;
 	encoder = &intel_encoder->base;
 
+	intel_encoder->devdata = devdata;
+
 	mutex_init(&dig_port->hdcp_mutex);
 
 	if (drm_encoder_init(&dev_priv->drm, &intel_encoder->base,
@@ -1377,10 +1361,7 @@ bool g4x_dp_init(struct drm_i915_private *dev_priv,
 	dig_port->hpd_pulse = intel_dp_hpd_pulse;
 
 	if (HAS_GMCH(dev_priv)) {
-		if (IS_GM45(dev_priv))
-			dig_port->connected = gm45_digital_port_connected;
-		else
-			dig_port->connected = g4x_digital_port_connected;
+		dig_port->connected = g4x_digital_port_connected;
 	} else {
 		if (port == PORT_A)
 			dig_port->connected = ilk_digital_port_connected;
@@ -1391,7 +1372,7 @@ bool g4x_dp_init(struct drm_i915_private *dev_priv,
 	if (port != PORT_A)
 		intel_infoframe_init(dig_port);
 
-	dig_port->aux_ch = intel_bios_port_aux_ch(dev_priv, port);
+	dig_port->aux_ch = intel_dp_aux_ch(intel_encoder);
 	if (!intel_dp_init_connector(dig_port, intel_connector))
 		goto err_init_connector;
 
diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.c b/drivers/gpu/drm/i915/display/g4x_hdmi.c
index 64c3b3990702..448ea26786e0 100644
--- a/drivers/gpu/drm/i915/display/g4x_hdmi.c
+++ b/drivers/gpu/drm/i915/display/g4x_hdmi.c
@@ -13,6 +13,7 @@
 #include "intel_de.h"
 #include "intel_display_power.h"
 #include "intel_display_types.h"
+#include "intel_dp_aux.h"
 #include "intel_dpio_phy.h"
 #include "intel_fifo_underrun.h"
 #include "intel_hdmi.h"
@@ -273,8 +274,8 @@ static void cpt_enable_hdmi(struct intel_atomic_state *state,
 	 */
 
 	if (pipe_config->pipe_bpp > 24) {
-		intel_de_write(dev_priv, TRANS_CHICKEN1(pipe),
-			       intel_de_read(dev_priv, TRANS_CHICKEN1(pipe)) | TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE);
+		intel_de_rmw(dev_priv, TRANS_CHICKEN1(pipe),
+			     0, TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE);
 
 		temp &= ~SDVO_COLOR_FORMAT_MASK;
 		temp |= SDVO_COLOR_FORMAT_8bpc;
@@ -290,8 +291,8 @@ static void cpt_enable_hdmi(struct intel_atomic_state *state,
 		intel_de_write(dev_priv, intel_hdmi->hdmi_reg, temp);
 		intel_de_posting_read(dev_priv, intel_hdmi->hdmi_reg);
 
-		intel_de_write(dev_priv, TRANS_CHICKEN1(pipe),
-			       intel_de_read(dev_priv, TRANS_CHICKEN1(pipe)) & ~TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE);
+		intel_de_rmw(dev_priv, TRANS_CHICKEN1(pipe),
+			     TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE, 0);
 	}
 
 	drm_WARN_ON(&dev_priv->drm, pipe_config->has_audio &&
@@ -548,10 +549,18 @@ intel_hdmi_hotplug(struct intel_encoder *encoder,
 void g4x_hdmi_init(struct drm_i915_private *dev_priv,
 		   i915_reg_t hdmi_reg, enum port port)
 {
+	const struct intel_bios_encoder_data *devdata;
 	struct intel_digital_port *dig_port;
 	struct intel_encoder *intel_encoder;
 	struct intel_connector *intel_connector;
 
+	devdata = intel_bios_encoder_data_lookup(dev_priv, port);
+
+	/* FIXME bail? */
+	if (!devdata)
+		drm_dbg_kms(&dev_priv->drm, "No VBT child device for HDMI-%c\n",
+			    port_name(port));
+
 	dig_port = kzalloc(sizeof(*dig_port), GFP_KERNEL);
 	if (!dig_port)
 		return;
@@ -564,6 +573,8 @@ void g4x_hdmi_init(struct drm_i915_private *dev_priv,
 
 	intel_encoder = &dig_port->base;
 
+	intel_encoder->devdata = devdata;
+
 	mutex_init(&dig_port->hdcp_mutex);
 
 	drm_encoder_init(&dev_priv->drm, &intel_encoder->base,
@@ -629,6 +640,6 @@ void g4x_hdmi_init(struct drm_i915_private *dev_priv,
 
 	intel_infoframe_init(dig_port);
 
-	dig_port->aux_ch = intel_bios_port_aux_ch(dev_priv, port);
+	dig_port->aux_ch = intel_dp_aux_ch(intel_encoder);
 	intel_hdmi_init_connector(dig_port, intel_connector);
 }
diff --git a/drivers/gpu/drm/i915/display/hsw_ips.c b/drivers/gpu/drm/i915/display/hsw_ips.c
index 83aa3800245f..8eca0de065b6 100644
--- a/drivers/gpu/drm/i915/display/hsw_ips.c
+++ b/drivers/gpu/drm/i915/display/hsw_ips.c
@@ -14,6 +14,7 @@ static void hsw_ips_enable(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	u32 val;
 
 	if (!crtc_state->ips_enabled)
 		return;
@@ -26,10 +27,15 @@ static void hsw_ips_enable(const struct intel_crtc_state *crtc_state)
 	drm_WARN_ON(&i915->drm,
 		    !(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
 
+	val = IPS_ENABLE;
+
+	if (i915->display.ips.false_color)
+		val |= IPS_FALSE_COLOR;
+
 	if (IS_BROADWELL(i915)) {
 		drm_WARN_ON(&i915->drm,
 			    snb_pcode_write(&i915->uncore, DISPLAY_IPS_CONTROL,
-					    IPS_ENABLE | IPS_PCODE_CONTROL));
+					    val | IPS_PCODE_CONTROL));
 		/*
 		 * Quoting Art Runyan: "its not safe to expect any particular
 		 * value in IPS_CTL bit 31 after enabling IPS through the
@@ -37,7 +43,7 @@ static void hsw_ips_enable(const struct intel_crtc_state *crtc_state)
 		 * so we need to just enable it and continue on.
 		 */
 	} else {
-		intel_de_write(i915, IPS_CTL, IPS_ENABLE);
+		intel_de_write(i915, IPS_CTL, val);
 		/*
 		 * The bit only becomes 1 in the next vblank, so this wait here
 		 * is essentially intel_wait_for_vblank. If we don't have this
@@ -267,3 +273,87 @@ void hsw_ips_get_config(struct intel_crtc_state *crtc_state)
 		crtc_state->ips_enabled = true;
 	}
 }
+
+static int hsw_ips_debugfs_false_color_get(void *data, u64 *val)
+{
+	struct intel_crtc *crtc = data;
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+
+	*val = i915->display.ips.false_color;
+
+	return 0;
+}
+
+static int hsw_ips_debugfs_false_color_set(void *data, u64 val)
+{
+	struct intel_crtc *crtc = data;
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct intel_crtc_state *crtc_state;
+	int ret;
+
+	ret = drm_modeset_lock(&crtc->base.mutex, NULL);
+	if (ret)
+		return ret;
+
+	i915->display.ips.false_color = val;
+
+	crtc_state = to_intel_crtc_state(crtc->base.state);
+
+	if (!crtc_state->hw.active)
+		goto unlock;
+
+	if (crtc_state->uapi.commit &&
+	    !try_wait_for_completion(&crtc_state->uapi.commit->hw_done))
+		goto unlock;
+
+	hsw_ips_enable(crtc_state);
+
+ unlock:
+	drm_modeset_unlock(&crtc->base.mutex);
+
+	return ret;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(hsw_ips_debugfs_false_color_fops,
+			 hsw_ips_debugfs_false_color_get,
+			 hsw_ips_debugfs_false_color_set,
+			 "%llu\n");
+
+static int hsw_ips_debugfs_status_show(struct seq_file *m, void *unused)
+{
+	struct intel_crtc *crtc = m->private;
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	intel_wakeref_t wakeref;
+
+	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
+
+	seq_printf(m, "Enabled by kernel parameter: %s\n",
+		   str_yes_no(i915->params.enable_ips));
+
+	if (DISPLAY_VER(i915) >= 8) {
+		seq_puts(m, "Currently: unknown\n");
+	} else {
+		if (intel_de_read(i915, IPS_CTL) & IPS_ENABLE)
+			seq_puts(m, "Currently: enabled\n");
+		else
+			seq_puts(m, "Currently: disabled\n");
+	}
+
+	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
+
+	return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(hsw_ips_debugfs_status);
+
+void hsw_ips_crtc_debugfs_add(struct intel_crtc *crtc)
+{
+	if (!hsw_crtc_supports_ips(crtc))
+		return;
+
+	debugfs_create_file("i915_ips_false_color", 0644, crtc->base.debugfs_entry,
+			    crtc, &hsw_ips_debugfs_false_color_fops);
+
+	debugfs_create_file("i915_ips_status", 0444, crtc->base.debugfs_entry,
+			    crtc, &hsw_ips_debugfs_status_fops);
+}
diff --git a/drivers/gpu/drm/i915/display/hsw_ips.h b/drivers/gpu/drm/i915/display/hsw_ips.h
index 4564dee497d7..4eb83b350791 100644
--- a/drivers/gpu/drm/i915/display/hsw_ips.h
+++ b/drivers/gpu/drm/i915/display/hsw_ips.h
@@ -22,5 +22,6 @@ bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state);
 int hsw_ips_compute_config(struct intel_atomic_state *state,
 			   struct intel_crtc *crtc);
 void hsw_ips_get_config(struct intel_crtc_state *crtc_state);
+void hsw_ips_crtc_debugfs_add(struct intel_crtc *crtc);
 
 #endif /* __HSW_IPS_H__ */
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/display/i9xx_wm.c
index 59714b1080d4..caef72d38798 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/display/i9xx_wm.c
@@ -1,49 +1,18 @@
+// SPDX-License-Identifier: MIT
 /*
- * Copyright © 2012 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- *
- * Authors:
- *    Eugeni Dodonov <eugeni.dodonov@intel.com>
- *
+ * Copyright © 2023 Intel Corporation
  */
 
-#include "display/intel_de.h"
-#include "display/intel_display.h"
-#include "display/intel_display_trace.h"
-#include "display/skl_watermark.h"
-
-#include "gt/intel_engine_regs.h"
-#include "gt/intel_gt.h"
-#include "gt/intel_gt_mcr.h"
-#include "gt/intel_gt_regs.h"
-
 #include "i915_drv.h"
+#include "i9xx_wm.h"
+#include "intel_atomic.h"
+#include "intel_display.h"
+#include "intel_display_trace.h"
 #include "intel_mchbar_regs.h"
-#include "intel_pm.h"
+#include "intel_wm.h"
+#include "skl_watermark.h"
 #include "vlv_sideband.h"
 
-struct drm_i915_clock_gating_funcs {
-	void (*init_clock_gating)(struct drm_i915_private *i915);
-};
-
 /* used in computing the new watermarks state */
 struct intel_wm_config {
 	unsigned int num_pipes_active;
@@ -51,180 +20,16 @@ struct intel_wm_config {
 	bool sprites_scaled;
 };
 
-static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	if (HAS_LLC(dev_priv)) {
-		/*
-		 * WaCompressedResourceDisplayNewHashMode:skl,kbl
-		 * Display WA #0390: skl,kbl
-		 *
-		 * Must match Sampler, Pixel Back End, and Media. See
-		 * WaCompressedResourceSamplerPbeMediaNewHashMode.
-		 */
-		intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
-	}
-
-	/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
-	intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
-
-	/* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
-	intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
-
-	/*
-	 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
-	 * Display WA #0859: skl,bxt,kbl,glk,cfl
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
-}
-
-static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	gen9_init_clock_gating(dev_priv);
-
-	/* WaDisableSDEUnitClockGating:bxt */
-	intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
-
-	/*
-	 * FIXME:
-	 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
-
-	/*
-	 * Wa: Backlight PWM may stop in the asserted state, causing backlight
-	 * to stay fully on.
-	 */
-	intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
-		   PWM1_GATING_DIS | PWM2_GATING_DIS);
-
-	/*
-	 * Lower the display internal timeout.
-	 * This is needed to avoid any hard hangs when DSI port PLL
-	 * is off and a MMIO access is attempted by any privilege
-	 * application, using batch buffers or any other means.
-	 */
-	intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
-
-	/*
-	 * WaFbcTurnOffFbcWatermark:bxt
-	 * Display WA #0562: bxt
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
-
-	/*
-	 * WaFbcHighMemBwCorruptionAvoidance:bxt
-	 * Display WA #0883: bxt
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
-}
-
-static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	gen9_init_clock_gating(dev_priv);
-
-	/*
-	 * WaDisablePWMClockGating:glk
-	 * Backlight PWM may stop in the asserted state, causing backlight
-	 * to stay fully on.
-	 */
-	intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
-		   PWM1_GATING_DIS | PWM2_GATING_DIS);
-}
-
-static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
-{
-	u32 tmp;
-
-	tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
-
-	switch (tmp & CLKCFG_FSB_MASK) {
-	case CLKCFG_FSB_533:
-		dev_priv->fsb_freq = 533; /* 133*4 */
-		break;
-	case CLKCFG_FSB_800:
-		dev_priv->fsb_freq = 800; /* 200*4 */
-		break;
-	case CLKCFG_FSB_667:
-		dev_priv->fsb_freq =  667; /* 167*4 */
-		break;
-	case CLKCFG_FSB_400:
-		dev_priv->fsb_freq = 400; /* 100*4 */
-		break;
-	}
-
-	switch (tmp & CLKCFG_MEM_MASK) {
-	case CLKCFG_MEM_533:
-		dev_priv->mem_freq = 533;
-		break;
-	case CLKCFG_MEM_667:
-		dev_priv->mem_freq = 667;
-		break;
-	case CLKCFG_MEM_800:
-		dev_priv->mem_freq = 800;
-		break;
-	}
-
-	/* detect pineview DDR3 setting */
-	tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
-	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
-}
-
-static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
-{
-	u16 ddrpll, csipll;
-
-	ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
-	csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
-
-	switch (ddrpll & 0xff) {
-	case 0xc:
-		dev_priv->mem_freq = 800;
-		break;
-	case 0x10:
-		dev_priv->mem_freq = 1066;
-		break;
-	case 0x14:
-		dev_priv->mem_freq = 1333;
-		break;
-	case 0x18:
-		dev_priv->mem_freq = 1600;
-		break;
-	default:
-		drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
-			ddrpll & 0xff);
-		dev_priv->mem_freq = 0;
-		break;
-	}
-
-	switch (csipll & 0x3ff) {
-	case 0x00c:
-		dev_priv->fsb_freq = 3200;
-		break;
-	case 0x00e:
-		dev_priv->fsb_freq = 3733;
-		break;
-	case 0x010:
-		dev_priv->fsb_freq = 4266;
-		break;
-	case 0x012:
-		dev_priv->fsb_freq = 4800;
-		break;
-	case 0x014:
-		dev_priv->fsb_freq = 5333;
-		break;
-	case 0x016:
-		dev_priv->fsb_freq = 5866;
-		break;
-	case 0x018:
-		dev_priv->fsb_freq = 6400;
-		break;
-	default:
-		drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
-			csipll & 0x3ff);
-		dev_priv->fsb_freq = 0;
-		break;
-	}
-}
+struct cxsr_latency {
+	bool is_desktop : 1;
+	bool is_ddr3 : 1;
+	u16 fsb_freq;
+	u16 mem_freq;
+	u16 display_sr;
+	u16 display_hpll_disable;
+	u16 cursor_sr;
+	u16 cursor_hpll_disable;
+};
 
 static const struct cxsr_latency cxsr_latency_table[] = {
 	{1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
@@ -361,7 +166,7 @@ static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enabl
 	} else if (IS_I915GM(dev_priv)) {
 		/*
 		 * FIXME can't find a bit like this for 915G, and
-		 * and yet it does have the related watermark in
+		 * yet it does have the related watermark in
 		 * FW_BLC_SELF. What's going on?
 		 */
 		was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
@@ -607,7 +412,7 @@ static const struct intel_watermark_params i830_a_wm_info = {
 
 static const struct intel_watermark_params i830_bc_wm_info = {
 	.fifo_size = I855GM_FIFO_SIZE,
-	.max_wm = I915_MAX_WM/2,
+	.max_wm = I915_MAX_WM / 2,
 	.default_wm = 1,
 	.guard_size = 2,
 	.cacheline_size = I830_FIFO_LINE_SIZE,
@@ -787,34 +592,6 @@ static bool is_enabling(int old, int new, int threshold)
 	return old < threshold && new >= threshold;
 }
 
-static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
-{
-	return dev_priv->display.wm.max_level + 1;
-}
-
-bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
-			    const struct intel_plane_state *plane_state)
-{
-	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
-
-	/* FIXME check the 'enable' instead */
-	if (!crtc_state->hw.active)
-		return false;
-
-	/*
-	 * Treat cursor with fb as always visible since cursor updates
-	 * can happen faster than the vrefresh rate, and the current
-	 * watermark code doesn't handle that correctly. Cursor updates
-	 * which set/clear the fb or change the cursor size are going
-	 * to get throttled by intel_legacy_cursor_update() to work
-	 * around this problem with the watermark code.
-	 */
-	if (plane->id == PLANE_CURSOR)
-		return plane_state->hw.fb != NULL;
-	else
-		return plane_state->uapi.visible;
-}
-
 static bool intel_crtc_active(struct intel_crtc *crtc)
 {
 	/* Be paranoid as we can arrive here with only partial
@@ -939,22 +716,22 @@ static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 		trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 
 	intel_uncore_write(&dev_priv->uncore, DSPFW1,
-		   FW_WM(wm->sr.plane, SR) |
-		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
-		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
-		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
+			   FW_WM(wm->sr.plane, SR) |
+			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
+			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
+			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 	intel_uncore_write(&dev_priv->uncore, DSPFW2,
-		   (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
-		   FW_WM(wm->sr.fbc, FBC_SR) |
-		   FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
-		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
-		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
-		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
+			   (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
+			   FW_WM(wm->sr.fbc, FBC_SR) |
+			   FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
+			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
+			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
+			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 	intel_uncore_write(&dev_priv->uncore, DSPFW3,
-		   (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
-		   FW_WM(wm->sr.cursor, CURSOR_SR) |
-		   FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
-		   FW_WM(wm->hpll.plane, HPLL_SR));
+			   (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
+			   FW_WM(wm->sr.cursor, CURSOR_SR) |
+			   FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
+			   FW_WM(wm->hpll.plane, HPLL_SR));
 
 	intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
 }
@@ -971,10 +748,10 @@ static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 		trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 
 		intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
-			   (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
-			   (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
-			   (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
-			   (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
+				   (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
+				   (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
+				   (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
+				   (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 	}
 
 	/*
@@ -989,50 +766,50 @@ static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 	intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
 
 	intel_uncore_write(&dev_priv->uncore, DSPFW1,
-		   FW_WM(wm->sr.plane, SR) |
-		   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
-		   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
-		   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
+			   FW_WM(wm->sr.plane, SR) |
+			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
+			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
+			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 	intel_uncore_write(&dev_priv->uncore, DSPFW2,
-		   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
-		   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
-		   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
+			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
+			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
+			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 	intel_uncore_write(&dev_priv->uncore, DSPFW3,
-		   FW_WM(wm->sr.cursor, CURSOR_SR));
+			   FW_WM(wm->sr.cursor, CURSOR_SR));
 
 	if (IS_CHERRYVIEW(dev_priv)) {
 		intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
-			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
-			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
+				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
+				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
 		intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
-			   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
-			   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
+				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
+				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
 		intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
-			   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
-			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
+				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
+				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
 		intel_uncore_write(&dev_priv->uncore, DSPHOWM,
-			   FW_WM(wm->sr.plane >> 9, SR_HI) |
-			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
-			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
-			   FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
-			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
-			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
-			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
-			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
-			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
-			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
+				   FW_WM(wm->sr.plane >> 9, SR_HI) |
+				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
+				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
+				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
+				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
+				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
+				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
+				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
+				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
+				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
 	} else {
 		intel_uncore_write(&dev_priv->uncore, DSPFW7,
-			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
-			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
+				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
+				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
 		intel_uncore_write(&dev_priv->uncore, DSPHOWM,
-			   FW_WM(wm->sr.plane >> 9, SR_HI) |
-			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
-			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
-			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
-			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
-			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
-			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
+				   FW_WM(wm->sr.plane >> 9, SR_HI) |
+				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
+				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
+				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
+				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
+				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
+				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
 	}
 
 	intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
@@ -1047,7 +824,7 @@ static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
 	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
 	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
 
-	dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL;
+	dev_priv->display.wm.num_levels = G4X_WM_LEVEL_HPLL + 1;
 }
 
 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
@@ -1154,7 +931,7 @@ static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 	bool dirty = false;
 
-	for (; level < intel_wm_num_levels(dev_priv); level++) {
+	for (; level < dev_priv->display.wm.num_levels; level++) {
 		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
 
 		dirty |= raw->plane[plane_id] != value;
@@ -1173,7 +950,7 @@ static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
 	/* NORMAL level doesn't have an FBC watermark */
 	level = max(level, G4X_WM_LEVEL_SR);
 
-	for (; level < intel_wm_num_levels(dev_priv); level++) {
+	for (; level < dev_priv->display.wm.num_levels; level++) {
 		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
 
 		dirty |= raw->fbc != value;
@@ -1192,7 +969,6 @@ static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
 {
 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
-	int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
 	enum plane_id plane_id = plane->id;
 	bool dirty = false;
 	int level;
@@ -1204,7 +980,7 @@ static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
 		goto out;
 	}
 
-	for (level = 0; level < num_levels; level++) {
+	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
 		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
 		int wm, max_wm;
 
@@ -1274,7 +1050,7 @@ static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 
-	if (level > dev_priv->display.wm.max_level)
+	if (level >= dev_priv->display.wm.num_levels)
 		return false;
 
 	return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
@@ -1610,13 +1386,13 @@ static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
 	/* all latencies in usec */
 	dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
 
-	dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2;
+	dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM2 + 1;
 
 	if (IS_CHERRYVIEW(dev_priv)) {
 		dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
 		dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
 
-		dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
+		dev_priv->display.wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;
 	}
 }
 
@@ -1752,7 +1528,7 @@ static void vlv_invalidate_wms(struct intel_crtc *crtc,
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
-	for (; level < intel_wm_num_levels(dev_priv); level++) {
+	for (; level < dev_priv->display.wm.num_levels; level++) {
 		enum plane_id plane_id;
 
 		for_each_plane_id_on_crtc(crtc, plane_id)
@@ -1779,10 +1555,9 @@ static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
 				 int level, enum plane_id plane_id, u16 value)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
-	int num_levels = intel_wm_num_levels(dev_priv);
 	bool dirty = false;
 
-	for (; level < num_levels; level++) {
+	for (; level < dev_priv->display.wm.num_levels; level++) {
 		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
 
 		dirty |= raw->plane[plane_id] != value;
@@ -1798,7 +1573,6 @@ static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 	enum plane_id plane_id = plane->id;
-	int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
 	int level;
 	bool dirty = false;
 
@@ -1807,7 +1581,7 @@ static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
 		goto out;
 	}
 
-	for (level = 0; level < num_levels; level++) {
+	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
 		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
 		int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
 		int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
@@ -1866,7 +1640,7 @@ static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
 	int level;
 
 	/* initially allow all levels */
-	wm_state->num_levels = intel_wm_num_levels(dev_priv);
+	wm_state->num_levels = dev_priv->display.wm.num_levels;
 	/*
 	 * Note that enabling cxsr with no primary/sprite planes
 	 * enabled can wedge the pipe. Hence we only allow cxsr
@@ -2129,7 +1903,7 @@ static void vlv_merge_wm(struct drm_i915_private *dev_priv,
 	struct intel_crtc *crtc;
 	int num_active_pipes = 0;
 
-	wm->level = dev_priv->display.wm.max_level;
+	wm->level = dev_priv->display.wm.num_levels - 1;
 	wm->cxsr = true;
 
 	for_each_intel_crtc(&dev_priv->drm, crtc) {
@@ -2836,6 +2610,8 @@ static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
 {
 	u64 sskpd;
 
+	i915->display.wm.num_levels = 5;
+
 	sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
 
 	wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
@@ -2851,6 +2627,8 @@ static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
 {
 	u32 sskpd;
 
+	i915->display.wm.num_levels = 4;
+
 	sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
 
 	wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
@@ -2863,6 +2641,8 @@ static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
 {
 	u32 mltr;
 
+	i915->display.wm.num_levels = 3;
+
 	mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
 
 	/* ILK primary LP0 latency is 700 ns */
@@ -2887,61 +2667,16 @@ static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
 		wm[0] = 13;
 }
 
-int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
-{
-	/* how many WM levels are we expecting */
-	if (HAS_HW_SAGV_WM(dev_priv))
-		return 5;
-	else if (DISPLAY_VER(dev_priv) >= 9)
-		return 7;
-	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
-		return 4;
-	else if (DISPLAY_VER(dev_priv) >= 6)
-		return 3;
-	else
-		return 2;
-}
-
-void intel_print_wm_latency(struct drm_i915_private *dev_priv,
-			    const char *name, const u16 wm[])
-{
-	int level, max_level = ilk_wm_max_level(dev_priv);
-
-	for (level = 0; level <= max_level; level++) {
-		unsigned int latency = wm[level];
-
-		if (latency == 0) {
-			drm_dbg_kms(&dev_priv->drm,
-				    "%s WM%d latency not provided\n",
-				    name, level);
-			continue;
-		}
-
-		/*
-		 * - latencies are in us on gen9.
-		 * - before then, WM1+ latency values are in 0.5us units
-		 */
-		if (DISPLAY_VER(dev_priv) >= 9)
-			latency *= 10;
-		else if (level > 0)
-			latency *= 5;
-
-		drm_dbg_kms(&dev_priv->drm,
-			    "%s WM%d latency %u (%u.%u usec)\n", name, level,
-			    wm[level], latency / 10, latency % 10);
-	}
-}
-
 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
 				    u16 wm[5], u16 min)
 {
-	int level, max_level = ilk_wm_max_level(dev_priv);
+	int level;
 
 	if (wm[0] >= min)
 		return false;
 
 	wm[0] = max(wm[0], min);
-	for (level = 1; level <= max_level; level++)
+	for (level = 1; level < dev_priv->display.wm.num_levels; level++)
 		wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
 
 	return true;
@@ -3061,8 +2796,8 @@ static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
 	const struct intel_plane_state *pristate = NULL;
 	const struct intel_plane_state *sprstate = NULL;
 	const struct intel_plane_state *curstate = NULL;
-	int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
 	struct ilk_wm_maximums max;
+	int level, usable_level;
 
 	pipe_wm = &crtc_state->wm.ilk.optimal;
 
@@ -3079,7 +2814,7 @@ static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
 	pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
 	pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
 
-	usable_level = max_level;
+	usable_level = dev_priv->display.wm.num_levels - 1;
 
 	/* ILK/SNB: LP2+ watermarks only w/o sprites */
 	if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
@@ -3133,7 +2868,7 @@ static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
 		intel_atomic_get_old_crtc_state(state, crtc);
 	struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
 	const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
-	int level, max_level = ilk_wm_max_level(dev_priv);
+	int level;
 
 	/*
 	 * Start with the final, target watermarks, then combine with the
@@ -3150,7 +2885,7 @@ static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
 	a->sprites_enabled |= b->sprites_enabled;
 	a->sprites_scaled |= b->sprites_scaled;
 
-	for (level = 0; level <= max_level; level++) {
+	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
 		struct intel_wm_level *a_wm = &a->wm[level];
 		const struct intel_wm_level *b_wm = &b->wm[level];
 
@@ -3221,8 +2956,8 @@ static void ilk_wm_merge(struct drm_i915_private *dev_priv,
 			 const struct ilk_wm_maximums *max,
 			 struct intel_pipe_wm *merged)
 {
-	int level, max_level = ilk_wm_max_level(dev_priv);
-	int last_enabled_level = max_level;
+	int level, num_levels = dev_priv->display.wm.num_levels;
+	int last_enabled_level = num_levels - 1;
 
 	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
 	if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
@@ -3233,7 +2968,7 @@ static void ilk_wm_merge(struct drm_i915_private *dev_priv,
 	merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
 
 	/* merge each WM1+ level */
-	for (level = 1; level <= max_level; level++) {
+	for (level = 1; level < num_levels; level++) {
 		struct intel_wm_level *wm = &merged->wm[level];
 
 		ilk_merge_wm_level(dev_priv, level, wm);
@@ -3258,7 +2993,7 @@ static void ilk_wm_merge(struct drm_i915_private *dev_priv,
 	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
 	if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
 	    dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
-		for (level = 2; level <= max_level; level++) {
+		for (level = 2; level < num_levels; level++) {
 			struct intel_wm_level *wm = &merged->wm[level];
 
 			wm->enable = false;
@@ -3346,17 +3081,18 @@ static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
 	}
 }
 
-/* Find the result with the highest level enabled. Check for enable_fbc_wm in
- * case both are at the same level. Prefer r1 in case they're the same. */
+/*
+ * Find the result with the highest level enabled. Check for enable_fbc_wm in
+ * case both are at the same level. Prefer r1 in case they're the same.
+ */
 static struct intel_pipe_wm *
 ilk_find_best_result(struct drm_i915_private *dev_priv,
 		     struct intel_pipe_wm *r1,
 		     struct intel_pipe_wm *r2)
 {
-	int level, max_level = ilk_wm_max_level(dev_priv);
-	int level1 = 0, level2 = 0;
+	int level, level1 = 0, level2 = 0;
 
-	for (level = 1; level <= max_level; level++) {
+	for (level = 1; level < dev_priv->display.wm.num_levels; level++) {
 		if (r1->wm[level].enable)
 			level1 = level;
 		if (r2->wm[level].enable)
@@ -3630,20 +3366,141 @@ static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
 		active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
 		active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
 	} else {
-		int level, max_level = ilk_wm_max_level(dev_priv);
+		int level;
 
 		/*
 		 * For inactive pipes, all watermark levels
 		 * should be marked as enabled but zeroed,
 		 * which is what we'd compute them to.
 		 */
-		for (level = 0; level <= max_level; level++)
+		for (level = 0; level < dev_priv->display.wm.num_levels; level++)
 			active->wm[level].enable = true;
 	}
 
 	crtc->wm.active.ilk = *active;
 }
 
+static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
+{
+	struct drm_plane *plane;
+	struct intel_crtc *crtc;
+
+	for_each_intel_crtc(state->dev, crtc) {
+		struct intel_crtc_state *crtc_state;
+
+		crtc_state = intel_atomic_get_crtc_state(state, crtc);
+		if (IS_ERR(crtc_state))
+			return PTR_ERR(crtc_state);
+
+		if (crtc_state->hw.active) {
+			/*
+			 * Preserve the inherited flag to avoid
+			 * taking the full modeset path.
+			 */
+			crtc_state->inherited = true;
+		}
+	}
+
+	drm_for_each_plane(plane, state->dev) {
+		struct drm_plane_state *plane_state;
+
+		plane_state = drm_atomic_get_plane_state(state, plane);
+		if (IS_ERR(plane_state))
+			return PTR_ERR(plane_state);
+	}
+
+	return 0;
+}
+
+/*
+ * Calculate what we think the watermarks should be for the state we've read
+ * out of the hardware and then immediately program those watermarks so that
+ * we ensure the hardware settings match our internal state.
+ *
+ * We can calculate what we think WM's should be by creating a duplicate of the
+ * current state (which was constructed during hardware readout) and running it
+ * through the atomic check code to calculate new watermark values in the
+ * state object.
+ */
+void ilk_wm_sanitize(struct drm_i915_private *dev_priv)
+{
+	struct drm_atomic_state *state;
+	struct intel_atomic_state *intel_state;
+	struct intel_crtc *crtc;
+	struct intel_crtc_state *crtc_state;
+	struct drm_modeset_acquire_ctx ctx;
+	int ret;
+	int i;
+
+	/* Only supported on platforms that use atomic watermark design */
+	if (!dev_priv->display.funcs.wm->optimize_watermarks)
+		return;
+
+	if (drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) >= 9))
+		return;
+
+	state = drm_atomic_state_alloc(&dev_priv->drm);
+	if (drm_WARN_ON(&dev_priv->drm, !state))
+		return;
+
+	intel_state = to_intel_atomic_state(state);
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+retry:
+	state->acquire_ctx = &ctx;
+
+	/*
+	 * Hardware readout is the only time we don't want to calculate
+	 * intermediate watermarks (since we don't trust the current
+	 * watermarks).
+	 */
+	if (!HAS_GMCH(dev_priv))
+		intel_state->skip_intermediate_wm = true;
+
+	ret = ilk_sanitize_watermarks_add_affected(state);
+	if (ret)
+		goto fail;
+
+	ret = intel_atomic_check(&dev_priv->drm, state);
+	if (ret)
+		goto fail;
+
+	/* Write calculated watermark values back */
+	for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
+		crtc_state->wm.need_postvbl_update = true;
+		intel_optimize_watermarks(intel_state, crtc);
+
+		to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
+	}
+
+fail:
+	if (ret == -EDEADLK) {
+		drm_atomic_state_clear(state);
+		drm_modeset_backoff(&ctx);
+		goto retry;
+	}
+
+	/*
+	 * If we fail here, it means that the hardware appears to be
+	 * programmed in a way that shouldn't be possible, given our
+	 * understanding of watermark requirements.  This might mean a
+	 * mistake in the hardware readout code or a mistake in the
+	 * watermark calculations for a given platform.  Raise a WARN
+	 * so that this is noticeable.
+	 *
+	 * If this actually happens, we'll have to just leave the
+	 * BIOS-programmed watermarks untouched and hope for the best.
+	 */
+	drm_WARN(&dev_priv->drm, ret,
+		 "Could not determine valid watermarks for inherited state\n");
+
+	drm_atomic_state_put(state);
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+}
+
 #define _FW_WM(value, plane) \
 	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
 #define _FW_WM_VLV(value, plane) \
@@ -3751,7 +3608,7 @@ static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
 #undef _FW_WM
 #undef _FW_WM_VLV
 
-void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
+static void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
 {
 	struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
 	struct intel_crtc *crtc;
@@ -3844,7 +3701,7 @@ void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
 		    str_yes_no(wm->fbc_en));
 }
 
-void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
+static void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
 {
 	struct intel_plane *plane;
 	struct intel_crtc *crtc;
@@ -3859,12 +3716,12 @@ void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
 		struct intel_plane_state *plane_state =
 			to_intel_plane_state(plane->base.state);
 		enum plane_id plane_id = plane->id;
-		int level, num_levels = intel_wm_num_levels(dev_priv);
+		int level;
 
 		if (plane_state->uapi.visible)
 			continue;
 
-		for (level = 0; level < num_levels; level++) {
+		for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
 			struct g4x_pipe_wm *raw =
 				&crtc_state->wm.g4x.raw[level];
 
@@ -3893,7 +3750,13 @@ void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 }
 
-void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
+static void g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
+{
+	g4x_wm_get_hw_state(i915);
+	g4x_wm_sanitize(i915);
+}
+
+static void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
 {
 	struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
 	struct intel_crtc *crtc;
@@ -3929,7 +3792,7 @@ void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
 			drm_dbg_kms(&dev_priv->drm,
 				    "Punit not acking DDR DVFS request, "
 				    "assuming DDR DVFS is disabled\n");
-			dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5;
+			dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
 		} else {
 			val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 			if ((val & FORCE_DDR_HIGH_FREQ) == 0)
@@ -3993,7 +3856,7 @@ void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
 		    wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
 }
 
-void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
+static void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
 {
 	struct intel_plane *plane;
 	struct intel_crtc *crtc;
@@ -4008,12 +3871,12 @@ void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
 		struct intel_plane_state *plane_state =
 			to_intel_plane_state(plane->base.state);
 		enum plane_id plane_id = plane->id;
-		int level, num_levels = intel_wm_num_levels(dev_priv);
+		int level;
 
 		if (plane_state->uapi.visible)
 			continue;
 
-		for (level = 0; level < num_levels; level++) {
+		for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
 			struct g4x_pipe_wm *raw =
 				&crtc_state->wm.vlv.raw[level];
 
@@ -4039,6 +3902,12 @@ void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 }
 
+static void vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
+{
+	vlv_wm_get_hw_state(i915);
+	vlv_wm_sanitize(i915);
+}
+
 /*
  * FIXME should probably kill this and improve
  * the real watermark readout/sanitation instead
@@ -4055,7 +3924,7 @@ static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
 	 */
 }
 
-void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
+static void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
 {
 	struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
 	struct intel_crtc *crtc;
@@ -4076,811 +3945,24 @@ void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
 	}
 
 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
-		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
+		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) &
+				    WM_MISC_DATA_PARTITION_5_6) ?
 			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
 	else if (IS_IVYBRIDGE(dev_priv))
-		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
+		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) &
+				    DISP_DATA_PARTITION_5_6) ?
 			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
 
 	hw->enable_fbc_wm =
 		!(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
 }
 
-static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	/*
-	 * On Ibex Peak and Cougar Point, we need to disable clock
-	 * gating for the panel power sequencer or it will fail to
-	 * start up when no ports are active.
-	 */
-	intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
-}
-
-static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
-{
-	enum pipe pipe;
-
-	for_each_pipe(dev_priv, pipe) {
-		intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
-
-		intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
-		intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
-	}
-}
-
-static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
-
-	/*
-	 * Required for FBC
-	 * WaFbcDisableDpfcClockGating:ilk
-	 */
-	dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
-		   ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
-		   ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
-
-	intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
-		   MARIUNIT_CLOCK_GATE_DISABLE |
-		   SVSMUNIT_CLOCK_GATE_DISABLE);
-	intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
-		   VFMUNIT_CLOCK_GATE_DISABLE);
-
-	/*
-	 * According to the spec the following bits should be set in
-	 * order to enable memory self-refresh
-	 * The bit 22/21 of 0x42004
-	 * The bit 5 of 0x42020
-	 * The bit 15 of 0x45000
-	 */
-	intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
-		   (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
-		    ILK_DPARB_GATE | ILK_VSDPFD_FULL));
-	dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
-	intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
-		   (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
-		    DISP_FBC_WM_DIS));
-
-	/*
-	 * Based on the document from hardware guys the following bits
-	 * should be set unconditionally in order to enable FBC.
-	 * The bit 22 of 0x42000
-	 * The bit 22 of 0x42004
-	 * The bit 7,8,9 of 0x42020.
-	 */
-	if (IS_IRONLAKE_M(dev_priv)) {
-		/* WaFbcAsynchFlipDisableFbcQueue:ilk */
-		intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
-		intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
-	}
-
-	intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
-
-	intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
-
-	g4x_disable_trickle_feed(dev_priv);
-
-	ibx_init_clock_gating(dev_priv);
-}
-
-static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	enum pipe pipe;
-	u32 val;
-
-	/*
-	 * On Ibex Peak and Cougar Point, we need to disable clock
-	 * gating for the panel power sequencer or it will fail to
-	 * start up when no ports are active.
-	 */
-	intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
-		   PCH_DPLUNIT_CLOCK_GATE_DISABLE |
-		   PCH_CPUNIT_CLOCK_GATE_DISABLE);
-	intel_uncore_rmw(&dev_priv->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
-	/* The below fixes the weird display corruption, a few pixels shifted
-	 * downward, on (only) LVDS of some HP laptops with IVY.
-	 */
-	for_each_pipe(dev_priv, pipe) {
-		val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
-		val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
-		val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
-		if (dev_priv->display.vbt.fdi_rx_polarity_inverted)
-			val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
-		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
-		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
-		intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
-	}
-	/* WADP0ClockGatingDisable */
-	for_each_pipe(dev_priv, pipe) {
-		intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
-			   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
-	}
-}
-
-static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
-{
-	u32 tmp;
-
-	tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
-	if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
-		drm_dbg_kms(&dev_priv->drm,
-			    "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
-			    tmp);
-}
-
-static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
-
-	intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
-
-	intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
-
-	intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
-		   intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
-		   GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
-		   GEN6_CSUNIT_CLOCK_GATE_DISABLE);
-
-	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
-	 * gating disable must be set.  Failure to set it results in
-	 * flickering pixels due to Z write ordering failures after
-	 * some amount of runtime in the Mesa "fire" demo, and Unigine
-	 * Sanctuary and Tropics, and apparently anything else with
-	 * alpha test or pixel discard.
-	 *
-	 * According to the spec, bit 11 (RCCUNIT) must also be set,
-	 * but we didn't debug actual testcases to find it out.
-	 *
-	 * WaDisableRCCUnitClockGating:snb
-	 * WaDisableRCPBUnitClockGating:snb
-	 */
-	intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
-		   GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
-		   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
-
-	/*
-	 * According to the spec the following bits should be
-	 * set in order to enable memory self-refresh and fbc:
-	 * The bit21 and bit22 of 0x42000
-	 * The bit21 and bit22 of 0x42004
-	 * The bit5 and bit7 of 0x42020
-	 * The bit14 of 0x70180
-	 * The bit14 of 0x71180
-	 *
-	 * WaFbcAsynchFlipDisableFbcQueue:snb
-	 */
-	intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
-		   intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
-		   ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
-	intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
-		   intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
-		   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
-	intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
-		   intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
-		   ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
-		   ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
-
-	g4x_disable_trickle_feed(dev_priv);
-
-	cpt_init_clock_gating(dev_priv);
-
-	gen6_check_mch_setup(dev_priv);
-}
-
-static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	/*
-	 * TODO: this bit should only be enabled when really needed, then
-	 * disabled when not needed anymore in order to save power.
-	 */
-	if (HAS_PCH_LPT_LP(dev_priv))
-		intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
-				 0, PCH_LP_PARTITION_LEVEL_DISABLE);
-
-	/* WADPOClockGatingDisable:hsw */
-	intel_uncore_rmw(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
-			 0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
-}
-
-static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
-{
-	if (HAS_PCH_LPT_LP(dev_priv)) {
-		u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
-
-		val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
-		intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
-	}
-}
-
-static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
-				   int general_prio_credits,
-				   int high_prio_credits)
-{
-	u32 misccpctl;
-	u32 val;
-
-	/* WaTempDisableDOPClkGating:bdw */
-	misccpctl = intel_uncore_rmw(&dev_priv->uncore, GEN7_MISCCPCTL,
-				     GEN7_DOP_CLOCK_GATE_ENABLE, 0);
-
-	val = intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
-	val &= ~L3_PRIO_CREDITS_MASK;
-	val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
-	val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
-	intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_L3SQCREG1, val);
-
-	/*
-	 * Wait at least 100 clocks before re-enabling clock gating.
-	 * See the definition of L3SQCREG1 in BSpec.
-	 */
-	intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
-	udelay(1);
-	intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
-}
-
-static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	/* Wa_1409120013:icl,ehl */
-	intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-			   DPFC_CHICKEN_COMP_DUMMY_PIXEL);
-
-	/*Wa_14010594013:icl, ehl */
-	intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
-			 0, ICL_DELAY_PMRSP);
-}
-
-static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	/* Wa_1409120013 */
-	if (DISPLAY_VER(dev_priv) == 12)
-		intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-				   DPFC_CHICKEN_COMP_DUMMY_PIXEL);
-
-	/* Wa_1409825376:tgl (pre-prod)*/
-	if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
-		intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, TGL_VRH_GATING_DIS);
-
-	/* Wa_14013723622:tgl,rkl,dg1,adl-s */
-	if (DISPLAY_VER(dev_priv) == 12)
-		intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
-				 CLKREQ_POLICY_MEM_UP_OVRD, 0);
-}
-
-static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	gen12lp_init_clock_gating(dev_priv);
-
-	/* Wa_22011091694:adlp */
-	intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
-
-	/* Bspec/49189 Initialize Sequence */
-	intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
-}
-
-static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	gen12lp_init_clock_gating(dev_priv);
-
-	/* Wa_1409836686:dg1[a0] */
-	if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
-		intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, DPT_GATING_DIS);
-}
-
-static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	/* Wa_22010146351:xehpsdv */
-	if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
-		intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
-}
-
-static void dg2_init_clock_gating(struct drm_i915_private *i915)
-{
-	/* Wa_22010954014:dg2 */
-	intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
-			 SGSI_SIDECLK_DIS);
-
-	/*
-	 * Wa_14010733611:dg2_g10
-	 * Wa_22010146351:dg2_g10
-	 */
-	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
-		intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
-				 SGR_DIS | SGGI_DIS);
-}
-
-static void pvc_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	/* Wa_14012385139:pvc */
-	if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
-		intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
-
-	/* Wa_22010954014:pvc */
-	if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
-		intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
-}
-
-static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	if (!HAS_PCH_CNP(dev_priv))
-		return;
-
-	/* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
-	intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
-}
-
-static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	cnp_init_clock_gating(dev_priv);
-	gen9_init_clock_gating(dev_priv);
-
-	/* WAC6entrylatency:cfl */
-	intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
-
-	/*
-	 * WaFbcTurnOffFbcWatermark:cfl
-	 * Display WA #0562: cfl
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
-
-	/*
-	 * WaFbcNukeOnHostModify:cfl
-	 * Display WA #0873: cfl
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-			 0, DPFC_NUKE_ON_ANY_MODIFICATION);
-}
-
-static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	gen9_init_clock_gating(dev_priv);
-
-	/* WAC6entrylatency:kbl */
-	intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
-
-	/* WaDisableSDEUnitClockGating:kbl */
-	if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
-		intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6,
-				 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
-
-	/* WaDisableGamClockGating:kbl */
-	if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
-		intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1,
-				 0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
-
-	/*
-	 * WaFbcTurnOffFbcWatermark:kbl
-	 * Display WA #0562: kbl
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
-
-	/*
-	 * WaFbcNukeOnHostModify:kbl
-	 * Display WA #0873: kbl
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-			 0, DPFC_NUKE_ON_ANY_MODIFICATION);
-}
-
-static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	gen9_init_clock_gating(dev_priv);
-
-	/* WaDisableDopClockGating:skl */
-	intel_uncore_rmw(&dev_priv->uncore, GEN7_MISCCPCTL,
-			 GEN7_DOP_CLOCK_GATE_ENABLE, 0);
-
-	/* WAC6entrylatency:skl */
-	intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
-
-	/*
-	 * WaFbcTurnOffFbcWatermark:skl
-	 * Display WA #0562: skl
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
-
-	/*
-	 * WaFbcNukeOnHostModify:skl
-	 * Display WA #0873: skl
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-			 0, DPFC_NUKE_ON_ANY_MODIFICATION);
-
-	/*
-	 * WaFbcHighMemBwCorruptionAvoidance:skl
-	 * Display WA #0883: skl
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
-}
-
-static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	enum pipe pipe;
-
-	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
-	intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
-
-	/* WaSwitchSolVfFArbitrationPriority:bdw */
-	intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
-
-	/* WaPsrDPAMaskVBlankInSRD:bdw */
-	intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, DPA_MASK_VBLANK_SRD);
-
-	for_each_pipe(dev_priv, pipe) {
-		/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
-		intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
-				 0, BDW_DPRS_MASK_VBLANK_SRD);
-	}
-
-	/* WaVSRefCountFullforceMissDisable:bdw */
-	/* WaDSRefCountFullforceMissDisable:bdw */
-	intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
-			 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
-
-	intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
-		   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
-
-	/* WaDisableSDEUnitClockGating:bdw */
-	intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
-
-	/* WaProgramL3SqcReg1Default:bdw */
-	gen8_set_l3sqc_credits(dev_priv, 30, 2);
-
-	/* WaKVMNotificationOnConfigChange:bdw */
-	intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR2_1,
-			 0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
-
-	lpt_init_clock_gating(dev_priv);
-
-	/* WaDisableDopClockGating:bdw
-	 *
-	 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
-	 * clock gating.
-	 */
-	intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
-}
-
-static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
-	intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
-
-	/* This is required by WaCatErrorRejectionIssue:hsw */
-	intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
-			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
-
-	/* WaSwitchSolVfFArbitrationPriority:hsw */
-	intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
-
-	lpt_init_clock_gating(dev_priv);
-}
-
-static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
-
-	/* WaFbcAsynchFlipDisableFbcQueue:ivb */
-	intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
-
-	/* WaDisableBackToBackFlipFix:ivb */
-	intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
-		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
-		   CHICKEN3_DGMG_DONE_FIX_DISABLE);
-
-	if (IS_IVB_GT1(dev_priv))
-		intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
-			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
-	else {
-		/* must write both registers */
-		intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
-			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
-		intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
-			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
-	}
-
-	/*
-	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
-	 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
-	 */
-	intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
-		   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
-
-	/* This is required by WaCatErrorRejectionIssue:ivb */
-	intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
-			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
-
-	g4x_disable_trickle_feed(dev_priv);
-
-	intel_uncore_rmw(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
-			 GEN6_MBC_SNPCR_MED);
-
-	if (!HAS_PCH_NOP(dev_priv))
-		cpt_init_clock_gating(dev_priv);
-
-	gen6_check_mch_setup(dev_priv);
-}
-
-static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	/* WaDisableBackToBackFlipFix:vlv */
-	intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
-		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
-		   CHICKEN3_DGMG_DONE_FIX_DISABLE);
-
-	/* WaDisableDopClockGating:vlv */
-	intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
-		   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
-
-	/* This is required by WaCatErrorRejectionIssue:vlv */
-	intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
-			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
-
-	/*
-	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
-	 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
-	 */
-	intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
-		   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
-
-	/* WaDisableL3Bank2xClockGate:vlv
-	 * Disabling L3 clock gating- MMIO 940c[25] = 1
-	 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
-	intel_uncore_rmw(&dev_priv->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
-
-	/*
-	 * WaDisableVLVClockGating_VBIIssue:vlv
-	 * Disable clock gating on th GCFG unit to prevent a delay
-	 * in the reporting of vblank events.
-	 */
-	intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
-}
-
-static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	/* WaVSRefCountFullforceMissDisable:chv */
-	/* WaDSRefCountFullforceMissDisable:chv */
-	intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
-			 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
-
-	/* WaDisableSemaphoreAndSyncFlipWait:chv */
-	intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
-		   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
-
-	/* WaDisableCSUnitClockGating:chv */
-	intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
-
-	/* WaDisableSDEUnitClockGating:chv */
-	intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
-
-	/*
-	 * WaProgramL3SqcReg1Default:chv
-	 * See gfxspecs/Related Documents/Performance Guide/
-	 * LSQC Setting Recommendations.
-	 */
-	gen8_set_l3sqc_credits(dev_priv, 38, 2);
-}
-
-static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	u32 dspclk_gate;
-
-	intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
-	intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
-		   GS_UNIT_CLOCK_GATE_DISABLE |
-		   CL_UNIT_CLOCK_GATE_DISABLE);
-	intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
-	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
-		OVRUNIT_CLOCK_GATE_DISABLE |
-		OVCUNIT_CLOCK_GATE_DISABLE;
-	if (IS_GM45(dev_priv))
-		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
-	intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D(dev_priv), dspclk_gate);
-
-	g4x_disable_trickle_feed(dev_priv);
-}
-
-static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	struct intel_uncore *uncore = &dev_priv->uncore;
-
-	intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
-	intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
-	intel_uncore_write(uncore, DSPCLK_GATE_D(dev_priv), 0);
-	intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
-	intel_uncore_write16(uncore, DEUC, 0);
-	intel_uncore_write(uncore,
-			   MI_ARB_STATE,
-			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
-}
-
-static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
-		   I965_RCC_CLOCK_GATE_DISABLE |
-		   I965_RCPB_CLOCK_GATE_DISABLE |
-		   I965_ISC_CLOCK_GATE_DISABLE |
-		   I965_FBC_CLOCK_GATE_DISABLE);
-	intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
-	intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
-		   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
-}
-
-static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
-
-	dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
-		DSTATE_DOT_CLOCK_GATING;
-	intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
-
-	if (IS_PINEVIEW(dev_priv))
-		intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
-				   _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
-
-	/* IIR "flip pending" means done if this bit is set */
-	intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
-			   _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
-
-	/* interrupts should cause a wake up from C3 */
-	intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
-
-	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
-	intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
-
-	intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
-		   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
-}
-
-static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
-
-	/* interrupts should cause a wake up from C3 */
-	intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
-		   _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
-
-	intel_uncore_write(&dev_priv->uncore, MEM_MODE,
-		   _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
-
-	/*
-	 * Have FBC ignore 3D activity since we use software
-	 * render tracking, and otherwise a pure 3D workload
-	 * (even if it just renders a single frame and then does
-	 * abosultely nothing) would not allow FBC to recompress
-	 * until a 2D blit occurs.
-	 */
-	intel_uncore_write(&dev_priv->uncore, SCPD0,
-		   _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
-}
-
-static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	intel_uncore_write(&dev_priv->uncore, MEM_MODE,
-		   _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
-		   _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
-}
-
-void intel_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
-}
-
-void intel_suspend_hw(struct drm_i915_private *dev_priv)
-{
-	if (HAS_PCH_LPT(dev_priv))
-		lpt_suspend_hw(dev_priv);
-}
-
-static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
-{
-	drm_dbg_kms(&dev_priv->drm,
-		    "No clock gating settings or workarounds applied.\n");
-}
-
-#define CG_FUNCS(platform)						\
-static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
-	.init_clock_gating = platform##_init_clock_gating,		\
-}
-
-CG_FUNCS(pvc);
-CG_FUNCS(dg2);
-CG_FUNCS(xehpsdv);
-CG_FUNCS(adlp);
-CG_FUNCS(dg1);
-CG_FUNCS(gen12lp);
-CG_FUNCS(icl);
-CG_FUNCS(cfl);
-CG_FUNCS(skl);
-CG_FUNCS(kbl);
-CG_FUNCS(bxt);
-CG_FUNCS(glk);
-CG_FUNCS(bdw);
-CG_FUNCS(chv);
-CG_FUNCS(hsw);
-CG_FUNCS(ivb);
-CG_FUNCS(vlv);
-CG_FUNCS(gen6);
-CG_FUNCS(ilk);
-CG_FUNCS(g4x);
-CG_FUNCS(i965gm);
-CG_FUNCS(i965g);
-CG_FUNCS(gen3);
-CG_FUNCS(i85x);
-CG_FUNCS(i830);
-CG_FUNCS(nop);
-#undef CG_FUNCS
-
-/**
- * intel_init_clock_gating_hooks - setup the clock gating hooks
- * @dev_priv: device private
- *
- * Setup the hooks that configure which clocks of a given platform can be
- * gated and also apply various GT and display specific workarounds for these
- * platforms. Note that some GT specific workarounds are applied separately
- * when GPU contexts or batchbuffers start their execution.
- */
-void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
-{
-	if (IS_PONTEVECCHIO(dev_priv))
-		dev_priv->clock_gating_funcs = &pvc_clock_gating_funcs;
-	else if (IS_DG2(dev_priv))
-		dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
-	else if (IS_XEHPSDV(dev_priv))
-		dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
-	else if (IS_ALDERLAKE_P(dev_priv))
-		dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
-	else if (IS_DG1(dev_priv))
-		dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
-	else if (GRAPHICS_VER(dev_priv) == 12)
-		dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
-	else if (GRAPHICS_VER(dev_priv) == 11)
-		dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
-	else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
-		dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
-	else if (IS_SKYLAKE(dev_priv))
-		dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
-	else if (IS_KABYLAKE(dev_priv))
-		dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
-	else if (IS_BROXTON(dev_priv))
-		dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
-	else if (IS_GEMINILAKE(dev_priv))
-		dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
-	else if (IS_BROADWELL(dev_priv))
-		dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
-	else if (IS_CHERRYVIEW(dev_priv))
-		dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
-	else if (IS_HASWELL(dev_priv))
-		dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
-	else if (IS_IVYBRIDGE(dev_priv))
-		dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
-	else if (IS_VALLEYVIEW(dev_priv))
-		dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
-	else if (GRAPHICS_VER(dev_priv) == 6)
-		dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
-	else if (GRAPHICS_VER(dev_priv) == 5)
-		dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
-	else if (IS_G4X(dev_priv))
-		dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
-	else if (IS_I965GM(dev_priv))
-		dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
-	else if (IS_I965G(dev_priv))
-		dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
-	else if (GRAPHICS_VER(dev_priv) == 3)
-		dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
-	else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
-		dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
-	else if (GRAPHICS_VER(dev_priv) == 2)
-		dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
-	else {
-		MISSING_CASE(INTEL_DEVID(dev_priv));
-		dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
-	}
-}
-
 static const struct intel_wm_funcs ilk_wm_funcs = {
 	.compute_pipe_wm = ilk_compute_pipe_wm,
 	.compute_intermediate_wm = ilk_compute_intermediate_wm,
 	.initial_watermarks = ilk_initial_watermarks,
 	.optimize_watermarks = ilk_optimize_watermarks,
+	.get_hw_state = ilk_wm_get_hw_state,
 };
 
 static const struct intel_wm_funcs vlv_wm_funcs = {
@@ -4889,6 +3971,7 @@ static const struct intel_wm_funcs vlv_wm_funcs = {
 	.initial_watermarks = vlv_initial_watermarks,
 	.optimize_watermarks = vlv_optimize_watermarks,
 	.atomic_update_watermarks = vlv_atomic_update_fifo,
+	.get_hw_state = vlv_wm_get_hw_state_and_sanitize,
 };
 
 static const struct intel_wm_funcs g4x_wm_funcs = {
@@ -4896,6 +3979,7 @@ static const struct intel_wm_funcs g4x_wm_funcs = {
 	.compute_intermediate_wm = g4x_compute_intermediate_wm,
 	.initial_watermarks = g4x_initial_watermarks,
 	.optimize_watermarks = g4x_optimize_watermarks,
+	.get_hw_state = g4x_wm_get_hw_state_and_sanitize,
 };
 
 static const struct intel_wm_funcs pnv_wm_funcs = {
@@ -4917,35 +4001,12 @@ static const struct intel_wm_funcs i845_wm_funcs = {
 static const struct intel_wm_funcs nop_funcs = {
 };
 
-/* Set up chip specific power management-related functions */
-void intel_init_pm(struct drm_i915_private *dev_priv)
+void i9xx_wm_init(struct drm_i915_private *dev_priv)
 {
-	if (DISPLAY_VER(dev_priv) >= 9) {
-		skl_wm_init(dev_priv);
-		return;
-	}
-
-	/* For cxsr */
-	if (IS_PINEVIEW(dev_priv))
-		pnv_get_mem_freq(dev_priv);
-	else if (GRAPHICS_VER(dev_priv) == 5)
-		ilk_get_mem_freq(dev_priv);
-
 	/* For FIFO watermark updates */
 	if (HAS_PCH_SPLIT(dev_priv)) {
 		ilk_setup_wm_latency(dev_priv);
-
-		if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] &&
-		     dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) ||
-		    (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] &&
-		     dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) {
-			dev_priv->display.funcs.wm = &ilk_wm_funcs;
-		} else {
-			drm_dbg_kms(&dev_priv->drm,
-				    "Failed to read display plane latency. "
-				    "Disable CxSR\n");
-			dev_priv->display.funcs.wm = &nop_funcs;
-		}
+		dev_priv->display.funcs.wm = &ilk_wm_funcs;
 	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		vlv_setup_wm_latency(dev_priv);
 		dev_priv->display.funcs.wm = &vlv_wm_funcs;
@@ -4966,8 +4027,9 @@ void intel_init_pm(struct drm_i915_private *dev_priv)
 			/* Disable CxSR and never update its watermark again */
 			intel_set_memory_cxsr(dev_priv, false);
 			dev_priv->display.funcs.wm = &nop_funcs;
-		} else
+		} else {
 			dev_priv->display.funcs.wm = &pnv_wm_funcs;
+		}
 	} else if (DISPLAY_VER(dev_priv) == 4) {
 		dev_priv->display.funcs.wm = &i965_wm_funcs;
 	} else if (DISPLAY_VER(dev_priv) == 3) {
@@ -4983,9 +4045,3 @@ void intel_init_pm(struct drm_i915_private *dev_priv)
 		dev_priv->display.funcs.wm = &nop_funcs;
 	}
 }
-
-void intel_pm_setup(struct drm_i915_private *dev_priv)
-{
-	dev_priv->runtime_pm.suspended = false;
-	atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
-}
diff --git a/drivers/gpu/drm/i915/display/i9xx_wm.h b/drivers/gpu/drm/i915/display/i9xx_wm.h
new file mode 100644
index 000000000000..a7875cbcd05a
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/i9xx_wm.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __I9XX_WM_H__
+#define __I9XX_WM_H__
+
+#include <linux/types.h>
+
+struct drm_i915_private;
+struct intel_crtc_state;
+struct intel_plane_state;
+
+int ilk_wm_max_level(const struct drm_i915_private *i915);
+bool ilk_disable_lp_wm(struct drm_i915_private *i915);
+void ilk_wm_sanitize(struct drm_i915_private *i915);
+bool intel_set_memory_cxsr(struct drm_i915_private *i915, bool enable);
+void i9xx_wm_init(struct drm_i915_private *i915);
+
+#endif /* __I9XX_WM_H__ */
diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c
index fc0eaf40dc94..ad78148e0788 100644
--- a/drivers/gpu/drm/i915/display/icl_dsi.c
+++ b/drivers/gpu/drm/i915/display/icl_dsi.c
@@ -45,6 +45,7 @@
 #include "intel_dsi_vbt.h"
 #include "intel_panel.h"
 #include "intel_vdsc.h"
+#include "intel_vdsc_regs.h"
 #include "skl_scaler.h"
 #include "skl_universal_plane.h"
 
@@ -207,7 +208,7 @@ void icl_dsi_frame_update(struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	u32 tmp, mode_flags;
+	u32 mode_flags;
 	enum port port;
 
 	mode_flags = crtc_state->mode_flags;
@@ -224,9 +225,7 @@ void icl_dsi_frame_update(struct intel_crtc_state *crtc_state)
 	else
 		return;
 
-	tmp = intel_de_read(dev_priv, DSI_CMD_FRMCTL(port));
-	tmp |= DSI_FRAME_UPDATE_REQUEST;
-	intel_de_write(dev_priv, DSI_CMD_FRMCTL(port), tmp);
+	intel_de_rmw(dev_priv, DSI_CMD_FRMCTL(port), 0, DSI_FRAME_UPDATE_REQUEST);
 }
 
 static void dsi_program_swing_and_deemphasis(struct intel_encoder *encoder)
@@ -234,7 +233,7 @@ static void dsi_program_swing_and_deemphasis(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum phy phy;
-	u32 tmp;
+	u32 tmp, mask, val;
 	int lane;
 
 	for_each_dsi_phy(phy, intel_dsi->phys) {
@@ -242,56 +241,35 @@ static void dsi_program_swing_and_deemphasis(struct intel_encoder *encoder)
 		 * Program voltage swing and pre-emphasis level values as per
 		 * table in BSPEC under DDI buffer programing
 		 */
+		mask = SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK;
+		val = SCALING_MODE_SEL(0x2) | TAP2_DISABLE | TAP3_DISABLE |
+		      RTERM_SELECT(0x6);
 		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN(0, phy));
-		tmp &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK);
-		tmp |= SCALING_MODE_SEL(0x2);
-		tmp |= TAP2_DISABLE | TAP3_DISABLE;
-		tmp |= RTERM_SELECT(0x6);
+		tmp &= ~mask;
+		tmp |= val;
 		intel_de_write(dev_priv, ICL_PORT_TX_DW5_GRP(phy), tmp);
+		intel_de_rmw(dev_priv, ICL_PORT_TX_DW5_AUX(phy), mask, val);
 
-		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_AUX(phy));
-		tmp &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK);
-		tmp |= SCALING_MODE_SEL(0x2);
-		tmp |= TAP2_DISABLE | TAP3_DISABLE;
-		tmp |= RTERM_SELECT(0x6);
-		intel_de_write(dev_priv, ICL_PORT_TX_DW5_AUX(phy), tmp);
-
+		mask = SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK |
+		       RCOMP_SCALAR_MASK;
+		val = SWING_SEL_UPPER(0x2) | SWING_SEL_LOWER(0x2) |
+		      RCOMP_SCALAR(0x98);
 		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW2_LN(0, phy));
-		tmp &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK |
-			 RCOMP_SCALAR_MASK);
-		tmp |= SWING_SEL_UPPER(0x2);
-		tmp |= SWING_SEL_LOWER(0x2);
-		tmp |= RCOMP_SCALAR(0x98);
+		tmp &= ~mask;
+		tmp |= val;
 		intel_de_write(dev_priv, ICL_PORT_TX_DW2_GRP(phy), tmp);
+		intel_de_rmw(dev_priv, ICL_PORT_TX_DW2_AUX(phy), mask, val);
 
-		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW2_AUX(phy));
-		tmp &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK |
-			 RCOMP_SCALAR_MASK);
-		tmp |= SWING_SEL_UPPER(0x2);
-		tmp |= SWING_SEL_LOWER(0x2);
-		tmp |= RCOMP_SCALAR(0x98);
-		intel_de_write(dev_priv, ICL_PORT_TX_DW2_AUX(phy), tmp);
-
-		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW4_AUX(phy));
-		tmp &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK |
-			 CURSOR_COEFF_MASK);
-		tmp |= POST_CURSOR_1(0x0);
-		tmp |= POST_CURSOR_2(0x0);
-		tmp |= CURSOR_COEFF(0x3f);
-		intel_de_write(dev_priv, ICL_PORT_TX_DW4_AUX(phy), tmp);
-
-		for (lane = 0; lane <= 3; lane++) {
-			/* Bspec: must not use GRP register for write */
-			tmp = intel_de_read(dev_priv,
-					    ICL_PORT_TX_DW4_LN(lane, phy));
-			tmp &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK |
-				 CURSOR_COEFF_MASK);
-			tmp |= POST_CURSOR_1(0x0);
-			tmp |= POST_CURSOR_2(0x0);
-			tmp |= CURSOR_COEFF(0x3f);
-			intel_de_write(dev_priv,
-				       ICL_PORT_TX_DW4_LN(lane, phy), tmp);
-		}
+		mask = POST_CURSOR_1_MASK | POST_CURSOR_2_MASK |
+		       CURSOR_COEFF_MASK;
+		val = POST_CURSOR_1(0x0) | POST_CURSOR_2(0x0) |
+		      CURSOR_COEFF(0x3f);
+		intel_de_rmw(dev_priv, ICL_PORT_TX_DW4_AUX(phy), mask, val);
+
+		/* Bspec: must not use GRP register for write */
+		for (lane = 0; lane <= 3; lane++)
+			intel_de_rmw(dev_priv, ICL_PORT_TX_DW4_LN(lane, phy),
+				     mask, val);
 	}
 }
 
@@ -322,7 +300,6 @@ static void configure_dual_link_mode(struct intel_encoder *encoder,
 	if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) {
 		const struct drm_display_mode *adjusted_mode =
 					&pipe_config->hw.adjusted_mode;
-		u32 dss_ctl2;
 		u16 hactive = adjusted_mode->crtc_hdisplay;
 		u16 dl_buffer_depth;
 
@@ -335,10 +312,8 @@ static void configure_dual_link_mode(struct intel_encoder *encoder,
 
 		dss_ctl1 &= ~LEFT_DL_BUF_TARGET_DEPTH_MASK;
 		dss_ctl1 |= LEFT_DL_BUF_TARGET_DEPTH(dl_buffer_depth);
-		dss_ctl2 = intel_de_read(dev_priv, dss_ctl2_reg);
-		dss_ctl2 &= ~RIGHT_DL_BUF_TARGET_DEPTH_MASK;
-		dss_ctl2 |= RIGHT_DL_BUF_TARGET_DEPTH(dl_buffer_depth);
-		intel_de_write(dev_priv, dss_ctl2_reg, dss_ctl2);
+		intel_de_rmw(dev_priv, dss_ctl2_reg, RIGHT_DL_BUF_TARGET_DEPTH_MASK,
+			     RIGHT_DL_BUF_TARGET_DEPTH(dl_buffer_depth));
 	} else {
 		/* Interleave */
 		dss_ctl1 |= DUAL_LINK_MODE_INTERLEAVE;
@@ -424,13 +399,10 @@ static void gen11_dsi_enable_io_power(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
-	u32 tmp;
 
-	for_each_dsi_port(port, intel_dsi->ports) {
-		tmp = intel_de_read(dev_priv, ICL_DSI_IO_MODECTL(port));
-		tmp |= COMBO_PHY_MODE_DSI;
-		intel_de_write(dev_priv, ICL_DSI_IO_MODECTL(port), tmp);
-	}
+	for_each_dsi_port(port, intel_dsi->ports)
+		intel_de_rmw(dev_priv, ICL_DSI_IO_MODECTL(port),
+			     0, COMBO_PHY_MODE_DSI);
 
 	get_dsi_io_power_domains(dev_priv, intel_dsi);
 }
@@ -456,26 +428,16 @@ static void gen11_dsi_config_phy_lanes_sequence(struct intel_encoder *encoder)
 
 	/* Step 4b(i) set loadgen select for transmit and aux lanes */
 	for_each_dsi_phy(phy, intel_dsi->phys) {
-		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW4_AUX(phy));
-		tmp &= ~LOADGEN_SELECT;
-		intel_de_write(dev_priv, ICL_PORT_TX_DW4_AUX(phy), tmp);
-		for (lane = 0; lane <= 3; lane++) {
-			tmp = intel_de_read(dev_priv,
-					    ICL_PORT_TX_DW4_LN(lane, phy));
-			tmp &= ~LOADGEN_SELECT;
-			if (lane != 2)
-				tmp |= LOADGEN_SELECT;
-			intel_de_write(dev_priv,
-				       ICL_PORT_TX_DW4_LN(lane, phy), tmp);
-		}
+		intel_de_rmw(dev_priv, ICL_PORT_TX_DW4_AUX(phy), LOADGEN_SELECT, 0);
+		for (lane = 0; lane <= 3; lane++)
+			intel_de_rmw(dev_priv, ICL_PORT_TX_DW4_LN(lane, phy),
+				     LOADGEN_SELECT, lane != 2 ? LOADGEN_SELECT : 0);
 	}
 
 	/* Step 4b(ii) set latency optimization for transmit and aux lanes */
 	for_each_dsi_phy(phy, intel_dsi->phys) {
-		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW2_AUX(phy));
-		tmp &= ~FRC_LATENCY_OPTIM_MASK;
-		tmp |= FRC_LATENCY_OPTIM_VAL(0x5);
-		intel_de_write(dev_priv, ICL_PORT_TX_DW2_AUX(phy), tmp);
+		intel_de_rmw(dev_priv, ICL_PORT_TX_DW2_AUX(phy),
+			     FRC_LATENCY_OPTIM_MASK, FRC_LATENCY_OPTIM_VAL(0x5));
 		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW2_LN(0, phy));
 		tmp &= ~FRC_LATENCY_OPTIM_MASK;
 		tmp |= FRC_LATENCY_OPTIM_VAL(0x5);
@@ -483,12 +445,8 @@ static void gen11_dsi_config_phy_lanes_sequence(struct intel_encoder *encoder)
 
 		/* For EHL, TGL, set latency optimization for PCS_DW1 lanes */
 		if (IS_JSL_EHL(dev_priv) || (DISPLAY_VER(dev_priv) >= 12)) {
-			tmp = intel_de_read(dev_priv,
-					    ICL_PORT_PCS_DW1_AUX(phy));
-			tmp &= ~LATENCY_OPTIM_MASK;
-			tmp |= LATENCY_OPTIM_VAL(0);
-			intel_de_write(dev_priv, ICL_PORT_PCS_DW1_AUX(phy),
-				       tmp);
+			intel_de_rmw(dev_priv, ICL_PORT_PCS_DW1_AUX(phy),
+				     LATENCY_OPTIM_MASK, LATENCY_OPTIM_VAL(0));
 
 			tmp = intel_de_read(dev_priv,
 					    ICL_PORT_PCS_DW1_LN(0, phy));
@@ -513,9 +471,7 @@ static void gen11_dsi_voltage_swing_program_seq(struct intel_encoder *encoder)
 		tmp = intel_de_read(dev_priv, ICL_PORT_PCS_DW1_LN(0, phy));
 		tmp &= ~COMMON_KEEPER_EN;
 		intel_de_write(dev_priv, ICL_PORT_PCS_DW1_GRP(phy), tmp);
-		tmp = intel_de_read(dev_priv, ICL_PORT_PCS_DW1_AUX(phy));
-		tmp &= ~COMMON_KEEPER_EN;
-		intel_de_write(dev_priv, ICL_PORT_PCS_DW1_AUX(phy), tmp);
+		intel_de_rmw(dev_priv, ICL_PORT_PCS_DW1_AUX(phy), COMMON_KEEPER_EN, 0);
 	}
 
 	/*
@@ -523,20 +479,15 @@ static void gen11_dsi_voltage_swing_program_seq(struct intel_encoder *encoder)
 	 * Note: loadgen select program is done
 	 * as part of lane phy sequence configuration
 	 */
-	for_each_dsi_phy(phy, intel_dsi->phys) {
-		tmp = intel_de_read(dev_priv, ICL_PORT_CL_DW5(phy));
-		tmp |= SUS_CLOCK_CONFIG;
-		intel_de_write(dev_priv, ICL_PORT_CL_DW5(phy), tmp);
-	}
+	for_each_dsi_phy(phy, intel_dsi->phys)
+		intel_de_rmw(dev_priv, ICL_PORT_CL_DW5(phy), 0, SUS_CLOCK_CONFIG);
 
 	/* Clear training enable to change swing values */
 	for_each_dsi_phy(phy, intel_dsi->phys) {
 		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN(0, phy));
 		tmp &= ~TX_TRAINING_EN;
 		intel_de_write(dev_priv, ICL_PORT_TX_DW5_GRP(phy), tmp);
-		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_AUX(phy));
-		tmp &= ~TX_TRAINING_EN;
-		intel_de_write(dev_priv, ICL_PORT_TX_DW5_AUX(phy), tmp);
+		intel_de_rmw(dev_priv, ICL_PORT_TX_DW5_AUX(phy), TX_TRAINING_EN, 0);
 	}
 
 	/* Program swing and de-emphasis */
@@ -547,9 +498,7 @@ static void gen11_dsi_voltage_swing_program_seq(struct intel_encoder *encoder)
 		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN(0, phy));
 		tmp |= TX_TRAINING_EN;
 		intel_de_write(dev_priv, ICL_PORT_TX_DW5_GRP(phy), tmp);
-		tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_AUX(phy));
-		tmp |= TX_TRAINING_EN;
-		intel_de_write(dev_priv, ICL_PORT_TX_DW5_AUX(phy), tmp);
+		intel_de_rmw(dev_priv, ICL_PORT_TX_DW5_AUX(phy), 0, TX_TRAINING_EN);
 	}
 }
 
@@ -557,13 +506,10 @@ static void gen11_dsi_enable_ddi_buffer(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
-	u32 tmp;
 	enum port port;
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		tmp = intel_de_read(dev_priv, DDI_BUF_CTL(port));
-		tmp |= DDI_BUF_CTL_ENABLE;
-		intel_de_write(dev_priv, DDI_BUF_CTL(port), tmp);
+		intel_de_rmw(dev_priv, DDI_BUF_CTL(port), 0, DDI_BUF_CTL_ENABLE);
 
 		if (wait_for_us(!(intel_de_read(dev_priv, DDI_BUF_CTL(port)) &
 				  DDI_BUF_IS_IDLE),
@@ -579,17 +525,13 @@ gen11_dsi_setup_dphy_timings(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
-	u32 tmp;
 	enum port port;
 	enum phy phy;
 
 	/* Program T-INIT master registers */
-	for_each_dsi_port(port, intel_dsi->ports) {
-		tmp = intel_de_read(dev_priv, ICL_DSI_T_INIT_MASTER(port));
-		tmp &= ~DSI_T_INIT_MASTER_MASK;
-		tmp |= intel_dsi->init_count;
-		intel_de_write(dev_priv, ICL_DSI_T_INIT_MASTER(port), tmp);
-	}
+	for_each_dsi_port(port, intel_dsi->ports)
+		intel_de_rmw(dev_priv, ICL_DSI_T_INIT_MASTER(port),
+			     DSI_T_INIT_MASTER_MASK, intel_dsi->init_count);
 
 	/* Program DPHY clock lanes timings */
 	for_each_dsi_port(port, intel_dsi->ports) {
@@ -620,31 +562,22 @@ gen11_dsi_setup_dphy_timings(struct intel_encoder *encoder,
 	if (DISPLAY_VER(dev_priv) == 11) {
 		if (afe_clk(encoder, crtc_state) <= 800000) {
 			for_each_dsi_port(port, intel_dsi->ports) {
-				tmp = intel_de_read(dev_priv,
-						    DPHY_TA_TIMING_PARAM(port));
-				tmp &= ~TA_SURE_MASK;
-				tmp |= TA_SURE_OVERRIDE | TA_SURE(0);
-				intel_de_write(dev_priv,
-					       DPHY_TA_TIMING_PARAM(port),
-					       tmp);
+				intel_de_rmw(dev_priv, DPHY_TA_TIMING_PARAM(port),
+					     TA_SURE_MASK,
+					     TA_SURE_OVERRIDE | TA_SURE(0));
 
 				/* shadow register inside display core */
-				tmp = intel_de_read(dev_priv,
-						    DSI_TA_TIMING_PARAM(port));
-				tmp &= ~TA_SURE_MASK;
-				tmp |= TA_SURE_OVERRIDE | TA_SURE(0);
-				intel_de_write(dev_priv,
-					       DSI_TA_TIMING_PARAM(port), tmp);
+				intel_de_rmw(dev_priv, DSI_TA_TIMING_PARAM(port),
+					     TA_SURE_MASK,
+					     TA_SURE_OVERRIDE | TA_SURE(0));
 			}
 		}
 	}
 
 	if (IS_JSL_EHL(dev_priv)) {
-		for_each_dsi_phy(phy, intel_dsi->phys) {
-			tmp = intel_de_read(dev_priv, ICL_DPHY_CHKN(phy));
-			tmp |= ICL_DPHY_CHKN_AFE_OVER_PPI_STRAP;
-			intel_de_write(dev_priv, ICL_DPHY_CHKN(phy), tmp);
-		}
+		for_each_dsi_phy(phy, intel_dsi->phys)
+			intel_de_rmw(dev_priv, ICL_DPHY_CHKN(phy),
+				     0, ICL_DPHY_CHKN_AFE_OVER_PPI_STRAP);
 	}
 }
 
@@ -836,11 +769,8 @@ gen11_dsi_configure_transcoder(struct intel_encoder *encoder,
 	if (intel_dsi->dual_link) {
 		for_each_dsi_port(port, intel_dsi->ports) {
 			dsi_trans = dsi_port_to_transcoder(port);
-			tmp = intel_de_read(dev_priv,
-					    TRANS_DDI_FUNC_CTL2(dsi_trans));
-			tmp |= PORT_SYNC_MODE_ENABLE;
-			intel_de_write(dev_priv,
-				       TRANS_DDI_FUNC_CTL2(dsi_trans), tmp);
+			intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL2(dsi_trans),
+				     0, PORT_SYNC_MODE_ENABLE);
 		}
 
 		/* configure stream splitting */
@@ -970,8 +900,8 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder,
 	/* program TRANS_HTOTAL register */
 	for_each_dsi_port(port, intel_dsi->ports) {
 		dsi_trans = dsi_port_to_transcoder(port);
-		intel_de_write(dev_priv, HTOTAL(dsi_trans),
-			       (hactive - 1) | ((htotal - 1) << 16));
+		intel_de_write(dev_priv, TRANS_HTOTAL(dsi_trans),
+			       HACTIVE(hactive - 1) | HTOTAL(htotal - 1));
 	}
 
 	/* TRANS_HSYNC register to be programmed only for video mode */
@@ -993,8 +923,8 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder,
 
 		for_each_dsi_port(port, intel_dsi->ports) {
 			dsi_trans = dsi_port_to_transcoder(port);
-			intel_de_write(dev_priv, HSYNC(dsi_trans),
-				       (hsync_start - 1) | ((hsync_end - 1) << 16));
+			intel_de_write(dev_priv, TRANS_HSYNC(dsi_trans),
+				       HSYNC_START(hsync_start - 1) | HSYNC_END(hsync_end - 1));
 		}
 	}
 
@@ -1007,8 +937,8 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder,
 		 * struct drm_display_mode.
 		 * For interlace mode: program required pixel minus 2
 		 */
-		intel_de_write(dev_priv, VTOTAL(dsi_trans),
-			       (vactive - 1) | ((vtotal - 1) << 16));
+		intel_de_write(dev_priv, TRANS_VTOTAL(dsi_trans),
+			       VACTIVE(vactive - 1) | VTOTAL(vtotal - 1));
 	}
 
 	if (vsync_end < vsync_start || vsync_end > vtotal)
@@ -1021,8 +951,8 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder,
 	if (is_vid_mode(intel_dsi)) {
 		for_each_dsi_port(port, intel_dsi->ports) {
 			dsi_trans = dsi_port_to_transcoder(port);
-			intel_de_write(dev_priv, VSYNC(dsi_trans),
-				       (vsync_start - 1) | ((vsync_end - 1) << 16));
+			intel_de_write(dev_priv, TRANS_VSYNC(dsi_trans),
+				       VSYNC_START(vsync_start - 1) | VSYNC_END(vsync_end - 1));
 		}
 	}
 
@@ -1035,17 +965,22 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder,
 	if (is_vid_mode(intel_dsi)) {
 		for_each_dsi_port(port, intel_dsi->ports) {
 			dsi_trans = dsi_port_to_transcoder(port);
-			intel_de_write(dev_priv, VSYNCSHIFT(dsi_trans),
+			intel_de_write(dev_priv, TRANS_VSYNCSHIFT(dsi_trans),
 				       vsync_shift);
 		}
 	}
 
-	/* program TRANS_VBLANK register, should be same as vtotal programmed */
+	/*
+	 * program TRANS_VBLANK register, should be same as vtotal programmed
+	 *
+	 * FIXME get rid of these local hacks and do it right,
+	 * this will not handle eg. delayed vblank correctly.
+	 */
 	if (DISPLAY_VER(dev_priv) >= 12) {
 		for_each_dsi_port(port, intel_dsi->ports) {
 			dsi_trans = dsi_port_to_transcoder(port);
-			intel_de_write(dev_priv, VBLANK(dsi_trans),
-				       (vactive - 1) | ((vtotal - 1) << 16));
+			intel_de_write(dev_priv, TRANS_VBLANK(dsi_trans),
+				       VBLANK_START(vactive - 1) | VBLANK_END(vtotal - 1));
 		}
 	}
 }
@@ -1056,17 +991,14 @@ static void gen11_dsi_enable_transcoder(struct intel_encoder *encoder)
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 	enum transcoder dsi_trans;
-	u32 tmp;
 
 	for_each_dsi_port(port, intel_dsi->ports) {
 		dsi_trans = dsi_port_to_transcoder(port);
-		tmp = intel_de_read(dev_priv, PIPECONF(dsi_trans));
-		tmp |= PIPECONF_ENABLE;
-		intel_de_write(dev_priv, PIPECONF(dsi_trans), tmp);
+		intel_de_rmw(dev_priv, TRANSCONF(dsi_trans), 0, TRANSCONF_ENABLE);
 
 		/* wait for transcoder to be enabled */
-		if (intel_de_wait_for_set(dev_priv, PIPECONF(dsi_trans),
-					  PIPECONF_STATE_ENABLE, 10))
+		if (intel_de_wait_for_set(dev_priv, TRANSCONF(dsi_trans),
+					  TRANSCONF_STATE_ENABLE, 10))
 			drm_err(&dev_priv->drm,
 				"DSI transcoder not enabled\n");
 	}
@@ -1079,7 +1011,7 @@ static void gen11_dsi_setup_timeouts(struct intel_encoder *encoder,
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 	enum transcoder dsi_trans;
-	u32 tmp, hs_tx_timeout, lp_rx_timeout, ta_timeout, divisor, mul;
+	u32 hs_tx_timeout, lp_rx_timeout, ta_timeout, divisor, mul;
 
 	/*
 	 * escape clock count calculation:
@@ -1099,26 +1031,23 @@ static void gen11_dsi_setup_timeouts(struct intel_encoder *encoder,
 		dsi_trans = dsi_port_to_transcoder(port);
 
 		/* program hst_tx_timeout */
-		tmp = intel_de_read(dev_priv, DSI_HSTX_TO(dsi_trans));
-		tmp &= ~HSTX_TIMEOUT_VALUE_MASK;
-		tmp |= HSTX_TIMEOUT_VALUE(hs_tx_timeout);
-		intel_de_write(dev_priv, DSI_HSTX_TO(dsi_trans), tmp);
+		intel_de_rmw(dev_priv, DSI_HSTX_TO(dsi_trans),
+			     HSTX_TIMEOUT_VALUE_MASK,
+			     HSTX_TIMEOUT_VALUE(hs_tx_timeout));
 
 		/* FIXME: DSI_CALIB_TO */
 
 		/* program lp_rx_host timeout */
-		tmp = intel_de_read(dev_priv, DSI_LPRX_HOST_TO(dsi_trans));
-		tmp &= ~LPRX_TIMEOUT_VALUE_MASK;
-		tmp |= LPRX_TIMEOUT_VALUE(lp_rx_timeout);
-		intel_de_write(dev_priv, DSI_LPRX_HOST_TO(dsi_trans), tmp);
+		intel_de_rmw(dev_priv, DSI_LPRX_HOST_TO(dsi_trans),
+			     LPRX_TIMEOUT_VALUE_MASK,
+			     LPRX_TIMEOUT_VALUE(lp_rx_timeout));
 
 		/* FIXME: DSI_PWAIT_TO */
 
 		/* program turn around timeout */
-		tmp = intel_de_read(dev_priv, DSI_TA_TO(dsi_trans));
-		tmp &= ~TA_TIMEOUT_VALUE_MASK;
-		tmp |= TA_TIMEOUT_VALUE(ta_timeout);
-		intel_de_write(dev_priv, DSI_TA_TO(dsi_trans), tmp);
+		intel_de_rmw(dev_priv, DSI_TA_TO(dsi_trans),
+			     TA_TIMEOUT_VALUE_MASK,
+			     TA_TIMEOUT_VALUE(ta_timeout));
 	}
 }
 
@@ -1322,19 +1251,16 @@ static void gen11_dsi_disable_transcoder(struct intel_encoder *encoder)
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 	enum transcoder dsi_trans;
-	u32 tmp;
 
 	for_each_dsi_port(port, intel_dsi->ports) {
 		dsi_trans = dsi_port_to_transcoder(port);
 
 		/* disable transcoder */
-		tmp = intel_de_read(dev_priv, PIPECONF(dsi_trans));
-		tmp &= ~PIPECONF_ENABLE;
-		intel_de_write(dev_priv, PIPECONF(dsi_trans), tmp);
+		intel_de_rmw(dev_priv, TRANSCONF(dsi_trans), TRANSCONF_ENABLE, 0);
 
 		/* wait for transcoder to be disabled */
-		if (intel_de_wait_for_clear(dev_priv, PIPECONF(dsi_trans),
-					    PIPECONF_STATE_ENABLE, 50))
+		if (intel_de_wait_for_clear(dev_priv, TRANSCONF(dsi_trans),
+					    TRANSCONF_STATE_ENABLE, 50))
 			drm_err(&dev_priv->drm,
 				"DSI trancoder not disabled\n");
 	}
@@ -1362,11 +1288,9 @@ static void gen11_dsi_deconfigure_trancoder(struct intel_encoder *encoder)
 
 	/* disable periodic update mode */
 	if (is_cmd_mode(intel_dsi)) {
-		for_each_dsi_port(port, intel_dsi->ports) {
-			tmp = intel_de_read(dev_priv, DSI_CMD_FRMCTL(port));
-			tmp &= ~DSI_PERIODIC_FRAME_UPDATE_ENABLE;
-			intel_de_write(dev_priv, DSI_CMD_FRMCTL(port), tmp);
-		}
+		for_each_dsi_port(port, intel_dsi->ports)
+			intel_de_rmw(dev_priv, DSI_CMD_FRMCTL(port),
+				     DSI_PERIODIC_FRAME_UPDATE_ENABLE, 0);
 	}
 
 	/* put dsi link in ULPS */
@@ -1386,20 +1310,16 @@ static void gen11_dsi_deconfigure_trancoder(struct intel_encoder *encoder)
 	/* disable ddi function */
 	for_each_dsi_port(port, intel_dsi->ports) {
 		dsi_trans = dsi_port_to_transcoder(port);
-		tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(dsi_trans));
-		tmp &= ~TRANS_DDI_FUNC_ENABLE;
-		intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(dsi_trans), tmp);
+		intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(dsi_trans),
+			     TRANS_DDI_FUNC_ENABLE, 0);
 	}
 
 	/* disable port sync mode if dual link */
 	if (intel_dsi->dual_link) {
 		for_each_dsi_port(port, intel_dsi->ports) {
 			dsi_trans = dsi_port_to_transcoder(port);
-			tmp = intel_de_read(dev_priv,
-					    TRANS_DDI_FUNC_CTL2(dsi_trans));
-			tmp &= ~PORT_SYNC_MODE_ENABLE;
-			intel_de_write(dev_priv,
-				       TRANS_DDI_FUNC_CTL2(dsi_trans), tmp);
+			intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL2(dsi_trans),
+				     PORT_SYNC_MODE_ENABLE, 0);
 		}
 	}
 }
@@ -1408,14 +1328,11 @@ static void gen11_dsi_disable_port(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
-	u32 tmp;
 	enum port port;
 
 	gen11_dsi_ungate_clocks(encoder);
 	for_each_dsi_port(port, intel_dsi->ports) {
-		tmp = intel_de_read(dev_priv, DDI_BUF_CTL(port));
-		tmp &= ~DDI_BUF_CTL_ENABLE;
-		intel_de_write(dev_priv, DDI_BUF_CTL(port), tmp);
+		intel_de_rmw(dev_priv, DDI_BUF_CTL(port), DDI_BUF_CTL_ENABLE, 0);
 
 		if (wait_for_us((intel_de_read(dev_priv, DDI_BUF_CTL(port)) &
 				 DDI_BUF_IS_IDLE),
@@ -1432,7 +1349,6 @@ static void gen11_dsi_disable_io_power(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
-	u32 tmp;
 
 	for_each_dsi_port(port, intel_dsi->ports) {
 		intel_wakeref_t wakeref;
@@ -1446,11 +1362,9 @@ static void gen11_dsi_disable_io_power(struct intel_encoder *encoder)
 	}
 
 	/* set mode to DDI */
-	for_each_dsi_port(port, intel_dsi->ports) {
-		tmp = intel_de_read(dev_priv, ICL_DSI_IO_MODECTL(port));
-		tmp &= ~COMBO_PHY_MODE_DSI;
-		intel_de_write(dev_priv, ICL_DSI_IO_MODECTL(port), tmp);
-	}
+	for_each_dsi_port(port, intel_dsi->ports)
+		intel_de_rmw(dev_priv, ICL_DSI_IO_MODECTL(port),
+			     COMBO_PHY_MODE_DSI, 0);
 }
 
 static void gen11_dsi_disable(struct intel_atomic_state *state,
@@ -1586,7 +1500,7 @@ static void gen11_dsi_get_config(struct intel_encoder *encoder,
 
 	gen11_dsi_get_timings(encoder, pipe_config);
 	pipe_config->output_types |= BIT(INTEL_OUTPUT_DSI);
-	pipe_config->pipe_bpp = bdw_get_pipemisc_bpp(crtc);
+	pipe_config->pipe_bpp = bdw_get_pipe_misc_bpp(crtc);
 
 	/* Get the details on which TE should be enabled */
 	if (is_cmd_mode(intel_dsi))
@@ -1638,8 +1552,6 @@ static int gen11_dsi_dsc_compute_config(struct intel_encoder *encoder,
 	if (crtc_state->dsc.slice_count > 1)
 		crtc_state->dsc.dsc_split = true;
 
-	vdsc_cfg->convert_rgb = true;
-
 	/* FIXME: initialize from VBT */
 	vdsc_cfg->rc_model_size = DSC_RC_MODEL_SIZE_CONST;
 
@@ -1766,8 +1678,8 @@ static bool gen11_dsi_get_hw_state(struct intel_encoder *encoder,
 			goto out;
 		}
 
-		tmp = intel_de_read(dev_priv, PIPECONF(dsi_trans));
-		ret = tmp & PIPECONF_ENABLE;
+		tmp = intel_de_read(dev_priv, TRANSCONF(dsi_trans));
+		ret = tmp & TRANSCONF_ENABLE;
 	}
 out:
 	intel_display_power_put(dev_priv, encoder->power_domain, wakeref);
diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.c b/drivers/gpu/drm/i915/display/intel_atomic_plane.c
index 1409bcfb6fd3..40de9f0f171b 100644
--- a/drivers/gpu/drm/i915/display/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.c
@@ -32,18 +32,17 @@
  */
 
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_blend.h>
 #include <drm/drm_fourcc.h>
 
-#include "gt/intel_rps.h"
-
 #include "i915_config.h"
 #include "intel_atomic_plane.h"
 #include "intel_cdclk.h"
+#include "intel_display_rps.h"
 #include "intel_display_trace.h"
 #include "intel_display_types.h"
 #include "intel_fb.h"
 #include "intel_fb_pin.h"
-#include "intel_sprite.h"
 #include "skl_scaler.h"
 #include "skl_watermark.h"
 
@@ -363,6 +362,7 @@ void intel_plane_set_invisible(struct intel_crtc_state *crtc_state,
 	crtc_state->scaled_planes &= ~BIT(plane->id);
 	crtc_state->nv12_planes &= ~BIT(plane->id);
 	crtc_state->c8_planes &= ~BIT(plane->id);
+	crtc_state->async_flip_planes &= ~BIT(plane->id);
 	crtc_state->data_rate[plane->id] = 0;
 	crtc_state->data_rate_y[plane->id] = 0;
 	crtc_state->rel_data_rate[plane->id] = 0;
@@ -582,8 +582,10 @@ static int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_cr
 			 intel_plane_is_scaled(new_plane_state))))
 		new_crtc_state->disable_lp_wm = true;
 
-	if (intel_plane_do_async_flip(plane, old_crtc_state, new_crtc_state))
+	if (intel_plane_do_async_flip(plane, old_crtc_state, new_crtc_state)) {
 		new_crtc_state->do_async_flip = true;
+		new_crtc_state->async_flip_planes |= BIT(plane->id);
+	}
 
 	return 0;
 }
@@ -938,62 +940,62 @@ int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state,
 	return 0;
 }
 
-struct wait_rps_boost {
-	struct wait_queue_entry wait;
-
-	struct drm_crtc *crtc;
-	struct i915_request *request;
-};
-
-static int do_rps_boost(struct wait_queue_entry *_wait,
-			unsigned mode, int sync, void *key)
+int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state)
 {
-	struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
-	struct i915_request *rq = wait->request;
+	struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
+	const struct drm_framebuffer *fb = plane_state->hw.fb;
+	struct drm_rect *src = &plane_state->uapi.src;
+	u32 src_x, src_y, src_w, src_h, hsub, vsub;
+	bool rotated = drm_rotation_90_or_270(plane_state->hw.rotation);
 
 	/*
-	 * If we missed the vblank, but the request is already running it
-	 * is reasonable to assume that it will complete before the next
-	 * vblank without our intervention, so leave RPS alone.
+	 * FIXME hsub/vsub vs. block size is a mess. Pre-tgl CCS
+	 * abuses hsub/vsub so we can't use them here. But as they
+	 * are limited to 32bpp RGB formats we don't actually need
+	 * to check anything.
 	 */
-	if (!i915_request_started(rq))
-		intel_rps_boost(rq);
-	i915_request_put(rq);
-
-	drm_crtc_vblank_put(wait->crtc);
-
-	list_del(&wait->wait.entry);
-	kfree(wait);
-	return 1;
-}
+	if (fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+	    fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS)
+		return 0;
 
-static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
-				       struct dma_fence *fence)
-{
-	struct wait_rps_boost *wait;
+	/*
+	 * Hardware doesn't handle subpixel coordinates.
+	 * Adjust to (macro)pixel boundary, but be careful not to
+	 * increase the source viewport size, because that could
+	 * push the downscaling factor out of bounds.
+	 */
+	src_x = src->x1 >> 16;
+	src_w = drm_rect_width(src) >> 16;
+	src_y = src->y1 >> 16;
+	src_h = drm_rect_height(src) >> 16;
 
-	if (!dma_fence_is_i915(fence))
-		return;
+	drm_rect_init(src, src_x << 16, src_y << 16,
+		      src_w << 16, src_h << 16);
 
-	if (DISPLAY_VER(to_i915(crtc->dev)) < 6)
-		return;
+	if (fb->format->format == DRM_FORMAT_RGB565 && rotated) {
+		hsub = 2;
+		vsub = 2;
+	} else {
+		hsub = fb->format->hsub;
+		vsub = fb->format->vsub;
+	}
 
-	if (drm_crtc_vblank_get(crtc))
-		return;
+	if (rotated)
+		hsub = vsub = max(hsub, vsub);
 
-	wait = kmalloc(sizeof(*wait), GFP_KERNEL);
-	if (!wait) {
-		drm_crtc_vblank_put(crtc);
-		return;
+	if (src_x % hsub || src_w % hsub) {
+		drm_dbg_kms(&i915->drm, "src x/w (%u, %u) must be a multiple of %u (rotated: %s)\n",
+			    src_x, src_w, hsub, str_yes_no(rotated));
+		return -EINVAL;
 	}
 
-	wait->request = to_request(dma_fence_get(fence));
-	wait->crtc = crtc;
-
-	wait->wait.func = do_rps_boost;
-	wait->wait.flags = 0;
+	if (src_y % vsub || src_h % vsub) {
+		drm_dbg_kms(&i915->drm, "src y/h (%u, %u) must be a multiple of %u (rotated: %s)\n",
+			    src_y, src_h, vsub, str_yes_no(rotated));
+		return -EINVAL;
+	}
 
-	add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
+	return 0;
 }
 
 /**
@@ -1086,13 +1088,13 @@ intel_prepare_plane_fb(struct drm_plane *_plane,
 		dma_resv_iter_begin(&cursor, obj->base.resv,
 				    DMA_RESV_USAGE_WRITE);
 		dma_resv_for_each_fence_unlocked(&cursor, fence) {
-			add_rps_boost_after_vblank(new_plane_state->hw.crtc,
-						   fence);
+			intel_display_rps_boost_after_vblank(new_plane_state->hw.crtc,
+							     fence);
 		}
 		dma_resv_iter_end(&cursor);
 	} else {
-		add_rps_boost_after_vblank(new_plane_state->hw.crtc,
-					   new_plane_state->uapi.fence);
+		intel_display_rps_boost_after_vblank(new_plane_state->hw.crtc,
+						     new_plane_state->uapi.fence);
 	}
 
 	/*
@@ -1103,10 +1105,7 @@ intel_prepare_plane_fb(struct drm_plane *_plane,
 	 * that are not quite steady state without resorting to forcing
 	 * maximum clocks following a vblank miss (see do_rps_boost()).
 	 */
-	if (!state->rps_interactive) {
-		intel_rps_mark_interactive(&to_gt(dev_priv)->rps, true);
-		state->rps_interactive = true;
-	}
+	intel_display_rps_mark_interactive(dev_priv, state, true);
 
 	return 0;
 
@@ -1137,10 +1136,7 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
 	if (!obj)
 		return;
 
-	if (state->rps_interactive) {
-		intel_rps_mark_interactive(&to_gt(dev_priv)->rps, false);
-		state->rps_interactive = false;
-	}
+	intel_display_rps_mark_interactive(dev_priv, state, false);
 
 	/* Should only be called after a successful intel_prepare_plane_fb()! */
 	intel_plane_unpin_fb(old_plane_state);
diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.h b/drivers/gpu/drm/i915/display/intel_atomic_plane.h
index 74b6d3b169a7..191dad0efc8e 100644
--- a/drivers/gpu/drm/i915/display/intel_atomic_plane.h
+++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.h
@@ -62,6 +62,7 @@ int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state,
 				      struct intel_crtc_state *crtc_state,
 				      int min_scale, int max_scale,
 				      bool can_position);
+int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state);
 void intel_plane_set_invisible(struct intel_crtc_state *crtc_state,
 			       struct intel_plane_state *plane_state);
 void intel_plane_helper_add(struct intel_plane *plane);
diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c
index a9335c856644..3d5a9bbc6fde 100644
--- a/drivers/gpu/drm/i915/display/intel_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_audio.c
@@ -581,8 +581,7 @@ static void enable_audio_dsc_wa(struct intel_encoder *encoder,
 				const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	enum pipe pipe = crtc->pipe;
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	unsigned int hblank_early_prog, samples_room;
 	unsigned int val;
 
@@ -592,32 +591,32 @@ static void enable_audio_dsc_wa(struct intel_encoder *encoder,
 	val = intel_de_read(i915, AUD_CONFIG_BE);
 
 	if (DISPLAY_VER(i915) == 11)
-		val |= HBLANK_EARLY_ENABLE_ICL(pipe);
+		val |= HBLANK_EARLY_ENABLE_ICL(cpu_transcoder);
 	else if (DISPLAY_VER(i915) >= 12)
-		val |= HBLANK_EARLY_ENABLE_TGL(pipe);
+		val |= HBLANK_EARLY_ENABLE_TGL(cpu_transcoder);
 
 	if (crtc_state->dsc.compression_enable &&
 	    crtc_state->hw.adjusted_mode.hdisplay >= 3840 &&
 	    crtc_state->hw.adjusted_mode.vdisplay >= 2160) {
 		/* Get hblank early enable value required */
-		val &= ~HBLANK_START_COUNT_MASK(pipe);
+		val &= ~HBLANK_START_COUNT_MASK(cpu_transcoder);
 		hblank_early_prog = calc_hblank_early_prog(encoder, crtc_state);
 		if (hblank_early_prog < 32)
-			val |= HBLANK_START_COUNT(pipe, HBLANK_START_COUNT_32);
+			val |= HBLANK_START_COUNT(cpu_transcoder, HBLANK_START_COUNT_32);
 		else if (hblank_early_prog < 64)
-			val |= HBLANK_START_COUNT(pipe, HBLANK_START_COUNT_64);
+			val |= HBLANK_START_COUNT(cpu_transcoder, HBLANK_START_COUNT_64);
 		else if (hblank_early_prog < 96)
-			val |= HBLANK_START_COUNT(pipe, HBLANK_START_COUNT_96);
+			val |= HBLANK_START_COUNT(cpu_transcoder, HBLANK_START_COUNT_96);
 		else
-			val |= HBLANK_START_COUNT(pipe, HBLANK_START_COUNT_128);
+			val |= HBLANK_START_COUNT(cpu_transcoder, HBLANK_START_COUNT_128);
 
 		/* Get samples room value required */
-		val &= ~NUMBER_SAMPLES_PER_LINE_MASK(pipe);
+		val &= ~NUMBER_SAMPLES_PER_LINE_MASK(cpu_transcoder);
 		samples_room = calc_samples_room(crtc_state);
 		if (samples_room < 3)
-			val |= NUMBER_SAMPLES_PER_LINE(pipe, samples_room);
+			val |= NUMBER_SAMPLES_PER_LINE(cpu_transcoder, samples_room);
 		else /* Program 0 i.e "All Samples available in buffer" */
-			val |= NUMBER_SAMPLES_PER_LINE(pipe, 0x0);
+			val |= NUMBER_SAMPLES_PER_LINE(cpu_transcoder, 0x0);
 	}
 
 	intel_de_write(i915, AUD_CONFIG_BE, val);
@@ -812,9 +811,9 @@ void intel_audio_codec_enable(struct intel_encoder *encoder,
 	struct i915_audio_component *acomp = i915->display.audio.component;
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	struct intel_audio_state *audio_state;
 	enum port port = encoder->port;
-	enum pipe pipe = crtc->pipe;
 
 	if (!crtc_state->has_audio)
 		return;
@@ -832,7 +831,7 @@ void intel_audio_codec_enable(struct intel_encoder *encoder,
 
 	mutex_lock(&i915->display.audio.mutex);
 
-	audio_state = &i915->display.audio.state[pipe];
+	audio_state = &i915->display.audio.state[cpu_transcoder];
 
 	audio_state->encoder = encoder;
 	BUILD_BUG_ON(sizeof(audio_state->eld) != sizeof(crtc_state->eld));
@@ -842,14 +841,14 @@ void intel_audio_codec_enable(struct intel_encoder *encoder,
 
 	if (acomp && acomp->base.audio_ops &&
 	    acomp->base.audio_ops->pin_eld_notify) {
-		/* audio drivers expect pipe = -1 to indicate Non-MST cases */
+		/* audio drivers expect cpu_transcoder = -1 to indicate Non-MST cases */
 		if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST))
-			pipe = -1;
+			cpu_transcoder = -1;
 		acomp->base.audio_ops->pin_eld_notify(acomp->base.audio_ops->audio_ptr,
-						      (int)port, (int)pipe);
+						      (int)port, (int)cpu_transcoder);
 	}
 
-	intel_lpe_audio_notify(i915, pipe, port, crtc_state->eld,
+	intel_lpe_audio_notify(i915, cpu_transcoder, port, crtc_state->eld,
 			       crtc_state->port_clock,
 			       intel_crtc_has_dp_encoder(crtc_state));
 }
@@ -871,9 +870,9 @@ void intel_audio_codec_disable(struct intel_encoder *encoder,
 	struct i915_audio_component *acomp = i915->display.audio.component;
 	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
 	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
+	enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 	struct intel_audio_state *audio_state;
 	enum port port = encoder->port;
-	enum pipe pipe = crtc->pipe;
 
 	if (!old_crtc_state->has_audio)
 		return;
@@ -890,7 +889,7 @@ void intel_audio_codec_disable(struct intel_encoder *encoder,
 
 	mutex_lock(&i915->display.audio.mutex);
 
-	audio_state = &i915->display.audio.state[pipe];
+	audio_state = &i915->display.audio.state[cpu_transcoder];
 
 	audio_state->encoder = NULL;
 	memset(audio_state->eld, 0, sizeof(audio_state->eld));
@@ -899,27 +898,26 @@ void intel_audio_codec_disable(struct intel_encoder *encoder,
 
 	if (acomp && acomp->base.audio_ops &&
 	    acomp->base.audio_ops->pin_eld_notify) {
-		/* audio drivers expect pipe = -1 to indicate Non-MST cases */
+		/* audio drivers expect cpu_transcoder = -1 to indicate Non-MST cases */
 		if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
-			pipe = -1;
+			cpu_transcoder = -1;
 		acomp->base.audio_ops->pin_eld_notify(acomp->base.audio_ops->audio_ptr,
-						      (int)port, (int)pipe);
+						      (int)port, (int)cpu_transcoder);
 	}
 
-	intel_lpe_audio_notify(i915, pipe, port, NULL, 0, false);
+	intel_lpe_audio_notify(i915, cpu_transcoder, port, NULL, 0, false);
 }
 
 static void intel_acomp_get_config(struct intel_encoder *encoder,
 				   struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	struct intel_audio_state *audio_state;
-	enum pipe pipe = crtc->pipe;
 
 	mutex_lock(&i915->display.audio.mutex);
 
-	audio_state = &i915->display.audio.state[pipe];
+	audio_state = &i915->display.audio.state[cpu_transcoder];
 
 	if (audio_state->encoder)
 		memcpy(crtc_state->eld, audio_state->eld, sizeof(audio_state->eld));
@@ -985,11 +983,7 @@ void intel_audio_cdclk_change_pre(struct drm_i915_private *i915)
 
 static void get_aud_ts_cdclk_m_n(int refclk, int cdclk, struct aud_ts_cdclk_m_n *aud_ts)
 {
-	if (refclk == 24000)
-		aud_ts->m = 12;
-	else
-		aud_ts->m = 15;
-
+	aud_ts->m = 60;
 	aud_ts->n = cdclk * aud_ts->m / 24000;
 }
 
@@ -1147,27 +1141,27 @@ static int i915_audio_component_get_cdclk_freq(struct device *kdev)
 }
 
 /*
- * get the intel audio state according to the parameter port and pipe
- * MST & (pipe >= 0): return the audio.state[pipe].encoder],
+ * get the intel audio state according to the parameter port and cpu_transcoder
+ * MST & (cpu_transcoder >= 0): return the audio.state[cpu_transcoder].encoder],
  *   when port is matched
- * MST & (pipe < 0): this is invalid
- * Non-MST & (pipe >= 0): only pipe = 0 (the first device entry)
+ * MST & (cpu_transcoder < 0): this is invalid
+ * Non-MST & (cpu_transcoder >= 0): only cpu_transcoder = 0 (the first device entry)
  *   will get the right intel_encoder with port matched
- * Non-MST & (pipe < 0): get the right intel_encoder with port matched
+ * Non-MST & (cpu_transcoder < 0): get the right intel_encoder with port matched
  */
 static struct intel_audio_state *find_audio_state(struct drm_i915_private *i915,
-						  int port, int pipe)
+						  int port, int cpu_transcoder)
 {
 	/* MST */
-	if (pipe >= 0) {
+	if (cpu_transcoder >= 0) {
 		struct intel_audio_state *audio_state;
 		struct intel_encoder *encoder;
 
 		if (drm_WARN_ON(&i915->drm,
-				pipe >= ARRAY_SIZE(i915->display.audio.state)))
+				cpu_transcoder >= ARRAY_SIZE(i915->display.audio.state)))
 			return NULL;
 
-		audio_state = &i915->display.audio.state[pipe];
+		audio_state = &i915->display.audio.state[cpu_transcoder];
 		encoder = audio_state->encoder;
 
 		if (encoder && encoder->port == port &&
@@ -1176,14 +1170,14 @@ static struct intel_audio_state *find_audio_state(struct drm_i915_private *i915,
 	}
 
 	/* Non-MST */
-	if (pipe > 0)
+	if (cpu_transcoder > 0)
 		return NULL;
 
-	for_each_pipe(i915, pipe) {
+	for_each_cpu_transcoder(i915, cpu_transcoder) {
 		struct intel_audio_state *audio_state;
 		struct intel_encoder *encoder;
 
-		audio_state = &i915->display.audio.state[pipe];
+		audio_state = &i915->display.audio.state[cpu_transcoder];
 		encoder = audio_state->encoder;
 
 		if (encoder && encoder->port == port &&
@@ -1195,7 +1189,7 @@ static struct intel_audio_state *find_audio_state(struct drm_i915_private *i915,
 }
 
 static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
-						int pipe, int rate)
+						int cpu_transcoder, int rate)
 {
 	struct drm_i915_private *i915 = kdev_to_i915(kdev);
 	struct i915_audio_component *acomp = i915->display.audio.component;
@@ -1211,7 +1205,7 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
 	cookie = i915_audio_component_get_power(kdev);
 	mutex_lock(&i915->display.audio.mutex);
 
-	audio_state = find_audio_state(i915, port, pipe);
+	audio_state = find_audio_state(i915, port, cpu_transcoder);
 	if (!audio_state) {
 		drm_dbg_kms(&i915->drm, "Not valid for port %c\n", port_name(port));
 		err = -ENODEV;
@@ -1223,7 +1217,7 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
 	/* FIXME stop using the legacy crtc pointer */
 	crtc = to_intel_crtc(encoder->base.crtc);
 
-	/* port must be valid now, otherwise the pipe will be invalid */
+	/* port must be valid now, otherwise the cpu_transcoder will be invalid */
 	acomp->aud_sample_rate[port] = rate;
 
 	/* FIXME get rid of the crtc->config stuff */
@@ -1236,7 +1230,7 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
 }
 
 static int i915_audio_component_get_eld(struct device *kdev, int port,
-					int pipe, bool *enabled,
+					int cpu_transcoder, bool *enabled,
 					unsigned char *buf, int max_bytes)
 {
 	struct drm_i915_private *i915 = kdev_to_i915(kdev);
@@ -1245,7 +1239,7 @@ static int i915_audio_component_get_eld(struct device *kdev, int port,
 
 	mutex_lock(&i915->display.audio.mutex);
 
-	audio_state = find_audio_state(i915, port, pipe);
+	audio_state = find_audio_state(i915, port, cpu_transcoder);
 	if (!audio_state) {
 		drm_dbg_kms(&i915->drm, "Not valid for port %c\n", port_name(port));
 		mutex_unlock(&i915->display.audio.mutex);
diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c
index a4e4b7f79e4d..2e8f17c04522 100644
--- a/drivers/gpu/drm/i915/display/intel_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_backlight.c
@@ -105,7 +105,8 @@ void intel_backlight_set_pwm_level(const struct drm_connector_state *conn_state,
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 
-	drm_dbg_kms(&i915->drm, "set backlight PWM = %d\n", val);
+	drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] set backlight PWM = %d\n",
+		    connector->base.base.id, connector->base.name, val);
 	panel->backlight.pwm_funcs->set(conn_state, val);
 }
 
@@ -283,7 +284,8 @@ intel_panel_actually_set_backlight(const struct drm_connector_state *conn_state,
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 
-	drm_dbg_kms(&i915->drm, "set backlight level = %d\n", level);
+	drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] set backlight level = %d\n",
+		    connector->base.base.id, connector->base.name, level);
 
 	panel->backlight.funcs->set(conn_state, level);
 }
@@ -345,27 +347,24 @@ static void lpt_disable_backlight(const struct drm_connector_state *old_conn_sta
 	 */
 	tmp = intel_de_read(i915, BLC_PWM_CPU_CTL2);
 	if (tmp & BLM_PWM_ENABLE) {
-		drm_dbg_kms(&i915->drm, "cpu backlight was enabled, disabling\n");
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] CPU backlight was enabled, disabling\n",
+			    connector->base.base.id, connector->base.name);
 		intel_de_write(i915, BLC_PWM_CPU_CTL2, tmp & ~BLM_PWM_ENABLE);
 	}
 
-	tmp = intel_de_read(i915, BLC_PWM_PCH_CTL1);
-	intel_de_write(i915, BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE);
+	intel_de_rmw(i915, BLC_PWM_PCH_CTL1, BLM_PCH_PWM_ENABLE, 0);
 }
 
 static void pch_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val)
 {
 	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
-	u32 tmp;
 
 	intel_backlight_set_pwm_level(old_conn_state, val);
 
-	tmp = intel_de_read(i915, BLC_PWM_CPU_CTL2);
-	intel_de_write(i915, BLC_PWM_CPU_CTL2, tmp & ~BLM_PWM_ENABLE);
+	intel_de_rmw(i915, BLC_PWM_CPU_CTL2, BLM_PWM_ENABLE, 0);
 
-	tmp = intel_de_read(i915, BLC_PWM_PCH_CTL1);
-	intel_de_write(i915, BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE);
+	intel_de_rmw(i915, BLC_PWM_PCH_CTL1, BLM_PCH_PWM_ENABLE, 0);
 }
 
 static void i9xx_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val)
@@ -376,12 +375,10 @@ static void i9xx_disable_backlight(const struct drm_connector_state *old_conn_st
 static void i965_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val)
 {
 	struct drm_i915_private *i915 = to_i915(old_conn_state->connector->dev);
-	u32 tmp;
 
 	intel_backlight_set_pwm_level(old_conn_state, val);
 
-	tmp = intel_de_read(i915, BLC_PWM_CTL2);
-	intel_de_write(i915, BLC_PWM_CTL2, tmp & ~BLM_PWM_ENABLE);
+	intel_de_rmw(i915, BLC_PWM_CTL2, BLM_PWM_ENABLE, 0);
 }
 
 static void vlv_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val)
@@ -389,12 +386,10 @@ static void vlv_disable_backlight(const struct drm_connector_state *old_conn_sta
 	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	enum pipe pipe = to_intel_crtc(old_conn_state->crtc)->pipe;
-	u32 tmp;
 
 	intel_backlight_set_pwm_level(old_conn_state, val);
 
-	tmp = intel_de_read(i915, VLV_BLC_PWM_CTL2(pipe));
-	intel_de_write(i915, VLV_BLC_PWM_CTL2(pipe), tmp & ~BLM_PWM_ENABLE);
+	intel_de_rmw(i915, VLV_BLC_PWM_CTL2(pipe), BLM_PWM_ENABLE, 0);
 }
 
 static void bxt_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val)
@@ -402,19 +397,14 @@ static void bxt_disable_backlight(const struct drm_connector_state *old_conn_sta
 	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	u32 tmp;
 
 	intel_backlight_set_pwm_level(old_conn_state, val);
 
-	tmp = intel_de_read(i915, BXT_BLC_PWM_CTL(panel->backlight.controller));
-	intel_de_write(i915, BXT_BLC_PWM_CTL(panel->backlight.controller),
-		       tmp & ~BXT_BLC_PWM_ENABLE);
+	intel_de_rmw(i915, BXT_BLC_PWM_CTL(panel->backlight.controller),
+		     BXT_BLC_PWM_ENABLE, 0);
 
-	if (panel->backlight.controller == 1) {
-		val = intel_de_read(i915, UTIL_PIN_CTL);
-		val &= ~UTIL_PIN_ENABLE;
-		intel_de_write(i915, UTIL_PIN_CTL, val);
-	}
+	if (panel->backlight.controller == 1)
+		intel_de_rmw(i915, UTIL_PIN_CTL, UTIL_PIN_ENABLE, 0);
 }
 
 static void cnp_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val)
@@ -422,13 +412,11 @@ static void cnp_disable_backlight(const struct drm_connector_state *old_conn_sta
 	struct intel_connector *connector = to_intel_connector(old_conn_state->connector);
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	u32 tmp;
 
 	intel_backlight_set_pwm_level(old_conn_state, val);
 
-	tmp = intel_de_read(i915, BXT_BLC_PWM_CTL(panel->backlight.controller));
-	intel_de_write(i915, BXT_BLC_PWM_CTL(panel->backlight.controller),
-		       tmp & ~BXT_BLC_PWM_ENABLE);
+	intel_de_rmw(i915, BXT_BLC_PWM_CTL(panel->backlight.controller),
+		     BXT_BLC_PWM_ENABLE, 0);
 }
 
 static void ext_pwm_disable_backlight(const struct drm_connector_state *old_conn_state, u32 level)
@@ -458,7 +446,8 @@ void intel_backlight_disable(const struct drm_connector_state *old_conn_state)
 	 * another client is not activated.
 	 */
 	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_CHANGING) {
-		drm_dbg_kms(&i915->drm, "Skipping backlight disable on vga switch\n");
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] Skipping backlight disable on vga switch\n",
+			    connector->base.base.id, connector->base.name);
 		return;
 	}
 
@@ -478,30 +467,24 @@ static void lpt_enable_backlight(const struct intel_crtc_state *crtc_state,
 	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	u32 pch_ctl1, pch_ctl2, schicken;
+	u32 pch_ctl1, pch_ctl2;
 
 	pch_ctl1 = intel_de_read(i915, BLC_PWM_PCH_CTL1);
 	if (pch_ctl1 & BLM_PCH_PWM_ENABLE) {
-		drm_dbg_kms(&i915->drm, "pch backlight already enabled\n");
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] PCH backlight already enabled\n",
+			    connector->base.base.id, connector->base.name);
 		pch_ctl1 &= ~BLM_PCH_PWM_ENABLE;
 		intel_de_write(i915, BLC_PWM_PCH_CTL1, pch_ctl1);
 	}
 
-	if (HAS_PCH_LPT(i915)) {
-		schicken = intel_de_read(i915, SOUTH_CHICKEN2);
-		if (panel->backlight.alternate_pwm_increment)
-			schicken |= LPT_PWM_GRANULARITY;
-		else
-			schicken &= ~LPT_PWM_GRANULARITY;
-		intel_de_write(i915, SOUTH_CHICKEN2, schicken);
-	} else {
-		schicken = intel_de_read(i915, SOUTH_CHICKEN1);
-		if (panel->backlight.alternate_pwm_increment)
-			schicken |= SPT_PWM_GRANULARITY;
-		else
-			schicken &= ~SPT_PWM_GRANULARITY;
-		intel_de_write(i915, SOUTH_CHICKEN1, schicken);
-	}
+	if (HAS_PCH_LPT(i915))
+		intel_de_rmw(i915, SOUTH_CHICKEN2, LPT_PWM_GRANULARITY,
+			     panel->backlight.alternate_pwm_increment ?
+			     LPT_PWM_GRANULARITY : 0);
+	else
+		intel_de_rmw(i915, SOUTH_CHICKEN1, SPT_PWM_GRANULARITY,
+			     panel->backlight.alternate_pwm_increment ?
+			     SPT_PWM_GRANULARITY : 0);
 
 	pch_ctl2 = panel->backlight.pwm_level_max << 16;
 	intel_de_write(i915, BLC_PWM_PCH_CTL2, pch_ctl2);
@@ -533,14 +516,16 @@ static void pch_enable_backlight(const struct intel_crtc_state *crtc_state,
 
 	cpu_ctl2 = intel_de_read(i915, BLC_PWM_CPU_CTL2);
 	if (cpu_ctl2 & BLM_PWM_ENABLE) {
-		drm_dbg_kms(&i915->drm, "cpu backlight already enabled\n");
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] CPU backlight already enabled\n",
+			    connector->base.base.id, connector->base.name);
 		cpu_ctl2 &= ~BLM_PWM_ENABLE;
 		intel_de_write(i915, BLC_PWM_CPU_CTL2, cpu_ctl2);
 	}
 
 	pch_ctl1 = intel_de_read(i915, BLC_PWM_PCH_CTL1);
 	if (pch_ctl1 & BLM_PCH_PWM_ENABLE) {
-		drm_dbg_kms(&i915->drm, "pch backlight already enabled\n");
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] PCH backlight already enabled\n",
+			    connector->base.base.id, connector->base.name);
 		pch_ctl1 &= ~BLM_PCH_PWM_ENABLE;
 		intel_de_write(i915, BLC_PWM_PCH_CTL1, pch_ctl1);
 	}
@@ -578,7 +563,8 @@ static void i9xx_enable_backlight(const struct intel_crtc_state *crtc_state,
 
 	ctl = intel_de_read(i915, BLC_PWM_CTL);
 	if (ctl & BACKLIGHT_DUTY_CYCLE_MASK_PNV) {
-		drm_dbg_kms(&i915->drm, "backlight already enabled\n");
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] backlight already enabled\n",
+			    connector->base.base.id, connector->base.name);
 		intel_de_write(i915, BLC_PWM_CTL, 0);
 	}
 
@@ -618,7 +604,8 @@ static void i965_enable_backlight(const struct intel_crtc_state *crtc_state,
 
 	ctl2 = intel_de_read(i915, BLC_PWM_CTL2);
 	if (ctl2 & BLM_PWM_ENABLE) {
-		drm_dbg_kms(&i915->drm, "backlight already enabled\n");
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] backlight already enabled\n",
+			    connector->base.base.id, connector->base.name);
 		ctl2 &= ~BLM_PWM_ENABLE;
 		intel_de_write(i915, BLC_PWM_CTL2, ctl2);
 	}
@@ -653,7 +640,8 @@ static void vlv_enable_backlight(const struct intel_crtc_state *crtc_state,
 
 	ctl2 = intel_de_read(i915, VLV_BLC_PWM_CTL2(pipe));
 	if (ctl2 & BLM_PWM_ENABLE) {
-		drm_dbg_kms(&i915->drm, "backlight already enabled\n");
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] backlight already enabled\n",
+			    connector->base.base.id, connector->base.name);
 		ctl2 &= ~BLM_PWM_ENABLE;
 		intel_de_write(i915, VLV_BLC_PWM_CTL2(pipe), ctl2);
 	}
@@ -685,7 +673,8 @@ static void bxt_enable_backlight(const struct intel_crtc_state *crtc_state,
 	if (panel->backlight.controller == 1) {
 		val = intel_de_read(i915, UTIL_PIN_CTL);
 		if (val & UTIL_PIN_ENABLE) {
-			drm_dbg_kms(&i915->drm, "util pin already enabled\n");
+			drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] utility pin already enabled\n",
+				    connector->base.base.id, connector->base.name);
 			val &= ~UTIL_PIN_ENABLE;
 			intel_de_write(i915, UTIL_PIN_CTL, val);
 		}
@@ -699,7 +688,8 @@ static void bxt_enable_backlight(const struct intel_crtc_state *crtc_state,
 
 	pwm_ctl = intel_de_read(i915, BXT_BLC_PWM_CTL(panel->backlight.controller));
 	if (pwm_ctl & BXT_BLC_PWM_ENABLE) {
-		drm_dbg_kms(&i915->drm, "backlight already enabled\n");
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] backlight already enabled\n",
+			    connector->base.base.id, connector->base.name);
 		pwm_ctl &= ~BXT_BLC_PWM_ENABLE;
 		intel_de_write(i915, BXT_BLC_PWM_CTL(panel->backlight.controller),
 			       pwm_ctl);
@@ -1270,6 +1260,10 @@ static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unus
 			       cpu_ctl2 & ~BLM_PWM_ENABLE);
 	}
 
+	drm_dbg_kms(&i915->drm,
+		    "[CONNECTOR:%d:%s] Using native PCH PWM for backlight control\n",
+		    connector->base.base.id, connector->base.name);
+
 	return 0;
 }
 
@@ -1297,6 +1291,10 @@ static int pch_setup_backlight(struct intel_connector *connector, enum pipe unus
 	panel->backlight.pwm_enabled = (cpu_ctl2 & BLM_PWM_ENABLE) &&
 		(pch_ctl1 & BLM_PCH_PWM_ENABLE);
 
+	drm_dbg_kms(&i915->drm,
+		    "[CONNECTOR:%d:%s] Using native PCH PWM for backlight control\n",
+		    connector->base.base.id, connector->base.name);
+
 	return 0;
 }
 
@@ -1335,6 +1333,10 @@ static int i9xx_setup_backlight(struct intel_connector *connector, enum pipe unu
 
 	panel->backlight.pwm_enabled = val != 0;
 
+	drm_dbg_kms(&i915->drm,
+		    "[CONNECTOR:%d:%s] Using native PWM for backlight control\n",
+		    connector->base.base.id, connector->base.name);
+
 	return 0;
 }
 
@@ -1364,6 +1366,10 @@ static int i965_setup_backlight(struct intel_connector *connector, enum pipe unu
 
 	panel->backlight.pwm_enabled = ctl2 & BLM_PWM_ENABLE;
 
+	drm_dbg_kms(&i915->drm,
+		    "[CONNECTOR:%d:%s] Using native PWM for backlight control\n",
+		    connector->base.base.id, connector->base.name);
+
 	return 0;
 }
 
@@ -1392,6 +1398,10 @@ static int vlv_setup_backlight(struct intel_connector *connector, enum pipe pipe
 
 	panel->backlight.pwm_enabled = ctl2 & BLM_PWM_ENABLE;
 
+	drm_dbg_kms(&i915->drm,
+		    "[CONNECTOR:%d:%s] Using native PWM for backlight control (on pipe %c)\n",
+		    connector->base.base.id, connector->base.name, pipe_name(pipe));
+
 	return 0;
 }
 
@@ -1428,6 +1438,11 @@ bxt_setup_backlight(struct intel_connector *connector, enum pipe unused)
 
 	panel->backlight.pwm_enabled = pwm_ctl & BXT_BLC_PWM_ENABLE;
 
+	drm_dbg_kms(&i915->drm,
+		    "[CONNECTOR:%d:%s] Using native PWM for backlight control (controller=%d)\n",
+		    connector->base.base.id, connector->base.name,
+		    panel->backlight.controller);
+
 	return 0;
 }
 
@@ -1468,7 +1483,8 @@ cnp_setup_backlight(struct intel_connector *connector, enum pipe unused)
 	 */
 	panel->backlight.controller = connector->panel.vbt.backlight.controller;
 	if (!cnp_backlight_controller_is_valid(i915, panel->backlight.controller)) {
-		drm_dbg_kms(&i915->drm, "Invalid backlight controller %d, assuming 0\n",
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] Invalid backlight controller %d, assuming 0\n",
+			    connector->base.base.id, connector->base.name,
 			    panel->backlight.controller);
 		panel->backlight.controller = 0;
 	}
@@ -1490,6 +1506,11 @@ cnp_setup_backlight(struct intel_connector *connector, enum pipe unused)
 
 	panel->backlight.pwm_enabled = pwm_ctl & BXT_BLC_PWM_ENABLE;
 
+	drm_dbg_kms(&i915->drm,
+		    "[CONNECTOR:%d:%s] Using native PCH PWM for backlight control (controller=%d)\n",
+		    connector->base.base.id, connector->base.name,
+		    panel->backlight.controller);
+
 	return 0;
 }
 
@@ -1511,8 +1532,8 @@ static int ext_pwm_setup_backlight(struct intel_connector *connector,
 	}
 
 	if (IS_ERR(panel->backlight.pwm)) {
-		drm_err(&i915->drm, "Failed to get the %s PWM chip\n",
-			desc);
+		drm_err(&i915->drm, "[CONNECTOR:%d:%s] Failed to get the %s PWM chip\n",
+			connector->base.base.id, connector->base.name, desc);
 		panel->backlight.pwm = NULL;
 		return -ENODEV;
 	}
@@ -1529,7 +1550,8 @@ static int ext_pwm_setup_backlight(struct intel_connector *connector,
 		level = intel_backlight_invert_pwm_level(connector, level);
 		panel->backlight.pwm_enabled = true;
 
-		drm_dbg_kms(&i915->drm, "PWM already enabled at freq %ld, VBT freq %d, level %d\n",
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] PWM already enabled at freq %ld, VBT freq %d, level %d\n",
+			    connector->base.base.id, connector->base.name,
 			    NSEC_PER_SEC / (unsigned long)panel->backlight.pwm_state.period,
 			    get_vbt_pwm_freq(connector), level);
 	} else {
@@ -1538,8 +1560,10 @@ static int ext_pwm_setup_backlight(struct intel_connector *connector,
 			NSEC_PER_SEC / get_vbt_pwm_freq(connector);
 	}
 
-	drm_info(&i915->drm, "Using %s PWM for LCD backlight control\n",
-		 desc);
+	drm_dbg_kms(&i915->drm,
+		    "[CONNECTOR:%d:%s] Using %s PWM for backlight control\n",
+		    connector->base.base.id, connector->base.name, desc);
+
 	return 0;
 }
 
@@ -1582,8 +1606,9 @@ static void intel_pwm_disable_backlight(const struct drm_connector_state *conn_s
 static int intel_pwm_setup_backlight(struct intel_connector *connector, enum pipe pipe)
 {
 	struct intel_panel *panel = &connector->panel;
-	int ret = panel->backlight.pwm_funcs->setup(connector, pipe);
+	int ret;
 
+	ret = panel->backlight.pwm_funcs->setup(connector, pipe);
 	if (ret < 0)
 		return ret;
 
@@ -1623,10 +1648,12 @@ int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe)
 	if (!connector->panel.vbt.backlight.present) {
 		if (intel_has_quirk(i915, QUIRK_BACKLIGHT_PRESENT)) {
 			drm_dbg_kms(&i915->drm,
-				    "no backlight present per VBT, but present per quirk\n");
+				    "[CONNECTOR:%d:%s] no backlight present per VBT, but present per quirk\n",
+				    connector->base.base.id, connector->base.name);
 		} else {
 			drm_dbg_kms(&i915->drm,
-				    "no backlight present per VBT\n");
+				    "[CONNECTOR:%d:%s] no backlight present per VBT\n",
+				    connector->base.base.id, connector->base.name);
 			return 0;
 		}
 	}
@@ -1642,16 +1669,16 @@ int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe)
 
 	if (ret) {
 		drm_dbg_kms(&i915->drm,
-			    "failed to setup backlight for connector %s\n",
-			    connector->base.name);
+			    "[CONNECTOR:%d:%s] failed to setup backlight\n",
+			    connector->base.base.id, connector->base.name);
 		return ret;
 	}
 
 	panel->backlight.present = true;
 
 	drm_dbg_kms(&i915->drm,
-		    "Connector %s backlight initialized, %s, brightness %u/%u\n",
-		    connector->base.name,
+		    "[CONNECTOR:%d:%s] backlight initialized, %s, brightness %u/%u\n",
+		    connector->base.base.id, connector->base.name,
 		    str_enabled_disabled(panel->backlight.enabled),
 		    panel->backlight.level, panel->backlight.max);
 
diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c
index 04b846440de6..75e69dffc5e9 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.c
+++ b/drivers/gpu/drm/i915/display/intel_bios.c
@@ -141,8 +141,8 @@ struct bdb_block_entry {
 };
 
 static const void *
-find_section(struct drm_i915_private *i915,
-	     enum bdb_block_id section_id)
+bdb_find_section(struct drm_i915_private *i915,
+		 enum bdb_block_id section_id)
 {
 	struct bdb_block_entry *entry;
 
@@ -201,7 +201,7 @@ static size_t lfp_data_min_size(struct drm_i915_private *i915)
 	const struct bdb_lvds_lfp_data_ptrs *ptrs;
 	size_t size;
 
-	ptrs = find_section(i915, BDB_LVDS_LFP_DATA_PTRS);
+	ptrs = bdb_find_section(i915, BDB_LVDS_LFP_DATA_PTRS);
 	if (!ptrs)
 		return 0;
 
@@ -630,7 +630,7 @@ static int vbt_get_panel_type(struct drm_i915_private *i915,
 {
 	const struct bdb_lvds_options *lvds_options;
 
-	lvds_options = find_section(i915, BDB_LVDS_OPTIONS);
+	lvds_options = bdb_find_section(i915, BDB_LVDS_OPTIONS);
 	if (!lvds_options)
 		return -1;
 
@@ -671,11 +671,11 @@ static int pnpid_get_panel_type(struct drm_i915_private *i915,
 
 	dump_pnp_id(i915, edid_id, "EDID");
 
-	ptrs = find_section(i915, BDB_LVDS_LFP_DATA_PTRS);
+	ptrs = bdb_find_section(i915, BDB_LVDS_LFP_DATA_PTRS);
 	if (!ptrs)
 		return -1;
 
-	data = find_section(i915, BDB_LVDS_LFP_DATA);
+	data = bdb_find_section(i915, BDB_LVDS_LFP_DATA);
 	if (!data)
 		return -1;
 
@@ -791,7 +791,7 @@ parse_panel_options(struct drm_i915_private *i915,
 	int panel_type = panel->vbt.panel_type;
 	int drrs_mode;
 
-	lvds_options = find_section(i915, BDB_LVDS_OPTIONS);
+	lvds_options = bdb_find_section(i915, BDB_LVDS_OPTIONS);
 	if (!lvds_options)
 		return;
 
@@ -881,11 +881,11 @@ parse_lfp_data(struct drm_i915_private *i915,
 	const struct lvds_pnp_id *pnp_id;
 	int panel_type = panel->vbt.panel_type;
 
-	ptrs = find_section(i915, BDB_LVDS_LFP_DATA_PTRS);
+	ptrs = bdb_find_section(i915, BDB_LVDS_LFP_DATA_PTRS);
 	if (!ptrs)
 		return;
 
-	data = find_section(i915, BDB_LVDS_LFP_DATA);
+	data = bdb_find_section(i915, BDB_LVDS_LFP_DATA);
 	if (!data)
 		return;
 
@@ -932,7 +932,7 @@ parse_generic_dtd(struct drm_i915_private *i915,
 	if (i915->display.vbt.version < 229)
 		return;
 
-	generic_dtd = find_section(i915, BDB_GENERIC_DTD);
+	generic_dtd = bdb_find_section(i915, BDB_GENERIC_DTD);
 	if (!generic_dtd)
 		return;
 
@@ -1011,7 +1011,7 @@ parse_lfp_backlight(struct drm_i915_private *i915,
 	int panel_type = panel->vbt.panel_type;
 	u16 level;
 
-	backlight_data = find_section(i915, BDB_LVDS_BACKLIGHT);
+	backlight_data = bdb_find_section(i915, BDB_LVDS_BACKLIGHT);
 	if (!backlight_data)
 		return;
 
@@ -1084,6 +1084,12 @@ parse_lfp_backlight(struct drm_i915_private *i915,
 		panel->vbt.backlight.min_brightness = entry->min_brightness;
 	}
 
+	if (i915->display.vbt.version >= 239)
+		panel->vbt.backlight.hdr_dpcd_refresh_timeout =
+			DIV_ROUND_UP(backlight_data->hdr_dpcd_refresh_timeout[panel_type], 100);
+	else
+		panel->vbt.backlight.hdr_dpcd_refresh_timeout = 30;
+
 	drm_dbg_kms(&i915->drm,
 		    "VBT backlight PWM modulation frequency %u Hz, "
 		    "active %s, min brightness %u, level %u, controller %u\n",
@@ -1113,14 +1119,14 @@ parse_sdvo_panel_data(struct drm_i915_private *i915,
 	if (index == -1) {
 		const struct bdb_sdvo_lvds_options *sdvo_lvds_options;
 
-		sdvo_lvds_options = find_section(i915, BDB_SDVO_LVDS_OPTIONS);
+		sdvo_lvds_options = bdb_find_section(i915, BDB_SDVO_LVDS_OPTIONS);
 		if (!sdvo_lvds_options)
 			return;
 
 		index = sdvo_lvds_options->panel_type;
 	}
 
-	dtds = find_section(i915, BDB_SDVO_PANEL_DTDS);
+	dtds = bdb_find_section(i915, BDB_SDVO_PANEL_DTDS);
 	if (!dtds)
 		return;
 
@@ -1156,7 +1162,7 @@ parse_general_features(struct drm_i915_private *i915)
 {
 	const struct bdb_general_features *general;
 
-	general = find_section(i915, BDB_GENERAL_FEATURES);
+	general = bdb_find_section(i915, BDB_GENERAL_FEATURES);
 	if (!general)
 		return;
 
@@ -1202,9 +1208,7 @@ child_device_ptr(const struct bdb_general_definitions *defs, int i)
 static void
 parse_sdvo_device_mapping(struct drm_i915_private *i915)
 {
-	struct sdvo_device_mapping *mapping;
 	const struct intel_bios_encoder_data *devdata;
-	const struct child_device_config *child;
 	int count = 0;
 
 	/*
@@ -1217,7 +1221,8 @@ parse_sdvo_device_mapping(struct drm_i915_private *i915)
 	}
 
 	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
-		child = &devdata->child;
+		const struct child_device_config *child = &devdata->child;
+		struct sdvo_device_mapping *mapping;
 
 		if (child->slave_addr != SLAVE_ADDR1 &&
 		    child->slave_addr != SLAVE_ADDR2) {
@@ -1280,7 +1285,7 @@ parse_driver_features(struct drm_i915_private *i915)
 {
 	const struct bdb_driver_features *driver;
 
-	driver = find_section(i915, BDB_DRIVER_FEATURES);
+	driver = bdb_find_section(i915, BDB_DRIVER_FEATURES);
 	if (!driver)
 		return;
 
@@ -1317,7 +1322,7 @@ parse_panel_driver_features(struct drm_i915_private *i915,
 {
 	const struct bdb_driver_features *driver;
 
-	driver = find_section(i915, BDB_DRIVER_FEATURES);
+	driver = bdb_find_section(i915, BDB_DRIVER_FEATURES);
 	if (!driver)
 		return;
 
@@ -1357,7 +1362,7 @@ parse_power_conservation_features(struct drm_i915_private *i915,
 	if (i915->display.vbt.version < 228)
 		return;
 
-	power = find_section(i915, BDB_LFP_POWER);
+	power = bdb_find_section(i915, BDB_LFP_POWER);
 	if (!power)
 		return;
 
@@ -1397,7 +1402,7 @@ parse_edp(struct drm_i915_private *i915,
 	const struct edp_fast_link_params *edp_link_params;
 	int panel_type = panel->vbt.panel_type;
 
-	edp = find_section(i915, BDB_EDP);
+	edp = bdb_find_section(i915, BDB_EDP);
 	if (!edp)
 		return;
 
@@ -1527,7 +1532,7 @@ parse_psr(struct drm_i915_private *i915,
 	const struct psr_table *psr_table;
 	int panel_type = panel->vbt.panel_type;
 
-	psr = find_section(i915, BDB_PSR);
+	psr = bdb_find_section(i915, BDB_PSR);
 	if (!psr) {
 		drm_dbg_kms(&i915->drm, "No PSR BDB found.\n");
 		return;
@@ -1688,7 +1693,7 @@ parse_mipi_config(struct drm_i915_private *i915,
 	/* Parse #52 for panel index used from panel_type already
 	 * parsed
 	 */
-	start = find_section(i915, BDB_MIPI_CONFIG);
+	start = bdb_find_section(i915, BDB_MIPI_CONFIG);
 	if (!start) {
 		drm_dbg_kms(&i915->drm, "No MIPI config BDB found");
 		return;
@@ -2000,7 +2005,7 @@ parse_mipi_sequence(struct drm_i915_private *i915,
 	if (panel->vbt.dsi.panel_id != MIPI_DSI_GENERIC_PANEL_ID)
 		return;
 
-	sequence = find_section(i915, BDB_MIPI_SEQUENCE);
+	sequence = bdb_find_section(i915, BDB_MIPI_SEQUENCE);
 	if (!sequence) {
 		drm_dbg_kms(&i915->drm,
 			    "No MIPI Sequence found, parsing complete\n");
@@ -2075,14 +2080,13 @@ parse_compression_parameters(struct drm_i915_private *i915)
 {
 	const struct bdb_compression_parameters *params;
 	struct intel_bios_encoder_data *devdata;
-	const struct child_device_config *child;
 	u16 block_size;
 	int index;
 
 	if (i915->display.vbt.version < 198)
 		return;
 
-	params = find_section(i915, BDB_COMPRESSION_PARAMETERS);
+	params = bdb_find_section(i915, BDB_COMPRESSION_PARAMETERS);
 	if (params) {
 		/* Sanity checks */
 		if (params->entry_size != sizeof(params->data[0])) {
@@ -2100,7 +2104,7 @@ parse_compression_parameters(struct drm_i915_private *i915)
 	}
 
 	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
-		child = &devdata->child;
+		const struct child_device_config *child = &devdata->child;
 
 		if (!child->compression_enable)
 			continue;
@@ -2226,14 +2230,14 @@ static u8 map_ddc_pin(struct drm_i915_private *i915, u8 vbt_pin)
 
 static enum port get_port_by_ddc_pin(struct drm_i915_private *i915, u8 ddc_pin)
 {
-	const struct intel_bios_encoder_data *devdata;
 	enum port port;
 
 	if (!ddc_pin)
 		return PORT_NONE;
 
 	for_each_port(port) {
-		devdata = i915->display.vbt.ports[port];
+		const struct intel_bios_encoder_data *devdata =
+			i915->display.vbt.ports[port];
 
 		if (devdata && ddc_pin == devdata->child.ddc_pin)
 			return port;
@@ -2292,14 +2296,14 @@ static void sanitize_ddc_pin(struct intel_bios_encoder_data *devdata,
 
 static enum port get_port_by_aux_ch(struct drm_i915_private *i915, u8 aux_ch)
 {
-	const struct intel_bios_encoder_data *devdata;
 	enum port port;
 
 	if (!aux_ch)
 		return PORT_NONE;
 
 	for_each_port(port) {
-		devdata = i915->display.vbt.ports[port];
+		const struct intel_bios_encoder_data *devdata =
+			i915->display.vbt.ports[port];
 
 		if (devdata && aux_ch == devdata->child.aux_channel)
 			return port;
@@ -2522,7 +2526,7 @@ static int parse_bdb_216_dp_max_link_rate(const int vbt_max_link_rate)
 	}
 }
 
-static int _intel_bios_dp_max_link_rate(const struct intel_bios_encoder_data *devdata)
+int intel_bios_dp_max_link_rate(const struct intel_bios_encoder_data *devdata)
 {
 	if (!devdata || devdata->i915->display.vbt.version < 216)
 		return 0;
@@ -2533,7 +2537,7 @@ static int _intel_bios_dp_max_link_rate(const struct intel_bios_encoder_data *de
 		return parse_bdb_216_dp_max_link_rate(devdata->child.dp_max_link_rate);
 }
 
-static int _intel_bios_dp_max_lane_count(const struct intel_bios_encoder_data *devdata)
+int intel_bios_dp_max_lane_count(const struct intel_bios_encoder_data *devdata)
 {
 	if (!devdata || devdata->i915->display.vbt.version < 244)
 		return 0;
@@ -2587,7 +2591,7 @@ intel_bios_encoder_supports_dp(const struct intel_bios_encoder_data *devdata)
 	return devdata->child.device_type & DEVICE_TYPE_DISPLAYPORT_OUTPUT;
 }
 
-static bool
+bool
 intel_bios_encoder_supports_edp(const struct intel_bios_encoder_data *devdata)
 {
 	return intel_bios_encoder_supports_dp(devdata) &&
@@ -2600,7 +2604,14 @@ intel_bios_encoder_supports_dsi(const struct intel_bios_encoder_data *devdata)
 	return devdata->child.device_type & DEVICE_TYPE_MIPI_OUTPUT;
 }
 
-static int _intel_bios_hdmi_level_shift(const struct intel_bios_encoder_data *devdata)
+bool
+intel_bios_encoder_is_lspcon(const struct intel_bios_encoder_data *devdata)
+{
+	return devdata && HAS_LSPCON(devdata->i915) && devdata->child.lspcon;
+}
+
+/* This is an index in the HDMI/DVI DDI buffer translation table, or -1 */
+int intel_bios_hdmi_level_shift(const struct intel_bios_encoder_data *devdata)
 {
 	if (!devdata || devdata->i915->display.vbt.version < 158)
 		return -1;
@@ -2608,7 +2619,7 @@ static int _intel_bios_hdmi_level_shift(const struct intel_bios_encoder_data *de
 	return devdata->child.hdmi_level_shifter_value;
 }
 
-static int _intel_bios_max_tmds_clock(const struct intel_bios_encoder_data *devdata)
+int intel_bios_hdmi_max_tmds_clock(const struct intel_bios_encoder_data *devdata)
 {
 	if (!devdata || devdata->i915->display.vbt.version < 204)
 		return 0;
@@ -2666,37 +2677,37 @@ static void print_ddi_port(const struct intel_bios_encoder_data *devdata,
 	drm_dbg_kms(&i915->drm,
 		    "Port %c VBT info: CRT:%d DVI:%d HDMI:%d DP:%d eDP:%d DSI:%d LSPCON:%d USB-Type-C:%d TBT:%d DSC:%d\n",
 		    port_name(port), is_crt, is_dvi, is_hdmi, is_dp, is_edp, is_dsi,
-		    HAS_LSPCON(i915) && child->lspcon,
+		    intel_bios_encoder_is_lspcon(devdata),
 		    supports_typec_usb, supports_tbt,
 		    devdata->dsc != NULL);
 
-	hdmi_level_shift = _intel_bios_hdmi_level_shift(devdata);
+	hdmi_level_shift = intel_bios_hdmi_level_shift(devdata);
 	if (hdmi_level_shift >= 0) {
 		drm_dbg_kms(&i915->drm,
 			    "Port %c VBT HDMI level shift: %d\n",
 			    port_name(port), hdmi_level_shift);
 	}
 
-	max_tmds_clock = _intel_bios_max_tmds_clock(devdata);
+	max_tmds_clock = intel_bios_hdmi_max_tmds_clock(devdata);
 	if (max_tmds_clock)
 		drm_dbg_kms(&i915->drm,
 			    "Port %c VBT HDMI max TMDS clock: %d kHz\n",
 			    port_name(port), max_tmds_clock);
 
 	/* I_boost config for SKL and above */
-	dp_boost_level = intel_bios_encoder_dp_boost_level(devdata);
+	dp_boost_level = intel_bios_dp_boost_level(devdata);
 	if (dp_boost_level)
 		drm_dbg_kms(&i915->drm,
 			    "Port %c VBT (e)DP boost level: %d\n",
 			    port_name(port), dp_boost_level);
 
-	hdmi_boost_level = intel_bios_encoder_hdmi_boost_level(devdata);
+	hdmi_boost_level = intel_bios_hdmi_boost_level(devdata);
 	if (hdmi_boost_level)
 		drm_dbg_kms(&i915->drm,
 			    "Port %c VBT HDMI boost level: %d\n",
 			    port_name(port), hdmi_boost_level);
 
-	dp_max_link_rate = _intel_bios_dp_max_link_rate(devdata);
+	dp_max_link_rate = intel_bios_dp_max_link_rate(devdata);
 	if (dp_max_link_rate)
 		drm_dbg_kms(&i915->drm,
 			    "Port %c VBT DP max link rate: %d\n",
@@ -2781,7 +2792,7 @@ parse_general_definitions(struct drm_i915_private *i915)
 	u16 block_size;
 	int bus_pin;
 
-	defs = find_section(i915, BDB_GENERAL_DEFINITIONS);
+	defs = bdb_find_section(i915, BDB_GENERAL_DEFINITIONS);
 	if (!defs) {
 		drm_dbg_kms(&i915->drm,
 			    "No general definition block is found, no devices defined.\n");
@@ -2811,7 +2822,7 @@ parse_general_definitions(struct drm_i915_private *i915)
 		expected_size = 37;
 	} else if (i915->display.vbt.version <= 215) {
 		expected_size = 38;
-	} else if (i915->display.vbt.version <= 237) {
+	} else if (i915->display.vbt.version <= 250) {
 		expected_size = 39;
 	} else {
 		expected_size = sizeof(*child);
@@ -3306,7 +3317,6 @@ void intel_bios_fini_panel(struct intel_panel *panel)
 bool intel_bios_is_tv_present(struct drm_i915_private *i915)
 {
 	const struct intel_bios_encoder_data *devdata;
-	const struct child_device_config *child;
 
 	if (!i915->display.vbt.int_tv_support)
 		return false;
@@ -3315,7 +3325,7 @@ bool intel_bios_is_tv_present(struct drm_i915_private *i915)
 		return true;
 
 	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
-		child = &devdata->child;
+		const struct child_device_config *child = &devdata->child;
 
 		/*
 		 * If the device type is not TV, continue.
@@ -3349,13 +3359,12 @@ bool intel_bios_is_tv_present(struct drm_i915_private *i915)
 bool intel_bios_is_lvds_present(struct drm_i915_private *i915, u8 *i2c_pin)
 {
 	const struct intel_bios_encoder_data *devdata;
-	const struct child_device_config *child;
 
 	if (list_empty(&i915->display.vbt.display_devices))
 		return true;
 
 	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
-		child = &devdata->child;
+		const struct child_device_config *child = &devdata->child;
 
 		/* If the device type is not LFP, continue.
 		 * We have to check both the new identifiers as well as the
@@ -3397,25 +3406,22 @@ bool intel_bios_is_lvds_present(struct drm_i915_private *i915, u8 *i2c_pin)
  */
 bool intel_bios_is_port_present(struct drm_i915_private *i915, enum port port)
 {
+	const struct intel_bios_encoder_data *devdata;
+
 	if (WARN_ON(!has_ddi_port_info(i915)))
 		return true;
 
-	return i915->display.vbt.ports[port];
-}
+	if (!is_port_valid(i915, port))
+		return false;
 
-/**
- * intel_bios_is_port_edp - is the device in given port eDP
- * @i915:	i915 device instance
- * @port:	port to check
- *
- * Return true if the device in %port is eDP.
- */
-bool intel_bios_is_port_edp(struct drm_i915_private *i915, enum port port)
-{
-	const struct intel_bios_encoder_data *devdata =
-		intel_bios_encoder_data_lookup(i915, port);
+	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
+		const struct child_device_config *child = &devdata->child;
+
+		if (dvo_port_to_port(i915, child->dvo_port) == port)
+			return true;
+	}
 
-	return devdata && intel_bios_encoder_supports_edp(devdata);
+	return false;
 }
 
 static bool intel_bios_encoder_supports_dp_dual_mode(const struct intel_bios_encoder_data *devdata)
@@ -3457,17 +3463,14 @@ bool intel_bios_is_dsi_present(struct drm_i915_private *i915,
 			       enum port *port)
 {
 	const struct intel_bios_encoder_data *devdata;
-	const struct child_device_config *child;
-	u8 dvo_port;
 
 	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
-		child = &devdata->child;
+		const struct child_device_config *child = &devdata->child;
+		u8 dvo_port = child->dvo_port;
 
 		if (!(child->device_type & DEVICE_TYPE_MIPI_OUTPUT))
 			continue;
 
-		dvo_port = child->dvo_port;
-
 		if (dsi_dvo_port_to_port(i915, dvo_port) == PORT_NONE) {
 			drm_dbg_kms(&i915->drm,
 				    "VBT has unsupported DSI port %c\n",
@@ -3554,10 +3557,9 @@ bool intel_bios_get_dsc_params(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	const struct intel_bios_encoder_data *devdata;
-	const struct child_device_config *child;
 
 	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
-		child = &devdata->child;
+		const struct child_device_config *child = &devdata->child;
 
 		if (!(child->device_type & DEVICE_TYPE_MIPI_OUTPUT))
 			continue;
@@ -3576,73 +3578,10 @@ bool intel_bios_get_dsc_params(struct intel_encoder *encoder,
 	return false;
 }
 
-/**
- * intel_bios_is_port_hpd_inverted - is HPD inverted for %port
- * @i915:	i915 device instance
- * @port:	port to check
- *
- * Return true if HPD should be inverted for %port.
- */
-bool
-intel_bios_is_port_hpd_inverted(const struct drm_i915_private *i915,
-				enum port port)
+static enum aux_ch map_aux_ch(struct drm_i915_private *i915, u8 aux_channel)
 {
-	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port];
-
-	if (drm_WARN_ON_ONCE(&i915->drm,
-			     !IS_GEMINILAKE(i915) && !IS_BROXTON(i915)))
-		return false;
-
-	return devdata && devdata->child.hpd_invert;
-}
-
-/**
- * intel_bios_is_lspcon_present - if LSPCON is attached on %port
- * @i915:	i915 device instance
- * @port:	port to check
- *
- * Return true if LSPCON is present on this port
- */
-bool
-intel_bios_is_lspcon_present(const struct drm_i915_private *i915,
-			     enum port port)
-{
-	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port];
-
-	return HAS_LSPCON(i915) && devdata && devdata->child.lspcon;
-}
-
-/**
- * intel_bios_is_lane_reversal_needed - if lane reversal needed on port
- * @i915:       i915 device instance
- * @port:       port to check
- *
- * Return true if port requires lane reversal
- */
-bool
-intel_bios_is_lane_reversal_needed(const struct drm_i915_private *i915,
-				   enum port port)
-{
-	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port];
-
-	return devdata && devdata->child.lane_reversal;
-}
-
-enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915,
-				   enum port port)
-{
-	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port];
 	enum aux_ch aux_ch;
 
-	if (!devdata || !devdata->child.aux_channel) {
-		aux_ch = (enum aux_ch)port;
-
-		drm_dbg_kms(&i915->drm,
-			    "using AUX %c for port %c (platform default)\n",
-			    aux_ch_name(aux_ch), port_name(port));
-		return aux_ch;
-	}
-
 	/*
 	 * RKL/DG1 VBT uses PHY based mapping. Combo PHYs A,B,C,D
 	 * map to DDI A,B,TC1,TC2 respectively.
@@ -3650,7 +3589,7 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915,
 	 * ADL-S VBT uses PHY based mapping. Combo PHYs A,B,C,D,E
 	 * map to DDI A,TC1,TC2,TC3,TC4 respectively.
 	 */
-	switch (devdata->child.aux_channel) {
+	switch (aux_channel) {
 	case DP_AUX_A:
 		aux_ch = AUX_CH_A;
 		break;
@@ -3711,35 +3650,23 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915,
 			aux_ch = AUX_CH_I;
 		break;
 	default:
-		MISSING_CASE(devdata->child.aux_channel);
+		MISSING_CASE(aux_channel);
 		aux_ch = AUX_CH_A;
 		break;
 	}
 
-	drm_dbg_kms(&i915->drm, "using AUX %c for port %c (VBT)\n",
-		    aux_ch_name(aux_ch), port_name(port));
-
 	return aux_ch;
 }
 
-int intel_bios_max_tmds_clock(struct intel_encoder *encoder)
+enum aux_ch intel_bios_dp_aux_ch(const struct intel_bios_encoder_data *devdata)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port];
+	if (!devdata || !devdata->child.aux_channel)
+		return AUX_CH_NONE;
 
-	return _intel_bios_max_tmds_clock(devdata);
+	return map_aux_ch(devdata->i915, devdata->child.aux_channel);
 }
 
-/* This is an index in the HDMI/DVI DDI buffer translation table, or -1 */
-int intel_bios_hdmi_level_shift(struct intel_encoder *encoder)
-{
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port];
-
-	return _intel_bios_hdmi_level_shift(devdata);
-}
-
-int intel_bios_encoder_dp_boost_level(const struct intel_bios_encoder_data *devdata)
+int intel_bios_dp_boost_level(const struct intel_bios_encoder_data *devdata)
 {
 	if (!devdata || devdata->i915->display.vbt.version < 196 || !devdata->child.iboost)
 		return 0;
@@ -3747,7 +3674,7 @@ int intel_bios_encoder_dp_boost_level(const struct intel_bios_encoder_data *devd
 	return translate_iboost(devdata->child.dp_iboost_level);
 }
 
-int intel_bios_encoder_hdmi_boost_level(const struct intel_bios_encoder_data *devdata)
+int intel_bios_hdmi_boost_level(const struct intel_bios_encoder_data *devdata)
 {
 	if (!devdata || devdata->i915->display.vbt.version < 196 || !devdata->child.iboost)
 		return 0;
@@ -3755,31 +3682,12 @@ int intel_bios_encoder_hdmi_boost_level(const struct intel_bios_encoder_data *de
 	return translate_iboost(devdata->child.hdmi_iboost_level);
 }
 
-int intel_bios_dp_max_link_rate(struct intel_encoder *encoder)
-{
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port];
-
-	return _intel_bios_dp_max_link_rate(devdata);
-}
-
-int intel_bios_dp_max_lane_count(struct intel_encoder *encoder)
-{
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port];
-
-	return _intel_bios_dp_max_lane_count(devdata);
-}
-
-int intel_bios_alternate_ddc_pin(struct intel_encoder *encoder)
+int intel_bios_hdmi_ddc_pin(const struct intel_bios_encoder_data *devdata)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port];
-
 	if (!devdata || !devdata->child.ddc_pin)
 		return 0;
 
-	return map_ddc_pin(i915, devdata->child.ddc_pin);
+	return map_ddc_pin(devdata->i915, devdata->child.ddc_pin);
 }
 
 bool intel_bios_encoder_supports_typec_usb(const struct intel_bios_encoder_data *devdata)
@@ -3792,6 +3700,16 @@ bool intel_bios_encoder_supports_tbt(const struct intel_bios_encoder_data *devda
 	return devdata->i915->display.vbt.version >= 209 && devdata->child.tbt;
 }
 
+bool intel_bios_encoder_lane_reversal(const struct intel_bios_encoder_data *devdata)
+{
+	return devdata && devdata->child.lane_reversal;
+}
+
+bool intel_bios_encoder_hpd_invert(const struct intel_bios_encoder_data *devdata)
+{
+	return devdata && devdata->child.hpd_invert;
+}
+
 const struct intel_bios_encoder_data *
 intel_bios_encoder_data_lookup(struct drm_i915_private *i915, enum port port)
 {
diff --git a/drivers/gpu/drm/i915/display/intel_bios.h b/drivers/gpu/drm/i915/display/intel_bios.h
index d221f784aa88..8a0730c9b48c 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.h
+++ b/drivers/gpu/drm/i915/display/intel_bios.h
@@ -38,6 +38,7 @@ struct intel_bios_encoder_data;
 struct intel_crtc_state;
 struct intel_encoder;
 struct intel_panel;
+enum aux_ch;
 enum port;
 
 enum intel_backlight_type {
@@ -248,21 +249,9 @@ bool intel_bios_is_port_present(struct drm_i915_private *dev_priv, enum port por
 bool intel_bios_is_port_edp(struct drm_i915_private *dev_priv, enum port port);
 bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum port port);
 bool intel_bios_is_dsi_present(struct drm_i915_private *dev_priv, enum port *port);
-bool intel_bios_is_port_hpd_inverted(const struct drm_i915_private *i915,
-				     enum port port);
-bool intel_bios_is_lspcon_present(const struct drm_i915_private *i915,
-				  enum port port);
-bool intel_bios_is_lane_reversal_needed(const struct drm_i915_private *i915,
-					enum port port);
-enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *dev_priv, enum port port);
 bool intel_bios_get_dsc_params(struct intel_encoder *encoder,
 			       struct intel_crtc_state *crtc_state,
 			       int dsc_max_bpc);
-int intel_bios_max_tmds_clock(struct intel_encoder *encoder);
-int intel_bios_hdmi_level_shift(struct intel_encoder *encoder);
-int intel_bios_dp_max_link_rate(struct intel_encoder *encoder);
-int intel_bios_dp_max_lane_count(struct intel_encoder *encoder);
-int intel_bios_alternate_ddc_pin(struct intel_encoder *encoder);
 bool intel_bios_port_supports_typec_usb(struct drm_i915_private *i915, enum port port);
 bool intel_bios_port_supports_tbt(struct drm_i915_private *i915, enum port port);
 
@@ -272,9 +261,19 @@ intel_bios_encoder_data_lookup(struct drm_i915_private *i915, enum port port);
 bool intel_bios_encoder_supports_dvi(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_hdmi(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_dp(const struct intel_bios_encoder_data *devdata);
+bool intel_bios_encoder_supports_edp(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_typec_usb(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_tbt(const struct intel_bios_encoder_data *devdata);
-int intel_bios_encoder_dp_boost_level(const struct intel_bios_encoder_data *devdata);
-int intel_bios_encoder_hdmi_boost_level(const struct intel_bios_encoder_data *devdata);
+bool intel_bios_encoder_is_lspcon(const struct intel_bios_encoder_data *devdata);
+bool intel_bios_encoder_lane_reversal(const struct intel_bios_encoder_data *devdata);
+bool intel_bios_encoder_hpd_invert(const struct intel_bios_encoder_data *devdata);
+enum aux_ch intel_bios_dp_aux_ch(const struct intel_bios_encoder_data *devdata);
+int intel_bios_dp_boost_level(const struct intel_bios_encoder_data *devdata);
+int intel_bios_dp_max_lane_count(const struct intel_bios_encoder_data *devdata);
+int intel_bios_dp_max_link_rate(const struct intel_bios_encoder_data *devdata);
+int intel_bios_hdmi_boost_level(const struct intel_bios_encoder_data *devdata);
+int intel_bios_hdmi_ddc_pin(const struct intel_bios_encoder_data *devdata);
+int intel_bios_hdmi_level_shift(const struct intel_bios_encoder_data *devdata);
+int intel_bios_hdmi_max_tmds_clock(const struct intel_bios_encoder_data *devdata);
 
 #endif /* _INTEL_BIOS_H_ */
diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c
index 1c236f02b380..202321ffbe2a 100644
--- a/drivers/gpu/drm/i915/display/intel_bw.c
+++ b/drivers/gpu/drm/i915/display/intel_bw.c
@@ -119,6 +119,32 @@ static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
 	return 0;
 }
 
+static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
+{
+	unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
+	unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
+	u16 qgv_points = 0, psf_points = 0;
+
+	/*
+	 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
+	 * it with failure if we try masking any unadvertised points.
+	 * So need to operate only with those returned from PCode.
+	 */
+	if (num_qgv_points > 0)
+		qgv_points = GENMASK(num_qgv_points - 1, 0);
+
+	if (num_psf_gv_points > 0)
+		psf_points = GENMASK(num_psf_gv_points - 1, 0);
+
+	return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
+}
+
+static bool is_sagv_enabled(struct drm_i915_private *i915, u16 points_mask)
+{
+	return !is_power_of_2(~points_mask & icl_qgv_points_mask(i915) &
+			      ICL_PCODE_REQ_QGV_PT_MASK);
+}
+
 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
 				  u32 points_mask)
 {
@@ -136,6 +162,9 @@ int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
 		return ret;
 	}
 
+	dev_priv->display.sagv.status = is_sagv_enabled(dev_priv, points_mask) ?
+		I915_SAGV_ENABLED : I915_SAGV_DISABLED;
+
 	return 0;
 }
 
@@ -965,26 +994,6 @@ int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
 	return 0;
 }
 
-static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
-{
-	unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
-	unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
-	u16 qgv_points = 0, psf_points = 0;
-
-	/*
-	 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
-	 * it with failure if we try masking any unadvertised points.
-	 * So need to operate only with those returned from PCode.
-	 */
-	if (num_qgv_points > 0)
-		qgv_points = GENMASK(num_qgv_points - 1, 0);
-
-	if (num_psf_gv_points > 0)
-		psf_points = GENMASK(num_psf_gv_points - 1, 0);
-
-	return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
-}
-
 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
 {
 	struct drm_i915_private *i915 = to_i915(state->base.dev);
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c
index 7e16b655c833..084a483f9776 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.c
@@ -1329,6 +1329,30 @@ static const struct intel_cdclk_vals adlp_cdclk_table[] = {
 	{}
 };
 
+static const struct intel_cdclk_vals rplu_cdclk_table[] = {
+	{ .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
+	{ .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
+	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
+	{ .refclk = 19200, .cdclk = 480000, .divider = 2, .ratio = 50 },
+	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
+	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
+
+	{ .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
+	{ .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
+	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
+	{ .refclk = 24000, .cdclk = 480000, .divider = 2, .ratio = 40 },
+	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
+	{ .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
+
+	{ .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
+	{ .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
+	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
+	{ .refclk = 38400, .cdclk = 480000, .divider = 2, .ratio = 25 },
+	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
+	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
+	{}
+};
+
 static const struct intel_cdclk_vals dg2_cdclk_table[] = {
 	{ .refclk = 38400, .cdclk = 163200, .divider = 2, .ratio = 34, .waveform = 0x8888 },
 	{ .refclk = 38400, .cdclk = 204000, .divider = 2, .ratio = 34, .waveform = 0x9248 },
@@ -1801,6 +1825,13 @@ static bool cdclk_compute_crawl_and_squash_midpoint(struct drm_i915_private *i91
 	return true;
 }
 
+static bool pll_enable_wa_needed(struct drm_i915_private *dev_priv)
+{
+	return ((IS_DG2(dev_priv) || IS_METEORLAKE(dev_priv)) &&
+		dev_priv->display.cdclk.hw.vco > 0 &&
+		HAS_CDCLK_SQUASH(dev_priv));
+}
+
 static void _bxt_set_cdclk(struct drm_i915_private *dev_priv,
 			   const struct intel_cdclk_config *cdclk_config,
 			   enum pipe pipe)
@@ -1815,9 +1846,13 @@ static void _bxt_set_cdclk(struct drm_i915_private *dev_priv,
 	    !cdclk_pll_is_unknown(dev_priv->display.cdclk.hw.vco)) {
 		if (dev_priv->display.cdclk.hw.vco != vco)
 			adlp_cdclk_pll_crawl(dev_priv, vco);
-	} else if (DISPLAY_VER(dev_priv) >= 11)
+	} else if (DISPLAY_VER(dev_priv) >= 11) {
+		/* wa_15010685871: dg2, mtl */
+		if (pll_enable_wa_needed(dev_priv))
+			dg2_cdclk_squash_program(dev_priv, 0);
+
 		icl_cdclk_pll_update(dev_priv, vco);
-	else
+	} else
 		bxt_cdclk_pll_update(dev_priv, vco);
 
 	waveform = cdclk_squash_waveform(dev_priv, cdclk);
@@ -3353,6 +3388,8 @@ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
 		/* Wa_22011320316:adl-p[a0] */
 		if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
 			dev_priv->display.cdclk.table = adlp_a_step_cdclk_table;
+		else if (IS_ADLP_RPLU(dev_priv))
+			dev_priv->display.cdclk.table = rplu_cdclk_table;
 		else
 			dev_priv->display.cdclk.table = adlp_cdclk_table;
 	} else if (IS_ROCKETLAKE(dev_priv)) {
diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c
index bd598a7f5047..36aac88143ac 100644
--- a/drivers/gpu/drm/i915/display/intel_color.c
+++ b/drivers/gpu/drm/i915/display/intel_color.c
@@ -262,7 +262,7 @@ static bool ilk_limited_range(const struct intel_crtc_state *crtc_state)
 	if (DISPLAY_VER(i915) >= 11)
 		return false;
 
-	/* pre-hsw have PIPECONF_COLOR_RANGE_SELECT */
+	/* pre-hsw have TRANSCONF_COLOR_RANGE_SELECT */
 	if (DISPLAY_VER(i915) < 7 || IS_IVYBRIDGE(i915))
 		return false;
 
@@ -653,7 +653,7 @@ static void ilk_color_commit_noarm(const struct intel_crtc_state *crtc_state)
 
 static void i9xx_color_commit_arm(const struct intel_crtc_state *crtc_state)
 {
-	/* update PIPECONF GAMMA_MODE */
+	/* update TRANSCONF GAMMA_MODE */
 	i9xx_set_pipeconf(crtc_state);
 }
 
@@ -662,7 +662,7 @@ static void ilk_color_commit_arm(const struct intel_crtc_state *crtc_state)
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 
-	/* update PIPECONF GAMMA_MODE */
+	/* update TRANSCONF GAMMA_MODE */
 	ilk_set_pipeconf(crtc_state);
 
 	intel_de_write_fw(i915, PIPE_CSC_MODE(crtc->pipe),
@@ -1329,8 +1329,11 @@ static void icl_load_luts(const struct intel_crtc_state *crtc_state)
 		break;
 	}
 
-	if (crtc_state->dsb)
-		intel_dsb_commit(crtc_state->dsb);
+	if (crtc_state->dsb) {
+		intel_dsb_finish(crtc_state->dsb);
+		intel_dsb_commit(crtc_state->dsb, false);
+		intel_dsb_wait(crtc_state->dsb);
+	}
 }
 
 static u32 chv_cgm_degamma_ldw(const struct drm_color_lut *color)
@@ -1461,6 +1464,9 @@ void intel_color_prepare_commit(struct intel_crtc_state *crtc_state)
 	/* FIXME DSB has issues loading LUTs, disable it for now */
 	return;
 
+	if (!crtc_state->pre_csc_lut && !crtc_state->post_csc_lut)
+		return;
+
 	crtc_state->dsb = intel_dsb_prepare(crtc, 1024);
 }
 
@@ -1581,6 +1587,8 @@ intel_color_add_affected_planes(struct intel_crtc_state *new_crtc_state)
 			return PTR_ERR(plane_state);
 
 		new_crtc_state->update_planes |= BIT(plane->id);
+		new_crtc_state->async_flip_planes = 0;
+		new_crtc_state->do_async_flip = false;
 
 		/* plane control register changes blocked by CxSR */
 		if (HAS_GMCH(i915))
diff --git a/drivers/gpu/drm/i915/display/intel_combo_phy.c b/drivers/gpu/drm/i915/display/intel_combo_phy.c
index 8b870b2dd4f9..922a6d87b553 100644
--- a/drivers/gpu/drm/i915/display/intel_combo_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_combo_phy.c
@@ -78,14 +78,11 @@ static void icl_set_procmon_ref_values(struct drm_i915_private *dev_priv,
 				       enum phy phy)
 {
 	const struct icl_procmon *procmon;
-	u32 val;
 
 	procmon = icl_get_procmon_ref_values(dev_priv, phy);
 
-	val = intel_de_read(dev_priv, ICL_PORT_COMP_DW1(phy));
-	val &= ~((0xff << 16) | 0xff);
-	val |= procmon->dw1;
-	intel_de_write(dev_priv, ICL_PORT_COMP_DW1(phy), val);
+	intel_de_rmw(dev_priv, ICL_PORT_COMP_DW1(phy),
+		     (0xff << 16) | 0xff, procmon->dw1);
 
 	intel_de_write(dev_priv, ICL_PORT_COMP_DW9(phy), procmon->dw9);
 	intel_de_write(dev_priv, ICL_PORT_COMP_DW10(phy), procmon->dw10);
@@ -236,8 +233,7 @@ static bool icl_combo_phy_verify_state(struct drm_i915_private *dev_priv,
 				     ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_DIV2);
 
 		ret &= check_phy_reg(dev_priv, phy, ICL_PORT_PCS_DW1_LN(0, phy),
-				     DCC_MODE_SELECT_MASK,
-				     DCC_MODE_SELECT_CONTINUOSLY);
+				     DCC_MODE_SELECT_MASK, RUN_DCC_ONCE);
 	}
 
 	ret &= icl_verify_procmon_ref_values(dev_priv, phy);
@@ -267,7 +263,6 @@ void intel_combo_phy_power_up_lanes(struct drm_i915_private *dev_priv,
 				    int lane_count, bool lane_reversal)
 {
 	u8 lane_mask;
-	u32 val;
 
 	if (is_dsi) {
 		drm_WARN_ON(&dev_priv->drm, lane_reversal);
@@ -308,10 +303,8 @@ void intel_combo_phy_power_up_lanes(struct drm_i915_private *dev_priv,
 		}
 	}
 
-	val = intel_de_read(dev_priv, ICL_PORT_CL_DW10(phy));
-	val &= ~PWR_DOWN_LN_MASK;
-	val |= lane_mask;
-	intel_de_write(dev_priv, ICL_PORT_CL_DW10(phy), val);
+	intel_de_rmw(dev_priv, ICL_PORT_CL_DW10(phy),
+		     PWR_DOWN_LN_MASK, lane_mask);
 }
 
 static void icl_combo_phys_init(struct drm_i915_private *dev_priv)
@@ -360,25 +353,19 @@ skip_phy_misc:
 
 			val = intel_de_read(dev_priv, ICL_PORT_PCS_DW1_LN(0, phy));
 			val &= ~DCC_MODE_SELECT_MASK;
-			val |= DCC_MODE_SELECT_CONTINUOSLY;
+			val |= RUN_DCC_ONCE;
 			intel_de_write(dev_priv, ICL_PORT_PCS_DW1_GRP(phy), val);
 		}
 
 		icl_set_procmon_ref_values(dev_priv, phy);
 
-		if (phy_is_master(dev_priv, phy)) {
-			val = intel_de_read(dev_priv, ICL_PORT_COMP_DW8(phy));
-			val |= IREFGEN;
-			intel_de_write(dev_priv, ICL_PORT_COMP_DW8(phy), val);
-		}
-
-		val = intel_de_read(dev_priv, ICL_PORT_COMP_DW0(phy));
-		val |= COMP_INIT;
-		intel_de_write(dev_priv, ICL_PORT_COMP_DW0(phy), val);
+		if (phy_is_master(dev_priv, phy))
+			intel_de_rmw(dev_priv, ICL_PORT_COMP_DW8(phy),
+				     0, IREFGEN);
 
-		val = intel_de_read(dev_priv, ICL_PORT_CL_DW5(phy));
-		val |= CL_POWER_DOWN_ENABLE;
-		intel_de_write(dev_priv, ICL_PORT_CL_DW5(phy), val);
+		intel_de_rmw(dev_priv, ICL_PORT_COMP_DW0(phy), 0, COMP_INIT);
+		intel_de_rmw(dev_priv, ICL_PORT_CL_DW5(phy),
+			     0, CL_POWER_DOWN_ENABLE);
 	}
 }
 
@@ -387,8 +374,6 @@ static void icl_combo_phys_uninit(struct drm_i915_private *dev_priv)
 	enum phy phy;
 
 	for_each_combo_phy_reverse(dev_priv, phy) {
-		u32 val;
-
 		if (phy == PHY_A &&
 		    !icl_combo_phy_verify_state(dev_priv, phy)) {
 			if (IS_TIGERLAKE(dev_priv) || IS_DG1(dev_priv)) {
@@ -410,14 +395,11 @@ static void icl_combo_phys_uninit(struct drm_i915_private *dev_priv)
 		if (!has_phy_misc(dev_priv, phy))
 			goto skip_phy_misc;
 
-		val = intel_de_read(dev_priv, ICL_PHY_MISC(phy));
-		val |= ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN;
-		intel_de_write(dev_priv, ICL_PHY_MISC(phy), val);
+		intel_de_rmw(dev_priv, ICL_PHY_MISC(phy), 0,
+			     ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN);
 
 skip_phy_misc:
-		val = intel_de_read(dev_priv, ICL_PORT_COMP_DW0(phy));
-		val &= ~COMP_INIT;
-		intel_de_write(dev_priv, ICL_PORT_COMP_DW0(phy), val);
+		intel_de_rmw(dev_priv, ICL_PORT_COMP_DW0(phy), COMP_INIT, 0);
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h b/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h
index 2ed65193ca19..b0983edccf3f 100644
--- a/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h
@@ -90,8 +90,8 @@
 #define ICL_PORT_PCS_DW1_AUX(phy)		_MMIO(_ICL_PORT_PCS_DW_AUX(1, phy))
 #define ICL_PORT_PCS_DW1_GRP(phy)		_MMIO(_ICL_PORT_PCS_DW_GRP(1, phy))
 #define ICL_PORT_PCS_DW1_LN(ln, phy)		_MMIO(_ICL_PORT_PCS_DW_LN(1, ln, phy))
-#define   DCC_MODE_SELECT_MASK			(0x3 << 20)
-#define   DCC_MODE_SELECT_CONTINUOSLY		(0x3 << 20)
+#define   DCC_MODE_SELECT_MASK			REG_GENMASK(21, 20)
+#define   RUN_DCC_ONCE				REG_FIELD_PREP(DCC_MODE_SELECT_MASK, 0)
 #define   COMMON_KEEPER_EN			(1 << 26)
 #define   LATENCY_OPTIM_MASK			(0x3 << 2)
 #define   LATENCY_OPTIM_VAL(x)			((x) << 2)
diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c
index 7267ffc7f539..38e9c61c2344 100644
--- a/drivers/gpu/drm/i915/display/intel_crt.c
+++ b/drivers/gpu/drm/i915/display/intel_crt.c
@@ -44,6 +44,7 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_fdi.h"
+#include "intel_fdi_regs.h"
 #include "intel_fifo_underrun.h"
 #include "intel_gmbus.h"
 #include "intel_hotplug.h"
@@ -260,7 +261,7 @@ static void hsw_post_disable_crt(struct intel_atomic_state *state,
 
 	ilk_pfit_disable(old_crtc_state);
 
-	intel_ddi_disable_pipe_clock(old_crtc_state);
+	intel_ddi_disable_transcoder_clock(old_crtc_state);
 
 	pch_post_disable_crt(state, encoder, old_crtc_state, old_conn_state);
 
@@ -300,7 +301,7 @@ static void hsw_pre_enable_crt(struct intel_atomic_state *state,
 
 	hsw_fdi_link_train(encoder, crtc_state);
 
-	intel_ddi_enable_pipe_clock(encoder, crtc_state);
+	intel_ddi_enable_transcoder_clock(encoder, crtc_state);
 }
 
 static void hsw_enable_crt(struct intel_atomic_state *state,
@@ -678,10 +679,11 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector)
 }
 
 static enum drm_connector_status
-intel_crt_load_detect(struct intel_crt *crt, u32 pipe)
+intel_crt_load_detect(struct intel_crt *crt, enum pipe pipe)
 {
 	struct drm_device *dev = crt->base.base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	enum transcoder cpu_transcoder = (enum transcoder)pipe;
 	u32 save_bclrpat;
 	u32 save_vtotal;
 	u32 vtotal, vactive;
@@ -693,25 +695,25 @@ intel_crt_load_detect(struct intel_crt *crt, u32 pipe)
 
 	drm_dbg_kms(&dev_priv->drm, "starting load-detect on CRT\n");
 
-	save_bclrpat = intel_de_read(dev_priv, BCLRPAT(pipe));
-	save_vtotal = intel_de_read(dev_priv, VTOTAL(pipe));
-	vblank = intel_de_read(dev_priv, VBLANK(pipe));
+	save_bclrpat = intel_de_read(dev_priv, BCLRPAT(cpu_transcoder));
+	save_vtotal = intel_de_read(dev_priv, TRANS_VTOTAL(cpu_transcoder));
+	vblank = intel_de_read(dev_priv, TRANS_VBLANK(cpu_transcoder));
 
-	vtotal = ((save_vtotal >> 16) & 0xfff) + 1;
-	vactive = (save_vtotal & 0x7ff) + 1;
+	vtotal = REG_FIELD_GET(VTOTAL_MASK, save_vtotal) + 1;
+	vactive = REG_FIELD_GET(VACTIVE_MASK, save_vtotal) + 1;
 
-	vblank_start = (vblank & 0xfff) + 1;
-	vblank_end = ((vblank >> 16) & 0xfff) + 1;
+	vblank_start = REG_FIELD_GET(VBLANK_START_MASK, vblank) + 1;
+	vblank_end = REG_FIELD_GET(VBLANK_END_MASK, vblank) + 1;
 
 	/* Set the border color to purple. */
-	intel_de_write(dev_priv, BCLRPAT(pipe), 0x500050);
+	intel_de_write(dev_priv, BCLRPAT(cpu_transcoder), 0x500050);
 
 	if (DISPLAY_VER(dev_priv) != 2) {
-		u32 pipeconf = intel_de_read(dev_priv, PIPECONF(pipe));
+		u32 transconf = intel_de_read(dev_priv, TRANSCONF(cpu_transcoder));
 
-		intel_de_write(dev_priv, PIPECONF(pipe),
-			       pipeconf | PIPECONF_FORCE_BORDER);
-		intel_de_posting_read(dev_priv, PIPECONF(pipe));
+		intel_de_write(dev_priv, TRANSCONF(cpu_transcoder),
+			       transconf | TRANSCONF_FORCE_BORDER);
+		intel_de_posting_read(dev_priv, TRANSCONF(cpu_transcoder));
 		/* Wait for next Vblank to substitue
 		 * border color for Color info */
 		intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(dev_priv, pipe));
@@ -720,7 +722,7 @@ intel_crt_load_detect(struct intel_crt *crt, u32 pipe)
 			connector_status_connected :
 			connector_status_disconnected;
 
-		intel_de_write(dev_priv, PIPECONF(pipe), pipeconf);
+		intel_de_write(dev_priv, TRANSCONF(cpu_transcoder), transconf);
 	} else {
 		bool restore_vblank = false;
 		int count, detect;
@@ -730,12 +732,13 @@ intel_crt_load_detect(struct intel_crt *crt, u32 pipe)
 		* Yes, this will flicker
 		*/
 		if (vblank_start <= vactive && vblank_end >= vtotal) {
-			u32 vsync = intel_de_read(dev_priv, VSYNC(pipe));
-			u32 vsync_start = (vsync & 0xffff) + 1;
+			u32 vsync = intel_de_read(dev_priv, TRANS_VSYNC(cpu_transcoder));
+			u32 vsync_start = REG_FIELD_GET(VSYNC_START_MASK, vsync) + 1;
 
 			vblank_start = vsync_start;
-			intel_de_write(dev_priv, VBLANK(pipe),
-				       (vblank_start - 1) | ((vblank_end - 1) << 16));
+			intel_de_write(dev_priv, TRANS_VBLANK(cpu_transcoder),
+				       VBLANK_START(vblank_start - 1) |
+				       VBLANK_END(vblank_end - 1));
 			restore_vblank = true;
 		}
 		/* sample in the vertical border, selecting the larger one */
@@ -766,7 +769,7 @@ intel_crt_load_detect(struct intel_crt *crt, u32 pipe)
 
 		/* restore vblank if necessary */
 		if (restore_vblank)
-			intel_de_write(dev_priv, VBLANK(pipe), vblank);
+			intel_de_write(dev_priv, TRANS_VBLANK(cpu_transcoder), vblank);
 		/*
 		 * If more than 3/4 of the scanline detected a monitor,
 		 * then it is assumed to be present. This works even on i830,
@@ -779,7 +782,7 @@ intel_crt_load_detect(struct intel_crt *crt, u32 pipe)
 	}
 
 	/* Restore previous settings */
-	intel_de_write(dev_priv, BCLRPAT(pipe), save_bclrpat);
+	intel_de_write(dev_priv, BCLRPAT(cpu_transcoder), save_bclrpat);
 
 	return status;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_crtc.c b/drivers/gpu/drm/i915/display/intel_crtc.c
index d5b5d40ed817..ed45a6934854 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc.c
@@ -25,6 +25,7 @@
 #include "intel_display_types.h"
 #include "intel_drrs.h"
 #include "intel_dsi.h"
+#include "intel_fifo_underrun.h"
 #include "intel_pipe_crc.h"
 #include "intel_psr.h"
 #include "intel_sprite.h"
@@ -211,7 +212,7 @@ static void intel_crtc_destroy(struct drm_crtc *_crtc)
 
 static int intel_crtc_late_register(struct drm_crtc *crtc)
 {
-	intel_crtc_debugfs_add(crtc);
+	intel_crtc_debugfs_add(to_intel_crtc(crtc));
 	return 0;
 }
 
@@ -314,6 +315,8 @@ int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
 	}
 	crtc->plane_ids_mask |= BIT(primary->id);
 
+	intel_init_fifo_underrun_reporting(dev_priv, crtc, false);
+
 	for_each_sprite(dev_priv, pipe, sprite) {
 		struct intel_plane *plane;
 
diff --git a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
index 2422d6ef5777..54c8adc0702e 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
@@ -14,14 +14,16 @@
 static void intel_dump_crtc_timings(struct drm_i915_private *i915,
 				    const struct drm_display_mode *mode)
 {
-	drm_dbg_kms(&i915->drm, "crtc timings: %d %d %d %d %d %d %d %d %d, "
-		    "type: 0x%x flags: 0x%x\n",
+	drm_dbg_kms(&i915->drm, "crtc timings: clock=%d, "
+		    "hd=%d hb=%d-%d hs=%d-%d ht=%d, "
+		    "vd=%d vb=%d-%d vs=%d-%d vt=%d, "
+		    "flags=0x%x\n",
 		    mode->crtc_clock,
-		    mode->crtc_hdisplay, mode->crtc_hsync_start,
-		    mode->crtc_hsync_end, mode->crtc_htotal,
-		    mode->crtc_vdisplay, mode->crtc_vsync_start,
-		    mode->crtc_vsync_end, mode->crtc_vtotal,
-		    mode->type, mode->flags);
+		    mode->crtc_hdisplay, mode->crtc_hblank_start, mode->crtc_hblank_end,
+		    mode->crtc_hsync_start, mode->crtc_hsync_end, mode->crtc_htotal,
+		    mode->crtc_vdisplay, mode->crtc_vblank_start, mode->crtc_vblank_end,
+		    mode->crtc_vsync_start, mode->crtc_vsync_end, mode->crtc_vtotal,
+		    mode->flags);
 }
 
 static void
@@ -121,7 +123,7 @@ static const char * const output_format_str[] = {
 	[INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
 };
 
-static const char *output_formats(enum intel_output_format format)
+const char *intel_output_format_name(enum intel_output_format format)
 {
 	if (format >= ARRAY_SIZE(output_format_str))
 		return "invalid";
@@ -179,7 +181,7 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
 		    "active: %s, output_types: %s (0x%x), output format: %s\n",
 		    str_yes_no(pipe_config->hw.active),
 		    buf, pipe_config->output_types,
-		    output_formats(pipe_config->output_format));
+		    intel_output_format_name(pipe_config->output_format));
 
 	drm_dbg_kms(&i915->drm,
 		    "cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
diff --git a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.h b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.h
index 9399c35b7e5e..780f3f1190d7 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.h
+++ b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.h
@@ -8,9 +8,11 @@
 
 struct intel_crtc_state;
 struct intel_atomic_state;
+enum intel_output_format;
 
 void intel_crtc_state_dump(const struct intel_crtc_state *crtc_state,
 			   struct intel_atomic_state *state,
 			   const char *context);
+const char *intel_output_format_name(enum intel_output_format format);
 
 #endif /* __INTEL_CRTC_STATE_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c
index d190fa0d393b..31bef0427377 100644
--- a/drivers/gpu/drm/i915/display/intel_cursor.c
+++ b/drivers/gpu/drm/i915/display/intel_cursor.c
@@ -21,7 +21,6 @@
 #include "intel_fb_pin.h"
 #include "intel_frontbuffer.h"
 #include "intel_psr.h"
-#include "intel_sprite.h"
 #include "skl_watermark.h"
 
 /* Cursor formats */
@@ -532,9 +531,10 @@ static void i9xx_cursor_update_arm(struct intel_plane *plane,
 		skl_write_cursor_wm(plane, crtc_state);
 
 	if (plane_state)
-		intel_psr2_program_plane_sel_fetch(plane, crtc_state, plane_state, 0);
+		intel_psr2_program_plane_sel_fetch_arm(plane, crtc_state,
+						       plane_state);
 	else
-		intel_psr2_disable_plane_sel_fetch(plane, crtc_state);
+		intel_psr2_disable_plane_sel_fetch_arm(plane, crtc_state);
 
 	if (plane->cursor.base != base ||
 	    plane->cursor.size != fbc_ctl ||
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
index 254559abedfb..3a7b98837516 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi.c
@@ -47,6 +47,7 @@
 #include "intel_dkl_phy.h"
 #include "intel_dkl_phy_regs.h"
 #include "intel_dp.h"
+#include "intel_dp_aux.h"
 #include "intel_dp_link_training.h"
 #include "intel_dp_mst.h"
 #include "intel_dpio_phy.h"
@@ -64,9 +65,9 @@
 #include "intel_psr.h"
 #include "intel_quirks.h"
 #include "intel_snps_phy.h"
-#include "intel_sprite.h"
 #include "intel_tc.h"
 #include "intel_vdsc.h"
+#include "intel_vdsc_regs.h"
 #include "intel_vrr.h"
 #include "skl_scaler.h"
 #include "skl_universal_plane.h"
@@ -89,7 +90,7 @@ static int intel_ddi_hdmi_level(struct intel_encoder *encoder,
 {
 	int level;
 
-	level = intel_bios_hdmi_level_shift(encoder);
+	level = intel_bios_hdmi_level_shift(encoder->devdata);
 	if (level < 0)
 		level = trans->hdmi_default_entry;
 
@@ -126,7 +127,7 @@ void hsw_prepare_dp_ddi_buffers(struct intel_encoder *encoder,
 
 	/* If we're boosting the current, set bit 31 of trans1 */
 	if (has_iboost(dev_priv) &&
-	    intel_bios_encoder_dp_boost_level(encoder->devdata))
+	    intel_bios_dp_boost_level(encoder->devdata))
 		iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE;
 
 	for (i = 0; i < n_entries; i++) {
@@ -158,7 +159,7 @@ static void hsw_prepare_hdmi_ddi_buffers(struct intel_encoder *encoder,
 
 	/* If we're boosting the current, set bit 31 of trans1 */
 	if (has_iboost(dev_priv) &&
-	    intel_bios_encoder_hdmi_boost_level(encoder->devdata))
+	    intel_bios_hdmi_boost_level(encoder->devdata))
 		iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE;
 
 	/* Entry 9 is for HDMI: */
@@ -644,19 +645,14 @@ int intel_ddi_toggle_hdcp_bits(struct intel_encoder *intel_encoder,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	intel_wakeref_t wakeref;
 	int ret = 0;
-	u32 tmp;
 
 	wakeref = intel_display_power_get_if_enabled(dev_priv,
 						     intel_encoder->power_domain);
 	if (drm_WARN_ON(dev, !wakeref))
 		return -ENXIO;
 
-	tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder));
-	if (enable)
-		tmp |= hdcp_mask;
-	else
-		tmp &= ~hdcp_mask;
-	intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder), tmp);
+	intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder),
+		     hdcp_mask, enable ? hdcp_mask : 0);
 	intel_display_power_put(dev_priv, intel_encoder->power_domain, wakeref);
 	return ret;
 }
@@ -948,8 +944,8 @@ static void intel_ddi_get_power_domains(struct intel_encoder *encoder,
 	main_link_aux_power_domain_get(dig_port, crtc_state);
 }
 
-void intel_ddi_enable_pipe_clock(struct intel_encoder *encoder,
-				 const struct intel_crtc_state *crtc_state)
+void intel_ddi_enable_transcoder_clock(struct intel_encoder *encoder,
+				       const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
@@ -957,33 +953,34 @@ void intel_ddi_enable_pipe_clock(struct intel_encoder *encoder,
 	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
 	u32 val;
 
-	if (cpu_transcoder != TRANSCODER_EDP) {
-		if (DISPLAY_VER(dev_priv) >= 13)
-			val = TGL_TRANS_CLK_SEL_PORT(phy);
-		else if (DISPLAY_VER(dev_priv) >= 12)
-			val = TGL_TRANS_CLK_SEL_PORT(encoder->port);
-		else
-			val = TRANS_CLK_SEL_PORT(encoder->port);
+	if (cpu_transcoder == TRANSCODER_EDP)
+		return;
 
-		intel_de_write(dev_priv, TRANS_CLK_SEL(cpu_transcoder), val);
-	}
+	if (DISPLAY_VER(dev_priv) >= 13)
+		val = TGL_TRANS_CLK_SEL_PORT(phy);
+	else if (DISPLAY_VER(dev_priv) >= 12)
+		val = TGL_TRANS_CLK_SEL_PORT(encoder->port);
+	else
+		val = TRANS_CLK_SEL_PORT(encoder->port);
+
+	intel_de_write(dev_priv, TRANS_CLK_SEL(cpu_transcoder), val);
 }
 
-void intel_ddi_disable_pipe_clock(const struct intel_crtc_state *crtc_state)
+void intel_ddi_disable_transcoder_clock(const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	u32 val;
 
-	if (cpu_transcoder != TRANSCODER_EDP) {
-		if (DISPLAY_VER(dev_priv) >= 12)
-			intel_de_write(dev_priv,
-				       TRANS_CLK_SEL(cpu_transcoder),
-				       TGL_TRANS_CLK_SEL_DISABLED);
-		else
-			intel_de_write(dev_priv,
-				       TRANS_CLK_SEL(cpu_transcoder),
-				       TRANS_CLK_SEL_DISABLED);
-	}
+	if (cpu_transcoder == TRANSCODER_EDP)
+		return;
+
+	if (DISPLAY_VER(dev_priv) >= 12)
+		val = TGL_TRANS_CLK_SEL_DISABLED;
+	else
+		val = TRANS_CLK_SEL_DISABLED;
+
+	intel_de_write(dev_priv, TRANS_CLK_SEL(cpu_transcoder), val);
 }
 
 static void _skl_ddi_set_iboost(struct drm_i915_private *dev_priv,
@@ -1009,9 +1006,9 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder,
 	u8 iboost;
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
-		iboost = intel_bios_encoder_hdmi_boost_level(encoder->devdata);
+		iboost = intel_bios_hdmi_boost_level(encoder->devdata);
 	else
-		iboost = intel_bios_encoder_dp_boost_level(encoder->devdata);
+		iboost = intel_bios_dp_boost_level(encoder->devdata);
 
 	if (iboost == 0) {
 		const struct intel_ddi_buf_trans *trans;
@@ -2200,15 +2197,13 @@ static void intel_ddi_enable_fec(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp *intel_dp;
-	u32 val;
 
 	if (!crtc_state->fec_enable)
 		return;
 
 	intel_dp = enc_to_intel_dp(encoder);
-	val = intel_de_read(dev_priv, dp_tp_ctl_reg(encoder, crtc_state));
-	val |= DP_TP_CTL_FEC_ENABLE;
-	intel_de_write(dev_priv, dp_tp_ctl_reg(encoder, crtc_state), val);
+	intel_de_rmw(dev_priv, dp_tp_ctl_reg(encoder, crtc_state),
+		     0, DP_TP_CTL_FEC_ENABLE);
 }
 
 static void intel_ddi_disable_fec_state(struct intel_encoder *encoder,
@@ -2216,15 +2211,13 @@ static void intel_ddi_disable_fec_state(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp *intel_dp;
-	u32 val;
 
 	if (!crtc_state->fec_enable)
 		return;
 
 	intel_dp = enc_to_intel_dp(encoder);
-	val = intel_de_read(dev_priv, dp_tp_ctl_reg(encoder, crtc_state));
-	val &= ~DP_TP_CTL_FEC_ENABLE;
-	intel_de_write(dev_priv, dp_tp_ctl_reg(encoder, crtc_state), val);
+	intel_de_rmw(dev_priv, dp_tp_ctl_reg(encoder, crtc_state),
+		     DP_TP_CTL_FEC_ENABLE, 0);
 	intel_de_posting_read(dev_priv, dp_tp_ctl_reg(encoder, crtc_state));
 }
 
@@ -2387,7 +2380,7 @@ static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state,
 	 * 7.a Configure Transcoder Clock Select to direct the Port clock to the
 	 * Transcoder.
 	 */
-	intel_ddi_enable_pipe_clock(encoder, crtc_state);
+	intel_ddi_enable_transcoder_clock(encoder, crtc_state);
 
 	if (HAS_DP20(dev_priv))
 		intel_ddi_config_transcoder_dp2(encoder, crtc_state);
@@ -2514,7 +2507,7 @@ static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state,
 	intel_ddi_enable_fec(encoder, crtc_state);
 
 	if (!is_mst)
-		intel_ddi_enable_pipe_clock(encoder, crtc_state);
+		intel_ddi_enable_transcoder_clock(encoder, crtc_state);
 
 	intel_dsc_dp_pps_write(encoder, crtc_state);
 }
@@ -2526,6 +2519,10 @@ static void intel_ddi_pre_enable_dp(struct intel_atomic_state *state,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
+	if (HAS_DP20(dev_priv))
+		intel_dp_128b132b_sdp_crc16(enc_to_intel_dp(encoder),
+					    crtc_state);
+
 	if (DISPLAY_VER(dev_priv) >= 12)
 		tgl_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state);
 	else
@@ -2556,7 +2553,7 @@ static void intel_ddi_pre_enable_hdmi(struct intel_atomic_state *state,
 
 	icl_program_mg_dp_mode(dig_port, crtc_state);
 
-	intel_ddi_enable_pipe_clock(encoder, crtc_state);
+	intel_ddi_enable_transcoder_clock(encoder, crtc_state);
 
 	dig_port->set_infoframes(encoder,
 				 crtc_state->has_infoframe,
@@ -2622,12 +2619,9 @@ static void intel_disable_ddi_buf(struct intel_encoder *encoder,
 		wait = true;
 	}
 
-	if (intel_crtc_has_dp_encoder(crtc_state)) {
-		val = intel_de_read(dev_priv, dp_tp_ctl_reg(encoder, crtc_state));
-		val &= ~(DP_TP_CTL_ENABLE | DP_TP_CTL_LINK_TRAIN_MASK);
-		val |= DP_TP_CTL_LINK_TRAIN_PAT1;
-		intel_de_write(dev_priv, dp_tp_ctl_reg(encoder, crtc_state), val);
-	}
+	if (intel_crtc_has_dp_encoder(crtc_state))
+		intel_de_rmw(dev_priv, dp_tp_ctl_reg(encoder, crtc_state),
+			     DP_TP_CTL_ENABLE, 0);
 
 	/* Disable FEC in DP Sink */
 	intel_ddi_disable_fec_state(encoder, crtc_state);
@@ -2660,19 +2654,14 @@ static void intel_ddi_post_disable_dp(struct intel_atomic_state *state,
 	if (DISPLAY_VER(dev_priv) >= 12) {
 		if (is_mst) {
 			enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
-			u32 val;
 
-			val = intel_de_read(dev_priv,
-					    TRANS_DDI_FUNC_CTL(cpu_transcoder));
-			val &= ~(TGL_TRANS_DDI_PORT_MASK |
-				 TRANS_DDI_MODE_SELECT_MASK);
-			intel_de_write(dev_priv,
-				       TRANS_DDI_FUNC_CTL(cpu_transcoder),
-				       val);
+			intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder),
+				     TGL_TRANS_DDI_PORT_MASK | TRANS_DDI_MODE_SELECT_MASK,
+				     0);
 		}
 	} else {
 		if (!is_mst)
-			intel_ddi_disable_pipe_clock(old_crtc_state);
+			intel_ddi_disable_transcoder_clock(old_crtc_state);
 	}
 
 	intel_disable_ddi_buf(encoder, old_crtc_state);
@@ -2683,7 +2672,7 @@ static void intel_ddi_post_disable_dp(struct intel_atomic_state *state,
 	 * transcoder"
 	 */
 	if (DISPLAY_VER(dev_priv) >= 12)
-		intel_ddi_disable_pipe_clock(old_crtc_state);
+		intel_ddi_disable_transcoder_clock(old_crtc_state);
 
 	intel_pps_vdd_on(intel_dp);
 	intel_pps_off(intel_dp);
@@ -2709,12 +2698,12 @@ static void intel_ddi_post_disable_hdmi(struct intel_atomic_state *state,
 				 old_crtc_state, old_conn_state);
 
 	if (DISPLAY_VER(dev_priv) < 12)
-		intel_ddi_disable_pipe_clock(old_crtc_state);
+		intel_ddi_disable_transcoder_clock(old_crtc_state);
 
 	intel_disable_ddi_buf(encoder, old_crtc_state);
 
 	if (DISPLAY_VER(dev_priv) >= 12)
-		intel_ddi_disable_pipe_clock(old_crtc_state);
+		intel_ddi_disable_transcoder_clock(old_crtc_state);
 
 	intel_display_power_put(dev_priv,
 				dig_port->ddi_io_power_domain,
@@ -2731,9 +2720,6 @@ static void intel_ddi_post_disable(struct intel_atomic_state *state,
 				   const struct drm_connector_state *old_conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
-	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
-	bool is_tc_port = intel_phy_is_tc(dev_priv, phy);
 	struct intel_crtc *slave_crtc;
 
 	if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST)) {
@@ -2783,6 +2769,17 @@ static void intel_ddi_post_disable(struct intel_atomic_state *state,
 	else
 		intel_ddi_post_disable_dp(state, encoder, old_crtc_state,
 					  old_conn_state);
+}
+
+static void intel_ddi_post_pll_disable(struct intel_atomic_state *state,
+				       struct intel_encoder *encoder,
+				       const struct intel_crtc_state *old_crtc_state,
+				       const struct drm_connector_state *old_conn_state)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
+	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	bool is_tc_port = intel_phy_is_tc(i915, phy);
 
 	main_link_aux_power_domain_put(dig_port, old_crtc_state);
 
@@ -3063,37 +3060,23 @@ void intel_ddi_update_pipe(struct intel_atomic_state *state,
 	intel_hdcp_update_pipe(state, encoder, crtc_state, conn_state);
 }
 
-static void
-intel_ddi_update_prepare(struct intel_atomic_state *state,
-			 struct intel_encoder *encoder,
-			 struct intel_crtc *crtc)
+void intel_ddi_update_active_dpll(struct intel_atomic_state *state,
+				  struct intel_encoder *encoder,
+				  struct intel_crtc *crtc)
 {
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	struct intel_crtc_state *crtc_state =
-		crtc ? intel_atomic_get_new_crtc_state(state, crtc) : NULL;
-	int required_lanes = crtc_state ? crtc_state->lane_count : 1;
-
-	drm_WARN_ON(state->base.dev, crtc && crtc->active);
-
-	intel_tc_port_get_link(enc_to_dig_port(encoder),
-		               required_lanes);
-	if (crtc_state && crtc_state->hw.active) {
-		struct intel_crtc *slave_crtc;
-
-		intel_update_active_dpll(state, crtc, encoder);
+		intel_atomic_get_new_crtc_state(state, crtc);
+	struct intel_crtc *slave_crtc;
+	enum phy phy = intel_port_to_phy(i915, encoder->port);
 
-		for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc,
-						 intel_crtc_bigjoiner_slave_pipes(crtc_state))
-			intel_update_active_dpll(state, slave_crtc, encoder);
-	}
-}
+	if (!intel_phy_is_tc(i915, phy))
+		return;
 
-static void
-intel_ddi_update_complete(struct intel_atomic_state *state,
-			  struct intel_encoder *encoder,
-			  struct intel_crtc *crtc)
-{
-	intel_tc_port_put_link(enc_to_dig_port(encoder));
+	intel_update_active_dpll(state, crtc, encoder);
+	for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc,
+					 intel_crtc_bigjoiner_slave_pipes(crtc_state))
+		intel_update_active_dpll(state, slave_crtc, encoder);
 }
 
 static void
@@ -3107,8 +3090,13 @@ intel_ddi_pre_pll_enable(struct intel_atomic_state *state,
 	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
 	bool is_tc_port = intel_phy_is_tc(dev_priv, phy);
 
-	if (is_tc_port)
+	if (is_tc_port) {
+		struct intel_crtc *master_crtc =
+			to_intel_crtc(crtc_state->uapi.crtc);
+
 		intel_tc_port_get_link(dig_port, crtc_state->lane_count);
+		intel_ddi_update_active_dpll(state, encoder, master_crtc);
+	}
 
 	main_link_aux_power_domain_get(dig_port, crtc_state);
 
@@ -3153,8 +3141,7 @@ static void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp,
 			wait = true;
 		}
 
-		dp_tp_ctl &= ~(DP_TP_CTL_ENABLE | DP_TP_CTL_LINK_TRAIN_MASK);
-		dp_tp_ctl |= DP_TP_CTL_LINK_TRAIN_PAT1;
+		dp_tp_ctl &= ~DP_TP_CTL_ENABLE;
 		intel_de_write(dev_priv, dp_tp_ctl_reg(encoder, crtc_state), dp_tp_ctl);
 		intel_de_posting_read(dev_priv, dp_tp_ctl_reg(encoder, crtc_state));
 
@@ -3222,12 +3209,9 @@ static void intel_ddi_set_idle_link_train(struct intel_dp *intel_dp,
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum port port = encoder->port;
-	u32 val;
 
-	val = intel_de_read(dev_priv, dp_tp_ctl_reg(encoder, crtc_state));
-	val &= ~DP_TP_CTL_LINK_TRAIN_MASK;
-	val |= DP_TP_CTL_LINK_TRAIN_IDLE;
-	intel_de_write(dev_priv, dp_tp_ctl_reg(encoder, crtc_state), val);
+	intel_de_rmw(dev_priv, dp_tp_ctl_reg(encoder, crtc_state),
+		     DP_TP_CTL_LINK_TRAIN_MASK, DP_TP_CTL_LINK_TRAIN_IDLE);
 
 	/*
 	 * Until TGL on PORT_A we can have only eDP in SST mode. There the only
@@ -3559,6 +3543,37 @@ static void icl_ddi_combo_get_config(struct intel_encoder *encoder,
 	intel_ddi_get_config(encoder, crtc_state);
 }
 
+static bool icl_ddi_tc_pll_is_tbt(const struct intel_shared_dpll *pll)
+{
+	return pll->info->id == DPLL_ID_ICL_TBTPLL;
+}
+
+static enum icl_port_dpll_id
+icl_ddi_tc_port_pll_type(struct intel_encoder *encoder,
+			 const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	const struct intel_shared_dpll *pll = crtc_state->shared_dpll;
+
+	if (drm_WARN_ON(&i915->drm, !pll))
+		return ICL_PORT_DPLL_DEFAULT;
+
+	if (icl_ddi_tc_pll_is_tbt(pll))
+		return ICL_PORT_DPLL_DEFAULT;
+	else
+		return ICL_PORT_DPLL_MG_PHY;
+}
+
+enum icl_port_dpll_id
+intel_ddi_port_pll_type(struct intel_encoder *encoder,
+			const struct intel_crtc_state *crtc_state)
+{
+	if (!encoder->port_pll_type)
+		return ICL_PORT_DPLL_DEFAULT;
+
+	return encoder->port_pll_type(encoder, crtc_state);
+}
+
 static void icl_ddi_tc_get_clock(struct intel_encoder *encoder,
 				 struct intel_crtc_state *crtc_state,
 				 struct intel_shared_dpll *pll)
@@ -3571,7 +3586,7 @@ static void icl_ddi_tc_get_clock(struct intel_encoder *encoder,
 	if (drm_WARN_ON(&i915->drm, !pll))
 		return;
 
-	if (pll->info->id == DPLL_ID_ICL_TBTPLL)
+	if (icl_ddi_tc_pll_is_tbt(pll))
 		port_dpll_id = ICL_PORT_DPLL_DEFAULT;
 	else
 		port_dpll_id = ICL_PORT_DPLL_MG_PHY;
@@ -3584,7 +3599,7 @@ static void icl_ddi_tc_get_clock(struct intel_encoder *encoder,
 
 	icl_set_active_port_dpll(crtc_state, port_dpll_id);
 
-	if (crtc_state->shared_dpll->info->id == DPLL_ID_ICL_TBTPLL)
+	if (icl_ddi_tc_pll_is_tbt(crtc_state->shared_dpll))
 		crtc_state->port_clock = icl_calc_tbt_pll_link(i915, encoder->port);
 	else
 		crtc_state->port_clock = intel_dpll_get_freq(i915, crtc_state->shared_dpll,
@@ -3626,7 +3641,8 @@ static void intel_ddi_sync_state(struct intel_encoder *encoder,
 	enum phy phy = intel_port_to_phy(i915, encoder->port);
 
 	if (intel_phy_is_tc(i915, phy))
-		intel_tc_port_sanitize_mode(enc_to_dig_port(encoder));
+		intel_tc_port_sanitize_mode(enc_to_dig_port(encoder),
+					    crtc_state);
 
 	if (crtc_state && intel_crtc_has_dp_encoder(crtc_state))
 		intel_dp_sync_state(encoder, crtc_state);
@@ -3826,7 +3842,7 @@ static void intel_ddi_encoder_destroy(struct drm_encoder *encoder)
 
 	intel_dp_encoder_flush_work(encoder);
 	if (intel_phy_is_tc(i915, phy))
-		intel_tc_port_flush_work(dig_port);
+		intel_tc_port_cleanup(dig_port);
 	intel_display_power_flush_work(i915);
 
 	drm_encoder_cleanup(encoder);
@@ -3971,8 +3987,8 @@ static int intel_hdmi_reset_link(struct intel_encoder *encoder,
 
 	ret = drm_scdc_readb(adapter, SCDC_TMDS_CONFIG, &config);
 	if (ret < 0) {
-		drm_err(&dev_priv->drm, "Failed to read TMDS config: %d\n",
-			ret);
+		drm_err(&dev_priv->drm, "[CONNECTOR:%d:%s] Failed to read TMDS config: %d\n",
+			connector->base.base.id, connector->base.name, ret);
 		return 0;
 	}
 
@@ -4267,7 +4283,7 @@ static void intel_ddi_encoder_shutdown(struct intel_encoder *encoder)
 	if (!intel_phy_is_tc(i915, phy))
 		return;
 
-	intel_tc_port_flush_work(dig_port);
+	intel_tc_port_cleanup(dig_port);
 }
 
 #define port_tc_name(port) ((port) - PORT_TC1 + '1')
@@ -4305,7 +4321,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 		intel_bios_encoder_supports_hdmi(devdata);
 	init_dp = intel_bios_encoder_supports_dp(devdata);
 
-	if (intel_bios_is_lspcon_present(dev_priv, port)) {
+	if (intel_bios_encoder_is_lspcon(devdata)) {
 		/*
 		 * Lspcon device needs to be driven with DP connector
 		 * with special detection sequence. So make sure DP
@@ -4381,6 +4397,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 	encoder->pre_pll_enable = intel_ddi_pre_pll_enable;
 	encoder->pre_enable = intel_ddi_pre_enable;
 	encoder->disable = intel_disable_ddi;
+	encoder->post_pll_disable = intel_ddi_post_pll_disable;
 	encoder->post_disable = intel_ddi_post_disable;
 	encoder->update_pipe = intel_ddi_update_pipe;
 	encoder->get_hw_state = intel_ddi_get_hw_state;
@@ -4420,6 +4437,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 			encoder->enable_clock = jsl_ddi_tc_enable_clock;
 			encoder->disable_clock = jsl_ddi_tc_disable_clock;
 			encoder->is_clock_enabled = jsl_ddi_tc_is_clock_enabled;
+			encoder->port_pll_type = icl_ddi_tc_port_pll_type;
 			encoder->get_config = icl_ddi_combo_get_config;
 		} else {
 			encoder->enable_clock = icl_ddi_combo_enable_clock;
@@ -4432,6 +4450,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 			encoder->enable_clock = icl_ddi_tc_enable_clock;
 			encoder->disable_clock = icl_ddi_tc_disable_clock;
 			encoder->is_clock_enabled = icl_ddi_tc_is_clock_enabled;
+			encoder->port_pll_type = icl_ddi_tc_port_pll_type;
 			encoder->get_config = icl_ddi_tc_get_config;
 		} else {
 			encoder->enable_clock = icl_ddi_combo_enable_clock;
@@ -4500,56 +4519,50 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 			intel_de_read(dev_priv, DDI_BUF_CTL(port))
 			& (DDI_BUF_PORT_REVERSAL | DDI_A_4_LANES);
 
-	if (intel_bios_is_lane_reversal_needed(dev_priv, port))
+	if (intel_bios_encoder_lane_reversal(devdata))
 		dig_port->saved_port_bits |= DDI_BUF_PORT_REVERSAL;
 
 	dig_port->dp.output_reg = INVALID_MMIO_REG;
 	dig_port->max_lanes = intel_ddi_max_lanes(dig_port);
-	dig_port->aux_ch = intel_bios_port_aux_ch(dev_priv, port);
+	dig_port->aux_ch = intel_dp_aux_ch(encoder);
 
 	if (intel_phy_is_tc(dev_priv, phy)) {
 		bool is_legacy =
 			!intel_bios_encoder_supports_typec_usb(devdata) &&
 			!intel_bios_encoder_supports_tbt(devdata);
 
-		intel_tc_port_init(dig_port, is_legacy);
+		if (!is_legacy && init_hdmi) {
+			is_legacy = !init_dp;
 
-		encoder->update_prepare = intel_ddi_update_prepare;
-		encoder->update_complete = intel_ddi_update_complete;
-	}
-
-	drm_WARN_ON(&dev_priv->drm, port > PORT_I);
-	dig_port->ddi_io_power_domain = intel_display_power_ddi_io_domain(dev_priv, port);
+			drm_dbg_kms(&dev_priv->drm,
+				    "VBT says port %c is non-legacy TC and has HDMI (with DP: %s), assume it's %s\n",
+				    port_name(port),
+				    str_yes_no(init_dp),
+				    is_legacy ? "legacy" : "non-legacy");
+		}
 
-	if (init_dp) {
-		if (!intel_ddi_init_dp_connector(dig_port))
+		if (intel_tc_port_init(dig_port, is_legacy) < 0)
 			goto err;
-
-		dig_port->hpd_pulse = intel_dp_hpd_pulse;
-
-		if (dig_port->dp.mso_link_count)
-			encoder->pipe_mask = intel_ddi_splitter_pipe_mask(dev_priv);
 	}
 
-	/* In theory we don't need the encoder->type check, but leave it just in
-	 * case we have some really bad VBTs... */
-	if (encoder->type != INTEL_OUTPUT_EDP && init_hdmi) {
-		if (!intel_ddi_init_hdmi_connector(dig_port))
-			goto err;
-	}
+	drm_WARN_ON(&dev_priv->drm, port > PORT_I);
+	dig_port->ddi_io_power_domain = intel_display_power_ddi_io_domain(dev_priv, port);
 
 	if (DISPLAY_VER(dev_priv) >= 11) {
 		if (intel_phy_is_tc(dev_priv, phy))
 			dig_port->connected = intel_tc_port_connected;
 		else
 			dig_port->connected = lpt_digital_port_connected;
-	} else if (DISPLAY_VER(dev_priv) >= 8) {
-		if (port == PORT_A || IS_GEMINILAKE(dev_priv) ||
-		    IS_BROXTON(dev_priv))
+	} else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
+		dig_port->connected = bdw_digital_port_connected;
+	} else if (DISPLAY_VER(dev_priv) == 9) {
+		dig_port->connected = lpt_digital_port_connected;
+	} else if (IS_BROADWELL(dev_priv)) {
+		if (port == PORT_A)
 			dig_port->connected = bdw_digital_port_connected;
 		else
 			dig_port->connected = lpt_digital_port_connected;
-	} else {
+	} else if (IS_HASWELL(dev_priv)) {
 		if (port == PORT_A)
 			dig_port->connected = hsw_digital_port_connected;
 		else
@@ -4558,6 +4571,25 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 
 	intel_infoframe_init(dig_port);
 
+	if (init_dp) {
+		if (!intel_ddi_init_dp_connector(dig_port))
+			goto err;
+
+		dig_port->hpd_pulse = intel_dp_hpd_pulse;
+
+		if (dig_port->dp.mso_link_count)
+			encoder->pipe_mask = intel_ddi_splitter_pipe_mask(dev_priv);
+	}
+
+	/*
+	 * In theory we don't need the encoder->type check,
+	 * but leave it just in case we have some really bad VBTs...
+	 */
+	if (encoder->type != INTEL_OUTPUT_EDP && init_hdmi) {
+		if (!intel_ddi_init_hdmi_connector(dig_port))
+			goto err;
+	}
+
 	return;
 
 err:
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.h b/drivers/gpu/drm/i915/display/intel_ddi.h
index d39076facdce..2bc034042a93 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.h
+++ b/drivers/gpu/drm/i915/display/intel_ddi.h
@@ -40,6 +40,9 @@ void hsw_ddi_enable_clock(struct intel_encoder *encoder,
 			  const struct intel_crtc_state *crtc_state);
 void hsw_ddi_disable_clock(struct intel_encoder *encoder);
 bool hsw_ddi_is_clock_enabled(struct intel_encoder *encoder);
+enum icl_port_dpll_id
+intel_ddi_port_pll_type(struct intel_encoder *encoder,
+			const struct intel_crtc_state *crtc_state);
 void hsw_ddi_get_config(struct intel_encoder *encoder,
 			struct intel_crtc_state *crtc_state);
 struct intel_shared_dpll *icl_ddi_combo_get_pll(struct intel_encoder *encoder);
@@ -52,9 +55,9 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe);
 void intel_ddi_enable_transcoder_func(struct intel_encoder *encoder,
 				      const struct intel_crtc_state *crtc_state);
 void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state);
-void intel_ddi_enable_pipe_clock(struct intel_encoder *encoder,
-				 const struct intel_crtc_state *crtc_state);
-void intel_ddi_disable_pipe_clock(const  struct intel_crtc_state *crtc_state);
+void intel_ddi_enable_transcoder_clock(struct intel_encoder *encoder,
+				       const struct intel_crtc_state *crtc_state);
+void intel_ddi_disable_transcoder_clock(const  struct intel_crtc_state *crtc_state);
 void intel_ddi_set_dp_msa(const struct intel_crtc_state *crtc_state,
 			  const struct drm_connector_state *conn_state);
 bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector);
@@ -69,5 +72,8 @@ void intel_ddi_sanitize_encoder_pll_mapping(struct intel_encoder *encoder);
 int intel_ddi_level(struct intel_encoder *encoder,
 		    const struct intel_crtc_state *crtc_state,
 		    int lane);
+void intel_ddi_update_active_dpll(struct intel_atomic_state *state,
+				  struct intel_encoder *encoder,
+				  struct intel_crtc *crtc);
 
 #endif /* __INTEL_DDI_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index 63b4b73f47c6..3c29792137a5 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -55,6 +55,7 @@
 #include "i915_reg.h"
 #include "i915_utils.h"
 #include "i9xx_plane.h"
+#include "i9xx_wm.h"
 #include "icl_dsi.h"
 #include "intel_acpi.h"
 #include "intel_atomic.h"
@@ -62,6 +63,7 @@
 #include "intel_audio.h"
 #include "intel_bw.h"
 #include "intel_cdclk.h"
+#include "intel_clock_gating.h"
 #include "intel_color.h"
 #include "intel_crt.h"
 #include "intel_crtc.h"
@@ -94,6 +96,7 @@
 #include "intel_hotplug.h"
 #include "intel_hti.h"
 #include "intel_lvds.h"
+#include "intel_lvds_regs.h"
 #include "intel_modeset_setup.h"
 #include "intel_modeset_verify.h"
 #include "intel_overlay.h"
@@ -103,19 +106,19 @@
 #include "intel_pcode.h"
 #include "intel_pipe_crc.h"
 #include "intel_plane_initial.h"
-#include "intel_pm.h"
 #include "intel_pps.h"
 #include "intel_psr.h"
 #include "intel_quirks.h"
 #include "intel_sdvo.h"
 #include "intel_snps_phy.h"
-#include "intel_sprite.h"
 #include "intel_tc.h"
 #include "intel_tv.h"
 #include "intel_vblank.h"
 #include "intel_vdsc.h"
+#include "intel_vdsc_regs.h"
 #include "intel_vga.h"
 #include "intel_vrr.h"
+#include "intel_wm.h"
 #include "skl_scaler.h"
 #include "skl_universal_plane.h"
 #include "skl_watermark.h"
@@ -127,104 +130,9 @@
 static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state);
 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 static void hsw_set_transconf(const struct intel_crtc_state *crtc_state);
-static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
+static void bdw_set_pipe_misc(const struct intel_crtc_state *crtc_state);
 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state);
 
-/**
- * intel_update_watermarks - update FIFO watermark values based on current modes
- * @dev_priv: i915 device
- *
- * Calculate watermark values for the various WM regs based on current mode
- * and plane configuration.
- *
- * There are several cases to deal with here:
- *   - normal (i.e. non-self-refresh)
- *   - self-refresh (SR) mode
- *   - lines are large relative to FIFO size (buffer can hold up to 2)
- *   - lines are small relative to FIFO size (buffer can hold more than 2
- *     lines), so need to account for TLB latency
- *
- *   The normal calculation is:
- *     watermark = dotclock * bytes per pixel * latency
- *   where latency is platform & configuration dependent (we assume pessimal
- *   values here).
- *
- *   The SR calculation is:
- *     watermark = (trunc(latency/line time)+1) * surface width *
- *       bytes per pixel
- *   where
- *     line time = htotal / dotclock
- *     surface width = hdisplay for normal plane and 64 for cursor
- *   and latency is assumed to be high, as above.
- *
- * The final value programmed to the register should always be rounded up,
- * and include an extra 2 entries to account for clock crossings.
- *
- * We don't use the sprite, so we can ignore that.  And on Crestline we have
- * to set the non-SR watermarks to 8.
- */
-void intel_update_watermarks(struct drm_i915_private *dev_priv)
-{
-	if (dev_priv->display.funcs.wm->update_wm)
-		dev_priv->display.funcs.wm->update_wm(dev_priv);
-}
-
-static int intel_compute_pipe_wm(struct intel_atomic_state *state,
-				 struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (dev_priv->display.funcs.wm->compute_pipe_wm)
-		return dev_priv->display.funcs.wm->compute_pipe_wm(state, crtc);
-	return 0;
-}
-
-static int intel_compute_intermediate_wm(struct intel_atomic_state *state,
-					 struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (!dev_priv->display.funcs.wm->compute_intermediate_wm)
-		return 0;
-	if (drm_WARN_ON(&dev_priv->drm,
-			!dev_priv->display.funcs.wm->compute_pipe_wm))
-		return 0;
-	return dev_priv->display.funcs.wm->compute_intermediate_wm(state, crtc);
-}
-
-static bool intel_initial_watermarks(struct intel_atomic_state *state,
-				     struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (dev_priv->display.funcs.wm->initial_watermarks) {
-		dev_priv->display.funcs.wm->initial_watermarks(state, crtc);
-		return true;
-	}
-	return false;
-}
-
-static void intel_atomic_update_watermarks(struct intel_atomic_state *state,
-					   struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (dev_priv->display.funcs.wm->atomic_update_watermarks)
-		dev_priv->display.funcs.wm->atomic_update_watermarks(state, crtc);
-}
-
-static void intel_optimize_watermarks(struct intel_atomic_state *state,
-				      struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (dev_priv->display.funcs.wm->optimize_watermarks)
-		dev_priv->display.funcs.wm->optimize_watermarks(state, crtc);
-}
-
-static int intel_compute_global_watermarks(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (dev_priv->display.funcs.wm->compute_global_watermarks)
-		return dev_priv->display.funcs.wm->compute_global_watermarks(state);
-	return 0;
-}
-
 /* returns HPLL frequency in kHz */
 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 {
@@ -293,11 +201,11 @@ static void
 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 {
 	if (enable)
-		intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
-		               intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) | DUPS1_GATING_DIS | DUPS2_GATING_DIS);
+		intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe),
+			     0, DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 	else
-		intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
-		               intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
+		intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe),
+			     DUPS1_GATING_DIS | DUPS2_GATING_DIS, 0);
 }
 
 /* Wa_2006604312:icl,ehl */
@@ -306,11 +214,9 @@ icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 		       bool enable)
 {
 	if (enable)
-		intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
-		               intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
+		intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe), 0, DPFR_GATING_DIS);
 	else
-		intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
-		               intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
+		intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe), DPFR_GATING_DIS, 0);
 }
 
 /* Wa_1604331009:icl,jsl,ehl */
@@ -395,8 +301,8 @@ intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
 		enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 
 		/* Wait for the Pipe State to go off */
-		if (intel_de_wait_for_clear(dev_priv, PIPECONF(cpu_transcoder),
-					    PIPECONF_STATE_ENABLE, 100))
+		if (intel_de_wait_for_clear(dev_priv, TRANSCONF(cpu_transcoder),
+					    TRANSCONF_STATE_ENABLE, 100))
 			drm_WARN(&dev_priv->drm, 1, "pipe_off wait timed out\n");
 	} else {
 		intel_wait_for_pipe_scanline_stopped(crtc);
@@ -417,8 +323,8 @@ void assert_transcoder(struct drm_i915_private *dev_priv,
 	power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
 	wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
 	if (wakeref) {
-		u32 val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
-		cur_state = !!(val & PIPECONF_ENABLE);
+		u32 val = intel_de_read(dev_priv, TRANSCONF(cpu_transcoder));
+		cur_state = !!(val & TRANSCONF_ENABLE);
 
 		intel_display_power_put(dev_priv, power_domain, wakeref);
 	} else {
@@ -530,15 +436,15 @@ void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state)
 		intel_de_rmw(dev_priv, PIPE_ARB_CTL(pipe),
 			     0, PIPE_ARB_USE_PROG_SLOTS);
 
-	reg = PIPECONF(cpu_transcoder);
+	reg = TRANSCONF(cpu_transcoder);
 	val = intel_de_read(dev_priv, reg);
-	if (val & PIPECONF_ENABLE) {
+	if (val & TRANSCONF_ENABLE) {
 		/* we keep both pipes enabled on 830 */
 		drm_WARN_ON(&dev_priv->drm, !IS_I830(dev_priv));
 		return;
 	}
 
-	intel_de_write(dev_priv, reg, val | PIPECONF_ENABLE);
+	intel_de_write(dev_priv, reg, val | TRANSCONF_ENABLE);
 	intel_de_posting_read(dev_priv, reg);
 
 	/*
@@ -569,9 +475,9 @@ void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 	 */
 	assert_planes_disabled(crtc);
 
-	reg = PIPECONF(cpu_transcoder);
+	reg = TRANSCONF(cpu_transcoder);
 	val = intel_de_read(dev_priv, reg);
-	if ((val & PIPECONF_ENABLE) == 0)
+	if ((val & TRANSCONF_ENABLE) == 0)
 		return;
 
 	/*
@@ -579,11 +485,11 @@ void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 	 * so best keep it disabled when not needed.
 	 */
 	if (old_crtc_state->double_wide)
-		val &= ~PIPECONF_DOUBLE_WIDE;
+		val &= ~TRANSCONF_DOUBLE_WIDE;
 
 	/* Don't disable pipe or pipe PLLs if needed */
 	if (!IS_I830(dev_priv))
-		val &= ~PIPECONF_ENABLE;
+		val &= ~TRANSCONF_ENABLE;
 
 	if (DISPLAY_VER(dev_priv) >= 14)
 		intel_de_rmw(dev_priv, MTL_CHICKEN_TRANS(cpu_transcoder),
@@ -593,7 +499,7 @@ void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 			     FECSTALL_DIS_DPTSTREAM_DPTTG, 0);
 
 	intel_de_write(dev_priv, reg, val);
-	if ((val & PIPECONF_ENABLE) == 0)
+	if ((val & TRANSCONF_ENABLE) == 0)
 		intel_wait_for_pipe_off(old_crtc_state);
 }
 
@@ -944,7 +850,7 @@ void intel_display_finish_reset(struct drm_i915_private *i915)
 		 */
 		intel_pps_unlock_regs_wa(i915);
 		intel_modeset_init_hw(i915);
-		intel_init_clock_gating(i915);
+		intel_clock_gating_init(i915);
 		intel_hpd_init(i915);
 
 		ret = __intel_display_resume(i915, state, ctx);
@@ -1053,7 +959,7 @@ intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
 		num_encoders++;
 	}
 
-	drm_WARN(encoder->base.dev, num_encoders != 1,
+	drm_WARN(state->base.dev, num_encoders != 1,
 		 "%d encoders for pipe %c\n",
 		 num_encoders, pipe_name(master_crtc->pipe));
 
@@ -1255,7 +1161,8 @@ static void intel_crtc_async_flip_disable_wa(struct intel_atomic_state *state,
 		intel_atomic_get_old_crtc_state(state, crtc);
 	const struct intel_crtc_state *new_crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
-	u8 update_planes = new_crtc_state->update_planes;
+	u8 disable_async_flip_planes = old_crtc_state->async_flip_planes &
+				       ~new_crtc_state->async_flip_planes;
 	const struct intel_plane_state *old_plane_state;
 	struct intel_plane *plane;
 	bool need_vbl_wait = false;
@@ -1264,7 +1171,7 @@ static void intel_crtc_async_flip_disable_wa(struct intel_atomic_state *state,
 	for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
 		if (plane->need_async_flip_disable_wa &&
 		    plane->pipe == crtc->pipe &&
-		    update_planes & BIT(plane->id)) {
+		    disable_async_flip_planes & BIT(plane->id)) {
 			/*
 			 * Apart from the async flip bit we want to
 			 * preserve the old state for the plane.
@@ -1381,7 +1288,7 @@ static void intel_pre_plane_update(struct intel_atomic_state *state,
 	 * WA for platforms where async address update enable bit
 	 * is double buffered and only latched at start of vblank.
 	 */
-	if (old_crtc_state->uapi.async_flip && !new_crtc_state->uapi.async_flip)
+	if (old_crtc_state->async_flip_planes & ~new_crtc_state->async_flip_planes)
 		intel_crtc_async_flip_disable_wa(state, crtc);
 }
 
@@ -1413,36 +1320,11 @@ static void intel_crtc_disable_planes(struct intel_atomic_state *state,
 	intel_frontbuffer_flip(dev_priv, fb_bits);
 }
 
-/*
- * intel_connector_primary_encoder - get the primary encoder for a connector
- * @connector: connector for which to return the encoder
- *
- * Returns the primary encoder for a connector. There is a 1:1 mapping from
- * all connectors to their encoder, except for DP-MST connectors which have
- * both a virtual and a primary encoder. These DP-MST primary encoders can be
- * pointed to by as many DP-MST connectors as there are pipes.
- */
-static struct intel_encoder *
-intel_connector_primary_encoder(struct intel_connector *connector)
-{
-	struct intel_encoder *encoder;
-
-	if (connector->mst_port)
-		return &dp_to_dig_port(connector->mst_port)->base;
-
-	encoder = intel_attached_encoder(connector);
-	drm_WARN_ON(connector->base.dev, !encoder);
-
-	return encoder;
-}
-
 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
 {
 	struct drm_i915_private *i915 = to_i915(state->base.dev);
 	struct intel_crtc_state *new_crtc_state, *old_crtc_state;
 	struct intel_crtc *crtc;
-	struct drm_connector_state *new_conn_state;
-	struct drm_connector *connector;
 	int i;
 
 	/*
@@ -1458,57 +1340,6 @@ static void intel_encoders_update_prepare(struct intel_atomic_state *state)
 			new_crtc_state->dpll_hw_state = old_crtc_state->dpll_hw_state;
 		}
 	}
-
-	if (!state->modeset)
-		return;
-
-	for_each_new_connector_in_state(&state->base, connector, new_conn_state,
-					i) {
-		struct intel_connector *intel_connector;
-		struct intel_encoder *encoder;
-		struct intel_crtc *crtc;
-
-		if (!intel_connector_needs_modeset(state, connector))
-			continue;
-
-		intel_connector = to_intel_connector(connector);
-		encoder = intel_connector_primary_encoder(intel_connector);
-		if (!encoder->update_prepare)
-			continue;
-
-		crtc = new_conn_state->crtc ?
-			to_intel_crtc(new_conn_state->crtc) : NULL;
-		encoder->update_prepare(state, encoder, crtc);
-	}
-}
-
-static void intel_encoders_update_complete(struct intel_atomic_state *state)
-{
-	struct drm_connector_state *new_conn_state;
-	struct drm_connector *connector;
-	int i;
-
-	if (!state->modeset)
-		return;
-
-	for_each_new_connector_in_state(&state->base, connector, new_conn_state,
-					i) {
-		struct intel_connector *intel_connector;
-		struct intel_encoder *encoder;
-		struct intel_crtc *crtc;
-
-		if (!intel_connector_needs_modeset(state, connector))
-			continue;
-
-		intel_connector = to_intel_connector(connector);
-		encoder = intel_connector_primary_encoder(intel_connector);
-		if (!encoder->update_complete)
-			continue;
-
-		crtc = new_conn_state->crtc ?
-			to_intel_crtc(new_conn_state->crtc) : NULL;
-		encoder->update_complete(state, encoder, crtc);
-	}
 }
 
 static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state,
@@ -1804,12 +1635,10 @@ static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
 	enum transcoder transcoder = crtc_state->cpu_transcoder;
 	i915_reg_t reg = DISPLAY_VER(dev_priv) >= 14 ? MTL_CHICKEN_TRANS(transcoder) :
 			 CHICKEN_TRANS(transcoder);
-	u32 val;
 
-	val = intel_de_read(dev_priv, reg);
-	val &= ~HSW_FRAME_START_DELAY_MASK;
-	val |= HSW_FRAME_START_DELAY(crtc_state->framestart_delay - 1);
-	intel_de_write(dev_priv, reg, val);
+	intel_de_rmw(dev_priv, reg,
+		     HSW_FRAME_START_DELAY_MASK,
+		     HSW_FRAME_START_DELAY(crtc_state->framestart_delay - 1));
 }
 
 static void icl_ddi_bigjoiner_pre_enable(struct intel_atomic_state *state,
@@ -1849,7 +1678,7 @@ static void hsw_configure_cpu_transcoder(const struct intel_crtc_state *crtc_sta
 	intel_set_transcoder_timings(crtc_state);
 
 	if (cpu_transcoder != TRANSCODER_EDP)
-		intel_de_write(dev_priv, PIPE_MULT(cpu_transcoder),
+		intel_de_write(dev_priv, TRANS_MULT(cpu_transcoder),
 			       crtc_state->pixel_multiplier - 1);
 
 	hsw_set_frame_start_delay(crtc_state);
@@ -1890,7 +1719,7 @@ static void hsw_crtc_enable(struct intel_atomic_state *state,
 
 	intel_set_pipe_src_size(new_crtc_state);
 	if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
-		bdw_set_pipemisc(new_crtc_state);
+		bdw_set_pipe_misc(new_crtc_state);
 
 	if (!intel_crtc_is_bigjoiner_slave(new_crtc_state) &&
 	    !transcoder_is_dsi(cpu_transcoder))
@@ -2000,6 +1829,8 @@ static void ilk_crtc_disable(struct intel_atomic_state *state,
 
 	intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 	intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
+
+	intel_disable_shared_dpll(old_crtc_state);
 }
 
 static void hsw_crtc_disable(struct intel_atomic_state *state,
@@ -2018,6 +1849,10 @@ static void hsw_crtc_disable(struct intel_atomic_state *state,
 		intel_encoders_post_disable(state, crtc);
 	}
 
+	intel_disable_shared_dpll(old_crtc_state);
+
+	intel_encoders_post_pll_disable(state, crtc);
+
 	intel_dmc_disable_pipe(i915, crtc->pipe);
 }
 
@@ -2236,6 +2071,8 @@ static void valleyview_crtc_enable(struct intel_atomic_state *state,
 
 	intel_set_pipe_src_size(new_crtc_state);
 
+	intel_de_write(dev_priv, VLV_PIPE_MSA_MISC(pipe), 0);
+
 	if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
 		intel_de_write(dev_priv, CHV_BLEND(pipe), CHV_BLEND_LEGACY);
 		intel_de_write(dev_priv, CHV_CANVAS(pipe), 0);
@@ -2822,12 +2659,14 @@ static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_sta
 	enum pipe pipe = crtc->pipe;
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
-	u32 crtc_vtotal, crtc_vblank_end;
+	u32 crtc_vdisplay, crtc_vtotal, crtc_vblank_start, crtc_vblank_end;
 	int vsyncshift = 0;
 
 	/* We need to be careful not to changed the adjusted mode, for otherwise
 	 * the hw state checker will get angry at the mismatch. */
+	crtc_vdisplay = adjusted_mode->crtc_vdisplay;
 	crtc_vtotal = adjusted_mode->crtc_vtotal;
+	crtc_vblank_start = adjusted_mode->crtc_vblank_start;
 	crtc_vblank_end = adjusted_mode->crtc_vblank_end;
 
 	if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
@@ -2844,23 +2683,44 @@ static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_sta
 			vsyncshift += adjusted_mode->crtc_htotal;
 	}
 
+	/*
+	 * VBLANK_START no longer works on ADL+, instead we must use
+	 * TRANS_SET_CONTEXT_LATENCY to configure the pipe vblank start.
+	 */
+	if (DISPLAY_VER(dev_priv) >= 13) {
+		intel_de_write(dev_priv, TRANS_SET_CONTEXT_LATENCY(cpu_transcoder),
+			       crtc_vblank_start - crtc_vdisplay);
+
+		/*
+		 * VBLANK_START not used by hw, just clear it
+		 * to make it stand out in register dumps.
+		 */
+		crtc_vblank_start = 1;
+	}
+
 	if (DISPLAY_VER(dev_priv) > 3)
-		intel_de_write(dev_priv, VSYNCSHIFT(cpu_transcoder),
-		               vsyncshift);
-
-	intel_de_write(dev_priv, HTOTAL(cpu_transcoder),
-		       (adjusted_mode->crtc_hdisplay - 1) | ((adjusted_mode->crtc_htotal - 1) << 16));
-	intel_de_write(dev_priv, HBLANK(cpu_transcoder),
-		       (adjusted_mode->crtc_hblank_start - 1) | ((adjusted_mode->crtc_hblank_end - 1) << 16));
-	intel_de_write(dev_priv, HSYNC(cpu_transcoder),
-		       (adjusted_mode->crtc_hsync_start - 1) | ((adjusted_mode->crtc_hsync_end - 1) << 16));
-
-	intel_de_write(dev_priv, VTOTAL(cpu_transcoder),
-		       (adjusted_mode->crtc_vdisplay - 1) | ((crtc_vtotal - 1) << 16));
-	intel_de_write(dev_priv, VBLANK(cpu_transcoder),
-		       (adjusted_mode->crtc_vblank_start - 1) | ((crtc_vblank_end - 1) << 16));
-	intel_de_write(dev_priv, VSYNC(cpu_transcoder),
-		       (adjusted_mode->crtc_vsync_start - 1) | ((adjusted_mode->crtc_vsync_end - 1) << 16));
+		intel_de_write(dev_priv, TRANS_VSYNCSHIFT(cpu_transcoder),
+			       vsyncshift);
+
+	intel_de_write(dev_priv, TRANS_HTOTAL(cpu_transcoder),
+		       HACTIVE(adjusted_mode->crtc_hdisplay - 1) |
+		       HTOTAL(adjusted_mode->crtc_htotal - 1));
+	intel_de_write(dev_priv, TRANS_HBLANK(cpu_transcoder),
+		       HBLANK_START(adjusted_mode->crtc_hblank_start - 1) |
+		       HBLANK_END(adjusted_mode->crtc_hblank_end - 1));
+	intel_de_write(dev_priv, TRANS_HSYNC(cpu_transcoder),
+		       HSYNC_START(adjusted_mode->crtc_hsync_start - 1) |
+		       HSYNC_END(adjusted_mode->crtc_hsync_end - 1));
+
+	intel_de_write(dev_priv, TRANS_VTOTAL(cpu_transcoder),
+		       VACTIVE(crtc_vdisplay - 1) |
+		       VTOTAL(crtc_vtotal - 1));
+	intel_de_write(dev_priv, TRANS_VBLANK(cpu_transcoder),
+		       VBLANK_START(crtc_vblank_start - 1) |
+		       VBLANK_END(crtc_vblank_end - 1));
+	intel_de_write(dev_priv, TRANS_VSYNC(cpu_transcoder),
+		       VSYNC_START(adjusted_mode->crtc_vsync_start - 1) |
+		       VSYNC_END(adjusted_mode->crtc_vsync_end - 1));
 
 	/* Workaround: when the EDP input selection is B, the VTOTAL_B must be
 	 * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
@@ -2868,9 +2728,9 @@ static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_sta
 	 * bits. */
 	if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
 	    (pipe == PIPE_B || pipe == PIPE_C))
-		intel_de_write(dev_priv, VTOTAL(pipe),
-		               intel_de_read(dev_priv, VTOTAL(cpu_transcoder)));
-
+		intel_de_write(dev_priv, TRANS_VTOTAL(pipe),
+			       VACTIVE(crtc_vdisplay - 1) |
+			       VTOTAL(crtc_vtotal - 1));
 }
 
 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
@@ -2898,9 +2758,9 @@ static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state)
 
 	if (DISPLAY_VER(dev_priv) >= 9 ||
 	    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
-		return intel_de_read(dev_priv, PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK_HSW;
+		return intel_de_read(dev_priv, TRANSCONF(cpu_transcoder)) & TRANSCONF_INTERLACE_MASK_HSW;
 	else
-		return intel_de_read(dev_priv, PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK;
+		return intel_de_read(dev_priv, TRANSCONF(cpu_transcoder)) & TRANSCONF_INTERLACE_MASK;
 }
 
 static void intel_get_transcoder_timings(struct intel_crtc *crtc,
@@ -2909,43 +2769,47 @@ static void intel_get_transcoder_timings(struct intel_crtc *crtc,
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
+	struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
 	u32 tmp;
 
-	tmp = intel_de_read(dev_priv, HTOTAL(cpu_transcoder));
-	pipe_config->hw.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
-	pipe_config->hw.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
+	tmp = intel_de_read(dev_priv, TRANS_HTOTAL(cpu_transcoder));
+	adjusted_mode->crtc_hdisplay = REG_FIELD_GET(HACTIVE_MASK, tmp) + 1;
+	adjusted_mode->crtc_htotal = REG_FIELD_GET(HTOTAL_MASK, tmp) + 1;
 
 	if (!transcoder_is_dsi(cpu_transcoder)) {
-		tmp = intel_de_read(dev_priv, HBLANK(cpu_transcoder));
-		pipe_config->hw.adjusted_mode.crtc_hblank_start =
-							(tmp & 0xffff) + 1;
-		pipe_config->hw.adjusted_mode.crtc_hblank_end =
-						((tmp >> 16) & 0xffff) + 1;
+		tmp = intel_de_read(dev_priv, TRANS_HBLANK(cpu_transcoder));
+		adjusted_mode->crtc_hblank_start = REG_FIELD_GET(HBLANK_START_MASK, tmp) + 1;
+		adjusted_mode->crtc_hblank_end = REG_FIELD_GET(HBLANK_END_MASK, tmp) + 1;
 	}
-	tmp = intel_de_read(dev_priv, HSYNC(cpu_transcoder));
-	pipe_config->hw.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
-	pipe_config->hw.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
 
-	tmp = intel_de_read(dev_priv, VTOTAL(cpu_transcoder));
-	pipe_config->hw.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
-	pipe_config->hw.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
+	tmp = intel_de_read(dev_priv, TRANS_HSYNC(cpu_transcoder));
+	adjusted_mode->crtc_hsync_start = REG_FIELD_GET(HSYNC_START_MASK, tmp) + 1;
+	adjusted_mode->crtc_hsync_end = REG_FIELD_GET(HSYNC_END_MASK, tmp) + 1;
+
+	tmp = intel_de_read(dev_priv, TRANS_VTOTAL(cpu_transcoder));
+	adjusted_mode->crtc_vdisplay = REG_FIELD_GET(VACTIVE_MASK, tmp) + 1;
+	adjusted_mode->crtc_vtotal = REG_FIELD_GET(VTOTAL_MASK, tmp) + 1;
 
+	/* FIXME TGL+ DSI transcoders have this! */
 	if (!transcoder_is_dsi(cpu_transcoder)) {
-		tmp = intel_de_read(dev_priv, VBLANK(cpu_transcoder));
-		pipe_config->hw.adjusted_mode.crtc_vblank_start =
-							(tmp & 0xffff) + 1;
-		pipe_config->hw.adjusted_mode.crtc_vblank_end =
-						((tmp >> 16) & 0xffff) + 1;
+		tmp = intel_de_read(dev_priv, TRANS_VBLANK(cpu_transcoder));
+		adjusted_mode->crtc_vblank_start = REG_FIELD_GET(VBLANK_START_MASK, tmp) + 1;
+		adjusted_mode->crtc_vblank_end = REG_FIELD_GET(VBLANK_END_MASK, tmp) + 1;
 	}
-	tmp = intel_de_read(dev_priv, VSYNC(cpu_transcoder));
-	pipe_config->hw.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
-	pipe_config->hw.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
+	tmp = intel_de_read(dev_priv, TRANS_VSYNC(cpu_transcoder));
+	adjusted_mode->crtc_vsync_start = REG_FIELD_GET(VSYNC_START_MASK, tmp) + 1;
+	adjusted_mode->crtc_vsync_end = REG_FIELD_GET(VSYNC_END_MASK, tmp) + 1;
 
 	if (intel_pipe_is_interlaced(pipe_config)) {
-		pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
-		pipe_config->hw.adjusted_mode.crtc_vtotal += 1;
-		pipe_config->hw.adjusted_mode.crtc_vblank_end += 1;
+		adjusted_mode->flags |= DRM_MODE_FLAG_INTERLACE;
+		adjusted_mode->crtc_vtotal += 1;
+		adjusted_mode->crtc_vblank_end += 1;
 	}
+
+	if (DISPLAY_VER(dev_priv) >= 13 && !transcoder_is_dsi(cpu_transcoder))
+		adjusted_mode->crtc_vblank_start =
+			adjusted_mode->crtc_vdisplay +
+			intel_de_read(dev_priv, TRANS_SET_CONTEXT_LATENCY(cpu_transcoder));
 }
 
 static void intel_bigjoiner_adjust_pipe_src(struct intel_crtc_state *crtc_state)
@@ -2985,7 +2849,8 @@ void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	u32 pipeconf = 0;
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	u32 val = 0;
 
 	/*
 	 * - We keep both pipes enabled on 830
@@ -2993,18 +2858,18 @@ void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
 	 * - During fastset the pipe is already enabled and must remain so
 	 */
 	if (IS_I830(dev_priv) || !intel_crtc_needs_modeset(crtc_state))
-		pipeconf |= PIPECONF_ENABLE;
+		val |= TRANSCONF_ENABLE;
 
 	if (crtc_state->double_wide)
-		pipeconf |= PIPECONF_DOUBLE_WIDE;
+		val |= TRANSCONF_DOUBLE_WIDE;
 
 	/* only g4x and later have fancy bpc/dither controls */
 	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
 	    IS_CHERRYVIEW(dev_priv)) {
 		/* Bspec claims that we can't use dithering for 30bpp pipes. */
 		if (crtc_state->dither && crtc_state->pipe_bpp != 30)
-			pipeconf |= PIPECONF_DITHER_EN |
-				    PIPECONF_DITHER_TYPE_SP;
+			val |= TRANSCONF_DITHER_EN |
+				TRANSCONF_DITHER_TYPE_SP;
 
 		switch (crtc_state->pipe_bpp) {
 		default:
@@ -3012,13 +2877,13 @@ void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
 			MISSING_CASE(crtc_state->pipe_bpp);
 			fallthrough;
 		case 18:
-			pipeconf |= PIPECONF_BPC_6;
+			val |= TRANSCONF_BPC_6;
 			break;
 		case 24:
-			pipeconf |= PIPECONF_BPC_8;
+			val |= TRANSCONF_BPC_8;
 			break;
 		case 30:
-			pipeconf |= PIPECONF_BPC_10;
+			val |= TRANSCONF_BPC_10;
 			break;
 		}
 	}
@@ -3026,23 +2891,23 @@ void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
 	if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
 		if (DISPLAY_VER(dev_priv) < 4 ||
 		    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
-			pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
+			val |= TRANSCONF_INTERLACE_W_FIELD_INDICATION;
 		else
-			pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
+			val |= TRANSCONF_INTERLACE_W_SYNC_SHIFT;
 	} else {
-		pipeconf |= PIPECONF_INTERLACE_PROGRESSIVE;
+		val |= TRANSCONF_INTERLACE_PROGRESSIVE;
 	}
 
 	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 	     crtc_state->limited_color_range)
-		pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
+		val |= TRANSCONF_COLOR_RANGE_SELECT;
 
-	pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
+	val |= TRANSCONF_GAMMA_MODE(crtc_state->gamma_mode);
 
-	pipeconf |= PIPECONF_FRAME_START_DELAY(crtc_state->framestart_delay - 1);
+	val |= TRANSCONF_FRAME_START_DELAY(crtc_state->framestart_delay - 1);
 
-	intel_de_write(dev_priv, PIPECONF(crtc->pipe), pipeconf);
-	intel_de_posting_read(dev_priv, PIPECONF(crtc->pipe));
+	intel_de_write(dev_priv, TRANSCONF(cpu_transcoder), val);
+	intel_de_posting_read(dev_priv, TRANSCONF(cpu_transcoder));
 }
 
 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
@@ -3143,20 +3008,20 @@ static void chv_crtc_clock_get(struct intel_crtc *crtc,
 }
 
 static enum intel_output_format
-bdw_get_pipemisc_output_format(struct intel_crtc *crtc)
+bdw_get_pipe_misc_output_format(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 tmp;
 
-	tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe));
+	tmp = intel_de_read(dev_priv, PIPE_MISC(crtc->pipe));
 
-	if (tmp & PIPEMISC_YUV420_ENABLE) {
+	if (tmp & PIPE_MISC_YUV420_ENABLE) {
 		/* We support 4:2:0 in full blend mode only */
 		drm_WARN_ON(&dev_priv->drm,
-			    (tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0);
+			    (tmp & PIPE_MISC_YUV420_MODE_FULL_BLEND) == 0);
 
 		return INTEL_OUTPUT_FORMAT_YCBCR420;
-	} else if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
+	} else if (tmp & PIPE_MISC_OUTPUT_COLORSPACE_YUV) {
 		return INTEL_OUTPUT_FORMAT_YCBCR444;
 	} else {
 		return INTEL_OUTPUT_FORMAT_RGB;
@@ -3201,20 +3066,20 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 
 	ret = false;
 
-	tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe));
-	if (!(tmp & PIPECONF_ENABLE))
+	tmp = intel_de_read(dev_priv, TRANSCONF(pipe_config->cpu_transcoder));
+	if (!(tmp & TRANSCONF_ENABLE))
 		goto out;
 
 	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
 	    IS_CHERRYVIEW(dev_priv)) {
-		switch (tmp & PIPECONF_BPC_MASK) {
-		case PIPECONF_BPC_6:
+		switch (tmp & TRANSCONF_BPC_MASK) {
+		case TRANSCONF_BPC_6:
 			pipe_config->pipe_bpp = 18;
 			break;
-		case PIPECONF_BPC_8:
+		case TRANSCONF_BPC_8:
 			pipe_config->pipe_bpp = 24;
 			break;
-		case PIPECONF_BPC_10:
+		case TRANSCONF_BPC_10:
 			pipe_config->pipe_bpp = 30;
 			break;
 		default:
@@ -3224,12 +3089,12 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 	}
 
 	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
-	    (tmp & PIPECONF_COLOR_RANGE_SELECT))
+	    (tmp & TRANSCONF_COLOR_RANGE_SELECT))
 		pipe_config->limited_color_range = true;
 
-	pipe_config->gamma_mode = REG_FIELD_GET(PIPECONF_GAMMA_MODE_MASK_I9XX, tmp);
+	pipe_config->gamma_mode = REG_FIELD_GET(TRANSCONF_GAMMA_MODE_MASK_I9XX, tmp);
 
-	pipe_config->framestart_delay = REG_FIELD_GET(PIPECONF_FRAME_START_DELAY_MASK, tmp) + 1;
+	pipe_config->framestart_delay = REG_FIELD_GET(TRANSCONF_FRAME_START_DELAY_MASK, tmp) + 1;
 
 	if (IS_CHERRYVIEW(dev_priv))
 		pipe_config->cgm_mode = intel_de_read(dev_priv,
@@ -3239,7 +3104,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 	intel_color_get_config(pipe_config);
 
 	if (DISPLAY_VER(dev_priv) < 4)
-		pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
+		pipe_config->double_wide = tmp & TRANSCONF_DOUBLE_WIDE;
 
 	intel_get_transcoder_timings(crtc, pipe_config);
 	intel_get_pipe_src_size(crtc, pipe_config);
@@ -3309,7 +3174,7 @@ void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum pipe pipe = crtc->pipe;
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	u32 val = 0;
 
 	/*
@@ -3317,7 +3182,7 @@ void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
 	 * - During fastset the pipe is already enabled and must remain so
 	 */
 	if (!intel_crtc_needs_modeset(crtc_state))
-		val |= PIPECONF_ENABLE;
+		val |= TRANSCONF_ENABLE;
 
 	switch (crtc_state->pipe_bpp) {
 	default:
@@ -3325,26 +3190,26 @@ void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
 		MISSING_CASE(crtc_state->pipe_bpp);
 		fallthrough;
 	case 18:
-		val |= PIPECONF_BPC_6;
+		val |= TRANSCONF_BPC_6;
 		break;
 	case 24:
-		val |= PIPECONF_BPC_8;
+		val |= TRANSCONF_BPC_8;
 		break;
 	case 30:
-		val |= PIPECONF_BPC_10;
+		val |= TRANSCONF_BPC_10;
 		break;
 	case 36:
-		val |= PIPECONF_BPC_12;
+		val |= TRANSCONF_BPC_12;
 		break;
 	}
 
 	if (crtc_state->dither)
-		val |= PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP;
+		val |= TRANSCONF_DITHER_EN | TRANSCONF_DITHER_TYPE_SP;
 
 	if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
-		val |= PIPECONF_INTERLACE_IF_ID_ILK;
+		val |= TRANSCONF_INTERLACE_IF_ID_ILK;
 	else
-		val |= PIPECONF_INTERLACE_PF_PD_ILK;
+		val |= TRANSCONF_INTERLACE_PF_PD_ILK;
 
 	/*
 	 * This would end up with an odd purple hue over
@@ -3355,18 +3220,18 @@ void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
 
 	if (crtc_state->limited_color_range &&
 	    !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
-		val |= PIPECONF_COLOR_RANGE_SELECT;
+		val |= TRANSCONF_COLOR_RANGE_SELECT;
 
 	if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
-		val |= PIPECONF_OUTPUT_COLORSPACE_YUV709;
+		val |= TRANSCONF_OUTPUT_COLORSPACE_YUV709;
 
-	val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
+	val |= TRANSCONF_GAMMA_MODE(crtc_state->gamma_mode);
 
-	val |= PIPECONF_FRAME_START_DELAY(crtc_state->framestart_delay - 1);
-	val |= PIPECONF_MSA_TIMING_DELAY(crtc_state->msa_timing_delay);
+	val |= TRANSCONF_FRAME_START_DELAY(crtc_state->framestart_delay - 1);
+	val |= TRANSCONF_MSA_TIMING_DELAY(crtc_state->msa_timing_delay);
 
-	intel_de_write(dev_priv, PIPECONF(pipe), val);
-	intel_de_posting_read(dev_priv, PIPECONF(pipe));
+	intel_de_write(dev_priv, TRANSCONF(cpu_transcoder), val);
+	intel_de_posting_read(dev_priv, TRANSCONF(cpu_transcoder));
 }
 
 static void hsw_set_transconf(const struct intel_crtc_state *crtc_state)
@@ -3381,25 +3246,25 @@ static void hsw_set_transconf(const struct intel_crtc_state *crtc_state)
 	 * - During fastset the pipe is already enabled and must remain so
 	 */
 	if (!intel_crtc_needs_modeset(crtc_state))
-		val |= PIPECONF_ENABLE;
+		val |= TRANSCONF_ENABLE;
 
 	if (IS_HASWELL(dev_priv) && crtc_state->dither)
-		val |= PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP;
+		val |= TRANSCONF_DITHER_EN | TRANSCONF_DITHER_TYPE_SP;
 
 	if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
-		val |= PIPECONF_INTERLACE_IF_ID_ILK;
+		val |= TRANSCONF_INTERLACE_IF_ID_ILK;
 	else
-		val |= PIPECONF_INTERLACE_PF_PD_ILK;
+		val |= TRANSCONF_INTERLACE_PF_PD_ILK;
 
 	if (IS_HASWELL(dev_priv) &&
 	    crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
-		val |= PIPECONF_OUTPUT_COLORSPACE_YUV_HSW;
+		val |= TRANSCONF_OUTPUT_COLORSPACE_YUV_HSW;
 
-	intel_de_write(dev_priv, PIPECONF(cpu_transcoder), val);
-	intel_de_posting_read(dev_priv, PIPECONF(cpu_transcoder));
+	intel_de_write(dev_priv, TRANSCONF(cpu_transcoder), val);
+	intel_de_posting_read(dev_priv, TRANSCONF(cpu_transcoder));
 }
 
-static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
+static void bdw_set_pipe_misc(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
@@ -3407,18 +3272,18 @@ static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
 
 	switch (crtc_state->pipe_bpp) {
 	case 18:
-		val |= PIPEMISC_BPC_6;
+		val |= PIPE_MISC_BPC_6;
 		break;
 	case 24:
-		val |= PIPEMISC_BPC_8;
+		val |= PIPE_MISC_BPC_8;
 		break;
 	case 30:
-		val |= PIPEMISC_BPC_10;
+		val |= PIPE_MISC_BPC_10;
 		break;
 	case 36:
 		/* Port output 12BPC defined for ADLP+ */
 		if (DISPLAY_VER(dev_priv) > 12)
-			val |= PIPEMISC_BPC_12_ADLP;
+			val |= PIPE_MISC_BPC_12_ADLP;
 		break;
 	default:
 		MISSING_CASE(crtc_state->pipe_bpp);
@@ -3426,38 +3291,38 @@ static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
 	}
 
 	if (crtc_state->dither)
-		val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
+		val |= PIPE_MISC_DITHER_ENABLE | PIPE_MISC_DITHER_TYPE_SP;
 
 	if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
 	    crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
-		val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
+		val |= PIPE_MISC_OUTPUT_COLORSPACE_YUV;
 
 	if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
-		val |= PIPEMISC_YUV420_ENABLE |
-			PIPEMISC_YUV420_MODE_FULL_BLEND;
+		val |= PIPE_MISC_YUV420_ENABLE |
+			PIPE_MISC_YUV420_MODE_FULL_BLEND;
 
 	if (DISPLAY_VER(dev_priv) >= 11 && is_hdr_mode(crtc_state))
-		val |= PIPEMISC_HDR_MODE_PRECISION;
+		val |= PIPE_MISC_HDR_MODE_PRECISION;
 
 	if (DISPLAY_VER(dev_priv) >= 12)
-		val |= PIPEMISC_PIXEL_ROUNDING_TRUNC;
+		val |= PIPE_MISC_PIXEL_ROUNDING_TRUNC;
 
-	intel_de_write(dev_priv, PIPEMISC(crtc->pipe), val);
+	intel_de_write(dev_priv, PIPE_MISC(crtc->pipe), val);
 }
 
-int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
+int bdw_get_pipe_misc_bpp(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 tmp;
 
-	tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe));
+	tmp = intel_de_read(dev_priv, PIPE_MISC(crtc->pipe));
 
-	switch (tmp & PIPEMISC_BPC_MASK) {
-	case PIPEMISC_BPC_6:
+	switch (tmp & PIPE_MISC_BPC_MASK) {
+	case PIPE_MISC_BPC_6:
 		return 18;
-	case PIPEMISC_BPC_8:
+	case PIPE_MISC_BPC_8:
 		return 24;
-	case PIPEMISC_BPC_10:
+	case PIPE_MISC_BPC_10:
 		return 30;
 	/*
 	 * PORT OUTPUT 12 BPC defined for ADLP+.
@@ -3469,7 +3334,7 @@ int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
 	 * on older platforms, need to find a workaround for 12 BPC
 	 * MIPI DSI HW readout.
 	 */
-	case PIPEMISC_BPC_12_ADLP:
+	case PIPE_MISC_BPC_12_ADLP:
 		if (DISPLAY_VER(dev_priv) > 12)
 			return 36;
 		fallthrough;
@@ -3621,33 +3486,33 @@ static bool ilk_get_pipe_config(struct intel_crtc *crtc,
 	pipe_config->shared_dpll = NULL;
 
 	ret = false;
-	tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe));
-	if (!(tmp & PIPECONF_ENABLE))
+	tmp = intel_de_read(dev_priv, TRANSCONF(pipe_config->cpu_transcoder));
+	if (!(tmp & TRANSCONF_ENABLE))
 		goto out;
 
-	switch (tmp & PIPECONF_BPC_MASK) {
-	case PIPECONF_BPC_6:
+	switch (tmp & TRANSCONF_BPC_MASK) {
+	case TRANSCONF_BPC_6:
 		pipe_config->pipe_bpp = 18;
 		break;
-	case PIPECONF_BPC_8:
+	case TRANSCONF_BPC_8:
 		pipe_config->pipe_bpp = 24;
 		break;
-	case PIPECONF_BPC_10:
+	case TRANSCONF_BPC_10:
 		pipe_config->pipe_bpp = 30;
 		break;
-	case PIPECONF_BPC_12:
+	case TRANSCONF_BPC_12:
 		pipe_config->pipe_bpp = 36;
 		break;
 	default:
 		break;
 	}
 
-	if (tmp & PIPECONF_COLOR_RANGE_SELECT)
+	if (tmp & TRANSCONF_COLOR_RANGE_SELECT)
 		pipe_config->limited_color_range = true;
 
-	switch (tmp & PIPECONF_OUTPUT_COLORSPACE_MASK) {
-	case PIPECONF_OUTPUT_COLORSPACE_YUV601:
-	case PIPECONF_OUTPUT_COLORSPACE_YUV709:
+	switch (tmp & TRANSCONF_OUTPUT_COLORSPACE_MASK) {
+	case TRANSCONF_OUTPUT_COLORSPACE_YUV601:
+	case TRANSCONF_OUTPUT_COLORSPACE_YUV709:
 		pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
 		break;
 	default:
@@ -3655,11 +3520,11 @@ static bool ilk_get_pipe_config(struct intel_crtc *crtc,
 		break;
 	}
 
-	pipe_config->gamma_mode = REG_FIELD_GET(PIPECONF_GAMMA_MODE_MASK_ILK, tmp);
+	pipe_config->gamma_mode = REG_FIELD_GET(TRANSCONF_GAMMA_MODE_MASK_ILK, tmp);
 
-	pipe_config->framestart_delay = REG_FIELD_GET(PIPECONF_FRAME_START_DELAY_MASK, tmp) + 1;
+	pipe_config->framestart_delay = REG_FIELD_GET(TRANSCONF_FRAME_START_DELAY_MASK, tmp) + 1;
 
-	pipe_config->msa_timing_delay = REG_FIELD_GET(PIPECONF_MSA_TIMING_DELAY_MASK, tmp);
+	pipe_config->msa_timing_delay = REG_FIELD_GET(TRANSCONF_MSA_TIMING_DELAY_MASK, tmp);
 
 	pipe_config->csc_mode = intel_de_read(dev_priv,
 					      PIPE_CSC_MODE(crtc->pipe));
@@ -3936,9 +3801,9 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
 			pipe_config->pch_pfit.force_thru = true;
 	}
 
-	tmp = intel_de_read(dev_priv, PIPECONF(pipe_config->cpu_transcoder));
+	tmp = intel_de_read(dev_priv, TRANSCONF(pipe_config->cpu_transcoder));
 
-	return tmp & PIPECONF_ENABLE;
+	return tmp & TRANSCONF_ENABLE;
 }
 
 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
@@ -4042,15 +3907,15 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc,
 
 	if (IS_HASWELL(dev_priv)) {
 		u32 tmp = intel_de_read(dev_priv,
-					PIPECONF(pipe_config->cpu_transcoder));
+					TRANSCONF(pipe_config->cpu_transcoder));
 
-		if (tmp & PIPECONF_OUTPUT_COLORSPACE_YUV_HSW)
+		if (tmp & TRANSCONF_OUTPUT_COLORSPACE_YUV_HSW)
 			pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
 		else
 			pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
 	} else {
 		pipe_config->output_format =
-			bdw_get_pipemisc_output_format(crtc);
+			bdw_get_pipe_misc_output_format(crtc);
 	}
 
 	pipe_config->gamma_mode = intel_de_read(dev_priv,
@@ -4093,7 +3958,7 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc,
 	    !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
 		pipe_config->pixel_multiplier =
 			intel_de_read(dev_priv,
-				      PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
+				      TRANS_MULT(pipe_config->cpu_transcoder)) + 1;
 	} else {
 		pipe_config->pixel_multiplier = 1;
 	}
@@ -5443,6 +5308,20 @@ pipe_config_dp_vsc_sdp_mismatch(struct drm_i915_private *dev_priv,
 	}
 }
 
+/* Returns the length up to and including the last differing byte */
+static size_t
+memcmp_diff_len(const u8 *a, const u8 *b, size_t len)
+{
+	int i;
+
+	for (i = len - 1; i >= 0; i--) {
+		if (a[i] != b[i])
+			return i + 1;
+	}
+
+	return 0;
+}
+
 static void
 pipe_config_buffer_mismatch(struct drm_i915_private *dev_priv,
 			    bool fastset, const char *name,
@@ -5452,6 +5331,9 @@ pipe_config_buffer_mismatch(struct drm_i915_private *dev_priv,
 		if (!drm_debug_enabled(DRM_UT_KMS))
 			return;
 
+		/* only dump up to the last difference */
+		len = memcmp_diff_len(a, b, len);
+
 		drm_dbg_kms(&dev_priv->drm,
 			    "fastset mismatch in %s buffer\n", name);
 		print_hex_dump(KERN_DEBUG, "expected: ", DUMP_PREFIX_NONE,
@@ -5459,6 +5341,9 @@ pipe_config_buffer_mismatch(struct drm_i915_private *dev_priv,
 		print_hex_dump(KERN_DEBUG, "found: ", DUMP_PREFIX_NONE,
 			       16, 0, b, len, false);
 	} else {
+		/* only dump up to the last difference */
+		len = memcmp_diff_len(a, b, len);
+
 		drm_err(&dev_priv->drm, "mismatch in %s buffer\n", name);
 		print_hex_dump(KERN_ERR, "expected: ", DUMP_PREFIX_NONE,
 			       16, 0, a, len, false);
@@ -5947,73 +5832,13 @@ int intel_modeset_all_pipes(struct intel_atomic_state *state,
 			return ret;
 
 		crtc_state->update_planes |= crtc_state->active_planes;
+		crtc_state->async_flip_planes = 0;
+		crtc_state->do_async_flip = false;
 	}
 
 	return 0;
 }
 
-void intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
-{
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct drm_display_mode adjusted_mode;
-
-	drm_mode_init(&adjusted_mode, &crtc_state->hw.adjusted_mode);
-
-	if (crtc_state->vrr.enable) {
-		adjusted_mode.crtc_vtotal = crtc_state->vrr.vmax;
-		adjusted_mode.crtc_vblank_end = crtc_state->vrr.vmax;
-		adjusted_mode.crtc_vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
-		crtc->vmax_vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
-	}
-
-	drm_calc_timestamping_constants(&crtc->base, &adjusted_mode);
-
-	crtc->mode_flags = crtc_state->mode_flags;
-
-	/*
-	 * The scanline counter increments at the leading edge of hsync.
-	 *
-	 * On most platforms it starts counting from vtotal-1 on the
-	 * first active line. That means the scanline counter value is
-	 * always one less than what we would expect. Ie. just after
-	 * start of vblank, which also occurs at start of hsync (on the
-	 * last active line), the scanline counter will read vblank_start-1.
-	 *
-	 * On gen2 the scanline counter starts counting from 1 instead
-	 * of vtotal-1, so we have to subtract one (or rather add vtotal-1
-	 * to keep the value positive), instead of adding one.
-	 *
-	 * On HSW+ the behaviour of the scanline counter depends on the output
-	 * type. For DP ports it behaves like most other platforms, but on HDMI
-	 * there's an extra 1 line difference. So we need to add two instead of
-	 * one to the value.
-	 *
-	 * On VLV/CHV DSI the scanline counter would appear to increment
-	 * approx. 1/3 of a scanline before start of vblank. Unfortunately
-	 * that means we can't tell whether we're in vblank or not while
-	 * we're on that particular line. We must still set scanline_offset
-	 * to 1 so that the vblank timestamps come out correct when we query
-	 * the scanline counter from within the vblank interrupt handler.
-	 * However if queried just before the start of vblank we'll get an
-	 * answer that's slightly in the future.
-	 */
-	if (DISPLAY_VER(dev_priv) == 2) {
-		int vtotal;
-
-		vtotal = adjusted_mode.crtc_vtotal;
-		if (adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
-			vtotal /= 2;
-
-		crtc->scanline_offset = vtotal - 1;
-	} else if (HAS_DDI(dev_priv) &&
-		   intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
-		crtc->scanline_offset = 2;
-	} else {
-		crtc->scanline_offset = 1;
-	}
-}
-
 /*
  * This implements the workaround described in the "notes" section of the mode
  * set sequence documentation. When going from no pipes or single pipe to
@@ -6699,8 +6524,8 @@ static int intel_bigjoiner_add_affected_crtcs(struct intel_atomic_state *state)
  * @dev: drm device
  * @_state: state to validate
  */
-static int intel_atomic_check(struct drm_device *dev,
-			      struct drm_atomic_state *_state)
+int intel_atomic_check(struct drm_device *dev,
+		       struct drm_atomic_state *_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_atomic_state *state = to_intel_atomic_state(_state);
@@ -7018,7 +6843,7 @@ static void commit_pipe_pre_planes(struct intel_atomic_state *state,
 			intel_color_commit_arm(new_crtc_state);
 
 		if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
-			bdw_set_pipemisc(new_crtc_state);
+			bdw_set_pipe_misc(new_crtc_state);
 
 		if (intel_crtc_needs_fastset(new_crtc_state))
 			intel_pipe_fastset(old_crtc_state, new_crtc_state);
@@ -7077,6 +6902,12 @@ static void intel_update_crtc(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	bool modeset = intel_crtc_needs_modeset(new_crtc_state);
 
+	if (old_crtc_state->inherited ||
+	    intel_crtc_needs_modeset(new_crtc_state)) {
+		if (HAS_DPT(i915))
+			intel_dpt_configure(crtc);
+	}
+
 	if (!modeset) {
 		if (new_crtc_state->preload_luts &&
 		    intel_crtc_needs_color_update(new_crtc_state))
@@ -7140,7 +6971,6 @@ static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
 	dev_priv->display.funcs.display->crtc_disable(state, crtc);
 	crtc->active = false;
 	intel_fbc_disable(crtc);
-	intel_disable_shared_dpll(old_crtc_state);
 
 	if (!new_crtc_state->hw.active)
 		intel_initial_watermarks(state, crtc);
@@ -7539,8 +7369,6 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state)
 	/* Now enable the clocks, plane, pipe, and connectors that we set up. */
 	dev_priv->display.funcs.display->commit_modeset_enables(state);
 
-	intel_encoders_update_complete(state);
-
 	if (state->modeset)
 		intel_set_cdclk_post_plane_update(state);
 
@@ -8382,124 +8210,6 @@ void intel_modeset_init_hw(struct drm_i915_private *i915)
 	cdclk_state->logical = cdclk_state->actual = i915->display.cdclk.hw;
 }
 
-static int sanitize_watermarks_add_affected(struct drm_atomic_state *state)
-{
-	struct drm_plane *plane;
-	struct intel_crtc *crtc;
-
-	for_each_intel_crtc(state->dev, crtc) {
-		struct intel_crtc_state *crtc_state;
-
-		crtc_state = intel_atomic_get_crtc_state(state, crtc);
-		if (IS_ERR(crtc_state))
-			return PTR_ERR(crtc_state);
-
-		if (crtc_state->hw.active) {
-			/*
-			 * Preserve the inherited flag to avoid
-			 * taking the full modeset path.
-			 */
-			crtc_state->inherited = true;
-		}
-	}
-
-	drm_for_each_plane(plane, state->dev) {
-		struct drm_plane_state *plane_state;
-
-		plane_state = drm_atomic_get_plane_state(state, plane);
-		if (IS_ERR(plane_state))
-			return PTR_ERR(plane_state);
-	}
-
-	return 0;
-}
-
-/*
- * Calculate what we think the watermarks should be for the state we've read
- * out of the hardware and then immediately program those watermarks so that
- * we ensure the hardware settings match our internal state.
- *
- * We can calculate what we think WM's should be by creating a duplicate of the
- * current state (which was constructed during hardware readout) and running it
- * through the atomic check code to calculate new watermark values in the
- * state object.
- */
-static void sanitize_watermarks(struct drm_i915_private *dev_priv)
-{
-	struct drm_atomic_state *state;
-	struct intel_atomic_state *intel_state;
-	struct intel_crtc *crtc;
-	struct intel_crtc_state *crtc_state;
-	struct drm_modeset_acquire_ctx ctx;
-	int ret;
-	int i;
-
-	/* Only supported on platforms that use atomic watermark design */
-	if (!dev_priv->display.funcs.wm->optimize_watermarks)
-		return;
-
-	state = drm_atomic_state_alloc(&dev_priv->drm);
-	if (drm_WARN_ON(&dev_priv->drm, !state))
-		return;
-
-	intel_state = to_intel_atomic_state(state);
-
-	drm_modeset_acquire_init(&ctx, 0);
-
-retry:
-	state->acquire_ctx = &ctx;
-
-	/*
-	 * Hardware readout is the only time we don't want to calculate
-	 * intermediate watermarks (since we don't trust the current
-	 * watermarks).
-	 */
-	if (!HAS_GMCH(dev_priv))
-		intel_state->skip_intermediate_wm = true;
-
-	ret = sanitize_watermarks_add_affected(state);
-	if (ret)
-		goto fail;
-
-	ret = intel_atomic_check(&dev_priv->drm, state);
-	if (ret)
-		goto fail;
-
-	/* Write calculated watermark values back */
-	for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
-		crtc_state->wm.need_postvbl_update = true;
-		intel_optimize_watermarks(intel_state, crtc);
-
-		to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
-	}
-
-fail:
-	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
-		drm_modeset_backoff(&ctx);
-		goto retry;
-	}
-
-	/*
-	 * If we fail here, it means that the hardware appears to be
-	 * programmed in a way that shouldn't be possible, given our
-	 * understanding of watermark requirements.  This might mean a
-	 * mistake in the hardware readout code or a mistake in the
-	 * watermark calculations for a given platform.  Raise a WARN
-	 * so that this is noticeable.
-	 *
-	 * If this actually happens, we'll have to just leave the
-	 * BIOS-programmed watermarks untouched and hope for the best.
-	 */
-	drm_WARN(&dev_priv->drm, ret,
-		 "Could not determine valid watermarks for inherited state\n");
-
-	drm_atomic_state_put(state);
-
-	drm_modeset_drop_locks(&ctx);
-	drm_modeset_acquire_fini(&ctx);
-}
-
 static int intel_initial_commit(struct drm_device *dev)
 {
 	struct drm_atomic_state *state = NULL;
@@ -8660,12 +8370,16 @@ int intel_modeset_init_noirq(struct drm_i915_private *i915)
 		goto cleanup_bios;
 
 	/* FIXME: completely on the wrong abstraction layer */
+	ret = intel_power_domains_init(i915);
+	if (ret < 0)
+		goto cleanup_vga;
+
 	intel_power_domains_init_hw(i915, false);
 
 	if (!HAS_DISPLAY(i915))
 		return 0;
 
-	intel_dmc_ucode_init(i915);
+	intel_dmc_init(i915);
 
 	i915->display.wq.modeset = alloc_ordered_workqueue("i915_modeset", 0);
 	i915->display.wq.flip = alloc_workqueue("i915_flip", WQ_HIGHPRI |
@@ -8700,8 +8414,9 @@ int intel_modeset_init_noirq(struct drm_i915_private *i915)
 	return 0;
 
 cleanup_vga_client_pw_domain_dmc:
-	intel_dmc_ucode_fini(i915);
+	intel_dmc_fini(i915);
 	intel_power_domains_driver_remove(i915);
+cleanup_vga:
 	intel_vga_unregister(i915);
 cleanup_bios:
 	intel_bios_driver_remove(i915);
@@ -8720,7 +8435,7 @@ int intel_modeset_init_nogem(struct drm_i915_private *i915)
 	if (!HAS_DISPLAY(i915))
 		return 0;
 
-	intel_init_pm(i915);
+	intel_wm_init(i915);
 
 	intel_panel_sanitize_ssc(i915);
 
@@ -8776,7 +8491,7 @@ int intel_modeset_init_nogem(struct drm_i915_private *i915)
 	 * since the watermark calculation done here will use pstate->fb.
 	 */
 	if (!HAS_GMCH(i915))
-		sanitize_watermarks(i915);
+		ilk_wm_sanitize(i915);
 
 	return 0;
 }
@@ -8817,6 +8532,7 @@ int intel_modeset_init(struct drm_i915_private *i915)
 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
 	struct intel_crtc *crtc = intel_crtc_for_pipe(dev_priv, pipe);
+	enum transcoder cpu_transcoder = (enum transcoder)pipe;
 	/* 640x480@60Hz, ~25175 kHz */
 	struct dpll clock = {
 		.m1 = 18,
@@ -8843,13 +8559,20 @@ void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 		PLL_REF_INPUT_DREFCLK |
 		DPLL_VCO_ENABLE;
 
-	intel_de_write(dev_priv, HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
-	intel_de_write(dev_priv, HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
-	intel_de_write(dev_priv, HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
-	intel_de_write(dev_priv, VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
-	intel_de_write(dev_priv, VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
-	intel_de_write(dev_priv, VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
-	intel_de_write(dev_priv, PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
+	intel_de_write(dev_priv, TRANS_HTOTAL(cpu_transcoder),
+		       HACTIVE(640 - 1) | HTOTAL(800 - 1));
+	intel_de_write(dev_priv, TRANS_HBLANK(cpu_transcoder),
+		       HBLANK_START(640 - 1) | HBLANK_END(800 - 1));
+	intel_de_write(dev_priv, TRANS_HSYNC(cpu_transcoder),
+		       HSYNC_START(656 - 1) | HSYNC_END(752 - 1));
+	intel_de_write(dev_priv, TRANS_VTOTAL(cpu_transcoder),
+		       VACTIVE(480 - 1) | VTOTAL(525 - 1));
+	intel_de_write(dev_priv, TRANS_VBLANK(cpu_transcoder),
+		       VBLANK_START(480 - 1) | VBLANK_END(525 - 1));
+	intel_de_write(dev_priv, TRANS_VSYNC(cpu_transcoder),
+		       VSYNC_START(490 - 1) | VSYNC_END(492 - 1));
+	intel_de_write(dev_priv, PIPESRC(pipe),
+		       PIPESRC_WIDTH(640 - 1) | PIPESRC_HEIGHT(480 - 1));
 
 	intel_de_write(dev_priv, FP0(pipe), fp);
 	intel_de_write(dev_priv, FP1(pipe), fp);
@@ -8880,8 +8603,8 @@ void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 		udelay(150); /* wait for warmup */
 	}
 
-	intel_de_write(dev_priv, PIPECONF(pipe), PIPECONF_ENABLE);
-	intel_de_posting_read(dev_priv, PIPECONF(pipe));
+	intel_de_write(dev_priv, TRANSCONF(pipe), TRANSCONF_ENABLE);
+	intel_de_posting_read(dev_priv, TRANSCONF(pipe));
 
 	intel_wait_for_pipe_scanline_moving(crtc);
 }
@@ -8904,8 +8627,8 @@ void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 	drm_WARN_ON(&dev_priv->drm,
 		    intel_de_read(dev_priv, CURCNTR(PIPE_B)) & MCURSOR_MODE_MASK);
 
-	intel_de_write(dev_priv, PIPECONF(pipe), 0);
-	intel_de_posting_read(dev_priv, PIPECONF(pipe));
+	intel_de_write(dev_priv, TRANSCONF(pipe), 0);
+	intel_de_posting_read(dev_priv, TRANSCONF(pipe));
 
 	intel_wait_for_pipe_scanline_stopped(crtc);
 
@@ -9026,7 +8749,7 @@ void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915)
 /* part #3: call after gem init */
 void intel_modeset_driver_remove_nogem(struct drm_i915_private *i915)
 {
-	intel_dmc_ucode_fini(i915);
+	intel_dmc_fini(i915);
 
 	intel_power_domains_driver_remove(i915);
 
@@ -9061,14 +8784,14 @@ void intel_display_driver_register(struct drm_i915_private *i915)
 	if (!HAS_DISPLAY(i915))
 		return;
 
-	intel_display_debugfs_register(i915);
-
 	/* Must be done after probing outputs */
 	intel_opregion_register(i915);
 	intel_acpi_video_register(i915);
 
 	intel_audio_init(i915);
 
+	intel_display_debugfs_register(i915);
+
 	/*
 	 * Some ports require correctly set-up hpd registers for
 	 * detection to work properly (leading to ghost connected
@@ -9077,7 +8800,7 @@ void intel_display_driver_register(struct drm_i915_private *i915)
 	 * enabled. We do it last so that the async config cannot run
 	 * before the connectors are registered.
 	 */
-	intel_fbdev_initial_config_async(&i915->drm);
+	intel_fbdev_initial_config_async(i915);
 
 	/*
 	 * We need to coordinate the hotplugs with the asynchronous
diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h
index cb6f520cc575..287159bdeb0d 100644
--- a/drivers/gpu/drm/i915/display/intel_display.h
+++ b/drivers/gpu/drm/i915/display/intel_display.h
@@ -32,6 +32,7 @@
 
 enum drm_scaling_filter;
 struct dpll;
+struct drm_atomic_state;
 struct drm_connector;
 struct drm_device;
 struct drm_display_mode;
@@ -163,14 +164,9 @@ enum tc_port {
 	I915_MAX_TC_PORTS
 };
 
-enum tc_port_mode {
-	TC_PORT_DISCONNECTED,
-	TC_PORT_TBT_ALT,
-	TC_PORT_DP_ALT,
-	TC_PORT_LEGACY,
-};
-
 enum aux_ch {
+	AUX_CH_NONE = -1,
+
 	AUX_CH_A,
 	AUX_CH_B,
 	AUX_CH_C,
@@ -394,6 +390,7 @@ enum phy_fia {
 			     ((connector) = to_intel_connector((__state)->base.connectors[__i].ptr), \
 			     (new_connector_state) = to_intel_digital_connector_state((__state)->base.connectors[__i].new_state), 1))
 
+int intel_atomic_check(struct drm_device *dev, struct drm_atomic_state *state);
 int intel_atomic_add_affected_planes(struct intel_atomic_state *state,
 				     struct intel_crtc *crtc);
 u8 intel_calc_active_pipes(struct intel_atomic_state *state,
@@ -418,7 +415,6 @@ bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state);
 bool intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 			       const struct intel_crtc_state *pipe_config,
 			       bool fastset);
-void intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state);
 
 void intel_plane_destroy(struct drm_plane *plane);
 void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
@@ -507,7 +503,7 @@ void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
 				  struct intel_crtc_state *crtc_state);
 void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state);
 
-int bdw_get_pipemisc_bpp(struct intel_crtc *crtc);
+int bdw_get_pipe_misc_bpp(struct intel_crtc *crtc);
 unsigned int intel_plane_fence_y_offset(const struct intel_plane_state *plane_state);
 
 bool intel_plane_uses_fence(const struct intel_plane_state *plane_state);
diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h
index fb8670aa2932..e36f88a39b86 100644
--- a/drivers/gpu/drm/i915/display/intel_display_core.h
+++ b/drivers/gpu/drm/i915/display/intel_display_core.h
@@ -19,13 +19,12 @@
 #include "intel_cdclk.h"
 #include "intel_display_limits.h"
 #include "intel_display_power.h"
-#include "intel_dmc.h"
 #include "intel_dpll_mgr.h"
 #include "intel_fbc.h"
 #include "intel_global_state.h"
 #include "intel_gmbus.h"
 #include "intel_opregion.h"
-#include "intel_pm_types.h"
+#include "intel_wm_types.h"
 
 struct drm_i915_private;
 struct drm_property;
@@ -40,6 +39,7 @@ struct intel_cdclk_vals;
 struct intel_color_funcs;
 struct intel_crtc;
 struct intel_crtc_state;
+struct intel_dmc;
 struct intel_dpll_funcs;
 struct intel_dpll_mgr;
 struct intel_fbdev;
@@ -85,6 +85,7 @@ struct intel_wm_funcs {
 	void (*optimize_watermarks)(struct intel_atomic_state *state,
 				    struct intel_crtc *crtc);
 	int (*compute_global_watermarks)(struct intel_atomic_state *state);
+	void (*get_hw_state)(struct drm_i915_private *i915);
 };
 
 struct intel_audio_state {
@@ -102,7 +103,7 @@ struct intel_audio {
 	u32 freq_cntrl;
 
 	/* current audio state for the audio component hooks */
-	struct intel_audio_state state[I915_MAX_PIPES];
+	struct intel_audio_state state[I915_MAX_TRANSCODERS];
 
 	/* necessary resource sharing with HDMI LPE audio driver. */
 	struct {
@@ -182,6 +183,17 @@ struct intel_hotplug {
 	 * blocked behind the non-DP one.
 	 */
 	struct workqueue_struct *dp_wq;
+
+	/*
+	 * Flag to track if long HPDs need not to be processed
+	 *
+	 * Some panels generate long HPDs while keep connected to the port.
+	 * This can cause issues with CI tests results. In CI systems we
+	 * don't expect to disconnect the panels and could ignore the long
+	 * HPDs generated from the faulty panels. This flag can be used as
+	 * cue to ignore the long HPDs and can be set / unset using debugfs.
+	 */
+	bool ignore_long_hpd;
 };
 
 struct intel_vbt_data {
@@ -243,7 +255,7 @@ struct intel_wm {
 		struct g4x_wm_values g4x;
 	};
 
-	u8 max_level;
+	u8 num_levels;
 
 	/*
 	 * Should be held around atomic WM register writing; also
@@ -340,6 +352,11 @@ struct intel_display {
 	} dkl;
 
 	struct {
+		struct intel_dmc *dmc;
+		intel_wakeref_t wakeref;
+	} dmc;
+
+	struct {
 		/* VLV/CHV/BXT/GLK DSI MMIO register base address */
 		u32 mmio_base;
 	} dsi;
@@ -378,9 +395,15 @@ struct intel_display {
 	} gmbus;
 
 	struct {
-		struct i915_hdcp_comp_master *master;
+		struct i915_hdcp_master *master;
 		bool comp_added;
 
+		/*
+		 * HDCP message struct for allocation of memory which can be
+		 * reused when sending message to gsc cs.
+		 * this is only populated post Meteorlake
+		 */
+		struct intel_hdcp_gsc_message *hdcp_message;
 		/* Mutex to protect the above hdcp component related values. */
 		struct mutex comp_mutex;
 	} hdcp;
@@ -396,6 +419,10 @@ struct intel_display {
 	} hti;
 
 	struct {
+		bool false_color;
+	} ips;
+
+	struct {
 		struct i915_power_domains domains;
 
 		/* Shadow for DISPLAY_PHY_CONTROL which can't be safely read */
@@ -466,7 +493,6 @@ struct intel_display {
 
 	/* Grouping using named structs. Keep sorted. */
 	struct intel_audio audio;
-	struct intel_dmc dmc;
 	struct intel_dpll dpll;
 	struct intel_fbc *fbc[I915_MAX_FBCS];
 	struct intel_frontbuffer_tracking fb_tracking;
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
index 7bcd90384a46..45113ae107ba 100644
--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
@@ -8,10 +8,12 @@
 #include <drm/drm_debugfs.h>
 #include <drm/drm_fourcc.h>
 
+#include "hsw_ips.h"
 #include "i915_debugfs.h"
 #include "i915_irq.h"
 #include "i915_reg.h"
 #include "intel_de.h"
+#include "intel_crtc_state_dump.h"
 #include "intel_display_debugfs.h"
 #include "intel_display_power.h"
 #include "intel_display_power_well.h"
@@ -26,10 +28,10 @@
 #include "intel_hdmi.h"
 #include "intel_hotplug.h"
 #include "intel_panel.h"
-#include "intel_pm.h"
 #include "intel_psr.h"
+#include "intel_psr_regs.h"
 #include "intel_sprite.h"
-#include "skl_watermark.h"
+#include "intel_wm.h"
 
 static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)
 {
@@ -49,33 +51,6 @@ static int i915_frontbuffer_tracking(struct seq_file *m, void *unused)
 	return 0;
 }
 
-static int i915_ips_status(struct seq_file *m, void *unused)
-{
-	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	intel_wakeref_t wakeref;
-
-	if (!HAS_IPS(dev_priv))
-		return -ENODEV;
-
-	wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
-
-	seq_printf(m, "Enabled by kernel parameter: %s\n",
-		   str_yes_no(dev_priv->params.enable_ips));
-
-	if (DISPLAY_VER(dev_priv) >= 8) {
-		seq_puts(m, "Currently: unknown\n");
-	} else {
-		if (intel_de_read(dev_priv, IPS_CTL) & IPS_ENABLE)
-			seq_puts(m, "Currently: enabled\n");
-		else
-			seq_puts(m, "Currently: disabled\n");
-	}
-
-	intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
-
-	return 0;
-}
-
 static int i915_sr_status(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -169,269 +144,6 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 	return 0;
 }
 
-static int i915_psr_sink_status_show(struct seq_file *m, void *data)
-{
-	u8 val;
-	static const char * const sink_status[] = {
-		"inactive",
-		"transition to active, capture and display",
-		"active, display from RFB",
-		"active, capture and display on sink device timings",
-		"transition to inactive, capture and display, timing re-sync",
-		"reserved",
-		"reserved",
-		"sink internal error",
-	};
-	struct drm_connector *connector = m->private;
-	struct intel_dp *intel_dp =
-		intel_attached_dp(to_intel_connector(connector));
-	int ret;
-
-	if (!CAN_PSR(intel_dp)) {
-		seq_puts(m, "PSR Unsupported\n");
-		return -ENODEV;
-	}
-
-	if (connector->status != connector_status_connected)
-		return -ENODEV;
-
-	ret = drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_STATUS, &val);
-
-	if (ret == 1) {
-		const char *str = "unknown";
-
-		val &= DP_PSR_SINK_STATE_MASK;
-		if (val < ARRAY_SIZE(sink_status))
-			str = sink_status[val];
-		seq_printf(m, "Sink PSR status: 0x%x [%s]\n", val, str);
-	} else {
-		return ret;
-	}
-
-	return 0;
-}
-DEFINE_SHOW_ATTRIBUTE(i915_psr_sink_status);
-
-static void
-psr_source_status(struct intel_dp *intel_dp, struct seq_file *m)
-{
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
-	const char *status = "unknown";
-	u32 val, status_val;
-
-	if (intel_dp->psr.psr2_enabled) {
-		static const char * const live_status[] = {
-			"IDLE",
-			"CAPTURE",
-			"CAPTURE_FS",
-			"SLEEP",
-			"BUFON_FW",
-			"ML_UP",
-			"SU_STANDBY",
-			"FAST_SLEEP",
-			"DEEP_SLEEP",
-			"BUF_ON",
-			"TG_ON"
-		};
-		val = intel_de_read(dev_priv,
-				    EDP_PSR2_STATUS(intel_dp->psr.transcoder));
-		status_val = REG_FIELD_GET(EDP_PSR2_STATUS_STATE_MASK, val);
-		if (status_val < ARRAY_SIZE(live_status))
-			status = live_status[status_val];
-	} else {
-		static const char * const live_status[] = {
-			"IDLE",
-			"SRDONACK",
-			"SRDENT",
-			"BUFOFF",
-			"BUFON",
-			"AUXACK",
-			"SRDOFFACK",
-			"SRDENT_ON",
-		};
-		val = intel_de_read(dev_priv,
-				    EDP_PSR_STATUS(intel_dp->psr.transcoder));
-		status_val = (val & EDP_PSR_STATUS_STATE_MASK) >>
-			      EDP_PSR_STATUS_STATE_SHIFT;
-		if (status_val < ARRAY_SIZE(live_status))
-			status = live_status[status_val];
-	}
-
-	seq_printf(m, "Source PSR status: %s [0x%08x]\n", status, val);
-}
-
-static int intel_psr_status(struct seq_file *m, struct intel_dp *intel_dp)
-{
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
-	struct intel_psr *psr = &intel_dp->psr;
-	intel_wakeref_t wakeref;
-	const char *status;
-	bool enabled;
-	u32 val;
-
-	seq_printf(m, "Sink support: %s", str_yes_no(psr->sink_support));
-	if (psr->sink_support)
-		seq_printf(m, " [0x%02x]", intel_dp->psr_dpcd[0]);
-	seq_puts(m, "\n");
-
-	if (!psr->sink_support)
-		return 0;
-
-	wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
-	mutex_lock(&psr->lock);
-
-	if (psr->enabled)
-		status = psr->psr2_enabled ? "PSR2 enabled" : "PSR1 enabled";
-	else
-		status = "disabled";
-	seq_printf(m, "PSR mode: %s\n", status);
-
-	if (!psr->enabled) {
-		seq_printf(m, "PSR sink not reliable: %s\n",
-			   str_yes_no(psr->sink_not_reliable));
-
-		goto unlock;
-	}
-
-	if (psr->psr2_enabled) {
-		val = intel_de_read(dev_priv,
-				    EDP_PSR2_CTL(intel_dp->psr.transcoder));
-		enabled = val & EDP_PSR2_ENABLE;
-	} else {
-		val = intel_de_read(dev_priv,
-				    EDP_PSR_CTL(intel_dp->psr.transcoder));
-		enabled = val & EDP_PSR_ENABLE;
-	}
-	seq_printf(m, "Source PSR ctl: %s [0x%08x]\n",
-		   str_enabled_disabled(enabled), val);
-	psr_source_status(intel_dp, m);
-	seq_printf(m, "Busy frontbuffer bits: 0x%08x\n",
-		   psr->busy_frontbuffer_bits);
-
-	/*
-	 * SKL+ Perf counter is reset to 0 everytime DC state is entered
-	 */
-	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
-		val = intel_de_read(dev_priv,
-				    EDP_PSR_PERF_CNT(intel_dp->psr.transcoder));
-		val &= EDP_PSR_PERF_CNT_MASK;
-		seq_printf(m, "Performance counter: %u\n", val);
-	}
-
-	if (psr->debug & I915_PSR_DEBUG_IRQ) {
-		seq_printf(m, "Last attempted entry at: %lld\n",
-			   psr->last_entry_attempt);
-		seq_printf(m, "Last exit at: %lld\n", psr->last_exit);
-	}
-
-	if (psr->psr2_enabled) {
-		u32 su_frames_val[3];
-		int frame;
-
-		/*
-		 * Reading all 3 registers before hand to minimize crossing a
-		 * frame boundary between register reads
-		 */
-		for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame += 3) {
-			val = intel_de_read(dev_priv,
-					    PSR2_SU_STATUS(intel_dp->psr.transcoder, frame));
-			su_frames_val[frame / 3] = val;
-		}
-
-		seq_puts(m, "Frame:\tPSR2 SU blocks:\n");
-
-		for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame++) {
-			u32 su_blocks;
-
-			su_blocks = su_frames_val[frame / 3] &
-				    PSR2_SU_STATUS_MASK(frame);
-			su_blocks = su_blocks >> PSR2_SU_STATUS_SHIFT(frame);
-			seq_printf(m, "%d\t%d\n", frame, su_blocks);
-		}
-
-		seq_printf(m, "PSR2 selective fetch: %s\n",
-			   str_enabled_disabled(psr->psr2_sel_fetch_enabled));
-	}
-
-unlock:
-	mutex_unlock(&psr->lock);
-	intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
-
-	return 0;
-}
-
-static int i915_edp_psr_status(struct seq_file *m, void *data)
-{
-	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct intel_dp *intel_dp = NULL;
-	struct intel_encoder *encoder;
-
-	if (!HAS_PSR(dev_priv))
-		return -ENODEV;
-
-	/* Find the first EDP which supports PSR */
-	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
-		intel_dp = enc_to_intel_dp(encoder);
-		break;
-	}
-
-	if (!intel_dp)
-		return -ENODEV;
-
-	return intel_psr_status(m, intel_dp);
-}
-
-static int
-i915_edp_psr_debug_set(void *data, u64 val)
-{
-	struct drm_i915_private *dev_priv = data;
-	struct intel_encoder *encoder;
-	intel_wakeref_t wakeref;
-	int ret = -ENODEV;
-
-	if (!HAS_PSR(dev_priv))
-		return ret;
-
-	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
-		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-
-		drm_dbg_kms(&dev_priv->drm, "Setting PSR debug to %llx\n", val);
-
-		wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
-
-		// TODO: split to each transcoder's PSR debug state
-		ret = intel_psr_debug_set(intel_dp, val);
-
-		intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
-	}
-
-	return ret;
-}
-
-static int
-i915_edp_psr_debug_get(void *data, u64 *val)
-{
-	struct drm_i915_private *dev_priv = data;
-	struct intel_encoder *encoder;
-
-	if (!HAS_PSR(dev_priv))
-		return -ENODEV;
-
-	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
-		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-
-		// TODO: split to each transcoder's PSR debug state
-		*val = READ_ONCE(intel_dp->psr.debug);
-		return 0;
-	}
-
-	return -ENODEV;
-}
-
-DEFINE_SIMPLE_ATTRIBUTE(i915_edp_psr_debug_fops,
-			i915_edp_psr_debug_get, i915_edp_psr_debug_set,
-			"%llu\n");
-
 static int i915_power_domain_info(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *i915 = node_to_i915(m->private);
@@ -832,10 +544,10 @@ static const struct file_operations crtc_updates_fops = {
 	.write = crtc_updates_write
 };
 
-static void crtc_updates_add(struct drm_crtc *crtc)
+static void crtc_updates_add(struct intel_crtc *crtc)
 {
-	debugfs_create_file("i915_update_info", 0644, crtc->debugfs_entry,
-			    to_intel_crtc(crtc), &crtc_updates_fops);
+	debugfs_create_file("i915_update_info", 0644, crtc->base.debugfs_entry,
+			    crtc, &crtc_updates_fops);
 }
 
 #else
@@ -845,7 +557,7 @@ static void crtc_updates_info(struct seq_file *m,
 {
 }
 
-static void crtc_updates_add(struct drm_crtc *crtc)
+static void crtc_updates_add(struct intel_crtc *crtc)
 {
 }
 #endif
@@ -1282,237 +994,6 @@ static int i915_displayport_test_type_show(struct seq_file *m, void *data)
 }
 DEFINE_SHOW_ATTRIBUTE(i915_displayport_test_type);
 
-static void wm_latency_show(struct seq_file *m, const u16 wm[8])
-{
-	struct drm_i915_private *dev_priv = m->private;
-	int level;
-	int num_levels;
-
-	if (IS_CHERRYVIEW(dev_priv))
-		num_levels = 3;
-	else if (IS_VALLEYVIEW(dev_priv))
-		num_levels = 1;
-	else if (IS_G4X(dev_priv))
-		num_levels = 3;
-	else
-		num_levels = ilk_wm_max_level(dev_priv) + 1;
-
-	drm_modeset_lock_all(&dev_priv->drm);
-
-	for (level = 0; level < num_levels; level++) {
-		unsigned int latency = wm[level];
-
-		/*
-		 * - WM1+ latency values in 0.5us units
-		 * - latencies are in us on gen9/vlv/chv
-		 */
-		if (DISPLAY_VER(dev_priv) >= 9 ||
-		    IS_VALLEYVIEW(dev_priv) ||
-		    IS_CHERRYVIEW(dev_priv) ||
-		    IS_G4X(dev_priv))
-			latency *= 10;
-		else if (level > 0)
-			latency *= 5;
-
-		seq_printf(m, "WM%d %u (%u.%u usec)\n",
-			   level, wm[level], latency / 10, latency % 10);
-	}
-
-	drm_modeset_unlock_all(&dev_priv->drm);
-}
-
-static int pri_wm_latency_show(struct seq_file *m, void *data)
-{
-	struct drm_i915_private *dev_priv = m->private;
-	const u16 *latencies;
-
-	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->display.wm.skl_latency;
-	else
-		latencies = dev_priv->display.wm.pri_latency;
-
-	wm_latency_show(m, latencies);
-
-	return 0;
-}
-
-static int spr_wm_latency_show(struct seq_file *m, void *data)
-{
-	struct drm_i915_private *dev_priv = m->private;
-	const u16 *latencies;
-
-	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->display.wm.skl_latency;
-	else
-		latencies = dev_priv->display.wm.spr_latency;
-
-	wm_latency_show(m, latencies);
-
-	return 0;
-}
-
-static int cur_wm_latency_show(struct seq_file *m, void *data)
-{
-	struct drm_i915_private *dev_priv = m->private;
-	const u16 *latencies;
-
-	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->display.wm.skl_latency;
-	else
-		latencies = dev_priv->display.wm.cur_latency;
-
-	wm_latency_show(m, latencies);
-
-	return 0;
-}
-
-static int pri_wm_latency_open(struct inode *inode, struct file *file)
-{
-	struct drm_i915_private *dev_priv = inode->i_private;
-
-	if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv))
-		return -ENODEV;
-
-	return single_open(file, pri_wm_latency_show, dev_priv);
-}
-
-static int spr_wm_latency_open(struct inode *inode, struct file *file)
-{
-	struct drm_i915_private *dev_priv = inode->i_private;
-
-	if (HAS_GMCH(dev_priv))
-		return -ENODEV;
-
-	return single_open(file, spr_wm_latency_show, dev_priv);
-}
-
-static int cur_wm_latency_open(struct inode *inode, struct file *file)
-{
-	struct drm_i915_private *dev_priv = inode->i_private;
-
-	if (HAS_GMCH(dev_priv))
-		return -ENODEV;
-
-	return single_open(file, cur_wm_latency_show, dev_priv);
-}
-
-static ssize_t wm_latency_write(struct file *file, const char __user *ubuf,
-				size_t len, loff_t *offp, u16 wm[8])
-{
-	struct seq_file *m = file->private_data;
-	struct drm_i915_private *dev_priv = m->private;
-	u16 new[8] = { 0 };
-	int num_levels;
-	int level;
-	int ret;
-	char tmp[32];
-
-	if (IS_CHERRYVIEW(dev_priv))
-		num_levels = 3;
-	else if (IS_VALLEYVIEW(dev_priv))
-		num_levels = 1;
-	else if (IS_G4X(dev_priv))
-		num_levels = 3;
-	else
-		num_levels = ilk_wm_max_level(dev_priv) + 1;
-
-	if (len >= sizeof(tmp))
-		return -EINVAL;
-
-	if (copy_from_user(tmp, ubuf, len))
-		return -EFAULT;
-
-	tmp[len] = '\0';
-
-	ret = sscanf(tmp, "%hu %hu %hu %hu %hu %hu %hu %hu",
-		     &new[0], &new[1], &new[2], &new[3],
-		     &new[4], &new[5], &new[6], &new[7]);
-	if (ret != num_levels)
-		return -EINVAL;
-
-	drm_modeset_lock_all(&dev_priv->drm);
-
-	for (level = 0; level < num_levels; level++)
-		wm[level] = new[level];
-
-	drm_modeset_unlock_all(&dev_priv->drm);
-
-	return len;
-}
-
-
-static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf,
-				    size_t len, loff_t *offp)
-{
-	struct seq_file *m = file->private_data;
-	struct drm_i915_private *dev_priv = m->private;
-	u16 *latencies;
-
-	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->display.wm.skl_latency;
-	else
-		latencies = dev_priv->display.wm.pri_latency;
-
-	return wm_latency_write(file, ubuf, len, offp, latencies);
-}
-
-static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf,
-				    size_t len, loff_t *offp)
-{
-	struct seq_file *m = file->private_data;
-	struct drm_i915_private *dev_priv = m->private;
-	u16 *latencies;
-
-	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->display.wm.skl_latency;
-	else
-		latencies = dev_priv->display.wm.spr_latency;
-
-	return wm_latency_write(file, ubuf, len, offp, latencies);
-}
-
-static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf,
-				    size_t len, loff_t *offp)
-{
-	struct seq_file *m = file->private_data;
-	struct drm_i915_private *dev_priv = m->private;
-	u16 *latencies;
-
-	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->display.wm.skl_latency;
-	else
-		latencies = dev_priv->display.wm.cur_latency;
-
-	return wm_latency_write(file, ubuf, len, offp, latencies);
-}
-
-static const struct file_operations i915_pri_wm_latency_fops = {
-	.owner = THIS_MODULE,
-	.open = pri_wm_latency_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.write = pri_wm_latency_write
-};
-
-static const struct file_operations i915_spr_wm_latency_fops = {
-	.owner = THIS_MODULE,
-	.open = spr_wm_latency_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.write = spr_wm_latency_write
-};
-
-static const struct file_operations i915_cur_wm_latency_fops = {
-	.owner = THIS_MODULE,
-	.open = cur_wm_latency_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.write = cur_wm_latency_write
-};
-
 static ssize_t
 i915_fifo_underrun_reset_write(struct file *filp,
 			       const char __user *ubuf,
@@ -1574,12 +1055,10 @@ static const struct file_operations i915_fifo_underrun_reset_ops = {
 
 static const struct drm_info_list intel_display_debugfs_list[] = {
 	{"i915_frontbuffer_tracking", i915_frontbuffer_tracking, 0},
-	{"i915_ips_status", i915_ips_status, 0},
 	{"i915_sr_status", i915_sr_status, 0},
 	{"i915_opregion", i915_opregion, 0},
 	{"i915_vbt", i915_vbt, 0},
 	{"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
-	{"i915_edp_psr_status", i915_edp_psr_status, 0},
 	{"i915_power_domain_info", i915_power_domain_info, 0},
 	{"i915_display_info", i915_display_info, 0},
 	{"i915_shared_dplls_info", i915_shared_dplls_info, 0},
@@ -1593,13 +1072,9 @@ static const struct {
 	const struct file_operations *fops;
 } intel_display_debugfs_files[] = {
 	{"i915_fifo_underrun_reset", &i915_fifo_underrun_reset_ops},
-	{"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
-	{"i915_spr_wm_latency", &i915_spr_wm_latency_fops},
-	{"i915_cur_wm_latency", &i915_cur_wm_latency_fops},
 	{"i915_dp_test_data", &i915_displayport_test_data_fops},
 	{"i915_dp_test_type", &i915_displayport_test_type_fops},
 	{"i915_dp_test_active", &i915_displayport_test_active_fops},
-	{"i915_edp_psr_debug", &i915_edp_psr_debug_fops},
 };
 
 void intel_display_debugfs_register(struct drm_i915_private *i915)
@@ -1622,7 +1097,8 @@ void intel_display_debugfs_register(struct drm_i915_private *i915)
 	intel_dmc_debugfs_register(i915);
 	intel_fbc_debugfs_register(i915);
 	intel_hpd_debugfs_register(i915);
-	skl_watermark_ipc_debugfs_register(i915);
+	intel_psr_debugfs_register(i915);
+	intel_wm_debugfs_register(i915);
 }
 
 static int i915_panel_show(struct seq_file *m, void *data)
@@ -1674,16 +1150,6 @@ out:
 }
 DEFINE_SHOW_ATTRIBUTE(i915_hdcp_sink_capability);
 
-static int i915_psr_status_show(struct seq_file *m, void *data)
-{
-	struct drm_connector *connector = m->private;
-	struct intel_dp *intel_dp =
-		intel_attached_dp(to_intel_connector(connector));
-
-	return intel_psr_status(m, intel_dp);
-}
-DEFINE_SHOW_ATTRIBUTE(i915_psr_status);
-
 static int i915_lpsp_capability_show(struct seq_file *m, void *data)
 {
 	struct drm_connector *connector = m->private;
@@ -1770,6 +1236,13 @@ static int i915_dsc_fec_support_show(struct seq_file *m, void *data)
 			   str_yes_no(crtc_state->dsc.compression_enable));
 		seq_printf(m, "DSC_Sink_Support: %s\n",
 			   str_yes_no(drm_dp_sink_supports_dsc(intel_dp->dsc_dpcd)));
+		seq_printf(m, "DSC_Output_Format_Sink_Support: RGB: %s YCBCR420: %s YCBCR444: %s\n",
+			   str_yes_no(drm_dp_dsc_sink_supports_format(intel_dp->dsc_dpcd,
+								      DP_DSC_RGB)),
+			   str_yes_no(drm_dp_dsc_sink_supports_format(intel_dp->dsc_dpcd,
+								      DP_DSC_YCbCr420_Native)),
+			   str_yes_no(drm_dp_dsc_sink_supports_format(intel_dp->dsc_dpcd,
+								      DP_DSC_YCbCr444)));
 		seq_printf(m, "Force_DSC_Enable: %s\n",
 			   str_yes_no(intel_dp->force_dsc_en));
 		if (!intel_dp_is_edp(intel_dp))
@@ -1895,13 +1368,80 @@ static const struct file_operations i915_dsc_bpc_fops = {
 	.write = i915_dsc_bpc_write
 };
 
+static int i915_dsc_output_format_show(struct seq_file *m, void *data)
+{
+	struct drm_connector *connector = m->private;
+	struct drm_device *dev = connector->dev;
+	struct drm_crtc *crtc;
+	struct intel_crtc_state *crtc_state;
+	struct intel_encoder *encoder = intel_attached_encoder(to_intel_connector(connector));
+	int ret;
+
+	if (!encoder)
+		return -ENODEV;
+
+	ret = drm_modeset_lock_single_interruptible(&dev->mode_config.connection_mutex);
+	if (ret)
+		return ret;
+
+	crtc = connector->state->crtc;
+	if (connector->status != connector_status_connected || !crtc) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	crtc_state = to_intel_crtc_state(crtc->state);
+	seq_printf(m, "DSC_Output_Format: %s\n",
+		   intel_output_format_name(crtc_state->output_format));
+
+out:	drm_modeset_unlock(&dev->mode_config.connection_mutex);
+
+	return ret;
+}
+
+static ssize_t i915_dsc_output_format_write(struct file *file,
+					    const char __user *ubuf,
+					    size_t len, loff_t *offp)
+{
+	struct drm_connector *connector =
+		((struct seq_file *)file->private_data)->private;
+	struct intel_encoder *encoder = intel_attached_encoder(to_intel_connector(connector));
+	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+	int dsc_output_format = 0;
+	int ret;
+
+	ret = kstrtoint_from_user(ubuf, len, 0, &dsc_output_format);
+	if (ret < 0)
+		return ret;
+
+	intel_dp->force_dsc_output_format = dsc_output_format;
+	*offp += len;
+
+	return len;
+}
+
+static int i915_dsc_output_format_open(struct inode *inode,
+				       struct file *file)
+{
+	return single_open(file, i915_dsc_output_format_show, inode->i_private);
+}
+
+static const struct file_operations i915_dsc_output_format_fops = {
+	.owner = THIS_MODULE,
+	.open = i915_dsc_output_format_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = i915_dsc_output_format_write
+};
+
 /*
  * Returns the Current CRTC's bpc.
  * Example usage: cat /sys/kernel/debug/dri/0/crtc-0/i915_current_bpc
  */
 static int i915_current_bpc_show(struct seq_file *m, void *data)
 {
-	struct intel_crtc *crtc = to_intel_crtc(m->private);
+	struct intel_crtc *crtc = m->private;
 	struct intel_crtc_state *crtc_state;
 	int ret;
 
@@ -1918,9 +1458,20 @@ static int i915_current_bpc_show(struct seq_file *m, void *data)
 }
 DEFINE_SHOW_ATTRIBUTE(i915_current_bpc);
 
+/* Pipe may differ from crtc index if pipes are fused off */
+static int intel_crtc_pipe_show(struct seq_file *m, void *unused)
+{
+	struct intel_crtc *crtc = m->private;
+
+	seq_printf(m, "%c\n", pipe_name(crtc->pipe));
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(intel_crtc_pipe);
+
 /**
  * intel_connector_debugfs_add - add i915 specific connector debugfs files
- * @connector: pointer to a registered drm_connector
+ * @intel_connector: pointer to a registered drm_connector
  *
  * Cleanup will be done by drm_connector_unregister() through a call to
  * drm_debugfs_connector_remove().
@@ -1936,19 +1487,11 @@ void intel_connector_debugfs_add(struct intel_connector *intel_connector)
 		return;
 
 	intel_drrs_connector_debugfs_add(intel_connector);
+	intel_psr_connector_debugfs_add(intel_connector);
 
-	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
+	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP)
 		debugfs_create_file("i915_panel_timings", S_IRUGO, root,
 				    connector, &i915_panel_fops);
-		debugfs_create_file("i915_psr_sink_status", S_IRUGO, root,
-				    connector, &i915_psr_sink_status_fops);
-	}
-
-	if (HAS_PSR(dev_priv) &&
-	    connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
-		debugfs_create_file("i915_psr_status", 0444, root,
-				    connector, &i915_psr_status_fops);
-	}
 
 	if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort ||
 	    connector->connector_type == DRM_MODE_CONNECTOR_HDMIA ||
@@ -1966,6 +1509,9 @@ void intel_connector_debugfs_add(struct intel_connector *intel_connector)
 
 		debugfs_create_file("i915_dsc_bpc", 0644, root,
 				    connector, &i915_dsc_bpc_fops);
+
+		debugfs_create_file("i915_dsc_output_format", 0644, root,
+				    connector, &i915_dsc_output_format_fops);
 	}
 
 	if (connector->connector_type == DRM_MODE_CONNECTOR_DSI ||
@@ -1983,15 +1529,20 @@ void intel_connector_debugfs_add(struct intel_connector *intel_connector)
  *
  * Failure to add debugfs entries should generally be ignored.
  */
-void intel_crtc_debugfs_add(struct drm_crtc *crtc)
+void intel_crtc_debugfs_add(struct intel_crtc *crtc)
 {
-	if (!crtc->debugfs_entry)
+	struct dentry *root = crtc->base.debugfs_entry;
+
+	if (!root)
 		return;
 
 	crtc_updates_add(crtc);
-	intel_drrs_crtc_debugfs_add(to_intel_crtc(crtc));
-	intel_fbc_crtc_debugfs_add(to_intel_crtc(crtc));
+	intel_drrs_crtc_debugfs_add(crtc);
+	intel_fbc_crtc_debugfs_add(crtc);
+	hsw_ips_crtc_debugfs_add(crtc);
 
-	debugfs_create_file("i915_current_bpc", 0444, crtc->debugfs_entry, crtc,
+	debugfs_create_file("i915_current_bpc", 0444, root, crtc,
 			    &i915_current_bpc_fops);
+	debugfs_create_file("i915_pipe", 0444, root, crtc,
+			    &intel_crtc_pipe_fops);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.h b/drivers/gpu/drm/i915/display/intel_display_debugfs.h
index d3a79c07c384..e1f479b7acd1 100644
--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.h
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.h
@@ -6,18 +6,18 @@
 #ifndef __INTEL_DISPLAY_DEBUGFS_H__
 #define __INTEL_DISPLAY_DEBUGFS_H__
 
-struct drm_crtc;
 struct drm_i915_private;
 struct intel_connector;
+struct intel_crtc;
 
 #ifdef CONFIG_DEBUG_FS
 void intel_display_debugfs_register(struct drm_i915_private *i915);
 void intel_connector_debugfs_add(struct intel_connector *connector);
-void intel_crtc_debugfs_add(struct drm_crtc *crtc);
+void intel_crtc_debugfs_add(struct intel_crtc *crtc);
 #else
 static inline void intel_display_debugfs_register(struct drm_i915_private *i915) {}
 static inline void intel_connector_debugfs_add(struct intel_connector *connector) {}
-static inline void intel_crtc_debugfs_add(struct drm_crtc *crtc) {}
+static inline void intel_crtc_debugfs_add(struct intel_crtc *crtc) {}
 #endif
 
 #endif /* __INTEL_DISPLAY_DEBUGFS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c
index 1a23ecd4623a..7c9f4288329e 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power.c
@@ -19,8 +19,10 @@
 #include "intel_mchbar_regs.h"
 #include "intel_pch_refclk.h"
 #include "intel_pcode.h"
+#include "intel_pps_regs.h"
 #include "intel_snps_phy.h"
 #include "skl_watermark.h"
+#include "skl_watermark_regs.h"
 #include "vlv_sideband.h"
 
 #define for_each_power_domain_well(__dev_priv, __power_well, __domain)	\
@@ -264,9 +266,10 @@ bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 }
 
 static u32
-sanitize_target_dc_state(struct drm_i915_private *dev_priv,
+sanitize_target_dc_state(struct drm_i915_private *i915,
 			 u32 target_dc_state)
 {
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 	static const u32 states[] = {
 		DC_STATE_EN_UPTO_DC6,
 		DC_STATE_EN_UPTO_DC5,
@@ -279,7 +282,7 @@ sanitize_target_dc_state(struct drm_i915_private *dev_priv,
 		if (target_dc_state != states[i])
 			continue;
 
-		if (dev_priv->display.dmc.allowed_dc_mask & target_dc_state)
+		if (power_domains->allowed_dc_mask & target_dc_state)
 			break;
 
 		target_dc_state = states[i + 1];
@@ -312,7 +315,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
 
 	state = sanitize_target_dc_state(dev_priv, state);
 
-	if (state == dev_priv->display.dmc.target_dc_state)
+	if (state == power_domains->target_dc_state)
 		goto unlock;
 
 	dc_off_enabled = intel_power_well_is_enabled(dev_priv, power_well);
@@ -323,7 +326,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
 	if (!dc_off_enabled)
 		intel_power_well_enable(dev_priv, power_well);
 
-	dev_priv->display.dmc.target_dc_state = state;
+	power_domains->target_dc_state = state;
 
 	if (!dc_off_enabled)
 		intel_power_well_disable(dev_priv, power_well);
@@ -696,7 +699,7 @@ out_verify:
 }
 
 /**
- * intel_display_power_put_async - release a power domain reference asynchronously
+ * __intel_display_power_put_async - release a power domain reference asynchronously
  * @i915: i915 device instance
  * @domain: power domain to reference
  * @wakeref: wakeref acquired for the reference that is being released
@@ -992,10 +995,10 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
 	dev_priv->params.disable_power_well =
 		sanitize_disable_power_well_option(dev_priv,
 						   dev_priv->params.disable_power_well);
-	dev_priv->display.dmc.allowed_dc_mask =
+	power_domains->allowed_dc_mask =
 		get_allowed_dc_mask(dev_priv, dev_priv->params.enable_dc);
 
-	dev_priv->display.dmc.target_dc_state =
+	power_domains->target_dc_state =
 		sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 
 	mutex_init(&power_domains->lock);
@@ -1181,8 +1184,10 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
 				"CPU PWM2 enabled\n");
 	I915_STATE_WARN(intel_de_read(dev_priv, BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
 			"PCH PWM1 enabled\n");
-	I915_STATE_WARN(intel_de_read(dev_priv, UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
-			"Utility pin enabled\n");
+	I915_STATE_WARN((intel_de_read(dev_priv, UTIL_PIN_CTL) &
+			 (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) ==
+			(UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
+			"Utility pin enabled in PWM mode\n");
 	I915_STATE_WARN(intel_de_read(dev_priv, PCH_GTC_CTL) & PCH_GTC_ENABLE,
 			"PCH GTC enabled\n");
 
@@ -1260,9 +1265,7 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
 		drm_err(&dev_priv->drm, "D_COMP RCOMP still in progress\n");
 
 	if (allow_power_down) {
-		val = intel_de_read(dev_priv, LCPLL_CTL);
-		val |= LCPLL_POWER_DOWN_ALLOW;
-		intel_de_write(dev_priv, LCPLL_CTL, val);
+		intel_de_rmw(dev_priv, LCPLL_CTL, 0, LCPLL_POWER_DOWN_ALLOW);
 		intel_de_posting_read(dev_priv, LCPLL_CTL);
 	}
 }
@@ -1306,9 +1309,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
 		drm_err(&dev_priv->drm, "LCPLL not locked yet\n");
 
 	if (val & LCPLL_CD_SOURCE_FCLK) {
-		val = intel_de_read(dev_priv, LCPLL_CTL);
-		val &= ~LCPLL_CD_SOURCE_FCLK;
-		intel_de_write(dev_priv, LCPLL_CTL, val);
+		intel_de_rmw(dev_priv, LCPLL_CTL, LCPLL_CD_SOURCE_FCLK, 0);
 
 		if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
 				 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
@@ -1347,15 +1348,11 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
  */
 static void hsw_enable_pc8(struct drm_i915_private *dev_priv)
 {
-	u32 val;
-
 	drm_dbg_kms(&dev_priv->drm, "Enabling package C8+\n");
 
-	if (HAS_PCH_LPT_LP(dev_priv)) {
-		val = intel_de_read(dev_priv, SOUTH_DSPCLK_GATE_D);
-		val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
-		intel_de_write(dev_priv, SOUTH_DSPCLK_GATE_D, val);
-	}
+	if (HAS_PCH_LPT_LP(dev_priv))
+		intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D,
+			     PCH_LP_PARTITION_LEVEL_DISABLE, 0);
 
 	lpt_disable_clkout_dp(dev_priv);
 	hsw_disable_lcpll(dev_priv, true, true);
@@ -1363,25 +1360,21 @@ static void hsw_enable_pc8(struct drm_i915_private *dev_priv)
 
 static void hsw_disable_pc8(struct drm_i915_private *dev_priv)
 {
-	u32 val;
-
 	drm_dbg_kms(&dev_priv->drm, "Disabling package C8+\n");
 
 	hsw_restore_lcpll(dev_priv);
 	intel_init_pch_refclk(dev_priv);
 
-	if (HAS_PCH_LPT_LP(dev_priv)) {
-		val = intel_de_read(dev_priv, SOUTH_DSPCLK_GATE_D);
-		val |= PCH_LP_PARTITION_LEVEL_DISABLE;
-		intel_de_write(dev_priv, SOUTH_DSPCLK_GATE_D, val);
-	}
+	if (HAS_PCH_LPT_LP(dev_priv))
+		intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D,
+			     0, PCH_LP_PARTITION_LEVEL_DISABLE);
 }
 
 static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
 				      bool enable)
 {
 	i915_reg_t reg;
-	u32 reset_bits, val;
+	u32 reset_bits;
 
 	if (IS_IVYBRIDGE(dev_priv)) {
 		reg = GEN7_MSG_CTL;
@@ -1394,14 +1387,7 @@ static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
 	if (DISPLAY_VER(dev_priv) >= 14)
 		reset_bits |= MTL_RESET_PICA_HANDSHAKE_EN;
 
-	val = intel_de_read(dev_priv, reg);
-
-	if (enable)
-		val |= reset_bits;
-	else
-		val &= ~reset_bits;
-
-	intel_de_write(dev_priv, reg, val);
+	intel_de_rmw(dev_priv, reg, reset_bits, enable ? reset_bits : 0);
 }
 
 static void skl_display_core_init(struct drm_i915_private *dev_priv,
@@ -1580,10 +1566,8 @@ static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv)
 		return;
 
 	if (IS_ALDERLAKE_S(dev_priv) ||
-	    IS_DG1_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) ||
-	    IS_RKL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) ||
-	    IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
-		/* Wa_1409767108:tgl,dg1,adl-s */
+	    IS_RKL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
+		/* Wa_1409767108 */
 		table = wa_1409767108_buddy_page_masks;
 	else
 		table = tgl_buddy_page_masks;
@@ -1618,7 +1602,6 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv,
 {
 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct i915_power_well *well;
-	u32 val;
 
 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 
@@ -1646,6 +1629,10 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv,
 	intel_power_well_enable(dev_priv, well);
 	mutex_unlock(&power_domains->lock);
 
+	if (DISPLAY_VER(dev_priv) == 14)
+		intel_de_rmw(dev_priv, DC_STATE_EN,
+			     HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH, 0);
+
 	/* 4. Enable CDCLK. */
 	intel_cdclk_init_hw(dev_priv);
 
@@ -1670,11 +1657,10 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv,
 		intel_dmc_load_program(dev_priv);
 
 	/* Wa_14011508470:tgl,dg1,rkl,adl-s,adl-p */
-	if (DISPLAY_VER(dev_priv) >= 12) {
-		val = DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
-		      DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR;
-		intel_de_rmw(dev_priv, GEN11_CHICKEN_DCPR_2, 0, val);
-	}
+	if (DISPLAY_VER(dev_priv) >= 12)
+		intel_de_rmw(dev_priv, GEN11_CHICKEN_DCPR_2, 0,
+			     DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
+			     DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR);
 
 	/* Wa_14011503030:xelpd */
 	if (DISPLAY_VER(dev_priv) >= 13)
@@ -1700,6 +1686,10 @@ static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
 	/* 3. Disable CD clock */
 	intel_cdclk_uninit_hw(dev_priv);
 
+	if (DISPLAY_VER(dev_priv) == 14)
+		intel_de_rmw(dev_priv, DC_STATE_EN, 0,
+			     HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH);
+
 	/*
 	 * 4. Disable Power Well 1 (PG1).
 	 *    The AUX IO power wells are toggled on demand, so they are already
@@ -2055,7 +2045,7 @@ void intel_power_domains_suspend(struct drm_i915_private *i915,
 	 * resources as required and also enable deeper system power states
 	 * that would be blocked if the firmware was inactive.
 	 */
-	if (!(i915->display.dmc.allowed_dc_mask & DC_STATE_EN_DC9) &&
+	if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC9) &&
 	    suspend_mode == I915_DRM_SUSPEND_IDLE &&
 	    intel_dmc_has_payload(i915)) {
 		intel_display_power_flush_work(i915);
@@ -2244,22 +2234,22 @@ void intel_display_power_suspend(struct drm_i915_private *i915)
 
 void intel_display_power_resume(struct drm_i915_private *i915)
 {
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
+
 	if (DISPLAY_VER(i915) >= 11) {
 		bxt_disable_dc9(i915);
 		icl_display_core_init(i915, true);
 		if (intel_dmc_has_payload(i915)) {
-			if (i915->display.dmc.allowed_dc_mask &
-			    DC_STATE_EN_UPTO_DC6)
+			if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
 				skl_enable_dc6(i915);
-			else if (i915->display.dmc.allowed_dc_mask &
-				 DC_STATE_EN_UPTO_DC5)
+			else if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
 				gen9_enable_dc5(i915);
 		}
 	} else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
 		bxt_disable_dc9(i915);
 		bxt_display_core_init(i915, true);
 		if (intel_dmc_has_payload(i915) &&
-		    (i915->display.dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
+		    (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
 			gen9_enable_dc5(i915);
 	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
 		hsw_disable_pc8(i915);
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.h b/drivers/gpu/drm/i915/display/intel_display_power.h
index 2154d900b1aa..8e96be8e6330 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.h
+++ b/drivers/gpu/drm/i915/display/intel_display_power.h
@@ -137,6 +137,10 @@ struct i915_power_domains {
 	bool display_core_suspended;
 	int power_well_count;
 
+	u32 dc_state;
+	u32 target_dc_state;
+	u32 allowed_dc_mask;
+
 	intel_wakeref_t init_wakeref;
 	intel_wakeref_t disable_wakeref;
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c
index 8710dd41ffd4..62bafcbc7937 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c
@@ -15,6 +15,7 @@
 #include "intel_dkl_phy.h"
 #include "intel_dkl_phy_regs.h"
 #include "intel_dmc.h"
+#include "intel_dp_aux_regs.h"
 #include "intel_dpio_phy.h"
 #include "intel_dpll.h"
 #include "intel_hotplug.h"
@@ -333,7 +334,6 @@ static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 {
 	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
-	u32 val;
 
 	if (power_well->desc->has_fuses) {
 		enum skl_power_gate pg;
@@ -356,9 +356,7 @@ static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 			gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 	}
 
-	val = intel_de_read(dev_priv, regs->driver);
-	intel_de_write(dev_priv, regs->driver,
-		       val | HSW_PWR_WELL_CTL_REQ(pw_idx));
+	intel_de_rmw(dev_priv, regs->driver, 0, HSW_PWR_WELL_CTL_REQ(pw_idx));
 
 	hsw_wait_for_power_well_enable(dev_priv, power_well, false);
 
@@ -380,17 +378,27 @@ static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 {
 	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
-	u32 val;
 
 	hsw_power_well_pre_disable(dev_priv,
 				   power_well->desc->irq_pipe_mask);
 
-	val = intel_de_read(dev_priv, regs->driver);
-	intel_de_write(dev_priv, regs->driver,
-		       val & ~HSW_PWR_WELL_CTL_REQ(pw_idx));
+	intel_de_rmw(dev_priv, regs->driver, HSW_PWR_WELL_CTL_REQ(pw_idx), 0);
 	hsw_wait_for_power_well_disable(dev_priv, power_well);
 }
 
+static bool intel_port_is_edp(struct drm_i915_private *i915, enum port port)
+{
+	struct intel_encoder *encoder;
+
+	for_each_intel_encoder(&i915->drm, encoder) {
+		if (encoder->type == INTEL_OUTPUT_EDP &&
+		    encoder->port == port)
+			return true;
+	}
+
+	return false;
+}
+
 static void
 icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 				    struct i915_power_well *power_well)
@@ -398,29 +406,22 @@ icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 	enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
-	u32 val;
 
 	drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 
-	val = intel_de_read(dev_priv, regs->driver);
-	intel_de_write(dev_priv, regs->driver,
-		       val | HSW_PWR_WELL_CTL_REQ(pw_idx));
+	intel_de_rmw(dev_priv, regs->driver, 0, HSW_PWR_WELL_CTL_REQ(pw_idx));
 
-	if (DISPLAY_VER(dev_priv) < 12) {
-		val = intel_de_read(dev_priv, ICL_PORT_CL_DW12(phy));
-		intel_de_write(dev_priv, ICL_PORT_CL_DW12(phy),
-			       val | ICL_LANE_ENABLE_AUX);
-	}
+	if (DISPLAY_VER(dev_priv) < 12)
+		intel_de_rmw(dev_priv, ICL_PORT_CL_DW12(phy),
+			     0, ICL_LANE_ENABLE_AUX);
 
 	hsw_wait_for_power_well_enable(dev_priv, power_well, false);
 
 	/* Display WA #1178: icl */
 	if (pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B &&
-	    !intel_bios_is_port_edp(dev_priv, (enum port)phy)) {
-		val = intel_de_read(dev_priv, ICL_AUX_ANAOVRD1(pw_idx));
-		val |= ICL_AUX_ANAOVRD1_ENABLE | ICL_AUX_ANAOVRD1_LDO_BYPASS;
-		intel_de_write(dev_priv, ICL_AUX_ANAOVRD1(pw_idx), val);
-	}
+	    !intel_port_is_edp(dev_priv, (enum port)phy))
+		intel_de_rmw(dev_priv, ICL_AUX_ANAOVRD1(pw_idx),
+			     0, ICL_AUX_ANAOVRD1_ENABLE | ICL_AUX_ANAOVRD1_LDO_BYPASS);
 }
 
 static void
@@ -430,17 +431,12 @@ icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
 	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 	enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
-	u32 val;
 
 	drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 
-	val = intel_de_read(dev_priv, ICL_PORT_CL_DW12(phy));
-	intel_de_write(dev_priv, ICL_PORT_CL_DW12(phy),
-		       val & ~ICL_LANE_ENABLE_AUX);
+	intel_de_rmw(dev_priv, ICL_PORT_CL_DW12(phy), ICL_LANE_ENABLE_AUX, 0);
 
-	val = intel_de_read(dev_priv, regs->driver);
-	intel_de_write(dev_priv, regs->driver,
-		       val & ~HSW_PWR_WELL_CTL_REQ(pw_idx));
+	intel_de_rmw(dev_priv, regs->driver, HSW_PWR_WELL_CTL_REQ(pw_idx), 0);
 
 	hsw_wait_for_power_well_disable(dev_priv, power_well);
 }
@@ -502,19 +498,15 @@ icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 	bool is_tbt = power_well->desc->is_tc_tbt;
 	bool timeout_expected;
-	u32 val;
 
 	icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port);
 
-	val = intel_de_read(dev_priv, DP_AUX_CH_CTL(aux_ch));
-	val &= ~DP_AUX_CH_CTL_TBT_IO;
-	if (is_tbt)
-		val |= DP_AUX_CH_CTL_TBT_IO;
-	intel_de_write(dev_priv, DP_AUX_CH_CTL(aux_ch), val);
+	intel_de_rmw(dev_priv, DP_AUX_CH_CTL(aux_ch),
+		     DP_AUX_CH_CTL_TBT_IO, is_tbt ? DP_AUX_CH_CTL_TBT_IO : 0);
 
-	val = intel_de_read(dev_priv, regs->driver);
-	intel_de_write(dev_priv, regs->driver,
-		       val | HSW_PWR_WELL_CTL_REQ(i915_power_well_instance(power_well)->hsw.idx));
+	intel_de_rmw(dev_priv, regs->driver,
+		     0,
+		     HSW_PWR_WELL_CTL_REQ(i915_power_well_instance(power_well)->hsw.idx));
 
 	/*
 	 * An AUX timeout is expected if the TBT DP tunnel is down,
@@ -700,19 +692,20 @@ static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 	return mask;
 }
 
-void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
+void gen9_sanitize_dc_state(struct drm_i915_private *i915)
 {
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 	u32 val;
 
-	if (!HAS_DISPLAY(dev_priv))
+	if (!HAS_DISPLAY(i915))
 		return;
 
-	val = intel_de_read(dev_priv, DC_STATE_EN) & gen9_dc_mask(dev_priv);
+	val = intel_de_read(i915, DC_STATE_EN) & gen9_dc_mask(i915);
 
-	drm_dbg_kms(&dev_priv->drm,
+	drm_dbg_kms(&i915->drm,
 		    "Resetting DC state tracking from %02x to %02x\n",
-		    dev_priv->display.dmc.dc_state, val);
-	dev_priv->display.dmc.dc_state = val;
+		    power_domains->dc_state, val);
+	power_domains->dc_state = val;
 }
 
 /**
@@ -740,6 +733,7 @@ void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
  */
 void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state)
 {
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	u32 val;
 	u32 mask;
 
@@ -747,8 +741,8 @@ void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state)
 		return;
 
 	if (drm_WARN_ON_ONCE(&dev_priv->drm,
-			     state & ~dev_priv->display.dmc.allowed_dc_mask))
-		state &= dev_priv->display.dmc.allowed_dc_mask;
+			     state & ~power_domains->allowed_dc_mask))
+		state &= power_domains->allowed_dc_mask;
 
 	val = intel_de_read(dev_priv, DC_STATE_EN);
 	mask = gen9_dc_mask(dev_priv);
@@ -756,16 +750,16 @@ void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state)
 		    val & mask, state);
 
 	/* Check if DMC is ignoring our DC state requests */
-	if ((val & mask) != dev_priv->display.dmc.dc_state)
+	if ((val & mask) != power_domains->dc_state)
 		drm_err(&dev_priv->drm, "DC state mismatch (0x%x -> 0x%x)\n",
-			dev_priv->display.dmc.dc_state, val & mask);
+			power_domains->dc_state, val & mask);
 
 	val &= ~mask;
 	val |= state;
 
 	gen9_write_dc_state(dev_priv, val);
 
-	dev_priv->display.dmc.dc_state = val & mask;
+	power_domains->dc_state = val & mask;
 }
 
 static void tgl_enable_dc3co(struct drm_i915_private *dev_priv)
@@ -776,12 +770,8 @@ static void tgl_enable_dc3co(struct drm_i915_private *dev_priv)
 
 static void tgl_disable_dc3co(struct drm_i915_private *dev_priv)
 {
-	u32 val;
-
 	drm_dbg_kms(&dev_priv->drm, "Disabling DC3CO\n");
-	val = intel_de_read(dev_priv, DC_STATE_EN);
-	val &= ~DC_STATE_DC3CO_STATUS;
-	intel_de_write(dev_priv, DC_STATE_EN, val);
+	intel_de_rmw(dev_priv, DC_STATE_EN, DC_STATE_DC3CO_STATUS, 0);
 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 	/*
 	 * Delay of 200us DC3CO Exit time B.Spec 49196
@@ -820,8 +810,8 @@ void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 
 	/* Wa Display #1183: skl,kbl,cfl */
 	if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv))
-		intel_de_write(dev_priv, GEN8_CHICKEN_DCPR_1,
-			       intel_de_read(dev_priv, GEN8_CHICKEN_DCPR_1) | SKL_SELECT_ALTERNATE_DC_EXIT);
+		intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
+			     0, SKL_SELECT_ALTERNATE_DC_EXIT);
 
 	gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 }
@@ -829,8 +819,10 @@ void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
 {
 	drm_WARN_ONCE(&dev_priv->drm,
-		      intel_de_read(dev_priv, UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
-		      "Backlight is not disabled.\n");
+		      (intel_de_read(dev_priv, UTIL_PIN_CTL) &
+		       (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) ==
+		      (UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
+		      "Utility pin enabled in PWM mode\n");
 	drm_WARN_ONCE(&dev_priv->drm,
 		      (intel_de_read(dev_priv, DC_STATE_EN) &
 		       DC_STATE_EN_UPTO_DC6),
@@ -847,8 +839,8 @@ void skl_enable_dc6(struct drm_i915_private *dev_priv)
 
 	/* Wa Display #1183: skl,kbl,cfl */
 	if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv))
-		intel_de_write(dev_priv, GEN8_CHICKEN_DCPR_1,
-			       intel_de_read(dev_priv, GEN8_CHICKEN_DCPR_1) | SKL_SELECT_ALTERNATE_DC_EXIT);
+		intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
+			     0, SKL_SELECT_ALTERNATE_DC_EXIT);
 
 	gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 }
@@ -957,9 +949,10 @@ static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
 
 void gen9_disable_dc_states(struct drm_i915_private *dev_priv)
 {
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct intel_cdclk_config cdclk_config = {};
 
-	if (dev_priv->display.dmc.target_dc_state == DC_STATE_EN_DC3CO) {
+	if (power_domains->target_dc_state == DC_STATE_EN_DC3CO) {
 		tgl_disable_dc3co(dev_priv);
 		return;
 	}
@@ -998,10 +991,12 @@ static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
 					   struct i915_power_well *power_well)
 {
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
+
 	if (!intel_dmc_has_payload(dev_priv))
 		return;
 
-	switch (dev_priv->display.dmc.target_dc_state) {
+	switch (power_domains->target_dc_state) {
 	case DC_STATE_EN_DC3CO:
 		tgl_enable_dc3co(dev_priv);
 		break;
@@ -1033,9 +1028,9 @@ static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
 					 struct i915_power_well *power_well)
 {
-	if ((intel_de_read(dev_priv, PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
+	if ((intel_de_read(dev_priv, TRANSCONF(PIPE_A)) & TRANSCONF_ENABLE) == 0)
 		i830_enable_pipe(dev_priv, PIPE_A);
-	if ((intel_de_read(dev_priv, PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
+	if ((intel_de_read(dev_priv, TRANSCONF(PIPE_B)) & TRANSCONF_ENABLE) == 0)
 		i830_enable_pipe(dev_priv, PIPE_B);
 }
 
@@ -1049,8 +1044,8 @@ static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
 static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
 					  struct i915_power_well *power_well)
 {
-	return intel_de_read(dev_priv, PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
-		intel_de_read(dev_priv, PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
+	return intel_de_read(dev_priv, TRANSCONF(PIPE_A)) & TRANSCONF_ENABLE &&
+		intel_de_read(dev_priv, TRANSCONF(PIPE_B)) & TRANSCONF_ENABLE;
 }
 
 static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
@@ -1149,18 +1144,14 @@ static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 
 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
 {
-	u32 val;
-
 	/*
 	 * On driver load, a pipe may be active and driving a DSI display.
 	 * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
 	 * (and never recovering) in this case. intel_dsi_post_disable() will
 	 * clear it when we turn off the display.
 	 */
-	val = intel_de_read(dev_priv, DSPCLK_GATE_D(dev_priv));
-	val &= DPOUNIT_CLOCK_GATE_DISABLE;
-	val |= VRHUNIT_CLOCK_GATE_DISABLE;
-	intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), val);
+	intel_de_rmw(dev_priv, DSPCLK_GATE_D(dev_priv),
+		     ~DPOUNIT_CLOCK_GATE_DISABLE, VRHUNIT_CLOCK_GATE_DISABLE);
 
 	/*
 	 * Disable trickle feed and enable pnd deadline calculation
@@ -1276,8 +1267,7 @@ static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 	 * both PLLs disabled, or we risk losing DPIO and PLL
 	 * synchronization.
 	 */
-	intel_de_write(dev_priv, DPIO_CTL,
-		       intel_de_read(dev_priv, DPIO_CTL) | DPIO_CMNRST);
+	intel_de_rmw(dev_priv, DPIO_CTL, 0, DPIO_CMNRST);
 }
 
 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
@@ -1289,8 +1279,7 @@ static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 		assert_pll_disabled(dev_priv, pipe);
 
 	/* Assert common reset */
-	intel_de_write(dev_priv, DPIO_CTL,
-		       intel_de_read(dev_priv, DPIO_CTL) & ~DPIO_CMNRST);
+	intel_de_rmw(dev_priv, DPIO_CTL, DPIO_CMNRST, 0);
 
 	vlv_set_power_well(dev_priv, power_well, false);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_reg_defs.h b/drivers/gpu/drm/i915/display/intel_display_reg_defs.h
index 02605418ff08..755c1ea8225c 100644
--- a/drivers/gpu/drm/i915/display/intel_display_reg_defs.h
+++ b/drivers/gpu/drm/i915/display/intel_display_reg_defs.h
@@ -13,7 +13,7 @@
 #define VLV_DISPLAY_BASE		0x180000
 
 /*
- * Named helper wrappers around _PICK_EVEN() and _PICK().
+ * Named helper wrappers around _PICK_EVEN() and _PICK_EVEN_2RANGES().
  */
 #define _PIPE(pipe, a, b)		_PICK_EVEN(pipe, a, b)
 #define _PLANE(plane, a, b)		_PICK_EVEN(plane, a, b)
@@ -29,12 +29,8 @@
 #define _MMIO_PLL(pll, a, b)		_MMIO(_PLL(pll, a, b))
 #define _MMIO_PHY(phy, a, b)		_MMIO(_PHY(phy, a, b))
 
-#define _PHY3(phy, ...)			_PICK(phy, __VA_ARGS__)
-
-#define _MMIO_PIPE3(pipe, a, b, c)	_MMIO(_PICK(pipe, a, b, c))
-#define _MMIO_PORT3(pipe, a, b, c)	_MMIO(_PICK(pipe, a, b, c))
-#define _MMIO_PHY3(phy, a, b, c)	_MMIO(_PHY3(phy, a, b, c))
-#define _MMIO_PLL3(pll, ...)		_MMIO(_PICK(pll, __VA_ARGS__))
+#define _MMIO_PIPE3(pipe, a, b, c)	_MMIO(_PICK_EVEN_2RANGES(pipe, 1, a, a, b, c))
+#define _MMIO_PORT3(pipe, a, b, c)	_MMIO(_PICK_EVEN_2RANGES(pipe, 1, a, a, b, c))
 
 /*
  * Device info offset array based helpers for groups of registers with unevenly
diff --git a/drivers/gpu/drm/i915/display/intel_display_rps.c b/drivers/gpu/drm/i915/display/intel_display_rps.c
new file mode 100644
index 000000000000..918d0327169a
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_display_rps.c
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#include <drm/drm_crtc.h>
+#include <drm/drm_vblank.h>
+
+#include "gt/intel_rps.h"
+#include "i915_drv.h"
+#include "intel_display_rps.h"
+#include "intel_display_types.h"
+
+struct wait_rps_boost {
+	struct wait_queue_entry wait;
+
+	struct drm_crtc *crtc;
+	struct i915_request *request;
+};
+
+static int do_rps_boost(struct wait_queue_entry *_wait,
+			unsigned mode, int sync, void *key)
+{
+	struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
+	struct i915_request *rq = wait->request;
+
+	/*
+	 * If we missed the vblank, but the request is already running it
+	 * is reasonable to assume that it will complete before the next
+	 * vblank without our intervention, so leave RPS alone.
+	 */
+	if (!i915_request_started(rq))
+		intel_rps_boost(rq);
+	i915_request_put(rq);
+
+	drm_crtc_vblank_put(wait->crtc);
+
+	list_del(&wait->wait.entry);
+	kfree(wait);
+	return 1;
+}
+
+void intel_display_rps_boost_after_vblank(struct drm_crtc *crtc,
+					  struct dma_fence *fence)
+{
+	struct wait_rps_boost *wait;
+
+	if (!dma_fence_is_i915(fence))
+		return;
+
+	if (DISPLAY_VER(to_i915(crtc->dev)) < 6)
+		return;
+
+	if (drm_crtc_vblank_get(crtc))
+		return;
+
+	wait = kmalloc(sizeof(*wait), GFP_KERNEL);
+	if (!wait) {
+		drm_crtc_vblank_put(crtc);
+		return;
+	}
+
+	wait->request = to_request(dma_fence_get(fence));
+	wait->crtc = crtc;
+
+	wait->wait.func = do_rps_boost;
+	wait->wait.flags = 0;
+
+	add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
+}
+
+void intel_display_rps_mark_interactive(struct drm_i915_private *i915,
+					struct intel_atomic_state *state,
+					bool interactive)
+{
+	if (state->rps_interactive == interactive)
+		return;
+
+	intel_rps_mark_interactive(&to_gt(i915)->rps, interactive);
+	state->rps_interactive = interactive;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_display_rps.h b/drivers/gpu/drm/i915/display/intel_display_rps.h
new file mode 100644
index 000000000000..e19009c2371a
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_display_rps.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_DISPLAY_RPS_H__
+#define __INTEL_DISPLAY_RPS_H__
+
+#include <linux/types.h>
+
+struct dma_fence;
+struct drm_crtc;
+struct drm_i915_private;
+struct intel_atomic_state;
+
+void intel_display_rps_boost_after_vblank(struct drm_crtc *crtc,
+					  struct dma_fence *fence);
+void intel_display_rps_mark_interactive(struct drm_i915_private *i915,
+					struct intel_atomic_state *state,
+					bool interactive);
+
+#endif /* __INTEL_DISPLAY_RPS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index 582234f0c49a..47395b39c8f4 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -43,7 +43,7 @@
 #include <drm/drm_rect.h>
 #include <drm/drm_vblank.h>
 #include <drm/drm_vblank_work.h>
-#include <drm/i915_mei_hdcp_interface.h>
+#include <drm/i915_hdcp_interface.h>
 #include <media/cec-notifier.h>
 
 #include "i915_vma.h"
@@ -53,13 +53,14 @@
 #include "intel_display_limits.h"
 #include "intel_display_power.h"
 #include "intel_dpll_mgr.h"
-#include "intel_pm_types.h"
+#include "intel_wm_types.h"
 
 struct drm_printer;
 struct __intel_global_objs_state;
 struct intel_ddi_buf_trans;
 struct intel_fbc;
 struct intel_connector;
+struct intel_tc_port;
 
 /*
  * Display related stuff
@@ -169,9 +170,6 @@ struct intel_encoder {
 	int (*compute_config_late)(struct intel_encoder *,
 				   struct intel_crtc_state *,
 				   struct drm_connector_state *);
-	void (*update_prepare)(struct intel_atomic_state *,
-			       struct intel_encoder *,
-			       struct intel_crtc *);
 	void (*pre_pll_enable)(struct intel_atomic_state *,
 			       struct intel_encoder *,
 			       const struct intel_crtc_state *,
@@ -184,9 +182,6 @@ struct intel_encoder {
 		       struct intel_encoder *,
 		       const struct intel_crtc_state *,
 		       const struct drm_connector_state *);
-	void (*update_complete)(struct intel_atomic_state *,
-				struct intel_encoder *,
-				struct intel_crtc *);
 	void (*disable)(struct intel_atomic_state *,
 			struct intel_encoder *,
 			const struct intel_crtc_state *,
@@ -255,6 +250,11 @@ struct intel_encoder {
 	 * Returns whether the port clock is enabled or not.
 	 */
 	bool (*is_clock_enabled)(struct intel_encoder *encoder);
+	/*
+	 * Returns the PLL type the port uses.
+	 */
+	enum icl_port_dpll_id (*port_pll_type)(struct intel_encoder *encoder,
+					       const struct intel_crtc_state *crtc_state);
 	const struct intel_ddi_buf_trans *(*get_buf_trans)(struct intel_encoder *encoder,
 							   const struct intel_crtc_state *crtc_state,
 							   int *n_entries);
@@ -326,6 +326,7 @@ struct intel_vbt_panel_data {
 	struct {
 		u16 pwm_freq_hz;
 		u16 brightness_precision_bits;
+		u16 hdr_dpcd_refresh_timeout;
 		bool present;
 		bool active_low_pwm;
 		u8 min_brightness;	/* min_brightness/255 of max */
@@ -1151,6 +1152,7 @@ struct intel_crtc_state {
 	bool has_psr2;
 	bool enable_psr2_sel_fetch;
 	bool req_psr2_sdp_prior_scanline;
+	bool wm_level_disabled;
 	u32 dc3co_exitline;
 	u16 su_y_granularity;
 	struct drm_dp_vsc_sdp psr_vsc;
@@ -1249,6 +1251,9 @@ struct intel_crtc_state {
 	/* bitmask of planes that will be updated during the commit */
 	u8 update_planes;
 
+	/* bitmask of planes with async flip active */
+	u8 async_flip_planes;
+
 	u8 framestart_delay; /* 1-4 */
 	u8 msa_timing_delay; /* 0-3 */
 
@@ -1502,17 +1507,6 @@ struct intel_watermark_params {
 	u8 cacheline_size;
 };
 
-struct cxsr_latency {
-	bool is_desktop : 1;
-	bool is_ddr3 : 1;
-	u16 fsb_freq;
-	u16 mem_freq;
-	u16 display_sr;
-	u16 display_hpll_disable;
-	u16 cursor_sr;
-	u16 cursor_hpll_disable;
-};
-
 #define to_intel_atomic_state(x) container_of(x, struct intel_atomic_state, base)
 #define to_intel_crtc(x) container_of(x, struct intel_crtc, base)
 #define to_intel_crtc_state(x) container_of(x, struct intel_crtc_state, uapi)
@@ -1742,6 +1736,7 @@ struct intel_dp {
 
 	/* Display stream compression testing */
 	bool force_dsc_en;
+	int force_dsc_output_format;
 	int force_dsc_bpc;
 
 	bool hobl_failed;
@@ -1782,16 +1777,7 @@ struct intel_digital_port {
 	intel_wakeref_t ddi_io_wakeref;
 	intel_wakeref_t aux_wakeref;
 
-	struct mutex tc_lock;	/* protects the TypeC port mode */
-	intel_wakeref_t tc_lock_wakeref;
-	enum intel_display_power_domain tc_lock_power_domain;
-	struct delayed_work tc_disconnect_phy_work;
-	int tc_link_refcount;
-	bool tc_legacy_port:1;
-	char tc_port_name[8];
-	enum tc_port_mode tc_mode;
-	enum phy_fia tc_phy_fia;
-	u8 tc_phy_fia_idx;
+	struct intel_tc_port *tc;
 
 	/* protects num_hdcp_streams reference count, hdcp_port_data and hdcp_auth_status */
 	struct mutex hdcp_mutex;
diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c
index 3485d5e6dd3c..8a88de67ff0a 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc.c
+++ b/drivers/gpu/drm/i915/display/intel_dmc.c
@@ -38,6 +38,39 @@
  * low-power state and comes back to normal.
  */
 
+enum intel_dmc_id {
+	DMC_FW_MAIN = 0,
+	DMC_FW_PIPEA,
+	DMC_FW_PIPEB,
+	DMC_FW_PIPEC,
+	DMC_FW_PIPED,
+	DMC_FW_MAX
+};
+
+struct intel_dmc {
+	struct drm_i915_private *i915;
+	struct work_struct work;
+	const char *fw_path;
+	u32 max_fw_size; /* bytes */
+	u32 version;
+	struct dmc_fw_info {
+		u32 mmio_count;
+		i915_reg_t mmioaddr[20];
+		u32 mmiodata[20];
+		u32 dmc_offset;
+		u32 start_mmioaddr;
+		u32 dmc_fw_size; /*dwords */
+		u32 *payload;
+		bool present;
+	} dmc_info[DMC_FW_MAX];
+};
+
+/* Note: This may be NULL. */
+static struct intel_dmc *i915_to_dmc(struct drm_i915_private *i915)
+{
+	return i915->display.dmc.dmc;
+}
+
 #define DMC_VERSION(major, minor)	((major) << 16 | (minor))
 #define DMC_VERSION_MAJOR(version)	((version) >> 16)
 #define DMC_VERSION_MINOR(version)	((version) & 0xffff)
@@ -56,10 +89,13 @@
 	__stringify(major) "_"			\
 	__stringify(minor) ".bin"
 
+#define XELPDP_DMC_MAX_FW_SIZE		0x7000
 #define DISPLAY_VER13_DMC_MAX_FW_SIZE	0x20000
-
 #define DISPLAY_VER12_DMC_MAX_FW_SIZE	ICL_DMC_MAX_FW_SIZE
 
+#define MTL_DMC_PATH			DMC_PATH(mtl)
+MODULE_FIRMWARE(MTL_DMC_PATH);
+
 #define DG2_DMC_PATH			DMC_LEGACY_PATH(dg2, 2, 08)
 MODULE_FIRMWARE(DG2_DMC_PATH);
 
@@ -249,9 +285,19 @@ struct stepping_info {
 	char substepping;
 };
 
-static bool has_dmc_id_fw(struct drm_i915_private *i915, int dmc_id)
+#define for_each_dmc_id(__dmc_id) \
+	for ((__dmc_id) = DMC_FW_MAIN; (__dmc_id) < DMC_FW_MAX; (__dmc_id)++)
+
+static bool is_valid_dmc_id(enum intel_dmc_id dmc_id)
 {
-	return i915->display.dmc.dmc_info[dmc_id].payload;
+	return dmc_id >= DMC_FW_MAIN && dmc_id < DMC_FW_MAX;
+}
+
+static bool has_dmc_id_fw(struct drm_i915_private *i915, enum intel_dmc_id dmc_id)
+{
+	struct intel_dmc *dmc = i915_to_dmc(i915);
+
+	return dmc && dmc->dmc_info[dmc_id].payload;
 }
 
 bool intel_dmc_has_payload(struct drm_i915_private *i915)
@@ -270,12 +316,12 @@ intel_get_stepping_info(struct drm_i915_private *i915,
 	return si;
 }
 
-static void gen9_set_dc_state_debugmask(struct drm_i915_private *dev_priv)
+static void gen9_set_dc_state_debugmask(struct drm_i915_private *i915)
 {
 	/* The below bit doesn't need to be cleared ever afterwards */
-	intel_de_rmw(dev_priv, DC_STATE_DEBUG, 0,
+	intel_de_rmw(i915, DC_STATE_DEBUG, 0,
 		     DC_STATE_DEBUG_MASK_CORES | DC_STATE_DEBUG_MASK_MEMORY_UP);
-	intel_de_posting_read(dev_priv, DC_STATE_DEBUG);
+	intel_de_posting_read(i915, DC_STATE_DEBUG);
 }
 
 static void disable_event_handler(struct drm_i915_private *i915,
@@ -315,26 +361,23 @@ disable_flip_queue_event(struct drm_i915_private *i915,
 }
 
 static bool
-get_flip_queue_event_regs(struct drm_i915_private *i915, int dmc_id,
+get_flip_queue_event_regs(struct drm_i915_private *i915, enum intel_dmc_id dmc_id,
 			  i915_reg_t *ctl_reg, i915_reg_t *htp_reg)
 {
-	switch (dmc_id) {
-	case DMC_FW_MAIN:
+	if (dmc_id == DMC_FW_MAIN) {
 		if (DISPLAY_VER(i915) == 12) {
 			*ctl_reg = DMC_EVT_CTL(i915, dmc_id, 3);
 			*htp_reg = DMC_EVT_HTP(i915, dmc_id, 3);
 
 			return true;
 		}
-		break;
-	case DMC_FW_PIPEA ... DMC_FW_PIPED:
+	} else if (dmc_id >= DMC_FW_PIPEA && dmc_id <= DMC_FW_PIPED) {
 		if (IS_DG2(i915)) {
 			*ctl_reg = DMC_EVT_CTL(i915, dmc_id, 2);
 			*htp_reg = DMC_EVT_HTP(i915, dmc_id, 2);
 
 			return true;
 		}
-		break;
 	}
 
 	return false;
@@ -343,13 +386,13 @@ get_flip_queue_event_regs(struct drm_i915_private *i915, int dmc_id,
 static void
 disable_all_flip_queue_events(struct drm_i915_private *i915)
 {
-	int dmc_id;
+	enum intel_dmc_id dmc_id;
 
 	/* TODO: check if the following applies to all D13+ platforms. */
 	if (!IS_DG2(i915) && !IS_TIGERLAKE(i915))
 		return;
 
-	for (dmc_id = 0; dmc_id < DMC_FW_MAX; dmc_id++) {
+	for_each_dmc_id(dmc_id) {
 		i915_reg_t ctl_reg;
 		i915_reg_t htp_reg;
 
@@ -365,22 +408,22 @@ disable_all_flip_queue_events(struct drm_i915_private *i915)
 
 static void disable_all_event_handlers(struct drm_i915_private *i915)
 {
-	int id;
+	enum intel_dmc_id dmc_id;
 
 	/* TODO: disable the event handlers on pre-GEN12 platforms as well */
 	if (DISPLAY_VER(i915) < 12)
 		return;
 
-	for (id = DMC_FW_MAIN; id < DMC_FW_MAX; id++) {
+	for_each_dmc_id(dmc_id) {
 		int handler;
 
-		if (!has_dmc_id_fw(i915, id))
+		if (!has_dmc_id_fw(i915, dmc_id))
 			continue;
 
 		for (handler = 0; handler < DMC_EVENT_HANDLER_COUNT_GEN12; handler++)
 			disable_event_handler(i915,
-					      DMC_EVT_CTL(i915, id, handler),
-					      DMC_EVT_HTP(i915, id, handler));
+					      DMC_EVT_CTL(i915, dmc_id, handler),
+					      DMC_EVT_HTP(i915, dmc_id, handler));
 	}
 }
 
@@ -426,7 +469,9 @@ static void pipedmc_clock_gating_wa(struct drm_i915_private *i915, bool enable)
 
 void intel_dmc_enable_pipe(struct drm_i915_private *i915, enum pipe pipe)
 {
-	if (!has_dmc_id_fw(i915, PIPE_TO_DMC_ID(pipe)))
+	enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(pipe);
+
+	if (!is_valid_dmc_id(dmc_id) || !has_dmc_id_fw(i915, dmc_id))
 		return;
 
 	if (DISPLAY_VER(i915) >= 14)
@@ -437,7 +482,9 @@ void intel_dmc_enable_pipe(struct drm_i915_private *i915, enum pipe pipe)
 
 void intel_dmc_disable_pipe(struct drm_i915_private *i915, enum pipe pipe)
 {
-	if (!has_dmc_id_fw(i915, PIPE_TO_DMC_ID(pipe)))
+	enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(pipe);
+
+	if (!is_valid_dmc_id(dmc_id) || !has_dmc_id_fw(i915, dmc_id))
 		return;
 
 	if (DISPLAY_VER(i915) >= 14)
@@ -448,57 +495,59 @@ void intel_dmc_disable_pipe(struct drm_i915_private *i915, enum pipe pipe)
 
 /**
  * intel_dmc_load_program() - write the firmware from memory to register.
- * @dev_priv: i915 drm device.
+ * @i915: i915 drm device.
  *
  * DMC firmware is read from a .bin file and kept in internal memory one time.
  * Everytime display comes back from low power state this function is called to
  * copy the firmware from internal memory to registers.
  */
-void intel_dmc_load_program(struct drm_i915_private *dev_priv)
+void intel_dmc_load_program(struct drm_i915_private *i915)
 {
-	struct intel_dmc *dmc = &dev_priv->display.dmc;
-	u32 id, i;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
+	struct intel_dmc *dmc = i915_to_dmc(i915);
+	enum intel_dmc_id dmc_id;
+	u32 i;
 
-	if (!intel_dmc_has_payload(dev_priv))
+	if (!intel_dmc_has_payload(i915))
 		return;
 
-	pipedmc_clock_gating_wa(dev_priv, true);
+	pipedmc_clock_gating_wa(i915, true);
 
-	disable_all_event_handlers(dev_priv);
+	disable_all_event_handlers(i915);
 
-	assert_rpm_wakelock_held(&dev_priv->runtime_pm);
+	assert_rpm_wakelock_held(&i915->runtime_pm);
 
 	preempt_disable();
 
-	for (id = 0; id < DMC_FW_MAX; id++) {
-		for (i = 0; i < dmc->dmc_info[id].dmc_fw_size; i++) {
-			intel_de_write_fw(dev_priv,
-					  DMC_PROGRAM(dmc->dmc_info[id].start_mmioaddr, i),
-					  dmc->dmc_info[id].payload[i]);
+	for_each_dmc_id(dmc_id) {
+		for (i = 0; i < dmc->dmc_info[dmc_id].dmc_fw_size; i++) {
+			intel_de_write_fw(i915,
+					  DMC_PROGRAM(dmc->dmc_info[dmc_id].start_mmioaddr, i),
+					  dmc->dmc_info[dmc_id].payload[i]);
 		}
 	}
 
 	preempt_enable();
 
-	for (id = 0; id < DMC_FW_MAX; id++) {
-		for (i = 0; i < dmc->dmc_info[id].mmio_count; i++) {
-			intel_de_write(dev_priv, dmc->dmc_info[id].mmioaddr[i],
-				       dmc->dmc_info[id].mmiodata[i]);
+	for_each_dmc_id(dmc_id) {
+		for (i = 0; i < dmc->dmc_info[dmc_id].mmio_count; i++) {
+			intel_de_write(i915, dmc->dmc_info[dmc_id].mmioaddr[i],
+				       dmc->dmc_info[dmc_id].mmiodata[i]);
 		}
 	}
 
-	dev_priv->display.dmc.dc_state = 0;
+	power_domains->dc_state = 0;
 
-	gen9_set_dc_state_debugmask(dev_priv);
+	gen9_set_dc_state_debugmask(i915);
 
 	/*
 	 * Flip queue events need to be disabled before enabling DC5/6.
 	 * i915 doesn't use the flip queue feature, so disable it already
 	 * here.
 	 */
-	disable_all_flip_queue_events(dev_priv);
+	disable_all_flip_queue_events(i915);
 
-	pipedmc_clock_gating_wa(dev_priv, false);
+	pipedmc_clock_gating_wa(i915, false);
 }
 
 /**
@@ -520,8 +569,11 @@ void intel_dmc_disable_program(struct drm_i915_private *i915)
 
 void assert_dmc_loaded(struct drm_i915_private *i915)
 {
-	drm_WARN_ONCE(&i915->drm,
-		      !intel_de_read(i915, DMC_PROGRAM(i915->display.dmc.dmc_info[DMC_FW_MAIN].start_mmioaddr, 0)),
+	struct intel_dmc *dmc = i915_to_dmc(i915);
+
+	drm_WARN_ONCE(&i915->drm, !dmc, "DMC not initialized\n");
+	drm_WARN_ONCE(&i915->drm, dmc &&
+		      !intel_de_read(i915, DMC_PROGRAM(dmc->dmc_info[DMC_FW_MAIN].start_mmioaddr, 0)),
 		      "DMC program storage start is NULL\n");
 	drm_WARN_ONCE(&i915->drm, !intel_de_read(i915, DMC_SSP_BASE),
 		      "DMC SSP Base Not fine\n");
@@ -556,15 +608,15 @@ static void dmc_set_fw_offset(struct intel_dmc *dmc,
 			      const struct stepping_info *si,
 			      u8 package_ver)
 {
-	unsigned int i, id;
-
-	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc);
+	struct drm_i915_private *i915 = dmc->i915;
+	enum intel_dmc_id dmc_id;
+	unsigned int i;
 
 	for (i = 0; i < num_entries; i++) {
-		id = package_ver <= 1 ? DMC_FW_MAIN : fw_info[i].dmc_id;
+		dmc_id = package_ver <= 1 ? DMC_FW_MAIN : fw_info[i].dmc_id;
 
-		if (id >= DMC_FW_MAX) {
-			drm_dbg(&i915->drm, "Unsupported firmware id: %u\n", id);
+		if (!is_valid_dmc_id(dmc_id)) {
+			drm_dbg(&i915->drm, "Unsupported firmware id: %u\n", dmc_id);
 			continue;
 		}
 
@@ -572,29 +624,24 @@ static void dmc_set_fw_offset(struct intel_dmc *dmc,
 		 * check for the stepping since we already found a previous FW
 		 * for this id.
 		 */
-		if (dmc->dmc_info[id].present)
+		if (dmc->dmc_info[dmc_id].present)
 			continue;
 
 		if (fw_info_matches_stepping(&fw_info[i], si)) {
-			dmc->dmc_info[id].present = true;
-			dmc->dmc_info[id].dmc_offset = fw_info[i].offset;
+			dmc->dmc_info[dmc_id].present = true;
+			dmc->dmc_info[dmc_id].dmc_offset = fw_info[i].offset;
 		}
 	}
 }
 
 static bool dmc_mmio_addr_sanity_check(struct intel_dmc *dmc,
 				       const u32 *mmioaddr, u32 mmio_count,
-				       int header_ver, u8 dmc_id)
+				       int header_ver, enum intel_dmc_id dmc_id)
 {
-	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc);
+	struct drm_i915_private *i915 = dmc->i915;
 	u32 start_range, end_range;
 	int i;
 
-	if (dmc_id >= DMC_FW_MAX) {
-		drm_warn(&i915->drm, "Unsupported firmware id %u\n", dmc_id);
-		return false;
-	}
-
 	if (header_ver == 1) {
 		start_range = DMC_MMIO_START_RANGE;
 		end_range = DMC_MMIO_END_RANGE;
@@ -622,9 +669,9 @@ static bool dmc_mmio_addr_sanity_check(struct intel_dmc *dmc,
 
 static u32 parse_dmc_fw_header(struct intel_dmc *dmc,
 			       const struct intel_dmc_header_base *dmc_header,
-			       size_t rem_size, u8 dmc_id)
+			       size_t rem_size, enum intel_dmc_id dmc_id)
 {
-	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc);
+	struct drm_i915_private *i915 = dmc->i915;
 	struct dmc_fw_info *dmc_info = &dmc->dmc_info[dmc_id];
 	unsigned int header_len_bytes, dmc_header_size, payload_size, i;
 	const u32 *mmioaddr, *mmiodata;
@@ -735,7 +782,7 @@ parse_dmc_fw_package(struct intel_dmc *dmc,
 		     const struct stepping_info *si,
 		     size_t rem_size)
 {
-	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc);
+	struct drm_i915_private *i915 = dmc->i915;
 	u32 package_size = sizeof(struct intel_package_header);
 	u32 num_entries, max_entries;
 	const struct intel_fw_info *fw_info;
@@ -789,7 +836,7 @@ static u32 parse_dmc_fw_css(struct intel_dmc *dmc,
 			    struct intel_css_header *css_header,
 			    size_t rem_size)
 {
-	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc);
+	struct drm_i915_private *i915 = dmc->i915;
 
 	if (rem_size < sizeof(struct intel_css_header)) {
 		drm_err(&i915->drm, "Truncated DMC firmware, refusing.\n");
@@ -809,18 +856,17 @@ static u32 parse_dmc_fw_css(struct intel_dmc *dmc,
 	return sizeof(struct intel_css_header);
 }
 
-static void parse_dmc_fw(struct drm_i915_private *dev_priv,
-			 const struct firmware *fw)
+static void parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
 {
+	struct drm_i915_private *i915 = dmc->i915;
 	struct intel_css_header *css_header;
 	struct intel_package_header *package_header;
 	struct intel_dmc_header_base *dmc_header;
-	struct intel_dmc *dmc = &dev_priv->display.dmc;
 	struct stepping_info display_info = { '*', '*'};
-	const struct stepping_info *si = intel_get_stepping_info(dev_priv, &display_info);
+	const struct stepping_info *si = intel_get_stepping_info(i915, &display_info);
+	enum intel_dmc_id dmc_id;
 	u32 readcount = 0;
 	u32 r, offset;
-	int id;
 
 	if (!fw)
 		return;
@@ -841,34 +887,33 @@ static void parse_dmc_fw(struct drm_i915_private *dev_priv,
 
 	readcount += r;
 
-	for (id = 0; id < DMC_FW_MAX; id++) {
-		if (!dev_priv->display.dmc.dmc_info[id].present)
+	for_each_dmc_id(dmc_id) {
+		if (!dmc->dmc_info[dmc_id].present)
 			continue;
 
-		offset = readcount + dmc->dmc_info[id].dmc_offset * 4;
+		offset = readcount + dmc->dmc_info[dmc_id].dmc_offset * 4;
 		if (offset > fw->size) {
-			drm_err(&dev_priv->drm, "Reading beyond the fw_size\n");
+			drm_err(&i915->drm, "Reading beyond the fw_size\n");
 			continue;
 		}
 
 		dmc_header = (struct intel_dmc_header_base *)&fw->data[offset];
-		parse_dmc_fw_header(dmc, dmc_header, fw->size - offset, id);
+		parse_dmc_fw_header(dmc, dmc_header, fw->size - offset, dmc_id);
 	}
 }
 
-static void intel_dmc_runtime_pm_get(struct drm_i915_private *dev_priv)
+static void intel_dmc_runtime_pm_get(struct drm_i915_private *i915)
 {
-	drm_WARN_ON(&dev_priv->drm, dev_priv->display.dmc.wakeref);
-	dev_priv->display.dmc.wakeref =
-		intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
+	drm_WARN_ON(&i915->drm, i915->display.dmc.wakeref);
+	i915->display.dmc.wakeref = intel_display_power_get(i915, POWER_DOMAIN_INIT);
 }
 
-static void intel_dmc_runtime_pm_put(struct drm_i915_private *dev_priv)
+static void intel_dmc_runtime_pm_put(struct drm_i915_private *i915)
 {
 	intel_wakeref_t wakeref __maybe_unused =
-		fetch_and_zero(&dev_priv->display.dmc.wakeref);
+		fetch_and_zero(&i915->display.dmc.wakeref);
 
-	intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
+	intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
 }
 
 static const char *dmc_fallback_path(struct drm_i915_private *i915)
@@ -881,46 +926,40 @@ static const char *dmc_fallback_path(struct drm_i915_private *i915)
 
 static void dmc_load_work_fn(struct work_struct *work)
 {
-	struct drm_i915_private *dev_priv;
-	struct intel_dmc *dmc;
+	struct intel_dmc *dmc = container_of(work, typeof(*dmc), work);
+	struct drm_i915_private *i915 = dmc->i915;
 	const struct firmware *fw = NULL;
 	const char *fallback_path;
 	int err;
 
-	dev_priv = container_of(work, typeof(*dev_priv), display.dmc.work);
-	dmc = &dev_priv->display.dmc;
-
-	err = request_firmware(&fw, dev_priv->display.dmc.fw_path, dev_priv->drm.dev);
+	err = request_firmware(&fw, dmc->fw_path, i915->drm.dev);
 
-	if (err == -ENOENT && !dev_priv->params.dmc_firmware_path) {
-		fallback_path = dmc_fallback_path(dev_priv);
+	if (err == -ENOENT && !i915->params.dmc_firmware_path) {
+		fallback_path = dmc_fallback_path(i915);
 		if (fallback_path) {
-			drm_dbg_kms(&dev_priv->drm,
-				    "%s not found, falling back to %s\n",
-				    dmc->fw_path,
-				    fallback_path);
-			err = request_firmware(&fw, fallback_path, dev_priv->drm.dev);
+			drm_dbg_kms(&i915->drm, "%s not found, falling back to %s\n",
+				    dmc->fw_path, fallback_path);
+			err = request_firmware(&fw, fallback_path, i915->drm.dev);
 			if (err == 0)
-				dev_priv->display.dmc.fw_path = fallback_path;
+				dmc->fw_path = fallback_path;
 		}
 	}
 
-	parse_dmc_fw(dev_priv, fw);
+	parse_dmc_fw(dmc, fw);
 
-	if (intel_dmc_has_payload(dev_priv)) {
-		intel_dmc_load_program(dev_priv);
-		intel_dmc_runtime_pm_put(dev_priv);
+	if (intel_dmc_has_payload(i915)) {
+		intel_dmc_load_program(i915);
+		intel_dmc_runtime_pm_put(i915);
 
-		drm_info(&dev_priv->drm,
-			 "Finished loading DMC firmware %s (v%u.%u)\n",
-			 dev_priv->display.dmc.fw_path, DMC_VERSION_MAJOR(dmc->version),
+		drm_info(&i915->drm, "Finished loading DMC firmware %s (v%u.%u)\n",
+			 dmc->fw_path, DMC_VERSION_MAJOR(dmc->version),
 			 DMC_VERSION_MINOR(dmc->version));
 	} else {
-		drm_notice(&dev_priv->drm,
+		drm_notice(&i915->drm,
 			   "Failed to load DMC firmware %s."
 			   " Disabling runtime power management.\n",
 			   dmc->fw_path);
-		drm_notice(&dev_priv->drm, "DMC firmware homepage: %s",
+		drm_notice(&i915->drm, "DMC firmware homepage: %s",
 			   INTEL_UC_FIRMWARE_URL);
 	}
 
@@ -928,19 +967,17 @@ static void dmc_load_work_fn(struct work_struct *work)
 }
 
 /**
- * intel_dmc_ucode_init() - initialize the firmware loading.
- * @dev_priv: i915 drm device.
+ * intel_dmc_init() - initialize the firmware loading.
+ * @i915: i915 drm device.
  *
  * This function is called at the time of loading the display driver to read
  * firmware from a .bin file and copied into a internal memory.
  */
-void intel_dmc_ucode_init(struct drm_i915_private *dev_priv)
+void intel_dmc_init(struct drm_i915_private *i915)
 {
-	struct intel_dmc *dmc = &dev_priv->display.dmc;
-
-	INIT_WORK(&dev_priv->display.dmc.work, dmc_load_work_fn);
+	struct intel_dmc *dmc;
 
-	if (!HAS_DMC(dev_priv))
+	if (!HAS_DMC(i915))
 		return;
 
 	/*
@@ -951,168 +988,195 @@ void intel_dmc_ucode_init(struct drm_i915_private *dev_priv)
 	 * suspend as runtime suspend *requires* a working DMC for whatever
 	 * reason.
 	 */
-	intel_dmc_runtime_pm_get(dev_priv);
+	intel_dmc_runtime_pm_get(i915);
+
+	dmc = kzalloc(sizeof(*dmc), GFP_KERNEL);
+	if (!dmc)
+		return;
+
+	dmc->i915 = i915;
 
-	if (IS_DG2(dev_priv)) {
+	INIT_WORK(&dmc->work, dmc_load_work_fn);
+
+	if (IS_METEORLAKE(i915)) {
+		dmc->fw_path = MTL_DMC_PATH;
+		dmc->max_fw_size = XELPDP_DMC_MAX_FW_SIZE;
+	} else if (IS_DG2(i915)) {
 		dmc->fw_path = DG2_DMC_PATH;
 		dmc->max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE;
-	} else if (IS_ALDERLAKE_P(dev_priv)) {
+	} else if (IS_ALDERLAKE_P(i915)) {
 		dmc->fw_path = ADLP_DMC_PATH;
 		dmc->max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE;
-	} else if (IS_ALDERLAKE_S(dev_priv)) {
+	} else if (IS_ALDERLAKE_S(i915)) {
 		dmc->fw_path = ADLS_DMC_PATH;
 		dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
-	} else if (IS_DG1(dev_priv)) {
+	} else if (IS_DG1(i915)) {
 		dmc->fw_path = DG1_DMC_PATH;
 		dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
-	} else if (IS_ROCKETLAKE(dev_priv)) {
+	} else if (IS_ROCKETLAKE(i915)) {
 		dmc->fw_path = RKL_DMC_PATH;
 		dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
-	} else if (IS_TIGERLAKE(dev_priv)) {
+	} else if (IS_TIGERLAKE(i915)) {
 		dmc->fw_path = TGL_DMC_PATH;
 		dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
-	} else if (DISPLAY_VER(dev_priv) == 11) {
+	} else if (DISPLAY_VER(i915) == 11) {
 		dmc->fw_path = ICL_DMC_PATH;
 		dmc->max_fw_size = ICL_DMC_MAX_FW_SIZE;
-	} else if (IS_GEMINILAKE(dev_priv)) {
+	} else if (IS_GEMINILAKE(i915)) {
 		dmc->fw_path = GLK_DMC_PATH;
 		dmc->max_fw_size = GLK_DMC_MAX_FW_SIZE;
-	} else if (IS_KABYLAKE(dev_priv) ||
-		   IS_COFFEELAKE(dev_priv) ||
-		   IS_COMETLAKE(dev_priv)) {
+	} else if (IS_KABYLAKE(i915) ||
+		   IS_COFFEELAKE(i915) ||
+		   IS_COMETLAKE(i915)) {
 		dmc->fw_path = KBL_DMC_PATH;
 		dmc->max_fw_size = KBL_DMC_MAX_FW_SIZE;
-	} else if (IS_SKYLAKE(dev_priv)) {
+	} else if (IS_SKYLAKE(i915)) {
 		dmc->fw_path = SKL_DMC_PATH;
 		dmc->max_fw_size = SKL_DMC_MAX_FW_SIZE;
-	} else if (IS_BROXTON(dev_priv)) {
+	} else if (IS_BROXTON(i915)) {
 		dmc->fw_path = BXT_DMC_PATH;
 		dmc->max_fw_size = BXT_DMC_MAX_FW_SIZE;
 	}
 
-	if (dev_priv->params.dmc_firmware_path) {
-		if (strlen(dev_priv->params.dmc_firmware_path) == 0) {
-			dmc->fw_path = NULL;
-			drm_info(&dev_priv->drm,
+	if (i915->params.dmc_firmware_path) {
+		if (strlen(i915->params.dmc_firmware_path) == 0) {
+			drm_info(&i915->drm,
 				 "Disabling DMC firmware and runtime PM\n");
-			return;
+			goto out;
 		}
 
-		dmc->fw_path = dev_priv->params.dmc_firmware_path;
+		dmc->fw_path = i915->params.dmc_firmware_path;
 	}
 
 	if (!dmc->fw_path) {
-		drm_dbg_kms(&dev_priv->drm,
+		drm_dbg_kms(&i915->drm,
 			    "No known DMC firmware for platform, disabling runtime PM\n");
-		return;
+		goto out;
 	}
 
-	drm_dbg_kms(&dev_priv->drm, "Loading %s\n", dmc->fw_path);
-	schedule_work(&dev_priv->display.dmc.work);
+	i915->display.dmc.dmc = dmc;
+
+	drm_dbg_kms(&i915->drm, "Loading %s\n", dmc->fw_path);
+	schedule_work(&dmc->work);
+
+	return;
+
+out:
+	kfree(dmc);
 }
 
 /**
- * intel_dmc_ucode_suspend() - prepare DMC firmware before system suspend
- * @dev_priv: i915 drm device
+ * intel_dmc_suspend() - prepare DMC firmware before system suspend
+ * @i915: i915 drm device
  *
  * Prepare the DMC firmware before entering system suspend. This includes
  * flushing pending work items and releasing any resources acquired during
  * init.
  */
-void intel_dmc_ucode_suspend(struct drm_i915_private *dev_priv)
+void intel_dmc_suspend(struct drm_i915_private *i915)
 {
-	if (!HAS_DMC(dev_priv))
+	struct intel_dmc *dmc = i915_to_dmc(i915);
+
+	if (!HAS_DMC(i915))
 		return;
 
-	flush_work(&dev_priv->display.dmc.work);
+	if (dmc)
+		flush_work(&dmc->work);
 
 	/* Drop the reference held in case DMC isn't loaded. */
-	if (!intel_dmc_has_payload(dev_priv))
-		intel_dmc_runtime_pm_put(dev_priv);
+	if (!intel_dmc_has_payload(i915))
+		intel_dmc_runtime_pm_put(i915);
 }
 
 /**
- * intel_dmc_ucode_resume() - init DMC firmware during system resume
- * @dev_priv: i915 drm device
+ * intel_dmc_resume() - init DMC firmware during system resume
+ * @i915: i915 drm device
  *
  * Reinitialize the DMC firmware during system resume, reacquiring any
- * resources released in intel_dmc_ucode_suspend().
+ * resources released in intel_dmc_suspend().
  */
-void intel_dmc_ucode_resume(struct drm_i915_private *dev_priv)
+void intel_dmc_resume(struct drm_i915_private *i915)
 {
-	if (!HAS_DMC(dev_priv))
+	if (!HAS_DMC(i915))
 		return;
 
 	/*
 	 * Reacquire the reference to keep RPM disabled in case DMC isn't
 	 * loaded.
 	 */
-	if (!intel_dmc_has_payload(dev_priv))
-		intel_dmc_runtime_pm_get(dev_priv);
+	if (!intel_dmc_has_payload(i915))
+		intel_dmc_runtime_pm_get(i915);
 }
 
 /**
- * intel_dmc_ucode_fini() - unload the DMC firmware.
- * @dev_priv: i915 drm device.
+ * intel_dmc_fini() - unload the DMC firmware.
+ * @i915: i915 drm device.
  *
  * Firmmware unloading includes freeing the internal memory and reset the
  * firmware loading status.
  */
-void intel_dmc_ucode_fini(struct drm_i915_private *dev_priv)
+void intel_dmc_fini(struct drm_i915_private *i915)
 {
-	int id;
+	struct intel_dmc *dmc = i915_to_dmc(i915);
+	enum intel_dmc_id dmc_id;
 
-	if (!HAS_DMC(dev_priv))
+	if (!HAS_DMC(i915))
 		return;
 
-	intel_dmc_ucode_suspend(dev_priv);
-	drm_WARN_ON(&dev_priv->drm, dev_priv->display.dmc.wakeref);
+	intel_dmc_suspend(i915);
+	drm_WARN_ON(&i915->drm, i915->display.dmc.wakeref);
 
-	for (id = 0; id < DMC_FW_MAX; id++)
-		kfree(dev_priv->display.dmc.dmc_info[id].payload);
+	if (dmc) {
+		for_each_dmc_id(dmc_id)
+			kfree(dmc->dmc_info[dmc_id].payload);
+
+		kfree(dmc);
+		i915->display.dmc.dmc = NULL;
+	}
 }
 
 void intel_dmc_print_error_state(struct drm_i915_error_state_buf *m,
 				 struct drm_i915_private *i915)
 {
-	struct intel_dmc *dmc = &i915->display.dmc;
+	struct intel_dmc *dmc = i915_to_dmc(i915);
 
 	if (!HAS_DMC(i915))
 		return;
 
+	i915_error_printf(m, "DMC initialized: %s\n", str_yes_no(dmc));
 	i915_error_printf(m, "DMC loaded: %s\n",
 			  str_yes_no(intel_dmc_has_payload(i915)));
-	i915_error_printf(m, "DMC fw version: %d.%d\n",
-			  DMC_VERSION_MAJOR(dmc->version),
-			  DMC_VERSION_MINOR(dmc->version));
+	if (dmc)
+		i915_error_printf(m, "DMC fw version: %d.%d\n",
+				  DMC_VERSION_MAJOR(dmc->version),
+				  DMC_VERSION_MINOR(dmc->version));
 }
 
 static int intel_dmc_debugfs_status_show(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *i915 = m->private;
+	struct intel_dmc *dmc = i915_to_dmc(i915);
 	intel_wakeref_t wakeref;
-	struct intel_dmc *dmc;
 	i915_reg_t dc5_reg, dc6_reg = INVALID_MMIO_REG;
 
 	if (!HAS_DMC(i915))
 		return -ENODEV;
 
-	dmc = &i915->display.dmc;
-
 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 
+	seq_printf(m, "DMC initialized: %s\n", str_yes_no(dmc));
 	seq_printf(m, "fw loaded: %s\n",
 		   str_yes_no(intel_dmc_has_payload(i915)));
-	seq_printf(m, "path: %s\n", dmc->fw_path);
+	seq_printf(m, "path: %s\n", dmc ? dmc->fw_path : "N/A");
 	seq_printf(m, "Pipe A fw needed: %s\n",
 		   str_yes_no(GRAPHICS_VER(i915) >= 12));
 	seq_printf(m, "Pipe A fw loaded: %s\n",
-		   str_yes_no(dmc->dmc_info[DMC_FW_PIPEA].payload));
+		   str_yes_no(has_dmc_id_fw(i915, DMC_FW_PIPEA)));
 	seq_printf(m, "Pipe B fw needed: %s\n",
 		   str_yes_no(IS_ALDERLAKE_P(i915) ||
 			      DISPLAY_VER(i915) >= 14));
 	seq_printf(m, "Pipe B fw loaded: %s\n",
-		   str_yes_no(dmc->dmc_info[DMC_FW_PIPEB].payload));
+		   str_yes_no(has_dmc_id_fw(i915, DMC_FW_PIPEB)));
 
 	if (!intel_dmc_has_payload(i915))
 		goto out;
@@ -1146,9 +1210,10 @@ static int intel_dmc_debugfs_status_show(struct seq_file *m, void *unused)
 		seq_printf(m, "DC5 -> DC6 count: %d\n",
 			   intel_de_read(i915, dc6_reg));
 
-out:
 	seq_printf(m, "program base: 0x%08x\n",
 		   intel_de_read(i915, DMC_PROGRAM(dmc->dmc_info[DMC_FW_MAIN].start_mmioaddr, 0)));
+
+out:
 	seq_printf(m, "ssp base: 0x%08x\n",
 		   intel_de_read(i915, DMC_SSP_BASE));
 	seq_printf(m, "htp: 0x%08x\n", intel_de_read(i915, DMC_HTP_SKL));
diff --git a/drivers/gpu/drm/i915/display/intel_dmc.h b/drivers/gpu/drm/i915/display/intel_dmc.h
index fd1725de4289..fd607afff2ef 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc.h
+++ b/drivers/gpu/drm/i915/display/intel_dmc.h
@@ -6,54 +6,20 @@
 #ifndef __INTEL_DMC_H__
 #define __INTEL_DMC_H__
 
-#include "i915_reg_defs.h"
-#include "intel_wakeref.h"
-#include <linux/workqueue.h>
+#include <linux/types.h>
 
 struct drm_i915_error_state_buf;
 struct drm_i915_private;
-
 enum pipe;
 
-enum {
-	DMC_FW_MAIN = 0,
-	DMC_FW_PIPEA,
-	DMC_FW_PIPEB,
-	DMC_FW_PIPEC,
-	DMC_FW_PIPED,
-	DMC_FW_MAX
-};
-
-struct intel_dmc {
-	struct work_struct work;
-	const char *fw_path;
-	u32 max_fw_size; /* bytes */
-	u32 version;
-	struct dmc_fw_info {
-		u32 mmio_count;
-		i915_reg_t mmioaddr[20];
-		u32 mmiodata[20];
-		u32 dmc_offset;
-		u32 start_mmioaddr;
-		u32 dmc_fw_size; /*dwords */
-		u32 *payload;
-		bool present;
-	} dmc_info[DMC_FW_MAX];
-
-	u32 dc_state;
-	u32 target_dc_state;
-	u32 allowed_dc_mask;
-	intel_wakeref_t wakeref;
-};
-
-void intel_dmc_ucode_init(struct drm_i915_private *i915);
+void intel_dmc_init(struct drm_i915_private *i915);
 void intel_dmc_load_program(struct drm_i915_private *i915);
 void intel_dmc_disable_program(struct drm_i915_private *i915);
 void intel_dmc_enable_pipe(struct drm_i915_private *i915, enum pipe pipe);
 void intel_dmc_disable_pipe(struct drm_i915_private *i915, enum pipe pipe);
-void intel_dmc_ucode_fini(struct drm_i915_private *i915);
-void intel_dmc_ucode_suspend(struct drm_i915_private *i915);
-void intel_dmc_ucode_resume(struct drm_i915_private *i915);
+void intel_dmc_fini(struct drm_i915_private *i915);
+void intel_dmc_suspend(struct drm_i915_private *i915);
+void intel_dmc_resume(struct drm_i915_private *i915);
 bool intel_dmc_has_payload(struct drm_i915_private *i915);
 void intel_dmc_debugfs_register(struct drm_i915_private *i915);
 void intel_dmc_print_error_state(struct drm_i915_error_state_buf *m,
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 62cbab7402e9..f0bace9d98a1 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -76,6 +76,7 @@
 #include "intel_tc.h"
 #include "intel_vdsc.h"
 #include "intel_vrr.h"
+#include "intel_crtc_state_dump.h"
 
 /* DP DSC throughput values used for slice count calculations KPixels/s */
 #define DP_DSC_PEAK_PIXEL_RATE			2720000
@@ -288,7 +289,7 @@ static int intel_dp_max_common_rate(struct intel_dp *intel_dp)
 
 static int intel_dp_max_source_lane_count(struct intel_digital_port *dig_port)
 {
-	int vbt_max_lanes = intel_bios_dp_max_lane_count(&dig_port->base);
+	int vbt_max_lanes = intel_bios_dp_max_lane_count(dig_port->base.devdata);
 	int max_lanes = dig_port->max_lanes;
 
 	if (vbt_max_lanes)
@@ -425,7 +426,7 @@ static int vbt_max_link_rate(struct intel_dp *intel_dp)
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	int max_rate;
 
-	max_rate = intel_bios_dp_max_link_rate(encoder);
+	max_rate = intel_bios_dp_max_link_rate(encoder->devdata);
 
 	if (intel_dp_is_edp(intel_dp)) {
 		struct intel_connector *connector = intel_dp->attached_connector;
@@ -687,6 +688,12 @@ u32 intel_dp_dsc_nearest_valid_bpp(struct drm_i915_private *i915, u32 bpp, u32 p
 	/* From XE_LPD onwards we support from bpc upto uncompressed bpp-1 BPPs */
 	if (DISPLAY_VER(i915) >= 13) {
 		bits_per_pixel = min(bits_per_pixel, pipe_bpp - 1);
+
+		/*
+		 * According to BSpec, 27 is the max DSC output bpp,
+		 * 8 is the min DSC output bpp
+		 */
+		bits_per_pixel = clamp_t(u32, bits_per_pixel, 8, 27);
 	} else {
 		/* Find the nearest match in the array of known BPPs from VESA */
 		for (i = 0; i < ARRAY_SIZE(valid_dsc_bpp) - 1; i++) {
@@ -716,9 +723,19 @@ u16 intel_dp_dsc_get_output_bpp(struct drm_i915_private *i915,
 	 * (LinkSymbolClock)* 8 * (TimeSlots / 64)
 	 * for SST -> TimeSlots is 64(i.e all TimeSlots that are available)
 	 * for MST -> TimeSlots has to be calculated, based on mode requirements
+	 *
+	 * Due to FEC overhead, the available bw is reduced to 97.2261%.
+	 * To support the given mode:
+	 * Bandwidth required should be <= Available link Bandwidth * FEC Overhead
+	 * =>ModeClock * bits_per_pixel <= Available Link Bandwidth * FEC Overhead
+	 * =>bits_per_pixel <= Available link Bandwidth * FEC Overhead / ModeClock
+	 * =>bits_per_pixel <= (NumberOfLanes * LinkSymbolClock) * 8 (TimeSlots / 64) /
+	 *		       (ModeClock / FEC Overhead)
+	 * =>bits_per_pixel <= (NumberOfLanes * LinkSymbolClock * TimeSlots) /
+	 *		       (ModeClock / FEC Overhead * 8)
 	 */
-	bits_per_pixel = DIV_ROUND_UP((link_clock * lane_count) * timeslots,
-				      intel_dp_mode_to_fec_clock(mode_clock) * 8);
+	bits_per_pixel = ((link_clock * lane_count) * timeslots) /
+			 (intel_dp_mode_to_fec_clock(mode_clock) * 8);
 
 	drm_dbg_kms(&i915->drm, "Max link bpp is %u for %u timeslots "
 				"total bw %u pixel clock %u\n",
@@ -771,6 +788,13 @@ u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp,
 		min_slice_count = DIV_ROUND_UP(mode_clock,
 					       DP_DSC_MAX_ENC_THROUGHPUT_1);
 
+	/*
+	 * Due to some DSC engine BW limitations, we need to enable second
+	 * slice and VDSC engine, whenever we approach close enough to max CDCLK
+	 */
+	if (mode_clock >= ((i915->display.cdclk.max_cdclk_freq * 85) / 100))
+		min_slice_count = max_t(u8, min_slice_count, 2);
+
 	max_slice_width = drm_dp_dsc_sink_max_slice_width(intel_dp->dsc_dpcd);
 	if (max_slice_width < DP_DSC_MIN_SLICE_WIDTH_VALUE) {
 		drm_dbg_kms(&i915->drm,
@@ -810,6 +834,9 @@ intel_dp_output_format(struct intel_connector *connector,
 {
 	struct intel_dp *intel_dp = intel_attached_dp(connector);
 
+	if (intel_dp->force_dsc_output_format)
+		return intel_dp->force_dsc_output_format;
+
 	if (!connector->base.ycbcr_420_allowed || !ycbcr_420_output)
 		return INTEL_OUTPUT_FORMAT_RGB;
 
@@ -1415,6 +1442,28 @@ static int intel_dp_sink_dsc_version_minor(struct intel_dp *intel_dp)
 		DP_DSC_MINOR_SHIFT;
 }
 
+static int intel_dp_get_slice_height(int vactive)
+{
+	int slice_height;
+
+	/*
+	 * VDSC 1.2a spec in Section 3.8 Options for Slices implies that 108
+	 * lines is an optimal slice height, but any size can be used as long as
+	 * vertical active integer multiple and maximum vertical slice count
+	 * requirements are met.
+	 */
+	for (slice_height = 108; slice_height <= vactive; slice_height += 2)
+		if (vactive % slice_height == 0)
+			return slice_height;
+
+	/*
+	 * Highly unlikely we reach here as most of the resolutions will end up
+	 * finding appropriate slice_height in above loop but returning
+	 * slice_height as 2 here as it should work with all resolutions.
+	 */
+	return 2;
+}
+
 static int intel_dp_dsc_compute_params(struct intel_encoder *encoder,
 				       struct intel_crtc_state *crtc_state)
 {
@@ -1433,17 +1482,7 @@ static int intel_dp_dsc_compute_params(struct intel_encoder *encoder,
 	vdsc_cfg->rc_model_size = DSC_RC_MODEL_SIZE_CONST;
 	vdsc_cfg->pic_height = crtc_state->hw.adjusted_mode.crtc_vdisplay;
 
-	/*
-	 * Slice Height of 8 works for all currently available panels. So start
-	 * with that if pic_height is an integral multiple of 8. Eventually add
-	 * logic to try multiple slice heights.
-	 */
-	if (vdsc_cfg->pic_height % 8 == 0)
-		vdsc_cfg->slice_height = 8;
-	else if (vdsc_cfg->pic_height % 4 == 0)
-		vdsc_cfg->slice_height = 4;
-	else
-		vdsc_cfg->slice_height = 2;
+	vdsc_cfg->slice_height = intel_dp_get_slice_height(vdsc_cfg->pic_height);
 
 	ret = intel_dsc_compute_params(crtc_state);
 	if (ret)
@@ -1455,9 +1494,10 @@ static int intel_dp_dsc_compute_params(struct intel_encoder *encoder,
 	vdsc_cfg->dsc_version_minor =
 		min(intel_dp_source_dsc_version_minor(intel_dp),
 		    intel_dp_sink_dsc_version_minor(intel_dp));
-
-	vdsc_cfg->convert_rgb = intel_dp->dsc_dpcd[DP_DSC_DEC_COLOR_FORMAT_CAP - DP_DSC_SUPPORT] &
-		DP_DSC_RGB;
+	if (vdsc_cfg->convert_rgb)
+		vdsc_cfg->convert_rgb =
+			intel_dp->dsc_dpcd[DP_DSC_DEC_COLOR_FORMAT_CAP - DP_DSC_SUPPORT] &
+			DP_DSC_RGB;
 
 	line_buf_depth = drm_dp_dsc_sink_line_buf_depth(intel_dp->dsc_dpcd);
 	if (!line_buf_depth) {
@@ -1480,6 +1520,31 @@ static int intel_dp_dsc_compute_params(struct intel_encoder *encoder,
 	return drm_dsc_compute_rc_parameters(vdsc_cfg);
 }
 
+static bool intel_dp_dsc_supports_format(struct intel_dp *intel_dp,
+					 enum intel_output_format output_format)
+{
+	u8 sink_dsc_format;
+
+	switch (output_format) {
+	case INTEL_OUTPUT_FORMAT_RGB:
+		sink_dsc_format = DP_DSC_RGB;
+		break;
+	case INTEL_OUTPUT_FORMAT_YCBCR444:
+		sink_dsc_format = DP_DSC_YCbCr444;
+		break;
+	case INTEL_OUTPUT_FORMAT_YCBCR420:
+		if (min(intel_dp_source_dsc_version_minor(intel_dp),
+			intel_dp_sink_dsc_version_minor(intel_dp)) < 2)
+			return false;
+		sink_dsc_format = DP_DSC_YCbCr420_Native;
+		break;
+	default:
+		return false;
+	}
+
+	return drm_dp_dsc_sink_supports_format(intel_dp->dsc_dpcd, sink_dsc_format);
+}
+
 int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 				struct intel_crtc_state *pipe_config,
 				struct drm_connector_state *conn_state,
@@ -1500,6 +1565,9 @@ int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 	if (!intel_dp_supports_dsc(intel_dp, pipe_config))
 		return -EINVAL;
 
+	if (!intel_dp_dsc_supports_format(intel_dp, pipe_config->output_format))
+		return -EINVAL;
+
 	if (compute_pipe_bpp)
 		pipe_bpp = intel_dp_dsc_compute_bpp(intel_dp, conn_state->max_requested_bpc);
 	else
@@ -1547,6 +1615,15 @@ int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 							    pipe_config->bigjoiner_pipes,
 							    pipe_bpp,
 							    timeslots);
+			/*
+			 * According to DSC 1.2a Section 4.1.1 Table 4.1 the maximum
+			 * supported PPS value can be 63.9375 and with the further
+			 * mention that bpp should be programmed double the target bpp
+			 * restricting our target bpp to be 31.9375 at max
+			 */
+			if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
+				dsc_max_output_bpp = min_t(u16, dsc_max_output_bpp, 31 << 4);
+
 			if (!dsc_max_output_bpp) {
 				drm_dbg_kms(&dev_priv->drm,
 					    "Compressed BPP not supported\n");
@@ -1585,16 +1662,8 @@ int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 	 * is greater than the maximum Cdclock and if slice count is even
 	 * then we need to use 2 VDSC instances.
 	 */
-	if (adjusted_mode->crtc_clock > dev_priv->display.cdclk.max_cdclk_freq ||
-	    pipe_config->bigjoiner_pipes) {
-		if (pipe_config->dsc.slice_count > 1) {
-			pipe_config->dsc.dsc_split = true;
-		} else {
-			drm_dbg_kms(&dev_priv->drm,
-				    "Cannot split stream to use 2 VDSC instances\n");
-			return -EINVAL;
-		}
-	}
+	if (pipe_config->bigjoiner_pipes || pipe_config->dsc.slice_count > 1)
+		pipe_config->dsc.dsc_split = true;
 
 	ret = intel_dp_dsc_compute_params(&dig_port->base, pipe_config);
 	if (ret < 0) {
@@ -1727,7 +1796,7 @@ bool intel_dp_limited_color_range(const struct intel_crtc_state *crtc_state,
 	 * Our YCbCr output is always limited range.
 	 * crtc_state->limited_color_range only applies to RGB,
 	 * and it must never be set for YCbCr or we risk setting
-	 * some conflicting bits in PIPECONF which will mess up
+	 * some conflicting bits in TRANSCONF which will mess up
 	 * the colors on the monitor.
 	 */
 	if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
@@ -1991,7 +2060,6 @@ intel_dp_drrs_compute_config(struct intel_connector *connector,
 }
 
 static bool intel_dp_has_audio(struct intel_encoder *encoder,
-			       const struct intel_crtc_state *crtc_state,
 			       const struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
@@ -2057,7 +2125,7 @@ intel_dp_audio_compute_config(struct intel_encoder *encoder,
 	struct drm_connector *connector = conn_state->connector;
 
 	pipe_config->sdp_split_enable =
-		intel_dp_has_audio(encoder, pipe_config, conn_state) &&
+		intel_dp_has_audio(encoder, conn_state) &&
 		intel_dp_is_uhbr(pipe_config);
 
 	drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] SDP split enable: %s\n",
@@ -2081,7 +2149,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 		pipe_config->has_pch_encoder = true;
 
 	pipe_config->has_audio =
-		intel_dp_has_audio(encoder, pipe_config, conn_state) &&
+		intel_dp_has_audio(encoder, conn_state) &&
 		intel_audio_compute_config(encoder, pipe_config, conn_state);
 
 	fixed_mode = intel_panel_fixed_mode(connector, adjusted_mode);
@@ -2281,10 +2349,15 @@ intel_edp_init_source_oui(struct intel_dp *intel_dp, bool careful)
 
 void intel_dp_wait_source_oui(struct intel_dp *intel_dp)
 {
+	struct intel_connector *connector = intel_dp->attached_connector;
 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 
-	drm_dbg_kms(&i915->drm, "Performing OUI wait\n");
-	wait_remaining_ms_from_jiffies(intel_dp->last_oui_write, 30);
+	drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] Performing OUI wait (%u ms)\n",
+		    connector->base.base.id, connector->base.name,
+		    connector->panel.vbt.backlight.hdr_dpcd_refresh_timeout);
+
+	wait_remaining_ms_from_jiffies(intel_dp->last_oui_write,
+				       connector->panel.vbt.backlight.hdr_dpcd_refresh_timeout);
 }
 
 /* If the device supports it, try to set the power state appropriately */
@@ -4851,7 +4924,7 @@ intel_dp_connector_register(struct drm_connector *connector)
 	if (!ret)
 		drm_dp_cec_register_connector(&intel_dp->aux, connector);
 
-	if (!intel_bios_is_lspcon_present(i915, dig_port->base.port))
+	if (!intel_bios_encoder_is_lspcon(dig_port->base.devdata))
 		return ret;
 
 	/*
@@ -5129,8 +5202,9 @@ intel_dp_hpd_pulse(struct intel_digital_port *dig_port, bool long_hpd)
 	return IRQ_HANDLED;
 }
 
-/* check the VBT to see whether the eDP is on another port */
-bool intel_dp_is_port_edp(struct drm_i915_private *dev_priv, enum port port)
+static bool _intel_dp_is_port_edp(struct drm_i915_private *dev_priv,
+				  const struct intel_bios_encoder_data *devdata,
+				  enum port port)
 {
 	/*
 	 * eDP not supported on g4x. so bail out early just
@@ -5142,13 +5216,24 @@ bool intel_dp_is_port_edp(struct drm_i915_private *dev_priv, enum port port)
 	if (DISPLAY_VER(dev_priv) < 9 && port == PORT_A)
 		return true;
 
-	return intel_bios_is_port_edp(dev_priv, port);
+	return devdata && intel_bios_encoder_supports_edp(devdata);
+}
+
+bool intel_dp_is_port_edp(struct drm_i915_private *i915, enum port port)
+{
+	const struct intel_bios_encoder_data *devdata =
+		intel_bios_encoder_data_lookup(i915, port);
+
+	return _intel_dp_is_port_edp(i915, devdata, port);
 }
 
 static bool
-has_gamut_metadata_dip(struct drm_i915_private *i915, enum port port)
+has_gamut_metadata_dip(struct intel_encoder *encoder)
 {
-	if (intel_bios_is_lspcon_present(i915, port))
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
+
+	if (intel_bios_encoder_is_lspcon(encoder->devdata))
 		return false;
 
 	if (DISPLAY_VER(i915) >= 11)
@@ -5183,14 +5268,14 @@ intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connect
 		drm_connector_attach_max_bpc_property(connector, 6, 12);
 
 	/* Register HDMI colorspace for case of lspcon */
-	if (intel_bios_is_lspcon_present(dev_priv, port)) {
+	if (intel_bios_encoder_is_lspcon(dp_to_dig_port(intel_dp)->base.devdata)) {
 		drm_connector_attach_content_type_property(connector);
 		intel_attach_hdmi_colorspace_property(connector);
 	} else {
 		intel_attach_dp_colorspace_property(connector);
 	}
 
-	if (has_gamut_metadata_dip(dev_priv, port))
+	if (has_gamut_metadata_dip(&dp_to_dig_port(intel_dp)->base))
 		drm_connector_attach_hdr_output_metadata_property(connector);
 
 	if (HAS_VRR(dev_priv))
@@ -5232,11 +5317,6 @@ static void intel_edp_backlight_setup(struct intel_dp *intel_dp,
 
 		if (pipe != PIPE_A && pipe != PIPE_B)
 			pipe = PIPE_A;
-
-		drm_dbg_kms(&i915->drm,
-			    "[CONNECTOR:%d:%s] using pipe %c for initial backlight setup\n",
-			    connector->base.base.id, connector->base.name,
-			    pipe_name(pipe));
 	}
 
 	intel_backlight_setup(connector, pipe);
@@ -5412,7 +5492,7 @@ intel_dp_init_connector(struct intel_digital_port *dig_port,
 	intel_dp->DP = intel_de_read(dev_priv, intel_dp->output_reg);
 	intel_dp->attached_connector = intel_connector;
 
-	if (intel_dp_is_port_edp(dev_priv, port)) {
+	if (_intel_dp_is_port_edp(dev_priv, intel_encoder->devdata, port)) {
 		/*
 		 * Currently we don't support eDP on TypeC ports, although in
 		 * theory it could work on TypeC legacy ports.
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c
index 5a176bfb10a2..705915d50565 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c
@@ -6,9 +6,11 @@
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "i915_trace.h"
+#include "intel_bios.h"
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_dp_aux.h"
+#include "intel_dp_aux_regs.h"
 #include "intel_pps.h"
 #include "intel_tc.h"
 
@@ -117,6 +119,32 @@ static u32 skl_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 	return index ? 0 : 1;
 }
 
+static int intel_dp_aux_sync_len(void)
+{
+	int precharge = 16; /* 10-16 */
+	int preamble = 16;
+
+	return precharge + preamble;
+}
+
+static int intel_dp_aux_fw_sync_len(void)
+{
+	int precharge = 16; /* 10-16 */
+	int preamble = 8;
+
+	return precharge + preamble;
+}
+
+static int g4x_dp_aux_precharge_len(void)
+{
+	int precharge_min = 10;
+	int preamble = 16;
+
+	/* HW wants the length of the extra precharge in 2us units */
+	return (intel_dp_aux_sync_len() -
+		precharge_min - preamble) / 2;
+}
+
 static u32 g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
 				int send_bytes,
 				u32 aux_clock_divider)
@@ -139,7 +167,7 @@ static u32 g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
 	       timeout |
 	       DP_AUX_CH_CTL_RECEIVE_ERROR |
 	       (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) |
-	       (3 << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) |
+	       (g4x_dp_aux_precharge_len() << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) |
 	       (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT);
 }
 
@@ -163,8 +191,8 @@ static u32 skl_get_aux_send_ctl(struct intel_dp *intel_dp,
 	      DP_AUX_CH_CTL_TIME_OUT_MAX |
 	      DP_AUX_CH_CTL_RECEIVE_ERROR |
 	      (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) |
-	      DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(32) |
-	      DP_AUX_CH_CTL_SYNC_PULSE_SKL(32);
+	      DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(intel_dp_aux_fw_sync_len()) |
+	      DP_AUX_CH_CTL_SYNC_PULSE_SKL(intel_dp_aux_sync_len());
 
 	if (intel_tc_port_in_tbt_alt_mode(dig_port))
 		ret |= DP_AUX_CH_CTL_TBT_IO;
@@ -204,8 +232,19 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp,
 	for (i = 0; i < ARRAY_SIZE(ch_data); i++)
 		ch_data[i] = intel_dp->aux_ch_data_reg(intel_dp, i);
 
-	if (is_tc_port)
+	if (is_tc_port) {
 		intel_tc_port_lock(dig_port);
+		/*
+		 * Abort transfers on a disconnected port as required by
+		 * DP 1.4a link CTS 4.2.1.5, also avoiding the long AUX
+		 * timeouts that would otherwise happen.
+		 * TODO: abort the transfer on non-TC ports as well.
+		 */
+		if (!intel_tc_port_connected_locked(&dig_port->base)) {
+			ret = -ENXIO;
+			goto out_unlock;
+		}
+	}
 
 	aux_domain = intel_aux_power_domain(dig_port);
 
@@ -366,7 +405,7 @@ out:
 
 	intel_pps_unlock(intel_dp, pps_wakeref);
 	intel_display_power_put_async(i915, aux_domain, aux_wakeref);
-
+out_unlock:
 	if (is_tc_port)
 		intel_tc_port_unlock(dig_port);
 
@@ -737,3 +776,37 @@ void intel_dp_aux_init(struct intel_dp *intel_dp)
 	intel_dp->aux.transfer = intel_dp_aux_transfer;
 	cpu_latency_qos_add_request(&intel_dp->pm_qos, PM_QOS_DEFAULT_VALUE);
 }
+
+static enum aux_ch default_aux_ch(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+
+	/* SKL has DDI E but no AUX E */
+	if (DISPLAY_VER(i915) == 9 && encoder->port == PORT_E)
+		return AUX_CH_A;
+
+	return (enum aux_ch)encoder->port;
+}
+
+enum aux_ch intel_dp_aux_ch(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum aux_ch aux_ch;
+
+	aux_ch = intel_bios_dp_aux_ch(encoder->devdata);
+	if (aux_ch != AUX_CH_NONE) {
+		drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s] using AUX %c (VBT)\n",
+			    encoder->base.base.id, encoder->base.name,
+			    aux_ch_name(aux_ch));
+		return aux_ch;
+	}
+
+	aux_ch = default_aux_ch(encoder);
+
+	drm_dbg_kms(&i915->drm,
+		    "[ENCODER:%d:%s] using AUX %c (platform default)\n",
+		    encoder->base.base.id, encoder->base.name,
+		    aux_ch_name(aux_ch));
+
+	return aux_ch;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.h b/drivers/gpu/drm/i915/display/intel_dp_aux.h
index 738577537bc7..138e340f94ee 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux.h
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux.h
@@ -6,9 +6,13 @@
 #ifndef __INTEL_DP_AUX_H__
 #define __INTEL_DP_AUX_H__
 
+enum aux_ch;
 struct intel_dp;
+struct intel_encoder;
 
 void intel_dp_aux_fini(struct intel_dp *intel_dp);
 void intel_dp_aux_init(struct intel_dp *intel_dp);
 
+enum aux_ch intel_dp_aux_ch(struct intel_encoder *encoder);
+
 #endif /* __INTEL_DP_AUX_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
index 83af95bce98d..95cc5251843e 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
@@ -105,6 +105,11 @@ enum intel_dp_aux_backlight_modparam {
 	INTEL_DP_AUX_BACKLIGHT_FORCE_INTEL = 3,
 };
 
+static bool is_intel_tcon_cap(const u8 tcon_cap[4])
+{
+	return tcon_cap[0] >= 1;
+}
+
 /* Intel EDP backlight callbacks */
 static bool
 intel_dp_aux_supports_hdr_backlight(struct intel_connector *connector)
@@ -125,14 +130,12 @@ intel_dp_aux_supports_hdr_backlight(struct intel_connector *connector)
 	if (!(tcon_cap[1] & INTEL_EDP_HDR_TCON_BRIGHTNESS_NITS_CAP))
 		return false;
 
-	if (tcon_cap[0] >= 1) {
-		drm_dbg_kms(&i915->drm, "Detected Intel HDR backlight interface version %d\n",
-			    tcon_cap[0]);
-	} else {
-		drm_dbg_kms(&i915->drm, "Detected unsupported HDR backlight interface version %d\n",
-			    tcon_cap[0]);
+	drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] Detected %s HDR backlight interface version %d\n",
+		    connector->base.base.id, connector->base.name,
+		    is_intel_tcon_cap(tcon_cap) ? "Intel" : "unsupported", tcon_cap[0]);
+
+	if (!is_intel_tcon_cap(tcon_cap))
 		return false;
-	}
 
 	/*
 	 * If we don't have HDR static metadata there is no way to
@@ -147,7 +150,8 @@ intel_dp_aux_supports_hdr_backlight(struct intel_connector *connector)
 	    !(connector->base.hdr_sink_metadata.hdmi_type1.metadata_type &
 	      BIT(HDMI_STATIC_METADATA_TYPE1))) {
 		drm_info(&i915->drm,
-			 "Panel is missing HDR static metadata. Possible support for Intel HDR backlight interface is not used. If your backlight controls don't work try booting with i915.enable_dpcd_backlight=%d. needs this, please file a _new_ bug report on drm/i915, see " FDO_BUG_URL " for details.\n",
+			 "[CONNECTOR:%d:%s] Panel is missing HDR static metadata. Possible support for Intel HDR backlight interface is not used. If your backlight controls don't work try booting with i915.enable_dpcd_backlight=%d. needs this, please file a _new_ bug report on drm/i915, see " FDO_BUG_URL " for details.\n",
+			 connector->base.base.id, connector->base.name,
 			 INTEL_DP_AUX_BACKLIGHT_FORCE_INTEL);
 		return false;
 	}
@@ -168,7 +172,8 @@ intel_dp_aux_hdr_get_backlight(struct intel_connector *connector, enum pipe pipe
 	u8 buf[2] = { 0 };
 
 	if (drm_dp_dpcd_readb(&intel_dp->aux, INTEL_EDP_HDR_GETSET_CTRL_PARAMS, &tmp) != 1) {
-		drm_err(&i915->drm, "Failed to read current backlight mode from DPCD\n");
+		drm_err(&i915->drm, "[CONNECTOR:%d:%s] Failed to read current backlight mode from DPCD\n",
+			connector->base.base.id, connector->base.name);
 		return 0;
 	}
 
@@ -185,7 +190,8 @@ intel_dp_aux_hdr_get_backlight(struct intel_connector *connector, enum pipe pipe
 
 	if (drm_dp_dpcd_read(&intel_dp->aux, INTEL_EDP_BRIGHTNESS_NITS_LSB, buf,
 			     sizeof(buf)) != sizeof(buf)) {
-		drm_err(&i915->drm, "Failed to read brightness from DPCD\n");
+		drm_err(&i915->drm, "[CONNECTOR:%d:%s] Failed to read brightness from DPCD\n",
+			connector->base.base.id, connector->base.name);
 		return 0;
 	}
 
@@ -205,7 +211,8 @@ intel_dp_aux_hdr_set_aux_backlight(const struct drm_connector_state *conn_state,
 
 	if (drm_dp_dpcd_write(&intel_dp->aux, INTEL_EDP_BRIGHTNESS_NITS_LSB, buf,
 			      sizeof(buf)) != sizeof(buf))
-		drm_err(dev, "Failed to write brightness level to DPCD\n");
+		drm_err(dev, "[CONNECTOR:%d:%s] Failed to write brightness level to DPCD\n",
+			connector->base.base.id, connector->base.name);
 }
 
 static void
@@ -238,7 +245,8 @@ intel_dp_aux_hdr_enable_backlight(const struct intel_crtc_state *crtc_state,
 
 	ret = drm_dp_dpcd_readb(&intel_dp->aux, INTEL_EDP_HDR_GETSET_CTRL_PARAMS, &old_ctrl);
 	if (ret != 1) {
-		drm_err(&i915->drm, "Failed to read current backlight control mode: %d\n", ret);
+		drm_err(&i915->drm, "[CONNECTOR:%d:%s] Failed to read current backlight control mode: %d\n",
+			connector->base.base.id, connector->base.name, ret);
 		return;
 	}
 
@@ -254,9 +262,10 @@ intel_dp_aux_hdr_enable_backlight(const struct intel_crtc_state *crtc_state,
 		ctrl &= ~INTEL_EDP_HDR_TCON_BRIGHTNESS_AUX_ENABLE;
 	}
 
-	if (ctrl != old_ctrl)
-		if (drm_dp_dpcd_writeb(&intel_dp->aux, INTEL_EDP_HDR_GETSET_CTRL_PARAMS, ctrl) != 1)
-			drm_err(&i915->drm, "Failed to configure DPCD brightness controls\n");
+	if (ctrl != old_ctrl &&
+	    drm_dp_dpcd_writeb(&intel_dp->aux, INTEL_EDP_HDR_GETSET_CTRL_PARAMS, ctrl) != 1)
+		drm_err(&i915->drm, "[CONNECTOR:%d:%s] Failed to configure DPCD brightness controls\n",
+			connector->base.base.id, connector->base.name);
 }
 
 static void
@@ -273,6 +282,11 @@ intel_dp_aux_hdr_disable_backlight(const struct drm_connector_state *conn_state,
 	panel->backlight.pwm_funcs->disable(conn_state, intel_backlight_invert_pwm_level(connector, 0));
 }
 
+static const char *dpcd_vs_pwm_str(bool aux)
+{
+	return aux ? "DPCD" : "PWM";
+}
+
 static int
 intel_dp_aux_hdr_setup_backlight(struct intel_connector *connector, enum pipe pipe)
 {
@@ -282,15 +296,16 @@ intel_dp_aux_hdr_setup_backlight(struct intel_connector *connector, enum pipe pi
 		&connector->base.display_info.luminance_range;
 	int ret;
 
-	if (panel->backlight.edp.intel.sdr_uses_aux) {
-		drm_dbg_kms(&i915->drm, "SDR backlight is controlled through DPCD\n");
-	} else {
-		drm_dbg_kms(&i915->drm, "SDR backlight is controlled through PWM\n");
+	drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] SDR backlight is controlled through %s\n",
+		    connector->base.base.id, connector->base.name,
+		    dpcd_vs_pwm_str(panel->backlight.edp.intel.sdr_uses_aux));
 
+	if (!panel->backlight.edp.intel.sdr_uses_aux) {
 		ret = panel->backlight.pwm_funcs->setup(connector, pipe);
 		if (ret < 0) {
 			drm_err(&i915->drm,
-				"Failed to setup SDR backlight controls through PWM: %d\n", ret);
+				"[CONNECTOR:%d:%s] Failed to setup SDR backlight controls through PWM: %d\n",
+				connector->base.base.id, connector->base.name, ret);
 			return ret;
 		}
 	}
@@ -303,8 +318,10 @@ intel_dp_aux_hdr_setup_backlight(struct intel_connector *connector, enum pipe pi
 		panel->backlight.min = 0;
 	}
 
-	drm_dbg_kms(&i915->drm, "Using backlight range %d..%d\n", panel->backlight.min,
-		    panel->backlight.max);
+	drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] Using AUX HDR interface for backlight control (range %d..%d)\n",
+		    connector->base.base.id, connector->base.name,
+		    panel->backlight.min, panel->backlight.max);
+
 
 	panel->backlight.level = intel_dp_aux_hdr_get_backlight(connector, pipe);
 	panel->backlight.enabled = panel->backlight.level != 0;
@@ -386,12 +403,19 @@ static int intel_dp_aux_vesa_setup_backlight(struct intel_connector *connector,
 	if (ret < 0)
 		return ret;
 
+	drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] AUX VESA backlight enable is controlled through %s\n",
+		    connector->base.base.id, connector->base.name,
+		    dpcd_vs_pwm_str(panel->backlight.edp.vesa.info.aux_enable));
+	drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] AUX VESA backlight level is controlled through %s\n",
+		    connector->base.base.id, connector->base.name,
+		    dpcd_vs_pwm_str(panel->backlight.edp.vesa.info.aux_set));
+
 	if (!panel->backlight.edp.vesa.info.aux_set || !panel->backlight.edp.vesa.info.aux_enable) {
 		ret = panel->backlight.pwm_funcs->setup(connector, pipe);
 		if (ret < 0) {
 			drm_err(&i915->drm,
-				"Failed to setup PWM backlight controls for eDP backlight: %d\n",
-				ret);
+				"[CONNECTOR:%d:%s] Failed to setup PWM backlight controls for eDP backlight: %d\n",
+				connector->base.base.id, connector->base.name, ret);
 			return ret;
 		}
 	}
@@ -418,6 +442,9 @@ static int intel_dp_aux_vesa_setup_backlight(struct intel_connector *connector,
 		}
 	}
 
+	drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] Using AUX VESA interface for backlight control\n",
+		    connector->base.base.id, connector->base.name);
+
 	return 0;
 }
 
@@ -428,7 +455,8 @@ intel_dp_aux_supports_vesa_backlight(struct intel_connector *connector)
 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 
 	if (drm_edp_backlight_supported(intel_dp->edp_dpcd)) {
-		drm_dbg_kms(&i915->drm, "AUX Backlight Control Supported!\n");
+		drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] AUX Backlight Control Supported!\n",
+			    connector->base.base.id, connector->base.name);
 		return true;
 	}
 	return false;
@@ -504,13 +532,15 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 	 * interfaces is to probe for Intel's first, and VESA's second.
 	 */
 	if (try_intel_interface && intel_dp_aux_supports_hdr_backlight(connector)) {
-		drm_dbg_kms(dev, "Using Intel proprietary eDP backlight controls\n");
+		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Using Intel proprietary eDP backlight controls\n",
+			    connector->base.base.id, connector->base.name);
 		panel->backlight.funcs = &intel_dp_hdr_bl_funcs;
 		return 0;
 	}
 
 	if (try_vesa_interface && intel_dp_aux_supports_vesa_backlight(connector)) {
-		drm_dbg_kms(dev, "Using VESA eDP backlight controls\n");
+		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Using VESA eDP backlight controls\n",
+			    connector->base.base.id, connector->base.name);
 		panel->backlight.funcs = &intel_dp_vesa_bl_funcs;
 		return 0;
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_regs.h b/drivers/gpu/drm/i915/display/intel_dp_aux_regs.h
new file mode 100644
index 000000000000..5702f318d537
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux_regs.h
@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_DP_AUX_REGS_H__
+#define __INTEL_DP_AUX_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+/*
+ * The aux channel provides a way to talk to the signal sink for DDC etc. Max
+ * packet size supported is 20 bytes in each direction, hence the 5 fixed data
+ * registers
+ */
+#define _DPA_AUX_CH_CTL		(DISPLAY_MMIO_BASE(dev_priv) + 0x64010)
+#define _DPA_AUX_CH_DATA1	(DISPLAY_MMIO_BASE(dev_priv) + 0x64014)
+
+#define _DPB_AUX_CH_CTL		(DISPLAY_MMIO_BASE(dev_priv) + 0x64110)
+#define _DPB_AUX_CH_DATA1	(DISPLAY_MMIO_BASE(dev_priv) + 0x64114)
+
+#define DP_AUX_CH_CTL(aux_ch)	_MMIO_PORT(aux_ch, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL)
+#define DP_AUX_CH_DATA(aux_ch, i)	_MMIO(_PORT(aux_ch, _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */
+
+#define _XELPDP_USBC1_AUX_CH_CTL	0x16F210
+#define _XELPDP_USBC2_AUX_CH_CTL	0x16F410
+#define _XELPDP_USBC3_AUX_CH_CTL	0x16F610
+#define _XELPDP_USBC4_AUX_CH_CTL	0x16F810
+
+#define XELPDP_DP_AUX_CH_CTL(aux_ch)		_MMIO(_PICK(aux_ch, \
+						       _DPA_AUX_CH_CTL, \
+						       _DPB_AUX_CH_CTL, \
+						       0, /* port/aux_ch C is non-existent */ \
+						       _XELPDP_USBC1_AUX_CH_CTL, \
+						       _XELPDP_USBC2_AUX_CH_CTL, \
+						       _XELPDP_USBC3_AUX_CH_CTL, \
+						       _XELPDP_USBC4_AUX_CH_CTL))
+
+#define _XELPDP_USBC1_AUX_CH_DATA1      0x16F214
+#define _XELPDP_USBC2_AUX_CH_DATA1      0x16F414
+#define _XELPDP_USBC3_AUX_CH_DATA1      0x16F614
+#define _XELPDP_USBC4_AUX_CH_DATA1      0x16F814
+
+#define XELPDP_DP_AUX_CH_DATA(aux_ch, i)	_MMIO(_PICK(aux_ch, \
+						       _DPA_AUX_CH_DATA1, \
+						       _DPB_AUX_CH_DATA1, \
+						       0, /* port/aux_ch C is non-existent */ \
+						       _XELPDP_USBC1_AUX_CH_DATA1, \
+						       _XELPDP_USBC2_AUX_CH_DATA1, \
+						       _XELPDP_USBC3_AUX_CH_DATA1, \
+						       _XELPDP_USBC4_AUX_CH_DATA1) + (i) * 4)
+
+#define   DP_AUX_CH_CTL_SEND_BUSY	    (1 << 31)
+#define   DP_AUX_CH_CTL_DONE		    (1 << 30)
+#define   DP_AUX_CH_CTL_INTERRUPT	    (1 << 29)
+#define   DP_AUX_CH_CTL_TIME_OUT_ERROR	    (1 << 28)
+#define   DP_AUX_CH_CTL_TIME_OUT_400us	    (0 << 26)
+#define   DP_AUX_CH_CTL_TIME_OUT_600us	    (1 << 26)
+#define   DP_AUX_CH_CTL_TIME_OUT_800us	    (2 << 26)
+#define   DP_AUX_CH_CTL_TIME_OUT_MAX	    (3 << 26) /* Varies per platform */
+#define   DP_AUX_CH_CTL_TIME_OUT_MASK	    (3 << 26)
+#define   DP_AUX_CH_CTL_RECEIVE_ERROR	    (1 << 25)
+#define   DP_AUX_CH_CTL_MESSAGE_SIZE_MASK    (0x1f << 20)
+#define   DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT   20
+#define   XELPDP_DP_AUX_CH_CTL_POWER_REQUEST REG_BIT(19)
+#define   XELPDP_DP_AUX_CH_CTL_POWER_STATUS  REG_BIT(18)
+#define   DP_AUX_CH_CTL_PRECHARGE_2US_MASK   (0xf << 16)
+#define   DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT  16
+#define   DP_AUX_CH_CTL_AUX_AKSV_SELECT	    (1 << 15)
+#define   DP_AUX_CH_CTL_MANCHESTER_TEST	    (1 << 14)
+#define   DP_AUX_CH_CTL_SYNC_TEST	    (1 << 13)
+#define   DP_AUX_CH_CTL_DEGLITCH_TEST	    (1 << 12)
+#define   DP_AUX_CH_CTL_PRECHARGE_TEST	    (1 << 11)
+#define   DP_AUX_CH_CTL_BIT_CLOCK_2X_MASK    (0x7ff)
+#define   DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT   0
+#define   DP_AUX_CH_CTL_PSR_DATA_AUX_REG_SKL	(1 << 14)
+#define   DP_AUX_CH_CTL_FS_DATA_AUX_REG_SKL	(1 << 13)
+#define   DP_AUX_CH_CTL_GTC_DATA_AUX_REG_SKL	(1 << 12)
+#define   DP_AUX_CH_CTL_TBT_IO			(1 << 11)
+#define   DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL_MASK (0x1f << 5)
+#define   DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(c) (((c) - 1) << 5)
+#define   DP_AUX_CH_CTL_SYNC_PULSE_SKL(c)   ((c) - 1)
+
+#endif /* __INTEL_DP_AUX_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
index 3d3efcf02011..d638054c74ac 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
@@ -1379,10 +1379,6 @@ intel_dp_128b132b_lane_cds(struct intel_dp *intel_dp,
 		}
 	}
 
-	/* FIXME: Should DP_TRAINING_PATTERN_DISABLE be written first? */
-	if (intel_dp->set_idle_link_train)
-		intel_dp->set_idle_link_train(intel_dp, crtc_state);
-
 	return true;
 }
 
@@ -1433,7 +1429,11 @@ intel_dp_128b132b_link_train(struct intel_dp *intel_dp,
 void intel_dp_start_link_train(struct intel_dp *intel_dp,
 			       const struct intel_crtc_state *crtc_state)
 {
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
+	struct intel_connector *connector = intel_dp->attached_connector;
+	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	bool passed;
+
 	/*
 	 * TODO: Reiniting LTTPRs here won't be needed once proper connector
 	 * HW state readout is added.
@@ -1451,6 +1451,46 @@ void intel_dp_start_link_train(struct intel_dp *intel_dp,
 	else
 		passed = intel_dp_link_train_all_phys(intel_dp, crtc_state, lttpr_count);
 
+	/*
+	 * Ignore the link failure in CI
+	 *
+	 * In fixed enviroments like CI, sometimes unexpected long HPDs are
+	 * generated by the displays. If ignore_long_hpd flag is set, such long
+	 * HPDs are ignored. And probably as a consequence of these ignored
+	 * long HPDs, subsequent link trainings are failed resulting into CI
+	 * execution failures.
+	 *
+	 * For test cases which rely on the link training or processing of HPDs
+	 * ignore_long_hpd flag can unset from the testcase.
+	 */
+	if (!passed && i915->display.hotplug.ignore_long_hpd) {
+		drm_dbg_kms(&i915->drm,
+			    "[CONNECTOR:%d:%s][ENCODER:%d:%s] Ignore the link failure\n",
+			    connector->base.base.id, connector->base.name,
+			    encoder->base.base.id, encoder->base.name);
+		return;
+	}
+
 	if (!passed)
 		intel_dp_schedule_fallback_link_training(intel_dp, crtc_state);
 }
+
+void intel_dp_128b132b_sdp_crc16(struct intel_dp *intel_dp,
+				 const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
+
+	/*
+	 * VIDEO_DIP_CTL register bit 31 should be set to '0' to not
+	 * disable SDP CRC. This is applicable for Display version 13.
+	 * Default value of bit 31 is '0' hence discarding the write
+	 * TODO: Corrective actions on SDP corruption yet to be defined
+	 */
+	if (intel_dp_is_uhbr(crtc_state))
+		/* DP v2.0 SCR on SDP CRC16 for 128b/132b Link Layer */
+		drm_dp_dpcd_writeb(&intel_dp->aux,
+				   DP_SDP_ERROR_DETECTION_CONFIGURATION,
+				   DP_SDP_CRC16_128B132B_EN);
+
+	drm_dbg_kms(&i915->drm, "DP2.0 SDP CRC16 for 128b/132b enabled\n");
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.h b/drivers/gpu/drm/i915/display/intel_dp_link_training.h
index 7fa1c0833096..2c8f2775891b 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_link_training.h
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.h
@@ -39,4 +39,6 @@ static inline u8 intel_dp_training_pattern_symbol(u8 pattern)
 	return pattern & ~DP_LINK_SCRAMBLING_DISABLE;
 }
 
+void intel_dp_128b132b_sdp_crc16(struct intel_dp *intel_dp,
+				 const struct intel_crtc_state *crtc_state);
 #endif /* __INTEL_DP_LINK_TRAINING_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c
index 7c9b328bc2d7..2c49d9ab86a2 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c
@@ -45,6 +45,27 @@
 #include "intel_hotplug.h"
 #include "skl_scaler.h"
 
+static int intel_dp_mst_check_constraints(struct drm_i915_private *i915, int bpp,
+					  const struct drm_display_mode *adjusted_mode,
+					  struct intel_crtc_state *crtc_state,
+					  bool dsc)
+{
+	if (intel_dp_is_uhbr(crtc_state) && DISPLAY_VER(i915) <= 13 && dsc) {
+		int output_bpp = bpp;
+		/* DisplayPort 2 128b/132b, bits per lane is always 32 */
+		int symbol_clock = crtc_state->port_clock / 32;
+
+		if (output_bpp * adjusted_mode->crtc_clock >=
+		    symbol_clock * 72) {
+			drm_dbg_kms(&i915->drm, "UHBR check failed(required bw %d available %d)\n",
+				    output_bpp * adjusted_mode->crtc_clock, symbol_clock * 72);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
 static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder,
 						struct intel_crtc_state *crtc_state,
 						int max_bpp,
@@ -81,12 +102,16 @@ static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder,
 	}
 
 	for (bpp = max_bpp; bpp >= min_bpp; bpp -= step) {
+		drm_dbg_kms(&i915->drm, "Trying bpp %d\n", bpp);
+
+		ret = intel_dp_mst_check_constraints(i915, bpp, adjusted_mode, crtc_state, dsc);
+		if (ret)
+			continue;
+
 		crtc_state->pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock,
 						       dsc ? bpp << 4 : bpp,
 						       dsc);
 
-		drm_dbg_kms(&i915->drm, "Trying bpp %d\n", bpp);
-
 		slots = drm_dp_atomic_find_time_slots(state, &intel_dp->mst_mgr,
 						      connector->port,
 						      crtc_state->pbn);
@@ -104,8 +129,8 @@ static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder,
 		}
 	}
 
-	/* Despite slots are non-zero, we still failed the atomic check */
-	if (ret && slots >= 0)
+	/* We failed to find a proper bpp/timeslots, return error */
+	if (ret)
 		slots = ret;
 
 	if (slots < 0) {
@@ -611,7 +636,7 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 	 * no clock to the transcoder"
 	 */
 	if (DISPLAY_VER(dev_priv) < 12 || !last_mst_stream)
-		intel_ddi_disable_pipe_clock(old_crtc_state);
+		intel_ddi_disable_transcoder_clock(old_crtc_state);
 
 
 	intel_mst->connector = NULL;
@@ -623,6 +648,20 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 		    intel_dp->active_mst_links);
 }
 
+static void intel_mst_post_pll_disable_dp(struct intel_atomic_state *state,
+					  struct intel_encoder *encoder,
+					  const struct intel_crtc_state *old_crtc_state,
+					  const struct drm_connector_state *old_conn_state)
+{
+	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder);
+	struct intel_digital_port *dig_port = intel_mst->primary;
+	struct intel_dp *intel_dp = &dig_port->dp;
+
+	if (intel_dp->active_mst_links == 0 &&
+	    dig_port->base.post_pll_disable)
+		dig_port->base.post_pll_disable(state, encoder, old_crtc_state, old_conn_state);
+}
+
 static void intel_mst_pre_pll_enable_dp(struct intel_atomic_state *state,
 					struct intel_encoder *encoder,
 					const struct intel_crtc_state *pipe_config,
@@ -635,6 +674,13 @@ static void intel_mst_pre_pll_enable_dp(struct intel_atomic_state *state,
 	if (intel_dp->active_mst_links == 0)
 		dig_port->base.pre_pll_enable(state, &dig_port->base,
 						    pipe_config, NULL);
+	else
+		/*
+		 * The port PLL state needs to get updated for secondary
+		 * streams as for the primary stream.
+		 */
+		intel_ddi_update_active_dpll(state, &dig_port->base,
+					     to_intel_crtc(pipe_config->uapi.crtc));
 }
 
 static void intel_mst_pre_enable_dp(struct intel_atomic_state *state,
@@ -691,7 +737,7 @@ static void intel_mst_pre_enable_dp(struct intel_atomic_state *state,
 	 * here for the following ones.
 	 */
 	if (DISPLAY_VER(dev_priv) < 12 || !first_mst_stream)
-		intel_ddi_enable_pipe_clock(encoder, pipe_config);
+		intel_ddi_enable_transcoder_clock(encoder, pipe_config);
 
 	intel_ddi_set_dp_msa(pipe_config, conn_state);
 }
@@ -1146,6 +1192,7 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *dig_port, enum pipe
 	intel_encoder->compute_config_late = intel_dp_mst_compute_config_late;
 	intel_encoder->disable = intel_mst_disable_dp;
 	intel_encoder->post_disable = intel_mst_post_disable_dp;
+	intel_encoder->post_pll_disable = intel_mst_post_pll_disable_dp;
 	intel_encoder->update_pipe = intel_ddi_update_pipe;
 	intel_encoder->pre_pll_enable = intel_mst_pre_pll_enable_dp;
 	intel_encoder->pre_enable = intel_mst_pre_enable_dp;
diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.c b/drivers/gpu/drm/i915/display/intel_dpio_phy.c
index 565c06de2432..62b93d097e44 100644
--- a/drivers/gpu/drm/i915/display/intel_dpio_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.c
@@ -389,9 +389,7 @@ static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv,
 			"force reprogramming it\n", phy);
 	}
 
-	val = intel_de_read(dev_priv, BXT_P_CR_GT_DISP_PWRON);
-	val |= phy_info->pwron_mask;
-	intel_de_write(dev_priv, BXT_P_CR_GT_DISP_PWRON, val);
+	intel_de_rmw(dev_priv, BXT_P_CR_GT_DISP_PWRON, 0, phy_info->pwron_mask);
 
 	/*
 	 * The PHY registers start out inaccessible and respond to reads with
@@ -410,27 +408,19 @@ static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv,
 			phy);
 
 	/* Program PLL Rcomp code offset */
-	val = intel_de_read(dev_priv, BXT_PORT_CL1CM_DW9(phy));
-	val &= ~IREF0RC_OFFSET_MASK;
-	val |= 0xE4 << IREF0RC_OFFSET_SHIFT;
-	intel_de_write(dev_priv, BXT_PORT_CL1CM_DW9(phy), val);
+	intel_de_rmw(dev_priv, BXT_PORT_CL1CM_DW9(phy), IREF0RC_OFFSET_MASK,
+		     0xE4 << IREF0RC_OFFSET_SHIFT);
 
-	val = intel_de_read(dev_priv, BXT_PORT_CL1CM_DW10(phy));
-	val &= ~IREF1RC_OFFSET_MASK;
-	val |= 0xE4 << IREF1RC_OFFSET_SHIFT;
-	intel_de_write(dev_priv, BXT_PORT_CL1CM_DW10(phy), val);
+	intel_de_rmw(dev_priv, BXT_PORT_CL1CM_DW10(phy), IREF1RC_OFFSET_MASK,
+		     0xE4 << IREF1RC_OFFSET_SHIFT);
 
 	/* Program power gating */
-	val = intel_de_read(dev_priv, BXT_PORT_CL1CM_DW28(phy));
-	val |= OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN |
-		SUS_CLK_CONFIG;
-	intel_de_write(dev_priv, BXT_PORT_CL1CM_DW28(phy), val);
-
-	if (phy_info->dual_channel) {
-		val = intel_de_read(dev_priv, BXT_PORT_CL2CM_DW6(phy));
-		val |= DW6_OLDO_DYN_PWR_DOWN_EN;
-		intel_de_write(dev_priv, BXT_PORT_CL2CM_DW6(phy), val);
-	}
+	intel_de_rmw(dev_priv, BXT_PORT_CL1CM_DW28(phy), 0,
+		     OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | SUS_CLK_CONFIG);
+
+	if (phy_info->dual_channel)
+		intel_de_rmw(dev_priv, BXT_PORT_CL2CM_DW6(phy), 0,
+			     DW6_OLDO_DYN_PWR_DOWN_EN);
 
 	if (phy_info->rcomp_phy != -1) {
 		u32 grc_code;
@@ -449,34 +439,25 @@ static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv,
 			   val << GRC_CODE_SLOW_SHIFT |
 			   val;
 		intel_de_write(dev_priv, BXT_PORT_REF_DW6(phy), grc_code);
-
-		val = intel_de_read(dev_priv, BXT_PORT_REF_DW8(phy));
-		val |= GRC_DIS | GRC_RDY_OVRD;
-		intel_de_write(dev_priv, BXT_PORT_REF_DW8(phy), val);
+		intel_de_rmw(dev_priv, BXT_PORT_REF_DW8(phy),
+			     0, GRC_DIS | GRC_RDY_OVRD);
 	}
 
 	if (phy_info->reset_delay)
 		udelay(phy_info->reset_delay);
 
-	val = intel_de_read(dev_priv, BXT_PHY_CTL_FAMILY(phy));
-	val |= COMMON_RESET_DIS;
-	intel_de_write(dev_priv, BXT_PHY_CTL_FAMILY(phy), val);
+	intel_de_rmw(dev_priv, BXT_PHY_CTL_FAMILY(phy), 0, COMMON_RESET_DIS);
 }
 
 void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 {
 	const struct bxt_ddi_phy_info *phy_info;
-	u32 val;
 
 	phy_info = bxt_get_phy_info(dev_priv, phy);
 
-	val = intel_de_read(dev_priv, BXT_PHY_CTL_FAMILY(phy));
-	val &= ~COMMON_RESET_DIS;
-	intel_de_write(dev_priv, BXT_PHY_CTL_FAMILY(phy), val);
+	intel_de_rmw(dev_priv, BXT_PHY_CTL_FAMILY(phy), COMMON_RESET_DIS, 0);
 
-	val = intel_de_read(dev_priv, BXT_P_CR_GT_DISP_PWRON);
-	val &= ~phy_info->pwron_mask;
-	intel_de_write(dev_priv, BXT_P_CR_GT_DISP_PWRON, val);
+	intel_de_rmw(dev_priv, BXT_P_CR_GT_DISP_PWRON, phy_info->pwron_mask, 0);
 }
 
 void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
index 380368eff31a..22fc908b7e5d 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
@@ -608,10 +608,8 @@ static void hsw_ddi_wrpll_disable(struct drm_i915_private *dev_priv,
 				  struct intel_shared_dpll *pll)
 {
 	const enum intel_dpll_id id = pll->info->id;
-	u32 val;
 
-	val = intel_de_read(dev_priv, WRPLL_CTL(id));
-	intel_de_write(dev_priv, WRPLL_CTL(id), val & ~WRPLL_PLL_ENABLE);
+	intel_de_rmw(dev_priv, WRPLL_CTL(id), WRPLL_PLL_ENABLE, 0);
 	intel_de_posting_read(dev_priv, WRPLL_CTL(id));
 
 	/*
@@ -626,10 +624,8 @@ static void hsw_ddi_spll_disable(struct drm_i915_private *dev_priv,
 				 struct intel_shared_dpll *pll)
 {
 	enum intel_dpll_id id = pll->info->id;
-	u32 val;
 
-	val = intel_de_read(dev_priv, SPLL_CTL);
-	intel_de_write(dev_priv, SPLL_CTL, val & ~SPLL_PLL_ENABLE);
+	intel_de_rmw(dev_priv, SPLL_CTL, SPLL_PLL_ENABLE, 0);
 	intel_de_posting_read(dev_priv, SPLL_CTL);
 
 	/*
@@ -1238,16 +1234,10 @@ static void skl_ddi_pll_write_ctrl1(struct drm_i915_private *dev_priv,
 				    struct intel_shared_dpll *pll)
 {
 	const enum intel_dpll_id id = pll->info->id;
-	u32 val;
 
-	val = intel_de_read(dev_priv, DPLL_CTRL1);
-
-	val &= ~(DPLL_CTRL1_HDMI_MODE(id) |
-		 DPLL_CTRL1_SSC(id) |
-		 DPLL_CTRL1_LINK_RATE_MASK(id));
-	val |= pll->state.hw_state.ctrl1 << (id * 6);
-
-	intel_de_write(dev_priv, DPLL_CTRL1, val);
+	intel_de_rmw(dev_priv, DPLL_CTRL1,
+		     DPLL_CTRL1_HDMI_MODE(id) | DPLL_CTRL1_SSC(id) | DPLL_CTRL1_LINK_RATE_MASK(id),
+		     pll->state.hw_state.ctrl1 << (id * 6));
 	intel_de_posting_read(dev_priv, DPLL_CTRL1);
 }
 
@@ -1265,8 +1255,7 @@ static void skl_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	intel_de_posting_read(dev_priv, regs[id].cfgcr2);
 
 	/* the enable bit is always bit 31 */
-	intel_de_write(dev_priv, regs[id].ctl,
-		       intel_de_read(dev_priv, regs[id].ctl) | LCPLL_PLL_ENABLE);
+	intel_de_rmw(dev_priv, regs[id].ctl, 0, LCPLL_PLL_ENABLE);
 
 	if (intel_de_wait_for_set(dev_priv, DPLL_STATUS, DPLL_LOCK(id), 5))
 		drm_err(&dev_priv->drm, "DPLL %d not locked\n", id);
@@ -1285,8 +1274,7 @@ static void skl_ddi_pll_disable(struct drm_i915_private *dev_priv,
 	const enum intel_dpll_id id = pll->info->id;
 
 	/* the enable bit is always bit 31 */
-	intel_de_write(dev_priv, regs[id].ctl,
-		       intel_de_read(dev_priv, regs[id].ctl) & ~LCPLL_PLL_ENABLE);
+	intel_de_rmw(dev_priv, regs[id].ctl, LCPLL_PLL_ENABLE, 0);
 	intel_de_posting_read(dev_priv, regs[id].ctl);
 }
 
@@ -1902,14 +1890,11 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	bxt_port_to_phy_channel(dev_priv, port, &phy, &ch);
 
 	/* Non-SSC reference */
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL_ENABLE(port));
-	temp |= PORT_PLL_REF_SEL;
-	intel_de_write(dev_priv, BXT_PORT_PLL_ENABLE(port), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL_ENABLE(port), 0, PORT_PLL_REF_SEL);
 
 	if (IS_GEMINILAKE(dev_priv)) {
-		temp = intel_de_read(dev_priv, BXT_PORT_PLL_ENABLE(port));
-		temp |= PORT_PLL_POWER_ENABLE;
-		intel_de_write(dev_priv, BXT_PORT_PLL_ENABLE(port), temp);
+		intel_de_rmw(dev_priv, BXT_PORT_PLL_ENABLE(port),
+			     0, PORT_PLL_POWER_ENABLE);
 
 		if (wait_for_us((intel_de_read(dev_priv, BXT_PORT_PLL_ENABLE(port)) &
 				 PORT_PLL_POWER_STATE), 200))
@@ -1918,39 +1903,28 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	}
 
 	/* Disable 10 bit clock */
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL_EBB_4(phy, ch));
-	temp &= ~PORT_PLL_10BIT_CLK_ENABLE;
-	intel_de_write(dev_priv, BXT_PORT_PLL_EBB_4(phy, ch), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL_EBB_4(phy, ch),
+		     PORT_PLL_10BIT_CLK_ENABLE, 0);
 
 	/* Write P1 & P2 */
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL_EBB_0(phy, ch));
-	temp &= ~(PORT_PLL_P1_MASK | PORT_PLL_P2_MASK);
-	temp |= pll->state.hw_state.ebb0;
-	intel_de_write(dev_priv, BXT_PORT_PLL_EBB_0(phy, ch), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL_EBB_0(phy, ch),
+		     PORT_PLL_P1_MASK | PORT_PLL_P2_MASK, pll->state.hw_state.ebb0);
 
 	/* Write M2 integer */
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL(phy, ch, 0));
-	temp &= ~PORT_PLL_M2_INT_MASK;
-	temp |= pll->state.hw_state.pll0;
-	intel_de_write(dev_priv, BXT_PORT_PLL(phy, ch, 0), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL(phy, ch, 0),
+		     PORT_PLL_M2_INT_MASK, pll->state.hw_state.pll0);
 
 	/* Write N */
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL(phy, ch, 1));
-	temp &= ~PORT_PLL_N_MASK;
-	temp |= pll->state.hw_state.pll1;
-	intel_de_write(dev_priv, BXT_PORT_PLL(phy, ch, 1), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL(phy, ch, 1),
+		     PORT_PLL_N_MASK, pll->state.hw_state.pll1);
 
 	/* Write M2 fraction */
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL(phy, ch, 2));
-	temp &= ~PORT_PLL_M2_FRAC_MASK;
-	temp |= pll->state.hw_state.pll2;
-	intel_de_write(dev_priv, BXT_PORT_PLL(phy, ch, 2), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL(phy, ch, 2),
+		     PORT_PLL_M2_FRAC_MASK, pll->state.hw_state.pll2);
 
 	/* Write M2 fraction enable */
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL(phy, ch, 3));
-	temp &= ~PORT_PLL_M2_FRAC_ENABLE;
-	temp |= pll->state.hw_state.pll3;
-	intel_de_write(dev_priv, BXT_PORT_PLL(phy, ch, 3), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL(phy, ch, 3),
+		     PORT_PLL_M2_FRAC_ENABLE, pll->state.hw_state.pll3);
 
 	/* Write coeff */
 	temp = intel_de_read(dev_priv, BXT_PORT_PLL(phy, ch, 6));
@@ -1961,15 +1935,11 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	intel_de_write(dev_priv, BXT_PORT_PLL(phy, ch, 6), temp);
 
 	/* Write calibration val */
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL(phy, ch, 8));
-	temp &= ~PORT_PLL_TARGET_CNT_MASK;
-	temp |= pll->state.hw_state.pll8;
-	intel_de_write(dev_priv, BXT_PORT_PLL(phy, ch, 8), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL(phy, ch, 8),
+		     PORT_PLL_TARGET_CNT_MASK, pll->state.hw_state.pll8);
 
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL(phy, ch, 9));
-	temp &= ~PORT_PLL_LOCK_THRESHOLD_MASK;
-	temp |= pll->state.hw_state.pll9;
-	intel_de_write(dev_priv, BXT_PORT_PLL(phy, ch, 9), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL(phy, ch, 9),
+		     PORT_PLL_LOCK_THRESHOLD_MASK, pll->state.hw_state.pll9);
 
 	temp = intel_de_read(dev_priv, BXT_PORT_PLL(phy, ch, 10));
 	temp &= ~PORT_PLL_DCO_AMP_OVR_EN_H;
@@ -1986,9 +1956,7 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	intel_de_write(dev_priv, BXT_PORT_PLL_EBB_4(phy, ch), temp);
 
 	/* Enable PLL */
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL_ENABLE(port));
-	temp |= PORT_PLL_ENABLE;
-	intel_de_write(dev_priv, BXT_PORT_PLL_ENABLE(port), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL_ENABLE(port), 0, PORT_PLL_ENABLE);
 	intel_de_posting_read(dev_priv, BXT_PORT_PLL_ENABLE(port));
 
 	if (wait_for_us((intel_de_read(dev_priv, BXT_PORT_PLL_ENABLE(port)) & PORT_PLL_LOCK),
@@ -2016,17 +1984,13 @@ static void bxt_ddi_pll_disable(struct drm_i915_private *dev_priv,
 					struct intel_shared_dpll *pll)
 {
 	enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */
-	u32 temp;
 
-	temp = intel_de_read(dev_priv, BXT_PORT_PLL_ENABLE(port));
-	temp &= ~PORT_PLL_ENABLE;
-	intel_de_write(dev_priv, BXT_PORT_PLL_ENABLE(port), temp);
+	intel_de_rmw(dev_priv, BXT_PORT_PLL_ENABLE(port), PORT_PLL_ENABLE, 0);
 	intel_de_posting_read(dev_priv, BXT_PORT_PLL_ENABLE(port));
 
 	if (IS_GEMINILAKE(dev_priv)) {
-		temp = intel_de_read(dev_priv, BXT_PORT_PLL_ENABLE(port));
-		temp &= ~PORT_PLL_POWER_ENABLE;
-		intel_de_write(dev_priv, BXT_PORT_PLL_ENABLE(port), temp);
+		intel_de_rmw(dev_priv, BXT_PORT_PLL_ENABLE(port),
+			     PORT_PLL_POWER_ENABLE, 0);
 
 		if (wait_for_us(!(intel_de_read(dev_priv, BXT_PORT_PLL_ENABLE(port)) &
 				  PORT_PLL_POWER_STATE), 200))
@@ -3641,8 +3605,8 @@ static void icl_dpll_write(struct drm_i915_private *dev_priv,
 			 !i915_mmio_reg_valid(div0_reg));
 	if (dev_priv->display.vbt.override_afc_startup &&
 	    i915_mmio_reg_valid(div0_reg))
-		intel_de_rmw(dev_priv, div0_reg, TGL_DPLL0_DIV0_AFC_STARTUP_MASK,
-			     hw_state->div0);
+		intel_de_rmw(dev_priv, div0_reg,
+			     TGL_DPLL0_DIV0_AFC_STARTUP_MASK, hw_state->div0);
 	intel_de_posting_read(dev_priv, cfgcr1_reg);
 }
 
@@ -3651,7 +3615,6 @@ static void icl_mg_pll_write(struct drm_i915_private *dev_priv,
 {
 	struct intel_dpll_hw_state *hw_state = &pll->state.hw_state;
 	enum tc_port tc_port = icl_pll_id_to_tc_port(pll->info->id);
-	u32 val;
 
 	/*
 	 * Some of the following registers have reserved fields, so program
@@ -3659,23 +3622,19 @@ static void icl_mg_pll_write(struct drm_i915_private *dev_priv,
 	 * during the calc/readout phase if the mask depends on some other HW
 	 * state like refclk, see icl_calc_mg_pll_state().
 	 */
-	val = intel_de_read(dev_priv, MG_REFCLKIN_CTL(tc_port));
-	val &= ~MG_REFCLKIN_CTL_OD_2_MUX_MASK;
-	val |= hw_state->mg_refclkin_ctl;
-	intel_de_write(dev_priv, MG_REFCLKIN_CTL(tc_port), val);
+	intel_de_rmw(dev_priv, MG_REFCLKIN_CTL(tc_port),
+		     MG_REFCLKIN_CTL_OD_2_MUX_MASK, hw_state->mg_refclkin_ctl);
 
-	val = intel_de_read(dev_priv, MG_CLKTOP2_CORECLKCTL1(tc_port));
-	val &= ~MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK;
-	val |= hw_state->mg_clktop2_coreclkctl1;
-	intel_de_write(dev_priv, MG_CLKTOP2_CORECLKCTL1(tc_port), val);
+	intel_de_rmw(dev_priv, MG_CLKTOP2_CORECLKCTL1(tc_port),
+		     MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK,
+		     hw_state->mg_clktop2_coreclkctl1);
 
-	val = intel_de_read(dev_priv, MG_CLKTOP2_HSCLKCTL(tc_port));
-	val &= ~(MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK |
-		 MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK |
-		 MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK |
-		 MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK);
-	val |= hw_state->mg_clktop2_hsclkctl;
-	intel_de_write(dev_priv, MG_CLKTOP2_HSCLKCTL(tc_port), val);
+	intel_de_rmw(dev_priv, MG_CLKTOP2_HSCLKCTL(tc_port),
+		     MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK |
+		     MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK |
+		     MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK |
+		     MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK,
+		     hw_state->mg_clktop2_hsclkctl);
 
 	intel_de_write(dev_priv, MG_PLL_DIV0(tc_port), hw_state->mg_pll_div0);
 	intel_de_write(dev_priv, MG_PLL_DIV1(tc_port), hw_state->mg_pll_div1);
@@ -3684,15 +3643,12 @@ static void icl_mg_pll_write(struct drm_i915_private *dev_priv,
 		       hw_state->mg_pll_frac_lock);
 	intel_de_write(dev_priv, MG_PLL_SSC(tc_port), hw_state->mg_pll_ssc);
 
-	val = intel_de_read(dev_priv, MG_PLL_BIAS(tc_port));
-	val &= ~hw_state->mg_pll_bias_mask;
-	val |= hw_state->mg_pll_bias;
-	intel_de_write(dev_priv, MG_PLL_BIAS(tc_port), val);
+	intel_de_rmw(dev_priv, MG_PLL_BIAS(tc_port),
+		     hw_state->mg_pll_bias_mask, hw_state->mg_pll_bias);
 
-	val = intel_de_read(dev_priv, MG_PLL_TDC_COLDST_BIAS(tc_port));
-	val &= ~hw_state->mg_pll_tdc_coldst_bias_mask;
-	val |= hw_state->mg_pll_tdc_coldst_bias;
-	intel_de_write(dev_priv, MG_PLL_TDC_COLDST_BIAS(tc_port), val);
+	intel_de_rmw(dev_priv, MG_PLL_TDC_COLDST_BIAS(tc_port),
+		     hw_state->mg_pll_tdc_coldst_bias_mask,
+		     hw_state->mg_pll_tdc_coldst_bias);
 
 	intel_de_posting_read(dev_priv, MG_PLL_TDC_COLDST_BIAS(tc_port));
 }
@@ -3766,11 +3722,7 @@ static void icl_pll_power_enable(struct drm_i915_private *dev_priv,
 				 struct intel_shared_dpll *pll,
 				 i915_reg_t enable_reg)
 {
-	u32 val;
-
-	val = intel_de_read(dev_priv, enable_reg);
-	val |= PLL_POWER_ENABLE;
-	intel_de_write(dev_priv, enable_reg, val);
+	intel_de_rmw(dev_priv, enable_reg, 0, PLL_POWER_ENABLE);
 
 	/*
 	 * The spec says we need to "wait" but it also says it should be
@@ -3785,11 +3737,7 @@ static void icl_pll_enable(struct drm_i915_private *dev_priv,
 			   struct intel_shared_dpll *pll,
 			   i915_reg_t enable_reg)
 {
-	u32 val;
-
-	val = intel_de_read(dev_priv, enable_reg);
-	val |= PLL_ENABLE;
-	intel_de_write(dev_priv, enable_reg, val);
+	intel_de_rmw(dev_priv, enable_reg, 0, PLL_ENABLE);
 
 	/* Timeout is actually 600us. */
 	if (intel_de_wait_for_set(dev_priv, enable_reg, PLL_LOCK, 1))
@@ -3815,8 +3763,7 @@ static void adlp_cmtg_clock_gating_wa(struct drm_i915_private *i915, struct inte
 	 * since TRANS_CMTG_CHICKEN is only accessible while DPLL0 is enabled.
 	 */
 	val = intel_de_read(i915, TRANS_CMTG_CHICKEN);
-	val = intel_de_read(i915, TRANS_CMTG_CHICKEN);
-	intel_de_write(i915, TRANS_CMTG_CHICKEN, DISABLE_DPT_CLK_GATING);
+	val = intel_de_rmw(i915, TRANS_CMTG_CHICKEN, ~0, DISABLE_DPT_CLK_GATING);
 	if (drm_WARN_ON(&i915->drm, val & ~DISABLE_DPT_CLK_GATING))
 		drm_dbg_kms(&i915->drm, "Unexpected flags in TRANS_CMTG_CHICKEN: %08x\n", val);
 }
@@ -3900,8 +3847,6 @@ static void icl_pll_disable(struct drm_i915_private *dev_priv,
 			    struct intel_shared_dpll *pll,
 			    i915_reg_t enable_reg)
 {
-	u32 val;
-
 	/* The first steps are done by intel_ddi_post_disable(). */
 
 	/*
@@ -3910,9 +3855,7 @@ static void icl_pll_disable(struct drm_i915_private *dev_priv,
 	 * nothing here.
 	 */
 
-	val = intel_de_read(dev_priv, enable_reg);
-	val &= ~PLL_ENABLE;
-	intel_de_write(dev_priv, enable_reg, val);
+	intel_de_rmw(dev_priv, enable_reg, PLL_ENABLE, 0);
 
 	/* Timeout is actually 1us. */
 	if (intel_de_wait_for_clear(dev_priv, enable_reg, PLL_LOCK, 1))
@@ -3920,9 +3863,7 @@ static void icl_pll_disable(struct drm_i915_private *dev_priv,
 
 	/* DVFS post sequence would be here. See the comment above. */
 
-	val = intel_de_read(dev_priv, enable_reg);
-	val &= ~PLL_POWER_ENABLE;
-	intel_de_write(dev_priv, enable_reg, val);
+	intel_de_rmw(dev_priv, enable_reg, PLL_POWER_ENABLE, 0);
 
 	/*
 	 * The spec says we need to "wait" but it also says it should be
diff --git a/drivers/gpu/drm/i915/display/intel_dpt.c b/drivers/gpu/drm/i915/display/intel_dpt.c
index 2a9f40a2b3ed..b8027392144d 100644
--- a/drivers/gpu/drm/i915/display/intel_dpt.c
+++ b/drivers/gpu/drm/i915/display/intel_dpt.c
@@ -9,6 +9,8 @@
 #include "gt/gen8_ppgtt.h"
 
 #include "i915_drv.h"
+#include "i915_reg.h"
+#include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_dpt.h"
 #include "intel_fb.h"
@@ -313,3 +315,26 @@ void intel_dpt_destroy(struct i915_address_space *vm)
 	dpt->obj->is_dpt = false;
 	i915_vm_put(&dpt->vm);
 }
+
+void intel_dpt_configure(struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+
+	if (DISPLAY_VER(i915) == 14) {
+		enum pipe pipe = crtc->pipe;
+		enum plane_id plane_id;
+
+		for_each_plane_id_on_crtc(crtc, plane_id) {
+			if (plane_id == PLANE_CURSOR)
+				continue;
+
+			intel_de_rmw(i915, PLANE_CHICKEN(pipe, plane_id),
+				     PLANE_CHICKEN_DISABLE_DPT,
+				     i915->params.enable_dpt ? 0 : PLANE_CHICKEN_DISABLE_DPT);
+		}
+	} else if (DISPLAY_VER(i915) == 13) {
+		intel_de_rmw(i915, CHICKEN_MISC_2,
+			     CHICKEN_MISC_DISABLE_DPT,
+			     i915->params.enable_dpt ? 0 : CHICKEN_MISC_DISABLE_DPT);
+	}
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dpt.h b/drivers/gpu/drm/i915/display/intel_dpt.h
index e18a9f767b11..d9a166550185 100644
--- a/drivers/gpu/drm/i915/display/intel_dpt.h
+++ b/drivers/gpu/drm/i915/display/intel_dpt.h
@@ -10,6 +10,7 @@ struct drm_i915_private;
 
 struct i915_address_space;
 struct i915_vma;
+struct intel_crtc;
 struct intel_framebuffer;
 
 void intel_dpt_destroy(struct i915_address_space *vm);
@@ -19,5 +20,6 @@ void intel_dpt_suspend(struct drm_i915_private *i915);
 void intel_dpt_resume(struct drm_i915_private *i915);
 struct i915_address_space *
 intel_dpt_create(struct intel_framebuffer *fb);
+void intel_dpt_configure(struct intel_crtc *crtc);
 
 #endif /* __INTEL_DPT_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_drrs.c b/drivers/gpu/drm/i915/display/intel_drrs.c
index 29c6421cd666..760e63cdc0c8 100644
--- a/drivers/gpu/drm/i915/display/intel_drrs.c
+++ b/drivers/gpu/drm/i915/display/intel_drrs.c
@@ -68,21 +68,15 @@ intel_drrs_set_refresh_rate_pipeconf(struct intel_crtc *crtc,
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum transcoder cpu_transcoder = crtc->drrs.cpu_transcoder;
-	u32 val, bit;
+	u32 bit;
 
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		bit = PIPECONF_REFRESH_RATE_ALT_VLV;
+		bit = TRANSCONF_REFRESH_RATE_ALT_VLV;
 	else
-		bit = PIPECONF_REFRESH_RATE_ALT_ILK;
+		bit = TRANSCONF_REFRESH_RATE_ALT_ILK;
 
-	val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
-
-	if (refresh_rate == DRRS_REFRESH_RATE_LOW)
-		val |= bit;
-	else
-		val &= ~bit;
-
-	intel_de_write(dev_priv, PIPECONF(cpu_transcoder), val);
+	intel_de_rmw(dev_priv, TRANSCONF(cpu_transcoder),
+		     bit, refresh_rate == DRRS_REFRESH_RATE_LOW ? bit : 0);
 }
 
 static void
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c
index 96bc117fd6a0..bed058d2c3ac 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb.c
@@ -11,6 +11,7 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_dsb.h"
+#include "intel_dsb_regs.h"
 
 struct i915_vma;
 
@@ -88,7 +89,8 @@ static bool assert_dsb_has_room(struct intel_dsb *dsb)
 
 	/* each instruction is 2 dwords */
 	return !drm_WARN(&i915->drm, dsb->free_pos > dsb->size - 2,
-			 "DSB buffer overflow\n");
+			 "[CRTC:%d:%s] DSB %d buffer overflow\n",
+			 crtc->base.base.id, crtc->base.name, dsb->id);
 }
 
 static bool is_dsb_busy(struct drm_i915_private *i915, enum pipe pipe,
@@ -198,7 +200,7 @@ void intel_dsb_reg_write(struct intel_dsb *dsb,
 	}
 }
 
-static u32 intel_dsb_align_tail(struct intel_dsb *dsb)
+static void intel_dsb_align_tail(struct intel_dsb *dsb)
 {
 	u32 aligned_tail, tail;
 
@@ -210,49 +212,58 @@ static u32 intel_dsb_align_tail(struct intel_dsb *dsb)
 		       aligned_tail - tail);
 
 	dsb->free_pos = aligned_tail / 4;
+}
 
-	return aligned_tail;
+void intel_dsb_finish(struct intel_dsb *dsb)
+{
+	intel_dsb_align_tail(dsb);
 }
 
 /**
  * intel_dsb_commit() - Trigger workload execution of DSB.
  * @dsb: DSB context
+ * @wait_for_vblank: wait for vblank before executing
  *
  * This function is used to do actual write to hardware using DSB.
  */
-void intel_dsb_commit(struct intel_dsb *dsb)
+void intel_dsb_commit(struct intel_dsb *dsb, bool wait_for_vblank)
 {
 	struct intel_crtc *crtc = dsb->crtc;
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 	u32 tail;
 
-	tail = intel_dsb_align_tail(dsb);
-	if (tail == 0)
+	tail = dsb->free_pos * 4;
+	if (drm_WARN_ON(&dev_priv->drm, !IS_ALIGNED(tail, CACHELINE_BYTES)))
 		return;
 
 	if (is_dsb_busy(dev_priv, pipe, dsb->id)) {
-		drm_err(&dev_priv->drm, "DSB engine is busy.\n");
-		goto reset;
+		drm_err(&dev_priv->drm, "[CRTC:%d:%s] DSB %d is busy\n",
+			crtc->base.base.id, crtc->base.name, dsb->id);
+		return;
 	}
 
 	intel_de_write(dev_priv, DSB_CTRL(pipe, dsb->id),
+		       (wait_for_vblank ? DSB_WAIT_FOR_VBLANK : 0) |
 		       DSB_ENABLE);
 	intel_de_write(dev_priv, DSB_HEAD(pipe, dsb->id),
 		       i915_ggtt_offset(dsb->vma));
 	intel_de_write(dev_priv, DSB_TAIL(pipe, dsb->id),
 		       i915_ggtt_offset(dsb->vma) + tail);
+}
 
-	drm_dbg_kms(&dev_priv->drm,
-		    "DSB execution started - head 0x%x, tail 0x%x\n",
-		    i915_ggtt_offset(dsb->vma),
-		    i915_ggtt_offset(dsb->vma) + tail);
+void intel_dsb_wait(struct intel_dsb *dsb)
+{
+	struct intel_crtc *crtc = dsb->crtc;
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum pipe pipe = crtc->pipe;
 
 	if (wait_for(!is_dsb_busy(dev_priv, pipe, dsb->id), 1))
 		drm_err(&dev_priv->drm,
-			"Timed out waiting for DSB workload completion.\n");
+			"[CRTC:%d:%s] DSB %d timed out waiting for idle\n",
+			crtc->base.base.id, crtc->base.name, dsb->id);
 
-reset:
+	/* Attempt to reset it */
 	dsb->free_pos = 0;
 	dsb->ins_start_offset = 0;
 	intel_de_write(dev_priv, DSB_CTRL(pipe, dsb->id), 0);
@@ -325,7 +336,8 @@ out_put_rpm:
 	kfree(dsb);
 out:
 	drm_info_once(&i915->drm,
-		      "DSB queue setup failed, will fallback to MMIO for display HW programming\n");
+		      "[CRTC:%d:%s] DSB %d queue setup failed, will fallback to MMIO for display HW programming\n",
+		      crtc->base.base.id, crtc->base.name, DSB1);
 
 	return NULL;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.h b/drivers/gpu/drm/i915/display/intel_dsb.h
index 05c221b6d0a4..b8148b47022d 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.h
+++ b/drivers/gpu/drm/i915/display/intel_dsb.h
@@ -15,9 +15,12 @@ struct intel_dsb;
 
 struct intel_dsb *intel_dsb_prepare(struct intel_crtc *crtc,
 				    unsigned int max_cmds);
+void intel_dsb_finish(struct intel_dsb *dsb);
 void intel_dsb_cleanup(struct intel_dsb *dsb);
 void intel_dsb_reg_write(struct intel_dsb *dsb,
 			 i915_reg_t reg, u32 val);
-void intel_dsb_commit(struct intel_dsb *dsb);
+void intel_dsb_commit(struct intel_dsb *dsb,
+		      bool wait_for_vblank);
+void intel_dsb_wait(struct intel_dsb *dsb);
 
 #endif
diff --git a/drivers/gpu/drm/i915/display/intel_dsb_regs.h b/drivers/gpu/drm/i915/display/intel_dsb_regs.h
new file mode 100644
index 000000000000..12535d478775
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dsb_regs.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_DSB_REGS_H__
+#define __INTEL_DSB_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+/* This register controls the Display State Buffer (DSB) engines. */
+#define _DSBSL_INSTANCE_BASE		0x70B00
+#define DSBSL_INSTANCE(pipe, id)	(_DSBSL_INSTANCE_BASE + \
+					 (pipe) * 0x1000 + (id) * 0x100)
+#define DSB_HEAD(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x0)
+#define DSB_TAIL(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x4)
+#define DSB_CTRL(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x8)
+#define   DSB_ENABLE			REG_BIT(31)
+#define   DSB_BUF_REITERATE		REG_BIT(29)
+#define   DSB_WAIT_FOR_VBLANK		REG_BIT(28)
+#define   DSB_WAIT_FOR_LINE_IN		REG_BIT(27)
+#define   DSB_HALT			REG_BIT(16)
+#define   DSB_NON_POSTED		REG_BIT(8)
+#define   DSB_STATUS_BUSY		REG_BIT(0)
+#define DSB_MMIOCTRL(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0xc)
+#define   DSB_MMIO_DEAD_CLOCKS_ENABLE	REG_BIT(31)
+#define   DSB_MMIO_DEAD_CLOCKS_COUNT_MASK	REG_GENMASK(15, 8)
+#define   DSB_MMIO_DEAD_CLOCKS_COUNT(x)	REG_FIELD_PREP(DSB_MMIO_DEAD_CLOCK_COUNT_MASK, (x))
+#define   DSB_MMIO_CYCLES_MASK		REG_GENMASK(7, 0)
+#define   DSB_MMIO_CYCLES(x)		REG_FIELD_PREP(DSB_MMIO_CYCLES_MASK, (x))
+#define DSB_POLLFUNC(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x10)
+#define   DSB_POLL_ENABLE		REG_BIT(31)
+#define   DSB_POLL_WAIT_MASK		REG_GENMASK(30, 23)
+#define   DSB_POLL_WAIT(x)		REG_FIELD_PREP(DSB_POLL_WAIT_MASK, (x)) /* usec */
+#define   DSB_POLL_COUNT_MASK		REG_GENMASK(22, 15)
+#define   DSB_POLL_COUNT(x)		REG_FIELD_PREP(DSB_POLL_COUNT_MASK, (x))
+#define DSB_DEBUG(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x14)
+#define DSB_POLLMASK(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x1c)
+#define DSB_STATUS(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x24)
+#define DSB_INTERRUPT(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x28)
+#define   DSB_ATS_FAULT_INT_EN		REG_BIT(20)
+#define   DSB_GTT_FAULT_INT_EN		REG_BIT(19)
+#define   DSB_RSPTIMEOUT_INT_EN		REG_BIT(18)
+#define   DSB_POLL_ERR_INT_EN		REG_BIT(17)
+#define   DSB_PROG_INT_EN		REG_BIT(16)
+#define   DSB_ATS_FAULT_INT_STATUS	REG_BIT(4)
+#define   DSB_GTT_FAULT_INT_STATUS	REG_BIT(3)
+#define   DSB_RSPTIMEOUT_INT_STATUS	REG_BIT(2)
+#define   DSB_POLL_ERR_INT_STATUS	REG_BIT(1)
+#define   DSB_PROG_INT_STATUS		REG_BIT(0)
+#define DSB_CURRENT_HEAD(pipe, id)	_MMIO(DSBSL_INSTANCE(pipe, id) + 0x2c)
+#define DSB_RM_TIMEOUT(pipe, id)	_MMIO(DSBSL_INSTANCE(pipe, id) + 0x30)
+#define   DSB_RM_CLAIM_TIMEOUT		REG_BIT(31)
+#define   DSB_RM_READY_TIMEOUT		REG_BIT(30)
+#define   DSB_RM_CLAIM_TIMEOUT_COUNT_MASK	REG_GENMASK(23, 16)
+#define   DSB_RM_CLAIM_TIMEOUT_COUNT(x)	REG_FIELD_PREP(DSB_RM_CLAIM_TIMEOUT_COUNT_MASK, (x)) /* clocks */
+#define   DSB_RM_READY_TIMEOUT_VALUE_MASK	REG_GENMASK(15, 0)
+#define   DSB_RM_READY_TIMEOUT_VALUE(x)	REG_FIELD_PREP(DSB_RM_READY_TIMEOUT_VALUE, (x)) /* usec */
+#define DSB_RMTIMEOUTREG_CAPTURE(pipe, id)	_MMIO(DSBSL_INSTANCE(pipe, id) + 0x34)
+#define DSB_PMCTRL(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x38)
+#define DSB_PMCTRL_2(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x3c)
+#define DSB_PF_LN_LOWER(pipe, id)	_MMIO(DSBSL_INSTANCE(pipe, id) + 0x40)
+#define DSB_PF_LN_UPPER(pipe, id)	_MMIO(DSBSL_INSTANCE(pipe, id) + 0x44)
+#define DSB_BUFRPT_CNT(pipe, id)	_MMIO(DSBSL_INSTANCE(pipe, id) + 0x48)
+#define DSB_CHICKEN(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0xf0)
+
+#endif /* __INTEL_DSB_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c b/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c
index 20e466d843ce..049443245310 100644
--- a/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c
@@ -162,6 +162,7 @@ static void dcs_enable_backlight(const struct intel_crtc_state *crtc_state,
 static int dcs_setup_backlight(struct intel_connector *connector,
 			       enum pipe unused)
 {
+	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 
 	if (panel->vbt.backlight.brightness_precision_bits > 8)
@@ -171,6 +172,10 @@ static int dcs_setup_backlight(struct intel_connector *connector,
 
 	panel->backlight.level = panel->backlight.max;
 
+	drm_dbg_kms(&i915->drm,
+		    "[CONNECTOR:%d:%s] Using DCS for backlight control\n",
+		    connector->base.base.id, connector->base.name);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_dsi_vbt.c b/drivers/gpu/drm/i915/display/intel_dsi_vbt.c
index 2cbc1292ab38..695b0d69a4cb 100644
--- a/drivers/gpu/drm/i915/display/intel_dsi_vbt.c
+++ b/drivers/gpu/drm/i915/display/intel_dsi_vbt.c
@@ -46,6 +46,7 @@
 #include "intel_dsi.h"
 #include "intel_dsi_vbt.h"
 #include "intel_gmbus_regs.h"
+#include "intel_pps_regs.h"
 #include "vlv_dsi.h"
 #include "vlv_dsi_regs.h"
 #include "vlv_sideband.h"
diff --git a/drivers/gpu/drm/i915/display/intel_dvo.c b/drivers/gpu/drm/i915/display/intel_dvo.c
index 0be8105cb18a..eb2dcd866cc8 100644
--- a/drivers/gpu/drm/i915/display/intel_dvo.c
+++ b/drivers/gpu/drm/i915/display/intel_dvo.c
@@ -444,11 +444,8 @@ static bool intel_dvo_init_dev(struct drm_i915_private *dev_priv,
 	 * the clock enabled before we attempt to initialize
 	 * the device.
 	 */
-	for_each_pipe(dev_priv, pipe) {
-		dpll[pipe] = intel_de_read(dev_priv, DPLL(pipe));
-		intel_de_write(dev_priv, DPLL(pipe),
-			       dpll[pipe] | DPLL_DVO_2X_MODE);
-	}
+	for_each_pipe(dev_priv, pipe)
+		dpll[pipe] = intel_de_rmw(dev_priv, DPLL(pipe), 0, DPLL_DVO_2X_MODE);
 
 	ret = dvo->dev_ops->init(&intel_dvo->dev, i2c);
 
diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c
index 93d0e46e5481..e5f637897b5e 100644
--- a/drivers/gpu/drm/i915/display/intel_fb.c
+++ b/drivers/gpu/drm/i915/display/intel_fb.c
@@ -716,14 +716,15 @@ static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
 	}
 }
 
-static bool intel_modifier_uses_dpt(struct drm_i915_private *i915, u64 modifier)
+bool intel_fb_modifier_uses_dpt(struct drm_i915_private *i915, u64 modifier)
 {
-	return DISPLAY_VER(i915) >= 13 && modifier != DRM_FORMAT_MOD_LINEAR;
+	return HAS_DPT(i915) && modifier != DRM_FORMAT_MOD_LINEAR;
 }
 
 bool intel_fb_uses_dpt(const struct drm_framebuffer *fb)
 {
-	return fb && intel_modifier_uses_dpt(to_i915(fb->dev), fb->modifier);
+	return fb && to_i915(fb->dev)->params.enable_dpt &&
+		intel_fb_modifier_uses_dpt(to_i915(fb->dev), fb->modifier);
 }
 
 unsigned int intel_cursor_alignment(const struct drm_i915_private *i915)
@@ -1189,7 +1190,7 @@ bool intel_fb_needs_pot_stride_remap(const struct intel_framebuffer *fb)
 {
 	struct drm_i915_private *i915 = to_i915(fb->base.dev);
 
-	return IS_ALDERLAKE_P(i915) && fb->base.modifier != DRM_FORMAT_MOD_LINEAR;
+	return IS_ALDERLAKE_P(i915) && intel_fb_uses_dpt(&fb->base);
 }
 
 static int intel_fb_pitch(const struct intel_framebuffer *fb, int color_plane, unsigned int rotation)
@@ -1705,7 +1706,7 @@ u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
 	 * The new CCS hash mode makes remapping impossible
 	 */
 	if (DISPLAY_VER(dev_priv) < 4 || intel_fb_is_ccs_modifier(modifier) ||
-	    intel_modifier_uses_dpt(dev_priv, modifier))
+	    intel_fb_modifier_uses_dpt(dev_priv, modifier))
 		return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
 	else if (DISPLAY_VER(dev_priv) >= 7)
 		return 256 * 1024;
@@ -2007,6 +2008,7 @@ int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 
 		vm = intel_dpt_create(intel_fb);
 		if (IS_ERR(vm)) {
+			drm_dbg_kms(&dev_priv->drm, "failed to create DPT\n");
 			ret = PTR_ERR(vm);
 			goto err;
 		}
@@ -2017,11 +2019,14 @@ int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 	ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
 	if (ret) {
 		drm_err(&dev_priv->drm, "framebuffer init failed %d\n", ret);
-		goto err;
+		goto err_free_dpt;
 	}
 
 	return 0;
 
+err_free_dpt:
+	if (intel_fb_uses_dpt(fb))
+		intel_dpt_destroy(intel_fb->dpt_vm);
 err:
 	intel_frontbuffer_put(intel_fb->frontbuffer);
 	return ret;
@@ -2046,6 +2051,7 @@ intel_user_framebuffer_create(struct drm_device *dev,
 	if (HAS_LMEM(i915) && !i915_gem_object_can_migrate(obj, INTEL_REGION_LMEM_0)) {
 		/* object is "remote", not in local memory */
 		i915_gem_object_put(obj);
+		drm_dbg_kms(&i915->drm, "framebuffer must reside in local memory\n");
 		return ERR_PTR(-EREMOTE);
 	}
 
diff --git a/drivers/gpu/drm/i915/display/intel_fb.h b/drivers/gpu/drm/i915/display/intel_fb.h
index 4662b812b934..e85167d6bc34 100644
--- a/drivers/gpu/drm/i915/display/intel_fb.h
+++ b/drivers/gpu/drm/i915/display/intel_fb.h
@@ -92,6 +92,7 @@ intel_user_framebuffer_create(struct drm_device *dev,
 			      struct drm_file *filp,
 			      const struct drm_mode_fb_cmd2 *user_mode_cmd);
 
+bool intel_fb_modifier_uses_dpt(struct drm_i915_private *i915, u64 modifier);
 bool intel_fb_uses_dpt(const struct drm_framebuffer *fb);
 
 #endif /* __INTEL_FB_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c
index 38825b30db16..673bcdfb7ff6 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/display/intel_fbdev.c
@@ -573,9 +573,9 @@ static void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
 		intel_fbdev_unregister(to_i915(ifbdev->helper.dev));
 }
 
-void intel_fbdev_initial_config_async(struct drm_device *dev)
+void intel_fbdev_initial_config_async(struct drm_i915_private *dev_priv)
 {
-	struct intel_fbdev *ifbdev = to_i915(dev)->display.fbdev.fbdev;
+	struct intel_fbdev *ifbdev = dev_priv->display.fbdev.fbdev;
 
 	if (!ifbdev)
 		return;
@@ -718,9 +718,9 @@ void intel_fbdev_output_poll_changed(struct drm_device *dev)
 		drm_fb_helper_hotplug_event(&ifbdev->helper);
 }
 
-void intel_fbdev_restore_mode(struct drm_device *dev)
+void intel_fbdev_restore_mode(struct drm_i915_private *dev_priv)
 {
-	struct intel_fbdev *ifbdev = to_i915(dev)->display.fbdev.fbdev;
+	struct intel_fbdev *ifbdev = dev_priv->display.fbdev.fbdev;
 
 	if (!ifbdev)
 		return;
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.h b/drivers/gpu/drm/i915/display/intel_fbdev.h
index 0e95e9472fa3..04fd523a5023 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev.h
+++ b/drivers/gpu/drm/i915/display/intel_fbdev.h
@@ -15,12 +15,12 @@ struct intel_framebuffer;
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
 int intel_fbdev_init(struct drm_device *dev);
-void intel_fbdev_initial_config_async(struct drm_device *dev);
+void intel_fbdev_initial_config_async(struct drm_i915_private *dev_priv);
 void intel_fbdev_unregister(struct drm_i915_private *dev_priv);
 void intel_fbdev_fini(struct drm_i915_private *dev_priv);
 void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous);
 void intel_fbdev_output_poll_changed(struct drm_device *dev);
-void intel_fbdev_restore_mode(struct drm_device *dev);
+void intel_fbdev_restore_mode(struct drm_i915_private *dev_priv);
 struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbdev *fbdev);
 #else
 static inline int intel_fbdev_init(struct drm_device *dev)
@@ -28,7 +28,7 @@ static inline int intel_fbdev_init(struct drm_device *dev)
 	return 0;
 }
 
-static inline void intel_fbdev_initial_config_async(struct drm_device *dev)
+static inline void intel_fbdev_initial_config_async(struct drm_i915_private *dev_priv)
 {
 }
 
@@ -48,7 +48,7 @@ static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
 {
 }
 
-static inline void intel_fbdev_restore_mode(struct drm_device *dev)
+static inline void intel_fbdev_restore_mode(struct drm_i915_private *i915)
 {
 }
 static inline struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbdev *fbdev)
diff --git a/drivers/gpu/drm/i915/display/intel_fdi.c b/drivers/gpu/drm/i915/display/intel_fdi.c
index 063f1da4f229..55283677c45a 100644
--- a/drivers/gpu/drm/i915/display/intel_fdi.c
+++ b/drivers/gpu/drm/i915/display/intel_fdi.c
@@ -12,6 +12,7 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_fdi.h"
+#include "intel_fdi_regs.h"
 
 struct intel_fdi_funcs {
 	void (*fdi_link_train)(struct intel_crtc *crtc,
@@ -366,8 +367,7 @@ void intel_fdi_normal_train(struct intel_crtc *crtc)
 
 	/* IVB wants error correction enabled */
 	if (IS_IVYBRIDGE(dev_priv))
-		intel_de_write(dev_priv, reg,
-			       intel_de_read(dev_priv, reg) | FDI_FS_ERRC_ENABLE | FDI_FE_ERRC_ENABLE);
+		intel_de_rmw(dev_priv, reg, 0, FDI_FS_ERRC_ENABLE | FDI_FE_ERRC_ENABLE);
 }
 
 /* The FDI link training functions for ILK/Ibexpeak. */
@@ -439,19 +439,11 @@ static void ilk_fdi_link_train(struct intel_crtc *crtc,
 		drm_err(&dev_priv->drm, "FDI train 1 fail!\n");
 
 	/* Train 2 */
-	reg = FDI_TX_CTL(pipe);
-	temp = intel_de_read(dev_priv, reg);
-	temp &= ~FDI_LINK_TRAIN_NONE;
-	temp |= FDI_LINK_TRAIN_PATTERN_2;
-	intel_de_write(dev_priv, reg, temp);
-
-	reg = FDI_RX_CTL(pipe);
-	temp = intel_de_read(dev_priv, reg);
-	temp &= ~FDI_LINK_TRAIN_NONE;
-	temp |= FDI_LINK_TRAIN_PATTERN_2;
-	intel_de_write(dev_priv, reg, temp);
-
-	intel_de_posting_read(dev_priv, reg);
+	intel_de_rmw(dev_priv, FDI_TX_CTL(pipe),
+		     FDI_LINK_TRAIN_NONE, FDI_LINK_TRAIN_PATTERN_2);
+	intel_de_rmw(dev_priv, FDI_RX_CTL(pipe),
+		     FDI_LINK_TRAIN_NONE, FDI_LINK_TRAIN_PATTERN_2);
+	intel_de_posting_read(dev_priv, FDI_RX_CTL(pipe));
 	udelay(150);
 
 	reg = FDI_RX_IIR(pipe);
@@ -538,13 +530,9 @@ static void gen6_fdi_link_train(struct intel_crtc *crtc,
 	udelay(150);
 
 	for (i = 0; i < 4; i++) {
-		reg = FDI_TX_CTL(pipe);
-		temp = intel_de_read(dev_priv, reg);
-		temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
-		temp |= snb_b_fdi_train_param[i];
-		intel_de_write(dev_priv, reg, temp);
-
-		intel_de_posting_read(dev_priv, reg);
+		intel_de_rmw(dev_priv, FDI_TX_CTL(pipe),
+			     FDI_LINK_TRAIN_VOL_EMP_MASK, snb_b_fdi_train_param[i]);
+		intel_de_posting_read(dev_priv, FDI_TX_CTL(pipe));
 		udelay(500);
 
 		for (retry = 0; retry < 5; retry++) {
@@ -593,13 +581,9 @@ static void gen6_fdi_link_train(struct intel_crtc *crtc,
 	udelay(150);
 
 	for (i = 0; i < 4; i++) {
-		reg = FDI_TX_CTL(pipe);
-		temp = intel_de_read(dev_priv, reg);
-		temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
-		temp |= snb_b_fdi_train_param[i];
-		intel_de_write(dev_priv, reg, temp);
-
-		intel_de_posting_read(dev_priv, reg);
+		intel_de_rmw(dev_priv, FDI_TX_CTL(pipe),
+			     FDI_LINK_TRAIN_VOL_EMP_MASK, snb_b_fdi_train_param[i]);
+		intel_de_posting_read(dev_priv, FDI_TX_CTL(pipe));
 		udelay(500);
 
 		for (retry = 0; retry < 5; retry++) {
@@ -719,19 +703,13 @@ static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
 		}
 
 		/* Train 2 */
-		reg = FDI_TX_CTL(pipe);
-		temp = intel_de_read(dev_priv, reg);
-		temp &= ~FDI_LINK_TRAIN_NONE_IVB;
-		temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
-		intel_de_write(dev_priv, reg, temp);
-
-		reg = FDI_RX_CTL(pipe);
-		temp = intel_de_read(dev_priv, reg);
-		temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
-		temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
-		intel_de_write(dev_priv, reg, temp);
-
-		intel_de_posting_read(dev_priv, reg);
+		intel_de_rmw(dev_priv, FDI_TX_CTL(pipe),
+			     FDI_LINK_TRAIN_NONE_IVB,
+			     FDI_LINK_TRAIN_PATTERN_2_IVB);
+		intel_de_rmw(dev_priv, FDI_RX_CTL(pipe),
+			     FDI_LINK_TRAIN_PATTERN_MASK_CPT,
+			     FDI_LINK_TRAIN_PATTERN_2_CPT);
+		intel_de_posting_read(dev_priv, FDI_RX_CTL(pipe));
 		udelay(2); /* should be 1.5us */
 
 		for (i = 0; i < 4; i++) {
@@ -837,9 +815,8 @@ void hsw_fdi_link_train(struct intel_encoder *encoder,
 		udelay(30);
 
 		/* Unset FDI_RX_MISC pwrdn lanes */
-		temp = intel_de_read(dev_priv, FDI_RX_MISC(PIPE_A));
-		temp &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK);
-		intel_de_write(dev_priv, FDI_RX_MISC(PIPE_A), temp);
+		intel_de_rmw(dev_priv, FDI_RX_MISC(PIPE_A),
+			     FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK, 0);
 		intel_de_posting_read(dev_priv, FDI_RX_MISC(PIPE_A));
 
 		/* Wait for FDI auto training time */
@@ -865,25 +842,19 @@ void hsw_fdi_link_train(struct intel_encoder *encoder,
 		intel_de_write(dev_priv, FDI_RX_CTL(PIPE_A), rx_ctl_val);
 		intel_de_posting_read(dev_priv, FDI_RX_CTL(PIPE_A));
 
-		temp = intel_de_read(dev_priv, DDI_BUF_CTL(PORT_E));
-		temp &= ~DDI_BUF_CTL_ENABLE;
-		intel_de_write(dev_priv, DDI_BUF_CTL(PORT_E), temp);
+		intel_de_rmw(dev_priv, DDI_BUF_CTL(PORT_E), DDI_BUF_CTL_ENABLE, 0);
 		intel_de_posting_read(dev_priv, DDI_BUF_CTL(PORT_E));
 
 		/* Disable DP_TP_CTL and FDI_RX_CTL and retry */
-		temp = intel_de_read(dev_priv, DP_TP_CTL(PORT_E));
-		temp &= ~(DP_TP_CTL_ENABLE | DP_TP_CTL_LINK_TRAIN_MASK);
-		temp |= DP_TP_CTL_LINK_TRAIN_PAT1;
-		intel_de_write(dev_priv, DP_TP_CTL(PORT_E), temp);
+		intel_de_rmw(dev_priv, DP_TP_CTL(PORT_E), DP_TP_CTL_ENABLE, 0);
 		intel_de_posting_read(dev_priv, DP_TP_CTL(PORT_E));
 
 		intel_wait_ddi_buf_idle(dev_priv, PORT_E);
 
 		/* Reset FDI_RX_MISC pwrdn lanes */
-		temp = intel_de_read(dev_priv, FDI_RX_MISC(PIPE_A));
-		temp &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK);
-		temp |= FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2);
-		intel_de_write(dev_priv, FDI_RX_MISC(PIPE_A), temp);
+		intel_de_rmw(dev_priv, FDI_RX_MISC(PIPE_A),
+			     FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK,
+			     FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2));
 		intel_de_posting_read(dev_priv, FDI_RX_MISC(PIPE_A));
 	}
 
@@ -898,7 +869,6 @@ void hsw_fdi_link_train(struct intel_encoder *encoder,
 void hsw_fdi_disable(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	u32 val;
 
 	/*
 	 * Bspec lists this as both step 13 (before DDI_BUF_CTL disable)
@@ -906,30 +876,15 @@ void hsw_fdi_disable(struct intel_encoder *encoder)
 	 * step 13 is the correct place for it. Step 18 is where it was
 	 * originally before the BUN.
 	 */
-	val = intel_de_read(dev_priv, FDI_RX_CTL(PIPE_A));
-	val &= ~FDI_RX_ENABLE;
-	intel_de_write(dev_priv, FDI_RX_CTL(PIPE_A), val);
-
-	val = intel_de_read(dev_priv, DDI_BUF_CTL(PORT_E));
-	val &= ~DDI_BUF_CTL_ENABLE;
-	intel_de_write(dev_priv, DDI_BUF_CTL(PORT_E), val);
-
+	intel_de_rmw(dev_priv, FDI_RX_CTL(PIPE_A), FDI_RX_ENABLE, 0);
+	intel_de_rmw(dev_priv, DDI_BUF_CTL(PORT_E), DDI_BUF_CTL_ENABLE, 0);
 	intel_wait_ddi_buf_idle(dev_priv, PORT_E);
-
 	intel_ddi_disable_clock(encoder);
-
-	val = intel_de_read(dev_priv, FDI_RX_MISC(PIPE_A));
-	val &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK);
-	val |= FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2);
-	intel_de_write(dev_priv, FDI_RX_MISC(PIPE_A), val);
-
-	val = intel_de_read(dev_priv, FDI_RX_CTL(PIPE_A));
-	val &= ~FDI_PCDCLK;
-	intel_de_write(dev_priv, FDI_RX_CTL(PIPE_A), val);
-
-	val = intel_de_read(dev_priv, FDI_RX_CTL(PIPE_A));
-	val &= ~FDI_RX_PLL_ENABLE;
-	intel_de_write(dev_priv, FDI_RX_CTL(PIPE_A), val);
+	intel_de_rmw(dev_priv, FDI_RX_MISC(PIPE_A),
+		     FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK,
+		     FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2));
+	intel_de_rmw(dev_priv, FDI_RX_CTL(PIPE_A), FDI_PCDCLK, 0);
+	intel_de_rmw(dev_priv, FDI_RX_CTL(PIPE_A), FDI_RX_PLL_ENABLE, 0);
 }
 
 void ilk_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
@@ -945,16 +900,14 @@ void ilk_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
 	temp = intel_de_read(dev_priv, reg);
 	temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
 	temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
-	temp |= (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
+	temp |= (intel_de_read(dev_priv, TRANSCONF(pipe)) & TRANSCONF_BPC_MASK) << 11;
 	intel_de_write(dev_priv, reg, temp | FDI_RX_PLL_ENABLE);
 
 	intel_de_posting_read(dev_priv, reg);
 	udelay(200);
 
 	/* Switch from Rawclk to PCDclk */
-	temp = intel_de_read(dev_priv, reg);
-	intel_de_write(dev_priv, reg, temp | FDI_PCDCLK);
-
+	intel_de_rmw(dev_priv, reg, 0, FDI_PCDCLK);
 	intel_de_posting_read(dev_priv, reg);
 	udelay(200);
 
@@ -974,28 +927,18 @@ void ilk_fdi_pll_disable(struct intel_crtc *crtc)
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
-	u32 temp;
 
 	/* Switch from PCDclk to Rawclk */
-	reg = FDI_RX_CTL(pipe);
-	temp = intel_de_read(dev_priv, reg);
-	intel_de_write(dev_priv, reg, temp & ~FDI_PCDCLK);
+	intel_de_rmw(dev_priv, FDI_RX_CTL(pipe), FDI_PCDCLK, 0);
 
 	/* Disable CPU FDI TX PLL */
-	reg = FDI_TX_CTL(pipe);
-	temp = intel_de_read(dev_priv, reg);
-	intel_de_write(dev_priv, reg, temp & ~FDI_TX_PLL_ENABLE);
-
-	intel_de_posting_read(dev_priv, reg);
+	intel_de_rmw(dev_priv, FDI_TX_CTL(pipe), FDI_TX_PLL_ENABLE, 0);
+	intel_de_posting_read(dev_priv, FDI_TX_CTL(pipe));
 	udelay(100);
 
-	reg = FDI_RX_CTL(pipe);
-	temp = intel_de_read(dev_priv, reg);
-	intel_de_write(dev_priv, reg, temp & ~FDI_RX_PLL_ENABLE);
-
 	/* Wait for the clocks to turn off. */
-	intel_de_posting_read(dev_priv, reg);
+	intel_de_rmw(dev_priv, FDI_RX_CTL(pipe), FDI_RX_PLL_ENABLE, 0);
+	intel_de_posting_read(dev_priv, FDI_RX_CTL(pipe));
 	udelay(100);
 }
 
@@ -1007,15 +950,13 @@ void ilk_fdi_disable(struct intel_crtc *crtc)
 	u32 temp;
 
 	/* disable CPU FDI tx and PCH FDI rx */
-	reg = FDI_TX_CTL(pipe);
-	temp = intel_de_read(dev_priv, reg);
-	intel_de_write(dev_priv, reg, temp & ~FDI_TX_ENABLE);
-	intel_de_posting_read(dev_priv, reg);
+	intel_de_rmw(dev_priv, FDI_TX_CTL(pipe), FDI_TX_ENABLE, 0);
+	intel_de_posting_read(dev_priv, FDI_TX_CTL(pipe));
 
 	reg = FDI_RX_CTL(pipe);
 	temp = intel_de_read(dev_priv, reg);
 	temp &= ~(0x7 << 16);
-	temp |= (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
+	temp |= (intel_de_read(dev_priv, TRANSCONF(pipe)) & TRANSCONF_BPC_MASK) << 11;
 	intel_de_write(dev_priv, reg, temp & ~FDI_RX_ENABLE);
 
 	intel_de_posting_read(dev_priv, reg);
@@ -1027,11 +968,8 @@ void ilk_fdi_disable(struct intel_crtc *crtc)
 			       FDI_RX_PHASE_SYNC_POINTER_OVR);
 
 	/* still set train pattern 1 */
-	reg = FDI_TX_CTL(pipe);
-	temp = intel_de_read(dev_priv, reg);
-	temp &= ~FDI_LINK_TRAIN_NONE;
-	temp |= FDI_LINK_TRAIN_PATTERN_1;
-	intel_de_write(dev_priv, reg, temp);
+	intel_de_rmw(dev_priv, FDI_TX_CTL(pipe),
+		     FDI_LINK_TRAIN_NONE, FDI_LINK_TRAIN_PATTERN_1);
 
 	reg = FDI_RX_CTL(pipe);
 	temp = intel_de_read(dev_priv, reg);
@@ -1042,9 +980,9 @@ void ilk_fdi_disable(struct intel_crtc *crtc)
 		temp &= ~FDI_LINK_TRAIN_NONE;
 		temp |= FDI_LINK_TRAIN_PATTERN_1;
 	}
-	/* BPC in FDI rx is consistent with that in PIPECONF */
+	/* BPC in FDI rx is consistent with that in TRANSCONF */
 	temp &= ~(0x07 << 16);
-	temp |= (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
+	temp |= (intel_de_read(dev_priv, TRANSCONF(pipe)) & TRANSCONF_BPC_MASK) << 11;
 	intel_de_write(dev_priv, reg, temp);
 
 	intel_de_posting_read(dev_priv, reg);
diff --git a/drivers/gpu/drm/i915/display/intel_fdi_regs.h b/drivers/gpu/drm/i915/display/intel_fdi_regs.h
new file mode 100644
index 000000000000..853b834c35a9
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_fdi_regs.h
@@ -0,0 +1,151 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_FDI_REGS_H__
+#define __INTEL_FDI_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+#define FDI_PLL_BIOS_0  _MMIO(0x46000)
+#define  FDI_PLL_FB_CLOCK_MASK  0xff
+#define FDI_PLL_BIOS_1  _MMIO(0x46004)
+#define FDI_PLL_BIOS_2  _MMIO(0x46008)
+#define DISPLAY_PORT_PLL_BIOS_0         _MMIO(0x4600c)
+#define DISPLAY_PORT_PLL_BIOS_1         _MMIO(0x46010)
+#define DISPLAY_PORT_PLL_BIOS_2         _MMIO(0x46014)
+
+#define FDI_PLL_FREQ_CTL        _MMIO(0x46030)
+#define  FDI_PLL_FREQ_CHANGE_REQUEST    (1 << 24)
+#define  FDI_PLL_FREQ_LOCK_LIMIT_MASK   0xfff00
+#define  FDI_PLL_FREQ_DISABLE_COUNT_LIMIT_MASK  0xff
+
+#define _FDI_RXA_CHICKEN        0xc200c
+#define _FDI_RXB_CHICKEN        0xc2010
+#define  FDI_RX_PHASE_SYNC_POINTER_OVR	(1 << 1)
+#define  FDI_RX_PHASE_SYNC_POINTER_EN	(1 << 0)
+#define FDI_RX_CHICKEN(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_CHICKEN, _FDI_RXB_CHICKEN)
+
+/* CPU: FDI_TX */
+#define _FDI_TXA_CTL            0x60100
+#define _FDI_TXB_CTL            0x61100
+#define FDI_TX_CTL(pipe)	_MMIO_PIPE(pipe, _FDI_TXA_CTL, _FDI_TXB_CTL)
+#define  FDI_TX_DISABLE         (0 << 31)
+#define  FDI_TX_ENABLE          (1 << 31)
+#define  FDI_LINK_TRAIN_PATTERN_1       (0 << 28)
+#define  FDI_LINK_TRAIN_PATTERN_2       (1 << 28)
+#define  FDI_LINK_TRAIN_PATTERN_IDLE    (2 << 28)
+#define  FDI_LINK_TRAIN_NONE            (3 << 28)
+#define  FDI_LINK_TRAIN_VOLTAGE_0_4V    (0 << 25)
+#define  FDI_LINK_TRAIN_VOLTAGE_0_6V    (1 << 25)
+#define  FDI_LINK_TRAIN_VOLTAGE_0_8V    (2 << 25)
+#define  FDI_LINK_TRAIN_VOLTAGE_1_2V    (3 << 25)
+#define  FDI_LINK_TRAIN_PRE_EMPHASIS_NONE (0 << 22)
+#define  FDI_LINK_TRAIN_PRE_EMPHASIS_1_5X (1 << 22)
+#define  FDI_LINK_TRAIN_PRE_EMPHASIS_2X   (2 << 22)
+#define  FDI_LINK_TRAIN_PRE_EMPHASIS_3X   (3 << 22)
+/* ILK always use 400mV 0dB for voltage swing and pre-emphasis level.
+   SNB has different settings. */
+/* SNB A-stepping */
+#define  FDI_LINK_TRAIN_400MV_0DB_SNB_A		(0x38 << 22)
+#define  FDI_LINK_TRAIN_400MV_6DB_SNB_A		(0x02 << 22)
+#define  FDI_LINK_TRAIN_600MV_3_5DB_SNB_A	(0x01 << 22)
+#define  FDI_LINK_TRAIN_800MV_0DB_SNB_A		(0x0 << 22)
+/* SNB B-stepping */
+#define  FDI_LINK_TRAIN_400MV_0DB_SNB_B		(0x0 << 22)
+#define  FDI_LINK_TRAIN_400MV_6DB_SNB_B		(0x3a << 22)
+#define  FDI_LINK_TRAIN_600MV_3_5DB_SNB_B	(0x39 << 22)
+#define  FDI_LINK_TRAIN_800MV_0DB_SNB_B		(0x38 << 22)
+#define  FDI_LINK_TRAIN_VOL_EMP_MASK		(0x3f << 22)
+#define  FDI_DP_PORT_WIDTH_SHIFT		19
+#define  FDI_DP_PORT_WIDTH_MASK			(7 << FDI_DP_PORT_WIDTH_SHIFT)
+#define  FDI_DP_PORT_WIDTH(width)           (((width) - 1) << FDI_DP_PORT_WIDTH_SHIFT)
+#define  FDI_TX_ENHANCE_FRAME_ENABLE    (1 << 18)
+/* Ironlake: hardwired to 1 */
+#define  FDI_TX_PLL_ENABLE              (1 << 14)
+
+/* Ivybridge has different bits for lolz */
+#define  FDI_LINK_TRAIN_PATTERN_1_IVB       (0 << 8)
+#define  FDI_LINK_TRAIN_PATTERN_2_IVB       (1 << 8)
+#define  FDI_LINK_TRAIN_PATTERN_IDLE_IVB    (2 << 8)
+#define  FDI_LINK_TRAIN_NONE_IVB            (3 << 8)
+
+/* both Tx and Rx */
+#define  FDI_COMPOSITE_SYNC		(1 << 11)
+#define  FDI_LINK_TRAIN_AUTO		(1 << 10)
+#define  FDI_SCRAMBLING_ENABLE          (0 << 7)
+#define  FDI_SCRAMBLING_DISABLE         (1 << 7)
+
+/* FDI_RX, FDI_X is hard-wired to Transcoder_X */
+#define _FDI_RXA_CTL             0xf000c
+#define _FDI_RXB_CTL             0xf100c
+#define FDI_RX_CTL(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_CTL, _FDI_RXB_CTL)
+#define  FDI_RX_ENABLE          (1 << 31)
+/* train, dp width same as FDI_TX */
+#define  FDI_FS_ERRC_ENABLE		(1 << 27)
+#define  FDI_FE_ERRC_ENABLE		(1 << 26)
+#define  FDI_RX_POLARITY_REVERSED_LPT	(1 << 16)
+#define  FDI_8BPC                       (0 << 16)
+#define  FDI_10BPC                      (1 << 16)
+#define  FDI_6BPC                       (2 << 16)
+#define  FDI_12BPC                      (3 << 16)
+#define  FDI_RX_LINK_REVERSAL_OVERRIDE  (1 << 15)
+#define  FDI_DMI_LINK_REVERSE_MASK      (1 << 14)
+#define  FDI_RX_PLL_ENABLE              (1 << 13)
+#define  FDI_FS_ERR_CORRECT_ENABLE      (1 << 11)
+#define  FDI_FE_ERR_CORRECT_ENABLE      (1 << 10)
+#define  FDI_FS_ERR_REPORT_ENABLE       (1 << 9)
+#define  FDI_FE_ERR_REPORT_ENABLE       (1 << 8)
+#define  FDI_RX_ENHANCE_FRAME_ENABLE    (1 << 6)
+#define  FDI_PCDCLK	                (1 << 4)
+/* CPT */
+#define  FDI_AUTO_TRAINING			(1 << 10)
+#define  FDI_LINK_TRAIN_PATTERN_1_CPT		(0 << 8)
+#define  FDI_LINK_TRAIN_PATTERN_2_CPT		(1 << 8)
+#define  FDI_LINK_TRAIN_PATTERN_IDLE_CPT	(2 << 8)
+#define  FDI_LINK_TRAIN_NORMAL_CPT		(3 << 8)
+#define  FDI_LINK_TRAIN_PATTERN_MASK_CPT	(3 << 8)
+
+#define _FDI_RXA_MISC			0xf0010
+#define _FDI_RXB_MISC			0xf1010
+#define  FDI_RX_PWRDN_LANE1_MASK	(3 << 26)
+#define  FDI_RX_PWRDN_LANE1_VAL(x)	((x) << 26)
+#define  FDI_RX_PWRDN_LANE0_MASK	(3 << 24)
+#define  FDI_RX_PWRDN_LANE0_VAL(x)	((x) << 24)
+#define  FDI_RX_TP1_TO_TP2_48		(2 << 20)
+#define  FDI_RX_TP1_TO_TP2_64		(3 << 20)
+#define  FDI_RX_FDI_DELAY_90		(0x90 << 0)
+#define FDI_RX_MISC(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_MISC, _FDI_RXB_MISC)
+
+#define _FDI_RXA_TUSIZE1        0xf0030
+#define _FDI_RXA_TUSIZE2        0xf0038
+#define _FDI_RXB_TUSIZE1        0xf1030
+#define _FDI_RXB_TUSIZE2        0xf1038
+#define FDI_RX_TUSIZE1(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_TUSIZE1, _FDI_RXB_TUSIZE1)
+#define FDI_RX_TUSIZE2(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_TUSIZE2, _FDI_RXB_TUSIZE2)
+
+/* FDI_RX interrupt register format */
+#define FDI_RX_INTER_LANE_ALIGN         (1 << 10)
+#define FDI_RX_SYMBOL_LOCK              (1 << 9) /* train 2 */
+#define FDI_RX_BIT_LOCK                 (1 << 8) /* train 1 */
+#define FDI_RX_TRAIN_PATTERN_2_FAIL     (1 << 7)
+#define FDI_RX_FS_CODE_ERR              (1 << 6)
+#define FDI_RX_FE_CODE_ERR              (1 << 5)
+#define FDI_RX_SYMBOL_ERR_RATE_ABOVE    (1 << 4)
+#define FDI_RX_HDCP_LINK_FAIL           (1 << 3)
+#define FDI_RX_PIXEL_FIFO_OVERFLOW      (1 << 2)
+#define FDI_RX_CROSS_CLOCK_OVERFLOW     (1 << 1)
+#define FDI_RX_SYMBOL_QUEUE_OVERFLOW    (1 << 0)
+
+#define _FDI_RXA_IIR            0xf0014
+#define _FDI_RXA_IMR            0xf0018
+#define _FDI_RXB_IIR            0xf1014
+#define _FDI_RXB_IMR            0xf1018
+#define FDI_RX_IIR(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_IIR, _FDI_RXB_IIR)
+#define FDI_RX_IMR(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_IMR, _FDI_RXB_IMR)
+
+#define FDI_PLL_CTL_1           _MMIO(0xfe000)
+#define FDI_PLL_CTL_2           _MMIO(0xfe004)
+
+#endif /* __INTEL_FDI_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
index d636d21fa9ce..b708a62e509a 100644
--- a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
@@ -31,6 +31,7 @@
 #include "intel_display_types.h"
 #include "intel_fbc.h"
 #include "intel_fifo_underrun.h"
+#include "intel_pch_display.h"
 
 /**
  * DOC: fifo underrun handling
@@ -509,3 +510,22 @@ void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv)
 
 	spin_unlock_irq(&dev_priv->irq_lock);
 }
+
+void intel_init_fifo_underrun_reporting(struct drm_i915_private *i915,
+					struct intel_crtc *crtc,
+					bool enable)
+{
+	crtc->cpu_fifo_underrun_disabled = !enable;
+
+	/*
+	 * We track the PCH trancoder underrun reporting state
+	 * within the crtc. With crtc for pipe A housing the underrun
+	 * reporting state for PCH transcoder A, crtc for pipe B housing
+	 * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
+	 * and marking underrun reporting as disabled for the non-existing
+	 * PCH transcoders B and C would prevent enabling the south
+	 * error interrupt (see cpt_can_enable_serr_int()).
+	 */
+	if (intel_has_pch_trancoder(i915, crtc->pipe))
+		crtc->pch_fifo_underrun_disabled = !enable;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_fifo_underrun.h b/drivers/gpu/drm/i915/display/intel_fifo_underrun.h
index 2e47d7d3c101..b00d8abebcf9 100644
--- a/drivers/gpu/drm/i915/display/intel_fifo_underrun.h
+++ b/drivers/gpu/drm/i915/display/intel_fifo_underrun.h
@@ -9,8 +9,11 @@
 #include <linux/types.h>
 
 struct drm_i915_private;
+struct intel_crtc;
 enum pipe;
 
+void intel_init_fifo_underrun_reporting(struct drm_i915_private *i915,
+					struct intel_crtc *crtc, bool enable);
 bool intel_set_cpu_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
 					   enum pipe pipe, bool enable);
 bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c
index 0bc4f6b48e80..3ddfc8080ee8 100644
--- a/drivers/gpu/drm/i915/display/intel_gmbus.c
+++ b/drivers/gpu/drm/i915/display/intel_gmbus.c
@@ -215,41 +215,23 @@ intel_gmbus_reset(struct drm_i915_private *i915)
 static void pnv_gmbus_clock_gating(struct drm_i915_private *i915,
 				   bool enable)
 {
-	u32 val;
-
 	/* When using bit bashing for I2C, this bit needs to be set to 1 */
-	val = intel_de_read(i915, DSPCLK_GATE_D(i915));
-	if (!enable)
-		val |= PNV_GMBUSUNIT_CLOCK_GATE_DISABLE;
-	else
-		val &= ~PNV_GMBUSUNIT_CLOCK_GATE_DISABLE;
-	intel_de_write(i915, DSPCLK_GATE_D(i915), val);
+	intel_de_rmw(i915, DSPCLK_GATE_D(i915), PNV_GMBUSUNIT_CLOCK_GATE_DISABLE,
+		     !enable ? PNV_GMBUSUNIT_CLOCK_GATE_DISABLE : 0);
 }
 
 static void pch_gmbus_clock_gating(struct drm_i915_private *i915,
 				   bool enable)
 {
-	u32 val;
-
-	val = intel_de_read(i915, SOUTH_DSPCLK_GATE_D);
-	if (!enable)
-		val |= PCH_GMBUSUNIT_CLOCK_GATE_DISABLE;
-	else
-		val &= ~PCH_GMBUSUNIT_CLOCK_GATE_DISABLE;
-	intel_de_write(i915, SOUTH_DSPCLK_GATE_D, val);
+	intel_de_rmw(i915, SOUTH_DSPCLK_GATE_D, PCH_GMBUSUNIT_CLOCK_GATE_DISABLE,
+		     !enable ? PCH_GMBUSUNIT_CLOCK_GATE_DISABLE : 0);
 }
 
 static void bxt_gmbus_clock_gating(struct drm_i915_private *i915,
 				   bool enable)
 {
-	u32 val;
-
-	val = intel_de_read(i915, GEN9_CLKGATE_DIS_4);
-	if (!enable)
-		val |= BXT_GMBUS_GATING_DIS;
-	else
-		val &= ~BXT_GMBUS_GATING_DIS;
-	intel_de_write(i915, GEN9_CLKGATE_DIS_4, val);
+	intel_de_rmw(i915, GEN9_CLKGATE_DIS_4, BXT_GMBUS_GATING_DIS,
+		     !enable ? BXT_GMBUS_GATING_DIS : 0);
 }
 
 static u32 get_reserved(struct intel_gmbus *bus)
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c
index 6406fd487ee5..650232c4892b 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp.c
+++ b/drivers/gpu/drm/i915/display/intel_hdcp.c
@@ -23,6 +23,7 @@
 #include "intel_display_power_well.h"
 #include "intel_display_types.h"
 #include "intel_hdcp.h"
+#include "intel_hdcp_gsc.h"
 #include "intel_hdcp_regs.h"
 #include "intel_pcode.h"
 
@@ -203,13 +204,20 @@ bool intel_hdcp2_capable(struct intel_connector *connector)
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_hdcp *hdcp = &connector->hdcp;
+	struct intel_gt *gt = dev_priv->media_gt;
+	struct intel_gsc_uc *gsc = &gt->uc.gsc;
 	bool capable = false;
 
 	/* I915 support for HDCP2.2 */
 	if (!hdcp->hdcp2_supported)
 		return false;
 
-	/* MEI interface is solid */
+	/* If MTL+ make sure gsc is loaded and proxy is setup */
+	if (intel_hdcp_gsc_cs_required(dev_priv))
+		if (!intel_uc_fw_is_running(&gsc->fw))
+			return false;
+
+	/* MEI/GSC interface is solid depending on which is used */
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
 	if (!dev_priv->display.hdcp.comp_added ||  !dev_priv->display.hdcp.master) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
@@ -943,8 +951,7 @@ static int _intel_hdcp_disable(struct intel_connector *connector)
 
 	repeater_ctl = intel_hdcp_get_repeater_ctl(dev_priv, cpu_transcoder,
 						   port);
-	intel_de_write(dev_priv, HDCP_REP_CTL,
-		       intel_de_read(dev_priv, HDCP_REP_CTL) & ~repeater_ctl);
+	intel_de_rmw(dev_priv, HDCP_REP_CTL, repeater_ctl, 0);
 
 	ret = hdcp->shim->toggle_signalling(dig_port, cpu_transcoder, false);
 	if (ret) {
@@ -1143,18 +1150,18 @@ hdcp2_prepare_ake_init(struct intel_connector *connector,
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct hdcp_port_data *data = &dig_port->hdcp_port_data;
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->initiate_hdcp2_session(comp->mei_dev, data, ake_data);
+	ret = arbiter->ops->initiate_hdcp2_session(arbiter->hdcp_dev, data, ake_data);
 	if (ret)
 		drm_dbg_kms(&dev_priv->drm, "Prepare_ake_init failed. %d\n",
 			    ret);
@@ -1173,18 +1180,18 @@ hdcp2_verify_rx_cert_prepare_km(struct intel_connector *connector,
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct hdcp_port_data *data = &dig_port->hdcp_port_data;
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->verify_receiver_cert_prepare_km(comp->mei_dev, data,
+	ret = arbiter->ops->verify_receiver_cert_prepare_km(arbiter->hdcp_dev, data,
 							 rx_cert, paired,
 							 ek_pub_km, msg_sz);
 	if (ret < 0)
@@ -1201,18 +1208,18 @@ static int hdcp2_verify_hprime(struct intel_connector *connector,
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct hdcp_port_data *data = &dig_port->hdcp_port_data;
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->verify_hprime(comp->mei_dev, data, rx_hprime);
+	ret = arbiter->ops->verify_hprime(arbiter->hdcp_dev, data, rx_hprime);
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Verify hprime failed. %d\n", ret);
 	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
@@ -1227,18 +1234,18 @@ hdcp2_store_pairing_info(struct intel_connector *connector,
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct hdcp_port_data *data = &dig_port->hdcp_port_data;
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->store_pairing_info(comp->mei_dev, data, pairing_info);
+	ret = arbiter->ops->store_pairing_info(arbiter->hdcp_dev, data, pairing_info);
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Store pairing info failed. %d\n",
 			    ret);
@@ -1254,18 +1261,18 @@ hdcp2_prepare_lc_init(struct intel_connector *connector,
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct hdcp_port_data *data = &dig_port->hdcp_port_data;
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->initiate_locality_check(comp->mei_dev, data, lc_init);
+	ret = arbiter->ops->initiate_locality_check(arbiter->hdcp_dev, data, lc_init);
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Prepare lc_init failed. %d\n",
 			    ret);
@@ -1281,18 +1288,18 @@ hdcp2_verify_lprime(struct intel_connector *connector,
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct hdcp_port_data *data = &dig_port->hdcp_port_data;
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->verify_lprime(comp->mei_dev, data, rx_lprime);
+	ret = arbiter->ops->verify_lprime(arbiter->hdcp_dev, data, rx_lprime);
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Verify L_Prime failed. %d\n",
 			    ret);
@@ -1307,18 +1314,18 @@ static int hdcp2_prepare_skey(struct intel_connector *connector,
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct hdcp_port_data *data = &dig_port->hdcp_port_data;
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->get_session_key(comp->mei_dev, data, ske_data);
+	ret = arbiter->ops->get_session_key(arbiter->hdcp_dev, data, ske_data);
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Get session key failed. %d\n",
 			    ret);
@@ -1336,20 +1343,21 @@ hdcp2_verify_rep_topology_prepare_ack(struct intel_connector *connector,
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct hdcp_port_data *data = &dig_port->hdcp_port_data;
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->repeater_check_flow_prepare_ack(comp->mei_dev, data,
-							 rep_topology,
-							 rep_send_ack);
+	ret = arbiter->ops->repeater_check_flow_prepare_ack(arbiter->hdcp_dev,
+							    data,
+							    rep_topology,
+							    rep_send_ack);
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm,
 			    "Verify rep topology failed. %d\n", ret);
@@ -1365,18 +1373,18 @@ hdcp2_verify_mprime(struct intel_connector *connector,
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct hdcp_port_data *data = &dig_port->hdcp_port_data;
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->verify_mprime(comp->mei_dev, data, stream_ready);
+	ret = arbiter->ops->verify_mprime(arbiter->hdcp_dev, data, stream_ready);
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Verify mprime failed. %d\n", ret);
 	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
@@ -1389,18 +1397,18 @@ static int hdcp2_authenticate_port(struct intel_connector *connector)
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct hdcp_port_data *data = &dig_port->hdcp_port_data;
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->enable_hdcp_authentication(comp->mei_dev, data);
+	ret = arbiter->ops->enable_hdcp_authentication(arbiter->hdcp_dev, data);
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Enable hdcp auth failed. %d\n",
 			    ret);
@@ -1409,22 +1417,22 @@ static int hdcp2_authenticate_port(struct intel_connector *connector)
 	return ret;
 }
 
-static int hdcp2_close_mei_session(struct intel_connector *connector)
+static int hdcp2_close_session(struct intel_connector *connector)
 {
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct i915_hdcp_comp_master *comp;
+	struct i915_hdcp_master *arbiter;
 	int ret;
 
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	comp = dev_priv->display.hdcp.master;
+	arbiter = dev_priv->display.hdcp.master;
 
-	if (!comp || !comp->ops) {
+	if (!arbiter || !arbiter->ops) {
 		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
-	ret = comp->ops->close_hdcp_session(comp->mei_dev,
+	ret = arbiter->ops->close_hdcp_session(arbiter->hdcp_dev,
 					     &dig_port->hdcp_port_data);
 	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
@@ -1433,7 +1441,7 @@ static int hdcp2_close_mei_session(struct intel_connector *connector)
 
 static int hdcp2_deauthenticate_port(struct intel_connector *connector)
 {
-	return hdcp2_close_mei_session(connector);
+	return hdcp2_close_session(connector);
 }
 
 /* Authentication flow starts from here */
@@ -1819,12 +1827,10 @@ static int hdcp2_enable_encryption(struct intel_connector *connector)
 	}
 
 	if (intel_de_read(dev_priv, HDCP2_STATUS(dev_priv, cpu_transcoder, port)) &
-	    LINK_AUTH_STATUS) {
+	    LINK_AUTH_STATUS)
 		/* Link is Authenticated. Now set for Encryption */
-		intel_de_write(dev_priv,
-			       HDCP2_CTL(dev_priv, cpu_transcoder, port),
-			       intel_de_read(dev_priv, HDCP2_CTL(dev_priv, cpu_transcoder, port)) | CTL_LINK_ENCRYPTION_REQ);
-	}
+		intel_de_rmw(dev_priv, HDCP2_CTL(dev_priv, cpu_transcoder, port),
+			     0, CTL_LINK_ENCRYPTION_REQ);
 
 	ret = intel_de_wait_for_set(dev_priv,
 				    HDCP2_STATUS(dev_priv, cpu_transcoder,
@@ -1848,8 +1854,8 @@ static int hdcp2_disable_encryption(struct intel_connector *connector)
 	drm_WARN_ON(&dev_priv->drm, !(intel_de_read(dev_priv, HDCP2_STATUS(dev_priv, cpu_transcoder, port)) &
 				      LINK_ENCRYPTION_STATUS));
 
-	intel_de_write(dev_priv, HDCP2_CTL(dev_priv, cpu_transcoder, port),
-		       intel_de_read(dev_priv, HDCP2_CTL(dev_priv, cpu_transcoder, port)) & ~CTL_LINK_ENCRYPTION_REQ);
+	intel_de_rmw(dev_priv, HDCP2_CTL(dev_priv, cpu_transcoder, port),
+		     CTL_LINK_ENCRYPTION_REQ, 0);
 
 	ret = intel_de_wait_for_clear(dev_priv,
 				      HDCP2_STATUS(dev_priv, cpu_transcoder,
@@ -2145,8 +2151,8 @@ static int i915_hdcp_component_bind(struct device *i915_kdev,
 
 	drm_dbg(&dev_priv->drm, "I915 HDCP comp bind\n");
 	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
-	dev_priv->display.hdcp.master = (struct i915_hdcp_comp_master *)data;
-	dev_priv->display.hdcp.master->mei_dev = mei_kdev;
+	dev_priv->display.hdcp.master = (struct i915_hdcp_master *)data;
+	dev_priv->display.hdcp.master->hdcp_dev = mei_kdev;
 	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return 0;
@@ -2163,30 +2169,30 @@ static void i915_hdcp_component_unbind(struct device *i915_kdev,
 	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 }
 
-static const struct component_ops i915_hdcp_component_ops = {
+static const struct component_ops i915_hdcp_ops = {
 	.bind   = i915_hdcp_component_bind,
 	.unbind = i915_hdcp_component_unbind,
 };
 
-static enum mei_fw_ddi intel_get_mei_fw_ddi_index(enum port port)
+static enum hdcp_ddi intel_get_hdcp_ddi_index(enum port port)
 {
 	switch (port) {
 	case PORT_A:
-		return MEI_DDI_A;
+		return HDCP_DDI_A;
 	case PORT_B ... PORT_F:
-		return (enum mei_fw_ddi)port;
+		return (enum hdcp_ddi)port;
 	default:
-		return MEI_DDI_INVALID_PORT;
+		return HDCP_DDI_INVALID_PORT;
 	}
 }
 
-static enum mei_fw_tc intel_get_mei_fw_tc(enum transcoder cpu_transcoder)
+static enum hdcp_transcoder intel_get_hdcp_transcoder(enum transcoder cpu_transcoder)
 {
 	switch (cpu_transcoder) {
 	case TRANSCODER_A ... TRANSCODER_D:
-		return (enum mei_fw_tc)(cpu_transcoder | 0x10);
+		return (enum hdcp_transcoder)(cpu_transcoder | 0x10);
 	default: /* eDP, DSI TRANSCODERS are non HDCP capable */
-		return MEI_INVALID_TRANSCODER;
+		return HDCP_INVALID_TRANSCODER;
 	}
 }
 
@@ -2200,20 +2206,20 @@ static int initialize_hdcp_port_data(struct intel_connector *connector,
 	enum port port = dig_port->base.port;
 
 	if (DISPLAY_VER(dev_priv) < 12)
-		data->fw_ddi = intel_get_mei_fw_ddi_index(port);
+		data->hdcp_ddi = intel_get_hdcp_ddi_index(port);
 	else
 		/*
-		 * As per ME FW API expectation, for GEN 12+, fw_ddi is filled
+		 * As per ME FW API expectation, for GEN 12+, hdcp_ddi is filled
 		 * with zero(INVALID PORT index).
 		 */
-		data->fw_ddi = MEI_DDI_INVALID_PORT;
+		data->hdcp_ddi = HDCP_DDI_INVALID_PORT;
 
 	/*
-	 * As associated transcoder is set and modified at modeset, here fw_tc
+	 * As associated transcoder is set and modified at modeset, here hdcp_transcoder
 	 * is initialized to zero (invalid transcoder index). This will be
 	 * retained for <Gen12 forever.
 	 */
-	data->fw_tc = MEI_INVALID_TRANSCODER;
+	data->hdcp_transcoder = HDCP_INVALID_TRANSCODER;
 
 	data->port_type = (u8)HDCP_PORT_TYPE_INTEGRATED;
 	data->protocol = (u8)shim->protocol;
@@ -2235,6 +2241,9 @@ static int initialize_hdcp_port_data(struct intel_connector *connector,
 
 static bool is_hdcp2_supported(struct drm_i915_private *dev_priv)
 {
+	if (intel_hdcp_gsc_cs_required(dev_priv))
+		return true;
+
 	if (!IS_ENABLED(CONFIG_INTEL_MEI_HDCP))
 		return false;
 
@@ -2256,10 +2265,14 @@ void intel_hdcp_component_init(struct drm_i915_private *dev_priv)
 
 	dev_priv->display.hdcp.comp_added = true;
 	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
-	ret = component_add_typed(dev_priv->drm.dev, &i915_hdcp_component_ops,
-				  I915_COMPONENT_HDCP);
+	if (intel_hdcp_gsc_cs_required(dev_priv))
+		ret = intel_hdcp_gsc_init(dev_priv);
+	else
+		ret = component_add_typed(dev_priv->drm.dev, &i915_hdcp_ops,
+					  I915_COMPONENT_HDCP);
+
 	if (ret < 0) {
-		drm_dbg_kms(&dev_priv->drm, "Failed at component add(%d)\n",
+		drm_dbg_kms(&dev_priv->drm, "Failed at fw component add(%d)\n",
 			    ret);
 		mutex_lock(&dev_priv->display.hdcp.comp_mutex);
 		dev_priv->display.hdcp.comp_added = false;
@@ -2350,7 +2363,8 @@ int intel_hdcp_enable(struct intel_connector *connector,
 	}
 
 	if (DISPLAY_VER(dev_priv) >= 12)
-		dig_port->hdcp_port_data.fw_tc = intel_get_mei_fw_tc(hdcp->cpu_transcoder);
+		dig_port->hdcp_port_data.hdcp_transcoder =
+			intel_get_hdcp_transcoder(hdcp->cpu_transcoder);
 
 	/*
 	 * Considering that HDCP2.2 is more secure than HDCP1.4, If the setup
@@ -2485,7 +2499,10 @@ void intel_hdcp_component_fini(struct drm_i915_private *dev_priv)
 	dev_priv->display.hdcp.comp_added = false;
 	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
-	component_del(dev_priv->drm.dev, &i915_hdcp_component_ops);
+	if (intel_hdcp_gsc_cs_required(dev_priv))
+		intel_hdcp_gsc_fini(dev_priv);
+	else
+		component_del(dev_priv->drm.dev, &i915_hdcp_ops);
 }
 
 void intel_hdcp_cleanup(struct intel_connector *connector)
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp_gsc.c b/drivers/gpu/drm/i915/display/intel_hdcp_gsc.c
new file mode 100644
index 000000000000..7e52aea6aa17
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_hdcp_gsc.c
@@ -0,0 +1,831 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2023, Intel Corporation.
+ */
+
+#include <drm/i915_hdcp_interface.h>
+
+#include "display/intel_hdcp_gsc.h"
+#include "gem/i915_gem_region.h"
+#include "gt/uc/intel_gsc_uc_heci_cmd_submit.h"
+#include "i915_drv.h"
+#include "i915_utils.h"
+
+bool intel_hdcp_gsc_cs_required(struct drm_i915_private *i915)
+{
+	return DISPLAY_VER(i915) >= 14;
+}
+
+static int
+gsc_hdcp_initiate_session(struct device *dev, struct hdcp_port_data *data,
+			  struct hdcp2_ake_init *ake_data)
+{
+	struct wired_cmd_initiate_hdcp2_session_in session_init_in = { { 0 } };
+	struct wired_cmd_initiate_hdcp2_session_out
+						session_init_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+
+	if (!dev || !data || !ake_data)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	session_init_in.header.api_version = HDCP_API_VERSION;
+	session_init_in.header.command_id = WIRED_INITIATE_HDCP2_SESSION;
+	session_init_in.header.status = FW_HDCP_STATUS_SUCCESS;
+	session_init_in.header.buffer_len =
+				WIRED_CMD_BUF_LEN_INITIATE_HDCP2_SESSION_IN;
+
+	session_init_in.port.integrated_port_type = data->port_type;
+	session_init_in.port.physical_port = (u8)data->hdcp_ddi;
+	session_init_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
+	session_init_in.protocol = data->protocol;
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)&session_init_in,
+				       sizeof(session_init_in),
+				       (u8 *)&session_init_out,
+				       sizeof(session_init_out));
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
+		return byte;
+	}
+
+	if (session_init_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "FW cmd 0x%08X Failed. Status: 0x%X\n",
+			    WIRED_INITIATE_HDCP2_SESSION,
+			    session_init_out.header.status);
+		return -EIO;
+	}
+
+	ake_data->msg_id = HDCP_2_2_AKE_INIT;
+	ake_data->tx_caps = session_init_out.tx_caps;
+	memcpy(ake_data->r_tx, session_init_out.r_tx, HDCP_2_2_RTX_LEN);
+
+	return 0;
+}
+
+static int
+gsc_hdcp_verify_receiver_cert_prepare_km(struct device *dev,
+					 struct hdcp_port_data *data,
+					 struct hdcp2_ake_send_cert *rx_cert,
+					 bool *km_stored,
+					 struct hdcp2_ake_no_stored_km
+								*ek_pub_km,
+					 size_t *msg_sz)
+{
+	struct wired_cmd_verify_receiver_cert_in verify_rxcert_in = { { 0 } };
+	struct wired_cmd_verify_receiver_cert_out verify_rxcert_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+
+	if (!dev || !data || !rx_cert || !km_stored || !ek_pub_km || !msg_sz)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	verify_rxcert_in.header.api_version = HDCP_API_VERSION;
+	verify_rxcert_in.header.command_id = WIRED_VERIFY_RECEIVER_CERT;
+	verify_rxcert_in.header.status = FW_HDCP_STATUS_SUCCESS;
+	verify_rxcert_in.header.buffer_len =
+				WIRED_CMD_BUF_LEN_VERIFY_RECEIVER_CERT_IN;
+
+	verify_rxcert_in.port.integrated_port_type = data->port_type;
+	verify_rxcert_in.port.physical_port = (u8)data->hdcp_ddi;
+	verify_rxcert_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
+
+	verify_rxcert_in.cert_rx = rx_cert->cert_rx;
+	memcpy(verify_rxcert_in.r_rx, &rx_cert->r_rx, HDCP_2_2_RRX_LEN);
+	memcpy(verify_rxcert_in.rx_caps, rx_cert->rx_caps, HDCP_2_2_RXCAPS_LEN);
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)&verify_rxcert_in,
+				       sizeof(verify_rxcert_in),
+				       (u8 *)&verify_rxcert_out,
+				       sizeof(verify_rxcert_out));
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed: %zd\n", byte);
+		return byte;
+	}
+
+	if (verify_rxcert_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "FW cmd 0x%08X Failed. Status: 0x%X\n",
+			    WIRED_VERIFY_RECEIVER_CERT,
+			    verify_rxcert_out.header.status);
+		return -EIO;
+	}
+
+	*km_stored = !!verify_rxcert_out.km_stored;
+	if (verify_rxcert_out.km_stored) {
+		ek_pub_km->msg_id = HDCP_2_2_AKE_STORED_KM;
+		*msg_sz = sizeof(struct hdcp2_ake_stored_km);
+	} else {
+		ek_pub_km->msg_id = HDCP_2_2_AKE_NO_STORED_KM;
+		*msg_sz = sizeof(struct hdcp2_ake_no_stored_km);
+	}
+
+	memcpy(ek_pub_km->e_kpub_km, &verify_rxcert_out.ekm_buff,
+	       sizeof(verify_rxcert_out.ekm_buff));
+
+	return 0;
+}
+
+static int
+gsc_hdcp_verify_hprime(struct device *dev, struct hdcp_port_data *data,
+		       struct hdcp2_ake_send_hprime *rx_hprime)
+{
+	struct wired_cmd_ake_send_hprime_in send_hprime_in = { { 0 } };
+	struct wired_cmd_ake_send_hprime_out send_hprime_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+
+	if (!dev || !data || !rx_hprime)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	send_hprime_in.header.api_version = HDCP_API_VERSION;
+	send_hprime_in.header.command_id = WIRED_AKE_SEND_HPRIME;
+	send_hprime_in.header.status = FW_HDCP_STATUS_SUCCESS;
+	send_hprime_in.header.buffer_len = WIRED_CMD_BUF_LEN_AKE_SEND_HPRIME_IN;
+
+	send_hprime_in.port.integrated_port_type = data->port_type;
+	send_hprime_in.port.physical_port = (u8)data->hdcp_ddi;
+	send_hprime_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
+
+	memcpy(send_hprime_in.h_prime, rx_hprime->h_prime,
+	       HDCP_2_2_H_PRIME_LEN);
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)&send_hprime_in,
+				       sizeof(send_hprime_in),
+				       (u8 *)&send_hprime_out,
+				       sizeof(send_hprime_out));
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
+		return byte;
+	}
+
+	if (send_hprime_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "FW cmd 0x%08X Failed. Status: 0x%X\n",
+			    WIRED_AKE_SEND_HPRIME, send_hprime_out.header.status);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int
+gsc_hdcp_store_pairing_info(struct device *dev, struct hdcp_port_data *data,
+			    struct hdcp2_ake_send_pairing_info *pairing_info)
+{
+	struct wired_cmd_ake_send_pairing_info_in pairing_info_in = { { 0 } };
+	struct wired_cmd_ake_send_pairing_info_out pairing_info_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+
+	if (!dev || !data || !pairing_info)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	pairing_info_in.header.api_version = HDCP_API_VERSION;
+	pairing_info_in.header.command_id = WIRED_AKE_SEND_PAIRING_INFO;
+	pairing_info_in.header.status = FW_HDCP_STATUS_SUCCESS;
+	pairing_info_in.header.buffer_len =
+					WIRED_CMD_BUF_LEN_SEND_PAIRING_INFO_IN;
+
+	pairing_info_in.port.integrated_port_type = data->port_type;
+	pairing_info_in.port.physical_port = (u8)data->hdcp_ddi;
+	pairing_info_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
+
+	memcpy(pairing_info_in.e_kh_km, pairing_info->e_kh_km,
+	       HDCP_2_2_E_KH_KM_LEN);
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)&pairing_info_in,
+				       sizeof(pairing_info_in),
+				       (u8 *)&pairing_info_out,
+				       sizeof(pairing_info_out));
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
+		return byte;
+	}
+
+	if (pairing_info_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "FW cmd 0x%08X failed. Status: 0x%X\n",
+			    WIRED_AKE_SEND_PAIRING_INFO,
+			    pairing_info_out.header.status);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int
+gsc_hdcp_initiate_locality_check(struct device *dev,
+				 struct hdcp_port_data *data,
+				 struct hdcp2_lc_init *lc_init_data)
+{
+	struct wired_cmd_init_locality_check_in lc_init_in = { { 0 } };
+	struct wired_cmd_init_locality_check_out lc_init_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+
+	if (!dev || !data || !lc_init_data)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	lc_init_in.header.api_version = HDCP_API_VERSION;
+	lc_init_in.header.command_id = WIRED_INIT_LOCALITY_CHECK;
+	lc_init_in.header.status = FW_HDCP_STATUS_SUCCESS;
+	lc_init_in.header.buffer_len = WIRED_CMD_BUF_LEN_INIT_LOCALITY_CHECK_IN;
+
+	lc_init_in.port.integrated_port_type = data->port_type;
+	lc_init_in.port.physical_port = (u8)data->hdcp_ddi;
+	lc_init_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)&lc_init_in, sizeof(lc_init_in),
+				       (u8 *)&lc_init_out, sizeof(lc_init_out));
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
+		return byte;
+	}
+
+	if (lc_init_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "FW cmd 0x%08X Failed. status: 0x%X\n",
+			    WIRED_INIT_LOCALITY_CHECK, lc_init_out.header.status);
+		return -EIO;
+	}
+
+	lc_init_data->msg_id = HDCP_2_2_LC_INIT;
+	memcpy(lc_init_data->r_n, lc_init_out.r_n, HDCP_2_2_RN_LEN);
+
+	return 0;
+}
+
+static int
+gsc_hdcp_verify_lprime(struct device *dev, struct hdcp_port_data *data,
+		       struct hdcp2_lc_send_lprime *rx_lprime)
+{
+	struct wired_cmd_validate_locality_in verify_lprime_in = { { 0 } };
+	struct wired_cmd_validate_locality_out verify_lprime_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+
+	if (!dev || !data || !rx_lprime)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	verify_lprime_in.header.api_version = HDCP_API_VERSION;
+	verify_lprime_in.header.command_id = WIRED_VALIDATE_LOCALITY;
+	verify_lprime_in.header.status = FW_HDCP_STATUS_SUCCESS;
+	verify_lprime_in.header.buffer_len =
+					WIRED_CMD_BUF_LEN_VALIDATE_LOCALITY_IN;
+
+	verify_lprime_in.port.integrated_port_type = data->port_type;
+	verify_lprime_in.port.physical_port = (u8)data->hdcp_ddi;
+	verify_lprime_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
+
+	memcpy(verify_lprime_in.l_prime, rx_lprime->l_prime,
+	       HDCP_2_2_L_PRIME_LEN);
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)&verify_lprime_in,
+				       sizeof(verify_lprime_in),
+				       (u8 *)&verify_lprime_out,
+				       sizeof(verify_lprime_out));
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
+		return byte;
+	}
+
+	if (verify_lprime_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "FW cmd 0x%08X failed. status: 0x%X\n",
+			    WIRED_VALIDATE_LOCALITY,
+			    verify_lprime_out.header.status);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int gsc_hdcp_get_session_key(struct device *dev,
+				    struct hdcp_port_data *data,
+				    struct hdcp2_ske_send_eks *ske_data)
+{
+	struct wired_cmd_get_session_key_in get_skey_in = { { 0 } };
+	struct wired_cmd_get_session_key_out get_skey_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+
+	if (!dev || !data || !ske_data)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	get_skey_in.header.api_version = HDCP_API_VERSION;
+	get_skey_in.header.command_id = WIRED_GET_SESSION_KEY;
+	get_skey_in.header.status = FW_HDCP_STATUS_SUCCESS;
+	get_skey_in.header.buffer_len = WIRED_CMD_BUF_LEN_GET_SESSION_KEY_IN;
+
+	get_skey_in.port.integrated_port_type = data->port_type;
+	get_skey_in.port.physical_port = (u8)data->hdcp_ddi;
+	get_skey_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)&get_skey_in, sizeof(get_skey_in),
+				       (u8 *)&get_skey_out, sizeof(get_skey_out));
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
+		return byte;
+	}
+
+	if (get_skey_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "FW cmd 0x%08X failed. status: 0x%X\n",
+			    WIRED_GET_SESSION_KEY, get_skey_out.header.status);
+		return -EIO;
+	}
+
+	ske_data->msg_id = HDCP_2_2_SKE_SEND_EKS;
+	memcpy(ske_data->e_dkey_ks, get_skey_out.e_dkey_ks,
+	       HDCP_2_2_E_DKEY_KS_LEN);
+	memcpy(ske_data->riv, get_skey_out.r_iv, HDCP_2_2_RIV_LEN);
+
+	return 0;
+}
+
+static int
+gsc_hdcp_repeater_check_flow_prepare_ack(struct device *dev,
+					 struct hdcp_port_data *data,
+					 struct hdcp2_rep_send_receiverid_list
+							*rep_topology,
+					 struct hdcp2_rep_send_ack
+							*rep_send_ack)
+{
+	struct wired_cmd_verify_repeater_in verify_repeater_in = { { 0 } };
+	struct wired_cmd_verify_repeater_out verify_repeater_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+
+	if (!dev || !rep_topology || !rep_send_ack || !data)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	verify_repeater_in.header.api_version = HDCP_API_VERSION;
+	verify_repeater_in.header.command_id = WIRED_VERIFY_REPEATER;
+	verify_repeater_in.header.status = FW_HDCP_STATUS_SUCCESS;
+	verify_repeater_in.header.buffer_len =
+					WIRED_CMD_BUF_LEN_VERIFY_REPEATER_IN;
+
+	verify_repeater_in.port.integrated_port_type = data->port_type;
+	verify_repeater_in.port.physical_port = (u8)data->hdcp_ddi;
+	verify_repeater_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
+
+	memcpy(verify_repeater_in.rx_info, rep_topology->rx_info,
+	       HDCP_2_2_RXINFO_LEN);
+	memcpy(verify_repeater_in.seq_num_v, rep_topology->seq_num_v,
+	       HDCP_2_2_SEQ_NUM_LEN);
+	memcpy(verify_repeater_in.v_prime, rep_topology->v_prime,
+	       HDCP_2_2_V_PRIME_HALF_LEN);
+	memcpy(verify_repeater_in.receiver_ids, rep_topology->receiver_ids,
+	       HDCP_2_2_RECEIVER_IDS_MAX_LEN);
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)&verify_repeater_in,
+				       sizeof(verify_repeater_in),
+				       (u8 *)&verify_repeater_out,
+				       sizeof(verify_repeater_out));
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
+		return byte;
+	}
+
+	if (verify_repeater_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "FW cmd 0x%08X failed. status: 0x%X\n",
+			    WIRED_VERIFY_REPEATER,
+			    verify_repeater_out.header.status);
+		return -EIO;
+	}
+
+	memcpy(rep_send_ack->v, verify_repeater_out.v,
+	       HDCP_2_2_V_PRIME_HALF_LEN);
+	rep_send_ack->msg_id = HDCP_2_2_REP_SEND_ACK;
+
+	return 0;
+}
+
+static int gsc_hdcp_verify_mprime(struct device *dev,
+				  struct hdcp_port_data *data,
+				  struct hdcp2_rep_stream_ready *stream_ready)
+{
+	struct wired_cmd_repeater_auth_stream_req_in *verify_mprime_in;
+	struct wired_cmd_repeater_auth_stream_req_out
+					verify_mprime_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+	size_t cmd_size;
+
+	if (!dev || !stream_ready || !data)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	cmd_size = struct_size(verify_mprime_in, streams, data->k);
+	if (cmd_size == SIZE_MAX)
+		return -EINVAL;
+
+	verify_mprime_in = kzalloc(cmd_size, GFP_KERNEL);
+	if (!verify_mprime_in)
+		return -ENOMEM;
+
+	verify_mprime_in->header.api_version = HDCP_API_VERSION;
+	verify_mprime_in->header.command_id = WIRED_REPEATER_AUTH_STREAM_REQ;
+	verify_mprime_in->header.status = FW_HDCP_STATUS_SUCCESS;
+	verify_mprime_in->header.buffer_len = cmd_size  - sizeof(verify_mprime_in->header);
+
+	verify_mprime_in->port.integrated_port_type = data->port_type;
+	verify_mprime_in->port.physical_port = (u8)data->hdcp_ddi;
+	verify_mprime_in->port.attached_transcoder = (u8)data->hdcp_transcoder;
+
+	memcpy(verify_mprime_in->m_prime, stream_ready->m_prime, HDCP_2_2_MPRIME_LEN);
+	drm_hdcp_cpu_to_be24(verify_mprime_in->seq_num_m, data->seq_num_m);
+
+	memcpy(verify_mprime_in->streams, data->streams,
+	       array_size(data->k, sizeof(*data->streams)));
+
+	verify_mprime_in->k = cpu_to_be16(data->k);
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)verify_mprime_in, cmd_size,
+				       (u8 *)&verify_mprime_out,
+				       sizeof(verify_mprime_out));
+	kfree(verify_mprime_in);
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
+		return byte;
+	}
+
+	if (verify_mprime_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "FW cmd 0x%08X failed. status: 0x%X\n",
+			    WIRED_REPEATER_AUTH_STREAM_REQ,
+			    verify_mprime_out.header.status);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int gsc_hdcp_enable_authentication(struct device *dev,
+					  struct hdcp_port_data *data)
+{
+	struct wired_cmd_enable_auth_in enable_auth_in = { { 0 } };
+	struct wired_cmd_enable_auth_out enable_auth_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+
+	if (!dev || !data)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	enable_auth_in.header.api_version = HDCP_API_VERSION;
+	enable_auth_in.header.command_id = WIRED_ENABLE_AUTH;
+	enable_auth_in.header.status = FW_HDCP_STATUS_SUCCESS;
+	enable_auth_in.header.buffer_len = WIRED_CMD_BUF_LEN_ENABLE_AUTH_IN;
+
+	enable_auth_in.port.integrated_port_type = data->port_type;
+	enable_auth_in.port.physical_port = (u8)data->hdcp_ddi;
+	enable_auth_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
+	enable_auth_in.stream_type = data->streams[0].stream_type;
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)&enable_auth_in,
+				       sizeof(enable_auth_in),
+				       (u8 *)&enable_auth_out,
+				       sizeof(enable_auth_out));
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
+		return byte;
+	}
+
+	if (enable_auth_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "FW cmd 0x%08X failed. status: 0x%X\n",
+			    WIRED_ENABLE_AUTH, enable_auth_out.header.status);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int
+gsc_hdcp_close_session(struct device *dev, struct hdcp_port_data *data)
+{
+	struct wired_cmd_close_session_in session_close_in = { { 0 } };
+	struct wired_cmd_close_session_out session_close_out = { { 0 } };
+	struct drm_i915_private *i915;
+	ssize_t byte;
+
+	if (!dev || !data)
+		return -EINVAL;
+
+	i915 = kdev_to_i915(dev);
+	if (!i915) {
+		dev_err(dev, "DRM not initialized, aborting HDCP.\n");
+		return -ENODEV;
+	}
+
+	session_close_in.header.api_version = HDCP_API_VERSION;
+	session_close_in.header.command_id = WIRED_CLOSE_SESSION;
+	session_close_in.header.status = FW_HDCP_STATUS_SUCCESS;
+	session_close_in.header.buffer_len =
+				WIRED_CMD_BUF_LEN_CLOSE_SESSION_IN;
+
+	session_close_in.port.integrated_port_type = data->port_type;
+	session_close_in.port.physical_port = (u8)data->hdcp_ddi;
+	session_close_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
+
+	byte = intel_hdcp_gsc_msg_send(i915, (u8 *)&session_close_in,
+				       sizeof(session_close_in),
+				       (u8 *)&session_close_out,
+				       sizeof(session_close_out));
+	if (byte < 0) {
+		drm_dbg_kms(&i915->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
+		return byte;
+	}
+
+	if (session_close_out.header.status != FW_HDCP_STATUS_SUCCESS) {
+		drm_dbg_kms(&i915->drm, "Session Close Failed. status: 0x%X\n",
+			    session_close_out.header.status);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static const struct i915_hdcp_ops gsc_hdcp_ops = {
+	.initiate_hdcp2_session = gsc_hdcp_initiate_session,
+	.verify_receiver_cert_prepare_km =
+				gsc_hdcp_verify_receiver_cert_prepare_km,
+	.verify_hprime = gsc_hdcp_verify_hprime,
+	.store_pairing_info = gsc_hdcp_store_pairing_info,
+	.initiate_locality_check = gsc_hdcp_initiate_locality_check,
+	.verify_lprime = gsc_hdcp_verify_lprime,
+	.get_session_key = gsc_hdcp_get_session_key,
+	.repeater_check_flow_prepare_ack =
+				gsc_hdcp_repeater_check_flow_prepare_ack,
+	.verify_mprime = gsc_hdcp_verify_mprime,
+	.enable_hdcp_authentication = gsc_hdcp_enable_authentication,
+	.close_hdcp_session = gsc_hdcp_close_session,
+};
+
+/*This function helps allocate memory for the command that we will send to gsc cs */
+static int intel_hdcp_gsc_initialize_message(struct drm_i915_private *i915,
+					     struct intel_hdcp_gsc_message *hdcp_message)
+{
+	struct intel_gt *gt = i915->media_gt;
+	struct drm_i915_gem_object *obj = NULL;
+	struct i915_vma *vma = NULL;
+	void *cmd;
+	int err;
+
+	/* allocate object of one page for HDCP command memory and store it */
+	obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
+
+	if (IS_ERR(obj)) {
+		drm_err(&i915->drm, "Failed to allocate HDCP streaming command!\n");
+		return PTR_ERR(obj);
+	}
+
+	cmd = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(i915, obj, true));
+	if (IS_ERR(cmd)) {
+		drm_err(&i915->drm, "Failed to map gsc message page!\n");
+		err = PTR_ERR(cmd);
+		goto out_unpin;
+	}
+
+	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto out_unmap;
+	}
+
+	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
+	if (err)
+		goto out_unmap;
+
+	memset(cmd, 0, obj->base.size);
+
+	hdcp_message->hdcp_cmd = cmd;
+	hdcp_message->vma = vma;
+
+	return 0;
+
+out_unmap:
+	i915_gem_object_unpin_map(obj);
+out_unpin:
+	i915_gem_object_put(obj);
+	return err;
+}
+
+static int intel_hdcp_gsc_hdcp2_init(struct drm_i915_private *i915)
+{
+	struct intel_hdcp_gsc_message *hdcp_message;
+	int ret;
+
+	hdcp_message = kzalloc(sizeof(*hdcp_message), GFP_KERNEL);
+
+	if (!hdcp_message)
+		return -ENOMEM;
+
+	/*
+	 * NOTE: No need to lock the comp mutex here as it is already
+	 * going to be taken before this function called
+	 */
+	i915->display.hdcp.hdcp_message = hdcp_message;
+	ret = intel_hdcp_gsc_initialize_message(i915, hdcp_message);
+
+	if (ret)
+		drm_err(&i915->drm, "Could not initialize hdcp_message\n");
+
+	return ret;
+}
+
+static void intel_hdcp_gsc_free_message(struct drm_i915_private *i915)
+{
+	struct intel_hdcp_gsc_message *hdcp_message =
+					i915->display.hdcp.hdcp_message;
+
+	i915_vma_unpin_and_release(&hdcp_message->vma, I915_VMA_RELEASE_MAP);
+	kfree(hdcp_message);
+}
+
+int intel_hdcp_gsc_init(struct drm_i915_private *i915)
+{
+	struct i915_hdcp_master *data;
+	int ret;
+
+	data = kzalloc(sizeof(struct i915_hdcp_master), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	mutex_lock(&i915->display.hdcp.comp_mutex);
+	i915->display.hdcp.master = data;
+	i915->display.hdcp.master->hdcp_dev = i915->drm.dev;
+	i915->display.hdcp.master->ops = &gsc_hdcp_ops;
+	ret = intel_hdcp_gsc_hdcp2_init(i915);
+	mutex_unlock(&i915->display.hdcp.comp_mutex);
+
+	return ret;
+}
+
+void intel_hdcp_gsc_fini(struct drm_i915_private *i915)
+{
+	intel_hdcp_gsc_free_message(i915);
+	kfree(i915->display.hdcp.master);
+}
+
+static int intel_gsc_send_sync(struct drm_i915_private *i915,
+			       struct intel_gsc_mtl_header *header, u64 addr,
+			       size_t msg_out_len)
+{
+	struct intel_gt *gt = i915->media_gt;
+	int ret;
+
+	header->flags = 0;
+	ret = intel_gsc_uc_heci_cmd_submit_packet(&gt->uc.gsc, addr,
+						  header->message_size,
+						  addr,
+						  msg_out_len + sizeof(*header));
+	if (ret) {
+		drm_err(&i915->drm, "failed to send gsc HDCP msg (%d)\n", ret);
+		return ret;
+	}
+
+	/*
+	 * Checking validity marker for memory sanity
+	 */
+	if (header->validity_marker != GSC_HECI_VALIDITY_MARKER) {
+		drm_err(&i915->drm, "invalid validity marker\n");
+		return -EINVAL;
+	}
+
+	if (header->status != 0) {
+		drm_err(&i915->drm, "header status indicates error %d\n",
+			header->status);
+		return -EINVAL;
+	}
+
+	if (header->flags & GSC_OUTFLAG_MSG_PENDING)
+		return -EAGAIN;
+
+	return 0;
+}
+
+/*
+ * This function can now be used for sending requests and will also handle
+ * receipt of reply messages hence no different function of message retrieval
+ * is required. We will initialize intel_hdcp_gsc_message structure then add
+ * gsc cs memory header as stated in specs after which the normal HDCP payload
+ * will follow
+ */
+ssize_t intel_hdcp_gsc_msg_send(struct drm_i915_private *i915, u8 *msg_in,
+				size_t msg_in_len, u8 *msg_out,
+				size_t msg_out_len)
+{
+	struct intel_gt *gt = i915->media_gt;
+	struct intel_gsc_mtl_header *header;
+	const size_t max_msg_size = PAGE_SIZE - sizeof(*header);
+	struct intel_hdcp_gsc_message *hdcp_message;
+	u64 addr, host_session_id;
+	u32 reply_size, msg_size;
+	int ret, tries = 0;
+
+	if (!intel_uc_uses_gsc_uc(&gt->uc))
+		return -ENODEV;
+
+	if (msg_in_len > max_msg_size || msg_out_len > max_msg_size)
+		return -ENOSPC;
+
+	hdcp_message = i915->display.hdcp.hdcp_message;
+	header = hdcp_message->hdcp_cmd;
+	addr = i915_ggtt_offset(hdcp_message->vma);
+
+	msg_size = msg_in_len + sizeof(*header);
+	memset(header, 0, msg_size);
+	get_random_bytes(&host_session_id, sizeof(u64));
+	intel_gsc_uc_heci_cmd_emit_mtl_header(header, HECI_MEADDRESS_HDCP,
+					      msg_size, host_session_id);
+	memcpy(hdcp_message->hdcp_cmd + sizeof(*header), msg_in, msg_in_len);
+
+	/*
+	 * Keep sending request in case the pending bit is set no need to add
+	 * message handle as we are using same address hence loc. of header is
+	 * same and it will contain the message handle. we will send the message
+	 * 20 times each message 50 ms apart
+	 */
+	do {
+		ret = intel_gsc_send_sync(i915, header, addr, msg_out_len);
+
+		/* Only try again if gsc says so */
+		if (ret != -EAGAIN)
+			break;
+
+		msleep(50);
+
+	} while (++tries < 20);
+
+	if (ret)
+		goto err;
+
+	/* we use the same mem for the reply, so header is in the same loc */
+	reply_size = header->message_size - sizeof(*header);
+	if (reply_size > msg_out_len) {
+		drm_warn(&i915->drm, "caller with insufficient HDCP reply size %u (%d)\n",
+			 reply_size, (u32)msg_out_len);
+		reply_size = msg_out_len;
+	} else if (reply_size != msg_out_len) {
+		drm_dbg_kms(&i915->drm, "caller unexpected HCDP reply size %u (%d)\n",
+			    reply_size, (u32)msg_out_len);
+	}
+
+	memcpy(msg_out, hdcp_message->hdcp_cmd + sizeof(*header), msg_out_len);
+
+err:
+	return ret;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp_gsc.h b/drivers/gpu/drm/i915/display/intel_hdcp_gsc.h
new file mode 100644
index 000000000000..5cc9fd2e88f6
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_hdcp_gsc.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_HDCP_GSC_H__
+#define __INTEL_HDCP_GSC_H__
+
+#include <linux/err.h>
+#include <linux/types.h>
+
+struct drm_i915_private;
+
+struct intel_hdcp_gsc_message {
+	struct i915_vma *vma;
+	void *hdcp_cmd;
+};
+
+bool intel_hdcp_gsc_cs_required(struct drm_i915_private *i915);
+ssize_t intel_hdcp_gsc_msg_send(struct drm_i915_private *i915, u8 *msg_in,
+				size_t msg_in_len, u8 *msg_out,
+				size_t msg_out_len);
+int intel_hdcp_gsc_init(struct drm_i915_private *i915);
+void intel_hdcp_gsc_fini(struct drm_i915_private *i915);
+
+#endif /* __INTEL_HDCP_GCS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index c0ce6d3dc505..a690a5616506 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -238,15 +238,11 @@ static void g4x_read_infoframe(struct intel_encoder *encoder,
 			       void *frame, ssize_t len)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	u32 val, *data = frame;
+	u32 *data = frame;
 	int i;
 
-	val = intel_de_read(dev_priv, VIDEO_DIP_CTL);
-
-	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
-	val |= g4x_infoframe_index(type);
-
-	intel_de_write(dev_priv, VIDEO_DIP_CTL, val);
+	intel_de_rmw(dev_priv, VIDEO_DIP_CTL,
+		     VIDEO_DIP_SELECT_MASK | 0xf, g4x_infoframe_index(type));
 
 	for (i = 0; i < len; i += 4)
 		*data++ = intel_de_read(dev_priv, VIDEO_DIP_DATA);
@@ -314,15 +310,11 @@ static void ibx_read_infoframe(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	u32 val, *data = frame;
+	u32 *data = frame;
 	int i;
 
-	val = intel_de_read(dev_priv, TVIDEO_DIP_CTL(crtc->pipe));
-
-	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
-	val |= g4x_infoframe_index(type);
-
-	intel_de_write(dev_priv, TVIDEO_DIP_CTL(crtc->pipe), val);
+	intel_de_rmw(dev_priv, TVIDEO_DIP_CTL(crtc->pipe),
+		     VIDEO_DIP_SELECT_MASK | 0xf, g4x_infoframe_index(type));
 
 	for (i = 0; i < len; i += 4)
 		*data++ = intel_de_read(dev_priv, TVIDEO_DIP_DATA(crtc->pipe));
@@ -396,15 +388,11 @@ static void cpt_read_infoframe(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	u32 val, *data = frame;
+	u32 *data = frame;
 	int i;
 
-	val = intel_de_read(dev_priv, TVIDEO_DIP_CTL(crtc->pipe));
-
-	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
-	val |= g4x_infoframe_index(type);
-
-	intel_de_write(dev_priv, TVIDEO_DIP_CTL(crtc->pipe), val);
+	intel_de_rmw(dev_priv, TVIDEO_DIP_CTL(crtc->pipe),
+		     VIDEO_DIP_SELECT_MASK | 0xf, g4x_infoframe_index(type));
 
 	for (i = 0; i < len; i += 4)
 		*data++ = intel_de_read(dev_priv, TVIDEO_DIP_DATA(crtc->pipe));
@@ -472,15 +460,11 @@ static void vlv_read_infoframe(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	u32 val, *data = frame;
+	u32 *data = frame;
 	int i;
 
-	val = intel_de_read(dev_priv, VLV_TVIDEO_DIP_CTL(crtc->pipe));
-
-	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
-	val |= g4x_infoframe_index(type);
-
-	intel_de_write(dev_priv, VLV_TVIDEO_DIP_CTL(crtc->pipe), val);
+	intel_de_rmw(dev_priv, VLV_TVIDEO_DIP_CTL(crtc->pipe),
+		     VIDEO_DIP_SELECT_MASK | 0xf, g4x_infoframe_index(type));
 
 	for (i = 0; i < len; i += 4)
 		*data++ = intel_de_read(dev_priv,
@@ -1795,7 +1779,7 @@ static int intel_hdmi_source_max_tmds_clock(struct intel_encoder *encoder)
 	else
 		max_tmds_clock = 165000;
 
-	vbt_max_tmds_clock = intel_bios_max_tmds_clock(encoder);
+	vbt_max_tmds_clock = intel_bios_hdmi_max_tmds_clock(encoder->devdata);
 	if (vbt_max_tmds_clock)
 		max_tmds_clock = min(max_tmds_clock, vbt_max_tmds_clock);
 
@@ -2152,7 +2136,7 @@ bool intel_hdmi_limited_color_range(const struct intel_crtc_state *crtc_state,
 	 * Our YCbCr output is always limited range.
 	 * crtc_state->limited_color_range only applies to RGB,
 	 * and it must never be set for YCbCr or we risk setting
-	 * some conflicting bits in PIPECONF which will mess up
+	 * some conflicting bits in TRANSCONF which will mess up
 	 * the colors on the monitor.
 	 */
 	if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
@@ -2240,6 +2224,25 @@ static bool intel_hdmi_is_cloned(const struct intel_crtc_state *crtc_state)
 		!is_power_of_2(crtc_state->uapi.encoder_mask);
 }
 
+static bool source_supports_scrambling(struct intel_encoder *encoder)
+{
+	/*
+	 * Gen 10+ support HDMI 2.0 : the max tmds clock is 594MHz, and
+	 * scrambling is supported.
+	 * But there seem to be cases where certain platforms that support
+	 * HDMI 2.0, have an HDMI1.4 retimer chip, and the max tmds clock is
+	 * capped by VBT to less than 340MHz.
+	 *
+	 * In such cases when an HDMI2.0 sink is connected, it creates a
+	 * problem : the platform and the sink both support scrambling but the
+	 * HDMI 1.4 retimer chip doesn't.
+	 *
+	 * So go for scrambling, based on the max tmds clock taking into account,
+	 * restrictions coming from VBT.
+	 */
+	return intel_hdmi_source_max_tmds_clock(encoder) > 340000;
+}
+
 int intel_hdmi_compute_config(struct intel_encoder *encoder,
 			      struct intel_crtc_state *pipe_config,
 			      struct drm_connector_state *conn_state)
@@ -2302,7 +2305,7 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder,
 
 	pipe_config->lane_count = 4;
 
-	if (scdc->scrambling.supported && DISPLAY_VER(dev_priv) >= 10) {
+	if (scdc->scrambling.supported && source_supports_scrambling(encoder)) {
 		if (scdc->scrambling.low_rates)
 			pipe_config->hdmi_scrambling = true;
 
@@ -2643,11 +2646,8 @@ bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder,
 				       bool scrambling)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
 	struct drm_scrambling *sink_scrambling =
 		&connector->display_info.hdmi.scdc.scrambling;
-	struct i2c_adapter *adapter =
-		intel_gmbus_get_adapter(dev_priv, intel_hdmi->ddc_bus);
 
 	if (!sink_scrambling->supported)
 		return true;
@@ -2658,9 +2658,8 @@ bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder,
 		    str_yes_no(scrambling), high_tmds_clock_ratio ? 40 : 10);
 
 	/* Set TMDS bit clock ratio to 1/40 or 1/10, and enable/disable scrambling */
-	return drm_scdc_set_high_tmds_clock_ratio(adapter,
-						  high_tmds_clock_ratio) &&
-		drm_scdc_set_scrambling(adapter, scrambling);
+	return drm_scdc_set_high_tmds_clock_ratio(connector, high_tmds_clock_ratio) &&
+		drm_scdc_set_scrambling(connector, scrambling);
 }
 
 static u8 chv_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
@@ -2852,11 +2851,12 @@ static u8 intel_hdmi_ddc_pin(struct intel_encoder *encoder)
 	enum port port = encoder->port;
 	u8 ddc_pin;
 
-	ddc_pin = intel_bios_alternate_ddc_pin(encoder);
+	ddc_pin = intel_bios_hdmi_ddc_pin(encoder->devdata);
 	if (ddc_pin) {
 		drm_dbg_kms(&dev_priv->drm,
-			    "Using DDC pin 0x%x for port %c (VBT)\n",
-			    ddc_pin, port_name(port));
+			    "[ENCODER:%d:%s] Using DDC pin 0x%x (VBT)\n",
+			    encoder->base.base.id, encoder->base.name,
+			    ddc_pin);
 		return ddc_pin;
 	}
 
@@ -2882,8 +2882,9 @@ static u8 intel_hdmi_ddc_pin(struct intel_encoder *encoder)
 		ddc_pin = g4x_port_to_ddc_pin(dev_priv, port);
 
 	drm_dbg_kms(&dev_priv->drm,
-		    "Using DDC pin 0x%x for port %c (platform default)\n",
-		    ddc_pin, port_name(port));
+		    "[ENCODER:%d:%s] Using DDC pin 0x%x (platform default)\n",
+		    encoder->base.base.id, encoder->base.name,
+		    ddc_pin);
 
 	return ddc_pin;
 }
@@ -2904,7 +2905,7 @@ void intel_infoframe_init(struct intel_digital_port *dig_port)
 		dig_port->set_infoframes = g4x_set_infoframes;
 		dig_port->infoframes_enabled = g4x_infoframes_enabled;
 	} else if (HAS_DDI(dev_priv)) {
-		if (intel_bios_is_lspcon_present(dev_priv, dig_port->base.port)) {
+		if (intel_bios_encoder_is_lspcon(dig_port->base.devdata)) {
 			dig_port->write_infoframe = lspcon_write_infoframe;
 			dig_port->read_infoframe = lspcon_read_infoframe;
 			dig_port->set_infoframes = lspcon_set_infoframes;
diff --git a/drivers/gpu/drm/i915/display/intel_hotplug.c b/drivers/gpu/drm/i915/display/intel_hotplug.c
index 907ab7526cb4..b12900446828 100644
--- a/drivers/gpu/drm/i915/display/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/display/intel_hotplug.c
@@ -389,6 +389,13 @@ static void i915_hotplug_work_func(struct work_struct *work)
 
 	spin_unlock_irq(&dev_priv->irq_lock);
 
+	/* Skip calling encode hotplug handlers if ignore long HPD set*/
+	if (dev_priv->display.hotplug.ignore_long_hpd) {
+		drm_dbg_kms(&dev_priv->drm, "Ignore HPD flag on - skip encoder hotplug handlers\n");
+		mutex_unlock(&dev_priv->drm.mode_config.mutex);
+		return;
+	}
+
 	drm_connector_list_iter_begin(&dev_priv->drm, &conn_iter);
 	for_each_intel_connector_iter(connector, &conn_iter) {
 		enum hpd_pin pin;
@@ -940,4 +947,6 @@ void intel_hpd_debugfs_register(struct drm_i915_private *i915)
 			    i915, &i915_hpd_storm_ctl_fops);
 	debugfs_create_file("i915_hpd_short_storm_ctl", 0644, minor->debugfs_root,
 			    i915, &i915_hpd_short_storm_ctl_fops);
+	debugfs_create_bool("i915_ignore_long_hpd", 0644, minor->debugfs_root,
+			    &i915->display.hotplug.ignore_long_hpd);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_lpe_audio.c b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
index 8aaaef4d7856..5863763de530 100644
--- a/drivers/gpu/drm/i915/display/intel_lpe_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
@@ -315,7 +315,7 @@ void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv)
  * intel_lpe_audio_notify() - notify lpe audio event
  * audio driver and i915
  * @dev_priv: the i915 drm device private data
- * @pipe: pipe
+ * @cpu_transcoder: CPU transcoder
  * @port: port
  * @eld : ELD data
  * @ls_clock: Link symbol clock in kHz
@@ -324,7 +324,7 @@ void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv)
  * Notify lpe audio driver of eld change.
  */
 void intel_lpe_audio_notify(struct drm_i915_private *dev_priv,
-			    enum pipe pipe, enum port port,
+			    enum transcoder cpu_transcoder, enum port port,
 			    const void *eld, int ls_clock, bool dp_output)
 {
 	unsigned long irqflags;
@@ -344,7 +344,7 @@ void intel_lpe_audio_notify(struct drm_i915_private *dev_priv,
 
 	if (eld != NULL) {
 		memcpy(ppdata->eld, eld, HDMI_MAX_ELD_BYTES);
-		ppdata->pipe = pipe;
+		ppdata->pipe = cpu_transcoder;
 		ppdata->ls_clock = ls_clock;
 		ppdata->dp_output = dp_output;
 
diff --git a/drivers/gpu/drm/i915/display/intel_lpe_audio.h b/drivers/gpu/drm/i915/display/intel_lpe_audio.h
index f848c5038714..0beecac267ae 100644
--- a/drivers/gpu/drm/i915/display/intel_lpe_audio.h
+++ b/drivers/gpu/drm/i915/display/intel_lpe_audio.h
@@ -8,15 +8,15 @@
 
 #include <linux/types.h>
 
-enum pipe;
 enum port;
+enum transcoder;
 struct drm_i915_private;
 
 int  intel_lpe_audio_init(struct drm_i915_private *dev_priv);
 void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv);
 void intel_lpe_audio_irq_handler(struct drm_i915_private *dev_priv);
 void intel_lpe_audio_notify(struct drm_i915_private *dev_priv,
-			    enum pipe pipe, enum port port,
+			    enum transcoder cpu_transcoder, enum port port,
 			    const void *eld, int ls_clock, bool dp_output);
 
 #endif /* __INTEL_LPE_AUDIO_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_lspcon.c b/drivers/gpu/drm/i915/display/intel_lspcon.c
index 9ff1c0b223ad..bb3b5355a0d9 100644
--- a/drivers/gpu/drm/i915/display/intel_lspcon.c
+++ b/drivers/gpu/drm/i915/display/intel_lspcon.c
@@ -689,7 +689,7 @@ void lspcon_resume(struct intel_digital_port *dig_port)
 	struct drm_i915_private *i915 = to_i915(dev);
 	enum drm_lspcon_mode expected_mode;
 
-	if (!intel_bios_is_lspcon_present(i915, dig_port->base.port))
+	if (!intel_bios_encoder_is_lspcon(dig_port->base.devdata))
 		return;
 
 	if (!lspcon->active) {
diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c
index a1557d84ce0a..0de44b3631cd 100644
--- a/drivers/gpu/drm/i915/display/intel_lvds.c
+++ b/drivers/gpu/drm/i915/display/intel_lvds.c
@@ -49,7 +49,9 @@
 #include "intel_fdi.h"
 #include "intel_gmbus.h"
 #include "intel_lvds.h"
+#include "intel_lvds_regs.h"
 #include "intel_panel.h"
+#include "intel_pps_regs.h"
 
 /* Private structure for the integrated LVDS support */
 struct intel_lvds_pps {
@@ -84,18 +86,18 @@ static struct intel_lvds_encoder *to_lvds_encoder(struct intel_encoder *encoder)
 	return container_of(encoder, struct intel_lvds_encoder, base);
 }
 
-bool intel_lvds_port_enabled(struct drm_i915_private *dev_priv,
+bool intel_lvds_port_enabled(struct drm_i915_private *i915,
 			     i915_reg_t lvds_reg, enum pipe *pipe)
 {
 	u32 val;
 
-	val = intel_de_read(dev_priv, lvds_reg);
+	val = intel_de_read(i915, lvds_reg);
 
 	/* asserts want to know the pipe even if the port is disabled */
-	if (HAS_PCH_CPT(dev_priv))
-		*pipe = (val & LVDS_PIPE_SEL_MASK_CPT) >> LVDS_PIPE_SEL_SHIFT_CPT;
+	if (HAS_PCH_CPT(i915))
+		*pipe = REG_FIELD_GET(LVDS_PIPE_SEL_MASK_CPT, val);
 	else
-		*pipe = (val & LVDS_PIPE_SEL_MASK) >> LVDS_PIPE_SEL_SHIFT;
+		*pipe = REG_FIELD_GET(LVDS_PIPE_SEL_MASK, val);
 
 	return val & LVDS_PORT_EN;
 }
@@ -103,31 +105,30 @@ bool intel_lvds_port_enabled(struct drm_i915_private *dev_priv,
 static bool intel_lvds_get_hw_state(struct intel_encoder *encoder,
 				    enum pipe *pipe)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(encoder);
 	intel_wakeref_t wakeref;
 	bool ret;
 
-	wakeref = intel_display_power_get_if_enabled(dev_priv,
-						     encoder->power_domain);
+	wakeref = intel_display_power_get_if_enabled(i915, encoder->power_domain);
 	if (!wakeref)
 		return false;
 
-	ret = intel_lvds_port_enabled(dev_priv, lvds_encoder->reg, pipe);
+	ret = intel_lvds_port_enabled(i915, lvds_encoder->reg, pipe);
 
-	intel_display_power_put(dev_priv, encoder->power_domain, wakeref);
+	intel_display_power_put(i915, encoder->power_domain, wakeref);
 
 	return ret;
 }
 
 static void intel_lvds_get_config(struct intel_encoder *encoder,
-				  struct intel_crtc_state *pipe_config)
+				  struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(encoder);
 	u32 tmp, flags = 0;
 
-	pipe_config->output_types |= BIT(INTEL_OUTPUT_LVDS);
+	crtc_state->output_types |= BIT(INTEL_OUTPUT_LVDS);
 
 	tmp = intel_de_read(dev_priv, lvds_encoder->reg);
 	if (tmp & LVDS_HSYNC_POLARITY)
@@ -139,20 +140,20 @@ static void intel_lvds_get_config(struct intel_encoder *encoder,
 	else
 		flags |= DRM_MODE_FLAG_PVSYNC;
 
-	pipe_config->hw.adjusted_mode.flags |= flags;
+	crtc_state->hw.adjusted_mode.flags |= flags;
 
 	if (DISPLAY_VER(dev_priv) < 5)
-		pipe_config->gmch_pfit.lvds_border_bits =
+		crtc_state->gmch_pfit.lvds_border_bits =
 			tmp & LVDS_BORDER_ENABLE;
 
 	/* gen2/3 store dither state in pfit control, needs to match */
 	if (DISPLAY_VER(dev_priv) < 4) {
 		tmp = intel_de_read(dev_priv, PFIT_CONTROL);
 
-		pipe_config->gmch_pfit.control |= tmp & PANEL_8TO6_DITHER_ENABLE;
+		crtc_state->gmch_pfit.control |= tmp & PANEL_8TO6_DITHER_ENABLE;
 	}
 
-	pipe_config->hw.adjusted_mode.crtc_clock = pipe_config->port_clock;
+	crtc_state->hw.adjusted_mode.crtc_clock = crtc_state->port_clock;
 }
 
 static void intel_lvds_pps_get_hw_state(struct drm_i915_private *dev_priv,
@@ -216,41 +217,44 @@ static void intel_lvds_pps_init_hw(struct drm_i915_private *dev_priv,
 	intel_de_write(dev_priv, PP_CONTROL(0), val);
 
 	intel_de_write(dev_priv, PP_ON_DELAYS(0),
-		       REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, pps->port) | REG_FIELD_PREP(PANEL_POWER_UP_DELAY_MASK, pps->t1_t2) | REG_FIELD_PREP(PANEL_LIGHT_ON_DELAY_MASK, pps->t5));
+		       REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, pps->port) |
+		       REG_FIELD_PREP(PANEL_POWER_UP_DELAY_MASK, pps->t1_t2) |
+		       REG_FIELD_PREP(PANEL_LIGHT_ON_DELAY_MASK, pps->t5));
 
 	intel_de_write(dev_priv, PP_OFF_DELAYS(0),
-		       REG_FIELD_PREP(PANEL_POWER_DOWN_DELAY_MASK, pps->t3) | REG_FIELD_PREP(PANEL_LIGHT_OFF_DELAY_MASK, pps->tx));
+		       REG_FIELD_PREP(PANEL_POWER_DOWN_DELAY_MASK, pps->t3) |
+		       REG_FIELD_PREP(PANEL_LIGHT_OFF_DELAY_MASK, pps->tx));
 
 	intel_de_write(dev_priv, PP_DIVISOR(0),
-		       REG_FIELD_PREP(PP_REFERENCE_DIVIDER_MASK, pps->divider) | REG_FIELD_PREP(PANEL_POWER_CYCLE_DELAY_MASK, DIV_ROUND_UP(pps->t4, 1000) + 1));
+		       REG_FIELD_PREP(PP_REFERENCE_DIVIDER_MASK, pps->divider) |
+		       REG_FIELD_PREP(PANEL_POWER_CYCLE_DELAY_MASK, DIV_ROUND_UP(pps->t4, 1000) + 1));
 }
 
 static void intel_pre_enable_lvds(struct intel_atomic_state *state,
 				  struct intel_encoder *encoder,
-				  const struct intel_crtc_state *pipe_config,
+				  const struct intel_crtc_state *crtc_state,
 				  const struct drm_connector_state *conn_state)
 {
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(encoder);
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
-	const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 	enum pipe pipe = crtc->pipe;
 	u32 temp;
 
-	if (HAS_PCH_SPLIT(dev_priv)) {
-		assert_fdi_rx_pll_disabled(dev_priv, pipe);
-		assert_shared_dpll_disabled(dev_priv,
-					    pipe_config->shared_dpll);
+	if (HAS_PCH_SPLIT(i915)) {
+		assert_fdi_rx_pll_disabled(i915, pipe);
+		assert_shared_dpll_disabled(i915, crtc_state->shared_dpll);
 	} else {
-		assert_pll_disabled(dev_priv, pipe);
+		assert_pll_disabled(i915, pipe);
 	}
 
-	intel_lvds_pps_init_hw(dev_priv, &lvds_encoder->init_pps);
+	intel_lvds_pps_init_hw(i915, &lvds_encoder->init_pps);
 
 	temp = lvds_encoder->init_lvds_val;
 	temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
 
-	if (HAS_PCH_CPT(dev_priv)) {
+	if (HAS_PCH_CPT(i915)) {
 		temp &= ~LVDS_PIPE_SEL_MASK_CPT;
 		temp |= LVDS_PIPE_SEL_CPT(pipe);
 	} else {
@@ -260,7 +264,7 @@ static void intel_pre_enable_lvds(struct intel_atomic_state *state,
 
 	/* set the corresponsding LVDS_BORDER bit */
 	temp &= ~LVDS_BORDER_ENABLE;
-	temp |= pipe_config->gmch_pfit.lvds_border_bits;
+	temp |= crtc_state->gmch_pfit.lvds_border_bits;
 
 	/*
 	 * Set the B0-B3 data pairs corresponding to whether we're going to
@@ -283,14 +287,14 @@ static void intel_pre_enable_lvds(struct intel_atomic_state *state,
 	/*
 	 * Set the dithering flag on LVDS as needed, note that there is no
 	 * special lvds dither control bit on pch-split platforms, dithering is
-	 * only controlled through the PIPECONF reg.
+	 * only controlled through the TRANSCONF reg.
 	 */
-	if (DISPLAY_VER(dev_priv) == 4) {
+	if (DISPLAY_VER(i915) == 4) {
 		/*
 		 * Bspec wording suggests that LVDS port dithering only exists
 		 * for 18bpp panels.
 		 */
-		if (pipe_config->dither && pipe_config->pipe_bpp == 18)
+		if (crtc_state->dither && crtc_state->pipe_bpp == 18)
 			temp |= LVDS_ENABLE_DITHER;
 		else
 			temp &= ~LVDS_ENABLE_DITHER;
@@ -301,7 +305,7 @@ static void intel_pre_enable_lvds(struct intel_atomic_state *state,
 	if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
 		temp |= LVDS_VSYNC_POLARITY;
 
-	intel_de_write(dev_priv, lvds_encoder->reg, temp);
+	intel_de_write(i915, lvds_encoder->reg, temp);
 }
 
 /*
@@ -309,25 +313,22 @@ static void intel_pre_enable_lvds(struct intel_atomic_state *state,
  */
 static void intel_enable_lvds(struct intel_atomic_state *state,
 			      struct intel_encoder *encoder,
-			      const struct intel_crtc_state *pipe_config,
+			      const struct intel_crtc_state *crtc_state,
 			      const struct drm_connector_state *conn_state)
 {
-	struct drm_device *dev = encoder->base.dev;
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(encoder);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	intel_de_write(dev_priv, lvds_encoder->reg,
-		       intel_de_read(dev_priv, lvds_encoder->reg) | LVDS_PORT_EN);
+	intel_de_rmw(dev_priv, lvds_encoder->reg, 0, LVDS_PORT_EN);
 
-	intel_de_write(dev_priv, PP_CONTROL(0),
-		       intel_de_read(dev_priv, PP_CONTROL(0)) | PANEL_POWER_ON);
+	intel_de_rmw(dev_priv, PP_CONTROL(0), 0, PANEL_POWER_ON);
 	intel_de_posting_read(dev_priv, lvds_encoder->reg);
 
 	if (intel_de_wait_for_set(dev_priv, PP_STATUS(0), PP_ON, 5000))
 		drm_err(&dev_priv->drm,
 			"timed out waiting for panel to power on\n");
 
-	intel_backlight_enable(pipe_config, conn_state);
+	intel_backlight_enable(crtc_state, conn_state);
 }
 
 static void intel_disable_lvds(struct intel_atomic_state *state,
@@ -338,14 +339,12 @@ static void intel_disable_lvds(struct intel_atomic_state *state,
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(encoder);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	intel_de_write(dev_priv, PP_CONTROL(0),
-		       intel_de_read(dev_priv, PP_CONTROL(0)) & ~PANEL_POWER_ON);
+	intel_de_rmw(dev_priv, PP_CONTROL(0), PANEL_POWER_ON, 0);
 	if (intel_de_wait_for_clear(dev_priv, PP_STATUS(0), PP_ON, 1000))
 		drm_err(&dev_priv->drm,
 			"timed out waiting for panel to power off\n");
 
-	intel_de_write(dev_priv, lvds_encoder->reg,
-		       intel_de_read(dev_priv, lvds_encoder->reg) & ~LVDS_PORT_EN);
+	intel_de_rmw(dev_priv, lvds_encoder->reg, LVDS_PORT_EN, 0);
 	intel_de_posting_read(dev_priv, lvds_encoder->reg);
 }
 
@@ -386,19 +385,19 @@ static void intel_lvds_shutdown(struct intel_encoder *encoder)
 }
 
 static enum drm_mode_status
-intel_lvds_mode_valid(struct drm_connector *connector,
+intel_lvds_mode_valid(struct drm_connector *_connector,
 		      struct drm_display_mode *mode)
 {
-	struct intel_connector *intel_connector = to_intel_connector(connector);
+	struct intel_connector *connector = to_intel_connector(_connector);
 	const struct drm_display_mode *fixed_mode =
-		intel_panel_fixed_mode(intel_connector, mode);
-	int max_pixclk = to_i915(connector->dev)->max_dotclk_freq;
+		intel_panel_fixed_mode(connector, mode);
+	int max_pixclk = to_i915(connector->base.dev)->max_dotclk_freq;
 	enum drm_mode_status status;
 
 	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
 		return MODE_NO_DBLESCAN;
 
-	status = intel_panel_mode_valid(intel_connector, mode);
+	status = intel_panel_mode_valid(connector, mode);
 	if (status != MODE_OK)
 		return status;
 
@@ -408,23 +407,21 @@ intel_lvds_mode_valid(struct drm_connector *connector,
 	return MODE_OK;
 }
 
-static int intel_lvds_compute_config(struct intel_encoder *intel_encoder,
-				     struct intel_crtc_state *pipe_config,
+static int intel_lvds_compute_config(struct intel_encoder *encoder,
+				     struct intel_crtc_state *crtc_state,
 				     struct drm_connector_state *conn_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
-	struct intel_lvds_encoder *lvds_encoder =
-		to_lvds_encoder(intel_encoder);
-	struct intel_connector *intel_connector =
-		lvds_encoder->attached_connector;
-	struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
-	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(encoder);
+	struct intel_connector *connector = lvds_encoder->attached_connector;
+	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	unsigned int lvds_bpp;
 	int ret;
 
 	/* Should never happen!! */
-	if (DISPLAY_VER(dev_priv) < 4 && crtc->pipe == 0) {
-		drm_err(&dev_priv->drm, "Can't support LVDS on pipe A\n");
+	if (DISPLAY_VER(i915) < 4 && crtc->pipe == 0) {
+		drm_err(&i915->drm, "Can't support LVDS on pipe A\n");
 		return -EINVAL;
 	}
 
@@ -433,14 +430,14 @@ static int intel_lvds_compute_config(struct intel_encoder *intel_encoder,
 	else
 		lvds_bpp = 6*3;
 
-	if (lvds_bpp != pipe_config->pipe_bpp && !pipe_config->bw_constrained) {
-		drm_dbg_kms(&dev_priv->drm,
+	if (lvds_bpp != crtc_state->pipe_bpp && !crtc_state->bw_constrained) {
+		drm_dbg_kms(&i915->drm,
 			    "forcing display bpp (was %d) to LVDS (%d)\n",
-			    pipe_config->pipe_bpp, lvds_bpp);
-		pipe_config->pipe_bpp = lvds_bpp;
+			    crtc_state->pipe_bpp, lvds_bpp);
+		crtc_state->pipe_bpp = lvds_bpp;
 	}
 
-	pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
+	crtc_state->output_format = INTEL_OUTPUT_FORMAT_RGB;
 
 	/*
 	 * We have timings from the BIOS for the panel, put them in
@@ -448,17 +445,17 @@ static int intel_lvds_compute_config(struct intel_encoder *intel_encoder,
 	 * with the panel scaling set up to source from the H/VDisplay
 	 * of the original mode.
 	 */
-	ret = intel_panel_compute_config(intel_connector, adjusted_mode);
+	ret = intel_panel_compute_config(connector, adjusted_mode);
 	if (ret)
 		return ret;
 
 	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)
 		return -EINVAL;
 
-	if (HAS_PCH_SPLIT(dev_priv))
-		pipe_config->has_pch_encoder = true;
+	if (HAS_PCH_SPLIT(i915))
+		crtc_state->has_pch_encoder = true;
 
-	ret = intel_panel_fitting(pipe_config, conn_state);
+	ret = intel_panel_fitting(crtc_state, conn_state);
 	if (ret)
 		return ret;
 
@@ -474,19 +471,19 @@ static int intel_lvds_compute_config(struct intel_encoder *intel_encoder,
 /*
  * Return the list of DDC modes if available, or the BIOS fixed mode otherwise.
  */
-static int intel_lvds_get_modes(struct drm_connector *connector)
+static int intel_lvds_get_modes(struct drm_connector *_connector)
 {
-	struct intel_connector *intel_connector = to_intel_connector(connector);
-	const struct drm_edid *fixed_edid = intel_connector->panel.fixed_edid;
+	struct intel_connector *connector = to_intel_connector(_connector);
+	const struct drm_edid *fixed_edid = connector->panel.fixed_edid;
 
 	/* Use panel fixed edid if we have one */
 	if (!IS_ERR_OR_NULL(fixed_edid)) {
-		drm_edid_connector_update(connector, fixed_edid);
+		drm_edid_connector_update(&connector->base, fixed_edid);
 
-		return drm_edid_connector_add_modes(connector);
+		return drm_edid_connector_add_modes(&connector->base);
 	}
 
-	return intel_panel_get_modes(intel_connector);
+	return intel_panel_get_modes(connector);
 }
 
 static const struct drm_connector_helper_funcs intel_lvds_connector_helper_funcs = {
@@ -585,12 +582,12 @@ static const struct dmi_system_id intel_no_lvds[] = {
 	},
 	{
 		.callback = intel_no_lvds_dmi_callback,
-                .ident = "AOpen i45GMx-I",
-                .matches = {
-                        DMI_MATCH(DMI_BOARD_VENDOR, "AOpen"),
-                        DMI_MATCH(DMI_BOARD_NAME, "i45GMx-I"),
-                },
-        },
+		.ident = "AOpen i45GMx-I",
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "AOpen"),
+			DMI_MATCH(DMI_BOARD_NAME, "i45GMx-I"),
+		},
+	},
 	{
 		.callback = intel_no_lvds_dmi_callback,
 		.ident = "Aopen i945GTt-VFA",
@@ -607,14 +604,14 @@ static const struct dmi_system_id intel_no_lvds[] = {
 		},
 	},
 	{
-                .callback = intel_no_lvds_dmi_callback,
-                .ident = "Clientron E830",
-                .matches = {
-                        DMI_MATCH(DMI_SYS_VENDOR, "Clientron"),
-                        DMI_MATCH(DMI_PRODUCT_NAME, "E830"),
-                },
-        },
-        {
+		.callback = intel_no_lvds_dmi_callback,
+		.ident = "Clientron E830",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Clientron"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "E830"),
+		},
+	},
+	{
 		.callback = intel_no_lvds_dmi_callback,
 		.ident = "Asus EeeBox PC EB1007",
 		.matches = {
@@ -764,11 +761,11 @@ static const struct dmi_system_id intel_dual_link_lvds[] = {
 	{ }	/* terminating entry */
 };
 
-struct intel_encoder *intel_get_lvds_encoder(struct drm_i915_private *dev_priv)
+struct intel_encoder *intel_get_lvds_encoder(struct drm_i915_private *i915)
 {
 	struct intel_encoder *encoder;
 
-	for_each_intel_encoder(&dev_priv->drm, encoder) {
+	for_each_intel_encoder(&i915->drm, encoder) {
 		if (encoder->type == INTEL_OUTPUT_LVDS)
 			return encoder;
 	}
@@ -776,24 +773,24 @@ struct intel_encoder *intel_get_lvds_encoder(struct drm_i915_private *dev_priv)
 	return NULL;
 }
 
-bool intel_is_dual_link_lvds(struct drm_i915_private *dev_priv)
+bool intel_is_dual_link_lvds(struct drm_i915_private *i915)
 {
-	struct intel_encoder *encoder = intel_get_lvds_encoder(dev_priv);
+	struct intel_encoder *encoder = intel_get_lvds_encoder(i915);
 
 	return encoder && to_lvds_encoder(encoder)->is_dual_link;
 }
 
 static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(lvds_encoder->base.base.dev);
+	struct drm_i915_private *i915 = to_i915(lvds_encoder->base.base.dev);
 	struct intel_connector *connector = lvds_encoder->attached_connector;
 	const struct drm_display_mode *fixed_mode =
 		intel_panel_preferred_fixed_mode(connector);
 	unsigned int val;
 
 	/* use the module option value if specified */
-	if (dev_priv->params.lvds_channel_mode > 0)
-		return dev_priv->params.lvds_channel_mode == 2;
+	if (i915->params.lvds_channel_mode > 0)
+		return i915->params.lvds_channel_mode == 2;
 
 	/* single channel LVDS is limited to 112 MHz */
 	if (fixed_mode->clock > 112999)
@@ -808,8 +805,8 @@ static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder)
 	 * we need to check "the value to be set" in VBT when LVDS
 	 * register is uninitialized.
 	 */
-	val = intel_de_read(dev_priv, lvds_encoder->reg);
-	if (HAS_PCH_CPT(dev_priv))
+	val = intel_de_read(i915, lvds_encoder->reg);
+	if (HAS_PCH_CPT(i915))
 		val &= ~(LVDS_DETECTED | LVDS_PIPE_SEL_MASK_CPT);
 	else
 		val &= ~(LVDS_DETECTED | LVDS_PIPE_SEL_MASK);
@@ -826,56 +823,54 @@ static void intel_lvds_add_properties(struct drm_connector *connector)
 
 /**
  * intel_lvds_init - setup LVDS connectors on this device
- * @dev_priv: i915 device
+ * @i915: i915 device
  *
  * Create the connector, register the LVDS DDC bus, and try to figure out what
  * modes we can display on the LVDS panel (if present).
  */
-void intel_lvds_init(struct drm_i915_private *dev_priv)
+void intel_lvds_init(struct drm_i915_private *i915)
 {
 	struct intel_lvds_encoder *lvds_encoder;
-	struct intel_encoder *intel_encoder;
-	struct intel_connector *intel_connector;
-	struct drm_connector *connector;
-	struct drm_encoder *encoder;
+	struct intel_connector *connector;
 	const struct drm_edid *drm_edid;
+	struct intel_encoder *encoder;
 	i915_reg_t lvds_reg;
 	u32 lvds;
 	u8 pin;
 
 	/* Skip init on machines we know falsely report LVDS */
 	if (dmi_check_system(intel_no_lvds)) {
-		drm_WARN(&dev_priv->drm, !dev_priv->display.vbt.int_lvds_support,
+		drm_WARN(&i915->drm, !i915->display.vbt.int_lvds_support,
 			 "Useless DMI match. Internal LVDS support disabled by VBT\n");
 		return;
 	}
 
-	if (!dev_priv->display.vbt.int_lvds_support) {
-		drm_dbg_kms(&dev_priv->drm,
+	if (!i915->display.vbt.int_lvds_support) {
+		drm_dbg_kms(&i915->drm,
 			    "Internal LVDS support disabled by VBT\n");
 		return;
 	}
 
-	if (HAS_PCH_SPLIT(dev_priv))
+	if (HAS_PCH_SPLIT(i915))
 		lvds_reg = PCH_LVDS;
 	else
 		lvds_reg = LVDS;
 
-	lvds = intel_de_read(dev_priv, lvds_reg);
+	lvds = intel_de_read(i915, lvds_reg);
 
-	if (HAS_PCH_SPLIT(dev_priv)) {
+	if (HAS_PCH_SPLIT(i915)) {
 		if ((lvds & LVDS_DETECTED) == 0)
 			return;
 	}
 
 	pin = GMBUS_PIN_PANEL;
-	if (!intel_bios_is_lvds_present(dev_priv, &pin)) {
+	if (!intel_bios_is_lvds_present(i915, &pin)) {
 		if ((lvds & LVDS_PORT_EN) == 0) {
-			drm_dbg_kms(&dev_priv->drm,
+			drm_dbg_kms(&i915->drm,
 				    "LVDS is not present in VBT\n");
 			return;
 		}
-		drm_dbg_kms(&dev_priv->drm,
+		drm_dbg_kms(&i915->drm,
 			    "LVDS is not present in VBT, but enabled anyway\n");
 	}
 
@@ -883,57 +878,55 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 	if (!lvds_encoder)
 		return;
 
-	intel_connector = intel_connector_alloc();
-	if (!intel_connector) {
+	connector = intel_connector_alloc();
+	if (!connector) {
 		kfree(lvds_encoder);
 		return;
 	}
 
-	lvds_encoder->attached_connector = intel_connector;
+	lvds_encoder->attached_connector = connector;
+	encoder = &lvds_encoder->base;
 
-	intel_encoder = &lvds_encoder->base;
-	encoder = &intel_encoder->base;
-	connector = &intel_connector->base;
-	drm_connector_init(&dev_priv->drm, &intel_connector->base, &intel_lvds_connector_funcs,
+	drm_connector_init(&i915->drm, &connector->base, &intel_lvds_connector_funcs,
 			   DRM_MODE_CONNECTOR_LVDS);
 
-	drm_encoder_init(&dev_priv->drm, &intel_encoder->base, &intel_lvds_enc_funcs,
+	drm_encoder_init(&i915->drm, &encoder->base, &intel_lvds_enc_funcs,
 			 DRM_MODE_ENCODER_LVDS, "LVDS");
 
-	intel_encoder->enable = intel_enable_lvds;
-	intel_encoder->pre_enable = intel_pre_enable_lvds;
-	intel_encoder->compute_config = intel_lvds_compute_config;
-	if (HAS_PCH_SPLIT(dev_priv)) {
-		intel_encoder->disable = pch_disable_lvds;
-		intel_encoder->post_disable = pch_post_disable_lvds;
+	encoder->enable = intel_enable_lvds;
+	encoder->pre_enable = intel_pre_enable_lvds;
+	encoder->compute_config = intel_lvds_compute_config;
+	if (HAS_PCH_SPLIT(i915)) {
+		encoder->disable = pch_disable_lvds;
+		encoder->post_disable = pch_post_disable_lvds;
 	} else {
-		intel_encoder->disable = gmch_disable_lvds;
+		encoder->disable = gmch_disable_lvds;
 	}
-	intel_encoder->get_hw_state = intel_lvds_get_hw_state;
-	intel_encoder->get_config = intel_lvds_get_config;
-	intel_encoder->update_pipe = intel_backlight_update;
-	intel_encoder->shutdown = intel_lvds_shutdown;
-	intel_connector->get_hw_state = intel_connector_get_hw_state;
-
-	intel_connector_attach_encoder(intel_connector, intel_encoder);
-
-	intel_encoder->type = INTEL_OUTPUT_LVDS;
-	intel_encoder->power_domain = POWER_DOMAIN_PORT_OTHER;
-	intel_encoder->port = PORT_NONE;
-	intel_encoder->cloneable = 0;
-	if (DISPLAY_VER(dev_priv) < 4)
-		intel_encoder->pipe_mask = BIT(PIPE_B);
+	encoder->get_hw_state = intel_lvds_get_hw_state;
+	encoder->get_config = intel_lvds_get_config;
+	encoder->update_pipe = intel_backlight_update;
+	encoder->shutdown = intel_lvds_shutdown;
+	connector->get_hw_state = intel_connector_get_hw_state;
+
+	intel_connector_attach_encoder(connector, encoder);
+
+	encoder->type = INTEL_OUTPUT_LVDS;
+	encoder->power_domain = POWER_DOMAIN_PORT_OTHER;
+	encoder->port = PORT_NONE;
+	encoder->cloneable = 0;
+	if (DISPLAY_VER(i915) < 4)
+		encoder->pipe_mask = BIT(PIPE_B);
 	else
-		intel_encoder->pipe_mask = ~0;
+		encoder->pipe_mask = ~0;
 
-	drm_connector_helper_add(connector, &intel_lvds_connector_helper_funcs);
-	connector->display_info.subpixel_order = SubPixelHorizontalRGB;
+	drm_connector_helper_add(&connector->base, &intel_lvds_connector_helper_funcs);
+	connector->base.display_info.subpixel_order = SubPixelHorizontalRGB;
 
 	lvds_encoder->reg = lvds_reg;
 
-	intel_lvds_add_properties(connector);
+	intel_lvds_add_properties(&connector->base);
 
-	intel_lvds_pps_get_hw_state(dev_priv, &lvds_encoder->init_pps);
+	intel_lvds_pps_get_hw_state(i915, &lvds_encoder->init_pps);
 	lvds_encoder->init_lvds_val = lvds;
 
 	/*
@@ -948,13 +941,13 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 	 * Attempt to get the fixed panel mode from DDC.  Assume that the
 	 * preferred mode is the right one.
 	 */
-	mutex_lock(&dev_priv->drm.mode_config.mutex);
+	mutex_lock(&i915->drm.mode_config.mutex);
 	if (vga_switcheroo_handler_flags() & VGA_SWITCHEROO_CAN_SWITCH_DDC) {
 		const struct edid *edid;
 
 		/* FIXME: Make drm_get_edid_switcheroo() return drm_edid */
-		edid = drm_get_edid_switcheroo(connector,
-					       intel_gmbus_get_adapter(dev_priv, pin));
+		edid = drm_get_edid_switcheroo(&connector->base,
+					       intel_gmbus_get_adapter(i915, pin));
 		if (edid) {
 			drm_edid = drm_edid_alloc(edid, (edid->extensions + 1) * EDID_LENGTH);
 			kfree(edid);
@@ -962,49 +955,49 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 			drm_edid = NULL;
 		}
 	} else {
-		drm_edid = drm_edid_read_ddc(connector,
-					     intel_gmbus_get_adapter(dev_priv, pin));
+		drm_edid = drm_edid_read_ddc(&connector->base,
+					     intel_gmbus_get_adapter(i915, pin));
 	}
 	if (drm_edid) {
-		if (drm_edid_connector_update(connector, drm_edid) ||
-		    !drm_edid_connector_add_modes(connector)) {
-			drm_edid_connector_update(connector, NULL);
+		if (drm_edid_connector_update(&connector->base, drm_edid) ||
+		    !drm_edid_connector_add_modes(&connector->base)) {
+			drm_edid_connector_update(&connector->base, NULL);
 			drm_edid_free(drm_edid);
 			drm_edid = ERR_PTR(-EINVAL);
 		}
 	} else {
 		drm_edid = ERR_PTR(-ENOENT);
 	}
-	intel_bios_init_panel_late(dev_priv, &intel_connector->panel, NULL,
+	intel_bios_init_panel_late(i915, &connector->panel, NULL,
 				   IS_ERR(drm_edid) ? NULL : drm_edid);
 
 	/* Try EDID first */
-	intel_panel_add_edid_fixed_modes(intel_connector, true);
+	intel_panel_add_edid_fixed_modes(connector, true);
 
 	/* Failed to get EDID, what about VBT? */
-	if (!intel_panel_preferred_fixed_mode(intel_connector))
-		intel_panel_add_vbt_lfp_fixed_mode(intel_connector);
+	if (!intel_panel_preferred_fixed_mode(connector))
+		intel_panel_add_vbt_lfp_fixed_mode(connector);
 
 	/*
 	 * If we didn't get a fixed mode from EDID or VBT, try checking
 	 * if the panel is already turned on.  If so, assume that
 	 * whatever is currently programmed is the correct mode.
 	 */
-	if (!intel_panel_preferred_fixed_mode(intel_connector))
-		intel_panel_add_encoder_fixed_mode(intel_connector, intel_encoder);
+	if (!intel_panel_preferred_fixed_mode(connector))
+		intel_panel_add_encoder_fixed_mode(connector, encoder);
 
-	mutex_unlock(&dev_priv->drm.mode_config.mutex);
+	mutex_unlock(&i915->drm.mode_config.mutex);
 
 	/* If we still don't have a mode after all that, give up. */
-	if (!intel_panel_preferred_fixed_mode(intel_connector))
+	if (!intel_panel_preferred_fixed_mode(connector))
 		goto failed;
 
-	intel_panel_init(intel_connector, drm_edid);
+	intel_panel_init(connector, drm_edid);
 
-	intel_backlight_setup(intel_connector, INVALID_PIPE);
+	intel_backlight_setup(connector, INVALID_PIPE);
 
 	lvds_encoder->is_dual_link = compute_is_dual_link_lvds(lvds_encoder);
-	drm_dbg_kms(&dev_priv->drm, "detected %s-link lvds configuration\n",
+	drm_dbg_kms(&i915->drm, "detected %s-link lvds configuration\n",
 		    lvds_encoder->is_dual_link ? "dual" : "single");
 
 	lvds_encoder->a3_power = lvds & LVDS_A3_POWER_MASK;
@@ -1012,10 +1005,10 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 	return;
 
 failed:
-	drm_dbg_kms(&dev_priv->drm, "No LVDS modes found, disabling.\n");
-	drm_connector_cleanup(connector);
-	drm_encoder_cleanup(encoder);
+	drm_dbg_kms(&i915->drm, "No LVDS modes found, disabling.\n");
+	drm_connector_cleanup(&connector->base);
+	drm_encoder_cleanup(&encoder->base);
 	kfree(lvds_encoder);
-	intel_connector_free(intel_connector);
+	intel_connector_free(connector);
 	return;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_lvds_regs.h b/drivers/gpu/drm/i915/display/intel_lvds_regs.h
new file mode 100644
index 000000000000..47c1832819ee
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_lvds_regs.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_LVDS_REGS_H__
+#define __INTEL_LVDS_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+/* LVDS port control */
+#define LVDS		_MMIO(0x61180)
+/*
+ * Enables the LVDS port.  This bit must be set before DPLLs are enabled, as
+ * the DPLL semantics change when the LVDS is assigned to that pipe.
+ */
+#define   LVDS_PORT_EN			REG_BIT(31)
+/* Selects pipe B for LVDS data.  Must be set on pre-965. */
+#define   LVDS_PIPE_SEL_MASK		REG_BIT(30)
+#define   LVDS_PIPE_SEL(pipe)		REG_FIELD_PREP(LVDS_PIPE_SEL_MASK, (pipe))
+#define   LVDS_PIPE_SEL_MASK_CPT	REG_GENMASK(30, 29)
+#define   LVDS_PIPE_SEL_CPT(pipe)	REG_FIELD_PREP(LVDS_PIPE_SEL_MASK_CPT, (pipe))
+/* LVDS dithering flag on 965/g4x platform */
+#define   LVDS_ENABLE_DITHER		REG_BIT(25)
+/* LVDS sync polarity flags. Set to invert (i.e. negative) */
+#define   LVDS_VSYNC_POLARITY		REG_BIT(21)
+#define   LVDS_HSYNC_POLARITY		REG_BIT(20)
+
+/* Enable border for unscaled (or aspect-scaled) display */
+#define   LVDS_BORDER_ENABLE		REG_BIT(15)
+/*
+ * Enables the A0-A2 data pairs and CLKA, containing 18 bits of color data per
+ * pixel.
+ */
+#define   LVDS_A0A2_CLKA_POWER_MASK	REG_GENMASK(9, 8)
+#define   LVDS_A0A2_CLKA_POWER_DOWN	REG_FIELD_PREP(LVDS_A0A2_CLKA_POWER_MASK, 0)
+#define   LVDS_A0A2_CLKA_POWER_UP	REG_FIELD_PREP(LVDS_A0A2_CLKA_POWER_MASK, 3)
+/*
+ * Controls the A3 data pair, which contains the additional LSBs for 24 bit
+ * mode.  Only enabled if LVDS_A0A2_CLKA_POWER_UP also indicates it should be
+ * on.
+ */
+#define   LVDS_A3_POWER_MASK		REG_GENMASK(7, 6)
+#define   LVDS_A3_POWER_DOWN		REG_FIELD_PREP(LVDS_A3_POWER_MASK, 0)
+#define   LVDS_A3_POWER_UP		REG_FIELD_PREP(LVDS_A3_POWER_MASK, 3)
+/*
+ * Controls the CLKB pair.  This should only be set when LVDS_B0B3_POWER_UP
+ * is set.
+ */
+#define   LVDS_CLKB_POWER_MASK		REG_GENMASK(5, 4)
+#define   LVDS_CLKB_POWER_DOWN		REG_FIELD_PREP(LVDS_CLKB_POWER_MASK, 0)
+#define   LVDS_CLKB_POWER_UP		REG_FIELD_PREP(LVDS_CLKB_POWER_MASK, 3)
+/*
+ * Controls the B0-B3 data pairs.  This must be set to match the DPLL p2
+ * setting for whether we are in dual-channel mode.  The B3 pair will
+ * additionally only be powered up when LVDS_A3_POWER_UP is set.
+ */
+#define   LVDS_B0B3_POWER_MASK		REG_GENMASK(3, 2)
+#define   LVDS_B0B3_POWER_DOWN		REG_FIELD_PREP(LVDS_B0B3_POWER_MASK, 0)
+#define   LVDS_B0B3_POWER_UP		REG_FIELD_PREP(LVDS_B0B3_POWER_MASK, 3)
+
+#define PCH_LVDS	_MMIO(0xe1180)
+#define   LVDS_DETECTED			REG_BIT(1)
+
+#endif /* __INTEL_LVDS_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_mg_phy_regs.h b/drivers/gpu/drm/i915/display/intel_mg_phy_regs.h
index 0e8248bce52d..0306ade2bc30 100644
--- a/drivers/gpu/drm/i915/display/intel_mg_phy_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_mg_phy_regs.h
@@ -142,7 +142,9 @@
 #define FIA1_BASE			0x163000
 #define FIA2_BASE			0x16E000
 #define FIA3_BASE			0x16F000
-#define _FIA(fia)			_PICK((fia), FIA1_BASE, FIA2_BASE, FIA3_BASE)
+#define _FIA(fia)			_PICK_EVEN_2RANGES((fia), 1,		\
+							   FIA1_BASE, FIA1_BASE,\
+							   FIA2_BASE, FIA3_BASE)
 #define _MMIO_FIA(fia, off)		_MMIO(_FIA(fia) + (off))
 
 /* ICL PHY DFLEX registers */
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_setup.c b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
index 52cdbd4fc2fa..134b943f1953 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_setup.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
@@ -11,6 +11,7 @@
 
 #include "i915_drv.h"
 #include "i915_reg.h"
+#include "i9xx_wm.h"
 #include "intel_atomic.h"
 #include "intel_bw.h"
 #include "intel_color.h"
@@ -21,9 +22,12 @@
 #include "intel_display.h"
 #include "intel_display_power.h"
 #include "intel_display_types.h"
+#include "intel_dmc.h"
+#include "intel_fifo_underrun.h"
 #include "intel_modeset_setup.h"
 #include "intel_pch_display.h"
-#include "intel_pm.h"
+#include "intel_vblank.h"
+#include "intel_wm.h"
 #include "skl_watermark.h"
 
 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
@@ -96,7 +100,6 @@ static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
 
 	intel_fbc_disable(crtc);
 	intel_update_watermarks(i915);
-	intel_disable_shared_dpll(crtc_state);
 
 	intel_display_power_put_all_in_set(i915, &crtc->enabled_power_domains);
 
@@ -234,12 +237,9 @@ static void intel_sanitize_fifo_underrun_reporting(const struct intel_crtc_state
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 
-	if (!crtc_state->hw.active && !HAS_GMCH(i915))
-		return;
-
 	/*
-	 * We start out with underrun reporting disabled to avoid races.
-	 * For correct bookkeeping mark this on active crtcs.
+	 * We start out with underrun reporting disabled on active
+	 * pipes to avoid races.
 	 *
 	 * Also on gmch platforms we dont have any hardware bits to
 	 * disable the underrun reporting. Which means we need to start
@@ -250,19 +250,9 @@ static void intel_sanitize_fifo_underrun_reporting(const struct intel_crtc_state
 	 * No protection against concurrent access is required - at
 	 * worst a fifo underrun happens which also sets this to false.
 	 */
-	crtc->cpu_fifo_underrun_disabled = true;
-
-	/*
-	 * We track the PCH trancoder underrun reporting state
-	 * within the crtc. With crtc for pipe A housing the underrun
-	 * reporting state for PCH transcoder A, crtc for pipe B housing
-	 * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
-	 * and marking underrun reporting as disabled for the non-existing
-	 * PCH transcoders B and C would prevent enabling the south
-	 * error interrupt (see cpt_can_enable_serr_int()).
-	 */
-	if (intel_has_pch_trancoder(i915, crtc->pipe))
-		crtc->pch_fifo_underrun_disabled = true;
+	intel_init_fifo_underrun_reporting(i915, crtc,
+					   !crtc_state->hw.active &&
+					   !HAS_GMCH(i915));
 }
 
 static void intel_sanitize_crtc(struct intel_crtc *crtc,
@@ -647,17 +637,14 @@ static void intel_early_display_was(struct drm_i915_private *i915)
 	 * Also known as Wa_14010480278.
 	 */
 	if (IS_DISPLAY_VER(i915, 10, 12))
-		intel_de_write(i915, GEN9_CLKGATE_DIS_0,
-			       intel_de_read(i915, GEN9_CLKGATE_DIS_0) | DARBF_GATING_DIS);
+		intel_de_rmw(i915, GEN9_CLKGATE_DIS_0, 0, DARBF_GATING_DIS);
 
-	if (IS_HASWELL(i915)) {
-		/*
-		 * WaRsPkgCStateDisplayPMReq:hsw
-		 * System hang if this isn't done before disabling all planes!
-		 */
-		intel_de_write(i915, CHICKEN_PAR1_1,
-			       intel_de_read(i915, CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
-	}
+	/*
+	 * WaRsPkgCStateDisplayPMReq:hsw
+	 * System hang if this isn't done before disabling all planes!
+	 */
+	if (IS_HASWELL(i915))
+		intel_de_rmw(i915, CHICKEN_PAR1_1, 0, FORCE_ARB_IDLE_PLANES);
 
 	if (IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) {
 		/* Display WA #1142:kbl,cfl,cml */
@@ -723,18 +710,7 @@ void intel_modeset_setup_hw_state(struct drm_i915_private *i915,
 
 	intel_dpll_sanitize_state(i915);
 
-	if (IS_G4X(i915)) {
-		g4x_wm_get_hw_state(i915);
-		g4x_wm_sanitize(i915);
-	} else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
-		vlv_wm_get_hw_state(i915);
-		vlv_wm_sanitize(i915);
-	} else if (DISPLAY_VER(i915) >= 9) {
-		skl_wm_get_hw_state(i915);
-		skl_wm_sanitize(i915);
-	} else if (HAS_PCH_SPLIT(i915)) {
-		ilk_wm_get_hw_state(i915);
-	}
+	intel_wm_get_hw_state(i915);
 
 	for_each_intel_crtc(&i915->drm, crtc) {
 		struct intel_crtc_state *crtc_state =
diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c
index b8dce0576512..b7973a05d022 100644
--- a/drivers/gpu/drm/i915/display/intel_opregion.c
+++ b/drivers/gpu/drm/i915/display/intel_opregion.c
@@ -1159,13 +1159,10 @@ void intel_opregion_register(struct drm_i915_private *i915)
 	intel_opregion_resume(i915);
 }
 
-void intel_opregion_resume(struct drm_i915_private *i915)
+static void intel_opregion_resume_display(struct drm_i915_private *i915)
 {
 	struct intel_opregion *opregion = &i915->display.opregion;
 
-	if (!opregion->header)
-		return;
-
 	if (opregion->acpi) {
 		intel_didl_outputs(i915);
 		intel_setup_cadls(i915);
@@ -1186,18 +1183,24 @@ void intel_opregion_resume(struct drm_i915_private *i915)
 
 	/* Some platforms abuse the _DSM to enable MUX */
 	intel_dsm_get_bios_data_funcs_supported(i915);
-
-	intel_opregion_notify_adapter(i915, PCI_D0);
 }
 
-void intel_opregion_suspend(struct drm_i915_private *i915, pci_power_t state)
+void intel_opregion_resume(struct drm_i915_private *i915)
 {
 	struct intel_opregion *opregion = &i915->display.opregion;
 
 	if (!opregion->header)
 		return;
 
-	intel_opregion_notify_adapter(i915, state);
+	if (HAS_DISPLAY(i915))
+		intel_opregion_resume_display(i915);
+
+	intel_opregion_notify_adapter(i915, PCI_D0);
+}
+
+static void intel_opregion_suspend_display(struct drm_i915_private *i915)
+{
+	struct intel_opregion *opregion = &i915->display.opregion;
 
 	if (opregion->asle)
 		opregion->asle->ardy = ASLE_ARDY_NOT_READY;
@@ -1208,6 +1211,19 @@ void intel_opregion_suspend(struct drm_i915_private *i915, pci_power_t state)
 		opregion->acpi->drdy = 0;
 }
 
+void intel_opregion_suspend(struct drm_i915_private *i915, pci_power_t state)
+{
+	struct intel_opregion *opregion = &i915->display.opregion;
+
+	if (!opregion->header)
+		return;
+
+	intel_opregion_notify_adapter(i915, state);
+
+	if (HAS_DISPLAY(i915))
+		intel_opregion_suspend_display(i915);
+}
+
 void intel_opregion_unregister(struct drm_i915_private *i915)
 {
 	struct intel_opregion *opregion = &i915->display.opregion;
@@ -1221,6 +1237,14 @@ void intel_opregion_unregister(struct drm_i915_private *i915)
 		unregister_acpi_notifier(&opregion->acpi_notifier);
 		opregion->acpi_notifier.notifier_call = NULL;
 	}
+}
+
+void intel_opregion_cleanup(struct drm_i915_private *i915)
+{
+	struct intel_opregion *opregion = &i915->display.opregion;
+
+	if (!opregion->header)
+		return;
 
 	/* just clear all opregion memory pointers now */
 	memunmap(opregion->header);
diff --git a/drivers/gpu/drm/i915/display/intel_opregion.h b/drivers/gpu/drm/i915/display/intel_opregion.h
index d02e6696a050..fd2ea8ef0fa2 100644
--- a/drivers/gpu/drm/i915/display/intel_opregion.h
+++ b/drivers/gpu/drm/i915/display/intel_opregion.h
@@ -60,6 +60,7 @@ struct intel_opregion {
 #ifdef CONFIG_ACPI
 
 int intel_opregion_setup(struct drm_i915_private *dev_priv);
+void intel_opregion_cleanup(struct drm_i915_private *i915);
 
 void intel_opregion_register(struct drm_i915_private *dev_priv);
 void intel_opregion_unregister(struct drm_i915_private *dev_priv);
@@ -85,6 +86,10 @@ static inline int intel_opregion_setup(struct drm_i915_private *dev_priv)
 	return 0;
 }
 
+static inline void intel_opregion_cleanup(struct drm_i915_private *i915)
+{
+}
+
 static inline void intel_opregion_register(struct drm_i915_private *dev_priv)
 {
 }
diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c
index 42aa04bac261..ce2a34a25211 100644
--- a/drivers/gpu/drm/i915/display/intel_panel.c
+++ b/drivers/gpu/drm/i915/display/intel_panel.c
@@ -39,6 +39,7 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_drrs.h"
+#include "intel_lvds_regs.h"
 #include "intel_panel.h"
 #include "intel_quirks.h"
 
diff --git a/drivers/gpu/drm/i915/display/intel_pch_display.c b/drivers/gpu/drm/i915/display/intel_pch_display.c
index cecc0d007cf3..2411fe4dee8b 100644
--- a/drivers/gpu/drm/i915/display/intel_pch_display.c
+++ b/drivers/gpu/drm/i915/display/intel_pch_display.c
@@ -9,7 +9,9 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_fdi.h"
+#include "intel_fdi_regs.h"
 #include "intel_lvds.h"
+#include "intel_lvds_regs.h"
 #include "intel_pch_display.h"
 #include "intel_pch_refclk.h"
 #include "intel_pps.h"
@@ -219,20 +221,20 @@ static void ilk_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_s
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 
 	intel_de_write(dev_priv, PCH_TRANS_HTOTAL(pch_transcoder),
-		       intel_de_read(dev_priv, HTOTAL(cpu_transcoder)));
+		       intel_de_read(dev_priv, TRANS_HTOTAL(cpu_transcoder)));
 	intel_de_write(dev_priv, PCH_TRANS_HBLANK(pch_transcoder),
-		       intel_de_read(dev_priv, HBLANK(cpu_transcoder)));
+		       intel_de_read(dev_priv, TRANS_HBLANK(cpu_transcoder)));
 	intel_de_write(dev_priv, PCH_TRANS_HSYNC(pch_transcoder),
-		       intel_de_read(dev_priv, HSYNC(cpu_transcoder)));
+		       intel_de_read(dev_priv, TRANS_HSYNC(cpu_transcoder)));
 
 	intel_de_write(dev_priv, PCH_TRANS_VTOTAL(pch_transcoder),
-		       intel_de_read(dev_priv, VTOTAL(cpu_transcoder)));
+		       intel_de_read(dev_priv, TRANS_VTOTAL(cpu_transcoder)));
 	intel_de_write(dev_priv, PCH_TRANS_VBLANK(pch_transcoder),
-		       intel_de_read(dev_priv, VBLANK(cpu_transcoder)));
+		       intel_de_read(dev_priv, TRANS_VBLANK(cpu_transcoder)));
 	intel_de_write(dev_priv, PCH_TRANS_VSYNC(pch_transcoder),
-		       intel_de_read(dev_priv, VSYNC(cpu_transcoder)));
+		       intel_de_read(dev_priv, TRANS_VSYNC(cpu_transcoder)));
 	intel_de_write(dev_priv, PCH_TRANS_VSYNCSHIFT(pch_transcoder),
-		       intel_de_read(dev_priv, VSYNCSHIFT(cpu_transcoder)));
+		       intel_de_read(dev_priv, TRANS_VSYNCSHIFT(cpu_transcoder)));
 }
 
 static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
@@ -266,7 +268,7 @@ static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
 
 	reg = PCH_TRANSCONF(pipe);
 	val = intel_de_read(dev_priv, reg);
-	pipeconf_val = intel_de_read(dev_priv, PIPECONF(pipe));
+	pipeconf_val = intel_de_read(dev_priv, TRANSCONF(pipe));
 
 	if (HAS_PCH_IBX(dev_priv)) {
 		/* Configure frame start delay to match the CPU */
@@ -278,15 +280,15 @@ static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
 		 * that in pipeconf reg. For HDMI we must use 8bpc
 		 * here for both 8bpc and 12bpc.
 		 */
-		val &= ~PIPECONF_BPC_MASK;
+		val &= ~TRANSCONF_BPC_MASK;
 		if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
-			val |= PIPECONF_BPC_8;
+			val |= TRANSCONF_BPC_8;
 		else
-			val |= pipeconf_val & PIPECONF_BPC_MASK;
+			val |= pipeconf_val & TRANSCONF_BPC_MASK;
 	}
 
 	val &= ~TRANS_INTERLACE_MASK;
-	if ((pipeconf_val & PIPECONF_INTERLACE_MASK_ILK) == PIPECONF_INTERLACE_IF_ID_ILK) {
+	if ((pipeconf_val & TRANSCONF_INTERLACE_MASK_ILK) == TRANSCONF_INTERLACE_IF_ID_ILK) {
 		if (HAS_PCH_IBX(dev_priv) &&
 		    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
 			val |= TRANS_INTERLACE_LEGACY_VSYNC_IBX;
@@ -307,7 +309,6 @@ static void ilk_disable_pch_transcoder(struct intel_crtc *crtc)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 	i915_reg_t reg;
-	u32 val;
 
 	/* FDI relies on the transcoder */
 	assert_fdi_tx_disabled(dev_priv, pipe);
@@ -317,21 +318,16 @@ static void ilk_disable_pch_transcoder(struct intel_crtc *crtc)
 	assert_pch_ports_disabled(dev_priv, pipe);
 
 	reg = PCH_TRANSCONF(pipe);
-	val = intel_de_read(dev_priv, reg);
-	val &= ~TRANS_ENABLE;
-	intel_de_write(dev_priv, reg, val);
+	intel_de_rmw(dev_priv, reg, TRANS_ENABLE, 0);
 	/* wait for PCH transcoder off, transcoder state */
 	if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
 		drm_err(&dev_priv->drm, "failed to disable transcoder %c\n",
 			pipe_name(pipe));
 
-	if (HAS_PCH_CPT(dev_priv)) {
+	if (HAS_PCH_CPT(dev_priv))
 		/* Workaround: Clear the timing override chicken bit again. */
-		reg = TRANS_CHICKEN2(pipe);
-		val = intel_de_read(dev_priv, reg);
-		val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
-		intel_de_write(dev_priv, reg, val);
-	}
+		intel_de_rmw(dev_priv, TRANS_CHICKEN2(pipe),
+			     TRANS_CHICKEN2_TIMING_OVERRIDE, 0);
 }
 
 void ilk_pch_pre_enable(struct intel_atomic_state *state,
@@ -414,7 +410,7 @@ void ilk_pch_enable(struct intel_atomic_state *state,
 	    intel_crtc_has_dp_encoder(crtc_state)) {
 		const struct drm_display_mode *adjusted_mode =
 			&crtc_state->hw.adjusted_mode;
-		u32 bpc = (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
+		u32 bpc = (intel_de_read(dev_priv, TRANSCONF(pipe)) & TRANSCONF_BPC_MASK) >> 5;
 		i915_reg_t reg = TRANS_DP_CTL(pipe);
 		enum port port;
 
@@ -456,21 +452,14 @@ void ilk_pch_post_disable(struct intel_atomic_state *state,
 	ilk_disable_pch_transcoder(crtc);
 
 	if (HAS_PCH_CPT(dev_priv)) {
-		i915_reg_t reg;
-		u32 temp;
-
 		/* disable TRANS_DP_CTL */
-		reg = TRANS_DP_CTL(pipe);
-		temp = intel_de_read(dev_priv, reg);
-		temp &= ~(TRANS_DP_OUTPUT_ENABLE |
-			  TRANS_DP_PORT_SEL_MASK);
-		temp |= TRANS_DP_PORT_SEL_NONE;
-		intel_de_write(dev_priv, reg, temp);
+		intel_de_rmw(dev_priv, TRANS_DP_CTL(pipe),
+			     TRANS_DP_OUTPUT_ENABLE | TRANS_DP_PORT_SEL_MASK,
+			     TRANS_DP_PORT_SEL_NONE);
 
 		/* disable DPLL_SEL */
-		temp = intel_de_read(dev_priv, PCH_DPLL_SEL);
-		temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
-		intel_de_write(dev_priv, PCH_DPLL_SEL, temp);
+		intel_de_rmw(dev_priv, PCH_DPLL_SEL,
+			     TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe), 0);
 	}
 
 	ilk_fdi_pll_disable(crtc);
@@ -565,9 +554,9 @@ static void lpt_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
 	intel_de_write(dev_priv, TRANS_CHICKEN2(PIPE_A), val);
 
 	val = TRANS_ENABLE;
-	pipeconf_val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
+	pipeconf_val = intel_de_read(dev_priv, TRANSCONF(cpu_transcoder));
 
-	if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) == PIPECONF_INTERLACE_IF_ID_ILK)
+	if ((pipeconf_val & TRANSCONF_INTERLACE_MASK_HSW) == TRANSCONF_INTERLACE_IF_ID_ILK)
 		val |= TRANS_INTERLACE_INTERLACED;
 	else
 		val |= TRANS_INTERLACE_PROGRESSIVE;
@@ -580,20 +569,14 @@ static void lpt_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
 
 static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 {
-	u32 val;
-
-	val = intel_de_read(dev_priv, LPT_TRANSCONF);
-	val &= ~TRANS_ENABLE;
-	intel_de_write(dev_priv, LPT_TRANSCONF, val);
+	intel_de_rmw(dev_priv, LPT_TRANSCONF, TRANS_ENABLE, 0);
 	/* wait for PCH transcoder off, transcoder state */
 	if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
 				    TRANS_STATE_ENABLE, 50))
 		drm_err(&dev_priv->drm, "Failed to disable PCH transcoder\n");
 
 	/* Workaround: clear timing override bit. */
-	val = intel_de_read(dev_priv, TRANS_CHICKEN2(PIPE_A));
-	val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
-	intel_de_write(dev_priv, TRANS_CHICKEN2(PIPE_A), val);
+	intel_de_rmw(dev_priv, TRANS_CHICKEN2(PIPE_A), TRANS_CHICKEN2_TIMING_OVERRIDE, 0);
 }
 
 void lpt_pch_enable(struct intel_atomic_state *state,
diff --git a/drivers/gpu/drm/i915/display/intel_pch_refclk.c b/drivers/gpu/drm/i915/display/intel_pch_refclk.c
index 3657b2940702..f4c09cc37a5e 100644
--- a/drivers/gpu/drm/i915/display/intel_pch_refclk.c
+++ b/drivers/gpu/drm/i915/display/intel_pch_refclk.c
@@ -12,19 +12,13 @@
 
 static void lpt_fdi_reset_mphy(struct drm_i915_private *dev_priv)
 {
-	u32 tmp;
-
-	tmp = intel_de_read(dev_priv, SOUTH_CHICKEN2);
-	tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
-	intel_de_write(dev_priv, SOUTH_CHICKEN2, tmp);
+	intel_de_rmw(dev_priv, SOUTH_CHICKEN2, 0, FDI_MPHY_IOSFSB_RESET_CTL);
 
 	if (wait_for_us(intel_de_read(dev_priv, SOUTH_CHICKEN2) &
 			FDI_MPHY_IOSFSB_RESET_STATUS, 100))
 		drm_err(&dev_priv->drm, "FDI mPHY reset assert timeout\n");
 
-	tmp = intel_de_read(dev_priv, SOUTH_CHICKEN2);
-	tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
-	intel_de_write(dev_priv, SOUTH_CHICKEN2, tmp);
+	intel_de_rmw(dev_priv, SOUTH_CHICKEN2, FDI_MPHY_IOSFSB_RESET_CTL, 0);
 
 	if (wait_for_us((intel_de_read(dev_priv, SOUTH_CHICKEN2) &
 			 FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c
index 7b21438edd9b..7f9926672a6a 100644
--- a/drivers/gpu/drm/i915/display/intel_pps.c
+++ b/drivers/gpu/drm/i915/display/intel_pps.c
@@ -13,7 +13,9 @@
 #include "intel_dpio_phy.h"
 #include "intel_dpll.h"
 #include "intel_lvds.h"
+#include "intel_lvds_regs.h"
 #include "intel_pps.h"
+#include "intel_pps_regs.h"
 #include "intel_quirks.h"
 
 static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv,
@@ -1534,17 +1536,13 @@ static void pps_init_registers(struct intel_dp *intel_dp, bool force_disable_vdd
 	/*
 	 * Compute the divisor for the pp clock, simply match the Bspec formula.
 	 */
-	if (i915_mmio_reg_valid(regs.pp_div)) {
+	if (i915_mmio_reg_valid(regs.pp_div))
 		intel_de_write(dev_priv, regs.pp_div,
 			       REG_FIELD_PREP(PP_REFERENCE_DIVIDER_MASK, (100 * div) / 2 - 1) | REG_FIELD_PREP(PANEL_POWER_CYCLE_DELAY_MASK, DIV_ROUND_UP(seq->t11_t12, 1000)));
-	} else {
-		u32 pp_ctl;
-
-		pp_ctl = intel_de_read(dev_priv, regs.pp_ctrl);
-		pp_ctl &= ~BXT_POWER_CYCLE_DELAY_MASK;
-		pp_ctl |= REG_FIELD_PREP(BXT_POWER_CYCLE_DELAY_MASK, DIV_ROUND_UP(seq->t11_t12, 1000));
-		intel_de_write(dev_priv, regs.pp_ctrl, pp_ctl);
-	}
+	else
+		intel_de_rmw(dev_priv, regs.pp_ctrl, BXT_POWER_CYCLE_DELAY_MASK,
+			     REG_FIELD_PREP(BXT_POWER_CYCLE_DELAY_MASK,
+					    DIV_ROUND_UP(seq->t11_t12, 1000)));
 
 	drm_dbg_kms(&dev_priv->drm,
 		    "panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n",
diff --git a/drivers/gpu/drm/i915/display/intel_pps_regs.h b/drivers/gpu/drm/i915/display/intel_pps_regs.h
new file mode 100644
index 000000000000..60edd2a27100
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_pps_regs.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_PPS_REGS_H__
+#define __INTEL_PPS_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+/* Panel power sequencing */
+#define PPS_BASE			0x61200
+#define VLV_PPS_BASE			(VLV_DISPLAY_BASE + PPS_BASE)
+#define PCH_PPS_BASE			0xC7200
+
+#define _MMIO_PPS(pps_idx, reg)		_MMIO(dev_priv->display.pps.mmio_base -	\
+					      PPS_BASE + (reg) +	\
+					      (pps_idx) * 0x100)
+
+#define _PP_STATUS			0x61200
+#define PP_STATUS(pps_idx)		_MMIO_PPS(pps_idx, _PP_STATUS)
+#define   PP_ON				REG_BIT(31)
+/*
+ * Indicates that all dependencies of the panel are on:
+ *
+ * - PLL enabled
+ * - pipe enabled
+ * - LVDS/DVOB/DVOC on
+ */
+#define   PP_READY			REG_BIT(30)
+#define   PP_SEQUENCE_MASK		REG_GENMASK(29, 28)
+#define   PP_SEQUENCE_NONE		REG_FIELD_PREP(PP_SEQUENCE_MASK, 0)
+#define   PP_SEQUENCE_POWER_UP		REG_FIELD_PREP(PP_SEQUENCE_MASK, 1)
+#define   PP_SEQUENCE_POWER_DOWN	REG_FIELD_PREP(PP_SEQUENCE_MASK, 2)
+#define   PP_CYCLE_DELAY_ACTIVE		REG_BIT(27)
+#define   PP_SEQUENCE_STATE_MASK	REG_GENMASK(3, 0)
+#define   PP_SEQUENCE_STATE_OFF_IDLE	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x0)
+#define   PP_SEQUENCE_STATE_OFF_S0_1	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x1)
+#define   PP_SEQUENCE_STATE_OFF_S0_2	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x2)
+#define   PP_SEQUENCE_STATE_OFF_S0_3	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x3)
+#define   PP_SEQUENCE_STATE_ON_IDLE	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x8)
+#define   PP_SEQUENCE_STATE_ON_S1_1	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x9)
+#define   PP_SEQUENCE_STATE_ON_S1_2	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xa)
+#define   PP_SEQUENCE_STATE_ON_S1_3	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xb)
+#define   PP_SEQUENCE_STATE_RESET	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xf)
+
+#define _PP_CONTROL			0x61204
+#define PP_CONTROL(pps_idx)		_MMIO_PPS(pps_idx, _PP_CONTROL)
+#define  PANEL_UNLOCK_MASK		REG_GENMASK(31, 16)
+#define  PANEL_UNLOCK_REGS		REG_FIELD_PREP(PANEL_UNLOCK_MASK, 0xabcd)
+#define  BXT_POWER_CYCLE_DELAY_MASK	REG_GENMASK(8, 4)
+#define  EDP_FORCE_VDD			REG_BIT(3)
+#define  EDP_BLC_ENABLE			REG_BIT(2)
+#define  PANEL_POWER_RESET		REG_BIT(1)
+#define  PANEL_POWER_ON			REG_BIT(0)
+
+#define _PP_ON_DELAYS			0x61208
+#define PP_ON_DELAYS(pps_idx)		_MMIO_PPS(pps_idx, _PP_ON_DELAYS)
+#define  PANEL_PORT_SELECT_MASK		REG_GENMASK(31, 30)
+#define  PANEL_PORT_SELECT_LVDS		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 0)
+#define  PANEL_PORT_SELECT_DPA		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 1)
+#define  PANEL_PORT_SELECT_DPC		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 2)
+#define  PANEL_PORT_SELECT_DPD		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 3)
+#define  PANEL_PORT_SELECT_VLV(port)	REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, port)
+#define  PANEL_POWER_UP_DELAY_MASK	REG_GENMASK(28, 16)
+#define  PANEL_LIGHT_ON_DELAY_MASK	REG_GENMASK(12, 0)
+
+#define _PP_OFF_DELAYS			0x6120C
+#define PP_OFF_DELAYS(pps_idx)		_MMIO_PPS(pps_idx, _PP_OFF_DELAYS)
+#define  PANEL_POWER_DOWN_DELAY_MASK	REG_GENMASK(28, 16)
+#define  PANEL_LIGHT_OFF_DELAY_MASK	REG_GENMASK(12, 0)
+
+#define _PP_DIVISOR			0x61210
+#define PP_DIVISOR(pps_idx)		_MMIO_PPS(pps_idx, _PP_DIVISOR)
+#define  PP_REFERENCE_DIVIDER_MASK	REG_GENMASK(31, 8)
+#define  PANEL_POWER_CYCLE_DELAY_MASK	REG_GENMASK(4, 0)
+
+#endif /* __INTEL_PPS_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
index 9f1a0bebae24..6badfff2b4a2 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.c
+++ b/drivers/gpu/drm/i915/display/intel_psr.c
@@ -34,6 +34,7 @@
 #include "intel_dp_aux.h"
 #include "intel_hdmi.h"
 #include "intel_psr.h"
+#include "intel_psr_regs.h"
 #include "intel_snps_phy.h"
 #include "skl_universal_plane.h"
 
@@ -152,7 +153,7 @@ static void psr_irq_control(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	i915_reg_t imr_reg;
-	u32 mask, val;
+	u32 mask;
 
 	if (DISPLAY_VER(dev_priv) >= 12)
 		imr_reg = TRANS_PSR_IMR(intel_dp->psr.transcoder);
@@ -164,10 +165,7 @@ static void psr_irq_control(struct intel_dp *intel_dp)
 		mask |= psr_irq_post_exit_bit_get(intel_dp) |
 			psr_irq_pre_entry_bit_get(intel_dp);
 
-	val = intel_de_read(dev_priv, imr_reg);
-	val &= ~psr_irq_mask_get(intel_dp);
-	val |= ~mask;
-	intel_de_write(dev_priv, imr_reg, val);
+	intel_de_rmw(dev_priv, imr_reg, psr_irq_mask_get(intel_dp), ~mask);
 }
 
 static void psr_event_print(struct drm_i915_private *i915,
@@ -245,8 +243,6 @@ void intel_psr_irq_handler(struct intel_dp *intel_dp, u32 psr_iir)
 	}
 
 	if (psr_iir & psr_irq_psr_error_bit_get(intel_dp)) {
-		u32 val;
-
 		drm_warn(&dev_priv->drm, "[transcoder %s] PSR aux error\n",
 			 transcoder_name(cpu_transcoder));
 
@@ -260,9 +256,7 @@ void intel_psr_irq_handler(struct intel_dp *intel_dp, u32 psr_iir)
 		 * again so we don't care about unmask the interruption
 		 * or unset irq_aux_error.
 		 */
-		val = intel_de_read(dev_priv, imr_reg);
-		val |= psr_irq_psr_error_bit_get(intel_dp);
-		intel_de_write(dev_priv, imr_reg, val);
+		intel_de_rmw(dev_priv, imr_reg, 0, psr_irq_psr_error_bit_get(intel_dp));
 
 		schedule_work(&intel_dp->psr.work);
 	}
@@ -526,6 +520,17 @@ static u32 intel_psr2_get_tp_time(struct intel_dp *intel_dp)
 	return val;
 }
 
+static int psr2_block_count_lines(struct intel_dp *intel_dp)
+{
+	return intel_dp->psr.io_wake_lines < 9 &&
+		intel_dp->psr.fast_wake_lines < 9 ? 8 : 12;
+}
+
+static int psr2_block_count(struct intel_dp *intel_dp)
+{
+	return psr2_block_count_lines(intel_dp) / 4;
+}
+
 static void hsw_activate_psr2(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
@@ -543,11 +548,10 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
 	val |= intel_psr2_get_tp_time(intel_dp);
 
 	if (DISPLAY_VER(dev_priv) >= 12) {
-		if (intel_dp->psr.io_wake_lines < 9 &&
-		    intel_dp->psr.fast_wake_lines < 9)
-			val |= TGL_EDP_PSR2_BLOCK_COUNT_NUM_2;
-		else
+		if (psr2_block_count(intel_dp) > 2)
 			val |= TGL_EDP_PSR2_BLOCK_COUNT_NUM_3;
+		else
+			val |= TGL_EDP_PSR2_BLOCK_COUNT_NUM_2;
 	}
 
 	/* Wa_22012278275:adl-p */
@@ -589,12 +593,6 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
 	if (intel_dp->psr.psr2_sel_fetch_enabled) {
 		u32 tmp;
 
-		/* Wa_1408330847 */
-		if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
-			intel_de_rmw(dev_priv, CHICKEN_PAR1_1,
-				     DIS_RAM_BYPASS_PSR2_MAN_TRACK,
-				     DIS_RAM_BYPASS_PSR2_MAN_TRACK);
-
 		tmp = intel_de_read(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder));
 		drm_WARN_ON(&dev_priv->drm, !(tmp & PSR2_MAN_TRK_CTL_ENABLE));
 	} else if (HAS_PSR2_SEL_FETCH(dev_priv)) {
@@ -635,13 +633,10 @@ static void psr2_program_idle_frames(struct intel_dp *intel_dp,
 				     u32 idle_frames)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
-	u32 val;
 
 	idle_frames <<=  EDP_PSR2_IDLE_FRAME_SHIFT;
-	val = intel_de_read(dev_priv, EDP_PSR2_CTL(intel_dp->psr.transcoder));
-	val &= ~EDP_PSR2_IDLE_FRAME_MASK;
-	val |= idle_frames;
-	intel_de_write(dev_priv, EDP_PSR2_CTL(intel_dp->psr.transcoder), val);
+	intel_de_rmw(dev_priv, EDP_PSR2_CTL(intel_dp->psr.transcoder),
+		     EDP_PSR2_IDLE_FRAME_MASK, idle_frames);
 }
 
 static void tgl_psr2_enable_dc3co(struct intel_dp *intel_dp)
@@ -706,6 +701,7 @@ tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp,
 {
 	const u32 crtc_vdisplay = crtc_state->uapi.adjusted_mode.crtc_vdisplay;
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	u32 exit_scanlines;
 
 	/*
@@ -722,7 +718,7 @@ tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp,
 	if (crtc_state->enable_psr2_sel_fetch)
 		return;
 
-	if (!(dev_priv->display.dmc.allowed_dc_mask & DC_STATE_EN_DC3CO))
+	if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC3CO))
 		return;
 
 	if (!dc3co_is_pipe_port_compatible(intel_dp, crtc_state))
@@ -763,13 +759,6 @@ static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp,
 		return false;
 	}
 
-	/* Wa_14010254185 Wa_14010103792 */
-	if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) {
-		drm_dbg_kms(&dev_priv->drm,
-			    "PSR2 sel fetch not enabled, missing the implementation of WAs\n");
-		return false;
-	}
-
 	return crtc_state->enable_psr2_sel_fetch = true;
 }
 
@@ -980,6 +969,15 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 		return false;
 	}
 
+	/* Vblank >= PSR2_CTL Block Count Number maximum line count */
+	if (crtc_state->hw.adjusted_mode.crtc_vblank_end -
+	    crtc_state->hw.adjusted_mode.crtc_vblank_start <
+	    psr2_block_count_lines(intel_dp)) {
+		drm_dbg_kms(&dev_priv->drm,
+			    "PSR2 not enabled, too short vblank time\n");
+		return false;
+	}
+
 	if (HAS_PSR2_SEL_FETCH(dev_priv)) {
 		if (!intel_psr2_sel_fetch_config_valid(intel_dp, crtc_state) &&
 		    !HAS_PSR_HW_TRACKING(dev_priv)) {
@@ -989,13 +987,6 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 		}
 	}
 
-	/* Wa_2209313811 */
-	if (!crtc_state->enable_psr2_sel_fetch &&
-	    IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) {
-		drm_dbg_kms(&dev_priv->drm, "PSR2 HW tracking is not supported this Display stepping\n");
-		goto unsupported;
-	}
-
 	if (!psr2_granularity_check(intel_dp, crtc_state)) {
 		drm_dbg_kms(&dev_priv->drm, "PSR2 not enabled, SU granularity not compatible\n");
 		goto unsupported;
@@ -1115,7 +1106,7 @@ void intel_psr_get_config(struct intel_encoder *encoder,
 	}
 
 	if (DISPLAY_VER(dev_priv) >= 12) {
-		val = intel_de_read(dev_priv, EXITLINE(intel_dp->psr.transcoder));
+		val = intel_de_read(dev_priv, TRANS_EXITLINE(intel_dp->psr.transcoder));
 		val &= EXITLINE_MASK;
 		pipe_config->dc3co_exitline = val;
 	}
@@ -1163,6 +1154,34 @@ static u32 wa_16013835468_bit_get(struct intel_dp *intel_dp)
 	}
 }
 
+/*
+ * Wa_16013835468
+ * Wa_14015648006
+ */
+static void wm_optimization_wa(struct intel_dp *intel_dp,
+			       const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	bool set_wa_bit = false;
+
+	/* Wa_14015648006 */
+	if (IS_MTL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) ||
+	    IS_DISPLAY_VER(dev_priv, 11, 13))
+		set_wa_bit |= crtc_state->wm_level_disabled;
+
+	/* Wa_16013835468 */
+	if (DISPLAY_VER(dev_priv) == 12)
+		set_wa_bit |= crtc_state->hw.adjusted_mode.crtc_vblank_start !=
+			crtc_state->hw.adjusted_mode.crtc_vdisplay;
+
+	if (set_wa_bit)
+		intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
+			     0, wa_16013835468_bit_get(intel_dp));
+	else
+		intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
+			     wa_16013835468_bit_get(intel_dp), 0);
+}
+
 static void intel_psr_enable_source(struct intel_dp *intel_dp,
 				    const struct intel_crtc_state *crtc_state)
 {
@@ -1189,19 +1208,13 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp,
 
 	psr_irq_control(intel_dp);
 
-	if (intel_dp->psr.dc3co_exitline) {
-		u32 val;
-
-		/*
-		 * TODO: if future platforms supports DC3CO in more than one
-		 * transcoder, EXITLINE will need to be unset when disabling PSR
-		 */
-		val = intel_de_read(dev_priv, EXITLINE(cpu_transcoder));
-		val &= ~EXITLINE_MASK;
-		val |= intel_dp->psr.dc3co_exitline << EXITLINE_SHIFT;
-		val |= EXITLINE_ENABLE;
-		intel_de_write(dev_priv, EXITLINE(cpu_transcoder), val);
-	}
+	/*
+	 * TODO: if future platforms supports DC3CO in more than one
+	 * transcoder, EXITLINE will need to be unset when disabling PSR
+	 */
+	if (intel_dp->psr.dc3co_exitline)
+		intel_de_rmw(dev_priv, TRANS_EXITLINE(cpu_transcoder), EXITLINE_MASK,
+			     intel_dp->psr.dc3co_exitline << EXITLINE_SHIFT | EXITLINE_ENABLE);
 
 	if (HAS_PSR_HW_TRACKING(dev_priv) && HAS_PSR2_SEL_FETCH(dev_priv))
 		intel_de_rmw(dev_priv, CHICKEN_PAR1_1, IGNORE_PSR2_HW_TRACKING,
@@ -1212,18 +1225,7 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp,
 	 * Wa_16013835468
 	 * Wa_14015648006
 	 */
-	if (IS_MTL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) ||
-	    IS_DISPLAY_VER(dev_priv, 12, 13)) {
-		u16 vtotal, vblank;
-
-		vtotal = crtc_state->uapi.adjusted_mode.crtc_vtotal -
-			crtc_state->uapi.adjusted_mode.crtc_vdisplay;
-		vblank = crtc_state->uapi.adjusted_mode.crtc_vblank_end -
-			crtc_state->uapi.adjusted_mode.crtc_vblank_start;
-		if (vblank > vtotal)
-			intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, 0,
-				     wa_16013835468_bit_get(intel_dp));
-	}
+	wm_optimization_wa(intel_dp, crtc_state);
 
 	if (intel_dp->psr.psr2_enabled) {
 		if (DISPLAY_VER(dev_priv) == 9)
@@ -1243,13 +1245,6 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp,
 			intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder), 0,
 				     ADLP_1_BASED_X_GRANULARITY);
 
-		/* Wa_16011168373:adl-p */
-		if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
-			intel_de_rmw(dev_priv,
-				     TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder),
-				     TRANS_SET_CONTEXT_LATENCY_MASK,
-				     TRANS_SET_CONTEXT_LATENCY_VALUE(1));
-
 		/* Wa_16012604467:adlp,mtl[a0,b0] */
 		if (IS_MTL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
 			intel_de_rmw(dev_priv,
@@ -1404,28 +1399,15 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp)
 	intel_psr_exit(intel_dp);
 	intel_psr_wait_exit_locked(intel_dp);
 
-	/* Wa_1408330847 */
-	if (intel_dp->psr.psr2_sel_fetch_enabled &&
-	    IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
-		intel_de_rmw(dev_priv, CHICKEN_PAR1_1,
-			     DIS_RAM_BYPASS_PSR2_MAN_TRACK, 0);
-
 	/*
 	 * Wa_16013835468
 	 * Wa_14015648006
 	 */
-	if (IS_MTL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) ||
-	    IS_DISPLAY_VER(dev_priv, 12, 13))
+	if (DISPLAY_VER(dev_priv) >= 11)
 		intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
 			     wa_16013835468_bit_get(intel_dp), 0);
 
 	if (intel_dp->psr.psr2_enabled) {
-		/* Wa_16011168373:adl-p */
-		if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
-			intel_de_rmw(dev_priv,
-				     TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder),
-				     TRANS_SET_CONTEXT_LATENCY_MASK, 0);
-
 		/* Wa_16012604467:adlp,mtl[a0,b0] */
 		if (IS_MTL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
 			intel_de_rmw(dev_priv,
@@ -1591,8 +1573,8 @@ static void psr_force_hw_tracking_exit(struct intel_dp *intel_dp)
 	intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);
 }
 
-void intel_psr2_disable_plane_sel_fetch(struct intel_plane *plane,
-					const struct intel_crtc_state *crtc_state)
+void intel_psr2_disable_plane_sel_fetch_arm(struct intel_plane *plane,
+					    const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	enum pipe pipe = plane->pipe;
@@ -1603,10 +1585,28 @@ void intel_psr2_disable_plane_sel_fetch(struct intel_plane *plane,
 	intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id), 0);
 }
 
-void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane,
-					const struct intel_crtc_state *crtc_state,
-					const struct intel_plane_state *plane_state,
-					int color_plane)
+void intel_psr2_program_plane_sel_fetch_arm(struct intel_plane *plane,
+					    const struct intel_crtc_state *crtc_state,
+					    const struct intel_plane_state *plane_state)
+{
+	struct drm_i915_private *i915 = to_i915(plane->base.dev);
+	enum pipe pipe = plane->pipe;
+
+	if (!crtc_state->enable_psr2_sel_fetch)
+		return;
+
+	if (plane->id == PLANE_CURSOR)
+		intel_de_write_fw(i915, PLANE_SEL_FETCH_CTL(pipe, plane->id),
+				  plane_state->ctl);
+	else
+		intel_de_write_fw(i915, PLANE_SEL_FETCH_CTL(pipe, plane->id),
+				  PLANE_SEL_FETCH_CTL_ENABLE);
+}
+
+void intel_psr2_program_plane_sel_fetch_noarm(struct intel_plane *plane,
+					      const struct intel_crtc_state *crtc_state,
+					      const struct intel_plane_state *plane_state,
+					      int color_plane)
 {
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	enum pipe pipe = plane->pipe;
@@ -1617,11 +1617,8 @@ void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane,
 	if (!crtc_state->enable_psr2_sel_fetch)
 		return;
 
-	if (plane->id == PLANE_CURSOR) {
-		intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id),
-				  plane_state->ctl);
+	if (plane->id == PLANE_CURSOR)
 		return;
-	}
 
 	clip = &plane_state->psr2_sel_fetch_area;
 
@@ -1649,9 +1646,6 @@ void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane,
 	val = (drm_rect_height(clip) - 1) << 16;
 	val |= (drm_rect_width(&plane_state->uapi.src) >> 16) - 1;
 	intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_SIZE(pipe, plane->id), val);
-
-	intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id),
-			  PLANE_SEL_FETCH_CTL_ENABLE);
 }
 
 void intel_psr2_program_trans_man_trk_ctl(const struct intel_crtc_state *crtc_state)
@@ -1975,14 +1969,20 @@ void intel_psr_pre_plane_update(struct intel_atomic_state *state,
 		 * - PSR disabled in new state
 		 * - All planes will go inactive
 		 * - Changing between PSR versions
+		 * - Display WA #1136: skl, bxt
 		 */
 		needs_to_disable |= intel_crtc_needs_modeset(new_crtc_state);
 		needs_to_disable |= !new_crtc_state->has_psr;
 		needs_to_disable |= !new_crtc_state->active_planes;
 		needs_to_disable |= new_crtc_state->has_psr2 != psr->psr2_enabled;
+		needs_to_disable |= DISPLAY_VER(i915) < 11 &&
+			new_crtc_state->wm_level_disabled;
 
 		if (psr->enabled && needs_to_disable)
 			intel_psr_disable_locked(intel_dp);
+		else if (psr->enabled && new_crtc_state->wm_level_disabled)
+			/* Wa_14015648006 */
+			wm_optimization_wa(intel_dp, new_crtc_state);
 
 		mutex_unlock(&psr->lock);
 	}
@@ -2001,23 +2001,29 @@ static void _intel_psr_post_plane_update(const struct intel_atomic_state *state,
 					     crtc_state->uapi.encoder_mask) {
 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 		struct intel_psr *psr = &intel_dp->psr;
+		bool keep_disabled = false;
 
 		mutex_lock(&psr->lock);
 
-		if (psr->sink_not_reliable)
-			goto exit;
-
 		drm_WARN_ON(&dev_priv->drm, psr->enabled && !crtc_state->active_planes);
 
-		/* Only enable if there is active planes */
-		if (!psr->enabled && crtc_state->active_planes)
+		keep_disabled |= psr->sink_not_reliable;
+		keep_disabled |= !crtc_state->active_planes;
+
+		/* Display WA #1136: skl, bxt */
+		keep_disabled |= DISPLAY_VER(dev_priv) < 11 &&
+			crtc_state->wm_level_disabled;
+
+		if (!psr->enabled && !keep_disabled)
 			intel_psr_enable_locked(intel_dp, crtc_state);
+		else if (psr->enabled && !crtc_state->wm_level_disabled)
+			/* Wa_14015648006 */
+			wm_optimization_wa(intel_dp, crtc_state);
 
 		/* Force a PSR exit when enabling CRC to avoid CRC timeouts */
 		if (crtc_state->crc_enabled && psr->enabled)
 			psr_force_hw_tracking_exit(intel_dp);
 
-exit:
 		mutex_unlock(&psr->lock);
 	}
 }
@@ -2691,3 +2697,302 @@ void intel_psr_unlock(const struct intel_crtc_state *crtc_state)
 		break;
 	}
 }
+
+static void
+psr_source_status(struct intel_dp *intel_dp, struct seq_file *m)
+{
+	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	const char *status = "unknown";
+	u32 val, status_val;
+
+	if (intel_dp->psr.psr2_enabled) {
+		static const char * const live_status[] = {
+			"IDLE",
+			"CAPTURE",
+			"CAPTURE_FS",
+			"SLEEP",
+			"BUFON_FW",
+			"ML_UP",
+			"SU_STANDBY",
+			"FAST_SLEEP",
+			"DEEP_SLEEP",
+			"BUF_ON",
+			"TG_ON"
+		};
+		val = intel_de_read(dev_priv,
+				    EDP_PSR2_STATUS(intel_dp->psr.transcoder));
+		status_val = REG_FIELD_GET(EDP_PSR2_STATUS_STATE_MASK, val);
+		if (status_val < ARRAY_SIZE(live_status))
+			status = live_status[status_val];
+	} else {
+		static const char * const live_status[] = {
+			"IDLE",
+			"SRDONACK",
+			"SRDENT",
+			"BUFOFF",
+			"BUFON",
+			"AUXACK",
+			"SRDOFFACK",
+			"SRDENT_ON",
+		};
+		val = intel_de_read(dev_priv,
+				    EDP_PSR_STATUS(intel_dp->psr.transcoder));
+		status_val = (val & EDP_PSR_STATUS_STATE_MASK) >>
+			      EDP_PSR_STATUS_STATE_SHIFT;
+		if (status_val < ARRAY_SIZE(live_status))
+			status = live_status[status_val];
+	}
+
+	seq_printf(m, "Source PSR status: %s [0x%08x]\n", status, val);
+}
+
+static int intel_psr_status(struct seq_file *m, struct intel_dp *intel_dp)
+{
+	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct intel_psr *psr = &intel_dp->psr;
+	intel_wakeref_t wakeref;
+	const char *status;
+	bool enabled;
+	u32 val;
+
+	seq_printf(m, "Sink support: %s", str_yes_no(psr->sink_support));
+	if (psr->sink_support)
+		seq_printf(m, " [0x%02x]", intel_dp->psr_dpcd[0]);
+	seq_puts(m, "\n");
+
+	if (!psr->sink_support)
+		return 0;
+
+	wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
+	mutex_lock(&psr->lock);
+
+	if (psr->enabled)
+		status = psr->psr2_enabled ? "PSR2 enabled" : "PSR1 enabled";
+	else
+		status = "disabled";
+	seq_printf(m, "PSR mode: %s\n", status);
+
+	if (!psr->enabled) {
+		seq_printf(m, "PSR sink not reliable: %s\n",
+			   str_yes_no(psr->sink_not_reliable));
+
+		goto unlock;
+	}
+
+	if (psr->psr2_enabled) {
+		val = intel_de_read(dev_priv,
+				    EDP_PSR2_CTL(intel_dp->psr.transcoder));
+		enabled = val & EDP_PSR2_ENABLE;
+	} else {
+		val = intel_de_read(dev_priv,
+				    EDP_PSR_CTL(intel_dp->psr.transcoder));
+		enabled = val & EDP_PSR_ENABLE;
+	}
+	seq_printf(m, "Source PSR ctl: %s [0x%08x]\n",
+		   str_enabled_disabled(enabled), val);
+	psr_source_status(intel_dp, m);
+	seq_printf(m, "Busy frontbuffer bits: 0x%08x\n",
+		   psr->busy_frontbuffer_bits);
+
+	/*
+	 * SKL+ Perf counter is reset to 0 everytime DC state is entered
+	 */
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
+		val = intel_de_read(dev_priv,
+				    EDP_PSR_PERF_CNT(intel_dp->psr.transcoder));
+		val &= EDP_PSR_PERF_CNT_MASK;
+		seq_printf(m, "Performance counter: %u\n", val);
+	}
+
+	if (psr->debug & I915_PSR_DEBUG_IRQ) {
+		seq_printf(m, "Last attempted entry at: %lld\n",
+			   psr->last_entry_attempt);
+		seq_printf(m, "Last exit at: %lld\n", psr->last_exit);
+	}
+
+	if (psr->psr2_enabled) {
+		u32 su_frames_val[3];
+		int frame;
+
+		/*
+		 * Reading all 3 registers before hand to minimize crossing a
+		 * frame boundary between register reads
+		 */
+		for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame += 3) {
+			val = intel_de_read(dev_priv,
+					    PSR2_SU_STATUS(intel_dp->psr.transcoder, frame));
+			su_frames_val[frame / 3] = val;
+		}
+
+		seq_puts(m, "Frame:\tPSR2 SU blocks:\n");
+
+		for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame++) {
+			u32 su_blocks;
+
+			su_blocks = su_frames_val[frame / 3] &
+				    PSR2_SU_STATUS_MASK(frame);
+			su_blocks = su_blocks >> PSR2_SU_STATUS_SHIFT(frame);
+			seq_printf(m, "%d\t%d\n", frame, su_blocks);
+		}
+
+		seq_printf(m, "PSR2 selective fetch: %s\n",
+			   str_enabled_disabled(psr->psr2_sel_fetch_enabled));
+	}
+
+unlock:
+	mutex_unlock(&psr->lock);
+	intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
+
+	return 0;
+}
+
+static int i915_edp_psr_status_show(struct seq_file *m, void *data)
+{
+	struct drm_i915_private *dev_priv = m->private;
+	struct intel_dp *intel_dp = NULL;
+	struct intel_encoder *encoder;
+
+	if (!HAS_PSR(dev_priv))
+		return -ENODEV;
+
+	/* Find the first EDP which supports PSR */
+	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
+		intel_dp = enc_to_intel_dp(encoder);
+		break;
+	}
+
+	if (!intel_dp)
+		return -ENODEV;
+
+	return intel_psr_status(m, intel_dp);
+}
+DEFINE_SHOW_ATTRIBUTE(i915_edp_psr_status);
+
+static int
+i915_edp_psr_debug_set(void *data, u64 val)
+{
+	struct drm_i915_private *dev_priv = data;
+	struct intel_encoder *encoder;
+	intel_wakeref_t wakeref;
+	int ret = -ENODEV;
+
+	if (!HAS_PSR(dev_priv))
+		return ret;
+
+	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
+		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+
+		drm_dbg_kms(&dev_priv->drm, "Setting PSR debug to %llx\n", val);
+
+		wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
+
+		// TODO: split to each transcoder's PSR debug state
+		ret = intel_psr_debug_set(intel_dp, val);
+
+		intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
+	}
+
+	return ret;
+}
+
+static int
+i915_edp_psr_debug_get(void *data, u64 *val)
+{
+	struct drm_i915_private *dev_priv = data;
+	struct intel_encoder *encoder;
+
+	if (!HAS_PSR(dev_priv))
+		return -ENODEV;
+
+	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
+		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+
+		// TODO: split to each transcoder's PSR debug state
+		*val = READ_ONCE(intel_dp->psr.debug);
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(i915_edp_psr_debug_fops,
+			i915_edp_psr_debug_get, i915_edp_psr_debug_set,
+			"%llu\n");
+
+void intel_psr_debugfs_register(struct drm_i915_private *i915)
+{
+	struct drm_minor *minor = i915->drm.primary;
+
+	debugfs_create_file("i915_edp_psr_debug", 0644, minor->debugfs_root,
+			    i915, &i915_edp_psr_debug_fops);
+
+	debugfs_create_file("i915_edp_psr_status", 0444, minor->debugfs_root,
+			    i915, &i915_edp_psr_status_fops);
+}
+
+static int i915_psr_sink_status_show(struct seq_file *m, void *data)
+{
+	struct intel_connector *connector = m->private;
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	static const char * const sink_status[] = {
+		"inactive",
+		"transition to active, capture and display",
+		"active, display from RFB",
+		"active, capture and display on sink device timings",
+		"transition to inactive, capture and display, timing re-sync",
+		"reserved",
+		"reserved",
+		"sink internal error",
+	};
+	const char *str;
+	int ret;
+	u8 val;
+
+	if (!CAN_PSR(intel_dp)) {
+		seq_puts(m, "PSR Unsupported\n");
+		return -ENODEV;
+	}
+
+	if (connector->base.status != connector_status_connected)
+		return -ENODEV;
+
+	ret = drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_STATUS, &val);
+	if (ret != 1)
+		return ret < 0 ? ret : -EIO;
+
+	val &= DP_PSR_SINK_STATE_MASK;
+	if (val < ARRAY_SIZE(sink_status))
+		str = sink_status[val];
+	else
+		str = "unknown";
+
+	seq_printf(m, "Sink PSR status: 0x%x [%s]\n", val, str);
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(i915_psr_sink_status);
+
+static int i915_psr_status_show(struct seq_file *m, void *data)
+{
+	struct intel_connector *connector = m->private;
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+
+	return intel_psr_status(m, intel_dp);
+}
+DEFINE_SHOW_ATTRIBUTE(i915_psr_status);
+
+void intel_psr_connector_debugfs_add(struct intel_connector *connector)
+{
+	struct drm_i915_private *i915 = to_i915(connector->base.dev);
+	struct dentry *root = connector->base.debugfs_entry;
+
+	if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
+		return;
+
+	debugfs_create_file("i915_psr_sink_status", 0444, root,
+			    connector, &i915_psr_sink_status_fops);
+
+	if (HAS_PSR(i915))
+		debugfs_create_file("i915_psr_status", 0444, root,
+				    connector, &i915_psr_status_fops);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h
index 2ac3a46cccc5..0b95e8aa615f 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.h
+++ b/drivers/gpu/drm/i915/display/intel_psr.h
@@ -13,6 +13,7 @@ struct drm_connector;
 struct drm_connector_state;
 struct drm_i915_private;
 struct intel_atomic_state;
+struct intel_connector;
 struct intel_crtc;
 struct intel_crtc_state;
 struct intel_dp;
@@ -46,16 +47,22 @@ bool intel_psr_enabled(struct intel_dp *intel_dp);
 int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 				struct intel_crtc *crtc);
 void intel_psr2_program_trans_man_trk_ctl(const struct intel_crtc_state *crtc_state);
-void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane,
-					const struct intel_crtc_state *crtc_state,
-					const struct intel_plane_state *plane_state,
-					int color_plane);
-void intel_psr2_disable_plane_sel_fetch(struct intel_plane *plane,
-					const struct intel_crtc_state *crtc_state);
+void intel_psr2_program_plane_sel_fetch_noarm(struct intel_plane *plane,
+					      const struct intel_crtc_state *crtc_state,
+					      const struct intel_plane_state *plane_state,
+					      int color_plane);
+void intel_psr2_program_plane_sel_fetch_arm(struct intel_plane *plane,
+					    const struct intel_crtc_state *crtc_state,
+					    const struct intel_plane_state *plane_state);
+
+void intel_psr2_disable_plane_sel_fetch_arm(struct intel_plane *plane,
+					    const struct intel_crtc_state *crtc_state);
 void intel_psr_pause(struct intel_dp *intel_dp);
 void intel_psr_resume(struct intel_dp *intel_dp);
 
 void intel_psr_lock(const struct intel_crtc_state *crtc_state);
 void intel_psr_unlock(const struct intel_crtc_state *crtc_state);
+void intel_psr_connector_debugfs_add(struct intel_connector *connector);
+void intel_psr_debugfs_register(struct drm_i915_private *i915);
 
 #endif /* __INTEL_PSR_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_psr_regs.h b/drivers/gpu/drm/i915/display/intel_psr_regs.h
new file mode 100644
index 000000000000..958d8cabc44b
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_psr_regs.h
@@ -0,0 +1,260 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_PSR_REGS_H__
+#define __INTEL_PSR_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+#define TRANS_EXITLINE(trans)	_MMIO_TRANS2((trans), _TRANS_EXITLINE_A)
+#define   EXITLINE_ENABLE	REG_BIT(31)
+#define   EXITLINE_MASK		REG_GENMASK(12, 0)
+#define   EXITLINE_SHIFT	0
+
+/*
+ * HSW+ eDP PSR registers
+ *
+ * HSW PSR registers are relative to DDIA(_DDI_BUF_CTL_A + 0x800) with just one
+ * instance of it
+ */
+#define _SRD_CTL_A				0x60800
+#define _SRD_CTL_EDP				0x6f800
+#define EDP_PSR_CTL(tran)			_MMIO_TRANS2(tran, _SRD_CTL_A)
+#define   EDP_PSR_ENABLE			(1 << 31)
+#define   BDW_PSR_SINGLE_FRAME			(1 << 30)
+#define   EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK	(1 << 29) /* SW can't modify */
+#define   EDP_PSR_LINK_STANDBY			(1 << 27)
+#define   EDP_PSR_MIN_LINK_ENTRY_TIME_MASK	(3 << 25)
+#define   EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES	(0 << 25)
+#define   EDP_PSR_MIN_LINK_ENTRY_TIME_4_LINES	(1 << 25)
+#define   EDP_PSR_MIN_LINK_ENTRY_TIME_2_LINES	(2 << 25)
+#define   EDP_PSR_MIN_LINK_ENTRY_TIME_0_LINES	(3 << 25)
+#define   EDP_PSR_MAX_SLEEP_TIME_SHIFT		20
+#define   EDP_PSR_SKIP_AUX_EXIT			(1 << 12)
+#define   EDP_PSR_TP1_TP2_SEL			(0 << 11)
+#define   EDP_PSR_TP1_TP3_SEL			(1 << 11)
+#define   EDP_PSR_CRC_ENABLE			(1 << 10) /* BDW+ */
+#define   EDP_PSR_TP2_TP3_TIME_500us		(0 << 8)
+#define   EDP_PSR_TP2_TP3_TIME_100us		(1 << 8)
+#define   EDP_PSR_TP2_TP3_TIME_2500us		(2 << 8)
+#define   EDP_PSR_TP2_TP3_TIME_0us		(3 << 8)
+#define   EDP_PSR_TP4_TIME_0US			(3 << 6) /* ICL+ */
+#define   EDP_PSR_TP1_TIME_500us		(0 << 4)
+#define   EDP_PSR_TP1_TIME_100us		(1 << 4)
+#define   EDP_PSR_TP1_TIME_2500us		(2 << 4)
+#define   EDP_PSR_TP1_TIME_0us			(3 << 4)
+#define   EDP_PSR_IDLE_FRAME_SHIFT		0
+
+/*
+ * Until TGL, IMR/IIR are fixed at 0x648xx. On TGL+ those registers are relative
+ * to transcoder and bits defined for each one as if using no shift (i.e. as if
+ * it was for TRANSCODER_EDP)
+ */
+#define EDP_PSR_IMR				_MMIO(0x64834)
+#define EDP_PSR_IIR				_MMIO(0x64838)
+#define _PSR_IMR_A				0x60814
+#define _PSR_IIR_A				0x60818
+#define TRANS_PSR_IMR(tran)			_MMIO_TRANS2(tran, _PSR_IMR_A)
+#define TRANS_PSR_IIR(tran)			_MMIO_TRANS2(tran, _PSR_IIR_A)
+#define   _EDP_PSR_TRANS_SHIFT(trans)		((trans) == TRANSCODER_EDP ? \
+						 0 : ((trans) - TRANSCODER_A + 1) * 8)
+#define   TGL_PSR_MASK			REG_GENMASK(2, 0)
+#define   TGL_PSR_ERROR			REG_BIT(2)
+#define   TGL_PSR_POST_EXIT		REG_BIT(1)
+#define   TGL_PSR_PRE_ENTRY		REG_BIT(0)
+#define   EDP_PSR_MASK(trans)		(TGL_PSR_MASK <<		\
+					 _EDP_PSR_TRANS_SHIFT(trans))
+#define   EDP_PSR_ERROR(trans)		(TGL_PSR_ERROR <<		\
+					 _EDP_PSR_TRANS_SHIFT(trans))
+#define   EDP_PSR_POST_EXIT(trans)	(TGL_PSR_POST_EXIT <<		\
+					 _EDP_PSR_TRANS_SHIFT(trans))
+#define   EDP_PSR_PRE_ENTRY(trans)	(TGL_PSR_PRE_ENTRY <<		\
+					 _EDP_PSR_TRANS_SHIFT(trans))
+
+#define _SRD_AUX_DATA_A				0x60814
+#define _SRD_AUX_DATA_EDP			0x6f814
+#define EDP_PSR_AUX_DATA(tran, i)		_MMIO_TRANS2(tran, _SRD_AUX_DATA_A + (i) + 4) /* 5 registers */
+
+#define _SRD_STATUS_A				0x60840
+#define _SRD_STATUS_EDP				0x6f840
+#define EDP_PSR_STATUS(tran)			_MMIO_TRANS2(tran, _SRD_STATUS_A)
+#define   EDP_PSR_STATUS_STATE_MASK		(7 << 29)
+#define   EDP_PSR_STATUS_STATE_SHIFT		29
+#define   EDP_PSR_STATUS_STATE_IDLE		(0 << 29)
+#define   EDP_PSR_STATUS_STATE_SRDONACK		(1 << 29)
+#define   EDP_PSR_STATUS_STATE_SRDENT		(2 << 29)
+#define   EDP_PSR_STATUS_STATE_BUFOFF		(3 << 29)
+#define   EDP_PSR_STATUS_STATE_BUFON		(4 << 29)
+#define   EDP_PSR_STATUS_STATE_AUXACK		(5 << 29)
+#define   EDP_PSR_STATUS_STATE_SRDOFFACK	(6 << 29)
+#define   EDP_PSR_STATUS_LINK_MASK		(3 << 26)
+#define   EDP_PSR_STATUS_LINK_FULL_OFF		(0 << 26)
+#define   EDP_PSR_STATUS_LINK_FULL_ON		(1 << 26)
+#define   EDP_PSR_STATUS_LINK_STANDBY		(2 << 26)
+#define   EDP_PSR_STATUS_MAX_SLEEP_TIMER_SHIFT	20
+#define   EDP_PSR_STATUS_MAX_SLEEP_TIMER_MASK	0x1f
+#define   EDP_PSR_STATUS_COUNT_SHIFT		16
+#define   EDP_PSR_STATUS_COUNT_MASK		0xf
+#define   EDP_PSR_STATUS_AUX_ERROR		(1 << 15)
+#define   EDP_PSR_STATUS_AUX_SENDING		(1 << 12)
+#define   EDP_PSR_STATUS_SENDING_IDLE		(1 << 9)
+#define   EDP_PSR_STATUS_SENDING_TP2_TP3	(1 << 8)
+#define   EDP_PSR_STATUS_SENDING_TP1		(1 << 4)
+#define   EDP_PSR_STATUS_IDLE_MASK		0xf
+
+#define _SRD_PERF_CNT_A			0x60844
+#define _SRD_PERF_CNT_EDP		0x6f844
+#define EDP_PSR_PERF_CNT(tran)		_MMIO_TRANS2(tran, _SRD_PERF_CNT_A)
+#define   EDP_PSR_PERF_CNT_MASK		0xffffff
+
+/* PSR_MASK on SKL+ */
+#define _SRD_DEBUG_A				0x60860
+#define _SRD_DEBUG_EDP				0x6f860
+#define EDP_PSR_DEBUG(tran)			_MMIO_TRANS2(tran, _SRD_DEBUG_A)
+#define   EDP_PSR_DEBUG_MASK_MAX_SLEEP         (1 << 28)
+#define   EDP_PSR_DEBUG_MASK_LPSP              (1 << 27)
+#define   EDP_PSR_DEBUG_MASK_MEMUP             (1 << 26)
+#define   EDP_PSR_DEBUG_MASK_HPD               (1 << 25)
+#define   EDP_PSR_DEBUG_MASK_DISP_REG_WRITE    (1 << 16) /* Reserved in ICL+ */
+#define   EDP_PSR_DEBUG_EXIT_ON_PIXEL_UNDERRUN (1 << 15) /* SKL+ */
+
+#define _PSR2_CTL_A				0x60900
+#define _PSR2_CTL_EDP				0x6f900
+#define EDP_PSR2_CTL(tran)			_MMIO_TRANS2(tran, _PSR2_CTL_A)
+#define   EDP_PSR2_ENABLE			(1 << 31)
+#define   EDP_SU_TRACK_ENABLE			(1 << 30) /* up to adl-p */
+#define   TGL_EDP_PSR2_BLOCK_COUNT_NUM_2	(0 << 28)
+#define   TGL_EDP_PSR2_BLOCK_COUNT_NUM_3	(1 << 28)
+#define   EDP_Y_COORDINATE_ENABLE		REG_BIT(25) /* display 10, 11 and 12 */
+#define   EDP_PSR2_SU_SDP_SCANLINE		REG_BIT(25) /* display 13+ */
+#define   EDP_MAX_SU_DISABLE_TIME(t)		((t) << 20)
+#define   EDP_MAX_SU_DISABLE_TIME_MASK		(0x1f << 20)
+#define   EDP_PSR2_IO_BUFFER_WAKE_MAX_LINES	8
+#define   EDP_PSR2_IO_BUFFER_WAKE(lines)	((EDP_PSR2_IO_BUFFER_WAKE_MAX_LINES - (lines)) << 13)
+#define   EDP_PSR2_IO_BUFFER_WAKE_MASK		(3 << 13)
+#define   TGL_EDP_PSR2_IO_BUFFER_WAKE_MIN_LINES	5
+#define   TGL_EDP_PSR2_IO_BUFFER_WAKE_SHIFT	13
+#define   TGL_EDP_PSR2_IO_BUFFER_WAKE(lines)	(((lines) - TGL_EDP_PSR2_IO_BUFFER_WAKE_MIN_LINES) << TGL_EDP_PSR2_IO_BUFFER_WAKE_SHIFT)
+#define   TGL_EDP_PSR2_IO_BUFFER_WAKE_MASK	(7 << 13)
+#define   EDP_PSR2_FAST_WAKE_MAX_LINES		8
+#define   EDP_PSR2_FAST_WAKE(lines)		((EDP_PSR2_FAST_WAKE_MAX_LINES - (lines)) << 11)
+#define   EDP_PSR2_FAST_WAKE_MASK		(3 << 11)
+#define   TGL_EDP_PSR2_FAST_WAKE_MIN_LINES	5
+#define   TGL_EDP_PSR2_FAST_WAKE_MIN_SHIFT	10
+#define   TGL_EDP_PSR2_FAST_WAKE(lines)		(((lines) - TGL_EDP_PSR2_FAST_WAKE_MIN_LINES) << TGL_EDP_PSR2_FAST_WAKE_MIN_SHIFT)
+#define   TGL_EDP_PSR2_FAST_WAKE_MASK		(7 << 10)
+#define   EDP_PSR2_TP2_TIME_500us		(0 << 8)
+#define   EDP_PSR2_TP2_TIME_100us		(1 << 8)
+#define   EDP_PSR2_TP2_TIME_2500us		(2 << 8)
+#define   EDP_PSR2_TP2_TIME_50us		(3 << 8)
+#define   EDP_PSR2_TP2_TIME_MASK		(3 << 8)
+#define   EDP_PSR2_FRAME_BEFORE_SU_SHIFT	4
+#define   EDP_PSR2_FRAME_BEFORE_SU_MASK		(0xf << 4)
+#define   EDP_PSR2_FRAME_BEFORE_SU(a)		((a) << 4)
+#define   EDP_PSR2_IDLE_FRAME_MASK		0xf
+#define   EDP_PSR2_IDLE_FRAME_SHIFT		0
+
+#define _PSR_EVENT_TRANS_A			0x60848
+#define _PSR_EVENT_TRANS_B			0x61848
+#define _PSR_EVENT_TRANS_C			0x62848
+#define _PSR_EVENT_TRANS_D			0x63848
+#define _PSR_EVENT_TRANS_EDP			0x6f848
+#define PSR_EVENT(tran)				_MMIO_TRANS2(tran, _PSR_EVENT_TRANS_A)
+#define  PSR_EVENT_PSR2_WD_TIMER_EXPIRE		(1 << 17)
+#define  PSR_EVENT_PSR2_DISABLED		(1 << 16)
+#define  PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN	(1 << 15)
+#define  PSR_EVENT_SU_CRC_FIFO_UNDERRUN		(1 << 14)
+#define  PSR_EVENT_GRAPHICS_RESET		(1 << 12)
+#define  PSR_EVENT_PCH_INTERRUPT		(1 << 11)
+#define  PSR_EVENT_MEMORY_UP			(1 << 10)
+#define  PSR_EVENT_FRONT_BUFFER_MODIFY		(1 << 9)
+#define  PSR_EVENT_WD_TIMER_EXPIRE		(1 << 8)
+#define  PSR_EVENT_PIPE_REGISTERS_UPDATE	(1 << 6)
+#define  PSR_EVENT_REGISTER_UPDATE		(1 << 5) /* Reserved in ICL+ */
+#define  PSR_EVENT_HDCP_ENABLE			(1 << 4)
+#define  PSR_EVENT_KVMR_SESSION_ENABLE		(1 << 3)
+#define  PSR_EVENT_VBI_ENABLE			(1 << 2)
+#define  PSR_EVENT_LPSP_MODE_EXIT		(1 << 1)
+#define  PSR_EVENT_PSR_DISABLE			(1 << 0)
+
+#define _PSR2_STATUS_A				0x60940
+#define _PSR2_STATUS_EDP			0x6f940
+#define EDP_PSR2_STATUS(tran)			_MMIO_TRANS2(tran, _PSR2_STATUS_A)
+#define EDP_PSR2_STATUS_STATE_MASK		REG_GENMASK(31, 28)
+#define EDP_PSR2_STATUS_STATE_DEEP_SLEEP	REG_FIELD_PREP(EDP_PSR2_STATUS_STATE_MASK, 0x8)
+
+#define _PSR2_SU_STATUS_A		0x60914
+#define _PSR2_SU_STATUS_EDP		0x6f914
+#define _PSR2_SU_STATUS(tran, index)	_MMIO_TRANS2(tran, _PSR2_SU_STATUS_A + (index) * 4)
+#define PSR2_SU_STATUS(tran, frame)	(_PSR2_SU_STATUS(tran, (frame) / 3))
+#define PSR2_SU_STATUS_SHIFT(frame)	(((frame) % 3) * 10)
+#define PSR2_SU_STATUS_MASK(frame)	(0x3ff << PSR2_SU_STATUS_SHIFT(frame))
+#define PSR2_SU_STATUS_FRAMES		8
+
+#define _PSR2_MAN_TRK_CTL_A					0x60910
+#define _PSR2_MAN_TRK_CTL_EDP					0x6f910
+#define PSR2_MAN_TRK_CTL(tran)					_MMIO_TRANS2(tran, _PSR2_MAN_TRK_CTL_A)
+#define  PSR2_MAN_TRK_CTL_ENABLE				REG_BIT(31)
+#define  PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK		REG_GENMASK(30, 21)
+#define  PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(val)		REG_FIELD_PREP(PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK, val)
+#define  PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK		REG_GENMASK(20, 11)
+#define  PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(val)		REG_FIELD_PREP(PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK, val)
+#define  PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME			REG_BIT(3)
+#define  PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME		REG_BIT(2)
+#define  PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE		REG_BIT(1)
+#define  ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK	REG_GENMASK(28, 16)
+#define  ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(val)	REG_FIELD_PREP(ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK, val)
+#define  ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK		REG_GENMASK(12, 0)
+#define  ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(val)		REG_FIELD_PREP(ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK, val)
+#define  ADLP_PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE		REG_BIT(31)
+#define  ADLP_PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME		REG_BIT(14)
+#define  ADLP_PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME		REG_BIT(13)
+
+#define _SEL_FETCH_PLANE_BASE_1_A		0x70890
+#define _SEL_FETCH_PLANE_BASE_2_A		0x708B0
+#define _SEL_FETCH_PLANE_BASE_3_A		0x708D0
+#define _SEL_FETCH_PLANE_BASE_4_A		0x708F0
+#define _SEL_FETCH_PLANE_BASE_5_A		0x70920
+#define _SEL_FETCH_PLANE_BASE_6_A		0x70940
+#define _SEL_FETCH_PLANE_BASE_7_A		0x70960
+#define _SEL_FETCH_PLANE_BASE_CUR_A		0x70880
+#define _SEL_FETCH_PLANE_BASE_1_B		0x71890
+
+#define _SEL_FETCH_PLANE_BASE_A(plane) _PICK(plane, \
+					     _SEL_FETCH_PLANE_BASE_1_A, \
+					     _SEL_FETCH_PLANE_BASE_2_A, \
+					     _SEL_FETCH_PLANE_BASE_3_A, \
+					     _SEL_FETCH_PLANE_BASE_4_A, \
+					     _SEL_FETCH_PLANE_BASE_5_A, \
+					     _SEL_FETCH_PLANE_BASE_6_A, \
+					     _SEL_FETCH_PLANE_BASE_7_A, \
+					     _SEL_FETCH_PLANE_BASE_CUR_A)
+#define _SEL_FETCH_PLANE_BASE_1(pipe) _PIPE(pipe, _SEL_FETCH_PLANE_BASE_1_A, _SEL_FETCH_PLANE_BASE_1_B)
+#define _SEL_FETCH_PLANE_BASE(pipe, plane) (_SEL_FETCH_PLANE_BASE_1(pipe) - \
+					    _SEL_FETCH_PLANE_BASE_1_A + \
+					    _SEL_FETCH_PLANE_BASE_A(plane))
+
+#define _SEL_FETCH_PLANE_CTL_1_A		0x70890
+#define PLANE_SEL_FETCH_CTL(pipe, plane) _MMIO(_SEL_FETCH_PLANE_BASE(pipe, plane) + \
+					       _SEL_FETCH_PLANE_CTL_1_A - \
+					       _SEL_FETCH_PLANE_BASE_1_A)
+#define PLANE_SEL_FETCH_CTL_ENABLE		REG_BIT(31)
+
+#define _SEL_FETCH_PLANE_POS_1_A		0x70894
+#define PLANE_SEL_FETCH_POS(pipe, plane) _MMIO(_SEL_FETCH_PLANE_BASE(pipe, plane) + \
+					       _SEL_FETCH_PLANE_POS_1_A - \
+					       _SEL_FETCH_PLANE_BASE_1_A)
+
+#define _SEL_FETCH_PLANE_SIZE_1_A		0x70898
+#define PLANE_SEL_FETCH_SIZE(pipe, plane) _MMIO(_SEL_FETCH_PLANE_BASE(pipe, plane) + \
+						_SEL_FETCH_PLANE_SIZE_1_A - \
+						_SEL_FETCH_PLANE_BASE_1_A)
+
+#define _SEL_FETCH_PLANE_OFFSET_1_A		0x7089C
+#define PLANE_SEL_FETCH_OFFSET(pipe, plane) _MMIO(_SEL_FETCH_PLANE_BASE(pipe, plane) + \
+						  _SEL_FETCH_PLANE_OFFSET_1_A - \
+						  _SEL_FETCH_PLANE_BASE_1_A)
+
+#endif /* __INTEL_PSR_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_qp_tables.c b/drivers/gpu/drm/i915/display/intel_qp_tables.c
index 6f8e4ec5c0fb..6e86c0971d24 100644
--- a/drivers/gpu/drm/i915/display/intel_qp_tables.c
+++ b/drivers/gpu/drm/i915/display/intel_qp_tables.c
@@ -17,6 +17,15 @@
 /* from BPP 6 to 36 in steps of 0.5 */
 #define RC_RANGE_QP444_12BPC_MAX_NUM_BPP	61
 
+/* from BPP 6 to 24 in steps of 0.5 */
+#define RC_RANGE_QP420_8BPC_MAX_NUM_BPP		17
+
+/* from BPP 6 to 30 in steps of 0.5 */
+#define RC_RANGE_QP420_10BPC_MAX_NUM_BPP	23
+
+/* from BPP 6 to 36 in steps of 0.5 */
+#define RC_RANGE_QP420_12BPC_MAX_NUM_BPP	29
+
 /*
  * These qp tables are as per the C model
  * and it has the rows pointing to bpps which increment
@@ -283,26 +292,182 @@ static const u8 rc_range_maxqp444_12bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP444_12BPC
 	  11, 11, 10, 10, 10, 10, 10, 9, 9, 8, 8, 8, 8, 8, 7, 7, 6, 6, 6, 6, 5, 5, 4 }
 };
 
-#define PARAM_TABLE(_minmax, _bpc, _row, _col)  do { \
-	if (bpc == (_bpc)) \
-		return rc_range_##_minmax##qp444_##_bpc##bpc[_row][_col]; \
+static const u8 rc_range_minqp420_8bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP420_8BPC_MAX_NUM_BPP] = {
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 3, 3, 3, 3, 3, 2, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0 },
+	{ 3, 3, 3, 3, 3, 2, 2, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 },
+	{ 3, 3, 3, 3, 3, 3, 2, 2, 1, 1, 1, 1, 1, 1, 1, 0, 0 },
+	{ 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 0 },
+	{ 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 0 },
+	{ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 1, 1 },
+	{ 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 3, 3, 3, 3, 2, 1, 1 },
+	{ 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 3, 2, 2, 1 },
+	{ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 3, 3, 2, 1 },
+	{ 9, 8, 8, 7, 7, 7, 7, 7, 7, 6, 5, 5, 4, 3, 3, 3, 2 },
+	{ 13, 12, 12, 11, 10, 10, 9, 8, 8, 7, 6, 6, 5, 5, 4, 4, 3 }
+};
+
+static const u8 rc_range_maxqp420_8bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP420_8BPC_MAX_NUM_BPP] = {
+	{ 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 4, 4, 4, 4, 4, 3, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0 },
+	{ 5, 5, 5, 5, 5, 4, 3, 2, 1, 1, 1, 1, 1, 1, 0, 0, 0 },
+	{ 6, 6, 6, 6, 6, 5, 4, 3, 2, 2, 2, 1, 1, 1, 1, 0, 0 },
+	{ 7, 7, 7, 7, 7, 5, 4, 3, 2, 2, 2, 2, 2, 1, 1, 1, 0 },
+	{ 7, 7, 7, 7, 7, 6, 5, 4, 3, 3, 3, 2, 2, 2, 1, 1, 0 },
+	{ 7, 7, 7, 7, 7, 6, 5, 4, 3, 3, 3, 3, 2, 2, 2, 1, 1 },
+	{ 8, 8, 8, 8, 8, 7, 6, 5, 4, 4, 4, 3, 3, 2, 2, 2, 1 },
+	{ 9, 9, 9, 8, 8, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1 },
+	{ 10, 10, 9, 9, 9, 8, 7, 6, 5, 5, 5, 4, 4, 3, 3, 2, 2 },
+	{ 10, 10, 10, 9, 9, 8, 8, 7, 6, 6, 5, 5, 4, 4, 3, 2, 2 },
+	{ 11, 11, 10, 10, 9, 9, 8, 7, 7, 6, 6, 5, 5, 4, 3, 3, 2 },
+	{ 11, 11, 11, 10, 9, 9, 9, 8, 7, 7, 6, 5, 5, 4, 4, 3, 2 },
+	{ 13, 12, 12, 11, 10, 10, 9, 8, 8, 7, 6, 6, 5, 4, 4, 4, 3 },
+	{ 14, 13, 13, 12, 11, 11, 10, 9, 9, 8, 7, 7, 6, 6, 5, 5, 4 }
+};
+
+static const u8 rc_range_minqp420_10bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP420_10BPC_MAX_NUM_BPP] = {
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 4, 4, 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 4, 4, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 5, 5, 5, 4, 4, 4, 4, 4, 4, 3, 3, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0 },
+	{ 7, 7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 3, 3, 2, 2, 2, 1, 1, 1, 0, 0, 0, 0 },
+	{ 7, 7, 7, 7, 7, 6, 5, 5, 5, 5, 5, 4, 3, 3, 2, 2, 1, 1, 1, 1, 1, 0, 0 },
+	{ 7, 7, 7, 7, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 3, 2, 2, 2, 2, 1, 1, 1, 0 },
+	{ 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 5, 4, 4, 4, 3, 2, 2, 2, 1, 1, 1, 0 },
+	{ 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 2, 1, 1 },
+	{ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1 },
+	{ 9, 9, 9, 9, 9, 8, 8, 8, 8, 8, 7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1 },
+	{ 9, 9, 9, 9, 9, 9, 8, 8, 8, 8, 8, 8, 8, 7, 6, 6, 5, 4, 4, 3, 3, 2, 1 },
+	{ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8, 8, 7, 7, 6, 5, 4, 4, 3, 3, 2, 1 },
+	{ 13, 12, 12, 11, 11, 11, 11, 11, 11, 10, 9, 9, 8, 7, 7, 6, 5, 5, 4, 3, 3,
+	  2, 2 },
+	{ 17, 16, 16, 15, 14, 14, 13, 12, 12, 11, 10, 10, 10, 9, 8, 8, 7, 6, 6, 5,
+	  5, 4, 4 }
+};
+
+static const u8 rc_range_maxqp420_10bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP420_10BPC_MAX_NUM_BPP] = {
+	{ 8, 8, 7, 6, 4, 4, 3, 3, 2, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 8, 8, 8, 7, 6, 5, 4, 4, 3, 3, 3, 3, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0 },
+	{ 9, 9, 9, 8, 8, 7, 6, 5, 4, 3, 3, 3, 3, 3, 2, 1, 1, 1, 0, 0, 0, 0, 0 },
+	{ 10, 10, 10, 9, 9, 8, 7, 6, 5, 4, 4, 3, 3, 3, 3, 2, 1, 1, 1, 1, 1, 0,
+	  0 },
+	{ 11, 11, 11, 10, 10, 8, 7, 6, 5, 4, 4, 4, 4, 3, 3, 3, 2, 2, 2, 1, 1, 1,
+	  0 },
+	{ 11, 11, 11, 10, 10, 9, 8, 7, 6, 6, 6, 5, 4, 4, 3, 3, 2, 2, 2, 2, 2, 1,
+	  1 },
+	{ 11, 11, 11, 11, 11, 10, 9, 8, 7, 7, 7, 6, 5, 5, 4, 3, 3, 3, 3, 2, 2, 2,
+	  1 },
+	{ 12, 12, 12, 12, 12, 11, 10, 9, 8, 8, 8, 7, 6, 5, 5, 4, 3, 3, 3, 2, 2,
+	  2, 1 },
+	{ 13, 13, 13, 12, 12, 11, 10, 10, 9, 9, 8, 8, 7, 7, 6, 5, 4, 4, 3, 3, 3,
+	  2, 2 },
+	{ 14, 14, 13, 13, 13, 12, 11, 10, 9, 9, 9, 8, 8, 7, 7, 6, 5, 4, 4, 3, 3,
+	  2, 2 },
+	{ 14, 14, 14, 13, 13, 12, 12, 11, 10, 10, 9, 9, 8, 8, 7, 6, 5, 5, 4, 4,
+	  3, 3, 2 },
+	{ 15, 15, 14, 14, 13, 13, 12, 11, 11, 10, 10, 9, 9, 8, 7, 7, 6, 5, 5, 4,
+	  4, 3, 2 },
+	{ 15, 15, 15, 14, 13, 13, 13, 12, 11, 11, 10, 9, 9, 8, 8, 7, 6, 5, 5, 4,
+	  4, 3, 2 },
+	{ 17, 16, 16, 15, 14, 14, 13, 12, 12, 11, 10, 10, 9, 8, 8, 7, 6, 6, 5, 4,
+	  4, 3, 3 },
+	{ 18, 17, 17, 16, 15, 15, 14, 13, 13, 12, 11, 11, 11, 10, 9, 9, 8, 7, 7,
+	  6, 6, 5, 5 }
+};
+
+static const u8 rc_range_minqp420_12bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP420_12BPC_MAX_NUM_BPP] = {
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	  0, 0, 0, 0, 0 },
+	{ 4, 4, 4, 4, 4, 4, 3, 3, 2, 2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0,
+	  0, 0, 0, 0, 0 },
+	{ 9, 8, 8, 7, 7, 6, 5, 5, 4, 4, 4, 4, 3, 3, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0,
+	  0, 0, 0, 0, 0 },
+	{ 10, 9, 9, 8, 8, 8, 7, 7, 6, 6, 6, 5, 5, 4, 4, 3, 2, 2, 1, 1, 1, 0, 0, 0,
+	  0, 0, 0, 0, 0 },
+	{ 11, 10, 10, 10, 10, 9, 9, 8, 7, 6, 6, 6, 6, 5, 5, 4, 3, 3, 3, 2, 2, 1,
+	  0, 0, 0, 0, 0, 0, 0 },
+	{ 11, 11, 11, 11, 11, 10, 10, 9, 9, 9, 9, 8, 7, 6, 5, 5, 4, 4, 3, 3, 3, 2,
+	  1, 1, 0, 0, 0, 0, 0 },
+	{ 11, 11, 11, 11, 11, 11, 10, 10, 9, 9, 9, 8, 8, 7, 6, 5, 5, 5, 5, 4, 3, 3,
+	  2, 1, 1, 1, 1, 1, 0 },
+	{ 11, 11, 11, 11, 11, 11, 11, 10, 10, 10, 10, 9, 8, 8, 8, 7, 6, 6, 5, 4, 4,
+	  3, 2, 2, 1, 1, 1, 1, 1 },
+	{ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, 6, 5,
+	  5, 4, 4, 2, 2, 1, 1, 1, 1 },
+	{ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, 6,
+	  5, 4, 4, 3, 2, 2, 1, 1, 1 },
+	{ 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7,
+	  6, 5, 4, 3, 3, 2, 2, 1, 1 },
+	{ 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 11, 10, 10, 9, 8, 8,
+	  7, 7, 6, 5, 4, 3, 3, 2, 2, 1 },
+	{ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 11, 11, 10, 9, 8, 8,
+	  7, 7, 6, 5, 4, 4, 3, 2, 2, 1 },
+	{ 15, 15, 15, 15, 15, 15, 15, 15, 15, 14, 13, 13, 12, 11, 11, 10, 9, 9, 8,
+	  8, 7, 6, 6, 5, 4, 4, 3, 3, 2 },
+	{ 21, 20, 20, 19, 18, 18, 17, 16, 16, 15, 14, 14, 14, 13, 12, 12, 11, 10,
+	  10, 10, 9, 8, 8, 7, 6, 6, 5, 5, 4 }
+};
+
+static const u8 rc_range_maxqp420_12bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP420_12BPC_MAX_NUM_BPP] = {
+	{ 11, 10, 9, 8, 6, 6, 5, 5, 4, 3, 3, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 0, 0,
+	  0, 0, 0, 0, 0, 0 },
+	{ 12, 11, 11, 10, 9, 8, 7, 7, 6, 6, 5, 5, 4, 3, 3, 2, 1, 1, 1, 1, 1, 1,
+	  1, 0, 0, 0, 0, 0, 0 },
+	{ 13, 12, 12, 11, 11, 10, 9, 8, 7, 6, 6, 6, 5, 5, 4, 3, 3, 2, 1, 1, 1, 1,
+	  1, 0, 0, 0, 0, 0, 0 },
+	{ 14, 13, 13, 12, 12, 11, 10, 9, 8, 7, 7, 6, 6, 5, 5, 4, 3, 3, 2, 2, 2, 1,
+	  1, 1, 0, 0, 0, 0, 0 },
+	{ 15, 14, 14, 13, 13, 11, 10, 9, 8, 7, 7, 7, 7, 6, 6, 5, 4, 4, 4, 3, 3, 2,
+	  1, 1, 1, 0, 0, 0, 0 },
+	{ 15, 15, 15, 14, 14, 13, 12, 11, 10, 10, 10, 9, 8, 7, 6, 6, 5, 5, 4, 4,
+	  4, 3, 2, 2, 1, 1, 0, 0, 0 },
+	{ 15, 15, 15, 15, 15, 14, 13, 12, 11, 11, 11, 10, 9, 8, 7, 6, 6, 6, 6, 5,
+	  4, 4, 3, 2, 2, 2, 1, 1, 0 },
+	{ 16, 16, 16, 16, 16, 15, 14, 13, 12, 12, 12, 11, 10, 9, 9, 8, 7, 7, 6, 5,
+	  5, 4, 3, 3, 2, 2, 2, 1, 1 },
+	{ 17, 17, 17, 16, 16, 15, 14, 14, 13, 13, 12, 12, 11, 11, 10, 9, 8, 8, 7,
+	  6, 6, 5, 5, 3, 3, 2, 2, 1, 1 },
+	{ 18, 18, 17, 17, 17, 16, 15, 14, 13, 13, 13, 12, 12, 11, 11, 10, 9, 8, 8,
+	  7, 6, 5, 5, 4, 3, 3, 2, 2, 1 },
+	{ 18, 18, 18, 17, 17, 16, 16, 15, 14, 14, 13, 13, 12, 12, 11, 10, 9, 9, 8,
+	  8, 7, 6, 5, 4, 4, 3, 3, 2, 2 },
+	{ 19, 19, 18, 18, 17, 17, 16, 15, 15, 14, 14, 13, 13, 12, 11, 11, 10, 9,
+	  9, 8, 8, 7, 6, 5, 4, 4, 3, 3, 2 },
+	{ 19, 19, 19, 18, 17, 17, 17, 16, 15, 15, 14, 13, 13, 12, 12, 11, 10, 9,
+	  9, 8, 8, 7, 6, 5, 5, 4, 3, 3, 2 },
+	{ 21, 20, 20, 19, 18, 18, 17, 16, 16, 15, 14, 14, 13, 12, 12, 11, 10, 10,
+	  9, 9, 8, 7, 7, 6, 5, 5, 4, 4, 3 },
+	{ 22, 21, 21, 20, 19, 19, 18, 17, 17, 16, 15, 15, 15, 14, 13, 13, 12, 11,
+	  11, 11, 10, 9, 9, 8, 7, 7, 6, 6, 5 }
+};
+
+#define PARAM_TABLE(_minmax, _bpc, _row, _col, _is_420)  do { \
+	if (bpc == (_bpc)) {	\
+		if (_is_420)	\
+			return rc_range_##_minmax##qp420_##_bpc##bpc[_row][_col]; \
+		else	\
+			return rc_range_##_minmax##qp444_##_bpc##bpc[_row][_col]; \
+	}	\
 } while (0)
 
-u8 intel_lookup_range_min_qp(int bpc, int buf_i, int bpp_i)
+u8 intel_lookup_range_min_qp(int bpc, int buf_i, int bpp_i, bool is_420)
 {
-	PARAM_TABLE(min, 8, buf_i, bpp_i);
-	PARAM_TABLE(min, 10, buf_i, bpp_i);
-	PARAM_TABLE(min, 12, buf_i, bpp_i);
+	PARAM_TABLE(min, 8, buf_i, bpp_i, is_420);
+	PARAM_TABLE(min, 10, buf_i, bpp_i, is_420);
+	PARAM_TABLE(min, 12, buf_i, bpp_i, is_420);
 
 	MISSING_CASE(bpc);
 	return 0;
 }
 
-u8 intel_lookup_range_max_qp(int bpc, int buf_i, int bpp_i)
+u8 intel_lookup_range_max_qp(int bpc, int buf_i, int bpp_i, bool is_420)
 {
-	PARAM_TABLE(max, 8, buf_i, bpp_i);
-	PARAM_TABLE(max, 10, buf_i, bpp_i);
-	PARAM_TABLE(max, 12, buf_i, bpp_i);
+	PARAM_TABLE(max, 8, buf_i, bpp_i, is_420);
+	PARAM_TABLE(max, 10, buf_i, bpp_i, is_420);
+	PARAM_TABLE(max, 12, buf_i, bpp_i, is_420);
 
 	MISSING_CASE(bpc);
 	return 0;
diff --git a/drivers/gpu/drm/i915/display/intel_qp_tables.h b/drivers/gpu/drm/i915/display/intel_qp_tables.h
index 9fb3c36bd7c6..a9ff9ca29938 100644
--- a/drivers/gpu/drm/i915/display/intel_qp_tables.h
+++ b/drivers/gpu/drm/i915/display/intel_qp_tables.h
@@ -8,7 +8,7 @@
 
 #include <linux/types.h>
 
-u8 intel_lookup_range_min_qp(int bpc, int buf_i, int bpp_i);
-u8 intel_lookup_range_max_qp(int bpc, int buf_i, int bpp_i);
+u8 intel_lookup_range_min_qp(int bpc, int buf_i, int bpp_i, bool is_420);
+u8 intel_lookup_range_max_qp(int bpc, int buf_i, int bpp_i, bool is_420);
 
 #endif
diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c
index e6b4d24b9cd0..25034bbf1445 100644
--- a/drivers/gpu/drm/i915/display/intel_sprite.c
+++ b/drivers/gpu/drm/i915/display/intel_sprite.c
@@ -32,85 +32,20 @@
 
 #include <linux/string_helpers.h>
 
-#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_color_mgmt.h>
-#include <drm/drm_crtc.h>
-#include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_rect.h>
 
 #include "i915_drv.h"
 #include "i915_reg.h"
-#include "i915_vgpu.h"
 #include "i9xx_plane.h"
 #include "intel_atomic_plane.h"
-#include "intel_crtc.h"
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_fb.h"
-#include "intel_frontbuffer.h"
 #include "intel_sprite.h"
-#include "intel_vrr.h"
-
-int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state)
-{
-	struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
-	const struct drm_framebuffer *fb = plane_state->hw.fb;
-	struct drm_rect *src = &plane_state->uapi.src;
-	u32 src_x, src_y, src_w, src_h, hsub, vsub;
-	bool rotated = drm_rotation_90_or_270(plane_state->hw.rotation);
-
-	/*
-	 * FIXME hsub/vsub vs. block size is a mess. Pre-tgl CCS
-	 * abuses hsub/vsub so we can't use them here. But as they
-	 * are limited to 32bpp RGB formats we don't actually need
-	 * to check anything.
-	 */
-	if (fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
-	    fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS)
-		return 0;
-
-	/*
-	 * Hardware doesn't handle subpixel coordinates.
-	 * Adjust to (macro)pixel boundary, but be careful not to
-	 * increase the source viewport size, because that could
-	 * push the downscaling factor out of bounds.
-	 */
-	src_x = src->x1 >> 16;
-	src_w = drm_rect_width(src) >> 16;
-	src_y = src->y1 >> 16;
-	src_h = drm_rect_height(src) >> 16;
-
-	drm_rect_init(src, src_x << 16, src_y << 16,
-		      src_w << 16, src_h << 16);
-
-	if (fb->format->format == DRM_FORMAT_RGB565 && rotated) {
-		hsub = 2;
-		vsub = 2;
-	} else {
-		hsub = fb->format->hsub;
-		vsub = fb->format->vsub;
-	}
-
-	if (rotated)
-		hsub = vsub = max(hsub, vsub);
-
-	if (src_x % hsub || src_w % hsub) {
-		drm_dbg_kms(&i915->drm, "src x/w (%u, %u) must be a multiple of %u (rotated: %s)\n",
-			    src_x, src_w, hsub, str_yes_no(rotated));
-		return -EINVAL;
-	}
-
-	if (src_y % vsub || src_h % vsub) {
-		drm_dbg_kms(&i915->drm, "src y/h (%u, %u) must be a multiple of %u (rotated: %s)\n",
-			    src_y, src_h, vsub, str_yes_no(rotated));
-		return -EINVAL;
-	}
-
-	return 0;
-}
 
 static void i9xx_plane_linear_gamma(u16 gamma[8])
 {
@@ -1217,7 +1152,8 @@ g4x_sprite_update_arm(struct intel_plane *plane,
 	}
 
 	intel_de_write_fw(dev_priv, DVSLINOFF(pipe), linear_offset);
-	intel_de_write_fw(dev_priv, DVSTILEOFF(pipe), (y << 16) | x);
+	intel_de_write_fw(dev_priv, DVSTILEOFF(pipe),
+			  DVS_OFFSET_Y(y) | DVS_OFFSET_X(x));
 
 	/*
 	 * The control register self-arms if the plane was previously
@@ -1448,124 +1384,6 @@ vlv_sprite_check(struct intel_crtc_state *crtc_state,
 	return 0;
 }
 
-static bool has_dst_key_in_primary_plane(struct drm_i915_private *dev_priv)
-{
-	return DISPLAY_VER(dev_priv) >= 9;
-}
-
-static void intel_plane_set_ckey(struct intel_plane_state *plane_state,
-				 const struct drm_intel_sprite_colorkey *set)
-{
-	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
-	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
-	struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
-
-	*key = *set;
-
-	/*
-	 * We want src key enabled on the
-	 * sprite and not on the primary.
-	 */
-	if (plane->id == PLANE_PRIMARY &&
-	    set->flags & I915_SET_COLORKEY_SOURCE)
-		key->flags = 0;
-
-	/*
-	 * On SKL+ we want dst key enabled on
-	 * the primary and not on the sprite.
-	 */
-	if (DISPLAY_VER(dev_priv) >= 9 && plane->id != PLANE_PRIMARY &&
-	    set->flags & I915_SET_COLORKEY_DESTINATION)
-		key->flags = 0;
-}
-
-int intel_sprite_set_colorkey_ioctl(struct drm_device *dev, void *data,
-				    struct drm_file *file_priv)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_intel_sprite_colorkey *set = data;
-	struct drm_plane *plane;
-	struct drm_plane_state *plane_state;
-	struct drm_atomic_state *state;
-	struct drm_modeset_acquire_ctx ctx;
-	int ret = 0;
-
-	/* ignore the pointless "none" flag */
-	set->flags &= ~I915_SET_COLORKEY_NONE;
-
-	if (set->flags & ~(I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE))
-		return -EINVAL;
-
-	/* Make sure we don't try to enable both src & dest simultaneously */
-	if ((set->flags & (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE)) == (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE))
-		return -EINVAL;
-
-	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
-	    set->flags & I915_SET_COLORKEY_DESTINATION)
-		return -EINVAL;
-
-	plane = drm_plane_find(dev, file_priv, set->plane_id);
-	if (!plane || plane->type != DRM_PLANE_TYPE_OVERLAY)
-		return -ENOENT;
-
-	/*
-	 * SKL+ only plane 2 can do destination keying against plane 1.
-	 * Also multiple planes can't do destination keying on the same
-	 * pipe simultaneously.
-	 */
-	if (DISPLAY_VER(dev_priv) >= 9 &&
-	    to_intel_plane(plane)->id >= PLANE_SPRITE1 &&
-	    set->flags & I915_SET_COLORKEY_DESTINATION)
-		return -EINVAL;
-
-	drm_modeset_acquire_init(&ctx, 0);
-
-	state = drm_atomic_state_alloc(plane->dev);
-	if (!state) {
-		ret = -ENOMEM;
-		goto out;
-	}
-	state->acquire_ctx = &ctx;
-
-	while (1) {
-		plane_state = drm_atomic_get_plane_state(state, plane);
-		ret = PTR_ERR_OR_ZERO(plane_state);
-		if (!ret)
-			intel_plane_set_ckey(to_intel_plane_state(plane_state), set);
-
-		/*
-		 * On some platforms we have to configure
-		 * the dst colorkey on the primary plane.
-		 */
-		if (!ret && has_dst_key_in_primary_plane(dev_priv)) {
-			struct intel_crtc *crtc =
-				intel_crtc_for_pipe(dev_priv,
-						    to_intel_plane(plane)->pipe);
-
-			plane_state = drm_atomic_get_plane_state(state,
-								 crtc->base.primary);
-			ret = PTR_ERR_OR_ZERO(plane_state);
-			if (!ret)
-				intel_plane_set_ckey(to_intel_plane_state(plane_state), set);
-		}
-
-		if (!ret)
-			ret = drm_atomic_commit(state);
-
-		if (ret != -EDEADLK)
-			break;
-
-		drm_atomic_state_clear(state);
-		drm_modeset_backoff(&ctx);
-	}
-
-	drm_atomic_state_put(state);
-out:
-	drm_modeset_drop_locks(&ctx);
-	drm_modeset_acquire_fini(&ctx);
-	return ret;
-}
-
 static const u32 g4x_sprite_formats[] = {
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_YUYV,
diff --git a/drivers/gpu/drm/i915/display/intel_sprite_uapi.c b/drivers/gpu/drm/i915/display/intel_sprite_uapi.c
new file mode 100644
index 000000000000..70a391083751
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_sprite_uapi.c
@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#include "i915_drv.h"
+#include "intel_crtc.h"
+#include "intel_display_types.h"
+#include "intel_sprite_uapi.h"
+
+static bool has_dst_key_in_primary_plane(struct drm_i915_private *dev_priv)
+{
+	return DISPLAY_VER(dev_priv) >= 9;
+}
+
+static void intel_plane_set_ckey(struct intel_plane_state *plane_state,
+				 const struct drm_intel_sprite_colorkey *set)
+{
+	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
+
+	*key = *set;
+
+	/*
+	 * We want src key enabled on the
+	 * sprite and not on the primary.
+	 */
+	if (plane->id == PLANE_PRIMARY &&
+	    set->flags & I915_SET_COLORKEY_SOURCE)
+		key->flags = 0;
+
+	/*
+	 * On SKL+ we want dst key enabled on
+	 * the primary and not on the sprite.
+	 */
+	if (DISPLAY_VER(dev_priv) >= 9 && plane->id != PLANE_PRIMARY &&
+	    set->flags & I915_SET_COLORKEY_DESTINATION)
+		key->flags = 0;
+}
+
+int intel_sprite_set_colorkey_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file_priv)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_intel_sprite_colorkey *set = data;
+	struct drm_plane *plane;
+	struct drm_plane_state *plane_state;
+	struct drm_atomic_state *state;
+	struct drm_modeset_acquire_ctx ctx;
+	int ret = 0;
+
+	/* ignore the pointless "none" flag */
+	set->flags &= ~I915_SET_COLORKEY_NONE;
+
+	if (set->flags & ~(I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE))
+		return -EINVAL;
+
+	/* Make sure we don't try to enable both src & dest simultaneously */
+	if ((set->flags & (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE)) == (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE))
+		return -EINVAL;
+
+	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
+	    set->flags & I915_SET_COLORKEY_DESTINATION)
+		return -EINVAL;
+
+	plane = drm_plane_find(dev, file_priv, set->plane_id);
+	if (!plane || plane->type != DRM_PLANE_TYPE_OVERLAY)
+		return -ENOENT;
+
+	/*
+	 * SKL+ only plane 2 can do destination keying against plane 1.
+	 * Also multiple planes can't do destination keying on the same
+	 * pipe simultaneously.
+	 */
+	if (DISPLAY_VER(dev_priv) >= 9 &&
+	    to_intel_plane(plane)->id >= PLANE_SPRITE1 &&
+	    set->flags & I915_SET_COLORKEY_DESTINATION)
+		return -EINVAL;
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	state = drm_atomic_state_alloc(plane->dev);
+	if (!state) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	state->acquire_ctx = &ctx;
+
+	while (1) {
+		plane_state = drm_atomic_get_plane_state(state, plane);
+		ret = PTR_ERR_OR_ZERO(plane_state);
+		if (!ret)
+			intel_plane_set_ckey(to_intel_plane_state(plane_state), set);
+
+		/*
+		 * On some platforms we have to configure
+		 * the dst colorkey on the primary plane.
+		 */
+		if (!ret && has_dst_key_in_primary_plane(dev_priv)) {
+			struct intel_crtc *crtc =
+				intel_crtc_for_pipe(dev_priv,
+						    to_intel_plane(plane)->pipe);
+
+			plane_state = drm_atomic_get_plane_state(state,
+								 crtc->base.primary);
+			ret = PTR_ERR_OR_ZERO(plane_state);
+			if (!ret)
+				intel_plane_set_ckey(to_intel_plane_state(plane_state), set);
+		}
+
+		if (!ret)
+			ret = drm_atomic_commit(state);
+
+		if (ret != -EDEADLK)
+			break;
+
+		drm_atomic_state_clear(state);
+		drm_modeset_backoff(&ctx);
+	}
+
+	drm_atomic_state_put(state);
+out:
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+	return ret;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_sprite_uapi.h b/drivers/gpu/drm/i915/display/intel_sprite_uapi.h
new file mode 100644
index 000000000000..3eb50025acaf
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_sprite_uapi.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_SPRITE_UAPI_H__
+#define __INTEL_SPRITE_UAPI_H__
+
+struct drm_device;
+struct drm_file;
+
+int intel_sprite_set_colorkey_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file_priv);
+
+#endif /* __INTEL_SPRITE_UAPI_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c
index be510b9c0d07..3b60995e9dfb 100644
--- a/drivers/gpu/drm/i915/display/intel_tc.c
+++ b/drivers/gpu/drm/i915/display/intel_tc.c
@@ -5,6 +5,7 @@
 
 #include "i915_drv.h"
 #include "i915_reg.h"
+#include "intel_ddi.h"
 #include "intel_de.h"
 #include "intel_display.h"
 #include "intel_display_power_map.h"
@@ -14,6 +15,52 @@
 #include "intel_mg_phy_regs.h"
 #include "intel_tc.h"
 
+enum tc_port_mode {
+	TC_PORT_DISCONNECTED,
+	TC_PORT_TBT_ALT,
+	TC_PORT_DP_ALT,
+	TC_PORT_LEGACY,
+};
+
+struct intel_tc_port;
+
+struct intel_tc_phy_ops {
+	enum intel_display_power_domain (*cold_off_domain)(struct intel_tc_port *tc);
+	u32 (*hpd_live_status)(struct intel_tc_port *tc);
+	bool (*is_ready)(struct intel_tc_port *tc);
+	bool (*is_owned)(struct intel_tc_port *tc);
+	void (*get_hw_state)(struct intel_tc_port *tc);
+	bool (*connect)(struct intel_tc_port *tc, int required_lanes);
+	void (*disconnect)(struct intel_tc_port *tc);
+	void (*init)(struct intel_tc_port *tc);
+};
+
+struct intel_tc_port {
+	struct intel_digital_port *dig_port;
+
+	const struct intel_tc_phy_ops *phy_ops;
+
+	struct mutex lock;	/* protects the TypeC port mode */
+	intel_wakeref_t lock_wakeref;
+#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
+	enum intel_display_power_domain lock_power_domain;
+#endif
+	struct delayed_work disconnect_phy_work;
+	int link_refcount;
+	bool legacy_port:1;
+	char port_name[8];
+	enum tc_port_mode mode;
+	enum tc_port_mode init_mode;
+	enum phy_fia phy_fia;
+	u8 phy_fia_idx;
+};
+
+static enum intel_display_power_domain
+tc_phy_cold_off_domain(struct intel_tc_port *);
+static u32 tc_phy_hpd_live_status(struct intel_tc_port *tc);
+static bool tc_phy_is_ready(struct intel_tc_port *tc);
+static enum tc_port_mode tc_phy_get_current_mode(struct intel_tc_port *tc);
+
 static const char *tc_port_mode_name(enum tc_port_mode mode)
 {
 	static const char * const names[] = {
@@ -29,13 +76,24 @@ static const char *tc_port_mode_name(enum tc_port_mode mode)
 	return names[mode];
 }
 
+static struct intel_tc_port *to_tc_port(struct intel_digital_port *dig_port)
+{
+	return dig_port->tc;
+}
+
+static struct drm_i915_private *tc_to_i915(struct intel_tc_port *tc)
+{
+	return to_i915(tc->dig_port->base.base.dev);
+}
+
 static bool intel_tc_port_in_mode(struct intel_digital_port *dig_port,
 				  enum tc_port_mode mode)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
 	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
+	struct intel_tc_port *tc = to_tc_port(dig_port);
 
-	return intel_phy_is_tc(i915, phy) && dig_port->tc_mode == mode;
+	return intel_phy_is_tc(i915, phy) && tc->mode == mode;
 }
 
 bool intel_tc_port_in_tbt_alt_mode(struct intel_digital_port *dig_port)
@@ -53,109 +111,178 @@ bool intel_tc_port_in_legacy_mode(struct intel_digital_port *dig_port)
 	return intel_tc_port_in_mode(dig_port, TC_PORT_LEGACY);
 }
 
+/*
+ * The display power domains used for TC ports depending on the
+ * platform and TC mode (legacy, DP-alt, TBT):
+ *
+ * POWER_DOMAIN_DISPLAY_CORE:
+ * --------------------------
+ * ADLP/all modes:
+ *   - TCSS/IOM access for PHY ready state.
+ * ADLP+/all modes:
+ *   - DE/north-,south-HPD ISR access for HPD live state.
+ *
+ * POWER_DOMAIN_PORT_DDI_LANES_<port>:
+ * -----------------------------------
+ * ICL+/all modes:
+ *   - DE/DDI_BUF access for port enabled state.
+ * ADLP/all modes:
+ *   - DE/DDI_BUF access for PHY owned state.
+ *
+ * POWER_DOMAIN_AUX_USBC<TC port index>:
+ * -------------------------------------
+ * ICL/legacy mode:
+ *   - TCSS/IOM,FIA access for PHY ready, owned and HPD live state
+ *   - TCSS/PHY: block TC-cold power state for using the PHY AUX and
+ *     main lanes.
+ * ADLP/legacy, DP-alt modes:
+ *   - TCSS/PHY: block TC-cold power state for using the PHY AUX and
+ *     main lanes.
+ *
+ * POWER_DOMAIN_TC_COLD_OFF:
+ * -------------------------
+ * TGL/legacy, DP-alt modes:
+ *   - TCSS/IOM,FIA access for PHY ready, owned and HPD live state
+ *   - TCSS/PHY: block TC-cold power state for using the PHY AUX and
+ *     main lanes.
+ *
+ * ICL, TGL, ADLP/TBT mode:
+ *   - TCSS/IOM,FIA access for HPD live state
+ *   - TCSS/TBT: block TC-cold power state for using the (TBT DP-IN)
+ *     AUX and main lanes.
+ */
 bool intel_tc_cold_requires_aux_pw(struct intel_digital_port *dig_port)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct intel_tc_port *tc = to_tc_port(dig_port);
 
-	return (DISPLAY_VER(i915) == 11 && dig_port->tc_legacy_port) ||
-		IS_ALDERLAKE_P(i915);
+	return tc_phy_cold_off_domain(tc) ==
+	       intel_display_power_legacy_aux_domain(i915, dig_port->aux_ch);
 }
 
-static enum intel_display_power_domain
-tc_cold_get_power_domain(struct intel_digital_port *dig_port, enum tc_port_mode mode)
+static intel_wakeref_t
+__tc_cold_block(struct intel_tc_port *tc, enum intel_display_power_domain *domain)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct drm_i915_private *i915 = tc_to_i915(tc);
 
-	if (mode == TC_PORT_TBT_ALT || !intel_tc_cold_requires_aux_pw(dig_port))
-		return POWER_DOMAIN_TC_COLD_OFF;
+	*domain = tc_phy_cold_off_domain(tc);
 
-	return intel_display_power_legacy_aux_domain(i915, dig_port->aux_ch);
+	return intel_display_power_get(i915, *domain);
 }
 
 static intel_wakeref_t
-tc_cold_block_in_mode(struct intel_digital_port *dig_port, enum tc_port_mode mode,
-		      enum intel_display_power_domain *domain)
+tc_cold_block(struct intel_tc_port *tc)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-
-	*domain = tc_cold_get_power_domain(dig_port, mode);
+	enum intel_display_power_domain domain;
+	intel_wakeref_t wakeref;
 
-	return intel_display_power_get(i915, *domain);
+	wakeref = __tc_cold_block(tc, &domain);
+#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
+	tc->lock_power_domain = domain;
+#endif
+	return wakeref;
 }
 
-static intel_wakeref_t
-tc_cold_block(struct intel_digital_port *dig_port, enum intel_display_power_domain *domain)
+static void
+__tc_cold_unblock(struct intel_tc_port *tc, enum intel_display_power_domain domain,
+		  intel_wakeref_t wakeref)
 {
-	return tc_cold_block_in_mode(dig_port, dig_port->tc_mode, domain);
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+
+	intel_display_power_put(i915, domain, wakeref);
 }
 
 static void
-tc_cold_unblock(struct intel_digital_port *dig_port, enum intel_display_power_domain domain,
-		intel_wakeref_t wakeref)
+tc_cold_unblock(struct intel_tc_port *tc, intel_wakeref_t wakeref)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	enum intel_display_power_domain domain = tc_phy_cold_off_domain(tc);
 
-	/*
-	 * wakeref == -1, means some error happened saving save_depot_stack but
-	 * power should still be put down and 0 is a invalid save_depot_stack
-	 * id so can be used to skip it for non TC legacy ports.
-	 */
-	if (wakeref == 0)
-		return;
+#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
+	drm_WARN_ON(&tc_to_i915(tc)->drm, tc->lock_power_domain != domain);
+#endif
+	__tc_cold_unblock(tc, domain, wakeref);
+}
 
-	intel_display_power_put(i915, domain, wakeref);
+static void
+assert_display_core_power_enabled(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+
+	drm_WARN_ON(&i915->drm,
+		    !intel_display_power_is_enabled(i915, POWER_DOMAIN_DISPLAY_CORE));
 }
 
 static void
-assert_tc_cold_blocked(struct intel_digital_port *dig_port)
+assert_tc_cold_blocked(struct intel_tc_port *tc)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct drm_i915_private *i915 = tc_to_i915(tc);
 	bool enabled;
 
 	enabled = intel_display_power_is_enabled(i915,
-						 tc_cold_get_power_domain(dig_port,
-									  dig_port->tc_mode));
+						 tc_phy_cold_off_domain(tc));
 	drm_WARN_ON(&i915->drm, !enabled);
 }
 
+static enum intel_display_power_domain
+tc_port_power_domain(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	enum tc_port tc_port = intel_port_to_tc(i915, tc->dig_port->base.port);
+
+	return POWER_DOMAIN_PORT_DDI_LANES_TC1 + tc_port - TC_PORT_1;
+}
+
+static void
+assert_tc_port_power_enabled(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+
+	drm_WARN_ON(&i915->drm,
+		    !intel_display_power_is_enabled(i915, tc_port_power_domain(tc)));
+}
+
 u32 intel_tc_port_get_lane_mask(struct intel_digital_port *dig_port)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct intel_tc_port *tc = to_tc_port(dig_port);
 	u32 lane_mask;
 
-	lane_mask = intel_de_read(i915, PORT_TX_DFLEXDPSP(dig_port->tc_phy_fia));
+	lane_mask = intel_de_read(i915, PORT_TX_DFLEXDPSP(tc->phy_fia));
 
 	drm_WARN_ON(&i915->drm, lane_mask == 0xffffffff);
-	assert_tc_cold_blocked(dig_port);
+	assert_tc_cold_blocked(tc);
 
-	lane_mask &= DP_LANE_ASSIGNMENT_MASK(dig_port->tc_phy_fia_idx);
-	return lane_mask >> DP_LANE_ASSIGNMENT_SHIFT(dig_port->tc_phy_fia_idx);
+	lane_mask &= DP_LANE_ASSIGNMENT_MASK(tc->phy_fia_idx);
+	return lane_mask >> DP_LANE_ASSIGNMENT_SHIFT(tc->phy_fia_idx);
 }
 
 u32 intel_tc_port_get_pin_assignment_mask(struct intel_digital_port *dig_port)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct intel_tc_port *tc = to_tc_port(dig_port);
 	u32 pin_mask;
 
-	pin_mask = intel_de_read(i915, PORT_TX_DFLEXPA1(dig_port->tc_phy_fia));
+	pin_mask = intel_de_read(i915, PORT_TX_DFLEXPA1(tc->phy_fia));
 
 	drm_WARN_ON(&i915->drm, pin_mask == 0xffffffff);
-	assert_tc_cold_blocked(dig_port);
+	assert_tc_cold_blocked(tc);
 
-	return (pin_mask & DP_PIN_ASSIGNMENT_MASK(dig_port->tc_phy_fia_idx)) >>
-	       DP_PIN_ASSIGNMENT_SHIFT(dig_port->tc_phy_fia_idx);
+	return (pin_mask & DP_PIN_ASSIGNMENT_MASK(tc->phy_fia_idx)) >>
+	       DP_PIN_ASSIGNMENT_SHIFT(tc->phy_fia_idx);
 }
 
 int intel_tc_port_fia_max_lane_count(struct intel_digital_port *dig_port)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct intel_tc_port *tc = to_tc_port(dig_port);
+	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
 	intel_wakeref_t wakeref;
 	u32 lane_mask;
 
-	if (dig_port->tc_mode != TC_PORT_DP_ALT)
+	if (!intel_phy_is_tc(i915, phy) || tc->mode != TC_PORT_DP_ALT)
 		return 4;
 
-	assert_tc_cold_blocked(dig_port);
+	assert_tc_cold_blocked(tc);
 
 	lane_mask = 0;
 	with_intel_display_power(i915, POWER_DOMAIN_DISPLAY_CORE, wakeref)
@@ -182,45 +309,51 @@ void intel_tc_port_set_fia_lane_count(struct intel_digital_port *dig_port,
 				      int required_lanes)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct intel_tc_port *tc = to_tc_port(dig_port);
 	bool lane_reversal = dig_port->saved_port_bits & DDI_BUF_PORT_REVERSAL;
 	u32 val;
 
 	drm_WARN_ON(&i915->drm,
-		    lane_reversal && dig_port->tc_mode != TC_PORT_LEGACY);
+		    lane_reversal && tc->mode != TC_PORT_LEGACY);
 
-	assert_tc_cold_blocked(dig_port);
+	assert_tc_cold_blocked(tc);
 
-	val = intel_de_read(i915, PORT_TX_DFLEXDPMLE1(dig_port->tc_phy_fia));
-	val &= ~DFLEXDPMLE1_DPMLETC_MASK(dig_port->tc_phy_fia_idx);
+	val = intel_de_read(i915, PORT_TX_DFLEXDPMLE1(tc->phy_fia));
+	val &= ~DFLEXDPMLE1_DPMLETC_MASK(tc->phy_fia_idx);
 
 	switch (required_lanes) {
 	case 1:
 		val |= lane_reversal ?
-			DFLEXDPMLE1_DPMLETC_ML3(dig_port->tc_phy_fia_idx) :
-			DFLEXDPMLE1_DPMLETC_ML0(dig_port->tc_phy_fia_idx);
+			DFLEXDPMLE1_DPMLETC_ML3(tc->phy_fia_idx) :
+			DFLEXDPMLE1_DPMLETC_ML0(tc->phy_fia_idx);
 		break;
 	case 2:
 		val |= lane_reversal ?
-			DFLEXDPMLE1_DPMLETC_ML3_2(dig_port->tc_phy_fia_idx) :
-			DFLEXDPMLE1_DPMLETC_ML1_0(dig_port->tc_phy_fia_idx);
+			DFLEXDPMLE1_DPMLETC_ML3_2(tc->phy_fia_idx) :
+			DFLEXDPMLE1_DPMLETC_ML1_0(tc->phy_fia_idx);
 		break;
 	case 4:
-		val |= DFLEXDPMLE1_DPMLETC_ML3_0(dig_port->tc_phy_fia_idx);
+		val |= DFLEXDPMLE1_DPMLETC_ML3_0(tc->phy_fia_idx);
 		break;
 	default:
 		MISSING_CASE(required_lanes);
 	}
 
-	intel_de_write(i915, PORT_TX_DFLEXDPMLE1(dig_port->tc_phy_fia), val);
+	intel_de_write(i915, PORT_TX_DFLEXDPMLE1(tc->phy_fia), val);
 }
 
-static void tc_port_fixup_legacy_flag(struct intel_digital_port *dig_port,
+static void tc_port_fixup_legacy_flag(struct intel_tc_port *tc,
 				      u32 live_status_mask)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct drm_i915_private *i915 = tc_to_i915(tc);
 	u32 valid_hpd_mask;
 
-	if (dig_port->tc_legacy_port)
+	drm_WARN_ON(&i915->drm, tc->mode != TC_PORT_DISCONNECTED);
+
+	if (hweight32(live_status_mask) != 1)
+		return;
+
+	if (tc->legacy_port)
 		valid_hpd_mask = BIT(TC_PORT_LEGACY);
 	else
 		valid_hpd_mask = BIT(TC_PORT_DP_ALT) |
@@ -232,80 +365,79 @@ static void tc_port_fixup_legacy_flag(struct intel_digital_port *dig_port,
 	/* If live status mismatches the VBT flag, trust the live status. */
 	drm_dbg_kms(&i915->drm,
 		    "Port %s: live status %08x mismatches the legacy port flag %08x, fixing flag\n",
-		    dig_port->tc_port_name, live_status_mask, valid_hpd_mask);
+		    tc->port_name, live_status_mask, valid_hpd_mask);
 
-	dig_port->tc_legacy_port = !dig_port->tc_legacy_port;
+	tc->legacy_port = !tc->legacy_port;
 }
 
-static u32 icl_tc_port_live_status_mask(struct intel_digital_port *dig_port)
+static void tc_phy_load_fia_params(struct intel_tc_port *tc, bool modular_fia)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	u32 isr_bit = i915->display.hotplug.pch_hpd[dig_port->base.hpd_pin];
-	u32 mask = 0;
-	u32 val;
-
-	val = intel_de_read(i915, PORT_TX_DFLEXDPSP(dig_port->tc_phy_fia));
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	enum port port = tc->dig_port->base.port;
+	enum tc_port tc_port = intel_port_to_tc(i915, port);
 
-	if (val == 0xffffffff) {
-		drm_dbg_kms(&i915->drm,
-			    "Port %s: PHY in TCCOLD, nothing connected\n",
-			    dig_port->tc_port_name);
-		return mask;
+	/*
+	 * Each Modular FIA instance houses 2 TC ports. In SOC that has more
+	 * than two TC ports, there are multiple instances of Modular FIA.
+	 */
+	if (modular_fia) {
+		tc->phy_fia = tc_port / 2;
+		tc->phy_fia_idx = tc_port % 2;
+	} else {
+		tc->phy_fia = FIA1;
+		tc->phy_fia_idx = tc_port;
 	}
+}
 
-	if (val & TC_LIVE_STATE_TBT(dig_port->tc_phy_fia_idx))
-		mask |= BIT(TC_PORT_TBT_ALT);
-	if (val & TC_LIVE_STATE_TC(dig_port->tc_phy_fia_idx))
-		mask |= BIT(TC_PORT_DP_ALT);
-
-	if (intel_de_read(i915, SDEISR) & isr_bit)
-		mask |= BIT(TC_PORT_LEGACY);
+/*
+ * ICL TC PHY handlers
+ * -------------------
+ */
+static enum intel_display_power_domain
+icl_tc_phy_cold_off_domain(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	struct intel_digital_port *dig_port = tc->dig_port;
 
-	/* The sink can be connected only in a single mode. */
-	if (!drm_WARN_ON_ONCE(&i915->drm, hweight32(mask) > 1))
-		tc_port_fixup_legacy_flag(dig_port, mask);
+	if (tc->legacy_port)
+		return intel_display_power_legacy_aux_domain(i915, dig_port->aux_ch);
 
-	return mask;
+	return POWER_DOMAIN_TC_COLD_OFF;
 }
 
-static u32 adl_tc_port_live_status_mask(struct intel_digital_port *dig_port)
+static u32 icl_tc_phy_hpd_live_status(struct intel_tc_port *tc)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum tc_port tc_port = intel_port_to_tc(i915, dig_port->base.port);
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	struct intel_digital_port *dig_port = tc->dig_port;
 	u32 isr_bit = i915->display.hotplug.pch_hpd[dig_port->base.hpd_pin];
-	u32 val, mask = 0;
+	intel_wakeref_t wakeref;
+	u32 fia_isr;
+	u32 pch_isr;
+	u32 mask = 0;
 
-	/*
-	 * On ADL-P HW/FW will wake from TCCOLD to complete the read access of
-	 * registers in IOM. Note that this doesn't apply to PHY and FIA
-	 * registers.
-	 */
-	val = intel_de_read(i915, TCSS_DDI_STATUS(tc_port));
-	if (val & TCSS_DDI_STATUS_HPD_LIVE_STATUS_ALT)
-		mask |= BIT(TC_PORT_DP_ALT);
-	if (val & TCSS_DDI_STATUS_HPD_LIVE_STATUS_TBT)
+	with_intel_display_power(i915, tc_phy_cold_off_domain(tc), wakeref) {
+		fia_isr = intel_de_read(i915, PORT_TX_DFLEXDPSP(tc->phy_fia));
+		pch_isr = intel_de_read(i915, SDEISR);
+	}
+
+	if (fia_isr == 0xffffffff) {
+		drm_dbg_kms(&i915->drm,
+			    "Port %s: PHY in TCCOLD, nothing connected\n",
+			    tc->port_name);
+		return mask;
+	}
+
+	if (fia_isr & TC_LIVE_STATE_TBT(tc->phy_fia_idx))
 		mask |= BIT(TC_PORT_TBT_ALT);
+	if (fia_isr & TC_LIVE_STATE_TC(tc->phy_fia_idx))
+		mask |= BIT(TC_PORT_DP_ALT);
 
-	if (intel_de_read(i915, SDEISR) & isr_bit)
+	if (pch_isr & isr_bit)
 		mask |= BIT(TC_PORT_LEGACY);
 
-	/* The sink can be connected only in a single mode. */
-	if (!drm_WARN_ON(&i915->drm, hweight32(mask) > 1))
-		tc_port_fixup_legacy_flag(dig_port, mask);
-
 	return mask;
 }
 
-static u32 tc_port_live_status_mask(struct intel_digital_port *dig_port)
-{
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-
-	if (IS_ALDERLAKE_P(i915))
-		return adl_tc_port_live_status_mask(dig_port);
-
-	return icl_tc_port_live_status_mask(dig_port);
-}
-
 /*
  * Return the PHY status complete flag indicating that display can acquire the
  * PHY ownership. The IOM firmware sets this flag when a DP-alt or legacy sink
@@ -314,136 +446,80 @@ static u32 tc_port_live_status_mask(struct intel_digital_port *dig_port)
  * owned by the TBT subsystem and so switching the ownership to display is not
  * required.
  */
-static bool icl_tc_phy_status_complete(struct intel_digital_port *dig_port)
+static bool icl_tc_phy_is_ready(struct intel_tc_port *tc)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct drm_i915_private *i915 = tc_to_i915(tc);
 	u32 val;
 
-	val = intel_de_read(i915, PORT_TX_DFLEXDPPMS(dig_port->tc_phy_fia));
-	if (val == 0xffffffff) {
-		drm_dbg_kms(&i915->drm,
-			    "Port %s: PHY in TCCOLD, assuming not complete\n",
-			    dig_port->tc_port_name);
-		return false;
-	}
-
-	return val & DP_PHY_MODE_STATUS_COMPLETED(dig_port->tc_phy_fia_idx);
-}
-
-/*
- * Return the PHY status complete flag indicating that display can acquire the
- * PHY ownership. The IOM firmware sets this flag when it's ready to switch
- * the ownership to display, regardless of what sink is connected (TBT-alt,
- * DP-alt, legacy or nothing). For TBT-alt sinks the PHY is owned by the TBT
- * subsystem and so switching the ownership to display is not required.
- */
-static bool adl_tc_phy_status_complete(struct intel_digital_port *dig_port)
-{
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum tc_port tc_port = intel_port_to_tc(i915, dig_port->base.port);
-	u32 val;
+	assert_tc_cold_blocked(tc);
 
-	val = intel_de_read(i915, TCSS_DDI_STATUS(tc_port));
+	val = intel_de_read(i915, PORT_TX_DFLEXDPPMS(tc->phy_fia));
 	if (val == 0xffffffff) {
 		drm_dbg_kms(&i915->drm,
-			    "Port %s: PHY in TCCOLD, assuming not complete\n",
-			    dig_port->tc_port_name);
+			    "Port %s: PHY in TCCOLD, assuming not ready\n",
+			    tc->port_name);
 		return false;
 	}
 
-	return val & TCSS_DDI_STATUS_READY;
+	return val & DP_PHY_MODE_STATUS_COMPLETED(tc->phy_fia_idx);
 }
 
-static bool tc_phy_status_complete(struct intel_digital_port *dig_port)
-{
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-
-	if (IS_ALDERLAKE_P(i915))
-		return adl_tc_phy_status_complete(dig_port);
-
-	return icl_tc_phy_status_complete(dig_port);
-}
-
-static bool icl_tc_phy_take_ownership(struct intel_digital_port *dig_port,
+static bool icl_tc_phy_take_ownership(struct intel_tc_port *tc,
 				      bool take)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct drm_i915_private *i915 = tc_to_i915(tc);
 	u32 val;
 
-	val = intel_de_read(i915, PORT_TX_DFLEXDPCSSS(dig_port->tc_phy_fia));
+	assert_tc_cold_blocked(tc);
+
+	val = intel_de_read(i915, PORT_TX_DFLEXDPCSSS(tc->phy_fia));
 	if (val == 0xffffffff) {
 		drm_dbg_kms(&i915->drm,
 			    "Port %s: PHY in TCCOLD, can't %s ownership\n",
-			    dig_port->tc_port_name, take ? "take" : "release");
+			    tc->port_name, take ? "take" : "release");
 
 		return false;
 	}
 
-	val &= ~DP_PHY_MODE_STATUS_NOT_SAFE(dig_port->tc_phy_fia_idx);
+	val &= ~DP_PHY_MODE_STATUS_NOT_SAFE(tc->phy_fia_idx);
 	if (take)
-		val |= DP_PHY_MODE_STATUS_NOT_SAFE(dig_port->tc_phy_fia_idx);
+		val |= DP_PHY_MODE_STATUS_NOT_SAFE(tc->phy_fia_idx);
 
-	intel_de_write(i915, PORT_TX_DFLEXDPCSSS(dig_port->tc_phy_fia), val);
+	intel_de_write(i915, PORT_TX_DFLEXDPCSSS(tc->phy_fia), val);
 
 	return true;
 }
 
-static bool adl_tc_phy_take_ownership(struct intel_digital_port *dig_port,
-				      bool take)
+static bool icl_tc_phy_is_owned(struct intel_tc_port *tc)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum port port = dig_port->base.port;
-
-	intel_de_rmw(i915, DDI_BUF_CTL(port), DDI_BUF_CTL_TC_PHY_OWNERSHIP,
-		     take ? DDI_BUF_CTL_TC_PHY_OWNERSHIP : 0);
-
-	return true;
-}
-
-static bool tc_phy_take_ownership(struct intel_digital_port *dig_port, bool take)
-{
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-
-	if (IS_ALDERLAKE_P(i915))
-		return adl_tc_phy_take_ownership(dig_port, take);
-
-	return icl_tc_phy_take_ownership(dig_port, take);
-}
-
-static bool icl_tc_phy_is_owned(struct intel_digital_port *dig_port)
-{
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct drm_i915_private *i915 = tc_to_i915(tc);
 	u32 val;
 
-	val = intel_de_read(i915, PORT_TX_DFLEXDPCSSS(dig_port->tc_phy_fia));
+	assert_tc_cold_blocked(tc);
+
+	val = intel_de_read(i915, PORT_TX_DFLEXDPCSSS(tc->phy_fia));
 	if (val == 0xffffffff) {
 		drm_dbg_kms(&i915->drm,
 			    "Port %s: PHY in TCCOLD, assume not owned\n",
-			    dig_port->tc_port_name);
+			    tc->port_name);
 		return false;
 	}
 
-	return val & DP_PHY_MODE_STATUS_NOT_SAFE(dig_port->tc_phy_fia_idx);
+	return val & DP_PHY_MODE_STATUS_NOT_SAFE(tc->phy_fia_idx);
 }
 
-static bool adl_tc_phy_is_owned(struct intel_digital_port *dig_port)
+static void icl_tc_phy_get_hw_state(struct intel_tc_port *tc)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum port port = dig_port->base.port;
-	u32 val;
-
-	val = intel_de_read(i915, DDI_BUF_CTL(port));
-	return val & DDI_BUF_CTL_TC_PHY_OWNERSHIP;
-}
+	enum intel_display_power_domain domain;
+	intel_wakeref_t tc_cold_wref;
 
-static bool tc_phy_is_owned(struct intel_digital_port *dig_port)
-{
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	tc_cold_wref = __tc_cold_block(tc, &domain);
 
-	if (IS_ALDERLAKE_P(i915))
-		return adl_tc_phy_is_owned(dig_port);
+	tc->mode = tc_phy_get_current_mode(tc);
+	if (tc->mode != TC_PORT_DISCONNECTED)
+		tc->lock_wakeref = tc_cold_block(tc);
 
-	return icl_tc_phy_is_owned(dig_port);
+	__tc_cold_unblock(tc, domain, tc_cold_wref);
 }
 
 /*
@@ -457,151 +533,590 @@ static bool tc_phy_is_owned(struct intel_digital_port *dig_port)
  * connect and disconnect to cleanly transfer ownership with the controller and
  * set the type-C power state.
  */
-static void icl_tc_phy_connect(struct intel_digital_port *dig_port,
-			       int required_lanes)
+static bool tc_phy_verify_legacy_or_dp_alt_mode(struct intel_tc_port *tc,
+						int required_lanes)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	u32 live_status_mask;
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	struct intel_digital_port *dig_port = tc->dig_port;
 	int max_lanes;
 
-	if (!tc_phy_status_complete(dig_port)) {
-		drm_dbg_kms(&i915->drm, "Port %s: PHY not ready\n",
-			    dig_port->tc_port_name);
-		goto out_set_tbt_alt_mode;
-	}
-
-	live_status_mask = tc_port_live_status_mask(dig_port);
-	if (!(live_status_mask & (BIT(TC_PORT_DP_ALT) | BIT(TC_PORT_LEGACY))) &&
-	    !dig_port->tc_legacy_port) {
-		drm_dbg_kms(&i915->drm, "Port %s: PHY ownership not required (live status %02x)\n",
-			    dig_port->tc_port_name, live_status_mask);
-		goto out_set_tbt_alt_mode;
-	}
-
-	if (!tc_phy_take_ownership(dig_port, true) &&
-	    !drm_WARN_ON(&i915->drm, dig_port->tc_legacy_port))
-		goto out_set_tbt_alt_mode;
-
 	max_lanes = intel_tc_port_fia_max_lane_count(dig_port);
-	if (dig_port->tc_legacy_port) {
+	if (tc->mode == TC_PORT_LEGACY) {
 		drm_WARN_ON(&i915->drm, max_lanes != 4);
-		dig_port->tc_mode = TC_PORT_LEGACY;
-
-		return;
+		return true;
 	}
 
+	drm_WARN_ON(&i915->drm, tc->mode != TC_PORT_DP_ALT);
+
 	/*
 	 * Now we have to re-check the live state, in case the port recently
 	 * became disconnected. Not necessary for legacy mode.
 	 */
-	if (!(tc_port_live_status_mask(dig_port) & BIT(TC_PORT_DP_ALT))) {
+	if (!(tc_phy_hpd_live_status(tc) & BIT(TC_PORT_DP_ALT))) {
 		drm_dbg_kms(&i915->drm, "Port %s: PHY sudden disconnect\n",
-			    dig_port->tc_port_name);
-		goto out_release_phy;
+			    tc->port_name);
+		return false;
 	}
 
 	if (max_lanes < required_lanes) {
 		drm_dbg_kms(&i915->drm,
 			    "Port %s: PHY max lanes %d < required lanes %d\n",
-			    dig_port->tc_port_name,
+			    tc->port_name,
 			    max_lanes, required_lanes);
-		goto out_release_phy;
+		return false;
 	}
 
-	dig_port->tc_mode = TC_PORT_DP_ALT;
+	return true;
+}
+
+static bool icl_tc_phy_connect(struct intel_tc_port *tc,
+			       int required_lanes)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+
+	tc->lock_wakeref = tc_cold_block(tc);
+
+	if (tc->mode == TC_PORT_TBT_ALT)
+		return true;
 
-	return;
+	if ((!tc_phy_is_ready(tc) ||
+	     !icl_tc_phy_take_ownership(tc, true)) &&
+	    !drm_WARN_ON(&i915->drm, tc->mode == TC_PORT_LEGACY)) {
+		drm_dbg_kms(&i915->drm, "Port %s: can't take PHY ownership (ready %s)\n",
+			    tc->port_name,
+			    str_yes_no(tc_phy_is_ready(tc)));
+		goto out_unblock_tc_cold;
+	}
+
+
+	if (!tc_phy_verify_legacy_or_dp_alt_mode(tc, required_lanes))
+		goto out_release_phy;
+
+	return true;
 
 out_release_phy:
-	tc_phy_take_ownership(dig_port, false);
-out_set_tbt_alt_mode:
-	dig_port->tc_mode = TC_PORT_TBT_ALT;
+	icl_tc_phy_take_ownership(tc, false);
+out_unblock_tc_cold:
+	tc_cold_unblock(tc, fetch_and_zero(&tc->lock_wakeref));
+
+	return false;
 }
 
 /*
  * See the comment at the connect function. This implements the Disconnect
  * Flow.
  */
-static void icl_tc_phy_disconnect(struct intel_digital_port *dig_port)
+static void icl_tc_phy_disconnect(struct intel_tc_port *tc)
 {
-	switch (dig_port->tc_mode) {
+	switch (tc->mode) {
 	case TC_PORT_LEGACY:
 	case TC_PORT_DP_ALT:
-		tc_phy_take_ownership(dig_port, false);
+		icl_tc_phy_take_ownership(tc, false);
 		fallthrough;
 	case TC_PORT_TBT_ALT:
-		dig_port->tc_mode = TC_PORT_DISCONNECTED;
-		fallthrough;
-	case TC_PORT_DISCONNECTED:
+		tc_cold_unblock(tc, fetch_and_zero(&tc->lock_wakeref));
 		break;
 	default:
-		MISSING_CASE(dig_port->tc_mode);
+		MISSING_CASE(tc->mode);
 	}
 }
 
-static bool icl_tc_phy_is_connected(struct intel_digital_port *dig_port)
+static void icl_tc_phy_init(struct intel_tc_port *tc)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	tc_phy_load_fia_params(tc, false);
+}
+
+static const struct intel_tc_phy_ops icl_tc_phy_ops = {
+	.cold_off_domain = icl_tc_phy_cold_off_domain,
+	.hpd_live_status = icl_tc_phy_hpd_live_status,
+	.is_ready = icl_tc_phy_is_ready,
+	.is_owned = icl_tc_phy_is_owned,
+	.get_hw_state = icl_tc_phy_get_hw_state,
+	.connect = icl_tc_phy_connect,
+	.disconnect = icl_tc_phy_disconnect,
+	.init = icl_tc_phy_init,
+};
+
+/*
+ * TGL TC PHY handlers
+ * -------------------
+ */
+static enum intel_display_power_domain
+tgl_tc_phy_cold_off_domain(struct intel_tc_port *tc)
+{
+	return POWER_DOMAIN_TC_COLD_OFF;
+}
+
+static void tgl_tc_phy_init(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	intel_wakeref_t wakeref;
+	u32 val;
+
+	with_intel_display_power(i915, tc_phy_cold_off_domain(tc), wakeref)
+		val = intel_de_read(i915, PORT_TX_DFLEXDPSP(FIA1));
+
+	drm_WARN_ON(&i915->drm, val == 0xffffffff);
 
-	if (!tc_phy_status_complete(dig_port)) {
-		drm_dbg_kms(&i915->drm, "Port %s: PHY status not complete\n",
-			    dig_port->tc_port_name);
-		return dig_port->tc_mode == TC_PORT_TBT_ALT;
+	tc_phy_load_fia_params(tc, val & MODULAR_FIA_MASK);
+}
+
+static const struct intel_tc_phy_ops tgl_tc_phy_ops = {
+	.cold_off_domain = tgl_tc_phy_cold_off_domain,
+	.hpd_live_status = icl_tc_phy_hpd_live_status,
+	.is_ready = icl_tc_phy_is_ready,
+	.is_owned = icl_tc_phy_is_owned,
+	.get_hw_state = icl_tc_phy_get_hw_state,
+	.connect = icl_tc_phy_connect,
+	.disconnect = icl_tc_phy_disconnect,
+	.init = tgl_tc_phy_init,
+};
+
+/*
+ * ADLP TC PHY handlers
+ * --------------------
+ */
+static enum intel_display_power_domain
+adlp_tc_phy_cold_off_domain(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	struct intel_digital_port *dig_port = tc->dig_port;
+
+	if (tc->mode != TC_PORT_TBT_ALT)
+		return intel_display_power_legacy_aux_domain(i915, dig_port->aux_ch);
+
+	return POWER_DOMAIN_TC_COLD_OFF;
+}
+
+static u32 adlp_tc_phy_hpd_live_status(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	struct intel_digital_port *dig_port = tc->dig_port;
+	enum hpd_pin hpd_pin = dig_port->base.hpd_pin;
+	u32 cpu_isr_bits = i915->display.hotplug.hpd[hpd_pin];
+	u32 pch_isr_bit = i915->display.hotplug.pch_hpd[hpd_pin];
+	intel_wakeref_t wakeref;
+	u32 cpu_isr;
+	u32 pch_isr;
+	u32 mask = 0;
+
+	with_intel_display_power(i915, POWER_DOMAIN_DISPLAY_CORE, wakeref) {
+		cpu_isr = intel_de_read(i915, GEN11_DE_HPD_ISR);
+		pch_isr = intel_de_read(i915, SDEISR);
 	}
 
-	/* On ADL-P the PHY complete flag is set in TBT mode as well. */
-	if (IS_ALDERLAKE_P(i915) && dig_port->tc_mode == TC_PORT_TBT_ALT)
-		return true;
+	if (cpu_isr & (cpu_isr_bits & GEN11_DE_TC_HOTPLUG_MASK))
+		mask |= BIT(TC_PORT_DP_ALT);
+	if (cpu_isr & (cpu_isr_bits & GEN11_DE_TBT_HOTPLUG_MASK))
+		mask |= BIT(TC_PORT_TBT_ALT);
+
+	if (pch_isr & pch_isr_bit)
+		mask |= BIT(TC_PORT_LEGACY);
+
+	return mask;
+}
+
+/*
+ * Return the PHY status complete flag indicating that display can acquire the
+ * PHY ownership. The IOM firmware sets this flag when it's ready to switch
+ * the ownership to display, regardless of what sink is connected (TBT-alt,
+ * DP-alt, legacy or nothing). For TBT-alt sinks the PHY is owned by the TBT
+ * subsystem and so switching the ownership to display is not required.
+ */
+static bool adlp_tc_phy_is_ready(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	enum tc_port tc_port = intel_port_to_tc(i915, tc->dig_port->base.port);
+	u32 val;
 
-	if (!tc_phy_is_owned(dig_port)) {
-		drm_dbg_kms(&i915->drm, "Port %s: PHY not owned\n",
-			    dig_port->tc_port_name);
+	assert_display_core_power_enabled(tc);
 
+	val = intel_de_read(i915, TCSS_DDI_STATUS(tc_port));
+	if (val == 0xffffffff) {
+		drm_dbg_kms(&i915->drm,
+			    "Port %s: PHY in TCCOLD, assuming not ready\n",
+			    tc->port_name);
 		return false;
 	}
 
-	return dig_port->tc_mode == TC_PORT_DP_ALT ||
-	       dig_port->tc_mode == TC_PORT_LEGACY;
+	return val & TCSS_DDI_STATUS_READY;
+}
+
+static bool adlp_tc_phy_take_ownership(struct intel_tc_port *tc,
+				       bool take)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	enum port port = tc->dig_port->base.port;
+
+	assert_tc_port_power_enabled(tc);
+
+	intel_de_rmw(i915, DDI_BUF_CTL(port), DDI_BUF_CTL_TC_PHY_OWNERSHIP,
+		     take ? DDI_BUF_CTL_TC_PHY_OWNERSHIP : 0);
+
+	return true;
+}
+
+static bool adlp_tc_phy_is_owned(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	enum port port = tc->dig_port->base.port;
+	u32 val;
+
+	assert_tc_port_power_enabled(tc);
+
+	val = intel_de_read(i915, DDI_BUF_CTL(port));
+	return val & DDI_BUF_CTL_TC_PHY_OWNERSHIP;
+}
+
+static void adlp_tc_phy_get_hw_state(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	enum intel_display_power_domain port_power_domain =
+		tc_port_power_domain(tc);
+	intel_wakeref_t port_wakeref;
+
+	port_wakeref = intel_display_power_get(i915, port_power_domain);
+
+	tc->mode = tc_phy_get_current_mode(tc);
+	if (tc->mode != TC_PORT_DISCONNECTED)
+		tc->lock_wakeref = tc_cold_block(tc);
+
+	intel_display_power_put(i915, port_power_domain, port_wakeref);
+}
+
+static bool adlp_tc_phy_connect(struct intel_tc_port *tc, int required_lanes)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	enum intel_display_power_domain port_power_domain =
+		tc_port_power_domain(tc);
+	intel_wakeref_t port_wakeref;
+
+	if (tc->mode == TC_PORT_TBT_ALT) {
+		tc->lock_wakeref = tc_cold_block(tc);
+		return true;
+	}
+
+	port_wakeref = intel_display_power_get(i915, port_power_domain);
+
+	if (!adlp_tc_phy_take_ownership(tc, true) &&
+	    !drm_WARN_ON(&i915->drm, tc->mode == TC_PORT_LEGACY)) {
+		drm_dbg_kms(&i915->drm, "Port %s: can't take PHY ownership\n",
+			    tc->port_name);
+		goto out_put_port_power;
+	}
+
+	if (!tc_phy_is_ready(tc) &&
+	    !drm_WARN_ON(&i915->drm, tc->mode == TC_PORT_LEGACY)) {
+		drm_dbg_kms(&i915->drm, "Port %s: PHY not ready\n",
+			    tc->port_name);
+		goto out_release_phy;
+	}
+
+	tc->lock_wakeref = tc_cold_block(tc);
+
+	if (!tc_phy_verify_legacy_or_dp_alt_mode(tc, required_lanes))
+		goto out_unblock_tc_cold;
+
+	intel_display_power_put(i915, port_power_domain, port_wakeref);
+
+	return true;
+
+out_unblock_tc_cold:
+	tc_cold_unblock(tc, fetch_and_zero(&tc->lock_wakeref));
+out_release_phy:
+	adlp_tc_phy_take_ownership(tc, false);
+out_put_port_power:
+	intel_display_power_put(i915, port_power_domain, port_wakeref);
+
+	return false;
+}
+
+static void adlp_tc_phy_disconnect(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	enum intel_display_power_domain port_power_domain =
+		tc_port_power_domain(tc);
+	intel_wakeref_t port_wakeref;
+
+	port_wakeref = intel_display_power_get(i915, port_power_domain);
+
+	tc_cold_unblock(tc, fetch_and_zero(&tc->lock_wakeref));
+
+	switch (tc->mode) {
+	case TC_PORT_LEGACY:
+	case TC_PORT_DP_ALT:
+		adlp_tc_phy_take_ownership(tc, false);
+		fallthrough;
+	case TC_PORT_TBT_ALT:
+		break;
+	default:
+		MISSING_CASE(tc->mode);
+	}
+
+	intel_display_power_put(i915, port_power_domain, port_wakeref);
+}
+
+static void adlp_tc_phy_init(struct intel_tc_port *tc)
+{
+	tc_phy_load_fia_params(tc, true);
+}
+
+static const struct intel_tc_phy_ops adlp_tc_phy_ops = {
+	.cold_off_domain = adlp_tc_phy_cold_off_domain,
+	.hpd_live_status = adlp_tc_phy_hpd_live_status,
+	.is_ready = adlp_tc_phy_is_ready,
+	.is_owned = adlp_tc_phy_is_owned,
+	.get_hw_state = adlp_tc_phy_get_hw_state,
+	.connect = adlp_tc_phy_connect,
+	.disconnect = adlp_tc_phy_disconnect,
+	.init = adlp_tc_phy_init,
+};
+
+/*
+ * Generic TC PHY handlers
+ * -----------------------
+ */
+static enum intel_display_power_domain
+tc_phy_cold_off_domain(struct intel_tc_port *tc)
+{
+	return tc->phy_ops->cold_off_domain(tc);
+}
+
+static u32 tc_phy_hpd_live_status(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	u32 mask;
+
+	mask = tc->phy_ops->hpd_live_status(tc);
+
+	/* The sink can be connected only in a single mode. */
+	drm_WARN_ON_ONCE(&i915->drm, hweight32(mask) > 1);
+
+	return mask;
+}
+
+static bool tc_phy_is_ready(struct intel_tc_port *tc)
+{
+	return tc->phy_ops->is_ready(tc);
+}
+
+static bool tc_phy_is_owned(struct intel_tc_port *tc)
+{
+	return tc->phy_ops->is_owned(tc);
+}
+
+static void tc_phy_get_hw_state(struct intel_tc_port *tc)
+{
+	tc->phy_ops->get_hw_state(tc);
+}
+
+static bool tc_phy_is_ready_and_owned(struct intel_tc_port *tc,
+				      bool phy_is_ready, bool phy_is_owned)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+
+	drm_WARN_ON(&i915->drm, phy_is_owned && !phy_is_ready);
+
+	return phy_is_ready && phy_is_owned;
+}
+
+static bool tc_phy_is_connected(struct intel_tc_port *tc,
+				enum icl_port_dpll_id port_pll_type)
+{
+	struct intel_encoder *encoder = &tc->dig_port->base;
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	bool phy_is_ready = tc_phy_is_ready(tc);
+	bool phy_is_owned = tc_phy_is_owned(tc);
+	bool is_connected;
+
+	if (tc_phy_is_ready_and_owned(tc, phy_is_ready, phy_is_owned))
+		is_connected = port_pll_type == ICL_PORT_DPLL_MG_PHY;
+	else
+		is_connected = port_pll_type == ICL_PORT_DPLL_DEFAULT;
+
+	drm_dbg_kms(&i915->drm,
+		    "Port %s: PHY connected: %s (ready: %s, owned: %s, pll_type: %s)\n",
+		    tc->port_name,
+		    str_yes_no(is_connected),
+		    str_yes_no(phy_is_ready),
+		    str_yes_no(phy_is_owned),
+		    port_pll_type == ICL_PORT_DPLL_DEFAULT ? "tbt" : "non-tbt");
+
+	return is_connected;
+}
+
+static void tc_phy_wait_for_ready(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+
+	if (wait_for(tc_phy_is_ready(tc), 100))
+		drm_err(&i915->drm, "Port %s: timeout waiting for PHY ready\n",
+			tc->port_name);
 }
 
 static enum tc_port_mode
-intel_tc_port_get_current_mode(struct intel_digital_port *dig_port)
+hpd_mask_to_tc_mode(u32 live_status_mask)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	u32 live_status_mask = tc_port_live_status_mask(dig_port);
-	enum tc_port_mode mode;
+	if (live_status_mask)
+		return fls(live_status_mask) - 1;
 
-	if (!tc_phy_is_owned(dig_port) ||
-	    drm_WARN_ON(&i915->drm, !tc_phy_status_complete(dig_port)))
+	return TC_PORT_DISCONNECTED;
+}
+
+static enum tc_port_mode
+tc_phy_hpd_live_mode(struct intel_tc_port *tc)
+{
+	u32 live_status_mask = tc_phy_hpd_live_status(tc);
+
+	return hpd_mask_to_tc_mode(live_status_mask);
+}
+
+static enum tc_port_mode
+get_tc_mode_in_phy_owned_state(struct intel_tc_port *tc,
+			       enum tc_port_mode live_mode)
+{
+	switch (live_mode) {
+	case TC_PORT_LEGACY:
+	case TC_PORT_DP_ALT:
+		return live_mode;
+	default:
+		MISSING_CASE(live_mode);
+		fallthrough;
+	case TC_PORT_TBT_ALT:
+	case TC_PORT_DISCONNECTED:
+		if (tc->legacy_port)
+			return TC_PORT_LEGACY;
+		else
+			return TC_PORT_DP_ALT;
+	}
+}
+
+static enum tc_port_mode
+get_tc_mode_in_phy_not_owned_state(struct intel_tc_port *tc,
+				   enum tc_port_mode live_mode)
+{
+	switch (live_mode) {
+	case TC_PORT_LEGACY:
+		return TC_PORT_DISCONNECTED;
+	case TC_PORT_DP_ALT:
+	case TC_PORT_TBT_ALT:
 		return TC_PORT_TBT_ALT;
+	default:
+		MISSING_CASE(live_mode);
+		fallthrough;
+	case TC_PORT_DISCONNECTED:
+		if (tc->legacy_port)
+			return TC_PORT_DISCONNECTED;
+		else
+			return TC_PORT_TBT_ALT;
+	}
+}
+
+static enum tc_port_mode
+tc_phy_get_current_mode(struct intel_tc_port *tc)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	enum tc_port_mode live_mode = tc_phy_hpd_live_mode(tc);
+	bool phy_is_ready;
+	bool phy_is_owned;
+	enum tc_port_mode mode;
+
+	/*
+	 * For legacy ports the IOM firmware initializes the PHY during boot-up
+	 * and system resume whether or not a sink is connected. Wait here for
+	 * the initialization to get ready.
+	 */
+	if (tc->legacy_port)
+		tc_phy_wait_for_ready(tc);
 
-	mode = dig_port->tc_legacy_port ? TC_PORT_LEGACY : TC_PORT_DP_ALT;
-	if (live_status_mask) {
-		enum tc_port_mode live_mode = fls(live_status_mask) - 1;
+	phy_is_ready = tc_phy_is_ready(tc);
+	phy_is_owned = tc_phy_is_owned(tc);
 
-		if (!drm_WARN_ON(&i915->drm, live_mode == TC_PORT_TBT_ALT))
-			mode = live_mode;
+	if (!tc_phy_is_ready_and_owned(tc, phy_is_ready, phy_is_owned)) {
+		mode = get_tc_mode_in_phy_not_owned_state(tc, live_mode);
+	} else {
+		drm_WARN_ON(&i915->drm, live_mode == TC_PORT_TBT_ALT);
+		mode = get_tc_mode_in_phy_owned_state(tc, live_mode);
 	}
 
+	drm_dbg_kms(&i915->drm,
+		    "Port %s: PHY mode: %s (ready: %s, owned: %s, HPD: %s)\n",
+		    tc->port_name,
+		    tc_port_mode_name(mode),
+		    str_yes_no(phy_is_ready),
+		    str_yes_no(phy_is_owned),
+		    tc_port_mode_name(live_mode));
+
 	return mode;
 }
 
+static enum tc_port_mode default_tc_mode(struct intel_tc_port *tc)
+{
+	if (tc->legacy_port)
+		return TC_PORT_LEGACY;
+
+	return TC_PORT_TBT_ALT;
+}
+
 static enum tc_port_mode
-intel_tc_port_get_target_mode(struct intel_digital_port *dig_port)
+hpd_mask_to_target_mode(struct intel_tc_port *tc, u32 live_status_mask)
 {
-	u32 live_status_mask = tc_port_live_status_mask(dig_port);
+	enum tc_port_mode mode = hpd_mask_to_tc_mode(live_status_mask);
 
-	if (live_status_mask)
-		return fls(live_status_mask) - 1;
+	if (mode != TC_PORT_DISCONNECTED)
+		return mode;
 
-	return TC_PORT_TBT_ALT;
+	return default_tc_mode(tc);
+}
+
+static enum tc_port_mode
+tc_phy_get_target_mode(struct intel_tc_port *tc)
+{
+	u32 live_status_mask = tc_phy_hpd_live_status(tc);
+
+	return hpd_mask_to_target_mode(tc, live_status_mask);
+}
+
+static void tc_phy_connect(struct intel_tc_port *tc, int required_lanes)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	u32 live_status_mask = tc_phy_hpd_live_status(tc);
+	bool connected;
+
+	tc_port_fixup_legacy_flag(tc, live_status_mask);
+
+	tc->mode = hpd_mask_to_target_mode(tc, live_status_mask);
+
+	connected = tc->phy_ops->connect(tc, required_lanes);
+	if (!connected && tc->mode != default_tc_mode(tc)) {
+		tc->mode = default_tc_mode(tc);
+		connected = tc->phy_ops->connect(tc, required_lanes);
+	}
+
+	drm_WARN_ON(&i915->drm, !connected);
+}
+
+static void tc_phy_disconnect(struct intel_tc_port *tc)
+{
+	if (tc->mode != TC_PORT_DISCONNECTED) {
+		tc->phy_ops->disconnect(tc);
+		tc->mode = TC_PORT_DISCONNECTED;
+	}
+}
+
+static void tc_phy_init(struct intel_tc_port *tc)
+{
+	mutex_lock(&tc->lock);
+	tc->phy_ops->init(tc);
+	mutex_unlock(&tc->lock);
 }
 
-static void intel_tc_port_reset_mode(struct intel_digital_port *dig_port,
+static void intel_tc_port_reset_mode(struct intel_tc_port *tc,
 				     int required_lanes, bool force_disconnect)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum tc_port_mode old_tc_mode = dig_port->tc_mode;
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	struct intel_digital_port *dig_port = tc->dig_port;
+	enum tc_port_mode old_tc_mode = tc->mode;
 
 	intel_display_power_flush_work(i915);
 	if (!intel_tc_cold_requires_aux_pw(dig_port)) {
@@ -613,58 +1128,48 @@ static void intel_tc_port_reset_mode(struct intel_digital_port *dig_port,
 		drm_WARN_ON(&i915->drm, aux_powered);
 	}
 
-	icl_tc_phy_disconnect(dig_port);
+	tc_phy_disconnect(tc);
 	if (!force_disconnect)
-		icl_tc_phy_connect(dig_port, required_lanes);
+		tc_phy_connect(tc, required_lanes);
 
 	drm_dbg_kms(&i915->drm, "Port %s: TC port mode reset (%s -> %s)\n",
-		    dig_port->tc_port_name,
+		    tc->port_name,
 		    tc_port_mode_name(old_tc_mode),
-		    tc_port_mode_name(dig_port->tc_mode));
+		    tc_port_mode_name(tc->mode));
 }
 
-static bool intel_tc_port_needs_reset(struct intel_digital_port *dig_port)
+static bool intel_tc_port_needs_reset(struct intel_tc_port *tc)
 {
-	return intel_tc_port_get_target_mode(dig_port) != dig_port->tc_mode;
+	return tc_phy_get_target_mode(tc) != tc->mode;
 }
 
-static void intel_tc_port_update_mode(struct intel_digital_port *dig_port,
+static void intel_tc_port_update_mode(struct intel_tc_port *tc,
 				      int required_lanes, bool force_disconnect)
 {
-	enum intel_display_power_domain domain;
-	intel_wakeref_t wref;
-	bool needs_reset = force_disconnect;
-
-	if (!needs_reset) {
-		/* Get power domain required to check the hotplug live status. */
-		wref = tc_cold_block(dig_port, &domain);
-		needs_reset = intel_tc_port_needs_reset(dig_port);
-		tc_cold_unblock(dig_port, domain, wref);
-	}
-
-	if (!needs_reset)
-		return;
-
-	/* Get power domain required for resetting the mode. */
-	wref = tc_cold_block_in_mode(dig_port, TC_PORT_DISCONNECTED, &domain);
-
-	intel_tc_port_reset_mode(dig_port, required_lanes, force_disconnect);
+	if (force_disconnect ||
+	    intel_tc_port_needs_reset(tc))
+		intel_tc_port_reset_mode(tc, required_lanes, force_disconnect);
+}
 
-	/* Get power domain matching the new mode after reset. */
-	tc_cold_unblock(dig_port, dig_port->tc_lock_power_domain,
-			fetch_and_zero(&dig_port->tc_lock_wakeref));
-	if (dig_port->tc_mode != TC_PORT_DISCONNECTED)
-		dig_port->tc_lock_wakeref = tc_cold_block(dig_port,
-							  &dig_port->tc_lock_power_domain);
+static void __intel_tc_port_get_link(struct intel_tc_port *tc)
+{
+	tc->link_refcount++;
+}
 
-	tc_cold_unblock(dig_port, domain, wref);
+static void __intel_tc_port_put_link(struct intel_tc_port *tc)
+{
+	tc->link_refcount--;
 }
 
-static void
-intel_tc_port_link_init_refcount(struct intel_digital_port *dig_port,
-				 int refcount)
+static bool tc_port_is_enabled(struct intel_tc_port *tc)
 {
-	dig_port->tc_link_refcount = refcount;
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	struct intel_digital_port *dig_port = tc->dig_port;
+
+	assert_tc_port_power_enabled(tc);
+
+	return intel_de_read(i915, DDI_BUF_CTL(dig_port->base.port)) &
+	       DDI_BUF_CTL_ENABLE;
 }
 
 /**
@@ -677,85 +1182,119 @@ intel_tc_port_link_init_refcount(struct intel_digital_port *dig_port,
 void intel_tc_port_init_mode(struct intel_digital_port *dig_port)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	intel_wakeref_t tc_cold_wref;
-	enum intel_display_power_domain domain;
+	struct intel_tc_port *tc = to_tc_port(dig_port);
+	bool update_mode = false;
 
-	mutex_lock(&dig_port->tc_lock);
+	mutex_lock(&tc->lock);
 
-	drm_WARN_ON(&i915->drm, dig_port->tc_mode != TC_PORT_DISCONNECTED);
-	drm_WARN_ON(&i915->drm, dig_port->tc_lock_wakeref);
-	drm_WARN_ON(&i915->drm, dig_port->tc_link_refcount);
+	drm_WARN_ON(&i915->drm, tc->mode != TC_PORT_DISCONNECTED);
+	drm_WARN_ON(&i915->drm, tc->lock_wakeref);
+	drm_WARN_ON(&i915->drm, tc->link_refcount);
 
-	tc_cold_wref = tc_cold_block(dig_port, &domain);
+	tc_phy_get_hw_state(tc);
+	/*
+	 * Save the initial mode for the state check in
+	 * intel_tc_port_sanitize_mode().
+	 */
+	tc->init_mode = tc->mode;
+
+	/*
+	 * The PHY needs to be connected for AUX to work during HW readout and
+	 * MST topology resume, but the PHY mode can only be changed if the
+	 * port is disabled.
+	 *
+	 * An exception is the case where BIOS leaves the PHY incorrectly
+	 * disconnected on an enabled legacy port. Work around that by
+	 * connecting the PHY even though the port is enabled. This doesn't
+	 * cause a problem as the PHY ownership state is ignored by the
+	 * IOM/TCSS firmware (only display can own the PHY in that case).
+	 */
+	if (!tc_port_is_enabled(tc)) {
+		update_mode = true;
+	} else if (tc->mode == TC_PORT_DISCONNECTED) {
+		drm_WARN_ON(&i915->drm, !tc->legacy_port);
+		drm_err(&i915->drm,
+			"Port %s: PHY disconnected on enabled port, connecting it\n",
+			tc->port_name);
+		update_mode = true;
+	}
+
+	if (update_mode)
+		intel_tc_port_update_mode(tc, 1, false);
+
+	/* Prevent changing tc->mode until intel_tc_port_sanitize_mode() is called. */
+	__intel_tc_port_get_link(tc);
 
-	dig_port->tc_mode = intel_tc_port_get_current_mode(dig_port);
-	/* Prevent changing dig_port->tc_mode until intel_tc_port_sanitize_mode() is called. */
-	intel_tc_port_link_init_refcount(dig_port, 1);
-	dig_port->tc_lock_wakeref = tc_cold_block(dig_port, &dig_port->tc_lock_power_domain);
+	mutex_unlock(&tc->lock);
+}
+
+static bool tc_port_has_active_links(struct intel_tc_port *tc,
+				     const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = tc_to_i915(tc);
+	struct intel_digital_port *dig_port = tc->dig_port;
+	enum icl_port_dpll_id pll_type = ICL_PORT_DPLL_DEFAULT;
+	int active_links = 0;
 
-	tc_cold_unblock(dig_port, domain, tc_cold_wref);
+	if (dig_port->dp.is_mst) {
+		/* TODO: get the PLL type for MST, once HW readout is done for it. */
+		active_links = intel_dp_mst_encoder_active_links(dig_port);
+	} else if (crtc_state && crtc_state->hw.active) {
+		pll_type = intel_ddi_port_pll_type(&dig_port->base, crtc_state);
+		active_links = 1;
+	}
 
-	drm_dbg_kms(&i915->drm, "Port %s: init mode (%s)\n",
-		    dig_port->tc_port_name,
-		    tc_port_mode_name(dig_port->tc_mode));
+	if (active_links && !tc_phy_is_connected(tc, pll_type))
+		drm_err(&i915->drm,
+			"Port %s: PHY disconnected with %d active link(s)\n",
+			tc->port_name, active_links);
 
-	mutex_unlock(&dig_port->tc_lock);
+	return active_links;
 }
 
 /**
  * intel_tc_port_sanitize_mode: Sanitize the given port's TypeC mode
  * @dig_port: digital port
+ * @crtc_state: atomic state of CRTC connected to @dig_port
  *
  * Sanitize @dig_port's TypeC mode wrt. the encoder's state right after driver
  * loading and system resume:
  * If the encoder is enabled keep the TypeC mode/PHY connected state locked until
  * the encoder is disabled.
  * If the encoder is disabled make sure the PHY is disconnected.
+ * @crtc_state is valid if @dig_port is enabled, NULL otherwise.
  */
-void intel_tc_port_sanitize_mode(struct intel_digital_port *dig_port)
+void intel_tc_port_sanitize_mode(struct intel_digital_port *dig_port,
+				 const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	struct intel_encoder *encoder = &dig_port->base;
-	int active_links = 0;
-
-	mutex_lock(&dig_port->tc_lock);
+	struct intel_tc_port *tc = to_tc_port(dig_port);
 
-	if (dig_port->dp.is_mst)
-		active_links = intel_dp_mst_encoder_active_links(dig_port);
-	else if (encoder->base.crtc)
-		active_links = to_intel_crtc(encoder->base.crtc)->active;
-
-	drm_WARN_ON(&i915->drm, dig_port->tc_link_refcount != 1);
-	intel_tc_port_link_init_refcount(dig_port, active_links);
+	mutex_lock(&tc->lock);
 
-	if (active_links) {
-		if (!icl_tc_phy_is_connected(dig_port))
-			drm_dbg_kms(&i915->drm,
-				    "Port %s: PHY disconnected with %d active link(s)\n",
-				    dig_port->tc_port_name, active_links);
-	} else {
+	drm_WARN_ON(&i915->drm, tc->link_refcount != 1);
+	if (!tc_port_has_active_links(tc, crtc_state)) {
 		/*
 		 * TBT-alt is the default mode in any case the PHY ownership is not
 		 * held (regardless of the sink's connected live state), so
 		 * we'll just switch to disconnected mode from it here without
 		 * a note.
 		 */
-		if (dig_port->tc_mode != TC_PORT_TBT_ALT)
+		if (tc->init_mode != TC_PORT_TBT_ALT &&
+		    tc->init_mode != TC_PORT_DISCONNECTED)
 			drm_dbg_kms(&i915->drm,
 				    "Port %s: PHY left in %s mode on disabled port, disconnecting it\n",
-				    dig_port->tc_port_name,
-				    tc_port_mode_name(dig_port->tc_mode));
-		icl_tc_phy_disconnect(dig_port);
-
-		tc_cold_unblock(dig_port, dig_port->tc_lock_power_domain,
-				fetch_and_zero(&dig_port->tc_lock_wakeref));
+				    tc->port_name,
+				    tc_port_mode_name(tc->init_mode));
+		tc_phy_disconnect(tc);
+		__intel_tc_port_put_link(tc);
 	}
 
 	drm_dbg_kms(&i915->drm, "Port %s: sanitize mode (%s)\n",
-		    dig_port->tc_port_name,
-		    tc_port_mode_name(dig_port->tc_mode));
+		    tc->port_name,
+		    tc_port_mode_name(tc->mode));
 
-	mutex_unlock(&dig_port->tc_lock);
+	mutex_unlock(&tc->lock);
 }
 
 /*
@@ -768,63 +1307,73 @@ void intel_tc_port_sanitize_mode(struct intel_digital_port *dig_port)
  * connected ports are usable, and avoids exposing to the users objects they
  * can't really use.
  */
-bool intel_tc_port_connected(struct intel_encoder *encoder)
+bool intel_tc_port_connected_locked(struct intel_encoder *encoder)
 {
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
-	bool is_connected;
+	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct intel_tc_port *tc = to_tc_port(dig_port);
+	u32 mask = ~0;
 
-	intel_tc_port_lock(dig_port);
+	drm_WARN_ON(&i915->drm, !intel_tc_port_ref_held(dig_port));
 
-	is_connected = tc_port_live_status_mask(dig_port) &
-		       BIT(dig_port->tc_mode);
+	if (tc->mode != TC_PORT_DISCONNECTED)
+		mask = BIT(tc->mode);
 
-	intel_tc_port_unlock(dig_port);
+	return tc_phy_hpd_live_status(tc) & mask;
+}
+
+bool intel_tc_port_connected(struct intel_encoder *encoder)
+{
+	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
+	struct intel_tc_port *tc = to_tc_port(dig_port);
+	bool is_connected;
+
+	mutex_lock(&tc->lock);
+	is_connected = intel_tc_port_connected_locked(encoder);
+	mutex_unlock(&tc->lock);
 
 	return is_connected;
 }
 
-static void __intel_tc_port_lock(struct intel_digital_port *dig_port,
+static void __intel_tc_port_lock(struct intel_tc_port *tc,
 				 int required_lanes)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct drm_i915_private *i915 = tc_to_i915(tc);
 
-	mutex_lock(&dig_port->tc_lock);
+	mutex_lock(&tc->lock);
 
-	cancel_delayed_work(&dig_port->tc_disconnect_phy_work);
+	cancel_delayed_work(&tc->disconnect_phy_work);
 
-	if (!dig_port->tc_link_refcount)
-		intel_tc_port_update_mode(dig_port, required_lanes,
+	if (!tc->link_refcount)
+		intel_tc_port_update_mode(tc, required_lanes,
 					  false);
 
-	drm_WARN_ON(&i915->drm, dig_port->tc_mode == TC_PORT_DISCONNECTED);
-	drm_WARN_ON(&i915->drm, dig_port->tc_mode != TC_PORT_TBT_ALT &&
-				!tc_phy_is_owned(dig_port));
+	drm_WARN_ON(&i915->drm, tc->mode == TC_PORT_DISCONNECTED);
+	drm_WARN_ON(&i915->drm, tc->mode != TC_PORT_TBT_ALT &&
+				!tc_phy_is_owned(tc));
 }
 
 void intel_tc_port_lock(struct intel_digital_port *dig_port)
 {
-	__intel_tc_port_lock(dig_port, 1);
+	__intel_tc_port_lock(to_tc_port(dig_port), 1);
 }
 
-/**
- * intel_tc_port_disconnect_phy_work: disconnect TypeC PHY from display port
- * @dig_port: digital port
- *
+/*
  * Disconnect the given digital port from its TypeC PHY (handing back the
  * control of the PHY to the TypeC subsystem). This will happen in a delayed
  * manner after each aux transactions and modeset disables.
  */
 static void intel_tc_port_disconnect_phy_work(struct work_struct *work)
 {
-	struct intel_digital_port *dig_port =
-		container_of(work, struct intel_digital_port, tc_disconnect_phy_work.work);
+	struct intel_tc_port *tc =
+		container_of(work, struct intel_tc_port, disconnect_phy_work.work);
 
-	mutex_lock(&dig_port->tc_lock);
+	mutex_lock(&tc->lock);
 
-	if (!dig_port->tc_link_refcount)
-		intel_tc_port_update_mode(dig_port, 1, true);
+	if (!tc->link_refcount)
+		intel_tc_port_update_mode(tc, 1, true);
 
-	mutex_unlock(&dig_port->tc_lock);
+	mutex_unlock(&tc->lock);
 }
 
 /**
@@ -835,105 +1384,91 @@ static void intel_tc_port_disconnect_phy_work(struct work_struct *work)
  */
 void intel_tc_port_flush_work(struct intel_digital_port *dig_port)
 {
-	flush_delayed_work(&dig_port->tc_disconnect_phy_work);
+	flush_delayed_work(&to_tc_port(dig_port)->disconnect_phy_work);
 }
 
 void intel_tc_port_unlock(struct intel_digital_port *dig_port)
 {
-	if (!dig_port->tc_link_refcount && dig_port->tc_mode != TC_PORT_DISCONNECTED)
-		queue_delayed_work(system_unbound_wq, &dig_port->tc_disconnect_phy_work,
+	struct intel_tc_port *tc = to_tc_port(dig_port);
+
+	if (!tc->link_refcount && tc->mode != TC_PORT_DISCONNECTED)
+		queue_delayed_work(system_unbound_wq, &tc->disconnect_phy_work,
 				   msecs_to_jiffies(1000));
 
-	mutex_unlock(&dig_port->tc_lock);
+	mutex_unlock(&tc->lock);
 }
 
 bool intel_tc_port_ref_held(struct intel_digital_port *dig_port)
 {
-	return mutex_is_locked(&dig_port->tc_lock) ||
-	       dig_port->tc_link_refcount;
+	struct intel_tc_port *tc = to_tc_port(dig_port);
+
+	return mutex_is_locked(&tc->lock) ||
+	       tc->link_refcount;
 }
 
 void intel_tc_port_get_link(struct intel_digital_port *dig_port,
 			    int required_lanes)
 {
-	__intel_tc_port_lock(dig_port, required_lanes);
-	dig_port->tc_link_refcount++;
+	struct intel_tc_port *tc = to_tc_port(dig_port);
+
+	__intel_tc_port_lock(tc, required_lanes);
+	__intel_tc_port_get_link(tc);
 	intel_tc_port_unlock(dig_port);
 }
 
 void intel_tc_port_put_link(struct intel_digital_port *dig_port)
 {
+	struct intel_tc_port *tc = to_tc_port(dig_port);
+
 	intel_tc_port_lock(dig_port);
-	--dig_port->tc_link_refcount;
+	__intel_tc_port_put_link(tc);
 	intel_tc_port_unlock(dig_port);
-
-	/*
-	 * Disconnecting the PHY after the PHY's PLL gets disabled may
-	 * hang the system on ADL-P, so disconnect the PHY here synchronously.
-	 * TODO: remove this once the root cause of the ordering requirement
-	 * is found/fixed.
-	 */
-	intel_tc_port_flush_work(dig_port);
 }
 
-static bool
-tc_has_modular_fia(struct drm_i915_private *i915, struct intel_digital_port *dig_port)
+int intel_tc_port_init(struct intel_digital_port *dig_port, bool is_legacy)
 {
-	enum intel_display_power_domain domain;
-	intel_wakeref_t wakeref;
-	u32 val;
+	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
+	struct intel_tc_port *tc;
+	enum port port = dig_port->base.port;
+	enum tc_port tc_port = intel_port_to_tc(i915, port);
 
-	if (!INTEL_INFO(i915)->display.has_modular_fia)
-		return false;
+	if (drm_WARN_ON(&i915->drm, tc_port == TC_PORT_NONE))
+		return -EINVAL;
 
-	mutex_lock(&dig_port->tc_lock);
-	wakeref = tc_cold_block(dig_port, &domain);
-	val = intel_de_read(i915, PORT_TX_DFLEXDPSP(FIA1));
-	tc_cold_unblock(dig_port, domain, wakeref);
-	mutex_unlock(&dig_port->tc_lock);
+	tc = kzalloc(sizeof(*tc), GFP_KERNEL);
+	if (!tc)
+		return -ENOMEM;
 
-	drm_WARN_ON(&i915->drm, val == 0xffffffff);
+	dig_port->tc = tc;
+	tc->dig_port = dig_port;
 
-	return val & MODULAR_FIA_MASK;
-}
+	if (DISPLAY_VER(i915) >= 13)
+		tc->phy_ops = &adlp_tc_phy_ops;
+	else if (DISPLAY_VER(i915) >= 12)
+		tc->phy_ops = &tgl_tc_phy_ops;
+	else
+		tc->phy_ops = &icl_tc_phy_ops;
 
-static void
-tc_port_load_fia_params(struct drm_i915_private *i915, struct intel_digital_port *dig_port)
-{
-	enum port port = dig_port->base.port;
-	enum tc_port tc_port = intel_port_to_tc(i915, port);
+	snprintf(tc->port_name, sizeof(tc->port_name),
+		 "%c/TC#%d", port_name(port), tc_port + 1);
 
-	/*
-	 * Each Modular FIA instance houses 2 TC ports. In SOC that has more
-	 * than two TC ports, there are multiple instances of Modular FIA.
-	 */
-	if (tc_has_modular_fia(i915, dig_port)) {
-		dig_port->tc_phy_fia = tc_port / 2;
-		dig_port->tc_phy_fia_idx = tc_port % 2;
-	} else {
-		dig_port->tc_phy_fia = FIA1;
-		dig_port->tc_phy_fia_idx = tc_port;
-	}
-}
+	mutex_init(&tc->lock);
+	INIT_DELAYED_WORK(&tc->disconnect_phy_work, intel_tc_port_disconnect_phy_work);
+	tc->legacy_port = is_legacy;
+	tc->mode = TC_PORT_DISCONNECTED;
+	tc->link_refcount = 0;
 
-void intel_tc_port_init(struct intel_digital_port *dig_port, bool is_legacy)
-{
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum port port = dig_port->base.port;
-	enum tc_port tc_port = intel_port_to_tc(i915, port);
+	tc_phy_init(tc);
 
-	if (drm_WARN_ON(&i915->drm, tc_port == TC_PORT_NONE))
-		return;
+	intel_tc_port_init_mode(dig_port);
 
-	snprintf(dig_port->tc_port_name, sizeof(dig_port->tc_port_name),
-		 "%c/TC#%d", port_name(port), tc_port + 1);
+	return 0;
+}
 
-	mutex_init(&dig_port->tc_lock);
-	INIT_DELAYED_WORK(&dig_port->tc_disconnect_phy_work, intel_tc_port_disconnect_phy_work);
-	dig_port->tc_legacy_port = is_legacy;
-	dig_port->tc_mode = TC_PORT_DISCONNECTED;
-	dig_port->tc_link_refcount = 0;
-	tc_port_load_fia_params(i915, dig_port);
+void intel_tc_port_cleanup(struct intel_digital_port *dig_port)
+{
+	intel_tc_port_flush_work(dig_port);
 
-	intel_tc_port_init_mode(dig_port);
+	kfree(dig_port->tc);
+	dig_port->tc = NULL;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_tc.h b/drivers/gpu/drm/i915/display/intel_tc.h
index d54082e2d5e8..dd0810f9ea95 100644
--- a/drivers/gpu/drm/i915/display/intel_tc.h
+++ b/drivers/gpu/drm/i915/display/intel_tc.h
@@ -6,9 +6,9 @@
 #ifndef __INTEL_TC_H__
 #define __INTEL_TC_H__
 
-#include <linux/mutex.h>
 #include <linux/types.h>
 
+struct intel_crtc_state;
 struct intel_digital_port;
 struct intel_encoder;
 
@@ -17,6 +17,7 @@ bool intel_tc_port_in_dp_alt_mode(struct intel_digital_port *dig_port);
 bool intel_tc_port_in_legacy_mode(struct intel_digital_port *dig_port);
 
 bool intel_tc_port_connected(struct intel_encoder *encoder);
+bool intel_tc_port_connected_locked(struct intel_encoder *encoder);
 
 u32 intel_tc_port_get_lane_mask(struct intel_digital_port *dig_port);
 u32 intel_tc_port_get_pin_assignment_mask(struct intel_digital_port *dig_port);
@@ -25,7 +26,8 @@ void intel_tc_port_set_fia_lane_count(struct intel_digital_port *dig_port,
 				      int required_lanes);
 
 void intel_tc_port_init_mode(struct intel_digital_port *dig_port);
-void intel_tc_port_sanitize_mode(struct intel_digital_port *dig_port);
+void intel_tc_port_sanitize_mode(struct intel_digital_port *dig_port,
+				 const struct intel_crtc_state *crtc_state);
 void intel_tc_port_lock(struct intel_digital_port *dig_port);
 void intel_tc_port_unlock(struct intel_digital_port *dig_port);
 void intel_tc_port_flush_work(struct intel_digital_port *dig_port);
@@ -34,7 +36,8 @@ void intel_tc_port_get_link(struct intel_digital_port *dig_port,
 void intel_tc_port_put_link(struct intel_digital_port *dig_port);
 bool intel_tc_port_ref_held(struct intel_digital_port *dig_port);
 
-void intel_tc_port_init(struct intel_digital_port *dig_port, bool is_legacy);
+int intel_tc_port_init(struct intel_digital_port *dig_port, bool is_legacy);
+void intel_tc_port_cleanup(struct intel_digital_port *dig_port);
 
 bool intel_tc_cold_requires_aux_pw(struct intel_digital_port *dig_port);
 
diff --git a/drivers/gpu/drm/i915/display/intel_tv.c b/drivers/gpu/drm/i915/display/intel_tv.c
index b986bf075889..557ec5b62afa 100644
--- a/drivers/gpu/drm/i915/display/intel_tv.c
+++ b/drivers/gpu/drm/i915/display/intel_tv.c
@@ -35,8 +35,8 @@
 #include <drm/drm_edid.h>
 
 #include "i915_drv.h"
-#include "i915_reg.h"
 #include "i915_irq.h"
+#include "i915_reg.h"
 #include "intel_connector.h"
 #include "intel_crtc.h"
 #include "intel_de.h"
@@ -44,6 +44,7 @@
 #include "intel_dpll.h"
 #include "intel_hotplug.h"
 #include "intel_tv.h"
+#include "intel_tv_regs.h"
 
 enum tv_margin {
 	TV_MARGIN_LEFT, TV_MARGIN_TOP,
@@ -930,8 +931,7 @@ intel_enable_tv(struct intel_atomic_state *state,
 	/* Prevents vblank waits from timing out in intel_tv_detect_type() */
 	intel_crtc_wait_for_next_vblank(to_intel_crtc(pipe_config->uapi.crtc));
 
-	intel_de_write(dev_priv, TV_CTL,
-		       intel_de_read(dev_priv, TV_CTL) | TV_ENC_ENABLE);
+	intel_de_rmw(dev_priv, TV_CTL, 0, TV_ENC_ENABLE);
 }
 
 static void
@@ -943,8 +943,7 @@ intel_disable_tv(struct intel_atomic_state *state,
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	intel_de_write(dev_priv, TV_CTL,
-		       intel_de_read(dev_priv, TV_CTL) & ~TV_ENC_ENABLE);
+	intel_de_rmw(dev_priv, TV_CTL, TV_ENC_ENABLE, 0);
 }
 
 static const struct tv_mode *intel_tv_mode_find(const struct drm_connector_state *conn_state)
diff --git a/drivers/gpu/drm/i915/display/intel_tv_regs.h b/drivers/gpu/drm/i915/display/intel_tv_regs.h
new file mode 100644
index 000000000000..ab25aeb3c423
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_tv_regs.h
@@ -0,0 +1,490 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_TV_REGS_H__
+#define __INTEL_TV_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+/* TV port control */
+#define TV_CTL			_MMIO(0x68000)
+/* Enables the TV encoder */
+# define TV_ENC_ENABLE			(1 << 31)
+/* Sources the TV encoder input from pipe B instead of A. */
+# define TV_ENC_PIPE_SEL_SHIFT		30
+# define TV_ENC_PIPE_SEL_MASK		(1 << 30)
+# define TV_ENC_PIPE_SEL(pipe)		((pipe) << 30)
+/* Outputs composite video (DAC A only) */
+# define TV_ENC_OUTPUT_COMPOSITE	(0 << 28)
+/* Outputs SVideo video (DAC B/C) */
+# define TV_ENC_OUTPUT_SVIDEO		(1 << 28)
+/* Outputs Component video (DAC A/B/C) */
+# define TV_ENC_OUTPUT_COMPONENT	(2 << 28)
+/* Outputs Composite and SVideo (DAC A/B/C) */
+# define TV_ENC_OUTPUT_SVIDEO_COMPOSITE	(3 << 28)
+# define TV_TRILEVEL_SYNC		(1 << 21)
+/* Enables slow sync generation (945GM only) */
+# define TV_SLOW_SYNC			(1 << 20)
+/* Selects 4x oversampling for 480i and 576p */
+# define TV_OVERSAMPLE_4X		(0 << 18)
+/* Selects 2x oversampling for 720p and 1080i */
+# define TV_OVERSAMPLE_2X		(1 << 18)
+/* Selects no oversampling for 1080p */
+# define TV_OVERSAMPLE_NONE		(2 << 18)
+/* Selects 8x oversampling */
+# define TV_OVERSAMPLE_8X		(3 << 18)
+# define TV_OVERSAMPLE_MASK		(3 << 18)
+/* Selects progressive mode rather than interlaced */
+# define TV_PROGRESSIVE			(1 << 17)
+/* Sets the colorburst to PAL mode.  Required for non-M PAL modes. */
+# define TV_PAL_BURST			(1 << 16)
+/* Field for setting delay of Y compared to C */
+# define TV_YC_SKEW_MASK		(7 << 12)
+/* Enables a fix for 480p/576p standard definition modes on the 915GM only */
+# define TV_ENC_SDP_FIX			(1 << 11)
+/*
+ * Enables a fix for the 915GM only.
+ *
+ * Not sure what it does.
+ */
+# define TV_ENC_C0_FIX			(1 << 10)
+/* Bits that must be preserved by software */
+# define TV_CTL_SAVE			((1 << 11) | (3 << 9) | (7 << 6) | 0xf)
+# define TV_FUSE_STATE_MASK		(3 << 4)
+/* Read-only state that reports all features enabled */
+# define TV_FUSE_STATE_ENABLED		(0 << 4)
+/* Read-only state that reports that Macrovision is disabled in hardware*/
+# define TV_FUSE_STATE_NO_MACROVISION	(1 << 4)
+/* Read-only state that reports that TV-out is disabled in hardware. */
+# define TV_FUSE_STATE_DISABLED		(2 << 4)
+/* Normal operation */
+# define TV_TEST_MODE_NORMAL		(0 << 0)
+/* Encoder test pattern 1 - combo pattern */
+# define TV_TEST_MODE_PATTERN_1		(1 << 0)
+/* Encoder test pattern 2 - full screen vertical 75% color bars */
+# define TV_TEST_MODE_PATTERN_2		(2 << 0)
+/* Encoder test pattern 3 - full screen horizontal 75% color bars */
+# define TV_TEST_MODE_PATTERN_3		(3 << 0)
+/* Encoder test pattern 4 - random noise */
+# define TV_TEST_MODE_PATTERN_4		(4 << 0)
+/* Encoder test pattern 5 - linear color ramps */
+# define TV_TEST_MODE_PATTERN_5		(5 << 0)
+/*
+ * This test mode forces the DACs to 50% of full output.
+ *
+ * This is used for load detection in combination with TVDAC_SENSE_MASK
+ */
+# define TV_TEST_MODE_MONITOR_DETECT	(7 << 0)
+# define TV_TEST_MODE_MASK		(7 << 0)
+
+#define TV_DAC			_MMIO(0x68004)
+# define TV_DAC_SAVE		0x00ffff00
+/*
+ * Reports that DAC state change logic has reported change (RO).
+ *
+ * This gets cleared when TV_DAC_STATE_EN is cleared
+*/
+# define TVDAC_STATE_CHG		(1 << 31)
+# define TVDAC_SENSE_MASK		(7 << 28)
+/* Reports that DAC A voltage is above the detect threshold */
+# define TVDAC_A_SENSE			(1 << 30)
+/* Reports that DAC B voltage is above the detect threshold */
+# define TVDAC_B_SENSE			(1 << 29)
+/* Reports that DAC C voltage is above the detect threshold */
+# define TVDAC_C_SENSE			(1 << 28)
+/*
+ * Enables DAC state detection logic, for load-based TV detection.
+ *
+ * The PLL of the chosen pipe (in TV_CTL) must be running, and the encoder set
+ * to off, for load detection to work.
+ */
+# define TVDAC_STATE_CHG_EN		(1 << 27)
+/* Sets the DAC A sense value to high */
+# define TVDAC_A_SENSE_CTL		(1 << 26)
+/* Sets the DAC B sense value to high */
+# define TVDAC_B_SENSE_CTL		(1 << 25)
+/* Sets the DAC C sense value to high */
+# define TVDAC_C_SENSE_CTL		(1 << 24)
+/* Overrides the ENC_ENABLE and DAC voltage levels */
+# define DAC_CTL_OVERRIDE		(1 << 7)
+/* Sets the slew rate.  Must be preserved in software */
+# define ENC_TVDAC_SLEW_FAST		(1 << 6)
+# define DAC_A_1_3_V			(0 << 4)
+# define DAC_A_1_1_V			(1 << 4)
+# define DAC_A_0_7_V			(2 << 4)
+# define DAC_A_MASK			(3 << 4)
+# define DAC_B_1_3_V			(0 << 2)
+# define DAC_B_1_1_V			(1 << 2)
+# define DAC_B_0_7_V			(2 << 2)
+# define DAC_B_MASK			(3 << 2)
+# define DAC_C_1_3_V			(0 << 0)
+# define DAC_C_1_1_V			(1 << 0)
+# define DAC_C_0_7_V			(2 << 0)
+# define DAC_C_MASK			(3 << 0)
+
+/*
+ * CSC coefficients are stored in a floating point format with 9 bits of
+ * mantissa and 2 or 3 bits of exponent.  The exponent is represented as 2**-n,
+ * where 2-bit exponents are unsigned n, and 3-bit exponents are signed n with
+ * -1 (0x3) being the only legal negative value.
+ */
+#define TV_CSC_Y		_MMIO(0x68010)
+# define TV_RY_MASK			0x07ff0000
+# define TV_RY_SHIFT			16
+# define TV_GY_MASK			0x00000fff
+# define TV_GY_SHIFT			0
+
+#define TV_CSC_Y2		_MMIO(0x68014)
+# define TV_BY_MASK			0x07ff0000
+# define TV_BY_SHIFT			16
+/*
+ * Y attenuation for component video.
+ *
+ * Stored in 1.9 fixed point.
+ */
+# define TV_AY_MASK			0x000003ff
+# define TV_AY_SHIFT			0
+
+#define TV_CSC_U		_MMIO(0x68018)
+# define TV_RU_MASK			0x07ff0000
+# define TV_RU_SHIFT			16
+# define TV_GU_MASK			0x000007ff
+# define TV_GU_SHIFT			0
+
+#define TV_CSC_U2		_MMIO(0x6801c)
+# define TV_BU_MASK			0x07ff0000
+# define TV_BU_SHIFT			16
+/*
+ * U attenuation for component video.
+ *
+ * Stored in 1.9 fixed point.
+ */
+# define TV_AU_MASK			0x000003ff
+# define TV_AU_SHIFT			0
+
+#define TV_CSC_V		_MMIO(0x68020)
+# define TV_RV_MASK			0x0fff0000
+# define TV_RV_SHIFT			16
+# define TV_GV_MASK			0x000007ff
+# define TV_GV_SHIFT			0
+
+#define TV_CSC_V2		_MMIO(0x68024)
+# define TV_BV_MASK			0x07ff0000
+# define TV_BV_SHIFT			16
+/*
+ * V attenuation for component video.
+ *
+ * Stored in 1.9 fixed point.
+ */
+# define TV_AV_MASK			0x000007ff
+# define TV_AV_SHIFT			0
+
+#define TV_CLR_KNOBS		_MMIO(0x68028)
+/* 2s-complement brightness adjustment */
+# define TV_BRIGHTNESS_MASK		0xff000000
+# define TV_BRIGHTNESS_SHIFT		24
+/* Contrast adjustment, as a 2.6 unsigned floating point number */
+# define TV_CONTRAST_MASK		0x00ff0000
+# define TV_CONTRAST_SHIFT		16
+/* Saturation adjustment, as a 2.6 unsigned floating point number */
+# define TV_SATURATION_MASK		0x0000ff00
+# define TV_SATURATION_SHIFT		8
+/* Hue adjustment, as an integer phase angle in degrees */
+# define TV_HUE_MASK			0x000000ff
+# define TV_HUE_SHIFT			0
+
+#define TV_CLR_LEVEL		_MMIO(0x6802c)
+/* Controls the DAC level for black */
+# define TV_BLACK_LEVEL_MASK		0x01ff0000
+# define TV_BLACK_LEVEL_SHIFT		16
+/* Controls the DAC level for blanking */
+# define TV_BLANK_LEVEL_MASK		0x000001ff
+# define TV_BLANK_LEVEL_SHIFT		0
+
+#define TV_H_CTL_1		_MMIO(0x68030)
+/* Number of pixels in the hsync. */
+# define TV_HSYNC_END_MASK		0x1fff0000
+# define TV_HSYNC_END_SHIFT		16
+/* Total number of pixels minus one in the line (display and blanking). */
+# define TV_HTOTAL_MASK			0x00001fff
+# define TV_HTOTAL_SHIFT		0
+
+#define TV_H_CTL_2		_MMIO(0x68034)
+/* Enables the colorburst (needed for non-component color) */
+# define TV_BURST_ENA			(1 << 31)
+/* Offset of the colorburst from the start of hsync, in pixels minus one. */
+# define TV_HBURST_START_SHIFT		16
+# define TV_HBURST_START_MASK		0x1fff0000
+/* Length of the colorburst */
+# define TV_HBURST_LEN_SHIFT		0
+# define TV_HBURST_LEN_MASK		0x0001fff
+
+#define TV_H_CTL_3		_MMIO(0x68038)
+/* End of hblank, measured in pixels minus one from start of hsync */
+# define TV_HBLANK_END_SHIFT		16
+# define TV_HBLANK_END_MASK		0x1fff0000
+/* Start of hblank, measured in pixels minus one from start of hsync */
+# define TV_HBLANK_START_SHIFT		0
+# define TV_HBLANK_START_MASK		0x0001fff
+
+#define TV_V_CTL_1		_MMIO(0x6803c)
+/* XXX */
+# define TV_NBR_END_SHIFT		16
+# define TV_NBR_END_MASK		0x07ff0000
+/* XXX */
+# define TV_VI_END_F1_SHIFT		8
+# define TV_VI_END_F1_MASK		0x00003f00
+/* XXX */
+# define TV_VI_END_F2_SHIFT		0
+# define TV_VI_END_F2_MASK		0x0000003f
+
+#define TV_V_CTL_2		_MMIO(0x68040)
+/* Length of vsync, in half lines */
+# define TV_VSYNC_LEN_MASK		0x07ff0000
+# define TV_VSYNC_LEN_SHIFT		16
+/* Offset of the start of vsync in field 1, measured in one less than the
+ * number of half lines.
+ */
+# define TV_VSYNC_START_F1_MASK		0x00007f00
+# define TV_VSYNC_START_F1_SHIFT	8
+/*
+ * Offset of the start of vsync in field 2, measured in one less than the
+ * number of half lines.
+ */
+# define TV_VSYNC_START_F2_MASK		0x0000007f
+# define TV_VSYNC_START_F2_SHIFT	0
+
+#define TV_V_CTL_3		_MMIO(0x68044)
+/* Enables generation of the equalization signal */
+# define TV_EQUAL_ENA			(1 << 31)
+/* Length of vsync, in half lines */
+# define TV_VEQ_LEN_MASK		0x007f0000
+# define TV_VEQ_LEN_SHIFT		16
+/* Offset of the start of equalization in field 1, measured in one less than
+ * the number of half lines.
+ */
+# define TV_VEQ_START_F1_MASK		0x0007f00
+# define TV_VEQ_START_F1_SHIFT		8
+/*
+ * Offset of the start of equalization in field 2, measured in one less than
+ * the number of half lines.
+ */
+# define TV_VEQ_START_F2_MASK		0x000007f
+# define TV_VEQ_START_F2_SHIFT		0
+
+#define TV_V_CTL_4		_MMIO(0x68048)
+/*
+ * Offset to start of vertical colorburst, measured in one less than the
+ * number of lines from vertical start.
+ */
+# define TV_VBURST_START_F1_MASK	0x003f0000
+# define TV_VBURST_START_F1_SHIFT	16
+/*
+ * Offset to the end of vertical colorburst, measured in one less than the
+ * number of lines from the start of NBR.
+ */
+# define TV_VBURST_END_F1_MASK		0x000000ff
+# define TV_VBURST_END_F1_SHIFT		0
+
+#define TV_V_CTL_5		_MMIO(0x6804c)
+/*
+ * Offset to start of vertical colorburst, measured in one less than the
+ * number of lines from vertical start.
+ */
+# define TV_VBURST_START_F2_MASK	0x003f0000
+# define TV_VBURST_START_F2_SHIFT	16
+/*
+ * Offset to the end of vertical colorburst, measured in one less than the
+ * number of lines from the start of NBR.
+ */
+# define TV_VBURST_END_F2_MASK		0x000000ff
+# define TV_VBURST_END_F2_SHIFT		0
+
+#define TV_V_CTL_6		_MMIO(0x68050)
+/*
+ * Offset to start of vertical colorburst, measured in one less than the
+ * number of lines from vertical start.
+ */
+# define TV_VBURST_START_F3_MASK	0x003f0000
+# define TV_VBURST_START_F3_SHIFT	16
+/*
+ * Offset to the end of vertical colorburst, measured in one less than the
+ * number of lines from the start of NBR.
+ */
+# define TV_VBURST_END_F3_MASK		0x000000ff
+# define TV_VBURST_END_F3_SHIFT		0
+
+#define TV_V_CTL_7		_MMIO(0x68054)
+/*
+ * Offset to start of vertical colorburst, measured in one less than the
+ * number of lines from vertical start.
+ */
+# define TV_VBURST_START_F4_MASK	0x003f0000
+# define TV_VBURST_START_F4_SHIFT	16
+/*
+ * Offset to the end of vertical colorburst, measured in one less than the
+ * number of lines from the start of NBR.
+ */
+# define TV_VBURST_END_F4_MASK		0x000000ff
+# define TV_VBURST_END_F4_SHIFT		0
+
+#define TV_SC_CTL_1		_MMIO(0x68060)
+/* Turns on the first subcarrier phase generation DDA */
+# define TV_SC_DDA1_EN			(1 << 31)
+/* Turns on the first subcarrier phase generation DDA */
+# define TV_SC_DDA2_EN			(1 << 30)
+/* Turns on the first subcarrier phase generation DDA */
+# define TV_SC_DDA3_EN			(1 << 29)
+/* Sets the subcarrier DDA to reset frequency every other field */
+# define TV_SC_RESET_EVERY_2		(0 << 24)
+/* Sets the subcarrier DDA to reset frequency every fourth field */
+# define TV_SC_RESET_EVERY_4		(1 << 24)
+/* Sets the subcarrier DDA to reset frequency every eighth field */
+# define TV_SC_RESET_EVERY_8		(2 << 24)
+/* Sets the subcarrier DDA to never reset the frequency */
+# define TV_SC_RESET_NEVER		(3 << 24)
+/* Sets the peak amplitude of the colorburst.*/
+# define TV_BURST_LEVEL_MASK		0x00ff0000
+# define TV_BURST_LEVEL_SHIFT		16
+/* Sets the increment of the first subcarrier phase generation DDA */
+# define TV_SCDDA1_INC_MASK		0x00000fff
+# define TV_SCDDA1_INC_SHIFT		0
+
+#define TV_SC_CTL_2		_MMIO(0x68064)
+/* Sets the rollover for the second subcarrier phase generation DDA */
+# define TV_SCDDA2_SIZE_MASK		0x7fff0000
+# define TV_SCDDA2_SIZE_SHIFT		16
+/* Sets the increent of the second subcarrier phase generation DDA */
+# define TV_SCDDA2_INC_MASK		0x00007fff
+# define TV_SCDDA2_INC_SHIFT		0
+
+#define TV_SC_CTL_3		_MMIO(0x68068)
+/* Sets the rollover for the third subcarrier phase generation DDA */
+# define TV_SCDDA3_SIZE_MASK		0x7fff0000
+# define TV_SCDDA3_SIZE_SHIFT		16
+/* Sets the increent of the third subcarrier phase generation DDA */
+# define TV_SCDDA3_INC_MASK		0x00007fff
+# define TV_SCDDA3_INC_SHIFT		0
+
+#define TV_WIN_POS		_MMIO(0x68070)
+/* X coordinate of the display from the start of horizontal active */
+# define TV_XPOS_MASK			0x1fff0000
+# define TV_XPOS_SHIFT			16
+/* Y coordinate of the display from the start of vertical active (NBR) */
+# define TV_YPOS_MASK			0x00000fff
+# define TV_YPOS_SHIFT			0
+
+#define TV_WIN_SIZE		_MMIO(0x68074)
+/* Horizontal size of the display window, measured in pixels*/
+# define TV_XSIZE_MASK			0x1fff0000
+# define TV_XSIZE_SHIFT			16
+/*
+ * Vertical size of the display window, measured in pixels.
+ *
+ * Must be even for interlaced modes.
+ */
+# define TV_YSIZE_MASK			0x00000fff
+# define TV_YSIZE_SHIFT			0
+
+#define TV_FILTER_CTL_1		_MMIO(0x68080)
+/*
+ * Enables automatic scaling calculation.
+ *
+ * If set, the rest of the registers are ignored, and the calculated values can
+ * be read back from the register.
+ */
+# define TV_AUTO_SCALE			(1 << 31)
+/*
+ * Disables the vertical filter.
+ *
+ * This is required on modes more than 1024 pixels wide */
+# define TV_V_FILTER_BYPASS		(1 << 29)
+/* Enables adaptive vertical filtering */
+# define TV_VADAPT			(1 << 28)
+# define TV_VADAPT_MODE_MASK		(3 << 26)
+/* Selects the least adaptive vertical filtering mode */
+# define TV_VADAPT_MODE_LEAST		(0 << 26)
+/* Selects the moderately adaptive vertical filtering mode */
+# define TV_VADAPT_MODE_MODERATE	(1 << 26)
+/* Selects the most adaptive vertical filtering mode */
+# define TV_VADAPT_MODE_MOST		(3 << 26)
+/*
+ * Sets the horizontal scaling factor.
+ *
+ * This should be the fractional part of the horizontal scaling factor divided
+ * by the oversampling rate.  TV_HSCALE should be less than 1, and set to:
+ *
+ * (src width - 1) / ((oversample * dest width) - 1)
+ */
+# define TV_HSCALE_FRAC_MASK		0x00003fff
+# define TV_HSCALE_FRAC_SHIFT		0
+
+#define TV_FILTER_CTL_2		_MMIO(0x68084)
+/*
+ * Sets the integer part of the 3.15 fixed-point vertical scaling factor.
+ *
+ * TV_VSCALE should be (src height - 1) / ((interlace * dest height) - 1)
+ */
+# define TV_VSCALE_INT_MASK		0x00038000
+# define TV_VSCALE_INT_SHIFT		15
+/*
+ * Sets the fractional part of the 3.15 fixed-point vertical scaling factor.
+ *
+ * \sa TV_VSCALE_INT_MASK
+ */
+# define TV_VSCALE_FRAC_MASK		0x00007fff
+# define TV_VSCALE_FRAC_SHIFT		0
+
+#define TV_FILTER_CTL_3		_MMIO(0x68088)
+/*
+ * Sets the integer part of the 3.15 fixed-point vertical scaling factor.
+ *
+ * TV_VSCALE should be (src height - 1) / (1/4 * (dest height - 1))
+ *
+ * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes.
+ */
+# define TV_VSCALE_IP_INT_MASK		0x00038000
+# define TV_VSCALE_IP_INT_SHIFT		15
+/*
+ * Sets the fractional part of the 3.15 fixed-point vertical scaling factor.
+ *
+ * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes.
+ *
+ * \sa TV_VSCALE_IP_INT_MASK
+ */
+# define TV_VSCALE_IP_FRAC_MASK		0x00007fff
+# define TV_VSCALE_IP_FRAC_SHIFT		0
+
+#define TV_CC_CONTROL		_MMIO(0x68090)
+# define TV_CC_ENABLE			(1 << 31)
+/*
+ * Specifies which field to send the CC data in.
+ *
+ * CC data is usually sent in field 0.
+ */
+# define TV_CC_FID_MASK			(1 << 27)
+# define TV_CC_FID_SHIFT		27
+/* Sets the horizontal position of the CC data.  Usually 135. */
+# define TV_CC_HOFF_MASK		0x03ff0000
+# define TV_CC_HOFF_SHIFT		16
+/* Sets the vertical position of the CC data.  Usually 21 */
+# define TV_CC_LINE_MASK		0x0000003f
+# define TV_CC_LINE_SHIFT		0
+
+#define TV_CC_DATA		_MMIO(0x68094)
+# define TV_CC_RDY			(1 << 31)
+/* Second word of CC data to be transmitted. */
+# define TV_CC_DATA_2_MASK		0x007f0000
+# define TV_CC_DATA_2_SHIFT		16
+/* First word of CC data to be transmitted. */
+# define TV_CC_DATA_1_MASK		0x0000007f
+# define TV_CC_DATA_1_SHIFT		0
+
+#define TV_H_LUMA(i)		_MMIO(0x68100 + (i) * 4) /* 60 registers */
+#define TV_H_CHROMA(i)		_MMIO(0x68200 + (i) * 4) /* 60 registers */
+#define TV_V_LUMA(i)		_MMIO(0x68300 + (i) * 4) /* 43 registers */
+#define TV_V_CHROMA(i)		_MMIO(0x68400 + (i) * 4) /* 43 registers */
+
+#endif /* __INTEL_TV_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_vblank.c b/drivers/gpu/drm/i915/display/intel_vblank.c
index 4c83e2320bca..f8bf9810527d 100644
--- a/drivers/gpu/drm/i915/display/intel_vblank.c
+++ b/drivers/gpu/drm/i915/display/intel_vblank.c
@@ -8,6 +8,7 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_vblank.h"
+#include "intel_vrr.h"
 
 /*
  * This timing diagram depicts the video signal in and
@@ -26,7 +27,7 @@
  *           |
  *           |          frame start:
  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
- *           |          may be shifted forward 1-3 extra lines via PIPECONF
+ *           |          may be shifted forward 1-3 extra lines via TRANSCONF
  *           |          |
  *           |          |  start of vsync:
  *           |          |  generate vsync interrupt
@@ -54,7 +55,7 @@
  * Summary:
  * - most events happen at the start of horizontal sync
  * - frame start happens at the start of horizontal blank, 1-4 lines
- *   (depending on PIPECONF settings) after the start of vblank
+ *   (depending on TRANSCONF settings) after the start of vblank
  * - gen3/4 pixel and frame counter are synchronized with the start
  *   of horizontal active on the first line of vertical active
  */
@@ -439,3 +440,94 @@ void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
 {
 	wait_for_pipe_scanline_moving(crtc, true);
 }
+
+static int intel_crtc_scanline_offset(const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+
+	/*
+	 * The scanline counter increments at the leading edge of hsync.
+	 *
+	 * On most platforms it starts counting from vtotal-1 on the
+	 * first active line. That means the scanline counter value is
+	 * always one less than what we would expect. Ie. just after
+	 * start of vblank, which also occurs at start of hsync (on the
+	 * last active line), the scanline counter will read vblank_start-1.
+	 *
+	 * On gen2 the scanline counter starts counting from 1 instead
+	 * of vtotal-1, so we have to subtract one (or rather add vtotal-1
+	 * to keep the value positive), instead of adding one.
+	 *
+	 * On HSW+ the behaviour of the scanline counter depends on the output
+	 * type. For DP ports it behaves like most other platforms, but on HDMI
+	 * there's an extra 1 line difference. So we need to add two instead of
+	 * one to the value.
+	 *
+	 * On VLV/CHV DSI the scanline counter would appear to increment
+	 * approx. 1/3 of a scanline before start of vblank. Unfortunately
+	 * that means we can't tell whether we're in vblank or not while
+	 * we're on that particular line. We must still set scanline_offset
+	 * to 1 so that the vblank timestamps come out correct when we query
+	 * the scanline counter from within the vblank interrupt handler.
+	 * However if queried just before the start of vblank we'll get an
+	 * answer that's slightly in the future.
+	 */
+	if (DISPLAY_VER(i915) == 2) {
+		int vtotal;
+
+		vtotal = adjusted_mode->crtc_vtotal;
+		if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
+			vtotal /= 2;
+
+		return vtotal - 1;
+	} else if (HAS_DDI(i915) && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
+		return 2;
+	} else {
+		return 1;
+	}
+}
+
+void intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct drm_display_mode adjusted_mode;
+	int vmax_vblank_start = 0;
+	unsigned long irqflags;
+
+	drm_mode_init(&adjusted_mode, &crtc_state->hw.adjusted_mode);
+
+	if (crtc_state->vrr.enable) {
+		adjusted_mode.crtc_vtotal = crtc_state->vrr.vmax;
+		adjusted_mode.crtc_vblank_end = crtc_state->vrr.vmax;
+		adjusted_mode.crtc_vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
+		vmax_vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
+	}
+
+	/*
+	 * Belts and suspenders locking to guarantee everyone sees 100%
+	 * consistent state during fastset seamless refresh rate changes.
+	 *
+	 * vblank_time_lock takes care of all drm_vblank.c stuff, and
+	 * uncore.lock takes care of __intel_get_crtc_scanline() which
+	 * may get called elsewhere as well.
+	 *
+	 * TODO maybe just protect everything (including
+	 * __intel_get_crtc_scanline()) with vblank_time_lock?
+	 * Need to audit everything to make sure it's safe.
+	 */
+	spin_lock_irqsave(&i915->drm.vblank_time_lock, irqflags);
+	spin_lock(&i915->uncore.lock);
+
+	drm_calc_timestamping_constants(&crtc->base, &adjusted_mode);
+
+	crtc->vmax_vblank_start = vmax_vblank_start;
+
+	crtc->mode_flags = crtc_state->mode_flags;
+
+	crtc->scanline_offset = intel_crtc_scanline_offset(crtc_state);
+
+	spin_unlock(&i915->uncore.lock);
+	spin_unlock_irqrestore(&i915->drm.vblank_time_lock, irqflags);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_vblank.h b/drivers/gpu/drm/i915/display/intel_vblank.h
index c9fea2c2a990..0884db7e76ae 100644
--- a/drivers/gpu/drm/i915/display/intel_vblank.h
+++ b/drivers/gpu/drm/i915/display/intel_vblank.h
@@ -11,6 +11,7 @@
 
 struct drm_crtc;
 struct intel_crtc;
+struct intel_crtc_state;
 
 u32 i915_get_vblank_counter(struct drm_crtc *crtc);
 u32 g4x_get_vblank_counter(struct drm_crtc *crtc);
@@ -19,5 +20,6 @@ bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error,
 int intel_get_crtc_scanline(struct intel_crtc *crtc);
 void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc);
 void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc);
+void intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state);
 
 #endif /* __INTEL_VBLANK_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c
index 207b2a648d32..8e787c13d26d 100644
--- a/drivers/gpu/drm/i915/display/intel_vdsc.c
+++ b/drivers/gpu/drm/i915/display/intel_vdsc.c
@@ -17,6 +17,7 @@
 #include "intel_dsi.h"
 #include "intel_qp_tables.h"
 #include "intel_vdsc.h"
+#include "intel_vdsc_regs.h"
 
 enum ROW_INDEX_BPP {
 	ROW_INDEX_6BPP = 0,
@@ -422,9 +423,9 @@ calculate_rc_params(struct rc_parameters *rc,
 	for (buf_i = 0; buf_i < DSC_NUM_BUF_RANGES; buf_i++) {
 		/* Read range_minqp and range_max_qp from qp tables */
 		rc->rc_range_params[buf_i].range_min_qp =
-			intel_lookup_range_min_qp(bpc, buf_i, bpp_i);
+			intel_lookup_range_min_qp(bpc, buf_i, bpp_i, vdsc_cfg->native_420);
 		rc->rc_range_params[buf_i].range_max_qp =
-			intel_lookup_range_max_qp(bpc, buf_i, bpp_i);
+			intel_lookup_range_max_qp(bpc, buf_i, bpp_i, vdsc_cfg->native_420);
 
 		/* Calculate range_bgp_offset */
 		if (bpp <= 6) {
@@ -447,6 +448,29 @@ calculate_rc_params(struct rc_parameters *rc,
 	}
 }
 
+static int intel_dsc_slice_dimensions_valid(struct intel_crtc_state *pipe_config,
+					    struct drm_dsc_config *vdsc_cfg)
+{
+	if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_RGB ||
+	    pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) {
+		if (vdsc_cfg->slice_height > 4095)
+			return -EINVAL;
+		if (vdsc_cfg->slice_height * vdsc_cfg->slice_width < 15000)
+			return -EINVAL;
+	} else if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) {
+		if (vdsc_cfg->slice_width % 2)
+			return -EINVAL;
+		if (vdsc_cfg->slice_height % 2)
+			return -EINVAL;
+		if (vdsc_cfg->slice_height > 4094)
+			return -EINVAL;
+		if (vdsc_cfg->slice_height * vdsc_cfg->slice_width < 30000)
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
 int intel_dsc_compute_params(struct intel_crtc_state *pipe_config)
 {
 	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
@@ -455,19 +479,64 @@ int intel_dsc_compute_params(struct intel_crtc_state *pipe_config)
 	u16 compressed_bpp = pipe_config->dsc.compressed_bpp;
 	const struct rc_parameters *rc_params;
 	struct rc_parameters *rc = NULL;
+	int err;
 	u8 i = 0;
 
 	vdsc_cfg->pic_width = pipe_config->hw.adjusted_mode.crtc_hdisplay;
 	vdsc_cfg->slice_width = DIV_ROUND_UP(vdsc_cfg->pic_width,
 					     pipe_config->dsc.slice_count);
 
-	/* Gen 11 does not support YCbCr */
+	err = intel_dsc_slice_dimensions_valid(pipe_config, vdsc_cfg);
+
+	if (err) {
+		drm_dbg_kms(&dev_priv->drm, "Slice dimension requirements not met\n");
+		return err;
+	}
+
+	/*
+	 * According to DSC 1.2 specs if colorspace is YCbCr then convert_rgb is 0
+	 * else 1
+	 */
+	vdsc_cfg->convert_rgb = pipe_config->output_format != INTEL_OUTPUT_FORMAT_YCBCR420 &&
+				pipe_config->output_format != INTEL_OUTPUT_FORMAT_YCBCR444;
+
+	if (DISPLAY_VER(dev_priv) >= 14 &&
+	    pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
+		vdsc_cfg->native_420 = true;
+	/* We do not support YcBCr422 as of now */
+	vdsc_cfg->native_422 = false;
 	vdsc_cfg->simple_422 = false;
 	/* Gen 11 does not support VBR */
 	vdsc_cfg->vbr_enable = false;
 
 	/* Gen 11 only supports integral values of bpp */
 	vdsc_cfg->bits_per_pixel = compressed_bpp << 4;
+
+	/*
+	 * According to DSC 1.2 specs in Section 4.1 if native_420 is set:
+	 * -We need to double the current bpp.
+	 * -second_line_bpg_offset is 12 in general and equal to 2*(slice_height-1) if slice
+	 * height < 8.
+	 * -second_line_offset_adj is 512 as shown by emperical values to yeild best chroma
+	 * preservation in second line.
+	 * -nsl_bpg_offset is calculated as second_line_offset/slice_height -1 then rounded
+	 * up to 16 fractional bits, we left shift second line offset by 11 to preserve 11
+	 * fractional bits.
+	 */
+	if (vdsc_cfg->native_420) {
+		vdsc_cfg->bits_per_pixel <<= 1;
+
+		if (vdsc_cfg->slice_height >= 8)
+			vdsc_cfg->second_line_bpg_offset = 12;
+		else
+			vdsc_cfg->second_line_bpg_offset =
+				2 * (vdsc_cfg->slice_height - 1);
+
+		vdsc_cfg->second_line_offset_adj = 512;
+		vdsc_cfg->nsl_bpg_offset = DIV_ROUND_UP(vdsc_cfg->second_line_bpg_offset << 11,
+							vdsc_cfg->slice_height - 1);
+	}
+
 	vdsc_cfg->bits_per_component = pipe_config->pipe_bpp / 3;
 
 	for (i = 0; i < DSC_NUM_BUF_RANGES - 1; i++) {
@@ -594,8 +663,13 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state)
 		DSC_VER_MIN_SHIFT |
 		vdsc_cfg->bits_per_component << DSC_BPC_SHIFT |
 		vdsc_cfg->line_buf_depth << DSC_LINE_BUF_DEPTH_SHIFT;
-	if (vdsc_cfg->dsc_version_minor == 2)
+	if (vdsc_cfg->dsc_version_minor == 2) {
 		pps_val |= DSC_ALT_ICH_SEL;
+		if (vdsc_cfg->native_420)
+			pps_val |= DSC_NATIVE_420_ENABLE;
+		if (vdsc_cfg->native_422)
+			pps_val |= DSC_NATIVE_422_ENABLE;
+	}
 	if (vdsc_cfg->block_pred_enable)
 		pps_val |= DSC_BLOCK_PREDICTION;
 	if (vdsc_cfg->convert_rgb)
@@ -906,6 +980,33 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state)
 				       pps_val);
 	}
 
+	if (DISPLAY_VER(dev_priv) >= 14) {
+		/* Populate PICTURE_PARAMETER_SET_17 registers */
+		pps_val = 0;
+		pps_val |= DSC_SL_BPG_OFFSET(vdsc_cfg->second_line_bpg_offset);
+		drm_dbg_kms(&dev_priv->drm, "PPS17 = 0x%08x\n", pps_val);
+		intel_de_write(dev_priv,
+			       MTL_DSC0_PICTURE_PARAMETER_SET_17(pipe),
+			       pps_val);
+		if (crtc_state->dsc.dsc_split)
+			intel_de_write(dev_priv,
+				       MTL_DSC1_PICTURE_PARAMETER_SET_17(pipe),
+				       pps_val);
+
+		/* Populate PICTURE_PARAMETER_SET_18 registers */
+		pps_val = 0;
+		pps_val |= DSC_NSL_BPG_OFFSET(vdsc_cfg->nsl_bpg_offset) |
+			   DSC_SL_OFFSET_ADJ(vdsc_cfg->second_line_offset_adj);
+		drm_dbg_kms(&dev_priv->drm, "PPS18 = 0x%08x\n", pps_val);
+		intel_de_write(dev_priv,
+			       MTL_DSC0_PICTURE_PARAMETER_SET_18(pipe),
+			       pps_val);
+		if (crtc_state->dsc.dsc_split)
+			intel_de_write(dev_priv,
+				       MTL_DSC1_PICTURE_PARAMETER_SET_18(pipe),
+				       pps_val);
+	}
+
 	/* Populate the RC_BUF_THRESH registers */
 	memset(rc_buf_thresh_dword, 0, sizeof(rc_buf_thresh_dword));
 	for (i = 0; i < DSC_NUM_BUF_RANGES - 1; i++) {
@@ -1180,7 +1281,7 @@ void intel_dsc_get_config(struct intel_crtc_state *crtc_state)
 	enum pipe pipe = crtc->pipe;
 	enum intel_display_power_domain power_domain;
 	intel_wakeref_t wakeref;
-	u32 dss_ctl1, dss_ctl2, val;
+	u32 dss_ctl1, dss_ctl2, pps0 = 0, pps1 = 0;
 
 	if (!intel_dsc_source_support(crtc_state))
 		return;
@@ -1203,13 +1304,21 @@ void intel_dsc_get_config(struct intel_crtc_state *crtc_state)
 
 	/* FIXME: add more state readout as needed */
 
-	/* PPS1 */
-	if (!is_pipe_dsc(crtc, cpu_transcoder))
-		val = intel_de_read(dev_priv, DSCA_PICTURE_PARAMETER_SET_1);
-	else
-		val = intel_de_read(dev_priv,
-				    ICL_DSC0_PICTURE_PARAMETER_SET_1(pipe));
-	vdsc_cfg->bits_per_pixel = val;
+	/* PPS0 & PPS1 */
+	if (!is_pipe_dsc(crtc, cpu_transcoder)) {
+		pps1 = intel_de_read(dev_priv, DSCA_PICTURE_PARAMETER_SET_1);
+	} else {
+		pps0 = intel_de_read(dev_priv,
+				     ICL_DSC0_PICTURE_PARAMETER_SET_0(pipe));
+		pps1 = intel_de_read(dev_priv,
+				     ICL_DSC0_PICTURE_PARAMETER_SET_1(pipe));
+	}
+
+	vdsc_cfg->bits_per_pixel = pps1;
+
+	if (pps0 & DSC_NATIVE_420_ENABLE)
+		vdsc_cfg->bits_per_pixel >>= 1;
+
 	crtc_state->dsc.compressed_bpp = vdsc_cfg->bits_per_pixel >> 4;
 out:
 	intel_display_power_put(dev_priv, power_domain, wakeref);
diff --git a/drivers/gpu/drm/i915/display/intel_vdsc_regs.h b/drivers/gpu/drm/i915/display/intel_vdsc_regs.h
new file mode 100644
index 000000000000..b71f00b5c761
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_vdsc_regs.h
@@ -0,0 +1,489 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_VDSC_REGS_H__
+#define __INTEL_VDSC_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+/* Display Stream Splitter Control */
+#define DSS_CTL1				_MMIO(0x67400)
+#define  SPLITTER_ENABLE			(1 << 31)
+#define  JOINER_ENABLE				(1 << 30)
+#define  DUAL_LINK_MODE_INTERLEAVE		(1 << 24)
+#define  DUAL_LINK_MODE_FRONTBACK		(0 << 24)
+#define  OVERLAP_PIXELS_MASK			(0xf << 16)
+#define  OVERLAP_PIXELS(pixels)			((pixels) << 16)
+#define  LEFT_DL_BUF_TARGET_DEPTH_MASK		(0xfff << 0)
+#define  LEFT_DL_BUF_TARGET_DEPTH(pixels)	((pixels) << 0)
+#define  MAX_DL_BUFFER_TARGET_DEPTH		0x5a0
+
+#define DSS_CTL2				_MMIO(0x67404)
+#define  LEFT_BRANCH_VDSC_ENABLE		(1 << 31)
+#define  RIGHT_BRANCH_VDSC_ENABLE		(1 << 15)
+#define  RIGHT_DL_BUF_TARGET_DEPTH_MASK		(0xfff << 0)
+#define  RIGHT_DL_BUF_TARGET_DEPTH(pixels)	((pixels) << 0)
+
+#define _ICL_PIPE_DSS_CTL1_PB			0x78200
+#define _ICL_PIPE_DSS_CTL1_PC			0x78400
+#define ICL_PIPE_DSS_CTL1(pipe)			_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_PIPE_DSS_CTL1_PB, \
+							   _ICL_PIPE_DSS_CTL1_PC)
+#define  BIG_JOINER_ENABLE			(1 << 29)
+#define  MASTER_BIG_JOINER_ENABLE		(1 << 28)
+#define  VGA_CENTERING_ENABLE			(1 << 27)
+#define  SPLITTER_CONFIGURATION_MASK		REG_GENMASK(26, 25)
+#define  SPLITTER_CONFIGURATION_2_SEGMENT	REG_FIELD_PREP(SPLITTER_CONFIGURATION_MASK, 0)
+#define  SPLITTER_CONFIGURATION_4_SEGMENT	REG_FIELD_PREP(SPLITTER_CONFIGURATION_MASK, 1)
+#define  UNCOMPRESSED_JOINER_MASTER		(1 << 21)
+#define  UNCOMPRESSED_JOINER_SLAVE		(1 << 20)
+
+#define _ICL_PIPE_DSS_CTL2_PB			0x78204
+#define _ICL_PIPE_DSS_CTL2_PC			0x78404
+#define ICL_PIPE_DSS_CTL2(pipe)			_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_PIPE_DSS_CTL2_PB, \
+							   _ICL_PIPE_DSS_CTL2_PC)
+
+/* MTL Display Stream Compression registers */
+#define _MTL_DSC0_PICTURE_PARAMETER_SET_17_PB	0x782B4
+#define _MTL_DSC1_PICTURE_PARAMETER_SET_17_PB	0x783B4
+#define _MTL_DSC0_PICTURE_PARAMETER_SET_17_PC	0x784B4
+#define _MTL_DSC1_PICTURE_PARAMETER_SET_17_PC	0x785B4
+#define MTL_DSC0_PICTURE_PARAMETER_SET_17(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _MTL_DSC0_PICTURE_PARAMETER_SET_17_PB, \
+							   _MTL_DSC0_PICTURE_PARAMETER_SET_17_PC)
+#define MTL_DSC1_PICTURE_PARAMETER_SET_17(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _MTL_DSC1_PICTURE_PARAMETER_SET_17_PB, \
+							   _MTL_DSC1_PICTURE_PARAMETER_SET_17_PC)
+#define DSC_SL_BPG_OFFSET(offset)		((offset) << 27)
+
+#define _MTL_DSC0_PICTURE_PARAMETER_SET_18_PB	0x782B8
+#define _MTL_DSC1_PICTURE_PARAMETER_SET_18_PB	0x783B8
+#define _MTL_DSC0_PICTURE_PARAMETER_SET_18_PC	0x784B8
+#define _MTL_DSC1_PICTURE_PARAMETER_SET_18_PC	0x785B8
+#define MTL_DSC0_PICTURE_PARAMETER_SET_18(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _MTL_DSC0_PICTURE_PARAMETER_SET_18_PB, \
+							   _MTL_DSC0_PICTURE_PARAMETER_SET_18_PC)
+#define MTL_DSC1_PICTURE_PARAMETER_SET_18(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _MTL_DSC1_PICTURE_PARAMETER_SET_18_PB, \
+							   _MTL_DSC1_PICTURE_PARAMETER_SET_18_PC)
+#define DSC_NSL_BPG_OFFSET(offset)		((offset) << 16)
+#define DSC_SL_OFFSET_ADJ(offset)		((offset) << 0)
+
+/* Icelake Display Stream Compression Registers */
+#define DSCA_PICTURE_PARAMETER_SET_0		_MMIO(0x6B200)
+#define DSCC_PICTURE_PARAMETER_SET_0		_MMIO(0x6BA00)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_0_PB	0x78270
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_0_PB	0x78370
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_0_PC	0x78470
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_0_PC	0x78570
+#define ICL_DSC0_PICTURE_PARAMETER_SET_0(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_0_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_0_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_0(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_0_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_0_PC)
+#define  DSC_NATIVE_422_ENABLE		BIT(23)
+#define  DSC_NATIVE_420_ENABLE		BIT(22)
+#define  DSC_ALT_ICH_SEL		(1 << 20)
+#define  DSC_VBR_ENABLE			(1 << 19)
+#define  DSC_422_ENABLE			(1 << 18)
+#define  DSC_COLOR_SPACE_CONVERSION	(1 << 17)
+#define  DSC_BLOCK_PREDICTION		(1 << 16)
+#define  DSC_LINE_BUF_DEPTH_SHIFT	12
+#define  DSC_BPC_SHIFT			8
+#define  DSC_VER_MIN_SHIFT		4
+#define  DSC_VER_MAJ			(0x1 << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_1		_MMIO(0x6B204)
+#define DSCC_PICTURE_PARAMETER_SET_1		_MMIO(0x6BA04)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_1_PB	0x78274
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_1_PB	0x78374
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_1_PC	0x78474
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_1_PC	0x78574
+#define ICL_DSC0_PICTURE_PARAMETER_SET_1(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_1_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_1_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_1(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_1_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_1_PC)
+#define  DSC_BPP(bpp)				((bpp) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_2		_MMIO(0x6B208)
+#define DSCC_PICTURE_PARAMETER_SET_2		_MMIO(0x6BA08)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_2_PB	0x78278
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_2_PB	0x78378
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_2_PC	0x78478
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_2_PC	0x78578
+#define ICL_DSC0_PICTURE_PARAMETER_SET_2(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_2_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_2_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_2(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+					    _ICL_DSC1_PICTURE_PARAMETER_SET_2_PB, \
+					    _ICL_DSC1_PICTURE_PARAMETER_SET_2_PC)
+#define  DSC_PIC_WIDTH(pic_width)	((pic_width) << 16)
+#define  DSC_PIC_HEIGHT(pic_height)	((pic_height) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_3		_MMIO(0x6B20C)
+#define DSCC_PICTURE_PARAMETER_SET_3		_MMIO(0x6BA0C)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_3_PB	0x7827C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_3_PB	0x7837C
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_3_PC	0x7847C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_3_PC	0x7857C
+#define ICL_DSC0_PICTURE_PARAMETER_SET_3(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_3_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_3_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_3(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_3_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_3_PC)
+#define  DSC_SLICE_WIDTH(slice_width)   ((slice_width) << 16)
+#define  DSC_SLICE_HEIGHT(slice_height) ((slice_height) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_4		_MMIO(0x6B210)
+#define DSCC_PICTURE_PARAMETER_SET_4		_MMIO(0x6BA10)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_4_PB	0x78280
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_4_PB	0x78380
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_4_PC	0x78480
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_4_PC	0x78580
+#define ICL_DSC0_PICTURE_PARAMETER_SET_4(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_4_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_4_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_4(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_4_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_4_PC)
+#define  DSC_INITIAL_DEC_DELAY(dec_delay)       ((dec_delay) << 16)
+#define  DSC_INITIAL_XMIT_DELAY(xmit_delay)     ((xmit_delay) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_5		_MMIO(0x6B214)
+#define DSCC_PICTURE_PARAMETER_SET_5		_MMIO(0x6BA14)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_5_PB	0x78284
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_5_PB	0x78384
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_5_PC	0x78484
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_5_PC	0x78584
+#define ICL_DSC0_PICTURE_PARAMETER_SET_5(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_5_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_5_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_5(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_5_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_5_PC)
+#define  DSC_SCALE_DEC_INT(scale_dec)	((scale_dec) << 16)
+#define  DSC_SCALE_INC_INT(scale_inc)		((scale_inc) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_6		_MMIO(0x6B218)
+#define DSCC_PICTURE_PARAMETER_SET_6		_MMIO(0x6BA18)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_6_PB	0x78288
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_6_PB	0x78388
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_6_PC	0x78488
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_6_PC	0x78588
+#define ICL_DSC0_PICTURE_PARAMETER_SET_6(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_6_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_6_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_6(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_6_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_6_PC)
+#define  DSC_FLATNESS_MAX_QP(max_qp)		((max_qp) << 24)
+#define  DSC_FLATNESS_MIN_QP(min_qp)		((min_qp) << 16)
+#define  DSC_FIRST_LINE_BPG_OFFSET(offset)	((offset) << 8)
+#define  DSC_INITIAL_SCALE_VALUE(value)		((value) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_7		_MMIO(0x6B21C)
+#define DSCC_PICTURE_PARAMETER_SET_7		_MMIO(0x6BA1C)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_7_PB	0x7828C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_7_PB	0x7838C
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_7_PC	0x7848C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_7_PC	0x7858C
+#define ICL_DSC0_PICTURE_PARAMETER_SET_7(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							    _ICL_DSC0_PICTURE_PARAMETER_SET_7_PB, \
+							    _ICL_DSC0_PICTURE_PARAMETER_SET_7_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_7(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							    _ICL_DSC1_PICTURE_PARAMETER_SET_7_PB, \
+							    _ICL_DSC1_PICTURE_PARAMETER_SET_7_PC)
+#define  DSC_NFL_BPG_OFFSET(bpg_offset)		((bpg_offset) << 16)
+#define  DSC_SLICE_BPG_OFFSET(bpg_offset)	((bpg_offset) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_8		_MMIO(0x6B220)
+#define DSCC_PICTURE_PARAMETER_SET_8		_MMIO(0x6BA20)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_8_PB	0x78290
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_8_PB	0x78390
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_8_PC	0x78490
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_8_PC	0x78590
+#define ICL_DSC0_PICTURE_PARAMETER_SET_8(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_8_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_8_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_8(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_8_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_8_PC)
+#define  DSC_INITIAL_OFFSET(initial_offset)		((initial_offset) << 16)
+#define  DSC_FINAL_OFFSET(final_offset)			((final_offset) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_9		_MMIO(0x6B224)
+#define DSCC_PICTURE_PARAMETER_SET_9		_MMIO(0x6BA24)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_9_PB	0x78294
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_9_PB	0x78394
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_9_PC	0x78494
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_9_PC	0x78594
+#define ICL_DSC0_PICTURE_PARAMETER_SET_9(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_9_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_9_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_9(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_9_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_9_PC)
+#define  DSC_RC_EDGE_FACTOR(rc_edge_fact)	((rc_edge_fact) << 16)
+#define  DSC_RC_MODEL_SIZE(rc_model_size)	((rc_model_size) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_10		_MMIO(0x6B228)
+#define DSCC_PICTURE_PARAMETER_SET_10		_MMIO(0x6BA28)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_10_PB	0x78298
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_10_PB	0x78398
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_10_PC	0x78498
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_10_PC	0x78598
+#define ICL_DSC0_PICTURE_PARAMETER_SET_10(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_10_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_10_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_10(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_10_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_10_PC)
+#define  DSC_RC_TARGET_OFF_LOW(rc_tgt_off_low)		((rc_tgt_off_low) << 20)
+#define  DSC_RC_TARGET_OFF_HIGH(rc_tgt_off_high)	((rc_tgt_off_high) << 16)
+#define  DSC_RC_QUANT_INC_LIMIT1(lim)			((lim) << 8)
+#define  DSC_RC_QUANT_INC_LIMIT0(lim)			((lim) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_11		_MMIO(0x6B22C)
+#define DSCC_PICTURE_PARAMETER_SET_11		_MMIO(0x6BA2C)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_11_PB	0x7829C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_11_PB	0x7839C
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_11_PC	0x7849C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_11_PC	0x7859C
+#define ICL_DSC0_PICTURE_PARAMETER_SET_11(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_11_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_11_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_11(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_11_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_11_PC)
+
+#define DSCA_PICTURE_PARAMETER_SET_12		_MMIO(0x6B260)
+#define DSCC_PICTURE_PARAMETER_SET_12		_MMIO(0x6BA60)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_12_PB	0x782A0
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_12_PB	0x783A0
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_12_PC	0x784A0
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_12_PC	0x785A0
+#define ICL_DSC0_PICTURE_PARAMETER_SET_12(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_12_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_12_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_12(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_12_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_12_PC)
+
+#define DSCA_PICTURE_PARAMETER_SET_13		_MMIO(0x6B264)
+#define DSCC_PICTURE_PARAMETER_SET_13		_MMIO(0x6BA64)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_13_PB	0x782A4
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_13_PB	0x783A4
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_13_PC	0x784A4
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_13_PC	0x785A4
+#define ICL_DSC0_PICTURE_PARAMETER_SET_13(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_13_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_13_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_13(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_13_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_13_PC)
+
+#define DSCA_PICTURE_PARAMETER_SET_14		_MMIO(0x6B268)
+#define DSCC_PICTURE_PARAMETER_SET_14		_MMIO(0x6BA68)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_14_PB	0x782A8
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_14_PB	0x783A8
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_14_PC	0x784A8
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_14_PC	0x785A8
+#define ICL_DSC0_PICTURE_PARAMETER_SET_14(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_14_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_14_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_14(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_14_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_14_PC)
+
+#define DSCA_PICTURE_PARAMETER_SET_15		_MMIO(0x6B26C)
+#define DSCC_PICTURE_PARAMETER_SET_15		_MMIO(0x6BA6C)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_15_PB	0x782AC
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_15_PB	0x783AC
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_15_PC	0x784AC
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_15_PC	0x785AC
+#define ICL_DSC0_PICTURE_PARAMETER_SET_15(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_15_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_15_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_15(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_15_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_15_PC)
+
+#define DSCA_PICTURE_PARAMETER_SET_16		_MMIO(0x6B270)
+#define DSCC_PICTURE_PARAMETER_SET_16		_MMIO(0x6BA70)
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_16_PB	0x782B0
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_16_PB	0x783B0
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_16_PC	0x784B0
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_16_PC	0x785B0
+#define ICL_DSC0_PICTURE_PARAMETER_SET_16(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_16_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_16_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_16(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_16_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_16_PC)
+#define  DSC_SLICE_ROW_PER_FRAME(slice_row_per_frame)	((slice_row_per_frame) << 20)
+#define  DSC_SLICE_PER_LINE(slice_per_line)		((slice_per_line) << 16)
+#define  DSC_SLICE_CHUNK_SIZE(slice_chunk_size)		((slice_chunk_size) << 0)
+
+/* Icelake Rate Control Buffer Threshold Registers */
+#define DSCA_RC_BUF_THRESH_0			_MMIO(0x6B230)
+#define DSCA_RC_BUF_THRESH_0_UDW		_MMIO(0x6B230 + 4)
+#define DSCC_RC_BUF_THRESH_0			_MMIO(0x6BA30)
+#define DSCC_RC_BUF_THRESH_0_UDW		_MMIO(0x6BA30 + 4)
+#define _ICL_DSC0_RC_BUF_THRESH_0_PB		(0x78254)
+#define _ICL_DSC0_RC_BUF_THRESH_0_UDW_PB	(0x78254 + 4)
+#define _ICL_DSC1_RC_BUF_THRESH_0_PB		(0x78354)
+#define _ICL_DSC1_RC_BUF_THRESH_0_UDW_PB	(0x78354 + 4)
+#define _ICL_DSC0_RC_BUF_THRESH_0_PC		(0x78454)
+#define _ICL_DSC0_RC_BUF_THRESH_0_UDW_PC	(0x78454 + 4)
+#define _ICL_DSC1_RC_BUF_THRESH_0_PC		(0x78554)
+#define _ICL_DSC1_RC_BUF_THRESH_0_UDW_PC	(0x78554 + 4)
+#define ICL_DSC0_RC_BUF_THRESH_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC0_RC_BUF_THRESH_0_PB, \
+						_ICL_DSC0_RC_BUF_THRESH_0_PC)
+#define ICL_DSC0_RC_BUF_THRESH_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC0_RC_BUF_THRESH_0_UDW_PB, \
+						_ICL_DSC0_RC_BUF_THRESH_0_UDW_PC)
+#define ICL_DSC1_RC_BUF_THRESH_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC1_RC_BUF_THRESH_0_PB, \
+						_ICL_DSC1_RC_BUF_THRESH_0_PC)
+#define ICL_DSC1_RC_BUF_THRESH_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC1_RC_BUF_THRESH_0_UDW_PB, \
+						_ICL_DSC1_RC_BUF_THRESH_0_UDW_PC)
+
+#define DSCA_RC_BUF_THRESH_1			_MMIO(0x6B238)
+#define DSCA_RC_BUF_THRESH_1_UDW		_MMIO(0x6B238 + 4)
+#define DSCC_RC_BUF_THRESH_1			_MMIO(0x6BA38)
+#define DSCC_RC_BUF_THRESH_1_UDW		_MMIO(0x6BA38 + 4)
+#define _ICL_DSC0_RC_BUF_THRESH_1_PB		(0x7825C)
+#define _ICL_DSC0_RC_BUF_THRESH_1_UDW_PB	(0x7825C + 4)
+#define _ICL_DSC1_RC_BUF_THRESH_1_PB		(0x7835C)
+#define _ICL_DSC1_RC_BUF_THRESH_1_UDW_PB	(0x7835C + 4)
+#define _ICL_DSC0_RC_BUF_THRESH_1_PC		(0x7845C)
+#define _ICL_DSC0_RC_BUF_THRESH_1_UDW_PC	(0x7845C + 4)
+#define _ICL_DSC1_RC_BUF_THRESH_1_PC		(0x7855C)
+#define _ICL_DSC1_RC_BUF_THRESH_1_UDW_PC	(0x7855C + 4)
+#define ICL_DSC0_RC_BUF_THRESH_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC0_RC_BUF_THRESH_1_PB, \
+						_ICL_DSC0_RC_BUF_THRESH_1_PC)
+#define ICL_DSC0_RC_BUF_THRESH_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC0_RC_BUF_THRESH_1_UDW_PB, \
+						_ICL_DSC0_RC_BUF_THRESH_1_UDW_PC)
+#define ICL_DSC1_RC_BUF_THRESH_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC1_RC_BUF_THRESH_1_PB, \
+						_ICL_DSC1_RC_BUF_THRESH_1_PC)
+#define ICL_DSC1_RC_BUF_THRESH_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC1_RC_BUF_THRESH_1_UDW_PB, \
+						_ICL_DSC1_RC_BUF_THRESH_1_UDW_PC)
+
+/* Icelake DSC Rate Control Range Parameter Registers */
+#define DSCA_RC_RANGE_PARAMETERS_0		_MMIO(0x6B240)
+#define DSCA_RC_RANGE_PARAMETERS_0_UDW		_MMIO(0x6B240 + 4)
+#define DSCC_RC_RANGE_PARAMETERS_0		_MMIO(0x6BA40)
+#define DSCC_RC_RANGE_PARAMETERS_0_UDW		_MMIO(0x6BA40 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_PB	(0x78208)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PB	(0x78208 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_PB	(0x78308)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PB	(0x78308 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_PC	(0x78408)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PC	(0x78408 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_PC	(0x78508)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PC	(0x78508 + 4)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_0_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_0_PC)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_0_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_0_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PC)
+#define RC_BPG_OFFSET_SHIFT			10
+#define RC_MAX_QP_SHIFT				5
+#define RC_MIN_QP_SHIFT				0
+
+#define DSCA_RC_RANGE_PARAMETERS_1		_MMIO(0x6B248)
+#define DSCA_RC_RANGE_PARAMETERS_1_UDW		_MMIO(0x6B248 + 4)
+#define DSCC_RC_RANGE_PARAMETERS_1		_MMIO(0x6BA48)
+#define DSCC_RC_RANGE_PARAMETERS_1_UDW		_MMIO(0x6BA48 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_PB	(0x78210)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PB	(0x78210 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_PB	(0x78310)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PB	(0x78310 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_PC	(0x78410)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PC	(0x78410 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_PC	(0x78510)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PC	(0x78510 + 4)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_1_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_1_PC)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_1_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_1_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PC)
+
+#define DSCA_RC_RANGE_PARAMETERS_2		_MMIO(0x6B250)
+#define DSCA_RC_RANGE_PARAMETERS_2_UDW		_MMIO(0x6B250 + 4)
+#define DSCC_RC_RANGE_PARAMETERS_2		_MMIO(0x6BA50)
+#define DSCC_RC_RANGE_PARAMETERS_2_UDW		_MMIO(0x6BA50 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_PB	(0x78218)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PB	(0x78218 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_PB	(0x78318)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PB	(0x78318 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_PC	(0x78418)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PC	(0x78418 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_PC	(0x78518)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PC	(0x78518 + 4)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_2(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_2_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_2_PC)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_2(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_2_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_2_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PC)
+
+#define DSCA_RC_RANGE_PARAMETERS_3		_MMIO(0x6B258)
+#define DSCA_RC_RANGE_PARAMETERS_3_UDW		_MMIO(0x6B258 + 4)
+#define DSCC_RC_RANGE_PARAMETERS_3		_MMIO(0x6BA58)
+#define DSCC_RC_RANGE_PARAMETERS_3_UDW		_MMIO(0x6BA58 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_PB	(0x78220)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PB	(0x78220 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_PB	(0x78320)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PB	(0x78320 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_PC	(0x78420)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PC	(0x78420 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_PC	(0x78520)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PC	(0x78520 + 4)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_3(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_3_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_3_PC)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_3(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_3_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_3_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PC)
+
+#endif /* __INTEL_VDSC_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c
index 5ff6aed9575e..4228f26b4c11 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr.c
+++ b/drivers/gpu/drm/i915/display/intel_vrr.c
@@ -144,17 +144,11 @@ intel_vrr_compute_config(struct intel_crtc_state *crtc_state,
 	 * is deprecated.
 	 */
 	if (DISPLAY_VER(i915) >= 13) {
-		/*
-		 * FIXME: Subtract Window2 delay from below value.
-		 *
-		 * Window2 specifies time required to program DSB (Window2) in
-		 * number of scan lines. Assuming 0 for no DSB.
-		 */
 		crtc_state->vrr.guardband =
-			crtc_state->vrr.vmin + 1 - adjusted_mode->crtc_vdisplay;
+			crtc_state->vrr.vmin + 1 - adjusted_mode->crtc_vblank_start;
 	} else {
 		crtc_state->vrr.pipeline_full =
-			min(255, crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay -
+			min(255, crtc_state->vrr.vmin - adjusted_mode->crtc_vblank_start -
 			    crtc_state->framestart_delay - 1);
 	}
 
diff --git a/drivers/gpu/drm/i915/display/intel_wm.c b/drivers/gpu/drm/i915/display/intel_wm.c
new file mode 100644
index 000000000000..b615449e70b4
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_wm.c
@@ -0,0 +1,408 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#include "i915_drv.h"
+#include "i9xx_wm.h"
+#include "intel_display_types.h"
+#include "intel_wm.h"
+#include "skl_watermark.h"
+
+/**
+ * intel_update_watermarks - update FIFO watermark values based on current modes
+ * @i915: i915 device
+ *
+ * Calculate watermark values for the various WM regs based on current mode
+ * and plane configuration.
+ *
+ * There are several cases to deal with here:
+ *   - normal (i.e. non-self-refresh)
+ *   - self-refresh (SR) mode
+ *   - lines are large relative to FIFO size (buffer can hold up to 2)
+ *   - lines are small relative to FIFO size (buffer can hold more than 2
+ *     lines), so need to account for TLB latency
+ *
+ *   The normal calculation is:
+ *     watermark = dotclock * bytes per pixel * latency
+ *   where latency is platform & configuration dependent (we assume pessimal
+ *   values here).
+ *
+ *   The SR calculation is:
+ *     watermark = (trunc(latency/line time)+1) * surface width *
+ *       bytes per pixel
+ *   where
+ *     line time = htotal / dotclock
+ *     surface width = hdisplay for normal plane and 64 for cursor
+ *   and latency is assumed to be high, as above.
+ *
+ * The final value programmed to the register should always be rounded up,
+ * and include an extra 2 entries to account for clock crossings.
+ *
+ * We don't use the sprite, so we can ignore that.  And on Crestline we have
+ * to set the non-SR watermarks to 8.
+ */
+void intel_update_watermarks(struct drm_i915_private *i915)
+{
+	if (i915->display.funcs.wm->update_wm)
+		i915->display.funcs.wm->update_wm(i915);
+}
+
+int intel_compute_pipe_wm(struct intel_atomic_state *state,
+			  struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+
+	if (i915->display.funcs.wm->compute_pipe_wm)
+		return i915->display.funcs.wm->compute_pipe_wm(state, crtc);
+
+	return 0;
+}
+
+int intel_compute_intermediate_wm(struct intel_atomic_state *state,
+				  struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+
+	if (!i915->display.funcs.wm->compute_intermediate_wm)
+		return 0;
+
+	if (drm_WARN_ON(&i915->drm, !i915->display.funcs.wm->compute_pipe_wm))
+		return 0;
+
+	return i915->display.funcs.wm->compute_intermediate_wm(state, crtc);
+}
+
+bool intel_initial_watermarks(struct intel_atomic_state *state,
+			      struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+
+	if (i915->display.funcs.wm->initial_watermarks) {
+		i915->display.funcs.wm->initial_watermarks(state, crtc);
+		return true;
+	}
+
+	return false;
+}
+
+void intel_atomic_update_watermarks(struct intel_atomic_state *state,
+				    struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+
+	if (i915->display.funcs.wm->atomic_update_watermarks)
+		i915->display.funcs.wm->atomic_update_watermarks(state, crtc);
+}
+
+void intel_optimize_watermarks(struct intel_atomic_state *state,
+			       struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+
+	if (i915->display.funcs.wm->optimize_watermarks)
+		i915->display.funcs.wm->optimize_watermarks(state, crtc);
+}
+
+int intel_compute_global_watermarks(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+
+	if (i915->display.funcs.wm->compute_global_watermarks)
+		return i915->display.funcs.wm->compute_global_watermarks(state);
+
+	return 0;
+}
+
+void intel_wm_get_hw_state(struct drm_i915_private *i915)
+{
+	if (i915->display.funcs.wm->get_hw_state)
+		return i915->display.funcs.wm->get_hw_state(i915);
+}
+
+bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
+			    const struct intel_plane_state *plane_state)
+{
+	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
+
+	/* FIXME check the 'enable' instead */
+	if (!crtc_state->hw.active)
+		return false;
+
+	/*
+	 * Treat cursor with fb as always visible since cursor updates
+	 * can happen faster than the vrefresh rate, and the current
+	 * watermark code doesn't handle that correctly. Cursor updates
+	 * which set/clear the fb or change the cursor size are going
+	 * to get throttled by intel_legacy_cursor_update() to work
+	 * around this problem with the watermark code.
+	 */
+	if (plane->id == PLANE_CURSOR)
+		return plane_state->hw.fb != NULL;
+	else
+		return plane_state->uapi.visible;
+}
+
+void intel_print_wm_latency(struct drm_i915_private *dev_priv,
+			    const char *name, const u16 wm[])
+{
+	int level;
+
+	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
+		unsigned int latency = wm[level];
+
+		if (latency == 0) {
+			drm_dbg_kms(&dev_priv->drm,
+				    "%s WM%d latency not provided\n",
+				    name, level);
+			continue;
+		}
+
+		/*
+		 * - latencies are in us on gen9.
+		 * - before then, WM1+ latency values are in 0.5us units
+		 */
+		if (DISPLAY_VER(dev_priv) >= 9)
+			latency *= 10;
+		else if (level > 0)
+			latency *= 5;
+
+		drm_dbg_kms(&dev_priv->drm,
+			    "%s WM%d latency %u (%u.%u usec)\n", name, level,
+			    wm[level], latency / 10, latency % 10);
+	}
+}
+
+void intel_wm_init(struct drm_i915_private *i915)
+{
+	if (DISPLAY_VER(i915) >= 9)
+		skl_wm_init(i915);
+	else
+		i9xx_wm_init(i915);
+}
+
+static void wm_latency_show(struct seq_file *m, const u16 wm[8])
+{
+	struct drm_i915_private *dev_priv = m->private;
+	int level;
+
+	drm_modeset_lock_all(&dev_priv->drm);
+
+	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
+		unsigned int latency = wm[level];
+
+		/*
+		 * - WM1+ latency values in 0.5us units
+		 * - latencies are in us on gen9/vlv/chv
+		 */
+		if (DISPLAY_VER(dev_priv) >= 9 ||
+		    IS_VALLEYVIEW(dev_priv) ||
+		    IS_CHERRYVIEW(dev_priv) ||
+		    IS_G4X(dev_priv))
+			latency *= 10;
+		else if (level > 0)
+			latency *= 5;
+
+		seq_printf(m, "WM%d %u (%u.%u usec)\n",
+			   level, wm[level], latency / 10, latency % 10);
+	}
+
+	drm_modeset_unlock_all(&dev_priv->drm);
+}
+
+static int pri_wm_latency_show(struct seq_file *m, void *data)
+{
+	struct drm_i915_private *dev_priv = m->private;
+	const u16 *latencies;
+
+	if (DISPLAY_VER(dev_priv) >= 9)
+		latencies = dev_priv->display.wm.skl_latency;
+	else
+		latencies = dev_priv->display.wm.pri_latency;
+
+	wm_latency_show(m, latencies);
+
+	return 0;
+}
+
+static int spr_wm_latency_show(struct seq_file *m, void *data)
+{
+	struct drm_i915_private *dev_priv = m->private;
+	const u16 *latencies;
+
+	if (DISPLAY_VER(dev_priv) >= 9)
+		latencies = dev_priv->display.wm.skl_latency;
+	else
+		latencies = dev_priv->display.wm.spr_latency;
+
+	wm_latency_show(m, latencies);
+
+	return 0;
+}
+
+static int cur_wm_latency_show(struct seq_file *m, void *data)
+{
+	struct drm_i915_private *dev_priv = m->private;
+	const u16 *latencies;
+
+	if (DISPLAY_VER(dev_priv) >= 9)
+		latencies = dev_priv->display.wm.skl_latency;
+	else
+		latencies = dev_priv->display.wm.cur_latency;
+
+	wm_latency_show(m, latencies);
+
+	return 0;
+}
+
+static int pri_wm_latency_open(struct inode *inode, struct file *file)
+{
+	struct drm_i915_private *dev_priv = inode->i_private;
+
+	if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv))
+		return -ENODEV;
+
+	return single_open(file, pri_wm_latency_show, dev_priv);
+}
+
+static int spr_wm_latency_open(struct inode *inode, struct file *file)
+{
+	struct drm_i915_private *dev_priv = inode->i_private;
+
+	if (HAS_GMCH(dev_priv))
+		return -ENODEV;
+
+	return single_open(file, spr_wm_latency_show, dev_priv);
+}
+
+static int cur_wm_latency_open(struct inode *inode, struct file *file)
+{
+	struct drm_i915_private *dev_priv = inode->i_private;
+
+	if (HAS_GMCH(dev_priv))
+		return -ENODEV;
+
+	return single_open(file, cur_wm_latency_show, dev_priv);
+}
+
+static ssize_t wm_latency_write(struct file *file, const char __user *ubuf,
+				size_t len, loff_t *offp, u16 wm[8])
+{
+	struct seq_file *m = file->private_data;
+	struct drm_i915_private *dev_priv = m->private;
+	u16 new[8] = { 0 };
+	int level;
+	int ret;
+	char tmp[32];
+
+	if (len >= sizeof(tmp))
+		return -EINVAL;
+
+	if (copy_from_user(tmp, ubuf, len))
+		return -EFAULT;
+
+	tmp[len] = '\0';
+
+	ret = sscanf(tmp, "%hu %hu %hu %hu %hu %hu %hu %hu",
+		     &new[0], &new[1], &new[2], &new[3],
+		     &new[4], &new[5], &new[6], &new[7]);
+	if (ret != dev_priv->display.wm.num_levels)
+		return -EINVAL;
+
+	drm_modeset_lock_all(&dev_priv->drm);
+
+	for (level = 0; level < dev_priv->display.wm.num_levels; level++)
+		wm[level] = new[level];
+
+	drm_modeset_unlock_all(&dev_priv->drm);
+
+	return len;
+}
+
+static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf,
+				    size_t len, loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	struct drm_i915_private *dev_priv = m->private;
+	u16 *latencies;
+
+	if (DISPLAY_VER(dev_priv) >= 9)
+		latencies = dev_priv->display.wm.skl_latency;
+	else
+		latencies = dev_priv->display.wm.pri_latency;
+
+	return wm_latency_write(file, ubuf, len, offp, latencies);
+}
+
+static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf,
+				    size_t len, loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	struct drm_i915_private *dev_priv = m->private;
+	u16 *latencies;
+
+	if (DISPLAY_VER(dev_priv) >= 9)
+		latencies = dev_priv->display.wm.skl_latency;
+	else
+		latencies = dev_priv->display.wm.spr_latency;
+
+	return wm_latency_write(file, ubuf, len, offp, latencies);
+}
+
+static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf,
+				    size_t len, loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	struct drm_i915_private *dev_priv = m->private;
+	u16 *latencies;
+
+	if (DISPLAY_VER(dev_priv) >= 9)
+		latencies = dev_priv->display.wm.skl_latency;
+	else
+		latencies = dev_priv->display.wm.cur_latency;
+
+	return wm_latency_write(file, ubuf, len, offp, latencies);
+}
+
+static const struct file_operations i915_pri_wm_latency_fops = {
+	.owner = THIS_MODULE,
+	.open = pri_wm_latency_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = pri_wm_latency_write
+};
+
+static const struct file_operations i915_spr_wm_latency_fops = {
+	.owner = THIS_MODULE,
+	.open = spr_wm_latency_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = spr_wm_latency_write
+};
+
+static const struct file_operations i915_cur_wm_latency_fops = {
+	.owner = THIS_MODULE,
+	.open = cur_wm_latency_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = cur_wm_latency_write
+};
+
+void intel_wm_debugfs_register(struct drm_i915_private *i915)
+{
+	struct drm_minor *minor = i915->drm.primary;
+
+	debugfs_create_file("i915_pri_wm_latency", 0644, minor->debugfs_root,
+			    i915, &i915_pri_wm_latency_fops);
+
+	debugfs_create_file("i915_spr_wm_latency", 0644, minor->debugfs_root,
+			    i915, &i915_spr_wm_latency_fops);
+
+	debugfs_create_file("i915_cur_wm_latency", 0644, minor->debugfs_root,
+			    i915, &i915_cur_wm_latency_fops);
+
+	skl_watermark_debugfs_register(i915);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_wm.h b/drivers/gpu/drm/i915/display/intel_wm.h
new file mode 100644
index 000000000000..48429ac140d2
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_wm.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_WM_H__
+#define __INTEL_WM_H__
+
+#include <linux/types.h>
+
+struct drm_i915_private;
+struct intel_atomic_state;
+struct intel_crtc;
+struct intel_crtc_state;
+struct intel_plane_state;
+
+void intel_update_watermarks(struct drm_i915_private *i915);
+int intel_compute_pipe_wm(struct intel_atomic_state *state,
+			  struct intel_crtc *crtc);
+int intel_compute_intermediate_wm(struct intel_atomic_state *state,
+				  struct intel_crtc *crtc);
+bool intel_initial_watermarks(struct intel_atomic_state *state,
+			      struct intel_crtc *crtc);
+void intel_atomic_update_watermarks(struct intel_atomic_state *state,
+				    struct intel_crtc *crtc);
+void intel_optimize_watermarks(struct intel_atomic_state *state,
+			       struct intel_crtc *crtc);
+int intel_compute_global_watermarks(struct intel_atomic_state *state);
+void intel_wm_get_hw_state(struct drm_i915_private *i915);
+bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
+			    const struct intel_plane_state *plane_state);
+void intel_print_wm_latency(struct drm_i915_private *i915,
+			    const char *name, const u16 wm[]);
+void intel_wm_init(struct drm_i915_private *i915);
+void intel_wm_debugfs_register(struct drm_i915_private *i915);
+
+#endif /* __INTEL_WM_H__ */
diff --git a/drivers/gpu/drm/i915/intel_pm_types.h b/drivers/gpu/drm/i915/display/intel_wm_types.h
index 93152537b420..628b7c0ce484 100644
--- a/drivers/gpu/drm/i915/intel_pm_types.h
+++ b/drivers/gpu/drm/i915/display/intel_wm_types.h
@@ -3,12 +3,12 @@
  * Copyright © 2021 Intel Corporation
  */
 
-#ifndef __INTEL_PM_TYPES_H__
-#define __INTEL_PM_TYPES_H__
+#ifndef __INTEL_WM_TYPES_H__
+#define __INTEL_WM_TYPES_H__
 
 #include <linux/types.h>
 
-#include "display/intel_display_limits.h"
+#include "intel_display_limits.h"
 
 enum intel_ddb_partitioning {
 	INTEL_DDB_PART_1_2,
@@ -73,4 +73,4 @@ static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1,
 	return false;
 }
 
-#endif /* __INTEL_PM_TYPES_H__ */
+#endif /* __INTEL_WM_TYPES_H__ */
diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c
index 9b172a1e90de..8ea0598a5a07 100644
--- a/drivers/gpu/drm/i915/display/skl_universal_plane.c
+++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c
@@ -17,7 +17,6 @@
 #include "intel_fb.h"
 #include "intel_fbc.h"
 #include "intel_psr.h"
-#include "intel_sprite.h"
 #include "skl_scaler.h"
 #include "skl_universal_plane.h"
 #include "skl_watermark.h"
@@ -642,7 +641,7 @@ icl_plane_disable_arm(struct intel_plane *plane,
 
 	skl_write_plane_wm(plane, crtc_state);
 
-	intel_psr2_disable_plane_sel_fetch(plane, crtc_state);
+	intel_psr2_disable_plane_sel_fetch_arm(plane, crtc_state);
 	intel_de_write_fw(dev_priv, PLANE_CTL(pipe, plane_id), 0);
 	intel_de_write_fw(dev_priv, PLANE_SURF(pipe, plane_id), 0);
 }
@@ -1260,7 +1259,7 @@ icl_plane_update_noarm(struct intel_plane *plane,
 	if (plane_state->force_black)
 		icl_plane_csc_load_black(plane);
 
-	intel_psr2_program_plane_sel_fetch(plane, crtc_state, plane_state, color_plane);
+	intel_psr2_program_plane_sel_fetch_noarm(plane, crtc_state, plane_state, color_plane);
 }
 
 static void
@@ -1287,6 +1286,8 @@ icl_plane_update_arm(struct intel_plane *plane,
 	if (plane_state->scaler_id >= 0)
 		skl_program_plane_scaler(plane, crtc_state, plane_state);
 
+	intel_psr2_program_plane_sel_fetch_arm(plane, crtc_state, plane_state);
+
 	/*
 	 * The control register self-arms if the plane was previously
 	 * disabled. Try to make the plane enable atomic by writing
@@ -2180,7 +2181,7 @@ static bool gen12_plane_has_mc_ccs(struct drm_i915_private *i915,
 	if (DISPLAY_VER(i915) < 12)
 		return false;
 
-	/* Wa_14010477008:tgl[a0..c0],rkl[all],dg1[all] */
+	/* Wa_14010477008 */
 	if (IS_DG1(i915) || IS_ROCKETLAKE(i915) ||
 	    IS_TGL_DISPLAY_STEP(i915, STEP_A0, STEP_D0))
 		return false;
@@ -2474,6 +2475,12 @@ skl_get_initial_plane_config(struct intel_crtc *crtc,
 		goto error;
 	}
 
+	if (!dev_priv->params.enable_dpt &&
+	    intel_fb_modifier_uses_dpt(dev_priv, fb->modifier)) {
+		drm_dbg_kms(&dev_priv->drm, "DPT disabled, skipping initial FB\n");
+		goto error;
+	}
+
 	/*
 	 * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
 	 * while i915 HW rotation is clockwise, thats why this swapping.
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c
index d1670cc3eff2..1c7e6468f3e3 100644
--- a/drivers/gpu/drm/i915/display/skl_watermark.c
+++ b/drivers/gpu/drm/i915/display/skl_watermark.c
@@ -5,21 +5,23 @@
 
 #include <drm/drm_blend.h>
 
+#include "i915_drv.h"
+#include "i915_fixed.h"
+#include "i915_reg.h"
+#include "i9xx_wm.h"
 #include "intel_atomic.h"
 #include "intel_atomic_plane.h"
 #include "intel_bw.h"
+#include "intel_crtc.h"
 #include "intel_de.h"
 #include "intel_display.h"
 #include "intel_display_power.h"
 #include "intel_display_types.h"
 #include "intel_fb.h"
-#include "skl_watermark.h"
-
-#include "i915_drv.h"
-#include "i915_fixed.h"
-#include "i915_reg.h"
 #include "intel_pcode.h"
-#include "intel_pm.h"
+#include "intel_wm.h"
+#include "skl_watermark.h"
+#include "skl_watermark_regs.h"
 
 static void skl_sagv_disable(struct drm_i915_private *i915);
 
@@ -64,7 +66,7 @@ static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
 static bool
 intel_has_sagv(struct drm_i915_private *i915)
 {
-	return DISPLAY_VER(i915) >= 9 && !IS_LP(i915) &&
+	return HAS_SAGV(i915) &&
 		i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
 }
 
@@ -92,7 +94,7 @@ intel_sagv_block_time(struct drm_i915_private *i915)
 		return val;
 	} else if (DISPLAY_VER(i915) == 11) {
 		return 10;
-	} else if (DISPLAY_VER(i915) == 9 && !IS_LP(i915)) {
+	} else if (HAS_SAGV(i915)) {
 		return 30;
 	} else {
 		return 0;
@@ -101,7 +103,7 @@ intel_sagv_block_time(struct drm_i915_private *i915)
 
 static void intel_sagv_init(struct drm_i915_private *i915)
 {
-	if (!intel_has_sagv(i915))
+	if (!HAS_SAGV(i915))
 		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 
 	/*
@@ -359,7 +361,7 @@ static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 			continue;
 
 		/* Find the highest enabled wm level for this plane */
-		for (level = ilk_wm_max_level(i915);
+		for (level = i915->display.wm.num_levels - 1;
 		     !wm->wm[level].enable; --level)
 		     { }
 
@@ -410,6 +412,9 @@ static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 
+	if (!i915->params.enable_sagv)
+		return false;
+
 	if (DISPLAY_VER(i915) >= 12)
 		return tgl_crtc_can_enable_sagv(crtc_state);
 	else
@@ -704,16 +709,38 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
 				 const struct skl_wm_level *result_prev,
 				 struct skl_wm_level *result /* out */);
 
+static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level,
+				   const struct skl_wm_params *wp)
+{
+	unsigned int latency = i915->display.wm.skl_latency[level];
+
+	if (latency == 0)
+		return 0;
+
+	/*
+	 * WaIncreaseLatencyIPCEnabled: kbl,cfl
+	 * Display WA #1141: kbl,cfl
+	 */
+	if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
+	    skl_watermark_ipc_enabled(i915))
+		latency += 4;
+
+	if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled)
+		latency += 15;
+
+	return latency;
+}
+
 static unsigned int
 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
 		      int num_active)
 {
 	struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
-	int level, max_level = ilk_wm_max_level(i915);
 	struct skl_wm_level wm = {};
 	int ret, min_ddb_alloc = 0;
 	struct skl_wm_params wp;
+	int level;
 
 	ret = skl_compute_wm_params(crtc_state, 256,
 				    drm_format_info(DRM_FORMAT_ARGB8888),
@@ -722,8 +749,8 @@ skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
 				    crtc_state->pixel_rate, &wp, 0);
 	drm_WARN_ON(&i915->drm, ret);
 
-	for (level = 0; level <= max_level; level++) {
-		unsigned int latency = i915->display.wm.skl_latency[level];
+	for (level = 0; level < i915->display.wm.num_levels; level++) {
+		unsigned int latency = skl_wm_latency(i915, level, &wp);
 
 		skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
 		if (wm.min_ddb_alloc == U16_MAX)
@@ -1407,16 +1434,22 @@ skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
 	}
 }
 
-static bool icl_need_wm1_wa(struct drm_i915_private *i915,
-			    enum plane_id plane_id)
+static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level,
+				const struct skl_plane_wm *wm)
 {
 	/*
 	 * Wa_1408961008:icl, ehl
 	 * Wa_14012656716:tgl, adl
-	 * Underruns with WM1+ disabled
+	 * Wa_14017887344:icl
+	 * Wa_14017868169:adl, tgl
+	 * Due to some power saving optimizations, different subsystems
+	 * like PSR, might still use even disabled wm level registers,
+	 * for "reference", so lets keep at least the values sane.
+	 * Considering amount of WA requiring us to do similar things, was
+	 * decided to simply do it for all of the platforms, as those wm
+	 * levels are disabled, this isn't going to do harm anyway.
 	 */
-	return DISPLAY_VER(i915) == 11 ||
-	       (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR);
+	return level > 0 && !wm->wm[level].enable;
 }
 
 struct skl_plane_ddb_iter {
@@ -1492,7 +1525,7 @@ skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
 	 * Find the highest watermark level for which we can satisfy the block
 	 * requirement of active planes.
 	 */
-	for (level = ilk_wm_max_level(i915); level >= 0; level--) {
+	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
 		blocks = 0;
 		for_each_plane_id_on_crtc(crtc, plane_id) {
 			const struct skl_plane_wm *wm =
@@ -1568,7 +1601,7 @@ skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
 	 * all levels as "enabled."  Go back now and disable the ones
 	 * that aren't actually possible.
 	 */
-	for (level++; level <= ilk_wm_max_level(i915); level++) {
+	for (level++; level < i915->display.wm.num_levels; level++) {
 		for_each_plane_id_on_crtc(crtc, plane_id) {
 			const struct skl_ddb_entry *ddb =
 				&crtc_state->wm.skl.plane_ddb[plane_id];
@@ -1585,12 +1618,10 @@ skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
 			else
 				skl_check_wm_level(&wm->wm[level], ddb);
 
-			if (icl_need_wm1_wa(i915, plane_id) &&
-			    level == 1 && !wm->wm[level].enable &&
-			    wm->wm[0].enable) {
-				wm->wm[level].blocks = wm->wm[0].blocks;
-				wm->wm[level].lines = wm->wm[0].lines;
-				wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
+			if (skl_need_wm_copy_wa(i915, level, wm)) {
+				wm->wm[level].blocks = wm->wm[level - 1].blocks;
+				wm->wm[level].lines = wm->wm[level - 1].lines;
+				wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
 			}
 		}
 	}
@@ -1835,17 +1866,6 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
 		return;
 	}
 
-	/*
-	 * WaIncreaseLatencyIPCEnabled: kbl,cfl
-	 * Display WA #1141: kbl,cfl
-	 */
-	if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
-	    skl_watermark_ipc_enabled(i915))
-		latency += 4;
-
-	if (skl_needs_memory_bw_wa(i915) && wp->x_tiled)
-		latency += 15;
-
 	method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
 				 wp->cpp, latency, wp->dbuf_block_size);
 	method2 = skl_wm_method2(wp->plane_pixel_rate,
@@ -1967,12 +1987,12 @@ skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
 		      struct skl_wm_level *levels)
 {
 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
-	int level, max_level = ilk_wm_max_level(i915);
 	struct skl_wm_level *result_prev = &levels[0];
+	int level;
 
-	for (level = 0; level <= max_level; level++) {
+	for (level = 0; level < i915->display.wm.num_levels; level++) {
 		struct skl_wm_level *result = &levels[level];
-		unsigned int latency = i915->display.wm.skl_latency[level];
+		unsigned int latency = skl_wm_latency(i915, level, wm_params);
 
 		skl_compute_plane_wm(crtc_state, plane, level, latency,
 				     wm_params, result_prev, result);
@@ -1992,7 +2012,8 @@ static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
 	unsigned int latency = 0;
 
 	if (i915->display.sagv.block_time_us)
-		latency = i915->display.sagv.block_time_us + i915->display.wm.skl_latency[0];
+		latency = i915->display.sagv.block_time_us +
+			skl_wm_latency(i915, 0, wm_params);
 
 	skl_compute_plane_wm(crtc_state, plane, 0, latency,
 			     wm_params, &levels[0],
@@ -2184,6 +2205,117 @@ static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
 	return 0;
 }
 
+static bool
+skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
+			int wm0_lines, int latency)
+{
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->hw.adjusted_mode;
+
+	/* FIXME missing scaler and DSC pre-fill time */
+	return crtc_state->framestart_delay +
+		intel_usecs_to_scanlines(adjusted_mode, latency) +
+		wm0_lines >
+		adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
+}
+
+static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	enum plane_id plane_id;
+	int wm0_lines = 0;
+
+	for_each_plane_id_on_crtc(crtc, plane_id) {
+		const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
+
+		/* FIXME what about !skl_wm_has_lines() platforms? */
+		wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
+	}
+
+	return wm0_lines;
+}
+
+static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
+				       int wm0_lines)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	int level;
+
+	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
+		int latency;
+
+		/* FIXME should we care about the latency w/a's? */
+		latency = skl_wm_latency(i915, level, NULL);
+		if (latency == 0)
+			continue;
+
+		/* FIXME is it correct to use 0 latency for wm0 here? */
+		if (level == 0)
+			latency = 0;
+
+		if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
+			return level;
+	}
+
+	return -EINVAL;
+}
+
+static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	int wm0_lines, level;
+
+	if (!crtc_state->hw.active)
+		return 0;
+
+	wm0_lines = skl_max_wm0_lines(crtc_state);
+
+	level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
+	if (level < 0)
+		return level;
+
+	/*
+	 * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
+	 * based on whether we're limited by the vblank duration.
+	 */
+	crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1;
+
+	for (level++; level < i915->display.wm.num_levels; level++) {
+		enum plane_id plane_id;
+
+		for_each_plane_id_on_crtc(crtc, plane_id) {
+			struct skl_plane_wm *wm =
+				&crtc_state->wm.skl.optimal.planes[plane_id];
+
+			/*
+			 * FIXME just clear enable or flag the entire
+			 * thing as bad via min_ddb_alloc=U16_MAX?
+			 */
+			wm->wm[level].enable = false;
+			wm->uv_wm[level].enable = false;
+		}
+	}
+
+	if (DISPLAY_VER(i915) >= 12 &&
+	    i915->display.sagv.block_time_us &&
+	    skl_is_vblank_too_short(crtc_state, wm0_lines,
+				    i915->display.sagv.block_time_us)) {
+		enum plane_id plane_id;
+
+		for_each_plane_id_on_crtc(crtc, plane_id) {
+			struct skl_plane_wm *wm =
+				&crtc_state->wm.skl.optimal.planes[plane_id];
+
+			wm->sagv.wm0.enable = false;
+			wm->sagv.trans_wm.enable = false;
+		}
+	}
+
+	return 0;
+}
+
 static int skl_build_pipe_wm(struct intel_atomic_state *state,
 			     struct intel_crtc *crtc)
 {
@@ -2213,7 +2345,7 @@ static int skl_build_pipe_wm(struct intel_atomic_state *state,
 
 	crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
 
-	return 0;
+	return skl_wm_check_vblank(crtc_state);
 }
 
 static void skl_ddb_entry_write(struct drm_i915_private *i915,
@@ -2248,7 +2380,6 @@ void skl_write_plane_wm(struct intel_plane *plane,
 			const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(plane->base.dev);
-	int level, max_level = ilk_wm_max_level(i915);
 	enum plane_id plane_id = plane->id;
 	enum pipe pipe = plane->pipe;
 	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
@@ -2256,8 +2387,9 @@ void skl_write_plane_wm(struct intel_plane *plane,
 		&crtc_state->wm.skl.plane_ddb[plane_id];
 	const struct skl_ddb_entry *ddb_y =
 		&crtc_state->wm.skl.plane_ddb_y[plane_id];
+	int level;
 
-	for (level = 0; level <= max_level; level++)
+	for (level = 0; level < i915->display.wm.num_levels; level++)
 		skl_write_wm_level(i915, PLANE_WM(pipe, plane_id, level),
 				   skl_plane_wm_level(pipe_wm, plane_id, level));
 
@@ -2285,14 +2417,14 @@ void skl_write_cursor_wm(struct intel_plane *plane,
 			 const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(plane->base.dev);
-	int level, max_level = ilk_wm_max_level(i915);
 	enum plane_id plane_id = plane->id;
 	enum pipe pipe = plane->pipe;
 	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
 	const struct skl_ddb_entry *ddb =
 		&crtc_state->wm.skl.plane_ddb[plane_id];
+	int level;
 
-	for (level = 0; level <= max_level; level++)
+	for (level = 0; level < i915->display.wm.num_levels; level++)
 		skl_write_wm_level(i915, CUR_WM(pipe, level),
 				   skl_plane_wm_level(pipe_wm, plane_id, level));
 
@@ -2324,9 +2456,9 @@ static bool skl_plane_wm_equals(struct drm_i915_private *i915,
 				const struct skl_plane_wm *wm1,
 				const struct skl_plane_wm *wm2)
 {
-	int level, max_level = ilk_wm_max_level(i915);
+	int level;
 
-	for (level = 0; level <= max_level; level++) {
+	for (level = 0; level < i915->display.wm.num_levels; level++) {
 		/*
 		 * We don't check uv_wm as the hardware doesn't actually
 		 * use it. It only gets used for calculating the required
@@ -2398,6 +2530,8 @@ skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
 			return PTR_ERR(plane_state);
 
 		new_crtc_state->update_planes |= BIT(plane_id);
+		new_crtc_state->async_flip_planes = 0;
+		new_crtc_state->do_async_flip = false;
 	}
 
 	return 0;
@@ -2674,9 +2808,9 @@ static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
 					 const struct skl_pipe_wm *new_pipe_wm)
 {
 	struct drm_i915_private *i915 = to_i915(plane->base.dev);
-	int level, max_level = ilk_wm_max_level(i915);
+	int level;
 
-	for (level = 0; level <= max_level; level++) {
+	for (level = 0; level < i915->display.wm.num_levels; level++) {
 		/*
 		 * We don't check uv_wm as the hardware doesn't actually
 		 * use it. It only gets used for calculating the required
@@ -2755,6 +2889,8 @@ static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
 			return PTR_ERR(plane_state);
 
 		new_crtc_state->update_planes |= BIT(plane_id);
+		new_crtc_state->async_flip_planes = 0;
+		new_crtc_state->do_async_flip = false;
 	}
 
 	return 0;
@@ -2810,16 +2946,14 @@ static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
 {
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
-	int level, max_level;
 	enum plane_id plane_id;
+	int level;
 	u32 val;
 
-	max_level = ilk_wm_max_level(i915);
-
 	for_each_plane_id_on_crtc(crtc, plane_id) {
 		struct skl_plane_wm *wm = &out->planes[plane_id];
 
-		for (level = 0; level <= max_level; level++) {
+		for (level = 0; level < i915->display.wm.num_levels; level++) {
 			if (plane_id != PLANE_CURSOR)
 				val = intel_de_read(i915, PLANE_WM(pipe, plane_id, level));
 			else
@@ -2856,7 +2990,7 @@ static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
 	}
 }
 
-void skl_wm_get_hw_state(struct drm_i915_private *i915)
+static void skl_wm_get_hw_state(struct drm_i915_private *i915)
 {
 	struct intel_dbuf_state *dbuf_state =
 		to_intel_dbuf_state(i915->display.dbuf.obj.state);
@@ -2956,7 +3090,7 @@ static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
 	return false;
 }
 
-void skl_wm_sanitize(struct drm_i915_private *i915)
+static void skl_wm_sanitize(struct drm_i915_private *i915)
 {
 	struct intel_crtc *crtc;
 
@@ -2992,6 +3126,12 @@ void skl_wm_sanitize(struct drm_i915_private *i915)
 	}
 }
 
+static void skl_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
+{
+	skl_wm_get_hw_state(i915);
+	skl_wm_sanitize(i915);
+}
+
 void intel_wm_state_verify(struct intel_crtc *crtc,
 			   struct intel_crtc_state *new_crtc_state)
 {
@@ -3002,9 +3142,9 @@ void intel_wm_state_verify(struct intel_crtc *crtc,
 		struct skl_pipe_wm wm;
 	} *hw;
 	const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
-	int level, max_level = ilk_wm_max_level(i915);
 	struct intel_plane *plane;
 	u8 hw_enabled_slices;
+	int level;
 
 	if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
 		return;
@@ -3031,7 +3171,7 @@ void intel_wm_state_verify(struct intel_crtc *crtc,
 		const struct skl_wm_level *hw_wm_level, *sw_wm_level;
 
 		/* Watermarks */
-		for (level = 0; level <= max_level; level++) {
+		for (level = 0; level < i915->display.wm.num_levels; level++) {
 			hw_wm_level = &hw->wm.planes[plane->id].wm[level];
 			sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
 
@@ -3153,7 +3293,7 @@ void skl_watermark_ipc_init(struct drm_i915_private *i915)
 
 static void
 adjust_wm_latency(struct drm_i915_private *i915,
-		  u16 wm[], int max_level, int read_latency)
+		  u16 wm[], int num_levels, int read_latency)
 {
 	bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
 	int i, level;
@@ -3163,12 +3303,12 @@ adjust_wm_latency(struct drm_i915_private *i915,
 	 * need to be disabled. We make sure to sanitize the values out
 	 * of the punit to satisfy this requirement.
 	 */
-	for (level = 1; level <= max_level; level++) {
+	for (level = 1; level < num_levels; level++) {
 		if (wm[level] == 0) {
-			for (i = level + 1; i <= max_level; i++)
+			for (i = level + 1; i < num_levels; i++)
 				wm[i] = 0;
 
-			max_level = level - 1;
+			num_levels = level;
 			break;
 		}
 	}
@@ -3181,7 +3321,7 @@ adjust_wm_latency(struct drm_i915_private *i915,
 	 * from the punit when level 0 response data is 0us.
 	 */
 	if (wm[0] == 0) {
-		for (level = 0; level <= max_level; level++)
+		for (level = 0; level < num_levels; level++)
 			wm[level] += read_latency;
 	}
 
@@ -3197,7 +3337,7 @@ adjust_wm_latency(struct drm_i915_private *i915,
 
 static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
 {
-	int max_level = ilk_wm_max_level(i915);
+	int num_levels = i915->display.wm.num_levels;
 	u32 val;
 
 	val = intel_de_read(i915, MTL_LATENCY_LP0_LP1);
@@ -3212,12 +3352,12 @@ static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
 	wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
 	wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
 
-	adjust_wm_latency(i915, wm, max_level, 6);
+	adjust_wm_latency(i915, wm, num_levels, 6);
 }
 
 static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
 {
-	int max_level = ilk_wm_max_level(i915);
+	int num_levels = i915->display.wm.num_levels;
 	int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
 	int mult = IS_DG2(i915) ? 2 : 1;
 	u32 val;
@@ -3249,11 +3389,16 @@ static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
 	wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
 	wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
 
-	adjust_wm_latency(i915, wm, max_level, read_latency);
+	adjust_wm_latency(i915, wm, num_levels, read_latency);
 }
 
 static void skl_setup_wm_latency(struct drm_i915_private *i915)
 {
+	if (HAS_HW_SAGV_WM(i915))
+		i915->display.wm.num_levels = 6;
+	else
+		i915->display.wm.num_levels = 8;
+
 	if (DISPLAY_VER(i915) >= 14)
 		mtl_read_wm_latency(i915, i915->display.wm.skl_latency);
 	else
@@ -3264,6 +3409,7 @@ static void skl_setup_wm_latency(struct drm_i915_private *i915)
 
 static const struct intel_wm_funcs skl_wm_funcs = {
 	.compute_global_watermarks = skl_compute_wm,
+	.get_hw_state = skl_wm_get_hw_state_and_sanitize,
 };
 
 void skl_wm_init(struct drm_i915_private *i915)
@@ -3541,13 +3687,35 @@ static const struct file_operations skl_watermark_ipc_status_fops = {
 	.write = skl_watermark_ipc_status_write
 };
 
-void skl_watermark_ipc_debugfs_register(struct drm_i915_private *i915)
+static int intel_sagv_status_show(struct seq_file *m, void *unused)
+{
+	struct drm_i915_private *i915 = m->private;
+	static const char * const sagv_status[] = {
+		[I915_SAGV_UNKNOWN] = "unknown",
+		[I915_SAGV_DISABLED] = "disabled",
+		[I915_SAGV_ENABLED] = "enabled",
+		[I915_SAGV_NOT_CONTROLLED] = "not controlled",
+	};
+
+	seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915)));
+	seq_printf(m, "SAGV modparam: %s\n", str_enabled_disabled(i915->params.enable_sagv));
+	seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
+	seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
+
+	return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
+
+void skl_watermark_debugfs_register(struct drm_i915_private *i915)
 {
 	struct drm_minor *minor = i915->drm.primary;
 
-	if (!HAS_IPC(i915))
-		return;
+	if (HAS_IPC(i915))
+		debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
+				    &skl_watermark_ipc_status_fops);
 
-	debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
-			    &skl_watermark_ipc_status_fops);
+	if (HAS_SAGV(i915))
+		debugfs_create_file("i915_sagv_status", 0444, minor->debugfs_root, i915,
+				    &intel_sagv_status_fops);
 }
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.h b/drivers/gpu/drm/i915/display/skl_watermark.h
index 37954c472070..f91a3d4ddc07 100644
--- a/drivers/gpu/drm/i915/display/skl_watermark.h
+++ b/drivers/gpu/drm/i915/display/skl_watermark.h
@@ -10,7 +10,7 @@
 
 #include "intel_display_limits.h"
 #include "intel_global_state.h"
-#include "intel_pm_types.h"
+#include "intel_wm_types.h"
 
 struct drm_i915_private;
 struct intel_atomic_state;
@@ -38,16 +38,13 @@ bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
 				 const struct skl_ddb_entry *entries,
 				 int num_entries, int ignore_idx);
 
-void skl_wm_get_hw_state(struct drm_i915_private *i915);
-void skl_wm_sanitize(struct drm_i915_private *i915);
-
 void intel_wm_state_verify(struct intel_crtc *crtc,
 			   struct intel_crtc_state *new_crtc_state);
 
 void skl_watermark_ipc_init(struct drm_i915_private *i915);
 void skl_watermark_ipc_update(struct drm_i915_private *i915);
 bool skl_watermark_ipc_enabled(struct drm_i915_private *i915);
-void skl_watermark_ipc_debugfs_register(struct drm_i915_private *i915);
+void skl_watermark_debugfs_register(struct drm_i915_private *i915);
 
 void skl_wm_init(struct drm_i915_private *i915);
 
diff --git a/drivers/gpu/drm/i915/display/skl_watermark_regs.h b/drivers/gpu/drm/i915/display/skl_watermark_regs.h
new file mode 100644
index 000000000000..628c5920ad49
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/skl_watermark_regs.h
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __SKL_WATERMARK_REGS_H__
+#define __SKL_WATERMARK_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+#define _PIPEA_MBUS_DBOX_CTL			0x7003C
+#define _PIPEB_MBUS_DBOX_CTL			0x7103C
+#define PIPE_MBUS_DBOX_CTL(pipe)		_MMIO_PIPE(pipe, _PIPEA_MBUS_DBOX_CTL, \
+							   _PIPEB_MBUS_DBOX_CTL)
+#define MBUS_DBOX_B2B_TRANSACTIONS_MAX_MASK	REG_GENMASK(24, 20) /* tgl+ */
+#define MBUS_DBOX_B2B_TRANSACTIONS_MAX(x)	REG_FIELD_PREP(MBUS_DBOX_B2B_TRANSACTIONS_MAX_MASK, x)
+#define MBUS_DBOX_B2B_TRANSACTIONS_DELAY_MASK	REG_GENMASK(19, 17) /* tgl+ */
+#define MBUS_DBOX_B2B_TRANSACTIONS_DELAY(x)	REG_FIELD_PREP(MBUS_DBOX_B2B_TRANSACTIONS_DELAY_MASK, x)
+#define MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN	REG_BIT(16) /* tgl+ */
+#define MBUS_DBOX_BW_CREDIT_MASK		REG_GENMASK(15, 14)
+#define MBUS_DBOX_BW_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, x)
+#define MBUS_DBOX_BW_4CREDITS_MTL		REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, 0x2)
+#define MBUS_DBOX_BW_8CREDITS_MTL		REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, 0x3)
+#define MBUS_DBOX_B_CREDIT_MASK			REG_GENMASK(12, 8)
+#define MBUS_DBOX_B_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_B_CREDIT_MASK, x)
+#define MBUS_DBOX_I_CREDIT_MASK			REG_GENMASK(7, 5)
+#define MBUS_DBOX_I_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_I_CREDIT_MASK, x)
+#define MBUS_DBOX_A_CREDIT_MASK			REG_GENMASK(3, 0)
+#define MBUS_DBOX_A_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_A_CREDIT_MASK, x)
+
+#define MBUS_UBOX_CTL			_MMIO(0x4503C)
+#define MBUS_BBOX_CTL_S1		_MMIO(0x45040)
+#define MBUS_BBOX_CTL_S2		_MMIO(0x45044)
+
+#define MBUS_CTL			_MMIO(0x4438C)
+#define MBUS_JOIN			REG_BIT(31)
+#define MBUS_HASHING_MODE_MASK		REG_BIT(30)
+#define MBUS_HASHING_MODE_2x2		REG_FIELD_PREP(MBUS_HASHING_MODE_MASK, 0)
+#define MBUS_HASHING_MODE_1x4		REG_FIELD_PREP(MBUS_HASHING_MODE_MASK, 1)
+#define MBUS_JOIN_PIPE_SELECT_MASK	REG_GENMASK(28, 26)
+#define MBUS_JOIN_PIPE_SELECT(pipe)	REG_FIELD_PREP(MBUS_JOIN_PIPE_SELECT_MASK, pipe)
+#define MBUS_JOIN_PIPE_SELECT_NONE	MBUS_JOIN_PIPE_SELECT(7)
+
+/* Watermark register definitions for SKL */
+#define _CUR_WM_A_0		0x70140
+#define _CUR_WM_B_0		0x71140
+#define _CUR_WM_SAGV_A		0x70158
+#define _CUR_WM_SAGV_B		0x71158
+#define _CUR_WM_SAGV_TRANS_A	0x7015C
+#define _CUR_WM_SAGV_TRANS_B	0x7115C
+#define _CUR_WM_TRANS_A		0x70168
+#define _CUR_WM_TRANS_B		0x71168
+#define _PLANE_WM_1_A_0		0x70240
+#define _PLANE_WM_1_B_0		0x71240
+#define _PLANE_WM_2_A_0		0x70340
+#define _PLANE_WM_2_B_0		0x71340
+#define _PLANE_WM_SAGV_1_A	0x70258
+#define _PLANE_WM_SAGV_1_B	0x71258
+#define _PLANE_WM_SAGV_2_A	0x70358
+#define _PLANE_WM_SAGV_2_B	0x71358
+#define _PLANE_WM_SAGV_TRANS_1_A	0x7025C
+#define _PLANE_WM_SAGV_TRANS_1_B	0x7125C
+#define _PLANE_WM_SAGV_TRANS_2_A	0x7035C
+#define _PLANE_WM_SAGV_TRANS_2_B	0x7135C
+#define _PLANE_WM_TRANS_1_A	0x70268
+#define _PLANE_WM_TRANS_1_B	0x71268
+#define _PLANE_WM_TRANS_2_A	0x70368
+#define _PLANE_WM_TRANS_2_B	0x71368
+#define   PLANE_WM_EN		(1 << 31)
+#define   PLANE_WM_IGNORE_LINES	(1 << 30)
+#define   PLANE_WM_LINES_MASK	REG_GENMASK(26, 14)
+#define   PLANE_WM_BLOCKS_MASK	REG_GENMASK(11, 0)
+
+#define _CUR_WM_0(pipe) _PIPE(pipe, _CUR_WM_A_0, _CUR_WM_B_0)
+#define CUR_WM(pipe, level) _MMIO(_CUR_WM_0(pipe) + ((4) * (level)))
+#define CUR_WM_SAGV(pipe) _MMIO_PIPE(pipe, _CUR_WM_SAGV_A, _CUR_WM_SAGV_B)
+#define CUR_WM_SAGV_TRANS(pipe) _MMIO_PIPE(pipe, _CUR_WM_SAGV_TRANS_A, _CUR_WM_SAGV_TRANS_B)
+#define CUR_WM_TRANS(pipe) _MMIO_PIPE(pipe, _CUR_WM_TRANS_A, _CUR_WM_TRANS_B)
+#define _PLANE_WM_1(pipe) _PIPE(pipe, _PLANE_WM_1_A_0, _PLANE_WM_1_B_0)
+#define _PLANE_WM_2(pipe) _PIPE(pipe, _PLANE_WM_2_A_0, _PLANE_WM_2_B_0)
+#define _PLANE_WM_BASE(pipe, plane) \
+	_PLANE(plane, _PLANE_WM_1(pipe), _PLANE_WM_2(pipe))
+#define PLANE_WM(pipe, plane, level) \
+	_MMIO(_PLANE_WM_BASE(pipe, plane) + ((4) * (level)))
+#define _PLANE_WM_SAGV_1(pipe) \
+	_PIPE(pipe, _PLANE_WM_SAGV_1_A, _PLANE_WM_SAGV_1_B)
+#define _PLANE_WM_SAGV_2(pipe) \
+	_PIPE(pipe, _PLANE_WM_SAGV_2_A, _PLANE_WM_SAGV_2_B)
+#define PLANE_WM_SAGV(pipe, plane) \
+	_MMIO(_PLANE(plane, _PLANE_WM_SAGV_1(pipe), _PLANE_WM_SAGV_2(pipe)))
+#define _PLANE_WM_SAGV_TRANS_1(pipe) \
+	_PIPE(pipe, _PLANE_WM_SAGV_TRANS_1_A, _PLANE_WM_SAGV_TRANS_1_B)
+#define _PLANE_WM_SAGV_TRANS_2(pipe) \
+	_PIPE(pipe, _PLANE_WM_SAGV_TRANS_2_A, _PLANE_WM_SAGV_TRANS_2_B)
+#define PLANE_WM_SAGV_TRANS(pipe, plane) \
+	_MMIO(_PLANE(plane, _PLANE_WM_SAGV_TRANS_1(pipe), _PLANE_WM_SAGV_TRANS_2(pipe)))
+#define _PLANE_WM_TRANS_1(pipe) \
+	_PIPE(pipe, _PLANE_WM_TRANS_1_A, _PLANE_WM_TRANS_1_B)
+#define _PLANE_WM_TRANS_2(pipe) \
+	_PIPE(pipe, _PLANE_WM_TRANS_2_A, _PLANE_WM_TRANS_2_B)
+#define PLANE_WM_TRANS(pipe, plane) \
+	_MMIO(_PLANE(plane, _PLANE_WM_TRANS_1(pipe), _PLANE_WM_TRANS_2(pipe)))
+
+#define _PLANE_BUF_CFG_1_B			0x7127c
+#define _PLANE_BUF_CFG_2_B			0x7137c
+#define _PLANE_BUF_CFG_1(pipe)	\
+	_PIPE(pipe, _PLANE_BUF_CFG_1_A, _PLANE_BUF_CFG_1_B)
+#define _PLANE_BUF_CFG_2(pipe)	\
+	_PIPE(pipe, _PLANE_BUF_CFG_2_A, _PLANE_BUF_CFG_2_B)
+#define PLANE_BUF_CFG(pipe, plane)	\
+	_MMIO_PLANE(plane, _PLANE_BUF_CFG_1(pipe), _PLANE_BUF_CFG_2(pipe))
+
+#define _PLANE_NV12_BUF_CFG_1_B		0x71278
+#define _PLANE_NV12_BUF_CFG_2_B		0x71378
+#define _PLANE_NV12_BUF_CFG_1(pipe)	\
+	_PIPE(pipe, _PLANE_NV12_BUF_CFG_1_A, _PLANE_NV12_BUF_CFG_1_B)
+#define _PLANE_NV12_BUF_CFG_2(pipe)	\
+	_PIPE(pipe, _PLANE_NV12_BUF_CFG_2_A, _PLANE_NV12_BUF_CFG_2_B)
+#define PLANE_NV12_BUF_CFG(pipe, plane)	\
+	_MMIO_PLANE(plane, _PLANE_NV12_BUF_CFG_1(pipe), _PLANE_NV12_BUF_CFG_2(pipe))
+
+/* SKL new cursor registers */
+#define _CUR_BUF_CFG_A				0x7017c
+#define _CUR_BUF_CFG_B				0x7117c
+#define CUR_BUF_CFG(pipe)	_MMIO_PIPE(pipe, _CUR_BUF_CFG_A, _CUR_BUF_CFG_B)
+
+/*
+ * The below are numbered starting from "S1" on gen11/gen12, but starting
+ * with display 13, the bspec switches to a 0-based numbering scheme
+ * (although the addresses stay the same so new S0 = old S1, new S1 = old S2).
+ * We'll just use the 0-based numbering here for all platforms since it's the
+ * way things will be named by the hardware team going forward, plus it's more
+ * consistent with how most of the rest of our registers are named.
+ */
+#define _DBUF_CTL_S0				0x45008
+#define _DBUF_CTL_S1				0x44FE8
+#define _DBUF_CTL_S2				0x44300
+#define _DBUF_CTL_S3				0x44304
+#define DBUF_CTL_S(slice)			_MMIO(_PICK(slice, \
+							    _DBUF_CTL_S0, \
+							    _DBUF_CTL_S1, \
+							    _DBUF_CTL_S2, \
+							    _DBUF_CTL_S3))
+#define  DBUF_POWER_REQUEST			REG_BIT(31)
+#define  DBUF_POWER_STATE			REG_BIT(30)
+#define  DBUF_TRACKER_STATE_SERVICE_MASK	REG_GENMASK(23, 19)
+#define  DBUF_TRACKER_STATE_SERVICE(x)		REG_FIELD_PREP(DBUF_TRACKER_STATE_SERVICE_MASK, x)
+#define  DBUF_MIN_TRACKER_STATE_SERVICE_MASK	REG_GENMASK(18, 16) /* ADL-P+ */
+#define  DBUF_MIN_TRACKER_STATE_SERVICE(x)		REG_FIELD_PREP(DBUF_MIN_TRACKER_STATE_SERVICE_MASK, x) /* ADL-P+ */
+
+#define MTL_LATENCY_LP0_LP1		_MMIO(0x45780)
+#define MTL_LATENCY_LP2_LP3		_MMIO(0x45784)
+#define MTL_LATENCY_LP4_LP5		_MMIO(0x45788)
+#define  MTL_LATENCY_LEVEL_EVEN_MASK	REG_GENMASK(12, 0)
+#define  MTL_LATENCY_LEVEL_ODD_MASK	REG_GENMASK(28, 16)
+
+#define MTL_LATENCY_SAGV		_MMIO(0x4578c)
+#define   MTL_LATENCY_QCLK_SAGV		REG_GENMASK(12, 0)
+
+#endif /* __SKL_WATERMARK_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c
index 2289f6b1b4eb..028965ab442d 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi.c
@@ -331,32 +331,23 @@ static bool glk_dsi_enable_io(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
-	u32 tmp;
 	bool cold_boot = false;
 
 	/* Set the MIPI mode
 	 * If MIPI_Mode is off, then writing to LP_Wake bit is not reflecting.
 	 * Power ON MIPI IO first and then write into IO reset and LP wake bits
 	 */
-	for_each_dsi_port(port, intel_dsi->ports) {
-		tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
-		intel_de_write(dev_priv, MIPI_CTRL(port),
-			       tmp | GLK_MIPIIO_ENABLE);
-	}
+	for_each_dsi_port(port, intel_dsi->ports)
+		intel_de_rmw(dev_priv, MIPI_CTRL(port), 0, GLK_MIPIIO_ENABLE);
 
 	/* Put the IO into reset */
-	tmp = intel_de_read(dev_priv, MIPI_CTRL(PORT_A));
-	tmp &= ~GLK_MIPIIO_RESET_RELEASED;
-	intel_de_write(dev_priv, MIPI_CTRL(PORT_A), tmp);
+	intel_de_rmw(dev_priv, MIPI_CTRL(PORT_A), GLK_MIPIIO_RESET_RELEASED, 0);
 
 	/* Program LP Wake */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
-		if (!(intel_de_read(dev_priv, MIPI_DEVICE_READY(port)) & DEVICE_READY))
-			tmp &= ~GLK_LP_WAKE;
-		else
-			tmp |= GLK_LP_WAKE;
-		intel_de_write(dev_priv, MIPI_CTRL(port), tmp);
+		u32 tmp = intel_de_read(dev_priv, MIPI_DEVICE_READY(port));
+		intel_de_rmw(dev_priv, MIPI_CTRL(port),
+			     GLK_LP_WAKE, (tmp & DEVICE_READY) ? GLK_LP_WAKE : 0);
 	}
 
 	/* Wait for Pwr ACK */
@@ -380,7 +371,6 @@ static void glk_dsi_device_ready(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
-	u32 val;
 
 	/* Wait for MIPI PHY status bit to set */
 	for_each_dsi_port(port, intel_dsi->ports) {
@@ -390,24 +380,18 @@ static void glk_dsi_device_ready(struct intel_encoder *encoder)
 	}
 
 	/* Get IO out of reset */
-	val = intel_de_read(dev_priv, MIPI_CTRL(PORT_A));
-	intel_de_write(dev_priv, MIPI_CTRL(PORT_A),
-		       val | GLK_MIPIIO_RESET_RELEASED);
+	intel_de_rmw(dev_priv, MIPI_CTRL(PORT_A), 0, GLK_MIPIIO_RESET_RELEASED);
 
 	/* Get IO out of Low power state*/
 	for_each_dsi_port(port, intel_dsi->ports) {
 		if (!(intel_de_read(dev_priv, MIPI_DEVICE_READY(port)) & DEVICE_READY)) {
-			val = intel_de_read(dev_priv, MIPI_DEVICE_READY(port));
-			val &= ~ULPS_STATE_MASK;
-			val |= DEVICE_READY;
-			intel_de_write(dev_priv, MIPI_DEVICE_READY(port), val);
+			intel_de_rmw(dev_priv, MIPI_DEVICE_READY(port),
+				     ULPS_STATE_MASK, DEVICE_READY);
 			usleep_range(10, 15);
 		} else {
 			/* Enter ULPS */
-			val = intel_de_read(dev_priv, MIPI_DEVICE_READY(port));
-			val &= ~ULPS_STATE_MASK;
-			val |= (ULPS_STATE_ENTER | DEVICE_READY);
-			intel_de_write(dev_priv, MIPI_DEVICE_READY(port), val);
+			intel_de_rmw(dev_priv, MIPI_DEVICE_READY(port),
+				     ULPS_STATE_MASK, ULPS_STATE_ENTER | DEVICE_READY);
 
 			/* Wait for ULPS active */
 			if (intel_de_wait_for_clear(dev_priv, MIPI_CTRL(port),
@@ -415,20 +399,15 @@ static void glk_dsi_device_ready(struct intel_encoder *encoder)
 				drm_err(&dev_priv->drm, "ULPS not active\n");
 
 			/* Exit ULPS */
-			val = intel_de_read(dev_priv, MIPI_DEVICE_READY(port));
-			val &= ~ULPS_STATE_MASK;
-			val |= (ULPS_STATE_EXIT | DEVICE_READY);
-			intel_de_write(dev_priv, MIPI_DEVICE_READY(port), val);
+			intel_de_rmw(dev_priv, MIPI_DEVICE_READY(port),
+				     ULPS_STATE_MASK, ULPS_STATE_EXIT | DEVICE_READY);
 
 			/* Enter Normal Mode */
-			val = intel_de_read(dev_priv, MIPI_DEVICE_READY(port));
-			val &= ~ULPS_STATE_MASK;
-			val |= (ULPS_STATE_NORMAL_OPERATION | DEVICE_READY);
-			intel_de_write(dev_priv, MIPI_DEVICE_READY(port), val);
-
-			val = intel_de_read(dev_priv, MIPI_CTRL(port));
-			val &= ~GLK_LP_WAKE;
-			intel_de_write(dev_priv, MIPI_CTRL(port), val);
+			intel_de_rmw(dev_priv, MIPI_DEVICE_READY(port),
+				     ULPS_STATE_MASK,
+				     ULPS_STATE_NORMAL_OPERATION | DEVICE_READY);
+
+			intel_de_rmw(dev_priv, MIPI_CTRL(port), GLK_LP_WAKE, 0);
 		}
 	}
 
@@ -460,9 +439,7 @@ static void bxt_dsi_device_ready(struct intel_encoder *encoder)
 
 	/* Enable MIPI PHY transparent latch */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		val = intel_de_read(dev_priv, BXT_MIPI_PORT_CTRL(port));
-		intel_de_write(dev_priv, BXT_MIPI_PORT_CTRL(port),
-			       val | LP_OUTPUT_HOLD);
+		intel_de_rmw(dev_priv, BXT_MIPI_PORT_CTRL(port), 0, LP_OUTPUT_HOLD);
 		usleep_range(2000, 2500);
 	}
 
@@ -482,7 +459,6 @@ static void vlv_dsi_device_ready(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
-	u32 val;
 
 	drm_dbg_kms(&dev_priv->drm, "\n");
 
@@ -505,9 +481,7 @@ static void vlv_dsi_device_ready(struct intel_encoder *encoder)
 		 * Common bit for both MIPI Port A & MIPI Port C
 		 * No similar bit in MIPI Port C reg
 		 */
-		val = intel_de_read(dev_priv, MIPI_PORT_CTRL(PORT_A));
-		intel_de_write(dev_priv, MIPI_PORT_CTRL(PORT_A),
-			       val | LP_OUTPUT_HOLD);
+		intel_de_rmw(dev_priv, MIPI_PORT_CTRL(PORT_A), 0, LP_OUTPUT_HOLD);
 		usleep_range(1000, 1500);
 
 		intel_de_write(dev_priv, MIPI_DEVICE_READY(port),
@@ -537,15 +511,11 @@ static void glk_dsi_enter_low_power_mode(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
-	u32 val;
 
 	/* Enter ULPS */
-	for_each_dsi_port(port, intel_dsi->ports) {
-		val = intel_de_read(dev_priv, MIPI_DEVICE_READY(port));
-		val &= ~ULPS_STATE_MASK;
-		val |= (ULPS_STATE_ENTER | DEVICE_READY);
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port), val);
-	}
+	for_each_dsi_port(port, intel_dsi->ports)
+		intel_de_rmw(dev_priv, MIPI_DEVICE_READY(port),
+			     ULPS_STATE_MASK, ULPS_STATE_ENTER | DEVICE_READY);
 
 	/* Wait for MIPI PHY status bit to unset */
 	for_each_dsi_port(port, intel_dsi->ports) {
@@ -568,12 +538,9 @@ static void glk_dsi_disable_mipi_io(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
-	u32 tmp;
 
 	/* Put the IO into reset */
-	tmp = intel_de_read(dev_priv, MIPI_CTRL(PORT_A));
-	tmp &= ~GLK_MIPIIO_RESET_RELEASED;
-	intel_de_write(dev_priv, MIPI_CTRL(PORT_A), tmp);
+	intel_de_rmw(dev_priv, MIPI_CTRL(PORT_A), GLK_MIPIIO_RESET_RELEASED, 0);
 
 	/* Wait for MIPI PHY status bit to unset */
 	for_each_dsi_port(port, intel_dsi->ports) {
@@ -583,11 +550,8 @@ static void glk_dsi_disable_mipi_io(struct intel_encoder *encoder)
 	}
 
 	/* Clear MIPI mode */
-	for_each_dsi_port(port, intel_dsi->ports) {
-		tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
-		tmp &= ~GLK_MIPIIO_ENABLE;
-		intel_de_write(dev_priv, MIPI_CTRL(port), tmp);
-	}
+	for_each_dsi_port(port, intel_dsi->ports)
+		intel_de_rmw(dev_priv, MIPI_CTRL(port), GLK_MIPIIO_ENABLE, 0);
 }
 
 static void glk_dsi_clear_device_ready(struct intel_encoder *encoder)
@@ -607,7 +571,6 @@ static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 		/* Common bit for both MIPI Port A & MIPI Port C on VLV/CHV */
 		i915_reg_t port_ctrl = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ?
 			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(PORT_A);
-		u32 val;
 
 		intel_de_write(dev_priv, MIPI_DEVICE_READY(port),
 			       DEVICE_READY | ULPS_STATE_ENTER);
@@ -631,8 +594,7 @@ static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 			drm_err(&dev_priv->drm, "DSI LP not going Low\n");
 
 		/* Disable MIPI PHY transparent latch */
-		val = intel_de_read(dev_priv, port_ctrl);
-		intel_de_write(dev_priv, port_ctrl, val & ~LP_OUTPUT_HOLD);
+		intel_de_rmw(dev_priv, port_ctrl, LP_OUTPUT_HOLD, 0);
 		usleep_range(1000, 1500);
 
 		intel_de_write(dev_priv, MIPI_DEVICE_READY(port), 0x00);
@@ -649,23 +611,17 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder,
 	enum port port;
 
 	if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) {
-		u32 temp;
+		u32 temp = intel_dsi->pixel_overlap;
+
 		if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
-			for_each_dsi_port(port, intel_dsi->ports) {
-				temp = intel_de_read(dev_priv,
-						     MIPI_CTRL(port));
-				temp &= ~BXT_PIXEL_OVERLAP_CNT_MASK |
-					intel_dsi->pixel_overlap <<
-					BXT_PIXEL_OVERLAP_CNT_SHIFT;
-				intel_de_write(dev_priv, MIPI_CTRL(port),
-					       temp);
-			}
+			for_each_dsi_port(port, intel_dsi->ports)
+				intel_de_rmw(dev_priv, MIPI_CTRL(port),
+					     BXT_PIXEL_OVERLAP_CNT_MASK,
+					     temp << BXT_PIXEL_OVERLAP_CNT_SHIFT);
 		} else {
-			temp = intel_de_read(dev_priv, VLV_CHICKEN_3);
-			temp &= ~PIXEL_OVERLAP_CNT_MASK |
-					intel_dsi->pixel_overlap <<
-					PIXEL_OVERLAP_CNT_SHIFT;
-			intel_de_write(dev_priv, VLV_CHICKEN_3, temp);
+			intel_de_rmw(dev_priv, VLV_CHICKEN_3,
+				     PIXEL_OVERLAP_CNT_MASK,
+				     temp << PIXEL_OVERLAP_CNT_SHIFT);
 		}
 	}
 
@@ -709,11 +665,9 @@ static void intel_dsi_port_disable(struct intel_encoder *encoder)
 	for_each_dsi_port(port, intel_dsi->ports) {
 		i915_reg_t port_ctrl = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ?
 			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
-		u32 temp;
 
 		/* de-assert ip_tg_enable signal */
-		temp = intel_de_read(dev_priv, port_ctrl);
-		intel_de_write(dev_priv, port_ctrl, temp & ~DPI_ENABLE);
+		intel_de_rmw(dev_priv, port_ctrl, DPI_ENABLE, 0);
 		intel_de_posting_read(dev_priv, port_ctrl);
 	}
 }
@@ -787,7 +741,6 @@ static void intel_dsi_pre_enable(struct intel_atomic_state *state,
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 	enum port port;
-	u32 val;
 	bool glk_cold_boot = false;
 
 	drm_dbg_kms(&dev_priv->drm, "\n");
@@ -810,9 +763,7 @@ static void intel_dsi_pre_enable(struct intel_atomic_state *state,
 
 	if (IS_BROXTON(dev_priv)) {
 		/* Add MIPI IO reset programming for modeset */
-		val = intel_de_read(dev_priv, BXT_P_CR_GT_DISP_PWRON);
-		intel_de_write(dev_priv, BXT_P_CR_GT_DISP_PWRON,
-			       val | MIPIO_RST_CTRL);
+		intel_de_rmw(dev_priv, BXT_P_CR_GT_DISP_PWRON, 0, MIPIO_RST_CTRL);
 
 		/* Power up DSI regulator */
 		intel_de_write(dev_priv, BXT_P_DSI_REGULATOR_CFG, STAP_SELECT);
@@ -820,12 +771,9 @@ static void intel_dsi_pre_enable(struct intel_atomic_state *state,
 	}
 
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
-		u32 val;
-
 		/* Disable DPOunit clock gating, can stall pipe */
-		val = intel_de_read(dev_priv, DSPCLK_GATE_D(dev_priv));
-		val |= DPOUNIT_CLOCK_GATE_DISABLE;
-		intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), val);
+		intel_de_rmw(dev_priv, DSPCLK_GATE_D(dev_priv),
+			     0, DPOUNIT_CLOCK_GATE_DISABLE);
 	}
 
 	if (!IS_GEMINILAKE(dev_priv))
@@ -949,7 +897,6 @@ static void intel_dsi_post_disable(struct intel_atomic_state *state,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
-	u32 val;
 
 	drm_dbg_kms(&dev_priv->drm, "\n");
 
@@ -987,21 +934,16 @@ static void intel_dsi_post_disable(struct intel_atomic_state *state,
 			       HS_IO_CTRL_SELECT);
 
 		/* Add MIPI IO reset programming for modeset */
-		val = intel_de_read(dev_priv, BXT_P_CR_GT_DISP_PWRON);
-		intel_de_write(dev_priv, BXT_P_CR_GT_DISP_PWRON,
-			       val & ~MIPIO_RST_CTRL);
+		intel_de_rmw(dev_priv, BXT_P_CR_GT_DISP_PWRON, MIPIO_RST_CTRL, 0);
 	}
 
 	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
 		bxt_dsi_pll_disable(encoder);
 	} else {
-		u32 val;
-
 		vlv_dsi_pll_disable(encoder);
 
-		val = intel_de_read(dev_priv, DSPCLK_GATE_D(dev_priv));
-		val &= ~DPOUNIT_CLOCK_GATE_DISABLE;
-		intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), val);
+		intel_de_rmw(dev_priv, DSPCLK_GATE_D(dev_priv),
+			     DPOUNIT_CLOCK_GATE_DISABLE, 0);
 	}
 
 	/* Assert reset */
@@ -1058,7 +1000,7 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 		 */
 		if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 		    port == PORT_C)
-			enabled = intel_de_read(dev_priv, PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
+			enabled = intel_de_read(dev_priv, TRANSCONF(PIPE_B)) & TRANSCONF_ENABLE;
 
 		/* Try command mode if video mode not enabled */
 		if (!enabled) {
@@ -1130,7 +1072,7 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
 	bpp = mipi_dsi_pixel_format_to_bpp(
 			pixel_format_from_register_bits(fmt));
 
-	pipe_config->pipe_bpp = bdw_get_pipemisc_bpp(crtc);
+	pipe_config->pipe_bpp = bdw_get_pipe_misc_bpp(crtc);
 
 	/* Enable Frame time stamo based scanline reporting */
 	pipe_config->mode_flags |=
@@ -1432,11 +1374,8 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 		} else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
 			enum pipe pipe = crtc->pipe;
 
-			tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
-			tmp &= ~BXT_PIPE_SELECT_MASK;
-
-			tmp |= BXT_PIPE_SELECT(pipe);
-			intel_de_write(dev_priv, MIPI_CTRL(port), tmp);
+			intel_de_rmw(dev_priv, MIPI_CTRL(port),
+				     BXT_PIPE_SELECT_MASK, BXT_PIPE_SELECT(pipe));
 		}
 
 		/* XXX: why here, why like this? handling in irq handler?! */
@@ -1605,7 +1544,6 @@ static void intel_dsi_unprepare(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
-	u32 val;
 
 	if (IS_GEMINILAKE(dev_priv))
 		return;
@@ -1620,9 +1558,7 @@ static void intel_dsi_unprepare(struct intel_encoder *encoder)
 			vlv_dsi_reset_clocks(encoder, port);
 		intel_de_write(dev_priv, MIPI_EOT_DISABLE(port), CLOCKSTOP);
 
-		val = intel_de_read(dev_priv, MIPI_DSI_FUNC_PRG(port));
-		val &= ~VID_MODE_FORMAT_MASK;
-		intel_de_write(dev_priv, MIPI_DSI_FUNC_PRG(port), val);
+		intel_de_rmw(dev_priv, MIPI_DSI_FUNC_PRG(port), VID_MODE_FORMAT_MASK, 0);
 
 		intel_de_write(dev_priv, MIPI_DEVICE_READY(port), 0x1);
 	}
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
index af7402127cd9..b697badbbe71 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
@@ -302,13 +302,10 @@ bool bxt_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
 void bxt_dsi_pll_disable(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	u32 val;
 
 	drm_dbg_kms(&dev_priv->drm, "\n");
 
-	val = intel_de_read(dev_priv, BXT_DSI_PLL_ENABLE);
-	val &= ~BXT_DSI_PLL_DO_ENABLE;
-	intel_de_write(dev_priv, BXT_DSI_PLL_ENABLE, val);
+	intel_de_rmw(dev_priv, BXT_DSI_PLL_ENABLE, BXT_DSI_PLL_DO_ENABLE, 0);
 
 	/*
 	 * PLL lock should deassert within 200us.
@@ -542,7 +539,6 @@ void bxt_dsi_pll_enable(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
-	u32 val;
 
 	drm_dbg_kms(&dev_priv->drm, "\n");
 
@@ -559,9 +555,7 @@ void bxt_dsi_pll_enable(struct intel_encoder *encoder,
 	}
 
 	/* Enable DSI PLL */
-	val = intel_de_read(dev_priv, BXT_DSI_PLL_ENABLE);
-	val |= BXT_DSI_PLL_DO_ENABLE;
-	intel_de_write(dev_priv, BXT_DSI_PLL_ENABLE, val);
+	intel_de_rmw(dev_priv, BXT_DSI_PLL_ENABLE, 0, BXT_DSI_PLL_DO_ENABLE);
 
 	/* Timeout and fail if PLL not locked */
 	if (intel_de_wait_for_set(dev_priv, BXT_DSI_PLL_ENABLE,
@@ -589,13 +583,9 @@ void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 		tmp &= ~(BXT_MIPI_RX_ESCLK_LOWER_FIXDIV_MASK(port));
 		intel_de_write(dev_priv, BXT_MIPI_CLOCK_CTL, tmp);
 	} else {
-		tmp = intel_de_read(dev_priv, MIPIO_TXESC_CLK_DIV1);
-		tmp &= ~GLK_TX_ESC_CLK_DIV1_MASK;
-		intel_de_write(dev_priv, MIPIO_TXESC_CLK_DIV1, tmp);
+		intel_de_rmw(dev_priv, MIPIO_TXESC_CLK_DIV1, GLK_TX_ESC_CLK_DIV1_MASK, 0);
 
-		tmp = intel_de_read(dev_priv, MIPIO_TXESC_CLK_DIV2);
-		tmp &= ~GLK_TX_ESC_CLK_DIV2_MASK;
-		intel_de_write(dev_priv, MIPIO_TXESC_CLK_DIV2, tmp);
+		intel_de_rmw(dev_priv, MIPIO_TXESC_CLK_DIV2, GLK_TX_ESC_CLK_DIV2_MASK, 0);
 	}
 	intel_de_write(dev_priv, MIPI_EOT_DISABLE(port), CLOCKSTOP);
 }
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 6d639ca24dfb..5402a7bbcb1d 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -364,7 +364,7 @@ static int set_proto_ctx_vm(struct drm_i915_file_private *fpriv,
 			    struct i915_gem_proto_context *pc,
 			    const struct drm_i915_gem_context_param *args)
 {
-	struct drm_i915_private *i915 = fpriv->dev_priv;
+	struct drm_i915_private *i915 = fpriv->i915;
 	struct i915_address_space *vm;
 
 	if (args->size)
@@ -733,7 +733,7 @@ static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv,
 			         struct i915_gem_proto_context *pc,
 			         const struct drm_i915_gem_context_param *args)
 {
-	struct drm_i915_private *i915 = fpriv->dev_priv;
+	struct drm_i915_private *i915 = fpriv->i915;
 	struct set_proto_ctx_engines set = { .i915 = i915 };
 	struct i915_context_param_engines __user *user =
 		u64_to_user_ptr(args->value);
@@ -813,7 +813,7 @@ static int set_proto_ctx_sseu(struct drm_i915_file_private *fpriv,
 			      struct i915_gem_proto_context *pc,
 			      struct drm_i915_gem_context_param *args)
 {
-	struct drm_i915_private *i915 = fpriv->dev_priv;
+	struct drm_i915_private *i915 = fpriv->i915;
 	struct drm_i915_gem_context_param_sseu user_sseu;
 	struct intel_sseu *sseu;
 	int ret;
@@ -913,7 +913,7 @@ static int set_proto_ctx_param(struct drm_i915_file_private *fpriv,
 		break;
 
 	case I915_CONTEXT_PARAM_PRIORITY:
-		ret = validate_priority(fpriv->dev_priv, args);
+		ret = validate_priority(fpriv->i915, args);
 		if (!ret)
 			pc->sched.priority = args->value;
 		break;
@@ -934,12 +934,12 @@ static int set_proto_ctx_param(struct drm_i915_file_private *fpriv,
 		if (args->size)
 			ret = -EINVAL;
 		else
-			ret = proto_context_set_persistence(fpriv->dev_priv, pc,
+			ret = proto_context_set_persistence(fpriv->i915, pc,
 							    args->value);
 		break;
 
 	case I915_CONTEXT_PARAM_PROTECTED_CONTENT:
-		ret = proto_context_set_protected(fpriv->dev_priv, pc,
+		ret = proto_context_set_protected(fpriv->i915, pc,
 						  args->value);
 		break;
 
@@ -1770,7 +1770,7 @@ void i915_gem_context_close(struct drm_file *file)
 	unsigned long idx;
 
 	xa_for_each(&file_priv->proto_context_xa, idx, pc)
-		proto_context_close(file_priv->dev_priv, pc);
+		proto_context_close(file_priv->i915, pc);
 	xa_destroy(&file_priv->proto_context_xa);
 	mutex_destroy(&file_priv->proto_context_lock);
 
@@ -2206,7 +2206,7 @@ finalize_create_context_locked(struct drm_i915_file_private *file_priv,
 
 	lockdep_assert_held(&file_priv->proto_context_lock);
 
-	ctx = i915_gem_create_context(file_priv->dev_priv, pc);
+	ctx = i915_gem_create_context(file_priv->i915, pc);
 	if (IS_ERR(ctx))
 		return ctx;
 
@@ -2223,7 +2223,7 @@ finalize_create_context_locked(struct drm_i915_file_private *file_priv,
 
 	old = xa_erase(&file_priv->proto_context_xa, id);
 	GEM_BUG_ON(old != pc);
-	proto_context_close(file_priv->dev_priv, pc);
+	proto_context_close(file_priv->i915, pc);
 
 	return ctx;
 }
@@ -2352,7 +2352,7 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
 	GEM_WARN_ON(ctx && pc);
 
 	if (pc)
-		proto_context_close(file_priv->dev_priv, pc);
+		proto_context_close(file_priv->i915, pc);
 
 	if (ctx)
 		context_close(ctx);
@@ -2505,7 +2505,7 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 			 * GEM_CONTEXT_CREATE starting with graphics
 			 * version 13.
 			 */
-			WARN_ON(GRAPHICS_VER(file_priv->dev_priv) > 12);
+			WARN_ON(GRAPHICS_VER(file_priv->i915) > 12);
 			ret = set_proto_ctx_param(file_priv, pc, args);
 		} else {
 			ret = -ENOENT;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_create.c b/drivers/gpu/drm/i915/gem/i915_gem_create.c
index e76c9703680e..bfe1dbda4cb7 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_create.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_create.c
@@ -144,7 +144,8 @@ object_free:
 }
 
 /**
- * Creates a new object using the same path as DRM_I915_GEM_CREATE_EXT
+ * __i915_gem_object_create_user - Creates a new object using the same path as
+ *                                 DRM_I915_GEM_CREATE_EXT
  * @i915: i915 private
  * @size: size of the buffer, in bytes
  * @placements: possible placement regions, in priority order
@@ -215,7 +216,7 @@ i915_gem_dumb_create(struct drm_file *file,
 }
 
 /**
- * Creates a new mm object and returns a handle to it.
+ * i915_gem_create_ioctl - Creates a new mm object and returns a handle to it.
  * @dev: drm device pointer
  * @data: ioctl data blob
  * @file: drm file pointer
@@ -399,7 +400,7 @@ static const i915_user_extension_fn create_extensions[] = {
 };
 
 /**
- * Creates a new mm object and returns a handle to it.
+ * i915_gem_create_ext_ioctl - Creates a new mm object and returns a handle to it.
  * @dev: drm device pointer
  * @data: ioctl data blob
  * @file: drm file pointer
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
index 497de40b8e68..d2d5a24301b2 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
@@ -116,7 +116,8 @@ void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj)
 }
 
 /**
- * Moves a single object to the WC read, and possibly write domain.
+ * i915_gem_object_set_to_wc_domain - Moves a single object to the WC read, and
+ *                                    possibly write domain.
  * @obj: object to act on
  * @write: ask for write access or read only
  *
@@ -177,7 +178,8 @@ i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
 }
 
 /**
- * Moves a single object to the GTT read, and possibly write domain.
+ * i915_gem_object_set_to_gtt_domain - Moves a single object to the GTT read,
+ *                                     and possibly write domain.
  * @obj: object to act on
  * @write: ask for write access or read only
  *
@@ -246,7 +248,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 }
 
 /**
- * Changes the cache-level of an object across all VMA.
+ * i915_gem_object_set_cache_level - Changes the cache-level of an object across all VMA.
  * @obj: object to act on
  * @cache_level: new cache level to set for the object
  *
@@ -467,7 +469,8 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 }
 
 /**
- * Moves a single object to the CPU read, and possibly write domain.
+ * i915_gem_object_set_to_cpu_domain - Moves a single object to the CPU read,
+ *                                     and possibly write domain.
  * @obj: object to act on
  * @write: requesting write or read-only access
  *
@@ -511,7 +514,8 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 }
 
 /**
- * Called when user space prepares to use an object with the CPU, either
+ * i915_gem_set_domain_ioctl - Called when user space prepares to use an
+ *                             object with the CPU, either
  * through the mmap ioctl's mapping or a GTT mapping.
  * @dev: drm device
  * @data: ioctl data blob
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index 9dce2957b4e5..3aeede6aee4d 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -2449,11 +2449,6 @@ static int eb_submit(struct i915_execbuffer *eb)
 	return err;
 }
 
-static int num_vcs_engines(struct drm_i915_private *i915)
-{
-	return hweight_long(VDBOX_MASK(to_gt(i915)));
-}
-
 /*
  * Find one BSD ring to dispatch the corresponding BSD command.
  * The engine index is returned.
@@ -2467,7 +2462,7 @@ gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
 	/* Check whether the file_priv has already selected one ring. */
 	if ((int)file_priv->bsd_engine < 0)
 		file_priv->bsd_engine =
-			get_random_u32_below(num_vcs_engines(dev_priv));
+			get_random_u32_below(dev_priv->engine_uabi_class_count[I915_ENGINE_CLASS_VIDEO]);
 
 	return file_priv->bsd_engine;
 }
@@ -2655,7 +2650,8 @@ eb_select_legacy_ring(struct i915_execbuffer *eb)
 		return -1;
 	}
 
-	if (user_ring_id == I915_EXEC_BSD && num_vcs_engines(i915) > 1) {
+	if (user_ring_id == I915_EXEC_BSD &&
+	    i915->engine_uabi_class_count[I915_ENGINE_CLASS_VIDEO] > 1) {
 		unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
 
 		if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
index e6d4efde4fc5..4666bb82f312 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
@@ -875,7 +875,7 @@ int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj,
 	return ret < 0 ? ret : 0;
 }
 
-/**
+/*
  * i915_gem_object_has_unknown_state - Return true if the object backing pages are
  * in an unknown_state. This means that userspace must NEVER be allowed to touch
  * the pages, with either the GPU or CPU.
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
index 90a967374b1a..8ac376c24aa2 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
@@ -890,8 +890,9 @@ i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type,
 	if (HAS_LMEMBAR_SMEM_STOLEN(i915)) {
 		/*
 		 * MTL dsm size is in GGC register.
-		 * Also MTL uses offset to DSMBASE in ptes, so i915
-		 * uses dsm_base = 0 to setup stolen region.
+		 * Also MTL uses offset to GSMBASE in ptes, so i915
+		 * uses dsm_base = 8MBs to setup stolen region, since
+		 * DSMBASE = GSMBASE + 8MB.
 		 */
 		ret = mtl_get_gms_size(uncore);
 		if (ret < 0) {
@@ -899,27 +900,25 @@ i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type,
 			return ERR_PTR(ret);
 		}
 
-		dsm_base = 0;
+		dsm_base = SZ_8M;
 		dsm_size = (resource_size_t)(ret * SZ_1M);
 
 		GEM_BUG_ON(pci_resource_len(pdev, GEN12_LMEM_BAR) != SZ_256M);
-		GEM_BUG_ON((dsm_size + SZ_8M) > lmem_size);
+		GEM_BUG_ON((dsm_base + dsm_size) > lmem_size);
 	} else {
 		/* Use DSM base address instead for stolen memory */
 		dsm_base = intel_uncore_read64(uncore, GEN12_DSMBASE) & GEN12_BDSM_MASK;
 		if (WARN_ON(lmem_size < dsm_base))
 			return ERR_PTR(-ENODEV);
-		dsm_size = lmem_size - dsm_base;
+		dsm_size = ALIGN_DOWN(lmem_size - dsm_base, SZ_1M);
 	}
 
-	io_size = dsm_size;
-	if (HAS_LMEMBAR_SMEM_STOLEN(i915)) {
-		io_start = pci_resource_start(pdev, GEN12_LMEM_BAR) + SZ_8M;
-	} else if (pci_resource_len(pdev, GEN12_LMEM_BAR) < lmem_size) {
+	if (pci_resource_len(pdev, GEN12_LMEM_BAR) < lmem_size) {
 		io_start = 0;
 		io_size = 0;
 	} else {
 		io_start = pci_resource_start(pdev, GEN12_LMEM_BAR) + dsm_base;
+		io_size = dsm_size;
 	}
 
 	min_page_size = HAS_64K_PAGES(i915) ? I915_GTT_PAGE_SIZE_64K :
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
index 4758f21c91e1..9227f8146a58 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
@@ -472,7 +472,7 @@ static int i915_ttm_shrink(struct drm_i915_gem_object *obj, unsigned int flags)
 	struct ttm_placement place = {};
 	int ret;
 
-	if (!bo->ttm || bo->resource->mem_type != TTM_PL_SYSTEM)
+	if (!bo->ttm || i915_ttm_cpu_maps_iomem(bo->resource))
 		return 0;
 
 	GEM_BUG_ON(!i915_tt->is_shmem);
@@ -511,7 +511,13 @@ static void i915_ttm_delete_mem_notify(struct ttm_buffer_object *bo)
 {
 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
 
-	if (bo->resource && !i915_ttm_is_ghost_object(bo)) {
+	/*
+	 * This gets called twice by ttm, so long as we have a ttm resource or
+	 * ttm_tt then we can still safely call this. Due to pipeline-gutting,
+	 * we maybe have NULL bo->resource, but in that case we should always
+	 * have a ttm alive (like if the pages are swapped out).
+	 */
+	if ((bo->resource || bo->ttm) && !i915_ttm_is_ghost_object(bo)) {
 		__i915_gem_object_pages_fini(obj);
 		i915_ttm_free_cached_io_rsgt(obj);
 	}
@@ -1268,7 +1274,7 @@ void i915_ttm_bo_destroy(struct ttm_buffer_object *bo)
 	}
 }
 
-/**
+/*
  * __i915_gem_ttm_object_init - Initialize a ttm-backed i915 gem object
  * @mem: The initial memory region for the object.
  * @obj: The gem object.
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.h b/drivers/gpu/drm/i915/gem/i915_gem_ttm.h
index 2a94a99ef76b..f8f6bed1b297 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.h
@@ -98,7 +98,7 @@ static inline bool i915_ttm_gtt_binds_lmem(struct ttm_resource *mem)
 static inline bool i915_ttm_cpu_maps_iomem(struct ttm_resource *mem)
 {
 	/* Once / if we support GGTT, this is also false for cached ttm_tts */
-	return mem->mem_type != I915_PL_SYSTEM;
+	return mem && mem->mem_type != I915_PL_SYSTEM;
 }
 
 bool i915_ttm_resource_mappable(struct ttm_resource *res);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c
index 76dd9e5e1a8b..dd188dfcc423 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c
@@ -253,6 +253,7 @@ static struct dma_fence *i915_ttm_accel_move(struct ttm_buffer_object *bo,
  * @_src_iter: Storage space for the source kmap iterator.
  * @dst_iter: Pointer to the destination kmap iterator.
  * @src_iter: Pointer to the source kmap iterator.
+ * @num_pages: Number of pages
  * @clear: Whether to clear instead of copy.
  * @src_rsgt: Refcounted scatter-gather list of source memory.
  * @dst_rsgt: Refcounted scatter-gather list of destination memory.
@@ -557,6 +558,8 @@ out:
  * i915_ttm_move - The TTM move callback used by i915.
  * @bo: The buffer object.
  * @evict: Whether this is an eviction.
+ * @ctx: Pointer to a struct ttm_operation_ctx indicating how the waits should be
+ *       performed if waiting
  * @dst_mem: The destination ttm resource.
  * @hop: If we need multihop, what temporary memory type to move to.
  *
@@ -711,6 +714,10 @@ int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst,
 
 	assert_object_held(dst);
 	assert_object_held(src);
+
+	if (GEM_WARN_ON(!src_bo->resource || !dst_bo->resource))
+		return -EINVAL;
+
 	i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
 
 	ret = dma_resv_reserve_fences(src_bo->base.resv, 1);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c
index 7e67742bc65e..ad649523d5e0 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c
@@ -53,7 +53,7 @@ static int i915_ttm_backup(struct i915_gem_apply_to_region *apply,
 	unsigned int flags;
 	int err = 0;
 
-	if (bo->resource->mem_type == I915_PL_SYSTEM || obj->ttm.backup)
+	if (!i915_ttm_cpu_maps_iomem(bo->resource) || obj->ttm.backup)
 		return 0;
 
 	if (pm_apply->allow_gpu && i915_gem_object_evictable(obj))
@@ -144,8 +144,7 @@ void i915_ttm_recover_region(struct intel_memory_region *mr)
 /**
  * i915_ttm_backup_region - Back up all objects of a region to smem.
  * @mr: The memory region
- * @allow_gpu: Whether to allow the gpu blitter for this backup.
- * @backup_pinned: Backup also pinned objects.
+ * @flags: TTM backup flags
  *
  * Loops over all objects of a region and either evicts them if they are
  * evictable or backs them up using a backup object if they are pinned.
@@ -187,7 +186,10 @@ static int i915_ttm_restore(struct i915_gem_apply_to_region *apply,
 		return err;
 
 	/* Content may have been swapped. */
-	err = ttm_tt_populate(backup_bo->bdev, backup_bo->ttm, &ctx);
+	if (!backup_bo->resource)
+		err = ttm_bo_validate(backup_bo, i915_ttm_sys_placement(), &ctx);
+	if (!err)
+		err = ttm_tt_populate(backup_bo->bdev, backup_bo->ttm, &ctx);
 	if (!err) {
 		err = i915_gem_obj_copy_ttm(obj, backup, pm_apply->allow_gpu,
 					    false);
@@ -209,7 +211,7 @@ static int i915_ttm_restore(struct i915_gem_apply_to_region *apply,
 /**
  * i915_ttm_restore_region - Restore backed-up objects of a region from smem.
  * @mr: The memory region
- * @allow_gpu: Whether to allow the gpu blitter to recover.
+ * @flags: TTM backup flags
  *
  * Loops over all objects of a region and if they are backed-up, restores
  * them from smem.
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_wait.c b/drivers/gpu/drm/i915/gem/i915_gem_wait.c
index e6e01c2a74a6..4a33ad2d122b 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_wait.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_wait.c
@@ -161,7 +161,7 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 }
 
 /**
- * Waits for rendering to the object to be completed
+ * i915_gem_object_wait - Waits for rendering to the object to be completed
  * @obj: i915 gem object
  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
  * @timeout: how long to wait
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
index 3bb1f7f0110e..ff81af4c8202 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
@@ -108,31 +108,30 @@ struct tiled_blits {
 	u32 height;
 };
 
-static bool supports_x_tiling(const struct drm_i915_private *i915)
+static bool fastblit_supports_x_tiling(const struct drm_i915_private *i915)
 {
 	int gen = GRAPHICS_VER(i915);
 
+	/* XY_FAST_COPY_BLT does not exist on pre-gen9 platforms */
+	drm_WARN_ON(&i915->drm, gen < 9);
+
 	if (gen < 12)
 		return true;
 
-	if (!HAS_LMEM(i915) || IS_DG1(i915))
+	if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
 		return false;
 
-	return true;
+	return HAS_DISPLAY(i915);
 }
 
 static bool fast_blit_ok(const struct blit_buffer *buf)
 {
-	int gen = GRAPHICS_VER(buf->vma->vm->i915);
-
-	if (gen < 9)
+	/* XY_FAST_COPY_BLT does not exist on pre-gen9 platforms */
+	if (GRAPHICS_VER(buf->vma->vm->i915) < 9)
 		return false;
 
-	if (gen < 12)
-		return true;
-
 	/* filter out platforms with unsupported X-tile support in fastblit */
-	if (buf->tiling == CLIENT_TILING_X && !supports_x_tiling(buf->vma->vm->i915))
+	if (buf->tiling == CLIENT_TILING_X && !fastblit_supports_x_tiling(buf->vma->vm->i915))
 		return false;
 
 	return true;
diff --git a/drivers/gpu/drm/i915/gt/intel_context.h b/drivers/gpu/drm/i915/gt/intel_context.h
index 0a8d553da3f4..48f888c3da08 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.h
+++ b/drivers/gpu/drm/i915/gt/intel_context.h
@@ -14,6 +14,7 @@
 #include "i915_drv.h"
 #include "intel_context_types.h"
 #include "intel_engine_types.h"
+#include "intel_gt_pm.h"
 #include "intel_ring_types.h"
 #include "intel_timeline_types.h"
 #include "i915_trace.h"
@@ -207,8 +208,11 @@ void intel_context_exit_engine(struct intel_context *ce);
 static inline void intel_context_enter(struct intel_context *ce)
 {
 	lockdep_assert_held(&ce->timeline->mutex);
-	if (!ce->active_count++)
-		ce->ops->enter(ce);
+	if (ce->active_count++)
+		return;
+
+	ce->ops->enter(ce);
+	intel_gt_pm_get(ce->vm->gt);
 }
 
 static inline void intel_context_mark_active(struct intel_context *ce)
@@ -222,8 +226,11 @@ static inline void intel_context_exit(struct intel_context *ce)
 {
 	lockdep_assert_held(&ce->timeline->mutex);
 	GEM_BUG_ON(!ce->active_count);
-	if (!--ce->active_count)
-		ce->ops->exit(ce);
+	if (--ce->active_count)
+		return;
+
+	intel_gt_pm_put_async(ce->vm->gt);
+	ce->ops->exit(ce);
 }
 
 static inline struct intel_context *intel_context_get(struct intel_context *ce)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index d4e29da74612..5c6c9a6d469c 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -9,6 +9,7 @@
 
 #include "gem/i915_gem_context.h"
 #include "gem/i915_gem_internal.h"
+#include "gt/intel_gt_print.h"
 #include "gt/intel_gt_regs.h"
 
 #include "i915_cmd_parser.h"
@@ -1143,12 +1144,130 @@ err_put:
 	return ret;
 }
 
+static int intel_engine_init_tlb_invalidation(struct intel_engine_cs *engine)
+{
+	static const union intel_engine_tlb_inv_reg gen8_regs[] = {
+		[RENDER_CLASS].reg		= GEN8_RTCR,
+		[VIDEO_DECODE_CLASS].reg	= GEN8_M1TCR, /* , GEN8_M2TCR */
+		[VIDEO_ENHANCEMENT_CLASS].reg	= GEN8_VTCR,
+		[COPY_ENGINE_CLASS].reg		= GEN8_BTCR,
+	};
+	static const union intel_engine_tlb_inv_reg gen12_regs[] = {
+		[RENDER_CLASS].reg		= GEN12_GFX_TLB_INV_CR,
+		[VIDEO_DECODE_CLASS].reg	= GEN12_VD_TLB_INV_CR,
+		[VIDEO_ENHANCEMENT_CLASS].reg	= GEN12_VE_TLB_INV_CR,
+		[COPY_ENGINE_CLASS].reg		= GEN12_BLT_TLB_INV_CR,
+		[COMPUTE_CLASS].reg		= GEN12_COMPCTX_TLB_INV_CR,
+	};
+	static const union intel_engine_tlb_inv_reg xehp_regs[] = {
+		[RENDER_CLASS].mcr_reg		  = XEHP_GFX_TLB_INV_CR,
+		[VIDEO_DECODE_CLASS].mcr_reg	  = XEHP_VD_TLB_INV_CR,
+		[VIDEO_ENHANCEMENT_CLASS].mcr_reg = XEHP_VE_TLB_INV_CR,
+		[COPY_ENGINE_CLASS].mcr_reg	  = XEHP_BLT_TLB_INV_CR,
+		[COMPUTE_CLASS].mcr_reg		  = XEHP_COMPCTX_TLB_INV_CR,
+	};
+	static const union intel_engine_tlb_inv_reg xelpmp_regs[] = {
+		[VIDEO_DECODE_CLASS].reg	  = GEN12_VD_TLB_INV_CR,
+		[VIDEO_ENHANCEMENT_CLASS].reg     = GEN12_VE_TLB_INV_CR,
+		[OTHER_CLASS].reg		  = XELPMP_GSC_TLB_INV_CR,
+	};
+	struct drm_i915_private *i915 = engine->i915;
+	const unsigned int instance = engine->instance;
+	const unsigned int class = engine->class;
+	const union intel_engine_tlb_inv_reg *regs;
+	union intel_engine_tlb_inv_reg reg;
+	unsigned int num = 0;
+	u32 val;
+
+	/*
+	 * New platforms should not be added with catch-all-newer (>=)
+	 * condition so that any later platform added triggers the below warning
+	 * and in turn mandates a human cross-check of whether the invalidation
+	 * flows have compatible semantics.
+	 *
+	 * For instance with the 11.00 -> 12.00 transition three out of five
+	 * respective engine registers were moved to masked type. Then after the
+	 * 12.00 -> 12.50 transition multi cast handling is required too.
+	 */
+
+	if (engine->gt->type == GT_MEDIA) {
+		if (MEDIA_VER_FULL(i915) == IP_VER(13, 0)) {
+			regs = xelpmp_regs;
+			num = ARRAY_SIZE(xelpmp_regs);
+		}
+	} else {
+		if (GRAPHICS_VER_FULL(i915) == IP_VER(12, 71) ||
+		    GRAPHICS_VER_FULL(i915) == IP_VER(12, 70) ||
+		    GRAPHICS_VER_FULL(i915) == IP_VER(12, 50) ||
+		    GRAPHICS_VER_FULL(i915) == IP_VER(12, 55)) {
+			regs = xehp_regs;
+			num = ARRAY_SIZE(xehp_regs);
+		} else if (GRAPHICS_VER_FULL(i915) == IP_VER(12, 0) ||
+			   GRAPHICS_VER_FULL(i915) == IP_VER(12, 10)) {
+			regs = gen12_regs;
+			num = ARRAY_SIZE(gen12_regs);
+		} else if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) <= 11) {
+			regs = gen8_regs;
+			num = ARRAY_SIZE(gen8_regs);
+		} else if (GRAPHICS_VER(i915) < 8) {
+			return 0;
+		}
+	}
+
+	if (gt_WARN_ONCE(engine->gt, !num,
+			 "Platform does not implement TLB invalidation!"))
+		return -ENODEV;
+
+	if (gt_WARN_ON_ONCE(engine->gt,
+			    class >= num ||
+			    (!regs[class].reg.reg &&
+			     !regs[class].mcr_reg.reg)))
+		return -ERANGE;
+
+	reg = regs[class];
+
+	if (regs == xelpmp_regs && class == OTHER_CLASS) {
+		/*
+		 * There's only a single GSC instance, but it uses register bit
+		 * 1 instead of either 0 or OTHER_GSC_INSTANCE.
+		 */
+		GEM_WARN_ON(instance != OTHER_GSC_INSTANCE);
+		val = 1;
+	} else if (regs == gen8_regs && class == VIDEO_DECODE_CLASS && instance == 1) {
+		reg.reg = GEN8_M2TCR;
+		val = 0;
+	} else {
+		val = instance;
+	}
+
+	val = BIT(val);
+
+	engine->tlb_inv.mcr = regs == xehp_regs;
+	engine->tlb_inv.reg = reg;
+	engine->tlb_inv.done = val;
+
+	if (GRAPHICS_VER(i915) >= 12 &&
+	    (engine->class == VIDEO_DECODE_CLASS ||
+	     engine->class == VIDEO_ENHANCEMENT_CLASS ||
+	     engine->class == COMPUTE_CLASS ||
+	     engine->class == OTHER_CLASS))
+		engine->tlb_inv.request = _MASKED_BIT_ENABLE(val);
+	else
+		engine->tlb_inv.request = val;
+
+	return 0;
+}
+
 static int engine_setup_common(struct intel_engine_cs *engine)
 {
 	int err;
 
 	init_llist_head(&engine->barrier_tasks);
 
+	err = intel_engine_init_tlb_invalidation(engine);
+	if (err)
+		return err;
+
 	err = init_status_page(engine);
 	if (err)
 		return err;
@@ -1309,8 +1428,8 @@ create_kernel_context(struct intel_engine_cs *engine)
 						  &kernel, "kernel_context");
 }
 
-/**
- * intel_engines_init_common - initialize cengine state which might require hw access
+/*
+ * engine_init_common - initialize engine state which might require hw access
  * @engine: Engine to initialize.
  *
  * Initializes @engine@ structure members shared between legacy and execlists
@@ -1939,13 +2058,13 @@ static const char *repr_timer(const struct timer_list *t)
 static void intel_engine_print_registers(struct intel_engine_cs *engine,
 					 struct drm_printer *m)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
+	struct drm_i915_private *i915 = engine->i915;
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	u64 addr;
 
-	if (engine->id == RENDER_CLASS && IS_GRAPHICS_VER(dev_priv, 4, 7))
+	if (engine->id == RENDER_CLASS && IS_GRAPHICS_VER(i915, 4, 7))
 		drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID));
-	if (HAS_EXECLISTS(dev_priv)) {
+	if (HAS_EXECLISTS(i915)) {
 		drm_printf(m, "\tEL_STAT_HI: 0x%08x\n",
 			   ENGINE_READ(engine, RING_EXECLIST_STATUS_HI));
 		drm_printf(m, "\tEL_STAT_LO: 0x%08x\n",
@@ -1966,7 +2085,7 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine,
 			   ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : "");
 	}
 
-	if (GRAPHICS_VER(dev_priv) >= 6) {
+	if (GRAPHICS_VER(i915) >= 6) {
 		drm_printf(m, "\tRING_IMR:   0x%08x\n",
 			   ENGINE_READ(engine, RING_IMR));
 		drm_printf(m, "\tRING_ESR:   0x%08x\n",
@@ -1983,15 +2102,15 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine,
 	addr = intel_engine_get_last_batch_head(engine);
 	drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
 		   upper_32_bits(addr), lower_32_bits(addr));
-	if (GRAPHICS_VER(dev_priv) >= 8)
+	if (GRAPHICS_VER(i915) >= 8)
 		addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW);
-	else if (GRAPHICS_VER(dev_priv) >= 4)
+	else if (GRAPHICS_VER(i915) >= 4)
 		addr = ENGINE_READ(engine, RING_DMA_FADD);
 	else
 		addr = ENGINE_READ(engine, DMA_FADD_I8XX);
 	drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
 		   upper_32_bits(addr), lower_32_bits(addr));
-	if (GRAPHICS_VER(dev_priv) >= 4) {
+	if (GRAPHICS_VER(i915) >= 4) {
 		drm_printf(m, "\tIPEIR: 0x%08x\n",
 			   ENGINE_READ(engine, RING_IPEIR));
 		drm_printf(m, "\tIPEHR: 0x%08x\n",
@@ -2001,7 +2120,7 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine,
 		drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR));
 	}
 
-	if (HAS_EXECLISTS(dev_priv) && !intel_engine_uses_guc(engine)) {
+	if (HAS_EXECLISTS(i915) && !intel_engine_uses_guc(engine)) {
 		struct i915_request * const *port, *rq;
 		const u32 *hws =
 			&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
@@ -2067,7 +2186,7 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine,
 		}
 		rcu_read_unlock();
 		i915_sched_engine_active_unlock_bh(engine->sched_engine);
-	} else if (GRAPHICS_VER(dev_priv) > 6) {
+	} else if (GRAPHICS_VER(i915) > 6) {
 		drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
 			   ENGINE_READ(engine, RING_PP_DIR_BASE));
 		drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
index e971b153fda9..ee531a5c142c 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
@@ -115,6 +115,15 @@ __queue_and_release_pm(struct i915_request *rq,
 	ENGINE_TRACE(engine, "parking\n");
 
 	/*
+	 * Open coded one half of intel_context_enter, which we have to omit
+	 * here (see the large comment below) and because the other part must
+	 * not be called due constructing directly with __i915_request_create
+	 * which increments active count via intel_context_mark_active.
+	 */
+	GEM_BUG_ON(rq->context->active_count != 1);
+	__intel_gt_pm_get(engine->gt);
+
+	/*
 	 * We have to serialise all potential retirement paths with our
 	 * submission, as we don't want to underflow either the
 	 * engine->wakeref.counter or our timeline->active_count.
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 4fd54fb8810f..960291f88fd6 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -53,6 +53,8 @@ struct intel_gt;
 struct intel_ring;
 struct intel_uncore;
 struct intel_breadcrumbs;
+struct intel_engine_cs;
+struct i915_perf_group;
 
 typedef u32 intel_engine_mask_t;
 #define ALL_ENGINES ((intel_engine_mask_t)~0ul)
@@ -341,6 +343,18 @@ struct intel_engine_guc_stats {
 	u64 start_gt_clk;
 };
 
+union intel_engine_tlb_inv_reg {
+	i915_reg_t	reg;
+	i915_mcr_reg_t	mcr_reg;
+};
+
+struct intel_engine_tlb_inv {
+	bool mcr;
+	union intel_engine_tlb_inv_reg reg;
+	u32 request;
+	u32 done;
+};
+
 struct intel_engine_cs {
 	struct drm_i915_private *i915;
 	struct intel_gt *gt;
@@ -372,6 +386,8 @@ struct intel_engine_cs {
 	u32 context_size;
 	u32 mmio_base;
 
+	struct intel_engine_tlb_inv tlb_inv;
+
 	/*
 	 * Some w/a require forcewake to be held (which prevents RC6) while
 	 * a particular engine is active. If so, we set fw_domain to which
@@ -603,6 +619,14 @@ struct intel_engine_cs {
 	} props, defaults;
 
 	I915_SELFTEST_DECLARE(struct fault_attr reset_timeout);
+
+	/*
+	 * The perf group maps to one OA unit which controls one OA buffer. All
+	 * reports corresponding to this engine will be reported to this OA
+	 * buffer. An engine will map to a single OA unit, but a single OA unit
+	 * can generate reports for multiple engines.
+	 */
+	struct i915_perf_group *oa_group;
 };
 
 static inline bool
diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c
index 842e69c7b21e..3c7f1ed92f5b 100644
--- a/drivers/gpu/drm/i915/gt/intel_ggtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c
@@ -291,6 +291,27 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 	ggtt->invalidate(ggtt);
 }
 
+static void gen8_ggtt_clear_range(struct i915_address_space *vm,
+				  u64 start, u64 length)
+{
+	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
+	unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
+	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
+	const gen8_pte_t scratch_pte = vm->scratch[0]->encode;
+	gen8_pte_t __iomem *gtt_base =
+		(gen8_pte_t __iomem *)ggtt->gsm + first_entry;
+	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
+	int i;
+
+	if (WARN(num_entries > max_entries,
+		 "First entry = %d; Num entries = %d (max=%d)\n",
+		 first_entry, num_entries, max_entries))
+		num_entries = max_entries;
+
+	for (i = 0; i < num_entries; i++)
+		gen8_set_pte(&gtt_base[i], scratch_pte);
+}
+
 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
 				  dma_addr_t addr,
 				  u64 offset,
@@ -551,8 +572,12 @@ static int init_ggtt(struct i915_ggtt *ggtt)
 		 * paths, and we trust that 0 will remain reserved. However,
 		 * the only likely reason for failure to insert is a driver
 		 * bug, which we expect to cause other failures...
+		 *
+		 * Since CPU can perform speculative reads on error capture
+		 * (write-combining allows it) add scratch page after error
+		 * capture to avoid DMAR errors.
 		 */
-		ggtt->error_capture.size = I915_GTT_PAGE_SIZE;
+		ggtt->error_capture.size = 2 * I915_GTT_PAGE_SIZE;
 		ggtt->error_capture.color = I915_COLOR_UNEVICTABLE;
 		if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture))
 			drm_mm_insert_node_in_range(&ggtt->vm.mm,
@@ -562,11 +587,15 @@ static int init_ggtt(struct i915_ggtt *ggtt)
 						    0, ggtt->mappable_end,
 						    DRM_MM_INSERT_LOW);
 	}
-	if (drm_mm_node_allocated(&ggtt->error_capture))
+	if (drm_mm_node_allocated(&ggtt->error_capture)) {
+		u64 start = ggtt->error_capture.start;
+		u64 size = ggtt->error_capture.size;
+
+		ggtt->vm.scratch_range(&ggtt->vm, start, size);
 		drm_dbg(&ggtt->vm.i915->drm,
 			"Reserved GGTT:[%llx, %llx] for use by error capture\n",
-			ggtt->error_capture.start,
-			ggtt->error_capture.start + ggtt->error_capture.size);
+			start, start + size);
+	}
 
 	/*
 	 * The upper portion of the GuC address space has a sizeable hole
@@ -919,6 +948,7 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 	ggtt->vm.cleanup = gen6_gmch_remove;
 	ggtt->vm.insert_page = gen8_ggtt_insert_page;
 	ggtt->vm.clear_range = nop_clear_range;
+	ggtt->vm.scratch_range = gen8_ggtt_clear_range;
 
 	ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
 
@@ -1082,6 +1112,7 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt)
 	ggtt->vm.clear_range = nop_clear_range;
 	if (!HAS_FULL_PPGTT(i915))
 		ggtt->vm.clear_range = gen6_ggtt_clear_range;
+	ggtt->vm.scratch_range = gen6_ggtt_clear_range;
 	ggtt->vm.insert_page = gen6_ggtt_insert_page;
 	ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
 	ggtt->vm.cleanup = gen6_gmch_remove;
@@ -1257,6 +1288,10 @@ void i915_ggtt_resume(struct i915_ggtt *ggtt)
 
 	flush = i915_ggtt_resume_vm(&ggtt->vm);
 
+	if (drm_mm_node_allocated(&ggtt->error_capture))
+		ggtt->vm.scratch_range(&ggtt->vm, ggtt->error_capture.start,
+				       ggtt->error_capture.size);
+
 	ggtt->invalidate(ggtt);
 
 	if (flush)
diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_gmch.c b/drivers/gpu/drm/i915/gt/intel_ggtt_gmch.c
index 77c793812eb4..d6a74ae2527b 100644
--- a/drivers/gpu/drm/i915/gt/intel_ggtt_gmch.c
+++ b/drivers/gpu/drm/i915/gt/intel_ggtt_gmch.c
@@ -102,6 +102,7 @@ int intel_ggtt_gmch_probe(struct i915_ggtt *ggtt)
 	ggtt->vm.insert_page = gmch_ggtt_insert_page;
 	ggtt->vm.insert_entries = gmch_ggtt_insert_entries;
 	ggtt->vm.clear_range = gmch_ggtt_clear_range;
+	ggtt->vm.scratch_range = gmch_ggtt_clear_range;
 	ggtt->vm.cleanup = gmch_ggtt_remove;
 
 	ggtt->invalidate = gmch_ggtt_invalidate;
diff --git a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h
index 2af1ae3831df..5d143e2a8db0 100644
--- a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h
+++ b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h
@@ -394,6 +394,7 @@
 #define MI_LOAD_URB_MEM         MI_INSTR(0x2C, 0)
 #define MI_STORE_URB_MEM        MI_INSTR(0x2D, 0)
 #define MI_CONDITIONAL_BATCH_BUFFER_END MI_INSTR(0x36, 0)
+#define  MI_DO_COMPARE		REG_BIT(21)
 
 #define STATE_BASE_ADDRESS \
 	((0x3 << 29) | (0x0 << 27) | (0x1 << 24) | (0x1 << 16))
@@ -439,6 +440,8 @@
 #define GSC_FW_LOAD GSC_INSTR(1, 0, 2)
 #define   HECI1_FW_LIMIT_VALID (1 << 31)
 
+#define GSC_HECI_CMD_PKT GSC_INSTR(0, 0, 6)
+
 /*
  * Used to convert any address to canonical form.
  * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
diff --git a/drivers/gpu/drm/i915/gt/intel_gsc.h b/drivers/gpu/drm/i915/gt/intel_gsc.h
index fcac1775e9c3..7ab3ca0f9f26 100644
--- a/drivers/gpu/drm/i915/gt/intel_gsc.h
+++ b/drivers/gpu/drm/i915/gt/intel_gsc.h
@@ -33,7 +33,7 @@ struct intel_gsc {
 	} intf[INTEL_GSC_NUM_INTERFACES];
 };
 
-void intel_gsc_init(struct intel_gsc *gsc, struct drm_i915_private *dev_priv);
+void intel_gsc_init(struct intel_gsc *gsc, struct drm_i915_private *i915);
 void intel_gsc_fini(struct intel_gsc *gsc);
 void intel_gsc_irq_handler(struct intel_gt *gt, u32 iir);
 
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index 40d357cf8b04..7a008e829d4d 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -28,7 +28,6 @@
 #include "intel_migrate.h"
 #include "intel_mocs.h"
 #include "intel_pci_config.h"
-#include "intel_pm.h"
 #include "intel_rc6.h"
 #include "intel_renderstate.h"
 #include "intel_rps.h"
@@ -785,6 +784,29 @@ void intel_gt_driver_unregister(struct intel_gt *gt)
 	intel_gsc_fini(&gt->gsc);
 
 	/*
+	 * If we unload the driver and wedge before the GSC worker is complete,
+	 * the worker will hit an error on its submission to the GSC engine and
+	 * then exit. This is hard to hit for a user, but it is reproducible
+	 * with skipping selftests. The error is handled gracefully by the
+	 * worker, so there are no functional issues, but we still end up with
+	 * an error message in dmesg, which is something we want to avoid as
+	 * this is a supported scenario. We could modify the worker to better
+	 * handle a wedging occurring during its execution, but that gets
+	 * complicated for a couple of reasons:
+	 * - We do want the error on runtime wedging, because there are
+	 *   implications for subsystems outside of GT (i.e., PXP, HDCP), it's
+	 *   only the error on driver unload that we want to silence.
+	 * - The worker is responsible for multiple submissions (GSC FW load,
+	 *   HuC auth, SW proxy), so all of those will have to be adapted to
+	 *   handle the wedged_on_fini scenario.
+	 * Therefore, it's much simpler to just wait for the worker to be done
+	 * before wedging on driver removal, also considering that the worker
+	 * will likely already be idle in the great majority of non-selftest
+	 * scenarios.
+	 */
+	intel_gsc_uc_flush_work(&gt->uc.gsc);
+
+	/*
 	 * Upon unregistering the device to prevent any new users, cancel
 	 * all in-flight requests so that we can quickly unbind the active
 	 * resources.
@@ -982,35 +1004,6 @@ void intel_gt_info_print(const struct intel_gt_info *info,
 	intel_sseu_dump(&info->sseu, p);
 }
 
-struct reg_and_bit {
-	union {
-		i915_reg_t reg;
-		i915_mcr_reg_t mcr_reg;
-	};
-	u32 bit;
-};
-
-static struct reg_and_bit
-get_reg_and_bit(const struct intel_engine_cs *engine, const bool gen8,
-		const i915_reg_t *regs, const unsigned int num)
-{
-	const unsigned int class = engine->class;
-	struct reg_and_bit rb = { };
-
-	if (gt_WARN_ON_ONCE(engine->gt, class >= num || !regs[class].reg))
-		return rb;
-
-	rb.reg = regs[class];
-	if (gen8 && class == VIDEO_DECODE_CLASS)
-		rb.reg.reg += 4 * engine->instance; /* GEN8_M2TCR */
-	else
-		rb.bit = engine->instance;
-
-	rb.bit = BIT(rb.bit);
-
-	return rb;
-}
-
 /*
  * HW architecture suggest typical invalidation time at 40us,
  * with pessimistic cases up to 100us and a recommendation to
@@ -1024,14 +1017,20 @@ get_reg_and_bit(const struct intel_engine_cs *engine, const bool gen8,
  * but are now considered MCR registers.  Since they exist within a GAM range,
  * the primary instance of the register rolls up the status from each unit.
  */
-static int wait_for_invalidate(struct intel_gt *gt, struct reg_and_bit rb)
+static int wait_for_invalidate(struct intel_engine_cs *engine)
 {
-	if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50))
-		return intel_gt_mcr_wait_for_reg(gt, rb.mcr_reg, rb.bit, 0,
+	if (engine->tlb_inv.mcr)
+		return intel_gt_mcr_wait_for_reg(engine->gt,
+						 engine->tlb_inv.reg.mcr_reg,
+						 engine->tlb_inv.done,
+						 0,
 						 TLB_INVAL_TIMEOUT_US,
 						 TLB_INVAL_TIMEOUT_MS);
 	else
-		return __intel_wait_for_register_fw(gt->uncore, rb.reg, rb.bit, 0,
+		return __intel_wait_for_register_fw(engine->gt->uncore,
+						    engine->tlb_inv.reg.reg,
+						    engine->tlb_inv.done,
+						    0,
 						    TLB_INVAL_TIMEOUT_US,
 						    TLB_INVAL_TIMEOUT_MS,
 						    NULL);
@@ -1039,62 +1038,14 @@ static int wait_for_invalidate(struct intel_gt *gt, struct reg_and_bit rb)
 
 static void mmio_invalidate_full(struct intel_gt *gt)
 {
-	static const i915_reg_t gen8_regs[] = {
-		[RENDER_CLASS]			= GEN8_RTCR,
-		[VIDEO_DECODE_CLASS]		= GEN8_M1TCR, /* , GEN8_M2TCR */
-		[VIDEO_ENHANCEMENT_CLASS]	= GEN8_VTCR,
-		[COPY_ENGINE_CLASS]		= GEN8_BTCR,
-	};
-	static const i915_reg_t gen12_regs[] = {
-		[RENDER_CLASS]			= GEN12_GFX_TLB_INV_CR,
-		[VIDEO_DECODE_CLASS]		= GEN12_VD_TLB_INV_CR,
-		[VIDEO_ENHANCEMENT_CLASS]	= GEN12_VE_TLB_INV_CR,
-		[COPY_ENGINE_CLASS]		= GEN12_BLT_TLB_INV_CR,
-		[COMPUTE_CLASS]			= GEN12_COMPCTX_TLB_INV_CR,
-	};
-	static const i915_mcr_reg_t xehp_regs[] = {
-		[RENDER_CLASS]			= XEHP_GFX_TLB_INV_CR,
-		[VIDEO_DECODE_CLASS]		= XEHP_VD_TLB_INV_CR,
-		[VIDEO_ENHANCEMENT_CLASS]	= XEHP_VE_TLB_INV_CR,
-		[COPY_ENGINE_CLASS]		= XEHP_BLT_TLB_INV_CR,
-		[COMPUTE_CLASS]			= XEHP_COMPCTX_TLB_INV_CR,
-	};
 	struct drm_i915_private *i915 = gt->i915;
 	struct intel_uncore *uncore = gt->uncore;
 	struct intel_engine_cs *engine;
 	intel_engine_mask_t awake, tmp;
 	enum intel_engine_id id;
-	const i915_reg_t *regs;
-	unsigned int num = 0;
 	unsigned long flags;
 
-	/*
-	 * New platforms should not be added with catch-all-newer (>=)
-	 * condition so that any later platform added triggers the below warning
-	 * and in turn mandates a human cross-check of whether the invalidation
-	 * flows have compatible semantics.
-	 *
-	 * For instance with the 11.00 -> 12.00 transition three out of five
-	 * respective engine registers were moved to masked type. Then after the
-	 * 12.00 -> 12.50 transition multi cast handling is required too.
-	 */
-
-	if (GRAPHICS_VER_FULL(i915) == IP_VER(12, 50) ||
-	    GRAPHICS_VER_FULL(i915) == IP_VER(12, 55)) {
-		regs = NULL;
-		num = ARRAY_SIZE(xehp_regs);
-	} else if (GRAPHICS_VER_FULL(i915) == IP_VER(12, 0) ||
-		   GRAPHICS_VER_FULL(i915) == IP_VER(12, 10)) {
-		regs = gen12_regs;
-		num = ARRAY_SIZE(gen12_regs);
-	} else if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) <= 11) {
-		regs = gen8_regs;
-		num = ARRAY_SIZE(gen8_regs);
-	} else if (GRAPHICS_VER(i915) < 8) {
-		return;
-	}
-
-	if (gt_WARN_ONCE(gt, !num, "Platform does not implement TLB invalidation!"))
+	if (GRAPHICS_VER(i915) < 8)
 		return;
 
 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
@@ -1104,33 +1055,18 @@ static void mmio_invalidate_full(struct intel_gt *gt)
 
 	awake = 0;
 	for_each_engine(engine, gt, id) {
-		struct reg_and_bit rb;
-
 		if (!intel_engine_pm_is_awake(engine))
 			continue;
 
-		if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
-			u32 val = BIT(engine->instance);
-
-			if (engine->class == VIDEO_DECODE_CLASS ||
-			    engine->class == VIDEO_ENHANCEMENT_CLASS ||
-			    engine->class == COMPUTE_CLASS)
-				val = _MASKED_BIT_ENABLE(val);
+		if (engine->tlb_inv.mcr)
 			intel_gt_mcr_multicast_write_fw(gt,
-							xehp_regs[engine->class],
-							val);
-		} else {
-			rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num);
-			if (!i915_mmio_reg_offset(rb.reg))
-				continue;
-
-			if (GRAPHICS_VER(i915) == 12 && (engine->class == VIDEO_DECODE_CLASS ||
-			    engine->class == VIDEO_ENHANCEMENT_CLASS ||
-			    engine->class == COMPUTE_CLASS))
-				rb.bit = _MASKED_BIT_ENABLE(rb.bit);
-
-			intel_uncore_write_fw(uncore, rb.reg, rb.bit);
-		}
+							engine->tlb_inv.reg.mcr_reg,
+							engine->tlb_inv.request);
+		else
+			intel_uncore_write_fw(uncore,
+					      engine->tlb_inv.reg.reg,
+					      engine->tlb_inv.request);
+
 		awake |= engine->mask;
 	}
 
@@ -1149,17 +1085,9 @@ static void mmio_invalidate_full(struct intel_gt *gt)
 	intel_gt_mcr_unlock(gt, flags);
 
 	for_each_engine_masked(engine, gt, awake, tmp) {
-		struct reg_and_bit rb;
-
-		if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
-			rb.mcr_reg = xehp_regs[engine->class];
-			rb.bit = BIT(engine->instance);
-		} else {
-			rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num);
-		}
-
-		if (wait_for_invalidate(gt, rb))
-			gt_err_ratelimited(gt, "%s TLB invalidation did not complete in %ums!\n",
+		if (wait_for_invalidate(engine))
+			gt_err_ratelimited(gt,
+					   "%s TLB invalidation did not complete in %ums!\n",
 					   engine->name, TLB_INVAL_TIMEOUT_MS);
 	}
 
@@ -1205,3 +1133,7 @@ unlock:
 		mutex_unlock(&gt->tlb.invalidate_lock);
 	}
 }
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftest_tlb.c"
+#endif
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_debugfs.c b/drivers/gpu/drm/i915/gt/intel_gt_debugfs.c
index 5fc2df01aa0d..4dc23b8d3aa2 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_debugfs.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_debugfs.c
@@ -83,11 +83,13 @@ static void gt_debugfs_register(struct intel_gt *gt, struct dentry *root)
 void intel_gt_debugfs_register(struct intel_gt *gt)
 {
 	struct dentry *root;
+	char gtname[4];
 
 	if (!gt->i915->drm.primary->debugfs_root)
 		return;
 
-	root = debugfs_create_dir("gt", gt->i915->drm.primary->debugfs_root);
+	snprintf(gtname, sizeof(gtname), "gt%u", gt->info.id);
+	root = debugfs_create_dir(gtname, gt->i915->drm.primary->debugfs_root);
 	if (IS_ERR(root))
 		return;
 
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_mcr.c b/drivers/gpu/drm/i915/gt/intel_gt_mcr.c
index 3bb1c701d5ff..0b414eae1683 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_mcr.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_mcr.c
@@ -35,7 +35,7 @@
  * ignored.
  */
 
-#define HAS_MSLICE_STEERING(dev_priv)	(INTEL_INFO(dev_priv)->has_mslice_steering)
+#define HAS_MSLICE_STEERING(i915)	(INTEL_INFO(i915)->has_mslice_steering)
 
 static const char * const intel_steering_types[] = {
 	"L3BANK",
@@ -364,6 +364,7 @@ static u32 rw_with_mcr_steering(struct intel_gt *gt,
  *          function call.
  */
 void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags)
+	__acquires(&gt->mcr_lock)
 {
 	unsigned long __flags;
 	int err = 0;
@@ -410,6 +411,7 @@ void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags)
  * Context: Releases gt->mcr_lock
  */
 void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags)
+	__releases(&gt->mcr_lock)
 {
 	spin_unlock_irqrestore(&gt->mcr_lock, flags);
 
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
index 56b993f6e7dc..e02cb90723ae 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
@@ -17,7 +17,6 @@
 #include "intel_gt_print.h"
 #include "intel_gt_requests.h"
 #include "intel_llc.h"
-#include "intel_pm.h"
 #include "intel_rc6.h"
 #include "intel_rps.h"
 #include "intel_wakeref.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_print.h b/drivers/gpu/drm/i915/gt/intel_gt_print.h
index 5d9da355ce24..55a336a9ff06 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_print.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_print.h
@@ -28,6 +28,9 @@
 #define gt_err_ratelimited(_gt, _fmt, ...) \
 	drm_err_ratelimited(&(_gt)->i915->drm, "GT%u: " _fmt, (_gt)->info.id, ##__VA_ARGS__)
 
+#define gt_notice_ratelimited(_gt, _fmt, ...) \
+	dev_notice_ratelimited((_gt)->i915->drm.dev, "GT%u: " _fmt, (_gt)->info.id, ##__VA_ARGS__)
+
 #define gt_probe_error(_gt, _fmt, ...) \
 	do { \
 		if (i915_error_injected()) \
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_regs.h b/drivers/gpu/drm/i915/gt/intel_gt_regs.h
index be0f6e305c88..fd1f9cd35e9d 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_regs.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_regs.h
@@ -9,8 +9,6 @@
 #include "i915_reg_defs.h"
 #include "display/intel_display_reg_defs.h"	/* VLV_DISPLAY_BASE */
 
-#define MCR_REG(offset)	((const i915_mcr_reg_t){ .reg = (offset) })
-
 /*
  * The perf control registers are technically multicast registers, but the
  * driver never needs to read/write them directly; we only use them to build
@@ -480,6 +478,9 @@
 #define   HDC_FORCE_NON_COHERENT		(1 << 4)
 #define   HDC_BARRIER_PERFORMANCE_DISABLE	(1 << 10)
 
+#define COMMON_SLICE_CHICKEN4			_MMIO(0x7300)
+#define   DISABLE_TDC_LOAD_BALANCING_CALC	REG_BIT(6)
+
 #define GEN8_HDC_CHICKEN1			_MMIO(0x7304)
 
 #define GEN11_COMMON_SLICE_CHICKEN3		_MMIO(0x7304)
@@ -769,9 +770,6 @@
 #define GEN10_DFR_RATIO_EN_AND_CHICKEN		MCR_REG(0x9550)
 #define   DFR_DISABLE				(1 << 9)
 
-#define INF_UNIT_LEVEL_CLKGATE			MCR_REG(0x9560)
-#define   CGPSF_CLKGATE_DIS			(1 << 3)
-
 #define MICRO_BP0_0				_MMIO(0x9800)
 #define MICRO_BP0_2				_MMIO(0x9804)
 #define MICRO_BP0_1				_MMIO(0x9808)
@@ -1093,6 +1091,7 @@
 #define XEHP_BLT_TLB_INV_CR			MCR_REG(0xcee4)
 #define GEN12_COMPCTX_TLB_INV_CR		_MMIO(0xcf04)
 #define XEHP_COMPCTX_TLB_INV_CR			MCR_REG(0xcf04)
+#define XELPMP_GSC_TLB_INV_CR			_MMIO(0xcf04)   /* media GT only */
 
 #define XEHP_MERT_MOD_CTRL			MCR_REG(0xcf28)
 #define RENDER_MOD_CTRL				MCR_REG(0xcf2c)
@@ -1145,6 +1144,8 @@
 #define   ENABLE_SMALLPL			REG_BIT(15)
 #define   SC_DISABLE_POWER_OPTIMIZATION_EBB	REG_BIT(9)
 #define   GEN11_SAMPLER_ENABLE_HEADLESS_MSG	REG_BIT(5)
+#define   MTL_DISABLE_SAMPLER_SC_OOO		REG_BIT(3)
+#define   GEN11_INDIRECT_STATE_BASE_ADDR_OVERRIDE	REG_BIT(0)
 
 #define GEN9_HALF_SLICE_CHICKEN7		MCR_REG(0xe194)
 #define   DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA	REG_BIT(15)
@@ -1156,7 +1157,13 @@
 #define   ENABLE_EU_COUNT_FOR_TDL_FLUSH		REG_BIT(10)
 #define   DISABLE_ECC				REG_BIT(5)
 #define   FLOAT_BLEND_OPTIMIZATION_ENABLE	REG_BIT(4)
+/*
+ * We have both ENABLE and DISABLE defines below using the same bit because the
+ * meaning depends on the target platform. There are no platform prefix for them
+ * because different steppings of DG2 pick one or the other semantics.
+ */
 #define   ENABLE_PREFETCH_INTO_IC		REG_BIT(3)
+#define   DISABLE_PREFETCH_INTO_IC		REG_BIT(3)
 
 #define EU_PERF_CNTL0				PERF_REG(0xe458)
 #define EU_PERF_CNTL4				PERF_REG(0xe45c)
@@ -1171,7 +1178,9 @@
 #define   THREAD_EX_ARB_MODE_RR_AFTER_DEP	REG_FIELD_PREP(THREAD_EX_ARB_MODE, 0x2)
 
 #define HSW_ROW_CHICKEN3			_MMIO(0xe49c)
+#define GEN9_ROW_CHICKEN3			MCR_REG(0xe49c)
 #define   HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE	(1 << 6)
+#define   MTL_DISABLE_FIX_FOR_EOT_FLUSH		REG_BIT(9)
 
 #define GEN8_ROW_CHICKEN			MCR_REG(0xe4f0)
 #define   FLOW_CONTROL_ENABLE			REG_BIT(15)
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_sysfs.c b/drivers/gpu/drm/i915/gt/intel_gt_sysfs.c
index 6629e4c72b6b..33cba406b569 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_sysfs.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_sysfs.c
@@ -72,7 +72,7 @@ static void kobj_gt_release(struct kobject *kobj)
 {
 }
 
-static struct kobj_type kobj_gt_type = {
+static const struct kobj_type kobj_gt_type = {
 	.release = kobj_gt_release,
 	.sysfs_ops = &kobj_sysfs_ops,
 	.default_groups = id_groups,
diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h
index 5a775310d3fc..69ce55f517f5 100644
--- a/drivers/gpu/drm/i915/gt/intel_gtt.h
+++ b/drivers/gpu/drm/i915/gt/intel_gtt.h
@@ -298,6 +298,8 @@ struct i915_address_space {
 				  u64 start, u64 length);
 	void (*clear_range)(struct i915_address_space *vm,
 			    u64 start, u64 length);
+	void (*scratch_range)(struct i915_address_space *vm,
+			      u64 start, u64 length);
 	void (*insert_page)(struct i915_address_space *vm,
 			    dma_addr_t addr,
 			    u64 offset,
diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c
index f4150f61f39c..8f3cd68d14f8 100644
--- a/drivers/gpu/drm/i915/gt/intel_rc6.c
+++ b/drivers/gpu/drm/i915/gt/intel_rc6.c
@@ -420,6 +420,21 @@ static void vlv_rc6_enable(struct intel_rc6 *rc6)
 	    GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
 }
 
+bool intel_check_bios_c6_setup(struct intel_rc6 *rc6)
+{
+	if (!rc6->bios_state_captured) {
+		struct intel_uncore *uncore = rc6_to_uncore(rc6);
+		intel_wakeref_t wakeref;
+
+		with_intel_runtime_pm(uncore->rpm, wakeref)
+			rc6->bios_rc_state = intel_uncore_read(uncore, GEN6_RC_STATE);
+
+		rc6->bios_state_captured = true;
+	}
+
+	return rc6->bios_rc_state & RC_SW_TARGET_STATE_MASK;
+}
+
 static bool bxt_check_bios_rc6_setup(struct intel_rc6 *rc6)
 {
 	struct intel_uncore *uncore = rc6_to_uncore(rc6);
@@ -503,6 +518,13 @@ static bool rc6_supported(struct intel_rc6 *rc6)
 		return false;
 	}
 
+	if (IS_METEORLAKE(gt->i915) &&
+	    !intel_check_bios_c6_setup(rc6)) {
+		drm_notice(&i915->drm,
+			   "C6 disabled by BIOS\n");
+		return false;
+	}
+
 	if (IS_MTL_MEDIA_STEP(gt->i915, STEP_A0, STEP_B0) &&
 	    gt->type == GT_MEDIA) {
 		drm_notice(&i915->drm,
@@ -707,9 +729,14 @@ void intel_rc6_disable(struct intel_rc6 *rc6)
 void intel_rc6_fini(struct intel_rc6 *rc6)
 {
 	struct drm_i915_gem_object *pctx;
+	struct intel_uncore *uncore = rc6_to_uncore(rc6);
 
 	intel_rc6_disable(rc6);
 
+	/* We want the BIOS C6 state preserved across loads for MTL */
+	if (IS_METEORLAKE(rc6_to_i915(rc6)) && rc6->bios_state_captured)
+		set(uncore, GEN6_RC_STATE, rc6->bios_rc_state);
+
 	pctx = fetch_and_zero(&rc6->pctx);
 	if (pctx)
 		i915_gem_object_put(pctx);
diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.h b/drivers/gpu/drm/i915/gt/intel_rc6.h
index 456fa668a276..e137c2c397c2 100644
--- a/drivers/gpu/drm/i915/gt/intel_rc6.h
+++ b/drivers/gpu/drm/i915/gt/intel_rc6.h
@@ -27,4 +27,6 @@ u64 intel_rc6_residency_us(struct intel_rc6 *rc6, enum intel_rc6_res_type id);
 void intel_rc6_print_residency(struct seq_file *m, const char *title,
 			       enum intel_rc6_res_type id);
 
+bool intel_check_bios_c6_setup(struct intel_rc6 *rc6);
+
 #endif /* INTEL_RC6_H */
diff --git a/drivers/gpu/drm/i915/gt/intel_rc6_types.h b/drivers/gpu/drm/i915/gt/intel_rc6_types.h
index fa23c4dce00b..cd4587098162 100644
--- a/drivers/gpu/drm/i915/gt/intel_rc6_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_rc6_types.h
@@ -29,6 +29,7 @@ struct intel_rc6 {
 	u64 cur_residency[INTEL_RC6_RES_MAX];
 
 	u32 ctl_enable;
+	u32 bios_rc_state;
 
 	struct drm_i915_gem_object *pctx;
 
@@ -36,6 +37,7 @@ struct intel_rc6 {
 	bool enabled : 1;
 	bool manual : 1;
 	bool wakeref : 1;
+	bool bios_state_captured : 1;
 };
 
 #endif /* INTEL_RC6_TYPES_H */
diff --git a/drivers/gpu/drm/i915/gt/intel_region_lmem.c b/drivers/gpu/drm/i915/gt/intel_region_lmem.c
index f3ad93db0b21..2a3217e2890f 100644
--- a/drivers/gpu/drm/i915/gt/intel_region_lmem.c
+++ b/drivers/gpu/drm/i915/gt/intel_region_lmem.c
@@ -54,6 +54,7 @@ static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t
 	struct resource *root_res;
 	resource_size_t rebar_size;
 	resource_size_t current_size;
+	intel_wakeref_t wakeref;
 	u32 pci_cmd;
 	int i;
 
@@ -102,15 +103,25 @@ static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t
 		return;
 	}
 
-	/* First disable PCI memory decoding references */
-	pci_read_config_dword(pdev, PCI_COMMAND, &pci_cmd);
-	pci_write_config_dword(pdev, PCI_COMMAND,
-			       pci_cmd & ~PCI_COMMAND_MEMORY);
+	/*
+	 * Releasing forcewake during BAR resizing results in later forcewake
+	 * ack timeouts and former can happen any time - it is asynchronous.
+	 * Grabbing all forcewakes prevents it.
+	 */
+	with_intel_runtime_pm(i915->uncore.rpm, wakeref) {
+		intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
 
-	_resize_bar(i915, GEN12_LMEM_BAR, rebar_size);
+		/* First disable PCI memory decoding references */
+		pci_read_config_dword(pdev, PCI_COMMAND, &pci_cmd);
+		pci_write_config_dword(pdev, PCI_COMMAND,
+				       pci_cmd & ~PCI_COMMAND_MEMORY);
 
-	pci_assign_unassigned_bus_resources(pdev->bus);
-	pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd);
+		_resize_bar(i915, GEN12_LMEM_BAR, rebar_size);
+
+		pci_assign_unassigned_bus_resources(pdev->bus);
+		pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd);
+		intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
+	}
 }
 #else
 static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t lmem_size) {}
@@ -158,7 +169,7 @@ static const struct intel_memory_region_ops intel_region_lmem_ops = {
 static bool get_legacy_lowmem_region(struct intel_uncore *uncore,
 				     u64 *start, u32 *size)
 {
-	if (!IS_DG1_GRAPHICS_STEP(uncore->i915, STEP_A0, STEP_C0))
+	if (!IS_DG1(uncore->i915))
 		return false;
 
 	*start = 0;
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index 0bb9094fdacd..797ea8340467 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -14,6 +14,8 @@
 
 #include "gt/intel_gt_regs.h"
 
+#include "gt/uc/intel_gsc_fw.h"
+
 #include "i915_drv.h"
 #include "i915_file_private.h"
 #include "i915_gpu_error.h"
@@ -268,10 +270,28 @@ out:
 static int gen6_hw_domain_reset(struct intel_gt *gt, u32 hw_domain_mask)
 {
 	struct intel_uncore *uncore = gt->uncore;
-	int loops = 2;
+	int loops;
 	int err;
 
 	/*
+	 * On some platforms, e.g. Jasperlake, we see that the engine register
+	 * state is not cleared until shortly after GDRST reports completion,
+	 * causing a failure as we try to immediately resume while the internal
+	 * state is still in flux. If we immediately repeat the reset, the
+	 * second reset appears to serialise with the first, and since it is a
+	 * no-op, the registers should retain their reset value. However, there
+	 * is still a concern that upon leaving the second reset, the internal
+	 * engine state is still in flux and not ready for resuming.
+	 *
+	 * Starting on MTL, there are some prep steps that we need to do when
+	 * resetting some engines that need to be applied every time we write to
+	 * GEN6_GDRST. As those are time consuming (tens of ms), we don't want
+	 * to perform that twice, so, since the Jasperlake issue hasn't been
+	 * observed on MTL, we avoid repeating the reset on newer platforms.
+	 */
+	loops = GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70) ? 2 : 1;
+
+	/*
 	 * GEN6_GDRST is not in the gt power well, no need to check
 	 * for fifo space for the write or forcewake the chip for
 	 * the read
@@ -279,20 +299,7 @@ static int gen6_hw_domain_reset(struct intel_gt *gt, u32 hw_domain_mask)
 	do {
 		intel_uncore_write_fw(uncore, GEN6_GDRST, hw_domain_mask);
 
-		/*
-		 * Wait for the device to ack the reset requests.
-		 *
-		 * On some platforms, e.g. Jasperlake, we see that the
-		 * engine register state is not cleared until shortly after
-		 * GDRST reports completion, causing a failure as we try
-		 * to immediately resume while the internal state is still
-		 * in flux. If we immediately repeat the reset, the second
-		 * reset appears to serialise with the first, and since
-		 * it is a no-op, the registers should retain their reset
-		 * value. However, there is still a concern that upon
-		 * leaving the second reset, the internal engine state
-		 * is still in flux and not ready for resuming.
-		 */
+		/* Wait for the device to ack the reset requests. */
 		err = __intel_wait_for_register_fw(uncore, GEN6_GDRST,
 						   hw_domain_mask, 0,
 						   2000, 0,
@@ -690,6 +697,74 @@ static reset_func intel_get_gpu_reset(const struct intel_gt *gt)
 		return NULL;
 }
 
+static int __reset_guc(struct intel_gt *gt)
+{
+	u32 guc_domain =
+		GRAPHICS_VER(gt->i915) >= 11 ? GEN11_GRDOM_GUC : GEN9_GRDOM_GUC;
+
+	return gen6_hw_domain_reset(gt, guc_domain);
+}
+
+static bool needs_wa_14015076503(struct intel_gt *gt, intel_engine_mask_t engine_mask)
+{
+	if (!IS_METEORLAKE(gt->i915) || !HAS_ENGINE(gt, GSC0))
+		return false;
+
+	if (!__HAS_ENGINE(engine_mask, GSC0))
+		return false;
+
+	return intel_gsc_uc_fw_init_done(&gt->uc.gsc);
+}
+
+static intel_engine_mask_t
+wa_14015076503_start(struct intel_gt *gt, intel_engine_mask_t engine_mask, bool first)
+{
+	if (!needs_wa_14015076503(gt, engine_mask))
+		return engine_mask;
+
+	/*
+	 * wa_14015076503: if the GSC FW is loaded, we need to alert it that
+	 * we're going to do a GSC engine reset and then wait for 200ms for the
+	 * FW to get ready for it. However, if this is the first ALL_ENGINES
+	 * reset attempt and the GSC is not busy, we can try to instead reset
+	 * the GuC and all the other engines individually to avoid the 200ms
+	 * wait.
+	 * Skipping the GSC engine is safe because, differently from other
+	 * engines, the GSCCS only role is to forward the commands to the GSC
+	 * FW, so it doesn't have any HW outside of the CS itself and therefore
+	 * it has no state that we don't explicitly re-init on resume or on
+	 * context switch LRC or power context). The HW for the GSC uC is
+	 * managed by the GSC FW so we don't need to care about that.
+	 */
+	if (engine_mask == ALL_ENGINES && first && intel_engine_is_idle(gt->engine[GSC0])) {
+		__reset_guc(gt);
+		engine_mask = gt->info.engine_mask & ~BIT(GSC0);
+	} else {
+		intel_uncore_rmw(gt->uncore,
+				 HECI_H_GS1(MTL_GSC_HECI2_BASE),
+				 0, HECI_H_GS1_ER_PREP);
+
+		/* make sure the reset bit is clear when writing the CSR reg */
+		intel_uncore_rmw(gt->uncore,
+				 HECI_H_CSR(MTL_GSC_HECI2_BASE),
+				 HECI_H_CSR_RST, HECI_H_CSR_IG);
+		msleep(200);
+	}
+
+	return engine_mask;
+}
+
+static void
+wa_14015076503_end(struct intel_gt *gt, intel_engine_mask_t engine_mask)
+{
+	if (!needs_wa_14015076503(gt, engine_mask))
+		return;
+
+	intel_uncore_rmw(gt->uncore,
+			 HECI_H_GS1(MTL_GSC_HECI2_BASE),
+			 HECI_H_GS1_ER_PREP, 0);
+}
+
 int __intel_gt_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask)
 {
 	const int retries = engine_mask == ALL_ENGINES ? RESET_MAX_RETRIES : 1;
@@ -707,10 +782,16 @@ int __intel_gt_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask)
 	 */
 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 	for (retry = 0; ret == -ETIMEDOUT && retry < retries; retry++) {
-		GT_TRACE(gt, "engine_mask=%x\n", engine_mask);
+		intel_engine_mask_t reset_mask;
+
+		reset_mask = wa_14015076503_start(gt, engine_mask, !retry);
+
+		GT_TRACE(gt, "engine_mask=%x\n", reset_mask);
 		preempt_disable();
-		ret = reset(gt, engine_mask, retry);
+		ret = reset(gt, reset_mask, retry);
 		preempt_enable();
+
+		wa_14015076503_end(gt, reset_mask);
 	}
 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 
@@ -735,14 +816,12 @@ bool intel_has_reset_engine(const struct intel_gt *gt)
 
 int intel_reset_guc(struct intel_gt *gt)
 {
-	u32 guc_domain =
-		GRAPHICS_VER(gt->i915) >= 11 ? GEN11_GRDOM_GUC : GEN9_GRDOM_GUC;
 	int ret;
 
 	GEM_BUG_ON(!HAS_GT_UC(gt->i915));
 
 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
-	ret = gen6_hw_domain_reset(gt, guc_domain);
+	ret = __reset_guc(gt);
 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 
 	return ret;
diff --git a/drivers/gpu/drm/i915/gt/intel_reset_types.h b/drivers/gpu/drm/i915/gt/intel_reset_types.h
index 9312b29f5a97..80351f0a856c 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_reset_types.h
@@ -51,7 +51,7 @@ struct intel_reset {
 
 	/**
 	 * Waitqueue to signal when the reset has completed. Used by clients
-	 * that wait for dev_priv->mm.wedged to settle.
+	 * that wait for i915->mm.wedged to settle.
 	 */
 	wait_queue_head_t queue;
 
diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
index 827adb0cfaea..3fd795c3263f 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
@@ -1052,9 +1052,9 @@ static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
 
 static void ring_release(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
+	struct drm_i915_private *i915 = engine->i915;
 
-	drm_WARN_ON(&dev_priv->drm, GRAPHICS_VER(dev_priv) > 2 &&
+	drm_WARN_ON(&i915->drm, GRAPHICS_VER(i915) > 2 &&
 		    (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
 
 	intel_engine_cleanup_common(engine);
diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c
index 2c92fa9d1942..b2671ac59dc0 100644
--- a/drivers/gpu/drm/i915/gt/intel_rps.c
+++ b/drivers/gpu/drm/i915/gt/intel_rps.c
@@ -1677,7 +1677,6 @@ static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
 static void vlv_rps_init(struct intel_rps *rps)
 {
 	struct drm_i915_private *i915 = rps_to_i915(rps);
-	u32 val;
 
 	vlv_iosf_sb_get(i915,
 			BIT(VLV_IOSF_SB_PUNIT) |
@@ -1686,21 +1685,6 @@ static void vlv_rps_init(struct intel_rps *rps)
 
 	vlv_init_gpll_ref_freq(rps);
 
-	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
-	switch ((val >> 6) & 3) {
-	case 0:
-	case 1:
-		i915->mem_freq = 800;
-		break;
-	case 2:
-		i915->mem_freq = 1066;
-		break;
-	case 3:
-		i915->mem_freq = 1333;
-		break;
-	}
-	drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
-
 	rps->max_freq = vlv_rps_max_freq(rps);
 	rps->rp0_freq = rps->max_freq;
 	drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
@@ -1727,7 +1711,6 @@ static void vlv_rps_init(struct intel_rps *rps)
 static void chv_rps_init(struct intel_rps *rps)
 {
 	struct drm_i915_private *i915 = rps_to_i915(rps);
-	u32 val;
 
 	vlv_iosf_sb_get(i915,
 			BIT(VLV_IOSF_SB_PUNIT) |
@@ -1736,18 +1719,6 @@ static void chv_rps_init(struct intel_rps *rps)
 
 	vlv_init_gpll_ref_freq(rps);
 
-	val = vlv_cck_read(i915, CCK_FUSE_REG);
-
-	switch ((val >> 2) & 0x7) {
-	case 3:
-		i915->mem_freq = 2000;
-		break;
-	default:
-		i915->mem_freq = 1600;
-		break;
-	}
-	drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
-
 	rps->max_freq = chv_rps_max_freq(rps);
 	rps->rp0_freq = rps->max_freq;
 	drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
@@ -2675,7 +2646,7 @@ bool rps_read_mask_mmio(struct intel_rps *rps,
 
 static struct drm_i915_private __rcu *ips_mchdev;
 
-/**
+/*
  * Tells the intel_ips driver that the i915 driver is now loaded, if
  * IPS got loaded first.
  *
diff --git a/drivers/gpu/drm/i915/gt/intel_rps_types.h b/drivers/gpu/drm/i915/gt/intel_rps_types.h
index 9173ec75f2b8..6507fa3f6d1e 100644
--- a/drivers/gpu/drm/i915/gt/intel_rps_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_rps_types.h
@@ -57,7 +57,7 @@ struct intel_rps {
 
 	/*
 	 * work, interrupts_enabled and pm_iir are protected by
-	 * dev_priv->irq_lock
+	 * i915->irq_lock
 	 */
 	struct timer_list timer;
 	struct work_struct work;
diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.c b/drivers/gpu/drm/i915/gt/intel_sseu.c
index 6c6198a257ac..1141f875f5bd 100644
--- a/drivers/gpu/drm/i915/gt/intel_sseu.c
+++ b/drivers/gpu/drm/i915/gt/intel_sseu.c
@@ -6,6 +6,7 @@
 #include <linux/string_helpers.h>
 
 #include "i915_drv.h"
+#include "i915_perf_types.h"
 #include "intel_engine_regs.h"
 #include "intel_gt_regs.h"
 #include "intel_sseu.h"
@@ -677,7 +678,7 @@ u32 intel_sseu_make_rpcs(struct intel_gt *gt,
 	 * If i915/perf is active, we want a stable powergating configuration
 	 * on the system. Use the configuration pinned by i915/perf.
 	 */
-	if (gt->perf.exclusive_stream)
+	if (gt->perf.group && gt->perf.group[PERF_GROUP_OAG].exclusive_stream)
 		req_sseu = &gt->perf.sseu;
 
 	slices = hweight8(req_sseu->slice_mask);
diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c
index 485c5cc5d0f9..b925ef47304b 100644
--- a/drivers/gpu/drm/i915/gt/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c
@@ -743,9 +743,13 @@ static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
 	       FF_MODE2_GS_TIMER_224,
 	       0, false);
 
-	if (!IS_DG1(i915))
+	if (!IS_DG1(i915)) {
 		/* Wa_1806527549 */
 		wa_masked_en(wal, HIZ_CHICKEN, HZ_DEPTH_TEST_LE_GE_OPT_DISABLE);
+
+		/* Wa_1606376872 */
+		wa_masked_en(wal, COMMON_SLICE_CHICKEN4, DISABLE_TDC_LOAD_BALANCING_CALC);
+	}
 }
 
 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
@@ -1470,54 +1474,17 @@ gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 }
 
 static void
-tgl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
-{
-	struct drm_i915_private *i915 = gt->i915;
-
-	gen12_gt_workarounds_init(gt, wal);
-
-	/* Wa_1409420604:tgl */
-	if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
-		wa_mcr_write_or(wal,
-				SUBSLICE_UNIT_LEVEL_CLKGATE2,
-				CPSSUNIT_CLKGATE_DIS);
-
-	/* Wa_1607087056:tgl also know as BUG:1409180338 */
-	if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
-		wa_write_or(wal,
-			    GEN11_SLICE_UNIT_LEVEL_CLKGATE,
-			    L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
-
-	/* Wa_1408615072:tgl[a0] */
-	if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
-		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
-			    VSUNIT_CLKGATE_DIS_TGL);
-}
-
-static void
 dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 {
-	struct drm_i915_private *i915 = gt->i915;
-
 	gen12_gt_workarounds_init(gt, wal);
 
-	/* Wa_1607087056:dg1 */
-	if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
-		wa_write_or(wal,
-			    GEN11_SLICE_UNIT_LEVEL_CLKGATE,
-			    L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
-
 	/* Wa_1409420604:dg1 */
-	if (IS_DG1(i915))
-		wa_mcr_write_or(wal,
-				SUBSLICE_UNIT_LEVEL_CLKGATE2,
-				CPSSUNIT_CLKGATE_DIS);
+	wa_mcr_write_or(wal, SUBSLICE_UNIT_LEVEL_CLKGATE2,
+			CPSSUNIT_CLKGATE_DIS);
 
 	/* Wa_1408615072:dg1 */
 	/* Empirical testing shows this register is unaffected by engine reset. */
-	if (IS_DG1(i915))
-		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
-			    VSUNIT_CLKGATE_DIS_TGL);
+	wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, VSUNIT_CLKGATE_DIS_TGL);
 }
 
 static void
@@ -1530,6 +1497,12 @@ xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 	/* Wa_1409757795:xehpsdv */
 	wa_mcr_write_or(wal, SCCGCTL94DC, CG3DDISURB);
 
+	/* Wa_18011725039:xehpsdv */
+	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) {
+		wa_mcr_masked_dis(wal, MLTICTXCTL, TDONRENDER);
+		wa_mcr_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH);
+	}
+
 	/* Wa_16011155590:xehpsdv */
 	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
 		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
@@ -1579,6 +1552,9 @@ xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 	/* Wa_14014368820:xehpsdv */
 	wa_mcr_write_or(wal, XEHP_GAMCNTRL_CTRL,
 			INVALIDATION_BROADCAST_MODE_DIS | GLOBAL_INVALIDATION_MODE);
+
+	/* Wa_14010670810:xehpsdv */
+	wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
 }
 
 static void
@@ -1681,13 +1657,6 @@ dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 	/* Wa_14014830051:dg2 */
 	wa_mcr_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN);
 
-	/*
-	 * The following are not actually "workarounds" but rather
-	 * recommended tuning settings documented in the bspec's
-	 * performance guide section.
-	 */
-	wa_mcr_write_or(wal, XEHP_SQCM, EN_32B_ACCESS);
-
 	/* Wa_14015795083 */
 	wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE);
 
@@ -1700,6 +1669,9 @@ dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 	/* Wa_1509235366:dg2 */
 	wa_mcr_write_or(wal, XEHP_GAMCNTRL_CTRL,
 			INVALIDATION_BROADCAST_MODE_DIS | GLOBAL_INVALIDATION_MODE);
+
+	/* Wa_14010648519:dg2 */
+	wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
 }
 
 static void
@@ -1715,6 +1687,9 @@ pvc_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 	wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB);
 	wa_mcr_write_or(wal, XEHP_VDBX_MOD_CTRL, FORCE_MISS_FTLB);
 	wa_mcr_write_or(wal, XEHP_VEBX_MOD_CTRL, FORCE_MISS_FTLB);
+
+	/* Wa_16016694945 */
+	wa_mcr_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC);
 }
 
 static void
@@ -1755,11 +1730,38 @@ xelpmp_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 	debug_dump_steering(gt);
 }
 
+/*
+ * The bspec performance guide has recommended MMIO tuning settings.  These
+ * aren't truly "workarounds" but we want to program them through the
+ * workaround infrastructure to make sure they're (re)applied at the proper
+ * times.
+ *
+ * The programming in this function is for settings that persist through
+ * engine resets and also are not part of any engine's register state context.
+ * I.e., settings that only need to be re-applied in the event of a full GT
+ * reset.
+ */
+static void gt_tuning_settings(struct intel_gt *gt, struct i915_wa_list *wal)
+{
+	if (IS_PONTEVECCHIO(gt->i915)) {
+		wa_mcr_write(wal, XEHPC_L3SCRUB,
+			     SCRUB_CL_DWNGRADE_SHARED | SCRUB_RATE_4B_PER_CLK);
+		wa_mcr_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_HOSTCACHEEN);
+	}
+
+	if (IS_DG2(gt->i915)) {
+		wa_mcr_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS);
+		wa_mcr_write_or(wal, XEHP_SQCM, EN_32B_ACCESS);
+	}
+}
+
 static void
 gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal)
 {
 	struct drm_i915_private *i915 = gt->i915;
 
+	gt_tuning_settings(gt, wal);
+
 	if (gt->type == GT_MEDIA) {
 		if (MEDIA_VER(i915) >= 13)
 			xelpmp_gt_workarounds_init(gt, wal);
@@ -1779,8 +1781,6 @@ gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal)
 		xehpsdv_gt_workarounds_init(gt, wal);
 	else if (IS_DG1(i915))
 		dg1_gt_workarounds_init(gt, wal);
-	else if (IS_TIGERLAKE(i915))
-		tgl_gt_workarounds_init(gt, wal);
 	else if (GRAPHICS_VER(i915) == 12)
 		gen12_gt_workarounds_init(gt, wal);
 	else if (GRAPHICS_VER(i915) == 11)
@@ -2187,37 +2187,20 @@ static void tgl_whitelist_build(struct intel_engine_cs *engine)
 
 		/* Wa_1806527549:tgl */
 		whitelist_reg(w, HIZ_CHICKEN);
+
+		/* Required by recommended tuning setting (not a workaround) */
+		whitelist_reg(w, GEN11_COMMON_SLICE_CHICKEN3);
+
 		break;
 	default:
 		break;
 	}
 }
 
-static void dg1_whitelist_build(struct intel_engine_cs *engine)
-{
-	struct i915_wa_list *w = &engine->whitelist;
-
-	tgl_whitelist_build(engine);
-
-	/* GEN:BUG:1409280441:dg1 */
-	if (IS_DG1_GRAPHICS_STEP(engine->i915, STEP_A0, STEP_B0) &&
-	    (engine->class == RENDER_CLASS ||
-	     engine->class == COPY_ENGINE_CLASS))
-		whitelist_reg_ext(w, RING_ID(engine->mmio_base),
-				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
-}
-
-static void xehpsdv_whitelist_build(struct intel_engine_cs *engine)
-{
-	allow_read_ctx_timestamp(engine);
-}
-
 static void dg2_whitelist_build(struct intel_engine_cs *engine)
 {
 	struct i915_wa_list *w = &engine->whitelist;
 
-	allow_read_ctx_timestamp(engine);
-
 	switch (engine->class) {
 	case RENDER_CLASS:
 		/*
@@ -2234,6 +2217,9 @@ static void dg2_whitelist_build(struct intel_engine_cs *engine)
 					  RING_FORCE_TO_NONPRIV_ACCESS_RD |
 					  RING_FORCE_TO_NONPRIV_RANGE_4);
 
+		/* Required by recommended tuning setting (not a workaround) */
+		whitelist_mcr_reg(w, XEHP_COMMON_SLICE_CHICKEN3);
+
 		break;
 	case COMPUTE_CLASS:
 		/* Wa_16011157294:dg2_g10 */
@@ -2265,12 +2251,25 @@ static void blacklist_trtt(struct intel_engine_cs *engine)
 
 static void pvc_whitelist_build(struct intel_engine_cs *engine)
 {
-	allow_read_ctx_timestamp(engine);
-
 	/* Wa_16014440446:pvc */
 	blacklist_trtt(engine);
 }
 
+static void mtl_whitelist_build(struct intel_engine_cs *engine)
+{
+	struct i915_wa_list *w = &engine->whitelist;
+
+	switch (engine->class) {
+	case RENDER_CLASS:
+		/* Required by recommended tuning setting (not a workaround) */
+		whitelist_mcr_reg(w, XEHP_COMMON_SLICE_CHICKEN3);
+
+		break;
+	default:
+		break;
+	}
+}
+
 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *i915 = engine->i915;
@@ -2279,15 +2278,13 @@ void intel_engine_init_whitelist(struct intel_engine_cs *engine)
 	wa_init_start(w, engine->gt, "whitelist", engine->name);
 
 	if (IS_METEORLAKE(i915))
-		; /* noop; none at this time */
+		mtl_whitelist_build(engine);
 	else if (IS_PONTEVECCHIO(i915))
 		pvc_whitelist_build(engine);
 	else if (IS_DG2(i915))
 		dg2_whitelist_build(engine);
 	else if (IS_XEHPSDV(i915))
-		xehpsdv_whitelist_build(engine);
-	else if (IS_DG1(i915))
-		dg1_whitelist_build(engine);
+		; /* none needed */
 	else if (GRAPHICS_VER(i915) == 12)
 		tgl_whitelist_build(engine);
 	else if (GRAPHICS_VER(i915) == 11)
@@ -2452,16 +2449,12 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 				 MDQ_ARBITRATION_MODE | UGM_BACKUP_MODE);
 	}
 
-	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) {
+	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
 		/* Wa_22010430635:dg2 */
 		wa_mcr_masked_en(wal,
 				 GEN9_ROW_CHICKEN4,
 				 GEN12_DISABLE_GRF_CLEAR);
 
-		/* Wa_14010648519:dg2 */
-		wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
-	}
-
 	/* Wa_14013202645:dg2 */
 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) ||
 	    IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0))
@@ -2482,27 +2475,6 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 			   true);
 	}
 
-	if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
-	    IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) {
-		/*
-		 * Wa_1607138336:tgl[a0],dg1[a0]
-		 * Wa_1607063988:tgl[a0],dg1[a0]
-		 */
-		wa_write_or(wal,
-			    GEN9_CTX_PREEMPT_REG,
-			    GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
-	}
-
-	if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) {
-		/*
-		 * Wa_1606679103:tgl
-		 * (see also Wa_1606682166:icl)
-		 */
-		wa_write_or(wal,
-			    GEN7_SARCHKMD,
-			    GEN7_DISABLE_SAMPLER_PREFETCH);
-	}
-
 	if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || IS_DG1(i915) ||
 	    IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
 		/* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */
@@ -2532,30 +2504,22 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 	}
 
 	if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) ||
-	    IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
 	    IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
-		/* Wa_1409804808:tgl,rkl,dg1[a0],adl-s,adl-p */
+		/* Wa_1409804808 */
 		wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2,
 				 GEN12_PUSH_CONST_DEREF_HOLD_DIS);
 
-		/*
-		 * Wa_1409085225:tgl
-		 * Wa_14010229206:tgl,rkl,dg1[a0],adl-s,adl-p
-		 */
+		/* Wa_14010229206 */
 		wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
 	}
 
-	if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
-	    IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) || IS_ALDERLAKE_P(i915)) {
+	if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) || IS_ALDERLAKE_P(i915)) {
 		/*
-		 * Wa_1607030317:tgl
-		 * Wa_1607186500:tgl
-		 * Wa_1607297627:tgl,rkl,dg1[a0],adlp
+		 * Wa_1607297627
 		 *
 		 * On TGL and RKL there are multiple entries for this WA in the
 		 * BSpec; some indicate this is an A0-only WA, others indicate
 		 * it applies to all steppings so we trust the "all steppings."
-		 * For DG1 this only applies to A0.
 		 */
 		wa_masked_en(wal,
 			     RING_PSMI_CTL(RENDER_RING_BASE),
@@ -2975,16 +2939,8 @@ static void
 add_render_compute_tuning_settings(struct drm_i915_private *i915,
 				   struct i915_wa_list *wal)
 {
-	if (IS_PONTEVECCHIO(i915)) {
-		wa_mcr_write(wal, XEHPC_L3SCRUB,
-			     SCRUB_CL_DWNGRADE_SHARED | SCRUB_RATE_4B_PER_CLK);
-		wa_mcr_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_HOSTCACHEEN);
-	}
-
-	if (IS_DG2(i915)) {
-		wa_mcr_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS);
+	if (IS_DG2(i915))
 		wa_mcr_write_clr_set(wal, RT_CTRL, STACKID_CTRL, STACKID_CTRL_512);
-	}
 
 	/*
 	 * This tuning setting proves beneficial only on ATS-M designs; the
@@ -3015,6 +2971,44 @@ general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_li
 
 	add_render_compute_tuning_settings(i915, wal);
 
+	if (GRAPHICS_VER(i915) >= 11) {
+		/* This is not a Wa (although referred to as
+		 * WaSetInidrectStateOverride in places), this allows
+		 * applications that reference sampler states through
+		 * the BindlessSamplerStateBaseAddress to have their
+		 * border color relative to DynamicStateBaseAddress
+		 * rather than BindlessSamplerStateBaseAddress.
+		 *
+		 * Otherwise SAMPLER_STATE border colors have to be
+		 * copied in multiple heaps (DynamicStateBaseAddress &
+		 * BindlessSamplerStateBaseAddress)
+		 *
+		 * BSpec: 46052
+		 */
+		wa_mcr_masked_en(wal,
+				 GEN10_SAMPLER_MODE,
+				 GEN11_INDIRECT_STATE_BASE_ADDR_OVERRIDE);
+	}
+
+	if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_B0, STEP_FOREVER) ||
+	    IS_MTL_GRAPHICS_STEP(i915, P, STEP_B0, STEP_FOREVER))
+		/* Wa_14017856879 */
+		wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN3, MTL_DISABLE_FIX_FOR_EOT_FLUSH);
+
+	if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) ||
+	    IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0))
+		/*
+		 * Wa_14017066071
+		 * Wa_14017654203
+		 */
+		wa_mcr_masked_en(wal, GEN10_SAMPLER_MODE,
+				 MTL_DISABLE_SAMPLER_SC_OOO);
+
+	if (IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0))
+		/* Wa_22015279794 */
+		wa_mcr_masked_en(wal, GEN10_CACHE_MODE_SS,
+				 DISABLE_PREFETCH_INTO_IC);
+
 	if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) ||
 	    IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0) ||
 	    IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER) ||
@@ -3066,11 +3060,6 @@ general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_li
 			   0, false);
 	}
 
-	if (IS_PONTEVECCHIO(i915)) {
-		/* Wa_16016694945 */
-		wa_mcr_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC);
-	}
-
 	if (IS_XEHPSDV(i915)) {
 		/* Wa_1409954639 */
 		wa_mcr_masked_en(wal,
@@ -3082,18 +3071,9 @@ general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_li
 				 GEN9_ROW_CHICKEN4,
 				 GEN12_DISABLE_GRF_CLEAR);
 
-		/* Wa_14010670810:xehpsdv */
-		wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
-
 		/* Wa_14010449647:xehpsdv */
 		wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
 				 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
-
-		/* Wa_18011725039:xehpsdv */
-		if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) {
-			wa_mcr_masked_dis(wal, MLTICTXCTL, TDONRENDER);
-			wa_mcr_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH);
-		}
 	}
 
 	if (IS_DG2(i915) || IS_PONTEVECCHIO(i915)) {
diff --git a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c
index b46425aeb2f0..0971241707ce 100644
--- a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c
+++ b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c
@@ -63,8 +63,8 @@ static void measure_clocks(struct intel_engine_cs *engine,
 
 		udelay(1000);
 
-		dt[i] = ktime_sub(ktime_get(), dt[i]);
 		cycles[i] += read_timestamp(engine);
+		dt[i] = ktime_sub(ktime_get(), dt[i]);
 		local_irq_enable();
 	}
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_llc.c b/drivers/gpu/drm/i915/gt/selftest_llc.c
index cfd736d88939..779fadcec7c4 100644
--- a/drivers/gpu/drm/i915/gt/selftest_llc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_llc.c
@@ -3,7 +3,6 @@
  * Copyright © 2019 Intel Corporation
  */
 
-#include "intel_pm.h" /* intel_gpu_freq() */
 #include "selftest_llc.h"
 #include "intel_rps.h"
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c
index 6755bbc4ebda..84e77e8dbba1 100644
--- a/drivers/gpu/drm/i915/gt/selftest_rps.c
+++ b/drivers/gpu/drm/i915/gt/selftest_rps.c
@@ -299,13 +299,13 @@ int live_rps_clock_interval(void *arg)
 			for (i = 0; i < 5; i++) {
 				preempt_disable();
 
-				dt_[i] = ktime_get();
 				cycles_[i] = -intel_uncore_read_fw(gt->uncore, GEN6_RP_CUR_UP_EI);
+				dt_[i] = ktime_get();
 
 				udelay(1000);
 
-				dt_[i] = ktime_sub(ktime_get(), dt_[i]);
 				cycles_[i] += intel_uncore_read_fw(gt->uncore, GEN6_RP_CUR_UP_EI);
+				dt_[i] = ktime_sub(ktime_get(), dt_[i]);
 
 				preempt_enable();
 			}
@@ -537,8 +537,8 @@ static u64 __measure_frequency(u32 *cntr, int duration_ms)
 {
 	u64 dc, dt;
 
-	dt = ktime_get();
 	dc = READ_ONCE(*cntr);
+	dt = ktime_get();
 	usleep_range(1000 * duration_ms, 2000 * duration_ms);
 	dc = READ_ONCE(*cntr) - dc;
 	dt = ktime_get() - dt;
@@ -566,8 +566,8 @@ static u64 __measure_cs_frequency(struct intel_engine_cs *engine,
 {
 	u64 dc, dt;
 
-	dt = ktime_get();
 	dc = intel_uncore_read_fw(engine->uncore, CS_GPR(0));
+	dt = ktime_get();
 	usleep_range(1000 * duration_ms, 2000 * duration_ms);
 	dc = intel_uncore_read_fw(engine->uncore, CS_GPR(0)) - dc;
 	dt = ktime_get() - dt;
@@ -1094,8 +1094,8 @@ static u64 __measure_power(int duration_ms)
 {
 	u64 dE, dt;
 
-	dt = ktime_get();
 	dE = librapl_energy_uJ();
+	dt = ktime_get();
 	usleep_range(1000 * duration_ms, 2000 * duration_ms);
 	dE = librapl_energy_uJ() - dE;
 	dt = ktime_get() - dt;
diff --git a/drivers/gpu/drm/i915/gt/selftest_timeline.c b/drivers/gpu/drm/i915/gt/selftest_timeline.c
index 522d0190509c..9f536c251179 100644
--- a/drivers/gpu/drm/i915/gt/selftest_timeline.c
+++ b/drivers/gpu/drm/i915/gt/selftest_timeline.c
@@ -825,7 +825,8 @@ static bool cmp_gte(u32 a, u32 b)
 	return a >= b;
 }
 
-static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt)
+static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt,
+			 struct intel_timeline *tl)
 {
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
@@ -834,7 +835,10 @@ static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt)
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
 
-	w->map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
+	/* keep the same cache settings as timeline */
+	i915_gem_object_set_cache_coherency(obj, tl->hwsp_ggtt->obj->cache_level);
+	w->map = i915_gem_object_pin_map_unlocked(obj,
+						  page_unmask_bits(tl->hwsp_ggtt->obj->mm.mapping));
 	if (IS_ERR(w->map)) {
 		i915_gem_object_put(obj);
 		return PTR_ERR(w->map);
@@ -1004,8 +1008,10 @@ static int live_hwsp_read(void *arg)
 	if (!tl->has_initial_breadcrumb)
 		goto out_free;
 
+	selftest_tl_pin(tl);
+
 	for (i = 0; i < ARRAY_SIZE(watcher); i++) {
-		err = setup_watcher(&watcher[i], gt);
+		err = setup_watcher(&watcher[i], gt, tl);
 		if (err)
 			goto out;
 	}
@@ -1160,6 +1166,8 @@ out:
 	for (i = 0; i < ARRAY_SIZE(watcher); i++)
 		cleanup_watcher(&watcher[i]);
 
+	intel_timeline_unpin(tl);
+
 	if (igt_flush_test(gt->i915))
 		err = -EIO;
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_tlb.c b/drivers/gpu/drm/i915/gt/selftest_tlb.c
new file mode 100644
index 000000000000..e6cac1f15d6e
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/selftest_tlb.c
@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#include "i915_selftest.h"
+
+#include "gem/i915_gem_internal.h"
+#include "gem/i915_gem_region.h"
+
+#include "gen8_engine_cs.h"
+#include "i915_gem_ww.h"
+#include "intel_engine_regs.h"
+#include "intel_gpu_commands.h"
+#include "intel_context.h"
+#include "intel_gt.h"
+#include "intel_ring.h"
+
+#include "selftests/igt_flush_test.h"
+#include "selftests/i915_random.h"
+
+static void vma_set_qw(struct i915_vma *vma, u64 addr, u64 val)
+{
+	GEM_BUG_ON(addr < i915_vma_offset(vma));
+	GEM_BUG_ON(addr >= i915_vma_offset(vma) + i915_vma_size(vma) + sizeof(val));
+	memset64(page_mask_bits(vma->obj->mm.mapping) +
+		 (addr - i915_vma_offset(vma)), val, 1);
+}
+
+static int
+pte_tlbinv(struct intel_context *ce,
+	   struct i915_vma *va,
+	   struct i915_vma *vb,
+	   u64 align,
+	   void (*tlbinv)(struct i915_address_space *vm, u64 addr, u64 length),
+	   u64 length,
+	   struct rnd_state *prng)
+{
+	struct drm_i915_gem_object *batch;
+	struct drm_mm_node vb_node;
+	struct i915_request *rq;
+	struct i915_vma *vma;
+	u64 addr;
+	int err;
+	u32 *cs;
+
+	batch = i915_gem_object_create_internal(ce->vm->i915, 4096);
+	if (IS_ERR(batch))
+		return PTR_ERR(batch);
+
+	vma = i915_vma_instance(batch, ce->vm, NULL);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto out;
+	}
+
+	err = i915_vma_pin(vma, 0, 0, PIN_USER);
+	if (err)
+		goto out;
+
+	/* Pin va at random but aligned offset after vma */
+	addr = round_up(vma->node.start + vma->node.size, align);
+	/* MI_CONDITIONAL_BATCH_BUFFER_END limits address to 48b */
+	addr = igt_random_offset(prng, addr, min(ce->vm->total, BIT_ULL(48)),
+				 va->size, align);
+	err = i915_vma_pin(va,  0, 0, addr | PIN_OFFSET_FIXED | PIN_USER);
+	if (err) {
+		pr_err("Cannot pin at %llx+%llx\n", addr, va->size);
+		goto out;
+	}
+	GEM_BUG_ON(i915_vma_offset(va) != addr);
+	if (vb != va) {
+		vb_node = vb->node;
+		vb->node = va->node; /* overwrites the _same_ PTE  */
+	}
+
+	/*
+	 * Now choose random dword at the 1st pinned page.
+	 *
+	 * SZ_64K pages on dg1 require that the whole PT be marked
+	 * containing 64KiB entries. So we make sure that vma
+	 * covers the whole PT, despite being randomly aligned to 64KiB
+	 * and restrict our sampling to the 2MiB PT within where
+	 * we know that we will be using 64KiB pages.
+	 */
+	if (align == SZ_64K)
+		addr = round_up(addr, SZ_2M);
+	addr = igt_random_offset(prng, addr, addr + align, 8, 8);
+
+	if (va != vb)
+		pr_info("%s(%s): Sampling %llx, with alignment %llx, using PTE size %x (phys %x, sg %x), invalidate:%llx+%llx\n",
+			ce->engine->name, va->obj->mm.region->name ?: "smem",
+			addr, align, va->resource->page_sizes_gtt,
+			va->page_sizes.phys, va->page_sizes.sg,
+			addr & -length, length);
+
+	cs = i915_gem_object_pin_map_unlocked(batch, I915_MAP_WC);
+	*cs++ = MI_NOOP; /* for later termination */
+	/*
+	 * Sample the target to see if we spot the updated backing store.
+	 * Gen8 VCS compares immediate value with bitwise-and of two
+	 * consecutive DWORDS pointed by addr, other gen/engines compare value
+	 * with DWORD pointed by addr. Moreover we want to exercise DWORD size
+	 * invalidations. To fulfill all these requirements below values
+	 * have been chosen.
+	 */
+	*cs++ = MI_CONDITIONAL_BATCH_BUFFER_END | MI_DO_COMPARE | 2;
+	*cs++ = 0; /* break if *addr == 0 */
+	*cs++ = lower_32_bits(addr);
+	*cs++ = upper_32_bits(addr);
+	vma_set_qw(va, addr, -1);
+	vma_set_qw(vb, addr, 0);
+
+	/* Keep sampling until we get bored */
+	*cs++ = MI_BATCH_BUFFER_START | BIT(8) | 1;
+	*cs++ = lower_32_bits(i915_vma_offset(vma));
+	*cs++ = upper_32_bits(i915_vma_offset(vma));
+
+	i915_gem_object_flush_map(batch);
+
+	rq = i915_request_create(ce);
+	if (IS_ERR(rq)) {
+		err = PTR_ERR(rq);
+		goto out_va;
+	}
+
+	err = rq->engine->emit_bb_start(rq, i915_vma_offset(vma), 0, 0);
+	if (err) {
+		i915_request_add(rq);
+		goto out_va;
+	}
+
+	i915_request_get(rq);
+	i915_request_add(rq);
+
+	/* Short sleep to sanitycheck the batch is spinning before we begin */
+	msleep(10);
+	if (va == vb) {
+		if (!i915_request_completed(rq)) {
+			pr_err("%s(%s): Semaphore sanitycheck failed %llx, with alignment %llx, using PTE size %x (phys %x, sg %x)\n",
+			       ce->engine->name, va->obj->mm.region->name ?: "smem",
+			       addr, align, va->resource->page_sizes_gtt,
+			       va->page_sizes.phys, va->page_sizes.sg);
+			err = -EIO;
+		}
+	} else if (!i915_request_completed(rq)) {
+		struct i915_vma_resource vb_res = {
+			.bi.pages = vb->obj->mm.pages,
+			.bi.page_sizes = vb->obj->mm.page_sizes,
+			.start = i915_vma_offset(vb),
+			.vma_size = i915_vma_size(vb)
+		};
+		unsigned int pte_flags = 0;
+
+		/* Flip the PTE between A and B */
+		if (i915_gem_object_is_lmem(vb->obj))
+			pte_flags |= PTE_LM;
+		ce->vm->insert_entries(ce->vm, &vb_res, 0, pte_flags);
+
+		/* Flush the PTE update to concurrent HW */
+		tlbinv(ce->vm, addr & -length, length);
+
+		if (wait_for(i915_request_completed(rq), HZ / 2)) {
+			pr_err("%s: Request did not complete; the COND_BBE did not read the updated PTE\n",
+			       ce->engine->name);
+			err = -EINVAL;
+		}
+	} else {
+		pr_err("Spinner ended unexpectedly\n");
+		err = -EIO;
+	}
+	i915_request_put(rq);
+
+	cs = page_mask_bits(batch->mm.mapping);
+	*cs = MI_BATCH_BUFFER_END;
+	wmb();
+
+out_va:
+	if (vb != va)
+		vb->node = vb_node;
+	i915_vma_unpin(va);
+	if (i915_vma_unbind_unlocked(va))
+		err = -EIO;
+out:
+	i915_gem_object_put(batch);
+	return err;
+}
+
+static struct drm_i915_gem_object *create_lmem(struct intel_gt *gt)
+{
+	/*
+	 * Allocation of largest possible page size allows to test all types
+	 * of pages.
+	 */
+	return i915_gem_object_create_lmem(gt->i915, SZ_1G, I915_BO_ALLOC_CONTIGUOUS);
+}
+
+static struct drm_i915_gem_object *create_smem(struct intel_gt *gt)
+{
+	/*
+	 * SZ_64K pages require covering the whole 2M PT (gen8 to tgl/dg1).
+	 * While that does not require the whole 2M block to be contiguous
+	 * it is easier to make it so, since we need that for SZ_2M pagees.
+	 * Since we randomly offset the start of the vma, we need a 4M object
+	 * so that there is a 2M range within it is suitable for SZ_64K PTE.
+	 */
+	return i915_gem_object_create_internal(gt->i915, SZ_4M);
+}
+
+static int
+mem_tlbinv(struct intel_gt *gt,
+	   struct drm_i915_gem_object *(*create_fn)(struct intel_gt *),
+	   void (*tlbinv)(struct i915_address_space *vm, u64 addr, u64 length))
+{
+	unsigned int ppgtt_size = RUNTIME_INFO(gt->i915)->ppgtt_size;
+	struct intel_engine_cs *engine;
+	struct drm_i915_gem_object *A, *B;
+	struct i915_ppgtt *ppgtt;
+	struct i915_vma *va, *vb;
+	enum intel_engine_id id;
+	I915_RND_STATE(prng);
+	void *vaddr;
+	int err;
+
+	/*
+	 * Check that the TLB invalidate is able to revoke an active
+	 * page. We load a page into a spinning COND_BBE loop and then
+	 * remap that page to a new physical address. The old address, and
+	 * so the loop keeps spinning, is retained in the TLB cache until
+	 * we issue an invalidate.
+	 */
+
+	A = create_fn(gt);
+	if (IS_ERR(A))
+		return PTR_ERR(A);
+
+	vaddr = i915_gem_object_pin_map_unlocked(A, I915_MAP_WC);
+	if (IS_ERR(vaddr)) {
+		err = PTR_ERR(vaddr);
+		goto out_a;
+	}
+
+	B = create_fn(gt);
+	if (IS_ERR(B)) {
+		err = PTR_ERR(B);
+		goto out_a;
+	}
+
+	vaddr = i915_gem_object_pin_map_unlocked(B, I915_MAP_WC);
+	if (IS_ERR(vaddr)) {
+		err = PTR_ERR(vaddr);
+		goto out_b;
+	}
+
+	GEM_BUG_ON(A->base.size != B->base.size);
+	if ((A->mm.page_sizes.phys | B->mm.page_sizes.phys) & (A->base.size - 1))
+		pr_warn("Failed to allocate contiguous pages for size %zx\n",
+			A->base.size);
+
+	ppgtt = i915_ppgtt_create(gt, 0);
+	if (IS_ERR(ppgtt)) {
+		err = PTR_ERR(ppgtt);
+		goto out_b;
+	}
+
+	va = i915_vma_instance(A, &ppgtt->vm, NULL);
+	if (IS_ERR(va)) {
+		err = PTR_ERR(va);
+		goto out_vm;
+	}
+
+	vb = i915_vma_instance(B, &ppgtt->vm, NULL);
+	if (IS_ERR(vb)) {
+		err = PTR_ERR(vb);
+		goto out_vm;
+	}
+
+	err = 0;
+	for_each_engine(engine, gt, id) {
+		struct i915_gem_ww_ctx ww;
+		struct intel_context *ce;
+		int bit;
+
+		ce = intel_context_create(engine);
+		if (IS_ERR(ce)) {
+			err = PTR_ERR(ce);
+			break;
+		}
+
+		i915_vm_put(ce->vm);
+		ce->vm = i915_vm_get(&ppgtt->vm);
+
+		for_i915_gem_ww(&ww, err, true)
+			err = intel_context_pin_ww(ce, &ww);
+		if (err)
+			goto err_put;
+
+		for_each_set_bit(bit,
+				 (unsigned long *)&RUNTIME_INFO(gt->i915)->page_sizes,
+				 BITS_PER_TYPE(RUNTIME_INFO(gt->i915)->page_sizes)) {
+			unsigned int len;
+
+			if (BIT_ULL(bit) < i915_vm_obj_min_alignment(va->vm, va->obj))
+				continue;
+
+			/* sanitycheck the semaphore wake up */
+			err = pte_tlbinv(ce, va, va,
+					 BIT_ULL(bit),
+					 NULL, SZ_4K,
+					 &prng);
+			if (err)
+				goto err_unpin;
+
+			for (len = 2; len <= ppgtt_size; len = min(2 * len, ppgtt_size)) {
+				err = pte_tlbinv(ce, va, vb,
+						 BIT_ULL(bit),
+						 tlbinv,
+						 BIT_ULL(len),
+						 &prng);
+				if (err)
+					goto err_unpin;
+				if (len == ppgtt_size)
+					break;
+			}
+		}
+err_unpin:
+		intel_context_unpin(ce);
+err_put:
+		intel_context_put(ce);
+		if (err)
+			break;
+	}
+
+	if (igt_flush_test(gt->i915))
+		err = -EIO;
+
+out_vm:
+	i915_vm_put(&ppgtt->vm);
+out_b:
+	i915_gem_object_put(B);
+out_a:
+	i915_gem_object_put(A);
+	return err;
+}
+
+static void tlbinv_full(struct i915_address_space *vm, u64 addr, u64 length)
+{
+	intel_gt_invalidate_tlb(vm->gt, intel_gt_tlb_seqno(vm->gt) | 1);
+}
+
+static int invalidate_full(void *arg)
+{
+	struct intel_gt *gt = arg;
+	int err;
+
+	if (GRAPHICS_VER(gt->i915) < 8)
+		return 0; /* TLB invalidate not implemented */
+
+	err = mem_tlbinv(gt, create_smem, tlbinv_full);
+	if (err == 0)
+		err = mem_tlbinv(gt, create_lmem, tlbinv_full);
+	if (err == -ENODEV || err == -ENXIO)
+		err = 0;
+
+	return err;
+}
+
+int intel_tlb_live_selftests(struct drm_i915_private *i915)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(invalidate_full),
+	};
+	struct intel_gt *gt;
+	unsigned int i;
+
+	for_each_gt(gt, i915, i) {
+		int err;
+
+		if (intel_gt_is_wedged(gt))
+			continue;
+
+		err = intel_gt_live_subtests(tests, gt);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/gt/sysfs_engines.c b/drivers/gpu/drm/i915/gt/sysfs_engines.c
index f2d9858d827c..021f51d9b456 100644
--- a/drivers/gpu/drm/i915/gt/sysfs_engines.c
+++ b/drivers/gpu/drm/i915/gt/sysfs_engines.c
@@ -24,37 +24,37 @@ static struct intel_engine_cs *kobj_to_engine(struct kobject *kobj)
 static ssize_t
 name_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%s\n", kobj_to_engine(kobj)->name);
+	return sysfs_emit(buf, "%s\n", kobj_to_engine(kobj)->name);
 }
 
-static struct kobj_attribute name_attr =
+static const struct kobj_attribute name_attr =
 __ATTR(name, 0444, name_show, NULL);
 
 static ssize_t
 class_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", kobj_to_engine(kobj)->uabi_class);
+	return sysfs_emit(buf, "%d\n", kobj_to_engine(kobj)->uabi_class);
 }
 
-static struct kobj_attribute class_attr =
+static const struct kobj_attribute class_attr =
 __ATTR(class, 0444, class_show, NULL);
 
 static ssize_t
 inst_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", kobj_to_engine(kobj)->uabi_instance);
+	return sysfs_emit(buf, "%d\n", kobj_to_engine(kobj)->uabi_instance);
 }
 
-static struct kobj_attribute inst_attr =
+static const struct kobj_attribute inst_attr =
 __ATTR(instance, 0444, inst_show, NULL);
 
 static ssize_t
 mmio_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
-	return sprintf(buf, "0x%x\n", kobj_to_engine(kobj)->mmio_base);
+	return sysfs_emit(buf, "0x%x\n", kobj_to_engine(kobj)->mmio_base);
 }
 
-static struct kobj_attribute mmio_attr =
+static const struct kobj_attribute mmio_attr =
 __ATTR(mmio_base, 0444, mmio_show, NULL);
 
 static const char * const vcs_caps[] = {
@@ -107,11 +107,9 @@ __caps_show(struct intel_engine_cs *engine,
 	for_each_set_bit(n, &caps, show_unknown ? BITS_PER_LONG : count) {
 		if (n >= count || !repr[n]) {
 			if (GEM_WARN_ON(show_unknown))
-				len += snprintf(buf + len, PAGE_SIZE - len,
-						"[%x] ", n);
+				len += sysfs_emit_at(buf, len, "[%x] ", n);
 		} else {
-			len += snprintf(buf + len, PAGE_SIZE - len,
-					"%s ", repr[n]);
+			len += sysfs_emit_at(buf, len, "%s ", repr[n]);
 		}
 		if (GEM_WARN_ON(len >= PAGE_SIZE))
 			break;
@@ -127,7 +125,7 @@ caps_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 	return __caps_show(engine, engine->uabi_capabilities, buf, true);
 }
 
-static struct kobj_attribute caps_attr =
+static const struct kobj_attribute caps_attr =
 __ATTR(capabilities, 0444, caps_show, NULL);
 
 static ssize_t
@@ -136,7 +134,7 @@ all_caps_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 	return __caps_show(kobj_to_engine(kobj), -1, buf, false);
 }
 
-static struct kobj_attribute all_caps_attr =
+static const struct kobj_attribute all_caps_attr =
 __ATTR(known_capabilities, 0444, all_caps_show, NULL);
 
 static ssize_t
@@ -182,10 +180,10 @@ max_spin_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
 	struct intel_engine_cs *engine = kobj_to_engine(kobj);
 
-	return sprintf(buf, "%lu\n", engine->props.max_busywait_duration_ns);
+	return sysfs_emit(buf, "%lu\n", engine->props.max_busywait_duration_ns);
 }
 
-static struct kobj_attribute max_spin_attr =
+static const struct kobj_attribute max_spin_attr =
 __ATTR(max_busywait_duration_ns, 0644, max_spin_show, max_spin_store);
 
 static ssize_t
@@ -193,10 +191,10 @@ max_spin_default(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
 	struct intel_engine_cs *engine = kobj_to_engine(kobj);
 
-	return sprintf(buf, "%lu\n", engine->defaults.max_busywait_duration_ns);
+	return sysfs_emit(buf, "%lu\n", engine->defaults.max_busywait_duration_ns);
 }
 
-static struct kobj_attribute max_spin_def =
+static const struct kobj_attribute max_spin_def =
 __ATTR(max_busywait_duration_ns, 0444, max_spin_default, NULL);
 
 static ssize_t
@@ -236,10 +234,10 @@ timeslice_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
 	struct intel_engine_cs *engine = kobj_to_engine(kobj);
 
-	return sprintf(buf, "%lu\n", engine->props.timeslice_duration_ms);
+	return sysfs_emit(buf, "%lu\n", engine->props.timeslice_duration_ms);
 }
 
-static struct kobj_attribute timeslice_duration_attr =
+static const struct kobj_attribute timeslice_duration_attr =
 __ATTR(timeslice_duration_ms, 0644, timeslice_show, timeslice_store);
 
 static ssize_t
@@ -247,10 +245,10 @@ timeslice_default(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
 	struct intel_engine_cs *engine = kobj_to_engine(kobj);
 
-	return sprintf(buf, "%lu\n", engine->defaults.timeslice_duration_ms);
+	return sysfs_emit(buf, "%lu\n", engine->defaults.timeslice_duration_ms);
 }
 
-static struct kobj_attribute timeslice_duration_def =
+static const struct kobj_attribute timeslice_duration_def =
 __ATTR(timeslice_duration_ms, 0444, timeslice_default, NULL);
 
 static ssize_t
@@ -287,10 +285,10 @@ stop_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
 	struct intel_engine_cs *engine = kobj_to_engine(kobj);
 
-	return sprintf(buf, "%lu\n", engine->props.stop_timeout_ms);
+	return sysfs_emit(buf, "%lu\n", engine->props.stop_timeout_ms);
 }
 
-static struct kobj_attribute stop_timeout_attr =
+static const struct kobj_attribute stop_timeout_attr =
 __ATTR(stop_timeout_ms, 0644, stop_show, stop_store);
 
 static ssize_t
@@ -298,10 +296,10 @@ stop_default(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
 	struct intel_engine_cs *engine = kobj_to_engine(kobj);
 
-	return sprintf(buf, "%lu\n", engine->defaults.stop_timeout_ms);
+	return sysfs_emit(buf, "%lu\n", engine->defaults.stop_timeout_ms);
 }
 
-static struct kobj_attribute stop_timeout_def =
+static const struct kobj_attribute stop_timeout_def =
 __ATTR(stop_timeout_ms, 0444, stop_default, NULL);
 
 static ssize_t
@@ -343,10 +341,10 @@ preempt_timeout_show(struct kobject *kobj, struct kobj_attribute *attr,
 {
 	struct intel_engine_cs *engine = kobj_to_engine(kobj);
 
-	return sprintf(buf, "%lu\n", engine->props.preempt_timeout_ms);
+	return sysfs_emit(buf, "%lu\n", engine->props.preempt_timeout_ms);
 }
 
-static struct kobj_attribute preempt_timeout_attr =
+static const struct kobj_attribute preempt_timeout_attr =
 __ATTR(preempt_timeout_ms, 0644, preempt_timeout_show, preempt_timeout_store);
 
 static ssize_t
@@ -355,10 +353,10 @@ preempt_timeout_default(struct kobject *kobj, struct kobj_attribute *attr,
 {
 	struct intel_engine_cs *engine = kobj_to_engine(kobj);
 
-	return sprintf(buf, "%lu\n", engine->defaults.preempt_timeout_ms);
+	return sysfs_emit(buf, "%lu\n", engine->defaults.preempt_timeout_ms);
 }
 
-static struct kobj_attribute preempt_timeout_def =
+static const struct kobj_attribute preempt_timeout_def =
 __ATTR(preempt_timeout_ms, 0444, preempt_timeout_default, NULL);
 
 static ssize_t
@@ -399,10 +397,10 @@ heartbeat_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
 	struct intel_engine_cs *engine = kobj_to_engine(kobj);
 
-	return sprintf(buf, "%lu\n", engine->props.heartbeat_interval_ms);
+	return sysfs_emit(buf, "%lu\n", engine->props.heartbeat_interval_ms);
 }
 
-static struct kobj_attribute heartbeat_interval_attr =
+static const struct kobj_attribute heartbeat_interval_attr =
 __ATTR(heartbeat_interval_ms, 0644, heartbeat_show, heartbeat_store);
 
 static ssize_t
@@ -410,10 +408,10 @@ heartbeat_default(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
 	struct intel_engine_cs *engine = kobj_to_engine(kobj);
 
-	return sprintf(buf, "%lu\n", engine->defaults.heartbeat_interval_ms);
+	return sysfs_emit(buf, "%lu\n", engine->defaults.heartbeat_interval_ms);
 }
 
-static struct kobj_attribute heartbeat_interval_def =
+static const struct kobj_attribute heartbeat_interval_def =
 __ATTR(heartbeat_interval_ms, 0444, heartbeat_default, NULL);
 
 static void kobj_engine_release(struct kobject *kobj)
@@ -421,7 +419,7 @@ static void kobj_engine_release(struct kobject *kobj)
 	kfree(kobj);
 }
 
-static struct kobj_type kobj_engine_type = {
+static const struct kobj_type kobj_engine_type = {
 	.release = kobj_engine_release,
 	.sysfs_ops = &kobj_sysfs_ops
 };
@@ -449,7 +447,7 @@ kobj_engine(struct kobject *dir, struct intel_engine_cs *engine)
 
 static void add_defaults(struct kobj_engine *parent)
 {
-	static const struct attribute *files[] = {
+	static const struct attribute * const files[] = {
 		&max_spin_def.attr,
 		&stop_timeout_def.attr,
 #if CONFIG_DRM_I915_HEARTBEAT_INTERVAL
@@ -485,7 +483,7 @@ static void add_defaults(struct kobj_engine *parent)
 
 void intel_engines_add_sysfs(struct drm_i915_private *i915)
 {
-	static const struct attribute *files[] = {
+	static const struct attribute * const files[] = {
 		&name_attr.attr,
 		&class_attr.attr,
 		&inst_attr.attr,
diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h
index 8085fb181274..bcb1129b3610 100644
--- a/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h
+++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h
@@ -21,6 +21,9 @@ enum intel_guc_load_status {
 	INTEL_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH       = 0x02,
 	INTEL_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH       = 0x03,
 	INTEL_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE      = 0x04,
+	INTEL_GUC_LOAD_STATUS_HWCONFIG_START                   = 0x05,
+	INTEL_GUC_LOAD_STATUS_HWCONFIG_DONE                    = 0x06,
+	INTEL_GUC_LOAD_STATUS_HWCONFIG_ERROR                   = 0x07,
 	INTEL_GUC_LOAD_STATUS_GDT_DONE                         = 0x10,
 	INTEL_GUC_LOAD_STATUS_IDT_DONE                         = 0x20,
 	INTEL_GUC_LOAD_STATUS_LAPIC_DONE                       = 0x30,
@@ -38,4 +41,18 @@ enum intel_guc_load_status {
 	INTEL_GUC_LOAD_STATUS_READY                            = 0xF0,
 };
 
+enum intel_bootrom_load_status {
+	INTEL_BOOTROM_STATUS_NO_KEY_FOUND                 = 0x13,
+	INTEL_BOOTROM_STATUS_AES_PROD_KEY_FOUND           = 0x1A,
+	INTEL_BOOTROM_STATUS_RSA_FAILED                   = 0x50,
+	INTEL_BOOTROM_STATUS_PAVPC_FAILED                 = 0x73,
+	INTEL_BOOTROM_STATUS_WOPCM_FAILED                 = 0x74,
+	INTEL_BOOTROM_STATUS_LOADLOC_FAILED               = 0x75,
+	INTEL_BOOTROM_STATUS_JUMP_PASSED                  = 0x76,
+	INTEL_BOOTROM_STATUS_JUMP_FAILED                  = 0x77,
+	INTEL_BOOTROM_STATUS_RC6CTXCONFIG_FAILED          = 0x79,
+	INTEL_BOOTROM_STATUS_MPUMAP_INCORRECT             = 0x7A,
+	INTEL_BOOTROM_STATUS_EXCEPTION                    = 0x7E,
+};
+
 #endif /* _ABI_GUC_ERRORS_ABI_H */
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c
index e73d4440c5e8..1d9fdfb11268 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c
@@ -6,6 +6,7 @@
 #include "gt/intel_engine_pm.h"
 #include "gt/intel_gpu_commands.h"
 #include "gt/intel_gt.h"
+#include "gt/intel_gt_print.h"
 #include "gt/intel_ring.h"
 #include "intel_gsc_fw.h"
 
@@ -88,9 +89,8 @@ out_rq:
 	i915_request_put(rq);
 
 	if (err)
-		drm_err(&gsc_uc_to_gt(gsc)->i915->drm,
-			"Request submission for GSC load failed (%d)\n",
-			err);
+		gt_err(gsc_uc_to_gt(gsc), "Request submission for GSC load failed %pe\n",
+		       ERR_PTR(err));
 
 	return err;
 }
@@ -200,8 +200,7 @@ int intel_gsc_uc_fw_upload(struct intel_gsc_uc *gsc)
 	/* FW is not fully operational until we enable SW proxy */
 	intel_uc_fw_change_status(gsc_fw, INTEL_UC_FIRMWARE_TRANSFERRED);
 
-	drm_info(&gt->i915->drm, "Loaded GSC firmware %s\n",
-		 gsc_fw->file_selected.path);
+	gt_info(gt, "Loaded GSC firmware %s\n", gsc_fw->file_selected.path);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.h
index 4b5dbb44afb4..f4c1106bb2a9 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.h
@@ -9,7 +9,9 @@
 #include <linux/types.h>
 
 struct intel_gsc_uc;
+struct intel_uncore;
 
 int intel_gsc_uc_fw_upload(struct intel_gsc_uc *gsc);
 bool intel_gsc_uc_fw_init_done(struct intel_gsc_uc *gsc);
+
 #endif
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.c
index fd21dbd2663b..2d5b70b3384c 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.c
@@ -6,6 +6,7 @@
 #include <linux/types.h>
 
 #include "gt/intel_gt.h"
+#include "gt/intel_gt_print.h"
 #include "intel_gsc_uc.h"
 #include "intel_gsc_fw.h"
 #include "i915_drv.h"
@@ -59,7 +60,6 @@ int intel_gsc_uc_init(struct intel_gsc_uc *gsc)
 {
 	static struct lock_class_key gsc_lock;
 	struct intel_gt *gt = gsc_uc_to_gt(gsc);
-	struct drm_i915_private *i915 = gt->i915;
 	struct intel_engine_cs *engine = gt->engine[GSC0];
 	struct intel_context *ce;
 	struct i915_vma *vma;
@@ -81,8 +81,7 @@ int intel_gsc_uc_init(struct intel_gsc_uc *gsc)
 						I915_GEM_HWS_GSC_ADDR,
 						&gsc_lock, "gsc_context");
 	if (IS_ERR(ce)) {
-		drm_err(&gt->i915->drm,
-			"failed to create GSC CS ctx for FW communication\n");
+		gt_err(gt, "failed to create GSC CS ctx for FW communication\n");
 		err =  PTR_ERR(ce);
 		goto out_vma;
 	}
@@ -98,7 +97,7 @@ out_vma:
 out_fw:
 	intel_uc_fw_fini(&gsc->fw);
 out:
-	i915_probe_error(i915, "failed with %d\n", err);
+	gt_probe_error(gt, "GSC init failed %pe\n", ERR_PTR(err));
 	return err;
 }
 
@@ -117,7 +116,7 @@ void intel_gsc_uc_fini(struct intel_gsc_uc *gsc)
 	intel_uc_fw_fini(&gsc->fw);
 }
 
-void intel_gsc_uc_suspend(struct intel_gsc_uc *gsc)
+void intel_gsc_uc_flush_work(struct intel_gsc_uc *gsc)
 {
 	if (!intel_uc_fw_is_loadable(&gsc->fw))
 		return;
@@ -125,6 +124,25 @@ void intel_gsc_uc_suspend(struct intel_gsc_uc *gsc)
 	flush_work(&gsc->work);
 }
 
+void intel_gsc_uc_resume(struct intel_gsc_uc *gsc)
+{
+	if (!intel_uc_fw_is_loadable(&gsc->fw))
+		return;
+
+	/*
+	 * we only want to start the GSC worker from here in the actual resume
+	 * flow and not during driver load. This is because GSC load is slow and
+	 * therefore we want to make sure that the default state init completes
+	 * first to not slow down the init thread. A separate call to
+	 * intel_gsc_uc_load_start will ensure that the GSC is loaded during
+	 * driver load.
+	 */
+	if (!gsc_uc_to_gt(gsc)->engine[GSC0]->default_state)
+		return;
+
+	intel_gsc_uc_load_start(gsc);
+}
+
 void intel_gsc_uc_load_start(struct intel_gsc_uc *gsc)
 {
 	if (!intel_uc_fw_is_loadable(&gsc->fw))
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.h b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.h
index 03fd0a8e8db1..5f50fa1ff8b9 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.h
@@ -26,6 +26,8 @@ void intel_gsc_uc_init_early(struct intel_gsc_uc *gsc);
 int intel_gsc_uc_init(struct intel_gsc_uc *gsc);
 void intel_gsc_uc_fini(struct intel_gsc_uc *gsc);
 void intel_gsc_uc_suspend(struct intel_gsc_uc *gsc);
+void intel_gsc_uc_resume(struct intel_gsc_uc *gsc);
+void intel_gsc_uc_flush_work(struct intel_gsc_uc *gsc);
 void intel_gsc_uc_load_start(struct intel_gsc_uc *gsc);
 
 static inline bool intel_gsc_uc_is_supported(struct intel_gsc_uc *gsc)
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.c b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.c
new file mode 100644
index 000000000000..ea0da06e2f39
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.c
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#include "gt/intel_engine_pm.h"
+#include "gt/intel_gpu_commands.h"
+#include "gt/intel_gt.h"
+#include "gt/intel_ring.h"
+#include "intel_gsc_uc_heci_cmd_submit.h"
+
+struct gsc_heci_pkt {
+	u64 addr_in;
+	u32 size_in;
+	u64 addr_out;
+	u32 size_out;
+};
+
+static int emit_gsc_heci_pkt(struct i915_request *rq, struct gsc_heci_pkt *pkt)
+{
+	u32 *cs;
+
+	cs = intel_ring_begin(rq, 8);
+	if (IS_ERR(cs))
+		return PTR_ERR(cs);
+
+	*cs++ = GSC_HECI_CMD_PKT;
+	*cs++ = lower_32_bits(pkt->addr_in);
+	*cs++ = upper_32_bits(pkt->addr_in);
+	*cs++ = pkt->size_in;
+	*cs++ = lower_32_bits(pkt->addr_out);
+	*cs++ = upper_32_bits(pkt->addr_out);
+	*cs++ = pkt->size_out;
+	*cs++ = 0;
+
+	intel_ring_advance(rq, cs);
+
+	return 0;
+}
+
+int intel_gsc_uc_heci_cmd_submit_packet(struct intel_gsc_uc *gsc, u64 addr_in,
+					u32 size_in, u64 addr_out,
+					u32 size_out)
+{
+	struct intel_context *ce = gsc->ce;
+	struct i915_request *rq;
+	struct gsc_heci_pkt pkt = {
+	.addr_in = addr_in,
+	.size_in = size_in,
+	.addr_out = addr_out,
+	.size_out = size_out
+	};
+	int err;
+
+	if (!ce)
+		return -ENODEV;
+
+	rq = i915_request_create(ce);
+	if (IS_ERR(rq))
+		return PTR_ERR(rq);
+
+	if (ce->engine->emit_init_breadcrumb) {
+		err = ce->engine->emit_init_breadcrumb(rq);
+		if (err)
+			goto out_rq;
+	}
+
+	err = emit_gsc_heci_pkt(rq, &pkt);
+
+	if (err)
+		goto out_rq;
+
+	err = ce->engine->emit_flush(rq, 0);
+
+out_rq:
+	i915_request_get(rq);
+
+	if (unlikely(err))
+		i915_request_set_error_once(rq, err);
+
+	i915_request_add(rq);
+
+	if (!err && i915_request_wait(rq, 0, msecs_to_jiffies(500)) < 0)
+		err = -ETIME;
+
+	i915_request_put(rq);
+
+	if (err)
+		drm_err(&gsc_uc_to_gt(gsc)->i915->drm,
+			"Request submission for GSC heci cmd failed (%d)\n",
+			err);
+
+	return err;
+}
+
+void intel_gsc_uc_heci_cmd_emit_mtl_header(struct intel_gsc_mtl_header *header,
+					   u8 heci_client_id, u32 message_size,
+					   u64 host_session_id)
+{
+	host_session_id &= ~HOST_SESSION_MASK;
+	if (heci_client_id == HECI_MEADDRESS_PXP)
+		host_session_id |= HOST_SESSION_PXP_SINGLE;
+
+	header->validity_marker = GSC_HECI_VALIDITY_MARKER;
+	header->heci_client_id = heci_client_id;
+	header->host_session_handle = host_session_id;
+	header->header_version = MTL_GSC_HEADER_VERSION;
+	header->message_size = message_size;
+}
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.h b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.h
new file mode 100644
index 000000000000..3d56ae501991
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.h
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef _INTEL_GSC_UC_HECI_CMD_SUBMIT_H_
+#define _INTEL_GSC_UC_HECI_CMD_SUBMIT_H_
+
+#include <linux/types.h>
+
+struct intel_gsc_uc;
+struct intel_gsc_mtl_header {
+	u32 validity_marker;
+#define GSC_HECI_VALIDITY_MARKER 0xA578875A
+
+	u8 heci_client_id;
+#define HECI_MEADDRESS_PXP 17
+#define HECI_MEADDRESS_HDCP 18
+
+	u8 reserved1;
+
+	u16 header_version;
+#define MTL_GSC_HEADER_VERSION 1
+
+	/*
+	 * FW allows host to decide host_session handle
+	 * as it sees fit.
+	 * For intertracebility reserving select bits(60-63)
+	 * to differentiate caller-target subsystem
+	 * 0000 - HDCP
+	 * 0001 - PXP Single Session
+	 */
+	u64 host_session_handle;
+#define HOST_SESSION_MASK	REG_GENMASK64(63, 60)
+#define HOST_SESSION_PXP_SINGLE BIT_ULL(60)
+	u64 gsc_message_handle;
+
+	u32 message_size; /* lower 20 bits only, upper 12 are reserved */
+
+	/*
+	 * Flags mask:
+	 * Bit 0: Pending
+	 * Bit 1: Session Cleanup;
+	 * Bits 2-15: Flags
+	 * Bits 16-31: Extension Size
+	 * According to internal spec flags are either input or output
+	 * we distinguish the flags using OUTFLAG or INFLAG
+	 */
+	u32 flags;
+#define GSC_OUTFLAG_MSG_PENDING	1
+
+	u32 status;
+} __packed;
+
+int intel_gsc_uc_heci_cmd_submit_packet(struct intel_gsc_uc *gsc,
+					u64 addr_in, u32 size_in,
+					u64 addr_out, u32 size_out);
+void intel_gsc_uc_heci_cmd_emit_mtl_header(struct intel_gsc_mtl_header *header,
+					   u8 heci_client_id, u32 message_size,
+					   u64 host_session_id);
+#endif
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
index bb4dfe707a7d..e46aac1a41e6 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
@@ -42,6 +42,8 @@ struct intel_guc {
 	/** @capture: the error-state-capture module's data and objects */
 	struct intel_guc_state_capture *capture;
 
+	struct dentry *dbgfs_node;
+
 	/** @sched_engine: Global engine used to submit requests to GuC */
 	struct i915_sched_engine *sched_engine;
 	/**
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c
index 710999d7189e..cf49188db6a6 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c
@@ -15,6 +15,7 @@
 #include "guc_capture_fwif.h"
 #include "intel_guc_capture.h"
 #include "intel_guc_fwif.h"
+#include "intel_guc_print.h"
 #include "i915_drv.h"
 #include "i915_gpu_error.h"
 #include "i915_irq.h"
@@ -353,7 +354,6 @@ guc_capture_alloc_steered_lists_xe_hpg(struct intel_guc *guc,
 				       u32 ipver)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
-	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	struct sseu_dev_info *sseu;
 	int slice, subslice, i, iter, num_steer_regs, num_tot_regs = 0;
 	const struct __guc_mmio_reg_descr_group *list;
@@ -402,7 +402,7 @@ guc_capture_alloc_steered_lists_xe_hpg(struct intel_guc *guc,
 		}
 	}
 
-	drm_dbg(&i915->drm, "GuC-capture found %d-ext-regs.\n", num_tot_regs);
+	guc_dbg(guc, "capture found %d ext-regs.\n", num_tot_regs);
 	guc->capture->extlists = extlists;
 }
 
@@ -477,7 +477,6 @@ guc_capture_list_init(struct intel_guc *guc, u32 owner, u32 type, u32 classid,
 		      struct guc_mmio_reg *ptr, u16 num_entries)
 {
 	u32 i = 0, j = 0;
-	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	const struct __guc_mmio_reg_descr_group *reglists = guc->capture->reglists;
 	struct __guc_mmio_reg_descr_group *extlists = guc->capture->extlists;
 	const struct __guc_mmio_reg_descr_group *match;
@@ -509,8 +508,7 @@ guc_capture_list_init(struct intel_guc *guc, u32 owner, u32 type, u32 classid,
 		}
 	}
 	if (i < num_entries)
-		drm_dbg(&i915->drm, "GuC-capture: Init reglist short %d out %d.\n",
-			(int)i, (int)num_entries);
+		guc_dbg(guc, "Got short capture reglist init: %d out %d.\n", i, num_entries);
 
 	return 0;
 }
@@ -540,12 +538,11 @@ guc_capture_getlistsize(struct intel_guc *guc, u32 owner, u32 type, u32 classid,
 			size_t *size, bool is_purpose_est)
 {
 	struct intel_guc_state_capture *gc = guc->capture;
-	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	struct __guc_capture_ads_cache *cache = &gc->ads_cache[owner][type][classid];
 	int num_regs;
 
 	if (!gc->reglists) {
-		drm_warn(&i915->drm, "GuC-capture: No reglist on this device\n");
+		guc_warn(guc, "No capture reglist for this device\n");
 		return -ENODEV;
 	}
 
@@ -557,9 +554,9 @@ guc_capture_getlistsize(struct intel_guc *guc, u32 owner, u32 type, u32 classid,
 	if (!is_purpose_est && owner == GUC_CAPTURE_LIST_INDEX_PF &&
 	    !guc_capture_get_one_list(gc->reglists, owner, type, classid)) {
 		if (type == GUC_CAPTURE_LIST_TYPE_GLOBAL)
-			drm_warn(&i915->drm, "Missing GuC-Err-Cap reglist Global!\n");
+			guc_warn(guc, "Missing capture reglist: global!\n");
 		else
-			drm_warn(&i915->drm, "Missing GuC-Err-Cap reglist %s(%u):%s(%u)!\n",
+			guc_warn(guc, "Missing capture reglist: %s(%u):%s(%u)!\n",
 				 __stringify_type(type), type,
 				 __stringify_engclass(classid), classid);
 		return -ENODATA;
@@ -592,7 +589,6 @@ intel_guc_capture_getlist(struct intel_guc *guc, u32 owner, u32 type, u32 classi
 {
 	struct intel_guc_state_capture *gc = guc->capture;
 	struct __guc_capture_ads_cache *cache = &gc->ads_cache[owner][type][classid];
-	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	struct guc_debug_capture_list *listnode;
 	int ret, num_regs;
 	u8 *caplist, *tmp;
@@ -623,7 +619,7 @@ intel_guc_capture_getlist(struct intel_guc *guc, u32 owner, u32 type, u32 classi
 
 	caplist = kzalloc(size, GFP_KERNEL);
 	if (!caplist) {
-		drm_dbg(&i915->drm, "GuC-capture: failed to alloc cached caplist");
+		guc_dbg(guc, "Failed to alloc cached register capture list");
 		return -ENOMEM;
 	}
 
@@ -653,7 +649,6 @@ intel_guc_capture_getnullheader(struct intel_guc *guc,
 				void **outptr, size_t *size)
 {
 	struct intel_guc_state_capture *gc = guc->capture;
-	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	int tmp = sizeof(u32) * 4;
 	void *null_header;
 
@@ -665,7 +660,7 @@ intel_guc_capture_getnullheader(struct intel_guc *guc,
 
 	null_header = kzalloc(tmp, GFP_KERNEL);
 	if (!null_header) {
-		drm_dbg(&i915->drm, "GuC-capture: failed to alloc cached nulllist");
+		guc_dbg(guc, "Failed to alloc cached register capture null list");
 		return -ENOMEM;
 	}
 
@@ -727,7 +722,6 @@ guc_capture_output_min_size_est(struct intel_guc *guc)
 
 static void check_guc_capture_size(struct intel_guc *guc)
 {
-	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	int min_size = guc_capture_output_min_size_est(guc);
 	int spare_size = min_size * GUC_CAPTURE_OVERBUFFER_MULTIPLIER;
 	u32 buffer_size = intel_guc_log_section_size_capture(&guc->log);
@@ -741,13 +735,13 @@ static void check_guc_capture_size(struct intel_guc *guc)
 	 * INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE.
 	 */
 	if (min_size < 0)
-		drm_warn(&i915->drm, "Failed to calculate GuC error state capture buffer minimum size: %d!\n",
+		guc_warn(guc, "Failed to calculate error state capture buffer minimum size: %d!\n",
 			 min_size);
 	else if (min_size > buffer_size)
-		drm_warn(&i915->drm, "GuC error state capture buffer maybe small: %d < %d\n",
+		guc_warn(guc, "Error state capture buffer maybe small: %d < %d\n",
 			 buffer_size, min_size);
 	else if (spare_size > buffer_size)
-		drm_dbg(&i915->drm, "GuC error state capture buffer lacks spare size: %d < %d (min = %d)\n",
+		guc_dbg(guc, "Error state capture buffer lacks spare size: %d < %d (min = %d)\n",
 			buffer_size, spare_size, min_size);
 }
 
@@ -848,7 +842,6 @@ static int
 guc_capture_log_remove_dw(struct intel_guc *guc, struct __guc_capture_bufstate *buf,
 			  u32 *dw)
 {
-	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	int tries = 2;
 	int avail = 0;
 	u32 *src_data;
@@ -865,7 +858,7 @@ guc_capture_log_remove_dw(struct intel_guc *guc, struct __guc_capture_bufstate *
 			return 4;
 		}
 		if (avail)
-			drm_dbg(&i915->drm, "GuC-Cap-Logs not dword aligned, skipping.\n");
+			guc_dbg(guc, "Register capture log not dword aligned, skipping.\n");
 		buf->rd = 0;
 	}
 
@@ -1118,13 +1111,12 @@ static void
 __guc_capture_create_prealloc_nodes(struct intel_guc *guc)
 {
 	struct __guc_capture_parsed_output *node = NULL;
-	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	int i;
 
 	for (i = 0; i < PREALLOC_NODES_MAX_COUNT; ++i) {
 		node = guc_capture_alloc_one_node(guc);
 		if (!node) {
-			drm_warn(&i915->drm, "GuC Capture pre-alloc-cache failure\n");
+			guc_warn(guc, "Register capture pre-alloc-cache failure\n");
 			/* dont free the priors, use what we got and cleanup at shutdown */
 			return;
 		}
@@ -1169,7 +1161,6 @@ guc_capture_create_prealloc_nodes(struct intel_guc *guc)
 static int
 guc_capture_extract_reglists(struct intel_guc *guc, struct __guc_capture_bufstate *buf)
 {
-	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	struct guc_state_capture_group_header_t ghdr = {0};
 	struct guc_state_capture_header_t hdr = {0};
 	struct __guc_capture_parsed_output *node = NULL;
@@ -1183,7 +1174,7 @@ guc_capture_extract_reglists(struct intel_guc *guc, struct __guc_capture_bufstat
 	if (!i)
 		return -ENODATA;
 	if (i % sizeof(u32)) {
-		drm_warn(&i915->drm, "GuC Capture new entries unaligned\n");
+		guc_warn(guc, "Got mis-aligned register capture entries\n");
 		ret = -EIO;
 		goto bailout;
 	}
@@ -1301,7 +1292,7 @@ guc_capture_extract_reglists(struct intel_guc *guc, struct __guc_capture_bufstat
 				break;
 			}
 			if (datatype != GUC_CAPTURE_LIST_TYPE_GLOBAL)
-				drm_dbg(&i915->drm, "GuC Capture missing global dump: %08x!\n",
+				guc_dbg(guc, "Register capture missing global dump: %08x!\n",
 					datatype);
 		}
 		node->is_partial = is_partial;
@@ -1322,7 +1313,7 @@ guc_capture_extract_reglists(struct intel_guc *guc, struct __guc_capture_bufstat
 
 		numregs = FIELD_GET(CAP_HDR_NUM_MMIOS, hdr.num_mmios);
 		if (numregs > guc->capture->max_mmio_per_node) {
-			drm_dbg(&i915->drm, "GuC Capture list extraction clipped by prealloc!\n");
+			guc_dbg(guc, "Register capture list extraction clipped by prealloc!\n");
 			numregs = guc->capture->max_mmio_per_node;
 		}
 		node->reginfo[datatype].num_regs = numregs;
@@ -1367,7 +1358,6 @@ static void __guc_capture_process_output(struct intel_guc *guc)
 {
 	unsigned int buffer_size, read_offset, write_offset, full_count;
 	struct intel_uc *uc = container_of(guc, typeof(*uc), guc);
-	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	struct guc_log_buffer_state log_buf_state_local;
 	struct guc_log_buffer_state *log_buf_state;
 	struct __guc_capture_bufstate buf;
@@ -1403,7 +1393,8 @@ static void __guc_capture_process_output(struct intel_guc *guc)
 		write_offset = buffer_size;
 	} else if (unlikely((read_offset > buffer_size) ||
 			(write_offset > buffer_size))) {
-		drm_err(&i915->drm, "invalid GuC log capture buffer state!\n");
+		guc_err(guc, "Register capture buffer in invalid state: read = 0x%X, size = 0x%X!\n",
+			read_offset, buffer_size);
 		/* copy whole buffer as offsets are unreliable */
 		read_offset = 0;
 		write_offset = buffer_size;
@@ -1607,13 +1598,11 @@ void intel_guc_capture_get_matching_node(struct intel_gt *gt,
 					 struct intel_context *ce)
 {
 	struct __guc_capture_parsed_output *n, *ntmp;
-	struct drm_i915_private *i915;
 	struct intel_guc *guc;
 
 	if (!gt || !ee || !ce)
 		return;
 
-	i915 = gt->i915;
 	guc = &gt->uc.guc;
 	if (!guc->capture)
 		return;
@@ -1627,9 +1616,8 @@ void intel_guc_capture_get_matching_node(struct intel_gt *gt,
 	list_for_each_entry_safe(n, ntmp, &guc->capture->outlist, link) {
 		if (n->eng_inst == GUC_ID_TO_ENGINE_INSTANCE(ee->engine->guc_id) &&
 		    n->eng_class == GUC_ID_TO_ENGINE_CLASS(ee->engine->guc_id) &&
-		    n->guc_id && n->guc_id == ce->guc_id.id &&
-		    (n->lrca & CTX_GTT_ADDRESS_MASK) && (n->lrca & CTX_GTT_ADDRESS_MASK) ==
-		    (ce->lrc.lrca & CTX_GTT_ADDRESS_MASK)) {
+		    n->guc_id == ce->guc_id.id &&
+		    (n->lrca & CTX_GTT_ADDRESS_MASK) == (ce->lrc.lrca & CTX_GTT_ADDRESS_MASK)) {
 			list_del(&n->link);
 			ee->guc_capture_node = n;
 			ee->guc_capture = guc->capture;
@@ -1637,7 +1625,9 @@ void intel_guc_capture_get_matching_node(struct intel_gt *gt,
 			return;
 		}
 	}
-	drm_dbg(&i915->drm, "GuC capture can't match ee to node\n");
+
+	guc_warn(guc, "No register capture node found for 0x%04X / 0x%08X\n",
+		 ce->guc_id.id, ce->lrc.lrca);
 }
 
 void intel_guc_capture_process(struct intel_guc *guc)
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c
index 69133420c78b..6fda3aec5c66 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c
@@ -12,6 +12,7 @@
 #include "gt/intel_gt.h"
 #include "gt/intel_gt_mcr.h"
 #include "gt/intel_gt_regs.h"
+#include "gt/intel_rps.h"
 #include "intel_guc_fw.h"
 #include "intel_guc_print.h"
 #include "i915_drv.h"
@@ -88,31 +89,80 @@ static int guc_xfer_rsa(struct intel_uc_fw *guc_fw,
 /*
  * Read the GuC status register (GUC_STATUS) and store it in the
  * specified location; then return a boolean indicating whether
- * the value matches either of two values representing completion
- * of the GuC boot process.
+ * the value matches either completion or a known failure code.
  *
  * This is used for polling the GuC status in a wait_for()
  * loop below.
  */
-static inline bool guc_ready(struct intel_uncore *uncore, u32 *status)
+static inline bool guc_load_done(struct intel_uncore *uncore, u32 *status, bool *success)
 {
 	u32 val = intel_uncore_read(uncore, GUC_STATUS);
 	u32 uk_val = REG_FIELD_GET(GS_UKERNEL_MASK, val);
+	u32 br_val = REG_FIELD_GET(GS_BOOTROM_MASK, val);
 
 	*status = val;
-	return uk_val == INTEL_GUC_LOAD_STATUS_READY;
+	switch (uk_val) {
+	case INTEL_GUC_LOAD_STATUS_READY:
+		*success = true;
+		return true;
+
+	case INTEL_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH:
+	case INTEL_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH:
+	case INTEL_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE:
+	case INTEL_GUC_LOAD_STATUS_HWCONFIG_ERROR:
+	case INTEL_GUC_LOAD_STATUS_DPC_ERROR:
+	case INTEL_GUC_LOAD_STATUS_EXCEPTION:
+	case INTEL_GUC_LOAD_STATUS_INIT_DATA_INVALID:
+	case INTEL_GUC_LOAD_STATUS_MPU_DATA_INVALID:
+	case INTEL_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
+		*success = false;
+		return true;
+	}
+
+	switch (br_val) {
+	case INTEL_BOOTROM_STATUS_NO_KEY_FOUND:
+	case INTEL_BOOTROM_STATUS_RSA_FAILED:
+	case INTEL_BOOTROM_STATUS_PAVPC_FAILED:
+	case INTEL_BOOTROM_STATUS_WOPCM_FAILED:
+	case INTEL_BOOTROM_STATUS_LOADLOC_FAILED:
+	case INTEL_BOOTROM_STATUS_JUMP_FAILED:
+	case INTEL_BOOTROM_STATUS_RC6CTXCONFIG_FAILED:
+	case INTEL_BOOTROM_STATUS_MPUMAP_INCORRECT:
+	case INTEL_BOOTROM_STATUS_EXCEPTION:
+		*success = false;
+		return true;
+	}
+
+	return false;
 }
 
+/*
+ * Use a longer timeout for debug builds so that problems can be detected
+ * and analysed. But a shorter timeout for releases so that user's don't
+ * wait forever to find out there is a problem. Note that the only reason
+ * an end user should hit the timeout is in case of extreme thermal throttling.
+ * And a system that is that hot during boot is probably dead anyway!
+ */
+#if defined(CONFIG_DRM_I915_DEBUG_GEM)
+#define GUC_LOAD_RETRY_LIMIT	20
+#else
+#define GUC_LOAD_RETRY_LIMIT	3
+#endif
+
 static int guc_wait_ucode(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
 	struct intel_uncore *uncore = gt->uncore;
+	ktime_t before, after, delta;
+	bool success;
 	u32 status;
-	int ret;
+	int ret, count;
+	u64 delta_ms;
+	u32 before_freq;
 
 	/*
 	 * Wait for the GuC to start up.
-	 * NB: Docs recommend not using the interrupt for completion.
+	 *
 	 * Measurements indicate this should take no more than 20ms
 	 * (assuming the GT clock is at maximum frequency). So, a
 	 * timeout here indicates that the GuC has failed and is unusable.
@@ -126,29 +176,80 @@ static int guc_wait_ucode(struct intel_guc *guc)
 	 * issues to be resolved. In the meantime bump the timeout to
 	 * 200ms. Even at slowest clock, this should be sufficient. And
 	 * in the working case, a larger timeout makes no difference.
+	 *
+	 * IFWI updates have also been seen to cause sporadic failures due to
+	 * the requested frequency not being granted and thus the firmware
+	 * load is attempted at minimum frequency. That can lead to load times
+	 * in the seconds range. However, there is a limit on how long an
+	 * individual wait_for() can wait. So wrap it in a loop.
 	 */
-	ret = wait_for(guc_ready(uncore, &status), 200);
-	if (ret) {
-		guc_info(guc, "load failed: status = 0x%08X\n", status);
-		guc_info(guc, "load failed: status: Reset = %d, "
-			"BootROM = 0x%02X, UKernel = 0x%02X, "
-			"MIA = 0x%02X, Auth = 0x%02X\n",
-			REG_FIELD_GET(GS_MIA_IN_RESET, status),
-			REG_FIELD_GET(GS_BOOTROM_MASK, status),
-			REG_FIELD_GET(GS_UKERNEL_MASK, status),
-			REG_FIELD_GET(GS_MIA_MASK, status),
-			REG_FIELD_GET(GS_AUTH_STATUS_MASK, status));
-
-		if ((status & GS_BOOTROM_MASK) == GS_BOOTROM_RSA_FAILED) {
+	before_freq = intel_rps_read_actual_frequency(&uncore->gt->rps);
+	before = ktime_get();
+	for (count = 0; count < GUC_LOAD_RETRY_LIMIT; count++) {
+		ret = wait_for(guc_load_done(uncore, &status, &success), 1000);
+		if (!ret || !success)
+			break;
+
+		guc_dbg(guc, "load still in progress, count = %d, freq = %dMHz\n",
+			count, intel_rps_read_actual_frequency(&uncore->gt->rps));
+	}
+	after = ktime_get();
+	delta = ktime_sub(after, before);
+	delta_ms = ktime_to_ms(delta);
+	if (ret || !success) {
+		u32 ukernel = REG_FIELD_GET(GS_UKERNEL_MASK, status);
+		u32 bootrom = REG_FIELD_GET(GS_BOOTROM_MASK, status);
+
+		guc_info(guc, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz, ret = %d\n",
+			 status, delta_ms, intel_rps_read_actual_frequency(&uncore->gt->rps), ret);
+		guc_info(guc, "load failed: status: Reset = %d, BootROM = 0x%02X, UKernel = 0x%02X, MIA = 0x%02X, Auth = 0x%02X\n",
+			 REG_FIELD_GET(GS_MIA_IN_RESET, status),
+			 bootrom, ukernel,
+			 REG_FIELD_GET(GS_MIA_MASK, status),
+			 REG_FIELD_GET(GS_AUTH_STATUS_MASK, status));
+
+		switch (bootrom) {
+		case INTEL_BOOTROM_STATUS_NO_KEY_FOUND:
+			guc_info(guc, "invalid key requested, header = 0x%08X\n",
+				 intel_uncore_read(uncore, GUC_HEADER_INFO));
+			ret = -ENOEXEC;
+			break;
+
+		case INTEL_BOOTROM_STATUS_RSA_FAILED:
 			guc_info(guc, "firmware signature verification failed\n");
 			ret = -ENOEXEC;
+			break;
 		}
 
-		if (REG_FIELD_GET(GS_UKERNEL_MASK, status) == INTEL_GUC_LOAD_STATUS_EXCEPTION) {
+		switch (ukernel) {
+		case INTEL_GUC_LOAD_STATUS_EXCEPTION:
 			guc_info(guc, "firmware exception. EIP: %#x\n",
 				 intel_uncore_read(uncore, SOFT_SCRATCH(13)));
 			ret = -ENXIO;
+			break;
+
+		case INTEL_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
+			guc_info(guc, "illegal register in save/restore workaround list\n");
+			ret = -EPERM;
+			break;
+
+		case INTEL_GUC_LOAD_STATUS_HWCONFIG_START:
+			guc_info(guc, "still extracting hwconfig table.\n");
+			ret = -ETIMEDOUT;
+			break;
 		}
+
+		/* Uncommon/unexpected error, see earlier status code print for details */
+		if (ret == 0)
+			ret = -ENXIO;
+	} else if (delta_ms > 200) {
+		guc_warn(guc, "excessive init time: %lldms! [freq = %dMHz, before = %dMHz, status = 0x%08X, count = %d, ret = %d]\n",
+			 delta_ms, intel_rps_read_actual_frequency(&uncore->gt->rps),
+			 before_freq, status, count, ret);
+	} else {
+		guc_dbg(guc, "init took %lldms, freq = %dMHz, before = %dMHz, status = 0x%08X, count = %d, ret = %d\n",
+			delta_ms, intel_rps_read_actual_frequency(&uncore->gt->rps),
+			before_freq, status, count, ret);
 	}
 
 	return ret;
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_hwconfig.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_hwconfig.c
index 4781fccc2687..852bea0208ce 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_hwconfig.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_hwconfig.c
@@ -102,7 +102,7 @@ static bool has_table(struct drm_i915_private *i915)
 	return false;
 }
 
-/**
+/*
  * intel_guc_hwconfig_init - Initialize the HWConfig
  *
  * Retrieve the HWConfig table from the GuC and save it locally.
@@ -136,7 +136,7 @@ static int guc_hwconfig_init(struct intel_gt *gt)
 	return 0;
 }
 
-/**
+/*
  * intel_gt_init_hwconfig - Initialize the HWConfig if available
  *
  * Retrieve the HWConfig table if available on the current platform.
@@ -149,7 +149,7 @@ int intel_gt_init_hwconfig(struct intel_gt *gt)
 	return guc_hwconfig_init(gt);
 }
 
-/**
+/*
  * intel_gt_fini_hwconfig - Finalize the HWConfig
  *
  * Free up the memory allocation holding the table.
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c
index c3792ddeec80..55bc8b55fbc0 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c
@@ -333,8 +333,7 @@ bool intel_guc_check_log_buf_overflow(struct intel_guc_log *log,
 			log->stats[type].sampled_overflow += 16;
 		}
 
-		dev_notice_ratelimited(guc_to_gt(log_to_guc(log))->i915->drm.dev,
-				       "GuC log buffer overflow\n");
+		guc_notice_ratelimited(log_to_guc(log), "log buffer overflow\n");
 	}
 
 	return overflow;
@@ -521,7 +520,7 @@ void intel_guc_log_init_early(struct intel_guc_log *log)
 static int guc_log_relay_create(struct intel_guc_log *log)
 {
 	struct intel_guc *guc = log_to_guc(log);
-	struct drm_i915_private *dev_priv = guc_to_gt(guc)->i915;
+	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	struct rchan *guc_log_relay_chan;
 	size_t n_subbufs, subbuf_size;
 	int ret;
@@ -543,10 +542,13 @@ static int guc_log_relay_create(struct intel_guc_log *log)
 	 */
 	n_subbufs = 8;
 
+	if (!guc->dbgfs_node)
+		return -ENOENT;
+
 	guc_log_relay_chan = relay_open("guc_log",
-					dev_priv->drm.primary->debugfs_root,
+					guc->dbgfs_node,
 					subbuf_size, n_subbufs,
-					&relay_callbacks, dev_priv);
+					&relay_callbacks, i915);
 	if (!guc_log_relay_chan) {
 		guc_err(guc, "Couldn't create relay channel for logging\n");
 
@@ -571,7 +573,7 @@ static void guc_log_relay_destroy(struct intel_guc_log *log)
 static void guc_log_copy_debuglogs_for_relay(struct intel_guc_log *log)
 {
 	struct intel_guc *guc = log_to_guc(log);
-	struct drm_i915_private *dev_priv = guc_to_gt(guc)->i915;
+	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	intel_wakeref_t wakeref;
 
 	_guc_log_copy_debuglogs_for_relay(log);
@@ -580,7 +582,7 @@ static void guc_log_copy_debuglogs_for_relay(struct intel_guc_log *log)
 	 * Generally device is expected to be active only at this
 	 * time, so get/put should be really quick.
 	 */
-	with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref)
+	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
 		guc_action_flush_log_complete(guc);
 }
 
@@ -662,7 +664,7 @@ void intel_guc_log_destroy(struct intel_guc_log *log)
 int intel_guc_log_set_level(struct intel_guc_log *log, u32 level)
 {
 	struct intel_guc *guc = log_to_guc(log);
-	struct drm_i915_private *dev_priv = guc_to_gt(guc)->i915;
+	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 	intel_wakeref_t wakeref;
 	int ret = 0;
 
@@ -676,12 +678,12 @@ int intel_guc_log_set_level(struct intel_guc_log *log, u32 level)
 	if (level < GUC_LOG_LEVEL_DISABLED || level > GUC_LOG_LEVEL_MAX)
 		return -EINVAL;
 
-	mutex_lock(&dev_priv->drm.struct_mutex);
+	mutex_lock(&i915->drm.struct_mutex);
 
 	if (log->level == level)
 		goto out_unlock;
 
-	with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref)
+	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
 		ret = guc_action_control_log(guc,
 					     GUC_LOG_LEVEL_IS_VERBOSE(level),
 					     GUC_LOG_LEVEL_IS_ENABLED(level),
@@ -694,7 +696,7 @@ int intel_guc_log_set_level(struct intel_guc_log *log, u32 level)
 	log->level = level;
 
 out_unlock:
-	mutex_unlock(&dev_priv->drm.struct_mutex);
+	mutex_unlock(&i915->drm.struct_mutex);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_print.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_print.h
index e75989d4ba06..2465d05638b4 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_print.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_print.h
@@ -30,6 +30,9 @@
 #define guc_err_ratelimited(_guc, _fmt, ...) \
 	guc_printk((_guc), err_ratelimited, _fmt, ##__VA_ARGS__)
 
+#define guc_notice_ratelimited(_guc, _fmt, ...) \
+	guc_printk((_guc), notice_ratelimited, _fmt, ##__VA_ARGS__)
+
 #define guc_probe_error(_guc, _fmt, ...) \
 	guc_printk((_guc), probe_error, _fmt, ##__VA_ARGS__)
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_rc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_rc.c
index 8f8dd05835c5..1adec6de223c 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_rc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_rc.c
@@ -6,6 +6,7 @@
 #include <linux/string_helpers.h>
 
 #include "intel_guc_rc.h"
+#include "intel_guc_print.h"
 #include "gt/intel_gt.h"
 #include "i915_drv.h"
 
@@ -59,13 +60,12 @@ static int __guc_rc_control(struct intel_guc *guc, bool enable)
 
 	ret = guc_action_control_gucrc(guc, enable);
 	if (ret) {
-		i915_probe_error(guc_to_gt(guc)->i915, "Failed to %s GuC RC (%pe)\n",
-				 str_enable_disable(enable), ERR_PTR(ret));
+		guc_probe_error(guc, "Failed to %s RC (%pe)\n",
+				str_enable_disable(enable), ERR_PTR(ret));
 		return ret;
 	}
 
-	drm_info(&gt->i915->drm, "GuC RC: %s\n",
-		 str_enabled_disabled(enable));
+	guc_info(guc, "RC %s\n", str_enabled_disabled(enable));
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h
index 9915de32e894..3fd798837502 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h
@@ -18,8 +18,6 @@
 #define   GS_MIA_IN_RESET		  (0x01 << GS_RESET_SHIFT)
 #define   GS_BOOTROM_SHIFT		1
 #define   GS_BOOTROM_MASK		  (0x7F << GS_BOOTROM_SHIFT)
-#define   GS_BOOTROM_RSA_FAILED		  (0x50 << GS_BOOTROM_SHIFT)
-#define   GS_BOOTROM_JUMP_PASSED	  (0x76 << GS_BOOTROM_SHIFT)
 #define   GS_UKERNEL_SHIFT		8
 #define   GS_UKERNEL_MASK		  (0xFF << GS_UKERNEL_SHIFT)
 #define   GS_MIA_SHIFT			16
@@ -32,6 +30,8 @@
 #define   GS_AUTH_STATUS_BAD		  (0x01 << GS_AUTH_STATUS_SHIFT)
 #define   GS_AUTH_STATUS_GOOD		  (0x02 << GS_AUTH_STATUS_SHIFT)
 
+#define GUC_HEADER_INFO			_MMIO(0xc014)
+
 #define SOFT_SCRATCH(n)			_MMIO(0xc180 + (n) * 4)
 #define SOFT_SCRATCH_COUNT		16
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
index 63464933cbce..026d73855f36 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
@@ -9,6 +9,7 @@
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "intel_guc_slpc.h"
+#include "intel_guc_print.h"
 #include "intel_mchbar_regs.h"
 #include "gt/intel_gt.h"
 #include "gt/intel_gt_regs.h"
@@ -171,14 +172,12 @@ static int guc_action_slpc_query(struct intel_guc *guc, u32 offset)
 static int slpc_query_task_state(struct intel_guc_slpc *slpc)
 {
 	struct intel_guc *guc = slpc_to_guc(slpc);
-	struct drm_i915_private *i915 = slpc_to_i915(slpc);
 	u32 offset = intel_guc_ggtt_offset(guc, slpc->vma);
 	int ret;
 
 	ret = guc_action_slpc_query(guc, offset);
 	if (unlikely(ret))
-		i915_probe_error(i915, "Failed to query task state (%pe)\n",
-				 ERR_PTR(ret));
+		guc_probe_error(guc, "Failed to query task state: %pe\n", ERR_PTR(ret));
 
 	drm_clflush_virt_range(slpc->vaddr, SLPC_PAGE_SIZE_BYTES);
 
@@ -188,15 +187,14 @@ static int slpc_query_task_state(struct intel_guc_slpc *slpc)
 static int slpc_set_param(struct intel_guc_slpc *slpc, u8 id, u32 value)
 {
 	struct intel_guc *guc = slpc_to_guc(slpc);
-	struct drm_i915_private *i915 = slpc_to_i915(slpc);
 	int ret;
 
 	GEM_BUG_ON(id >= SLPC_MAX_PARAM);
 
 	ret = guc_action_slpc_set_param(guc, id, value);
 	if (ret)
-		i915_probe_error(i915, "Failed to set param %d to %u (%pe)\n",
-				 id, value, ERR_PTR(ret));
+		guc_probe_error(guc, "Failed to set param %d to %u: %pe\n",
+				id, value, ERR_PTR(ret));
 
 	return ret;
 }
@@ -212,8 +210,8 @@ static int slpc_unset_param(struct intel_guc_slpc *slpc, u8 id)
 
 static int slpc_force_min_freq(struct intel_guc_slpc *slpc, u32 freq)
 {
-	struct drm_i915_private *i915 = slpc_to_i915(slpc);
 	struct intel_guc *guc = slpc_to_guc(slpc);
+	struct drm_i915_private *i915 = slpc_to_i915(slpc);
 	intel_wakeref_t wakeref;
 	int ret = 0;
 
@@ -236,9 +234,8 @@ static int slpc_force_min_freq(struct intel_guc_slpc *slpc, u32 freq)
 					SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ,
 					freq);
 		if (ret)
-			drm_notice(&i915->drm,
-				   "Failed to send set_param for min freq(%d): (%d)\n",
-				   freq, ret);
+			guc_notice(guc, "Failed to send set_param for min freq(%d): %pe\n",
+				   freq, ERR_PTR(ret));
 	}
 
 	return ret;
@@ -267,7 +264,6 @@ static void slpc_boost_work(struct work_struct *work)
 int intel_guc_slpc_init(struct intel_guc_slpc *slpc)
 {
 	struct intel_guc *guc = slpc_to_guc(slpc);
-	struct drm_i915_private *i915 = slpc_to_i915(slpc);
 	u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data));
 	int err;
 
@@ -275,9 +271,7 @@ int intel_guc_slpc_init(struct intel_guc_slpc *slpc)
 
 	err = intel_guc_allocate_and_map_vma(guc, size, &slpc->vma, (void **)&slpc->vaddr);
 	if (unlikely(err)) {
-		i915_probe_error(i915,
-				 "Failed to allocate SLPC struct (err=%pe)\n",
-				 ERR_PTR(err));
+		guc_probe_error(guc, "Failed to allocate SLPC struct: %pe\n", ERR_PTR(err));
 		return err;
 	}
 
@@ -338,7 +332,6 @@ static int guc_action_slpc_reset(struct intel_guc *guc, u32 offset)
 
 static int slpc_reset(struct intel_guc_slpc *slpc)
 {
-	struct drm_i915_private *i915 = slpc_to_i915(slpc);
 	struct intel_guc *guc = slpc_to_guc(slpc);
 	u32 offset = intel_guc_ggtt_offset(guc, slpc->vma);
 	int ret;
@@ -346,15 +339,14 @@ static int slpc_reset(struct intel_guc_slpc *slpc)
 	ret = guc_action_slpc_reset(guc, offset);
 
 	if (unlikely(ret < 0)) {
-		i915_probe_error(i915, "SLPC reset action failed (%pe)\n",
-				 ERR_PTR(ret));
+		guc_probe_error(guc, "SLPC reset action failed: %pe\n", ERR_PTR(ret));
 		return ret;
 	}
 
 	if (!ret) {
 		if (wait_for(slpc_is_running(slpc), SLPC_RESET_TIMEOUT_MS)) {
-			i915_probe_error(i915, "SLPC not enabled! State = %s\n",
-					 slpc_get_state_string(slpc));
+			guc_probe_error(guc, "SLPC not enabled! State = %s\n",
+					slpc_get_state_string(slpc));
 			return -EIO;
 		}
 	}
@@ -495,8 +487,8 @@ int intel_guc_slpc_set_min_freq(struct intel_guc_slpc *slpc, u32 val)
 			     SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY,
 			     val < slpc->rp1_freq);
 	if (ret) {
-		i915_probe_error(i915, "Failed to toggle efficient freq (%pe)\n",
-				 ERR_PTR(ret));
+		guc_probe_error(slpc_to_guc(slpc), "Failed to toggle efficient freq: %pe\n",
+				ERR_PTR(ret));
 		goto out;
 	}
 
@@ -611,15 +603,12 @@ static int slpc_set_softlimits(struct intel_guc_slpc *slpc)
 
 static bool is_slpc_min_freq_rpmax(struct intel_guc_slpc *slpc)
 {
-	struct drm_i915_private *i915 = slpc_to_i915(slpc);
 	int slpc_min_freq;
 	int ret;
 
 	ret = intel_guc_slpc_get_min_freq(slpc, &slpc_min_freq);
 	if (ret) {
-		drm_err(&i915->drm,
-			"Failed to get min freq: (%d)\n",
-			ret);
+		guc_err(slpc_to_guc(slpc), "Failed to get min freq: %pe\n", ERR_PTR(ret));
 		return false;
 	}
 
@@ -685,9 +674,8 @@ int intel_guc_slpc_override_gucrc_mode(struct intel_guc_slpc *slpc, u32 mode)
 	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
 		ret = slpc_set_param(slpc, SLPC_PARAM_PWRGATE_RC_MODE, mode);
 		if (ret)
-			drm_err(&i915->drm,
-				"Override gucrc mode %d failed %d\n",
-				mode, ret);
+			guc_err(slpc_to_guc(slpc), "Override RC mode %d failed: %pe\n",
+				mode, ERR_PTR(ret));
 	}
 
 	return ret;
@@ -702,9 +690,7 @@ int intel_guc_slpc_unset_gucrc_mode(struct intel_guc_slpc *slpc)
 	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
 		ret = slpc_unset_param(slpc, SLPC_PARAM_PWRGATE_RC_MODE);
 		if (ret)
-			drm_err(&i915->drm,
-				"Unsetting gucrc mode failed %d\n",
-				ret);
+			guc_err(slpc_to_guc(slpc), "Unsetting RC mode failed: %pe\n", ERR_PTR(ret));
 	}
 
 	return ret;
@@ -725,7 +711,7 @@ int intel_guc_slpc_unset_gucrc_mode(struct intel_guc_slpc *slpc)
  */
 int intel_guc_slpc_enable(struct intel_guc_slpc *slpc)
 {
-	struct drm_i915_private *i915 = slpc_to_i915(slpc);
+	struct intel_guc *guc = slpc_to_guc(slpc);
 	int ret;
 
 	GEM_BUG_ON(!slpc->vma);
@@ -734,8 +720,7 @@ int intel_guc_slpc_enable(struct intel_guc_slpc *slpc)
 
 	ret = slpc_reset(slpc);
 	if (unlikely(ret < 0)) {
-		i915_probe_error(i915, "SLPC Reset event returned (%pe)\n",
-				 ERR_PTR(ret));
+		guc_probe_error(guc, "SLPC Reset event returned: %pe\n", ERR_PTR(ret));
 		return ret;
 	}
 
@@ -743,7 +728,7 @@ int intel_guc_slpc_enable(struct intel_guc_slpc *slpc)
 	if (unlikely(ret < 0))
 		return ret;
 
-	intel_guc_pm_intrmsk_enable(to_gt(i915));
+	intel_guc_pm_intrmsk_enable(slpc_to_gt(slpc));
 
 	slpc_get_rp_values(slpc);
 
@@ -753,16 +738,14 @@ int intel_guc_slpc_enable(struct intel_guc_slpc *slpc)
 	/* Set SLPC max limit to RP0 */
 	ret = slpc_use_fused_rp0(slpc);
 	if (unlikely(ret)) {
-		i915_probe_error(i915, "Failed to set SLPC max to RP0 (%pe)\n",
-				 ERR_PTR(ret));
+		guc_probe_error(guc, "Failed to set SLPC max to RP0: %pe\n", ERR_PTR(ret));
 		return ret;
 	}
 
 	/* Revert SLPC min/max to softlimits if necessary */
 	ret = slpc_set_softlimits(slpc);
 	if (unlikely(ret)) {
-		i915_probe_error(i915, "Failed to set SLPC softlimits (%pe)\n",
-				 ERR_PTR(ret));
+		guc_probe_error(guc, "Failed to set SLPC softlimits: %pe\n", ERR_PTR(ret));
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
index 53f3ed3244d5..88e881b100cf 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
@@ -1352,6 +1352,16 @@ static ktime_t guc_engine_busyness(struct intel_engine_cs *engine, ktime_t *now)
 	return ns_to_ktime(total);
 }
 
+static void guc_enable_busyness_worker(struct intel_guc *guc)
+{
+	mod_delayed_work(system_highpri_wq, &guc->timestamp.work, guc->timestamp.ping_delay);
+}
+
+static void guc_cancel_busyness_worker(struct intel_guc *guc)
+{
+	cancel_delayed_work_sync(&guc->timestamp.work);
+}
+
 static void __reset_guc_busyness_stats(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
@@ -1360,7 +1370,7 @@ static void __reset_guc_busyness_stats(struct intel_guc *guc)
 	unsigned long flags;
 	ktime_t unused;
 
-	cancel_delayed_work_sync(&guc->timestamp.work);
+	guc_cancel_busyness_worker(guc);
 
 	spin_lock_irqsave(&guc->timestamp.lock, flags);
 
@@ -1416,8 +1426,7 @@ static void guc_timestamp_ping(struct work_struct *wrk)
 
 	intel_gt_reset_unlock(gt, srcu);
 
-	mod_delayed_work(system_highpri_wq, &guc->timestamp.work,
-			 guc->timestamp.ping_delay);
+	guc_enable_busyness_worker(guc);
 }
 
 static int guc_action_enable_usage_stats(struct intel_guc *guc)
@@ -1432,20 +1441,26 @@ static int guc_action_enable_usage_stats(struct intel_guc *guc)
 	return intel_guc_send(guc, action, ARRAY_SIZE(action));
 }
 
-static void guc_init_engine_stats(struct intel_guc *guc)
+static int guc_init_engine_stats(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
 	intel_wakeref_t wakeref;
+	int ret;
 
-	mod_delayed_work(system_highpri_wq, &guc->timestamp.work,
-			 guc->timestamp.ping_delay);
+	with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)
+		ret = guc_action_enable_usage_stats(guc);
 
-	with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref) {
-		int ret = guc_action_enable_usage_stats(guc);
+	if (ret)
+		guc_err(guc, "Failed to enable usage stats: %pe\n", ERR_PTR(ret));
+	else
+		guc_enable_busyness_worker(guc);
 
-		if (ret)
-			guc_err(guc, "Failed to enable usage stats: %pe\n", ERR_PTR(ret));
-	}
+	return ret;
+}
+
+static void guc_fini_engine_stats(struct intel_guc *guc)
+{
+	guc_cancel_busyness_worker(guc);
 }
 
 void intel_guc_busyness_park(struct intel_gt *gt)
@@ -1460,7 +1475,7 @@ void intel_guc_busyness_park(struct intel_gt *gt)
 	 * and causes an unclaimed register access warning. Cancel the worker
 	 * synchronously here.
 	 */
-	cancel_delayed_work_sync(&guc->timestamp.work);
+	guc_cancel_busyness_worker(guc);
 
 	/*
 	 * Before parking, we should sample engine busyness stats if we need to.
@@ -1487,8 +1502,7 @@ void intel_guc_busyness_unpark(struct intel_gt *gt)
 	spin_lock_irqsave(&guc->timestamp.lock, flags);
 	guc_update_pm_timestamp(guc, &unused);
 	spin_unlock_irqrestore(&guc->timestamp.lock, flags);
-	mod_delayed_work(system_highpri_wq, &guc->timestamp.work,
-			 guc->timestamp.ping_delay);
+	guc_enable_busyness_worker(guc);
 }
 
 static inline bool
@@ -4102,9 +4116,11 @@ static void guc_set_default_submission(struct intel_engine_cs *engine)
 	engine->submit_request = guc_submit_request;
 }
 
-static inline void guc_kernel_context_pin(struct intel_guc *guc,
-					  struct intel_context *ce)
+static inline int guc_kernel_context_pin(struct intel_guc *guc,
+					 struct intel_context *ce)
 {
+	int ret;
+
 	/*
 	 * Note: we purposefully do not check the returns below because
 	 * the registration can only fail if a reset is just starting.
@@ -4112,16 +4128,24 @@ static inline void guc_kernel_context_pin(struct intel_guc *guc,
 	 * isn't happening and even it did this code would be run again.
 	 */
 
-	if (context_guc_id_invalid(ce))
-		pin_guc_id(guc, ce);
+	if (context_guc_id_invalid(ce)) {
+		ret = pin_guc_id(guc, ce);
+
+		if (ret < 0)
+			return ret;
+	}
 
 	if (!test_bit(CONTEXT_GUC_INIT, &ce->flags))
 		guc_context_init(ce);
 
-	try_context_registration(ce, true);
+	ret = try_context_registration(ce, true);
+	if (ret)
+		unpin_guc_id(guc, ce);
+
+	return ret;
 }
 
-static inline void guc_init_lrc_mapping(struct intel_guc *guc)
+static inline int guc_init_submission(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
 	struct intel_engine_cs *engine;
@@ -4148,9 +4172,17 @@ static inline void guc_init_lrc_mapping(struct intel_guc *guc)
 		struct intel_context *ce;
 
 		list_for_each_entry(ce, &engine->pinned_contexts_list,
-				    pinned_contexts_link)
-			guc_kernel_context_pin(guc, ce);
+				    pinned_contexts_link) {
+			int ret = guc_kernel_context_pin(guc, ce);
+
+			if (ret) {
+				/* No point in trying to clean up as i915 will wedge on failure */
+				return ret;
+			}
+		}
 	}
+
+	return 0;
 }
 
 static void guc_release(struct intel_engine_cs *engine)
@@ -4393,30 +4425,57 @@ static int guc_init_global_schedule_policy(struct intel_guc *guc)
 	return ret;
 }
 
-void intel_guc_submission_enable(struct intel_guc *guc)
+static void guc_route_semaphores(struct intel_guc *guc, bool to_guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
+	u32 val;
 
-	/* Enable and route to GuC */
-	if (GRAPHICS_VER(gt->i915) >= 12)
-		intel_uncore_write(gt->uncore, GEN12_GUC_SEM_INTR_ENABLES,
-				   GUC_SEM_INTR_ROUTE_TO_GUC |
-				   GUC_SEM_INTR_ENABLE_ALL);
+	if (GRAPHICS_VER(gt->i915) < 12)
+		return;
+
+	if (to_guc)
+		val = GUC_SEM_INTR_ROUTE_TO_GUC | GUC_SEM_INTR_ENABLE_ALL;
+	else
+		val = 0;
 
-	guc_init_lrc_mapping(guc);
-	guc_init_engine_stats(guc);
-	guc_init_global_schedule_policy(guc);
+	intel_uncore_write(gt->uncore, GEN12_GUC_SEM_INTR_ENABLES, val);
 }
 
-void intel_guc_submission_disable(struct intel_guc *guc)
+int intel_guc_submission_enable(struct intel_guc *guc)
 {
-	struct intel_gt *gt = guc_to_gt(guc);
+	int ret;
+
+	/* Semaphore interrupt enable and route to GuC */
+	guc_route_semaphores(guc, true);
 
-	/* Note: By the time we're here, GuC may have already been reset */
+	ret = guc_init_submission(guc);
+	if (ret)
+		goto fail_sem;
+
+	ret = guc_init_engine_stats(guc);
+	if (ret)
+		goto fail_sem;
+
+	ret = guc_init_global_schedule_policy(guc);
+	if (ret)
+		goto fail_stats;
+
+	return 0;
+
+fail_stats:
+	guc_fini_engine_stats(guc);
+fail_sem:
+	guc_route_semaphores(guc, false);
+	return ret;
+}
+
+/* Note: By the time we're here, GuC may have already been reset */
+void intel_guc_submission_disable(struct intel_guc *guc)
+{
+	guc_cancel_busyness_worker(guc);
 
-	/* Disable and route to host */
-	if (GRAPHICS_VER(gt->i915) >= 12)
-		intel_uncore_write(gt->uncore, GEN12_GUC_SEM_INTR_ENABLES, 0x0);
+	/* Semaphore interrupt disable and route to host */
+	guc_route_semaphores(guc, false);
 }
 
 static bool __guc_submission_supported(struct intel_guc *guc)
@@ -4660,9 +4719,10 @@ static void guc_handle_context_reset(struct intel_guc *guc,
 {
 	trace_intel_context_reset(ce);
 
-	drm_dbg(&guc_to_gt(guc)->i915->drm, "Got GuC reset of 0x%04X, exiting = %d, banned = %d\n",
-		ce->guc_id.id, test_bit(CONTEXT_EXITING, &ce->flags),
-		test_bit(CONTEXT_BANNED, &ce->flags));
+	guc_dbg(guc, "Got context reset notification: 0x%04X on %s, exiting = %s, banned = %s\n",
+		ce->guc_id.id, ce->engine->name,
+		str_yes_no(intel_context_is_exiting(ce)),
+		str_yes_no(intel_context_is_banned(ce)));
 
 	if (likely(intel_context_is_schedulable(ce))) {
 		capture_error_state(guc, ce);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h
index 5a95a9f0a8e3..c57b29cdb1a6 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h
@@ -15,7 +15,7 @@ struct intel_engine_cs;
 
 void intel_guc_submission_init_early(struct intel_guc *guc);
 int intel_guc_submission_init(struct intel_guc *guc);
-void intel_guc_submission_enable(struct intel_guc *guc);
+int intel_guc_submission_enable(struct intel_guc *guc);
 void intel_guc_submission_disable(struct intel_guc *guc);
 void intel_guc_submission_fini(struct intel_guc *guc);
 int intel_guc_preempt_work_create(struct intel_guc *guc);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.c b/drivers/gpu/drm/i915/gt/uc/intel_huc.c
index 0c103ca160d1..04724ff56ded 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_huc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.c
@@ -6,6 +6,7 @@
 #include <linux/types.h>
 
 #include "gt/intel_gt.h"
+#include "gt/intel_gt_print.h"
 #include "intel_guc_reg.h"
 #include "intel_huc.h"
 #include "i915_drv.h"
@@ -13,6 +14,15 @@
 #include <linux/device/bus.h>
 #include <linux/mei_aux.h>
 
+#define huc_printk(_huc, _level, _fmt, ...) \
+	gt_##_level(huc_to_gt(_huc), "HuC: " _fmt, ##__VA_ARGS__)
+#define huc_err(_huc, _fmt, ...)	huc_printk((_huc), err, _fmt, ##__VA_ARGS__)
+#define huc_warn(_huc, _fmt, ...)	huc_printk((_huc), warn, _fmt, ##__VA_ARGS__)
+#define huc_notice(_huc, _fmt, ...)	huc_printk((_huc), notice, _fmt, ##__VA_ARGS__)
+#define huc_info(_huc, _fmt, ...)	huc_printk((_huc), info, _fmt, ##__VA_ARGS__)
+#define huc_dbg(_huc, _fmt, ...)	huc_printk((_huc), dbg, _fmt, ##__VA_ARGS__)
+#define huc_probe_error(_huc, _fmt, ...) huc_printk((_huc), probe_error, _fmt, ##__VA_ARGS__)
+
 /**
  * DOC: HuC
  *
@@ -107,11 +117,9 @@ static enum hrtimer_restart huc_delayed_load_timer_callback(struct hrtimer *hrti
 
 	if (!intel_huc_is_authenticated(huc)) {
 		if (huc->delayed_load.status == INTEL_HUC_WAITING_ON_GSC)
-			drm_notice(&huc_to_gt(huc)->i915->drm,
-				   "timed out waiting for MEI GSC init to load HuC\n");
+			huc_notice(huc, "timed out waiting for MEI GSC\n");
 		else if (huc->delayed_load.status == INTEL_HUC_WAITING_ON_PXP)
-			drm_notice(&huc_to_gt(huc)->i915->drm,
-				   "timed out waiting for MEI PXP init to load HuC\n");
+			huc_notice(huc, "timed out waiting for MEI PXP\n");
 		else
 			MISSING_CASE(huc->delayed_load.status);
 
@@ -174,8 +182,7 @@ static int gsc_notifier(struct notifier_block *nb, unsigned long action, void *d
 
 	case BUS_NOTIFY_DRIVER_NOT_BOUND: /* mei driver fails to be bound */
 	case BUS_NOTIFY_UNBIND_DRIVER: /* mei driver about to be unbound */
-		drm_info(&huc_to_gt(huc)->i915->drm,
-			 "mei driver not bound, disabling HuC load\n");
+		huc_info(huc, "MEI driver not bound, disabling load\n");
 		gsc_init_error(huc);
 		break;
 	}
@@ -183,7 +190,7 @@ static int gsc_notifier(struct notifier_block *nb, unsigned long action, void *d
 	return 0;
 }
 
-void intel_huc_register_gsc_notifier(struct intel_huc *huc, struct bus_type *bus)
+void intel_huc_register_gsc_notifier(struct intel_huc *huc, const struct bus_type *bus)
 {
 	int ret;
 
@@ -193,14 +200,13 @@ void intel_huc_register_gsc_notifier(struct intel_huc *huc, struct bus_type *bus
 	huc->delayed_load.nb.notifier_call = gsc_notifier;
 	ret = bus_register_notifier(bus, &huc->delayed_load.nb);
 	if (ret) {
-		drm_err(&huc_to_gt(huc)->i915->drm,
-			"failed to register GSC notifier\n");
+		huc_err(huc, "failed to register GSC notifier %pe\n", ERR_PTR(ret));
 		huc->delayed_load.nb.notifier_call = NULL;
 		gsc_init_error(huc);
 	}
 }
 
-void intel_huc_unregister_gsc_notifier(struct intel_huc *huc, struct bus_type *bus)
+void intel_huc_unregister_gsc_notifier(struct intel_huc *huc, const struct bus_type *bus)
 {
 	if (!huc->delayed_load.nb.notifier_call)
 		return;
@@ -313,29 +319,25 @@ static int check_huc_loading_mode(struct intel_huc *huc)
 			      GSC_LOADS_HUC;
 
 	if (fw_needs_gsc != hw_uses_gsc) {
-		drm_err(&gt->i915->drm,
-			"mismatch between HuC FW (%s) and HW (%s) load modes\n",
-			HUC_LOAD_MODE_STRING(fw_needs_gsc),
-			HUC_LOAD_MODE_STRING(hw_uses_gsc));
+		huc_err(huc, "mismatch between FW (%s) and HW (%s) load modes\n",
+			HUC_LOAD_MODE_STRING(fw_needs_gsc), HUC_LOAD_MODE_STRING(hw_uses_gsc));
 		return -ENOEXEC;
 	}
 
 	/* make sure we can access the GSC via the mei driver if we need it */
 	if (!(IS_ENABLED(CONFIG_INTEL_MEI_PXP) && IS_ENABLED(CONFIG_INTEL_MEI_GSC)) &&
 	    fw_needs_gsc) {
-		drm_info(&gt->i915->drm,
-			 "Can't load HuC due to missing MEI modules\n");
+		huc_info(huc, "can't load due to missing MEI modules\n");
 		return -EIO;
 	}
 
-	drm_dbg(&gt->i915->drm, "GSC loads huc=%s\n", str_yes_no(fw_needs_gsc));
+	huc_dbg(huc, "loaded by GSC = %s\n", str_yes_no(fw_needs_gsc));
 
 	return 0;
 }
 
 int intel_huc_init(struct intel_huc *huc)
 {
-	struct drm_i915_private *i915 = huc_to_gt(huc)->i915;
 	int err;
 
 	err = check_huc_loading_mode(huc);
@@ -352,7 +354,7 @@ int intel_huc_init(struct intel_huc *huc)
 
 out:
 	intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_INIT_FAIL);
-	drm_info(&i915->drm, "HuC init failed with %d\n", err);
+	huc_info(huc, "initialization failed %pe\n", ERR_PTR(err));
 	return err;
 }
 
@@ -396,13 +398,13 @@ int intel_huc_wait_for_auth_complete(struct intel_huc *huc)
 	delayed_huc_load_complete(huc);
 
 	if (ret) {
-		drm_err(&gt->i915->drm, "HuC: Firmware not verified %d\n", ret);
+		huc_err(huc, "firmware not verified %pe\n", ERR_PTR(ret));
 		intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_LOAD_FAIL);
 		return ret;
 	}
 
 	intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_RUNNING);
-	drm_info(&gt->i915->drm, "HuC authenticated\n");
+	huc_info(huc, "authenticated!\n");
 	return 0;
 }
 
@@ -437,7 +439,7 @@ int intel_huc_auth(struct intel_huc *huc)
 
 	ret = intel_guc_auth_huc(guc, intel_guc_ggtt_offset(guc, huc->fw.rsa_data));
 	if (ret) {
-		DRM_ERROR("HuC: GuC did not ack Auth request %d\n", ret);
+		huc_err(huc, "authentication by GuC failed %pe\n", ERR_PTR(ret));
 		goto fail;
 	}
 
@@ -449,7 +451,7 @@ int intel_huc_auth(struct intel_huc *huc)
 	return 0;
 
 fail:
-	i915_probe_error(gt->i915, "HuC: Authentication failed %d\n", ret);
+	huc_probe_error(huc, "authentication failed %pe\n", ERR_PTR(ret));
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.h b/drivers/gpu/drm/i915/gt/uc/intel_huc.h
index db555b3c1f56..0789184d81a2 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_huc.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.h
@@ -52,8 +52,8 @@ int intel_huc_check_status(struct intel_huc *huc);
 void intel_huc_update_auth_status(struct intel_huc *huc);
 bool intel_huc_is_authenticated(struct intel_huc *huc);
 
-void intel_huc_register_gsc_notifier(struct intel_huc *huc, struct bus_type *bus);
-void intel_huc_unregister_gsc_notifier(struct intel_huc *huc, struct bus_type *bus);
+void intel_huc_register_gsc_notifier(struct intel_huc *huc, const struct bus_type *bus);
+void intel_huc_unregister_gsc_notifier(struct intel_huc *huc, const struct bus_type *bus);
 
 static inline bool intel_huc_is_supported(struct intel_huc *huc)
 {
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c
index de7f987cf611..4ccb4be4c9cb 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c
@@ -83,15 +83,15 @@ static int __intel_uc_reset_hw(struct intel_uc *uc)
 
 static void __confirm_options(struct intel_uc *uc)
 {
-	struct drm_i915_private *i915 = uc_to_gt(uc)->i915;
+	struct intel_gt *gt = uc_to_gt(uc);
+	struct drm_i915_private *i915 = gt->i915;
 
-	drm_dbg(&i915->drm,
-		"enable_guc=%d (guc:%s submission:%s huc:%s slpc:%s)\n",
-		i915->params.enable_guc,
-		str_yes_no(intel_uc_wants_guc(uc)),
-		str_yes_no(intel_uc_wants_guc_submission(uc)),
-		str_yes_no(intel_uc_wants_huc(uc)),
-		str_yes_no(intel_uc_wants_guc_slpc(uc)));
+	gt_dbg(gt, "enable_guc=%d (guc:%s submission:%s huc:%s slpc:%s)\n",
+	       i915->params.enable_guc,
+	       str_yes_no(intel_uc_wants_guc(uc)),
+	       str_yes_no(intel_uc_wants_guc_submission(uc)),
+	       str_yes_no(intel_uc_wants_huc(uc)),
+	       str_yes_no(intel_uc_wants_guc_slpc(uc)));
 
 	if (i915->params.enable_guc == 0) {
 		GEM_BUG_ON(intel_uc_wants_guc(uc));
@@ -102,26 +102,22 @@ static void __confirm_options(struct intel_uc *uc)
 	}
 
 	if (!intel_uc_supports_guc(uc))
-		drm_info(&i915->drm,
-			 "Incompatible option enable_guc=%d - %s\n",
-			 i915->params.enable_guc, "GuC is not supported!");
+		gt_info(gt,  "Incompatible option enable_guc=%d - %s\n",
+			i915->params.enable_guc, "GuC is not supported!");
 
 	if (i915->params.enable_guc & ENABLE_GUC_LOAD_HUC &&
 	    !intel_uc_supports_huc(uc))
-		drm_info(&i915->drm,
-			 "Incompatible option enable_guc=%d - %s\n",
-			 i915->params.enable_guc, "HuC is not supported!");
+		gt_info(gt, "Incompatible option enable_guc=%d - %s\n",
+			i915->params.enable_guc, "HuC is not supported!");
 
 	if (i915->params.enable_guc & ENABLE_GUC_SUBMISSION &&
 	    !intel_uc_supports_guc_submission(uc))
-		drm_info(&i915->drm,
-			 "Incompatible option enable_guc=%d - %s\n",
-			 i915->params.enable_guc, "GuC submission is N/A");
+		gt_info(gt, "Incompatible option enable_guc=%d - %s\n",
+			i915->params.enable_guc, "GuC submission is N/A");
 
 	if (i915->params.enable_guc & ~ENABLE_GUC_MASK)
-		drm_info(&i915->drm,
-			 "Incompatible option enable_guc=%d - %s\n",
-			 i915->params.enable_guc, "undocumented flag");
+		gt_info(gt, "Incompatible option enable_guc=%d - %s\n",
+			i915->params.enable_guc, "undocumented flag");
 }
 
 void intel_uc_init_early(struct intel_uc *uc)
@@ -143,6 +139,7 @@ void intel_uc_init_early(struct intel_uc *uc)
 void intel_uc_init_late(struct intel_uc *uc)
 {
 	intel_guc_init_late(&uc->guc);
+	intel_gsc_uc_load_start(&uc->gsc);
 }
 
 void intel_uc_driver_late_release(struct intel_uc *uc)
@@ -535,8 +532,11 @@ static int __uc_init_hw(struct intel_uc *uc)
 	else
 		intel_huc_auth(huc);
 
-	if (intel_uc_uses_guc_submission(uc))
-		intel_guc_submission_enable(guc);
+	if (intel_uc_uses_guc_submission(uc)) {
+		ret = intel_guc_submission_enable(guc);
+		if (ret)
+			goto err_log_capture;
+	}
 
 	if (intel_uc_uses_guc_slpc(uc)) {
 		ret = intel_guc_slpc_enable(&guc->slpc);
@@ -547,12 +547,8 @@ static int __uc_init_hw(struct intel_uc *uc)
 		intel_rps_lower_unslice(&uc_to_gt(uc)->rps);
 	}
 
-	intel_gsc_uc_load_start(&uc->gsc);
-
-	gt_info(gt, "GuC submission %s\n",
-		str_enabled_disabled(intel_uc_uses_guc_submission(uc)));
-	gt_info(gt, "GuC SLPC %s\n",
-		str_enabled_disabled(intel_uc_uses_guc_slpc(uc)));
+	guc_info(guc, "submission %s\n", str_enabled_disabled(intel_uc_uses_guc_submission(uc)));
+	guc_info(guc, "SLPC %s\n", str_enabled_disabled(intel_uc_uses_guc_slpc(uc)));
 
 	return 0;
 
@@ -678,7 +674,7 @@ void intel_uc_suspend(struct intel_uc *uc)
 	int err;
 
 	/* flush the GSC worker */
-	intel_gsc_uc_suspend(&uc->gsc);
+	intel_gsc_uc_flush_work(&uc->gsc);
 
 	if (!intel_guc_is_ready(guc)) {
 		guc->interrupts.enabled = false;
@@ -720,6 +716,8 @@ static int __uc_resume(struct intel_uc *uc, bool enable_communication)
 		return err;
 	}
 
+	intel_gsc_uc_resume(&uc->gsc);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c
index 284d6fbc2d08..2f93cc4e408a 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c
@@ -54,6 +54,8 @@ void intel_uc_debugfs_register(struct intel_uc *uc, struct dentry *gt_root)
 	if (IS_ERR(root))
 		return;
 
+	uc->guc.dbgfs_node = root;
+
 	intel_gt_debugfs_register_files(root, files, ARRAY_SIZE(files), uc);
 
 	intel_guc_debugfs_register(&uc->guc, root);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
index 65672ff82605..24765c30a0e1 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
@@ -11,6 +11,7 @@
 #include <drm/drm_print.h>
 
 #include "gem/i915_gem_lmem.h"
+#include "gt/intel_gt_print.h"
 #include "intel_uc_fw.h"
 #include "intel_uc_fw_abi.h"
 #include "i915_drv.h"
@@ -44,11 +45,10 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw,
 			       enum intel_uc_fw_status status)
 {
 	uc_fw->__status =  status;
-	drm_dbg(&__uc_fw_to_gt(uc_fw)->i915->drm,
-		"%s firmware -> %s\n",
-		intel_uc_fw_type_repr(uc_fw->type),
-		status == INTEL_UC_FIRMWARE_SELECTED ?
-		uc_fw->file_selected.path : intel_uc_fw_status_repr(status));
+	gt_dbg(__uc_fw_to_gt(uc_fw), "%s firmware -> %s\n",
+	       intel_uc_fw_type_repr(uc_fw->type),
+	       status == INTEL_UC_FIRMWARE_SELECTED ?
+	       uc_fw->file_selected.path : intel_uc_fw_status_repr(status));
 }
 #endif
 
@@ -488,12 +488,25 @@ static void __force_fw_fetch_failures(struct intel_uc_fw *uc_fw, int e)
 	}
 }
 
-static int check_gsc_manifest(const struct firmware *fw,
+static int check_gsc_manifest(struct intel_gt *gt,
+			      const struct firmware *fw,
 			      struct intel_uc_fw *uc_fw)
 {
 	u32 *dw = (u32 *)fw->data;
-	u32 version_hi = dw[HUC_GSC_VERSION_HI_DW];
-	u32 version_lo = dw[HUC_GSC_VERSION_LO_DW];
+	u32 version_hi, version_lo;
+	size_t min_size;
+
+	/* Check the size of the blob before examining buffer contents */
+	min_size = sizeof(u32) * (HUC_GSC_VERSION_LO_DW + 1);
+	if (unlikely(fw->size < min_size)) {
+		gt_warn(gt, "%s firmware %s: invalid size: %zu < %zu\n",
+			intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
+			fw->size, min_size);
+		return -ENODATA;
+	}
+
+	version_hi = dw[HUC_GSC_VERSION_HI_DW];
+	version_lo = dw[HUC_GSC_VERSION_LO_DW];
 
 	uc_fw->file_selected.ver.major = FIELD_GET(HUC_GSC_MAJOR_VER_HI_MASK, version_hi);
 	uc_fw->file_selected.ver.minor = FIELD_GET(HUC_GSC_MINOR_VER_HI_MASK, version_hi);
@@ -562,15 +575,14 @@ static int check_ccs_header(struct intel_gt *gt,
 			    const struct firmware *fw,
 			    struct intel_uc_fw *uc_fw)
 {
-	struct drm_i915_private *i915 = gt->i915;
 	struct uc_css_header *css;
 	size_t size;
 
 	/* Check the size of the blob before examining buffer contents */
 	if (unlikely(fw->size < sizeof(struct uc_css_header))) {
-		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu < %zu\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
-			 fw->size, sizeof(struct uc_css_header));
+		gt_warn(gt, "%s firmware %s: invalid size: %zu < %zu\n",
+			intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
+			fw->size, sizeof(struct uc_css_header));
 		return -ENODATA;
 	}
 
@@ -580,10 +592,9 @@ static int check_ccs_header(struct intel_gt *gt,
 	size = (css->header_size_dw - css->key_size_dw - css->modulus_size_dw -
 		css->exponent_size_dw) * sizeof(u32);
 	if (unlikely(size != sizeof(struct uc_css_header))) {
-		drm_warn(&i915->drm,
-			 "%s firmware %s: unexpected header size: %zu != %zu\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
-			 fw->size, sizeof(struct uc_css_header));
+		gt_warn(gt, "%s firmware %s: unexpected header size: %zu != %zu\n",
+			intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
+			fw->size, sizeof(struct uc_css_header));
 		return -EPROTO;
 	}
 
@@ -596,18 +607,18 @@ static int check_ccs_header(struct intel_gt *gt,
 	/* At least, it should have header, uCode and RSA. Size of all three. */
 	size = sizeof(struct uc_css_header) + uc_fw->ucode_size + uc_fw->rsa_size;
 	if (unlikely(fw->size < size)) {
-		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu < %zu\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
-			 fw->size, size);
+		gt_warn(gt, "%s firmware %s: invalid size: %zu < %zu\n",
+			intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
+			fw->size, size);
 		return -ENOEXEC;
 	}
 
 	/* Sanity check whether this fw is not larger than whole WOPCM memory */
 	size = __intel_uc_fw_get_upload_size(uc_fw);
 	if (unlikely(size >= gt->wopcm.size)) {
-		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu > %zu\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
-			 size, (size_t)gt->wopcm.size);
+		gt_warn(gt, "%s firmware %s: invalid size: %zu > %zu\n",
+			intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
+			size, (size_t)gt->wopcm.size);
 		return -E2BIG;
 	}
 
@@ -635,20 +646,20 @@ static bool guc_check_version_range(struct intel_uc_fw *uc_fw)
 	 */
 
 	if (!is_ver_8bit(&uc_fw->file_selected.ver)) {
-		drm_warn(&__uc_fw_to_gt(uc_fw)->i915->drm, "%s firmware: invalid file version: 0x%02X:%02X:%02X\n",
-			 intel_uc_fw_type_repr(uc_fw->type),
-			 uc_fw->file_selected.ver.major,
-			 uc_fw->file_selected.ver.minor,
-			 uc_fw->file_selected.ver.patch);
+		gt_warn(__uc_fw_to_gt(uc_fw), "%s firmware: invalid file version: 0x%02X:%02X:%02X\n",
+			intel_uc_fw_type_repr(uc_fw->type),
+			uc_fw->file_selected.ver.major,
+			uc_fw->file_selected.ver.minor,
+			uc_fw->file_selected.ver.patch);
 		return false;
 	}
 
 	if (!is_ver_8bit(&guc->submission_version)) {
-		drm_warn(&__uc_fw_to_gt(uc_fw)->i915->drm, "%s firmware: invalid submit version: 0x%02X:%02X:%02X\n",
-			 intel_uc_fw_type_repr(uc_fw->type),
-			 guc->submission_version.major,
-			 guc->submission_version.minor,
-			 guc->submission_version.patch);
+		gt_warn(__uc_fw_to_gt(uc_fw), "%s firmware: invalid submit version: 0x%02X:%02X:%02X\n",
+			intel_uc_fw_type_repr(uc_fw->type),
+			guc->submission_version.major,
+			guc->submission_version.minor,
+			guc->submission_version.patch);
 		return false;
 	}
 
@@ -666,7 +677,7 @@ static int check_fw_header(struct intel_gt *gt,
 		return 0;
 
 	if (uc_fw->loaded_via_gsc)
-		err = check_gsc_manifest(fw, uc_fw);
+		err = check_gsc_manifest(gt, fw, uc_fw);
 	else
 		err = check_ccs_header(gt, fw, uc_fw);
 	if (err)
@@ -687,10 +698,9 @@ static int try_firmware_load(struct intel_uc_fw *uc_fw, const struct firmware **
 		return err;
 
 	if ((*fw)->size > INTEL_UC_RSVD_GGTT_PER_FW) {
-		drm_err(&gt->i915->drm,
-			"%s firmware %s: size (%zuKB) exceeds max supported size (%uKB)\n",
-			intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
-			(*fw)->size / SZ_1K, INTEL_UC_RSVD_GGTT_PER_FW / SZ_1K);
+		gt_err(gt, "%s firmware %s: size (%zuKB) exceeds max supported size (%uKB)\n",
+		       intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
+		       (*fw)->size / SZ_1K, INTEL_UC_RSVD_GGTT_PER_FW / SZ_1K);
 
 		/* try to find another blob to load */
 		release_firmware(*fw);
@@ -768,10 +778,10 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw)
 	if (uc_fw->file_wanted.ver.major && uc_fw->file_selected.ver.major) {
 		/* Check the file's major version was as it claimed */
 		if (uc_fw->file_selected.ver.major != uc_fw->file_wanted.ver.major) {
-			drm_notice(&i915->drm, "%s firmware %s: unexpected version: %u.%u != %u.%u\n",
-				   intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
-				   uc_fw->file_selected.ver.major, uc_fw->file_selected.ver.minor,
-				   uc_fw->file_wanted.ver.major, uc_fw->file_wanted.ver.minor);
+			gt_notice(gt, "%s firmware %s: unexpected version: %u.%u != %u.%u\n",
+				  intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
+				  uc_fw->file_selected.ver.major, uc_fw->file_selected.ver.minor,
+				  uc_fw->file_wanted.ver.major, uc_fw->file_wanted.ver.minor);
 			if (!intel_uc_fw_is_overridden(uc_fw)) {
 				err = -ENOEXEC;
 				goto fail;
@@ -786,16 +796,14 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw)
 		/* Preserve the version that was really wanted */
 		memcpy(&uc_fw->file_wanted, &file_ideal, sizeof(uc_fw->file_wanted));
 
-		drm_notice(&i915->drm,
-			   "%s firmware %s (%d.%d) is recommended, but only %s (%d.%d) was found\n",
-			   intel_uc_fw_type_repr(uc_fw->type),
-			   uc_fw->file_wanted.path,
-			   uc_fw->file_wanted.ver.major, uc_fw->file_wanted.ver.minor,
-			   uc_fw->file_selected.path,
-			   uc_fw->file_selected.ver.major, uc_fw->file_selected.ver.minor);
-		drm_info(&i915->drm,
-			 "Consider updating your linux-firmware pkg or downloading from %s\n",
-			 INTEL_UC_FIRMWARE_URL);
+		gt_notice(gt, "%s firmware %s (%d.%d) is recommended, but only %s (%d.%d) was found\n",
+			  intel_uc_fw_type_repr(uc_fw->type),
+			  uc_fw->file_wanted.path,
+			  uc_fw->file_wanted.ver.major, uc_fw->file_wanted.ver.minor,
+			  uc_fw->file_selected.path,
+			  uc_fw->file_selected.ver.major, uc_fw->file_selected.ver.minor);
+		gt_info(gt, "Consider updating your linux-firmware pkg or downloading from %s\n",
+			INTEL_UC_FIRMWARE_URL);
 	}
 
 	if (HAS_LMEM(i915)) {
@@ -823,10 +831,10 @@ fail:
 				  INTEL_UC_FIRMWARE_MISSING :
 				  INTEL_UC_FIRMWARE_ERROR);
 
-	i915_probe_error(i915, "%s firmware %s: fetch failed with error %d\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, err);
-	drm_info(&i915->drm, "%s firmware(s) can be downloaded from %s\n",
-		 intel_uc_fw_type_repr(uc_fw->type), INTEL_UC_FIRMWARE_URL);
+	gt_probe_error(gt, "%s firmware %s: fetch failed %pe\n",
+		       intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, ERR_PTR(err));
+	gt_info(gt, "%s firmware(s) can be downloaded from %s\n",
+		intel_uc_fw_type_repr(uc_fw->type), INTEL_UC_FIRMWARE_URL);
 
 	release_firmware(fw);		/* OK even if fw is NULL */
 	return err;
@@ -932,9 +940,9 @@ static int uc_fw_xfer(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags)
 	/* Wait for DMA to finish */
 	ret = intel_wait_for_register_fw(uncore, DMA_CTRL, START_DMA, 0, 100);
 	if (ret)
-		drm_err(&gt->i915->drm, "DMA for %s fw failed, DMA_CTRL=%u\n",
-			intel_uc_fw_type_repr(uc_fw->type),
-			intel_uncore_read_fw(uncore, DMA_CTRL));
+		gt_err(gt, "DMA for %s fw failed, DMA_CTRL=%u\n",
+		       intel_uc_fw_type_repr(uc_fw->type),
+		       intel_uncore_read_fw(uncore, DMA_CTRL));
 
 	/* Disable the bits once DMA is over */
 	intel_uncore_write_fw(uncore, DMA_CTRL, _MASKED_BIT_DISABLE(dma_flags));
@@ -950,9 +958,8 @@ int intel_uc_fw_mark_load_failed(struct intel_uc_fw *uc_fw, int err)
 
 	GEM_BUG_ON(!intel_uc_fw_is_loadable(uc_fw));
 
-	i915_probe_error(gt->i915, "Failed to load %s firmware %s (%d)\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
-			 err);
+	gt_probe_error(gt, "Failed to load %s firmware %s %pe\n",
+		       intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, ERR_PTR(err));
 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_LOAD_FAIL);
 
 	return err;
@@ -1078,15 +1085,15 @@ int intel_uc_fw_init(struct intel_uc_fw *uc_fw)
 
 	err = i915_gem_object_pin_pages_unlocked(uc_fw->obj);
 	if (err) {
-		DRM_DEBUG_DRIVER("%s fw pin-pages err=%d\n",
-				 intel_uc_fw_type_repr(uc_fw->type), err);
+		gt_dbg(__uc_fw_to_gt(uc_fw), "%s fw pin-pages failed %pe\n",
+		       intel_uc_fw_type_repr(uc_fw->type), ERR_PTR(err));
 		goto out;
 	}
 
 	err = uc_fw_rsa_data_create(uc_fw);
 	if (err) {
-		DRM_DEBUG_DRIVER("%s fw rsa data creation failed, err=%d\n",
-				 intel_uc_fw_type_repr(uc_fw->type), err);
+		gt_dbg(__uc_fw_to_gt(uc_fw), "%s fw rsa data creation failed %pe\n",
+		       intel_uc_fw_type_repr(uc_fw->type), ERR_PTR(err));
 		goto out_unpin;
 	}
 
diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
index e28518fe8b90..1fd760539f77 100644
--- a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
+++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
@@ -3,6 +3,8 @@
  * Copyright �� 2021 Intel Corporation
  */
 
+#include "gt/intel_gt_print.h"
+#include "intel_guc_print.h"
 #include "selftests/igt_spinner.h"
 #include "selftests/intel_scheduler_helpers.h"
 
@@ -65,7 +67,7 @@ static int intel_guc_scrub_ctbs(void *arg)
 		ce = intel_context_create(engine);
 		if (IS_ERR(ce)) {
 			ret = PTR_ERR(ce);
-			drm_err(&gt->i915->drm, "Failed to create context, %d: %d\n", i, ret);
+			gt_err(gt, "Failed to create context %d: %pe\n", i, ce);
 			goto err;
 		}
 
@@ -86,7 +88,7 @@ static int intel_guc_scrub_ctbs(void *arg)
 
 		if (IS_ERR(rq)) {
 			ret = PTR_ERR(rq);
-			drm_err(&gt->i915->drm, "Failed to create request, %d: %d\n", i, ret);
+			gt_err(gt, "Failed to create request %d: %pe\n", i, rq);
 			goto err;
 		}
 
@@ -96,7 +98,7 @@ static int intel_guc_scrub_ctbs(void *arg)
 	for (i = 0; i < 3; ++i) {
 		ret = i915_request_wait(last[i], 0, HZ);
 		if (ret < 0) {
-			drm_err(&gt->i915->drm, "Last request failed to complete: %d\n", ret);
+			gt_err(gt, "Last request failed to complete: %pe\n", ERR_PTR(ret));
 			goto err;
 		}
 		i915_request_put(last[i]);
@@ -113,7 +115,7 @@ static int intel_guc_scrub_ctbs(void *arg)
 	/* GT will not idle if G2H are lost */
 	ret = intel_gt_wait_for_idle(gt, HZ);
 	if (ret < 0) {
-		drm_err(&gt->i915->drm, "GT failed to idle: %d\n", ret);
+		gt_err(gt, "GT failed to idle: %pe\n", ERR_PTR(ret));
 		goto err;
 	}
 
@@ -153,7 +155,7 @@ static int intel_guc_steal_guc_ids(void *arg)
 
 	ce = kcalloc(GUC_MAX_CONTEXT_ID, sizeof(*ce), GFP_KERNEL);
 	if (!ce) {
-		drm_err(&gt->i915->drm, "Context array allocation failed\n");
+		guc_err(guc, "Context array allocation failed\n");
 		return -ENOMEM;
 	}
 
@@ -166,25 +168,25 @@ static int intel_guc_steal_guc_ids(void *arg)
 	ce[context_index] = intel_context_create(engine);
 	if (IS_ERR(ce[context_index])) {
 		ret = PTR_ERR(ce[context_index]);
+		guc_err(guc, "Failed to create context: %pe\n", ce[context_index]);
 		ce[context_index] = NULL;
-		drm_err(&gt->i915->drm, "Failed to create context: %d\n", ret);
 		goto err_wakeref;
 	}
 	ret = igt_spinner_init(&spin, engine->gt);
 	if (ret) {
-		drm_err(&gt->i915->drm, "Failed to create spinner: %d\n", ret);
+		guc_err(guc, "Failed to create spinner: %pe\n", ERR_PTR(ret));
 		goto err_contexts;
 	}
 	spin_rq = igt_spinner_create_request(&spin, ce[context_index],
 					     MI_ARB_CHECK);
 	if (IS_ERR(spin_rq)) {
 		ret = PTR_ERR(spin_rq);
-		drm_err(&gt->i915->drm, "Failed to create spinner request: %d\n", ret);
+		guc_err(guc, "Failed to create spinner request: %pe\n", spin_rq);
 		goto err_contexts;
 	}
 	ret = request_add_spin(spin_rq, &spin);
 	if (ret) {
-		drm_err(&gt->i915->drm, "Failed to add Spinner request: %d\n", ret);
+		guc_err(guc, "Failed to add Spinner request: %pe\n", ERR_PTR(ret));
 		goto err_spin_rq;
 	}
 
@@ -192,9 +194,9 @@ static int intel_guc_steal_guc_ids(void *arg)
 	while (ret != -EAGAIN) {
 		ce[++context_index] = intel_context_create(engine);
 		if (IS_ERR(ce[context_index])) {
-			ret = PTR_ERR(ce[context_index--]);
-			ce[context_index] = NULL;
-			drm_err(&gt->i915->drm, "Failed to create context: %d\n", ret);
+			ret = PTR_ERR(ce[context_index]);
+			guc_err(guc, "Failed to create context: %pe\n", ce[context_index]);
+			ce[context_index--] = NULL;
 			goto err_spin_rq;
 		}
 
@@ -203,8 +205,8 @@ static int intel_guc_steal_guc_ids(void *arg)
 			ret = PTR_ERR(rq);
 			rq = NULL;
 			if (ret != -EAGAIN) {
-				drm_err(&gt->i915->drm, "Failed to create request, %d: %d\n",
-					context_index, ret);
+				guc_err(guc, "Failed to create request %d: %pe\n",
+					context_index, ERR_PTR(ret));
 				goto err_spin_rq;
 			}
 		} else {
@@ -218,7 +220,7 @@ static int intel_guc_steal_guc_ids(void *arg)
 	igt_spinner_end(&spin);
 	ret = intel_selftest_wait_for_rq(spin_rq);
 	if (ret) {
-		drm_err(&gt->i915->drm, "Spin request failed to complete: %d\n", ret);
+		guc_err(guc, "Spin request failed to complete: %pe\n", ERR_PTR(ret));
 		i915_request_put(last);
 		goto err_spin_rq;
 	}
@@ -230,7 +232,7 @@ static int intel_guc_steal_guc_ids(void *arg)
 	ret = i915_request_wait(last, 0, HZ * 30);
 	i915_request_put(last);
 	if (ret < 0) {
-		drm_err(&gt->i915->drm, "Last request failed to complete: %d\n", ret);
+		guc_err(guc, "Last request failed to complete: %pe\n", ERR_PTR(ret));
 		goto err_spin_rq;
 	}
 
@@ -238,7 +240,7 @@ static int intel_guc_steal_guc_ids(void *arg)
 	rq = nop_user_request(ce[context_index], NULL);
 	if (IS_ERR(rq)) {
 		ret = PTR_ERR(rq);
-		drm_err(&gt->i915->drm, "Failed to steal guc_id, %d: %d\n", context_index, ret);
+		guc_err(guc, "Failed to steal guc_id %d: %pe\n", context_index, rq);
 		goto err_spin_rq;
 	}
 
@@ -246,20 +248,20 @@ static int intel_guc_steal_guc_ids(void *arg)
 	ret = i915_request_wait(rq, 0, HZ);
 	i915_request_put(rq);
 	if (ret < 0) {
-		drm_err(&gt->i915->drm, "Request with stolen guc_id failed to complete: %d\n", ret);
+		guc_err(guc, "Request with stolen guc_id failed to complete: %pe\n", ERR_PTR(ret));
 		goto err_spin_rq;
 	}
 
 	/* Wait for idle */
 	ret = intel_gt_wait_for_idle(gt, HZ * 30);
 	if (ret < 0) {
-		drm_err(&gt->i915->drm, "GT failed to idle: %d\n", ret);
+		guc_err(guc, "GT failed to idle: %pe\n", ERR_PTR(ret));
 		goto err_spin_rq;
 	}
 
 	/* Verify a guc_id was stolen */
 	if (guc->number_guc_id_stolen == number_guc_id_stolen) {
-		drm_err(&gt->i915->drm, "No guc_id was stolen");
+		guc_err(guc, "No guc_id was stolen");
 		ret = -EINVAL;
 	} else {
 		ret = 0;
diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c
index d91b58f70403..34b5d952e2bc 100644
--- a/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c
@@ -3,6 +3,7 @@
  * Copyright © 2022 Intel Corporation
  */
 
+#include "gt/intel_gt_print.h"
 #include "selftests/igt_spinner.h"
 #include "selftests/igt_reset.h"
 #include "selftests/intel_scheduler_helpers.h"
@@ -45,7 +46,7 @@ static int intel_hang_guc(void *arg)
 
 	ctx = kernel_context(gt->i915, NULL);
 	if (IS_ERR(ctx)) {
-		drm_err(&gt->i915->drm, "Failed get kernel context: %ld\n", PTR_ERR(ctx));
+		gt_err(gt, "Failed get kernel context: %pe\n", ctx);
 		return PTR_ERR(ctx);
 	}
 
@@ -54,7 +55,7 @@ static int intel_hang_guc(void *arg)
 	ce = intel_context_create(engine);
 	if (IS_ERR(ce)) {
 		ret = PTR_ERR(ce);
-		drm_err(&gt->i915->drm, "Failed to create spinner request: %d\n", ret);
+		gt_err(gt, "Failed to create spinner request: %pe\n", ce);
 		goto err;
 	}
 
@@ -63,13 +64,13 @@ static int intel_hang_guc(void *arg)
 	old_beat = engine->props.heartbeat_interval_ms;
 	ret = intel_engine_set_heartbeat(engine, BEAT_INTERVAL);
 	if (ret) {
-		drm_err(&gt->i915->drm, "Failed to boost heatbeat interval: %d\n", ret);
+		gt_err(gt, "Failed to boost heatbeat interval: %pe\n", ERR_PTR(ret));
 		goto err;
 	}
 
 	ret = igt_spinner_init(&spin, engine->gt);
 	if (ret) {
-		drm_err(&gt->i915->drm, "Failed to create spinner: %d\n", ret);
+		gt_err(gt, "Failed to create spinner: %pe\n", ERR_PTR(ret));
 		goto err;
 	}
 
@@ -77,28 +78,28 @@ static int intel_hang_guc(void *arg)
 	intel_context_put(ce);
 	if (IS_ERR(rq)) {
 		ret = PTR_ERR(rq);
-		drm_err(&gt->i915->drm, "Failed to create spinner request: %d\n", ret);
+		gt_err(gt, "Failed to create spinner request: %pe\n", rq);
 		goto err_spin;
 	}
 
 	ret = request_add_spin(rq, &spin);
 	if (ret) {
 		i915_request_put(rq);
-		drm_err(&gt->i915->drm, "Failed to add Spinner request: %d\n", ret);
+		gt_err(gt, "Failed to add Spinner request: %pe\n", ERR_PTR(ret));
 		goto err_spin;
 	}
 
 	ret = intel_reset_guc(gt);
 	if (ret) {
 		i915_request_put(rq);
-		drm_err(&gt->i915->drm, "Failed to reset GuC, ret = %d\n", ret);
+		gt_err(gt, "Failed to reset GuC: %pe\n", ERR_PTR(ret));
 		goto err_spin;
 	}
 
 	guc_status = intel_uncore_read(gt->uncore, GUC_STATUS);
 	if (!(guc_status & GS_MIA_IN_RESET)) {
 		i915_request_put(rq);
-		drm_err(&gt->i915->drm, "GuC failed to reset: status = 0x%08X\n", guc_status);
+		gt_err(gt, "Failed to reset GuC: status = 0x%08X\n", guc_status);
 		ret = -EIO;
 		goto err_spin;
 	}
@@ -107,12 +108,12 @@ static int intel_hang_guc(void *arg)
 	ret = intel_selftest_wait_for_rq(rq);
 	i915_request_put(rq);
 	if (ret) {
-		drm_err(&gt->i915->drm, "Request failed to complete: %d\n", ret);
+		gt_err(gt, "Request failed to complete: %pe\n", ERR_PTR(ret));
 		goto err_spin;
 	}
 
 	if (i915_reset_count(global) == reset_count) {
-		drm_err(&gt->i915->drm, "Failed to record a GPU reset\n");
+		gt_err(gt, "Failed to record a GPU reset\n");
 		ret = -EINVAL;
 		goto err_spin;
 	}
@@ -132,7 +133,7 @@ err_spin:
 		ret = intel_selftest_wait_for_rq(rq);
 		i915_request_put(rq);
 		if (ret) {
-			drm_err(&gt->i915->drm, "No-op failed to complete: %d\n", ret);
+			gt_err(gt, "No-op failed to complete: %pe\n", ERR_PTR(ret));
 			goto err;
 		}
 	}
diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
index d17982c36d25..a40e7c32e613 100644
--- a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
+++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
@@ -3,6 +3,7 @@
  * Copyright �� 2019 Intel Corporation
  */
 
+#include "gt/intel_gt_print.h"
 #include "selftests/igt_spinner.h"
 #include "selftests/igt_reset.h"
 #include "selftests/intel_scheduler_helpers.h"
@@ -115,30 +116,30 @@ static int __intel_guc_multi_lrc_basic(struct intel_gt *gt, unsigned int class)
 
 	parent = multi_lrc_create_parent(gt, class, 0);
 	if (IS_ERR(parent)) {
-		drm_err(&gt->i915->drm, "Failed creating contexts: %ld", PTR_ERR(parent));
+		gt_err(gt, "Failed creating contexts: %pe\n", parent);
 		return PTR_ERR(parent);
 	} else if (!parent) {
-		drm_dbg(&gt->i915->drm, "Not enough engines in class: %d", class);
+		gt_dbg(gt, "Not enough engines in class: %d\n", class);
 		return 0;
 	}
 
 	rq = multi_lrc_nop_request(parent);
 	if (IS_ERR(rq)) {
 		ret = PTR_ERR(rq);
-		drm_err(&gt->i915->drm, "Failed creating requests: %d", ret);
+		gt_err(gt, "Failed creating requests: %pe\n", rq);
 		goto out;
 	}
 
 	ret = intel_selftest_wait_for_rq(rq);
 	if (ret)
-		drm_err(&gt->i915->drm, "Failed waiting on request: %d", ret);
+		gt_err(gt, "Failed waiting on request: %pe\n", ERR_PTR(ret));
 
 	i915_request_put(rq);
 
 	if (ret >= 0) {
 		ret = intel_gt_wait_for_idle(gt, HZ * 5);
 		if (ret < 0)
-			drm_err(&gt->i915->drm, "GT failed to idle: %d\n", ret);
+			gt_err(gt, "GT failed to idle: %pe\n", ERR_PTR(ret));
 	}
 
 out:
diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c
index 4d898b14de93..e0c5dfb788eb 100644
--- a/drivers/gpu/drm/i915/gvt/display.c
+++ b/drivers/gpu/drm/i915/gvt/display.c
@@ -63,7 +63,7 @@ static int edp_pipe_is_enabled(struct intel_vgpu *vgpu)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915;
 
-	if (!(vgpu_vreg_t(vgpu, PIPECONF(_PIPE_EDP)) & PIPECONF_ENABLE))
+	if (!(vgpu_vreg_t(vgpu, TRANSCONF(TRANSCODER_EDP)) & TRANSCONF_ENABLE))
 		return 0;
 
 	if (!(vgpu_vreg(vgpu, _TRANS_DDI_FUNC_CTL_EDP) & TRANS_DDI_FUNC_ENABLE))
@@ -79,7 +79,7 @@ int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe)
 			pipe < PIPE_A || pipe >= I915_MAX_PIPES))
 		return -EINVAL;
 
-	if (vgpu_vreg_t(vgpu, PIPECONF(pipe)) & PIPECONF_ENABLE)
+	if (vgpu_vreg_t(vgpu, TRANSCONF(pipe)) & TRANSCONF_ENABLE)
 		return 1;
 
 	if (edp_pipe_is_enabled(vgpu) &&
@@ -187,8 +187,8 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu)
 			  GEN8_DE_PORT_HOTPLUG(HPD_PORT_C));
 
 		for_each_pipe(dev_priv, pipe) {
-			vgpu_vreg_t(vgpu, PIPECONF(pipe)) &=
-				~(PIPECONF_ENABLE | PIPECONF_STATE_ENABLE);
+			vgpu_vreg_t(vgpu, TRANSCONF(pipe)) &=
+				~(TRANSCONF_ENABLE | TRANSCONF_STATE_ENABLE);
 			vgpu_vreg_t(vgpu, DSPCNTR(pipe)) &= ~DISP_ENABLE;
 			vgpu_vreg_t(vgpu, SPRCTL(pipe)) &= ~SPRITE_ENABLE;
 			vgpu_vreg_t(vgpu, CURCNTR(pipe)) &= ~MCURSOR_MODE_MASK;
@@ -248,8 +248,8 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu)
 		 *   TRANSCODER_A can be enabled. PORT_x depends on the input of
 		 *   setup_virtual_dp_monitor.
 		 */
-		vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_ENABLE;
-		vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_STATE_ENABLE;
+		vgpu_vreg_t(vgpu, TRANSCONF(TRANSCODER_A)) |= TRANSCONF_ENABLE;
+		vgpu_vreg_t(vgpu, TRANSCONF(TRANSCODER_A)) |= TRANSCONF_STATE_ENABLE;
 
 		/*
 		 * Golden M/N are calculated based on:
@@ -506,7 +506,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu)
 		vgpu_vreg_t(vgpu, CURCNTR(pipe)) |= MCURSOR_MODE_DISABLE;
 	}
 
-	vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_ENABLE;
+	vgpu_vreg_t(vgpu, TRANSCONF(TRANSCODER_A)) |= TRANSCONF_ENABLE;
 }
 
 static void clean_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num)
@@ -584,7 +584,7 @@ static int setup_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num,
  * @turnon: Turn ON/OFF vblank_timer
  *
  * This function is used to turn on/off or update the per-vGPU vblank_timer
- * when PIPECONF is enabled or disabled. vblank_timer period is also updated
+ * when TRANSCONF is enabled or disabled. vblank_timer period is also updated
  * if guest changed the refresh rate.
  *
  */
diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c
index 1b509c1a1e33..7c49a3d673a5 100644
--- a/drivers/gpu/drm/i915/gvt/edid.c
+++ b/drivers/gpu/drm/i915/gvt/edid.c
@@ -32,6 +32,7 @@
  *
  */
 
+#include "display/intel_dp_aux_regs.h"
 #include "display/intel_gmbus_regs.h"
 #include "gvt.h"
 #include "i915_drv.h"
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index 735fc83e7026..4b45a041ac5c 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -43,8 +43,13 @@
 #include "intel_mchbar_regs.h"
 #include "display/intel_display_types.h"
 #include "display/intel_dmc_regs.h"
+#include "display/intel_dp_aux_regs.h"
 #include "display/intel_dpio_phy.h"
 #include "display/intel_fbc.h"
+#include "display/intel_fdi_regs.h"
+#include "display/intel_pps_regs.h"
+#include "display/intel_psr_regs.h"
+#include "display/skl_watermark_regs.h"
 #include "display/vlv_dsi_pll_regs.h"
 #include "gt/intel_gt_regs.h"
 
@@ -666,8 +671,8 @@ static void vgpu_update_refresh_rate(struct intel_vgpu *vgpu)
 	link_n = vgpu_vreg_t(vgpu, PIPE_LINK_N1(TRANSCODER_A));
 
 	/* Get H/V total from transcoder timing */
-	htotal = (vgpu_vreg_t(vgpu, HTOTAL(TRANSCODER_A)) >> TRANS_HTOTAL_SHIFT);
-	vtotal = (vgpu_vreg_t(vgpu, VTOTAL(TRANSCODER_A)) >> TRANS_VTOTAL_SHIFT);
+	htotal = (vgpu_vreg_t(vgpu, TRANS_HTOTAL(TRANSCODER_A)) >> TRANS_HTOTAL_SHIFT);
+	vtotal = (vgpu_vreg_t(vgpu, TRANS_VTOTAL(TRANSCODER_A)) >> TRANS_VTOTAL_SHIFT);
 
 	if (dp_br && link_n && htotal && vtotal) {
 		u64 pixel_clk = 0;
@@ -697,12 +702,12 @@ static int pipeconf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 	write_vreg(vgpu, offset, p_data, bytes);
 	data = vgpu_vreg(vgpu, offset);
 
-	if (data & PIPECONF_ENABLE) {
-		vgpu_vreg(vgpu, offset) |= PIPECONF_STATE_ENABLE;
+	if (data & TRANSCONF_ENABLE) {
+		vgpu_vreg(vgpu, offset) |= TRANSCONF_STATE_ENABLE;
 		vgpu_update_refresh_rate(vgpu);
 		vgpu_update_vblank_emulation(vgpu, true);
 	} else {
-		vgpu_vreg(vgpu, offset) &= ~PIPECONF_STATE_ENABLE;
+		vgpu_vreg(vgpu, offset) &= ~TRANSCONF_STATE_ENABLE;
 		vgpu_update_vblank_emulation(vgpu, false);
 	}
 	return 0;
@@ -2262,10 +2267,10 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_DFH(GEN7_HALF_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 
 	/* display */
-	MMIO_DH(PIPECONF(PIPE_A), D_ALL, NULL, pipeconf_mmio_write);
-	MMIO_DH(PIPECONF(PIPE_B), D_ALL, NULL, pipeconf_mmio_write);
-	MMIO_DH(PIPECONF(PIPE_C), D_ALL, NULL, pipeconf_mmio_write);
-	MMIO_DH(PIPECONF(_PIPE_EDP), D_ALL, NULL, pipeconf_mmio_write);
+	MMIO_DH(TRANSCONF(TRANSCODER_A), D_ALL, NULL, pipeconf_mmio_write);
+	MMIO_DH(TRANSCONF(TRANSCODER_B), D_ALL, NULL, pipeconf_mmio_write);
+	MMIO_DH(TRANSCONF(TRANSCODER_C), D_ALL, NULL, pipeconf_mmio_write);
+	MMIO_DH(TRANSCONF(TRANSCODER_EDP), D_ALL, NULL, pipeconf_mmio_write);
 	MMIO_DH(DSPSURF(PIPE_A), D_ALL, NULL, pri_surf_mmio_write);
 	MMIO_DH(REG_50080(PIPE_A, PLANE_PRIMARY), D_ALL, NULL,
 		reg50080_mmio_write);
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 45773ce1deac..80c2bf98e341 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -52,7 +52,6 @@
 #include "i915_irq.h"
 #include "i915_scheduler.h"
 #include "intel_mchbar_regs.h"
-#include "intel_pm.h"
 
 static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)
 {
@@ -575,14 +574,34 @@ static int i915_wa_registers(struct seq_file *m, void *unused)
 static int i915_wedged_get(void *data, u64 *val)
 {
 	struct drm_i915_private *i915 = data;
+	struct intel_gt *gt;
+	unsigned int i;
 
-	return intel_gt_debugfs_reset_show(to_gt(i915), val);
+	*val = 0;
+
+	for_each_gt(gt, i915, i) {
+		int ret;
+
+		ret = intel_gt_debugfs_reset_show(gt, val);
+		if (ret)
+			return ret;
+
+		/* at least one tile should be wedged */
+		if (*val)
+			break;
+	}
+
+	return 0;
 }
 
 static int i915_wedged_set(void *data, u64 val)
 {
 	struct drm_i915_private *i915 = data;
-	intel_gt_debugfs_reset_store(to_gt(i915), val);
+	struct intel_gt *gt;
+	unsigned int i;
+
+	for_each_gt(gt, i915, i)
+		intel_gt_debugfs_reset_store(gt, val);
 
 	return 0;
 }
@@ -733,7 +752,11 @@ static int i915_sseu_status(struct seq_file *m, void *unused)
 static int i915_forcewake_open(struct inode *inode, struct file *file)
 {
 	struct drm_i915_private *i915 = inode->i_private;
-	intel_gt_pm_debugfs_forcewake_user_open(to_gt(i915));
+	struct intel_gt *gt;
+	unsigned int i;
+
+	for_each_gt(gt, i915, i)
+		intel_gt_pm_debugfs_forcewake_user_open(gt);
 
 	return 0;
 }
@@ -741,7 +764,11 @@ static int i915_forcewake_open(struct inode *inode, struct file *file)
 static int i915_forcewake_release(struct inode *inode, struct file *file)
 {
 	struct drm_i915_private *i915 = inode->i_private;
-	intel_gt_pm_debugfs_forcewake_user_release(to_gt(i915));
+	struct intel_gt *gt;
+	unsigned int i;
+
+	for_each_gt(gt, i915, i)
+		intel_gt_pm_debugfs_forcewake_user_release(gt);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c
index cf1c0970ecb4..93fdc40d724f 100644
--- a/drivers/gpu/drm/i915/i915_driver.c
+++ b/drivers/gpu/drm/i915/i915_driver.c
@@ -79,11 +79,11 @@
 #include "soc/intel_dram.h"
 #include "soc/intel_gmch.h"
 
-#include "i915_file_private.h"
 #include "i915_debugfs.h"
 #include "i915_driver.h"
 #include "i915_drm_client.h"
 #include "i915_drv.h"
+#include "i915_file_private.h"
 #include "i915_getparam.h"
 #include "i915_hwmon.h"
 #include "i915_ioc32.h"
@@ -97,11 +97,11 @@
 #include "i915_sysfs.h"
 #include "i915_utils.h"
 #include "i915_vgpu.h"
+#include "intel_clock_gating.h"
 #include "intel_gvt.h"
 #include "intel_memory_region.h"
 #include "intel_pci_config.h"
 #include "intel_pcode.h"
-#include "intel_pm.h"
 #include "intel_region_ttm.h"
 #include "vlv_suspend.h"
 
@@ -167,6 +167,8 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
 	pre |= IS_KABYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
 	pre |= IS_GEMINILAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x3;
 	pre |= IS_ICELAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x7;
+	pre |= IS_TIGERLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
+	pre |= IS_DG1(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
 
 	if (pre) {
 		drm_err(&dev_priv->drm, "This is a pre-production stepping. "
@@ -248,22 +250,14 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
 	/* This must be called before any calls to HAS_PCH_* */
 	intel_detect_pch(dev_priv);
 
-	intel_pm_setup(dev_priv);
-	ret = intel_power_domains_init(dev_priv);
-	if (ret < 0)
-		goto err_gem;
 	intel_irq_init(dev_priv);
 	intel_init_display_hooks(dev_priv);
-	intel_init_clock_gating_hooks(dev_priv);
+	intel_clock_gating_hooks_init(dev_priv);
 
 	intel_detect_preproduction_hw(dev_priv);
 
 	return 0;
 
-err_gem:
-	i915_gem_cleanup_early(dev_priv);
-	intel_gt_driver_late_release_all(dev_priv);
-	i915_drm_clients_fini(&dev_priv->clients);
 err_rootgt:
 	intel_region_ttm_device_fini(dev_priv);
 err_ttm:
@@ -475,7 +469,9 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 	if (ret)
 		return ret;
 
-	i915_perf_init(dev_priv);
+	ret = i915_perf_init(dev_priv);
+	if (ret)
+		return ret;
 
 	ret = i915_ggtt_probe_hw(dev_priv);
 	if (ret)
@@ -489,13 +485,17 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 	if (ret)
 		goto err_ggtt;
 
-	ret = intel_memory_regions_hw_probe(dev_priv);
+	/*
+	 * Make sure we probe lmem before we probe stolen-lmem. The BAR size
+	 * might be different due to bar resizing.
+	 */
+	ret = intel_gt_tiles_init(dev_priv);
 	if (ret)
 		goto err_ggtt;
 
-	ret = intel_gt_tiles_init(dev_priv);
+	ret = intel_memory_regions_hw_probe(dev_priv);
 	if (ret)
-		goto err_mem_regions;
+		goto err_ggtt;
 
 	ret = i915_ggtt_enable_hw(dev_priv);
 	if (ret) {
@@ -537,7 +537,7 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 
 	ret = i915_pcode_init(dev_priv);
 	if (ret)
-		goto err_msi;
+		goto err_opregion;
 
 	/*
 	 * Fill the dram structure to get the system dram info. This will be
@@ -558,6 +558,8 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 
 	return 0;
 
+err_opregion:
+	intel_opregion_cleanup(dev_priv);
 err_msi:
 	if (pdev->msi_enabled)
 		pci_disable_msi(pdev);
@@ -583,6 +585,8 @@ static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)
 
 	i915_perf_fini(dev_priv);
 
+	intel_opregion_cleanup(dev_priv);
+
 	if (pdev->msi_enabled)
 		pci_disable_msi(pdev);
 
@@ -936,7 +940,9 @@ static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
  */
 static void i915_driver_lastclose(struct drm_device *dev)
 {
-	intel_fbdev_restore_mode(dev);
+	struct drm_i915_private *i915 = to_i915(dev);
+
+	intel_fbdev_restore_mode(i915);
 
 	vga_switcheroo_process_delayed_switch();
 }
@@ -1002,7 +1008,7 @@ void i915_driver_shutdown(struct drm_i915_private *i915)
 	intel_suspend_encoders(i915);
 	intel_shutdown_encoders(i915);
 
-	intel_dmc_ucode_suspend(i915);
+	intel_dmc_suspend(i915);
 
 	i915_gem_suspend(i915);
 
@@ -1032,6 +1038,13 @@ static bool suspend_to_idle(struct drm_i915_private *dev_priv)
 	return false;
 }
 
+static void i915_drm_complete(struct drm_device *dev)
+{
+	struct drm_i915_private *i915 = to_i915(dev);
+
+	intel_pxp_resume_complete(i915->pxp);
+}
+
 static int i915_drm_prepare(struct drm_device *dev)
 {
 	struct drm_i915_private *i915 = to_i915(dev);
@@ -1072,8 +1085,6 @@ static int i915_drm_suspend(struct drm_device *dev)
 
 	intel_suspend_encoders(dev_priv);
 
-	intel_suspend_hw(dev_priv);
-
 	/* Must be called before GGTT is suspended. */
 	intel_dpt_suspend(dev_priv);
 	i915_ggtt_suspend(to_gt(dev_priv)->ggtt);
@@ -1087,7 +1098,7 @@ static int i915_drm_suspend(struct drm_device *dev)
 
 	dev_priv->suspend_count++;
 
-	intel_dmc_ucode_suspend(dev_priv);
+	intel_dmc_suspend(dev_priv);
 
 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
 
@@ -1208,7 +1219,7 @@ static int i915_drm_resume(struct drm_device *dev)
 	/* Must be called after GGTT is resumed. */
 	intel_dpt_resume(dev_priv);
 
-	intel_dmc_ucode_resume(dev_priv);
+	intel_dmc_resume(dev_priv);
 
 	i915_restore_display(dev_priv);
 	intel_pps_unlock_regs_wa(dev_priv);
@@ -1232,10 +1243,8 @@ static int i915_drm_resume(struct drm_device *dev)
 
 	i915_gem_resume(dev_priv);
 
-	intel_pxp_resume(dev_priv->pxp);
-
 	intel_modeset_init_hw(dev_priv);
-	intel_init_clock_gating(dev_priv);
+	intel_clock_gating_init(dev_priv);
 	intel_hpd_init(dev_priv);
 
 	/* MST sideband requires HPD interrupts enabled */
@@ -1425,6 +1434,16 @@ static int i915_pm_resume(struct device *kdev)
 	return i915_drm_resume(&i915->drm);
 }
 
+static void i915_pm_complete(struct device *kdev)
+{
+	struct drm_i915_private *i915 = kdev_to_i915(kdev);
+
+	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
+		return;
+
+	i915_drm_complete(&i915->drm);
+}
+
 /* freeze: before creating the hibernation_image */
 static int i915_pm_freeze(struct device *kdev)
 {
@@ -1645,6 +1664,7 @@ const struct dev_pm_ops i915_pm_ops = {
 	.suspend_late = i915_pm_suspend_late,
 	.resume_early = i915_pm_resume_early,
 	.resume = i915_pm_resume,
+	.complete = i915_pm_complete,
 
 	/*
 	 * S4 event handlers
diff --git a/drivers/gpu/drm/i915/i915_drm_client.c b/drivers/gpu/drm/i915/i915_drm_client.c
index b09d1d386574..e8fa172ebe5e 100644
--- a/drivers/gpu/drm/i915/i915_drm_client.c
+++ b/drivers/gpu/drm/i915/i915_drm_client.c
@@ -130,7 +130,7 @@ void i915_drm_client_fdinfo(struct seq_file *m, struct file *f)
 {
 	struct drm_file *file = f->private_data;
 	struct drm_i915_file_private *file_priv = file->driver_priv;
-	struct drm_i915_private *i915 = file_priv->dev_priv;
+	struct drm_i915_private *i915 = file_priv->i915;
 	struct i915_drm_client *client = file_priv->client;
 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 	unsigned int i;
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 4295306487c7..e771fdc3099c 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -580,6 +580,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 	IS_SUBPLATFORM(dev_priv, INTEL_ALDERLAKE_P, INTEL_SUBPLATFORM_N)
 #define IS_ADLP_RPLP(dev_priv) \
 	IS_SUBPLATFORM(dev_priv, INTEL_ALDERLAKE_P, INTEL_SUBPLATFORM_RPL)
+#define IS_ADLP_RPLU(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_ALDERLAKE_P, INTEL_SUBPLATFORM_RPLU)
 #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \
 				    (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00)
 #define IS_BDW_ULT(dev_priv) \
@@ -653,22 +655,9 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 	(IS_TIGERLAKE(__i915) && \
 	 IS_DISPLAY_STEP(__i915, since, until))
 
-#define IS_TGL_UY_GRAPHICS_STEP(__i915, since, until) \
-	(IS_TGL_UY(__i915) && \
-	 IS_GRAPHICS_STEP(__i915, since, until))
-
-#define IS_TGL_GRAPHICS_STEP(__i915, since, until) \
-	(IS_TIGERLAKE(__i915) && !IS_TGL_UY(__i915)) && \
-	 IS_GRAPHICS_STEP(__i915, since, until))
-
 #define IS_RKL_DISPLAY_STEP(p, since, until) \
 	(IS_ROCKETLAKE(p) && IS_DISPLAY_STEP(p, since, until))
 
-#define IS_DG1_GRAPHICS_STEP(p, since, until) \
-	(IS_DG1(p) && IS_GRAPHICS_STEP(p, since, until))
-#define IS_DG1_DISPLAY_STEP(p, since, until) \
-	(IS_DG1(p) && IS_DISPLAY_STEP(p, since, until))
-
 #define IS_ADLS_DISPLAY_STEP(__i915, since, until) \
 	(IS_ALDERLAKE_S(__i915) && \
 	 IS_DISPLAY_STEP(__i915, since, until))
@@ -824,6 +813,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 #define HAS_FBC(dev_priv)	(RUNTIME_INFO(dev_priv)->fbc_mask != 0)
 #define HAS_CUR_FBC(dev_priv)	(!HAS_GMCH(dev_priv) && DISPLAY_VER(dev_priv) >= 7)
 
+#define HAS_DPT(dev_priv)	(DISPLAY_VER(dev_priv) >= 13)
+
 #define HAS_IPS(dev_priv)	(IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv))
 
 #define HAS_DP_MST(dev_priv)	(INTEL_INFO(dev_priv)->display.has_dp_mst)
@@ -869,6 +860,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 	(INTEL_INFO(dev_priv)->has_oa_bpc_reporting)
 #define HAS_OA_SLICE_CONTRIB_LIMITS(dev_priv) \
 	(INTEL_INFO(dev_priv)->has_oa_slice_contrib_limits)
+#define HAS_OAM(dev_priv) \
+	(INTEL_INFO(dev_priv)->has_oam)
 
 /*
  * Set this flag, when platform requires 64K GTT page sizes or larger for
@@ -876,7 +869,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
  */
 #define HAS_64K_PAGES(dev_priv) (INTEL_INFO(dev_priv)->has_64k_pages)
 
-#define HAS_IPC(dev_priv)		 (INTEL_INFO(dev_priv)->display.has_ipc)
+#define HAS_IPC(dev_priv)		(INTEL_INFO(dev_priv)->display.has_ipc)
+#define HAS_SAGV(dev_priv)		(DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv))
 
 #define HAS_REGION(i915, i) (RUNTIME_INFO(i915)->memory_regions & (i))
 #define HAS_LMEM(i915) HAS_REGION(i915, REGION_LMEM)
diff --git a/drivers/gpu/drm/i915/i915_file_private.h b/drivers/gpu/drm/i915/i915_file_private.h
index f42877869692..c9cb8eecacde 100644
--- a/drivers/gpu/drm/i915/i915_file_private.h
+++ b/drivers/gpu/drm/i915/i915_file_private.h
@@ -15,7 +15,7 @@ struct drm_file;
 struct i915_drm_client;
 
 struct drm_i915_file_private {
-	struct drm_i915_private *dev_priv;
+	struct drm_i915_private *i915;
 
 	union {
 		struct drm_file *file;
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 35950fa91406..0a78bdbd36b1 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -58,7 +58,7 @@
 #include "i915_file_private.h"
 #include "i915_trace.h"
 #include "i915_vgpu.h"
-#include "intel_pm.h"
+#include "intel_clock_gating.h"
 
 static int
 insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
@@ -444,7 +444,7 @@ out_rpm:
 }
 
 /**
- * Reads data from the object referenced by handle.
+ * i915_gem_pread_ioctl - Reads data from the object referenced by handle.
  * @dev: drm device pointer
  * @data: ioctl data blob
  * @file: drm file pointer
@@ -533,7 +533,7 @@ ggtt_write(struct io_mapping *mapping,
 }
 
 /**
- * This is the fast pwrite path, where we copy the data directly from the
+ * i915_gem_gtt_pwrite_fast - This is the fast pwrite path, where we copy the data directly from the
  * user into the GTT, uncached.
  * @obj: i915 GEM object
  * @args: pwrite arguments structure
@@ -723,7 +723,7 @@ err_unlock:
 }
 
 /**
- * Writes data to the object referenced by handle.
+ * i915_gem_pwrite_ioctl - Writes data to the object referenced by handle.
  * @dev: drm device
  * @data: ioctl data blob
  * @file: drm file
@@ -808,7 +808,7 @@ err:
 }
 
 /**
- * Called when user space has done writes to this buffer
+ * i915_gem_sw_finish_ioctl - Called when user space has done writes to this buffer
  * @dev: drm device
  * @data: ioctl data blob
  * @file: drm file
@@ -1164,7 +1164,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 	}
 
 	/*
-	 * Despite its name intel_init_clock_gating applies both display
+	 * Despite its name intel_clock_gating_init applies both display
 	 * clock gating workarounds; GT mmio workarounds and the occasional
 	 * GT power context workaround. Worse, sometimes it includes a context
 	 * register workaround which we need to apply before we record the
@@ -1172,7 +1172,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 	 *
 	 * FIXME: break up the workarounds and apply them at the right time!
 	 */
-	intel_init_clock_gating(dev_priv);
+	intel_clock_gating_init(dev_priv);
 
 	for_each_gt(gt, dev_priv, i) {
 		ret = intel_gt_init(gt);
@@ -1216,7 +1216,7 @@ err_unlock:
 		/* Minimal basic recovery for KMS */
 		ret = i915_ggtt_enable_hw(dev_priv);
 		i915_ggtt_resume(to_gt(dev_priv)->ggtt);
-		intel_init_clock_gating(dev_priv);
+		intel_clock_gating_init(dev_priv);
 	}
 
 	i915_gem_drain_freed_objects(dev_priv);
@@ -1313,7 +1313,7 @@ int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
 	}
 
 	file->driver_priv = file_priv;
-	file_priv->dev_priv = i915;
+	file_priv->i915 = i915;
 	file_priv->file = file;
 	file_priv->client = client;
 
diff --git a/drivers/gpu/drm/i915/i915_getparam.c b/drivers/gpu/drm/i915/i915_getparam.c
index 61ef2d9cfa62..2238e096c957 100644
--- a/drivers/gpu/drm/i915/i915_getparam.c
+++ b/drivers/gpu/drm/i915/i915_getparam.c
@@ -173,7 +173,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data,
 		value = INTEL_INFO(i915)->has_coherent_ggtt;
 		break;
 	case I915_PARAM_PERF_REVISION:
-		value = i915_perf_ioctl_version();
+		value = i915_perf_ioctl_version(i915);
 		break;
 	case I915_PARAM_OA_TIMESTAMP_FREQUENCY:
 		value = i915_perf_oa_timestamp_frequency(i915);
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 904f21e1380c..f020c0086fbc 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -505,6 +505,7 @@ static void error_print_context(struct drm_i915_error_state_buf *m,
 		   header, ctx->comm, ctx->pid, ctx->sched_attr.priority,
 		   ctx->guilty, ctx->active,
 		   ctx->total_runtime, ctx->avg_runtime);
+	err_printf(m, "  context timeline seqno %u\n", ctx->hwsp_seqno);
 }
 
 static struct i915_vma_coredump *
@@ -1395,6 +1396,8 @@ static bool record_context(struct i915_gem_context_coredump *e,
 	e->sched_attr = ctx->sched;
 	e->guilty = atomic_read(&ctx->guilty_count);
 	e->active = atomic_read(&ctx->active_count);
+	e->hwsp_seqno = (ce->timeline && ce->timeline->hwsp_seqno) ?
+				*ce->timeline->hwsp_seqno : ~0U;
 
 	e->total_runtime = intel_context_get_total_runtime_ns(ce);
 	e->avg_runtime = intel_context_get_avg_runtime_ns(ce);
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h
index 56027ffbce51..a91932cc6531 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.h
+++ b/drivers/gpu/drm/i915/i915_gpu_error.h
@@ -107,6 +107,7 @@ struct intel_engine_coredump {
 		int active;
 		int guilty;
 		struct i915_sched_attr sched_attr;
+		u32 hwsp_seqno;
 	} context;
 
 	struct i915_vma_coredump *vma;
diff --git a/drivers/gpu/drm/i915/i915_hwmon.c b/drivers/gpu/drm/i915/i915_hwmon.c
index 1225bc432f0d..8e7dccc8d3a0 100644
--- a/drivers/gpu/drm/i915/i915_hwmon.c
+++ b/drivers/gpu/drm/i915/i915_hwmon.c
@@ -99,20 +99,6 @@ hwm_field_read_and_scale(struct hwm_drvdata *ddat, i915_reg_t rgadr,
 	return mul_u64_u32_shr(reg_value, scale_factor, nshift);
 }
 
-static void
-hwm_field_scale_and_write(struct hwm_drvdata *ddat, i915_reg_t rgadr,
-			  int nshift, unsigned int scale_factor, long lval)
-{
-	u32 nval;
-
-	/* Computation in 64-bits to avoid overflow. Round to nearest. */
-	nval = DIV_ROUND_CLOSEST_ULL((u64)lval << nshift, scale_factor);
-
-	hwm_locked_with_pm_intel_uncore_rmw(ddat, rgadr,
-					    PKG_PWR_LIM_1,
-					    REG_FIELD_PREP(PKG_PWR_LIM_1, nval));
-}
-
 /*
  * hwm_energy - Obtain energy value
  *
@@ -232,11 +218,15 @@ hwm_power1_max_interval_store(struct device *dev,
 	/* val in hw units */
 	val = DIV_ROUND_CLOSEST_ULL((u64)val << hwmon->scl_shift_time, SF_TIME);
 	/* Convert to 1.x * power(2,y) */
-	if (!val)
-		return -EINVAL;
-	y = ilog2(val);
-	/* x = (val - (1 << y)) >> (y - 2); */
-	x = (val - (1ul << y)) << x_w >> y;
+	if (!val) {
+		/* Avoid ilog2(0) */
+		y = 0;
+		x = 0;
+	} else {
+		y = ilog2(val);
+		/* x = (val - (1 << y)) >> (y - 2); */
+		x = (val - (1ul << y)) << x_w >> y;
+	}
 
 	rxy = REG_FIELD_PREP(PKG_PWR_LIM_1_TIME_X, x) | REG_FIELD_PREP(PKG_PWR_LIM_1_TIME_Y, y);
 
@@ -359,6 +349,8 @@ hwm_power_is_visible(const struct hwm_drvdata *ddat, u32 attr, int chan)
 	}
 }
 
+#define PL1_DISABLE 0
+
 /*
  * HW allows arbitrary PL1 limits to be set but silently clamps these values to
  * "typical but not guaranteed" min/max values in rg.pkg_power_sku. Follow the
@@ -372,6 +364,14 @@ hwm_power_max_read(struct hwm_drvdata *ddat, long *val)
 	intel_wakeref_t wakeref;
 	u64 r, min, max;
 
+	/* Check if PL1 limit is disabled */
+	with_intel_runtime_pm(ddat->uncore->rpm, wakeref)
+		r = intel_uncore_read(ddat->uncore, hwmon->rg.pkg_rapl_limit);
+	if (!(r & PKG_PWR_LIM_1_EN)) {
+		*val = PL1_DISABLE;
+		return 0;
+	}
+
 	*val = hwm_field_read_and_scale(ddat,
 					hwmon->rg.pkg_rapl_limit,
 					PKG_PWR_LIM_1,
@@ -392,6 +392,38 @@ hwm_power_max_read(struct hwm_drvdata *ddat, long *val)
 }
 
 static int
+hwm_power_max_write(struct hwm_drvdata *ddat, long val)
+{
+	struct i915_hwmon *hwmon = ddat->hwmon;
+	intel_wakeref_t wakeref;
+	u32 nval;
+
+	/* Disable PL1 limit and verify, because the limit cannot be disabled on all platforms */
+	if (val == PL1_DISABLE) {
+		mutex_lock(&hwmon->hwmon_lock);
+		with_intel_runtime_pm(ddat->uncore->rpm, wakeref) {
+			intel_uncore_rmw(ddat->uncore, hwmon->rg.pkg_rapl_limit,
+					 PKG_PWR_LIM_1_EN, 0);
+			nval = intel_uncore_read(ddat->uncore, hwmon->rg.pkg_rapl_limit);
+		}
+		mutex_unlock(&hwmon->hwmon_lock);
+
+		if (nval & PKG_PWR_LIM_1_EN)
+			return -ENODEV;
+		return 0;
+	}
+
+	/* Computation in 64-bits to avoid overflow. Round to nearest. */
+	nval = DIV_ROUND_CLOSEST_ULL((u64)val << hwmon->scl_shift_power, SF_POWER);
+	nval = PKG_PWR_LIM_1_EN | REG_FIELD_PREP(PKG_PWR_LIM_1, nval);
+
+	hwm_locked_with_pm_intel_uncore_rmw(ddat, hwmon->rg.pkg_rapl_limit,
+					    PKG_PWR_LIM_1_EN | PKG_PWR_LIM_1,
+					    nval);
+	return 0;
+}
+
+static int
 hwm_power_read(struct hwm_drvdata *ddat, u32 attr, int chan, long *val)
 {
 	struct i915_hwmon *hwmon = ddat->hwmon;
@@ -425,16 +457,11 @@ hwm_power_read(struct hwm_drvdata *ddat, u32 attr, int chan, long *val)
 static int
 hwm_power_write(struct hwm_drvdata *ddat, u32 attr, int chan, long val)
 {
-	struct i915_hwmon *hwmon = ddat->hwmon;
 	u32 uval;
 
 	switch (attr) {
 	case hwmon_power_max:
-		hwm_field_scale_and_write(ddat,
-					  hwmon->rg.pkg_rapl_limit,
-					  hwmon->scl_shift_power,
-					  SF_POWER, val);
-		return 0;
+		return hwm_power_max_write(ddat, val);
 	case hwmon_power_crit:
 		uval = DIV_ROUND_CLOSEST_ULL(val << POWER_SETUP_I1_SHIFT, SF_POWER);
 		return hwm_pcode_write_i1(ddat->uncore->i915, uval);
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 240d5e198904..d24bdea65a3d 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -37,10 +37,12 @@
 #include "display/intel_de.h"
 #include "display/intel_display_trace.h"
 #include "display/intel_display_types.h"
+#include "display/intel_fdi_regs.h"
 #include "display/intel_fifo_underrun.h"
 #include "display/intel_hotplug.h"
 #include "display/intel_lpe_audio.h"
 #include "display/intel_psr.h"
+#include "display/intel_psr_regs.h"
 
 #include "gt/intel_breadcrumbs.h"
 #include "gt/intel_gt.h"
@@ -52,7 +54,6 @@
 #include "i915_driver.h"
 #include "i915_drv.h"
 #include "i915_irq.h"
-#include "intel_pm.h"
 
 /**
  * DOC: interrupt handling
@@ -81,8 +82,7 @@ static inline void pmu_irq_stats(struct drm_i915_private *i915,
 }
 
 typedef bool (*long_pulse_detect_func)(enum hpd_pin pin, u32 val);
-typedef u32 (*hotplug_enables_func)(struct drm_i915_private *i915,
-				    enum hpd_pin pin);
+typedef u32 (*hotplug_enables_func)(struct intel_encoder *encoder);
 
 static const u32 hpd_ilk[HPD_NUM_PINS] = {
 	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
@@ -199,6 +199,8 @@ static void intel_hpd_init_pins(struct drm_i915_private *dev_priv)
 		hpd->hpd = hpd_gen11;
 	else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
 		hpd->hpd = hpd_bxt;
+	else if (DISPLAY_VER(dev_priv) == 9)
+		hpd->hpd = NULL; /* no north HPD on SKL */
 	else if (DISPLAY_VER(dev_priv) >= 8)
 		hpd->hpd = hpd_bdw;
 	else if (DISPLAY_VER(dev_priv) >= 7)
@@ -884,7 +886,7 @@ static u32 intel_hpd_hotplug_enables(struct drm_i915_private *i915,
 	u32 hotplug = 0;
 
 	for_each_intel_encoder(&i915->drm, encoder)
-		hotplug |= hotplug_enables(i915, encoder->hpd_pin);
+		hotplug |= hotplug_enables(encoder);
 
 	return hotplug;
 }
@@ -2835,10 +2837,11 @@ static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
 	spin_unlock_irq(&dev_priv->irq_lock);
 }
 
-static u32 ibx_hotplug_enables(struct drm_i915_private *i915,
-			       enum hpd_pin pin)
+static u32 ibx_hotplug_enables(struct intel_encoder *encoder)
 {
-	switch (pin) {
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+
+	switch (encoder->hpd_pin) {
 	case HPD_PORT_A:
 		/*
 		 * When CPU and PCH are on the same package, port A
@@ -2890,31 +2893,29 @@ static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
 	ibx_hpd_detection_setup(dev_priv);
 }
 
-static u32 icp_ddi_hotplug_enables(struct drm_i915_private *i915,
-				   enum hpd_pin pin)
+static u32 icp_ddi_hotplug_enables(struct intel_encoder *encoder)
 {
-	switch (pin) {
+	switch (encoder->hpd_pin) {
 	case HPD_PORT_A:
 	case HPD_PORT_B:
 	case HPD_PORT_C:
 	case HPD_PORT_D:
-		return SHOTPLUG_CTL_DDI_HPD_ENABLE(pin);
+		return SHOTPLUG_CTL_DDI_HPD_ENABLE(encoder->hpd_pin);
 	default:
 		return 0;
 	}
 }
 
-static u32 icp_tc_hotplug_enables(struct drm_i915_private *i915,
-				  enum hpd_pin pin)
+static u32 icp_tc_hotplug_enables(struct intel_encoder *encoder)
 {
-	switch (pin) {
+	switch (encoder->hpd_pin) {
 	case HPD_PORT_TC1:
 	case HPD_PORT_TC2:
 	case HPD_PORT_TC3:
 	case HPD_PORT_TC4:
 	case HPD_PORT_TC5:
 	case HPD_PORT_TC6:
-		return ICP_TC_HPD_ENABLE(pin);
+		return ICP_TC_HPD_ENABLE(encoder->hpd_pin);
 	default:
 		return 0;
 	}
@@ -2958,17 +2959,16 @@ static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv)
 	icp_tc_hpd_detection_setup(dev_priv);
 }
 
-static u32 gen11_hotplug_enables(struct drm_i915_private *i915,
-				 enum hpd_pin pin)
+static u32 gen11_hotplug_enables(struct intel_encoder *encoder)
 {
-	switch (pin) {
+	switch (encoder->hpd_pin) {
 	case HPD_PORT_TC1:
 	case HPD_PORT_TC2:
 	case HPD_PORT_TC3:
 	case HPD_PORT_TC4:
 	case HPD_PORT_TC5:
 	case HPD_PORT_TC6:
-		return GEN11_HOTPLUG_CTL_ENABLE(pin);
+		return GEN11_HOTPLUG_CTL_ENABLE(encoder->hpd_pin);
 	default:
 		return 0;
 	}
@@ -3031,10 +3031,9 @@ static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
 		icp_hpd_irq_setup(dev_priv);
 }
 
-static u32 spt_hotplug_enables(struct drm_i915_private *i915,
-			       enum hpd_pin pin)
+static u32 spt_hotplug_enables(struct intel_encoder *encoder)
 {
-	switch (pin) {
+	switch (encoder->hpd_pin) {
 	case HPD_PORT_A:
 		return PORTA_HOTPLUG_ENABLE;
 	case HPD_PORT_B:
@@ -3048,10 +3047,9 @@ static u32 spt_hotplug_enables(struct drm_i915_private *i915,
 	}
 }
 
-static u32 spt_hotplug2_enables(struct drm_i915_private *i915,
-				enum hpd_pin pin)
+static u32 spt_hotplug2_enables(struct intel_encoder *encoder)
 {
-	switch (pin) {
+	switch (encoder->hpd_pin) {
 	case HPD_PORT_E:
 		return PORTE_HOTPLUG_ENABLE;
 	default:
@@ -3094,10 +3092,9 @@ static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
 	spt_hpd_detection_setup(dev_priv);
 }
 
-static u32 ilk_hotplug_enables(struct drm_i915_private *i915,
-			       enum hpd_pin pin)
+static u32 ilk_hotplug_enables(struct intel_encoder *encoder)
 {
-	switch (pin) {
+	switch (encoder->hpd_pin) {
 	case HPD_PORT_A:
 		return DIGITAL_PORTA_HOTPLUG_ENABLE |
 			DIGITAL_PORTA_PULSE_DURATION_2ms;
@@ -3135,25 +3132,24 @@ static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
 	ibx_hpd_irq_setup(dev_priv);
 }
 
-static u32 bxt_hotplug_enables(struct drm_i915_private *i915,
-			       enum hpd_pin pin)
+static u32 bxt_hotplug_enables(struct intel_encoder *encoder)
 {
 	u32 hotplug;
 
-	switch (pin) {
+	switch (encoder->hpd_pin) {
 	case HPD_PORT_A:
 		hotplug = PORTA_HOTPLUG_ENABLE;
-		if (intel_bios_is_port_hpd_inverted(i915, PORT_A))
+		if (intel_bios_encoder_hpd_invert(encoder->devdata))
 			hotplug |= BXT_DDIA_HPD_INVERT;
 		return hotplug;
 	case HPD_PORT_B:
 		hotplug = PORTB_HOTPLUG_ENABLE;
-		if (intel_bios_is_port_hpd_inverted(i915, PORT_B))
+		if (intel_bios_encoder_hpd_invert(encoder->devdata))
 			hotplug |= BXT_DDIB_HPD_INVERT;
 		return hotplug;
 	case HPD_PORT_C:
 		hotplug = PORTC_HOTPLUG_ENABLE;
-		if (intel_bios_is_port_hpd_inverted(i915, PORT_C))
+		if (intel_bios_encoder_hpd_invert(encoder->devdata))
 			hotplug |= BXT_DDIC_HPD_INVERT;
 		return hotplug;
 	default:
@@ -3471,15 +3467,33 @@ static void i8xx_irq_reset(struct drm_i915_private *dev_priv)
 	dev_priv->irq_mask = ~0u;
 }
 
+static u32 i9xx_error_mask(struct drm_i915_private *i915)
+{
+	/*
+	 * On gen2/3 FBC generates (seemingly spurious)
+	 * display INVALID_GTT/INVALID_GTT_PTE table errors.
+	 *
+	 * Also gen3 bspec has this to say:
+	 * "DISPA_INVALID_GTT_PTE
+	 "  [DevNapa] : Reserved. This bit does not reflect the page
+	 "              table error for the display plane A."
+	 *
+	 * Unfortunately we can't mask off individual PGTBL_ER bits,
+	 * so we just have to mask off all page table errors via EMR.
+	 */
+	if (HAS_FBC(i915))
+		return ~I915_ERROR_MEMORY_REFRESH;
+	else
+		return ~(I915_ERROR_PAGE_TABLE |
+			 I915_ERROR_MEMORY_REFRESH);
+}
+
 static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv)
 {
 	struct intel_uncore *uncore = &dev_priv->uncore;
 	u16 enable_mask;
 
-	intel_uncore_write16(uncore,
-			     EMR,
-			     ~(I915_ERROR_PAGE_TABLE |
-			       I915_ERROR_MEMORY_REFRESH));
+	intel_uncore_write16(uncore, EMR, i9xx_error_mask(dev_priv));
 
 	/* Unmask the interrupts that we always want on. */
 	dev_priv->irq_mask =
@@ -3510,9 +3524,7 @@ static void i8xx_error_irq_ack(struct drm_i915_private *i915,
 	u16 emr;
 
 	*eir = intel_uncore_read16(uncore, EIR);
-
-	if (*eir)
-		intel_uncore_write16(uncore, EIR, *eir);
+	intel_uncore_write16(uncore, EIR, *eir);
 
 	*eir_stuck = intel_uncore_read16(uncore, EIR);
 	if (*eir_stuck == 0)
@@ -3541,6 +3553,9 @@ static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
 	if (eir_stuck)
 		drm_dbg(&dev_priv->drm, "EIR stuck: 0x%04x, masked\n",
 			eir_stuck);
+
+	drm_dbg(&dev_priv->drm, "PGTBL_ER: 0x%08x\n",
+		intel_uncore_read(&dev_priv->uncore, PGTBL_ER));
 }
 
 static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
@@ -3548,7 +3563,8 @@ static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
 {
 	u32 emr;
 
-	*eir = intel_uncore_rmw(&dev_priv->uncore, EIR, 0, 0);
+	*eir = intel_uncore_read(&dev_priv->uncore, EIR);
+	intel_uncore_write(&dev_priv->uncore, EIR, *eir);
 
 	*eir_stuck = intel_uncore_read(&dev_priv->uncore, EIR);
 	if (*eir_stuck == 0)
@@ -3564,7 +3580,8 @@ static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
 	 * (or by a GPU reset) so we mask any bit that
 	 * remains set.
 	 */
-	emr = intel_uncore_rmw(&dev_priv->uncore, EMR, ~0, 0xffffffff);
+	emr = intel_uncore_read(&dev_priv->uncore, EMR);
+	intel_uncore_write(&dev_priv->uncore, EMR, 0xffffffff);
 	intel_uncore_write(&dev_priv->uncore, EMR, emr | *eir_stuck);
 }
 
@@ -3576,6 +3593,9 @@ static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
 	if (eir_stuck)
 		drm_dbg(&dev_priv->drm, "EIR stuck: 0x%08x, masked\n",
 			eir_stuck);
+
+	drm_dbg(&dev_priv->drm, "PGTBL_ER: 0x%08x\n",
+		intel_uncore_read(&dev_priv->uncore, PGTBL_ER));
 }
 
 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
@@ -3645,8 +3665,7 @@ static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
 	struct intel_uncore *uncore = &dev_priv->uncore;
 	u32 enable_mask;
 
-	intel_uncore_write(uncore, EMR, ~(I915_ERROR_PAGE_TABLE |
-					  I915_ERROR_MEMORY_REFRESH));
+	intel_uncore_write(uncore, EMR, i9xx_error_mask(dev_priv));
 
 	/* Unmask the interrupts that we always want on. */
 	dev_priv->irq_mask =
@@ -3749,26 +3768,31 @@ static void i965_irq_reset(struct drm_i915_private *dev_priv)
 	dev_priv->irq_mask = ~0u;
 }
 
-static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
+static u32 i965_error_mask(struct drm_i915_private *i915)
 {
-	struct intel_uncore *uncore = &dev_priv->uncore;
-	u32 enable_mask;
-	u32 error_mask;
-
 	/*
 	 * Enable some error detection, note the instruction error mask
 	 * bit is reserved, so we leave it masked.
+	 *
+	 * i965 FBC no longer generates spurious GTT errors,
+	 * so we can always enable the page table errors.
 	 */
-	if (IS_G4X(dev_priv)) {
-		error_mask = ~(GM45_ERROR_PAGE_TABLE |
-			       GM45_ERROR_MEM_PRIV |
-			       GM45_ERROR_CP_PRIV |
-			       I915_ERROR_MEMORY_REFRESH);
-	} else {
-		error_mask = ~(I915_ERROR_PAGE_TABLE |
-			       I915_ERROR_MEMORY_REFRESH);
-	}
-	intel_uncore_write(uncore, EMR, error_mask);
+	if (IS_G4X(i915))
+		return ~(GM45_ERROR_PAGE_TABLE |
+			 GM45_ERROR_MEM_PRIV |
+			 GM45_ERROR_CP_PRIV |
+			 I915_ERROR_MEMORY_REFRESH);
+	else
+		return ~(I915_ERROR_PAGE_TABLE |
+			 I915_ERROR_MEMORY_REFRESH);
+}
+
+static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
+{
+	struct intel_uncore *uncore = &dev_priv->uncore;
+	u32 enable_mask;
+
+	intel_uncore_write(uncore, EMR, i965_error_mask(dev_priv));
 
 	/* Unmask the interrupts that we always want on. */
 	dev_priv->irq_mask =
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index ade744cccfea..0a171b57fd8f 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -121,6 +121,9 @@ i915_param_named_unsafe(enable_psr2_sel_fetch, bool, 0400,
 	"(0=disabled, 1=enabled) "
 	"Default: 0");
 
+i915_param_named_unsafe(enable_sagv, bool, 0600,
+	"Enable system agent voltage/frequency scaling (SAGV) (default: true)");
+
 i915_param_named_unsafe(force_probe, charp, 0400,
 	"Force probe options for specified supported devices. "
 	"See CONFIG_DRM_I915_FORCE_PROBE for details.");
@@ -131,6 +134,9 @@ i915_param_named_unsafe(disable_power_well, int, 0400,
 
 i915_param_named_unsafe(enable_ips, int, 0400, "Enable IPS (default: true)");
 
+i915_param_named_unsafe(enable_dpt, bool, 0400,
+	"Enable display page table (DPT) (default: true)");
+
 i915_param_named(fastboot, int, 0400,
 	"Try to skip unnecessary mode sets at boot time "
 	"(0=disabled, 1=enabled) "
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 3f51f90145b6..68abf0ad6c00 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -54,8 +54,10 @@ struct drm_printer;
 	param(int, enable_dc, -1, 0400) \
 	param(int, enable_fbc, -1, 0600) \
 	param(int, enable_psr, -1, 0600) \
+	param(bool, enable_dpt, true, 0400) \
 	param(bool, psr_safest_params, false, 0400) \
 	param(bool, enable_psr2_sel_fetch, true, 0400) \
+	param(bool, enable_sagv, true, 0600) \
 	param(int, disable_power_well, -1, 0400) \
 	param(int, enable_ips, 1, 0600) \
 	param(int, invert_brightness, 0, 0600) \
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index a8d942b16223..cddb6e197972 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -896,7 +896,6 @@ static const struct intel_device_info jsl_info = {
 static const struct intel_device_info tgl_info = {
 	GEN12_FEATURES,
 	PLATFORM(INTEL_TIGERLAKE),
-	.display.has_modular_fia = 1,
 	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 };
@@ -996,7 +995,6 @@ static const struct intel_device_info adl_p_info = {
 			       BIT(TRANSCODER_C) | BIT(TRANSCODER_D) |
 			       BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1),
 	.display.has_cdclk_crawl = 1,
-	.display.has_modular_fia = 1,
 	.display.has_psr_hw_tracking = 0,
 	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
@@ -1028,6 +1026,7 @@ static const struct intel_device_info adl_p_info = {
 	.has_mslice_steering = 1, \
 	.has_oa_bpc_reporting = 1, \
 	.has_oa_slice_contrib_limits = 1, \
+	.has_oam = 1, \
 	.has_rc6 = 1, \
 	.has_reset_engine = 1, \
 	.has_rps = 1, \
@@ -1143,7 +1142,6 @@ static const struct intel_device_info mtl_info = {
 	.__runtime.graphics.ip.rel = 70,
 	.__runtime.media.ip.ver = 13,
 	PLATFORM(INTEL_METEORLAKE),
-	.display.has_modular_fia = 1,
 	.extra_gt_list = xelpmp_extra_gt,
 	.has_flat_ccs = 0,
 	.has_gmd_id = 1,
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index 004074936300..050b8ae7b8e7 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -192,6 +192,7 @@
  */
 
 #include <linux/anon_inodes.h>
+#include <linux/nospec.h>
 #include <linux/sizes.h>
 #include <linux/uuid.h>
 
@@ -208,6 +209,7 @@
 #include "gt/intel_gt_regs.h"
 #include "gt/intel_lrc.h"
 #include "gt/intel_lrc_reg.h"
+#include "gt/intel_rc6.h"
 #include "gt/intel_ring.h"
 #include "gt/uc/intel_guc_slpc.h"
 
@@ -326,6 +328,12 @@ static const struct i915_oa_format oa_formats[I915_OA_FORMAT_MAX] = {
 	[I915_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256 },
 	[I915_OAR_FORMAT_A32u40_A4u32_B8_C8]    = { 5, 256 },
 	[I915_OA_FORMAT_A24u40_A14u32_B8_C8]    = { 5, 256 },
+	[I915_OAM_FORMAT_MPEC8u64_B8_C8]	= { 1, 192, TYPE_OAM, HDR_64_BIT },
+	[I915_OAM_FORMAT_MPEC8u32_B8_C8]	= { 2, 128, TYPE_OAM, HDR_64_BIT },
+};
+
+static const u32 mtl_oa_base[] = {
+	[PERF_GROUP_OAM_SAMEDIA_0] = 0x393000,
 };
 
 #define SAMPLE_OA_REPORT      (1<<0)
@@ -418,11 +426,17 @@ static void free_oa_config_bo(struct i915_oa_config_bo *oa_bo)
 	kfree(oa_bo);
 }
 
+static inline const
+struct i915_perf_regs *__oa_regs(struct i915_perf_stream *stream)
+{
+	return &stream->engine->oa_group->regs;
+}
+
 static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream)
 {
 	struct intel_uncore *uncore = stream->uncore;
 
-	return intel_uncore_read(uncore, GEN12_OAG_OATAILPTR) &
+	return intel_uncore_read(uncore, __oa_regs(stream)->oa_tail_ptr) &
 	       GEN12_OAG_OATAILPTR_MASK;
 }
 
@@ -441,6 +455,67 @@ static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream)
 	return oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
 }
 
+#define oa_report_header_64bit(__s) \
+	((__s)->oa_buffer.format->header == HDR_64_BIT)
+
+static u64 oa_report_id(struct i915_perf_stream *stream, void *report)
+{
+	return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report;
+}
+
+static u64 oa_report_reason(struct i915_perf_stream *stream, void *report)
+{
+	return (oa_report_id(stream, report) >> OAREPORT_REASON_SHIFT) &
+	       (GRAPHICS_VER(stream->perf->i915) == 12 ?
+		OAREPORT_REASON_MASK_EXTENDED :
+		OAREPORT_REASON_MASK);
+}
+
+static void oa_report_id_clear(struct i915_perf_stream *stream, u32 *report)
+{
+	if (oa_report_header_64bit(stream))
+		*(u64 *)report = 0;
+	else
+		*report = 0;
+}
+
+static bool oa_report_ctx_invalid(struct i915_perf_stream *stream, void *report)
+{
+	return !(oa_report_id(stream, report) &
+	       stream->perf->gen8_valid_ctx_bit) &&
+	       GRAPHICS_VER(stream->perf->i915) <= 11;
+}
+
+static u64 oa_timestamp(struct i915_perf_stream *stream, void *report)
+{
+	return oa_report_header_64bit(stream) ?
+		*((u64 *)report + 1) :
+		*((u32 *)report + 1);
+}
+
+static void oa_timestamp_clear(struct i915_perf_stream *stream, u32 *report)
+{
+	if (oa_report_header_64bit(stream))
+		*(u64 *)&report[2] = 0;
+	else
+		report[1] = 0;
+}
+
+static u32 oa_context_id(struct i915_perf_stream *stream, u32 *report)
+{
+	u32 ctx_id = oa_report_header_64bit(stream) ? report[4] : report[2];
+
+	return ctx_id & stream->specific_ctx_id_mask;
+}
+
+static void oa_context_id_squash(struct i915_perf_stream *stream, u32 *report)
+{
+	if (oa_report_header_64bit(stream))
+		report[4] = INVALID_CTX_ID;
+	else
+		report[2] = INVALID_CTX_ID;
+}
+
 /**
  * oa_buffer_check_unlocked - check for data and update tail ptr state
  * @stream: i915 stream instance
@@ -473,6 +548,7 @@ static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream)
 	bool pollin;
 	u32 hw_tail;
 	u64 now;
+	u32 partial_report_size;
 
 	/* We have to consider the (unlikely) possibility that read() errors
 	 * could result in an OA buffer reset which might reset the head and
@@ -482,10 +558,15 @@ static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream)
 
 	hw_tail = stream->perf->ops.oa_hw_tail_read(stream);
 
-	/* The tail pointer increases in 64 byte increments,
-	 * not in report_size steps...
+	/* The tail pointer increases in 64 byte increments, not in report_size
+	 * steps. Also the report size may not be a power of 2. Compute
+	 * potentially partially landed report in the OA buffer
 	 */
-	hw_tail &= ~(report_size - 1);
+	partial_report_size = OA_TAKEN(hw_tail, stream->oa_buffer.tail);
+	partial_report_size %= report_size;
+
+	/* Subtract partial amount off the tail */
+	hw_tail = gtt_offset + OA_TAKEN(hw_tail, partial_report_size);
 
 	now = ktime_get_mono_fast_ns();
 
@@ -509,21 +590,22 @@ static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream)
 		hw_tail -= gtt_offset;
 		tail = hw_tail;
 
-		/* Walk the stream backward until we find a report with dword 0
-		 * & 1 not at 0. Since the circular buffer pointers progress by
-		 * increments of 64 bytes and that reports can be up to 256
-		 * bytes long, we can't tell whether a report has fully landed
-		 * in memory before the first 2 dwords of the following report
-		 * have effectively landed.
+		/* Walk the stream backward until we find a report with report
+		 * id and timestmap not at 0. Since the circular buffer pointers
+		 * progress by increments of 64 bytes and that reports can be up
+		 * to 256 bytes long, we can't tell whether a report has fully
+		 * landed in memory before the report id and timestamp of the
+		 * following report have effectively landed.
 		 *
 		 * This is assuming that the writes of the OA unit land in
 		 * memory in the order they were written to.
 		 * If not : (╯°□°）╯︵ ┻━┻
 		 */
 		while (OA_TAKEN(tail, aged_tail) >= report_size) {
-			u32 *report32 = (void *)(stream->oa_buffer.vaddr + tail);
+			void *report = stream->oa_buffer.vaddr + tail;
 
-			if (report32[0] != 0 || report32[1] != 0)
+			if (oa_report_id(stream, report) ||
+			    oa_timestamp(stream, report))
 				break;
 
 			tail = (tail - report_size) & (OA_BUFFER_SIZE - 1);
@@ -607,6 +689,8 @@ static int append_oa_sample(struct i915_perf_stream *stream,
 {
 	int report_size = stream->oa_buffer.format->size;
 	struct drm_i915_perf_record_header header;
+	int report_size_partial;
+	u8 *oa_buf_end;
 
 	header.type = DRM_I915_PERF_RECORD_SAMPLE;
 	header.pad = 0;
@@ -620,8 +704,20 @@ static int append_oa_sample(struct i915_perf_stream *stream,
 		return -EFAULT;
 	buf += sizeof(header);
 
-	if (copy_to_user(buf, report, report_size))
+	oa_buf_end = stream->oa_buffer.vaddr + OA_BUFFER_SIZE;
+	report_size_partial = oa_buf_end - report;
+
+	if (report_size_partial < report_size) {
+		if (copy_to_user(buf, report, report_size_partial))
+			return -EFAULT;
+		buf += report_size_partial;
+
+		if (copy_to_user(buf, stream->oa_buffer.vaddr,
+				 report_size - report_size_partial))
+			return -EFAULT;
+	} else if (copy_to_user(buf, report, report_size)) {
 		return -EFAULT;
+	}
 
 	(*offset) += header.size;
 
@@ -685,12 +781,11 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream,
 	 * An out of bounds or misaligned head or tail pointer implies a driver
 	 * bug since we validate + align the tail pointers we read from the
 	 * hardware and we are in full control of the head pointer which should
-	 * only be incremented by multiples of the report size (notably also
-	 * all a power of two).
+	 * only be incremented by multiples of the report size.
 	 */
 	if (drm_WARN_ONCE(&uncore->i915->drm,
-			  head > OA_BUFFER_SIZE || head % report_size ||
-			  tail > OA_BUFFER_SIZE || tail % report_size,
+			  head > OA_BUFFER_SIZE ||
+			  tail > OA_BUFFER_SIZE,
 			  "Inconsistent OA buffer pointers: head = %u, tail = %u\n",
 			  head, tail))
 		return -EIO;
@@ -702,39 +797,19 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream,
 		u8 *report = oa_buf_base + head;
 		u32 *report32 = (void *)report;
 		u32 ctx_id;
-		u32 reason;
-
-		/*
-		 * All the report sizes factor neatly into the buffer
-		 * size so we never expect to see a report split
-		 * between the beginning and end of the buffer.
-		 *
-		 * Given the initial alignment check a misalignment
-		 * here would imply a driver bug that would result
-		 * in an overrun.
-		 */
-		if (drm_WARN_ON(&uncore->i915->drm,
-				(OA_BUFFER_SIZE - head) < report_size)) {
-			drm_err(&uncore->i915->drm,
-				"Spurious OA head ptr: non-integral report offset\n");
-			break;
-		}
+		u64 reason;
 
 		/*
 		 * The reason field includes flags identifying what
 		 * triggered this specific report (mostly timer
 		 * triggered or e.g. due to a context switch).
 		 *
-		 * This field is never expected to be zero so we can
-		 * check that the report isn't invalid before copying
-		 * it to userspace...
+		 * In MMIO triggered reports, some platforms do not set the
+		 * reason bit in this field and it is valid to have a reason
+		 * field of zero.
 		 */
-		reason = ((report32[0] >> OAREPORT_REASON_SHIFT) &
-			  (GRAPHICS_VER(stream->perf->i915) == 12 ?
-			   OAREPORT_REASON_MASK_EXTENDED :
-			   OAREPORT_REASON_MASK));
-
-		ctx_id = report32[2] & stream->specific_ctx_id_mask;
+		reason = oa_report_reason(stream, report);
+		ctx_id = oa_context_id(stream, report32);
 
 		/*
 		 * Squash whatever is in the CTX_ID field if it's marked as
@@ -744,9 +819,10 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream,
 		 * Note: that we don't clear the valid_ctx_bit so userspace can
 		 * understand that the ID has been squashed by the kernel.
 		 */
-		if (!(report32[0] & stream->perf->gen8_valid_ctx_bit) &&
-		    GRAPHICS_VER(stream->perf->i915) <= 11)
-			ctx_id = report32[2] = INVALID_CTX_ID;
+		if (oa_report_ctx_invalid(stream, report)) {
+			ctx_id = INVALID_CTX_ID;
+			oa_context_id_squash(stream, report32);
+		}
 
 		/*
 		 * NB: For Gen 8 the OA unit no longer supports clock gating
@@ -790,7 +866,7 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream,
 			 */
 			if (stream->ctx &&
 			    stream->specific_ctx_id != ctx_id) {
-				report32[2] = INVALID_CTX_ID;
+				oa_context_id_squash(stream, report32);
 			}
 
 			ret = append_oa_sample(stream, buf, count, offset,
@@ -802,18 +878,19 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream,
 		}
 
 		/*
-		 * Clear out the first 2 dword as a mean to detect unlanded
+		 * Clear out the report id and timestamp as a means to detect unlanded
 		 * reports.
 		 */
-		report32[0] = 0;
-		report32[1] = 0;
+		oa_report_id_clear(stream, report32);
+		oa_timestamp_clear(stream, report32);
 	}
 
 	if (start_offset != *offset) {
 		i915_reg_t oaheadptr;
 
 		oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ?
-			    GEN12_OAG_OAHEADPTR : GEN8_OAHEADPTR;
+			    __oa_regs(stream)->oa_head_ptr :
+			    GEN8_OAHEADPTR;
 
 		spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
 
@@ -866,7 +943,8 @@ static int gen8_oa_read(struct i915_perf_stream *stream,
 		return -EIO;
 
 	oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ?
-		       GEN12_OAG_OASTATUS : GEN8_OASTATUS;
+		       __oa_regs(stream)->oa_status :
+		       GEN8_OASTATUS;
 
 	oastatus = intel_uncore_read(uncore, oastatus_reg);
 
@@ -1570,12 +1648,23 @@ free_noa_wait(struct i915_perf_stream *stream)
 	i915_vma_unpin_and_release(&stream->noa_wait, 0);
 }
 
+static bool engine_supports_oa(const struct intel_engine_cs *engine)
+{
+	return engine->oa_group;
+}
+
+static bool engine_supports_oa_format(struct intel_engine_cs *engine, int type)
+{
+	return engine->oa_group && engine->oa_group->type == type;
+}
+
 static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
 {
 	struct i915_perf *perf = stream->perf;
 	struct intel_gt *gt = stream->engine->gt;
+	struct i915_perf_group *g = stream->engine->oa_group;
 
-	if (WARN_ON(stream != gt->perf.exclusive_stream))
+	if (WARN_ON(stream != g->exclusive_stream))
 		return;
 
 	/*
@@ -1584,7 +1673,7 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
 	 *
 	 * See i915_oa_init_reg_state() and lrc_configure_all_contexts()
 	 */
-	WRITE_ONCE(gt->perf.exclusive_stream, NULL);
+	WRITE_ONCE(g->exclusive_stream, NULL);
 	perf->ops.disable_metric_set(stream);
 
 	free_oa_buffer(stream);
@@ -1720,8 +1809,8 @@ static void gen12_init_oa_buffer(struct i915_perf_stream *stream)
 
 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
 
-	intel_uncore_write(uncore, GEN12_OAG_OASTATUS, 0);
-	intel_uncore_write(uncore, GEN12_OAG_OAHEADPTR,
+	intel_uncore_write(uncore, __oa_regs(stream)->oa_status, 0);
+	intel_uncore_write(uncore, __oa_regs(stream)->oa_head_ptr,
 			   gtt_offset & GEN12_OAG_OAHEADPTR_MASK);
 	stream->oa_buffer.head = gtt_offset;
 
@@ -1733,9 +1822,9 @@ static void gen12_init_oa_buffer(struct i915_perf_stream *stream)
 	 *  to enable proper functionality of the overflow
 	 *  bit."
 	 */
-	intel_uncore_write(uncore, GEN12_OAG_OABUFFER, gtt_offset |
+	intel_uncore_write(uncore, __oa_regs(stream)->oa_buffer, gtt_offset |
 			   OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT);
-	intel_uncore_write(uncore, GEN12_OAG_OATAILPTR,
+	intel_uncore_write(uncore, __oa_regs(stream)->oa_tail_ptr,
 			   gtt_offset & GEN12_OAG_OATAILPTR_MASK);
 
 	/* Mark that we need updated tail pointers to read from... */
@@ -2495,7 +2584,8 @@ err_add_request:
 	return err;
 }
 
-static int gen8_configure_context(struct i915_gem_context *ctx,
+static int gen8_configure_context(struct i915_perf_stream *stream,
+				  struct i915_gem_context *ctx,
 				  struct flex *flex, unsigned int count)
 {
 	struct i915_gem_engines_iter it;
@@ -2636,7 +2726,7 @@ oa_configure_all_contexts(struct i915_perf_stream *stream,
 
 		spin_unlock(&i915->gem.contexts.lock);
 
-		err = gen8_configure_context(ctx, regs, num_regs);
+		err = gen8_configure_context(stream, ctx, regs, num_regs);
 		if (err) {
 			i915_gem_context_put(ctx);
 			return err;
@@ -2681,6 +2771,9 @@ gen12_configure_all_contexts(struct i915_perf_stream *stream,
 		},
 	};
 
+	if (stream->engine->class != RENDER_CLASS)
+		return 0;
+
 	return oa_configure_all_contexts(stream,
 					 regs, ARRAY_SIZE(regs),
 					 active);
@@ -2810,7 +2903,7 @@ gen12_enable_metric_set(struct i915_perf_stream *stream,
 				   _MASKED_BIT_ENABLE(GEN12_DISABLE_DOP_GATING));
 	}
 
-	intel_uncore_write(uncore, GEN12_OAG_OA_DEBUG,
+	intel_uncore_write(uncore, __oa_regs(stream)->oa_debug,
 			   /* Disable clk ratio reports, like previous Gens. */
 			   _MASKED_BIT_ENABLE(GEN12_OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
 					      GEN12_OAG_OA_DEBUG_INCLUDE_CLK_RATIO) |
@@ -2820,7 +2913,7 @@ gen12_enable_metric_set(struct i915_perf_stream *stream,
 			    */
 			   oag_report_ctx_switches(stream));
 
-	intel_uncore_write(uncore, GEN12_OAG_OAGLBCTXCTRL, periodic ?
+	intel_uncore_write(uncore, __oa_regs(stream)->oa_ctx_ctrl, periodic ?
 			   (GEN12_OAG_OAGLBCTXCTRL_COUNTER_RESUME |
 			    GEN12_OAG_OAGLBCTXCTRL_TIMER_ENABLE |
 			    (period_exponent << GEN12_OAG_OAGLBCTXCTRL_TIMER_PERIOD_SHIFT))
@@ -2974,8 +3067,8 @@ static void gen8_oa_enable(struct i915_perf_stream *stream)
 
 static void gen12_oa_enable(struct i915_perf_stream *stream)
 {
-	struct intel_uncore *uncore = stream->uncore;
-	u32 report_format = stream->oa_buffer.format->format;
+	const struct i915_perf_regs *regs;
+	u32 val;
 
 	/*
 	 * If we don't want OA reports from the OA buffer, then we don't even
@@ -2986,9 +3079,11 @@ static void gen12_oa_enable(struct i915_perf_stream *stream)
 
 	gen12_init_oa_buffer(stream);
 
-	intel_uncore_write(uncore, GEN12_OAG_OACONTROL,
-			   (report_format << GEN12_OAG_OACONTROL_OA_COUNTER_FORMAT_SHIFT) |
-			   GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE);
+	regs = __oa_regs(stream);
+	val = (stream->oa_buffer.format->format << regs->oa_ctrl_counter_format_shift) |
+	      GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE;
+
+	intel_uncore_write(stream->uncore, regs->oa_ctrl, val);
 }
 
 /**
@@ -3040,9 +3135,9 @@ static void gen12_oa_disable(struct i915_perf_stream *stream)
 {
 	struct intel_uncore *uncore = stream->uncore;
 
-	intel_uncore_write(uncore, GEN12_OAG_OACONTROL, 0);
+	intel_uncore_write(uncore, __oa_regs(stream)->oa_ctrl, 0);
 	if (intel_wait_for_register(uncore,
-				    GEN12_OAG_OACONTROL,
+				    __oa_regs(stream)->oa_ctrl,
 				    GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE, 0,
 				    50))
 		drm_err(&stream->perf->i915->drm,
@@ -3182,6 +3277,7 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 {
 	struct drm_i915_private *i915 = stream->perf->i915;
 	struct i915_perf *perf = stream->perf;
+	struct i915_perf_group *g;
 	struct intel_gt *gt;
 	int ret;
 
@@ -3191,6 +3287,7 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 		return -EINVAL;
 	}
 	gt = props->engine->gt;
+	g = props->engine->oa_group;
 
 	/*
 	 * If the sysfs metrics/ directory wasn't registered for some
@@ -3221,7 +3318,7 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 	 * counter reports and marshal to the appropriate client
 	 * we currently only allow exclusive access
 	 */
-	if (gt->perf.exclusive_stream) {
+	if (g->exclusive_stream) {
 		drm_dbg(&stream->perf->i915->drm,
 			"OA unit already in use\n");
 		return -EBUSY;
@@ -3316,7 +3413,7 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 	stream->ops = &i915_oa_stream_ops;
 
 	stream->engine->gt->perf.sseu = props->sseu;
-	WRITE_ONCE(gt->perf.exclusive_stream, stream);
+	WRITE_ONCE(g->exclusive_stream, stream);
 
 	ret = i915_perf_stream_enable_sync(stream);
 	if (ret) {
@@ -3339,7 +3436,7 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 	return 0;
 
 err_enable:
-	WRITE_ONCE(gt->perf.exclusive_stream, NULL);
+	WRITE_ONCE(g->exclusive_stream, NULL);
 	perf->ops.disable_metric_set(stream);
 
 	free_oa_buffer(stream);
@@ -3373,7 +3470,7 @@ void i915_oa_init_reg_state(const struct intel_context *ce,
 		return;
 
 	/* perf.exclusive_stream serialised by lrc_configure_all_contexts() */
-	stream = READ_ONCE(engine->gt->perf.exclusive_stream);
+	stream = READ_ONCE(engine->oa_group->exclusive_stream);
 	if (stream && GRAPHICS_VER(stream->perf->i915) < 12)
 		gen8_update_reg_state_unlocked(ce, stream);
 }
@@ -3940,41 +4037,35 @@ static int read_properties_unlocked(struct i915_perf *perf,
 				    u32 n_props,
 				    struct perf_open_properties *props)
 {
+	struct drm_i915_gem_context_param_sseu user_sseu;
+	const struct i915_oa_format *f;
 	u64 __user *uprop = uprops;
+	bool config_instance = false;
+	bool config_class = false;
+	bool config_sseu = false;
+	u8 class, instance;
 	u32 i;
 	int ret;
 
 	memset(props, 0, sizeof(struct perf_open_properties));
 	props->poll_oa_period = DEFAULT_POLL_PERIOD_NS;
 
-	if (!n_props) {
-		drm_dbg(&perf->i915->drm,
-			"No i915 perf properties given\n");
-		return -EINVAL;
-	}
-
-	/* At the moment we only support using i915-perf on the RCS. */
-	props->engine = intel_engine_lookup_user(perf->i915,
-						 I915_ENGINE_CLASS_RENDER,
-						 0);
-	if (!props->engine) {
-		drm_dbg(&perf->i915->drm,
-			"No RENDER-capable engines\n");
-		return -EINVAL;
-	}
-
 	/* Considering that ID = 0 is reserved and assuming that we don't
 	 * (currently) expect any configurations to ever specify duplicate
 	 * values for a particular property ID then the last _PROP_MAX value is
 	 * one greater than the maximum number of properties we expect to get
 	 * from userspace.
 	 */
-	if (n_props >= DRM_I915_PERF_PROP_MAX) {
+	if (!n_props || n_props >= DRM_I915_PERF_PROP_MAX) {
 		drm_dbg(&perf->i915->drm,
-			"More i915 perf properties specified than exist\n");
+			"Invalid number of i915 perf properties given\n");
 		return -EINVAL;
 	}
 
+	/* Defaults when class:instance is not passed */
+	class = I915_ENGINE_CLASS_RENDER;
+	instance = 0;
+
 	for (i = 0; i < n_props; i++) {
 		u64 oa_period, oa_freq_hz;
 		u64 id, value;
@@ -4069,8 +4160,6 @@ static int read_properties_unlocked(struct i915_perf *perf,
 			props->hold_preemption = !!value;
 			break;
 		case DRM_I915_PERF_PROP_GLOBAL_SSEU: {
-			struct drm_i915_gem_context_param_sseu user_sseu;
-
 			if (GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 50)) {
 				drm_dbg(&perf->i915->drm,
 					"SSEU config not supported on gfx %x\n",
@@ -4085,14 +4174,7 @@ static int read_properties_unlocked(struct i915_perf *perf,
 					"Unable to copy global sseu parameter\n");
 				return -EFAULT;
 			}
-
-			ret = get_sseu_config(&props->sseu, props->engine, &user_sseu);
-			if (ret) {
-				drm_dbg(&perf->i915->drm,
-					"Invalid SSEU configuration\n");
-				return ret;
-			}
-			props->has_sseu = true;
+			config_sseu = true;
 			break;
 		}
 		case DRM_I915_PERF_PROP_POLL_OA_PERIOD:
@@ -4104,7 +4186,15 @@ static int read_properties_unlocked(struct i915_perf *perf,
 			}
 			props->poll_oa_period = value;
 			break;
-		case DRM_I915_PERF_PROP_MAX:
+		case DRM_I915_PERF_PROP_OA_ENGINE_CLASS:
+			class = (u8)value;
+			config_class = true;
+			break;
+		case DRM_I915_PERF_PROP_OA_ENGINE_INSTANCE:
+			instance = (u8)value;
+			config_instance = true;
+			break;
+		default:
 			MISSING_CASE(id);
 			return -EINVAL;
 		}
@@ -4112,6 +4202,60 @@ static int read_properties_unlocked(struct i915_perf *perf,
 		uprop += 2;
 	}
 
+	if ((config_class && !config_instance) ||
+	    (config_instance && !config_class)) {
+		drm_dbg(&perf->i915->drm,
+			"OA engine-class and engine-instance parameters must be passed together\n");
+		return -EINVAL;
+	}
+
+	props->engine = intel_engine_lookup_user(perf->i915, class, instance);
+	if (!props->engine) {
+		drm_dbg(&perf->i915->drm,
+			"OA engine class and instance invalid %d:%d\n",
+			class, instance);
+		return -EINVAL;
+	}
+
+	if (!engine_supports_oa(props->engine)) {
+		drm_dbg(&perf->i915->drm,
+			"Engine not supported by OA %d:%d\n",
+			class, instance);
+		return -EINVAL;
+	}
+
+	/*
+	 * Wa_14017512683: mtl[a0..c0): Use of OAM must be preceded with Media
+	 * C6 disable in BIOS. Fail if Media C6 is enabled on steppings where OAM
+	 * does not work as expected.
+	 */
+	if (IS_MTL_MEDIA_STEP(props->engine->i915, STEP_A0, STEP_C0) &&
+	    props->engine->oa_group->type == TYPE_OAM &&
+	    intel_check_bios_c6_setup(&props->engine->gt->rc6)) {
+		drm_dbg(&perf->i915->drm,
+			"OAM requires media C6 to be disabled in BIOS\n");
+		return -EINVAL;
+	}
+
+	i = array_index_nospec(props->oa_format, I915_OA_FORMAT_MAX);
+	f = &perf->oa_formats[i];
+	if (!engine_supports_oa_format(props->engine, f->type)) {
+		drm_dbg(&perf->i915->drm,
+			"Invalid OA format %d for class %d\n",
+			f->type, props->engine->class);
+		return -EINVAL;
+	}
+
+	if (config_sseu) {
+		ret = get_sseu_config(&props->sseu, props->engine, &user_sseu);
+		if (ret) {
+			drm_dbg(&perf->i915->drm,
+				"Invalid SSEU configuration\n");
+			return ret;
+		}
+		props->has_sseu = true;
+	}
+
 	return 0;
 }
 
@@ -4282,6 +4426,14 @@ static const struct i915_range gen12_oa_b_counters[] = {
 	{}
 };
 
+static const struct i915_range mtl_oam_b_counters[] = {
+	{ .start = 0x393000, .end = 0x39301c },	/* GEN12_OAM_STARTTRIG1[1-8] */
+	{ .start = 0x393020, .end = 0x39303c },	/* GEN12_OAM_REPORTTRIG1[1-8] */
+	{ .start = 0x393040, .end = 0x39307c },	/* GEN12_OAM_CEC[0-7][0-1] */
+	{ .start = 0x393200, .end = 0x39323C },	/* MPES[0-7] */
+	{}
+};
+
 static const struct i915_range xehp_oa_b_counters[] = {
 	{ .start = 0xdc48, .end = 0xdc48 },	/* OAA_ENABLE_REG */
 	{ .start = 0xdd00, .end = 0xdd48 },	/* OAG_LCE0_0 - OAA_LENABLE_REG */
@@ -4335,6 +4487,8 @@ static const struct i915_range mtl_oa_mux_regs[] = {
 	{ .start = 0x0d0c, .end = 0x0d2c },	/* NOA_CONFIG[0-8] */
 	{ .start = 0x9840, .end = 0x9840 },	/* GDT_CHICKEN_BITS */
 	{ .start = 0x9884, .end = 0x9888 },	/* NOA_WRITE */
+	{ .start = 0x38d100, .end = 0x38d114},	/* VISACTL */
+	{}
 };
 
 static bool gen7_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
@@ -4372,10 +4526,20 @@ static bool gen12_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
 	return reg_in_range_table(addr, gen12_oa_b_counters);
 }
 
+static bool mtl_is_valid_oam_b_counter_addr(struct i915_perf *perf, u32 addr)
+{
+	if (HAS_OAM(perf->i915) &&
+	    GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 70))
+		return reg_in_range_table(addr, mtl_oam_b_counters);
+
+	return false;
+}
+
 static bool xehp_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
 {
 	return reg_in_range_table(addr, xehp_oa_b_counters) ||
-		reg_in_range_table(addr, gen12_oa_b_counters);
+		reg_in_range_table(addr, gen12_oa_b_counters) ||
+		mtl_is_valid_oam_b_counter_addr(perf, addr);
 }
 
 static bool gen12_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
@@ -4740,6 +4904,136 @@ static struct ctl_table oa_table[] = {
 	{}
 };
 
+static u32 num_perf_groups_per_gt(struct intel_gt *gt)
+{
+	return 1;
+}
+
+static u32 __oam_engine_group(struct intel_engine_cs *engine)
+{
+	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 70)) {
+		/*
+		 * There's 1 SAMEDIA gt and 1 OAM per SAMEDIA gt. All media slices
+		 * within the gt use the same OAM. All MTL SKUs list 1 SA MEDIA.
+		 */
+		drm_WARN_ON(&engine->i915->drm,
+			    engine->gt->type != GT_MEDIA);
+
+		return PERF_GROUP_OAM_SAMEDIA_0;
+	}
+
+	return PERF_GROUP_INVALID;
+}
+
+static u32 __oa_engine_group(struct intel_engine_cs *engine)
+{
+	switch (engine->class) {
+	case RENDER_CLASS:
+		return PERF_GROUP_OAG;
+
+	case VIDEO_DECODE_CLASS:
+	case VIDEO_ENHANCEMENT_CLASS:
+		return __oam_engine_group(engine);
+
+	default:
+		return PERF_GROUP_INVALID;
+	}
+}
+
+static struct i915_perf_regs __oam_regs(u32 base)
+{
+	return (struct i915_perf_regs) {
+		base,
+		GEN12_OAM_HEAD_POINTER(base),
+		GEN12_OAM_TAIL_POINTER(base),
+		GEN12_OAM_BUFFER(base),
+		GEN12_OAM_CONTEXT_CONTROL(base),
+		GEN12_OAM_CONTROL(base),
+		GEN12_OAM_DEBUG(base),
+		GEN12_OAM_STATUS(base),
+		GEN12_OAM_CONTROL_COUNTER_FORMAT_SHIFT,
+	};
+}
+
+static struct i915_perf_regs __oag_regs(void)
+{
+	return (struct i915_perf_regs) {
+		0,
+		GEN12_OAG_OAHEADPTR,
+		GEN12_OAG_OATAILPTR,
+		GEN12_OAG_OABUFFER,
+		GEN12_OAG_OAGLBCTXCTRL,
+		GEN12_OAG_OACONTROL,
+		GEN12_OAG_OA_DEBUG,
+		GEN12_OAG_OASTATUS,
+		GEN12_OAG_OACONTROL_OA_COUNTER_FORMAT_SHIFT,
+	};
+}
+
+static void oa_init_groups(struct intel_gt *gt)
+{
+	int i, num_groups = gt->perf.num_perf_groups;
+
+	for (i = 0; i < num_groups; i++) {
+		struct i915_perf_group *g = &gt->perf.group[i];
+
+		/* Fused off engines can result in a group with num_engines == 0 */
+		if (g->num_engines == 0)
+			continue;
+
+		if (i == PERF_GROUP_OAG && gt->type != GT_MEDIA) {
+			g->regs = __oag_regs();
+			g->type = TYPE_OAG;
+		} else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
+			g->regs = __oam_regs(mtl_oa_base[i]);
+			g->type = TYPE_OAM;
+		}
+	}
+}
+
+static int oa_init_gt(struct intel_gt *gt)
+{
+	u32 num_groups = num_perf_groups_per_gt(gt);
+	struct intel_engine_cs *engine;
+	struct i915_perf_group *g;
+	intel_engine_mask_t tmp;
+
+	g = kcalloc(num_groups, sizeof(*g), GFP_KERNEL);
+	if (!g)
+		return -ENOMEM;
+
+	for_each_engine_masked(engine, gt, ALL_ENGINES, tmp) {
+		u32 index = __oa_engine_group(engine);
+
+		engine->oa_group = NULL;
+		if (index < num_groups) {
+			g[index].num_engines++;
+			engine->oa_group = &g[index];
+		}
+	}
+
+	gt->perf.num_perf_groups = num_groups;
+	gt->perf.group = g;
+
+	oa_init_groups(gt);
+
+	return 0;
+}
+
+static int oa_init_engine_groups(struct i915_perf *perf)
+{
+	struct intel_gt *gt;
+	int i, ret;
+
+	for_each_gt(gt, perf->i915, i) {
+		ret = oa_init_gt(gt);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static void oa_init_supported_formats(struct i915_perf *perf)
 {
 	struct drm_i915_private *i915 = perf->i915;
@@ -4780,9 +5074,15 @@ static void oa_init_supported_formats(struct i915_perf *perf)
 		break;
 
 	case INTEL_DG2:
+		oa_format_add(perf, I915_OAR_FORMAT_A32u40_A4u32_B8_C8);
+		oa_format_add(perf, I915_OA_FORMAT_A24u40_A14u32_B8_C8);
+		break;
+
 	case INTEL_METEORLAKE:
 		oa_format_add(perf, I915_OAR_FORMAT_A32u40_A4u32_B8_C8);
 		oa_format_add(perf, I915_OA_FORMAT_A24u40_A14u32_B8_C8);
+		oa_format_add(perf, I915_OAM_FORMAT_MPEC8u64_B8_C8);
+		oa_format_add(perf, I915_OAM_FORMAT_MPEC8u32_B8_C8);
 		break;
 
 	default:
@@ -4830,7 +5130,7 @@ static void i915_perf_init_info(struct drm_i915_private *i915)
  * Note: i915-perf initialization is split into an 'init' and 'register'
  * phase with the i915_perf_register() exposing state to userspace.
  */
-void i915_perf_init(struct drm_i915_private *i915)
+int i915_perf_init(struct drm_i915_private *i915)
 {
 	struct i915_perf *perf = &i915->perf;
 
@@ -4906,7 +5206,7 @@ void i915_perf_init(struct drm_i915_private *i915)
 
 	if (perf->ops.enable_metric_set) {
 		struct intel_gt *gt;
-		int i;
+		int i, ret;
 
 		for_each_gt(gt, i915, i)
 			mutex_init(&gt->perf.lock);
@@ -4945,8 +5245,17 @@ void i915_perf_init(struct drm_i915_private *i915)
 
 		perf->i915 = i915;
 
+		ret = oa_init_engine_groups(perf);
+		if (ret) {
+			drm_err(&i915->drm,
+				"OA initialization failed %d\n", ret);
+			return ret;
+		}
+
 		oa_init_supported_formats(perf);
 	}
+
+	return 0;
 }
 
 static int destroy_config(int id, void *p, void *data)
@@ -4973,10 +5282,15 @@ void i915_perf_sysctl_unregister(void)
 void i915_perf_fini(struct drm_i915_private *i915)
 {
 	struct i915_perf *perf = &i915->perf;
+	struct intel_gt *gt;
+	int i;
 
 	if (!perf->i915)
 		return;
 
+	for_each_gt(gt, perf->i915, i)
+		kfree(gt->perf.group);
+
 	idr_for_each(&perf->metrics_idr, destroy_config, perf);
 	idr_destroy(&perf->metrics_idr);
 
@@ -4989,7 +5303,7 @@ void i915_perf_fini(struct drm_i915_private *i915)
  *
  * This version number is used by userspace to detect available features.
  */
-int i915_perf_ioctl_version(void)
+int i915_perf_ioctl_version(struct drm_i915_private *i915)
 {
 	/*
 	 * 1: Initial version
@@ -5010,8 +5324,30 @@ int i915_perf_ioctl_version(void)
 	 *
 	 * 5: Add DRM_I915_PERF_PROP_POLL_OA_PERIOD parameter that controls the
 	 *    interval for the hrtimer used to check for OA data.
+	 *
+	 * 6: Add DRM_I915_PERF_PROP_OA_ENGINE_CLASS and
+	 *    DRM_I915_PERF_PROP_OA_ENGINE_INSTANCE
+	 *
+	 * 7: Add support for video decode and enhancement classes.
+	 */
+
+	/*
+	 * Wa_14017512683: mtl[a0..c0): Use of OAM must be preceded with Media
+	 * C6 disable in BIOS. If Media C6 is enabled in BIOS, return version 6
+	 * to indicate that OA media is not supported.
 	 */
-	return 5;
+	if (IS_MTL_MEDIA_STEP(i915, STEP_A0, STEP_C0)) {
+		struct intel_gt *gt;
+		int i;
+
+		for_each_gt(gt, i915, i) {
+			if (gt->type == GT_MEDIA &&
+			    intel_check_bios_c6_setup(&gt->rc6))
+				return 6;
+		}
+	}
+
+	return 7;
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/i915_perf.h b/drivers/gpu/drm/i915/i915_perf.h
index f96e09a4af04..accf626f2b13 100644
--- a/drivers/gpu/drm/i915/i915_perf.h
+++ b/drivers/gpu/drm/i915/i915_perf.h
@@ -18,11 +18,11 @@ struct i915_oa_config;
 struct intel_context;
 struct intel_engine_cs;
 
-void i915_perf_init(struct drm_i915_private *i915);
+int i915_perf_init(struct drm_i915_private *i915);
 void i915_perf_fini(struct drm_i915_private *i915);
 void i915_perf_register(struct drm_i915_private *i915);
 void i915_perf_unregister(struct drm_i915_private *i915);
-int i915_perf_ioctl_version(void);
+int i915_perf_ioctl_version(struct drm_i915_private *i915);
 int i915_perf_sysctl_register(void);
 void i915_perf_sysctl_unregister(void);
 
diff --git a/drivers/gpu/drm/i915/i915_perf_oa_regs.h b/drivers/gpu/drm/i915/i915_perf_oa_regs.h
index 381d94101610..ba103875e19f 100644
--- a/drivers/gpu/drm/i915/i915_perf_oa_regs.h
+++ b/drivers/gpu/drm/i915/i915_perf_oa_regs.h
@@ -138,4 +138,82 @@
 #define   GEN12_SQCNT1_PMON_ENABLE		REG_BIT(30)
 #define   GEN12_SQCNT1_OABPC			REG_BIT(29)
 
+/* Gen12 OAM unit */
+#define GEN12_OAM_HEAD_POINTER_OFFSET   (0x1a0)
+#define  GEN12_OAM_HEAD_POINTER_MASK    0xffffffc0
+
+#define GEN12_OAM_TAIL_POINTER_OFFSET   (0x1a4)
+#define  GEN12_OAM_TAIL_POINTER_MASK    0xffffffc0
+
+#define GEN12_OAM_BUFFER_OFFSET         (0x1a8)
+#define  GEN12_OAM_BUFFER_SIZE_MASK     (0x7)
+#define  GEN12_OAM_BUFFER_SIZE_SHIFT    (3)
+#define  GEN12_OAM_BUFFER_MEMORY_SELECT REG_BIT(0) /* 0: PPGTT, 1: GGTT */
+
+#define GEN12_OAM_CONTEXT_CONTROL_OFFSET              (0x1bc)
+#define  GEN12_OAM_CONTEXT_CONTROL_TIMER_PERIOD_SHIFT 2
+#define  GEN12_OAM_CONTEXT_CONTROL_TIMER_ENABLE       REG_BIT(1)
+#define  GEN12_OAM_CONTEXT_CONTROL_COUNTER_RESUME     REG_BIT(0)
+
+#define GEN12_OAM_CONTROL_OFFSET                (0x194)
+#define  GEN12_OAM_CONTROL_COUNTER_FORMAT_SHIFT 1
+#define  GEN12_OAM_CONTROL_COUNTER_ENABLE       REG_BIT(0)
+
+#define GEN12_OAM_DEBUG_OFFSET                      (0x198)
+#define  GEN12_OAM_DEBUG_BUFFER_SIZE_SELECT         REG_BIT(12)
+#define  GEN12_OAM_DEBUG_INCLUDE_CLK_RATIO          REG_BIT(6)
+#define  GEN12_OAM_DEBUG_DISABLE_CLK_RATIO_REPORTS  REG_BIT(5)
+#define  GEN12_OAM_DEBUG_DISABLE_GO_1_0_REPORTS     REG_BIT(2)
+#define  GEN12_OAM_DEBUG_DISABLE_CTX_SWITCH_REPORTS REG_BIT(1)
+
+#define GEN12_OAM_STATUS_OFFSET            (0x19c)
+#define  GEN12_OAM_STATUS_COUNTER_OVERFLOW REG_BIT(2)
+#define  GEN12_OAM_STATUS_BUFFER_OVERFLOW  REG_BIT(1)
+#define  GEN12_OAM_STATUS_REPORT_LOST      REG_BIT(0)
+
+#define GEN12_OAM_MMIO_TRG_OFFSET	(0x1d0)
+
+#define GEN12_OAM_MMIO_TRG(base) \
+	_MMIO((base) + GEN12_OAM_MMIO_TRG_OFFSET)
+
+#define GEN12_OAM_HEAD_POINTER(base) \
+	_MMIO((base) + GEN12_OAM_HEAD_POINTER_OFFSET)
+#define GEN12_OAM_TAIL_POINTER(base) \
+	_MMIO((base) + GEN12_OAM_TAIL_POINTER_OFFSET)
+#define GEN12_OAM_BUFFER(base) \
+	_MMIO((base) + GEN12_OAM_BUFFER_OFFSET)
+#define GEN12_OAM_CONTEXT_CONTROL(base) \
+	_MMIO((base) + GEN12_OAM_CONTEXT_CONTROL_OFFSET)
+#define GEN12_OAM_CONTROL(base) \
+	_MMIO((base) + GEN12_OAM_CONTROL_OFFSET)
+#define GEN12_OAM_DEBUG(base) \
+	_MMIO((base) + GEN12_OAM_DEBUG_OFFSET)
+#define GEN12_OAM_STATUS(base) \
+	_MMIO((base) + GEN12_OAM_STATUS_OFFSET)
+
+#define GEN12_OAM_CEC0_0_OFFSET		(0x40)
+#define GEN12_OAM_CEC7_1_OFFSET		(0x7c)
+#define GEN12_OAM_CEC0_0(base) \
+	_MMIO((base) + GEN12_OAM_CEC0_0_OFFSET)
+#define GEN12_OAM_CEC7_1(base) \
+	_MMIO((base) + GEN12_OAM_CEC7_1_OFFSET)
+
+#define GEN12_OAM_STARTTRIG1_OFFSET	(0x00)
+#define GEN12_OAM_STARTTRIG8_OFFSET	(0x1c)
+#define GEN12_OAM_STARTTRIG1(base) \
+	_MMIO((base) + GEN12_OAM_STARTTRIG1_OFFSET)
+#define GEN12_OAM_STARTTRIG8(base) \
+	_MMIO((base) + GEN12_OAM_STARTTRIG8_OFFSET)
+
+#define GEN12_OAM_REPORTTRIG1_OFFSET	(0x20)
+#define GEN12_OAM_REPORTTRIG8_OFFSET	(0x3c)
+#define GEN12_OAM_REPORTTRIG1(base) \
+	_MMIO((base) + GEN12_OAM_REPORTTRIG1_OFFSET)
+#define GEN12_OAM_REPORTTRIG8(base) \
+	_MMIO((base) + GEN12_OAM_REPORTTRIG8_OFFSET)
+
+#define GEN12_OAM_PERF_COUNTER_B0_OFFSET	(0x84)
+#define GEN12_OAM_PERF_COUNTER_B(base, idx) \
+	_MMIO((base) + GEN12_OAM_PERF_COUNTER_B0_OFFSET + 4 * (idx))
+
 #endif /* __INTEL_PERF_OA_REGS__ */
diff --git a/drivers/gpu/drm/i915/i915_perf_types.h b/drivers/gpu/drm/i915/i915_perf_types.h
index 4d5d8c365d9e..66dd5f74de05 100644
--- a/drivers/gpu/drm/i915/i915_perf_types.h
+++ b/drivers/gpu/drm/i915/i915_perf_types.h
@@ -17,8 +17,10 @@
 #include <linux/wait.h>
 #include <uapi/drm/i915_drm.h>
 
+#include "gt/intel_engine_types.h"
 #include "gt/intel_sseu.h"
 #include "i915_reg_defs.h"
+#include "intel_uncore.h"
 #include "intel_wakeref.h"
 
 struct drm_i915_private;
@@ -30,9 +32,41 @@ struct i915_vma;
 struct intel_context;
 struct intel_engine_cs;
 
+enum {
+	PERF_GROUP_OAG = 0,
+	PERF_GROUP_OAM_SAMEDIA_0 = 0,
+
+	PERF_GROUP_MAX,
+	PERF_GROUP_INVALID = U32_MAX,
+};
+
+enum report_header {
+	HDR_32_BIT = 0,
+	HDR_64_BIT,
+};
+
+struct i915_perf_regs {
+	u32 base;
+	i915_reg_t oa_head_ptr;
+	i915_reg_t oa_tail_ptr;
+	i915_reg_t oa_buffer;
+	i915_reg_t oa_ctx_ctrl;
+	i915_reg_t oa_ctrl;
+	i915_reg_t oa_debug;
+	i915_reg_t oa_status;
+	u32 oa_ctrl_counter_format_shift;
+};
+
+enum oa_type {
+	TYPE_OAG,
+	TYPE_OAM,
+};
+
 struct i915_oa_format {
 	u32 format;
 	int size;
+	int type;
+	enum report_header header;
 };
 
 struct i915_oa_reg {
@@ -390,6 +424,30 @@ struct i915_oa_ops {
 	u32 (*oa_hw_tail_read)(struct i915_perf_stream *stream);
 };
 
+struct i915_perf_group {
+	/*
+	 * @exclusive_stream: The stream currently using the OA unit. This is
+	 * sometimes accessed outside a syscall associated to its file
+	 * descriptor.
+	 */
+	struct i915_perf_stream *exclusive_stream;
+
+	/*
+	 * @num_engines: The number of engines using this OA unit.
+	 */
+	u32 num_engines;
+
+	/*
+	 * @regs: OA buffer register group for programming the OA unit.
+	 */
+	struct i915_perf_regs regs;
+
+	/*
+	 * @type: Type of OA unit - OAM, OAG etc.
+	 */
+	enum oa_type type;
+};
+
 struct i915_perf_gt {
 	/*
 	 * Lock associated with anything below within this structure.
@@ -402,12 +460,15 @@ struct i915_perf_gt {
 	 */
 	struct intel_sseu sseu;
 
+	/**
+	 * @num_perf_groups: number of perf groups per gt.
+	 */
+	u32 num_perf_groups;
+
 	/*
-	 * @exclusive_stream: The stream currently using the OA unit. This is
-	 * sometimes accessed outside a syscall associated to its file
-	 * descriptor.
+	 * @group: list of OA groups - one for each OA buffer.
 	 */
-	struct i915_perf_stream *exclusive_stream;
+	struct i915_perf_group *group;
 };
 
 struct i915_perf {
diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c
index 6d422b056f8a..7ece883a7d95 100644
--- a/drivers/gpu/drm/i915/i915_pmu.c
+++ b/drivers/gpu/drm/i915/i915_pmu.c
@@ -17,7 +17,6 @@
 
 #include "i915_drv.h"
 #include "i915_pmu.h"
-#include "intel_pm.h"
 
 /* Frequency for the sampling timer for events which need it. */
 #define FREQUENCY 200
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 747b53b567a0..c4197e31962e 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -116,6 +116,9 @@
  *  #define GEN8_BAR                    _MMIO(0xb888)
  */
 
+#define GU_CNTL_PROTECTED		_MMIO(0x10100C)
+#define   DEPRESENT			REG_BIT(9)
+
 #define GU_CNTL				_MMIO(0x101010)
 #define   LMEM_INIT			REG_BIT(7)
 #define   DRIVERFLR			REG_BIT(31)
@@ -541,9 +544,10 @@
 #define _BXT_PHY0_BASE			0x6C000
 #define _BXT_PHY1_BASE			0x162000
 #define _BXT_PHY2_BASE			0x163000
-#define BXT_PHY_BASE(phy)		_PHY3((phy), _BXT_PHY0_BASE, \
-						     _BXT_PHY1_BASE, \
-						     _BXT_PHY2_BASE)
+#define BXT_PHY_BASE(phy)							\
+	 _PICK_EVEN_2RANGES(phy, 1,						\
+			    _BXT_PHY0_BASE, _BXT_PHY0_BASE,			\
+			    _BXT_PHY1_BASE, _BXT_PHY2_BASE)
 
 #define _BXT_PHY(phy, reg)						\
 	_MMIO(BXT_PHY_BASE(phy) - _BXT_PHY0_BASE + (reg))
@@ -566,13 +570,14 @@
 #define BXT_PHY_CTL(port)		_MMIO_PORT(port, _BXT_PHY_CTL_DDI_A, \
 							 _BXT_PHY_CTL_DDI_B)
 
-#define _PHY_CTL_FAMILY_EDP		0x64C80
 #define _PHY_CTL_FAMILY_DDI		0x64C90
+#define _PHY_CTL_FAMILY_EDP		0x64C80
 #define _PHY_CTL_FAMILY_DDI_C		0x64CA0
 #define   COMMON_RESET_DIS		(1 << 31)
-#define BXT_PHY_CTL_FAMILY(phy)		_MMIO_PHY3((phy), _PHY_CTL_FAMILY_DDI, \
-							  _PHY_CTL_FAMILY_EDP, \
-							  _PHY_CTL_FAMILY_DDI_C)
+#define BXT_PHY_CTL_FAMILY(phy)							\
+	 _MMIO(_PICK_EVEN_2RANGES(phy, 1,					\
+				  _PHY_CTL_FAMILY_DDI, _PHY_CTL_FAMILY_DDI,	\
+				  _PHY_CTL_FAMILY_EDP, _PHY_CTL_FAMILY_DDI_C))
 
 /* BXT PHY PLL registers */
 #define _PORT_PLL_A			0x46074
@@ -923,8 +928,18 @@
 #define DG1_GSC_HECI2_BASE	0x00259000
 #define DG2_GSC_HECI1_BASE	0x00373000
 #define DG2_GSC_HECI2_BASE	0x00374000
+#define MTL_GSC_HECI1_BASE	0x00116000
+#define MTL_GSC_HECI2_BASE	0x00117000
 
+#define HECI_H_CSR(base)	_MMIO((base) + 0x4)
+#define   HECI_H_CSR_IE		REG_BIT(0)
+#define   HECI_H_CSR_IS		REG_BIT(1)
+#define   HECI_H_CSR_IG		REG_BIT(2)
+#define   HECI_H_CSR_RDY	REG_BIT(3)
+#define   HECI_H_CSR_RST	REG_BIT(4)
 
+#define HECI_H_GS1(base)	_MMIO((base) + 0xc4c)
+#define   HECI_H_GS1_ER_PREP	REG_BIT(0)
 
 #define HSW_GTT_CACHE_EN	_MMIO(0x4024)
 #define   GTT_CACHE_EN_ALL	0xF0007FFF
@@ -1038,9 +1053,11 @@
 #define _MBUS_ABOX0_CTL			0x45038
 #define _MBUS_ABOX1_CTL			0x45048
 #define _MBUS_ABOX2_CTL			0x4504C
-#define MBUS_ABOX_CTL(x)		_MMIO(_PICK(x, _MBUS_ABOX0_CTL, \
-						    _MBUS_ABOX1_CTL, \
-						    _MBUS_ABOX2_CTL))
+#define MBUS_ABOX_CTL(x)							\
+	_MMIO(_PICK_EVEN_2RANGES(x, 2,						\
+				 _MBUS_ABOX0_CTL, _MBUS_ABOX1_CTL,		\
+				 _MBUS_ABOX2_CTL, _MBUS_ABOX2_CTL))
+
 #define MBUS_ABOX_BW_CREDIT_MASK	(3 << 20)
 #define MBUS_ABOX_BW_CREDIT(x)		((x) << 20)
 #define MBUS_ABOX_B_CREDIT_MASK		(0xF << 16)
@@ -1050,39 +1067,6 @@
 #define MBUS_ABOX_BT_CREDIT_POOL1_MASK	(0x1F << 0)
 #define MBUS_ABOX_BT_CREDIT_POOL1(x)	((x) << 0)
 
-#define _PIPEA_MBUS_DBOX_CTL			0x7003C
-#define _PIPEB_MBUS_DBOX_CTL			0x7103C
-#define PIPE_MBUS_DBOX_CTL(pipe)		_MMIO_PIPE(pipe, _PIPEA_MBUS_DBOX_CTL, \
-							   _PIPEB_MBUS_DBOX_CTL)
-#define MBUS_DBOX_B2B_TRANSACTIONS_MAX_MASK	REG_GENMASK(24, 20) /* tgl+ */
-#define MBUS_DBOX_B2B_TRANSACTIONS_MAX(x)	REG_FIELD_PREP(MBUS_DBOX_B2B_TRANSACTIONS_MAX_MASK, x)
-#define MBUS_DBOX_B2B_TRANSACTIONS_DELAY_MASK	REG_GENMASK(19, 17) /* tgl+ */
-#define MBUS_DBOX_B2B_TRANSACTIONS_DELAY(x)	REG_FIELD_PREP(MBUS_DBOX_B2B_TRANSACTIONS_DELAY_MASK, x)
-#define MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN	REG_BIT(16) /* tgl+ */
-#define MBUS_DBOX_BW_CREDIT_MASK		REG_GENMASK(15, 14)
-#define MBUS_DBOX_BW_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, x)
-#define MBUS_DBOX_BW_4CREDITS_MTL		REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, 0x2)
-#define MBUS_DBOX_BW_8CREDITS_MTL		REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, 0x3)
-#define MBUS_DBOX_B_CREDIT_MASK			REG_GENMASK(12, 8)
-#define MBUS_DBOX_B_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_B_CREDIT_MASK, x)
-#define MBUS_DBOX_I_CREDIT_MASK			REG_GENMASK(7, 5)
-#define MBUS_DBOX_I_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_I_CREDIT_MASK, x)
-#define MBUS_DBOX_A_CREDIT_MASK			REG_GENMASK(3, 0)
-#define MBUS_DBOX_A_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_A_CREDIT_MASK, x)
-
-#define MBUS_UBOX_CTL			_MMIO(0x4503C)
-#define MBUS_BBOX_CTL_S1		_MMIO(0x45040)
-#define MBUS_BBOX_CTL_S2		_MMIO(0x45044)
-
-#define MBUS_CTL			_MMIO(0x4438C)
-#define MBUS_JOIN			REG_BIT(31)
-#define MBUS_HASHING_MODE_MASK		REG_BIT(30)
-#define MBUS_HASHING_MODE_2x2		REG_FIELD_PREP(MBUS_HASHING_MODE_MASK, 0)
-#define MBUS_HASHING_MODE_1x4		REG_FIELD_PREP(MBUS_HASHING_MODE_MASK, 1)
-#define MBUS_JOIN_PIPE_SELECT_MASK	REG_GENMASK(28, 26)
-#define MBUS_JOIN_PIPE_SELECT(pipe)	REG_FIELD_PREP(MBUS_JOIN_PIPE_SELECT_MASK, pipe)
-#define MBUS_JOIN_PIPE_SELECT_NONE	MBUS_JOIN_PIPE_SELECT(7)
-
 /* Make render/texture TLB fetches lower priorty than associated data
  *   fetches. This is not turned on by default
  */
@@ -1390,7 +1374,8 @@
 #define IVB_FBC_RT_BASE_UPPER		_MMIO(0x7024)
 
 #define IPS_CTL		_MMIO(0x43408)
-#define   IPS_ENABLE	(1 << 31)
+#define   IPS_ENABLE		REG_BIT(31)
+#define   IPS_FALSE_COLOR	REG_BIT(4)
 
 #define MSG_FBC_REND_STATE(fbc_id)	_MMIO_PIPE((fbc_id), 0x50380, 0x50384)
 #define   FBC_REND_NUKE			REG_BIT(2)
@@ -1730,10 +1715,11 @@
 #define   PALETTE_10BIT_BLUE_EXP_MASK	REG_GENMASK(7, 6)
 #define   PALETTE_10BIT_BLUE_MANT_MASK	REG_GENMASK(5, 2)
 #define   PALETTE_10BIT_BLUE_UDW_MASK	REG_GENMASK(1, 0)
-#define PALETTE(pipe, i)	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + \
-				      _PICK((pipe), _PALETTE_A,		\
-					    _PALETTE_B, _CHV_PALETTE_C) + \
-				      (i) * 4)
+#define PALETTE(pipe, i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) +			\
+			       _PICK_EVEN_2RANGES(pipe, 2,			\
+						  _PALETTE_A, _PALETTE_B,	\
+						  _CHV_PALETTE_C, _CHV_PALETTE_C) + \
+						  (i) * 4)
 
 #define PEG_BAND_GAP_DATA	_MMIO(0x14d68)
 
@@ -1908,48 +1894,72 @@
 #define PIPE_CRC_RES_RES1_I915(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_RES1_A_I915)
 #define PIPE_CRC_RES_RES2_G4X(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_RES2_A_G4X)
 
-/* Pipe A timing regs */
-#define _HTOTAL_A	0x60000
-#define _HBLANK_A	0x60004
-#define _HSYNC_A	0x60008
-#define _VTOTAL_A	0x6000c
-#define _VBLANK_A	0x60010
-#define _VSYNC_A	0x60014
-#define _EXITLINE_A	0x60018
-#define _PIPEASRC	0x6001c
+/* Pipe/transcoder A timing regs */
+#define _TRANS_HTOTAL_A		0x60000
+#define   HTOTAL_MASK			REG_GENMASK(31, 16)
+#define   HTOTAL(htotal)		REG_FIELD_PREP(HTOTAL_MASK, (htotal))
+#define   HACTIVE_MASK			REG_GENMASK(15, 0)
+#define   HACTIVE(hdisplay)		REG_FIELD_PREP(HACTIVE_MASK, (hdisplay))
+#define _TRANS_HBLANK_A		0x60004
+#define   HBLANK_END_MASK		REG_GENMASK(31, 16)
+#define   HBLANK_END(hblank_end)	REG_FIELD_PREP(HBLANK_END_MASK, (hblank_end))
+#define   HBLANK_START_MASK		REG_GENMASK(15, 0)
+#define   HBLANK_START(hblank_start)	REG_FIELD_PREP(HBLANK_START_MASK, (hblank_start))
+#define _TRANS_HSYNC_A		0x60008
+#define   HSYNC_END_MASK		REG_GENMASK(31, 16)
+#define   HSYNC_END(hsync_end)		REG_FIELD_PREP(HSYNC_END_MASK, (hsync_end))
+#define   HSYNC_START_MASK		REG_GENMASK(15, 0)
+#define   HSYNC_START(hsync_start)	REG_FIELD_PREP(HSYNC_START_MASK, (hsync_start))
+#define _TRANS_VTOTAL_A		0x6000c
+#define   VTOTAL_MASK			REG_GENMASK(31, 16)
+#define   VTOTAL(vtotal)		REG_FIELD_PREP(VTOTAL_MASK, (vtotal))
+#define   VACTIVE_MASK			REG_GENMASK(15, 0)
+#define   VACTIVE(vdisplay)		REG_FIELD_PREP(VACTIVE_MASK, (vdisplay))
+#define _TRANS_VBLANK_A		0x60010
+#define   VBLANK_END_MASK		REG_GENMASK(31, 16)
+#define   VBLANK_END(vblank_end)	REG_FIELD_PREP(VBLANK_END_MASK, (vblank_end))
+#define   VBLANK_START_MASK		REG_GENMASK(15, 0)
+#define   VBLANK_START(vblank_start)	REG_FIELD_PREP(VBLANK_START_MASK, (vblank_start))
+#define _TRANS_VSYNC_A		0x60014
+#define   VSYNC_END_MASK		REG_GENMASK(31, 16)
+#define   VSYNC_END(vsync_end)		REG_FIELD_PREP(VSYNC_END_MASK, (vsync_end))
+#define   VSYNC_START_MASK		REG_GENMASK(15, 0)
+#define   VSYNC_START(vsync_start)	REG_FIELD_PREP(VSYNC_START_MASK, (vsync_start))
+#define _TRANS_EXITLINE_A	0x60018
+#define _PIPEASRC		0x6001c
 #define   PIPESRC_WIDTH_MASK	REG_GENMASK(31, 16)
 #define   PIPESRC_WIDTH(w)	REG_FIELD_PREP(PIPESRC_WIDTH_MASK, (w))
 #define   PIPESRC_HEIGHT_MASK	REG_GENMASK(15, 0)
 #define   PIPESRC_HEIGHT(h)	REG_FIELD_PREP(PIPESRC_HEIGHT_MASK, (h))
-#define _BCLRPAT_A	0x60020
-#define _VSYNCSHIFT_A	0x60028
-#define _PIPE_MULT_A	0x6002c
-
-/* Pipe B timing regs */
-#define _HTOTAL_B	0x61000
-#define _HBLANK_B	0x61004
-#define _HSYNC_B	0x61008
-#define _VTOTAL_B	0x6100c
-#define _VBLANK_B	0x61010
-#define _VSYNC_B	0x61014
-#define _PIPEBSRC	0x6101c
-#define _BCLRPAT_B	0x61020
-#define _VSYNCSHIFT_B	0x61028
-#define _PIPE_MULT_B	0x6102c
+#define _BCLRPAT_A		0x60020
+#define _TRANS_VSYNCSHIFT_A	0x60028
+#define _TRANS_MULT_A		0x6002c
+
+/* Pipe/transcoder B timing regs */
+#define _TRANS_HTOTAL_B		0x61000
+#define _TRANS_HBLANK_B		0x61004
+#define _TRANS_HSYNC_B		0x61008
+#define _TRANS_VTOTAL_B		0x6100c
+#define _TRANS_VBLANK_B		0x61010
+#define _TRANS_VSYNC_B		0x61014
+#define _PIPEBSRC		0x6101c
+#define _BCLRPAT_B		0x61020
+#define _TRANS_VSYNCSHIFT_B	0x61028
+#define _TRANS_MULT_B		0x6102c
 
 /* DSI 0 timing regs */
-#define _HTOTAL_DSI0		0x6b000
-#define _HSYNC_DSI0		0x6b008
-#define _VTOTAL_DSI0		0x6b00c
-#define _VSYNC_DSI0		0x6b014
-#define _VSYNCSHIFT_DSI0	0x6b028
+#define _TRANS_HTOTAL_DSI0	0x6b000
+#define _TRANS_HSYNC_DSI0	0x6b008
+#define _TRANS_VTOTAL_DSI0	0x6b00c
+#define _TRANS_VSYNC_DSI0	0x6b014
+#define _TRANS_VSYNCSHIFT_DSI0	0x6b028
 
 /* DSI 1 timing regs */
-#define _HTOTAL_DSI1		0x6b800
-#define _HSYNC_DSI1		0x6b808
-#define _VTOTAL_DSI1		0x6b80c
-#define _VSYNC_DSI1		0x6b814
-#define _VSYNCSHIFT_DSI1	0x6b828
+#define _TRANS_HTOTAL_DSI1	0x6b800
+#define _TRANS_HSYNC_DSI1	0x6b808
+#define _TRANS_VTOTAL_DSI1	0x6b80c
+#define _TRANS_VSYNC_DSI1	0x6b814
+#define _TRANS_VSYNCSHIFT_DSI1	0x6b828
 
 #define TRANSCODER_A_OFFSET 0x60000
 #define TRANSCODER_B_OFFSET 0x61000
@@ -1960,21 +1970,16 @@
 #define TRANSCODER_DSI0_OFFSET	0x6b000
 #define TRANSCODER_DSI1_OFFSET	0x6b800
 
-#define HTOTAL(trans)		_MMIO_TRANS2(trans, _HTOTAL_A)
-#define HBLANK(trans)		_MMIO_TRANS2(trans, _HBLANK_A)
-#define HSYNC(trans)		_MMIO_TRANS2(trans, _HSYNC_A)
-#define VTOTAL(trans)		_MMIO_TRANS2(trans, _VTOTAL_A)
-#define VBLANK(trans)		_MMIO_TRANS2(trans, _VBLANK_A)
-#define VSYNC(trans)		_MMIO_TRANS2(trans, _VSYNC_A)
-#define BCLRPAT(trans)		_MMIO_TRANS2(trans, _BCLRPAT_A)
-#define VSYNCSHIFT(trans)	_MMIO_TRANS2(trans, _VSYNCSHIFT_A)
-#define PIPESRC(trans)		_MMIO_TRANS2(trans, _PIPEASRC)
-#define PIPE_MULT(trans)	_MMIO_TRANS2(trans, _PIPE_MULT_A)
-
-#define EXITLINE(trans)		_MMIO_TRANS2(trans, _EXITLINE_A)
-#define   EXITLINE_ENABLE	REG_BIT(31)
-#define   EXITLINE_MASK		REG_GENMASK(12, 0)
-#define   EXITLINE_SHIFT	0
+#define TRANS_HTOTAL(trans)	_MMIO_TRANS2((trans), _TRANS_HTOTAL_A)
+#define TRANS_HBLANK(trans)	_MMIO_TRANS2((trans), _TRANS_HBLANK_A)
+#define TRANS_HSYNC(trans)	_MMIO_TRANS2((trans), _TRANS_HSYNC_A)
+#define TRANS_VTOTAL(trans)	_MMIO_TRANS2((trans), _TRANS_VTOTAL_A)
+#define TRANS_VBLANK(trans)	_MMIO_TRANS2((trans), _TRANS_VBLANK_A)
+#define TRANS_VSYNC(trans)	_MMIO_TRANS2((trans), _TRANS_VSYNC_A)
+#define BCLRPAT(trans)		_MMIO_TRANS2((trans), _BCLRPAT_A)
+#define TRANS_VSYNCSHIFT(trans)	_MMIO_TRANS2((trans), _TRANS_VSYNCSHIFT_A)
+#define PIPESRC(pipe)		_MMIO_TRANS2((pipe), _PIPEASRC)
+#define TRANS_MULT(trans)	_MMIO_TRANS2((trans), _TRANS_MULT_A)
 
 /* VRR registers */
 #define _TRANS_VRR_CTL_A		0x60420
@@ -2069,309 +2074,6 @@
 #define   TRANS_PUSH_EN			REG_BIT(31)
 #define   TRANS_PUSH_SEND		REG_BIT(30)
 
-/*
- * HSW+ eDP PSR registers
- *
- * HSW PSR registers are relative to DDIA(_DDI_BUF_CTL_A + 0x800) with just one
- * instance of it
- */
-#define _SRD_CTL_A				0x60800
-#define _SRD_CTL_EDP				0x6f800
-#define EDP_PSR_CTL(tran)			_MMIO_TRANS2(tran, _SRD_CTL_A)
-#define   EDP_PSR_ENABLE			(1 << 31)
-#define   BDW_PSR_SINGLE_FRAME			(1 << 30)
-#define   EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK	(1 << 29) /* SW can't modify */
-#define   EDP_PSR_LINK_STANDBY			(1 << 27)
-#define   EDP_PSR_MIN_LINK_ENTRY_TIME_MASK	(3 << 25)
-#define   EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES	(0 << 25)
-#define   EDP_PSR_MIN_LINK_ENTRY_TIME_4_LINES	(1 << 25)
-#define   EDP_PSR_MIN_LINK_ENTRY_TIME_2_LINES	(2 << 25)
-#define   EDP_PSR_MIN_LINK_ENTRY_TIME_0_LINES	(3 << 25)
-#define   EDP_PSR_MAX_SLEEP_TIME_SHIFT		20
-#define   EDP_PSR_SKIP_AUX_EXIT			(1 << 12)
-#define   EDP_PSR_TP1_TP2_SEL			(0 << 11)
-#define   EDP_PSR_TP1_TP3_SEL			(1 << 11)
-#define   EDP_PSR_CRC_ENABLE			(1 << 10) /* BDW+ */
-#define   EDP_PSR_TP2_TP3_TIME_500us		(0 << 8)
-#define   EDP_PSR_TP2_TP3_TIME_100us		(1 << 8)
-#define   EDP_PSR_TP2_TP3_TIME_2500us		(2 << 8)
-#define   EDP_PSR_TP2_TP3_TIME_0us		(3 << 8)
-#define   EDP_PSR_TP4_TIME_0US			(3 << 6) /* ICL+ */
-#define   EDP_PSR_TP1_TIME_500us		(0 << 4)
-#define   EDP_PSR_TP1_TIME_100us		(1 << 4)
-#define   EDP_PSR_TP1_TIME_2500us		(2 << 4)
-#define   EDP_PSR_TP1_TIME_0us			(3 << 4)
-#define   EDP_PSR_IDLE_FRAME_SHIFT		0
-
-/*
- * Until TGL, IMR/IIR are fixed at 0x648xx. On TGL+ those registers are relative
- * to transcoder and bits defined for each one as if using no shift (i.e. as if
- * it was for TRANSCODER_EDP)
- */
-#define EDP_PSR_IMR				_MMIO(0x64834)
-#define EDP_PSR_IIR				_MMIO(0x64838)
-#define _PSR_IMR_A				0x60814
-#define _PSR_IIR_A				0x60818
-#define TRANS_PSR_IMR(tran)			_MMIO_TRANS2(tran, _PSR_IMR_A)
-#define TRANS_PSR_IIR(tran)			_MMIO_TRANS2(tran, _PSR_IIR_A)
-#define   _EDP_PSR_TRANS_SHIFT(trans)		((trans) == TRANSCODER_EDP ? \
-						 0 : ((trans) - TRANSCODER_A + 1) * 8)
-#define   TGL_PSR_MASK			REG_GENMASK(2, 0)
-#define   TGL_PSR_ERROR			REG_BIT(2)
-#define   TGL_PSR_POST_EXIT		REG_BIT(1)
-#define   TGL_PSR_PRE_ENTRY		REG_BIT(0)
-#define   EDP_PSR_MASK(trans)		(TGL_PSR_MASK <<		\
-					 _EDP_PSR_TRANS_SHIFT(trans))
-#define   EDP_PSR_ERROR(trans)		(TGL_PSR_ERROR <<		\
-					 _EDP_PSR_TRANS_SHIFT(trans))
-#define   EDP_PSR_POST_EXIT(trans)	(TGL_PSR_POST_EXIT <<		\
-					 _EDP_PSR_TRANS_SHIFT(trans))
-#define   EDP_PSR_PRE_ENTRY(trans)	(TGL_PSR_PRE_ENTRY <<		\
-					 _EDP_PSR_TRANS_SHIFT(trans))
-
-#define _SRD_AUX_DATA_A				0x60814
-#define _SRD_AUX_DATA_EDP			0x6f814
-#define EDP_PSR_AUX_DATA(tran, i)		_MMIO_TRANS2(tran, _SRD_AUX_DATA_A + (i) + 4) /* 5 registers */
-
-#define _SRD_STATUS_A				0x60840
-#define _SRD_STATUS_EDP				0x6f840
-#define EDP_PSR_STATUS(tran)			_MMIO_TRANS2(tran, _SRD_STATUS_A)
-#define   EDP_PSR_STATUS_STATE_MASK		(7 << 29)
-#define   EDP_PSR_STATUS_STATE_SHIFT		29
-#define   EDP_PSR_STATUS_STATE_IDLE		(0 << 29)
-#define   EDP_PSR_STATUS_STATE_SRDONACK		(1 << 29)
-#define   EDP_PSR_STATUS_STATE_SRDENT		(2 << 29)
-#define   EDP_PSR_STATUS_STATE_BUFOFF		(3 << 29)
-#define   EDP_PSR_STATUS_STATE_BUFON		(4 << 29)
-#define   EDP_PSR_STATUS_STATE_AUXACK		(5 << 29)
-#define   EDP_PSR_STATUS_STATE_SRDOFFACK	(6 << 29)
-#define   EDP_PSR_STATUS_LINK_MASK		(3 << 26)
-#define   EDP_PSR_STATUS_LINK_FULL_OFF		(0 << 26)
-#define   EDP_PSR_STATUS_LINK_FULL_ON		(1 << 26)
-#define   EDP_PSR_STATUS_LINK_STANDBY		(2 << 26)
-#define   EDP_PSR_STATUS_MAX_SLEEP_TIMER_SHIFT	20
-#define   EDP_PSR_STATUS_MAX_SLEEP_TIMER_MASK	0x1f
-#define   EDP_PSR_STATUS_COUNT_SHIFT		16
-#define   EDP_PSR_STATUS_COUNT_MASK		0xf
-#define   EDP_PSR_STATUS_AUX_ERROR		(1 << 15)
-#define   EDP_PSR_STATUS_AUX_SENDING		(1 << 12)
-#define   EDP_PSR_STATUS_SENDING_IDLE		(1 << 9)
-#define   EDP_PSR_STATUS_SENDING_TP2_TP3	(1 << 8)
-#define   EDP_PSR_STATUS_SENDING_TP1		(1 << 4)
-#define   EDP_PSR_STATUS_IDLE_MASK		0xf
-
-#define _SRD_PERF_CNT_A			0x60844
-#define _SRD_PERF_CNT_EDP		0x6f844
-#define EDP_PSR_PERF_CNT(tran)		_MMIO_TRANS2(tran, _SRD_PERF_CNT_A)
-#define   EDP_PSR_PERF_CNT_MASK		0xffffff
-
-/* PSR_MASK on SKL+ */
-#define _SRD_DEBUG_A				0x60860
-#define _SRD_DEBUG_EDP				0x6f860
-#define EDP_PSR_DEBUG(tran)			_MMIO_TRANS2(tran, _SRD_DEBUG_A)
-#define   EDP_PSR_DEBUG_MASK_MAX_SLEEP         (1 << 28)
-#define   EDP_PSR_DEBUG_MASK_LPSP              (1 << 27)
-#define   EDP_PSR_DEBUG_MASK_MEMUP             (1 << 26)
-#define   EDP_PSR_DEBUG_MASK_HPD               (1 << 25)
-#define   EDP_PSR_DEBUG_MASK_DISP_REG_WRITE    (1 << 16) /* Reserved in ICL+ */
-#define   EDP_PSR_DEBUG_EXIT_ON_PIXEL_UNDERRUN (1 << 15) /* SKL+ */
-
-#define _PSR2_CTL_A				0x60900
-#define _PSR2_CTL_EDP				0x6f900
-#define EDP_PSR2_CTL(tran)			_MMIO_TRANS2(tran, _PSR2_CTL_A)
-#define   EDP_PSR2_ENABLE			(1 << 31)
-#define   EDP_SU_TRACK_ENABLE			(1 << 30) /* up to adl-p */
-#define   TGL_EDP_PSR2_BLOCK_COUNT_NUM_2	(0 << 28)
-#define   TGL_EDP_PSR2_BLOCK_COUNT_NUM_3	(1 << 28)
-#define   EDP_Y_COORDINATE_ENABLE		REG_BIT(25) /* display 10, 11 and 12 */
-#define   EDP_PSR2_SU_SDP_SCANLINE		REG_BIT(25) /* display 13+ */
-#define   EDP_MAX_SU_DISABLE_TIME(t)		((t) << 20)
-#define   EDP_MAX_SU_DISABLE_TIME_MASK		(0x1f << 20)
-#define   EDP_PSR2_IO_BUFFER_WAKE_MAX_LINES	8
-#define   EDP_PSR2_IO_BUFFER_WAKE(lines)	((EDP_PSR2_IO_BUFFER_WAKE_MAX_LINES - (lines)) << 13)
-#define   EDP_PSR2_IO_BUFFER_WAKE_MASK		(3 << 13)
-#define   TGL_EDP_PSR2_IO_BUFFER_WAKE_MIN_LINES	5
-#define   TGL_EDP_PSR2_IO_BUFFER_WAKE_SHIFT	13
-#define   TGL_EDP_PSR2_IO_BUFFER_WAKE(lines)	(((lines) - TGL_EDP_PSR2_IO_BUFFER_WAKE_MIN_LINES) << TGL_EDP_PSR2_IO_BUFFER_WAKE_SHIFT)
-#define   TGL_EDP_PSR2_IO_BUFFER_WAKE_MASK	(7 << 13)
-#define   EDP_PSR2_FAST_WAKE_MAX_LINES		8
-#define   EDP_PSR2_FAST_WAKE(lines)		((EDP_PSR2_FAST_WAKE_MAX_LINES - (lines)) << 11)
-#define   EDP_PSR2_FAST_WAKE_MASK		(3 << 11)
-#define   TGL_EDP_PSR2_FAST_WAKE_MIN_LINES	5
-#define   TGL_EDP_PSR2_FAST_WAKE_MIN_SHIFT	10
-#define   TGL_EDP_PSR2_FAST_WAKE(lines)		(((lines) - TGL_EDP_PSR2_FAST_WAKE_MIN_LINES) << TGL_EDP_PSR2_FAST_WAKE_MIN_SHIFT)
-#define   TGL_EDP_PSR2_FAST_WAKE_MASK		(7 << 10)
-#define   EDP_PSR2_TP2_TIME_500us		(0 << 8)
-#define   EDP_PSR2_TP2_TIME_100us		(1 << 8)
-#define   EDP_PSR2_TP2_TIME_2500us		(2 << 8)
-#define   EDP_PSR2_TP2_TIME_50us		(3 << 8)
-#define   EDP_PSR2_TP2_TIME_MASK		(3 << 8)
-#define   EDP_PSR2_FRAME_BEFORE_SU_SHIFT	4
-#define   EDP_PSR2_FRAME_BEFORE_SU_MASK		(0xf << 4)
-#define   EDP_PSR2_FRAME_BEFORE_SU(a)		((a) << 4)
-#define   EDP_PSR2_IDLE_FRAME_MASK		0xf
-#define   EDP_PSR2_IDLE_FRAME_SHIFT		0
-
-#define _PSR_EVENT_TRANS_A			0x60848
-#define _PSR_EVENT_TRANS_B			0x61848
-#define _PSR_EVENT_TRANS_C			0x62848
-#define _PSR_EVENT_TRANS_D			0x63848
-#define _PSR_EVENT_TRANS_EDP			0x6f848
-#define PSR_EVENT(tran)				_MMIO_TRANS2(tran, _PSR_EVENT_TRANS_A)
-#define  PSR_EVENT_PSR2_WD_TIMER_EXPIRE		(1 << 17)
-#define  PSR_EVENT_PSR2_DISABLED		(1 << 16)
-#define  PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN	(1 << 15)
-#define  PSR_EVENT_SU_CRC_FIFO_UNDERRUN		(1 << 14)
-#define  PSR_EVENT_GRAPHICS_RESET		(1 << 12)
-#define  PSR_EVENT_PCH_INTERRUPT		(1 << 11)
-#define  PSR_EVENT_MEMORY_UP			(1 << 10)
-#define  PSR_EVENT_FRONT_BUFFER_MODIFY		(1 << 9)
-#define  PSR_EVENT_WD_TIMER_EXPIRE		(1 << 8)
-#define  PSR_EVENT_PIPE_REGISTERS_UPDATE	(1 << 6)
-#define  PSR_EVENT_REGISTER_UPDATE		(1 << 5) /* Reserved in ICL+ */
-#define  PSR_EVENT_HDCP_ENABLE			(1 << 4)
-#define  PSR_EVENT_KVMR_SESSION_ENABLE		(1 << 3)
-#define  PSR_EVENT_VBI_ENABLE			(1 << 2)
-#define  PSR_EVENT_LPSP_MODE_EXIT		(1 << 1)
-#define  PSR_EVENT_PSR_DISABLE			(1 << 0)
-
-#define _PSR2_STATUS_A				0x60940
-#define _PSR2_STATUS_EDP			0x6f940
-#define EDP_PSR2_STATUS(tran)			_MMIO_TRANS2(tran, _PSR2_STATUS_A)
-#define EDP_PSR2_STATUS_STATE_MASK		REG_GENMASK(31, 28)
-#define EDP_PSR2_STATUS_STATE_DEEP_SLEEP	REG_FIELD_PREP(EDP_PSR2_STATUS_STATE_MASK, 0x8)
-
-#define _PSR2_SU_STATUS_A		0x60914
-#define _PSR2_SU_STATUS_EDP		0x6f914
-#define _PSR2_SU_STATUS(tran, index)	_MMIO_TRANS2(tran, _PSR2_SU_STATUS_A + (index) * 4)
-#define PSR2_SU_STATUS(tran, frame)	(_PSR2_SU_STATUS(tran, (frame) / 3))
-#define PSR2_SU_STATUS_SHIFT(frame)	(((frame) % 3) * 10)
-#define PSR2_SU_STATUS_MASK(frame)	(0x3ff << PSR2_SU_STATUS_SHIFT(frame))
-#define PSR2_SU_STATUS_FRAMES		8
-
-#define _PSR2_MAN_TRK_CTL_A					0x60910
-#define _PSR2_MAN_TRK_CTL_EDP					0x6f910
-#define PSR2_MAN_TRK_CTL(tran)					_MMIO_TRANS2(tran, _PSR2_MAN_TRK_CTL_A)
-#define  PSR2_MAN_TRK_CTL_ENABLE				REG_BIT(31)
-#define  PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK		REG_GENMASK(30, 21)
-#define  PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(val)		REG_FIELD_PREP(PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK, val)
-#define  PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK		REG_GENMASK(20, 11)
-#define  PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(val)		REG_FIELD_PREP(PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK, val)
-#define  PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME			REG_BIT(3)
-#define  PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME		REG_BIT(2)
-#define  PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE		REG_BIT(1)
-#define  ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK	REG_GENMASK(28, 16)
-#define  ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(val)	REG_FIELD_PREP(ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK, val)
-#define  ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK		REG_GENMASK(12, 0)
-#define  ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(val)		REG_FIELD_PREP(ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR_MASK, val)
-#define  ADLP_PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE		REG_BIT(31)
-#define  ADLP_PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME		REG_BIT(14)
-#define  ADLP_PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME		REG_BIT(13)
-
-/* Icelake DSC Rate Control Range Parameter Registers */
-#define DSCA_RC_RANGE_PARAMETERS_0		_MMIO(0x6B240)
-#define DSCA_RC_RANGE_PARAMETERS_0_UDW		_MMIO(0x6B240 + 4)
-#define DSCC_RC_RANGE_PARAMETERS_0		_MMIO(0x6BA40)
-#define DSCC_RC_RANGE_PARAMETERS_0_UDW		_MMIO(0x6BA40 + 4)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_PB	(0x78208)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PB	(0x78208 + 4)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_PB	(0x78308)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PB	(0x78308 + 4)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_PC	(0x78408)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PC	(0x78408 + 4)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_PC	(0x78508)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PC	(0x78508 + 4)
-#define ICL_DSC0_RC_RANGE_PARAMETERS_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_0_PB, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_0_PC)
-#define ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PB, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PC)
-#define ICL_DSC1_RC_RANGE_PARAMETERS_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_0_PB, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_0_PC)
-#define ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PB, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PC)
-#define RC_BPG_OFFSET_SHIFT			10
-#define RC_MAX_QP_SHIFT				5
-#define RC_MIN_QP_SHIFT				0
-
-#define DSCA_RC_RANGE_PARAMETERS_1		_MMIO(0x6B248)
-#define DSCA_RC_RANGE_PARAMETERS_1_UDW		_MMIO(0x6B248 + 4)
-#define DSCC_RC_RANGE_PARAMETERS_1		_MMIO(0x6BA48)
-#define DSCC_RC_RANGE_PARAMETERS_1_UDW		_MMIO(0x6BA48 + 4)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_PB	(0x78210)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PB	(0x78210 + 4)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_PB	(0x78310)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PB	(0x78310 + 4)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_PC	(0x78410)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PC	(0x78410 + 4)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_PC	(0x78510)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PC	(0x78510 + 4)
-#define ICL_DSC0_RC_RANGE_PARAMETERS_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_1_PB, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_1_PC)
-#define ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PB, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PC)
-#define ICL_DSC1_RC_RANGE_PARAMETERS_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_1_PB, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_1_PC)
-#define ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PB, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PC)
-
-#define DSCA_RC_RANGE_PARAMETERS_2		_MMIO(0x6B250)
-#define DSCA_RC_RANGE_PARAMETERS_2_UDW		_MMIO(0x6B250 + 4)
-#define DSCC_RC_RANGE_PARAMETERS_2		_MMIO(0x6BA50)
-#define DSCC_RC_RANGE_PARAMETERS_2_UDW		_MMIO(0x6BA50 + 4)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_PB	(0x78218)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PB	(0x78218 + 4)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_PB	(0x78318)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PB	(0x78318 + 4)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_PC	(0x78418)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PC	(0x78418 + 4)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_PC	(0x78518)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PC	(0x78518 + 4)
-#define ICL_DSC0_RC_RANGE_PARAMETERS_2(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_2_PB, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_2_PC)
-#define ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PB, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PC)
-#define ICL_DSC1_RC_RANGE_PARAMETERS_2(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_2_PB, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_2_PC)
-#define ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PB, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PC)
-
-#define DSCA_RC_RANGE_PARAMETERS_3		_MMIO(0x6B258)
-#define DSCA_RC_RANGE_PARAMETERS_3_UDW		_MMIO(0x6B258 + 4)
-#define DSCC_RC_RANGE_PARAMETERS_3		_MMIO(0x6BA58)
-#define DSCC_RC_RANGE_PARAMETERS_3_UDW		_MMIO(0x6BA58 + 4)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_PB	(0x78220)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PB	(0x78220 + 4)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_PB	(0x78320)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PB	(0x78320 + 4)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_PC	(0x78420)
-#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PC	(0x78420 + 4)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_PC	(0x78520)
-#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PC	(0x78520 + 4)
-#define ICL_DSC0_RC_RANGE_PARAMETERS_3(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_3_PB, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_3_PC)
-#define ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PB, \
-							_ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PC)
-#define ICL_DSC1_RC_RANGE_PARAMETERS_3(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_3_PB, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_3_PC)
-#define ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PB, \
-							_ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PC)
-
 /* VGA port control */
 #define ADPA			_MMIO(0x61100)
 #define PCH_ADPA                _MMIO(0xe1100)
@@ -2453,18 +2155,7 @@
 #define CRT_HOTPLUG_DETECT_VOLTAGE_475MV	(1 << 2)
 
 #define PORT_HOTPLUG_STAT	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61114)
-/*
- * HDMI/DP bits are g4x+
- *
- * WARNING: Bspec for hpd status bits on gen4 seems to be completely confused.
- * Please check the detailed lore in the commit message for for experimental
- * evidence.
- */
-/* Bspec says GM45 should match G4X/VLV/CHV, but reality disagrees */
-#define   PORTD_HOTPLUG_LIVE_STATUS_GM45	(1 << 29)
-#define   PORTC_HOTPLUG_LIVE_STATUS_GM45	(1 << 28)
-#define   PORTB_HOTPLUG_LIVE_STATUS_GM45	(1 << 27)
-/* G4X/VLV/CHV DP/HDMI bits again match Bspec */
+/* HDMI/DP bits are g4x+ */
 #define   PORTD_HOTPLUG_LIVE_STATUS_G4X		(1 << 27)
 #define   PORTC_HOTPLUG_LIVE_STATUS_G4X		(1 << 28)
 #define   PORTB_HOTPLUG_LIVE_STATUS_G4X		(1 << 29)
@@ -2594,59 +2285,6 @@
 #define   SDVO_PIPE_SEL_MASK_CHV		(3 << 24)
 #define   SDVO_PIPE_SEL_CHV(pipe)		((pipe) << 24)
 
-/* LVDS port control */
-#define LVDS			_MMIO(0x61180)
-/*
- * Enables the LVDS port.  This bit must be set before DPLLs are enabled, as
- * the DPLL semantics change when the LVDS is assigned to that pipe.
- */
-#define   LVDS_PORT_EN			(1 << 31)
-/* Selects pipe B for LVDS data.  Must be set on pre-965. */
-#define   LVDS_PIPE_SEL_SHIFT		30
-#define   LVDS_PIPE_SEL_MASK		(1 << 30)
-#define   LVDS_PIPE_SEL(pipe)		((pipe) << 30)
-#define   LVDS_PIPE_SEL_SHIFT_CPT	29
-#define   LVDS_PIPE_SEL_MASK_CPT	(3 << 29)
-#define   LVDS_PIPE_SEL_CPT(pipe)	((pipe) << 29)
-/* LVDS dithering flag on 965/g4x platform */
-#define   LVDS_ENABLE_DITHER		(1 << 25)
-/* LVDS sync polarity flags. Set to invert (i.e. negative) */
-#define   LVDS_VSYNC_POLARITY		(1 << 21)
-#define   LVDS_HSYNC_POLARITY		(1 << 20)
-
-/* Enable border for unscaled (or aspect-scaled) display */
-#define   LVDS_BORDER_ENABLE		(1 << 15)
-/*
- * Enables the A0-A2 data pairs and CLKA, containing 18 bits of color data per
- * pixel.
- */
-#define   LVDS_A0A2_CLKA_POWER_MASK	(3 << 8)
-#define   LVDS_A0A2_CLKA_POWER_DOWN	(0 << 8)
-#define   LVDS_A0A2_CLKA_POWER_UP	(3 << 8)
-/*
- * Controls the A3 data pair, which contains the additional LSBs for 24 bit
- * mode.  Only enabled if LVDS_A0A2_CLKA_POWER_UP also indicates it should be
- * on.
- */
-#define   LVDS_A3_POWER_MASK		(3 << 6)
-#define   LVDS_A3_POWER_DOWN		(0 << 6)
-#define   LVDS_A3_POWER_UP		(3 << 6)
-/*
- * Controls the CLKB pair.  This should only be set when LVDS_B0B3_POWER_UP
- * is set.
- */
-#define   LVDS_CLKB_POWER_MASK		(3 << 4)
-#define   LVDS_CLKB_POWER_DOWN		(0 << 4)
-#define   LVDS_CLKB_POWER_UP		(3 << 4)
-/*
- * Controls the B0-B3 data pairs.  This must be set to match the DPLL p2
- * setting for whether we are in dual-channel mode.  The B3 pair will
- * additionally only be powered up when LVDS_A3_POWER_UP is set.
- */
-#define   LVDS_B0B3_POWER_MASK		(3 << 2)
-#define   LVDS_B0B3_POWER_DOWN		(0 << 2)
-#define   LVDS_B0B3_POWER_UP		(3 << 2)
-
 /* Video Data Island Packet control */
 #define VIDEO_DIP_DATA		_MMIO(0x61178)
 /* Read the description of VIDEO_DIP_DATA (before Haswell) or VIDEO_DIP_ECC
@@ -2692,73 +2330,6 @@
 #define   VIDEO_DIP_ENABLE_GMP_HSW	(1 << 4)
 #define   VIDEO_DIP_ENABLE_SPD_HSW	(1 << 0)
 
-/* Panel power sequencing */
-#define PPS_BASE			0x61200
-#define VLV_PPS_BASE			(VLV_DISPLAY_BASE + PPS_BASE)
-#define PCH_PPS_BASE			0xC7200
-
-#define _MMIO_PPS(pps_idx, reg)		_MMIO(dev_priv->display.pps.mmio_base -	\
-					      PPS_BASE + (reg) +	\
-					      (pps_idx) * 0x100)
-
-#define _PP_STATUS			0x61200
-#define PP_STATUS(pps_idx)		_MMIO_PPS(pps_idx, _PP_STATUS)
-#define   PP_ON				REG_BIT(31)
-/*
- * Indicates that all dependencies of the panel are on:
- *
- * - PLL enabled
- * - pipe enabled
- * - LVDS/DVOB/DVOC on
- */
-#define   PP_READY			REG_BIT(30)
-#define   PP_SEQUENCE_MASK		REG_GENMASK(29, 28)
-#define   PP_SEQUENCE_NONE		REG_FIELD_PREP(PP_SEQUENCE_MASK, 0)
-#define   PP_SEQUENCE_POWER_UP		REG_FIELD_PREP(PP_SEQUENCE_MASK, 1)
-#define   PP_SEQUENCE_POWER_DOWN	REG_FIELD_PREP(PP_SEQUENCE_MASK, 2)
-#define   PP_CYCLE_DELAY_ACTIVE		REG_BIT(27)
-#define   PP_SEQUENCE_STATE_MASK	REG_GENMASK(3, 0)
-#define   PP_SEQUENCE_STATE_OFF_IDLE	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x0)
-#define   PP_SEQUENCE_STATE_OFF_S0_1	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x1)
-#define   PP_SEQUENCE_STATE_OFF_S0_2	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x2)
-#define   PP_SEQUENCE_STATE_OFF_S0_3	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x3)
-#define   PP_SEQUENCE_STATE_ON_IDLE	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x8)
-#define   PP_SEQUENCE_STATE_ON_S1_1	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x9)
-#define   PP_SEQUENCE_STATE_ON_S1_2	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xa)
-#define   PP_SEQUENCE_STATE_ON_S1_3	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xb)
-#define   PP_SEQUENCE_STATE_RESET	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xf)
-
-#define _PP_CONTROL			0x61204
-#define PP_CONTROL(pps_idx)		_MMIO_PPS(pps_idx, _PP_CONTROL)
-#define  PANEL_UNLOCK_MASK		REG_GENMASK(31, 16)
-#define  PANEL_UNLOCK_REGS		REG_FIELD_PREP(PANEL_UNLOCK_MASK, 0xabcd)
-#define  BXT_POWER_CYCLE_DELAY_MASK	REG_GENMASK(8, 4)
-#define  EDP_FORCE_VDD			REG_BIT(3)
-#define  EDP_BLC_ENABLE			REG_BIT(2)
-#define  PANEL_POWER_RESET		REG_BIT(1)
-#define  PANEL_POWER_ON			REG_BIT(0)
-
-#define _PP_ON_DELAYS			0x61208
-#define PP_ON_DELAYS(pps_idx)		_MMIO_PPS(pps_idx, _PP_ON_DELAYS)
-#define  PANEL_PORT_SELECT_MASK		REG_GENMASK(31, 30)
-#define  PANEL_PORT_SELECT_LVDS		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 0)
-#define  PANEL_PORT_SELECT_DPA		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 1)
-#define  PANEL_PORT_SELECT_DPC		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 2)
-#define  PANEL_PORT_SELECT_DPD		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 3)
-#define  PANEL_PORT_SELECT_VLV(port)	REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, port)
-#define  PANEL_POWER_UP_DELAY_MASK	REG_GENMASK(28, 16)
-#define  PANEL_LIGHT_ON_DELAY_MASK	REG_GENMASK(12, 0)
-
-#define _PP_OFF_DELAYS			0x6120C
-#define PP_OFF_DELAYS(pps_idx)		_MMIO_PPS(pps_idx, _PP_OFF_DELAYS)
-#define  PANEL_POWER_DOWN_DELAY_MASK	REG_GENMASK(28, 16)
-#define  PANEL_LIGHT_OFF_DELAY_MASK	REG_GENMASK(12, 0)
-
-#define _PP_DIVISOR			0x61210
-#define PP_DIVISOR(pps_idx)		_MMIO_PPS(pps_idx, _PP_DIVISOR)
-#define  PP_REFERENCE_DIVIDER_MASK	REG_GENMASK(31, 8)
-#define  PANEL_POWER_CYCLE_DELAY_MASK	REG_GENMASK(4, 0)
-
 /* Panel fitting */
 #define PFIT_CONTROL	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61230)
 #define   PFIT_ENABLE		(1 << 31)
@@ -2796,485 +2367,6 @@
 #define PCH_GTC_CTL		_MMIO(0xe7000)
 #define   PCH_GTC_ENABLE	(1 << 31)
 
-/* TV port control */
-#define TV_CTL			_MMIO(0x68000)
-/* Enables the TV encoder */
-# define TV_ENC_ENABLE			(1 << 31)
-/* Sources the TV encoder input from pipe B instead of A. */
-# define TV_ENC_PIPE_SEL_SHIFT		30
-# define TV_ENC_PIPE_SEL_MASK		(1 << 30)
-# define TV_ENC_PIPE_SEL(pipe)		((pipe) << 30)
-/* Outputs composite video (DAC A only) */
-# define TV_ENC_OUTPUT_COMPOSITE	(0 << 28)
-/* Outputs SVideo video (DAC B/C) */
-# define TV_ENC_OUTPUT_SVIDEO		(1 << 28)
-/* Outputs Component video (DAC A/B/C) */
-# define TV_ENC_OUTPUT_COMPONENT	(2 << 28)
-/* Outputs Composite and SVideo (DAC A/B/C) */
-# define TV_ENC_OUTPUT_SVIDEO_COMPOSITE	(3 << 28)
-# define TV_TRILEVEL_SYNC		(1 << 21)
-/* Enables slow sync generation (945GM only) */
-# define TV_SLOW_SYNC			(1 << 20)
-/* Selects 4x oversampling for 480i and 576p */
-# define TV_OVERSAMPLE_4X		(0 << 18)
-/* Selects 2x oversampling for 720p and 1080i */
-# define TV_OVERSAMPLE_2X		(1 << 18)
-/* Selects no oversampling for 1080p */
-# define TV_OVERSAMPLE_NONE		(2 << 18)
-/* Selects 8x oversampling */
-# define TV_OVERSAMPLE_8X		(3 << 18)
-# define TV_OVERSAMPLE_MASK		(3 << 18)
-/* Selects progressive mode rather than interlaced */
-# define TV_PROGRESSIVE			(1 << 17)
-/* Sets the colorburst to PAL mode.  Required for non-M PAL modes. */
-# define TV_PAL_BURST			(1 << 16)
-/* Field for setting delay of Y compared to C */
-# define TV_YC_SKEW_MASK		(7 << 12)
-/* Enables a fix for 480p/576p standard definition modes on the 915GM only */
-# define TV_ENC_SDP_FIX			(1 << 11)
-/*
- * Enables a fix for the 915GM only.
- *
- * Not sure what it does.
- */
-# define TV_ENC_C0_FIX			(1 << 10)
-/* Bits that must be preserved by software */
-# define TV_CTL_SAVE			((1 << 11) | (3 << 9) | (7 << 6) | 0xf)
-# define TV_FUSE_STATE_MASK		(3 << 4)
-/* Read-only state that reports all features enabled */
-# define TV_FUSE_STATE_ENABLED		(0 << 4)
-/* Read-only state that reports that Macrovision is disabled in hardware*/
-# define TV_FUSE_STATE_NO_MACROVISION	(1 << 4)
-/* Read-only state that reports that TV-out is disabled in hardware. */
-# define TV_FUSE_STATE_DISABLED		(2 << 4)
-/* Normal operation */
-# define TV_TEST_MODE_NORMAL		(0 << 0)
-/* Encoder test pattern 1 - combo pattern */
-# define TV_TEST_MODE_PATTERN_1		(1 << 0)
-/* Encoder test pattern 2 - full screen vertical 75% color bars */
-# define TV_TEST_MODE_PATTERN_2		(2 << 0)
-/* Encoder test pattern 3 - full screen horizontal 75% color bars */
-# define TV_TEST_MODE_PATTERN_3		(3 << 0)
-/* Encoder test pattern 4 - random noise */
-# define TV_TEST_MODE_PATTERN_4		(4 << 0)
-/* Encoder test pattern 5 - linear color ramps */
-# define TV_TEST_MODE_PATTERN_5		(5 << 0)
-/*
- * This test mode forces the DACs to 50% of full output.
- *
- * This is used for load detection in combination with TVDAC_SENSE_MASK
- */
-# define TV_TEST_MODE_MONITOR_DETECT	(7 << 0)
-# define TV_TEST_MODE_MASK		(7 << 0)
-
-#define TV_DAC			_MMIO(0x68004)
-# define TV_DAC_SAVE		0x00ffff00
-/*
- * Reports that DAC state change logic has reported change (RO).
- *
- * This gets cleared when TV_DAC_STATE_EN is cleared
-*/
-# define TVDAC_STATE_CHG		(1 << 31)
-# define TVDAC_SENSE_MASK		(7 << 28)
-/* Reports that DAC A voltage is above the detect threshold */
-# define TVDAC_A_SENSE			(1 << 30)
-/* Reports that DAC B voltage is above the detect threshold */
-# define TVDAC_B_SENSE			(1 << 29)
-/* Reports that DAC C voltage is above the detect threshold */
-# define TVDAC_C_SENSE			(1 << 28)
-/*
- * Enables DAC state detection logic, for load-based TV detection.
- *
- * The PLL of the chosen pipe (in TV_CTL) must be running, and the encoder set
- * to off, for load detection to work.
- */
-# define TVDAC_STATE_CHG_EN		(1 << 27)
-/* Sets the DAC A sense value to high */
-# define TVDAC_A_SENSE_CTL		(1 << 26)
-/* Sets the DAC B sense value to high */
-# define TVDAC_B_SENSE_CTL		(1 << 25)
-/* Sets the DAC C sense value to high */
-# define TVDAC_C_SENSE_CTL		(1 << 24)
-/* Overrides the ENC_ENABLE and DAC voltage levels */
-# define DAC_CTL_OVERRIDE		(1 << 7)
-/* Sets the slew rate.  Must be preserved in software */
-# define ENC_TVDAC_SLEW_FAST		(1 << 6)
-# define DAC_A_1_3_V			(0 << 4)
-# define DAC_A_1_1_V			(1 << 4)
-# define DAC_A_0_7_V			(2 << 4)
-# define DAC_A_MASK			(3 << 4)
-# define DAC_B_1_3_V			(0 << 2)
-# define DAC_B_1_1_V			(1 << 2)
-# define DAC_B_0_7_V			(2 << 2)
-# define DAC_B_MASK			(3 << 2)
-# define DAC_C_1_3_V			(0 << 0)
-# define DAC_C_1_1_V			(1 << 0)
-# define DAC_C_0_7_V			(2 << 0)
-# define DAC_C_MASK			(3 << 0)
-
-/*
- * CSC coefficients are stored in a floating point format with 9 bits of
- * mantissa and 2 or 3 bits of exponent.  The exponent is represented as 2**-n,
- * where 2-bit exponents are unsigned n, and 3-bit exponents are signed n with
- * -1 (0x3) being the only legal negative value.
- */
-#define TV_CSC_Y		_MMIO(0x68010)
-# define TV_RY_MASK			0x07ff0000
-# define TV_RY_SHIFT			16
-# define TV_GY_MASK			0x00000fff
-# define TV_GY_SHIFT			0
-
-#define TV_CSC_Y2		_MMIO(0x68014)
-# define TV_BY_MASK			0x07ff0000
-# define TV_BY_SHIFT			16
-/*
- * Y attenuation for component video.
- *
- * Stored in 1.9 fixed point.
- */
-# define TV_AY_MASK			0x000003ff
-# define TV_AY_SHIFT			0
-
-#define TV_CSC_U		_MMIO(0x68018)
-# define TV_RU_MASK			0x07ff0000
-# define TV_RU_SHIFT			16
-# define TV_GU_MASK			0x000007ff
-# define TV_GU_SHIFT			0
-
-#define TV_CSC_U2		_MMIO(0x6801c)
-# define TV_BU_MASK			0x07ff0000
-# define TV_BU_SHIFT			16
-/*
- * U attenuation for component video.
- *
- * Stored in 1.9 fixed point.
- */
-# define TV_AU_MASK			0x000003ff
-# define TV_AU_SHIFT			0
-
-#define TV_CSC_V		_MMIO(0x68020)
-# define TV_RV_MASK			0x0fff0000
-# define TV_RV_SHIFT			16
-# define TV_GV_MASK			0x000007ff
-# define TV_GV_SHIFT			0
-
-#define TV_CSC_V2		_MMIO(0x68024)
-# define TV_BV_MASK			0x07ff0000
-# define TV_BV_SHIFT			16
-/*
- * V attenuation for component video.
- *
- * Stored in 1.9 fixed point.
- */
-# define TV_AV_MASK			0x000007ff
-# define TV_AV_SHIFT			0
-
-#define TV_CLR_KNOBS		_MMIO(0x68028)
-/* 2s-complement brightness adjustment */
-# define TV_BRIGHTNESS_MASK		0xff000000
-# define TV_BRIGHTNESS_SHIFT		24
-/* Contrast adjustment, as a 2.6 unsigned floating point number */
-# define TV_CONTRAST_MASK		0x00ff0000
-# define TV_CONTRAST_SHIFT		16
-/* Saturation adjustment, as a 2.6 unsigned floating point number */
-# define TV_SATURATION_MASK		0x0000ff00
-# define TV_SATURATION_SHIFT		8
-/* Hue adjustment, as an integer phase angle in degrees */
-# define TV_HUE_MASK			0x000000ff
-# define TV_HUE_SHIFT			0
-
-#define TV_CLR_LEVEL		_MMIO(0x6802c)
-/* Controls the DAC level for black */
-# define TV_BLACK_LEVEL_MASK		0x01ff0000
-# define TV_BLACK_LEVEL_SHIFT		16
-/* Controls the DAC level for blanking */
-# define TV_BLANK_LEVEL_MASK		0x000001ff
-# define TV_BLANK_LEVEL_SHIFT		0
-
-#define TV_H_CTL_1		_MMIO(0x68030)
-/* Number of pixels in the hsync. */
-# define TV_HSYNC_END_MASK		0x1fff0000
-# define TV_HSYNC_END_SHIFT		16
-/* Total number of pixels minus one in the line (display and blanking). */
-# define TV_HTOTAL_MASK			0x00001fff
-# define TV_HTOTAL_SHIFT		0
-
-#define TV_H_CTL_2		_MMIO(0x68034)
-/* Enables the colorburst (needed for non-component color) */
-# define TV_BURST_ENA			(1 << 31)
-/* Offset of the colorburst from the start of hsync, in pixels minus one. */
-# define TV_HBURST_START_SHIFT		16
-# define TV_HBURST_START_MASK		0x1fff0000
-/* Length of the colorburst */
-# define TV_HBURST_LEN_SHIFT		0
-# define TV_HBURST_LEN_MASK		0x0001fff
-
-#define TV_H_CTL_3		_MMIO(0x68038)
-/* End of hblank, measured in pixels minus one from start of hsync */
-# define TV_HBLANK_END_SHIFT		16
-# define TV_HBLANK_END_MASK		0x1fff0000
-/* Start of hblank, measured in pixels minus one from start of hsync */
-# define TV_HBLANK_START_SHIFT		0
-# define TV_HBLANK_START_MASK		0x0001fff
-
-#define TV_V_CTL_1		_MMIO(0x6803c)
-/* XXX */
-# define TV_NBR_END_SHIFT		16
-# define TV_NBR_END_MASK		0x07ff0000
-/* XXX */
-# define TV_VI_END_F1_SHIFT		8
-# define TV_VI_END_F1_MASK		0x00003f00
-/* XXX */
-# define TV_VI_END_F2_SHIFT		0
-# define TV_VI_END_F2_MASK		0x0000003f
-
-#define TV_V_CTL_2		_MMIO(0x68040)
-/* Length of vsync, in half lines */
-# define TV_VSYNC_LEN_MASK		0x07ff0000
-# define TV_VSYNC_LEN_SHIFT		16
-/* Offset of the start of vsync in field 1, measured in one less than the
- * number of half lines.
- */
-# define TV_VSYNC_START_F1_MASK		0x00007f00
-# define TV_VSYNC_START_F1_SHIFT	8
-/*
- * Offset of the start of vsync in field 2, measured in one less than the
- * number of half lines.
- */
-# define TV_VSYNC_START_F2_MASK		0x0000007f
-# define TV_VSYNC_START_F2_SHIFT	0
-
-#define TV_V_CTL_3		_MMIO(0x68044)
-/* Enables generation of the equalization signal */
-# define TV_EQUAL_ENA			(1 << 31)
-/* Length of vsync, in half lines */
-# define TV_VEQ_LEN_MASK		0x007f0000
-# define TV_VEQ_LEN_SHIFT		16
-/* Offset of the start of equalization in field 1, measured in one less than
- * the number of half lines.
- */
-# define TV_VEQ_START_F1_MASK		0x0007f00
-# define TV_VEQ_START_F1_SHIFT		8
-/*
- * Offset of the start of equalization in field 2, measured in one less than
- * the number of half lines.
- */
-# define TV_VEQ_START_F2_MASK		0x000007f
-# define TV_VEQ_START_F2_SHIFT		0
-
-#define TV_V_CTL_4		_MMIO(0x68048)
-/*
- * Offset to start of vertical colorburst, measured in one less than the
- * number of lines from vertical start.
- */
-# define TV_VBURST_START_F1_MASK	0x003f0000
-# define TV_VBURST_START_F1_SHIFT	16
-/*
- * Offset to the end of vertical colorburst, measured in one less than the
- * number of lines from the start of NBR.
- */
-# define TV_VBURST_END_F1_MASK		0x000000ff
-# define TV_VBURST_END_F1_SHIFT		0
-
-#define TV_V_CTL_5		_MMIO(0x6804c)
-/*
- * Offset to start of vertical colorburst, measured in one less than the
- * number of lines from vertical start.
- */
-# define TV_VBURST_START_F2_MASK	0x003f0000
-# define TV_VBURST_START_F2_SHIFT	16
-/*
- * Offset to the end of vertical colorburst, measured in one less than the
- * number of lines from the start of NBR.
- */
-# define TV_VBURST_END_F2_MASK		0x000000ff
-# define TV_VBURST_END_F2_SHIFT		0
-
-#define TV_V_CTL_6		_MMIO(0x68050)
-/*
- * Offset to start of vertical colorburst, measured in one less than the
- * number of lines from vertical start.
- */
-# define TV_VBURST_START_F3_MASK	0x003f0000
-# define TV_VBURST_START_F3_SHIFT	16
-/*
- * Offset to the end of vertical colorburst, measured in one less than the
- * number of lines from the start of NBR.
- */
-# define TV_VBURST_END_F3_MASK		0x000000ff
-# define TV_VBURST_END_F3_SHIFT		0
-
-#define TV_V_CTL_7		_MMIO(0x68054)
-/*
- * Offset to start of vertical colorburst, measured in one less than the
- * number of lines from vertical start.
- */
-# define TV_VBURST_START_F4_MASK	0x003f0000
-# define TV_VBURST_START_F4_SHIFT	16
-/*
- * Offset to the end of vertical colorburst, measured in one less than the
- * number of lines from the start of NBR.
- */
-# define TV_VBURST_END_F4_MASK		0x000000ff
-# define TV_VBURST_END_F4_SHIFT		0
-
-#define TV_SC_CTL_1		_MMIO(0x68060)
-/* Turns on the first subcarrier phase generation DDA */
-# define TV_SC_DDA1_EN			(1 << 31)
-/* Turns on the first subcarrier phase generation DDA */
-# define TV_SC_DDA2_EN			(1 << 30)
-/* Turns on the first subcarrier phase generation DDA */
-# define TV_SC_DDA3_EN			(1 << 29)
-/* Sets the subcarrier DDA to reset frequency every other field */
-# define TV_SC_RESET_EVERY_2		(0 << 24)
-/* Sets the subcarrier DDA to reset frequency every fourth field */
-# define TV_SC_RESET_EVERY_4		(1 << 24)
-/* Sets the subcarrier DDA to reset frequency every eighth field */
-# define TV_SC_RESET_EVERY_8		(2 << 24)
-/* Sets the subcarrier DDA to never reset the frequency */
-# define TV_SC_RESET_NEVER		(3 << 24)
-/* Sets the peak amplitude of the colorburst.*/
-# define TV_BURST_LEVEL_MASK		0x00ff0000
-# define TV_BURST_LEVEL_SHIFT		16
-/* Sets the increment of the first subcarrier phase generation DDA */
-# define TV_SCDDA1_INC_MASK		0x00000fff
-# define TV_SCDDA1_INC_SHIFT		0
-
-#define TV_SC_CTL_2		_MMIO(0x68064)
-/* Sets the rollover for the second subcarrier phase generation DDA */
-# define TV_SCDDA2_SIZE_MASK		0x7fff0000
-# define TV_SCDDA2_SIZE_SHIFT		16
-/* Sets the increent of the second subcarrier phase generation DDA */
-# define TV_SCDDA2_INC_MASK		0x00007fff
-# define TV_SCDDA2_INC_SHIFT		0
-
-#define TV_SC_CTL_3		_MMIO(0x68068)
-/* Sets the rollover for the third subcarrier phase generation DDA */
-# define TV_SCDDA3_SIZE_MASK		0x7fff0000
-# define TV_SCDDA3_SIZE_SHIFT		16
-/* Sets the increent of the third subcarrier phase generation DDA */
-# define TV_SCDDA3_INC_MASK		0x00007fff
-# define TV_SCDDA3_INC_SHIFT		0
-
-#define TV_WIN_POS		_MMIO(0x68070)
-/* X coordinate of the display from the start of horizontal active */
-# define TV_XPOS_MASK			0x1fff0000
-# define TV_XPOS_SHIFT			16
-/* Y coordinate of the display from the start of vertical active (NBR) */
-# define TV_YPOS_MASK			0x00000fff
-# define TV_YPOS_SHIFT			0
-
-#define TV_WIN_SIZE		_MMIO(0x68074)
-/* Horizontal size of the display window, measured in pixels*/
-# define TV_XSIZE_MASK			0x1fff0000
-# define TV_XSIZE_SHIFT			16
-/*
- * Vertical size of the display window, measured in pixels.
- *
- * Must be even for interlaced modes.
- */
-# define TV_YSIZE_MASK			0x00000fff
-# define TV_YSIZE_SHIFT			0
-
-#define TV_FILTER_CTL_1		_MMIO(0x68080)
-/*
- * Enables automatic scaling calculation.
- *
- * If set, the rest of the registers are ignored, and the calculated values can
- * be read back from the register.
- */
-# define TV_AUTO_SCALE			(1 << 31)
-/*
- * Disables the vertical filter.
- *
- * This is required on modes more than 1024 pixels wide */
-# define TV_V_FILTER_BYPASS		(1 << 29)
-/* Enables adaptive vertical filtering */
-# define TV_VADAPT			(1 << 28)
-# define TV_VADAPT_MODE_MASK		(3 << 26)
-/* Selects the least adaptive vertical filtering mode */
-# define TV_VADAPT_MODE_LEAST		(0 << 26)
-/* Selects the moderately adaptive vertical filtering mode */
-# define TV_VADAPT_MODE_MODERATE	(1 << 26)
-/* Selects the most adaptive vertical filtering mode */
-# define TV_VADAPT_MODE_MOST		(3 << 26)
-/*
- * Sets the horizontal scaling factor.
- *
- * This should be the fractional part of the horizontal scaling factor divided
- * by the oversampling rate.  TV_HSCALE should be less than 1, and set to:
- *
- * (src width - 1) / ((oversample * dest width) - 1)
- */
-# define TV_HSCALE_FRAC_MASK		0x00003fff
-# define TV_HSCALE_FRAC_SHIFT		0
-
-#define TV_FILTER_CTL_2		_MMIO(0x68084)
-/*
- * Sets the integer part of the 3.15 fixed-point vertical scaling factor.
- *
- * TV_VSCALE should be (src height - 1) / ((interlace * dest height) - 1)
- */
-# define TV_VSCALE_INT_MASK		0x00038000
-# define TV_VSCALE_INT_SHIFT		15
-/*
- * Sets the fractional part of the 3.15 fixed-point vertical scaling factor.
- *
- * \sa TV_VSCALE_INT_MASK
- */
-# define TV_VSCALE_FRAC_MASK		0x00007fff
-# define TV_VSCALE_FRAC_SHIFT		0
-
-#define TV_FILTER_CTL_3		_MMIO(0x68088)
-/*
- * Sets the integer part of the 3.15 fixed-point vertical scaling factor.
- *
- * TV_VSCALE should be (src height - 1) / (1/4 * (dest height - 1))
- *
- * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes.
- */
-# define TV_VSCALE_IP_INT_MASK		0x00038000
-# define TV_VSCALE_IP_INT_SHIFT		15
-/*
- * Sets the fractional part of the 3.15 fixed-point vertical scaling factor.
- *
- * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes.
- *
- * \sa TV_VSCALE_IP_INT_MASK
- */
-# define TV_VSCALE_IP_FRAC_MASK		0x00007fff
-# define TV_VSCALE_IP_FRAC_SHIFT		0
-
-#define TV_CC_CONTROL		_MMIO(0x68090)
-# define TV_CC_ENABLE			(1 << 31)
-/*
- * Specifies which field to send the CC data in.
- *
- * CC data is usually sent in field 0.
- */
-# define TV_CC_FID_MASK			(1 << 27)
-# define TV_CC_FID_SHIFT		27
-/* Sets the horizontal position of the CC data.  Usually 135. */
-# define TV_CC_HOFF_MASK		0x03ff0000
-# define TV_CC_HOFF_SHIFT		16
-/* Sets the vertical position of the CC data.  Usually 21 */
-# define TV_CC_LINE_MASK		0x0000003f
-# define TV_CC_LINE_SHIFT		0
-
-#define TV_CC_DATA		_MMIO(0x68094)
-# define TV_CC_RDY			(1 << 31)
-/* Second word of CC data to be transmitted. */
-# define TV_CC_DATA_2_MASK		0x007f0000
-# define TV_CC_DATA_2_SHIFT		16
-/* First word of CC data to be transmitted. */
-# define TV_CC_DATA_1_MASK		0x0000007f
-# define TV_CC_DATA_1_SHIFT		0
-
-#define TV_H_LUMA(i)		_MMIO(0x68100 + (i) * 4) /* 60 registers */
-#define TV_H_CHROMA(i)		_MMIO(0x68200 + (i) * 4) /* 60 registers */
-#define TV_V_LUMA(i)		_MMIO(0x68300 + (i) * 4) /* 43 registers */
-#define TV_V_CHROMA(i)		_MMIO(0x68400 + (i) * 4) /* 43 registers */
-
 /* Display Port */
 #define DP_A			_MMIO(0x64000) /* eDP */
 #define DP_B			_MMIO(0x64100)
@@ -3368,79 +2460,6 @@
 /* A fantasy */
 #define   DP_DETECTED			(1 << 2)
 
-/* The aux channel provides a way to talk to the
- * signal sink for DDC etc. Max packet size supported
- * is 20 bytes in each direction, hence the 5 fixed
- * data registers
- */
-#define _DPA_AUX_CH_CTL		(DISPLAY_MMIO_BASE(dev_priv) + 0x64010)
-#define _DPA_AUX_CH_DATA1	(DISPLAY_MMIO_BASE(dev_priv) + 0x64014)
-
-#define _DPB_AUX_CH_CTL		(DISPLAY_MMIO_BASE(dev_priv) + 0x64110)
-#define _DPB_AUX_CH_DATA1	(DISPLAY_MMIO_BASE(dev_priv) + 0x64114)
-
-#define DP_AUX_CH_CTL(aux_ch)	_MMIO_PORT(aux_ch, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL)
-#define DP_AUX_CH_DATA(aux_ch, i)	_MMIO(_PORT(aux_ch, _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */
-
-#define _XELPDP_USBC1_AUX_CH_CTL	0x16F210
-#define _XELPDP_USBC2_AUX_CH_CTL	0x16F410
-#define _XELPDP_USBC3_AUX_CH_CTL	0x16F610
-#define _XELPDP_USBC4_AUX_CH_CTL	0x16F810
-
-#define XELPDP_DP_AUX_CH_CTL(aux_ch)		_MMIO(_PICK(aux_ch, \
-						       _DPA_AUX_CH_CTL, \
-						       _DPB_AUX_CH_CTL, \
-						       0, /* port/aux_ch C is non-existent */ \
-						       _XELPDP_USBC1_AUX_CH_CTL, \
-						       _XELPDP_USBC2_AUX_CH_CTL, \
-						       _XELPDP_USBC3_AUX_CH_CTL, \
-						       _XELPDP_USBC4_AUX_CH_CTL))
-
-#define _XELPDP_USBC1_AUX_CH_DATA1      0x16F214
-#define _XELPDP_USBC2_AUX_CH_DATA1      0x16F414
-#define _XELPDP_USBC3_AUX_CH_DATA1      0x16F614
-#define _XELPDP_USBC4_AUX_CH_DATA1      0x16F814
-
-#define XELPDP_DP_AUX_CH_DATA(aux_ch, i)	_MMIO(_PICK(aux_ch, \
-						       _DPA_AUX_CH_DATA1, \
-						       _DPB_AUX_CH_DATA1, \
-						       0, /* port/aux_ch C is non-existent */ \
-						       _XELPDP_USBC1_AUX_CH_DATA1, \
-						       _XELPDP_USBC2_AUX_CH_DATA1, \
-						       _XELPDP_USBC3_AUX_CH_DATA1, \
-						       _XELPDP_USBC4_AUX_CH_DATA1) + (i) * 4)
-
-#define   DP_AUX_CH_CTL_SEND_BUSY	    (1 << 31)
-#define   DP_AUX_CH_CTL_DONE		    (1 << 30)
-#define   DP_AUX_CH_CTL_INTERRUPT	    (1 << 29)
-#define   DP_AUX_CH_CTL_TIME_OUT_ERROR	    (1 << 28)
-#define   DP_AUX_CH_CTL_TIME_OUT_400us	    (0 << 26)
-#define   DP_AUX_CH_CTL_TIME_OUT_600us	    (1 << 26)
-#define   DP_AUX_CH_CTL_TIME_OUT_800us	    (2 << 26)
-#define   DP_AUX_CH_CTL_TIME_OUT_MAX	    (3 << 26) /* Varies per platform */
-#define   DP_AUX_CH_CTL_TIME_OUT_MASK	    (3 << 26)
-#define   DP_AUX_CH_CTL_RECEIVE_ERROR	    (1 << 25)
-#define   DP_AUX_CH_CTL_MESSAGE_SIZE_MASK    (0x1f << 20)
-#define   DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT   20
-#define   XELPDP_DP_AUX_CH_CTL_POWER_REQUEST REG_BIT(19)
-#define   XELPDP_DP_AUX_CH_CTL_POWER_STATUS  REG_BIT(18)
-#define   DP_AUX_CH_CTL_PRECHARGE_2US_MASK   (0xf << 16)
-#define   DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT  16
-#define   DP_AUX_CH_CTL_AUX_AKSV_SELECT	    (1 << 15)
-#define   DP_AUX_CH_CTL_MANCHESTER_TEST	    (1 << 14)
-#define   DP_AUX_CH_CTL_SYNC_TEST	    (1 << 13)
-#define   DP_AUX_CH_CTL_DEGLITCH_TEST	    (1 << 12)
-#define   DP_AUX_CH_CTL_PRECHARGE_TEST	    (1 << 11)
-#define   DP_AUX_CH_CTL_BIT_CLOCK_2X_MASK    (0x7ff)
-#define   DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT   0
-#define   DP_AUX_CH_CTL_PSR_DATA_AUX_REG_SKL	(1 << 14)
-#define   DP_AUX_CH_CTL_FS_DATA_AUX_REG_SKL	(1 << 13)
-#define   DP_AUX_CH_CTL_GTC_DATA_AUX_REG_SKL	(1 << 12)
-#define   DP_AUX_CH_CTL_TBT_IO			(1 << 11)
-#define   DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL_MASK (0x1f << 5)
-#define   DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(c) (((c) - 1) << 5)
-#define   DP_AUX_CH_CTL_SYNC_PULSE_SKL(c)   ((c) - 1)
-
 /*
  * Computing GMCH M and N values for the Display Port link
  *
@@ -3494,61 +2513,61 @@
 #define _PIPEADSL		0x70000
 #define   PIPEDSL_CURR_FIELD	REG_BIT(31) /* ctg+ */
 #define   PIPEDSL_LINE_MASK	REG_GENMASK(19, 0)
-#define _PIPEACONF		0x70008
-#define   PIPECONF_ENABLE			REG_BIT(31)
-#define   PIPECONF_DOUBLE_WIDE			REG_BIT(30) /* pre-i965 */
-#define   PIPECONF_STATE_ENABLE			REG_BIT(30) /* i965+ */
-#define   PIPECONF_DSI_PLL_LOCKED		REG_BIT(29) /* vlv & pipe A only */
-#define   PIPECONF_FRAME_START_DELAY_MASK	REG_GENMASK(28, 27) /* pre-hsw */
-#define   PIPECONF_FRAME_START_DELAY(x)		REG_FIELD_PREP(PIPECONF_FRAME_START_DELAY_MASK, (x)) /* pre-hsw: 0-3 */
-#define   PIPECONF_PIPE_LOCKED			REG_BIT(25)
-#define   PIPECONF_FORCE_BORDER			REG_BIT(25)
-#define   PIPECONF_GAMMA_MODE_MASK_I9XX		REG_BIT(24) /* gmch */
-#define   PIPECONF_GAMMA_MODE_MASK_ILK		REG_GENMASK(25, 24) /* ilk-ivb */
-#define   PIPECONF_GAMMA_MODE_8BIT		REG_FIELD_PREP(PIPECONF_GAMMA_MODE_MASK, 0)
-#define   PIPECONF_GAMMA_MODE_10BIT		REG_FIELD_PREP(PIPECONF_GAMMA_MODE_MASK, 1)
-#define   PIPECONF_GAMMA_MODE_12BIT		REG_FIELD_PREP(PIPECONF_GAMMA_MODE_MASK_ILK, 2) /* ilk-ivb */
-#define   PIPECONF_GAMMA_MODE_SPLIT		REG_FIELD_PREP(PIPECONF_GAMMA_MODE_MASK_ILK, 3) /* ivb */
-#define   PIPECONF_GAMMA_MODE(x)		REG_FIELD_PREP(PIPECONF_GAMMA_MODE_MASK_ILK, (x)) /* pass in GAMMA_MODE_MODE_* */
-#define   PIPECONF_INTERLACE_MASK		REG_GENMASK(23, 21) /* gen3+ */
-#define   PIPECONF_INTERLACE_PROGRESSIVE	REG_FIELD_PREP(PIPECONF_INTERLACE_MASK, 0)
-#define   PIPECONF_INTERLACE_W_SYNC_SHIFT_PANEL	REG_FIELD_PREP(PIPECONF_INTERLACE_MASK, 4) /* gen4 only */
-#define   PIPECONF_INTERLACE_W_SYNC_SHIFT	REG_FIELD_PREP(PIPECONF_INTERLACE_MASK, 5) /* gen4 only */
-#define   PIPECONF_INTERLACE_W_FIELD_INDICATION	REG_FIELD_PREP(PIPECONF_INTERLACE_MASK, 6)
-#define   PIPECONF_INTERLACE_FIELD_0_ONLY	REG_FIELD_PREP(PIPECONF_INTERLACE_MASK, 7) /* gen3 only */
+#define _TRANSACONF		0x70008
+#define   TRANSCONF_ENABLE			REG_BIT(31)
+#define   TRANSCONF_DOUBLE_WIDE			REG_BIT(30) /* pre-i965 */
+#define   TRANSCONF_STATE_ENABLE			REG_BIT(30) /* i965+ */
+#define   TRANSCONF_DSI_PLL_LOCKED		REG_BIT(29) /* vlv & pipe A only */
+#define   TRANSCONF_FRAME_START_DELAY_MASK	REG_GENMASK(28, 27) /* pre-hsw */
+#define   TRANSCONF_FRAME_START_DELAY(x)		REG_FIELD_PREP(TRANSCONF_FRAME_START_DELAY_MASK, (x)) /* pre-hsw: 0-3 */
+#define   TRANSCONF_PIPE_LOCKED			REG_BIT(25)
+#define   TRANSCONF_FORCE_BORDER			REG_BIT(25)
+#define   TRANSCONF_GAMMA_MODE_MASK_I9XX		REG_BIT(24) /* gmch */
+#define   TRANSCONF_GAMMA_MODE_MASK_ILK		REG_GENMASK(25, 24) /* ilk-ivb */
+#define   TRANSCONF_GAMMA_MODE_8BIT		REG_FIELD_PREP(TRANSCONF_GAMMA_MODE_MASK, 0)
+#define   TRANSCONF_GAMMA_MODE_10BIT		REG_FIELD_PREP(TRANSCONF_GAMMA_MODE_MASK, 1)
+#define   TRANSCONF_GAMMA_MODE_12BIT		REG_FIELD_PREP(TRANSCONF_GAMMA_MODE_MASK_ILK, 2) /* ilk-ivb */
+#define   TRANSCONF_GAMMA_MODE_SPLIT		REG_FIELD_PREP(TRANSCONF_GAMMA_MODE_MASK_ILK, 3) /* ivb */
+#define   TRANSCONF_GAMMA_MODE(x)		REG_FIELD_PREP(TRANSCONF_GAMMA_MODE_MASK_ILK, (x)) /* pass in GAMMA_MODE_MODE_* */
+#define   TRANSCONF_INTERLACE_MASK		REG_GENMASK(23, 21) /* gen3+ */
+#define   TRANSCONF_INTERLACE_PROGRESSIVE	REG_FIELD_PREP(TRANSCONF_INTERLACE_MASK, 0)
+#define   TRANSCONF_INTERLACE_W_SYNC_SHIFT_PANEL	REG_FIELD_PREP(TRANSCONF_INTERLACE_MASK, 4) /* gen4 only */
+#define   TRANSCONF_INTERLACE_W_SYNC_SHIFT	REG_FIELD_PREP(TRANSCONF_INTERLACE_MASK, 5) /* gen4 only */
+#define   TRANSCONF_INTERLACE_W_FIELD_INDICATION	REG_FIELD_PREP(TRANSCONF_INTERLACE_MASK, 6)
+#define   TRANSCONF_INTERLACE_FIELD_0_ONLY	REG_FIELD_PREP(TRANSCONF_INTERLACE_MASK, 7) /* gen3 only */
 /*
  * ilk+: PF/D=progressive fetch/display, IF/D=interlaced fetch/display,
  * DBL=power saving pixel doubling, PF-ID* requires panel fitter
  */
-#define   PIPECONF_INTERLACE_MASK_ILK		REG_GENMASK(23, 21) /* ilk+ */
-#define   PIPECONF_INTERLACE_MASK_HSW		REG_GENMASK(22, 21) /* hsw+ */
-#define   PIPECONF_INTERLACE_PF_PD_ILK		REG_FIELD_PREP(PIPECONF_INTERLACE_MASK_ILK, 0)
-#define   PIPECONF_INTERLACE_PF_ID_ILK		REG_FIELD_PREP(PIPECONF_INTERLACE_MASK_ILK, 1)
-#define   PIPECONF_INTERLACE_IF_ID_ILK		REG_FIELD_PREP(PIPECONF_INTERLACE_MASK_ILK, 3)
-#define   PIPECONF_INTERLACE_IF_ID_DBL_ILK	REG_FIELD_PREP(PIPECONF_INTERLACE_MASK_ILK, 4) /* ilk/snb only */
-#define   PIPECONF_INTERLACE_PF_ID_DBL_ILK	REG_FIELD_PREP(PIPECONF_INTERLACE_MASK_ILK, 5) /* ilk/snb only */
-#define   PIPECONF_REFRESH_RATE_ALT_ILK		REG_BIT(20)
-#define   PIPECONF_MSA_TIMING_DELAY_MASK	REG_GENMASK(19, 18) /* ilk/snb/ivb */
-#define   PIPECONF_MSA_TIMING_DELAY(x)		REG_FIELD_PREP(PIPECONF_MSA_TIMING_DELAY_MASK, (x))
-#define   PIPECONF_CXSR_DOWNCLOCK		REG_BIT(16)
-#define   PIPECONF_REFRESH_RATE_ALT_VLV		REG_BIT(14)
-#define   PIPECONF_COLOR_RANGE_SELECT		REG_BIT(13)
-#define   PIPECONF_OUTPUT_COLORSPACE_MASK	REG_GENMASK(12, 11) /* ilk-ivb */
-#define   PIPECONF_OUTPUT_COLORSPACE_RGB	REG_FIELD_PREP(PIPECONF_OUTPUT_COLORSPACE_MASK, 0) /* ilk-ivb */
-#define   PIPECONF_OUTPUT_COLORSPACE_YUV601	REG_FIELD_PREP(PIPECONF_OUTPUT_COLORSPACE_MASK, 1) /* ilk-ivb */
-#define   PIPECONF_OUTPUT_COLORSPACE_YUV709	REG_FIELD_PREP(PIPECONF_OUTPUT_COLORSPACE_MASK, 2) /* ilk-ivb */
-#define   PIPECONF_OUTPUT_COLORSPACE_YUV_HSW	REG_BIT(11) /* hsw only */
-#define   PIPECONF_BPC_MASK			REG_GENMASK(7, 5) /* ctg-ivb */
-#define   PIPECONF_BPC_8			REG_FIELD_PREP(PIPECONF_BPC_MASK, 0)
-#define   PIPECONF_BPC_10			REG_FIELD_PREP(PIPECONF_BPC_MASK, 1)
-#define   PIPECONF_BPC_6			REG_FIELD_PREP(PIPECONF_BPC_MASK, 2)
-#define   PIPECONF_BPC_12			REG_FIELD_PREP(PIPECONF_BPC_MASK, 3)
-#define   PIPECONF_DITHER_EN			REG_BIT(4)
-#define   PIPECONF_DITHER_TYPE_MASK		REG_GENMASK(3, 2)
-#define   PIPECONF_DITHER_TYPE_SP		REG_FIELD_PREP(PIPECONF_DITHER_TYPE_MASK, 0)
-#define   PIPECONF_DITHER_TYPE_ST1		REG_FIELD_PREP(PIPECONF_DITHER_TYPE_MASK, 1)
-#define   PIPECONF_DITHER_TYPE_ST2		REG_FIELD_PREP(PIPECONF_DITHER_TYPE_MASK, 2)
-#define   PIPECONF_DITHER_TYPE_TEMP		REG_FIELD_PREP(PIPECONF_DITHER_TYPE_MASK, 3)
+#define   TRANSCONF_INTERLACE_MASK_ILK		REG_GENMASK(23, 21) /* ilk+ */
+#define   TRANSCONF_INTERLACE_MASK_HSW		REG_GENMASK(22, 21) /* hsw+ */
+#define   TRANSCONF_INTERLACE_PF_PD_ILK		REG_FIELD_PREP(TRANSCONF_INTERLACE_MASK_ILK, 0)
+#define   TRANSCONF_INTERLACE_PF_ID_ILK		REG_FIELD_PREP(TRANSCONF_INTERLACE_MASK_ILK, 1)
+#define   TRANSCONF_INTERLACE_IF_ID_ILK		REG_FIELD_PREP(TRANSCONF_INTERLACE_MASK_ILK, 3)
+#define   TRANSCONF_INTERLACE_IF_ID_DBL_ILK	REG_FIELD_PREP(TRANSCONF_INTERLACE_MASK_ILK, 4) /* ilk/snb only */
+#define   TRANSCONF_INTERLACE_PF_ID_DBL_ILK	REG_FIELD_PREP(TRANSCONF_INTERLACE_MASK_ILK, 5) /* ilk/snb only */
+#define   TRANSCONF_REFRESH_RATE_ALT_ILK		REG_BIT(20)
+#define   TRANSCONF_MSA_TIMING_DELAY_MASK	REG_GENMASK(19, 18) /* ilk/snb/ivb */
+#define   TRANSCONF_MSA_TIMING_DELAY(x)		REG_FIELD_PREP(TRANSCONF_MSA_TIMING_DELAY_MASK, (x))
+#define   TRANSCONF_CXSR_DOWNCLOCK		REG_BIT(16)
+#define   TRANSCONF_REFRESH_RATE_ALT_VLV		REG_BIT(14)
+#define   TRANSCONF_COLOR_RANGE_SELECT		REG_BIT(13)
+#define   TRANSCONF_OUTPUT_COLORSPACE_MASK	REG_GENMASK(12, 11) /* ilk-ivb */
+#define   TRANSCONF_OUTPUT_COLORSPACE_RGB	REG_FIELD_PREP(TRANSCONF_OUTPUT_COLORSPACE_MASK, 0) /* ilk-ivb */
+#define   TRANSCONF_OUTPUT_COLORSPACE_YUV601	REG_FIELD_PREP(TRANSCONF_OUTPUT_COLORSPACE_MASK, 1) /* ilk-ivb */
+#define   TRANSCONF_OUTPUT_COLORSPACE_YUV709	REG_FIELD_PREP(TRANSCONF_OUTPUT_COLORSPACE_MASK, 2) /* ilk-ivb */
+#define   TRANSCONF_OUTPUT_COLORSPACE_YUV_HSW	REG_BIT(11) /* hsw only */
+#define   TRANSCONF_BPC_MASK			REG_GENMASK(7, 5) /* ctg-ivb */
+#define   TRANSCONF_BPC_8			REG_FIELD_PREP(TRANSCONF_BPC_MASK, 0)
+#define   TRANSCONF_BPC_10			REG_FIELD_PREP(TRANSCONF_BPC_MASK, 1)
+#define   TRANSCONF_BPC_6			REG_FIELD_PREP(TRANSCONF_BPC_MASK, 2)
+#define   TRANSCONF_BPC_12			REG_FIELD_PREP(TRANSCONF_BPC_MASK, 3)
+#define   TRANSCONF_DITHER_EN			REG_BIT(4)
+#define   TRANSCONF_DITHER_TYPE_MASK		REG_GENMASK(3, 2)
+#define   TRANSCONF_DITHER_TYPE_SP		REG_FIELD_PREP(TRANSCONF_DITHER_TYPE_MASK, 0)
+#define   TRANSCONF_DITHER_TYPE_ST1		REG_FIELD_PREP(TRANSCONF_DITHER_TYPE_MASK, 1)
+#define   TRANSCONF_DITHER_TYPE_ST2		REG_FIELD_PREP(TRANSCONF_DITHER_TYPE_MASK, 2)
+#define   TRANSCONF_DITHER_TYPE_TEMP		REG_FIELD_PREP(TRANSCONF_DITHER_TYPE_MASK, 3)
 #define _PIPEASTAT		0x70024
 #define   PIPE_FIFO_UNDERRUN_STATUS		(1UL << 31)
 #define   SPRITE1_FLIP_DONE_INT_EN_VLV		(1UL << 30)
@@ -3617,7 +2636,7 @@
 #define PIPE_DSI0_OFFSET	0x7b000
 #define PIPE_DSI1_OFFSET	0x7b800
 
-#define PIPECONF(pipe)		_MMIO_PIPE2(pipe, _PIPEACONF)
+#define TRANSCONF(trans)	_MMIO_PIPE2((trans), _TRANSACONF)
 #define PIPEDSL(pipe)		_MMIO_PIPE2(pipe, _PIPEADSL)
 #define PIPEFRAME(pipe)		_MMIO_PIPE2(pipe, _PIPEAFRAMEHIGH)
 #define PIPEFRAMEPIXEL(pipe)	_MMIO_PIPE2(pipe, _PIPEAFRAMEPIXEL)
@@ -3633,36 +2652,38 @@
 
 #define _PIPE_MISC_A			0x70030
 #define _PIPE_MISC_B			0x71030
-#define   PIPEMISC_YUV420_ENABLE		REG_BIT(27) /* glk+ */
-#define   PIPEMISC_YUV420_MODE_FULL_BLEND	REG_BIT(26) /* glk+ */
-#define   PIPEMISC_HDR_MODE_PRECISION		REG_BIT(23) /* icl+ */
-#define   PIPEMISC_OUTPUT_COLORSPACE_YUV	REG_BIT(11)
-#define   PIPEMISC_PIXEL_ROUNDING_TRUNC		REG_BIT(8) /* tgl+ */
+#define   PIPE_MISC_YUV420_ENABLE		REG_BIT(27) /* glk+ */
+#define   PIPE_MISC_YUV420_MODE_FULL_BLEND	REG_BIT(26) /* glk+ */
+#define   PIPE_MISC_HDR_MODE_PRECISION		REG_BIT(23) /* icl+ */
+#define   PIPE_MISC_OUTPUT_COLORSPACE_YUV	REG_BIT(11)
+#define   PIPE_MISC_PIXEL_ROUNDING_TRUNC		REG_BIT(8) /* tgl+ */
 /*
  * For Display < 13, Bits 5-7 of PIPE MISC represent DITHER BPC with
  * valid values of: 6, 8, 10 BPC.
  * ADLP+, the bits 5-7 represent PORT OUTPUT BPC with valid values of:
  * 6, 8, 10, 12 BPC.
  */
-#define   PIPEMISC_BPC_MASK			REG_GENMASK(7, 5)
-#define   PIPEMISC_BPC_8			REG_FIELD_PREP(PIPEMISC_BPC_MASK, 0)
-#define   PIPEMISC_BPC_10			REG_FIELD_PREP(PIPEMISC_BPC_MASK, 1)
-#define   PIPEMISC_BPC_6			REG_FIELD_PREP(PIPEMISC_BPC_MASK, 2)
-#define   PIPEMISC_BPC_12_ADLP			REG_FIELD_PREP(PIPEMISC_BPC_MASK, 4) /* adlp+ */
-#define   PIPEMISC_DITHER_ENABLE		REG_BIT(4)
-#define   PIPEMISC_DITHER_TYPE_MASK		REG_GENMASK(3, 2)
-#define   PIPEMISC_DITHER_TYPE_SP		REG_FIELD_PREP(PIPEMISC_DITHER_TYPE_MASK, 0)
-#define   PIPEMISC_DITHER_TYPE_ST1		REG_FIELD_PREP(PIPEMISC_DITHER_TYPE_MASK, 1)
-#define   PIPEMISC_DITHER_TYPE_ST2		REG_FIELD_PREP(PIPEMISC_DITHER_TYPE_MASK, 2)
-#define   PIPEMISC_DITHER_TYPE_TEMP		REG_FIELD_PREP(PIPEMISC_DITHER_TYPE_MASK, 3)
-#define PIPEMISC(pipe)			_MMIO_PIPE2(pipe, _PIPE_MISC_A)
+#define   PIPE_MISC_BPC_MASK			REG_GENMASK(7, 5)
+#define   PIPE_MISC_BPC_8			REG_FIELD_PREP(PIPE_MISC_BPC_MASK, 0)
+#define   PIPE_MISC_BPC_10			REG_FIELD_PREP(PIPE_MISC_BPC_MASK, 1)
+#define   PIPE_MISC_BPC_6			REG_FIELD_PREP(PIPE_MISC_BPC_MASK, 2)
+#define   PIPE_MISC_BPC_12_ADLP			REG_FIELD_PREP(PIPE_MISC_BPC_MASK, 4) /* adlp+ */
+#define   PIPE_MISC_DITHER_ENABLE		REG_BIT(4)
+#define   PIPE_MISC_DITHER_TYPE_MASK		REG_GENMASK(3, 2)
+#define   PIPE_MISC_DITHER_TYPE_SP		REG_FIELD_PREP(PIPE_MISC_DITHER_TYPE_MASK, 0)
+#define   PIPE_MISC_DITHER_TYPE_ST1		REG_FIELD_PREP(PIPE_MISC_DITHER_TYPE_MASK, 1)
+#define   PIPE_MISC_DITHER_TYPE_ST2		REG_FIELD_PREP(PIPE_MISC_DITHER_TYPE_MASK, 2)
+#define   PIPE_MISC_DITHER_TYPE_TEMP		REG_FIELD_PREP(PIPE_MISC_DITHER_TYPE_MASK, 3)
+#define PIPE_MISC(pipe)			_MMIO_PIPE(pipe, _PIPE_MISC_A, _PIPE_MISC_B)
 
 #define _PIPE_MISC2_A					0x7002C
 #define _PIPE_MISC2_B					0x7102C
 #define   PIPE_MISC2_BUBBLE_COUNTER_MASK	REG_GENMASK(31, 24)
 #define   PIPE_MISC2_BUBBLE_COUNTER_SCALER_EN	REG_FIELD_PREP(PIPE_MISC2_BUBBLE_COUNTER_MASK, 80)
 #define   PIPE_MISC2_BUBBLE_COUNTER_SCALER_DIS	REG_FIELD_PREP(PIPE_MISC2_BUBBLE_COUNTER_MASK, 20)
-#define PIPE_MISC2(pipe)					_MMIO_PIPE2(pipe, _PIPE_MISC2_A)
+#define   PIPE_MISC2_FLIP_INFO_PLANE_SEL_MASK		REG_GENMASK(2, 0) /* tgl+ */
+#define   PIPE_MISC2_FLIP_INFO_PLANE_SEL(plane_id)	REG_FIELD_PREP(PIPE_MISC2_FLIP_INFO_PLANE_SEL_MASK, (plane_id))
+#define PIPE_MISC2(pipe)		_MMIO_PIPE(pipe, _PIPE_MISC2_A, _PIPE_MISC2_B)
 
 /* Skylake+ pipe bottom (background) color */
 #define _SKL_BOTTOM_COLOR_A		0x70034
@@ -3943,66 +2964,6 @@
 #define I965_CURSOR_MAX_WM	32
 #define I965_CURSOR_DFT_WM	8
 
-/* Watermark register definitions for SKL */
-#define _CUR_WM_A_0		0x70140
-#define _CUR_WM_B_0		0x71140
-#define _CUR_WM_SAGV_A		0x70158
-#define _CUR_WM_SAGV_B		0x71158
-#define _CUR_WM_SAGV_TRANS_A	0x7015C
-#define _CUR_WM_SAGV_TRANS_B	0x7115C
-#define _CUR_WM_TRANS_A		0x70168
-#define _CUR_WM_TRANS_B		0x71168
-#define _PLANE_WM_1_A_0		0x70240
-#define _PLANE_WM_1_B_0		0x71240
-#define _PLANE_WM_2_A_0		0x70340
-#define _PLANE_WM_2_B_0		0x71340
-#define _PLANE_WM_SAGV_1_A	0x70258
-#define _PLANE_WM_SAGV_1_B	0x71258
-#define _PLANE_WM_SAGV_2_A	0x70358
-#define _PLANE_WM_SAGV_2_B	0x71358
-#define _PLANE_WM_SAGV_TRANS_1_A	0x7025C
-#define _PLANE_WM_SAGV_TRANS_1_B	0x7125C
-#define _PLANE_WM_SAGV_TRANS_2_A	0x7035C
-#define _PLANE_WM_SAGV_TRANS_2_B	0x7135C
-#define _PLANE_WM_TRANS_1_A	0x70268
-#define _PLANE_WM_TRANS_1_B	0x71268
-#define _PLANE_WM_TRANS_2_A	0x70368
-#define _PLANE_WM_TRANS_2_B	0x71368
-#define   PLANE_WM_EN		(1 << 31)
-#define   PLANE_WM_IGNORE_LINES	(1 << 30)
-#define   PLANE_WM_LINES_MASK	REG_GENMASK(26, 14)
-#define   PLANE_WM_BLOCKS_MASK	REG_GENMASK(11, 0)
-
-#define _CUR_WM_0(pipe) _PIPE(pipe, _CUR_WM_A_0, _CUR_WM_B_0)
-#define CUR_WM(pipe, level) _MMIO(_CUR_WM_0(pipe) + ((4) * (level)))
-#define CUR_WM_SAGV(pipe) _MMIO_PIPE(pipe, _CUR_WM_SAGV_A, _CUR_WM_SAGV_B)
-#define CUR_WM_SAGV_TRANS(pipe) _MMIO_PIPE(pipe, _CUR_WM_SAGV_TRANS_A, _CUR_WM_SAGV_TRANS_B)
-#define CUR_WM_TRANS(pipe) _MMIO_PIPE(pipe, _CUR_WM_TRANS_A, _CUR_WM_TRANS_B)
-#define _PLANE_WM_1(pipe) _PIPE(pipe, _PLANE_WM_1_A_0, _PLANE_WM_1_B_0)
-#define _PLANE_WM_2(pipe) _PIPE(pipe, _PLANE_WM_2_A_0, _PLANE_WM_2_B_0)
-#define _PLANE_WM_BASE(pipe, plane) \
-	_PLANE(plane, _PLANE_WM_1(pipe), _PLANE_WM_2(pipe))
-#define PLANE_WM(pipe, plane, level) \
-	_MMIO(_PLANE_WM_BASE(pipe, plane) + ((4) * (level)))
-#define _PLANE_WM_SAGV_1(pipe) \
-	_PIPE(pipe, _PLANE_WM_SAGV_1_A, _PLANE_WM_SAGV_1_B)
-#define _PLANE_WM_SAGV_2(pipe) \
-	_PIPE(pipe, _PLANE_WM_SAGV_2_A, _PLANE_WM_SAGV_2_B)
-#define PLANE_WM_SAGV(pipe, plane) \
-	_MMIO(_PLANE(plane, _PLANE_WM_SAGV_1(pipe), _PLANE_WM_SAGV_2(pipe)))
-#define _PLANE_WM_SAGV_TRANS_1(pipe) \
-	_PIPE(pipe, _PLANE_WM_SAGV_TRANS_1_A, _PLANE_WM_SAGV_TRANS_1_B)
-#define _PLANE_WM_SAGV_TRANS_2(pipe) \
-	_PIPE(pipe, _PLANE_WM_SAGV_TRANS_2_A, _PLANE_WM_SAGV_TRANS_2_B)
-#define PLANE_WM_SAGV_TRANS(pipe, plane) \
-	_MMIO(_PLANE(plane, _PLANE_WM_SAGV_TRANS_1(pipe), _PLANE_WM_SAGV_TRANS_2(pipe)))
-#define _PLANE_WM_TRANS_1(pipe) \
-	_PIPE(pipe, _PLANE_WM_TRANS_1_A, _PLANE_WM_TRANS_1_B)
-#define _PLANE_WM_TRANS_2(pipe) \
-	_PIPE(pipe, _PLANE_WM_TRANS_2_A, _PLANE_WM_TRANS_2_B)
-#define PLANE_WM_TRANS(pipe, plane) \
-	_MMIO(_PLANE(plane, _PLANE_WM_TRANS_1(pipe), _PLANE_WM_TRANS_2(pipe)))
-
 /* define the Watermark register on Ironlake */
 #define _WM0_PIPEA_ILK		0x45100
 #define _WM0_PIPEB_ILK		0x45104
@@ -4112,6 +3073,7 @@
 #define   CUR_FBC_EN			REG_BIT(31)
 #define   CUR_FBC_HEIGHT_MASK		REG_GENMASK(7, 0)
 #define   CUR_FBC_HEIGHT(h)		REG_FIELD_PREP(CUR_FBC_HEIGHT_MASK, (h))
+#define _CUR_CHICKEN_A		0x700a4 /* mtl+ */
 #define _CURASURFLIVE		0x700ac /* g4x+ */
 #define _CURBCNTR		0x700c0
 #define _CURBBASE		0x700c4
@@ -4126,6 +3088,7 @@
 #define CURPOS(pipe) _MMIO_CURSOR2(pipe, _CURAPOS)
 #define CURSIZE(pipe) _MMIO_CURSOR2(pipe, _CURASIZE)
 #define CUR_FBC_CTL(pipe) _MMIO_CURSOR2(pipe, _CUR_FBC_CTL_A)
+#define CUR_CHICKEN(pipe) _MMIO_CURSOR2(pipe, _CUR_CHICKEN_A)
 #define CURSURFLIVE(pipe) _MMIO_CURSOR2(pipe, _CURASURFLIVE)
 
 #define CURSOR_A_OFFSET 0x70080
@@ -4257,7 +3220,7 @@
 
 /* Pipe B */
 #define _PIPEBDSL		(DISPLAY_MMIO_BASE(dev_priv) + 0x71000)
-#define _PIPEBCONF		(DISPLAY_MMIO_BASE(dev_priv) + 0x71008)
+#define _TRANSBCONF		(DISPLAY_MMIO_BASE(dev_priv) + 0x71008)
 #define _PIPEBSTAT		(DISPLAY_MMIO_BASE(dev_priv) + 0x71024)
 #define _PIPEBFRAMEHIGH		0x71040
 #define _PIPEBFRAMEPIXEL	0x71044
@@ -4528,6 +3491,7 @@
 #define   SP_CONST_ALPHA_ENABLE		REG_BIT(31)
 #define   SP_CONST_ALPHA_MASK		REG_GENMASK(7, 0)
 #define   SP_CONST_ALPHA(alpha)		REG_FIELD_PREP(SP_CONST_ALPHA_MASK, (alpha))
+#define _SPASURFLIVE		(VLV_DISPLAY_BASE + 0x721ac)
 #define _SPACLRC0		(VLV_DISPLAY_BASE + 0x721d0)
 #define   SP_CONTRAST_MASK		REG_GENMASK(26, 18)
 #define   SP_CONTRAST(x)		REG_FIELD_PREP(SP_CONTRAST_MASK, (x)) /* u3.6 */
@@ -4551,6 +3515,7 @@
 #define _SPBKEYMAXVAL		(VLV_DISPLAY_BASE + 0x722a0)
 #define _SPBTILEOFF		(VLV_DISPLAY_BASE + 0x722a4)
 #define _SPBCONSTALPHA		(VLV_DISPLAY_BASE + 0x722a8)
+#define _SPBSURFLIVE		(VLV_DISPLAY_BASE + 0x722ac)
 #define _SPBCLRC0		(VLV_DISPLAY_BASE + 0x722d0)
 #define _SPBCLRC1		(VLV_DISPLAY_BASE + 0x722d4)
 #define _SPBGAMC		(VLV_DISPLAY_BASE + 0x722e0)
@@ -4571,6 +3536,7 @@
 #define SPKEYMAXVAL(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPAKEYMAXVAL, _SPBKEYMAXVAL)
 #define SPTILEOFF(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPATILEOFF, _SPBTILEOFF)
 #define SPCONSTALPHA(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPACONSTALPHA, _SPBCONSTALPHA)
+#define SPSURFLIVE(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPASURFLIVE, _SPBSURFLIVE)
 #define SPCLRC0(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPACLRC0, _SPBCLRC0)
 #define SPCLRC1(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPACLRC1, _SPBCLRC1)
 #define SPGAMC(pipe, plane_id, i)	_MMIO(_VLV_SPR((pipe), (plane_id), _SPAGAMC, _SPBGAMC) + (5 - (i)) * 4) /* 6 x u0.10 */
@@ -4722,10 +3688,13 @@
 #define _PLANE_KEYVAL_2_A			0x70294
 #define _PLANE_KEYMSK_1_A			0x70198
 #define _PLANE_KEYMSK_2_A			0x70298
-#define  PLANE_KEYMSK_ALPHA_ENABLE		(1 << 31)
+#define   PLANE_KEYMSK_ALPHA_ENABLE		REG_BIT(31)
 #define _PLANE_KEYMAX_1_A			0x701a0
 #define _PLANE_KEYMAX_2_A			0x702a0
-#define  PLANE_KEYMAX_ALPHA(a)			((a) << 24)
+#define   PLANE_KEYMAX_ALPHA_MASK		REG_GENMASK(31, 24)
+#define   PLANE_KEYMAX_ALPHA(a)			REG_FIELD_PREP(PLANE_KEYMAX_ALPHA_MASK, (a))
+#define _PLANE_SURFLIVE_1_A			0x701ac
+#define _PLANE_SURFLIVE_2_A			0x702ac
 #define _PLANE_CC_VAL_1_A			0x701b4
 #define _PLANE_CC_VAL_2_A			0x702b4
 #define _PLANE_AUX_DIST_1_A			0x701c0
@@ -4772,8 +3741,16 @@
 #define   PLANE_COLOR_ALPHA_DISABLE			REG_FIELD_PREP(PLANE_COLOR_ALPHA_MASK, 0)
 #define   PLANE_COLOR_ALPHA_SW_PREMULTIPLY		REG_FIELD_PREP(PLANE_COLOR_ALPHA_MASK, 2)
 #define   PLANE_COLOR_ALPHA_HW_PREMULTIPLY		REG_FIELD_PREP(PLANE_COLOR_ALPHA_MASK, 3)
+#define _PLANE_CHICKEN_1_A			0x7026C /* tgl+ */
+#define _PLANE_CHICKEN_2_A			0x7036C /* tgl+ */
+#define   PLANE_CHICKEN_DISABLE_DPT		REG_BIT(19) /* mtl+ */
 #define _PLANE_BUF_CFG_1_A			0x7027c
 #define _PLANE_BUF_CFG_2_A			0x7037c
+/* skl+: 10 bits, icl+ 11 bits, adlp+ 12 bits */
+#define   PLANE_BUF_END_MASK		REG_GENMASK(27, 16)
+#define   PLANE_BUF_END(end)		REG_FIELD_PREP(PLANE_BUF_END_MASK, (end))
+#define   PLANE_BUF_START_MASK		REG_GENMASK(11, 0)
+#define   PLANE_BUF_START(start)	REG_FIELD_PREP(PLANE_BUF_START_MASK, (start))
 #define _PLANE_NV12_BUF_CFG_1_A		0x70278
 #define _PLANE_NV12_BUF_CFG_2_A		0x70378
 
@@ -4910,28 +3887,19 @@
 #define PLANE_KEYMAX(pipe, plane)	\
 	_MMIO_PLANE(plane, _PLANE_KEYMAX_1(pipe), _PLANE_KEYMAX_2(pipe))
 
-#define _PLANE_BUF_CFG_1_B			0x7127c
-#define _PLANE_BUF_CFG_2_B			0x7137c
-/* skl+: 10 bits, icl+ 11 bits, adlp+ 12 bits */
-#define   PLANE_BUF_END_MASK		REG_GENMASK(27, 16)
-#define   PLANE_BUF_END(end)		REG_FIELD_PREP(PLANE_BUF_END_MASK, (end))
-#define   PLANE_BUF_START_MASK		REG_GENMASK(11, 0)
-#define   PLANE_BUF_START(start)	REG_FIELD_PREP(PLANE_BUF_START_MASK, (start))
-#define _PLANE_BUF_CFG_1(pipe)	\
-	_PIPE(pipe, _PLANE_BUF_CFG_1_A, _PLANE_BUF_CFG_1_B)
-#define _PLANE_BUF_CFG_2(pipe)	\
-	_PIPE(pipe, _PLANE_BUF_CFG_2_A, _PLANE_BUF_CFG_2_B)
-#define PLANE_BUF_CFG(pipe, plane)	\
-	_MMIO_PLANE(plane, _PLANE_BUF_CFG_1(pipe), _PLANE_BUF_CFG_2(pipe))
-
-#define _PLANE_NV12_BUF_CFG_1_B		0x71278
-#define _PLANE_NV12_BUF_CFG_2_B		0x71378
-#define _PLANE_NV12_BUF_CFG_1(pipe)	\
-	_PIPE(pipe, _PLANE_NV12_BUF_CFG_1_A, _PLANE_NV12_BUF_CFG_1_B)
-#define _PLANE_NV12_BUF_CFG_2(pipe)	\
-	_PIPE(pipe, _PLANE_NV12_BUF_CFG_2_A, _PLANE_NV12_BUF_CFG_2_B)
-#define PLANE_NV12_BUF_CFG(pipe, plane)	\
-	_MMIO_PLANE(plane, _PLANE_NV12_BUF_CFG_1(pipe), _PLANE_NV12_BUF_CFG_2(pipe))
+#define _PLANE_SURFLIVE_1_B			0x711ac
+#define _PLANE_SURFLIVE_2_B			0x712ac
+#define _PLANE_SURFLIVE_1(pipe)	_PIPE(pipe, _PLANE_SURFLIVE_1_A, _PLANE_SURFLIVE_1_B)
+#define _PLANE_SURFLIVE_2(pipe)	_PIPE(pipe, _PLANE_SURFLIVE_2_A, _PLANE_SURFLIVE_2_B)
+#define PLANE_SURFLIVE(pipe, plane)	\
+	_MMIO_PLANE(plane, _PLANE_SURFLIVE_1(pipe), _PLANE_SURFLIVE_2(pipe))
+
+#define _PLANE_CHICKEN_1_B			0x7126c
+#define _PLANE_CHICKEN_2_B			0x7136c
+#define _PLANE_CHICKEN_1(pipe)	_PIPE(pipe, _PLANE_CHICKEN_1_A, _PLANE_CHICKEN_1_B)
+#define _PLANE_CHICKEN_2(pipe)	_PIPE(pipe, _PLANE_CHICKEN_2_A, _PLANE_CHICKEN_2_B)
+#define PLANE_CHICKEN(pipe, plane) \
+	_MMIO_PLANE(plane, _PLANE_CHICKEN_1(pipe), _PLANE_CHICKEN_2(pipe))
 
 #define _PLANE_AUX_DIST_1_B		0x711c0
 #define _PLANE_AUX_DIST_2_B		0x712c0
@@ -4970,56 +3938,6 @@
 #define PLANE_COLOR_CTL(pipe, plane)	\
 	_MMIO_PLANE(plane, _PLANE_COLOR_CTL_1(pipe), _PLANE_COLOR_CTL_2(pipe))
 
-#define _SEL_FETCH_PLANE_BASE_1_A		0x70890
-#define _SEL_FETCH_PLANE_BASE_2_A		0x708B0
-#define _SEL_FETCH_PLANE_BASE_3_A		0x708D0
-#define _SEL_FETCH_PLANE_BASE_4_A		0x708F0
-#define _SEL_FETCH_PLANE_BASE_5_A		0x70920
-#define _SEL_FETCH_PLANE_BASE_6_A		0x70940
-#define _SEL_FETCH_PLANE_BASE_7_A		0x70960
-#define _SEL_FETCH_PLANE_BASE_CUR_A		0x70880
-#define _SEL_FETCH_PLANE_BASE_1_B		0x71890
-
-#define _SEL_FETCH_PLANE_BASE_A(plane) _PICK(plane, \
-					     _SEL_FETCH_PLANE_BASE_1_A, \
-					     _SEL_FETCH_PLANE_BASE_2_A, \
-					     _SEL_FETCH_PLANE_BASE_3_A, \
-					     _SEL_FETCH_PLANE_BASE_4_A, \
-					     _SEL_FETCH_PLANE_BASE_5_A, \
-					     _SEL_FETCH_PLANE_BASE_6_A, \
-					     _SEL_FETCH_PLANE_BASE_7_A, \
-					     _SEL_FETCH_PLANE_BASE_CUR_A)
-#define _SEL_FETCH_PLANE_BASE_1(pipe) _PIPE(pipe, _SEL_FETCH_PLANE_BASE_1_A, _SEL_FETCH_PLANE_BASE_1_B)
-#define _SEL_FETCH_PLANE_BASE(pipe, plane) (_SEL_FETCH_PLANE_BASE_1(pipe) - \
-					    _SEL_FETCH_PLANE_BASE_1_A + \
-					    _SEL_FETCH_PLANE_BASE_A(plane))
-
-#define _SEL_FETCH_PLANE_CTL_1_A		0x70890
-#define PLANE_SEL_FETCH_CTL(pipe, plane) _MMIO(_SEL_FETCH_PLANE_BASE(pipe, plane) + \
-					       _SEL_FETCH_PLANE_CTL_1_A - \
-					       _SEL_FETCH_PLANE_BASE_1_A)
-#define PLANE_SEL_FETCH_CTL_ENABLE		REG_BIT(31)
-
-#define _SEL_FETCH_PLANE_POS_1_A		0x70894
-#define PLANE_SEL_FETCH_POS(pipe, plane) _MMIO(_SEL_FETCH_PLANE_BASE(pipe, plane) + \
-					       _SEL_FETCH_PLANE_POS_1_A - \
-					       _SEL_FETCH_PLANE_BASE_1_A)
-
-#define _SEL_FETCH_PLANE_SIZE_1_A		0x70898
-#define PLANE_SEL_FETCH_SIZE(pipe, plane) _MMIO(_SEL_FETCH_PLANE_BASE(pipe, plane) + \
-						_SEL_FETCH_PLANE_SIZE_1_A - \
-						_SEL_FETCH_PLANE_BASE_1_A)
-
-#define _SEL_FETCH_PLANE_OFFSET_1_A		0x7089C
-#define PLANE_SEL_FETCH_OFFSET(pipe, plane) _MMIO(_SEL_FETCH_PLANE_BASE(pipe, plane) + \
-						  _SEL_FETCH_PLANE_OFFSET_1_A - \
-						  _SEL_FETCH_PLANE_BASE_1_A)
-
-/* SKL new cursor registers */
-#define _CUR_BUF_CFG_A				0x7017c
-#define _CUR_BUF_CFG_B				0x7117c
-#define CUR_BUF_CFG(pipe)	_MMIO_PIPE(pipe, _CUR_BUF_CFG_A, _CUR_BUF_CFG_B)
-
 /* VBIOS regs */
 #define VGACNTRL		_MMIO(0x71400)
 # define VGA_DISP_DISABLE			(1 << 31)
@@ -5049,14 +3967,6 @@
 #define  RR_HW_LOW_POWER_FRAMES_MASK    0xff
 #define  RR_HW_HIGH_POWER_FRAMES_MASK   0xff00
 
-#define FDI_PLL_BIOS_0  _MMIO(0x46000)
-#define  FDI_PLL_FB_CLOCK_MASK  0xff
-#define FDI_PLL_BIOS_1  _MMIO(0x46004)
-#define FDI_PLL_BIOS_2  _MMIO(0x46008)
-#define DISPLAY_PORT_PLL_BIOS_0         _MMIO(0x4600c)
-#define DISPLAY_PORT_PLL_BIOS_1         _MMIO(0x46010)
-#define DISPLAY_PORT_PLL_BIOS_2         _MMIO(0x46014)
-
 #define PCH_3DCGDIS0		_MMIO(0x46020)
 # define MARIUNIT_CLOCK_GATE_DISABLE		(1 << 18)
 # define SVSMUNIT_CLOCK_GATE_DISABLE		(1 << 1)
@@ -5064,12 +3974,6 @@
 #define PCH_3DCGDIS1		_MMIO(0x46024)
 # define VFMUNIT_CLOCK_GATE_DISABLE		(1 << 11)
 
-#define FDI_PLL_FREQ_CTL        _MMIO(0x46030)
-#define  FDI_PLL_FREQ_CHANGE_REQUEST    (1 << 24)
-#define  FDI_PLL_FREQ_LOCK_LIMIT_MASK   0xfff00
-#define  FDI_PLL_FREQ_DISABLE_COUNT_LIMIT_MASK  0xff
-
-
 #define _PIPEA_DATA_M1		0x60030
 #define _PIPEA_DATA_N1		0x60034
 #define _PIPEA_DATA_M2		0x60038
@@ -5434,6 +4338,7 @@
 #define  GEN8_PIPE_CDCLK_CRC_DONE	(1 << 28)
 #define  XELPD_PIPE_SOFT_UNDERRUN	(1 << 22)
 #define  XELPD_PIPE_HARD_UNDERRUN	(1 << 21)
+#define  GEN12_PIPE_VBLANK_UNMOD	(1 << 19)
 #define  GEN8_PIPE_CURSOR_FAULT		(1 << 10)
 #define  GEN8_PIPE_SPRITE_FAULT		(1 << 9)
 #define  GEN8_PIPE_PRIMARY_FAULT	(1 << 8)
@@ -5621,6 +4526,7 @@
 #define  KVM_CONFIG_CHANGE_NOTIFICATION_SELECT	(1 << 14)
 
 #define CHICKEN_MISC_2		_MMIO(0x42084)
+#define  CHICKEN_MISC_DISABLE_DPT	REG_BIT(30) /* adl,dg2 */
 #define  KBL_ARB_FILL_SPARE_14	REG_BIT(14)
 #define  KBL_ARB_FILL_SPARE_13	REG_BIT(13)
 #define  GLK_CL2_PWR_DOWN	(1 << 12)
@@ -5691,30 +4597,6 @@
 #define  DISP_DATA_PARTITION_5_6	(1 << 6)
 #define  DISP_IPC_ENABLE		(1 << 3)
 
-/*
- * The below are numbered starting from "S1" on gen11/gen12, but starting
- * with display 13, the bspec switches to a 0-based numbering scheme
- * (although the addresses stay the same so new S0 = old S1, new S1 = old S2).
- * We'll just use the 0-based numbering here for all platforms since it's the
- * way things will be named by the hardware team going forward, plus it's more
- * consistent with how most of the rest of our registers are named.
- */
-#define _DBUF_CTL_S0				0x45008
-#define _DBUF_CTL_S1				0x44FE8
-#define _DBUF_CTL_S2				0x44300
-#define _DBUF_CTL_S3				0x44304
-#define DBUF_CTL_S(slice)			_MMIO(_PICK(slice, \
-							    _DBUF_CTL_S0, \
-							    _DBUF_CTL_S1, \
-							    _DBUF_CTL_S2, \
-							    _DBUF_CTL_S3))
-#define  DBUF_POWER_REQUEST			REG_BIT(31)
-#define  DBUF_POWER_STATE			REG_BIT(30)
-#define  DBUF_TRACKER_STATE_SERVICE_MASK	REG_GENMASK(23, 19)
-#define  DBUF_TRACKER_STATE_SERVICE(x)		REG_FIELD_PREP(DBUF_TRACKER_STATE_SERVICE_MASK, x)
-#define  DBUF_MIN_TRACKER_STATE_SERVICE_MASK	REG_GENMASK(18, 16) /* ADL-P+ */
-#define  DBUF_MIN_TRACKER_STATE_SERVICE(x)		REG_FIELD_PREP(DBUF_MIN_TRACKER_STATE_SERVICE_MASK, x) /* ADL-P+ */
-
 #define GEN7_MSG_CTL	_MMIO(0x45010)
 #define  WAIT_FOR_PCH_RESET_ACK		(1 << 1)
 #define  WAIT_FOR_PCH_FLR_ACK		(1 << 0)
@@ -6258,12 +5140,6 @@
 #define  LPT_PWM_GRANULARITY		(1 << 5)
 #define  DPLS_EDP_PPS_FIX_DIS		(1 << 0)
 
-#define _FDI_RXA_CHICKEN        0xc200c
-#define _FDI_RXB_CHICKEN        0xc2010
-#define  FDI_RX_PHASE_SYNC_POINTER_OVR	(1 << 1)
-#define  FDI_RX_PHASE_SYNC_POINTER_EN	(1 << 0)
-#define FDI_RX_CHICKEN(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_CHICKEN, _FDI_RXB_CHICKEN)
-
 #define SOUTH_DSPCLK_GATE_D	_MMIO(0xc2020)
 #define  PCH_GMBUSUNIT_CLOCK_GATE_DISABLE (1 << 31)
 #define  PCH_DPLUNIT_CLOCK_GATE_DISABLE (1 << 30)
@@ -6273,130 +5149,6 @@
 #define  CNP_PWM_CGE_GATING_DISABLE (1 << 13)
 #define  PCH_LP_PARTITION_LEVEL_DISABLE  (1 << 12)
 
-/* CPU: FDI_TX */
-#define _FDI_TXA_CTL            0x60100
-#define _FDI_TXB_CTL            0x61100
-#define FDI_TX_CTL(pipe)	_MMIO_PIPE(pipe, _FDI_TXA_CTL, _FDI_TXB_CTL)
-#define  FDI_TX_DISABLE         (0 << 31)
-#define  FDI_TX_ENABLE          (1 << 31)
-#define  FDI_LINK_TRAIN_PATTERN_1       (0 << 28)
-#define  FDI_LINK_TRAIN_PATTERN_2       (1 << 28)
-#define  FDI_LINK_TRAIN_PATTERN_IDLE    (2 << 28)
-#define  FDI_LINK_TRAIN_NONE            (3 << 28)
-#define  FDI_LINK_TRAIN_VOLTAGE_0_4V    (0 << 25)
-#define  FDI_LINK_TRAIN_VOLTAGE_0_6V    (1 << 25)
-#define  FDI_LINK_TRAIN_VOLTAGE_0_8V    (2 << 25)
-#define  FDI_LINK_TRAIN_VOLTAGE_1_2V    (3 << 25)
-#define  FDI_LINK_TRAIN_PRE_EMPHASIS_NONE (0 << 22)
-#define  FDI_LINK_TRAIN_PRE_EMPHASIS_1_5X (1 << 22)
-#define  FDI_LINK_TRAIN_PRE_EMPHASIS_2X   (2 << 22)
-#define  FDI_LINK_TRAIN_PRE_EMPHASIS_3X   (3 << 22)
-/* ILK always use 400mV 0dB for voltage swing and pre-emphasis level.
-   SNB has different settings. */
-/* SNB A-stepping */
-#define  FDI_LINK_TRAIN_400MV_0DB_SNB_A		(0x38 << 22)
-#define  FDI_LINK_TRAIN_400MV_6DB_SNB_A		(0x02 << 22)
-#define  FDI_LINK_TRAIN_600MV_3_5DB_SNB_A	(0x01 << 22)
-#define  FDI_LINK_TRAIN_800MV_0DB_SNB_A		(0x0 << 22)
-/* SNB B-stepping */
-#define  FDI_LINK_TRAIN_400MV_0DB_SNB_B		(0x0 << 22)
-#define  FDI_LINK_TRAIN_400MV_6DB_SNB_B		(0x3a << 22)
-#define  FDI_LINK_TRAIN_600MV_3_5DB_SNB_B	(0x39 << 22)
-#define  FDI_LINK_TRAIN_800MV_0DB_SNB_B		(0x38 << 22)
-#define  FDI_LINK_TRAIN_VOL_EMP_MASK		(0x3f << 22)
-#define  FDI_DP_PORT_WIDTH_SHIFT		19
-#define  FDI_DP_PORT_WIDTH_MASK			(7 << FDI_DP_PORT_WIDTH_SHIFT)
-#define  FDI_DP_PORT_WIDTH(width)           (((width) - 1) << FDI_DP_PORT_WIDTH_SHIFT)
-#define  FDI_TX_ENHANCE_FRAME_ENABLE    (1 << 18)
-/* Ironlake: hardwired to 1 */
-#define  FDI_TX_PLL_ENABLE              (1 << 14)
-
-/* Ivybridge has different bits for lolz */
-#define  FDI_LINK_TRAIN_PATTERN_1_IVB       (0 << 8)
-#define  FDI_LINK_TRAIN_PATTERN_2_IVB       (1 << 8)
-#define  FDI_LINK_TRAIN_PATTERN_IDLE_IVB    (2 << 8)
-#define  FDI_LINK_TRAIN_NONE_IVB            (3 << 8)
-
-/* both Tx and Rx */
-#define  FDI_COMPOSITE_SYNC		(1 << 11)
-#define  FDI_LINK_TRAIN_AUTO		(1 << 10)
-#define  FDI_SCRAMBLING_ENABLE          (0 << 7)
-#define  FDI_SCRAMBLING_DISABLE         (1 << 7)
-
-/* FDI_RX, FDI_X is hard-wired to Transcoder_X */
-#define _FDI_RXA_CTL             0xf000c
-#define _FDI_RXB_CTL             0xf100c
-#define FDI_RX_CTL(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_CTL, _FDI_RXB_CTL)
-#define  FDI_RX_ENABLE          (1 << 31)
-/* train, dp width same as FDI_TX */
-#define  FDI_FS_ERRC_ENABLE		(1 << 27)
-#define  FDI_FE_ERRC_ENABLE		(1 << 26)
-#define  FDI_RX_POLARITY_REVERSED_LPT	(1 << 16)
-#define  FDI_8BPC                       (0 << 16)
-#define  FDI_10BPC                      (1 << 16)
-#define  FDI_6BPC                       (2 << 16)
-#define  FDI_12BPC                      (3 << 16)
-#define  FDI_RX_LINK_REVERSAL_OVERRIDE  (1 << 15)
-#define  FDI_DMI_LINK_REVERSE_MASK      (1 << 14)
-#define  FDI_RX_PLL_ENABLE              (1 << 13)
-#define  FDI_FS_ERR_CORRECT_ENABLE      (1 << 11)
-#define  FDI_FE_ERR_CORRECT_ENABLE      (1 << 10)
-#define  FDI_FS_ERR_REPORT_ENABLE       (1 << 9)
-#define  FDI_FE_ERR_REPORT_ENABLE       (1 << 8)
-#define  FDI_RX_ENHANCE_FRAME_ENABLE    (1 << 6)
-#define  FDI_PCDCLK	                (1 << 4)
-/* CPT */
-#define  FDI_AUTO_TRAINING			(1 << 10)
-#define  FDI_LINK_TRAIN_PATTERN_1_CPT		(0 << 8)
-#define  FDI_LINK_TRAIN_PATTERN_2_CPT		(1 << 8)
-#define  FDI_LINK_TRAIN_PATTERN_IDLE_CPT	(2 << 8)
-#define  FDI_LINK_TRAIN_NORMAL_CPT		(3 << 8)
-#define  FDI_LINK_TRAIN_PATTERN_MASK_CPT	(3 << 8)
-
-#define _FDI_RXA_MISC			0xf0010
-#define _FDI_RXB_MISC			0xf1010
-#define  FDI_RX_PWRDN_LANE1_MASK	(3 << 26)
-#define  FDI_RX_PWRDN_LANE1_VAL(x)	((x) << 26)
-#define  FDI_RX_PWRDN_LANE0_MASK	(3 << 24)
-#define  FDI_RX_PWRDN_LANE0_VAL(x)	((x) << 24)
-#define  FDI_RX_TP1_TO_TP2_48		(2 << 20)
-#define  FDI_RX_TP1_TO_TP2_64		(3 << 20)
-#define  FDI_RX_FDI_DELAY_90		(0x90 << 0)
-#define FDI_RX_MISC(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_MISC, _FDI_RXB_MISC)
-
-#define _FDI_RXA_TUSIZE1        0xf0030
-#define _FDI_RXA_TUSIZE2        0xf0038
-#define _FDI_RXB_TUSIZE1        0xf1030
-#define _FDI_RXB_TUSIZE2        0xf1038
-#define FDI_RX_TUSIZE1(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_TUSIZE1, _FDI_RXB_TUSIZE1)
-#define FDI_RX_TUSIZE2(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_TUSIZE2, _FDI_RXB_TUSIZE2)
-
-/* FDI_RX interrupt register format */
-#define FDI_RX_INTER_LANE_ALIGN         (1 << 10)
-#define FDI_RX_SYMBOL_LOCK              (1 << 9) /* train 2 */
-#define FDI_RX_BIT_LOCK                 (1 << 8) /* train 1 */
-#define FDI_RX_TRAIN_PATTERN_2_FAIL     (1 << 7)
-#define FDI_RX_FS_CODE_ERR              (1 << 6)
-#define FDI_RX_FE_CODE_ERR              (1 << 5)
-#define FDI_RX_SYMBOL_ERR_RATE_ABOVE    (1 << 4)
-#define FDI_RX_HDCP_LINK_FAIL           (1 << 3)
-#define FDI_RX_PIXEL_FIFO_OVERFLOW      (1 << 2)
-#define FDI_RX_CROSS_CLOCK_OVERFLOW     (1 << 1)
-#define FDI_RX_SYMBOL_QUEUE_OVERFLOW    (1 << 0)
-
-#define _FDI_RXA_IIR            0xf0014
-#define _FDI_RXA_IMR            0xf0018
-#define _FDI_RXB_IIR            0xf1014
-#define _FDI_RXB_IMR            0xf1018
-#define FDI_RX_IIR(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_IIR, _FDI_RXB_IIR)
-#define FDI_RX_IMR(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_IMR, _FDI_RXB_IMR)
-
-#define FDI_PLL_CTL_1           _MMIO(0xfe000)
-#define FDI_PLL_CTL_2           _MMIO(0xfe004)
-
-#define PCH_LVDS	_MMIO(0xe1180)
-#define  LVDS_DETECTED	(1 << 1)
-
 #define _PCH_DP_B		0xe4100
 #define PCH_DP_B		_MMIO(_PCH_DP_B)
 #define _PCH_DPB_AUX_CH_CTL	0xe4110
@@ -7217,21 +5969,23 @@ enum skl_power_gate {
 							ADLS_DPCLKA_DDIK_SEL_MASK)
 
 /* ICL PLL */
-#define DPLL0_ENABLE		0x46010
-#define DPLL1_ENABLE		0x46014
+#define _DPLL0_ENABLE		0x46010
+#define _DPLL1_ENABLE		0x46014
 #define _ADLS_DPLL2_ENABLE	0x46018
 #define _ADLS_DPLL3_ENABLE	0x46030
-#define  PLL_ENABLE		(1 << 31)
-#define  PLL_LOCK		(1 << 30)
-#define  PLL_POWER_ENABLE	(1 << 27)
-#define  PLL_POWER_STATE	(1 << 26)
-#define ICL_DPLL_ENABLE(pll)	_MMIO_PLL3(pll, DPLL0_ENABLE, DPLL1_ENABLE, \
-					   _ADLS_DPLL2_ENABLE, _ADLS_DPLL3_ENABLE)
+#define   PLL_ENABLE		REG_BIT(31)
+#define   PLL_LOCK		REG_BIT(30)
+#define   PLL_POWER_ENABLE	REG_BIT(27)
+#define   PLL_POWER_STATE	REG_BIT(26)
+#define ICL_DPLL_ENABLE(pll)	_MMIO(_PICK_EVEN_2RANGES(pll, 3,			\
+							_DPLL0_ENABLE, _DPLL1_ENABLE,	\
+							_ADLS_DPLL3_ENABLE, _ADLS_DPLL3_ENABLE))
 
 #define _DG2_PLL3_ENABLE	0x4601C
 
-#define DG2_PLL_ENABLE(pll) _MMIO_PLL3(pll, DPLL0_ENABLE, DPLL1_ENABLE, \
-				       _ADLS_DPLL2_ENABLE, _DG2_PLL3_ENABLE)
+#define DG2_PLL_ENABLE(pll)	_MMIO(_PICK_EVEN_2RANGES(pll, 3,			\
+							_DPLL0_ENABLE, _DPLL1_ENABLE,	\
+							_DG2_PLL3_ENABLE, _DG2_PLL3_ENABLE))
 
 #define TBT_PLL_ENABLE		_MMIO(0x46020)
 
@@ -7239,13 +5993,14 @@ enum skl_power_gate {
 #define _MG_PLL2_ENABLE		0x46034
 #define _MG_PLL3_ENABLE		0x46038
 #define _MG_PLL4_ENABLE		0x4603C
-/* Bits are the same as DPLL0_ENABLE */
+/* Bits are the same as _DPLL0_ENABLE */
 #define MG_PLL_ENABLE(tc_port)	_MMIO_PORT((tc_port), _MG_PLL1_ENABLE, \
 					   _MG_PLL2_ENABLE)
 
 /* DG1 PLL */
-#define DG1_DPLL_ENABLE(pll)    _MMIO_PLL3(pll, DPLL0_ENABLE, DPLL1_ENABLE, \
-					   _MG_PLL1_ENABLE, _MG_PLL2_ENABLE)
+#define DG1_DPLL_ENABLE(pll)    _MMIO(_PICK_EVEN_2RANGES(pll, 2,			\
+							_DPLL0_ENABLE, _DPLL1_ENABLE,	\
+							_MG_PLL1_ENABLE, _MG_PLL2_ENABLE))
 
 /* ADL-P Type C PLL */
 #define PORTTC1_PLL_ENABLE	0x46038
@@ -7305,9 +6060,9 @@ enum skl_power_gate {
 #define _TGL_DPLL0_CFGCR0		0x164284
 #define _TGL_DPLL1_CFGCR0		0x16428C
 #define _TGL_TBTPLL_CFGCR0		0x16429C
-#define TGL_DPLL_CFGCR0(pll)		_MMIO_PLL3(pll, _TGL_DPLL0_CFGCR0, \
-						  _TGL_DPLL1_CFGCR0, \
-						  _TGL_TBTPLL_CFGCR0)
+#define TGL_DPLL_CFGCR0(pll)		_MMIO(_PICK_EVEN_2RANGES(pll, 2,		\
+					      _TGL_DPLL0_CFGCR0, _TGL_DPLL1_CFGCR0,	\
+					      _TGL_TBTPLL_CFGCR0, _TGL_TBTPLL_CFGCR0))
 #define RKL_DPLL_CFGCR0(pll)		_MMIO_PLL(pll, _TGL_DPLL0_CFGCR0, \
 						  _TGL_DPLL1_CFGCR0)
 
@@ -7320,40 +6075,36 @@ enum skl_power_gate {
 #define _TGL_DPLL0_CFGCR1		0x164288
 #define _TGL_DPLL1_CFGCR1		0x164290
 #define _TGL_TBTPLL_CFGCR1		0x1642A0
-#define TGL_DPLL_CFGCR1(pll)		_MMIO_PLL3(pll, _TGL_DPLL0_CFGCR1, \
-						   _TGL_DPLL1_CFGCR1, \
-						   _TGL_TBTPLL_CFGCR1)
+#define TGL_DPLL_CFGCR1(pll)		_MMIO(_PICK_EVEN_2RANGES(pll, 2,		\
+					      _TGL_DPLL0_CFGCR1, _TGL_DPLL1_CFGCR1,	\
+					      _TGL_TBTPLL_CFGCR1, _TGL_TBTPLL_CFGCR1))
 #define RKL_DPLL_CFGCR1(pll)		_MMIO_PLL(pll, _TGL_DPLL0_CFGCR1, \
 						  _TGL_DPLL1_CFGCR1)
 
 #define _DG1_DPLL2_CFGCR0		0x16C284
 #define _DG1_DPLL3_CFGCR0		0x16C28C
-#define DG1_DPLL_CFGCR0(pll)		_MMIO_PLL3(pll, _TGL_DPLL0_CFGCR0, \
-						   _TGL_DPLL1_CFGCR0, \
-						   _DG1_DPLL2_CFGCR0, \
-						   _DG1_DPLL3_CFGCR0)
+#define DG1_DPLL_CFGCR0(pll)		_MMIO(_PICK_EVEN_2RANGES(pll, 2,		\
+					      _TGL_DPLL0_CFGCR0, _TGL_DPLL1_CFGCR0,	\
+					      _DG1_DPLL2_CFGCR0, _DG1_DPLL3_CFGCR0))
 
 #define _DG1_DPLL2_CFGCR1               0x16C288
 #define _DG1_DPLL3_CFGCR1               0x16C290
-#define DG1_DPLL_CFGCR1(pll)            _MMIO_PLL3(pll, _TGL_DPLL0_CFGCR1, \
-						   _TGL_DPLL1_CFGCR1, \
-						   _DG1_DPLL2_CFGCR1, \
-						   _DG1_DPLL3_CFGCR1)
+#define DG1_DPLL_CFGCR1(pll)            _MMIO(_PICK_EVEN_2RANGES(pll, 2,		\
+					      _TGL_DPLL0_CFGCR1, _TGL_DPLL1_CFGCR1,	\
+					      _DG1_DPLL2_CFGCR1, _DG1_DPLL3_CFGCR1))
 
 /* For ADL-S DPLL4_CFGCR0/1 are used to control DPLL2 */
-#define _ADLS_DPLL3_CFGCR0		0x1642C0
 #define _ADLS_DPLL4_CFGCR0		0x164294
-#define ADLS_DPLL_CFGCR0(pll)		_MMIO_PLL3(pll, _TGL_DPLL0_CFGCR0, \
-						   _TGL_DPLL1_CFGCR0, \
-						   _ADLS_DPLL4_CFGCR0, \
-						   _ADLS_DPLL3_CFGCR0)
+#define _ADLS_DPLL3_CFGCR0		0x1642C0
+#define ADLS_DPLL_CFGCR0(pll)		_MMIO(_PICK_EVEN_2RANGES(pll, 2,		\
+					      _TGL_DPLL0_CFGCR0, _TGL_DPLL1_CFGCR0,	\
+					      _ADLS_DPLL4_CFGCR0, _ADLS_DPLL3_CFGCR0))
 
-#define _ADLS_DPLL3_CFGCR1		0x1642C4
 #define _ADLS_DPLL4_CFGCR1		0x164298
-#define ADLS_DPLL_CFGCR1(pll)		_MMIO_PLL3(pll, _TGL_DPLL0_CFGCR1, \
-						   _TGL_DPLL1_CFGCR1, \
-						   _ADLS_DPLL4_CFGCR1, \
-						   _ADLS_DPLL3_CFGCR1)
+#define _ADLS_DPLL3_CFGCR1		0x1642C4
+#define ADLS_DPLL_CFGCR1(pll)		_MMIO(_PICK_EVEN_2RANGES(pll, 2,		\
+					      _TGL_DPLL0_CFGCR1, _TGL_DPLL1_CFGCR1,	\
+					      _ADLS_DPLL4_CFGCR1, _ADLS_DPLL3_CFGCR1))
 
 /* BXT display engine PLL */
 #define BXT_DE_PLL_CTL			_MMIO(0x6d000)
@@ -7373,6 +6124,8 @@ enum skl_power_gate {
 #define  DC_STATE_DISABLE		0
 #define  DC_STATE_EN_DC3CO		REG_BIT(30)
 #define  DC_STATE_DC3CO_STATUS		REG_BIT(29)
+#define  HOLD_PHY_CLKREQ_PG1_LATCH	REG_BIT(21)
+#define  HOLD_PHY_PG1_LATCH		REG_BIT(20)
 #define  DC_STATE_EN_UPTO_DC5		(1 << 0)
 #define  DC_STATE_EN_DC9		(1 << 3)
 #define  DC_STATE_EN_UPTO_DC6		(2 << 0)
@@ -7596,8 +6349,8 @@ enum skl_power_gate {
 
 #define _PLANE_CSC_RY_GY_1(pipe)	_PIPE(pipe, _PLANE_CSC_RY_GY_1_A, \
 					      _PLANE_CSC_RY_GY_1_B)
-#define _PLANE_CSC_RY_GY_2(pipe)	_PIPE(pipe, _PLANE_INPUT_CSC_RY_GY_2_A, \
-					      _PLANE_INPUT_CSC_RY_GY_2_B)
+#define _PLANE_CSC_RY_GY_2(pipe)	_PIPE(pipe, _PLANE_CSC_RY_GY_2_A, \
+					      _PLANE_CSC_RY_GY_2_B)
 #define PLANE_CSC_COEFF(pipe, plane, index)	_MMIO_PLANE(plane, \
 							    _PLANE_CSC_RY_GY_1(pipe) +  (index) * 4, \
 							    _PLANE_CSC_RY_GY_2(pipe) + (index) * 4)
@@ -7682,47 +6435,29 @@ enum skl_power_gate {
 #define  GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_SHIFT	12
 #define  GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_MASK	(0xf << 12)
 
+/* g4x+, except vlv/chv! */
 #define _PIPE_FRMTMSTMP_A		0x70048
+#define _PIPE_FRMTMSTMP_B		0x71048
 #define PIPE_FRMTMSTMP(pipe)		\
-			_MMIO_PIPE2(pipe, _PIPE_FRMTMSTMP_A)
-
-/* Display Stream Splitter Control */
-#define DSS_CTL1				_MMIO(0x67400)
-#define  SPLITTER_ENABLE			(1 << 31)
-#define  JOINER_ENABLE				(1 << 30)
-#define  DUAL_LINK_MODE_INTERLEAVE		(1 << 24)
-#define  DUAL_LINK_MODE_FRONTBACK		(0 << 24)
-#define  OVERLAP_PIXELS_MASK			(0xf << 16)
-#define  OVERLAP_PIXELS(pixels)			((pixels) << 16)
-#define  LEFT_DL_BUF_TARGET_DEPTH_MASK		(0xfff << 0)
-#define  LEFT_DL_BUF_TARGET_DEPTH(pixels)	((pixels) << 0)
-#define  MAX_DL_BUFFER_TARGET_DEPTH		0x5a0
-
-#define DSS_CTL2				_MMIO(0x67404)
-#define  LEFT_BRANCH_VDSC_ENABLE		(1 << 31)
-#define  RIGHT_BRANCH_VDSC_ENABLE		(1 << 15)
-#define  RIGHT_DL_BUF_TARGET_DEPTH_MASK		(0xfff << 0)
-#define  RIGHT_DL_BUF_TARGET_DEPTH(pixels)	((pixels) << 0)
-
-#define _ICL_PIPE_DSS_CTL1_PB			0x78200
-#define _ICL_PIPE_DSS_CTL1_PC			0x78400
-#define ICL_PIPE_DSS_CTL1(pipe)			_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_PIPE_DSS_CTL1_PB, \
-							   _ICL_PIPE_DSS_CTL1_PC)
-#define  BIG_JOINER_ENABLE			(1 << 29)
-#define  MASTER_BIG_JOINER_ENABLE		(1 << 28)
-#define  VGA_CENTERING_ENABLE			(1 << 27)
-#define  SPLITTER_CONFIGURATION_MASK		REG_GENMASK(26, 25)
-#define  SPLITTER_CONFIGURATION_2_SEGMENT	REG_FIELD_PREP(SPLITTER_CONFIGURATION_MASK, 0)
-#define  SPLITTER_CONFIGURATION_4_SEGMENT	REG_FIELD_PREP(SPLITTER_CONFIGURATION_MASK, 1)
-#define  UNCOMPRESSED_JOINER_MASTER		(1 << 21)
-#define  UNCOMPRESSED_JOINER_SLAVE		(1 << 20)
-
-#define _ICL_PIPE_DSS_CTL2_PB			0x78204
-#define _ICL_PIPE_DSS_CTL2_PC			0x78404
-#define ICL_PIPE_DSS_CTL2(pipe)			_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_PIPE_DSS_CTL2_PB, \
-							   _ICL_PIPE_DSS_CTL2_PC)
+	_MMIO_PIPE(pipe, _PIPE_FRMTMSTMP_A, _PIPE_FRMTMSTMP_B)
+
+/* g4x+, except vlv/chv! */
+#define _PIPE_FLIPTMSTMP_A		0x7004C
+#define _PIPE_FLIPTMSTMP_B		0x7104C
+#define PIPE_FLIPTMSTMP(pipe)		\
+	_MMIO_PIPE(pipe, _PIPE_FLIPTMSTMP_A, _PIPE_FLIPTMSTMP_B)
+
+/* tgl+ */
+#define _PIPE_FLIPDONETMSTMP_A		0x70054
+#define _PIPE_FLIPDONETMSTMP_B		0x71054
+#define PIPE_FLIPDONETIMSTMP(pipe)	\
+	_MMIO_PIPE(pipe, _PIPE_FLIPDONETMSTMP_A, _PIPE_FLIPDONETMSTMP_B)
+
+#define _VLV_PIPE_MSA_MISC_A			0x70048
+#define VLV_PIPE_MSA_MISC(pipe)		\
+			_MMIO_PIPE2(pipe, _VLV_PIPE_MSA_MISC_A)
+#define   VLV_MSA_MISC1_HW_ENABLE			REG_BIT(31)
+#define   VLV_MSA_MISC1_SW_S3D_MASK			REG_GENMASK(2, 0) /* MSA MISC1 3:1 */
 
 #define GGC				_MMIO(0x108040)
 #define   GMS_MASK			REG_GENMASK(15, 8)
@@ -7747,314 +6482,6 @@ enum skl_power_gate {
 #define  ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN	(1 << 23)
 #define  DG2_PHY_DP_TX_ACK_MASK			REG_GENMASK(23, 20)
 
-/* Icelake Display Stream Compression Registers */
-#define DSCA_PICTURE_PARAMETER_SET_0		_MMIO(0x6B200)
-#define DSCC_PICTURE_PARAMETER_SET_0		_MMIO(0x6BA00)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_0_PB	0x78270
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_0_PB	0x78370
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_0_PC	0x78470
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_0_PC	0x78570
-#define ICL_DSC0_PICTURE_PARAMETER_SET_0(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_0_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_0_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_0(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_0_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_0_PC)
-#define  DSC_ALT_ICH_SEL		(1 << 20)
-#define  DSC_VBR_ENABLE			(1 << 19)
-#define  DSC_422_ENABLE			(1 << 18)
-#define  DSC_COLOR_SPACE_CONVERSION	(1 << 17)
-#define  DSC_BLOCK_PREDICTION		(1 << 16)
-#define  DSC_LINE_BUF_DEPTH_SHIFT	12
-#define  DSC_BPC_SHIFT			8
-#define  DSC_VER_MIN_SHIFT		4
-#define  DSC_VER_MAJ			(0x1 << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_1		_MMIO(0x6B204)
-#define DSCC_PICTURE_PARAMETER_SET_1		_MMIO(0x6BA04)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_1_PB	0x78274
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_1_PB	0x78374
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_1_PC	0x78474
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_1_PC	0x78574
-#define ICL_DSC0_PICTURE_PARAMETER_SET_1(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_1_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_1_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_1(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_1_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_1_PC)
-#define  DSC_BPP(bpp)				((bpp) << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_2		_MMIO(0x6B208)
-#define DSCC_PICTURE_PARAMETER_SET_2		_MMIO(0x6BA08)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_2_PB	0x78278
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_2_PB	0x78378
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_2_PC	0x78478
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_2_PC	0x78578
-#define ICL_DSC0_PICTURE_PARAMETER_SET_2(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_2_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_2_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_2(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-					    _ICL_DSC1_PICTURE_PARAMETER_SET_2_PB, \
-					    _ICL_DSC1_PICTURE_PARAMETER_SET_2_PC)
-#define  DSC_PIC_WIDTH(pic_width)	((pic_width) << 16)
-#define  DSC_PIC_HEIGHT(pic_height)	((pic_height) << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_3		_MMIO(0x6B20C)
-#define DSCC_PICTURE_PARAMETER_SET_3		_MMIO(0x6BA0C)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_3_PB	0x7827C
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_3_PB	0x7837C
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_3_PC	0x7847C
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_3_PC	0x7857C
-#define ICL_DSC0_PICTURE_PARAMETER_SET_3(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_3_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_3_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_3(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_3_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_3_PC)
-#define  DSC_SLICE_WIDTH(slice_width)   ((slice_width) << 16)
-#define  DSC_SLICE_HEIGHT(slice_height) ((slice_height) << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_4		_MMIO(0x6B210)
-#define DSCC_PICTURE_PARAMETER_SET_4		_MMIO(0x6BA10)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_4_PB	0x78280
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_4_PB	0x78380
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_4_PC	0x78480
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_4_PC	0x78580
-#define ICL_DSC0_PICTURE_PARAMETER_SET_4(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_4_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_4_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_4(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_4_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_4_PC)
-#define  DSC_INITIAL_DEC_DELAY(dec_delay)       ((dec_delay) << 16)
-#define  DSC_INITIAL_XMIT_DELAY(xmit_delay)     ((xmit_delay) << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_5		_MMIO(0x6B214)
-#define DSCC_PICTURE_PARAMETER_SET_5		_MMIO(0x6BA14)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_5_PB	0x78284
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_5_PB	0x78384
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_5_PC	0x78484
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_5_PC	0x78584
-#define ICL_DSC0_PICTURE_PARAMETER_SET_5(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_5_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_5_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_5(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_5_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_5_PC)
-#define  DSC_SCALE_DEC_INT(scale_dec)	((scale_dec) << 16)
-#define  DSC_SCALE_INC_INT(scale_inc)		((scale_inc) << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_6		_MMIO(0x6B218)
-#define DSCC_PICTURE_PARAMETER_SET_6		_MMIO(0x6BA18)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_6_PB	0x78288
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_6_PB	0x78388
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_6_PC	0x78488
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_6_PC	0x78588
-#define ICL_DSC0_PICTURE_PARAMETER_SET_6(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_6_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_6_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_6(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_6_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_6_PC)
-#define  DSC_FLATNESS_MAX_QP(max_qp)		((max_qp) << 24)
-#define  DSC_FLATNESS_MIN_QP(min_qp)		((min_qp) << 16)
-#define  DSC_FIRST_LINE_BPG_OFFSET(offset)	((offset) << 8)
-#define  DSC_INITIAL_SCALE_VALUE(value)		((value) << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_7		_MMIO(0x6B21C)
-#define DSCC_PICTURE_PARAMETER_SET_7		_MMIO(0x6BA1C)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_7_PB	0x7828C
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_7_PB	0x7838C
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_7_PC	0x7848C
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_7_PC	0x7858C
-#define ICL_DSC0_PICTURE_PARAMETER_SET_7(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							    _ICL_DSC0_PICTURE_PARAMETER_SET_7_PB, \
-							    _ICL_DSC0_PICTURE_PARAMETER_SET_7_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_7(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							    _ICL_DSC1_PICTURE_PARAMETER_SET_7_PB, \
-							    _ICL_DSC1_PICTURE_PARAMETER_SET_7_PC)
-#define  DSC_NFL_BPG_OFFSET(bpg_offset)		((bpg_offset) << 16)
-#define  DSC_SLICE_BPG_OFFSET(bpg_offset)	((bpg_offset) << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_8		_MMIO(0x6B220)
-#define DSCC_PICTURE_PARAMETER_SET_8		_MMIO(0x6BA20)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_8_PB	0x78290
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_8_PB	0x78390
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_8_PC	0x78490
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_8_PC	0x78590
-#define ICL_DSC0_PICTURE_PARAMETER_SET_8(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_8_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_8_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_8(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_8_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_8_PC)
-#define  DSC_INITIAL_OFFSET(initial_offset)		((initial_offset) << 16)
-#define  DSC_FINAL_OFFSET(final_offset)			((final_offset) << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_9		_MMIO(0x6B224)
-#define DSCC_PICTURE_PARAMETER_SET_9		_MMIO(0x6BA24)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_9_PB	0x78294
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_9_PB	0x78394
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_9_PC	0x78494
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_9_PC	0x78594
-#define ICL_DSC0_PICTURE_PARAMETER_SET_9(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_9_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_9_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_9(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_9_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_9_PC)
-#define  DSC_RC_EDGE_FACTOR(rc_edge_fact)	((rc_edge_fact) << 16)
-#define  DSC_RC_MODEL_SIZE(rc_model_size)	((rc_model_size) << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_10		_MMIO(0x6B228)
-#define DSCC_PICTURE_PARAMETER_SET_10		_MMIO(0x6BA28)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_10_PB	0x78298
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_10_PB	0x78398
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_10_PC	0x78498
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_10_PC	0x78598
-#define ICL_DSC0_PICTURE_PARAMETER_SET_10(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_10_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_10_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_10(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_10_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_10_PC)
-#define  DSC_RC_TARGET_OFF_LOW(rc_tgt_off_low)		((rc_tgt_off_low) << 20)
-#define  DSC_RC_TARGET_OFF_HIGH(rc_tgt_off_high)	((rc_tgt_off_high) << 16)
-#define  DSC_RC_QUANT_INC_LIMIT1(lim)			((lim) << 8)
-#define  DSC_RC_QUANT_INC_LIMIT0(lim)			((lim) << 0)
-
-#define DSCA_PICTURE_PARAMETER_SET_11		_MMIO(0x6B22C)
-#define DSCC_PICTURE_PARAMETER_SET_11		_MMIO(0x6BA2C)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_11_PB	0x7829C
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_11_PB	0x7839C
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_11_PC	0x7849C
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_11_PC	0x7859C
-#define ICL_DSC0_PICTURE_PARAMETER_SET_11(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_11_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_11_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_11(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_11_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_11_PC)
-
-#define DSCA_PICTURE_PARAMETER_SET_12		_MMIO(0x6B260)
-#define DSCC_PICTURE_PARAMETER_SET_12		_MMIO(0x6BA60)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_12_PB	0x782A0
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_12_PB	0x783A0
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_12_PC	0x784A0
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_12_PC	0x785A0
-#define ICL_DSC0_PICTURE_PARAMETER_SET_12(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_12_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_12_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_12(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_12_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_12_PC)
-
-#define DSCA_PICTURE_PARAMETER_SET_13		_MMIO(0x6B264)
-#define DSCC_PICTURE_PARAMETER_SET_13		_MMIO(0x6BA64)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_13_PB	0x782A4
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_13_PB	0x783A4
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_13_PC	0x784A4
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_13_PC	0x785A4
-#define ICL_DSC0_PICTURE_PARAMETER_SET_13(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_13_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_13_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_13(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_13_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_13_PC)
-
-#define DSCA_PICTURE_PARAMETER_SET_14		_MMIO(0x6B268)
-#define DSCC_PICTURE_PARAMETER_SET_14		_MMIO(0x6BA68)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_14_PB	0x782A8
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_14_PB	0x783A8
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_14_PC	0x784A8
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_14_PC	0x785A8
-#define ICL_DSC0_PICTURE_PARAMETER_SET_14(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_14_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_14_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_14(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_14_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_14_PC)
-
-#define DSCA_PICTURE_PARAMETER_SET_15		_MMIO(0x6B26C)
-#define DSCC_PICTURE_PARAMETER_SET_15		_MMIO(0x6BA6C)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_15_PB	0x782AC
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_15_PB	0x783AC
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_15_PC	0x784AC
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_15_PC	0x785AC
-#define ICL_DSC0_PICTURE_PARAMETER_SET_15(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_15_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_15_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_15(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_15_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_15_PC)
-
-#define DSCA_PICTURE_PARAMETER_SET_16		_MMIO(0x6B270)
-#define DSCC_PICTURE_PARAMETER_SET_16		_MMIO(0x6BA70)
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_16_PB	0x782B0
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_16_PB	0x783B0
-#define _ICL_DSC0_PICTURE_PARAMETER_SET_16_PC	0x784B0
-#define _ICL_DSC1_PICTURE_PARAMETER_SET_16_PC	0x785B0
-#define ICL_DSC0_PICTURE_PARAMETER_SET_16(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_16_PB, \
-							   _ICL_DSC0_PICTURE_PARAMETER_SET_16_PC)
-#define ICL_DSC1_PICTURE_PARAMETER_SET_16(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_16_PB, \
-							   _ICL_DSC1_PICTURE_PARAMETER_SET_16_PC)
-#define  DSC_SLICE_ROW_PER_FRAME(slice_row_per_frame)	((slice_row_per_frame) << 20)
-#define  DSC_SLICE_PER_LINE(slice_per_line)		((slice_per_line) << 16)
-#define  DSC_SLICE_CHUNK_SIZE(slice_chunk_size)		((slice_chunk_size) << 0)
-
-/* Icelake Rate Control Buffer Threshold Registers */
-#define DSCA_RC_BUF_THRESH_0			_MMIO(0x6B230)
-#define DSCA_RC_BUF_THRESH_0_UDW		_MMIO(0x6B230 + 4)
-#define DSCC_RC_BUF_THRESH_0			_MMIO(0x6BA30)
-#define DSCC_RC_BUF_THRESH_0_UDW		_MMIO(0x6BA30 + 4)
-#define _ICL_DSC0_RC_BUF_THRESH_0_PB		(0x78254)
-#define _ICL_DSC0_RC_BUF_THRESH_0_UDW_PB	(0x78254 + 4)
-#define _ICL_DSC1_RC_BUF_THRESH_0_PB		(0x78354)
-#define _ICL_DSC1_RC_BUF_THRESH_0_UDW_PB	(0x78354 + 4)
-#define _ICL_DSC0_RC_BUF_THRESH_0_PC		(0x78454)
-#define _ICL_DSC0_RC_BUF_THRESH_0_UDW_PC	(0x78454 + 4)
-#define _ICL_DSC1_RC_BUF_THRESH_0_PC		(0x78554)
-#define _ICL_DSC1_RC_BUF_THRESH_0_UDW_PC	(0x78554 + 4)
-#define ICL_DSC0_RC_BUF_THRESH_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-						_ICL_DSC0_RC_BUF_THRESH_0_PB, \
-						_ICL_DSC0_RC_BUF_THRESH_0_PC)
-#define ICL_DSC0_RC_BUF_THRESH_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-						_ICL_DSC0_RC_BUF_THRESH_0_UDW_PB, \
-						_ICL_DSC0_RC_BUF_THRESH_0_UDW_PC)
-#define ICL_DSC1_RC_BUF_THRESH_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-						_ICL_DSC1_RC_BUF_THRESH_0_PB, \
-						_ICL_DSC1_RC_BUF_THRESH_0_PC)
-#define ICL_DSC1_RC_BUF_THRESH_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-						_ICL_DSC1_RC_BUF_THRESH_0_UDW_PB, \
-						_ICL_DSC1_RC_BUF_THRESH_0_UDW_PC)
-
-#define DSCA_RC_BUF_THRESH_1			_MMIO(0x6B238)
-#define DSCA_RC_BUF_THRESH_1_UDW		_MMIO(0x6B238 + 4)
-#define DSCC_RC_BUF_THRESH_1			_MMIO(0x6BA38)
-#define DSCC_RC_BUF_THRESH_1_UDW		_MMIO(0x6BA38 + 4)
-#define _ICL_DSC0_RC_BUF_THRESH_1_PB		(0x7825C)
-#define _ICL_DSC0_RC_BUF_THRESH_1_UDW_PB	(0x7825C + 4)
-#define _ICL_DSC1_RC_BUF_THRESH_1_PB		(0x7835C)
-#define _ICL_DSC1_RC_BUF_THRESH_1_UDW_PB	(0x7835C + 4)
-#define _ICL_DSC0_RC_BUF_THRESH_1_PC		(0x7845C)
-#define _ICL_DSC0_RC_BUF_THRESH_1_UDW_PC	(0x7845C + 4)
-#define _ICL_DSC1_RC_BUF_THRESH_1_PC		(0x7855C)
-#define _ICL_DSC1_RC_BUF_THRESH_1_UDW_PC	(0x7855C + 4)
-#define ICL_DSC0_RC_BUF_THRESH_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-						_ICL_DSC0_RC_BUF_THRESH_1_PB, \
-						_ICL_DSC0_RC_BUF_THRESH_1_PC)
-#define ICL_DSC0_RC_BUF_THRESH_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-						_ICL_DSC0_RC_BUF_THRESH_1_UDW_PB, \
-						_ICL_DSC0_RC_BUF_THRESH_1_UDW_PC)
-#define ICL_DSC1_RC_BUF_THRESH_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
-						_ICL_DSC1_RC_BUF_THRESH_1_PB, \
-						_ICL_DSC1_RC_BUF_THRESH_1_PC)
-#define ICL_DSC1_RC_BUF_THRESH_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
-						_ICL_DSC1_RC_BUF_THRESH_1_UDW_PB, \
-						_ICL_DSC1_RC_BUF_THRESH_1_UDW_PC)
-
 #define PORT_TX_DFLEXDPSP(fia)			_MMIO_FIA((fia), 0x008A0)
 #define   MODULAR_FIA_MASK			(1 << 4)
 #define   TC_LIVE_STATE_TBT(idx)		(1 << ((idx) * 8 + 6))
@@ -8091,16 +6518,6 @@ enum skl_power_gate {
 #define OROM_OFFSET				_MMIO(0x1020c0)
 #define   OROM_OFFSET_MASK			REG_GENMASK(20, 16)
 
-/* This register controls the Display State Buffer (DSB) engines. */
-#define _DSBSL_INSTANCE_BASE		0x70B00
-#define DSBSL_INSTANCE(pipe, id)	(_DSBSL_INSTANCE_BASE + \
-					 (pipe) * 0x1000 + (id) * 0x100)
-#define DSB_HEAD(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x0)
-#define DSB_TAIL(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x4)
-#define DSB_CTRL(pipe, id)		_MMIO(DSBSL_INSTANCE(pipe, id) + 0x8)
-#define   DSB_ENABLE			(1 << 31)
-#define   DSB_STATUS_BUSY		(1 << 0)
-
 #define CLKREQ_POLICY			_MMIO(0x101038)
 #define  CLKREQ_POLICY_MEM_UP_OVRD	REG_BIT(1)
 
@@ -8112,26 +6529,18 @@ enum skl_power_gate {
 #define MTL_CLKGATE_DIS_TRANS(trans)			_MMIO_TRANS2(trans, _MTL_CLKGATE_DIS_TRANS_A)
 #define  MTL_CLKGATE_DIS_TRANS_DMASC_GATING_DIS		REG_BIT(7)
 
-#define MTL_LATENCY_LP0_LP1		_MMIO(0x45780)
-#define MTL_LATENCY_LP2_LP3		_MMIO(0x45784)
-#define MTL_LATENCY_LP4_LP5		_MMIO(0x45788)
-#define  MTL_LATENCY_LEVEL_EVEN_MASK	REG_GENMASK(12, 0)
-#define  MTL_LATENCY_LEVEL_ODD_MASK	REG_GENMASK(28, 16)
-
-#define MTL_LATENCY_SAGV		_MMIO(0x4578b)
-#define   MTL_LATENCY_QCLK_SAGV		REG_GENMASK(12, 0)
-
 #define MTL_MEM_SS_INFO_GLOBAL			_MMIO(0x45700)
 #define   MTL_N_OF_ENABLED_QGV_POINTS_MASK	REG_GENMASK(11, 8)
 #define   MTL_N_OF_POPULATED_CH_MASK		REG_GENMASK(7, 4)
 #define   MTL_DDR_TYPE_MASK			REG_GENMASK(3, 0)
 
-#define MTL_MEM_SS_INFO_QGV_POINT_LOW(point)	 _MMIO(0x45710 + (point) * 2)
+#define MTL_MEM_SS_INFO_QGV_POINT_OFFSET	0x45710
+#define MTL_MEM_SS_INFO_QGV_POINT_LOW(point)	_MMIO(MTL_MEM_SS_INFO_QGV_POINT_OFFSET + (point) * 8)
 #define   MTL_TRCD_MASK			REG_GENMASK(31, 24)
 #define   MTL_TRP_MASK			REG_GENMASK(23, 16)
 #define   MTL_DCLK_MASK			REG_GENMASK(15, 0)
 
-#define MTL_MEM_SS_INFO_QGV_POINT_HIGH(point)	 _MMIO(0x45714 + (point) * 2)
+#define MTL_MEM_SS_INFO_QGV_POINT_HIGH(point)	_MMIO(MTL_MEM_SS_INFO_QGV_POINT_OFFSET + (point) * 8 + 4)
 #define   MTL_TRAS_MASK			REG_GENMASK(16, 8)
 #define   MTL_TRDPRE_MASK		REG_GENMASK(7, 0)
 
diff --git a/drivers/gpu/drm/i915/i915_reg_defs.h b/drivers/gpu/drm/i915/i915_reg_defs.h
index be43580a6979..db26de6b57bc 100644
--- a/drivers/gpu/drm/i915/i915_reg_defs.h
+++ b/drivers/gpu/drm/i915/i915_reg_defs.h
@@ -120,6 +120,35 @@
 #define _PICK_EVEN(__index, __a, __b) ((__a) + (__index) * ((__b) - (__a)))
 
 /*
+ * Like _PICK_EVEN(), but supports 2 ranges of evenly spaced address offsets.
+ * @__c_index corresponds to the index in which the second range starts to be
+ * used. Using math interval notation, the first range is used for indexes [ 0,
+ * @__c_index), while the second range is used for [ @__c_index, ... ). Example:
+ *
+ * #define _FOO_A			0xf000
+ * #define _FOO_B			0xf004
+ * #define _FOO_C			0xf008
+ * #define _SUPER_FOO_A			0xa000
+ * #define _SUPER_FOO_B			0xa100
+ * #define FOO(x)			_MMIO(_PICK_EVEN_2RANGES(x, 3,		\
+ *					      _FOO_A, _FOO_B,			\
+ *					      _SUPER_FOO_A, _SUPER_FOO_B))
+ *
+ * This expands to:
+ *	0: 0xf000,
+ *	1: 0xf004,
+ *	2: 0xf008,
+ *	3: 0xa000,
+ *	4: 0xa100,
+ *	5: 0xa200,
+ *	...
+ */
+#define _PICK_EVEN_2RANGES(__index, __c_index, __a, __b, __c, __d)		\
+	(BUILD_BUG_ON_ZERO(!__is_constexpr(__c_index)) +			\
+	 ((__index) < (__c_index) ? _PICK_EVEN(__index, __a, __b) :		\
+				   _PICK_EVEN((__index) - (__c_index), __c, __d)))
+
+/*
  * Given the arbitrary numbers in varargs, pick the 0-based __index'th number.
  *
  * Always prefer _PICK_EVEN() over this if the numbers are evenly spaced.
@@ -136,6 +165,8 @@ typedef struct {
 	u32 reg;
 } i915_mcr_reg_t;
 
+#define MCR_REG(offset)	((const i915_mcr_reg_t){ .reg = (offset) })
+
 #define INVALID_MMIO_REG _MMIO(0)
 
 /*
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index 7503dcb9043b..630a732aaecc 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -48,7 +48,6 @@
 #include "i915_driver.h"
 #include "i915_drv.h"
 #include "i915_trace.h"
-#include "intel_pm.h"
 
 struct execute_cb {
 	struct irq_work work;
diff --git a/drivers/gpu/drm/i915/i915_scatterlist.c b/drivers/gpu/drm/i915/i915_scatterlist.c
index 7c7a86921b1c..e93d2538f298 100644
--- a/drivers/gpu/drm/i915/i915_scatterlist.c
+++ b/drivers/gpu/drm/i915/i915_scatterlist.c
@@ -56,7 +56,7 @@ static const struct i915_refct_sgt_ops rsgt_ops = {
 /**
  * i915_refct_sgt_init - Initialize a struct i915_refct_sgt with default ops
  * @rsgt: The struct i915_refct_sgt to initialize.
- * size: The size of the underlying memory buffer.
+ * @size: The size of the underlying memory buffer.
  */
 void i915_refct_sgt_init(struct i915_refct_sgt *rsgt, size_t size)
 {
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index 595e8b574990..e88bb4f04305 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -37,7 +37,6 @@
 
 #include "i915_drv.h"
 #include "i915_sysfs.h"
-#include "intel_pm.h"
 
 struct drm_i915_private *kdev_minor_to_i915(struct device *kdev)
 {
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
index a72698a2dbc8..a1bc804cfa15 100644
--- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
+++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
@@ -139,13 +139,6 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
 
 	mutex_unlock(&bman->lock);
 
-	if (place->lpfn - place->fpfn == n_pages)
-		bman_res->base.start = place->fpfn;
-	else if (lpfn <= bman->visible_size)
-		bman_res->base.start = 0;
-	else
-		bman_res->base.start = bman->visible_size;
-
 	*res = &bman_res->base;
 	return 0;
 
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index f51fd9fd4c89..20a44788999e 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -739,6 +739,7 @@ bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color)
 /**
  * i915_vma_insert - finds a slot for the vma in its address space
  * @vma: the vma
+ * @ww: An optional struct i915_gem_ww_ctx
  * @size: requested size in bytes (can be larger than the VMA)
  * @alignment: required alignment
  * @flags: mask of PIN_* flags to use
@@ -1714,7 +1715,7 @@ static void release_references(struct i915_vma *vma, struct intel_gt *gt,
 	i915_vma_free(vma);
 }
 
-/**
+/*
  * i915_vma_destroy_locked - Remove all weak reference to the vma and put
  * the initial reference.
  *
diff --git a/drivers/gpu/drm/i915/intel_clock_gating.c b/drivers/gpu/drm/i915/intel_clock_gating.c
new file mode 100644
index 000000000000..2c5302bcba19
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_clock_gating.c
@@ -0,0 +1,888 @@
+/*
+ * Copyright © 2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eugeni Dodonov <eugeni.dodonov@intel.com>
+ *
+ */
+
+#include "display/intel_de.h"
+#include "display/intel_display.h"
+#include "display/intel_display_trace.h"
+#include "display/skl_watermark.h"
+
+#include "gt/intel_engine_regs.h"
+#include "gt/intel_gt.h"
+#include "gt/intel_gt_mcr.h"
+#include "gt/intel_gt_regs.h"
+
+#include "i915_drv.h"
+#include "intel_clock_gating.h"
+#include "intel_mchbar_regs.h"
+#include "vlv_sideband.h"
+
+struct drm_i915_clock_gating_funcs {
+	void (*init_clock_gating)(struct drm_i915_private *i915);
+};
+
+static void gen9_init_clock_gating(struct drm_i915_private *i915)
+{
+	if (HAS_LLC(i915)) {
+		/*
+		 * WaCompressedResourceDisplayNewHashMode:skl,kbl
+		 * Display WA #0390: skl,kbl
+		 *
+		 * Must match Sampler, Pixel Back End, and Media. See
+		 * WaCompressedResourceSamplerPbeMediaNewHashMode.
+		 */
+		intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
+	}
+
+	/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
+	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
+
+	/* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
+	intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
+
+	/*
+	 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
+	 * Display WA #0859: skl,bxt,kbl,glk,cfl
+	 */
+	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
+}
+
+static void bxt_init_clock_gating(struct drm_i915_private *i915)
+{
+	gen9_init_clock_gating(i915);
+
+	/* WaDisableSDEUnitClockGating:bxt */
+	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
+
+	/*
+	 * FIXME:
+	 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
+	 */
+	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
+
+	/*
+	 * Wa: Backlight PWM may stop in the asserted state, causing backlight
+	 * to stay fully on.
+	 */
+	intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
+			   intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
+			   PWM1_GATING_DIS | PWM2_GATING_DIS);
+
+	/*
+	 * Lower the display internal timeout.
+	 * This is needed to avoid any hard hangs when DSI port PLL
+	 * is off and a MMIO access is attempted by any privilege
+	 * application, using batch buffers or any other means.
+	 */
+	intel_uncore_write(&i915->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
+
+	/*
+	 * WaFbcTurnOffFbcWatermark:bxt
+	 * Display WA #0562: bxt
+	 */
+	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+
+	/*
+	 * WaFbcHighMemBwCorruptionAvoidance:bxt
+	 * Display WA #0883: bxt
+	 */
+	intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
+}
+
+static void glk_init_clock_gating(struct drm_i915_private *i915)
+{
+	gen9_init_clock_gating(i915);
+
+	/*
+	 * WaDisablePWMClockGating:glk
+	 * Backlight PWM may stop in the asserted state, causing backlight
+	 * to stay fully on.
+	 */
+	intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
+			   intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
+			   PWM1_GATING_DIS | PWM2_GATING_DIS);
+}
+
+static void ibx_init_clock_gating(struct drm_i915_private *i915)
+{
+	/*
+	 * On Ibex Peak and Cougar Point, we need to disable clock
+	 * gating for the panel power sequencer or it will fail to
+	 * start up when no ports are active.
+	 */
+	intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
+}
+
+static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
+{
+	enum pipe pipe;
+
+	for_each_pipe(dev_priv, pipe) {
+		intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
+
+		intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
+		intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
+	}
+}
+
+static void ilk_init_clock_gating(struct drm_i915_private *i915)
+{
+	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
+
+	/*
+	 * Required for FBC
+	 * WaFbcDisableDpfcClockGating:ilk
+	 */
+	dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
+		   ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
+		   ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
+
+	intel_uncore_write(&i915->uncore, PCH_3DCGDIS0,
+			   MARIUNIT_CLOCK_GATE_DISABLE |
+			   SVSMUNIT_CLOCK_GATE_DISABLE);
+	intel_uncore_write(&i915->uncore, PCH_3DCGDIS1,
+			   VFMUNIT_CLOCK_GATE_DISABLE);
+
+	/*
+	 * According to the spec the following bits should be set in
+	 * order to enable memory self-refresh
+	 * The bit 22/21 of 0x42004
+	 * The bit 5 of 0x42020
+	 * The bit 15 of 0x45000
+	 */
+	intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
+			   (intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
+			    ILK_DPARB_GATE | ILK_VSDPFD_FULL));
+	dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
+	intel_uncore_write(&i915->uncore, DISP_ARB_CTL,
+			   (intel_uncore_read(&i915->uncore, DISP_ARB_CTL) |
+			    DISP_FBC_WM_DIS));
+
+	/*
+	 * Based on the document from hardware guys the following bits
+	 * should be set unconditionally in order to enable FBC.
+	 * The bit 22 of 0x42000
+	 * The bit 22 of 0x42004
+	 * The bit 7,8,9 of 0x42020.
+	 */
+	if (IS_IRONLAKE_M(i915)) {
+		/* WaFbcAsynchFlipDisableFbcQueue:ilk */
+		intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
+		intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
+	}
+
+	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
+
+	intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
+
+	g4x_disable_trickle_feed(i915);
+
+	ibx_init_clock_gating(i915);
+}
+
+static void cpt_init_clock_gating(struct drm_i915_private *i915)
+{
+	enum pipe pipe;
+	u32 val;
+
+	/*
+	 * On Ibex Peak and Cougar Point, we need to disable clock
+	 * gating for the panel power sequencer or it will fail to
+	 * start up when no ports are active.
+	 */
+	intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
+			   PCH_DPLUNIT_CLOCK_GATE_DISABLE |
+			   PCH_CPUNIT_CLOCK_GATE_DISABLE);
+	intel_uncore_rmw(&i915->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
+	/* The below fixes the weird display corruption, a few pixels shifted
+	 * downward, on (only) LVDS of some HP laptops with IVY.
+	 */
+	for_each_pipe(i915, pipe) {
+		val = intel_uncore_read(&i915->uncore, TRANS_CHICKEN2(pipe));
+		val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
+		val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
+		if (i915->display.vbt.fdi_rx_polarity_inverted)
+			val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
+		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
+		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
+		intel_uncore_write(&i915->uncore, TRANS_CHICKEN2(pipe), val);
+	}
+	/* WADP0ClockGatingDisable */
+	for_each_pipe(i915, pipe) {
+		intel_uncore_write(&i915->uncore, TRANS_CHICKEN1(pipe),
+				   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
+	}
+}
+
+static void gen6_check_mch_setup(struct drm_i915_private *i915)
+{
+	u32 tmp;
+
+	tmp = intel_uncore_read(&i915->uncore, MCH_SSKPD);
+	if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
+		drm_dbg_kms(&i915->drm,
+			    "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
+			    tmp);
+}
+
+static void gen6_init_clock_gating(struct drm_i915_private *i915)
+{
+	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
+
+	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
+
+	intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
+
+	intel_uncore_write(&i915->uncore, GEN6_UCGCTL1,
+			   intel_uncore_read(&i915->uncore, GEN6_UCGCTL1) |
+			   GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
+			   GEN6_CSUNIT_CLOCK_GATE_DISABLE);
+
+	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
+	 * gating disable must be set.  Failure to set it results in
+	 * flickering pixels due to Z write ordering failures after
+	 * some amount of runtime in the Mesa "fire" demo, and Unigine
+	 * Sanctuary and Tropics, and apparently anything else with
+	 * alpha test or pixel discard.
+	 *
+	 * According to the spec, bit 11 (RCCUNIT) must also be set,
+	 * but we didn't debug actual testcases to find it out.
+	 *
+	 * WaDisableRCCUnitClockGating:snb
+	 * WaDisableRCPBUnitClockGating:snb
+	 */
+	intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
+			   GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
+			   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
+
+	/*
+	 * According to the spec the following bits should be
+	 * set in order to enable memory self-refresh and fbc:
+	 * The bit21 and bit22 of 0x42000
+	 * The bit21 and bit22 of 0x42004
+	 * The bit5 and bit7 of 0x42020
+	 * The bit14 of 0x70180
+	 * The bit14 of 0x71180
+	 *
+	 * WaFbcAsynchFlipDisableFbcQueue:snb
+	 */
+	intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN1,
+			   intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN1) |
+			   ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
+	intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
+			   intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
+			   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
+	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D,
+			   intel_uncore_read(&i915->uncore, ILK_DSPCLK_GATE_D) |
+			   ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
+			   ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
+
+	g4x_disable_trickle_feed(i915);
+
+	cpt_init_clock_gating(i915);
+
+	gen6_check_mch_setup(i915);
+}
+
+static void lpt_init_clock_gating(struct drm_i915_private *i915)
+{
+	/*
+	 * TODO: this bit should only be enabled when really needed, then
+	 * disabled when not needed anymore in order to save power.
+	 */
+	if (HAS_PCH_LPT_LP(i915))
+		intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D,
+				 0, PCH_LP_PARTITION_LEVEL_DISABLE);
+
+	/* WADPOClockGatingDisable:hsw */
+	intel_uncore_rmw(&i915->uncore, TRANS_CHICKEN1(PIPE_A),
+			 0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
+}
+
+static void gen8_set_l3sqc_credits(struct drm_i915_private *i915,
+				   int general_prio_credits,
+				   int high_prio_credits)
+{
+	u32 misccpctl;
+	u32 val;
+
+	/* WaTempDisableDOPClkGating:bdw */
+	misccpctl = intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
+				     GEN7_DOP_CLOCK_GATE_ENABLE, 0);
+
+	val = intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
+	val &= ~L3_PRIO_CREDITS_MASK;
+	val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
+	val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
+	intel_gt_mcr_multicast_write(to_gt(i915), GEN8_L3SQCREG1, val);
+
+	/*
+	 * Wait at least 100 clocks before re-enabling clock gating.
+	 * See the definition of L3SQCREG1 in BSpec.
+	 */
+	intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
+	udelay(1);
+	intel_uncore_write(&i915->uncore, GEN7_MISCCPCTL, misccpctl);
+}
+
+static void icl_init_clock_gating(struct drm_i915_private *i915)
+{
+	/* Wa_1409120013:icl,ehl */
+	intel_uncore_write(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
+			   DPFC_CHICKEN_COMP_DUMMY_PIXEL);
+
+	/*Wa_14010594013:icl, ehl */
+	intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1,
+			 0, ICL_DELAY_PMRSP);
+}
+
+static void gen12lp_init_clock_gating(struct drm_i915_private *i915)
+{
+	/* Wa_1409120013 */
+	if (DISPLAY_VER(i915) == 12)
+		intel_uncore_write(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
+				   DPFC_CHICKEN_COMP_DUMMY_PIXEL);
+
+	/* Wa_14013723622:tgl,rkl,dg1,adl-s */
+	if (DISPLAY_VER(i915) == 12)
+		intel_uncore_rmw(&i915->uncore, CLKREQ_POLICY,
+				 CLKREQ_POLICY_MEM_UP_OVRD, 0);
+}
+
+static void adlp_init_clock_gating(struct drm_i915_private *i915)
+{
+	gen12lp_init_clock_gating(i915);
+
+	/* Wa_22011091694:adlp */
+	intel_de_rmw(i915, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
+
+	/* Bspec/49189 Initialize Sequence */
+	intel_de_rmw(i915, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
+}
+
+static void xehpsdv_init_clock_gating(struct drm_i915_private *i915)
+{
+	/* Wa_22010146351:xehpsdv */
+	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
+		intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
+}
+
+static void dg2_init_clock_gating(struct drm_i915_private *i915)
+{
+	/* Wa_22010954014:dg2 */
+	intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
+			 SGSI_SIDECLK_DIS);
+
+	/*
+	 * Wa_14010733611:dg2_g10
+	 * Wa_22010146351:dg2_g10
+	 */
+	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
+		intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
+				 SGR_DIS | SGGI_DIS);
+}
+
+static void pvc_init_clock_gating(struct drm_i915_private *i915)
+{
+	/* Wa_14012385139:pvc */
+	if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
+		intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
+
+	/* Wa_22010954014:pvc */
+	if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
+		intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
+}
+
+static void cnp_init_clock_gating(struct drm_i915_private *i915)
+{
+	if (!HAS_PCH_CNP(i915))
+		return;
+
+	/* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
+	intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
+}
+
+static void cfl_init_clock_gating(struct drm_i915_private *i915)
+{
+	cnp_init_clock_gating(i915);
+	gen9_init_clock_gating(i915);
+
+	/* WAC6entrylatency:cfl */
+	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
+
+	/*
+	 * WaFbcTurnOffFbcWatermark:cfl
+	 * Display WA #0562: cfl
+	 */
+	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+
+	/*
+	 * WaFbcNukeOnHostModify:cfl
+	 * Display WA #0873: cfl
+	 */
+	intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
+			 0, DPFC_NUKE_ON_ANY_MODIFICATION);
+}
+
+static void kbl_init_clock_gating(struct drm_i915_private *i915)
+{
+	gen9_init_clock_gating(i915);
+
+	/* WAC6entrylatency:kbl */
+	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
+
+	/* WaDisableSDEUnitClockGating:kbl */
+	if (IS_KBL_GRAPHICS_STEP(i915, 0, STEP_C0))
+		intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6,
+				 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
+
+	/* WaDisableGamClockGating:kbl */
+	if (IS_KBL_GRAPHICS_STEP(i915, 0, STEP_C0))
+		intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1,
+				 0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
+
+	/*
+	 * WaFbcTurnOffFbcWatermark:kbl
+	 * Display WA #0562: kbl
+	 */
+	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+
+	/*
+	 * WaFbcNukeOnHostModify:kbl
+	 * Display WA #0873: kbl
+	 */
+	intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
+			 0, DPFC_NUKE_ON_ANY_MODIFICATION);
+}
+
+static void skl_init_clock_gating(struct drm_i915_private *i915)
+{
+	gen9_init_clock_gating(i915);
+
+	/* WaDisableDopClockGating:skl */
+	intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
+			 GEN7_DOP_CLOCK_GATE_ENABLE, 0);
+
+	/* WAC6entrylatency:skl */
+	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
+
+	/*
+	 * WaFbcTurnOffFbcWatermark:skl
+	 * Display WA #0562: skl
+	 */
+	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+
+	/*
+	 * WaFbcNukeOnHostModify:skl
+	 * Display WA #0873: skl
+	 */
+	intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
+			 0, DPFC_NUKE_ON_ANY_MODIFICATION);
+
+	/*
+	 * WaFbcHighMemBwCorruptionAvoidance:skl
+	 * Display WA #0883: skl
+	 */
+	intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
+}
+
+static void bdw_init_clock_gating(struct drm_i915_private *i915)
+{
+	enum pipe pipe;
+
+	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
+	intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
+
+	/* WaSwitchSolVfFArbitrationPriority:bdw */
+	intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
+
+	/* WaPsrDPAMaskVBlankInSRD:bdw */
+	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, DPA_MASK_VBLANK_SRD);
+
+	for_each_pipe(i915, pipe) {
+		/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
+		intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
+				 0, BDW_DPRS_MASK_VBLANK_SRD);
+	}
+
+	/* WaVSRefCountFullforceMissDisable:bdw */
+	/* WaDSRefCountFullforceMissDisable:bdw */
+	intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
+			 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
+
+	intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
+			   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
+
+	/* WaDisableSDEUnitClockGating:bdw */
+	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
+
+	/* WaProgramL3SqcReg1Default:bdw */
+	gen8_set_l3sqc_credits(i915, 30, 2);
+
+	/* WaKVMNotificationOnConfigChange:bdw */
+	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR2_1,
+			 0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
+
+	lpt_init_clock_gating(i915);
+
+	/* WaDisableDopClockGating:bdw
+	 *
+	 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
+	 * clock gating.
+	 */
+	intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
+}
+
+static void hsw_init_clock_gating(struct drm_i915_private *i915)
+{
+	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
+	intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
+
+	/* This is required by WaCatErrorRejectionIssue:hsw */
+	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
+			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
+
+	/* WaSwitchSolVfFArbitrationPriority:hsw */
+	intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
+
+	lpt_init_clock_gating(i915);
+}
+
+static void ivb_init_clock_gating(struct drm_i915_private *i915)
+{
+	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
+
+	/* WaFbcAsynchFlipDisableFbcQueue:ivb */
+	intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
+
+	/* WaDisableBackToBackFlipFix:ivb */
+	intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
+			   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
+			   CHICKEN3_DGMG_DONE_FIX_DISABLE);
+
+	if (IS_IVB_GT1(i915))
+		intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
+				   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+	else {
+		/* must write both registers */
+		intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
+				   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+		intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2_GT2,
+				   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+	}
+
+	/*
+	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
+	 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
+	 */
+	intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
+			   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
+
+	/* This is required by WaCatErrorRejectionIssue:ivb */
+	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
+			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
+
+	g4x_disable_trickle_feed(i915);
+
+	intel_uncore_rmw(&i915->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
+			 GEN6_MBC_SNPCR_MED);
+
+	if (!HAS_PCH_NOP(i915))
+		cpt_init_clock_gating(i915);
+
+	gen6_check_mch_setup(i915);
+}
+
+static void vlv_init_clock_gating(struct drm_i915_private *i915)
+{
+	/* WaDisableBackToBackFlipFix:vlv */
+	intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
+			   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
+			   CHICKEN3_DGMG_DONE_FIX_DISABLE);
+
+	/* WaDisableDopClockGating:vlv */
+	intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
+			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+
+	/* This is required by WaCatErrorRejectionIssue:vlv */
+	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
+			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
+
+	/*
+	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
+	 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
+	 */
+	intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
+			   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
+
+	/* WaDisableL3Bank2xClockGate:vlv
+	 * Disabling L3 clock gating- MMIO 940c[25] = 1
+	 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
+	intel_uncore_rmw(&i915->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
+
+	/*
+	 * WaDisableVLVClockGating_VBIIssue:vlv
+	 * Disable clock gating on th GCFG unit to prevent a delay
+	 * in the reporting of vblank events.
+	 */
+	intel_uncore_write(&i915->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
+}
+
+static void chv_init_clock_gating(struct drm_i915_private *i915)
+{
+	/* WaVSRefCountFullforceMissDisable:chv */
+	/* WaDSRefCountFullforceMissDisable:chv */
+	intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
+			 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
+
+	/* WaDisableSemaphoreAndSyncFlipWait:chv */
+	intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
+			   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
+
+	/* WaDisableCSUnitClockGating:chv */
+	intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
+
+	/* WaDisableSDEUnitClockGating:chv */
+	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
+
+	/*
+	 * WaProgramL3SqcReg1Default:chv
+	 * See gfxspecs/Related Documents/Performance Guide/
+	 * LSQC Setting Recommendations.
+	 */
+	gen8_set_l3sqc_credits(i915, 38, 2);
+}
+
+static void g4x_init_clock_gating(struct drm_i915_private *i915)
+{
+	u32 dspclk_gate;
+
+	intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, 0);
+	intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
+			   GS_UNIT_CLOCK_GATE_DISABLE |
+			   CL_UNIT_CLOCK_GATE_DISABLE);
+	intel_uncore_write(&i915->uncore, RAMCLK_GATE_D, 0);
+	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
+		OVRUNIT_CLOCK_GATE_DISABLE |
+		OVCUNIT_CLOCK_GATE_DISABLE;
+	if (IS_GM45(i915))
+		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
+	intel_uncore_write(&i915->uncore, DSPCLK_GATE_D(i915), dspclk_gate);
+
+	g4x_disable_trickle_feed(i915);
+}
+
+static void i965gm_init_clock_gating(struct drm_i915_private *i915)
+{
+	struct intel_uncore *uncore = &i915->uncore;
+
+	intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
+	intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
+	intel_uncore_write(uncore, DSPCLK_GATE_D(i915), 0);
+	intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
+	intel_uncore_write16(uncore, DEUC, 0);
+	intel_uncore_write(uncore,
+			   MI_ARB_STATE,
+			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
+}
+
+static void i965g_init_clock_gating(struct drm_i915_private *i915)
+{
+	intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
+			   I965_RCC_CLOCK_GATE_DISABLE |
+			   I965_RCPB_CLOCK_GATE_DISABLE |
+			   I965_ISC_CLOCK_GATE_DISABLE |
+			   I965_FBC_CLOCK_GATE_DISABLE);
+	intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, 0);
+	intel_uncore_write(&i915->uncore, MI_ARB_STATE,
+			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
+}
+
+static void gen3_init_clock_gating(struct drm_i915_private *i915)
+{
+	u32 dstate = intel_uncore_read(&i915->uncore, D_STATE);
+
+	dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
+		DSTATE_DOT_CLOCK_GATING;
+	intel_uncore_write(&i915->uncore, D_STATE, dstate);
+
+	if (IS_PINEVIEW(i915))
+		intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
+				   _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
+
+	/* IIR "flip pending" means done if this bit is set */
+	intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
+			   _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
+
+	/* interrupts should cause a wake up from C3 */
+	intel_uncore_write(&i915->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
+
+	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
+	intel_uncore_write(&i915->uncore, MI_ARB_STATE,
+			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
+
+	intel_uncore_write(&i915->uncore, MI_ARB_STATE,
+			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
+}
+
+static void i85x_init_clock_gating(struct drm_i915_private *i915)
+{
+	intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
+
+	/* interrupts should cause a wake up from C3 */
+	intel_uncore_write(&i915->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
+			   _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
+
+	intel_uncore_write(&i915->uncore, MEM_MODE,
+			   _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
+
+	/*
+	 * Have FBC ignore 3D activity since we use software
+	 * render tracking, and otherwise a pure 3D workload
+	 * (even if it just renders a single frame and then does
+	 * abosultely nothing) would not allow FBC to recompress
+	 * until a 2D blit occurs.
+	 */
+	intel_uncore_write(&i915->uncore, SCPD0,
+			   _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
+}
+
+static void i830_init_clock_gating(struct drm_i915_private *i915)
+{
+	intel_uncore_write(&i915->uncore, MEM_MODE,
+			   _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
+			   _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
+}
+
+void intel_clock_gating_init(struct drm_i915_private *i915)
+{
+	i915->clock_gating_funcs->init_clock_gating(i915);
+}
+
+static void nop_init_clock_gating(struct drm_i915_private *i915)
+{
+	drm_dbg_kms(&i915->drm,
+		    "No clock gating settings or workarounds applied.\n");
+}
+
+#define CG_FUNCS(platform)						\
+static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
+	.init_clock_gating = platform##_init_clock_gating,		\
+}
+
+CG_FUNCS(pvc);
+CG_FUNCS(dg2);
+CG_FUNCS(xehpsdv);
+CG_FUNCS(adlp);
+CG_FUNCS(gen12lp);
+CG_FUNCS(icl);
+CG_FUNCS(cfl);
+CG_FUNCS(skl);
+CG_FUNCS(kbl);
+CG_FUNCS(bxt);
+CG_FUNCS(glk);
+CG_FUNCS(bdw);
+CG_FUNCS(chv);
+CG_FUNCS(hsw);
+CG_FUNCS(ivb);
+CG_FUNCS(vlv);
+CG_FUNCS(gen6);
+CG_FUNCS(ilk);
+CG_FUNCS(g4x);
+CG_FUNCS(i965gm);
+CG_FUNCS(i965g);
+CG_FUNCS(gen3);
+CG_FUNCS(i85x);
+CG_FUNCS(i830);
+CG_FUNCS(nop);
+#undef CG_FUNCS
+
+/**
+ * intel_clock_gating_hooks_init - setup the clock gating hooks
+ * @i915: device private
+ *
+ * Setup the hooks that configure which clocks of a given platform can be
+ * gated and also apply various GT and display specific workarounds for these
+ * platforms. Note that some GT specific workarounds are applied separately
+ * when GPU contexts or batchbuffers start their execution.
+ */
+void intel_clock_gating_hooks_init(struct drm_i915_private *i915)
+{
+	if (IS_METEORLAKE(i915))
+		i915->clock_gating_funcs = &nop_clock_gating_funcs;
+	else if (IS_PONTEVECCHIO(i915))
+		i915->clock_gating_funcs = &pvc_clock_gating_funcs;
+	else if (IS_DG2(i915))
+		i915->clock_gating_funcs = &dg2_clock_gating_funcs;
+	else if (IS_XEHPSDV(i915))
+		i915->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
+	else if (IS_ALDERLAKE_P(i915))
+		i915->clock_gating_funcs = &adlp_clock_gating_funcs;
+	else if (GRAPHICS_VER(i915) == 12)
+		i915->clock_gating_funcs = &gen12lp_clock_gating_funcs;
+	else if (GRAPHICS_VER(i915) == 11)
+		i915->clock_gating_funcs = &icl_clock_gating_funcs;
+	else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
+		i915->clock_gating_funcs = &cfl_clock_gating_funcs;
+	else if (IS_SKYLAKE(i915))
+		i915->clock_gating_funcs = &skl_clock_gating_funcs;
+	else if (IS_KABYLAKE(i915))
+		i915->clock_gating_funcs = &kbl_clock_gating_funcs;
+	else if (IS_BROXTON(i915))
+		i915->clock_gating_funcs = &bxt_clock_gating_funcs;
+	else if (IS_GEMINILAKE(i915))
+		i915->clock_gating_funcs = &glk_clock_gating_funcs;
+	else if (IS_BROADWELL(i915))
+		i915->clock_gating_funcs = &bdw_clock_gating_funcs;
+	else if (IS_CHERRYVIEW(i915))
+		i915->clock_gating_funcs = &chv_clock_gating_funcs;
+	else if (IS_HASWELL(i915))
+		i915->clock_gating_funcs = &hsw_clock_gating_funcs;
+	else if (IS_IVYBRIDGE(i915))
+		i915->clock_gating_funcs = &ivb_clock_gating_funcs;
+	else if (IS_VALLEYVIEW(i915))
+		i915->clock_gating_funcs = &vlv_clock_gating_funcs;
+	else if (GRAPHICS_VER(i915) == 6)
+		i915->clock_gating_funcs = &gen6_clock_gating_funcs;
+	else if (GRAPHICS_VER(i915) == 5)
+		i915->clock_gating_funcs = &ilk_clock_gating_funcs;
+	else if (IS_G4X(i915))
+		i915->clock_gating_funcs = &g4x_clock_gating_funcs;
+	else if (IS_I965GM(i915))
+		i915->clock_gating_funcs = &i965gm_clock_gating_funcs;
+	else if (IS_I965G(i915))
+		i915->clock_gating_funcs = &i965g_clock_gating_funcs;
+	else if (GRAPHICS_VER(i915) == 3)
+		i915->clock_gating_funcs = &gen3_clock_gating_funcs;
+	else if (IS_I85X(i915) || IS_I865G(i915))
+		i915->clock_gating_funcs = &i85x_clock_gating_funcs;
+	else if (GRAPHICS_VER(i915) == 2)
+		i915->clock_gating_funcs = &i830_clock_gating_funcs;
+	else {
+		MISSING_CASE(INTEL_DEVID(i915));
+		i915->clock_gating_funcs = &nop_clock_gating_funcs;
+	}
+}
diff --git a/drivers/gpu/drm/i915/intel_clock_gating.h b/drivers/gpu/drm/i915/intel_clock_gating.h
new file mode 100644
index 000000000000..5b4e4c55b2c2
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_clock_gating.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_CLOCK_GATING_H__
+#define __INTEL_CLOCK_GATING_H__
+
+struct drm_i915_private;
+
+void intel_clock_gating_init(struct drm_i915_private *i915);
+void intel_clock_gating_hooks_init(struct drm_i915_private *i915);
+
+#endif /* __INTEL_CLOCK_GATING_H__ */
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 98769e5f2c3d..fc5cd14adfcc 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -119,9 +119,14 @@ void intel_device_info_print(const struct intel_device_info *info,
 		drm_printf(p, "display version: %u\n",
 			   runtime->display.ip.ver);
 
+	drm_printf(p, "graphics stepping: %s\n", intel_step_name(runtime->step.graphics_step));
+	drm_printf(p, "media stepping: %s\n", intel_step_name(runtime->step.media_step));
+	drm_printf(p, "display stepping: %s\n", intel_step_name(runtime->step.display_step));
+	drm_printf(p, "base die stepping: %s\n", intel_step_name(runtime->step.basedie_step));
+
 	drm_printf(p, "gt: %d\n", info->gt);
-	drm_printf(p, "memory-regions: %x\n", runtime->memory_regions);
-	drm_printf(p, "page-sizes: %x\n", runtime->page_sizes);
+	drm_printf(p, "memory-regions: 0x%x\n", runtime->memory_regions);
+	drm_printf(p, "page-sizes: 0x%x\n", runtime->page_sizes);
 	drm_printf(p, "platform: %s\n", intel_platform_name(info->platform));
 	drm_printf(p, "ppgtt-size: %d\n", runtime->ppgtt_size);
 	drm_printf(p, "ppgtt-type: %d\n", runtime->ppgtt_type);
@@ -202,6 +207,10 @@ static const u16 subplatform_rpl_ids[] = {
 	INTEL_RPLP_IDS(0),
 };
 
+static const u16 subplatform_rplu_ids[] = {
+	INTEL_RPLU_IDS(0),
+};
+
 static const u16 subplatform_g10_ids[] = {
 	INTEL_DG2_G10_IDS(0),
 	INTEL_ATS_M150_IDS(0),
@@ -269,6 +278,9 @@ static void intel_device_info_subplatform_init(struct drm_i915_private *i915)
 	} else if (find_devid(devid, subplatform_rpl_ids,
 			      ARRAY_SIZE(subplatform_rpl_ids))) {
 		mask = BIT(INTEL_SUBPLATFORM_RPL);
+		if (find_devid(devid, subplatform_rplu_ids,
+			       ARRAY_SIZE(subplatform_rplu_ids)))
+			mask |= BIT(INTEL_SUBPLATFORM_RPLU);
 	} else if (find_devid(devid, subplatform_g10_ids,
 			      ARRAY_SIZE(subplatform_g10_ids))) {
 		mask = BIT(INTEL_SUBPLATFORM_G10);
@@ -436,6 +448,14 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 			runtime->num_sprites[pipe] = 1;
 	}
 
+	if (HAS_DISPLAY(dev_priv) &&
+	    (IS_DGFX(dev_priv) || DISPLAY_VER(dev_priv) >= 14) &&
+	    !(intel_de_read(dev_priv, GU_CNTL_PROTECTED) & DEPRESENT)) {
+		drm_info(&dev_priv->drm, "Display not present, disabling\n");
+
+		runtime->pipe_mask = 0;
+	}
+
 	if (HAS_DISPLAY(dev_priv) && IS_GRAPHICS_VER(dev_priv, 7, 8) &&
 	    HAS_PCH_SPLIT(dev_priv)) {
 		u32 fuse_strap = intel_de_read(dev_priv, FUSE_STRAP);
@@ -457,8 +477,6 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 			drm_info(&dev_priv->drm,
 				 "Display fused off, disabling\n");
 			runtime->pipe_mask = 0;
-			runtime->cpu_transcoder_mask = 0;
-			runtime->fbc_mask = 0;
 		} else if (fuse_strap & IVB_PIPE_C_DISABLE) {
 			drm_info(&dev_priv->drm, "PipeC fused off\n");
 			runtime->pipe_mask &= ~BIT(PIPE_C);
@@ -535,5 +553,5 @@ void intel_driver_caps_print(const struct intel_driver_caps *caps,
 {
 	drm_printf(p, "Has logical contexts? %s\n",
 		   str_yes_no(caps->has_logical_contexts));
-	drm_printf(p, "scheduler: %x\n", caps->scheduler);
+	drm_printf(p, "scheduler: 0x%x\n", caps->scheduler);
 }
diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h
index 80bda653d61b..080a4557899b 100644
--- a/drivers/gpu/drm/i915/intel_device_info.h
+++ b/drivers/gpu/drm/i915/intel_device_info.h
@@ -127,6 +127,7 @@ enum intel_platform {
  * bit set
  */
 #define INTEL_SUBPLATFORM_N    1
+#define INTEL_SUBPLATFORM_RPLU  2
 
 /* MTL */
 #define INTEL_SUBPLATFORM_M	0
@@ -166,6 +167,7 @@ enum intel_ppgtt_type {
 	func(has_mslice_steering); \
 	func(has_oa_bpc_reporting); \
 	func(has_oa_slice_contrib_limits); \
+	func(has_oam); \
 	func(has_one_eu_per_fuse_bit); \
 	func(has_pxp); \
 	func(has_rc6); \
@@ -191,7 +193,6 @@ enum intel_ppgtt_type {
 	func(has_hotplug); \
 	func(has_hti); \
 	func(has_ipc); \
-	func(has_modular_fia); \
 	func(has_overlay); \
 	func(has_psr); \
 	func(has_psr_hw_tracking); \
diff --git a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
index 1f4805aa2b08..5d08774029cc 100644
--- a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
+++ b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
@@ -7,7 +7,12 @@
 #include "display/intel_backlight_regs.h"
 #include "display/intel_display_types.h"
 #include "display/intel_dmc_regs.h"
+#include "display/intel_dp_aux_regs.h"
 #include "display/intel_dpio_phy.h"
+#include "display/intel_fdi_regs.h"
+#include "display/intel_lvds_regs.h"
+#include "display/intel_psr_regs.h"
+#include "display/skl_watermark_regs.h"
 #include "display/vlv_dsi_pll_regs.h"
 #include "gt/intel_gt_regs.h"
 #include "gvt/gvt.h"
@@ -117,10 +122,10 @@ static int iterate_generic_mmio(struct intel_gvt_mmio_table_iter *iter)
 	MMIO_D(PIPEDSL(PIPE_B));
 	MMIO_D(PIPEDSL(PIPE_C));
 	MMIO_D(PIPEDSL(_PIPE_EDP));
-	MMIO_D(PIPECONF(PIPE_A));
-	MMIO_D(PIPECONF(PIPE_B));
-	MMIO_D(PIPECONF(PIPE_C));
-	MMIO_D(PIPECONF(_PIPE_EDP));
+	MMIO_D(TRANSCONF(TRANSCODER_A));
+	MMIO_D(TRANSCONF(TRANSCODER_B));
+	MMIO_D(TRANSCONF(TRANSCODER_C));
+	MMIO_D(TRANSCONF(TRANSCODER_EDP));
 	MMIO_D(PIPESTAT(PIPE_A));
 	MMIO_D(PIPESTAT(PIPE_B));
 	MMIO_D(PIPESTAT(PIPE_C));
@@ -218,41 +223,41 @@ static int iterate_generic_mmio(struct intel_gvt_mmio_table_iter *iter)
 	MMIO_D(SPRSCALE(PIPE_C));
 	MMIO_D(SPRSURFLIVE(PIPE_C));
 	MMIO_D(REG_50080(PIPE_C, PLANE_SPRITE0));
-	MMIO_D(HTOTAL(TRANSCODER_A));
-	MMIO_D(HBLANK(TRANSCODER_A));
-	MMIO_D(HSYNC(TRANSCODER_A));
-	MMIO_D(VTOTAL(TRANSCODER_A));
-	MMIO_D(VBLANK(TRANSCODER_A));
-	MMIO_D(VSYNC(TRANSCODER_A));
+	MMIO_D(TRANS_HTOTAL(TRANSCODER_A));
+	MMIO_D(TRANS_HBLANK(TRANSCODER_A));
+	MMIO_D(TRANS_HSYNC(TRANSCODER_A));
+	MMIO_D(TRANS_VTOTAL(TRANSCODER_A));
+	MMIO_D(TRANS_VBLANK(TRANSCODER_A));
+	MMIO_D(TRANS_VSYNC(TRANSCODER_A));
 	MMIO_D(BCLRPAT(TRANSCODER_A));
-	MMIO_D(VSYNCSHIFT(TRANSCODER_A));
+	MMIO_D(TRANS_VSYNCSHIFT(TRANSCODER_A));
 	MMIO_D(PIPESRC(TRANSCODER_A));
-	MMIO_D(HTOTAL(TRANSCODER_B));
-	MMIO_D(HBLANK(TRANSCODER_B));
-	MMIO_D(HSYNC(TRANSCODER_B));
-	MMIO_D(VTOTAL(TRANSCODER_B));
-	MMIO_D(VBLANK(TRANSCODER_B));
-	MMIO_D(VSYNC(TRANSCODER_B));
+	MMIO_D(TRANS_HTOTAL(TRANSCODER_B));
+	MMIO_D(TRANS_HBLANK(TRANSCODER_B));
+	MMIO_D(TRANS_HSYNC(TRANSCODER_B));
+	MMIO_D(TRANS_VTOTAL(TRANSCODER_B));
+	MMIO_D(TRANS_VBLANK(TRANSCODER_B));
+	MMIO_D(TRANS_VSYNC(TRANSCODER_B));
 	MMIO_D(BCLRPAT(TRANSCODER_B));
-	MMIO_D(VSYNCSHIFT(TRANSCODER_B));
+	MMIO_D(TRANS_VSYNCSHIFT(TRANSCODER_B));
 	MMIO_D(PIPESRC(TRANSCODER_B));
-	MMIO_D(HTOTAL(TRANSCODER_C));
-	MMIO_D(HBLANK(TRANSCODER_C));
-	MMIO_D(HSYNC(TRANSCODER_C));
-	MMIO_D(VTOTAL(TRANSCODER_C));
-	MMIO_D(VBLANK(TRANSCODER_C));
-	MMIO_D(VSYNC(TRANSCODER_C));
+	MMIO_D(TRANS_HTOTAL(TRANSCODER_C));
+	MMIO_D(TRANS_HBLANK(TRANSCODER_C));
+	MMIO_D(TRANS_HSYNC(TRANSCODER_C));
+	MMIO_D(TRANS_VTOTAL(TRANSCODER_C));
+	MMIO_D(TRANS_VBLANK(TRANSCODER_C));
+	MMIO_D(TRANS_VSYNC(TRANSCODER_C));
 	MMIO_D(BCLRPAT(TRANSCODER_C));
-	MMIO_D(VSYNCSHIFT(TRANSCODER_C));
+	MMIO_D(TRANS_VSYNCSHIFT(TRANSCODER_C));
 	MMIO_D(PIPESRC(TRANSCODER_C));
-	MMIO_D(HTOTAL(TRANSCODER_EDP));
-	MMIO_D(HBLANK(TRANSCODER_EDP));
-	MMIO_D(HSYNC(TRANSCODER_EDP));
-	MMIO_D(VTOTAL(TRANSCODER_EDP));
-	MMIO_D(VBLANK(TRANSCODER_EDP));
-	MMIO_D(VSYNC(TRANSCODER_EDP));
+	MMIO_D(TRANS_HTOTAL(TRANSCODER_EDP));
+	MMIO_D(TRANS_HBLANK(TRANSCODER_EDP));
+	MMIO_D(TRANS_HSYNC(TRANSCODER_EDP));
+	MMIO_D(TRANS_VTOTAL(TRANSCODER_EDP));
+	MMIO_D(TRANS_VBLANK(TRANSCODER_EDP));
+	MMIO_D(TRANS_VSYNC(TRANSCODER_EDP));
 	MMIO_D(BCLRPAT(TRANSCODER_EDP));
-	MMIO_D(VSYNCSHIFT(TRANSCODER_EDP));
+	MMIO_D(TRANS_VSYNCSHIFT(TRANSCODER_EDP));
 	MMIO_D(PIPE_DATA_M1(TRANSCODER_A));
 	MMIO_D(PIPE_DATA_N1(TRANSCODER_A));
 	MMIO_D(PIPE_DATA_M2(TRANSCODER_A));
@@ -493,9 +498,9 @@ static int iterate_generic_mmio(struct intel_gvt_mmio_table_iter *iter)
 	MMIO_D(GAMMA_MODE(PIPE_A));
 	MMIO_D(GAMMA_MODE(PIPE_B));
 	MMIO_D(GAMMA_MODE(PIPE_C));
-	MMIO_D(PIPE_MULT(PIPE_A));
-	MMIO_D(PIPE_MULT(PIPE_B));
-	MMIO_D(PIPE_MULT(PIPE_C));
+	MMIO_D(TRANS_MULT(TRANSCODER_A));
+	MMIO_D(TRANS_MULT(TRANSCODER_B));
+	MMIO_D(TRANS_MULT(TRANSCODER_C));
 	MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_A));
 	MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_B));
 	MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_C));
@@ -788,9 +793,9 @@ static int iterate_bdw_plus_mmio(struct intel_gvt_mmio_table_iter *iter)
 	MMIO_RING_D(RING_REG);
 #undef RING_REG
 
-	MMIO_D(PIPEMISC(PIPE_A));
-	MMIO_D(PIPEMISC(PIPE_B));
-	MMIO_D(PIPEMISC(PIPE_C));
+	MMIO_D(PIPE_MISC(PIPE_A));
+	MMIO_D(PIPE_MISC(PIPE_B));
+	MMIO_D(PIPE_MISC(PIPE_C));
 	MMIO_D(_MMIO(0x1c1d0));
 	MMIO_D(GEN6_MBCUNIT_SNPCR);
 	MMIO_D(GEN7_MISCCPCTL);
diff --git a/drivers/gpu/drm/i915/intel_pcode.c b/drivers/gpu/drm/i915/intel_pcode.c
index a234d9b4ed14..3db2ba439bb5 100644
--- a/drivers/gpu/drm/i915/intel_pcode.c
+++ b/drivers/gpu/drm/i915/intel_pcode.c
@@ -204,15 +204,42 @@ out:
 #undef COND
 }
 
+static int pcode_init_wait(struct intel_uncore *uncore, int timeout_ms)
+{
+	if (__intel_wait_for_register_fw(uncore,
+					 GEN6_PCODE_MAILBOX,
+					 GEN6_PCODE_READY, 0,
+					 500, timeout_ms,
+					 NULL))
+		return -EPROBE_DEFER;
+
+	return skl_pcode_request(uncore,
+				 DG1_PCODE_STATUS,
+				 DG1_UNCORE_GET_INIT_STATUS,
+				 DG1_UNCORE_INIT_STATUS_COMPLETE,
+				 DG1_UNCORE_INIT_STATUS_COMPLETE, timeout_ms);
+}
+
 int intel_pcode_init(struct intel_uncore *uncore)
 {
+	int err;
+
 	if (!IS_DGFX(uncore->i915))
 		return 0;
 
-	return skl_pcode_request(uncore, DG1_PCODE_STATUS,
-				 DG1_UNCORE_GET_INIT_STATUS,
-				 DG1_UNCORE_INIT_STATUS_COMPLETE,
-				 DG1_UNCORE_INIT_STATUS_COMPLETE, 180000);
+	/*
+	 * Wait 10 seconds so that the punit to settle and complete
+	 * any outstanding transactions upon module load
+	 */
+	err = pcode_init_wait(uncore, 10000);
+
+	if (err) {
+		drm_notice(&uncore->i915->drm,
+			   "Waiting for HW initialisation...\n");
+		err = pcode_init_wait(uncore, 180000);
+	}
+
+	return err;
 }
 
 int snb_pcode_read_p(struct intel_uncore *uncore, u32 mbcmd, u32 p1, u32 p2, u32 *val)
diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h
deleted file mode 100644
index c09b872d65c8..000000000000
--- a/drivers/gpu/drm/i915/intel_pm.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2019 Intel Corporation
- */
-
-#ifndef __INTEL_PM_H__
-#define __INTEL_PM_H__
-
-#include <linux/types.h>
-
-struct drm_i915_private;
-struct intel_crtc_state;
-struct intel_plane_state;
-
-void intel_init_clock_gating(struct drm_i915_private *dev_priv);
-void intel_suspend_hw(struct drm_i915_private *dev_priv);
-int ilk_wm_max_level(const struct drm_i915_private *dev_priv);
-void intel_init_pm(struct drm_i915_private *dev_priv);
-void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv);
-void intel_pm_setup(struct drm_i915_private *dev_priv);
-void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv);
-void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv);
-void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv);
-void g4x_wm_sanitize(struct drm_i915_private *dev_priv);
-void vlv_wm_sanitize(struct drm_i915_private *dev_priv);
-bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv);
-bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
-			    const struct intel_plane_state *plane_state);
-void intel_print_wm_latency(struct drm_i915_private *dev_priv,
-			    const char *name, const u16 wm[]);
-
-bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable);
-
-#endif /* __INTEL_PM_H__ */
diff --git a/drivers/gpu/drm/i915/intel_region_ttm.c b/drivers/gpu/drm/i915/intel_region_ttm.c
index b7fbd5abb42a..bf6097e7433d 100644
--- a/drivers/gpu/drm/i915/intel_region_ttm.c
+++ b/drivers/gpu/drm/i915/intel_region_ttm.c
@@ -181,6 +181,7 @@ intel_region_ttm_resource_to_rsgt(struct intel_memory_region *mem,
 /**
  * intel_region_ttm_resource_alloc - Allocate memory resources from a region
  * @mem: The memory region,
+ * @offset: BO offset
  * @size: The requested size in bytes
  * @flags: Allocation flags
  *
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index 129746713d07..cf5122299b6b 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -652,6 +652,8 @@ void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm)
 
 	rpm->kdev = kdev;
 	rpm->available = HAS_RUNTIME_PM(i915);
+	rpm->suspended = false;
+	atomic_set(&rpm->wakeref_count, 0);
 
 	init_intel_runtime_pm_wakeref(rpm);
 	INIT_LIST_HEAD(&rpm->lmem_userfault_list);
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 8dee9e62a73e..796ebfe6c550 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -32,7 +32,6 @@
 #include "i915_reg.h"
 #include "i915_trace.h"
 #include "i915_vgpu.h"
-#include "intel_pm.h"
 
 #define FORCEWAKE_ACK_TIMEOUT_MS 50
 #define GT_FIFO_TIMEOUT_MS	 10
@@ -178,12 +177,19 @@ wait_ack_set(const struct intel_uncore_forcewake_domain *d,
 static inline void
 fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
 {
-	if (wait_ack_clear(d, FORCEWAKE_KERNEL)) {
+	if (!wait_ack_clear(d, FORCEWAKE_KERNEL))
+		return;
+
+	if (fw_ack(d) == ~0)
+		drm_err(&d->uncore->i915->drm,
+			"%s: MMIO unreliable (forcewake register returns 0xFFFFFFFF)!\n",
+			intel_uncore_forcewake_domain_to_str(d->id));
+	else
 		drm_err(&d->uncore->i915->drm,
 			"%s: timed out waiting for forcewake ack to clear.\n",
 			intel_uncore_forcewake_domain_to_str(d->id));
-		add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */
-	}
+
+	add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */
 }
 
 enum ack_type {
@@ -2460,7 +2466,7 @@ static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
 
 static void uncore_unmap_mmio(struct drm_device *drm, void *regs)
 {
-	iounmap(regs);
+	iounmap((void __iomem *)regs);
 }
 
 int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr)
@@ -2491,7 +2497,8 @@ int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr)
 		return -EIO;
 	}
 
-	return drmm_add_action_or_reset(&i915->drm, uncore_unmap_mmio, uncore->regs);
+	return drmm_add_action_or_reset(&i915->drm, uncore_unmap_mmio,
+					(void __force *)uncore->regs);
 }
 
 void intel_uncore_init_early(struct intel_uncore *uncore,
@@ -2602,11 +2609,45 @@ static int uncore_forcewake_init(struct intel_uncore *uncore)
 	return 0;
 }
 
+static int sanity_check_mmio_access(struct intel_uncore *uncore)
+{
+	struct drm_i915_private *i915 = uncore->i915;
+
+	if (GRAPHICS_VER(i915) < 8)
+		return 0;
+
+	/*
+	 * Sanitycheck that MMIO access to the device is working properly.  If
+	 * the CPU is unable to communcate with a PCI device, BAR reads will
+	 * return 0xFFFFFFFF.  Let's make sure the device isn't in this state
+	 * before we start trying to access registers.
+	 *
+	 * We use the primary GT's forcewake register as our guinea pig since
+	 * it's been around since HSW and it's a masked register so the upper
+	 * 16 bits can never read back as 1's if device access is operating
+	 * properly.
+	 *
+	 * If MMIO isn't working, we'll wait up to 2 seconds to see if it
+	 * recovers, then give up.
+	 */
+#define COND (__raw_uncore_read32(uncore, FORCEWAKE_MT) != ~0)
+	if (wait_for(COND, 2000) == -ETIMEDOUT) {
+		drm_err(&i915->drm, "Device is non-operational; MMIO access returns 0xFFFFFFFF!\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
 int intel_uncore_init_mmio(struct intel_uncore *uncore)
 {
 	struct drm_i915_private *i915 = uncore->i915;
 	int ret;
 
+	ret = sanity_check_mmio_access(uncore);
+	if (ret)
+		return ret;
+
 	/*
 	 * The boot firmware initializes local memory and assesses its health.
 	 * If memory training fails, the punit will have been instructed to
@@ -2748,14 +2789,25 @@ static void driver_initiated_flr(struct intel_uncore *uncore)
 	/* Trigger the actual Driver-FLR */
 	intel_uncore_rmw_fw(uncore, GU_CNTL, 0, DRIVERFLR);
 
+	/* Wait for hardware teardown to complete */
+	ret = intel_wait_for_register_fw(uncore, GU_CNTL,
+					 DRIVERFLR, 0,
+					 flr_timeout_ms);
+	if (ret) {
+		drm_err(&i915->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
+		return;
+	}
+
+	/* Wait for hardware/firmware re-init to complete */
 	ret = intel_wait_for_register_fw(uncore, GU_DEBUG,
 					 DRIVERFLR_STATUS, DRIVERFLR_STATUS,
 					 flr_timeout_ms);
 	if (ret) {
-		drm_err(&i915->drm, "wait for Driver-FLR completion failed! %d\n", ret);
+		drm_err(&i915->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
 		return;
 	}
 
+	/* Clear sticky completion status */
 	intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_wakeref.h b/drivers/gpu/drm/i915/intel_wakeref.h
index 71b8a63f6f10..0b6b4852ab23 100644
--- a/drivers/gpu/drm/i915/intel_wakeref.h
+++ b/drivers/gpu/drm/i915/intel_wakeref.h
@@ -105,7 +105,7 @@ __intel_wakeref_get(struct intel_wakeref *wf)
 }
 
 /**
- * intel_wakeref_get_if_in_use: Acquire the wakeref
+ * intel_wakeref_get_if_active: Acquire the wakeref
  * @wf: the wakeref
  *
  * Acquire a hold on the wakeref, but only if the wakeref is already
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp.c b/drivers/gpu/drm/i915/pxp/intel_pxp.c
index cfc9af8b3d21..9d4c7724e98e 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp.c
@@ -270,6 +270,60 @@ static bool pxp_component_bound(struct intel_pxp *pxp)
 	return bound;
 }
 
+static int __pxp_global_teardown_final(struct intel_pxp *pxp)
+{
+	if (!pxp->arb_is_valid)
+		return 0;
+	/*
+	 * To ensure synchronous and coherent session teardown completion
+	 * in response to suspend or shutdown triggers, don't use a worker.
+	 */
+	intel_pxp_mark_termination_in_progress(pxp);
+	intel_pxp_terminate(pxp, false);
+
+	if (!wait_for_completion_timeout(&pxp->termination, msecs_to_jiffies(250)))
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static int __pxp_global_teardown_restart(struct intel_pxp *pxp)
+{
+	if (pxp->arb_is_valid)
+		return 0;
+	/*
+	 * The arb-session is currently inactive and we are doing a reset and restart
+	 * due to a runtime event. Use the worker that was designed for this.
+	 */
+	pxp_queue_termination(pxp);
+
+	if (!wait_for_completion_timeout(&pxp->termination, msecs_to_jiffies(250)))
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+void intel_pxp_end(struct intel_pxp *pxp)
+{
+	struct drm_i915_private *i915 = pxp->ctrl_gt->i915;
+	intel_wakeref_t wakeref;
+
+	if (!intel_pxp_is_enabled(pxp))
+		return;
+
+	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
+
+	mutex_lock(&pxp->arb_mutex);
+
+	if (__pxp_global_teardown_final(pxp))
+		drm_dbg(&i915->drm, "PXP end timed out\n");
+
+	mutex_unlock(&pxp->arb_mutex);
+
+	intel_pxp_fini_hw(pxp);
+	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
+}
+
 /*
  * the arb session is restarted from the irq work when we receive the
  * termination completion interrupt
@@ -286,16 +340,9 @@ int intel_pxp_start(struct intel_pxp *pxp)
 
 	mutex_lock(&pxp->arb_mutex);
 
-	if (pxp->arb_is_valid)
-		goto unlock;
-
-	pxp_queue_termination(pxp);
-
-	if (!wait_for_completion_timeout(&pxp->termination,
-					msecs_to_jiffies(250))) {
-		ret = -ETIMEDOUT;
+	ret = __pxp_global_teardown_restart(pxp);
+	if (ret)
 		goto unlock;
-	}
 
 	/* make sure the compiler doesn't optimize the double access */
 	barrier();
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp.h b/drivers/gpu/drm/i915/pxp/intel_pxp.h
index 04440fada711..3ded0890cd27 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp.h
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp.h
@@ -24,8 +24,10 @@ void intel_pxp_init_hw(struct intel_pxp *pxp);
 void intel_pxp_fini_hw(struct intel_pxp *pxp);
 
 void intel_pxp_mark_termination_in_progress(struct intel_pxp *pxp);
+void intel_pxp_tee_end_arb_fw_session(struct intel_pxp *pxp, u32 arb_session_id);
 
 int intel_pxp_start(struct intel_pxp *pxp);
+void intel_pxp_end(struct intel_pxp *pxp);
 
 int intel_pxp_key_check(struct intel_pxp *pxp,
 			struct drm_i915_gem_object *obj,
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_cmd_interface_42.h b/drivers/gpu/drm/i915/pxp/intel_pxp_cmd_interface_42.h
index 739f9072fa5f..26f7d9f01bf3 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_cmd_interface_42.h
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_cmd_interface_42.h
@@ -12,6 +12,9 @@
 /* PXP-Opcode for Init Session */
 #define PXP42_CMDID_INIT_SESSION 0x1e
 
+/* PXP-Opcode for Invalidate Stream Key */
+#define PXP42_CMDID_INVALIDATE_STREAM_KEY 0x00000007
+
 /* PXP-Input-Packet: Init Session (Arb-Session) */
 struct pxp42_create_arb_in {
 	struct pxp_cmd_header header;
@@ -25,4 +28,16 @@ struct pxp42_create_arb_out {
 	struct pxp_cmd_header header;
 } __packed;
 
+/* PXP-Input-Packet: Invalidate Stream Key */
+struct pxp42_inv_stream_key_in {
+	struct pxp_cmd_header header;
+	u32 rsvd[3];
+} __packed;
+
+/* PXP-Output-Packet: Invalidate Stream Key */
+struct pxp42_inv_stream_key_out {
+	struct pxp_cmd_header header;
+	u32 rsvd;
+} __packed;
+
 #endif /* __INTEL_PXP_FW_INTERFACE_42_H__ */
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_cmd_interface_cmn.h b/drivers/gpu/drm/i915/pxp/intel_pxp_cmd_interface_cmn.h
index aaa8187a0afb..6f6541d5e49a 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_cmd_interface_cmn.h
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_cmd_interface_cmn.h
@@ -18,6 +18,9 @@
 enum pxp_status {
 	PXP_STATUS_SUCCESS = 0x0,
 	PXP_STATUS_ERROR_API_VERSION = 0x1002,
+	PXP_STATUS_NOT_READY = 0x100e,
+	PXP_STATUS_PLATFCONFIG_KF1_NOVERIF = 0x101a,
+	PXP_STATUS_PLATFCONFIG_KF1_BAD = 0x101f,
 	PXP_STATUS_OP_NOT_PERMITTED = 0x4013
 };
 
@@ -28,6 +31,9 @@ struct pxp_cmd_header {
 	union {
 		u32 status; /* out */
 		u32 stream_id; /* in */
+#define PXP_CMDHDR_EXTDATA_SESSION_VALID GENMASK(0, 0)
+#define PXP_CMDHDR_EXTDATA_APP_TYPE GENMASK(1, 1)
+#define PXP_CMDHDR_EXTDATA_SESSION_ID GENMASK(17, 2)
 	};
 	/* Length of the message (excluding the header) */
 	u32 buffer_len;
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_huc.c b/drivers/gpu/drm/i915/pxp/intel_pxp_huc.c
index 64609d1b1c0f..23431c36b60b 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_huc.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_huc.c
@@ -38,7 +38,7 @@ int intel_pxp_huc_load_and_auth(struct intel_pxp *pxp)
 	huc_in.header.command_id  = PXP43_CMDID_START_HUC_AUTH;
 	huc_in.header.status      = 0;
 	huc_in.header.buffer_len  = sizeof(huc_in.huc_base_address);
-	huc_in.huc_base_address   = huc_phys_addr;
+	huc_in.huc_base_address   = cpu_to_le64(huc_phys_addr);
 
 	err = intel_pxp_tee_stream_message(pxp, client_id, fence_id,
 					   &huc_in, sizeof(huc_in),
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_pm.c b/drivers/gpu/drm/i915/pxp/intel_pxp_pm.c
index 892d39cc61c1..4f836b317424 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_pm.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_pm.c
@@ -16,7 +16,7 @@ void intel_pxp_suspend_prepare(struct intel_pxp *pxp)
 	if (!intel_pxp_is_enabled(pxp))
 		return;
 
-	pxp->arb_is_valid = false;
+	intel_pxp_end(pxp);
 
 	intel_pxp_invalidate(pxp);
 }
@@ -34,7 +34,7 @@ void intel_pxp_suspend(struct intel_pxp *pxp)
 	}
 }
 
-void intel_pxp_resume(struct intel_pxp *pxp)
+void intel_pxp_resume_complete(struct intel_pxp *pxp)
 {
 	if (!intel_pxp_is_enabled(pxp))
 		return;
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h b/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h
index 586be769104f..06b46f535b42 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h
@@ -11,7 +11,7 @@ struct intel_pxp;
 #ifdef CONFIG_DRM_I915_PXP
 void intel_pxp_suspend_prepare(struct intel_pxp *pxp);
 void intel_pxp_suspend(struct intel_pxp *pxp);
-void intel_pxp_resume(struct intel_pxp *pxp);
+void intel_pxp_resume_complete(struct intel_pxp *pxp);
 void intel_pxp_runtime_suspend(struct intel_pxp *pxp);
 #else
 static inline void intel_pxp_suspend_prepare(struct intel_pxp *pxp)
@@ -22,7 +22,7 @@ static inline void intel_pxp_suspend(struct intel_pxp *pxp)
 {
 }
 
-static inline void intel_pxp_resume(struct intel_pxp *pxp)
+static inline void intel_pxp_resume_complete(struct intel_pxp *pxp)
 {
 }
 
@@ -32,6 +32,6 @@ static inline void intel_pxp_runtime_suspend(struct intel_pxp *pxp)
 #endif
 static inline void intel_pxp_runtime_resume(struct intel_pxp *pxp)
 {
-	intel_pxp_resume(pxp);
+	intel_pxp_resume_complete(pxp);
 }
 #endif /* __INTEL_PXP_PM_H__ */
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_session.c b/drivers/gpu/drm/i915/pxp/intel_pxp_session.c
index ae413580b81a..7de849cb6c47 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_session.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_session.c
@@ -74,7 +74,7 @@ static int pxp_create_arb_session(struct intel_pxp *pxp)
 
 	ret = pxp_wait_for_session_state(pxp, ARB_SESSION, true);
 	if (ret) {
-		drm_err(&gt->i915->drm, "arb session failed to go in play\n");
+		drm_dbg(&gt->i915->drm, "arb session failed to go in play\n");
 		return ret;
 	}
 	drm_dbg(&gt->i915->drm, "PXP ARB session is alive\n");
@@ -110,14 +110,16 @@ static int pxp_terminate_arb_session_and_global(struct intel_pxp *pxp)
 
 	intel_uncore_write(gt->uncore, PXP_GLOBAL_TERMINATE, 1);
 
+	intel_pxp_tee_end_arb_fw_session(pxp, ARB_SESSION);
+
 	return ret;
 }
 
-static void pxp_terminate(struct intel_pxp *pxp)
+void intel_pxp_terminate(struct intel_pxp *pxp, bool post_invalidation_needs_restart)
 {
 	int ret;
 
-	pxp->hw_state_invalidated = true;
+	pxp->hw_state_invalidated = post_invalidation_needs_restart;
 
 	/*
 	 * if we fail to submit the termination there is no point in waiting for
@@ -165,7 +167,7 @@ static void pxp_session_work(struct work_struct *work)
 
 	if (events & PXP_TERMINATION_REQUEST) {
 		events &= ~PXP_TERMINATION_COMPLETE;
-		pxp_terminate(pxp);
+		intel_pxp_terminate(pxp, true);
 	}
 
 	if (events & PXP_TERMINATION_COMPLETE)
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_session.h b/drivers/gpu/drm/i915/pxp/intel_pxp_session.h
index 903ac52cffa1..ba5788127109 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_session.h
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_session.h
@@ -12,9 +12,14 @@ struct intel_pxp;
 
 #ifdef CONFIG_DRM_I915_PXP
 void intel_pxp_session_management_init(struct intel_pxp *pxp);
+void intel_pxp_terminate(struct intel_pxp *pxp, bool post_invalidation_needs_restart);
 #else
 static inline void intel_pxp_session_management_init(struct intel_pxp *pxp)
 {
 }
+
+static inline void intel_pxp_terminate(struct intel_pxp *pxp, bool post_invalidation_needs_restart)
+{
+}
 #endif
 #endif /* __INTEL_PXP_SESSION_H__ */
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_tee.c b/drivers/gpu/drm/i915/pxp/intel_pxp_tee.c
index 73aa8015f828..a2846b1dbbee 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_tee.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_tee.c
@@ -19,6 +19,37 @@
 #include "intel_pxp_tee.h"
 #include "intel_pxp_types.h"
 
+static bool
+is_fw_err_platform_config(u32 type)
+{
+	switch (type) {
+	case PXP_STATUS_ERROR_API_VERSION:
+	case PXP_STATUS_PLATFCONFIG_KF1_NOVERIF:
+	case PXP_STATUS_PLATFCONFIG_KF1_BAD:
+		return true;
+	default:
+		break;
+	}
+	return false;
+}
+
+static const char *
+fw_err_to_string(u32 type)
+{
+	switch (type) {
+	case PXP_STATUS_ERROR_API_VERSION:
+		return "ERR_API_VERSION";
+	case PXP_STATUS_NOT_READY:
+		return "ERR_NOT_READY";
+	case PXP_STATUS_PLATFCONFIG_KF1_NOVERIF:
+	case PXP_STATUS_PLATFCONFIG_KF1_BAD:
+		return "ERR_PLATFORM_CONFIG";
+	default:
+		break;
+	}
+	return NULL;
+}
+
 static int intel_pxp_tee_io_message(struct intel_pxp *pxp,
 				    void *msg_in, u32 msg_in_size,
 				    void *msg_out, u32 msg_out_max_size,
@@ -127,6 +158,12 @@ static int i915_pxp_tee_component_bind(struct device *i915_kdev,
 	intel_wakeref_t wakeref;
 	int ret = 0;
 
+	if (!HAS_HECI_PXP(i915)) {
+		pxp->dev_link = device_link_add(i915_kdev, tee_kdev, DL_FLAG_STATELESS);
+		if (drm_WARN_ON(&i915->drm, !pxp->dev_link))
+			return -ENODEV;
+	}
+
 	mutex_lock(&pxp->tee_mutex);
 	pxp->pxp_component = data;
 	pxp->pxp_component->tee_dev = tee_kdev;
@@ -169,6 +206,11 @@ static void i915_pxp_tee_component_unbind(struct device *i915_kdev,
 	mutex_lock(&pxp->tee_mutex);
 	pxp->pxp_component = NULL;
 	mutex_unlock(&pxp->tee_mutex);
+
+	if (pxp->dev_link) {
+		device_link_del(pxp->dev_link);
+		pxp->dev_link = NULL;
+	}
 }
 
 static const struct component_ops i915_pxp_tee_component_ops = {
@@ -296,15 +338,68 @@ int intel_pxp_tee_cmd_create_arb_session(struct intel_pxp *pxp,
 				       &msg_out, sizeof(msg_out),
 				       NULL);
 
-	if (ret)
-		drm_err(&i915->drm, "Failed to send tee msg ret=[%d]\n", ret);
-	else if (msg_out.header.status == PXP_STATUS_ERROR_API_VERSION)
-		drm_dbg(&i915->drm, "PXP firmware version unsupported, requested: "
-			"CMD-ID-[0x%08x] on API-Ver-[0x%08x]\n",
-			msg_in.header.command_id, msg_in.header.api_version);
-	else if (msg_out.header.status != 0x0)
-		drm_warn(&i915->drm, "PXP firmware failed arb session init request ret=[0x%08x]\n",
-			 msg_out.header.status);
+	if (ret) {
+		drm_err(&i915->drm, "Failed to send tee msg init arb session, ret=[%d]\n", ret);
+	} else if (msg_out.header.status != 0) {
+		if (is_fw_err_platform_config(msg_out.header.status)) {
+			drm_info_once(&i915->drm,
+				      "PXP init-arb-session-%d failed due to BIOS/SOC:0x%08x:%s\n",
+				      arb_session_id, msg_out.header.status,
+				      fw_err_to_string(msg_out.header.status));
+		} else {
+			drm_dbg(&i915->drm, "PXP init-arb-session--%d failed 0x%08x:%st:\n",
+				arb_session_id, msg_out.header.status,
+				fw_err_to_string(msg_out.header.status));
+			drm_dbg(&i915->drm, "     cmd-detail: ID=[0x%08x],API-Ver-[0x%08x]\n",
+				msg_in.header.command_id, msg_in.header.api_version);
+		}
+	}
 
 	return ret;
 }
+
+void intel_pxp_tee_end_arb_fw_session(struct intel_pxp *pxp, u32 session_id)
+{
+	struct drm_i915_private *i915 = pxp->ctrl_gt->i915;
+	struct pxp42_inv_stream_key_in msg_in = {0};
+	struct pxp42_inv_stream_key_out msg_out = {0};
+	int ret, trials = 0;
+
+try_again:
+	memset(&msg_in, 0, sizeof(msg_in));
+	memset(&msg_out, 0, sizeof(msg_out));
+	msg_in.header.api_version = PXP_APIVER(4, 2);
+	msg_in.header.command_id = PXP42_CMDID_INVALIDATE_STREAM_KEY;
+	msg_in.header.buffer_len = sizeof(msg_in) - sizeof(msg_in.header);
+
+	msg_in.header.stream_id = FIELD_PREP(PXP_CMDHDR_EXTDATA_SESSION_VALID, 1);
+	msg_in.header.stream_id |= FIELD_PREP(PXP_CMDHDR_EXTDATA_APP_TYPE, 0);
+	msg_in.header.stream_id |= FIELD_PREP(PXP_CMDHDR_EXTDATA_SESSION_ID, session_id);
+
+	ret = intel_pxp_tee_io_message(pxp,
+				       &msg_in, sizeof(msg_in),
+				       &msg_out, sizeof(msg_out),
+				       NULL);
+
+	/* Cleanup coherency between GT and Firmware is critical, so try again if it fails */
+	if ((ret || msg_out.header.status != 0x0) && ++trials < 3)
+		goto try_again;
+
+	if (ret) {
+		drm_err(&i915->drm, "Failed to send tee msg for inv-stream-key-%u, ret=[%d]\n",
+			session_id, ret);
+	} else if (msg_out.header.status != 0) {
+		if (is_fw_err_platform_config(msg_out.header.status)) {
+			drm_info_once(&i915->drm,
+				      "PXP inv-stream-key-%u failed due to BIOS/SOC :0x%08x:%s\n",
+				      session_id, msg_out.header.status,
+				      fw_err_to_string(msg_out.header.status));
+		} else {
+			drm_dbg(&i915->drm, "PXP inv-stream-key-%u failed 0x%08x:%s:\n",
+				session_id, msg_out.header.status,
+				fw_err_to_string(msg_out.header.status));
+			drm_dbg(&i915->drm, "     cmd-detail: ID=[0x%08x],API-Ver-[0x%08x]\n",
+				msg_in.header.command_id, msg_in.header.api_version);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_types.h b/drivers/gpu/drm/i915/pxp/intel_pxp_types.h
index 7dc5f08d1583..007de49e1ea4 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_types.h
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_types.h
@@ -32,6 +32,9 @@ struct intel_pxp {
 	 * which are protected by &tee_mutex.
 	 */
 	struct i915_pxp_component *pxp_component;
+
+	/* @dev_link: Enforce module relationship for power management ordering. */
+	struct device_link *dev_link;
 	/**
 	 * @pxp_component_added: track if the pxp component has been added.
 	 * Set and cleared in tee init and fini functions respectively.
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index 01e75160a84a..154801f1c468 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -69,8 +69,10 @@ static int fake_get_pages(struct drm_i915_gem_object *obj)
 
 	rem = round_up(obj->base.size, BIT(31)) >> 31;
 	/* restricted by sg_alloc_table */
-	if (overflows_type(rem, unsigned int))
+	if (overflows_type(rem, unsigned int)) {
+		kfree(pages);
 		return -E2BIG;
+	}
 
 	if (sg_alloc_table(pages, rem, GFP)) {
 		kfree(pages);
@@ -1940,361 +1942,6 @@ out_put:
 	return err;
 }
 
-static int context_sync(struct intel_context *ce)
-{
-	struct i915_request *rq;
-	long timeout;
-
-	rq = intel_context_create_request(ce);
-	if (IS_ERR(rq))
-		return PTR_ERR(rq);
-
-	i915_request_get(rq);
-	i915_request_add(rq);
-
-	timeout = i915_request_wait(rq, 0, HZ / 5);
-	i915_request_put(rq);
-
-	return timeout < 0 ? -EIO : 0;
-}
-
-static struct i915_request *
-submit_batch(struct intel_context *ce, u64 addr)
-{
-	struct i915_request *rq;
-	int err;
-
-	rq = intel_context_create_request(ce);
-	if (IS_ERR(rq))
-		return rq;
-
-	err = 0;
-	if (rq->engine->emit_init_breadcrumb) /* detect a hang */
-		err = rq->engine->emit_init_breadcrumb(rq);
-	if (err == 0)
-		err = rq->engine->emit_bb_start(rq, addr, 0, 0);
-
-	if (err == 0)
-		i915_request_get(rq);
-	i915_request_add(rq);
-
-	return err ? ERR_PTR(err) : rq;
-}
-
-static u32 *spinner(u32 *batch, int i)
-{
-	return batch + i * 64 / sizeof(*batch) + 4;
-}
-
-static void end_spin(u32 *batch, int i)
-{
-	*spinner(batch, i) = MI_BATCH_BUFFER_END;
-	wmb();
-}
-
-static int igt_cs_tlb(void *arg)
-{
-	const unsigned int count = PAGE_SIZE / 64;
-	const unsigned int chunk_size = count * PAGE_SIZE;
-	struct drm_i915_private *i915 = arg;
-	struct drm_i915_gem_object *bbe, *act, *out;
-	struct i915_gem_engines_iter it;
-	struct i915_address_space *vm;
-	struct i915_gem_context *ctx;
-	struct intel_context *ce;
-	struct i915_vma *vma;
-	I915_RND_STATE(prng);
-	struct file *file;
-	unsigned int i;
-	u32 *result;
-	u32 *batch;
-	int err = 0;
-
-	/*
-	 * Our mission here is to fool the hardware to execute something
-	 * from scratch as it has not seen the batch move (due to missing
-	 * the TLB invalidate).
-	 */
-
-	file = mock_file(i915);
-	if (IS_ERR(file))
-		return PTR_ERR(file);
-
-	ctx = live_context(i915, file);
-	if (IS_ERR(ctx)) {
-		err = PTR_ERR(ctx);
-		goto out_unlock;
-	}
-
-	vm = i915_gem_context_get_eb_vm(ctx);
-	if (i915_is_ggtt(vm))
-		goto out_vm;
-
-	/* Create two pages; dummy we prefill the TLB, and intended */
-	bbe = i915_gem_object_create_internal(i915, PAGE_SIZE);
-	if (IS_ERR(bbe)) {
-		err = PTR_ERR(bbe);
-		goto out_vm;
-	}
-
-	batch = i915_gem_object_pin_map_unlocked(bbe, I915_MAP_WC);
-	if (IS_ERR(batch)) {
-		err = PTR_ERR(batch);
-		goto out_put_bbe;
-	}
-	memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32));
-	i915_gem_object_flush_map(bbe);
-	i915_gem_object_unpin_map(bbe);
-
-	act = i915_gem_object_create_internal(i915, PAGE_SIZE);
-	if (IS_ERR(act)) {
-		err = PTR_ERR(act);
-		goto out_put_bbe;
-	}
-
-	/* Track the execution of each request by writing into different slot */
-	batch = i915_gem_object_pin_map_unlocked(act, I915_MAP_WC);
-	if (IS_ERR(batch)) {
-		err = PTR_ERR(batch);
-		goto out_put_act;
-	}
-	for (i = 0; i < count; i++) {
-		u32 *cs = batch + i * 64 / sizeof(*cs);
-		u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32);
-
-		GEM_BUG_ON(GRAPHICS_VER(i915) < 6);
-		cs[0] = MI_STORE_DWORD_IMM_GEN4;
-		if (GRAPHICS_VER(i915) >= 8) {
-			cs[1] = lower_32_bits(addr);
-			cs[2] = upper_32_bits(addr);
-			cs[3] = i;
-			cs[4] = MI_NOOP;
-			cs[5] = MI_BATCH_BUFFER_START_GEN8;
-		} else {
-			cs[1] = 0;
-			cs[2] = lower_32_bits(addr);
-			cs[3] = i;
-			cs[4] = MI_NOOP;
-			cs[5] = MI_BATCH_BUFFER_START;
-		}
-	}
-
-	out = i915_gem_object_create_internal(i915, PAGE_SIZE);
-	if (IS_ERR(out)) {
-		err = PTR_ERR(out);
-		goto out_put_batch;
-	}
-	i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED);
-
-	vma = i915_vma_instance(out, vm, NULL);
-	if (IS_ERR(vma)) {
-		err = PTR_ERR(vma);
-		goto out_put_out;
-	}
-
-	err = i915_vma_pin(vma, 0, 0,
-			   PIN_USER |
-			   PIN_OFFSET_FIXED |
-			   (vm->total - PAGE_SIZE));
-	if (err)
-		goto out_put_out;
-	GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
-
-	result = i915_gem_object_pin_map_unlocked(out, I915_MAP_WB);
-	if (IS_ERR(result)) {
-		err = PTR_ERR(result);
-		goto out_put_out;
-	}
-
-	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
-		IGT_TIMEOUT(end_time);
-		unsigned long pass = 0;
-
-		if (!intel_engine_can_store_dword(ce->engine))
-			continue;
-
-		while (!__igt_timeout(end_time, NULL)) {
-			struct i915_vm_pt_stash stash = {};
-			struct i915_request *rq;
-			struct i915_gem_ww_ctx ww;
-			struct i915_vma_resource *vma_res;
-			u64 offset;
-
-			offset = igt_random_offset(&prng,
-						   0, vm->total - PAGE_SIZE,
-						   chunk_size, PAGE_SIZE);
-
-			memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32));
-
-			vma = i915_vma_instance(bbe, vm, NULL);
-			if (IS_ERR(vma)) {
-				err = PTR_ERR(vma);
-				goto end;
-			}
-
-			i915_gem_object_lock(bbe, NULL);
-			err = i915_vma_get_pages(vma);
-			i915_gem_object_unlock(bbe);
-			if (err)
-				goto end;
-
-			vma_res = i915_vma_resource_alloc();
-			if (IS_ERR(vma_res)) {
-				i915_vma_put_pages(vma);
-				err = PTR_ERR(vma_res);
-				goto end;
-			}
-
-			i915_gem_ww_ctx_init(&ww, false);
-retry:
-			err = i915_vm_lock_objects(vm, &ww);
-			if (err)
-				goto end_ww;
-
-			err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size);
-			if (err)
-				goto end_ww;
-
-			err = i915_vm_map_pt_stash(vm, &stash);
-			if (!err)
-				vm->allocate_va_range(vm, &stash, offset, chunk_size);
-			i915_vm_free_pt_stash(vm, &stash);
-end_ww:
-			if (err == -EDEADLK) {
-				err = i915_gem_ww_ctx_backoff(&ww);
-				if (!err)
-					goto retry;
-			}
-			i915_gem_ww_ctx_fini(&ww);
-			if (err) {
-				kfree(vma_res);
-				goto end;
-			}
-
-			i915_vma_resource_init_from_vma(vma_res, vma);
-			/* Prime the TLB with the dummy pages */
-			for (i = 0; i < count; i++) {
-				vma_res->start = offset + i * PAGE_SIZE;
-				vm->insert_entries(vm, vma_res, I915_CACHE_NONE,
-						   0);
-
-				rq = submit_batch(ce, vma_res->start);
-				if (IS_ERR(rq)) {
-					err = PTR_ERR(rq);
-					i915_vma_resource_fini(vma_res);
-					kfree(vma_res);
-					goto end;
-				}
-				i915_request_put(rq);
-			}
-			i915_vma_resource_fini(vma_res);
-			i915_vma_put_pages(vma);
-
-			err = context_sync(ce);
-			if (err) {
-				pr_err("%s: dummy setup timed out\n",
-				       ce->engine->name);
-				kfree(vma_res);
-				goto end;
-			}
-
-			vma = i915_vma_instance(act, vm, NULL);
-			if (IS_ERR(vma)) {
-				kfree(vma_res);
-				err = PTR_ERR(vma);
-				goto end;
-			}
-
-			i915_gem_object_lock(act, NULL);
-			err = i915_vma_get_pages(vma);
-			i915_gem_object_unlock(act);
-			if (err) {
-				kfree(vma_res);
-				goto end;
-			}
-
-			i915_vma_resource_init_from_vma(vma_res, vma);
-			/* Replace the TLB with target batches */
-			for (i = 0; i < count; i++) {
-				struct i915_request *rq;
-				u32 *cs = batch + i * 64 / sizeof(*cs);
-				u64 addr;
-
-				vma_res->start = offset + i * PAGE_SIZE;
-				vm->insert_entries(vm, vma_res, I915_CACHE_NONE, 0);
-
-				addr = vma_res->start + i * 64;
-				cs[4] = MI_NOOP;
-				cs[6] = lower_32_bits(addr);
-				cs[7] = upper_32_bits(addr);
-				wmb();
-
-				rq = submit_batch(ce, addr);
-				if (IS_ERR(rq)) {
-					err = PTR_ERR(rq);
-					i915_vma_resource_fini(vma_res);
-					kfree(vma_res);
-					goto end;
-				}
-
-				/* Wait until the context chain has started */
-				if (i == 0) {
-					while (READ_ONCE(result[i]) &&
-					       !i915_request_completed(rq))
-						cond_resched();
-				} else {
-					end_spin(batch, i - 1);
-				}
-
-				i915_request_put(rq);
-			}
-			end_spin(batch, count - 1);
-
-			i915_vma_resource_fini(vma_res);
-			kfree(vma_res);
-			i915_vma_put_pages(vma);
-
-			err = context_sync(ce);
-			if (err) {
-				pr_err("%s: writes timed out\n",
-				       ce->engine->name);
-				goto end;
-			}
-
-			for (i = 0; i < count; i++) {
-				if (result[i] != i) {
-					pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n",
-					       ce->engine->name, pass,
-					       offset, i, result[i], i);
-					err = -EINVAL;
-					goto end;
-				}
-			}
-
-			vm->clear_range(vm, offset, chunk_size);
-			pass++;
-		}
-	}
-end:
-	if (igt_flush_test(i915))
-		err = -EIO;
-	i915_gem_context_unlock_engines(ctx);
-	i915_gem_object_unpin_map(out);
-out_put_out:
-	i915_gem_object_put(out);
-out_put_batch:
-	i915_gem_object_unpin_map(act);
-out_put_act:
-	i915_gem_object_put(act);
-out_put_bbe:
-	i915_gem_object_put(bbe);
-out_vm:
-	i915_vm_put(vm);
-out_unlock:
-	fput(file);
-	return err;
-}
-
 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
 {
 	static const struct i915_subtest tests[] = {
@@ -2314,7 +1961,6 @@ int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_ggtt_fill),
 		SUBTEST(igt_ggtt_page),
 		SUBTEST(igt_ggtt_misaligned_pin),
-		SUBTEST(igt_cs_tlb),
 	};
 
 	GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total));
diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
index aaf8a380e5c7..5aee6c9a8295 100644
--- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
@@ -25,6 +25,7 @@ selftest(gt_lrc, intel_lrc_live_selftests)
 selftest(gt_mocs, intel_mocs_live_selftests)
 selftest(gt_pm, intel_gt_pm_live_selftests)
 selftest(gt_heartbeat, intel_heartbeat_live_selftests)
+selftest(gt_tlb, intel_tlb_live_selftests)
 selftest(requests, i915_request_live_selftests)
 selftest(migrate, intel_migrate_live_selftests)
 selftest(active, i915_active_live_selftests)
diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c
index 6fe22b096bdd..a9b79888c193 100644
--- a/drivers/gpu/drm/i915/selftests/i915_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_request.c
@@ -957,18 +957,18 @@ static int live_cancel_request(void *arg)
 	return 0;
 }
 
-static struct i915_vma *empty_batch(struct drm_i915_private *i915)
+static struct i915_vma *empty_batch(struct intel_gt *gt)
 {
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
 	u32 *cmd;
 	int err;
 
-	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
+	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
 	if (IS_ERR(obj))
 		return ERR_CAST(obj);
 
-	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
+	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(cmd)) {
 		err = PTR_ERR(cmd);
 		goto err;
@@ -979,15 +979,15 @@ static struct i915_vma *empty_batch(struct drm_i915_private *i915)
 	__i915_gem_object_flush_map(obj, 0, 64);
 	i915_gem_object_unpin_map(obj);
 
-	intel_gt_chipset_flush(to_gt(i915));
+	intel_gt_chipset_flush(gt);
 
-	vma = i915_vma_instance(obj, &to_gt(i915)->ggtt->vm, NULL);
+	vma = i915_vma_instance(obj, gt->vm, NULL);
 	if (IS_ERR(vma)) {
 		err = PTR_ERR(vma);
 		goto err;
 	}
 
-	err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_GLOBAL);
+	err = i915_vma_pin(vma, 0, 0, PIN_USER);
 	if (err)
 		goto err;
 
@@ -1005,6 +1005,14 @@ err:
 	return ERR_PTR(err);
 }
 
+static int emit_bb_start(struct i915_request *rq, struct i915_vma *batch)
+{
+	return rq->engine->emit_bb_start(rq,
+					 i915_vma_offset(batch),
+					 i915_vma_size(batch),
+					 0);
+}
+
 static struct i915_request *
 empty_request(struct intel_engine_cs *engine,
 	      struct i915_vma *batch)
@@ -1016,10 +1024,7 @@ empty_request(struct intel_engine_cs *engine,
 	if (IS_ERR(request))
 		return request;
 
-	err = engine->emit_bb_start(request,
-				    i915_vma_offset(batch),
-				    i915_vma_size(batch),
-				    I915_DISPATCH_SECURE);
+	err = emit_bb_start(request, batch);
 	if (err)
 		goto out_request;
 
@@ -1034,8 +1039,7 @@ static int live_empty_request(void *arg)
 	struct drm_i915_private *i915 = arg;
 	struct intel_engine_cs *engine;
 	struct igt_live_test t;
-	struct i915_vma *batch;
-	int err = 0;
+	int err;
 
 	/*
 	 * Submit various sized batches of empty requests, to each engine
@@ -1043,16 +1047,17 @@ static int live_empty_request(void *arg)
 	 * the overhead of submitting requests to the hardware.
 	 */
 
-	batch = empty_batch(i915);
-	if (IS_ERR(batch))
-		return PTR_ERR(batch);
-
 	for_each_uabi_engine(engine, i915) {
 		IGT_TIMEOUT(end_time);
 		struct i915_request *request;
+		struct i915_vma *batch;
 		unsigned long n, prime;
 		ktime_t times[2] = {};
 
+		batch = empty_batch(engine->gt);
+		if (IS_ERR(batch))
+			return PTR_ERR(batch);
+
 		err = igt_live_test_begin(&t, i915, __func__, engine->name);
 		if (err)
 			goto out_batch;
@@ -1100,27 +1105,29 @@ static int live_empty_request(void *arg)
 			engine->name,
 			ktime_to_ns(times[0]),
 			prime, div64_u64(ktime_to_ns(times[1]), prime));
+out_batch:
+		i915_vma_unpin(batch);
+		i915_vma_put(batch);
+		if (err)
+			break;
 	}
 
-out_batch:
-	i915_vma_unpin(batch);
-	i915_vma_put(batch);
 	return err;
 }
 
-static struct i915_vma *recursive_batch(struct drm_i915_private *i915)
+static struct i915_vma *recursive_batch(struct intel_gt *gt)
 {
 	struct drm_i915_gem_object *obj;
-	const int ver = GRAPHICS_VER(i915);
+	const int ver = GRAPHICS_VER(gt->i915);
 	struct i915_vma *vma;
 	u32 *cmd;
 	int err;
 
-	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
+	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
 	if (IS_ERR(obj))
 		return ERR_CAST(obj);
 
-	vma = i915_vma_instance(obj, to_gt(i915)->vm, NULL);
+	vma = i915_vma_instance(obj, gt->vm, NULL);
 	if (IS_ERR(vma)) {
 		err = PTR_ERR(vma);
 		goto err;
@@ -1152,7 +1159,7 @@ static struct i915_vma *recursive_batch(struct drm_i915_private *i915)
 	__i915_gem_object_flush_map(obj, 0, 64);
 	i915_gem_object_unpin_map(obj);
 
-	intel_gt_chipset_flush(to_gt(i915));
+	intel_gt_chipset_flush(gt);
 
 	return vma;
 
@@ -1186,7 +1193,6 @@ static int live_all_engines(void *arg)
 	struct intel_engine_cs *engine;
 	struct i915_request **request;
 	struct igt_live_test t;
-	struct i915_vma *batch;
 	unsigned int idx;
 	int err;
 
@@ -1204,42 +1210,44 @@ static int live_all_engines(void *arg)
 	if (err)
 		goto out_free;
 
-	batch = recursive_batch(i915);
-	if (IS_ERR(batch)) {
-		err = PTR_ERR(batch);
-		pr_err("%s: Unable to create batch, err=%d\n", __func__, err);
-		goto out_free;
-	}
-
-	i915_vma_lock(batch);
-
 	idx = 0;
 	for_each_uabi_engine(engine, i915) {
+		struct i915_vma *batch;
+
+		batch = recursive_batch(engine->gt);
+		if (IS_ERR(batch)) {
+			err = PTR_ERR(batch);
+			pr_err("%s: Unable to create batch, err=%d\n",
+			       __func__, err);
+			goto out_free;
+		}
+
+		i915_vma_lock(batch);
 		request[idx] = intel_engine_create_kernel_request(engine);
 		if (IS_ERR(request[idx])) {
 			err = PTR_ERR(request[idx]);
 			pr_err("%s: Request allocation failed with err=%d\n",
 			       __func__, err);
-			goto out_request;
+			goto out_unlock;
 		}
+		GEM_BUG_ON(request[idx]->context->vm != batch->vm);
 
 		err = i915_vma_move_to_active(batch, request[idx], 0);
 		GEM_BUG_ON(err);
 
-		err = engine->emit_bb_start(request[idx],
-					    i915_vma_offset(batch),
-					    i915_vma_size(batch),
-					    0);
+		err = emit_bb_start(request[idx], batch);
 		GEM_BUG_ON(err);
 		request[idx]->batch = batch;
 
 		i915_request_get(request[idx]);
 		i915_request_add(request[idx]);
 		idx++;
+out_unlock:
+		i915_vma_unlock(batch);
+		if (err)
+			goto out_request;
 	}
 
-	i915_vma_unlock(batch);
-
 	idx = 0;
 	for_each_uabi_engine(engine, i915) {
 		if (i915_request_completed(request[idx])) {
@@ -1251,17 +1259,23 @@ static int live_all_engines(void *arg)
 		idx++;
 	}
 
-	err = recursive_batch_resolve(batch);
-	if (err) {
-		pr_err("%s: failed to resolve batch, err=%d\n", __func__, err);
-		goto out_request;
+	idx = 0;
+	for_each_uabi_engine(engine, i915) {
+		err = recursive_batch_resolve(request[idx]->batch);
+		if (err) {
+			pr_err("%s: failed to resolve batch, err=%d\n",
+			       __func__, err);
+			goto out_request;
+		}
+		idx++;
 	}
 
 	idx = 0;
 	for_each_uabi_engine(engine, i915) {
+		struct i915_request *rq = request[idx];
 		long timeout;
 
-		timeout = i915_request_wait(request[idx], 0,
+		timeout = i915_request_wait(rq, 0,
 					    MAX_SCHEDULE_TIMEOUT);
 		if (timeout < 0) {
 			err = timeout;
@@ -1270,8 +1284,10 @@ static int live_all_engines(void *arg)
 			goto out_request;
 		}
 
-		GEM_BUG_ON(!i915_request_completed(request[idx]));
-		i915_request_put(request[idx]);
+		GEM_BUG_ON(!i915_request_completed(rq));
+		i915_vma_unpin(rq->batch);
+		i915_vma_put(rq->batch);
+		i915_request_put(rq);
 		request[idx] = NULL;
 		idx++;
 	}
@@ -1281,12 +1297,18 @@ static int live_all_engines(void *arg)
 out_request:
 	idx = 0;
 	for_each_uabi_engine(engine, i915) {
-		if (request[idx])
-			i915_request_put(request[idx]);
+		struct i915_request *rq = request[idx];
+
+		if (!rq)
+			continue;
+
+		if (rq->batch) {
+			i915_vma_unpin(rq->batch);
+			i915_vma_put(rq->batch);
+		}
+		i915_request_put(rq);
 		idx++;
 	}
-	i915_vma_unpin(batch);
-	i915_vma_put(batch);
 out_free:
 	kfree(request);
 	return err;
@@ -1322,7 +1344,7 @@ static int live_sequential_engines(void *arg)
 	for_each_uabi_engine(engine, i915) {
 		struct i915_vma *batch;
 
-		batch = recursive_batch(i915);
+		batch = recursive_batch(engine->gt);
 		if (IS_ERR(batch)) {
 			err = PTR_ERR(batch);
 			pr_err("%s: Unable to create batch for %s, err=%d\n",
@@ -1338,6 +1360,7 @@ static int live_sequential_engines(void *arg)
 			       __func__, engine->name, err);
 			goto out_unlock;
 		}
+		GEM_BUG_ON(request[idx]->context->vm != batch->vm);
 
 		if (prev) {
 			err = i915_request_await_dma_fence(request[idx],
@@ -1353,10 +1376,7 @@ static int live_sequential_engines(void *arg)
 		err = i915_vma_move_to_active(batch, request[idx], 0);
 		GEM_BUG_ON(err);
 
-		err = engine->emit_bb_start(request[idx],
-					    i915_vma_offset(batch),
-					    i915_vma_size(batch),
-					    0);
+		err = emit_bb_start(request[idx], batch);
 		GEM_BUG_ON(err);
 		request[idx]->batch = batch;
 
diff --git a/drivers/gpu/drm/i915/soc/intel_dram.c b/drivers/gpu/drm/i915/soc/intel_dram.c
index bba8cb6e8ae4..9f0651d48d41 100644
--- a/drivers/gpu/drm/i915/soc/intel_dram.c
+++ b/drivers/gpu/drm/i915/soc/intel_dram.c
@@ -10,6 +10,7 @@
 #include "intel_dram.h"
 #include "intel_mchbar_regs.h"
 #include "intel_pcode.h"
+#include "vlv_sideband.h"
 
 struct dram_dimm_info {
 	u16 size;
@@ -42,6 +43,155 @@ static const char *intel_dram_type_str(enum intel_dram_type type)
 
 #undef DRAM_TYPE_STR
 
+static void pnv_detect_mem_freq(struct drm_i915_private *dev_priv)
+{
+	u32 tmp;
+
+	tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
+
+	switch (tmp & CLKCFG_FSB_MASK) {
+	case CLKCFG_FSB_533:
+		dev_priv->fsb_freq = 533; /* 133*4 */
+		break;
+	case CLKCFG_FSB_800:
+		dev_priv->fsb_freq = 800; /* 200*4 */
+		break;
+	case CLKCFG_FSB_667:
+		dev_priv->fsb_freq =  667; /* 167*4 */
+		break;
+	case CLKCFG_FSB_400:
+		dev_priv->fsb_freq = 400; /* 100*4 */
+		break;
+	}
+
+	switch (tmp & CLKCFG_MEM_MASK) {
+	case CLKCFG_MEM_533:
+		dev_priv->mem_freq = 533;
+		break;
+	case CLKCFG_MEM_667:
+		dev_priv->mem_freq = 667;
+		break;
+	case CLKCFG_MEM_800:
+		dev_priv->mem_freq = 800;
+		break;
+	}
+
+	/* detect pineview DDR3 setting */
+	tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
+	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
+}
+
+static void ilk_detect_mem_freq(struct drm_i915_private *dev_priv)
+{
+	u16 ddrpll, csipll;
+
+	ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
+	switch (ddrpll & 0xff) {
+	case 0xc:
+		dev_priv->mem_freq = 800;
+		break;
+	case 0x10:
+		dev_priv->mem_freq = 1066;
+		break;
+	case 0x14:
+		dev_priv->mem_freq = 1333;
+		break;
+	case 0x18:
+		dev_priv->mem_freq = 1600;
+		break;
+	default:
+		drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
+			ddrpll & 0xff);
+		dev_priv->mem_freq = 0;
+		break;
+	}
+
+	csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
+	switch (csipll & 0x3ff) {
+	case 0x00c:
+		dev_priv->fsb_freq = 3200;
+		break;
+	case 0x00e:
+		dev_priv->fsb_freq = 3733;
+		break;
+	case 0x010:
+		dev_priv->fsb_freq = 4266;
+		break;
+	case 0x012:
+		dev_priv->fsb_freq = 4800;
+		break;
+	case 0x014:
+		dev_priv->fsb_freq = 5333;
+		break;
+	case 0x016:
+		dev_priv->fsb_freq = 5866;
+		break;
+	case 0x018:
+		dev_priv->fsb_freq = 6400;
+		break;
+	default:
+		drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
+			csipll & 0x3ff);
+		dev_priv->fsb_freq = 0;
+		break;
+	}
+}
+
+static void chv_detect_mem_freq(struct drm_i915_private *i915)
+{
+	u32 val;
+
+	vlv_iosf_sb_get(i915, BIT(VLV_IOSF_SB_CCK));
+	val = vlv_cck_read(i915, CCK_FUSE_REG);
+	vlv_iosf_sb_put(i915, BIT(VLV_IOSF_SB_CCK));
+
+	switch ((val >> 2) & 0x7) {
+	case 3:
+		i915->mem_freq = 2000;
+		break;
+	default:
+		i915->mem_freq = 1600;
+		break;
+	}
+}
+
+static void vlv_detect_mem_freq(struct drm_i915_private *i915)
+{
+	u32 val;
+
+	vlv_iosf_sb_get(i915, BIT(VLV_IOSF_SB_PUNIT));
+	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
+	vlv_iosf_sb_put(i915, BIT(VLV_IOSF_SB_PUNIT));
+
+	switch ((val >> 6) & 3) {
+	case 0:
+	case 1:
+		i915->mem_freq = 800;
+		break;
+	case 2:
+		i915->mem_freq = 1066;
+		break;
+	case 3:
+		i915->mem_freq = 1333;
+		break;
+	}
+}
+
+static void detect_mem_freq(struct drm_i915_private *i915)
+{
+	if (IS_PINEVIEW(i915))
+		pnv_detect_mem_freq(i915);
+	else if (GRAPHICS_VER(i915) == 5)
+		ilk_detect_mem_freq(i915);
+	else if (IS_CHERRYVIEW(i915))
+		chv_detect_mem_freq(i915);
+	else if (IS_VALLEYVIEW(i915))
+		vlv_detect_mem_freq(i915);
+
+	if (i915->mem_freq)
+		drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
+}
+
 static int intel_dimm_num_devices(const struct dram_dimm_info *dimm)
 {
 	return dimm->ranks * 64 / (dimm->width ?: 1);
@@ -507,6 +657,8 @@ void intel_dram_detect(struct drm_i915_private *i915)
 	struct dram_info *dram_info = &i915->dram_info;
 	int ret;
 
+	detect_mem_freq(i915);
+
 	if (GRAPHICS_VER(i915) < 9 || IS_DG2(i915) || !HAS_DISPLAY(i915))
 		return;
 
diff --git a/drivers/gpu/drm/i915/vlv_suspend.c b/drivers/gpu/drm/i915/vlv_suspend.c
index 02e63ed77f60..94595dde2b96 100644
--- a/drivers/gpu/drm/i915/vlv_suspend.c
+++ b/drivers/gpu/drm/i915/vlv_suspend.c
@@ -12,7 +12,7 @@
 #include "i915_reg.h"
 #include "i915_trace.h"
 #include "i915_utils.h"
-#include "intel_pm.h"
+#include "intel_clock_gating.h"
 #include "vlv_suspend.h"
 
 #include "gt/intel_gt_regs.h"
@@ -451,7 +451,7 @@ int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume)
 	vlv_check_no_gt_access(dev_priv);
 
 	if (rpm_resume)
-		intel_init_clock_gating(dev_priv);
+		intel_clock_gating_init(dev_priv);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/imx/Kconfig b/drivers/gpu/drm/imx/Kconfig
index e5749927fd6c..03535a15dd8f 100644
--- a/drivers/gpu/drm/imx/Kconfig
+++ b/drivers/gpu/drm/imx/Kconfig
@@ -2,3 +2,4 @@
 
 source "drivers/gpu/drm/imx/dcss/Kconfig"
 source "drivers/gpu/drm/imx/ipuv3/Kconfig"
+source "drivers/gpu/drm/imx/lcdc/Kconfig"
diff --git a/drivers/gpu/drm/imx/Makefile b/drivers/gpu/drm/imx/Makefile
index 909622864716..86f38e7c7422 100644
--- a/drivers/gpu/drm/imx/Makefile
+++ b/drivers/gpu/drm/imx/Makefile
@@ -2,3 +2,4 @@
 
 obj-$(CONFIG_DRM_IMX_DCSS) += dcss/
 obj-$(CONFIG_DRM_IMX) += ipuv3/
+obj-$(CONFIG_DRM_IMX_LCDC) += lcdc/
diff --git a/drivers/gpu/drm/imx/dcss/dcss-kms.c b/drivers/gpu/drm/imx/dcss/dcss-kms.c
index dab5e664920d..896de946f8df 100644
--- a/drivers/gpu/drm/imx/dcss/dcss-kms.c
+++ b/drivers/gpu/drm/imx/dcss/dcss-kms.c
@@ -7,7 +7,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge_connector.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_of.h>
@@ -145,7 +145,7 @@ struct dcss_kms_dev *dcss_kms_attach(struct dcss_dev *dcss)
 	if (ret)
 		goto cleanup_crtc;
 
-	drm_fbdev_generic_setup(drm, 32);
+	drm_fbdev_dma_setup(drm, 32);
 
 	return kms;
 
diff --git a/drivers/gpu/drm/imx/ipuv3/Kconfig b/drivers/gpu/drm/imx/ipuv3/Kconfig
index bb278a369575..bacf0655ebaf 100644
--- a/drivers/gpu/drm/imx/ipuv3/Kconfig
+++ b/drivers/gpu/drm/imx/ipuv3/Kconfig
@@ -4,7 +4,7 @@ config DRM_IMX
 	select DRM_KMS_HELPER
 	select VIDEOMODE_HELPERS
 	select DRM_GEM_DMA_HELPER
-	depends on DRM && (ARCH_MXC || ARCH_MULTIPLATFORM || COMPILE_TEST)
+	depends on DRM && (ARCH_MXC || COMPILE_TEST)
 	depends on IMX_IPUV3_CORE
 	help
 	  enable i.MX graphics support
diff --git a/drivers/gpu/drm/imx/ipuv3/imx-drm-core.c b/drivers/gpu/drm/imx/ipuv3/imx-drm-core.c
index e060fa6cbcb9..4a866ac60fff 100644
--- a/drivers/gpu/drm/imx/ipuv3/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/ipuv3/imx-drm-core.c
@@ -16,7 +16,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_managed.h>
@@ -249,7 +249,7 @@ static int imx_drm_bind(struct device *dev)
 	if (ret)
 		goto err_poll_fini;
 
-	drm_fbdev_generic_setup(drm, legacyfb_depth);
+	drm_fbdev_dma_setup(drm, legacyfb_depth);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/imx/lcdc/Kconfig b/drivers/gpu/drm/imx/lcdc/Kconfig
new file mode 100644
index 000000000000..7e57922bbd9d
--- /dev/null
+++ b/drivers/gpu/drm/imx/lcdc/Kconfig
@@ -0,0 +1,7 @@
+config DRM_IMX_LCDC
+      tristate "Freescale i.MX LCDC displays"
+      depends on DRM && (ARCH_MXC || COMPILE_TEST)
+      select DRM_GEM_DMA_HELPER
+      select DRM_KMS_HELPER
+      help
+        Found on i.MX1, i.MX21, i.MX25 and i.MX27.
diff --git a/drivers/gpu/drm/imx/lcdc/Makefile b/drivers/gpu/drm/imx/lcdc/Makefile
new file mode 100644
index 000000000000..e84daa432c2e
--- /dev/null
+++ b/drivers/gpu/drm/imx/lcdc/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_DRM_IMX_LCDC) += imx-lcdc.o
diff --git a/drivers/gpu/drm/imx/lcdc/imx-lcdc.c b/drivers/gpu/drm/imx/lcdc/imx-lcdc.c
new file mode 100644
index 000000000000..8e6d457917da
--- /dev/null
+++ b/drivers/gpu/drm/imx/lcdc/imx-lcdc.c
@@ -0,0 +1,546 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// SPDX-FileCopyrightText: 2020 Marian Cichy <M.Cichy@pengutronix.de>
+
+#include <drm/drm_bridge.h>
+#include <drm/drm_bridge_connector.h>
+#include <drm/drm_damage_helper.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fb_dma_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_gem_dma_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_vblank.h>
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#define IMX21LCDC_LSSAR         0x0000 /* LCDC Screen Start Address Register */
+#define IMX21LCDC_LSR           0x0004 /* LCDC Size Register */
+#define IMX21LCDC_LVPWR         0x0008 /* LCDC Virtual Page Width Register */
+#define IMX21LCDC_LCPR          0x000C /* LCDC Cursor Position Register */
+#define IMX21LCDC_LCWHB         0x0010 /* LCDC Cursor Width Height and Blink Register*/
+#define IMX21LCDC_LCCMR         0x0014 /* LCDC Color Cursor Mapping Register */
+#define IMX21LCDC_LPCR          0x0018 /* LCDC Panel Configuration Register */
+#define IMX21LCDC_LHCR          0x001C /* LCDC Horizontal Configuration Register */
+#define IMX21LCDC_LVCR          0x0020 /* LCDC Vertical Configuration Register */
+#define IMX21LCDC_LPOR          0x0024 /* LCDC Panning Offset Register */
+#define IMX21LCDC_LSCR          0x0028 /* LCDC Sharp Configuration Register */
+#define IMX21LCDC_LPCCR         0x002C /* LCDC PWM Contrast Control Register */
+#define IMX21LCDC_LDCR          0x0030 /* LCDC DMA Control Register */
+#define IMX21LCDC_LRMCR         0x0034 /* LCDC Refresh Mode Control Register */
+#define IMX21LCDC_LICR          0x0038 /* LCDC Interrupt Configuration Register */
+#define IMX21LCDC_LIER          0x003C /* LCDC Interrupt Enable Register */
+#define IMX21LCDC_LISR          0x0040 /* LCDC Interrupt Status Register */
+#define IMX21LCDC_LGWSAR        0x0050 /* LCDC Graphic Window Start Address Register */
+#define IMX21LCDC_LGWSR         0x0054 /* LCDC Graph Window Size Register */
+#define IMX21LCDC_LGWVPWR       0x0058 /* LCDC Graphic Window Virtual Page Width Register */
+#define IMX21LCDC_LGWPOR        0x005C /* LCDC Graphic Window Panning Offset Register */
+#define IMX21LCDC_LGWPR         0x0060 /* LCDC Graphic Window Position Register */
+#define IMX21LCDC_LGWCR         0x0064 /* LCDC Graphic Window Control Register */
+#define IMX21LCDC_LGWDCR        0x0068 /* LCDC Graphic Window DMA Control Register */
+#define IMX21LCDC_LAUSCR        0x0080 /* LCDC AUS Mode Control Register */
+#define IMX21LCDC_LAUSCCR       0x0084 /* LCDC AUS Mode Cursor Control Register */
+#define IMX21LCDC_BGLUT         0x0800 /* Background Lookup Table */
+#define IMX21LCDC_GWLUT         0x0C00 /* Graphic Window Lookup Table */
+
+#define IMX21LCDC_LCPR_CC0 BIT(30) /* Cursor Control Bit 0 */
+#define IMX21LCDC_LCPR_CC1 BIT(31) /* Cursor Control Bit 1 */
+
+/* Values HSYNC, VSYNC and Framesize Register */
+#define IMX21LCDC_LHCR_HWIDTH		GENMASK(31, 26)
+#define IMX21LCDC_LHCR_HFPORCH		GENMASK(15, 8)		/* H_WAIT_1 in the i.MX25 Reference manual */
+#define IMX21LCDC_LHCR_HBPORCH		GENMASK(7, 0)		/* H_WAIT_2 in the i.MX25 Reference manual */
+
+#define IMX21LCDC_LVCR_VWIDTH		GENMASK(31, 26)
+#define IMX21LCDC_LVCR_VFPORCH		GENMASK(15, 8)		/* V_WAIT_1 in the i.MX25 Reference manual */
+#define IMX21LCDC_LVCR_VBPORCH		GENMASK(7, 0)		/* V_WAIT_2 in the i.MX25 Reference manual */
+
+#define IMX21LCDC_LSR_XMAX		GENMASK(25, 20)
+#define IMX21LCDC_LSR_YMAX		GENMASK(9, 0)
+
+/* Values for LPCR Register */
+#define IMX21LCDC_LPCR_PCD		GENMASK(5, 0)
+#define IMX21LCDC_LPCR_SHARP		BIT(6)
+#define IMX21LCDC_LPCR_SCLKSEL		BIT(7)
+#define IMX21LCDC_LPCR_ACD		GENMASK(14, 8)
+#define IMX21LCDC_LPCR_ACDSEL		BIT(15)
+#define IMX21LCDC_LPCR_REV_VS		BIT(16)
+#define IMX21LCDC_LPCR_SWAP_SEL		BIT(17)
+#define IMX21LCDC_LPCR_END_SEL		BIT(18)
+#define IMX21LCDC_LPCR_SCLKIDLE		BIT(19)
+#define IMX21LCDC_LPCR_OEPOL		BIT(20)
+#define IMX21LCDC_LPCR_CLKPOL		BIT(21)
+#define IMX21LCDC_LPCR_LPPOL		BIT(22)
+#define IMX21LCDC_LPCR_FLMPOL		BIT(23)
+#define IMX21LCDC_LPCR_PIXPOL		BIT(24)
+#define IMX21LCDC_LPCR_BPIX		GENMASK(27, 25)
+#define IMX21LCDC_LPCR_PBSIZ		GENMASK(29, 28)
+#define IMX21LCDC_LPCR_COLOR		BIT(30)
+#define IMX21LCDC_LPCR_TFT		BIT(31)
+
+#define INTR_EOF BIT(1) /* VBLANK Interrupt Bit */
+
+#define BPP_RGB565	0x05
+#define BPP_XRGB8888	0x07
+
+#define LCDC_MIN_XRES 64
+#define LCDC_MIN_YRES 64
+
+#define LCDC_MAX_XRES 1024
+#define LCDC_MAX_YRES 1024
+
+struct imx_lcdc {
+	struct drm_device drm;
+	struct drm_simple_display_pipe pipe;
+	struct drm_connector *connector;
+	void __iomem *base;
+
+	struct clk *clk_ipg;
+	struct clk *clk_ahb;
+	struct clk *clk_per;
+};
+
+static const u32 imx_lcdc_formats[] = {
+	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
+};
+
+static inline struct imx_lcdc *imx_lcdc_from_drmdev(struct drm_device *drm)
+{
+	return container_of(drm, struct imx_lcdc, drm);
+}
+
+static unsigned int imx_lcdc_get_format(unsigned int drm_format)
+{
+	switch (drm_format) {
+	default:
+		DRM_WARN("Format not supported - fallback to XRGB8888\n");
+		fallthrough;
+
+	case DRM_FORMAT_XRGB8888:
+		return BPP_XRGB8888;
+
+	case DRM_FORMAT_RGB565:
+		return BPP_RGB565;
+	}
+}
+
+static void imx_lcdc_update_hw_registers(struct drm_simple_display_pipe *pipe,
+					 struct drm_plane_state *old_state,
+					 bool mode_set)
+{
+	struct drm_crtc *crtc = &pipe->crtc;
+	struct drm_plane_state *new_state = pipe->plane.state;
+	struct drm_framebuffer *fb = new_state->fb;
+	struct imx_lcdc *lcdc = imx_lcdc_from_drmdev(pipe->crtc.dev);
+	u32 lpcr, lvcr, lhcr;
+	u32 framesize;
+	dma_addr_t addr;
+
+	addr = drm_fb_dma_get_gem_addr(fb, new_state, 0);
+	/* The LSSAR register specifies the LCD screen start address (SSA). */
+	writel(addr, lcdc->base + IMX21LCDC_LSSAR);
+
+	if (!mode_set)
+		return;
+
+	/* Disable PER clock to make register write possible */
+	if (old_state && old_state->crtc && old_state->crtc->enabled)
+		clk_disable_unprepare(lcdc->clk_per);
+
+	/* Framesize */
+	framesize = FIELD_PREP(IMX21LCDC_LSR_XMAX, crtc->mode.hdisplay >> 4) |
+		FIELD_PREP(IMX21LCDC_LSR_YMAX, crtc->mode.vdisplay);
+	writel(framesize, lcdc->base + IMX21LCDC_LSR);
+
+	/* HSYNC */
+	lhcr = FIELD_PREP(IMX21LCDC_LHCR_HFPORCH, crtc->mode.hsync_start - crtc->mode.hdisplay - 1) |
+		FIELD_PREP(IMX21LCDC_LHCR_HWIDTH, crtc->mode.hsync_end - crtc->mode.hsync_start - 1) |
+		FIELD_PREP(IMX21LCDC_LHCR_HBPORCH, crtc->mode.htotal - crtc->mode.hsync_end - 3);
+	writel(lhcr, lcdc->base + IMX21LCDC_LHCR);
+
+	/* VSYNC */
+	lvcr = FIELD_PREP(IMX21LCDC_LVCR_VFPORCH, crtc->mode.vsync_start - crtc->mode.vdisplay) |
+		FIELD_PREP(IMX21LCDC_LVCR_VWIDTH, crtc->mode.vsync_end - crtc->mode.vsync_start) |
+		FIELD_PREP(IMX21LCDC_LVCR_VBPORCH, crtc->mode.vtotal - crtc->mode.vsync_end);
+	writel(lvcr, lcdc->base + IMX21LCDC_LVCR);
+
+	lpcr = readl(lcdc->base + IMX21LCDC_LPCR);
+	lpcr &= ~IMX21LCDC_LPCR_BPIX;
+	lpcr |= FIELD_PREP(IMX21LCDC_LPCR_BPIX, imx_lcdc_get_format(fb->format->format));
+	writel(lpcr, lcdc->base + IMX21LCDC_LPCR);
+
+	/* Virtual Page Width */
+	writel(new_state->fb->pitches[0] / 4, lcdc->base + IMX21LCDC_LVPWR);
+
+	/* Enable PER clock */
+	if (new_state->crtc->enabled)
+		clk_prepare_enable(lcdc->clk_per);
+}
+
+static void imx_lcdc_pipe_enable(struct drm_simple_display_pipe *pipe,
+				 struct drm_crtc_state *crtc_state,
+				 struct drm_plane_state *plane_state)
+{
+	int ret;
+	int clk_div;
+	int bpp;
+	struct imx_lcdc *lcdc = imx_lcdc_from_drmdev(pipe->crtc.dev);
+	struct drm_display_mode *mode = &pipe->crtc.mode;
+	struct drm_display_info *disp_info = &lcdc->connector->display_info;
+	const int hsync_pol = (mode->flags & DRM_MODE_FLAG_PHSYNC) ? 0 : 1;
+	const int vsync_pol = (mode->flags & DRM_MODE_FLAG_PVSYNC) ? 0 : 1;
+	const int data_enable_pol =
+		(disp_info->bus_flags & DRM_BUS_FLAG_DE_HIGH) ? 0 : 1;
+	const int clk_pol =
+		(disp_info->bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE) ? 0 : 1;
+
+	clk_div = DIV_ROUND_CLOSEST_ULL(clk_get_rate(lcdc->clk_per),
+					mode->clock * 1000);
+	bpp = imx_lcdc_get_format(plane_state->fb->format->format);
+
+	writel(FIELD_PREP(IMX21LCDC_LPCR_PCD, clk_div - 1) |
+	       FIELD_PREP(IMX21LCDC_LPCR_LPPOL, hsync_pol) |
+	       FIELD_PREP(IMX21LCDC_LPCR_FLMPOL, vsync_pol) |
+	       FIELD_PREP(IMX21LCDC_LPCR_OEPOL, data_enable_pol) |
+	       FIELD_PREP(IMX21LCDC_LPCR_TFT, 1) |
+	       FIELD_PREP(IMX21LCDC_LPCR_COLOR, 1) |
+	       FIELD_PREP(IMX21LCDC_LPCR_PBSIZ, 3) |
+	       FIELD_PREP(IMX21LCDC_LPCR_BPIX, bpp) |
+	       FIELD_PREP(IMX21LCDC_LPCR_SCLKSEL, 1) |
+	       FIELD_PREP(IMX21LCDC_LPCR_PIXPOL, 0) |
+	       FIELD_PREP(IMX21LCDC_LPCR_CLKPOL, clk_pol),
+	       lcdc->base + IMX21LCDC_LPCR);
+
+	/* 0px panning offset */
+	writel(0x00000000, lcdc->base + IMX21LCDC_LPOR);
+
+	/* disable hardware cursor */
+	writel(readl(lcdc->base + IMX21LCDC_LCPR) & ~(IMX21LCDC_LCPR_CC0 | IMX21LCDC_LCPR_CC1),
+	       lcdc->base + IMX21LCDC_LCPR);
+
+	ret = clk_prepare_enable(lcdc->clk_ipg);
+	if (ret) {
+		dev_err(pipe->crtc.dev->dev, "Cannot enable ipg clock: %pe\n", ERR_PTR(ret));
+		return;
+	}
+	ret = clk_prepare_enable(lcdc->clk_ahb);
+	if (ret) {
+		dev_err(pipe->crtc.dev->dev, "Cannot enable ahb clock: %pe\n", ERR_PTR(ret));
+
+		clk_disable_unprepare(lcdc->clk_ipg);
+
+		return;
+	}
+
+	imx_lcdc_update_hw_registers(pipe, NULL, true);
+
+	/* Enable VBLANK Interrupt */
+	writel(INTR_EOF, lcdc->base + IMX21LCDC_LIER);
+}
+
+static void imx_lcdc_pipe_disable(struct drm_simple_display_pipe *pipe)
+{
+	struct imx_lcdc *lcdc = imx_lcdc_from_drmdev(pipe->crtc.dev);
+	struct drm_crtc *crtc = &lcdc->pipe.crtc;
+	struct drm_pending_vblank_event *event;
+
+	clk_disable_unprepare(lcdc->clk_ahb);
+	clk_disable_unprepare(lcdc->clk_ipg);
+
+	if (pipe->crtc.enabled)
+		clk_disable_unprepare(lcdc->clk_per);
+
+	spin_lock_irq(&lcdc->drm.event_lock);
+	event = crtc->state->event;
+	if (event) {
+		crtc->state->event = NULL;
+		drm_crtc_send_vblank_event(crtc, event);
+	}
+	spin_unlock_irq(&lcdc->drm.event_lock);
+
+	/* Disable VBLANK Interrupt */
+	writel(0, lcdc->base + IMX21LCDC_LIER);
+}
+
+static int imx_lcdc_pipe_check(struct drm_simple_display_pipe *pipe,
+			       struct drm_plane_state *plane_state,
+			       struct drm_crtc_state *crtc_state)
+{
+	const struct drm_display_mode *mode = &crtc_state->mode;
+	const struct drm_display_mode *old_mode = &pipe->crtc.state->mode;
+
+	if (mode->hdisplay < LCDC_MIN_XRES || mode->hdisplay > LCDC_MAX_XRES ||
+	    mode->vdisplay < LCDC_MIN_YRES || mode->vdisplay > LCDC_MAX_YRES ||
+	    mode->hdisplay % 0x10) { /* must be multiple of 16 */
+		drm_err(pipe->crtc.dev, "unsupported display mode (%u x %u)\n",
+			mode->hdisplay, mode->vdisplay);
+		return -EINVAL;
+	}
+
+	crtc_state->mode_changed =
+		old_mode->hdisplay != mode->hdisplay ||
+		old_mode->vdisplay != mode->vdisplay;
+
+	return 0;
+}
+
+static void imx_lcdc_pipe_update(struct drm_simple_display_pipe *pipe,
+				 struct drm_plane_state *old_state)
+{
+	struct drm_crtc *crtc = &pipe->crtc;
+	struct drm_pending_vblank_event *event = crtc->state->event;
+	struct drm_plane_state *new_state = pipe->plane.state;
+	struct drm_framebuffer *fb = new_state->fb;
+	struct drm_framebuffer *old_fb = old_state->fb;
+	struct drm_crtc *old_crtc = old_state->crtc;
+	bool mode_changed = false;
+
+	if (old_fb && old_fb->format != fb->format)
+		mode_changed = true;
+	else if (old_crtc != crtc)
+		mode_changed = true;
+
+	imx_lcdc_update_hw_registers(pipe, old_state, mode_changed);
+
+	if (event) {
+		crtc->state->event = NULL;
+
+		spin_lock_irq(&crtc->dev->event_lock);
+
+		if (crtc->state->active && drm_crtc_vblank_get(crtc) == 0)
+			drm_crtc_arm_vblank_event(crtc, event);
+		else
+			drm_crtc_send_vblank_event(crtc, event);
+
+		spin_unlock_irq(&crtc->dev->event_lock);
+	}
+}
+
+static const struct drm_simple_display_pipe_funcs imx_lcdc_pipe_funcs = {
+	.enable = imx_lcdc_pipe_enable,
+	.disable = imx_lcdc_pipe_disable,
+	.check = imx_lcdc_pipe_check,
+	.update = imx_lcdc_pipe_update,
+};
+
+static const struct drm_mode_config_funcs imx_lcdc_mode_config_funcs = {
+	.fb_create = drm_gem_fb_create_with_dirty,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static const struct drm_mode_config_helper_funcs imx_lcdc_mode_config_helpers = {
+	.atomic_commit_tail = drm_atomic_helper_commit_tail_rpm,
+};
+
+static void imx_lcdc_release(struct drm_device *drm)
+{
+	struct imx_lcdc *lcdc = imx_lcdc_from_drmdev(drm);
+
+	drm_kms_helper_poll_fini(drm);
+	kfree(lcdc);
+}
+
+DEFINE_DRM_GEM_DMA_FOPS(imx_lcdc_drm_fops);
+
+static struct drm_driver imx_lcdc_drm_driver = {
+	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
+	.fops = &imx_lcdc_drm_fops,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
+	.release = imx_lcdc_release,
+	.name = "imx-lcdc",
+	.desc = "i.MX LCDC driver",
+	.date = "20200716",
+};
+
+static const struct of_device_id imx_lcdc_of_dev_id[] = {
+	{
+		.compatible = "fsl,imx21-lcdc",
+	},
+	{
+		.compatible = "fsl,imx25-lcdc",
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, imx_lcdc_of_dev_id);
+
+static irqreturn_t imx_lcdc_irq_handler(int irq, void *arg)
+{
+	struct imx_lcdc *lcdc = arg;
+	struct drm_crtc *crtc = &lcdc->pipe.crtc;
+	unsigned int status;
+
+	status = readl(lcdc->base + IMX21LCDC_LISR);
+
+	if (status & INTR_EOF) {
+		drm_crtc_handle_vblank(crtc);
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
+static int imx_lcdc_probe(struct platform_device *pdev)
+{
+	struct imx_lcdc *lcdc;
+	struct drm_device *drm;
+	struct drm_bridge *bridge;
+	int irq;
+	int ret;
+	struct device *dev = &pdev->dev;
+
+	lcdc = devm_drm_dev_alloc(dev, &imx_lcdc_drm_driver,
+				  struct imx_lcdc, drm);
+	if (!lcdc)
+		return -ENOMEM;
+
+	drm = &lcdc->drm;
+
+	lcdc->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(lcdc->base))
+		return dev_err_probe(dev, PTR_ERR(lcdc->base), "Cannot get IO memory\n");
+
+	bridge = devm_drm_of_get_bridge(dev, dev->of_node, 0, 0);
+	if (IS_ERR(bridge))
+		return dev_err_probe(dev, PTR_ERR(bridge), "Failed to find bridge\n");
+
+	/* Get Clocks */
+	lcdc->clk_ipg = devm_clk_get(dev, "ipg");
+	if (IS_ERR(lcdc->clk_ipg))
+		return dev_err_probe(dev, PTR_ERR(lcdc->clk_ipg), "Failed to get %s clk\n", "ipg");
+
+	lcdc->clk_ahb = devm_clk_get(dev, "ahb");
+	if (IS_ERR(lcdc->clk_ahb))
+		return dev_err_probe(dev, PTR_ERR(lcdc->clk_ahb), "Failed to get %s clk\n", "ahb");
+
+	lcdc->clk_per = devm_clk_get(dev, "per");
+	if (IS_ERR(lcdc->clk_per))
+		return dev_err_probe(dev, PTR_ERR(lcdc->clk_per), "Failed to get %s clk\n", "per");
+
+	ret = dma_set_mask_and_coherent(drm->dev, DMA_BIT_MASK(32));
+	if (ret)
+		return dev_err_probe(dev, ret, "Cannot set DMA Mask\n");
+
+	/* Modeset init */
+	ret = drmm_mode_config_init(drm);
+	if (ret)
+		return dev_err_probe(dev, ret, "Cannot initialize mode configuration structure\n");
+
+	/* CRTC, Plane, Encoder */
+	ret = drm_simple_display_pipe_init(drm, &lcdc->pipe,
+					   &imx_lcdc_pipe_funcs,
+					   imx_lcdc_formats,
+					   ARRAY_SIZE(imx_lcdc_formats), NULL, NULL);
+	if (ret < 0)
+		return dev_err_probe(drm->dev, ret, "Cannot setup simple display pipe\n");
+
+	ret = drm_vblank_init(drm, drm->mode_config.num_crtc);
+	if (ret < 0)
+		return dev_err_probe(drm->dev, ret, "Failed to initialize vblank\n");
+
+	ret = drm_bridge_attach(&lcdc->pipe.encoder, bridge, NULL, DRM_BRIDGE_ATTACH_NO_CONNECTOR);
+	if (ret)
+		return dev_err_probe(drm->dev, ret, "Cannot attach bridge\n");
+
+	lcdc->connector = drm_bridge_connector_init(drm, &lcdc->pipe.encoder);
+	if (IS_ERR(lcdc->connector))
+		return dev_err_probe(drm->dev, PTR_ERR(lcdc->connector), "Cannot init bridge connector\n");
+
+	drm_connector_attach_encoder(lcdc->connector, &lcdc->pipe.encoder);
+
+	/*
+	 * The LCDC controller does not have an enable bit. The
+	 * controller starts directly when the clocks are enabled.
+	 * If the clocks are enabled when the controller is not yet
+	 * programmed with proper register values (enabled at the
+	 * bootloader, for example) then it just goes into some undefined
+	 * state.
+	 * To avoid this issue, let's enable and disable LCDC IPG,
+	 * PER and AHB clock so that we force some kind of 'reset'
+	 * to the LCDC block.
+	 */
+
+	ret = clk_prepare_enable(lcdc->clk_ipg);
+	if (ret)
+		return dev_err_probe(dev, ret, "Cannot enable ipg clock\n");
+	clk_disable_unprepare(lcdc->clk_ipg);
+
+	ret = clk_prepare_enable(lcdc->clk_per);
+	if (ret)
+		return dev_err_probe(dev, ret, "Cannot enable per clock\n");
+	clk_disable_unprepare(lcdc->clk_per);
+
+	ret = clk_prepare_enable(lcdc->clk_ahb);
+	if (ret)
+		return dev_err_probe(dev, ret, "Cannot enable ahb clock\n");
+	clk_disable_unprepare(lcdc->clk_ahb);
+
+	drm->mode_config.min_width = LCDC_MIN_XRES;
+	drm->mode_config.max_width = LCDC_MAX_XRES;
+	drm->mode_config.min_height = LCDC_MIN_YRES;
+	drm->mode_config.max_height = LCDC_MAX_YRES;
+	drm->mode_config.preferred_depth = 16;
+	drm->mode_config.funcs = &imx_lcdc_mode_config_funcs;
+	drm->mode_config.helper_private = &imx_lcdc_mode_config_helpers;
+
+	drm_mode_config_reset(drm);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		ret = irq;
+		return ret;
+	}
+
+	ret = devm_request_irq(dev, irq, imx_lcdc_irq_handler, 0, "imx-lcdc", lcdc);
+	if (ret < 0)
+		return dev_err_probe(drm->dev, ret, "Failed to install IRQ handler\n");
+
+	platform_set_drvdata(pdev, drm);
+
+	ret = drm_dev_register(&lcdc->drm, 0);
+	if (ret)
+		return dev_err_probe(dev, ret, "Cannot register device\n");
+
+	drm_fbdev_generic_setup(drm, 0);
+
+	return 0;
+}
+
+static int imx_lcdc_remove(struct platform_device *pdev)
+{
+	struct drm_device *drm = platform_get_drvdata(pdev);
+
+	drm_dev_unregister(drm);
+	drm_atomic_helper_shutdown(drm);
+
+	return 0;
+}
+
+static void imx_lcdc_shutdown(struct platform_device *pdev)
+{
+	drm_atomic_helper_shutdown(platform_get_drvdata(pdev));
+}
+
+static struct platform_driver imx_lcdc_driver = {
+	.driver = {
+		.name = "imx-lcdc",
+		.of_match_table = imx_lcdc_of_dev_id,
+	},
+	.probe = imx_lcdc_probe,
+	.remove = imx_lcdc_remove,
+	.shutdown = imx_lcdc_shutdown,
+};
+module_platform_driver(imx_lcdc_driver);
+
+MODULE_AUTHOR("Marian Cichy <M.Cichy@pengutronix.de>");
+MODULE_DESCRIPTION("Freescale i.MX LCDC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/kmb/kmb_drv.c b/drivers/gpu/drm/kmb/kmb_drv.c
index d29c678f6c91..24035b53441c 100644
--- a/drivers/gpu/drm/kmb/kmb_drv.c
+++ b/drivers/gpu/drm/kmb/kmb_drv.c
@@ -15,7 +15,7 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
@@ -562,7 +562,7 @@ static int kmb_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_register;
 
-	drm_fbdev_generic_setup(&kmb->drm, 0);
+	drm_fbdev_dma_setup(&kmb->drm, 0);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/lima/lima_drv.c b/drivers/gpu/drm/lima/lima_drv.c
index 7b8d7178d09a..39cab4a55f57 100644
--- a/drivers/gpu/drm/lima/lima_drv.c
+++ b/drivers/gpu/drm/lima/lima_drv.c
@@ -392,8 +392,10 @@ static int lima_pdev_probe(struct platform_device *pdev)
 
 	/* Allocate and initialize the DRM device. */
 	ddev = drm_dev_alloc(&lima_drm_driver, &pdev->dev);
-	if (IS_ERR(ddev))
-		return PTR_ERR(ddev);
+	if (IS_ERR(ddev)) {
+		err = PTR_ERR(ddev);
+		goto err_out0;
+	}
 
 	ddev->dev_private = ldev;
 	ldev->ddev = ddev;
diff --git a/drivers/gpu/drm/lima/lima_gem.c b/drivers/gpu/drm/lima/lima_gem.c
index 0f1ca0b0db49..10252dc11a22 100644
--- a/drivers/gpu/drm/lima/lima_gem.c
+++ b/drivers/gpu/drm/lima/lima_gem.c
@@ -277,21 +277,13 @@ static int lima_gem_add_deps(struct drm_file *file, struct lima_submit *submit)
 	int i, err;
 
 	for (i = 0; i < ARRAY_SIZE(submit->in_sync); i++) {
-		struct dma_fence *fence = NULL;
-
 		if (!submit->in_sync[i])
 			continue;
 
-		err = drm_syncobj_find_fence(file, submit->in_sync[i],
-					     0, 0, &fence);
+		err = drm_sched_job_add_syncobj_dependency(&submit->task->base, file,
+							   submit->in_sync[i], 0);
 		if (err)
 			return err;
-
-		err = drm_sched_job_add_dependency(&submit->task->base, fence);
-		if (err) {
-			dma_fence_put(fence);
-			return err;
-		}
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/logicvc/logicvc_drm.c b/drivers/gpu/drm/logicvc/logicvc_drm.c
index 2fb23697740a..c35c453fd025 100644
--- a/drivers/gpu/drm/logicvc/logicvc_drm.c
+++ b/drivers/gpu/drm/logicvc/logicvc_drm.c
@@ -17,7 +17,7 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_print.h>
 
@@ -449,7 +449,7 @@ static int logicvc_drm_probe(struct platform_device *pdev)
 		preferred_bpp = 32;
 		break;
 	}
-	drm_fbdev_generic_setup(drm_dev, preferred_bpp);
+	drm_fbdev_dma_setup(drm_dev, preferred_bpp);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/mcde/mcde_drv.c b/drivers/gpu/drm/mcde/mcde_drv.c
index 4aedb050d2a5..a8cd86c06c14 100644
--- a/drivers/gpu/drm/mcde/mcde_drv.c
+++ b/drivers/gpu/drm/mcde/mcde_drv.c
@@ -69,7 +69,7 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_dma_helper.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -94,7 +94,7 @@
 #define MCDE_PID_MAJOR_VERSION_MASK 0xFF000000
 
 static const struct drm_mode_config_funcs mcde_mode_config_funcs = {
-	.fb_create = drm_gem_fb_create_with_dirty,
+	.fb_create = drm_gem_fb_create,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -237,7 +237,7 @@ static int mcde_drm_bind(struct device *dev)
 	if (ret < 0)
 		goto unbind;
 
-	drm_fbdev_generic_setup(drm, 32);
+	drm_fbdev_dma_setup(drm, 32);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/mediatek/Makefile b/drivers/gpu/drm/mediatek/Makefile
index 3517d1c65cd7..d4d193f60271 100644
--- a/drivers/gpu/drm/mediatek/Makefile
+++ b/drivers/gpu/drm/mediatek/Makefile
@@ -6,6 +6,7 @@ mediatek-drm-y := mtk_disp_aal.o \
 		  mtk_disp_gamma.o \
 		  mtk_disp_merge.o \
 		  mtk_disp_ovl.o \
+		  mtk_disp_ovl_adaptor.o \
 		  mtk_disp_rdma.o \
 		  mtk_drm_crtc.o \
 		  mtk_drm_ddp_comp.o \
@@ -14,6 +15,7 @@ mediatek-drm-y := mtk_disp_aal.o \
 		  mtk_drm_plane.o \
 		  mtk_dsi.o \
 		  mtk_dpi.o \
+		  mtk_ethdr.o \
 		  mtk_mdp_rdma.o
 
 obj-$(CONFIG_DRM_MEDIATEK) += mediatek-drm.o
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_drv.h b/drivers/gpu/drm/mediatek/mtk_disp_drv.h
index 33e61a136bbc..2254038519e1 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_drv.h
+++ b/drivers/gpu/drm/mediatek/mtk_disp_drv.h
@@ -7,6 +7,8 @@
 #define _MTK_DISP_DRV_H_
 
 #include <linux/soc/mediatek/mtk-cmdq.h>
+#include <linux/soc/mediatek/mtk-mmsys.h>
+#include <linux/soc/mediatek/mtk-mutex.h>
 #include "mtk_drm_plane.h"
 #include "mtk_mdp_rdma.h"
 
@@ -96,6 +98,34 @@ void mtk_ovl_register_vblank_cb(struct device *dev,
 void mtk_ovl_unregister_vblank_cb(struct device *dev);
 void mtk_ovl_enable_vblank(struct device *dev);
 void mtk_ovl_disable_vblank(struct device *dev);
+const u32 *mtk_ovl_get_formats(struct device *dev);
+size_t mtk_ovl_get_num_formats(struct device *dev);
+
+void mtk_ovl_adaptor_add_comp(struct device *dev, struct mtk_mutex *mutex);
+void mtk_ovl_adaptor_remove_comp(struct device *dev, struct mtk_mutex *mutex);
+void mtk_ovl_adaptor_connect(struct device *dev, struct device *mmsys_dev,
+			     unsigned int next);
+void mtk_ovl_adaptor_disconnect(struct device *dev, struct device *mmsys_dev,
+				unsigned int next);
+int mtk_ovl_adaptor_clk_enable(struct device *dev);
+void mtk_ovl_adaptor_clk_disable(struct device *dev);
+void mtk_ovl_adaptor_config(struct device *dev, unsigned int w,
+			    unsigned int h, unsigned int vrefresh,
+			    unsigned int bpc, struct cmdq_pkt *cmdq_pkt);
+void mtk_ovl_adaptor_layer_config(struct device *dev, unsigned int idx,
+				  struct mtk_plane_state *state,
+				  struct cmdq_pkt *cmdq_pkt);
+void mtk_ovl_adaptor_register_vblank_cb(struct device *dev, void (*vblank_cb)(void *),
+					void *vblank_cb_data);
+void mtk_ovl_adaptor_unregister_vblank_cb(struct device *dev);
+void mtk_ovl_adaptor_enable_vblank(struct device *dev);
+void mtk_ovl_adaptor_disable_vblank(struct device *dev);
+void mtk_ovl_adaptor_start(struct device *dev);
+void mtk_ovl_adaptor_stop(struct device *dev);
+unsigned int mtk_ovl_adaptor_layer_nr(struct device *dev);
+struct device *mtk_ovl_adaptor_dma_dev_get(struct device *dev);
+const u32 *mtk_ovl_adaptor_get_formats(struct device *dev);
+size_t mtk_ovl_adaptor_get_num_formats(struct device *dev);
 
 void mtk_rdma_bypass_shadow(struct device *dev);
 int mtk_rdma_clk_enable(struct device *dev);
@@ -115,6 +145,8 @@ void mtk_rdma_register_vblank_cb(struct device *dev,
 void mtk_rdma_unregister_vblank_cb(struct device *dev);
 void mtk_rdma_enable_vblank(struct device *dev);
 void mtk_rdma_disable_vblank(struct device *dev);
+const u32 *mtk_rdma_get_formats(struct device *dev);
+size_t mtk_rdma_get_num_formats(struct device *dev);
 
 int mtk_mdp_rdma_clk_enable(struct device *dev);
 void mtk_mdp_rdma_clk_disable(struct device *dev);
@@ -122,4 +154,7 @@ void mtk_mdp_rdma_start(struct device *dev, struct cmdq_pkt *cmdq_pkt);
 void mtk_mdp_rdma_stop(struct device *dev, struct cmdq_pkt *cmdq_pkt);
 void mtk_mdp_rdma_config(struct device *dev, struct mtk_mdp_rdma_cfg *cfg,
 			 struct cmdq_pkt *cmdq_pkt);
+const u32 *mtk_mdp_rdma_get_formats(struct device *dev);
+size_t mtk_mdp_rdma_get_num_formats(struct device *dev);
+
 #endif
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
index 9d8c986700ee..8f52cc1f3fba 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
@@ -42,6 +42,7 @@
 #define DISP_REG_OVL_RDMA_CTRL(n)		(0x00c0 + 0x20 * (n))
 #define DISP_REG_OVL_RDMA_GMC(n)		(0x00c8 + 0x20 * (n))
 #define DISP_REG_OVL_ADDR_MT2701		0x0040
+#define DISP_REG_OVL_CLRFMT_EXT			0x02D0
 #define DISP_REG_OVL_ADDR_MT8173		0x0f40
 #define DISP_REG_OVL_ADDR(ovl, n)		((ovl)->data->addr + 0x20 * (n))
 #define DISP_REG_OVL_HDR_ADDR(ovl, n)		((ovl)->data->addr + 0x20 * (n) + 0x04)
@@ -62,11 +63,45 @@
 					0 : OVL_CON_CLRFMT_RGB)
 #define OVL_CON_CLRFMT_RGB888(ovl)	((ovl)->data->fmt_rgb565_is_0 ? \
 					OVL_CON_CLRFMT_RGB : 0)
+#define OVL_CON_CLRFMT_BIT_DEPTH_MASK(ovl)	(0xFF << 4 * (ovl))
+#define OVL_CON_CLRFMT_BIT_DEPTH(depth, ovl)	(depth << 4 * (ovl))
+#define OVL_CON_CLRFMT_8_BIT			0x00
+#define OVL_CON_CLRFMT_10_BIT			0x01
 #define	OVL_CON_AEN		BIT(8)
 #define	OVL_CON_ALPHA		0xff
 #define	OVL_CON_VIRT_FLIP	BIT(9)
 #define	OVL_CON_HORZ_FLIP	BIT(10)
 
+static const u32 mt8173_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_YUYV,
+};
+
+static const u32 mt8195_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ARGB2101010,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_BGRA1010102,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_YUYV,
+};
+
 struct mtk_disp_ovl_data {
 	unsigned int addr;
 	unsigned int gmc_bits;
@@ -74,6 +109,9 @@ struct mtk_disp_ovl_data {
 	bool fmt_rgb565_is_0;
 	bool smi_id_en;
 	bool supports_afbc;
+	const u32 *formats;
+	size_t num_formats;
+	bool supports_clrfmt_ext;
 };
 
 /*
@@ -139,6 +177,20 @@ void mtk_ovl_disable_vblank(struct device *dev)
 	writel_relaxed(0x0, ovl->regs + DISP_REG_OVL_INTEN);
 }
 
+const u32 *mtk_ovl_get_formats(struct device *dev)
+{
+	struct mtk_disp_ovl *ovl = dev_get_drvdata(dev);
+
+	return ovl->data->formats;
+}
+
+size_t mtk_ovl_get_num_formats(struct device *dev)
+{
+	struct mtk_disp_ovl *ovl = dev_get_drvdata(dev);
+
+	return ovl->data->num_formats;
+}
+
 int mtk_ovl_clk_enable(struct device *dev)
 {
 	struct mtk_disp_ovl *ovl = dev_get_drvdata(dev);
@@ -189,6 +241,30 @@ static void mtk_ovl_set_afbc(struct mtk_disp_ovl *ovl, struct cmdq_pkt *cmdq_pkt
 			   DISP_REG_OVL_DATAPATH_CON, OVL_LAYER_AFBC_EN(idx));
 }
 
+static void mtk_ovl_set_bit_depth(struct device *dev, int idx, u32 format,
+				  struct cmdq_pkt *cmdq_pkt)
+{
+	struct mtk_disp_ovl *ovl = dev_get_drvdata(dev);
+	unsigned int reg;
+	unsigned int bit_depth = OVL_CON_CLRFMT_8_BIT;
+
+	if (!ovl->data->supports_clrfmt_ext)
+		return;
+
+	reg = readl(ovl->regs + DISP_REG_OVL_CLRFMT_EXT);
+	reg &= ~OVL_CON_CLRFMT_BIT_DEPTH_MASK(idx);
+
+	if (format == DRM_FORMAT_RGBA1010102 ||
+	    format == DRM_FORMAT_BGRA1010102 ||
+	    format == DRM_FORMAT_ARGB2101010)
+		bit_depth = OVL_CON_CLRFMT_10_BIT;
+
+	reg |= OVL_CON_CLRFMT_BIT_DEPTH(bit_depth, idx);
+
+	mtk_ddp_write(cmdq_pkt, reg, &ovl->cmdq_reg,
+		      ovl->regs, DISP_REG_OVL_CLRFMT_EXT);
+}
+
 void mtk_ovl_config(struct device *dev, unsigned int w,
 		    unsigned int h, unsigned int vrefresh,
 		    unsigned int bpc, struct cmdq_pkt *cmdq_pkt)
@@ -303,9 +379,11 @@ static unsigned int ovl_fmt_convert(struct mtk_disp_ovl *ovl, unsigned int fmt)
 		return OVL_CON_CLRFMT_ARGB8888;
 	case DRM_FORMAT_BGRX8888:
 	case DRM_FORMAT_BGRA8888:
+	case DRM_FORMAT_BGRA1010102:
 		return OVL_CON_CLRFMT_ARGB8888 | OVL_CON_BYTE_SWAP;
 	case DRM_FORMAT_XRGB8888:
 	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_ARGB2101010:
 		return OVL_CON_CLRFMT_RGBA8888;
 	case DRM_FORMAT_XBGR8888:
 	case DRM_FORMAT_ABGR8888:
@@ -389,6 +467,7 @@ void mtk_ovl_layer_config(struct device *dev, unsigned int idx,
 				      &ovl->cmdq_reg, ovl->regs, DISP_REG_OVL_PITCH_MSB(idx));
 	}
 
+	mtk_ovl_set_bit_depth(dev, idx, fmt, cmdq_pkt);
 	mtk_ovl_layer_on(dev, idx, cmdq_pkt);
 }
 
@@ -496,6 +575,8 @@ static const struct mtk_disp_ovl_data mt2701_ovl_driver_data = {
 	.gmc_bits = 8,
 	.layer_nr = 4,
 	.fmt_rgb565_is_0 = false,
+	.formats = mt8173_formats,
+	.num_formats = ARRAY_SIZE(mt8173_formats),
 };
 
 static const struct mtk_disp_ovl_data mt8173_ovl_driver_data = {
@@ -503,6 +584,8 @@ static const struct mtk_disp_ovl_data mt8173_ovl_driver_data = {
 	.gmc_bits = 8,
 	.layer_nr = 4,
 	.fmt_rgb565_is_0 = true,
+	.formats = mt8173_formats,
+	.num_formats = ARRAY_SIZE(mt8173_formats),
 };
 
 static const struct mtk_disp_ovl_data mt8183_ovl_driver_data = {
@@ -510,6 +593,8 @@ static const struct mtk_disp_ovl_data mt8183_ovl_driver_data = {
 	.gmc_bits = 10,
 	.layer_nr = 4,
 	.fmt_rgb565_is_0 = true,
+	.formats = mt8173_formats,
+	.num_formats = ARRAY_SIZE(mt8173_formats),
 };
 
 static const struct mtk_disp_ovl_data mt8183_ovl_2l_driver_data = {
@@ -517,6 +602,8 @@ static const struct mtk_disp_ovl_data mt8183_ovl_2l_driver_data = {
 	.gmc_bits = 10,
 	.layer_nr = 2,
 	.fmt_rgb565_is_0 = true,
+	.formats = mt8173_formats,
+	.num_formats = ARRAY_SIZE(mt8173_formats),
 };
 
 static const struct mtk_disp_ovl_data mt8192_ovl_driver_data = {
@@ -525,6 +612,8 @@ static const struct mtk_disp_ovl_data mt8192_ovl_driver_data = {
 	.layer_nr = 4,
 	.fmt_rgb565_is_0 = true,
 	.smi_id_en = true,
+	.formats = mt8173_formats,
+	.num_formats = ARRAY_SIZE(mt8173_formats),
 };
 
 static const struct mtk_disp_ovl_data mt8192_ovl_2l_driver_data = {
@@ -533,6 +622,8 @@ static const struct mtk_disp_ovl_data mt8192_ovl_2l_driver_data = {
 	.layer_nr = 2,
 	.fmt_rgb565_is_0 = true,
 	.smi_id_en = true,
+	.formats = mt8173_formats,
+	.num_formats = ARRAY_SIZE(mt8173_formats),
 };
 
 static const struct mtk_disp_ovl_data mt8195_ovl_driver_data = {
@@ -542,6 +633,9 @@ static const struct mtk_disp_ovl_data mt8195_ovl_driver_data = {
 	.fmt_rgb565_is_0 = true,
 	.smi_id_en = true,
 	.supports_afbc = true,
+	.formats = mt8195_formats,
+	.num_formats = ARRAY_SIZE(mt8195_formats),
+	.supports_clrfmt_ext = true,
 };
 
 static const struct of_device_id mtk_disp_ovl_driver_dt_match[] = {
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c
new file mode 100644
index 000000000000..c0a38f5217ee
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c
@@ -0,0 +1,547 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2021 MediaTek Inc.
+ */
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_of.h>
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <linux/soc/mediatek/mtk-cmdq.h>
+#include <linux/soc/mediatek/mtk-mmsys.h>
+#include <linux/soc/mediatek/mtk-mutex.h>
+
+#include "mtk_disp_drv.h"
+#include "mtk_drm_crtc.h"
+#include "mtk_drm_ddp_comp.h"
+#include "mtk_drm_drv.h"
+#include "mtk_ethdr.h"
+
+#define MTK_OVL_ADAPTOR_RDMA_MAX_WIDTH 1920
+#define MTK_OVL_ADAPTOR_LAYER_NUM 4
+
+enum mtk_ovl_adaptor_comp_type {
+	OVL_ADAPTOR_TYPE_RDMA = 0,
+	OVL_ADAPTOR_TYPE_MERGE,
+	OVL_ADAPTOR_TYPE_ETHDR,
+	OVL_ADAPTOR_TYPE_NUM,
+};
+
+enum mtk_ovl_adaptor_comp_id {
+	OVL_ADAPTOR_MDP_RDMA0,
+	OVL_ADAPTOR_MDP_RDMA1,
+	OVL_ADAPTOR_MDP_RDMA2,
+	OVL_ADAPTOR_MDP_RDMA3,
+	OVL_ADAPTOR_MDP_RDMA4,
+	OVL_ADAPTOR_MDP_RDMA5,
+	OVL_ADAPTOR_MDP_RDMA6,
+	OVL_ADAPTOR_MDP_RDMA7,
+	OVL_ADAPTOR_MERGE0,
+	OVL_ADAPTOR_MERGE1,
+	OVL_ADAPTOR_MERGE2,
+	OVL_ADAPTOR_MERGE3,
+	OVL_ADAPTOR_ETHDR0,
+	OVL_ADAPTOR_ID_MAX
+};
+
+struct ovl_adaptor_comp_match {
+	enum mtk_ovl_adaptor_comp_type type;
+	int alias_id;
+};
+
+struct mtk_disp_ovl_adaptor {
+	struct device *ovl_adaptor_comp[OVL_ADAPTOR_ID_MAX];
+	struct device *mmsys_dev;
+	bool children_bound;
+};
+
+static const char * const private_comp_stem[OVL_ADAPTOR_TYPE_NUM] = {
+	[OVL_ADAPTOR_TYPE_RDMA]		= "vdo1-rdma",
+	[OVL_ADAPTOR_TYPE_MERGE]	= "merge",
+	[OVL_ADAPTOR_TYPE_ETHDR]	= "ethdr",
+};
+
+static const struct ovl_adaptor_comp_match comp_matches[OVL_ADAPTOR_ID_MAX] = {
+	[OVL_ADAPTOR_MDP_RDMA0]	= { OVL_ADAPTOR_TYPE_RDMA, 0 },
+	[OVL_ADAPTOR_MDP_RDMA1]	= { OVL_ADAPTOR_TYPE_RDMA, 1 },
+	[OVL_ADAPTOR_MDP_RDMA2]	= { OVL_ADAPTOR_TYPE_RDMA, 2 },
+	[OVL_ADAPTOR_MDP_RDMA3]	= { OVL_ADAPTOR_TYPE_RDMA, 3 },
+	[OVL_ADAPTOR_MDP_RDMA4]	= { OVL_ADAPTOR_TYPE_RDMA, 4 },
+	[OVL_ADAPTOR_MDP_RDMA5]	= { OVL_ADAPTOR_TYPE_RDMA, 5 },
+	[OVL_ADAPTOR_MDP_RDMA6]	= { OVL_ADAPTOR_TYPE_RDMA, 6 },
+	[OVL_ADAPTOR_MDP_RDMA7]	= { OVL_ADAPTOR_TYPE_RDMA, 7 },
+	[OVL_ADAPTOR_MERGE0]	= { OVL_ADAPTOR_TYPE_MERGE, 1 },
+	[OVL_ADAPTOR_MERGE1]	= { OVL_ADAPTOR_TYPE_MERGE, 2 },
+	[OVL_ADAPTOR_MERGE2]	= { OVL_ADAPTOR_TYPE_MERGE, 3 },
+	[OVL_ADAPTOR_MERGE3]	= { OVL_ADAPTOR_TYPE_MERGE, 4 },
+	[OVL_ADAPTOR_ETHDR0]	= { OVL_ADAPTOR_TYPE_ETHDR, 0 },
+};
+
+void mtk_ovl_adaptor_layer_config(struct device *dev, unsigned int idx,
+				  struct mtk_plane_state *state,
+				  struct cmdq_pkt *cmdq_pkt)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+	struct mtk_plane_pending_state *pending = &state->pending;
+	struct mtk_mdp_rdma_cfg rdma_config = {0};
+	struct device *rdma_l;
+	struct device *rdma_r;
+	struct device *merge;
+	struct device *ethdr;
+	const struct drm_format_info *fmt_info = drm_format_info(pending->format);
+	bool use_dual_pipe = false;
+	unsigned int align_width;
+	unsigned int l_w = 0;
+	unsigned int r_w = 0;
+
+	dev_dbg(dev, "%s+ idx:%d, enable:%d, fmt:0x%x\n", __func__, idx,
+		pending->enable, pending->format);
+	dev_dbg(dev, "addr 0x%pad, fb w:%d, {%d,%d,%d,%d}\n",
+		&pending->addr, (pending->pitch / fmt_info->cpp[0]),
+		pending->x, pending->y, pending->width, pending->height);
+
+	rdma_l = ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_MDP_RDMA0 + 2 * idx];
+	rdma_r = ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_MDP_RDMA0 + 2 * idx + 1];
+	merge = ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_MERGE0 + idx];
+	ethdr = ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_ETHDR0];
+
+	if (!pending->enable) {
+		mtk_merge_stop_cmdq(merge, cmdq_pkt);
+		mtk_mdp_rdma_stop(rdma_l, cmdq_pkt);
+		mtk_mdp_rdma_stop(rdma_r, cmdq_pkt);
+		mtk_ethdr_layer_config(ethdr, idx, state, cmdq_pkt);
+		return;
+	}
+
+	/* ETHDR is in 1T2P domain, width needs to be 2 pixels align */
+	align_width = ALIGN_DOWN(pending->width, 2);
+
+	if (align_width > MTK_OVL_ADAPTOR_RDMA_MAX_WIDTH)
+		use_dual_pipe = true;
+
+	if (use_dual_pipe) {
+		l_w = (align_width / 2) + ((pending->width / 2) % 2);
+		r_w = align_width - l_w;
+	} else {
+		l_w = align_width;
+	}
+	mtk_merge_advance_config(merge, l_w, r_w, pending->height, 0, 0, cmdq_pkt);
+	mtk_mmsys_merge_async_config(ovl_adaptor->mmsys_dev, idx, align_width / 2,
+				     pending->height, cmdq_pkt);
+
+	rdma_config.width = l_w;
+	rdma_config.height = pending->height;
+	rdma_config.addr0 = pending->addr;
+	rdma_config.pitch = pending->pitch;
+	rdma_config.fmt = pending->format;
+	rdma_config.color_encoding = pending->color_encoding;
+	mtk_mdp_rdma_config(rdma_l, &rdma_config, cmdq_pkt);
+
+	if (use_dual_pipe) {
+		rdma_config.x_left = l_w;
+		rdma_config.width = r_w;
+		mtk_mdp_rdma_config(rdma_r, &rdma_config, cmdq_pkt);
+	}
+
+	mtk_merge_start_cmdq(merge, cmdq_pkt);
+
+	mtk_mdp_rdma_start(rdma_l, cmdq_pkt);
+	if (use_dual_pipe)
+		mtk_mdp_rdma_start(rdma_r, cmdq_pkt);
+	else
+		mtk_mdp_rdma_stop(rdma_r, cmdq_pkt);
+
+	mtk_ethdr_layer_config(ethdr, idx, state, cmdq_pkt);
+}
+
+void mtk_ovl_adaptor_config(struct device *dev, unsigned int w,
+			    unsigned int h, unsigned int vrefresh,
+			    unsigned int bpc, struct cmdq_pkt *cmdq_pkt)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+
+	mtk_ethdr_config(ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_ETHDR0], w, h,
+			 vrefresh, bpc, cmdq_pkt);
+}
+
+void mtk_ovl_adaptor_start(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+
+	mtk_ethdr_start(ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_ETHDR0]);
+}
+
+void mtk_ovl_adaptor_stop(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+
+	mtk_ethdr_stop(ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_ETHDR0]);
+}
+
+int mtk_ovl_adaptor_clk_enable(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+	struct device *comp;
+	int ret;
+	int i;
+
+	for (i = 0; i < OVL_ADAPTOR_MERGE0; i++) {
+		comp = ovl_adaptor->ovl_adaptor_comp[i];
+		ret = pm_runtime_get_sync(comp);
+		if (ret < 0) {
+			dev_err(dev, "Failed to enable power domain %d, err %d\n", i, ret);
+			goto pwr_err;
+		}
+	}
+
+	for (i = 0; i < OVL_ADAPTOR_ID_MAX; i++) {
+		comp = ovl_adaptor->ovl_adaptor_comp[i];
+
+		if (i < OVL_ADAPTOR_MERGE0)
+			ret = mtk_mdp_rdma_clk_enable(comp);
+		else if (i < OVL_ADAPTOR_ETHDR0)
+			ret = mtk_merge_clk_enable(comp);
+		else
+			ret = mtk_ethdr_clk_enable(comp);
+		if (ret) {
+			dev_err(dev, "Failed to enable clock %d, err %d\n", i, ret);
+			goto clk_err;
+		}
+	}
+
+	return ret;
+
+clk_err:
+	while (--i >= 0) {
+		comp = ovl_adaptor->ovl_adaptor_comp[i];
+		if (i < OVL_ADAPTOR_MERGE0)
+			mtk_mdp_rdma_clk_disable(comp);
+		else if (i < OVL_ADAPTOR_ETHDR0)
+			mtk_merge_clk_disable(comp);
+		else
+			mtk_ethdr_clk_disable(comp);
+	}
+	i = OVL_ADAPTOR_MERGE0;
+
+pwr_err:
+	while (--i >= 0)
+		pm_runtime_put(ovl_adaptor->ovl_adaptor_comp[i]);
+
+	return ret;
+}
+
+void mtk_ovl_adaptor_clk_disable(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+	struct device *comp;
+	int i;
+
+	for (i = 0; i < OVL_ADAPTOR_ID_MAX; i++) {
+		comp = ovl_adaptor->ovl_adaptor_comp[i];
+
+		if (i < OVL_ADAPTOR_MERGE0) {
+			mtk_mdp_rdma_clk_disable(comp);
+			pm_runtime_put(comp);
+		} else if (i < OVL_ADAPTOR_ETHDR0) {
+			mtk_merge_clk_disable(comp);
+		} else {
+			mtk_ethdr_clk_disable(comp);
+		}
+	}
+}
+
+unsigned int mtk_ovl_adaptor_layer_nr(struct device *dev)
+{
+	return MTK_OVL_ADAPTOR_LAYER_NUM;
+}
+
+struct device *mtk_ovl_adaptor_dma_dev_get(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+
+	return ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_MDP_RDMA0];
+}
+
+void mtk_ovl_adaptor_register_vblank_cb(struct device *dev, void (*vblank_cb)(void *),
+					void *vblank_cb_data)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+
+	mtk_ethdr_register_vblank_cb(ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_ETHDR0],
+				     vblank_cb, vblank_cb_data);
+}
+
+void mtk_ovl_adaptor_unregister_vblank_cb(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+
+	mtk_ethdr_unregister_vblank_cb(ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_ETHDR0]);
+}
+
+void mtk_ovl_adaptor_enable_vblank(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+
+	mtk_ethdr_enable_vblank(ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_ETHDR0]);
+}
+
+void mtk_ovl_adaptor_disable_vblank(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+
+	mtk_ethdr_disable_vblank(ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_ETHDR0]);
+}
+
+const u32 *mtk_ovl_adaptor_get_formats(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+
+	return mtk_mdp_rdma_get_formats(ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_MDP_RDMA0]);
+}
+
+size_t mtk_ovl_adaptor_get_num_formats(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *ovl_adaptor = dev_get_drvdata(dev);
+
+	return mtk_mdp_rdma_get_num_formats(ovl_adaptor->ovl_adaptor_comp[OVL_ADAPTOR_MDP_RDMA0]);
+}
+
+void mtk_ovl_adaptor_add_comp(struct device *dev, struct mtk_mutex *mutex)
+{
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MDP_RDMA0);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MDP_RDMA1);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MDP_RDMA2);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MDP_RDMA3);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MDP_RDMA4);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MDP_RDMA5);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MDP_RDMA6);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MDP_RDMA7);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MERGE1);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MERGE2);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MERGE3);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_MERGE4);
+	mtk_mutex_add_comp(mutex, DDP_COMPONENT_ETHDR_MIXER);
+}
+
+void mtk_ovl_adaptor_remove_comp(struct device *dev, struct mtk_mutex *mutex)
+{
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MDP_RDMA0);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MDP_RDMA1);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MDP_RDMA2);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MDP_RDMA3);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MDP_RDMA4);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MDP_RDMA5);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MDP_RDMA6);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MDP_RDMA7);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MERGE1);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MERGE2);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MERGE3);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_MERGE4);
+	mtk_mutex_remove_comp(mutex, DDP_COMPONENT_ETHDR_MIXER);
+}
+
+void mtk_ovl_adaptor_connect(struct device *dev, struct device *mmsys_dev, unsigned int next)
+{
+	mtk_mmsys_ddp_connect(mmsys_dev, DDP_COMPONENT_MDP_RDMA0, DDP_COMPONENT_MERGE1);
+	mtk_mmsys_ddp_connect(mmsys_dev, DDP_COMPONENT_MDP_RDMA1, DDP_COMPONENT_MERGE1);
+	mtk_mmsys_ddp_connect(mmsys_dev, DDP_COMPONENT_MDP_RDMA2, DDP_COMPONENT_MERGE2);
+	mtk_mmsys_ddp_connect(mmsys_dev, DDP_COMPONENT_MERGE1, DDP_COMPONENT_ETHDR_MIXER);
+	mtk_mmsys_ddp_connect(mmsys_dev, DDP_COMPONENT_MERGE2, DDP_COMPONENT_ETHDR_MIXER);
+	mtk_mmsys_ddp_connect(mmsys_dev, DDP_COMPONENT_MERGE3, DDP_COMPONENT_ETHDR_MIXER);
+	mtk_mmsys_ddp_connect(mmsys_dev, DDP_COMPONENT_MERGE4, DDP_COMPONENT_ETHDR_MIXER);
+	mtk_mmsys_ddp_connect(mmsys_dev, DDP_COMPONENT_ETHDR_MIXER, next);
+}
+
+void mtk_ovl_adaptor_disconnect(struct device *dev, struct device *mmsys_dev, unsigned int next)
+{
+	mtk_mmsys_ddp_disconnect(mmsys_dev, DDP_COMPONENT_MDP_RDMA0, DDP_COMPONENT_MERGE1);
+	mtk_mmsys_ddp_disconnect(mmsys_dev, DDP_COMPONENT_MDP_RDMA1, DDP_COMPONENT_MERGE1);
+	mtk_mmsys_ddp_disconnect(mmsys_dev, DDP_COMPONENT_MDP_RDMA2, DDP_COMPONENT_MERGE2);
+	mtk_mmsys_ddp_disconnect(mmsys_dev, DDP_COMPONENT_MERGE1, DDP_COMPONENT_ETHDR_MIXER);
+	mtk_mmsys_ddp_disconnect(mmsys_dev, DDP_COMPONENT_MERGE2, DDP_COMPONENT_ETHDR_MIXER);
+	mtk_mmsys_ddp_disconnect(mmsys_dev, DDP_COMPONENT_MERGE3, DDP_COMPONENT_ETHDR_MIXER);
+	mtk_mmsys_ddp_disconnect(mmsys_dev, DDP_COMPONENT_MERGE4, DDP_COMPONENT_ETHDR_MIXER);
+	mtk_mmsys_ddp_disconnect(mmsys_dev, DDP_COMPONENT_ETHDR_MIXER, next);
+}
+
+static int ovl_adaptor_comp_get_id(struct device *dev, struct device_node *node,
+				   enum mtk_ovl_adaptor_comp_type type)
+{
+	int alias_id = of_alias_get_id(node, private_comp_stem[type]);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(comp_matches); i++)
+		if (comp_matches[i].type == type &&
+		    comp_matches[i].alias_id == alias_id)
+			return i;
+
+	dev_warn(dev, "Failed to get id. type: %d, alias: %d\n", type, alias_id);
+	return -EINVAL;
+}
+
+static const struct of_device_id mtk_ovl_adaptor_comp_dt_ids[] = {
+	{
+		.compatible = "mediatek,mt8195-vdo1-rdma",
+		.data = (void *)OVL_ADAPTOR_TYPE_RDMA,
+	}, {
+		.compatible = "mediatek,mt8195-disp-merge",
+		.data = (void *)OVL_ADAPTOR_TYPE_MERGE,
+	}, {
+		.compatible = "mediatek,mt8195-disp-ethdr",
+		.data = (void *)OVL_ADAPTOR_TYPE_ETHDR,
+	},
+	{},
+};
+
+static int compare_of(struct device *dev, void *data)
+{
+	return dev->of_node == data;
+}
+
+static int ovl_adaptor_comp_init(struct device *dev, struct component_match **match)
+{
+	struct mtk_disp_ovl_adaptor *priv = dev_get_drvdata(dev);
+	struct device_node *node, *parent;
+	struct platform_device *comp_pdev;
+
+	parent = dev->parent->parent->of_node->parent;
+
+	for_each_child_of_node(parent, node) {
+		const struct of_device_id *of_id;
+		enum mtk_ovl_adaptor_comp_type type;
+		int id;
+
+		of_id = of_match_node(mtk_ovl_adaptor_comp_dt_ids, node);
+		if (!of_id)
+			continue;
+
+		if (!of_device_is_available(node)) {
+			dev_dbg(dev, "Skipping disabled component %pOF\n",
+				node);
+			continue;
+		}
+
+		type = (enum mtk_ovl_adaptor_comp_type)of_id->data;
+		id = ovl_adaptor_comp_get_id(dev, node, type);
+		if (id < 0) {
+			dev_warn(dev, "Skipping unknown component %pOF\n",
+				 node);
+			continue;
+		}
+
+		comp_pdev = of_find_device_by_node(node);
+		if (!comp_pdev)
+			return -EPROBE_DEFER;
+
+		priv->ovl_adaptor_comp[id] = &comp_pdev->dev;
+
+		drm_of_component_match_add(dev, match, compare_of, node);
+		dev_dbg(dev, "Adding component match for %pOF\n", node);
+	}
+
+	if (!*match) {
+		dev_err(dev, "No match device for ovl_adaptor\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int mtk_disp_ovl_adaptor_comp_bind(struct device *dev, struct device *master,
+					  void *data)
+{
+	struct mtk_disp_ovl_adaptor *priv = dev_get_drvdata(dev);
+
+	if (!priv->children_bound)
+		return -EPROBE_DEFER;
+
+	return 0;
+}
+
+static void mtk_disp_ovl_adaptor_comp_unbind(struct device *dev, struct device *master,
+					     void *data)
+{
+}
+
+static const struct component_ops mtk_disp_ovl_adaptor_comp_ops = {
+	.bind	= mtk_disp_ovl_adaptor_comp_bind,
+	.unbind = mtk_disp_ovl_adaptor_comp_unbind,
+};
+
+static int mtk_disp_ovl_adaptor_master_bind(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *priv = dev_get_drvdata(dev);
+	int ret;
+
+	ret = component_bind_all(dev, priv->mmsys_dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "component_bind_all failed!\n");
+
+	priv->children_bound = true;
+	return 0;
+}
+
+static void mtk_disp_ovl_adaptor_master_unbind(struct device *dev)
+{
+	struct mtk_disp_ovl_adaptor *priv = dev_get_drvdata(dev);
+
+	priv->children_bound = false;
+}
+
+static const struct component_master_ops mtk_disp_ovl_adaptor_master_ops = {
+	.bind		= mtk_disp_ovl_adaptor_master_bind,
+	.unbind		= mtk_disp_ovl_adaptor_master_unbind,
+};
+
+static int mtk_disp_ovl_adaptor_probe(struct platform_device *pdev)
+{
+	struct mtk_disp_ovl_adaptor *priv;
+	struct device *dev = &pdev->dev;
+	struct component_match *match = NULL;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, priv);
+
+	ret = ovl_adaptor_comp_init(dev, &match);
+	if (ret < 0)
+		return ret;
+
+	priv->mmsys_dev = pdev->dev.platform_data;
+
+	component_master_add_with_match(dev, &mtk_disp_ovl_adaptor_master_ops, match);
+
+	pm_runtime_enable(dev);
+
+	ret = component_add(dev, &mtk_disp_ovl_adaptor_comp_ops);
+	if (ret != 0) {
+		pm_runtime_disable(dev);
+		dev_err(dev, "Failed to add component: %d\n", ret);
+	}
+
+	return ret;
+}
+
+static int mtk_disp_ovl_adaptor_remove(struct platform_device *pdev)
+{
+	component_master_del(&pdev->dev, &mtk_disp_ovl_adaptor_master_ops);
+	pm_runtime_disable(&pdev->dev);
+	return 0;
+}
+
+struct platform_driver mtk_disp_ovl_adaptor_driver = {
+	.probe		= mtk_disp_ovl_adaptor_probe,
+	.remove		= mtk_disp_ovl_adaptor_remove,
+	.driver		= {
+		.name	= "mediatek-disp-ovl-adaptor",
+		.owner	= THIS_MODULE,
+	},
+};
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
index a5a0c3bac35d..d4df17ad600a 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
@@ -55,8 +55,24 @@
 
 #define RDMA_MEM_GMC				0x40402020
 
+static const u32 mt8173_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_YUYV,
+};
+
 struct mtk_disp_rdma_data {
 	unsigned int fifo_size;
+	const u32 *formats;
+	size_t num_formats;
 };
 
 /*
@@ -127,6 +143,20 @@ void mtk_rdma_disable_vblank(struct device *dev)
 	rdma_update_bits(dev, DISP_REG_RDMA_INT_ENABLE, RDMA_FRAME_END_INT, 0);
 }
 
+const u32 *mtk_rdma_get_formats(struct device *dev)
+{
+	struct mtk_disp_rdma *rdma = dev_get_drvdata(dev);
+
+	return rdma->data->formats;
+}
+
+size_t mtk_rdma_get_num_formats(struct device *dev)
+{
+	struct mtk_disp_rdma *rdma = dev_get_drvdata(dev);
+
+	return rdma->data->num_formats;
+}
+
 int mtk_rdma_clk_enable(struct device *dev)
 {
 	struct mtk_disp_rdma *rdma = dev_get_drvdata(dev);
@@ -361,18 +391,26 @@ static int mtk_disp_rdma_remove(struct platform_device *pdev)
 
 static const struct mtk_disp_rdma_data mt2701_rdma_driver_data = {
 	.fifo_size = SZ_4K,
+	.formats = mt8173_formats,
+	.num_formats = ARRAY_SIZE(mt8173_formats),
 };
 
 static const struct mtk_disp_rdma_data mt8173_rdma_driver_data = {
 	.fifo_size = SZ_8K,
+	.formats = mt8173_formats,
+	.num_formats = ARRAY_SIZE(mt8173_formats),
 };
 
 static const struct mtk_disp_rdma_data mt8183_rdma_driver_data = {
 	.fifo_size = 5 * SZ_1K,
+	.formats = mt8173_formats,
+	.num_formats = ARRAY_SIZE(mt8173_formats),
 };
 
 static const struct mtk_disp_rdma_data mt8195_rdma_driver_data = {
 	.fifo_size = 1920,
+	.formats = mt8173_formats,
+	.num_formats = ARRAY_SIZE(mt8173_formats),
 };
 
 static const struct of_device_id mtk_disp_rdma_driver_dt_match[] = {
diff --git a/drivers/gpu/drm/mediatek/mtk_dp.c b/drivers/gpu/drm/mediatek/mtk_dp.c
index 1f94fcc144d3..64eee77452c0 100644
--- a/drivers/gpu/drm/mediatek/mtk_dp.c
+++ b/drivers/gpu/drm/mediatek/mtk_dp.c
@@ -806,10 +806,9 @@ static int mtk_dp_aux_wait_for_completion(struct mtk_dp *mtk_dp, bool is_read)
 }
 
 static int mtk_dp_aux_do_transfer(struct mtk_dp *mtk_dp, bool is_read, u8 cmd,
-				  u32 addr, u8 *buf, size_t length)
+				  u32 addr, u8 *buf, size_t length, u8 *reply_cmd)
 {
 	int ret;
-	u32 reply_cmd;
 
 	if (is_read && (length > DP_AUX_MAX_PAYLOAD_BYTES ||
 			(cmd == DP_AUX_NATIVE_READ && !length)))
@@ -841,10 +840,10 @@ static int mtk_dp_aux_do_transfer(struct mtk_dp *mtk_dp, bool is_read, u8 cmd,
 	/* Wait for feedback from sink device. */
 	ret = mtk_dp_aux_wait_for_completion(mtk_dp, is_read);
 
-	reply_cmd = mtk_dp_read(mtk_dp, MTK_DP_AUX_P0_3624) &
-		    AUX_RX_REPLY_COMMAND_AUX_TX_P0_MASK;
+	*reply_cmd = mtk_dp_read(mtk_dp, MTK_DP_AUX_P0_3624) &
+		     AUX_RX_REPLY_COMMAND_AUX_TX_P0_MASK;
 
-	if (ret || reply_cmd) {
+	if (ret) {
 		u32 phy_status = mtk_dp_read(mtk_dp, MTK_DP_AUX_P0_3628) &
 				 AUX_RX_PHY_STATE_AUX_TX_P0_MASK;
 		if (phy_status != AUX_RX_PHY_STATE_AUX_TX_P0_RX_IDLE) {
@@ -1823,7 +1822,8 @@ static irqreturn_t mtk_dp_hpd_event_thread(int hpd, void *dev)
 	spin_unlock_irqrestore(&mtk_dp->irq_thread_lock, flags);
 
 	if (status & MTK_DP_THREAD_CABLE_STATE_CHG) {
-		drm_helper_hpd_irq_event(mtk_dp->bridge.dev);
+		if (mtk_dp->bridge.dev)
+			drm_helper_hpd_irq_event(mtk_dp->bridge.dev);
 
 		if (!mtk_dp->train_info.cable_plugged_in) {
 			mtk_dp_disable_sdp_aui(mtk_dp);
@@ -2070,7 +2070,7 @@ static ssize_t mtk_dp_aux_transfer(struct drm_dp_aux *mtk_aux,
 		ret = mtk_dp_aux_do_transfer(mtk_dp, is_read, request,
 					     msg->address + accessed_bytes,
 					     msg->buffer + accessed_bytes,
-					     to_access);
+					     to_access, &msg->reply);
 
 		if (ret) {
 			drm_info(mtk_dp->drm_dev,
@@ -2080,7 +2080,6 @@ static ssize_t mtk_dp_aux_transfer(struct drm_dp_aux *mtk_aux,
 		accessed_bytes += to_access;
 	} while (accessed_bytes < msg->size);
 
-	msg->reply = DP_AUX_NATIVE_REPLY_ACK | DP_AUX_I2C_REPLY_ACK;
 	return msg->size;
 err:
 	msg->reply = DP_AUX_NATIVE_REPLY_NACK | DP_AUX_I2C_REPLY_NACK;
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
index 5071f1263216..d40142842f85 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
@@ -58,6 +58,7 @@ struct mtk_drm_crtc {
 #endif
 
 	struct device			*mmsys_dev;
+	struct device			*dma_dev;
 	struct mtk_mutex		*mutex;
 	unsigned int			ddp_comp_nr;
 	struct mtk_ddp_comp		**ddp_comp;
@@ -378,13 +379,17 @@ static int mtk_crtc_ddp_hw_init(struct mtk_drm_crtc *mtk_crtc)
 	}
 
 	for (i = 0; i < mtk_crtc->ddp_comp_nr - 1; i++) {
-		mtk_mmsys_ddp_connect(mtk_crtc->mmsys_dev,
-				      mtk_crtc->ddp_comp[i]->id,
-				      mtk_crtc->ddp_comp[i + 1]->id);
-		mtk_mutex_add_comp(mtk_crtc->mutex,
-					mtk_crtc->ddp_comp[i]->id);
+		if (!mtk_ddp_comp_connect(mtk_crtc->ddp_comp[i], mtk_crtc->mmsys_dev,
+					  mtk_crtc->ddp_comp[i + 1]->id))
+			mtk_mmsys_ddp_connect(mtk_crtc->mmsys_dev,
+					      mtk_crtc->ddp_comp[i]->id,
+					      mtk_crtc->ddp_comp[i + 1]->id);
+		if (!mtk_ddp_comp_add(mtk_crtc->ddp_comp[i], mtk_crtc->mutex))
+			mtk_mutex_add_comp(mtk_crtc->mutex,
+					   mtk_crtc->ddp_comp[i]->id);
 	}
-	mtk_mutex_add_comp(mtk_crtc->mutex, mtk_crtc->ddp_comp[i]->id);
+	if (!mtk_ddp_comp_add(mtk_crtc->ddp_comp[i], mtk_crtc->mutex))
+		mtk_mutex_add_comp(mtk_crtc->mutex, mtk_crtc->ddp_comp[i]->id);
 	mtk_mutex_enable(mtk_crtc->mutex);
 
 	for (i = 0; i < mtk_crtc->ddp_comp_nr; i++) {
@@ -433,17 +438,22 @@ static void mtk_crtc_ddp_hw_fini(struct mtk_drm_crtc *mtk_crtc)
 	}
 
 	for (i = 0; i < mtk_crtc->ddp_comp_nr; i++)
-		mtk_mutex_remove_comp(mtk_crtc->mutex,
-					   mtk_crtc->ddp_comp[i]->id);
+		if (!mtk_ddp_comp_remove(mtk_crtc->ddp_comp[i], mtk_crtc->mutex))
+			mtk_mutex_remove_comp(mtk_crtc->mutex,
+					      mtk_crtc->ddp_comp[i]->id);
 	mtk_mutex_disable(mtk_crtc->mutex);
 	for (i = 0; i < mtk_crtc->ddp_comp_nr - 1; i++) {
-		mtk_mmsys_ddp_disconnect(mtk_crtc->mmsys_dev,
-					 mtk_crtc->ddp_comp[i]->id,
-					 mtk_crtc->ddp_comp[i + 1]->id);
-		mtk_mutex_remove_comp(mtk_crtc->mutex,
-					   mtk_crtc->ddp_comp[i]->id);
+		if (!mtk_ddp_comp_disconnect(mtk_crtc->ddp_comp[i], mtk_crtc->mmsys_dev,
+					     mtk_crtc->ddp_comp[i + 1]->id))
+			mtk_mmsys_ddp_disconnect(mtk_crtc->mmsys_dev,
+						 mtk_crtc->ddp_comp[i]->id,
+						 mtk_crtc->ddp_comp[i + 1]->id);
+		if (!mtk_ddp_comp_remove(mtk_crtc->ddp_comp[i], mtk_crtc->mutex))
+			mtk_mutex_remove_comp(mtk_crtc->mutex,
+					      mtk_crtc->ddp_comp[i]->id);
 	}
-	mtk_mutex_remove_comp(mtk_crtc->mutex, mtk_crtc->ddp_comp[i]->id);
+	if (!mtk_ddp_comp_remove(mtk_crtc->ddp_comp[i], mtk_crtc->mutex))
+		mtk_mutex_remove_comp(mtk_crtc->mutex, mtk_crtc->ddp_comp[i]->id);
 	mtk_crtc_ddp_clk_disable(mtk_crtc);
 	mtk_mutex_unprepare(mtk_crtc->mutex);
 
@@ -856,7 +866,9 @@ static int mtk_drm_crtc_init_comp_planes(struct drm_device *drm_dev,
 				BIT(pipe),
 				mtk_drm_crtc_plane_type(mtk_crtc->layer_nr,
 							num_planes),
-				mtk_ddp_comp_supported_rotations(comp));
+				mtk_ddp_comp_supported_rotations(comp),
+				mtk_ddp_comp_get_formats(comp),
+				mtk_ddp_comp_get_num_formats(comp));
 		if (ret)
 			return ret;
 
@@ -865,22 +877,36 @@ static int mtk_drm_crtc_init_comp_planes(struct drm_device *drm_dev,
 	return 0;
 }
 
+struct device *mtk_drm_crtc_dma_dev_get(struct drm_crtc *crtc)
+{
+	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+
+	return mtk_crtc->dma_dev;
+}
+
 int mtk_drm_crtc_create(struct drm_device *drm_dev,
-			const enum mtk_ddp_comp_id *path, unsigned int path_len)
+			const unsigned int *path, unsigned int path_len,
+			int priv_data_index)
 {
 	struct mtk_drm_private *priv = drm_dev->dev_private;
 	struct device *dev = drm_dev->dev;
 	struct mtk_drm_crtc *mtk_crtc;
 	unsigned int num_comp_planes = 0;
-	int pipe = priv->num_pipes;
 	int ret;
 	int i;
 	bool has_ctm = false;
 	uint gamma_lut_size = 0;
+	struct drm_crtc *tmp;
+	int crtc_i = 0;
 
 	if (!path)
 		return 0;
 
+	priv = priv->all_drm_private[priv_data_index];
+
+	drm_for_each_crtc(tmp, drm_dev)
+		crtc_i++;
+
 	for (i = 0; i < path_len; i++) {
 		enum mtk_ddp_comp_id comp_id = path[i];
 		struct device_node *node;
@@ -889,10 +915,13 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 		node = priv->comp_node[comp_id];
 		comp = &priv->ddp_comp[comp_id];
 
-		if (!node) {
+		/* Not all drm components have a DTS device node, such as ovl_adaptor,
+		 * which is the drm bring up sub driver
+		 */
+		if (!node && comp_id != DDP_COMPONENT_DRM_OVL_ADAPTOR) {
 			dev_info(dev,
-				 "Not creating crtc %d because component %d is disabled or missing\n",
-				 pipe, comp_id);
+				"Not creating crtc %d because component %d is disabled or missing\n",
+				crtc_i, comp_id);
 			return 0;
 		}
 
@@ -922,7 +951,7 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 	}
 
 	for (i = 0; i < mtk_crtc->ddp_comp_nr; i++) {
-		enum mtk_ddp_comp_id comp_id = path[i];
+		unsigned int comp_id = path[i];
 		struct mtk_ddp_comp *comp;
 
 		comp = &priv->ddp_comp[comp_id];
@@ -950,29 +979,35 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 
 	for (i = 0; i < mtk_crtc->ddp_comp_nr; i++) {
 		ret = mtk_drm_crtc_init_comp_planes(drm_dev, mtk_crtc, i,
-						    pipe);
+						    crtc_i);
 		if (ret)
 			return ret;
 	}
 
-	ret = mtk_drm_crtc_init(drm_dev, mtk_crtc, pipe);
+	/*
+	 * Default to use the first component as the dma dev.
+	 * In the case of ovl_adaptor sub driver, it needs to use the
+	 * dma_dev_get function to get representative dma dev.
+	 */
+	mtk_crtc->dma_dev = mtk_ddp_comp_dma_dev_get(&priv->ddp_comp[path[0]]);
+
+	ret = mtk_drm_crtc_init(drm_dev, mtk_crtc, crtc_i);
 	if (ret < 0)
 		return ret;
 
 	if (gamma_lut_size)
 		drm_mode_crtc_set_gamma_size(&mtk_crtc->base, gamma_lut_size);
 	drm_crtc_enable_color_mgmt(&mtk_crtc->base, 0, has_ctm, gamma_lut_size);
-	priv->num_pipes++;
 	mutex_init(&mtk_crtc->hw_lock);
 
 #if IS_REACHABLE(CONFIG_MTK_CMDQ)
+	i = priv->mbox_index++;
 	mtk_crtc->cmdq_client.client.dev = mtk_crtc->mmsys_dev;
 	mtk_crtc->cmdq_client.client.tx_block = false;
 	mtk_crtc->cmdq_client.client.knows_txdone = true;
 	mtk_crtc->cmdq_client.client.rx_callback = ddp_cmdq_cb;
 	mtk_crtc->cmdq_client.chan =
-			mbox_request_channel(&mtk_crtc->cmdq_client.client,
-					     drm_crtc_index(&mtk_crtc->base));
+			mbox_request_channel(&mtk_crtc->cmdq_client.client, i);
 	if (IS_ERR(mtk_crtc->cmdq_client.chan)) {
 		dev_dbg(dev, "mtk_crtc %d failed to create mailbox client, writing register by CPU now\n",
 			drm_crtc_index(&mtk_crtc->base));
@@ -982,7 +1017,7 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 	if (mtk_crtc->cmdq_client.chan) {
 		ret = of_property_read_u32_index(priv->mutex_node,
 						 "mediatek,gce-events",
-						 drm_crtc_index(&mtk_crtc->base),
+						 i,
 						 &mtk_crtc->cmdq_event);
 		if (ret) {
 			dev_dbg(dev, "mtk_crtc %d failed to get mediatek,gce-events property\n",
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.h b/drivers/gpu/drm/mediatek/mtk_drm_crtc.h
index cb9a36c48d4f..3e9046993d09 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.h
@@ -16,11 +16,13 @@
 
 void mtk_drm_crtc_commit(struct drm_crtc *crtc);
 int mtk_drm_crtc_create(struct drm_device *drm_dev,
-			const enum mtk_ddp_comp_id *path,
-			unsigned int path_len);
+			const unsigned int *path,
+			unsigned int path_len,
+			int priv_data_index);
 int mtk_drm_crtc_plane_check(struct drm_crtc *crtc, struct drm_plane *plane,
 			     struct mtk_plane_state *state);
 void mtk_drm_crtc_async_update(struct drm_crtc *crtc, struct drm_plane *plane,
 			       struct drm_atomic_state *plane_state);
+struct device *mtk_drm_crtc_dma_dev_get(struct drm_crtc *crtc);
 
 #endif /* MTK_DRM_CRTC_H */
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
index 6b6d5335c834..f114da4d36a9 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
@@ -359,6 +359,8 @@ static const struct mtk_ddp_comp_funcs ddp_ovl = {
 	.layer_config = mtk_ovl_layer_config,
 	.bgclr_in_on = mtk_ovl_bgclr_in_on,
 	.bgclr_in_off = mtk_ovl_bgclr_in_off,
+	.get_formats = mtk_ovl_get_formats,
+	.get_num_formats = mtk_ovl_get_num_formats,
 };
 
 static const struct mtk_ddp_comp_funcs ddp_postmask = {
@@ -381,6 +383,8 @@ static const struct mtk_ddp_comp_funcs ddp_rdma = {
 	.disable_vblank = mtk_rdma_disable_vblank,
 	.layer_nr = mtk_rdma_layer_nr,
 	.layer_config = mtk_rdma_layer_config,
+	.get_formats = mtk_rdma_get_formats,
+	.get_num_formats = mtk_rdma_get_num_formats,
 };
 
 static const struct mtk_ddp_comp_funcs ddp_ufoe = {
@@ -389,6 +393,27 @@ static const struct mtk_ddp_comp_funcs ddp_ufoe = {
 	.start = mtk_ufoe_start,
 };
 
+static const struct mtk_ddp_comp_funcs ddp_ovl_adaptor = {
+	.clk_enable = mtk_ovl_adaptor_clk_enable,
+	.clk_disable = mtk_ovl_adaptor_clk_disable,
+	.config = mtk_ovl_adaptor_config,
+	.start = mtk_ovl_adaptor_start,
+	.stop = mtk_ovl_adaptor_stop,
+	.layer_nr = mtk_ovl_adaptor_layer_nr,
+	.layer_config = mtk_ovl_adaptor_layer_config,
+	.register_vblank_cb = mtk_ovl_adaptor_register_vblank_cb,
+	.unregister_vblank_cb = mtk_ovl_adaptor_unregister_vblank_cb,
+	.enable_vblank = mtk_ovl_adaptor_enable_vblank,
+	.disable_vblank = mtk_ovl_adaptor_disable_vblank,
+	.dma_dev_get = mtk_ovl_adaptor_dma_dev_get,
+	.connect = mtk_ovl_adaptor_connect,
+	.disconnect = mtk_ovl_adaptor_disconnect,
+	.add = mtk_ovl_adaptor_add_comp,
+	.remove = mtk_ovl_adaptor_remove_comp,
+	.get_formats = mtk_ovl_adaptor_get_formats,
+	.get_num_formats = mtk_ovl_adaptor_get_num_formats,
+};
+
 static const char * const mtk_ddp_comp_stem[MTK_DDP_COMP_TYPE_MAX] = {
 	[MTK_DISP_AAL] = "aal",
 	[MTK_DISP_BLS] = "bls",
@@ -402,6 +427,7 @@ static const char * const mtk_ddp_comp_stem[MTK_DDP_COMP_TYPE_MAX] = {
 	[MTK_DISP_OD] = "od",
 	[MTK_DISP_OVL] = "ovl",
 	[MTK_DISP_OVL_2L] = "ovl-2l",
+	[MTK_DISP_OVL_ADAPTOR] = "ovl_adaptor",
 	[MTK_DISP_POSTMASK] = "postmask",
 	[MTK_DISP_PWM] = "pwm",
 	[MTK_DISP_RDMA] = "rdma",
@@ -418,53 +444,54 @@ struct mtk_ddp_comp_match {
 	const struct mtk_ddp_comp_funcs *funcs;
 };
 
-static const struct mtk_ddp_comp_match mtk_ddp_matches[DDP_COMPONENT_ID_MAX] = {
-	[DDP_COMPONENT_AAL0]		= { MTK_DISP_AAL,	0, &ddp_aal },
-	[DDP_COMPONENT_AAL1]		= { MTK_DISP_AAL,	1, &ddp_aal },
-	[DDP_COMPONENT_BLS]		= { MTK_DISP_BLS,	0, NULL },
-	[DDP_COMPONENT_CCORR]		= { MTK_DISP_CCORR,	0, &ddp_ccorr },
-	[DDP_COMPONENT_COLOR0]		= { MTK_DISP_COLOR,	0, &ddp_color },
-	[DDP_COMPONENT_COLOR1]		= { MTK_DISP_COLOR,	1, &ddp_color },
-	[DDP_COMPONENT_DITHER0]		= { MTK_DISP_DITHER,	0, &ddp_dither },
-	[DDP_COMPONENT_DP_INTF0]	= { MTK_DP_INTF,	0, &ddp_dpi },
-	[DDP_COMPONENT_DP_INTF1]	= { MTK_DP_INTF,	1, &ddp_dpi },
-	[DDP_COMPONENT_DPI0]		= { MTK_DPI,		0, &ddp_dpi },
-	[DDP_COMPONENT_DPI1]		= { MTK_DPI,		1, &ddp_dpi },
-	[DDP_COMPONENT_DSC0]		= { MTK_DISP_DSC,	0, &ddp_dsc },
-	[DDP_COMPONENT_DSC1]		= { MTK_DISP_DSC,	1, &ddp_dsc },
-	[DDP_COMPONENT_DSI0]		= { MTK_DSI,		0, &ddp_dsi },
-	[DDP_COMPONENT_DSI1]		= { MTK_DSI,		1, &ddp_dsi },
-	[DDP_COMPONENT_DSI2]		= { MTK_DSI,		2, &ddp_dsi },
-	[DDP_COMPONENT_DSI3]		= { MTK_DSI,		3, &ddp_dsi },
-	[DDP_COMPONENT_GAMMA]		= { MTK_DISP_GAMMA,	0, &ddp_gamma },
-	[DDP_COMPONENT_MERGE0]		= { MTK_DISP_MERGE,	0, &ddp_merge },
-	[DDP_COMPONENT_MERGE1]		= { MTK_DISP_MERGE,	1, &ddp_merge },
-	[DDP_COMPONENT_MERGE2]		= { MTK_DISP_MERGE,	2, &ddp_merge },
-	[DDP_COMPONENT_MERGE3]		= { MTK_DISP_MERGE,	3, &ddp_merge },
-	[DDP_COMPONENT_MERGE4]		= { MTK_DISP_MERGE,	4, &ddp_merge },
-	[DDP_COMPONENT_MERGE5]		= { MTK_DISP_MERGE,	5, &ddp_merge },
-	[DDP_COMPONENT_OD0]		= { MTK_DISP_OD,	0, &ddp_od },
-	[DDP_COMPONENT_OD1]		= { MTK_DISP_OD,	1, &ddp_od },
-	[DDP_COMPONENT_OVL0]		= { MTK_DISP_OVL,	0, &ddp_ovl },
-	[DDP_COMPONENT_OVL1]		= { MTK_DISP_OVL,	1, &ddp_ovl },
-	[DDP_COMPONENT_OVL_2L0]		= { MTK_DISP_OVL_2L,	0, &ddp_ovl },
-	[DDP_COMPONENT_OVL_2L1]		= { MTK_DISP_OVL_2L,	1, &ddp_ovl },
-	[DDP_COMPONENT_OVL_2L2]		= { MTK_DISP_OVL_2L,    2, &ddp_ovl },
-	[DDP_COMPONENT_POSTMASK0]	= { MTK_DISP_POSTMASK,	0, &ddp_postmask },
-	[DDP_COMPONENT_PWM0]		= { MTK_DISP_PWM,	0, NULL },
-	[DDP_COMPONENT_PWM1]		= { MTK_DISP_PWM,	1, NULL },
-	[DDP_COMPONENT_PWM2]		= { MTK_DISP_PWM,	2, NULL },
-	[DDP_COMPONENT_RDMA0]		= { MTK_DISP_RDMA,	0, &ddp_rdma },
-	[DDP_COMPONENT_RDMA1]		= { MTK_DISP_RDMA,	1, &ddp_rdma },
-	[DDP_COMPONENT_RDMA2]		= { MTK_DISP_RDMA,	2, &ddp_rdma },
-	[DDP_COMPONENT_RDMA4]		= { MTK_DISP_RDMA,      4, &ddp_rdma },
-	[DDP_COMPONENT_UFOE]		= { MTK_DISP_UFOE,	0, &ddp_ufoe },
-	[DDP_COMPONENT_WDMA0]		= { MTK_DISP_WDMA,	0, NULL },
-	[DDP_COMPONENT_WDMA1]		= { MTK_DISP_WDMA,	1, NULL },
+static const struct mtk_ddp_comp_match mtk_ddp_matches[DDP_COMPONENT_DRM_ID_MAX] = {
+	[DDP_COMPONENT_AAL0]		= { MTK_DISP_AAL,		0, &ddp_aal },
+	[DDP_COMPONENT_AAL1]		= { MTK_DISP_AAL,		1, &ddp_aal },
+	[DDP_COMPONENT_BLS]		= { MTK_DISP_BLS,		0, NULL },
+	[DDP_COMPONENT_CCORR]		= { MTK_DISP_CCORR,		0, &ddp_ccorr },
+	[DDP_COMPONENT_COLOR0]		= { MTK_DISP_COLOR,		0, &ddp_color },
+	[DDP_COMPONENT_COLOR1]		= { MTK_DISP_COLOR,		1, &ddp_color },
+	[DDP_COMPONENT_DITHER0]		= { MTK_DISP_DITHER,		0, &ddp_dither },
+	[DDP_COMPONENT_DP_INTF0]	= { MTK_DP_INTF,		0, &ddp_dpi },
+	[DDP_COMPONENT_DP_INTF1]	= { MTK_DP_INTF,		1, &ddp_dpi },
+	[DDP_COMPONENT_DPI0]		= { MTK_DPI,			0, &ddp_dpi },
+	[DDP_COMPONENT_DPI1]		= { MTK_DPI,			1, &ddp_dpi },
+	[DDP_COMPONENT_DRM_OVL_ADAPTOR]	= { MTK_DISP_OVL_ADAPTOR,	0, &ddp_ovl_adaptor },
+	[DDP_COMPONENT_DSC0]		= { MTK_DISP_DSC,		0, &ddp_dsc },
+	[DDP_COMPONENT_DSC1]		= { MTK_DISP_DSC,		1, &ddp_dsc },
+	[DDP_COMPONENT_DSI0]		= { MTK_DSI,			0, &ddp_dsi },
+	[DDP_COMPONENT_DSI1]		= { MTK_DSI,			1, &ddp_dsi },
+	[DDP_COMPONENT_DSI2]		= { MTK_DSI,			2, &ddp_dsi },
+	[DDP_COMPONENT_DSI3]		= { MTK_DSI,			3, &ddp_dsi },
+	[DDP_COMPONENT_GAMMA]		= { MTK_DISP_GAMMA,		0, &ddp_gamma },
+	[DDP_COMPONENT_MERGE0]		= { MTK_DISP_MERGE,		0, &ddp_merge },
+	[DDP_COMPONENT_MERGE1]		= { MTK_DISP_MERGE,		1, &ddp_merge },
+	[DDP_COMPONENT_MERGE2]		= { MTK_DISP_MERGE,		2, &ddp_merge },
+	[DDP_COMPONENT_MERGE3]		= { MTK_DISP_MERGE,		3, &ddp_merge },
+	[DDP_COMPONENT_MERGE4]		= { MTK_DISP_MERGE,		4, &ddp_merge },
+	[DDP_COMPONENT_MERGE5]		= { MTK_DISP_MERGE,		5, &ddp_merge },
+	[DDP_COMPONENT_OD0]		= { MTK_DISP_OD,		0, &ddp_od },
+	[DDP_COMPONENT_OD1]		= { MTK_DISP_OD,		1, &ddp_od },
+	[DDP_COMPONENT_OVL0]		= { MTK_DISP_OVL,		0, &ddp_ovl },
+	[DDP_COMPONENT_OVL1]		= { MTK_DISP_OVL,		1, &ddp_ovl },
+	[DDP_COMPONENT_OVL_2L0]		= { MTK_DISP_OVL_2L,		0, &ddp_ovl },
+	[DDP_COMPONENT_OVL_2L1]		= { MTK_DISP_OVL_2L,		1, &ddp_ovl },
+	[DDP_COMPONENT_OVL_2L2]		= { MTK_DISP_OVL_2L,		2, &ddp_ovl },
+	[DDP_COMPONENT_POSTMASK0]	= { MTK_DISP_POSTMASK,		0, &ddp_postmask },
+	[DDP_COMPONENT_PWM0]		= { MTK_DISP_PWM,		0, NULL },
+	[DDP_COMPONENT_PWM1]		= { MTK_DISP_PWM,		1, NULL },
+	[DDP_COMPONENT_PWM2]		= { MTK_DISP_PWM,		2, NULL },
+	[DDP_COMPONENT_RDMA0]		= { MTK_DISP_RDMA,		0, &ddp_rdma },
+	[DDP_COMPONENT_RDMA1]		= { MTK_DISP_RDMA,		1, &ddp_rdma },
+	[DDP_COMPONENT_RDMA2]		= { MTK_DISP_RDMA,		2, &ddp_rdma },
+	[DDP_COMPONENT_RDMA4]		= { MTK_DISP_RDMA,		4, &ddp_rdma },
+	[DDP_COMPONENT_UFOE]		= { MTK_DISP_UFOE,		0, &ddp_ufoe },
+	[DDP_COMPONENT_WDMA0]		= { MTK_DISP_WDMA,		0, NULL },
+	[DDP_COMPONENT_WDMA1]		= { MTK_DISP_WDMA,		1, NULL },
 };
 
 static bool mtk_drm_find_comp_in_ddp(struct device *dev,
-				     const enum mtk_ddp_comp_id *path,
+				     const unsigned int *path,
 				     unsigned int path_len,
 				     struct mtk_ddp_comp *ddp_comp)
 {
@@ -517,7 +544,7 @@ unsigned int mtk_drm_find_possible_crtc_by_comp(struct drm_device *drm,
 }
 
 int mtk_ddp_comp_init(struct device_node *node, struct mtk_ddp_comp *comp,
-		      enum mtk_ddp_comp_id comp_id)
+		      unsigned int comp_id)
 {
 	struct platform_device *comp_pdev;
 	enum mtk_ddp_comp_type type;
@@ -526,19 +553,24 @@ int mtk_ddp_comp_init(struct device_node *node, struct mtk_ddp_comp *comp,
 	int ret;
 #endif
 
-	if (comp_id < 0 || comp_id >= DDP_COMPONENT_ID_MAX)
+	if (comp_id < 0 || comp_id >= DDP_COMPONENT_DRM_ID_MAX)
 		return -EINVAL;
 
 	type = mtk_ddp_matches[comp_id].type;
 
 	comp->id = comp_id;
 	comp->funcs = mtk_ddp_matches[comp_id].funcs;
-	comp_pdev = of_find_device_by_node(node);
-	if (!comp_pdev) {
-		DRM_INFO("Waiting for device %s\n", node->full_name);
-		return -EPROBE_DEFER;
+	/* Not all drm components have a DTS device node, such as ovl_adaptor,
+	 * which is the drm bring up sub driver
+	 */
+	if (node) {
+		comp_pdev = of_find_device_by_node(node);
+		if (!comp_pdev) {
+			DRM_INFO("Waiting for device %s\n", node->full_name);
+			return -EPROBE_DEFER;
+		}
+		comp->dev = &comp_pdev->dev;
 	}
-	comp->dev = &comp_pdev->dev;
 
 	if (type == MTK_DISP_AAL ||
 	    type == MTK_DISP_BLS ||
@@ -548,6 +580,7 @@ int mtk_ddp_comp_init(struct device_node *node, struct mtk_ddp_comp *comp,
 	    type == MTK_DISP_MERGE ||
 	    type == MTK_DISP_OVL ||
 	    type == MTK_DISP_OVL_2L ||
+	    type == MTK_DISP_OVL_ADAPTOR ||
 	    type == MTK_DISP_PWM ||
 	    type == MTK_DISP_RDMA ||
 	    type == MTK_DPI ||
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h
index 2d0052c23dcb..febcaeef16a1 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h
@@ -9,6 +9,7 @@
 #include <linux/io.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
 #include <linux/soc/mediatek/mtk-mmsys.h>
+#include <linux/soc/mediatek/mtk-mutex.h>
 
 struct device;
 struct device_node;
@@ -30,6 +31,7 @@ enum mtk_ddp_comp_type {
 	MTK_DISP_OD,
 	MTK_DISP_OVL,
 	MTK_DISP_OVL_2L,
+	MTK_DISP_OVL_ADAPTOR,
 	MTK_DISP_POSTMASK,
 	MTK_DISP_PWM,
 	MTK_DISP_RDMA,
@@ -71,12 +73,19 @@ struct mtk_ddp_comp_funcs {
 	void (*bgclr_in_off)(struct device *dev);
 	void (*ctm_set)(struct device *dev,
 			struct drm_crtc_state *state);
+	struct device * (*dma_dev_get)(struct device *dev);
+	const u32 *(*get_formats)(struct device *dev);
+	size_t (*get_num_formats)(struct device *dev);
+	void (*connect)(struct device *dev, struct device *mmsys_dev, unsigned int next);
+	void (*disconnect)(struct device *dev, struct device *mmsys_dev, unsigned int next);
+	void (*add)(struct device *dev, struct mtk_mutex *mutex);
+	void (*remove)(struct device *dev, struct mtk_mutex *mutex);
 };
 
 struct mtk_ddp_comp {
 	struct device *dev;
 	int irq;
-	enum mtk_ddp_comp_id id;
+	unsigned int id;
 	const struct mtk_ddp_comp_funcs *funcs;
 };
 
@@ -203,13 +212,76 @@ static inline void mtk_ddp_ctm_set(struct mtk_ddp_comp *comp,
 		comp->funcs->ctm_set(comp->dev, state);
 }
 
+static inline struct device *mtk_ddp_comp_dma_dev_get(struct mtk_ddp_comp *comp)
+{
+	if (comp->funcs && comp->funcs->dma_dev_get)
+		return comp->funcs->dma_dev_get(comp->dev);
+	return comp->dev;
+}
+
+static inline
+const u32 *mtk_ddp_comp_get_formats(struct mtk_ddp_comp *comp)
+{
+	if (comp->funcs && comp->funcs->get_formats)
+		return comp->funcs->get_formats(comp->dev);
+
+	return NULL;
+}
+
+static inline
+size_t mtk_ddp_comp_get_num_formats(struct mtk_ddp_comp *comp)
+{
+	if (comp->funcs && comp->funcs->get_num_formats)
+		return comp->funcs->get_num_formats(comp->dev);
+
+	return 0;
+}
+
+static inline bool mtk_ddp_comp_add(struct mtk_ddp_comp *comp, struct mtk_mutex *mutex)
+{
+	if (comp->funcs && comp->funcs->add) {
+		comp->funcs->add(comp->dev, mutex);
+		return true;
+	}
+	return false;
+}
+
+static inline bool mtk_ddp_comp_remove(struct mtk_ddp_comp *comp, struct mtk_mutex *mutex)
+{
+	if (comp->funcs && comp->funcs->remove) {
+		comp->funcs->remove(comp->dev, mutex);
+		return true;
+	}
+	return false;
+}
+
+static inline bool mtk_ddp_comp_connect(struct mtk_ddp_comp *comp, struct device *mmsys_dev,
+					unsigned int next)
+{
+	if (comp->funcs && comp->funcs->connect) {
+		comp->funcs->connect(comp->dev, mmsys_dev, next);
+		return true;
+	}
+	return false;
+}
+
+static inline bool mtk_ddp_comp_disconnect(struct mtk_ddp_comp *comp, struct device *mmsys_dev,
+					   unsigned int next)
+{
+	if (comp->funcs && comp->funcs->disconnect) {
+		comp->funcs->disconnect(comp->dev, mmsys_dev, next);
+		return true;
+	}
+	return false;
+}
+
 int mtk_ddp_comp_get_id(struct device_node *node,
 			enum mtk_ddp_comp_type comp_type);
 unsigned int mtk_drm_find_possible_crtc_by_comp(struct drm_device *drm,
 						struct device *dev);
 int mtk_ddp_comp_init(struct device_node *comp_node, struct mtk_ddp_comp *comp,
-		      enum mtk_ddp_comp_id comp_id);
-enum mtk_ddp_comp_type mtk_ddp_comp_get_type(enum mtk_ddp_comp_id comp_id);
+		      unsigned int comp_id);
+enum mtk_ddp_comp_type mtk_ddp_comp_get_type(unsigned int comp_id);
 void mtk_ddp_write(struct cmdq_pkt *cmdq_pkt, unsigned int value,
 		   struct cmdq_client_reg *cmdq_reg, void __iomem *regs,
 		   unsigned int offset);
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
index a13b36ac03a1..6dcb4ba2466c 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
@@ -4,8 +4,6 @@
  * Author: YT SHEN <yt.shen@mediatek.com>
  */
 
-#include <linux/clk.h>
-#include <linux/clk-provider.h>
 #include <linux/component.h>
 #include <linux/iommu.h>
 #include <linux/module.h>
@@ -60,7 +58,7 @@ static const struct drm_mode_config_funcs mtk_drm_mode_config_funcs = {
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-static const enum mtk_ddp_comp_id mt2701_mtk_ddp_main[] = {
+static const unsigned int mt2701_mtk_ddp_main[] = {
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_RDMA0,
 	DDP_COMPONENT_COLOR0,
@@ -68,12 +66,12 @@ static const enum mtk_ddp_comp_id mt2701_mtk_ddp_main[] = {
 	DDP_COMPONENT_DSI0,
 };
 
-static const enum mtk_ddp_comp_id mt2701_mtk_ddp_ext[] = {
+static const unsigned int mt2701_mtk_ddp_ext[] = {
 	DDP_COMPONENT_RDMA1,
 	DDP_COMPONENT_DPI0,
 };
 
-static const enum mtk_ddp_comp_id mt7623_mtk_ddp_main[] = {
+static const unsigned int mt7623_mtk_ddp_main[] = {
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_RDMA0,
 	DDP_COMPONENT_COLOR0,
@@ -81,12 +79,12 @@ static const enum mtk_ddp_comp_id mt7623_mtk_ddp_main[] = {
 	DDP_COMPONENT_DPI0,
 };
 
-static const enum mtk_ddp_comp_id mt7623_mtk_ddp_ext[] = {
+static const unsigned int mt7623_mtk_ddp_ext[] = {
 	DDP_COMPONENT_RDMA1,
 	DDP_COMPONENT_DSI0,
 };
 
-static const enum mtk_ddp_comp_id mt2712_mtk_ddp_main[] = {
+static const unsigned int mt2712_mtk_ddp_main[] = {
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_COLOR0,
 	DDP_COMPONENT_AAL0,
@@ -96,7 +94,7 @@ static const enum mtk_ddp_comp_id mt2712_mtk_ddp_main[] = {
 	DDP_COMPONENT_PWM0,
 };
 
-static const enum mtk_ddp_comp_id mt2712_mtk_ddp_ext[] = {
+static const unsigned int mt2712_mtk_ddp_ext[] = {
 	DDP_COMPONENT_OVL1,
 	DDP_COMPONENT_COLOR1,
 	DDP_COMPONENT_AAL1,
@@ -106,13 +104,13 @@ static const enum mtk_ddp_comp_id mt2712_mtk_ddp_ext[] = {
 	DDP_COMPONENT_PWM1,
 };
 
-static const enum mtk_ddp_comp_id mt2712_mtk_ddp_third[] = {
+static const unsigned int mt2712_mtk_ddp_third[] = {
 	DDP_COMPONENT_RDMA2,
 	DDP_COMPONENT_DSI3,
 	DDP_COMPONENT_PWM2,
 };
 
-static enum mtk_ddp_comp_id mt8167_mtk_ddp_main[] = {
+static unsigned int mt8167_mtk_ddp_main[] = {
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_COLOR0,
 	DDP_COMPONENT_CCORR,
@@ -123,7 +121,7 @@ static enum mtk_ddp_comp_id mt8167_mtk_ddp_main[] = {
 	DDP_COMPONENT_DSI0,
 };
 
-static const enum mtk_ddp_comp_id mt8173_mtk_ddp_main[] = {
+static const unsigned int mt8173_mtk_ddp_main[] = {
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_COLOR0,
 	DDP_COMPONENT_AAL0,
@@ -134,7 +132,7 @@ static const enum mtk_ddp_comp_id mt8173_mtk_ddp_main[] = {
 	DDP_COMPONENT_PWM0,
 };
 
-static const enum mtk_ddp_comp_id mt8173_mtk_ddp_ext[] = {
+static const unsigned int mt8173_mtk_ddp_ext[] = {
 	DDP_COMPONENT_OVL1,
 	DDP_COMPONENT_COLOR1,
 	DDP_COMPONENT_GAMMA,
@@ -142,7 +140,7 @@ static const enum mtk_ddp_comp_id mt8173_mtk_ddp_ext[] = {
 	DDP_COMPONENT_DPI0,
 };
 
-static const enum mtk_ddp_comp_id mt8183_mtk_ddp_main[] = {
+static const unsigned int mt8183_mtk_ddp_main[] = {
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_OVL_2L0,
 	DDP_COMPONENT_RDMA0,
@@ -154,13 +152,13 @@ static const enum mtk_ddp_comp_id mt8183_mtk_ddp_main[] = {
 	DDP_COMPONENT_DSI0,
 };
 
-static const enum mtk_ddp_comp_id mt8183_mtk_ddp_ext[] = {
+static const unsigned int mt8183_mtk_ddp_ext[] = {
 	DDP_COMPONENT_OVL_2L1,
 	DDP_COMPONENT_RDMA1,
 	DDP_COMPONENT_DPI0,
 };
 
-static const enum mtk_ddp_comp_id mt8186_mtk_ddp_main[] = {
+static const unsigned int mt8186_mtk_ddp_main[] = {
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_RDMA0,
 	DDP_COMPONENT_COLOR0,
@@ -172,13 +170,25 @@ static const enum mtk_ddp_comp_id mt8186_mtk_ddp_main[] = {
 	DDP_COMPONENT_DSI0,
 };
 
-static const enum mtk_ddp_comp_id mt8186_mtk_ddp_ext[] = {
+static const unsigned int mt8186_mtk_ddp_ext[] = {
 	DDP_COMPONENT_OVL_2L0,
 	DDP_COMPONENT_RDMA1,
 	DDP_COMPONENT_DPI0,
 };
 
-static const enum mtk_ddp_comp_id mt8192_mtk_ddp_main[] = {
+static const unsigned int mt8188_mtk_ddp_main[] = {
+	DDP_COMPONENT_OVL0,
+	DDP_COMPONENT_RDMA0,
+	DDP_COMPONENT_COLOR0,
+	DDP_COMPONENT_CCORR,
+	DDP_COMPONENT_AAL0,
+	DDP_COMPONENT_GAMMA,
+	DDP_COMPONENT_POSTMASK0,
+	DDP_COMPONENT_DITHER0,
+	DDP_COMPONENT_DP_INTF0,
+};
+
+static const unsigned int mt8192_mtk_ddp_main[] = {
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_OVL_2L0,
 	DDP_COMPONENT_RDMA0,
@@ -191,13 +201,13 @@ static const enum mtk_ddp_comp_id mt8192_mtk_ddp_main[] = {
 	DDP_COMPONENT_DSI0,
 };
 
-static const enum mtk_ddp_comp_id mt8192_mtk_ddp_ext[] = {
+static const unsigned int mt8192_mtk_ddp_ext[] = {
 	DDP_COMPONENT_OVL_2L2,
 	DDP_COMPONENT_RDMA4,
 	DDP_COMPONENT_DPI0,
 };
 
-static const enum mtk_ddp_comp_id mt8195_mtk_ddp_main[] = {
+static const unsigned int mt8195_mtk_ddp_main[] = {
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_RDMA0,
 	DDP_COMPONENT_COLOR0,
@@ -210,19 +220,19 @@ static const enum mtk_ddp_comp_id mt8195_mtk_ddp_main[] = {
 	DDP_COMPONENT_DP_INTF0,
 };
 
+static const unsigned int mt8195_mtk_ddp_ext[] = {
+	DDP_COMPONENT_DRM_OVL_ADAPTOR,
+	DDP_COMPONENT_MERGE5,
+	DDP_COMPONENT_DP_INTF1,
+};
+
 static const struct mtk_mmsys_driver_data mt2701_mmsys_driver_data = {
 	.main_path = mt2701_mtk_ddp_main,
 	.main_len = ARRAY_SIZE(mt2701_mtk_ddp_main),
 	.ext_path = mt2701_mtk_ddp_ext,
 	.ext_len = ARRAY_SIZE(mt2701_mtk_ddp_ext),
 	.shadow_register = true,
-};
-
-static const struct mtk_mmsys_match_data mt2701_mmsys_match_data = {
-	.num_drv_data = 1,
-	.drv_data = {
-		&mt2701_mmsys_driver_data,
-	},
+	.mmsys_dev_num = 1,
 };
 
 static const struct mtk_mmsys_driver_data mt7623_mmsys_driver_data = {
@@ -231,13 +241,7 @@ static const struct mtk_mmsys_driver_data mt7623_mmsys_driver_data = {
 	.ext_path = mt7623_mtk_ddp_ext,
 	.ext_len = ARRAY_SIZE(mt7623_mtk_ddp_ext),
 	.shadow_register = true,
-};
-
-static const struct mtk_mmsys_match_data mt7623_mmsys_match_data = {
-	.num_drv_data = 1,
-	.drv_data = {
-		&mt7623_mmsys_driver_data,
-	},
+	.mmsys_dev_num = 1,
 };
 
 static const struct mtk_mmsys_driver_data mt2712_mmsys_driver_data = {
@@ -247,25 +251,13 @@ static const struct mtk_mmsys_driver_data mt2712_mmsys_driver_data = {
 	.ext_len = ARRAY_SIZE(mt2712_mtk_ddp_ext),
 	.third_path = mt2712_mtk_ddp_third,
 	.third_len = ARRAY_SIZE(mt2712_mtk_ddp_third),
-};
-
-static const struct mtk_mmsys_match_data mt2712_mmsys_match_data = {
-	.num_drv_data = 1,
-	.drv_data = {
-		&mt2712_mmsys_driver_data,
-	},
+	.mmsys_dev_num = 1,
 };
 
 static const struct mtk_mmsys_driver_data mt8167_mmsys_driver_data = {
 	.main_path = mt8167_mtk_ddp_main,
 	.main_len = ARRAY_SIZE(mt8167_mtk_ddp_main),
-};
-
-static const struct mtk_mmsys_match_data mt8167_mmsys_match_data = {
-	.num_drv_data = 1,
-	.drv_data = {
-		&mt8167_mmsys_driver_data,
-	},
+	.mmsys_dev_num = 1,
 };
 
 static const struct mtk_mmsys_driver_data mt8173_mmsys_driver_data = {
@@ -273,13 +265,7 @@ static const struct mtk_mmsys_driver_data mt8173_mmsys_driver_data = {
 	.main_len = ARRAY_SIZE(mt8173_mtk_ddp_main),
 	.ext_path = mt8173_mtk_ddp_ext,
 	.ext_len = ARRAY_SIZE(mt8173_mtk_ddp_ext),
-};
-
-static const struct mtk_mmsys_match_data mt8173_mmsys_match_data = {
-	.num_drv_data = 1,
-	.drv_data = {
-		&mt8173_mmsys_driver_data,
-	},
+	.mmsys_dev_num = 1,
 };
 
 static const struct mtk_mmsys_driver_data mt8183_mmsys_driver_data = {
@@ -287,13 +273,7 @@ static const struct mtk_mmsys_driver_data mt8183_mmsys_driver_data = {
 	.main_len = ARRAY_SIZE(mt8183_mtk_ddp_main),
 	.ext_path = mt8183_mtk_ddp_ext,
 	.ext_len = ARRAY_SIZE(mt8183_mtk_ddp_ext),
-};
-
-static const struct mtk_mmsys_match_data mt8183_mmsys_match_data = {
-	.num_drv_data = 1,
-	.drv_data = {
-		&mt8183_mmsys_driver_data,
-	},
+	.mmsys_dev_num = 1,
 };
 
 static const struct mtk_mmsys_driver_data mt8186_mmsys_driver_data = {
@@ -301,13 +281,12 @@ static const struct mtk_mmsys_driver_data mt8186_mmsys_driver_data = {
 	.main_len = ARRAY_SIZE(mt8186_mtk_ddp_main),
 	.ext_path = mt8186_mtk_ddp_ext,
 	.ext_len = ARRAY_SIZE(mt8186_mtk_ddp_ext),
+	.mmsys_dev_num = 1,
 };
 
-static const struct mtk_mmsys_match_data mt8186_mmsys_match_data = {
-	.num_drv_data = 1,
-	.drv_data = {
-		&mt8186_mmsys_driver_data,
-	},
+static const struct mtk_mmsys_driver_data mt8188_vdosys0_driver_data = {
+	.main_path = mt8188_mtk_ddp_main,
+	.main_len = ARRAY_SIZE(mt8188_mtk_ddp_main),
 };
 
 static const struct mtk_mmsys_driver_data mt8192_mmsys_driver_data = {
@@ -315,56 +294,133 @@ static const struct mtk_mmsys_driver_data mt8192_mmsys_driver_data = {
 	.main_len = ARRAY_SIZE(mt8192_mtk_ddp_main),
 	.ext_path = mt8192_mtk_ddp_ext,
 	.ext_len = ARRAY_SIZE(mt8192_mtk_ddp_ext),
-};
-
-static const struct mtk_mmsys_match_data mt8192_mmsys_match_data = {
-	.num_drv_data = 1,
-	.drv_data = {
-		&mt8192_mmsys_driver_data,
-	},
+	.mmsys_dev_num = 1,
 };
 
 static const struct mtk_mmsys_driver_data mt8195_vdosys0_driver_data = {
-	.io_start = 0x1c01a000,
 	.main_path = mt8195_mtk_ddp_main,
 	.main_len = ARRAY_SIZE(mt8195_mtk_ddp_main),
+	.mmsys_dev_num = 2,
 };
 
 static const struct mtk_mmsys_driver_data mt8195_vdosys1_driver_data = {
-	.io_start = 0x1c100000,
+	.ext_path = mt8195_mtk_ddp_ext,
+	.ext_len = ARRAY_SIZE(mt8195_mtk_ddp_ext),
+	.mmsys_id = 1,
+	.mmsys_dev_num = 2,
 };
 
-static const struct mtk_mmsys_match_data mt8195_mmsys_match_data = {
-	.num_drv_data = 1,
-	.drv_data = {
-		&mt8195_vdosys0_driver_data,
-		&mt8195_vdosys1_driver_data,
-	},
+static const struct of_device_id mtk_drm_of_ids[] = {
+	{ .compatible = "mediatek,mt2701-mmsys",
+	  .data = &mt2701_mmsys_driver_data},
+	{ .compatible = "mediatek,mt7623-mmsys",
+	  .data = &mt7623_mmsys_driver_data},
+	{ .compatible = "mediatek,mt2712-mmsys",
+	  .data = &mt2712_mmsys_driver_data},
+	{ .compatible = "mediatek,mt8167-mmsys",
+	  .data = &mt8167_mmsys_driver_data},
+	{ .compatible = "mediatek,mt8173-mmsys",
+	  .data = &mt8173_mmsys_driver_data},
+	{ .compatible = "mediatek,mt8183-mmsys",
+	  .data = &mt8183_mmsys_driver_data},
+	{ .compatible = "mediatek,mt8186-mmsys",
+	  .data = &mt8186_mmsys_driver_data},
+	{ .compatible = "mediatek,mt8188-vdosys0",
+	  .data = &mt8188_vdosys0_driver_data},
+	{ .compatible = "mediatek,mt8192-mmsys",
+	  .data = &mt8192_mmsys_driver_data},
+	{ .compatible = "mediatek,mt8195-mmsys",
+	  .data = &mt8195_vdosys0_driver_data},
+	{ .compatible = "mediatek,mt8195-vdosys0",
+	  .data = &mt8195_vdosys0_driver_data},
+	{ .compatible = "mediatek,mt8195-vdosys1",
+	  .data = &mt8195_vdosys1_driver_data},
+	{ }
 };
+MODULE_DEVICE_TABLE(of, mtk_drm_of_ids);
+
+static int mtk_drm_match(struct device *dev, void *data)
+{
+	if (!strncmp(dev_name(dev), "mediatek-drm", sizeof("mediatek-drm") - 1))
+		return true;
+	return false;
+}
+
+static bool mtk_drm_get_all_drm_priv(struct device *dev)
+{
+	struct mtk_drm_private *drm_priv = dev_get_drvdata(dev);
+	struct mtk_drm_private *all_drm_priv[MAX_CRTC];
+	struct device_node *phandle = dev->parent->of_node;
+	const struct of_device_id *of_id;
+	struct device_node *node;
+	struct device *drm_dev;
+	int cnt = 0;
+	int i, j;
+
+	for_each_child_of_node(phandle->parent, node) {
+		struct platform_device *pdev;
+
+		of_id = of_match_node(mtk_drm_of_ids, node);
+		if (!of_id)
+			continue;
+
+		pdev = of_find_device_by_node(node);
+		if (!pdev)
+			continue;
+
+		drm_dev = device_find_child(&pdev->dev, NULL, mtk_drm_match);
+		if (!drm_dev || !dev_get_drvdata(drm_dev))
+			continue;
+
+		all_drm_priv[cnt] = dev_get_drvdata(drm_dev);
+		if (all_drm_priv[cnt] && all_drm_priv[cnt]->mtk_drm_bound)
+			cnt++;
+	}
+
+	if (drm_priv->data->mmsys_dev_num == cnt) {
+		for (i = 0; i < cnt; i++)
+			for (j = 0; j < cnt; j++)
+				all_drm_priv[j]->all_drm_private[i] = all_drm_priv[i];
+
+		return true;
+	}
+
+	return false;
+}
+
+static bool mtk_drm_find_mmsys_comp(struct mtk_drm_private *private, int comp_id)
+{
+	const struct mtk_mmsys_driver_data *drv_data = private->data;
+	int i;
+
+	if (drv_data->main_path)
+		for (i = 0; i < drv_data->main_len; i++)
+			if (drv_data->main_path[i] == comp_id)
+				return true;
+
+	if (drv_data->ext_path)
+		for (i = 0; i < drv_data->ext_len; i++)
+			if (drv_data->ext_path[i] == comp_id)
+				return true;
+
+	if (drv_data->third_path)
+		for (i = 0; i < drv_data->third_len; i++)
+			if (drv_data->third_path[i] == comp_id)
+				return true;
+
+	return false;
+}
 
 static int mtk_drm_kms_init(struct drm_device *drm)
 {
 	struct mtk_drm_private *private = drm->dev_private;
-	struct platform_device *pdev;
-	struct device_node *np;
-	struct device *dma_dev;
-	int ret;
+	struct mtk_drm_private *priv_n;
+	struct device *dma_dev = NULL;
+	int ret, i, j;
 
 	if (drm_firmware_drivers_only())
 		return -ENODEV;
 
-	if (!iommu_present(&platform_bus_type))
-		return -EPROBE_DEFER;
-
-	pdev = of_find_device_by_node(private->mutex_node);
-	if (!pdev) {
-		dev_err(drm->dev, "Waiting for disp-mutex device %pOF\n",
-			private->mutex_node);
-		of_node_put(private->mutex_node);
-		return -EPROBE_DEFER;
-	}
-	private->mutex_dev = &pdev->dev;
-
 	ret = drmm_mode_config_init(drm);
 	if (ret)
 		goto put_mutex_dev;
@@ -382,9 +438,12 @@ static int mtk_drm_kms_init(struct drm_device *drm)
 	drm->mode_config.funcs = &mtk_drm_mode_config_funcs;
 	drm->mode_config.helper_private = &mtk_drm_mode_config_helpers;
 
-	ret = component_bind_all(drm->dev, drm);
-	if (ret)
-		goto put_mutex_dev;
+	for (i = 0; i < private->data->mmsys_dev_num; i++) {
+		drm->dev_private = private->all_drm_private[i];
+		ret = component_bind_all(private->all_drm_private[i]->dev, drm);
+		if (ret)
+			goto put_mutex_dev;
+	}
 
 	/*
 	 * Ensure internal panels are at the top of the connector list before
@@ -393,37 +452,53 @@ static int mtk_drm_kms_init(struct drm_device *drm)
 	drm_helper_move_panel_connectors_to_head(drm);
 
 	/*
-	 * We currently support two fixed data streams, each optional,
-	 * and each statically assigned to a crtc:
-	 * OVL0 -> COLOR0 -> AAL -> OD -> RDMA0 -> UFOE -> DSI0 ...
+	 * 1. We currently support two fixed data streams, each optional,
+	 *    and each statically assigned to a crtc:
+	 *    OVL0 -> COLOR0 -> AAL -> OD -> RDMA0 -> UFOE -> DSI0 ...
+	 * 2. For multi mmsys architecture, crtc path data are located in
+	 *    different drm private data structures. Loop through crtc index to
+	 *    create crtc from the main path and then ext_path and finally the
+	 *    third path.
 	 */
-	ret = mtk_drm_crtc_create(drm, private->data->main_path,
-				  private->data->main_len);
-	if (ret < 0)
-		goto err_component_unbind;
-	/* ... and OVL1 -> COLOR1 -> GAMMA -> RDMA1 -> DPI0. */
-	ret = mtk_drm_crtc_create(drm, private->data->ext_path,
-				  private->data->ext_len);
-	if (ret < 0)
-		goto err_component_unbind;
-
-	ret = mtk_drm_crtc_create(drm, private->data->third_path,
-				  private->data->third_len);
-	if (ret < 0)
-		goto err_component_unbind;
+	for (i = 0; i < MAX_CRTC; i++) {
+		for (j = 0; j < private->data->mmsys_dev_num; j++) {
+			priv_n = private->all_drm_private[j];
+
+			if (i == 0 && priv_n->data->main_len) {
+				ret = mtk_drm_crtc_create(drm, priv_n->data->main_path,
+							  priv_n->data->main_len, j);
+				if (ret)
+					goto err_component_unbind;
+
+				continue;
+			} else if (i == 1 && priv_n->data->ext_len) {
+				ret = mtk_drm_crtc_create(drm, priv_n->data->ext_path,
+							  priv_n->data->ext_len, j);
+				if (ret)
+					goto err_component_unbind;
+
+				continue;
+			} else if (i == 2 && priv_n->data->third_len) {
+				ret = mtk_drm_crtc_create(drm, priv_n->data->third_path,
+							  priv_n->data->third_len, j);
+				if (ret)
+					goto err_component_unbind;
+
+				continue;
+			}
+		}
+	}
 
 	/* Use OVL device for all DMA memory allocations */
-	np = private->comp_node[private->data->main_path[0]] ?:
-	     private->comp_node[private->data->ext_path[0]];
-	pdev = of_find_device_by_node(np);
-	if (!pdev) {
+	dma_dev = mtk_drm_crtc_dma_dev_get(drm_crtc_from_index(drm, 0));
+	if (!dma_dev) {
 		ret = -ENODEV;
 		dev_err(drm->dev, "Need at least one OVL device\n");
 		goto err_component_unbind;
 	}
 
-	dma_dev = &pdev->dev;
-	private->dma_dev = dma_dev;
+	for (i = 0; i < private->data->mmsys_dev_num; i++)
+		private->all_drm_private[i]->dma_dev = dma_dev;
 
 	/*
 	 * Configure the DMA segment size to make sure we get contiguous IOVA
@@ -445,9 +520,12 @@ static int mtk_drm_kms_init(struct drm_device *drm)
 	return 0;
 
 err_component_unbind:
-	component_unbind_all(drm->dev, drm);
+	for (i = 0; i < private->data->mmsys_dev_num; i++)
+		component_unbind_all(private->all_drm_private[i]->dev, drm);
 put_mutex_dev:
-	put_device(private->mutex_dev);
+	for (i = 0; i < private->data->mmsys_dev_num; i++)
+		put_device(private->all_drm_private[i]->mutex_dev);
+
 	return ret;
 }
 
@@ -492,18 +570,44 @@ static const struct drm_driver mtk_drm_driver = {
 	.minor = DRIVER_MINOR,
 };
 
+static int compare_dev(struct device *dev, void *data)
+{
+	return dev == (struct device *)data;
+}
+
 static int mtk_drm_bind(struct device *dev)
 {
 	struct mtk_drm_private *private = dev_get_drvdata(dev);
+	struct platform_device *pdev;
 	struct drm_device *drm;
-	int ret;
+	int ret, i;
+
+	if (!iommu_present(&platform_bus_type))
+		return -EPROBE_DEFER;
+
+	pdev = of_find_device_by_node(private->mutex_node);
+	if (!pdev) {
+		dev_err(dev, "Waiting for disp-mutex device %pOF\n",
+			private->mutex_node);
+		of_node_put(private->mutex_node);
+		return -EPROBE_DEFER;
+	}
+
+	private->mutex_dev = &pdev->dev;
+	private->mtk_drm_bound = true;
+	private->dev = dev;
+
+	if (!mtk_drm_get_all_drm_priv(dev))
+		return 0;
 
 	drm = drm_dev_alloc(&mtk_drm_driver, dev);
 	if (IS_ERR(drm))
 		return PTR_ERR(drm);
 
+	private->drm_master = true;
 	drm->dev_private = private;
-	private->drm = drm;
+	for (i = 0; i < private->data->mmsys_dev_num; i++)
+		private->all_drm_private[i]->drm = drm;
 
 	ret = mtk_drm_kms_init(drm);
 	if (ret < 0)
@@ -529,10 +633,14 @@ static void mtk_drm_unbind(struct device *dev)
 {
 	struct mtk_drm_private *private = dev_get_drvdata(dev);
 
-	drm_dev_unregister(private->drm);
-	mtk_drm_kms_deinit(private->drm);
-	drm_dev_put(private->drm);
-	private->num_pipes = 0;
+	/* for multi mmsys dev, unregister drm dev in mmsys master */
+	if (private->drm_master) {
+		drm_dev_unregister(private->drm);
+		mtk_drm_kms_deinit(private->drm);
+		drm_dev_put(private->drm);
+	}
+	private->mtk_drm_bound = false;
+	private->drm_master = false;
 	private->drm = NULL;
 }
 
@@ -588,6 +696,8 @@ static const struct of_device_id mtk_ddp_comp_dt_ids[] = {
 	  .data = (void *)MTK_DISP_MUTEX },
 	{ .compatible = "mediatek,mt8186-disp-mutex",
 	  .data = (void *)MTK_DISP_MUTEX },
+	{ .compatible = "mediatek,mt8188-disp-mutex",
+	  .data = (void *)MTK_DISP_MUTEX },
 	{ .compatible = "mediatek,mt8192-disp-mutex",
 	  .data = (void *)MTK_DISP_MUTEX },
 	{ .compatible = "mediatek,mt8195-disp-mutex",
@@ -657,58 +767,15 @@ static const struct of_device_id mtk_ddp_comp_dt_ids[] = {
 	{ }
 };
 
-static const struct of_device_id mtk_drm_of_ids[] = {
-	{ .compatible = "mediatek,mt2701-mmsys",
-	  .data = &mt2701_mmsys_match_data},
-	{ .compatible = "mediatek,mt7623-mmsys",
-	  .data = &mt7623_mmsys_match_data},
-	{ .compatible = "mediatek,mt2712-mmsys",
-	  .data = &mt2712_mmsys_match_data},
-	{ .compatible = "mediatek,mt8167-mmsys",
-	  .data = &mt8167_mmsys_match_data},
-	{ .compatible = "mediatek,mt8173-mmsys",
-	  .data = &mt8173_mmsys_match_data},
-	{ .compatible = "mediatek,mt8183-mmsys",
-	  .data = &mt8183_mmsys_match_data},
-	{ .compatible = "mediatek,mt8186-mmsys",
-	  .data = &mt8186_mmsys_match_data},
-	{ .compatible = "mediatek,mt8192-mmsys",
-	  .data = &mt8192_mmsys_match_data},
-	{ .compatible = "mediatek,mt8195-mmsys",
-	  .data = &mt8195_mmsys_match_data},
-	{ }
-};
-MODULE_DEVICE_TABLE(of, mtk_drm_of_ids);
-
-static int mtk_drm_find_match_data(struct device *dev,
-				   const struct mtk_mmsys_match_data *match_data)
-{
-	int i;
-	struct platform_device *pdev = of_find_device_by_node(dev->parent->of_node);
-	struct resource *res;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		dev_err(dev, "failed to get parent resource\n");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < match_data->num_drv_data; i++)
-		if (match_data->drv_data[i]->io_start == res->start)
-			return i;
-
-	return -EINVAL;
-}
-
 static int mtk_drm_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *phandle = dev->parent->of_node;
 	const struct of_device_id *of_id;
-	const struct mtk_mmsys_match_data *match_data;
 	struct mtk_drm_private *private;
 	struct device_node *node;
 	struct component_match *match = NULL;
+	struct platform_device *ovl_adaptor;
 	int ret;
 	int i;
 
@@ -726,18 +793,24 @@ static int mtk_drm_probe(struct platform_device *pdev)
 	if (!of_id)
 		return -ENODEV;
 
-	match_data = of_id->data;
-	if (match_data->num_drv_data > 1) {
-		/* This SoC has multiple mmsys channels */
-		ret = mtk_drm_find_match_data(dev, match_data);
-		if (ret < 0) {
-			dev_err(dev, "Couldn't get match driver data\n");
-			return ret;
-		}
-		private->data = match_data->drv_data[ret];
-	} else {
-		dev_dbg(dev, "Using single mmsys channel\n");
-		private->data = match_data->drv_data[0];
+	private->data = of_id->data;
+
+	private->all_drm_private = devm_kmalloc_array(dev, private->data->mmsys_dev_num,
+						      sizeof(*private->all_drm_private),
+						      GFP_KERNEL);
+	if (!private->all_drm_private)
+		return -ENOMEM;
+
+	/* Bringup ovl_adaptor */
+	if (mtk_drm_find_mmsys_comp(private, DDP_COMPONENT_DRM_OVL_ADAPTOR)) {
+		ovl_adaptor = platform_device_register_data(dev, "mediatek-disp-ovl-adaptor",
+							    PLATFORM_DEVID_AUTO,
+							    (void *)private->mmsys_dev,
+							    sizeof(*private->mmsys_dev));
+		private->ddp_comp[DDP_COMPONENT_DRM_OVL_ADAPTOR].dev = &ovl_adaptor->dev;
+		mtk_ddp_comp_init(NULL, &private->ddp_comp[DDP_COMPONENT_DRM_OVL_ADAPTOR],
+				  DDP_COMPONENT_DRM_OVL_ADAPTOR);
+		component_match_add(dev, &match, compare_dev, &ovl_adaptor->dev);
 	}
 
 	/* Iterate over sibling DISP function blocks */
@@ -759,7 +832,13 @@ static int mtk_drm_probe(struct platform_device *pdev)
 		comp_type = (enum mtk_ddp_comp_type)of_id->data;
 
 		if (comp_type == MTK_DISP_MUTEX) {
-			private->mutex_node = of_node_get(node);
+			int id;
+
+			id = of_alias_get_id(node, "mutex");
+			if (id < 0 || id == private->data->mmsys_id) {
+				private->mutex_node = of_node_get(node);
+				dev_dbg(dev, "get mutex for mmsys %d", private->data->mmsys_id);
+			}
 			continue;
 		}
 
@@ -770,6 +849,9 @@ static int mtk_drm_probe(struct platform_device *pdev)
 			continue;
 		}
 
+		if (!mtk_drm_find_mmsys_comp(private, comp_id))
+			continue;
+
 		private->comp_node[comp_id] = of_node_get(node);
 
 		/*
@@ -784,6 +866,7 @@ static int mtk_drm_probe(struct platform_device *pdev)
 		    comp_type == MTK_DISP_MERGE ||
 		    comp_type == MTK_DISP_OVL ||
 		    comp_type == MTK_DISP_OVL_2L ||
+		    comp_type == MTK_DISP_OVL_ADAPTOR ||
 		    comp_type == MTK_DISP_RDMA ||
 		    comp_type == MTK_DP_INTF ||
 		    comp_type == MTK_DPI ||
@@ -821,7 +904,7 @@ err_pm:
 	pm_runtime_disable(dev);
 err_node:
 	of_node_put(private->mutex_node);
-	for (i = 0; i < DDP_COMPONENT_ID_MAX; i++)
+	for (i = 0; i < DDP_COMPONENT_DRM_ID_MAX; i++)
 		of_node_put(private->comp_node[i]);
 	return ret;
 }
@@ -834,7 +917,7 @@ static int mtk_drm_remove(struct platform_device *pdev)
 	component_master_del(&pdev->dev, &mtk_drm_ops);
 	pm_runtime_disable(&pdev->dev);
 	of_node_put(private->mutex_node);
-	for (i = 0; i < DDP_COMPONENT_ID_MAX; i++)
+	for (i = 0; i < DDP_COMPONENT_DRM_ID_MAX; i++)
 		of_node_put(private->comp_node[i]);
 
 	return 0;
@@ -845,16 +928,20 @@ static int mtk_drm_sys_prepare(struct device *dev)
 	struct mtk_drm_private *private = dev_get_drvdata(dev);
 	struct drm_device *drm = private->drm;
 
-	return drm_mode_config_helper_suspend(drm);
+	if (private->drm_master)
+		return drm_mode_config_helper_suspend(drm);
+	else
+		return 0;
 }
 
 static void mtk_drm_sys_complete(struct device *dev)
 {
 	struct mtk_drm_private *private = dev_get_drvdata(dev);
 	struct drm_device *drm = private->drm;
-	int ret;
+	int ret = 0;
 
-	ret = drm_mode_config_helper_resume(drm);
+	if (private->drm_master)
+		ret = drm_mode_config_helper_resume(drm);
 	if (ret)
 		dev_err(dev, "Failed to resume\n");
 }
@@ -879,11 +966,13 @@ static struct platform_driver * const mtk_drm_drivers[] = {
 	&mtk_disp_color_driver,
 	&mtk_disp_gamma_driver,
 	&mtk_disp_merge_driver,
+	&mtk_disp_ovl_adaptor_driver,
 	&mtk_disp_ovl_driver,
 	&mtk_disp_rdma_driver,
 	&mtk_dpi_driver,
 	&mtk_drm_platform_driver,
 	&mtk_dsi_driver,
+	&mtk_ethdr_driver,
 	&mtk_mdp_rdma_driver,
 };
 
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.h b/drivers/gpu/drm/mediatek/mtk_drm_drv.h
index 7b37b5cf9629..eb2fd45941f0 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.h
@@ -11,6 +11,8 @@
 
 #define MAX_CRTC	3
 #define MAX_CONNECTOR	2
+#define DDP_COMPONENT_DRM_OVL_ADAPTOR (DDP_COMPONENT_ID_MAX + 1)
+#define DDP_COMPONENT_DRM_ID_MAX (DDP_COMPONENT_DRM_OVL_ADAPTOR + 1)
 
 struct device;
 struct device_node;
@@ -21,35 +23,33 @@ struct drm_property;
 struct regmap;
 
 struct mtk_mmsys_driver_data {
-	const resource_size_t io_start;
-	const enum mtk_ddp_comp_id *main_path;
+	const unsigned int *main_path;
 	unsigned int main_len;
-	const enum mtk_ddp_comp_id *ext_path;
+	const unsigned int *ext_path;
 	unsigned int ext_len;
-	const enum mtk_ddp_comp_id *third_path;
+	const unsigned int *third_path;
 	unsigned int third_len;
 
 	bool shadow_register;
-};
-
-struct mtk_mmsys_match_data {
-	unsigned short num_drv_data;
-	const struct mtk_mmsys_driver_data *drv_data[];
+	unsigned int mmsys_id;
+	unsigned int mmsys_dev_num;
 };
 
 struct mtk_drm_private {
 	struct drm_device *drm;
 	struct device *dma_dev;
-
-	unsigned int num_pipes;
-
+	bool mtk_drm_bound;
+	bool drm_master;
+	struct device *dev;
 	struct device_node *mutex_node;
 	struct device *mutex_dev;
 	struct device *mmsys_dev;
-	struct device_node *comp_node[DDP_COMPONENT_ID_MAX];
-	struct mtk_ddp_comp ddp_comp[DDP_COMPONENT_ID_MAX];
+	struct device_node *comp_node[DDP_COMPONENT_DRM_ID_MAX];
+	struct mtk_ddp_comp ddp_comp[DDP_COMPONENT_DRM_ID_MAX];
 	const struct mtk_mmsys_driver_data *data;
 	struct drm_atomic_state *suspend_state;
+	unsigned int mbox_index;
+	struct mtk_drm_private **all_drm_private;
 };
 
 extern struct platform_driver mtk_disp_aal_driver;
@@ -57,10 +57,12 @@ extern struct platform_driver mtk_disp_ccorr_driver;
 extern struct platform_driver mtk_disp_color_driver;
 extern struct platform_driver mtk_disp_gamma_driver;
 extern struct platform_driver mtk_disp_merge_driver;
+extern struct platform_driver mtk_disp_ovl_adaptor_driver;
 extern struct platform_driver mtk_disp_ovl_driver;
 extern struct platform_driver mtk_disp_rdma_driver;
 extern struct platform_driver mtk_dpi_driver;
 extern struct platform_driver mtk_dsi_driver;
+extern struct platform_driver mtk_ethdr_driver;
 extern struct platform_driver mtk_mdp_rdma_driver;
 
 #endif /* MTK_DRM_DRV_H */
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.c b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
index d54fbf34b000..31f9420aff6f 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_plane.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
@@ -19,20 +19,6 @@
 #include "mtk_drm_gem.h"
 #include "mtk_drm_plane.h"
 
-static const u32 formats[] = {
-	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_BGRX8888,
-	DRM_FORMAT_BGRA8888,
-	DRM_FORMAT_ABGR8888,
-	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_RGB888,
-	DRM_FORMAT_BGR888,
-	DRM_FORMAT_RGB565,
-	DRM_FORMAT_UYVY,
-	DRM_FORMAT_YUYV,
-};
-
 static const u64 modifiers[] = {
 	DRM_FORMAT_MOD_LINEAR,
 	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_32x8 |
@@ -315,13 +301,19 @@ static const struct drm_plane_helper_funcs mtk_plane_helper_funcs = {
 
 int mtk_plane_init(struct drm_device *dev, struct drm_plane *plane,
 		   unsigned long possible_crtcs, enum drm_plane_type type,
-		   unsigned int supported_rotations)
+		   unsigned int supported_rotations, const u32 *formats,
+		   size_t num_formats)
 {
 	int err;
 
+	if (!formats || !num_formats) {
+		DRM_ERROR("no formats for plane\n");
+		return -EINVAL;
+	}
+
 	err = drm_universal_plane_init(dev, plane, possible_crtcs,
 				       &mtk_plane_funcs, formats,
-				       ARRAY_SIZE(formats), modifiers, type, NULL);
+				       num_formats, modifiers, type, NULL);
 	if (err) {
 		DRM_ERROR("failed to initialize plane\n");
 		return err;
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.h b/drivers/gpu/drm/mediatek/mtk_drm_plane.h
index 8f39011cdbfc..99aff7da0831 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_plane.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_plane.h
@@ -48,6 +48,7 @@ to_mtk_plane_state(struct drm_plane_state *state)
 
 int mtk_plane_init(struct drm_device *dev, struct drm_plane *plane,
 		   unsigned long possible_crtcs, enum drm_plane_type type,
-		   unsigned int supported_rotations);
+		   unsigned int supported_rotations, const u32 *formats,
+		   size_t num_formats);
 
 #endif
diff --git a/drivers/gpu/drm/mediatek/mtk_ethdr.c b/drivers/gpu/drm/mediatek/mtk_ethdr.c
new file mode 100644
index 000000000000..73dc4da3ba3b
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_ethdr.c
@@ -0,0 +1,370 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2021 MediaTek Inc.
+ */
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/soc/mediatek/mtk-cmdq.h>
+#include <linux/soc/mediatek/mtk-mmsys.h>
+
+#include "mtk_drm_crtc.h"
+#include "mtk_drm_ddp_comp.h"
+#include "mtk_drm_drv.h"
+#include "mtk_ethdr.h"
+
+#define MIX_INTEN			0x4
+#define MIX_FME_CPL_INTEN			BIT(1)
+#define MIX_INTSTA			0x8
+#define MIX_EN				0xc
+#define MIX_RST				0x14
+#define MIX_ROI_SIZE			0x18
+#define MIX_DATAPATH_CON		0x1c
+#define OUTPUT_NO_RND				BIT(3)
+#define SOURCE_RGB_SEL				BIT(7)
+#define BACKGROUND_RELAY			(4 << 9)
+#define MIX_ROI_BGCLR			0x20
+#define BGCLR_BLACK				0xff000000
+#define MIX_SRC_CON			0x24
+#define MIX_SRC_L0_EN				BIT(0)
+#define MIX_L_SRC_CON(n)		(0x28 + 0x18 * (n))
+#define NON_PREMULTI_SOURCE			(2 << 12)
+#define MIX_L_SRC_SIZE(n)		(0x30 + 0x18 * (n))
+#define MIX_L_SRC_OFFSET(n)		(0x34 + 0x18 * (n))
+#define MIX_FUNC_DCM0			0x120
+#define MIX_FUNC_DCM1			0x124
+#define MIX_FUNC_DCM_ENABLE			0xffffffff
+
+#define HDR_VDO_FE_0804_HDR_DM_FE	0x804
+#define HDR_VDO_FE_0804_BYPASS_ALL		0xfd
+#define HDR_GFX_FE_0204_GFX_HDR_FE	0x204
+#define HDR_GFX_FE_0204_BYPASS_ALL		0xfd
+#define HDR_VDO_BE_0204_VDO_DM_BE	0x204
+#define HDR_VDO_BE_0204_BYPASS_ALL		0x7e
+
+#define MIXER_INX_MODE_BYPASS			0
+#define MIXER_INX_MODE_EVEN_EXTEND		1
+#define DEFAULT_9BIT_ALPHA			0x100
+#define	MIXER_ALPHA_AEN				BIT(8)
+#define	MIXER_ALPHA				0xff
+#define ETHDR_CLK_NUM				13
+
+enum mtk_ethdr_comp_id {
+	ETHDR_MIXER,
+	ETHDR_VDO_FE0,
+	ETHDR_VDO_FE1,
+	ETHDR_GFX_FE0,
+	ETHDR_GFX_FE1,
+	ETHDR_VDO_BE,
+	ETHDR_ADL_DS,
+	ETHDR_ID_MAX
+};
+
+struct mtk_ethdr_comp {
+	struct device		*dev;
+	void __iomem		*regs;
+	struct cmdq_client_reg	cmdq_base;
+};
+
+struct mtk_ethdr {
+	struct mtk_ethdr_comp	ethdr_comp[ETHDR_ID_MAX];
+	struct clk_bulk_data	ethdr_clk[ETHDR_CLK_NUM];
+	struct device		*mmsys_dev;
+	void			(*vblank_cb)(void *data);
+	void			*vblank_cb_data;
+	int			irq;
+	struct reset_control	*reset_ctl;
+};
+
+static const char * const ethdr_clk_str[] = {
+	"ethdr_top",
+	"mixer",
+	"vdo_fe0",
+	"vdo_fe1",
+	"gfx_fe0",
+	"gfx_fe1",
+	"vdo_be",
+	"adl_ds",
+	"vdo_fe0_async",
+	"vdo_fe1_async",
+	"gfx_fe0_async",
+	"gfx_fe1_async",
+	"vdo_be_async",
+};
+
+void mtk_ethdr_register_vblank_cb(struct device *dev,
+				  void (*vblank_cb)(void *),
+				  void *vblank_cb_data)
+{
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+
+	priv->vblank_cb = vblank_cb;
+	priv->vblank_cb_data = vblank_cb_data;
+}
+
+void mtk_ethdr_unregister_vblank_cb(struct device *dev)
+{
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+
+	priv->vblank_cb = NULL;
+	priv->vblank_cb_data = NULL;
+}
+
+void mtk_ethdr_enable_vblank(struct device *dev)
+{
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+
+	writel(MIX_FME_CPL_INTEN, priv->ethdr_comp[ETHDR_MIXER].regs + MIX_INTEN);
+}
+
+void mtk_ethdr_disable_vblank(struct device *dev)
+{
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+
+	writel(0x0, priv->ethdr_comp[ETHDR_MIXER].regs + MIX_INTEN);
+}
+
+static irqreturn_t mtk_ethdr_irq_handler(int irq, void *dev_id)
+{
+	struct mtk_ethdr *priv = dev_id;
+
+	writel(0x0, priv->ethdr_comp[ETHDR_MIXER].regs + MIX_INTSTA);
+
+	if (!priv->vblank_cb)
+		return IRQ_NONE;
+
+	priv->vblank_cb(priv->vblank_cb_data);
+
+	return IRQ_HANDLED;
+}
+
+void mtk_ethdr_layer_config(struct device *dev, unsigned int idx,
+			    struct mtk_plane_state *state,
+			    struct cmdq_pkt *cmdq_pkt)
+{
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+	struct mtk_ethdr_comp *mixer = &priv->ethdr_comp[ETHDR_MIXER];
+	struct mtk_plane_pending_state *pending = &state->pending;
+	unsigned int offset = (pending->x & 1) << 31 | pending->y << 16 | pending->x;
+	unsigned int align_width = ALIGN_DOWN(pending->width, 2);
+	unsigned int alpha_con = 0;
+
+	dev_dbg(dev, "%s+ idx:%d", __func__, idx);
+
+	if (idx >= 4)
+		return;
+
+	if (!pending->enable) {
+		mtk_ddp_write(cmdq_pkt, 0, &mixer->cmdq_base, mixer->regs, MIX_L_SRC_SIZE(idx));
+		return;
+	}
+
+	if (state->base.fb && state->base.fb->format->has_alpha)
+		alpha_con = MIXER_ALPHA_AEN | MIXER_ALPHA;
+
+	mtk_mmsys_mixer_in_config(priv->mmsys_dev, idx + 1, alpha_con ? false : true,
+				  DEFAULT_9BIT_ALPHA,
+				  pending->x & 1 ? MIXER_INX_MODE_EVEN_EXTEND :
+				  MIXER_INX_MODE_BYPASS, align_width / 2 - 1, cmdq_pkt);
+
+	mtk_ddp_write(cmdq_pkt, pending->height << 16 | align_width, &mixer->cmdq_base,
+		      mixer->regs, MIX_L_SRC_SIZE(idx));
+	mtk_ddp_write(cmdq_pkt, offset, &mixer->cmdq_base, mixer->regs, MIX_L_SRC_OFFSET(idx));
+	mtk_ddp_write_mask(cmdq_pkt, alpha_con, &mixer->cmdq_base, mixer->regs, MIX_L_SRC_CON(idx),
+			   0x1ff);
+	mtk_ddp_write_mask(cmdq_pkt, BIT(idx), &mixer->cmdq_base, mixer->regs, MIX_SRC_CON,
+			   BIT(idx));
+}
+
+void mtk_ethdr_config(struct device *dev, unsigned int w,
+		      unsigned int h, unsigned int vrefresh,
+		      unsigned int bpc, struct cmdq_pkt *cmdq_pkt)
+{
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+	struct mtk_ethdr_comp *vdo_fe0 = &priv->ethdr_comp[ETHDR_VDO_FE0];
+	struct mtk_ethdr_comp *vdo_fe1 = &priv->ethdr_comp[ETHDR_VDO_FE1];
+	struct mtk_ethdr_comp *gfx_fe0 = &priv->ethdr_comp[ETHDR_GFX_FE0];
+	struct mtk_ethdr_comp *gfx_fe1 = &priv->ethdr_comp[ETHDR_GFX_FE1];
+	struct mtk_ethdr_comp *vdo_be = &priv->ethdr_comp[ETHDR_VDO_BE];
+	struct mtk_ethdr_comp *mixer = &priv->ethdr_comp[ETHDR_MIXER];
+
+	dev_dbg(dev, "%s-w:%d, h:%d\n", __func__, w, h);
+
+	mtk_ddp_write(cmdq_pkt, HDR_VDO_FE_0804_BYPASS_ALL, &vdo_fe0->cmdq_base,
+		      vdo_fe0->regs, HDR_VDO_FE_0804_HDR_DM_FE);
+
+	mtk_ddp_write(cmdq_pkt, HDR_VDO_FE_0804_BYPASS_ALL, &vdo_fe1->cmdq_base,
+		      vdo_fe1->regs, HDR_VDO_FE_0804_HDR_DM_FE);
+
+	mtk_ddp_write(cmdq_pkt, HDR_GFX_FE_0204_BYPASS_ALL, &gfx_fe0->cmdq_base,
+		      gfx_fe0->regs, HDR_GFX_FE_0204_GFX_HDR_FE);
+
+	mtk_ddp_write(cmdq_pkt, HDR_GFX_FE_0204_BYPASS_ALL, &gfx_fe1->cmdq_base,
+		      gfx_fe1->regs, HDR_GFX_FE_0204_GFX_HDR_FE);
+
+	mtk_ddp_write(cmdq_pkt, HDR_VDO_BE_0204_BYPASS_ALL, &vdo_be->cmdq_base,
+		      vdo_be->regs, HDR_VDO_BE_0204_VDO_DM_BE);
+
+	mtk_ddp_write(cmdq_pkt, MIX_FUNC_DCM_ENABLE, &mixer->cmdq_base, mixer->regs, MIX_FUNC_DCM0);
+	mtk_ddp_write(cmdq_pkt, MIX_FUNC_DCM_ENABLE, &mixer->cmdq_base, mixer->regs, MIX_FUNC_DCM1);
+	mtk_ddp_write(cmdq_pkt, h << 16 | w, &mixer->cmdq_base, mixer->regs, MIX_ROI_SIZE);
+	mtk_ddp_write(cmdq_pkt, BGCLR_BLACK, &mixer->cmdq_base, mixer->regs, MIX_ROI_BGCLR);
+	mtk_ddp_write(cmdq_pkt, NON_PREMULTI_SOURCE, &mixer->cmdq_base, mixer->regs,
+		      MIX_L_SRC_CON(0));
+	mtk_ddp_write(cmdq_pkt, NON_PREMULTI_SOURCE, &mixer->cmdq_base, mixer->regs,
+		      MIX_L_SRC_CON(1));
+	mtk_ddp_write(cmdq_pkt, NON_PREMULTI_SOURCE, &mixer->cmdq_base, mixer->regs,
+		      MIX_L_SRC_CON(2));
+	mtk_ddp_write(cmdq_pkt, NON_PREMULTI_SOURCE, &mixer->cmdq_base, mixer->regs,
+		      MIX_L_SRC_CON(3));
+	mtk_ddp_write(cmdq_pkt, 0x0, &mixer->cmdq_base, mixer->regs, MIX_L_SRC_SIZE(0));
+	mtk_ddp_write(cmdq_pkt, OUTPUT_NO_RND | SOURCE_RGB_SEL | BACKGROUND_RELAY,
+		      &mixer->cmdq_base, mixer->regs, MIX_DATAPATH_CON);
+	mtk_ddp_write_mask(cmdq_pkt, MIX_SRC_L0_EN, &mixer->cmdq_base, mixer->regs,
+			   MIX_SRC_CON, MIX_SRC_L0_EN);
+
+	mtk_mmsys_hdr_config(priv->mmsys_dev, w / 2, h, cmdq_pkt);
+	mtk_mmsys_mixer_in_channel_swap(priv->mmsys_dev, 4, 0, cmdq_pkt);
+}
+
+void mtk_ethdr_start(struct device *dev)
+{
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+	struct mtk_ethdr_comp *mixer = &priv->ethdr_comp[ETHDR_MIXER];
+
+	writel(1, mixer->regs + MIX_EN);
+}
+
+void mtk_ethdr_stop(struct device *dev)
+{
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+	struct mtk_ethdr_comp *mixer = &priv->ethdr_comp[ETHDR_MIXER];
+
+	writel(0, mixer->regs + MIX_EN);
+	writel(1, mixer->regs + MIX_RST);
+	reset_control_reset(priv->reset_ctl);
+	writel(0, mixer->regs + MIX_RST);
+}
+
+int mtk_ethdr_clk_enable(struct device *dev)
+{
+	int ret;
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+
+	ret = clk_bulk_prepare_enable(ETHDR_CLK_NUM, priv->ethdr_clk);
+	if (ret)
+		dev_err(dev,
+			"ethdr_clk prepare enable failed\n");
+	return ret;
+}
+
+void mtk_ethdr_clk_disable(struct device *dev)
+{
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+
+	clk_bulk_disable_unprepare(ETHDR_CLK_NUM, priv->ethdr_clk);
+}
+
+static int mtk_ethdr_bind(struct device *dev, struct device *master,
+			  void *data)
+{
+	struct mtk_ethdr *priv = dev_get_drvdata(dev);
+
+	priv->mmsys_dev = data;
+	return 0;
+}
+
+static void mtk_ethdr_unbind(struct device *dev, struct device *master, void *data)
+{
+}
+
+static const struct component_ops mtk_ethdr_component_ops = {
+	.bind	= mtk_ethdr_bind,
+	.unbind = mtk_ethdr_unbind,
+};
+
+static int mtk_ethdr_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mtk_ethdr *priv;
+	int ret;
+	int i;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	for (i = 0; i < ETHDR_ID_MAX; i++) {
+		priv->ethdr_comp[i].dev = dev;
+		priv->ethdr_comp[i].regs = of_iomap(dev->of_node, i);
+#if IS_REACHABLE(CONFIG_MTK_CMDQ)
+		ret = cmdq_dev_get_client_reg(dev,
+					      &priv->ethdr_comp[i].cmdq_base, i);
+		if (ret)
+			dev_dbg(dev, "get mediatek,gce-client-reg fail!\n");
+#endif
+		dev_dbg(dev, "[DRM]regs:0x%p, node:%d\n", priv->ethdr_comp[i].regs, i);
+	}
+
+	for (i = 0; i < ETHDR_CLK_NUM; i++)
+		priv->ethdr_clk[i].id = ethdr_clk_str[i];
+	ret = devm_clk_bulk_get_optional(dev, ETHDR_CLK_NUM, priv->ethdr_clk);
+	if (ret)
+		return ret;
+
+	priv->irq = platform_get_irq(pdev, 0);
+	if (priv->irq < 0)
+		priv->irq = 0;
+
+	if (priv->irq) {
+		ret = devm_request_irq(dev, priv->irq, mtk_ethdr_irq_handler,
+				       IRQF_TRIGGER_NONE, dev_name(dev), priv);
+		if (ret < 0) {
+			dev_err(dev, "Failed to request irq %d: %d\n", priv->irq, ret);
+			return ret;
+		}
+	}
+
+	priv->reset_ctl = devm_reset_control_array_get_optional_exclusive(dev);
+	if (IS_ERR(priv->reset_ctl)) {
+		dev_err_probe(dev, PTR_ERR(priv->reset_ctl), "cannot get ethdr reset control\n");
+		return PTR_ERR(priv->reset_ctl);
+	}
+
+	platform_set_drvdata(pdev, priv);
+
+	ret = component_add(dev, &mtk_ethdr_component_ops);
+	if (ret)
+		dev_notice(dev, "Failed to add component: %d\n", ret);
+
+	return ret;
+}
+
+static int mtk_ethdr_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &mtk_ethdr_component_ops);
+	return 0;
+}
+
+static const struct of_device_id mtk_ethdr_driver_dt_match[] = {
+	{ .compatible = "mediatek,mt8195-disp-ethdr"},
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, mtk_ethdr_driver_dt_match);
+
+struct platform_driver mtk_ethdr_driver = {
+	.probe		= mtk_ethdr_probe,
+	.remove		= mtk_ethdr_remove,
+	.driver		= {
+		.name	= "mediatek-disp-ethdr",
+		.owner	= THIS_MODULE,
+		.of_match_table = mtk_ethdr_driver_dt_match,
+	},
+};
diff --git a/drivers/gpu/drm/mediatek/mtk_ethdr.h b/drivers/gpu/drm/mediatek/mtk_ethdr.h
new file mode 100644
index 000000000000..81af9edea3f7
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_ethdr.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2021 MediaTek Inc.
+ */
+
+#ifndef __MTK_ETHDR_H__
+#define __MTK_ETHDR_H__
+
+void mtk_ethdr_start(struct device *dev);
+void mtk_ethdr_stop(struct device *dev);
+int mtk_ethdr_clk_enable(struct device *dev);
+void mtk_ethdr_clk_disable(struct device *dev);
+void mtk_ethdr_config(struct device *dev, unsigned int w,
+		      unsigned int h, unsigned int vrefresh,
+		      unsigned int bpc, struct cmdq_pkt *cmdq_pkt);
+void mtk_ethdr_layer_config(struct device *dev, unsigned int idx,
+			    struct mtk_plane_state *state,
+			    struct cmdq_pkt *cmdq_pkt);
+void mtk_ethdr_register_vblank_cb(struct device *dev,
+				  void (*vblank_cb)(void *),
+				  void *vblank_cb_data);
+void mtk_ethdr_unregister_vblank_cb(struct device *dev);
+void mtk_ethdr_enable_vblank(struct device *dev);
+void mtk_ethdr_disable_vblank(struct device *dev);
+#endif
diff --git a/drivers/gpu/drm/mediatek/mtk_mdp_rdma.c b/drivers/gpu/drm/mediatek/mtk_mdp_rdma.c
index eecfa98ff52e..e06db6e56b5f 100644
--- a/drivers/gpu/drm/mediatek/mtk_mdp_rdma.c
+++ b/drivers/gpu/drm/mediatek/mtk_mdp_rdma.c
@@ -62,6 +62,20 @@
 #define RDMA_CSC_FULL709_TO_RGB			5
 #define RDMA_CSC_BT601_TO_RGB			6
 
+static const u32 formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_YUYV,
+};
+
 enum rdma_format {
 	RDMA_INPUT_FORMAT_RGB565 = 0,
 	RDMA_INPUT_FORMAT_RGB888 = 1,
@@ -219,6 +233,16 @@ void mtk_mdp_rdma_config(struct device *dev, struct mtk_mdp_rdma_cfg *cfg,
 			   MDP_RDMA_MF_CLIP_SIZE, FLD_MF_CLIP_H);
 }
 
+const u32 *mtk_mdp_rdma_get_formats(struct device *dev)
+{
+	return formats;
+}
+
+size_t mtk_mdp_rdma_get_num_formats(struct device *dev)
+{
+	return ARRAY_SIZE(formats);
+}
+
 int mtk_mdp_rdma_clk_enable(struct device *dev)
 {
 	struct mtk_mdp_rdma *rdma = dev_get_drvdata(dev);
diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c
index 7caf937c3c90..bb72fda9106d 100644
--- a/drivers/gpu/drm/meson/meson_drv.c
+++ b/drivers/gpu/drm/meson/meson_drv.c
@@ -18,7 +18,7 @@
 #include <drm/drm_aperture.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
@@ -353,7 +353,7 @@ static int meson_drv_bind_master(struct device *dev, bool has_components)
 	if (ret)
 		goto uninstall_irq;
 
-	drm_fbdev_generic_setup(drm, 32);
+	drm_fbdev_dma_setup(drm, 32);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/meson/meson_venc.c b/drivers/gpu/drm/meson/meson_venc.c
index 3c55ed003359..fcd532db19c1 100644
--- a/drivers/gpu/drm/meson/meson_venc.c
+++ b/drivers/gpu/drm/meson/meson_venc.c
@@ -866,10 +866,10 @@ meson_venc_hdmi_supported_mode(const struct drm_display_mode *mode)
 			    DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC))
 		return MODE_BAD;
 
-	if (mode->hdisplay < 640 || mode->hdisplay > 1920)
+	if (mode->hdisplay < 400 || mode->hdisplay > 1920)
 		return MODE_BAD_HVALUE;
 
-	if (mode->vdisplay < 480 || mode->vdisplay > 1200)
+	if (mode->vdisplay < 480 || mode->vdisplay > 1920)
 		return MODE_BAD_VVALUE;
 
 	return MODE_OK;
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h
index 9e604dbb8e44..57c7edcab602 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.h
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.h
@@ -375,12 +375,15 @@ int mgag200_primary_plane_helper_atomic_check(struct drm_plane *plane,
 					      struct drm_atomic_state *new_state);
 void mgag200_primary_plane_helper_atomic_update(struct drm_plane *plane,
 						struct drm_atomic_state *old_state);
+void mgag200_primary_plane_helper_atomic_enable(struct drm_plane *plane,
+						struct drm_atomic_state *state);
 void mgag200_primary_plane_helper_atomic_disable(struct drm_plane *plane,
 						 struct drm_atomic_state *old_state);
 #define MGAG200_PRIMARY_PLANE_HELPER_FUNCS \
 	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS, \
 	.atomic_check = mgag200_primary_plane_helper_atomic_check, \
 	.atomic_update = mgag200_primary_plane_helper_atomic_update, \
+	.atomic_enable = mgag200_primary_plane_helper_atomic_enable, \
 	.atomic_disable = mgag200_primary_plane_helper_atomic_disable
 
 #define MGAG200_PRIMARY_PLANE_FUNCS \
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index 0a5aaf78172a..0f2dd26755df 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -501,10 +501,6 @@ void mgag200_primary_plane_helper_atomic_update(struct drm_plane *plane,
 	struct drm_framebuffer *fb = plane_state->fb;
 	struct drm_atomic_helper_damage_iter iter;
 	struct drm_rect damage;
-	u8 seq1;
-
-	if (!fb)
-		return;
 
 	drm_atomic_helper_damage_iter_init(&iter, old_plane_state, plane_state);
 	drm_atomic_for_each_plane_damage(&iter, &damage) {
@@ -514,13 +510,19 @@ void mgag200_primary_plane_helper_atomic_update(struct drm_plane *plane,
 	/* Always scanout image at VRAM offset 0 */
 	mgag200_set_startadd(mdev, (u32)0);
 	mgag200_set_offset(mdev, fb);
+}
 
-	if (!old_plane_state->crtc && plane_state->crtc) { // enabling
-		RREG_SEQ(0x01, seq1);
-		seq1 &= ~MGAREG_SEQ1_SCROFF;
-		WREG_SEQ(0x01, seq1);
-		msleep(20);
-	}
+void mgag200_primary_plane_helper_atomic_enable(struct drm_plane *plane,
+						struct drm_atomic_state *state)
+{
+	struct drm_device *dev = plane->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	u8 seq1;
+
+	RREG_SEQ(0x01, seq1);
+	seq1 &= ~MGAREG_SEQ1_SCROFF;
+	WREG_SEQ(0x01, seq1);
+	msleep(20);
 }
 
 void mgag200_primary_plane_helper_atomic_disable(struct drm_plane *plane,
diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
index 949b18a29a55..85f5ab1d552c 100644
--- a/drivers/gpu/drm/msm/Kconfig
+++ b/drivers/gpu/drm/msm/Kconfig
@@ -9,6 +9,7 @@ config DRM_MSM
 	depends on QCOM_OCMEM || QCOM_OCMEM=n
 	depends on QCOM_LLCC || QCOM_LLCC=n
 	depends on QCOM_COMMAND_DB || QCOM_COMMAND_DB=n
+	depends on PM
 	select IOMMU_IO_PGTABLE
 	select QCOM_MDT_LOADER if ARCH_QCOM
 	select REGULATOR
@@ -28,6 +29,7 @@ config DRM_MSM
 	select SYNC_FILE
 	select PM_OPP
 	select NVMEM
+	select PM_GENERIC_DOMAINS
 	help
 	  DRM/KMS driver for MSM/snapdragon.
 
diff --git a/drivers/gpu/drm/msm/adreno/a2xx.xml.h b/drivers/gpu/drm/msm/adreno/a2xx.xml.h
index afa6023346c4..f87a1312f580 100644
--- a/drivers/gpu/drm/msm/adreno/a2xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a2xx.xml.h
@@ -8,21 +8,21 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a2xx.xml                (  90810 bytes, from 2021-06-21 15:24:24)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  14609 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  69086 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113358 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149512 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 184954 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11331 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   6038 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2924 bytes, from 2021-07-22 15:21:56)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a2xx.xml                (  91929 bytes, from 2023-02-28 23:52:27)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  15434 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  74995 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113474 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149590 bytes, from 2023-02-14 19:37:12)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 198949 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11404 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   9055 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2976 bytes, from 2023-03-10 18:32:52)
+
+Copyright (C) 2013-2023 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -1060,6 +1060,12 @@ enum a2xx_mh_perfcnt_select {
 	AXI_TOTAL_READ_REQUEST_DATA_BEATS = 181,
 };
 
+enum perf_mode_cnt {
+	PERF_STATE_RESET = 0,
+	PERF_STATE_ENABLE = 1,
+	PERF_STATE_FREEZE = 2,
+};
+
 enum adreno_mmu_clnt_beh {
 	BEH_NEVR = 0,
 	BEH_TRAN_RNG = 1,
@@ -1307,6 +1313,18 @@ static inline uint32_t A2XX_MH_MMU_VA_RANGE_VA_BASE(uint32_t val)
 #define A2XX_RBBM_PM_OVERRIDE1_MH_TCROQ_SCLK_PM_OVERRIDE	0x80000000
 
 #define REG_A2XX_RBBM_PM_OVERRIDE2				0x0000039d
+#define A2XX_RBBM_PM_OVERRIDE2_PA_REG_SCLK_PM_OVERRIDE		0x00000001
+#define A2XX_RBBM_PM_OVERRIDE2_PA_PA_SCLK_PM_OVERRIDE		0x00000002
+#define A2XX_RBBM_PM_OVERRIDE2_PA_AG_SCLK_PM_OVERRIDE		0x00000004
+#define A2XX_RBBM_PM_OVERRIDE2_VGT_REG_SCLK_PM_OVERRIDE		0x00000008
+#define A2XX_RBBM_PM_OVERRIDE2_VGT_FIFOS_SCLK_PM_OVERRIDE	0x00000010
+#define A2XX_RBBM_PM_OVERRIDE2_VGT_VGT_SCLK_PM_OVERRIDE		0x00000020
+#define A2XX_RBBM_PM_OVERRIDE2_DEBUG_PERF_SCLK_PM_OVERRIDE	0x00000040
+#define A2XX_RBBM_PM_OVERRIDE2_PERM_SCLK_PM_OVERRIDE		0x00000080
+#define A2XX_RBBM_PM_OVERRIDE2_GC_GA_GMEM0_PM_OVERRIDE		0x00000100
+#define A2XX_RBBM_PM_OVERRIDE2_GC_GA_GMEM1_PM_OVERRIDE		0x00000200
+#define A2XX_RBBM_PM_OVERRIDE2_GC_GA_GMEM2_PM_OVERRIDE		0x00000400
+#define A2XX_RBBM_PM_OVERRIDE2_GC_GA_GMEM3_PM_OVERRIDE		0x00000800
 
 #define REG_A2XX_RBBM_DEBUG_OUT					0x000003a0
 
@@ -1334,6 +1352,12 @@ static inline uint32_t A2XX_MH_MMU_VA_RANGE_VA_BASE(uint32_t val)
 #define REG_A2XX_RBBM_PERIPHID2					0x000003fa
 
 #define REG_A2XX_CP_PERFMON_CNTL				0x00000444
+#define A2XX_CP_PERFMON_CNTL_PERF_MODE_CNT__MASK		0x00000007
+#define A2XX_CP_PERFMON_CNTL_PERF_MODE_CNT__SHIFT		0
+static inline uint32_t A2XX_CP_PERFMON_CNTL_PERF_MODE_CNT(enum perf_mode_cnt val)
+{
+	return ((val) << A2XX_CP_PERFMON_CNTL_PERF_MODE_CNT__SHIFT) & A2XX_CP_PERFMON_CNTL_PERF_MODE_CNT__MASK;
+}
 
 #define REG_A2XX_CP_PERFCOUNTER_SELECT				0x00000445
 
diff --git a/drivers/gpu/drm/msm/adreno/a3xx.xml.h b/drivers/gpu/drm/msm/adreno/a3xx.xml.h
index 520ae3f375a1..237b564445be 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a3xx.xml.h
@@ -8,21 +8,21 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a2xx.xml                (  90810 bytes, from 2021-06-21 15:24:24)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  14609 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  69086 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113358 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149512 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 184954 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11331 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   6038 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2924 bytes, from 2021-07-22 15:21:56)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a2xx.xml                (  91929 bytes, from 2023-02-28 23:52:27)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  15434 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  74995 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113474 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149590 bytes, from 2023-02-14 19:37:12)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 198949 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11404 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   9055 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2976 bytes, from 2023-03-10 18:32:52)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
index 948785ed07bb..c86b377f6f0d 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
@@ -477,6 +477,16 @@ static struct msm_gpu_state *a3xx_gpu_state_get(struct msm_gpu *gpu)
 	return state;
 }
 
+static u64 a3xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
+{
+	u64 busy_cycles;
+
+	busy_cycles = gpu_read64(gpu, REG_A3XX_RBBM_PERFCTR_RBBM_1_LO);
+	*out_sample_rate = clk_get_rate(gpu->core_clk);
+
+	return busy_cycles;
+}
+
 static u32 a3xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 {
 	ring->memptrs->rptr = gpu_read(gpu, REG_AXXX_CP_RB_RPTR);
@@ -498,6 +508,7 @@ static const struct adreno_gpu_funcs funcs = {
 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
 		.show = adreno_show,
 #endif
+		.gpu_busy = a3xx_gpu_busy,
 		.gpu_state_get = a3xx_gpu_state_get,
 		.gpu_state_put = adreno_gpu_state_put,
 		.create_address_space = adreno_create_address_space,
diff --git a/drivers/gpu/drm/msm/adreno/a4xx.xml.h b/drivers/gpu/drm/msm/adreno/a4xx.xml.h
index 7e5c21015d10..ff5f1e98a5fc 100644
--- a/drivers/gpu/drm/msm/adreno/a4xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a4xx.xml.h
@@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a2xx.xml                (  90810 bytes, from 2021-06-21 15:24:24)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  14609 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  69086 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113358 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149512 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 184954 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11331 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   6038 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2924 bytes, from 2021-07-22 15:21:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a2xx.xml                (  91929 bytes, from 2023-02-28 23:52:27)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  15434 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  74995 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113474 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149590 bytes, from 2023-02-14 19:37:12)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 198949 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11404 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   9055 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2976 bytes, from 2023-03-10 18:32:52)
 
 Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
@@ -3159,6 +3159,18 @@ static inline uint32_t A4XX_TPL1_TP_TEX_COUNT_GS(uint32_t val)
 #define REG_A4XX_TPL1_TP_GS_BORDER_COLOR_BASE_ADDR		0x0000238d
 
 #define REG_A4XX_TPL1_TP_FS_TEX_COUNT				0x000023a0
+#define A4XX_TPL1_TP_FS_TEX_COUNT_FS__MASK			0x000000ff
+#define A4XX_TPL1_TP_FS_TEX_COUNT_FS__SHIFT			0
+static inline uint32_t A4XX_TPL1_TP_FS_TEX_COUNT_FS(uint32_t val)
+{
+	return ((val) << A4XX_TPL1_TP_FS_TEX_COUNT_FS__SHIFT) & A4XX_TPL1_TP_FS_TEX_COUNT_FS__MASK;
+}
+#define A4XX_TPL1_TP_FS_TEX_COUNT_CS__MASK			0x0000ff00
+#define A4XX_TPL1_TP_FS_TEX_COUNT_CS__SHIFT			8
+static inline uint32_t A4XX_TPL1_TP_FS_TEX_COUNT_CS(uint32_t val)
+{
+	return ((val) << A4XX_TPL1_TP_FS_TEX_COUNT_CS__SHIFT) & A4XX_TPL1_TP_FS_TEX_COUNT_CS__MASK;
+}
 
 #define REG_A4XX_TPL1_TP_FS_BORDER_COLOR_BASE_ADDR		0x000023a1
 
diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
index 3e09d3a7a0ac..715436cb3996 100644
--- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
@@ -611,6 +611,16 @@ static int a4xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
 	return 0;
 }
 
+static u64 a4xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
+{
+	u64 busy_cycles;
+
+	busy_cycles = gpu_read64(gpu, REG_A4XX_RBBM_PERFCTR_RBBM_1_LO);
+	*out_sample_rate = clk_get_rate(gpu->core_clk);
+
+	return busy_cycles;
+}
+
 static u32 a4xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 {
 	ring->memptrs->rptr = gpu_read(gpu, REG_A4XX_CP_RB_RPTR);
@@ -632,6 +642,7 @@ static const struct adreno_gpu_funcs funcs = {
 #if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
 		.show = adreno_show,
 #endif
+		.gpu_busy = a4xx_gpu_busy,
 		.gpu_state_get = a4xx_gpu_state_get,
 		.gpu_state_put = adreno_gpu_state_put,
 		.create_address_space = adreno_create_address_space,
diff --git a/drivers/gpu/drm/msm/adreno/a5xx.xml.h b/drivers/gpu/drm/msm/adreno/a5xx.xml.h
index 2505b4e43ca0..03b7ee592b11 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a5xx.xml.h
@@ -8,21 +8,21 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a2xx.xml                (  90810 bytes, from 2021-06-21 15:24:24)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  14609 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  69086 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113358 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149512 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 184954 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11331 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   6038 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2924 bytes, from 2021-07-22 15:21:56)
-
-Copyright (C) 2013-2022 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a2xx.xml                (  91929 bytes, from 2023-02-28 23:52:27)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  15434 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  74995 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113474 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149590 bytes, from 2023-02-14 19:37:12)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 198949 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11404 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   9055 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2976 bytes, from 2023-03-10 18:32:52)
+
+Copyright (C) 2013-2023 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -4218,6 +4218,7 @@ static inline uint32_t A5XX_SP_CS_CONFIG_SHADEROBJOFFSET(uint32_t val)
 #define REG_A5XX_SP_FS_CONFIG_MAX_CONST				0x0000e58b
 
 #define REG_A5XX_SP_VS_CTRL_REG0				0x0000e590
+#define A5XX_SP_VS_CTRL_REG0_BUFFER				0x00000004
 #define A5XX_SP_VS_CTRL_REG0_THREADSIZE__MASK			0x00000008
 #define A5XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT			3
 static inline uint32_t A5XX_SP_VS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
@@ -4316,6 +4317,7 @@ static inline uint32_t A5XX_SP_VS_VPC_DST_REG_OUTLOC3(uint32_t val)
 #define REG_A5XX_SP_VS_OBJ_START_HI				0x0000e5ad
 
 #define REG_A5XX_SP_FS_CTRL_REG0				0x0000e5c0
+#define A5XX_SP_FS_CTRL_REG0_BUFFER				0x00000004
 #define A5XX_SP_FS_CTRL_REG0_THREADSIZE__MASK			0x00000008
 #define A5XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT			3
 static inline uint32_t A5XX_SP_FS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
@@ -4406,6 +4408,7 @@ static inline uint32_t A5XX_SP_FS_MRT_REG_COLOR_FORMAT(enum a5xx_color_fmt val)
 #define REG_A5XX_UNKNOWN_E5DB					0x0000e5db
 
 #define REG_A5XX_SP_CS_CTRL_REG0				0x0000e5f0
+#define A5XX_SP_CS_CTRL_REG0_BUFFER				0x00000004
 #define A5XX_SP_CS_CTRL_REG0_THREADSIZE__MASK			0x00000008
 #define A5XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT			3
 static inline uint32_t A5XX_SP_CS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
@@ -4440,6 +4443,7 @@ static inline uint32_t A5XX_SP_CS_CTRL_REG0_BRANCHSTACK(uint32_t val)
 #define REG_A5XX_SP_CS_OBJ_START_HI				0x0000e5f4
 
 #define REG_A5XX_SP_HS_CTRL_REG0				0x0000e600
+#define A5XX_SP_HS_CTRL_REG0_BUFFER				0x00000004
 #define A5XX_SP_HS_CTRL_REG0_THREADSIZE__MASK			0x00000008
 #define A5XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT			3
 static inline uint32_t A5XX_SP_HS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
@@ -4474,6 +4478,7 @@ static inline uint32_t A5XX_SP_HS_CTRL_REG0_BRANCHSTACK(uint32_t val)
 #define REG_A5XX_SP_HS_OBJ_START_HI				0x0000e604
 
 #define REG_A5XX_SP_DS_CTRL_REG0				0x0000e610
+#define A5XX_SP_DS_CTRL_REG0_BUFFER				0x00000004
 #define A5XX_SP_DS_CTRL_REG0_THREADSIZE__MASK			0x00000008
 #define A5XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT			3
 static inline uint32_t A5XX_SP_DS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
@@ -4508,6 +4513,7 @@ static inline uint32_t A5XX_SP_DS_CTRL_REG0_BRANCHSTACK(uint32_t val)
 #define REG_A5XX_SP_DS_OBJ_START_HI				0x0000e62d
 
 #define REG_A5XX_SP_GS_CTRL_REG0				0x0000e640
+#define A5XX_SP_GS_CTRL_REG0_BUFFER				0x00000004
 #define A5XX_SP_GS_CTRL_REG0_THREADSIZE__MASK			0x00000008
 #define A5XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT			3
 static inline uint32_t A5XX_SP_GS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
@@ -4665,11 +4671,11 @@ static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(uint32_t val)
 {
 	return ((val) << A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK;
 }
-#define A5XX_HLSQ_CONTROL_2_REG_SIZE__MASK			0xff000000
-#define A5XX_HLSQ_CONTROL_2_REG_SIZE__SHIFT			24
-static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_SIZE(uint32_t val)
+#define A5XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK			0xff000000
+#define A5XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT		24
+static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_CENTERRHW(uint32_t val)
 {
-	return ((val) << A5XX_HLSQ_CONTROL_2_REG_SIZE__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_SIZE__MASK;
+	return ((val) << A5XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK;
 }
 
 #define REG_A5XX_HLSQ_CONTROL_3_REG				0x0000e787
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
index a1e006ec5dce..1e8d2982d603 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
@@ -567,7 +567,7 @@ static void a5xx_ucode_check_version(struct a5xx_gpu *a5xx_gpu,
 	msm_gem_put_vaddr(obj);
 }
 
-static int a5xx_ucode_init(struct msm_gpu *gpu)
+static int a5xx_ucode_load(struct msm_gpu *gpu)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
@@ -605,9 +605,24 @@ static int a5xx_ucode_init(struct msm_gpu *gpu)
 		a5xx_ucode_check_version(a5xx_gpu, a5xx_gpu->pfp_bo);
 	}
 
-	gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, a5xx_gpu->pm4_iova);
+	if (a5xx_gpu->has_whereami) {
+		if (!a5xx_gpu->shadow_bo) {
+			a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
+				sizeof(u32) * gpu->nr_rings,
+				MSM_BO_WC | MSM_BO_MAP_PRIV,
+				gpu->aspace, &a5xx_gpu->shadow_bo,
+				&a5xx_gpu->shadow_iova);
 
-	gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, a5xx_gpu->pfp_iova);
+			if (IS_ERR(a5xx_gpu->shadow))
+				return PTR_ERR(a5xx_gpu->shadow);
+
+			msm_gem_object_set_name(a5xx_gpu->shadow_bo, "shadow");
+		}
+	} else if (gpu->nr_rings > 1) {
+		/* Disable preemption if WHERE_AM_I isn't available */
+		a5xx_preempt_fini(gpu);
+		gpu->nr_rings = 1;
+	}
 
 	return 0;
 }
@@ -900,9 +915,8 @@ static int a5xx_hw_init(struct msm_gpu *gpu)
 	if (adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))
 		a5xx_gpmu_ucode_init(gpu);
 
-	ret = a5xx_ucode_init(gpu);
-	if (ret)
-		return ret;
+	gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, a5xx_gpu->pm4_iova);
+	gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, a5xx_gpu->pfp_iova);
 
 	/* Set the ringbuffer address */
 	gpu_write64(gpu, REG_A5XX_CP_RB_BASE, gpu->rb[0]->iova);
@@ -916,27 +930,10 @@ static int a5xx_hw_init(struct msm_gpu *gpu)
 	gpu_write(gpu, REG_A5XX_CP_RB_CNTL,
 		MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
 
-	/* Create a privileged buffer for the RPTR shadow */
-	if (a5xx_gpu->has_whereami) {
-		if (!a5xx_gpu->shadow_bo) {
-			a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
-				sizeof(u32) * gpu->nr_rings,
-				MSM_BO_WC | MSM_BO_MAP_PRIV,
-				gpu->aspace, &a5xx_gpu->shadow_bo,
-				&a5xx_gpu->shadow_iova);
-
-			if (IS_ERR(a5xx_gpu->shadow))
-				return PTR_ERR(a5xx_gpu->shadow);
-
-			msm_gem_object_set_name(a5xx_gpu->shadow_bo, "shadow");
-		}
-
+	/* Configure the RPTR shadow if needed: */
+	if (a5xx_gpu->shadow_bo) {
 		gpu_write64(gpu, REG_A5XX_CP_RB_RPTR_ADDR,
 			    shadowptr(a5xx_gpu, gpu->rb[0]));
-	} else if (gpu->nr_rings > 1) {
-		/* Disable preemption if WHERE_AM_I isn't available */
-		a5xx_preempt_fini(gpu);
-		gpu->nr_rings = 1;
 	}
 
 	a5xx_preempt_hw_init(gpu);
@@ -1099,14 +1096,19 @@ bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
 {
 	struct msm_gpu *gpu = arg;
-	pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
-			iova, flags,
+	struct adreno_smmu_fault_info *info = data;
+	char block[12] = "unknown";
+	u32 scratch[] = {
 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),
 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),
 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),
-			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)));
+			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)),
+	};
 
-	return 0;
+	if (info)
+		snprintf(block, sizeof(block), "%x", info->fsynr1);
+
+	return adreno_fault_handler(gpu, iova, flags, info, block, scratch);
 }
 
 static void a5xx_cp_err_irq(struct msm_gpu *gpu)
@@ -1682,6 +1684,7 @@ static const struct adreno_gpu_funcs funcs = {
 		.get_param = adreno_get_param,
 		.set_param = adreno_set_param,
 		.hw_init = a5xx_hw_init,
+		.ucode_load = a5xx_ucode_load,
 		.pm_suspend = a5xx_pm_suspend,
 		.pm_resume = a5xx_pm_resume,
 		.recover = a5xx_recover,
@@ -1743,6 +1746,7 @@ struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
 	struct a5xx_gpu *a5xx_gpu = NULL;
 	struct adreno_gpu *adreno_gpu;
 	struct msm_gpu *gpu;
+	unsigned int nr_rings;
 	int ret;
 
 	if (!pdev) {
@@ -1763,7 +1767,12 @@ struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
 
 	check_speed_bin(&pdev->dev);
 
-	ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 4);
+	nr_rings = 4;
+
+	if (adreno_is_a510(adreno_gpu))
+		nr_rings = 1;
+
+	ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, nr_rings);
 	if (ret) {
 		a5xx_destroy(&(a5xx_gpu->base.base));
 		return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/msm/adreno/a6xx.xml.h b/drivers/gpu/drm/msm/adreno/a6xx.xml.h
index a92788019376..4dc3be6ed45d 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx.xml.h
@@ -8,21 +8,21 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a2xx.xml                (  90810 bytes, from 2021-06-21 15:24:24)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  14609 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  69086 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113358 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149512 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 184954 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11331 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   6038 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2924 bytes, from 2021-07-22 15:21:56)
-
-Copyright (C) 2013-2022 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a2xx.xml                (  91929 bytes, from 2023-02-28 23:52:27)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  15434 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  74995 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113474 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149590 bytes, from 2023-02-14 19:37:12)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 198949 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11404 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   9055 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2976 bytes, from 2023-03-10 18:32:52)
+
+Copyright (C) 2013-2023 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -911,6 +911,7 @@ enum a6xx_ztest_mode {
 	A6XX_EARLY_Z = 0,
 	A6XX_LATE_Z = 1,
 	A6XX_EARLY_LRZ_LATE_Z = 2,
+	A6XX_INVALID_ZTEST = 3,
 };
 
 enum a6xx_sequenced_thread_dist {
@@ -946,6 +947,12 @@ enum a6xx_buffers_location {
 	BUFFERS_IN_SYSMEM = 3,
 };
 
+enum a6xx_lrz_dir_status {
+	LRZ_DIR_LE = 1,
+	LRZ_DIR_GE = 2,
+	LRZ_DIR_INVALID = 3,
+};
+
 enum a6xx_fragcoord_sample_mode {
 	FRAGCOORD_CENTER = 0,
 	FRAGCOORD_SAMPLE = 3,
@@ -978,6 +985,11 @@ enum a6xx_threadsize {
 	THREAD128 = 1,
 };
 
+enum a6xx_bindless_descriptor_size {
+	BINDLESS_DESCRIPTOR_16B = 1,
+	BINDLESS_DESCRIPTOR_64B = 3,
+};
+
 enum a6xx_isam_mode {
 	ISAMMODE_GL = 2,
 };
@@ -1030,6 +1042,8 @@ enum a6xx_tex_type {
 
 #define A6XX_RBBM_INT_0_MASK_RBBM_GPU_IDLE			0x00000001
 #define A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR			0x00000002
+#define A6XX_RBBM_INT_0_MASK_CP_IPC_INTR_0			0x00000010
+#define A6XX_RBBM_INT_0_MASK_CP_IPC_INTR_1			0x00000020
 #define A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW	0x00000040
 #define A6XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR			0x00000080
 #define A6XX_RBBM_INT_0_MASK_CP_SW				0x00000100
@@ -1040,15 +1054,19 @@ enum a6xx_tex_type {
 #define A6XX_RBBM_INT_0_MASK_CP_IB2				0x00002000
 #define A6XX_RBBM_INT_0_MASK_CP_IB1				0x00004000
 #define A6XX_RBBM_INT_0_MASK_CP_RB				0x00008000
+#define A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPT			0x00008000
+#define A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPTLPAC			0x00010000
 #define A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS			0x00020000
 #define A6XX_RBBM_INT_0_MASK_CP_WT_DONE_TS			0x00040000
 #define A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS			0x00100000
+#define A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS_LPAC		0x00200000
 #define A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW		0x00400000
 #define A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT			0x00800000
 #define A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS			0x01000000
 #define A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR			0x02000000
 #define A6XX_RBBM_INT_0_MASK_DEBBUS_INTR_0			0x04000000
 #define A6XX_RBBM_INT_0_MASK_DEBBUS_INTR_1			0x08000000
+#define A6XX_RBBM_INT_0_MASK_TSBWRITEERROR			0x10000000
 #define A6XX_RBBM_INT_0_MASK_ISDB_CPU_IRQ			0x40000000
 #define A6XX_RBBM_INT_0_MASK_ISDB_UNDER_DEBUG			0x80000000
 #define A6XX_CP_INT_CP_OPCODE_ERROR				0x00000001
@@ -1058,15 +1076,21 @@ enum a6xx_tex_type {
 #define A6XX_CP_INT_CP_AHB_ERROR				0x00000020
 #define A6XX_CP_INT_CP_VSD_PARITY_ERROR				0x00000040
 #define A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR			0x00000080
+#define A6XX_CP_INT_CP_OPCODE_ERROR_LPAC			0x00000100
+#define A6XX_CP_INT_CP_UCODE_ERROR_LPAC				0x00000200
+#define A6XX_CP_INT_CP_HW_FAULT_ERROR_LPAC			0x00000400
+#define A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR_LPAC		0x00000800
+#define A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR_LPAC			0x00001000
+#define A6XX_CP_INT_CP_OPCODE_ERROR_BV				0x00002000
+#define A6XX_CP_INT_CP_UCODE_ERROR_BV				0x00004000
+#define A6XX_CP_INT_CP_HW_FAULT_ERROR_BV			0x00008000
+#define A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR_BV		0x00010000
+#define A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR_BV			0x00020000
 #define REG_A6XX_CP_RB_BASE					0x00000800
 
-#define REG_A6XX_CP_RB_BASE_HI					0x00000801
-
 #define REG_A6XX_CP_RB_CNTL					0x00000802
 
-#define REG_A6XX_CP_RB_RPTR_ADDR_LO				0x00000804
-
-#define REG_A6XX_CP_RB_RPTR_ADDR_HI				0x00000805
+#define REG_A6XX_CP_RB_RPTR_ADDR				0x00000804
 
 #define REG_A6XX_CP_RB_RPTR					0x00000806
 
@@ -1083,26 +1107,28 @@ enum a6xx_tex_type {
 
 #define REG_A6XX_CP_PROTECT_STATUS				0x00000824
 
+#define REG_A6XX_CP_STATUS_1					0x00000825
+
 #define REG_A6XX_CP_SQE_INSTR_BASE				0x00000830
 
 #define REG_A6XX_CP_MISC_CNTL					0x00000840
 
-#define REG_A6XX_CP_CHICKEN_DBG					0x00000841
-
 #define REG_A6XX_CP_APRIV_CNTL					0x00000844
 
+#define REG_A6XX_CP_PREEMPT_THRESHOLD				0x000008c0
+
 #define REG_A6XX_CP_ROQ_THRESHOLDS_1				0x000008c1
-#define A6XX_CP_ROQ_THRESHOLDS_1_RB_LO__MASK			0x000000ff
-#define A6XX_CP_ROQ_THRESHOLDS_1_RB_LO__SHIFT			0
-static inline uint32_t A6XX_CP_ROQ_THRESHOLDS_1_RB_LO(uint32_t val)
+#define A6XX_CP_ROQ_THRESHOLDS_1_MRB_START__MASK		0x000000ff
+#define A6XX_CP_ROQ_THRESHOLDS_1_MRB_START__SHIFT		0
+static inline uint32_t A6XX_CP_ROQ_THRESHOLDS_1_MRB_START(uint32_t val)
 {
-	return ((val >> 2) << A6XX_CP_ROQ_THRESHOLDS_1_RB_LO__SHIFT) & A6XX_CP_ROQ_THRESHOLDS_1_RB_LO__MASK;
+	return ((val >> 2) << A6XX_CP_ROQ_THRESHOLDS_1_MRB_START__SHIFT) & A6XX_CP_ROQ_THRESHOLDS_1_MRB_START__MASK;
 }
-#define A6XX_CP_ROQ_THRESHOLDS_1_RB_HI__MASK			0x0000ff00
-#define A6XX_CP_ROQ_THRESHOLDS_1_RB_HI__SHIFT			8
-static inline uint32_t A6XX_CP_ROQ_THRESHOLDS_1_RB_HI(uint32_t val)
+#define A6XX_CP_ROQ_THRESHOLDS_1_VSD_START__MASK		0x0000ff00
+#define A6XX_CP_ROQ_THRESHOLDS_1_VSD_START__SHIFT		8
+static inline uint32_t A6XX_CP_ROQ_THRESHOLDS_1_VSD_START(uint32_t val)
 {
-	return ((val >> 2) << A6XX_CP_ROQ_THRESHOLDS_1_RB_HI__SHIFT) & A6XX_CP_ROQ_THRESHOLDS_1_RB_HI__MASK;
+	return ((val >> 2) << A6XX_CP_ROQ_THRESHOLDS_1_VSD_START__SHIFT) & A6XX_CP_ROQ_THRESHOLDS_1_VSD_START__MASK;
 }
 #define A6XX_CP_ROQ_THRESHOLDS_1_IB1_START__MASK		0x00ff0000
 #define A6XX_CP_ROQ_THRESHOLDS_1_IB1_START__SHIFT		16
@@ -1164,27 +1190,21 @@ static inline uint32_t A6XX_CP_PROTECT_REG_MASK_LEN(uint32_t val)
 
 #define REG_A6XX_CP_CONTEXT_SWITCH_CNTL				0x000008a0
 
-#define REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO			0x000008a1
-
-#define REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI			0x000008a2
+#define REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO			0x000008a1
 
-#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO	0x000008a3
+#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR	0x000008a3
 
-#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI	0x000008a4
+#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR	0x000008a5
 
-#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO	0x000008a5
+#define REG_A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR	0x000008a7
 
-#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI	0x000008a6
-
-#define REG_A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO	0x000008a7
-
-#define REG_A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI	0x000008a8
+#define REG_A7XX_CP_CONTEXT_SWITCH_LEVEL_STATUS			0x000008ab
 
 static inline uint32_t REG_A6XX_CP_PERFCTR_CP_SEL(uint32_t i0) { return 0x000008d0 + 0x1*i0; }
 
-#define REG_A6XX_CP_CRASH_SCRIPT_BASE_LO			0x00000900
+static inline uint32_t REG_A7XX_CP_BV_PERFCTR_CP_SEL(uint32_t i0) { return 0x000008e0 + 0x1*i0; }
 
-#define REG_A6XX_CP_CRASH_SCRIPT_BASE_HI			0x00000901
+#define REG_A6XX_CP_CRASH_SCRIPT_BASE				0x00000900
 
 #define REG_A6XX_CP_CRASH_DUMP_CNTL				0x00000902
 
@@ -1212,63 +1232,165 @@ static inline uint32_t REG_A6XX_CP_PERFCTR_CP_SEL(uint32_t i0) { return 0x000008
 
 #define REG_A6XX_CP_IB1_BASE					0x00000928
 
-#define REG_A6XX_CP_IB1_BASE_HI					0x00000929
-
 #define REG_A6XX_CP_IB1_REM_SIZE				0x0000092a
 
 #define REG_A6XX_CP_IB2_BASE					0x0000092b
 
-#define REG_A6XX_CP_IB2_BASE_HI					0x0000092c
-
 #define REG_A6XX_CP_IB2_REM_SIZE				0x0000092d
 
 #define REG_A6XX_CP_SDS_BASE					0x0000092e
 
-#define REG_A6XX_CP_SDS_BASE_HI					0x0000092f
-
 #define REG_A6XX_CP_SDS_REM_SIZE				0x00000930
 
 #define REG_A6XX_CP_MRB_BASE					0x00000931
 
-#define REG_A6XX_CP_MRB_BASE_HI					0x00000932
-
 #define REG_A6XX_CP_MRB_REM_SIZE				0x00000933
 
 #define REG_A6XX_CP_VSD_BASE					0x00000934
 
-#define REG_A6XX_CP_VSD_BASE_HI					0x00000935
+#define REG_A6XX_CP_ROQ_RB_STAT					0x00000939
+#define A6XX_CP_ROQ_RB_STAT_RPTR__MASK				0x000003ff
+#define A6XX_CP_ROQ_RB_STAT_RPTR__SHIFT				0
+static inline uint32_t A6XX_CP_ROQ_RB_STAT_RPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_RB_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_RB_STAT_RPTR__MASK;
+}
+#define A6XX_CP_ROQ_RB_STAT_WPTR__MASK				0x03ff0000
+#define A6XX_CP_ROQ_RB_STAT_WPTR__SHIFT				16
+static inline uint32_t A6XX_CP_ROQ_RB_STAT_WPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_RB_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_RB_STAT_WPTR__MASK;
+}
+
+#define REG_A6XX_CP_ROQ_IB1_STAT				0x0000093a
+#define A6XX_CP_ROQ_IB1_STAT_RPTR__MASK				0x000003ff
+#define A6XX_CP_ROQ_IB1_STAT_RPTR__SHIFT			0
+static inline uint32_t A6XX_CP_ROQ_IB1_STAT_RPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_IB1_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_IB1_STAT_RPTR__MASK;
+}
+#define A6XX_CP_ROQ_IB1_STAT_WPTR__MASK				0x03ff0000
+#define A6XX_CP_ROQ_IB1_STAT_WPTR__SHIFT			16
+static inline uint32_t A6XX_CP_ROQ_IB1_STAT_WPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_IB1_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_IB1_STAT_WPTR__MASK;
+}
+
+#define REG_A6XX_CP_ROQ_IB2_STAT				0x0000093b
+#define A6XX_CP_ROQ_IB2_STAT_RPTR__MASK				0x000003ff
+#define A6XX_CP_ROQ_IB2_STAT_RPTR__SHIFT			0
+static inline uint32_t A6XX_CP_ROQ_IB2_STAT_RPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_IB2_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_IB2_STAT_RPTR__MASK;
+}
+#define A6XX_CP_ROQ_IB2_STAT_WPTR__MASK				0x03ff0000
+#define A6XX_CP_ROQ_IB2_STAT_WPTR__SHIFT			16
+static inline uint32_t A6XX_CP_ROQ_IB2_STAT_WPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_IB2_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_IB2_STAT_WPTR__MASK;
+}
+
+#define REG_A6XX_CP_ROQ_SDS_STAT				0x0000093c
+#define A6XX_CP_ROQ_SDS_STAT_RPTR__MASK				0x000003ff
+#define A6XX_CP_ROQ_SDS_STAT_RPTR__SHIFT			0
+static inline uint32_t A6XX_CP_ROQ_SDS_STAT_RPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_SDS_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_SDS_STAT_RPTR__MASK;
+}
+#define A6XX_CP_ROQ_SDS_STAT_WPTR__MASK				0x03ff0000
+#define A6XX_CP_ROQ_SDS_STAT_WPTR__SHIFT			16
+static inline uint32_t A6XX_CP_ROQ_SDS_STAT_WPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_SDS_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_SDS_STAT_WPTR__MASK;
+}
+
+#define REG_A6XX_CP_ROQ_MRB_STAT				0x0000093d
+#define A6XX_CP_ROQ_MRB_STAT_RPTR__MASK				0x000003ff
+#define A6XX_CP_ROQ_MRB_STAT_RPTR__SHIFT			0
+static inline uint32_t A6XX_CP_ROQ_MRB_STAT_RPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_MRB_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_MRB_STAT_RPTR__MASK;
+}
+#define A6XX_CP_ROQ_MRB_STAT_WPTR__MASK				0x03ff0000
+#define A6XX_CP_ROQ_MRB_STAT_WPTR__SHIFT			16
+static inline uint32_t A6XX_CP_ROQ_MRB_STAT_WPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_MRB_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_MRB_STAT_WPTR__MASK;
+}
+
+#define REG_A6XX_CP_ROQ_VSD_STAT				0x0000093e
+#define A6XX_CP_ROQ_VSD_STAT_RPTR__MASK				0x000003ff
+#define A6XX_CP_ROQ_VSD_STAT_RPTR__SHIFT			0
+static inline uint32_t A6XX_CP_ROQ_VSD_STAT_RPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_VSD_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_VSD_STAT_RPTR__MASK;
+}
+#define A6XX_CP_ROQ_VSD_STAT_WPTR__MASK				0x03ff0000
+#define A6XX_CP_ROQ_VSD_STAT_WPTR__SHIFT			16
+static inline uint32_t A6XX_CP_ROQ_VSD_STAT_WPTR(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_VSD_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_VSD_STAT_WPTR__MASK;
+}
+
+#define REG_A6XX_CP_IB1_DWORDS					0x00000943
+
+#define REG_A6XX_CP_IB2_DWORDS					0x00000944
+
+#define REG_A6XX_CP_SDS_DWORDS					0x00000945
 
 #define REG_A6XX_CP_MRB_DWORDS					0x00000946
 
 #define REG_A6XX_CP_VSD_DWORDS					0x00000947
 
-#define REG_A6XX_CP_CSQ_IB1_STAT				0x00000949
-#define A6XX_CP_CSQ_IB1_STAT_REM__MASK				0xffff0000
-#define A6XX_CP_CSQ_IB1_STAT_REM__SHIFT				16
-static inline uint32_t A6XX_CP_CSQ_IB1_STAT_REM(uint32_t val)
+#define REG_A6XX_CP_ROQ_AVAIL_RB				0x00000948
+#define A6XX_CP_ROQ_AVAIL_RB_REM__MASK				0xffff0000
+#define A6XX_CP_ROQ_AVAIL_RB_REM__SHIFT				16
+static inline uint32_t A6XX_CP_ROQ_AVAIL_RB_REM(uint32_t val)
 {
-	return ((val) << A6XX_CP_CSQ_IB1_STAT_REM__SHIFT) & A6XX_CP_CSQ_IB1_STAT_REM__MASK;
+	return ((val) << A6XX_CP_ROQ_AVAIL_RB_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_RB_REM__MASK;
 }
 
-#define REG_A6XX_CP_CSQ_IB2_STAT				0x0000094a
-#define A6XX_CP_CSQ_IB2_STAT_REM__MASK				0xffff0000
-#define A6XX_CP_CSQ_IB2_STAT_REM__SHIFT				16
-static inline uint32_t A6XX_CP_CSQ_IB2_STAT_REM(uint32_t val)
+#define REG_A6XX_CP_ROQ_AVAIL_IB1				0x00000949
+#define A6XX_CP_ROQ_AVAIL_IB1_REM__MASK				0xffff0000
+#define A6XX_CP_ROQ_AVAIL_IB1_REM__SHIFT			16
+static inline uint32_t A6XX_CP_ROQ_AVAIL_IB1_REM(uint32_t val)
 {
-	return ((val) << A6XX_CP_CSQ_IB2_STAT_REM__SHIFT) & A6XX_CP_CSQ_IB2_STAT_REM__MASK;
+	return ((val) << A6XX_CP_ROQ_AVAIL_IB1_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_IB1_REM__MASK;
 }
 
-#define REG_A6XX_CP_MRQ_MRB_STAT				0x0000094c
-#define A6XX_CP_MRQ_MRB_STAT_REM__MASK				0xffff0000
-#define A6XX_CP_MRQ_MRB_STAT_REM__SHIFT				16
-static inline uint32_t A6XX_CP_MRQ_MRB_STAT_REM(uint32_t val)
+#define REG_A6XX_CP_ROQ_AVAIL_IB2				0x0000094a
+#define A6XX_CP_ROQ_AVAIL_IB2_REM__MASK				0xffff0000
+#define A6XX_CP_ROQ_AVAIL_IB2_REM__SHIFT			16
+static inline uint32_t A6XX_CP_ROQ_AVAIL_IB2_REM(uint32_t val)
 {
-	return ((val) << A6XX_CP_MRQ_MRB_STAT_REM__SHIFT) & A6XX_CP_MRQ_MRB_STAT_REM__MASK;
+	return ((val) << A6XX_CP_ROQ_AVAIL_IB2_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_IB2_REM__MASK;
 }
 
-#define REG_A6XX_CP_ALWAYS_ON_COUNTER_LO			0x00000980
+#define REG_A6XX_CP_ROQ_AVAIL_SDS				0x0000094b
+#define A6XX_CP_ROQ_AVAIL_SDS_REM__MASK				0xffff0000
+#define A6XX_CP_ROQ_AVAIL_SDS_REM__SHIFT			16
+static inline uint32_t A6XX_CP_ROQ_AVAIL_SDS_REM(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_AVAIL_SDS_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_SDS_REM__MASK;
+}
 
-#define REG_A6XX_CP_ALWAYS_ON_COUNTER_HI			0x00000981
+#define REG_A6XX_CP_ROQ_AVAIL_MRB				0x0000094c
+#define A6XX_CP_ROQ_AVAIL_MRB_REM__MASK				0xffff0000
+#define A6XX_CP_ROQ_AVAIL_MRB_REM__SHIFT			16
+static inline uint32_t A6XX_CP_ROQ_AVAIL_MRB_REM(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_AVAIL_MRB_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_MRB_REM__MASK;
+}
+
+#define REG_A6XX_CP_ROQ_AVAIL_VSD				0x0000094d
+#define A6XX_CP_ROQ_AVAIL_VSD_REM__MASK				0xffff0000
+#define A6XX_CP_ROQ_AVAIL_VSD_REM__SHIFT			16
+static inline uint32_t A6XX_CP_ROQ_AVAIL_VSD_REM(uint32_t val)
+{
+	return ((val) << A6XX_CP_ROQ_AVAIL_VSD_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_VSD_REM__MASK;
+}
+
+#define REG_A6XX_CP_ALWAYS_ON_COUNTER				0x00000980
 
 #define REG_A6XX_CP_AHB_CNTL					0x0000098d
 
@@ -1276,12 +1398,70 @@ static inline uint32_t A6XX_CP_MRQ_MRB_STAT_REM(uint32_t val)
 
 #define REG_A6XX_CP_APERTURE_CNTL_CD				0x00000a03
 
+#define REG_A7XX_CP_BV_PROTECT_STATUS				0x00000a61
+
+#define REG_A7XX_CP_BV_HW_FAULT					0x00000a64
+
+#define REG_A7XX_CP_BV_DRAW_STATE_ADDR				0x00000a81
+
+#define REG_A7XX_CP_BV_DRAW_STATE_DATA				0x00000a82
+
+#define REG_A7XX_CP_BV_ROQ_DBG_ADDR				0x00000a83
+
+#define REG_A7XX_CP_BV_ROQ_DBG_DATA				0x00000a84
+
+#define REG_A7XX_CP_BV_SQE_UCODE_DBG_ADDR			0x00000a85
+
+#define REG_A7XX_CP_BV_SQE_UCODE_DBG_DATA			0x00000a86
+
+#define REG_A7XX_CP_BV_SQE_STAT_ADDR				0x00000a87
+
+#define REG_A7XX_CP_BV_SQE_STAT_DATA				0x00000a88
+
+#define REG_A7XX_CP_BV_MEM_POOL_DBG_ADDR			0x00000a96
+
+#define REG_A7XX_CP_BV_MEM_POOL_DBG_DATA			0x00000a97
+
+#define REG_A7XX_CP_BV_RB_RPTR_ADDR				0x00000a98
+
+#define REG_A7XX_CP_RESOURCE_TBL_DBG_ADDR			0x00000a9a
+
+#define REG_A7XX_CP_RESOURCE_TBL_DBG_DATA			0x00000a9b
+
+#define REG_A7XX_CP_BV_APRIV_CNTL				0x00000ad0
+
+#define REG_A7XX_CP_BV_CHICKEN_DBG				0x00000ada
+
+#define REG_A7XX_CP_LPAC_DRAW_STATE_ADDR			0x00000b0a
+
+#define REG_A7XX_CP_LPAC_DRAW_STATE_DATA			0x00000b0b
+
+#define REG_A7XX_CP_LPAC_ROQ_DBG_ADDR				0x00000b0c
+
+#define REG_A7XX_CP_SQE_AC_UCODE_DBG_ADDR			0x00000b27
+
+#define REG_A7XX_CP_SQE_AC_UCODE_DBG_DATA			0x00000b28
+
+#define REG_A7XX_CP_SQE_AC_STAT_ADDR				0x00000b29
+
+#define REG_A7XX_CP_SQE_AC_STAT_DATA				0x00000b2a
+
+#define REG_A7XX_CP_LPAC_APRIV_CNTL				0x00000b31
+
 #define REG_A6XX_CP_LPAC_PROG_FIFO_SIZE				0x00000b34
 
+#define REG_A7XX_CP_LPAC_ROQ_DBG_DATA				0x00000b35
+
+#define REG_A7XX_CP_LPAC_FIFO_DBG_DATA				0x00000b36
+
+#define REG_A7XX_CP_LPAC_FIFO_DBG_ADDR				0x00000b40
+
 #define REG_A6XX_CP_LPAC_SQE_INSTR_BASE				0x00000b82
 
 #define REG_A6XX_VSC_ADDR_MODE_CNTL				0x00000c01
 
+#define REG_A6XX_RBBM_GPR0_CNTL					0x00000018
+
 #define REG_A6XX_RBBM_INT_0_STATUS				0x00000201
 
 #define REG_A6XX_RBBM_STATUS					0x00000210
@@ -1310,11 +1490,27 @@ static inline uint32_t A6XX_CP_MRQ_MRB_STAT_REM(uint32_t val)
 #define A6XX_RBBM_STATUS_CP_AHB_BUSY_CP_MASTER			0x00000002
 #define A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER			0x00000001
 
+#define REG_A6XX_RBBM_STATUS1					0x00000211
+
+#define REG_A6XX_RBBM_STATUS2					0x00000212
+
 #define REG_A6XX_RBBM_STATUS3					0x00000213
 #define A6XX_RBBM_STATUS3_SMMU_STALLED_ON_FAULT			0x01000000
 
 #define REG_A6XX_RBBM_VBIF_GX_RESET_STATUS			0x00000215
 
+#define REG_A7XX_RBBM_CLOCK_MODE_CP				0x00000260
+
+#define REG_A7XX_RBBM_CLOCK_MODE_BV_LRZ				0x00000284
+
+#define REG_A7XX_RBBM_CLOCK_MODE_BV_GRAS			0x00000285
+
+#define REG_A7XX_RBBM_CLOCK_MODE2_GRAS				0x00000286
+
+#define REG_A7XX_RBBM_CLOCK_MODE_BV_VFD				0x00000287
+
+#define REG_A7XX_RBBM_CLOCK_MODE_BV_GPC				0x00000288
+
 static inline uint32_t REG_A6XX_RBBM_PERFCTR_CP(uint32_t i0) { return 0x00000400 + 0x2*i0; }
 
 static inline uint32_t REG_A6XX_RBBM_PERFCTR_RBBM(uint32_t i0) { return 0x0000041c + 0x2*i0; }
@@ -1347,6 +1543,62 @@ static inline uint32_t REG_A6XX_RBBM_PERFCTR_LRZ(uint32_t i0) { return 0x000004e
 
 static inline uint32_t REG_A6XX_RBBM_PERFCTR_CMP(uint32_t i0) { return 0x000004f2 + 0x2*i0; }
 
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_CP(uint32_t i0) { return 0x00000300 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_RBBM(uint32_t i0) { return 0x0000031c + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_PC(uint32_t i0) { return 0x00000324 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_VFD(uint32_t i0) { return 0x00000334 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_HLSQ(uint32_t i0) { return 0x00000344 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_VPC(uint32_t i0) { return 0x00000350 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_CCU(uint32_t i0) { return 0x0000035c + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_TSE(uint32_t i0) { return 0x00000366 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_RAS(uint32_t i0) { return 0x0000036e + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_UCHE(uint32_t i0) { return 0x00000376 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_TP(uint32_t i0) { return 0x0000038e + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_SP(uint32_t i0) { return 0x000003a6 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_RB(uint32_t i0) { return 0x000003d6 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_VSC(uint32_t i0) { return 0x000003e6 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_LRZ(uint32_t i0) { return 0x000003ea + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_CMP(uint32_t i0) { return 0x000003f2 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_UFC(uint32_t i0) { return 0x000003fa + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR2_HLSQ(uint32_t i0) { return 0x00000410 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR2_CP(uint32_t i0) { return 0x0000041c + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR2_SP(uint32_t i0) { return 0x0000042a + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR2_TP(uint32_t i0) { return 0x00000442 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR2_UFC(uint32_t i0) { return 0x0000044e + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_BV_PC(uint32_t i0) { return 0x00000460 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_BV_VFD(uint32_t i0) { return 0x00000470 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_BV_VPC(uint32_t i0) { return 0x00000480 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_BV_TSE(uint32_t i0) { return 0x0000048c + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_BV_RAS(uint32_t i0) { return 0x00000494 + 0x2*i0; }
+
+static inline uint32_t REG_A7XX_RBBM_PERFCTR_BV_LRZ(uint32_t i0) { return 0x0000049c + 0x2*i0; }
+
 #define REG_A6XX_RBBM_PERFCTR_CNTL				0x00000500
 
 #define REG_A6XX_RBBM_PERFCTR_LOAD_CMD0				0x00000501
@@ -1371,6 +1623,10 @@ static inline uint32_t REG_A6XX_RBBM_PERFCTR_RBBM_SEL(uint32_t i0) { return 0x00
 
 #define REG_A6XX_RBBM_ISDB_CNT					0x00000533
 
+#define REG_A7XX_RBBM_NC_MODE_CNTL				0x00000534
+
+#define REG_A7XX_RBBM_SNAPSHOT_STATUS				0x00000535
+
 #define REG_A6XX_RBBM_PRIMCTR_0_LO				0x00000540
 
 #define REG_A6XX_RBBM_PRIMCTR_0_HI				0x00000541
@@ -1417,9 +1673,7 @@ static inline uint32_t REG_A6XX_RBBM_PERFCTR_RBBM_SEL(uint32_t i0) { return 0x00
 
 #define REG_A6XX_RBBM_SECVID_TRUST_CNTL				0x0000f400
 
-#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO		0x0000f800
-
-#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI		0x0000f801
+#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE			0x0000f800
 
 #define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE			0x0000f802
 
@@ -1427,6 +1681,8 @@ static inline uint32_t REG_A6XX_RBBM_PERFCTR_RBBM_SEL(uint32_t i0) { return 0x00
 
 #define REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL			0x0000f810
 
+#define REG_A7XX_RBBM_SECVID_TSB_STATUS				0x0000fc00
+
 #define REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL			0x00000010
 
 #define REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL			0x00000011
@@ -1438,12 +1694,18 @@ static inline uint32_t REG_A6XX_RBBM_PERFCTR_RBBM_SEL(uint32_t i0) { return 0x00
 #define REG_A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD			0x0000001c
 #define A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD_WAIT_GPU_IDLE		0x00000001
 
+#define REG_A7XX_RBBM_GBIF_HALT					0x00000016
+
+#define REG_A7XX_RBBM_GBIF_HALT_ACK				0x00000017
+
 #define REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL			0x0000001f
 
 #define REG_A6XX_RBBM_INT_CLEAR_CMD				0x00000037
 
 #define REG_A6XX_RBBM_INT_0_MASK				0x00000038
 
+#define REG_A7XX_RBBM_INT_2_MASK				0x0000003a
+
 #define REG_A6XX_RBBM_SP_HYST_CNT				0x00000042
 
 #define REG_A6XX_RBBM_SW_RESET_CMD				0x00000043
@@ -1674,6 +1936,8 @@ static inline uint32_t REG_A6XX_RBBM_PERFCTR_RBBM_SEL(uint32_t i0) { return 0x00
 
 #define REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE			0x00000122
 
+#define REG_A6XX_RBBM_LPAC_GBIF_CLIENT_QOS_CNTL			0x000005ff
+
 #define REG_A6XX_DBGC_CFG_DBGBUS_SEL_A				0x00000600
 
 #define REG_A6XX_DBGC_CFG_DBGBUS_SEL_B				0x00000601
@@ -1852,25 +2116,15 @@ static inline uint32_t REG_A6XX_VSC_PERFCTR_VSC_SEL(uint32_t i0) { return 0x0000
 
 #define REG_A6XX_UCHE_MODE_CNTL					0x00000e01
 
-#define REG_A6XX_UCHE_WRITE_RANGE_MAX_LO			0x00000e05
-
-#define REG_A6XX_UCHE_WRITE_RANGE_MAX_HI			0x00000e06
-
-#define REG_A6XX_UCHE_WRITE_THRU_BASE_LO			0x00000e07
+#define REG_A6XX_UCHE_WRITE_RANGE_MAX				0x00000e05
 
-#define REG_A6XX_UCHE_WRITE_THRU_BASE_HI			0x00000e08
+#define REG_A6XX_UCHE_WRITE_THRU_BASE				0x00000e07
 
-#define REG_A6XX_UCHE_TRAP_BASE_LO				0x00000e09
+#define REG_A6XX_UCHE_TRAP_BASE					0x00000e09
 
-#define REG_A6XX_UCHE_TRAP_BASE_HI				0x00000e0a
+#define REG_A6XX_UCHE_GMEM_RANGE_MIN				0x00000e0b
 
-#define REG_A6XX_UCHE_GMEM_RANGE_MIN_LO				0x00000e0b
-
-#define REG_A6XX_UCHE_GMEM_RANGE_MIN_HI				0x00000e0c
-
-#define REG_A6XX_UCHE_GMEM_RANGE_MAX_LO				0x00000e0d
-
-#define REG_A6XX_UCHE_GMEM_RANGE_MAX_HI				0x00000e0e
+#define REG_A6XX_UCHE_GMEM_RANGE_MAX				0x00000e0d
 
 #define REG_A6XX_UCHE_CACHE_WAYS				0x00000e17
 
@@ -1886,6 +2140,8 @@ static inline uint32_t A6XX_UCHE_CLIENT_PF_PERFSEL(uint32_t val)
 
 static inline uint32_t REG_A6XX_UCHE_PERFCTR_UCHE_SEL(uint32_t i0) { return 0x00000e1c + 0x1*i0; }
 
+#define REG_A6XX_UCHE_GBIF_GX_CONFIG				0x00000e3a
+
 #define REG_A6XX_UCHE_CMDQ_CONFIG				0x00000e3c
 
 #define REG_A6XX_VBIF_VERSION					0x00003000
@@ -1983,6 +2239,8 @@ static inline uint32_t A6XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL(uint32_t val)
 
 #define REG_A6XX_GBIF_PERF_PWR_CNT_EN				0x00003cc0
 
+#define REG_A6XX_GBIF_PERF_PWR_CNT_CLR				0x00003cc1
+
 #define REG_A6XX_GBIF_PERF_CNT_SEL				0x00003cc2
 
 #define REG_A6XX_GBIF_PERF_PWR_CNT_SEL				0x00003cc3
@@ -2105,7 +2363,7 @@ static inline uint32_t REG_A6XX_VSC_DRAW_STRM_SIZE_REG(uint32_t i0) { return 0x0
 #define A6XX_GRAS_CL_CNTL_CLIP_DISABLE				0x00000001
 #define A6XX_GRAS_CL_CNTL_ZNEAR_CLIP_DISABLE			0x00000002
 #define A6XX_GRAS_CL_CNTL_ZFAR_CLIP_DISABLE			0x00000004
-#define A6XX_GRAS_CL_CNTL_UNK5					0x00000020
+#define A6XX_GRAS_CL_CNTL_Z_CLAMP_ENABLE			0x00000020
 #define A6XX_GRAS_CL_CNTL_ZERO_GB_SCALE_Z			0x00000040
 #define A6XX_GRAS_CL_CNTL_VP_CLIP_CODE_IGNORE			0x00000080
 #define A6XX_GRAS_CL_CNTL_VP_XFORM_DISABLE			0x00000100
@@ -2420,11 +2678,12 @@ static inline uint32_t A6XX_GRAS_SC_CNTL_SEQUENCED_THREAD_DISTRIBUTION(enum a6xx
 {
 	return ((val) << A6XX_GRAS_SC_CNTL_SEQUENCED_THREAD_DISTRIBUTION__SHIFT) & A6XX_GRAS_SC_CNTL_SEQUENCED_THREAD_DISTRIBUTION__MASK;
 }
-#define A6XX_GRAS_SC_CNTL_UNK9__MASK				0x00000e00
-#define A6XX_GRAS_SC_CNTL_UNK9__SHIFT				9
-static inline uint32_t A6XX_GRAS_SC_CNTL_UNK9(uint32_t val)
+#define A6XX_GRAS_SC_CNTL_UNK9					0x00000200
+#define A6XX_GRAS_SC_CNTL_ROTATION__MASK			0x00000c00
+#define A6XX_GRAS_SC_CNTL_ROTATION__SHIFT			10
+static inline uint32_t A6XX_GRAS_SC_CNTL_ROTATION(uint32_t val)
 {
-	return ((val) << A6XX_GRAS_SC_CNTL_UNK9__SHIFT) & A6XX_GRAS_SC_CNTL_UNK9__MASK;
+	return ((val) << A6XX_GRAS_SC_CNTL_ROTATION__SHIFT) & A6XX_GRAS_SC_CNTL_ROTATION__MASK;
 }
 #define A6XX_GRAS_SC_CNTL_EARLYVIZOUTEN				0x00001000
 
@@ -2697,12 +2956,14 @@ static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y(uint32_t val)
 #define A6XX_GRAS_LRZ_CNTL_FC_ENABLE				0x00000008
 #define A6XX_GRAS_LRZ_CNTL_Z_TEST_ENABLE			0x00000010
 #define A6XX_GRAS_LRZ_CNTL_Z_BOUNDS_ENABLE			0x00000020
-#define A6XX_GRAS_LRZ_CNTL_UNK6__MASK				0x000003c0
-#define A6XX_GRAS_LRZ_CNTL_UNK6__SHIFT				6
-static inline uint32_t A6XX_GRAS_LRZ_CNTL_UNK6(uint32_t val)
+#define A6XX_GRAS_LRZ_CNTL_DIR__MASK				0x000000c0
+#define A6XX_GRAS_LRZ_CNTL_DIR__SHIFT				6
+static inline uint32_t A6XX_GRAS_LRZ_CNTL_DIR(enum a6xx_lrz_dir_status val)
 {
-	return ((val) << A6XX_GRAS_LRZ_CNTL_UNK6__SHIFT) & A6XX_GRAS_LRZ_CNTL_UNK6__MASK;
+	return ((val) << A6XX_GRAS_LRZ_CNTL_DIR__SHIFT) & A6XX_GRAS_LRZ_CNTL_DIR__MASK;
 }
+#define A6XX_GRAS_LRZ_CNTL_DIR_WRITE				0x00000100
+#define A6XX_GRAS_LRZ_CNTL_DISABLE_ON_WRONG_DIR			0x00000200
 
 #define REG_A6XX_GRAS_LRZ_PS_INPUT_CNTL				0x00008101
 #define A6XX_GRAS_LRZ_PS_INPUT_CNTL_SAMPLEID			0x00000001
@@ -2754,24 +3015,24 @@ static inline uint32_t A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(uint32_t val)
 #define REG_A6XX_GRAS_SAMPLE_CNTL				0x00008109
 #define A6XX_GRAS_SAMPLE_CNTL_PER_SAMP_MODE			0x00000001
 
-#define REG_A6XX_GRAS_UNKNOWN_810A				0x0000810a
-#define A6XX_GRAS_UNKNOWN_810A_UNK0__MASK			0x000007ff
-#define A6XX_GRAS_UNKNOWN_810A_UNK0__SHIFT			0
-static inline uint32_t A6XX_GRAS_UNKNOWN_810A_UNK0(uint32_t val)
+#define REG_A6XX_GRAS_LRZ_DEPTH_VIEW				0x0000810a
+#define A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_LAYER__MASK		0x000007ff
+#define A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_LAYER__SHIFT		0
+static inline uint32_t A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_LAYER(uint32_t val)
 {
-	return ((val) << A6XX_GRAS_UNKNOWN_810A_UNK0__SHIFT) & A6XX_GRAS_UNKNOWN_810A_UNK0__MASK;
+	return ((val) << A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_LAYER__SHIFT) & A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_LAYER__MASK;
 }
-#define A6XX_GRAS_UNKNOWN_810A_UNK16__MASK			0x07ff0000
-#define A6XX_GRAS_UNKNOWN_810A_UNK16__SHIFT			16
-static inline uint32_t A6XX_GRAS_UNKNOWN_810A_UNK16(uint32_t val)
+#define A6XX_GRAS_LRZ_DEPTH_VIEW_LAYER_COUNT__MASK		0x07ff0000
+#define A6XX_GRAS_LRZ_DEPTH_VIEW_LAYER_COUNT__SHIFT		16
+static inline uint32_t A6XX_GRAS_LRZ_DEPTH_VIEW_LAYER_COUNT(uint32_t val)
 {
-	return ((val) << A6XX_GRAS_UNKNOWN_810A_UNK16__SHIFT) & A6XX_GRAS_UNKNOWN_810A_UNK16__MASK;
+	return ((val) << A6XX_GRAS_LRZ_DEPTH_VIEW_LAYER_COUNT__SHIFT) & A6XX_GRAS_LRZ_DEPTH_VIEW_LAYER_COUNT__MASK;
 }
-#define A6XX_GRAS_UNKNOWN_810A_UNK28__MASK			0xf0000000
-#define A6XX_GRAS_UNKNOWN_810A_UNK28__SHIFT			28
-static inline uint32_t A6XX_GRAS_UNKNOWN_810A_UNK28(uint32_t val)
+#define A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_MIP_LEVEL__MASK		0xf0000000
+#define A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_MIP_LEVEL__SHIFT		28
+static inline uint32_t A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_MIP_LEVEL(uint32_t val)
 {
-	return ((val) << A6XX_GRAS_UNKNOWN_810A_UNK28__SHIFT) & A6XX_GRAS_UNKNOWN_810A_UNK28__MASK;
+	return ((val) << A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_MIP_LEVEL__SHIFT) & A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_MIP_LEVEL__MASK;
 }
 
 #define REG_A6XX_GRAS_UNKNOWN_8110				0x00008110
@@ -2900,6 +3161,8 @@ static inline uint32_t A6XX_GRAS_2D_RESOLVE_CNTL_2_Y(uint32_t val)
 
 #define REG_A6XX_GRAS_ADDR_MODE_CNTL				0x00008601
 
+#define REG_A7XX_GRAS_NC_MODE_CNTL				0x00008602
+
 static inline uint32_t REG_A6XX_GRAS_PERFCTR_TSE_SEL(uint32_t i0) { return 0x00008610 + 0x1*i0; }
 
 static inline uint32_t REG_A6XX_GRAS_PERFCTR_RAS_SEL(uint32_t i0) { return 0x00008614 + 0x1*i0; }
@@ -3126,7 +3389,7 @@ static inline uint32_t A6XX_RB_RENDER_CONTROL0_COORD_MASK(uint32_t val)
 
 #define REG_A6XX_RB_RENDER_CONTROL1				0x0000880a
 #define A6XX_RB_RENDER_CONTROL1_SAMPLEMASK			0x00000001
-#define A6XX_RB_RENDER_CONTROL1_UNK1				0x00000002
+#define A6XX_RB_RENDER_CONTROL1_POSTDEPTHCOVERAGE		0x00000002
 #define A6XX_RB_RENDER_CONTROL1_FACENESS			0x00000004
 #define A6XX_RB_RENDER_CONTROL1_SAMPLEID			0x00000008
 #define A6XX_RB_RENDER_CONTROL1_FRAGCOORDSAMPLEMODE__MASK	0x00000030
@@ -3135,7 +3398,7 @@ static inline uint32_t A6XX_RB_RENDER_CONTROL1_FRAGCOORDSAMPLEMODE(enum a6xx_fra
 {
 	return ((val) << A6XX_RB_RENDER_CONTROL1_FRAGCOORDSAMPLEMODE__SHIFT) & A6XX_RB_RENDER_CONTROL1_FRAGCOORDSAMPLEMODE__MASK;
 }
-#define A6XX_RB_RENDER_CONTROL1_SIZE				0x00000040
+#define A6XX_RB_RENDER_CONTROL1_CENTERRHW			0x00000040
 #define A6XX_RB_RENDER_CONTROL1_LINELENGTHEN			0x00000080
 #define A6XX_RB_RENDER_CONTROL1_FOVEATION			0x00000100
 
@@ -3691,7 +3954,7 @@ static inline uint32_t A6XX_RB_WINDOW_OFFSET_Y(uint32_t val)
 }
 
 #define REG_A6XX_RB_SAMPLE_COUNT_CONTROL			0x00008891
-#define A6XX_RB_SAMPLE_COUNT_CONTROL_UNK0			0x00000001
+#define A6XX_RB_SAMPLE_COUNT_CONTROL_DISABLE			0x00000001
 #define A6XX_RB_SAMPLE_COUNT_CONTROL_COPY			0x00000002
 
 #define REG_A6XX_RB_LRZ_CNTL					0x00008898
@@ -3783,12 +4046,12 @@ static inline uint32_t A6XX_RB_WINDOW_OFFSET2_Y(uint32_t val)
 	return ((val) << A6XX_RB_WINDOW_OFFSET2_Y__SHIFT) & A6XX_RB_WINDOW_OFFSET2_Y__MASK;
 }
 
-#define REG_A6XX_RB_MSAA_CNTL					0x000088d5
-#define A6XX_RB_MSAA_CNTL_SAMPLES__MASK				0x00000018
-#define A6XX_RB_MSAA_CNTL_SAMPLES__SHIFT			3
-static inline uint32_t A6XX_RB_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+#define REG_A6XX_RB_BLIT_GMEM_MSAA_CNTL				0x000088d5
+#define A6XX_RB_BLIT_GMEM_MSAA_CNTL_SAMPLES__MASK		0x00000018
+#define A6XX_RB_BLIT_GMEM_MSAA_CNTL_SAMPLES__SHIFT		3
+static inline uint32_t A6XX_RB_BLIT_GMEM_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
 {
-	return ((val) << A6XX_RB_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_RB_MSAA_CNTL_SAMPLES__MASK;
+	return ((val) << A6XX_RB_BLIT_GMEM_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_RB_BLIT_GMEM_MSAA_CNTL_SAMPLES__MASK;
 }
 
 #define REG_A6XX_RB_BLIT_BASE_GMEM				0x000088d6
@@ -3892,17 +4155,17 @@ static inline uint32_t A6XX_RB_BLIT_INFO_CLEAR_MASK(uint32_t val)
 {
 	return ((val) << A6XX_RB_BLIT_INFO_CLEAR_MASK__SHIFT) & A6XX_RB_BLIT_INFO_CLEAR_MASK__MASK;
 }
-#define A6XX_RB_BLIT_INFO_UNK8__MASK				0x00000300
-#define A6XX_RB_BLIT_INFO_UNK8__SHIFT				8
-static inline uint32_t A6XX_RB_BLIT_INFO_UNK8(uint32_t val)
+#define A6XX_RB_BLIT_INFO_LAST__MASK				0x00000300
+#define A6XX_RB_BLIT_INFO_LAST__SHIFT				8
+static inline uint32_t A6XX_RB_BLIT_INFO_LAST(uint32_t val)
 {
-	return ((val) << A6XX_RB_BLIT_INFO_UNK8__SHIFT) & A6XX_RB_BLIT_INFO_UNK8__MASK;
+	return ((val) << A6XX_RB_BLIT_INFO_LAST__SHIFT) & A6XX_RB_BLIT_INFO_LAST__MASK;
 }
-#define A6XX_RB_BLIT_INFO_UNK12__MASK				0x0000f000
-#define A6XX_RB_BLIT_INFO_UNK12__SHIFT				12
-static inline uint32_t A6XX_RB_BLIT_INFO_UNK12(uint32_t val)
+#define A6XX_RB_BLIT_INFO_BUFFER_ID__MASK			0x0000f000
+#define A6XX_RB_BLIT_INFO_BUFFER_ID__SHIFT			12
+static inline uint32_t A6XX_RB_BLIT_INFO_BUFFER_ID(uint32_t val)
 {
-	return ((val) << A6XX_RB_BLIT_INFO_UNK12__SHIFT) & A6XX_RB_BLIT_INFO_UNK12__MASK;
+	return ((val) << A6XX_RB_BLIT_INFO_BUFFER_ID__SHIFT) & A6XX_RB_BLIT_INFO_BUFFER_ID__MASK;
 }
 
 #define REG_A6XX_RB_UNKNOWN_88F0				0x000088f0
@@ -4173,16 +4436,23 @@ static inline uint32_t A6XX_RB_2D_DST_FLAGS_PLANE_PITCH(uint32_t val)
 
 #define REG_A6XX_RB_UNKNOWN_8E01				0x00008e01
 
-#define REG_A6XX_RB_UNKNOWN_8E04				0x00008e04
+#define REG_A6XX_RB_DBG_ECO_CNTL				0x00008e04
 
 #define REG_A6XX_RB_ADDR_MODE_CNTL				0x00008e05
 
 #define REG_A6XX_RB_CCU_CNTL					0x00008e07
-#define A6XX_RB_CCU_CNTL_COLOR_OFFSET__MASK			0xff800000
-#define A6XX_RB_CCU_CNTL_COLOR_OFFSET__SHIFT			23
-static inline uint32_t A6XX_RB_CCU_CNTL_COLOR_OFFSET(uint32_t val)
+#define A6XX_RB_CCU_CNTL_CONCURRENT_RESOLVE			0x00000004
+#define A6XX_RB_CCU_CNTL_DEPTH_OFFSET_HI__MASK			0x00000080
+#define A6XX_RB_CCU_CNTL_DEPTH_OFFSET_HI__SHIFT			7
+static inline uint32_t A6XX_RB_CCU_CNTL_DEPTH_OFFSET_HI(uint32_t val)
 {
-	return ((val >> 12) << A6XX_RB_CCU_CNTL_COLOR_OFFSET__SHIFT) & A6XX_RB_CCU_CNTL_COLOR_OFFSET__MASK;
+	return ((val) << A6XX_RB_CCU_CNTL_DEPTH_OFFSET_HI__SHIFT) & A6XX_RB_CCU_CNTL_DEPTH_OFFSET_HI__MASK;
+}
+#define A6XX_RB_CCU_CNTL_COLOR_OFFSET_HI__MASK			0x00000200
+#define A6XX_RB_CCU_CNTL_COLOR_OFFSET_HI__SHIFT			9
+static inline uint32_t A6XX_RB_CCU_CNTL_COLOR_OFFSET_HI(uint32_t val)
+{
+	return ((val) << A6XX_RB_CCU_CNTL_COLOR_OFFSET_HI__SHIFT) & A6XX_RB_CCU_CNTL_COLOR_OFFSET_HI__MASK;
 }
 #define A6XX_RB_CCU_CNTL_DEPTH_OFFSET__MASK			0x001ff000
 #define A6XX_RB_CCU_CNTL_DEPTH_OFFSET__SHIFT			12
@@ -4191,7 +4461,12 @@ static inline uint32_t A6XX_RB_CCU_CNTL_DEPTH_OFFSET(uint32_t val)
 	return ((val >> 12) << A6XX_RB_CCU_CNTL_DEPTH_OFFSET__SHIFT) & A6XX_RB_CCU_CNTL_DEPTH_OFFSET__MASK;
 }
 #define A6XX_RB_CCU_CNTL_GMEM					0x00400000
-#define A6XX_RB_CCU_CNTL_UNK2					0x00000004
+#define A6XX_RB_CCU_CNTL_COLOR_OFFSET__MASK			0xff800000
+#define A6XX_RB_CCU_CNTL_COLOR_OFFSET__SHIFT			23
+static inline uint32_t A6XX_RB_CCU_CNTL_COLOR_OFFSET(uint32_t val)
+{
+	return ((val >> 12) << A6XX_RB_CCU_CNTL_COLOR_OFFSET__SHIFT) & A6XX_RB_CCU_CNTL_COLOR_OFFSET__MASK;
+}
 
 #define REG_A6XX_RB_NC_MODE_CNTL				0x00008e08
 #define A6XX_RB_NC_MODE_CNTL_MODE				0x00000001
@@ -4225,6 +4500,8 @@ static inline uint32_t REG_A6XX_RB_PERFCTR_CCU_SEL(uint32_t i0) { return 0x00008
 
 static inline uint32_t REG_A6XX_RB_PERFCTR_CMP_SEL(uint32_t i0) { return 0x00008e2c + 0x1*i0; }
 
+static inline uint32_t REG_A7XX_RB_PERFCTR_UFC_SEL(uint32_t i0) { return 0x00008e30 + 0x1*i0; }
+
 #define REG_A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_HOST			0x00008e3b
 
 #define REG_A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_CD			0x00008e3d
@@ -4440,7 +4717,13 @@ static inline uint32_t A6XX_VPC_SO_BUFFER_SIZE(uint32_t val)
 	return ((val >> 2) << A6XX_VPC_SO_BUFFER_SIZE__SHIFT) & A6XX_VPC_SO_BUFFER_SIZE__MASK;
 }
 
-static inline uint32_t REG_A6XX_VPC_SO_NCOMP(uint32_t i0) { return 0x0000921d + 0x7*i0; }
+static inline uint32_t REG_A6XX_VPC_SO_BUFFER_STRIDE(uint32_t i0) { return 0x0000921d + 0x7*i0; }
+#define A6XX_VPC_SO_BUFFER_STRIDE__MASK				0x000003ff
+#define A6XX_VPC_SO_BUFFER_STRIDE__SHIFT			0
+static inline uint32_t A6XX_VPC_SO_BUFFER_STRIDE(uint32_t val)
+{
+	return ((val >> 2) << A6XX_VPC_SO_BUFFER_STRIDE__SHIFT) & A6XX_VPC_SO_BUFFER_STRIDE__MASK;
+}
 
 static inline uint32_t REG_A6XX_VPC_SO_BUFFER_OFFSET(uint32_t i0) { return 0x0000921e + 0x7*i0; }
 #define A6XX_VPC_SO_BUFFER_OFFSET__MASK				0xfffffffc
@@ -4597,7 +4880,7 @@ static inline uint32_t A6XX_VPC_SO_STREAM_CNTL_STREAM_ENABLE(uint32_t val)
 #define REG_A6XX_VPC_SO_DISABLE					0x00009306
 #define A6XX_VPC_SO_DISABLE_DISABLE				0x00000001
 
-#define REG_A6XX_VPC_UNKNOWN_9600				0x00009600
+#define REG_A6XX_VPC_DBG_ECO_CNTL				0x00009600
 
 #define REG_A6XX_VPC_ADDR_MODE_CNTL				0x00009601
 
@@ -4607,6 +4890,8 @@ static inline uint32_t A6XX_VPC_SO_STREAM_CNTL_STREAM_ENABLE(uint32_t val)
 
 static inline uint32_t REG_A6XX_VPC_PERFCTR_VPC_SEL(uint32_t i0) { return 0x00009604 + 0x1*i0; }
 
+static inline uint32_t REG_A7XX_VPC_PERFCTR_VPC_SEL(uint32_t i0) { return 0x0000960b + 0x1*i0; }
+
 #define REG_A6XX_PC_TESS_NUM_VERTEX				0x00009800
 
 #define REG_A6XX_PC_HS_INPUT_SIZE				0x00009801
@@ -4646,7 +4931,12 @@ static inline uint32_t A6XX_PC_TESS_CNTL_OUTPUT(enum a6xx_tess_output val)
 #define REG_A6XX_PC_PRIMID_PASSTHRU				0x00009806
 
 #define REG_A6XX_PC_SO_STREAM_CNTL				0x00009808
-#define A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE			0x00008000
+#define A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE__MASK		0x00078000
+#define A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE__SHIFT		15
+static inline uint32_t A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE(uint32_t val)
+{
+	return ((val) << A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE__SHIFT) & A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE__MASK;
+}
 
 #define REG_A6XX_PC_DGEN_SU_CONSERVATIVE_RAS_CNTL		0x0000980a
 #define A6XX_PC_DGEN_SU_CONSERVATIVE_RAS_CNTL_CONSERVATIVERASEN	0x00000001
@@ -4936,6 +5226,8 @@ static inline uint32_t A6XX_PC_BIN_DRAW_STRM(uint32_t val)
 
 static inline uint32_t REG_A6XX_PC_PERFCTR_PC_SEL(uint32_t i0) { return 0x00009e34 + 0x1*i0; }
 
+static inline uint32_t REG_A7XX_PC_PERFCTR_PC_SEL(uint32_t i0) { return 0x00009e42 + 0x1*i0; }
+
 #define REG_A6XX_PC_UNKNOWN_9E72				0x00009e72
 
 #define REG_A6XX_VFD_CONTROL_0					0x0000a000
@@ -5138,9 +5430,11 @@ static inline uint32_t A6XX_VFD_DEST_CNTL_INSTR_REGID(uint32_t val)
 
 static inline uint32_t REG_A6XX_VFD_PERFCTR_VFD_SEL(uint32_t i0) { return 0x0000a610 + 0x1*i0; }
 
+static inline uint32_t REG_A7XX_VFD_PERFCTR_VFD_SEL(uint32_t i0) { return 0x0000a610 + 0x1*i0; }
+
 #define REG_A6XX_SP_VS_CTRL_REG0				0x0000a800
 #define A6XX_SP_VS_CTRL_REG0_MERGEDREGS				0x00100000
-#define A6XX_SP_VS_CTRL_REG0_UNK21				0x00200000
+#define A6XX_SP_VS_CTRL_REG0_EARLYPREAMBLE			0x00200000
 #define A6XX_SP_VS_CTRL_REG0_THREADMODE__MASK			0x00000001
 #define A6XX_SP_VS_CTRL_REG0_THREADMODE__SHIFT			0
 static inline uint32_t A6XX_SP_VS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
@@ -5318,7 +5612,7 @@ static inline uint32_t A6XX_SP_VS_PVT_MEM_HW_STACK_OFFSET_OFFSET(uint32_t val)
 }
 
 #define REG_A6XX_SP_HS_CTRL_REG0				0x0000a830
-#define A6XX_SP_HS_CTRL_REG0_UNK20				0x00100000
+#define A6XX_SP_HS_CTRL_REG0_EARLYPREAMBLE			0x00100000
 #define A6XX_SP_HS_CTRL_REG0_THREADMODE__MASK			0x00000001
 #define A6XX_SP_HS_CTRL_REG0_THREADMODE__SHIFT			0
 static inline uint32_t A6XX_SP_HS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
@@ -5428,7 +5722,7 @@ static inline uint32_t A6XX_SP_HS_PVT_MEM_HW_STACK_OFFSET_OFFSET(uint32_t val)
 }
 
 #define REG_A6XX_SP_DS_CTRL_REG0				0x0000a840
-#define A6XX_SP_DS_CTRL_REG0_MERGEDREGS				0x00100000
+#define A6XX_SP_DS_CTRL_REG0_EARLYPREAMBLE			0x00100000
 #define A6XX_SP_DS_CTRL_REG0_THREADMODE__MASK			0x00000001
 #define A6XX_SP_DS_CTRL_REG0_THREADMODE__SHIFT			0
 static inline uint32_t A6XX_SP_DS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
@@ -5606,7 +5900,7 @@ static inline uint32_t A6XX_SP_DS_PVT_MEM_HW_STACK_OFFSET_OFFSET(uint32_t val)
 }
 
 #define REG_A6XX_SP_GS_CTRL_REG0				0x0000a870
-#define A6XX_SP_GS_CTRL_REG0_UNK20				0x00100000
+#define A6XX_SP_GS_CTRL_REG0_EARLYPREAMBLE			0x00100000
 #define A6XX_SP_GS_CTRL_REG0_THREADMODE__MASK			0x00000001
 #define A6XX_SP_GS_CTRL_REG0_THREADMODE__SHIFT			0
 static inline uint32_t A6XX_SP_GS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
@@ -5862,12 +6156,8 @@ static inline uint32_t A6XX_SP_FS_CTRL_REG0_THREADSIZE(enum a6xx_threadsize val)
 #define A6XX_SP_FS_CTRL_REG0_UNK24				0x01000000
 #define A6XX_SP_FS_CTRL_REG0_UNK25				0x02000000
 #define A6XX_SP_FS_CTRL_REG0_PIXLODENABLE			0x04000000
-#define A6XX_SP_FS_CTRL_REG0_UNK27__MASK			0x18000000
-#define A6XX_SP_FS_CTRL_REG0_UNK27__SHIFT			27
-static inline uint32_t A6XX_SP_FS_CTRL_REG0_UNK27(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_CTRL_REG0_UNK27__SHIFT) & A6XX_SP_FS_CTRL_REG0_UNK27__MASK;
-}
+#define A6XX_SP_FS_CTRL_REG0_UNK27				0x08000000
+#define A6XX_SP_FS_CTRL_REG0_EARLYPREAMBLE			0x10000000
 #define A6XX_SP_FS_CTRL_REG0_MERGEDREGS				0x80000000
 #define A6XX_SP_FS_CTRL_REG0_THREADMODE__MASK			0x00000001
 #define A6XX_SP_FS_CTRL_REG0_THREADMODE__SHIFT			0
@@ -6069,18 +6359,14 @@ static inline uint32_t A6XX_SP_FS_PREFETCH_CNTL_COUNT(uint32_t val)
 {
 	return ((val) << A6XX_SP_FS_PREFETCH_CNTL_COUNT__SHIFT) & A6XX_SP_FS_PREFETCH_CNTL_COUNT__MASK;
 }
-#define A6XX_SP_FS_PREFETCH_CNTL_UNK3				0x00000008
-#define A6XX_SP_FS_PREFETCH_CNTL_UNK4__MASK			0x00000ff0
-#define A6XX_SP_FS_PREFETCH_CNTL_UNK4__SHIFT			4
-static inline uint32_t A6XX_SP_FS_PREFETCH_CNTL_UNK4(uint32_t val)
+#define A6XX_SP_FS_PREFETCH_CNTL_IJ_WRITE_DISABLE		0x00000008
+#define A6XX_SP_FS_PREFETCH_CNTL_UNK4				0x00000010
+#define A6XX_SP_FS_PREFETCH_CNTL_WRITE_COLOR_TO_OUTPUT		0x00000020
+#define A6XX_SP_FS_PREFETCH_CNTL_UNK6__MASK			0x00007fc0
+#define A6XX_SP_FS_PREFETCH_CNTL_UNK6__SHIFT			6
+static inline uint32_t A6XX_SP_FS_PREFETCH_CNTL_UNK6(uint32_t val)
 {
-	return ((val) << A6XX_SP_FS_PREFETCH_CNTL_UNK4__SHIFT) & A6XX_SP_FS_PREFETCH_CNTL_UNK4__MASK;
-}
-#define A6XX_SP_FS_PREFETCH_CNTL_UNK12__MASK			0x00007000
-#define A6XX_SP_FS_PREFETCH_CNTL_UNK12__SHIFT			12
-static inline uint32_t A6XX_SP_FS_PREFETCH_CNTL_UNK12(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_PREFETCH_CNTL_UNK12__SHIFT) & A6XX_SP_FS_PREFETCH_CNTL_UNK12__MASK;
+	return ((val) << A6XX_SP_FS_PREFETCH_CNTL_UNK6__SHIFT) & A6XX_SP_FS_PREFETCH_CNTL_UNK6__MASK;
 }
 
 static inline uint32_t REG_A6XX_SP_FS_PREFETCH(uint32_t i0) { return 0x0000a99f + 0x1*i0; }
@@ -6117,9 +6403,11 @@ static inline uint32_t A6XX_SP_FS_PREFETCH_CMD_WRMASK(uint32_t val)
 	return ((val) << A6XX_SP_FS_PREFETCH_CMD_WRMASK__SHIFT) & A6XX_SP_FS_PREFETCH_CMD_WRMASK__MASK;
 }
 #define A6XX_SP_FS_PREFETCH_CMD_HALF				0x04000000
-#define A6XX_SP_FS_PREFETCH_CMD_CMD__MASK			0xf8000000
-#define A6XX_SP_FS_PREFETCH_CMD_CMD__SHIFT			27
-static inline uint32_t A6XX_SP_FS_PREFETCH_CMD_CMD(uint32_t val)
+#define A6XX_SP_FS_PREFETCH_CMD_UNK27				0x08000000
+#define A6XX_SP_FS_PREFETCH_CMD_BINDLESS			0x10000000
+#define A6XX_SP_FS_PREFETCH_CMD_CMD__MASK			0xe0000000
+#define A6XX_SP_FS_PREFETCH_CMD_CMD__SHIFT			29
+static inline uint32_t A6XX_SP_FS_PREFETCH_CMD_CMD(enum a6xx_tex_prefetch_cmd val)
 {
 	return ((val) << A6XX_SP_FS_PREFETCH_CMD_CMD__SHIFT) & A6XX_SP_FS_PREFETCH_CMD_CMD__MASK;
 }
@@ -6161,7 +6449,7 @@ static inline uint32_t A6XX_SP_CS_CTRL_REG0_THREADSIZE(enum a6xx_threadsize val)
 }
 #define A6XX_SP_CS_CTRL_REG0_UNK21				0x00200000
 #define A6XX_SP_CS_CTRL_REG0_UNK22				0x00400000
-#define A6XX_SP_CS_CTRL_REG0_SEPARATEPROLOG			0x00800000
+#define A6XX_SP_CS_CTRL_REG0_EARLYPREAMBLE			0x00800000
 #define A6XX_SP_CS_CTRL_REG0_MERGEDREGS				0x80000000
 #define A6XX_SP_CS_CTRL_REG0_THREADMODE__MASK			0x00000001
 #define A6XX_SP_CS_CTRL_REG0_THREADMODE__SHIFT			0
@@ -6355,7 +6643,19 @@ static inline uint32_t A6XX_SP_CS_TEX_CONST(uint32_t val)
 
 static inline uint32_t REG_A6XX_SP_CS_BINDLESS_BASE(uint32_t i0) { return 0x0000a9e8 + 0x2*i0; }
 
-static inline uint32_t REG_A6XX_SP_CS_BINDLESS_BASE_ADDR(uint32_t i0) { return 0x0000a9e8 + 0x2*i0; }
+static inline uint32_t REG_A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR(uint32_t i0) { return 0x0000a9e8 + 0x2*i0; }
+#define A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK	0x00000003
+#define A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT	0
+static inline uint32_t A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE(enum a6xx_bindless_descriptor_size val)
+{
+	return ((val) << A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT) & A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK;
+}
+#define A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK		0xfffffffc
+#define A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT		2
+static inline uint32_t A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR(uint32_t val)
+{
+	return ((val >> 2) << A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT) & A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK;
+}
 
 #define REG_A6XX_SP_CS_IBO					0x0000a9f2
 #define A6XX_SP_CS_IBO__MASK					0xffffffff
@@ -6406,7 +6706,19 @@ static inline uint32_t A6XX_SP_FS_CONFIG_NIBO(uint32_t val)
 
 static inline uint32_t REG_A6XX_SP_BINDLESS_BASE(uint32_t i0) { return 0x0000ab10 + 0x2*i0; }
 
-static inline uint32_t REG_A6XX_SP_BINDLESS_BASE_ADDR(uint32_t i0) { return 0x0000ab10 + 0x2*i0; }
+static inline uint32_t REG_A6XX_SP_BINDLESS_BASE_DESCRIPTOR(uint32_t i0) { return 0x0000ab10 + 0x2*i0; }
+#define A6XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK	0x00000003
+#define A6XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT	0
+static inline uint32_t A6XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE(enum a6xx_bindless_descriptor_size val)
+{
+	return ((val) << A6XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT) & A6XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK;
+}
+#define A6XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK		0xfffffffc
+#define A6XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT		2
+static inline uint32_t A6XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR(uint32_t val)
+{
+	return ((val >> 2) << A6XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT) & A6XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK;
+}
 
 #define REG_A6XX_SP_IBO						0x0000ab1a
 #define A6XX_SP_IBO__MASK					0xffffffff
@@ -6436,7 +6748,7 @@ static inline uint32_t A6XX_SP_2D_DST_FORMAT_MASK(uint32_t val)
 	return ((val) << A6XX_SP_2D_DST_FORMAT_MASK__SHIFT) & A6XX_SP_2D_DST_FORMAT_MASK__MASK;
 }
 
-#define REG_A6XX_SP_UNKNOWN_AE00				0x0000ae00
+#define REG_A6XX_SP_DBG_ECO_CNTL				0x0000ae00
 
 #define REG_A6XX_SP_ADDR_MODE_CNTL				0x0000ae01
 
@@ -6457,6 +6769,12 @@ static inline uint32_t A6XX_SP_2D_DST_FORMAT_MASK(uint32_t val)
 
 static inline uint32_t REG_A6XX_SP_PERFCTR_SP_SEL(uint32_t i0) { return 0x0000ae10 + 0x1*i0; }
 
+static inline uint32_t REG_A7XX_SP_PERFCTR_HLSQ_SEL(uint32_t i0) { return 0x0000ae60 + 0x1*i0; }
+
+#define REG_A7XX_SP_READ_SEL					0x0000ae6d
+
+static inline uint32_t REG_A7XX_SP_PERFCTR_SP_SEL(uint32_t i0) { return 0x0000ae80 + 0x1*i0; }
+
 #define REG_A6XX_SP_CONTEXT_SWITCH_GFX_PREEMPTION_SAFE_MODE	0x0000be22
 
 #define REG_A6XX_SP_PS_TP_BORDER_COLOR_BASE_ADDR		0x0000b180
@@ -6887,6 +7205,8 @@ static inline uint32_t A6XX_HLSQ_FS_CNTL_0_UNK2(uint32_t val)
 
 #define REG_A6XX_HLSQ_CONTROL_1_REG				0x0000b982
 
+#define REG_A7XX_HLSQ_CONTROL_1_REG				0x0000a9c7
+
 #define REG_A6XX_HLSQ_CONTROL_2_REG				0x0000b983
 #define A6XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK			0x000000ff
 #define A6XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT		0
@@ -6906,11 +7226,37 @@ static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(uint32_t val)
 {
 	return ((val) << A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK;
 }
-#define A6XX_HLSQ_CONTROL_2_REG_SIZE__MASK			0xff000000
-#define A6XX_HLSQ_CONTROL_2_REG_SIZE__SHIFT			24
-static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_SIZE(uint32_t val)
+#define A6XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK			0xff000000
+#define A6XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT		24
+static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_CENTERRHW(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK;
+}
+
+#define REG_A7XX_HLSQ_CONTROL_2_REG				0x0000a9c8
+#define A7XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK			0x000000ff
+#define A7XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT		0
+static inline uint32_t A7XX_HLSQ_CONTROL_2_REG_FACEREGID(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT) & A7XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK;
+}
+#define A7XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK			0x0000ff00
+#define A7XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT			8
+static inline uint32_t A7XX_HLSQ_CONTROL_2_REG_SAMPLEID(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT) & A7XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK;
+}
+#define A7XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK		0x00ff0000
+#define A7XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT		16
+static inline uint32_t A7XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT) & A7XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK;
+}
+#define A7XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK			0xff000000
+#define A7XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT		24
+static inline uint32_t A7XX_HLSQ_CONTROL_2_REG_CENTERRHW(uint32_t val)
 {
-	return ((val) << A6XX_HLSQ_CONTROL_2_REG_SIZE__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_SIZE__MASK;
+	return ((val) << A7XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT) & A7XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK;
 }
 
 #define REG_A6XX_HLSQ_CONTROL_3_REG				0x0000b984
@@ -6939,6 +7285,32 @@ static inline uint32_t A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID(uint32_t val)
 	return ((val) << A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT) & A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK;
 }
 
+#define REG_A7XX_HLSQ_CONTROL_3_REG				0x0000a9c9
+#define A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__MASK		0x000000ff
+#define A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__SHIFT		0
+static inline uint32_t A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__SHIFT) & A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__MASK;
+}
+#define A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__MASK		0x0000ff00
+#define A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__SHIFT		8
+static inline uint32_t A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__SHIFT) & A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__MASK;
+}
+#define A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__MASK		0x00ff0000
+#define A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__SHIFT	16
+static inline uint32_t A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__SHIFT) & A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__MASK;
+}
+#define A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK	0xff000000
+#define A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT	24
+static inline uint32_t A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT) & A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK;
+}
+
 #define REG_A6XX_HLSQ_CONTROL_4_REG				0x0000b985
 #define A6XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK		0x000000ff
 #define A6XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT		0
@@ -6965,6 +7337,32 @@ static inline uint32_t A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID(uint32_t val)
 	return ((val) << A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT) & A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK;
 }
 
+#define REG_A7XX_HLSQ_CONTROL_4_REG				0x0000a9ca
+#define A7XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK		0x000000ff
+#define A7XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT		0
+static inline uint32_t A7XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT) & A7XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK;
+}
+#define A7XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__MASK		0x0000ff00
+#define A7XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__SHIFT		8
+static inline uint32_t A7XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__SHIFT) & A7XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__MASK;
+}
+#define A7XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK		0x00ff0000
+#define A7XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT		16
+static inline uint32_t A7XX_HLSQ_CONTROL_4_REG_XYCOORDREGID(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT) & A7XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK;
+}
+#define A7XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK		0xff000000
+#define A7XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT		24
+static inline uint32_t A7XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT) & A7XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK;
+}
+
 #define REG_A6XX_HLSQ_CONTROL_5_REG				0x0000b986
 #define A6XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__MASK		0x000000ff
 #define A6XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__SHIFT		0
@@ -6979,6 +7377,20 @@ static inline uint32_t A6XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID(uint32_t va
 	return ((val) << A6XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__SHIFT) & A6XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__MASK;
 }
 
+#define REG_A7XX_HLSQ_CONTROL_5_REG				0x0000a9cb
+#define A7XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__MASK		0x000000ff
+#define A7XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__SHIFT		0
+static inline uint32_t A7XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__SHIFT) & A7XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__MASK;
+}
+#define A7XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__MASK	0x0000ff00
+#define A7XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__SHIFT	8
+static inline uint32_t A7XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID(uint32_t val)
+{
+	return ((val) << A7XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__SHIFT) & A7XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__MASK;
+}
+
 #define REG_A6XX_HLSQ_CS_CNTL					0x0000b987
 #define A6XX_HLSQ_CS_CNTL_CONSTLEN__MASK			0x000000ff
 #define A6XX_HLSQ_CS_CNTL_CONSTLEN__SHIFT			0
@@ -7124,7 +7536,19 @@ static inline uint32_t A6XX_HLSQ_LOAD_STATE_FRAG_EXT_SRC_ADDR(uint32_t val)
 
 static inline uint32_t REG_A6XX_HLSQ_CS_BINDLESS_BASE(uint32_t i0) { return 0x0000b9c0 + 0x2*i0; }
 
-static inline uint32_t REG_A6XX_HLSQ_CS_BINDLESS_BASE_ADDR(uint32_t i0) { return 0x0000b9c0 + 0x2*i0; }
+static inline uint32_t REG_A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR(uint32_t i0) { return 0x0000b9c0 + 0x2*i0; }
+#define A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK	0x00000003
+#define A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT	0
+static inline uint32_t A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE(enum a6xx_bindless_descriptor_size val)
+{
+	return ((val) << A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT) & A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK;
+}
+#define A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK	0xfffffffc
+#define A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT	2
+static inline uint32_t A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_ADDR(uint32_t val)
+{
+	return ((val >> 2) << A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT) & A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK;
+}
 
 #define REG_A6XX_HLSQ_CS_UNKNOWN_B9D0				0x0000b9d0
 #define A6XX_HLSQ_CS_UNKNOWN_B9D0_SHARED_SIZE__MASK		0x0000001f
@@ -7204,7 +7628,19 @@ static inline uint32_t A6XX_HLSQ_FS_CNTL_CONSTLEN(uint32_t val)
 
 static inline uint32_t REG_A6XX_HLSQ_BINDLESS_BASE(uint32_t i0) { return 0x0000bb20 + 0x2*i0; }
 
-static inline uint32_t REG_A6XX_HLSQ_BINDLESS_BASE_ADDR(uint32_t i0) { return 0x0000bb20 + 0x2*i0; }
+static inline uint32_t REG_A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR(uint32_t i0) { return 0x0000bb20 + 0x2*i0; }
+#define A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK	0x00000003
+#define A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT	0
+static inline uint32_t A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE(enum a6xx_bindless_descriptor_size val)
+{
+	return ((val) << A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT) & A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK;
+}
+#define A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK		0xfffffffc
+#define A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT		2
+static inline uint32_t A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_ADDR(uint32_t val)
+{
+	return ((val >> 2) << A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT) & A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK;
+}
 
 #define REG_A6XX_HLSQ_2D_EVENT_CMD				0x0000bd80
 #define A6XX_HLSQ_2D_EVENT_CMD_STATE_ID__MASK			0x0000ff00
@@ -7224,7 +7660,7 @@ static inline uint32_t A6XX_HLSQ_2D_EVENT_CMD_EVENT(enum vgt_event_type val)
 
 #define REG_A6XX_HLSQ_UNKNOWN_BE01				0x0000be01
 
-#define REG_A6XX_HLSQ_UNKNOWN_BE04				0x0000be04
+#define REG_A6XX_HLSQ_DBG_ECO_CNTL				0x0000be04
 
 #define REG_A6XX_HLSQ_ADDR_MODE_CNTL				0x0000be05
 
@@ -7234,6 +7670,8 @@ static inline uint32_t REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL(uint32_t i0) { return 0x00
 
 #define REG_A6XX_HLSQ_CONTEXT_SWITCH_GFX_PREEMPTION_SAFE_MODE	0x0000be22
 
+#define REG_A7XX_SP_AHB_READ_APERTURE				0x0000c000
+
 #define REG_A6XX_CP_EVENT_START					0x0000d600
 #define A6XX_CP_EVENT_START_STATE_ID__MASK			0x000000ff
 #define A6XX_CP_EVENT_START_STATE_ID__SHIFT			0
@@ -7426,7 +7864,18 @@ static inline uint32_t A6XX_TEX_CONST_1_HEIGHT(uint32_t val)
 }
 
 #define REG_A6XX_TEX_CONST_2					0x00000002
-#define A6XX_TEX_CONST_2_BUFFER					0x00000010
+#define A6XX_TEX_CONST_2_STRUCTSIZETEXELS__MASK			0x0000fff0
+#define A6XX_TEX_CONST_2_STRUCTSIZETEXELS__SHIFT		4
+static inline uint32_t A6XX_TEX_CONST_2_STRUCTSIZETEXELS(uint32_t val)
+{
+	return ((val) << A6XX_TEX_CONST_2_STRUCTSIZETEXELS__SHIFT) & A6XX_TEX_CONST_2_STRUCTSIZETEXELS__MASK;
+}
+#define A6XX_TEX_CONST_2_STARTOFFSETTEXELS__MASK		0x003f0000
+#define A6XX_TEX_CONST_2_STARTOFFSETTEXELS__SHIFT		16
+static inline uint32_t A6XX_TEX_CONST_2_STARTOFFSETTEXELS(uint32_t val)
+{
+	return ((val) << A6XX_TEX_CONST_2_STARTOFFSETTEXELS__SHIFT) & A6XX_TEX_CONST_2_STARTOFFSETTEXELS__MASK;
+}
 #define A6XX_TEX_CONST_2_PITCHALIGN__MASK			0x0000000f
 #define A6XX_TEX_CONST_2_PITCHALIGN__SHIFT			0
 static inline uint32_t A6XX_TEX_CONST_2_PITCHALIGN(uint32_t val)
@@ -7485,6 +7934,12 @@ static inline uint32_t A6XX_TEX_CONST_5_DEPTH(uint32_t val)
 }
 
 #define REG_A6XX_TEX_CONST_6					0x00000006
+#define A6XX_TEX_CONST_6_MIN_LOD_CLAMP__MASK			0x00000fff
+#define A6XX_TEX_CONST_6_MIN_LOD_CLAMP__SHIFT			0
+static inline uint32_t A6XX_TEX_CONST_6_MIN_LOD_CLAMP(float val)
+{
+	return ((((uint32_t)(val * 256.0))) << A6XX_TEX_CONST_6_MIN_LOD_CLAMP__SHIFT) & A6XX_TEX_CONST_6_MIN_LOD_CLAMP__MASK;
+}
 #define A6XX_TEX_CONST_6_PLANE_PITCH__MASK			0xffffff00
 #define A6XX_TEX_CONST_6_PLANE_PITCH__SHIFT			8
 static inline uint32_t A6XX_TEX_CONST_6_PLANE_PITCH(uint32_t val)
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
index 7f5bc73b2040..e16b4b3f8535 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -621,6 +621,8 @@ setup_pdc:
 	/* ensure no writes happen before the uCode is fully written */
 	wmb();
 
+	a6xx_rpmh_stop(gmu);
+
 err:
 	if (!IS_ERR_OR_NULL(pdcptr))
 		iounmap(pdcptr);
@@ -753,7 +755,6 @@ static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu)
 
 static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
 {
-	static bool rpmh_init;
 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 	int ret;
@@ -776,15 +777,9 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
 		/* Turn on register retention */
 		gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1);
 
-		/* We only need to load the RPMh microcode once */
-		if (!rpmh_init) {
-			a6xx_gmu_rpmh_init(gmu);
-			rpmh_init = true;
-		} else {
-			ret = a6xx_rpmh_start(gmu);
-			if (ret)
-				return ret;
-		}
+		ret = a6xx_rpmh_start(gmu);
+		if (ret)
+			return ret;
 
 		ret = a6xx_gmu_fw_load(gmu);
 		if (ret)
@@ -1482,6 +1477,12 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
 
 	pm_runtime_force_suspend(gmu->dev);
 
+	/*
+	 * Since cxpd is a virt device, the devlink with gmu-dev will be removed
+	 * automatically when we do detach
+	 */
+	dev_pm_domain_detach(gmu->cxpd, false);
+
 	if (!IS_ERR_OR_NULL(gmu->gxpd)) {
 		pm_runtime_disable(gmu->gxpd);
 		dev_pm_domain_detach(gmu->gxpd, false);
@@ -1504,6 +1505,17 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
 	gmu->initialized = false;
 }
 
+static int cxpd_notifier_cb(struct notifier_block *nb,
+			unsigned long action, void *data)
+{
+	struct a6xx_gmu *gmu = container_of(nb, struct a6xx_gmu, pd_nb);
+
+	if (action == GENPD_NOTIFY_OFF)
+		complete_all(&gmu->pd_gate);
+
+	return 0;
+}
+
 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 {
 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
@@ -1608,8 +1620,10 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 
 	if (adreno_is_a650_family(adreno_gpu)) {
 		gmu->rscc = a6xx_gmu_get_mmio(pdev, "rscc");
-		if (IS_ERR(gmu->rscc))
+		if (IS_ERR(gmu->rscc)) {
+			ret = -ENODEV;
 			goto err_mmio;
+		}
 	} else {
 		gmu->rscc = gmu->mmio + 0x23000;
 	}
@@ -1618,8 +1632,26 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 	gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq);
 	gmu->gmu_irq = a6xx_gmu_get_irq(gmu, pdev, "gmu", a6xx_gmu_irq);
 
-	if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0)
+	if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0) {
+		ret = -ENODEV;
 		goto err_mmio;
+	}
+
+	gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx");
+	if (IS_ERR(gmu->cxpd)) {
+		ret = PTR_ERR(gmu->cxpd);
+		goto err_mmio;
+	}
+
+	if (!device_link_add(gmu->dev, gmu->cxpd,
+					DL_FLAG_PM_RUNTIME)) {
+		ret = -ENODEV;
+		goto detach_cxpd;
+	}
+
+	init_completion(&gmu->pd_gate);
+	complete_all(&gmu->pd_gate);
+	gmu->pd_nb.notifier_call = cxpd_notifier_cb;
 
 	/*
 	 * Get a link to the GX power domain to reset the GPU in case of GMU
@@ -1633,10 +1665,16 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 	/* Set up the HFI queues */
 	a6xx_hfi_init(gmu);
 
+	/* Initialize RPMh */
+	a6xx_gmu_rpmh_init(gmu);
+
 	gmu->initialized = true;
 
 	return 0;
 
+detach_cxpd:
+	dev_pm_domain_detach(gmu->cxpd, false);
+
 err_mmio:
 	iounmap(gmu->mmio);
 	if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc"))
@@ -1644,8 +1682,6 @@ err_mmio:
 	free_irq(gmu->gmu_irq, gmu);
 	free_irq(gmu->hfi_irq, gmu);
 
-	ret = -ENODEV;
-
 err_memory:
 	a6xx_gmu_memory_free(gmu);
 err_put_device:
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
index e034935b3986..0bc3eb443fec 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
@@ -4,8 +4,10 @@
 #ifndef _A6XX_GMU_H_
 #define _A6XX_GMU_H_
 
+#include <linux/completion.h>
 #include <linux/iopoll.h>
 #include <linux/interrupt.h>
+#include <linux/notifier.h>
 #include "msm_drv.h"
 #include "a6xx_hfi.h"
 
@@ -56,6 +58,7 @@ struct a6xx_gmu {
 	int gmu_irq;
 
 	struct device *gxpd;
+	struct device *cxpd;
 
 	int idle_level;
 
@@ -89,6 +92,10 @@ struct a6xx_gmu {
 	bool initialized;
 	bool hung;
 	bool legacy; /* a618 or a630 */
+
+	/* For power domain callback */
+	struct notifier_block pd_nb;
+	struct completion pd_gate;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h
index 4a3230978c0e..9ab15d91aced 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h
@@ -8,21 +8,21 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a2xx.xml                (  90810 bytes, from 2021-06-21 15:24:24)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  14609 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  69086 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113358 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149512 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 184954 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11331 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   6038 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2924 bytes, from 2021-07-22 15:21:56)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a2xx.xml                (  91929 bytes, from 2023-02-28 23:52:27)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  15434 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  74995 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113474 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149590 bytes, from 2023-02-14 19:37:12)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 198949 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11404 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   9055 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2976 bytes, from 2023-03-10 18:32:52)
+
+Copyright (C) 2013-2023 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index 6faea5049f76..9fb214f150dd 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -10,7 +10,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/devfreq.h>
-#include <linux/reset.h>
+#include <linux/pm_domain.h>
 #include <linux/soc/qcom/llcc-qcom.h>
 
 #define GPU_PAS_ID 13
@@ -187,7 +187,7 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 	 * GPU registers so we need to add 0x1a800 to the register value on A630
 	 * to get the right value from PM4.
 	 */
-	get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
+	get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
 		rbmemptr_stats(ring, index, alwayson_start));
 
 	/* Invalidate CCU depth and color */
@@ -228,7 +228,7 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 
 	get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
 		rbmemptr_stats(ring, index, cpcycles_end));
-	get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
+	get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
 		rbmemptr_stats(ring, index, alwayson_end));
 
 	/* Write the fence to the scratch register */
@@ -247,7 +247,7 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 	OUT_RING(ring, submit->seqno);
 
 	trace_msm_gpu_submit_flush(submit,
-		gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO));
+		gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER));
 
 	a6xx_flush(gpu, ring);
 }
@@ -917,7 +917,7 @@ out:
 	return ret;
 }
 
-static int a6xx_ucode_init(struct msm_gpu *gpu)
+static int a6xx_ucode_load(struct msm_gpu *gpu)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
@@ -946,7 +946,23 @@ static int a6xx_ucode_init(struct msm_gpu *gpu)
 		}
 	}
 
-	gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE, a6xx_gpu->sqe_iova);
+	/*
+	 * Expanded APRIV and targets that support WHERE_AM_I both need a
+	 * privileged buffer to store the RPTR shadow
+	 */
+	if ((adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) &&
+	    !a6xx_gpu->shadow_bo) {
+		a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
+						      sizeof(u32) * gpu->nr_rings,
+						      MSM_BO_WC | MSM_BO_MAP_PRIV,
+						      gpu->aspace, &a6xx_gpu->shadow_bo,
+						      &a6xx_gpu->shadow_iova);
+
+		if (IS_ERR(a6xx_gpu->shadow))
+			return PTR_ERR(a6xx_gpu->shadow);
+
+		msm_gem_object_set_name(a6xx_gpu->shadow_bo, "shadow");
+	}
 
 	return 0;
 }
@@ -997,7 +1013,7 @@ static int hw_init(struct msm_gpu *gpu)
 	 * memory rendering at this point in time and we don't want to block off
 	 * part of the virtual memory space.
 	 */
-	gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 0x00000000);
+	gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE, 0x00000000);
 	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
 
 	/* Turn on 64 bit addressing for all blocks */
@@ -1037,18 +1053,15 @@ static int hw_init(struct msm_gpu *gpu)
 	gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);
 
 	/* Disable L2 bypass in the UCHE */
-	gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0);
-	gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff);
-	gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_LO, 0xfffff000);
-	gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff);
-	gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000);
-	gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff);
+	gpu_write64(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX, 0x0001ffffffffffc0llu);
+	gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, 0x0001fffffffff000llu);
+	gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, 0x0001fffffffff000llu);
 
 	if (!adreno_is_a650_family(adreno_gpu)) {
 		/* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
-		gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
+		gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN, 0x00100000);
 
-		gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO,
+		gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX,
 			0x00100000 + adreno_gpu->gmem - 1);
 	}
 
@@ -1135,9 +1148,7 @@ static int hw_init(struct msm_gpu *gpu)
 	if (ret)
 		goto out;
 
-	ret = a6xx_ucode_init(gpu);
-	if (ret)
-		goto out;
+	gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE, a6xx_gpu->sqe_iova);
 
 	/* Set the ringbuffer address */
 	gpu_write64(gpu, REG_A6XX_CP_RB_BASE, gpu->rb[0]->iova);
@@ -1152,26 +1163,9 @@ static int hw_init(struct msm_gpu *gpu)
 		gpu_write(gpu, REG_A6XX_CP_RB_CNTL,
 			MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
 
-	/*
-	 * Expanded APRIV and targets that support WHERE_AM_I both need a
-	 * privileged buffer to store the RPTR shadow
-	 */
-
-	if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) {
-		if (!a6xx_gpu->shadow_bo) {
-			a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
-				sizeof(u32) * gpu->nr_rings,
-				MSM_BO_WC | MSM_BO_MAP_PRIV,
-				gpu->aspace, &a6xx_gpu->shadow_bo,
-				&a6xx_gpu->shadow_iova);
-
-			if (IS_ERR(a6xx_gpu->shadow))
-				return PTR_ERR(a6xx_gpu->shadow);
-
-			msm_gem_object_set_name(a6xx_gpu->shadow_bo, "shadow");
-		}
-
-		gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR_LO,
+	/* Configure the RPTR shadow if needed: */
+	if (a6xx_gpu->shadow_bo) {
+		gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR,
 			shadowptr(a6xx_gpu, gpu->rb[0]));
 	}
 
@@ -1259,6 +1253,7 @@ static void a6xx_recover(struct msm_gpu *gpu)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
 	int i, active_submits;
 
 	adreno_dump_info(gpu);
@@ -1297,6 +1292,10 @@ static void a6xx_recover(struct msm_gpu *gpu)
 	 */
 	gpu->active_submits = 0;
 
+	reinit_completion(&gmu->pd_gate);
+	dev_pm_genpd_add_notifier(gmu->cxpd, &gmu->pd_nb);
+	dev_pm_genpd_synced_poweroff(gmu->cxpd);
+
 	/* Drop the rpm refcount from active submits */
 	if (active_submits)
 		pm_runtime_put(&gpu->pdev->dev);
@@ -1304,8 +1303,10 @@ static void a6xx_recover(struct msm_gpu *gpu)
 	/* And the final one from recover worker */
 	pm_runtime_put_sync(&gpu->pdev->dev);
 
-	/* Call into gpucc driver to poll for cx gdsc collapse */
-	reset_control_reset(gpu->cx_collapse);
+	if (!wait_for_completion_timeout(&gmu->pd_gate, msecs_to_jiffies(1000)))
+		DRM_DEV_ERROR(&gpu->pdev->dev, "cx gdsc didn't collapse\n");
+
+	dev_pm_genpd_remove_notifier(gmu->cxpd);
 
 	pm_runtime_use_autosuspend(&gpu->pdev->dev);
 
@@ -1361,73 +1362,23 @@ static const char *a6xx_fault_block(struct msm_gpu *gpu, u32 id)
 	return a6xx_uche_fault_block(gpu, id);
 }
 
-#define ARM_SMMU_FSR_TF                 BIT(1)
-#define ARM_SMMU_FSR_PF			BIT(3)
-#define ARM_SMMU_FSR_EF			BIT(4)
-
 static int a6xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
 {
 	struct msm_gpu *gpu = arg;
 	struct adreno_smmu_fault_info *info = data;
-	const char *type = "UNKNOWN";
-	const char *block;
-	bool do_devcoredump = info && !READ_ONCE(gpu->crashstate);
-
-	/*
-	 * If we aren't going to be resuming later from fault_worker, then do
-	 * it now.
-	 */
-	if (!do_devcoredump) {
-		gpu->aspace->mmu->funcs->resume_translation(gpu->aspace->mmu);
-	}
+	const char *block = "unknown";
 
-	/*
-	 * Print a default message if we couldn't get the data from the
-	 * adreno-smmu-priv
-	 */
-	if (!info) {
-		pr_warn_ratelimited("*** gpu fault: iova=%.16lx flags=%d (%u,%u,%u,%u)\n",
-			iova, flags,
+	u32 scratch[] = {
 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
-			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));
-
-		return 0;
-	}
-
-	if (info->fsr & ARM_SMMU_FSR_TF)
-		type = "TRANSLATION";
-	else if (info->fsr & ARM_SMMU_FSR_PF)
-		type = "PERMISSION";
-	else if (info->fsr & ARM_SMMU_FSR_EF)
-		type = "EXTERNAL";
-
-	block = a6xx_fault_block(gpu, info->fsynr1 & 0xff);
-
-	pr_warn_ratelimited("*** gpu fault: ttbr0=%.16llx iova=%.16lx dir=%s type=%s source=%s (%u,%u,%u,%u)\n",
-			info->ttbr0, iova,
-			flags & IOMMU_FAULT_WRITE ? "WRITE" : "READ",
-			type, block,
-			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
-			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
-			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
-			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));
-
-	if (do_devcoredump) {
-		/* Turn off the hangcheck timer to keep it from bothering us */
-		del_timer(&gpu->hangcheck_timer);
-
-		gpu->fault_info.ttbr0 = info->ttbr0;
-		gpu->fault_info.iova  = iova;
-		gpu->fault_info.flags = flags;
-		gpu->fault_info.type  = type;
-		gpu->fault_info.block = block;
+			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)),
+	};
 
-		kthread_queue_work(gpu->worker, &gpu->fault_work);
-	}
+	if (info)
+		block = a6xx_fault_block(gpu, info->fsynr1 & 0xff);
 
-	return 0;
+	return adreno_fault_handler(gpu, iova, flags, info, block, scratch);
 }
 
 static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)
@@ -1712,7 +1663,7 @@ static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
 	/* Force the GPU power on so we can read this register */
 	a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
 
-	*value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO);
+	*value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER);
 
 	a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
 
@@ -1848,8 +1799,8 @@ static bool a6xx_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 	 *     to prevent prefetching into an unrelated submit.  (And
 	 *     either way, at some point the ROQ will be full.)
 	 */
-	cp_state.ib1_rem += gpu_read(gpu, REG_A6XX_CP_CSQ_IB1_STAT) >> 16;
-	cp_state.ib2_rem += gpu_read(gpu, REG_A6XX_CP_CSQ_IB2_STAT) >> 16;
+	cp_state.ib1_rem += gpu_read(gpu, REG_A6XX_CP_ROQ_AVAIL_IB1) >> 16;
+	cp_state.ib2_rem += gpu_read(gpu, REG_A6XX_CP_ROQ_AVAIL_IB2) >> 16;
 
 	progress = !!memcmp(&cp_state, &ring->last_cp_state, sizeof(cp_state));
 
@@ -1886,6 +1837,31 @@ static u32 a619_get_speed_bin(u32 fuse)
 	return UINT_MAX;
 }
 
+static u32 a640_get_speed_bin(u32 fuse)
+{
+	if (fuse == 0)
+		return 0;
+	else if (fuse == 1)
+		return 1;
+
+	return UINT_MAX;
+}
+
+static u32 a650_get_speed_bin(u32 fuse)
+{
+	if (fuse == 0)
+		return 0;
+	else if (fuse == 1)
+		return 1;
+	/* Yep, 2 and 3 are swapped! :/ */
+	else if (fuse == 2)
+		return 3;
+	else if (fuse == 3)
+		return 2;
+
+	return UINT_MAX;
+}
+
 static u32 adreno_7c3_get_speed_bin(u32 fuse)
 {
 	if (fuse == 0)
@@ -1911,6 +1887,12 @@ static u32 fuse_to_supp_hw(struct device *dev, struct adreno_rev rev, u32 fuse)
 	if (adreno_cmp_rev(ADRENO_REV(6, 3, 5, ANY_ID), rev))
 		val = adreno_7c3_get_speed_bin(fuse);
 
+	if (adreno_cmp_rev(ADRENO_REV(6, 4, 0, ANY_ID), rev))
+		val = a640_get_speed_bin(fuse);
+
+	if (adreno_cmp_rev(ADRENO_REV(6, 5, 0, ANY_ID), rev))
+		val = a650_get_speed_bin(fuse);
+
 	if (val == UINT_MAX) {
 		DRM_DEV_ERROR(dev,
 			"missing support for speed-bin: %u. Some OPPs may not be supported by hardware\n",
@@ -1954,6 +1936,7 @@ static const struct adreno_gpu_funcs funcs = {
 		.get_param = adreno_get_param,
 		.set_param = adreno_set_param,
 		.hw_init = a6xx_hw_init,
+		.ucode_load = a6xx_ucode_load,
 		.pm_suspend = a6xx_pm_suspend,
 		.pm_resume = a6xx_pm_resume,
 		.recover = a6xx_recover,
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
index b7e217d00a22..30ecdff363e7 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
@@ -147,7 +147,7 @@ static int a6xx_crashdumper_run(struct msm_gpu *gpu,
 	/* Make sure all pending memory writes are posted */
 	wmb();
 
-	gpu_write64(gpu, REG_A6XX_CP_CRASH_SCRIPT_BASE_LO, dumper->iova);
+	gpu_write64(gpu, REG_A6XX_CP_CRASH_SCRIPT_BASE, dumper->iova);
 
 	gpu_write(gpu, REG_A6XX_CP_CRASH_DUMP_CNTL, 1);
 
diff --git a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
index abb037ccc02b..51c320a2e5c0 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
@@ -8,21 +8,21 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a2xx.xml                (  90810 bytes, from 2021-06-21 15:24:24)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  14609 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  69086 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113358 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149512 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 184954 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11331 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   6038 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2924 bytes, from 2021-07-22 15:21:56)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a2xx.xml                (  91929 bytes, from 2023-02-28 23:52:27)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  15434 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  74995 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113474 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149590 bytes, from 2023-02-14 19:37:12)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 198949 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11404 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   9055 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2976 bytes, from 2023-03-10 18:32:52)
+
+Copyright (C) 2013-2023 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -49,11 +49,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
 enum chip {
-	A2XX = 0,
-	A3XX = 0,
-	A4XX = 0,
-	A5XX = 0,
-	A6XX = 0,
+	A2XX = 2,
+	A3XX = 3,
+	A4XX = 4,
+	A5XX = 5,
+	A6XX = 6,
+	A7XX = 7,
 };
 
 enum adreno_pa_su_sc_draw {
@@ -210,6 +211,17 @@ enum a5xx_line_mode {
 	RECTANGULAR = 1,
 };
 
+enum a6xx_tex_prefetch_cmd {
+	TEX_PREFETCH_UNK0 = 0,
+	TEX_PREFETCH_SAM = 1,
+	TEX_PREFETCH_GATHER4R = 2,
+	TEX_PREFETCH_GATHER4G = 3,
+	TEX_PREFETCH_GATHER4B = 4,
+	TEX_PREFETCH_GATHER4A = 5,
+	TEX_PREFETCH_UNK6 = 6,
+	TEX_PREFETCH_UNK7 = 7,
+};
+
 #define REG_AXXX_CP_RB_BASE					0x000001c0
 
 #define REG_AXXX_CP_RB_CNTL					0x000001c1
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index c5c4c93b3689..8cff86e9d35c 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -432,31 +432,35 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev)
 	if (ret)
 		return NULL;
 
+	if (gpu->funcs->ucode_load) {
+		ret = gpu->funcs->ucode_load(gpu);
+		if (ret)
+			return NULL;
+	}
+
 	/*
 	 * Now that we have firmware loaded, and are ready to begin
 	 * booting the gpu, go ahead and enable runpm:
 	 */
 	pm_runtime_enable(&pdev->dev);
 
-	/* Make sure pm runtime is active and reset any previous errors */
-	pm_runtime_set_active(&pdev->dev);
-
 	ret = pm_runtime_get_sync(&pdev->dev);
 	if (ret < 0) {
-		pm_runtime_put_sync(&pdev->dev);
+		pm_runtime_put_noidle(&pdev->dev);
 		DRM_DEV_ERROR(dev->dev, "Couldn't power up the GPU: %d\n", ret);
-		return NULL;
+		goto err_disable_rpm;
 	}
 
 	mutex_lock(&gpu->lock);
 	ret = msm_gpu_hw_init(gpu);
 	mutex_unlock(&gpu->lock);
-	pm_runtime_put_autosuspend(&pdev->dev);
 	if (ret) {
 		DRM_DEV_ERROR(dev->dev, "gpu hw init failed: %d\n", ret);
-		return NULL;
+		goto err_put_rpm;
 	}
 
+	pm_runtime_put_autosuspend(&pdev->dev);
+
 #ifdef CONFIG_DEBUG_FS
 	if (gpu->funcs->debugfs_init) {
 		gpu->funcs->debugfs_init(gpu, dev->primary);
@@ -465,6 +469,13 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev)
 #endif
 
 	return gpu;
+
+err_put_rpm:
+	pm_runtime_put_sync_suspend(&pdev->dev);
+err_disable_rpm:
+	pm_runtime_disable(&pdev->dev);
+
+	return NULL;
 }
 
 static int find_chipid(struct device *dev, struct adreno_rev *rev)
@@ -548,6 +559,10 @@ static int adreno_bind(struct device *dev, struct device *master, void *data)
 		return PTR_ERR(gpu);
 	}
 
+	ret = dev_pm_opp_of_find_icc_paths(dev, NULL);
+	if (ret)
+		return ret;
+
 	return 0;
 }
 
@@ -566,8 +581,8 @@ static void adreno_unbind(struct device *dev, struct device *master,
 }
 
 static const struct component_ops a3xx_ops = {
-		.bind   = adreno_bind,
-		.unbind = adreno_unbind,
+	.bind   = adreno_bind,
+	.unbind = adreno_unbind,
 };
 
 static void adreno_device_register_headless(void)
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index ce6b76c45b6f..6934cee07d42 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -208,7 +208,7 @@ adreno_iommu_create_address_space(struct msm_gpu *gpu,
 	struct msm_gem_address_space *aspace;
 	u64 start, size;
 
-	mmu = msm_iommu_new(&pdev->dev, quirks);
+	mmu = msm_iommu_gpu_new(&pdev->dev, gpu, quirks);
 	if (IS_ERR_OR_NULL(mmu))
 		return ERR_CAST(mmu);
 
@@ -246,6 +246,66 @@ u64 adreno_private_address_space_size(struct msm_gpu *gpu)
 	return SZ_4G;
 }
 
+#define ARM_SMMU_FSR_TF                 BIT(1)
+#define ARM_SMMU_FSR_PF			BIT(3)
+#define ARM_SMMU_FSR_EF			BIT(4)
+
+int adreno_fault_handler(struct msm_gpu *gpu, unsigned long iova, int flags,
+			 struct adreno_smmu_fault_info *info, const char *block,
+			 u32 scratch[4])
+{
+	const char *type = "UNKNOWN";
+	bool do_devcoredump = info && !READ_ONCE(gpu->crashstate);
+
+	/*
+	 * If we aren't going to be resuming later from fault_worker, then do
+	 * it now.
+	 */
+	if (!do_devcoredump) {
+		gpu->aspace->mmu->funcs->resume_translation(gpu->aspace->mmu);
+	}
+
+	/*
+	 * Print a default message if we couldn't get the data from the
+	 * adreno-smmu-priv
+	 */
+	if (!info) {
+		pr_warn_ratelimited("*** gpu fault: iova=%.16lx flags=%d (%u,%u,%u,%u)\n",
+			iova, flags,
+			scratch[0], scratch[1], scratch[2], scratch[3]);
+
+		return 0;
+	}
+
+	if (info->fsr & ARM_SMMU_FSR_TF)
+		type = "TRANSLATION";
+	else if (info->fsr & ARM_SMMU_FSR_PF)
+		type = "PERMISSION";
+	else if (info->fsr & ARM_SMMU_FSR_EF)
+		type = "EXTERNAL";
+
+	pr_warn_ratelimited("*** gpu fault: ttbr0=%.16llx iova=%.16lx dir=%s type=%s source=%s (%u,%u,%u,%u)\n",
+			info->ttbr0, iova,
+			flags & IOMMU_FAULT_WRITE ? "WRITE" : "READ",
+			type, block,
+			scratch[0], scratch[1], scratch[2], scratch[3]);
+
+	if (do_devcoredump) {
+		/* Turn off the hangcheck timer to keep it from bothering us */
+		del_timer(&gpu->hangcheck_timer);
+
+		gpu->fault_info.ttbr0 = info->ttbr0;
+		gpu->fault_info.iova  = iova;
+		gpu->fault_info.flags = flags;
+		gpu->fault_info.type  = type;
+		gpu->fault_info.block = block;
+
+		kthread_queue_work(gpu->worker, &gpu->fault_work);
+	}
+
+	return 0;
+}
+
 int adreno_get_param(struct msm_gpu *gpu, struct msm_file_private *ctx,
 		     uint32_t param, uint64_t *value, uint32_t *len)
 {
@@ -503,16 +563,9 @@ struct drm_gem_object *adreno_fw_create_bo(struct msm_gpu *gpu,
 
 int adreno_hw_init(struct msm_gpu *gpu)
 {
-	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
-	int ret, i;
-
 	VERB("%s", gpu->name);
 
-	ret = adreno_load_fw(adreno_gpu);
-	if (ret)
-		return ret;
-
-	for (i = 0; i < gpu->nr_rings; i++) {
+	for (int i = 0; i < gpu->nr_rings; i++) {
 		struct msm_ringbuffer *ring = gpu->rb[i];
 
 		if (!ring)
@@ -922,73 +975,46 @@ void adreno_wait_ring(struct msm_ringbuffer *ring, uint32_t ndwords)
 			ring->id);
 }
 
-/* Get legacy powerlevels from qcom,gpu-pwrlevels and populate the opp table */
-static int adreno_get_legacy_pwrlevels(struct device *dev)
-{
-	struct device_node *child, *node;
-	int ret;
-
-	node = of_get_compatible_child(dev->of_node, "qcom,gpu-pwrlevels");
-	if (!node) {
-		DRM_DEV_DEBUG(dev, "Could not find the GPU powerlevels\n");
-		return -ENXIO;
-	}
-
-	for_each_child_of_node(node, child) {
-		unsigned int val;
-
-		ret = of_property_read_u32(child, "qcom,gpu-freq", &val);
-		if (ret)
-			continue;
-
-		/*
-		 * Skip the intentionally bogus clock value found at the bottom
-		 * of most legacy frequency tables
-		 */
-		if (val != 27000000)
-			dev_pm_opp_add(dev, val, 0);
-	}
-
-	of_node_put(node);
-
-	return 0;
-}
-
-static void adreno_get_pwrlevels(struct device *dev,
+static int adreno_get_pwrlevels(struct device *dev,
 		struct msm_gpu *gpu)
 {
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 	unsigned long freq = ULONG_MAX;
 	struct dev_pm_opp *opp;
 	int ret;
 
 	gpu->fast_rate = 0;
 
-	/* You down with OPP? */
-	if (!of_find_property(dev->of_node, "operating-points-v2", NULL))
-		ret = adreno_get_legacy_pwrlevels(dev);
-	else {
-		ret = devm_pm_opp_of_add_table(dev);
-		if (ret)
-			DRM_DEV_ERROR(dev, "Unable to set the OPP table\n");
-	}
-
-	if (!ret) {
-		/* Find the fastest defined rate */
-		opp = dev_pm_opp_find_freq_floor(dev, &freq);
-		if (!IS_ERR(opp)) {
-			gpu->fast_rate = freq;
-			dev_pm_opp_put(opp);
+	/* devm_pm_opp_of_add_table may error out but will still create an OPP table */
+	ret = devm_pm_opp_of_add_table(dev);
+	if (ret == -ENODEV) {
+		/* Special cases for ancient hw with ancient DT bindings */
+		if (adreno_is_a2xx(adreno_gpu)) {
+			dev_warn(dev, "Unable to find the OPP table. Falling back to 200 MHz.\n");
+			dev_pm_opp_add(dev, 200000000, 0);
+		} else if (adreno_is_a320(adreno_gpu)) {
+			dev_warn(dev, "Unable to find the OPP table. Falling back to 450 MHz.\n");
+			dev_pm_opp_add(dev, 450000000, 0);
+		} else {
+			DRM_DEV_ERROR(dev, "Unable to find the OPP table\n");
+			return -ENODEV;
 		}
+	} else if (ret) {
+		DRM_DEV_ERROR(dev, "Unable to set the OPP table\n");
+		return ret;
 	}
 
-	if (!gpu->fast_rate) {
-		dev_warn(dev,
-			"Could not find a clock rate. Using a reasonable default\n");
-		/* Pick a suitably safe clock speed for any target */
-		gpu->fast_rate = 200000000;
-	}
+	/* Find the fastest defined rate */
+	opp = dev_pm_opp_find_freq_floor(dev, &freq);
+	if (IS_ERR(opp))
+		return PTR_ERR(opp);
+
+	gpu->fast_rate = freq;
+	dev_pm_opp_put(opp);
 
 	DBG("fast_rate=%u, slow_rate=27000000", gpu->fast_rate);
+
+	return 0;
 }
 
 int adreno_gpu_ocmem_init(struct device *dev, struct adreno_gpu *adreno_gpu,
@@ -1046,6 +1072,24 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	struct adreno_rev *rev = &config->rev;
 	const char *gpu_name;
 	u32 speedbin;
+	int ret;
+
+	/* Only handle the core clock when GMU is not in use */
+	if (config->rev.core < 6) {
+		/*
+		 * This can only be done before devm_pm_opp_of_add_table(), or
+		 * dev_pm_opp_set_config() will WARN_ON()
+		 */
+		if (IS_ERR(devm_clk_get(dev, "core"))) {
+			/*
+			 * If "core" is absent, go for the legacy clock name.
+			 * If we got this far in probing, it's a given one of
+			 * them exists.
+			 */
+			devm_pm_opp_set_clkname(dev, "core_clk");
+		} else
+			devm_pm_opp_set_clkname(dev, "core");
+	}
 
 	adreno_gpu->funcs = funcs;
 	adreno_gpu->info = adreno_info(config->rev);
@@ -1070,7 +1114,9 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 
 	adreno_gpu_config.nr_rings = nr_rings;
 
-	adreno_get_pwrlevels(dev, gpu);
+	ret = adreno_get_pwrlevels(dev, gpu);
+	if (ret)
+		return ret;
 
 	pm_runtime_set_autosuspend_delay(dev,
 		adreno_gpu->info->inactive_period);
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index b4f9b1343d63..f62612a5c70f 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -341,6 +341,10 @@ adreno_iommu_create_address_space(struct msm_gpu *gpu,
 				  struct platform_device *pdev,
 				  unsigned long quirks);
 
+int adreno_fault_handler(struct msm_gpu *gpu, unsigned long iova, int flags,
+			 struct adreno_smmu_fault_info *info, const char *block,
+			 u32 scratch[4]);
+
 int adreno_read_speedbin(struct device *dev, u32 *speedbin);
 
 /*
diff --git a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
index 7aecf920f9b9..8a4a2d161a29 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
@@ -8,21 +8,21 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a2xx.xml                (  90810 bytes, from 2021-06-21 15:24:24)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  14609 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  69086 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2021-11-24 23:05:10)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113358 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149512 bytes, from 2022-01-31 23:06:21)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 184954 bytes, from 2022-03-03 16:41:33)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11331 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   6038 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2924 bytes, from 2021-07-22 15:21:56)
-
-Copyright (C) 2013-2022 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno.xml                     (    594 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml        (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a2xx.xml                (  91929 bytes, from 2023-02-28 23:52:27)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml       (  15434 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml          (  74995 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a3xx.xml                (  84231 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a4xx.xml                ( 113474 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a5xx.xml                ( 149590 bytes, from 2023-02-14 19:37:12)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx.xml                ( 198949 bytes, from 2023-03-20 18:06:23)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml            (  11404 bytes, from 2023-03-10 18:32:53)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/ocmem.xml               (   1773 bytes, from 2022-08-02 16:38:43)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_control_regs.xml (   9055 bytes, from 2023-03-10 18:32:52)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pipe_regs.xml    (   2976 bytes, from 2023-03-10 18:32:52)
+
+Copyright (C) 2013-2023 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -76,6 +76,10 @@ enum vgt_event_type {
 	VS_FETCH_DONE = 27,
 	FACENESS_FLUSH = 28,
 	WT_DONE_TS = 8,
+	START_FRAGMENT_CTRS = 13,
+	STOP_FRAGMENT_CTRS = 14,
+	START_COMPUTE_CTRS = 15,
+	STOP_COMPUTE_CTRS = 16,
 	FLUSH_SO_0 = 17,
 	FLUSH_SO_1 = 18,
 	FLUSH_SO_2 = 19,
@@ -86,7 +90,7 @@ enum vgt_event_type {
 	PC_CCU_FLUSH_DEPTH_TS = 28,
 	PC_CCU_FLUSH_COLOR_TS = 29,
 	BLIT = 30,
-	UNK_25 = 37,
+	LRZ_CLEAR = 37,
 	LRZ_FLUSH = 38,
 	BLIT_OP_FILL_2D = 39,
 	BLIT_OP_COPY_2D = 40,
@@ -95,6 +99,20 @@ enum vgt_event_type {
 	UNK_2C = 44,
 	UNK_2D = 45,
 	CACHE_INVALIDATE = 49,
+	LABEL = 63,
+	CCU_INVALIDATE_DEPTH = 24,
+	CCU_INVALIDATE_COLOR = 25,
+	CCU_RESOLVE_CLEAN = 26,
+	CCU_FLUSH_DEPTH = 28,
+	CCU_FLUSH_COLOR = 29,
+	CCU_RESOLVE = 30,
+	CCU_END_RESOLVE_GROUP = 31,
+	CCU_CLEAN_DEPTH = 32,
+	CCU_CLEAN_COLOR = 33,
+	CACHE_RESET = 48,
+	CACHE_CLEAN = 49,
+	CACHE_FLUSH7 = 50,
+	CACHE_INVALIDATE7 = 51,
 };
 
 enum pc_di_primtype {
@@ -290,6 +308,9 @@ enum adreno_pm4_type3_packets {
 	IN_INCR_UPDT_CONST = 86,
 	IN_INCR_UPDT_INSTR = 87,
 	PKT4 = 4,
+	IN_IB_END = 10,
+	IN_GMU_INTERRUPT = 11,
+	IN_PREEMPT = 15,
 	CP_SCRATCH_WRITE = 76,
 	CP_REG_TO_MEM_OFFSET_MEM = 116,
 	CP_REG_TO_MEM_OFFSET_REG = 114,
@@ -297,10 +318,20 @@ enum adreno_pm4_type3_packets {
 	CP_WAIT_TWO_REGS = 112,
 	CP_MEMCPY = 117,
 	CP_SET_BIN_DATA5_OFFSET = 46,
+	CP_CONTEXT_SWITCH = 84,
 	CP_SET_CTXSWITCH_IB = 85,
 	CP_REG_WRITE = 109,
 	CP_START_BIN = 80,
 	CP_END_BIN = 81,
+	CP_PREEMPT_DISABLE = 108,
+	CP_WAIT_TIMESTAMP = 20,
+	CP_THREAD_CONTROL = 23,
+	CP_CONTEXT_REG_BUNCH2 = 93,
+	CP_UNK15 = 21,
+	CP_UNK16 = 22,
+	CP_UNK18 = 24,
+	CP_UNK1B = 27,
+	CP_UNK49 = 73,
 };
 
 enum adreno_state_block {
@@ -480,6 +511,13 @@ enum reg_tracker {
 	TRACK_CNTL_REG = 1,
 	TRACK_RENDER_CNTL = 2,
 	UNK_EVENT_WRITE = 4,
+	TRACK_LRZ = 8,
+};
+
+enum cp_thread {
+	CP_SET_THREAD_BR = 1,
+	CP_SET_THREAD_BV = 2,
+	CP_SET_THREAD_BOTH = 3,
 };
 
 #define REG_CP_LOAD_STATE_0					0x00000000
@@ -1256,6 +1294,10 @@ static inline uint32_t CP_SET_BIN_DATA5_6_BIN_PRIM_STRM_HI(uint32_t val)
 	return ((val) << CP_SET_BIN_DATA5_6_BIN_PRIM_STRM_HI__SHIFT) & CP_SET_BIN_DATA5_6_BIN_PRIM_STRM_HI__MASK;
 }
 
+#define REG_CP_SET_BIN_DATA5_7					0x00000007
+
+#define REG_CP_SET_BIN_DATA5_9					0x00000009
+
 #define REG_CP_SET_BIN_DATA5_OFFSET_0				0x00000000
 #define CP_SET_BIN_DATA5_OFFSET_0_VSC_SIZE__MASK		0x003f0000
 #define CP_SET_BIN_DATA5_OFFSET_0_VSC_SIZE__SHIFT		16
@@ -2202,7 +2244,18 @@ static inline uint32_t A6XX_CP_REG_TEST_0_BIT(uint32_t val)
 {
 	return ((val) << A6XX_CP_REG_TEST_0_BIT__SHIFT) & A6XX_CP_REG_TEST_0_BIT__MASK;
 }
-#define A6XX_CP_REG_TEST_0_WAIT_FOR_ME				0x02000000
+#define A6XX_CP_REG_TEST_0_SKIP_WAIT_FOR_ME			0x02000000
+#define A6XX_CP_REG_TEST_0_PRED_BIT__MASK			0x7c000000
+#define A6XX_CP_REG_TEST_0_PRED_BIT__SHIFT			26
+static inline uint32_t A6XX_CP_REG_TEST_0_PRED_BIT(uint32_t val)
+{
+	return ((val) << A6XX_CP_REG_TEST_0_PRED_BIT__SHIFT) & A6XX_CP_REG_TEST_0_PRED_BIT__MASK;
+}
+#define A6XX_CP_REG_TEST_0_PRED_UPDATE				0x80000000
+
+#define REG_A6XX_CP_REG_TEST_PRED_MASK				0x00000001
+
+#define REG_A6XX_CP_REG_TEST_PRED_VAL				0x00000002
 
 #define REG_CP_COND_REG_EXEC_0					0x00000000
 #define CP_COND_REG_EXEC_0_REG0__MASK				0x0003ffff
@@ -2211,6 +2264,12 @@ static inline uint32_t CP_COND_REG_EXEC_0_REG0(uint32_t val)
 {
 	return ((val) << CP_COND_REG_EXEC_0_REG0__SHIFT) & CP_COND_REG_EXEC_0_REG0__MASK;
 }
+#define CP_COND_REG_EXEC_0_PRED_BIT__MASK			0x007c0000
+#define CP_COND_REG_EXEC_0_PRED_BIT__SHIFT			18
+static inline uint32_t CP_COND_REG_EXEC_0_PRED_BIT(uint32_t val)
+{
+	return ((val) << CP_COND_REG_EXEC_0_PRED_BIT__SHIFT) & CP_COND_REG_EXEC_0_PRED_BIT__MASK;
+}
 #define CP_COND_REG_EXEC_0_BINNING				0x02000000
 #define CP_COND_REG_EXEC_0_GMEM					0x04000000
 #define CP_COND_REG_EXEC_0_SYSMEM				0x08000000
@@ -2308,13 +2367,17 @@ static inline uint32_t CP_SET_CTXSWITCH_IB_2_TYPE(enum ctxswitch_ib val)
 }
 
 #define REG_CP_REG_WRITE_0					0x00000000
-#define CP_REG_WRITE_0_TRACKER__MASK				0x00000007
+#define CP_REG_WRITE_0_TRACKER__MASK				0x0000000f
 #define CP_REG_WRITE_0_TRACKER__SHIFT				0
 static inline uint32_t CP_REG_WRITE_0_TRACKER(enum reg_tracker val)
 {
 	return ((val) << CP_REG_WRITE_0_TRACKER__SHIFT) & CP_REG_WRITE_0_TRACKER__MASK;
 }
 
+#define REG_CP_REG_WRITE_1					0x00000001
+
+#define REG_CP_REG_WRITE_2					0x00000002
+
 #define REG_CP_SMMU_TABLE_UPDATE_0				0x00000000
 #define CP_SMMU_TABLE_UPDATE_0_TTBR0_LO__MASK			0xffffffff
 #define CP_SMMU_TABLE_UPDATE_0_TTBR0_LO__SHIFT			0
@@ -2361,5 +2424,21 @@ static inline uint32_t CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK(uint32_t val)
 
 #define REG_CP_START_BIN_BODY_DWORDS				0x00000004
 
+#define REG_CP_WAIT_TIMESTAMP_0					0x00000000
+
+#define REG_CP_WAIT_TIMESTAMP_ADDR				0x00000001
+
+#define REG_CP_WAIT_TIMESTAMP_TIMESTAMP				0x00000003
+
+#define REG_CP_THREAD_CONTROL_0					0x00000000
+#define CP_THREAD_CONTROL_0_THREAD__MASK			0x00000003
+#define CP_THREAD_CONTROL_0_THREAD__SHIFT			0
+static inline uint32_t CP_THREAD_CONTROL_0_THREAD(enum cp_thread val)
+{
+	return ((val) << CP_THREAD_CONTROL_0_THREAD__SHIFT) & CP_THREAD_CONTROL_0_THREAD__MASK;
+}
+#define CP_THREAD_CONTROL_0_CONCURRENT_BIN_DISABLE		0x08000000
+#define CP_THREAD_CONTROL_0_SYNC_THREADS			0x80000000
+
 
 #endif /* ADRENO_PM4_XML */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h
new file mode 100644
index 000000000000..2b3ae84057df
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h
@@ -0,0 +1,210 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_3_0_MSM8998_H
+#define _DPU_3_0_MSM8998_H
+
+static const struct dpu_caps msm8998_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0x7,
+	.qseed_type = DPU_SSPP_SCALER_QSEED3,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.has_3d_merge = true,
+	.max_linewidth = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+	.max_hdeci_exp = MAX_HORZ_DECIMATION,
+	.max_vdeci_exp = MAX_VERT_DECIMATION,
+};
+
+static const struct dpu_ubwc_cfg msm8998_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_10,
+	.highest_bank_bit = 0x2,
+};
+
+static const struct dpu_mdp_cfg msm8998_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x458,
+	.features = 0,
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG1] = { .reg_off = 0x2b4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG2] = { .reg_off = 0x2bc, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG3] = { .reg_off = 0x2c4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2c4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA3] = { .reg_off = 0x2c4, .bit_off = 12 },
+	.clk_ctrls[DPU_CLK_CTRL_CURSOR0] = { .reg_off = 0x3a8, .bit_off = 15 },
+	.clk_ctrls[DPU_CLK_CTRL_CURSOR1] = { .reg_off = 0x3b0, .bit_off = 15 },
+	},
+};
+
+static const struct dpu_ctl_cfg msm8998_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x1000, .len = 0x94,
+	.features = BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x1200, .len = 0x94,
+	.features = 0,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x1400, .len = 0x94,
+	.features = BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x1600, .len = 0x94,
+	.features = 0,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+	{
+	.name = "ctl_4", .id = CTL_4,
+	.base = 0x1800, .len = 0x94,
+	.features = 0,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
+	},
+};
+
+static const struct dpu_sspp_cfg msm8998_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x1ac, VIG_MSM8998_MASK,
+		msm8998_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, 0x1ac, VIG_MSM8998_MASK,
+		msm8998_vig_sblk_1, 4, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
+	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, 0x1ac, VIG_MSM8998_MASK,
+		msm8998_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
+	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, 0x1ac, VIG_MSM8998_MASK,
+		msm8998_vig_sblk_3, 12, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x1ac, DMA_MSM8998_MASK,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x1ac, DMA_MSM8998_MASK,
+		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x1ac, DMA_CURSOR_MSM8998_MASK,
+		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, 0x1ac, DMA_CURSOR_MSM8998_MASK,
+		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
+};
+
+static const struct dpu_lm_cfg msm8998_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_MSM8998_MASK,
+		&msm8998_lm_sblk, PINGPONG_0, LM_2, DSPP_0),
+	LM_BLK("lm_1", LM_1, 0x45000, MIXER_MSM8998_MASK,
+		&msm8998_lm_sblk, PINGPONG_1, LM_5, DSPP_1),
+	LM_BLK("lm_2", LM_2, 0x46000, MIXER_MSM8998_MASK,
+		&msm8998_lm_sblk, PINGPONG_2, LM_0, 0),
+	LM_BLK("lm_3", LM_3, 0x47000, MIXER_MSM8998_MASK,
+		&msm8998_lm_sblk, PINGPONG_MAX, 0, 0),
+	LM_BLK("lm_4", LM_4, 0x48000, MIXER_MSM8998_MASK,
+		&msm8998_lm_sblk, PINGPONG_MAX, 0, 0),
+	LM_BLK("lm_5", LM_5, 0x49000, MIXER_MSM8998_MASK,
+		&msm8998_lm_sblk, PINGPONG_3, LM_1, 0),
+};
+
+static const struct dpu_pingpong_cfg msm8998_pp[] = {
+	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x70000, 0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
+	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x70800, 0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
+	PP_BLK("pingpong_2", PINGPONG_2, 0x71000, 0, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
+	PP_BLK("pingpong_3", PINGPONG_3, 0x71800, 0, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
+};
+
+static const struct dpu_dspp_cfg msm8998_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_MSM8998_MASK,
+		 &msm8998_dspp_sblk),
+	DSPP_BLK("dspp_1", DSPP_1, 0x56000, DSPP_MSM8998_MASK,
+		 &msm8998_dspp_sblk),
+};
+
+static const struct dpu_intf_cfg msm8998_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x6a000, 0x280, INTF_DP, 0, 25, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	INTF_BLK("intf_1", INTF_1, 0x6a800, 0x280, INTF_DSI, 0, 25, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+	INTF_BLK("intf_2", INTF_2, 0x6b000, 0x280, INTF_DSI, 1, 25, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
+	INTF_BLK("intf_3", INTF_3, 0x6b800, 0x280, INTF_HDMI, 0, 25, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
+};
+
+static const struct dpu_perf_cfg msm8998_perf_data = {
+	.max_bw_low = 6700000,
+	.max_bw_high = 6700000,
+	.min_core_ib = 2400000,
+	.min_llcc_ib = 800000,
+	.min_dram_ib = 800000,
+	.undersized_prefill_lines = 2,
+	.xtra_prefill_lines = 2,
+	.dest_scale_prefill_lines = 3,
+	.macrotile_prefill_lines = 4,
+	.yuv_nv12_prefill_lines = 8,
+	.linear_prefill_lines = 1,
+	.downscaling_prefill_lines = 1,
+	.amortizable_threshold = 25,
+	.min_prefill_lines = 25,
+	.danger_lut_tbl = {0xf, 0xffff, 0x0},
+	.safe_lut_tbl = {0xfffc, 0xff00, 0xffff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(msm8998_qos_linear),
+		.entries = msm8998_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(msm8998_qos_macrotile),
+		.entries = msm8998_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(msm8998_qos_nrt),
+		.entries = msm8998_qos_nrt
+		},
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 200,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_msm8998_cfg = {
+	.caps = &msm8998_dpu_caps,
+	.ubwc = &msm8998_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(msm8998_mdp),
+	.mdp = msm8998_mdp,
+	.ctl_count = ARRAY_SIZE(msm8998_ctl),
+	.ctl = msm8998_ctl,
+	.sspp_count = ARRAY_SIZE(msm8998_sspp),
+	.sspp = msm8998_sspp,
+	.mixer_count = ARRAY_SIZE(msm8998_lm),
+	.mixer = msm8998_lm,
+	.dspp_count = ARRAY_SIZE(msm8998_dspp),
+	.dspp = msm8998_dspp,
+	.pingpong_count = ARRAY_SIZE(msm8998_pp),
+	.pingpong = msm8998_pp,
+	.intf_count = ARRAY_SIZE(msm8998_intf),
+	.intf = msm8998_intf,
+	.vbif_count = ARRAY_SIZE(msm8998_vbif),
+	.vbif = msm8998_vbif,
+	.reg_dma_count = 0,
+	.perf = &msm8998_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_INTR) | \
+		     BIT(MDP_INTF1_INTR) | \
+		     BIT(MDP_INTF2_INTR) | \
+		     BIT(MDP_INTF3_INTR) | \
+		     BIT(MDP_INTF4_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_4_0_sdm845.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_4_0_sdm845.h
new file mode 100644
index 000000000000..ceca741e93c9
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_4_0_sdm845.h
@@ -0,0 +1,210 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_4_0_SDM845_H
+#define _DPU_4_0_SDM845_H
+
+static const struct dpu_caps sdm845_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0xb,
+	.qseed_type = DPU_SSPP_SCALER_QSEED3,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.has_3d_merge = true,
+	.max_linewidth = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+	.max_hdeci_exp = MAX_HORZ_DECIMATION,
+	.max_vdeci_exp = MAX_VERT_DECIMATION,
+};
+
+static const struct dpu_ubwc_cfg sdm845_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_20,
+	.highest_bank_bit = 0x2,
+};
+
+static const struct dpu_mdp_cfg sdm845_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x45c,
+	.features = BIT(DPU_MDP_AUDIO_SELECT),
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG1] = { .reg_off = 0x2b4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG2] = { .reg_off = 0x2bc, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG3] = { .reg_off = 0x2c4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2bc, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA3] = { .reg_off = 0x2c4, .bit_off = 8 },
+	},
+};
+
+static const struct dpu_ctl_cfg sdm845_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x1000, .len = 0xe4,
+	.features = BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x1200, .len = 0xe4,
+	.features = BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x1400, .len = 0xe4,
+	.features = 0,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x1600, .len = 0xe4,
+	.features = 0,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+	{
+	.name = "ctl_4", .id = CTL_4,
+	.base = 0x1800, .len = 0xe4,
+	.features = 0,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
+	},
+};
+
+static const struct dpu_sspp_cfg sdm845_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x1c8, VIG_SDM845_MASK_SDMA,
+		sdm845_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, 0x1c8, VIG_SDM845_MASK_SDMA,
+		sdm845_vig_sblk_1, 4, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
+	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, 0x1c8, VIG_SDM845_MASK_SDMA,
+		sdm845_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
+	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, 0x1c8, VIG_SDM845_MASK_SDMA,
+		sdm845_vig_sblk_3, 12, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x1c8, DMA_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x1c8, DMA_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x1c8, DMA_CURSOR_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, 0x1c8, DMA_CURSOR_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
+};
+
+static const struct dpu_lm_cfg sdm845_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_0, LM_1, 0),
+	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_1, LM_0, 0),
+	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_2, LM_5, 0),
+	LM_BLK("lm_3", LM_3, 0x0, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_MAX, 0, 0),
+	LM_BLK("lm_4", LM_4, 0x0, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_MAX, 0, 0),
+	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_3, LM_2, 0),
+};
+
+static const struct dpu_pingpong_cfg sdm845_pp[] = {
+	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x70000, 0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
+	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x70800, 0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
+	PP_BLK("pingpong_2", PINGPONG_2, 0x71000, 0, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
+	PP_BLK("pingpong_3", PINGPONG_3, 0x71800, 0, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
+};
+
+static const struct dpu_dsc_cfg sdm845_dsc[] = {
+	DSC_BLK("dsc_0", DSC_0, 0x80000, 0),
+	DSC_BLK("dsc_1", DSC_1, 0x80400, 0),
+	DSC_BLK("dsc_2", DSC_2, 0x80800, 0),
+	DSC_BLK("dsc_3", DSC_3, 0x80c00, 0),
+};
+
+static const struct dpu_intf_cfg sdm845_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x6a000, 0x280, INTF_DP, 0, 24, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	INTF_BLK("intf_1", INTF_1, 0x6a800, 0x280, INTF_DSI, 0, 24, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+	INTF_BLK("intf_2", INTF_2, 0x6b000, 0x280, INTF_DSI, 1, 24, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
+	INTF_BLK("intf_3", INTF_3, 0x6b800, 0x280, INTF_DP, 1, 24, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
+};
+
+static const struct dpu_perf_cfg sdm845_perf_data = {
+	.max_bw_low = 6800000,
+	.max_bw_high = 6800000,
+	.min_core_ib = 2400000,
+	.min_llcc_ib = 800000,
+	.min_dram_ib = 800000,
+	.undersized_prefill_lines = 2,
+	.xtra_prefill_lines = 2,
+	.dest_scale_prefill_lines = 3,
+	.macrotile_prefill_lines = 4,
+	.yuv_nv12_prefill_lines = 8,
+	.linear_prefill_lines = 1,
+	.downscaling_prefill_lines = 1,
+	.amortizable_threshold = 25,
+	.min_prefill_lines = 24,
+	.danger_lut_tbl = {0xf, 0xffff, 0x0},
+	.safe_lut_tbl = {0xfff0, 0xf000, 0xffff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sdm845_qos_linear),
+		.entries = sdm845_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sdm845_qos_macrotile),
+		.entries = sdm845_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sdm845_qos_nrt),
+		.entries = sdm845_qos_nrt
+		},
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sdm845_cfg = {
+	.caps = &sdm845_dpu_caps,
+	.ubwc = &sdm845_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sdm845_mdp),
+	.mdp = sdm845_mdp,
+	.ctl_count = ARRAY_SIZE(sdm845_ctl),
+	.ctl = sdm845_ctl,
+	.sspp_count = ARRAY_SIZE(sdm845_sspp),
+	.sspp = sdm845_sspp,
+	.mixer_count = ARRAY_SIZE(sdm845_lm),
+	.mixer = sdm845_lm,
+	.pingpong_count = ARRAY_SIZE(sdm845_pp),
+	.pingpong = sdm845_pp,
+	.dsc_count = ARRAY_SIZE(sdm845_dsc),
+	.dsc = sdm845_dsc,
+	.intf_count = ARRAY_SIZE(sdm845_intf),
+	.intf = sdm845_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.reg_dma_count = 1,
+	.dma_cfg = &sdm845_regdma,
+	.perf = &sdm845_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_INTR) | \
+		     BIT(MDP_INTF1_INTR) | \
+		     BIT(MDP_INTF2_INTR) | \
+		     BIT(MDP_INTF3_INTR) | \
+		     BIT(MDP_AD4_0_INTR) | \
+		     BIT(MDP_AD4_1_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_0_sm8150.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_0_sm8150.h
new file mode 100644
index 000000000000..282d410269ff
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_0_sm8150.h
@@ -0,0 +1,237 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_5_0_SM8150_H
+#define _DPU_5_0_SM8150_H
+
+static const struct dpu_caps sm8150_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0xb,
+	.qseed_type = DPU_SSPP_SCALER_QSEED3,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.has_3d_merge = true,
+	.max_linewidth = 4096,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+	.max_hdeci_exp = MAX_HORZ_DECIMATION,
+	.max_vdeci_exp = MAX_VERT_DECIMATION,
+};
+
+static const struct dpu_ubwc_cfg sm8150_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_30,
+	.highest_bank_bit = 0x2,
+};
+
+static const struct dpu_mdp_cfg sm8150_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x45c,
+	.features = BIT(DPU_MDP_AUDIO_SELECT),
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG1] = { .reg_off = 0x2b4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG2] = { .reg_off = 0x2bc, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG3] = { .reg_off = 0x2c4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2bc, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA3] = { .reg_off = 0x2c4, .bit_off = 8 },
+	},
+};
+
+/* FIXME: get rid of DPU_CTL_SPLIT_DISPLAY in favour of proper ACTIVE_CTL support */
+static const struct dpu_ctl_cfg sm8150_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x1000, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x1200, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x1400, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x1600, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+	{
+	.name = "ctl_4", .id = CTL_4,
+	.base = 0x1800, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
+	},
+	{
+	.name = "ctl_5", .id = CTL_5,
+	.base = 0x1a00, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
+	},
+};
+
+static const struct dpu_sspp_cfg sm8150_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x1f0, VIG_SDM845_MASK,
+		sdm845_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, 0x1f0, VIG_SDM845_MASK,
+		sdm845_vig_sblk_1, 4, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
+	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, 0x1f0, VIG_SDM845_MASK,
+		sdm845_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
+	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, 0x1f0, VIG_SDM845_MASK,
+		sdm845_vig_sblk_3, 12, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x1f0, DMA_SDM845_MASK,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x1f0, DMA_SDM845_MASK,
+		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x1f0, DMA_CURSOR_SDM845_MASK,
+		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, 0x1f0, DMA_CURSOR_SDM845_MASK,
+		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
+};
+
+static const struct dpu_lm_cfg sm8150_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
+	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_1, LM_0, DSPP_1),
+	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_2, LM_3, 0),
+	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_3, LM_2, 0),
+	LM_BLK("lm_4", LM_4, 0x48000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_4, LM_5, 0),
+	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_5, LM_4, 0),
+};
+
+static const struct dpu_dspp_cfg sm8150_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_1", DSPP_1, 0x56000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_2", DSPP_2, 0x58000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_3", DSPP_3, 0x5a000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+};
+
+static const struct dpu_pingpong_cfg sm8150_pp[] = {
+	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x70000, MERGE_3D_0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
+	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x70800, MERGE_3D_0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
+	PP_BLK("pingpong_2", PINGPONG_2, 0x71000, MERGE_3D_1, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
+	PP_BLK("pingpong_3", PINGPONG_3, 0x71800, MERGE_3D_1, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
+	PP_BLK("pingpong_4", PINGPONG_4, 0x72000, MERGE_3D_2, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30),
+			-1),
+	PP_BLK("pingpong_5", PINGPONG_5, 0x72800, MERGE_3D_2, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31),
+			-1),
+};
+
+static const struct dpu_merge_3d_cfg sm8150_merge_3d[] = {
+	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x83000),
+	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x83100),
+	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x83200),
+};
+
+static const struct dpu_dsc_cfg sm8150_dsc[] = {
+	DSC_BLK("dsc_0", DSC_0, 0x80000, BIT(DPU_DSC_OUTPUT_CTRL)),
+	DSC_BLK("dsc_1", DSC_1, 0x80400, BIT(DPU_DSC_OUTPUT_CTRL)),
+	DSC_BLK("dsc_2", DSC_2, 0x80800, BIT(DPU_DSC_OUTPUT_CTRL)),
+	DSC_BLK("dsc_3", DSC_3, 0x80c00, BIT(DPU_DSC_OUTPUT_CTRL)),
+};
+
+static const struct dpu_intf_cfg sm8150_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x6a000, 0x280, INTF_DP, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	INTF_BLK("intf_1", INTF_1, 0x6a800, 0x2bc, INTF_DSI, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+	INTF_BLK("intf_2", INTF_2, 0x6b000, 0x2bc, INTF_DSI, 1, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
+	INTF_BLK("intf_3", INTF_3, 0x6b800, 0x280, INTF_DP, 1, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
+};
+
+static const struct dpu_perf_cfg sm8150_perf_data = {
+	.max_bw_low = 12800000,
+	.max_bw_high = 12800000,
+	.min_core_ib = 2400000,
+	.min_llcc_ib = 800000,
+	.min_dram_ib = 800000,
+	.min_prefill_lines = 24,
+	.danger_lut_tbl = {0xf, 0xffff, 0x0},
+	.safe_lut_tbl = {0xfff8, 0xf000, 0xffff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sm8150_qos_linear),
+		.entries = sm8150_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+		/* TODO: macrotile-qseed is different from macrotile */
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sm8150_cfg = {
+	.caps = &sm8150_dpu_caps,
+	.ubwc = &sm8150_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sm8150_mdp),
+	.mdp = sm8150_mdp,
+	.ctl_count = ARRAY_SIZE(sm8150_ctl),
+	.ctl = sm8150_ctl,
+	.sspp_count = ARRAY_SIZE(sm8150_sspp),
+	.sspp = sm8150_sspp,
+	.mixer_count = ARRAY_SIZE(sm8150_lm),
+	.mixer = sm8150_lm,
+	.dspp_count = ARRAY_SIZE(sm8150_dspp),
+	.dspp = sm8150_dspp,
+	.dsc_count = ARRAY_SIZE(sm8150_dsc),
+	.dsc = sm8150_dsc,
+	.pingpong_count = ARRAY_SIZE(sm8150_pp),
+	.pingpong = sm8150_pp,
+	.merge_3d_count = ARRAY_SIZE(sm8150_merge_3d),
+	.merge_3d = sm8150_merge_3d,
+	.intf_count = ARRAY_SIZE(sm8150_intf),
+	.intf = sm8150_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.reg_dma_count = 1,
+	.dma_cfg = &sm8150_regdma,
+	.perf = &sm8150_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_INTR) | \
+		     BIT(MDP_INTF1_INTR) | \
+		     BIT(MDP_INTF2_INTR) | \
+		     BIT(MDP_INTF3_INTR) | \
+		     BIT(MDP_AD4_0_INTR) | \
+		     BIT(MDP_AD4_1_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_1_sc8180x.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_1_sc8180x.h
new file mode 100644
index 000000000000..c57400265f28
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_5_1_sc8180x.h
@@ -0,0 +1,217 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_5_1_SC8180X_H
+#define _DPU_5_1_SC8180X_H
+
+static const struct dpu_caps sc8180x_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0xb,
+	.qseed_type = DPU_SSPP_SCALER_QSEED3,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.has_3d_merge = true,
+	.max_linewidth = 4096,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+	.max_hdeci_exp = MAX_HORZ_DECIMATION,
+	.max_vdeci_exp = MAX_VERT_DECIMATION,
+};
+
+static const struct dpu_ubwc_cfg sc8180x_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_30,
+	.highest_bank_bit = 0x3,
+};
+
+static const struct dpu_mdp_cfg sc8180x_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x45c,
+	.features = BIT(DPU_MDP_AUDIO_SELECT),
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG1] = { .reg_off = 0x2b4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG2] = { .reg_off = 0x2bc, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG3] = { .reg_off = 0x2c4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2bc, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA3] = { .reg_off = 0x2c4, .bit_off = 8 },
+	},
+};
+
+static const struct dpu_ctl_cfg sc8180x_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x1000, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x1200, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x1400, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x1600, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+	{
+	.name = "ctl_4", .id = CTL_4,
+	.base = 0x1800, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
+	},
+	{
+	.name = "ctl_5", .id = CTL_5,
+	.base = 0x1a00, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
+	},
+};
+
+static const struct dpu_sspp_cfg sc8180x_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x1f0, VIG_SDM845_MASK,
+		sdm845_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, 0x1f0, VIG_SDM845_MASK,
+		sdm845_vig_sblk_1, 4, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
+	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, 0x1f0, VIG_SDM845_MASK,
+		sdm845_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
+	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, 0x1f0, VIG_SDM845_MASK,
+		sdm845_vig_sblk_3, 12, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x1f0, DMA_SDM845_MASK,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x1f0, DMA_SDM845_MASK,
+		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x1f0, DMA_CURSOR_SDM845_MASK,
+		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, 0x1f0, DMA_CURSOR_SDM845_MASK,
+		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
+};
+
+static const struct dpu_lm_cfg sc8180x_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_0, LM_1, 0),
+	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_1, LM_0, 0),
+	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_2, LM_3, 0),
+	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_3, LM_2, 0),
+	LM_BLK("lm_4", LM_4, 0x48000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_4, LM_5, 0),
+	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_5, LM_4, 0),
+};
+
+static const struct dpu_pingpong_cfg sc8180x_pp[] = {
+	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x70000, MERGE_3D_0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
+	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x70800, MERGE_3D_0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
+	PP_BLK("pingpong_2", PINGPONG_2, 0x71000, MERGE_3D_1, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
+	PP_BLK("pingpong_3", PINGPONG_3, 0x71800, MERGE_3D_1, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
+	PP_BLK("pingpong_4", PINGPONG_4, 0x72000, MERGE_3D_2, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30),
+			-1),
+	PP_BLK("pingpong_5", PINGPONG_5, 0x72800, MERGE_3D_2, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31),
+			-1),
+};
+
+static const struct dpu_merge_3d_cfg sc8180x_merge_3d[] = {
+	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x83000),
+	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x83100),
+	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x83200),
+};
+
+static const struct dpu_intf_cfg sc8180x_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x6a000, 0x280, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	INTF_BLK("intf_1", INTF_1, 0x6a800, 0x2bc, INTF_DSI, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+	INTF_BLK("intf_2", INTF_2, 0x6b000, 0x2bc, INTF_DSI, 1, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
+	/* INTF_3 is for MST, wired to INTF_DP 0 and 1, use dummy index until this is supported */
+	INTF_BLK("intf_3", INTF_3, 0x6b800, 0x280, INTF_DP, 999, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
+	INTF_BLK("intf_4", INTF_4, 0x6c000, 0x280, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 20, 21),
+	INTF_BLK("intf_5", INTF_5, 0x6c800, 0x280, INTF_DP, MSM_DP_CONTROLLER_2, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 22, 23),
+};
+
+static const struct dpu_perf_cfg sc8180x_perf_data = {
+	.max_bw_low = 9600000,
+	.max_bw_high = 9600000,
+	.min_core_ib = 2400000,
+	.min_llcc_ib = 800000,
+	.min_dram_ib = 800000,
+	.danger_lut_tbl = {0xf, 0xffff, 0x0},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
+		.entries = sc7180_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+		/* TODO: macrotile-qseed is different from macrotile */
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sc8180x_cfg = {
+	.caps = &sc8180x_dpu_caps,
+	.ubwc = &sc8180x_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sc8180x_mdp),
+	.mdp = sc8180x_mdp,
+	.ctl_count = ARRAY_SIZE(sc8180x_ctl),
+	.ctl = sc8180x_ctl,
+	.sspp_count = ARRAY_SIZE(sc8180x_sspp),
+	.sspp = sc8180x_sspp,
+	.mixer_count = ARRAY_SIZE(sc8180x_lm),
+	.mixer = sc8180x_lm,
+	.pingpong_count = ARRAY_SIZE(sc8180x_pp),
+	.pingpong = sc8180x_pp,
+	.merge_3d_count = ARRAY_SIZE(sc8180x_merge_3d),
+	.merge_3d = sc8180x_merge_3d,
+	.intf_count = ARRAY_SIZE(sc8180x_intf),
+	.intf = sc8180x_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.reg_dma_count = 1,
+	.dma_cfg = &sm8150_regdma,
+	.perf = &sc8180x_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_INTR) | \
+		     BIT(MDP_INTF1_INTR) | \
+		     BIT(MDP_INTF2_INTR) | \
+		     BIT(MDP_INTF3_INTR) | \
+		     BIT(MDP_INTF4_INTR) | \
+		     BIT(MDP_INTF5_INTR) | \
+		     BIT(MDP_AD4_0_INTR) | \
+		     BIT(MDP_AD4_1_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_0_sm8250.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_0_sm8250.h
new file mode 100644
index 000000000000..2c40229ea515
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_0_sm8250.h
@@ -0,0 +1,244 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_6_0_SM8250_H
+#define _DPU_6_0_SM8250_H
+
+static const struct dpu_caps sm8250_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0xb,
+	.qseed_type = DPU_SSPP_SCALER_QSEED4,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.has_3d_merge = true,
+	.max_linewidth = 4096,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+};
+
+static const struct dpu_ubwc_cfg sm8250_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_40,
+	.highest_bank_bit = 0x3, /* TODO: 2 for LP_DDR4 */
+	.ubwc_swizzle = 0x6,
+};
+
+static const struct dpu_mdp_cfg sm8250_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x494,
+	.features = 0,
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG1] = { .reg_off = 0x2b4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG2] = { .reg_off = 0x2bc, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG3] = { .reg_off = 0x2c4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2bc, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA3] = { .reg_off = 0x2c4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_REG_DMA] = { .reg_off = 0x2bc, .bit_off = 20 },
+	.clk_ctrls[DPU_CLK_CTRL_WB2] = { .reg_off = 0x3b8, .bit_off = 24 },
+	},
+};
+
+/* FIXME: get rid of DPU_CTL_SPLIT_DISPLAY in favour of proper ACTIVE_CTL support */
+static const struct dpu_ctl_cfg sm8250_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x1000, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x1200, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x1400, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x1600, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+	{
+	.name = "ctl_4", .id = CTL_4,
+	.base = 0x1800, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
+	},
+	{
+	.name = "ctl_5", .id = CTL_5,
+	.base = 0x1a00, .len = 0x1e0,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
+	},
+};
+
+static const struct dpu_sspp_cfg sm8250_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x1f8, VIG_SC7180_MASK_SDMA,
+		sm8250_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, 0x1f8, VIG_SC7180_MASK_SDMA,
+		sm8250_vig_sblk_1, 4, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
+	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, 0x1f8, VIG_SC7180_MASK_SDMA,
+		sm8250_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
+	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, 0x1f8, VIG_SC7180_MASK_SDMA,
+		sm8250_vig_sblk_3, 12, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x1f8, DMA_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x1f8, DMA_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x1f8, DMA_CURSOR_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, 0x1f8, DMA_CURSOR_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
+};
+
+static const struct dpu_lm_cfg sm8250_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
+	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_1, LM_0, DSPP_1),
+	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_2, LM_3, 0),
+	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_3, LM_2, 0),
+	LM_BLK("lm_4", LM_4, 0x48000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_4, LM_5, 0),
+	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_5, LM_4, 0),
+};
+
+static const struct dpu_dspp_cfg sm8250_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_1", DSPP_1, 0x56000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_2", DSPP_2, 0x58000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_3", DSPP_3, 0x5a000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+};
+
+static const struct dpu_pingpong_cfg sm8250_pp[] = {
+	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x70000, MERGE_3D_0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
+	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x70800, MERGE_3D_0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
+	PP_BLK("pingpong_2", PINGPONG_2, 0x71000, MERGE_3D_1, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
+	PP_BLK("pingpong_3", PINGPONG_3, 0x71800, MERGE_3D_1, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
+	PP_BLK("pingpong_4", PINGPONG_4, 0x72000, MERGE_3D_2, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30),
+			-1),
+	PP_BLK("pingpong_5", PINGPONG_5, 0x72800, MERGE_3D_2, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31),
+			-1),
+};
+
+static const struct dpu_merge_3d_cfg sm8250_merge_3d[] = {
+	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x83000),
+	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x83100),
+	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x83200),
+};
+
+static const struct dpu_dsc_cfg sm8250_dsc[] = {
+	DSC_BLK("dsc_0", DSC_0, 0x80000, BIT(DPU_DSC_OUTPUT_CTRL)),
+	DSC_BLK("dsc_1", DSC_1, 0x80400, BIT(DPU_DSC_OUTPUT_CTRL)),
+	DSC_BLK("dsc_2", DSC_2, 0x80800, BIT(DPU_DSC_OUTPUT_CTRL)),
+	DSC_BLK("dsc_3", DSC_3, 0x80c00, BIT(DPU_DSC_OUTPUT_CTRL)),
+};
+
+static const struct dpu_intf_cfg sm8250_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x6a000, 0x280, INTF_DP, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	INTF_BLK("intf_1", INTF_1, 0x6a800, 0x2c0, INTF_DSI, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+	INTF_BLK("intf_2", INTF_2, 0x6b000, 0x2c0, INTF_DSI, 1, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
+	INTF_BLK("intf_3", INTF_3, 0x6b800, 0x280, INTF_DP, 1, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
+};
+
+static const struct dpu_wb_cfg sm8250_wb[] = {
+	WB_BLK("wb_2", WB_2, 0x65000, WB_SM8250_MASK, DPU_CLK_CTRL_WB2, 6,
+			VBIF_RT, MDP_SSPP_TOP0_INTR, 4096, 4),
+};
+
+static const struct dpu_perf_cfg sm8250_perf_data = {
+	.max_bw_low = 13700000,
+	.max_bw_high = 16600000,
+	.min_core_ib = 4800000,
+	.min_llcc_ib = 0,
+	.min_dram_ib = 800000,
+	.min_prefill_lines = 35,
+	.danger_lut_tbl = {0xf, 0xffff, 0x0},
+	.safe_lut_tbl = {0xfff0, 0xff00, 0xffff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
+		.entries = sc7180_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+		/* TODO: macrotile-qseed is different from macrotile */
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sm8250_cfg = {
+	.caps = &sm8250_dpu_caps,
+	.ubwc = &sm8250_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sm8250_mdp),
+	.mdp = sm8250_mdp,
+	.ctl_count = ARRAY_SIZE(sm8250_ctl),
+	.ctl = sm8250_ctl,
+	.sspp_count = ARRAY_SIZE(sm8250_sspp),
+	.sspp = sm8250_sspp,
+	.mixer_count = ARRAY_SIZE(sm8250_lm),
+	.mixer = sm8250_lm,
+	.dspp_count = ARRAY_SIZE(sm8250_dspp),
+	.dspp = sm8250_dspp,
+	.dsc_count = ARRAY_SIZE(sm8250_dsc),
+	.dsc = sm8250_dsc,
+	.pingpong_count = ARRAY_SIZE(sm8250_pp),
+	.pingpong = sm8250_pp,
+	.merge_3d_count = ARRAY_SIZE(sm8250_merge_3d),
+	.merge_3d = sm8250_merge_3d,
+	.intf_count = ARRAY_SIZE(sm8250_intf),
+	.intf = sm8250_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.wb_count = ARRAY_SIZE(sm8250_wb),
+	.wb = sm8250_wb,
+	.reg_dma_count = 1,
+	.dma_cfg = &sm8250_regdma,
+	.perf = &sm8250_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_INTR) | \
+		     BIT(MDP_INTF1_INTR) | \
+		     BIT(MDP_INTF2_INTR) | \
+		     BIT(MDP_INTF3_INTR) | \
+		     BIT(MDP_INTF4_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_2_sc7180.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_2_sc7180.h
new file mode 100644
index 000000000000..8799ed757119
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_2_sc7180.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_6_2_SC7180_H
+#define _DPU_6_2_SC7180_H
+
+static const struct dpu_caps sc7180_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0x9,
+	.qseed_type = DPU_SSPP_SCALER_QSEED4,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.max_linewidth = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+};
+
+static const struct dpu_ubwc_cfg sc7180_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_20,
+	.highest_bank_bit = 0x3,
+};
+
+static const struct dpu_mdp_cfg sc7180_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x494,
+	.features = 0,
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2c4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_WB2] = { .reg_off = 0x3b8, .bit_off = 24 },
+	},
+};
+
+static const struct dpu_ctl_cfg sc7180_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x1000, .len = 0x1dc,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x1200, .len = 0x1dc,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x1400, .len = 0x1dc,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+};
+
+static const struct dpu_sspp_cfg sc7180_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x1f8, VIG_SC7180_MASK,
+		sc7180_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x1f8, DMA_SDM845_MASK,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x1f8, DMA_CURSOR_SDM845_MASK,
+		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x1f8, DMA_CURSOR_SDM845_MASK,
+		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+};
+
+static const struct dpu_lm_cfg sc7180_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
+		&sc7180_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
+	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
+		&sc7180_lm_sblk, PINGPONG_1, LM_0, 0),
+};
+
+static const struct dpu_dspp_cfg sc7180_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
+		 &sc7180_dspp_sblk),
+};
+
+static const struct dpu_pingpong_cfg sc7180_pp[] = {
+	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x70000, 0, sdm845_pp_sblk_te, -1, -1),
+	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x70800, 0, sdm845_pp_sblk_te, -1, -1),
+};
+
+static const struct dpu_intf_cfg sc7180_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x6a000, 0x280, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	INTF_BLK("intf_1", INTF_1, 0x6a800, 0x2c0, INTF_DSI, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+};
+
+static const struct dpu_wb_cfg sc7180_wb[] = {
+	WB_BLK("wb_2", WB_2, 0x65000, WB_SM8250_MASK, DPU_CLK_CTRL_WB2, 6,
+			VBIF_RT, MDP_SSPP_TOP0_INTR, 4096, 4),
+};
+
+static const struct dpu_perf_cfg sc7180_perf_data = {
+	.max_bw_low = 6800000,
+	.max_bw_high = 6800000,
+	.min_core_ib = 2400000,
+	.min_llcc_ib = 800000,
+	.min_dram_ib = 1600000,
+	.min_prefill_lines = 24,
+	.danger_lut_tbl = {0xff, 0xffff, 0x0},
+	.safe_lut_tbl = {0xfff0, 0xff00, 0xffff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
+		.entries = sc7180_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sc7180_cfg = {
+	.caps = &sc7180_dpu_caps,
+	.ubwc = &sc7180_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sc7180_mdp),
+	.mdp = sc7180_mdp,
+	.ctl_count = ARRAY_SIZE(sc7180_ctl),
+	.ctl = sc7180_ctl,
+	.sspp_count = ARRAY_SIZE(sc7180_sspp),
+	.sspp = sc7180_sspp,
+	.mixer_count = ARRAY_SIZE(sc7180_lm),
+	.mixer = sc7180_lm,
+	.dspp_count = ARRAY_SIZE(sc7180_dspp),
+	.dspp = sc7180_dspp,
+	.pingpong_count = ARRAY_SIZE(sc7180_pp),
+	.pingpong = sc7180_pp,
+	.intf_count = ARRAY_SIZE(sc7180_intf),
+	.intf = sc7180_intf,
+	.wb_count = ARRAY_SIZE(sc7180_wb),
+	.wb = sc7180_wb,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.reg_dma_count = 1,
+	.dma_cfg = &sdm845_regdma,
+	.perf = &sc7180_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_INTR) | \
+		     BIT(MDP_INTF1_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_3_sm6115.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_3_sm6115.h
new file mode 100644
index 000000000000..6f04d8f85c92
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_3_sm6115.h
@@ -0,0 +1,129 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_6_3_SM6115_H
+#define _DPU_6_3_SM6115_H
+
+static const struct dpu_caps sm6115_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_LINE_WIDTH,
+	.max_mixer_blendstages = 0x4,
+	.qseed_type = DPU_SSPP_SCALER_QSEED4,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.max_linewidth = 2160,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+};
+
+static const struct dpu_ubwc_cfg sm6115_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_10,
+	.highest_bank_bit = 0x1,
+	.ubwc_swizzle = 0x7,
+};
+
+static const struct dpu_mdp_cfg sm6115_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x494,
+	.features = 0,
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	},
+};
+
+static const struct dpu_ctl_cfg sm6115_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x1000, .len = 0x1dc,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+};
+
+static const struct dpu_sspp_cfg sm6115_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x1f8, VIG_SC7180_MASK,
+		sm6115_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x1f8, DMA_SDM845_MASK,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+};
+
+static const struct dpu_lm_cfg sm6115_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_QCM2290_MASK,
+		&qcm2290_lm_sblk, PINGPONG_0, 0, DSPP_0),
+};
+
+static const struct dpu_dspp_cfg sm6115_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+};
+
+static const struct dpu_pingpong_cfg sm6115_pp[] = {
+	PP_BLK("pingpong_0", PINGPONG_0, 0x70000, 0, sdm845_pp_sblk,
+		DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+		DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
+};
+
+static const struct dpu_intf_cfg sm6115_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x00000, 0x280, INTF_NONE, 0, 0, 0, 0, 0, 0),
+	INTF_BLK("intf_1", INTF_1, 0x6a800, 0x2c0, INTF_DSI, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+};
+
+static const struct dpu_perf_cfg sm6115_perf_data = {
+	.max_bw_low = 3100000,
+	.max_bw_high = 4000000,
+	.min_core_ib = 2400000,
+	.min_llcc_ib = 800000,
+	.min_dram_ib = 800000,
+	.min_prefill_lines = 24,
+	.danger_lut_tbl = {0xff, 0xffff, 0x0},
+	.safe_lut_tbl = {0xfff0, 0xff00, 0xffff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
+		.entries = sc7180_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+		/* TODO: macrotile-qseed is different from macrotile */
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sm6115_cfg = {
+	.caps = &sm6115_dpu_caps,
+	.ubwc = &sm6115_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sm6115_mdp),
+	.mdp = sm6115_mdp,
+	.ctl_count = ARRAY_SIZE(sm6115_ctl),
+	.ctl = sm6115_ctl,
+	.sspp_count = ARRAY_SIZE(sm6115_sspp),
+	.sspp = sm6115_sspp,
+	.mixer_count = ARRAY_SIZE(sm6115_lm),
+	.mixer = sm6115_lm,
+	.dspp_count = ARRAY_SIZE(sm6115_dspp),
+	.dspp = sm6115_dspp,
+	.pingpong_count = ARRAY_SIZE(sm6115_pp),
+	.pingpong = sm6115_pp,
+	.intf_count = ARRAY_SIZE(sm6115_intf),
+	.intf = sm6115_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.perf = &sm6115_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_INTR) | \
+		     BIT(MDP_INTF1_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_5_qcm2290.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_5_qcm2290.h
new file mode 100644
index 000000000000..303492d62a5c
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_6_5_qcm2290.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_6_5_QCM2290_H
+#define _DPU_6_5_QCM2290_H
+
+static const struct dpu_caps qcm2290_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_LINE_WIDTH,
+	.max_mixer_blendstages = 0x4,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.max_linewidth = 2160,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+};
+
+static const struct dpu_ubwc_cfg qcm2290_ubwc_cfg = {
+	.highest_bank_bit = 0x2,
+};
+
+static const struct dpu_mdp_cfg qcm2290_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x494,
+	.features = 0,
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	},
+};
+
+static const struct dpu_ctl_cfg qcm2290_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x1000, .len = 0x1dc,
+	.features = BIT(DPU_CTL_ACTIVE_CFG),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+};
+
+static const struct dpu_sspp_cfg qcm2290_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x1f8, VIG_QCM2290_MASK,
+		 qcm2290_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x1f8, DMA_SDM845_MASK,
+		 qcm2290_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+};
+
+static const struct dpu_lm_cfg qcm2290_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_QCM2290_MASK,
+		&qcm2290_lm_sblk, PINGPONG_0, 0, DSPP_0),
+};
+
+static const struct dpu_dspp_cfg qcm2290_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+};
+
+static const struct dpu_pingpong_cfg qcm2290_pp[] = {
+	PP_BLK("pingpong_0", PINGPONG_0, 0x70000, 0, sdm845_pp_sblk,
+		DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+		DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
+};
+
+static const struct dpu_intf_cfg qcm2290_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x00000, 0x280, INTF_NONE, 0, 0, 0, 0, 0, 0),
+	INTF_BLK("intf_1", INTF_1, 0x6a800, 0x2c0, INTF_DSI, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+};
+
+static const struct dpu_perf_cfg qcm2290_perf_data = {
+	.max_bw_low = 2700000,
+	.max_bw_high = 2700000,
+	.min_core_ib = 1300000,
+	.min_llcc_ib = 0,
+	.min_dram_ib = 1600000,
+	.min_prefill_lines = 24,
+	.danger_lut_tbl = {0xff, 0x0, 0x0},
+	.safe_lut_tbl = {0xfff0, 0x0, 0x0},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(qcm2290_qos_linear),
+		.entries = qcm2290_qos_linear
+		},
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_qcm2290_cfg = {
+	.caps = &qcm2290_dpu_caps,
+	.ubwc = &qcm2290_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(qcm2290_mdp),
+	.mdp = qcm2290_mdp,
+	.ctl_count = ARRAY_SIZE(qcm2290_ctl),
+	.ctl = qcm2290_ctl,
+	.sspp_count = ARRAY_SIZE(qcm2290_sspp),
+	.sspp = qcm2290_sspp,
+	.mixer_count = ARRAY_SIZE(qcm2290_lm),
+	.mixer = qcm2290_lm,
+	.dspp_count = ARRAY_SIZE(qcm2290_dspp),
+	.dspp = qcm2290_dspp,
+	.pingpong_count = ARRAY_SIZE(qcm2290_pp),
+	.pingpong = qcm2290_pp,
+	.intf_count = ARRAY_SIZE(qcm2290_intf),
+	.intf = qcm2290_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.perf = &qcm2290_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_INTR) | \
+		     BIT(MDP_INTF1_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_0_sm8350.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_0_sm8350.h
new file mode 100644
index 000000000000..ca107ca8de46
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_0_sm8350.h
@@ -0,0 +1,226 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_7_0_SM8350_H
+#define _DPU_7_0_SM8350_H
+
+static const struct dpu_caps sm8350_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0xb,
+	.qseed_type = DPU_SSPP_SCALER_QSEED4,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.has_3d_merge = true,
+	.max_linewidth = 4096,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+};
+
+static const struct dpu_ubwc_cfg sm8350_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_40,
+	.highest_bank_bit = 0x3, /* TODO: 2 for LP_DDR4 */
+};
+
+static const struct dpu_mdp_cfg sm8350_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x494,
+	.features = 0,
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG1] = { .reg_off = 0x2b4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG2] = { .reg_off = 0x2bc, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG3] = { .reg_off = 0x2c4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2bc, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA3] = { .reg_off = 0x2c4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_REG_DMA] = { .reg_off = 0x2bc, .bit_off = 20 },
+	},
+};
+
+/* FIXME: get rid of DPU_CTL_SPLIT_DISPLAY in favour of proper ACTIVE_CTL support */
+static const struct dpu_ctl_cfg sm8350_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x15000, .len = 0x1e8,
+	.features = BIT(DPU_CTL_SPLIT_DISPLAY) | CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x16000, .len = 0x1e8,
+	.features = BIT(DPU_CTL_SPLIT_DISPLAY) | CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x17000, .len = 0x1e8,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x18000, .len = 0x1e8,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+	{
+	.name = "ctl_4", .id = CTL_4,
+	.base = 0x19000, .len = 0x1e8,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
+	},
+	{
+	.name = "ctl_5", .id = CTL_5,
+	.base = 0x1a000, .len = 0x1e8,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
+	},
+};
+
+static const struct dpu_sspp_cfg sm8350_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x1f8, VIG_SC7180_MASK,
+		sm8250_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, 0x1f8, VIG_SC7180_MASK,
+		sm8250_vig_sblk_1, 4, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
+	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, 0x1f8, VIG_SC7180_MASK,
+		sm8250_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
+	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, 0x1f8, VIG_SC7180_MASK,
+		sm8250_vig_sblk_3, 12, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x1f8, DMA_SDM845_MASK,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x1f8, DMA_SDM845_MASK,
+		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x1f8, DMA_CURSOR_SDM845_MASK,
+		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, 0x1f8, DMA_CURSOR_SDM845_MASK,
+		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
+};
+
+static const struct dpu_lm_cfg sm8350_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
+	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_1, LM_0, DSPP_1),
+	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_2, LM_3, 0),
+	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_3, LM_2, 0),
+	LM_BLK("lm_4", LM_4, 0x48000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_4, LM_5, 0),
+	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_5, LM_4, 0),
+};
+
+static const struct dpu_dspp_cfg sm8350_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_1", DSPP_1, 0x56000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_2", DSPP_2, 0x58000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_3", DSPP_3, 0x5a000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+};
+
+static const struct dpu_pingpong_cfg sm8350_pp[] = {
+	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x69000, MERGE_3D_0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
+	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x6a000, MERGE_3D_0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
+	PP_BLK("pingpong_2", PINGPONG_2, 0x6b000, MERGE_3D_1, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
+	PP_BLK("pingpong_3", PINGPONG_3, 0x6c000, MERGE_3D_1, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
+	PP_BLK("pingpong_4", PINGPONG_4, 0x6d000, MERGE_3D_2, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30),
+			-1),
+	PP_BLK("pingpong_5", PINGPONG_5, 0x6e000, MERGE_3D_2, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31),
+			-1),
+};
+
+static const struct dpu_merge_3d_cfg sm8350_merge_3d[] = {
+	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x4e000),
+	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x4f000),
+	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x50000),
+};
+
+static const struct dpu_intf_cfg sm8350_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x34000, 0x280, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	INTF_BLK("intf_1", INTF_1, 0x35000, 0x2c4, INTF_DSI, 0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+	INTF_BLK("intf_2", INTF_2, 0x36000, 0x2c4, INTF_DSI, 1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
+	INTF_BLK("intf_3", INTF_3, 0x37000, 0x280, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
+};
+
+static const struct dpu_perf_cfg sm8350_perf_data = {
+	.max_bw_low = 11800000,
+	.max_bw_high = 15500000,
+	.min_core_ib = 2500000,
+	.min_llcc_ib = 0,
+	.min_dram_ib = 800000,
+	.min_prefill_lines = 40,
+	/* FIXME: lut tables */
+	.danger_lut_tbl = {0x3ffff, 0x3ffff, 0x0},
+	.safe_lut_tbl = {0xfe00, 0xfe00, 0xffff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
+		.entries = sc7180_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+		/* TODO: macrotile-qseed is different from macrotile */
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sm8350_cfg = {
+	.caps = &sm8350_dpu_caps,
+	.ubwc = &sm8350_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sm8350_mdp),
+	.mdp = sm8350_mdp,
+	.ctl_count = ARRAY_SIZE(sm8350_ctl),
+	.ctl = sm8350_ctl,
+	.sspp_count = ARRAY_SIZE(sm8350_sspp),
+	.sspp = sm8350_sspp,
+	.mixer_count = ARRAY_SIZE(sm8350_lm),
+	.mixer = sm8350_lm,
+	.dspp_count = ARRAY_SIZE(sm8350_dspp),
+	.dspp = sm8350_dspp,
+	.pingpong_count = ARRAY_SIZE(sm8350_pp),
+	.pingpong = sm8350_pp,
+	.merge_3d_count = ARRAY_SIZE(sm8350_merge_3d),
+	.merge_3d = sm8350_merge_3d,
+	.intf_count = ARRAY_SIZE(sm8350_intf),
+	.intf = sm8350_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.reg_dma_count = 1,
+	.dma_cfg = &sm8350_regdma,
+	.perf = &sm8350_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_7xxx_INTR) | \
+		     BIT(MDP_INTF1_7xxx_INTR) | \
+		     BIT(MDP_INTF2_7xxx_INTR) | \
+		     BIT(MDP_INTF3_7xxx_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h
new file mode 100644
index 000000000000..5957de185984
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h
@@ -0,0 +1,158 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_7_2_SC7280_H
+#define _DPU_7_2_SC7280_H
+
+static const struct dpu_caps sc7280_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0x7,
+	.qseed_type = DPU_SSPP_SCALER_QSEED4,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.max_linewidth = 2400,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+};
+
+static const struct dpu_ubwc_cfg sc7280_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_30,
+	.highest_bank_bit = 0x1,
+	.ubwc_swizzle = 0x6,
+};
+
+static const struct dpu_mdp_cfg sc7280_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x2014,
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2c4, .bit_off = 8 },
+	},
+};
+
+static const struct dpu_ctl_cfg sc7280_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x15000, .len = 0x1e8,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x16000, .len = 0x1e8,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x17000, .len = 0x1e8,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x18000, .len = 0x1e8,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+};
+
+static const struct dpu_sspp_cfg sc7280_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x1f8, VIG_SC7280_MASK_SDMA,
+		sc7280_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x1f8, DMA_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x1f8, DMA_CURSOR_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x1f8, DMA_CURSOR_SDM845_MASK_SDMA,
+		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+};
+
+static const struct dpu_lm_cfg sc7280_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
+		&sc7180_lm_sblk, PINGPONG_0, 0, DSPP_0),
+	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
+		&sc7180_lm_sblk, PINGPONG_2, LM_3, 0),
+	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK,
+		&sc7180_lm_sblk, PINGPONG_3, LM_2, 0),
+};
+
+static const struct dpu_dspp_cfg sc7280_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
+		 &sc7180_dspp_sblk),
+};
+
+static const struct dpu_pingpong_cfg sc7280_pp[] = {
+	PP_BLK("pingpong_0", PINGPONG_0, 0x69000, 0, sc7280_pp_sblk, -1, -1),
+	PP_BLK("pingpong_1", PINGPONG_1, 0x6a000, 0, sc7280_pp_sblk, -1, -1),
+	PP_BLK("pingpong_2", PINGPONG_2, 0x6b000, 0, sc7280_pp_sblk, -1, -1),
+	PP_BLK("pingpong_3", PINGPONG_3, 0x6c000, 0, sc7280_pp_sblk, -1, -1),
+};
+
+static const struct dpu_intf_cfg sc7280_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x34000, 0x280, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	INTF_BLK("intf_1", INTF_1, 0x35000, 0x2c4, INTF_DSI, 0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+	INTF_BLK("intf_5", INTF_5, 0x39000, 0x280, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 22, 23),
+};
+
+static const struct dpu_perf_cfg sc7280_perf_data = {
+	.max_bw_low = 4700000,
+	.max_bw_high = 8800000,
+	.min_core_ib = 2500000,
+	.min_llcc_ib = 0,
+	.min_dram_ib = 1600000,
+	.min_prefill_lines = 24,
+	.danger_lut_tbl = {0xffff, 0xffff, 0x0},
+	.safe_lut_tbl = {0xff00, 0xff00, 0xffff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sc7280_cfg = {
+	.caps = &sc7280_dpu_caps,
+	.ubwc = &sc7280_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sc7280_mdp),
+	.mdp = sc7280_mdp,
+	.ctl_count = ARRAY_SIZE(sc7280_ctl),
+	.ctl = sc7280_ctl,
+	.sspp_count = ARRAY_SIZE(sc7280_sspp),
+	.sspp = sc7280_sspp,
+	.dspp_count = ARRAY_SIZE(sc7280_dspp),
+	.dspp = sc7280_dspp,
+	.mixer_count = ARRAY_SIZE(sc7280_lm),
+	.mixer = sc7280_lm,
+	.pingpong_count = ARRAY_SIZE(sc7280_pp),
+	.pingpong = sc7280_pp,
+	.intf_count = ARRAY_SIZE(sc7280_intf),
+	.intf = sc7280_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.perf = &sc7280_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_7xxx_INTR) | \
+		     BIT(MDP_INTF1_7xxx_INTR) | \
+		     BIT(MDP_INTF5_7xxx_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h
new file mode 100644
index 000000000000..9aab110b8c44
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_0_sc8280xp.h
@@ -0,0 +1,222 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_8_0_SC8280XP_H
+#define _DPU_8_0_SC8280XP_H
+
+static const struct dpu_caps sc8280xp_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 11,
+	.qseed_type = DPU_SSPP_SCALER_QSEED4,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.has_3d_merge = true,
+	.max_linewidth = 5120,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+};
+
+static const struct dpu_ubwc_cfg sc8280xp_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_40,
+	.highest_bank_bit = 2,
+	.ubwc_swizzle = 6,
+};
+
+static const struct dpu_mdp_cfg sc8280xp_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x494,
+	.features = BIT(DPU_MDP_PERIPH_0_REMOVED),
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG1] = { .reg_off = 0x2b4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG2] = { .reg_off = 0x2bc, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG3] = { .reg_off = 0x2c4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2bc, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA3] = { .reg_off = 0x2c4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_REG_DMA] = { .reg_off = 0x2bc, .bit_off = 20 },
+	},
+};
+
+static const struct dpu_ctl_cfg sc8280xp_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x15000, .len = 0x204,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x16000, .len = 0x204,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x17000, .len = 0x204,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x18000, .len = 0x204,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+	{
+	.name = "ctl_4", .id = CTL_4,
+	.base = 0x19000, .len = 0x204,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
+	},
+	{
+	.name = "ctl_5", .id = CTL_5,
+	.base = 0x1a000, .len = 0x204,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
+	},
+};
+
+static const struct dpu_sspp_cfg sc8280xp_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x2ac, VIG_SC7180_MASK,
+		 sm8250_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, 0x2ac, VIG_SC7180_MASK,
+		 sm8250_vig_sblk_1, 4, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
+	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, 0x2ac, VIG_SC7180_MASK,
+		 sm8250_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
+	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, 0x2ac, VIG_SC7180_MASK,
+		 sm8250_vig_sblk_3, 12, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x2ac, DMA_SDM845_MASK,
+		 sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x2ac, DMA_SDM845_MASK,
+		 sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x2ac, DMA_CURSOR_SDM845_MASK,
+		 sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, 0x2ac, DMA_CURSOR_SDM845_MASK,
+		 sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
+};
+
+static const struct dpu_lm_cfg sc8280xp_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
+	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_1, LM_0, DSPP_1),
+	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_2, LM_3, DSPP_2),
+	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_3, LM_2, DSPP_3),
+	LM_BLK("lm_4", LM_4, 0x48000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_4, LM_5, 0),
+	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_5, LM_4, 0),
+};
+
+static const struct dpu_dspp_cfg sc8280xp_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_1", DSPP_1, 0x56000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_2", DSPP_2, 0x58000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_3", DSPP_3, 0x5a000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+};
+
+static const struct dpu_pingpong_cfg sc8280xp_pp[] = {
+	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x69000, MERGE_3D_0, sdm845_pp_sblk_te,
+		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8), -1),
+	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x6a000, MERGE_3D_0, sdm845_pp_sblk_te,
+		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9), -1),
+	PP_BLK_TE("pingpong_2", PINGPONG_2, 0x6b000, MERGE_3D_1, sdm845_pp_sblk_te,
+		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10), -1),
+	PP_BLK_TE("pingpong_3", PINGPONG_3, 0x6c000, MERGE_3D_1, sdm845_pp_sblk_te,
+		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11), -1),
+	PP_BLK_TE("pingpong_4", PINGPONG_4, 0x6d000, MERGE_3D_2, sdm845_pp_sblk_te,
+		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30), -1),
+	PP_BLK_TE("pingpong_5", PINGPONG_5, 0x6e000, MERGE_3D_2, sdm845_pp_sblk_te,
+		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31), -1),
+};
+
+static const struct dpu_merge_3d_cfg sc8280xp_merge_3d[] = {
+	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x4e000),
+	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x4f000),
+	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x50000),
+};
+
+/* TODO: INTF 3, 8 and 7 are used for MST, marked as INTF_NONE for now */
+static const struct dpu_intf_cfg sc8280xp_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x34000, 0x280, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	INTF_BLK("intf_1", INTF_1, 0x35000, 0x300, INTF_DSI, 0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+	INTF_BLK("intf_2", INTF_2, 0x36000, 0x300, INTF_DSI, 1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
+	INTF_BLK("intf_3", INTF_3, 0x37000, 0x280, INTF_NONE, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
+	INTF_BLK("intf_4", INTF_4, 0x38000, 0x280, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 20, 21),
+	INTF_BLK("intf_5", INTF_5, 0x39000, 0x280, INTF_DP, MSM_DP_CONTROLLER_3, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 22, 23),
+	INTF_BLK("intf_6", INTF_6, 0x3a000, 0x280, INTF_DP, MSM_DP_CONTROLLER_2, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 16, 17),
+	INTF_BLK("intf_7", INTF_7, 0x3b000, 0x280, INTF_NONE, MSM_DP_CONTROLLER_2, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 18, 19),
+	INTF_BLK("intf_8", INTF_8, 0x3c000, 0x280, INTF_NONE, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 12, 13),
+};
+
+static const struct dpu_perf_cfg sc8280xp_perf_data = {
+	.max_bw_low = 13600000,
+	.max_bw_high = 18200000,
+	.min_core_ib = 2500000,
+	.min_llcc_ib = 0,
+	.min_dram_ib = 800000,
+	.danger_lut_tbl = {0xf, 0xffff, 0x0},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sc8180x_qos_linear),
+		.entries = sc8180x_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sc8180x_qos_macrotile),
+		.entries = sc8180x_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+		/* TODO: macrotile-qseed is different from macrotile */
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sc8280xp_cfg = {
+	.caps = &sc8280xp_dpu_caps,
+	.ubwc = &sc8280xp_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sc8280xp_mdp),
+	.mdp = sc8280xp_mdp,
+	.ctl_count = ARRAY_SIZE(sc8280xp_ctl),
+	.ctl = sc8280xp_ctl,
+	.sspp_count = ARRAY_SIZE(sc8280xp_sspp),
+	.sspp = sc8280xp_sspp,
+	.mixer_count = ARRAY_SIZE(sc8280xp_lm),
+	.mixer = sc8280xp_lm,
+	.dspp_count = ARRAY_SIZE(sc8280xp_dspp),
+	.dspp = sc8280xp_dspp,
+	.pingpong_count = ARRAY_SIZE(sc8280xp_pp),
+	.pingpong = sc8280xp_pp,
+	.merge_3d_count = ARRAY_SIZE(sc8280xp_merge_3d),
+	.merge_3d = sc8280xp_merge_3d,
+	.intf_count = ARRAY_SIZE(sc8280xp_intf),
+	.intf = sc8280xp_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.reg_dma_count = 1,
+	.dma_cfg = &sc8280xp_regdma,
+	.perf = &sc8280xp_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_7xxx_INTR) | \
+		     BIT(MDP_INTF1_7xxx_INTR) | \
+		     BIT(MDP_INTF2_7xxx_INTR) | \
+		     BIT(MDP_INTF3_7xxx_INTR) | \
+		     BIT(MDP_INTF4_7xxx_INTR) | \
+		     BIT(MDP_INTF5_7xxx_INTR) | \
+		     BIT(MDP_INTF6_7xxx_INTR) | \
+		     BIT(MDP_INTF7_7xxx_INTR) | \
+		     BIT(MDP_INTF8_7xxx_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h
new file mode 100644
index 000000000000..02a259b6b426
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_8_1_sm8450.h
@@ -0,0 +1,234 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_8_1_SM8450_H
+#define _DPU_8_1_SM8450_H
+
+static const struct dpu_caps sm8450_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0xb,
+	.qseed_type = DPU_SSPP_SCALER_QSEED4,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.has_3d_merge = true,
+	.max_linewidth = 5120,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+};
+
+static const struct dpu_ubwc_cfg sm8450_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_40,
+	.highest_bank_bit = 0x3, /* TODO: 2 for LP_DDR4 */
+	.ubwc_swizzle = 0x6,
+};
+
+static const struct dpu_mdp_cfg sm8450_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x494,
+	.features = BIT(DPU_MDP_PERIPH_0_REMOVED),
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG1] = { .reg_off = 0x2b4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG2] = { .reg_off = 0x2bc, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG3] = { .reg_off = 0x2c4, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2bc, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA3] = { .reg_off = 0x2c4, .bit_off = 8 },
+	.clk_ctrls[DPU_CLK_CTRL_REG_DMA] = { .reg_off = 0x2bc, .bit_off = 20 },
+	},
+};
+
+/* FIXME: get rid of DPU_CTL_SPLIT_DISPLAY in favour of proper ACTIVE_CTL support */
+static const struct dpu_ctl_cfg sm8450_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x15000, .len = 0x204,
+	.features = BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_SPLIT_DISPLAY) | BIT(DPU_CTL_FETCH_ACTIVE),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x16000, .len = 0x204,
+	.features = BIT(DPU_CTL_SPLIT_DISPLAY) | CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x17000, .len = 0x204,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x18000, .len = 0x204,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+	{
+	.name = "ctl_4", .id = CTL_4,
+	.base = 0x19000, .len = 0x204,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
+	},
+	{
+	.name = "ctl_5", .id = CTL_5,
+	.base = 0x1a000, .len = 0x204,
+	.features = CTL_SC7280_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
+	},
+};
+
+static const struct dpu_sspp_cfg sm8450_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x32c, VIG_SC7180_MASK,
+		sm8250_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, 0x32c, VIG_SC7180_MASK,
+		sm8250_vig_sblk_1, 4, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
+	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, 0x32c, VIG_SC7180_MASK,
+		sm8250_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
+	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, 0x32c, VIG_SC7180_MASK,
+		sm8250_vig_sblk_3, 12, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x32c, DMA_SDM845_MASK,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x32c, DMA_SDM845_MASK,
+		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x32c, DMA_CURSOR_SDM845_MASK,
+		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, 0x32c, DMA_CURSOR_SDM845_MASK,
+		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
+};
+
+static const struct dpu_lm_cfg sm8450_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
+	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_1, LM_0, DSPP_1),
+	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_2, LM_3, 0),
+	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_3, LM_2, 0),
+	LM_BLK("lm_4", LM_4, 0x48000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_4, LM_5, 0),
+	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_5, LM_4, 0),
+};
+
+static const struct dpu_dspp_cfg sm8450_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_1", DSPP_1, 0x56000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_2", DSPP_2, 0x58000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_3", DSPP_3, 0x5a000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+};
+/* FIXME: interrupts */
+static const struct dpu_pingpong_cfg sm8450_pp[] = {
+	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x69000, MERGE_3D_0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
+	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x6a000, MERGE_3D_0, sdm845_pp_sblk_te,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
+	PP_BLK("pingpong_2", PINGPONG_2, 0x6b000, MERGE_3D_1, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
+	PP_BLK("pingpong_3", PINGPONG_3, 0x6c000, MERGE_3D_1, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
+	PP_BLK("pingpong_4", PINGPONG_4, 0x6d000, MERGE_3D_2, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30),
+			-1),
+	PP_BLK("pingpong_5", PINGPONG_5, 0x6e000, MERGE_3D_2, sdm845_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31),
+			-1),
+	PP_BLK("pingpong_6", PINGPONG_6, 0x65800, MERGE_3D_3, sdm845_pp_sblk,
+			-1,
+			-1),
+	PP_BLK("pingpong_7", PINGPONG_7, 0x65c00, MERGE_3D_3, sdm845_pp_sblk,
+			-1,
+			-1),
+};
+
+static const struct dpu_merge_3d_cfg sm8450_merge_3d[] = {
+	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x4e000),
+	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x4f000),
+	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x50000),
+	MERGE_3D_BLK("merge_3d_3", MERGE_3D_3, 0x65f00),
+};
+
+static const struct dpu_intf_cfg sm8450_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x34000, 0x280, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	INTF_BLK("intf_1", INTF_1, 0x35000, 0x300, INTF_DSI, 0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+	INTF_BLK("intf_2", INTF_2, 0x36000, 0x300, INTF_DSI, 1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
+	INTF_BLK("intf_3", INTF_3, 0x37000, 0x280, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
+};
+
+static const struct dpu_perf_cfg sm8450_perf_data = {
+	.max_bw_low = 13600000,
+	.max_bw_high = 18200000,
+	.min_core_ib = 2500000,
+	.min_llcc_ib = 0,
+	.min_dram_ib = 800000,
+	.min_prefill_lines = 35,
+	/* FIXME: lut tables */
+	.danger_lut_tbl = {0x3ffff, 0x3ffff, 0x0},
+	.safe_lut_tbl = {0xfe00, 0xfe00, 0xffff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
+		.entries = sc7180_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+		/* TODO: macrotile-qseed is different from macrotile */
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sm8450_cfg = {
+	.caps = &sm8450_dpu_caps,
+	.ubwc = &sm8450_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sm8450_mdp),
+	.mdp = sm8450_mdp,
+	.ctl_count = ARRAY_SIZE(sm8450_ctl),
+	.ctl = sm8450_ctl,
+	.sspp_count = ARRAY_SIZE(sm8450_sspp),
+	.sspp = sm8450_sspp,
+	.mixer_count = ARRAY_SIZE(sm8450_lm),
+	.mixer = sm8450_lm,
+	.dspp_count = ARRAY_SIZE(sm8450_dspp),
+	.dspp = sm8450_dspp,
+	.pingpong_count = ARRAY_SIZE(sm8450_pp),
+	.pingpong = sm8450_pp,
+	.merge_3d_count = ARRAY_SIZE(sm8450_merge_3d),
+	.merge_3d = sm8450_merge_3d,
+	.intf_count = ARRAY_SIZE(sm8450_intf),
+	.intf = sm8450_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.reg_dma_count = 1,
+	.dma_cfg = &sm8450_regdma,
+	.perf = &sm8450_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_7xxx_INTR) | \
+		     BIT(MDP_INTF1_7xxx_INTR) | \
+		     BIT(MDP_INTF2_7xxx_INTR) | \
+		     BIT(MDP_INTF3_7xxx_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h
new file mode 100644
index 000000000000..9e403034093f
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_9_0_sm8550.h
@@ -0,0 +1,239 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_9_0_SM8550_H
+#define _DPU_9_0_SM8550_H
+
+static const struct dpu_caps sm8550_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0xb,
+	.qseed_type = DPU_SSPP_SCALER_QSEED4,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.has_3d_merge = true,
+	.max_linewidth = 5120,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+};
+
+static const struct dpu_ubwc_cfg sm8550_ubwc_cfg = {
+	.ubwc_version = DPU_HW_UBWC_VER_40,
+	.highest_bank_bit = 0x3, /* TODO: 2 for LP_DDR4 */
+};
+
+static const struct dpu_mdp_cfg sm8550_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0, .len = 0x494,
+	.features = BIT(DPU_MDP_PERIPH_0_REMOVED),
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x4330, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG1] = { .reg_off = 0x6330, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG2] = { .reg_off = 0x8330, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_VIG3] = { .reg_off = 0xa330, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x24330, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x26330, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x28330, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA3] = { .reg_off = 0x2a330, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA4] = { .reg_off = 0x2c330, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_DMA5] = { .reg_off = 0x2e330, .bit_off = 0 },
+	.clk_ctrls[DPU_CLK_CTRL_REG_DMA] = { .reg_off = 0x2bc, .bit_off = 20 },
+	},
+};
+
+/* FIXME: get rid of DPU_CTL_SPLIT_DISPLAY in favour of proper ACTIVE_CTL support */
+static const struct dpu_ctl_cfg sm8550_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x15000, .len = 0x290,
+	.features = CTL_SM8550_MASK | BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x16000, .len = 0x290,
+	.features = CTL_SM8550_MASK | BIT(DPU_CTL_SPLIT_DISPLAY),
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x17000, .len = 0x290,
+	.features = CTL_SM8550_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x18000, .len = 0x290,
+	.features = CTL_SM8550_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+	{
+	.name = "ctl_4", .id = CTL_4,
+	.base = 0x19000, .len = 0x290,
+	.features = CTL_SM8550_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
+	},
+	{
+	.name = "ctl_5", .id = CTL_5,
+	.base = 0x1a000, .len = 0x290,
+	.features = CTL_SM8550_MASK,
+	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
+	},
+};
+
+static const struct dpu_sspp_cfg sm8550_sspp[] = {
+	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, 0x344, VIG_SC7180_MASK,
+		sm8550_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
+	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, 0x344, VIG_SC7180_MASK,
+		sm8550_vig_sblk_1, 4, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
+	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, 0x344, VIG_SC7180_MASK,
+		sm8550_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
+	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, 0x344, VIG_SC7180_MASK,
+		sm8550_vig_sblk_3, 12, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
+	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, 0x344, DMA_SDM845_MASK,
+		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
+	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, 0x344, DMA_SDM845_MASK,
+		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
+	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, 0x344, DMA_SDM845_MASK,
+		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
+	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, 0x344, DMA_SDM845_MASK,
+		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
+	SSPP_BLK("sspp_12", SSPP_DMA4, 0x2c000, 0x344, DMA_CURSOR_SDM845_MASK,
+		sm8550_dma_sblk_4, 14, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA4),
+	SSPP_BLK("sspp_13", SSPP_DMA5, 0x2e000, 0x344, DMA_CURSOR_SDM845_MASK,
+		sm8550_dma_sblk_5, 15, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA5),
+};
+
+static const struct dpu_lm_cfg sm8550_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
+	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_1, LM_0, DSPP_1),
+	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_2, LM_3, 0),
+	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_3, LM_2, 0),
+	LM_BLK("lm_4", LM_4, 0x48000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_4, LM_5, 0),
+	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK,
+		&sdm845_lm_sblk, PINGPONG_5, LM_4, 0),
+};
+
+static const struct dpu_dspp_cfg sm8550_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_1", DSPP_1, 0x56000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_2", DSPP_2, 0x58000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+	DSPP_BLK("dspp_3", DSPP_3, 0x5a000, DSPP_SC7180_MASK,
+		 &sm8150_dspp_sblk),
+};
+static const struct dpu_pingpong_cfg sm8550_pp[] = {
+	PP_BLK_DIPHER("pingpong_0", PINGPONG_0, 0x69000, MERGE_3D_0, sc7280_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+			-1),
+	PP_BLK_DIPHER("pingpong_1", PINGPONG_1, 0x6a000, MERGE_3D_0, sc7280_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
+			-1),
+	PP_BLK_DIPHER("pingpong_2", PINGPONG_2, 0x6b000, MERGE_3D_1, sc7280_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
+			-1),
+	PP_BLK_DIPHER("pingpong_3", PINGPONG_3, 0x6c000, MERGE_3D_1, sc7280_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
+			-1),
+	PP_BLK_DIPHER("pingpong_4", PINGPONG_4, 0x6d000, MERGE_3D_2, sc7280_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30),
+			-1),
+	PP_BLK_DIPHER("pingpong_5", PINGPONG_5, 0x6e000, MERGE_3D_2, sc7280_pp_sblk,
+			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31),
+			-1),
+	PP_BLK_DIPHER("pingpong_6", PINGPONG_6, 0x66000, MERGE_3D_3, sc7280_pp_sblk,
+			-1,
+			-1),
+	PP_BLK_DIPHER("pingpong_7", PINGPONG_7, 0x66400, MERGE_3D_3, sc7280_pp_sblk,
+			-1,
+			-1),
+};
+
+static const struct dpu_merge_3d_cfg sm8550_merge_3d[] = {
+	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x4e000),
+	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x4f000),
+	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x50000),
+	MERGE_3D_BLK("merge_3d_3", MERGE_3D_3, 0x66700),
+};
+
+static const struct dpu_intf_cfg sm8550_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x34000, 0x280, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
+	/* TODO TE sub-blocks for intf1 & intf2 */
+	INTF_BLK("intf_1", INTF_1, 0x35000, 0x300, INTF_DSI, 0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
+	INTF_BLK("intf_2", INTF_2, 0x36000, 0x300, INTF_DSI, 1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
+	INTF_BLK("intf_3", INTF_3, 0x37000, 0x280, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
+};
+
+static const struct dpu_perf_cfg sm8550_perf_data = {
+	.max_bw_low = 13600000,
+	.max_bw_high = 18200000,
+	.min_core_ib = 2500000,
+	.min_llcc_ib = 0,
+	.min_dram_ib = 800000,
+	.min_prefill_lines = 35,
+	/* FIXME: lut tables */
+	.danger_lut_tbl = {0x3ffff, 0x3ffff, 0x0},
+	.safe_lut_tbl = {0xfe00, 0xfe00, 0xffff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
+		.entries = sc7180_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+		/* TODO: macrotile-qseed is different from macrotile */
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+const struct dpu_mdss_cfg dpu_sm8550_cfg = {
+	.caps = &sm8550_dpu_caps,
+	.ubwc = &sm8550_ubwc_cfg,
+	.mdp_count = ARRAY_SIZE(sm8550_mdp),
+	.mdp = sm8550_mdp,
+	.ctl_count = ARRAY_SIZE(sm8550_ctl),
+	.ctl = sm8550_ctl,
+	.sspp_count = ARRAY_SIZE(sm8550_sspp),
+	.sspp = sm8550_sspp,
+	.mixer_count = ARRAY_SIZE(sm8550_lm),
+	.mixer = sm8550_lm,
+	.dspp_count = ARRAY_SIZE(sm8550_dspp),
+	.dspp = sm8550_dspp,
+	.pingpong_count = ARRAY_SIZE(sm8550_pp),
+	.pingpong = sm8550_pp,
+	.merge_3d_count = ARRAY_SIZE(sm8550_merge_3d),
+	.merge_3d = sm8550_merge_3d,
+	.intf_count = ARRAY_SIZE(sm8550_intf),
+	.intf = sm8550_intf,
+	.vbif_count = ARRAY_SIZE(sdm845_vbif),
+	.vbif = sdm845_vbif,
+	.reg_dma_count = 1,
+	.dma_cfg = &sm8450_regdma,
+	.perf = &sm8550_perf_data,
+	.mdss_irqs = BIT(MDP_SSPP_TOP0_INTR) | \
+		     BIT(MDP_SSPP_TOP0_INTR2) | \
+		     BIT(MDP_SSPP_TOP0_HIST_INTR) | \
+		     BIT(MDP_INTF0_7xxx_INTR) | \
+		     BIT(MDP_INTF1_7xxx_INTR) | \
+		     BIT(MDP_INTF2_7xxx_INTR) | \
+		     BIT(MDP_INTF3_7xxx_INTR),
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
index f29a339a3705..cc66ddffe672 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
@@ -21,6 +21,7 @@
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_rect.h>
 #include <drm/drm_vblank.h>
+#include <drm/drm_self_refresh_helper.h>
 
 #include "dpu_kms.h"
 #include "dpu_hw_lm.h"
@@ -400,6 +401,47 @@ static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc)
 	}
 }
 
+static void _dpu_crtc_blend_setup_pipe(struct drm_crtc *crtc,
+				       struct drm_plane *plane,
+				       struct dpu_crtc_mixer *mixer,
+				       u32 num_mixers,
+				       enum dpu_stage stage,
+				       struct dpu_format *format,
+				       uint64_t modifier,
+				       struct dpu_sw_pipe *pipe,
+				       unsigned int stage_idx,
+				       struct dpu_hw_stage_cfg *stage_cfg
+				      )
+{
+	uint32_t lm_idx;
+	enum dpu_sspp sspp_idx;
+	struct drm_plane_state *state;
+
+	sspp_idx = pipe->sspp->idx;
+
+	state = plane->state;
+
+	trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
+				   state, to_dpu_plane_state(state), stage_idx,
+				   format->base.pixel_format,
+				   modifier);
+
+	DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d multirect_idx %d\n",
+			 crtc->base.id,
+			 stage,
+			 plane->base.id,
+			 sspp_idx - SSPP_NONE,
+			 state->fb ? state->fb->base.id : -1,
+			 pipe->multirect_index);
+
+	stage_cfg->stage[stage][stage_idx] = sspp_idx;
+	stage_cfg->multirect_index[stage][stage_idx] = pipe->multirect_index;
+
+	/* blend config update */
+	for (lm_idx = 0; lm_idx < num_mixers; lm_idx++)
+		mixer[lm_idx].lm_ctl->ops.update_pending_flush_sspp(mixer[lm_idx].lm_ctl, sspp_idx);
+}
+
 static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 	struct dpu_crtc *dpu_crtc, struct dpu_crtc_mixer *mixer,
 	struct dpu_hw_stage_cfg *stage_cfg)
@@ -412,15 +454,12 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 	struct dpu_format *format;
 	struct dpu_hw_ctl *ctl = mixer->lm_ctl;
 
-	uint32_t stage_idx, lm_idx;
-	int zpos_cnt[DPU_STAGE_MAX + 1] = { 0 };
+	uint32_t lm_idx;
 	bool bg_alpha_enable = false;
 	DECLARE_BITMAP(fetch_active, SSPP_MAX);
 
 	memset(fetch_active, 0, sizeof(fetch_active));
 	drm_atomic_crtc_for_each_plane(plane, crtc) {
-		enum dpu_sspp sspp_idx;
-
 		state = plane->state;
 		if (!state)
 			continue;
@@ -431,40 +470,30 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 		pstate = to_dpu_plane_state(state);
 		fb = state->fb;
 
-		sspp_idx = dpu_plane_pipe(plane);
-		set_bit(sspp_idx, fetch_active);
-
-		DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d\n",
-				crtc->base.id,
-				pstate->stage,
-				plane->base.id,
-				sspp_idx - SSPP_VIG0,
-				state->fb ? state->fb->base.id : -1);
-
 		format = to_dpu_format(msm_framebuffer_format(pstate->base.fb));
 
 		if (pstate->stage == DPU_STAGE_BASE && format->alpha_enable)
 			bg_alpha_enable = true;
 
-		stage_idx = zpos_cnt[pstate->stage]++;
-		stage_cfg->stage[pstate->stage][stage_idx] =
-					sspp_idx;
-		stage_cfg->multirect_index[pstate->stage][stage_idx] =
-					pstate->multirect_index;
-
-		trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
-					   state, pstate, stage_idx,
-					   sspp_idx - SSPP_VIG0,
-					   format->base.pixel_format,
-					   fb ? fb->modifier : 0);
+		set_bit(pstate->pipe.sspp->idx, fetch_active);
+		_dpu_crtc_blend_setup_pipe(crtc, plane,
+					   mixer, cstate->num_mixers,
+					   pstate->stage,
+					   format, fb ? fb->modifier : 0,
+					   &pstate->pipe, 0, stage_cfg);
+
+		if (pstate->r_pipe.sspp) {
+			set_bit(pstate->r_pipe.sspp->idx, fetch_active);
+			_dpu_crtc_blend_setup_pipe(crtc, plane,
+						   mixer, cstate->num_mixers,
+						   pstate->stage,
+						   format, fb ? fb->modifier : 0,
+						   &pstate->r_pipe, 1, stage_cfg);
+		}
 
 		/* blend config update */
 		for (lm_idx = 0; lm_idx < cstate->num_mixers; lm_idx++) {
-			_dpu_crtc_setup_blend_cfg(mixer + lm_idx,
-						pstate, format);
-
-			mixer[lm_idx].lm_ctl->ops.update_pending_flush_sspp(mixer[lm_idx].lm_ctl,
-									    sspp_idx);
+			_dpu_crtc_setup_blend_cfg(mixer + lm_idx, pstate, format);
 
 			if (bg_alpha_enable && !format->alpha_enable)
 				mixer[lm_idx].mixer_op_mode = 0;
@@ -767,7 +796,7 @@ static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc)
 
 		/* stage config flush mask */
 		ctl->ops.update_pending_flush_dspp(ctl,
-			mixer[i].hw_dspp->idx);
+			mixer[i].hw_dspp->idx, DPU_DSPP_PCC);
 	}
 }
 
@@ -1021,6 +1050,18 @@ static void dpu_crtc_disable(struct drm_crtc *crtc,
 
 	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
 
+	/* If disable is triggered while in self refresh mode,
+	 * reset the encoder software state so that in enable
+	 * it won't trigger a warn while assigning crtc.
+	 */
+	if (old_crtc_state->self_refresh_active) {
+		drm_for_each_encoder_mask(encoder, crtc->dev,
+					old_crtc_state->encoder_mask) {
+			dpu_encoder_assign_crtc(encoder, NULL);
+		}
+		return;
+	}
+
 	/* Disable/save vblank irq handling */
 	drm_crtc_vblank_off(crtc);
 
@@ -1032,7 +1073,14 @@ static void dpu_crtc_disable(struct drm_crtc *crtc,
 		 */
 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
 			release_bandwidth = true;
-		dpu_encoder_assign_crtc(encoder, NULL);
+
+		/*
+		 * If disable is triggered during psr active(e.g: screen dim in PSR),
+		 * we will need encoder->crtc connection to process the device sleep &
+		 * preserve it during psr sequence.
+		 */
+		if (!crtc->state->self_refresh_active)
+			dpu_encoder_assign_crtc(encoder, NULL);
 	}
 
 	/* wait for frame_event_done completion */
@@ -1080,6 +1128,9 @@ static void dpu_crtc_enable(struct drm_crtc *crtc,
 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 	struct drm_encoder *encoder;
 	bool request_bandwidth = false;
+	struct drm_crtc_state *old_crtc_state;
+
+	old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc);
 
 	pm_runtime_get_sync(crtc->dev->dev);
 
@@ -1102,25 +1153,23 @@ static void dpu_crtc_enable(struct drm_crtc *crtc,
 	trace_dpu_crtc_enable(DRMID(crtc), true, dpu_crtc);
 	dpu_crtc->enabled = true;
 
-	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
-		dpu_encoder_assign_crtc(encoder, crtc);
+	if (!old_crtc_state->self_refresh_active) {
+		drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
+			dpu_encoder_assign_crtc(encoder, crtc);
+	}
 
 	/* Enable/restore vblank irq handling */
 	drm_crtc_vblank_on(crtc);
 }
 
-struct plane_state {
-	struct dpu_plane_state *dpu_pstate;
-	const struct drm_plane_state *drm_pstate;
-	int stage;
-	u32 pipe_id;
-};
-
 static bool dpu_crtc_needs_dirtyfb(struct drm_crtc_state *cstate)
 {
 	struct drm_crtc *crtc = cstate->crtc;
 	struct drm_encoder *encoder;
 
+	if (cstate->self_refresh_active)
+		return true;
+
 	drm_for_each_encoder_mask (encoder, crtc->dev, cstate->encoder_mask) {
 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_CMD) {
 			return true;
@@ -1137,151 +1186,46 @@ static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
 									  crtc);
 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc_state);
-	struct plane_state *pstates;
 
 	const struct drm_plane_state *pstate;
 	struct drm_plane *plane;
-	struct drm_display_mode *mode;
 
-	int cnt = 0, rc = 0, mixer_width = 0, i, z_pos;
+	int rc = 0;
 
-	struct dpu_multirect_plane_states multirect_plane[DPU_STAGE_MAX * 2];
-	int multirect_count = 0;
-	const struct drm_plane_state *pipe_staged[SSPP_MAX];
-	int left_zpos_cnt = 0, right_zpos_cnt = 0;
-	struct drm_rect crtc_rect = { 0 };
 	bool needs_dirtyfb = dpu_crtc_needs_dirtyfb(crtc_state);
 
-	pstates = kzalloc(sizeof(*pstates) * DPU_STAGE_MAX * 4, GFP_KERNEL);
-	if (!pstates)
-		return -ENOMEM;
-
-	if (!crtc_state->enable || !crtc_state->active) {
+	if (!crtc_state->enable || !drm_atomic_crtc_effectively_active(crtc_state)) {
 		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, active %d, skip atomic_check\n",
 				crtc->base.id, crtc_state->enable,
 				crtc_state->active);
 		memset(&cstate->new_perf, 0, sizeof(cstate->new_perf));
-		goto end;
+		return 0;
 	}
 
-	mode = &crtc_state->adjusted_mode;
 	DRM_DEBUG_ATOMIC("%s: check\n", dpu_crtc->name);
 
 	/* force a full mode set if active state changed */
 	if (crtc_state->active_changed)
 		crtc_state->mode_changed = true;
 
-	memset(pipe_staged, 0, sizeof(pipe_staged));
-
-	if (cstate->num_mixers) {
-		mixer_width = mode->hdisplay / cstate->num_mixers;
-
+	if (cstate->num_mixers)
 		_dpu_crtc_setup_lm_bounds(crtc, crtc_state);
-	}
-
-	crtc_rect.x2 = mode->hdisplay;
-	crtc_rect.y2 = mode->vdisplay;
 
-	 /* get plane state for all drm planes associated with crtc state */
+	/* FIXME: move this to dpu_plane_atomic_check? */
 	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
 		struct dpu_plane_state *dpu_pstate = to_dpu_plane_state(pstate);
-		struct drm_rect dst, clip = crtc_rect;
 
 		if (IS_ERR_OR_NULL(pstate)) {
 			rc = PTR_ERR(pstate);
 			DPU_ERROR("%s: failed to get plane%d state, %d\n",
 					dpu_crtc->name, plane->base.id, rc);
-			goto end;
+			return rc;
 		}
-		if (cnt >= DPU_STAGE_MAX * 4)
-			continue;
 
 		if (!pstate->visible)
 			continue;
 
-		pstates[cnt].dpu_pstate = dpu_pstate;
-		pstates[cnt].drm_pstate = pstate;
-		pstates[cnt].stage = pstate->normalized_zpos;
-		pstates[cnt].pipe_id = dpu_plane_pipe(plane);
-
 		dpu_pstate->needs_dirtyfb = needs_dirtyfb;
-
-		if (pipe_staged[pstates[cnt].pipe_id]) {
-			multirect_plane[multirect_count].r0 =
-				pipe_staged[pstates[cnt].pipe_id];
-			multirect_plane[multirect_count].r1 = pstate;
-			multirect_count++;
-
-			pipe_staged[pstates[cnt].pipe_id] = NULL;
-		} else {
-			pipe_staged[pstates[cnt].pipe_id] = pstate;
-		}
-
-		cnt++;
-
-		dst = drm_plane_state_dest(pstate);
-		if (!drm_rect_intersect(&clip, &dst)) {
-			DPU_ERROR("invalid vertical/horizontal destination\n");
-			DPU_ERROR("display: " DRM_RECT_FMT " plane: "
-				  DRM_RECT_FMT "\n", DRM_RECT_ARG(&crtc_rect),
-				  DRM_RECT_ARG(&dst));
-			rc = -E2BIG;
-			goto end;
-		}
-	}
-
-	for (i = 1; i < SSPP_MAX; i++) {
-		if (pipe_staged[i])
-			dpu_plane_clear_multirect(pipe_staged[i]);
-	}
-
-	z_pos = -1;
-	for (i = 0; i < cnt; i++) {
-		/* reset counts at every new blend stage */
-		if (pstates[i].stage != z_pos) {
-			left_zpos_cnt = 0;
-			right_zpos_cnt = 0;
-			z_pos = pstates[i].stage;
-		}
-
-		/* verify z_pos setting before using it */
-		if (z_pos >= DPU_STAGE_MAX - DPU_STAGE_0) {
-			DPU_ERROR("> %d plane stages assigned\n",
-					DPU_STAGE_MAX - DPU_STAGE_0);
-			rc = -EINVAL;
-			goto end;
-		} else if (pstates[i].drm_pstate->crtc_x < mixer_width) {
-			if (left_zpos_cnt == 2) {
-				DPU_ERROR("> 2 planes @ stage %d on left\n",
-					z_pos);
-				rc = -EINVAL;
-				goto end;
-			}
-			left_zpos_cnt++;
-
-		} else {
-			if (right_zpos_cnt == 2) {
-				DPU_ERROR("> 2 planes @ stage %d on right\n",
-					z_pos);
-				rc = -EINVAL;
-				goto end;
-			}
-			right_zpos_cnt++;
-		}
-
-		pstates[i].dpu_pstate->stage = z_pos + DPU_STAGE_0;
-		DRM_DEBUG_ATOMIC("%s: zpos %d\n", dpu_crtc->name, z_pos);
-	}
-
-	for (i = 0; i < multirect_count; i++) {
-		if (dpu_plane_validate_multirect_v2(&multirect_plane[i])) {
-			DPU_ERROR(
-			"multirect validation failed for planes (%d - %d)\n",
-					multirect_plane[i].r0->plane->base.id,
-					multirect_plane[i].r1->plane->base.id);
-			rc = -EINVAL;
-			goto end;
-		}
 	}
 
 	atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
@@ -1290,74 +1234,10 @@ static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
 	if (rc) {
 		DPU_ERROR("crtc%d failed performance check %d\n",
 				crtc->base.id, rc);
-		goto end;
+		return rc;
 	}
 
-	/* validate source split:
-	 * use pstates sorted by stage to check planes on same stage
-	 * we assume that all pipes are in source split so its valid to compare
-	 * without taking into account left/right mixer placement
-	 */
-	for (i = 1; i < cnt; i++) {
-		struct plane_state *prv_pstate, *cur_pstate;
-		struct drm_rect left_rect, right_rect;
-		int32_t left_pid, right_pid;
-		int32_t stage;
-
-		prv_pstate = &pstates[i - 1];
-		cur_pstate = &pstates[i];
-		if (prv_pstate->stage != cur_pstate->stage)
-			continue;
-
-		stage = cur_pstate->stage;
-
-		left_pid = prv_pstate->dpu_pstate->base.plane->base.id;
-		left_rect = drm_plane_state_dest(prv_pstate->drm_pstate);
-
-		right_pid = cur_pstate->dpu_pstate->base.plane->base.id;
-		right_rect = drm_plane_state_dest(cur_pstate->drm_pstate);
-
-		if (right_rect.x1 < left_rect.x1) {
-			swap(left_pid, right_pid);
-			swap(left_rect, right_rect);
-		}
-
-		/**
-		 * - planes are enumerated in pipe-priority order such that
-		 *   planes with lower drm_id must be left-most in a shared
-		 *   blend-stage when using source split.
-		 * - planes in source split must be contiguous in width
-		 * - planes in source split must have same dest yoff and height
-		 */
-		if (right_pid < left_pid) {
-			DPU_ERROR(
-				"invalid src split cfg. priority mismatch. stage: %d left: %d right: %d\n",
-				stage, left_pid, right_pid);
-			rc = -EINVAL;
-			goto end;
-		} else if (right_rect.x1 != drm_rect_width(&left_rect)) {
-			DPU_ERROR("non-contiguous coordinates for src split. "
-				  "stage: %d left: " DRM_RECT_FMT " right: "
-				  DRM_RECT_FMT "\n", stage,
-				  DRM_RECT_ARG(&left_rect),
-				  DRM_RECT_ARG(&right_rect));
-			rc = -EINVAL;
-			goto end;
-		} else if (left_rect.y1 != right_rect.y1 ||
-			   drm_rect_height(&left_rect) != drm_rect_height(&right_rect)) {
-			DPU_ERROR("source split at stage: %d. invalid "
-				  "yoff/height: left: " DRM_RECT_FMT " right: "
-				  DRM_RECT_FMT "\n", stage,
-				  DRM_RECT_ARG(&left_rect),
-				  DRM_RECT_ARG(&right_rect));
-			rc = -EINVAL;
-			goto end;
-		}
-	}
-
-end:
-	kfree(pstates);
-	return rc;
+	return 0;
 }
 
 int dpu_crtc_vblank(struct drm_crtc *crtc, bool en)
@@ -1474,8 +1354,16 @@ static int _dpu_debugfs_status_show(struct seq_file *s, void *data)
 		seq_printf(s, "\tdst x:%4d dst_y:%4d dst_w:%4d dst_h:%4d\n",
 			state->crtc_x, state->crtc_y, state->crtc_w,
 			state->crtc_h);
-		seq_printf(s, "\tmultirect: mode: %d index: %d\n",
-			pstate->multirect_mode, pstate->multirect_index);
+		seq_printf(s, "\tsspp[0]:%s\n",
+			   pstate->pipe.sspp->cap->name);
+		seq_printf(s, "\tmultirect[0]: mode: %d index: %d\n",
+			pstate->pipe.multirect_mode, pstate->pipe.multirect_index);
+		if (pstate->r_pipe.sspp) {
+			seq_printf(s, "\tsspp[1]:%s\n",
+				   pstate->r_pipe.sspp->cap->name);
+			seq_printf(s, "\tmultirect[1]: mode: %d index: %d\n",
+				   pstate->r_pipe.multirect_mode, pstate->r_pipe.multirect_index);
+		}
 
 		seq_puts(s, "\n");
 	}
@@ -1577,7 +1465,7 @@ struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane,
 {
 	struct drm_crtc *crtc = NULL;
 	struct dpu_crtc *dpu_crtc = NULL;
-	int i;
+	int i, ret;
 
 	dpu_crtc = kzalloc(sizeof(*dpu_crtc), GFP_KERNEL);
 	if (!dpu_crtc)
@@ -1614,6 +1502,13 @@ struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane,
 	/* initialize event handling */
 	spin_lock_init(&dpu_crtc->event_lock);
 
+	ret = drm_self_refresh_helper_init(crtc);
+	if (ret) {
+		DPU_ERROR("Failed to initialize %s with self-refresh helpers %d\n",
+			crtc->name, ret);
+		return ERR_PTR(ret);
+	}
+
 	DRM_DEBUG_KMS("%s: successfully initialized crtc\n", dpu_crtc->name);
 	return crtc;
 }
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
index 758261e8ac73..1dc5dbe58572 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
@@ -12,6 +12,7 @@
 #include <linux/kthread.h>
 #include <linux/seq_file.h>
 
+#include <drm/drm_atomic.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_file.h>
 #include <drm/drm_probe_helper.h>
@@ -544,7 +545,8 @@ bool dpu_encoder_use_dsc_merge(struct drm_encoder *drm_enc)
 static struct msm_display_topology dpu_encoder_get_topology(
 			struct dpu_encoder_virt *dpu_enc,
 			struct dpu_kms *dpu_kms,
-			struct drm_display_mode *mode)
+			struct drm_display_mode *mode,
+			struct drm_crtc_state *crtc_state)
 {
 	struct msm_display_topology topology = {0};
 	int i, intf_count = 0;
@@ -562,8 +564,7 @@ static struct msm_display_topology dpu_encoder_get_topology(
 	 * 1 LM, 1 INTF
 	 * 2 LM, 1 INTF (stream merge to support high resolution interfaces)
 	 *
-	 * Adding color blocks only to primary interface if available in
-	 * sufficient number
+	 * Add dspps to the reservation requirements if ctm is requested
 	 */
 	if (intf_count == 2)
 		topology.num_lm = 2;
@@ -572,11 +573,8 @@ static struct msm_display_topology dpu_encoder_get_topology(
 	else
 		topology.num_lm = (mode->hdisplay > MAX_HDISPLAY_SPLIT) ? 2 : 1;
 
-	if (dpu_enc->disp_info.intf_type == DRM_MODE_ENCODER_DSI) {
-		if (dpu_kms->catalog->dspp &&
-			(dpu_kms->catalog->dspp_count >= topology.num_lm))
-			topology.num_dspp = topology.num_lm;
-	}
+	if (crtc_state->ctm)
+		topology.num_dspp = topology.num_lm;
 
 	topology.num_intf = intf_count;
 
@@ -637,25 +635,22 @@ static int dpu_encoder_virt_atomic_check(
 		if (ret) {
 			DPU_ERROR_ENC(dpu_enc,
 					"mode unsupported, phys idx %d\n", i);
-			break;
+			return ret;
 		}
 	}
 
-	topology = dpu_encoder_get_topology(dpu_enc, dpu_kms, adj_mode);
+	topology = dpu_encoder_get_topology(dpu_enc, dpu_kms, adj_mode, crtc_state);
 
-	/* Reserve dynamic resources now. */
-	if (!ret) {
-		/*
-		 * Release and Allocate resources on every modeset
-		 * Dont allocate when active is false.
-		 */
-		if (drm_atomic_crtc_needs_modeset(crtc_state)) {
-			dpu_rm_release(global_state, drm_enc);
+	/*
+	 * Release and Allocate resources on every modeset
+	 * Dont allocate when active is false.
+	 */
+	if (drm_atomic_crtc_needs_modeset(crtc_state)) {
+		dpu_rm_release(global_state, drm_enc);
 
-			if (!crtc_state->active_changed || crtc_state->active)
-				ret = dpu_rm_reserve(&dpu_kms->rm, global_state,
-						drm_enc, crtc_state, topology);
-		}
+		if (!crtc_state->active_changed || crtc_state->enable)
+			ret = dpu_rm_reserve(&dpu_kms->rm, global_state,
+					drm_enc, crtc_state, topology);
 	}
 
 	trace_dpu_enc_atomic_check_flags(DRMID(drm_enc), adj_mode->flags);
@@ -1171,7 +1166,8 @@ out:
 	mutex_unlock(&dpu_enc->enc_lock);
 }
 
-static void dpu_encoder_virt_enable(struct drm_encoder *drm_enc)
+static void dpu_encoder_virt_atomic_enable(struct drm_encoder *drm_enc,
+					struct drm_atomic_state *state)
 {
 	struct dpu_encoder_virt *dpu_enc = NULL;
 	int ret = 0;
@@ -1207,14 +1203,28 @@ out:
 	mutex_unlock(&dpu_enc->enc_lock);
 }
 
-static void dpu_encoder_virt_disable(struct drm_encoder *drm_enc)
+static void dpu_encoder_virt_atomic_disable(struct drm_encoder *drm_enc,
+					struct drm_atomic_state *state)
 {
 	struct dpu_encoder_virt *dpu_enc = NULL;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_state = NULL;
 	int i = 0;
 
 	dpu_enc = to_dpu_encoder_virt(drm_enc);
 	DPU_DEBUG_ENC(dpu_enc, "\n");
 
+	crtc = drm_atomic_get_old_crtc_for_encoder(state, drm_enc);
+	if (crtc)
+		old_state = drm_atomic_get_old_crtc_state(state, crtc);
+
+	/*
+	 * The encoder is already disabled if self refresh mode was set earlier,
+	 * in the old_state for the corresponding crtc.
+	 */
+	if (old_state && old_state->self_refresh_active)
+		return;
+
 	mutex_lock(&dpu_enc->enc_lock);
 	dpu_enc->enabled = false;
 
@@ -2078,25 +2088,6 @@ void dpu_encoder_helper_phys_cleanup(struct dpu_encoder_phys *phys_enc)
 	ctl->ops.clear_pending_flush(ctl);
 }
 
-void dpu_encoder_prepare_commit(struct drm_encoder *drm_enc)
-{
-	struct dpu_encoder_virt *dpu_enc;
-	struct dpu_encoder_phys *phys;
-	int i;
-
-	if (!drm_enc) {
-		DPU_ERROR("invalid encoder\n");
-		return;
-	}
-	dpu_enc = to_dpu_encoder_virt(drm_enc);
-
-	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
-		phys = dpu_enc->phys_encs[i];
-		if (phys->ops.prepare_commit)
-			phys->ops.prepare_commit(phys);
-	}
-}
-
 #ifdef CONFIG_DEBUG_FS
 static int _dpu_encoder_status_show(struct seq_file *s, void *data)
 {
@@ -2388,8 +2379,8 @@ static void dpu_encoder_frame_done_timeout(struct timer_list *t)
 
 static const struct drm_encoder_helper_funcs dpu_encoder_helper_funcs = {
 	.atomic_mode_set = dpu_encoder_virt_atomic_mode_set,
-	.disable = dpu_encoder_virt_disable,
-	.enable = dpu_encoder_virt_enable,
+	.atomic_disable = dpu_encoder_virt_atomic_disable,
+	.atomic_enable = dpu_encoder_virt_atomic_enable,
 	.atomic_check = dpu_encoder_virt_atomic_check,
 };
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
index 9e7236ef34e6..2c9ef8d1b877 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
@@ -147,13 +147,6 @@ int dpu_encoder_setup(struct drm_device *dev, struct drm_encoder *enc,
 		struct msm_display_info *disp_info);
 
 /**
- * dpu_encoder_prepare_commit - prepare encoder at the very beginning of an
- *	atomic commit, before any registers are written
- * @drm_enc:    Pointer to previously created drm encoder structure
- */
-void dpu_encoder_prepare_commit(struct drm_encoder *drm_enc);
-
-/**
  * dpu_encoder_set_idle_timeout - set the idle timeout for video
  *                    and command mode encoders.
  * @drm_enc:    Pointer to previously created drm encoder structure
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
index c8f4a62a9536..74470d068622 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
@@ -40,6 +40,8 @@
 
 #define DPU_ENC_MAX_POLL_TIMEOUT_US	2000
 
+static void dpu_encoder_phys_cmd_enable_te(struct dpu_encoder_phys *phys_enc);
+
 static bool dpu_encoder_phys_cmd_is_master(struct dpu_encoder_phys *phys_enc)
 {
 	return (phys_enc->split_role != ENC_ROLE_SLAVE);
@@ -565,6 +567,8 @@ static void dpu_encoder_phys_cmd_prepare_for_kickoff(
 			  phys_enc->hw_pp->idx - PINGPONG_0);
 	}
 
+	dpu_encoder_phys_cmd_enable_te(phys_enc);
+
 	DPU_DEBUG_CMDENC(cmd_enc, "pp:%d pending_cnt %d\n",
 			phys_enc->hw_pp->idx - PINGPONG_0,
 			atomic_read(&phys_enc->pending_kickoff_cnt));
@@ -586,8 +590,7 @@ static bool dpu_encoder_phys_cmd_is_ongoing_pptx(
 	return false;
 }
 
-static void dpu_encoder_phys_cmd_prepare_commit(
-		struct dpu_encoder_phys *phys_enc)
+static void dpu_encoder_phys_cmd_enable_te(struct dpu_encoder_phys *phys_enc)
 {
 	struct dpu_encoder_phys_cmd *cmd_enc =
 		to_dpu_encoder_phys_cmd(phys_enc);
@@ -732,7 +735,6 @@ static void dpu_encoder_phys_cmd_trigger_start(
 static void dpu_encoder_phys_cmd_init_ops(
 		struct dpu_encoder_phys_ops *ops)
 {
-	ops->prepare_commit = dpu_encoder_phys_cmd_prepare_commit;
 	ops->is_master = dpu_encoder_phys_cmd_is_master;
 	ops->atomic_mode_set = dpu_encoder_phys_cmd_atomic_mode_set;
 	ops->enable = dpu_encoder_phys_cmd_enable;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
index 48c48106b16a..3a374292f311 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
@@ -523,6 +523,7 @@ static void dpu_encoder_phys_vid_disable(struct dpu_encoder_phys *phys_enc)
 {
 	unsigned long lock_flags;
 	int ret;
+	struct intf_status intf_status = {0};
 
 	if (!phys_enc->parent || !phys_enc->parent->dev) {
 		DPU_ERROR("invalid encoder/device\n");
@@ -567,6 +568,27 @@ static void dpu_encoder_phys_vid_disable(struct dpu_encoder_phys *phys_enc)
 		}
 	}
 
+	if (phys_enc->hw_intf && phys_enc->hw_intf->ops.get_status)
+		phys_enc->hw_intf->ops.get_status(phys_enc->hw_intf, &intf_status);
+
+	/*
+	 * Wait for a vsync if timing en status is on after timing engine
+	 * is disabled.
+	 */
+	if (intf_status.is_en && dpu_encoder_phys_vid_is_master(phys_enc)) {
+		spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
+		dpu_encoder_phys_inc_pending(phys_enc);
+		spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
+		ret = dpu_encoder_phys_vid_wait_for_vblank(phys_enc);
+		if (ret) {
+			atomic_set(&phys_enc->pending_kickoff_cnt, 0);
+			DRM_ERROR("wait disable failed: id:%u intf:%d ret:%d\n",
+				  DRMID(phys_enc->parent),
+				  phys_enc->hw_intf->idx - INTF_0, ret);
+		}
+	}
+
+	dpu_encoder_helper_phys_cleanup(phys_enc);
 	phys_enc->enable_state = DPU_ENC_DISABLED;
 }
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
index d95540309d4d..e366ab134249 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
@@ -536,6 +536,16 @@ static const struct dpu_format dpu_format_map_ubwc[] = {
 		true, 4, DPU_FORMAT_FLAG_DX | DPU_FORMAT_FLAG_COMPRESSED,
 		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
 
+	/* XRGB2101010 and ARGB2101010 purposely have the same color
+	* ordering.  The hardware only supports ARGB2101010 UBWC
+	* natively.
+	*/
+	INTERLEAVED_RGB_FMT_TILED(ARGB2101010,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, DPU_FORMAT_FLAG_DX | DPU_FORMAT_FLAG_COMPRESSED,
+		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
+
 	PSEUDO_YUV_FMT_TILED(NV12,
 		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
 		C1_B_Cb, C2_R_Cr,
@@ -591,6 +601,7 @@ static int _dpu_format_get_media_color_ubwc(const struct dpu_format *fmt)
 		{DRM_FORMAT_XBGR8888, COLOR_FMT_RGBA8888_UBWC},
 		{DRM_FORMAT_XRGB8888, COLOR_FMT_RGBA8888_UBWC},
 		{DRM_FORMAT_ABGR2101010, COLOR_FMT_RGBA1010102_UBWC},
+		{DRM_FORMAT_ARGB2101010, COLOR_FMT_RGBA1010102_UBWC},
 		{DRM_FORMAT_XRGB2101010, COLOR_FMT_RGBA1010102_UBWC},
 		{DRM_FORMAT_XBGR2101010, COLOR_FMT_RGBA1010102_UBWC},
 		{DRM_FORMAT_BGR565, COLOR_FMT_RGB565_UBWC},
@@ -918,8 +929,7 @@ int dpu_format_populate_layout(
 		struct drm_framebuffer *fb,
 		struct dpu_hw_fmt_layout *layout)
 {
-	uint32_t plane_addr[DPU_MAX_PLANES];
-	int i, ret;
+	int ret;
 
 	if (!fb || !layout) {
 		DRM_ERROR("invalid arguments\n");
@@ -940,9 +950,6 @@ int dpu_format_populate_layout(
 	if (ret)
 		return ret;
 
-	for (i = 0; i < DPU_MAX_PLANES; ++i)
-		plane_addr[i] = layout->plane_addr[i];
-
 	/* Populate the addresses given the fb */
 	if (DPU_FORMAT_IS_UBWC(layout->format) ||
 			DPU_FORMAT_IS_TILE(layout->format))
@@ -950,10 +957,6 @@ int dpu_format_populate_layout(
 	else
 		ret = _dpu_format_populate_addrs_linear(aspace, fb, layout);
 
-	/* check if anything changed */
-	if (!ret && !memcmp(plane_addr, layout->plane_addr, sizeof(plane_addr)))
-		ret = -EAGAIN;
-
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
index 497c9e1673ab..03f162af1a50 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
@@ -27,9 +27,15 @@
 #define VIG_SDM845_MASK \
 	(VIG_MASK | BIT(DPU_SSPP_QOS_8LVL) | BIT(DPU_SSPP_SCALER_QSEED3))
 
+#define VIG_SDM845_MASK_SDMA \
+	(VIG_SDM845_MASK | BIT(DPU_SSPP_SMART_DMA_V2))
+
 #define VIG_SC7180_MASK \
 	(VIG_MASK | BIT(DPU_SSPP_QOS_8LVL) | BIT(DPU_SSPP_SCALER_QSEED4))
 
+#define VIG_SC7180_MASK_SDMA \
+	(VIG_SC7180_MASK | BIT(DPU_SSPP_SMART_DMA_V2))
+
 #define VIG_QCM2290_MASK (VIG_BASE_MASK | BIT(DPU_SSPP_QOS_8LVL))
 
 #define DMA_MSM8998_MASK \
@@ -40,6 +46,9 @@
 #define VIG_SC7280_MASK \
 	(VIG_SC7180_MASK | BIT(DPU_SSPP_INLINE_ROTATION))
 
+#define VIG_SC7280_MASK_SDMA \
+	(VIG_SC7280_MASK | BIT(DPU_SSPP_SMART_DMA_V2))
+
 #define DMA_SDM845_MASK \
 	(BIT(DPU_SSPP_SRC) | BIT(DPU_SSPP_QOS) | BIT(DPU_SSPP_QOS_8LVL) |\
 	BIT(DPU_SSPP_TS_PREFILL) | BIT(DPU_SSPP_TS_PREFILL_REC1) |\
@@ -48,6 +57,12 @@
 #define DMA_CURSOR_SDM845_MASK \
 	(DMA_SDM845_MASK | BIT(DPU_SSPP_CURSOR))
 
+#define DMA_SDM845_MASK_SDMA \
+	(DMA_SDM845_MASK | BIT(DPU_SSPP_SMART_DMA_V2))
+
+#define DMA_CURSOR_SDM845_MASK_SDMA \
+	(DMA_CURSOR_SDM845_MASK | BIT(DPU_SSPP_SMART_DMA_V2))
+
 #define DMA_CURSOR_MSM8998_MASK \
 	(DMA_MSM8998_MASK | BIT(DPU_SSPP_CURSOR))
 
@@ -66,7 +81,10 @@
 	(PINGPONG_SDM845_MASK | BIT(DPU_PINGPONG_TE2))
 
 #define CTL_SC7280_MASK \
-	(BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_FETCH_ACTIVE) | BIT(DPU_CTL_VM_CFG))
+	(BIT(DPU_CTL_ACTIVE_CFG) | \
+	 BIT(DPU_CTL_FETCH_ACTIVE) | \
+	 BIT(DPU_CTL_VM_CFG) | \
+	 BIT(DPU_CTL_DSPP_SUB_BLOCK_FLUSH))
 
 #define CTL_SM8550_MASK \
 	(CTL_SC7280_MASK | BIT(DPU_CTL_HAS_LAYER_EXT4))
@@ -79,83 +97,11 @@
 
 #define INTF_SDM845_MASK (0)
 
-#define INTF_SC7180_MASK BIT(DPU_INTF_INPUT_CTRL) | BIT(DPU_INTF_TE)
+#define INTF_SC7180_MASK \
+	(BIT(DPU_INTF_INPUT_CTRL) | BIT(DPU_INTF_TE) | BIT(DPU_INTF_STATUS_SUPPORTED))
 
 #define INTF_SC7280_MASK INTF_SC7180_MASK | BIT(DPU_DATA_HCTL_EN)
 
-#define IRQ_SDM845_MASK (BIT(MDP_SSPP_TOP0_INTR) | \
-			 BIT(MDP_SSPP_TOP0_INTR2) | \
-			 BIT(MDP_SSPP_TOP0_HIST_INTR) | \
-			 BIT(MDP_INTF0_INTR) | \
-			 BIT(MDP_INTF1_INTR) | \
-			 BIT(MDP_INTF2_INTR) | \
-			 BIT(MDP_INTF3_INTR) | \
-			 BIT(MDP_AD4_0_INTR) | \
-			 BIT(MDP_AD4_1_INTR))
-
-#define IRQ_SC7180_MASK (BIT(MDP_SSPP_TOP0_INTR) | \
-			 BIT(MDP_SSPP_TOP0_INTR2) | \
-			 BIT(MDP_SSPP_TOP0_HIST_INTR) | \
-			 BIT(MDP_INTF0_INTR) | \
-			 BIT(MDP_INTF1_INTR))
-
-#define IRQ_SC7280_MASK (BIT(MDP_SSPP_TOP0_INTR) | \
-			 BIT(MDP_SSPP_TOP0_INTR2) | \
-			 BIT(MDP_SSPP_TOP0_HIST_INTR) | \
-			 BIT(MDP_INTF0_7xxx_INTR) | \
-			 BIT(MDP_INTF1_7xxx_INTR) | \
-			 BIT(MDP_INTF5_7xxx_INTR))
-
-#define IRQ_SM8250_MASK (BIT(MDP_SSPP_TOP0_INTR) | \
-			 BIT(MDP_SSPP_TOP0_INTR2) | \
-			 BIT(MDP_SSPP_TOP0_HIST_INTR) | \
-			 BIT(MDP_INTF0_INTR) | \
-			 BIT(MDP_INTF1_INTR) | \
-			 BIT(MDP_INTF2_INTR) | \
-			 BIT(MDP_INTF3_INTR) | \
-			 BIT(MDP_INTF4_INTR))
-
-#define IRQ_SM8350_MASK (BIT(MDP_SSPP_TOP0_INTR) | \
-			 BIT(MDP_SSPP_TOP0_INTR2) | \
-			 BIT(MDP_SSPP_TOP0_HIST_INTR) | \
-			 BIT(MDP_INTF0_7xxx_INTR) | \
-			 BIT(MDP_INTF1_7xxx_INTR) | \
-			 BIT(MDP_INTF2_7xxx_INTR) | \
-			 BIT(MDP_INTF3_7xxx_INTR))
-
-#define IRQ_SC8180X_MASK (BIT(MDP_SSPP_TOP0_INTR) | \
-			  BIT(MDP_SSPP_TOP0_INTR2) | \
-			  BIT(MDP_SSPP_TOP0_HIST_INTR) | \
-			  BIT(MDP_INTF0_INTR) | \
-			  BIT(MDP_INTF1_INTR) | \
-			  BIT(MDP_INTF2_INTR) | \
-			  BIT(MDP_INTF3_INTR) | \
-			  BIT(MDP_INTF4_INTR) | \
-			  BIT(MDP_INTF5_INTR) | \
-			  BIT(MDP_AD4_0_INTR) | \
-			  BIT(MDP_AD4_1_INTR))
-
-#define IRQ_SC8280XP_MASK (BIT(MDP_SSPP_TOP0_INTR) | \
-			   BIT(MDP_SSPP_TOP0_INTR2) | \
-			   BIT(MDP_SSPP_TOP0_HIST_INTR) | \
-			   BIT(MDP_INTF0_7xxx_INTR) | \
-			   BIT(MDP_INTF1_7xxx_INTR) | \
-			   BIT(MDP_INTF2_7xxx_INTR) | \
-			   BIT(MDP_INTF3_7xxx_INTR) | \
-			   BIT(MDP_INTF4_7xxx_INTR) | \
-			   BIT(MDP_INTF5_7xxx_INTR) | \
-			   BIT(MDP_INTF6_7xxx_INTR) | \
-			   BIT(MDP_INTF7_7xxx_INTR) | \
-			   BIT(MDP_INTF8_7xxx_INTR))
-
-#define IRQ_SM8450_MASK (BIT(MDP_SSPP_TOP0_INTR) | \
-			 BIT(MDP_SSPP_TOP0_INTR2) | \
-			 BIT(MDP_SSPP_TOP0_HIST_INTR) | \
-			 BIT(MDP_INTF0_7xxx_INTR) | \
-			 BIT(MDP_INTF1_7xxx_INTR) | \
-			 BIT(MDP_INTF2_7xxx_INTR) | \
-			 BIT(MDP_INTF3_7xxx_INTR))
-
 #define WB_SM8250_MASK (BIT(DPU_WB_LINE_MODE) | \
 			 BIT(DPU_WB_UBWC) | \
 			 BIT(DPU_WB_YUV_CONFIG) | \
@@ -188,6 +134,7 @@ static const uint32_t plane_formats[] = {
 	DRM_FORMAT_RGBX8888,
 	DRM_FORMAT_BGRX8888,
 	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ARGB2101010,
 	DRM_FORMAT_XRGB2101010,
 	DRM_FORMAT_RGB888,
 	DRM_FORMAT_BGR888,
@@ -217,6 +164,7 @@ static const uint32_t plane_formats_yuv[] = {
 	DRM_FORMAT_RGBA8888,
 	DRM_FORMAT_BGRX8888,
 	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_ARGB2101010,
 	DRM_FORMAT_XRGB2101010,
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_XBGR8888,
@@ -295,789 +243,6 @@ static const uint32_t wb2_formats[] = {
 };
 
 /*************************************************************
- * DPU sub blocks config
- *************************************************************/
-/* DPU top level caps */
-static const struct dpu_caps msm8998_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.max_mixer_blendstages = 0x7,
-	.qseed_type = DPU_SSPP_SCALER_QSEED3,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V1,
-	.ubwc_version = DPU_HW_UBWC_VER_10,
-	.has_src_split = true,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.has_3d_merge = true,
-	.max_linewidth = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-	.max_hdeci_exp = MAX_HORZ_DECIMATION,
-	.max_vdeci_exp = MAX_VERT_DECIMATION,
-};
-
-static const struct dpu_caps qcm2290_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_LINE_WIDTH,
-	.max_mixer_blendstages = 0x4,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.max_linewidth = 2160,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-};
-
-static const struct dpu_caps sdm845_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.max_mixer_blendstages = 0xb,
-	.qseed_type = DPU_SSPP_SCALER_QSEED3,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2,
-	.ubwc_version = DPU_HW_UBWC_VER_20,
-	.has_src_split = true,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.has_3d_merge = true,
-	.max_linewidth = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-	.max_hdeci_exp = MAX_HORZ_DECIMATION,
-	.max_vdeci_exp = MAX_VERT_DECIMATION,
-};
-
-static const struct dpu_caps sc7180_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.max_mixer_blendstages = 0x9,
-	.qseed_type = DPU_SSPP_SCALER_QSEED4,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2,
-	.ubwc_version = DPU_HW_UBWC_VER_20,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.max_linewidth = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-};
-
-static const struct dpu_caps sm6115_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_LINE_WIDTH,
-	.max_mixer_blendstages = 0x4,
-	.qseed_type = DPU_SSPP_SCALER_QSEED4,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2, /* TODO: v2.5 */
-	.ubwc_version = DPU_HW_UBWC_VER_10,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.max_linewidth = 2160,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-};
-
-static const struct dpu_caps sm8150_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.max_mixer_blendstages = 0xb,
-	.qseed_type = DPU_SSPP_SCALER_QSEED3,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2, /* TODO: v2.5 */
-	.ubwc_version = DPU_HW_UBWC_VER_30,
-	.has_src_split = true,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.has_3d_merge = true,
-	.max_linewidth = 4096,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-	.max_hdeci_exp = MAX_HORZ_DECIMATION,
-	.max_vdeci_exp = MAX_VERT_DECIMATION,
-};
-
-static const struct dpu_caps sc8180x_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.max_mixer_blendstages = 0xb,
-	.qseed_type = DPU_SSPP_SCALER_QSEED3,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2, /* TODO: v2.5 */
-	.ubwc_version = DPU_HW_UBWC_VER_30,
-	.has_src_split = true,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.has_3d_merge = true,
-	.max_linewidth = 4096,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-	.max_hdeci_exp = MAX_HORZ_DECIMATION,
-	.max_vdeci_exp = MAX_VERT_DECIMATION,
-};
-
-static const struct dpu_caps sc8280xp_dpu_caps = {
-	.max_mixer_width = 2560,
-	.max_mixer_blendstages = 11,
-	.qseed_type = DPU_SSPP_SCALER_QSEED4,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2, /* TODO: v2.5 */
-	.ubwc_version = DPU_HW_UBWC_VER_40,
-	.has_src_split = true,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.has_3d_merge = true,
-	.max_linewidth = 5120,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-};
-
-static const struct dpu_caps sm8250_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.max_mixer_blendstages = 0xb,
-	.qseed_type = DPU_SSPP_SCALER_QSEED4,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2, /* TODO: v2.5 */
-	.ubwc_version = DPU_HW_UBWC_VER_40,
-	.has_src_split = true,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.has_3d_merge = true,
-	.max_linewidth = 4096,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-};
-
-static const struct dpu_caps sm8350_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.max_mixer_blendstages = 0xb,
-	.qseed_type = DPU_SSPP_SCALER_QSEED4,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2, /* TODO: v2.5 */
-	.ubwc_version = DPU_HW_UBWC_VER_40,
-	.has_src_split = true,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.has_3d_merge = true,
-	.max_linewidth = 4096,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-};
-
-static const struct dpu_caps sm8450_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.max_mixer_blendstages = 0xb,
-	.qseed_type = DPU_SSPP_SCALER_QSEED4,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2, /* TODO: v2.5 */
-	.ubwc_version = DPU_HW_UBWC_VER_40,
-	.has_src_split = true,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.has_3d_merge = true,
-	.max_linewidth = 5120,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-};
-
-static const struct dpu_caps sm8550_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.max_mixer_blendstages = 0xb,
-	.qseed_type = DPU_SSPP_SCALER_QSEED4,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2, /* TODO: v2.5 */
-	.ubwc_version = DPU_HW_UBWC_VER_40,
-	.has_src_split = true,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.has_3d_merge = true,
-	.max_linewidth = 5120,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-};
-
-static const struct dpu_caps sc7280_dpu_caps = {
-	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
-	.max_mixer_blendstages = 0x7,
-	.qseed_type = DPU_SSPP_SCALER_QSEED4,
-	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2,
-	.ubwc_version = DPU_HW_UBWC_VER_30,
-	.has_dim_layer = true,
-	.has_idle_pc = true,
-	.max_linewidth = 2400,
-	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
-};
-
-static const struct dpu_mdp_cfg msm8998_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x458,
-	.features = 0,
-	.highest_bank_bit = 0x2,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-			.reg_off = 0x2AC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG1] = {
-			.reg_off = 0x2B4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG2] = {
-			.reg_off = 0x2BC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG3] = {
-			.reg_off = 0x2C4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-			.reg_off = 0x2AC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA1] = {
-			.reg_off = 0x2B4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA2] = {
-			.reg_off = 0x2C4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA3] = {
-			.reg_off = 0x2C4, .bit_off = 12},
-	.clk_ctrls[DPU_CLK_CTRL_CURSOR0] = {
-			.reg_off = 0x3A8, .bit_off = 15},
-	.clk_ctrls[DPU_CLK_CTRL_CURSOR1] = {
-			.reg_off = 0x3B0, .bit_off = 15},
-	},
-};
-
-static const struct dpu_mdp_cfg sdm845_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x45C,
-	.features = BIT(DPU_MDP_AUDIO_SELECT),
-	.highest_bank_bit = 0x2,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-			.reg_off = 0x2AC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG1] = {
-			.reg_off = 0x2B4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG2] = {
-			.reg_off = 0x2BC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG3] = {
-			.reg_off = 0x2C4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-			.reg_off = 0x2AC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA1] = {
-			.reg_off = 0x2B4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA2] = {
-			.reg_off = 0x2BC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA3] = {
-			.reg_off = 0x2C4, .bit_off = 8},
-	},
-};
-
-static const struct dpu_mdp_cfg sc7180_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x494,
-	.features = 0,
-	.highest_bank_bit = 0x3,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-		.reg_off = 0x2AC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-		.reg_off = 0x2AC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA1] = {
-		.reg_off = 0x2B4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA2] = {
-		.reg_off = 0x2C4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_WB2] = {
-		.reg_off = 0x3B8, .bit_off = 24},
-	},
-};
-
-static const struct dpu_mdp_cfg sc8180x_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x45C,
-	.features = BIT(DPU_MDP_AUDIO_SELECT),
-	.highest_bank_bit = 0x3,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-			.reg_off = 0x2AC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG1] = {
-			.reg_off = 0x2B4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG2] = {
-			.reg_off = 0x2BC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG3] = {
-			.reg_off = 0x2C4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-			.reg_off = 0x2AC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA1] = {
-			.reg_off = 0x2B4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA2] = {
-			.reg_off = 0x2BC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA3] = {
-			.reg_off = 0x2C4, .bit_off = 8},
-	},
-};
-
-static const struct dpu_mdp_cfg sm6115_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x494,
-	.features = 0,
-	.highest_bank_bit = 0x1,
-	.ubwc_swizzle = 0x7,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-		.reg_off = 0x2ac, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-		.reg_off = 0x2ac, .bit_off = 8},
-	},
-};
-
-static const struct dpu_mdp_cfg sm8250_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x494,
-	.features = 0,
-	.highest_bank_bit = 0x3, /* TODO: 2 for LP_DDR4 */
-	.ubwc_swizzle = 0x6,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-			.reg_off = 0x2AC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG1] = {
-			.reg_off = 0x2B4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG2] = {
-			.reg_off = 0x2BC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG3] = {
-			.reg_off = 0x2C4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-			.reg_off = 0x2AC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA1] = {
-			.reg_off = 0x2B4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA2] = {
-			.reg_off = 0x2BC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA3] = {
-			.reg_off = 0x2C4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_REG_DMA] = {
-			.reg_off = 0x2BC, .bit_off = 20},
-	.clk_ctrls[DPU_CLK_CTRL_WB2] = {
-			.reg_off = 0x3B8, .bit_off = 24},
-	},
-};
-
-static const struct dpu_mdp_cfg sm8350_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x494,
-	.features = 0,
-	.highest_bank_bit = 0x3, /* TODO: 2 for LP_DDR4 */
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-			.reg_off = 0x2ac, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG1] = {
-			.reg_off = 0x2b4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG2] = {
-			.reg_off = 0x2bc, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG3] = {
-			.reg_off = 0x2c4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-			.reg_off = 0x2ac, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA1] = {
-			.reg_off = 0x2b4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA2] = {
-			.reg_off = 0x2bc, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA3] = {
-			.reg_off = 0x2c4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_REG_DMA] = {
-			.reg_off = 0x2bc, .bit_off = 20},
-	},
-};
-
-static const struct dpu_mdp_cfg sm8450_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x494,
-	.features = BIT(DPU_MDP_PERIPH_0_REMOVED),
-	.highest_bank_bit = 0x3, /* TODO: 2 for LP_DDR4 */
-	.ubwc_swizzle = 0x6,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-			.reg_off = 0x2AC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG1] = {
-			.reg_off = 0x2B4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG2] = {
-			.reg_off = 0x2BC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG3] = {
-			.reg_off = 0x2C4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-			.reg_off = 0x2AC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA1] = {
-			.reg_off = 0x2B4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA2] = {
-			.reg_off = 0x2BC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA3] = {
-			.reg_off = 0x2C4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_REG_DMA] = {
-			.reg_off = 0x2BC, .bit_off = 20},
-	},
-};
-
-static const struct dpu_mdp_cfg sc7280_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x2014,
-	.highest_bank_bit = 0x1,
-	.ubwc_swizzle = 0x6,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-		.reg_off = 0x2AC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-		.reg_off = 0x2AC, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA1] = {
-		.reg_off = 0x2B4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA2] = {
-		.reg_off = 0x2C4, .bit_off = 8},
-	},
-};
-
-static const struct dpu_mdp_cfg sc8280xp_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x494,
-	.features = BIT(DPU_MDP_PERIPH_0_REMOVED),
-	.highest_bank_bit = 2,
-	.ubwc_swizzle = 6,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = { .reg_off = 0x2ac, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG1] = { .reg_off = 0x2b4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG2] = { .reg_off = 0x2bc, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG3] = { .reg_off = 0x2c4, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = { .reg_off = 0x2ac, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA1] = { .reg_off = 0x2b4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA2] = { .reg_off = 0x2bc, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_DMA3] = { .reg_off = 0x2c4, .bit_off = 8},
-	.clk_ctrls[DPU_CLK_CTRL_REG_DMA] = { .reg_off = 0x2bc, .bit_off = 20},
-	},
-};
-
-static const struct dpu_mdp_cfg sm8550_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0, .len = 0x494,
-	.features = BIT(DPU_MDP_PERIPH_0_REMOVED),
-	.highest_bank_bit = 0x3, /* TODO: 2 for LP_DDR4 */
-	.ubwc_swizzle = 0x6,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-			.reg_off = 0x4330, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG1] = {
-			.reg_off = 0x6330, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG2] = {
-			.reg_off = 0x8330, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_VIG3] = {
-			.reg_off = 0xa330, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-			.reg_off = 0x24330, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA1] = {
-			.reg_off = 0x26330, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA2] = {
-			.reg_off = 0x28330, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA3] = {
-			.reg_off = 0x2a330, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA4] = {
-			.reg_off = 0x2c330, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA5] = {
-			.reg_off = 0x2e330, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_REG_DMA] = {
-			.reg_off = 0x2bc, .bit_off = 20},
-	},
-};
-
-static const struct dpu_mdp_cfg qcm2290_mdp[] = {
-	{
-	.name = "top_0", .id = MDP_TOP,
-	.base = 0x0, .len = 0x494,
-	.features = 0,
-	.highest_bank_bit = 0x2,
-	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
-		.reg_off = 0x2AC, .bit_off = 0},
-	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
-		.reg_off = 0x2AC, .bit_off = 8},
-	},
-};
-
-/*************************************************************
- * CTL sub blocks config
- *************************************************************/
-static const struct dpu_ctl_cfg msm8998_ctl[] = {
-	{
-	.name = "ctl_0", .id = CTL_0,
-	.base = 0x1000, .len = 0x94,
-	.features = BIT(DPU_CTL_SPLIT_DISPLAY),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
-	},
-	{
-	.name = "ctl_1", .id = CTL_1,
-	.base = 0x1200, .len = 0x94,
-	.features = 0,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
-	},
-	{
-	.name = "ctl_2", .id = CTL_2,
-	.base = 0x1400, .len = 0x94,
-	.features = BIT(DPU_CTL_SPLIT_DISPLAY),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
-	},
-	{
-	.name = "ctl_3", .id = CTL_3,
-	.base = 0x1600, .len = 0x94,
-	.features = 0,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
-	},
-	{
-	.name = "ctl_4", .id = CTL_4,
-	.base = 0x1800, .len = 0x94,
-	.features = 0,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
-	},
-};
-
-static const struct dpu_ctl_cfg sdm845_ctl[] = {
-	{
-	.name = "ctl_0", .id = CTL_0,
-	.base = 0x1000, .len = 0xE4,
-	.features = BIT(DPU_CTL_SPLIT_DISPLAY),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
-	},
-	{
-	.name = "ctl_1", .id = CTL_1,
-	.base = 0x1200, .len = 0xE4,
-	.features = BIT(DPU_CTL_SPLIT_DISPLAY),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
-	},
-	{
-	.name = "ctl_2", .id = CTL_2,
-	.base = 0x1400, .len = 0xE4,
-	.features = 0,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
-	},
-	{
-	.name = "ctl_3", .id = CTL_3,
-	.base = 0x1600, .len = 0xE4,
-	.features = 0,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
-	},
-	{
-	.name = "ctl_4", .id = CTL_4,
-	.base = 0x1800, .len = 0xE4,
-	.features = 0,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
-	},
-};
-
-static const struct dpu_ctl_cfg sc7180_ctl[] = {
-	{
-	.name = "ctl_0", .id = CTL_0,
-	.base = 0x1000, .len = 0x1dc,
-	.features = BIT(DPU_CTL_ACTIVE_CFG),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
-	},
-	{
-	.name = "ctl_1", .id = CTL_1,
-	.base = 0x1200, .len = 0x1dc,
-	.features = BIT(DPU_CTL_ACTIVE_CFG),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
-	},
-	{
-	.name = "ctl_2", .id = CTL_2,
-	.base = 0x1400, .len = 0x1dc,
-	.features = BIT(DPU_CTL_ACTIVE_CFG),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
-	},
-};
-
-static const struct dpu_ctl_cfg sc8280xp_ctl[] = {
-	{
-	.name = "ctl_0", .id = CTL_0,
-	.base = 0x15000, .len = 0x204,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
-	},
-	{
-	.name = "ctl_1", .id = CTL_1,
-	.base = 0x16000, .len = 0x204,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
-	},
-	{
-	.name = "ctl_2", .id = CTL_2,
-	.base = 0x17000, .len = 0x204,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
-	},
-	{
-	.name = "ctl_3", .id = CTL_3,
-	.base = 0x18000, .len = 0x204,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
-	},
-	{
-	.name = "ctl_4", .id = CTL_4,
-	.base = 0x19000, .len = 0x204,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
-	},
-	{
-	.name = "ctl_5", .id = CTL_5,
-	.base = 0x1a000, .len = 0x204,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
-	},
-};
-
-static const struct dpu_ctl_cfg sm8150_ctl[] = {
-	{
-	.name = "ctl_0", .id = CTL_0,
-	.base = 0x1000, .len = 0x1e0,
-	.features = BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_SPLIT_DISPLAY),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
-	},
-	{
-	.name = "ctl_1", .id = CTL_1,
-	.base = 0x1200, .len = 0x1e0,
-	.features = BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_SPLIT_DISPLAY),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
-	},
-	{
-	.name = "ctl_2", .id = CTL_2,
-	.base = 0x1400, .len = 0x1e0,
-	.features = BIT(DPU_CTL_ACTIVE_CFG),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
-	},
-	{
-	.name = "ctl_3", .id = CTL_3,
-	.base = 0x1600, .len = 0x1e0,
-	.features = BIT(DPU_CTL_ACTIVE_CFG),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
-	},
-	{
-	.name = "ctl_4", .id = CTL_4,
-	.base = 0x1800, .len = 0x1e0,
-	.features = BIT(DPU_CTL_ACTIVE_CFG),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
-	},
-	{
-	.name = "ctl_5", .id = CTL_5,
-	.base = 0x1a00, .len = 0x1e0,
-	.features = BIT(DPU_CTL_ACTIVE_CFG),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
-	},
-};
-
-static const struct dpu_ctl_cfg sm8350_ctl[] = {
-	{
-	.name = "ctl_0", .id = CTL_0,
-	.base = 0x15000, .len = 0x1e8,
-	.features = BIT(DPU_CTL_SPLIT_DISPLAY) | CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
-	},
-	{
-	.name = "ctl_1", .id = CTL_1,
-	.base = 0x16000, .len = 0x1e8,
-	.features = BIT(DPU_CTL_SPLIT_DISPLAY) | CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
-	},
-	{
-	.name = "ctl_2", .id = CTL_2,
-	.base = 0x17000, .len = 0x1e8,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
-	},
-	{
-	.name = "ctl_3", .id = CTL_3,
-	.base = 0x18000, .len = 0x1e8,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
-	},
-	{
-	.name = "ctl_4", .id = CTL_4,
-	.base = 0x19000, .len = 0x1e8,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
-	},
-	{
-	.name = "ctl_5", .id = CTL_5,
-	.base = 0x1a000, .len = 0x1e8,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
-	},
-};
-
-static const struct dpu_ctl_cfg sm8450_ctl[] = {
-	{
-	.name = "ctl_0", .id = CTL_0,
-	.base = 0x15000, .len = 0x204,
-	.features = BIT(DPU_CTL_ACTIVE_CFG) | BIT(DPU_CTL_SPLIT_DISPLAY) | BIT(DPU_CTL_FETCH_ACTIVE),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
-	},
-	{
-	.name = "ctl_1", .id = CTL_1,
-	.base = 0x16000, .len = 0x204,
-	.features = BIT(DPU_CTL_SPLIT_DISPLAY) | CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
-	},
-	{
-	.name = "ctl_2", .id = CTL_2,
-	.base = 0x17000, .len = 0x204,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
-	},
-	{
-	.name = "ctl_3", .id = CTL_3,
-	.base = 0x18000, .len = 0x204,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
-	},
-	{
-	.name = "ctl_4", .id = CTL_4,
-	.base = 0x19000, .len = 0x204,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
-	},
-	{
-	.name = "ctl_5", .id = CTL_5,
-	.base = 0x1a000, .len = 0x204,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
-	},
-};
-
-static const struct dpu_ctl_cfg sm8550_ctl[] = {
-	{
-	.name = "ctl_0", .id = CTL_0,
-	.base = 0x15000, .len = 0x290,
-	.features = CTL_SM8550_MASK | BIT(DPU_CTL_SPLIT_DISPLAY),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
-	},
-	{
-	.name = "ctl_1", .id = CTL_1,
-	.base = 0x16000, .len = 0x290,
-	.features = CTL_SM8550_MASK | BIT(DPU_CTL_SPLIT_DISPLAY),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
-	},
-	{
-	.name = "ctl_2", .id = CTL_2,
-	.base = 0x17000, .len = 0x290,
-	.features = CTL_SM8550_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
-	},
-	{
-	.name = "ctl_3", .id = CTL_3,
-	.base = 0x18000, .len = 0x290,
-	.features = CTL_SM8550_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
-	},
-	{
-	.name = "ctl_4", .id = CTL_4,
-	.base = 0x19000, .len = 0x290,
-	.features = CTL_SM8550_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13),
-	},
-	{
-	.name = "ctl_5", .id = CTL_5,
-	.base = 0x1a000, .len = 0x290,
-	.features = CTL_SM8550_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 23),
-	},
-};
-
-static const struct dpu_ctl_cfg sc7280_ctl[] = {
-	{
-	.name = "ctl_0", .id = CTL_0,
-	.base = 0x15000, .len = 0x1E8,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
-	},
-	{
-	.name = "ctl_1", .id = CTL_1,
-	.base = 0x16000, .len = 0x1E8,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
-	},
-	{
-	.name = "ctl_2", .id = CTL_2,
-	.base = 0x17000, .len = 0x1E8,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
-	},
-	{
-	.name = "ctl_3", .id = CTL_3,
-	.base = 0x18000, .len = 0x1E8,
-	.features = CTL_SC7280_MASK,
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
-	},
-};
-
-static const struct dpu_ctl_cfg qcm2290_ctl[] = {
-	{
-	.name = "ctl_0", .id = CTL_0,
-	.base = 0x1000, .len = 0x1dc,
-	.features = BIT(DPU_CTL_ACTIVE_CFG),
-	.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
-	},
-};
-
-/*************************************************************
  * SSPP sub blocks config
  *************************************************************/
 
@@ -1164,11 +329,11 @@ static const struct dpu_sspp_sub_blks sdm845_dma_sblk_1 = _DMA_SBLK("9", 2);
 static const struct dpu_sspp_sub_blks sdm845_dma_sblk_2 = _DMA_SBLK("10", 3);
 static const struct dpu_sspp_sub_blks sdm845_dma_sblk_3 = _DMA_SBLK("11", 4);
 
-#define SSPP_BLK(_name, _id, _base, _features, \
+#define SSPP_BLK(_name, _id, _base, _len, _features, \
 		_sblk, _xinid, _type, _clkctrl) \
 	{ \
 	.name = _name, .id = _id, \
-	.base = _base, .len = 0x1c8, \
+	.base = _base, .len = _len, \
 	.features = _features, \
 	.sblk = &_sblk, \
 	.xin_id = _xinid, \
@@ -1176,71 +341,15 @@ static const struct dpu_sspp_sub_blks sdm845_dma_sblk_3 = _DMA_SBLK("11", 4);
 	.clk_ctrl = _clkctrl \
 	}
 
-static const struct dpu_sspp_cfg msm8998_sspp[] = {
-	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_MSM8998_MASK,
-		msm8998_vig_sblk_0, 0,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
-	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, VIG_MSM8998_MASK,
-		msm8998_vig_sblk_1, 4,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
-	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, VIG_MSM8998_MASK,
-		msm8998_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
-	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, VIG_MSM8998_MASK,
-		msm8998_vig_sblk_3, 12,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
-	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000,  DMA_MSM8998_MASK,
-		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
-	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000,  DMA_MSM8998_MASK,
-		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
-	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000,  DMA_CURSOR_MSM8998_MASK,
-		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
-	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000,  DMA_CURSOR_MSM8998_MASK,
-		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
-};
-
-static const struct dpu_sspp_cfg sdm845_sspp[] = {
-	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_SDM845_MASK,
-		sdm845_vig_sblk_0, 0,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
-	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, VIG_SDM845_MASK,
-		sdm845_vig_sblk_1, 4,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
-	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, VIG_SDM845_MASK,
-		sdm845_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
-	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, VIG_SDM845_MASK,
-		sdm845_vig_sblk_3, 12,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
-	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
-	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
-	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000,  DMA_CURSOR_SDM845_MASK,
-		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
-	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000,  DMA_CURSOR_SDM845_MASK,
-		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
-};
-
 static const struct dpu_sspp_sub_blks sc7180_vig_sblk_0 =
 				_VIG_SBLK("0", 4, DPU_SSPP_SCALER_QSEED4);
 
 static const struct dpu_sspp_sub_blks sc7280_vig_sblk_0 =
 			_VIG_SBLK_ROT("0", 4, DPU_SSPP_SCALER_QSEED4, &dpu_rot_sc7280_cfg_v2);
 
-static const struct dpu_sspp_cfg sc7180_sspp[] = {
-	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_SC7180_MASK,
-		sc7180_vig_sblk_0, 0,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
-	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
-	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000,  DMA_CURSOR_SDM845_MASK,
-		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
-	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000,  DMA_CURSOR_SDM845_MASK,
-		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
-};
-
 static const struct dpu_sspp_sub_blks sm6115_vig_sblk_0 =
 				_VIG_SBLK("0", 2, DPU_SSPP_SCALER_QSEED4);
 
-static const struct dpu_sspp_cfg sm6115_sspp[] = {
-	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_SC7180_MASK,
-		sm6115_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
-	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
-};
-
 static const struct dpu_sspp_sub_blks sm8250_vig_sblk_0 =
 				_VIG_SBLK("0", 5, DPU_SSPP_SCALER_QSEED4);
 static const struct dpu_sspp_sub_blks sm8250_vig_sblk_1 =
@@ -1250,53 +359,6 @@ static const struct dpu_sspp_sub_blks sm8250_vig_sblk_2 =
 static const struct dpu_sspp_sub_blks sm8250_vig_sblk_3 =
 				_VIG_SBLK("3", 8, DPU_SSPP_SCALER_QSEED4);
 
-static const struct dpu_sspp_cfg sm8250_sspp[] = {
-	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_SC7180_MASK,
-		sm8250_vig_sblk_0, 0,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
-	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, VIG_SC7180_MASK,
-		sm8250_vig_sblk_1, 4,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
-	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, VIG_SC7180_MASK,
-		sm8250_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
-	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, VIG_SC7180_MASK,
-		sm8250_vig_sblk_3, 12,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
-	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
-	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
-	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000,  DMA_CURSOR_SDM845_MASK,
-		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
-	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000,  DMA_CURSOR_SDM845_MASK,
-		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
-};
-
-static const struct dpu_sspp_sub_blks sm8450_vig_sblk_0 =
-				_VIG_SBLK("0", 5, DPU_SSPP_SCALER_QSEED4);
-static const struct dpu_sspp_sub_blks sm8450_vig_sblk_1 =
-				_VIG_SBLK("1", 6, DPU_SSPP_SCALER_QSEED4);
-static const struct dpu_sspp_sub_blks sm8450_vig_sblk_2 =
-				_VIG_SBLK("2", 7, DPU_SSPP_SCALER_QSEED4);
-static const struct dpu_sspp_sub_blks sm8450_vig_sblk_3 =
-				_VIG_SBLK("3", 8, DPU_SSPP_SCALER_QSEED4);
-
-static const struct dpu_sspp_cfg sm8450_sspp[] = {
-	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_SC7180_MASK,
-		sm8450_vig_sblk_0, 0,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
-	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, VIG_SC7180_MASK,
-		sm8450_vig_sblk_1, 4,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
-	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, VIG_SC7180_MASK,
-		sm8450_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
-	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, VIG_SC7180_MASK,
-		sm8450_vig_sblk_3, 12,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
-	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
-	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
-	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000,  DMA_CURSOR_SDM845_MASK,
-		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
-	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000,  DMA_CURSOR_SDM845_MASK,
-		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
-};
-
 static const struct dpu_sspp_sub_blks sm8550_vig_sblk_0 =
 				_VIG_SBLK("0", 7, DPU_SSPP_SCALER_QSEED4);
 static const struct dpu_sspp_sub_blks sm8550_vig_sblk_1 =
@@ -1308,68 +370,6 @@ static const struct dpu_sspp_sub_blks sm8550_vig_sblk_3 =
 static const struct dpu_sspp_sub_blks sm8550_dma_sblk_4 = _DMA_SBLK("12", 5);
 static const struct dpu_sspp_sub_blks sm8550_dma_sblk_5 = _DMA_SBLK("13", 6);
 
-static const struct dpu_sspp_cfg sm8550_sspp[] = {
-	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_SC7180_MASK,
-		sm8550_vig_sblk_0, 0,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
-	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, VIG_SC7180_MASK,
-		sm8550_vig_sblk_1, 4,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
-	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, VIG_SC7180_MASK,
-		sm8550_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
-	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, VIG_SC7180_MASK,
-		sm8550_vig_sblk_3, 12,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
-	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
-	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
-	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
-	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
-	SSPP_BLK("sspp_12", SSPP_DMA4, 0x2c000,  DMA_CURSOR_SDM845_MASK,
-		sm8550_dma_sblk_4, 14, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA4),
-	SSPP_BLK("sspp_13", SSPP_DMA5, 0x2e000,  DMA_CURSOR_SDM845_MASK,
-		sm8550_dma_sblk_5, 15, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA5),
-};
-
-static const struct dpu_sspp_cfg sc7280_sspp[] = {
-	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_SC7280_MASK,
-		sc7280_vig_sblk_0, 0,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
-	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000,  DMA_SDM845_MASK,
-		sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
-	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000,  DMA_CURSOR_SDM845_MASK,
-		sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
-	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000,  DMA_CURSOR_SDM845_MASK,
-		sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
-};
-
-static const struct dpu_sspp_sub_blks sc8280xp_vig_sblk_0 =
-				_VIG_SBLK("0", 5, DPU_SSPP_SCALER_QSEED4);
-static const struct dpu_sspp_sub_blks sc8280xp_vig_sblk_1 =
-				_VIG_SBLK("1", 6, DPU_SSPP_SCALER_QSEED4);
-static const struct dpu_sspp_sub_blks sc8280xp_vig_sblk_2 =
-				_VIG_SBLK("2", 7, DPU_SSPP_SCALER_QSEED4);
-static const struct dpu_sspp_sub_blks sc8280xp_vig_sblk_3 =
-				_VIG_SBLK("3", 8, DPU_SSPP_SCALER_QSEED4);
-
-static const struct dpu_sspp_cfg sc8280xp_sspp[] = {
-	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_SC7180_MASK,
-		 sc8280xp_vig_sblk_0, 0,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
-	SSPP_BLK("sspp_1", SSPP_VIG1, 0x6000, VIG_SC7180_MASK,
-		 sc8280xp_vig_sblk_1, 4,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG1),
-	SSPP_BLK("sspp_2", SSPP_VIG2, 0x8000, VIG_SC7180_MASK,
-		 sc8280xp_vig_sblk_2, 8, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG2),
-	SSPP_BLK("sspp_3", SSPP_VIG3, 0xa000, VIG_SC7180_MASK,
-		 sc8280xp_vig_sblk_3, 12,  SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG3),
-	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000, DMA_SDM845_MASK,
-		 sdm845_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
-	SSPP_BLK("sspp_9", SSPP_DMA1, 0x26000, DMA_SDM845_MASK,
-		 sdm845_dma_sblk_1, 5, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA1),
-	SSPP_BLK("sspp_10", SSPP_DMA2, 0x28000, DMA_CURSOR_SDM845_MASK,
-		 sdm845_dma_sblk_2, 9, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA2),
-	SSPP_BLK("sspp_11", SSPP_DMA3, 0x2a000, DMA_CURSOR_SDM845_MASK,
-		 sdm845_dma_sblk_3, 13, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA3),
-};
-
 #define _VIG_SBLK_NOSCALE(num, sdma_pri) \
 	{ \
 	.maxdwnscale = SSPP_UNITY_SCALE, \
@@ -1386,13 +386,6 @@ static const struct dpu_sspp_cfg sc8280xp_sspp[] = {
 static const struct dpu_sspp_sub_blks qcm2290_vig_sblk_0 = _VIG_SBLK_NOSCALE("0", 2);
 static const struct dpu_sspp_sub_blks qcm2290_dma_sblk_0 = _DMA_SBLK("8", 1);
 
-static const struct dpu_sspp_cfg qcm2290_sspp[] = {
-	SSPP_BLK("sspp_0", SSPP_VIG0, 0x4000, VIG_QCM2290_MASK,
-		 qcm2290_vig_sblk_0, 0, SSPP_TYPE_VIG, DPU_CLK_CTRL_VIG0),
-	SSPP_BLK("sspp_8", SSPP_DMA0, 0x24000,  DMA_SDM845_MASK,
-		 qcm2290_dma_sblk_0, 1, SSPP_TYPE_DMA, DPU_CLK_CTRL_DMA0),
-};
-
 /*************************************************************
  * MIXER sub blocks config
  *************************************************************/
@@ -1419,21 +412,6 @@ static const struct dpu_lm_sub_blks msm8998_lm_sblk = {
 	},
 };
 
-static const struct dpu_lm_cfg msm8998_lm[] = {
-	LM_BLK("lm_0", LM_0, 0x44000, MIXER_MSM8998_MASK,
-		&msm8998_lm_sblk, PINGPONG_0, LM_2, DSPP_0),
-	LM_BLK("lm_1", LM_1, 0x45000, MIXER_MSM8998_MASK,
-		&msm8998_lm_sblk, PINGPONG_1, LM_5, DSPP_1),
-	LM_BLK("lm_2", LM_2, 0x46000, MIXER_MSM8998_MASK,
-		&msm8998_lm_sblk, PINGPONG_2, LM_0, 0),
-	LM_BLK("lm_3", LM_3, 0x47000, MIXER_MSM8998_MASK,
-		&msm8998_lm_sblk, PINGPONG_MAX, 0, 0),
-	LM_BLK("lm_4", LM_4, 0x48000, MIXER_MSM8998_MASK,
-		&msm8998_lm_sblk, PINGPONG_MAX, 0, 0),
-	LM_BLK("lm_5", LM_5, 0x49000, MIXER_MSM8998_MASK,
-		&msm8998_lm_sblk, PINGPONG_3, LM_1, 0),
-};
-
 /* SDM845 */
 
 static const struct dpu_lm_sub_blks sdm845_lm_sblk = {
@@ -1445,21 +423,6 @@ static const struct dpu_lm_sub_blks sdm845_lm_sblk = {
 	},
 };
 
-static const struct dpu_lm_cfg sdm845_lm[] = {
-	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_0, LM_1, 0),
-	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_1, LM_0, 0),
-	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_2, LM_5, 0),
-	LM_BLK("lm_3", LM_3, 0x0, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_MAX, 0, 0),
-	LM_BLK("lm_4", LM_4, 0x0, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_MAX, 0, 0),
-	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_3, LM_2, 0),
-};
-
 /* SC7180 */
 
 static const struct dpu_lm_sub_blks sc7180_lm_sblk = {
@@ -1470,50 +433,6 @@ static const struct dpu_lm_sub_blks sc7180_lm_sblk = {
 	},
 };
 
-static const struct dpu_lm_cfg sc7180_lm[] = {
-	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
-		&sc7180_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
-	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
-		&sc7180_lm_sblk, PINGPONG_1, LM_0, 0),
-};
-
-/* SC8280XP */
-
-static const struct dpu_lm_cfg sc8280xp_lm[] = {
-	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
-	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_1, LM_0, DSPP_1),
-	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_2, LM_3, DSPP_2),
-	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_3, LM_2, DSPP_3),
-	LM_BLK("lm_4", LM_4, 0x48000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_4, LM_5, 0),
-	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK, &sdm845_lm_sblk, PINGPONG_5, LM_4, 0),
-};
-
-/* SM8150 */
-
-static const struct dpu_lm_cfg sm8150_lm[] = {
-	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
-	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_1, LM_0, DSPP_1),
-	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_2, LM_3, 0),
-	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_3, LM_2, 0),
-	LM_BLK("lm_4", LM_4, 0x48000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_4, LM_5, 0),
-	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_5, LM_4, 0),
-};
-
-static const struct dpu_lm_cfg sc7280_lm[] = {
-	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
-		&sc7180_lm_sblk, PINGPONG_0, 0, DSPP_0),
-	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
-		&sc7180_lm_sblk, PINGPONG_2, LM_3, 0),
-	LM_BLK("lm_3", LM_3, 0x47000, MIXER_SDM845_MASK,
-		&sc7180_lm_sblk, PINGPONG_3, LM_2, 0),
-};
-
 /* QCM2290 */
 
 static const struct dpu_lm_sub_blks qcm2290_lm_sblk = {
@@ -1524,11 +443,6 @@ static const struct dpu_lm_sub_blks qcm2290_lm_sblk = {
 	},
 };
 
-static const struct dpu_lm_cfg qcm2290_lm[] = {
-	LM_BLK("lm_0", LM_0, 0x44000, MIXER_QCM2290_MASK,
-		&qcm2290_lm_sblk, PINGPONG_0, 0, DSPP_0),
-};
-
 /*************************************************************
  * DSPP sub blocks config
  *************************************************************/
@@ -1557,34 +471,6 @@ static const struct dpu_dspp_sub_blks sm8150_dspp_sblk = {
 		.sblk = _sblk \
 		}
 
-static const struct dpu_dspp_cfg msm8998_dspp[] = {
-	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_MSM8998_MASK,
-		 &msm8998_dspp_sblk),
-	DSPP_BLK("dspp_1", DSPP_1, 0x56000, DSPP_MSM8998_MASK,
-		 &msm8998_dspp_sblk),
-};
-
-static const struct dpu_dspp_cfg sc7180_dspp[] = {
-	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
-		 &sc7180_dspp_sblk),
-};
-
-static const struct dpu_dspp_cfg sm8150_dspp[] = {
-	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
-		 &sm8150_dspp_sblk),
-	DSPP_BLK("dspp_1", DSPP_1, 0x56000, DSPP_SC7180_MASK,
-		 &sm8150_dspp_sblk),
-	DSPP_BLK("dspp_2", DSPP_2, 0x58000, DSPP_SC7180_MASK,
-		 &sm8150_dspp_sblk),
-	DSPP_BLK("dspp_3", DSPP_3, 0x5a000, DSPP_SC7180_MASK,
-		 &sm8150_dspp_sblk),
-};
-
-static const struct dpu_dspp_cfg qcm2290_dspp[] = {
-	DSPP_BLK("dspp_0", DSPP_0, 0x54000, DSPP_SC7180_MASK,
-		 &sm8150_dspp_sblk),
-};
-
 /*************************************************************
  * PINGPONG sub blocks config
  *************************************************************/
@@ -1636,151 +522,6 @@ static const struct dpu_pingpong_sub_blks sc7280_pp_sblk = {
 	.intr_rdptr = _rdptr, \
 	}
 
-static const struct dpu_pingpong_cfg sdm845_pp[] = {
-	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x70000, 0, sdm845_pp_sblk_te,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
-	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x70800, 0, sdm845_pp_sblk_te,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
-	PP_BLK("pingpong_2", PINGPONG_2, 0x71000, 0, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
-	PP_BLK("pingpong_3", PINGPONG_3, 0x71800, 0, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
-};
-
-static struct dpu_pingpong_cfg sc7180_pp[] = {
-	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x70000, 0, sdm845_pp_sblk_te, -1, -1),
-	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x70800, 0, sdm845_pp_sblk_te, -1, -1),
-};
-
-static struct dpu_pingpong_cfg sc8280xp_pp[] = {
-	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x69000, MERGE_3D_0, sdm845_pp_sblk_te,
-		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8), -1),
-	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x6a000, MERGE_3D_0, sdm845_pp_sblk_te,
-		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9), -1),
-	PP_BLK_TE("pingpong_2", PINGPONG_2, 0x6b000, MERGE_3D_1, sdm845_pp_sblk_te,
-		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10), -1),
-	PP_BLK_TE("pingpong_3", PINGPONG_3, 0x6c000, MERGE_3D_1, sdm845_pp_sblk_te,
-		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11), -1),
-	PP_BLK_TE("pingpong_4", PINGPONG_4, 0x6d000, MERGE_3D_2, sdm845_pp_sblk_te,
-		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30), -1),
-	PP_BLK_TE("pingpong_5", PINGPONG_5, 0x6e000, MERGE_3D_2, sdm845_pp_sblk_te,
-		  DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31), -1),
-};
-
-static const struct dpu_pingpong_cfg sm8150_pp[] = {
-	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x70000, MERGE_3D_0, sdm845_pp_sblk_te,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
-	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x70800, MERGE_3D_0, sdm845_pp_sblk_te,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
-	PP_BLK("pingpong_2", PINGPONG_2, 0x71000, MERGE_3D_1, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
-	PP_BLK("pingpong_3", PINGPONG_3, 0x71800, MERGE_3D_1, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
-	PP_BLK("pingpong_4", PINGPONG_4, 0x72000, MERGE_3D_2, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30),
-			-1),
-	PP_BLK("pingpong_5", PINGPONG_5, 0x72800, MERGE_3D_2, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31),
-			-1),
-};
-
-static const struct dpu_pingpong_cfg sm8350_pp[] = {
-	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x69000, MERGE_3D_0, sdm845_pp_sblk_te,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
-	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x6a000, MERGE_3D_0, sdm845_pp_sblk_te,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
-	PP_BLK("pingpong_2", PINGPONG_2, 0x6b000, MERGE_3D_1, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
-	PP_BLK("pingpong_3", PINGPONG_3, 0x6c000, MERGE_3D_1, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
-	PP_BLK("pingpong_4", PINGPONG_4, 0x6d000, MERGE_3D_2, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30),
-			-1),
-	PP_BLK("pingpong_5", PINGPONG_5, 0x6e000, MERGE_3D_2, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31),
-			-1),
-};
-
-static const struct dpu_pingpong_cfg sc7280_pp[] = {
-	PP_BLK("pingpong_0", PINGPONG_0, 0x69000, 0, sc7280_pp_sblk, -1, -1),
-	PP_BLK("pingpong_1", PINGPONG_1, 0x6a000, 0, sc7280_pp_sblk, -1, -1),
-	PP_BLK("pingpong_2", PINGPONG_2, 0x6b000, 0, sc7280_pp_sblk, -1, -1),
-	PP_BLK("pingpong_3", PINGPONG_3, 0x6c000, 0, sc7280_pp_sblk, -1, -1),
-};
-
-static struct dpu_pingpong_cfg qcm2290_pp[] = {
-	PP_BLK("pingpong_0", PINGPONG_0, 0x70000, 0, sdm845_pp_sblk,
-		DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
-		DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
-};
-
-/* FIXME: interrupts */
-static const struct dpu_pingpong_cfg sm8450_pp[] = {
-	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x69000, MERGE_3D_0, sdm845_pp_sblk_te,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 12)),
-	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x6a000, MERGE_3D_0, sdm845_pp_sblk_te,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 13)),
-	PP_BLK("pingpong_2", PINGPONG_2, 0x6b000, MERGE_3D_1, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 14)),
-	PP_BLK("pingpong_3", PINGPONG_3, 0x6c000, MERGE_3D_1, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 15)),
-	PP_BLK("pingpong_4", PINGPONG_4, 0x6d000, MERGE_3D_2, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30),
-			-1),
-	PP_BLK("pingpong_5", PINGPONG_5, 0x6e000, MERGE_3D_2, sdm845_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31),
-			-1),
-	PP_BLK("pingpong_6", PINGPONG_6, 0x65800, MERGE_3D_3, sdm845_pp_sblk,
-			-1,
-			-1),
-	PP_BLK("pingpong_7", PINGPONG_7, 0x65c00, MERGE_3D_3, sdm845_pp_sblk,
-			-1,
-			-1),
-};
-
-static const struct dpu_pingpong_cfg sm8550_pp[] = {
-	PP_BLK_DIPHER("pingpong_0", PINGPONG_0, 0x69000, MERGE_3D_0, sc7280_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
-			-1),
-	PP_BLK_DIPHER("pingpong_1", PINGPONG_1, 0x6a000, MERGE_3D_0, sc7280_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 9),
-			-1),
-	PP_BLK_DIPHER("pingpong_2", PINGPONG_2, 0x6b000, MERGE_3D_1, sc7280_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
-			-1),
-	PP_BLK_DIPHER("pingpong_3", PINGPONG_3, 0x6c000, MERGE_3D_1, sc7280_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
-			-1),
-	PP_BLK_DIPHER("pingpong_4", PINGPONG_4, 0x6d000, MERGE_3D_2, sc7280_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 30),
-			-1),
-	PP_BLK_DIPHER("pingpong_5", PINGPONG_5, 0x6e000, MERGE_3D_2, sc7280_pp_sblk,
-			DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 31),
-			-1),
-	PP_BLK_DIPHER("pingpong_6", PINGPONG_6, 0x66000, MERGE_3D_3, sc7280_pp_sblk,
-			-1,
-			-1),
-	PP_BLK_DIPHER("pingpong_7", PINGPONG_7, 0x66400, MERGE_3D_3, sc7280_pp_sblk,
-			-1,
-			-1),
-};
-
 /*************************************************************
  * MERGE_3D sub blocks config
  *************************************************************/
@@ -1792,32 +533,6 @@ static const struct dpu_pingpong_cfg sm8550_pp[] = {
 	.sblk = NULL \
 	}
 
-static const struct dpu_merge_3d_cfg sm8150_merge_3d[] = {
-	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x83000),
-	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x83100),
-	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x83200),
-};
-
-static const struct dpu_merge_3d_cfg sm8350_merge_3d[] = {
-	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x4e000),
-	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x4f000),
-	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x50000),
-};
-
-static const struct dpu_merge_3d_cfg sm8450_merge_3d[] = {
-	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x4e000),
-	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x4f000),
-	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x50000),
-	MERGE_3D_BLK("merge_3d_3", MERGE_3D_3, 0x65f00),
-};
-
-static const struct dpu_merge_3d_cfg sm8550_merge_3d[] = {
-	MERGE_3D_BLK("merge_3d_0", MERGE_3D_0, 0x4e000),
-	MERGE_3D_BLK("merge_3d_1", MERGE_3D_1, 0x4f000),
-	MERGE_3D_BLK("merge_3d_2", MERGE_3D_2, 0x50000),
-	MERGE_3D_BLK("merge_3d_3", MERGE_3D_3, 0x66700),
-};
-
 /*************************************************************
  * DSC sub blocks config
  *************************************************************/
@@ -1828,27 +543,13 @@ static const struct dpu_merge_3d_cfg sm8550_merge_3d[] = {
 	.features = _features, \
 	}
 
-static struct dpu_dsc_cfg sdm845_dsc[] = {
-	DSC_BLK("dsc_0", DSC_0, 0x80000, 0),
-	DSC_BLK("dsc_1", DSC_1, 0x80400, 0),
-	DSC_BLK("dsc_2", DSC_2, 0x80800, 0),
-	DSC_BLK("dsc_3", DSC_3, 0x80c00, 0),
-};
-
-static struct dpu_dsc_cfg sm8150_dsc[] = {
-	DSC_BLK("dsc_0", DSC_0, 0x80000, BIT(DPU_DSC_OUTPUT_CTRL)),
-	DSC_BLK("dsc_1", DSC_1, 0x80400, BIT(DPU_DSC_OUTPUT_CTRL)),
-	DSC_BLK("dsc_2", DSC_2, 0x80800, BIT(DPU_DSC_OUTPUT_CTRL)),
-	DSC_BLK("dsc_3", DSC_3, 0x80c00, BIT(DPU_DSC_OUTPUT_CTRL)),
-};
-
 /*************************************************************
  * INTF sub blocks config
  *************************************************************/
-#define INTF_BLK(_name, _id, _base, _type, _ctrl_id, _progfetch, _features, _reg, _underrun_bit, _vsync_bit) \
+#define INTF_BLK(_name, _id, _base, _len, _type, _ctrl_id, _progfetch, _features, _reg, _underrun_bit, _vsync_bit) \
 	{\
 	.name = _name, .id = _id, \
-	.base = _base, .len = 0x280, \
+	.base = _base, .len = _len, \
 	.features = _features, \
 	.type = _type, \
 	.controller_id = _ctrl_id, \
@@ -1857,88 +558,6 @@ static struct dpu_dsc_cfg sm8150_dsc[] = {
 	.intr_vsync = DPU_IRQ_IDX(_reg, _vsync_bit), \
 	}
 
-static const struct dpu_intf_cfg msm8998_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, 0, 25, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
-	INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, 25, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-	INTF_BLK("intf_2", INTF_2, 0x6B000, INTF_DSI, 1, 25, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
-	INTF_BLK("intf_3", INTF_3, 0x6B800, INTF_HDMI, 0, 25, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
-};
-
-static const struct dpu_intf_cfg sdm845_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, 0, 24, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
-	INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, 24, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-	INTF_BLK("intf_2", INTF_2, 0x6B000, INTF_DSI, 1, 24, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
-	INTF_BLK("intf_3", INTF_3, 0x6B800, INTF_DP, 1, 24, INTF_SDM845_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
-};
-
-static const struct dpu_intf_cfg sc7180_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
-	INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-};
-
-static const struct dpu_intf_cfg sm8150_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
-	INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-	INTF_BLK("intf_2", INTF_2, 0x6B000, INTF_DSI, 1, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
-	INTF_BLK("intf_3", INTF_3, 0x6B800, INTF_DP, 1, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
-};
-
-static const struct dpu_intf_cfg sc7280_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x34000, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
-	INTF_BLK("intf_1", INTF_1, 0x35000, INTF_DSI, 0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-	INTF_BLK("intf_5", INTF_5, 0x39000, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 22, 23),
-};
-
-static const struct dpu_intf_cfg sm8350_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x34000, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
-	INTF_BLK("intf_1", INTF_1, 0x35000, INTF_DSI, 0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-	INTF_BLK("intf_2", INTF_2, 0x36000, INTF_DSI, 1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
-	INTF_BLK("intf_3", INTF_3, 0x37000, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
-};
-
-static const struct dpu_intf_cfg sc8180x_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
-	INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-	INTF_BLK("intf_2", INTF_2, 0x6B000, INTF_DSI, 1, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
-	/* INTF_3 is for MST, wired to INTF_DP 0 and 1, use dummy index until this is supported */
-	INTF_BLK("intf_3", INTF_3, 0x6B800, INTF_DP, 999, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
-	INTF_BLK("intf_4", INTF_4, 0x6C000, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 20, 21),
-	INTF_BLK("intf_5", INTF_5, 0x6C800, INTF_DP, MSM_DP_CONTROLLER_2, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 22, 23),
-};
-
-/* TODO: INTF 3, 8 and 7 are used for MST, marked as INTF_NONE for now */
-static const struct dpu_intf_cfg sc8280xp_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x34000, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
-	INTF_BLK("intf_1", INTF_1, 0x35000, INTF_DSI, 0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-	INTF_BLK("intf_2", INTF_2, 0x36000, INTF_DSI, 1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
-	INTF_BLK("intf_3", INTF_3, 0x37000, INTF_NONE, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
-	INTF_BLK("intf_4", INTF_4, 0x38000, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 20, 21),
-	INTF_BLK("intf_5", INTF_5, 0x39000, INTF_DP, MSM_DP_CONTROLLER_3, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 22, 23),
-	INTF_BLK("intf_6", INTF_6, 0x3a000, INTF_DP, MSM_DP_CONTROLLER_2, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 16, 17),
-	INTF_BLK("intf_7", INTF_7, 0x3b000, INTF_NONE, MSM_DP_CONTROLLER_2, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 18, 19),
-	INTF_BLK("intf_8", INTF_8, 0x3c000, INTF_NONE, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 12, 13),
-};
-
-static const struct dpu_intf_cfg qcm2290_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x00000, INTF_NONE, 0, 0, 0, 0, 0, 0),
-	INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0, 24, INTF_SC7180_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-};
-
-static const struct dpu_intf_cfg sm8450_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x34000, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
-	INTF_BLK("intf_1", INTF_1, 0x35000, INTF_DSI, 0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-	INTF_BLK("intf_2", INTF_2, 0x36000, INTF_DSI, 1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
-	INTF_BLK("intf_3", INTF_3, 0x37000, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
-};
-
-static const struct dpu_intf_cfg sm8550_intf[] = {
-	INTF_BLK("intf_0", INTF_0, 0x34000, INTF_DP, MSM_DP_CONTROLLER_0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 24, 25),
-	/* TODO TE sub-blocks for intf1 & intf2 */
-	INTF_BLK("intf_1", INTF_1, 0x35000, INTF_DSI, 0, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 26, 27),
-	INTF_BLK("intf_2", INTF_2, 0x36000, INTF_DSI, 1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 28, 29),
-	INTF_BLK("intf_3", INTF_3, 0x37000, INTF_DP, MSM_DP_CONTROLLER_1, 24, INTF_SC7280_MASK, MDP_SSPP_TOP0_INTR, 30, 31),
-};
-
 /*************************************************************
  * Writeback blocks config
  *************************************************************/
@@ -1957,11 +576,6 @@ static const struct dpu_intf_cfg sm8550_intf[] = {
 	.intr_wb_done = DPU_IRQ_IDX(_reg, _wb_done_bit) \
 	}
 
-static const struct dpu_wb_cfg sm8250_wb[] = {
-	WB_BLK("wb_2", WB_2, 0x65000, WB_SM8250_MASK, DPU_CLK_CTRL_WB2, 6,
-			VBIF_RT, MDP_SSPP_TOP0_INTR, 4096, 4),
-};
-
 /*************************************************************
  * VBIF sub blocks config
  *************************************************************/
@@ -2166,715 +780,26 @@ static const struct dpu_qos_lut_entry sc7180_qos_nrt[] = {
 	{.fl = 0, .lut = 0x0},
 };
 
-static const struct dpu_perf_cfg msm8998_perf_data = {
-	.max_bw_low = 6700000,
-	.max_bw_high = 6700000,
-	.min_core_ib = 2400000,
-	.min_llcc_ib = 800000,
-	.min_dram_ib = 800000,
-	.undersized_prefill_lines = 2,
-	.xtra_prefill_lines = 2,
-	.dest_scale_prefill_lines = 3,
-	.macrotile_prefill_lines = 4,
-	.yuv_nv12_prefill_lines = 8,
-	.linear_prefill_lines = 1,
-	.downscaling_prefill_lines = 1,
-	.amortizable_threshold = 25,
-	.min_prefill_lines = 25,
-	.danger_lut_tbl = {0xf, 0xffff, 0x0},
-	.safe_lut_tbl = {0xfffc, 0xff00, 0xffff},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(msm8998_qos_linear),
-		.entries = msm8998_qos_linear
-		},
-		{.nentry = ARRAY_SIZE(msm8998_qos_macrotile),
-		.entries = msm8998_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(msm8998_qos_nrt),
-		.entries = msm8998_qos_nrt
-		},
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 200,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg sdm845_perf_data = {
-	.max_bw_low = 6800000,
-	.max_bw_high = 6800000,
-	.min_core_ib = 2400000,
-	.min_llcc_ib = 800000,
-	.min_dram_ib = 800000,
-	.undersized_prefill_lines = 2,
-	.xtra_prefill_lines = 2,
-	.dest_scale_prefill_lines = 3,
-	.macrotile_prefill_lines = 4,
-	.yuv_nv12_prefill_lines = 8,
-	.linear_prefill_lines = 1,
-	.downscaling_prefill_lines = 1,
-	.amortizable_threshold = 25,
-	.min_prefill_lines = 24,
-	.danger_lut_tbl = {0xf, 0xffff, 0x0},
-	.safe_lut_tbl = {0xfff0, 0xf000, 0xffff},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(sdm845_qos_linear),
-		.entries = sdm845_qos_linear
-		},
-		{.nentry = ARRAY_SIZE(sdm845_qos_macrotile),
-		.entries = sdm845_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sdm845_qos_nrt),
-		.entries = sdm845_qos_nrt
-		},
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg sc7180_perf_data = {
-	.max_bw_low = 6800000,
-	.max_bw_high = 6800000,
-	.min_core_ib = 2400000,
-	.min_llcc_ib = 800000,
-	.min_dram_ib = 1600000,
-	.min_prefill_lines = 24,
-	.danger_lut_tbl = {0xff, 0xffff, 0x0},
-	.safe_lut_tbl = {0xfff0, 0xff00, 0xffff},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
-		.entries = sc7180_qos_linear
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
-		.entries = sc7180_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
-		.entries = sc7180_qos_nrt
-		},
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg sm6115_perf_data = {
-	.max_bw_low = 3100000,
-	.max_bw_high = 4000000,
-	.min_core_ib = 2400000,
-	.min_llcc_ib = 800000,
-	.min_dram_ib = 800000,
-	.min_prefill_lines = 24,
-	.danger_lut_tbl = {0xff, 0xffff, 0x0},
-	.safe_lut_tbl = {0xfff0, 0xff00, 0xffff},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
-		.entries = sc7180_qos_linear
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
-		.entries = sc7180_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
-		.entries = sc7180_qos_nrt
-		},
-		/* TODO: macrotile-qseed is different from macrotile */
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg sm8150_perf_data = {
-	.max_bw_low = 12800000,
-	.max_bw_high = 12800000,
-	.min_core_ib = 2400000,
-	.min_llcc_ib = 800000,
-	.min_dram_ib = 800000,
-	.min_prefill_lines = 24,
-	.danger_lut_tbl = {0xf, 0xffff, 0x0},
-	.safe_lut_tbl = {0xfff8, 0xf000, 0xffff},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(sm8150_qos_linear),
-		.entries = sm8150_qos_linear
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
-		.entries = sc7180_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
-		.entries = sc7180_qos_nrt
-		},
-		/* TODO: macrotile-qseed is different from macrotile */
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg sc8180x_perf_data = {
-	.max_bw_low = 9600000,
-	.max_bw_high = 9600000,
-	.min_core_ib = 2400000,
-	.min_llcc_ib = 800000,
-	.min_dram_ib = 800000,
-	.danger_lut_tbl = {0xf, 0xffff, 0x0},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
-		.entries = sc7180_qos_linear
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
-		.entries = sc7180_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
-		.entries = sc7180_qos_nrt
-		},
-		/* TODO: macrotile-qseed is different from macrotile */
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg sc8280xp_perf_data = {
-	.max_bw_low = 13600000,
-	.max_bw_high = 18200000,
-	.min_core_ib = 2500000,
-	.min_llcc_ib = 0,
-	.min_dram_ib = 800000,
-	.danger_lut_tbl = {0xf, 0xffff, 0x0},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(sc8180x_qos_linear),
-		.entries = sc8180x_qos_linear
-		},
-		{.nentry = ARRAY_SIZE(sc8180x_qos_macrotile),
-		.entries = sc8180x_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
-		.entries = sc7180_qos_nrt
-		},
-		/* TODO: macrotile-qseed is different from macrotile */
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg sm8250_perf_data = {
-	.max_bw_low = 13700000,
-	.max_bw_high = 16600000,
-	.min_core_ib = 4800000,
-	.min_llcc_ib = 0,
-	.min_dram_ib = 800000,
-	.min_prefill_lines = 35,
-	.danger_lut_tbl = {0xf, 0xffff, 0x0},
-	.safe_lut_tbl = {0xfff0, 0xff00, 0xffff},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
-		.entries = sc7180_qos_linear
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
-		.entries = sc7180_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
-		.entries = sc7180_qos_nrt
-		},
-		/* TODO: macrotile-qseed is different from macrotile */
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg sm8450_perf_data = {
-	.max_bw_low = 13600000,
-	.max_bw_high = 18200000,
-	.min_core_ib = 2500000,
-	.min_llcc_ib = 0,
-	.min_dram_ib = 800000,
-	.min_prefill_lines = 35,
-	/* FIXME: lut tables */
-	.danger_lut_tbl = {0x3ffff, 0x3ffff, 0x0},
-	.safe_lut_tbl = {0xfe00, 0xfe00, 0xffff},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
-		.entries = sc7180_qos_linear
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
-		.entries = sc7180_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
-		.entries = sc7180_qos_nrt
-		},
-		/* TODO: macrotile-qseed is different from macrotile */
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg sc7280_perf_data = {
-	.max_bw_low = 4700000,
-	.max_bw_high = 8800000,
-	.min_core_ib = 2500000,
-	.min_llcc_ib = 0,
-	.min_dram_ib = 1600000,
-	.min_prefill_lines = 24,
-	.danger_lut_tbl = {0xffff, 0xffff, 0x0},
-	.safe_lut_tbl = {0xff00, 0xff00, 0xffff},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
-		.entries = sc7180_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
-		.entries = sc7180_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
-		.entries = sc7180_qos_nrt
-		},
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg sm8350_perf_data = {
-	.max_bw_low = 11800000,
-	.max_bw_high = 15500000,
-	.min_core_ib = 2500000,
-	.min_llcc_ib = 0,
-	.min_dram_ib = 800000,
-	.min_prefill_lines = 40,
-	/* FIXME: lut tables */
-	.danger_lut_tbl = {0x3ffff, 0x3ffff, 0x0},
-	.safe_lut_tbl = {0xfe00, 0xfe00, 0xffff},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
-		.entries = sc7180_qos_linear
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
-		.entries = sc7180_qos_macrotile
-		},
-		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
-		.entries = sc7180_qos_nrt
-		},
-		/* TODO: macrotile-qseed is different from macrotile */
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
-
-static const struct dpu_perf_cfg qcm2290_perf_data = {
-	.max_bw_low = 2700000,
-	.max_bw_high = 2700000,
-	.min_core_ib = 1300000,
-	.min_llcc_ib = 0,
-	.min_dram_ib = 1600000,
-	.min_prefill_lines = 24,
-	.danger_lut_tbl = {0xff, 0x0, 0x0},
-	.safe_lut_tbl = {0xfff0, 0x0, 0x0},
-	.qos_lut_tbl = {
-		{.nentry = ARRAY_SIZE(qcm2290_qos_linear),
-		.entries = qcm2290_qos_linear
-		},
-	},
-	.cdp_cfg = {
-		{.rd_enable = 1, .wr_enable = 1},
-		{.rd_enable = 1, .wr_enable = 0}
-	},
-	.clk_inefficiency_factor = 105,
-	.bw_inefficiency_factor = 120,
-};
 /*************************************************************
  * Hardware catalog
  *************************************************************/
 
-static const struct dpu_mdss_cfg msm8998_dpu_cfg = {
-	.caps = &msm8998_dpu_caps,
-	.mdp_count = ARRAY_SIZE(msm8998_mdp),
-	.mdp = msm8998_mdp,
-	.ctl_count = ARRAY_SIZE(msm8998_ctl),
-	.ctl = msm8998_ctl,
-	.sspp_count = ARRAY_SIZE(msm8998_sspp),
-	.sspp = msm8998_sspp,
-	.mixer_count = ARRAY_SIZE(msm8998_lm),
-	.mixer = msm8998_lm,
-	.dspp_count = ARRAY_SIZE(msm8998_dspp),
-	.dspp = msm8998_dspp,
-	.pingpong_count = ARRAY_SIZE(sdm845_pp),
-	.pingpong = sdm845_pp,
-	.intf_count = ARRAY_SIZE(msm8998_intf),
-	.intf = msm8998_intf,
-	.vbif_count = ARRAY_SIZE(msm8998_vbif),
-	.vbif = msm8998_vbif,
-	.reg_dma_count = 0,
-	.perf = &msm8998_perf_data,
-	.mdss_irqs = IRQ_SM8250_MASK,
-};
+#include "catalog/dpu_3_0_msm8998.h"
 
-static const struct dpu_mdss_cfg sdm845_dpu_cfg = {
-	.caps = &sdm845_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sdm845_mdp),
-	.mdp = sdm845_mdp,
-	.ctl_count = ARRAY_SIZE(sdm845_ctl),
-	.ctl = sdm845_ctl,
-	.sspp_count = ARRAY_SIZE(sdm845_sspp),
-	.sspp = sdm845_sspp,
-	.mixer_count = ARRAY_SIZE(sdm845_lm),
-	.mixer = sdm845_lm,
-	.pingpong_count = ARRAY_SIZE(sdm845_pp),
-	.pingpong = sdm845_pp,
-	.dsc_count = ARRAY_SIZE(sdm845_dsc),
-	.dsc = sdm845_dsc,
-	.intf_count = ARRAY_SIZE(sdm845_intf),
-	.intf = sdm845_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.reg_dma_count = 1,
-	.dma_cfg = &sdm845_regdma,
-	.perf = &sdm845_perf_data,
-	.mdss_irqs = IRQ_SDM845_MASK,
-};
+#include "catalog/dpu_4_0_sdm845.h"
 
-static const struct dpu_mdss_cfg sc7180_dpu_cfg = {
-	.caps = &sc7180_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sc7180_mdp),
-	.mdp = sc7180_mdp,
-	.ctl_count = ARRAY_SIZE(sc7180_ctl),
-	.ctl = sc7180_ctl,
-	.sspp_count = ARRAY_SIZE(sc7180_sspp),
-	.sspp = sc7180_sspp,
-	.mixer_count = ARRAY_SIZE(sc7180_lm),
-	.mixer = sc7180_lm,
-	.dspp_count = ARRAY_SIZE(sc7180_dspp),
-	.dspp = sc7180_dspp,
-	.pingpong_count = ARRAY_SIZE(sc7180_pp),
-	.pingpong = sc7180_pp,
-	.intf_count = ARRAY_SIZE(sc7180_intf),
-	.intf = sc7180_intf,
-	.wb_count = ARRAY_SIZE(sm8250_wb),
-	.wb = sm8250_wb,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.reg_dma_count = 1,
-	.dma_cfg = &sdm845_regdma,
-	.perf = &sc7180_perf_data,
-	.mdss_irqs = IRQ_SC7180_MASK,
-};
-
-static const struct dpu_mdss_cfg sm6115_dpu_cfg = {
-	.caps = &sm6115_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sm6115_mdp),
-	.mdp = sm6115_mdp,
-	.ctl_count = ARRAY_SIZE(qcm2290_ctl),
-	.ctl = qcm2290_ctl,
-	.sspp_count = ARRAY_SIZE(sm6115_sspp),
-	.sspp = sm6115_sspp,
-	.mixer_count = ARRAY_SIZE(qcm2290_lm),
-	.mixer = qcm2290_lm,
-	.dspp_count = ARRAY_SIZE(qcm2290_dspp),
-	.dspp = qcm2290_dspp,
-	.pingpong_count = ARRAY_SIZE(qcm2290_pp),
-	.pingpong = qcm2290_pp,
-	.intf_count = ARRAY_SIZE(qcm2290_intf),
-	.intf = qcm2290_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.perf = &sm6115_perf_data,
-	.mdss_irqs = IRQ_SC7180_MASK,
-};
-
-static const struct dpu_mdss_cfg sm8150_dpu_cfg = {
-	.caps = &sm8150_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sdm845_mdp),
-	.mdp = sdm845_mdp,
-	.ctl_count = ARRAY_SIZE(sm8150_ctl),
-	.ctl = sm8150_ctl,
-	.sspp_count = ARRAY_SIZE(sdm845_sspp),
-	.sspp = sdm845_sspp,
-	.mixer_count = ARRAY_SIZE(sm8150_lm),
-	.mixer = sm8150_lm,
-	.dspp_count = ARRAY_SIZE(sm8150_dspp),
-	.dspp = sm8150_dspp,
-	.dsc_count = ARRAY_SIZE(sm8150_dsc),
-	.dsc = sm8150_dsc,
-	.pingpong_count = ARRAY_SIZE(sm8150_pp),
-	.pingpong = sm8150_pp,
-	.merge_3d_count = ARRAY_SIZE(sm8150_merge_3d),
-	.merge_3d = sm8150_merge_3d,
-	.intf_count = ARRAY_SIZE(sm8150_intf),
-	.intf = sm8150_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.reg_dma_count = 1,
-	.dma_cfg = &sm8150_regdma,
-	.perf = &sm8150_perf_data,
-	.mdss_irqs = IRQ_SDM845_MASK,
-};
-
-static const struct dpu_mdss_cfg sc8180x_dpu_cfg = {
-	.caps = &sc8180x_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sc8180x_mdp),
-	.mdp = sc8180x_mdp,
-	.ctl_count = ARRAY_SIZE(sm8150_ctl),
-	.ctl = sm8150_ctl,
-	.sspp_count = ARRAY_SIZE(sdm845_sspp),
-	.sspp = sdm845_sspp,
-	.mixer_count = ARRAY_SIZE(sm8150_lm),
-	.mixer = sm8150_lm,
-	.pingpong_count = ARRAY_SIZE(sm8150_pp),
-	.pingpong = sm8150_pp,
-	.merge_3d_count = ARRAY_SIZE(sm8150_merge_3d),
-	.merge_3d = sm8150_merge_3d,
-	.intf_count = ARRAY_SIZE(sc8180x_intf),
-	.intf = sc8180x_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.reg_dma_count = 1,
-	.dma_cfg = &sm8150_regdma,
-	.perf = &sc8180x_perf_data,
-	.mdss_irqs = IRQ_SC8180X_MASK,
-};
-
-static const struct dpu_mdss_cfg sc8280xp_dpu_cfg = {
-	.caps = &sc8280xp_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sc8280xp_mdp),
-	.mdp = sc8280xp_mdp,
-	.ctl_count = ARRAY_SIZE(sc8280xp_ctl),
-	.ctl = sc8280xp_ctl,
-	.sspp_count = ARRAY_SIZE(sc8280xp_sspp),
-	.sspp = sc8280xp_sspp,
-	.mixer_count = ARRAY_SIZE(sc8280xp_lm),
-	.mixer = sc8280xp_lm,
-	.dspp_count = ARRAY_SIZE(sm8150_dspp),
-	.dspp = sm8150_dspp,
-	.pingpong_count = ARRAY_SIZE(sc8280xp_pp),
-	.pingpong = sc8280xp_pp,
-	.merge_3d_count = ARRAY_SIZE(sm8350_merge_3d),
-	.merge_3d = sm8350_merge_3d,
-	.intf_count = ARRAY_SIZE(sc8280xp_intf),
-	.intf = sc8280xp_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.reg_dma_count = 1,
-	.dma_cfg = &sc8280xp_regdma,
-	.perf = &sc8280xp_perf_data,
-	.mdss_irqs = IRQ_SC8280XP_MASK,
-};
-
-static const struct dpu_mdss_cfg sm8250_dpu_cfg = {
-	.caps = &sm8250_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sm8250_mdp),
-	.mdp = sm8250_mdp,
-	.ctl_count = ARRAY_SIZE(sm8150_ctl),
-	.ctl = sm8150_ctl,
-	.sspp_count = ARRAY_SIZE(sm8250_sspp),
-	.sspp = sm8250_sspp,
-	.mixer_count = ARRAY_SIZE(sm8150_lm),
-	.mixer = sm8150_lm,
-	.dspp_count = ARRAY_SIZE(sm8150_dspp),
-	.dspp = sm8150_dspp,
-	.dsc_count = ARRAY_SIZE(sm8150_dsc),
-	.dsc = sm8150_dsc,
-	.pingpong_count = ARRAY_SIZE(sm8150_pp),
-	.pingpong = sm8150_pp,
-	.merge_3d_count = ARRAY_SIZE(sm8150_merge_3d),
-	.merge_3d = sm8150_merge_3d,
-	.intf_count = ARRAY_SIZE(sm8150_intf),
-	.intf = sm8150_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.wb_count = ARRAY_SIZE(sm8250_wb),
-	.wb = sm8250_wb,
-	.reg_dma_count = 1,
-	.dma_cfg = &sm8250_regdma,
-	.perf = &sm8250_perf_data,
-	.mdss_irqs = IRQ_SM8250_MASK,
-};
-
-static const struct dpu_mdss_cfg sm8350_dpu_cfg = {
-	.caps = &sm8350_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sm8350_mdp),
-	.mdp = sm8350_mdp,
-	.ctl_count = ARRAY_SIZE(sm8350_ctl),
-	.ctl = sm8350_ctl,
-	.sspp_count = ARRAY_SIZE(sm8250_sspp),
-	.sspp = sm8250_sspp,
-	.mixer_count = ARRAY_SIZE(sm8150_lm),
-	.mixer = sm8150_lm,
-	.dspp_count = ARRAY_SIZE(sm8150_dspp),
-	.dspp = sm8150_dspp,
-	.pingpong_count = ARRAY_SIZE(sm8350_pp),
-	.pingpong = sm8350_pp,
-	.merge_3d_count = ARRAY_SIZE(sm8350_merge_3d),
-	.merge_3d = sm8350_merge_3d,
-	.intf_count = ARRAY_SIZE(sm8350_intf),
-	.intf = sm8350_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.reg_dma_count = 1,
-	.dma_cfg = &sm8350_regdma,
-	.perf = &sm8350_perf_data,
-	.mdss_irqs = IRQ_SM8350_MASK,
-};
+#include "catalog/dpu_5_0_sm8150.h"
+#include "catalog/dpu_5_1_sc8180x.h"
 
-static const struct dpu_mdss_cfg sm8450_dpu_cfg = {
-	.caps = &sm8450_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sm8450_mdp),
-	.mdp = sm8450_mdp,
-	.ctl_count = ARRAY_SIZE(sm8450_ctl),
-	.ctl = sm8450_ctl,
-	.sspp_count = ARRAY_SIZE(sm8450_sspp),
-	.sspp = sm8450_sspp,
-	.mixer_count = ARRAY_SIZE(sm8150_lm),
-	.mixer = sm8150_lm,
-	.dspp_count = ARRAY_SIZE(sm8150_dspp),
-	.dspp = sm8150_dspp,
-	.pingpong_count = ARRAY_SIZE(sm8450_pp),
-	.pingpong = sm8450_pp,
-	.merge_3d_count = ARRAY_SIZE(sm8450_merge_3d),
-	.merge_3d = sm8450_merge_3d,
-	.intf_count = ARRAY_SIZE(sm8450_intf),
-	.intf = sm8450_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.reg_dma_count = 1,
-	.dma_cfg = &sm8450_regdma,
-	.perf = &sm8450_perf_data,
-	.mdss_irqs = IRQ_SM8450_MASK,
-};
-
-static const struct dpu_mdss_cfg sm8550_dpu_cfg = {
-	.caps = &sm8550_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sm8550_mdp),
-	.mdp = sm8550_mdp,
-	.ctl_count = ARRAY_SIZE(sm8550_ctl),
-	.ctl = sm8550_ctl,
-	.sspp_count = ARRAY_SIZE(sm8550_sspp),
-	.sspp = sm8550_sspp,
-	.mixer_count = ARRAY_SIZE(sm8150_lm),
-	.mixer = sm8150_lm,
-	.dspp_count = ARRAY_SIZE(sm8150_dspp),
-	.dspp = sm8150_dspp,
-	.pingpong_count = ARRAY_SIZE(sm8550_pp),
-	.pingpong = sm8550_pp,
-	.merge_3d_count = ARRAY_SIZE(sm8550_merge_3d),
-	.merge_3d = sm8550_merge_3d,
-	.intf_count = ARRAY_SIZE(sm8550_intf),
-	.intf = sm8550_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.reg_dma_count = 1,
-	.dma_cfg = &sm8450_regdma,
-	.perf = &sm8450_perf_data,
-	.mdss_irqs = IRQ_SM8450_MASK,
-};
-
-static const struct dpu_mdss_cfg sc7280_dpu_cfg = {
-	.caps = &sc7280_dpu_caps,
-	.mdp_count = ARRAY_SIZE(sc7280_mdp),
-	.mdp = sc7280_mdp,
-	.ctl_count = ARRAY_SIZE(sc7280_ctl),
-	.ctl = sc7280_ctl,
-	.sspp_count = ARRAY_SIZE(sc7280_sspp),
-	.sspp = sc7280_sspp,
-	.dspp_count = ARRAY_SIZE(sc7180_dspp),
-	.dspp = sc7180_dspp,
-	.mixer_count = ARRAY_SIZE(sc7280_lm),
-	.mixer = sc7280_lm,
-	.pingpong_count = ARRAY_SIZE(sc7280_pp),
-	.pingpong = sc7280_pp,
-	.intf_count = ARRAY_SIZE(sc7280_intf),
-	.intf = sc7280_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.perf = &sc7280_perf_data,
-	.mdss_irqs = IRQ_SC7280_MASK,
-};
-
-static const struct dpu_mdss_cfg qcm2290_dpu_cfg = {
-	.caps = &qcm2290_dpu_caps,
-	.mdp_count = ARRAY_SIZE(qcm2290_mdp),
-	.mdp = qcm2290_mdp,
-	.ctl_count = ARRAY_SIZE(qcm2290_ctl),
-	.ctl = qcm2290_ctl,
-	.sspp_count = ARRAY_SIZE(qcm2290_sspp),
-	.sspp = qcm2290_sspp,
-	.mixer_count = ARRAY_SIZE(qcm2290_lm),
-	.mixer = qcm2290_lm,
-	.dspp_count = ARRAY_SIZE(qcm2290_dspp),
-	.dspp = qcm2290_dspp,
-	.pingpong_count = ARRAY_SIZE(qcm2290_pp),
-	.pingpong = qcm2290_pp,
-	.intf_count = ARRAY_SIZE(qcm2290_intf),
-	.intf = qcm2290_intf,
-	.vbif_count = ARRAY_SIZE(sdm845_vbif),
-	.vbif = sdm845_vbif,
-	.perf = &qcm2290_perf_data,
-	.mdss_irqs = IRQ_SC7180_MASK,
-};
-
-static const struct dpu_mdss_hw_cfg_handler cfg_handler[] = {
-	{ .hw_rev = DPU_HW_VER_300, .dpu_cfg = &msm8998_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_301, .dpu_cfg = &msm8998_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_400, .dpu_cfg = &sdm845_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_401, .dpu_cfg = &sdm845_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_500, .dpu_cfg = &sm8150_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_501, .dpu_cfg = &sm8150_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_510, .dpu_cfg = &sc8180x_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_600, .dpu_cfg = &sm8250_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_620, .dpu_cfg = &sc7180_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_630, .dpu_cfg = &sm6115_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_650, .dpu_cfg = &qcm2290_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_700, .dpu_cfg = &sm8350_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_720, .dpu_cfg = &sc7280_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_800, .dpu_cfg = &sc8280xp_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_810, .dpu_cfg = &sm8450_dpu_cfg},
-	{ .hw_rev = DPU_HW_VER_900, .dpu_cfg = &sm8550_dpu_cfg},
-};
-
-const struct dpu_mdss_cfg *dpu_hw_catalog_init(u32 hw_rev)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(cfg_handler); i++) {
-		if (cfg_handler[i].hw_rev == hw_rev)
-			return cfg_handler[i].dpu_cfg;
-	}
+#include "catalog/dpu_6_0_sm8250.h"
+#include "catalog/dpu_6_2_sc7180.h"
+#include "catalog/dpu_6_3_sm6115.h"
+#include "catalog/dpu_6_5_qcm2290.h"
 
-	DPU_ERROR("unsupported chipset id:%X\n", hw_rev);
+#include "catalog/dpu_7_0_sm8350.h"
+#include "catalog/dpu_7_2_sc7280.h"
 
-	return ERR_PTR(-ENODEV);
-}
+#include "catalog/dpu_8_0_sc8280xp.h"
+#include "catalog/dpu_8_1_sm8450.h"
 
+#include "catalog/dpu_9_0_sm8550.h"
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
index e6590302b3bf..71584cd56fd7 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
@@ -19,47 +19,6 @@
  */
 #define MAX_BLOCKS    12
 
-#define DPU_HW_VER(MAJOR, MINOR, STEP) (((MAJOR & 0xF) << 28)    |\
-		((MINOR & 0xFFF) << 16)  |\
-		(STEP & 0xFFFF))
-
-#define DPU_HW_MAJOR(rev)		((rev) >> 28)
-#define DPU_HW_MINOR(rev)		(((rev) >> 16) & 0xFFF)
-#define DPU_HW_STEP(rev)		((rev) & 0xFFFF)
-#define DPU_HW_MAJOR_MINOR(rev)		((rev) >> 16)
-
-#define IS_DPU_MAJOR_MINOR_SAME(rev1, rev2)   \
-	(DPU_HW_MAJOR_MINOR((rev1)) == DPU_HW_MAJOR_MINOR((rev2)))
-
-#define DPU_HW_VER_170	DPU_HW_VER(1, 7, 0) /* 8996 v1.0 */
-#define DPU_HW_VER_171	DPU_HW_VER(1, 7, 1) /* 8996 v2.0 */
-#define DPU_HW_VER_172	DPU_HW_VER(1, 7, 2) /* 8996 v3.0 */
-#define DPU_HW_VER_300	DPU_HW_VER(3, 0, 0) /* 8998 v1.0 */
-#define DPU_HW_VER_301	DPU_HW_VER(3, 0, 1) /* 8998 v1.1 */
-#define DPU_HW_VER_400	DPU_HW_VER(4, 0, 0) /* sdm845 v1.0 */
-#define DPU_HW_VER_401	DPU_HW_VER(4, 0, 1) /* sdm845 v2.0 */
-#define DPU_HW_VER_410	DPU_HW_VER(4, 1, 0) /* sdm670 v1.0 */
-#define DPU_HW_VER_500	DPU_HW_VER(5, 0, 0) /* sm8150 v1.0 */
-#define DPU_HW_VER_501	DPU_HW_VER(5, 0, 1) /* sm8150 v2.0 */
-#define DPU_HW_VER_510	DPU_HW_VER(5, 1, 1) /* sc8180 */
-#define DPU_HW_VER_600	DPU_HW_VER(6, 0, 0) /* sm8250 */
-#define DPU_HW_VER_620	DPU_HW_VER(6, 2, 0) /* sc7180 v1.0 */
-#define DPU_HW_VER_630	DPU_HW_VER(6, 3, 0) /* sm6115|sm4250 */
-#define DPU_HW_VER_650	DPU_HW_VER(6, 5, 0) /* qcm2290|sm4125 */
-#define DPU_HW_VER_700	DPU_HW_VER(7, 0, 0) /* sm8350 */
-#define DPU_HW_VER_720	DPU_HW_VER(7, 2, 0) /* sc7280 */
-#define DPU_HW_VER_800	DPU_HW_VER(8, 0, 0) /* sc8280xp */
-#define DPU_HW_VER_810	DPU_HW_VER(8, 1, 0) /* sm8450 */
-#define DPU_HW_VER_900	DPU_HW_VER(9, 0, 0) /* sm8550 */
-
-#define IS_MSM8996_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_170)
-#define IS_MSM8998_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_300)
-#define IS_SDM845_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_400)
-#define IS_SDM670_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_410)
-#define IS_SDM855_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_500)
-#define IS_SC7180_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_620)
-#define IS_SC7280_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_720)
-
 #define DPU_HW_BLK_NAME_LEN	16
 
 #define MAX_IMG_WIDTH 0x3fff
@@ -169,10 +128,12 @@ enum {
  * DSPP sub-blocks
  * @DPU_DSPP_PCC             Panel color correction block
  * @DPU_DSPP_GC              Gamma correction block
+ * @DPU_DSPP_IGC             Inverse gamma correction block
  */
 enum {
 	DPU_DSPP_PCC = 0x1,
 	DPU_DSPP_GC,
+	DPU_DSPP_IGC,
 	DPU_DSPP_MAX
 };
 
@@ -200,6 +161,7 @@ enum {
  * @DPU_CTL_FETCH_ACTIVE:	Active CTL for fetch HW (SSPPs)
  * @DPU_CTL_VM_CFG:		CTL config to support multiple VMs
  * @DPU_CTL_HAS_LAYER_EXT4:	CTL has the CTL_LAYER_EXT4 register
+ * @DPU_CTL_DSPP_BLOCK_FLUSH:	CTL config to support dspp sub-block flush
  * @DPU_CTL_MAX
  */
 enum {
@@ -208,22 +170,25 @@ enum {
 	DPU_CTL_FETCH_ACTIVE,
 	DPU_CTL_VM_CFG,
 	DPU_CTL_HAS_LAYER_EXT4,
+	DPU_CTL_DSPP_SUB_BLOCK_FLUSH,
 	DPU_CTL_MAX
 };
 
 /**
  * INTF sub-blocks
- * @DPU_INTF_INPUT_CTRL         Supports the setting of pp block from which
- *                              pixel data arrives to this INTF
- * @DPU_INTF_TE                 INTF block has TE configuration support
- * @DPU_DATA_HCTL_EN            Allows data to be transferred at different rate
-                                than video timing
+ * @DPU_INTF_INPUT_CTRL             Supports the setting of pp block from which
+ *                                  pixel data arrives to this INTF
+ * @DPU_INTF_TE                     INTF block has TE configuration support
+ * @DPU_DATA_HCTL_EN                Allows data to be transferred at different rate
+ *                                  than video timing
+ * @DPU_INTF_STATUS_SUPPORTED       INTF block has INTF_STATUS register
  * @DPU_INTF_MAX
  */
 enum {
 	DPU_INTF_INPUT_CTRL = 0x1,
 	DPU_INTF_TE,
 	DPU_DATA_HCTL_EN,
+	DPU_INTF_STATUS_SUPPORTED,
 	DPU_INTF_MAX
 };
 
@@ -393,8 +358,6 @@ struct dpu_rotation_cfg {
  * @max_mixer_blendstages max layer mixer blend stages or
  *                       supported z order
  * @qseed_type         qseed2 or qseed3 support.
- * @smart_dma_rev      Supported version of SmartDMA feature.
- * @ubwc_version       UBWC feature version (0x0 for not supported)
  * @has_src_split      source split feature status
  * @has_dim_layer      dim layer feature status
  * @has_idle_pc        indicate if idle power collapse feature is supported
@@ -408,8 +371,6 @@ struct dpu_caps {
 	u32 max_mixer_width;
 	u32 max_mixer_blendstages;
 	u32 qseed_type;
-	u32 smart_dma_rev;
-	u32 ubwc_version;
 	bool has_src_split;
 	bool has_dim_layer;
 	bool has_idle_pc;
@@ -538,15 +499,24 @@ struct dpu_clk_ctrl_reg {
  * @id:                index identifying this block
  * @base:              register base offset to mdss
  * @features           bit mask identifying sub-blocks/features
- * @highest_bank_bit:  UBWC parameter
- * @ubwc_swizzle:      ubwc default swizzle setting
  * @clk_ctrls          clock control register definition
  */
 struct dpu_mdp_cfg {
 	DPU_HW_BLK_INFO;
+	struct dpu_clk_ctrl_reg clk_ctrls[DPU_CLK_CTRL_MAX];
+};
+
+/**
+ * struct dpu_ubwc_cfg - UBWC and memory configuration
+ *
+ * @ubwc_version       UBWC feature version (0x0 for not supported)
+ * @highest_bank_bit:  UBWC parameter
+ * @ubwc_swizzle:      ubwc default swizzle setting
+ */
+struct dpu_ubwc_cfg {
+	u32 ubwc_version;
 	u32 highest_bank_bit;
 	u32 ubwc_swizzle;
-	struct dpu_clk_ctrl_reg clk_ctrls[DPU_CLK_CTRL_MAX];
 };
 
 /* struct dpu_ctl_cfg : MDP CTL instance info
@@ -848,6 +818,8 @@ struct dpu_perf_cfg {
 struct dpu_mdss_cfg {
 	const struct dpu_caps *caps;
 
+	const struct dpu_ubwc_cfg *ubwc;
+
 	u32 mdp_count;
 	const struct dpu_mdp_cfg *mdp;
 
@@ -867,7 +839,7 @@ struct dpu_mdss_cfg {
 	const struct dpu_merge_3d_cfg *merge_3d;
 
 	u32 dsc_count;
-	struct dpu_dsc_cfg *dsc;
+	const struct dpu_dsc_cfg *dsc;
 
 	u32 intf_count;
 	const struct dpu_intf_cfg *intf;
@@ -896,18 +868,18 @@ struct dpu_mdss_cfg {
 	unsigned long mdss_irqs;
 };
 
-struct dpu_mdss_hw_cfg_handler {
-	u32 hw_rev;
-	const struct dpu_mdss_cfg *dpu_cfg;
-};
-
-/**
- * dpu_hw_catalog_init - dpu hardware catalog init API retrieves
- * hardcoded target specific catalog information in config structure
- * @hw_rev:       caller needs provide the hardware revision.
- *
- * Return: dpu config structure
- */
-const struct dpu_mdss_cfg *dpu_hw_catalog_init(u32 hw_rev);
+extern const struct dpu_mdss_cfg dpu_msm8998_cfg;
+extern const struct dpu_mdss_cfg dpu_sdm845_cfg;
+extern const struct dpu_mdss_cfg dpu_sm8150_cfg;
+extern const struct dpu_mdss_cfg dpu_sc8180x_cfg;
+extern const struct dpu_mdss_cfg dpu_sm8250_cfg;
+extern const struct dpu_mdss_cfg dpu_sc7180_cfg;
+extern const struct dpu_mdss_cfg dpu_sm6115_cfg;
+extern const struct dpu_mdss_cfg dpu_qcm2290_cfg;
+extern const struct dpu_mdss_cfg dpu_sm8350_cfg;
+extern const struct dpu_mdss_cfg dpu_sc7280_cfg;
+extern const struct dpu_mdss_cfg dpu_sc8280xp_cfg;
+extern const struct dpu_mdss_cfg dpu_sm8450_cfg;
+extern const struct dpu_mdss_cfg dpu_sm8550_cfg;
 
 #endif /* _DPU_HW_CATALOG_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
index 6c53ea560ffa..bbdc95ce374a 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
@@ -26,15 +26,16 @@
 #define   CTL_SW_RESET                  0x030
 #define   CTL_LAYER_EXTN_OFFSET         0x40
 #define   CTL_MERGE_3D_ACTIVE           0x0E4
+#define   CTL_DSC_ACTIVE                0x0E8
 #define   CTL_WB_ACTIVE                 0x0EC
 #define   CTL_INTF_ACTIVE               0x0F4
+#define   CTL_FETCH_PIPE_ACTIVE         0x0FC
 #define   CTL_MERGE_3D_FLUSH            0x100
-#define   CTL_DSC_ACTIVE                0x0E8
 #define   CTL_DSC_FLUSH                0x104
 #define   CTL_WB_FLUSH                  0x108
 #define   CTL_INTF_FLUSH                0x110
 #define   CTL_INTF_MASTER               0x134
-#define   CTL_FETCH_PIPE_ACTIVE         0x0FC
+#define   CTL_DSPP_n_FLUSH(n)           ((0x13C) + ((n) * 4))
 
 #define CTL_MIXER_BORDER_OUT            BIT(24)
 #define CTL_FLUSH_MASK_CTL              BIT(17)
@@ -44,6 +45,7 @@
 #define  DSC_IDX        22
 #define  INTF_IDX       31
 #define WB_IDX          16
+#define  DSPP_IDX       29  /* From DPU hw rev 7.x.x */
 #define CTL_INVALID_BIT                 0xffff
 #define CTL_DEFAULT_GROUP_ID		0xf
 
@@ -115,6 +117,9 @@ static inline void dpu_hw_ctl_clear_pending_flush(struct dpu_hw_ctl *ctx)
 	trace_dpu_hw_ctl_clear_pending_flush(ctx->pending_flush_mask,
 				     dpu_hw_ctl_get_flush_register(ctx));
 	ctx->pending_flush_mask = 0x0;
+
+	memset(ctx->pending_dspp_flush_mask, 0,
+		sizeof(ctx->pending_dspp_flush_mask));
 }
 
 static inline void dpu_hw_ctl_update_pending_flush(struct dpu_hw_ctl *ctx,
@@ -132,6 +137,8 @@ static u32 dpu_hw_ctl_get_pending_flush(struct dpu_hw_ctl *ctx)
 
 static inline void dpu_hw_ctl_trigger_flush_v1(struct dpu_hw_ctl *ctx)
 {
+	int dspp;
+
 	if (ctx->pending_flush_mask & BIT(MERGE_3D_IDX))
 		DPU_REG_WRITE(&ctx->hw, CTL_MERGE_3D_FLUSH,
 				ctx->pending_merge_3d_flush_mask);
@@ -142,6 +149,13 @@ static inline void dpu_hw_ctl_trigger_flush_v1(struct dpu_hw_ctl *ctx)
 		DPU_REG_WRITE(&ctx->hw, CTL_WB_FLUSH,
 				ctx->pending_wb_flush_mask);
 
+	if (ctx->pending_flush_mask & BIT(DSPP_IDX))
+		for (dspp = DSPP_0; dspp < DSPP_MAX; dspp++) {
+			if (ctx->pending_dspp_flush_mask[dspp - DSPP_0])
+				DPU_REG_WRITE(&ctx->hw,
+				CTL_DSPP_n_FLUSH(dspp - DSPP_0),
+				ctx->pending_dspp_flush_mask[dspp - DSPP_0]);
+		}
 	DPU_REG_WRITE(&ctx->hw, CTL_FLUSH, ctx->pending_flush_mask);
 }
 
@@ -289,7 +303,7 @@ static void dpu_hw_ctl_update_pending_flush_merge_3d_v1(struct dpu_hw_ctl *ctx,
 }
 
 static void dpu_hw_ctl_update_pending_flush_dspp(struct dpu_hw_ctl *ctx,
-	enum dpu_dspp dspp)
+	enum dpu_dspp dspp, u32 dspp_sub_blk)
 {
 	switch (dspp) {
 	case DSPP_0:
@@ -309,6 +323,29 @@ static void dpu_hw_ctl_update_pending_flush_dspp(struct dpu_hw_ctl *ctx,
 	}
 }
 
+static void dpu_hw_ctl_update_pending_flush_dspp_sub_blocks(
+	struct dpu_hw_ctl *ctx,	enum dpu_dspp dspp, u32 dspp_sub_blk)
+{
+	if (dspp >= DSPP_MAX)
+		return;
+
+	switch (dspp_sub_blk) {
+	case DPU_DSPP_IGC:
+		ctx->pending_dspp_flush_mask[dspp - DSPP_0] |= BIT(2);
+		break;
+	case DPU_DSPP_PCC:
+		ctx->pending_dspp_flush_mask[dspp - DSPP_0] |= BIT(4);
+		break;
+	case DPU_DSPP_GC:
+		ctx->pending_dspp_flush_mask[dspp - DSPP_0] |= BIT(5);
+		break;
+	default:
+		return;
+	}
+
+	ctx->pending_flush_mask |= BIT(DSPP_IDX);
+}
+
 static u32 dpu_hw_ctl_poll_reset_status(struct dpu_hw_ctl *ctx, u32 timeout_us)
 {
 	struct dpu_hw_blk_reg_map *c = &ctx->hw;
@@ -630,7 +667,11 @@ static void _setup_ctl_ops(struct dpu_hw_ctl_ops *ops,
 	ops->setup_blendstage = dpu_hw_ctl_setup_blendstage;
 	ops->update_pending_flush_sspp = dpu_hw_ctl_update_pending_flush_sspp;
 	ops->update_pending_flush_mixer = dpu_hw_ctl_update_pending_flush_mixer;
-	ops->update_pending_flush_dspp = dpu_hw_ctl_update_pending_flush_dspp;
+	if (cap & BIT(DPU_CTL_DSPP_SUB_BLOCK_FLUSH))
+		ops->update_pending_flush_dspp = dpu_hw_ctl_update_pending_flush_dspp_sub_blocks;
+	else
+		ops->update_pending_flush_dspp = dpu_hw_ctl_update_pending_flush_dspp;
+
 	if (cap & BIT(DPU_CTL_FETCH_ACTIVE))
 		ops->set_active_pipes = dpu_hw_ctl_set_fetch_pipe_active;
 };
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
index 96c012ec8467..78611a831697 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
@@ -152,9 +152,11 @@ struct dpu_hw_ctl_ops {
 	 * No effect on hardware
 	 * @ctx       : ctl path ctx pointer
 	 * @blk       : DSPP block index
+	 * @dspp_sub_blk : DSPP sub-block index
 	 */
 	void (*update_pending_flush_dspp)(struct dpu_hw_ctl *ctx,
-		enum dpu_dspp blk);
+		enum dpu_dspp blk, u32 dspp_sub_blk);
+
 	/**
 	 * Write the value of the pending_flush_mask to hardware
 	 * @ctx       : ctl path ctx pointer
@@ -242,6 +244,7 @@ struct dpu_hw_ctl {
 	u32 pending_intf_flush_mask;
 	u32 pending_wb_flush_mask;
 	u32 pending_merge_3d_flush_mask;
+	u32 pending_dspp_flush_mask[DSPP_MAX - DSPP_0];
 
 	/* ops */
 	struct dpu_hw_ctl_ops ops;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c
index 619926da1441..4e1396575e6a 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c
@@ -175,7 +175,7 @@ static void dpu_hw_dsc_bind_pingpong_blk(
 	DPU_REG_WRITE(c, dsc_ctl_offset, mux_cfg);
 }
 
-static struct dpu_dsc_cfg *_dsc_offset(enum dpu_dsc dsc,
+static const struct dpu_dsc_cfg *_dsc_offset(enum dpu_dsc dsc,
 				       const struct dpu_mdss_cfg *m,
 				       void __iomem *addr,
 				       struct dpu_hw_blk_reg_map *b)
@@ -207,7 +207,7 @@ struct dpu_hw_dsc *dpu_hw_dsc_init(enum dpu_dsc idx, void __iomem *addr,
 				   const struct dpu_mdss_cfg *m)
 {
 	struct dpu_hw_dsc *c;
-	struct dpu_dsc_cfg *cfg;
+	const struct dpu_dsc_cfg *cfg;
 
 	c = kzalloc(sizeof(*c), GFP_KERNEL);
 	if (!c)
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c
index 7ce66bf3f4c8..84ee2efa9c66 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c
@@ -62,6 +62,7 @@
 #define   INTF_LINE_COUNT               0x0B0
 
 #define   INTF_MUX                      0x25C
+#define   INTF_STATUS                   0x26C
 
 #define INTF_CFG_ACTIVE_H_EN	BIT(29)
 #define INTF_CFG_ACTIVE_V_EN	BIT(30)
@@ -297,8 +298,13 @@ static void dpu_hw_intf_get_status(
 		struct intf_status *s)
 {
 	struct dpu_hw_blk_reg_map *c = &intf->hw;
+	unsigned long cap = intf->cap->features;
+
+	if (cap & BIT(DPU_INTF_STATUS_SUPPORTED))
+		s->is_en = DPU_REG_READ(c, INTF_STATUS) & BIT(0);
+	else
+		s->is_en = DPU_REG_READ(c, INTF_TIMING_ENGINE_EN);
 
-	s->is_en = DPU_REG_READ(c, INTF_TIMING_ENGINE_EN);
 	s->is_prog_fetch_en = !!(DPU_REG_READ(c, INTF_CONFIG) & BIT(31));
 	if (s->is_en) {
 		s->frame_count = DPU_REG_READ(c, INTF_FRAME_COUNT);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
index 4246ab0b3bee..cf70a9bd1034 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
@@ -136,7 +136,7 @@
 #define TS_CLK			19200000
 
 
-static int _sspp_subblk_offset(struct dpu_hw_pipe *ctx,
+static int _sspp_subblk_offset(struct dpu_hw_sspp *ctx,
 		int s_id,
 		u32 *idx)
 {
@@ -168,17 +168,16 @@ static int _sspp_subblk_offset(struct dpu_hw_pipe *ctx,
 	return rc;
 }
 
-static void dpu_hw_sspp_setup_multirect(struct dpu_hw_pipe *ctx,
-		enum dpu_sspp_multirect_index index,
-		enum dpu_sspp_multirect_mode mode)
+static void dpu_hw_sspp_setup_multirect(struct dpu_sw_pipe *pipe)
 {
+	struct dpu_hw_sspp *ctx = pipe->sspp;
 	u32 mode_mask;
 	u32 idx;
 
 	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
 		return;
 
-	if (index == DPU_SSPP_RECT_SOLO) {
+	if (pipe->multirect_index == DPU_SSPP_RECT_SOLO) {
 		/**
 		 * if rect index is RECT_SOLO, we cannot expect a
 		 * virtual plane sharing the same SSPP id. So we go
@@ -187,8 +186,8 @@ static void dpu_hw_sspp_setup_multirect(struct dpu_hw_pipe *ctx,
 		mode_mask = 0;
 	} else {
 		mode_mask = DPU_REG_READ(&ctx->hw, SSPP_MULTIRECT_OPMODE + idx);
-		mode_mask |= index;
-		if (mode == DPU_SSPP_MULTIRECT_TIME_MX)
+		mode_mask |= pipe->multirect_index;
+		if (pipe->multirect_mode == DPU_SSPP_MULTIRECT_TIME_MX)
 			mode_mask |= BIT(2);
 		else
 			mode_mask &= ~BIT(2);
@@ -197,7 +196,7 @@ static void dpu_hw_sspp_setup_multirect(struct dpu_hw_pipe *ctx,
 	DPU_REG_WRITE(&ctx->hw, SSPP_MULTIRECT_OPMODE + idx, mode_mask);
 }
 
-static void _sspp_setup_opmode(struct dpu_hw_pipe *ctx,
+static void _sspp_setup_opmode(struct dpu_hw_sspp *ctx,
 		u32 mask, u8 en)
 {
 	u32 idx;
@@ -218,7 +217,7 @@ static void _sspp_setup_opmode(struct dpu_hw_pipe *ctx,
 	DPU_REG_WRITE(&ctx->hw, SSPP_VIG_OP_MODE + idx, opmode);
 }
 
-static void _sspp_setup_csc10_opmode(struct dpu_hw_pipe *ctx,
+static void _sspp_setup_csc10_opmode(struct dpu_hw_sspp *ctx,
 		u32 mask, u8 en)
 {
 	u32 idx;
@@ -239,10 +238,10 @@ static void _sspp_setup_csc10_opmode(struct dpu_hw_pipe *ctx,
 /*
  * Setup source pixel format, flip,
  */
-static void dpu_hw_sspp_setup_format(struct dpu_hw_pipe *ctx,
-		const struct dpu_format *fmt, u32 flags,
-		enum dpu_sspp_multirect_index rect_mode)
+static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe,
+		const struct dpu_format *fmt, u32 flags)
 {
+	struct dpu_hw_sspp *ctx = pipe->sspp;
 	struct dpu_hw_blk_reg_map *c;
 	u32 chroma_samp, unpack, src_format;
 	u32 opmode = 0;
@@ -253,7 +252,8 @@ static void dpu_hw_sspp_setup_format(struct dpu_hw_pipe *ctx,
 	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx) || !fmt)
 		return;
 
-	if (rect_mode == DPU_SSPP_RECT_SOLO || rect_mode == DPU_SSPP_RECT_0) {
+	if (pipe->multirect_index == DPU_SSPP_RECT_SOLO ||
+	    pipe->multirect_index == DPU_SSPP_RECT_0) {
 		op_mode_off = SSPP_SRC_OP_MODE;
 		unpack_pat_off = SSPP_SRC_UNPACK_PATTERN;
 		format_off = SSPP_SRC_FORMAT;
@@ -307,25 +307,25 @@ static void dpu_hw_sspp_setup_format(struct dpu_hw_pipe *ctx,
 		src_format |= (fmt->fetch_mode & 3) << 30; /*FRAME_FORMAT */
 		DPU_REG_WRITE(c, SSPP_FETCH_CONFIG,
 			DPU_FETCH_CONFIG_RESET_VALUE |
-			ctx->mdp->highest_bank_bit << 18);
-		switch (ctx->catalog->caps->ubwc_version) {
+			ctx->ubwc->highest_bank_bit << 18);
+		switch (ctx->ubwc->ubwc_version) {
 		case DPU_HW_UBWC_VER_10:
 			fast_clear = fmt->alpha_enable ? BIT(31) : 0;
 			DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL,
-					fast_clear | (ctx->mdp->ubwc_swizzle & 0x1) |
+					fast_clear | (ctx->ubwc->ubwc_swizzle & 0x1) |
 					BIT(8) |
-					(ctx->mdp->highest_bank_bit << 4));
+					(ctx->ubwc->highest_bank_bit << 4));
 			break;
 		case DPU_HW_UBWC_VER_20:
 			fast_clear = fmt->alpha_enable ? BIT(31) : 0;
 			DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL,
-					fast_clear | (ctx->mdp->ubwc_swizzle) |
-					(ctx->mdp->highest_bank_bit << 4));
+					fast_clear | (ctx->ubwc->ubwc_swizzle) |
+					(ctx->ubwc->highest_bank_bit << 4));
 			break;
 		case DPU_HW_UBWC_VER_30:
 			DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL,
-					BIT(30) | (ctx->mdp->ubwc_swizzle) |
-					(ctx->mdp->highest_bank_bit << 4));
+					BIT(30) | (ctx->ubwc->ubwc_swizzle) |
+					(ctx->ubwc->highest_bank_bit << 4));
 			break;
 		case DPU_HW_UBWC_VER_40:
 			DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL,
@@ -360,7 +360,7 @@ static void dpu_hw_sspp_setup_format(struct dpu_hw_pipe *ctx,
 	DPU_REG_WRITE(c, SSPP_UBWC_ERROR_STATUS + idx, BIT(31));
 }
 
-static void dpu_hw_sspp_setup_pe_config(struct dpu_hw_pipe *ctx,
+static void dpu_hw_sspp_setup_pe_config(struct dpu_hw_sspp *ctx,
 		struct dpu_hw_pixel_ext *pe_ext)
 {
 	struct dpu_hw_blk_reg_map *c;
@@ -418,23 +418,22 @@ static void dpu_hw_sspp_setup_pe_config(struct dpu_hw_pipe *ctx,
 			tot_req_pixels[3]);
 }
 
-static void _dpu_hw_sspp_setup_scaler3(struct dpu_hw_pipe *ctx,
-		struct dpu_hw_pipe_cfg *sspp,
-		void *scaler_cfg)
+static void _dpu_hw_sspp_setup_scaler3(struct dpu_hw_sspp *ctx,
+		struct dpu_hw_scaler3_cfg *scaler3_cfg,
+		const struct dpu_format *format)
 {
 	u32 idx;
-	struct dpu_hw_scaler3_cfg *scaler3_cfg = scaler_cfg;
 
-	if (_sspp_subblk_offset(ctx, DPU_SSPP_SCALER_QSEED3, &idx) || !sspp
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SCALER_QSEED3, &idx)
 		|| !scaler3_cfg)
 		return;
 
 	dpu_hw_setup_scaler3(&ctx->hw, scaler3_cfg, idx,
 			ctx->cap->sblk->scaler_blk.version,
-			sspp->layout.format);
+			format);
 }
 
-static u32 _dpu_hw_sspp_get_scaler3_ver(struct dpu_hw_pipe *ctx)
+static u32 _dpu_hw_sspp_get_scaler3_ver(struct dpu_hw_sspp *ctx)
 {
 	u32 idx;
 
@@ -447,12 +446,12 @@ static u32 _dpu_hw_sspp_get_scaler3_ver(struct dpu_hw_pipe *ctx)
 /*
  * dpu_hw_sspp_setup_rects()
  */
-static void dpu_hw_sspp_setup_rects(struct dpu_hw_pipe *ctx,
-		struct dpu_hw_pipe_cfg *cfg,
-		enum dpu_sspp_multirect_index rect_index)
+static void dpu_hw_sspp_setup_rects(struct dpu_sw_pipe *pipe,
+		struct dpu_sw_pipe_cfg *cfg)
 {
+	struct dpu_hw_sspp *ctx = pipe->sspp;
 	struct dpu_hw_blk_reg_map *c;
-	u32 src_size, src_xy, dst_size, dst_xy, ystride0, ystride1;
+	u32 src_size, src_xy, dst_size, dst_xy;
 	u32 src_size_off, src_xy_off, out_size_off, out_xy_off;
 	u32 idx;
 
@@ -461,7 +460,8 @@ static void dpu_hw_sspp_setup_rects(struct dpu_hw_pipe *ctx,
 
 	c = &ctx->hw;
 
-	if (rect_index == DPU_SSPP_RECT_SOLO || rect_index == DPU_SSPP_RECT_0) {
+	if (pipe->multirect_index == DPU_SSPP_RECT_SOLO ||
+	    pipe->multirect_index == DPU_SSPP_RECT_0) {
 		src_size_off = SSPP_SRC_SIZE;
 		src_xy_off = SSPP_SRC_XY;
 		out_size_off = SSPP_OUT_SIZE;
@@ -482,68 +482,69 @@ static void dpu_hw_sspp_setup_rects(struct dpu_hw_pipe *ctx,
 	dst_size = (drm_rect_height(&cfg->dst_rect) << 16) |
 		drm_rect_width(&cfg->dst_rect);
 
-	if (rect_index == DPU_SSPP_RECT_SOLO) {
-		ystride0 = (cfg->layout.plane_pitch[0]) |
-			(cfg->layout.plane_pitch[1] << 16);
-		ystride1 = (cfg->layout.plane_pitch[2]) |
-			(cfg->layout.plane_pitch[3] << 16);
-	} else {
-		ystride0 = DPU_REG_READ(c, SSPP_SRC_YSTRIDE0 + idx);
-		ystride1 = DPU_REG_READ(c, SSPP_SRC_YSTRIDE1 + idx);
-
-		if (rect_index == DPU_SSPP_RECT_0) {
-			ystride0 = (ystride0 & 0xFFFF0000) |
-				(cfg->layout.plane_pitch[0] & 0x0000FFFF);
-			ystride1 = (ystride1 & 0xFFFF0000)|
-				(cfg->layout.plane_pitch[2] & 0x0000FFFF);
-		} else {
-			ystride0 = (ystride0 & 0x0000FFFF) |
-				((cfg->layout.plane_pitch[0] << 16) &
-				 0xFFFF0000);
-			ystride1 = (ystride1 & 0x0000FFFF) |
-				((cfg->layout.plane_pitch[2] << 16) &
-				 0xFFFF0000);
-		}
-	}
-
 	/* rectangle register programming */
 	DPU_REG_WRITE(c, src_size_off + idx, src_size);
 	DPU_REG_WRITE(c, src_xy_off + idx, src_xy);
 	DPU_REG_WRITE(c, out_size_off + idx, dst_size);
 	DPU_REG_WRITE(c, out_xy_off + idx, dst_xy);
-
-	DPU_REG_WRITE(c, SSPP_SRC_YSTRIDE0 + idx, ystride0);
-	DPU_REG_WRITE(c, SSPP_SRC_YSTRIDE1 + idx, ystride1);
 }
 
-static void dpu_hw_sspp_setup_sourceaddress(struct dpu_hw_pipe *ctx,
-		struct dpu_hw_pipe_cfg *cfg,
-		enum dpu_sspp_multirect_index rect_mode)
+static void dpu_hw_sspp_setup_sourceaddress(struct dpu_sw_pipe *pipe,
+		struct dpu_hw_fmt_layout *layout)
 {
+	struct dpu_hw_sspp *ctx = pipe->sspp;
+	u32 ystride0, ystride1;
 	int i;
 	u32 idx;
 
 	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
 		return;
 
-	if (rect_mode == DPU_SSPP_RECT_SOLO) {
-		for (i = 0; i < ARRAY_SIZE(cfg->layout.plane_addr); i++)
+	if (pipe->multirect_index == DPU_SSPP_RECT_SOLO) {
+		for (i = 0; i < ARRAY_SIZE(layout->plane_addr); i++)
 			DPU_REG_WRITE(&ctx->hw, SSPP_SRC0_ADDR + idx + i * 0x4,
-					cfg->layout.plane_addr[i]);
-	} else if (rect_mode == DPU_SSPP_RECT_0) {
+					layout->plane_addr[i]);
+	} else if (pipe->multirect_index == DPU_SSPP_RECT_0) {
 		DPU_REG_WRITE(&ctx->hw, SSPP_SRC0_ADDR + idx,
-				cfg->layout.plane_addr[0]);
+				layout->plane_addr[0]);
 		DPU_REG_WRITE(&ctx->hw, SSPP_SRC2_ADDR + idx,
-				cfg->layout.plane_addr[2]);
+				layout->plane_addr[2]);
 	} else {
 		DPU_REG_WRITE(&ctx->hw, SSPP_SRC1_ADDR + idx,
-				cfg->layout.plane_addr[0]);
+				layout->plane_addr[0]);
 		DPU_REG_WRITE(&ctx->hw, SSPP_SRC3_ADDR + idx,
-				cfg->layout.plane_addr[2]);
+				layout->plane_addr[2]);
 	}
+
+	if (pipe->multirect_index == DPU_SSPP_RECT_SOLO) {
+		ystride0 = (layout->plane_pitch[0]) |
+			(layout->plane_pitch[1] << 16);
+		ystride1 = (layout->plane_pitch[2]) |
+			(layout->plane_pitch[3] << 16);
+	} else {
+		ystride0 = DPU_REG_READ(&ctx->hw, SSPP_SRC_YSTRIDE0 + idx);
+		ystride1 = DPU_REG_READ(&ctx->hw, SSPP_SRC_YSTRIDE1 + idx);
+
+		if (pipe->multirect_index == DPU_SSPP_RECT_0) {
+			ystride0 = (ystride0 & 0xFFFF0000) |
+				(layout->plane_pitch[0] & 0x0000FFFF);
+			ystride1 = (ystride1 & 0xFFFF0000)|
+				(layout->plane_pitch[2] & 0x0000FFFF);
+		} else {
+			ystride0 = (ystride0 & 0x0000FFFF) |
+				((layout->plane_pitch[0] << 16) &
+				 0xFFFF0000);
+			ystride1 = (ystride1 & 0x0000FFFF) |
+				((layout->plane_pitch[2] << 16) &
+				 0xFFFF0000);
+		}
+	}
+
+	DPU_REG_WRITE(&ctx->hw, SSPP_SRC_YSTRIDE0 + idx, ystride0);
+	DPU_REG_WRITE(&ctx->hw, SSPP_SRC_YSTRIDE1 + idx, ystride1);
 }
 
-static void dpu_hw_sspp_setup_csc(struct dpu_hw_pipe *ctx,
+static void dpu_hw_sspp_setup_csc(struct dpu_hw_sspp *ctx,
 		const struct dpu_csc_cfg *data)
 {
 	u32 idx;
@@ -560,22 +561,28 @@ static void dpu_hw_sspp_setup_csc(struct dpu_hw_pipe *ctx,
 	dpu_hw_csc_setup(&ctx->hw, idx, data, csc10);
 }
 
-static void dpu_hw_sspp_setup_solidfill(struct dpu_hw_pipe *ctx, u32 color, enum
-		dpu_sspp_multirect_index rect_index)
+static void dpu_hw_sspp_setup_solidfill(struct dpu_sw_pipe *pipe, u32 color)
 {
+	struct dpu_hw_sspp *ctx = pipe->sspp;
+	struct dpu_hw_fmt_layout cfg;
 	u32 idx;
 
 	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
 		return;
 
-	if (rect_index == DPU_SSPP_RECT_SOLO || rect_index == DPU_SSPP_RECT_0)
+	/* cleanup source addresses */
+	memset(&cfg, 0, sizeof(cfg));
+	ctx->ops.setup_sourceaddress(pipe, &cfg);
+
+	if (pipe->multirect_index == DPU_SSPP_RECT_SOLO ||
+	    pipe->multirect_index == DPU_SSPP_RECT_0)
 		DPU_REG_WRITE(&ctx->hw, SSPP_SRC_CONSTANT_COLOR + idx, color);
 	else
 		DPU_REG_WRITE(&ctx->hw, SSPP_SRC_CONSTANT_COLOR_REC1 + idx,
 				color);
 }
 
-static void dpu_hw_sspp_setup_danger_safe_lut(struct dpu_hw_pipe *ctx,
+static void dpu_hw_sspp_setup_danger_safe_lut(struct dpu_hw_sspp *ctx,
 			u32 danger_lut,
 			u32 safe_lut)
 {
@@ -588,7 +595,7 @@ static void dpu_hw_sspp_setup_danger_safe_lut(struct dpu_hw_pipe *ctx,
 	DPU_REG_WRITE(&ctx->hw, SSPP_SAFE_LUT + idx, safe_lut);
 }
 
-static void dpu_hw_sspp_setup_creq_lut(struct dpu_hw_pipe *ctx,
+static void dpu_hw_sspp_setup_creq_lut(struct dpu_hw_sspp *ctx,
 			u64 creq_lut)
 {
 	u32 idx;
@@ -605,7 +612,7 @@ static void dpu_hw_sspp_setup_creq_lut(struct dpu_hw_pipe *ctx,
 	}
 }
 
-static void dpu_hw_sspp_setup_qos_ctrl(struct dpu_hw_pipe *ctx,
+static void dpu_hw_sspp_setup_qos_ctrl(struct dpu_hw_sspp *ctx,
 		struct dpu_hw_pipe_qos_cfg *cfg)
 {
 	u32 idx;
@@ -630,10 +637,10 @@ static void dpu_hw_sspp_setup_qos_ctrl(struct dpu_hw_pipe *ctx,
 	DPU_REG_WRITE(&ctx->hw, SSPP_QOS_CTRL + idx, qos_ctrl);
 }
 
-static void dpu_hw_sspp_setup_cdp(struct dpu_hw_pipe *ctx,
-		struct dpu_hw_cdp_cfg *cfg,
-		enum dpu_sspp_multirect_index index)
+static void dpu_hw_sspp_setup_cdp(struct dpu_sw_pipe *pipe,
+		struct dpu_hw_cdp_cfg *cfg)
 {
+	struct dpu_hw_sspp *ctx = pipe->sspp;
 	u32 idx;
 	u32 cdp_cntl = 0;
 	u32 cdp_cntl_offset = 0;
@@ -644,7 +651,8 @@ static void dpu_hw_sspp_setup_cdp(struct dpu_hw_pipe *ctx,
 	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
 		return;
 
-	if (index == DPU_SSPP_RECT_SOLO || index == DPU_SSPP_RECT_0)
+	if (pipe->multirect_index == DPU_SSPP_RECT_SOLO ||
+	    pipe->multirect_index == DPU_SSPP_RECT_0)
 		cdp_cntl_offset = SSPP_CDP_CNTL;
 	else
 		cdp_cntl_offset = SSPP_CDP_CNTL_REC1;
@@ -661,7 +669,7 @@ static void dpu_hw_sspp_setup_cdp(struct dpu_hw_pipe *ctx,
 	DPU_REG_WRITE(&ctx->hw, cdp_cntl_offset, cdp_cntl);
 }
 
-static void _setup_layer_ops(struct dpu_hw_pipe *c,
+static void _setup_layer_ops(struct dpu_hw_sspp *c,
 		unsigned long features)
 {
 	if (test_bit(DPU_SSPP_SRC, &features)) {
@@ -699,7 +707,8 @@ static void _setup_layer_ops(struct dpu_hw_pipe *c,
 }
 
 #ifdef CONFIG_DEBUG_FS
-int _dpu_hw_sspp_init_debugfs(struct dpu_hw_pipe *hw_pipe, struct dpu_kms *kms, struct dentry *entry)
+int _dpu_hw_sspp_init_debugfs(struct dpu_hw_sspp *hw_pipe, struct dpu_kms *kms,
+			      struct dentry *entry)
 {
 	const struct dpu_sspp_cfg *cfg = hw_pipe->cap;
 	const struct dpu_sspp_sub_blks *sblk = cfg->sblk;
@@ -783,10 +792,10 @@ static const struct dpu_sspp_cfg *_sspp_offset(enum dpu_sspp sspp,
 	return ERR_PTR(-ENOMEM);
 }
 
-struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx,
+struct dpu_hw_sspp *dpu_hw_sspp_init(enum dpu_sspp idx,
 		void __iomem *addr, const struct dpu_mdss_cfg *catalog)
 {
-	struct dpu_hw_pipe *hw_pipe;
+	struct dpu_hw_sspp *hw_pipe;
 	const struct dpu_sspp_cfg *cfg;
 
 	if (!addr || !catalog)
@@ -804,7 +813,7 @@ struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx,
 
 	/* Assign ops */
 	hw_pipe->catalog = catalog;
-	hw_pipe->mdp = &catalog->mdp[0];
+	hw_pipe->ubwc = catalog->ubwc;
 	hw_pipe->idx = idx;
 	hw_pipe->cap = cfg;
 	_setup_layer_ops(hw_pipe, hw_pipe->cap->features);
@@ -812,7 +821,7 @@ struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx,
 	return hw_pipe;
 }
 
-void dpu_hw_sspp_destroy(struct dpu_hw_pipe *ctx)
+void dpu_hw_sspp_destroy(struct dpu_hw_sspp *ctx)
 {
 	kfree(ctx);
 }
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
index 0c95b7e64f6c..74b98b6b3bc3 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
@@ -10,7 +10,7 @@
 #include "dpu_hw_util.h"
 #include "dpu_formats.h"
 
-struct dpu_hw_pipe;
+struct dpu_hw_sspp;
 
 /**
  * Flags
@@ -153,20 +153,14 @@ struct dpu_hw_pixel_ext {
 };
 
 /**
- * struct dpu_hw_pipe_cfg : Pipe description
- * @layout:    format layout information for programming buffer to hardware
+ * struct dpu_sw_pipe_cfg : software pipe configuration
  * @src_rect:  src ROI, caller takes into account the different operations
  *             such as decimation, flip etc to program this field
  * @dest_rect: destination ROI.
- * @index:     index of the rectangle of SSPP
- * @mode:      parallel or time multiplex multirect mode
  */
-struct dpu_hw_pipe_cfg {
-	struct dpu_hw_fmt_layout layout;
+struct dpu_sw_pipe_cfg {
 	struct drm_rect src_rect;
 	struct drm_rect dst_rect;
-	enum dpu_sspp_multirect_index index;
-	enum dpu_sspp_multirect_mode mode;
 };
 
 /**
@@ -202,6 +196,18 @@ struct dpu_hw_pipe_ts_cfg {
 };
 
 /**
+ * struct dpu_sw_pipe - software pipe description
+ * @sspp:      backing SSPP pipe
+ * @index:     index of the rectangle of SSPP
+ * @mode:      parallel or time multiplex multirect mode
+ */
+struct dpu_sw_pipe {
+	struct dpu_hw_sspp *sspp;
+	enum dpu_sspp_multirect_index multirect_index;
+	enum dpu_sspp_multirect_mode multirect_mode;
+};
+
+/**
  * struct dpu_hw_sspp_ops - interface to the SSPP Hw driver functions
  * Caller must call the init function to get the pipe context for each pipe
  * Assumption is these functions will be called after clocks are enabled
@@ -209,77 +215,65 @@ struct dpu_hw_pipe_ts_cfg {
 struct dpu_hw_sspp_ops {
 	/**
 	 * setup_format - setup pixel format cropping rectangle, flip
-	 * @ctx: Pointer to pipe context
+	 * @pipe: Pointer to software pipe context
 	 * @cfg: Pointer to pipe config structure
 	 * @flags: Extra flags for format config
-	 * @index: rectangle index in multirect
 	 */
-	void (*setup_format)(struct dpu_hw_pipe *ctx,
-			const struct dpu_format *fmt, u32 flags,
-			enum dpu_sspp_multirect_index index);
+	void (*setup_format)(struct dpu_sw_pipe *pipe,
+			     const struct dpu_format *fmt, u32 flags);
 
 	/**
 	 * setup_rects - setup pipe ROI rectangles
-	 * @ctx: Pointer to pipe context
+	 * @pipe: Pointer to software pipe context
 	 * @cfg: Pointer to pipe config structure
-	 * @index: rectangle index in multirect
 	 */
-	void (*setup_rects)(struct dpu_hw_pipe *ctx,
-			struct dpu_hw_pipe_cfg *cfg,
-			enum dpu_sspp_multirect_index index);
+	void (*setup_rects)(struct dpu_sw_pipe *pipe,
+			    struct dpu_sw_pipe_cfg *cfg);
 
 	/**
 	 * setup_pe - setup pipe pixel extension
 	 * @ctx: Pointer to pipe context
 	 * @pe_ext: Pointer to pixel ext settings
 	 */
-	void (*setup_pe)(struct dpu_hw_pipe *ctx,
+	void (*setup_pe)(struct dpu_hw_sspp *ctx,
 			struct dpu_hw_pixel_ext *pe_ext);
 
 	/**
 	 * setup_sourceaddress - setup pipe source addresses
-	 * @ctx: Pointer to pipe context
-	 * @cfg: Pointer to pipe config structure
-	 * @index: rectangle index in multirect
+	 * @pipe: Pointer to software pipe context
+	 * @layout: format layout information for programming buffer to hardware
 	 */
-	void (*setup_sourceaddress)(struct dpu_hw_pipe *ctx,
-			struct dpu_hw_pipe_cfg *cfg,
-			enum dpu_sspp_multirect_index index);
+	void (*setup_sourceaddress)(struct dpu_sw_pipe *ctx,
+				    struct dpu_hw_fmt_layout *layout);
 
 	/**
 	 * setup_csc - setup color space coversion
 	 * @ctx: Pointer to pipe context
 	 * @data: Pointer to config structure
 	 */
-	void (*setup_csc)(struct dpu_hw_pipe *ctx, const struct dpu_csc_cfg *data);
+	void (*setup_csc)(struct dpu_hw_sspp *ctx, const struct dpu_csc_cfg *data);
 
 	/**
 	 * setup_solidfill - enable/disable colorfill
-	 * @ctx: Pointer to pipe context
+	 * @pipe: Pointer to software pipe context
 	 * @const_color: Fill color value
 	 * @flags: Pipe flags
-	 * @index: rectangle index in multirect
 	 */
-	void (*setup_solidfill)(struct dpu_hw_pipe *ctx, u32 color,
-			enum dpu_sspp_multirect_index index);
+	void (*setup_solidfill)(struct dpu_sw_pipe *pipe, u32 color);
 
 	/**
 	 * setup_multirect - setup multirect configuration
-	 * @ctx: Pointer to pipe context
-	 * @index: rectangle index in multirect
-	 * @mode: parallel fetch / time multiplex multirect mode
+	 * @pipe: Pointer to software pipe context
 	 */
 
-	void (*setup_multirect)(struct dpu_hw_pipe *ctx,
-			enum dpu_sspp_multirect_index index,
-			enum dpu_sspp_multirect_mode mode);
+	void (*setup_multirect)(struct dpu_sw_pipe *pipe);
 
 	/**
 	 * setup_sharpening - setup sharpening
 	 * @ctx: Pointer to pipe context
 	 * @cfg: Pointer to config structure
 	 */
-	void (*setup_sharpening)(struct dpu_hw_pipe *ctx,
+	void (*setup_sharpening)(struct dpu_hw_sspp *ctx,
 			struct dpu_hw_sharp_cfg *cfg);
 
 	/**
@@ -289,7 +283,7 @@ struct dpu_hw_sspp_ops {
 	 * @safe_lut: LUT for generate safe level based on fill level
 	 *
 	 */
-	void (*setup_danger_safe_lut)(struct dpu_hw_pipe *ctx,
+	void (*setup_danger_safe_lut)(struct dpu_hw_sspp *ctx,
 			u32 danger_lut,
 			u32 safe_lut);
 
@@ -299,7 +293,7 @@ struct dpu_hw_sspp_ops {
 	 * @creq_lut: LUT for generate creq level based on fill level
 	 *
 	 */
-	void (*setup_creq_lut)(struct dpu_hw_pipe *ctx,
+	void (*setup_creq_lut)(struct dpu_hw_sspp *ctx,
 			u64 creq_lut);
 
 	/**
@@ -308,7 +302,7 @@ struct dpu_hw_sspp_ops {
 	 * @cfg: Pointer to pipe QoS configuration
 	 *
 	 */
-	void (*setup_qos_ctrl)(struct dpu_hw_pipe *ctx,
+	void (*setup_qos_ctrl)(struct dpu_hw_sspp *ctx,
 			struct dpu_hw_pipe_qos_cfg *cfg);
 
 	/**
@@ -316,51 +310,48 @@ struct dpu_hw_sspp_ops {
 	 * @ctx: Pointer to pipe context
 	 * @cfg: Pointer to histogram configuration
 	 */
-	void (*setup_histogram)(struct dpu_hw_pipe *ctx,
+	void (*setup_histogram)(struct dpu_hw_sspp *ctx,
 			void *cfg);
 
 	/**
 	 * setup_scaler - setup scaler
-	 * @ctx: Pointer to pipe context
-	 * @pipe_cfg: Pointer to pipe configuration
-	 * @scaler_cfg: Pointer to scaler configuration
+	 * @scaler3_cfg: Pointer to scaler configuration
+	 * @format: pixel format parameters
 	 */
-	void (*setup_scaler)(struct dpu_hw_pipe *ctx,
-		struct dpu_hw_pipe_cfg *pipe_cfg,
-		void *scaler_cfg);
+	void (*setup_scaler)(struct dpu_hw_sspp *ctx,
+		struct dpu_hw_scaler3_cfg *scaler3_cfg,
+		const struct dpu_format *format);
 
 	/**
 	 * get_scaler_ver - get scaler h/w version
 	 * @ctx: Pointer to pipe context
 	 */
-	u32 (*get_scaler_ver)(struct dpu_hw_pipe *ctx);
+	u32 (*get_scaler_ver)(struct dpu_hw_sspp *ctx);
 
 	/**
 	 * setup_cdp - setup client driven prefetch
-	 * @ctx: Pointer to pipe context
+	 * @pipe: Pointer to software pipe context
 	 * @cfg: Pointer to cdp configuration
-	 * @index: rectangle index in multirect
 	 */
-	void (*setup_cdp)(struct dpu_hw_pipe *ctx,
-			struct dpu_hw_cdp_cfg *cfg,
-			enum dpu_sspp_multirect_index index);
+	void (*setup_cdp)(struct dpu_sw_pipe *pipe,
+			  struct dpu_hw_cdp_cfg *cfg);
 };
 
 /**
- * struct dpu_hw_pipe - pipe description
+ * struct dpu_hw_sspp - pipe description
  * @base: hardware block base structure
  * @hw: block hardware details
  * @catalog: back pointer to catalog
- * @mdp: pointer to associated mdp portion of the catalog
+ * @ubwc: ubwc configuration data
  * @idx: pipe index
  * @cap: pointer to layer_cfg
  * @ops: pointer to operations possible for this pipe
  */
-struct dpu_hw_pipe {
+struct dpu_hw_sspp {
 	struct dpu_hw_blk base;
 	struct dpu_hw_blk_reg_map hw;
 	const struct dpu_mdss_cfg *catalog;
-	const struct dpu_mdp_cfg *mdp;
+	const struct dpu_ubwc_cfg *ubwc;
 
 	/* Pipe */
 	enum dpu_sspp idx;
@@ -378,7 +369,7 @@ struct dpu_kms;
  * @addr: Mapped register io address of MDP
  * @catalog : Pointer to mdss catalog data
  */
-struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx,
+struct dpu_hw_sspp *dpu_hw_sspp_init(enum dpu_sspp idx,
 		void __iomem *addr, const struct dpu_mdss_cfg *catalog);
 
 /**
@@ -386,10 +377,10 @@ struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx,
  * should be called during Hw pipe cleanup.
  * @ctx:  Pointer to SSPP driver context returned by dpu_hw_sspp_init
  */
-void dpu_hw_sspp_destroy(struct dpu_hw_pipe *ctx);
+void dpu_hw_sspp_destroy(struct dpu_hw_sspp *ctx);
 
-void dpu_debugfs_sspp_init(struct dpu_kms *dpu_kms, struct dentry *debugfs_root);
-int _dpu_hw_sspp_init_debugfs(struct dpu_hw_pipe *hw_pipe, struct dpu_kms *kms, struct dentry *entry);
+int _dpu_hw_sspp_init_debugfs(struct dpu_hw_sspp *hw_pipe, struct dpu_kms *kms,
+			      struct dentry *entry);
 
 #endif /*_DPU_HW_SSPP_H */
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
index a683bd9b5a04..0e7a68714e9e 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
@@ -250,6 +250,24 @@ void dpu_debugfs_create_regset32(const char *name, umode_t mode,
 	debugfs_create_file(name, mode, parent, regset, &dpu_regset32_fops);
 }
 
+static void dpu_debugfs_sspp_init(struct dpu_kms *dpu_kms, struct dentry *debugfs_root)
+{
+	struct dentry *entry = debugfs_create_dir("sspp", debugfs_root);
+	int i;
+
+	if (IS_ERR(entry))
+		return;
+
+	for (i = SSPP_NONE; i < SSPP_MAX; i++) {
+		struct dpu_hw_sspp *hw = dpu_rm_get_sspp(&dpu_kms->rm, i);
+
+		if (!hw)
+			continue;
+
+		_dpu_hw_sspp_init_debugfs(hw, dpu_kms, entry);
+	}
+}
+
 static int dpu_kms_debugfs_init(struct msm_kms *kms, struct drm_minor *minor)
 {
 	struct dpu_kms *dpu_kms = to_dpu_kms(kms);
@@ -411,40 +429,6 @@ static void dpu_kms_disable_commit(struct msm_kms *kms)
 	pm_runtime_put_sync(&dpu_kms->pdev->dev);
 }
 
-static ktime_t dpu_kms_vsync_time(struct msm_kms *kms, struct drm_crtc *crtc)
-{
-	struct drm_encoder *encoder;
-
-	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask) {
-		ktime_t vsync_time;
-
-		if (dpu_encoder_vsync_time(encoder, &vsync_time) == 0)
-			return vsync_time;
-	}
-
-	return ktime_get();
-}
-
-static void dpu_kms_prepare_commit(struct msm_kms *kms,
-		struct drm_atomic_state *state)
-{
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *crtc_state;
-	struct drm_encoder *encoder;
-	int i;
-
-	if (!kms)
-		return;
-
-	/* Call prepare_commit for all affected encoders */
-	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
-		drm_for_each_encoder_mask(encoder, crtc->dev,
-					  crtc_state->encoder_mask) {
-			dpu_encoder_prepare_commit(encoder);
-		}
-	}
-}
-
 static void dpu_kms_flush_commit(struct msm_kms *kms, unsigned crtc_mask)
 {
 	struct dpu_kms *dpu_kms = to_dpu_kms(kms);
@@ -491,7 +475,7 @@ static void dpu_kms_wait_for_commit_done(struct msm_kms *kms,
 		return;
 	}
 
-	if (!crtc->state->active) {
+	if (!drm_atomic_crtc_effectively_active(crtc->state)) {
 		DPU_DEBUG("[crtc:%d] not active\n", crtc->base.id);
 		return;
 	}
@@ -953,8 +937,6 @@ static const struct msm_kms_funcs kms_funcs = {
 	.irq             = dpu_core_irq,
 	.enable_commit   = dpu_kms_enable_commit,
 	.disable_commit  = dpu_kms_disable_commit,
-	.vsync_time      = dpu_kms_vsync_time,
-	.prepare_commit  = dpu_kms_prepare_commit,
 	.flush_commit    = dpu_kms_flush_commit,
 	.wait_flush      = dpu_kms_wait_flush,
 	.complete_commit = dpu_kms_complete_commit,
@@ -1013,6 +995,7 @@ static int dpu_kms_hw_init(struct msm_kms *kms)
 	struct dpu_kms *dpu_kms;
 	struct drm_device *dev;
 	int i, rc = -EINVAL;
+	u32 core_rev;
 
 	if (!kms) {
 		DPU_ERROR("invalid kms\n");
@@ -1062,17 +1045,14 @@ static int dpu_kms_hw_init(struct msm_kms *kms)
 	if (rc < 0)
 		goto error;
 
-	dpu_kms->core_rev = readl_relaxed(dpu_kms->mmio + 0x0);
+	core_rev = readl_relaxed(dpu_kms->mmio + 0x0);
 
-	pr_info("dpu hardware revision:0x%x\n", dpu_kms->core_rev);
+	pr_info("dpu hardware revision:0x%x\n", core_rev);
 
-	dpu_kms->catalog = dpu_hw_catalog_init(dpu_kms->core_rev);
-	if (IS_ERR_OR_NULL(dpu_kms->catalog)) {
-		rc = PTR_ERR(dpu_kms->catalog);
-		if (!dpu_kms->catalog)
-			rc = -EINVAL;
-		DPU_ERROR("catalog init failed: %d\n", rc);
-		dpu_kms->catalog = NULL;
+	dpu_kms->catalog = of_device_get_match_data(dev->dev);
+	if (!dpu_kms->catalog) {
+		DPU_ERROR("device config not known!\n");
+		rc = -EINVAL;
 		goto power_error;
 	}
 
@@ -1298,19 +1278,19 @@ static const struct dev_pm_ops dpu_pm_ops = {
 };
 
 static const struct of_device_id dpu_dt_match[] = {
-	{ .compatible = "qcom,msm8998-dpu", },
-	{ .compatible = "qcom,qcm2290-dpu", },
-	{ .compatible = "qcom,sdm845-dpu", },
-	{ .compatible = "qcom,sc7180-dpu", },
-	{ .compatible = "qcom,sc7280-dpu", },
-	{ .compatible = "qcom,sc8180x-dpu", },
-	{ .compatible = "qcom,sc8280xp-dpu", },
-	{ .compatible = "qcom,sm6115-dpu", },
-	{ .compatible = "qcom,sm8150-dpu", },
-	{ .compatible = "qcom,sm8250-dpu", },
-	{ .compatible = "qcom,sm8350-dpu", },
-	{ .compatible = "qcom,sm8450-dpu", },
-	{ .compatible = "qcom,sm8550-dpu", },
+	{ .compatible = "qcom,msm8998-dpu", .data = &dpu_msm8998_cfg, },
+	{ .compatible = "qcom,qcm2290-dpu", .data = &dpu_qcm2290_cfg, },
+	{ .compatible = "qcom,sdm845-dpu", .data = &dpu_sdm845_cfg, },
+	{ .compatible = "qcom,sc7180-dpu", .data = &dpu_sc7180_cfg, },
+	{ .compatible = "qcom,sc7280-dpu", .data = &dpu_sc7280_cfg, },
+	{ .compatible = "qcom,sc8180x-dpu", .data = &dpu_sc8180x_cfg, },
+	{ .compatible = "qcom,sc8280xp-dpu", .data = &dpu_sc8280xp_cfg, },
+	{ .compatible = "qcom,sm6115-dpu", .data = &dpu_sm6115_cfg, },
+	{ .compatible = "qcom,sm8150-dpu", .data = &dpu_sm8150_cfg, },
+	{ .compatible = "qcom,sm8250-dpu", .data = &dpu_sm8250_cfg, },
+	{ .compatible = "qcom,sm8350-dpu", .data = &dpu_sm8350_cfg, },
+	{ .compatible = "qcom,sm8450-dpu", .data = &dpu_sm8450_cfg, },
+	{ .compatible = "qcom,sm8550-dpu", .data = &dpu_sm8550_cfg, },
 	{}
 };
 MODULE_DEVICE_TABLE(of, dpu_dt_match);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
index ed80ed6784ee..aca39a4689f4 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
@@ -68,7 +68,6 @@
 struct dpu_kms {
 	struct msm_kms base;
 	struct drm_device *dev;
-	int core_rev;
 	const struct dpu_mdss_cfg *catalog;
 
 	/* io/register spaces: */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
index bfd5be89e8b8..14b5cfe30611 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
@@ -47,13 +47,6 @@
 #define DPU_PLANE_COLOR_FILL_FLAG	BIT(31)
 #define DPU_ZPOS_MAX 255
 
-/* multirect rect index */
-enum {
-	R0,
-	R1,
-	R_MAX
-};
-
 /*
  * Default Preload Values
  */
@@ -69,6 +62,7 @@ static const uint32_t qcom_compressed_supported_formats[] = {
 	DRM_FORMAT_ARGB8888,
 	DRM_FORMAT_XBGR8888,
 	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB2101010,
 	DRM_FORMAT_XRGB2101010,
 	DRM_FORMAT_BGR565,
 
@@ -104,7 +98,6 @@ struct dpu_plane {
 
 	enum dpu_sspp pipe;
 
-	struct dpu_hw_pipe *pipe_hw;
 	uint32_t color_fill;
 	bool is_error;
 	bool is_rt_pipe;
@@ -128,21 +121,19 @@ static struct dpu_kms *_dpu_plane_get_kms(struct drm_plane *plane)
 
 /**
  * _dpu_plane_calc_bw - calculate bandwidth required for a plane
- * @plane: Pointer to drm plane.
- * @fb:   Pointer to framebuffer associated with the given plane
+ * @catalog: Points to dpu catalog structure
+ * @fmt: Pointer to source buffer format
+ * @mode: Pointer to drm display mode
  * @pipe_cfg: Pointer to pipe configuration
  * Result: Updates calculated bandwidth in the plane state.
  * BW Equation: src_w * src_h * bpp * fps * (v_total / v_dest)
  * Prefill BW Equation: line src bytes * line_time
  */
-static void _dpu_plane_calc_bw(struct drm_plane *plane,
-	struct drm_framebuffer *fb,
-	struct dpu_hw_pipe_cfg *pipe_cfg)
+static u64 _dpu_plane_calc_bw(const struct dpu_mdss_cfg *catalog,
+	const struct dpu_format *fmt,
+	const struct drm_display_mode *mode,
+	struct dpu_sw_pipe_cfg *pipe_cfg)
 {
-	struct dpu_plane_state *pstate;
-	struct drm_display_mode *mode;
-	const struct dpu_format *fmt = NULL;
-	struct dpu_kms *dpu_kms = _dpu_plane_get_kms(plane);
 	int src_width, src_height, dst_height, fps;
 	u64 plane_prefill_bw;
 	u64 plane_bw;
@@ -150,11 +141,6 @@ static void _dpu_plane_calc_bw(struct drm_plane *plane,
 	u64 scale_factor;
 	int vbp, vpw, vfp;
 
-	pstate = to_dpu_plane_state(plane->state);
-	mode = &plane->state->crtc->mode;
-
-	fmt = dpu_get_dpu_format_ext(fb->format->format, fb->modifier);
-
 	src_width = drm_rect_width(&pipe_cfg->src_rect);
 	src_height = drm_rect_height(&pipe_cfg->src_rect);
 	dst_height = drm_rect_height(&pipe_cfg->dst_rect);
@@ -162,7 +148,7 @@ static void _dpu_plane_calc_bw(struct drm_plane *plane,
 	vbp = mode->vtotal - mode->vsync_end;
 	vpw = mode->vsync_end - mode->vsync_start;
 	vfp = mode->vsync_start - mode->vdisplay;
-	hw_latency_lines =  dpu_kms->catalog->perf->min_prefill_lines;
+	hw_latency_lines =  catalog->perf->min_prefill_lines;
 	scale_factor = src_height > dst_height ?
 		mult_frac(src_height, 1, dst_height) : 1;
 
@@ -182,61 +168,60 @@ static void _dpu_plane_calc_bw(struct drm_plane *plane,
 		do_div(plane_prefill_bw, hw_latency_lines);
 
 
-	pstate->plane_fetch_bw = max(plane_bw, plane_prefill_bw);
+	return max(plane_bw, plane_prefill_bw);
 }
 
 /**
  * _dpu_plane_calc_clk - calculate clock required for a plane
- * @plane: Pointer to drm plane.
+ * @mode: Pointer to drm display mode
  * @pipe_cfg: Pointer to pipe configuration
  * Result: Updates calculated clock in the plane state.
  * Clock equation: dst_w * v_total * fps * (src_h / dst_h)
  */
-static void _dpu_plane_calc_clk(struct drm_plane *plane, struct dpu_hw_pipe_cfg *pipe_cfg)
+static u64 _dpu_plane_calc_clk(const struct drm_display_mode *mode,
+		struct dpu_sw_pipe_cfg *pipe_cfg)
 {
-	struct dpu_plane_state *pstate;
-	struct drm_display_mode *mode;
 	int dst_width, src_height, dst_height, fps;
-
-	pstate = to_dpu_plane_state(plane->state);
-	mode = &plane->state->crtc->mode;
+	u64 plane_clk;
 
 	src_height = drm_rect_height(&pipe_cfg->src_rect);
 	dst_width = drm_rect_width(&pipe_cfg->dst_rect);
 	dst_height = drm_rect_height(&pipe_cfg->dst_rect);
 	fps = drm_mode_vrefresh(mode);
 
-	pstate->plane_clk =
+	plane_clk =
 		dst_width * mode->vtotal * fps;
 
 	if (src_height > dst_height) {
-		pstate->plane_clk *= src_height;
-		do_div(pstate->plane_clk, dst_height);
+		plane_clk *= src_height;
+		do_div(plane_clk, dst_height);
 	}
+
+	return plane_clk;
 }
 
 /**
  * _dpu_plane_calc_fill_level - calculate fill level of the given source format
  * @plane:		Pointer to drm plane
+ * @pipe:		Pointer to software pipe
  * @fmt:		Pointer to source buffer format
  * @src_width:		width of source buffer
  * Return: fill level corresponding to the source buffer/format or 0 if error
  */
 static int _dpu_plane_calc_fill_level(struct drm_plane *plane,
+		struct dpu_sw_pipe *pipe,
 		const struct dpu_format *fmt, u32 src_width)
 {
 	struct dpu_plane *pdpu;
-	struct dpu_plane_state *pstate;
 	u32 fixed_buff_size;
 	u32 total_fl;
 
-	if (!fmt || !plane->state || !src_width || !fmt->bpp) {
+	if (!fmt || !pipe || !src_width || !fmt->bpp) {
 		DPU_ERROR("invalid arguments\n");
 		return 0;
 	}
 
 	pdpu = to_dpu_plane(plane);
-	pstate = to_dpu_plane_state(plane->state);
 	fixed_buff_size = pdpu->catalog->caps->pixel_ram_size;
 
 	/* FIXME: in multirect case account for the src_width of all the planes */
@@ -252,7 +237,7 @@ static int _dpu_plane_calc_fill_level(struct drm_plane *plane,
 				((src_width + 32) * fmt->bpp);
 		}
 	} else {
-		if (pstate->multirect_mode == DPU_SSPP_MULTIRECT_PARALLEL) {
+		if (pipe->multirect_mode == DPU_SSPP_MULTIRECT_PARALLEL) {
 			total_fl = (fixed_buff_size / 2) * 2 /
 				((src_width + 32) * fmt->bpp);
 		} else {
@@ -262,7 +247,7 @@ static int _dpu_plane_calc_fill_level(struct drm_plane *plane,
 	}
 
 	DPU_DEBUG_PLANE(pdpu, "pnum:%d fmt: %4.4s w:%u fl:%u\n",
-			pdpu->pipe - SSPP_VIG0,
+			pipe->sspp->idx - SSPP_VIG0,
 			(char *)&fmt->base.pixel_format,
 			src_width, total_fl);
 
@@ -272,24 +257,22 @@ static int _dpu_plane_calc_fill_level(struct drm_plane *plane,
 /**
  * _dpu_plane_set_qos_lut - set QoS LUT of the given plane
  * @plane:		Pointer to drm plane
- * @fb:			Pointer to framebuffer associated with the given plane
+ * @pipe:		Pointer to software pipe
+ * @fmt:		Pointer to source buffer format
  * @pipe_cfg:		Pointer to pipe configuration
  */
 static void _dpu_plane_set_qos_lut(struct drm_plane *plane,
-		struct drm_framebuffer *fb, struct dpu_hw_pipe_cfg *pipe_cfg)
+		struct dpu_sw_pipe *pipe,
+		const struct dpu_format *fmt, struct dpu_sw_pipe_cfg *pipe_cfg)
 {
 	struct dpu_plane *pdpu = to_dpu_plane(plane);
-	const struct dpu_format *fmt = NULL;
 	u64 qos_lut;
 	u32 total_fl = 0, lut_usage;
 
 	if (!pdpu->is_rt_pipe) {
 		lut_usage = DPU_QOS_LUT_USAGE_NRT;
 	} else {
-		fmt = dpu_get_dpu_format_ext(
-				fb->format->format,
-				fb->modifier);
-		total_fl = _dpu_plane_calc_fill_level(plane, fmt,
+		total_fl = _dpu_plane_calc_fill_level(plane, pipe, fmt,
 				drm_rect_width(&pipe_cfg->src_rect));
 
 		if (fmt && DPU_FORMAT_IS_LINEAR(fmt))
@@ -301,7 +284,7 @@ static void _dpu_plane_set_qos_lut(struct drm_plane *plane,
 	qos_lut = _dpu_hw_get_qos_lut(
 			&pdpu->catalog->perf->qos_lut_tbl[lut_usage], total_fl);
 
-	trace_dpu_perf_set_qos_luts(pdpu->pipe - SSPP_VIG0,
+	trace_dpu_perf_set_qos_luts(pipe->sspp->idx - SSPP_VIG0,
 			(fmt) ? fmt->base.pixel_format : 0,
 			pdpu->is_rt_pipe, total_fl, qos_lut, lut_usage);
 
@@ -310,19 +293,20 @@ static void _dpu_plane_set_qos_lut(struct drm_plane *plane,
 			fmt ? (char *)&fmt->base.pixel_format : NULL,
 			pdpu->is_rt_pipe, total_fl, qos_lut);
 
-	pdpu->pipe_hw->ops.setup_creq_lut(pdpu->pipe_hw, qos_lut);
+	pipe->sspp->ops.setup_creq_lut(pipe->sspp, qos_lut);
 }
 
 /**
  * _dpu_plane_set_danger_lut - set danger/safe LUT of the given plane
  * @plane:		Pointer to drm plane
- * @fb:			Pointer to framebuffer associated with the given plane
+ * @pipe:		Pointer to software pipe
+ * @fmt:		Pointer to source buffer format
  */
 static void _dpu_plane_set_danger_lut(struct drm_plane *plane,
-		struct drm_framebuffer *fb)
+		struct dpu_sw_pipe *pipe,
+		const struct dpu_format *fmt)
 {
 	struct dpu_plane *pdpu = to_dpu_plane(plane);
-	const struct dpu_format *fmt = NULL;
 	u32 danger_lut, safe_lut;
 
 	if (!pdpu->is_rt_pipe) {
@@ -331,10 +315,6 @@ static void _dpu_plane_set_danger_lut(struct drm_plane *plane,
 		safe_lut = pdpu->catalog->perf->safe_lut_tbl
 				[DPU_QOS_LUT_USAGE_NRT];
 	} else {
-		fmt = dpu_get_dpu_format_ext(
-				fb->format->format,
-				fb->modifier);
-
 		if (fmt && DPU_FORMAT_IS_LINEAR(fmt)) {
 			danger_lut = pdpu->catalog->perf->danger_lut_tbl
 					[DPU_QOS_LUT_USAGE_LINEAR];
@@ -361,17 +341,19 @@ static void _dpu_plane_set_danger_lut(struct drm_plane *plane,
 		danger_lut,
 		safe_lut);
 
-	pdpu->pipe_hw->ops.setup_danger_safe_lut(pdpu->pipe_hw,
+	pipe->sspp->ops.setup_danger_safe_lut(pipe->sspp,
 			danger_lut, safe_lut);
 }
 
 /**
  * _dpu_plane_set_qos_ctrl - set QoS control of the given plane
  * @plane:		Pointer to drm plane
+ * @pipe:		Pointer to software pipe
  * @enable:		true to enable QoS control
  * @flags:		QoS control mode (enum dpu_plane_qos)
  */
 static void _dpu_plane_set_qos_ctrl(struct drm_plane *plane,
+	struct dpu_sw_pipe *pipe,
 	bool enable, u32 flags)
 {
 	struct dpu_plane *pdpu = to_dpu_plane(plane);
@@ -380,9 +362,9 @@ static void _dpu_plane_set_qos_ctrl(struct drm_plane *plane,
 	memset(&pipe_qos_cfg, 0, sizeof(pipe_qos_cfg));
 
 	if (flags & DPU_PLANE_QOS_VBLANK_CTRL) {
-		pipe_qos_cfg.creq_vblank = pdpu->pipe_hw->cap->sblk->creq_vblank;
+		pipe_qos_cfg.creq_vblank = pipe->sspp->cap->sblk->creq_vblank;
 		pipe_qos_cfg.danger_vblank =
-				pdpu->pipe_hw->cap->sblk->danger_vblank;
+				pipe->sspp->cap->sblk->danger_vblank;
 		pipe_qos_cfg.vblank_en = enable;
 	}
 
@@ -408,32 +390,35 @@ static void _dpu_plane_set_qos_ctrl(struct drm_plane *plane,
 		pipe_qos_cfg.danger_vblank,
 		pdpu->is_rt_pipe);
 
-	pdpu->pipe_hw->ops.setup_qos_ctrl(pdpu->pipe_hw,
+	pipe->sspp->ops.setup_qos_ctrl(pipe->sspp,
 			&pipe_qos_cfg);
 }
 
 /**
  * _dpu_plane_set_ot_limit - set OT limit for the given plane
  * @plane:		Pointer to drm plane
- * @crtc:		Pointer to drm crtc
+ * @pipe:		Pointer to software pipe
  * @pipe_cfg:		Pointer to pipe configuration
+ * @frame_rate:		CRTC's frame rate
  */
 static void _dpu_plane_set_ot_limit(struct drm_plane *plane,
-		struct drm_crtc *crtc, struct dpu_hw_pipe_cfg *pipe_cfg)
+		struct dpu_sw_pipe *pipe,
+		struct dpu_sw_pipe_cfg *pipe_cfg,
+		int frame_rate)
 {
 	struct dpu_plane *pdpu = to_dpu_plane(plane);
 	struct dpu_vbif_set_ot_params ot_params;
 	struct dpu_kms *dpu_kms = _dpu_plane_get_kms(plane);
 
 	memset(&ot_params, 0, sizeof(ot_params));
-	ot_params.xin_id = pdpu->pipe_hw->cap->xin_id;
-	ot_params.num = pdpu->pipe_hw->idx - SSPP_NONE;
+	ot_params.xin_id = pipe->sspp->cap->xin_id;
+	ot_params.num = pipe->sspp->idx - SSPP_NONE;
 	ot_params.width = drm_rect_width(&pipe_cfg->src_rect);
 	ot_params.height = drm_rect_height(&pipe_cfg->src_rect);
 	ot_params.is_wfd = !pdpu->is_rt_pipe;
-	ot_params.frame_rate = drm_mode_vrefresh(&crtc->mode);
+	ot_params.frame_rate = frame_rate;
 	ot_params.vbif_idx = VBIF_RT;
-	ot_params.clk_ctrl = pdpu->pipe_hw->cap->clk_ctrl;
+	ot_params.clk_ctrl = pipe->sspp->cap->clk_ctrl;
 	ot_params.rd = true;
 
 	dpu_vbif_set_ot_limit(dpu_kms, &ot_params);
@@ -442,8 +427,10 @@ static void _dpu_plane_set_ot_limit(struct drm_plane *plane,
 /**
  * _dpu_plane_set_qos_remap - set vbif QoS for the given plane
  * @plane:		Pointer to drm plane
+ * @pipe:		Pointer to software pipe
  */
-static void _dpu_plane_set_qos_remap(struct drm_plane *plane)
+static void _dpu_plane_set_qos_remap(struct drm_plane *plane,
+		struct dpu_sw_pipe *pipe)
 {
 	struct dpu_plane *pdpu = to_dpu_plane(plane);
 	struct dpu_vbif_set_qos_params qos_params;
@@ -451,9 +438,9 @@ static void _dpu_plane_set_qos_remap(struct drm_plane *plane)
 
 	memset(&qos_params, 0, sizeof(qos_params));
 	qos_params.vbif_idx = VBIF_RT;
-	qos_params.clk_ctrl = pdpu->pipe_hw->cap->clk_ctrl;
-	qos_params.xin_id = pdpu->pipe_hw->cap->xin_id;
-	qos_params.num = pdpu->pipe_hw->idx - SSPP_VIG0;
+	qos_params.clk_ctrl = pipe->sspp->cap->clk_ctrl;
+	qos_params.xin_id = pipe->sspp->cap->xin_id;
+	qos_params.num = pipe->sspp->idx - SSPP_VIG0;
 	qos_params.is_rt = pdpu->is_rt_pipe;
 
 	DPU_DEBUG_PLANE(pdpu, "pipe:%d vbif:%d xin:%d rt:%d, clk_ctrl:%d\n",
@@ -465,39 +452,15 @@ static void _dpu_plane_set_qos_remap(struct drm_plane *plane)
 	dpu_vbif_set_qos_remap(dpu_kms, &qos_params);
 }
 
-static void _dpu_plane_set_scanout(struct drm_plane *plane,
-		struct dpu_plane_state *pstate,
-		struct dpu_hw_pipe_cfg *pipe_cfg,
-		struct drm_framebuffer *fb)
-{
-	struct dpu_plane *pdpu = to_dpu_plane(plane);
-	struct dpu_kms *kms = _dpu_plane_get_kms(&pdpu->base);
-	struct msm_gem_address_space *aspace = kms->base.aspace;
-	int ret;
-
-	ret = dpu_format_populate_layout(aspace, fb, &pipe_cfg->layout);
-	if (ret == -EAGAIN)
-		DPU_DEBUG_PLANE(pdpu, "not updating same src addrs\n");
-	else if (ret)
-		DPU_ERROR_PLANE(pdpu, "failed to get format layout, %d\n", ret);
-	else if (pdpu->pipe_hw->ops.setup_sourceaddress) {
-		trace_dpu_plane_set_scanout(pdpu->pipe_hw->idx,
-					    &pipe_cfg->layout,
-					    pstate->multirect_index);
-		pdpu->pipe_hw->ops.setup_sourceaddress(pdpu->pipe_hw, pipe_cfg,
-						pstate->multirect_index);
-	}
-}
-
-static void _dpu_plane_setup_scaler3(struct dpu_plane *pdpu,
-		struct dpu_plane_state *pstate,
+static void _dpu_plane_setup_scaler3(struct dpu_hw_sspp *pipe_hw,
 		uint32_t src_w, uint32_t src_h, uint32_t dst_w, uint32_t dst_h,
 		struct dpu_hw_scaler3_cfg *scale_cfg,
 		const struct dpu_format *fmt,
-		uint32_t chroma_subsmpl_h, uint32_t chroma_subsmpl_v)
+		uint32_t chroma_subsmpl_h, uint32_t chroma_subsmpl_v,
+		unsigned int rotation)
 {
 	uint32_t i;
-	bool inline_rotation = pstate->rotation & DRM_MODE_ROTATE_90;
+	bool inline_rotation = rotation & DRM_MODE_ROTATE_90;
 
 	/*
 	 * For inline rotation cases, scaler config is post-rotation,
@@ -536,7 +499,7 @@ static void _dpu_plane_setup_scaler3(struct dpu_plane *pdpu,
 			scale_cfg->src_height[i] /= chroma_subsmpl_v;
 		}
 
-		if (pdpu->pipe_hw->cap->features &
+		if (pipe_hw->cap->features &
 			BIT(DPU_SSPP_SCALER_QSEED4)) {
 			scale_cfg->preload_x[i] = DPU_QSEED4_DEFAULT_PRELOAD_H;
 			scale_cfg->preload_y[i] = DPU_QSEED4_DEFAULT_PRELOAD_V;
@@ -607,36 +570,28 @@ static const struct dpu_csc_cfg dpu_csc10_YUV2RGB_601L = {
 	{ 0x00, 0x3ff, 0x00, 0x3ff, 0x00, 0x3ff,},
 };
 
-static const struct dpu_csc_cfg *_dpu_plane_get_csc(struct dpu_plane *pdpu, const struct dpu_format *fmt)
+static const struct dpu_csc_cfg *_dpu_plane_get_csc(struct dpu_sw_pipe *pipe,
+						    const struct dpu_format *fmt)
 {
 	const struct dpu_csc_cfg *csc_ptr;
 
-	if (!pdpu) {
-		DPU_ERROR("invalid plane\n");
-		return NULL;
-	}
-
 	if (!DPU_FORMAT_IS_YUV(fmt))
 		return NULL;
 
-	if (BIT(DPU_SSPP_CSC_10BIT) & pdpu->pipe_hw->cap->features)
+	if (BIT(DPU_SSPP_CSC_10BIT) & pipe->sspp->cap->features)
 		csc_ptr = &dpu_csc10_YUV2RGB_601L;
 	else
 		csc_ptr = &dpu_csc_YUV2RGB_601L;
 
-	DPU_DEBUG_PLANE(pdpu, "using 0x%X 0x%X 0x%X...\n",
-			csc_ptr->csc_mv[0],
-			csc_ptr->csc_mv[1],
-			csc_ptr->csc_mv[2]);
-
 	return csc_ptr;
 }
 
-static void _dpu_plane_setup_scaler(struct dpu_plane *pdpu,
-		struct dpu_plane_state *pstate,
+static void _dpu_plane_setup_scaler(struct dpu_sw_pipe *pipe,
 		const struct dpu_format *fmt, bool color_fill,
-		struct dpu_hw_pipe_cfg *pipe_cfg)
+		struct dpu_sw_pipe_cfg *pipe_cfg,
+		unsigned int rotation)
 {
+	struct dpu_hw_sspp *pipe_hw = pipe->sspp;
 	const struct drm_format_info *info = drm_format_info(fmt->base.pixel_format);
 	struct dpu_hw_scaler3_cfg scaler3_cfg;
 	struct dpu_hw_pixel_ext pixel_ext;
@@ -650,20 +605,21 @@ static void _dpu_plane_setup_scaler(struct dpu_plane *pdpu,
 
 	/* don't chroma subsample if decimating */
 	/* update scaler. calculate default config for QSEED3 */
-	_dpu_plane_setup_scaler3(pdpu, pstate,
+	_dpu_plane_setup_scaler3(pipe_hw,
 			src_width,
 			src_height,
 			dst_width,
 			dst_height,
 			&scaler3_cfg, fmt,
-			info->hsub, info->vsub);
+			info->hsub, info->vsub,
+			rotation);
 
 	/* configure pixel extension based on scalar config */
 	_dpu_plane_setup_pixel_ext(&scaler3_cfg, &pixel_ext,
 			src_width, src_height, info->hsub, info->vsub);
 
-	if (pdpu->pipe_hw->ops.setup_pe)
-		pdpu->pipe_hw->ops.setup_pe(pdpu->pipe_hw,
+	if (pipe_hw->ops.setup_pe)
+		pipe_hw->ops.setup_pe(pipe_hw,
 				&pixel_ext);
 
 	/**
@@ -671,11 +627,44 @@ static void _dpu_plane_setup_scaler(struct dpu_plane *pdpu,
 	 * bypassed. Still we need to update alpha and bitwidth
 	 * ONLY for RECT0
 	 */
-	if (pdpu->pipe_hw->ops.setup_scaler &&
-			pstate->multirect_index != DPU_SSPP_RECT_1)
-		pdpu->pipe_hw->ops.setup_scaler(pdpu->pipe_hw,
-				pipe_cfg,
-				&scaler3_cfg);
+	if (pipe_hw->ops.setup_scaler &&
+			pipe->multirect_index != DPU_SSPP_RECT_1)
+		pipe_hw->ops.setup_scaler(pipe_hw,
+				&scaler3_cfg,
+				fmt);
+}
+
+static void _dpu_plane_color_fill_pipe(struct dpu_plane_state *pstate,
+				       struct dpu_sw_pipe *pipe,
+				       struct drm_rect *dst_rect,
+				       u32 fill_color,
+				       const struct dpu_format *fmt)
+{
+	struct dpu_sw_pipe_cfg pipe_cfg;
+
+	/* update sspp */
+	if (!pipe->sspp->ops.setup_solidfill)
+		return;
+
+	pipe->sspp->ops.setup_solidfill(pipe, fill_color);
+
+	/* override scaler/decimation if solid fill */
+	pipe_cfg.dst_rect = *dst_rect;
+
+	pipe_cfg.src_rect.x1 = 0;
+	pipe_cfg.src_rect.y1 = 0;
+	pipe_cfg.src_rect.x2 =
+		drm_rect_width(&pipe_cfg.dst_rect);
+	pipe_cfg.src_rect.y2 =
+		drm_rect_height(&pipe_cfg.dst_rect);
+
+	if (pipe->sspp->ops.setup_format)
+		pipe->sspp->ops.setup_format(pipe, fmt, DPU_SSPP_SOLID_FILL);
+
+	if (pipe->sspp->ops.setup_rects)
+		pipe->sspp->ops.setup_rects(pipe, &pipe_cfg);
+
+	_dpu_plane_setup_scaler(pipe, fmt, true, &pipe_cfg, pstate->rotation);
 }
 
 /**
@@ -683,15 +672,14 @@ static void _dpu_plane_setup_scaler(struct dpu_plane *pdpu,
  * @pdpu:   Pointer to DPU plane object
  * @color:  RGB fill color value, [23..16] Blue, [15..8] Green, [7..0] Red
  * @alpha:  8-bit fill alpha value, 255 selects 100% alpha
- * Returns: 0 on success
  */
-static int _dpu_plane_color_fill(struct dpu_plane *pdpu,
+static void _dpu_plane_color_fill(struct dpu_plane *pdpu,
 		uint32_t color, uint32_t alpha)
 {
 	const struct dpu_format *fmt;
 	const struct drm_plane *plane = &pdpu->base;
 	struct dpu_plane_state *pstate = to_dpu_plane_state(plane->state);
-	struct dpu_hw_pipe_cfg pipe_cfg;
+	u32 fill_color = (color & 0xFFFFFF) | ((alpha & 0xFF) << 24);
 
 	DPU_DEBUG_PLANE(pdpu, "\n");
 
@@ -700,156 +688,17 @@ static int _dpu_plane_color_fill(struct dpu_plane *pdpu,
 	 * h/w only supports RGB variants
 	 */
 	fmt = dpu_get_dpu_format(DRM_FORMAT_ABGR8888);
+	/* should not happen ever */
+	if (!fmt)
+		return;
 
 	/* update sspp */
-	if (fmt && pdpu->pipe_hw->ops.setup_solidfill) {
-		pdpu->pipe_hw->ops.setup_solidfill(pdpu->pipe_hw,
-				(color & 0xFFFFFF) | ((alpha & 0xFF) << 24),
-				pstate->multirect_index);
-
-		/* override scaler/decimation if solid fill */
-		pipe_cfg.dst_rect = pstate->base.dst;
-
-		pipe_cfg.src_rect.x1 = 0;
-		pipe_cfg.src_rect.y1 = 0;
-		pipe_cfg.src_rect.x2 =
-			drm_rect_width(&pipe_cfg.dst_rect);
-		pipe_cfg.src_rect.y2 =
-			drm_rect_height(&pipe_cfg.dst_rect);
-
-		if (pdpu->pipe_hw->ops.setup_format)
-			pdpu->pipe_hw->ops.setup_format(pdpu->pipe_hw,
-					fmt, DPU_SSPP_SOLID_FILL,
-					pstate->multirect_index);
-
-		if (pdpu->pipe_hw->ops.setup_rects)
-			pdpu->pipe_hw->ops.setup_rects(pdpu->pipe_hw,
-					&pipe_cfg,
-					pstate->multirect_index);
-
-		_dpu_plane_setup_scaler(pdpu, pstate, fmt, true, &pipe_cfg);
-	}
-
-	return 0;
-}
-
-void dpu_plane_clear_multirect(const struct drm_plane_state *drm_state)
-{
-	struct dpu_plane_state *pstate = to_dpu_plane_state(drm_state);
-
-	pstate->multirect_index = DPU_SSPP_RECT_SOLO;
-	pstate->multirect_mode = DPU_SSPP_MULTIRECT_NONE;
-}
-
-int dpu_plane_validate_multirect_v2(struct dpu_multirect_plane_states *plane)
-{
-	struct dpu_plane_state *pstate[R_MAX];
-	const struct drm_plane_state *drm_state[R_MAX];
-	struct drm_rect src[R_MAX], dst[R_MAX];
-	struct dpu_plane *dpu_plane[R_MAX];
-	const struct dpu_format *fmt[R_MAX];
-	int i, buffer_lines;
-	unsigned int max_tile_height = 1;
-	bool parallel_fetch_qualified = true;
-	bool has_tiled_rect = false;
-
-	for (i = 0; i < R_MAX; i++) {
-		const struct msm_format *msm_fmt;
-
-		drm_state[i] = i ? plane->r1 : plane->r0;
-		msm_fmt = msm_framebuffer_format(drm_state[i]->fb);
-		fmt[i] = to_dpu_format(msm_fmt);
-
-		if (DPU_FORMAT_IS_UBWC(fmt[i])) {
-			has_tiled_rect = true;
-			if (fmt[i]->tile_height > max_tile_height)
-				max_tile_height = fmt[i]->tile_height;
-		}
-	}
-
-	for (i = 0; i < R_MAX; i++) {
-		int width_threshold;
-
-		pstate[i] = to_dpu_plane_state(drm_state[i]);
-		dpu_plane[i] = to_dpu_plane(drm_state[i]->plane);
-
-		if (pstate[i] == NULL) {
-			DPU_ERROR("DPU plane state of plane id %d is NULL\n",
-				drm_state[i]->plane->base.id);
-			return -EINVAL;
-		}
-
-		src[i].x1 = drm_state[i]->src_x >> 16;
-		src[i].y1 = drm_state[i]->src_y >> 16;
-		src[i].x2 = src[i].x1 + (drm_state[i]->src_w >> 16);
-		src[i].y2 = src[i].y1 + (drm_state[i]->src_h >> 16);
-
-		dst[i] = drm_plane_state_dest(drm_state[i]);
-
-		if (drm_rect_calc_hscale(&src[i], &dst[i], 1, 1) != 1 ||
-		    drm_rect_calc_vscale(&src[i], &dst[i], 1, 1) != 1) {
-			DPU_ERROR_PLANE(dpu_plane[i],
-				"scaling is not supported in multirect mode\n");
-			return -EINVAL;
-		}
-
-		if (DPU_FORMAT_IS_YUV(fmt[i])) {
-			DPU_ERROR_PLANE(dpu_plane[i],
-				"Unsupported format for multirect mode\n");
-			return -EINVAL;
-		}
-
-		/**
-		 * SSPP PD_MEM is split half - one for each RECT.
-		 * Tiled formats need 5 lines of buffering while fetching
-		 * whereas linear formats need only 2 lines.
-		 * So we cannot support more than half of the supported SSPP
-		 * width for tiled formats.
-		 */
-		width_threshold = dpu_plane[i]->catalog->caps->max_linewidth;
-		if (has_tiled_rect)
-			width_threshold /= 2;
-
-		if (parallel_fetch_qualified &&
-		    drm_rect_width(&src[i]) > width_threshold)
-			parallel_fetch_qualified = false;
-
-	}
-
-	/* Validate RECT's and set the mode */
-
-	/* Prefer PARALLEL FETCH Mode over TIME_MX Mode */
-	if (parallel_fetch_qualified) {
-		pstate[R0]->multirect_mode = DPU_SSPP_MULTIRECT_PARALLEL;
-		pstate[R1]->multirect_mode = DPU_SSPP_MULTIRECT_PARALLEL;
-
-		goto done;
-	}
-
-	/* TIME_MX Mode */
-	buffer_lines = 2 * max_tile_height;
+	_dpu_plane_color_fill_pipe(pstate, &pstate->pipe, &pstate->pipe_cfg.dst_rect,
+				   fill_color, fmt);
 
-	if (dst[R1].y1 >= dst[R0].y2 + buffer_lines ||
-	    dst[R0].y1 >= dst[R1].y2 + buffer_lines) {
-		pstate[R0]->multirect_mode = DPU_SSPP_MULTIRECT_TIME_MX;
-		pstate[R1]->multirect_mode = DPU_SSPP_MULTIRECT_TIME_MX;
-	} else {
-		DPU_ERROR(
-			"No multirect mode possible for the planes (%d - %d)\n",
-			drm_state[R0]->plane->base.id,
-			drm_state[R1]->plane->base.id);
-		return -EINVAL;
-	}
-
-done:
-	pstate[R0]->multirect_index = DPU_SSPP_RECT_0;
-	pstate[R1]->multirect_index = DPU_SSPP_RECT_1;
-
-	DPU_DEBUG_PLANE(dpu_plane[R0], "R0: %d - %d\n",
-		pstate[R0]->multirect_mode, pstate[R0]->multirect_index);
-	DPU_DEBUG_PLANE(dpu_plane[R1], "R1: %d - %d\n",
-		pstate[R1]->multirect_mode, pstate[R1]->multirect_index);
-	return 0;
+	if (pstate->r_pipe.sspp)
+		_dpu_plane_color_fill_pipe(pstate, &pstate->r_pipe, &pstate->r_pipe_cfg.dst_rect,
+					   fill_color, fmt);
 }
 
 static int dpu_plane_prepare_fb(struct drm_plane *plane,
@@ -914,25 +763,6 @@ static void dpu_plane_cleanup_fb(struct drm_plane *plane,
 				old_pstate->needs_dirtyfb);
 }
 
-static bool dpu_plane_validate_src(struct drm_rect *src,
-				   struct drm_rect *fb_rect,
-				   uint32_t min_src_size)
-{
-	/* Ensure fb size is supported */
-	if (drm_rect_width(fb_rect) > MAX_IMG_WIDTH ||
-	    drm_rect_height(fb_rect) > MAX_IMG_HEIGHT)
-		return false;
-
-	/* Ensure src rect is above the minimum size */
-	if (drm_rect_width(src) < min_src_size ||
-	    drm_rect_height(src) < min_src_size)
-		return false;
-
-	/* Ensure src is fully encapsulated in fb */
-	return drm_rect_intersect(fb_rect, src) &&
-		drm_rect_equals(fb_rect, src);
-}
-
 static int dpu_plane_check_inline_rotation(struct dpu_plane *pdpu,
 						const struct dpu_sspp_sub_blks *sblk,
 						struct drm_rect src, const struct dpu_format *fmt)
@@ -961,6 +791,53 @@ static int dpu_plane_check_inline_rotation(struct dpu_plane *pdpu,
 	return 0;
 }
 
+static int dpu_plane_atomic_check_pipe(struct dpu_plane *pdpu,
+		struct dpu_sw_pipe *pipe,
+		struct dpu_sw_pipe_cfg *pipe_cfg,
+		const struct dpu_format *fmt)
+{
+	uint32_t min_src_size;
+
+	min_src_size = DPU_FORMAT_IS_YUV(fmt) ? 2 : 1;
+
+	if (DPU_FORMAT_IS_YUV(fmt) &&
+	    (!(pipe->sspp->cap->features & DPU_SSPP_SCALER) ||
+	     !(pipe->sspp->cap->features & DPU_SSPP_CSC_ANY))) {
+		DPU_DEBUG_PLANE(pdpu,
+				"plane doesn't have scaler/csc for yuv\n");
+		return -EINVAL;
+	}
+
+	/* check src bounds */
+	if (drm_rect_width(&pipe_cfg->src_rect) < min_src_size ||
+	    drm_rect_height(&pipe_cfg->src_rect) < min_src_size) {
+		DPU_DEBUG_PLANE(pdpu, "invalid source " DRM_RECT_FMT "\n",
+				DRM_RECT_ARG(&pipe_cfg->src_rect));
+		return -E2BIG;
+	}
+
+	/* valid yuv image */
+	if (DPU_FORMAT_IS_YUV(fmt) &&
+	    (pipe_cfg->src_rect.x1 & 0x1 ||
+	     pipe_cfg->src_rect.y1 & 0x1 ||
+	     drm_rect_width(&pipe_cfg->src_rect) & 0x1 ||
+	     drm_rect_height(&pipe_cfg->src_rect) & 0x1)) {
+		DPU_DEBUG_PLANE(pdpu, "invalid yuv source " DRM_RECT_FMT "\n",
+				DRM_RECT_ARG(&pipe_cfg->src_rect));
+		return -EINVAL;
+	}
+
+	/* min dst support */
+	if (drm_rect_width(&pipe_cfg->dst_rect) < 0x1 ||
+	    drm_rect_height(&pipe_cfg->dst_rect) < 0x1) {
+		DPU_DEBUG_PLANE(pdpu, "invalid dest rect " DRM_RECT_FMT "\n",
+				DRM_RECT_ARG(&pipe_cfg->dst_rect));
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int dpu_plane_atomic_check(struct drm_plane *plane,
 				  struct drm_atomic_state *state)
 {
@@ -969,14 +846,18 @@ static int dpu_plane_atomic_check(struct drm_plane *plane,
 	int ret = 0, min_scale;
 	struct dpu_plane *pdpu = to_dpu_plane(plane);
 	struct dpu_plane_state *pstate = to_dpu_plane_state(new_plane_state);
+	struct dpu_sw_pipe *pipe = &pstate->pipe;
+	struct dpu_sw_pipe *r_pipe = &pstate->r_pipe;
 	const struct drm_crtc_state *crtc_state = NULL;
 	const struct dpu_format *fmt;
-	struct drm_rect src, dst, fb_rect = { 0 };
-	uint32_t min_src_size, max_linewidth;
+	struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg;
+	struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->r_pipe_cfg;
+	struct drm_rect fb_rect = { 0 };
+	uint32_t max_linewidth;
 	unsigned int rotation;
 	uint32_t supported_rotations;
-	const struct dpu_sspp_cfg *pipe_hw_caps = pdpu->pipe_hw->cap;
-	const struct dpu_sspp_sub_blks *sblk = pdpu->pipe_hw->cap->sblk;
+	const struct dpu_sspp_cfg *pipe_hw_caps = pstate->pipe.sspp->cap;
+	const struct dpu_sspp_sub_blks *sblk = pstate->pipe.sspp->cap->sblk;
 
 	if (new_plane_state->crtc)
 		crtc_state = drm_atomic_get_new_crtc_state(state,
@@ -994,55 +875,99 @@ static int dpu_plane_atomic_check(struct drm_plane *plane,
 	if (!new_plane_state->visible)
 		return 0;
 
-	src.x1 = new_plane_state->src_x >> 16;
-	src.y1 = new_plane_state->src_y >> 16;
-	src.x2 = src.x1 + (new_plane_state->src_w >> 16);
-	src.y2 = src.y1 + (new_plane_state->src_h >> 16);
+	pipe->multirect_index = DPU_SSPP_RECT_SOLO;
+	pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE;
+	r_pipe->multirect_index = DPU_SSPP_RECT_SOLO;
+	r_pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE;
+	r_pipe->sspp = NULL;
+
+	pstate->stage = DPU_STAGE_0 + pstate->base.normalized_zpos;
+	if (pstate->stage >= pdpu->catalog->caps->max_mixer_blendstages) {
+		DPU_ERROR("> %d plane stages assigned\n",
+			  pdpu->catalog->caps->max_mixer_blendstages - DPU_STAGE_0);
+		return -EINVAL;
+	}
+
+	pipe_cfg->src_rect = new_plane_state->src;
+
+	/* state->src is 16.16, src_rect is not */
+	pipe_cfg->src_rect.x1 >>= 16;
+	pipe_cfg->src_rect.x2 >>= 16;
+	pipe_cfg->src_rect.y1 >>= 16;
+	pipe_cfg->src_rect.y2 >>= 16;
 
-	dst = drm_plane_state_dest(new_plane_state);
+	pipe_cfg->dst_rect = new_plane_state->dst;
 
 	fb_rect.x2 = new_plane_state->fb->width;
 	fb_rect.y2 = new_plane_state->fb->height;
 
-	max_linewidth = pdpu->catalog->caps->max_linewidth;
+	/* Ensure fb size is supported */
+	if (drm_rect_width(&fb_rect) > MAX_IMG_WIDTH ||
+	    drm_rect_height(&fb_rect) > MAX_IMG_HEIGHT) {
+		DPU_DEBUG_PLANE(pdpu, "invalid framebuffer " DRM_RECT_FMT "\n",
+				DRM_RECT_ARG(&fb_rect));
+		return -E2BIG;
+	}
 
 	fmt = to_dpu_format(msm_framebuffer_format(new_plane_state->fb));
 
-	min_src_size = DPU_FORMAT_IS_YUV(fmt) ? 2 : 1;
+	max_linewidth = pdpu->catalog->caps->max_linewidth;
 
-	if (DPU_FORMAT_IS_YUV(fmt) &&
-		(!(pipe_hw_caps->features & DPU_SSPP_SCALER) ||
-		 !(pipe_hw_caps->features & DPU_SSPP_CSC_ANY))) {
-		DPU_DEBUG_PLANE(pdpu,
-				"plane doesn't have scaler/csc for yuv\n");
-		return -EINVAL;
+	if (drm_rect_width(&pipe_cfg->src_rect) > max_linewidth) {
+		/*
+		 * In parallel multirect case only the half of the usual width
+		 * is supported for tiled formats. If we are here, we know that
+		 * full width is more than max_linewidth, thus each rect is
+		 * wider than allowed.
+		 */
+		if (DPU_FORMAT_IS_UBWC(fmt)) {
+			DPU_DEBUG_PLANE(pdpu, "invalid src " DRM_RECT_FMT " line:%u, tiled format\n",
+					DRM_RECT_ARG(&pipe_cfg->src_rect), max_linewidth);
+			return -E2BIG;
+		}
 
-	/* check src bounds */
-	} else if (!dpu_plane_validate_src(&src, &fb_rect, min_src_size)) {
-		DPU_DEBUG_PLANE(pdpu, "invalid source " DRM_RECT_FMT "\n",
-				DRM_RECT_ARG(&src));
-		return -E2BIG;
+		if (drm_rect_width(&pipe_cfg->src_rect) > 2 * max_linewidth) {
+			DPU_DEBUG_PLANE(pdpu, "invalid src " DRM_RECT_FMT " line:%u\n",
+					DRM_RECT_ARG(&pipe_cfg->src_rect), max_linewidth);
+			return -E2BIG;
+		}
 
-	/* valid yuv image */
-	} else if (DPU_FORMAT_IS_YUV(fmt) &&
-		   (src.x1 & 0x1 || src.y1 & 0x1 ||
-		    drm_rect_width(&src) & 0x1 ||
-		    drm_rect_height(&src) & 0x1)) {
-		DPU_DEBUG_PLANE(pdpu, "invalid yuv source " DRM_RECT_FMT "\n",
-				DRM_RECT_ARG(&src));
-		return -EINVAL;
+		if (drm_rect_width(&pipe_cfg->src_rect) != drm_rect_width(&pipe_cfg->dst_rect) ||
+		    drm_rect_height(&pipe_cfg->src_rect) != drm_rect_height(&pipe_cfg->dst_rect) ||
+		    (!test_bit(DPU_SSPP_SMART_DMA_V1, &pipe->sspp->cap->features) &&
+		     !test_bit(DPU_SSPP_SMART_DMA_V2, &pipe->sspp->cap->features)) ||
+		    DPU_FORMAT_IS_YUV(fmt)) {
+			DPU_DEBUG_PLANE(pdpu, "invalid src " DRM_RECT_FMT " line:%u, can't use split source\n",
+					DRM_RECT_ARG(&pipe_cfg->src_rect), max_linewidth);
+			return -E2BIG;
+		}
 
-	/* min dst support */
-	} else if (drm_rect_width(&dst) < 0x1 || drm_rect_height(&dst) < 0x1) {
-		DPU_DEBUG_PLANE(pdpu, "invalid dest rect " DRM_RECT_FMT "\n",
-				DRM_RECT_ARG(&dst));
-		return -EINVAL;
+		/*
+		 * Use multirect for wide plane. We do not support dynamic
+		 * assignment of SSPPs, so we know the configuration.
+		 */
+		pipe->multirect_index = DPU_SSPP_RECT_0;
+		pipe->multirect_mode = DPU_SSPP_MULTIRECT_PARALLEL;
+
+		r_pipe->sspp = pipe->sspp;
+		r_pipe->multirect_index = DPU_SSPP_RECT_1;
+		r_pipe->multirect_mode = DPU_SSPP_MULTIRECT_PARALLEL;
+
+		*r_pipe_cfg = *pipe_cfg;
+		pipe_cfg->src_rect.x2 = (pipe_cfg->src_rect.x1 + pipe_cfg->src_rect.x2) >> 1;
+		pipe_cfg->dst_rect.x2 = (pipe_cfg->dst_rect.x1 + pipe_cfg->dst_rect.x2) >> 1;
+		r_pipe_cfg->src_rect.x1 = pipe_cfg->src_rect.x2;
+		r_pipe_cfg->dst_rect.x1 = pipe_cfg->dst_rect.x2;
+	}
 
-	/* check decimated source width */
-	} else if (drm_rect_width(&src) > max_linewidth) {
-		DPU_DEBUG_PLANE(pdpu, "invalid src " DRM_RECT_FMT " line:%u\n",
-				DRM_RECT_ARG(&src), max_linewidth);
-		return -E2BIG;
+	ret = dpu_plane_atomic_check_pipe(pdpu, pipe, pipe_cfg, fmt);
+	if (ret)
+		return ret;
+
+	if (r_pipe->sspp) {
+		ret = dpu_plane_atomic_check_pipe(pdpu, r_pipe, r_pipe_cfg, fmt);
+		if (ret)
+			return ret;
 	}
 
 	supported_rotations = DRM_MODE_REFLECT_MASK | DRM_MODE_ROTATE_0;
@@ -1055,7 +980,7 @@ static int dpu_plane_atomic_check(struct drm_plane *plane,
 
 	if ((pipe_hw_caps->features & BIT(DPU_SSPP_INLINE_ROTATION)) &&
 		(rotation & DRM_MODE_ROTATE_90)) {
-		ret = dpu_plane_check_inline_rotation(pdpu, sblk, src, fmt);
+		ret = dpu_plane_check_inline_rotation(pdpu, sblk, pipe_cfg->src_rect, fmt);
 		if (ret)
 			return ret;
 	}
@@ -1066,6 +991,28 @@ static int dpu_plane_atomic_check(struct drm_plane *plane,
 	return 0;
 }
 
+static void dpu_plane_flush_csc(struct dpu_plane *pdpu, struct dpu_sw_pipe *pipe)
+{
+	const struct dpu_format *format =
+		to_dpu_format(msm_framebuffer_format(pdpu->base.state->fb));
+	const struct dpu_csc_cfg *csc_ptr;
+
+	if (!pipe->sspp || !pipe->sspp->ops.setup_csc)
+		return;
+
+	csc_ptr = _dpu_plane_get_csc(pipe, format);
+	if (!csc_ptr)
+		return;
+
+	DPU_DEBUG_PLANE(pdpu, "using 0x%X 0x%X 0x%X...\n",
+			csc_ptr->csc_mv[0],
+			csc_ptr->csc_mv[1],
+			csc_ptr->csc_mv[2]);
+
+	pipe->sspp->ops.setup_csc(pipe->sspp, csc_ptr);
+
+}
+
 void dpu_plane_flush(struct drm_plane *plane)
 {
 	struct dpu_plane *pdpu;
@@ -1089,12 +1036,9 @@ void dpu_plane_flush(struct drm_plane *plane)
 	else if (pdpu->color_fill & DPU_PLANE_COLOR_FILL_FLAG)
 		/* force 100% alpha */
 		_dpu_plane_color_fill(pdpu, pdpu->color_fill, 0xFF);
-	else if (pdpu->pipe_hw && pdpu->pipe_hw->ops.setup_csc) {
-		const struct dpu_format *fmt = to_dpu_format(msm_framebuffer_format(plane->state->fb));
-		const struct dpu_csc_cfg *csc_ptr = _dpu_plane_get_csc(pdpu, fmt);
-
-		if (csc_ptr)
-			pdpu->pipe_hw->ops.setup_csc(pdpu->pipe_hw, csc_ptr);
+	else {
+		dpu_plane_flush_csc(pdpu, &pstate->pipe);
+		dpu_plane_flush_csc(pdpu, &pstate->r_pipe);
 	}
 
 	/* flag h/w flush complete */
@@ -1118,45 +1062,24 @@ void dpu_plane_set_error(struct drm_plane *plane, bool error)
 	pdpu->is_error = error;
 }
 
-static void dpu_plane_sspp_atomic_update(struct drm_plane *plane)
+static void dpu_plane_sspp_update_pipe(struct drm_plane *plane,
+				       struct dpu_sw_pipe *pipe,
+				       struct dpu_sw_pipe_cfg *pipe_cfg,
+				       const struct dpu_format *fmt,
+				       int frame_rate,
+				       struct dpu_hw_fmt_layout *layout)
 {
 	uint32_t src_flags;
 	struct dpu_plane *pdpu = to_dpu_plane(plane);
 	struct drm_plane_state *state = plane->state;
 	struct dpu_plane_state *pstate = to_dpu_plane_state(state);
-	struct drm_crtc *crtc = state->crtc;
-	struct drm_framebuffer *fb = state->fb;
-	bool is_rt_pipe;
-	const struct dpu_format *fmt =
-		to_dpu_format(msm_framebuffer_format(fb));
-	struct dpu_hw_pipe_cfg pipe_cfg;
-
-	memset(&pipe_cfg, 0, sizeof(struct dpu_hw_pipe_cfg));
 
-	_dpu_plane_set_scanout(plane, pstate, &pipe_cfg, fb);
-
-	pstate->pending = true;
-
-	is_rt_pipe = (dpu_crtc_get_client_type(crtc) != NRT_CLIENT);
-	pstate->needs_qos_remap |= (is_rt_pipe != pdpu->is_rt_pipe);
-	pdpu->is_rt_pipe = is_rt_pipe;
-
-	_dpu_plane_set_qos_ctrl(plane, false, DPU_PLANE_QOS_PANIC_CTRL);
-
-	DPU_DEBUG_PLANE(pdpu, "FB[%u] " DRM_RECT_FP_FMT "->crtc%u " DRM_RECT_FMT
-			", %4.4s ubwc %d\n", fb->base.id, DRM_RECT_FP_ARG(&state->src),
-			crtc->base.id, DRM_RECT_ARG(&state->dst),
-			(char *)&fmt->base.pixel_format, DPU_FORMAT_IS_UBWC(fmt));
-
-	pipe_cfg.src_rect = state->src;
-
-	/* state->src is 16.16, src_rect is not */
-	pipe_cfg.src_rect.x1 >>= 16;
-	pipe_cfg.src_rect.x2 >>= 16;
-	pipe_cfg.src_rect.y1 >>= 16;
-	pipe_cfg.src_rect.y2 >>= 16;
+	if (layout && pipe->sspp->ops.setup_sourceaddress) {
+		trace_dpu_plane_set_scanout(pipe, layout);
+		pipe->sspp->ops.setup_sourceaddress(pipe, layout);
+	}
 
-	pipe_cfg.dst_rect = state->dst;
+	_dpu_plane_set_qos_ctrl(plane, pipe, false, DPU_PLANE_QOS_PANIC_CTRL);
 
 	/* override for color fill */
 	if (pdpu->color_fill & DPU_PLANE_COLOR_FILL_FLAG) {
@@ -1164,21 +1087,18 @@ static void dpu_plane_sspp_atomic_update(struct drm_plane *plane)
 		return;
 	}
 
-	if (pdpu->pipe_hw->ops.setup_rects) {
-		pdpu->pipe_hw->ops.setup_rects(pdpu->pipe_hw,
-				&pipe_cfg,
-				pstate->multirect_index);
+	if (pipe->sspp->ops.setup_rects) {
+		pipe->sspp->ops.setup_rects(pipe,
+				pipe_cfg);
 	}
 
-	_dpu_plane_setup_scaler(pdpu, pstate, fmt, false, &pipe_cfg);
+	_dpu_plane_setup_scaler(pipe, fmt, false, pipe_cfg, pstate->rotation);
 
-	if (pdpu->pipe_hw->ops.setup_multirect)
-		pdpu->pipe_hw->ops.setup_multirect(
-				pdpu->pipe_hw,
-				pstate->multirect_index,
-				pstate->multirect_mode);
+	if (pipe->sspp->ops.setup_multirect)
+		pipe->sspp->ops.setup_multirect(
+				pipe);
 
-	if (pdpu->pipe_hw->ops.setup_format) {
+	if (pipe->sspp->ops.setup_format) {
 		unsigned int rotation = pstate->rotation;
 
 		src_flags = 0x0;
@@ -1193,10 +1113,9 @@ static void dpu_plane_sspp_atomic_update(struct drm_plane *plane)
 			src_flags |= DPU_SSPP_ROT_90;
 
 		/* update format */
-		pdpu->pipe_hw->ops.setup_format(pdpu->pipe_hw, fmt, src_flags,
-				pstate->multirect_index);
+		pipe->sspp->ops.setup_format(pipe, fmt, src_flags);
 
-		if (pdpu->pipe_hw->ops.setup_cdp) {
+		if (pipe->sspp->ops.setup_cdp) {
 			struct dpu_hw_cdp_cfg cdp_cfg;
 
 			memset(&cdp_cfg, 0, sizeof(struct dpu_hw_cdp_cfg));
@@ -1210,35 +1129,100 @@ static void dpu_plane_sspp_atomic_update(struct drm_plane *plane)
 					DPU_FORMAT_IS_TILE(fmt);
 			cdp_cfg.preload_ahead = DPU_SSPP_CDP_PRELOAD_AHEAD_64;
 
-			pdpu->pipe_hw->ops.setup_cdp(pdpu->pipe_hw, &cdp_cfg, pstate->multirect_index);
+			pipe->sspp->ops.setup_cdp(pipe, &cdp_cfg);
 		}
 	}
 
-	_dpu_plane_set_qos_lut(plane, fb, &pipe_cfg);
-	_dpu_plane_set_danger_lut(plane, fb);
+	_dpu_plane_set_qos_lut(plane, pipe, fmt, pipe_cfg);
+	_dpu_plane_set_danger_lut(plane, pipe, fmt);
 
 	if (plane->type != DRM_PLANE_TYPE_CURSOR) {
-		_dpu_plane_set_qos_ctrl(plane, true, DPU_PLANE_QOS_PANIC_CTRL);
-		_dpu_plane_set_ot_limit(plane, crtc, &pipe_cfg);
+		_dpu_plane_set_qos_ctrl(plane, pipe, true, DPU_PLANE_QOS_PANIC_CTRL);
+		_dpu_plane_set_ot_limit(plane, pipe, pipe_cfg, frame_rate);
 	}
 
-	if (pstate->needs_qos_remap) {
-		pstate->needs_qos_remap = false;
-		_dpu_plane_set_qos_remap(plane);
+	if (pstate->needs_qos_remap)
+		_dpu_plane_set_qos_remap(plane, pipe);
+}
+
+static void dpu_plane_sspp_atomic_update(struct drm_plane *plane)
+{
+	struct dpu_plane *pdpu = to_dpu_plane(plane);
+	struct drm_plane_state *state = plane->state;
+	struct dpu_plane_state *pstate = to_dpu_plane_state(state);
+	struct dpu_sw_pipe *pipe = &pstate->pipe;
+	struct dpu_sw_pipe *r_pipe = &pstate->r_pipe;
+	struct drm_crtc *crtc = state->crtc;
+	struct drm_framebuffer *fb = state->fb;
+	bool is_rt_pipe;
+	const struct dpu_format *fmt =
+		to_dpu_format(msm_framebuffer_format(fb));
+	struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg;
+	struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->r_pipe_cfg;
+	struct dpu_kms *kms = _dpu_plane_get_kms(&pdpu->base);
+	struct msm_gem_address_space *aspace = kms->base.aspace;
+	struct dpu_hw_fmt_layout layout;
+	bool layout_valid = false;
+	int ret;
+
+	ret = dpu_format_populate_layout(aspace, fb, &layout);
+	if (ret)
+		DPU_ERROR_PLANE(pdpu, "failed to get format layout, %d\n", ret);
+	else
+		layout_valid = true;
+
+	pstate->pending = true;
+
+	is_rt_pipe = (dpu_crtc_get_client_type(crtc) != NRT_CLIENT);
+	pstate->needs_qos_remap |= (is_rt_pipe != pdpu->is_rt_pipe);
+	pdpu->is_rt_pipe = is_rt_pipe;
+
+	DPU_DEBUG_PLANE(pdpu, "FB[%u] " DRM_RECT_FP_FMT "->crtc%u " DRM_RECT_FMT
+			", %4.4s ubwc %d\n", fb->base.id, DRM_RECT_FP_ARG(&state->src),
+			crtc->base.id, DRM_RECT_ARG(&state->dst),
+			(char *)&fmt->base.pixel_format, DPU_FORMAT_IS_UBWC(fmt));
+
+	dpu_plane_sspp_update_pipe(plane, pipe, pipe_cfg, fmt,
+				   drm_mode_vrefresh(&crtc->mode),
+				   layout_valid ? &layout : NULL);
+
+	if (r_pipe->sspp) {
+		dpu_plane_sspp_update_pipe(plane, r_pipe, r_pipe_cfg, fmt,
+					   drm_mode_vrefresh(&crtc->mode),
+					   layout_valid ? &layout : NULL);
 	}
 
-	_dpu_plane_calc_bw(plane, fb, &pipe_cfg);
+	if (pstate->needs_qos_remap)
+		pstate->needs_qos_remap = false;
+
+	pstate->plane_fetch_bw = _dpu_plane_calc_bw(pdpu->catalog, fmt,
+						    &crtc->mode, pipe_cfg);
+
+	pstate->plane_clk = _dpu_plane_calc_clk(&crtc->mode, pipe_cfg);
 
-	_dpu_plane_calc_clk(plane, &pipe_cfg);
+	if (r_pipe->sspp) {
+		pstate->plane_fetch_bw += _dpu_plane_calc_bw(pdpu->catalog, fmt, &crtc->mode, r_pipe_cfg);
+
+		pstate->plane_clk = max(pstate->plane_clk, _dpu_plane_calc_clk(&crtc->mode, r_pipe_cfg));
+	}
 }
 
 static void _dpu_plane_atomic_disable(struct drm_plane *plane)
 {
 	struct drm_plane_state *state = plane->state;
 	struct dpu_plane_state *pstate = to_dpu_plane_state(state);
+	struct dpu_sw_pipe *r_pipe = &pstate->r_pipe;
 
 	trace_dpu_plane_disable(DRMID(plane), false,
-				pstate->multirect_mode);
+				pstate->pipe.multirect_mode);
+
+	if (r_pipe->sspp) {
+		r_pipe->multirect_index = DPU_SSPP_RECT_SOLO;
+		r_pipe->multirect_mode = DPU_SSPP_MULTIRECT_NONE;
+
+		if (r_pipe->sspp->ops.setup_multirect)
+			r_pipe->sspp->ops.setup_multirect(r_pipe);
+	}
 
 	pstate->pending = true;
 }
@@ -1264,19 +1248,22 @@ static void dpu_plane_atomic_update(struct drm_plane *plane,
 static void dpu_plane_destroy(struct drm_plane *plane)
 {
 	struct dpu_plane *pdpu = plane ? to_dpu_plane(plane) : NULL;
+	struct dpu_plane_state *pstate;
 
 	DPU_DEBUG_PLANE(pdpu, "\n");
 
 	if (pdpu) {
-		_dpu_plane_set_qos_ctrl(plane, false, DPU_PLANE_QOS_PANIC_CTRL);
+		pstate = to_dpu_plane_state(plane->state);
+		_dpu_plane_set_qos_ctrl(plane, &pstate->pipe, false, DPU_PLANE_QOS_PANIC_CTRL);
+
+		if (pstate->r_pipe.sspp)
+			_dpu_plane_set_qos_ctrl(plane, &pstate->r_pipe, false, DPU_PLANE_QOS_PANIC_CTRL);
 
 		mutex_destroy(&pdpu->lock);
 
 		/* this will destroy the states as well */
 		drm_plane_cleanup(plane);
 
-		dpu_hw_sspp_destroy(pdpu->pipe_hw);
-
 		kfree(pdpu);
 	}
 }
@@ -1352,18 +1339,36 @@ static void dpu_plane_atomic_print_state(struct drm_printer *p,
 		const struct drm_plane_state *state)
 {
 	const struct dpu_plane_state *pstate = to_dpu_plane_state(state);
-	const struct dpu_plane *pdpu = to_dpu_plane(state->plane);
+	const struct dpu_sw_pipe *pipe = &pstate->pipe;
+	const struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg;
+	const struct dpu_sw_pipe *r_pipe = &pstate->r_pipe;
+	const struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->r_pipe_cfg;
 
 	drm_printf(p, "\tstage=%d\n", pstate->stage);
-	drm_printf(p, "\tsspp=%s\n", pdpu->pipe_hw->cap->name);
-	drm_printf(p, "\tmultirect_mode=%s\n", dpu_get_multirect_mode(pstate->multirect_mode));
-	drm_printf(p, "\tmultirect_index=%s\n", dpu_get_multirect_index(pstate->multirect_index));
+
+	drm_printf(p, "\tsspp[0]=%s\n", pipe->sspp->cap->name);
+	drm_printf(p, "\tmultirect_mode[0]=%s\n", dpu_get_multirect_mode(pipe->multirect_mode));
+	drm_printf(p, "\tmultirect_index[0]=%s\n",
+		   dpu_get_multirect_index(pipe->multirect_index));
+	drm_printf(p, "\tsrc[0]=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&pipe_cfg->src_rect));
+	drm_printf(p, "\tdst[0]=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&pipe_cfg->dst_rect));
+
+	if (r_pipe->sspp) {
+		drm_printf(p, "\tsspp[1]=%s\n", r_pipe->sspp->cap->name);
+		drm_printf(p, "\tmultirect_mode[1]=%s\n",
+			   dpu_get_multirect_mode(r_pipe->multirect_mode));
+		drm_printf(p, "\tmultirect_index[1]=%s\n",
+			   dpu_get_multirect_index(r_pipe->multirect_index));
+		drm_printf(p, "\tsrc[1]=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&r_pipe_cfg->src_rect));
+		drm_printf(p, "\tdst[1]=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&r_pipe_cfg->dst_rect));
+	}
 }
 
 static void dpu_plane_reset(struct drm_plane *plane)
 {
 	struct dpu_plane *pdpu;
 	struct dpu_plane_state *pstate;
+	struct dpu_kms *dpu_kms = _dpu_plane_get_kms(plane);
 
 	if (!plane) {
 		DPU_ERROR("invalid plane\n");
@@ -1385,6 +1390,16 @@ static void dpu_plane_reset(struct drm_plane *plane)
 		return;
 	}
 
+	/*
+	 * Set the SSPP here until we have proper virtualized DPU planes.
+	 * This is the place where the state is allocated, so fill it fully.
+	 */
+	pstate->pipe.sspp = dpu_rm_get_sspp(&dpu_kms->rm, pdpu->pipe);
+	pstate->pipe.multirect_index = DPU_SSPP_RECT_SOLO;
+	pstate->pipe.multirect_mode = DPU_SSPP_MULTIRECT_NONE;
+
+	pstate->r_pipe.sspp = NULL;
+
 	__drm_atomic_helper_plane_reset(plane, &pstate->base);
 }
 
@@ -1392,31 +1407,18 @@ static void dpu_plane_reset(struct drm_plane *plane)
 void dpu_plane_danger_signal_ctrl(struct drm_plane *plane, bool enable)
 {
 	struct dpu_plane *pdpu = to_dpu_plane(plane);
+	struct dpu_plane_state *pstate = to_dpu_plane_state(plane->state);
 	struct dpu_kms *dpu_kms = _dpu_plane_get_kms(plane);
 
 	if (!pdpu->is_rt_pipe)
 		return;
 
 	pm_runtime_get_sync(&dpu_kms->pdev->dev);
-	_dpu_plane_set_qos_ctrl(plane, enable, DPU_PLANE_QOS_PANIC_CTRL);
+	_dpu_plane_set_qos_ctrl(plane, &pstate->pipe, enable, DPU_PLANE_QOS_PANIC_CTRL);
+	if (pstate->r_pipe.sspp)
+		_dpu_plane_set_qos_ctrl(plane, &pstate->r_pipe, enable, DPU_PLANE_QOS_PANIC_CTRL);
 	pm_runtime_put_sync(&dpu_kms->pdev->dev);
 }
-
-/* SSPP live inside dpu_plane private data only. Enumerate them here. */
-void dpu_debugfs_sspp_init(struct dpu_kms *dpu_kms, struct dentry *debugfs_root)
-{
-	struct drm_plane *plane;
-	struct dentry *entry = debugfs_create_dir("sspp", debugfs_root);
-
-	if (IS_ERR(entry))
-		return;
-
-	drm_for_each_plane(plane, dpu_kms->dev) {
-		struct dpu_plane *pdpu = to_dpu_plane(plane);
-
-		_dpu_hw_sspp_init_debugfs(pdpu->pipe_hw, dpu_kms, entry);
-	}
-}
 #endif
 
 static bool dpu_plane_format_mod_supported(struct drm_plane *plane,
@@ -1450,11 +1452,6 @@ static const struct drm_plane_helper_funcs dpu_plane_helper_funcs = {
 		.atomic_update = dpu_plane_atomic_update,
 };
 
-enum dpu_sspp dpu_plane_pipe(struct drm_plane *plane)
-{
-	return plane ? to_dpu_plane(plane)->pipe : SSPP_NONE;
-}
-
 /* initialize plane */
 struct drm_plane *dpu_plane_init(struct drm_device *dev,
 		uint32_t pipe, enum drm_plane_type type,
@@ -1465,6 +1462,7 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev,
 	struct dpu_plane *pdpu;
 	struct msm_drm_private *priv = dev->dev_private;
 	struct dpu_kms *kms = to_dpu_kms(priv->kms);
+	struct dpu_hw_sspp *pipe_hw;
 	uint32_t num_formats;
 	uint32_t supported_rotations;
 	int ret = -EINVAL;
@@ -1482,24 +1480,20 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev,
 	pdpu->pipe = pipe;
 
 	/* initialize underlying h/w driver */
-	pdpu->pipe_hw = dpu_hw_sspp_init(pipe, kms->mmio, kms->catalog);
-	if (IS_ERR(pdpu->pipe_hw)) {
-		DPU_ERROR("[%u]SSPP init failed\n", pipe);
-		ret = PTR_ERR(pdpu->pipe_hw);
+	pipe_hw = dpu_rm_get_sspp(&kms->rm, pipe);
+	if (!pipe_hw || !pipe_hw->cap || !pipe_hw->cap->sblk) {
+		DPU_ERROR("[%u]SSPP is invalid\n", pipe);
 		goto clean_plane;
-	} else if (!pdpu->pipe_hw->cap || !pdpu->pipe_hw->cap->sblk) {
-		DPU_ERROR("[%u]SSPP init returned invalid cfg\n", pipe);
-		goto clean_sspp;
 	}
 
-	format_list = pdpu->pipe_hw->cap->sblk->format_list;
-	num_formats = pdpu->pipe_hw->cap->sblk->num_formats;
+	format_list = pipe_hw->cap->sblk->format_list;
+	num_formats = pipe_hw->cap->sblk->num_formats;
 
 	ret = drm_universal_plane_init(dev, plane, 0xff, &dpu_plane_funcs,
 				format_list, num_formats,
 				supported_format_modifiers, type, NULL);
 	if (ret)
-		goto clean_sspp;
+		goto clean_plane;
 
 	pdpu->catalog = kms->catalog;
 
@@ -1515,7 +1509,7 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev,
 
 	supported_rotations = DRM_MODE_REFLECT_MASK | DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
 
-	if (pdpu->pipe_hw->cap->features & BIT(DPU_SSPP_INLINE_ROTATION))
+	if (pipe_hw->cap->features & BIT(DPU_SSPP_INLINE_ROTATION))
 		supported_rotations |= DRM_MODE_ROTATE_MASK;
 
 	drm_plane_create_rotation_property(plane,
@@ -1532,9 +1526,6 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev,
 					pipe, plane->base.id);
 	return plane;
 
-clean_sspp:
-	if (pdpu && pdpu->pipe_hw)
-		dpu_hw_sspp_destroy(pdpu->pipe_hw);
 clean_plane:
 	kfree(pdpu);
 	return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
index b7b1b05199c2..abd6b21a049b 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
@@ -18,6 +18,10 @@
  * struct dpu_plane_state: Define dpu extension of drm plane state object
  * @base:	base drm plane state object
  * @aspace:	pointer to address space for input/output buffers
+ * @pipe:	software pipe description
+ * @r_pipe:	software pipe description of the second pipe
+ * @pipe_cfg:	software pipe configuration
+ * @r_pipe_cfg:	software pipe configuration for the second pipe
  * @stage:	assigned by crtc blender
  * @needs_qos_remap: qos remap settings need to be updated
  * @multirect_index: index of the rectangle of SSPP
@@ -31,10 +35,12 @@
 struct dpu_plane_state {
 	struct drm_plane_state base;
 	struct msm_gem_address_space *aspace;
+	struct dpu_sw_pipe pipe;
+	struct dpu_sw_pipe r_pipe;
+	struct dpu_sw_pipe_cfg pipe_cfg;
+	struct dpu_sw_pipe_cfg r_pipe_cfg;
 	enum dpu_stage stage;
 	bool needs_qos_remap;
-	uint32_t multirect_index;
-	uint32_t multirect_mode;
 	bool pending;
 
 	u64 plane_fetch_bw;
@@ -44,27 +50,10 @@ struct dpu_plane_state {
 	unsigned int rotation;
 };
 
-/**
- * struct dpu_multirect_plane_states: Defines multirect pair of drm plane states
- * @r0: drm plane configured on rect 0
- * @r1: drm plane configured on rect 1
- */
-struct dpu_multirect_plane_states {
-	const struct drm_plane_state *r0;
-	const struct drm_plane_state *r1;
-};
-
 #define to_dpu_plane_state(x) \
 	container_of(x, struct dpu_plane_state, base)
 
 /**
- * dpu_plane_pipe - return sspp identifier for the given plane
- * @plane:   Pointer to DRM plane object
- * Returns: sspp identifier of the given plane
- */
-enum dpu_sspp dpu_plane_pipe(struct drm_plane *plane);
-
-/**
  * dpu_plane_flush - final plane operations before commit flush
  * @plane: Pointer to drm plane structure
  */
@@ -89,19 +78,6 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev,
 		unsigned long possible_crtcs);
 
 /**
- * dpu_plane_validate_multirecti_v2 - validate the multirect planes
- *				      against hw limitations
- * @plane: drm plate states of the multirect pair
- */
-int dpu_plane_validate_multirect_v2(struct dpu_multirect_plane_states *plane);
-
-/**
- * dpu_plane_clear_multirect - clear multirect bits for the given pipe
- * @drm_state: Pointer to DRM plane state
- */
-void dpu_plane_clear_multirect(const struct drm_plane_state *drm_state);
-
-/**
  * dpu_plane_color_fill - enables color fill on plane
  * @plane:  Pointer to DRM plane object
  * @color:  RGB fill color value, [23..16] Blue, [15..8] Green, [7..0] Red
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
index 66c1b70d244f..f4dda88a73f7 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
@@ -8,6 +8,7 @@
 #include "dpu_hw_lm.h"
 #include "dpu_hw_ctl.h"
 #include "dpu_hw_pingpong.h"
+#include "dpu_hw_sspp.h"
 #include "dpu_hw_intf.h"
 #include "dpu_hw_wb.h"
 #include "dpu_hw_dspp.h"
@@ -91,6 +92,9 @@ int dpu_rm_destroy(struct dpu_rm *rm)
 	for (i = 0; i < ARRAY_SIZE(rm->hw_wb); i++)
 		dpu_hw_wb_destroy(rm->hw_wb[i]);
 
+	for (i = 0; i < ARRAY_SIZE(rm->hw_sspp); i++)
+		dpu_hw_sspp_destroy(rm->hw_sspp[i]);
+
 	return 0;
 }
 
@@ -255,6 +259,24 @@ int dpu_rm_init(struct dpu_rm *rm,
 		rm->dsc_blks[dsc->id - DSC_0] = &hw->base;
 	}
 
+	for (i = 0; i < cat->sspp_count; i++) {
+		struct dpu_hw_sspp *hw;
+		const struct dpu_sspp_cfg *sspp = &cat->sspp[i];
+
+		if (sspp->id < SSPP_NONE || sspp->id >= SSPP_MAX) {
+			DPU_ERROR("skip intf %d with invalid id\n", sspp->id);
+			continue;
+		}
+
+		hw = dpu_hw_sspp_init(sspp->id, mmio, cat);
+		if (IS_ERR(hw)) {
+			rc = PTR_ERR(hw);
+			DPU_ERROR("failed sspp object creation: err %d\n", rc);
+			goto fail;
+		}
+		rm->hw_sspp[sspp->id - SSPP_NONE] = hw;
+	}
+
 	return 0;
 
 fail:
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
index 59de72b381f9..d62c2edb2460 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
@@ -21,6 +21,7 @@ struct dpu_global_state;
  * @hw_intf: array of intf hardware resources
  * @hw_wb: array of wb hardware resources
  * @dspp_blks: array of dspp hardware resources
+ * @hw_sspp: array of sspp hardware resources
  */
 struct dpu_rm {
 	struct dpu_hw_blk *pingpong_blks[PINGPONG_MAX - PINGPONG_0];
@@ -31,6 +32,7 @@ struct dpu_rm {
 	struct dpu_hw_blk *dspp_blks[DSPP_MAX - DSPP_0];
 	struct dpu_hw_blk *merge_3d_blks[MERGE_3D_MAX - MERGE_3D_0];
 	struct dpu_hw_blk *dsc_blks[DSC_MAX - DSC_0];
+	struct dpu_hw_sspp *hw_sspp[SSPP_MAX - SSPP_NONE];
 };
 
 /**
@@ -108,5 +110,15 @@ static inline struct dpu_hw_wb *dpu_rm_get_wb(struct dpu_rm *rm, enum dpu_wb wb_
 	return rm->hw_wb[wb_idx - WB_0];
 }
 
+/**
+ * dpu_rm_get_sspp - Return a struct dpu_hw_sspp instance given it's index.
+ * @rm: DPU Resource Manager handle
+ * @sspp_idx: SSPP index
+ */
+static inline struct dpu_hw_sspp *dpu_rm_get_sspp(struct dpu_rm *rm, enum dpu_sspp sspp_idx)
+{
+	return rm->hw_sspp[sspp_idx - SSPP_NONE];
+}
+
 #endif /* __DPU_RM_H__ */
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h
index 76169f406505..0ad148cc2fb8 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h
@@ -633,9 +633,9 @@ TRACE_EVENT(dpu_enc_phys_vid_irq_ctrl,
 TRACE_EVENT(dpu_crtc_setup_mixer,
 	TP_PROTO(uint32_t crtc_id, uint32_t plane_id,
 		 struct drm_plane_state *state, struct dpu_plane_state *pstate,
-		 uint32_t stage_idx, enum dpu_sspp sspp, uint32_t pixel_format,
+		 uint32_t stage_idx, uint32_t pixel_format,
 		 uint64_t modifier),
-	TP_ARGS(crtc_id, plane_id, state, pstate, stage_idx, sspp,
+	TP_ARGS(crtc_id, plane_id, state, pstate, stage_idx,
 		pixel_format, modifier),
 	TP_STRUCT__entry(
 		__field(	uint32_t,		crtc_id		)
@@ -659,9 +659,9 @@ TRACE_EVENT(dpu_crtc_setup_mixer,
 		__entry->dst_rect = drm_plane_state_dest(state);
 		__entry->stage_idx = stage_idx;
 		__entry->stage = pstate->stage;
-		__entry->sspp = sspp;
-		__entry->multirect_idx = pstate->multirect_index;
-		__entry->multirect_mode = pstate->multirect_mode;
+		__entry->sspp = pstate->pipe.sspp->idx;
+		__entry->multirect_idx = pstate->pipe.multirect_index;
+		__entry->multirect_mode = pstate->pipe.multirect_mode;
 		__entry->pixel_format = pixel_format;
 		__entry->modifier = modifier;
 	),
@@ -762,18 +762,17 @@ TRACE_EVENT(dpu_crtc_disable_frame_pending,
 );
 
 TRACE_EVENT(dpu_plane_set_scanout,
-	TP_PROTO(enum dpu_sspp index, struct dpu_hw_fmt_layout *layout,
-		 enum dpu_sspp_multirect_index multirect_index),
-	TP_ARGS(index, layout, multirect_index),
+	TP_PROTO(struct dpu_sw_pipe *pipe, struct dpu_hw_fmt_layout *layout),
+	TP_ARGS(pipe, layout),
 	TP_STRUCT__entry(
 		__field(	enum dpu_sspp,			index	)
 		__field_struct(	struct dpu_hw_fmt_layout,	layout	)
 		__field(	enum dpu_sspp_multirect_index,	multirect_index)
 	),
 	TP_fast_assign(
-		__entry->index = index;
+		__entry->index = pipe->sspp->idx;
 		__entry->layout = *layout;
-		__entry->multirect_index = multirect_index;
+		__entry->multirect_index = pipe->multirect_index;
 	),
 	TP_printk("index:%d layout:{%ux%u @ [%u/%u, %u/%u, %u/%u, %u/%u]} "
 		  "multirect_index:%d", __entry->index, __entry->layout.width,
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h b/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h
index a2b6422948ec..cc8fde450884 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
index 9a1a0769575d..6e37072ed302 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
@@ -84,10 +84,6 @@ static void mdp4_disable_commit(struct msm_kms *kms)
 	mdp4_disable(mdp4_kms);
 }
 
-static void mdp4_prepare_commit(struct msm_kms *kms, struct drm_atomic_state *state)
-{
-}
-
 static void mdp4_flush_commit(struct msm_kms *kms, unsigned crtc_mask)
 {
 	/* TODO */
@@ -154,7 +150,6 @@ static const struct mdp_kms_funcs kms_funcs = {
 		.disable_vblank  = mdp4_disable_vblank,
 		.enable_commit   = mdp4_enable_commit,
 		.disable_commit  = mdp4_disable_commit,
-		.prepare_commit  = mdp4_prepare_commit,
 		.flush_commit    = mdp4_flush_commit,
 		.wait_flush      = mdp4_wait_flush,
 		.complete_commit = mdp4_complete_commit,
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h
index 86fc44b518cb..270e11c904bd 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c
index 1f1555aa02d2..2eec2d78f32a 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c
@@ -655,7 +655,7 @@ static const struct mdp5_cfg_hw msm8x96_config = {
 	.max_clk = 412500000,
 };
 
-const struct mdp5_cfg_hw msm8x76_config = {
+static const struct mdp5_cfg_hw msm8x76_config = {
 	.name = "msm8x76",
 	.mdp = {
 		.count = 1,
diff --git a/drivers/gpu/drm/msm/disp/mdp_common.xml.h b/drivers/gpu/drm/msm/disp/mdp_common.xml.h
index be759106b621..4dd8d7db2862 100644
--- a/drivers/gpu/drm/msm/disp/mdp_common.xml.h
+++ b/drivers/gpu/drm/msm/disp/mdp_common.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c
index b73031cd48e4..e75b97127c0d 100644
--- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c
+++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.c
@@ -129,9 +129,6 @@ void msm_disp_snapshot_destroy(struct drm_device *drm_dev)
 	}
 
 	priv = drm_dev->dev_private;
-	if (!priv->kms)
-		return;
-
 	kms = priv->kms;
 
 	if (kms->dump_worker)
diff --git a/drivers/gpu/drm/msm/dp/dp_aux.c b/drivers/gpu/drm/msm/dp/dp_aux.c
index cc3efed593aa..8e3b677f35e6 100644
--- a/drivers/gpu/drm/msm/dp/dp_aux.c
+++ b/drivers/gpu/drm/msm/dp/dp_aux.c
@@ -162,47 +162,6 @@ static ssize_t dp_aux_cmd_fifo_rx(struct dp_aux_private *aux,
 	return i;
 }
 
-static void dp_aux_native_handler(struct dp_aux_private *aux, u32 isr)
-{
-	if (isr & DP_INTR_AUX_I2C_DONE)
-		aux->aux_error_num = DP_AUX_ERR_NONE;
-	else if (isr & DP_INTR_WRONG_ADDR)
-		aux->aux_error_num = DP_AUX_ERR_ADDR;
-	else if (isr & DP_INTR_TIMEOUT)
-		aux->aux_error_num = DP_AUX_ERR_TOUT;
-	if (isr & DP_INTR_NACK_DEFER)
-		aux->aux_error_num = DP_AUX_ERR_NACK;
-	if (isr & DP_INTR_AUX_ERROR) {
-		aux->aux_error_num = DP_AUX_ERR_PHY;
-		dp_catalog_aux_clear_hw_interrupts(aux->catalog);
-	}
-}
-
-static void dp_aux_i2c_handler(struct dp_aux_private *aux, u32 isr)
-{
-	if (isr & DP_INTR_AUX_I2C_DONE) {
-		if (isr & (DP_INTR_I2C_NACK | DP_INTR_I2C_DEFER))
-			aux->aux_error_num = DP_AUX_ERR_NACK;
-		else
-			aux->aux_error_num = DP_AUX_ERR_NONE;
-	} else {
-		if (isr & DP_INTR_WRONG_ADDR)
-			aux->aux_error_num = DP_AUX_ERR_ADDR;
-		else if (isr & DP_INTR_TIMEOUT)
-			aux->aux_error_num = DP_AUX_ERR_TOUT;
-		if (isr & DP_INTR_NACK_DEFER)
-			aux->aux_error_num = DP_AUX_ERR_NACK_DEFER;
-		if (isr & DP_INTR_I2C_NACK)
-			aux->aux_error_num = DP_AUX_ERR_NACK;
-		if (isr & DP_INTR_I2C_DEFER)
-			aux->aux_error_num = DP_AUX_ERR_DEFER;
-		if (isr & DP_INTR_AUX_ERROR) {
-			aux->aux_error_num = DP_AUX_ERR_PHY;
-			dp_catalog_aux_clear_hw_interrupts(aux->catalog);
-		}
-	}
-}
-
 static void dp_aux_update_offset_and_segment(struct dp_aux_private *aux,
 					     struct drm_dp_aux_msg *input_msg)
 {
@@ -409,14 +368,14 @@ exit:
 	return ret;
 }
 
-void dp_aux_isr(struct drm_dp_aux *dp_aux)
+irqreturn_t dp_aux_isr(struct drm_dp_aux *dp_aux)
 {
 	u32 isr;
 	struct dp_aux_private *aux;
 
 	if (!dp_aux) {
 		DRM_ERROR("invalid input\n");
-		return;
+		return IRQ_NONE;
 	}
 
 	aux = container_of(dp_aux, struct dp_aux_private, dp_aux);
@@ -425,17 +384,48 @@ void dp_aux_isr(struct drm_dp_aux *dp_aux)
 
 	/* no interrupts pending, return immediately */
 	if (!isr)
-		return;
+		return IRQ_NONE;
 
-	if (!aux->cmd_busy)
-		return;
+	if (!aux->cmd_busy) {
+		DRM_ERROR("Unexpected DP AUX IRQ %#010x when not busy\n", isr);
+		return IRQ_NONE;
+	}
 
-	if (aux->native)
-		dp_aux_native_handler(aux, isr);
-	else
-		dp_aux_i2c_handler(aux, isr);
+	/*
+	 * The logic below assumes only one error bit is set (other than "done"
+	 * which can apparently be set at the same time as some of the other
+	 * bits). Warn if more than one get set so we know we need to improve
+	 * the logic.
+	 */
+	if (hweight32(isr & ~DP_INTR_AUX_XFER_DONE) > 1)
+		DRM_WARN("Some DP AUX interrupts unhandled: %#010x\n", isr);
+
+	if (isr & DP_INTR_AUX_ERROR) {
+		aux->aux_error_num = DP_AUX_ERR_PHY;
+		dp_catalog_aux_clear_hw_interrupts(aux->catalog);
+	} else if (isr & DP_INTR_NACK_DEFER) {
+		aux->aux_error_num = DP_AUX_ERR_NACK_DEFER;
+	} else if (isr & DP_INTR_WRONG_ADDR) {
+		aux->aux_error_num = DP_AUX_ERR_ADDR;
+	} else if (isr & DP_INTR_TIMEOUT) {
+		aux->aux_error_num = DP_AUX_ERR_TOUT;
+	} else if (!aux->native && (isr & DP_INTR_I2C_NACK)) {
+		aux->aux_error_num = DP_AUX_ERR_NACK;
+	} else if (!aux->native && (isr & DP_INTR_I2C_DEFER)) {
+		if (isr & DP_INTR_AUX_XFER_DONE)
+			aux->aux_error_num = DP_AUX_ERR_NACK;
+		else
+			aux->aux_error_num = DP_AUX_ERR_DEFER;
+	} else if (isr & DP_INTR_AUX_XFER_DONE) {
+		aux->aux_error_num = DP_AUX_ERR_NONE;
+	} else {
+		DRM_WARN("Unexpected interrupt: %#010x\n", isr);
+		return IRQ_NONE;
+	}
 
 	complete(&aux->comp);
+
+	return IRQ_HANDLED;
 }
 
 void dp_aux_reconfig(struct drm_dp_aux *dp_aux)
diff --git a/drivers/gpu/drm/msm/dp/dp_aux.h b/drivers/gpu/drm/msm/dp/dp_aux.h
index e930974bcb5b..511305da4f66 100644
--- a/drivers/gpu/drm/msm/dp/dp_aux.h
+++ b/drivers/gpu/drm/msm/dp/dp_aux.h
@@ -11,7 +11,7 @@
 
 int dp_aux_register(struct drm_dp_aux *dp_aux);
 void dp_aux_unregister(struct drm_dp_aux *dp_aux);
-void dp_aux_isr(struct drm_dp_aux *dp_aux);
+irqreturn_t dp_aux_isr(struct drm_dp_aux *dp_aux);
 void dp_aux_init(struct drm_dp_aux *dp_aux);
 void dp_aux_deinit(struct drm_dp_aux *dp_aux);
 void dp_aux_reconfig(struct drm_dp_aux *dp_aux);
diff --git a/drivers/gpu/drm/msm/dp/dp_catalog.c b/drivers/gpu/drm/msm/dp/dp_catalog.c
index 676279d0ca8d..7a8cf1c8233d 100644
--- a/drivers/gpu/drm/msm/dp/dp_catalog.c
+++ b/drivers/gpu/drm/msm/dp/dp_catalog.c
@@ -27,7 +27,7 @@
 #define DP_INTF_CONFIG_DATABUS_WIDEN     BIT(4)
 
 #define DP_INTERRUPT_STATUS1 \
-	(DP_INTR_AUX_I2C_DONE| \
+	(DP_INTR_AUX_XFER_DONE| \
 	DP_INTR_WRONG_ADDR | DP_INTR_TIMEOUT | \
 	DP_INTR_NACK_DEFER | DP_INTR_WRONG_DATA_CNT | \
 	DP_INTR_I2C_NACK | DP_INTR_I2C_DEFER | \
@@ -47,6 +47,14 @@
 #define DP_INTERRUPT_STATUS2_MASK \
 	(DP_INTERRUPT_STATUS2 << DP_INTERRUPT_STATUS_MASK_SHIFT)
 
+#define DP_INTERRUPT_STATUS4 \
+	(PSR_UPDATE_INT | PSR_CAPTURE_INT | PSR_EXIT_INT | \
+	PSR_UPDATE_ERROR_INT | PSR_WAKE_ERROR_INT)
+
+#define DP_INTERRUPT_MASK4 \
+	(PSR_UPDATE_MASK | PSR_CAPTURE_MASK | PSR_EXIT_MASK | \
+	PSR_UPDATE_ERROR_MASK | PSR_WAKE_ERROR_MASK)
+
 struct dp_catalog_private {
 	struct device *dev;
 	struct drm_device *drm_dev;
@@ -359,6 +367,23 @@ void dp_catalog_ctrl_lane_mapping(struct dp_catalog *dp_catalog)
 			ln_mapping);
 }
 
+void dp_catalog_ctrl_psr_mainlink_enable(struct dp_catalog *dp_catalog,
+						bool enable)
+{
+	u32 val;
+	struct dp_catalog_private *catalog = container_of(dp_catalog,
+				struct dp_catalog_private, dp_catalog);
+
+	val = dp_read_link(catalog, REG_DP_MAINLINK_CTRL);
+
+	if (enable)
+		val |= DP_MAINLINK_CTRL_ENABLE;
+	else
+		val &= ~DP_MAINLINK_CTRL_ENABLE;
+
+	dp_write_link(catalog, REG_DP_MAINLINK_CTRL, val);
+}
+
 void dp_catalog_ctrl_mainlink_ctrl(struct dp_catalog *dp_catalog,
 						bool enable)
 {
@@ -610,6 +635,47 @@ void dp_catalog_ctrl_hpd_config(struct dp_catalog *dp_catalog)
 	dp_write_aux(catalog, REG_DP_DP_HPD_CTRL, DP_DP_HPD_CTRL_HPD_EN);
 }
 
+static void dp_catalog_enable_sdp(struct dp_catalog_private *catalog)
+{
+	/* trigger sdp */
+	dp_write_link(catalog, MMSS_DP_SDP_CFG3, UPDATE_SDP);
+	dp_write_link(catalog, MMSS_DP_SDP_CFG3, 0x0);
+}
+
+void dp_catalog_ctrl_config_psr(struct dp_catalog *dp_catalog)
+{
+	struct dp_catalog_private *catalog = container_of(dp_catalog,
+				struct dp_catalog_private, dp_catalog);
+	u32 config;
+
+	/* enable PSR1 function */
+	config = dp_read_link(catalog, REG_PSR_CONFIG);
+	config |= PSR1_SUPPORTED;
+	dp_write_link(catalog, REG_PSR_CONFIG, config);
+
+	dp_write_ahb(catalog, REG_DP_INTR_MASK4, DP_INTERRUPT_MASK4);
+	dp_catalog_enable_sdp(catalog);
+}
+
+void dp_catalog_ctrl_set_psr(struct dp_catalog *dp_catalog, bool enter)
+{
+	struct dp_catalog_private *catalog = container_of(dp_catalog,
+			struct dp_catalog_private, dp_catalog);
+	u32 cmd;
+
+	cmd = dp_read_link(catalog, REG_PSR_CMD);
+
+	cmd &= ~(PSR_ENTER | PSR_EXIT);
+
+	if (enter)
+		cmd |= PSR_ENTER;
+	else
+		cmd |= PSR_EXIT;
+
+	dp_catalog_enable_sdp(catalog);
+	dp_write_link(catalog, REG_PSR_CMD, cmd);
+}
+
 u32 dp_catalog_link_is_connected(struct dp_catalog *dp_catalog)
 {
 	struct dp_catalog_private *catalog = container_of(dp_catalog,
@@ -645,6 +711,20 @@ u32 dp_catalog_hpd_get_intr_status(struct dp_catalog *dp_catalog)
 	return isr & (mask | ~DP_DP_HPD_INT_MASK);
 }
 
+u32 dp_catalog_ctrl_read_psr_interrupt_status(struct dp_catalog *dp_catalog)
+{
+	struct dp_catalog_private *catalog = container_of(dp_catalog,
+				struct dp_catalog_private, dp_catalog);
+	u32 intr, intr_ack;
+
+	intr = dp_read_ahb(catalog, REG_DP_INTR_STATUS4);
+	intr_ack = (intr & DP_INTERRUPT_STATUS4)
+			<< DP_INTERRUPT_STATUS_ACK_SHIFT;
+	dp_write_ahb(catalog, REG_DP_INTR_STATUS4, intr_ack);
+
+	return intr;
+}
+
 int dp_catalog_ctrl_get_interrupt(struct dp_catalog *dp_catalog)
 {
 	struct dp_catalog_private *catalog = container_of(dp_catalog,
diff --git a/drivers/gpu/drm/msm/dp/dp_catalog.h b/drivers/gpu/drm/msm/dp/dp_catalog.h
index 1f717f45c115..82376a2697ef 100644
--- a/drivers/gpu/drm/msm/dp/dp_catalog.h
+++ b/drivers/gpu/drm/msm/dp/dp_catalog.h
@@ -13,7 +13,7 @@
 
 /* interrupts */
 #define DP_INTR_HPD		BIT(0)
-#define DP_INTR_AUX_I2C_DONE	BIT(3)
+#define DP_INTR_AUX_XFER_DONE	BIT(3)
 #define DP_INTR_WRONG_ADDR	BIT(6)
 #define DP_INTR_TIMEOUT		BIT(9)
 #define DP_INTR_NACK_DEFER	BIT(12)
@@ -93,6 +93,7 @@ void dp_catalog_ctrl_state_ctrl(struct dp_catalog *dp_catalog, u32 state);
 void dp_catalog_ctrl_config_ctrl(struct dp_catalog *dp_catalog, u32 config);
 void dp_catalog_ctrl_lane_mapping(struct dp_catalog *dp_catalog);
 void dp_catalog_ctrl_mainlink_ctrl(struct dp_catalog *dp_catalog, bool enable);
+void dp_catalog_ctrl_psr_mainlink_enable(struct dp_catalog *dp_catalog, bool enable);
 void dp_catalog_ctrl_config_misc(struct dp_catalog *dp_catalog, u32 cc, u32 tb);
 void dp_catalog_ctrl_config_msa(struct dp_catalog *dp_catalog, u32 rate,
 				u32 stream_rate_khz, bool fixed_nvid);
@@ -104,12 +105,15 @@ void dp_catalog_ctrl_enable_irq(struct dp_catalog *dp_catalog, bool enable);
 void dp_catalog_hpd_config_intr(struct dp_catalog *dp_catalog,
 			u32 intr_mask, bool en);
 void dp_catalog_ctrl_hpd_config(struct dp_catalog *dp_catalog);
+void dp_catalog_ctrl_config_psr(struct dp_catalog *dp_catalog);
+void dp_catalog_ctrl_set_psr(struct dp_catalog *dp_catalog, bool enter);
 u32 dp_catalog_link_is_connected(struct dp_catalog *dp_catalog);
 u32 dp_catalog_hpd_get_intr_status(struct dp_catalog *dp_catalog);
 void dp_catalog_ctrl_phy_reset(struct dp_catalog *dp_catalog);
 int dp_catalog_ctrl_update_vx_px(struct dp_catalog *dp_catalog, u8 v_level,
 				u8 p_level);
 int dp_catalog_ctrl_get_interrupt(struct dp_catalog *dp_catalog);
+u32 dp_catalog_ctrl_read_psr_interrupt_status(struct dp_catalog *dp_catalog);
 void dp_catalog_ctrl_update_transfer_unit(struct dp_catalog *dp_catalog,
 				u32 dp_tu, u32 valid_boundary,
 				u32 valid_boundary2);
diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c
index dd26ca651a05..a7a5c7e0ab92 100644
--- a/drivers/gpu/drm/msm/dp/dp_ctrl.c
+++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c
@@ -22,6 +22,7 @@
 
 #define DP_KHZ_TO_HZ 1000
 #define IDLE_PATTERN_COMPLETION_TIMEOUT_JIFFIES	(30 * HZ / 1000) /* 30 ms */
+#define PSR_OPERATION_COMPLETION_TIMEOUT_JIFFIES       (300 * HZ / 1000) /* 300 ms */
 #define WAIT_FOR_VIDEO_READY_TIMEOUT_JIFFIES (HZ / 2)
 
 #define DP_CTRL_INTR_READY_FOR_VIDEO     BIT(0)
@@ -80,6 +81,7 @@ struct dp_ctrl_private {
 	struct dp_catalog *catalog;
 
 	struct completion idle_comp;
+	struct completion psr_op_comp;
 	struct completion video_comp;
 };
 
@@ -153,6 +155,9 @@ static void dp_ctrl_config_ctrl(struct dp_ctrl_private *ctrl)
 	config |= DP_CONFIGURATION_CTRL_STATIC_DYNAMIC_CN;
 	config |= DP_CONFIGURATION_CTRL_SYNC_ASYNC_CLK;
 
+	if (ctrl->panel->psr_cap.version)
+		config |= DP_CONFIGURATION_CTRL_SEND_VSC;
+
 	dp_catalog_ctrl_config_ctrl(ctrl->catalog, config);
 }
 
@@ -1375,6 +1380,64 @@ void dp_ctrl_reset_irq_ctrl(struct dp_ctrl *dp_ctrl, bool enable)
 	dp_catalog_ctrl_enable_irq(ctrl->catalog, enable);
 }
 
+void dp_ctrl_config_psr(struct dp_ctrl *dp_ctrl)
+{
+	u8 cfg;
+	struct dp_ctrl_private *ctrl = container_of(dp_ctrl,
+			struct dp_ctrl_private, dp_ctrl);
+
+	if (!ctrl->panel->psr_cap.version)
+		return;
+
+	dp_catalog_ctrl_config_psr(ctrl->catalog);
+
+	cfg = DP_PSR_ENABLE;
+	drm_dp_dpcd_write(ctrl->aux, DP_PSR_EN_CFG, &cfg, 1);
+}
+
+void dp_ctrl_set_psr(struct dp_ctrl *dp_ctrl, bool enter)
+{
+	struct dp_ctrl_private *ctrl = container_of(dp_ctrl,
+			struct dp_ctrl_private, dp_ctrl);
+
+	if (!ctrl->panel->psr_cap.version)
+		return;
+
+	/*
+	 * When entering PSR,
+	 * 1. Send PSR enter SDP and wait for the PSR_UPDATE_INT
+	 * 2. Turn off video
+	 * 3. Disable the mainlink
+	 *
+	 * When exiting PSR,
+	 * 1. Enable the mainlink
+	 * 2. Send the PSR exit SDP
+	 */
+	if (enter) {
+		reinit_completion(&ctrl->psr_op_comp);
+		dp_catalog_ctrl_set_psr(ctrl->catalog, true);
+
+		if (!wait_for_completion_timeout(&ctrl->psr_op_comp,
+			PSR_OPERATION_COMPLETION_TIMEOUT_JIFFIES)) {
+			DRM_ERROR("PSR_ENTRY timedout\n");
+			dp_catalog_ctrl_set_psr(ctrl->catalog, false);
+			return;
+		}
+
+		dp_ctrl_push_idle(dp_ctrl);
+		dp_catalog_ctrl_state_ctrl(ctrl->catalog, 0);
+
+		dp_catalog_ctrl_psr_mainlink_enable(ctrl->catalog, false);
+	} else {
+		dp_catalog_ctrl_psr_mainlink_enable(ctrl->catalog, true);
+
+		dp_catalog_ctrl_set_psr(ctrl->catalog, false);
+		dp_catalog_ctrl_state_ctrl(ctrl->catalog, DP_STATE_CTRL_SEND_VIDEO);
+		dp_ctrl_wait4video_ready(ctrl);
+		dp_catalog_ctrl_state_ctrl(ctrl->catalog, 0);
+	}
+}
+
 void dp_ctrl_phy_init(struct dp_ctrl *dp_ctrl)
 {
 	struct dp_ctrl_private *ctrl;
@@ -1979,27 +2042,49 @@ int dp_ctrl_off(struct dp_ctrl *dp_ctrl)
 	return ret;
 }
 
-void dp_ctrl_isr(struct dp_ctrl *dp_ctrl)
+irqreturn_t dp_ctrl_isr(struct dp_ctrl *dp_ctrl)
 {
 	struct dp_ctrl_private *ctrl;
 	u32 isr;
+	irqreturn_t ret = IRQ_NONE;
 
 	if (!dp_ctrl)
-		return;
+		return IRQ_NONE;
 
 	ctrl = container_of(dp_ctrl, struct dp_ctrl_private, dp_ctrl);
 
+	if (ctrl->panel->psr_cap.version) {
+		isr = dp_catalog_ctrl_read_psr_interrupt_status(ctrl->catalog);
+
+		if (isr)
+			complete(&ctrl->psr_op_comp);
+
+		if (isr & PSR_EXIT_INT)
+			drm_dbg_dp(ctrl->drm_dev, "PSR exit done\n");
+
+		if (isr & PSR_UPDATE_INT)
+			drm_dbg_dp(ctrl->drm_dev, "PSR frame update done\n");
+
+		if (isr & PSR_CAPTURE_INT)
+			drm_dbg_dp(ctrl->drm_dev, "PSR frame capture done\n");
+	}
+
 	isr = dp_catalog_ctrl_get_interrupt(ctrl->catalog);
 
+
 	if (isr & DP_CTRL_INTR_READY_FOR_VIDEO) {
 		drm_dbg_dp(ctrl->drm_dev, "dp_video_ready\n");
 		complete(&ctrl->video_comp);
+		ret = IRQ_HANDLED;
 	}
 
 	if (isr & DP_CTRL_INTR_IDLE_PATTERN_SENT) {
 		drm_dbg_dp(ctrl->drm_dev, "idle_patterns_sent\n");
 		complete(&ctrl->idle_comp);
+		ret = IRQ_HANDLED;
 	}
+
+	return ret;
 }
 
 struct dp_ctrl *dp_ctrl_get(struct device *dev, struct dp_link *link,
@@ -2035,6 +2120,7 @@ struct dp_ctrl *dp_ctrl_get(struct device *dev, struct dp_link *link,
 		dev_err(dev, "failed to add DP OPP table\n");
 
 	init_completion(&ctrl->idle_comp);
+	init_completion(&ctrl->psr_op_comp);
 	init_completion(&ctrl->video_comp);
 
 	/* in parameters */
diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.h b/drivers/gpu/drm/msm/dp/dp_ctrl.h
index 9f29734af81c..f712780149fd 100644
--- a/drivers/gpu/drm/msm/dp/dp_ctrl.h
+++ b/drivers/gpu/drm/msm/dp/dp_ctrl.h
@@ -25,7 +25,7 @@ int dp_ctrl_off_link_stream(struct dp_ctrl *dp_ctrl);
 int dp_ctrl_off_link(struct dp_ctrl *dp_ctrl);
 int dp_ctrl_off(struct dp_ctrl *dp_ctrl);
 void dp_ctrl_push_idle(struct dp_ctrl *dp_ctrl);
-void dp_ctrl_isr(struct dp_ctrl *dp_ctrl);
+irqreturn_t dp_ctrl_isr(struct dp_ctrl *dp_ctrl);
 void dp_ctrl_handle_sink_request(struct dp_ctrl *dp_ctrl);
 struct dp_ctrl *dp_ctrl_get(struct device *dev, struct dp_link *link,
 			struct dp_panel *panel,	struct drm_dp_aux *aux,
@@ -37,4 +37,7 @@ void dp_ctrl_phy_init(struct dp_ctrl *dp_ctrl);
 void dp_ctrl_phy_exit(struct dp_ctrl *dp_ctrl);
 void dp_ctrl_irq_phy_exit(struct dp_ctrl *dp_ctrl);
 
+void dp_ctrl_set_psr(struct dp_ctrl *dp_ctrl, bool enable);
+void dp_ctrl_config_psr(struct dp_ctrl *dp_ctrl);
+
 #endif /* _DP_CTRL_H_ */
diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c
index bde1a7ce442f..3e13acdfa7e5 100644
--- a/drivers/gpu/drm/msm/dp/dp_display.c
+++ b/drivers/gpu/drm/msm/dp/dp_display.c
@@ -406,6 +406,8 @@ static int dp_display_process_hpd_high(struct dp_display_private *dp)
 
 	edid = dp->panel->edid;
 
+	dp->dp_display.psr_supported = dp->panel->psr_cap.version;
+
 	dp->audio_supported = drm_detect_monitor_audio(edid);
 	dp_panel_handle_sink_request(dp->panel);
 
@@ -910,6 +912,10 @@ static int dp_display_post_enable(struct msm_dp *dp_display)
 
 	/* signal the connect event late to synchronize video and display */
 	dp_display_handle_plugged_change(dp_display, true);
+
+	if (dp_display->psr_supported)
+		dp_ctrl_config_psr(dp->ctrl);
+
 	return 0;
 }
 
@@ -990,14 +996,6 @@ enum drm_mode_status dp_bridge_mode_valid(struct drm_bridge *bridge,
 		return -EINVAL;
 	}
 
-	/*
-	 * The eDP controller currently does not have a reliable way of
-	 * enabling panel power to read sink capabilities. So, we rely
-	 * on the panel driver to populate only supported modes for now.
-	 */
-	if (dp->is_edp)
-		return MODE_OK;
-
 	if (mode->clock > DP_MAX_PIXEL_CLK_KHZ)
 		return MODE_CLOCK_HIGH;
 
@@ -1104,6 +1102,19 @@ static void dp_display_config_hpd(struct dp_display_private *dp)
 	enable_irq(dp->irq);
 }
 
+void dp_display_set_psr(struct msm_dp *dp_display, bool enter)
+{
+	struct dp_display_private *dp;
+
+	if (!dp_display) {
+		DRM_ERROR("invalid params\n");
+		return;
+	}
+
+	dp = container_of(dp_display, struct dp_display_private, dp_display);
+	dp_ctrl_set_psr(dp->ctrl, enter);
+}
+
 static int hpd_event_thread(void *data)
 {
 	struct dp_display_private *dp_priv;
@@ -1204,7 +1215,7 @@ static int dp_hpd_event_thread_start(struct dp_display_private *dp_priv)
 static irqreturn_t dp_display_irq_handler(int irq, void *dev_id)
 {
 	struct dp_display_private *dp = dev_id;
-	irqreturn_t ret = IRQ_HANDLED;
+	irqreturn_t ret = IRQ_NONE;
 	u32 hpd_isr_status;
 
 	if (!dp) {
@@ -1232,13 +1243,15 @@ static irqreturn_t dp_display_irq_handler(int irq, void *dev_id)
 
 		if (hpd_isr_status & DP_DP_HPD_UNPLUG_INT_MASK)
 			dp_add_event(dp, EV_HPD_UNPLUG_INT, 0, 0);
+
+		ret = IRQ_HANDLED;
 	}
 
 	/* DP controller isr */
-	dp_ctrl_isr(dp->ctrl);
+	ret |= dp_ctrl_isr(dp->ctrl);
 
 	/* DP aux isr */
-	dp_aux_isr(dp->aux);
+	ret |= dp_aux_isr(dp->aux);
 
 	return ret;
 }
@@ -1652,7 +1665,8 @@ int msm_dp_modeset_init(struct msm_dp *dp_display, struct drm_device *dev,
 	return 0;
 }
 
-void dp_bridge_enable(struct drm_bridge *drm_bridge)
+void dp_bridge_atomic_enable(struct drm_bridge *drm_bridge,
+			     struct drm_bridge_state *old_bridge_state)
 {
 	struct msm_dp_bridge *dp_bridge = to_dp_bridge(drm_bridge);
 	struct msm_dp *dp = dp_bridge->dp_display;
@@ -1707,7 +1721,8 @@ void dp_bridge_enable(struct drm_bridge *drm_bridge)
 	mutex_unlock(&dp_display->event_mutex);
 }
 
-void dp_bridge_disable(struct drm_bridge *drm_bridge)
+void dp_bridge_atomic_disable(struct drm_bridge *drm_bridge,
+			      struct drm_bridge_state *old_bridge_state)
 {
 	struct msm_dp_bridge *dp_bridge = to_dp_bridge(drm_bridge);
 	struct msm_dp *dp = dp_bridge->dp_display;
@@ -1718,7 +1733,8 @@ void dp_bridge_disable(struct drm_bridge *drm_bridge)
 	dp_ctrl_push_idle(dp_display->ctrl);
 }
 
-void dp_bridge_post_disable(struct drm_bridge *drm_bridge)
+void dp_bridge_atomic_post_disable(struct drm_bridge *drm_bridge,
+				   struct drm_bridge_state *old_bridge_state)
 {
 	struct msm_dp_bridge *dp_bridge = to_dp_bridge(drm_bridge);
 	struct msm_dp *dp = dp_bridge->dp_display;
diff --git a/drivers/gpu/drm/msm/dp/dp_display.h b/drivers/gpu/drm/msm/dp/dp_display.h
index 371337d0fae2..1e9415ab15d8 100644
--- a/drivers/gpu/drm/msm/dp/dp_display.h
+++ b/drivers/gpu/drm/msm/dp/dp_display.h
@@ -29,6 +29,7 @@ struct msm_dp {
 
 	u32 max_dp_lanes;
 	struct dp_audio *dp_audio;
+	bool psr_supported;
 };
 
 int dp_display_set_plugged_cb(struct msm_dp *dp_display,
@@ -39,5 +40,6 @@ bool dp_display_check_video_test(struct msm_dp *dp_display);
 int dp_display_get_test_bpp(struct msm_dp *dp_display);
 void dp_display_signal_audio_start(struct msm_dp *dp_display);
 void dp_display_signal_audio_complete(struct msm_dp *dp_display);
+void dp_display_set_psr(struct msm_dp *dp, bool enter);
 
 #endif /* _DP_DISPLAY_H_ */
diff --git a/drivers/gpu/drm/msm/dp/dp_drm.c b/drivers/gpu/drm/msm/dp/dp_drm.c
index 275370f21115..785d76639497 100644
--- a/drivers/gpu/drm/msm/dp/dp_drm.c
+++ b/drivers/gpu/drm/msm/dp/dp_drm.c
@@ -94,9 +94,9 @@ static const struct drm_bridge_funcs dp_bridge_ops = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state   = drm_atomic_helper_bridge_destroy_state,
 	.atomic_reset           = drm_atomic_helper_bridge_reset,
-	.enable       = dp_bridge_enable,
-	.disable      = dp_bridge_disable,
-	.post_disable = dp_bridge_post_disable,
+	.atomic_enable          = dp_bridge_atomic_enable,
+	.atomic_disable         = dp_bridge_atomic_disable,
+	.atomic_post_disable    = dp_bridge_atomic_post_disable,
 	.mode_set     = dp_bridge_mode_set,
 	.mode_valid   = dp_bridge_mode_valid,
 	.get_modes    = dp_bridge_get_modes,
@@ -107,6 +107,171 @@ static const struct drm_bridge_funcs dp_bridge_ops = {
 	.hpd_notify   = dp_bridge_hpd_notify,
 };
 
+static int edp_bridge_atomic_check(struct drm_bridge *drm_bridge,
+				   struct drm_bridge_state *bridge_state,
+				   struct drm_crtc_state *crtc_state,
+				   struct drm_connector_state *conn_state)
+{
+	struct msm_dp *dp = to_dp_bridge(drm_bridge)->dp_display;
+
+	if (WARN_ON(!conn_state))
+		return -ENODEV;
+
+	conn_state->self_refresh_aware = dp->psr_supported;
+
+	if (!conn_state->crtc || !crtc_state)
+		return 0;
+
+	if (crtc_state->self_refresh_active && !dp->psr_supported)
+		return -EINVAL;
+
+	return 0;
+}
+
+static void edp_bridge_atomic_enable(struct drm_bridge *drm_bridge,
+				     struct drm_bridge_state *old_bridge_state)
+{
+	struct drm_atomic_state *atomic_state = old_bridge_state->base.state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state;
+	struct msm_dp_bridge *dp_bridge = to_dp_bridge(drm_bridge);
+	struct msm_dp *dp = dp_bridge->dp_display;
+
+	/*
+	 * Check the old state of the crtc to determine if the panel
+	 * was put into psr state previously by the edp_bridge_atomic_disable.
+	 * If the panel is in psr, just exit psr state and skip the full
+	 * bridge enable sequence.
+	 */
+	crtc = drm_atomic_get_new_crtc_for_encoder(atomic_state,
+						   drm_bridge->encoder);
+	if (!crtc)
+		return;
+
+	old_crtc_state = drm_atomic_get_old_crtc_state(atomic_state, crtc);
+
+	if (old_crtc_state && old_crtc_state->self_refresh_active) {
+		dp_display_set_psr(dp, false);
+		return;
+	}
+
+	dp_bridge_atomic_enable(drm_bridge, old_bridge_state);
+}
+
+static void edp_bridge_atomic_disable(struct drm_bridge *drm_bridge,
+				      struct drm_bridge_state *old_bridge_state)
+{
+	struct drm_atomic_state *atomic_state = old_bridge_state->base.state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state = NULL, *old_crtc_state = NULL;
+	struct msm_dp_bridge *dp_bridge = to_dp_bridge(drm_bridge);
+	struct msm_dp *dp = dp_bridge->dp_display;
+
+	crtc = drm_atomic_get_old_crtc_for_encoder(atomic_state,
+						   drm_bridge->encoder);
+	if (!crtc)
+		goto out;
+
+	new_crtc_state = drm_atomic_get_new_crtc_state(atomic_state, crtc);
+	if (!new_crtc_state)
+		goto out;
+
+	old_crtc_state = drm_atomic_get_old_crtc_state(atomic_state, crtc);
+	if (!old_crtc_state)
+		goto out;
+
+	/*
+	 * Set self refresh mode if current crtc state is active.
+	 *
+	 * If old crtc state is active, then this is a display disable
+	 * call while the sink is in psr state. So, exit psr here.
+	 * The eDP controller will be disabled in the
+	 * edp_bridge_atomic_post_disable function.
+	 *
+	 * We observed sink is stuck in self refresh if psr exit is skipped
+	 * when display disable occurs while the sink is in psr state.
+	 */
+	if (new_crtc_state->self_refresh_active) {
+		dp_display_set_psr(dp, true);
+		return;
+	} else if (old_crtc_state->self_refresh_active) {
+		dp_display_set_psr(dp, false);
+		return;
+	}
+
+out:
+	dp_bridge_atomic_disable(drm_bridge, old_bridge_state);
+}
+
+static void edp_bridge_atomic_post_disable(struct drm_bridge *drm_bridge,
+				struct drm_bridge_state *old_bridge_state)
+{
+	struct drm_atomic_state *atomic_state = old_bridge_state->base.state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state = NULL;
+
+	crtc = drm_atomic_get_old_crtc_for_encoder(atomic_state,
+						   drm_bridge->encoder);
+	if (!crtc)
+		return;
+
+	new_crtc_state = drm_atomic_get_new_crtc_state(atomic_state, crtc);
+	if (!new_crtc_state)
+		return;
+
+	/*
+	 * Self refresh mode is already set in edp_bridge_atomic_disable.
+	 */
+	if (new_crtc_state->self_refresh_active)
+		return;
+
+	dp_bridge_atomic_post_disable(drm_bridge, old_bridge_state);
+}
+
+/**
+ * edp_bridge_mode_valid - callback to determine if specified mode is valid
+ * @bridge: Pointer to drm bridge structure
+ * @info: display info
+ * @mode: Pointer to drm mode structure
+ * Returns: Validity status for specified mode
+ */
+static enum drm_mode_status edp_bridge_mode_valid(struct drm_bridge *bridge,
+					  const struct drm_display_info *info,
+					  const struct drm_display_mode *mode)
+{
+	struct msm_dp *dp;
+	int mode_pclk_khz = mode->clock;
+
+	dp = to_dp_bridge(bridge)->dp_display;
+
+	if (!dp || !mode_pclk_khz || !dp->connector) {
+		DRM_ERROR("invalid params\n");
+		return -EINVAL;
+	}
+
+	if (mode->clock > DP_MAX_PIXEL_CLK_KHZ)
+		return MODE_CLOCK_HIGH;
+
+	/*
+	 * The eDP controller currently does not have a reliable way of
+	 * enabling panel power to read sink capabilities. So, we rely
+	 * on the panel driver to populate only supported modes for now.
+	 */
+	return MODE_OK;
+}
+
+static const struct drm_bridge_funcs edp_bridge_ops = {
+	.atomic_enable = edp_bridge_atomic_enable,
+	.atomic_disable = edp_bridge_atomic_disable,
+	.atomic_post_disable = edp_bridge_atomic_post_disable,
+	.mode_set = dp_bridge_mode_set,
+	.mode_valid = edp_bridge_mode_valid,
+	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_check = edp_bridge_atomic_check,
+};
+
 struct drm_bridge *dp_bridge_init(struct msm_dp *dp_display, struct drm_device *dev,
 			struct drm_encoder *encoder)
 {
@@ -121,7 +286,7 @@ struct drm_bridge *dp_bridge_init(struct msm_dp *dp_display, struct drm_device *
 	dp_bridge->dp_display = dp_display;
 
 	bridge = &dp_bridge->bridge;
-	bridge->funcs = &dp_bridge_ops;
+	bridge->funcs = dp_display->is_edp ? &edp_bridge_ops : &dp_bridge_ops;
 	bridge->type = dp_display->connector_type;
 
 	/*
diff --git a/drivers/gpu/drm/msm/dp/dp_drm.h b/drivers/gpu/drm/msm/dp/dp_drm.h
index 250f7c66201f..afe79b85e183 100644
--- a/drivers/gpu/drm/msm/dp/dp_drm.h
+++ b/drivers/gpu/drm/msm/dp/dp_drm.h
@@ -23,9 +23,12 @@ struct drm_connector *dp_drm_connector_init(struct msm_dp *dp_display, struct dr
 struct drm_bridge *dp_bridge_init(struct msm_dp *dp_display, struct drm_device *dev,
 			struct drm_encoder *encoder);
 
-void dp_bridge_enable(struct drm_bridge *drm_bridge);
-void dp_bridge_disable(struct drm_bridge *drm_bridge);
-void dp_bridge_post_disable(struct drm_bridge *drm_bridge);
+void dp_bridge_atomic_enable(struct drm_bridge *drm_bridge,
+			     struct drm_bridge_state *old_bridge_state);
+void dp_bridge_atomic_disable(struct drm_bridge *drm_bridge,
+			      struct drm_bridge_state *old_bridge_state);
+void dp_bridge_atomic_post_disable(struct drm_bridge *drm_bridge,
+				   struct drm_bridge_state *old_bridge_state);
 enum drm_mode_status dp_bridge_mode_valid(struct drm_bridge *bridge,
 					  const struct drm_display_info *info,
 					  const struct drm_display_mode *mode);
diff --git a/drivers/gpu/drm/msm/dp/dp_link.c b/drivers/gpu/drm/msm/dp/dp_link.c
index f1f1d646539d..42427129acea 100644
--- a/drivers/gpu/drm/msm/dp/dp_link.c
+++ b/drivers/gpu/drm/msm/dp/dp_link.c
@@ -937,6 +937,38 @@ static int dp_link_process_phy_test_pattern_request(
 	return 0;
 }
 
+static bool dp_link_read_psr_error_status(struct dp_link_private *link)
+{
+	u8 status;
+
+	drm_dp_dpcd_read(link->aux, DP_PSR_ERROR_STATUS, &status, 1);
+
+	if (status & DP_PSR_LINK_CRC_ERROR)
+		DRM_ERROR("PSR LINK CRC ERROR\n");
+	else if (status & DP_PSR_RFB_STORAGE_ERROR)
+		DRM_ERROR("PSR RFB STORAGE ERROR\n");
+	else if (status & DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR)
+		DRM_ERROR("PSR VSC SDP UNCORRECTABLE ERROR\n");
+	else
+		return false;
+
+	return true;
+}
+
+static bool dp_link_psr_capability_changed(struct dp_link_private *link)
+{
+	u8 status;
+
+	drm_dp_dpcd_read(link->aux, DP_PSR_ESI, &status, 1);
+
+	if (status & DP_PSR_CAPS_CHANGE) {
+		drm_dbg_dp(link->drm_dev, "PSR Capability Change\n");
+		return true;
+	}
+
+	return false;
+}
+
 static u8 get_link_status(const u8 link_status[DP_LINK_STATUS_SIZE], int r)
 {
 	return link_status[r - DP_LANE0_1_STATUS];
@@ -1055,6 +1087,10 @@ int dp_link_process_request(struct dp_link *dp_link)
 		dp_link->sink_request |= DP_TEST_LINK_TRAINING;
 	} else if (!dp_link_process_phy_test_pattern_request(link)) {
 		dp_link->sink_request |= DP_TEST_LINK_PHY_TEST_PATTERN;
+	} else if (dp_link_read_psr_error_status(link)) {
+		DRM_ERROR("PSR IRQ_HPD received\n");
+	} else if (dp_link_psr_capability_changed(link)) {
+		drm_dbg_dp(link->drm_dev, "PSR Capability changed");
 	} else {
 		ret = dp_link_process_link_status_update(link);
 		if (!ret) {
diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c
index 1800d8963f8a..42d52510ffd4 100644
--- a/drivers/gpu/drm/msm/dp/dp_panel.c
+++ b/drivers/gpu/drm/msm/dp/dp_panel.c
@@ -20,6 +20,27 @@ struct dp_panel_private {
 	bool aux_cfg_update_done;
 };
 
+static void dp_panel_read_psr_cap(struct dp_panel_private *panel)
+{
+	ssize_t rlen;
+	struct dp_panel *dp_panel;
+
+	dp_panel = &panel->dp_panel;
+
+	/* edp sink */
+	if (dp_panel->dpcd[DP_EDP_CONFIGURATION_CAP]) {
+		rlen = drm_dp_dpcd_read(panel->aux, DP_PSR_SUPPORT,
+				&dp_panel->psr_cap, sizeof(dp_panel->psr_cap));
+		if (rlen == sizeof(dp_panel->psr_cap)) {
+			drm_dbg_dp(panel->drm_dev,
+				"psr version: 0x%x, psr_cap: 0x%x\n",
+				dp_panel->psr_cap.version,
+				dp_panel->psr_cap.capabilities);
+		} else
+			DRM_ERROR("failed to read psr info, rlen=%zd\n", rlen);
+	}
+}
+
 static int dp_panel_read_dpcd(struct dp_panel *dp_panel)
 {
 	int rc = 0;
@@ -107,6 +128,7 @@ static int dp_panel_read_dpcd(struct dp_panel *dp_panel)
 		}
 	}
 
+	dp_panel_read_psr_cap(panel);
 end:
 	return rc;
 }
diff --git a/drivers/gpu/drm/msm/dp/dp_panel.h b/drivers/gpu/drm/msm/dp/dp_panel.h
index f04d0210b5cd..45208b45eb53 100644
--- a/drivers/gpu/drm/msm/dp/dp_panel.h
+++ b/drivers/gpu/drm/msm/dp/dp_panel.h
@@ -34,6 +34,11 @@ struct dp_panel_in {
 	struct dp_catalog *catalog;
 };
 
+struct dp_panel_psr {
+	u8 version;
+	u8 capabilities;
+};
+
 struct dp_panel {
 	/* dpcd raw data */
 	u8 dpcd[DP_RECEIVER_CAP_SIZE + 1];
@@ -46,6 +51,7 @@ struct dp_panel {
 	struct edid *edid;
 	struct drm_connector *connector;
 	struct dp_display_mode dp_mode;
+	struct dp_panel_psr psr_cap;
 	bool video_test;
 
 	u32 vic;
diff --git a/drivers/gpu/drm/msm/dp/dp_reg.h b/drivers/gpu/drm/msm/dp/dp_reg.h
index 268602803d9a..ea85a691e72b 100644
--- a/drivers/gpu/drm/msm/dp/dp_reg.h
+++ b/drivers/gpu/drm/msm/dp/dp_reg.h
@@ -22,6 +22,20 @@
 #define REG_DP_INTR_STATUS2			(0x00000024)
 #define REG_DP_INTR_STATUS3			(0x00000028)
 
+#define REG_DP_INTR_STATUS4			(0x0000002C)
+#define PSR_UPDATE_INT				(0x00000001)
+#define PSR_CAPTURE_INT				(0x00000004)
+#define PSR_EXIT_INT				(0x00000010)
+#define PSR_UPDATE_ERROR_INT			(0x00000040)
+#define PSR_WAKE_ERROR_INT			(0x00000100)
+
+#define REG_DP_INTR_MASK4			(0x00000030)
+#define PSR_UPDATE_MASK				(0x00000001)
+#define PSR_CAPTURE_MASK			(0x00000002)
+#define PSR_EXIT_MASK				(0x00000004)
+#define PSR_UPDATE_ERROR_MASK			(0x00000008)
+#define PSR_WAKE_ERROR_MASK			(0x00000010)
+
 #define REG_DP_DP_HPD_CTRL			(0x00000000)
 #define DP_DP_HPD_CTRL_HPD_EN			(0x00000001)
 
@@ -164,6 +178,16 @@
 #define MMSS_DP_AUDIO_TIMING_RBR_48		(0x00000094)
 #define MMSS_DP_AUDIO_TIMING_HBR_48		(0x00000098)
 
+#define REG_PSR_CONFIG				(0x00000100)
+#define DISABLE_PSR				(0x00000000)
+#define PSR1_SUPPORTED				(0x00000001)
+#define PSR2_WITHOUT_FRAMESYNC			(0x00000002)
+#define PSR2_WITH_FRAMESYNC			(0x00000003)
+
+#define REG_PSR_CMD				(0x00000110)
+#define PSR_ENTER				(0x00000001)
+#define PSR_EXIT				(0x00000002)
+
 #define MMSS_DP_PSR_CRC_RG			(0x00000154)
 #define MMSS_DP_PSR_CRC_B			(0x00000158)
 
@@ -184,6 +208,9 @@
 #define MMSS_DP_AUDIO_STREAM_0			(0x00000240)
 #define MMSS_DP_AUDIO_STREAM_1			(0x00000244)
 
+#define MMSS_DP_SDP_CFG3			(0x0000024c)
+#define UPDATE_SDP				(0x00000001)
+
 #define MMSS_DP_EXTENSION_0			(0x00000250)
 #define MMSS_DP_EXTENSION_1			(0x00000254)
 #define MMSS_DP_EXTENSION_2			(0x00000258)
diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c
index 31fdee2052be..baab79ab6e74 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.c
+++ b/drivers/gpu/drm/msm/dsi/dsi.c
@@ -4,7 +4,6 @@
  */
 
 #include "dsi.h"
-#include "dsi_cfg.h"
 
 bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi)
 {
@@ -173,8 +172,10 @@ static int dsi_dev_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id dt_match[] = {
-	{ .compatible = "qcom,mdss-dsi-ctrl", .data = NULL /* autodetect cfg */ },
-	{ .compatible = "qcom,dsi-ctrl-6g-qcm2290", .data = &qcm2290_dsi_cfg_handler },
+	{ .compatible = "qcom,mdss-dsi-ctrl" },
+
+	/* Deprecated, don't use */
+	{ .compatible = "qcom,dsi-ctrl-6g-qcm2290" },
 	{}
 };
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi.xml.h b/drivers/gpu/drm/msm/dsi/dsi.xml.h
index d1b2a17b0a66..a4a154601114 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.xml.h
+++ b/drivers/gpu/drm/msm/dsi/dsi.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -785,4 +785,5 @@ static inline uint32_t DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM0_SLICE_WIDTH(ui
 	return ((val) << DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM0_SLICE_WIDTH__SHIFT) & DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM0_SLICE_WIDTH__MASK;
 }
 
+
 #endif /* DSI_XML */
diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.c b/drivers/gpu/drm/msm/dsi/dsi_cfg.c
index 6d21f0b33411..29ccd755cc2e 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_cfg.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.c
@@ -21,8 +21,9 @@ static const struct msm_dsi_config apq8064_dsi_cfg = {
 	.num_regulators = ARRAY_SIZE(apq8064_dsi_regulators),
 	.bus_clk_names = dsi_v2_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_v2_bus_clk_names),
-	.io_start = { 0x4700000, 0x5800000 },
-	.num_dsi = 2,
+	.io_start = {
+		{ 0x4700000, 0x5800000 },
+	},
 };
 
 static const char * const dsi_6g_bus_clk_names[] = {
@@ -41,46 +42,40 @@ static const struct msm_dsi_config msm8974_apq8084_dsi_cfg = {
 	.num_regulators = ARRAY_SIZE(msm8974_apq8084_regulators),
 	.bus_clk_names = dsi_6g_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names),
-	.io_start = { 0xfd922800, 0xfd922b00 },
-	.num_dsi = 2,
+	.io_start = {
+		{ 0xfd922800, 0xfd922b00 },
+	},
 };
 
-static const char * const dsi_8916_bus_clk_names[] = {
+static const char * const dsi_v1_3_1_clk_names[] = {
 	"mdp_core", "iface", "bus",
 };
 
-static const struct regulator_bulk_data msm8916_dsi_regulators[] = {
+static const struct regulator_bulk_data dsi_v1_3_1_regulators[] = {
 	{ .supply = "vdda", .init_load_uA = 100000 },	/* 1.2 V */
 	{ .supply = "vddio", .init_load_uA = 100000 },	/* 1.8 V */
 };
 
 static const struct msm_dsi_config msm8916_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.regulator_data = msm8916_dsi_regulators,
-	.num_regulators = ARRAY_SIZE(msm8916_dsi_regulators),
-	.bus_clk_names = dsi_8916_bus_clk_names,
-	.num_bus_clks = ARRAY_SIZE(dsi_8916_bus_clk_names),
-	.io_start = { 0x1a98000 },
-	.num_dsi = 1,
-};
-
-static const char * const dsi_8976_bus_clk_names[] = {
-	"mdp_core", "iface", "bus",
-};
-
-static const struct regulator_bulk_data msm8976_dsi_regulators[] = {
-	{ .supply = "vdda", .init_load_uA = 100000 },	/* 1.2 V */
-	{ .supply = "vddio", .init_load_uA = 100000 },	/* 1.8 V */
+	.regulator_data = dsi_v1_3_1_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_v1_3_1_regulators),
+	.bus_clk_names = dsi_v1_3_1_clk_names,
+	.num_bus_clks = ARRAY_SIZE(dsi_v1_3_1_clk_names),
+	.io_start = {
+		{ 0x1a98000 },
+	},
 };
 
 static const struct msm_dsi_config msm8976_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.regulator_data = msm8976_dsi_regulators,
-	.num_regulators = ARRAY_SIZE(msm8976_dsi_regulators),
-	.bus_clk_names = dsi_8976_bus_clk_names,
-	.num_bus_clks = ARRAY_SIZE(dsi_8976_bus_clk_names),
-	.io_start = { 0x1a94000, 0x1a96000 },
-	.num_dsi = 2,
+	.regulator_data = dsi_v1_3_1_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_v1_3_1_regulators),
+	.bus_clk_names = dsi_v1_3_1_clk_names,
+	.num_bus_clks = ARRAY_SIZE(dsi_v1_3_1_clk_names),
+	.io_start = {
+		{ 0x1a94000, 0x1a96000 },
+	},
 };
 
 static const struct regulator_bulk_data msm8994_dsi_regulators[] = {
@@ -98,12 +93,9 @@ static const struct msm_dsi_config msm8994_dsi_cfg = {
 	.num_regulators = ARRAY_SIZE(msm8994_dsi_regulators),
 	.bus_clk_names = dsi_6g_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names),
-	.io_start = { 0xfd998000, 0xfd9a0000 },
-	.num_dsi = 2,
-};
-
-static const char * const dsi_8996_bus_clk_names[] = {
-	"mdp_core", "iface", "bus", "core_mmss",
+	.io_start = {
+		{ 0xfd998000, 0xfd9a0000 },
+	},
 };
 
 static const struct regulator_bulk_data msm8996_dsi_regulators[] = {
@@ -116,10 +108,11 @@ static const struct msm_dsi_config msm8996_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
 	.regulator_data = msm8996_dsi_regulators,
 	.num_regulators = ARRAY_SIZE(msm8996_dsi_regulators),
-	.bus_clk_names = dsi_8996_bus_clk_names,
-	.num_bus_clks = ARRAY_SIZE(dsi_8996_bus_clk_names),
-	.io_start = { 0x994000, 0x996000 },
-	.num_dsi = 2,
+	.bus_clk_names = dsi_6g_bus_clk_names,
+	.num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names),
+	.io_start = {
+		{ 0x994000, 0x996000 },
+	},
 };
 
 static const char * const dsi_msm8998_bus_clk_names[] = {
@@ -137,8 +130,9 @@ static const struct msm_dsi_config msm8998_dsi_cfg = {
 	.num_regulators = ARRAY_SIZE(msm8998_dsi_regulators),
 	.bus_clk_names = dsi_msm8998_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_msm8998_bus_clk_names),
-	.io_start = { 0xc994000, 0xc996000 },
-	.num_dsi = 2,
+	.io_start = {
+		{ 0xc994000, 0xc996000 },
+	},
 };
 
 static const char * const dsi_sdm660_bus_clk_names[] = {
@@ -155,30 +149,29 @@ static const struct msm_dsi_config sdm660_dsi_cfg = {
 	.num_regulators = ARRAY_SIZE(sdm660_dsi_regulators),
 	.bus_clk_names = dsi_sdm660_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_sdm660_bus_clk_names),
-	.io_start = { 0xc994000, 0xc996000 },
-	.num_dsi = 2,
+	.io_start = {
+		{ 0xc994000, 0xc996000 },
+	},
 };
 
-static const char * const dsi_sdm845_bus_clk_names[] = {
+static const char * const dsi_v2_4_clk_names[] = {
 	"iface", "bus",
 };
 
-static const char * const dsi_sc7180_bus_clk_names[] = {
-	"iface", "bus",
-};
-
-static const struct regulator_bulk_data sdm845_dsi_regulators[] = {
+static const struct regulator_bulk_data dsi_v2_4_regulators[] = {
 	{ .supply = "vdda", .init_load_uA = 21800 },	/* 1.2 V */
 };
 
 static const struct msm_dsi_config sdm845_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.regulator_data = sdm845_dsi_regulators,
-	.num_regulators = ARRAY_SIZE(sdm845_dsi_regulators),
-	.bus_clk_names = dsi_sdm845_bus_clk_names,
-	.num_bus_clks = ARRAY_SIZE(dsi_sdm845_bus_clk_names),
-	.io_start = { 0xae94000, 0xae96000 },
-	.num_dsi = 2,
+	.regulator_data = dsi_v2_4_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_v2_4_regulators),
+	.bus_clk_names = dsi_v2_4_clk_names,
+	.num_bus_clks = ARRAY_SIZE(dsi_v2_4_clk_names),
+	.io_start = {
+		{ 0xae94000, 0xae96000 }, /* SDM845 / SDM670 */
+		{ 0x5e94000 }, /* QCM2290 / SM6115 / SM6125 / SM6375 */
+	},
 };
 
 static const struct regulator_bulk_data sm8550_dsi_regulators[] = {
@@ -189,28 +182,11 @@ static const struct msm_dsi_config sm8550_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
 	.regulator_data = sm8550_dsi_regulators,
 	.num_regulators = ARRAY_SIZE(sm8550_dsi_regulators),
-	.bus_clk_names = dsi_sdm845_bus_clk_names,
-	.num_bus_clks = ARRAY_SIZE(dsi_sdm845_bus_clk_names),
-	.io_start = { 0xae94000, 0xae96000 },
-	.num_dsi = 2,
-};
-
-static const struct regulator_bulk_data sc7180_dsi_regulators[] = {
-	{ .supply = "vdda", .init_load_uA = 21800 },	/* 1.2 V */
-};
-
-static const struct msm_dsi_config sc7180_dsi_cfg = {
-	.io_offset = DSI_6G_REG_SHIFT,
-	.regulator_data = sc7180_dsi_regulators,
-	.num_regulators = ARRAY_SIZE(sc7180_dsi_regulators),
-	.bus_clk_names = dsi_sc7180_bus_clk_names,
-	.num_bus_clks = ARRAY_SIZE(dsi_sc7180_bus_clk_names),
-	.io_start = { 0xae94000 },
-	.num_dsi = 1,
-};
-
-static const char * const dsi_sc7280_bus_clk_names[] = {
-	"iface", "bus",
+	.bus_clk_names = dsi_v2_4_clk_names,
+	.num_bus_clks = ARRAY_SIZE(dsi_v2_4_clk_names),
+	.io_start = {
+		{ 0xae94000, 0xae96000 },
+	},
 };
 
 static const struct regulator_bulk_data sc7280_dsi_regulators[] = {
@@ -221,28 +197,11 @@ static const struct msm_dsi_config sc7280_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
 	.regulator_data = sc7280_dsi_regulators,
 	.num_regulators = ARRAY_SIZE(sc7280_dsi_regulators),
-	.bus_clk_names = dsi_sc7280_bus_clk_names,
-	.num_bus_clks = ARRAY_SIZE(dsi_sc7280_bus_clk_names),
-	.io_start = { 0xae94000, 0xae96000 },
-	.num_dsi = 2,
-};
-
-static const char * const dsi_qcm2290_bus_clk_names[] = {
-	"iface", "bus",
-};
-
-static const struct regulator_bulk_data qcm2290_dsi_cfg_regulators[] = {
-	{ .supply = "vdda", .init_load_uA = 21800 },	/* 1.2 V */
-};
-
-static const struct msm_dsi_config qcm2290_dsi_cfg = {
-	.io_offset = DSI_6G_REG_SHIFT,
-	.regulator_data = qcm2290_dsi_cfg_regulators,
-	.num_regulators = ARRAY_SIZE(qcm2290_dsi_cfg_regulators),
-	.bus_clk_names = dsi_qcm2290_bus_clk_names,
-	.num_bus_clks = ARRAY_SIZE(dsi_qcm2290_bus_clk_names),
-	.io_start = { 0x5e94000 },
-	.num_dsi = 1,
+	.bus_clk_names = dsi_v2_4_clk_names,
+	.num_bus_clks = ARRAY_SIZE(dsi_v2_4_clk_names),
+	.io_start = {
+		{ 0xae94000, 0xae96000 },
+	},
 };
 
 static const struct msm_dsi_host_cfg_ops msm_dsi_v2_host_ops = {
@@ -311,7 +270,7 @@ static const struct msm_dsi_cfg_handler dsi_cfg_handlers[] = {
 	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_4_0,
 		&sdm845_dsi_cfg, &msm_dsi_6g_v2_host_ops},
 	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_4_1,
-		&sc7180_dsi_cfg, &msm_dsi_6g_v2_host_ops},
+		&sdm845_dsi_cfg, &msm_dsi_6g_v2_host_ops},
 	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_5_0,
 		&sc7280_dsi_cfg, &msm_dsi_6g_v2_host_ops},
 	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_6_0,
@@ -335,9 +294,3 @@ const struct msm_dsi_cfg_handler *msm_dsi_cfg_get(u32 major, u32 minor)
 
 	return cfg_hnd;
 }
-
-/*  Non autodetect configs */
-const struct msm_dsi_cfg_handler qcm2290_dsi_cfg_handler = {
-	.cfg = &qcm2290_dsi_cfg,
-	.ops = &msm_dsi_6g_v2_host_ops,
-};
diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.h b/drivers/gpu/drm/msm/dsi/dsi_cfg.h
index 44be4a88aa83..91bdaf50bb1a 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_cfg.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.h
@@ -32,14 +32,16 @@
 
 #define DSI_6G_REG_SHIFT	4
 
+/* Maximum number of configurations matched against the same hw revision */
+#define VARIANTS_MAX			2
+
 struct msm_dsi_config {
 	u32 io_offset;
 	const struct regulator_bulk_data *regulator_data;
 	int num_regulators;
 	const char * const *bus_clk_names;
 	const int num_bus_clks;
-	const resource_size_t io_start[DSI_MAX];
-	const int num_dsi;
+	const resource_size_t io_start[VARIANTS_MAX][DSI_MAX];
 };
 
 struct msm_dsi_host_cfg_ops {
@@ -63,8 +65,5 @@ struct msm_dsi_cfg_handler {
 
 const struct msm_dsi_cfg_handler *msm_dsi_cfg_get(u32 major, u32 minor);
 
-/* Non autodetect configs */
-extern const struct msm_dsi_cfg_handler qcm2290_dsi_cfg_handler;
-
 #endif /* __MSM_DSI_CFG_H__ */
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index 18fa30e1e858..961689a255c4 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -214,10 +214,6 @@ static const struct msm_dsi_cfg_handler *dsi_get_config(
 	int ret;
 	u32 major = 0, minor = 0;
 
-	cfg_hnd = device_get_match_data(dev);
-	if (cfg_hnd)
-		return cfg_hnd;
-
 	ahb_clk = msm_clk_get(msm_host->pdev, "iface");
 	if (IS_ERR(ahb_clk)) {
 		pr_err("%s: cannot get interface clock\n", __func__);
@@ -1862,16 +1858,16 @@ static int dsi_host_get_id(struct msm_dsi_host *msm_host)
 	struct platform_device *pdev = msm_host->pdev;
 	const struct msm_dsi_config *cfg = msm_host->cfg_hnd->cfg;
 	struct resource *res;
-	int i;
+	int i, j;
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dsi_ctrl");
 	if (!res)
 		return -EINVAL;
 
-	for (i = 0; i < cfg->num_dsi; i++) {
-		if (cfg->io_start[i] == res->start)
-			return i;
-	}
+	for (i = 0; i < VARIANTS_MAX; i++)
+		for (j = 0; j < DSI_MAX; j++)
+			if (cfg->io_start[i][j] == res->start)
+				return j;
 
 	return -EINVAL;
 }
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_10nm.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_10nm.xml.h
index 8b1be69ccf89..a2ae8777e59e 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_10nm.xml.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_phy_10nm.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_14nm.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_14nm.xml.h
index 515f1fa605bf..24e2fdc0cde1 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_14nm.xml.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_phy_14nm.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_20nm.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_20nm.xml.h
index 81e4622eb358..6352541f37e9 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_20nm.xml.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_phy_20nm.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_28nm.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_28nm.xml.h
index 8c7db35c12c8..178bd4fd7893 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_28nm.xml.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_phy_28nm.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_28nm_8960.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_28nm_8960.xml.h
index 44eeca31a811..5f900bb53519 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_28nm_8960.xml.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_phy_28nm_8960.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_7nm.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_7nm.xml.h
index 5bc061797003..584cbd0205ef 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_7nm.xml.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_phy_7nm.xml.h
@@ -8,24 +8,24 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
 
 Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
diff --git a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
index 03bc322d0487..7062f7164216 100644
--- a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
+++ b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/dsi/sfpb.xml.h b/drivers/gpu/drm/msm/dsi/sfpb.xml.h
index 2ae711cbec36..344a1a1620cd 100644
--- a/drivers/gpu/drm/msm/dsi/sfpb.xml.h
+++ b/drivers/gpu/drm/msm/dsi/sfpb.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
 
-Copyright (C) 2013-2021 by the following authors:
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
index c9eb26df67c0..973b460486a5 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -776,10 +776,28 @@ static inline uint32_t HDMI_8x60_PHY_REG1_OUTVOL_SWING_CTRL(uint32_t val)
 
 #define REG_HDMI_8x74_ANA_CFG1					0x00000004
 
+#define REG_HDMI_8x74_ANA_CFG2					0x00000008
+
+#define REG_HDMI_8x74_ANA_CFG3					0x0000000c
+
 #define REG_HDMI_8x74_PD_CTRL0					0x00000010
 
 #define REG_HDMI_8x74_PD_CTRL1					0x00000014
 
+#define REG_HDMI_8x74_GLB_CFG					0x00000018
+
+#define REG_HDMI_8x74_DCC_CFG0					0x0000001c
+
+#define REG_HDMI_8x74_DCC_CFG1					0x00000020
+
+#define REG_HDMI_8x74_TXCAL_CFG0				0x00000024
+
+#define REG_HDMI_8x74_TXCAL_CFG1				0x00000028
+
+#define REG_HDMI_8x74_TXCAL_CFG2				0x0000002c
+
+#define REG_HDMI_8x74_TXCAL_CFG3				0x00000030
+
 #define REG_HDMI_8x74_BIST_CFG0					0x00000034
 
 #define REG_HDMI_8x74_BIST_PATN0				0x0000003c
@@ -790,6 +808,8 @@ static inline uint32_t HDMI_8x60_PHY_REG1_OUTVOL_SWING_CTRL(uint32_t val)
 
 #define REG_HDMI_8x74_BIST_PATN3				0x00000048
 
+#define REG_HDMI_8x74_STATUS					0x0000005c
+
 #define REG_HDMI_28nm_PHY_PLL_REFCLK_CFG			0x00000000
 
 #define REG_HDMI_28nm_PHY_PLL_POSTDIV1_CFG			0x00000004
@@ -877,6 +897,8 @@ static inline uint32_t HDMI_8x60_PHY_REG1_OUTVOL_SWING_CTRL(uint32_t val)
 
 #define REG_HDMI_28nm_PHY_PLL_DEBUG_BUS_SEL			0x000000a0
 
+#define REG_HDMI_28nm_PHY_PLL_STATUS				0x000000c0
+
 #define REG_HDMI_8996_PHY_CFG					0x00000000
 
 #define REG_HDMI_8996_PHY_PD_CTL				0x00000004
diff --git a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
index 3edc698c4df5..498801526695 100644
--- a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
+++ b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
@@ -8,26 +8,26 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/tmp/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2020-12-31 19:26:32)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi.xml               (  17560 bytes, from 2021-09-16 22:37:02)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2021-07-22 15:21:56)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-03 01:18:13)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  41874 bytes, from 2021-01-30 18:25:22)
-- /home/robclark/tmp/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2021-01-30 18:25:22)
-
-Copyright (C) 2013-2021 by the following authors:
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
+- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
+
+Copyright (C) 2013-2022 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/msm_atomic.c b/drivers/gpu/drm/msm/msm_atomic.c
index 1686fbb611fd..d77fa9793c54 100644
--- a/drivers/gpu/drm/msm/msm_atomic.c
+++ b/drivers/gpu/drm/msm/msm_atomic.c
@@ -179,6 +179,24 @@ static unsigned get_crtc_mask(struct drm_atomic_state *state)
 	return mask;
 }
 
+int msm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
+{
+	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
+	struct drm_crtc *crtc;
+	int i;
+
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
+				      new_crtc_state, i) {
+		if ((old_crtc_state->ctm && !new_crtc_state->ctm) ||
+		    (!old_crtc_state->ctm && new_crtc_state->ctm)) {
+			new_crtc_state->mode_changed = true;
+			state->allow_modeset = true;
+		}
+	}
+
+	return drm_atomic_helper_check(dev, state);
+}
+
 void msm_atomic_commit_tail(struct drm_atomic_state *state)
 {
 	struct drm_device *dev = state->dev;
@@ -186,8 +204,7 @@ void msm_atomic_commit_tail(struct drm_atomic_state *state)
 	struct msm_kms *kms = priv->kms;
 	struct drm_crtc *async_crtc = NULL;
 	unsigned crtc_mask = get_crtc_mask(state);
-	bool async = kms->funcs->vsync_time &&
-			can_do_async(state, &async_crtc);
+	bool async = can_do_async(state, &async_crtc);
 
 	trace_msm_atomic_commit_tail_start(async, crtc_mask);
 
@@ -206,7 +223,8 @@ void msm_atomic_commit_tail(struct drm_atomic_state *state)
 	 * Now that there is no in-progress flush, prepare the
 	 * current update:
 	 */
-	kms->funcs->prepare_commit(kms, state);
+	if (kms->funcs->prepare_commit)
+		kms->funcs->prepare_commit(kms, state);
 
 	/*
 	 * Push atomic updates down to hardware:
@@ -231,7 +249,9 @@ void msm_atomic_commit_tail(struct drm_atomic_state *state)
 
 			kms->pending_crtc_mask |= crtc_mask;
 
-			vsync_time = kms->funcs->vsync_time(kms, async_crtc);
+			if (drm_crtc_next_vblank_start(async_crtc, &vsync_time))
+				goto fallback;
+
 			wakeup_time = ktime_sub(vsync_time, ms_to_ktime(1));
 
 			msm_hrtimer_queue_work(&timer->work, wakeup_time,
@@ -253,6 +273,7 @@ void msm_atomic_commit_tail(struct drm_atomic_state *state)
 		return;
 	}
 
+fallback:
 	/*
 	 * If there is any async flush pending on updated crtcs, fold
 	 * them into the current flush.
diff --git a/drivers/gpu/drm/msm/msm_debugfs.c b/drivers/gpu/drm/msm/msm_debugfs.c
index d6ecff0ab618..9c0e633a3a61 100644
--- a/drivers/gpu/drm/msm/msm_debugfs.c
+++ b/drivers/gpu/drm/msm/msm_debugfs.c
@@ -10,6 +10,7 @@
 #include <linux/fault-inject.h>
 
 #include <drm/drm_debugfs.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_file.h>
 #include <drm/drm_framebuffer.h>
 
@@ -241,12 +242,11 @@ static int msm_fb_show(struct seq_file *m, void *arg)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
-	struct msm_drm_private *priv = dev->dev_private;
 	struct drm_framebuffer *fb, *fbdev_fb = NULL;
 
-	if (priv->fbdev) {
+	if (dev->fb_helper && dev->fb_helper->fb) {
 		seq_printf(m, "fbcon ");
-		fbdev_fb = priv->fbdev->fb;
+		fbdev_fb = dev->fb_helper->fb;
 		msm_framebuffer_describe(fbdev_fb, m);
 	}
 
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index aca48c868c14..b4cfa44a8a5c 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -8,10 +8,12 @@
 #include <linux/dma-mapping.h>
 #include <linux/fault-inject.h>
 #include <linux/kthread.h>
+#include <linux/of_address.h>
 #include <linux/sched/mm.h>
 #include <linux/uaccess.h>
 #include <uapi/linux/sched/types.h>
 
+#include <drm/drm_aperture.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_file.h>
@@ -46,15 +48,17 @@
  * - 1.8.0 - Add MSM_BO_CACHED_COHERENT for supported GPUs (a6xx)
  * - 1.9.0 - Add MSM_SUBMIT_FENCE_SN_IN
  * - 1.10.0 - Add MSM_SUBMIT_BO_NO_IMPLICIT
+ * - 1.11.0 - Add wait boost (MSM_WAIT_FENCE_BOOST, MSM_PREP_BOOST)
  */
 #define MSM_VERSION_MAJOR	1
 #define MSM_VERSION_MINOR	10
 #define MSM_VERSION_PATCHLEVEL	0
 
+static void msm_deinit_vram(struct drm_device *ddev);
+
 static const struct drm_mode_config_funcs mode_config_funcs = {
 	.fb_create = msm_framebuffer_create,
-	.output_poll_changed = drm_fb_helper_output_poll_changed,
-	.atomic_check = drm_atomic_helper_check,
+	.atomic_check = msm_atomic_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
@@ -62,12 +66,6 @@ static const struct drm_mode_config_helper_funcs mode_config_helper_funcs = {
 	.atomic_commit_tail = msm_atomic_commit_tail,
 };
 
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-static bool fbdev = true;
-MODULE_PARM_DESC(fbdev, "Enable fbdev compat layer");
-module_param(fbdev, bool, 0600);
-#endif
-
 static char *vram = "16m";
 MODULE_PARM_DESC(vram, "Configure VRAM size (for devices without IOMMU/GPUMMU)");
 module_param(vram, charp, 0);
@@ -150,9 +148,6 @@ static void msm_irq_uninstall(struct drm_device *dev)
 	struct msm_drm_private *priv = dev->dev_private;
 	struct msm_kms *kms = priv->kms;
 
-	if (!priv->kms)
-		return;
-
 	kms->funcs->irq_uninstall(kms);
 	if (kms->irq_requested)
 		free_irq(kms->irq, dev);
@@ -240,12 +235,8 @@ static int msm_drm_uninit(struct device *dev)
 	msm_perf_debugfs_cleanup(priv);
 	msm_rd_debugfs_cleanup(priv);
 
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	if (fbdev && priv->fbdev)
-		msm_fbdev_free(ddev);
-#endif
-
-	msm_disp_snapshot_destroy(ddev);
+	if (kms)
+		msm_disp_snapshot_destroy(ddev);
 
 	drm_mode_config_cleanup(ddev);
 
@@ -253,30 +244,27 @@ static int msm_drm_uninit(struct device *dev)
 		drm_bridge_remove(priv->bridges[i]);
 	priv->num_bridges = 0;
 
-	pm_runtime_get_sync(dev);
-	msm_irq_uninstall(ddev);
-	pm_runtime_put_sync(dev);
+	if (kms) {
+		pm_runtime_get_sync(dev);
+		msm_irq_uninstall(ddev);
+		pm_runtime_put_sync(dev);
+	}
 
 	if (kms && kms->funcs)
 		kms->funcs->destroy(kms);
 
-	if (priv->vram.paddr) {
-		unsigned long attrs = DMA_ATTR_NO_KERNEL_MAPPING;
-		drm_mm_takedown(&priv->vram.mm);
-		dma_free_attrs(dev, priv->vram.size, NULL,
-			       priv->vram.paddr, attrs);
-	}
+	msm_deinit_vram(ddev);
 
 	component_unbind_all(dev, ddev);
 
 	ddev->dev_private = NULL;
+	drm_dev_put(ddev);
+
 	destroy_workqueue(priv->wq);
 
 	return 0;
 }
 
-#include <linux/of_address.h>
-
 struct msm_gem_address_space *msm_kms_init_aspace(struct drm_device *dev)
 {
 	struct msm_gem_address_space *aspace;
@@ -401,6 +389,19 @@ static int msm_init_vram(struct drm_device *dev)
 	return ret;
 }
 
+static void msm_deinit_vram(struct drm_device *ddev)
+{
+	struct msm_drm_private *priv = ddev->dev_private;
+	unsigned long attrs = DMA_ATTR_NO_KERNEL_MAPPING;
+
+	if (!priv->vram.paddr)
+		return;
+
+	drm_mm_takedown(&priv->vram.mm);
+	dma_free_attrs(ddev->dev, priv->vram.size, NULL, priv->vram.paddr,
+			attrs);
+}
+
 static int msm_drm_init(struct device *dev, const struct drm_driver *drv)
 {
 	struct msm_drm_private *priv = dev_get_drvdata(dev);
@@ -420,8 +421,10 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv)
 	priv->dev = ddev;
 
 	priv->wq = alloc_ordered_workqueue("msm", 0);
-	if (!priv->wq)
-		return -ENOMEM;
+	if (!priv->wq) {
+		ret = -ENOMEM;
+		goto err_put_dev;
+	}
 
 	INIT_LIST_HEAD(&priv->objects);
 	mutex_init(&priv->obj_lock);
@@ -444,12 +447,17 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv)
 
 	ret = msm_init_vram(ddev);
 	if (ret)
-		goto err_drm_dev_put;
+		goto err_cleanup_mode_config;
 
 	/* Bind all our sub-components: */
 	ret = component_bind_all(dev, ddev);
 	if (ret)
-		goto err_drm_dev_put;
+		goto err_deinit_vram;
+
+	/* the fw fb could be anywhere in memory */
+	ret = drm_aperture_remove_framebuffers(false, drv);
+	if (ret)
+		goto err_msm_uninit;
 
 	dma_set_max_seg_size(dev, UINT_MAX);
 
@@ -529,23 +537,30 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv)
 	}
 	drm_mode_config_reset(ddev);
 
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	if (kms && fbdev)
-		priv->fbdev = msm_fbdev_init(ddev);
-#endif
-
 	ret = msm_debugfs_late_init(ddev);
 	if (ret)
 		goto err_msm_uninit;
 
 	drm_kms_helper_poll_init(ddev);
 
+	if (kms)
+		msm_fbdev_setup(ddev);
+
 	return 0;
 
 err_msm_uninit:
 	msm_drm_uninit(dev);
-err_drm_dev_put:
+
+	return ret;
+
+err_deinit_vram:
+	msm_deinit_vram(ddev);
+err_cleanup_mode_config:
+	drm_mode_config_cleanup(ddev);
+	destroy_workqueue(priv->wq);
+err_put_dev:
 	drm_dev_put(ddev);
+
 	return ret;
 }
 
@@ -899,7 +914,7 @@ static int msm_ioctl_gem_info(struct drm_device *dev, void *data,
 }
 
 static int wait_fence(struct msm_gpu_submitqueue *queue, uint32_t fence_id,
-		      ktime_t timeout)
+		      ktime_t timeout, uint32_t flags)
 {
 	struct dma_fence *fence;
 	int ret;
@@ -918,17 +933,18 @@ static int wait_fence(struct msm_gpu_submitqueue *queue, uint32_t fence_id,
 	 * retired, so if the fence is not found it means there is nothing
 	 * to wait for
 	 */
-	ret = mutex_lock_interruptible(&queue->idr_lock);
-	if (ret)
-		return ret;
+	spin_lock(&queue->idr_lock);
 	fence = idr_find(&queue->fence_idr, fence_id);
 	if (fence)
 		fence = dma_fence_get_rcu(fence);
-	mutex_unlock(&queue->idr_lock);
+	spin_unlock(&queue->idr_lock);
 
 	if (!fence)
 		return 0;
 
+	if (flags & MSM_WAIT_FENCE_BOOST)
+		dma_fence_set_deadline(fence, ktime_get());
+
 	ret = dma_fence_wait_timeout(fence, true, timeout_to_jiffies(&timeout));
 	if (ret == 0) {
 		ret = -ETIMEDOUT;
@@ -949,8 +965,8 @@ static int msm_ioctl_wait_fence(struct drm_device *dev, void *data,
 	struct msm_gpu_submitqueue *queue;
 	int ret;
 
-	if (args->pad) {
-		DRM_ERROR("invalid pad: %08x\n", args->pad);
+	if (args->flags & ~MSM_WAIT_FENCE_FLAGS) {
+		DRM_ERROR("invalid flags: %08x\n", args->flags);
 		return -EINVAL;
 	}
 
@@ -961,7 +977,7 @@ static int msm_ioctl_wait_fence(struct drm_device *dev, void *data,
 	if (!queue)
 		return -ENOENT;
 
-	ret = wait_fence(queue, args->fence, to_ktime(args->timeout));
+	ret = wait_fence(queue, args->fence, to_ktime(args->timeout), args->flags);
 
 	msm_submitqueue_put(queue);
 
@@ -1068,7 +1084,6 @@ static const struct drm_driver msm_driver = {
 				DRIVER_SYNCOBJ,
 	.open               = msm_open,
 	.postclose           = msm_postclose,
-	.lastclose          = drm_fb_helper_lastclose,
 	.dumb_create        = msm_gem_dumb_create,
 	.dumb_map_offset    = msm_gem_dumb_map_offset,
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index 9f0c184b02a0..e13a8cbd61c9 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -29,7 +29,6 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_fb_helper.h>
 #include <drm/display/drm_dsc.h>
 #include <drm/msm_drm.h>
 #include <drm/drm_gem.h>
@@ -129,8 +128,6 @@ struct msm_drm_private {
 	bool is_a2xx;
 	bool has_cached_coherent;
 
-	struct drm_fb_helper *fbdev;
-
 	struct msm_rd_state *rd;       /* debugfs to dump all submits */
 	struct msm_rd_state *hangrd;   /* debugfs to dump hanging submits */
 	struct msm_perf_state *perf;
@@ -261,6 +258,7 @@ int msm_atomic_init_pending_timer(struct msm_pending_timer *timer,
 		struct msm_kms *kms, int crtc_idx);
 void msm_atomic_destroy_pending_timer(struct msm_pending_timer *timer);
 void msm_atomic_commit_tail(struct drm_atomic_state *state);
+int msm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state);
 struct drm_atomic_state *msm_atomic_state_alloc(struct drm_device *dev);
 void msm_atomic_state_clear(struct drm_atomic_state *state);
 void msm_atomic_state_free(struct drm_atomic_state *state);
@@ -306,8 +304,13 @@ struct drm_framebuffer *msm_framebuffer_create(struct drm_device *dev,
 struct drm_framebuffer * msm_alloc_stolen_fb(struct drm_device *dev,
 		int w, int h, int p, uint32_t format);
 
-struct drm_fb_helper *msm_fbdev_init(struct drm_device *dev);
-void msm_fbdev_free(struct drm_device *dev);
+#ifdef CONFIG_DRM_FBDEV_EMULATION
+void msm_fbdev_setup(struct drm_device *dev);
+#else
+static inline void msm_fbdev_setup(struct drm_device *dev)
+{
+}
+#endif
 
 struct hdmi;
 #ifdef CONFIG_DRM_MSM_HDMI
@@ -548,7 +551,7 @@ static inline unsigned long timeout_to_jiffies(const ktime_t *timeout)
 		remaining_jiffies = ktime_divns(rem, NSEC_PER_SEC / HZ);
 	}
 
-	return clamp(remaining_jiffies, 0LL, (s64)INT_MAX);
+	return clamp(remaining_jiffies, 1LL, (s64)INT_MAX);
 }
 
 /* Driver helpers */
diff --git a/drivers/gpu/drm/msm/msm_fbdev.c b/drivers/gpu/drm/msm/msm_fbdev.c
index d26aa52217ce..2ebc86381e1c 100644
--- a/drivers/gpu/drm/msm/msm_fbdev.c
+++ b/drivers/gpu/drm/msm/msm_fbdev.c
@@ -4,8 +4,8 @@
  * Author: Rob Clark <robdclark@gmail.com>
  */
 
-#include <drm/drm_aperture.h>
-#include <drm/drm_crtc.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
@@ -15,18 +15,40 @@
 #include "msm_gem.h"
 #include "msm_kms.h"
 
-static int msm_fbdev_mmap(struct fb_info *info, struct vm_area_struct *vma);
+static bool fbdev = true;
+MODULE_PARM_DESC(fbdev, "Enable fbdev compat layer");
+module_param(fbdev, bool, 0600);
 
 /*
  * fbdev funcs, to implement legacy fbdev interface on top of drm driver
  */
 
-#define to_msm_fbdev(x) container_of(x, struct msm_fbdev, base)
+static int msm_fbdev_mmap(struct fb_info *info, struct vm_area_struct *vma)
+{
+	struct drm_fb_helper *helper = (struct drm_fb_helper *)info->par;
+	struct drm_gem_object *bo = msm_framebuffer_bo(helper->fb, 0);
 
-struct msm_fbdev {
-	struct drm_fb_helper base;
-	struct drm_framebuffer *fb;
-};
+	return drm_gem_prime_mmap(bo, vma);
+}
+
+static void msm_fbdev_fb_destroy(struct fb_info *info)
+{
+	struct drm_fb_helper *helper = (struct drm_fb_helper *)info->par;
+	struct drm_framebuffer *fb = helper->fb;
+	struct drm_gem_object *bo = msm_framebuffer_bo(fb, 0);
+
+	DBG();
+
+	drm_fb_helper_fini(helper);
+
+	/* this will free the backing object */
+	msm_gem_put_vaddr(bo);
+	drm_framebuffer_remove(fb);
+
+	drm_client_release(&helper->client);
+	drm_fb_helper_unprepare(helper);
+	kfree(helper);
+}
 
 static const struct fb_ops msm_fb_ops = {
 	.owner = THIS_MODULE,
@@ -41,21 +63,12 @@ static const struct fb_ops msm_fb_ops = {
 	.fb_copyarea = drm_fb_helper_sys_copyarea,
 	.fb_imageblit = drm_fb_helper_sys_imageblit,
 	.fb_mmap = msm_fbdev_mmap,
+	.fb_destroy = msm_fbdev_fb_destroy,
 };
 
-static int msm_fbdev_mmap(struct fb_info *info, struct vm_area_struct *vma)
-{
-	struct drm_fb_helper *helper = (struct drm_fb_helper *)info->par;
-	struct msm_fbdev *fbdev = to_msm_fbdev(helper);
-	struct drm_gem_object *bo = msm_framebuffer_bo(fbdev->fb, 0);
-
-	return drm_gem_prime_mmap(bo, vma);
-}
-
 static int msm_fbdev_create(struct drm_fb_helper *helper,
 		struct drm_fb_helper_surface_size *sizes)
 {
-	struct msm_fbdev *fbdev = to_msm_fbdev(helper);
 	struct drm_device *dev = helper->dev;
 	struct msm_drm_private *priv = dev->dev_private;
 	struct drm_framebuffer *fb = NULL;
@@ -102,7 +115,6 @@ static int msm_fbdev_create(struct drm_fb_helper *helper,
 
 	DBG("fbi=%p, dev=%p", fbi, dev);
 
-	fbdev->fb = fb;
 	helper->fb = fb;
 
 	fbi->fbops = &msm_fb_ops;
@@ -119,7 +131,7 @@ static int msm_fbdev_create(struct drm_fb_helper *helper,
 	fbi->fix.smem_len = bo->size;
 
 	DBG("par=%p, %dx%d", fbi->par, fbi->var.xres, fbi->var.yres);
-	DBG("allocated %dx%d fb", fbdev->fb->width, fbdev->fb->height);
+	DBG("allocated %dx%d fb", fb->width, fb->height);
 
 	return 0;
 
@@ -132,73 +144,98 @@ static const struct drm_fb_helper_funcs msm_fb_helper_funcs = {
 	.fb_probe = msm_fbdev_create,
 };
 
-/* initialize fbdev helper */
-struct drm_fb_helper *msm_fbdev_init(struct drm_device *dev)
+/*
+ * struct drm_client
+ */
+
+static void msm_fbdev_client_unregister(struct drm_client_dev *client)
 {
-	struct msm_drm_private *priv = dev->dev_private;
-	struct msm_fbdev *fbdev;
-	struct drm_fb_helper *helper;
-	int ret;
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+
+	if (fb_helper->info) {
+		drm_fb_helper_unregister_info(fb_helper);
+	} else {
+		drm_client_release(&fb_helper->client);
+		drm_fb_helper_unprepare(fb_helper);
+		kfree(fb_helper);
+	}
+}
 
-	fbdev = kzalloc(sizeof(*fbdev), GFP_KERNEL);
-	if (!fbdev)
-		return NULL;
+static int msm_fbdev_client_restore(struct drm_client_dev *client)
+{
+	drm_fb_helper_lastclose(client->dev);
 
-	helper = &fbdev->base;
+	return 0;
+}
 
-	drm_fb_helper_prepare(dev, helper, 32, &msm_fb_helper_funcs);
+static int msm_fbdev_client_hotplug(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+	struct drm_device *dev = client->dev;
+	int ret;
 
-	ret = drm_fb_helper_init(dev, helper);
-	if (ret) {
-		DRM_DEV_ERROR(dev->dev, "could not init fbdev: ret=%d\n", ret);
-		goto fail;
-	}
+	if (dev->fb_helper)
+		return drm_fb_helper_hotplug_event(dev->fb_helper);
 
-	/* the fw fb could be anywhere in memory */
-	ret = drm_aperture_remove_framebuffers(false, dev->driver);
+	ret = drm_fb_helper_init(dev, fb_helper);
 	if (ret)
-		goto fini;
+		goto err_drm_err;
 
-	ret = drm_fb_helper_initial_config(helper);
-	if (ret)
-		goto fini;
+	if (!drm_drv_uses_atomic_modeset(dev))
+		drm_helper_disable_unused_functions(dev);
 
-	priv->fbdev = helper;
+	ret = drm_fb_helper_initial_config(fb_helper);
+	if (ret)
+		goto err_drm_fb_helper_fini;
 
-	return helper;
+	return 0;
 
-fini:
-	drm_fb_helper_fini(helper);
-fail:
-	drm_fb_helper_unprepare(helper);
-	kfree(fbdev);
-	return NULL;
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(fb_helper);
+err_drm_err:
+	drm_err(dev, "Failed to setup fbdev emulation (ret=%d)\n", ret);
+	return ret;
 }
 
-void msm_fbdev_free(struct drm_device *dev)
-{
-	struct msm_drm_private *priv = dev->dev_private;
-	struct drm_fb_helper *helper = priv->fbdev;
-	struct msm_fbdev *fbdev;
+static const struct drm_client_funcs msm_fbdev_client_funcs = {
+	.owner		= THIS_MODULE,
+	.unregister	= msm_fbdev_client_unregister,
+	.restore	= msm_fbdev_client_restore,
+	.hotplug	= msm_fbdev_client_hotplug,
+};
 
-	DBG();
+/* initialize fbdev helper */
+void msm_fbdev_setup(struct drm_device *dev)
+{
+	struct drm_fb_helper *helper;
+	int ret;
 
-	drm_fb_helper_unregister_info(helper);
+	if (!fbdev)
+		return;
 
-	drm_fb_helper_fini(helper);
+	drm_WARN(dev, !dev->registered, "Device has not been registered.\n");
+	drm_WARN(dev, dev->fb_helper, "fb_helper is already set!\n");
 
-	fbdev = to_msm_fbdev(priv->fbdev);
+	helper = kzalloc(sizeof(*helper), GFP_KERNEL);
+	if (!helper)
+		return;
+	drm_fb_helper_prepare(dev, helper, 32, &msm_fb_helper_funcs);
 
-	/* this will free the backing object */
-	if (fbdev->fb) {
-		struct drm_gem_object *bo =
-			msm_framebuffer_bo(fbdev->fb, 0);
-		msm_gem_put_vaddr(bo);
-		drm_framebuffer_remove(fbdev->fb);
+	ret = drm_client_init(dev, &helper->client, "fbdev", &msm_fbdev_client_funcs);
+	if (ret) {
+		drm_err(dev, "Failed to register client: %d\n", ret);
+		goto err_drm_fb_helper_unprepare;
 	}
 
-	drm_fb_helper_unprepare(helper);
-	kfree(fbdev);
+	ret = msm_fbdev_client_hotplug(&helper->client);
+	if (ret)
+		drm_dbg_kms(dev, "client hotplug ret=%d\n", ret);
+
+	drm_client_register(&helper->client);
 
-	priv->fbdev = NULL;
+	return;
+
+err_drm_fb_helper_unprepare:
+	drm_fb_helper_unprepare(helper);
+	kfree(helper);
 }
diff --git a/drivers/gpu/drm/msm/msm_fence.c b/drivers/gpu/drm/msm/msm_fence.c
index 56641408ea74..96599ec3eb78 100644
--- a/drivers/gpu/drm/msm/msm_fence.c
+++ b/drivers/gpu/drm/msm/msm_fence.c
@@ -8,6 +8,35 @@
 
 #include "msm_drv.h"
 #include "msm_fence.h"
+#include "msm_gpu.h"
+
+static struct msm_gpu *fctx2gpu(struct msm_fence_context *fctx)
+{
+	struct msm_drm_private *priv = fctx->dev->dev_private;
+	return priv->gpu;
+}
+
+static enum hrtimer_restart deadline_timer(struct hrtimer *t)
+{
+	struct msm_fence_context *fctx = container_of(t,
+			struct msm_fence_context, deadline_timer);
+
+	kthread_queue_work(fctx2gpu(fctx)->worker, &fctx->deadline_work);
+
+	return HRTIMER_NORESTART;
+}
+
+static void deadline_work(struct kthread_work *work)
+{
+	struct msm_fence_context *fctx = container_of(work,
+			struct msm_fence_context, deadline_work);
+
+	/* If deadline fence has already passed, nothing to do: */
+	if (msm_fence_completed(fctx, fctx->next_deadline_fence))
+		return;
+
+	msm_devfreq_boost(fctx2gpu(fctx), 2);
+}
 
 
 struct msm_fence_context *
@@ -36,6 +65,13 @@ msm_fence_context_alloc(struct drm_device *dev, volatile uint32_t *fenceptr,
 	fctx->completed_fence = fctx->last_fence;
 	*fctx->fenceptr = fctx->last_fence;
 
+	hrtimer_init(&fctx->deadline_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+	fctx->deadline_timer.function = deadline_timer;
+
+	kthread_init_work(&fctx->deadline_work, deadline_work);
+
+	fctx->next_deadline = ktime_get();
+
 	return fctx;
 }
 
@@ -62,6 +98,8 @@ void msm_update_fence(struct msm_fence_context *fctx, uint32_t fence)
 	spin_lock_irqsave(&fctx->spinlock, flags);
 	if (fence_after(fence, fctx->completed_fence))
 		fctx->completed_fence = fence;
+	if (msm_fence_completed(fctx, fctx->next_deadline_fence))
+		hrtimer_cancel(&fctx->deadline_timer);
 	spin_unlock_irqrestore(&fctx->spinlock, flags);
 }
 
@@ -92,14 +130,50 @@ static bool msm_fence_signaled(struct dma_fence *fence)
 	return msm_fence_completed(f->fctx, f->base.seqno);
 }
 
+static void msm_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
+{
+	struct msm_fence *f = to_msm_fence(fence);
+	struct msm_fence_context *fctx = f->fctx;
+	unsigned long flags;
+	ktime_t now;
+
+	spin_lock_irqsave(&fctx->spinlock, flags);
+	now = ktime_get();
+
+	if (ktime_after(now, fctx->next_deadline) ||
+			ktime_before(deadline, fctx->next_deadline)) {
+		fctx->next_deadline = deadline;
+		fctx->next_deadline_fence =
+			max(fctx->next_deadline_fence, (uint32_t)fence->seqno);
+
+		/*
+		 * Set timer to trigger boost 3ms before deadline, or
+		 * if we are already less than 3ms before the deadline
+		 * schedule boost work immediately.
+		 */
+		deadline = ktime_sub(deadline, ms_to_ktime(3));
+
+		if (ktime_after(now, deadline)) {
+			kthread_queue_work(fctx2gpu(fctx)->worker,
+					&fctx->deadline_work);
+		} else {
+			hrtimer_start(&fctx->deadline_timer, deadline,
+					HRTIMER_MODE_ABS);
+		}
+	}
+
+	spin_unlock_irqrestore(&fctx->spinlock, flags);
+}
+
 static const struct dma_fence_ops msm_fence_ops = {
 	.get_driver_name = msm_fence_get_driver_name,
 	.get_timeline_name = msm_fence_get_timeline_name,
 	.signaled = msm_fence_signaled,
+	.set_deadline = msm_fence_set_deadline,
 };
 
 struct dma_fence *
-msm_fence_alloc(struct msm_fence_context *fctx)
+msm_fence_alloc(void)
 {
 	struct msm_fence *f;
 
@@ -107,10 +181,16 @@ msm_fence_alloc(struct msm_fence_context *fctx)
 	if (!f)
 		return ERR_PTR(-ENOMEM);
 
+	return &f->base;
+}
+
+void
+msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx)
+{
+	struct msm_fence *f = to_msm_fence(fence);
+
 	f->fctx = fctx;
 
 	dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
 		       fctx->context, ++fctx->last_fence);
-
-	return &f->base;
 }
diff --git a/drivers/gpu/drm/msm/msm_fence.h b/drivers/gpu/drm/msm/msm_fence.h
index 7f1798c54cd1..148196375a0b 100644
--- a/drivers/gpu/drm/msm/msm_fence.h
+++ b/drivers/gpu/drm/msm/msm_fence.h
@@ -52,6 +52,26 @@ struct msm_fence_context {
 	volatile uint32_t *fenceptr;
 
 	spinlock_t spinlock;
+
+	/*
+	 * TODO this doesn't really deal with multiple deadlines, like
+	 * if userspace got multiple frames ahead.. OTOH atomic updates
+	 * don't queue, so maybe that is ok
+	 */
+
+	/** next_deadline: Time of next deadline */
+	ktime_t next_deadline;
+
+	/**
+	 * next_deadline_fence:
+	 *
+	 * Fence value for next pending deadline.  The deadline timer is
+	 * canceled when this fence is signaled.
+	 */
+	uint32_t next_deadline_fence;
+
+	struct hrtimer deadline_timer;
+	struct kthread_work deadline_work;
 };
 
 struct msm_fence_context * msm_fence_context_alloc(struct drm_device *dev,
@@ -61,7 +81,8 @@ void msm_fence_context_free(struct msm_fence_context *fctx);
 bool msm_fence_completed(struct msm_fence_context *fctx, uint32_t fence);
 void msm_update_fence(struct msm_fence_context *fctx, uint32_t fence);
 
-struct dma_fence * msm_fence_alloc(struct msm_fence_context *fctx);
+struct dma_fence * msm_fence_alloc(void);
+void msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx);
 
 static inline bool
 fence_before(uint32_t a, uint32_t b)
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index c2fb98a94bc3..db6c4e281d75 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -19,8 +19,6 @@
 #include "msm_gpu.h"
 #include "msm_mmu.h"
 
-static void update_lru(struct drm_gem_object *obj);
-
 static dma_addr_t physaddr(struct drm_gem_object *obj)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
@@ -63,6 +61,49 @@ static void sync_for_cpu(struct msm_gem_object *msm_obj)
 	dma_unmap_sgtable(dev, msm_obj->sgt, DMA_BIDIRECTIONAL, 0);
 }
 
+static void update_lru_active(struct drm_gem_object *obj)
+{
+	struct msm_drm_private *priv = obj->dev->dev_private;
+	struct msm_gem_object *msm_obj = to_msm_bo(obj);
+
+	GEM_WARN_ON(!msm_obj->pages);
+
+	if (msm_obj->pin_count) {
+		drm_gem_lru_move_tail_locked(&priv->lru.pinned, obj);
+	} else if (msm_obj->madv == MSM_MADV_WILLNEED) {
+		drm_gem_lru_move_tail_locked(&priv->lru.willneed, obj);
+	} else {
+		GEM_WARN_ON(msm_obj->madv != MSM_MADV_DONTNEED);
+
+		drm_gem_lru_move_tail_locked(&priv->lru.dontneed, obj);
+	}
+}
+
+static void update_lru_locked(struct drm_gem_object *obj)
+{
+	struct msm_drm_private *priv = obj->dev->dev_private;
+	struct msm_gem_object *msm_obj = to_msm_bo(obj);
+
+	msm_gem_assert_locked(&msm_obj->base);
+
+	if (!msm_obj->pages) {
+		GEM_WARN_ON(msm_obj->pin_count);
+
+		drm_gem_lru_move_tail_locked(&priv->lru.unbacked, obj);
+	} else {
+		update_lru_active(obj);
+	}
+}
+
+static void update_lru(struct drm_gem_object *obj)
+{
+	struct msm_drm_private *priv = obj->dev->dev_private;
+
+	mutex_lock(&priv->lru.lock);
+	update_lru_locked(obj);
+	mutex_unlock(&priv->lru.lock);
+}
+
 /* allocate pages from VRAM carveout, used when no IOMMU: */
 static struct page **get_pages_vram(struct drm_gem_object *obj, int npages)
 {
@@ -180,6 +221,7 @@ static void put_pages(struct drm_gem_object *obj)
 
 static struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj)
 {
+	struct msm_drm_private *priv = obj->dev->dev_private;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct page **p;
 
@@ -190,10 +232,13 @@ static struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj)
 	}
 
 	p = get_pages(obj);
-	if (!IS_ERR(p)) {
-		to_msm_bo(obj)->pin_count++;
-		update_lru(obj);
-	}
+	if (IS_ERR(p))
+		return p;
+
+	mutex_lock(&priv->lru.lock);
+	msm_obj->pin_count++;
+	update_lru_locked(obj);
+	mutex_unlock(&priv->lru.lock);
 
 	return p;
 }
@@ -309,12 +354,10 @@ static struct msm_gem_vma *add_vma(struct drm_gem_object *obj,
 
 	msm_gem_assert_locked(obj);
 
-	vma = kzalloc(sizeof(*vma), GFP_KERNEL);
+	vma = msm_gem_vma_new(aspace);
 	if (!vma)
 		return ERR_PTR(-ENOMEM);
 
-	vma->aspace = aspace;
-
 	list_add_tail(&vma->list, &msm_obj->vmas);
 
 	return vma;
@@ -361,9 +404,9 @@ put_iova_spaces(struct drm_gem_object *obj, bool close)
 
 	list_for_each_entry(vma, &msm_obj->vmas, list) {
 		if (vma->aspace) {
-			msm_gem_purge_vma(vma->aspace, vma);
+			msm_gem_vma_purge(vma);
 			if (close)
-				msm_gem_close_vma(vma->aspace, vma);
+				msm_gem_vma_close(vma);
 		}
 	}
 }
@@ -399,7 +442,7 @@ static struct msm_gem_vma *get_vma_locked(struct drm_gem_object *obj,
 		if (IS_ERR(vma))
 			return vma;
 
-		ret = msm_gem_init_vma(aspace, vma, obj->size,
+		ret = msm_gem_vma_init(vma, obj->size,
 			range_start, range_end);
 		if (ret) {
 			del_vma(vma);
@@ -437,7 +480,7 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma)
 	if (IS_ERR(pages))
 		return PTR_ERR(pages);
 
-	ret = msm_gem_map_vma(vma->aspace, vma, prot, msm_obj->sgt, obj->size);
+	ret = msm_gem_vma_map(vma, prot, msm_obj->sgt, obj->size);
 	if (ret)
 		msm_gem_unpin_locked(obj);
 
@@ -446,14 +489,34 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma)
 
 void msm_gem_unpin_locked(struct drm_gem_object *obj)
 {
+	struct msm_drm_private *priv = obj->dev->dev_private;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
 	msm_gem_assert_locked(obj);
 
+	mutex_lock(&priv->lru.lock);
 	msm_obj->pin_count--;
 	GEM_WARN_ON(msm_obj->pin_count < 0);
+	update_lru_locked(obj);
+	mutex_unlock(&priv->lru.lock);
+}
 
-	update_lru(obj);
+/* Special unpin path for use in fence-signaling path, avoiding the need
+ * to hold the obj lock by only depending on things that a protected by
+ * the LRU lock.  In particular we know that that we already have backing
+ * and and that the object's dma_resv has the fence for the current
+ * submit/job which will prevent us racing against page eviction.
+ */
+void msm_gem_unpin_active(struct drm_gem_object *obj)
+{
+	struct msm_drm_private *priv = obj->dev->dev_private;
+	struct msm_gem_object *msm_obj = to_msm_bo(obj);
+
+	mutex_lock(&priv->lru.lock);
+	msm_obj->pin_count--;
+	GEM_WARN_ON(msm_obj->pin_count < 0);
+	update_lru_active(obj);
+	mutex_unlock(&priv->lru.lock);
 }
 
 struct msm_gem_vma *msm_gem_get_vma_locked(struct drm_gem_object *obj,
@@ -539,8 +602,8 @@ static int clear_iova(struct drm_gem_object *obj,
 	if (msm_gem_vma_inuse(vma))
 		return -EBUSY;
 
-	msm_gem_purge_vma(vma->aspace, vma);
-	msm_gem_close_vma(vma->aspace, vma);
+	msm_gem_vma_purge(vma);
+	msm_gem_vma_close(vma);
 	del_vma(vma);
 
 	return 0;
@@ -589,7 +652,7 @@ void msm_gem_unpin_iova(struct drm_gem_object *obj,
 	msm_gem_lock(obj);
 	vma = lookup_vma(obj, aspace);
 	if (!GEM_WARN_ON(!vma)) {
-		msm_gem_unpin_vma(vma);
+		msm_gem_vma_unpin(vma);
 		msm_gem_unpin_locked(obj);
 	}
 	msm_gem_unlock(obj);
@@ -628,6 +691,7 @@ fail:
 static void *get_vaddr(struct drm_gem_object *obj, unsigned madv)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
+	struct page **pages;
 	int ret = 0;
 
 	msm_gem_assert_locked(obj);
@@ -641,6 +705,10 @@ static void *get_vaddr(struct drm_gem_object *obj, unsigned madv)
 		return ERR_PTR(-EBUSY);
 	}
 
+	pages = msm_gem_pin_pages_locked(obj);
+	if (IS_ERR(pages))
+		return ERR_CAST(pages);
+
 	/* increment vmap_count *before* vmap() call, so shrinker can
 	 * check vmap_count (is_vunmapable()) outside of msm_obj lock.
 	 * This guarantees that we won't try to msm_gem_vunmap() this
@@ -650,25 +718,19 @@ static void *get_vaddr(struct drm_gem_object *obj, unsigned madv)
 	msm_obj->vmap_count++;
 
 	if (!msm_obj->vaddr) {
-		struct page **pages = get_pages(obj);
-		if (IS_ERR(pages)) {
-			ret = PTR_ERR(pages);
-			goto fail;
-		}
 		msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,
 				VM_MAP, msm_gem_pgprot(msm_obj, PAGE_KERNEL));
 		if (msm_obj->vaddr == NULL) {
 			ret = -ENOMEM;
 			goto fail;
 		}
-
-		update_lru(obj);
 	}
 
 	return msm_obj->vaddr;
 
 fail:
 	msm_obj->vmap_count--;
+	msm_gem_unpin_locked(obj);
 	return ERR_PTR(ret);
 }
 
@@ -707,6 +769,7 @@ void msm_gem_put_vaddr_locked(struct drm_gem_object *obj)
 	GEM_WARN_ON(msm_obj->vmap_count < 1);
 
 	msm_obj->vmap_count--;
+	msm_gem_unpin_locked(obj);
 }
 
 void msm_gem_put_vaddr(struct drm_gem_object *obj)
@@ -721,10 +784,13 @@ void msm_gem_put_vaddr(struct drm_gem_object *obj)
  */
 int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
 {
+	struct msm_drm_private *priv = obj->dev->dev_private;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
 	msm_gem_lock(obj);
 
+	mutex_lock(&priv->lru.lock);
+
 	if (msm_obj->madv != __MSM_MADV_PURGED)
 		msm_obj->madv = madv;
 
@@ -733,7 +799,9 @@ int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
 	/* If the obj is inactive, we might need to move it
 	 * between inactive lists
 	 */
-	update_lru(obj);
+	update_lru_locked(obj);
+
+	mutex_unlock(&priv->lru.lock);
 
 	msm_gem_unlock(obj);
 
@@ -743,6 +811,7 @@ int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
 void msm_gem_purge(struct drm_gem_object *obj)
 {
 	struct drm_device *dev = obj->dev;
+	struct msm_drm_private *priv = obj->dev->dev_private;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
 	msm_gem_assert_locked(obj);
@@ -759,7 +828,10 @@ void msm_gem_purge(struct drm_gem_object *obj)
 
 	put_iova_vmas(obj);
 
+	mutex_lock(&priv->lru.lock);
+	/* A one-way transition: */
 	msm_obj->madv = __MSM_MADV_PURGED;
+	mutex_unlock(&priv->lru.lock);
 
 	drm_gem_free_mmap_offset(obj);
 
@@ -806,29 +878,6 @@ void msm_gem_vunmap(struct drm_gem_object *obj)
 	msm_obj->vaddr = NULL;
 }
 
-static void update_lru(struct drm_gem_object *obj)
-{
-	struct msm_drm_private *priv = obj->dev->dev_private;
-	struct msm_gem_object *msm_obj = to_msm_bo(obj);
-
-	msm_gem_assert_locked(&msm_obj->base);
-
-	if (!msm_obj->pages) {
-		GEM_WARN_ON(msm_obj->pin_count);
-		GEM_WARN_ON(msm_obj->vmap_count);
-
-		drm_gem_lru_move_tail(&priv->lru.unbacked, obj);
-	} else if (msm_obj->pin_count || msm_obj->vmap_count) {
-		drm_gem_lru_move_tail(&priv->lru.pinned, obj);
-	} else if (msm_obj->madv == MSM_MADV_WILLNEED) {
-		drm_gem_lru_move_tail(&priv->lru.willneed, obj);
-	} else {
-		GEM_WARN_ON(msm_obj->madv != MSM_MADV_DONTNEED);
-
-		drm_gem_lru_move_tail(&priv->lru.dontneed, obj);
-	}
-}
-
 bool msm_gem_active(struct drm_gem_object *obj)
 {
 	msm_gem_assert_locked(obj);
@@ -846,6 +895,11 @@ int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout)
 		op & MSM_PREP_NOSYNC ? 0 : timeout_to_jiffies(timeout);
 	long ret;
 
+	if (op & MSM_PREP_BOOST) {
+		dma_resv_set_deadline(obj->resv, dma_resv_usage_rw(write),
+				      ktime_get());
+	}
+
 	ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(write),
 				    true,  remain);
 	if (ret == 0)
diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h
index c4844cf3a585..2bd6846c83a9 100644
--- a/drivers/gpu/drm/msm/msm_gem.h
+++ b/drivers/gpu/drm/msm/msm_gem.h
@@ -59,6 +59,7 @@ struct msm_fence_context;
 
 struct msm_gem_vma {
 	struct drm_mm_node node;
+	spinlock_t lock;
 	uint64_t iova;
 	struct msm_gem_address_space *aspace;
 	struct list_head list;    /* node in msm_gem_object::vmas */
@@ -69,19 +70,15 @@ struct msm_gem_vma {
 	struct msm_fence_context *fctx[MSM_GPU_MAX_RINGS];
 };
 
-int msm_gem_init_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma, int size,
+struct msm_gem_vma *msm_gem_vma_new(struct msm_gem_address_space *aspace);
+int msm_gem_vma_init(struct msm_gem_vma *vma, int size,
 		u64 range_start, u64 range_end);
 bool msm_gem_vma_inuse(struct msm_gem_vma *vma);
-void msm_gem_purge_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma);
-void msm_gem_unpin_vma(struct msm_gem_vma *vma);
-void msm_gem_unpin_vma_fenced(struct msm_gem_vma *vma, struct msm_fence_context *fctx);
-int msm_gem_map_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma, int prot,
-		struct sg_table *sgt, int size);
-void msm_gem_close_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma);
+void msm_gem_vma_purge(struct msm_gem_vma *vma);
+void msm_gem_vma_unpin(struct msm_gem_vma *vma);
+void msm_gem_vma_unpin_fenced(struct msm_gem_vma *vma, struct msm_fence_context *fctx);
+int msm_gem_vma_map(struct msm_gem_vma *vma, int prot, struct sg_table *sgt, int size);
+void msm_gem_vma_close(struct msm_gem_vma *vma);
 
 struct msm_gem_object {
 	struct drm_gem_object base;
@@ -89,7 +86,9 @@ struct msm_gem_object {
 	uint32_t flags;
 
 	/**
-	 * Advice: are the backing pages purgeable?
+	 * madv: are the backing pages purgeable?
+	 *
+	 * Protected by obj lock and LRU lock
 	 */
 	uint8_t madv;
 
@@ -117,6 +116,11 @@ struct msm_gem_object {
 
 	char name[32]; /* Identifier to print for the debugfs files */
 
+	/**
+	 * pin_count: Number of times the pages are pinned
+	 *
+	 * Protected by LRU lock.
+	 */
 	int pin_count;
 };
 #define to_msm_bo(x) container_of(x, struct msm_gem_object, base)
@@ -124,6 +128,7 @@ struct msm_gem_object {
 uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj);
 int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma);
 void msm_gem_unpin_locked(struct drm_gem_object *obj);
+void msm_gem_unpin_active(struct drm_gem_object *obj);
 struct msm_gem_vma *msm_gem_get_vma_locked(struct drm_gem_object *obj,
 					   struct msm_gem_address_space *aspace);
 int msm_gem_get_iova(struct drm_gem_object *obj,
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index ac8ed731f76d..aff18c2f600a 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -41,8 +41,16 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,
 	if (!submit)
 		return ERR_PTR(-ENOMEM);
 
+	submit->hw_fence = msm_fence_alloc();
+	if (IS_ERR(submit->hw_fence)) {
+		ret = PTR_ERR(submit->hw_fence);
+		kfree(submit);
+		return ERR_PTR(ret);
+	}
+
 	ret = drm_sched_job_init(&submit->base, queue->entity, queue);
 	if (ret) {
+		kfree(submit->hw_fence);
 		kfree(submit);
 		return ERR_PTR(ret);
 	}
@@ -72,9 +80,9 @@ void __msm_gem_submit_destroy(struct kref *kref)
 	unsigned i;
 
 	if (submit->fence_id) {
-		mutex_lock(&submit->queue->idr_lock);
+		spin_lock(&submit->queue->idr_lock);
 		idr_remove(&submit->queue->fence_idr, submit->fence_id);
-		mutex_unlock(&submit->queue->idr_lock);
+		spin_unlock(&submit->queue->idr_lock);
 	}
 
 	dma_fence_put(submit->user_fence);
@@ -242,7 +250,7 @@ static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,
 	submit->bos[i].flags &= ~cleanup_flags;
 
 	if (flags & BO_VMA_PINNED)
-		msm_gem_unpin_vma(submit->bos[i].vma);
+		msm_gem_vma_unpin(submit->bos[i].vma);
 
 	if (flags & BO_OBJ_PINNED)
 		msm_gem_unpin_locked(obj);
@@ -565,7 +573,6 @@ static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit,
 
 	for (i = 0; i < nr_in_syncobjs; ++i) {
 		uint64_t address = in_syncobjs_addr + i * syncobj_stride;
-		struct dma_fence *fence;
 
 		if (copy_from_user(&syncobj_desc,
 			           u64_to_user_ptr(address),
@@ -585,12 +592,8 @@ static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit,
 			break;
 		}
 
-		ret = drm_syncobj_find_fence(file, syncobj_desc.handle,
-		                             syncobj_desc.point, 0, &fence);
-		if (ret)
-			break;
-
-		ret = drm_sched_job_add_dependency(&submit->base, fence);
+		ret = drm_sched_job_add_syncobj_dependency(&submit->base, file,
+							   syncobj_desc.handle, syncobj_desc.point);
 		if (ret)
 			break;
 
@@ -874,7 +877,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 
 	submit->nr_cmds = i;
 
-	mutex_lock(&queue->idr_lock);
+	idr_preload(GFP_KERNEL);
+
+	spin_lock(&queue->idr_lock);
 
 	/*
 	 * If using userspace provided seqno fence, validate that the id
@@ -884,7 +889,8 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 	 */
 	if ((args->flags & MSM_SUBMIT_FENCE_SN_IN) &&
 			idr_find(&queue->fence_idr, args->fence)) {
-		mutex_unlock(&queue->idr_lock);
+		spin_unlock(&queue->idr_lock);
+		idr_preload_end();
 		ret = -EINVAL;
 		goto out;
 	}
@@ -902,7 +908,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 		submit->fence_id = args->fence;
 		ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence,
 				    &submit->fence_id, submit->fence_id,
-				    GFP_KERNEL);
+				    GFP_NOWAIT);
 		/*
 		 * We've already validated that the fence_id slot is valid,
 		 * so if idr_alloc_u32 failed, it is a kernel bug
@@ -915,10 +921,11 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 		 */
 		submit->fence_id = idr_alloc_cyclic(&queue->fence_idr,
 						    submit->user_fence, 1,
-						    INT_MAX, GFP_KERNEL);
+						    INT_MAX, GFP_NOWAIT);
 	}
 
-	mutex_unlock(&queue->idr_lock);
+	spin_unlock(&queue->idr_lock);
+	idr_preload_end();
 
 	if (submit->fence_id < 0) {
 		ret = submit->fence_id;
diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c
index c471aebcdbab..98287ed99960 100644
--- a/drivers/gpu/drm/msm/msm_gem_vma.c
+++ b/drivers/gpu/drm/msm/msm_gem_vma.c
@@ -40,25 +40,34 @@ msm_gem_address_space_get(struct msm_gem_address_space *aspace)
 
 bool msm_gem_vma_inuse(struct msm_gem_vma *vma)
 {
+	bool ret = true;
+
+	spin_lock(&vma->lock);
+
 	if (vma->inuse > 0)
-		return true;
+		goto out;
 
 	while (vma->fence_mask) {
 		unsigned idx = ffs(vma->fence_mask) - 1;
 
 		if (!msm_fence_completed(vma->fctx[idx], vma->fence[idx]))
-			return true;
+			goto out;
 
 		vma->fence_mask &= ~BIT(idx);
 	}
 
-	return false;
+	ret = false;
+
+out:
+	spin_unlock(&vma->lock);
+
+	return ret;
 }
 
 /* Actually unmap memory for the vma */
-void msm_gem_purge_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma)
+void msm_gem_vma_purge(struct msm_gem_vma *vma)
 {
+	struct msm_gem_address_space *aspace = vma->aspace;
 	unsigned size = vma->node.size;
 
 	/* Print a message if we try to purge a vma in use */
@@ -68,14 +77,12 @@ void msm_gem_purge_vma(struct msm_gem_address_space *aspace,
 	if (!vma->mapped)
 		return;
 
-	if (aspace->mmu)
-		aspace->mmu->funcs->unmap(aspace->mmu, vma->iova, size);
+	aspace->mmu->funcs->unmap(aspace->mmu, vma->iova, size);
 
 	vma->mapped = false;
 }
 
-/* Remove reference counts for the mapping */
-void msm_gem_unpin_vma(struct msm_gem_vma *vma)
+static void vma_unpin_locked(struct msm_gem_vma *vma)
 {
 	if (GEM_WARN_ON(!vma->inuse))
 		return;
@@ -83,50 +90,75 @@ void msm_gem_unpin_vma(struct msm_gem_vma *vma)
 		vma->inuse--;
 }
 
+/* Remove reference counts for the mapping */
+void msm_gem_vma_unpin(struct msm_gem_vma *vma)
+{
+	spin_lock(&vma->lock);
+	vma_unpin_locked(vma);
+	spin_unlock(&vma->lock);
+}
+
 /* Replace pin reference with fence: */
-void msm_gem_unpin_vma_fenced(struct msm_gem_vma *vma, struct msm_fence_context *fctx)
+void msm_gem_vma_unpin_fenced(struct msm_gem_vma *vma, struct msm_fence_context *fctx)
 {
+	spin_lock(&vma->lock);
 	vma->fctx[fctx->index] = fctx;
 	vma->fence[fctx->index] = fctx->last_fence;
 	vma->fence_mask |= BIT(fctx->index);
-	msm_gem_unpin_vma(vma);
+	vma_unpin_locked(vma);
+	spin_unlock(&vma->lock);
 }
 
 /* Map and pin vma: */
 int
-msm_gem_map_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma, int prot,
+msm_gem_vma_map(struct msm_gem_vma *vma, int prot,
 		struct sg_table *sgt, int size)
 {
-	int ret = 0;
+	struct msm_gem_address_space *aspace = vma->aspace;
+	int ret;
 
 	if (GEM_WARN_ON(!vma->iova))
 		return -EINVAL;
 
 	/* Increase the usage counter */
+	spin_lock(&vma->lock);
 	vma->inuse++;
+	spin_unlock(&vma->lock);
 
 	if (vma->mapped)
 		return 0;
 
 	vma->mapped = true;
 
-	if (aspace && aspace->mmu)
-		ret = aspace->mmu->funcs->map(aspace->mmu, vma->iova, sgt,
-				size, prot);
+	if (!aspace)
+		return 0;
+
+	/*
+	 * NOTE: iommu/io-pgtable can allocate pages, so we cannot hold
+	 * a lock across map/unmap which is also used in the job_run()
+	 * path, as this can cause deadlock in job_run() vs shrinker/
+	 * reclaim.
+	 *
+	 * Revisit this if we can come up with a scheme to pre-alloc pages
+	 * for the pgtable in map/unmap ops.
+	 */
+	ret = aspace->mmu->funcs->map(aspace->mmu, vma->iova, sgt, size, prot);
 
 	if (ret) {
 		vma->mapped = false;
+		spin_lock(&vma->lock);
 		vma->inuse--;
+		spin_unlock(&vma->lock);
 	}
 
 	return ret;
 }
 
 /* Close an iova.  Warn if it is still in use */
-void msm_gem_close_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma)
+void msm_gem_vma_close(struct msm_gem_vma *vma)
 {
+	struct msm_gem_address_space *aspace = vma->aspace;
+
 	GEM_WARN_ON(msm_gem_vma_inuse(vma) || vma->mapped);
 
 	spin_lock(&aspace->lock);
@@ -139,13 +171,30 @@ void msm_gem_close_vma(struct msm_gem_address_space *aspace,
 	msm_gem_address_space_put(aspace);
 }
 
+struct msm_gem_vma *msm_gem_vma_new(struct msm_gem_address_space *aspace)
+{
+	struct msm_gem_vma *vma;
+
+	vma = kzalloc(sizeof(*vma), GFP_KERNEL);
+	if (!vma)
+		return NULL;
+
+	spin_lock_init(&vma->lock);
+	vma->aspace = aspace;
+
+	return vma;
+}
+
 /* Initialize a new vma and allocate an iova for it */
-int msm_gem_init_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma, int size,
+int msm_gem_vma_init(struct msm_gem_vma *vma, int size,
 		u64 range_start, u64 range_end)
 {
+	struct msm_gem_address_space *aspace = vma->aspace;
 	int ret;
 
+	if (GEM_WARN_ON(!aspace))
+		return -EINVAL;
+
 	if (GEM_WARN_ON(vma->iova))
 		return -EBUSY;
 
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 380249500325..26ebda40be4f 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -16,7 +16,6 @@
 #include <generated/utsrelease.h>
 #include <linux/string_helpers.h>
 #include <linux/devcoredump.h>
-#include <linux/reset.h>
 #include <linux/sched/task.h>
 
 /*
@@ -59,7 +58,7 @@ static int disable_pwrrail(struct msm_gpu *gpu)
 static int enable_clk(struct msm_gpu *gpu)
 {
 	if (gpu->core_clk && gpu->fast_rate)
-		clk_set_rate(gpu->core_clk, gpu->fast_rate);
+		dev_pm_opp_set_rate(&gpu->pdev->dev, gpu->fast_rate);
 
 	/* Set the RBBM timer rate to 19.2Mhz */
 	if (gpu->rbbmtimer_clk)
@@ -78,7 +77,7 @@ static int disable_clk(struct msm_gpu *gpu)
 	 * will be rounded down to zero anyway so it all works out.
 	 */
 	if (gpu->core_clk)
-		clk_set_rate(gpu->core_clk, 27000000);
+		dev_pm_opp_set_rate(&gpu->pdev->dev, 27000000);
 
 	if (gpu->rbbmtimer_clk)
 		clk_set_rate(gpu->rbbmtimer_clk, 0);
@@ -935,9 +934,6 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	if (IS_ERR(gpu->gpu_cx))
 		gpu->gpu_cx = NULL;
 
-	gpu->cx_collapse = devm_reset_control_get_optional_exclusive(&pdev->dev,
-			"cx_collapse");
-
 	gpu->pdev = pdev;
 	platform_set_drvdata(pdev, &gpu->adreno_smmu);
 
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index fc1c0d8611a8..7a4fa1b8655b 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -13,7 +13,6 @@
 #include <linux/interconnect.h>
 #include <linux/pm_opp.h>
 #include <linux/regulator/consumer.h>
-#include <linux/reset.h>
 
 #include "msm_drv.h"
 #include "msm_fence.h"
@@ -50,6 +49,12 @@ struct msm_gpu_funcs {
 	int (*set_param)(struct msm_gpu *gpu, struct msm_file_private *ctx,
 			 uint32_t param, uint64_t value, uint32_t len);
 	int (*hw_init)(struct msm_gpu *gpu);
+
+	/**
+	 * @ucode_load: Optional hook to upload fw to GEM objs
+	 */
+	int (*ucode_load)(struct msm_gpu *gpu);
+
 	int (*pm_suspend)(struct msm_gpu *gpu);
 	int (*pm_resume)(struct msm_gpu *gpu);
 	void (*submit)(struct msm_gpu *gpu, struct msm_gem_submit *submit);
@@ -281,9 +286,6 @@ struct msm_gpu {
 	bool hw_apriv;
 
 	struct thermal_cooling_device *cooling;
-
-	/* To poll for cx gdsc collapse during gpu recovery */
-	struct reset_control *cx_collapse;
 };
 
 static inline struct msm_gpu *dev_to_gpu(struct device *dev)
@@ -499,7 +501,7 @@ struct msm_gpu_submitqueue {
 	struct msm_file_private *ctx;
 	struct list_head node;
 	struct idr fence_idr;
-	struct mutex idr_lock;
+	struct spinlock idr_lock;
 	struct mutex lock;
 	struct kref ref;
 	struct drm_sched_entity *entity;
diff --git a/drivers/gpu/drm/msm/msm_gpu_devfreq.c b/drivers/gpu/drm/msm/msm_gpu_devfreq.c
index e27dbf12b5e8..ea70c1c32d94 100644
--- a/drivers/gpu/drm/msm/msm_gpu_devfreq.c
+++ b/drivers/gpu/drm/msm/msm_gpu_devfreq.c
@@ -48,7 +48,7 @@ static int msm_devfreq_target(struct device *dev, unsigned long *freq,
 		gpu->funcs->gpu_set_freq(gpu, opp, df->suspended);
 		mutex_unlock(&df->lock);
 	} else {
-		clk_set_rate(gpu->core_clk, *freq);
+		dev_pm_opp_set_rate(dev, *freq);
 	}
 
 	dev_pm_opp_put(opp);
diff --git a/drivers/gpu/drm/msm/msm_io_utils.c b/drivers/gpu/drm/msm/msm_io_utils.c
index d02cd29ce829..59d2788c4510 100644
--- a/drivers/gpu/drm/msm/msm_io_utils.c
+++ b/drivers/gpu/drm/msm/msm_io_utils.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/interconnect.h>
+#include <linux/io.h>
 
 #include "msm_drv.h"
 
diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c
index c2507582ecf3..418e1e06cdde 100644
--- a/drivers/gpu/drm/msm/msm_iommu.c
+++ b/drivers/gpu/drm/msm/msm_iommu.c
@@ -237,13 +237,6 @@ struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent)
 	if (!ttbr1_cfg)
 		return ERR_PTR(-ENODEV);
 
-	/*
-	 * Defer setting the fault handler until we have a valid adreno_smmu
-	 * to avoid accidentially installing a GPU specific fault handler for
-	 * the display's iommu
-	 */
-	iommu_set_fault_handler(iommu->domain, msm_fault_handler, iommu);
-
 	pagetable = kzalloc(sizeof(*pagetable), GFP_KERNEL);
 	if (!pagetable)
 		return ERR_PTR(-ENOMEM);
@@ -271,9 +264,6 @@ struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent)
 	 * the arm-smmu driver as a trigger to set up TTBR0
 	 */
 	if (atomic_inc_return(&iommu->pagetables) == 1) {
-		/* Enable stall on iommu fault: */
-		adreno_smmu->set_stall(adreno_smmu->cookie, true);
-
 		ret = adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, &ttbr0_cfg);
 		if (ret) {
 			free_io_pgtable_ops(pagetable->pgtbl_ops);
@@ -302,6 +292,7 @@ static int msm_fault_handler(struct iommu_domain *domain, struct device *dev,
 		unsigned long iova, int flags, void *arg)
 {
 	struct msm_iommu *iommu = arg;
+	struct msm_mmu *mmu = &iommu->base;
 	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(iommu->base.dev);
 	struct adreno_smmu_fault_info info, *ptr = NULL;
 
@@ -314,6 +305,10 @@ static int msm_fault_handler(struct iommu_domain *domain, struct device *dev,
 		return iommu->base.handler(iommu->base.arg, iova, flags, ptr);
 
 	pr_warn_ratelimited("*** fault: iova=%16lx, flags=%d\n", iova, flags);
+
+	if (mmu->funcs->resume_translation)
+		mmu->funcs->resume_translation(mmu);
+
 	return 0;
 }
 
@@ -321,7 +316,8 @@ static void msm_iommu_resume_translation(struct msm_mmu *mmu)
 {
 	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(mmu->dev);
 
-	adreno_smmu->resume_translation(adreno_smmu->cookie, true);
+	if (adreno_smmu->resume_translation)
+		adreno_smmu->resume_translation(adreno_smmu->cookie, true);
 }
 
 static void msm_iommu_detach(struct msm_mmu *mmu)
@@ -406,3 +402,23 @@ struct msm_mmu *msm_iommu_new(struct device *dev, unsigned long quirks)
 
 	return &iommu->base;
 }
+
+struct msm_mmu *msm_iommu_gpu_new(struct device *dev, struct msm_gpu *gpu, unsigned long quirks)
+{
+	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev);
+	struct msm_iommu *iommu;
+	struct msm_mmu *mmu;
+
+	mmu = msm_iommu_new(dev, quirks);
+	if (IS_ERR(mmu))
+		return mmu;
+
+	iommu = to_msm_iommu(mmu);
+	iommu_set_fault_handler(iommu->domain, msm_fault_handler, iommu);
+
+	/* Enable stall on iommu fault: */
+	if (adreno_smmu->set_stall)
+		adreno_smmu->set_stall(adreno_smmu->cookie, true);
+
+	return mmu;
+}
diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h
index f8ed7588928c..086a3f1ff956 100644
--- a/drivers/gpu/drm/msm/msm_kms.h
+++ b/drivers/gpu/drm/msm/msm_kms.h
@@ -60,14 +60,6 @@ struct msm_kms_funcs {
 	void (*disable_commit)(struct msm_kms *kms);
 
 	/**
-	 * If the kms backend supports async commit, it should implement
-	 * this method to return the time of the next vsync.  This is
-	 * used to determine a time slightly before vsync, for the async
-	 * commit timer to run and complete an async commit.
-	 */
-	ktime_t (*vsync_time)(struct msm_kms *kms, struct drm_crtc *crtc);
-
-	/**
 	 * Prepare for atomic commit.  This is called after any previous
 	 * (async or otherwise) commit has completed.
 	 */
diff --git a/drivers/gpu/drm/msm/msm_mdss.c b/drivers/gpu/drm/msm/msm_mdss.c
index 02646e4bb4cd..e8c93731aaa1 100644
--- a/drivers/gpu/drm/msm/msm_mdss.c
+++ b/drivers/gpu/drm/msm/msm_mdss.c
@@ -16,9 +16,6 @@
 #include "msm_drv.h"
 #include "msm_kms.h"
 
-/* for DPU_HW_* defines */
-#include "disp/dpu1/dpu_hw_catalog.h"
-
 #define HW_REV				0x0
 #define HW_INTR_STATUS			0x0010
 
@@ -29,6 +26,16 @@
 
 #define MIN_IB_BW	400000000UL /* Min ib vote 400MB */
 
+struct msm_mdss_data {
+	u32 ubwc_version;
+	/* can be read from register 0x58 */
+	u32 ubwc_dec_version;
+	u32 ubwc_swizzle;
+	u32 ubwc_static;
+	u32 highest_bank_bit;
+	u32 macrotile_mode;
+};
+
 struct msm_mdss {
 	struct device *dev;
 
@@ -40,6 +47,7 @@ struct msm_mdss {
 		unsigned long enabled_mask;
 		struct irq_domain *domain;
 	} irq_controller;
+	const struct msm_mdss_data *mdss_data;
 	struct icc_path *path[2];
 	u32 num_paths;
 };
@@ -182,46 +190,40 @@ static int _msm_mdss_irq_domain_add(struct msm_mdss *msm_mdss)
 #define UBWC_3_0 0x30000000
 #define UBWC_4_0 0x40000000
 
-static void msm_mdss_setup_ubwc_dec_20(struct msm_mdss *msm_mdss,
-				       u32 ubwc_static)
+static void msm_mdss_setup_ubwc_dec_20(struct msm_mdss *msm_mdss)
 {
-	writel_relaxed(ubwc_static, msm_mdss->mmio + UBWC_STATIC);
+	const struct msm_mdss_data *data = msm_mdss->mdss_data;
+
+	writel_relaxed(data->ubwc_static, msm_mdss->mmio + UBWC_STATIC);
 }
 
-static void msm_mdss_setup_ubwc_dec_30(struct msm_mdss *msm_mdss,
-				       unsigned int ubwc_version,
-				       u32 ubwc_swizzle,
-				       u32 highest_bank_bit,
-				       u32 macrotile_mode)
+static void msm_mdss_setup_ubwc_dec_30(struct msm_mdss *msm_mdss)
 {
-	u32 value = (ubwc_swizzle & 0x1) |
-		    (highest_bank_bit & 0x3) << 4 |
-		    (macrotile_mode & 0x1) << 12;
+	const struct msm_mdss_data *data = msm_mdss->mdss_data;
+	u32 value = (data->ubwc_swizzle & 0x1) |
+		    (data->highest_bank_bit & 0x3) << 4 |
+		    (data->macrotile_mode & 0x1) << 12;
 
-	if (ubwc_version == UBWC_3_0)
+	if (data->ubwc_version == UBWC_3_0)
 		value |= BIT(10);
 
-	if (ubwc_version == UBWC_1_0)
+	if (data->ubwc_version == UBWC_1_0)
 		value |= BIT(8);
 
 	writel_relaxed(value, msm_mdss->mmio + UBWC_STATIC);
 }
 
-static void msm_mdss_setup_ubwc_dec_40(struct msm_mdss *msm_mdss,
-				       unsigned int ubwc_version,
-				       u32 ubwc_swizzle,
-				       u32 ubwc_static,
-				       u32 highest_bank_bit,
-				       u32 macrotile_mode)
+static void msm_mdss_setup_ubwc_dec_40(struct msm_mdss *msm_mdss)
 {
-	u32 value = (ubwc_swizzle & 0x7) |
-		    (ubwc_static & 0x1) << 3 |
-		    (highest_bank_bit & 0x7) << 4 |
-		    (macrotile_mode & 0x1) << 12;
+	const struct msm_mdss_data *data = msm_mdss->mdss_data;
+	u32 value = (data->ubwc_swizzle & 0x7) |
+		    (data->ubwc_static & 0x1) << 3 |
+		    (data->highest_bank_bit & 0x7) << 4 |
+		    (data->macrotile_mode & 0x1) << 12;
 
 	writel_relaxed(value, msm_mdss->mmio + UBWC_STATIC);
 
-	if (ubwc_version == UBWC_3_0) {
+	if (data->ubwc_version == UBWC_3_0) {
 		writel_relaxed(1, msm_mdss->mmio + UBWC_CTRL_2);
 		writel_relaxed(0, msm_mdss->mmio + UBWC_PREDICTION_MODE);
 	} else {
@@ -233,7 +235,6 @@ static void msm_mdss_setup_ubwc_dec_40(struct msm_mdss *msm_mdss,
 static int msm_mdss_enable(struct msm_mdss *msm_mdss)
 {
 	int ret;
-	u32 hw_rev;
 
 	/*
 	 * Several components have AXI clocks that can only be turned on if
@@ -249,57 +250,36 @@ static int msm_mdss_enable(struct msm_mdss *msm_mdss)
 	}
 
 	/*
-	 * HW_REV requires MDSS_MDP_CLK, which is not enabled by the mdss on
-	 * mdp5 hardware. Skip reading it for now.
+	 * Register access requires MDSS_MDP_CLK, which is not enabled by the
+	 * mdss on mdp5 hardware. Skip it for now.
 	 */
-	if (msm_mdss->is_mdp5)
+	if (msm_mdss->is_mdp5 || !msm_mdss->mdss_data)
 		return 0;
 
-	hw_rev = readl_relaxed(msm_mdss->mmio + HW_REV);
-	dev_dbg(msm_mdss->dev, "HW_REV: 0x%x\n", hw_rev);
-	dev_dbg(msm_mdss->dev, "UBWC_DEC_HW_VERSION: 0x%x\n",
-		readl_relaxed(msm_mdss->mmio + UBWC_DEC_HW_VERSION));
-
 	/*
 	 * ubwc config is part of the "mdss" region which is not accessible
 	 * from the rest of the driver. hardcode known configurations here
 	 *
 	 * Decoder version can be read from the UBWC_DEC_HW_VERSION reg,
-	 * UBWC_n and the rest of params comes from hw_catalog.
-	 * Unforunately this driver can not access hw catalog, so we have to
-	 * hardcode them here.
+	 * UBWC_n and the rest of params comes from hw data.
 	 */
-	switch (hw_rev) {
-	case DPU_HW_VER_500:
-	case DPU_HW_VER_501:
-		msm_mdss_setup_ubwc_dec_30(msm_mdss, UBWC_3_0, 0, 2, 0);
-		break;
-	case DPU_HW_VER_600:
-		/* TODO: highest_bank_bit = 2 for LP_DDR4 */
-		msm_mdss_setup_ubwc_dec_40(msm_mdss, UBWC_4_0, 6, 1, 3, 1);
-		break;
-	case DPU_HW_VER_620:
-		/* UBWC_2_0 */
-		msm_mdss_setup_ubwc_dec_20(msm_mdss, 0x1e);
+	switch (msm_mdss->mdss_data->ubwc_dec_version) {
+	case UBWC_2_0:
+		msm_mdss_setup_ubwc_dec_20(msm_mdss);
 		break;
-	case DPU_HW_VER_630:
-		/* UBWC_2_0 */
-		msm_mdss_setup_ubwc_dec_20(msm_mdss, 0x11f);
+	case UBWC_3_0:
+		msm_mdss_setup_ubwc_dec_30(msm_mdss);
 		break;
-	case DPU_HW_VER_700:
-		/* TODO: highest_bank_bit = 2 for LP_DDR4 */
-		msm_mdss_setup_ubwc_dec_40(msm_mdss, UBWC_4_0, 6, 1, 3, 1);
+	case UBWC_4_0:
+		msm_mdss_setup_ubwc_dec_40(msm_mdss);
 		break;
-	case DPU_HW_VER_720:
-		msm_mdss_setup_ubwc_dec_40(msm_mdss, UBWC_3_0, 6, 1, 1, 1);
-		break;
-	case DPU_HW_VER_800:
-		msm_mdss_setup_ubwc_dec_40(msm_mdss, UBWC_4_0, 6, 1, 2, 1);
-		break;
-	case DPU_HW_VER_810:
-	case DPU_HW_VER_900:
-		/* TODO: highest_bank_bit = 2 for LP_DDR4 */
-		msm_mdss_setup_ubwc_dec_40(msm_mdss, UBWC_4_0, 6, 1, 3, 1);
+	default:
+		dev_err(msm_mdss->dev, "Unsupported UBWC decoder version %x\n",
+			msm_mdss->mdss_data->ubwc_dec_version);
+		dev_err(msm_mdss->dev, "HW_REV: 0x%x\n",
+			readl_relaxed(msm_mdss->mmio + HW_REV));
+		dev_err(msm_mdss->dev, "UBWC_DEC_HW_VERSION: 0x%x\n",
+			readl_relaxed(msm_mdss->mmio + UBWC_DEC_HW_VERSION));
 		break;
 	}
 
@@ -490,6 +470,8 @@ static int mdss_probe(struct platform_device *pdev)
 	if (IS_ERR(mdss))
 		return PTR_ERR(mdss);
 
+	mdss->mdss_data = of_device_get_match_data(&pdev->dev);
+
 	platform_set_drvdata(pdev, mdss);
 
 	/*
@@ -519,21 +501,81 @@ static int mdss_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct msm_mdss_data sc7180_data = {
+	.ubwc_version = UBWC_2_0,
+	.ubwc_dec_version = UBWC_2_0,
+	.ubwc_static = 0x1e,
+};
+
+static const struct msm_mdss_data sc7280_data = {
+	.ubwc_version = UBWC_3_0,
+	.ubwc_dec_version = UBWC_4_0,
+	.ubwc_swizzle = 6,
+	.ubwc_static = 1,
+	.highest_bank_bit = 1,
+	.macrotile_mode = 1,
+};
+
+static const struct msm_mdss_data sc8180x_data = {
+	.ubwc_version = UBWC_3_0,
+	.ubwc_dec_version = UBWC_3_0,
+	.highest_bank_bit = 3,
+	.macrotile_mode = 1,
+};
+
+static const struct msm_mdss_data sc8280xp_data = {
+	.ubwc_version = UBWC_4_0,
+	.ubwc_dec_version = UBWC_4_0,
+	.ubwc_swizzle = 6,
+	.ubwc_static = 1,
+	.highest_bank_bit = 2,
+	.macrotile_mode = 1,
+};
+
+static const struct msm_mdss_data sdm845_data = {
+	.ubwc_version = UBWC_2_0,
+	.ubwc_dec_version = UBWC_2_0,
+	.highest_bank_bit = 2,
+};
+
+static const struct msm_mdss_data sm8150_data = {
+	.ubwc_version = UBWC_3_0,
+	.ubwc_dec_version = UBWC_3_0,
+	.highest_bank_bit = 2,
+};
+
+static const struct msm_mdss_data sm6115_data = {
+	.ubwc_version = UBWC_1_0,
+	.ubwc_dec_version = UBWC_2_0,
+	.ubwc_swizzle = 7,
+	.ubwc_static = 0x11f,
+};
+
+static const struct msm_mdss_data sm8250_data = {
+	.ubwc_version = UBWC_4_0,
+	.ubwc_dec_version = UBWC_4_0,
+	.ubwc_swizzle = 6,
+	.ubwc_static = 1,
+	/* TODO: highest_bank_bit = 2 for LP_DDR4 */
+	.highest_bank_bit = 3,
+	.macrotile_mode = 1,
+};
+
 static const struct of_device_id mdss_dt_match[] = {
 	{ .compatible = "qcom,mdss" },
 	{ .compatible = "qcom,msm8998-mdss" },
 	{ .compatible = "qcom,qcm2290-mdss" },
-	{ .compatible = "qcom,sdm845-mdss" },
-	{ .compatible = "qcom,sc7180-mdss" },
-	{ .compatible = "qcom,sc7280-mdss" },
-	{ .compatible = "qcom,sc8180x-mdss" },
-	{ .compatible = "qcom,sc8280xp-mdss" },
-	{ .compatible = "qcom,sm6115-mdss" },
-	{ .compatible = "qcom,sm8150-mdss" },
-	{ .compatible = "qcom,sm8250-mdss" },
-	{ .compatible = "qcom,sm8350-mdss" },
-	{ .compatible = "qcom,sm8450-mdss" },
-	{ .compatible = "qcom,sm8550-mdss" },
+	{ .compatible = "qcom,sdm845-mdss", .data = &sdm845_data },
+	{ .compatible = "qcom,sc7180-mdss", .data = &sc7180_data },
+	{ .compatible = "qcom,sc7280-mdss", .data = &sc7280_data },
+	{ .compatible = "qcom,sc8180x-mdss", .data = &sc8180x_data },
+	{ .compatible = "qcom,sc8280xp-mdss", .data = &sc8280xp_data },
+	{ .compatible = "qcom,sm6115-mdss", .data = &sm6115_data },
+	{ .compatible = "qcom,sm8150-mdss", .data = &sm8150_data },
+	{ .compatible = "qcom,sm8250-mdss", .data = &sm8250_data },
+	{ .compatible = "qcom,sm8350-mdss", .data = &sm8250_data },
+	{ .compatible = "qcom,sm8450-mdss", .data = &sm8250_data },
+	{ .compatible = "qcom,sm8550-mdss", .data = &sm8250_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, mdss_dt_match);
diff --git a/drivers/gpu/drm/msm/msm_mmu.h b/drivers/gpu/drm/msm/msm_mmu.h
index 74cd81e701ff..eb72d3645c1d 100644
--- a/drivers/gpu/drm/msm/msm_mmu.h
+++ b/drivers/gpu/drm/msm/msm_mmu.h
@@ -41,6 +41,7 @@ static inline void msm_mmu_init(struct msm_mmu *mmu, struct device *dev,
 }
 
 struct msm_mmu *msm_iommu_new(struct device *dev, unsigned long quirks);
+struct msm_mmu *msm_iommu_gpu_new(struct device *dev, struct msm_gpu *gpu, unsigned long quirks);
 struct msm_mmu *msm_gpummu_new(struct device *dev, struct msm_gpu *gpu);
 
 static inline void msm_mmu_set_fault_handler(struct msm_mmu *mmu, void *arg,
diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c
index 57a8e9564540..b60199184409 100644
--- a/drivers/gpu/drm/msm/msm_ringbuffer.c
+++ b/drivers/gpu/drm/msm/msm_ringbuffer.c
@@ -18,16 +18,14 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)
 	struct msm_gpu *gpu = submit->gpu;
 	int i;
 
-	submit->hw_fence = msm_fence_alloc(fctx);
+	msm_fence_init(submit->hw_fence, fctx);
 
 	for (i = 0; i < submit->nr_bos; i++) {
 		struct drm_gem_object *obj = &submit->bos[i].obj->base;
 
-		msm_gem_lock(obj);
-		msm_gem_unpin_vma_fenced(submit->bos[i].vma, fctx);
-		msm_gem_unpin_locked(obj);
+		msm_gem_vma_unpin_fenced(submit->bos[i].vma, fctx);
+		msm_gem_unpin_active(obj);
 		submit->bos[i].flags &= ~(BO_VMA_PINNED | BO_OBJ_PINNED);
-		msm_gem_unlock(obj);
 	}
 
 	/* TODO move submit path over to using a per-ring lock.. */
diff --git a/drivers/gpu/drm/msm/msm_submitqueue.c b/drivers/gpu/drm/msm/msm_submitqueue.c
index c6929e205b51..0e803125a325 100644
--- a/drivers/gpu/drm/msm/msm_submitqueue.c
+++ b/drivers/gpu/drm/msm/msm_submitqueue.c
@@ -200,7 +200,7 @@ int msm_submitqueue_create(struct drm_device *drm, struct msm_file_private *ctx,
 		*id = queue->id;
 
 	idr_init(&queue->fence_idr);
-	mutex_init(&queue->idr_lock);
+	spin_lock_init(&queue->idr_lock);
 	mutex_init(&queue->lock);
 
 	list_add_tail(&queue->node, &ctx->submitqueues);
diff --git a/drivers/gpu/drm/mxsfb/lcdif_drv.c b/drivers/gpu/drm/mxsfb/lcdif_drv.c
index cc2ceb301b96..6fb5b469ee5a 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_drv.c
+++ b/drivers/gpu/drm/mxsfb/lcdif_drv.c
@@ -16,7 +16,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_mode_config.h>
@@ -220,7 +220,7 @@ static int lcdif_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_unload;
 
-	drm_fbdev_generic_setup(drm, 32);
+	drm_fbdev_dma_setup(drm, 32);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_drv.c b/drivers/gpu/drm/mxsfb/mxsfb_drv.c
index b3ab86ad1b36..368b1fbd8305 100644
--- a/drivers/gpu/drm/mxsfb/mxsfb_drv.c
+++ b/drivers/gpu/drm/mxsfb/mxsfb_drv.c
@@ -20,7 +20,7 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_connector.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -365,7 +365,7 @@ static int mxsfb_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_unload;
 
-	drm_fbdev_generic_setup(drm, 32);
+	drm_fbdev_dma_setup(drm, 32);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 288eebc70a67..c2ec91cc845d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -1015,9 +1015,6 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict,
 	if (ret)
 		goto out_ntfy;
 
-	if (nvbo->bo.pin_count)
-		NV_WARN(drm, "Moving pinned object %p!\n", nvbo);
-
 	if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
 		ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile);
 		if (ret)
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index f77e44958037..ab9062e50977 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -645,7 +645,7 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
 				struct drm_nouveau_gem_pushbuf_reloc *reloc,
 				struct drm_nouveau_gem_pushbuf_bo *bo)
 {
-	long ret = 0;
+	int ret = 0;
 	unsigned i;
 
 	for (i = 0; i < req->nr_relocs; i++) {
@@ -653,6 +653,7 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
 		struct drm_nouveau_gem_pushbuf_bo *b;
 		struct nouveau_bo *nvbo;
 		uint32_t data;
+		long lret;
 
 		if (unlikely(r->bo_index >= req->nr_buffers)) {
 			NV_PRINTK(err, cli, "reloc bo index invalid\n");
@@ -703,13 +704,18 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
 				data |= r->vor;
 		}
 
-		ret = dma_resv_wait_timeout(nvbo->bo.base.resv,
-					    DMA_RESV_USAGE_BOOKKEEP,
-					    false, 15 * HZ);
-		if (ret == 0)
+		lret = dma_resv_wait_timeout(nvbo->bo.base.resv,
+					     DMA_RESV_USAGE_BOOKKEEP,
+					     false, 15 * HZ);
+		if (!lret)
 			ret = -EBUSY;
+		else if (lret > 0)
+			ret = 0;
+		else
+			ret = lret;
+
 		if (ret) {
-			NV_PRINTK(err, cli, "reloc wait_idle failed: %ld\n",
+			NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n",
 				  ret);
 			break;
 		}
diff --git a/drivers/gpu/drm/nouveau/nouveau_hwmon.c b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
index a7db7c31064b..e844be49e11e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hwmon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
@@ -41,7 +41,7 @@ static ssize_t
 nouveau_hwmon_show_temp1_auto_point1_pwm(struct device *d,
 					 struct device_attribute *a, char *buf)
 {
-	return snprintf(buf, PAGE_SIZE, "%d\n", 100);
+	return sysfs_emit(buf, "%d\n", 100);
 }
 static SENSOR_DEVICE_ATTR(temp1_auto_point1_pwm, 0444,
 			  nouveau_hwmon_show_temp1_auto_point1_pwm, NULL, 0);
@@ -54,8 +54,8 @@ nouveau_hwmon_temp1_auto_point1_temp(struct device *d,
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nvkm_therm *therm = nvxx_therm(&drm->client.device);
 
-	return snprintf(buf, PAGE_SIZE, "%d\n",
-	      therm->attr_get(therm, NVKM_THERM_ATTR_THRS_FAN_BOOST) * 1000);
+	return sysfs_emit(buf, "%d\n",
+			  therm->attr_get(therm, NVKM_THERM_ATTR_THRS_FAN_BOOST) * 1000);
 }
 static ssize_t
 nouveau_hwmon_set_temp1_auto_point1_temp(struct device *d,
@@ -87,8 +87,8 @@ nouveau_hwmon_temp1_auto_point1_temp_hyst(struct device *d,
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nvkm_therm *therm = nvxx_therm(&drm->client.device);
 
-	return snprintf(buf, PAGE_SIZE, "%d\n",
-	 therm->attr_get(therm, NVKM_THERM_ATTR_THRS_FAN_BOOST_HYST) * 1000);
+	return sysfs_emit(buf, "%d\n",
+			  therm->attr_get(therm, NVKM_THERM_ATTR_THRS_FAN_BOOST_HYST) * 1000);
 }
 static ssize_t
 nouveau_hwmon_set_temp1_auto_point1_temp_hyst(struct device *d,
diff --git a/drivers/gpu/drm/nouveau/nouveau_led.h b/drivers/gpu/drm/nouveau/nouveau_led.h
index 21a5775028cc..bc9bc7208da3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_led.h
+++ b/drivers/gpu/drm/nouveau/nouveau_led.h
@@ -27,7 +27,7 @@
 
 #include "nouveau_drv.h"
 
-struct led_classdev;
+#include <linux/leds.h>
 
 struct nouveau_led {
 	struct drm_device *dev;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c
index 115d0997fd62..4ebc030e40d1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c
@@ -606,7 +606,7 @@ gv100_disp_curs = {
 	.user = 73,
 };
 
-const struct nvkm_disp_mthd_list
+static const struct nvkm_disp_mthd_list
 gv100_disp_core_mthd_base = {
 	.mthd = 0x0000,
 	.addr = 0x000000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
index 5bb65258c36d..6c94451d0faa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
@@ -447,7 +447,7 @@ gf100_fifo_nonstall_allow(struct nvkm_event *event, int type, int index)
 	spin_unlock_irqrestore(&fifo->lock, flags);
 }
 
-void
+static void
 gf100_fifo_nonstall_block(struct nvkm_event *event, int type, int index)
 {
 	struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c
index b5836cbc29aa..93d628d7d508 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c
@@ -30,7 +30,7 @@
 #include <subdev/timer.h>
 #include <subdev/top.h>
 
-struct nvkm_cgrp *
+static struct nvkm_cgrp *
 nvkm_engn_cgrp_get(struct nvkm_engn *engn, unsigned long *pirqflags)
 {
 	struct nvkm_cgrp *cgrp = NULL;
diff --git a/drivers/gpu/drm/nouveau/nvkm/nvfw/acr.c b/drivers/gpu/drm/nouveau/nvkm/nvfw/acr.c
index 83a9c48bc58c..7ac90c495737 100644
--- a/drivers/gpu/drm/nouveau/nvkm/nvfw/acr.c
+++ b/drivers/gpu/drm/nouveau/nvkm/nvfw/acr.c
@@ -45,7 +45,7 @@ wpr_header_v1_dump(struct nvkm_subdev *subdev, const struct wpr_header_v1 *hdr)
 	nvkm_debug(subdev, "\tstatus        : %d\n", hdr->status);
 }
 
-void
+static void
 wpr_generic_header_dump(struct nvkm_subdev *subdev, const struct wpr_generic_header *hdr)
 {
 	nvkm_debug(subdev, "wprGenericHeader\n");
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c
index 1e2eabec1a76..5d28d30d09d5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c
@@ -47,7 +47,7 @@ ga100_mc_device_enabled(struct nvkm_mc *mc, u32 mask)
 	return (nvkm_rd32(mc->subdev.device, 0x000600) & mask) == mask;
 }
 
-const struct nvkm_mc_device_func
+static const struct nvkm_mc_device_func
 ga100_mc_device = {
 	.enabled = ga100_mc_device_enabled,
 	.enable = ga100_mc_device_enable,
diff --git a/drivers/gpu/drm/omapdrm/Kconfig b/drivers/gpu/drm/omapdrm/Kconfig
index 455e1a91f0e5..76ded1568bd0 100644
--- a/drivers/gpu/drm/omapdrm/Kconfig
+++ b/drivers/gpu/drm/omapdrm/Kconfig
@@ -2,7 +2,7 @@
 config DRM_OMAP
 	tristate "OMAP DRM"
 	depends on DRM && OF
-	depends on ARCH_OMAP2PLUS || ARCH_MULTIPLATFORM
+	depends on ARCH_OMAP2PLUS
 	select DRM_KMS_HELPER
 	select VIDEOMODE_HELPERS
 	select HDMI
diff --git a/drivers/gpu/drm/omapdrm/omap_debugfs.c b/drivers/gpu/drm/omapdrm/omap_debugfs.c
index bfb2ccb40bd1..a3d470468e5b 100644
--- a/drivers/gpu/drm/omapdrm/omap_debugfs.c
+++ b/drivers/gpu/drm/omapdrm/omap_debugfs.c
@@ -47,15 +47,15 @@ static int fb_show(struct seq_file *m, void *arg)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
-	struct omap_drm_private *priv = dev->dev_private;
+	struct drm_fb_helper *helper = dev->fb_helper;
 	struct drm_framebuffer *fb;
 
 	seq_printf(m, "fbcon ");
-	omap_framebuffer_describe(priv->fbdev->fb, m);
+	omap_framebuffer_describe(helper->fb, m);
 
 	mutex_lock(&dev->mode_config.fb_lock);
 	list_for_each_entry(fb, &dev->mode_config.fb_list, head) {
-		if (fb == priv->fbdev->fb)
+		if (fb == helper->fb)
 			continue;
 
 		seq_printf(m, "user ");
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index 699ed814e021..5ed549726104 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -6,6 +6,7 @@
 
 #include <linux/dma-mapping.h>
 #include <linux/platform_device.h>
+#include <linux/of.h>
 #include <linux/sort.h>
 #include <linux/sys_soc.h>
 
@@ -14,7 +15,6 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_bridge_connector.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_helper.h>
 #include <drm/drm_file.h>
 #include <drm/drm_ioctl.h>
 #include <drm/drm_panel.h>
@@ -24,6 +24,7 @@
 
 #include "omap_dmm_tiler.h"
 #include "omap_drv.h"
+#include "omap_fbdev.h"
 
 #define DRIVER_NAME		MODULE_NAME
 #define DRIVER_DESC		"OMAP DRM"
@@ -219,7 +220,6 @@ static const struct drm_mode_config_helper_funcs omap_mode_config_helper_funcs =
 
 static const struct drm_mode_config_funcs omap_mode_config_funcs = {
 	.fb_create = omap_framebuffer_create,
-	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = omap_atomic_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -652,7 +652,6 @@ static const struct drm_driver omap_drm_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM  |
 		DRIVER_ATOMIC | DRIVER_RENDER,
 	.open = dev_open,
-	.lastclose = drm_fb_helper_lastclose,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init = omap_debugfs_init,
 #endif
@@ -741,8 +740,6 @@ static int omapdrm_init(struct omap_drm_private *priv, struct device *dev)
 		goto err_cleanup_modeset;
 	}
 
-	omap_fbdev_init(ddev);
-
 	drm_kms_helper_poll_init(ddev);
 
 	/*
@@ -753,12 +750,12 @@ static int omapdrm_init(struct omap_drm_private *priv, struct device *dev)
 	if (ret)
 		goto err_cleanup_helpers;
 
+	omap_fbdev_setup(ddev);
+
 	return 0;
 
 err_cleanup_helpers:
 	drm_kms_helper_poll_fini(ddev);
-
-	omap_fbdev_fini(ddev);
 err_cleanup_modeset:
 	omap_modeset_fini(ddev);
 err_free_overlays:
@@ -784,8 +781,6 @@ static void omapdrm_cleanup(struct omap_drm_private *priv)
 
 	drm_kms_helper_poll_fini(ddev);
 
-	omap_fbdev_fini(ddev);
-
 	drm_atomic_helper_shutdown(ddev);
 
 	omap_modeset_fini(ddev);
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.h b/drivers/gpu/drm/omapdrm/omap_drv.h
index 825960fd3ea9..4c7217b35f6b 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.h
+++ b/drivers/gpu/drm/omapdrm/omap_drv.h
@@ -21,7 +21,6 @@
 #include "omap_crtc.h"
 #include "omap_encoder.h"
 #include "omap_fb.h"
-#include "omap_fbdev.h"
 #include "omap_gem.h"
 #include "omap_irq.h"
 #include "omap_plane.h"
@@ -77,8 +76,6 @@ struct omap_drm_private {
 
 	struct drm_private_obj glob_obj;
 
-	struct drm_fb_helper *fbdev;
-
 	struct workqueue_struct *wq;
 
 	/* lock for obj_list below */
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
index a6c8542087ec..b950e93b3846 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c
@@ -4,14 +4,17 @@
  * Author: Rob Clark <rob@ti.com>
  */
 
-#include <drm/drm_crtc.h>
-#include <drm/drm_util.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_file.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_util.h>
 
 #include "omap_drv.h"
+#include "omap_fbdev.h"
 
 MODULE_PARM_DESC(ywrap, "Enable ywrap scrolling (omap44xx and later, default 'y')");
 static bool ywrap_enabled = true;
@@ -25,8 +28,6 @@ module_param_named(ywrap, ywrap_enabled, bool, 0644);
 
 struct omap_fbdev {
 	struct drm_fb_helper base;
-	struct drm_framebuffer *fb;
-	struct drm_gem_object *bo;
 	bool ywrap_enabled;
 
 	/* for deferred dmm roll when getting called in atomic ctx */
@@ -38,12 +39,14 @@ static struct drm_fb_helper *get_fb(struct fb_info *fbi);
 static void pan_worker(struct work_struct *work)
 {
 	struct omap_fbdev *fbdev = container_of(work, struct omap_fbdev, work);
-	struct fb_info *fbi = fbdev->base.info;
+	struct drm_fb_helper *helper = &fbdev->base;
+	struct fb_info *fbi = helper->info;
+	struct drm_gem_object *bo = drm_gem_fb_get_obj(helper->fb, 0);
 	int npages;
 
 	/* DMM roll shifts in 4K pages: */
 	npages = fbi->fix.line_length >> PAGE_SHIFT;
-	omap_gem_roll(fbdev->bo, fbi->var.yoffset * npages);
+	omap_gem_roll(bo, fbi->var.yoffset * npages);
 }
 
 static int omap_fbdev_pan_display(struct fb_var_screeninfo *var,
@@ -71,6 +74,25 @@ fallback:
 	return drm_fb_helper_pan_display(var, fbi);
 }
 
+static void omap_fbdev_fb_destroy(struct fb_info *info)
+{
+	struct drm_fb_helper *helper = info->par;
+	struct drm_framebuffer *fb = helper->fb;
+	struct drm_gem_object *bo = drm_gem_fb_get_obj(fb, 0);
+	struct omap_fbdev *fbdev = to_omap_fbdev(helper);
+
+	DBG();
+
+	drm_fb_helper_fini(helper);
+
+	omap_gem_unpin(bo);
+	drm_framebuffer_remove(fb);
+
+	drm_client_release(&helper->client);
+	drm_fb_helper_unprepare(helper);
+	kfree(fbdev);
+}
+
 static const struct fb_ops omap_fb_ops = {
 	.owner = THIS_MODULE,
 
@@ -86,6 +108,8 @@ static const struct fb_ops omap_fb_ops = {
 	.fb_fillrect = drm_fb_helper_sys_fillrect,
 	.fb_copyarea = drm_fb_helper_sys_copyarea,
 	.fb_imageblit = drm_fb_helper_sys_imageblit,
+
+	.fb_destroy = omap_fbdev_fb_destroy,
 };
 
 static int omap_fbdev_create(struct drm_fb_helper *helper,
@@ -98,6 +122,7 @@ static int omap_fbdev_create(struct drm_fb_helper *helper,
 	union omap_gem_size gsize;
 	struct fb_info *fbi = NULL;
 	struct drm_mode_fb_cmd2 mode_cmd = {0};
+	struct drm_gem_object *bo;
 	dma_addr_t dma_addr;
 	int ret;
 
@@ -128,20 +153,20 @@ static int omap_fbdev_create(struct drm_fb_helper *helper,
 		.bytes = PAGE_ALIGN(mode_cmd.pitches[0] * mode_cmd.height),
 	};
 	DBG("allocating %d bytes for fb %d", gsize.bytes, dev->primary->index);
-	fbdev->bo = omap_gem_new(dev, gsize, OMAP_BO_SCANOUT | OMAP_BO_WC);
-	if (!fbdev->bo) {
+	bo = omap_gem_new(dev, gsize, OMAP_BO_SCANOUT | OMAP_BO_WC);
+	if (!bo) {
 		dev_err(dev->dev, "failed to allocate buffer object\n");
 		ret = -ENOMEM;
 		goto fail;
 	}
 
-	fb = omap_framebuffer_init(dev, &mode_cmd, &fbdev->bo);
+	fb = omap_framebuffer_init(dev, &mode_cmd, &bo);
 	if (IS_ERR(fb)) {
 		dev_err(dev->dev, "failed to allocate fb\n");
 		/* note: if fb creation failed, we can't rely on fb destroy
 		 * to unref the bo:
 		 */
-		drm_gem_object_put(fbdev->bo);
+		drm_gem_object_put(bo);
 		ret = PTR_ERR(fb);
 		goto fail;
 	}
@@ -154,7 +179,7 @@ static int omap_fbdev_create(struct drm_fb_helper *helper,
 	 * to it).  Then we just need to be sure that we are able to re-
 	 * pin it in case of an opps.
 	 */
-	ret = omap_gem_pin(fbdev->bo, &dma_addr);
+	ret = omap_gem_pin(bo, &dma_addr);
 	if (ret) {
 		dev_err(dev->dev, "could not pin framebuffer\n");
 		ret = -ENOMEM;
@@ -170,17 +195,16 @@ static int omap_fbdev_create(struct drm_fb_helper *helper,
 
 	DBG("fbi=%p, dev=%p", fbi, dev);
 
-	fbdev->fb = fb;
 	helper->fb = fb;
 
 	fbi->fbops = &omap_fb_ops;
 
 	drm_fb_helper_fill_info(fbi, helper, sizes);
 
-	fbi->screen_buffer = omap_gem_vaddr(fbdev->bo);
-	fbi->screen_size = fbdev->bo->size;
+	fbi->screen_buffer = omap_gem_vaddr(bo);
+	fbi->screen_size = bo->size;
 	fbi->fix.smem_start = dma_addr;
-	fbi->fix.smem_len = fbdev->bo->size;
+	fbi->fix.smem_len = bo->size;
 
 	/* if we have DMM, then we can use it for scrolling by just
 	 * shuffling pages around in DMM rather than doing sw blit.
@@ -193,7 +217,7 @@ static int omap_fbdev_create(struct drm_fb_helper *helper,
 
 
 	DBG("par=%p, %dx%d", fbi->par, fbi->var.xres, fbi->var.yres);
-	DBG("allocated %dx%d fb", fbdev->fb->width, fbdev->fb->height);
+	DBG("allocated %dx%d fb", fb->width, fb->height);
 
 	return 0;
 
@@ -220,75 +244,94 @@ static struct drm_fb_helper *get_fb(struct fb_info *fbi)
 	return fbi->par;
 }
 
-/* initialize fbdev helper */
-void omap_fbdev_init(struct drm_device *dev)
+/*
+ * struct drm_client
+ */
+
+static void omap_fbdev_client_unregister(struct drm_client_dev *client)
 {
-	struct omap_drm_private *priv = dev->dev_private;
-	struct omap_fbdev *fbdev = NULL;
-	struct drm_fb_helper *helper;
-	int ret = 0;
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
 
-	if (!priv->num_pipes)
-		return;
+	if (fb_helper->info) {
+		drm_fb_helper_unregister_info(fb_helper);
+	} else {
+		drm_client_release(&fb_helper->client);
+		drm_fb_helper_unprepare(fb_helper);
+		kfree(fb_helper);
+	}
+}
 
-	fbdev = kzalloc(sizeof(*fbdev), GFP_KERNEL);
-	if (!fbdev)
-		return;
+static int omap_fbdev_client_restore(struct drm_client_dev *client)
+{
+	drm_fb_helper_lastclose(client->dev);
 
-	INIT_WORK(&fbdev->work, pan_worker);
+	return 0;
+}
 
-	helper = &fbdev->base;
+static int omap_fbdev_client_hotplug(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+	struct drm_device *dev = client->dev;
+	int ret;
 
-	drm_fb_helper_prepare(dev, helper, 32, &omap_fb_helper_funcs);
+	if (dev->fb_helper)
+		return drm_fb_helper_hotplug_event(dev->fb_helper);
 
-	ret = drm_fb_helper_init(dev, helper);
+	ret = drm_fb_helper_init(dev, fb_helper);
 	if (ret)
-		goto fail;
+		goto err_drm_err;
 
-	ret = drm_fb_helper_initial_config(helper);
+	ret = drm_fb_helper_initial_config(fb_helper);
 	if (ret)
-		goto fini;
-
-	priv->fbdev = helper;
-
-	return;
+		goto err_drm_fb_helper_fini;
 
-fini:
-	drm_fb_helper_fini(helper);
-fail:
-	drm_fb_helper_unprepare(helper);
-	kfree(fbdev);
+	return 0;
 
-	dev_warn(dev->dev, "omap_fbdev_init failed\n");
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(fb_helper);
+err_drm_err:
+	drm_err(dev, "Failed to setup fbdev emulation (ret=%d)\n", ret);
+	return ret;
 }
 
-void omap_fbdev_fini(struct drm_device *dev)
+static const struct drm_client_funcs omap_fbdev_client_funcs = {
+	.owner		= THIS_MODULE,
+	.unregister	= omap_fbdev_client_unregister,
+	.restore	= omap_fbdev_client_restore,
+	.hotplug	= omap_fbdev_client_hotplug,
+};
+
+void omap_fbdev_setup(struct drm_device *dev)
 {
-	struct omap_drm_private *priv = dev->dev_private;
-	struct drm_fb_helper *helper = priv->fbdev;
 	struct omap_fbdev *fbdev;
+	struct drm_fb_helper *helper;
+	int ret;
 
-	DBG();
+	drm_WARN(dev, !dev->registered, "Device has not been registered.\n");
+	drm_WARN(dev, dev->fb_helper, "fb_helper is already set!\n");
 
-	if (!helper)
+	fbdev = kzalloc(sizeof(*fbdev), GFP_KERNEL);
+	if (!fbdev)
 		return;
+	helper = &fbdev->base;
 
-	drm_fb_helper_unregister_info(helper);
+	drm_fb_helper_prepare(dev, helper, 32, &omap_fb_helper_funcs);
 
-	drm_fb_helper_fini(helper);
+	ret = drm_client_init(dev, &helper->client, "fbdev", &omap_fbdev_client_funcs);
+	if (ret)
+		goto err_drm_client_init;
+
+	INIT_WORK(&fbdev->work, pan_worker);
 
-	fbdev = to_omap_fbdev(helper);
+	ret = omap_fbdev_client_hotplug(&helper->client);
+	if (ret)
+		drm_dbg_kms(dev, "client hotplug ret=%d\n", ret);
 
-	/* unpin the GEM object pinned in omap_fbdev_create() */
-	if (fbdev->bo)
-		omap_gem_unpin(fbdev->bo);
+	drm_client_register(&helper->client);
 
-	/* this will free the backing object */
-	if (fbdev->fb)
-		drm_framebuffer_remove(fbdev->fb);
+	return;
 
+err_drm_client_init:
 	drm_fb_helper_unprepare(helper);
 	kfree(fbdev);
-
-	priv->fbdev = NULL;
 }
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.h b/drivers/gpu/drm/omapdrm/omap_fbdev.h
index 74a68a5a6eab..74c691a8d45f 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.h
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.h
@@ -10,16 +10,11 @@
 #define __OMAPDRM_FBDEV_H__
 
 struct drm_device;
-struct drm_fb_helper;
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
-void omap_fbdev_init(struct drm_device *dev);
-void omap_fbdev_fini(struct drm_device *dev);
+void omap_fbdev_setup(struct drm_device *dev);
 #else
-static inline void omap_fbdev_init(struct drm_device *dev)
-{
-}
-static inline void omap_fbdev_fini(struct drm_device *dev)
+static inline void omap_fbdev_setup(struct drm_device *dev)
 {
 }
 #endif
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index 8eeee71c0000..2b9d6db7860b 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -318,6 +318,17 @@ config DRM_PANEL_LG_LG4573
 	  Say Y here if you want to enable support for LG4573 RGB panel.
 	  To compile this driver as a module, choose M here.
 
+config DRM_PANEL_MAGNACHIP_D53E6EA8966
+	tristate "Magnachip D53E6EA8966 DSI panel"
+	depends on OF && SPI
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_MIPI_DBI
+	help
+	  DRM panel driver for the Samsung AMS495QA01 panel controlled
+	  with the Magnachip D53E6EA8966 panel IC. This panel receives
+	  video data via DSI but commands via 9-bit SPI using DBI.
+
 config DRM_PANEL_NEC_NL8048HL11
 	tristate "NEC NL8048HL11 RGB panel"
 	depends on GPIOLIB && OF && SPI
@@ -377,6 +388,16 @@ config DRM_PANEL_NOVATEK_NT35950
 	  Sharp panels used in Sony Xperia Z5 Premium and XZ Premium
 	  mobile phones.
 
+config DRM_PANEL_NOVATEK_NT36523
+	tristate "Novatek NT36523 panel driver"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for the panels built
+	  around the Novatek NT36523 display controller, such as some
+	  Boe panels used in Xiaomi Mi Pad 5 and 5 Pro tablets.
+
 config DRM_PANEL_NOVATEK_NT36672A
 	tristate "Novatek NT36672A DSI panel"
 	depends on OF
@@ -685,6 +706,16 @@ config DRM_PANEL_SONY_ACX565AKM
 	  Say Y here if you want to enable support for the Sony ACX565AKM
 	  800x600 3.5" panel (found on the Nokia N900).
 
+config DRM_PANEL_SONY_TD4353_JDI
+	tristate "Sony TD4353 JDI panel"
+	depends on GPIOLIB && OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for the Sony Tama
+	  TD4353 JDI command mode panel as found on some Sony Xperia
+	  XZ2 and XZ2 Compact smartphones.
+
 config DRM_PANEL_SONY_TULIP_TRULY_NT35521
 	tristate "Sony Tulip Truly NT35521 panel"
 	depends on GPIOLIB && OF
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index c05aa9e23907..ff169781e82d 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -29,12 +29,14 @@ obj-$(CONFIG_DRM_PANEL_LEADTEK_LTK050H3146W) += panel-leadtek-ltk050h3146w.o
 obj-$(CONFIG_DRM_PANEL_LEADTEK_LTK500HD1829) += panel-leadtek-ltk500hd1829.o
 obj-$(CONFIG_DRM_PANEL_LG_LB035Q02) += panel-lg-lb035q02.o
 obj-$(CONFIG_DRM_PANEL_LG_LG4573) += panel-lg-lg4573.o
+obj-$(CONFIG_DRM_PANEL_MAGNACHIP_D53E6EA8966) += panel-magnachip-d53e6ea8966.o
 obj-$(CONFIG_DRM_PANEL_NEC_NL8048HL11) += panel-nec-nl8048hl11.o
 obj-$(CONFIG_DRM_PANEL_NEWVISION_NV3051D) += panel-newvision-nv3051d.o
 obj-$(CONFIG_DRM_PANEL_NEWVISION_NV3052C) += panel-newvision-nv3052c.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35510) += panel-novatek-nt35510.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35560) += panel-novatek-nt35560.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35950) += panel-novatek-nt35950.o
+obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36523) += panel-novatek-nt36523.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36672A) += panel-novatek-nt36672a.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT39016) += panel-novatek-nt39016.o
 obj-$(CONFIG_DRM_PANEL_MANTIX_MLAF057WE51) += panel-mantix-mlaf057we51.o
@@ -69,6 +71,7 @@ obj-$(CONFIG_DRM_PANEL_SITRONIX_ST7701) += panel-sitronix-st7701.o
 obj-$(CONFIG_DRM_PANEL_SITRONIX_ST7703) += panel-sitronix-st7703.o
 obj-$(CONFIG_DRM_PANEL_SITRONIX_ST7789V) += panel-sitronix-st7789v.o
 obj-$(CONFIG_DRM_PANEL_SONY_ACX565AKM) += panel-sony-acx565akm.o
+obj-$(CONFIG_DRM_PANEL_SONY_TD4353_JDI) += panel-sony-td4353-jdi.o
 obj-$(CONFIG_DRM_PANEL_SONY_TULIP_TRULY_NT35521) += panel-sony-tulip-truly-nt35521.o
 obj-$(CONFIG_DRM_PANEL_TDO_TL070WSH30) += panel-tdo-tl070wsh30.o
 obj-$(CONFIG_DRM_PANEL_TPO_TD028TTEC1) += panel-tpo-td028ttec1.o
diff --git a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
index c924f1124ebc..783234ae0f57 100644
--- a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
+++ b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
@@ -1131,6 +1131,103 @@ static const struct panel_init_cmd auo_b101uan08_3_init_cmd[] = {
 	{},
 };
 
+static const struct panel_init_cmd starry_qfh032011_53g_init_cmd[] = {
+	_INIT_DCS_CMD(0xB0, 0x01),
+	_INIT_DCS_CMD(0xC3, 0x4F),
+	_INIT_DCS_CMD(0xC4, 0x40),
+	_INIT_DCS_CMD(0xC5, 0x40),
+	_INIT_DCS_CMD(0xC6, 0x40),
+	_INIT_DCS_CMD(0xC7, 0x40),
+	_INIT_DCS_CMD(0xC8, 0x4D),
+	_INIT_DCS_CMD(0xC9, 0x52),
+	_INIT_DCS_CMD(0xCA, 0x51),
+	_INIT_DCS_CMD(0xCD, 0x5D),
+	_INIT_DCS_CMD(0xCE, 0x5B),
+	_INIT_DCS_CMD(0xCF, 0x4B),
+	_INIT_DCS_CMD(0xD0, 0x49),
+	_INIT_DCS_CMD(0xD1, 0x47),
+	_INIT_DCS_CMD(0xD2, 0x45),
+	_INIT_DCS_CMD(0xD3, 0x41),
+	_INIT_DCS_CMD(0xD7, 0x50),
+	_INIT_DCS_CMD(0xD8, 0x40),
+	_INIT_DCS_CMD(0xD9, 0x40),
+	_INIT_DCS_CMD(0xDA, 0x40),
+	_INIT_DCS_CMD(0xDB, 0x40),
+	_INIT_DCS_CMD(0xDC, 0x4E),
+	_INIT_DCS_CMD(0xDD, 0x52),
+	_INIT_DCS_CMD(0xDE, 0x51),
+	_INIT_DCS_CMD(0xE1, 0x5E),
+	_INIT_DCS_CMD(0xE2, 0x5C),
+	_INIT_DCS_CMD(0xE3, 0x4C),
+	_INIT_DCS_CMD(0xE4, 0x4A),
+	_INIT_DCS_CMD(0xE5, 0x48),
+	_INIT_DCS_CMD(0xE6, 0x46),
+	_INIT_DCS_CMD(0xE7, 0x42),
+	_INIT_DCS_CMD(0xB0, 0x03),
+	_INIT_DCS_CMD(0xBE, 0x03),
+	_INIT_DCS_CMD(0xCC, 0x44),
+	_INIT_DCS_CMD(0xC8, 0x07),
+	_INIT_DCS_CMD(0xC9, 0x05),
+	_INIT_DCS_CMD(0xCA, 0x42),
+	_INIT_DCS_CMD(0xCD, 0x3E),
+	_INIT_DCS_CMD(0xCF, 0x60),
+	_INIT_DCS_CMD(0xD2, 0x04),
+	_INIT_DCS_CMD(0xD3, 0x04),
+	_INIT_DCS_CMD(0xD4, 0x01),
+	_INIT_DCS_CMD(0xD5, 0x00),
+	_INIT_DCS_CMD(0xD6, 0x03),
+	_INIT_DCS_CMD(0xD7, 0x04),
+	_INIT_DCS_CMD(0xD9, 0x01),
+	_INIT_DCS_CMD(0xDB, 0x01),
+	_INIT_DCS_CMD(0xE4, 0xF0),
+	_INIT_DCS_CMD(0xE5, 0x0A),
+	_INIT_DCS_CMD(0xB0, 0x00),
+	_INIT_DCS_CMD(0xCC, 0x08),
+	_INIT_DCS_CMD(0xC2, 0x08),
+	_INIT_DCS_CMD(0xC4, 0x10),
+	_INIT_DCS_CMD(0xB0, 0x02),
+	_INIT_DCS_CMD(0xC0, 0x00),
+	_INIT_DCS_CMD(0xC1, 0x0A),
+	_INIT_DCS_CMD(0xC2, 0x20),
+	_INIT_DCS_CMD(0xC3, 0x24),
+	_INIT_DCS_CMD(0xC4, 0x23),
+	_INIT_DCS_CMD(0xC5, 0x29),
+	_INIT_DCS_CMD(0xC6, 0x23),
+	_INIT_DCS_CMD(0xC7, 0x1C),
+	_INIT_DCS_CMD(0xC8, 0x19),
+	_INIT_DCS_CMD(0xC9, 0x17),
+	_INIT_DCS_CMD(0xCA, 0x17),
+	_INIT_DCS_CMD(0xCB, 0x18),
+	_INIT_DCS_CMD(0xCC, 0x1A),
+	_INIT_DCS_CMD(0xCD, 0x1E),
+	_INIT_DCS_CMD(0xCE, 0x20),
+	_INIT_DCS_CMD(0xCF, 0x23),
+	_INIT_DCS_CMD(0xD0, 0x07),
+	_INIT_DCS_CMD(0xD1, 0x00),
+	_INIT_DCS_CMD(0xD2, 0x00),
+	_INIT_DCS_CMD(0xD3, 0x0A),
+	_INIT_DCS_CMD(0xD4, 0x13),
+	_INIT_DCS_CMD(0xD5, 0x1C),
+	_INIT_DCS_CMD(0xD6, 0x1A),
+	_INIT_DCS_CMD(0xD7, 0x13),
+	_INIT_DCS_CMD(0xD8, 0x17),
+	_INIT_DCS_CMD(0xD9, 0x1C),
+	_INIT_DCS_CMD(0xDA, 0x19),
+	_INIT_DCS_CMD(0xDB, 0x17),
+	_INIT_DCS_CMD(0xDC, 0x17),
+	_INIT_DCS_CMD(0xDD, 0x18),
+	_INIT_DCS_CMD(0xDE, 0x1A),
+	_INIT_DCS_CMD(0xDF, 0x1E),
+	_INIT_DCS_CMD(0xE0, 0x20),
+	_INIT_DCS_CMD(0xE1, 0x23),
+	_INIT_DCS_CMD(0xE2, 0x07),
+	_INIT_DCS_CMD(0X11),
+	_INIT_DELAY_CMD(120),
+	_INIT_DCS_CMD(0X29),
+	_INIT_DELAY_CMD(80),
+	{},
+};
+
 static inline struct boe_panel *to_boe_panel(struct drm_panel *panel)
 {
 	return container_of(panel, struct boe_panel, base);
@@ -1497,6 +1594,32 @@ static const struct panel_desc boe_tv105wum_nw0_desc = {
 	.init_cmds = boe_init_cmd,
 };
 
+static const struct drm_display_mode starry_qfh032011_53g_default_mode = {
+	.clock = 165731,
+	.hdisplay = 1200,
+	.hsync_start = 1200 + 100,
+	.hsync_end = 1200 + 100 + 10,
+	.htotal = 1200 + 100 + 10 + 100,
+	.vdisplay = 1920,
+	.vsync_start = 1920 + 14,
+	.vsync_end = 1920 + 14 + 10,
+	.vtotal = 1920 + 14 + 10 + 15,
+};
+
+static const struct panel_desc starry_qfh032011_53g_desc = {
+	.modes = &starry_qfh032011_53g_default_mode,
+	.bpc = 8,
+	.size = {
+		.width_mm = 135,
+		.height_mm = 216,
+	},
+	.lanes = 4,
+	.format = MIPI_DSI_FMT_RGB888,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
+		      MIPI_DSI_MODE_LPM,
+	.init_cmds = starry_qfh032011_53g_init_cmd,
+};
+
 static int boe_panel_get_modes(struct drm_panel *panel,
 			       struct drm_connector *connector)
 {
@@ -1667,6 +1790,9 @@ static const struct of_device_id boe_of_match[] = {
 	{ .compatible = "innolux,hj110iz-01a",
 	  .data = &inx_hj110iz_desc
 	},
+	{ .compatible = "starry,2081101qfh032011-53g",
+	  .data = &starry_qfh032011_53g_desc
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, boe_of_match);
diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c
index 01bfe0783304..e23ddab2126e 100644
--- a/drivers/gpu/drm/panel/panel-edp.c
+++ b/drivers/gpu/drm/panel/panel-edp.c
@@ -1871,6 +1871,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x1e9b, &delay_200_500_e50, "B133UAN02.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x1ea5, &delay_200_500_e50, "B116XAK01.6"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x582d, &delay_200_500_e50, "B133UAN01.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"),
 
@@ -1878,6 +1879,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x07d1, &boe_nv133fhm_n61.delay, "NV133FHM-N61"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x082d, &boe_nv133fhm_n61.delay, "NV133FHM-N62"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x094b, &delay_200_500_e50, "NT116WHM-N21"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x095f, &delay_200_500_e50, "NE135FBM-N41 v8.1"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x098d, &boe_nv110wtm_n61.delay, "NV110WTM-N61"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x09dd, &delay_200_500_e50, "NT116WHM-N21"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a5d, &delay_200_500_e50, "NV116WHM-N45"),
diff --git a/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c b/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
index 48c1702a863b..323c33c9c37a 100644
--- a/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
+++ b/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
@@ -167,6 +167,202 @@ static const struct drm_panel_funcs jadard_funcs = {
 	.get_modes = jadard_get_modes,
 };
 
+static const struct jadard_init_cmd radxa_display_8hd_ad002_init_cmds[] = {
+	{ .data = { 0xE0, 0x00 } },
+	{ .data = { 0xE1, 0x93 } },
+	{ .data = { 0xE2, 0x65 } },
+	{ .data = { 0xE3, 0xF8 } },
+	{ .data = { 0x80, 0x03 } },
+	{ .data = { 0xE0, 0x01 } },
+	{ .data = { 0x00, 0x00 } },
+	{ .data = { 0x01, 0x7E } },
+	{ .data = { 0x03, 0x00 } },
+	{ .data = { 0x04, 0x65 } },
+	{ .data = { 0x0C, 0x74 } },
+	{ .data = { 0x17, 0x00 } },
+	{ .data = { 0x18, 0xB7 } },
+	{ .data = { 0x19, 0x00 } },
+	{ .data = { 0x1A, 0x00 } },
+	{ .data = { 0x1B, 0xB7 } },
+	{ .data = { 0x1C, 0x00 } },
+	{ .data = { 0x24, 0xFE } },
+	{ .data = { 0x37, 0x19 } },
+	{ .data = { 0x38, 0x05 } },
+	{ .data = { 0x39, 0x00 } },
+	{ .data = { 0x3A, 0x01 } },
+	{ .data = { 0x3B, 0x01 } },
+	{ .data = { 0x3C, 0x70 } },
+	{ .data = { 0x3D, 0xFF } },
+	{ .data = { 0x3E, 0xFF } },
+	{ .data = { 0x3F, 0xFF } },
+	{ .data = { 0x40, 0x06 } },
+	{ .data = { 0x41, 0xA0 } },
+	{ .data = { 0x43, 0x1E } },
+	{ .data = { 0x44, 0x0F } },
+	{ .data = { 0x45, 0x28 } },
+	{ .data = { 0x4B, 0x04 } },
+	{ .data = { 0x55, 0x02 } },
+	{ .data = { 0x56, 0x01 } },
+	{ .data = { 0x57, 0xA9 } },
+	{ .data = { 0x58, 0x0A } },
+	{ .data = { 0x59, 0x0A } },
+	{ .data = { 0x5A, 0x37 } },
+	{ .data = { 0x5B, 0x19 } },
+	{ .data = { 0x5D, 0x78 } },
+	{ .data = { 0x5E, 0x63 } },
+	{ .data = { 0x5F, 0x54 } },
+	{ .data = { 0x60, 0x49 } },
+	{ .data = { 0x61, 0x45 } },
+	{ .data = { 0x62, 0x38 } },
+	{ .data = { 0x63, 0x3D } },
+	{ .data = { 0x64, 0x28 } },
+	{ .data = { 0x65, 0x43 } },
+	{ .data = { 0x66, 0x41 } },
+	{ .data = { 0x67, 0x43 } },
+	{ .data = { 0x68, 0x62 } },
+	{ .data = { 0x69, 0x50 } },
+	{ .data = { 0x6A, 0x57 } },
+	{ .data = { 0x6B, 0x49 } },
+	{ .data = { 0x6C, 0x44 } },
+	{ .data = { 0x6D, 0x37 } },
+	{ .data = { 0x6E, 0x23 } },
+	{ .data = { 0x6F, 0x10 } },
+	{ .data = { 0x70, 0x78 } },
+	{ .data = { 0x71, 0x63 } },
+	{ .data = { 0x72, 0x54 } },
+	{ .data = { 0x73, 0x49 } },
+	{ .data = { 0x74, 0x45 } },
+	{ .data = { 0x75, 0x38 } },
+	{ .data = { 0x76, 0x3D } },
+	{ .data = { 0x77, 0x28 } },
+	{ .data = { 0x78, 0x43 } },
+	{ .data = { 0x79, 0x41 } },
+	{ .data = { 0x7A, 0x43 } },
+	{ .data = { 0x7B, 0x62 } },
+	{ .data = { 0x7C, 0x50 } },
+	{ .data = { 0x7D, 0x57 } },
+	{ .data = { 0x7E, 0x49 } },
+	{ .data = { 0x7F, 0x44 } },
+	{ .data = { 0x80, 0x37 } },
+	{ .data = { 0x81, 0x23 } },
+	{ .data = { 0x82, 0x10 } },
+	{ .data = { 0xE0, 0x02 } },
+	{ .data = { 0x00, 0x47 } },
+	{ .data = { 0x01, 0x47 } },
+	{ .data = { 0x02, 0x45 } },
+	{ .data = { 0x03, 0x45 } },
+	{ .data = { 0x04, 0x4B } },
+	{ .data = { 0x05, 0x4B } },
+	{ .data = { 0x06, 0x49 } },
+	{ .data = { 0x07, 0x49 } },
+	{ .data = { 0x08, 0x41 } },
+	{ .data = { 0x09, 0x1F } },
+	{ .data = { 0x0A, 0x1F } },
+	{ .data = { 0x0B, 0x1F } },
+	{ .data = { 0x0C, 0x1F } },
+	{ .data = { 0x0D, 0x1F } },
+	{ .data = { 0x0E, 0x1F } },
+	{ .data = { 0x0F, 0x5F } },
+	{ .data = { 0x10, 0x5F } },
+	{ .data = { 0x11, 0x57 } },
+	{ .data = { 0x12, 0x77 } },
+	{ .data = { 0x13, 0x35 } },
+	{ .data = { 0x14, 0x1F } },
+	{ .data = { 0x15, 0x1F } },
+	{ .data = { 0x16, 0x46 } },
+	{ .data = { 0x17, 0x46 } },
+	{ .data = { 0x18, 0x44 } },
+	{ .data = { 0x19, 0x44 } },
+	{ .data = { 0x1A, 0x4A } },
+	{ .data = { 0x1B, 0x4A } },
+	{ .data = { 0x1C, 0x48 } },
+	{ .data = { 0x1D, 0x48 } },
+	{ .data = { 0x1E, 0x40 } },
+	{ .data = { 0x1F, 0x1F } },
+	{ .data = { 0x20, 0x1F } },
+	{ .data = { 0x21, 0x1F } },
+	{ .data = { 0x22, 0x1F } },
+	{ .data = { 0x23, 0x1F } },
+	{ .data = { 0x24, 0x1F } },
+	{ .data = { 0x25, 0x5F } },
+	{ .data = { 0x26, 0x5F } },
+	{ .data = { 0x27, 0x57 } },
+	{ .data = { 0x28, 0x77 } },
+	{ .data = { 0x29, 0x35 } },
+	{ .data = { 0x2A, 0x1F } },
+	{ .data = { 0x2B, 0x1F } },
+	{ .data = { 0x58, 0x40 } },
+	{ .data = { 0x59, 0x00 } },
+	{ .data = { 0x5A, 0x00 } },
+	{ .data = { 0x5B, 0x10 } },
+	{ .data = { 0x5C, 0x06 } },
+	{ .data = { 0x5D, 0x40 } },
+	{ .data = { 0x5E, 0x01 } },
+	{ .data = { 0x5F, 0x02 } },
+	{ .data = { 0x60, 0x30 } },
+	{ .data = { 0x61, 0x01 } },
+	{ .data = { 0x62, 0x02 } },
+	{ .data = { 0x63, 0x03 } },
+	{ .data = { 0x64, 0x6B } },
+	{ .data = { 0x65, 0x05 } },
+	{ .data = { 0x66, 0x0C } },
+	{ .data = { 0x67, 0x73 } },
+	{ .data = { 0x68, 0x09 } },
+	{ .data = { 0x69, 0x03 } },
+	{ .data = { 0x6A, 0x56 } },
+	{ .data = { 0x6B, 0x08 } },
+	{ .data = { 0x6C, 0x00 } },
+	{ .data = { 0x6D, 0x04 } },
+	{ .data = { 0x6E, 0x04 } },
+	{ .data = { 0x6F, 0x88 } },
+	{ .data = { 0x70, 0x00 } },
+	{ .data = { 0x71, 0x00 } },
+	{ .data = { 0x72, 0x06 } },
+	{ .data = { 0x73, 0x7B } },
+	{ .data = { 0x74, 0x00 } },
+	{ .data = { 0x75, 0xF8 } },
+	{ .data = { 0x76, 0x00 } },
+	{ .data = { 0x77, 0xD5 } },
+	{ .data = { 0x78, 0x2E } },
+	{ .data = { 0x79, 0x12 } },
+	{ .data = { 0x7A, 0x03 } },
+	{ .data = { 0x7B, 0x00 } },
+	{ .data = { 0x7C, 0x00 } },
+	{ .data = { 0x7D, 0x03 } },
+	{ .data = { 0x7E, 0x7B } },
+	{ .data = { 0xE0, 0x04 } },
+	{ .data = { 0x00, 0x0E } },
+	{ .data = { 0x02, 0xB3 } },
+	{ .data = { 0x09, 0x60 } },
+	{ .data = { 0x0E, 0x2A } },
+	{ .data = { 0x36, 0x59 } },
+	{ .data = { 0xE0, 0x00 } },
+};
+
+static const struct jadard_panel_desc radxa_display_8hd_ad002_desc = {
+	.mode = {
+		.clock		= 70000,
+
+		.hdisplay	= 800,
+		.hsync_start	= 800 + 40,
+		.hsync_end	= 800 + 40 + 18,
+		.htotal		= 800 + 40 + 18 + 20,
+
+		.vdisplay	= 1280,
+		.vsync_start	= 1280 + 20,
+		.vsync_end	= 1280 + 20 + 4,
+		.vtotal		= 1280 + 20 + 4 + 20,
+
+		.width_mm	= 127,
+		.height_mm	= 199,
+		.type		= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+	},
+	.lanes = 4,
+	.format = MIPI_DSI_FMT_RGB888,
+	.init_cmds = radxa_display_8hd_ad002_init_cmds,
+	.num_init_cmds = ARRAY_SIZE(radxa_display_8hd_ad002_init_cmds),
+};
+
 static const struct jadard_init_cmd cz101b4001_init_cmds[] = {
 	{ .data = { 0xE0, 0x00 } },
 	{ .data = { 0xE1, 0x93 } },
@@ -452,7 +648,18 @@ static void jadard_dsi_remove(struct mipi_dsi_device *dsi)
 }
 
 static const struct of_device_id jadard_of_match[] = {
-	{ .compatible = "chongzhou,cz101b4001", .data = &cz101b4001_desc },
+	{
+		.compatible = "chongzhou,cz101b4001",
+		.data = &cz101b4001_desc
+	},
+	{
+		.compatible = "radxa,display-10hd-ad001",
+		.data = &cz101b4001_desc
+	},
+	{
+		.compatible = "radxa,display-8hd-ad002",
+		.data = &radxa_display_8hd_ad002_desc
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, jadard_of_match);
diff --git a/drivers/gpu/drm/panel/panel-magnachip-d53e6ea8966.c b/drivers/gpu/drm/panel/panel-magnachip-d53e6ea8966.c
new file mode 100644
index 000000000000..26d358b9b85a
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-magnachip-d53e6ea8966.c
@@ -0,0 +1,522 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Magnachip d53e6ea8966 MIPI-DSI panel driver
+ * Copyright (C) 2023 Chris Morgan
+ */
+
+#include <drm/drm_mipi_dbi.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/media-bus-format.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/spi/spi.h>
+
+#include <video/mipi_display.h>
+
+/* Forward declaration for use in backlight function */
+struct d53e6ea8966;
+
+/* Panel info, unique to each panel */
+struct d53e6ea8966_panel_info {
+	/** @display_modes: the supported display modes */
+	const struct drm_display_mode *display_modes;
+	/** @num_modes: the number of supported display modes */
+	unsigned int num_modes;
+	/** @width_mm: panel width in mm */
+	u16 width_mm;
+	/** @height_mm: panel height in mm */
+	u16 height_mm;
+	/** @bus_flags: drm bus flags for panel */
+	u32 bus_flags;
+	/** @panel_init_seq: panel specific init sequence */
+	void (*panel_init_seq)(struct d53e6ea8966 *db);
+	/** @backlight_register: panel backlight registration or NULL */
+	int (*backlight_register)(struct d53e6ea8966 *db);
+};
+
+struct d53e6ea8966 {
+	/** @dev: the container device */
+	struct device *dev;
+	/** @dbi: the DBI bus abstraction handle */
+	struct mipi_dbi dbi;
+	/** @panel: the DRM panel instance for this device */
+	struct drm_panel panel;
+	/** @reset: reset GPIO line */
+	struct gpio_desc *reset;
+	/** @enable: enable GPIO line */
+	struct gpio_desc *enable;
+	/** @reg_vdd: VDD supply regulator for panel logic */
+	struct regulator *reg_vdd;
+	/** @reg_elvdd: ELVDD supply regulator for panel display */
+	struct regulator *reg_elvdd;
+	/** @dsi_dev: DSI child device (panel) */
+	struct mipi_dsi_device *dsi_dev;
+	/** @bl_dev: pseudo-backlight device for oled panel */
+	struct backlight_device *bl_dev;
+	/** @panel_info: struct containing panel timing and info */
+	const struct d53e6ea8966_panel_info *panel_info;
+};
+
+#define NUM_GAMMA_LEVELS	16
+#define GAMMA_TABLE_COUNT	23
+#define MAX_BRIGHTNESS		(NUM_GAMMA_LEVELS - 1)
+
+#define MCS_ELVSS_ON			0xb1
+#define MCS_TEMP_SWIRE			0xb2
+#define MCS_PASSWORD_0			0xf0
+#define MCS_PASSWORD_1			0xf1
+#define MCS_ANALOG_PWR_CTL_0		0xf4
+#define MCS_ANALOG_PWR_CTL_1		0xf5
+#define MCS_GTCON_SET			0xf7
+#define MCS_GATELESS_SIGNAL_SET		0xf8
+#define MCS_SET_GAMMA			0xf9
+
+static inline struct d53e6ea8966 *to_d53e6ea8966(struct drm_panel *panel)
+{
+	return container_of(panel, struct d53e6ea8966, panel);
+}
+
+/* Table of gamma values provided in datasheet */
+static u8 ams495qa01_gamma[NUM_GAMMA_LEVELS][GAMMA_TABLE_COUNT] = {
+	{0x01, 0x79, 0x78, 0x8d, 0xd9, 0xdf, 0xd5, 0xcb, 0xcf, 0xc5,
+	 0xe5, 0xe0, 0xe4, 0xdc, 0xb8, 0xd4, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x7d, 0x7c, 0x92, 0xd7, 0xdd, 0xd2, 0xcb, 0xd0, 0xc6,
+	 0xe5, 0xe1, 0xe3, 0xda, 0xbd, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x7f, 0x7e, 0x95, 0xd7, 0xde, 0xd2, 0xcb, 0xcf, 0xc5,
+	 0xe5, 0xe3, 0xe3, 0xda, 0xbf, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x82, 0x81, 0x99, 0xd6, 0xdd, 0xd1, 0xca, 0xcf, 0xc3,
+	 0xe4, 0xe3, 0xe3, 0xda, 0xc2, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x84, 0x83, 0x9b, 0xd7, 0xde, 0xd2, 0xc8, 0xce, 0xc2,
+	 0xe4, 0xe3, 0xe2, 0xd9, 0xc3, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x87, 0x86, 0x9f, 0xd6, 0xdd, 0xd1, 0xc7, 0xce, 0xc1,
+	 0xe4, 0xe3, 0xe2, 0xd9, 0xc6, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x89, 0x89, 0xa2, 0xd5, 0xdb, 0xcf, 0xc8, 0xcf, 0xc2,
+	 0xe3, 0xe3, 0xe1, 0xd9, 0xc7, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x8b, 0x8b, 0xa5, 0xd5, 0xdb, 0xcf, 0xc7, 0xce, 0xc0,
+	 0xe3, 0xe3, 0xe1, 0xd8, 0xc7, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x8d, 0x8d, 0xa7, 0xd5, 0xdb, 0xcf, 0xc6, 0xce, 0xc0,
+	 0xe4, 0xe4, 0xe1, 0xd7, 0xc8, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x8f, 0x8f, 0xaa, 0xd4, 0xdb, 0xce, 0xc6, 0xcd, 0xbf,
+	 0xe3, 0xe3, 0xe1, 0xd7, 0xca, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x91, 0x91, 0xac, 0xd3, 0xda, 0xce, 0xc5, 0xcd, 0xbe,
+	 0xe3, 0xe3, 0xe0, 0xd7, 0xca, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x93, 0x93, 0xaf, 0xd3, 0xda, 0xcd, 0xc5, 0xcd, 0xbe,
+	 0xe2, 0xe3, 0xdf, 0xd6, 0xca, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x95, 0x95, 0xb1, 0xd2, 0xd9, 0xcc, 0xc4, 0xcd, 0xbe,
+	 0xe2, 0xe3, 0xdf, 0xd7, 0xcc, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x99, 0x99, 0xb6, 0xd1, 0xd9, 0xcc, 0xc3, 0xcb, 0xbc,
+	 0xe2, 0xe4, 0xdf, 0xd6, 0xcc, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x9c, 0x9c, 0xba, 0xd0, 0xd8, 0xcb, 0xc3, 0xcb, 0xbb,
+	 0xe2, 0xe4, 0xdf, 0xd6, 0xce, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+	{0x01, 0x9f, 0x9f, 0xbe, 0xcf, 0xd7, 0xc9, 0xc2, 0xcb, 0xbb,
+	 0xe1, 0xe3, 0xde, 0xd6, 0xd0, 0xd3, 0xfa, 0xed, 0xe6, 0x2f,
+	 0x00, 0x2f},
+};
+
+/*
+ * Table of elvss values provided in datasheet and corresponds to
+ * gamma values.
+ */
+static u8 ams495qa01_elvss[NUM_GAMMA_LEVELS] = {
+	0x15, 0x15, 0x15, 0x15, 0x15, 0x15, 0x15, 0x15, 0x15, 0x15,
+	0x15, 0x15, 0x14, 0x14, 0x13, 0x12,
+};
+
+static int ams495qa01_update_gamma(struct mipi_dbi *dbi, int brightness)
+{
+	int tmp = brightness;
+
+	mipi_dbi_command_buf(dbi, MCS_SET_GAMMA, ams495qa01_gamma[tmp],
+			     ARRAY_SIZE(ams495qa01_gamma[tmp]));
+	mipi_dbi_command(dbi, MCS_SET_GAMMA, 0x00);
+
+	/* Undocumented command */
+	mipi_dbi_command(dbi, 0x26, 0x00);
+
+	mipi_dbi_command(dbi, MCS_TEMP_SWIRE, ams495qa01_elvss[tmp]);
+
+	return 0;
+}
+
+static void ams495qa01_panel_init(struct d53e6ea8966 *db)
+{
+	struct mipi_dbi *dbi = &db->dbi;
+
+	mipi_dbi_command(dbi, MCS_PASSWORD_0, 0x5a, 0x5a);
+	mipi_dbi_command(dbi, MCS_PASSWORD_1, 0x5a, 0x5a);
+
+	/* Undocumented commands */
+	mipi_dbi_command(dbi, 0xb0, 0x02);
+	mipi_dbi_command(dbi, 0xf3, 0x3b);
+
+	mipi_dbi_command(dbi, MCS_ANALOG_PWR_CTL_0, 0x33, 0x42, 0x00, 0x08);
+	mipi_dbi_command(dbi, MCS_ANALOG_PWR_CTL_1, 0x00, 0x06, 0x26, 0x35, 0x03);
+
+	/* Undocumented commands */
+	mipi_dbi_command(dbi, 0xf6, 0x02);
+	mipi_dbi_command(dbi, 0xc6, 0x0b, 0x00, 0x00, 0x3c, 0x00, 0x22,
+			 0x00, 0x00, 0x00, 0x00);
+
+	mipi_dbi_command(dbi, MCS_GTCON_SET, 0x20);
+	mipi_dbi_command(dbi, MCS_TEMP_SWIRE, 0x06, 0x06, 0x06, 0x06);
+	mipi_dbi_command(dbi, MCS_ELVSS_ON, 0x07, 0x00, 0x10);
+	mipi_dbi_command(dbi, MCS_GATELESS_SIGNAL_SET, 0x7f, 0x7a,
+			 0x89, 0x67, 0x26, 0x38, 0x00, 0x00, 0x09,
+			 0x67, 0x70, 0x88, 0x7a, 0x76, 0x05, 0x09,
+			 0x23, 0x23, 0x23);
+
+	/* Undocumented commands */
+	mipi_dbi_command(dbi, 0xb5, 0xff, 0xef, 0x35, 0x42, 0x0d, 0xd7,
+			 0xff, 0x07, 0xff, 0xff, 0xfd, 0x00, 0x01,
+			 0xff, 0x05, 0x12, 0x0f, 0xff, 0xff, 0xff,
+			 0xff);
+	mipi_dbi_command(dbi, 0xb4, 0x15);
+	mipi_dbi_command(dbi, 0xb3, 0x00);
+
+	ams495qa01_update_gamma(dbi, MAX_BRIGHTNESS);
+}
+
+static int d53e6ea8966_prepare(struct drm_panel *panel)
+{
+	struct d53e6ea8966 *db = to_d53e6ea8966(panel);
+	int ret;
+
+	/* Power up */
+	ret = regulator_enable(db->reg_vdd);
+	if (ret) {
+		dev_err(db->dev, "failed to enable vdd regulator: %d\n", ret);
+		return ret;
+	}
+
+	if (db->reg_elvdd) {
+		ret = regulator_enable(db->reg_elvdd);
+		if (ret) {
+			dev_err(db->dev,
+				"failed to enable elvdd regulator: %d\n", ret);
+			regulator_disable(db->reg_vdd);
+			return ret;
+		}
+	}
+
+	/* Enable */
+	if (db->enable)
+		gpiod_set_value_cansleep(db->enable, 1);
+
+	msleep(50);
+
+	/* Reset */
+	gpiod_set_value_cansleep(db->reset, 1);
+	usleep_range(1000, 5000);
+	gpiod_set_value_cansleep(db->reset, 0);
+	msleep(20);
+
+	db->panel_info->panel_init_seq(db);
+
+	return 0;
+}
+
+static int d53e6ea8966_enable(struct drm_panel *panel)
+{
+	struct d53e6ea8966 *db = to_d53e6ea8966(panel);
+	struct mipi_dbi *dbi = &db->dbi;
+
+	mipi_dbi_command(dbi, MIPI_DCS_EXIT_SLEEP_MODE);
+	msleep(200);
+	mipi_dbi_command(dbi, MIPI_DCS_SET_DISPLAY_ON);
+	usleep_range(10000, 15000);
+
+	return 0;
+}
+
+static int d53e6ea8966_disable(struct drm_panel *panel)
+{
+	struct d53e6ea8966 *db = to_d53e6ea8966(panel);
+	struct mipi_dbi *dbi = &db->dbi;
+
+	mipi_dbi_command(dbi, MIPI_DCS_SET_DISPLAY_OFF);
+	msleep(20);
+	mipi_dbi_command(dbi, MIPI_DCS_ENTER_SLEEP_MODE);
+	msleep(100);
+
+	return 0;
+}
+
+static int d53e6ea8966_unprepare(struct drm_panel *panel)
+{
+	struct d53e6ea8966 *db = to_d53e6ea8966(panel);
+
+	if (db->enable)
+		gpiod_set_value_cansleep(db->enable, 0);
+
+	gpiod_set_value_cansleep(db->reset, 1);
+
+	if (db->reg_elvdd)
+		regulator_disable(db->reg_elvdd);
+
+	regulator_disable(db->reg_vdd);
+	msleep(100);
+
+	return 0;
+}
+
+static int d53e6ea8966_get_modes(struct drm_panel *panel,
+				 struct drm_connector *connector)
+{
+	struct d53e6ea8966 *db = to_d53e6ea8966(panel);
+	const struct d53e6ea8966_panel_info *panel_info = db->panel_info;
+	struct drm_display_mode *mode;
+	static const u32 bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+	unsigned int i;
+
+	for (i = 0; i < panel_info->num_modes; i++) {
+		mode = drm_mode_duplicate(connector->dev,
+					  &panel_info->display_modes[i]);
+		if (!mode)
+			return -ENOMEM;
+
+		drm_mode_set_name(mode);
+		drm_mode_probed_add(connector, mode);
+	}
+
+	connector->display_info.bpc = 8;
+	connector->display_info.width_mm = panel_info->width_mm;
+	connector->display_info.height_mm = panel_info->height_mm;
+	connector->display_info.bus_flags = panel_info->bus_flags;
+
+	drm_display_info_set_bus_formats(&connector->display_info,
+					 &bus_format, 1);
+
+	return 1;
+}
+
+static const struct drm_panel_funcs d53e6ea8966_panel_funcs = {
+	.disable = d53e6ea8966_disable,
+	.enable = d53e6ea8966_enable,
+	.get_modes = d53e6ea8966_get_modes,
+	.prepare = d53e6ea8966_prepare,
+	.unprepare = d53e6ea8966_unprepare,
+};
+
+static int ams495qa01_set_brightness(struct backlight_device *bd)
+{
+	struct d53e6ea8966 *db = bl_get_data(bd);
+	struct mipi_dbi *dbi = &db->dbi;
+	int brightness = backlight_get_brightness(bd);
+
+	ams495qa01_update_gamma(dbi, brightness);
+
+	return 0;
+}
+
+static const struct backlight_ops ams495qa01_backlight_ops = {
+	.update_status	= ams495qa01_set_brightness,
+};
+
+static int ams495qa01_backlight_register(struct d53e6ea8966 *db)
+{
+	struct backlight_properties props = {
+		.type		= BACKLIGHT_RAW,
+		.brightness	= MAX_BRIGHTNESS,
+		.max_brightness = MAX_BRIGHTNESS,
+	};
+	struct device *dev = db->dev;
+	int ret = 0;
+
+	db->bl_dev = devm_backlight_device_register(dev, "panel", dev, db,
+						    &ams495qa01_backlight_ops,
+						    &props);
+	if (IS_ERR(db->bl_dev)) {
+		ret = PTR_ERR(db->bl_dev);
+		dev_err(dev, "error registering backlight device (%d)\n", ret);
+	}
+
+	return ret;
+}
+
+static int d53e6ea8966_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct mipi_dsi_host *dsi_host;
+	struct d53e6ea8966 *db;
+	int ret;
+	struct mipi_dsi_device_info info = {
+		.type = "d53e6ea8966",
+		.channel = 0,
+		.node = NULL,
+	};
+
+	db = devm_kzalloc(dev, sizeof(*db), GFP_KERNEL);
+	if (!db)
+		return -ENOMEM;
+
+	spi_set_drvdata(spi, db);
+
+	db->dev = dev;
+
+	db->panel_info = of_device_get_match_data(dev);
+	if (!db->panel_info)
+		return -EINVAL;
+
+	db->reg_vdd = devm_regulator_get(dev, "vdd");
+	if (IS_ERR(db->reg_vdd))
+		return dev_err_probe(dev, PTR_ERR(db->reg_vdd),
+				     "Failed to get vdd supply\n");
+
+	db->reg_elvdd = devm_regulator_get_optional(dev, "elvdd");
+	if (IS_ERR(db->reg_elvdd))
+		db->reg_elvdd = NULL;
+
+	db->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(db->reset)) {
+		ret = PTR_ERR(db->reset);
+		return dev_err_probe(dev, ret, "no RESET GPIO\n");
+	}
+
+	db->enable = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_LOW);
+	if (IS_ERR(db->enable)) {
+		ret = PTR_ERR(db->enable);
+		return dev_err_probe(dev, ret, "cannot get ENABLE GPIO\n");
+	}
+
+	ret = mipi_dbi_spi_init(spi, &db->dbi, NULL);
+	if (ret)
+		return dev_err_probe(dev, ret, "MIPI DBI init failed\n");
+
+	dsi_host = drm_of_get_dsi_bus(dev);
+	if (IS_ERR(dsi_host)) {
+		ret = PTR_ERR(dsi_host);
+		return dev_err_probe(dev, ret, "Error attaching DSI bus\n");
+	}
+
+	db->dsi_dev = devm_mipi_dsi_device_register_full(dev, dsi_host, &info);
+	if (IS_ERR(db->dsi_dev)) {
+		dev_err(dev, "failed to register dsi device: %ld\n",
+			PTR_ERR(db->dsi_dev));
+		return PTR_ERR(db->dsi_dev);
+	}
+
+	db->dsi_dev->lanes = 2;
+	db->dsi_dev->format = MIPI_DSI_FMT_RGB888;
+	db->dsi_dev->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+			  MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_NO_EOT_PACKET;
+
+	drm_panel_init(&db->panel, dev, &d53e6ea8966_panel_funcs,
+		       DRM_MODE_CONNECTOR_DSI);
+
+	if (db->panel_info->backlight_register) {
+		ret = db->panel_info->backlight_register(db);
+		if (ret < 0)
+			return ret;
+		db->panel.backlight = db->bl_dev;
+	}
+
+	drm_panel_add(&db->panel);
+
+	ret = devm_mipi_dsi_attach(dev, db->dsi_dev);
+	if (ret < 0) {
+		dev_err(dev, "mipi_dsi_attach failed: %d\n", ret);
+		drm_panel_remove(&db->panel);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void d53e6ea8966_remove(struct spi_device *spi)
+{
+	struct d53e6ea8966 *db = spi_get_drvdata(spi);
+
+	drm_panel_remove(&db->panel);
+}
+
+static const struct drm_display_mode ams495qa01_modes[] = {
+	{ /* 60hz */
+		.clock = 33500,
+		.hdisplay = 960,
+		.hsync_start = 960 + 10,
+		.hsync_end = 960 + 10 + 2,
+		.htotal = 960 + 10 + 2 + 10,
+		.vdisplay = 544,
+		.vsync_start = 544 + 10,
+		.vsync_end = 544 + 10 + 2,
+		.vtotal = 544 + 10 + 2 + 10,
+		.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+		.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+		},
+	{ /* 50hz */
+		.clock = 27800,
+		.hdisplay = 960,
+		.hsync_start = 960 + 10,
+		.hsync_end = 960 + 10 + 2,
+		.htotal = 960 + 10 + 2 + 10,
+		.vdisplay = 544,
+		.vsync_start = 544 + 10,
+		.vsync_end = 544 + 10 + 2,
+		.vtotal = 544 + 10 + 2 + 10,
+		.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+		.type = DRM_MODE_TYPE_DRIVER,
+	},
+};
+
+static const struct d53e6ea8966_panel_info ams495qa01_info = {
+	.display_modes = ams495qa01_modes,
+	.num_modes = ARRAY_SIZE(ams495qa01_modes),
+	.width_mm = 117,
+	.height_mm = 74,
+	.bus_flags = DRM_BUS_FLAG_DE_LOW | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
+	.panel_init_seq = ams495qa01_panel_init,
+	.backlight_register = ams495qa01_backlight_register,
+};
+
+static const struct of_device_id d53e6ea8966_match[] = {
+	{ .compatible = "samsung,ams495qa01", .data = &ams495qa01_info },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, d53e6ea8966_match);
+
+static const struct spi_device_id d53e6ea8966_ids[] = {
+	{ "ams495qa01", 0 },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(spi, d53e6ea8966_ids);
+
+static struct spi_driver d53e6ea8966_driver = {
+	.driver		= {
+		.name	= "d53e6ea8966-panel",
+		.of_match_table = d53e6ea8966_match,
+	},
+	.id_table	= d53e6ea8966_ids,
+	.probe		= d53e6ea8966_probe,
+	.remove		= d53e6ea8966_remove,
+};
+module_spi_driver(d53e6ea8966_driver);
+
+MODULE_AUTHOR("Chris Morgan <macromorgan@hotmail.com>");
+MODULE_DESCRIPTION("Magnachip d53e6ea8966 panel driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt35950.c b/drivers/gpu/drm/panel/panel-novatek-nt35950.c
index abf752b36a52..8b108ac80b55 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt35950.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt35950.c
@@ -585,8 +585,12 @@ static int nt35950_probe(struct mipi_dsi_device *dsi)
 		       DRM_MODE_CONNECTOR_DSI);
 
 	ret = drm_panel_of_backlight(&nt->panel);
-	if (ret)
+	if (ret) {
+		if (num_dsis == 2)
+			mipi_dsi_device_unregister(nt->dsi[1]);
+
 		return dev_err_probe(dev, ret, "Failed to get backlight\n");
+	}
 
 	drm_panel_add(&nt->panel);
 
@@ -602,6 +606,10 @@ static int nt35950_probe(struct mipi_dsi_device *dsi)
 
 		ret = mipi_dsi_attach(nt->dsi[i]);
 		if (ret < 0) {
+			/* If we fail to attach to either host, we're done */
+			if (num_dsis == 2)
+				mipi_dsi_device_unregister(nt->dsi[1]);
+
 			return dev_err_probe(dev, ret,
 					     "Cannot attach to DSI%d host.\n", i);
 		}
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt36523.c b/drivers/gpu/drm/panel/panel-novatek-nt36523.c
new file mode 100644
index 000000000000..d30dbbfb67b1
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-novatek-nt36523.c
@@ -0,0 +1,777 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Novatek NT36523 DriverIC panels driver
+ *
+ * Copyright (c) 2022, 2023 Jianhua Lu <lujianhua000@gmail.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of_graph.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+
+#define DSI_NUM_MIN 1
+
+#define mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, cmd, seq...)        \
+		do {                                                 \
+			mipi_dsi_dcs_write_seq(dsi0, cmd, seq);      \
+			mipi_dsi_dcs_write_seq(dsi1, cmd, seq);      \
+		} while (0)
+
+struct panel_info {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi[2];
+	const struct panel_desc *desc;
+
+	struct gpio_desc *reset_gpio;
+	struct backlight_device *backlight;
+	struct regulator *vddio;
+
+	bool prepared;
+};
+
+struct panel_desc {
+	unsigned int width_mm;
+	unsigned int height_mm;
+
+	unsigned int bpc;
+	unsigned int lanes;
+	unsigned long mode_flags;
+	enum mipi_dsi_pixel_format format;
+
+	const struct drm_display_mode *modes;
+	unsigned int num_modes;
+	const struct mipi_dsi_device_info dsi_info;
+	int (*init_sequence)(struct panel_info *pinfo);
+
+	bool is_dual_dsi;
+};
+
+static inline struct panel_info *to_panel_info(struct drm_panel *panel)
+{
+	return container_of(panel, struct panel_info, panel);
+}
+
+static int elish_boe_init_sequence(struct panel_info *pinfo)
+{
+	struct mipi_dsi_device *dsi0 = pinfo->dsi[0];
+	struct mipi_dsi_device *dsi1 = pinfo->dsi[1];
+	/* No datasheet, so write magic init sequence directly */
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb9, 0x05);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x20);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x18, 0x40);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb9, 0x02);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x23);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x00, 0x80);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x01, 0x84);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x05, 0x2d);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x06, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x07, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x08, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x09, 0x45);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x11, 0x02);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x12, 0x80);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x15, 0x83);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x16, 0x0c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x29, 0x0a);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x30, 0xff);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x31, 0xfe);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x32, 0xfd);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x33, 0xfb);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x34, 0xf8);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x35, 0xf5);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x36, 0xf3);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x37, 0xf2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x38, 0xf2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x39, 0xf2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3a, 0xef);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3b, 0xec);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3d, 0xe9);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3f, 0xe5);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x40, 0xe5);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x41, 0xe5);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x2a, 0x13);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x45, 0xff);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x46, 0xf4);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x47, 0xe7);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x48, 0xda);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x49, 0xcd);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4a, 0xc0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4b, 0xb3);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4c, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4d, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4e, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4f, 0x99);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x50, 0x80);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x51, 0x68);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x52, 0x66);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x53, 0x66);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x54, 0x66);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x2b, 0x0e);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x58, 0xff);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x59, 0xfb);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5a, 0xf7);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5b, 0xf3);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5c, 0xef);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5d, 0xe3);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5e, 0xda);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5f, 0xd8);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x60, 0xd8);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x61, 0xd8);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x62, 0xcb);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x63, 0xbf);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x64, 0xb3);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x65, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x66, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x67, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x2a);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x25, 0x47);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x30, 0x47);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x39, 0x47);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x26);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x19, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1a, 0xe0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1b, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1c, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x2a, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x2b, 0xe0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0xf0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x84, 0x08);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x85, 0x0c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x20);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x51, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x25);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x91, 0x1f);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x92, 0x0f);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x93, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x94, 0x18);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x95, 0x03);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x96, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb0, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x25);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x19, 0x1f);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1b, 0x1b);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x24);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb8, 0x28);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x27);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd0, 0x31);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd1, 0x20);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd2, 0x30);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd4, 0x08);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xde, 0x80);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xdf, 0x02);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x26);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x00, 0x81);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x01, 0xb0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x22);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x9f, 0x50);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x6f, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x70, 0x11);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x73, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x74, 0x49);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x76, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x77, 0x49);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xa0, 0x3f);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xa9, 0x50);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xaa, 0x28);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xab, 0x28);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xad, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb8, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb9, 0x49);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xba, 0x49);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbb, 0x49);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbe, 0x04);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbf, 0x49);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc0, 0x04);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc1, 0x59);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc2, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc5, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc6, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc7, 0x48);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xca, 0x43);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xcb, 0x3c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xce, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xcf, 0x43);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd0, 0x3c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd3, 0x43);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd4, 0x3c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd7, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xdc, 0x43);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xdd, 0x3c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xe1, 0x43);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xe2, 0x3c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xf2, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xf3, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xf4, 0x48);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x25);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x13, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x14, 0x23);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbc, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbd, 0x23);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x2a);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x97, 0x3c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x98, 0x02);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x99, 0x95);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x9a, 0x03);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x9b, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x9c, 0x0b);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x9d, 0x0a);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x9e, 0x90);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x22);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x9f, 0x50);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x23);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xa3, 0x50);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0xe0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x14, 0x60);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x16, 0xc0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4f, 0x02);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0xf0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3a, 0x08);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0xd0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x02, 0xaf);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x09, 0xee);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1c, 0x99);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1d, 0x09);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x51, 0x0f, 0xff);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x53, 0x2c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x35, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbb, 0x13);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3b, 0x03, 0xac, 0x1a, 0x04, 0x04);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x11);
+	msleep(70);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x29);
+
+	return 0;
+}
+
+static int elish_csot_init_sequence(struct panel_info *pinfo)
+{
+	struct mipi_dsi_device *dsi0 = pinfo->dsi[0];
+	struct mipi_dsi_device *dsi1 = pinfo->dsi[1];
+	/* No datasheet, so write magic init sequence directly */
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb9, 0x05);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x20);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x18, 0x40);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb9, 0x02);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0xd0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x02, 0xaf);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x00, 0x30);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x09, 0xee);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1c, 0x99);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1d, 0x09);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0xf0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3a, 0x08);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0xe0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4f, 0x02);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x20);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x58, 0x40);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x35, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x23);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x00, 0x80);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x01, 0x84);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x05, 0x2d);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x06, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x07, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x08, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x09, 0x45);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x11, 0x02);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x12, 0x80);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x15, 0x83);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x16, 0x0c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x29, 0x0a);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x30, 0xff);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x31, 0xfe);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x32, 0xfd);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x33, 0xfb);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x34, 0xf8);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x35, 0xf5);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x36, 0xf3);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x37, 0xf2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x38, 0xf2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x39, 0xf2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3a, 0xef);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3b, 0xec);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3d, 0xe9);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3f, 0xe5);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x40, 0xe5);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x41, 0xe5);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x2a, 0x13);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x45, 0xff);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x46, 0xf4);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x47, 0xe7);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x48, 0xda);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x49, 0xcd);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4a, 0xc0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4b, 0xb3);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4c, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4d, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4e, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x4f, 0x99);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x50, 0x80);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x51, 0x68);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x52, 0x66);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x53, 0x66);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x54, 0x66);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x2b, 0x0e);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x58, 0xff);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x59, 0xfb);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5a, 0xf7);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5b, 0xf3);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5c, 0xef);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5d, 0xe3);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5e, 0xda);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x5f, 0xd8);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x60, 0xd8);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x61, 0xd8);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x62, 0xcb);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x63, 0xbf);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x64, 0xb3);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x65, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x66, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x67, 0xb2);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x51, 0x0f, 0xff);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x53, 0x2c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x55, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbb, 0x13);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x3b, 0x03, 0xac, 0x1a, 0x04, 0x04);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x2a);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x25, 0x46);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x30, 0x46);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x39, 0x46);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x26);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x01, 0xb0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x19, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1a, 0xe0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1b, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1c, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x2a, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x2b, 0xe0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0xf0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x84, 0x08);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x85, 0x0c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x20);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x51, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x25);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x91, 0x1f);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x92, 0x0f);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x93, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x94, 0x18);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x95, 0x03);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x96, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb0, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x25);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x19, 0x1f);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x1b, 0x1b);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x24);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb8, 0x28);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x27);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd0, 0x31);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd1, 0x20);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd4, 0x08);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xde, 0x80);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xdf, 0x02);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x26);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x00, 0x81);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x01, 0xb0);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x22);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x6f, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x70, 0x11);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x73, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x74, 0x4d);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xa0, 0x3f);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xa9, 0x50);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xaa, 0x28);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xab, 0x28);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xad, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb8, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xb9, 0x4b);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xba, 0x96);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbb, 0x4b);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbe, 0x07);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbf, 0x4b);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc0, 0x07);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc1, 0x5c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc2, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc5, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc6, 0x3f);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xc7, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xca, 0x08);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xcb, 0x40);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xce, 0x00);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xcf, 0x08);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd0, 0x40);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd3, 0x08);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xd4, 0x40);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x25);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbc, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xbd, 0x1c);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x2a);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xfb, 0x01);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x9a, 0x03);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0xff, 0x10);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x11);
+	msleep(70);
+	mipi_dsi_dual_dcs_write_seq(dsi0, dsi1, 0x29);
+
+	return 0;
+}
+
+static const struct drm_display_mode elish_boe_modes[] = {
+	{
+		/* There is only one 120 Hz timing, but it doesn't work perfectly, 104 Hz preferred */
+		.clock = (1600 + 60 + 8 + 60) * (2560 + 26 + 4 + 168) * 104 / 1000,
+		.hdisplay = 1600,
+		.hsync_start = 1600 + 60,
+		.hsync_end = 1600 + 60 + 8,
+		.htotal = 1600 + 60 + 8 + 60,
+		.vdisplay = 2560,
+		.vsync_start = 2560 + 26,
+		.vsync_end = 2560 + 26 + 4,
+		.vtotal = 2560 + 26 + 4 + 168,
+	},
+};
+
+static const struct drm_display_mode elish_csot_modes[] = {
+	{
+		/* There is only one 120 Hz timing, but it doesn't work perfectly, 104 Hz preferred */
+		.clock = (1600 + 200 + 40 + 52) * (2560 + 26 + 4 + 168) * 104 / 1000,
+		.hdisplay = 1600,
+		.hsync_start = 1600 + 200,
+		.hsync_end = 1600 + 200 + 40,
+		.htotal = 1600 + 200 + 40 + 52,
+		.vdisplay = 2560,
+		.vsync_start = 2560 + 26,
+		.vsync_end = 2560 + 26 + 4,
+		.vtotal = 2560 + 26 + 4 + 168,
+	},
+};
+
+static const struct panel_desc elish_boe_desc = {
+	.modes = elish_boe_modes,
+	.num_modes = ARRAY_SIZE(elish_boe_modes),
+	.dsi_info = {
+		.type = "BOE-elish",
+		.channel = 0,
+		.node = NULL,
+	},
+	.width_mm = 127,
+	.height_mm = 203,
+	.bpc = 8,
+	.lanes = 3,
+	.format = MIPI_DSI_FMT_RGB888,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_CLOCK_NON_CONTINUOUS | MIPI_DSI_MODE_LPM,
+	.init_sequence = elish_boe_init_sequence,
+	.is_dual_dsi = true,
+};
+
+static const struct panel_desc elish_csot_desc = {
+	.modes = elish_csot_modes,
+	.num_modes = ARRAY_SIZE(elish_csot_modes),
+	.dsi_info = {
+		.type = "CSOT-elish",
+		.channel = 0,
+		.node = NULL,
+	},
+	.width_mm = 127,
+	.height_mm = 203,
+	.bpc = 8,
+	.lanes = 3,
+	.format = MIPI_DSI_FMT_RGB888,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_CLOCK_NON_CONTINUOUS | MIPI_DSI_MODE_LPM,
+	.init_sequence = elish_csot_init_sequence,
+	.is_dual_dsi = true,
+};
+
+static void nt36523_reset(struct panel_info *pinfo)
+{
+	gpiod_set_value_cansleep(pinfo->reset_gpio, 1);
+	usleep_range(12000, 13000);
+	gpiod_set_value_cansleep(pinfo->reset_gpio, 0);
+	usleep_range(12000, 13000);
+	gpiod_set_value_cansleep(pinfo->reset_gpio, 1);
+	usleep_range(12000, 13000);
+	gpiod_set_value_cansleep(pinfo->reset_gpio, 0);
+	usleep_range(12000, 13000);
+}
+
+static int nt36523_prepare(struct drm_panel *panel)
+{
+	struct panel_info *pinfo = to_panel_info(panel);
+	int ret;
+
+	if (pinfo->prepared)
+		return 0;
+
+	ret = regulator_enable(pinfo->vddio);
+	if (ret) {
+		dev_err(panel->dev, "failed to enable vddio regulator: %d\n", ret);
+		return ret;
+	}
+
+	nt36523_reset(pinfo);
+
+	ret = pinfo->desc->init_sequence(pinfo);
+	if (ret < 0) {
+		regulator_disable(pinfo->vddio);
+		dev_err(panel->dev, "failed to initialize panel: %d\n", ret);
+		return ret;
+	}
+
+	pinfo->prepared = true;
+
+	return 0;
+}
+
+static int nt36523_disable(struct drm_panel *panel)
+{
+	struct panel_info *pinfo = to_panel_info(panel);
+	int i, ret;
+
+	for (i = 0; i < DSI_NUM_MIN + pinfo->desc->is_dual_dsi; i++) {
+		ret = mipi_dsi_dcs_set_display_off(pinfo->dsi[i]);
+		if (ret < 0)
+			dev_err(&pinfo->dsi[i]->dev, "failed to set display off: %d\n", ret);
+	}
+
+	for (i = 0; i < DSI_NUM_MIN + pinfo->desc->is_dual_dsi; i++) {
+		ret = mipi_dsi_dcs_enter_sleep_mode(pinfo->dsi[i]);
+		if (ret < 0)
+			dev_err(&pinfo->dsi[i]->dev, "failed to enter sleep mode: %d\n", ret);
+	}
+
+	msleep(70);
+
+	return 0;
+}
+
+static int nt36523_unprepare(struct drm_panel *panel)
+{
+	struct panel_info *pinfo = to_panel_info(panel);
+
+	if (!pinfo->prepared)
+		return 0;
+
+	gpiod_set_value_cansleep(pinfo->reset_gpio, 1);
+	regulator_disable(pinfo->vddio);
+
+	pinfo->prepared = false;
+
+	return 0;
+}
+
+static void nt36523_remove(struct mipi_dsi_device *dsi)
+{
+	struct panel_info *pinfo = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = mipi_dsi_detach(pinfo->dsi[0]);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to detach from DSI0 host: %d\n", ret);
+
+	if (pinfo->desc->is_dual_dsi) {
+		ret = mipi_dsi_detach(pinfo->dsi[1]);
+		if (ret < 0)
+			dev_err(&pinfo->dsi[1]->dev, "failed to detach from DSI1 host: %d\n", ret);
+		mipi_dsi_device_unregister(pinfo->dsi[1]);
+	}
+
+	drm_panel_remove(&pinfo->panel);
+}
+
+static int nt36523_get_modes(struct drm_panel *panel,
+			       struct drm_connector *connector)
+{
+	struct panel_info *pinfo = to_panel_info(panel);
+	int i;
+
+	for (i = 0; i < pinfo->desc->num_modes; i++) {
+		const struct drm_display_mode *m = &pinfo->desc->modes[i];
+		struct drm_display_mode *mode;
+
+		mode = drm_mode_duplicate(connector->dev, m);
+		if (!mode) {
+			dev_err(panel->dev, "failed to add mode %ux%u@%u\n",
+				m->hdisplay, m->vdisplay, drm_mode_vrefresh(m));
+			return -ENOMEM;
+		}
+
+		mode->type = DRM_MODE_TYPE_DRIVER;
+		if (i == 0)
+			mode->type |= DRM_MODE_TYPE_PREFERRED;
+
+		drm_mode_set_name(mode);
+		drm_mode_probed_add(connector, mode);
+	}
+
+	connector->display_info.width_mm = pinfo->desc->width_mm;
+	connector->display_info.height_mm = pinfo->desc->height_mm;
+	connector->display_info.bpc = pinfo->desc->bpc;
+
+	return pinfo->desc->num_modes;
+}
+
+static const struct drm_panel_funcs nt36523_panel_funcs = {
+	.disable = nt36523_disable,
+	.prepare = nt36523_prepare,
+	.unprepare = nt36523_unprepare,
+	.get_modes = nt36523_get_modes,
+};
+
+static int nt36523_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct device_node *dsi1;
+	struct mipi_dsi_host *dsi1_host;
+	struct panel_info *pinfo;
+	const struct mipi_dsi_device_info *info;
+	int i, ret;
+
+	pinfo = devm_kzalloc(dev, sizeof(*pinfo), GFP_KERNEL);
+	if (!pinfo)
+		return -ENOMEM;
+
+	pinfo->vddio = devm_regulator_get(dev, "vddio");
+	if (IS_ERR(pinfo->vddio))
+		return dev_err_probe(dev, PTR_ERR(pinfo->vddio), "failed to get vddio regulator\n");
+
+	pinfo->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(pinfo->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(pinfo->reset_gpio), "failed to get reset gpio\n");
+
+	pinfo->desc = of_device_get_match_data(dev);
+	if (!pinfo->desc)
+		return -ENODEV;
+
+	/* If the panel is dual dsi, register DSI1 */
+	if (pinfo->desc->is_dual_dsi) {
+		info = &pinfo->desc->dsi_info;
+
+		dsi1 = of_graph_get_remote_node(dsi->dev.of_node, 1, -1);
+		if (!dsi1) {
+			dev_err(dev, "cannot get secondary DSI node.\n");
+			return -ENODEV;
+		}
+
+		dsi1_host = of_find_mipi_dsi_host_by_node(dsi1);
+		of_node_put(dsi1);
+		if (!dsi1_host)
+			return dev_err_probe(dev, -EPROBE_DEFER, "cannot get secondary DSI host\n");
+
+		pinfo->dsi[1] = mipi_dsi_device_register_full(dsi1_host, info);
+		if (!pinfo->dsi[1]) {
+			dev_err(dev, "cannot get secondary DSI device\n");
+			return -ENODEV;
+		}
+	}
+
+	pinfo->dsi[0] = dsi;
+	mipi_dsi_set_drvdata(dsi, pinfo);
+	drm_panel_init(&pinfo->panel, dev, &nt36523_panel_funcs, DRM_MODE_CONNECTOR_DSI);
+
+	ret = drm_panel_of_backlight(&pinfo->panel);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get backlight\n");
+
+	drm_panel_add(&pinfo->panel);
+
+	for (i = 0; i < DSI_NUM_MIN + pinfo->desc->is_dual_dsi; i++) {
+		pinfo->dsi[i]->lanes = pinfo->desc->lanes;
+		pinfo->dsi[i]->format = pinfo->desc->format;
+		pinfo->dsi[i]->mode_flags = pinfo->desc->mode_flags;
+
+		ret = mipi_dsi_attach(pinfo->dsi[i]);
+		if (ret < 0)
+			return dev_err_probe(dev, ret, "cannot attach to DSI%d host.\n", i);
+	}
+
+	return 0;
+}
+
+static const struct of_device_id nt36523_of_match[] = {
+	{
+		.compatible = "xiaomi,elish-boe-nt36523",
+		.data = &elish_boe_desc,
+	},
+	{
+		.compatible = "xiaomi,elish-csot-nt36523",
+		.data = &elish_csot_desc,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, nt36523_of_match);
+
+static struct mipi_dsi_driver nt36523_driver = {
+	.probe = nt36523_probe,
+	.remove = nt36523_remove,
+	.driver = {
+		.name = "panel-novatek-nt36523",
+		.of_match_table = nt36523_of_match,
+	},
+};
+module_mipi_dsi_driver(nt36523_driver);
+
+MODULE_AUTHOR("Jianhua Lu <lujianhua000@gmail.com>");
+MODULE_DESCRIPTION("DRM driver for Novatek NT36523 based MIPI DSI panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c b/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
index b4729a94c34a..898b892f1143 100644
--- a/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
+++ b/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
@@ -471,7 +471,7 @@ static int otm8009a_probe(struct mipi_dsi_device *dsi)
 		       DRM_MODE_CONNECTOR_DSI);
 
 	ctx->bl_dev = devm_backlight_device_register(dev, dev_name(dev),
-						     dsi->host->dev, ctx,
+						     dev, ctx,
 						     &otm8009a_backlight_ops,
 						     NULL);
 	if (IS_ERR(ctx->bl_dev)) {
diff --git a/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c b/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c
index 76160e5d43bd..c250ca36a5b3 100644
--- a/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c
+++ b/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/media-bus-format.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -48,6 +49,7 @@ struct seiko_panel {
 	const struct seiko_panel_desc *desc;
 	struct regulator *dvdd;
 	struct regulator *avdd;
+	struct gpio_desc *enable_gpio;
 };
 
 static inline struct seiko_panel *to_seiko_panel(struct drm_panel *panel)
@@ -139,6 +141,8 @@ static int seiko_panel_unprepare(struct drm_panel *panel)
 	if (!p->prepared)
 		return 0;
 
+	gpiod_set_value_cansleep(p->enable_gpio, 0);
+
 	regulator_disable(p->avdd);
 
 	/* Add a 100ms delay as per the panel datasheet */
@@ -174,6 +178,8 @@ static int seiko_panel_prepare(struct drm_panel *panel)
 		goto disable_dvdd;
 	}
 
+	gpiod_set_value_cansleep(p->enable_gpio, 1);
+
 	p->prepared = true;
 
 	return 0;
@@ -252,6 +258,12 @@ static int seiko_panel_probe(struct device *dev,
 	if (IS_ERR(panel->avdd))
 		return PTR_ERR(panel->avdd);
 
+	panel->enable_gpio = devm_gpiod_get_optional(dev, "enable",
+						     GPIOD_OUT_LOW);
+	if (IS_ERR(panel->enable_gpio))
+		return dev_err_probe(dev, PTR_ERR(panel->enable_gpio),
+				     "failed to request GPIO\n");
+
 	drm_panel_init(&panel->base, dev, &seiko_panel_funcs,
 		       DRM_MODE_CONNECTOR_DPI);
 
diff --git a/drivers/gpu/drm/panel/panel-sitronix-st7701.c b/drivers/gpu/drm/panel/panel-sitronix-st7701.c
index 0b8cf65172ff..7eae83aa0ea1 100644
--- a/drivers/gpu/drm/panel/panel-sitronix-st7701.c
+++ b/drivers/gpu/drm/panel/panel-sitronix-st7701.c
@@ -135,6 +135,7 @@ struct st7701 {
 	struct regulator_bulk_data supplies[2];
 	struct gpio_desc *reset;
 	unsigned int sleep_delay;
+	enum drm_panel_orientation orientation;
 };
 
 static inline struct st7701 *panel_to_st7701(struct drm_panel *panel)
@@ -397,6 +398,31 @@ static void dmt028vghmcmi_1a_gip_sequence(struct st7701 *st7701)
 	ST7701_DSI(st7701, 0x3A, 0x70);
 }
 
+static void kd50t048a_gip_sequence(struct st7701 *st7701)
+{
+	/**
+	 * ST7701_SPEC_V1.2 is unable to provide enough information above this
+	 * specific command sequence, so grab the same from vendor BSP driver.
+	 */
+	ST7701_DSI(st7701, 0xE0, 0x00, 0x00, 0x02);
+	ST7701_DSI(st7701, 0xE1, 0x08, 0x00, 0x0A, 0x00, 0x07, 0x00, 0x09,
+		   0x00, 0x00, 0x33, 0x33);
+	ST7701_DSI(st7701, 0xE2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		   0x00, 0x00, 0x00, 0x00, 0x00, 0x00);
+	ST7701_DSI(st7701, 0xE3, 0x00, 0x00, 0x33, 0x33);
+	ST7701_DSI(st7701, 0xE4, 0x44, 0x44);
+	ST7701_DSI(st7701, 0xE5, 0x0E, 0x60, 0xA0, 0xA0, 0x10, 0x60, 0xA0,
+		   0xA0, 0x0A, 0x60, 0xA0, 0xA0, 0x0C, 0x60, 0xA0, 0xA0);
+	ST7701_DSI(st7701, 0xE6, 0x00, 0x00, 0x33, 0x33);
+	ST7701_DSI(st7701, 0xE7, 0x44, 0x44);
+	ST7701_DSI(st7701, 0xE8, 0x0D, 0x60, 0xA0, 0xA0, 0x0F, 0x60, 0xA0,
+		   0xA0, 0x09, 0x60, 0xA0, 0xA0, 0x0B, 0x60, 0xA0, 0xA0);
+	ST7701_DSI(st7701, 0xEB, 0x02, 0x01, 0xE4, 0xE4, 0x44, 0x00, 0x40);
+	ST7701_DSI(st7701, 0xEC, 0x02, 0x01);
+	ST7701_DSI(st7701, 0xED, 0xAB, 0x89, 0x76, 0x54, 0x01, 0xFF, 0xFF,
+		   0xFF, 0xFF, 0xFF, 0xFF, 0x10, 0x45, 0x67, 0x98, 0xBA);
+}
+
 static int st7701_prepare(struct drm_panel *panel)
 {
 	struct st7701 *st7701 = panel_to_st7701(panel);
@@ -489,15 +515,29 @@ static int st7701_get_modes(struct drm_panel *panel,
 	connector->display_info.width_mm = desc_mode->width_mm;
 	connector->display_info.height_mm = desc_mode->height_mm;
 
+	/*
+	 * TODO: Remove once all drm drivers call
+	 * drm_connector_set_orientation_from_panel()
+	 */
+	drm_connector_set_panel_orientation(connector, st7701->orientation);
+
 	return 1;
 }
 
+static enum drm_panel_orientation st7701_get_orientation(struct drm_panel *panel)
+{
+	struct st7701 *st7701 = panel_to_st7701(panel);
+
+	return st7701->orientation;
+}
+
 static const struct drm_panel_funcs st7701_funcs = {
 	.disable	= st7701_disable,
 	.unprepare	= st7701_unprepare,
 	.prepare	= st7701_prepare,
 	.enable		= st7701_enable,
 	.get_modes	= st7701_get_modes,
+	.get_orientation = st7701_get_orientation,
 };
 
 static const struct drm_display_mode ts8550b_mode = {
@@ -700,6 +740,105 @@ static const struct st7701_panel_desc dmt028vghmcmi_1a_desc = {
 	.gip_sequence = dmt028vghmcmi_1a_gip_sequence,
 };
 
+static const struct drm_display_mode kd50t048a_mode = {
+	.clock          = 27500,
+
+	.hdisplay       = 480,
+	.hsync_start    = 480 + 2,
+	.hsync_end      = 480 + 2 + 10,
+	.htotal         = 480 + 2 + 10 + 2,
+
+	.vdisplay       = 854,
+	.vsync_start    = 854 + 2,
+	.vsync_end      = 854 + 2 + 2,
+	.vtotal         = 854 + 2 + 2 + 17,
+
+	.width_mm       = 69,
+	.height_mm      = 139,
+
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static const struct st7701_panel_desc kd50t048a_desc = {
+	.mode = &kd50t048a_mode,
+	.lanes = 2,
+	.format = MIPI_DSI_FMT_RGB888,
+	.panel_sleep_delay = 0,
+
+	.pv_gamma = {
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC0_MASK, 0),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC4_MASK, 0xd),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC8_MASK, 0x14),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC16_MASK, 0xd),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC24_MASK, 0x10),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC52_MASK, 0x5),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC80_MASK, 0x2),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC108_MASK, 0x8),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC147_MASK, 0x8),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC175_MASK, 0x1e),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC203_MASK, 0x5),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC231_MASK, 0x13),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC239_MASK, 0x11),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 2) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC247_MASK, 0x23),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC251_MASK, 0x29),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC255_MASK, 0x18)
+	},
+	.nv_gamma = {
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC0_MASK, 0),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC4_MASK, 0xc),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC8_MASK, 0x14),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC16_MASK, 0xc),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC24_MASK, 0x10),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC52_MASK, 0x5),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC80_MASK, 0x3),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC108_MASK, 0x8),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC147_MASK, 0x7),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC175_MASK, 0x20),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC203_MASK, 0x5),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC231_MASK, 0x13),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC239_MASK, 0x11),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 2) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC247_MASK, 0x24),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC251_MASK, 0x29),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC255_MASK, 0x18)
+	},
+	.nlinv = 1,
+	.vop_uv = 4887500,
+	.vcom_uv = 937500,
+	.vgh_mv = 15000,
+	.vgl_mv = -9510,
+	.avdd_mv = 6600,
+	.avcl_mv = -4400,
+	.gamma_op_bias = OP_BIAS_MIDDLE,
+	.input_op_bias = OP_BIAS_MIN,
+	.output_op_bias = OP_BIAS_MIN,
+	.t2d_ns = 1600,
+	.t3d_ns = 10400,
+	.eot_en = true,
+	.gip_sequence = kd50t048a_gip_sequence,
+};
+
 static int st7701_dsi_probe(struct mipi_dsi_device *dsi)
 {
 	const struct st7701_panel_desc *desc;
@@ -730,6 +869,10 @@ static int st7701_dsi_probe(struct mipi_dsi_device *dsi)
 		return PTR_ERR(st7701->reset);
 	}
 
+	ret = of_drm_get_panel_orientation(dsi->dev.of_node, &st7701->orientation);
+	if (ret < 0)
+		return dev_err_probe(&dsi->dev, ret, "Failed to get orientation\n");
+
 	drm_panel_init(&st7701->panel, &dsi->dev, &st7701_funcs,
 		       DRM_MODE_CONNECTOR_DSI);
 
@@ -775,6 +918,7 @@ static void st7701_dsi_remove(struct mipi_dsi_device *dsi)
 
 static const struct of_device_id st7701_of_match[] = {
 	{ .compatible = "densitron,dmt028vghmcmi-1a", .data = &dmt028vghmcmi_1a_desc },
+	{ .compatible = "elida,kd50t048a", .data = &kd50t048a_desc },
 	{ .compatible = "techstar,ts8550b", .data = &ts8550b_desc },
 	{ }
 };
diff --git a/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c b/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c
new file mode 100644
index 000000000000..8d8813dbaa45
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c
@@ -0,0 +1,329 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2022 Konrad Dybcio <konrad.dybcio@somainline.org>
+ *
+ * Generated with linux-mdss-dsi-panel-driver-generator with a
+ * substantial amount of manual adjustments.
+ *
+ * SONY Downstream kernel calls this one:
+ * - "JDI ID3" for Akari  (XZ2)
+ * - "JDI ID4" for Apollo (XZ2 Compact)
+ */
+
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+
+enum {
+	TYPE_TAMA_60HZ,
+	/*
+	 * Leaving room for expansion - SONY very often uses
+	 * *truly reliably* overclockable panels on their flagships!
+	 */
+};
+
+struct sony_td4353_jdi {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+	struct regulator_bulk_data supplies[3];
+	struct gpio_desc *panel_reset_gpio;
+	struct gpio_desc *touch_reset_gpio;
+	bool prepared;
+	int type;
+};
+
+static inline struct sony_td4353_jdi *to_sony_td4353_jdi(struct drm_panel *panel)
+{
+	return container_of(panel, struct sony_td4353_jdi, panel);
+}
+
+static int sony_td4353_jdi_on(struct sony_td4353_jdi *ctx)
+{
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	struct device *dev = &dsi->dev;
+	int ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_column_address(dsi, 0x0000, 1080 - 1);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set column address: %d\n", ret);
+		return ret;
+	}
+
+	ret = mipi_dsi_dcs_set_page_address(dsi, 0x0000, 2160 - 1);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set page address: %d\n", ret);
+		return ret;
+	}
+
+	ret = mipi_dsi_dcs_set_tear_scanline(dsi, 0);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set tear scanline: %d\n", ret);
+		return ret;
+	}
+
+	ret = mipi_dsi_dcs_set_tear_on(dsi, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set tear on: %d\n", ret);
+		return ret;
+	}
+
+	mipi_dsi_dcs_write_seq(dsi, MIPI_DCS_SET_ADDRESS_MODE, 0x00);
+
+	ret = mipi_dsi_dcs_set_pixel_format(dsi, 0x77);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set pixel format: %d\n", ret);
+		return ret;
+	}
+
+	mipi_dsi_dcs_write_seq(dsi, MIPI_DCS_SET_PARTIAL_ROWS,
+			  0x00, 0x00, 0x08, 0x6f);
+
+	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to exit sleep mode: %d\n", ret);
+		return ret;
+	}
+	msleep(70);
+
+	mipi_dsi_dcs_write_seq(dsi, MIPI_DCS_WRITE_MEMORY_START);
+
+	ret = mipi_dsi_dcs_set_display_on(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to turn display on: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int sony_td4353_jdi_off(struct sony_td4353_jdi *ctx)
+{
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	struct device *dev = &dsi->dev;
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_off(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set display off: %d\n", ret);
+		return ret;
+	}
+	msleep(22);
+
+	ret = mipi_dsi_dcs_set_tear_off(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set tear off: %d\n", ret);
+		return ret;
+	}
+
+	ret = mipi_dsi_dcs_enter_sleep_mode(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enter sleep mode: %d\n", ret);
+		return ret;
+	}
+	msleep(80);
+
+	return 0;
+}
+
+static void sony_td4353_assert_reset_gpios(struct sony_td4353_jdi *ctx, int mode)
+{
+	gpiod_set_value_cansleep(ctx->touch_reset_gpio, mode);
+	gpiod_set_value_cansleep(ctx->panel_reset_gpio, mode);
+	usleep_range(5000, 5100);
+}
+
+static int sony_td4353_jdi_prepare(struct drm_panel *panel)
+{
+	struct sony_td4353_jdi *ctx = to_sony_td4353_jdi(panel);
+	struct device *dev = &ctx->dsi->dev;
+	int ret;
+
+	if (ctx->prepared)
+		return 0;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enable regulators: %d\n", ret);
+		return ret;
+	}
+
+	msleep(100);
+
+	sony_td4353_assert_reset_gpios(ctx, 1);
+
+	ret = sony_td4353_jdi_on(ctx);
+	if (ret < 0) {
+		dev_err(dev, "Failed to power on panel: %d\n", ret);
+		sony_td4353_assert_reset_gpios(ctx, 0);
+		regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+		return ret;
+	}
+
+	ctx->prepared = true;
+	return 0;
+}
+
+static int sony_td4353_jdi_unprepare(struct drm_panel *panel)
+{
+	struct sony_td4353_jdi *ctx = to_sony_td4353_jdi(panel);
+	struct device *dev = &ctx->dsi->dev;
+	int ret;
+
+	if (!ctx->prepared)
+		return 0;
+
+	ret = sony_td4353_jdi_off(ctx);
+	if (ret < 0)
+		dev_err(dev, "Failed to power off panel: %d\n", ret);
+
+	sony_td4353_assert_reset_gpios(ctx, 0);
+	regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+
+	ctx->prepared = false;
+	return 0;
+}
+
+static const struct drm_display_mode sony_td4353_jdi_mode_tama_60hz = {
+	.clock = (1080 + 4 + 8 + 8) * (2160 + 259 + 8 + 8) * 60 / 1000,
+	.hdisplay = 1080,
+	.hsync_start = 1080 + 4,
+	.hsync_end = 1080 + 4 + 8,
+	.htotal = 1080 + 4 + 8 + 8,
+	.vdisplay = 2160,
+	.vsync_start = 2160 + 259,
+	.vsync_end = 2160 + 259 + 8,
+	.vtotal = 2160 + 259 + 8 + 8,
+	.width_mm = 64,
+	.height_mm = 128,
+};
+
+static int sony_td4353_jdi_get_modes(struct drm_panel *panel,
+				   struct drm_connector *connector)
+{
+	struct sony_td4353_jdi *ctx = to_sony_td4353_jdi(panel);
+	struct drm_display_mode *mode = NULL;
+
+	if (ctx->type == TYPE_TAMA_60HZ)
+		mode = drm_mode_duplicate(connector->dev, &sony_td4353_jdi_mode_tama_60hz);
+	else
+		return -EINVAL;
+
+	if (!mode)
+		return -ENOMEM;
+
+	drm_mode_set_name(mode);
+
+	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+	connector->display_info.width_mm = mode->width_mm;
+	connector->display_info.height_mm = mode->height_mm;
+	drm_mode_probed_add(connector, mode);
+
+	return 1;
+}
+
+static const struct drm_panel_funcs sony_td4353_jdi_panel_funcs = {
+	.prepare = sony_td4353_jdi_prepare,
+	.unprepare = sony_td4353_jdi_unprepare,
+	.get_modes = sony_td4353_jdi_get_modes,
+};
+
+static int sony_td4353_jdi_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct sony_td4353_jdi *ctx;
+	int ret;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->type = (uintptr_t)of_device_get_match_data(dev);
+
+	ctx->supplies[0].supply = "vddio";
+	ctx->supplies[1].supply = "vsp";
+	ctx->supplies[2].supply = "vsn";
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ctx->supplies),
+				      ctx->supplies);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to get regulators\n");
+
+	ctx->panel_reset_gpio = devm_gpiod_get(dev, "panel-reset", GPIOD_ASIS);
+	if (IS_ERR(ctx->panel_reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->panel_reset_gpio),
+				     "Failed to get panel-reset-gpios\n");
+
+	ctx->touch_reset_gpio = devm_gpiod_get(dev, "touch-reset", GPIOD_ASIS);
+	if (IS_ERR(ctx->touch_reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->touch_reset_gpio),
+				     "Failed to get touch-reset-gpios\n");
+
+	ctx->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_CLOCK_NON_CONTINUOUS;
+
+	drm_panel_init(&ctx->panel, dev, &sony_td4353_jdi_panel_funcs,
+		       DRM_MODE_CONNECTOR_DSI);
+
+	ret = drm_panel_of_backlight(&ctx->panel);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get backlight\n");
+
+	drm_panel_add(&ctx->panel);
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
+		drm_panel_remove(&ctx->panel);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void sony_td4353_jdi_remove(struct mipi_dsi_device *dsi)
+{
+	struct sony_td4353_jdi *ctx = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_remove(&ctx->panel);
+}
+
+static const struct of_device_id sony_td4353_jdi_of_match[] = {
+	{ .compatible = "sony,td4353-jdi-tama", .data = (void *)TYPE_TAMA_60HZ },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, sony_td4353_jdi_of_match);
+
+static struct mipi_dsi_driver sony_td4353_jdi_driver = {
+	.probe = sony_td4353_jdi_probe,
+	.remove = sony_td4353_jdi_remove,
+	.driver = {
+		.name = "panel-sony-td4353-jdi",
+		.of_match_table = sony_td4353_jdi_of_match,
+	},
+};
+module_mipi_dsi_driver(sony_td4353_jdi_driver);
+
+MODULE_AUTHOR("Konrad Dybcio <konrad.dybcio@somainline.org>");
+MODULE_DESCRIPTION("DRM panel driver for SONY Xperia XZ2/XZ2c JDI panel");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.c b/drivers/gpu/drm/panfrost/panfrost_devfreq.c
index fe5f12f16a63..58dfb15a8757 100644
--- a/drivers/gpu/drm/panfrost/panfrost_devfreq.c
+++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.c
@@ -4,6 +4,7 @@
 #include <linux/clk.h>
 #include <linux/devfreq.h>
 #include <linux/devfreq_cooling.h>
+#include <linux/nvmem-consumer.h>
 #include <linux/platform_device.h>
 #include <linux/pm_opp.h>
 
@@ -82,6 +83,31 @@ static struct devfreq_dev_profile panfrost_devfreq_profile = {
 	.get_dev_status = panfrost_devfreq_get_dev_status,
 };
 
+static int panfrost_read_speedbin(struct device *dev)
+{
+	u32 val;
+	int ret;
+
+	ret = nvmem_cell_read_variable_le_u32(dev, "speed-bin", &val);
+	if (ret) {
+		/*
+		 * -ENOENT means that this platform doesn't support speedbins
+		 * as it didn't declare any speed-bin nvmem: in this case, we
+		 * keep going without it; any other error means that we are
+		 * supposed to read the bin value, but we failed doing so.
+		 */
+		if (ret != -ENOENT) {
+			DRM_DEV_ERROR(dev, "Cannot read speed-bin (%d).", ret);
+			return ret;
+		}
+
+		return 0;
+	}
+	DRM_DEV_DEBUG(dev, "Using speed-bin = 0x%x\n", val);
+
+	return devm_pm_opp_set_supported_hw(dev, &val, 1);
+}
+
 int panfrost_devfreq_init(struct panfrost_device *pfdev)
 {
 	int ret;
@@ -101,6 +127,10 @@ int panfrost_devfreq_init(struct panfrost_device *pfdev)
 		return 0;
 	}
 
+	ret = panfrost_read_speedbin(dev);
+	if (ret)
+		return ret;
+
 	ret = devm_pm_opp_set_regulators(dev, pfdev->comp->supply_names);
 	if (ret) {
 		/* Continue if the optional regulator is missing */
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h
index d9ba68cffb77..b0126b9fbadc 100644
--- a/drivers/gpu/drm/panfrost/panfrost_device.h
+++ b/drivers/gpu/drm/panfrost/panfrost_device.h
@@ -23,7 +23,7 @@ struct panfrost_job;
 struct panfrost_perfcnt;
 
 #define NUM_JOB_SLOTS 3
-#define MAX_PM_DOMAINS 3
+#define MAX_PM_DOMAINS 5
 
 struct panfrost_features {
 	u16 id;
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
index abb0dadd8f63..bbada731bbbd 100644
--- a/drivers/gpu/drm/panfrost/panfrost_drv.c
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -220,15 +220,8 @@ panfrost_copy_in_sync(struct drm_device *dev,
 	}
 
 	for (i = 0; i < in_fence_count; i++) {
-		struct dma_fence *fence;
-
-		ret = drm_syncobj_find_fence(file_priv, handles[i], 0, 0,
-					     &fence);
-		if (ret)
-			goto fail;
-
-		ret = drm_sched_job_add_dependency(&job->base, fence);
-
+		ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv,
+							   handles[i], 0);
 		if (ret)
 			goto fail;
 	}
@@ -654,6 +647,14 @@ static const struct panfrost_compatible amlogic_data = {
 	.vendor_quirk = panfrost_gpu_amlogic_quirk,
 };
 
+/*
+ * The old data with two power supplies for MT8183 is here only to
+ * keep retro-compatibility with older devicetrees, as DVFS will
+ * not work with this one.
+ *
+ * On new devicetrees please use the _b variant with a single and
+ * coupled regulators instead.
+ */
 static const char * const mediatek_mt8183_supplies[] = { "mali", "sram", NULL };
 static const char * const mediatek_mt8183_pm_domains[] = { "core0", "core1", "core2" };
 static const struct panfrost_compatible mediatek_mt8183_data = {
@@ -663,6 +664,32 @@ static const struct panfrost_compatible mediatek_mt8183_data = {
 	.pm_domain_names = mediatek_mt8183_pm_domains,
 };
 
+static const char * const mediatek_mt8183_b_supplies[] = { "mali", NULL };
+static const struct panfrost_compatible mediatek_mt8183_b_data = {
+	.num_supplies = ARRAY_SIZE(mediatek_mt8183_b_supplies) - 1,
+	.supply_names = mediatek_mt8183_b_supplies,
+	.num_pm_domains = ARRAY_SIZE(mediatek_mt8183_pm_domains),
+	.pm_domain_names = mediatek_mt8183_pm_domains,
+};
+
+static const char * const mediatek_mt8186_pm_domains[] = { "core0", "core1" };
+static const struct panfrost_compatible mediatek_mt8186_data = {
+	.num_supplies = ARRAY_SIZE(mediatek_mt8183_b_supplies) - 1,
+	.supply_names = mediatek_mt8183_b_supplies,
+	.num_pm_domains = ARRAY_SIZE(mediatek_mt8186_pm_domains),
+	.pm_domain_names = mediatek_mt8186_pm_domains,
+};
+
+static const char * const mediatek_mt8192_supplies[] = { "mali", NULL };
+static const char * const mediatek_mt8192_pm_domains[] = { "core0", "core1", "core2",
+							   "core3", "core4" };
+static const struct panfrost_compatible mediatek_mt8192_data = {
+	.num_supplies = ARRAY_SIZE(mediatek_mt8192_supplies) - 1,
+	.supply_names = mediatek_mt8192_supplies,
+	.num_pm_domains = ARRAY_SIZE(mediatek_mt8192_pm_domains),
+	.pm_domain_names = mediatek_mt8192_pm_domains,
+};
+
 static const struct of_device_id dt_match[] = {
 	/* Set first to probe before the generic compatibles */
 	{ .compatible = "amlogic,meson-gxm-mali",
@@ -681,6 +708,9 @@ static const struct of_device_id dt_match[] = {
 	{ .compatible = "arm,mali-bifrost", .data = &default_data, },
 	{ .compatible = "arm,mali-valhall-jm", .data = &default_data, },
 	{ .compatible = "mediatek,mt8183-mali", .data = &mediatek_mt8183_data },
+	{ .compatible = "mediatek,mt8183b-mali", .data = &mediatek_mt8183_b_data },
+	{ .compatible = "mediatek,mt8186-mali", .data = &mediatek_mt8186_data },
+	{ .compatible = "mediatek,mt8192-mali", .data = &mediatek_mt8192_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, dt_match);
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c
index 6452e4e900dd..d28b99732dde 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gpu.c
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c
@@ -204,6 +204,14 @@ static const struct panfrost_model gpu_models[] = {
 
 	GPU_MODEL(g57, 0x9001,
 		GPU_REV(g57, 0, 0)),
+
+	/* MediaTek MT8192 has a Mali-G57 with a different GPU ID from the
+	 * standard. Arm's driver does not appear to handle this model.
+	 * ChromeOS has a hack downstream for it. Treat it as equivalent to
+	 * standard Mali-G57 for now.
+	 */
+	GPU_MODEL(g57, 0x9003,
+		GPU_REV(g57, 0, 0)),
 };
 
 static void panfrost_gpu_init_features(struct panfrost_device *pfdev)
diff --git a/drivers/gpu/drm/pl111/pl111_drv.c b/drivers/gpu/drm/pl111/pl111_drv.c
index 00deba0b7271..4b2a9e9753f6 100644
--- a/drivers/gpu/drm/pl111/pl111_drv.c
+++ b/drivers/gpu/drm/pl111/pl111_drv.c
@@ -48,7 +48,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -308,7 +308,7 @@ static int pl111_amba_probe(struct amba_device *amba_dev,
 	if (ret < 0)
 		goto dev_put;
 
-	drm_fbdev_generic_setup(drm, priv->variant->fb_bpp);
+	drm_fbdev_dma_setup(drm, priv->variant->fb_bpp);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index a92a5b0d4c25..1a82629bce3f 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -143,6 +143,17 @@ static int qxl_bo_move(struct ttm_buffer_object *bo, bool evict,
 	struct ttm_resource *old_mem = bo->resource;
 	int ret;
 
+	if (!old_mem) {
+		if (new_mem->mem_type != TTM_PL_SYSTEM) {
+			hop->mem_type = TTM_PL_SYSTEM;
+			hop->flags = TTM_PL_FLAG_TEMPORARY;
+			return -EMULTIHOP;
+		}
+
+		ttm_bo_move_null(bo, new_mem);
+		return 0;
+	}
+
 	qxl_bo_move_notify(bo, new_mem);
 
 	ret = ttm_bo_wait_ctx(bo, ctx);
diff --git a/drivers/gpu/drm/radeon/Kconfig b/drivers/gpu/drm/radeon/Kconfig
index 62a596d3a891..e19d77d58810 100644
--- a/drivers/gpu/drm/radeon/Kconfig
+++ b/drivers/gpu/drm/radeon/Kconfig
@@ -8,6 +8,7 @@ config DRM_RADEON
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HELPER
         select DRM_KMS_HELPER
+	select DRM_SUBALLOC_HELPER
         select DRM_TTM
 	select DRM_TTM_HELPER
 	select SND_HDA_COMPONENT if SND_HDA_CORE
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index bb4e56f2f170..a8734b7d0485 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -36,7 +36,7 @@ radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \
 	atom.o radeon_fence.o radeon_ttm.o radeon_object.o radeon_gart.o \
 	radeon_legacy_crtc.o radeon_legacy_encoders.o radeon_connectors.o \
 	radeon_encoders.o radeon_display.o radeon_cursor.o radeon_i2c.o \
-	radeon_clocks.o radeon_fb.o radeon_gem.o radeon_ring.o radeon_irq_kms.o \
+	radeon_clocks.o radeon_gem.o radeon_ring.o radeon_irq_kms.o \
 	radeon_cs.o radeon_bios.o radeon_benchmark.o r100.o r300.o r420.o \
 	rs400.o rs600.o rs690.o rv515.o r520.o r600.o rv770.o radeon_test.o \
 	r200.o radeon_legacy_tv.o r600_cs.o \
@@ -76,6 +76,7 @@ radeon-y += \
 	vce_v1_0.o \
 	vce_v2_0.o
 
+radeon-$(CONFIG_DRM_FBDEV_EMULATION) += radeon_fbdev.o
 radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
 radeon-$(CONFIG_ACPI) += radeon_acpi.o
 
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 57e20780a458..8afb03bbce29 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -79,6 +79,7 @@
 
 #include <drm/drm_gem.h>
 #include <drm/drm_audio_component.h>
+#include <drm/drm_suballoc.h>
 
 #include "radeon_family.h"
 #include "radeon_mode.h"
@@ -511,52 +512,12 @@ struct radeon_bo {
 };
 #define gem_to_radeon_bo(gobj) container_of((gobj), struct radeon_bo, tbo.base)
 
-/* sub-allocation manager, it has to be protected by another lock.
- * By conception this is an helper for other part of the driver
- * like the indirect buffer or semaphore, which both have their
- * locking.
- *
- * Principe is simple, we keep a list of sub allocation in offset
- * order (first entry has offset == 0, last entry has the highest
- * offset).
- *
- * When allocating new object we first check if there is room at
- * the end total_size - (last_object_offset + last_object_size) >=
- * alloc_size. If so we allocate new object there.
- *
- * When there is not enough room at the end, we start waiting for
- * each sub object until we reach object_offset+object_size >=
- * alloc_size, this object then become the sub object we return.
- *
- * Alignment can't be bigger than page size.
- *
- * Hole are not considered for allocation to keep things simple.
- * Assumption is that there won't be hole (all object on same
- * alignment).
- */
 struct radeon_sa_manager {
-	wait_queue_head_t	wq;
-	struct radeon_bo	*bo;
-	struct list_head	*hole;
-	struct list_head	flist[RADEON_NUM_RINGS];
-	struct list_head	olist;
-	unsigned		size;
-	uint64_t		gpu_addr;
-	void			*cpu_ptr;
-	uint32_t		domain;
-	uint32_t		align;
-};
-
-struct radeon_sa_bo;
-
-/* sub-allocation buffer */
-struct radeon_sa_bo {
-	struct list_head		olist;
-	struct list_head		flist;
-	struct radeon_sa_manager	*manager;
-	unsigned			soffset;
-	unsigned			eoffset;
-	struct radeon_fence		*fence;
+	struct drm_suballoc_manager	base;
+	struct radeon_bo		*bo;
+	uint64_t			gpu_addr;
+	void				*cpu_ptr;
+	u32 domain;
 };
 
 /*
@@ -569,6 +530,8 @@ struct radeon_gem {
 
 extern const struct drm_gem_object_funcs radeon_gem_object_funcs;
 
+int radeon_align_pitch(struct radeon_device *rdev, int width, int cpp, bool tiled);
+
 int radeon_gem_init(struct radeon_device *rdev);
 void radeon_gem_fini(struct radeon_device *rdev);
 int radeon_gem_object_create(struct radeon_device *rdev, unsigned long size,
@@ -587,7 +550,7 @@ int radeon_mode_dumb_mmap(struct drm_file *filp,
  * Semaphores.
  */
 struct radeon_semaphore {
-	struct radeon_sa_bo	*sa_bo;
+	struct drm_suballoc	*sa_bo;
 	signed			waiters;
 	uint64_t		gpu_addr;
 };
@@ -816,7 +779,7 @@ void radeon_irq_kms_disable_hpd(struct radeon_device *rdev, unsigned hpd_mask);
  */
 
 struct radeon_ib {
-	struct radeon_sa_bo		*sa_bo;
+	struct drm_suballoc		*sa_bo;
 	uint32_t			length_dw;
 	uint64_t			gpu_addr;
 	uint32_t			*ptr;
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index f34a7f63261d..901e75ec70ff 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -34,7 +34,6 @@
 #include <drm/drm_device.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_edid.h>
-#include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -1354,7 +1353,6 @@ radeon_user_framebuffer_create(struct drm_device *dev,
 
 static const struct drm_mode_config_funcs radeon_mode_funcs = {
 	.fb_create = radeon_user_framebuffer_create,
-	.output_poll_changed = drm_fb_helper_output_poll_changed,
 };
 
 static const struct drm_prop_enum_list radeon_tmds_pll_enum_list[] =
@@ -1642,7 +1640,6 @@ int radeon_modeset_init(struct radeon_device *rdev)
 	/* setup afmt */
 	radeon_afmt_init(rdev);
 
-	radeon_fbdev_init(rdev);
 	drm_kms_helper_poll_init(rdev->ddev);
 
 	/* do pm late init */
@@ -1657,7 +1654,6 @@ void radeon_modeset_fini(struct radeon_device *rdev)
 		drm_kms_helper_poll_fini(rdev->ddev);
 		radeon_hpd_fini(rdev);
 		drm_helper_force_disable_all(rdev->ddev);
-		radeon_fbdev_fini(rdev);
 		radeon_afmt_fini(rdev);
 		drm_mode_config_cleanup(rdev->ddev);
 		rdev->mode_info.mode_config_initialized = false;
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 716ab85a376b..e4374814f0ef 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -341,6 +341,8 @@ static int radeon_pci_probe(struct pci_dev *pdev,
 	if (ret)
 		goto err_agp;
 
+	radeon_fbdev_setup(dev->dev_private);
+
 	return 0;
 
 err_agp:
@@ -595,7 +597,6 @@ static const struct drm_driver kms_driver = {
 	.load = radeon_driver_load_kms,
 	.open = radeon_driver_open_kms,
 	.postclose = radeon_driver_postclose_kms,
-	.lastclose = radeon_driver_lastclose_kms,
 	.unload = radeon_driver_unload_kms,
 	.ioctls = radeon_ioctls_kms,
 	.num_ioctls = ARRAY_SIZE(radeon_ioctls_kms),
diff --git a/drivers/gpu/drm/radeon/radeon_drv.h b/drivers/gpu/drm/radeon/radeon_drv.h
index ac7970919c4d..2ffe0975ee54 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.h
+++ b/drivers/gpu/drm/radeon/radeon_drv.h
@@ -120,7 +120,6 @@ long radeon_drm_ioctl(struct file *filp,
 
 int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags);
 void radeon_driver_unload_kms(struct drm_device *dev);
-void radeon_driver_lastclose_kms(struct drm_device *dev);
 int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv);
 void radeon_driver_postclose_kms(struct drm_device *dev,
 				 struct drm_file *file_priv);
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
deleted file mode 100644
index c4807f0c43bc..000000000000
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Copyright © 2007 David Airlie
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *     David Airlie
- */
-
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/pm_runtime.h>
-#include <linux/slab.h>
-#include <linux/vga_switcheroo.h>
-
-#include <drm/drm_crtc.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_fb_helper.h>
-#include <drm/drm_fourcc.h>
-#include <drm/drm_framebuffer.h>
-#include <drm/radeon_drm.h>
-
-#include "radeon.h"
-
-/* object hierarchy -
- * this contains a helper + a radeon fb
- * the helper contains a pointer to radeon framebuffer baseclass.
- */
-struct radeon_fbdev {
-	struct drm_fb_helper helper; /* must be first */
-	struct drm_framebuffer fb;
-	struct radeon_device *rdev;
-};
-
-static int
-radeonfb_open(struct fb_info *info, int user)
-{
-	struct radeon_fbdev *rfbdev = info->par;
-	struct radeon_device *rdev = rfbdev->rdev;
-	int ret = pm_runtime_get_sync(rdev->ddev->dev);
-
-	if (ret < 0 && ret != -EACCES) {
-		pm_runtime_mark_last_busy(rdev->ddev->dev);
-		pm_runtime_put_autosuspend(rdev->ddev->dev);
-		return ret;
-	}
-	return 0;
-}
-
-static int
-radeonfb_release(struct fb_info *info, int user)
-{
-	struct radeon_fbdev *rfbdev = info->par;
-	struct radeon_device *rdev = rfbdev->rdev;
-
-	pm_runtime_mark_last_busy(rdev->ddev->dev);
-	pm_runtime_put_autosuspend(rdev->ddev->dev);
-	return 0;
-}
-
-static const struct fb_ops radeonfb_ops = {
-	.owner = THIS_MODULE,
-	DRM_FB_HELPER_DEFAULT_OPS,
-	.fb_open = radeonfb_open,
-	.fb_release = radeonfb_release,
-	.fb_read = drm_fb_helper_cfb_read,
-	.fb_write = drm_fb_helper_cfb_write,
-	.fb_fillrect = drm_fb_helper_cfb_fillrect,
-	.fb_copyarea = drm_fb_helper_cfb_copyarea,
-	.fb_imageblit = drm_fb_helper_cfb_imageblit,
-};
-
-
-int radeon_align_pitch(struct radeon_device *rdev, int width, int cpp, bool tiled)
-{
-	int aligned = width;
-	int align_large = (ASIC_IS_AVIVO(rdev)) || tiled;
-	int pitch_mask = 0;
-
-	switch (cpp) {
-	case 1:
-		pitch_mask = align_large ? 255 : 127;
-		break;
-	case 2:
-		pitch_mask = align_large ? 127 : 31;
-		break;
-	case 3:
-	case 4:
-		pitch_mask = align_large ? 63 : 15;
-		break;
-	}
-
-	aligned += pitch_mask;
-	aligned &= ~pitch_mask;
-	return aligned * cpp;
-}
-
-static void radeonfb_destroy_pinned_object(struct drm_gem_object *gobj)
-{
-	struct radeon_bo *rbo = gem_to_radeon_bo(gobj);
-	int ret;
-
-	ret = radeon_bo_reserve(rbo, false);
-	if (likely(ret == 0)) {
-		radeon_bo_kunmap(rbo);
-		radeon_bo_unpin(rbo);
-		radeon_bo_unreserve(rbo);
-	}
-	drm_gem_object_put(gobj);
-}
-
-static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev,
-					 struct drm_mode_fb_cmd2 *mode_cmd,
-					 struct drm_gem_object **gobj_p)
-{
-	const struct drm_format_info *info;
-	struct radeon_device *rdev = rfbdev->rdev;
-	struct drm_gem_object *gobj = NULL;
-	struct radeon_bo *rbo = NULL;
-	bool fb_tiled = false; /* useful for testing */
-	u32 tiling_flags = 0;
-	int ret;
-	int aligned_size, size;
-	int height = mode_cmd->height;
-	u32 cpp;
-
-	info = drm_get_format_info(rdev->ddev, mode_cmd);
-	cpp = info->cpp[0];
-
-	/* need to align pitch with crtc limits */
-	mode_cmd->pitches[0] = radeon_align_pitch(rdev, mode_cmd->width, cpp,
-						  fb_tiled);
-
-	if (rdev->family >= CHIP_R600)
-		height = ALIGN(mode_cmd->height, 8);
-	size = mode_cmd->pitches[0] * height;
-	aligned_size = ALIGN(size, PAGE_SIZE);
-	ret = radeon_gem_object_create(rdev, aligned_size, 0,
-				       RADEON_GEM_DOMAIN_VRAM,
-				       0, true, &gobj);
-	if (ret) {
-		pr_err("failed to allocate framebuffer (%d)\n", aligned_size);
-		return -ENOMEM;
-	}
-	rbo = gem_to_radeon_bo(gobj);
-
-	if (fb_tiled)
-		tiling_flags = RADEON_TILING_MACRO;
-
-#ifdef __BIG_ENDIAN
-	switch (cpp) {
-	case 4:
-		tiling_flags |= RADEON_TILING_SWAP_32BIT;
-		break;
-	case 2:
-		tiling_flags |= RADEON_TILING_SWAP_16BIT;
-		break;
-	default:
-		break;
-	}
-#endif
-
-	if (tiling_flags) {
-		ret = radeon_bo_set_tiling_flags(rbo,
-						 tiling_flags | RADEON_TILING_SURFACE,
-						 mode_cmd->pitches[0]);
-		if (ret)
-			dev_err(rdev->dev, "FB failed to set tiling flags\n");
-	}
-
-
-	ret = radeon_bo_reserve(rbo, false);
-	if (unlikely(ret != 0))
-		goto out_unref;
-	/* Only 27 bit offset for legacy CRTC */
-	ret = radeon_bo_pin_restricted(rbo, RADEON_GEM_DOMAIN_VRAM,
-				       ASIC_IS_AVIVO(rdev) ? 0 : 1 << 27,
-				       NULL);
-	if (ret) {
-		radeon_bo_unreserve(rbo);
-		goto out_unref;
-	}
-	if (fb_tiled)
-		radeon_bo_check_tiling(rbo, 0, 0);
-	ret = radeon_bo_kmap(rbo, NULL);
-	radeon_bo_unreserve(rbo);
-	if (ret)
-		goto out_unref;
-
-	*gobj_p = gobj;
-	return 0;
-out_unref:
-	radeonfb_destroy_pinned_object(gobj);
-	*gobj_p = NULL;
-	return ret;
-}
-
-static int radeonfb_create(struct drm_fb_helper *helper,
-			   struct drm_fb_helper_surface_size *sizes)
-{
-	struct radeon_fbdev *rfbdev =
-		container_of(helper, struct radeon_fbdev, helper);
-	struct radeon_device *rdev = rfbdev->rdev;
-	struct fb_info *info;
-	struct drm_framebuffer *fb = NULL;
-	struct drm_mode_fb_cmd2 mode_cmd;
-	struct drm_gem_object *gobj = NULL;
-	struct radeon_bo *rbo = NULL;
-	int ret;
-	unsigned long tmp;
-
-	mode_cmd.width = sizes->surface_width;
-	mode_cmd.height = sizes->surface_height;
-
-	/* avivo can't scanout real 24bpp */
-	if ((sizes->surface_bpp == 24) && ASIC_IS_AVIVO(rdev))
-		sizes->surface_bpp = 32;
-
-	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
-							  sizes->surface_depth);
-
-	ret = radeonfb_create_pinned_object(rfbdev, &mode_cmd, &gobj);
-	if (ret) {
-		DRM_ERROR("failed to create fbcon object %d\n", ret);
-		return ret;
-	}
-
-	rbo = gem_to_radeon_bo(gobj);
-
-	/* okay we have an object now allocate the framebuffer */
-	info = drm_fb_helper_alloc_info(helper);
-	if (IS_ERR(info)) {
-		ret = PTR_ERR(info);
-		goto out;
-	}
-
-	/* radeon resume is fragile and needs a vt switch to help it along */
-	info->skip_vt_switch = false;
-
-	ret = radeon_framebuffer_init(rdev->ddev, &rfbdev->fb, &mode_cmd, gobj);
-	if (ret) {
-		DRM_ERROR("failed to initialize framebuffer %d\n", ret);
-		goto out;
-	}
-
-	fb = &rfbdev->fb;
-
-	/* setup helper */
-	rfbdev->helper.fb = fb;
-
-	memset_io(rbo->kptr, 0x0, radeon_bo_size(rbo));
-
-	info->fbops = &radeonfb_ops;
-
-	tmp = radeon_bo_gpu_offset(rbo) - rdev->mc.vram_start;
-	info->fix.smem_start = rdev->mc.aper_base + tmp;
-	info->fix.smem_len = radeon_bo_size(rbo);
-	info->screen_base = rbo->kptr;
-	info->screen_size = radeon_bo_size(rbo);
-
-	drm_fb_helper_fill_info(info, &rfbdev->helper, sizes);
-
-	/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
-
-	if (info->screen_base == NULL) {
-		ret = -ENOSPC;
-		goto out;
-	}
-
-	DRM_INFO("fb mappable at 0x%lX\n",  info->fix.smem_start);
-	DRM_INFO("vram apper at 0x%lX\n",  (unsigned long)rdev->mc.aper_base);
-	DRM_INFO("size %lu\n", (unsigned long)radeon_bo_size(rbo));
-	DRM_INFO("fb depth is %d\n", fb->format->depth);
-	DRM_INFO("   pitch is %d\n", fb->pitches[0]);
-
-	vga_switcheroo_client_fb_set(rdev->pdev, info);
-	return 0;
-
-out:
-	if (fb && ret) {
-		drm_gem_object_put(gobj);
-		drm_framebuffer_unregister_private(fb);
-		drm_framebuffer_cleanup(fb);
-		kfree(fb);
-	}
-	return ret;
-}
-
-static int radeon_fbdev_destroy(struct drm_device *dev, struct radeon_fbdev *rfbdev)
-{
-	struct drm_framebuffer *fb = &rfbdev->fb;
-
-	drm_fb_helper_unregister_info(&rfbdev->helper);
-
-	if (fb->obj[0]) {
-		radeonfb_destroy_pinned_object(fb->obj[0]);
-		fb->obj[0] = NULL;
-		drm_framebuffer_unregister_private(fb);
-		drm_framebuffer_cleanup(fb);
-	}
-	drm_fb_helper_fini(&rfbdev->helper);
-
-	return 0;
-}
-
-static const struct drm_fb_helper_funcs radeon_fb_helper_funcs = {
-	.fb_probe = radeonfb_create,
-};
-
-int radeon_fbdev_init(struct radeon_device *rdev)
-{
-	struct radeon_fbdev *rfbdev;
-	int bpp_sel = 32;
-	int ret;
-
-	/* don't enable fbdev if no connectors */
-	if (list_empty(&rdev->ddev->mode_config.connector_list))
-		return 0;
-
-	/* select 8 bpp console on 8MB cards, or 16 bpp on RN50 or 32MB */
-	if (rdev->mc.real_vram_size <= (8*1024*1024))
-		bpp_sel = 8;
-	else if (ASIC_IS_RN50(rdev) ||
-		 rdev->mc.real_vram_size <= (32*1024*1024))
-		bpp_sel = 16;
-
-	rfbdev = kzalloc(sizeof(struct radeon_fbdev), GFP_KERNEL);
-	if (!rfbdev)
-		return -ENOMEM;
-
-	rfbdev->rdev = rdev;
-	rdev->mode_info.rfbdev = rfbdev;
-
-	drm_fb_helper_prepare(rdev->ddev, &rfbdev->helper, bpp_sel,
-			      &radeon_fb_helper_funcs);
-
-	ret = drm_fb_helper_init(rdev->ddev, &rfbdev->helper);
-	if (ret)
-		goto free;
-
-	/* disable all the possible outputs/crtcs before entering KMS mode */
-	drm_helper_disable_unused_functions(rdev->ddev);
-
-	ret = drm_fb_helper_initial_config(&rfbdev->helper);
-	if (ret)
-		goto fini;
-
-	return 0;
-
-fini:
-	drm_fb_helper_fini(&rfbdev->helper);
-free:
-	drm_fb_helper_unprepare(&rfbdev->helper);
-	kfree(rfbdev);
-	return ret;
-}
-
-void radeon_fbdev_fini(struct radeon_device *rdev)
-{
-	if (!rdev->mode_info.rfbdev)
-		return;
-
-	radeon_fbdev_destroy(rdev->ddev, rdev->mode_info.rfbdev);
-	drm_fb_helper_unprepare(&rdev->mode_info.rfbdev->helper);
-	kfree(rdev->mode_info.rfbdev);
-	rdev->mode_info.rfbdev = NULL;
-}
-
-void radeon_fbdev_set_suspend(struct radeon_device *rdev, int state)
-{
-	if (rdev->mode_info.rfbdev)
-		drm_fb_helper_set_suspend(&rdev->mode_info.rfbdev->helper, state);
-}
-
-bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj)
-{
-	if (!rdev->mode_info.rfbdev)
-		return false;
-
-	if (robj == gem_to_radeon_bo(rdev->mode_info.rfbdev->fb.obj[0]))
-		return true;
-	return false;
-}
diff --git a/drivers/gpu/drm/radeon/radeon_fbdev.c b/drivers/gpu/drm/radeon/radeon_fbdev.c
new file mode 100644
index 000000000000..fe76e29910ef
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_fbdev.c
@@ -0,0 +1,422 @@
+/*
+ * Copyright © 2007 David Airlie
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *     David Airlie
+ */
+
+#include <linux/pci.h>
+#include <linux/pm_runtime.h>
+#include <linux/vga_switcheroo.h>
+
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+
+#include "radeon.h"
+
+static void radeon_fbdev_destroy_pinned_object(struct drm_gem_object *gobj)
+{
+	struct radeon_bo *rbo = gem_to_radeon_bo(gobj);
+	int ret;
+
+	ret = radeon_bo_reserve(rbo, false);
+	if (likely(ret == 0)) {
+		radeon_bo_kunmap(rbo);
+		radeon_bo_unpin(rbo);
+		radeon_bo_unreserve(rbo);
+	}
+	drm_gem_object_put(gobj);
+}
+
+static int radeon_fbdev_create_pinned_object(struct drm_fb_helper *fb_helper,
+					     struct drm_mode_fb_cmd2 *mode_cmd,
+					     struct drm_gem_object **gobj_p)
+{
+	const struct drm_format_info *info;
+	struct radeon_device *rdev = fb_helper->dev->dev_private;
+	struct drm_gem_object *gobj = NULL;
+	struct radeon_bo *rbo = NULL;
+	bool fb_tiled = false; /* useful for testing */
+	u32 tiling_flags = 0;
+	int ret;
+	int aligned_size, size;
+	int height = mode_cmd->height;
+	u32 cpp;
+
+	info = drm_get_format_info(rdev->ddev, mode_cmd);
+	cpp = info->cpp[0];
+
+	/* need to align pitch with crtc limits */
+	mode_cmd->pitches[0] = radeon_align_pitch(rdev, mode_cmd->width, cpp,
+						  fb_tiled);
+
+	if (rdev->family >= CHIP_R600)
+		height = ALIGN(mode_cmd->height, 8);
+	size = mode_cmd->pitches[0] * height;
+	aligned_size = ALIGN(size, PAGE_SIZE);
+	ret = radeon_gem_object_create(rdev, aligned_size, 0,
+				       RADEON_GEM_DOMAIN_VRAM,
+				       0, true, &gobj);
+	if (ret) {
+		pr_err("failed to allocate framebuffer (%d)\n", aligned_size);
+		return -ENOMEM;
+	}
+	rbo = gem_to_radeon_bo(gobj);
+
+	if (fb_tiled)
+		tiling_flags = RADEON_TILING_MACRO;
+
+#ifdef __BIG_ENDIAN
+	switch (cpp) {
+	case 4:
+		tiling_flags |= RADEON_TILING_SWAP_32BIT;
+		break;
+	case 2:
+		tiling_flags |= RADEON_TILING_SWAP_16BIT;
+		break;
+	default:
+		break;
+	}
+#endif
+
+	if (tiling_flags) {
+		ret = radeon_bo_set_tiling_flags(rbo,
+						 tiling_flags | RADEON_TILING_SURFACE,
+						 mode_cmd->pitches[0]);
+		if (ret)
+			dev_err(rdev->dev, "FB failed to set tiling flags\n");
+	}
+
+	ret = radeon_bo_reserve(rbo, false);
+	if (unlikely(ret != 0))
+		goto err_radeon_fbdev_destroy_pinned_object;
+	/* Only 27 bit offset for legacy CRTC */
+	ret = radeon_bo_pin_restricted(rbo, RADEON_GEM_DOMAIN_VRAM,
+				       ASIC_IS_AVIVO(rdev) ? 0 : 1 << 27,
+				       NULL);
+	if (ret) {
+		radeon_bo_unreserve(rbo);
+		goto err_radeon_fbdev_destroy_pinned_object;
+	}
+	if (fb_tiled)
+		radeon_bo_check_tiling(rbo, 0, 0);
+	ret = radeon_bo_kmap(rbo, NULL);
+	radeon_bo_unreserve(rbo);
+	if (ret)
+		goto err_radeon_fbdev_destroy_pinned_object;
+
+	*gobj_p = gobj;
+	return 0;
+
+err_radeon_fbdev_destroy_pinned_object:
+	radeon_fbdev_destroy_pinned_object(gobj);
+	*gobj_p = NULL;
+	return ret;
+}
+
+/*
+ * Fbdev ops and struct fb_ops
+ */
+
+static int radeon_fbdev_fb_open(struct fb_info *info, int user)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+	struct radeon_device *rdev = fb_helper->dev->dev_private;
+	int ret;
+
+	ret = pm_runtime_get_sync(rdev->ddev->dev);
+	if (ret < 0 && ret != -EACCES)
+		goto err_pm_runtime_mark_last_busy;
+
+	return 0;
+
+err_pm_runtime_mark_last_busy:
+	pm_runtime_mark_last_busy(rdev->ddev->dev);
+	pm_runtime_put_autosuspend(rdev->ddev->dev);
+	return ret;
+}
+
+static int radeon_fbdev_fb_release(struct fb_info *info, int user)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+	struct radeon_device *rdev = fb_helper->dev->dev_private;
+
+	pm_runtime_mark_last_busy(rdev->ddev->dev);
+	pm_runtime_put_autosuspend(rdev->ddev->dev);
+
+	return 0;
+}
+
+static void radeon_fbdev_fb_destroy(struct fb_info *info)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+	struct drm_framebuffer *fb = fb_helper->fb;
+	struct drm_gem_object *gobj = drm_gem_fb_get_obj(fb, 0);
+
+	drm_fb_helper_fini(fb_helper);
+
+	drm_framebuffer_unregister_private(fb);
+	drm_framebuffer_cleanup(fb);
+	kfree(fb);
+	radeon_fbdev_destroy_pinned_object(gobj);
+
+	drm_client_release(&fb_helper->client);
+	drm_fb_helper_unprepare(fb_helper);
+	kfree(fb_helper);
+}
+
+static const struct fb_ops radeon_fbdev_fb_ops = {
+	.owner = THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
+	.fb_open = radeon_fbdev_fb_open,
+	.fb_release = radeon_fbdev_fb_release,
+	.fb_read = drm_fb_helper_cfb_read,
+	.fb_write = drm_fb_helper_cfb_write,
+	.fb_fillrect = drm_fb_helper_cfb_fillrect,
+	.fb_copyarea = drm_fb_helper_cfb_copyarea,
+	.fb_imageblit = drm_fb_helper_cfb_imageblit,
+	.fb_destroy = radeon_fbdev_fb_destroy,
+};
+
+/*
+ * Fbdev helpers and struct drm_fb_helper_funcs
+ */
+
+static int radeon_fbdev_fb_helper_fb_probe(struct drm_fb_helper *fb_helper,
+					   struct drm_fb_helper_surface_size *sizes)
+{
+	struct radeon_device *rdev = fb_helper->dev->dev_private;
+	struct drm_mode_fb_cmd2 mode_cmd = { };
+	struct fb_info *info;
+	struct drm_gem_object *gobj;
+	struct radeon_bo *rbo;
+	struct drm_framebuffer *fb;
+	int ret;
+	unsigned long tmp;
+
+	mode_cmd.width = sizes->surface_width;
+	mode_cmd.height = sizes->surface_height;
+
+	/* avivo can't scanout real 24bpp */
+	if ((sizes->surface_bpp == 24) && ASIC_IS_AVIVO(rdev))
+		sizes->surface_bpp = 32;
+
+	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
+							  sizes->surface_depth);
+
+	ret = radeon_fbdev_create_pinned_object(fb_helper, &mode_cmd, &gobj);
+	if (ret) {
+		DRM_ERROR("failed to create fbcon object %d\n", ret);
+		return ret;
+	}
+	rbo = gem_to_radeon_bo(gobj);
+
+	fb = kzalloc(sizeof(*fb), GFP_KERNEL);
+	if (!fb) {
+		ret = -ENOMEM;
+		goto err_radeon_fbdev_destroy_pinned_object;
+	}
+	ret = radeon_framebuffer_init(rdev->ddev, fb, &mode_cmd, gobj);
+	if (ret) {
+		DRM_ERROR("failed to initialize framebuffer %d\n", ret);
+		goto err_kfree;
+	}
+
+	/* setup helper */
+	fb_helper->fb = fb;
+
+	/* okay we have an object now allocate the framebuffer */
+	info = drm_fb_helper_alloc_info(fb_helper);
+	if (IS_ERR(info)) {
+		ret = PTR_ERR(info);
+		goto err_drm_framebuffer_unregister_private;
+	}
+
+	info->fbops = &radeon_fbdev_fb_ops;
+	info->flags = FBINFO_DEFAULT;
+	/* radeon resume is fragile and needs a vt switch to help it along */
+	info->skip_vt_switch = false;
+
+	drm_fb_helper_fill_info(info, fb_helper, sizes);
+
+	tmp = radeon_bo_gpu_offset(rbo) - rdev->mc.vram_start;
+	info->fix.smem_start = rdev->mc.aper_base + tmp;
+	info->fix.smem_len = radeon_bo_size(rbo);
+	info->screen_base = (__force void __iomem *)rbo->kptr;
+	info->screen_size = radeon_bo_size(rbo);
+
+	memset_io(info->screen_base, 0, info->screen_size);
+
+	/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
+
+	DRM_INFO("fb mappable at 0x%lX\n",  info->fix.smem_start);
+	DRM_INFO("vram apper at 0x%lX\n",  (unsigned long)rdev->mc.aper_base);
+	DRM_INFO("size %lu\n", (unsigned long)radeon_bo_size(rbo));
+	DRM_INFO("fb depth is %d\n", fb->format->depth);
+	DRM_INFO("   pitch is %d\n", fb->pitches[0]);
+
+	return 0;
+
+err_drm_framebuffer_unregister_private:
+	fb_helper->fb = NULL;
+	drm_framebuffer_unregister_private(fb);
+	drm_framebuffer_cleanup(fb);
+err_kfree:
+	kfree(fb);
+err_radeon_fbdev_destroy_pinned_object:
+	radeon_fbdev_destroy_pinned_object(gobj);
+	return ret;
+}
+
+static const struct drm_fb_helper_funcs radeon_fbdev_fb_helper_funcs = {
+	.fb_probe = radeon_fbdev_fb_helper_fb_probe,
+};
+
+/*
+ * Fbdev client and struct drm_client_funcs
+ */
+
+static void radeon_fbdev_client_unregister(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+	struct drm_device *dev = fb_helper->dev;
+	struct radeon_device *rdev = dev->dev_private;
+
+	if (fb_helper->info) {
+		vga_switcheroo_client_fb_set(rdev->pdev, NULL);
+		drm_fb_helper_unregister_info(fb_helper);
+	} else {
+		drm_client_release(&fb_helper->client);
+		drm_fb_helper_unprepare(fb_helper);
+		kfree(fb_helper);
+	}
+}
+
+static int radeon_fbdev_client_restore(struct drm_client_dev *client)
+{
+	drm_fb_helper_lastclose(client->dev);
+	vga_switcheroo_process_delayed_switch();
+
+	return 0;
+}
+
+static int radeon_fbdev_client_hotplug(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+	struct drm_device *dev = client->dev;
+	struct radeon_device *rdev = dev->dev_private;
+	int ret;
+
+	if (dev->fb_helper)
+		return drm_fb_helper_hotplug_event(dev->fb_helper);
+
+	ret = drm_fb_helper_init(dev, fb_helper);
+	if (ret)
+		goto err_drm_err;
+
+	if (!drm_drv_uses_atomic_modeset(dev))
+		drm_helper_disable_unused_functions(dev);
+
+	ret = drm_fb_helper_initial_config(fb_helper);
+	if (ret)
+		goto err_drm_fb_helper_fini;
+
+	vga_switcheroo_client_fb_set(rdev->pdev, fb_helper->info);
+
+	return 0;
+
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(fb_helper);
+err_drm_err:
+	drm_err(dev, "Failed to setup radeon fbdev emulation (ret=%d)\n", ret);
+	return ret;
+}
+
+static const struct drm_client_funcs radeon_fbdev_client_funcs = {
+	.owner		= THIS_MODULE,
+	.unregister	= radeon_fbdev_client_unregister,
+	.restore	= radeon_fbdev_client_restore,
+	.hotplug	= radeon_fbdev_client_hotplug,
+};
+
+void radeon_fbdev_setup(struct radeon_device *rdev)
+{
+	struct drm_fb_helper *fb_helper;
+	int bpp_sel = 32;
+	int ret;
+
+	if (rdev->mc.real_vram_size <= (8 * 1024 * 1024))
+		bpp_sel = 8;
+	else if (ASIC_IS_RN50(rdev) || rdev->mc.real_vram_size <= (32 * 1024 * 1024))
+		bpp_sel = 16;
+
+	fb_helper = kzalloc(sizeof(*fb_helper), GFP_KERNEL);
+	if (!fb_helper)
+		return;
+	drm_fb_helper_prepare(rdev->ddev, fb_helper, bpp_sel, &radeon_fbdev_fb_helper_funcs);
+
+	ret = drm_client_init(rdev->ddev, &fb_helper->client, "radeon-fbdev",
+			      &radeon_fbdev_client_funcs);
+	if (ret) {
+		drm_err(rdev->ddev, "Failed to register client: %d\n", ret);
+		goto err_drm_client_init;
+	}
+
+	ret = radeon_fbdev_client_hotplug(&fb_helper->client);
+	if (ret)
+		drm_dbg_kms(rdev->ddev, "client hotplug ret=%d\n", ret);
+
+	drm_client_register(&fb_helper->client);
+
+	return;
+
+err_drm_client_init:
+	drm_fb_helper_unprepare(fb_helper);
+	kfree(fb_helper);
+}
+
+void radeon_fbdev_set_suspend(struct radeon_device *rdev, int state)
+{
+	if (rdev->ddev->fb_helper)
+		drm_fb_helper_set_suspend(rdev->ddev->fb_helper, state);
+}
+
+bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj)
+{
+	struct drm_fb_helper *fb_helper = rdev->ddev->fb_helper;
+	struct drm_gem_object *gobj;
+
+	if (!fb_helper)
+		return false;
+
+	gobj = drm_gem_fb_get_obj(fb_helper->fb, 0);
+	if (!gobj)
+		return false;
+	if (gobj != &robj->tbo.base)
+		return false;
+
+	return true;
+}
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index 261fcbae88d7..bdc5af23f005 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -822,6 +822,30 @@ out:
 	return r;
 }
 
+int radeon_align_pitch(struct radeon_device *rdev, int width, int cpp, bool tiled)
+{
+	int aligned = width;
+	int align_large = (ASIC_IS_AVIVO(rdev)) || tiled;
+	int pitch_mask = 0;
+
+	switch (cpp) {
+	case 1:
+		pitch_mask = align_large ? 255 : 127;
+		break;
+	case 2:
+		pitch_mask = align_large ? 127 : 31;
+		break;
+	case 3:
+	case 4:
+		pitch_mask = align_large ? 63 : 15;
+		break;
+	}
+
+	aligned += pitch_mask;
+	aligned &= ~pitch_mask;
+	return aligned * cpp;
+}
+
 int radeon_mode_dumb_create(struct drm_file *file_priv,
 			    struct drm_device *dev,
 			    struct drm_mode_create_dumb *args)
diff --git a/drivers/gpu/drm/radeon/radeon_ib.c b/drivers/gpu/drm/radeon/radeon_ib.c
index 62b116727b4f..6a45a72488f9 100644
--- a/drivers/gpu/drm/radeon/radeon_ib.c
+++ b/drivers/gpu/drm/radeon/radeon_ib.c
@@ -61,7 +61,7 @@ int radeon_ib_get(struct radeon_device *rdev, int ring,
 {
 	int r;
 
-	r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256);
+	r = radeon_sa_bo_new(&rdev->ring_tmp_bo, &ib->sa_bo, size, 256);
 	if (r) {
 		dev_err(rdev->dev, "failed to get a new IB (%d)\n", r);
 		return r;
@@ -77,7 +77,7 @@ int radeon_ib_get(struct radeon_device *rdev, int ring,
 		/* ib pool is bound at RADEON_VA_IB_OFFSET in virtual address
 		 * space and soffset is the offset inside the pool bo
 		 */
-		ib->gpu_addr = ib->sa_bo->soffset + RADEON_VA_IB_OFFSET;
+		ib->gpu_addr = drm_suballoc_soffset(ib->sa_bo) + RADEON_VA_IB_OFFSET;
 	} else {
 		ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo);
 	}
@@ -97,7 +97,7 @@ int radeon_ib_get(struct radeon_device *rdev, int ring,
 void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib)
 {
 	radeon_sync_free(rdev, &ib->sync, ib->fence);
-	radeon_sa_bo_free(rdev, &ib->sa_bo, ib->fence);
+	radeon_sa_bo_free(&ib->sa_bo, ib->fence);
 	radeon_fence_unref(&ib->fence);
 }
 
@@ -201,8 +201,7 @@ int radeon_ib_pool_init(struct radeon_device *rdev)
 
 	if (rdev->family >= CHIP_BONAIRE) {
 		r = radeon_sa_bo_manager_init(rdev, &rdev->ring_tmp_bo,
-					      RADEON_IB_POOL_SIZE*64*1024,
-					      RADEON_GPU_PAGE_SIZE,
+					      RADEON_IB_POOL_SIZE*64*1024, 256,
 					      RADEON_GEM_DOMAIN_GTT,
 					      RADEON_GEM_GTT_WC);
 	} else {
@@ -210,8 +209,7 @@ int radeon_ib_pool_init(struct radeon_device *rdev)
 		 * to the command stream checking
 		 */
 		r = radeon_sa_bo_manager_init(rdev, &rdev->ring_tmp_bo,
-					      RADEON_IB_POOL_SIZE*64*1024,
-					      RADEON_GPU_PAGE_SIZE,
+					      RADEON_IB_POOL_SIZE*64*1024, 256,
 					      RADEON_GEM_DOMAIN_GTT, 0);
 	}
 	if (r) {
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 965161b8565b..e0214cf1b43b 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -32,7 +32,6 @@
 #include <linux/uaccess.h>
 #include <linux/vga_switcheroo.h>
 
-#include <drm/drm_fb_helper.h>
 #include <drm/drm_file.h>
 #include <drm/drm_ioctl.h>
 #include <drm/radeon_drm.h>
@@ -622,23 +621,6 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 	return 0;
 }
 
-
-/*
- * Outdated mess for old drm with Xorg being in charge (void function now).
- */
-/**
- * radeon_driver_lastclose_kms - drm callback for last close
- *
- * @dev: drm dev pointer
- *
- * Switch vga_switcheroo state after last close (all asics).
- */
-void radeon_driver_lastclose_kms(struct drm_device *dev)
-{
-	drm_fb_helper_lastclose(dev);
-	vga_switcheroo_process_delayed_switch();
-}
-
 /**
  * radeon_driver_open_kms - drm callback for open
  *
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index 3a59d016e8cd..1decdcec0264 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -229,8 +229,6 @@ enum radeon_dvo_chip {
 	DVO_SIL1178,
 };
 
-struct radeon_fbdev;
-
 struct radeon_afmt {
 	bool enabled;
 	int offset;
@@ -267,8 +265,6 @@ struct radeon_mode_info {
 	struct edid *bios_hardcoded_edid;
 	int bios_hardcoded_edid_size;
 
-	/* pointer to fbdev info structure */
-	struct radeon_fbdev *rfbdev;
 	/* firmware flags */
 	u16 firmware_flags;
 	/* pointer to backlight encoder */
@@ -943,17 +939,25 @@ void dce4_program_fmt(struct drm_encoder *encoder);
 void dce8_program_fmt(struct drm_encoder *encoder);
 
 /* fbdev layer */
-int radeon_fbdev_init(struct radeon_device *rdev);
-void radeon_fbdev_fini(struct radeon_device *rdev);
+#if defined(CONFIG_DRM_FBDEV_EMULATION)
+void radeon_fbdev_setup(struct radeon_device *rdev);
 void radeon_fbdev_set_suspend(struct radeon_device *rdev, int state);
 bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj);
+#else
+static inline void radeon_fbdev_setup(struct radeon_device *rdev)
+{ }
+static inline void radeon_fbdev_set_suspend(struct radeon_device *rdev, int state)
+{ }
+static inline bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj)
+{
+	return false;
+}
+#endif
 
 void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id);
 
 void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id);
 
-int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled);
-
 int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder, int fe_idx);
 void radeon_atom_release_dig_encoder(struct radeon_device *rdev, int enc_idx);
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h
index 0a6ef49e990a..39cc87a59a9a 100644
--- a/drivers/gpu/drm/radeon/radeon_object.h
+++ b/drivers/gpu/drm/radeon/radeon_object.h
@@ -169,15 +169,22 @@ extern void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence,
 /*
  * sub allocation
  */
+static inline struct radeon_sa_manager *
+to_radeon_sa_manager(struct drm_suballoc_manager *manager)
+{
+	return container_of(manager, struct radeon_sa_manager, base);
+}
 
-static inline uint64_t radeon_sa_bo_gpu_addr(struct radeon_sa_bo *sa_bo)
+static inline uint64_t radeon_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
 {
-	return sa_bo->manager->gpu_addr + sa_bo->soffset;
+	return to_radeon_sa_manager(sa_bo->manager)->gpu_addr +
+		drm_suballoc_soffset(sa_bo);
 }
 
-static inline void * radeon_sa_bo_cpu_addr(struct radeon_sa_bo *sa_bo)
+static inline void *radeon_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
 {
-	return sa_bo->manager->cpu_ptr + sa_bo->soffset;
+	return to_radeon_sa_manager(sa_bo->manager)->cpu_ptr +
+		drm_suballoc_soffset(sa_bo);
 }
 
 extern int radeon_sa_bo_manager_init(struct radeon_device *rdev,
@@ -190,12 +197,10 @@ extern int radeon_sa_bo_manager_start(struct radeon_device *rdev,
 				      struct radeon_sa_manager *sa_manager);
 extern int radeon_sa_bo_manager_suspend(struct radeon_device *rdev,
 					struct radeon_sa_manager *sa_manager);
-extern int radeon_sa_bo_new(struct radeon_device *rdev,
-			    struct radeon_sa_manager *sa_manager,
-			    struct radeon_sa_bo **sa_bo,
-			    unsigned size, unsigned align);
-extern void radeon_sa_bo_free(struct radeon_device *rdev,
-			      struct radeon_sa_bo **sa_bo,
+extern int radeon_sa_bo_new(struct radeon_sa_manager *sa_manager,
+			    struct drm_suballoc **sa_bo,
+			    unsigned int size, unsigned int align);
+extern void radeon_sa_bo_free(struct drm_suballoc **sa_bo,
 			      struct radeon_fence *fence);
 #if defined(CONFIG_DEBUG_FS)
 extern void radeon_sa_bo_dump_debug_info(struct radeon_sa_manager *sa_manager,
diff --git a/drivers/gpu/drm/radeon/radeon_sa.c b/drivers/gpu/drm/radeon/radeon_sa.c
index 0981948bd9ed..c87a57c9c592 100644
--- a/drivers/gpu/drm/radeon/radeon_sa.c
+++ b/drivers/gpu/drm/radeon/radeon_sa.c
@@ -44,53 +44,32 @@
 
 #include "radeon.h"
 
-static void radeon_sa_bo_remove_locked(struct radeon_sa_bo *sa_bo);
-static void radeon_sa_bo_try_free(struct radeon_sa_manager *sa_manager);
-
 int radeon_sa_bo_manager_init(struct radeon_device *rdev,
 			      struct radeon_sa_manager *sa_manager,
-			      unsigned size, u32 align, u32 domain, u32 flags)
+			      unsigned int size, u32 sa_align, u32 domain,
+			      u32 flags)
 {
-	int i, r;
-
-	init_waitqueue_head(&sa_manager->wq);
-	sa_manager->bo = NULL;
-	sa_manager->size = size;
-	sa_manager->domain = domain;
-	sa_manager->align = align;
-	sa_manager->hole = &sa_manager->olist;
-	INIT_LIST_HEAD(&sa_manager->olist);
-	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-		INIT_LIST_HEAD(&sa_manager->flist[i]);
-	}
+	int r;
 
-	r = radeon_bo_create(rdev, size, align, true,
+	r = radeon_bo_create(rdev, size, RADEON_GPU_PAGE_SIZE, true,
 			     domain, flags, NULL, NULL, &sa_manager->bo);
 	if (r) {
 		dev_err(rdev->dev, "(%d) failed to allocate bo for manager\n", r);
 		return r;
 	}
 
+	sa_manager->domain = domain;
+
+	drm_suballoc_manager_init(&sa_manager->base, size, sa_align);
+
 	return r;
 }
 
 void radeon_sa_bo_manager_fini(struct radeon_device *rdev,
 			       struct radeon_sa_manager *sa_manager)
 {
-	struct radeon_sa_bo *sa_bo, *tmp;
-
-	if (!list_empty(&sa_manager->olist)) {
-		sa_manager->hole = &sa_manager->olist,
-		radeon_sa_bo_try_free(sa_manager);
-		if (!list_empty(&sa_manager->olist)) {
-			dev_err(rdev->dev, "sa_manager is not empty, clearing anyway\n");
-		}
-	}
-	list_for_each_entry_safe(sa_bo, tmp, &sa_manager->olist, olist) {
-		radeon_sa_bo_remove_locked(sa_bo);
-	}
+	drm_suballoc_manager_fini(&sa_manager->base);
 	radeon_bo_unref(&sa_manager->bo);
-	sa_manager->size = 0;
 }
 
 int radeon_sa_bo_manager_start(struct radeon_device *rdev,
@@ -139,260 +118,34 @@ int radeon_sa_bo_manager_suspend(struct radeon_device *rdev,
 	return r;
 }
 
-static void radeon_sa_bo_remove_locked(struct radeon_sa_bo *sa_bo)
+int radeon_sa_bo_new(struct radeon_sa_manager *sa_manager,
+		     struct drm_suballoc **sa_bo,
+		     unsigned int size, unsigned int align)
 {
-	struct radeon_sa_manager *sa_manager = sa_bo->manager;
-	if (sa_manager->hole == &sa_bo->olist) {
-		sa_manager->hole = sa_bo->olist.prev;
-	}
-	list_del_init(&sa_bo->olist);
-	list_del_init(&sa_bo->flist);
-	radeon_fence_unref(&sa_bo->fence);
-	kfree(sa_bo);
-}
-
-static void radeon_sa_bo_try_free(struct radeon_sa_manager *sa_manager)
-{
-	struct radeon_sa_bo *sa_bo, *tmp;
-
-	if (sa_manager->hole->next == &sa_manager->olist)
-		return;
+	struct drm_suballoc *sa = drm_suballoc_new(&sa_manager->base, size,
+						   GFP_KERNEL, true, align);
 
-	sa_bo = list_entry(sa_manager->hole->next, struct radeon_sa_bo, olist);
-	list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) {
-		if (sa_bo->fence == NULL || !radeon_fence_signaled(sa_bo->fence)) {
-			return;
-		}
-		radeon_sa_bo_remove_locked(sa_bo);
+	if (IS_ERR(sa)) {
+		*sa_bo = NULL;
+		return PTR_ERR(sa);
 	}
-}
 
-static inline unsigned radeon_sa_bo_hole_soffset(struct radeon_sa_manager *sa_manager)
-{
-	struct list_head *hole = sa_manager->hole;
-
-	if (hole != &sa_manager->olist) {
-		return list_entry(hole, struct radeon_sa_bo, olist)->eoffset;
-	}
+	*sa_bo = sa;
 	return 0;
 }
 
-static inline unsigned radeon_sa_bo_hole_eoffset(struct radeon_sa_manager *sa_manager)
-{
-	struct list_head *hole = sa_manager->hole;
-
-	if (hole->next != &sa_manager->olist) {
-		return list_entry(hole->next, struct radeon_sa_bo, olist)->soffset;
-	}
-	return sa_manager->size;
-}
-
-static bool radeon_sa_bo_try_alloc(struct radeon_sa_manager *sa_manager,
-				   struct radeon_sa_bo *sa_bo,
-				   unsigned size, unsigned align)
-{
-	unsigned soffset, eoffset, wasted;
-
-	soffset = radeon_sa_bo_hole_soffset(sa_manager);
-	eoffset = radeon_sa_bo_hole_eoffset(sa_manager);
-	wasted = (align - (soffset % align)) % align;
-
-	if ((eoffset - soffset) >= (size + wasted)) {
-		soffset += wasted;
-
-		sa_bo->manager = sa_manager;
-		sa_bo->soffset = soffset;
-		sa_bo->eoffset = soffset + size;
-		list_add(&sa_bo->olist, sa_manager->hole);
-		INIT_LIST_HEAD(&sa_bo->flist);
-		sa_manager->hole = &sa_bo->olist;
-		return true;
-	}
-	return false;
-}
-
-/**
- * radeon_sa_event - Check if we can stop waiting
- *
- * @sa_manager: pointer to the sa_manager
- * @size: number of bytes we want to allocate
- * @align: alignment we need to match
- *
- * Check if either there is a fence we can wait for or
- * enough free memory to satisfy the allocation directly
- */
-static bool radeon_sa_event(struct radeon_sa_manager *sa_manager,
-			    unsigned size, unsigned align)
-{
-	unsigned soffset, eoffset, wasted;
-	int i;
-
-	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-		if (!list_empty(&sa_manager->flist[i])) {
-			return true;
-		}
-	}
-
-	soffset = radeon_sa_bo_hole_soffset(sa_manager);
-	eoffset = radeon_sa_bo_hole_eoffset(sa_manager);
-	wasted = (align - (soffset % align)) % align;
-
-	if ((eoffset - soffset) >= (size + wasted)) {
-		return true;
-	}
-
-	return false;
-}
-
-static bool radeon_sa_bo_next_hole(struct radeon_sa_manager *sa_manager,
-				   struct radeon_fence **fences,
-				   unsigned *tries)
-{
-	struct radeon_sa_bo *best_bo = NULL;
-	unsigned i, soffset, best, tmp;
-
-	/* if hole points to the end of the buffer */
-	if (sa_manager->hole->next == &sa_manager->olist) {
-		/* try again with its beginning */
-		sa_manager->hole = &sa_manager->olist;
-		return true;
-	}
-
-	soffset = radeon_sa_bo_hole_soffset(sa_manager);
-	/* to handle wrap around we add sa_manager->size */
-	best = sa_manager->size * 2;
-	/* go over all fence list and try to find the closest sa_bo
-	 * of the current last
-	 */
-	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-		struct radeon_sa_bo *sa_bo;
-
-		fences[i] = NULL;
-
-		if (list_empty(&sa_manager->flist[i])) {
-			continue;
-		}
-
-		sa_bo = list_first_entry(&sa_manager->flist[i],
-					 struct radeon_sa_bo, flist);
-
-		if (!radeon_fence_signaled(sa_bo->fence)) {
-			fences[i] = sa_bo->fence;
-			continue;
-		}
-
-		/* limit the number of tries each ring gets */
-		if (tries[i] > 2) {
-			continue;
-		}
-
-		tmp = sa_bo->soffset;
-		if (tmp < soffset) {
-			/* wrap around, pretend it's after */
-			tmp += sa_manager->size;
-		}
-		tmp -= soffset;
-		if (tmp < best) {
-			/* this sa bo is the closest one */
-			best = tmp;
-			best_bo = sa_bo;
-		}
-	}
-
-	if (best_bo) {
-		++tries[best_bo->fence->ring];
-		sa_manager->hole = best_bo->olist.prev;
-
-		/* we knew that this one is signaled,
-		   so it's save to remote it */
-		radeon_sa_bo_remove_locked(best_bo);
-		return true;
-	}
-	return false;
-}
-
-int radeon_sa_bo_new(struct radeon_device *rdev,
-		     struct radeon_sa_manager *sa_manager,
-		     struct radeon_sa_bo **sa_bo,
-		     unsigned size, unsigned align)
-{
-	struct radeon_fence *fences[RADEON_NUM_RINGS];
-	unsigned tries[RADEON_NUM_RINGS];
-	int i, r;
-
-	BUG_ON(align > sa_manager->align);
-	BUG_ON(size > sa_manager->size);
-
-	*sa_bo = kmalloc(sizeof(struct radeon_sa_bo), GFP_KERNEL);
-	if ((*sa_bo) == NULL) {
-		return -ENOMEM;
-	}
-	(*sa_bo)->manager = sa_manager;
-	(*sa_bo)->fence = NULL;
-	INIT_LIST_HEAD(&(*sa_bo)->olist);
-	INIT_LIST_HEAD(&(*sa_bo)->flist);
-
-	spin_lock(&sa_manager->wq.lock);
-	do {
-		for (i = 0; i < RADEON_NUM_RINGS; ++i)
-			tries[i] = 0;
-
-		do {
-			radeon_sa_bo_try_free(sa_manager);
-
-			if (radeon_sa_bo_try_alloc(sa_manager, *sa_bo,
-						   size, align)) {
-				spin_unlock(&sa_manager->wq.lock);
-				return 0;
-			}
-
-			/* see if we can skip over some allocations */
-		} while (radeon_sa_bo_next_hole(sa_manager, fences, tries));
-
-		for (i = 0; i < RADEON_NUM_RINGS; ++i)
-			radeon_fence_ref(fences[i]);
-
-		spin_unlock(&sa_manager->wq.lock);
-		r = radeon_fence_wait_any(rdev, fences, false);
-		for (i = 0; i < RADEON_NUM_RINGS; ++i)
-			radeon_fence_unref(&fences[i]);
-		spin_lock(&sa_manager->wq.lock);
-		/* if we have nothing to wait for block */
-		if (r == -ENOENT) {
-			r = wait_event_interruptible_locked(
-				sa_manager->wq, 
-				radeon_sa_event(sa_manager, size, align)
-			);
-		}
-
-	} while (!r);
-
-	spin_unlock(&sa_manager->wq.lock);
-	kfree(*sa_bo);
-	*sa_bo = NULL;
-	return r;
-}
-
-void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo **sa_bo,
+void radeon_sa_bo_free(struct drm_suballoc **sa_bo,
 		       struct radeon_fence *fence)
 {
-	struct radeon_sa_manager *sa_manager;
-
 	if (sa_bo == NULL || *sa_bo == NULL) {
 		return;
 	}
 
-	sa_manager = (*sa_bo)->manager;
-	spin_lock(&sa_manager->wq.lock);
-	if (fence && !radeon_fence_signaled(fence)) {
-		(*sa_bo)->fence = radeon_fence_ref(fence);
-		list_add_tail(&(*sa_bo)->flist,
-			      &sa_manager->flist[fence->ring]);
-	} else {
-		radeon_sa_bo_remove_locked(*sa_bo);
-	}
-	wake_up_all_locked(&sa_manager->wq);
-	spin_unlock(&sa_manager->wq.lock);
+	if (fence)
+		drm_suballoc_free(*sa_bo, &fence->base);
+	else
+		drm_suballoc_free(*sa_bo, NULL);
+
 	*sa_bo = NULL;
 }
 
@@ -400,25 +153,8 @@ void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo **sa_bo,
 void radeon_sa_bo_dump_debug_info(struct radeon_sa_manager *sa_manager,
 				  struct seq_file *m)
 {
-	struct radeon_sa_bo *i;
+	struct drm_printer p = drm_seq_file_printer(m);
 
-	spin_lock(&sa_manager->wq.lock);
-	list_for_each_entry(i, &sa_manager->olist, olist) {
-		uint64_t soffset = i->soffset + sa_manager->gpu_addr;
-		uint64_t eoffset = i->eoffset + sa_manager->gpu_addr;
-		if (&i->olist == sa_manager->hole) {
-			seq_printf(m, ">");
-		} else {
-			seq_printf(m, " ");
-		}
-		seq_printf(m, "[0x%010llx 0x%010llx] size %8lld",
-			   soffset, eoffset, eoffset - soffset);
-		if (i->fence) {
-			seq_printf(m, " protected by 0x%016llx on ring %d",
-				   i->fence->seq, i->fence->ring);
-		}
-		seq_printf(m, "\n");
-	}
-	spin_unlock(&sa_manager->wq.lock);
+	drm_suballoc_dump_debug_info(&sa_manager->base, &p, sa_manager->gpu_addr);
 }
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_semaphore.c b/drivers/gpu/drm/radeon/radeon_semaphore.c
index 221e59476f64..1f0a9a4ff5ae 100644
--- a/drivers/gpu/drm/radeon/radeon_semaphore.c
+++ b/drivers/gpu/drm/radeon/radeon_semaphore.c
@@ -40,7 +40,7 @@ int radeon_semaphore_create(struct radeon_device *rdev,
 	if (*semaphore == NULL) {
 		return -ENOMEM;
 	}
-	r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo,
+	r = radeon_sa_bo_new(&rdev->ring_tmp_bo,
 			     &(*semaphore)->sa_bo, 8, 8);
 	if (r) {
 		kfree(*semaphore);
@@ -100,7 +100,7 @@ void radeon_semaphore_free(struct radeon_device *rdev,
 		dev_err(rdev->dev, "semaphore %p has more waiters than signalers,"
 			" hardware lockup imminent!\n", *semaphore);
 	}
-	radeon_sa_bo_free(rdev, &(*semaphore)->sa_bo, fence);
+	radeon_sa_bo_free(&(*semaphore)->sa_bo, fence);
 	kfree(*semaphore);
 	*semaphore = NULL;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 1e8e287e113c..2220cdf6a3f6 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -211,13 +211,10 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict,
 	if (r)
 		return r;
 
-	/* Can't move a pinned BO */
 	rbo = container_of(bo, struct radeon_bo, tbo);
-	if (WARN_ON_ONCE(rbo->tbo.pin_count > 0))
-		return -EINVAL;
-
 	rdev = radeon_get_rdev(bo->bdev);
-	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
+	if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM &&
+			 bo->ttm == NULL)) {
 		ttm_bo_move_null(bo, new_mem);
 		goto out;
 	}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
index 008e172ed43b..d6d29be6b4f4 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -298,13 +298,26 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 		escr = params.escr;
 	}
 
-	if (rcdu->info->gen < 4) {
+	/*
+	 * The ESCR register only exists in DU channels that can output to an
+	 * LVDS or DPAT, and the OTAR register in DU channels that can output
+	 * to a DPAD.
+	 */
+	if ((rcdu->info->routes[RCAR_DU_OUTPUT_DPAD0].possible_crtcs |
+	     rcdu->info->routes[RCAR_DU_OUTPUT_DPAD1].possible_crtcs |
+	     rcdu->info->routes[RCAR_DU_OUTPUT_LVDS0].possible_crtcs |
+	     rcdu->info->routes[RCAR_DU_OUTPUT_LVDS1].possible_crtcs) &
+	    BIT(rcrtc->index)) {
 		dev_dbg(rcrtc->dev->dev, "%s: ESCR 0x%08x\n", __func__, escr);
 
 		rcar_du_crtc_write(rcrtc, rcrtc->index % 2 ? ESCR13 : ESCR02, escr);
-		rcar_du_crtc_write(rcrtc, rcrtc->index % 2 ? OTAR13 : OTAR02, 0);
 	}
 
+	if ((rcdu->info->routes[RCAR_DU_OUTPUT_DPAD0].possible_crtcs |
+	     rcdu->info->routes[RCAR_DU_OUTPUT_DPAD1].possible_crtcs) &
+	    BIT(rcrtc->index))
+		rcar_du_crtc_write(rcrtc, rcrtc->index % 2 ? OTAR13 : OTAR02, 0);
+
 	/* Signal polarities */
 	dsmr = ((mode->flags & DRM_MODE_FLAG_PVSYNC) ? DSMR_VSL : 0)
 	     | ((mode->flags & DRM_MODE_FLAG_PHSYNC) ? DSMR_HSL : 0)
@@ -749,16 +762,17 @@ static void rcar_du_crtc_atomic_enable(struct drm_crtc *crtc,
 
 	/*
 	 * On D3/E3 the dot clock is provided by the LVDS encoder attached to
-	 * the DU channel. We need to enable its clock output explicitly if
-	 * the LVDS output is disabled.
+	 * the DU channel. We need to enable its clock output explicitly before
+	 * starting the CRTC, as the bridge hasn't been enabled by the atomic
+	 * helpers yet.
 	 */
-	if (rcdu->info->lvds_clk_mask & BIT(rcrtc->index) &&
-	    rstate->outputs == BIT(RCAR_DU_OUTPUT_DPAD0)) {
+	if (rcdu->info->lvds_clk_mask & BIT(rcrtc->index)) {
+		bool dot_clk_only = rstate->outputs == BIT(RCAR_DU_OUTPUT_DPAD0);
 		struct drm_bridge *bridge = rcdu->lvds[rcrtc->index];
 		const struct drm_display_mode *mode =
 			&crtc->state->adjusted_mode;
 
-		rcar_lvds_pclk_enable(bridge, mode->clock * 1000);
+		rcar_lvds_pclk_enable(bridge, mode->clock * 1000, dot_clk_only);
 	}
 
 	/*
@@ -795,15 +809,16 @@ static void rcar_du_crtc_atomic_disable(struct drm_crtc *crtc,
 	rcar_du_crtc_stop(rcrtc);
 	rcar_du_crtc_put(rcrtc);
 
-	if (rcdu->info->lvds_clk_mask & BIT(rcrtc->index) &&
-	    rstate->outputs == BIT(RCAR_DU_OUTPUT_DPAD0)) {
+	if (rcdu->info->lvds_clk_mask & BIT(rcrtc->index)) {
+		bool dot_clk_only = rstate->outputs == BIT(RCAR_DU_OUTPUT_DPAD0);
 		struct drm_bridge *bridge = rcdu->lvds[rcrtc->index];
 
 		/*
 		 * Disable the LVDS clock output, see
-		 * rcar_du_crtc_atomic_enable().
+		 * rcar_du_crtc_atomic_enable(). When the LVDS output is used,
+		 * this also disables the LVDS encoder.
 		 */
-		rcar_lvds_pclk_disable(bridge);
+		rcar_lvds_pclk_disable(bridge, dot_clk_only);
 	}
 
 	if ((rcdu->info->dsi_clk_mask & BIT(rcrtc->index)) &&
@@ -815,7 +830,6 @@ static void rcar_du_crtc_atomic_disable(struct drm_crtc *crtc,
 		 * Disable the DSI clock output, see
 		 * rcar_du_crtc_atomic_enable().
 		 */
-
 		rcar_mipi_dsi_pclk_disable(bridge);
 	}
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
index b1787be31e92..7ecec7b04a8d 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
@@ -109,8 +109,8 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 	renc = drmm_encoder_alloc(&rcdu->ddev, struct rcar_du_encoder, base,
 				  &rcar_du_encoder_funcs, DRM_MODE_ENCODER_NONE,
 				  NULL);
-	if (!renc)
-		return -ENOMEM;
+	if (IS_ERR(renc))
+		return PTR_ERR(renc);
 
 	renc->output = output;
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_group.c b/drivers/gpu/drm/rcar-du/rcar_du_group.c
index 152602236377..2ccd2581f544 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_group.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_group.c
@@ -138,6 +138,7 @@ static void rcar_du_group_setup(struct rcar_du_group *rgrp)
 {
 	struct rcar_du_device *rcdu = rgrp->dev;
 	u32 defr7 = DEFR7_CODE;
+	u32 dorcr;
 
 	/* Enable extended features */
 	rcar_du_group_write(rgrp, DEFR, DEFR_CODE | DEFR_DEFE);
@@ -174,8 +175,15 @@ static void rcar_du_group_setup(struct rcar_du_group *rgrp)
 	/*
 	 * Use DS1PR and DS2PR to configure planes priorities and connects the
 	 * superposition 0 to DU0 pins. DU1 pins will be configured dynamically.
+	 *
+	 * Groups that have a single channel have a hardcoded configuration. On
+	 * Gen3 and newer, the documentation requires PG1T, DK1S and PG1D_DS1 to
+	 * always be set in this case.
 	 */
-	rcar_du_group_write(rgrp, DORCR, DORCR_PG1D_DS1 | DORCR_DPRS);
+	dorcr = DORCR_PG0D_DS0 | DORCR_DPRS;
+	if (rcdu->info->gen >= 3 && rgrp->num_crtcs == 1)
+		dorcr |= DORCR_PG1T | DORCR_DK1S | DORCR_PG1D_DS1;
+	rcar_du_group_write(rgrp, DORCR, dorcr);
 
 	/* Apply planes to CRTCs association. */
 	mutex_lock(&rgrp->lock);
@@ -349,7 +357,7 @@ int rcar_du_group_set_routing(struct rcar_du_group *rgrp)
 	struct rcar_du_device *rcdu = rgrp->dev;
 	u32 dorcr = rcar_du_group_read(rgrp, DORCR);
 
-	dorcr &= ~(DORCR_PG2T | DORCR_DK2S | DORCR_PG2D_MASK);
+	dorcr &= ~(DORCR_PG1T | DORCR_DK1S | DORCR_PG1D_MASK);
 
 	/*
 	 * Set the DPAD1 pins sources. Select CRTC 0 if explicitly requested and
@@ -357,9 +365,9 @@ int rcar_du_group_set_routing(struct rcar_du_group *rgrp)
 	 * by default.
 	 */
 	if (rcdu->dpad1_source == rgrp->index * 2)
-		dorcr |= DORCR_PG2D_DS1;
+		dorcr |= DORCR_PG1D_DS0;
 	else
-		dorcr |= DORCR_PG2T | DORCR_DK2S | DORCR_PG2D_DS2;
+		dorcr |= DORCR_PG1T | DORCR_DK1S | DORCR_PG1D_DS1;
 
 	rcar_du_group_write(rgrp, DORCR, dorcr);
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_regs.h b/drivers/gpu/drm/rcar-du/rcar_du_regs.h
index 789ae9285108..6c750fab6ebb 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_regs.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_regs.h
@@ -511,19 +511,19 @@
  */
 
 #define DORCR			0x11000
-#define DORCR_PG2T		(1 << 30)
-#define DORCR_DK2S		(1 << 28)
-#define DORCR_PG2D_DS1		(0 << 24)
-#define DORCR_PG2D_DS2		(1 << 24)
-#define DORCR_PG2D_FIX0		(2 << 24)
-#define DORCR_PG2D_DOOR		(3 << 24)
-#define DORCR_PG2D_MASK		(3 << 24)
-#define DORCR_DR1D		(1 << 21)
-#define DORCR_PG1D_DS1		(0 << 16)
-#define DORCR_PG1D_DS2		(1 << 16)
-#define DORCR_PG1D_FIX0		(2 << 16)
-#define DORCR_PG1D_DOOR		(3 << 16)
-#define DORCR_PG1D_MASK		(3 << 16)
+#define DORCR_PG1T		(1 << 30)
+#define DORCR_DK1S		(1 << 28)
+#define DORCR_PG1D_DS0		(0 << 24)
+#define DORCR_PG1D_DS1		(1 << 24)
+#define DORCR_PG1D_FIX0		(2 << 24)
+#define DORCR_PG1D_DOOR		(3 << 24)
+#define DORCR_PG1D_MASK		(3 << 24)
+#define DORCR_DR0D		(1 << 21)
+#define DORCR_PG0D_DS0		(0 << 16)
+#define DORCR_PG0D_DS1		(1 << 16)
+#define DORCR_PG0D_FIX0		(2 << 16)
+#define DORCR_PG0D_DOOR		(3 << 16)
+#define DORCR_PG0D_MASK		(3 << 16)
 #define DORCR_RGPV		(1 << 4)
 #define DORCR_DPRS		(1 << 0)
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
index fe90be51d64e..45c05d0ffc70 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
@@ -73,7 +73,7 @@ void rcar_du_vsp_enable(struct rcar_du_crtc *crtc)
 			.src.y2 = mode->vdisplay << 16,
 			.zpos = 0,
 		},
-		.format = rcar_du_format_info(DRM_FORMAT_ARGB8888),
+		.format = rcar_du_format_info(DRM_FORMAT_XRGB8888),
 		.source = RCAR_DU_PLANE_VSPD1,
 		.colorkey = 0,
 	};
diff --git a/drivers/gpu/drm/rcar-du/rcar_lvds.c b/drivers/gpu/drm/rcar-du/rcar_lvds.c
index 260ea5d8624e..ca215b588fd7 100644
--- a/drivers/gpu/drm/rcar-du/rcar_lvds.c
+++ b/drivers/gpu/drm/rcar-du/rcar_lvds.c
@@ -269,8 +269,8 @@ done:
 		pll->pll_m, pll->pll_n, pll->pll_e, pll->div);
 }
 
-static void __rcar_lvds_pll_setup_d3_e3(struct rcar_lvds *lvds,
-					unsigned int freq, bool dot_clock_only)
+static void rcar_lvds_pll_setup_d3_e3(struct rcar_lvds *lvds,
+				      unsigned int freq, bool dot_clock_only)
 {
 	struct pll_info pll = { .diff = (unsigned long)-1 };
 	u32 lvdpllcr;
@@ -305,52 +305,8 @@ static void __rcar_lvds_pll_setup_d3_e3(struct rcar_lvds *lvds,
 		rcar_lvds_write(lvds, LVDDIV, 0);
 }
 
-static void rcar_lvds_pll_setup_d3_e3(struct rcar_lvds *lvds, unsigned int freq)
-{
-	__rcar_lvds_pll_setup_d3_e3(lvds, freq, false);
-}
-
 /* -----------------------------------------------------------------------------
- * Clock - D3/E3 only
- */
-
-int rcar_lvds_pclk_enable(struct drm_bridge *bridge, unsigned long freq)
-{
-	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
-	int ret;
-
-	if (WARN_ON(!(lvds->info->quirks & RCAR_LVDS_QUIRK_EXT_PLL)))
-		return -ENODEV;
-
-	dev_dbg(lvds->dev, "enabling LVDS PLL, freq=%luHz\n", freq);
-
-	ret = pm_runtime_resume_and_get(lvds->dev);
-	if (ret)
-		return ret;
-
-	__rcar_lvds_pll_setup_d3_e3(lvds, freq, true);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(rcar_lvds_pclk_enable);
-
-void rcar_lvds_pclk_disable(struct drm_bridge *bridge)
-{
-	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
-
-	if (WARN_ON(!(lvds->info->quirks & RCAR_LVDS_QUIRK_EXT_PLL)))
-		return;
-
-	dev_dbg(lvds->dev, "disabling LVDS PLL\n");
-
-	rcar_lvds_write(lvds, LVDPLLCR, 0);
-
-	pm_runtime_put_sync(lvds->dev);
-}
-EXPORT_SYMBOL_GPL(rcar_lvds_pclk_disable);
-
-/* -----------------------------------------------------------------------------
- * Bridge
+ * Enable/disable
  */
 
 static enum rcar_lvds_mode rcar_lvds_get_lvds_mode(struct rcar_lvds *lvds,
@@ -394,10 +350,10 @@ static enum rcar_lvds_mode rcar_lvds_get_lvds_mode(struct rcar_lvds *lvds,
 	return mode;
 }
 
-static void __rcar_lvds_atomic_enable(struct drm_bridge *bridge,
-				      struct drm_atomic_state *state,
-				      struct drm_crtc *crtc,
-				      struct drm_connector *connector)
+static void rcar_lvds_enable(struct drm_bridge *bridge,
+			     struct drm_atomic_state *state,
+			     struct drm_crtc *crtc,
+			     struct drm_connector *connector)
 {
 	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
 	u32 lvdhcr;
@@ -410,8 +366,7 @@ static void __rcar_lvds_atomic_enable(struct drm_bridge *bridge,
 
 	/* Enable the companion LVDS encoder in dual-link mode. */
 	if (lvds->link_type != RCAR_LVDS_SINGLE_LINK && lvds->companion)
-		__rcar_lvds_atomic_enable(lvds->companion, state, crtc,
-					  connector);
+		rcar_lvds_enable(lvds->companion, state, crtc, connector);
 
 	/*
 	 * Hardcode the channels and control signals routing for now.
@@ -465,8 +420,12 @@ static void __rcar_lvds_atomic_enable(struct drm_bridge *bridge,
 	/*
 	 * PLL clock configuration on all instances but the companion in
 	 * dual-link mode.
+	 *
+	 * The extended PLL has been turned on by an explicit call to
+	 * rcar_lvds_pclk_enable() from the DU driver.
 	 */
-	if (lvds->link_type == RCAR_LVDS_SINGLE_LINK || lvds->companion) {
+	if ((lvds->link_type == RCAR_LVDS_SINGLE_LINK || lvds->companion) &&
+	    !(lvds->info->quirks & RCAR_LVDS_QUIRK_EXT_PLL)) {
 		const struct drm_crtc_state *crtc_state =
 			drm_atomic_get_new_crtc_state(state, crtc);
 		const struct drm_display_mode *mode =
@@ -531,22 +490,7 @@ static void __rcar_lvds_atomic_enable(struct drm_bridge *bridge,
 	rcar_lvds_write(lvds, LVDCR0, lvdcr0);
 }
 
-static void rcar_lvds_atomic_enable(struct drm_bridge *bridge,
-				    struct drm_bridge_state *old_bridge_state)
-{
-	struct drm_atomic_state *state = old_bridge_state->base.state;
-	struct drm_connector *connector;
-	struct drm_crtc *crtc;
-
-	connector = drm_atomic_get_new_connector_for_encoder(state,
-							     bridge->encoder);
-	crtc = drm_atomic_get_new_connector_state(state, connector)->crtc;
-
-	__rcar_lvds_atomic_enable(bridge, state, crtc, connector);
-}
-
-static void rcar_lvds_atomic_disable(struct drm_bridge *bridge,
-				     struct drm_bridge_state *old_bridge_state)
+static void rcar_lvds_disable(struct drm_bridge *bridge)
 {
 	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
 	u32 lvdcr0;
@@ -578,15 +522,99 @@ static void rcar_lvds_atomic_disable(struct drm_bridge *bridge,
 
 	rcar_lvds_write(lvds, LVDCR0, 0);
 	rcar_lvds_write(lvds, LVDCR1, 0);
-	rcar_lvds_write(lvds, LVDPLLCR, 0);
+
+	/* The extended PLL is turned off in rcar_lvds_pclk_disable(). */
+	if (!(lvds->info->quirks & RCAR_LVDS_QUIRK_EXT_PLL))
+		rcar_lvds_write(lvds, LVDPLLCR, 0);
 
 	/* Disable the companion LVDS encoder in dual-link mode. */
 	if (lvds->link_type != RCAR_LVDS_SINGLE_LINK && lvds->companion)
-		lvds->companion->funcs->atomic_disable(lvds->companion,
-						       old_bridge_state);
+		rcar_lvds_disable(lvds->companion);
+
+	pm_runtime_put_sync(lvds->dev);
+}
+
+/* -----------------------------------------------------------------------------
+ * Clock - D3/E3 only
+ */
+
+int rcar_lvds_pclk_enable(struct drm_bridge *bridge, unsigned long freq,
+			  bool dot_clk_only)
+{
+	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
+	int ret;
+
+	if (WARN_ON(!(lvds->info->quirks & RCAR_LVDS_QUIRK_EXT_PLL)))
+		return -ENODEV;
+
+	dev_dbg(lvds->dev, "enabling LVDS PLL, freq=%luHz\n", freq);
+
+	ret = pm_runtime_resume_and_get(lvds->dev);
+	if (ret)
+		return ret;
+
+	rcar_lvds_pll_setup_d3_e3(lvds, freq, dot_clk_only);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(rcar_lvds_pclk_enable);
+
+void rcar_lvds_pclk_disable(struct drm_bridge *bridge, bool dot_clk_only)
+{
+	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
+
+	if (WARN_ON(!(lvds->info->quirks & RCAR_LVDS_QUIRK_EXT_PLL)))
+		return;
+
+	dev_dbg(lvds->dev, "disabling LVDS PLL\n");
+
+	if (!dot_clk_only)
+		rcar_lvds_disable(bridge);
+
+	rcar_lvds_write(lvds, LVDPLLCR, 0);
 
 	pm_runtime_put_sync(lvds->dev);
 }
+EXPORT_SYMBOL_GPL(rcar_lvds_pclk_disable);
+
+/* -----------------------------------------------------------------------------
+ * Bridge
+ */
+
+static void rcar_lvds_atomic_enable(struct drm_bridge *bridge,
+				    struct drm_bridge_state *old_bridge_state)
+{
+	struct drm_atomic_state *state = old_bridge_state->base.state;
+	struct drm_connector *connector;
+	struct drm_crtc *crtc;
+
+	connector = drm_atomic_get_new_connector_for_encoder(state,
+							     bridge->encoder);
+	crtc = drm_atomic_get_new_connector_state(state, connector)->crtc;
+
+	rcar_lvds_enable(bridge, state, crtc, connector);
+}
+
+static void rcar_lvds_atomic_disable(struct drm_bridge *bridge,
+				     struct drm_bridge_state *old_bridge_state)
+{
+	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
+
+	/*
+	 * For D3 and E3, disabling the LVDS encoder before the DU would stall
+	 * the DU, causing a vblank wait timeout when stopping the DU. This has
+	 * been traced to clearing the LVEN bit, but the exact reason is
+	 * unknown. Keep the encoder enabled, it will be disabled by an explicit
+	 * call to rcar_lvds_pclk_disable() from the DU driver.
+	 *
+	 * We could clear the LVRES bit already to disable the LVDS output, but
+	 * that's likely pointless.
+	 */
+	if (lvds->info->quirks & RCAR_LVDS_QUIRK_EXT_PLL)
+		return;
+
+	rcar_lvds_disable(bridge);
+}
 
 static bool rcar_lvds_mode_fixup(struct drm_bridge *bridge,
 				 const struct drm_display_mode *mode,
@@ -922,14 +950,12 @@ static const struct rcar_lvds_device_info rcar_lvds_r8a77990_info = {
 	.gen = 3,
 	.quirks = RCAR_LVDS_QUIRK_GEN3_LVEN | RCAR_LVDS_QUIRK_EXT_PLL
 		| RCAR_LVDS_QUIRK_DUAL_LINK,
-	.pll_setup = rcar_lvds_pll_setup_d3_e3,
 };
 
 static const struct rcar_lvds_device_info rcar_lvds_r8a77995_info = {
 	.gen = 3,
 	.quirks = RCAR_LVDS_QUIRK_GEN3_LVEN | RCAR_LVDS_QUIRK_PWD
 		| RCAR_LVDS_QUIRK_EXT_PLL | RCAR_LVDS_QUIRK_DUAL_LINK,
-	.pll_setup = rcar_lvds_pll_setup_d3_e3,
 };
 
 static const struct of_device_id rcar_lvds_of_table[] = {
diff --git a/drivers/gpu/drm/rcar-du/rcar_lvds.h b/drivers/gpu/drm/rcar-du/rcar_lvds.h
index bee7033b60d6..887c63500000 100644
--- a/drivers/gpu/drm/rcar-du/rcar_lvds.h
+++ b/drivers/gpu/drm/rcar-du/rcar_lvds.h
@@ -13,17 +13,21 @@
 struct drm_bridge;
 
 #if IS_ENABLED(CONFIG_DRM_RCAR_LVDS)
-int rcar_lvds_pclk_enable(struct drm_bridge *bridge, unsigned long freq);
-void rcar_lvds_pclk_disable(struct drm_bridge *bridge);
+int rcar_lvds_pclk_enable(struct drm_bridge *bridge, unsigned long freq,
+			  bool dot_clk_only);
+void rcar_lvds_pclk_disable(struct drm_bridge *bridge, bool dot_clk_only);
 bool rcar_lvds_dual_link(struct drm_bridge *bridge);
 bool rcar_lvds_is_connected(struct drm_bridge *bridge);
 #else
 static inline int rcar_lvds_pclk_enable(struct drm_bridge *bridge,
-					unsigned long freq)
+					unsigned long freq, bool dot_clk_only)
 {
 	return -ENOSYS;
 }
-static inline void rcar_lvds_pclk_disable(struct drm_bridge *bridge) { }
+static inline void rcar_lvds_pclk_disable(struct drm_bridge *bridge,
+					  bool dot_clock_only)
+{
+}
 static inline bool rcar_lvds_dual_link(struct drm_bridge *bridge)
 {
 	return false;
diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
index 7901c3babc8c..917e79951aac 100644
--- a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
@@ -359,11 +359,6 @@ static inline void dsi_write(struct dw_mipi_dsi_rockchip *dsi, u32 reg, u32 val)
 	writel(val, dsi->base + reg);
 }
 
-static inline u32 dsi_read(struct dw_mipi_dsi_rockchip *dsi, u32 reg)
-{
-	return readl(dsi->base + reg);
-}
-
 static void dw_mipi_dsi_phy_write(struct dw_mipi_dsi_rockchip *dsi,
 				  u8 test_code,
 				  u8 test_data)
diff --git a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
index 2f4b8f64cbad..112699949db9 100644
--- a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
@@ -74,6 +74,7 @@ struct rockchip_hdmi {
 	struct regmap *regmap;
 	struct rockchip_encoder encoder;
 	const struct rockchip_hdmi_chip_data *chip_data;
+	const struct dw_hdmi_plat_data *plat_data;
 	struct clk *ref_clk;
 	struct clk *grf_clk;
 	struct dw_hdmi *hdmi;
@@ -161,6 +162,12 @@ static const struct dw_hdmi_mpll_config rockchip_mpll_cfg[] = {
 			{ 0x4064, 0x0003}
 		},
 	}, {
+		340000000, {
+			{ 0x0040, 0x0003 },
+			{ 0x3b4c, 0x0003 },
+			{ 0x5a64, 0x0003 },
+		},
+	}, {
 		~0UL, {
 			{ 0x00a0, 0x000a },
 			{ 0x2001, 0x000f },
@@ -186,6 +193,8 @@ static const struct dw_hdmi_curr_ctrl rockchip_cur_ctr[] = {
 	}, {
 		148500000, { 0x0000, 0x0038, 0x0038 },
 	}, {
+		600000000, { 0x0000, 0x0000, 0x0000 },
+	}, {
 		~0UL,      { 0x0000, 0x0000, 0x0000},
 	}
 };
@@ -241,23 +250,39 @@ static int rockchip_hdmi_parse_dt(struct rockchip_hdmi *hdmi)
 }
 
 static enum drm_mode_status
-dw_hdmi_rockchip_mode_valid(struct dw_hdmi *hdmi, void *data,
+dw_hdmi_rockchip_mode_valid(struct dw_hdmi *dw_hdmi, void *data,
 			    const struct drm_display_info *info,
 			    const struct drm_display_mode *mode)
 {
+	struct rockchip_hdmi *hdmi = data;
 	const struct dw_hdmi_mpll_config *mpll_cfg = rockchip_mpll_cfg;
 	int pclk = mode->clock * 1000;
-	bool valid = false;
+	bool exact_match = hdmi->plat_data->phy_force_vendor;
 	int i;
 
+	if (hdmi->ref_clk) {
+		int rpclk = clk_round_rate(hdmi->ref_clk, pclk);
+
+		if (abs(rpclk - pclk) > pclk / 1000)
+			return MODE_NOCLOCK;
+	}
+
 	for (i = 0; mpll_cfg[i].mpixelclock != (~0UL); i++) {
-		if (pclk == mpll_cfg[i].mpixelclock) {
-			valid = true;
-			break;
-		}
+		/*
+		 * For vendor specific phys force an exact match of the pixelclock
+		 * to preserve the original behaviour of the driver.
+		 */
+		if (exact_match && pclk == mpll_cfg[i].mpixelclock)
+			return MODE_OK;
+		/*
+		 * The Synopsys phy can work with pixelclocks up to the value given
+		 * in the corresponding mpll_cfg entry.
+		 */
+		if (!exact_match && pclk <= mpll_cfg[i].mpixelclock)
+			return MODE_OK;
 	}
 
-	return (valid) ? MODE_OK : MODE_BAD;
+	return MODE_BAD;
 }
 
 static void dw_hdmi_rockchip_encoder_disable(struct drm_encoder *encoder)
@@ -546,8 +571,10 @@ static int dw_hdmi_rockchip_bind(struct device *dev, struct device *master,
 		return -ENOMEM;
 
 	hdmi->dev = &pdev->dev;
+	hdmi->plat_data = plat_data;
 	hdmi->chip_data = plat_data->phy_data;
 	plat_data->phy_data = hdmi;
+	plat_data->priv_data = hdmi;
 	encoder = &hdmi->encoder.encoder;
 
 	encoder->possible_crtcs = drm_of_find_possible_crtcs(drm, dev->of_node);
@@ -640,6 +667,7 @@ static void dw_hdmi_rockchip_unbind(struct device *dev, struct device *master,
 	struct rockchip_hdmi *hdmi = dev_get_drvdata(dev);
 
 	dw_hdmi_unbind(hdmi->hdmi);
+	drm_encoder_cleanup(&hdmi->encoder.encoder);
 	clk_disable_unprepare(hdmi->ref_clk);
 
 	regulator_disable(hdmi->avdd_1v8);
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
index 8ea09d915c3c..b8f8b45ebf59 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
@@ -261,9 +261,6 @@ static int rockchip_drm_gem_object_mmap(struct drm_gem_object *obj,
 	else
 		ret = rockchip_drm_gem_object_mmap_dma(obj, vma);
 
-	if (ret)
-		drm_gem_vm_close(vma);
-
 	return ret;
 }
 
@@ -518,8 +515,14 @@ int rockchip_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
 
 	if (rk_obj->pages) {
-		void *vaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,
-				  pgprot_writecombine(PAGE_KERNEL));
+		void *vaddr;
+
+		if (rk_obj->kvaddr)
+			vaddr = rk_obj->kvaddr;
+		else
+			vaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,
+				     pgprot_writecombine(PAGE_KERNEL));
+
 		if (!vaddr)
 			return -ENOMEM;
 		iosys_map_set_vaddr(map, vaddr);
@@ -539,7 +542,8 @@ void rockchip_gem_prime_vunmap(struct drm_gem_object *obj,
 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
 
 	if (rk_obj->pages) {
-		vunmap(map->vaddr);
+		if (map->vaddr != rk_obj->kvaddr)
+			vunmap(map->vaddr);
 		return;
 	}
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index fa1f4ee6d195..d8f5e064a1ba 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -316,13 +316,10 @@ static int vop_convert_afbc_format(uint32_t format)
 	case DRM_FORMAT_RGB565:
 	case DRM_FORMAT_BGR565:
 		return AFBC_FMT_RGB565;
-	/* either of the below should not be reachable */
 	default:
-		DRM_WARN_ONCE("unsupported AFBC format[%08x]\n", format);
+		DRM_DEBUG_KMS("unsupported AFBC format[%08x]\n", format);
 		return -EINVAL;
 	}
-
-	return -EINVAL;
 }
 
 static uint16_t scl_vop_cal_scale(enum scale_mode mode, uint32_t src,
@@ -1174,6 +1171,17 @@ static void vop_crtc_disable_vblank(struct drm_crtc *crtc)
 	spin_unlock_irqrestore(&vop->irq_lock, flags);
 }
 
+static enum drm_mode_status vop_crtc_mode_valid(struct drm_crtc *crtc,
+						const struct drm_display_mode *mode)
+{
+	struct vop *vop = to_vop(crtc);
+
+	if (vop->data->max_output.width && mode->hdisplay > vop->data->max_output.width)
+		return MODE_BAD_HVALUE;
+
+	return MODE_OK;
+}
+
 static bool vop_crtc_mode_fixup(struct drm_crtc *crtc,
 				const struct drm_display_mode *mode,
 				struct drm_display_mode *adjusted_mode)
@@ -1585,6 +1593,7 @@ static void vop_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs vop_crtc_helper_funcs = {
+	.mode_valid = vop_crtc_mode_valid,
 	.mode_fixup = vop_crtc_mode_fixup,
 	.atomic_check = vop_crtc_atomic_check,
 	.atomic_begin = vop_crtc_atomic_begin,
@@ -2221,7 +2230,7 @@ static int vop_bind(struct device *dev, struct device *master, void *data)
 		goto err_disable_pm_runtime;
 
 	if (vop->data->feature & VOP_FEATURE_INTERNAL_RGB) {
-		vop->rgb = rockchip_rgb_init(dev, &vop->crtc, vop->drm_dev);
+		vop->rgb = rockchip_rgb_init(dev, &vop->crtc, vop->drm_dev, 0);
 		if (IS_ERR(vop->rgb)) {
 			ret = PTR_ERR(vop->rgb);
 			goto err_disable_pm_runtime;
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
index 8502849833d9..5f56e0597df8 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
@@ -42,6 +42,11 @@ enum vop_data_format {
 	VOP_FMT_YUV444SP,
 };
 
+struct vop_rect {
+	int width;
+	int height;
+};
+
 struct vop_reg {
 	uint32_t mask;
 	uint16_t offset;
@@ -225,6 +230,7 @@ struct vop_data {
 	const struct vop_win_data *win;
 	unsigned int win_size;
 	unsigned int lut_size;
+	struct vop_rect max_output;
 
 #define VOP_FEATURE_OUTPUT_RGB10	BIT(0)
 #define VOP_FEATURE_INTERNAL_RGB	BIT(1)
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
index ba3b81789509..ca73b8ccc29f 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
@@ -38,6 +38,7 @@
 #include "rockchip_drm_gem.h"
 #include "rockchip_drm_fb.h"
 #include "rockchip_drm_vop2.h"
+#include "rockchip_rgb.h"
 
 /*
  * VOP2 architecture
@@ -211,6 +212,9 @@ struct vop2 {
 	struct clk *hclk;
 	struct clk *aclk;
 
+	/* optional internal rgb encoder */
+	struct rockchip_rgb *rgb;
+
 	/* must be put at the end of the struct */
 	struct vop2_win win[];
 };
@@ -839,6 +843,8 @@ static void vop2_enable(struct vop2 *vop2)
 		return;
 	}
 
+	regcache_sync(vop2->map);
+
 	if (vop2->data->soc_id == 3566)
 		vop2_writel(vop2, RK3568_OTP_WIN_EN, 1);
 
@@ -867,6 +873,8 @@ static void vop2_disable(struct vop2 *vop2)
 
 	pm_runtime_put_sync(vop2->dev);
 
+	regcache_mark_dirty(vop2->map);
+
 	clk_disable_unprepare(vop2->aclk);
 	clk_disable_unprepare(vop2->hclk);
 }
@@ -1434,6 +1442,8 @@ static void rk3568_set_intf_mux(struct vop2_video_port *vp, int id,
 		die &= ~RK3568_SYS_DSP_INFACE_EN_RGB_MUX;
 		die |= RK3568_SYS_DSP_INFACE_EN_RGB |
 			   FIELD_PREP(RK3568_SYS_DSP_INFACE_EN_RGB_MUX, vp->id);
+		dip &= ~RK3568_DSP_IF_POL__RGB_LVDS_PIN_POL;
+		dip |= FIELD_PREP(RK3568_DSP_IF_POL__RGB_LVDS_PIN_POL, polflags);
 		if (polflags & POLFLAG_DCLK_INV)
 			regmap_write(vop2->grf, RK3568_GRF_VO_CON1, BIT(3 + 16) | BIT(3));
 		else
@@ -2245,7 +2255,7 @@ static struct vop2_video_port *find_vp_without_primary(struct vop2 *vop2)
 
 #define NR_LAYERS 6
 
-static int vop2_create_crtc(struct vop2 *vop2)
+static int vop2_create_crtcs(struct vop2 *vop2)
 {
 	const struct vop2_data *vop2_data = vop2->data;
 	struct drm_device *drm = vop2->drm;
@@ -2295,7 +2305,7 @@ static int vop2_create_crtc(struct vop2 *vop2)
 	nvp = 0;
 	for (i = 0; i < vop2->registered_num_wins; i++) {
 		struct vop2_win *win = &vop2->win[i];
-		u32 possible_crtcs;
+		u32 possible_crtcs = 0;
 
 		if (vop2->data->soc_id == 3566) {
 			/*
@@ -2370,15 +2380,44 @@ static int vop2_create_crtc(struct vop2 *vop2)
 	return 0;
 }
 
-static void vop2_destroy_crtc(struct drm_crtc *crtc)
+static void vop2_destroy_crtcs(struct vop2 *vop2)
 {
-	of_node_put(crtc->port);
+	struct drm_device *drm = vop2->drm;
+	struct list_head *crtc_list = &drm->mode_config.crtc_list;
+	struct list_head *plane_list = &drm->mode_config.plane_list;
+	struct drm_crtc *crtc, *tmpc;
+	struct drm_plane *plane, *tmpp;
+
+	list_for_each_entry_safe(plane, tmpp, plane_list, head)
+		drm_plane_cleanup(plane);
 
 	/*
 	 * Destroy CRTC after vop2_plane_destroy() since vop2_disable_plane()
 	 * references the CRTC.
 	 */
-	drm_crtc_cleanup(crtc);
+	list_for_each_entry_safe(crtc, tmpc, crtc_list, head) {
+		of_node_put(crtc->port);
+		drm_crtc_cleanup(crtc);
+	}
+}
+
+static int vop2_find_rgb_encoder(struct vop2 *vop2)
+{
+	struct device_node *node = vop2->dev->of_node;
+	struct device_node *endpoint;
+	int i;
+
+	for (i = 0; i < vop2->data->nr_vps; i++) {
+		endpoint = of_graph_get_endpoint_by_regs(node, i,
+							 ROCKCHIP_VOP2_EP_RGB0);
+		if (!endpoint)
+			continue;
+
+		of_node_put(endpoint);
+		return i;
+	}
+
+	return -ENOENT;
 }
 
 static struct reg_field vop2_cluster_regs[VOP2_WIN_MAX_REG] = {
@@ -2621,7 +2660,7 @@ static int vop2_bind(struct device *dev, struct device *master, void *data)
 		return -ENODEV;
 
 	/* Allocate vop2 struct and its vop2_win array */
-	alloc_size = sizeof(*vop2) + sizeof(*vop2->win) * vop2_data->win_size;
+	alloc_size = struct_size(vop2, win, vop2_data->win_size);
 	vop2 = devm_kzalloc(dev, alloc_size, GFP_KERNEL);
 	if (!vop2)
 		return -ENOMEM;
@@ -2644,6 +2683,8 @@ static int vop2_bind(struct device *dev, struct device *master, void *data)
 	vop2->len = resource_size(res);
 
 	vop2->map = devm_regmap_init_mmio(dev, vop2->regs, &vop2_regmap_config);
+	if (IS_ERR(vop2->map))
+		return PTR_ERR(vop2->map);
 
 	ret = vop2_win_init(vop2);
 	if (ret)
@@ -2682,33 +2723,45 @@ static int vop2_bind(struct device *dev, struct device *master, void *data)
 	if (ret)
 		return ret;
 
-	ret = vop2_create_crtc(vop2);
+	ret = vop2_create_crtcs(vop2);
 	if (ret)
 		return ret;
 
+	ret = vop2_find_rgb_encoder(vop2);
+	if (ret >= 0) {
+		vop2->rgb = rockchip_rgb_init(dev, &vop2->vps[ret].crtc,
+					      vop2->drm, ret);
+		if (IS_ERR(vop2->rgb)) {
+			if (PTR_ERR(vop2->rgb) == -EPROBE_DEFER) {
+				ret = PTR_ERR(vop2->rgb);
+				goto err_crtcs;
+			}
+			vop2->rgb = NULL;
+		}
+	}
+
 	rockchip_drm_dma_init_device(vop2->drm, vop2->dev);
 
 	pm_runtime_enable(&pdev->dev);
 
 	return 0;
+
+err_crtcs:
+	vop2_destroy_crtcs(vop2);
+
+	return ret;
 }
 
 static void vop2_unbind(struct device *dev, struct device *master, void *data)
 {
 	struct vop2 *vop2 = dev_get_drvdata(dev);
-	struct drm_device *drm = vop2->drm;
-	struct list_head *plane_list = &drm->mode_config.plane_list;
-	struct list_head *crtc_list = &drm->mode_config.crtc_list;
-	struct drm_crtc *crtc, *tmpc;
-	struct drm_plane *plane, *tmpp;
 
 	pm_runtime_disable(dev);
 
-	list_for_each_entry_safe(plane, tmpp, plane_list, head)
-		drm_plane_cleanup(plane);
+	if (vop2->rgb)
+		rockchip_rgb_fini(vop2->rgb);
 
-	list_for_each_entry_safe(crtc, tmpc, crtc_list, head)
-		vop2_destroy_crtc(crtc);
+	vop2_destroy_crtcs(vop2);
 }
 
 const struct component_ops vop2_component_ops = {
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h
index c727093a06d6..f1234a151130 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h
@@ -27,11 +27,6 @@ enum win_dly_mode {
 	VOP2_DLY_MODE_MAX,
 };
 
-struct vop_rect {
-	int width;
-	int height;
-};
-
 enum vop2_scale_up_mode {
 	VOP2_SCALE_UP_NRST_NBOR,
 	VOP2_SCALE_UP_BIL,
diff --git a/drivers/gpu/drm/rockchip/rockchip_rgb.c b/drivers/gpu/drm/rockchip/rockchip_rgb.c
index 75eb7cca3d82..c677b71ae516 100644
--- a/drivers/gpu/drm/rockchip/rockchip_rgb.c
+++ b/drivers/gpu/drm/rockchip/rockchip_rgb.c
@@ -22,13 +22,11 @@
 #include "rockchip_drm_vop.h"
 #include "rockchip_rgb.h"
 
-#define encoder_to_rgb(c) container_of(c, struct rockchip_rgb, encoder)
-
 struct rockchip_rgb {
 	struct device *dev;
 	struct drm_device *drm_dev;
 	struct drm_bridge *bridge;
-	struct drm_encoder encoder;
+	struct rockchip_encoder encoder;
 	struct drm_connector connector;
 	int output_mode;
 };
@@ -74,7 +72,8 @@ struct drm_encoder_helper_funcs rockchip_rgb_encoder_helper_funcs = {
 
 struct rockchip_rgb *rockchip_rgb_init(struct device *dev,
 				       struct drm_crtc *crtc,
-				       struct drm_device *drm_dev)
+				       struct drm_device *drm_dev,
+				       int video_port)
 {
 	struct rockchip_rgb *rgb;
 	struct drm_encoder *encoder;
@@ -92,7 +91,7 @@ struct rockchip_rgb *rockchip_rgb_init(struct device *dev,
 	rgb->dev = dev;
 	rgb->drm_dev = drm_dev;
 
-	port = of_graph_get_port_by_id(dev->of_node, 0);
+	port = of_graph_get_port_by_id(dev->of_node, video_port);
 	if (!port)
 		return ERR_PTR(-EINVAL);
 
@@ -105,8 +104,8 @@ struct rockchip_rgb *rockchip_rgb_init(struct device *dev,
 			continue;
 
 		child_count++;
-		ret = drm_of_find_panel_or_bridge(dev->of_node, 0, endpoint_id,
-						  &panel, &bridge);
+		ret = drm_of_find_panel_or_bridge(dev->of_node, video_port,
+						  endpoint_id, &panel, &bridge);
 		if (!ret) {
 			of_node_put(endpoint);
 			break;
@@ -125,7 +124,7 @@ struct rockchip_rgb *rockchip_rgb_init(struct device *dev,
 		return ERR_PTR(ret);
 	}
 
-	encoder = &rgb->encoder;
+	encoder = &rgb->encoder.encoder;
 	encoder->possible_crtcs = drm_crtc_mask(crtc);
 
 	ret = drm_simple_encoder_init(drm_dev, encoder, DRM_MODE_ENCODER_NONE);
@@ -161,6 +160,8 @@ struct rockchip_rgb *rockchip_rgb_init(struct device *dev,
 		goto err_free_encoder;
 	}
 
+	rgb->encoder.crtc_endpoint_id = endpoint_id;
+
 	ret = drm_connector_attach_encoder(connector, encoder);
 	if (ret < 0) {
 		DRM_DEV_ERROR(drm_dev->dev,
@@ -182,6 +183,6 @@ void rockchip_rgb_fini(struct rockchip_rgb *rgb)
 {
 	drm_panel_bridge_remove(rgb->bridge);
 	drm_connector_cleanup(&rgb->connector);
-	drm_encoder_cleanup(&rgb->encoder);
+	drm_encoder_cleanup(&rgb->encoder.encoder);
 }
 EXPORT_SYMBOL_GPL(rockchip_rgb_fini);
diff --git a/drivers/gpu/drm/rockchip/rockchip_rgb.h b/drivers/gpu/drm/rockchip/rockchip_rgb.h
index 27b9635124bc..1bd4e20e91eb 100644
--- a/drivers/gpu/drm/rockchip/rockchip_rgb.h
+++ b/drivers/gpu/drm/rockchip/rockchip_rgb.h
@@ -8,12 +8,14 @@
 #ifdef CONFIG_ROCKCHIP_RGB
 struct rockchip_rgb *rockchip_rgb_init(struct device *dev,
 				       struct drm_crtc *crtc,
-				       struct drm_device *drm_dev);
+				       struct drm_device *drm_dev,
+				       int video_port);
 void rockchip_rgb_fini(struct rockchip_rgb *rgb);
 #else
 static inline struct rockchip_rgb *rockchip_rgb_init(struct device *dev,
 						     struct drm_crtc *crtc,
-						     struct drm_device *drm_dev)
+						     struct drm_device *drm_dev,
+						     int video_port)
 {
 	return NULL;
 }
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
index 014f99e8928e..20ac7811c5eb 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
@@ -181,6 +181,7 @@ static const struct vop_data rk3036_vop = {
 	.output = &rk3036_output,
 	.win = rk3036_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3036_vop_win_data),
+	.max_output = { 1920, 1080 },
 };
 
 static const struct vop_win_phy rk3126_win1_data = {
@@ -213,6 +214,7 @@ static const struct vop_data rk3126_vop = {
 	.output = &rk3036_output,
 	.win = rk3126_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3126_vop_win_data),
+	.max_output = { 1920, 1080 },
 };
 
 static const int px30_vop_intrs[] = {
@@ -340,6 +342,7 @@ static const struct vop_data px30_vop_big = {
 	.output = &px30_output,
 	.win = px30_vop_big_win_data,
 	.win_size = ARRAY_SIZE(px30_vop_big_win_data),
+	.max_output = { 1920, 1080 },
 };
 
 static const struct vop_win_data px30_vop_lit_win_data[] = {
@@ -356,6 +359,7 @@ static const struct vop_data px30_vop_lit = {
 	.output = &px30_output,
 	.win = px30_vop_lit_win_data,
 	.win_size = ARRAY_SIZE(px30_vop_lit_win_data),
+	.max_output = { 1920, 1080 },
 };
 
 static const struct vop_scl_regs rk3066_win_scl = {
@@ -479,6 +483,7 @@ static const struct vop_data rk3066_vop = {
 	.output = &rk3066_output,
 	.win = rk3066_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3066_vop_win_data),
+	.max_output = { 1920, 1080 },
 };
 
 static const struct vop_scl_regs rk3188_win_scl = {
@@ -585,6 +590,7 @@ static const struct vop_data rk3188_vop = {
 	.win = rk3188_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3188_vop_win_data),
 	.feature = VOP_FEATURE_INTERNAL_RGB,
+	.max_output = { 2048, 1536 },
 };
 
 static const struct vop_scl_extension rk3288_win_full_scl_ext = {
@@ -732,6 +738,12 @@ static const struct vop_data rk3288_vop = {
 	.win = rk3288_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3288_vop_win_data),
 	.lut_size = 1024,
+	/*
+	 * This is the maximum resolution for the VOPB, the VOPL can only do
+	 * 2560x1600, but we can't distinguish them as they have the same
+	 * compatible.
+	 */
+	.max_output = { 3840, 2160 },
 };
 
 static const int rk3368_vop_intrs[] = {
@@ -833,6 +845,7 @@ static const struct vop_data rk3368_vop = {
 	.misc = &rk3368_misc,
 	.win = rk3368_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3368_vop_win_data),
+	.max_output = { 4096, 2160 },
 };
 
 static const struct vop_intr rk3366_vop_intr = {
@@ -854,6 +867,7 @@ static const struct vop_data rk3366_vop = {
 	.misc = &rk3368_misc,
 	.win = rk3368_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3368_vop_win_data),
+	.max_output = { 4096, 2160 },
 };
 
 static const struct vop_output rk3399_output = {
@@ -984,6 +998,7 @@ static const struct vop_data rk3399_vop_big = {
 	.win_size = ARRAY_SIZE(rk3399_vop_win_data),
 	.win_yuv2yuv = rk3399_vop_big_win_yuv2yuv_data,
 	.lut_size = 1024,
+	.max_output = { 4096, 2160 },
 };
 
 static const struct vop_win_data rk3399_vop_lit_win_data[] = {
@@ -1010,6 +1025,7 @@ static const struct vop_data rk3399_vop_lit = {
 	.win_size = ARRAY_SIZE(rk3399_vop_lit_win_data),
 	.win_yuv2yuv = rk3399_vop_lit_win_yuv2yuv_data,
 	.lut_size = 256,
+	.max_output = { 2560, 1600 },
 };
 
 static const struct vop_win_data rk3228_vop_win_data[] = {
@@ -1029,6 +1045,7 @@ static const struct vop_data rk3228_vop = {
 	.misc = &rk3368_misc,
 	.win = rk3228_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3228_vop_win_data),
+	.max_output = { 4096, 2160 },
 };
 
 static const struct vop_modeset rk3328_modeset = {
@@ -1100,6 +1117,7 @@ static const struct vop_data rk3328_vop = {
 	.misc = &rk3328_misc,
 	.win = rk3328_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3328_vop_win_data),
+	.max_output = { 4096, 2160 },
 };
 
 static const struct of_device_id vop_driver_dt_match[] = {
diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c
index 7fd869520ef2..fe9c6468e440 100644
--- a/drivers/gpu/drm/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/scheduler/sched_fence.c
@@ -123,6 +123,37 @@ static void drm_sched_fence_release_finished(struct dma_fence *f)
 	dma_fence_put(&fence->scheduled);
 }
 
+static void drm_sched_fence_set_deadline_finished(struct dma_fence *f,
+						  ktime_t deadline)
+{
+	struct drm_sched_fence *fence = to_drm_sched_fence(f);
+	struct dma_fence *parent;
+	unsigned long flags;
+
+	spin_lock_irqsave(&fence->lock, flags);
+
+	/* If we already have an earlier deadline, keep it: */
+	if (test_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, &f->flags) &&
+	    ktime_before(fence->deadline, deadline)) {
+		spin_unlock_irqrestore(&fence->lock, flags);
+		return;
+	}
+
+	fence->deadline = deadline;
+	set_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, &f->flags);
+
+	spin_unlock_irqrestore(&fence->lock, flags);
+
+	/*
+	 * smp_load_aquire() to ensure that if we are racing another
+	 * thread calling drm_sched_fence_set_parent(), that we see
+	 * the parent set before it calls test_bit(HAS_DEADLINE_BIT)
+	 */
+	parent = smp_load_acquire(&fence->parent);
+	if (parent)
+		dma_fence_set_deadline(parent, deadline);
+}
+
 static const struct dma_fence_ops drm_sched_fence_ops_scheduled = {
 	.get_driver_name = drm_sched_fence_get_driver_name,
 	.get_timeline_name = drm_sched_fence_get_timeline_name,
@@ -133,6 +164,7 @@ static const struct dma_fence_ops drm_sched_fence_ops_finished = {
 	.get_driver_name = drm_sched_fence_get_driver_name,
 	.get_timeline_name = drm_sched_fence_get_timeline_name,
 	.release = drm_sched_fence_release_finished,
+	.set_deadline = drm_sched_fence_set_deadline_finished,
 };
 
 struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f)
@@ -147,6 +179,20 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f)
 }
 EXPORT_SYMBOL(to_drm_sched_fence);
 
+void drm_sched_fence_set_parent(struct drm_sched_fence *s_fence,
+				struct dma_fence *fence)
+{
+	/*
+	 * smp_store_release() to ensure another thread racing us
+	 * in drm_sched_fence_set_deadline_finished() sees the
+	 * fence's parent set before test_bit()
+	 */
+	smp_store_release(&s_fence->parent, dma_fence_get(fence));
+	if (test_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT,
+		     &s_fence->finished.flags))
+		dma_fence_set_deadline(fence, s_fence->deadline);
+}
+
 struct drm_sched_fence *drm_sched_fence_alloc(struct drm_sched_entity *entity,
 					      void *owner)
 {
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 0e4378420271..fcd5bd7e5e8e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -53,6 +53,7 @@
 
 #include <drm/drm_print.h>
 #include <drm/drm_gem.h>
+#include <drm/drm_syncobj.h>
 #include <drm/gpu_scheduler.h>
 #include <drm/spsc_queue.h>
 
@@ -308,7 +309,8 @@ static void drm_sched_start_timeout(struct drm_gpu_scheduler *sched)
  */
 void drm_sched_fault(struct drm_gpu_scheduler *sched)
 {
-	mod_delayed_work(sched->timeout_wq, &sched->work_tdr, 0);
+	if (sched->ready)
+		mod_delayed_work(sched->timeout_wq, &sched->work_tdr, 0);
 }
 EXPORT_SYMBOL(drm_sched_fault);
 
@@ -719,6 +721,34 @@ int drm_sched_job_add_dependency(struct drm_sched_job *job,
 EXPORT_SYMBOL(drm_sched_job_add_dependency);
 
 /**
+ * drm_sched_job_add_syncobj_dependency - adds a syncobj's fence as a job dependency
+ * @job: scheduler job to add the dependencies to
+ * @file: drm file private pointer
+ * @handle: syncobj handle to lookup
+ * @point: timeline point
+ *
+ * This adds the fence matching the given syncobj to @job.
+ *
+ * Returns:
+ * 0 on success, or an error on failing to expand the array.
+ */
+int drm_sched_job_add_syncobj_dependency(struct drm_sched_job *job,
+					 struct drm_file *file,
+					 u32 handle,
+					 u32 point)
+{
+	struct dma_fence *fence;
+	int ret;
+
+	ret = drm_syncobj_find_fence(file, handle, point, 0, &fence);
+	if (ret)
+		return ret;
+
+	return drm_sched_job_add_dependency(job, fence);
+}
+EXPORT_SYMBOL(drm_sched_job_add_syncobj_dependency);
+
+/**
  * drm_sched_job_add_resv_dependencies - add all fences from the resv to the job
  * @job: scheduler job to add the dependencies to
  * @resv: the dma_resv object to get the fences from
@@ -1013,7 +1043,7 @@ static int drm_sched_main(void *param)
 		drm_sched_fence_scheduled(s_fence);
 
 		if (!IS_ERR_OR_NULL(fence)) {
-			s_fence->parent = dma_fence_get(fence);
+			drm_sched_fence_set_parent(s_fence, fence);
 			/* Drop for original kref_init of the fence */
 			dma_fence_put(fence);
 
diff --git a/drivers/gpu/drm/sti/Kconfig b/drivers/gpu/drm/sti/Kconfig
index f2a880c48485..3c7a5feff8de 100644
--- a/drivers/gpu/drm/sti/Kconfig
+++ b/drivers/gpu/drm/sti/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config DRM_STI
 	tristate "DRM Support for STMicroelectronics SoC stiH4xx Series"
-	depends on OF && DRM && (ARCH_STI || ARCH_MULTIPLATFORM)
+	depends on OF && DRM && ARCH_STI
 	select RESET_CONTROLLER
 	select DRM_KMS_HELPER
 	select DRM_GEM_DMA_HELPER
diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c
index ef6a4e63198f..1b87b5899f9e 100644
--- a/drivers/gpu/drm/sti/sti_drv.c
+++ b/drivers/gpu/drm/sti/sti_drv.c
@@ -14,7 +14,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_debugfs.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_of.h>
@@ -199,7 +199,7 @@ static int sti_bind(struct device *dev)
 
 	drm_mode_config_reset(ddev);
 
-	drm_fbdev_generic_setup(ddev, 32);
+	drm_fbdev_dma_setup(ddev, 32);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/stm/Kconfig b/drivers/gpu/drm/stm/Kconfig
index ded72f879482..fa49cde43bb2 100644
--- a/drivers/gpu/drm/stm/Kconfig
+++ b/drivers/gpu/drm/stm/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config DRM_STM
 	tristate "DRM Support for STMicroelectronics SoC Series"
-	depends on DRM && (ARCH_STM32 || ARCH_MULTIPLATFORM)
+	depends on DRM && ARCH_STM32
 	select DRM_KMS_HELPER
 	select DRM_GEM_DMA_HELPER
 	select DRM_PANEL_BRIDGE
diff --git a/drivers/gpu/drm/stm/drv.c b/drivers/gpu/drm/stm/drv.c
index 50410bd99dfe..422220df7d8c 100644
--- a/drivers/gpu/drm/stm/drv.c
+++ b/drivers/gpu/drm/stm/drv.c
@@ -18,7 +18,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
@@ -203,7 +203,7 @@ static int stm_drm_platform_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_put;
 
-	drm_fbdev_generic_setup(ddev, 16);
+	drm_fbdev_dma_setup(ddev, 16);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index 38070fc261f3..b11dbd50d73e 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -792,7 +792,7 @@ static int sun4i_backend_bind(struct device *dev, struct device *master,
 	dev_set_drvdata(dev, backend);
 	spin_lock_init(&backend->frontend_lock);
 
-	if (of_find_property(dev->of_node, "interconnects", NULL)) {
+	if (of_property_present(dev->of_node, "interconnects")) {
 		/*
 		 * This assume we have the same DMA constraints for all our the
 		 * devices in our pipeline (all the backends, but also the
diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c
index d6c741716167..e49f78a6a8cf 100644
--- a/drivers/gpu/drm/sun4i/sun4i_drv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_drv.c
@@ -17,7 +17,7 @@
 #include <drm/drm_aperture.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_of.h>
@@ -111,7 +111,7 @@ static int sun4i_drv_bind(struct device *dev)
 	if (ret)
 		goto finish_poll;
 
-	drm_fbdev_generic_setup(drm, 32);
+	drm_fbdev_dma_setup(drm, 32);
 
 	dev_set_drvdata(dev, drm);
 
diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.c b/drivers/gpu/drm/sun4i/sun8i_mixer.c
index bafee05f6b24..11d5244a5aa5 100644
--- a/drivers/gpu/drm/sun4i/sun8i_mixer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_mixer.c
@@ -391,7 +391,7 @@ static int sun8i_mixer_bind(struct device *dev, struct device *master,
 	mixer->engine.ops = &sun8i_engine_ops;
 	mixer->engine.node = dev->of_node;
 
-	if (of_find_property(dev->of_node, "iommus", NULL)) {
+	if (of_property_present(dev->of_node, "iommus")) {
 		/*
 		 * This assume we have the same DMA constraints for
 		 * all our the mixers in our pipeline. This sounds
diff --git a/drivers/gpu/drm/tegra/Kconfig b/drivers/gpu/drm/tegra/Kconfig
index c36323f1c7e6..56453ca277c2 100644
--- a/drivers/gpu/drm/tegra/Kconfig
+++ b/drivers/gpu/drm/tegra/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config DRM_TEGRA
 	tristate "NVIDIA Tegra DRM"
-	depends on ARCH_TEGRA || (ARM && COMPILE_TEST)
+	depends on ARCH_TEGRA || COMPILE_TEST
 	depends on COMMON_CLK
 	depends on DRM
 	depends on OF
diff --git a/drivers/gpu/drm/tegra/Makefile b/drivers/gpu/drm/tegra/Makefile
index bb0d2c144b55..6fc4b504e786 100644
--- a/drivers/gpu/drm/tegra/Makefile
+++ b/drivers/gpu/drm/tegra/Makefile
@@ -29,4 +29,6 @@ tegra-drm-y := \
 
 tegra-drm-y += trace.o
 
+tegra-drm-$(CONFIG_DRM_FBDEV_EMULATION) += fbdev.o
+
 obj-$(CONFIG_DRM_TEGRA) += tegra-drm.o
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index a67453cee883..6e78416e64b0 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -2381,7 +2381,6 @@ static int tegra_crtc_calculate_memory_bandwidth(struct drm_crtc *crtc,
 	const struct tegra_plane_state *tegra_state;
 	const struct drm_plane_state *plane_state;
 	struct tegra_dc *dc = to_tegra_dc(crtc);
-	const struct drm_crtc_state *old_state;
 	struct drm_crtc_state *new_state;
 	struct tegra_plane *tegra;
 	struct drm_plane *plane;
@@ -2396,7 +2395,6 @@ static int tegra_crtc_calculate_memory_bandwidth(struct drm_crtc *crtc,
 		return 0;
 
 	new_state = drm_atomic_get_new_crtc_state(state, crtc);
-	old_state = drm_atomic_get_old_crtc_state(state, crtc);
 
 	/*
 	 * For overlapping planes pixel's data is fetched for each plane at
@@ -3263,27 +3261,15 @@ disable_pm:
 	return err;
 }
 
-static int tegra_dc_remove(struct platform_device *pdev)
+static void tegra_dc_remove(struct platform_device *pdev)
 {
 	struct tegra_dc *dc = platform_get_drvdata(pdev);
-	int err;
 
-	err = host1x_client_unregister(&dc->client);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
-			err);
-		return err;
-	}
+	host1x_client_unregister(&dc->client);
 
-	err = tegra_dc_rgb_remove(dc);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to remove RGB output: %d\n", err);
-		return err;
-	}
+	tegra_dc_rgb_remove(dc);
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 struct platform_driver tegra_dc_driver = {
@@ -3292,5 +3278,5 @@ struct platform_driver tegra_dc_driver = {
 		.of_match_table = tegra_dc_of_match,
 	},
 	.probe = tegra_dc_probe,
-	.remove = tegra_dc_remove,
+	.remove_new = tegra_dc_remove,
 };
diff --git a/drivers/gpu/drm/tegra/dc.h b/drivers/gpu/drm/tegra/dc.h
index f902794d42cc..0559fa6b1bf7 100644
--- a/drivers/gpu/drm/tegra/dc.h
+++ b/drivers/gpu/drm/tegra/dc.h
@@ -169,7 +169,7 @@ void tegra_crtc_atomic_post_commit(struct drm_crtc *crtc,
 
 /* from rgb.c */
 int tegra_dc_rgb_probe(struct tegra_dc *dc);
-int tegra_dc_rgb_remove(struct tegra_dc *dc);
+void tegra_dc_rgb_remove(struct tegra_dc *dc);
 int tegra_dc_rgb_init(struct drm_device *drm, struct tegra_dc *dc);
 int tegra_dc_rgb_exit(struct tegra_dc *dc);
 
diff --git a/drivers/gpu/drm/tegra/dpaux.c b/drivers/gpu/drm/tegra/dpaux.c
index 3c84e73d5051..4d2677dcd831 100644
--- a/drivers/gpu/drm/tegra/dpaux.c
+++ b/drivers/gpu/drm/tegra/dpaux.c
@@ -579,7 +579,7 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int tegra_dpaux_remove(struct platform_device *pdev)
+static void tegra_dpaux_remove(struct platform_device *pdev)
 {
 	struct tegra_dpaux *dpaux = platform_get_drvdata(pdev);
 
@@ -594,8 +594,6 @@ static int tegra_dpaux_remove(struct platform_device *pdev)
 	mutex_lock(&dpaux_lock);
 	list_del(&dpaux->list);
 	mutex_unlock(&dpaux_lock);
-
-	return 0;
 }
 
 static int tegra_dpaux_suspend(struct device *dev)
@@ -695,7 +693,7 @@ struct platform_driver tegra_dpaux_driver = {
 		.pm = pm_ptr(&tegra_dpaux_pm_ops),
 	},
 	.probe = tegra_dpaux_probe,
-	.remove = tegra_dpaux_remove,
+	.remove_new = tegra_dpaux_remove,
 };
 
 struct drm_dp_aux *drm_dp_aux_find_by_of_node(struct device_node *np)
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index 6ca9f396e55b..85ba96cddd51 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -56,9 +56,6 @@ static int tegra_atomic_check(struct drm_device *drm,
 
 static const struct drm_mode_config_funcs tegra_drm_mode_config_funcs = {
 	.fb_create = tegra_fb_create,
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	.output_poll_changed = drm_fb_helper_output_poll_changed,
-#endif
 	.atomic_check = tegra_atomic_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -885,7 +882,6 @@ static const struct drm_driver tegra_drm_driver = {
 			   DRIVER_ATOMIC | DRIVER_RENDER | DRIVER_SYNCOBJ,
 	.open = tegra_drm_open,
 	.postclose = tegra_drm_postclose,
-	.lastclose = drm_fb_helper_lastclose,
 
 #if defined(CONFIG_DEBUG_FS)
 	.debugfs_init = tegra_debugfs_init,
@@ -1185,15 +1181,11 @@ static int host1x_drm_probe(struct host1x_device *dev)
 	drm->mode_config.funcs = &tegra_drm_mode_config_funcs;
 	drm->mode_config.helper_private = &tegra_drm_mode_config_helpers;
 
-	err = tegra_drm_fb_prepare(drm);
-	if (err < 0)
-		goto config;
-
 	drm_kms_helper_poll_init(drm);
 
 	err = host1x_device_init(dev);
 	if (err < 0)
-		goto fbdev;
+		goto poll;
 
 	/*
 	 * Now that all display controller have been initialized, the maximum
@@ -1256,18 +1248,14 @@ static int host1x_drm_probe(struct host1x_device *dev)
 	if (err < 0)
 		goto hub;
 
-	err = tegra_drm_fb_init(drm);
+	err = drm_dev_register(drm, 0);
 	if (err < 0)
 		goto hub;
 
-	err = drm_dev_register(drm, 0);
-	if (err < 0)
-		goto fb;
+	tegra_fbdev_setup(drm);
 
 	return 0;
 
-fb:
-	tegra_drm_fb_exit(drm);
 hub:
 	if (tegra->hub)
 		tegra_display_hub_cleanup(tegra->hub);
@@ -1280,10 +1268,8 @@ device:
 	}
 
 	host1x_device_exit(dev);
-fbdev:
+poll:
 	drm_kms_helper_poll_fini(drm);
-	tegra_drm_fb_free(drm);
-config:
 	drm_mode_config_cleanup(drm);
 domain:
 	if (tegra->domain)
@@ -1304,7 +1290,6 @@ static int host1x_drm_remove(struct host1x_device *dev)
 	drm_dev_unregister(drm);
 
 	drm_kms_helper_poll_fini(drm);
-	tegra_drm_fb_exit(drm);
 	drm_atomic_helper_shutdown(drm);
 	drm_mode_config_cleanup(drm);
 
diff --git a/drivers/gpu/drm/tegra/drm.h b/drivers/gpu/drm/tegra/drm.h
index 845e60f144c7..f9d18e8cf6ab 100644
--- a/drivers/gpu/drm/tegra/drm.h
+++ b/drivers/gpu/drm/tegra/drm.h
@@ -15,7 +15,6 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_encoder.h>
-#include <drm/drm_fb_helper.h>
 #include <drm/drm_fixed.h>
 #include <drm/drm_probe_helper.h>
 #include <uapi/drm/tegra_drm.h>
@@ -29,13 +28,6 @@
 
 struct reset_control;
 
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-struct tegra_fbdev {
-	struct drm_fb_helper base;
-	struct drm_framebuffer *fb;
-};
-#endif
-
 struct tegra_drm {
 	struct drm_device *drm;
 
@@ -53,10 +45,6 @@ struct tegra_drm {
 	struct mutex clients_lock;
 	struct list_head clients;
 
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	struct tegra_fbdev *fbdev;
-#endif
-
 	unsigned int hmask, vmask;
 	unsigned int pitch_align;
 	unsigned int num_crtcs;
@@ -196,13 +184,20 @@ struct tegra_bo *tegra_fb_get_plane(struct drm_framebuffer *framebuffer,
 bool tegra_fb_is_bottom_up(struct drm_framebuffer *framebuffer);
 int tegra_fb_get_tiling(struct drm_framebuffer *framebuffer,
 			struct tegra_bo_tiling *tiling);
+struct drm_framebuffer *tegra_fb_alloc(struct drm_device *drm,
+				       const struct drm_mode_fb_cmd2 *mode_cmd,
+				       struct tegra_bo **planes,
+				       unsigned int num_planes);
 struct drm_framebuffer *tegra_fb_create(struct drm_device *drm,
 					struct drm_file *file,
 					const struct drm_mode_fb_cmd2 *cmd);
-int tegra_drm_fb_prepare(struct drm_device *drm);
-void tegra_drm_fb_free(struct drm_device *drm);
-int tegra_drm_fb_init(struct drm_device *drm);
-void tegra_drm_fb_exit(struct drm_device *drm);
+
+#ifdef CONFIG_DRM_FBDEV_EMULATION
+void tegra_fbdev_setup(struct drm_device *drm);
+#else
+static inline void tegra_fbdev_setup(struct drm_device *drm)
+{ }
+#endif
 
 extern struct platform_driver tegra_display_hub_driver;
 extern struct platform_driver tegra_dc_driver;
diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
index de1333dc0d86..a9870c828374 100644
--- a/drivers/gpu/drm/tegra/dsi.c
+++ b/drivers/gpu/drm/tegra/dsi.c
@@ -912,6 +912,15 @@ static void tegra_dsi_encoder_enable(struct drm_encoder *encoder)
 	u32 value;
 	int err;
 
+	/* If the bootloader enabled DSI it needs to be disabled
+	 * in order for the panel initialization commands to be
+	 * properly sent.
+	 */
+	value = tegra_dsi_readl(dsi, DSI_POWER_CONTROL);
+
+	if (value & DSI_POWER_CONTROL_ENABLE)
+		tegra_dsi_disable(dsi);
+
 	err = tegra_dsi_prepare(dsi);
 	if (err < 0) {
 		dev_err(dsi->dev, "failed to prepare: %d\n", err);
@@ -1589,28 +1598,24 @@ static int tegra_dsi_probe(struct platform_device *pdev)
 	}
 
 	dsi->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(dsi->clk)) {
-		dev_err(&pdev->dev, "cannot get DSI clock\n");
-		return PTR_ERR(dsi->clk);
-	}
+	if (IS_ERR(dsi->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(dsi->clk),
+				     "cannot get DSI clock\n");
 
 	dsi->clk_lp = devm_clk_get(&pdev->dev, "lp");
-	if (IS_ERR(dsi->clk_lp)) {
-		dev_err(&pdev->dev, "cannot get low-power clock\n");
-		return PTR_ERR(dsi->clk_lp);
-	}
+	if (IS_ERR(dsi->clk_lp))
+		return dev_err_probe(&pdev->dev, PTR_ERR(dsi->clk_lp),
+				     "cannot get low-power clock\n");
 
 	dsi->clk_parent = devm_clk_get(&pdev->dev, "parent");
-	if (IS_ERR(dsi->clk_parent)) {
-		dev_err(&pdev->dev, "cannot get parent clock\n");
-		return PTR_ERR(dsi->clk_parent);
-	}
+	if (IS_ERR(dsi->clk_parent))
+		return dev_err_probe(&pdev->dev, PTR_ERR(dsi->clk_parent),
+				     "cannot get parent clock\n");
 
 	dsi->vdd = devm_regulator_get(&pdev->dev, "avdd-dsi-csi");
-	if (IS_ERR(dsi->vdd)) {
-		dev_err(&pdev->dev, "cannot get VDD supply\n");
-		return PTR_ERR(dsi->vdd);
-	}
+	if (IS_ERR(dsi->vdd))
+		return dev_err_probe(&pdev->dev, PTR_ERR(dsi->vdd),
+				     "cannot get VDD supply\n");
 
 	err = tegra_dsi_setup_clocks(dsi);
 	if (err < 0) {
@@ -1659,26 +1664,18 @@ mipi_free:
 	return err;
 }
 
-static int tegra_dsi_remove(struct platform_device *pdev)
+static void tegra_dsi_remove(struct platform_device *pdev)
 {
 	struct tegra_dsi *dsi = platform_get_drvdata(pdev);
-	int err;
 
 	pm_runtime_disable(&pdev->dev);
 
-	err = host1x_client_unregister(&dsi->client);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
-			err);
-		return err;
-	}
+	host1x_client_unregister(&dsi->client);
 
 	tegra_output_remove(&dsi->output);
 
 	mipi_dsi_host_unregister(&dsi->host);
 	tegra_mipi_free(dsi->mipi);
-
-	return 0;
 }
 
 static const struct of_device_id tegra_dsi_of_match[] = {
@@ -1696,5 +1693,5 @@ struct platform_driver tegra_dsi_driver = {
 		.of_match_table = tegra_dsi_of_match,
 	},
 	.probe = tegra_dsi_probe,
-	.remove = tegra_dsi_remove,
+	.remove_new = tegra_dsi_remove,
 };
diff --git a/drivers/gpu/drm/tegra/fb.c b/drivers/gpu/drm/tegra/fb.c
index bfebe2786d61..a719af1dc9a5 100644
--- a/drivers/gpu/drm/tegra/fb.c
+++ b/drivers/gpu/drm/tegra/fb.c
@@ -17,13 +17,6 @@
 #include "drm.h"
 #include "gem.h"
 
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-static inline struct tegra_fbdev *to_tegra_fbdev(struct drm_fb_helper *helper)
-{
-	return container_of(helper, struct tegra_fbdev, base);
-}
-#endif
-
 struct tegra_bo *tegra_fb_get_plane(struct drm_framebuffer *framebuffer,
 				    unsigned int index)
 {
@@ -108,10 +101,10 @@ static const struct drm_framebuffer_funcs tegra_fb_funcs = {
 	.create_handle = drm_gem_fb_create_handle,
 };
 
-static struct drm_framebuffer *tegra_fb_alloc(struct drm_device *drm,
-					      const struct drm_mode_fb_cmd2 *mode_cmd,
-					      struct tegra_bo **planes,
-					      unsigned int num_planes)
+struct drm_framebuffer *tegra_fb_alloc(struct drm_device *drm,
+				       const struct drm_mode_fb_cmd2 *mode_cmd,
+				       struct tegra_bo **planes,
+				       unsigned int num_planes)
 {
 	struct drm_framebuffer *fb;
 	unsigned int i;
@@ -186,230 +179,3 @@ unreference:
 
 	return ERR_PTR(err);
 }
-
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-static int tegra_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
-{
-	struct drm_fb_helper *helper = info->par;
-	struct tegra_bo *bo;
-	int err;
-
-	bo = tegra_fb_get_plane(helper->fb, 0);
-
-	err = drm_gem_mmap_obj(&bo->gem, bo->gem.size, vma);
-	if (err < 0)
-		return err;
-
-	return __tegra_gem_mmap(&bo->gem, vma);
-}
-
-static const struct fb_ops tegra_fb_ops = {
-	.owner = THIS_MODULE,
-	DRM_FB_HELPER_DEFAULT_OPS,
-	.fb_read = drm_fb_helper_sys_read,
-	.fb_write = drm_fb_helper_sys_write,
-	.fb_fillrect = drm_fb_helper_sys_fillrect,
-	.fb_copyarea = drm_fb_helper_sys_copyarea,
-	.fb_imageblit = drm_fb_helper_sys_imageblit,
-	.fb_mmap = tegra_fb_mmap,
-};
-
-static int tegra_fbdev_probe(struct drm_fb_helper *helper,
-			     struct drm_fb_helper_surface_size *sizes)
-{
-	struct tegra_fbdev *fbdev = to_tegra_fbdev(helper);
-	struct tegra_drm *tegra = helper->dev->dev_private;
-	struct drm_device *drm = helper->dev;
-	struct drm_mode_fb_cmd2 cmd = { 0 };
-	unsigned int bytes_per_pixel;
-	struct drm_framebuffer *fb;
-	unsigned long offset;
-	struct fb_info *info;
-	struct tegra_bo *bo;
-	size_t size;
-	int err;
-
-	bytes_per_pixel = DIV_ROUND_UP(sizes->surface_bpp, 8);
-
-	cmd.width = sizes->surface_width;
-	cmd.height = sizes->surface_height;
-	cmd.pitches[0] = round_up(sizes->surface_width * bytes_per_pixel,
-				  tegra->pitch_align);
-
-	cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
-						     sizes->surface_depth);
-
-	size = cmd.pitches[0] * cmd.height;
-
-	bo = tegra_bo_create(drm, size, 0);
-	if (IS_ERR(bo))
-		return PTR_ERR(bo);
-
-	info = drm_fb_helper_alloc_info(helper);
-	if (IS_ERR(info)) {
-		dev_err(drm->dev, "failed to allocate framebuffer info\n");
-		drm_gem_object_put(&bo->gem);
-		return PTR_ERR(info);
-	}
-
-	fbdev->fb = tegra_fb_alloc(drm, &cmd, &bo, 1);
-	if (IS_ERR(fbdev->fb)) {
-		err = PTR_ERR(fbdev->fb);
-		dev_err(drm->dev, "failed to allocate DRM framebuffer: %d\n",
-			err);
-		drm_gem_object_put(&bo->gem);
-		return PTR_ERR(fbdev->fb);
-	}
-
-	fb = fbdev->fb;
-	helper->fb = fb;
-	helper->info = info;
-
-	info->fbops = &tegra_fb_ops;
-
-	drm_fb_helper_fill_info(info, helper, sizes);
-
-	offset = info->var.xoffset * bytes_per_pixel +
-		 info->var.yoffset * fb->pitches[0];
-
-	if (bo->pages) {
-		bo->vaddr = vmap(bo->pages, bo->num_pages, VM_MAP,
-				 pgprot_writecombine(PAGE_KERNEL));
-		if (!bo->vaddr) {
-			dev_err(drm->dev, "failed to vmap() framebuffer\n");
-			err = -ENOMEM;
-			goto destroy;
-		}
-	}
-
-	info->screen_base = (void __iomem *)bo->vaddr + offset;
-	info->screen_size = size;
-	info->fix.smem_start = (unsigned long)(bo->iova + offset);
-	info->fix.smem_len = size;
-
-	return 0;
-
-destroy:
-	drm_framebuffer_remove(fb);
-	return err;
-}
-
-static const struct drm_fb_helper_funcs tegra_fb_helper_funcs = {
-	.fb_probe = tegra_fbdev_probe,
-};
-
-static struct tegra_fbdev *tegra_fbdev_create(struct drm_device *drm)
-{
-	struct tegra_fbdev *fbdev;
-
-	fbdev = kzalloc(sizeof(*fbdev), GFP_KERNEL);
-	if (!fbdev) {
-		dev_err(drm->dev, "failed to allocate DRM fbdev\n");
-		return ERR_PTR(-ENOMEM);
-	}
-
-	drm_fb_helper_prepare(drm, &fbdev->base, 32, &tegra_fb_helper_funcs);
-
-	return fbdev;
-}
-
-static void tegra_fbdev_free(struct tegra_fbdev *fbdev)
-{
-	drm_fb_helper_unprepare(&fbdev->base);
-	kfree(fbdev);
-}
-
-static int tegra_fbdev_init(struct tegra_fbdev *fbdev,
-			    unsigned int num_crtc,
-			    unsigned int max_connectors)
-{
-	struct drm_device *drm = fbdev->base.dev;
-	int err;
-
-	err = drm_fb_helper_init(drm, &fbdev->base);
-	if (err < 0) {
-		dev_err(drm->dev, "failed to initialize DRM FB helper: %d\n",
-			err);
-		return err;
-	}
-
-	err = drm_fb_helper_initial_config(&fbdev->base);
-	if (err < 0) {
-		dev_err(drm->dev, "failed to set initial configuration: %d\n",
-			err);
-		goto fini;
-	}
-
-	return 0;
-
-fini:
-	drm_fb_helper_fini(&fbdev->base);
-	return err;
-}
-
-static void tegra_fbdev_exit(struct tegra_fbdev *fbdev)
-{
-	drm_fb_helper_unregister_info(&fbdev->base);
-
-	if (fbdev->fb) {
-		struct tegra_bo *bo = tegra_fb_get_plane(fbdev->fb, 0);
-
-		/* Undo the special mapping we made in fbdev probe. */
-		if (bo && bo->pages) {
-			vunmap(bo->vaddr);
-			bo->vaddr = NULL;
-		}
-
-		drm_framebuffer_remove(fbdev->fb);
-	}
-
-	drm_fb_helper_fini(&fbdev->base);
-	tegra_fbdev_free(fbdev);
-}
-#endif
-
-int tegra_drm_fb_prepare(struct drm_device *drm)
-{
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	struct tegra_drm *tegra = drm->dev_private;
-
-	tegra->fbdev = tegra_fbdev_create(drm);
-	if (IS_ERR(tegra->fbdev))
-		return PTR_ERR(tegra->fbdev);
-#endif
-
-	return 0;
-}
-
-void tegra_drm_fb_free(struct drm_device *drm)
-{
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	struct tegra_drm *tegra = drm->dev_private;
-
-	tegra_fbdev_free(tegra->fbdev);
-#endif
-}
-
-int tegra_drm_fb_init(struct drm_device *drm)
-{
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	struct tegra_drm *tegra = drm->dev_private;
-	int err;
-
-	err = tegra_fbdev_init(tegra->fbdev, drm->mode_config.num_crtc,
-			       drm->mode_config.num_connector);
-	if (err < 0)
-		return err;
-#endif
-
-	return 0;
-}
-
-void tegra_drm_fb_exit(struct drm_device *drm)
-{
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	struct tegra_drm *tegra = drm->dev_private;
-
-	tegra_fbdev_exit(tegra->fbdev);
-#endif
-}
diff --git a/drivers/gpu/drm/tegra/fbdev.c b/drivers/gpu/drm/tegra/fbdev.c
new file mode 100644
index 000000000000..dca9eccae466
--- /dev/null
+++ b/drivers/gpu/drm/tegra/fbdev.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012-2013 Avionic Design GmbH
+ * Copyright (C) 2012 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Based on the KMS/FB DMA helpers
+ *   Copyright (C) 2012 Analog Devices Inc.
+ */
+
+#include <linux/console.h>
+#include <linux/vmalloc.h>
+
+#include <drm/drm_drv.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_modeset_helper.h>
+
+#include "drm.h"
+#include "gem.h"
+
+static int tegra_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
+{
+	struct drm_fb_helper *helper = info->par;
+	struct tegra_bo *bo;
+	int err;
+
+	bo = tegra_fb_get_plane(helper->fb, 0);
+
+	err = drm_gem_mmap_obj(&bo->gem, bo->gem.size, vma);
+	if (err < 0)
+		return err;
+
+	return __tegra_gem_mmap(&bo->gem, vma);
+}
+
+static void tegra_fbdev_fb_destroy(struct fb_info *info)
+{
+	struct drm_fb_helper *helper = info->par;
+	struct drm_framebuffer *fb = helper->fb;
+	struct tegra_bo *bo = tegra_fb_get_plane(fb, 0);
+
+	drm_fb_helper_fini(helper);
+
+	/* Undo the special mapping we made in fbdev probe. */
+	if (bo->pages) {
+		vunmap(bo->vaddr);
+		bo->vaddr = NULL;
+	}
+	drm_framebuffer_remove(fb);
+
+	drm_client_release(&helper->client);
+	drm_fb_helper_unprepare(helper);
+	kfree(helper);
+}
+
+static const struct fb_ops tegra_fb_ops = {
+	.owner = THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
+	.fb_read = drm_fb_helper_sys_read,
+	.fb_write = drm_fb_helper_sys_write,
+	.fb_fillrect = drm_fb_helper_sys_fillrect,
+	.fb_copyarea = drm_fb_helper_sys_copyarea,
+	.fb_imageblit = drm_fb_helper_sys_imageblit,
+	.fb_mmap = tegra_fb_mmap,
+	.fb_destroy = tegra_fbdev_fb_destroy,
+};
+
+static int tegra_fbdev_probe(struct drm_fb_helper *helper,
+			     struct drm_fb_helper_surface_size *sizes)
+{
+	struct tegra_drm *tegra = helper->dev->dev_private;
+	struct drm_device *drm = helper->dev;
+	struct drm_mode_fb_cmd2 cmd = { 0 };
+	unsigned int bytes_per_pixel;
+	struct drm_framebuffer *fb;
+	unsigned long offset;
+	struct fb_info *info;
+	struct tegra_bo *bo;
+	size_t size;
+	int err;
+
+	bytes_per_pixel = DIV_ROUND_UP(sizes->surface_bpp, 8);
+
+	cmd.width = sizes->surface_width;
+	cmd.height = sizes->surface_height;
+	cmd.pitches[0] = round_up(sizes->surface_width * bytes_per_pixel,
+				  tegra->pitch_align);
+
+	cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
+						     sizes->surface_depth);
+
+	size = cmd.pitches[0] * cmd.height;
+
+	bo = tegra_bo_create(drm, size, 0);
+	if (IS_ERR(bo))
+		return PTR_ERR(bo);
+
+	info = drm_fb_helper_alloc_info(helper);
+	if (IS_ERR(info)) {
+		dev_err(drm->dev, "failed to allocate framebuffer info\n");
+		drm_gem_object_put(&bo->gem);
+		return PTR_ERR(info);
+	}
+
+	fb = tegra_fb_alloc(drm, &cmd, &bo, 1);
+	if (IS_ERR(fb)) {
+		err = PTR_ERR(fb);
+		dev_err(drm->dev, "failed to allocate DRM framebuffer: %d\n",
+			err);
+		drm_gem_object_put(&bo->gem);
+		return PTR_ERR(fb);
+	}
+
+	helper->fb = fb;
+	helper->info = info;
+
+	info->fbops = &tegra_fb_ops;
+
+	drm_fb_helper_fill_info(info, helper, sizes);
+
+	offset = info->var.xoffset * bytes_per_pixel +
+		 info->var.yoffset * fb->pitches[0];
+
+	if (bo->pages) {
+		bo->vaddr = vmap(bo->pages, bo->num_pages, VM_MAP,
+				 pgprot_writecombine(PAGE_KERNEL));
+		if (!bo->vaddr) {
+			dev_err(drm->dev, "failed to vmap() framebuffer\n");
+			err = -ENOMEM;
+			goto destroy;
+		}
+	}
+
+	info->screen_base = (void __iomem *)bo->vaddr + offset;
+	info->screen_size = size;
+	info->fix.smem_start = (unsigned long)(bo->iova + offset);
+	info->fix.smem_len = size;
+
+	return 0;
+
+destroy:
+	drm_framebuffer_remove(fb);
+	return err;
+}
+
+static const struct drm_fb_helper_funcs tegra_fb_helper_funcs = {
+	.fb_probe = tegra_fbdev_probe,
+};
+
+/*
+ * struct drm_client
+ */
+
+static void tegra_fbdev_client_unregister(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+
+	if (fb_helper->info) {
+		drm_fb_helper_unregister_info(fb_helper);
+	} else {
+		drm_client_release(&fb_helper->client);
+		drm_fb_helper_unprepare(fb_helper);
+		kfree(fb_helper);
+	}
+}
+
+static int tegra_fbdev_client_restore(struct drm_client_dev *client)
+{
+	drm_fb_helper_lastclose(client->dev);
+
+	return 0;
+}
+
+static int tegra_fbdev_client_hotplug(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+	struct drm_device *dev = client->dev;
+	int ret;
+
+	if (dev->fb_helper)
+		return drm_fb_helper_hotplug_event(dev->fb_helper);
+
+	ret = drm_fb_helper_init(dev, fb_helper);
+	if (ret)
+		goto err_drm_err;
+
+	if (!drm_drv_uses_atomic_modeset(dev))
+		drm_helper_disable_unused_functions(dev);
+
+	ret = drm_fb_helper_initial_config(fb_helper);
+	if (ret)
+		goto err_drm_fb_helper_fini;
+
+	return 0;
+
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(fb_helper);
+err_drm_err:
+	drm_err(dev, "Failed to setup fbdev emulation (ret=%d)\n", ret);
+	return ret;
+}
+
+static const struct drm_client_funcs tegra_fbdev_client_funcs = {
+	.owner		= THIS_MODULE,
+	.unregister	= tegra_fbdev_client_unregister,
+	.restore	= tegra_fbdev_client_restore,
+	.hotplug	= tegra_fbdev_client_hotplug,
+};
+
+void tegra_fbdev_setup(struct drm_device *dev)
+{
+	struct drm_fb_helper *helper;
+	int ret;
+
+	drm_WARN(dev, !dev->registered, "Device has not been registered.\n");
+	drm_WARN(dev, dev->fb_helper, "fb_helper is already set!\n");
+
+	helper = kzalloc(sizeof(*helper), GFP_KERNEL);
+	if (!helper)
+		return;
+	drm_fb_helper_prepare(dev, helper, 32, &tegra_fb_helper_funcs);
+
+	ret = drm_client_init(dev, &helper->client, "fbdev", &tegra_fbdev_client_funcs);
+	if (ret)
+		goto err_drm_client_init;
+
+	ret = tegra_fbdev_client_hotplug(&helper->client);
+	if (ret)
+		drm_dbg_kms(dev, "client hotplug ret=%d\n", ret);
+
+	drm_client_register(&helper->client);
+
+	return;
+
+err_drm_client_init:
+	drm_fb_helper_unprepare(helper);
+	kfree(helper);
+}
diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c
index bce991a2ccc0..dea38892d6e6 100644
--- a/drivers/gpu/drm/tegra/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -13,6 +13,7 @@
 #include <linux/dma-buf.h>
 #include <linux/iommu.h>
 #include <linux/module.h>
+#include <linux/vmalloc.h>
 
 #include <drm/drm_drv.h>
 #include <drm/drm_prime.h>
diff --git a/drivers/gpu/drm/tegra/gr2d.c b/drivers/gpu/drm/tegra/gr2d.c
index e3bb4c99ed39..50f77fddda54 100644
--- a/drivers/gpu/drm/tegra/gr2d.c
+++ b/drivers/gpu/drm/tegra/gr2d.c
@@ -295,19 +295,11 @@ static int gr2d_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int gr2d_remove(struct platform_device *pdev)
+static void gr2d_remove(struct platform_device *pdev)
 {
 	struct gr2d *gr2d = platform_get_drvdata(pdev);
-	int err;
-
-	err = host1x_client_unregister(&gr2d->client.base);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
-			err);
-		return err;
-	}
 
-	return 0;
+	host1x_client_unregister(&gr2d->client.base);
 }
 
 static int __maybe_unused gr2d_runtime_suspend(struct device *dev)
@@ -403,5 +395,5 @@ struct platform_driver tegra_gr2d_driver = {
 		.pm = &tegra_gr2d_pm,
 	},
 	.probe = gr2d_probe,
-	.remove = gr2d_remove,
+	.remove_new = gr2d_remove,
 };
diff --git a/drivers/gpu/drm/tegra/gr3d.c b/drivers/gpu/drm/tegra/gr3d.c
index a1fd3113ea96..c026c2c916c1 100644
--- a/drivers/gpu/drm/tegra/gr3d.c
+++ b/drivers/gpu/drm/tegra/gr3d.c
@@ -550,19 +550,11 @@ static int gr3d_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int gr3d_remove(struct platform_device *pdev)
+static void gr3d_remove(struct platform_device *pdev)
 {
 	struct gr3d *gr3d = platform_get_drvdata(pdev);
-	int err;
 
-	err = host1x_client_unregister(&gr3d->client.base);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
-			err);
-		return err;
-	}
-
-	return 0;
+	host1x_client_unregister(&gr3d->client.base);
 }
 
 static int __maybe_unused gr3d_runtime_suspend(struct device *dev)
@@ -638,5 +630,5 @@ struct platform_driver tegra_gr3d_driver = {
 		.pm = &tegra_gr3d_pm,
 	},
 	.probe = gr3d_probe,
-	.remove = gr3d_remove,
+	.remove_new = gr3d_remove,
 };
diff --git a/drivers/gpu/drm/tegra/hdmi.c b/drivers/gpu/drm/tegra/hdmi.c
index 40ec3e6cf204..6eac54ae1205 100644
--- a/drivers/gpu/drm/tegra/hdmi.c
+++ b/drivers/gpu/drm/tegra/hdmi.c
@@ -1874,21 +1874,13 @@ static int tegra_hdmi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int tegra_hdmi_remove(struct platform_device *pdev)
+static void tegra_hdmi_remove(struct platform_device *pdev)
 {
 	struct tegra_hdmi *hdmi = platform_get_drvdata(pdev);
-	int err;
 
-	err = host1x_client_unregister(&hdmi->client);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
-			err);
-		return err;
-	}
+	host1x_client_unregister(&hdmi->client);
 
 	tegra_output_remove(&hdmi->output);
-
-	return 0;
 }
 
 struct platform_driver tegra_hdmi_driver = {
@@ -1897,5 +1889,5 @@ struct platform_driver tegra_hdmi_driver = {
 		.of_match_table = tegra_hdmi_of_match,
 	},
 	.probe = tegra_hdmi_probe,
-	.remove = tegra_hdmi_remove,
+	.remove_new = tegra_hdmi_remove,
 };
diff --git a/drivers/gpu/drm/tegra/hub.c b/drivers/gpu/drm/tegra/hub.c
index b872527a123c..916857361a91 100644
--- a/drivers/gpu/drm/tegra/hub.c
+++ b/drivers/gpu/drm/tegra/hub.c
@@ -1174,17 +1174,12 @@ unregister:
 	return err;
 }
 
-static int tegra_display_hub_remove(struct platform_device *pdev)
+static void tegra_display_hub_remove(struct platform_device *pdev)
 {
 	struct tegra_display_hub *hub = platform_get_drvdata(pdev);
 	unsigned int i;
-	int err;
 
-	err = host1x_client_unregister(&hub->client);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
-			err);
-	}
+	host1x_client_unregister(&hub->client);
 
 	for (i = 0; i < hub->soc->num_wgrps; i++) {
 		struct tegra_windowgroup *wgrp = &hub->wgrps[i];
@@ -1193,8 +1188,6 @@ static int tegra_display_hub_remove(struct platform_device *pdev)
 	}
 
 	pm_runtime_disable(&pdev->dev);
-
-	return err;
 }
 
 static const struct tegra_display_hub_soc tegra186_display_hub = {
@@ -1226,5 +1219,5 @@ struct platform_driver tegra_display_hub_driver = {
 		.of_match_table = tegra_display_hub_of_match,
 	},
 	.probe = tegra_display_hub_probe,
-	.remove = tegra_display_hub_remove,
+	.remove_new = tegra_display_hub_remove,
 };
diff --git a/drivers/gpu/drm/tegra/nvdec.c b/drivers/gpu/drm/tegra/nvdec.c
index 86c5818ac27b..ae78a81e5eef 100644
--- a/drivers/gpu/drm/tegra/nvdec.c
+++ b/drivers/gpu/drm/tegra/nvdec.c
@@ -547,21 +547,13 @@ exit_falcon:
 	return err;
 }
 
-static int nvdec_remove(struct platform_device *pdev)
+static void nvdec_remove(struct platform_device *pdev)
 {
 	struct nvdec *nvdec = platform_get_drvdata(pdev);
-	int err;
 
-	err = host1x_client_unregister(&nvdec->client.base);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
-			err);
-		return err;
-	}
+	host1x_client_unregister(&nvdec->client.base);
 
 	falcon_exit(&nvdec->falcon);
-
-	return 0;
 }
 
 static const struct dev_pm_ops nvdec_pm_ops = {
@@ -577,7 +569,7 @@ struct platform_driver tegra_nvdec_driver = {
 		.pm = &nvdec_pm_ops
 	},
 	.probe = nvdec_probe,
-	.remove = nvdec_remove,
+	.remove_new = nvdec_remove,
 };
 
 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
diff --git a/drivers/gpu/drm/tegra/output.c b/drivers/gpu/drm/tegra/output.c
index a8925dcd7edd..dc2dcb5ca1c8 100644
--- a/drivers/gpu/drm/tegra/output.c
+++ b/drivers/gpu/drm/tegra/output.c
@@ -4,6 +4,9 @@
  * Copyright (C) 2012 NVIDIA CORPORATION.  All rights reserved.
  */
 
+#include <linux/i2c.h>
+#include <linux/of.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
diff --git a/drivers/gpu/drm/tegra/plane.c b/drivers/gpu/drm/tegra/plane.c
index 10090116895f..ffe5f06b770d 100644
--- a/drivers/gpu/drm/tegra/plane.c
+++ b/drivers/gpu/drm/tegra/plane.c
@@ -777,21 +777,17 @@ int tegra_plane_interconnect_init(struct tegra_plane *plane)
 
 	plane->icc_mem = devm_of_icc_get(dev, icc_name);
 	err = PTR_ERR_OR_ZERO(plane->icc_mem);
-	if (err) {
-		dev_err_probe(dev, err, "failed to get %s interconnect\n",
-			      icc_name);
-		return err;
-	}
+	if (err)
+		return dev_err_probe(dev, err, "failed to get %s interconnect\n",
+				     icc_name);
 
 	/* plane B on T20/30 has a dedicated memory client for a 6-tap vertical filter */
 	if (plane->index == 1 && dc->soc->has_win_b_vfilter_mem_client) {
 		plane->icc_mem_vfilter = devm_of_icc_get(dev, "winb-vfilter");
 		err = PTR_ERR_OR_ZERO(plane->icc_mem_vfilter);
-		if (err) {
-			dev_err_probe(dev, err, "failed to get %s interconnect\n",
-				      "winb-vfilter");
-			return err;
-		}
+		if (err)
+			return dev_err_probe(dev, err, "failed to get %s interconnect\n",
+					     "winb-vfilter");
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/tegra/rgb.c b/drivers/gpu/drm/tegra/rgb.c
index ff8fce36d2aa..79566c9ea8ff 100644
--- a/drivers/gpu/drm/tegra/rgb.c
+++ b/drivers/gpu/drm/tegra/rgb.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/clk.h>
+#include <linux/of.h>
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge_connector.h>
@@ -250,12 +251,12 @@ int tegra_dc_rgb_probe(struct tegra_dc *dc)
 	return 0;
 }
 
-int tegra_dc_rgb_remove(struct tegra_dc *dc)
+void tegra_dc_rgb_remove(struct tegra_dc *dc)
 {
 	struct tegra_rgb *rgb;
 
 	if (!dc->rgb)
-		return 0;
+		return;
 
 	rgb = to_rgb(dc->rgb);
 	clk_put(rgb->pll_d2_out0);
@@ -263,8 +264,6 @@ int tegra_dc_rgb_remove(struct tegra_dc *dc)
 
 	tegra_output_remove(dc->rgb);
 	dc->rgb = NULL;
-
-	return 0;
 }
 
 int tegra_dc_rgb_init(struct drm_device *drm, struct tegra_dc *dc)
diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c
index 8af632740673..fbb63d755496 100644
--- a/drivers/gpu/drm/tegra/sor.c
+++ b/drivers/gpu/drm/tegra/sor.c
@@ -1153,7 +1153,7 @@ static int tegra_sor_compute_config(struct tegra_sor *sor,
 				    struct drm_dp_link *link)
 {
 	const u64 f = 100000, link_rate = link->rate * 1000;
-	const u64 pclk = mode->clock * 1000;
+	const u64 pclk = (u64)mode->clock * 1000;
 	u64 input, output, watermark, num;
 	struct tegra_sor_params params;
 	u32 num_syms_per_line;
@@ -2140,10 +2140,8 @@ static void tegra_sor_hdmi_disable_scrambling(struct tegra_sor *sor)
 
 static void tegra_sor_hdmi_scdc_disable(struct tegra_sor *sor)
 {
-	struct i2c_adapter *ddc = sor->output.ddc;
-
-	drm_scdc_set_high_tmds_clock_ratio(ddc, false);
-	drm_scdc_set_scrambling(ddc, false);
+	drm_scdc_set_high_tmds_clock_ratio(&sor->output.connector, false);
+	drm_scdc_set_scrambling(&sor->output.connector, false);
 
 	tegra_sor_hdmi_disable_scrambling(sor);
 }
@@ -2168,10 +2166,8 @@ static void tegra_sor_hdmi_enable_scrambling(struct tegra_sor *sor)
 
 static void tegra_sor_hdmi_scdc_enable(struct tegra_sor *sor)
 {
-	struct i2c_adapter *ddc = sor->output.ddc;
-
-	drm_scdc_set_high_tmds_clock_ratio(ddc, true);
-	drm_scdc_set_scrambling(ddc, true);
+	drm_scdc_set_high_tmds_clock_ratio(&sor->output.connector, true);
+	drm_scdc_set_scrambling(&sor->output.connector, true);
 
 	tegra_sor_hdmi_enable_scrambling(sor);
 }
@@ -2179,9 +2175,8 @@ static void tegra_sor_hdmi_scdc_enable(struct tegra_sor *sor)
 static void tegra_sor_hdmi_scdc_work(struct work_struct *work)
 {
 	struct tegra_sor *sor = container_of(work, struct tegra_sor, scdc.work);
-	struct i2c_adapter *ddc = sor->output.ddc;
 
-	if (!drm_scdc_get_scrambling_status(ddc)) {
+	if (!drm_scdc_get_scrambling_status(&sor->output.connector)) {
 		DRM_DEBUG_KMS("SCDC not scrambled\n");
 		tegra_sor_hdmi_scdc_enable(sor);
 	}
@@ -2964,11 +2959,9 @@ static int tegra_sor_hdmi_probe(struct tegra_sor *sor)
 	int err;
 
 	sor->avdd_io_supply = devm_regulator_get(sor->dev, "avdd-io-hdmi-dp");
-	if (IS_ERR(sor->avdd_io_supply)) {
-		dev_err(sor->dev, "cannot get AVDD I/O supply: %ld\n",
-			PTR_ERR(sor->avdd_io_supply));
-		return PTR_ERR(sor->avdd_io_supply);
-	}
+	if (IS_ERR(sor->avdd_io_supply))
+		return dev_err_probe(sor->dev, PTR_ERR(sor->avdd_io_supply),
+				     "cannot get AVDD I/O supply\n");
 
 	err = tegra_sor_enable_regulator(sor, sor->avdd_io_supply);
 	if (err < 0) {
@@ -2978,11 +2971,9 @@ static int tegra_sor_hdmi_probe(struct tegra_sor *sor)
 	}
 
 	sor->vdd_pll_supply = devm_regulator_get(sor->dev, "vdd-hdmi-dp-pll");
-	if (IS_ERR(sor->vdd_pll_supply)) {
-		dev_err(sor->dev, "cannot get VDD PLL supply: %ld\n",
-			PTR_ERR(sor->vdd_pll_supply));
-		return PTR_ERR(sor->vdd_pll_supply);
-	}
+	if (IS_ERR(sor->vdd_pll_supply))
+		return dev_err_probe(sor->dev, PTR_ERR(sor->vdd_pll_supply),
+				     "cannot get VDD PLL supply\n");
 
 	err = tegra_sor_enable_regulator(sor, sor->vdd_pll_supply);
 	if (err < 0) {
@@ -2992,11 +2983,9 @@ static int tegra_sor_hdmi_probe(struct tegra_sor *sor)
 	}
 
 	sor->hdmi_supply = devm_regulator_get(sor->dev, "hdmi");
-	if (IS_ERR(sor->hdmi_supply)) {
-		dev_err(sor->dev, "cannot get HDMI supply: %ld\n",
-			PTR_ERR(sor->hdmi_supply));
-		return PTR_ERR(sor->hdmi_supply);
-	}
+	if (IS_ERR(sor->hdmi_supply))
+		return dev_err_probe(sor->dev, PTR_ERR(sor->hdmi_supply),
+				     "cannot get HDMI supply\n");
 
 	err = tegra_sor_enable_regulator(sor, sor->hdmi_supply);
 	if (err < 0) {
@@ -3799,10 +3788,8 @@ static int tegra_sor_probe(struct platform_device *pdev)
 	}
 
 	err = platform_get_irq(pdev, 0);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to get IRQ: %d\n", err);
+	if (err < 0)
 		goto remove;
-	}
 
 	sor->irq = err;
 
@@ -3978,17 +3965,11 @@ put_aux:
 	return err;
 }
 
-static int tegra_sor_remove(struct platform_device *pdev)
+static void tegra_sor_remove(struct platform_device *pdev)
 {
 	struct tegra_sor *sor = platform_get_drvdata(pdev);
-	int err;
 
-	err = host1x_client_unregister(&sor->client);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
-			err);
-		return err;
-	}
+	host1x_client_unregister(&sor->client);
 
 	pm_runtime_disable(&pdev->dev);
 
@@ -3998,8 +3979,6 @@ static int tegra_sor_remove(struct platform_device *pdev)
 	}
 
 	tegra_output_remove(&sor->output);
-
-	return 0;
 }
 
 static int __maybe_unused tegra_sor_suspend(struct device *dev)
@@ -4059,5 +4038,5 @@ struct platform_driver tegra_sor_driver = {
 		.pm = &tegra_sor_pm_ops,
 	},
 	.probe = tegra_sor_probe,
-	.remove = tegra_sor_remove,
+	.remove_new = tegra_sor_remove,
 };
diff --git a/drivers/gpu/drm/tegra/vic.c b/drivers/gpu/drm/tegra/vic.c
index 531a71c72061..da7a038dca20 100644
--- a/drivers/gpu/drm/tegra/vic.c
+++ b/drivers/gpu/drm/tegra/vic.c
@@ -537,21 +537,13 @@ exit_falcon:
 	return err;
 }
 
-static int vic_remove(struct platform_device *pdev)
+static void vic_remove(struct platform_device *pdev)
 {
 	struct vic *vic = platform_get_drvdata(pdev);
-	int err;
 
-	err = host1x_client_unregister(&vic->client.base);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
-			err);
-		return err;
-	}
+	host1x_client_unregister(&vic->client.base);
 
 	falcon_exit(&vic->falcon);
-
-	return 0;
 }
 
 static const struct dev_pm_ops vic_pm_ops = {
@@ -566,7 +558,7 @@ struct platform_driver tegra_vic_driver = {
 		.pm = &vic_pm_ops
 	},
 	.probe = vic_probe,
-	.remove = vic_remove,
+	.remove_new = vic_remove,
 };
 
 #if IS_ENABLED(CONFIG_ARCH_TEGRA_124_SOC)
diff --git a/drivers/gpu/drm/tests/drm_format_helper_test.c b/drivers/gpu/drm/tests/drm_format_helper_test.c
index 34e80eb6d96e..474bb7a1c4ee 100644
--- a/drivers/gpu/drm/tests/drm_format_helper_test.c
+++ b/drivers/gpu/drm/tests/drm_format_helper_test.c
@@ -67,6 +67,11 @@ struct convert_to_argb2101010_result {
 	const u32 expected[TEST_BUF_SIZE];
 };
 
+struct convert_to_mono_result {
+	unsigned int dst_pitch;
+	const u8 expected[TEST_BUF_SIZE];
+};
+
 struct convert_xrgb8888_case {
 	const char *name;
 	unsigned int pitch;
@@ -82,6 +87,7 @@ struct convert_xrgb8888_case {
 	struct convert_to_argb8888_result argb8888_result;
 	struct convert_to_xrgb2101010_result xrgb2101010_result;
 	struct convert_to_argb2101010_result argb2101010_result;
+	struct convert_to_mono_result mono_result;
 };
 
 static struct convert_xrgb8888_case convert_xrgb8888_cases[] = {
@@ -131,6 +137,10 @@ static struct convert_xrgb8888_case convert_xrgb8888_cases[] = {
 			.dst_pitch = 0,
 			.expected = { 0xFFF00000 },
 		},
+		.mono_result = {
+			.dst_pitch = 0,
+			.expected = { 0b0 },
+		},
 	},
 	{
 		.name = "single_pixel_clip_rectangle",
@@ -181,6 +191,10 @@ static struct convert_xrgb8888_case convert_xrgb8888_cases[] = {
 			.dst_pitch = 0,
 			.expected = { 0xFFF00000 },
 		},
+		.mono_result = {
+			.dst_pitch = 0,
+			.expected = { 0b0 },
+		},
 	},
 	{
 		/* Well known colors: White, black, red, green, blue, magenta,
@@ -293,6 +307,15 @@ static struct convert_xrgb8888_case convert_xrgb8888_cases[] = {
 				0xFFFFFC00, 0xC00FFFFF,
 			},
 		},
+		.mono_result = {
+			.dst_pitch = 0,
+			.expected = {
+				0b01,
+				0b10,
+				0b00,
+				0b11,
+			},
+		},
 	},
 	{
 		/* Randomly picked colors. Full buffer within the clip area. */
@@ -300,96 +323,104 @@ static struct convert_xrgb8888_case convert_xrgb8888_cases[] = {
 		.pitch = 3 * 4,
 		.clip = DRM_RECT_INIT(0, 0, 3, 3),
 		.xrgb8888 = {
-			0xA10E449C, 0xB1114D05, 0xC1A80303,
-			0xD16C7073, 0xA20E449C, 0xB2114D05,
-			0xC2A80303, 0xD26C7073, 0xA30E449C,
+			0xA10E449C, 0xB1114D05, 0xC1A8F303,
+			0xD16CF073, 0xA20E449C, 0xB2114D05,
+			0xC2A80303, 0xD26CF073, 0xA30E449C,
 		},
 		.gray8_result = {
 			.dst_pitch = 5,
 			.expected = {
-				0x3C, 0x33, 0x34, 0x00, 0x00,
-				0x6F, 0x3C, 0x33, 0x00, 0x00,
-				0x34, 0x6F, 0x3C, 0x00, 0x00,
+				0x3C, 0x33, 0xC4, 0x00, 0x00,
+				0xBB, 0x3C, 0x33, 0x00, 0x00,
+				0x34, 0xBB, 0x3C, 0x00, 0x00,
 			},
 		},
 		.rgb332_result = {
 			.dst_pitch = 5,
 			.expected = {
-				0x0A, 0x08, 0xA0, 0x00, 0x00,
-				0x6D, 0x0A, 0x08, 0x00, 0x00,
-				0xA0, 0x6D, 0x0A, 0x00, 0x00,
+				0x0A, 0x08, 0xBC, 0x00, 0x00,
+				0x7D, 0x0A, 0x08, 0x00, 0x00,
+				0xA0, 0x7D, 0x0A, 0x00, 0x00,
 			},
 		},
 		.rgb565_result = {
 			.dst_pitch = 10,
 			.expected = {
-				0x0A33, 0x1260, 0xA800, 0x0000, 0x0000,
-				0x6B8E, 0x0A33, 0x1260, 0x0000, 0x0000,
-				0xA800, 0x6B8E, 0x0A33, 0x0000, 0x0000,
+				0x0A33, 0x1260, 0xAF80, 0x0000, 0x0000,
+				0x6F8E, 0x0A33, 0x1260, 0x0000, 0x0000,
+				0xA800, 0x6F8E, 0x0A33, 0x0000, 0x0000,
 			},
 			.expected_swab = {
-				0x330A, 0x6012, 0x00A8, 0x0000, 0x0000,
-				0x8E6B, 0x330A, 0x6012, 0x0000, 0x0000,
-				0x00A8, 0x8E6B, 0x330A, 0x0000, 0x0000,
+				0x330A, 0x6012, 0x80AF, 0x0000, 0x0000,
+				0x8E6F, 0x330A, 0x6012, 0x0000, 0x0000,
+				0x00A8, 0x8E6F, 0x330A, 0x0000, 0x0000,
 			},
 		},
 		.xrgb1555_result = {
 			.dst_pitch = 10,
 			.expected = {
-				0x0513, 0x0920, 0x5400, 0x0000, 0x0000,
-				0x35CE, 0x0513, 0x0920, 0x0000, 0x0000,
-				0x5400, 0x35CE, 0x0513, 0x0000, 0x0000,
+				0x0513, 0x0920, 0x57C0, 0x0000, 0x0000,
+				0x37CE, 0x0513, 0x0920, 0x0000, 0x0000,
+				0x5400, 0x37CE, 0x0513, 0x0000, 0x0000,
 			},
 		},
 		.argb1555_result = {
 			.dst_pitch = 10,
 			.expected = {
-				0x8513, 0x8920, 0xD400, 0x0000, 0x0000,
-				0xB5CE, 0x8513, 0x8920, 0x0000, 0x0000,
-				0xD400, 0xB5CE, 0x8513, 0x0000, 0x0000,
+				0x8513, 0x8920, 0xD7C0, 0x0000, 0x0000,
+				0xB7CE, 0x8513, 0x8920, 0x0000, 0x0000,
+				0xD400, 0xB7CE, 0x8513, 0x0000, 0x0000,
 			},
 		},
 		.rgba5551_result = {
 			.dst_pitch = 10,
 			.expected = {
-				0x0A27, 0x1241, 0xA801, 0x0000, 0x0000,
-				0x6B9D, 0x0A27, 0x1241, 0x0000, 0x0000,
-				0xA801, 0x6B9D, 0x0A27, 0x0000, 0x0000,
+				0x0A27, 0x1241, 0xAF81, 0x0000, 0x0000,
+				0x6F9D, 0x0A27, 0x1241, 0x0000, 0x0000,
+				0xA801, 0x6F9D, 0x0A27, 0x0000, 0x0000,
 			},
 		},
 		.rgb888_result = {
 			.dst_pitch = 15,
 			.expected = {
-				0x9C, 0x44, 0x0E, 0x05, 0x4D, 0x11, 0x03, 0x03, 0xA8,
+				0x9C, 0x44, 0x0E, 0x05, 0x4D, 0x11, 0x03, 0xF3, 0xA8,
 				0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-				0x73, 0x70, 0x6C, 0x9C, 0x44, 0x0E, 0x05, 0x4D, 0x11,
+				0x73, 0xF0, 0x6C, 0x9C, 0x44, 0x0E, 0x05, 0x4D, 0x11,
 				0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-				0x03, 0x03, 0xA8, 0x73, 0x70, 0x6C, 0x9C, 0x44, 0x0E,
+				0x03, 0x03, 0xA8, 0x73, 0xF0, 0x6C, 0x9C, 0x44, 0x0E,
 				0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 			},
 		},
 		.argb8888_result = {
 			.dst_pitch = 20,
 			.expected = {
-				0xFF0E449C, 0xFF114D05, 0xFFA80303, 0x00000000, 0x00000000,
-				0xFF6C7073, 0xFF0E449C, 0xFF114D05, 0x00000000, 0x00000000,
-				0xFFA80303, 0xFF6C7073, 0xFF0E449C, 0x00000000, 0x00000000,
+				0xFF0E449C, 0xFF114D05, 0xFFA8F303, 0x00000000, 0x00000000,
+				0xFF6CF073, 0xFF0E449C, 0xFF114D05, 0x00000000, 0x00000000,
+				0xFFA80303, 0xFF6CF073, 0xFF0E449C, 0x00000000, 0x00000000,
 			},
 		},
 		.xrgb2101010_result = {
 			.dst_pitch = 20,
 			.expected = {
-				0x03844672, 0x0444D414, 0x2A20300C, 0x00000000, 0x00000000,
-				0x1B1705CD, 0x03844672, 0x0444D414, 0x00000000, 0x00000000,
-				0x2A20300C, 0x1B1705CD, 0x03844672, 0x00000000, 0x00000000,
+				0x03844672, 0x0444D414, 0x2A2F3C0C, 0x00000000, 0x00000000,
+				0x1B1F0DCD, 0x03844672, 0x0444D414, 0x00000000, 0x00000000,
+				0x2A20300C, 0x1B1F0DCD, 0x03844672, 0x00000000, 0x00000000,
 			},
 		},
 		.argb2101010_result = {
 			.dst_pitch = 20,
 			.expected = {
-				0xC3844672, 0xC444D414, 0xEA20300C, 0x00000000, 0x00000000,
-				0xDB1705CD, 0xC3844672, 0xC444D414, 0x00000000, 0x00000000,
-				0xEA20300C, 0xDB1705CD, 0xC3844672, 0x00000000, 0x00000000,
+				0xC3844672, 0xC444D414, 0xEA2F3C0C, 0x00000000, 0x00000000,
+				0xDB1F0DCD, 0xC3844672, 0xC444D414, 0x00000000, 0x00000000,
+				0xEA20300C, 0xDB1F0DCD, 0xC3844672, 0x00000000, 0x00000000,
+			},
+		},
+		.mono_result = {
+			.dst_pitch = 2,
+			.expected = {
+				0b100, 0b000,
+				0b001, 0b000,
+				0b010, 0b000,
 			},
 		},
 	},
@@ -414,7 +445,7 @@ static size_t conversion_buf_size(u32 dst_format, unsigned int dst_pitch,
 		return -EINVAL;
 
 	if (!dst_pitch)
-		dst_pitch = drm_rect_width(clip) * dst_fi->cpp[0];
+		dst_pitch = drm_format_info_min_pitch(dst_fi, 0, drm_rect_width(clip));
 
 	return dst_pitch * drm_rect_height(clip);
 }
@@ -597,7 +628,7 @@ static void drm_test_fb_xrgb8888_to_xrgb1555(struct kunit *test)
 
 	drm_fb_xrgb8888_to_xrgb1555(&dst, &result->dst_pitch, &src, &fb, &params->clip);
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
-	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected, dst_size), 0);
+	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 }
 
 static void drm_test_fb_xrgb8888_to_argb1555(struct kunit *test)
@@ -628,7 +659,7 @@ static void drm_test_fb_xrgb8888_to_argb1555(struct kunit *test)
 
 	drm_fb_xrgb8888_to_argb1555(&dst, &result->dst_pitch, &src, &fb, &params->clip);
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
-	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected, dst_size), 0);
+	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 }
 
 static void drm_test_fb_xrgb8888_to_rgba5551(struct kunit *test)
@@ -659,7 +690,7 @@ static void drm_test_fb_xrgb8888_to_rgba5551(struct kunit *test)
 
 	drm_fb_xrgb8888_to_rgba5551(&dst, &result->dst_pitch, &src, &fb, &params->clip);
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
-	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected, dst_size), 0);
+	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 }
 
 static void drm_test_fb_xrgb8888_to_rgb888(struct kunit *test)
@@ -724,7 +755,7 @@ static void drm_test_fb_xrgb8888_to_argb8888(struct kunit *test)
 
 	drm_fb_xrgb8888_to_argb8888(&dst, &result->dst_pitch, &src, &fb, &params->clip);
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
-	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected, dst_size), 0);
+	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 }
 
 static void drm_test_fb_xrgb8888_to_xrgb2101010(struct kunit *test)
@@ -786,7 +817,37 @@ static void drm_test_fb_xrgb8888_to_argb2101010(struct kunit *test)
 
 	drm_fb_xrgb8888_to_argb2101010(&dst, &result->dst_pitch, &src, &fb, &params->clip);
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
-	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected, dst_size), 0);
+	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
+}
+
+static void drm_test_fb_xrgb8888_to_mono(struct kunit *test)
+{
+	const struct convert_xrgb8888_case *params = test->param_value;
+	const struct convert_to_mono_result *result = &params->mono_result;
+	size_t dst_size;
+	u8 *buf = NULL;
+	__le32 *xrgb8888 = NULL;
+	struct iosys_map dst, src;
+
+	struct drm_framebuffer fb = {
+		.format = drm_format_info(DRM_FORMAT_XRGB8888),
+		.pitches = { params->pitch, 0, 0 },
+	};
+
+	dst_size = conversion_buf_size(DRM_FORMAT_C1, result->dst_pitch, &params->clip);
+
+	KUNIT_ASSERT_GT(test, dst_size, 0);
+
+	buf = kunit_kzalloc(test, dst_size, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
+	iosys_map_set_vaddr(&dst, buf);
+
+	xrgb8888 = cpubuf_to_le32(test, params->xrgb8888, TEST_BUF_SIZE);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xrgb8888);
+	iosys_map_set_vaddr(&src, xrgb8888);
+
+	drm_fb_xrgb8888_to_mono(&dst, &result->dst_pitch, &src, &fb, &params->clip);
+	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 }
 
 static struct kunit_case drm_format_helper_test_cases[] = {
@@ -800,6 +861,7 @@ static struct kunit_case drm_format_helper_test_cases[] = {
 	KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_argb8888, convert_xrgb8888_gen_params),
 	KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_xrgb2101010, convert_xrgb8888_gen_params),
 	KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_argb2101010, convert_xrgb8888_gen_params),
+	KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_mono, convert_xrgb8888_gen_params),
 	{}
 };
 
diff --git a/drivers/gpu/drm/tests/drm_kunit_helpers.c b/drivers/gpu/drm/tests/drm_kunit_helpers.c
index e98b4150f556..4df47071dc88 100644
--- a/drivers/gpu/drm/tests/drm_kunit_helpers.c
+++ b/drivers/gpu/drm/tests/drm_kunit_helpers.c
@@ -19,14 +19,8 @@ static int fake_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int fake_remove(struct platform_device *pdev)
-{
-	return 0;
-}
-
 static struct platform_driver fake_platform_driver = {
 	.probe	= fake_probe,
-	.remove	= fake_remove,
 	.driver = {
 		.name	= KUNIT_DEVICE_NAME,
 	},
diff --git a/drivers/gpu/drm/tidss/tidss_dispc.c b/drivers/gpu/drm/tidss/tidss_dispc.c
index 165365b515e1..dca077411f77 100644
--- a/drivers/gpu/drm/tidss/tidss_dispc.c
+++ b/drivers/gpu/drm/tidss/tidss_dispc.c
@@ -1985,9 +1985,9 @@ dma_addr_t dispc_plane_state_p_uv_addr(const struct drm_plane_state *state)
 		(y * fb->pitches[1] / fb->format->vsub);
 }
 
-int dispc_plane_setup(struct dispc_device *dispc, u32 hw_plane,
-		      const struct drm_plane_state *state,
-		      u32 hw_videoport)
+void dispc_plane_setup(struct dispc_device *dispc, u32 hw_plane,
+		       const struct drm_plane_state *state,
+		       u32 hw_videoport)
 {
 	bool lite = dispc->feat->vid_lite[hw_plane];
 	u32 fourcc = state->fb->format->format;
@@ -2066,15 +2066,11 @@ int dispc_plane_setup(struct dispc_device *dispc, u32 hw_plane,
 	else
 		VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, 0,
 				28, 28);
-
-	return 0;
 }
 
-int dispc_plane_enable(struct dispc_device *dispc, u32 hw_plane, bool enable)
+void dispc_plane_enable(struct dispc_device *dispc, u32 hw_plane, bool enable)
 {
 	VID_REG_FLD_MOD(dispc, hw_plane, DISPC_VID_ATTRIBUTES, !!enable, 0, 0);
-
-	return 0;
 }
 
 static u32 dispc_vid_get_fifo_size(struct dispc_device *dispc, u32 hw_plane)
diff --git a/drivers/gpu/drm/tidss/tidss_dispc.h b/drivers/gpu/drm/tidss/tidss_dispc.h
index e49432f0abf5..946ed769caaf 100644
--- a/drivers/gpu/drm/tidss/tidss_dispc.h
+++ b/drivers/gpu/drm/tidss/tidss_dispc.h
@@ -123,10 +123,10 @@ int dispc_runtime_resume(struct dispc_device *dispc);
 int dispc_plane_check(struct dispc_device *dispc, u32 hw_plane,
 		      const struct drm_plane_state *state,
 		      u32 hw_videoport);
-int dispc_plane_setup(struct dispc_device *dispc, u32 hw_plane,
-		      const struct drm_plane_state *state,
-		      u32 hw_videoport);
-int dispc_plane_enable(struct dispc_device *dispc, u32 hw_plane, bool enable);
+void dispc_plane_setup(struct dispc_device *dispc, u32 hw_plane,
+		       const struct drm_plane_state *state,
+		       u32 hw_videoport);
+void dispc_plane_enable(struct dispc_device *dispc, u32 hw_plane, bool enable);
 const u32 *dispc_plane_formats(struct dispc_device *dispc, unsigned int *len);
 
 int dispc_init(struct tidss_device *tidss);
diff --git a/drivers/gpu/drm/tidss/tidss_drv.c b/drivers/gpu/drm/tidss/tidss_drv.c
index 2dac8727d2f4..3f5f27fb6ebc 100644
--- a/drivers/gpu/drm/tidss/tidss_drv.c
+++ b/drivers/gpu/drm/tidss/tidss_drv.c
@@ -13,7 +13,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_module.h>
@@ -179,7 +179,7 @@ static int tidss_probe(struct platform_device *pdev)
 		goto err_irq_uninstall;
 	}
 
-	drm_fbdev_generic_setup(ddev, 32);
+	drm_fbdev_dma_setup(ddev, 32);
 
 	dev_dbg(dev, "%s done\n", __func__);
 
diff --git a/drivers/gpu/drm/tidss/tidss_plane.c b/drivers/gpu/drm/tidss/tidss_plane.c
index fe2c41f0cd4f..6bdd6e4a955a 100644
--- a/drivers/gpu/drm/tidss/tidss_plane.c
+++ b/drivers/gpu/drm/tidss/tidss_plane.c
@@ -113,7 +113,6 @@ static void tidss_plane_atomic_update(struct drm_plane *plane,
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
 	u32 hw_videoport;
-	int ret;
 
 	dev_dbg(ddev->dev, "%s\n", __func__);
 
@@ -124,15 +123,17 @@ static void tidss_plane_atomic_update(struct drm_plane *plane,
 
 	hw_videoport = to_tidss_crtc(new_state->crtc)->hw_videoport;
 
-	ret = dispc_plane_setup(tidss->dispc, tplane->hw_plane_id,
-				new_state, hw_videoport);
+	dispc_plane_setup(tidss->dispc, tplane->hw_plane_id, new_state, hw_videoport);
+}
 
-	if (ret) {
-		dev_err(plane->dev->dev, "%s: Failed to setup plane %d\n",
-			__func__, tplane->hw_plane_id);
-		dispc_plane_enable(tidss->dispc, tplane->hw_plane_id, false);
-		return;
-	}
+static void tidss_plane_atomic_enable(struct drm_plane *plane,
+				      struct drm_atomic_state *state)
+{
+	struct drm_device *ddev = plane->dev;
+	struct tidss_device *tidss = to_tidss(ddev);
+	struct tidss_plane *tplane = to_tidss_plane(plane);
+
+	dev_dbg(ddev->dev, "%s\n", __func__);
 
 	dispc_plane_enable(tidss->dispc, tplane->hw_plane_id, true);
 }
@@ -160,6 +161,7 @@ static void drm_plane_destroy(struct drm_plane *plane)
 static const struct drm_plane_helper_funcs tidss_plane_helper_funcs = {
 	.atomic_check = tidss_plane_atomic_check,
 	.atomic_update = tidss_plane_atomic_update,
+	.atomic_enable = tidss_plane_atomic_enable,
 	.atomic_disable = tidss_plane_atomic_disable,
 };
 
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index 4ca426007dc8..fe56beea3e93 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -16,7 +16,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_debugfs.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -384,7 +384,7 @@ static int tilcdc_init(const struct drm_driver *ddrv, struct device *dev)
 		goto init_failed;
 	priv->is_registered = true;
 
-	drm_fbdev_generic_setup(ddev, bpp);
+	drm_fbdev_dma_setup(ddev, bpp);
 	return 0;
 
 init_failed:
diff --git a/drivers/gpu/drm/tiny/arcpgu.c b/drivers/gpu/drm/tiny/arcpgu.c
index 611bbee15071..e5b10e41554a 100644
--- a/drivers/gpu/drm/tiny/arcpgu.c
+++ b/drivers/gpu/drm/tiny/arcpgu.c
@@ -12,7 +12,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_fb_dma_helper.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_dma_helper.h>
@@ -394,7 +394,7 @@ static int arcpgu_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_unload;
 
-	drm_fbdev_generic_setup(&arcpgu->drm, 16);
+	drm_fbdev_dma_setup(&arcpgu->drm, 16);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/tiny/bochs.c b/drivers/gpu/drm/tiny/bochs.c
index 024346054c70..d254679a136e 100644
--- a/drivers/gpu/drm/tiny/bochs.c
+++ b/drivers/gpu/drm/tiny/bochs.c
@@ -545,7 +545,6 @@ static int bochs_kms_init(struct bochs_device *bochs)
 
 	bochs->dev->mode_config.preferred_depth = 24;
 	bochs->dev->mode_config.prefer_shadow = 0;
-	bochs->dev->mode_config.prefer_shadow_fbdev = 1;
 	bochs->dev->mode_config.quirk_addfb_prefer_host_byte_order = true;
 
 	bochs->dev->mode_config.funcs = &bochs_mode_funcs;
diff --git a/drivers/gpu/drm/tiny/cirrus.c b/drivers/gpu/drm/tiny/cirrus.c
index accfa52e78c5..594bc472862f 100644
--- a/drivers/gpu/drm/tiny/cirrus.c
+++ b/drivers/gpu/drm/tiny/cirrus.c
@@ -24,6 +24,7 @@
 #include <video/vga.h>
 
 #include <drm/drm_aperture.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_connector.h>
@@ -43,7 +44,6 @@
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_module.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #define DRIVER_NAME "cirrus"
 #define DRIVER_DESC "qemu cirrus vga"
@@ -56,16 +56,34 @@
 
 struct cirrus_device {
 	struct drm_device	       dev;
-	struct drm_simple_display_pipe pipe;
-	struct drm_connector	       conn;
-	unsigned int		       cpp;
-	unsigned int		       pitch;
+
+	/* modesetting pipeline */
+	struct drm_plane	       primary_plane;
+	struct drm_crtc		       crtc;
+	struct drm_encoder	       encoder;
+	struct drm_connector	       connector;
+
+	/* HW resources */
 	void __iomem		       *vram;
 	void __iomem		       *mmio;
 };
 
 #define to_cirrus(_dev) container_of(_dev, struct cirrus_device, dev)
 
+struct cirrus_primary_plane_state {
+	struct drm_shadow_plane_state base;
+
+	/* HW scanout buffer */
+	const struct drm_format_info   *format;
+	unsigned int		       pitch;
+};
+
+static inline struct cirrus_primary_plane_state *
+to_cirrus_primary_plane_state(struct drm_plane_state *plane_state)
+{
+	return container_of(plane_state, struct cirrus_primary_plane_state, base.base);
+};
+
 /* ------------------------------------------------------------------ */
 /*
  * The meat of this driver. The core passes us a mode and we have to program
@@ -126,46 +144,42 @@ static void wreg_hdr(struct cirrus_device *cirrus, u8 val)
 	iowrite8(val, cirrus->mmio + VGA_DAC_MASK);
 }
 
-static int cirrus_convert_to(struct drm_framebuffer *fb)
+static const struct drm_format_info *cirrus_convert_to(struct drm_framebuffer *fb)
 {
-	if (fb->format->cpp[0] == 4 && fb->pitches[0] > CIRRUS_MAX_PITCH) {
+	if (fb->format->format == DRM_FORMAT_XRGB8888 && fb->pitches[0] > CIRRUS_MAX_PITCH) {
 		if (fb->width * 3 <= CIRRUS_MAX_PITCH)
 			/* convert from XR24 to RG24 */
-			return 3;
+			return drm_format_info(DRM_FORMAT_RGB888);
 		else
 			/* convert from XR24 to RG16 */
-			return 2;
+			return drm_format_info(DRM_FORMAT_RGB565);
 	}
-	return 0;
+	return NULL;
 }
 
-static int cirrus_cpp(struct drm_framebuffer *fb)
+static const struct drm_format_info *cirrus_format(struct drm_framebuffer *fb)
 {
-	int convert_cpp = cirrus_convert_to(fb);
+	const struct drm_format_info *format = cirrus_convert_to(fb);
 
-	if (convert_cpp)
-		return convert_cpp;
-	return fb->format->cpp[0];
+	if (format)
+		return format;
+	return fb->format;
 }
 
 static int cirrus_pitch(struct drm_framebuffer *fb)
 {
-	int convert_cpp = cirrus_convert_to(fb);
+	const struct drm_format_info *format = cirrus_convert_to(fb);
 
-	if (convert_cpp)
-		return convert_cpp * fb->width;
+	if (format)
+		return drm_format_info_min_pitch(format, 0, fb->width);
 	return fb->pitches[0];
 }
 
 static void cirrus_set_start_address(struct cirrus_device *cirrus, u32 offset)
 {
-	int idx;
 	u32 addr;
 	u8 tmp;
 
-	if (!drm_dev_enter(&cirrus->dev, &idx))
-		return;
-
 	addr = offset >> 2;
 	wreg_crt(cirrus, 0x0c, (u8)((addr >> 8) & 0xff));
 	wreg_crt(cirrus, 0x0d, (u8)(addr & 0xff));
@@ -180,21 +194,14 @@ static void cirrus_set_start_address(struct cirrus_device *cirrus, u32 offset)
 	tmp &= 0x7f;
 	tmp |= (addr >> 12) & 0x80;
 	wreg_crt(cirrus, 0x1d, tmp);
-
-	drm_dev_exit(idx);
 }
 
-static int cirrus_mode_set(struct cirrus_device *cirrus,
-			   struct drm_display_mode *mode,
-			   struct drm_framebuffer *fb)
+static void cirrus_mode_set(struct cirrus_device *cirrus,
+			    struct drm_display_mode *mode)
 {
 	int hsyncstart, hsyncend, htotal, hdispend;
 	int vtotal, vdispend;
-	int tmp, idx;
-	int sr07 = 0, hdr = 0;
-
-	if (!drm_dev_enter(&cirrus->dev, &idx))
-		return -1;
+	int tmp;
 
 	htotal = mode->htotal / 8;
 	hsyncend = mode->hsync_end / 8;
@@ -258,46 +265,39 @@ static int cirrus_mode_set(struct cirrus_device *cirrus,
 
 	/* Disable Hercules/CGA compatibility */
 	wreg_crt(cirrus, VGA_CRTC_MODE, 0x03);
+}
+
+static void cirrus_format_set(struct cirrus_device *cirrus,
+			      const struct drm_format_info *format)
+{
+	u8 sr07, hdr;
 
 	sr07 = rreg_seq(cirrus, 0x07);
 	sr07 &= 0xe0;
-	hdr = 0;
 
-	cirrus->cpp = cirrus_cpp(fb);
-	switch (cirrus->cpp * 8) {
-	case 8:
+	switch (format->format) {
+	case DRM_FORMAT_C8:
 		sr07 |= 0x11;
+		hdr = 0x00;
 		break;
-	case 16:
+	case DRM_FORMAT_RGB565:
 		sr07 |= 0x17;
 		hdr = 0xc1;
 		break;
-	case 24:
+	case DRM_FORMAT_RGB888:
 		sr07 |= 0x15;
 		hdr = 0xc5;
 		break;
-	case 32:
+	case DRM_FORMAT_XRGB8888:
 		sr07 |= 0x19;
 		hdr = 0xc5;
 		break;
 	default:
-		drm_dev_exit(idx);
-		return -1;
+		return;
 	}
 
 	wreg_seq(cirrus, 0x7, sr07);
 
-	/* Program the pitch */
-	cirrus->pitch = cirrus_pitch(fb);
-	tmp = cirrus->pitch / 8;
-	wreg_crt(cirrus, VGA_CRTC_OFFSET, tmp);
-
-	/* Enable extended blanking and pitch bits, and enable full memory */
-	tmp = 0x22;
-	tmp |= (cirrus->pitch >> 7) & 0x10;
-	tmp |= (cirrus->pitch >> 6) & 0x40;
-	wreg_crt(cirrus, 0x1b, tmp);
-
 	/* Enable high-colour modes */
 	wreg_gfx(cirrus, VGA_GFX_MODE, 0x40);
 
@@ -305,208 +305,323 @@ static int cirrus_mode_set(struct cirrus_device *cirrus,
 	wreg_gfx(cirrus, VGA_GFX_MISC, 0x01);
 
 	wreg_hdr(cirrus, hdr);
+}
 
-	cirrus_set_start_address(cirrus, 0);
+static void cirrus_pitch_set(struct cirrus_device *cirrus, unsigned int pitch)
+{
+	u8 cr13, cr1b;
 
-	/* Unblank (needed on S3 resume, vgabios doesn't do it then) */
-	outb(0x20, 0x3c0);
+	/* Program the pitch */
+	cr13 = pitch / 8;
+	wreg_crt(cirrus, VGA_CRTC_OFFSET, cr13);
 
-	drm_dev_exit(idx);
-	return 0;
+	/* Enable extended blanking and pitch bits, and enable full memory */
+	cr1b = 0x22;
+	cr1b |= (pitch >> 7) & 0x10;
+	cr1b |= (pitch >> 6) & 0x40;
+	wreg_crt(cirrus, 0x1b, cr1b);
+
+	cirrus_set_start_address(cirrus, 0);
 }
 
-static int cirrus_fb_blit_rect(struct drm_framebuffer *fb,
-			       const struct iosys_map *vmap,
-			       struct drm_rect *rect)
-{
-	struct cirrus_device *cirrus = to_cirrus(fb->dev);
-	struct iosys_map dst;
-	int idx;
+/* ------------------------------------------------------------------ */
+/* cirrus display pipe						      */
 
-	if (!drm_dev_enter(&cirrus->dev, &idx))
-		return -ENODEV;
+static const uint32_t cirrus_primary_plane_formats[] = {
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_XRGB8888,
+};
 
-	iosys_map_set_vaddr_iomem(&dst, cirrus->vram);
+static const uint64_t cirrus_primary_plane_format_modifiers[] = {
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
+static int cirrus_primary_plane_helper_atomic_check(struct drm_plane *plane,
+						    struct drm_atomic_state *state)
+{
+	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct cirrus_primary_plane_state *new_primary_plane_state =
+		to_cirrus_primary_plane_state(new_plane_state);
+	struct drm_framebuffer *fb = new_plane_state->fb;
+	struct drm_crtc *new_crtc = new_plane_state->crtc;
+	struct drm_crtc_state *new_crtc_state = NULL;
+	int ret;
+	unsigned int pitch;
 
-	if (cirrus->cpp == fb->format->cpp[0]) {
-		iosys_map_incr(&dst, drm_fb_clip_offset(fb->pitches[0], fb->format, rect));
-		drm_fb_memcpy(&dst, fb->pitches, vmap, fb, rect);
+	if (new_crtc)
+		new_crtc_state = drm_atomic_get_new_crtc_state(state, new_crtc);
 
-	} else if (fb->format->cpp[0] == 4 && cirrus->cpp == 2) {
-		iosys_map_incr(&dst, drm_fb_clip_offset(cirrus->pitch, fb->format, rect));
-		drm_fb_xrgb8888_to_rgb565(&dst, &cirrus->pitch, vmap, fb, rect, false);
+	ret = drm_atomic_helper_check_plane_state(new_plane_state, new_crtc_state,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  false, false);
+	if (ret)
+		return ret;
+	else if (!new_plane_state->visible)
+		return 0;
 
-	} else if (fb->format->cpp[0] == 4 && cirrus->cpp == 3) {
-		iosys_map_incr(&dst, drm_fb_clip_offset(cirrus->pitch, fb->format, rect));
-		drm_fb_xrgb8888_to_rgb888(&dst, &cirrus->pitch, vmap, fb, rect);
+	pitch = cirrus_pitch(fb);
 
-	} else {
-		WARN_ON_ONCE("cpp mismatch");
-	}
+	/* validate size constraints */
+	if (pitch > CIRRUS_MAX_PITCH)
+		return -EINVAL;
+	else if (pitch * fb->height > CIRRUS_VRAM_SIZE)
+		return -EINVAL;
 
-	drm_dev_exit(idx);
+	new_primary_plane_state->format = cirrus_format(fb);
+	new_primary_plane_state->pitch = pitch;
 
 	return 0;
 }
 
-static int cirrus_fb_blit_fullscreen(struct drm_framebuffer *fb,
-				     const struct iosys_map *map)
+static void cirrus_primary_plane_helper_atomic_update(struct drm_plane *plane,
+						      struct drm_atomic_state *state)
 {
-	struct drm_rect fullscreen = {
-		.x1 = 0,
-		.x2 = fb->width,
-		.y1 = 0,
-		.y2 = fb->height,
-	};
-	return cirrus_fb_blit_rect(fb, map, &fullscreen);
-}
+	struct cirrus_device *cirrus = to_cirrus(plane->dev);
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct cirrus_primary_plane_state *primary_plane_state =
+		to_cirrus_primary_plane_state(plane_state);
+	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
+	struct drm_framebuffer *fb = plane_state->fb;
+	const struct drm_format_info *format = primary_plane_state->format;
+	unsigned int pitch = primary_plane_state->pitch;
+	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
+	struct cirrus_primary_plane_state *old_primary_plane_state =
+		to_cirrus_primary_plane_state(old_plane_state);
+	struct iosys_map vaddr = IOSYS_MAP_INIT_VADDR_IOMEM(cirrus->vram);
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_rect damage;
+	int idx;
 
-static int cirrus_check_size(int width, int height,
-			     struct drm_framebuffer *fb)
-{
-	int pitch = width * 2;
+	if (!fb)
+		return;
 
-	if (fb)
-		pitch = cirrus_pitch(fb);
+	if (!drm_dev_enter(&cirrus->dev, &idx))
+		return;
 
-	if (pitch > CIRRUS_MAX_PITCH)
-		return -EINVAL;
-	if (pitch * height > CIRRUS_VRAM_SIZE)
-		return -EINVAL;
-	return 0;
-}
+	if (old_primary_plane_state->format != format)
+		cirrus_format_set(cirrus, format);
+	if (old_primary_plane_state->pitch != pitch)
+		cirrus_pitch_set(cirrus, pitch);
 
-/* ------------------------------------------------------------------ */
-/* cirrus connector						      */
+	drm_atomic_helper_damage_iter_init(&iter, old_plane_state, plane_state);
+	drm_atomic_for_each_plane_damage(&iter, &damage) {
+		unsigned int offset = drm_fb_clip_offset(pitch, format, &damage);
+		struct iosys_map dst = IOSYS_MAP_INIT_OFFSET(&vaddr, offset);
 
-static int cirrus_conn_get_modes(struct drm_connector *conn)
-{
-	int count;
+		drm_fb_blit(&dst, &pitch, format->format, shadow_plane_state->data, fb, &damage);
+	}
 
-	count = drm_add_modes_noedid(conn,
-				     conn->dev->mode_config.max_width,
-				     conn->dev->mode_config.max_height);
-	drm_set_preferred_mode(conn, 1024, 768);
-	return count;
+	drm_dev_exit(idx);
 }
 
-static const struct drm_connector_helper_funcs cirrus_conn_helper_funcs = {
-	.get_modes = cirrus_conn_get_modes,
-};
-
-static const struct drm_connector_funcs cirrus_conn_funcs = {
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = drm_connector_cleanup,
-	.reset = drm_atomic_helper_connector_reset,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+static const struct drm_plane_helper_funcs cirrus_primary_plane_helper_funcs = {
+	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
+	.atomic_check = cirrus_primary_plane_helper_atomic_check,
+	.atomic_update = cirrus_primary_plane_helper_atomic_update,
 };
 
-static int cirrus_conn_init(struct cirrus_device *cirrus)
+static struct drm_plane_state *
+cirrus_primary_plane_atomic_duplicate_state(struct drm_plane *plane)
 {
-	drm_connector_helper_add(&cirrus->conn, &cirrus_conn_helper_funcs);
-	return drm_connector_init(&cirrus->dev, &cirrus->conn,
-				  &cirrus_conn_funcs, DRM_MODE_CONNECTOR_VGA);
+	struct drm_plane_state *plane_state = plane->state;
+	struct cirrus_primary_plane_state *primary_plane_state =
+		to_cirrus_primary_plane_state(plane_state);
+	struct cirrus_primary_plane_state *new_primary_plane_state;
+	struct drm_shadow_plane_state *new_shadow_plane_state;
 
-}
+	if (!plane_state)
+		return NULL;
 
-/* ------------------------------------------------------------------ */
-/* cirrus (simple) display pipe					      */
+	new_primary_plane_state = kzalloc(sizeof(*new_primary_plane_state), GFP_KERNEL);
+	if (!new_primary_plane_state)
+		return NULL;
+	new_shadow_plane_state = &new_primary_plane_state->base;
+
+	__drm_gem_duplicate_shadow_plane_state(plane, new_shadow_plane_state);
+	new_primary_plane_state->format = primary_plane_state->format;
+	new_primary_plane_state->pitch = primary_plane_state->pitch;
+
+	return &new_shadow_plane_state->base;
+}
 
-static enum drm_mode_status cirrus_pipe_mode_valid(struct drm_simple_display_pipe *pipe,
-						   const struct drm_display_mode *mode)
+static void cirrus_primary_plane_atomic_destroy_state(struct drm_plane *plane,
+						      struct drm_plane_state *plane_state)
 {
-	if (cirrus_check_size(mode->hdisplay, mode->vdisplay, NULL) < 0)
-		return MODE_BAD;
-	return MODE_OK;
+	struct cirrus_primary_plane_state *primary_plane_state =
+		to_cirrus_primary_plane_state(plane_state);
+
+	__drm_gem_destroy_shadow_plane_state(&primary_plane_state->base);
+	kfree(primary_plane_state);
 }
 
-static int cirrus_pipe_check(struct drm_simple_display_pipe *pipe,
-			     struct drm_plane_state *plane_state,
-			     struct drm_crtc_state *crtc_state)
+static void cirrus_reset_primary_plane(struct drm_plane *plane)
 {
-	struct drm_framebuffer *fb = plane_state->fb;
+	struct cirrus_primary_plane_state *primary_plane_state;
 
-	if (!fb)
-		return 0;
-	return cirrus_check_size(fb->width, fb->height, fb);
+	if (plane->state) {
+		cirrus_primary_plane_atomic_destroy_state(plane, plane->state);
+		plane->state = NULL; /* must be set to NULL here */
+	}
+
+	primary_plane_state = kzalloc(sizeof(*primary_plane_state), GFP_KERNEL);
+	if (!primary_plane_state)
+		return;
+	__drm_gem_reset_shadow_plane(plane, &primary_plane_state->base);
 }
 
-static void cirrus_pipe_enable(struct drm_simple_display_pipe *pipe,
-			       struct drm_crtc_state *crtc_state,
-			       struct drm_plane_state *plane_state)
+static const struct drm_plane_funcs cirrus_primary_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = drm_plane_cleanup,
+	.reset = cirrus_reset_primary_plane,
+	.atomic_duplicate_state = cirrus_primary_plane_atomic_duplicate_state,
+	.atomic_destroy_state = cirrus_primary_plane_atomic_destroy_state,
+};
+
+static int cirrus_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state)
 {
-	struct cirrus_device *cirrus = to_cirrus(pipe->crtc.dev);
-	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+	int ret;
+
+	if (!crtc_state->enable)
+		return 0;
+
+	ret = drm_atomic_helper_check_crtc_primary_plane(crtc_state);
+	if (ret)
+		return ret;
 
-	cirrus_mode_set(cirrus, &crtc_state->mode, plane_state->fb);
-	cirrus_fb_blit_fullscreen(plane_state->fb, &shadow_plane_state->data[0]);
+	return 0;
 }
 
-static void cirrus_pipe_update(struct drm_simple_display_pipe *pipe,
-			       struct drm_plane_state *old_state)
+static void cirrus_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+					     struct drm_atomic_state *state)
 {
-	struct cirrus_device *cirrus = to_cirrus(pipe->crtc.dev);
-	struct drm_plane_state *state = pipe->plane.state;
-	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(state);
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct drm_rect rect;
+	struct cirrus_device *cirrus = to_cirrus(crtc->dev);
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+	int idx;
+
+	if (!drm_dev_enter(&cirrus->dev, &idx))
+		return;
 
-	if (state->fb && cirrus->cpp != cirrus_cpp(state->fb))
-		cirrus_mode_set(cirrus, &crtc->mode, state->fb);
+	cirrus_mode_set(cirrus, &crtc_state->mode);
 
-	if (state->fb && drm_atomic_helper_damage_merged(old_state, state, &rect))
-		cirrus_fb_blit_rect(state->fb, &shadow_plane_state->data[0], &rect);
+	/* Unblank (needed on S3 resume, vgabios doesn't do it then) */
+	outb(VGA_AR_ENABLE_DISPLAY, VGA_ATT_W);
+
+	drm_dev_exit(idx);
 }
 
-static const struct drm_simple_display_pipe_funcs cirrus_pipe_funcs = {
-	.mode_valid = cirrus_pipe_mode_valid,
-	.check	    = cirrus_pipe_check,
-	.enable	    = cirrus_pipe_enable,
-	.update	    = cirrus_pipe_update,
-	DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS,
+static const struct drm_crtc_helper_funcs cirrus_crtc_helper_funcs = {
+	.atomic_check = cirrus_crtc_helper_atomic_check,
+	.atomic_enable = cirrus_crtc_helper_atomic_enable,
 };
 
-static const uint32_t cirrus_formats[] = {
-	DRM_FORMAT_RGB565,
-	DRM_FORMAT_RGB888,
-	DRM_FORMAT_XRGB8888,
+static const struct drm_crtc_funcs cirrus_crtc_funcs = {
+	.reset = drm_atomic_helper_crtc_reset,
+	.destroy = drm_crtc_cleanup,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 };
 
-static const uint64_t cirrus_modifiers[] = {
-	DRM_FORMAT_MOD_LINEAR,
-	DRM_FORMAT_MOD_INVALID
+static const struct drm_encoder_funcs cirrus_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+static int cirrus_connector_helper_get_modes(struct drm_connector *connector)
+{
+	int count;
+
+	count = drm_add_modes_noedid(connector,
+				     connector->dev->mode_config.max_width,
+				     connector->dev->mode_config.max_height);
+	drm_set_preferred_mode(connector, 1024, 768);
+	return count;
+}
+
+static const struct drm_connector_helper_funcs cirrus_connector_helper_funcs = {
+	.get_modes = cirrus_connector_helper_get_modes,
+};
+
+static const struct drm_connector_funcs cirrus_connector_funcs = {
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
 static int cirrus_pipe_init(struct cirrus_device *cirrus)
 {
-	return drm_simple_display_pipe_init(&cirrus->dev,
-					    &cirrus->pipe,
-					    &cirrus_pipe_funcs,
-					    cirrus_formats,
-					    ARRAY_SIZE(cirrus_formats),
-					    cirrus_modifiers,
-					    &cirrus->conn);
+	struct drm_device *dev = &cirrus->dev;
+	struct drm_plane *primary_plane;
+	struct drm_crtc *crtc;
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+	int ret;
+
+	primary_plane = &cirrus->primary_plane;
+	ret = drm_universal_plane_init(dev, primary_plane, 0,
+				       &cirrus_primary_plane_funcs,
+				       cirrus_primary_plane_formats,
+				       ARRAY_SIZE(cirrus_primary_plane_formats),
+				       cirrus_primary_plane_format_modifiers,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret)
+		return ret;
+	drm_plane_helper_add(primary_plane, &cirrus_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	crtc = &cirrus->crtc;
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&cirrus_crtc_funcs, NULL);
+	if (ret)
+		return ret;
+	drm_crtc_helper_add(crtc, &cirrus_crtc_helper_funcs);
+
+	encoder = &cirrus->encoder;
+	ret = drm_encoder_init(dev, encoder, &cirrus_encoder_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
+	if (ret)
+		return ret;
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+
+	connector = &cirrus->connector;
+	ret = drm_connector_init(dev, connector, &cirrus_connector_funcs,
+				 DRM_MODE_CONNECTOR_VGA);
+	if (ret)
+		return ret;
+	drm_connector_helper_add(connector, &cirrus_connector_helper_funcs);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret)
+		return ret;
+
+	return 0;
 }
 
 /* ------------------------------------------------------------------ */
 /* cirrus framebuffers & mode config				      */
 
-static struct drm_framebuffer*
-cirrus_fb_create(struct drm_device *dev, struct drm_file *file_priv,
-		 const struct drm_mode_fb_cmd2 *mode_cmd)
+static enum drm_mode_status cirrus_mode_config_mode_valid(struct drm_device *dev,
+							  const struct drm_display_mode *mode)
 {
-	if (mode_cmd->pixel_format != DRM_FORMAT_RGB565 &&
-	    mode_cmd->pixel_format != DRM_FORMAT_RGB888 &&
-	    mode_cmd->pixel_format != DRM_FORMAT_XRGB8888)
-		return ERR_PTR(-EINVAL);
-	if (cirrus_check_size(mode_cmd->width, mode_cmd->height, NULL) < 0)
-		return ERR_PTR(-EINVAL);
-	return drm_gem_fb_create_with_dirty(dev, file_priv, mode_cmd);
+	const struct drm_format_info *format = drm_format_info(DRM_FORMAT_XRGB8888);
+	uint64_t pitch = drm_format_info_min_pitch(format, 0, mode->hdisplay);
+
+	if (pitch * mode->vdisplay > CIRRUS_VRAM_SIZE)
+		return MODE_MEM;
+
+	return MODE_OK;
 }
 
 static const struct drm_mode_config_funcs cirrus_mode_config_funcs = {
-	.fb_create = cirrus_fb_create,
+	.fb_create = drm_gem_fb_create_with_dirty,
+	.mode_valid = cirrus_mode_config_mode_valid,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -589,10 +704,6 @@ static int cirrus_pci_probe(struct pci_dev *pdev,
 	if (ret)
 		return ret;
 
-	ret = cirrus_conn_init(cirrus);
-	if (ret < 0)
-		return ret;
-
 	ret = cirrus_pipe_init(cirrus);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/gpu/drm/tiny/ofdrm.c b/drivers/gpu/drm/tiny/ofdrm.c
index 6e349ca42485..76cd7f515bab 100644
--- a/drivers/gpu/drm/tiny/ofdrm.c
+++ b/drivers/gpu/drm/tiny/ofdrm.c
@@ -162,13 +162,9 @@ static bool display_get_big_endian_of(struct drm_device *dev, struct device_node
 	bool big_endian;
 
 #ifdef __BIG_ENDIAN
-	big_endian = true;
-	if (of_get_property(of_node, "little-endian", NULL))
-		big_endian = false;
+	big_endian = !of_property_read_bool(of_node, "little-endian");
 #else
-	big_endian = false;
-	if (of_get_property(of_node, "big-endian", NULL))
-		big_endian = true;
+	big_endian = of_property_read_bool(of_node, "big-endian");
 #endif
 
 	return big_endian;
diff --git a/drivers/gpu/drm/tiny/simpledrm.c b/drivers/gpu/drm/tiny/simpledrm.c
index 63881a3754f8..25e11ef11c4c 100644
--- a/drivers/gpu/drm/tiny/simpledrm.c
+++ b/drivers/gpu/drm/tiny/simpledrm.c
@@ -204,7 +204,7 @@ simplefb_get_memory_of(struct drm_device *dev, struct device_node *of_node)
 	if (err)
 		return ERR_PTR(err);
 
-	if (of_get_property(of_node, "reg", NULL))
+	if (of_property_present(of_node, "reg"))
 		drm_warn(dev, "preferring \"memory-region\" over \"reg\" property\n");
 
 	return res;
@@ -606,16 +606,12 @@ static const struct drm_mode_config_funcs simpledrm_mode_config_funcs = {
  */
 
 static struct drm_display_mode simpledrm_mode(unsigned int width,
-					      unsigned int height)
+					      unsigned int height,
+					      unsigned int width_mm,
+					      unsigned int height_mm)
 {
-	/*
-	 * Assume a monitor resolution of 96 dpi to
-	 * get a somewhat reasonable screen size.
-	 */
 	const struct drm_display_mode mode = {
-		DRM_MODE_INIT(60, width, height,
-			      DRM_MODE_RES_MM(width, 96ul),
-			      DRM_MODE_RES_MM(height, 96ul))
+		DRM_MODE_INIT(60, width, height, width_mm, height_mm)
 	};
 
 	return mode;
@@ -629,6 +625,8 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv,
 	struct simpledrm_device *sdev;
 	struct drm_device *dev;
 	int width, height, stride;
+	int width_mm = 0, height_mm = 0;
+	struct device_node *panel_node;
 	const struct drm_format_info *format;
 	struct resource *res, *mem = NULL;
 	struct drm_plane *primary_plane;
@@ -685,6 +683,12 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv,
 		mem = simplefb_get_memory_of(dev, of_node);
 		if (IS_ERR(mem))
 			return ERR_CAST(mem);
+		panel_node = of_parse_phandle(of_node, "panel", 0);
+		if (panel_node) {
+			simplefb_read_u32_of(dev, panel_node, "width-mm", &width_mm);
+			simplefb_read_u32_of(dev, panel_node, "height-mm", &height_mm);
+			of_node_put(panel_node);
+		}
 	} else {
 		drm_err(dev, "no simplefb configuration found\n");
 		return ERR_PTR(-ENODEV);
@@ -695,7 +699,16 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv,
 			return ERR_PTR(-EINVAL);
 	}
 
-	sdev->mode = simpledrm_mode(width, height);
+	/*
+	 * Assume a monitor resolution of 96 dpi if physical dimensions
+	 * are not specified to get a somewhat reasonable screen size.
+	 */
+	if (!width_mm)
+		width_mm = DRM_MODE_RES_MM(width, 96ul);
+	if (!height_mm)
+		height_mm = DRM_MODE_RES_MM(height, 96ul);
+
+	sdev->mode = simpledrm_mode(width, height, width_mm, height_mm);
 	sdev->format = format;
 	sdev->pitch = stride;
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index c286c6ffe07f..bd5dae4d1624 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -84,6 +84,7 @@ EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
  * ttm_bo_set_bulk_move - update BOs bulk move object
  *
  * @bo: The buffer object.
+ * @bulk: bulk move structure
  *
  * Update the BOs bulk move object, making sure that resources are added/removed
  * as well. A bulk move allows to move many resource on the LRU at once,
@@ -120,8 +121,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 	bool old_use_tt, new_use_tt;
 	int ret;
 
-	old_use_tt = bo->resource &&
-		ttm_manager_type(bdev, bo->resource->mem_type)->use_tt;
+	old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt;
 	new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt;
 
 	ttm_bo_unmap_virtual(bo);
@@ -463,7 +463,8 @@ bounce:
 	if (ret == -EMULTIHOP) {
 		ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop);
 		if (ret) {
-			pr_err("Buffer eviction failed\n");
+			if (ret != -ERESTARTSYS && ret != -EINTR)
+				pr_err("Buffer eviction failed\n");
 			ttm_resource_free(bo, &evict_mem);
 			goto out;
 		}
@@ -746,7 +747,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
  *
  * @bo: Pointer to a struct ttm_buffer_object. the data of which
  * we want to allocate space for.
- * @proposed_placement: Proposed new placement for the buffer object.
+ * @placement: Proposed new placement for the buffer object.
  * @mem: A struct ttm_resource.
  * @ctx: if and how to sleep, lock buffers and alloc memory
  *
@@ -892,14 +893,18 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,
 	if (!placement->num_placement && !placement->num_busy_placement)
 		return ttm_bo_pipeline_gutting(bo);
 
-	/*
-	 * Check whether we need to move buffer.
-	 */
-	if (!bo->resource || !ttm_resource_compat(bo->resource, placement)) {
-		ret = ttm_bo_move_buffer(bo, placement, ctx);
-		if (ret)
-			return ret;
-	}
+	/* Check whether we need to move buffer. */
+	if (bo->resource && ttm_resource_compat(bo->resource, placement))
+		return 0;
+
+	/* Moving of pinned BOs is forbidden */
+	if (bo->pin_count)
+		return -EINVAL;
+
+	ret = ttm_bo_move_buffer(bo, placement, ctx);
+	if (ret)
+		return ret;
+
 	/*
 	 * We might need to add a TTM.
 	 */
@@ -951,7 +956,6 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo,
 			 struct sg_table *sg, struct dma_resv *resv,
 			 void (*destroy) (struct ttm_buffer_object *))
 {
-	static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM };
 	int ret;
 
 	kref_init(&bo->kref);
@@ -968,12 +972,6 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo,
 		bo->base.resv = &bo->base._resv;
 	atomic_inc(&ttm_glob.bo_count);
 
-	ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource);
-	if (unlikely(ret)) {
-		ttm_bo_put(bo);
-		return ret;
-	}
-
 	/*
 	 * For ttm_bo_type_device buffers, allocate
 	 * address space from the device.
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 7635d7d6b13b..fd9fd3d15101 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -157,8 +157,8 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 	bool clear;
 	int ret = 0;
 
-	if (!src_mem)
-		return 0;
+	if (WARN_ON(!src_mem))
+		return -EINVAL;
 
 	src_man = ttm_manager_type(bdev, src_mem->mem_type);
 	if (ttm && ((ttm->page_flags & TTM_TT_FLAG_SWAPPED) ||
@@ -704,30 +704,23 @@ EXPORT_SYMBOL(ttm_bo_move_sync_cleanup);
  */
 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
 {
-	static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM };
 	struct ttm_buffer_object *ghost;
-	struct ttm_resource *sys_res;
 	struct ttm_tt *ttm;
 	int ret;
 
-	ret = ttm_resource_alloc(bo, &sys_mem, &sys_res);
-	if (ret)
-		return ret;
-
 	/* If already idle, no need for ghost object dance. */
 	if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP)) {
 		if (!bo->ttm) {
 			/* See comment below about clearing. */
 			ret = ttm_tt_create(bo, true);
 			if (ret)
-				goto error_free_sys_mem;
+				return ret;
 		} else {
 			ttm_tt_unpopulate(bo->bdev, bo->ttm);
 			if (bo->type == ttm_bo_type_device)
 				ttm_tt_mark_for_clear(bo->ttm);
 		}
 		ttm_resource_free(bo, &bo->resource);
-		ttm_bo_assign_mem(bo, sys_res);
 		return 0;
 	}
 
@@ -744,7 +737,7 @@ int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
 	ret = ttm_tt_create(bo, true);
 	swap(bo->ttm, ttm);
 	if (ret)
-		goto error_free_sys_mem;
+		return ret;
 
 	ret = ttm_buffer_object_transfer(bo, &ghost);
 	if (ret)
@@ -760,13 +753,9 @@ int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
 	dma_resv_unlock(&ghost->base._resv);
 	ttm_bo_put(ghost);
 	bo->ttm = ttm;
-	ttm_bo_assign_mem(bo, sys_res);
 	return 0;
 
 error_destroy_tt:
 	ttm_tt_destroy(bo->bdev, ttm);
-
-error_free_sys_mem:
-	ttm_resource_free(bo, &sys_res);
 	return ret;
 }
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index ca7744b852f5..4bca6b54520a 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -218,14 +218,21 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
 	prot = ttm_io_prot(bo, bo->resource, prot);
 	if (!bo->resource->bus.is_iomem) {
 		struct ttm_operation_ctx ctx = {
-			.interruptible = false,
+			.interruptible = true,
 			.no_wait_gpu = false,
 			.force_alloc = true
 		};
 
 		ttm = bo->ttm;
-		if (ttm_tt_populate(bdev, bo->ttm, &ctx))
-			return VM_FAULT_OOM;
+		err = ttm_tt_populate(bdev, bo->ttm, &ctx);
+		if (err) {
+			if (err == -EINTR || err == -ERESTARTSYS ||
+			    err == -EAGAIN)
+				return VM_FAULT_NOPAGE;
+
+			pr_debug("TTM fault hit %pe.\n", ERR_PTR(err));
+			return VM_FAULT_SIGBUS;
+		}
 	} else {
 		/* Iomem should not be marked encrypted */
 		prot = pgprot_decrypted(prot);
diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index ae2f19dc9f81..64a59f46f6c3 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -137,7 +137,6 @@ int ttm_global_swapout(struct ttm_operation_ctx *ctx, gfp_t gfp_flags)
 	mutex_unlock(&ttm_global_mutex);
 	return ret;
 }
-EXPORT_SYMBOL(ttm_global_swapout);
 
 int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
 		       gfp_t gfp_flags)
diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
index aa116a7bbae3..dfce896c4bae 100644
--- a/drivers/gpu/drm/ttm/ttm_pool.c
+++ b/drivers/gpu/drm/ttm/ttm_pool.c
@@ -368,6 +368,43 @@ static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order,
 }
 
 /**
+ * ttm_pool_free_range() - Free a range of TTM pages
+ * @pool: The pool used for allocating.
+ * @tt: The struct ttm_tt holding the page pointers.
+ * @caching: The page caching mode used by the range.
+ * @start_page: index for first page to free.
+ * @end_page: index for last page to free + 1.
+ *
+ * During allocation the ttm_tt page-vector may be populated with ranges of
+ * pages with different attributes if allocation hit an error without being
+ * able to completely fulfill the allocation. This function can be used
+ * to free these individual ranges.
+ */
+static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt,
+				enum ttm_caching caching,
+				pgoff_t start_page, pgoff_t end_page)
+{
+	struct page **pages = tt->pages;
+	unsigned int order;
+	pgoff_t i, nr;
+
+	for (i = start_page; i < end_page; i += nr, pages += nr) {
+		struct ttm_pool_type *pt = NULL;
+
+		order = ttm_pool_page_order(pool, *pages);
+		nr = (1UL << order);
+		if (tt->dma_address)
+			ttm_pool_unmap(pool, tt->dma_address[i], nr);
+
+		pt = ttm_pool_select_type(pool, caching, order);
+		if (pt)
+			ttm_pool_type_give(pt, *pages);
+		else
+			ttm_pool_free_page(pool, caching, order, *pages);
+	}
+}
+
+/**
  * ttm_pool_alloc - Fill a ttm_tt object
  *
  * @pool: ttm_pool to use
@@ -382,12 +419,14 @@ static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order,
 int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 		   struct ttm_operation_ctx *ctx)
 {
-	unsigned long num_pages = tt->num_pages;
+	pgoff_t num_pages = tt->num_pages;
 	dma_addr_t *dma_addr = tt->dma_address;
 	struct page **caching = tt->pages;
 	struct page **pages = tt->pages;
+	enum ttm_caching page_caching;
 	gfp_t gfp_flags = GFP_USER;
-	unsigned int i, order;
+	pgoff_t caching_divide;
+	unsigned int order;
 	struct page *p;
 	int r;
 
@@ -410,6 +449,7 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 	     order = min_t(unsigned int, order, __fls(num_pages))) {
 		struct ttm_pool_type *pt;
 
+		page_caching = tt->caching;
 		pt = ttm_pool_select_type(pool, tt->caching, order);
 		p = pt ? ttm_pool_type_take(pt) : NULL;
 		if (p) {
@@ -418,6 +458,7 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 			if (r)
 				goto error_free_page;
 
+			caching = pages;
 			do {
 				r = ttm_pool_page_allocated(pool, order, p,
 							    &dma_addr,
@@ -426,14 +467,15 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 				if (r)
 					goto error_free_page;
 
+				caching = pages;
 				if (num_pages < (1 << order))
 					break;
 
 				p = ttm_pool_type_take(pt);
 			} while (p);
-			caching = pages;
 		}
 
+		page_caching = ttm_cached;
 		while (num_pages >= (1 << order) &&
 		       (p = ttm_pool_alloc_page(pool, gfp_flags, order))) {
 
@@ -442,6 +484,7 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 							   tt->caching);
 				if (r)
 					goto error_free_page;
+				caching = pages;
 			}
 			r = ttm_pool_page_allocated(pool, order, p, &dma_addr,
 						    &num_pages, &pages);
@@ -468,15 +511,13 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 	return 0;
 
 error_free_page:
-	ttm_pool_free_page(pool, tt->caching, order, p);
+	ttm_pool_free_page(pool, page_caching, order, p);
 
 error_free_all:
 	num_pages = tt->num_pages - num_pages;
-	for (i = 0; i < num_pages; ) {
-		order = ttm_pool_page_order(pool, tt->pages[i]);
-		ttm_pool_free_page(pool, tt->caching, order, tt->pages[i]);
-		i += 1 << order;
-	}
+	caching_divide = caching - tt->pages;
+	ttm_pool_free_range(pool, tt, tt->caching, 0, caching_divide);
+	ttm_pool_free_range(pool, tt, ttm_cached, caching_divide, num_pages);
 
 	return r;
 }
@@ -492,27 +533,7 @@ EXPORT_SYMBOL(ttm_pool_alloc);
  */
 void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt)
 {
-	unsigned int i;
-
-	for (i = 0; i < tt->num_pages; ) {
-		struct page *p = tt->pages[i];
-		unsigned int order, num_pages;
-		struct ttm_pool_type *pt;
-
-		order = ttm_pool_page_order(pool, p);
-		num_pages = 1ULL << order;
-		if (tt->dma_address)
-			ttm_pool_unmap(pool, tt->dma_address[i], num_pages);
-
-		pt = ttm_pool_select_type(pool, tt->caching, order);
-		if (pt)
-			ttm_pool_type_give(pt, tt->pages[i]);
-		else
-			ttm_pool_free_page(pool, tt->caching, order,
-					   tt->pages[i]);
-
-		i += num_pages;
-	}
+	ttm_pool_free_range(pool, tt, tt->caching, 0, tt->num_pages);
 
 	while (atomic_long_read(&allocated_pages) > page_pool_size)
 		ttm_pool_shrink();
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index b8a826a24fb2..7333f7a87a2f 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -361,7 +361,6 @@ bool ttm_resource_compat(struct ttm_resource *res,
 
 	return false;
 }
-EXPORT_SYMBOL(ttm_resource_compat);
 
 void ttm_resource_set_bo(struct ttm_resource *res,
 			 struct ttm_buffer_object *bo)
diff --git a/drivers/gpu/drm/tve200/tve200_drv.c b/drivers/gpu/drm/tve200/tve200_drv.c
index 0d05c386d303..abd557332b28 100644
--- a/drivers/gpu/drm/tve200/tve200_drv.c
+++ b/drivers/gpu/drm/tve200/tve200_drv.c
@@ -40,7 +40,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
@@ -227,7 +227,7 @@ static int tve200_probe(struct platform_device *pdev)
 	 * Passing in 16 here will make the RGB565 mode the default
 	 * Passing in 32 will use XRGB8888 mode
 	 */
-	drm_fbdev_generic_setup(drm, 16);
+	drm_fbdev_dma_setup(drm, 16);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index 5da1806f3969..2e94ce788c71 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -397,20 +397,6 @@ v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
 }
 
 static int
-v3d_job_add_deps(struct drm_file *file_priv, struct v3d_job *job,
-		 u32 in_sync, u32 point)
-{
-	struct dma_fence *in_fence = NULL;
-	int ret;
-
-	ret = drm_syncobj_find_fence(file_priv, in_sync, point, 0, &in_fence);
-	if (ret == -EINVAL)
-		return ret;
-
-	return drm_sched_job_add_dependency(&job->base, in_fence);
-}
-
-static int
 v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
 	     void **container, size_t size, void (*free)(struct kref *ref),
 	     u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue)
@@ -447,14 +433,18 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
 					DRM_DEBUG("Failed to copy wait dep handle.\n");
 					goto fail_deps;
 				}
-				ret = v3d_job_add_deps(file_priv, job, in.handle, 0);
-				if (ret)
+				ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0);
+
+				// TODO: Investigate why this was filtered out for the IOCTL.
+				if (ret && ret != -ENOENT)
 					goto fail_deps;
 			}
 		}
 	} else {
-		ret = v3d_job_add_deps(file_priv, job, in_sync, 0);
-		if (ret)
+		ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0);
+
+		// TODO: Investigate why this was filtered out for the IOCTL.
+		if (ret && ret != -ENOENT)
 			goto fail_deps;
 	}
 
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index 0ccaee57fe9a..c8bf954042e0 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -33,7 +33,7 @@
 #include <drm/drm_aperture.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_vblank.h>
 
 #include <soc/bcm2835/raspberrypi-firmware.h>
@@ -387,7 +387,7 @@ static int vc4_drm_bind(struct device *dev)
 	if (ret < 0)
 		goto unbind_all;
 
-	drm_fbdev_generic_setup(drm, 16);
+	drm_fbdev_dma_setup(drm, 16);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 95069bb16821..8768566c610b 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -690,7 +690,7 @@ struct vc4_exec_info {
 	/* This is the array of BOs that were looked up at the start of exec.
 	 * Command validation will use indices into this array.
 	 */
-	struct drm_gem_dma_object **bo;
+	struct drm_gem_object **bo;
 	uint32_t bo_count;
 
 	/* List of BOs that are being written by the RCL.  Other than
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index 628d40ff3aa1..03648f954985 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -199,7 +199,7 @@ vc4_save_hang_state(struct drm_device *dev)
 			continue;
 
 		for (j = 0; j < exec[i]->bo_count; j++) {
-			bo = to_vc4_bo(&exec[i]->bo[j]->base);
+			bo = to_vc4_bo(exec[i]->bo[j]);
 
 			/* Retain BOs just in case they were marked purgeable.
 			 * This prevents the BO from being purged before
@@ -207,8 +207,8 @@ vc4_save_hang_state(struct drm_device *dev)
 			 */
 			WARN_ON(!refcount_read(&bo->usecnt));
 			refcount_inc(&bo->usecnt);
-			drm_gem_object_get(&exec[i]->bo[j]->base);
-			kernel_state->bo[k++] = &exec[i]->bo[j]->base;
+			drm_gem_object_get(exec[i]->bo[j]);
+			kernel_state->bo[k++] = exec[i]->bo[j];
 		}
 
 		list_for_each_entry(bo, &exec[i]->unref_list, unref_head) {
@@ -558,7 +558,7 @@ vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno)
 	unsigned i;
 
 	for (i = 0; i < exec->bo_count; i++) {
-		bo = to_vc4_bo(&exec->bo[i]->base);
+		bo = to_vc4_bo(exec->bo[i]);
 		bo->seqno = seqno;
 
 		dma_resv_add_fence(bo->base.base.resv, exec->fence,
@@ -585,11 +585,8 @@ vc4_unlock_bo_reservations(struct drm_device *dev,
 {
 	int i;
 
-	for (i = 0; i < exec->bo_count; i++) {
-		struct drm_gem_object *bo = &exec->bo[i]->base;
-
-		dma_resv_unlock(bo->resv);
-	}
+	for (i = 0; i < exec->bo_count; i++)
+		dma_resv_unlock(exec->bo[i]->resv);
 
 	ww_acquire_fini(acquire_ctx);
 }
@@ -614,7 +611,7 @@ vc4_lock_bo_reservations(struct drm_device *dev,
 
 retry:
 	if (contended_lock != -1) {
-		bo = &exec->bo[contended_lock]->base;
+		bo = exec->bo[contended_lock];
 		ret = dma_resv_lock_slow_interruptible(bo->resv, acquire_ctx);
 		if (ret) {
 			ww_acquire_done(acquire_ctx);
@@ -626,19 +623,19 @@ retry:
 		if (i == contended_lock)
 			continue;
 
-		bo = &exec->bo[i]->base;
+		bo = exec->bo[i];
 
 		ret = dma_resv_lock_interruptible(bo->resv, acquire_ctx);
 		if (ret) {
 			int j;
 
 			for (j = 0; j < i; j++) {
-				bo = &exec->bo[j]->base;
+				bo = exec->bo[j];
 				dma_resv_unlock(bo->resv);
 			}
 
 			if (contended_lock != -1 && contended_lock >= i) {
-				bo = &exec->bo[contended_lock]->base;
+				bo = exec->bo[contended_lock];
 
 				dma_resv_unlock(bo->resv);
 			}
@@ -659,7 +656,7 @@ retry:
 	 * before we commit the CL to the hardware.
 	 */
 	for (i = 0; i < exec->bo_count; i++) {
-		bo = &exec->bo[i]->base;
+		bo = exec->bo[i];
 
 		ret = dma_resv_reserve_fences(bo->resv, 1);
 		if (ret) {
@@ -749,7 +746,6 @@ vc4_cl_lookup_bos(struct drm_device *dev,
 		  struct vc4_exec_info *exec)
 {
 	struct drm_vc4_submit_cl *args = exec->args;
-	uint32_t *handles;
 	int ret = 0;
 	int i;
 
@@ -763,54 +759,18 @@ vc4_cl_lookup_bos(struct drm_device *dev,
 		return -EINVAL;
 	}
 
-	exec->bo = kvmalloc_array(exec->bo_count,
-				    sizeof(struct drm_gem_dma_object *),
-				    GFP_KERNEL | __GFP_ZERO);
-	if (!exec->bo) {
-		DRM_ERROR("Failed to allocate validated BO pointers\n");
-		return -ENOMEM;
-	}
-
-	handles = kvmalloc_array(exec->bo_count, sizeof(uint32_t), GFP_KERNEL);
-	if (!handles) {
-		ret = -ENOMEM;
-		DRM_ERROR("Failed to allocate incoming GEM handles\n");
-		goto fail;
-	}
-
-	if (copy_from_user(handles, u64_to_user_ptr(args->bo_handles),
-			   exec->bo_count * sizeof(uint32_t))) {
-		ret = -EFAULT;
-		DRM_ERROR("Failed to copy in GEM handles\n");
-		goto fail;
-	}
-
-	spin_lock(&file_priv->table_lock);
-	for (i = 0; i < exec->bo_count; i++) {
-		struct drm_gem_object *bo = idr_find(&file_priv->object_idr,
-						     handles[i]);
-		if (!bo) {
-			DRM_DEBUG("Failed to look up GEM BO %d: %d\n",
-				  i, handles[i]);
-			ret = -EINVAL;
-			break;
-		}
-
-		drm_gem_object_get(bo);
-		exec->bo[i] = (struct drm_gem_dma_object *)bo;
-	}
-	spin_unlock(&file_priv->table_lock);
+	ret = drm_gem_objects_lookup(file_priv, u64_to_user_ptr(args->bo_handles),
+				     exec->bo_count, &exec->bo);
 
 	if (ret)
 		goto fail_put_bo;
 
 	for (i = 0; i < exec->bo_count; i++) {
-		ret = vc4_bo_inc_usecnt(to_vc4_bo(&exec->bo[i]->base));
+		ret = vc4_bo_inc_usecnt(to_vc4_bo(exec->bo[i]));
 		if (ret)
 			goto fail_dec_usecnt;
 	}
 
-	kvfree(handles);
 	return 0;
 
 fail_dec_usecnt:
@@ -823,15 +783,13 @@ fail_dec_usecnt:
 	 * step.
 	 */
 	for (i-- ; i >= 0; i--)
-		vc4_bo_dec_usecnt(to_vc4_bo(&exec->bo[i]->base));
+		vc4_bo_dec_usecnt(to_vc4_bo(exec->bo[i]));
 
 fail_put_bo:
 	/* Release any reference to acquired objects. */
 	for (i = 0; i < exec->bo_count && exec->bo[i]; i++)
-		drm_gem_object_put(&exec->bo[i]->base);
+		drm_gem_object_put(exec->bo[i]);
 
-fail:
-	kvfree(handles);
 	kvfree(exec->bo);
 	exec->bo = NULL;
 	return ret;
@@ -974,10 +932,10 @@ vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
 
 	if (exec->bo) {
 		for (i = 0; i < exec->bo_count; i++) {
-			struct vc4_bo *bo = to_vc4_bo(&exec->bo[i]->base);
+			struct vc4_bo *bo = to_vc4_bo(exec->bo[i]);
 
 			vc4_bo_dec_usecnt(bo);
-			drm_gem_object_put(&exec->bo[i]->base);
+			drm_gem_object_put(exec->bo[i]);
 		}
 		kvfree(exec->bo);
 	}
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index ea22c9bf223a..06713d8b82b5 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -885,7 +885,8 @@ static void vc4_hdmi_set_infoframes(struct drm_encoder *encoder)
 static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
-	struct drm_device *drm = vc4_hdmi->connector.dev;
+	struct drm_connector *connector = &vc4_hdmi->connector;
+	struct drm_device *drm = connector->dev;
 	const struct drm_display_mode *mode = &vc4_hdmi->saved_adjusted_mode;
 	unsigned long flags;
 	int idx;
@@ -903,8 +904,8 @@ static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder)
 	if (!drm_dev_enter(drm, &idx))
 		return;
 
-	drm_scdc_set_high_tmds_clock_ratio(vc4_hdmi->ddc, true);
-	drm_scdc_set_scrambling(vc4_hdmi->ddc, true);
+	drm_scdc_set_high_tmds_clock_ratio(connector, true);
+	drm_scdc_set_scrambling(connector, true);
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 	HDMI_WRITE(HDMI_SCRAMBLER_CTL, HDMI_READ(HDMI_SCRAMBLER_CTL) |
@@ -922,7 +923,8 @@ static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder)
 static void vc4_hdmi_disable_scrambling(struct drm_encoder *encoder)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
-	struct drm_device *drm = vc4_hdmi->connector.dev;
+	struct drm_connector *connector = &vc4_hdmi->connector;
+	struct drm_device *drm = connector->dev;
 	unsigned long flags;
 	int idx;
 
@@ -944,8 +946,8 @@ static void vc4_hdmi_disable_scrambling(struct drm_encoder *encoder)
 		   ~VC5_HDMI_SCRAMBLER_CTL_ENABLE);
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
 
-	drm_scdc_set_scrambling(vc4_hdmi->ddc, false);
-	drm_scdc_set_high_tmds_clock_ratio(vc4_hdmi->ddc, false);
+	drm_scdc_set_scrambling(connector, false);
+	drm_scdc_set_high_tmds_clock_ratio(connector, false);
 
 	drm_dev_exit(idx);
 }
@@ -955,12 +957,13 @@ static void vc4_hdmi_scrambling_wq(struct work_struct *work)
 	struct vc4_hdmi *vc4_hdmi = container_of(to_delayed_work(work),
 						 struct vc4_hdmi,
 						 scrambling_work);
+	struct drm_connector *connector = &vc4_hdmi->connector;
 
-	if (drm_scdc_get_scrambling_status(vc4_hdmi->ddc))
+	if (drm_scdc_get_scrambling_status(connector))
 		return;
 
-	drm_scdc_set_high_tmds_clock_ratio(vc4_hdmi->ddc, true);
-	drm_scdc_set_scrambling(vc4_hdmi->ddc, true);
+	drm_scdc_set_high_tmds_clock_ratio(connector, true);
+	drm_scdc_set_scrambling(connector, true);
 
 	queue_delayed_work(system_wq, &vc4_hdmi->scrambling_work,
 			   msecs_to_jiffies(SCRAMBLING_POLLING_DELAY_MS));
@@ -1466,6 +1469,12 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
 	if (!drm_dev_enter(drm, &idx))
 		goto out;
 
+	ret = pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev);
+	if (ret < 0) {
+		DRM_ERROR("Failed to retain power domain: %d\n", ret);
+		goto err_dev_exit;
+	}
+
 	/*
 	 * As stated in RPi's vc4 firmware "HDMI state machine (HSM) clock must
 	 * be faster than pixel clock, infinitesimally faster, tested in
@@ -1482,17 +1491,13 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
 	 * Additionally, the AXI clock needs to be at least 25% of
 	 * pixel clock, but HSM ends up being the limiting factor.
 	 */
-	hsm_rate = max_t(unsigned long, 120000000, (tmds_char_rate / 100) * 101);
+	hsm_rate = max_t(unsigned long,
+			 HSM_MIN_CLOCK_FREQ,
+			 (tmds_char_rate / 100) * 101);
 	ret = clk_set_min_rate(vc4_hdmi->hsm_clock, hsm_rate);
 	if (ret) {
 		DRM_ERROR("Failed to set HSM clock rate: %d\n", ret);
-		goto err_dev_exit;
-	}
-
-	ret = pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev);
-	if (ret < 0) {
-		DRM_ERROR("Failed to retain power domain: %d\n", ret);
-		goto err_dev_exit;
+		goto err_put_runtime_pm;
 	}
 
 	ret = clk_set_rate(vc4_hdmi->pixel_clock, tmds_char_rate);
@@ -3018,7 +3023,7 @@ static int vc4_hdmi_cec_init(struct vc4_hdmi *vc4_hdmi)
 	struct device *dev = &pdev->dev;
 	int ret;
 
-	if (!of_find_property(dev->of_node, "interrupts", NULL)) {
+	if (!of_property_present(dev->of_node, "interrupts")) {
 		dev_warn(dev, "'interrupts' DT property is missing, no CEC\n");
 		return 0;
 	}
@@ -3188,16 +3193,9 @@ static int vc4_hdmi_init_resources(struct drm_device *drm,
 		DRM_ERROR("Failed to get HDMI state machine clock\n");
 		return PTR_ERR(vc4_hdmi->hsm_clock);
 	}
-
 	vc4_hdmi->audio_clock = vc4_hdmi->hsm_clock;
 	vc4_hdmi->cec_clock = vc4_hdmi->hsm_clock;
 
-	vc4_hdmi->hsm_rpm_clock = devm_clk_get(dev, "hdmi");
-	if (IS_ERR(vc4_hdmi->hsm_rpm_clock)) {
-		DRM_ERROR("Failed to get HDMI state machine clock\n");
-		return PTR_ERR(vc4_hdmi->hsm_rpm_clock);
-	}
-
 	return 0;
 }
 
@@ -3280,12 +3278,6 @@ static int vc5_hdmi_init_resources(struct drm_device *drm,
 		return PTR_ERR(vc4_hdmi->hsm_clock);
 	}
 
-	vc4_hdmi->hsm_rpm_clock = devm_clk_get(dev, "hdmi");
-	if (IS_ERR(vc4_hdmi->hsm_rpm_clock)) {
-		DRM_ERROR("Failed to get HDMI state machine clock\n");
-		return PTR_ERR(vc4_hdmi->hsm_rpm_clock);
-	}
-
 	vc4_hdmi->pixel_bvb_clock = devm_clk_get(dev, "bvb");
 	if (IS_ERR(vc4_hdmi->pixel_bvb_clock)) {
 		DRM_ERROR("Failed to get pixel bvb clock\n");
@@ -3349,7 +3341,7 @@ static int vc4_hdmi_runtime_suspend(struct device *dev)
 {
 	struct vc4_hdmi *vc4_hdmi = dev_get_drvdata(dev);
 
-	clk_disable_unprepare(vc4_hdmi->hsm_rpm_clock);
+	clk_disable_unprepare(vc4_hdmi->hsm_clock);
 
 	return 0;
 }
@@ -3362,16 +3354,7 @@ static int vc4_hdmi_runtime_resume(struct device *dev)
 	unsigned long rate;
 	int ret;
 
-	/*
-	 * The HSM clock is in the HDMI power domain, so we need to set
-	 * its frequency while the power domain is active so that it
-	 * keeps its rate.
-	 */
-	ret = clk_set_min_rate(vc4_hdmi->hsm_rpm_clock, HSM_MIN_CLOCK_FREQ);
-	if (ret)
-		return ret;
-
-	ret = clk_prepare_enable(vc4_hdmi->hsm_rpm_clock);
+	ret = clk_prepare_enable(vc4_hdmi->hsm_clock);
 	if (ret)
 		return ret;
 
@@ -3384,7 +3367,7 @@ static int vc4_hdmi_runtime_resume(struct device *dev)
 	 * case, it will lead to a silent CPU stall. Let's make sure we
 	 * prevent such a case.
 	 */
-	rate = clk_get_rate(vc4_hdmi->hsm_rpm_clock);
+	rate = clk_get_rate(vc4_hdmi->hsm_clock);
 	if (!rate) {
 		ret = -EINVAL;
 		goto err_disable_clk;
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.h b/drivers/gpu/drm/vc4/vc4_hdmi.h
index dc3ccd8002a0..e3619836ca17 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.h
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.h
@@ -164,7 +164,6 @@ struct vc4_hdmi {
 	struct clk *cec_clock;
 	struct clk *pixel_clock;
 	struct clk *hsm_clock;
-	struct clk *hsm_rpm_clock;
 	struct clk *audio_clock;
 	struct clk *pixel_bvb_clock;
 
diff --git a/drivers/gpu/drm/vc4/vc4_irq.c b/drivers/gpu/drm/vc4/vc4_irq.c
index 1e6db0121ccd..563b3dfeb9b9 100644
--- a/drivers/gpu/drm/vc4/vc4_irq.c
+++ b/drivers/gpu/drm/vc4/vc4_irq.c
@@ -57,8 +57,6 @@
 			 V3D_INT_FLDONE | \
 			 V3D_INT_FRDONE)
 
-DECLARE_WAIT_QUEUE_HEAD(render_wait);
-
 static void
 vc4_overflow_mem_work(struct work_struct *work)
 {
diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c
index 520231af4df9..7dff3ca5af6b 100644
--- a/drivers/gpu/drm/vc4/vc4_validate.c
+++ b/drivers/gpu/drm/vc4/vc4_validate.c
@@ -117,7 +117,7 @@ vc4_use_bo(struct vc4_exec_info *exec, uint32_t hindex)
 			  hindex, exec->bo_count);
 		return NULL;
 	}
-	obj = exec->bo[hindex];
+	obj = to_drm_gem_dma_obj(exec->bo[hindex]);
 	bo = to_vc4_bo(&obj->base);
 
 	if (bo->validated_shader) {
@@ -810,7 +810,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 			return -EINVAL;
 		}
 
-		bo[i] = exec->bo[src_handles[i]];
+		bo[i] = to_drm_gem_dma_obj(exec->bo[src_handles[i]]);
 		if (!bo[i])
 			return -EINVAL;
 	}
diff --git a/drivers/gpu/drm/vgem/vgem_drv.h b/drivers/gpu/drm/vgem/vgem_drv.h
index 0ed300317f87..34cf63e6fb3d 100644
--- a/drivers/gpu/drm/vgem/vgem_drv.h
+++ b/drivers/gpu/drm/vgem/vgem_drv.h
@@ -39,17 +39,6 @@ struct vgem_file {
 	struct mutex fence_mutex;
 };
 
-#define to_vgem_bo(x) container_of(x, struct drm_vgem_gem_object, base)
-struct drm_vgem_gem_object {
-	struct drm_gem_object base;
-
-	struct page **pages;
-	unsigned int pages_pin_count;
-	struct mutex pages_lock;
-
-	struct sg_table *table;
-};
-
 int vgem_fence_open(struct vgem_file *file);
 int vgem_fence_attach_ioctl(struct drm_device *dev,
 			    void *data,
diff --git a/drivers/gpu/drm/vgem/vgem_fence.c b/drivers/gpu/drm/vgem/vgem_fence.c
index c2a879734d40..e15754178395 100644
--- a/drivers/gpu/drm/vgem/vgem_fence.c
+++ b/drivers/gpu/drm/vgem/vgem_fence.c
@@ -249,4 +249,5 @@ void vgem_fence_close(struct vgem_file *vfile)
 {
 	idr_for_each(&vfile->fence_idr, __vgem_fence_idr_fini, vfile);
 	idr_destroy(&vfile->fence_idr);
+	mutex_destroy(&vfile->fence_mutex);
 }
diff --git a/drivers/gpu/drm/virtio/Kconfig b/drivers/gpu/drm/virtio/Kconfig
index 51ec7c3240c9..ea06ff2aa4b4 100644
--- a/drivers/gpu/drm/virtio/Kconfig
+++ b/drivers/gpu/drm/virtio/Kconfig
@@ -11,3 +11,14 @@ config DRM_VIRTIO_GPU
 	   QEMU based VMMs (like KVM or Xen).
 
 	   If unsure say M.
+
+config DRM_VIRTIO_GPU_KMS
+	bool "Virtio GPU driver modesetting support"
+	depends on DRM_VIRTIO_GPU
+	default y
+	help
+	   Enable modesetting support for virtio GPU driver.  This can be
+	   disabled in cases where only "headless" usage of the GPU is
+	   required.
+
+	   If unsure, say Y.
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index 9ea7611a9e0f..ad924a8502e9 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -336,6 +336,9 @@ int virtio_gpu_modeset_init(struct virtio_gpu_device *vgdev)
 {
 	int i, ret;
 
+	if (!vgdev->num_scanouts)
+		return 0;
+
 	ret = drmm_mode_config_init(vgdev->ddev);
 	if (ret)
 		return ret;
@@ -362,6 +365,9 @@ void virtio_gpu_modeset_fini(struct virtio_gpu_device *vgdev)
 {
 	int i;
 
+	if (!vgdev->num_scanouts)
+		return;
+
 	for (i = 0 ; i < vgdev->num_scanouts; ++i)
 		kfree(vgdev->outputs[i].edid);
 }
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c
index ae97b98750b6..add075681e18 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.c
@@ -172,6 +172,10 @@ MODULE_AUTHOR("Alon Levy");
 DEFINE_DRM_GEM_FOPS(virtio_gpu_driver_fops);
 
 static const struct drm_driver driver = {
+	/*
+	 * If KMS is disabled DRIVER_MODESET and DRIVER_ATOMIC are masked
+	 * out via drm_device::driver_features:
+	 */
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_RENDER | DRIVER_ATOMIC,
 	.open = virtio_gpu_driver_open,
 	.postclose = virtio_gpu_driver_postclose,
diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c
index 27b7f14dae89..5a3b5aaed1f3 100644
--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
+++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
@@ -43,11 +43,13 @@ static void virtio_gpu_config_changed_work_func(struct work_struct *work)
 	virtio_cread_le(vgdev->vdev, struct virtio_gpu_config,
 			events_read, &events_read);
 	if (events_read & VIRTIO_GPU_EVENT_DISPLAY) {
-		if (vgdev->has_edid)
-			virtio_gpu_cmd_get_edids(vgdev);
-		virtio_gpu_cmd_get_display_info(vgdev);
-		virtio_gpu_notify(vgdev);
-		drm_helper_hpd_irq_event(vgdev->ddev);
+		if (vgdev->num_scanouts) {
+			if (vgdev->has_edid)
+				virtio_gpu_cmd_get_edids(vgdev);
+			virtio_gpu_cmd_get_display_info(vgdev);
+			virtio_gpu_notify(vgdev);
+			drm_helper_hpd_irq_event(vgdev->ddev);
+		}
 		events_clear |= VIRTIO_GPU_EVENT_DISPLAY;
 	}
 	virtio_cwrite_le(vgdev->vdev, struct virtio_gpu_config,
@@ -223,12 +225,15 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
 			num_scanouts, &num_scanouts);
 	vgdev->num_scanouts = min_t(uint32_t, num_scanouts,
 				    VIRTIO_GPU_MAX_SCANOUTS);
-	if (!vgdev->num_scanouts) {
-		DRM_ERROR("num_scanouts is zero\n");
-		ret = -EINVAL;
-		goto err_scanouts;
+
+	if (!IS_ENABLED(CONFIG_DRM_VIRTIO_GPU_KMS) || !vgdev->num_scanouts) {
+		DRM_INFO("KMS disabled\n");
+		vgdev->num_scanouts = 0;
+		vgdev->has_edid = false;
+		dev->driver_features &= ~(DRIVER_MODESET | DRIVER_ATOMIC);
+	} else {
+		DRM_INFO("number of scanouts: %d\n", num_scanouts);
 	}
-	DRM_INFO("number of scanouts: %d\n", num_scanouts);
 
 	virtio_cread_le(vgdev->vdev, struct virtio_gpu_config,
 			num_capsets, &num_capsets);
@@ -244,12 +249,14 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
 
 	if (num_capsets)
 		virtio_gpu_get_capsets(vgdev, num_capsets);
-	if (vgdev->has_edid)
-		virtio_gpu_cmd_get_edids(vgdev);
-	virtio_gpu_cmd_get_display_info(vgdev);
-	virtio_gpu_notify(vgdev);
-	wait_event_timeout(vgdev->resp_wq, !vgdev->display_info_pending,
-			   5 * HZ);
+	if (vgdev->num_scanouts) {
+		if (vgdev->has_edid)
+			virtio_gpu_cmd_get_edids(vgdev);
+		virtio_gpu_cmd_get_display_info(vgdev);
+		virtio_gpu_notify(vgdev);
+		wait_event_timeout(vgdev->resp_wq, !vgdev->display_info_pending,
+				   5 * HZ);
+	}
 	return 0;
 
 err_scanouts:
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index 4c09e313bebc..a2e045f3a000 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -390,5 +390,9 @@ struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev,
 		return plane;
 
 	drm_plane_helper_add(plane, funcs);
+
+	if (type == DRM_PLANE_TYPE_PRIMARY)
+		drm_plane_enable_fb_damage_clips(plane);
+
 	return plane;
 }
diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c
index 1778a2081fd6..b1a00c0c25a7 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vq.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
@@ -923,8 +923,7 @@ void virtio_gpu_cmd_context_create(struct virtio_gpu_device *vgdev, uint32_t id,
 	cmd_p->hdr.ctx_id = cpu_to_le32(id);
 	cmd_p->nlen = cpu_to_le32(nlen);
 	cmd_p->context_init = cpu_to_le32(context_init);
-	strncpy(cmd_p->debug_name, name, sizeof(cmd_p->debug_name) - 1);
-	cmd_p->debug_name[sizeof(cmd_p->debug_name) - 1] = 0;
+	strscpy(cmd_p->debug_name, name, sizeof(cmd_p->debug_name));
 	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
 }
 
diff --git a/drivers/gpu/drm/vkms/vkms_output.c b/drivers/gpu/drm/vkms/vkms_output.c
index 991857125bb4..5ce70dd946aa 100644
--- a/drivers/gpu/drm/vkms/vkms_output.c
+++ b/drivers/gpu/drm/vkms/vkms_output.c
@@ -4,21 +4,19 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
-
-static void vkms_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_cleanup(connector);
-}
 
 static const struct drm_connector_funcs vkms_connector_funcs = {
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = vkms_connector_destroy,
+	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
+static const struct drm_encoder_funcs vkms_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static int vkms_conn_get_modes(struct drm_connector *connector)
 {
 	int count;
@@ -91,7 +89,8 @@ int vkms_output_init(struct vkms_device *vkmsdev, int index)
 
 	drm_connector_helper_add(connector, &vkms_conn_helper_funcs);
 
-	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_VIRTUAL);
+	ret = drm_encoder_init(dev, encoder, &vkms_encoder_funcs,
+			       DRM_MODE_ENCODER_VIRTUAL, NULL);
 	if (ret) {
 		DRM_ERROR("Failed to init encoder\n");
 		goto err_encoder;
diff --git a/drivers/gpu/drm/vkms/vkms_plane.c b/drivers/gpu/drm/vkms/vkms_plane.c
index b3f8a115cc23..c41cec7dcb70 100644
--- a/drivers/gpu/drm/vkms/vkms_plane.c
+++ b/drivers/gpu/drm/vkms/vkms_plane.c
@@ -132,7 +132,6 @@ static int vkms_plane_atomic_check(struct drm_plane *plane,
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
 	struct drm_crtc_state *crtc_state;
-	bool can_position = false;
 	int ret;
 
 	if (!new_plane_state->fb || WARN_ON(!new_plane_state->crtc))
@@ -143,20 +142,13 @@ static int vkms_plane_atomic_check(struct drm_plane *plane,
 	if (IS_ERR(crtc_state))
 		return PTR_ERR(crtc_state);
 
-	if (plane->type != DRM_PLANE_TYPE_PRIMARY)
-		can_position = true;
-
 	ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state,
 						  DRM_PLANE_NO_SCALING,
 						  DRM_PLANE_NO_SCALING,
-						  can_position, true);
+						  true, true);
 	if (ret != 0)
 		return ret;
 
-	/* for now primary plane must be visible and full screen */
-	if (!new_plane_state->visible && !can_position)
-		return -EINVAL;
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/vmwgfx/Makefile b/drivers/gpu/drm/vmwgfx/Makefile
index 2a644f035597..e94479d9cd5b 100644
--- a/drivers/gpu/drm/vmwgfx/Makefile
+++ b/drivers/gpu/drm/vmwgfx/Makefile
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \
 	    vmwgfx_ioctl.o vmwgfx_resource.o vmwgfx_ttm_buffer.o \
-	    vmwgfx_cmd.o vmwgfx_irq.o vmwgfx_ldu.o vmwgfx_ttm_glue.o \
+	    vmwgfx_cmd.o vmwgfx_irq.o vmwgfx_ldu.o \
 	    vmwgfx_overlay.o vmwgfx_gmrid_manager.o vmwgfx_fence.o \
 	    vmwgfx_bo.o vmwgfx_scrn.o vmwgfx_context.o \
 	    vmwgfx_surface.o vmwgfx_prime.o vmwgfx_mob.o vmwgfx_shader.o \
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
index 4dcf2eb7aa80..82094c137855 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright © 2011-2018 VMware, Inc., Palo Alto, CA., USA
+ * Copyright © 2011-2023 VMware, Inc., Palo Alto, CA., USA
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -26,40 +26,31 @@
  *
  **************************************************************************/
 
-#include <drm/ttm/ttm_placement.h>
-
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
-#include "ttm_object.h"
 
 
-/**
- * vmw_buffer_object - Convert a struct ttm_buffer_object to a struct
- * vmw_buffer_object.
- *
- * @bo: Pointer to the TTM buffer object.
- * Return: Pointer to the struct vmw_buffer_object embedding the
- * TTM buffer object.
- */
-static struct vmw_buffer_object *
-vmw_buffer_object(struct ttm_buffer_object *bo)
+#include <drm/ttm/ttm_placement.h>
+
+static void vmw_bo_release(struct vmw_bo *vbo)
 {
-	return container_of(bo, struct vmw_buffer_object, base);
+	vmw_bo_unmap(vbo);
+	drm_gem_object_release(&vbo->tbo.base);
 }
 
 /**
- * bo_is_vmw - check if the buffer object is a &vmw_buffer_object
- * @bo: ttm buffer object to be checked
+ * vmw_bo_free - vmw_bo destructor
  *
- * Uses destroy function associated with the object to determine if this is
- * a &vmw_buffer_object.
- *
- * Returns:
- * true if the object is of &vmw_buffer_object type, false if not.
+ * @bo: Pointer to the embedded struct ttm_buffer_object
  */
-static bool bo_is_vmw(struct ttm_buffer_object *bo)
+static void vmw_bo_free(struct ttm_buffer_object *bo)
 {
-	return bo->destroy == &vmw_bo_bo_free ||
-	       bo->destroy == &vmw_gem_destroy;
+	struct vmw_bo *vbo = to_vmw_bo(&bo->base);
+
+	WARN_ON(vbo->dirty);
+	WARN_ON(!RB_EMPTY_ROOT(&vbo->res_tree));
+	vmw_bo_release(vbo);
+	kfree(vbo);
 }
 
 /**
@@ -72,13 +63,13 @@ static bool bo_is_vmw(struct ttm_buffer_object *bo)
  * Return: Zero on success, Negative error code on failure. In particular
  * -ERESTARTSYS if interrupted by a signal
  */
-int vmw_bo_pin_in_placement(struct vmw_private *dev_priv,
-			    struct vmw_buffer_object *buf,
-			    struct ttm_placement *placement,
-			    bool interruptible)
+static int vmw_bo_pin_in_placement(struct vmw_private *dev_priv,
+				   struct vmw_bo *buf,
+				   struct ttm_placement *placement,
+				   bool interruptible)
 {
 	struct ttm_operation_ctx ctx = {interruptible, false };
-	struct ttm_buffer_object *bo = &buf->base;
+	struct ttm_buffer_object *bo = &buf->tbo;
 	int ret;
 
 	vmw_execbuf_release_pinned_bo(dev_priv);
@@ -87,12 +78,7 @@ int vmw_bo_pin_in_placement(struct vmw_private *dev_priv,
 	if (unlikely(ret != 0))
 		goto err;
 
-	if (buf->base.pin_count > 0)
-		ret = ttm_resource_compat(bo->resource, placement)
-			? 0 : -EINVAL;
-	else
-		ret = ttm_bo_validate(bo, placement, &ctx);
-
+	ret = ttm_bo_validate(bo, placement, &ctx);
 	if (!ret)
 		vmw_bo_pin_reserved(buf, true);
 
@@ -115,11 +101,11 @@ err:
  * -ERESTARTSYS if interrupted by a signal
  */
 int vmw_bo_pin_in_vram_or_gmr(struct vmw_private *dev_priv,
-			      struct vmw_buffer_object *buf,
+			      struct vmw_bo *buf,
 			      bool interruptible)
 {
 	struct ttm_operation_ctx ctx = {interruptible, false };
-	struct ttm_buffer_object *bo = &buf->base;
+	struct ttm_buffer_object *bo = &buf->tbo;
 	int ret;
 
 	vmw_execbuf_release_pinned_bo(dev_priv);
@@ -128,17 +114,17 @@ int vmw_bo_pin_in_vram_or_gmr(struct vmw_private *dev_priv,
 	if (unlikely(ret != 0))
 		goto err;
 
-	if (buf->base.pin_count > 0) {
-		ret = ttm_resource_compat(bo->resource, &vmw_vram_gmr_placement)
-			? 0 : -EINVAL;
-		goto out_unreserve;
-	}
-
-	ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, &ctx);
+	vmw_bo_placement_set(buf,
+			     VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM,
+			     VMW_BO_DOMAIN_GMR);
+	ret = ttm_bo_validate(bo, &buf->placement, &ctx);
 	if (likely(ret == 0) || ret == -ERESTARTSYS)
 		goto out_unreserve;
 
-	ret = ttm_bo_validate(bo, &vmw_vram_placement, &ctx);
+	vmw_bo_placement_set(buf,
+			     VMW_BO_DOMAIN_VRAM,
+			     VMW_BO_DOMAIN_VRAM);
+	ret = ttm_bo_validate(bo, &buf->placement, &ctx);
 
 out_unreserve:
 	if (!ret)
@@ -163,7 +149,7 @@ err:
  * -ERESTARTSYS if interrupted by a signal
  */
 int vmw_bo_pin_in_vram(struct vmw_private *dev_priv,
-		       struct vmw_buffer_object *buf,
+		       struct vmw_bo *buf,
 		       bool interruptible)
 {
 	return vmw_bo_pin_in_placement(dev_priv, buf, &vmw_vram_placement,
@@ -184,22 +170,13 @@ int vmw_bo_pin_in_vram(struct vmw_private *dev_priv,
  * -ERESTARTSYS if interrupted by a signal
  */
 int vmw_bo_pin_in_start_of_vram(struct vmw_private *dev_priv,
-				struct vmw_buffer_object *buf,
+				struct vmw_bo *buf,
 				bool interruptible)
 {
 	struct ttm_operation_ctx ctx = {interruptible, false };
-	struct ttm_buffer_object *bo = &buf->base;
-	struct ttm_placement placement;
-	struct ttm_place place;
+	struct ttm_buffer_object *bo = &buf->tbo;
 	int ret = 0;
 
-	place = vmw_vram_placement.placement[0];
-	place.lpfn = PFN_UP(bo->resource->size);
-	placement.num_placement = 1;
-	placement.placement = &place;
-	placement.num_busy_placement = 1;
-	placement.busy_placement = &place;
-
 	vmw_execbuf_release_pinned_bo(dev_priv);
 	ret = ttm_bo_reserve(bo, interruptible, false, NULL);
 	if (unlikely(ret != 0))
@@ -213,16 +190,19 @@ int vmw_bo_pin_in_start_of_vram(struct vmw_private *dev_priv,
 	if (bo->resource->mem_type == TTM_PL_VRAM &&
 	    bo->resource->start < PFN_UP(bo->resource->size) &&
 	    bo->resource->start > 0 &&
-	    buf->base.pin_count == 0) {
+	    buf->tbo.pin_count == 0) {
 		ctx.interruptible = false;
-		(void) ttm_bo_validate(bo, &vmw_sys_placement, &ctx);
+		vmw_bo_placement_set(buf,
+				     VMW_BO_DOMAIN_SYS,
+				     VMW_BO_DOMAIN_SYS);
+		(void)ttm_bo_validate(bo, &buf->placement, &ctx);
 	}
 
-	if (buf->base.pin_count > 0)
-		ret = ttm_resource_compat(bo->resource, &placement)
-			? 0 : -EINVAL;
-	else
-		ret = ttm_bo_validate(bo, &placement, &ctx);
+	vmw_bo_placement_set(buf,
+			     VMW_BO_DOMAIN_VRAM,
+			     VMW_BO_DOMAIN_VRAM);
+	buf->places[0].lpfn = PFN_UP(bo->resource->size);
+	ret = ttm_bo_validate(bo, &buf->placement, &ctx);
 
 	/* For some reason we didn't end up at the start of vram */
 	WARN_ON(ret == 0 && bo->resource->start != 0);
@@ -248,10 +228,10 @@ err_unlock:
  * -ERESTARTSYS if interrupted by a signal
  */
 int vmw_bo_unpin(struct vmw_private *dev_priv,
-		 struct vmw_buffer_object *buf,
+		 struct vmw_bo *buf,
 		 bool interruptible)
 {
-	struct ttm_buffer_object *bo = &buf->base;
+	struct ttm_buffer_object *bo = &buf->tbo;
 	int ret;
 
 	ret = ttm_bo_reserve(bo, interruptible, false, NULL);
@@ -293,12 +273,12 @@ void vmw_bo_get_guest_ptr(const struct ttm_buffer_object *bo,
  * @pin: Whether to pin or unpin.
  *
  */
-void vmw_bo_pin_reserved(struct vmw_buffer_object *vbo, bool pin)
+void vmw_bo_pin_reserved(struct vmw_bo *vbo, bool pin)
 {
 	struct ttm_operation_ctx ctx = { false, true };
 	struct ttm_place pl;
 	struct ttm_placement placement;
-	struct ttm_buffer_object *bo = &vbo->base;
+	struct ttm_buffer_object *bo = &vbo->tbo;
 	uint32_t old_mem_type = bo->resource->mem_type;
 	int ret;
 
@@ -341,9 +321,9 @@ void vmw_bo_pin_reserved(struct vmw_buffer_object *vbo, bool pin)
  * 3) Buffer object destruction
  *
  */
-void *vmw_bo_map_and_cache(struct vmw_buffer_object *vbo)
+void *vmw_bo_map_and_cache(struct vmw_bo *vbo)
 {
-	struct ttm_buffer_object *bo = &vbo->base;
+	struct ttm_buffer_object *bo = &vbo->tbo;
 	bool not_used;
 	void *virtual;
 	int ret;
@@ -366,96 +346,70 @@ void *vmw_bo_map_and_cache(struct vmw_buffer_object *vbo)
  * @vbo: The buffer object whose map we are tearing down.
  *
  * This function tears down a cached map set up using
- * vmw_buffer_object_map_and_cache().
+ * vmw_bo_map_and_cache().
  */
-void vmw_bo_unmap(struct vmw_buffer_object *vbo)
+void vmw_bo_unmap(struct vmw_bo *vbo)
 {
 	if (vbo->map.bo == NULL)
 		return;
 
 	ttm_bo_kunmap(&vbo->map);
+	vbo->map.bo = NULL;
 }
 
 
 /**
- * vmw_bo_bo_free - vmw buffer object destructor
- *
- * @bo: Pointer to the embedded struct ttm_buffer_object
- */
-void vmw_bo_bo_free(struct ttm_buffer_object *bo)
-{
-	struct vmw_buffer_object *vmw_bo = vmw_buffer_object(bo);
-
-	WARN_ON(vmw_bo->dirty);
-	WARN_ON(!RB_EMPTY_ROOT(&vmw_bo->res_tree));
-	vmw_bo_unmap(vmw_bo);
-	drm_gem_object_release(&bo->base);
-	kfree(vmw_bo);
-}
-
-/* default destructor */
-static void vmw_bo_default_destroy(struct ttm_buffer_object *bo)
-{
-	kfree(bo);
-}
-
-/**
- * vmw_bo_create_kernel - Create a pinned BO for internal kernel use.
+ * vmw_bo_init - Initialize a vmw buffer object
  *
  * @dev_priv: Pointer to the device private struct
- * @size: size of the BO we need
- * @placement: where to put it
- * @p_bo: resulting BO
+ * @vmw_bo: Buffer object to initialize
+ * @params: Parameters used to initialize the buffer object
+ * @destroy: The function used to delete the buffer object
+ * Returns: Zero on success, negative error code on error.
  *
- * Creates and pin a simple BO for in kernel use.
  */
-int vmw_bo_create_kernel(struct vmw_private *dev_priv, unsigned long size,
-			 struct ttm_placement *placement,
-			 struct ttm_buffer_object **p_bo)
+static int vmw_bo_init(struct vmw_private *dev_priv,
+		       struct vmw_bo *vmw_bo,
+		       struct vmw_bo_params *params,
+		       void (*destroy)(struct ttm_buffer_object *))
 {
 	struct ttm_operation_ctx ctx = {
-		.interruptible = false,
+		.interruptible = params->bo_type != ttm_bo_type_kernel,
 		.no_wait_gpu = false
 	};
-	struct ttm_buffer_object *bo;
+	struct ttm_device *bdev = &dev_priv->bdev;
 	struct drm_device *vdev = &dev_priv->drm;
 	int ret;
 
-	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
-	if (unlikely(!bo))
-		return -ENOMEM;
+	memset(vmw_bo, 0, sizeof(*vmw_bo));
 
-	size = ALIGN(size, PAGE_SIZE);
+	BUILD_BUG_ON(TTM_MAX_BO_PRIORITY <= 3);
+	vmw_bo->tbo.priority = 3;
+	vmw_bo->res_tree = RB_ROOT;
 
-	drm_gem_private_object_init(vdev, &bo->base, size);
+	params->size = ALIGN(params->size, PAGE_SIZE);
+	drm_gem_private_object_init(vdev, &vmw_bo->tbo.base, params->size);
 
-	ret = ttm_bo_init_reserved(&dev_priv->bdev, bo, ttm_bo_type_kernel,
-				   placement, 0, &ctx, NULL, NULL,
-				   vmw_bo_default_destroy);
+	vmw_bo_placement_set(vmw_bo, params->domain, params->busy_domain);
+	ret = ttm_bo_init_reserved(bdev, &vmw_bo->tbo, params->bo_type,
+				   &vmw_bo->placement, 0, &ctx, NULL,
+				   NULL, destroy);
 	if (unlikely(ret))
-		goto error_free;
+		return ret;
 
-	ttm_bo_pin(bo);
-	ttm_bo_unreserve(bo);
-	*p_bo = bo;
+	if (params->pin)
+		ttm_bo_pin(&vmw_bo->tbo);
+	ttm_bo_unreserve(&vmw_bo->tbo);
 
 	return 0;
-
-error_free:
-	kfree(bo);
-	return ret;
 }
 
 int vmw_bo_create(struct vmw_private *vmw,
-		  size_t size, struct ttm_placement *placement,
-		  bool interruptible, bool pin,
-		  void (*bo_free)(struct ttm_buffer_object *bo),
-		  struct vmw_buffer_object **p_bo)
+		  struct vmw_bo_params *params,
+		  struct vmw_bo **p_bo)
 {
 	int ret;
 
-	BUG_ON(!bo_free);
-
 	*p_bo = kmalloc(sizeof(**p_bo), GFP_KERNEL);
 	if (unlikely(!*p_bo)) {
 		DRM_ERROR("Failed to allocate a buffer.\n");
@@ -465,9 +419,7 @@ int vmw_bo_create(struct vmw_private *vmw,
 	/*
 	 * vmw_bo_init will delete the *p_bo object if it fails
 	 */
-	ret = vmw_bo_init(vmw, *p_bo, size,
-			  placement, interruptible, pin,
-			  bo_free);
+	ret = vmw_bo_init(vmw, *p_bo, params, vmw_bo_free);
 	if (unlikely(ret != 0))
 		goto out_error;
 
@@ -478,57 +430,7 @@ out_error:
 }
 
 /**
- * vmw_bo_init - Initialize a vmw buffer object
- *
- * @dev_priv: Pointer to the device private struct
- * @vmw_bo: Pointer to the struct vmw_buffer_object to initialize.
- * @size: Buffer object size in bytes.
- * @placement: Initial placement.
- * @interruptible: Whether waits should be performed interruptible.
- * @pin: If the BO should be created pinned at a fixed location.
- * @bo_free: The buffer object destructor.
- * Returns: Zero on success, negative error code on error.
- *
- * Note that on error, the code will free the buffer object.
- */
-int vmw_bo_init(struct vmw_private *dev_priv,
-		struct vmw_buffer_object *vmw_bo,
-		size_t size, struct ttm_placement *placement,
-		bool interruptible, bool pin,
-		void (*bo_free)(struct ttm_buffer_object *bo))
-{
-	struct ttm_operation_ctx ctx = {
-		.interruptible = interruptible,
-		.no_wait_gpu = false
-	};
-	struct ttm_device *bdev = &dev_priv->bdev;
-	struct drm_device *vdev = &dev_priv->drm;
-	int ret;
-
-	WARN_ON_ONCE(!bo_free);
-	memset(vmw_bo, 0, sizeof(*vmw_bo));
-	BUILD_BUG_ON(TTM_MAX_BO_PRIORITY <= 3);
-	vmw_bo->base.priority = 3;
-	vmw_bo->res_tree = RB_ROOT;
-
-	size = ALIGN(size, PAGE_SIZE);
-	drm_gem_private_object_init(vdev, &vmw_bo->base.base, size);
-
-	ret = ttm_bo_init_reserved(bdev, &vmw_bo->base, ttm_bo_type_device,
-				   placement, 0, &ctx, NULL, NULL, bo_free);
-	if (unlikely(ret)) {
-		return ret;
-	}
-
-	if (pin)
-		ttm_bo_pin(&vmw_bo->base);
-	ttm_bo_unreserve(&vmw_bo->base);
-
-	return 0;
-}
-
-/**
- * vmw_user_bo_synccpu_grab - Grab a struct vmw_buffer_object for cpu
+ * vmw_user_bo_synccpu_grab - Grab a struct vmw_bo for cpu
  * access, idling previous GPU operations on the buffer and optionally
  * blocking it for further command submissions.
  *
@@ -541,11 +443,11 @@ int vmw_bo_init(struct vmw_private *dev_priv,
  *
  * A blocking grab will be automatically released when @tfile is closed.
  */
-static int vmw_user_bo_synccpu_grab(struct vmw_buffer_object *vmw_bo,
+static int vmw_user_bo_synccpu_grab(struct vmw_bo *vmw_bo,
 				    uint32_t flags)
 {
 	bool nonblock = !!(flags & drm_vmw_synccpu_dontblock);
-	struct ttm_buffer_object *bo = &vmw_bo->base;
+	struct ttm_buffer_object *bo = &vmw_bo->tbo;
 	int ret;
 
 	if (flags & drm_vmw_synccpu_allow_cs) {
@@ -588,17 +490,17 @@ static int vmw_user_bo_synccpu_release(struct drm_file *filp,
 				       uint32_t handle,
 				       uint32_t flags)
 {
-	struct vmw_buffer_object *vmw_bo;
+	struct vmw_bo *vmw_bo;
 	int ret = vmw_user_bo_lookup(filp, handle, &vmw_bo);
 
 	if (!ret) {
 		if (!(flags & drm_vmw_synccpu_allow_cs)) {
 			atomic_dec(&vmw_bo->cpu_writers);
 		}
-		ttm_bo_put(&vmw_bo->base);
+		ttm_bo_put(&vmw_bo->tbo);
 	}
 
-	drm_gem_object_put(&vmw_bo->base.base);
+	drm_gem_object_put(&vmw_bo->tbo.base);
 	return ret;
 }
 
@@ -620,7 +522,7 @@ int vmw_user_bo_synccpu_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_vmw_synccpu_arg *arg =
 		(struct drm_vmw_synccpu_arg *) data;
-	struct vmw_buffer_object *vbo;
+	struct vmw_bo *vbo;
 	int ret;
 
 	if ((arg->flags & (drm_vmw_synccpu_read | drm_vmw_synccpu_write)) == 0
@@ -639,7 +541,7 @@ int vmw_user_bo_synccpu_ioctl(struct drm_device *dev, void *data,
 
 		ret = vmw_user_bo_synccpu_grab(vbo, arg->flags);
 		vmw_bo_unreference(&vbo);
-		drm_gem_object_put(&vbo->base.base);
+		drm_gem_object_put(&vbo->tbo.base);
 		if (unlikely(ret != 0)) {
 			if (ret == -ERESTARTSYS || ret == -EBUSY)
 				return -EBUSY;
@@ -683,8 +585,7 @@ int vmw_bo_unref_ioctl(struct drm_device *dev, void *data,
 	struct drm_vmw_unref_dmabuf_arg *arg =
 	    (struct drm_vmw_unref_dmabuf_arg *)data;
 
-	drm_gem_handle_delete(file_priv, arg->handle);
-	return 0;
+	return drm_gem_handle_delete(file_priv, arg->handle);
 }
 
 
@@ -694,14 +595,14 @@ int vmw_bo_unref_ioctl(struct drm_device *dev, void *data,
  * @filp: The file the handle is registered with.
  * @handle: The user buffer object handle
  * @out: Pointer to a where a pointer to the embedded
- * struct vmw_buffer_object should be placed.
+ * struct vmw_bo should be placed.
  * Return: Zero on success, Negative error code on error.
  *
  * The vmw buffer object pointer will be refcounted (both ttm and gem)
  */
 int vmw_user_bo_lookup(struct drm_file *filp,
-		       uint32_t handle,
-		       struct vmw_buffer_object **out)
+		       u32 handle,
+		       struct vmw_bo **out)
 {
 	struct drm_gem_object *gobj;
 
@@ -712,8 +613,8 @@ int vmw_user_bo_lookup(struct drm_file *filp,
 		return -ESRCH;
 	}
 
-	*out = gem_to_vmw_bo(gobj);
-	ttm_bo_get(&(*out)->base);
+	*out = to_vmw_bo(gobj);
+	ttm_bo_get(&(*out)->tbo);
 
 	return 0;
 }
@@ -734,8 +635,7 @@ void vmw_bo_fence_single(struct ttm_buffer_object *bo,
 			 struct vmw_fence_obj *fence)
 {
 	struct ttm_device *bdev = bo->bdev;
-	struct vmw_private *dev_priv =
-		container_of(bdev, struct vmw_private, bdev);
+	struct vmw_private *dev_priv = vmw_priv_from_ttm(bdev);
 	int ret;
 
 	if (fence == NULL)
@@ -771,7 +671,7 @@ int vmw_dumb_create(struct drm_file *file_priv,
 		    struct drm_mode_create_dumb *args)
 {
 	struct vmw_private *dev_priv = vmw_priv(dev);
-	struct vmw_buffer_object *vbo;
+	struct vmw_bo *vbo;
 	int cpp = DIV_ROUND_UP(args->bpp, 8);
 	int ret;
 
@@ -795,7 +695,7 @@ int vmw_dumb_create(struct drm_file *file_priv,
 						args->size, &args->handle,
 						&vbo);
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_put(&vbo->base.base);
+	drm_gem_object_put(&vbo->tbo.base);
 	return ret;
 }
 
@@ -806,12 +706,8 @@ int vmw_dumb_create(struct drm_file *file_priv,
  */
 void vmw_bo_swap_notify(struct ttm_buffer_object *bo)
 {
-	/* Is @bo embedded in a struct vmw_buffer_object? */
-	if (!bo_is_vmw(bo))
-		return;
-
 	/* Kill any cached kernel maps before swapout */
-	vmw_bo_unmap(vmw_buffer_object(bo));
+	vmw_bo_unmap(to_vmw_bo(&bo->base));
 }
 
 
@@ -828,13 +724,7 @@ void vmw_bo_swap_notify(struct ttm_buffer_object *bo)
 void vmw_bo_move_notify(struct ttm_buffer_object *bo,
 			struct ttm_resource *mem)
 {
-	struct vmw_buffer_object *vbo;
-
-	/* Make sure @bo is embedded in a struct vmw_buffer_object? */
-	if (!bo_is_vmw(bo))
-		return;
-
-	vbo = container_of(bo, struct vmw_buffer_object, base);
+	struct vmw_bo *vbo = to_vmw_bo(&bo->base);
 
 	/*
 	 * Kill any cached kernel maps before move to or from VRAM.
@@ -852,3 +742,98 @@ void vmw_bo_move_notify(struct ttm_buffer_object *bo,
 	if (mem->mem_type != VMW_PL_MOB && bo->resource->mem_type == VMW_PL_MOB)
 		vmw_resource_unbind_list(vbo);
 }
+
+static u32
+set_placement_list(struct ttm_place *pl, u32 domain)
+{
+	u32 n = 0;
+
+	/*
+	 * The placements are ordered according to our preferences
+	 */
+	if (domain & VMW_BO_DOMAIN_MOB) {
+		pl[n].mem_type = VMW_PL_MOB;
+		pl[n].flags = 0;
+		pl[n].fpfn = 0;
+		pl[n].lpfn = 0;
+		n++;
+	}
+	if (domain & VMW_BO_DOMAIN_GMR) {
+		pl[n].mem_type = VMW_PL_GMR;
+		pl[n].flags = 0;
+		pl[n].fpfn = 0;
+		pl[n].lpfn = 0;
+		n++;
+	}
+	if (domain & VMW_BO_DOMAIN_VRAM) {
+		pl[n].mem_type = TTM_PL_VRAM;
+		pl[n].flags = 0;
+		pl[n].fpfn = 0;
+		pl[n].lpfn = 0;
+		n++;
+	}
+	if (domain & VMW_BO_DOMAIN_WAITABLE_SYS) {
+		pl[n].mem_type = VMW_PL_SYSTEM;
+		pl[n].flags = 0;
+		pl[n].fpfn = 0;
+		pl[n].lpfn = 0;
+		n++;
+	}
+	if (domain & VMW_BO_DOMAIN_SYS) {
+		pl[n].mem_type = TTM_PL_SYSTEM;
+		pl[n].flags = 0;
+		pl[n].fpfn = 0;
+		pl[n].lpfn = 0;
+		n++;
+	}
+
+	WARN_ON(!n);
+	if (!n) {
+		pl[n].mem_type = TTM_PL_SYSTEM;
+		pl[n].flags = 0;
+		pl[n].fpfn = 0;
+		pl[n].lpfn = 0;
+		n++;
+	}
+	return n;
+}
+
+void vmw_bo_placement_set(struct vmw_bo *bo, u32 domain, u32 busy_domain)
+{
+	struct ttm_device *bdev = bo->tbo.bdev;
+	struct vmw_private *vmw = vmw_priv_from_ttm(bdev);
+	struct ttm_placement *pl = &bo->placement;
+	bool mem_compatible = false;
+	u32 i;
+
+	pl->placement = bo->places;
+	pl->num_placement = set_placement_list(bo->places, domain);
+
+	if (drm_debug_enabled(DRM_UT_DRIVER) && bo->tbo.resource) {
+		for (i = 0; i < pl->num_placement; ++i) {
+			if (bo->tbo.resource->mem_type == TTM_PL_SYSTEM ||
+			    bo->tbo.resource->mem_type == pl->placement[i].mem_type)
+				mem_compatible = true;
+		}
+		if (!mem_compatible)
+			drm_warn(&vmw->drm,
+				 "%s: Incompatible transition from "
+				 "bo->base.resource->mem_type = %u to domain = %u\n",
+				 __func__, bo->tbo.resource->mem_type, domain);
+	}
+
+	pl->busy_placement = bo->busy_places;
+	pl->num_busy_placement = set_placement_list(bo->busy_places, busy_domain);
+}
+
+void vmw_bo_placement_set_default_accelerated(struct vmw_bo *bo)
+{
+	struct ttm_device *bdev = bo->tbo.bdev;
+	struct vmw_private *vmw = vmw_priv_from_ttm(bdev);
+	u32 domain = VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM;
+
+	if (vmw->has_mob)
+		domain = VMW_BO_DOMAIN_MOB;
+
+	vmw_bo_placement_set(bo, domain, domain);
+}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.h b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.h
new file mode 100644
index 000000000000..50a836e70994
--- /dev/null
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.h
@@ -0,0 +1,203 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/**************************************************************************
+ *
+ * Copyright 2023 VMware, Inc., Palo Alto, CA., USA
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+#ifndef VMWGFX_BO_H
+#define VMWGFX_BO_H
+
+#include "device_include/svga_reg.h"
+
+#include <drm/ttm/ttm_bo.h>
+#include <drm/ttm/ttm_placement.h>
+
+#include <linux/rbtree_types.h>
+#include <linux/types.h>
+
+struct vmw_bo_dirty;
+struct vmw_fence_obj;
+struct vmw_private;
+struct vmw_resource;
+
+enum vmw_bo_domain {
+	VMW_BO_DOMAIN_SYS           = BIT(0),
+	VMW_BO_DOMAIN_WAITABLE_SYS  = BIT(1),
+	VMW_BO_DOMAIN_VRAM          = BIT(2),
+	VMW_BO_DOMAIN_GMR           = BIT(3),
+	VMW_BO_DOMAIN_MOB           = BIT(4),
+};
+
+struct vmw_bo_params {
+	u32 domain;
+	u32 busy_domain;
+	enum ttm_bo_type bo_type;
+	size_t size;
+	bool pin;
+};
+
+/**
+ * struct vmw_bo - TTM buffer object with vmwgfx additions
+ * @tbo: The TTM buffer object
+ * @placement: The preferred placement for this buffer object
+ * @places: The chosen places for the preferred placement.
+ * @busy_places: Chosen busy places for the preferred placement
+ * @map: Kmap object for semi-persistent mappings
+ * @res_tree: RB tree of resources using this buffer object as a backing MOB
+ * @res_prios: Eviction priority counts for attached resources
+ * @cpu_writers: Number of synccpu write grabs. Protected by reservation when
+ * increased. May be decreased without reservation.
+ * @dx_query_ctx: DX context if this buffer object is used as a DX query MOB
+ * @dirty: structure for user-space dirty-tracking
+ */
+struct vmw_bo {
+	struct ttm_buffer_object tbo;
+
+	struct ttm_placement placement;
+	struct ttm_place places[5];
+	struct ttm_place busy_places[5];
+
+	/* Protected by reservation */
+	struct ttm_bo_kmap_obj map;
+
+	struct rb_root res_tree;
+	u32 res_prios[TTM_MAX_BO_PRIORITY];
+
+	atomic_t cpu_writers;
+	/* Not ref-counted.  Protected by binding_mutex */
+	struct vmw_resource *dx_query_ctx;
+	struct vmw_bo_dirty *dirty;
+};
+
+void vmw_bo_placement_set(struct vmw_bo *bo, u32 domain, u32 busy_domain);
+void vmw_bo_placement_set_default_accelerated(struct vmw_bo *bo);
+
+int vmw_bo_create(struct vmw_private *dev_priv,
+		  struct vmw_bo_params *params,
+		  struct vmw_bo **p_bo);
+
+int vmw_bo_unref_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv);
+
+int vmw_bo_pin_in_vram(struct vmw_private *dev_priv,
+		       struct vmw_bo *buf,
+		       bool interruptible);
+int vmw_bo_pin_in_vram_or_gmr(struct vmw_private *dev_priv,
+			      struct vmw_bo *buf,
+			      bool interruptible);
+int vmw_bo_pin_in_start_of_vram(struct vmw_private *vmw_priv,
+				struct vmw_bo *bo,
+				bool interruptible);
+void vmw_bo_pin_reserved(struct vmw_bo *bo, bool pin);
+int vmw_bo_unpin(struct vmw_private *vmw_priv,
+		 struct vmw_bo *bo,
+		 bool interruptible);
+
+void vmw_bo_get_guest_ptr(const struct ttm_buffer_object *buf,
+			  SVGAGuestPtr *ptr);
+int vmw_user_bo_synccpu_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file_priv);
+void vmw_bo_fence_single(struct ttm_buffer_object *bo,
+			 struct vmw_fence_obj *fence);
+
+void *vmw_bo_map_and_cache(struct vmw_bo *vbo);
+void vmw_bo_unmap(struct vmw_bo *vbo);
+
+void vmw_bo_move_notify(struct ttm_buffer_object *bo,
+			struct ttm_resource *mem);
+void vmw_bo_swap_notify(struct ttm_buffer_object *bo);
+
+int vmw_user_bo_lookup(struct drm_file *filp,
+		       u32 handle,
+		       struct vmw_bo **out);
+/**
+ * vmw_bo_adjust_prio - Adjust the buffer object eviction priority
+ * according to attached resources
+ * @vbo: The struct vmw_bo
+ */
+static inline void vmw_bo_prio_adjust(struct vmw_bo *vbo)
+{
+	int i = ARRAY_SIZE(vbo->res_prios);
+
+	while (i--) {
+		if (vbo->res_prios[i]) {
+			vbo->tbo.priority = i;
+			return;
+		}
+	}
+
+	vbo->tbo.priority = 3;
+}
+
+/**
+ * vmw_bo_prio_add - Notify a buffer object of a newly attached resource
+ * eviction priority
+ * @vbo: The struct vmw_bo
+ * @prio: The resource priority
+ *
+ * After being notified, the code assigns the highest resource eviction priority
+ * to the backing buffer object (mob).
+ */
+static inline void vmw_bo_prio_add(struct vmw_bo *vbo, int prio)
+{
+	if (vbo->res_prios[prio]++ == 0)
+		vmw_bo_prio_adjust(vbo);
+}
+
+/**
+ * vmw_bo_used_prio_del - Notify a buffer object of a resource with a certain
+ * priority being removed
+ * @vbo: The struct vmw_bo
+ * @prio: The resource priority
+ *
+ * After being notified, the code assigns the highest resource eviction priority
+ * to the backing buffer object (mob).
+ */
+static inline void vmw_bo_prio_del(struct vmw_bo *vbo, int prio)
+{
+	if (--vbo->res_prios[prio] == 0)
+		vmw_bo_prio_adjust(vbo);
+}
+
+static inline void vmw_bo_unreference(struct vmw_bo **buf)
+{
+	struct vmw_bo *tmp_buf = *buf;
+
+	*buf = NULL;
+	if (tmp_buf)
+		ttm_bo_put(&tmp_buf->tbo);
+}
+
+static inline struct vmw_bo *vmw_bo_reference(struct vmw_bo *buf)
+{
+	ttm_bo_get(&buf->tbo);
+	return buf;
+}
+
+static inline struct vmw_bo *to_vmw_bo(struct drm_gem_object *gobj)
+{
+	return container_of((gobj), struct vmw_bo, tbo.base);
+}
+
+#endif // VMWGFX_BO_H
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
index 162dfeb1cc5a..195ff8792e5a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009-2020 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -24,13 +24,13 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  **************************************************************************/
-
-#include <linux/sched/signal.h>
+#include "vmwgfx_bo.h"
+#include "vmwgfx_drv.h"
+#include "vmwgfx_devcaps.h"
 
 #include <drm/ttm/ttm_placement.h>
 
-#include "vmwgfx_drv.h"
-#include "vmwgfx_devcaps.h"
+#include <linux/sched/signal.h>
 
 bool vmw_supports_3d(struct vmw_private *dev_priv)
 {
@@ -567,7 +567,7 @@ static int vmw_cmd_emit_dummy_legacy_query(struct vmw_private *dev_priv,
 	 * without writing to the query result structure.
 	 */
 
-	struct ttm_buffer_object *bo = &dev_priv->dummy_query_bo->base;
+	struct ttm_buffer_object *bo = &dev_priv->dummy_query_bo->tbo;
 	struct {
 		SVGA3dCmdHeader header;
 		SVGA3dCmdWaitForQuery body;
@@ -613,7 +613,7 @@ static int vmw_cmd_emit_dummy_gb_query(struct vmw_private *dev_priv,
 	 * without writing to the query result structure.
 	 */
 
-	struct ttm_buffer_object *bo = &dev_priv->dummy_query_bo->base;
+	struct ttm_buffer_object *bo = &dev_priv->dummy_query_bo->tbo;
 	struct {
 		SVGA3dCmdHeader header;
 		SVGA3dCmdWaitForGBQuery body;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
index 2b843ff4b437..94e8982f5616 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2015 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2015-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -25,12 +25,13 @@
  *
  **************************************************************************/
 
-#include <linux/dmapool.h>
-#include <linux/pci.h>
+#include "vmwgfx_bo.h"
+#include "vmwgfx_drv.h"
 
 #include <drm/ttm/ttm_bo.h>
 
-#include "vmwgfx_drv.h"
+#include <linux/dmapool.h>
+#include <linux/pci.h>
 
 /*
  * Size of inline command buffers. Try to make sure that a page size is a
@@ -79,7 +80,6 @@ struct vmw_cmdbuf_context {
  * frees are protected by @lock.
  * @cmd_space: Buffer object for the command buffer space, unless we were
  * able to make a contigous coherent DMA memory allocation, @handle. Immutable.
- * @map_obj: Mapping state for @cmd_space. Immutable.
  * @map: Pointer to command buffer space. May be a mapped buffer object or
  * a contigous coherent DMA memory allocation. Immutable.
  * @cur: Command buffer for small kernel command submissions. Protected by
@@ -116,8 +116,7 @@ struct vmw_cmdbuf_man {
 	struct vmw_cmdbuf_context ctx[SVGA_CB_CONTEXT_MAX];
 	struct list_head error;
 	struct drm_mm mm;
-	struct ttm_buffer_object *cmd_space;
-	struct ttm_bo_kmap_obj map_obj;
+	struct vmw_bo *cmd_space;
 	u8 *map;
 	struct vmw_cmdbuf_header *cur;
 	size_t cur_pos;
@@ -888,7 +887,7 @@ static int vmw_cmdbuf_space_pool(struct vmw_cmdbuf_man *man,
 	header->cmd = man->map + offset;
 	if (man->using_mob) {
 		cb_hdr->flags = SVGA_CB_FLAG_MOB;
-		cb_hdr->ptr.mob.mobid = man->cmd_space->resource->start;
+		cb_hdr->ptr.mob.mobid = man->cmd_space->tbo.resource->start;
 		cb_hdr->ptr.mob.mobOffset = offset;
 	} else {
 		cb_hdr->ptr.pa = (u64)man->handle + (u64)offset;
@@ -1221,7 +1220,6 @@ static int vmw_cmdbuf_startstop(struct vmw_cmdbuf_man *man, u32 context,
 int vmw_cmdbuf_set_pool_size(struct vmw_cmdbuf_man *man, size_t size)
 {
 	struct vmw_private *dev_priv = man->dev_priv;
-	bool dummy;
 	int ret;
 
 	if (man->has_pool)
@@ -1234,6 +1232,13 @@ int vmw_cmdbuf_set_pool_size(struct vmw_cmdbuf_man *man, size_t size)
 	if (man->map) {
 		man->using_mob = false;
 	} else {
+		struct vmw_bo_params bo_params = {
+			.domain = VMW_BO_DOMAIN_MOB,
+			.busy_domain = VMW_BO_DOMAIN_MOB,
+			.bo_type = ttm_bo_type_kernel,
+			.size = size,
+			.pin = true
+		};
 		/*
 		 * DMA memory failed. If we can have command buffers in a
 		 * MOB, try to use that instead. Note that this will
@@ -1244,19 +1249,12 @@ int vmw_cmdbuf_set_pool_size(struct vmw_cmdbuf_man *man, size_t size)
 		    !dev_priv->has_mob)
 			return -ENOMEM;
 
-		ret = vmw_bo_create_kernel(dev_priv, size,
-					   &vmw_mob_placement,
-					   &man->cmd_space);
+		ret = vmw_bo_create(dev_priv, &bo_params, &man->cmd_space);
 		if (ret)
 			return ret;
 
-		man->using_mob = true;
-		ret = ttm_bo_kmap(man->cmd_space, 0, size >> PAGE_SHIFT,
-				  &man->map_obj);
-		if (ret)
-			goto out_no_map;
-
-		man->map = ttm_kmap_obj_virtual(&man->map_obj, &dummy);
+		man->map = vmw_bo_map_and_cache(man->cmd_space);
+		man->using_mob = man->map;
 	}
 
 	man->size = size;
@@ -1276,14 +1274,6 @@ int vmw_cmdbuf_set_pool_size(struct vmw_cmdbuf_man *man, size_t size)
 		 (man->using_mob) ? "MOB" : "DMA");
 
 	return 0;
-
-out_no_map:
-	if (man->using_mob) {
-		ttm_bo_put(man->cmd_space);
-		man->cmd_space = NULL;
-	}
-
-	return ret;
 }
 
 /**
@@ -1382,14 +1372,11 @@ void vmw_cmdbuf_remove_pool(struct vmw_cmdbuf_man *man)
 	man->has_pool = false;
 	man->default_size = VMW_CMDBUF_INLINE_SIZE;
 	(void) vmw_cmdbuf_idle(man, false, 10*HZ);
-	if (man->using_mob) {
-		(void) ttm_bo_kunmap(&man->map_obj);
-		ttm_bo_put(man->cmd_space);
-		man->cmd_space = NULL;
-	} else {
+	if (man->using_mob)
+		vmw_bo_unreference(&man->cmd_space);
+	else
 		dma_free_coherent(man->dev_priv->drm.dev,
 				  man->size, man->map, man->handle);
-	}
 }
 
 /**
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
index e0f48cd9529b..ecc503e42790 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009-2015 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -27,9 +27,10 @@
 
 #include <drm/ttm/ttm_placement.h>
 
+#include "vmwgfx_binding.h"
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 #include "vmwgfx_resource_priv.h"
-#include "vmwgfx_binding.h"
 
 struct vmw_user_context {
 	struct ttm_base_object base;
@@ -38,7 +39,7 @@ struct vmw_user_context {
 	struct vmw_cmdbuf_res_manager *man;
 	struct vmw_resource *cotables[SVGA_COTABLE_MAX];
 	spinlock_t cotable_lock;
-	struct vmw_buffer_object *dx_query_mob;
+	struct vmw_bo *dx_query_mob;
 };
 
 static void vmw_user_context_free(struct vmw_resource *res);
@@ -72,10 +73,11 @@ const struct vmw_user_resource_conv *user_context_converter =
 
 static const struct vmw_res_func vmw_legacy_context_func = {
 	.res_type = vmw_res_context,
-	.needs_backup = false,
+	.needs_guest_memory = false,
 	.may_evict = false,
 	.type_name = "legacy contexts",
-	.backup_placement = NULL,
+	.domain = VMW_BO_DOMAIN_SYS,
+	.busy_domain = VMW_BO_DOMAIN_SYS,
 	.create = NULL,
 	.destroy = NULL,
 	.bind = NULL,
@@ -84,12 +86,13 @@ static const struct vmw_res_func vmw_legacy_context_func = {
 
 static const struct vmw_res_func vmw_gb_context_func = {
 	.res_type = vmw_res_context,
-	.needs_backup = true,
+	.needs_guest_memory = true,
 	.may_evict = true,
 	.prio = 3,
 	.dirty_prio = 3,
 	.type_name = "guest backed contexts",
-	.backup_placement = &vmw_mob_placement,
+	.domain = VMW_BO_DOMAIN_MOB,
+	.busy_domain = VMW_BO_DOMAIN_MOB,
 	.create = vmw_gb_context_create,
 	.destroy = vmw_gb_context_destroy,
 	.bind = vmw_gb_context_bind,
@@ -98,12 +101,13 @@ static const struct vmw_res_func vmw_gb_context_func = {
 
 static const struct vmw_res_func vmw_dx_context_func = {
 	.res_type = vmw_res_dx_context,
-	.needs_backup = true,
+	.needs_guest_memory = true,
 	.may_evict = true,
 	.prio = 3,
 	.dirty_prio = 3,
 	.type_name = "dx contexts",
-	.backup_placement = &vmw_mob_placement,
+	.domain = VMW_BO_DOMAIN_MOB,
+	.busy_domain = VMW_BO_DOMAIN_MOB,
 	.create = vmw_dx_context_create,
 	.destroy = vmw_dx_context_destroy,
 	.bind = vmw_dx_context_bind,
@@ -182,7 +186,7 @@ static int vmw_gb_context_init(struct vmw_private *dev_priv,
 	struct vmw_user_context *uctx =
 		container_of(res, struct vmw_user_context, res);
 
-	res->backup_size = (dx ? sizeof(SVGADXContextMobFormat) :
+	res->guest_memory_size = (dx ? sizeof(SVGADXContextMobFormat) :
 				 sizeof(SVGAGBContextData));
 	ret = vmw_resource_init(dev_priv, res, true,
 				res_free,
@@ -354,8 +358,8 @@ static int vmw_gb_context_bind(struct vmw_resource *res,
 	cmd->header.size = sizeof(cmd->body);
 	cmd->body.cid = res->id;
 	cmd->body.mobid = bo->resource->start;
-	cmd->body.validContents = res->backup_dirty;
-	res->backup_dirty = false;
+	cmd->body.validContents = res->guest_memory_dirty;
+	res->guest_memory_dirty = false;
 	vmw_cmd_commit(dev_priv, sizeof(*cmd));
 
 	return 0;
@@ -521,8 +525,8 @@ static int vmw_dx_context_bind(struct vmw_resource *res,
 	cmd->header.size = sizeof(cmd->body);
 	cmd->body.cid = res->id;
 	cmd->body.mobid = bo->resource->start;
-	cmd->body.validContents = res->backup_dirty;
-	res->backup_dirty = false;
+	cmd->body.validContents = res->guest_memory_dirty;
+	res->guest_memory_dirty = false;
 	vmw_cmd_commit(dev_priv, sizeof(*cmd));
 
 
@@ -853,7 +857,7 @@ vmw_context_binding_state(struct vmw_resource *ctx)
  * specified in the parameter.  0 otherwise.
  */
 int vmw_context_bind_dx_query(struct vmw_resource *ctx_res,
-			      struct vmw_buffer_object *mob)
+			      struct vmw_bo *mob)
 {
 	struct vmw_user_context *uctx =
 		container_of(ctx_res, struct vmw_user_context, res);
@@ -885,7 +889,7 @@ int vmw_context_bind_dx_query(struct vmw_resource *ctx_res,
  *
  * @ctx_res: The context resource
  */
-struct vmw_buffer_object *
+struct vmw_bo *
 vmw_context_get_dx_query_mob(struct vmw_resource *ctx_res)
 {
 	struct vmw_user_context *uctx =
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
index b78a10312fad..c0b24d1cacbf 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2014-2015 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2014-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -30,13 +30,14 @@
  * whenever the backing MOB is evicted.
  */
 
-#include <drm/ttm/ttm_placement.h>
-
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 #include "vmwgfx_mksstat.h"
 #include "vmwgfx_resource_priv.h"
 #include "vmwgfx_so.h"
 
+#include <drm/ttm/ttm_placement.h>
+
 /**
  * struct vmw_cotable - Context Object Table resource
  *
@@ -130,12 +131,13 @@ static int vmw_cotable_destroy(struct vmw_resource *res);
 
 static const struct vmw_res_func vmw_cotable_func = {
 	.res_type = vmw_res_cotable,
-	.needs_backup = true,
+	.needs_guest_memory = true,
 	.may_evict = true,
 	.prio = 3,
 	.dirty_prio = 3,
 	.type_name = "context guest backed object tables",
-	.backup_placement = &vmw_mob_placement,
+	.domain = VMW_BO_DOMAIN_MOB,
+	.busy_domain = VMW_BO_DOMAIN_MOB,
 	.create = vmw_cotable_create,
 	.destroy = vmw_cotable_destroy,
 	.bind = vmw_cotable_bind,
@@ -180,7 +182,7 @@ static int vmw_cotable_unscrub(struct vmw_resource *res)
 {
 	struct vmw_cotable *vcotbl = vmw_cotable(res);
 	struct vmw_private *dev_priv = res->dev_priv;
-	struct ttm_buffer_object *bo = &res->backup->base;
+	struct ttm_buffer_object *bo = &res->guest_memory_bo->tbo;
 	struct {
 		SVGA3dCmdHeader header;
 		SVGA3dCmdDXSetCOTable body;
@@ -228,7 +230,7 @@ static int vmw_cotable_bind(struct vmw_resource *res,
 	 * take the opportunity to correct the value here so that it's not
 	 * misused in the future.
 	 */
-	val_buf->bo = &res->backup->base;
+	val_buf->bo = &res->guest_memory_bo->tbo;
 
 	return vmw_cotable_unscrub(res);
 }
@@ -289,7 +291,7 @@ int vmw_cotable_scrub(struct vmw_resource *res, bool readback)
 		cmd0->body.cid = vcotbl->ctx->id;
 		cmd0->body.type = vcotbl->type;
 		cmd1 = (void *) &cmd0[1];
-		vcotbl->size_read_back = res->backup_size;
+		vcotbl->size_read_back = res->guest_memory_size;
 	}
 	cmd1->header.id = SVGA_3D_CMD_DX_SET_COTABLE;
 	cmd1->header.size = sizeof(cmd1->body);
@@ -371,12 +373,12 @@ static int vmw_cotable_readback(struct vmw_resource *res)
 		cmd->header.size = sizeof(cmd->body);
 		cmd->body.cid = vcotbl->ctx->id;
 		cmd->body.type = vcotbl->type;
-		vcotbl->size_read_back = res->backup_size;
+		vcotbl->size_read_back = res->guest_memory_size;
 		vmw_cmd_commit(dev_priv, sizeof(*cmd));
 	}
 
 	(void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
-	vmw_bo_fence_single(&res->backup->base, fence);
+	vmw_bo_fence_single(&res->guest_memory_bo->tbo, fence);
 	vmw_fence_obj_unreference(&fence);
 
 	return 0;
@@ -399,14 +401,21 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
 	struct ttm_operation_ctx ctx = { false, false };
 	struct vmw_private *dev_priv = res->dev_priv;
 	struct vmw_cotable *vcotbl = vmw_cotable(res);
-	struct vmw_buffer_object *buf, *old_buf = res->backup;
-	struct ttm_buffer_object *bo, *old_bo = &res->backup->base;
-	size_t old_size = res->backup_size;
+	struct vmw_bo *buf, *old_buf = res->guest_memory_bo;
+	struct ttm_buffer_object *bo, *old_bo = &res->guest_memory_bo->tbo;
+	size_t old_size = res->guest_memory_size;
 	size_t old_size_read_back = vcotbl->size_read_back;
 	size_t cur_size_read_back;
 	struct ttm_bo_kmap_obj old_map, new_map;
 	int ret;
 	size_t i;
+	struct vmw_bo_params bo_params = {
+		.domain = VMW_BO_DOMAIN_MOB,
+		.busy_domain = VMW_BO_DOMAIN_MOB,
+		.bo_type = ttm_bo_type_device,
+		.size = new_size,
+		.pin = true
+	};
 
 	MKS_STAT_TIME_DECL(MKSSTAT_KERN_COTABLE_RESIZE);
 	MKS_STAT_TIME_PUSH(MKSSTAT_KERN_COTABLE_RESIZE);
@@ -423,14 +432,13 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
 	 * for the new COTable. Initially pin the buffer object to make sure
 	 * we can use tryreserve without failure.
 	 */
-	ret = vmw_bo_create(dev_priv, new_size, &vmw_mob_placement,
-			    true, true, vmw_bo_bo_free, &buf);
+	ret = vmw_bo_create(dev_priv, &bo_params, &buf);
 	if (ret) {
 		DRM_ERROR("Failed initializing new cotable MOB.\n");
 		goto out_done;
 	}
 
-	bo = &buf->base;
+	bo = &buf->tbo;
 	WARN_ON_ONCE(ttm_bo_reserve(bo, false, true, NULL));
 
 	ret = ttm_bo_wait(old_bo, false, false);
@@ -464,15 +472,18 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
 	}
 
 	/* Unpin new buffer, and switch backup buffers. */
-	ret = ttm_bo_validate(bo, &vmw_mob_placement, &ctx);
+	vmw_bo_placement_set(buf,
+			     VMW_BO_DOMAIN_MOB,
+			     VMW_BO_DOMAIN_MOB);
+	ret = ttm_bo_validate(bo, &buf->placement, &ctx);
 	if (unlikely(ret != 0)) {
 		DRM_ERROR("Failed validating new COTable backup buffer.\n");
 		goto out_wait;
 	}
 
 	vmw_resource_mob_detach(res);
-	res->backup = buf;
-	res->backup_size = new_size;
+	res->guest_memory_bo = buf;
+	res->guest_memory_size = new_size;
 	vcotbl->size_read_back = cur_size_read_back;
 
 	/*
@@ -482,8 +493,8 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
 	ret = vmw_cotable_unscrub(res);
 	if (ret) {
 		DRM_ERROR("Failed switching COTable backup buffer.\n");
-		res->backup = old_buf;
-		res->backup_size = old_size;
+		res->guest_memory_bo = old_buf;
+		res->guest_memory_size = old_size;
 		vcotbl->size_read_back = old_size_read_back;
 		vmw_resource_mob_attach(res);
 		goto out_wait;
@@ -498,7 +509,7 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
 	if (unlikely(ret))
 		goto out_wait;
 
-	/* Release the pin acquired in vmw_bo_init */
+	/* Release the pin acquired in vmw_bo_create */
 	ttm_bo_unpin(bo);
 
 	MKS_STAT_TIME_POP(MKSSTAT_KERN_COTABLE_RESIZE);
@@ -533,7 +544,7 @@ out_done:
 static int vmw_cotable_create(struct vmw_resource *res)
 {
 	struct vmw_cotable *vcotbl = vmw_cotable(res);
-	size_t new_size = res->backup_size;
+	size_t new_size = res->guest_memory_size;
 	size_t needed_size;
 	int ret;
 
@@ -542,7 +553,7 @@ static int vmw_cotable_create(struct vmw_resource *res)
 	while (needed_size > new_size)
 		new_size *= 2;
 
-	if (likely(new_size <= res->backup_size)) {
+	if (likely(new_size <= res->guest_memory_size)) {
 		if (vcotbl->scrubbed && vmw_resource_mob_attached(res)) {
 			ret = vmw_cotable_unscrub(res);
 			if (ret)
@@ -606,12 +617,12 @@ struct vmw_resource *vmw_cotable_alloc(struct vmw_private *dev_priv,
 
 	INIT_LIST_HEAD(&vcotbl->resource_list);
 	vcotbl->res.id = type;
-	vcotbl->res.backup_size = PAGE_SIZE;
+	vcotbl->res.guest_memory_size = PAGE_SIZE;
 	num_entries = PAGE_SIZE / co_info[type].size;
 	if (num_entries < co_info[type].min_initial_entries) {
-		vcotbl->res.backup_size = co_info[type].min_initial_entries *
+		vcotbl->res.guest_memory_size = co_info[type].min_initial_entries *
 			co_info[type].size;
-		vcotbl->res.backup_size = PFN_ALIGN(vcotbl->res.backup_size);
+		vcotbl->res.guest_memory_size = PFN_ALIGN(vcotbl->res.guest_memory_size);
 	}
 
 	vcotbl->scrubbed = true;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 9ad28346aff7..8b24ecf60e3e 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009-2022 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -28,9 +28,10 @@
 
 #include "vmwgfx_drv.h"
 
+#include "vmwgfx_bo.h"
+#include "vmwgfx_binding.h"
 #include "vmwgfx_devcaps.h"
 #include "vmwgfx_mksstat.h"
-#include "vmwgfx_binding.h"
 #include "ttm_object.h"
 
 #include <drm/drm_aperture.h>
@@ -44,6 +45,9 @@
 #include <drm/ttm/ttm_placement.h>
 #include <generated/utsrelease.h>
 
+#ifdef CONFIG_X86
+#include <asm/hypervisor.h>
+#endif
 #include <linux/cc_platform.h>
 #include <linux/dma-mapping.h>
 #include <linux/module.h>
@@ -386,27 +390,32 @@ static void vmw_print_sm_type(struct vmw_private *dev_priv)
 static int vmw_dummy_query_bo_create(struct vmw_private *dev_priv)
 {
 	int ret;
-	struct vmw_buffer_object *vbo;
+	struct vmw_bo *vbo;
 	struct ttm_bo_kmap_obj map;
 	volatile SVGA3dQueryResult *result;
 	bool dummy;
+	struct vmw_bo_params bo_params = {
+		.domain = VMW_BO_DOMAIN_SYS,
+		.busy_domain = VMW_BO_DOMAIN_SYS,
+		.bo_type = ttm_bo_type_kernel,
+		.size = PAGE_SIZE,
+		.pin = true
+	};
 
 	/*
 	 * Create the vbo as pinned, so that a tryreserve will
 	 * immediately succeed. This is because we're the only
 	 * user of the bo currently.
 	 */
-	ret = vmw_bo_create(dev_priv, PAGE_SIZE,
-			    &vmw_sys_placement, false, true,
-			    &vmw_bo_bo_free, &vbo);
+	ret = vmw_bo_create(dev_priv, &bo_params, &vbo);
 	if (unlikely(ret != 0))
 		return ret;
 
-	ret = ttm_bo_reserve(&vbo->base, false, true, NULL);
+	ret = ttm_bo_reserve(&vbo->tbo, false, true, NULL);
 	BUG_ON(ret != 0);
 	vmw_bo_pin_reserved(vbo, true);
 
-	ret = ttm_bo_kmap(&vbo->base, 0, 1, &map);
+	ret = ttm_bo_kmap(&vbo->tbo, 0, 1, &map);
 	if (likely(ret == 0)) {
 		result = ttm_kmap_obj_virtual(&map, &dummy);
 		result->totalSize = sizeof(*result);
@@ -415,7 +424,7 @@ static int vmw_dummy_query_bo_create(struct vmw_private *dev_priv)
 		ttm_bo_kunmap(&map);
 	}
 	vmw_bo_pin_reserved(vbo, false);
-	ttm_bo_unreserve(&vbo->base);
+	ttm_bo_unreserve(&vbo->tbo);
 
 	if (unlikely(ret != 0)) {
 		DRM_ERROR("Dummy query buffer map failed.\n");
@@ -891,6 +900,16 @@ static int vmw_driver_load(struct vmw_private *dev_priv, u32 pci_id)
 				 cap2_names, ARRAY_SIZE(cap2_names));
 	}
 
+	if (!vmwgfx_supported(dev_priv)) {
+		vmw_disable_backdoor();
+		drm_err_once(&dev_priv->drm,
+			     "vmwgfx seems to be running on an unsupported hypervisor.");
+		drm_err_once(&dev_priv->drm,
+			     "This configuration is likely broken.");
+		drm_err_once(&dev_priv->drm,
+			     "Please switch to a supported graphics device to avoid problems.");
+	}
+
 	ret = vmw_dma_select_mode(dev_priv);
 	if (unlikely(ret != 0)) {
 		drm_info(&dev_priv->drm,
@@ -1314,6 +1333,22 @@ static void vmw_master_drop(struct drm_device *dev,
 	vmw_kms_legacy_hotspot_clear(dev_priv);
 }
 
+bool vmwgfx_supported(struct vmw_private *vmw)
+{
+#if defined(CONFIG_X86)
+	return hypervisor_is_type(X86_HYPER_VMWARE);
+#elif defined(CONFIG_ARM64)
+	/*
+	 * On aarch64 only svga3 is supported
+	 */
+	return vmw->pci_id == VMWGFX_PCI_ID_SVGA3;
+#else
+	drm_warn_once(&vmw->drm,
+		      "vmwgfx is running on an unknown architecture.");
+	return false;
+#endif
+}
+
 /**
  * __vmw_svga_enable - Enable SVGA mode, FIFO and use of VRAM.
  *
@@ -1565,7 +1600,7 @@ static const struct file_operations vmwgfx_driver_fops = {
 	.open = drm_open,
 	.release = drm_release,
 	.unlocked_ioctl = vmw_unlocked_ioctl,
-	.mmap = vmw_mmap,
+	.mmap = drm_gem_mmap,
 	.poll = drm_poll,
 	.read = drm_read,
 #if defined(CONFIG_COMPAT)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 203fa32cd4c1..3810a9984a7f 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
- * Copyright 2009-2022 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -117,32 +117,6 @@ struct vmwgfx_hash_item {
 	unsigned long key;
 };
 
-/**
- * struct vmw_buffer_object - TTM buffer object with vmwgfx additions
- * @base: The TTM buffer object
- * @res_tree: RB tree of resources using this buffer object as a backing MOB
- * @base_mapped_count: ttm BO mapping count; used by KMS atomic helpers.
- * @cpu_writers: Number of synccpu write grabs. Protected by reservation when
- * increased. May be decreased without reservation.
- * @dx_query_ctx: DX context if this buffer object is used as a DX query MOB
- * @map: Kmap object for semi-persistent mappings
- * @res_prios: Eviction priority counts for attached resources
- * @dirty: structure for user-space dirty-tracking
- */
-struct vmw_buffer_object {
-	struct ttm_buffer_object base;
-	struct rb_root res_tree;
-	/* For KMS atomic helpers: ttm bo mapping count */
-	atomic_t base_mapped_count;
-
-	atomic_t cpu_writers;
-	/* Not ref-counted.  Protected by binding_mutex */
-	struct vmw_resource *dx_query_ctx;
-	/* Protected by reservation */
-	struct ttm_bo_kmap_obj map;
-	u32 res_prios[TTM_MAX_BO_PRIORITY];
-	struct vmw_bo_dirty *dirty;
-};
 
 /**
  * struct vmw_validate_buffer - Carries validation info about buffers.
@@ -168,21 +142,23 @@ struct vmw_res_func;
  * @kref: For refcounting.
  * @dev_priv: Pointer to the device private for this resource. Immutable.
  * @id: Device id. Protected by @dev_priv::resource_lock.
- * @backup_size: Backup buffer size. Immutable.
- * @res_dirty: Resource contains data not yet in the backup buffer. Protected
- * by resource reserved.
- * @backup_dirty: Backup buffer contains data not yet in the HW resource.
+ * @guest_memory_size: Guest memory buffer size. Immutable.
+ * @res_dirty: Resource contains data not yet in the guest memory buffer.
  * Protected by resource reserved.
+ * @guest_memory_dirty: Guest memory buffer contains data not yet in the HW
+ * resource. Protected by resource reserved.
  * @coherent: Emulate coherency by tracking vm accesses.
- * @backup: The backup buffer if any. Protected by resource reserved.
- * @backup_offset: Offset into the backup buffer if any. Protected by resource
- * reserved. Note that only a few resource types can have a @backup_offset
- * different from zero.
+ * @guest_memory_bo: The guest memory buffer if any. Protected by resource
+ * reserved.
+ * @guest_memory_offset: Offset into the guest memory buffer if any. Protected
+ * by resource reserved. Note that only a few resource types can have a
+ * @guest_memory_offset different from zero.
  * @pin_count: The pin count for this resource. A pinned resource has a
  * pin-count greater than zero. It is not on the resource LRU lists and its
- * backup buffer is pinned. Hence it can't be evicted.
+ * guest memory buffer is pinned. Hence it can't be evicted.
  * @func: Method vtable for this resource. Immutable.
- * @mob_node; Node for the MOB backup rbtree. Protected by @backup reserved.
+ * @mob_node; Node for the MOB guest memory rbtree. Protected by
+ * @guest_memory_bo reserved.
  * @lru_head: List head for the LRU list. Protected by @dev_priv::resource_lock.
  * @binding_head: List head for the context binding list. Protected by
  * the @dev_priv::binding_mutex
@@ -190,18 +166,20 @@ struct vmw_res_func;
  * @hw_destroy: Callback to destroy the resource on the device, as part of
  * resource destruction.
  */
+struct vmw_bo;
+struct vmw_bo;
 struct vmw_resource_dirty;
 struct vmw_resource {
 	struct kref kref;
 	struct vmw_private *dev_priv;
 	int id;
 	u32 used_prio;
-	unsigned long backup_size;
+	unsigned long guest_memory_size;
 	u32 res_dirty : 1;
-	u32 backup_dirty : 1;
+	u32 guest_memory_dirty : 1;
 	u32 coherent : 1;
-	struct vmw_buffer_object *backup;
-	unsigned long backup_offset;
+	struct vmw_bo *guest_memory_bo;
+	unsigned long guest_memory_offset;
 	unsigned long pin_count;
 	const struct vmw_res_func *func;
 	struct rb_node mob_node;
@@ -446,7 +424,7 @@ struct vmw_sw_context{
 	struct drm_file *filp;
 	uint32_t *cmd_bounce;
 	uint32_t cmd_bounce_size;
-	struct vmw_buffer_object *cur_query_bo;
+	struct vmw_bo *cur_query_bo;
 	struct list_head bo_relocations;
 	struct list_head res_relocations;
 	uint32_t *buf_start;
@@ -458,7 +436,7 @@ struct vmw_sw_context{
 	struct list_head staged_cmd_res;
 	struct list_head ctx_list;
 	struct vmw_ctx_validation_info *dx_ctx_node;
-	struct vmw_buffer_object *dx_query_mob;
+	struct vmw_bo *dx_query_mob;
 	struct vmw_resource *dx_query_ctx;
 	struct vmw_cmdbuf_res_manager *man;
 	struct vmw_validation_context *ctx;
@@ -492,7 +470,7 @@ struct vmw_otable_batch {
 	unsigned num_otables;
 	struct vmw_otable *otables;
 	struct vmw_resource *context;
-	struct ttm_buffer_object *otable_bo;
+	struct vmw_bo *otable_bo;
 };
 
 enum {
@@ -632,8 +610,8 @@ struct vmw_private {
 	 * are protected by the cmdbuf mutex.
 	 */
 
-	struct vmw_buffer_object *dummy_query_bo;
-	struct vmw_buffer_object *pinned_bo;
+	struct vmw_bo *dummy_query_bo;
+	struct vmw_bo *pinned_bo;
 	uint32_t query_cid;
 	uint32_t query_cid_valid;
 	bool dummy_query_bo_pinned;
@@ -677,11 +655,6 @@ struct vmw_private {
 #endif
 };
 
-static inline struct vmw_buffer_object *gem_to_vmw_bo(struct drm_gem_object *gobj)
-{
-	return container_of((gobj), struct vmw_buffer_object, base.base);
-}
-
 static inline struct vmw_surface *vmw_res_to_srf(struct vmw_resource *res)
 {
 	return container_of(res, struct vmw_surface, res);
@@ -692,6 +665,11 @@ static inline struct vmw_private *vmw_priv(struct drm_device *dev)
 	return (struct vmw_private *)dev->dev_private;
 }
 
+static inline struct vmw_private *vmw_priv_from_ttm(struct ttm_device *bdev)
+{
+	return container_of(bdev, struct vmw_private, bdev);
+}
+
 static inline struct vmw_fpriv *vmw_fpriv(struct drm_file *file_priv)
 {
 	return (struct vmw_fpriv *)file_priv->driver_priv;
@@ -795,6 +773,7 @@ static inline u32 vmw_max_num_uavs(struct vmw_private *dev_priv)
 
 extern void vmw_svga_enable(struct vmw_private *dev_priv);
 extern void vmw_svga_disable(struct vmw_private *dev_priv);
+bool vmwgfx_supported(struct vmw_private *vmw);
 
 
 /**
@@ -825,7 +804,7 @@ extern int vmw_user_lookup_handle(struct vmw_private *dev_priv,
 				  struct drm_file *filp,
 				  uint32_t handle,
 				  struct vmw_surface **out_surf,
-				  struct vmw_buffer_object **out_buf);
+				  struct vmw_bo **out_buf);
 extern int vmw_user_resource_lookup_handle(
 	struct vmw_private *dev_priv,
 	struct ttm_object_file *tfile,
@@ -844,20 +823,20 @@ extern int vmw_user_stream_lookup(struct vmw_private *dev_priv,
 extern void vmw_resource_unreserve(struct vmw_resource *res,
 				   bool dirty_set,
 				   bool dirty,
-				   bool switch_backup,
-				   struct vmw_buffer_object *new_backup,
-				   unsigned long new_backup_offset);
+				   bool switch_guest_memory,
+				   struct vmw_bo *new_guest_memory,
+				   unsigned long new_guest_memory_offset);
 extern void vmw_query_move_notify(struct ttm_buffer_object *bo,
 				  struct ttm_resource *old_mem,
 				  struct ttm_resource *new_mem);
-extern int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob);
-extern void vmw_resource_evict_all(struct vmw_private *dev_priv);
-extern void vmw_resource_unbind_list(struct vmw_buffer_object *vbo);
+int vmw_query_readback_all(struct vmw_bo *dx_query_mob);
+void vmw_resource_evict_all(struct vmw_private *dev_priv);
+void vmw_resource_unbind_list(struct vmw_bo *vbo);
 void vmw_resource_mob_attach(struct vmw_resource *res);
 void vmw_resource_mob_detach(struct vmw_resource *res);
 void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start,
 			       pgoff_t end);
-int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start,
+int vmw_resources_clean(struct vmw_bo *vbo, pgoff_t start,
 			pgoff_t end, pgoff_t *num_prefault);
 
 /**
@@ -872,117 +851,15 @@ static inline bool vmw_resource_mob_attached(const struct vmw_resource *res)
 }
 
 /**
- * Buffer object helper functions - vmwgfx_bo.c
- */
-extern int vmw_bo_pin_in_placement(struct vmw_private *vmw_priv,
-				   struct vmw_buffer_object *bo,
-				   struct ttm_placement *placement,
-				   bool interruptible);
-extern int vmw_bo_pin_in_vram(struct vmw_private *dev_priv,
-			      struct vmw_buffer_object *buf,
-			      bool interruptible);
-extern int vmw_bo_pin_in_vram_or_gmr(struct vmw_private *dev_priv,
-				     struct vmw_buffer_object *buf,
-				     bool interruptible);
-extern int vmw_bo_pin_in_start_of_vram(struct vmw_private *vmw_priv,
-				       struct vmw_buffer_object *bo,
-				       bool interruptible);
-extern int vmw_bo_unpin(struct vmw_private *vmw_priv,
-			struct vmw_buffer_object *bo,
-			bool interruptible);
-extern void vmw_bo_get_guest_ptr(const struct ttm_buffer_object *buf,
-				 SVGAGuestPtr *ptr);
-extern void vmw_bo_pin_reserved(struct vmw_buffer_object *bo, bool pin);
-extern void vmw_bo_bo_free(struct ttm_buffer_object *bo);
-extern int vmw_bo_create_kernel(struct vmw_private *dev_priv,
-				unsigned long size,
-				struct ttm_placement *placement,
-				struct ttm_buffer_object **p_bo);
-extern int vmw_bo_create(struct vmw_private *dev_priv,
-			 size_t size, struct ttm_placement *placement,
-			 bool interruptible, bool pin,
-			 void (*bo_free)(struct ttm_buffer_object *bo),
-			 struct vmw_buffer_object **p_bo);
-extern int vmw_bo_init(struct vmw_private *dev_priv,
-		       struct vmw_buffer_object *vmw_bo,
-		       size_t size, struct ttm_placement *placement,
-		       bool interruptible, bool pin,
-		       void (*bo_free)(struct ttm_buffer_object *bo));
-extern int vmw_bo_unref_ioctl(struct drm_device *dev, void *data,
-			      struct drm_file *file_priv);
-extern int vmw_user_bo_synccpu_ioctl(struct drm_device *dev, void *data,
-				     struct drm_file *file_priv);
-extern int vmw_user_bo_lookup(struct drm_file *filp,
-			      uint32_t handle,
-			      struct vmw_buffer_object **out);
-extern void vmw_bo_fence_single(struct ttm_buffer_object *bo,
-				struct vmw_fence_obj *fence);
-extern void *vmw_bo_map_and_cache(struct vmw_buffer_object *vbo);
-extern void vmw_bo_unmap(struct vmw_buffer_object *vbo);
-extern void vmw_bo_move_notify(struct ttm_buffer_object *bo,
-			       struct ttm_resource *mem);
-extern void vmw_bo_swap_notify(struct ttm_buffer_object *bo);
-
-/**
- * vmw_bo_adjust_prio - Adjust the buffer object eviction priority
- * according to attached resources
- * @vbo: The struct vmw_buffer_object
- */
-static inline void vmw_bo_prio_adjust(struct vmw_buffer_object *vbo)
-{
-	int i = ARRAY_SIZE(vbo->res_prios);
-
-	while (i--) {
-		if (vbo->res_prios[i]) {
-			vbo->base.priority = i;
-			return;
-		}
-	}
-
-	vbo->base.priority = 3;
-}
-
-/**
- * vmw_bo_prio_add - Notify a buffer object of a newly attached resource
- * eviction priority
- * @vbo: The struct vmw_buffer_object
- * @prio: The resource priority
- *
- * After being notified, the code assigns the highest resource eviction priority
- * to the backing buffer object (mob).
- */
-static inline void vmw_bo_prio_add(struct vmw_buffer_object *vbo, int prio)
-{
-	if (vbo->res_prios[prio]++ == 0)
-		vmw_bo_prio_adjust(vbo);
-}
-
-/**
- * vmw_bo_prio_del - Notify a buffer object of a resource with a certain
- * priority being removed
- * @vbo: The struct vmw_buffer_object
- * @prio: The resource priority
- *
- * After being notified, the code assigns the highest resource eviction priority
- * to the backing buffer object (mob).
- */
-static inline void vmw_bo_prio_del(struct vmw_buffer_object *vbo, int prio)
-{
-	if (--vbo->res_prios[prio] == 0)
-		vmw_bo_prio_adjust(vbo);
-}
-
-/**
  * GEM related functionality - vmwgfx_gem.c
  */
 extern int vmw_gem_object_create_with_handle(struct vmw_private *dev_priv,
 					     struct drm_file *filp,
 					     uint32_t size,
 					     uint32_t *handle,
-					     struct vmw_buffer_object **p_vbo);
+					     struct vmw_bo **p_vbo);
 extern int vmw_gem_object_create_ioctl(struct drm_device *dev, void *data,
 				       struct drm_file *filp);
-extern void vmw_gem_destroy(struct ttm_buffer_object *bo);
 extern void vmw_debugfs_gem_init(struct vmw_private *vdev);
 
 /**
@@ -1056,29 +933,20 @@ vmw_is_cursor_bypass3_enabled(const struct vmw_private *dev_priv)
 }
 
 /**
- * TTM glue - vmwgfx_ttm_glue.c
- */
-
-extern int vmw_mmap(struct file *filp, struct vm_area_struct *vma);
-
-/**
  * TTM buffer object driver - vmwgfx_ttm_buffer.c
  */
 
 extern const size_t vmw_tt_size;
 extern struct ttm_placement vmw_vram_placement;
-extern struct ttm_placement vmw_vram_sys_placement;
 extern struct ttm_placement vmw_vram_gmr_placement;
 extern struct ttm_placement vmw_sys_placement;
-extern struct ttm_placement vmw_srf_placement;
-extern struct ttm_placement vmw_mob_placement;
-extern struct ttm_placement vmw_nonfixed_placement;
 extern struct ttm_device_funcs vmw_bo_driver;
 extern const struct vmw_sg_table *
 vmw_bo_sg_table(struct ttm_buffer_object *bo);
-extern int vmw_bo_create_and_populate(struct vmw_private *dev_priv,
-				      unsigned long bo_size,
-				      struct ttm_buffer_object **bo_p);
+int vmw_bo_create_and_populate(struct vmw_private *dev_priv,
+			       size_t bo_size,
+			       u32 domain,
+			       struct vmw_bo **bo_p);
 
 extern void vmw_piter_start(struct vmw_piter *viter,
 			    const struct vmw_sg_table *vsgt,
@@ -1297,8 +1165,8 @@ vmw_context_binding_state(struct vmw_resource *ctx);
 extern void vmw_dx_context_scrub_cotables(struct vmw_resource *ctx,
 					  bool readback);
 extern int vmw_context_bind_dx_query(struct vmw_resource *ctx_res,
-				     struct vmw_buffer_object *mob);
-extern struct vmw_buffer_object *
+				     struct vmw_bo *mob);
+extern struct vmw_bo *
 vmw_context_get_dx_query_mob(struct vmw_resource *ctx_res);
 
 
@@ -1491,6 +1359,7 @@ int vmw_bo_cpu_blit(struct ttm_buffer_object *dst,
 		    struct vmw_diff_cpy *diff);
 
 /* Host messaging -vmwgfx_msg.c: */
+void vmw_disable_backdoor(void);
 int vmw_host_get_guestinfo(const char *guest_info_param,
 			   char *buffer, size_t *length);
 __printf(1, 2) int vmw_host_printf(const char *fmt, ...);
@@ -1523,12 +1392,12 @@ int vmw_mksstat_remove_all(struct vmw_private *dev_priv);
 	DRM_DEBUG_DRIVER(fmt, ##__VA_ARGS__)
 
 /* Resource dirtying - vmwgfx_page_dirty.c */
-void vmw_bo_dirty_scan(struct vmw_buffer_object *vbo);
-int vmw_bo_dirty_add(struct vmw_buffer_object *vbo);
+void vmw_bo_dirty_scan(struct vmw_bo *vbo);
+int vmw_bo_dirty_add(struct vmw_bo *vbo);
 void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res);
 void vmw_bo_dirty_clear_res(struct vmw_resource *res);
-void vmw_bo_dirty_release(struct vmw_buffer_object *vbo);
-void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo,
+void vmw_bo_dirty_release(struct vmw_bo *vbo);
+void vmw_bo_dirty_unmap(struct vmw_bo *vbo,
 			pgoff_t start, pgoff_t end);
 vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf);
 vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf);
@@ -1561,22 +1430,6 @@ static inline struct vmw_surface *vmw_surface_reference(struct vmw_surface *srf)
 	return srf;
 }
 
-static inline void vmw_bo_unreference(struct vmw_buffer_object **buf)
-{
-	struct vmw_buffer_object *tmp_buf = *buf;
-
-	*buf = NULL;
-	if (tmp_buf != NULL)
-		ttm_bo_put(&tmp_buf->base);
-}
-
-static inline struct vmw_buffer_object *
-vmw_bo_reference(struct vmw_buffer_object *buf)
-{
-	ttm_bo_get(&buf->base);
-	return buf;
-}
-
 static inline void vmw_fifo_resource_inc(struct vmw_private *dev_priv)
 {
 	atomic_inc(&dev_priv->num_fifo_resources);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index 0590bb22c73a..6b9aa2b4ef54 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009 - 2022 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009 - 2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -24,17 +24,17 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  **************************************************************************/
-#include <linux/sync_file.h>
-#include <linux/hashtable.h>
-
+#include "vmwgfx_binding.h"
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
-#include "vmwgfx_reg.h"
+#include "vmwgfx_mksstat.h"
+#include "vmwgfx_so.h"
+
 #include <drm/ttm/ttm_bo.h>
 #include <drm/ttm/ttm_placement.h>
-#include "vmwgfx_so.h"
-#include "vmwgfx_binding.h"
-#include "vmwgfx_mksstat.h"
 
+#include <linux/sync_file.h>
+#include <linux/hashtable.h>
 
 /*
  * Helper macro to get dx_ctx_node if available otherwise print an error
@@ -65,7 +65,7 @@
  */
 struct vmw_relocation {
 	struct list_head head;
-	struct vmw_buffer_object *vbo;
+	struct vmw_bo *vbo;
 	union {
 		SVGAMobId *mob_loc;
 		SVGAGuestPtr *location;
@@ -149,7 +149,7 @@ static int vmw_resource_context_res_add(struct vmw_private *dev_priv,
 static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 SVGAMobId *id,
-				 struct vmw_buffer_object **vmw_bo_p);
+				 struct vmw_bo **vmw_bo_p);
 /**
  * vmw_ptr_diff - Compute the offset from a to b in bytes
  *
@@ -475,12 +475,16 @@ static int vmw_resource_context_res_add(struct vmw_private *dev_priv,
 
 	if (has_sm4_context(dev_priv) &&
 	    vmw_res_type(ctx) == vmw_res_dx_context) {
-		struct vmw_buffer_object *dx_query_mob;
+		struct vmw_bo *dx_query_mob;
 
 		dx_query_mob = vmw_context_get_dx_query_mob(ctx);
-		if (dx_query_mob)
+		if (dx_query_mob) {
+			vmw_bo_placement_set(dx_query_mob,
+					     VMW_BO_DOMAIN_MOB,
+					     VMW_BO_DOMAIN_MOB);
 			ret = vmw_validation_add_bo(sw_context->ctx,
-						    dx_query_mob, true, false);
+						    dx_query_mob);
+		}
 	}
 
 	mutex_unlock(&dev_priv->binding_mutex);
@@ -596,7 +600,7 @@ static int vmw_resources_reserve(struct vmw_sw_context *sw_context)
 		return ret;
 
 	if (sw_context->dx_query_mob) {
-		struct vmw_buffer_object *expected_dx_query_mob;
+		struct vmw_bo *expected_dx_query_mob;
 
 		expected_dx_query_mob =
 			vmw_context_get_dx_query_mob(sw_context->dx_query_ctx);
@@ -703,7 +707,7 @@ res_check_done:
 static int vmw_rebind_all_dx_query(struct vmw_resource *ctx_res)
 {
 	struct vmw_private *dev_priv = ctx_res->dev_priv;
-	struct vmw_buffer_object *dx_query_mob;
+	struct vmw_bo *dx_query_mob;
 	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXBindAllQuery);
 
 	dx_query_mob = vmw_context_get_dx_query_mob(ctx_res);
@@ -718,7 +722,7 @@ static int vmw_rebind_all_dx_query(struct vmw_resource *ctx_res)
 	cmd->header.id = SVGA_3D_CMD_DX_BIND_ALL_QUERY;
 	cmd->header.size = sizeof(cmd->body);
 	cmd->body.cid = ctx_res->id;
-	cmd->body.mobid = dx_query_mob->base.resource->start;
+	cmd->body.mobid = dx_query_mob->tbo.resource->start;
 	vmw_cmd_commit(dev_priv, sizeof(*cmd));
 
 	vmw_context_bind_dx_query(ctx_res, dx_query_mob);
@@ -1017,7 +1021,7 @@ static int vmw_cmd_present_check(struct vmw_private *dev_priv,
  * after successful submission of the current command batch.
  */
 static int vmw_query_bo_switch_prepare(struct vmw_private *dev_priv,
-				       struct vmw_buffer_object *new_query_bo,
+				       struct vmw_bo *new_query_bo,
 				       struct vmw_sw_context *sw_context)
 {
 	struct vmw_res_cache_entry *ctx_entry =
@@ -1029,24 +1033,24 @@ static int vmw_query_bo_switch_prepare(struct vmw_private *dev_priv,
 
 	if (unlikely(new_query_bo != sw_context->cur_query_bo)) {
 
-		if (unlikely(PFN_UP(new_query_bo->base.resource->size) > 4)) {
+		if (unlikely(PFN_UP(new_query_bo->tbo.resource->size) > 4)) {
 			VMW_DEBUG_USER("Query buffer too large.\n");
 			return -EINVAL;
 		}
 
 		if (unlikely(sw_context->cur_query_bo != NULL)) {
 			sw_context->needs_post_query_barrier = true;
+			vmw_bo_placement_set_default_accelerated(sw_context->cur_query_bo);
 			ret = vmw_validation_add_bo(sw_context->ctx,
-						    sw_context->cur_query_bo,
-						    dev_priv->has_mob, false);
+						    sw_context->cur_query_bo);
 			if (unlikely(ret != 0))
 				return ret;
 		}
 		sw_context->cur_query_bo = new_query_bo;
 
+		vmw_bo_placement_set_default_accelerated(dev_priv->dummy_query_bo);
 		ret = vmw_validation_add_bo(sw_context->ctx,
-					    dev_priv->dummy_query_bo,
-					    dev_priv->has_mob, false);
+					    dev_priv->dummy_query_bo);
 		if (unlikely(ret != 0))
 			return ret;
 	}
@@ -1145,9 +1149,9 @@ static void vmw_query_bo_switch_commit(struct vmw_private *dev_priv,
 static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 SVGAMobId *id,
-				 struct vmw_buffer_object **vmw_bo_p)
+				 struct vmw_bo **vmw_bo_p)
 {
-	struct vmw_buffer_object *vmw_bo;
+	struct vmw_bo *vmw_bo;
 	uint32_t handle = *id;
 	struct vmw_relocation *reloc;
 	int ret;
@@ -1158,9 +1162,10 @@ static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 		drm_dbg(&dev_priv->drm, "Could not find or use MOB buffer.\n");
 		return PTR_ERR(vmw_bo);
 	}
-	ret = vmw_validation_add_bo(sw_context->ctx, vmw_bo, true, false);
-	ttm_bo_put(&vmw_bo->base);
-	drm_gem_object_put(&vmw_bo->base.base);
+	vmw_bo_placement_set(vmw_bo, VMW_BO_DOMAIN_MOB, VMW_BO_DOMAIN_MOB);
+	ret = vmw_validation_add_bo(sw_context->ctx, vmw_bo);
+	ttm_bo_put(&vmw_bo->tbo);
+	drm_gem_object_put(&vmw_bo->tbo.base);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -1200,9 +1205,9 @@ static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 static int vmw_translate_guest_ptr(struct vmw_private *dev_priv,
 				   struct vmw_sw_context *sw_context,
 				   SVGAGuestPtr *ptr,
-				   struct vmw_buffer_object **vmw_bo_p)
+				   struct vmw_bo **vmw_bo_p)
 {
-	struct vmw_buffer_object *vmw_bo;
+	struct vmw_bo *vmw_bo;
 	uint32_t handle = ptr->gmrId;
 	struct vmw_relocation *reloc;
 	int ret;
@@ -1213,9 +1218,11 @@ static int vmw_translate_guest_ptr(struct vmw_private *dev_priv,
 		drm_dbg(&dev_priv->drm, "Could not find or use GMR region.\n");
 		return PTR_ERR(vmw_bo);
 	}
-	ret = vmw_validation_add_bo(sw_context->ctx, vmw_bo, false, false);
-	ttm_bo_put(&vmw_bo->base);
-	drm_gem_object_put(&vmw_bo->base.base);
+	vmw_bo_placement_set(vmw_bo, VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM,
+			     VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM);
+	ret = vmw_validation_add_bo(sw_context->ctx, vmw_bo);
+	ttm_bo_put(&vmw_bo->tbo);
+	drm_gem_object_put(&vmw_bo->tbo.base);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -1280,7 +1287,7 @@ static int vmw_cmd_dx_bind_query(struct vmw_private *dev_priv,
 				 SVGA3dCmdHeader *header)
 {
 	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXBindQuery);
-	struct vmw_buffer_object *vmw_bo;
+	struct vmw_bo *vmw_bo;
 	int ret;
 
 	cmd = container_of(header, typeof(*cmd), header);
@@ -1363,7 +1370,7 @@ static int vmw_cmd_end_gb_query(struct vmw_private *dev_priv,
 				struct vmw_sw_context *sw_context,
 				SVGA3dCmdHeader *header)
 {
-	struct vmw_buffer_object *vmw_bo;
+	struct vmw_bo *vmw_bo;
 	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdEndGBQuery);
 	int ret;
 
@@ -1393,7 +1400,7 @@ static int vmw_cmd_end_query(struct vmw_private *dev_priv,
 			     struct vmw_sw_context *sw_context,
 			     SVGA3dCmdHeader *header)
 {
-	struct vmw_buffer_object *vmw_bo;
+	struct vmw_bo *vmw_bo;
 	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdEndQuery);
 	int ret;
 
@@ -1439,7 +1446,7 @@ static int vmw_cmd_wait_gb_query(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 SVGA3dCmdHeader *header)
 {
-	struct vmw_buffer_object *vmw_bo;
+	struct vmw_bo *vmw_bo;
 	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdWaitForGBQuery);
 	int ret;
 
@@ -1467,7 +1474,7 @@ static int vmw_cmd_wait_query(struct vmw_private *dev_priv,
 			      struct vmw_sw_context *sw_context,
 			      SVGA3dCmdHeader *header)
 {
-	struct vmw_buffer_object *vmw_bo;
+	struct vmw_bo *vmw_bo;
 	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdWaitForQuery);
 	int ret;
 
@@ -1504,7 +1511,7 @@ static int vmw_cmd_dma(struct vmw_private *dev_priv,
 		       struct vmw_sw_context *sw_context,
 		       SVGA3dCmdHeader *header)
 {
-	struct vmw_buffer_object *vmw_bo = NULL;
+	struct vmw_bo *vmw_bo = NULL;
 	struct vmw_surface *srf = NULL;
 	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdSurfaceDMA);
 	int ret;
@@ -1528,7 +1535,7 @@ static int vmw_cmd_dma(struct vmw_private *dev_priv,
 		return ret;
 
 	/* Make sure DMA doesn't cross BO boundaries. */
-	bo_size = vmw_bo->base.base.size;
+	bo_size = vmw_bo->tbo.base.size;
 	if (unlikely(cmd->body.guest.ptr.offset > bo_size)) {
 		VMW_DEBUG_USER("Invalid DMA offset.\n");
 		return -EINVAL;
@@ -1551,7 +1558,7 @@ static int vmw_cmd_dma(struct vmw_private *dev_priv,
 
 	srf = vmw_res_to_srf(sw_context->res_cache[vmw_res_surface].res);
 
-	vmw_kms_cursor_snoop(srf, sw_context->fp->tfile, &vmw_bo->base, header);
+	vmw_kms_cursor_snoop(srf, sw_context->fp->tfile, &vmw_bo->tbo, header);
 
 	return 0;
 }
@@ -1670,7 +1677,7 @@ static int vmw_cmd_check_define_gmrfb(struct vmw_private *dev_priv,
 				      struct vmw_sw_context *sw_context,
 				      void *buf)
 {
-	struct vmw_buffer_object *vmw_bo;
+	struct vmw_bo *vmw_bo;
 
 	struct {
 		uint32_t header;
@@ -1701,7 +1708,7 @@ static int vmw_cmd_res_switch_backup(struct vmw_private *dev_priv,
 				     struct vmw_resource *res, uint32_t *buf_id,
 				     unsigned long backup_offset)
 {
-	struct vmw_buffer_object *vbo;
+	struct vmw_bo *vbo;
 	void *info;
 	int ret;
 
@@ -3754,7 +3761,7 @@ static void vmw_apply_relocations(struct vmw_sw_context *sw_context)
 	struct ttm_buffer_object *bo;
 
 	list_for_each_entry(reloc, &sw_context->bo_relocations, head) {
-		bo = &reloc->vbo->base;
+		bo = &reloc->vbo->tbo;
 		switch (bo->resource->mem_type) {
 		case TTM_PL_VRAM:
 			reloc->location->offset += bo->resource->start << PAGE_SHIFT;
@@ -4364,13 +4371,17 @@ void __vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv,
 	if (dev_priv->pinned_bo == NULL)
 		goto out_unlock;
 
-	ret = vmw_validation_add_bo(&val_ctx, dev_priv->pinned_bo, false,
-				    false);
+	vmw_bo_placement_set(dev_priv->pinned_bo,
+			     VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM,
+			     VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM);
+	ret = vmw_validation_add_bo(&val_ctx, dev_priv->pinned_bo);
 	if (ret)
 		goto out_no_reserve;
 
-	ret = vmw_validation_add_bo(&val_ctx, dev_priv->dummy_query_bo, false,
-				    false);
+	vmw_bo_placement_set(dev_priv->dummy_query_bo,
+			     VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM,
+			     VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM);
+	ret = vmw_validation_add_bo(&val_ctx, dev_priv->dummy_query_bo);
 	if (ret)
 		goto out_no_reserve;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
index 66cc35dc223e..2a0cda324703 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2011-2014 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2011-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_gem.c b/drivers/gpu/drm/vmwgfx/vmwgfx_gem.c
index 4d2c28e39f4e..c0da89e16e6f 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_gem.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_gem.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /*
- * Copyright 2021 VMware, Inc.
+ * Copyright 2021-2023 VMware, Inc.
  *
  * Permission is hereby granted, free of charge, to any person
  * obtaining a copy of this software and associated documentation
@@ -24,31 +24,17 @@
  *
  */
 
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 
 #include "drm/drm_prime.h"
 #include "drm/drm_gem_ttm_helper.h"
 
-/**
- * vmw_buffer_object - Convert a struct ttm_buffer_object to a struct
- * vmw_buffer_object.
- *
- * @bo: Pointer to the TTM buffer object.
- * Return: Pointer to the struct vmw_buffer_object embedding the
- * TTM buffer object.
- */
-static struct vmw_buffer_object *
-vmw_buffer_object(struct ttm_buffer_object *bo)
-{
-	return container_of(bo, struct vmw_buffer_object, base);
-}
-
 static void vmw_gem_object_free(struct drm_gem_object *gobj)
 {
 	struct ttm_buffer_object *bo = drm_gem_ttm_of_gem(gobj);
-	if (bo) {
+	if (bo)
 		ttm_bo_put(bo);
-	}
 }
 
 static int vmw_gem_object_open(struct drm_gem_object *obj,
@@ -65,7 +51,7 @@ static void vmw_gem_object_close(struct drm_gem_object *obj,
 static int vmw_gem_pin_private(struct drm_gem_object *obj, bool do_pin)
 {
 	struct ttm_buffer_object *bo = drm_gem_ttm_of_gem(obj);
-	struct vmw_buffer_object *vbo = vmw_buffer_object(bo);
+	struct vmw_bo *vbo = to_vmw_bo(obj);
 	int ret;
 
 	ret = ttm_bo_reserve(bo, false, false, NULL);
@@ -103,6 +89,13 @@ static struct sg_table *vmw_gem_object_get_sg_table(struct drm_gem_object *obj)
 	return drm_prime_pages_to_sg(obj->dev, vmw_tt->dma_ttm.pages, vmw_tt->dma_ttm.num_pages);
 }
 
+static const struct vm_operations_struct vmw_vm_ops = {
+	.pfn_mkwrite = vmw_bo_vm_mkwrite,
+	.page_mkwrite = vmw_bo_vm_mkwrite,
+	.fault = vmw_bo_vm_fault,
+	.open = ttm_bo_vm_open,
+	.close = ttm_bo_vm_close,
+};
 
 static const struct drm_gem_object_funcs vmw_gem_object_funcs = {
 	.free = vmw_gem_object_free,
@@ -115,43 +108,31 @@ static const struct drm_gem_object_funcs vmw_gem_object_funcs = {
 	.vmap = drm_gem_ttm_vmap,
 	.vunmap = drm_gem_ttm_vunmap,
 	.mmap = drm_gem_ttm_mmap,
+	.vm_ops = &vmw_vm_ops,
 };
 
-/**
- * vmw_gem_destroy - vmw buffer object destructor
- *
- * @bo: Pointer to the embedded struct ttm_buffer_object
- */
-void vmw_gem_destroy(struct ttm_buffer_object *bo)
-{
-	struct vmw_buffer_object *vbo = vmw_buffer_object(bo);
-
-	WARN_ON(vbo->dirty);
-	WARN_ON(!RB_EMPTY_ROOT(&vbo->res_tree));
-	vmw_bo_unmap(vbo);
-	drm_gem_object_release(&vbo->base.base);
-	kfree(vbo);
-}
-
 int vmw_gem_object_create_with_handle(struct vmw_private *dev_priv,
 				      struct drm_file *filp,
 				      uint32_t size,
 				      uint32_t *handle,
-				      struct vmw_buffer_object **p_vbo)
+				      struct vmw_bo **p_vbo)
 {
 	int ret;
-
-	ret = vmw_bo_create(dev_priv, size,
-			    (dev_priv->has_mob) ?
-				    &vmw_sys_placement :
-				    &vmw_vram_sys_placement,
-			    true, false, &vmw_gem_destroy, p_vbo);
+	struct vmw_bo_params params = {
+		.domain = (dev_priv->has_mob) ? VMW_BO_DOMAIN_SYS : VMW_BO_DOMAIN_VRAM,
+		.busy_domain = VMW_BO_DOMAIN_SYS,
+		.bo_type = ttm_bo_type_device,
+		.size = size,
+		.pin = false
+	};
+
+	ret = vmw_bo_create(dev_priv, &params, p_vbo);
 	if (ret != 0)
 		goto out_no_bo;
 
-	(*p_vbo)->base.base.funcs = &vmw_gem_object_funcs;
+	(*p_vbo)->tbo.base.funcs = &vmw_gem_object_funcs;
 
-	ret = drm_gem_handle_create(filp, &(*p_vbo)->base.base, handle);
+	ret = drm_gem_handle_create(filp, &(*p_vbo)->tbo.base, handle);
 out_no_bo:
 	return ret;
 }
@@ -165,7 +146,7 @@ int vmw_gem_object_create_ioctl(struct drm_device *dev, void *data,
 	    (union drm_vmw_alloc_dmabuf_arg *)data;
 	struct drm_vmw_alloc_dmabuf_req *req = &arg->req;
 	struct drm_vmw_dmabuf_rep *rep = &arg->rep;
-	struct vmw_buffer_object *vbo;
+	struct vmw_bo *vbo;
 	uint32_t handle;
 	int ret;
 
@@ -175,23 +156,23 @@ int vmw_gem_object_create_ioctl(struct drm_device *dev, void *data,
 		goto out_no_bo;
 
 	rep->handle = handle;
-	rep->map_handle = drm_vma_node_offset_addr(&vbo->base.base.vma_node);
+	rep->map_handle = drm_vma_node_offset_addr(&vbo->tbo.base.vma_node);
 	rep->cur_gmr_id = handle;
 	rep->cur_gmr_offset = 0;
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_put(&vbo->base.base);
+	drm_gem_object_put(&vbo->tbo.base);
 out_no_bo:
 	return ret;
 }
 
 #if defined(CONFIG_DEBUG_FS)
 
-static void vmw_bo_print_info(int id, struct vmw_buffer_object *bo, struct seq_file *m)
+static void vmw_bo_print_info(int id, struct vmw_bo *bo, struct seq_file *m)
 {
 	const char *placement;
 	const char *type;
 
-	switch (bo->base.resource->mem_type) {
+	switch (bo->tbo.resource->mem_type) {
 	case TTM_PL_SYSTEM:
 		placement = " CPU";
 		break;
@@ -212,7 +193,7 @@ static void vmw_bo_print_info(int id, struct vmw_buffer_object *bo, struct seq_f
 		break;
 	}
 
-	switch (bo->base.type) {
+	switch (bo->tbo.type) {
 	case ttm_bo_type_device:
 		type = "device";
 		break;
@@ -228,12 +209,12 @@ static void vmw_bo_print_info(int id, struct vmw_buffer_object *bo, struct seq_f
 	}
 
 	seq_printf(m, "\t\t0x%08x: %12zu bytes %s, type = %s",
-		   id, bo->base.base.size, placement, type);
+		   id, bo->tbo.base.size, placement, type);
 	seq_printf(m, ", priority = %u, pin_count = %u, GEM refs = %d, TTM refs = %d",
-		   bo->base.priority,
-		   bo->base.pin_count,
-		   kref_read(&bo->base.base.refcount),
-		   kref_read(&bo->base.kref));
+		   bo->tbo.priority,
+		   bo->tbo.pin_count,
+		   kref_read(&bo->tbo.base.refcount),
+		   kref_read(&bo->tbo.kref));
 	seq_puts(m, "\n");
 }
 
@@ -260,14 +241,14 @@ static int vmw_debugfs_gem_info_show(struct seq_file *m, void *unused)
 		 * Therefore, we need to protect this ->comm access using RCU.
 		 */
 		rcu_read_lock();
-		task = pid_task(file->pid, PIDTYPE_PID);
+		task = pid_task(file->pid, PIDTYPE_TGID);
 		seq_printf(m, "pid %8d command %s:\n", pid_nr(file->pid),
 			   task ? task->comm : "<unknown>");
 		rcu_read_unlock();
 
 		spin_lock(&file->table_lock);
 		idr_for_each_entry(&file->object_idr, gobj, id) {
-			struct vmw_buffer_object *bo = gem_to_vmw_bo(gobj);
+			struct vmw_bo *bo = to_vmw_bo(gobj);
 
 			vmw_bo_print_info(id, bo, m);
 		}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 445d619e1fdc..b62207be3363 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009-2022 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -24,8 +24,9 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  **************************************************************************/
-
 #include "vmwgfx_kms.h"
+
+#include "vmwgfx_bo.h"
 #include "vmw_surface_cache.h"
 
 #include <drm/drm_atomic.h>
@@ -152,9 +153,8 @@ static void vmw_cursor_update_mob(struct vmw_private *dev_priv,
 	SVGAGBCursorHeader *header;
 	SVGAGBAlphaCursorHeader *alpha_header;
 	const u32 image_size = width * height * sizeof(*image);
-	bool dummy;
 
-	header = ttm_kmap_obj_virtual(&vps->cursor.map, &dummy);
+	header = vmw_bo_map_and_cache(vps->cursor.bo);
 	alpha_header = &header->header.alphaHeader;
 
 	memset(header, 0, sizeof(*header));
@@ -169,7 +169,7 @@ static void vmw_cursor_update_mob(struct vmw_private *dev_priv,
 
 	memcpy(header + 1, image, image_size);
 	vmw_write(dev_priv, SVGA_REG_CURSOR_MOBID,
-		  vps->cursor.bo->resource->start);
+		  vps->cursor.bo->tbo.resource->start);
 }
 
 
@@ -184,13 +184,13 @@ static u32 vmw_du_cursor_mob_size(u32 w, u32 h)
  */
 static u32 *vmw_du_cursor_plane_acquire_image(struct vmw_plane_state *vps)
 {
-	bool dummy;
+	bool is_iomem;
 	if (vps->surf) {
 		if (vps->surf_mapped)
-			return vmw_bo_map_and_cache(vps->surf->res.backup);
+			return vmw_bo_map_and_cache(vps->surf->res.guest_memory_bo);
 		return vps->surf->snooper.image;
 	} else if (vps->bo)
-		return ttm_kmap_obj_virtual(&vps->bo->map, &dummy);
+		return ttm_kmap_obj_virtual(&vps->bo->map, &is_iomem);
 	return NULL;
 }
 
@@ -222,15 +222,13 @@ static bool vmw_du_cursor_plane_has_changed(struct vmw_plane_state *old_vps,
 	return changed;
 }
 
-static void vmw_du_destroy_cursor_mob(struct ttm_buffer_object **bo)
+static void vmw_du_destroy_cursor_mob(struct vmw_bo **vbo)
 {
-	if (!(*bo))
+	if (!(*vbo))
 		return;
 
-	ttm_bo_unpin(*bo);
-	ttm_bo_put(*bo);
-	kfree(*bo);
-	*bo = NULL;
+	ttm_bo_unpin(&(*vbo)->tbo);
+	vmw_bo_unreference(vbo);
 }
 
 static void vmw_du_put_cursor_mob(struct vmw_cursor_plane *vcp,
@@ -254,8 +252,8 @@ static void vmw_du_put_cursor_mob(struct vmw_cursor_plane *vcp,
 
 	/* Cache is full: See if this mob is bigger than an existing mob. */
 	for (i = 0; i < ARRAY_SIZE(vcp->cursor_mobs); i++) {
-		if (vcp->cursor_mobs[i]->base.size <
-		    vps->cursor.bo->base.size) {
+		if (vcp->cursor_mobs[i]->tbo.base.size <
+		    vps->cursor.bo->tbo.base.size) {
 			vmw_du_destroy_cursor_mob(&vcp->cursor_mobs[i]);
 			vcp->cursor_mobs[i] = vps->cursor.bo;
 			vps->cursor.bo = NULL;
@@ -288,7 +286,7 @@ static int vmw_du_get_cursor_mob(struct vmw_cursor_plane *vcp,
 		return -EINVAL;
 
 	if (vps->cursor.bo) {
-		if (vps->cursor.bo->base.size >= size)
+		if (vps->cursor.bo->tbo.base.size >= size)
 			return 0;
 		vmw_du_put_cursor_mob(vcp, vps);
 	}
@@ -296,26 +294,27 @@ static int vmw_du_get_cursor_mob(struct vmw_cursor_plane *vcp,
 	/* Look for an unused mob in the cache. */
 	for (i = 0; i < ARRAY_SIZE(vcp->cursor_mobs); i++) {
 		if (vcp->cursor_mobs[i] &&
-		    vcp->cursor_mobs[i]->base.size >= size) {
+		    vcp->cursor_mobs[i]->tbo.base.size >= size) {
 			vps->cursor.bo = vcp->cursor_mobs[i];
 			vcp->cursor_mobs[i] = NULL;
 			return 0;
 		}
 	}
 	/* Create a new mob if we can't find an existing one. */
-	ret = vmw_bo_create_kernel(dev_priv, size, &vmw_mob_placement,
-				   &vps->cursor.bo);
+	ret = vmw_bo_create_and_populate(dev_priv, size,
+					 VMW_BO_DOMAIN_MOB,
+					 &vps->cursor.bo);
 
 	if (ret != 0)
 		return ret;
 
 	/* Fence the mob creation so we are guarateed to have the mob */
-	ret = ttm_bo_reserve(vps->cursor.bo, false, false, NULL);
+	ret = ttm_bo_reserve(&vps->cursor.bo->tbo, false, false, NULL);
 	if (ret != 0)
 		goto teardown;
 
-	vmw_bo_fence_single(vps->cursor.bo, NULL);
-	ttm_bo_unreserve(vps->cursor.bo);
+	vmw_bo_fence_single(&vps->cursor.bo->tbo, NULL);
+	ttm_bo_unreserve(&vps->cursor.bo->tbo);
 	return 0;
 
 teardown:
@@ -363,7 +362,7 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf,
 	SVGA3dCopyBox *box;
 	unsigned box_count;
 	void *virtual;
-	bool dummy;
+	bool is_iomem;
 	struct vmw_dma_cmd {
 		SVGA3dCmdHeader header;
 		SVGA3dCmdSurfaceDMA dma;
@@ -423,7 +422,7 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf,
 	if (unlikely(ret != 0))
 		goto err_unreserve;
 
-	virtual = ttm_kmap_obj_virtual(&map, &dummy);
+	virtual = ttm_kmap_obj_virtual(&map, &is_iomem);
 
 	if (box->w == VMW_CURSOR_SNOOP_WIDTH && cmd->dma.guest.pitch == image_pitch) {
 		memcpy(srf->snooper.image, virtual,
@@ -573,39 +572,30 @@ vmw_du_cursor_plane_map_cm(struct vmw_plane_state *vps)
 {
 	int ret;
 	u32 size = vmw_du_cursor_mob_size(vps->base.crtc_w, vps->base.crtc_h);
-	struct ttm_buffer_object *bo = vps->cursor.bo;
+	struct ttm_buffer_object *bo;
 
-	if (!bo)
+	if (!vps->cursor.bo)
 		return -EINVAL;
 
+	bo = &vps->cursor.bo->tbo;
+
 	if (bo->base.size < size)
 		return -EINVAL;
 
-	if (vps->cursor.mapped)
+	if (vps->cursor.bo->map.virtual)
 		return 0;
 
 	ret = ttm_bo_reserve(bo, false, false, NULL);
-
 	if (unlikely(ret != 0))
 		return -ENOMEM;
 
-	ret = ttm_bo_kmap(bo, 0, PFN_UP(size), &vps->cursor.map);
-
-	/*
-	 * We just want to try to get mob bind to finish
-	 * so that the first write to SVGA_REG_CURSOR_MOBID
-	 * is done with a buffer that the device has already
-	 * seen
-	 */
-	(void) ttm_bo_wait(bo, false, false);
+	vmw_bo_map_and_cache(vps->cursor.bo);
 
 	ttm_bo_unreserve(bo);
 
 	if (unlikely(ret != 0))
 		return -ENOMEM;
 
-	vps->cursor.mapped = true;
-
 	return 0;
 }
 
@@ -622,19 +612,15 @@ static int
 vmw_du_cursor_plane_unmap_cm(struct vmw_plane_state *vps)
 {
 	int ret = 0;
-	struct ttm_buffer_object *bo = vps->cursor.bo;
-
-	if (!vps->cursor.mapped)
-		return 0;
+	struct vmw_bo *vbo = vps->cursor.bo;
 
-	if (!bo)
+	if (!vbo || !vbo->map.virtual)
 		return 0;
 
-	ret = ttm_bo_reserve(bo, true, false, NULL);
+	ret = ttm_bo_reserve(&vbo->tbo, true, false, NULL);
 	if (likely(ret == 0)) {
-		ttm_bo_kunmap(&vps->cursor.map);
-		ttm_bo_unreserve(bo);
-		vps->cursor.mapped = false;
+		vmw_bo_unmap(vbo);
+		ttm_bo_unreserve(&vbo->tbo);
 	}
 
 	return ret;
@@ -657,20 +643,19 @@ vmw_du_cursor_plane_cleanup_fb(struct drm_plane *plane,
 {
 	struct vmw_cursor_plane *vcp = vmw_plane_to_vcp(plane);
 	struct vmw_plane_state *vps = vmw_plane_state_to_vps(old_state);
-	bool dummy;
+	bool is_iomem;
 
 	if (vps->surf_mapped) {
-		vmw_bo_unmap(vps->surf->res.backup);
+		vmw_bo_unmap(vps->surf->res.guest_memory_bo);
 		vps->surf_mapped = false;
 	}
 
-	if (vps->bo && ttm_kmap_obj_virtual(&vps->bo->map, &dummy)) {
-		const int ret = ttm_bo_reserve(&vps->bo->base, true, false, NULL);
+	if (vps->bo && ttm_kmap_obj_virtual(&vps->bo->map, &is_iomem)) {
+		const int ret = ttm_bo_reserve(&vps->bo->tbo, true, false, NULL);
 
 		if (likely(ret == 0)) {
-			if (atomic_read(&vps->bo->base_mapped_count) == 0)
-			    ttm_bo_kunmap(&vps->bo->map);
-			ttm_bo_unreserve(&vps->bo->base);
+			ttm_bo_kunmap(&vps->bo->map);
+			ttm_bo_unreserve(&vps->bo->tbo);
 		}
 	}
 
@@ -736,29 +721,26 @@ vmw_du_cursor_plane_prepare_fb(struct drm_plane *plane,
 		 * reserve the ttm_buffer_object first which
 		 * vmw_bo_map_and_cache() omits.
 		 */
-		ret = ttm_bo_reserve(&vps->bo->base, true, false, NULL);
+		ret = ttm_bo_reserve(&vps->bo->tbo, true, false, NULL);
 
 		if (unlikely(ret != 0))
 			return -ENOMEM;
 
-		ret = ttm_bo_kmap(&vps->bo->base, 0, PFN_UP(size), &vps->bo->map);
-
-		if (likely(ret == 0))
-			atomic_inc(&vps->bo->base_mapped_count);
+		ret = ttm_bo_kmap(&vps->bo->tbo, 0, PFN_UP(size), &vps->bo->map);
 
-		ttm_bo_unreserve(&vps->bo->base);
+		ttm_bo_unreserve(&vps->bo->tbo);
 
 		if (unlikely(ret != 0))
 			return -ENOMEM;
-	} else if (vps->surf && !vps->bo && vps->surf->res.backup) {
+	} else if (vps->surf && !vps->bo && vps->surf->res.guest_memory_bo) {
 
 		WARN_ON(vps->surf->snooper.image);
-		ret = ttm_bo_reserve(&vps->surf->res.backup->base, true, false,
+		ret = ttm_bo_reserve(&vps->surf->res.guest_memory_bo->tbo, true, false,
 				     NULL);
 		if (unlikely(ret != 0))
 			return -ENOMEM;
-		vmw_bo_map_and_cache(vps->surf->res.backup);
-		ttm_bo_unreserve(&vps->surf->res.backup->base);
+		vmw_bo_map_and_cache(vps->surf->res.guest_memory_bo);
+		ttm_bo_unreserve(&vps->surf->res.guest_memory_bo->tbo);
 		vps->surf_mapped = true;
 	}
 
@@ -785,7 +767,6 @@ vmw_du_cursor_plane_atomic_update(struct drm_plane *plane,
 	struct vmw_plane_state *vps = vmw_plane_state_to_vps(new_state);
 	struct vmw_plane_state *old_vps = vmw_plane_state_to_vps(old_state);
 	s32 hotspot_x, hotspot_y;
-	bool dummy;
 
 	hotspot_x = du->hotspot_x;
 	hotspot_y = du->hotspot_y;
@@ -827,11 +808,6 @@ vmw_du_cursor_plane_atomic_update(struct drm_plane *plane,
 						hotspot_x, hotspot_y);
 	}
 
-	if (vps->bo) {
-		if (ttm_kmap_obj_virtual(&vps->bo->map, &dummy))
-			atomic_dec(&vps->bo->base_mapped_count);
-	}
-
 	du->cursor_x = new_state->crtc_x + du->set_gui_x;
 	du->cursor_y = new_state->crtc_y + du->set_gui_y;
 
@@ -935,7 +911,7 @@ int vmw_du_cursor_plane_atomic_check(struct drm_plane *plane,
 		WARN_ON(!surface);
 
 		if (!surface ||
-		    (!surface->snooper.image && !surface->res.backup)) {
+		    (!surface->snooper.image && !surface->res.guest_memory_bo)) {
 			DRM_ERROR("surface not suitable for cursor\n");
 			return -EINVAL;
 		}
@@ -1279,9 +1255,9 @@ int vmw_kms_readback(struct vmw_private *dev_priv,
 					    user_fence_rep, vclips, num_clips,
 					    NULL);
 	case vmw_du_screen_target:
-		return vmw_kms_stdu_dma(dev_priv, file_priv, vfb,
-					user_fence_rep, NULL, vclips, num_clips,
-					1, false, true, NULL);
+		return vmw_kms_stdu_readback(dev_priv, file_priv, vfb,
+					     user_fence_rep, NULL, vclips, num_clips,
+					     1, NULL);
 	default:
 		WARN_ONCE(true,
 			  "Readback called with invalid display system.\n");
@@ -1406,7 +1382,7 @@ static int vmw_framebuffer_bo_create_handle(struct drm_framebuffer *fb,
 	struct vmw_framebuffer_bo *vfbd =
 			vmw_framebuffer_to_vfbd(fb);
 
-	return drm_gem_handle_create(file_priv, &vfbd->buffer->base.base, handle);
+	return drm_gem_handle_create(file_priv, &vfbd->buffer->tbo.base, handle);
 }
 
 static void vmw_framebuffer_bo_destroy(struct drm_framebuffer *framebuffer)
@@ -1420,135 +1396,12 @@ static void vmw_framebuffer_bo_destroy(struct drm_framebuffer *framebuffer)
 	kfree(vfbd);
 }
 
-static int vmw_framebuffer_bo_dirty(struct drm_framebuffer *framebuffer,
-				    struct drm_file *file_priv,
-				    unsigned int flags, unsigned int color,
-				    struct drm_clip_rect *clips,
-				    unsigned int num_clips)
-{
-	struct vmw_private *dev_priv = vmw_priv(framebuffer->dev);
-	struct vmw_framebuffer_bo *vfbd =
-		vmw_framebuffer_to_vfbd(framebuffer);
-	struct drm_clip_rect norect;
-	int ret, increment = 1;
-
-	drm_modeset_lock_all(&dev_priv->drm);
-
-	if (!num_clips) {
-		num_clips = 1;
-		clips = &norect;
-		norect.x1 = norect.y1 = 0;
-		norect.x2 = framebuffer->width;
-		norect.y2 = framebuffer->height;
-	} else if (flags & DRM_MODE_FB_DIRTY_ANNOTATE_COPY) {
-		num_clips /= 2;
-		increment = 2;
-	}
-
-	switch (dev_priv->active_display_unit) {
-	case vmw_du_legacy:
-		ret = vmw_kms_ldu_do_bo_dirty(dev_priv, &vfbd->base, 0, 0,
-					      clips, num_clips, increment);
-		break;
-	default:
-		ret = -EINVAL;
-		WARN_ONCE(true, "Dirty called with invalid display system.\n");
-		break;
-	}
-
-	vmw_cmd_flush(dev_priv, false);
-
-	drm_modeset_unlock_all(&dev_priv->drm);
-
-	return ret;
-}
-
-static int vmw_framebuffer_bo_dirty_ext(struct drm_framebuffer *framebuffer,
-					struct drm_file *file_priv,
-					unsigned int flags, unsigned int color,
-					struct drm_clip_rect *clips,
-					unsigned int num_clips)
-{
-	struct vmw_private *dev_priv = vmw_priv(framebuffer->dev);
-
-	if (dev_priv->active_display_unit == vmw_du_legacy &&
-	    vmw_cmd_supported(dev_priv))
-		return vmw_framebuffer_bo_dirty(framebuffer, file_priv, flags,
-						color, clips, num_clips);
-
-	return drm_atomic_helper_dirtyfb(framebuffer, file_priv, flags, color,
-					 clips, num_clips);
-}
-
 static const struct drm_framebuffer_funcs vmw_framebuffer_bo_funcs = {
 	.create_handle = vmw_framebuffer_bo_create_handle,
 	.destroy = vmw_framebuffer_bo_destroy,
-	.dirty = vmw_framebuffer_bo_dirty_ext,
+	.dirty = drm_atomic_helper_dirtyfb,
 };
 
-/*
- * Pin the bofer in a location suitable for access by the
- * display system.
- */
-static int vmw_framebuffer_pin(struct vmw_framebuffer *vfb)
-{
-	struct vmw_private *dev_priv = vmw_priv(vfb->base.dev);
-	struct vmw_buffer_object *buf;
-	struct ttm_placement *placement;
-	int ret;
-
-	buf = vfb->bo ?  vmw_framebuffer_to_vfbd(&vfb->base)->buffer :
-		vmw_framebuffer_to_vfbs(&vfb->base)->surface->res.backup;
-
-	if (!buf)
-		return 0;
-
-	switch (dev_priv->active_display_unit) {
-	case vmw_du_legacy:
-		vmw_overlay_pause_all(dev_priv);
-		ret = vmw_bo_pin_in_start_of_vram(dev_priv, buf, false);
-		vmw_overlay_resume_all(dev_priv);
-		break;
-	case vmw_du_screen_object:
-	case vmw_du_screen_target:
-		if (vfb->bo) {
-			if (dev_priv->capabilities & SVGA_CAP_3D) {
-				/*
-				 * Use surface DMA to get content to
-				 * sreen target surface.
-				 */
-				placement = &vmw_vram_gmr_placement;
-			} else {
-				/* Use CPU blit. */
-				placement = &vmw_sys_placement;
-			}
-		} else {
-			/* Use surface / image update */
-			placement = &vmw_mob_placement;
-		}
-
-		return vmw_bo_pin_in_placement(dev_priv, buf, placement, false);
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-static int vmw_framebuffer_unpin(struct vmw_framebuffer *vfb)
-{
-	struct vmw_private *dev_priv = vmw_priv(vfb->base.dev);
-	struct vmw_buffer_object *buf;
-
-	buf = vfb->bo ?  vmw_framebuffer_to_vfbd(&vfb->base)->buffer :
-		vmw_framebuffer_to_vfbs(&vfb->base)->surface->res.backup;
-
-	if (WARN_ON(!buf))
-		return 0;
-
-	return vmw_bo_unpin(dev_priv, buf, false);
-}
-
 /**
  * vmw_create_bo_proxy - create a proxy surface for the buffer object
  *
@@ -1566,7 +1419,7 @@ static int vmw_framebuffer_unpin(struct vmw_framebuffer *vfb)
  */
 static int vmw_create_bo_proxy(struct drm_device *dev,
 			       const struct drm_mode_fb_cmd2 *mode_cmd,
-			       struct vmw_buffer_object *bo_mob,
+			       struct vmw_bo *bo_mob,
 			       struct vmw_surface **srf_out)
 {
 	struct vmw_surface_metadata metadata = {0};
@@ -1618,9 +1471,9 @@ static int vmw_create_bo_proxy(struct drm_device *dev,
 	/* Reserve and switch the backing mob. */
 	mutex_lock(&res->dev_priv->cmdbuf_mutex);
 	(void) vmw_resource_reserve(res, false, true);
-	vmw_bo_unreference(&res->backup);
-	res->backup = vmw_bo_reference(bo_mob);
-	res->backup_offset = 0;
+	vmw_bo_unreference(&res->guest_memory_bo);
+	res->guest_memory_bo = vmw_bo_reference(bo_mob);
+	res->guest_memory_offset = 0;
 	vmw_resource_unreserve(res, false, false, false, NULL, 0);
 	mutex_unlock(&res->dev_priv->cmdbuf_mutex);
 
@@ -1630,7 +1483,7 @@ static int vmw_create_bo_proxy(struct drm_device *dev,
 
 
 static int vmw_kms_new_framebuffer_bo(struct vmw_private *dev_priv,
-				      struct vmw_buffer_object *bo,
+				      struct vmw_bo *bo,
 				      struct vmw_framebuffer **out,
 				      const struct drm_mode_fb_cmd2
 				      *mode_cmd)
@@ -1642,7 +1495,7 @@ static int vmw_kms_new_framebuffer_bo(struct vmw_private *dev_priv,
 	int ret;
 
 	requested_size = mode_cmd->height * mode_cmd->pitches[0];
-	if (unlikely(requested_size > bo->base.base.size)) {
+	if (unlikely(requested_size > bo->tbo.base.size)) {
 		DRM_ERROR("Screen buffer object size is too small "
 			  "for requested mode.\n");
 		return -EINVAL;
@@ -1663,7 +1516,7 @@ static int vmw_kms_new_framebuffer_bo(struct vmw_private *dev_priv,
 		goto out_err1;
 	}
 
-	vfbd->base.base.obj[0] = &bo->base.base;
+	vfbd->base.base.obj[0] = &bo->tbo.base;
 	drm_helper_mode_fill_fb_struct(dev, &vfbd->base.base, mode_cmd);
 	vfbd->base.bo = true;
 	vfbd->buffer = vmw_bo_reference(bo);
@@ -1718,7 +1571,7 @@ vmw_kms_srf_ok(struct vmw_private *dev_priv, uint32_t width, uint32_t height)
  */
 struct vmw_framebuffer *
 vmw_kms_new_framebuffer(struct vmw_private *dev_priv,
-			struct vmw_buffer_object *bo,
+			struct vmw_bo *bo,
 			struct vmw_surface *surface,
 			bool only_2d,
 			const struct drm_mode_fb_cmd2 *mode_cmd)
@@ -1765,9 +1618,6 @@ vmw_kms_new_framebuffer(struct vmw_private *dev_priv,
 	if (ret)
 		return ERR_PTR(ret);
 
-	vfb->pin = vmw_framebuffer_pin;
-	vfb->unpin = vmw_framebuffer_unpin;
-
 	return vfb;
 }
 
@@ -1782,7 +1632,7 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	struct vmw_private *dev_priv = vmw_priv(dev);
 	struct vmw_framebuffer *vfb = NULL;
 	struct vmw_surface *surface = NULL;
-	struct vmw_buffer_object *bo = NULL;
+	struct vmw_bo *bo = NULL;
 	int ret;
 
 	/* returns either a bo or surface */
@@ -1817,7 +1667,7 @@ err_out:
 	/* vmw_user_lookup_handle takes one ref so does new_fb */
 	if (bo) {
 		vmw_bo_unreference(&bo);
-		drm_gem_object_put(&bo->base.base);
+		drm_gem_object_put(&bo->tbo.base);
 	}
 	if (surface)
 		vmw_surface_unreference(&surface);
@@ -2202,7 +2052,6 @@ int vmw_kms_init(struct vmw_private *dev_priv)
 	dev->mode_config.max_width = dev_priv->texture_max_width;
 	dev->mode_config.max_height = dev_priv->texture_max_height;
 	dev->mode_config.preferred_depth = dev_priv->assume_16bpp ? 16 : 32;
-	dev->mode_config.prefer_shadow_fbdev = !dev_priv->has_mob;
 
 	drm_mode_create_suggested_offset_properties(dev);
 	vmw_kms_create_hotplug_mode_update_property(dev_priv);
@@ -3076,8 +2925,20 @@ int vmw_du_helper_plane_update(struct vmw_du_update_plane *update)
 		struct vmw_framebuffer_bo *vfbbo =
 			container_of(update->vfb, typeof(*vfbbo), base);
 
-		ret = vmw_validation_add_bo(&val_ctx, vfbbo->buffer, false,
-					    update->cpu_blit);
+		/*
+		 * For screen targets we want a mappable bo, for everything else we want
+		 * accelerated i.e. host backed (vram or gmr) bo. If the display unit
+		 * is not screen target then mob's shouldn't be available.
+		 */
+		if (update->dev_priv->active_display_unit == vmw_du_screen_target) {
+			vmw_bo_placement_set(vfbbo->buffer,
+					     VMW_BO_DOMAIN_SYS | VMW_BO_DOMAIN_MOB | VMW_BO_DOMAIN_GMR,
+					     VMW_BO_DOMAIN_SYS | VMW_BO_DOMAIN_MOB | VMW_BO_DOMAIN_GMR);
+		} else {
+			WARN_ON(update->dev_priv->has_mob);
+			vmw_bo_placement_set_default_accelerated(vfbbo->buffer);
+		}
+		ret = vmw_validation_add_bo(&val_ctx, vfbbo->buffer);
 	} else {
 		struct vmw_framebuffer_surface *vfbs =
 			container_of(update->vfb, typeof(*vfbs), base);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index 4d6e7b555db7..db81e635dc06 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
- * Copyright 2009-2022 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -126,7 +126,6 @@ struct vmw_du_update_plane {
 	struct vmw_framebuffer *vfb;
 	struct vmw_fence_obj **out_fence;
 	struct mutex *mutex;
-	bool cpu_blit;
 	bool intr;
 };
 
@@ -217,8 +216,6 @@ struct vmw_kms_dirty {
  */
 struct vmw_framebuffer {
 	struct drm_framebuffer base;
-	int (*pin)(struct vmw_framebuffer *fb);
-	int (*unpin)(struct vmw_framebuffer *fb);
 	bool bo;
 	uint32_t user_handle;
 };
@@ -233,7 +230,7 @@ struct vmw_clip_rect {
 struct vmw_framebuffer_surface {
 	struct vmw_framebuffer base;
 	struct vmw_surface *surface;
-	struct vmw_buffer_object *buffer;
+	struct vmw_bo *buffer;
 	struct list_head head;
 	bool is_bo_proxy;  /* true if this is proxy surface for DMA buf */
 };
@@ -241,7 +238,7 @@ struct vmw_framebuffer_surface {
 
 struct vmw_framebuffer_bo {
 	struct vmw_framebuffer base;
-	struct vmw_buffer_object *buffer;
+	struct vmw_bo *buffer;
 };
 
 
@@ -273,9 +270,7 @@ struct vmw_crtc_state {
 };
 
 struct vmw_cursor_plane_state {
-	struct ttm_buffer_object *bo;
-	struct ttm_bo_kmap_obj map;
-	bool mapped;
+	struct vmw_bo *bo;
 	s32 hotspot_x;
 	s32 hotspot_y;
 };
@@ -293,7 +288,7 @@ struct vmw_cursor_plane_state {
 struct vmw_plane_state {
 	struct drm_plane_state base;
 	struct vmw_surface *surf;
-	struct vmw_buffer_object *bo;
+	struct vmw_bo *bo;
 
 	int content_fb_type;
 	unsigned long bo_size;
@@ -346,7 +341,7 @@ struct vmw_connector_state {
 struct vmw_cursor_plane {
 	struct drm_plane base;
 
-	struct ttm_buffer_object *cursor_mobs[3];
+	struct vmw_bo *cursor_mobs[3];
 };
 
 /**
@@ -364,7 +359,7 @@ struct vmw_display_unit {
 	struct vmw_cursor_plane cursor;
 
 	struct vmw_surface *cursor_surface;
-	struct vmw_buffer_object *cursor_bo;
+	struct vmw_bo *cursor_bo;
 	size_t cursor_age;
 
 	int cursor_x;
@@ -397,7 +392,7 @@ struct vmw_display_unit {
 
 struct vmw_validation_ctx {
 	struct vmw_resource *res;
-	struct vmw_buffer_object *buf;
+	struct vmw_bo *buf;
 };
 
 #define vmw_crtc_to_du(x) \
@@ -458,7 +453,7 @@ int vmw_kms_readback(struct vmw_private *dev_priv,
 		     uint32_t num_clips);
 struct vmw_framebuffer *
 vmw_kms_new_framebuffer(struct vmw_private *dev_priv,
-			struct vmw_buffer_object *bo,
+			struct vmw_bo *bo,
 			struct vmw_surface *surface,
 			bool only_2d,
 			const struct drm_mode_fb_cmd2 *mode_cmd);
@@ -512,11 +507,6 @@ void vmw_du_connector_destroy_state(struct drm_connector *connector,
  */
 int vmw_kms_ldu_init_display(struct vmw_private *dev_priv);
 int vmw_kms_ldu_close_display(struct vmw_private *dev_priv);
-int vmw_kms_ldu_do_bo_dirty(struct vmw_private *dev_priv,
-			    struct vmw_framebuffer *framebuffer,
-			    unsigned int flags, unsigned int color,
-			    struct drm_clip_rect *clips,
-			    unsigned int num_clips, int increment);
 int vmw_kms_update_proxy(struct vmw_resource *res,
 			 const struct drm_clip_rect *clips,
 			 unsigned num_clips,
@@ -566,17 +556,15 @@ int vmw_kms_stdu_surface_dirty(struct vmw_private *dev_priv,
 			       unsigned num_clips, int inc,
 			       struct vmw_fence_obj **out_fence,
 			       struct drm_crtc *crtc);
-int vmw_kms_stdu_dma(struct vmw_private *dev_priv,
-		     struct drm_file *file_priv,
-		     struct vmw_framebuffer *vfb,
-		     struct drm_vmw_fence_rep __user *user_fence_rep,
-		     struct drm_clip_rect *clips,
-		     struct drm_vmw_rect *vclips,
-		     uint32_t num_clips,
-		     int increment,
-		     bool to_surface,
-		     bool interruptible,
-		     struct drm_crtc *crtc);
+int vmw_kms_stdu_readback(struct vmw_private *dev_priv,
+			  struct drm_file *file_priv,
+			  struct vmw_framebuffer *vfb,
+			  struct drm_vmw_fence_rep __user *user_fence_rep,
+			  struct drm_clip_rect *clips,
+			  struct drm_vmw_rect *vclips,
+			  uint32_t num_clips,
+			  int increment,
+			  struct drm_crtc *crtc);
 
 int vmw_du_helper_plane_update(struct vmw_du_update_plane *update);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index a56e5d0ca3c6..a82fa9700370 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009-2022 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -25,11 +25,13 @@
  *
  **************************************************************************/
 
+#include "vmwgfx_bo.h"
+#include "vmwgfx_kms.h"
+
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_fourcc.h>
 
-#include "vmwgfx_kms.h"
 
 #define vmw_crtc_to_ldu(x) \
 	container_of(x, struct vmw_legacy_display_unit, base.crtc)
@@ -134,6 +136,47 @@ static int vmw_ldu_commit_list(struct vmw_private *dev_priv)
 	return 0;
 }
 
+/*
+ * Pin the buffer in a location suitable for access by the
+ * display system.
+ */
+static int vmw_ldu_fb_pin(struct vmw_framebuffer *vfb)
+{
+	struct vmw_private *dev_priv = vmw_priv(vfb->base.dev);
+	struct vmw_bo *buf;
+	int ret;
+
+	buf = vfb->bo ?  vmw_framebuffer_to_vfbd(&vfb->base)->buffer :
+		vmw_framebuffer_to_vfbs(&vfb->base)->surface->res.guest_memory_bo;
+
+	if (!buf)
+		return 0;
+	WARN_ON(dev_priv->active_display_unit != vmw_du_legacy);
+
+	if (dev_priv->active_display_unit == vmw_du_legacy) {
+		vmw_overlay_pause_all(dev_priv);
+		ret = vmw_bo_pin_in_start_of_vram(dev_priv, buf, false);
+		vmw_overlay_resume_all(dev_priv);
+	} else
+		ret = -EINVAL;
+
+	return ret;
+}
+
+static int vmw_ldu_fb_unpin(struct vmw_framebuffer *vfb)
+{
+	struct vmw_private *dev_priv = vmw_priv(vfb->base.dev);
+	struct vmw_bo *buf;
+
+	buf = vfb->bo ?  vmw_framebuffer_to_vfbd(&vfb->base)->buffer :
+		vmw_framebuffer_to_vfbs(&vfb->base)->surface->res.guest_memory_bo;
+
+	if (WARN_ON(!buf))
+		return 0;
+
+	return vmw_bo_unpin(dev_priv, buf, false);
+}
+
 static int vmw_ldu_del_active(struct vmw_private *vmw_priv,
 			      struct vmw_legacy_display_unit *ldu)
 {
@@ -145,8 +188,7 @@ static int vmw_ldu_del_active(struct vmw_private *vmw_priv,
 	list_del_init(&ldu->active);
 	if (--(ld->num_active) == 0) {
 		BUG_ON(!ld->fb);
-		if (ld->fb->unpin)
-			ld->fb->unpin(ld->fb);
+		WARN_ON(vmw_ldu_fb_unpin(ld->fb));
 		ld->fb = NULL;
 	}
 
@@ -163,11 +205,10 @@ static int vmw_ldu_add_active(struct vmw_private *vmw_priv,
 
 	BUG_ON(!ld->num_active && ld->fb);
 	if (vfb != ld->fb) {
-		if (ld->fb && ld->fb->unpin)
-			ld->fb->unpin(ld->fb);
+		if (ld->fb)
+			WARN_ON(vmw_ldu_fb_unpin(ld->fb));
 		vmw_svga_enable(vmw_priv);
-		if (vfb->pin)
-			vfb->pin(vfb);
+		WARN_ON(vmw_ldu_fb_pin(vfb));
 		ld->fb = vfb;
 	}
 
@@ -234,6 +275,7 @@ static const struct drm_crtc_funcs vmw_legacy_crtc_funcs = {
 	.atomic_duplicate_state = vmw_du_crtc_duplicate_state,
 	.atomic_destroy_state = vmw_du_crtc_destroy_state,
 	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
 };
 
 
@@ -273,6 +315,12 @@ static const struct
 drm_connector_helper_funcs vmw_ldu_connector_helper_funcs = {
 };
 
+static int vmw_kms_ldu_do_bo_dirty(struct vmw_private *dev_priv,
+				   struct vmw_framebuffer *framebuffer,
+				   unsigned int flags, unsigned int color,
+				   struct drm_mode_rect *clips,
+				   unsigned int num_clips);
+
 /*
  * Legacy Display Plane Functions
  */
@@ -291,7 +339,6 @@ vmw_ldu_primary_plane_atomic_update(struct drm_plane *plane,
 	struct drm_framebuffer *fb;
 	struct drm_crtc *crtc = new_state->crtc ?: old_state->crtc;
 
-
 	ldu = vmw_crtc_to_ldu(crtc);
 	dev_priv = vmw_priv(plane->dev);
 	fb       = new_state->fb;
@@ -304,8 +351,31 @@ vmw_ldu_primary_plane_atomic_update(struct drm_plane *plane,
 		vmw_ldu_del_active(dev_priv, ldu);
 
 	vmw_ldu_commit_list(dev_priv);
-}
 
+	if (vfb && vmw_cmd_supported(dev_priv)) {
+		struct drm_mode_rect fb_rect = {
+			.x1 = 0,
+			.y1 = 0,
+			.x2 = vfb->base.width,
+			.y2 = vfb->base.height
+		};
+		struct drm_mode_rect *damage_rects = drm_plane_get_damage_clips(new_state);
+		u32 rect_count = drm_plane_get_damage_clips_count(new_state);
+		int ret;
+
+		if (!damage_rects) {
+			damage_rects = &fb_rect;
+			rect_count = 1;
+		}
+
+		ret = vmw_kms_ldu_do_bo_dirty(dev_priv, vfb, 0, 0, damage_rects, rect_count);
+
+		drm_WARN_ONCE(plane->dev, ret,
+			"vmw_kms_ldu_do_bo_dirty failed with: ret=%d\n", ret);
+
+		vmw_cmd_flush(dev_priv, false);
+	}
+}
 
 static const struct drm_plane_funcs vmw_ldu_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
@@ -536,11 +606,11 @@ int vmw_kms_ldu_close_display(struct vmw_private *dev_priv)
 }
 
 
-int vmw_kms_ldu_do_bo_dirty(struct vmw_private *dev_priv,
-			    struct vmw_framebuffer *framebuffer,
-			    unsigned int flags, unsigned int color,
-			    struct drm_clip_rect *clips,
-			    unsigned int num_clips, int increment)
+static int vmw_kms_ldu_do_bo_dirty(struct vmw_private *dev_priv,
+				   struct vmw_framebuffer *framebuffer,
+				   unsigned int flags, unsigned int color,
+				   struct drm_mode_rect *clips,
+				   unsigned int num_clips)
 {
 	size_t fifo_size;
 	int i;
@@ -556,7 +626,7 @@ int vmw_kms_ldu_do_bo_dirty(struct vmw_private *dev_priv,
 		return -ENOMEM;
 
 	memset(cmd, 0, fifo_size);
-	for (i = 0; i < num_clips; i++, clips += increment) {
+	for (i = 0; i < num_clips; i++, clips++) {
 		cmd[i].header = SVGA_CMD_UPDATE;
 		cmd[i].body.x = clips->x1;
 		cmd[i].body.y = clips->y1;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c b/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
index 0a8cc28d6606..7055cbefc768 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2012-2021 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2012-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -25,10 +25,11 @@
  *
  **************************************************************************/
 
-#include <linux/highmem.h>
-
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 
+#include <linux/highmem.h>
+
 #ifdef CONFIG_64BIT
 #define VMW_PPN_SIZE 8
 #define VMW_MOBFMT_PTDEPTH_0 SVGA3D_MOBFMT_PT64_0
@@ -50,7 +51,7 @@
  * @pt_root_page    DMA address of the level 0 page of the page table.
  */
 struct vmw_mob {
-	struct ttm_buffer_object *pt_bo;
+	struct vmw_bo *pt_bo;
 	unsigned long num_pages;
 	unsigned pt_level;
 	dma_addr_t pt_root_page;
@@ -203,7 +204,7 @@ static void vmw_takedown_otable_base(struct vmw_private *dev_priv,
 	if (otable->page_table == NULL)
 		return;
 
-	bo = otable->page_table->pt_bo;
+	bo = &otable->page_table->pt_bo->tbo;
 	cmd = VMW_CMD_RESERVE(dev_priv, sizeof(*cmd));
 	if (unlikely(cmd == NULL))
 		return;
@@ -251,7 +252,9 @@ static int vmw_otable_batch_setup(struct vmw_private *dev_priv,
 		bo_size += otables[i].size;
 	}
 
-	ret = vmw_bo_create_and_populate(dev_priv, bo_size, &batch->otable_bo);
+	ret = vmw_bo_create_and_populate(dev_priv, bo_size,
+					 VMW_BO_DOMAIN_WAITABLE_SYS,
+					 &batch->otable_bo);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -260,7 +263,8 @@ static int vmw_otable_batch_setup(struct vmw_private *dev_priv,
 		if (!batch->otables[i].enabled)
 			continue;
 
-		ret = vmw_setup_otable_base(dev_priv, i, batch->otable_bo,
+		ret = vmw_setup_otable_base(dev_priv, i,
+					    &batch->otable_bo->tbo,
 					    offset,
 					    &otables[i]);
 		if (unlikely(ret != 0))
@@ -277,8 +281,8 @@ out_no_setup:
 						 &batch->otables[i]);
 	}
 
-	vmw_bo_unpin_unlocked(batch->otable_bo);
-	ttm_bo_put(batch->otable_bo);
+	vmw_bo_unpin_unlocked(&batch->otable_bo->tbo);
+	ttm_bo_put(&batch->otable_bo->tbo);
 	batch->otable_bo = NULL;
 	return ret;
 }
@@ -329,7 +333,7 @@ static void vmw_otable_batch_takedown(struct vmw_private *dev_priv,
 			       struct vmw_otable_batch *batch)
 {
 	SVGAOTableType i;
-	struct ttm_buffer_object *bo = batch->otable_bo;
+	struct ttm_buffer_object *bo = &batch->otable_bo->tbo;
 	int ret;
 
 	for (i = 0; i < batch->num_otables; ++i)
@@ -344,8 +348,7 @@ static void vmw_otable_batch_takedown(struct vmw_private *dev_priv,
 	ttm_bo_unpin(bo);
 	ttm_bo_unreserve(bo);
 
-	ttm_bo_put(batch->otable_bo);
-	batch->otable_bo = NULL;
+	vmw_bo_unreference(&batch->otable_bo);
 }
 
 /*
@@ -413,7 +416,9 @@ static int vmw_mob_pt_populate(struct vmw_private *dev_priv,
 {
 	BUG_ON(mob->pt_bo != NULL);
 
-	return vmw_bo_create_and_populate(dev_priv, mob->num_pages * PAGE_SIZE, &mob->pt_bo);
+	return vmw_bo_create_and_populate(dev_priv, mob->num_pages * PAGE_SIZE,
+					  VMW_BO_DOMAIN_WAITABLE_SYS,
+					  &mob->pt_bo);
 }
 
 /**
@@ -494,7 +499,7 @@ static void vmw_mob_pt_setup(struct vmw_mob *mob,
 			     unsigned long num_data_pages)
 {
 	unsigned long num_pt_pages = 0;
-	struct ttm_buffer_object *bo = mob->pt_bo;
+	struct ttm_buffer_object *bo = &mob->pt_bo->tbo;
 	struct vmw_piter save_pt_iter = {0};
 	struct vmw_piter pt_iter;
 	const struct vmw_sg_table *vsgt;
@@ -531,9 +536,8 @@ static void vmw_mob_pt_setup(struct vmw_mob *mob,
 void vmw_mob_destroy(struct vmw_mob *mob)
 {
 	if (mob->pt_bo) {
-		vmw_bo_unpin_unlocked(mob->pt_bo);
-		ttm_bo_put(mob->pt_bo);
-		mob->pt_bo = NULL;
+		vmw_bo_unpin_unlocked(&mob->pt_bo->tbo);
+		vmw_bo_unreference(&mob->pt_bo);
 	}
 	kfree(mob);
 }
@@ -552,7 +556,7 @@ void vmw_mob_unbind(struct vmw_private *dev_priv,
 		SVGA3dCmdDestroyGBMob body;
 	} *cmd;
 	int ret;
-	struct ttm_buffer_object *bo = mob->pt_bo;
+	struct ttm_buffer_object *bo = &mob->pt_bo->tbo;
 
 	if (bo) {
 		ret = ttm_bo_reserve(bo, false, true, NULL);
@@ -644,9 +648,8 @@ int vmw_mob_bind(struct vmw_private *dev_priv,
 out_no_cmd_space:
 	vmw_fifo_resource_dec(dev_priv);
 	if (pt_set_up) {
-		vmw_bo_unpin_unlocked(mob->pt_bo);
-		ttm_bo_put(mob->pt_bo);
-		mob->pt_bo = NULL;
+		vmw_bo_unpin_unlocked(&mob->pt_bo->tbo);
+		vmw_bo_unreference(&mob->pt_bo);
 	}
 
 	return -ENOMEM;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c b/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c
index e76976a95a1e..2651fe0ef518 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c
@@ -703,32 +703,6 @@ static inline void hypervisor_ppn_remove(PPN64 pfn)
 #define MKSSTAT_KERNEL_DESCRIPTION "vmwgfx"
 
 /**
- * mksstat_init_record: Initializes an MKSGuestStatCounter-based record
- * for the respective mksGuestStat index.
- *
- * @stat_idx: Index of the MKSGuestStatCounter-based mksGuestStat record.
- * @pstat: Pointer to array of MKSGuestStatCounterTime.
- * @pinfo: Pointer to array of MKSGuestStatInfoEntry.
- * @pstrs: Pointer to current end of the name/description sequence.
- * Return: Pointer to the new end of the names/description sequence.
- */
-
-static inline char *mksstat_init_record(mksstat_kern_stats_t stat_idx,
-	MKSGuestStatCounterTime *pstat, MKSGuestStatInfoEntry *pinfo, char *pstrs)
-{
-	char *const pstrd = pstrs + strlen(mksstat_kern_name_desc[stat_idx][0]) + 1;
-	strcpy(pstrs, mksstat_kern_name_desc[stat_idx][0]);
-	strcpy(pstrd, mksstat_kern_name_desc[stat_idx][1]);
-
-	pinfo[stat_idx].name.s = pstrs;
-	pinfo[stat_idx].description.s = pstrd;
-	pinfo[stat_idx].flags = MKS_GUEST_STAT_FLAG_NONE;
-	pinfo[stat_idx].stat.counter = (MKSGuestStatCounter *)&pstat[stat_idx];
-
-	return pstrd + strlen(mksstat_kern_name_desc[stat_idx][1]) + 1;
-}
-
-/**
  * mksstat_init_record_time: Initializes an MKSGuestStatCounterTime-based record
  * for the respective mksGuestStat index.
  *
@@ -1205,3 +1179,12 @@ int vmw_mksstat_remove_ioctl(struct drm_device *dev, void *data,
 
 	return -EAGAIN;
 }
+
+/**
+ * vmw_disable_backdoor: Disables all backdoor communication
+ * with the hypervisor.
+ */
+void vmw_disable_backdoor(void)
+{
+	vmw_msg_enabled = 0;
+}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
index b5b311f2a91a..7e112319a23c 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009-2014 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -24,19 +24,19 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  **************************************************************************/
-
-#include <drm/ttm/ttm_placement.h>
+#include "vmwgfx_bo.h"
+#include "vmwgfx_drv.h"
 
 #include "device_include/svga_overlay.h"
 #include "device_include/svga_escape.h"
 
-#include "vmwgfx_drv.h"
+#include <drm/ttm/ttm_placement.h>
 
 #define VMW_MAX_NUM_STREAMS 1
 #define VMW_OVERLAY_CAP_MASK (SVGA_FIFO_CAP_VIDEO | SVGA_FIFO_CAP_ESCAPE)
 
 struct vmw_stream {
-	struct vmw_buffer_object *buf;
+	struct vmw_bo *buf;
 	bool claimed;
 	bool paused;
 	struct drm_vmw_control_stream_arg saved;
@@ -53,12 +53,6 @@ struct vmw_overlay {
 	struct vmw_stream stream[VMW_MAX_NUM_STREAMS];
 };
 
-static inline struct vmw_overlay *vmw_overlay(struct drm_device *dev)
-{
-	struct vmw_private *dev_priv = vmw_priv(dev);
-	return dev_priv ? dev_priv->overlay_priv : NULL;
-}
-
 struct vmw_escape_header {
 	uint32_t cmd;
 	SVGAFifoCmdEscape body;
@@ -92,7 +86,7 @@ static inline void fill_flush(struct vmw_escape_video_flush *cmd,
  * -ERESTARTSYS if interrupted by a signal.
  */
 static int vmw_overlay_send_put(struct vmw_private *dev_priv,
-				struct vmw_buffer_object *buf,
+				struct vmw_bo *buf,
 				struct drm_vmw_control_stream_arg *arg,
 				bool interruptible)
 {
@@ -140,7 +134,7 @@ static int vmw_overlay_send_put(struct vmw_private *dev_priv,
 	for (i = 0; i < num_items; i++)
 		items[i].registerId = i;
 
-	vmw_bo_get_guest_ptr(&buf->base, &ptr);
+	vmw_bo_get_guest_ptr(&buf->tbo, &ptr);
 	ptr.offset += arg->offset;
 
 	items[SVGA_VIDEO_ENABLED].value     = true;
@@ -223,7 +217,7 @@ static int vmw_overlay_send_stop(struct vmw_private *dev_priv,
  * used with GMRs instead of being locked to vram.
  */
 static int vmw_overlay_move_buffer(struct vmw_private *dev_priv,
-				   struct vmw_buffer_object *buf,
+				   struct vmw_bo *buf,
 				   bool pin, bool inter)
 {
 	if (!pin)
@@ -295,7 +289,7 @@ static int vmw_overlay_stop(struct vmw_private *dev_priv,
  * -ERESTARTSYS if interrupted.
  */
 static int vmw_overlay_update_stream(struct vmw_private *dev_priv,
-				     struct vmw_buffer_object *buf,
+				     struct vmw_bo *buf,
 				     struct drm_vmw_control_stream_arg *arg,
 				     bool interruptible)
 {
@@ -433,7 +427,7 @@ int vmw_overlay_ioctl(struct drm_device *dev, void *data,
 	struct vmw_overlay *overlay = dev_priv->overlay_priv;
 	struct drm_vmw_control_stream_arg *arg =
 	    (struct drm_vmw_control_stream_arg *)data;
-	struct vmw_buffer_object *buf;
+	struct vmw_bo *buf;
 	struct vmw_resource *res;
 	int ret;
 
@@ -458,7 +452,7 @@ int vmw_overlay_ioctl(struct drm_device *dev, void *data,
 	ret = vmw_overlay_update_stream(dev_priv, buf, arg, true);
 
 	vmw_bo_unreference(&buf);
-	drm_gem_object_put(&buf->base.base);
+	drm_gem_object_put(&buf->tbo.base);
 
 out_unlock:
 	mutex_unlock(&overlay->mutex);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c
index f41f041559f4..74ff2812d66a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2019 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2019-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -24,6 +24,7 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  **************************************************************************/
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 
 /*
@@ -78,11 +79,11 @@ struct vmw_bo_dirty {
  * dirty structure with the results. This function may change the
  * dirty-tracking method.
  */
-static void vmw_bo_dirty_scan_pagetable(struct vmw_buffer_object *vbo)
+static void vmw_bo_dirty_scan_pagetable(struct vmw_bo *vbo)
 {
 	struct vmw_bo_dirty *dirty = vbo->dirty;
-	pgoff_t offset = drm_vma_node_start(&vbo->base.base.vma_node);
-	struct address_space *mapping = vbo->base.bdev->dev_mapping;
+	pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
+	struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
 	pgoff_t num_marked;
 
 	num_marked = clean_record_shared_mapping_range
@@ -116,26 +117,25 @@ static void vmw_bo_dirty_scan_pagetable(struct vmw_buffer_object *vbo)
  *
  * This function may change the dirty-tracking method.
  */
-static void vmw_bo_dirty_scan_mkwrite(struct vmw_buffer_object *vbo)
+static void vmw_bo_dirty_scan_mkwrite(struct vmw_bo *vbo)
 {
 	struct vmw_bo_dirty *dirty = vbo->dirty;
-	unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
-	struct address_space *mapping = vbo->base.bdev->dev_mapping;
+	unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
+	struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
 	pgoff_t num_marked;
 
 	if (dirty->end <= dirty->start)
 		return;
 
-	num_marked = wp_shared_mapping_range(vbo->base.bdev->dev_mapping,
-					dirty->start + offset,
-					dirty->end - dirty->start);
+	num_marked = wp_shared_mapping_range(vbo->tbo.bdev->dev_mapping,
+					     dirty->start + offset,
+					     dirty->end - dirty->start);
 
 	if (100UL * num_marked / dirty->bitmap_size >
-	    VMW_DIRTY_PERCENTAGE) {
+	    VMW_DIRTY_PERCENTAGE)
 		dirty->change_count++;
-	} else {
+	else
 		dirty->change_count = 0;
-	}
 
 	if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
 		pgoff_t start = 0;
@@ -160,7 +160,7 @@ static void vmw_bo_dirty_scan_mkwrite(struct vmw_buffer_object *vbo)
  *
  * This function may change the dirty tracking method.
  */
-void vmw_bo_dirty_scan(struct vmw_buffer_object *vbo)
+void vmw_bo_dirty_scan(struct vmw_bo *vbo)
 {
 	struct vmw_bo_dirty *dirty = vbo->dirty;
 
@@ -181,12 +181,12 @@ void vmw_bo_dirty_scan(struct vmw_buffer_object *vbo)
  * when calling unmap_mapping_range(). This function makes sure we pick
  * up all dirty pages.
  */
-static void vmw_bo_dirty_pre_unmap(struct vmw_buffer_object *vbo,
+static void vmw_bo_dirty_pre_unmap(struct vmw_bo *vbo,
 				   pgoff_t start, pgoff_t end)
 {
 	struct vmw_bo_dirty *dirty = vbo->dirty;
-	unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
-	struct address_space *mapping = vbo->base.bdev->dev_mapping;
+	unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
+	struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
 
 	if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end)
 		return;
@@ -206,11 +206,11 @@ static void vmw_bo_dirty_pre_unmap(struct vmw_buffer_object *vbo,
  *
  * This is similar to ttm_bo_unmap_virtual() except it takes a subrange.
  */
-void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo,
+void vmw_bo_dirty_unmap(struct vmw_bo *vbo,
 			pgoff_t start, pgoff_t end)
 {
-	unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
-	struct address_space *mapping = vbo->base.bdev->dev_mapping;
+	unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
+	struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
 
 	vmw_bo_dirty_pre_unmap(vbo, start, end);
 	unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT,
@@ -227,10 +227,10 @@ void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo,
  *
  * Return: Zero on success, -ENOMEM on memory allocation failure.
  */
-int vmw_bo_dirty_add(struct vmw_buffer_object *vbo)
+int vmw_bo_dirty_add(struct vmw_bo *vbo)
 {
 	struct vmw_bo_dirty *dirty = vbo->dirty;
-	pgoff_t num_pages = PFN_UP(vbo->base.resource->size);
+	pgoff_t num_pages = PFN_UP(vbo->tbo.resource->size);
 	size_t size;
 	int ret;
 
@@ -253,8 +253,8 @@ int vmw_bo_dirty_add(struct vmw_buffer_object *vbo)
 	if (num_pages < PAGE_SIZE / sizeof(pte_t)) {
 		dirty->method = VMW_BO_DIRTY_PAGETABLE;
 	} else {
-		struct address_space *mapping = vbo->base.bdev->dev_mapping;
-		pgoff_t offset = drm_vma_node_start(&vbo->base.base.vma_node);
+		struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
+		pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
 
 		dirty->method = VMW_BO_DIRTY_MKWRITE;
 
@@ -284,7 +284,7 @@ out_no_dirty:
  *
  * Return: Zero on success, -ENOMEM on memory allocation failure.
  */
-void vmw_bo_dirty_release(struct vmw_buffer_object *vbo)
+void vmw_bo_dirty_release(struct vmw_bo *vbo)
 {
 	struct vmw_bo_dirty *dirty = vbo->dirty;
 
@@ -306,11 +306,11 @@ void vmw_bo_dirty_release(struct vmw_buffer_object *vbo)
  */
 void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res)
 {
-	struct vmw_buffer_object *vbo = res->backup;
+	struct vmw_bo *vbo = res->guest_memory_bo;
 	struct vmw_bo_dirty *dirty = vbo->dirty;
 	pgoff_t start, cur, end;
-	unsigned long res_start = res->backup_offset;
-	unsigned long res_end = res->backup_offset + res->backup_size;
+	unsigned long res_start = res->guest_memory_offset;
+	unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;
 
 	WARN_ON_ONCE(res_start & ~PAGE_MASK);
 	res_start >>= PAGE_SHIFT;
@@ -351,9 +351,9 @@ void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res)
  */
 void vmw_bo_dirty_clear_res(struct vmw_resource *res)
 {
-	unsigned long res_start = res->backup_offset;
-	unsigned long res_end = res->backup_offset + res->backup_size;
-	struct vmw_buffer_object *vbo = res->backup;
+	unsigned long res_start = res->guest_memory_offset;
+	unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;
+	struct vmw_bo *vbo = res->guest_memory_bo;
 	struct vmw_bo_dirty *dirty = vbo->dirty;
 
 	res_start >>= PAGE_SHIFT;
@@ -380,8 +380,7 @@ vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf)
 	vm_fault_t ret;
 	unsigned long page_offset;
 	unsigned int save_flags;
-	struct vmw_buffer_object *vbo =
-		container_of(bo, typeof(*vbo), base);
+	struct vmw_bo *vbo = to_vmw_bo(&bo->base);
 
 	/*
 	 * mkwrite() doesn't handle the VM_FAULT_RETRY return value correctly.
@@ -419,8 +418,7 @@ vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)
 	struct vm_area_struct *vma = vmf->vma;
 	struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
 	    vma->vm_private_data;
-	struct vmw_buffer_object *vbo =
-		container_of(bo, struct vmw_buffer_object, base);
+	struct vmw_bo *vbo = to_vmw_bo(&bo->base);
 	pgoff_t num_prefault;
 	pgprot_t prot;
 	vm_fault_t ret;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index c7d645e5ec7b..71eeabf001c8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009-2015 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -27,9 +27,10 @@
 
 #include <drm/ttm/ttm_placement.h>
 
-#include "vmwgfx_resource_priv.h"
 #include "vmwgfx_binding.h"
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
+#include "vmwgfx_resource_priv.h"
 
 #define VMW_RES_EVICT_ERR_COUNT 10
 
@@ -39,10 +40,10 @@
  */
 void vmw_resource_mob_attach(struct vmw_resource *res)
 {
-	struct vmw_buffer_object *backup = res->backup;
-	struct rb_node **new = &backup->res_tree.rb_node, *parent = NULL;
+	struct vmw_bo *gbo = res->guest_memory_bo;
+	struct rb_node **new = &gbo->res_tree.rb_node, *parent = NULL;
 
-	dma_resv_assert_held(res->backup->base.base.resv);
+	dma_resv_assert_held(gbo->tbo.base.resv);
 	res->used_prio = (res->res_dirty) ? res->func->dirty_prio :
 		res->func->prio;
 
@@ -51,14 +52,14 @@ void vmw_resource_mob_attach(struct vmw_resource *res)
 			container_of(*new, struct vmw_resource, mob_node);
 
 		parent = *new;
-		new = (res->backup_offset < this->backup_offset) ?
+		new = (res->guest_memory_offset < this->guest_memory_offset) ?
 			&((*new)->rb_left) : &((*new)->rb_right);
 	}
 
 	rb_link_node(&res->mob_node, parent, new);
-	rb_insert_color(&res->mob_node, &backup->res_tree);
+	rb_insert_color(&res->mob_node, &gbo->res_tree);
 
-	vmw_bo_prio_add(backup, res->used_prio);
+	vmw_bo_prio_add(gbo, res->used_prio);
 }
 
 /**
@@ -67,13 +68,13 @@ void vmw_resource_mob_attach(struct vmw_resource *res)
  */
 void vmw_resource_mob_detach(struct vmw_resource *res)
 {
-	struct vmw_buffer_object *backup = res->backup;
+	struct vmw_bo *gbo = res->guest_memory_bo;
 
-	dma_resv_assert_held(backup->base.base.resv);
+	dma_resv_assert_held(gbo->tbo.base.resv);
 	if (vmw_resource_mob_attached(res)) {
-		rb_erase(&res->mob_node, &backup->res_tree);
+		rb_erase(&res->mob_node, &gbo->res_tree);
 		RB_CLEAR_NODE(&res->mob_node);
-		vmw_bo_prio_del(backup, res->used_prio);
+		vmw_bo_prio_del(gbo, res->used_prio);
 	}
 }
 
@@ -120,8 +121,8 @@ static void vmw_resource_release(struct kref *kref)
 	spin_lock(&dev_priv->resource_lock);
 	list_del_init(&res->lru_head);
 	spin_unlock(&dev_priv->resource_lock);
-	if (res->backup) {
-		struct ttm_buffer_object *bo = &res->backup->base;
+	if (res->guest_memory_bo) {
+		struct ttm_buffer_object *bo = &res->guest_memory_bo->tbo;
 
 		ret = ttm_bo_reserve(bo, false, false, NULL);
 		BUG_ON(ret);
@@ -133,14 +134,14 @@ static void vmw_resource_release(struct kref *kref)
 			val_buf.num_shared = 0;
 			res->func->unbind(res, false, &val_buf);
 		}
-		res->backup_dirty = false;
+		res->guest_memory_size = false;
 		vmw_resource_mob_detach(res);
 		if (res->dirty)
 			res->func->dirty_free(res);
 		if (res->coherent)
-			vmw_bo_dirty_release(res->backup);
+			vmw_bo_dirty_release(res->guest_memory_bo);
 		ttm_bo_unreserve(bo);
-		vmw_bo_unreference(&res->backup);
+		vmw_bo_unreference(&res->guest_memory_bo);
 	}
 
 	if (likely(res->hw_destroy != NULL)) {
@@ -223,9 +224,9 @@ int vmw_resource_init(struct vmw_private *dev_priv, struct vmw_resource *res,
 	INIT_LIST_HEAD(&res->lru_head);
 	INIT_LIST_HEAD(&res->binding_head);
 	res->id = -1;
-	res->backup = NULL;
-	res->backup_offset = 0;
-	res->backup_dirty = false;
+	res->guest_memory_bo = NULL;
+	res->guest_memory_offset = 0;
+	res->guest_memory_dirty = false;
 	res->res_dirty = false;
 	res->coherent = false;
 	res->used_prio = 3;
@@ -263,7 +264,7 @@ int vmw_user_resource_lookup_handle(struct vmw_private *dev_priv,
 	int ret = -EINVAL;
 
 	base = ttm_base_object_lookup(tfile, handle);
-	if (unlikely(base == NULL))
+	if (unlikely(!base))
 		return -EINVAL;
 
 	if (unlikely(ttm_base_object_type(base) != converter->object_type))
@@ -290,7 +291,7 @@ int vmw_user_lookup_handle(struct vmw_private *dev_priv,
 			   struct drm_file *filp,
 			   uint32_t handle,
 			   struct vmw_surface **out_surf,
-			   struct vmw_buffer_object **out_buf)
+			   struct vmw_bo **out_buf)
 {
 	struct ttm_object_file *tfile = vmw_fpriv(filp)->tfile;
 	struct vmw_resource *res;
@@ -312,32 +313,36 @@ int vmw_user_lookup_handle(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_resource_buf_alloc - Allocate a backup buffer for a resource.
+ * vmw_resource_buf_alloc - Allocate a guest memory buffer for a resource.
  *
- * @res:            The resource for which to allocate a backup buffer.
+ * @res:            The resource for which to allocate a gbo buffer.
  * @interruptible:  Whether any sleeps during allocation should be
  *                  performed while interruptible.
  */
 static int vmw_resource_buf_alloc(struct vmw_resource *res,
 				  bool interruptible)
 {
-	unsigned long size = PFN_ALIGN(res->backup_size);
-	struct vmw_buffer_object *backup;
+	unsigned long size = PFN_ALIGN(res->guest_memory_size);
+	struct vmw_bo *gbo;
+	struct vmw_bo_params bo_params = {
+		.domain = res->func->domain,
+		.busy_domain = res->func->busy_domain,
+		.bo_type = ttm_bo_type_device,
+		.size = res->guest_memory_size,
+		.pin = false
+	};
 	int ret;
 
-	if (likely(res->backup)) {
-		BUG_ON(res->backup->base.base.size < size);
+	if (likely(res->guest_memory_bo)) {
+		BUG_ON(res->guest_memory_bo->tbo.base.size < size);
 		return 0;
 	}
 
-	ret = vmw_bo_create(res->dev_priv, res->backup_size,
-			    res->func->backup_placement,
-			    interruptible, false,
-			    &vmw_bo_bo_free, &backup);
+	ret = vmw_bo_create(res->dev_priv, &bo_params, &gbo);
 	if (unlikely(ret != 0))
 		goto out_no_bo;
 
-	res->backup = backup;
+	res->guest_memory_bo = gbo;
 
 out_no_bo:
 	return ret;
@@ -369,13 +374,13 @@ static int vmw_resource_do_validate(struct vmw_resource *res,
 	}
 
 	if (func->bind &&
-	    ((func->needs_backup && !vmw_resource_mob_attached(res) &&
-	      val_buf->bo != NULL) ||
-	     (!func->needs_backup && val_buf->bo != NULL))) {
+	    ((func->needs_guest_memory && !vmw_resource_mob_attached(res) &&
+	      val_buf->bo) ||
+	     (!func->needs_guest_memory && val_buf->bo))) {
 		ret = func->bind(res, val_buf);
 		if (unlikely(ret != 0))
 			goto out_bind_failed;
-		if (func->needs_backup)
+		if (func->needs_guest_memory)
 			vmw_resource_mob_attach(res);
 	}
 
@@ -385,11 +390,11 @@ static int vmw_resource_do_validate(struct vmw_resource *res,
 	 */
 	if (func->dirty_alloc && vmw_resource_mob_attached(res) &&
 	    !res->coherent) {
-		if (res->backup->dirty && !res->dirty) {
+		if (res->guest_memory_bo->dirty && !res->dirty) {
 			ret = func->dirty_alloc(res);
 			if (ret)
 				return ret;
-		} else if (!res->backup->dirty && res->dirty) {
+		} else if (!res->guest_memory_bo->dirty && res->dirty) {
 			func->dirty_free(res);
 		}
 	}
@@ -400,12 +405,12 @@ static int vmw_resource_do_validate(struct vmw_resource *res,
 	 */
 	if (res->dirty) {
 		if (dirtying && !res->res_dirty) {
-			pgoff_t start = res->backup_offset >> PAGE_SHIFT;
+			pgoff_t start = res->guest_memory_offset >> PAGE_SHIFT;
 			pgoff_t end = __KERNEL_DIV_ROUND_UP
-				(res->backup_offset + res->backup_size,
+				(res->guest_memory_offset + res->guest_memory_size,
 				 PAGE_SIZE);
 
-			vmw_bo_dirty_unmap(res->backup, start, end);
+			vmw_bo_dirty_unmap(res->guest_memory_bo, start, end);
 		}
 
 		vmw_bo_dirty_transfer_to_res(res);
@@ -427,10 +432,10 @@ out_bind_failed:
  * @res:               Pointer to the struct vmw_resource to unreserve.
  * @dirty_set:         Change dirty status of the resource.
  * @dirty:             When changing dirty status indicates the new status.
- * @switch_backup:     Backup buffer has been switched.
- * @new_backup:        Pointer to new backup buffer if command submission
+ * @switch_guest_memory: Guest memory buffer has been switched.
+ * @new_guest_memory_bo: Pointer to new guest memory buffer if command submission
  *                     switched. May be NULL.
- * @new_backup_offset: New backup offset if @switch_backup is true.
+ * @new_guest_memory_offset: New gbo offset if @switch_guest_memory is true.
  *
  * Currently unreserving a resource means putting it back on the device's
  * resource lru list, so that it can be evicted if necessary.
@@ -438,42 +443,42 @@ out_bind_failed:
 void vmw_resource_unreserve(struct vmw_resource *res,
 			    bool dirty_set,
 			    bool dirty,
-			    bool switch_backup,
-			    struct vmw_buffer_object *new_backup,
-			    unsigned long new_backup_offset)
+			    bool switch_guest_memory,
+			    struct vmw_bo *new_guest_memory_bo,
+			    unsigned long new_guest_memory_offset)
 {
 	struct vmw_private *dev_priv = res->dev_priv;
 
 	if (!list_empty(&res->lru_head))
 		return;
 
-	if (switch_backup && new_backup != res->backup) {
-		if (res->backup) {
+	if (switch_guest_memory && new_guest_memory_bo != res->guest_memory_bo) {
+		if (res->guest_memory_bo) {
 			vmw_resource_mob_detach(res);
 			if (res->coherent)
-				vmw_bo_dirty_release(res->backup);
-			vmw_bo_unreference(&res->backup);
+				vmw_bo_dirty_release(res->guest_memory_bo);
+			vmw_bo_unreference(&res->guest_memory_bo);
 		}
 
-		if (new_backup) {
-			res->backup = vmw_bo_reference(new_backup);
+		if (new_guest_memory_bo) {
+			res->guest_memory_bo = vmw_bo_reference(new_guest_memory_bo);
 
 			/*
 			 * The validation code should already have added a
 			 * dirty tracker here.
 			 */
-			WARN_ON(res->coherent && !new_backup->dirty);
+			WARN_ON(res->coherent && !new_guest_memory_bo->dirty);
 
 			vmw_resource_mob_attach(res);
 		} else {
-			res->backup = NULL;
+			res->guest_memory_bo = NULL;
 		}
-	} else if (switch_backup && res->coherent) {
-		vmw_bo_dirty_release(res->backup);
+	} else if (switch_guest_memory && res->coherent) {
+		vmw_bo_dirty_release(res->guest_memory_bo);
 	}
 
-	if (switch_backup)
-		res->backup_offset = new_backup_offset;
+	if (switch_guest_memory)
+		res->guest_memory_offset = new_guest_memory_offset;
 
 	if (dirty_set)
 		res->res_dirty = dirty;
@@ -507,30 +512,32 @@ vmw_resource_check_buffer(struct ww_acquire_ctx *ticket,
 {
 	struct ttm_operation_ctx ctx = { true, false };
 	struct list_head val_list;
-	bool backup_dirty = false;
+	bool guest_memory_dirty = false;
 	int ret;
 
-	if (unlikely(res->backup == NULL)) {
+	if (unlikely(!res->guest_memory_bo)) {
 		ret = vmw_resource_buf_alloc(res, interruptible);
 		if (unlikely(ret != 0))
 			return ret;
 	}
 
 	INIT_LIST_HEAD(&val_list);
-	ttm_bo_get(&res->backup->base);
-	val_buf->bo = &res->backup->base;
+	ttm_bo_get(&res->guest_memory_bo->tbo);
+	val_buf->bo = &res->guest_memory_bo->tbo;
 	val_buf->num_shared = 0;
 	list_add_tail(&val_buf->head, &val_list);
 	ret = ttm_eu_reserve_buffers(ticket, &val_list, interruptible, NULL);
 	if (unlikely(ret != 0))
 		goto out_no_reserve;
 
-	if (res->func->needs_backup && !vmw_resource_mob_attached(res))
+	if (res->func->needs_guest_memory && !vmw_resource_mob_attached(res))
 		return 0;
 
-	backup_dirty = res->backup_dirty;
-	ret = ttm_bo_validate(&res->backup->base,
-			      res->func->backup_placement,
+	guest_memory_dirty = res->guest_memory_dirty;
+	vmw_bo_placement_set(res->guest_memory_bo, res->func->domain,
+			     res->func->busy_domain);
+	ret = ttm_bo_validate(&res->guest_memory_bo->tbo,
+			      &res->guest_memory_bo->placement,
 			      &ctx);
 
 	if (unlikely(ret != 0))
@@ -543,8 +550,8 @@ out_no_validate:
 out_no_reserve:
 	ttm_bo_put(val_buf->bo);
 	val_buf->bo = NULL;
-	if (backup_dirty)
-		vmw_bo_unreference(&res->backup);
+	if (guest_memory_dirty)
+		vmw_bo_unreference(&res->guest_memory_bo);
 
 	return ret;
 }
@@ -555,12 +562,13 @@ out_no_reserve:
  * @res:            The resource to reserve.
  *
  * This function takes the resource off the LRU list and make sure
- * a backup buffer is present for guest-backed resources. However,
- * the buffer may not be bound to the resource at this point.
+ * a guest memory buffer is present for guest-backed resources.
+ * However, the buffer may not be bound to the resource at this
+ * point.
  *
  */
 int vmw_resource_reserve(struct vmw_resource *res, bool interruptible,
-			 bool no_backup)
+			 bool no_guest_memory)
 {
 	struct vmw_private *dev_priv = res->dev_priv;
 	int ret;
@@ -569,13 +577,13 @@ int vmw_resource_reserve(struct vmw_resource *res, bool interruptible,
 	list_del_init(&res->lru_head);
 	spin_unlock(&dev_priv->resource_lock);
 
-	if (res->func->needs_backup && res->backup == NULL &&
-	    !no_backup) {
+	if (res->func->needs_guest_memory && !res->guest_memory_bo &&
+	    !no_guest_memory) {
 		ret = vmw_resource_buf_alloc(res, interruptible);
 		if (unlikely(ret != 0)) {
-			DRM_ERROR("Failed to allocate a backup buffer "
+			DRM_ERROR("Failed to allocate a guest memory buffer "
 				  "of size %lu. bytes\n",
-				  (unsigned long) res->backup_size);
+				  (unsigned long) res->guest_memory_size);
 			return ret;
 		}
 	}
@@ -585,10 +593,10 @@ int vmw_resource_reserve(struct vmw_resource *res, bool interruptible,
 
 /**
  * vmw_resource_backoff_reservation - Unreserve and unreference a
- *                                    backup buffer
+ *                                    guest memory buffer
  *.
  * @ticket:         The ww acquire ctx used for reservation.
- * @val_buf:        Backup buffer information.
+ * @val_buf:        Guest memory buffer information.
  */
 static void
 vmw_resource_backoff_reservation(struct ww_acquire_ctx *ticket,
@@ -630,14 +638,14 @@ static int vmw_resource_do_evict(struct ww_acquire_ctx *ticket,
 		return ret;
 
 	if (unlikely(func->unbind != NULL &&
-		     (!func->needs_backup || vmw_resource_mob_attached(res)))) {
+		     (!func->needs_guest_memory || vmw_resource_mob_attached(res)))) {
 		ret = func->unbind(res, res->res_dirty, &val_buf);
 		if (unlikely(ret != 0))
 			goto out_no_unbind;
 		vmw_resource_mob_detach(res);
 	}
 	ret = func->destroy(res);
-	res->backup_dirty = true;
+	res->guest_memory_dirty = true;
 	res->res_dirty = false;
 out_no_unbind:
 	vmw_resource_backoff_reservation(ticket, &val_buf);
@@ -676,8 +684,8 @@ int vmw_resource_validate(struct vmw_resource *res, bool intr,
 
 	val_buf.bo = NULL;
 	val_buf.num_shared = 0;
-	if (res->backup)
-		val_buf.bo = &res->backup->base;
+	if (res->guest_memory_bo)
+		val_buf.bo = &res->guest_memory_bo->tbo;
 	do {
 		ret = vmw_resource_do_validate(res, &val_buf, dirtying);
 		if (likely(ret != -EBUSY))
@@ -717,9 +725,9 @@ int vmw_resource_validate(struct vmw_resource *res, bool intr,
 
 	if (unlikely(ret != 0))
 		goto out_no_validate;
-	else if (!res->func->needs_backup && res->backup) {
+	else if (!res->func->needs_guest_memory && res->guest_memory_bo) {
 		WARN_ON_ONCE(vmw_resource_mob_attached(res));
-		vmw_bo_unreference(&res->backup);
+		vmw_bo_unreference(&res->guest_memory_bo);
 	}
 
 	return 0;
@@ -740,14 +748,14 @@ out_no_validate:
  * validation code, since resource validation and eviction
  * both require the backup buffer to be reserved.
  */
-void vmw_resource_unbind_list(struct vmw_buffer_object *vbo)
+void vmw_resource_unbind_list(struct vmw_bo *vbo)
 {
 	struct ttm_validate_buffer val_buf = {
-		.bo = &vbo->base,
+		.bo = &vbo->tbo,
 		.num_shared = 0
 	};
 
-	dma_resv_assert_held(vbo->base.base.resv);
+	dma_resv_assert_held(vbo->tbo.base.resv);
 	while (!RB_EMPTY_ROOT(&vbo->res_tree)) {
 		struct rb_node *node = vbo->res_tree.rb_node;
 		struct vmw_resource *res =
@@ -756,12 +764,12 @@ void vmw_resource_unbind_list(struct vmw_buffer_object *vbo)
 		if (!WARN_ON_ONCE(!res->func->unbind))
 			(void) res->func->unbind(res, res->res_dirty, &val_buf);
 
-		res->backup_dirty = true;
+		res->guest_memory_size = true;
 		res->res_dirty = false;
 		vmw_resource_mob_detach(res);
 	}
 
-	(void) ttm_bo_wait(&vbo->base, false, false);
+	(void) ttm_bo_wait(&vbo->tbo, false, false);
 }
 
 
@@ -773,7 +781,7 @@ void vmw_resource_unbind_list(struct vmw_buffer_object *vbo)
  * Read back cached states from the device if they exist.  This function
  * assumes binding_mutex is held.
  */
-int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob)
+int vmw_query_readback_all(struct vmw_bo *dx_query_mob)
 {
 	struct vmw_resource *dx_query_ctx;
 	struct vmw_private *dev_priv;
@@ -822,20 +830,19 @@ void vmw_query_move_notify(struct ttm_buffer_object *bo,
 			   struct ttm_resource *old_mem,
 			   struct ttm_resource *new_mem)
 {
-	struct vmw_buffer_object *dx_query_mob;
+	struct vmw_bo *dx_query_mob;
 	struct ttm_device *bdev = bo->bdev;
-	struct vmw_private *dev_priv;
-
-	dev_priv = container_of(bdev, struct vmw_private, bdev);
+	struct vmw_private *dev_priv = vmw_priv_from_ttm(bdev);
 
 	mutex_lock(&dev_priv->binding_mutex);
 
 	/* If BO is being moved from MOB to system memory */
-	if (new_mem->mem_type == TTM_PL_SYSTEM &&
+	if (old_mem &&
+	    new_mem->mem_type == TTM_PL_SYSTEM &&
 	    old_mem->mem_type == VMW_PL_MOB) {
 		struct vmw_fence_obj *fence;
 
-		dx_query_mob = container_of(bo, struct vmw_buffer_object, base);
+		dx_query_mob = to_vmw_bo(&bo->base);
 		if (!dx_query_mob || !dx_query_mob->dx_query_ctx) {
 			mutex_unlock(&dev_priv->binding_mutex);
 			return;
@@ -863,7 +870,7 @@ void vmw_query_move_notify(struct ttm_buffer_object *bo,
  */
 bool vmw_resource_needs_backup(const struct vmw_resource *res)
 {
-	return res->func->needs_backup;
+	return res->func->needs_guest_memory;
 }
 
 /**
@@ -959,21 +966,24 @@ int vmw_resource_pin(struct vmw_resource *res, bool interruptible)
 		goto out_no_reserve;
 
 	if (res->pin_count == 0) {
-		struct vmw_buffer_object *vbo = NULL;
+		struct vmw_bo *vbo = NULL;
 
-		if (res->backup) {
-			vbo = res->backup;
+		if (res->guest_memory_bo) {
+			vbo = res->guest_memory_bo;
 
-			ret = ttm_bo_reserve(&vbo->base, interruptible, false, NULL);
+			ret = ttm_bo_reserve(&vbo->tbo, interruptible, false, NULL);
 			if (ret)
 				goto out_no_validate;
-			if (!vbo->base.pin_count) {
+			if (!vbo->tbo.pin_count) {
+				vmw_bo_placement_set(vbo,
+						     res->func->domain,
+						     res->func->busy_domain);
 				ret = ttm_bo_validate
-					(&vbo->base,
-					 res->func->backup_placement,
+					(&vbo->tbo,
+					 &vbo->placement,
 					 &ctx);
 				if (ret) {
-					ttm_bo_unreserve(&vbo->base);
+					ttm_bo_unreserve(&vbo->tbo);
 					goto out_no_validate;
 				}
 			}
@@ -983,7 +993,7 @@ int vmw_resource_pin(struct vmw_resource *res, bool interruptible)
 		}
 		ret = vmw_resource_validate(res, interruptible, true);
 		if (vbo)
-			ttm_bo_unreserve(&vbo->base);
+			ttm_bo_unreserve(&vbo->tbo);
 		if (ret)
 			goto out_no_validate;
 	}
@@ -1016,12 +1026,12 @@ void vmw_resource_unpin(struct vmw_resource *res)
 	WARN_ON(ret);
 
 	WARN_ON(res->pin_count == 0);
-	if (--res->pin_count == 0 && res->backup) {
-		struct vmw_buffer_object *vbo = res->backup;
+	if (--res->pin_count == 0 && res->guest_memory_bo) {
+		struct vmw_bo *vbo = res->guest_memory_bo;
 
-		(void) ttm_bo_reserve(&vbo->base, false, false, NULL);
+		(void) ttm_bo_reserve(&vbo->tbo, false, false, NULL);
 		vmw_bo_pin_reserved(vbo, false);
-		ttm_bo_unreserve(&vbo->base);
+		ttm_bo_unreserve(&vbo->tbo);
 	}
 
 	vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
@@ -1062,7 +1072,7 @@ void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start,
  * @num_prefault: Returns how many pages including the first have been
  * cleaned and are ok to prefault
  */
-int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start,
+int vmw_resources_clean(struct vmw_bo *vbo, pgoff_t start,
 			pgoff_t end, pgoff_t *num_prefault)
 {
 	struct rb_node *cur = vbo->res_tree.rb_node;
@@ -1079,9 +1089,9 @@ int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start,
 		struct vmw_resource *cur_res =
 			container_of(cur, struct vmw_resource, mob_node);
 
-		if (cur_res->backup_offset >= res_end) {
+		if (cur_res->guest_memory_offset >= res_end) {
 			cur = cur->rb_left;
-		} else if (cur_res->backup_offset + cur_res->backup_size <=
+		} else if (cur_res->guest_memory_offset + cur_res->guest_memory_size <=
 			   res_start) {
 			cur = cur->rb_right;
 		} else {
@@ -1092,7 +1102,7 @@ int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start,
 	}
 
 	/*
-	 * In order of increasing backup_offset, clean dirty resources
+	 * In order of increasing guest_memory_offset, clean dirty resources
 	 * intersecting the range.
 	 */
 	while (found) {
@@ -1108,13 +1118,13 @@ int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start,
 
 			found->res_dirty = false;
 		}
-		last_cleaned = found->backup_offset + found->backup_size;
+		last_cleaned = found->guest_memory_offset + found->guest_memory_size;
 		cur = rb_next(&found->mob_node);
 		if (!cur)
 			break;
 
 		found = container_of(cur, struct vmw_resource, mob_node);
-		if (found->backup_offset >= res_end)
+		if (found->guest_memory_offset >= res_end)
 			break;
 	}
 
@@ -1123,7 +1133,7 @@ int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start,
 	 */
 	*num_prefault = 1;
 	if (last_cleaned > res_start) {
-		struct ttm_buffer_object *bo = &vbo->base;
+		struct ttm_buffer_object *bo = &vbo->tbo;
 
 		*num_prefault = __KERNEL_DIV_ROUND_UP(last_cleaned - res_start,
 						      PAGE_SIZE);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource_priv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_resource_priv.h
index 3b7438b2d289..aa7cbd396bea 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource_priv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource_priv.h
@@ -58,10 +58,11 @@ struct vmw_user_resource_conv {
  * struct vmw_res_func - members and functions common for a resource type
  *
  * @res_type:          Enum that identifies the lru list to use for eviction.
- * @needs_backup:      Whether the resource is guest-backed and needs
+ * @needs_guest_memory:Whether the resource is guest-backed and needs
  *                     persistent buffer storage.
  * @type_name:         String that identifies the resource type.
- * @backup_placement:  TTM placement for backup buffers.
+ * @domain:            TTM placement for guest memory buffers.
+ * @busy_domain:       TTM busy placement for guest memory buffers.
  * @may_evict          Whether the resource may be evicted.
  * @create:            Create a hardware resource.
  * @destroy:           Destroy a hardware resource.
@@ -81,9 +82,10 @@ struct vmw_user_resource_conv {
  */
 struct vmw_res_func {
 	enum vmw_res_type res_type;
-	bool needs_backup;
+	bool needs_guest_memory;
 	const char *type_name;
-	struct ttm_placement *backup_placement;
+	u32 domain;
+	u32 busy_domain;
 	bool may_evict;
 	u32 prio;
 	u32 dirty_prio;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index e1f36a09c59c..556a403b7eb5 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2011-2022 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2011-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -25,13 +25,14 @@
  *
  **************************************************************************/
 
+#include "vmwgfx_bo.h"
+#include "vmwgfx_kms.h"
+
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
 
-#include "vmwgfx_kms.h"
-
 #define vmw_crtc_to_sou(x) \
 	container_of(x, struct vmw_screen_object_unit, base.crtc)
 #define vmw_encoder_to_sou(x) \
@@ -89,7 +90,7 @@ struct vmw_screen_object_unit {
 	struct vmw_display_unit base;
 
 	unsigned long buffer_size; /**< Size of allocated buffer */
-	struct vmw_buffer_object *buffer; /**< Backing store buffer */
+	struct vmw_bo *buffer; /**< Backing store buffer */
 
 	bool defined;
 };
@@ -148,7 +149,7 @@ static int vmw_sou_fifo_create(struct vmw_private *dev_priv,
 	sou->base.set_gui_y = cmd->obj.root.y;
 
 	/* Ok to assume that buffer is pinned in vram */
-	vmw_bo_get_guest_ptr(&sou->buffer->base, &cmd->obj.backingStore.ptr);
+	vmw_bo_get_guest_ptr(&sou->buffer->tbo, &cmd->obj.backingStore.ptr);
 	cmd->obj.backingStore.pitch = mode->hdisplay * 4;
 
 	vmw_cmd_commit(dev_priv, fifo_size);
@@ -409,9 +410,13 @@ vmw_sou_primary_plane_prepare_fb(struct drm_plane *plane,
 	struct drm_crtc *crtc = plane->state->crtc ?: new_state->crtc;
 	struct vmw_plane_state *vps = vmw_plane_state_to_vps(new_state);
 	struct vmw_private *dev_priv;
-	size_t size;
 	int ret;
-
+	struct vmw_bo_params bo_params = {
+		.domain = VMW_BO_DOMAIN_VRAM,
+		.busy_domain = VMW_BO_DOMAIN_VRAM,
+		.bo_type = ttm_bo_type_device,
+		.pin = true
+	};
 
 	if (!new_fb) {
 		vmw_bo_unreference(&vps->bo);
@@ -420,11 +425,11 @@ vmw_sou_primary_plane_prepare_fb(struct drm_plane *plane,
 		return 0;
 	}
 
-	size = new_state->crtc_w * new_state->crtc_h * 4;
+	bo_params.size = new_state->crtc_w * new_state->crtc_h * 4;
 	dev_priv = vmw_priv(crtc->dev);
 
 	if (vps->bo) {
-		if (vps->bo_size == size) {
+		if (vps->bo_size == bo_params.size) {
 			/*
 			 * Note that this might temporarily up the pin-count
 			 * to 2, until cleanup_fb() is called.
@@ -443,16 +448,12 @@ vmw_sou_primary_plane_prepare_fb(struct drm_plane *plane,
 	 * resume the overlays, this is preferred to failing to alloc.
 	 */
 	vmw_overlay_pause_all(dev_priv);
-	ret = vmw_bo_create(dev_priv, size,
-			    &vmw_vram_placement,
-			    false, true, &vmw_bo_bo_free, &vps->bo);
+	ret = vmw_bo_create(dev_priv, &bo_params, &vps->bo);
 	vmw_overlay_resume_all(dev_priv);
-	if (ret) {
-		vps->bo = NULL; /* vmw_bo_init frees on error */
+	if (ret)
 		return ret;
-	}
 
-	vps->bo_size = size;
+	vps->bo_size = bo_params.size;
 
 	/*
 	 * TTM already thinks the buffer is pinned, but make sure the
@@ -489,7 +490,7 @@ static uint32_t vmw_sou_bo_define_gmrfb(struct vmw_du_update_plane *update,
 	gmr->body.format.colorDepth = depth;
 	gmr->body.format.reserved = 0;
 	gmr->body.bytesPerLine = update->vfb->base.pitches[0];
-	vmw_bo_get_guest_ptr(&vfbbo->buffer->base, &gmr->body.ptr);
+	vmw_bo_get_guest_ptr(&vfbbo->buffer->tbo, &gmr->body.ptr);
 
 	return sizeof(*gmr);
 }
@@ -546,7 +547,6 @@ static int vmw_sou_plane_update_bo(struct vmw_private *dev_priv,
 	bo_update.base.vfb = vfb;
 	bo_update.base.out_fence = out_fence;
 	bo_update.base.mutex = NULL;
-	bo_update.base.cpu_blit = false;
 	bo_update.base.intr = true;
 
 	bo_update.base.calc_fifo_size = vmw_sou_bo_fifo_size;
@@ -707,7 +707,6 @@ static int vmw_sou_plane_update_surface(struct vmw_private *dev_priv,
 	srf_update.base.vfb = vfb;
 	srf_update.base.out_fence = out_fence;
 	srf_update.base.mutex = &dev_priv->cmdbuf_mutex;
-	srf_update.base.cpu_blit = false;
 	srf_update.base.intr = true;
 
 	srf_update.base.calc_fifo_size = vmw_sou_surface_fifo_size;
@@ -947,7 +946,7 @@ int vmw_kms_sou_init_display(struct vmw_private *dev_priv)
 static int do_bo_define_gmrfb(struct vmw_private *dev_priv,
 				  struct vmw_framebuffer *framebuffer)
 {
-	struct vmw_buffer_object *buf =
+	struct vmw_bo *buf =
 		container_of(framebuffer, struct vmw_framebuffer_bo,
 			     base)->buffer;
 	int depth = framebuffer->base.format->depth;
@@ -973,7 +972,7 @@ static int do_bo_define_gmrfb(struct vmw_private *dev_priv,
 	cmd->body.format.reserved = 0;
 	cmd->body.bytesPerLine = framebuffer->base.pitches[0];
 	/* Buffer is reserved in vram or GMR */
-	vmw_bo_get_guest_ptr(&buf->base, &cmd->body.ptr);
+	vmw_bo_get_guest_ptr(&buf->tbo, &cmd->body.ptr);
 	vmw_cmd_commit(dev_priv, sizeof(*cmd));
 
 	return 0;
@@ -1216,14 +1215,16 @@ int vmw_kms_sou_do_bo_dirty(struct vmw_private *dev_priv,
 				struct vmw_fence_obj **out_fence,
 				struct drm_crtc *crtc)
 {
-	struct vmw_buffer_object *buf =
+	struct vmw_bo *buf =
 		container_of(framebuffer, struct vmw_framebuffer_bo,
 			     base)->buffer;
 	struct vmw_kms_dirty dirty;
 	DECLARE_VAL_CONTEXT(val_ctx, NULL, 0);
 	int ret;
 
-	ret = vmw_validation_add_bo(&val_ctx, buf, false, false);
+	vmw_bo_placement_set(buf, VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM,
+			     VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM);
+	ret = vmw_validation_add_bo(&val_ctx, buf);
 	if (ret)
 		return ret;
 
@@ -1323,13 +1324,15 @@ int vmw_kms_sou_readback(struct vmw_private *dev_priv,
 			 uint32_t num_clips,
 			 struct drm_crtc *crtc)
 {
-	struct vmw_buffer_object *buf =
+	struct vmw_bo *buf =
 		container_of(vfb, struct vmw_framebuffer_bo, base)->buffer;
 	struct vmw_kms_dirty dirty;
 	DECLARE_VAL_CONTEXT(val_ctx, NULL, 0);
 	int ret;
 
-	ret = vmw_validation_add_bo(&val_ctx, buf, false, false);
+	vmw_bo_placement_set(buf, VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM,
+			     VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM);
+	ret = vmw_validation_add_bo(&val_ctx, buf);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
index 51e83dfa1cac..e7226db8b242 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009-2015 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -27,9 +27,10 @@
 
 #include <drm/ttm/ttm_placement.h>
 
+#include "vmwgfx_binding.h"
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 #include "vmwgfx_resource_priv.h"
-#include "vmwgfx_binding.h"
 
 struct vmw_shader {
 	struct vmw_resource res;
@@ -88,12 +89,13 @@ const struct vmw_user_resource_conv *user_shader_converter =
 
 static const struct vmw_res_func vmw_gb_shader_func = {
 	.res_type = vmw_res_shader,
-	.needs_backup = true,
+	.needs_guest_memory = true,
 	.may_evict = true,
 	.prio = 3,
 	.dirty_prio = 3,
 	.type_name = "guest backed shaders",
-	.backup_placement = &vmw_mob_placement,
+	.domain = VMW_BO_DOMAIN_MOB,
+	.busy_domain = VMW_BO_DOMAIN_MOB,
 	.create = vmw_gb_shader_create,
 	.destroy = vmw_gb_shader_destroy,
 	.bind = vmw_gb_shader_bind,
@@ -102,12 +104,13 @@ static const struct vmw_res_func vmw_gb_shader_func = {
 
 static const struct vmw_res_func vmw_dx_shader_func = {
 	.res_type = vmw_res_shader,
-	.needs_backup = true,
+	.needs_guest_memory = true,
 	.may_evict = true,
 	.prio = 3,
 	.dirty_prio = 3,
 	.type_name = "dx shaders",
-	.backup_placement = &vmw_mob_placement,
+	.domain = VMW_BO_DOMAIN_MOB,
+	.busy_domain = VMW_BO_DOMAIN_MOB,
 	.create = vmw_dx_shader_create,
 	/*
 	 * The destroy callback is only called with a committed resource on
@@ -158,7 +161,7 @@ static int vmw_gb_shader_init(struct vmw_private *dev_priv,
 			      SVGA3dShaderType type,
 			      uint8_t num_input_sig,
 			      uint8_t num_output_sig,
-			      struct vmw_buffer_object *byte_code,
+			      struct vmw_bo *byte_code,
 			      void (*res_free) (struct vmw_resource *res))
 {
 	struct vmw_shader *shader = vmw_res_to_shader(res);
@@ -175,10 +178,10 @@ static int vmw_gb_shader_init(struct vmw_private *dev_priv,
 		return ret;
 	}
 
-	res->backup_size = size;
+	res->guest_memory_size = size;
 	if (byte_code) {
-		res->backup = vmw_bo_reference(byte_code);
-		res->backup_offset = offset;
+		res->guest_memory_bo = vmw_bo_reference(byte_code);
+		res->guest_memory_offset = offset;
 	}
 	shader->size = size;
 	shader->type = type;
@@ -259,8 +262,8 @@ static int vmw_gb_shader_bind(struct vmw_resource *res,
 	cmd->header.size = sizeof(cmd->body);
 	cmd->body.shid = res->id;
 	cmd->body.mobid = bo->resource->start;
-	cmd->body.offsetInBytes = res->backup_offset;
-	res->backup_dirty = false;
+	cmd->body.offsetInBytes = res->guest_memory_offset;
+	res->guest_memory_dirty = false;
 	vmw_cmd_commit(dev_priv, sizeof(*cmd));
 
 	return 0;
@@ -277,7 +280,7 @@ static int vmw_gb_shader_unbind(struct vmw_resource *res,
 	} *cmd;
 	struct vmw_fence_obj *fence;
 
-	BUG_ON(res->backup->base.resource->mem_type != VMW_PL_MOB);
+	BUG_ON(res->guest_memory_bo->tbo.resource->mem_type != VMW_PL_MOB);
 
 	cmd = VMW_CMD_RESERVE(dev_priv, sizeof(*cmd));
 	if (unlikely(cmd == NULL))
@@ -397,8 +400,8 @@ static int vmw_dx_shader_unscrub(struct vmw_resource *res)
 	cmd->header.size = sizeof(cmd->body);
 	cmd->body.cid = shader->ctx->id;
 	cmd->body.shid = shader->id;
-	cmd->body.mobid = res->backup->base.resource->start;
-	cmd->body.offsetInBytes = res->backup_offset;
+	cmd->body.mobid = res->guest_memory_bo->tbo.resource->start;
+	cmd->body.offsetInBytes = res->guest_memory_offset;
 	vmw_cmd_commit(dev_priv, sizeof(*cmd));
 
 	vmw_cotable_add_resource(shader->cotable, &shader->cotable_head);
@@ -508,7 +511,7 @@ static int vmw_dx_shader_unbind(struct vmw_resource *res,
 	struct vmw_fence_obj *fence;
 	int ret;
 
-	BUG_ON(res->backup->base.resource->mem_type != VMW_PL_MOB);
+	BUG_ON(res->guest_memory_bo->tbo.resource->mem_type != VMW_PL_MOB);
 
 	mutex_lock(&dev_priv->binding_mutex);
 	ret = vmw_dx_shader_scrub(res);
@@ -680,7 +683,7 @@ int vmw_shader_destroy_ioctl(struct drm_device *dev, void *data,
 }
 
 static int vmw_user_shader_alloc(struct vmw_private *dev_priv,
-				 struct vmw_buffer_object *buffer,
+				 struct vmw_bo *buffer,
 				 size_t shader_size,
 				 size_t offset,
 				 SVGA3dShaderType shader_type,
@@ -734,7 +737,7 @@ out:
 
 
 static struct vmw_resource *vmw_shader_alloc(struct vmw_private *dev_priv,
-					     struct vmw_buffer_object *buffer,
+					     struct vmw_bo *buffer,
 					     size_t shader_size,
 					     size_t offset,
 					     SVGA3dShaderType shader_type)
@@ -771,7 +774,7 @@ static int vmw_shader_define(struct drm_device *dev, struct drm_file *file_priv,
 {
 	struct vmw_private *dev_priv = vmw_priv(dev);
 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-	struct vmw_buffer_object *buffer = NULL;
+	struct vmw_bo *buffer = NULL;
 	SVGA3dShaderType shader_type;
 	int ret;
 
@@ -782,7 +785,7 @@ static int vmw_shader_define(struct drm_device *dev, struct drm_file *file_priv,
 			return ret;
 		}
 
-		if ((u64)buffer->base.base.size < (u64)size + (u64)offset) {
+		if ((u64)buffer->tbo.base.size < (u64)size + (u64)offset) {
 			VMW_DEBUG_USER("Illegal buffer- or shader size.\n");
 			ret = -EINVAL;
 			goto out_bad_arg;
@@ -807,7 +810,7 @@ static int vmw_shader_define(struct drm_device *dev, struct drm_file *file_priv,
 				    num_output_sig, tfile, shader_handle);
 out_bad_arg:
 	vmw_bo_unreference(&buffer);
-	drm_gem_object_put(&buffer->base.base);
+	drm_gem_object_put(&buffer->tbo.base);
 	return ret;
 }
 
@@ -884,28 +887,34 @@ int vmw_compat_shader_add(struct vmw_private *dev_priv,
 			  struct list_head *list)
 {
 	struct ttm_operation_ctx ctx = { false, true };
-	struct vmw_buffer_object *buf;
+	struct vmw_bo *buf;
 	struct ttm_bo_kmap_obj map;
 	bool is_iomem;
 	int ret;
 	struct vmw_resource *res;
+	struct vmw_bo_params bo_params = {
+		.domain = VMW_BO_DOMAIN_SYS,
+		.busy_domain = VMW_BO_DOMAIN_SYS,
+		.bo_type = ttm_bo_type_device,
+		.size = size,
+		.pin = true
+	};
 
 	if (!vmw_shader_id_ok(user_key, shader_type))
 		return -EINVAL;
 
-	ret = vmw_bo_create(dev_priv, size, &vmw_sys_placement,
-			    true, true, vmw_bo_bo_free, &buf);
+	ret = vmw_bo_create(dev_priv, &bo_params, &buf);
 	if (unlikely(ret != 0))
 		goto out;
 
-	ret = ttm_bo_reserve(&buf->base, false, true, NULL);
+	ret = ttm_bo_reserve(&buf->tbo, false, true, NULL);
 	if (unlikely(ret != 0))
 		goto no_reserve;
 
 	/* Map and copy shader bytecode. */
-	ret = ttm_bo_kmap(&buf->base, 0, PFN_UP(size), &map);
+	ret = ttm_bo_kmap(&buf->tbo, 0, PFN_UP(size), &map);
 	if (unlikely(ret != 0)) {
-		ttm_bo_unreserve(&buf->base);
+		ttm_bo_unreserve(&buf->tbo);
 		goto no_reserve;
 	}
 
@@ -913,9 +922,9 @@ int vmw_compat_shader_add(struct vmw_private *dev_priv,
 	WARN_ON(is_iomem);
 
 	ttm_bo_kunmap(&map);
-	ret = ttm_bo_validate(&buf->base, &vmw_sys_placement, &ctx);
+	ret = ttm_bo_validate(&buf->tbo, &buf->placement, &ctx);
 	WARN_ON(ret != 0);
-	ttm_bo_unreserve(&buf->base);
+	ttm_bo_unreserve(&buf->tbo);
 
 	res = vmw_shader_alloc(dev_priv, buf, size, 0, shader_type);
 	if (unlikely(ret != 0))
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_so.c b/drivers/gpu/drm/vmwgfx/vmwgfx_so.c
index 4ea32b01efc0..d199e718cb2d 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_so.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_so.c
@@ -24,6 +24,7 @@
  *
  **************************************************************************/
 
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 #include "vmwgfx_resource_priv.h"
 #include "vmwgfx_so.h"
@@ -70,7 +71,7 @@ struct vmw_view {
 	unsigned view_id;              /* Immutable */
 	u32 cmd_size;                  /* Immutable */
 	bool committed;                /* Protected by binding_mutex */
-	u32 cmd[1];                    /* Immutable */
+	u32 cmd[];		       /* Immutable */
 };
 
 static int vmw_view_create(struct vmw_resource *res);
@@ -81,10 +82,11 @@ static void vmw_view_commit_notify(struct vmw_resource *res,
 
 static const struct vmw_res_func vmw_view_func = {
 	.res_type = vmw_res_view,
-	.needs_backup = false,
+	.needs_guest_memory = false,
 	.may_evict = false,
 	.type_name = "DX view",
-	.backup_placement = NULL,
+	.domain = VMW_BO_DOMAIN_SYS,
+	.busy_domain = VMW_BO_DOMAIN_SYS,
 	.create = vmw_view_create,
 	.commit_notify = vmw_view_commit_notify,
 };
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index 0090abe89254..ba0c0e12cfe9 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /******************************************************************************
  *
- * COPYRIGHT (C) 2014-2022 VMware, Inc., Palo Alto, CA., USA
+ * COPYRIGHT (C) 2014-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -25,14 +25,15 @@
  *
  ******************************************************************************/
 
+#include "vmwgfx_bo.h"
+#include "vmwgfx_kms.h"
+#include "vmw_surface_cache.h"
+
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
 
-#include "vmwgfx_kms.h"
-#include "vmw_surface_cache.h"
-
 #define vmw_crtc_to_stdu(x) \
 	container_of(x, struct vmw_screen_target_display_unit, base.crtc)
 #define vmw_encoder_to_stdu(x) \
@@ -65,12 +66,11 @@ enum stdu_content_type {
  */
 struct vmw_stdu_dirty {
 	struct vmw_kms_dirty base;
-	SVGA3dTransferType  transfer;
 	s32 left, right, top, bottom;
 	s32 fb_left, fb_top;
 	u32 pitch;
 	union {
-		struct vmw_buffer_object *buf;
+		struct vmw_bo *buf;
 		u32 sid;
 	};
 };
@@ -136,12 +136,6 @@ static void vmw_stdu_destroy(struct vmw_screen_target_display_unit *stdu);
  * Screen Target Display Unit CRTC Functions
  *****************************************************************************/
 
-static bool vmw_stdu_use_cpu_blit(const struct vmw_private *vmw)
-{
-	return !(vmw->capabilities & SVGA_CAP_3D) || vmw->vram_size < (32 * 1024 * 1024);
-}
-
-
 /**
  * vmw_stdu_crtc_destroy - cleans up the STDU
  *
@@ -451,93 +445,6 @@ static void vmw_stdu_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 /**
- * vmw_stdu_bo_clip - Callback to encode a suface DMA command cliprect
- *
- * @dirty: The closure structure.
- *
- * Encodes a surface DMA command cliprect and updates the bounding box
- * for the DMA.
- */
-static void vmw_stdu_bo_clip(struct vmw_kms_dirty *dirty)
-{
-	struct vmw_stdu_dirty *ddirty =
-		container_of(dirty, struct vmw_stdu_dirty, base);
-	struct vmw_stdu_dma *cmd = dirty->cmd;
-	struct SVGA3dCopyBox *blit = (struct SVGA3dCopyBox *) &cmd[1];
-
-	blit += dirty->num_hits;
-	blit->srcx = dirty->fb_x;
-	blit->srcy = dirty->fb_y;
-	blit->x = dirty->unit_x1;
-	blit->y = dirty->unit_y1;
-	blit->d = 1;
-	blit->w = dirty->unit_x2 - dirty->unit_x1;
-	blit->h = dirty->unit_y2 - dirty->unit_y1;
-	dirty->num_hits++;
-
-	if (ddirty->transfer != SVGA3D_WRITE_HOST_VRAM)
-		return;
-
-	/* Destination bounding box */
-	ddirty->left = min_t(s32, ddirty->left, dirty->unit_x1);
-	ddirty->top = min_t(s32, ddirty->top, dirty->unit_y1);
-	ddirty->right = max_t(s32, ddirty->right, dirty->unit_x2);
-	ddirty->bottom = max_t(s32, ddirty->bottom, dirty->unit_y2);
-}
-
-/**
- * vmw_stdu_bo_fifo_commit - Callback to fill in and submit a DMA command.
- *
- * @dirty: The closure structure.
- *
- * Fills in the missing fields in a DMA command, and optionally encodes
- * a screen target update command, depending on transfer direction.
- */
-static void vmw_stdu_bo_fifo_commit(struct vmw_kms_dirty *dirty)
-{
-	struct vmw_stdu_dirty *ddirty =
-		container_of(dirty, struct vmw_stdu_dirty, base);
-	struct vmw_screen_target_display_unit *stdu =
-		container_of(dirty->unit, typeof(*stdu), base);
-	struct vmw_stdu_dma *cmd = dirty->cmd;
-	struct SVGA3dCopyBox *blit = (struct SVGA3dCopyBox *) &cmd[1];
-	SVGA3dCmdSurfaceDMASuffix *suffix =
-		(SVGA3dCmdSurfaceDMASuffix *) &blit[dirty->num_hits];
-	size_t blit_size = sizeof(*blit) * dirty->num_hits + sizeof(*suffix);
-
-	if (!dirty->num_hits) {
-		vmw_cmd_commit(dirty->dev_priv, 0);
-		return;
-	}
-
-	cmd->header.id = SVGA_3D_CMD_SURFACE_DMA;
-	cmd->header.size = sizeof(cmd->body) + blit_size;
-	vmw_bo_get_guest_ptr(&ddirty->buf->base, &cmd->body.guest.ptr);
-	cmd->body.guest.pitch = ddirty->pitch;
-	cmd->body.host.sid = stdu->display_srf->res.id;
-	cmd->body.host.face = 0;
-	cmd->body.host.mipmap = 0;
-	cmd->body.transfer = ddirty->transfer;
-	suffix->suffixSize = sizeof(*suffix);
-	suffix->maximumOffset = ddirty->buf->base.base.size;
-
-	if (ddirty->transfer == SVGA3D_WRITE_HOST_VRAM) {
-		blit_size += sizeof(struct vmw_stdu_update);
-
-		vmw_stdu_populate_update(&suffix[1], stdu->base.unit,
-					 ddirty->left, ddirty->right,
-					 ddirty->top, ddirty->bottom);
-	}
-
-	vmw_cmd_commit(dirty->dev_priv, sizeof(*cmd) + blit_size);
-
-	stdu->display_srf->res.res_dirty = true;
-	ddirty->left = ddirty->top = S32_MAX;
-	ddirty->right = ddirty->bottom = S32_MIN;
-}
-
-
-/**
  * vmw_stdu_bo_cpu_clip - Callback to encode a CPU blit
  *
  * @dirty: The closure structure.
@@ -597,62 +504,21 @@ static void vmw_stdu_bo_cpu_commit(struct vmw_kms_dirty *dirty)
 		return;
 
 	/* Assume we are blitting from Guest (bo) to Host (display_srf) */
-	dst_pitch = stdu->display_srf->metadata.base_size.width * stdu->cpp;
-	dst_bo = &stdu->display_srf->res.backup->base;
-	dst_offset = ddirty->top * dst_pitch + ddirty->left * stdu->cpp;
-
-	src_pitch = ddirty->pitch;
-	src_bo = &ddirty->buf->base;
-	src_offset = ddirty->fb_top * src_pitch + ddirty->fb_left * stdu->cpp;
-
-	/* Swap src and dst if the assumption was wrong. */
-	if (ddirty->transfer != SVGA3D_WRITE_HOST_VRAM) {
-		swap(dst_pitch, src_pitch);
-		swap(dst_bo, src_bo);
-		swap(src_offset, dst_offset);
-	}
+	src_pitch = stdu->display_srf->metadata.base_size.width * stdu->cpp;
+	src_bo = &stdu->display_srf->res.guest_memory_bo->tbo;
+	src_offset = ddirty->top * src_pitch + ddirty->left * stdu->cpp;
+
+	dst_pitch = ddirty->pitch;
+	dst_bo = &ddirty->buf->tbo;
+	dst_offset = ddirty->fb_top * dst_pitch + ddirty->fb_left * stdu->cpp;
 
 	(void) vmw_bo_cpu_blit(dst_bo, dst_offset, dst_pitch,
 			       src_bo, src_offset, src_pitch,
 			       width * stdu->cpp, height, &diff);
-
-	if (ddirty->transfer == SVGA3D_WRITE_HOST_VRAM &&
-	    drm_rect_visible(&diff.rect)) {
-		struct vmw_private *dev_priv;
-		struct vmw_stdu_update *cmd;
-		struct drm_clip_rect region;
-		int ret;
-
-		/* We are updating the actual surface, not a proxy */
-		region.x1 = diff.rect.x1;
-		region.x2 = diff.rect.x2;
-		region.y1 = diff.rect.y1;
-		region.y2 = diff.rect.y2;
-		ret = vmw_kms_update_proxy(&stdu->display_srf->res, &region,
-					   1, 1);
-		if (ret)
-			goto out_cleanup;
-
-
-		dev_priv = vmw_priv(stdu->base.crtc.dev);
-		cmd = VMW_CMD_RESERVE(dev_priv, sizeof(*cmd));
-		if (!cmd)
-			goto out_cleanup;
-
-		vmw_stdu_populate_update(cmd, stdu->base.unit,
-					 region.x1, region.x2,
-					 region.y1, region.y2);
-
-		vmw_cmd_commit(dev_priv, sizeof(*cmd));
-	}
-
-out_cleanup:
-	ddirty->left = ddirty->top = ddirty->fb_left = ddirty->fb_top = S32_MAX;
-	ddirty->right = ddirty->bottom = S32_MIN;
 }
 
 /**
- * vmw_kms_stdu_dma - Perform a DMA transfer between a buffer-object backed
+ * vmw_kms_stdu_readback - Perform a readback from a buffer-object backed
  * framebuffer and the screen target system.
  *
  * @dev_priv: Pointer to the device private structure.
@@ -665,9 +531,6 @@ out_cleanup:
  * be NULL.
  * @num_clips: Number of clip rects in @clips or @vclips.
  * @increment: Increment to use when looping over @clips or @vclips.
- * @to_surface: Whether to DMA to the screen target system as opposed to
- * from the screen target system.
- * @interruptible: Whether to perform waits interruptible if possible.
  * @crtc: If crtc is passed, perform stdu dma on that crtc only.
  *
  * If DMA-ing till the screen target system, the function will also notify
@@ -676,59 +539,49 @@ out_cleanup:
  * Returns 0 on success, negative error code on failure. -ERESTARTSYS if
  * interrupted.
  */
-int vmw_kms_stdu_dma(struct vmw_private *dev_priv,
-		     struct drm_file *file_priv,
-		     struct vmw_framebuffer *vfb,
-		     struct drm_vmw_fence_rep __user *user_fence_rep,
-		     struct drm_clip_rect *clips,
-		     struct drm_vmw_rect *vclips,
-		     uint32_t num_clips,
-		     int increment,
-		     bool to_surface,
-		     bool interruptible,
-		     struct drm_crtc *crtc)
+int vmw_kms_stdu_readback(struct vmw_private *dev_priv,
+			  struct drm_file *file_priv,
+			  struct vmw_framebuffer *vfb,
+			  struct drm_vmw_fence_rep __user *user_fence_rep,
+			  struct drm_clip_rect *clips,
+			  struct drm_vmw_rect *vclips,
+			  uint32_t num_clips,
+			  int increment,
+			  struct drm_crtc *crtc)
 {
-	struct vmw_buffer_object *buf =
+	struct vmw_bo *buf =
 		container_of(vfb, struct vmw_framebuffer_bo, base)->buffer;
 	struct vmw_stdu_dirty ddirty;
 	int ret;
-	bool cpu_blit = vmw_stdu_use_cpu_blit(dev_priv);
 	DECLARE_VAL_CONTEXT(val_ctx, NULL, 0);
 
 	/*
-	 * VMs without 3D support don't have the surface DMA command and
-	 * we'll be using a CPU blit, and the framebuffer should be moved out
-	 * of VRAM.
+	 * The GMR domain might seem confusing because it might seem like it should
+	 * never happen with screen targets but e.g. the xorg vmware driver issues
+	 * CMD_SURFACE_DMA for various pixmap updates which might transition our bo to
+	 * a GMR. Instead of forcing another transition we can optimize the readback
+	 * by reading directly from the GMR.
 	 */
-	ret = vmw_validation_add_bo(&val_ctx, buf, false, cpu_blit);
+	vmw_bo_placement_set(buf,
+			     VMW_BO_DOMAIN_MOB | VMW_BO_DOMAIN_SYS | VMW_BO_DOMAIN_GMR,
+			     VMW_BO_DOMAIN_MOB | VMW_BO_DOMAIN_SYS | VMW_BO_DOMAIN_GMR);
+	ret = vmw_validation_add_bo(&val_ctx, buf);
 	if (ret)
 		return ret;
 
-	ret = vmw_validation_prepare(&val_ctx, NULL, interruptible);
+	ret = vmw_validation_prepare(&val_ctx, NULL, true);
 	if (ret)
 		goto out_unref;
 
-	ddirty.transfer = (to_surface) ? SVGA3D_WRITE_HOST_VRAM :
-		SVGA3D_READ_HOST_VRAM;
 	ddirty.left = ddirty.top = S32_MAX;
 	ddirty.right = ddirty.bottom = S32_MIN;
 	ddirty.fb_left = ddirty.fb_top = S32_MAX;
 	ddirty.pitch = vfb->base.pitches[0];
 	ddirty.buf = buf;
-	ddirty.base.fifo_commit = vmw_stdu_bo_fifo_commit;
-	ddirty.base.clip = vmw_stdu_bo_clip;
-	ddirty.base.fifo_reserve_size = sizeof(struct vmw_stdu_dma) +
-		num_clips * sizeof(SVGA3dCopyBox) +
-		sizeof(SVGA3dCmdSurfaceDMASuffix);
-	if (to_surface)
-		ddirty.base.fifo_reserve_size += sizeof(struct vmw_stdu_update);
-
-
-	if (cpu_blit) {
-		ddirty.base.fifo_commit = vmw_stdu_bo_cpu_commit;
-		ddirty.base.clip = vmw_stdu_bo_cpu_clip;
-		ddirty.base.fifo_reserve_size = 0;
-	}
+
+	ddirty.base.fifo_commit = vmw_stdu_bo_cpu_commit;
+	ddirty.base.clip = vmw_stdu_bo_cpu_clip;
+	ddirty.base.fifo_reserve_size = 0;
 
 	ddirty.base.crtc = crtc;
 
@@ -1160,11 +1013,8 @@ vmw_stdu_primary_plane_prepare_fb(struct drm_plane *plane,
 	/*
 	 * This should only happen if the buffer object is too large to create a
 	 * proxy surface for.
-	 * If we are a 2D VM with a buffer object then we have to use CPU blit
-	 * so cache these mappings
 	 */
-	if (vps->content_fb_type == SEPARATE_BO &&
-	    vmw_stdu_use_cpu_blit(dev_priv))
+	if (vps->content_fb_type == SEPARATE_BO)
 		vps->cpp = new_fb->pitches[0] / new_fb->width;
 
 	return 0;
@@ -1174,14 +1024,6 @@ out_srf_unref:
 	return ret;
 }
 
-static uint32_t vmw_stdu_bo_fifo_size(struct vmw_du_update_plane *update,
-				      uint32_t num_hits)
-{
-	return sizeof(struct vmw_stdu_dma) + sizeof(SVGA3dCopyBox) * num_hits +
-		sizeof(SVGA3dCmdSurfaceDMASuffix) +
-		sizeof(struct vmw_stdu_update);
-}
-
 static uint32_t vmw_stdu_bo_fifo_size_cpu(struct vmw_du_update_plane *update,
 					  uint32_t num_hits)
 {
@@ -1189,68 +1031,6 @@ static uint32_t vmw_stdu_bo_fifo_size_cpu(struct vmw_du_update_plane *update,
 		sizeof(struct vmw_stdu_update);
 }
 
-static uint32_t vmw_stdu_bo_populate_dma(struct vmw_du_update_plane  *update,
-					 void *cmd, uint32_t num_hits)
-{
-	struct vmw_screen_target_display_unit *stdu;
-	struct vmw_framebuffer_bo *vfbbo;
-	struct vmw_stdu_dma *cmd_dma = cmd;
-
-	stdu = container_of(update->du, typeof(*stdu), base);
-	vfbbo = container_of(update->vfb, typeof(*vfbbo), base);
-
-	cmd_dma->header.id = SVGA_3D_CMD_SURFACE_DMA;
-	cmd_dma->header.size = sizeof(cmd_dma->body) +
-		sizeof(struct SVGA3dCopyBox) * num_hits +
-		sizeof(SVGA3dCmdSurfaceDMASuffix);
-	vmw_bo_get_guest_ptr(&vfbbo->buffer->base, &cmd_dma->body.guest.ptr);
-	cmd_dma->body.guest.pitch = update->vfb->base.pitches[0];
-	cmd_dma->body.host.sid = stdu->display_srf->res.id;
-	cmd_dma->body.host.face = 0;
-	cmd_dma->body.host.mipmap = 0;
-	cmd_dma->body.transfer = SVGA3D_WRITE_HOST_VRAM;
-
-	return sizeof(*cmd_dma);
-}
-
-static uint32_t vmw_stdu_bo_populate_clip(struct vmw_du_update_plane  *update,
-					  void *cmd, struct drm_rect *clip,
-					  uint32_t fb_x, uint32_t fb_y)
-{
-	struct SVGA3dCopyBox *box = cmd;
-
-	box->srcx = fb_x;
-	box->srcy = fb_y;
-	box->srcz = 0;
-	box->x = clip->x1;
-	box->y = clip->y1;
-	box->z = 0;
-	box->w = drm_rect_width(clip);
-	box->h = drm_rect_height(clip);
-	box->d = 1;
-
-	return sizeof(*box);
-}
-
-static uint32_t vmw_stdu_bo_populate_update(struct vmw_du_update_plane  *update,
-					    void *cmd, struct drm_rect *bb)
-{
-	struct vmw_screen_target_display_unit *stdu;
-	struct vmw_framebuffer_bo *vfbbo;
-	SVGA3dCmdSurfaceDMASuffix *suffix = cmd;
-
-	stdu = container_of(update->du, typeof(*stdu), base);
-	vfbbo = container_of(update->vfb, typeof(*vfbbo), base);
-
-	suffix->suffixSize = sizeof(*suffix);
-	suffix->maximumOffset = vfbbo->buffer->base.base.size;
-
-	vmw_stdu_populate_update(&suffix[1], stdu->base.unit, bb->x1, bb->x2,
-				 bb->y1, bb->y2);
-
-	return sizeof(*suffix) + sizeof(struct vmw_stdu_update);
-}
-
 static uint32_t vmw_stdu_bo_pre_clip_cpu(struct vmw_du_update_plane  *update,
 					 void *cmd, uint32_t num_hits)
 {
@@ -1300,11 +1080,11 @@ vmw_stdu_bo_populate_update_cpu(struct vmw_du_update_plane  *update, void *cmd,
 
 	diff.cpp = stdu->cpp;
 
-	dst_bo = &stdu->display_srf->res.backup->base;
+	dst_bo = &stdu->display_srf->res.guest_memory_bo->tbo;
 	dst_pitch = stdu->display_srf->metadata.base_size.width * stdu->cpp;
 	dst_offset = bb->y1 * dst_pitch + bb->x1 * stdu->cpp;
 
-	src_bo = &vfbbo->buffer->base;
+	src_bo = &vfbbo->buffer->tbo;
 	src_pitch = update->vfb->base.pitches[0];
 	src_offset = bo_update->fb_top * src_pitch + bo_update->fb_left *
 		stdu->cpp;
@@ -1368,24 +1148,12 @@ static int vmw_stdu_plane_update_bo(struct vmw_private *dev_priv,
 	bo_update.base.vfb = vfb;
 	bo_update.base.out_fence = out_fence;
 	bo_update.base.mutex = NULL;
-	bo_update.base.cpu_blit = vmw_stdu_use_cpu_blit(dev_priv);
 	bo_update.base.intr = false;
 
-	/*
-	 * VM without 3D support don't have surface DMA command and framebuffer
-	 * should be moved out of VRAM.
-	 */
-	if (bo_update.base.cpu_blit) {
-		bo_update.base.calc_fifo_size = vmw_stdu_bo_fifo_size_cpu;
-		bo_update.base.pre_clip = vmw_stdu_bo_pre_clip_cpu;
-		bo_update.base.clip = vmw_stdu_bo_clip_cpu;
-		bo_update.base.post_clip = vmw_stdu_bo_populate_update_cpu;
-	} else {
-		bo_update.base.calc_fifo_size = vmw_stdu_bo_fifo_size;
-		bo_update.base.pre_clip = vmw_stdu_bo_populate_dma;
-		bo_update.base.clip = vmw_stdu_bo_populate_clip;
-		bo_update.base.post_clip = vmw_stdu_bo_populate_update;
-	}
+	bo_update.base.calc_fifo_size = vmw_stdu_bo_fifo_size_cpu;
+	bo_update.base.pre_clip = vmw_stdu_bo_pre_clip_cpu;
+	bo_update.base.clip = vmw_stdu_bo_clip_cpu;
+	bo_update.base.post_clip = vmw_stdu_bo_populate_update_cpu;
 
 	return vmw_du_helper_plane_update(&bo_update.base);
 }
@@ -1548,7 +1316,6 @@ static int vmw_stdu_plane_update_surface(struct vmw_private *dev_priv,
 	srf_update.vfb = vfb;
 	srf_update.out_fence = out_fence;
 	srf_update.mutex = &dev_priv->cmdbuf_mutex;
-	srf_update.cpu_blit = false;
 	srf_update.intr = true;
 
 	if (vfbs->is_bo_proxy)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_streamoutput.c b/drivers/gpu/drm/vmwgfx/vmwgfx_streamoutput.c
index 2de97419d5c9..edcc40659038 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_streamoutput.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_streamoutput.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright © 2018-2019 VMware, Inc., Palo Alto, CA., USA
+ * Copyright © 2018-2023 VMware, Inc., Palo Alto, CA., USA
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -26,11 +26,12 @@
  *
  **************************************************************************/
 
-#include <drm/ttm/ttm_placement.h>
-
+#include "vmwgfx_binding.h"
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 #include "vmwgfx_resource_priv.h"
-#include "vmwgfx_binding.h"
+
+#include <drm/ttm/ttm_placement.h>
 
 /**
  * struct vmw_dx_streamoutput - Streamoutput resource metadata.
@@ -62,10 +63,11 @@ static void vmw_dx_streamoutput_commit_notify(struct vmw_resource *res,
 
 static const struct vmw_res_func vmw_dx_streamoutput_func = {
 	.res_type = vmw_res_streamoutput,
-	.needs_backup = true,
+	.needs_guest_memory = true,
 	.may_evict = false,
 	.type_name = "DX streamoutput",
-	.backup_placement = &vmw_mob_placement,
+	.domain = VMW_BO_DOMAIN_MOB,
+	.busy_domain = VMW_BO_DOMAIN_MOB,
 	.create = vmw_dx_streamoutput_create,
 	.destroy = NULL, /* Command buffer managed resource. */
 	.bind = vmw_dx_streamoutput_bind,
@@ -104,8 +106,8 @@ static int vmw_dx_streamoutput_unscrub(struct vmw_resource *res)
 	cmd->header.id = SVGA_3D_CMD_DX_BIND_STREAMOUTPUT;
 	cmd->header.size = sizeof(cmd->body);
 	cmd->body.soid = so->id;
-	cmd->body.mobid = res->backup->base.resource->start;
-	cmd->body.offsetInBytes = res->backup_offset;
+	cmd->body.mobid = res->guest_memory_bo->tbo.resource->start;
+	cmd->body.offsetInBytes = res->guest_memory_offset;
 	cmd->body.sizeInBytes = so->size;
 	vmw_cmd_commit(dev_priv, sizeof(*cmd));
 
@@ -195,7 +197,7 @@ static int vmw_dx_streamoutput_unbind(struct vmw_resource *res, bool readback,
 	struct vmw_fence_obj *fence;
 	int ret;
 
-	if (WARN_ON(res->backup->base.resource->mem_type != VMW_PL_MOB))
+	if (WARN_ON(res->guest_memory_bo->tbo.resource->mem_type != VMW_PL_MOB))
 		return -EINVAL;
 
 	mutex_lock(&dev_priv->binding_mutex);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
index dcfb003841b3..5db403ee8261 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009-2015 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -25,8 +25,7 @@
  *
  **************************************************************************/
 
-#include <drm/ttm/ttm_placement.h>
-
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 #include "vmwgfx_resource_priv.h"
 #include "vmwgfx_so.h"
@@ -34,6 +33,8 @@
 #include "vmw_surface_cache.h"
 #include "device_include/svga3d_surfacedefs.h"
 
+#include <drm/ttm/ttm_placement.h>
+
 #define SVGA3D_FLAGS_64(upper32, lower32) (((uint64_t)upper32 << 32) | lower32)
 #define SVGA3D_FLAGS_UPPER_32(svga3d_flags) (svga3d_flags >> 32)
 #define SVGA3D_FLAGS_LOWER_32(svga3d_flags) \
@@ -125,12 +126,13 @@ const struct vmw_user_resource_conv *user_surface_converter =
 
 static const struct vmw_res_func vmw_legacy_surface_func = {
 	.res_type = vmw_res_surface,
-	.needs_backup = false,
+	.needs_guest_memory = false,
 	.may_evict = true,
 	.prio = 1,
 	.dirty_prio = 1,
 	.type_name = "legacy surfaces",
-	.backup_placement = &vmw_srf_placement,
+	.domain = VMW_BO_DOMAIN_GMR,
+	.busy_domain = VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM,
 	.create = &vmw_legacy_srf_create,
 	.destroy = &vmw_legacy_srf_destroy,
 	.bind = &vmw_legacy_srf_bind,
@@ -139,12 +141,13 @@ static const struct vmw_res_func vmw_legacy_surface_func = {
 
 static const struct vmw_res_func vmw_gb_surface_func = {
 	.res_type = vmw_res_surface,
-	.needs_backup = true,
+	.needs_guest_memory = true,
 	.may_evict = true,
 	.prio = 1,
 	.dirty_prio = 2,
 	.type_name = "guest backed surfaces",
-	.backup_placement = &vmw_mob_placement,
+	.domain = VMW_BO_DOMAIN_MOB,
+	.busy_domain = VMW_BO_DOMAIN_MOB,
 	.create = vmw_gb_surface_create,
 	.destroy = vmw_gb_surface_destroy,
 	.bind = vmw_gb_surface_bind,
@@ -379,7 +382,7 @@ static void vmw_hw_surface_destroy(struct vmw_resource *res)
 		 */
 
 		mutex_lock(&dev_priv->cmdbuf_mutex);
-		dev_priv->used_memory_size -= res->backup_size;
+		dev_priv->used_memory_size -= res->guest_memory_size;
 		mutex_unlock(&dev_priv->cmdbuf_mutex);
 	}
 }
@@ -409,7 +412,7 @@ static int vmw_legacy_srf_create(struct vmw_resource *res)
 		return 0;
 
 	srf = vmw_res_to_srf(res);
-	if (unlikely(dev_priv->used_memory_size + res->backup_size >=
+	if (unlikely(dev_priv->used_memory_size + res->guest_memory_size >=
 		     dev_priv->memory_size))
 		return -EBUSY;
 
@@ -447,7 +450,7 @@ static int vmw_legacy_srf_create(struct vmw_resource *res)
 	 * Surface memory usage accounting.
 	 */
 
-	dev_priv->used_memory_size += res->backup_size;
+	dev_priv->used_memory_size += res->guest_memory_size;
 	return 0;
 
 out_no_fifo:
@@ -524,7 +527,7 @@ static int vmw_legacy_srf_dma(struct vmw_resource *res,
 static int vmw_legacy_srf_bind(struct vmw_resource *res,
 			       struct ttm_validate_buffer *val_buf)
 {
-	if (!res->backup_dirty)
+	if (!res->guest_memory_dirty)
 		return 0;
 
 	return vmw_legacy_srf_dma(res, val_buf, true);
@@ -583,7 +586,7 @@ static int vmw_legacy_srf_destroy(struct vmw_resource *res)
 	 * Surface memory usage accounting.
 	 */
 
-	dev_priv->used_memory_size -= res->backup_size;
+	dev_priv->used_memory_size -= res->guest_memory_size;
 
 	/*
 	 * Release the surface ID.
@@ -683,8 +686,8 @@ static void vmw_user_surface_base_release(struct ttm_base_object **p_base)
 	    container_of(base, struct vmw_user_surface, prime.base);
 	struct vmw_resource *res = &user_srf->srf.res;
 
-	if (res && res->backup)
-		drm_gem_object_put(&res->backup->base.base);
+	if (res->guest_memory_bo)
+		drm_gem_object_put(&res->guest_memory_bo->tbo.base);
 
 	*p_base = NULL;
 	vmw_resource_unreference(&res);
@@ -812,7 +815,7 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 			++cur_size;
 		}
 	}
-	res->backup_size = cur_bo_offset;
+	res->guest_memory_size = cur_bo_offset;
 	if (metadata->scanout &&
 	    metadata->num_sizes == 1 &&
 	    metadata->sizes[0].width == VMW_CURSOR_SNOOP_WIDTH &&
@@ -856,14 +859,14 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 
 		ret = vmw_gem_object_create_with_handle(dev_priv,
 							file_priv,
-							res->backup_size,
+							res->guest_memory_size,
 							&backup_handle,
-							&res->backup);
+							&res->guest_memory_bo);
 		if (unlikely(ret != 0)) {
 			vmw_resource_unreference(&res);
 			goto out_unlock;
 		}
-		vmw_bo_reference(res->backup);
+		vmw_bo_reference(res->guest_memory_bo);
 		/*
 		 * We don't expose the handle to the userspace and surface
 		 * already holds a gem reference
@@ -872,7 +875,7 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 	}
 
 	tmp = vmw_resource_reference(&srf->res);
-	ret = ttm_prime_object_init(tfile, res->backup_size, &user_srf->prime,
+	ret = ttm_prime_object_init(tfile, res->guest_memory_size, &user_srf->prime,
 				    req->shareable, VMW_RES_SURFACE,
 				    &vmw_user_surface_base_release);
 
@@ -1186,7 +1189,7 @@ static int vmw_gb_surface_bind(struct vmw_resource *res,
 
 	BUG_ON(bo->resource->mem_type != VMW_PL_MOB);
 
-	submit_size = sizeof(*cmd1) + (res->backup_dirty ? sizeof(*cmd2) : 0);
+	submit_size = sizeof(*cmd1) + (res->guest_memory_dirty ? sizeof(*cmd2) : 0);
 
 	cmd1 = VMW_CMD_RESERVE(dev_priv, submit_size);
 	if (unlikely(!cmd1))
@@ -1196,7 +1199,7 @@ static int vmw_gb_surface_bind(struct vmw_resource *res,
 	cmd1->header.size = sizeof(cmd1->body);
 	cmd1->body.sid = res->id;
 	cmd1->body.mobid = bo->resource->start;
-	if (res->backup_dirty) {
+	if (res->guest_memory_dirty) {
 		cmd2 = (void *) &cmd1[1];
 		cmd2->header.id = SVGA_3D_CMD_UPDATE_GB_SURFACE;
 		cmd2->header.size = sizeof(cmd2->body);
@@ -1204,12 +1207,12 @@ static int vmw_gb_surface_bind(struct vmw_resource *res,
 	}
 	vmw_cmd_commit(dev_priv, submit_size);
 
-	if (res->backup->dirty && res->backup_dirty) {
+	if (res->guest_memory_bo->dirty && res->guest_memory_dirty) {
 		/* We've just made a full upload. Cear dirty regions. */
 		vmw_bo_dirty_clear_res(res);
 	}
 
-	res->backup_dirty = false;
+	res->guest_memory_dirty = false;
 
 	return 0;
 }
@@ -1505,11 +1508,11 @@ vmw_gb_surface_define_internal(struct drm_device *dev,
 
 	if (req->base.buffer_handle != SVGA3D_INVALID_ID) {
 		ret = vmw_user_bo_lookup(file_priv, req->base.buffer_handle,
-					 &res->backup);
+					 &res->guest_memory_bo);
 		if (ret == 0) {
-			if (res->backup->base.base.size < res->backup_size) {
+			if (res->guest_memory_bo->tbo.base.size < res->guest_memory_size) {
 				VMW_DEBUG_USER("Surface backup buffer too small.\n");
-				vmw_bo_unreference(&res->backup);
+				vmw_bo_unreference(&res->guest_memory_bo);
 				ret = -EINVAL;
 				goto out_unlock;
 			} else {
@@ -1520,11 +1523,11 @@ vmw_gb_surface_define_internal(struct drm_device *dev,
 		   (drm_vmw_surface_flag_create_buffer |
 		    drm_vmw_surface_flag_coherent)) {
 		ret = vmw_gem_object_create_with_handle(dev_priv, file_priv,
-							res->backup_size,
+							res->guest_memory_size,
 							&backup_handle,
-							&res->backup);
+							&res->guest_memory_bo);
 		if (ret == 0)
-			vmw_bo_reference(res->backup);
+			vmw_bo_reference(res->guest_memory_bo);
 	}
 
 	if (unlikely(ret != 0)) {
@@ -1533,9 +1536,9 @@ vmw_gb_surface_define_internal(struct drm_device *dev,
 	}
 
 	if (req->base.drm_surface_flags & drm_vmw_surface_flag_coherent) {
-		struct vmw_buffer_object *backup = res->backup;
+		struct vmw_bo *backup = res->guest_memory_bo;
 
-		ttm_bo_reserve(&backup->base, false, false, NULL);
+		ttm_bo_reserve(&backup->tbo, false, false, NULL);
 		if (!res->func->dirty_alloc)
 			ret = -EINVAL;
 		if (!ret)
@@ -1544,7 +1547,7 @@ vmw_gb_surface_define_internal(struct drm_device *dev,
 			res->coherent = true;
 			ret = res->func->dirty_alloc(res);
 		}
-		ttm_bo_unreserve(&backup->base);
+		ttm_bo_unreserve(&backup->tbo);
 		if (ret) {
 			vmw_resource_unreference(&res);
 			goto out_unlock;
@@ -1553,7 +1556,7 @@ vmw_gb_surface_define_internal(struct drm_device *dev,
 	}
 
 	tmp = vmw_resource_reference(res);
-	ret = ttm_prime_object_init(tfile, res->backup_size, &user_srf->prime,
+	ret = ttm_prime_object_init(tfile, res->guest_memory_size, &user_srf->prime,
 				    req->base.drm_surface_flags &
 				    drm_vmw_surface_flag_shareable,
 				    VMW_RES_SURFACE,
@@ -1566,11 +1569,11 @@ vmw_gb_surface_define_internal(struct drm_device *dev,
 	}
 
 	rep->handle      = user_srf->prime.base.handle;
-	rep->backup_size = res->backup_size;
-	if (res->backup) {
+	rep->backup_size = res->guest_memory_size;
+	if (res->guest_memory_bo) {
 		rep->buffer_map_handle =
-			drm_vma_node_offset_addr(&res->backup->base.base.vma_node);
-		rep->buffer_size = res->backup->base.base.size;
+			drm_vma_node_offset_addr(&res->guest_memory_bo->tbo.base.vma_node);
+		rep->buffer_size = res->guest_memory_bo->tbo.base.size;
 		rep->buffer_handle = backup_handle;
 	} else {
 		rep->buffer_map_handle = 0;
@@ -1613,14 +1616,14 @@ vmw_gb_surface_reference_internal(struct drm_device *dev,
 
 	user_srf = container_of(base, struct vmw_user_surface, prime.base);
 	srf = &user_srf->srf;
-	if (!srf->res.backup) {
+	if (!srf->res.guest_memory_bo) {
 		DRM_ERROR("Shared GB surface is missing a backup buffer.\n");
 		goto out_bad_resource;
 	}
 	metadata = &srf->metadata;
 
 	mutex_lock(&dev_priv->cmdbuf_mutex); /* Protect res->backup */
-	ret = drm_gem_handle_create(file_priv, &srf->res.backup->base.base,
+	ret = drm_gem_handle_create(file_priv, &srf->res.guest_memory_bo->tbo.base,
 				    &backup_handle);
 	mutex_unlock(&dev_priv->cmdbuf_mutex);
 	if (ret != 0) {
@@ -1639,11 +1642,11 @@ vmw_gb_surface_reference_internal(struct drm_device *dev,
 	rep->creq.base.buffer_handle = backup_handle;
 	rep->creq.base.base_size = metadata->base_size;
 	rep->crep.handle = user_srf->prime.base.handle;
-	rep->crep.backup_size = srf->res.backup_size;
+	rep->crep.backup_size = srf->res.guest_memory_size;
 	rep->crep.buffer_handle = backup_handle;
 	rep->crep.buffer_map_handle =
-		drm_vma_node_offset_addr(&srf->res.backup->base.base.vma_node);
-	rep->crep.buffer_size = srf->res.backup->base.base.size;
+		drm_vma_node_offset_addr(&srf->res.guest_memory_bo->tbo.base.vma_node);
+	rep->crep.buffer_size = srf->res.guest_memory_bo->tbo.base.size;
 
 	rep->creq.version = drm_vmw_gb_surface_v1;
 	rep->creq.svga3d_flags_upper_32_bits =
@@ -1742,12 +1745,12 @@ static void vmw_surface_tex_dirty_range_add(struct vmw_resource *res,
 {
 	struct vmw_surface_dirty *dirty =
 		(struct vmw_surface_dirty *) res->dirty;
-	size_t backup_end = res->backup_offset + res->backup_size;
+	size_t backup_end = res->guest_memory_offset + res->guest_memory_size;
 	struct vmw_surface_loc loc1, loc2;
 	const struct vmw_surface_cache *cache;
 
-	start = max_t(size_t, start, res->backup_offset) - res->backup_offset;
-	end = min(end, backup_end) - res->backup_offset;
+	start = max_t(size_t, start, res->guest_memory_offset) - res->guest_memory_offset;
+	end = min(end, backup_end) - res->guest_memory_offset;
 	cache = &dirty->cache;
 	vmw_surface_get_loc(cache, &loc1, start);
 	vmw_surface_get_loc(cache, &loc2, end - 1);
@@ -1794,13 +1797,13 @@ static void vmw_surface_buf_dirty_range_add(struct vmw_resource *res,
 	struct vmw_surface_dirty *dirty =
 		(struct vmw_surface_dirty *) res->dirty;
 	const struct vmw_surface_cache *cache = &dirty->cache;
-	size_t backup_end = res->backup_offset + cache->mip_chain_bytes;
+	size_t backup_end = res->guest_memory_offset + cache->mip_chain_bytes;
 	SVGA3dBox *box = &dirty->boxes[0];
 	u32 box_c2;
 
 	box->h = box->d = 1;
-	start = max_t(size_t, start, res->backup_offset) - res->backup_offset;
-	end = min(end, backup_end) - res->backup_offset;
+	start = max_t(size_t, start, res->guest_memory_offset) - res->guest_memory_offset;
+	end = min(end, backup_end) - res->guest_memory_offset;
 	box_c2 = box->x + box->w;
 	if (box->w == 0 || box->x > start)
 		box->x = start;
@@ -1816,8 +1819,8 @@ static void vmw_surface_dirty_range_add(struct vmw_resource *res, size_t start,
 {
 	struct vmw_surface *srf = vmw_res_to_srf(res);
 
-	if (WARN_ON(end <= res->backup_offset ||
-		    start >= res->backup_offset + res->backup_size))
+	if (WARN_ON(end <= res->guest_memory_offset ||
+		    start >= res->guest_memory_offset + res->guest_memory_size))
 		return;
 
 	if (srf->metadata.format == SVGA3D_BUFFER)
@@ -2074,7 +2077,7 @@ int vmw_gb_surface_define(struct vmw_private *dev_priv,
 	if (metadata->flags & SVGA3D_SURFACE_MULTISAMPLE)
 		sample_count = metadata->multisample_count;
 
-	srf->res.backup_size =
+	srf->res.guest_memory_size =
 		vmw_surface_get_serialized_size_extended(
 				metadata->format,
 				metadata->base_size,
@@ -2083,7 +2086,7 @@ int vmw_gb_surface_define(struct vmw_private *dev_priv,
 				sample_count);
 
 	if (metadata->flags & SVGA3D_SURFACE_BIND_STREAM_OUTPUT)
-		srf->res.backup_size += sizeof(SVGA3dDXSOState);
+		srf->res.guest_memory_size += sizeof(SVGA3dDXSOState);
 
 	/*
 	 * Don't set SVGA3D_SURFACE_SCREENTARGET flag for a scanout surface with
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
index 856a352a72a6..af8562c95cc3 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright 2009-2015 VMware, Inc., Palo Alto, CA., USA
+ * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
@@ -25,6 +25,7 @@
  *
  **************************************************************************/
 
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 #include <drm/ttm/ttm_placement.h>
 
@@ -49,13 +50,6 @@ static const struct ttm_place gmr_placement_flags = {
 	.flags = 0
 };
 
-static const struct ttm_place mob_placement_flags = {
-	.fpfn = 0,
-	.lpfn = 0,
-	.mem_type = VMW_PL_MOB,
-	.flags = 0
-};
-
 struct ttm_placement vmw_vram_placement = {
 	.num_placement = 1,
 	.placement = &vram_placement_flags,
@@ -77,27 +71,6 @@ static const struct ttm_place vram_gmr_placement_flags[] = {
 	}
 };
 
-static const struct ttm_place gmr_vram_placement_flags[] = {
-	{
-		.fpfn = 0,
-		.lpfn = 0,
-		.mem_type = VMW_PL_GMR,
-		.flags = 0
-	}, {
-		.fpfn = 0,
-		.lpfn = 0,
-		.mem_type = TTM_PL_VRAM,
-		.flags = 0
-	}
-};
-
-static const struct ttm_place vmw_sys_placement_flags = {
-	.fpfn = 0,
-	.lpfn = 0,
-	.mem_type = VMW_PL_SYSTEM,
-	.flags = 0
-};
-
 struct ttm_placement vmw_vram_gmr_placement = {
 	.num_placement = 2,
 	.placement = vram_gmr_placement_flags,
@@ -105,13 +78,6 @@ struct ttm_placement vmw_vram_gmr_placement = {
 	.busy_placement = &gmr_placement_flags
 };
 
-struct ttm_placement vmw_vram_sys_placement = {
-	.num_placement = 1,
-	.placement = &vram_placement_flags,
-	.num_busy_placement = 1,
-	.busy_placement = &sys_placement_flags
-};
-
 struct ttm_placement vmw_sys_placement = {
 	.num_placement = 1,
 	.placement = &sys_placement_flags,
@@ -119,53 +85,6 @@ struct ttm_placement vmw_sys_placement = {
 	.busy_placement = &sys_placement_flags
 };
 
-struct ttm_placement vmw_pt_sys_placement = {
-	.num_placement = 1,
-	.placement = &vmw_sys_placement_flags,
-	.num_busy_placement = 1,
-	.busy_placement = &vmw_sys_placement_flags
-};
-
-static const struct ttm_place nonfixed_placement_flags[] = {
-	{
-		.fpfn = 0,
-		.lpfn = 0,
-		.mem_type = TTM_PL_SYSTEM,
-		.flags = 0
-	}, {
-		.fpfn = 0,
-		.lpfn = 0,
-		.mem_type = VMW_PL_GMR,
-		.flags = 0
-	}, {
-		.fpfn = 0,
-		.lpfn = 0,
-		.mem_type = VMW_PL_MOB,
-		.flags = 0
-	}
-};
-
-struct ttm_placement vmw_srf_placement = {
-	.num_placement = 1,
-	.num_busy_placement = 2,
-	.placement = &gmr_placement_flags,
-	.busy_placement = gmr_vram_placement_flags
-};
-
-struct ttm_placement vmw_mob_placement = {
-	.num_placement = 1,
-	.num_busy_placement = 1,
-	.placement = &mob_placement_flags,
-	.busy_placement = &mob_placement_flags
-};
-
-struct ttm_placement vmw_nonfixed_placement = {
-	.num_placement = 3,
-	.placement = nonfixed_placement_flags,
-	.num_busy_placement = 1,
-	.busy_placement = &sys_placement_flags
-};
-
 const size_t vmw_tt_size = sizeof(struct vmw_ttm_tt);
 
 /**
@@ -508,7 +427,7 @@ static struct ttm_tt *vmw_ttm_tt_create(struct ttm_buffer_object *bo,
 	if (!vmw_be)
 		return NULL;
 
-	vmw_be->dev_priv = container_of(bo->bdev, struct vmw_private, bdev);
+	vmw_be->dev_priv = vmw_priv_from_ttm(bo->bdev);
 	vmw_be->mob = NULL;
 
 	if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
@@ -534,7 +453,7 @@ static void vmw_evict_flags(struct ttm_buffer_object *bo,
 
 static int vmw_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem)
 {
-	struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev);
+	struct vmw_private *dev_priv = vmw_priv_from_ttm(bdev);
 
 	switch (mem->mem_type) {
 	case TTM_PL_SYSTEM:
@@ -596,9 +515,13 @@ static int vmw_move(struct ttm_buffer_object *bo,
 		    struct ttm_resource *new_mem,
 		    struct ttm_place *hop)
 {
-	struct ttm_resource_manager *old_man = ttm_manager_type(bo->bdev, bo->resource->mem_type);
-	struct ttm_resource_manager *new_man = ttm_manager_type(bo->bdev, new_mem->mem_type);
-	int ret;
+	struct ttm_resource_manager *new_man;
+	struct ttm_resource_manager *old_man = NULL;
+	int ret = 0;
+
+	new_man = ttm_manager_type(bo->bdev, new_mem->mem_type);
+	if (bo->resource)
+		old_man = ttm_manager_type(bo->bdev, bo->resource->mem_type);
 
 	if (new_man->use_tt && !vmw_memtype_is_system(new_mem->mem_type)) {
 		ret = vmw_ttm_bind(bo->bdev, bo->ttm, new_mem);
@@ -606,9 +529,15 @@ static int vmw_move(struct ttm_buffer_object *bo,
 			return ret;
 	}
 
+	if (!bo->resource || (bo->resource->mem_type == TTM_PL_SYSTEM &&
+			      bo->ttm == NULL)) {
+		ttm_bo_move_null(bo, new_mem);
+		return 0;
+	}
+
 	vmw_move_notify(bo, bo->resource, new_mem);
 
-	if (old_man->use_tt && new_man->use_tt) {
+	if (old_man && old_man->use_tt && new_man->use_tt) {
 		if (vmw_memtype_is_system(bo->resource->mem_type)) {
 			ttm_bo_move_null(bo, new_mem);
 			return 0;
@@ -645,34 +574,39 @@ struct ttm_device_funcs vmw_bo_driver = {
 };
 
 int vmw_bo_create_and_populate(struct vmw_private *dev_priv,
-			       unsigned long bo_size,
-			       struct ttm_buffer_object **bo_p)
+			       size_t bo_size, u32 domain,
+			       struct vmw_bo **bo_p)
 {
 	struct ttm_operation_ctx ctx = {
 		.interruptible = false,
 		.no_wait_gpu = false
 	};
-	struct ttm_buffer_object *bo;
+	struct vmw_bo *vbo;
 	int ret;
+	struct vmw_bo_params bo_params = {
+		.domain = domain,
+		.busy_domain = domain,
+		.bo_type = ttm_bo_type_kernel,
+		.size = bo_size,
+		.pin = true
+	};
 
-	ret = vmw_bo_create_kernel(dev_priv, bo_size,
-				   &vmw_pt_sys_placement,
-				   &bo);
+	ret = vmw_bo_create(dev_priv, &bo_params, &vbo);
 	if (unlikely(ret != 0))
 		return ret;
 
-	ret = ttm_bo_reserve(bo, false, true, NULL);
+	ret = ttm_bo_reserve(&vbo->tbo, false, true, NULL);
 	BUG_ON(ret != 0);
-	ret = vmw_ttm_populate(bo->bdev, bo->ttm, &ctx);
+	ret = vmw_ttm_populate(vbo->tbo.bdev, vbo->tbo.ttm, &ctx);
 	if (likely(ret == 0)) {
 		struct vmw_ttm_tt *vmw_tt =
-			container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm);
+			container_of(vbo->tbo.ttm, struct vmw_ttm_tt, dma_ttm);
 		ret = vmw_ttm_map_dma(vmw_tt);
 	}
 
-	ttm_bo_unreserve(bo);
+	ttm_bo_unreserve(&vbo->tbo);
 
 	if (likely(ret == 0))
-		*bo_p = bo;
+		*bo_p = vbo;
 	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_va.c b/drivers/gpu/drm/vmwgfx/vmwgfx_va.c
index 6ad744ae07f5..d140089e53d4 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_va.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_va.c
@@ -25,6 +25,7 @@
  *
  **************************************************************************/
 
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
 #include "vmwgfx_resource_priv.h"
 
@@ -80,10 +81,11 @@ static void vmw_stream_set_arg_handle(void *data, u32 handle)
 static const struct vmw_simple_resource_func va_stream_func = {
 	.res_func = {
 		.res_type = vmw_res_stream,
-		.needs_backup = false,
+		.needs_guest_memory = false,
 		.may_evict = false,
 		.type_name = "overlay stream",
-		.backup_placement = NULL,
+		.domain = VMW_BO_DOMAIN_SYS,
+		.busy_domain = VMW_BO_DOMAIN_SYS,
 		.create = NULL,
 		.destroy = NULL,
 		.bind = NULL,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c
index f5c4a40fb16d..aaacbdcbd742 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /**************************************************************************
  *
- * Copyright © 2018 - 2022 VMware, Inc., Palo Alto, CA., USA
+ * Copyright © 2018 - 2023 VMware, Inc., Palo Alto, CA., USA
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -25,9 +25,12 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  **************************************************************************/
-#include <linux/slab.h>
-#include "vmwgfx_validation.h"
+#include "vmwgfx_bo.h"
 #include "vmwgfx_drv.h"
+#include "vmwgfx_resource_priv.h"
+#include "vmwgfx_validation.h"
+
+#include <linux/slab.h>
 
 
 #define VMWGFX_VALIDATION_MEM_GRAN (16*PAGE_SIZE)
@@ -38,8 +41,6 @@
  * @hash: A hash entry used for the duplicate detection hash table.
  * @coherent_count: If switching backup buffers, number of new coherent
  * resources that will have this buffer as a backup buffer.
- * @as_mob: Validate as mob.
- * @cpu_blit: Validate for cpu blit access.
  *
  * Bit fields are used since these structures are allocated and freed in
  * large numbers and space conservation is desired.
@@ -48,21 +49,19 @@ struct vmw_validation_bo_node {
 	struct ttm_validate_buffer base;
 	struct vmwgfx_hash_item hash;
 	unsigned int coherent_count;
-	u32 as_mob : 1;
-	u32 cpu_blit : 1;
 };
 /**
  * struct vmw_validation_res_node - Resource validation metadata.
  * @head: List head for the resource validation list.
  * @hash: A hash entry used for the duplicate detection hash table.
  * @res: Reference counted resource pointer.
- * @new_backup: Non ref-counted pointer to new backup buffer to be assigned
- * to a resource.
- * @new_backup_offset: Offset into the new backup mob for resources that can
- * share MOBs.
+ * @new_guest_memory_bo: Non ref-counted pointer to new guest memory buffer
+ * to be assigned to a resource.
+ * @new_guest_memory_offset: Offset into the new backup mob for resources
+ * that can share MOBs.
  * @no_buffer_needed: Kernel does not need to allocate a MOB during validation,
  * the command stream provides a mob bind operation.
- * @switching_backup: The validation process is switching backup MOB.
+ * @switching_guest_memory_bo: The validation process is switching backup MOB.
  * @first_usage: True iff the resource has been seen only once in the current
  * validation batch.
  * @reserved: Whether the resource is currently reserved by this process.
@@ -77,10 +76,10 @@ struct vmw_validation_res_node {
 	struct list_head head;
 	struct vmwgfx_hash_item hash;
 	struct vmw_resource *res;
-	struct vmw_buffer_object *new_backup;
-	unsigned long new_backup_offset;
+	struct vmw_bo *new_guest_memory_bo;
+	unsigned long new_guest_memory_offset;
 	u32 no_buffer_needed : 1;
-	u32 switching_backup : 1;
+	u32 switching_guest_memory_bo : 1;
 	u32 first_usage : 1;
 	u32 reserved : 1;
 	u32 dirty : 1;
@@ -173,7 +172,7 @@ static void vmw_validation_mem_free(struct vmw_validation_context *ctx)
  */
 static struct vmw_validation_bo_node *
 vmw_validation_find_bo_dup(struct vmw_validation_context *ctx,
-			   struct vmw_buffer_object *vbo)
+			   struct vmw_bo *vbo)
 {
 	struct  vmw_validation_bo_node *bo_node = NULL;
 
@@ -194,7 +193,7 @@ vmw_validation_find_bo_dup(struct vmw_validation_context *ctx,
 		struct  vmw_validation_bo_node *entry;
 
 		list_for_each_entry(entry, &ctx->bo_list, base.head) {
-			if (entry->base.bo == &vbo->base) {
+			if (entry->base.bo == &vbo->tbo) {
 				bo_node = entry;
 				break;
 			}
@@ -258,26 +257,16 @@ out:
  * vmw_validation_add_bo - Add a buffer object to the validation context.
  * @ctx: The validation context.
  * @vbo: The buffer object.
- * @as_mob: Validate as mob, otherwise suitable for GMR operations.
- * @cpu_blit: Validate in a page-mappable location.
  *
  * Return: Zero on success, negative error code otherwise.
  */
 int vmw_validation_add_bo(struct vmw_validation_context *ctx,
-			  struct vmw_buffer_object *vbo,
-			  bool as_mob,
-			  bool cpu_blit)
+			  struct vmw_bo *vbo)
 {
 	struct vmw_validation_bo_node *bo_node;
 
 	bo_node = vmw_validation_find_bo_dup(ctx, vbo);
-	if (bo_node) {
-		if (bo_node->as_mob != as_mob ||
-		    bo_node->cpu_blit != cpu_blit) {
-			DRM_ERROR("Inconsistent buffer usage.\n");
-			return -EINVAL;
-		}
-	} else {
+	if (!bo_node) {
 		struct ttm_validate_buffer *val_buf;
 
 		bo_node = vmw_validation_mem_alloc(ctx, sizeof(*bo_node));
@@ -290,13 +279,11 @@ int vmw_validation_add_bo(struct vmw_validation_context *ctx,
 				bo_node->hash.key);
 		}
 		val_buf = &bo_node->base;
-		val_buf->bo = ttm_bo_get_unless_zero(&vbo->base);
+		val_buf->bo = ttm_bo_get_unless_zero(&vbo->tbo);
 		if (!val_buf->bo)
 			return -ESRCH;
 		val_buf->num_shared = 0;
 		list_add_tail(&val_buf->head, &ctx->bo_list);
-		bo_node->as_mob = as_mob;
-		bo_node->cpu_blit = cpu_blit;
 	}
 
 	return 0;
@@ -406,23 +393,23 @@ void vmw_validation_res_set_dirty(struct vmw_validation_context *ctx,
  * the resource.
  * @vbo: The new backup buffer object MOB. This buffer object needs to have
  * already been registered with the validation context.
- * @backup_offset: Offset into the new backup MOB.
+ * @guest_memory_offset: Offset into the new backup MOB.
  */
 void vmw_validation_res_switch_backup(struct vmw_validation_context *ctx,
 				      void *val_private,
-				      struct vmw_buffer_object *vbo,
-				      unsigned long backup_offset)
+				      struct vmw_bo *vbo,
+				      unsigned long guest_memory_offset)
 {
 	struct vmw_validation_res_node *val;
 
 	val = container_of(val_private, typeof(*val), private);
 
-	val->switching_backup = 1;
+	val->switching_guest_memory_bo = 1;
 	if (val->first_usage)
 		val->no_buffer_needed = 1;
 
-	val->new_backup = vbo;
-	val->new_backup_offset = backup_offset;
+	val->new_guest_memory_bo = vbo;
+	val->new_guest_memory_offset = guest_memory_offset;
 }
 
 /**
@@ -450,21 +437,22 @@ int vmw_validation_res_reserve(struct vmw_validation_context *ctx,
 			goto out_unreserve;
 
 		val->reserved = 1;
-		if (res->backup) {
-			struct vmw_buffer_object *vbo = res->backup;
+		if (res->guest_memory_bo) {
+			struct vmw_bo *vbo = res->guest_memory_bo;
 
-			ret = vmw_validation_add_bo
-				(ctx, vbo, vmw_resource_needs_backup(res),
-				 false);
+			vmw_bo_placement_set(vbo,
+					     res->func->domain,
+					     res->func->busy_domain);
+			ret = vmw_validation_add_bo(ctx, vbo);
 			if (ret)
 				goto out_unreserve;
 		}
 
-		if (val->switching_backup && val->new_backup &&
+		if (val->switching_guest_memory_bo && val->new_guest_memory_bo &&
 		    res->coherent) {
 			struct vmw_validation_bo_node *bo_node =
 				vmw_validation_find_bo_dup(ctx,
-							   val->new_backup);
+							   val->new_guest_memory_bo);
 
 			if (WARN_ON(!bo_node)) {
 				ret = -EINVAL;
@@ -507,9 +495,9 @@ void vmw_validation_res_unreserve(struct vmw_validation_context *ctx,
 				vmw_resource_unreserve(val->res,
 						       val->dirty_set,
 						       val->dirty,
-						       val->switching_backup,
-						       val->new_backup,
-						       val->new_backup_offset);
+						       val->switching_guest_memory_bo,
+						       val->new_guest_memory_bo,
+						       val->new_guest_memory_offset);
 		}
 }
 
@@ -517,17 +505,14 @@ void vmw_validation_res_unreserve(struct vmw_validation_context *ctx,
  * vmw_validation_bo_validate_single - Validate a single buffer object.
  * @bo: The TTM buffer object base.
  * @interruptible: Whether to perform waits interruptible if possible.
- * @validate_as_mob: Whether to validate in MOB memory.
  *
  * Return: Zero on success, -ERESTARTSYS if interrupted. Negative error
  * code on failure.
  */
-int vmw_validation_bo_validate_single(struct ttm_buffer_object *bo,
-				      bool interruptible,
-				      bool validate_as_mob)
+static int vmw_validation_bo_validate_single(struct ttm_buffer_object *bo,
+					     bool interruptible)
 {
-	struct vmw_buffer_object *vbo =
-		container_of(bo, struct vmw_buffer_object, base);
+	struct vmw_bo *vbo = to_vmw_bo(&bo->base);
 	struct ttm_operation_ctx ctx = {
 		.interruptible = interruptible,
 		.no_wait_gpu = false
@@ -537,30 +522,20 @@ int vmw_validation_bo_validate_single(struct ttm_buffer_object *bo,
 	if (atomic_read(&vbo->cpu_writers))
 		return -EBUSY;
 
-	if (vbo->base.pin_count > 0)
+	if (vbo->tbo.pin_count > 0)
 		return 0;
 
-	if (validate_as_mob)
-		return ttm_bo_validate(bo, &vmw_mob_placement, &ctx);
-
-	/**
-	 * Put BO in VRAM if there is space, otherwise as a GMR.
-	 * If there is no space in VRAM and GMR ids are all used up,
-	 * start evicting GMRs to make room. If the DMA buffer can't be
-	 * used as a GMR, this will return -ENOMEM.
-	 */
-
-	ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, &ctx);
+	ret = ttm_bo_validate(bo, &vbo->placement, &ctx);
 	if (ret == 0 || ret == -ERESTARTSYS)
 		return ret;
 
-	/**
-	 * If that failed, try VRAM again, this time evicting
+	/*
+	 * If that failed, try again, this time evicting
 	 * previous contents.
 	 */
+	ctx.allow_res_evict = true;
 
-	ret = ttm_bo_validate(bo, &vmw_vram_placement, &ctx);
-	return ret;
+	return ttm_bo_validate(bo, &vbo->placement, &ctx);
 }
 
 /**
@@ -578,21 +553,10 @@ int vmw_validation_bo_validate(struct vmw_validation_context *ctx, bool intr)
 	int ret;
 
 	list_for_each_entry(entry, &ctx->bo_list, base.head) {
-		struct vmw_buffer_object *vbo =
-			container_of(entry->base.bo, typeof(*vbo), base);
-
-		if (entry->cpu_blit) {
-			struct ttm_operation_ctx ttm_ctx = {
-				.interruptible = intr,
-				.no_wait_gpu = false
-			};
-
-			ret = ttm_bo_validate(entry->base.bo,
-					      &vmw_nonfixed_placement, &ttm_ctx);
-		} else {
-			ret = vmw_validation_bo_validate_single
-			(entry->base.bo, intr, entry->as_mob);
-		}
+		struct vmw_bo *vbo = to_vmw_bo(&entry->base.bo->base);
+
+		ret = vmw_validation_bo_validate_single(entry->base.bo, intr);
+
 		if (ret)
 			return ret;
 
@@ -639,7 +603,7 @@ int vmw_validation_res_validate(struct vmw_validation_context *ctx, bool intr)
 
 	list_for_each_entry(val, &ctx->resource_list, head) {
 		struct vmw_resource *res = val->res;
-		struct vmw_buffer_object *backup = res->backup;
+		struct vmw_bo *backup = res->guest_memory_bo;
 
 		ret = vmw_resource_validate(res, intr, val->dirty_set &&
 					    val->dirty);
@@ -650,12 +614,12 @@ int vmw_validation_res_validate(struct vmw_validation_context *ctx, bool intr)
 		}
 
 		/* Check if the resource switched backup buffer */
-		if (backup && res->backup && (backup != res->backup)) {
-			struct vmw_buffer_object *vbo = res->backup;
+		if (backup && res->guest_memory_bo && backup != res->guest_memory_bo) {
+			struct vmw_bo *vbo = res->guest_memory_bo;
 
-			ret = vmw_validation_add_bo
-				(ctx, vbo, vmw_resource_needs_backup(res),
-				 false);
+			vmw_bo_placement_set(vbo, res->func->domain,
+					     res->func->busy_domain);
+			ret = vmw_validation_add_bo(ctx, vbo);
 			if (ret)
 				return ret;
 		}
@@ -889,9 +853,7 @@ void vmw_validation_bo_backoff(struct vmw_validation_context *ctx)
 	list_for_each_entry(entry, &ctx->bo_list, base.head) {
 		if (entry->coherent_count) {
 			unsigned int coherent_count = entry->coherent_count;
-			struct vmw_buffer_object *vbo =
-				container_of(entry->base.bo, typeof(*vbo),
-					     base);
+			struct vmw_bo *vbo = to_vmw_bo(&entry->base.bo->base);
 
 			while (coherent_count--)
 				vmw_bo_dirty_release(vbo);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h
index ab9ec226f433..240ee0c4ebfd 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h
@@ -73,7 +73,7 @@ struct vmw_validation_context {
 	size_t total_mem;
 };
 
-struct vmw_buffer_object;
+struct vmw_bo;
 struct vmw_resource;
 struct vmw_fence_obj;
 
@@ -159,11 +159,7 @@ static inline unsigned int vmw_validation_align(unsigned int val)
 }
 
 int vmw_validation_add_bo(struct vmw_validation_context *ctx,
-			  struct vmw_buffer_object *vbo,
-			  bool as_mob, bool cpu_blit);
-int vmw_validation_bo_validate_single(struct ttm_buffer_object *bo,
-				      bool interruptible,
-				      bool validate_as_mob);
+			  struct vmw_bo *vbo);
 int vmw_validation_bo_validate(struct vmw_validation_context *ctx, bool intr);
 void vmw_validation_unref_lists(struct vmw_validation_context *ctx);
 int vmw_validation_add_resource(struct vmw_validation_context *ctx,
@@ -179,7 +175,7 @@ void vmw_validation_res_unreserve(struct vmw_validation_context *ctx,
 				  bool backoff);
 void vmw_validation_res_switch_backup(struct vmw_validation_context *ctx,
 				      void *val_private,
-				      struct vmw_buffer_object *vbo,
+				      struct vmw_bo *vbo,
 				      unsigned long backup_offset);
 int vmw_validation_res_validate(struct vmw_validation_context *ctx, bool intr);
 
diff --git a/drivers/gpu/drm/xlnx/zynqmp_kms.c b/drivers/gpu/drm/xlnx/zynqmp_kms.c
index 776ef5480206..a7f8611be6f4 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_kms.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_kms.c
@@ -19,7 +19,7 @@
 #include <drm/drm_device.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_encoder.h>
-#include <drm/drm_fbdev_generic.h>
+#include <drm/drm_fbdev_dma.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_dma_helper.h>
@@ -515,7 +515,7 @@ int zynqmp_dpsub_drm_init(struct zynqmp_dpsub *dpsub)
 		goto err_poll_fini;
 
 	/* Initialize fbdev generic emulation. */
-	drm_fbdev_generic_setup(drm, 24);
+	drm_fbdev_dma_setup(drm, 24);
 
 	return 0;
 
diff --git a/drivers/gpu/host1x/Kconfig b/drivers/gpu/host1x/Kconfig
index 1861a8180d3f..e6c78ae2003a 100644
--- a/drivers/gpu/host1x/Kconfig
+++ b/drivers/gpu/host1x/Kconfig
@@ -5,7 +5,7 @@ config TEGRA_HOST1X_CONTEXT_BUS
 
 config TEGRA_HOST1X
 	tristate "NVIDIA Tegra host1x driver"
-	depends on ARCH_TEGRA || (ARM && COMPILE_TEST)
+	depends on ARCH_TEGRA || COMPILE_TEST
 	select DMA_SHARED_BUFFER
 	select TEGRA_HOST1X_CONTEXT_BUS
 	select IOMMU_IOVA
diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c
index bc7271a00a94..4d16a3396c4a 100644
--- a/drivers/gpu/host1x/bus.c
+++ b/drivers/gpu/host1x/bus.c
@@ -803,7 +803,7 @@ EXPORT_SYMBOL(__host1x_client_register);
  * Removes a host1x client from its host1x controller instance. If a logical
  * device has already been initialized, it will be torn down.
  */
-int host1x_client_unregister(struct host1x_client *client)
+void host1x_client_unregister(struct host1x_client *client)
 {
 	struct host1x_client *c;
 	struct host1x *host1x;
@@ -815,7 +815,7 @@ int host1x_client_unregister(struct host1x_client *client)
 		err = host1x_del_client(host1x, client);
 		if (!err) {
 			mutex_unlock(&devices_lock);
-			return 0;
+			return;
 		}
 	}
 
@@ -832,8 +832,6 @@ int host1x_client_unregister(struct host1x_client *client)
 	mutex_unlock(&clients_lock);
 
 	host1x_bo_cache_destroy(&client->cache);
-
-	return 0;
 }
 EXPORT_SYMBOL(host1x_client_unregister);
 
diff --git a/drivers/gpu/host1x/context.c b/drivers/gpu/host1x/context.c
index 8beedcf080ab..9ad89d22c0ca 100644
--- a/drivers/gpu/host1x/context.c
+++ b/drivers/gpu/host1x/context.c
@@ -13,6 +13,11 @@
 #include "context.h"
 #include "dev.h"
 
+static void host1x_memory_context_release(struct device *dev)
+{
+	/* context device is freed in host1x_memory_context_list_free() */
+}
+
 int host1x_memory_context_list_init(struct host1x *host1x)
 {
 	struct host1x_memory_context_list *cdl = &host1x->context_list;
@@ -51,38 +56,41 @@ int host1x_memory_context_list_init(struct host1x *host1x)
 		dev_set_name(&ctx->dev, "host1x-ctx.%d", i);
 		ctx->dev.bus = &host1x_context_device_bus_type;
 		ctx->dev.parent = host1x->dev;
+		ctx->dev.release = host1x_memory_context_release;
 
 		dma_set_max_seg_size(&ctx->dev, UINT_MAX);
 
 		err = device_add(&ctx->dev);
 		if (err) {
 			dev_err(host1x->dev, "could not add context device %d: %d\n", i, err);
-			goto del_devices;
+			put_device(&ctx->dev);
+			goto unreg_devices;
 		}
 
 		err = of_dma_configure_id(&ctx->dev, node, true, &i);
 		if (err) {
 			dev_err(host1x->dev, "IOMMU configuration failed for context device %d: %d\n",
 				i, err);
-			device_del(&ctx->dev);
-			goto del_devices;
+			device_unregister(&ctx->dev);
+			goto unreg_devices;
 		}
 
 		if (!tegra_dev_iommu_get_stream_id(&ctx->dev, &ctx->stream_id) ||
 		    !device_iommu_mapped(&ctx->dev)) {
 			dev_err(host1x->dev, "Context device %d has no IOMMU!\n", i);
-			device_del(&ctx->dev);
-			goto del_devices;
+			device_unregister(&ctx->dev);
+			goto unreg_devices;
 		}
 	}
 
 	return 0;
 
-del_devices:
+unreg_devices:
 	while (i--)
-		device_del(&cdl->devs[i].dev);
+		device_unregister(&cdl->devs[i].dev);
 
 	kfree(cdl->devs);
+	cdl->devs = NULL;
 	cdl->len = 0;
 
 	return err;
@@ -93,7 +101,7 @@ void host1x_memory_context_list_free(struct host1x_memory_context_list *cdl)
 	unsigned int i;
 
 	for (i = 0; i < cdl->len; i++)
-		device_del(&cdl->devs[i].dev);
+		device_unregister(&cdl->devs[i].dev);
 
 	kfree(cdl->devs);
 	cdl->len = 0;
diff --git a/drivers/gpu/host1x/mipi.c b/drivers/gpu/host1x/mipi.c
index 2efe12dde8bc..4dcec535ec21 100644
--- a/drivers/gpu/host1x/mipi.c
+++ b/drivers/gpu/host1x/mipi.c
@@ -501,7 +501,6 @@ static int tegra_mipi_probe(struct platform_device *pdev)
 {
 	const struct of_device_id *match;
 	struct tegra_mipi *mipi;
-	struct resource *res;
 	int err;
 
 	match = of_match_node(tegra_mipi_of_match, pdev->dev.of_node);
@@ -515,8 +514,7 @@ static int tegra_mipi_probe(struct platform_device *pdev)
 	mipi->soc = match->data;
 	mipi->dev = &pdev->dev;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	mipi->regs = devm_ioremap_resource(&pdev->dev, res);
+	mipi->regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
 	if (IS_ERR(mipi->regs))
 		return PTR_ERR(mipi->regs);
 
diff --git a/drivers/gpu/host1x/syncpt.c b/drivers/gpu/host1x/syncpt.c
index 2d2007760eac..f63d14a57a1d 100644
--- a/drivers/gpu/host1x/syncpt.c
+++ b/drivers/gpu/host1x/syncpt.c
@@ -248,7 +248,13 @@ int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh, long timeout,
 	if (value)
 		*value = host1x_syncpt_load(sp);
 
-	if (wait_err == 0)
+	/*
+	 * Don't rely on dma_fence_wait_timeout return value,
+	 * since it returns zero both on timeout and if the
+	 * wait completed with 0 jiffies left.
+	 */
+	host1x_hw_syncpt_load(sp->host, sp);
+	if (wait_err == 0 && !host1x_syncpt_is_expired(sp, thresh))
 		return -EAGAIN;
 	else if (wait_err < 0)
 		return wait_err;
diff --git a/drivers/gpu/ipu-v3/Kconfig b/drivers/gpu/ipu-v3/Kconfig
index 061fb990c120..7dece2a53c5c 100644
--- a/drivers/gpu/ipu-v3/Kconfig
+++ b/drivers/gpu/ipu-v3/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config IMX_IPUV3_CORE
 	tristate "IPUv3 core support"
-	depends on SOC_IMX5 || SOC_IMX6Q || ARCH_MULTIPLATFORM || COMPILE_TEST
+	depends on SOC_IMX5 || SOC_IMX6Q || COMPILE_TEST
 	depends on DRM || !DRM # if DRM=m, this can't be 'y'
 	select BITREVERSE
 	select GENERIC_ALLOCATOR if DRM
diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_client.c b/drivers/hid/amd-sfh-hid/amd_sfh_client.c
index c751d12f5df8..d9b7b01900b5 100644
--- a/drivers/hid/amd-sfh-hid/amd_sfh_client.c
+++ b/drivers/hid/amd-sfh-hid/amd_sfh_client.c
@@ -147,6 +147,7 @@ static const char *get_sensor_name(int idx)
 	case mag_idx:
 		return "magnetometer";
 	case als_idx:
+	case ACS_IDX: /* ambient color sensor */
 		return "ALS";
 	case HPD_IDX:
 		return "HPD";
diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_hid.h b/drivers/hid/amd-sfh-hid/amd_sfh_hid.h
index 528036892c9d..97296f587bc7 100644
--- a/drivers/hid/amd-sfh-hid/amd_sfh_hid.h
+++ b/drivers/hid/amd-sfh-hid/amd_sfh_hid.h
@@ -11,7 +11,7 @@
 #ifndef AMDSFH_HID_H
 #define AMDSFH_HID_H
 
-#define MAX_HID_DEVICES		5
+#define MAX_HID_DEVICES		6
 #define AMD_SFH_HID_VENDOR	0x1022
 #define AMD_SFH_HID_PRODUCT	0x0001
 
diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c
index 47774b9ab3de..2530fa98b568 100644
--- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c
+++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c
@@ -29,6 +29,7 @@
 #define MAGNO_EN	BIT(2)
 #define HPD_EN		BIT(16)
 #define ALS_EN		BIT(19)
+#define ACS_EN		BIT(22)
 
 static int sensor_mask_override = -1;
 module_param_named(sensor_mask, sensor_mask_override, int, 0444);
@@ -233,6 +234,9 @@ int amd_mp2_get_sensor_num(struct amd_mp2_dev *privdata, u8 *sensor_id)
 	if (HPD_EN & activestatus)
 		sensor_id[num_of_sensors++] = HPD_IDX;
 
+	if (ACS_EN & activestatus)
+		sensor_id[num_of_sensors++] = ACS_IDX;
+
 	return num_of_sensors;
 }
 
@@ -367,6 +371,14 @@ init_done:
 	return devm_add_action_or_reset(&pdev->dev, privdata->mp2_ops->remove, privdata);
 }
 
+static void amd_sfh_shutdown(struct pci_dev *pdev)
+{
+	struct amd_mp2_dev *mp2 = pci_get_drvdata(pdev);
+
+	if (mp2 && mp2->mp2_ops)
+		mp2->mp2_ops->stop_all(mp2);
+}
+
 static int __maybe_unused amd_mp2_pci_resume(struct device *dev)
 {
 	struct amd_mp2_dev *mp2 = dev_get_drvdata(dev);
@@ -401,6 +413,7 @@ static struct pci_driver amd_mp2_pci_driver = {
 	.id_table	= amd_mp2_pci_tbl,
 	.probe		= amd_mp2_pci_probe,
 	.driver.pm	= &amd_mp2_pm_ops,
+	.shutdown	= amd_sfh_shutdown,
 };
 module_pci_driver(amd_mp2_pci_driver);
 
diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h
index dfb7cabd82ef..70add75fc506 100644
--- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h
+++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h
@@ -23,6 +23,7 @@
 #define V2_STATUS	0x2
 
 #define HPD_IDX		16
+#define ACS_IDX		22
 
 #define SENSOR_DISCOVERY_STATUS_MASK		GENMASK(5, 3)
 #define SENSOR_DISCOVERY_STATUS_SHIFT		3
diff --git a/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c b/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c
index f9a8c02d5a7b..8716a05950c8 100644
--- a/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c
+++ b/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c
@@ -48,6 +48,7 @@ static int get_report_descriptor(int sensor_idx, u8 *rep_desc)
 		       sizeof(comp3_report_descriptor));
 		break;
 	case als_idx: /* ambient light sensor */
+	case ACS_IDX: /* ambient color sensor */
 		memset(rep_desc, 0, sizeof(als_report_descriptor));
 		memcpy(rep_desc, als_report_descriptor,
 		       sizeof(als_report_descriptor));
@@ -97,6 +98,7 @@ static u32 get_descr_sz(int sensor_idx, int descriptor_name)
 		}
 		break;
 	case als_idx:
+	case ACS_IDX: /* ambient color sensor */
 		switch (descriptor_name) {
 		case descr_size:
 			return sizeof(als_report_descriptor);
@@ -174,6 +176,7 @@ static u8 get_feature_report(int sensor_idx, int report_id, u8 *feature_report)
 		report_size = sizeof(magno_feature);
 		break;
 	case als_idx:  /* ambient light sensor */
+	case ACS_IDX: /* ambient color sensor */
 		get_common_features(&als_feature.common_property, report_id);
 		als_feature.als_change_sesnitivity = HID_DEFAULT_SENSITIVITY;
 		als_feature.als_sensitivity_min = HID_DEFAULT_MIN_VALUE;
@@ -245,6 +248,7 @@ static u8 get_input_report(u8 current_index, int sensor_idx, int report_id,
 		report_size = sizeof(magno_input);
 		break;
 	case als_idx: /* Als */
+	case ACS_IDX: /* ambient color sensor */
 		get_common_inputs(&als_input.common_property, report_id);
 		/* For ALS ,V2 Platforms uses C2P_MSG5 register instead of DRAM access method */
 		if (supported_input == V2_STATUS)
diff --git a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_desc.c b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_desc.c
index 0609fea581c9..6f0d332ccf51 100644
--- a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_desc.c
+++ b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_desc.c
@@ -218,7 +218,7 @@ static u8 get_input_rep(u8 current_index, int sensor_idx, int report_id,
 			     OFFSET_SENSOR_DATA_DEFAULT;
 		memcpy_fromio(&als_data, sensoraddr, sizeof(struct sfh_als_data));
 		get_common_inputs(&als_input.common_property, report_id);
-		als_input.illuminance_value = als_data.lux;
+		als_input.illuminance_value = float_to_int(als_data.lux);
 		report_size = sizeof(als_input);
 		memcpy(input_report, &als_input, sizeof(als_input));
 		break;
diff --git a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c
index a1d6e08fab7d..bb8bd7892b67 100644
--- a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c
+++ b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c
@@ -112,6 +112,7 @@ static int amd_sfh1_1_hid_client_init(struct amd_mp2_dev *privdata)
 	cl_data->num_hid_devices = amd_sfh_get_sensor_num(privdata, &cl_data->sensor_idx[0]);
 	if (cl_data->num_hid_devices == 0)
 		return -ENODEV;
+	cl_data->is_any_sensor_enabled = false;
 
 	INIT_DELAYED_WORK(&cl_data->work, amd_sfh_work);
 	INIT_DELAYED_WORK(&cl_data->work_buffer, amd_sfh_work_buffer);
@@ -170,6 +171,7 @@ static int amd_sfh1_1_hid_client_init(struct amd_mp2_dev *privdata)
 		status = (status == 0) ? SENSOR_ENABLED : SENSOR_DISABLED;
 
 		if (status == SENSOR_ENABLED) {
+			cl_data->is_any_sensor_enabled = true;
 			cl_data->sensor_sts[i] = SENSOR_ENABLED;
 			rc = amdtp_hid_probe(i, cl_data);
 			if (rc) {
@@ -186,12 +188,21 @@ static int amd_sfh1_1_hid_client_init(struct amd_mp2_dev *privdata)
 					cl_data->sensor_sts[i]);
 				goto cleanup;
 			}
+		} else {
+			cl_data->sensor_sts[i] = SENSOR_DISABLED;
 		}
 		dev_dbg(dev, "sid 0x%x (%s) status 0x%x\n",
 			cl_data->sensor_idx[i], get_sensor_name(cl_data->sensor_idx[i]),
 			cl_data->sensor_sts[i]);
 	}
 
+	if (!cl_data->is_any_sensor_enabled) {
+		dev_warn(dev, "Failed to discover, sensors not enabled is %d\n",
+			 cl_data->is_any_sensor_enabled);
+		rc = -EOPNOTSUPP;
+		goto cleanup;
+	}
+
 	schedule_delayed_work(&cl_data->work_buffer, msecs_to_jiffies(AMD_SFH_IDLE_LOOP));
 	return 0;
 
diff --git a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c
index c6df959ec725..4f81ef2d4f56 100644
--- a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c
+++ b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c
@@ -16,11 +16,11 @@ static int amd_sfh_wait_response(struct amd_mp2_dev *mp2, u8 sid, u32 cmd_id)
 {
 	struct sfh_cmd_response cmd_resp;
 
-	/* Get response with status within a max of 1600 ms timeout */
+	/* Get response with status within a max of 10000 ms timeout */
 	if (!readl_poll_timeout(mp2->mmio + AMD_P2C_MSG(0), cmd_resp.resp,
 				(cmd_resp.response.response == 0 &&
 				cmd_resp.response.cmd_id == cmd_id && (sid == 0xff ||
-				cmd_resp.response.sensor_id == sid)), 500, 1600000))
+				cmd_resp.response.sensor_id == sid)), 500, 10000000))
 		return cmd_resp.response.response;
 
 	return -1;
@@ -33,6 +33,7 @@ static void amd_start_sensor(struct amd_mp2_dev *privdata, struct amd_mp2_sensor
 	cmd_base.ul = 0;
 	cmd_base.cmd.cmd_id = ENABLE_SENSOR;
 	cmd_base.cmd.intr_disable = 0;
+	cmd_base.cmd.sub_cmd_value = 1;
 	cmd_base.cmd.sensor_id = info.sensor_idx;
 
 	writel(cmd_base.ul, privdata->mmio + AMD_C2P_MSG(0));
@@ -45,6 +46,7 @@ static void amd_stop_sensor(struct amd_mp2_dev *privdata, u16 sensor_idx)
 	cmd_base.ul = 0;
 	cmd_base.cmd.cmd_id = DISABLE_SENSOR;
 	cmd_base.cmd.intr_disable = 0;
+	cmd_base.cmd.sub_cmd_value = 1;
 	cmd_base.cmd.sensor_id = sensor_idx;
 
 	writeq(0x0, privdata->mmio + AMD_C2P_MSG(1));
@@ -56,8 +58,10 @@ static void amd_stop_all_sensor(struct amd_mp2_dev *privdata)
 	struct sfh_cmd_base cmd_base;
 
 	cmd_base.ul = 0;
-	cmd_base.cmd.cmd_id = STOP_ALL_SENSORS;
+	cmd_base.cmd.cmd_id = DISABLE_SENSOR;
 	cmd_base.cmd.intr_disable = 0;
+	/* 0xf indicates all sensors */
+	cmd_base.cmd.sensor_id = 0xf;
 
 	writel(cmd_base.ul, privdata->mmio + AMD_C2P_MSG(0));
 }
diff --git a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.h b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.h
index ae47a369dc05..9d31d5b510eb 100644
--- a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.h
+++ b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.h
@@ -33,9 +33,9 @@ struct sfh_cmd_base {
 		struct {
 			u32 sensor_id		: 4;
 			u32 cmd_id		: 4;
-			u32 sub_cmd_id		: 6;
-			u32 length		: 12;
-			u32 rsvd		: 5;
+			u32 sub_cmd_id		: 8;
+			u32 sub_cmd_value	: 12;
+			u32 rsvd		: 3;
 			u32 intr_disable	: 1;
 		} cmd;
 	};
@@ -133,7 +133,7 @@ struct sfh_mag_data {
 
 struct sfh_als_data {
 	struct sfh_common_data commondata;
-	u16 lux;
+	u32 lux;
 };
 
 struct hpd_status {
diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c
index 8a034a555d4c..d9ef45fcaeab 100644
--- a/drivers/hid/bpf/hid_bpf_dispatch.c
+++ b/drivers/hid/bpf/hid_bpf_dispatch.c
@@ -342,9 +342,6 @@ hid_bpf_release_context(struct hid_bpf_ctx *ctx)
 {
 	struct hid_bpf_ctx_kern *ctx_kern;
 
-	if (!ctx)
-		return;
-
 	ctx_kern = container_of(ctx, struct hid_bpf_ctx_kern, ctx);
 
 	kfree(ctx_kern);
diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c
index 1ccab8aa326c..cc535d2d6e8c 100644
--- a/drivers/hid/hid-apple.c
+++ b/drivers/hid/hid-apple.c
@@ -22,6 +22,7 @@
 #include <linux/slab.h>
 #include <linux/timer.h>
 #include <linux/string.h>
+#include <linux/leds.h>
 
 #include "hid-ids.h"
 
@@ -875,14 +876,16 @@ static const struct hid_device_id apple_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_ANSI),
 		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_ISO),
-		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN },
+		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN |
+			APPLE_ISO_TILDE_QUIRK },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_JIS),
 		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN |
 			APPLE_RDESC_JIS },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_ANSI),
 		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_ISO),
-		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN },
+		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN |
+			APPLE_ISO_TILDE_QUIRK },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_JIS),
 		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN |
 			APPLE_RDESC_JIS },
@@ -901,7 +904,8 @@ static const struct hid_device_id apple_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_ANSI),
 		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_ISO),
-		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN },
+		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN |
+			APPLE_ISO_TILDE_QUIRK },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_JIS),
 		.driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN |
 			APPLE_RDESC_JIS },
@@ -942,31 +946,31 @@ static const struct hid_device_id apple_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_ANSI),
 		.driver_data = APPLE_HAS_FN },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_ISO),
-		.driver_data = APPLE_HAS_FN },
+		.driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_JIS),
 		.driver_data = APPLE_HAS_FN | APPLE_RDESC_JIS },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_ANSI),
 		.driver_data = APPLE_HAS_FN },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_ISO),
-		.driver_data = APPLE_HAS_FN },
+		.driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_JIS),
 		.driver_data = APPLE_HAS_FN | APPLE_RDESC_JIS },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_ANSI),
 		.driver_data = APPLE_HAS_FN },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_ISO),
-		.driver_data = APPLE_HAS_FN },
+		.driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_JIS),
 		.driver_data = APPLE_HAS_FN | APPLE_RDESC_JIS },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_ANSI),
 		.driver_data = APPLE_HAS_FN },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_ISO),
-		.driver_data = APPLE_HAS_FN },
+		.driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_JIS),
 		.driver_data = APPLE_HAS_FN | APPLE_RDESC_JIS },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_ANSI),
 		.driver_data = APPLE_HAS_FN },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_ISO),
-		.driver_data = APPLE_HAS_FN },
+		.driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_JIS),
 		.driver_data = APPLE_HAS_FN | APPLE_RDESC_JIS },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_ANSI),
diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c
index d1094bb1aa42..01a27579ec02 100644
--- a/drivers/hid/hid-asus.c
+++ b/drivers/hid/hid-asus.c
@@ -30,6 +30,7 @@
 #include <linux/input/mt.h>
 #include <linux/usb.h> /* For to_usb_interface for T100 touchpad intf check */
 #include <linux/power_supply.h>
+#include <linux/leds.h>
 
 #include "hid-ids.h"
 
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index c2e9b6d1fd7d..d79e946acdcb 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -415,6 +415,7 @@
 #define I2C_DEVICE_ID_HP_SPECTRE_X360_15	0x2817
 #define I2C_DEVICE_ID_HP_SPECTRE_X360_13_AW0020NG  0x29DF
 #define I2C_DEVICE_ID_ASUS_TP420IA_TOUCHSCREEN 0x2BC8
+#define I2C_DEVICE_ID_ASUS_GV301RA_TOUCHSCREEN 0x2C82
 #define USB_DEVICE_ID_ASUS_UX550VE_TOUCHSCREEN	0x2544
 #define USB_DEVICE_ID_ASUS_UX550_TOUCHSCREEN	0x2706
 #define I2C_DEVICE_ID_SURFACE_GO_TOUCHSCREEN	0x261A
@@ -721,12 +722,19 @@
 #define USB_DEVICE_ID_GENIUS_MANTICORE	0x0153
 #define USB_DEVICE_ID_GENIUS_GX_IMPERATOR	0x4018
 #define USB_DEVICE_ID_KYE_GPEN_560	0x5003
+#define USB_DEVICE_ID_KYE_EASYPEN_M406	0x5005
+#define USB_DEVICE_ID_KYE_EASYPEN_M506	0x500F
 #define USB_DEVICE_ID_KYE_EASYPEN_I405X	0x5010
 #define USB_DEVICE_ID_KYE_MOUSEPEN_I608X	0x5011
-#define USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2	0x501a
+#define USB_DEVICE_ID_KYE_EASYPEN_M406W	0x5012
 #define USB_DEVICE_ID_KYE_EASYPEN_M610X	0x5013
+#define USB_DEVICE_ID_KYE_EASYPEN_340	0x5014
 #define USB_DEVICE_ID_KYE_PENSKETCH_M912	0x5015
+#define USB_DEVICE_ID_KYE_MOUSEPEN_M508WX	0x5016
+#define USB_DEVICE_ID_KYE_MOUSEPEN_M508X	0x5017
 #define USB_DEVICE_ID_KYE_EASYPEN_M406XE	0x5019
+#define USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2	0x501A
+#define USB_DEVICE_ID_KYE_PENSKETCH_T609A	0x501B
 
 #define USB_VENDOR_ID_LABTEC		0x1020
 #define USB_DEVICE_ID_LABTEC_WIRELESS_KEYBOARD	0x0006
diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c
index 5c65a584b3fa..a1d2690a1a0d 100644
--- a/drivers/hid/hid-input.c
+++ b/drivers/hid/hid-input.c
@@ -372,6 +372,8 @@ static const struct hid_device_id hid_battery_quirks[] = {
 	  HID_BATTERY_QUIRK_IGNORE },
 	{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_ASUS_TP420IA_TOUCHSCREEN),
 	  HID_BATTERY_QUIRK_IGNORE },
+	{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_ASUS_GV301RA_TOUCHSCREEN),
+	  HID_BATTERY_QUIRK_IGNORE },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_ASUS_UX550_TOUCHSCREEN),
 	  HID_BATTERY_QUIRK_IGNORE },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_ASUS_UX550VE_TOUCHSCREEN),
@@ -1267,6 +1269,16 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel
 			return;
 		}
 		goto unknown;
+	case HID_UP_CAMERA:
+		switch (usage->hid & HID_USAGE) {
+		case 0x020:
+			map_key_clear(KEY_CAMERA_FOCUS);	break;
+		case 0x021:
+			map_key_clear(KEY_CAMERA);		break;
+		default:
+			goto ignore;
+		}
+		break;
 
 	case HID_UP_HPVENDOR:	/* Reported on a Dutch layout HP5308 */
 		set_bit(EV_REP, input->evbit);
diff --git a/drivers/hid/hid-kye.c b/drivers/hid/hid-kye.c
index da903138eee4..eb9bf2829937 100644
--- a/drivers/hid/hid-kye.c
+++ b/drivers/hid/hid-kye.c
@@ -5,361 +5,216 @@
  *  Copyright (c) 2009 Jiri Kosina
  *  Copyright (c) 2009 Tomas Hanak
  *  Copyright (c) 2012 Nikolai Kondrashov
+ *  Copyright (c) 2023 David Yang
  */
 
-/*
- */
-
+#include <asm-generic/unaligned.h>
 #include <linux/device.h>
 #include <linux/hid.h>
 #include <linux/module.h>
 
 #include "hid-ids.h"
 
-/* Original EasyPen i405X report descriptor size */
-#define EASYPEN_I405X_RDESC_ORIG_SIZE	476
-
-/* Fixed EasyPen i405X report descriptor */
-static __u8 easypen_i405x_rdesc_fixed[] = {
-	0x06, 0x00, 0xFF, /*  Usage Page (FF00h),             */
-	0x09, 0x01,       /*  Usage (01h),                    */
-	0xA1, 0x01,       /*  Collection (Application),       */
-	0x85, 0x05,       /*    Report ID (5),                */
-	0x09, 0x01,       /*    Usage (01h),                  */
-	0x15, 0x80,       /*    Logical Minimum (-128),       */
-	0x25, 0x7F,       /*    Logical Maximum (127),        */
-	0x75, 0x08,       /*    Report Size (8),              */
-	0x95, 0x07,       /*    Report Count (7),             */
-	0xB1, 0x02,       /*    Feature (Variable),           */
-	0xC0,             /*  End Collection,                 */
-	0x05, 0x0D,       /*  Usage Page (Digitizer),         */
-	0x09, 0x01,       /*  Usage (Digitizer),              */
-	0xA1, 0x01,       /*  Collection (Application),       */
-	0x85, 0x10,       /*    Report ID (16),               */
-	0x09, 0x20,       /*    Usage (Stylus),               */
-	0xA0,             /*    Collection (Physical),        */
-	0x14,             /*      Logical Minimum (0),        */
-	0x25, 0x01,       /*      Logical Maximum (1),        */
-	0x75, 0x01,       /*      Report Size (1),            */
-	0x09, 0x42,       /*      Usage (Tip Switch),         */
-	0x09, 0x44,       /*      Usage (Barrel Switch),      */
-	0x09, 0x46,       /*      Usage (Tablet Pick),        */
-	0x95, 0x03,       /*      Report Count (3),           */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0x95, 0x04,       /*      Report Count (4),           */
-	0x81, 0x03,       /*      Input (Constant, Variable), */
-	0x09, 0x32,       /*      Usage (In Range),           */
-	0x95, 0x01,       /*      Report Count (1),           */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0x75, 0x10,       /*      Report Size (16),           */
-	0x95, 0x01,       /*      Report Count (1),           */
-	0xA4,             /*      Push,                       */
-	0x05, 0x01,       /*      Usage Page (Desktop),       */
-	0x55, 0xFD,       /*      Unit Exponent (-3),         */
-	0x65, 0x13,       /*      Unit (Inch),                */
-	0x34,             /*      Physical Minimum (0),       */
-	0x09, 0x30,       /*      Usage (X),                  */
-	0x46, 0x7C, 0x15, /*      Physical Maximum (5500),    */
-	0x26, 0x00, 0x37, /*      Logical Maximum (14080),    */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0x09, 0x31,       /*      Usage (Y),                  */
-	0x46, 0xA0, 0x0F, /*      Physical Maximum (4000),    */
-	0x26, 0x00, 0x28, /*      Logical Maximum (10240),    */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0xB4,             /*      Pop,                        */
-	0x09, 0x30,       /*      Usage (Tip Pressure),       */
-	0x26, 0xFF, 0x03, /*      Logical Maximum (1023),     */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0xC0,             /*    End Collection,               */
-	0xC0              /*  End Collection                  */
+/* Data gathered from Database/VID0458_PID????/Vista/TBoard/default.xml in ioTablet driver
+ *
+ * TODO:
+ *   - Add battery and sleep support for EasyPen M406W and MousePen M508WX
+ *   - Investigate ScrollZ.MiceFMT buttons of EasyPen M406
+ */
+
+static const __u8 easypen_m406_control_rdesc[] = {
+	0x05, 0x0C,        /*  Usage Page (Consumer),    */
+	0x09, 0x01,        /*  Usage (Consumer Control), */
+	0xA1, 0x01,        /*  Collection (Application), */
+	0x85, 0x12,        /*    Report ID (18),         */
+	0x0A, 0x45, 0x02,  /*    Usage (AC Rotate),      */
+	0x09, 0x40,        /*    Usage (Menu),           */
+	0x0A, 0x2F, 0x02,  /*    Usage (AC Zoom),        */
+	0x0A, 0x46, 0x02,  /*    Usage (AC Resize),      */
+	0x0A, 0x1A, 0x02,  /*    Usage (AC Undo),        */
+	0x0A, 0x6A, 0x02,  /*    Usage (AC Delete),      */
+	0x0A, 0x24, 0x02,  /*    Usage (AC Back),        */
+	0x0A, 0x25, 0x02,  /*    Usage (AC Forward),     */
+	0x14,              /*    Logical Minimum (0),    */
+	0x25, 0x01,        /*    Logical Maximum (1),    */
+	0x75, 0x01,        /*    Report Size (1),        */
+	0x95, 0x08,        /*    Report Count (8),       */
+	0x81, 0x02,        /*    Input (Variable),       */
+	0x95, 0x30,        /*    Report Count (48),      */
+	0x81, 0x01,        /*    Input (Constant),       */
+	0xC0               /*  End Collection            */
 };
 
-/* Original MousePen i608X report descriptor size */
-#define MOUSEPEN_I608X_RDESC_ORIG_SIZE	476
-
-/* Fixed MousePen i608X report descriptor */
-static __u8 mousepen_i608x_rdesc_fixed[] = {
-	0x06, 0x00, 0xFF, /*  Usage Page (FF00h),             */
-	0x09, 0x01,       /*  Usage (01h),                    */
-	0xA1, 0x01,       /*  Collection (Application),       */
-	0x85, 0x05,       /*    Report ID (5),                */
-	0x09, 0x01,       /*    Usage (01h),                  */
-	0x15, 0x80,       /*    Logical Minimum (-128),       */
-	0x25, 0x7F,       /*    Logical Maximum (127),        */
-	0x75, 0x08,       /*    Report Size (8),              */
-	0x95, 0x07,       /*    Report Count (7),             */
-	0xB1, 0x02,       /*    Feature (Variable),           */
-	0xC0,             /*  End Collection,                 */
-	0x05, 0x0D,       /*  Usage Page (Digitizer),         */
-	0x09, 0x01,       /*  Usage (Digitizer),              */
-	0xA1, 0x01,       /*  Collection (Application),       */
-	0x85, 0x10,       /*    Report ID (16),               */
-	0x09, 0x20,       /*    Usage (Stylus),               */
-	0xA0,             /*    Collection (Physical),        */
-	0x14,             /*      Logical Minimum (0),        */
-	0x25, 0x01,       /*      Logical Maximum (1),        */
-	0x75, 0x01,       /*      Report Size (1),            */
-	0x09, 0x42,       /*      Usage (Tip Switch),         */
-	0x09, 0x44,       /*      Usage (Barrel Switch),      */
-	0x09, 0x46,       /*      Usage (Tablet Pick),        */
-	0x95, 0x03,       /*      Report Count (3),           */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0x95, 0x04,       /*      Report Count (4),           */
-	0x81, 0x03,       /*      Input (Constant, Variable), */
-	0x09, 0x32,       /*      Usage (In Range),           */
-	0x95, 0x01,       /*      Report Count (1),           */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0x75, 0x10,       /*      Report Size (16),           */
-	0x95, 0x01,       /*      Report Count (1),           */
-	0xA4,             /*      Push,                       */
-	0x05, 0x01,       /*      Usage Page (Desktop),       */
-	0x55, 0xFD,       /*      Unit Exponent (-3),         */
-	0x65, 0x13,       /*      Unit (Inch),                */
-	0x34,             /*      Physical Minimum (0),       */
-	0x09, 0x30,       /*      Usage (X),                  */
-	0x46, 0x40, 0x1F, /*      Physical Maximum (8000),    */
-	0x26, 0x00, 0x50, /*      Logical Maximum (20480),    */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0x09, 0x31,       /*      Usage (Y),                  */
-	0x46, 0x70, 0x17, /*      Physical Maximum (6000),    */
-	0x26, 0x00, 0x3C, /*      Logical Maximum (15360),    */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0xB4,             /*      Pop,                        */
-	0x09, 0x30,       /*      Usage (Tip Pressure),       */
-	0x26, 0xFF, 0x03, /*      Logical Maximum (1023),     */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0xC0,             /*    End Collection,               */
-	0xC0,             /*  End Collection,                 */
-	0x05, 0x01,       /*  Usage Page (Desktop),           */
-	0x09, 0x02,       /*  Usage (Mouse),                  */
-	0xA1, 0x01,       /*  Collection (Application),       */
-	0x85, 0x11,       /*    Report ID (17),               */
-	0x09, 0x01,       /*    Usage (Pointer),              */
-	0xA0,             /*    Collection (Physical),        */
-	0x14,             /*      Logical Minimum (0),        */
-	0xA4,             /*      Push,                       */
-	0x05, 0x09,       /*      Usage Page (Button),        */
-	0x75, 0x01,       /*      Report Size (1),            */
-	0x19, 0x01,       /*      Usage Minimum (01h),        */
-	0x29, 0x03,       /*      Usage Maximum (03h),        */
-	0x25, 0x01,       /*      Logical Maximum (1),        */
-	0x95, 0x03,       /*      Report Count (3),           */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0x95, 0x05,       /*      Report Count (5),           */
-	0x81, 0x01,       /*      Input (Constant),           */
-	0xB4,             /*      Pop,                        */
-	0x95, 0x01,       /*      Report Count (1),           */
-	0xA4,             /*      Push,                       */
-	0x55, 0xFD,       /*      Unit Exponent (-3),         */
-	0x65, 0x13,       /*      Unit (Inch),                */
-	0x34,             /*      Physical Minimum (0),       */
-	0x75, 0x10,       /*      Report Size (16),           */
-	0x09, 0x30,       /*      Usage (X),                  */
-	0x46, 0x40, 0x1F, /*      Physical Maximum (8000),    */
-	0x26, 0x00, 0x50, /*      Logical Maximum (20480),    */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0x09, 0x31,       /*      Usage (Y),                  */
-	0x46, 0x70, 0x17, /*      Physical Maximum (6000),    */
-	0x26, 0x00, 0x3C, /*      Logical Maximum (15360),    */
-	0x81, 0x02,       /*      Input (Variable),           */
-	0xB4,             /*      Pop,                        */
-	0x75, 0x08,       /*      Report Size (8),            */
-	0x09, 0x38,       /*      Usage (Wheel),              */
-	0x15, 0xFF,       /*      Logical Minimum (-1),       */
-	0x25, 0x01,       /*      Logical Maximum (1),        */
-	0x81, 0x06,       /*      Input (Variable, Relative), */
-	0x81, 0x01,       /*      Input (Constant),           */
-	0xC0,             /*    End Collection,               */
-	0xC0              /*  End Collection                  */
+static const __u8 easypen_m506_control_rdesc[] = {
+	0x05, 0x0C,        /*  Usage Page (Consumer),    */
+	0x09, 0x01,        /*  Usage (Consumer Control), */
+	0xA1, 0x01,        /*  Collection (Application), */
+	0x85, 0x12,        /*    Report ID (18),         */
+	0x0A, 0x6A, 0x02,  /*    Usage (AC Delete),      */
+	0x0A, 0x1A, 0x02,  /*    Usage (AC Undo),        */
+	0x0A, 0x2D, 0x02,  /*    Usage (AC Zoom In),     */
+	0x0A, 0x2E, 0x02,  /*    Usage (AC Zoom Out),    */
+	0x14,              /*    Logical Minimum (0),    */
+	0x25, 0x01,        /*    Logical Maximum (1),    */
+	0x75, 0x01,        /*    Report Size (1),        */
+	0x95, 0x04,        /*    Report Count (4),       */
+	0x81, 0x02,        /*    Input (Variable),       */
+	0x95, 0x34,        /*    Report Count (52),      */
+	0x81, 0x01,        /*    Input (Constant),       */
+	0xC0               /*  End Collection            */
 };
 
-/* Original MousePen i608X v2 report descriptor size */
-#define MOUSEPEN_I608X_V2_RDESC_ORIG_SIZE	482
+static const __u8 easypen_m406w_control_rdesc[] = {
+	0x05, 0x0C,        /*  Usage Page (Consumer),    */
+	0x09, 0x01,        /*  Usage (Consumer Control), */
+	0xA1, 0x01,        /*  Collection (Application), */
+	0x85, 0x12,        /*    Report ID (18),         */
+	0x0A, 0x6A, 0x02,  /*    Usage (AC Delete),      */
+	0x0A, 0x1A, 0x02,  /*    Usage (AC Undo),        */
+	0x0A, 0x01, 0x02,  /*    Usage (AC New),         */
+	0x09, 0x40,        /*    Usage (Menu),           */
+	0x14,              /*    Logical Minimum (0),    */
+	0x25, 0x01,        /*    Logical Maximum (1),    */
+	0x75, 0x01,        /*    Report Size (1),        */
+	0x95, 0x04,        /*    Report Count (4),       */
+	0x81, 0x02,        /*    Input (Variable),       */
+	0x95, 0x34,        /*    Report Count (52),      */
+	0x81, 0x01,        /*    Input (Constant),       */
+	0xC0               /*  End Collection            */
+};
 
-/* Fixed MousePen i608X v2 report descriptor */
-static __u8 mousepen_i608x_v2_rdesc_fixed[] = {
-	0x06, 0x00, 0xFF,             /*  Usage Page (FF00h),             */
-	0x09, 0x01,                   /*  Usage (01h),                    */
-	0xA1, 0x01,                   /*  Collection (Application),       */
-	0x85, 0x05,                   /*    Report ID (5),                */
-	0x09, 0x01,                   /*    Usage (01h),                  */
-	0x15, 0x80,                   /*    Logical Minimum (-128),       */
-	0x25, 0x7F,                   /*    Logical Maximum (127),        */
-	0x75, 0x08,                   /*    Report Size (8),              */
-	0x95, 0x07,                   /*    Report Count (7),             */
-	0xB1, 0x02,                   /*    Feature (Variable),           */
-	0xC0,                         /*  End Collection,                 */
-	0x05, 0x0D,                   /*  Usage Page (Digitizer),         */
-	0x09, 0x01,                   /*  Usage (Digitizer),              */
-	0xA1, 0x01,                   /*  Collection (Application),       */
-	0x85, 0x10,                   /*    Report ID (16),               */
-	0x09, 0x20,                   /*    Usage (Stylus),               */
-	0xA0,                         /*    Collection (Physical),        */
-	0x14,                         /*      Logical Minimum (0),        */
-	0x25, 0x01,                   /*      Logical Maximum (1),        */
-	0x75, 0x01,                   /*      Report Size (1),            */
-	0x09, 0x42,                   /*      Usage (Tip Switch),         */
-	0x09, 0x44,                   /*      Usage (Barrel Switch),      */
-	0x09, 0x46,                   /*      Usage (Tablet Pick),        */
-	0x95, 0x03,                   /*      Report Count (3),           */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0x95, 0x04,                   /*      Report Count (4),           */
-	0x81, 0x03,                   /*      Input (Constant, Variable), */
-	0x09, 0x32,                   /*      Usage (In Range),           */
-	0x95, 0x01,                   /*      Report Count (1),           */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0x75, 0x10,                   /*      Report Size (16),           */
-	0x95, 0x01,                   /*      Report Count (1),           */
-	0xA4,                         /*      Push,                       */
-	0x05, 0x01,                   /*      Usage Page (Desktop),       */
-	0x55, 0xFD,                   /*      Unit Exponent (-3),         */
-	0x65, 0x13,                   /*      Unit (Inch),                */
-	0x34,                         /*      Physical Minimum (0),       */
-	0x09, 0x30,                   /*      Usage (X),                  */
-	0x46, 0x40, 0x1F,             /*      Physical Maximum (8000),    */
-	0x27, 0x00, 0xA0, 0x00, 0x00, /*      Logical Maximum (40960),    */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0x09, 0x31,                   /*      Usage (Y),                  */
-	0x46, 0x70, 0x17,             /*      Physical Maximum (6000),    */
-	0x26, 0x00, 0x78,             /*      Logical Maximum (30720),    */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0xB4,                         /*      Pop,                        */
-	0x09, 0x30,                   /*      Usage (Tip Pressure),       */
-	0x26, 0xFF, 0x07,             /*      Logical Maximum (2047),     */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0xC0,                         /*    End Collection,               */
-	0xC0,                         /*  End Collection,                 */
-	0x05, 0x01,                   /*  Usage Page (Desktop),           */
-	0x09, 0x02,                   /*  Usage (Mouse),                  */
-	0xA1, 0x01,                   /*  Collection (Application),       */
-	0x85, 0x11,                   /*    Report ID (17),               */
-	0x09, 0x01,                   /*    Usage (Pointer),              */
-	0xA0,                         /*    Collection (Physical),        */
-	0x14,                         /*      Logical Minimum (0),        */
-	0xA4,                         /*      Push,                       */
-	0x05, 0x09,                   /*      Usage Page (Button),        */
-	0x75, 0x01,                   /*      Report Size (1),            */
-	0x19, 0x01,                   /*      Usage Minimum (01h),        */
-	0x29, 0x03,                   /*      Usage Maximum (03h),        */
-	0x25, 0x01,                   /*      Logical Maximum (1),        */
-	0x95, 0x03,                   /*      Report Count (3),           */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0x95, 0x05,                   /*      Report Count (5),           */
-	0x81, 0x01,                   /*      Input (Constant),           */
-	0xB4,                         /*      Pop,                        */
-	0x95, 0x01,                   /*      Report Count (1),           */
-	0xA4,                         /*      Push,                       */
-	0x55, 0xFD,                   /*      Unit Exponent (-3),         */
-	0x65, 0x13,                   /*      Unit (Inch),                */
-	0x34,                         /*      Physical Minimum (0),       */
-	0x75, 0x10,                   /*      Report Size (16),           */
-	0x09, 0x30,                   /*      Usage (X),                  */
-	0x46, 0x40, 0x1F,             /*      Physical Maximum (8000),    */
-	0x27, 0x00, 0xA0, 0x00, 0x00, /*      Logical Maximum (40960),    */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0x09, 0x31,                   /*      Usage (Y),                  */
-	0x46, 0x70, 0x17,             /*      Physical Maximum (6000),    */
-	0x26, 0x00, 0x78,             /*      Logical Maximum (30720),    */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0xB4,                         /*      Pop,                        */
-	0x75, 0x08,                   /*      Report Size (8),            */
-	0x09, 0x38,                   /*      Usage (Wheel),              */
-	0x15, 0xFF,                   /*      Logical Minimum (-1),       */
-	0x25, 0x01,                   /*      Logical Maximum (1),        */
-	0x81, 0x06,                   /*      Input (Variable, Relative), */
-	0x81, 0x01,                   /*      Input (Constant),           */
-	0xC0,                         /*    End Collection,               */
-	0xC0                          /*  End Collection                  */
+static const __u8 easypen_m610x_control_rdesc[] = {
+	0x05, 0x0C,        /*  Usage Page (Consumer),       */
+	0x09, 0x01,        /*  Usage (Consumer Control),    */
+	0xA1, 0x01,        /*  Collection (Application),    */
+	0x85, 0x12,        /*    Report ID (18),            */
+	0x0A, 0x1A, 0x02,  /*    Usage (AC Undo),           */
+	0x0A, 0x79, 0x02,  /*    Usage (AC Redo Or Repeat), */
+	0x0A, 0x2D, 0x02,  /*    Usage (AC Zoom In),        */
+	0x0A, 0x2E, 0x02,  /*    Usage (AC Zoom Out),       */
+	0x14,              /*    Logical Minimum (0),       */
+	0x25, 0x01,        /*    Logical Maximum (1),       */
+	0x75, 0x01,        /*    Report Size (1),           */
+	0x95, 0x04,        /*    Report Count (4),          */
+	0x81, 0x02,        /*    Input (Variable),          */
+	0x95, 0x34,        /*    Report Count (52),         */
+	0x81, 0x01,        /*    Input (Constant),          */
+	0xC0               /*  End Collection               */
 };
 
-/* Original EasyPen M610X report descriptor size */
-#define EASYPEN_M610X_RDESC_ORIG_SIZE	476
+static const __u8 pensketch_m912_control_rdesc[] = {
+	0x05, 0x0C,        /*  Usage Page (Consumer),        */
+	0x09, 0x01,        /*  Usage (Consumer Control),     */
+	0xA1, 0x01,        /*  Collection (Application),     */
+	0x85, 0x12,        /*    Report ID (18),             */
+	0x14,              /*    Logical Minimum (0),        */
+	0x25, 0x01,        /*    Logical Maximum (1),        */
+	0x75, 0x01,        /*    Report Size (1),            */
+	0x95, 0x08,        /*    Report Count (8),           */
+	0x05, 0x0C,        /*    Usage Page (Consumer),      */
+	0x0A, 0x6A, 0x02,  /*    Usage (AC Delete),          */
+	0x0A, 0x1A, 0x02,  /*    Usage (AC Undo),            */
+	0x0A, 0x01, 0x02,  /*    Usage (AC New),             */
+	0x0A, 0x2F, 0x02,  /*    Usage (AC Zoom),            */
+	0x0A, 0x25, 0x02,  /*    Usage (AC Forward),         */
+	0x0A, 0x24, 0x02,  /*    Usage (AC Back),            */
+	0x0A, 0x2D, 0x02,  /*    Usage (AC Zoom In),         */
+	0x0A, 0x2E, 0x02,  /*    Usage (AC Zoom Out),        */
+	0x81, 0x02,        /*    Input (Variable),           */
+	0x95, 0x30,        /*    Report Count (48),          */
+	0x81, 0x03,        /*    Input (Constant, Variable), */
+	0xC0               /*  End Collection                */
+};
 
-/* Fixed EasyPen M610X report descriptor */
-static __u8 easypen_m610x_rdesc_fixed[] = {
-	0x06, 0x00, 0xFF,             /*  Usage Page (FF00h),             */
-	0x09, 0x01,                   /*  Usage (01h),                    */
-	0xA1, 0x01,                   /*  Collection (Application),       */
-	0x85, 0x05,                   /*    Report ID (5),                */
-	0x09, 0x01,                   /*    Usage (01h),                  */
-	0x15, 0x80,                   /*    Logical Minimum (-128),       */
-	0x25, 0x7F,                   /*    Logical Maximum (127),        */
-	0x75, 0x08,                   /*    Report Size (8),              */
-	0x95, 0x07,                   /*    Report Count (7),             */
-	0xB1, 0x02,                   /*    Feature (Variable),           */
-	0xC0,                         /*  End Collection,                 */
-	0x05, 0x0D,                   /*  Usage Page (Digitizer),         */
-	0x09, 0x01,                   /*  Usage (Digitizer),              */
-	0xA1, 0x01,                   /*  Collection (Application),       */
-	0x85, 0x10,                   /*    Report ID (16),               */
-	0x09, 0x20,                   /*    Usage (Stylus),               */
-	0xA0,                         /*    Collection (Physical),        */
-	0x14,                         /*      Logical Minimum (0),        */
-	0x25, 0x01,                   /*      Logical Maximum (1),        */
-	0x75, 0x01,                   /*      Report Size (1),            */
-	0x09, 0x42,                   /*      Usage (Tip Switch),         */
-	0x09, 0x44,                   /*      Usage (Barrel Switch),      */
-	0x09, 0x46,                   /*      Usage (Tablet Pick),        */
-	0x95, 0x03,                   /*      Report Count (3),           */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0x95, 0x04,                   /*      Report Count (4),           */
-	0x81, 0x03,                   /*      Input (Constant, Variable), */
-	0x09, 0x32,                   /*      Usage (In Range),           */
-	0x95, 0x01,                   /*      Report Count (1),           */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0x75, 0x10,                   /*      Report Size (16),           */
-	0x95, 0x01,                   /*      Report Count (1),           */
-	0xA4,                         /*      Push,                       */
-	0x05, 0x01,                   /*      Usage Page (Desktop),       */
-	0x55, 0xFD,                   /*      Unit Exponent (-3),         */
-	0x65, 0x13,                   /*      Unit (Inch),                */
-	0x34,                         /*      Physical Minimum (0),       */
-	0x09, 0x30,                   /*      Usage (X),                  */
-	0x46, 0x10, 0x27,             /*      Physical Maximum (10000),   */
-	0x27, 0x00, 0xA0, 0x00, 0x00, /*      Logical Maximum (40960),    */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0x09, 0x31,                   /*      Usage (Y),                  */
-	0x46, 0x6A, 0x18,             /*      Physical Maximum (6250),    */
-	0x26, 0x00, 0x64,             /*      Logical Maximum (25600),    */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0xB4,                         /*      Pop,                        */
-	0x09, 0x30,                   /*      Usage (Tip Pressure),       */
-	0x26, 0xFF, 0x03,             /*      Logical Maximum (1023),     */
-	0x81, 0x02,                   /*      Input (Variable),           */
-	0xC0,                         /*    End Collection,               */
-	0xC0,                         /*  End Collection,                 */
-	0x05, 0x0C,                   /*  Usage Page (Consumer),          */
-	0x09, 0x01,                   /*  Usage (Consumer Control),       */
-	0xA1, 0x01,                   /*  Collection (Application),       */
-	0x85, 0x12,                   /*    Report ID (18),               */
-	0x14,                         /*    Logical Minimum (0),          */
-	0x25, 0x01,                   /*    Logical Maximum (1),          */
-	0x75, 0x01,                   /*    Report Size (1),              */
-	0x95, 0x04,                   /*    Report Count (4),             */
-	0x0A, 0x1A, 0x02,             /*    Usage (AC Undo),              */
-	0x0A, 0x79, 0x02,             /*    Usage (AC Redo Or Repeat),    */
-	0x0A, 0x2D, 0x02,             /*    Usage (AC Zoom In),           */
-	0x0A, 0x2E, 0x02,             /*    Usage (AC Zoom Out),          */
-	0x81, 0x02,                   /*    Input (Variable),             */
-	0x95, 0x01,                   /*    Report Count (1),             */
-	0x75, 0x14,                   /*    Report Size (20),             */
-	0x81, 0x03,                   /*    Input (Constant, Variable),   */
-	0x75, 0x20,                   /*    Report Size (32),             */
-	0x81, 0x03,                   /*    Input (Constant, Variable),   */
-	0xC0                          /*  End Collection                  */
+static const __u8 mousepen_m508wx_control_rdesc[] = {
+	0x05, 0x0C,        /*  Usage Page (Consumer),    */
+	0x09, 0x01,        /*  Usage (Consumer Control), */
+	0xA1, 0x01,        /*  Collection (Application), */
+	0x85, 0x12,        /*    Report ID (18),         */
+	0x0A, 0x1A, 0x02,  /*    Usage (AC Undo),        */
+	0x0A, 0x6A, 0x02,  /*    Usage (AC Delete),      */
+	0x0A, 0x2D, 0x02,  /*    Usage (AC Zoom In),     */
+	0x0A, 0x2E, 0x02,  /*    Usage (AC Zoom Out),    */
+	0x14,              /*    Logical Minimum (0),    */
+	0x25, 0x01,        /*    Logical Maximum (1),    */
+	0x75, 0x01,        /*    Report Size (1),        */
+	0x95, 0x04,        /*    Report Count (4),       */
+	0x81, 0x02,        /*    Input (Variable),       */
+	0x95, 0x34,        /*    Report Count (52),      */
+	0x81, 0x01,        /*    Input (Constant),       */
+	0xC0               /*  End Collection            */
 };
 
+static const __u8 mousepen_m508x_control_rdesc[] = {
+	0x05, 0x0C,        /*  Usage Page (Consumer),        */
+	0x09, 0x01,        /*  Usage (Consumer Control),     */
+	0xA1, 0x01,        /*  Collection (Application),     */
+	0x85, 0x12,        /*    Report ID (18),             */
+	0x0A, 0x01, 0x02,  /*    Usage (AC New),             */
+	0x09, 0x40,        /*    Usage (Menu),               */
+	0x0A, 0x6A, 0x02,  /*    Usage (AC Delete),          */
+	0x0A, 0x1A, 0x02,  /*    Usage (AC Undo),            */
+	0x14,              /*    Logical Minimum (0),        */
+	0x25, 0x01,        /*    Logical Maximum (1),        */
+	0x75, 0x01,        /*    Report Size (1),            */
+	0x95, 0x04,        /*    Report Count (4),           */
+	0x81, 0x02,        /*    Input (Variable),           */
+	0x81, 0x01,        /*    Input (Constant),           */
+	0x15, 0xFF,        /*    Logical Minimum (-1),       */
+	0x95, 0x10,        /*    Report Count (16),          */
+	0x81, 0x01,        /*    Input (Constant),           */
+	0x0A, 0x35, 0x02,  /*    Usage (AC Scroll),          */
+	0x0A, 0x2F, 0x02,  /*    Usage (AC Zoom),            */
+	0x0A, 0x38, 0x02,  /*    Usage (AC Pan),             */
+	0x75, 0x08,        /*    Report Size (8),            */
+	0x95, 0x03,        /*    Report Count (3),           */
+	0x81, 0x06,        /*    Input (Variable, Relative), */
+	0x95, 0x01,        /*    Report Count (1),           */
+	0x81, 0x01,        /*    Input (Constant),           */
+	0xC0               /*  End Collection                */
+};
+
+static const __u8 easypen_m406xe_control_rdesc[] = {
+	0x05, 0x0C,        /*  Usage Page (Consumer),          */
+	0x09, 0x01,        /*  Usage (Consumer Control),       */
+	0xA1, 0x01,        /*  Collection (Application),       */
+	0x85, 0x12,        /*      Report ID (18),             */
+	0x14,              /*      Logical Minimum (0),        */
+	0x25, 0x01,        /*      Logical Maximum (1),        */
+	0x75, 0x01,        /*      Report Size (1),            */
+	0x95, 0x04,        /*      Report Count (4),           */
+	0x0A, 0x79, 0x02,  /*      Usage (AC Redo Or Repeat),  */
+	0x0A, 0x1A, 0x02,  /*      Usage (AC Undo),            */
+	0x0A, 0x2D, 0x02,  /*      Usage (AC Zoom In),         */
+	0x0A, 0x2E, 0x02,  /*      Usage (AC Zoom Out),        */
+	0x81, 0x02,        /*      Input (Variable),           */
+	0x95, 0x34,        /*      Report Count (52),          */
+	0x81, 0x03,        /*      Input (Constant, Variable), */
+	0xC0               /*  End Collection                  */
+};
 
-/* Original PenSketch M912 report descriptor size */
-#define PENSKETCH_M912_RDESC_ORIG_SIZE	482
+static const __u8 pensketch_t609a_control_rdesc[] = {
+	0x05, 0x0C,        /*  Usage Page (Consumer),    */
+	0x09, 0x01,        /*  Usage (Consumer Control), */
+	0xA1, 0x01,        /*  Collection (Application), */
+	0x85, 0x12,        /*    Report ID (18),         */
+	0x0A, 0x6A, 0x02,  /*    Usage (AC Delete),      */
+	0x14,              /*    Logical Minimum (0),    */
+	0x25, 0x01,        /*    Logical Maximum (1),    */
+	0x75, 0x01,        /*    Report Size (1),        */
+	0x95, 0x08,        /*    Report Count (8),       */
+	0x81, 0x02,        /*    Input (Variable),       */
+	0x95, 0x37,        /*    Report Count (55),      */
+	0x81, 0x01,        /*    Input (Constant),       */
+	0xC0               /*  End Collection            */
+};
 
-/* Fixed PenSketch M912 report descriptor */
-static __u8 pensketch_m912_rdesc_fixed[] = {
-	0x05, 0x01,                   /*  Usage Page (Desktop),           */
-	0x08,                         /*  Usage (00h),                    */
+/* Fix indexes in kye_tablet_fixup if you change this */
+static const __u8 kye_tablet_rdesc[] = {
+	0x06, 0x00, 0xFF,             /*  Usage Page (FF00h),             */
+	0x09, 0x01,                   /*  Usage (01h),                    */
 	0xA1, 0x01,                   /*  Collection (Application),       */
 	0x85, 0x05,                   /*    Report ID (5),                */
-	0x06, 0x00, 0xFF,             /*    Usage Page (FF00h),           */
 	0x09, 0x01,                   /*    Usage (01h),                  */
 	0x15, 0x81,                   /*    Logical Minimum (-127),       */
 	0x25, 0x7F,                   /*    Logical Maximum (127),        */
@@ -382,30 +237,29 @@ static __u8 pensketch_m912_rdesc_fixed[] = {
 	0x95, 0x03,                   /*      Report Count (3),           */
 	0x81, 0x02,                   /*      Input (Variable),           */
 	0x95, 0x04,                   /*      Report Count (4),           */
-	0x81, 0x03,                   /*      Input (Constant, Variable), */
+	0x81, 0x01,                   /*      Input (Constant),           */
 	0x09, 0x32,                   /*      Usage (In Range),           */
 	0x95, 0x01,                   /*      Report Count (1),           */
 	0x81, 0x02,                   /*      Input (Variable),           */
 	0x75, 0x10,                   /*      Report Size (16),           */
-	0x95, 0x01,                   /*      Report Count (1),           */
 	0xA4,                         /*      Push,                       */
 	0x05, 0x01,                   /*      Usage Page (Desktop),       */
-	0x55, 0xFD,                   /*      Unit Exponent (-3),         */
-	0x65, 0x13,                   /*      Unit (Inch),                */
-	0x14,                         /*      Logical Minimum (0),        */
-	0x34,                         /*      Physical Minimum (0),       */
 	0x09, 0x30,                   /*      Usage (X),                  */
-	0x27, 0x00, 0xF0, 0x00, 0x00, /*      Logical Maximum (61440),    */
-	0x46, 0xE0, 0x2E,             /*      Physical Maximum (12000),   */
+	0x27, 0xFF, 0x7F, 0x00, 0x00, /*      Logical Maximum (32767),    */
+	0x34,                         /*      Physical Minimum (0),       */
+	0x47, 0x00, 0x00, 0x00, 0x00, /*      Physical Maximum (0),       */
+	0x65, 0x11,                   /*      Unit (Centimeter),          */
+	0x55, 0x00,                   /*      Unit Exponent (0),          */
+	0x75, 0x10,                   /*      Report Size (16),           */
 	0x81, 0x02,                   /*      Input (Variable),           */
 	0x09, 0x31,                   /*      Usage (Y),                  */
-	0x27, 0x00, 0xB4, 0x00, 0x00, /*      Logical Maximum (46080),    */
-	0x46, 0x28, 0x23,             /*      Physical Maximum (9000),    */
+	0x27, 0xFF, 0x7F, 0x00, 0x00, /*      Logical Maximum (32767),    */
+	0x47, 0x00, 0x00, 0x00, 0x00, /*      Physical Maximum (0),       */
 	0x81, 0x02,                   /*      Input (Variable),           */
 	0xB4,                         /*      Pop,                        */
+	0x05, 0x0D,                   /*      Usage Page (Digitizer),     */
 	0x09, 0x30,                   /*      Usage (Tip Pressure),       */
-	0x14,                         /*      Logical Minimum (0),        */
-	0x26, 0xFF, 0x07,             /*      Logical Maximum (2047),     */
+	0x27, 0xFF, 0x07, 0x00, 0x00, /*      Logical Maximum (2047),     */
 	0x81, 0x02,                   /*      Input (Variable),           */
 	0xC0,                         /*    End Collection,               */
 	0xC0,                         /*  End Collection,                 */
@@ -416,146 +270,98 @@ static __u8 pensketch_m912_rdesc_fixed[] = {
 	0x09, 0x21,                   /*    Usage (Puck),                 */
 	0xA0,                         /*    Collection (Physical),        */
 	0x05, 0x09,                   /*      Usage Page (Button),        */
-	0x75, 0x01,                   /*      Report Size (1),            */
 	0x19, 0x01,                   /*      Usage Minimum (01h),        */
 	0x29, 0x03,                   /*      Usage Maximum (03h),        */
 	0x14,                         /*      Logical Minimum (0),        */
 	0x25, 0x01,                   /*      Logical Maximum (1),        */
+	0x75, 0x01,                   /*      Report Size (1),            */
 	0x95, 0x03,                   /*      Report Count (3),           */
 	0x81, 0x02,                   /*      Input (Variable),           */
 	0x95, 0x04,                   /*      Report Count (4),           */
 	0x81, 0x01,                   /*      Input (Constant),           */
+	0x05, 0x0D,                   /*      Usage Page (Digitizer),     */
+	0x09, 0x32,                   /*      Usage (In Range),           */
 	0x95, 0x01,                   /*      Report Count (1),           */
-	0x0B, 0x32, 0x00, 0x0D, 0x00, /*      Usage (Digitizer In Range), */
-	0x14,                         /*      Logical Minimum (0),        */
-	0x25, 0x01,                   /*      Logical Maximum (1),        */
 	0x81, 0x02,                   /*      Input (Variable),           */
-	0xA4,                         /*      Push,                       */
 	0x05, 0x01,                   /*      Usage Page (Desktop),       */
-	0x75, 0x10,                   /*      Report Size (16),           */
-	0x95, 0x01,                   /*      Report Count (1),           */
-	0x55, 0xFD,                   /*      Unit Exponent (-3),         */
-	0x65, 0x13,                   /*      Unit (Inch),                */
-	0x14,                         /*      Logical Minimum (0),        */
-	0x34,                         /*      Physical Minimum (0),       */
+	0xA4,                         /*      Push,                       */
 	0x09, 0x30,                   /*      Usage (X),                  */
-	0x27, 0x00, 0xF0, 0x00, 0x00, /*      Logical Maximum (61440),    */
-	0x46, 0xE0, 0x2E,             /*      Physical Maximum (12000),   */
+	0x27, 0xFF, 0x7F, 0x00, 0x00, /*      Logical Maximum (32767),    */
+	0x34,                         /*      Physical Minimum (0),       */
+	0x47, 0x00, 0x00, 0x00, 0x00, /*      Physical Maximum (0),       */
+	0x65, 0x11,                   /*      Unit (Centimeter),          */
+	0x55, 0x00,                   /*      Unit Exponent (0),          */
+	0x75, 0x10,                   /*      Report Size (16),           */
 	0x81, 0x02,                   /*      Input (Variable),           */
 	0x09, 0x31,                   /*      Usage (Y),                  */
-	0x27, 0x00, 0xB4, 0x00, 0x00, /*      Logical Maximum (46080),    */
-	0x46, 0x28, 0x23,             /*      Physical Maximum (9000),    */
+	0x27, 0xFF, 0x7F, 0x00, 0x00, /*      Logical Maximum (32767),    */
+	0x47, 0x00, 0x00, 0x00, 0x00, /*      Physical Maximum (0),       */
 	0x81, 0x02,                   /*      Input (Variable),           */
+	0xB4,                         /*      Pop,                        */
 	0x09, 0x38,                   /*      Usage (Wheel),              */
+	0x15, 0xFF,                   /*      Logical Minimum (-1),       */
 	0x75, 0x08,                   /*      Report Size (8),            */
 	0x95, 0x01,                   /*      Report Count (1),           */
-	0x15, 0xFF,                   /*      Logical Minimum (-1),       */
-	0x25, 0x01,                   /*      Logical Maximum (1),        */
-	0x34,                         /*      Physical Minimum (0),       */
-	0x44,                         /*      Physical Maximum (0),       */
 	0x81, 0x06,                   /*      Input (Variable, Relative), */
-	0xB4,                         /*      Pop,                        */
+	0x81, 0x01,                   /*      Input (Constant),           */
 	0xC0,                         /*    End Collection,               */
-	0xC0,                         /*  End Collection,                 */
-	0x05, 0x0C,                   /*  Usage Page (Consumer),          */
-	0x09, 0x01,                   /*  Usage (Consumer Control),       */
-	0xA1, 0x01,                   /*  Collection (Application),       */
-	0x85, 0x12,                   /*    Report ID (18),               */
-	0x14,                         /*    Logical Minimum (0),          */
-	0x25, 0x01,                   /*    Logical Maximum (1),          */
-	0x75, 0x01,                   /*    Report Size (1),              */
-	0x95, 0x08,                   /*    Report Count (8),             */
-	0x05, 0x0C,                   /*    Usage Page (Consumer),        */
-	0x0A, 0x6A, 0x02,             /*    Usage (AC Delete),            */
-	0x0A, 0x1A, 0x02,             /*    Usage (AC Undo),              */
-	0x0A, 0x01, 0x02,             /*    Usage (AC New),               */
-	0x0A, 0x2F, 0x02,             /*    Usage (AC Zoom),              */
-	0x0A, 0x25, 0x02,             /*    Usage (AC Forward),           */
-	0x0A, 0x24, 0x02,             /*    Usage (AC Back),              */
-	0x0A, 0x2D, 0x02,             /*    Usage (AC Zoom In),           */
-	0x0A, 0x2E, 0x02,             /*    Usage (AC Zoom Out),          */
-	0x81, 0x02,                   /*    Input (Variable),             */
-	0x95, 0x30,                   /*    Report Count (48),            */
-	0x81, 0x03,                   /*    Input (Constant, Variable),   */
 	0xC0                          /*  End Collection                  */
 };
 
-/* Original EasyPen M406XE report descriptor size */
-#define EASYPEN_M406XE_RDESC_ORIG_SIZE	476
-
-/* Fixed EasyPen M406XE  report descriptor */
-static __u8 easypen_m406xe_rdesc_fixed[] = {
-	0x05, 0x01,         /*  Usage Page (Desktop),               */
-	0x09, 0x01,         /*  Usage (01h),                        */
-	0xA1, 0x01,         /*  Collection (Application),           */
-	0x85, 0x05,         /*      Report ID (5),                  */
-	0x09, 0x01,         /*      Usage (01h),                    */
-	0x15, 0x80,         /*      Logical Minimum (-128),         */
-	0x25, 0x7F,         /*      Logical Maximum (127),          */
-	0x75, 0x08,         /*      Report Size (8),                */
-	0x95, 0x07,         /*      Report Count (7),               */
-	0xB1, 0x02,         /*      Feature (Variable),             */
-	0xC0,               /*  End Collection,                     */
-	0x05, 0x0D,         /*  Usage Page (Digitizer),             */
-	0x09, 0x01,         /*  Usage (Digitizer),                  */
-	0xA1, 0x01,         /*  Collection (Application),           */
-	0x85, 0x10,         /*      Report ID (16),                 */
-	0x09, 0x20,         /*      Usage (Stylus),                 */
-	0xA0,               /*      Collection (Physical),          */
-	0x14,               /*          Logical Minimum (0),        */
-	0x25, 0x01,         /*          Logical Maximum (1),        */
-	0x75, 0x01,         /*          Report Size (1),            */
-	0x09, 0x42,         /*          Usage (Tip Switch),         */
-	0x09, 0x44,         /*          Usage (Barrel Switch),      */
-	0x09, 0x46,         /*          Usage (Tablet Pick),        */
-	0x95, 0x03,         /*          Report Count (3),           */
-	0x81, 0x02,         /*          Input (Variable),           */
-	0x95, 0x04,         /*          Report Count (4),           */
-	0x81, 0x03,         /*          Input (Constant, Variable), */
-	0x09, 0x32,         /*          Usage (In Range),           */
-	0x95, 0x01,         /*          Report Count (1),           */
-	0x81, 0x02,         /*          Input (Variable),           */
-	0x75, 0x10,         /*          Report Size (16),           */
-	0x95, 0x01,         /*          Report Count (1),           */
-	0xA4,               /*          Push,                       */
-	0x05, 0x01,         /*          Usage Page (Desktop),       */
-	0x55, 0xFD,         /*          Unit Exponent (-3),         */
-	0x65, 0x13,         /*          Unit (Inch),                */
-	0x34,               /*          Physical Minimum (0),       */
-	0x09, 0x30,         /*          Usage (X),                  */
-	0x46, 0x70, 0x17,   /*          Physical Maximum (6000),    */
-	0x26, 0x00, 0x3C,   /*          Logical Maximum (15360),    */
-	0x81, 0x02,         /*          Input (Variable),           */
-	0x09, 0x31,         /*          Usage (Y),                  */
-	0x46, 0xA0, 0x0F,   /*          Physical Maximum (4000),    */
-	0x26, 0x00, 0x28,   /*          Logical Maximum (10240),    */
-	0x81, 0x02,         /*          Input (Variable),           */
-	0xB4,               /*          Pop,                        */
-	0x09, 0x30,         /*          Usage (Tip Pressure),       */
-	0x26, 0xFF, 0x03,   /*          Logical Maximum (1023),     */
-	0x81, 0x02,         /*          Input (Variable),           */
-	0xC0,               /*      End Collection,                 */
-	0xC0,               /*  End Collection                      */
-	0x05, 0x0C,         /*  Usage Page (Consumer),              */
-	0x09, 0x01,         /*  Usage (Consumer Control),           */
-	0xA1, 0x01,         /*  Collection (Application),           */
-	0x85, 0x12,         /*      Report ID (18),                 */
-	0x14,               /*      Logical Minimum (0),            */
-	0x25, 0x01,         /*      Logical Maximum (1),            */
-	0x75, 0x01,         /*      Report Size (1),                */
-	0x95, 0x04,         /*      Report Count (4),               */
-	0x0A, 0x79, 0x02,   /*      Usage (AC Redo Or Repeat),      */
-	0x0A, 0x1A, 0x02,   /*      Usage (AC Undo),                */
-	0x0A, 0x2D, 0x02,   /*      Usage (AC Zoom In),             */
-	0x0A, 0x2E, 0x02,   /*      Usage (AC Zoom Out),            */
-	0x81, 0x02,         /*      Input (Variable),               */
-	0x95, 0x34,         /*      Report Count (52),              */
-	0x81, 0x03,         /*      Input (Constant, Variable),     */
-	0xC0                /*  End Collection                      */
+static const struct kye_tablet_info {
+	__u32 product;
+	__s32 x_logical_maximum;
+	__s32 y_logical_maximum;
+	__s32 pressure_logical_maximum;
+	__s32 x_physical_maximum;
+	__s32 y_physical_maximum;
+	__s8 unit_exponent;
+	__s8 unit;
+	bool has_punk;
+	unsigned int control_rsize;
+	const __u8 *control_rdesc;
+} kye_tablets_info[] = {
+	{USB_DEVICE_ID_KYE_EASYPEN_M406,  /* 0x5005 */
+		15360, 10240, 1023,    6,   4,  0, 0x13, false,
+		sizeof(easypen_m406_control_rdesc), easypen_m406_control_rdesc},
+	{USB_DEVICE_ID_KYE_EASYPEN_M506,  /* 0x500F */
+		24576, 20480, 1023,    6,   5,  0, 0x13, false,
+		sizeof(easypen_m506_control_rdesc), easypen_m506_control_rdesc},
+	{USB_DEVICE_ID_KYE_EASYPEN_I405X,  /* 0x5010 */
+		14080, 10240, 1023,   55,  40, -1, 0x13, false},
+	{USB_DEVICE_ID_KYE_MOUSEPEN_I608X,  /* 0x5011 */
+		20480, 15360, 2047,    8,   6,  0, 0x13,  true},
+	{USB_DEVICE_ID_KYE_EASYPEN_M406W,  /* 0x5012 */
+		15360, 10240, 1023,    6,   4,  0, 0x13, false,
+		sizeof(easypen_m406w_control_rdesc), easypen_m406w_control_rdesc},
+	{USB_DEVICE_ID_KYE_EASYPEN_M610X,  /* 0x5013 */
+		40960, 25600, 1023, 1000, 625, -2, 0x13, false,
+		sizeof(easypen_m610x_control_rdesc), easypen_m610x_control_rdesc},
+	{USB_DEVICE_ID_KYE_EASYPEN_340,  /* 0x5014 */
+		10240,  7680, 1023,    4,   3,  0, 0x13, false},
+	{USB_DEVICE_ID_KYE_PENSKETCH_M912,  /* 0x5015 */
+		61440, 46080, 2047,   12,   9,  0, 0x13,  true,
+		sizeof(pensketch_m912_control_rdesc), pensketch_m912_control_rdesc},
+	{USB_DEVICE_ID_KYE_MOUSEPEN_M508WX,  /* 0x5016 */
+		40960, 25600, 2047,    8,   5,  0, 0x13,  true,
+		sizeof(mousepen_m508wx_control_rdesc), mousepen_m508wx_control_rdesc},
+	{USB_DEVICE_ID_KYE_MOUSEPEN_M508X,  /* 0x5017 */
+		40960, 25600, 2047,    8,   5,  0, 0x13,  true,
+		sizeof(mousepen_m508x_control_rdesc), mousepen_m508x_control_rdesc},
+	{USB_DEVICE_ID_KYE_EASYPEN_M406XE,  /* 0x5019 */
+		15360, 10240, 1023,    6,   4,  0, 0x13, false,
+		sizeof(easypen_m406xe_control_rdesc), easypen_m406xe_control_rdesc},
+	{USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2,  /* 0x501A */
+		40960, 30720, 2047,    8,   6,  0, 0x13,  true},
+	{USB_DEVICE_ID_KYE_PENSKETCH_T609A,  /* 0x501B */
+		43520, 28160, 1023,   85,  55, -1, 0x13, false,
+		sizeof(pensketch_t609a_control_rdesc), pensketch_t609a_control_rdesc},
+	{}
 };
 
 static __u8 *kye_consumer_control_fixup(struct hid_device *hdev, __u8 *rdesc,
-		unsigned int *rsize, int offset, const char *device_name) {
+		unsigned int *rsize, int offset, const char *device_name)
+{
 	/*
 	 * the fixup that need to be done:
 	 *   - change Usage Maximum in the Consumer Control
@@ -574,6 +380,79 @@ static __u8 *kye_consumer_control_fixup(struct hid_device *hdev, __u8 *rdesc,
 	return rdesc;
 }
 
+/*
+ * Fix tablet descriptor of so-called "DataFormat 2".
+ *
+ * Though we may achieve a usable descriptor from original vendor-defined one,
+ * some problems exist:
+ *  - Their Logical Maximum never exceed 32767 (7F FF), though device do report
+ *    values greater than that;
+ *  - Physical Maximums are arbitrarily filled (always equal to Logical
+ *    Maximum);
+ *  - Detail for control buttons are not provided (a vendor-defined Usage Page
+ *    with fixed content).
+ *
+ * Thus we use a pre-defined parameter table rather than digging it from
+ * original descriptor.
+ *
+ * We may as well write a fallback routine for unrecognized kye tablet, but it's
+ * clear kye are unlikely to produce new models in the foreseeable future, so we
+ * simply enumerate all possible models.
+ */
+static __u8 *kye_tablet_fixup(struct hid_device *hdev, __u8 *rdesc, unsigned int *rsize)
+{
+	const struct kye_tablet_info *info;
+	unsigned int newsize;
+
+	if (*rsize < sizeof(kye_tablet_rdesc)) {
+		hid_warn(hdev,
+			 "tablet report size too small, or kye_tablet_rdesc unexpectedly large\n");
+		return rdesc;
+	}
+
+	for (info = kye_tablets_info; info->product; info++) {
+		if (hdev->product == info->product)
+			break;
+	}
+
+	if (!info->product) {
+		hid_err(hdev, "tablet unknown, someone forget to add kye_tablet_info entry?\n");
+		return rdesc;
+	}
+
+	newsize = info->has_punk ? sizeof(kye_tablet_rdesc) : 112;
+	memcpy(rdesc, kye_tablet_rdesc, newsize);
+
+	put_unaligned_le32(info->x_logical_maximum, rdesc + 66);
+	put_unaligned_le32(info->x_physical_maximum, rdesc + 72);
+	rdesc[77] = info->unit;
+	rdesc[79] = info->unit_exponent;
+	put_unaligned_le32(info->y_logical_maximum, rdesc + 87);
+	put_unaligned_le32(info->y_physical_maximum, rdesc + 92);
+	put_unaligned_le32(info->pressure_logical_maximum, rdesc + 104);
+
+	if (info->has_punk) {
+		put_unaligned_le32(info->x_logical_maximum, rdesc + 156);
+		put_unaligned_le32(info->x_physical_maximum, rdesc + 162);
+		rdesc[167] = info->unit;
+		rdesc[169] = info->unit_exponent;
+		put_unaligned_le32(info->y_logical_maximum, rdesc + 177);
+		put_unaligned_le32(info->y_physical_maximum, rdesc + 182);
+	}
+
+	if (info->control_rsize) {
+		if (newsize + info->control_rsize > *rsize)
+			hid_err(hdev, "control rdesc unexpectedly large");
+		else {
+			memcpy(rdesc + newsize, info->control_rdesc, info->control_rsize);
+			newsize += info->control_rsize;
+		}
+	}
+
+	*rsize = newsize;
+	return rdesc;
+}
+
 static __u8 *kye_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		unsigned int *rsize)
 {
@@ -602,66 +481,37 @@ static __u8 *kye_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 			rdesc[74] = 0x08;
 		}
 		break;
-	case USB_DEVICE_ID_KYE_EASYPEN_I405X:
-		if (*rsize == EASYPEN_I405X_RDESC_ORIG_SIZE) {
-			rdesc = easypen_i405x_rdesc_fixed;
-			*rsize = sizeof(easypen_i405x_rdesc_fixed);
-		}
-		break;
-	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X:
-		if (*rsize == MOUSEPEN_I608X_RDESC_ORIG_SIZE) {
-			rdesc = mousepen_i608x_rdesc_fixed;
-			*rsize = sizeof(mousepen_i608x_rdesc_fixed);
-		}
-		break;
-	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2:
-		if (*rsize == MOUSEPEN_I608X_V2_RDESC_ORIG_SIZE) {
-			rdesc = mousepen_i608x_v2_rdesc_fixed;
-			*rsize = sizeof(mousepen_i608x_v2_rdesc_fixed);
-		}
-		break;
-	case USB_DEVICE_ID_KYE_EASYPEN_M610X:
-		if (*rsize == EASYPEN_M610X_RDESC_ORIG_SIZE) {
-			rdesc = easypen_m610x_rdesc_fixed;
-			*rsize = sizeof(easypen_m610x_rdesc_fixed);
-		}
-		break;
-	case USB_DEVICE_ID_KYE_EASYPEN_M406XE:
-		if (*rsize == EASYPEN_M406XE_RDESC_ORIG_SIZE) {
-			rdesc = easypen_m406xe_rdesc_fixed;
-			*rsize = sizeof(easypen_m406xe_rdesc_fixed);
-		}
-		break;
-	case USB_DEVICE_ID_KYE_PENSKETCH_M912:
-		if (*rsize == PENSKETCH_M912_RDESC_ORIG_SIZE) {
-			rdesc = pensketch_m912_rdesc_fixed;
-			*rsize = sizeof(pensketch_m912_rdesc_fixed);
-		}
-		break;
 	case USB_DEVICE_ID_GENIUS_GILA_GAMING_MOUSE:
 		rdesc = kye_consumer_control_fixup(hdev, rdesc, rsize, 104,
 					"Genius Gila Gaming Mouse");
 		break;
+	case USB_DEVICE_ID_GENIUS_MANTICORE:
+		rdesc = kye_consumer_control_fixup(hdev, rdesc, rsize, 104,
+					"Genius Manticore Keyboard");
+		break;
 	case USB_DEVICE_ID_GENIUS_GX_IMPERATOR:
 		rdesc = kye_consumer_control_fixup(hdev, rdesc, rsize, 83,
 					"Genius Gx Imperator Keyboard");
 		break;
-	case USB_DEVICE_ID_GENIUS_MANTICORE:
-		rdesc = kye_consumer_control_fixup(hdev, rdesc, rsize, 104,
-					"Genius Manticore Keyboard");
+	case USB_DEVICE_ID_KYE_EASYPEN_M406:
+	case USB_DEVICE_ID_KYE_EASYPEN_M506:
+	case USB_DEVICE_ID_KYE_EASYPEN_I405X:
+	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X:
+	case USB_DEVICE_ID_KYE_EASYPEN_M406W:
+	case USB_DEVICE_ID_KYE_EASYPEN_M610X:
+	case USB_DEVICE_ID_KYE_EASYPEN_340:
+	case USB_DEVICE_ID_KYE_PENSKETCH_M912:
+	case USB_DEVICE_ID_KYE_MOUSEPEN_M508WX:
+	case USB_DEVICE_ID_KYE_MOUSEPEN_M508X:
+	case USB_DEVICE_ID_KYE_EASYPEN_M406XE:
+	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2:
+	case USB_DEVICE_ID_KYE_PENSKETCH_T609A:
+		rdesc = kye_tablet_fixup(hdev, rdesc, rsize);
 		break;
 	}
 	return rdesc;
 }
 
-/**
- * kye_tablet_enable() - Enable fully-functional tablet mode by setting a special feature report.
- *
- * @hdev:	HID device
- *
- * The specific report ID and data were discovered by sniffing the
- * Windows driver traffic.
- */
 static int kye_tablet_enable(struct hid_device *hdev)
 {
 	struct list_head *list;
@@ -688,6 +538,15 @@ static int kye_tablet_enable(struct hid_device *hdev)
 
 	value = report->field[0]->value;
 
+	/*
+	 * The code is for DataFormat 2 of config xml. They have no obvious
+	 * meaning (at least not configurable in Windows driver) except enabling
+	 * fully-functional tablet mode (absolute positioning). Otherwise, the
+	 * tablet acts like a relative mouse.
+	 *
+	 * Though there're magic codes for DataFormat 3 and 4, no devices use
+	 * these DataFormats.
+	 */
 	value[0] = 0x12;
 	value[1] = 0x10;
 	value[2] = 0x11;
@@ -717,26 +576,33 @@ static int kye_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	}
 
 	switch (id->product) {
+	case USB_DEVICE_ID_GENIUS_MANTICORE:
+		/*
+		 * The manticore keyboard needs to have all the interfaces
+		 * opened at least once to be fully functional.
+		 */
+		if (hid_hw_open(hdev))
+			hid_hw_close(hdev);
+		break;
+	case USB_DEVICE_ID_KYE_EASYPEN_M406:
+	case USB_DEVICE_ID_KYE_EASYPEN_M506:
 	case USB_DEVICE_ID_KYE_EASYPEN_I405X:
 	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X:
-	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2:
+	case USB_DEVICE_ID_KYE_EASYPEN_M406W:
 	case USB_DEVICE_ID_KYE_EASYPEN_M610X:
-	case USB_DEVICE_ID_KYE_EASYPEN_M406XE:
+	case USB_DEVICE_ID_KYE_EASYPEN_340:
 	case USB_DEVICE_ID_KYE_PENSKETCH_M912:
+	case USB_DEVICE_ID_KYE_MOUSEPEN_M508WX:
+	case USB_DEVICE_ID_KYE_MOUSEPEN_M508X:
+	case USB_DEVICE_ID_KYE_EASYPEN_M406XE:
+	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2:
+	case USB_DEVICE_ID_KYE_PENSKETCH_T609A:
 		ret = kye_tablet_enable(hdev);
 		if (ret) {
 			hid_err(hdev, "tablet enabling failed\n");
 			goto enabling_err;
 		}
 		break;
-	case USB_DEVICE_ID_GENIUS_MANTICORE:
-		/*
-		 * The manticore keyboard needs to have all the interfaces
-		 * opened at least once to be fully functional.
-		 */
-		if (hid_hw_open(hdev))
-			hid_hw_close(hdev);
-		break;
 	}
 
 	return 0;
@@ -749,23 +615,37 @@ err:
 static const struct hid_device_id kye_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_ERGO_525V) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
+				USB_DEVICE_ID_GENIUS_GILA_GAMING_MOUSE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
+				USB_DEVICE_ID_GENIUS_MANTICORE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
+				USB_DEVICE_ID_GENIUS_GX_IMPERATOR) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
+				USB_DEVICE_ID_KYE_EASYPEN_M406) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
+				USB_DEVICE_ID_KYE_EASYPEN_M506) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
 				USB_DEVICE_ID_KYE_EASYPEN_I405X) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
 				USB_DEVICE_ID_KYE_MOUSEPEN_I608X) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
-				USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2) },
+				USB_DEVICE_ID_KYE_EASYPEN_M406W) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
 				USB_DEVICE_ID_KYE_EASYPEN_M610X) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
-				USB_DEVICE_ID_KYE_EASYPEN_M406XE) },
+				USB_DEVICE_ID_KYE_EASYPEN_340) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
-				USB_DEVICE_ID_GENIUS_GILA_GAMING_MOUSE) },
+				USB_DEVICE_ID_KYE_PENSKETCH_M912) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
-				USB_DEVICE_ID_GENIUS_GX_IMPERATOR) },
+				USB_DEVICE_ID_KYE_MOUSEPEN_M508WX) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
-				USB_DEVICE_ID_GENIUS_MANTICORE) },
+				USB_DEVICE_ID_KYE_MOUSEPEN_M508X) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
-				USB_DEVICE_ID_KYE_PENSKETCH_M912) },
+				USB_DEVICE_ID_KYE_EASYPEN_M406XE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
+				USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
+				USB_DEVICE_ID_KYE_PENSKETCH_T609A) },
 	{ }
 };
 MODULE_DEVICE_TABLE(hid, kye_devices);
diff --git a/drivers/hid/hid-lg-g15.c b/drivers/hid/hid-lg-g15.c
index c8f82bcbf1ab..acbec1dcf196 100644
--- a/drivers/hid/hid-lg-g15.c
+++ b/drivers/hid/hid-lg-g15.c
@@ -7,6 +7,7 @@
 
 #include <linux/device.h>
 #include <linux/hid.h>
+#include <linux/leds.h>
 #include <linux/module.h>
 #include <linux/random.h>
 #include <linux/sched.h>
diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c
index 5fc88a063297..0fcfd85fea0f 100644
--- a/drivers/hid/hid-logitech-hidpp.c
+++ b/drivers/hid/hid-logitech-hidpp.c
@@ -74,6 +74,7 @@ MODULE_PARM_DESC(disable_tap_to_click,
 #define HIDPP_QUIRK_HIDPP_EXTRA_MOUSE_BTNS	BIT(27)
 #define HIDPP_QUIRK_HIDPP_CONSUMER_VENDOR_KEYS	BIT(28)
 #define HIDPP_QUIRK_HI_RES_SCROLL_1P0		BIT(29)
+#define HIDPP_QUIRK_WIRELESS_STATUS		BIT(30)
 
 /* These are just aliases for now */
 #define HIDPP_QUIRK_KBD_SCROLL_WHEEL HIDPP_QUIRK_HIDPP_WHEELS
@@ -94,6 +95,7 @@ MODULE_PARM_DESC(disable_tap_to_click,
 #define HIDPP_CAPABILITY_HIDPP20_HI_RES_WHEEL	BIT(7)
 #define HIDPP_CAPABILITY_HIDPP20_HI_RES_SCROLL	BIT(8)
 #define HIDPP_CAPABILITY_HIDPP10_FAST_SCROLL	BIT(9)
+#define HIDPP_CAPABILITY_ADC_MEASUREMENT	BIT(10)
 
 #define lg_map_key_clear(c)  hid_map_usage_clear(hi, usage, bit, max, EV_KEY, (c))
 
@@ -145,6 +147,7 @@ struct hidpp_battery {
 	u8 feature_index;
 	u8 solar_feature_index;
 	u8 voltage_feature_index;
+	u8 adc_measurement_feature_index;
 	struct power_supply_desc desc;
 	struct power_supply *ps;
 	char name[64];
@@ -471,6 +474,26 @@ static void hidpp_prefix_name(char **name, int name_length)
 	*name = new_name;
 }
 
+/*
+ * Updates the USB wireless_status based on whether the headset
+ * is turned on and reachable.
+ */
+static void hidpp_update_usb_wireless_status(struct hidpp_device *hidpp)
+{
+	struct hid_device *hdev = hidpp->hid_dev;
+	struct usb_interface *intf;
+
+	if (!(hidpp->quirks & HIDPP_QUIRK_WIRELESS_STATUS))
+		return;
+	if (!hid_is_usb(hdev))
+		return;
+
+	intf = to_usb_interface(hdev->dev.parent);
+	usb_set_wireless_status(intf, hidpp->battery.online ?
+				USB_WIRELESS_STATUS_CONNECTED :
+				USB_WIRELESS_STATUS_DISCONNECTED);
+}
+
 /**
  * hidpp_scroll_counter_handle_scroll() - Send high- and low-resolution scroll
  *                                        events given a high-resolution wheel
@@ -853,8 +876,7 @@ static int hidpp_unifying_init(struct hidpp_device *hidpp)
 	if (ret)
 		return ret;
 
-	snprintf(hdev->uniq, sizeof(hdev->uniq), "%04x-%4phD",
-		 hdev->product, &serial);
+	snprintf(hdev->uniq, sizeof(hdev->uniq), "%4phD", &serial);
 	dbg_hid("HID++ Unifying: Got serial: %s\n", hdev->uniq);
 
 	name = hidpp_unifying_get_name(hidpp);
@@ -948,6 +970,54 @@ print_version:
 }
 
 /* -------------------------------------------------------------------------- */
+/* 0x0003: Device Information                                                 */
+/* -------------------------------------------------------------------------- */
+
+#define HIDPP_PAGE_DEVICE_INFORMATION			0x0003
+
+#define CMD_GET_DEVICE_INFO				0x00
+
+static int hidpp_get_serial(struct hidpp_device *hidpp, u32 *serial)
+{
+	struct hidpp_report response;
+	u8 feature_type;
+	u8 feature_index;
+	int ret;
+
+	ret = hidpp_root_get_feature(hidpp, HIDPP_PAGE_DEVICE_INFORMATION,
+				     &feature_index,
+				     &feature_type);
+	if (ret)
+		return ret;
+
+	ret = hidpp_send_fap_command_sync(hidpp, feature_index,
+					  CMD_GET_DEVICE_INFO,
+					  NULL, 0, &response);
+	if (ret)
+		return ret;
+
+	/* See hidpp_unifying_get_serial() */
+	*serial = *((u32 *)&response.rap.params[1]);
+	return 0;
+}
+
+static int hidpp_serial_init(struct hidpp_device *hidpp)
+{
+	struct hid_device *hdev = hidpp->hid_dev;
+	u32 serial;
+	int ret;
+
+	ret = hidpp_get_serial(hidpp, &serial);
+	if (ret)
+		return ret;
+
+	snprintf(hdev->uniq, sizeof(hdev->uniq), "%4phD", &serial);
+	dbg_hid("HID++ DeviceInformation: Got serial: %s\n", hdev->uniq);
+
+	return 0;
+}
+
+/* -------------------------------------------------------------------------- */
 /* 0x0005: GetDeviceNameType                                                  */
 /* -------------------------------------------------------------------------- */
 
@@ -1357,7 +1427,7 @@ static int hidpp20_map_battery_capacity(struct hid_device *hid_dev, int voltage)
 	 * there are a few devices that use different battery technology.
 	 */
 
-	static const int voltages[] = {
+	static const int voltages[100] = {
 		4186, 4156, 4143, 4133, 4122, 4113, 4103, 4094, 4086, 4075,
 		4067, 4059, 4051, 4043, 4035, 4027, 4019, 4011, 4003, 3997,
 		3989, 3983, 3976, 3969, 3961, 3955, 3949, 3942, 3935, 3929,
@@ -1372,8 +1442,6 @@ static int hidpp20_map_battery_capacity(struct hid_device *hid_dev, int voltage)
 
 	int i;
 
-	BUILD_BUG_ON(ARRAY_SIZE(voltages) != 100);
-
 	if (unlikely(voltage < 3500 || voltage >= 5000))
 		hid_warn_once(hid_dev,
 			      "%s: possibly using the wrong voltage curve\n",
@@ -1746,6 +1814,164 @@ static int hidpp_set_wireless_feature_index(struct hidpp_device *hidpp)
 }
 
 /* -------------------------------------------------------------------------- */
+/* 0x1f20: ADC measurement                                                    */
+/* -------------------------------------------------------------------------- */
+
+#define HIDPP_PAGE_ADC_MEASUREMENT 0x1f20
+
+#define CMD_ADC_MEASUREMENT_GET_ADC_MEASUREMENT 0x00
+
+#define EVENT_ADC_MEASUREMENT_STATUS_BROADCAST 0x00
+
+static int hidpp20_map_adc_measurement_1f20_capacity(struct hid_device *hid_dev, int voltage)
+{
+	/* NB: This voltage curve doesn't necessarily map perfectly to all
+	 * devices that implement the ADC_MEASUREMENT feature. This is because
+	 * there are a few devices that use different battery technology.
+	 *
+	 * Adapted from:
+	 * https://github.com/Sapd/HeadsetControl/blob/acd972be0468e039b93aae81221f20a54d2d60f7/src/devices/logitech_g633_g933_935.c#L44-L52
+	 */
+	static const int voltages[100] = {
+		4030, 4024, 4018, 4011, 4003, 3994, 3985, 3975, 3963, 3951,
+		3937, 3922, 3907, 3893, 3880, 3868, 3857, 3846, 3837, 3828,
+		3820, 3812, 3805, 3798, 3791, 3785, 3779, 3773, 3768, 3762,
+		3757, 3752, 3747, 3742, 3738, 3733, 3729, 3724, 3720, 3716,
+		3712, 3708, 3704, 3700, 3696, 3692, 3688, 3685, 3681, 3677,
+		3674, 3670, 3667, 3663, 3660, 3657, 3653, 3650, 3646, 3643,
+		3640, 3637, 3633, 3630, 3627, 3624, 3620, 3617, 3614, 3611,
+		3608, 3604, 3601, 3598, 3595, 3592, 3589, 3585, 3582, 3579,
+		3576, 3573, 3569, 3566, 3563, 3560, 3556, 3553, 3550, 3546,
+		3543, 3539, 3536, 3532, 3529, 3525, 3499, 3466, 3433, 3399,
+	};
+
+	int i;
+
+	if (voltage == 0)
+		return 0;
+
+	if (unlikely(voltage < 3400 || voltage >= 5000))
+		hid_warn_once(hid_dev,
+			      "%s: possibly using the wrong voltage curve\n",
+			      __func__);
+
+	for (i = 0; i < ARRAY_SIZE(voltages); i++) {
+		if (voltage >= voltages[i])
+			return ARRAY_SIZE(voltages) - i;
+	}
+
+	return 0;
+}
+
+static int hidpp20_map_adc_measurement_1f20(u8 data[3], int *voltage)
+{
+	int status;
+	u8 flags;
+
+	flags = data[2];
+
+	switch (flags) {
+	case 0x01:
+		status = POWER_SUPPLY_STATUS_DISCHARGING;
+		break;
+	case 0x03:
+		status = POWER_SUPPLY_STATUS_CHARGING;
+		break;
+	case 0x07:
+		status = POWER_SUPPLY_STATUS_FULL;
+		break;
+	case 0x0F:
+	default:
+		status = POWER_SUPPLY_STATUS_UNKNOWN;
+		break;
+	}
+
+	*voltage = get_unaligned_be16(data);
+
+	dbg_hid("Parsed 1f20 data as flag 0x%02x voltage %dmV\n",
+		flags, *voltage);
+
+	return status;
+}
+
+/* Return value is whether the device is online */
+static bool hidpp20_get_adc_measurement_1f20(struct hidpp_device *hidpp,
+						 u8 feature_index,
+						 int *status, int *voltage)
+{
+	struct hidpp_report response;
+	int ret;
+	u8 *params = (u8 *)response.fap.params;
+
+	*status = POWER_SUPPLY_STATUS_UNKNOWN;
+	*voltage = 0;
+	ret = hidpp_send_fap_command_sync(hidpp, feature_index,
+					  CMD_ADC_MEASUREMENT_GET_ADC_MEASUREMENT,
+					  NULL, 0, &response);
+
+	if (ret > 0) {
+		hid_dbg(hidpp->hid_dev, "%s: received protocol error 0x%02x\n",
+			__func__, ret);
+		return false;
+	}
+
+	*status = hidpp20_map_adc_measurement_1f20(params, voltage);
+	return true;
+}
+
+static int hidpp20_query_adc_measurement_info_1f20(struct hidpp_device *hidpp)
+{
+	u8 feature_type;
+
+	if (hidpp->battery.adc_measurement_feature_index == 0xff) {
+		int ret;
+
+		ret = hidpp_root_get_feature(hidpp, HIDPP_PAGE_ADC_MEASUREMENT,
+					     &hidpp->battery.adc_measurement_feature_index,
+					     &feature_type);
+		if (ret)
+			return ret;
+
+		hidpp->capabilities |= HIDPP_CAPABILITY_ADC_MEASUREMENT;
+	}
+
+	hidpp->battery.online = hidpp20_get_adc_measurement_1f20(hidpp,
+								 hidpp->battery.adc_measurement_feature_index,
+								 &hidpp->battery.status,
+								 &hidpp->battery.voltage);
+	hidpp->battery.capacity = hidpp20_map_adc_measurement_1f20_capacity(hidpp->hid_dev,
+									    hidpp->battery.voltage);
+	hidpp_update_usb_wireless_status(hidpp);
+
+	return 0;
+}
+
+static int hidpp20_adc_measurement_event_1f20(struct hidpp_device *hidpp,
+					    u8 *data, int size)
+{
+	struct hidpp_report *report = (struct hidpp_report *)data;
+	int status, voltage;
+
+	if (report->fap.feature_index != hidpp->battery.adc_measurement_feature_index ||
+		report->fap.funcindex_clientid != EVENT_ADC_MEASUREMENT_STATUS_BROADCAST)
+		return 0;
+
+	status = hidpp20_map_adc_measurement_1f20(report->fap.params, &voltage);
+
+	hidpp->battery.online = status != POWER_SUPPLY_STATUS_UNKNOWN;
+
+	if (voltage != hidpp->battery.voltage || status != hidpp->battery.status) {
+		hidpp->battery.status = status;
+		hidpp->battery.voltage = voltage;
+		hidpp->battery.capacity = hidpp20_map_adc_measurement_1f20_capacity(hidpp->hid_dev, voltage);
+		if (hidpp->battery.ps)
+			power_supply_changed(hidpp->battery.ps);
+		hidpp_update_usb_wireless_status(hidpp);
+	}
+	return 0;
+}
+
+/* -------------------------------------------------------------------------- */
 /* 0x2120: Hi-resolution scrolling                                            */
 /* -------------------------------------------------------------------------- */
 
@@ -3663,6 +3889,9 @@ static int hidpp_raw_hidpp_event(struct hidpp_device *hidpp, u8 *data,
 		ret = hidpp20_battery_voltage_event(hidpp, data, size);
 		if (ret != 0)
 			return ret;
+		ret = hidpp20_adc_measurement_event_1f20(hidpp, data, size);
+		if (ret != 0)
+			return ret;
 	}
 
 	if (hidpp->capabilities & HIDPP_CAPABILITY_HIDPP10_BATTERY) {
@@ -3786,6 +4015,7 @@ static int hidpp_initialize_battery(struct hidpp_device *hidpp)
 	hidpp->battery.feature_index = 0xff;
 	hidpp->battery.solar_feature_index = 0xff;
 	hidpp->battery.voltage_feature_index = 0xff;
+	hidpp->battery.adc_measurement_feature_index = 0xff;
 
 	if (hidpp->protocol_major >= 2) {
 		if (hidpp->quirks & HIDPP_QUIRK_CLASS_K750)
@@ -3799,6 +4029,8 @@ static int hidpp_initialize_battery(struct hidpp_device *hidpp)
 				ret = hidpp20_query_battery_info_1004(hidpp);
 			if (ret)
 				ret = hidpp20_query_battery_voltage_info(hidpp);
+			if (ret)
+				ret = hidpp20_query_adc_measurement_info_1f20(hidpp);
 		}
 
 		if (ret)
@@ -3828,7 +4060,8 @@ static int hidpp_initialize_battery(struct hidpp_device *hidpp)
 
 	if (hidpp->capabilities & HIDPP_CAPABILITY_BATTERY_MILEAGE ||
 	    hidpp->capabilities & HIDPP_CAPABILITY_BATTERY_PERCENTAGE ||
-	    hidpp->capabilities & HIDPP_CAPABILITY_BATTERY_VOLTAGE)
+	    hidpp->capabilities & HIDPP_CAPABILITY_BATTERY_VOLTAGE ||
+	    hidpp->capabilities & HIDPP_CAPABILITY_ADC_MEASUREMENT)
 		battery_props[num_battery_props++] =
 				POWER_SUPPLY_PROP_CAPACITY;
 
@@ -3836,7 +4069,8 @@ static int hidpp_initialize_battery(struct hidpp_device *hidpp)
 		battery_props[num_battery_props++] =
 				POWER_SUPPLY_PROP_CAPACITY_LEVEL;
 
-	if (hidpp->capabilities & HIDPP_CAPABILITY_BATTERY_VOLTAGE)
+	if (hidpp->capabilities & HIDPP_CAPABILITY_BATTERY_VOLTAGE ||
+	    hidpp->capabilities & HIDPP_CAPABILITY_ADC_MEASUREMENT)
 		battery_props[num_battery_props++] =
 			POWER_SUPPLY_PROP_VOLTAGE_NOW;
 
@@ -4009,6 +4243,8 @@ static void hidpp_connect_event(struct hidpp_device *hidpp)
 			hidpp20_query_battery_voltage_info(hidpp);
 		else if (hidpp->capabilities & HIDPP_CAPABILITY_UNIFIED_BATTERY)
 			hidpp20_query_battery_info_1004(hidpp);
+		else if (hidpp->capabilities & HIDPP_CAPABILITY_ADC_MEASUREMENT)
+			hidpp20_query_adc_measurement_info_1f20(hidpp);
 		else
 			hidpp20_query_battery_info_1000(hidpp);
 	}
@@ -4210,6 +4446,8 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id)
 
 	if (hidpp->quirks & HIDPP_QUIRK_UNIFYING)
 		hidpp_unifying_init(hidpp);
+	else if (hid_is_usb(hidpp->hid_dev))
+		hidpp_serial_init(hidpp);
 
 	connected = hidpp_root_get_protocol_version(hidpp) == 0;
 	atomic_set(&hidpp->connected, connected);
@@ -4379,6 +4617,10 @@ static const struct hid_device_id hidpp_devices[] = {
 	{ /* Logitech G Pro Gaming Mouse over USB */
 	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC088) },
 
+	{ /* G935 Gaming Headset */
+	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0x0a87),
+		.driver_data = HIDPP_QUIRK_WIRELESS_STATUS },
+
 	{ /* MX5000 keyboard over Bluetooth */
 	  HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb305),
 	  .driver_data = HIDPP_QUIRK_HIDPP_CONSUMER_VENDOR_KEYS },
diff --git a/drivers/hid/hid-mcp2221.c b/drivers/hid/hid-mcp2221.c
index f74a977cf8f8..72883e0ce757 100644
--- a/drivers/hid/hid-mcp2221.c
+++ b/drivers/hid/hid-mcp2221.c
@@ -79,8 +79,8 @@ struct mcp_get_gpio {
 	u8 cmd;
 	u8 dummy;
 	struct {
-		u8 direction;
 		u8 value;
+		u8 direction;
 	} gpio[MCP_NGPIO];
 } __packed;
 
@@ -594,7 +594,7 @@ static int mcp_gpio_get(struct gpio_chip *gc,
 
 	mcp->txbuf[0] = MCP2221_GPIO_GET;
 
-	mcp->gp_idx = offsetof(struct mcp_get_gpio, gpio[offset].value);
+	mcp->gp_idx = offsetof(struct mcp_get_gpio, gpio[offset]);
 
 	mutex_lock(&mcp->lock);
 	ret = mcp_send_data_req_status(mcp, mcp->txbuf, 1);
@@ -675,7 +675,7 @@ static int mcp_gpio_get_direction(struct gpio_chip *gc,
 
 	mcp->txbuf[0] = MCP2221_GPIO_GET;
 
-	mcp->gp_idx = offsetof(struct mcp_get_gpio, gpio[offset].direction);
+	mcp->gp_idx = offsetof(struct mcp_get_gpio, gpio[offset]);
 
 	mutex_lock(&mcp->lock);
 	ret = mcp_send_data_req_status(mcp, mcp->txbuf, 1);
diff --git a/drivers/hid/hid-nintendo.c b/drivers/hid/hid-nintendo.c
index 5bfc0c450460..250f5d2f888a 100644
--- a/drivers/hid/hid-nintendo.c
+++ b/drivers/hid/hid-nintendo.c
@@ -433,7 +433,9 @@ struct joycon_ctlr {
 	u8 usb_ack_match;
 	u8 subcmd_ack_match;
 	bool received_input_report;
+	unsigned int last_input_report_msecs;
 	unsigned int last_subcmd_sent_msecs;
+	unsigned int consecutive_valid_report_deltas;
 
 	/* factory calibration data */
 	struct joycon_stick_cal left_stick_cal_x;
@@ -543,19 +545,54 @@ static void joycon_wait_for_input_report(struct joycon_ctlr *ctlr)
  * Sending subcommands and/or rumble data at too high a rate can cause bluetooth
  * controller disconnections.
  */
+#define JC_INPUT_REPORT_MIN_DELTA	8
+#define JC_INPUT_REPORT_MAX_DELTA	17
+#define JC_SUBCMD_TX_OFFSET_MS		4
+#define JC_SUBCMD_VALID_DELTA_REQ	3
+#define JC_SUBCMD_RATE_MAX_ATTEMPTS	500
+#define JC_SUBCMD_RATE_LIMITER_USB_MS	20
+#define JC_SUBCMD_RATE_LIMITER_BT_MS	60
+#define JC_SUBCMD_RATE_LIMITER_MS(ctlr)	((ctlr)->hdev->bus == BUS_USB ? JC_SUBCMD_RATE_LIMITER_USB_MS : JC_SUBCMD_RATE_LIMITER_BT_MS)
 static void joycon_enforce_subcmd_rate(struct joycon_ctlr *ctlr)
 {
-	static const unsigned int max_subcmd_rate_ms = 25;
-	unsigned int current_ms = jiffies_to_msecs(jiffies);
-	unsigned int delta_ms = current_ms - ctlr->last_subcmd_sent_msecs;
+	unsigned int current_ms;
+	unsigned long subcmd_delta;
+	int consecutive_valid_deltas = 0;
+	int attempts = 0;
+	unsigned long flags;
+
+	if (unlikely(ctlr->ctlr_state != JOYCON_CTLR_STATE_READ))
+		return;
 
-	while (delta_ms < max_subcmd_rate_ms &&
-	       ctlr->ctlr_state == JOYCON_CTLR_STATE_READ) {
+	do {
 		joycon_wait_for_input_report(ctlr);
 		current_ms = jiffies_to_msecs(jiffies);
-		delta_ms = current_ms - ctlr->last_subcmd_sent_msecs;
+		subcmd_delta = current_ms - ctlr->last_subcmd_sent_msecs;
+
+		spin_lock_irqsave(&ctlr->lock, flags);
+		consecutive_valid_deltas = ctlr->consecutive_valid_report_deltas;
+		spin_unlock_irqrestore(&ctlr->lock, flags);
+
+		attempts++;
+	} while ((consecutive_valid_deltas < JC_SUBCMD_VALID_DELTA_REQ ||
+		  subcmd_delta < JC_SUBCMD_RATE_LIMITER_MS(ctlr)) &&
+		 ctlr->ctlr_state == JOYCON_CTLR_STATE_READ &&
+		 attempts < JC_SUBCMD_RATE_MAX_ATTEMPTS);
+
+	if (attempts >= JC_SUBCMD_RATE_MAX_ATTEMPTS) {
+		hid_warn(ctlr->hdev, "%s: exceeded max attempts", __func__);
+		return;
 	}
+
 	ctlr->last_subcmd_sent_msecs = current_ms;
+
+	/*
+	 * Wait a short time after receiving an input report before
+	 * transmitting. This should reduce odds of a TX coinciding with an RX.
+	 * Minimizing concurrent BT traffic with the controller seems to lower
+	 * the rate of disconnections.
+	 */
+	msleep(JC_SUBCMD_TX_OFFSET_MS);
 }
 
 static int joycon_hid_send_sync(struct joycon_ctlr *ctlr, u8 *data, size_t len,
@@ -1223,6 +1260,7 @@ static void joycon_parse_report(struct joycon_ctlr *ctlr,
 	u8 tmp;
 	u32 btns;
 	unsigned long msecs = jiffies_to_msecs(jiffies);
+	unsigned long report_delta_ms = msecs - ctlr->last_input_report_msecs;
 
 	spin_lock_irqsave(&ctlr->lock, flags);
 	if (IS_ENABLED(CONFIG_NINTENDO_FF) && rep->vibrator_report &&
@@ -1364,6 +1402,31 @@ static void joycon_parse_report(struct joycon_ctlr *ctlr,
 
 	input_sync(dev);
 
+	spin_lock_irqsave(&ctlr->lock, flags);
+	ctlr->last_input_report_msecs = msecs;
+	/*
+	 * Was this input report a reasonable time delta compared to the prior
+	 * report? We use this information to decide when a safe time is to send
+	 * rumble packets or subcommand packets.
+	 */
+	if (report_delta_ms >= JC_INPUT_REPORT_MIN_DELTA &&
+	    report_delta_ms <= JC_INPUT_REPORT_MAX_DELTA) {
+		if (ctlr->consecutive_valid_report_deltas < JC_SUBCMD_VALID_DELTA_REQ)
+			ctlr->consecutive_valid_report_deltas++;
+	} else {
+		ctlr->consecutive_valid_report_deltas = 0;
+	}
+	/*
+	 * Our consecutive valid report tracking is only relevant for
+	 * bluetooth-connected controllers. For USB devices, we're beholden to
+	 * USB's underlying polling rate anyway. Always set to the consecutive
+	 * delta requirement.
+	 */
+	if (ctlr->hdev->bus == BUS_USB)
+		ctlr->consecutive_valid_report_deltas = JC_SUBCMD_VALID_DELTA_REQ;
+
+	spin_unlock_irqrestore(&ctlr->lock, flags);
+
 	/*
 	 * Immediately after receiving a report is the most reliable time to
 	 * send a subcommand to the controller. Wake any subcommand senders
@@ -1527,6 +1590,7 @@ static int joycon_set_rumble(struct joycon_ctlr *ctlr, u16 amp_r, u16 amp_l,
 	u16 freq_l_low;
 	u16 freq_l_high;
 	unsigned long flags;
+	int next_rq_head;
 
 	spin_lock_irqsave(&ctlr->lock, flags);
 	freq_r_low = ctlr->rumble_rl_freq;
@@ -1547,8 +1611,21 @@ static int joycon_set_rumble(struct joycon_ctlr *ctlr, u16 amp_r, u16 amp_l,
 	joycon_encode_rumble(data, freq_l_low, freq_l_high, amp);
 
 	spin_lock_irqsave(&ctlr->lock, flags);
-	if (++ctlr->rumble_queue_head >= JC_RUMBLE_QUEUE_SIZE)
-		ctlr->rumble_queue_head = 0;
+
+	next_rq_head = ctlr->rumble_queue_head + 1;
+	if (next_rq_head >= JC_RUMBLE_QUEUE_SIZE)
+		next_rq_head = 0;
+
+	/* Did we overrun the circular buffer?
+	 * If so, be sure we keep the latest intended rumble state.
+	 */
+	if (next_rq_head == ctlr->rumble_queue_tail) {
+		hid_dbg(ctlr->hdev, "rumble queue is full");
+		/* overwrite the prior value at the end of the circular buf */
+		next_rq_head = ctlr->rumble_queue_head;
+	}
+
+	ctlr->rumble_queue_head = next_rq_head;
 	memcpy(ctlr->rumble_data[ctlr->rumble_queue_head], data,
 	       JC_RUMBLE_DATA_SIZE);
 
@@ -2128,7 +2205,7 @@ static int nintendo_hid_probe(struct hid_device *hdev,
 
 	ctlr->hdev = hdev;
 	ctlr->ctlr_state = JOYCON_CTLR_STATE_INIT;
-	ctlr->rumble_queue_head = JC_RUMBLE_QUEUE_SIZE - 1;
+	ctlr->rumble_queue_head = 0;
 	ctlr->rumble_queue_tail = 0;
 	hid_set_drvdata(hdev, ctlr);
 	mutex_init(&ctlr->output_mutex);
diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c
index 66e64350f138..804fc03600cc 100644
--- a/drivers/hid/hid-quirks.c
+++ b/drivers/hid/hid-quirks.c
@@ -104,12 +104,20 @@ static const struct hid_device_id hid_quirks[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_PIXART_OEM_USB_OPTICAL_MOUSE_1f4a), HID_QUIRK_ALWAYS_POLL },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_IDEACOM, USB_DEVICE_ID_IDEACOM_IDC6680), HID_QUIRK_MULTI_INPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_INNOMEDIA, USB_DEVICE_ID_INNEX_GENESIS_ATARI), HID_QUIRK_MULTI_INPUT },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M610X), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_PIXART_USB_OPTICAL_MOUSE_ID2), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M406), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M506), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_I405X), HID_QUIRK_MULTI_INPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X), HID_QUIRK_MULTI_INPUT },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M406W), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M610X), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_340), HID_QUIRK_MULTI_INPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_PENSKETCH_M912), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_M508WX), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_M508X), HID_QUIRK_MULTI_INPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M406XE), HID_QUIRK_MULTI_INPUT },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_PIXART_USB_OPTICAL_MOUSE_ID2), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_PENSKETCH_T609A), HID_QUIRK_MULTI_INPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_OPTICAL_USB_MOUSE_600E), HID_QUIRK_ALWAYS_POLL },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_608D), HID_QUIRK_ALWAYS_POLL },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_6019), HID_QUIRK_ALWAYS_POLL },
diff --git a/drivers/hid/hid-roccat-arvo.c b/drivers/hid/hid-roccat-arvo.c
index d94ee0539421..ea6b79b3aeeb 100644
--- a/drivers/hid/hid-roccat-arvo.c
+++ b/drivers/hid/hid-roccat-arvo.c
@@ -433,7 +433,7 @@ static int __init arvo_init(void)
 {
 	int retval;
 
-	arvo_class = class_create(THIS_MODULE, "arvo");
+	arvo_class = class_create("arvo");
 	if (IS_ERR(arvo_class))
 		return PTR_ERR(arvo_class);
 	arvo_class->dev_groups = arvo_groups;
diff --git a/drivers/hid/hid-roccat-isku.c b/drivers/hid/hid-roccat-isku.c
index e95d59cd8d07..3903a2cea00c 100644
--- a/drivers/hid/hid-roccat-isku.c
+++ b/drivers/hid/hid-roccat-isku.c
@@ -435,7 +435,7 @@ static struct hid_driver isku_driver = {
 static int __init isku_init(void)
 {
 	int retval;
-	isku_class = class_create(THIS_MODULE, "isku");
+	isku_class = class_create("isku");
 	if (IS_ERR(isku_class))
 		return PTR_ERR(isku_class);
 	isku_class->dev_groups = isku_groups;
diff --git a/drivers/hid/hid-roccat-kone.c b/drivers/hid/hid-roccat-kone.c
index 76da04801ca9..945ae236fb45 100644
--- a/drivers/hid/hid-roccat-kone.c
+++ b/drivers/hid/hid-roccat-kone.c
@@ -890,7 +890,7 @@ static int __init kone_init(void)
 	int retval;
 
 	/* class name has to be same as driver name */
-	kone_class = class_create(THIS_MODULE, "kone");
+	kone_class = class_create("kone");
 	if (IS_ERR(kone_class))
 		return PTR_ERR(kone_class);
 	kone_class->dev_groups = kone_groups;
diff --git a/drivers/hid/hid-roccat-koneplus.c b/drivers/hid/hid-roccat-koneplus.c
index 1896c69ea512..97b83b6f53dd 100644
--- a/drivers/hid/hid-roccat-koneplus.c
+++ b/drivers/hid/hid-roccat-koneplus.c
@@ -549,7 +549,7 @@ static int __init koneplus_init(void)
 	int retval;
 
 	/* class name has to be same as driver name */
-	koneplus_class = class_create(THIS_MODULE, "koneplus");
+	koneplus_class = class_create("koneplus");
 	if (IS_ERR(koneplus_class))
 		return PTR_ERR(koneplus_class);
 	koneplus_class->dev_groups = koneplus_groups;
diff --git a/drivers/hid/hid-roccat-konepure.c b/drivers/hid/hid-roccat-konepure.c
index cf8eeb33a125..a297756f2410 100644
--- a/drivers/hid/hid-roccat-konepure.c
+++ b/drivers/hid/hid-roccat-konepure.c
@@ -207,7 +207,7 @@ static int __init konepure_init(void)
 {
 	int retval;
 
-	konepure_class = class_create(THIS_MODULE, "konepure");
+	konepure_class = class_create("konepure");
 	if (IS_ERR(konepure_class))
 		return PTR_ERR(konepure_class);
 	konepure_class->dev_groups = konepure_groups;
diff --git a/drivers/hid/hid-roccat-kovaplus.c b/drivers/hid/hid-roccat-kovaplus.c
index 6fb9b9563769..1a1d96e11683 100644
--- a/drivers/hid/hid-roccat-kovaplus.c
+++ b/drivers/hid/hid-roccat-kovaplus.c
@@ -638,7 +638,7 @@ static int __init kovaplus_init(void)
 {
 	int retval;
 
-	kovaplus_class = class_create(THIS_MODULE, "kovaplus");
+	kovaplus_class = class_create("kovaplus");
 	if (IS_ERR(kovaplus_class))
 		return PTR_ERR(kovaplus_class);
 	kovaplus_class->dev_groups = kovaplus_groups;
diff --git a/drivers/hid/hid-roccat-pyra.c b/drivers/hid/hid-roccat-pyra.c
index 4fcc8e7d276f..15528c3b013c 100644
--- a/drivers/hid/hid-roccat-pyra.c
+++ b/drivers/hid/hid-roccat-pyra.c
@@ -585,7 +585,7 @@ static int __init pyra_init(void)
 	int retval;
 
 	/* class name has to be same as driver name */
-	pyra_class = class_create(THIS_MODULE, "pyra");
+	pyra_class = class_create("pyra");
 	if (IS_ERR(pyra_class))
 		return PTR_ERR(pyra_class);
 	pyra_class->dev_groups = pyra_groups;
diff --git a/drivers/hid/hid-roccat-ryos.c b/drivers/hid/hid-roccat-ryos.c
index 5bf1971a2b14..0eb17a3b925d 100644
--- a/drivers/hid/hid-roccat-ryos.c
+++ b/drivers/hid/hid-roccat-ryos.c
@@ -216,7 +216,7 @@ static int __init ryos_init(void)
 {
 	int retval;
 
-	ryos_class = class_create(THIS_MODULE, "ryos");
+	ryos_class = class_create("ryos");
 	if (IS_ERR(ryos_class))
 		return PTR_ERR(ryos_class);
 	ryos_class->dev_groups = ryos_groups;
diff --git a/drivers/hid/hid-roccat-savu.c b/drivers/hid/hid-roccat-savu.c
index a784bb4ee651..93be7acef673 100644
--- a/drivers/hid/hid-roccat-savu.c
+++ b/drivers/hid/hid-roccat-savu.c
@@ -204,7 +204,7 @@ static int __init savu_init(void)
 {
 	int retval;
 
-	savu_class = class_create(THIS_MODULE, "savu");
+	savu_class = class_create("savu");
 	if (IS_ERR(savu_class))
 		return PTR_ERR(savu_class);
 	savu_class->dev_groups = savu_groups;
diff --git a/drivers/hid/hid-steelseries.c b/drivers/hid/hid-steelseries.c
index 37353c41cba7..aae3afc4107a 100644
--- a/drivers/hid/hid-steelseries.c
+++ b/drivers/hid/hid-steelseries.c
@@ -11,6 +11,7 @@
 #include <linux/device.h>
 #include <linux/hid.h>
 #include <linux/module.h>
+#include <linux/leds.h>
 
 #include "hid-ids.h"
 
diff --git a/drivers/hid/hidraw.c b/drivers/hid/hidraw.c
index 197b1e7bf029..93e62b161501 100644
--- a/drivers/hid/hidraw.c
+++ b/drivers/hid/hidraw.c
@@ -618,7 +618,7 @@ int __init hidraw_init(void)
 
 	hidraw_major = MAJOR(dev_id);
 
-	hidraw_class = class_create(THIS_MODULE, "hidraw");
+	hidraw_class = class_create("hidraw");
 	if (IS_ERR(hidraw_class)) {
 		result = PTR_ERR(hidraw_class);
 		goto error_cdev;
diff --git a/drivers/hid/i2c-hid/Kconfig b/drivers/hid/i2c-hid/Kconfig
index 4439be7fa74d..3be17109301a 100644
--- a/drivers/hid/i2c-hid/Kconfig
+++ b/drivers/hid/i2c-hid/Kconfig
@@ -23,12 +23,14 @@ config I2C_HID_ACPI
 
 config I2C_HID_OF
 	tristate "HID over I2C transport layer Open Firmware driver"
-	depends on OF
+	# No "depends on OF" because this can also be used for manually
+	# (board-file) instantiated "hid-over-i2c" type i2c-clients.
 	select I2C_HID_CORE
 	help
 	  Say Y here if you use a keyboard, a touchpad, a touchscreen, or any
 	  other HID based devices which is connected to your computer via I2C.
-	  This driver supports Open Firmware (Device Tree)-based systems.
+	  This driver supports Open Firmware (Device Tree)-based systems as
+	  well as binding to manually (board-file) instantiated i2c-hid-clients.
 
 	  If unsure, say N.
 
diff --git a/drivers/hid/i2c-hid/i2c-hid-of.c b/drivers/hid/i2c-hid/i2c-hid-of.c
index 10176568133a..855f53092f4e 100644
--- a/drivers/hid/i2c-hid/i2c-hid-of.c
+++ b/drivers/hid/i2c-hid/i2c-hid-of.c
@@ -21,6 +21,7 @@
 
 #include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/gpio/consumer.h>
 #include <linux/hid.h>
 #include <linux/i2c.h>
 #include <linux/kernel.h>
@@ -35,8 +36,10 @@ struct i2c_hid_of {
 	struct i2chid_ops ops;
 
 	struct i2c_client *client;
+	struct gpio_desc *reset_gpio;
 	struct regulator_bulk_data supplies[2];
 	int post_power_delay_ms;
+	int post_reset_delay_ms;
 };
 
 static int i2c_hid_of_power_up(struct i2chid_ops *ops)
@@ -55,6 +58,10 @@ static int i2c_hid_of_power_up(struct i2chid_ops *ops)
 	if (ihid_of->post_power_delay_ms)
 		msleep(ihid_of->post_power_delay_ms);
 
+	gpiod_set_value_cansleep(ihid_of->reset_gpio, 0);
+	if (ihid_of->post_reset_delay_ms)
+		msleep(ihid_of->post_reset_delay_ms);
+
 	return 0;
 }
 
@@ -62,6 +69,7 @@ static void i2c_hid_of_power_down(struct i2chid_ops *ops)
 {
 	struct i2c_hid_of *ihid_of = container_of(ops, struct i2c_hid_of, ops);
 
+	gpiod_set_value_cansleep(ihid_of->reset_gpio, 1);
 	regulator_bulk_disable(ARRAY_SIZE(ihid_of->supplies),
 			       ihid_of->supplies);
 }
@@ -75,33 +83,43 @@ static int i2c_hid_of_probe(struct i2c_client *client)
 	int ret;
 	u32 val;
 
-	ihid_of = devm_kzalloc(&client->dev, sizeof(*ihid_of), GFP_KERNEL);
+	ihid_of = devm_kzalloc(dev, sizeof(*ihid_of), GFP_KERNEL);
 	if (!ihid_of)
 		return -ENOMEM;
 
 	ihid_of->ops.power_up = i2c_hid_of_power_up;
 	ihid_of->ops.power_down = i2c_hid_of_power_down;
 
-	ret = of_property_read_u32(dev->of_node, "hid-descr-addr", &val);
+	ret = device_property_read_u32(dev, "hid-descr-addr", &val);
 	if (ret) {
-		dev_err(&client->dev, "HID register address not provided\n");
+		dev_err(dev, "HID register address not provided\n");
 		return -ENODEV;
 	}
 	if (val >> 16) {
-		dev_err(&client->dev, "Bad HID register address: 0x%08x\n",
-			val);
+		dev_err(dev, "Bad HID register address: 0x%08x\n", val);
 		return -EINVAL;
 	}
 	hid_descriptor_address = val;
 
-	if (!device_property_read_u32(&client->dev, "post-power-on-delay-ms",
-				      &val))
+	if (!device_property_read_u32(dev, "post-power-on-delay-ms", &val))
 		ihid_of->post_power_delay_ms = val;
 
+	/*
+	 * Note this is a kernel internal device-property set by x86 platform code,
+	 * this MUST not be used in devicetree files without first adding it to
+	 * the DT bindings.
+	 */
+	if (!device_property_read_u32(dev, "post-reset-deassert-delay-ms", &val))
+		ihid_of->post_reset_delay_ms = val;
+
+	/* Start out with reset asserted */
+	ihid_of->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ihid_of->reset_gpio))
+		return PTR_ERR(ihid_of->reset_gpio);
+
 	ihid_of->supplies[0].supply = "vdd";
 	ihid_of->supplies[1].supply = "vddl";
-	ret = devm_regulator_bulk_get(&client->dev,
-				      ARRAY_SIZE(ihid_of->supplies),
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ihid_of->supplies),
 				      ihid_of->supplies);
 	if (ret)
 		return ret;
@@ -116,11 +134,13 @@ static int i2c_hid_of_probe(struct i2c_client *client)
 				  hid_descriptor_address, quirks);
 }
 
+#ifdef CONFIG_OF
 static const struct of_device_id i2c_hid_of_match[] = {
 	{ .compatible = "hid-over-i2c" },
 	{},
 };
 MODULE_DEVICE_TABLE(of, i2c_hid_of_match);
+#endif
 
 static const struct i2c_device_id i2c_hid_of_id_table[] = {
 	{ "hid", 0 },
diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c
index fb538a6c4add..8214896adada 100644
--- a/drivers/hid/wacom_sys.c
+++ b/drivers/hid/wacom_sys.c
@@ -2372,13 +2372,6 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 	if (error)
 		goto fail;
 
-	if (!(features->device_type & WACOM_DEVICETYPE_WL_MONITOR) &&
-	     (features->quirks & WACOM_QUIRK_BATTERY)) {
-		error = wacom_initialize_battery(wacom);
-		if (error)
-			goto fail;
-	}
-
 	error = wacom_register_inputs(wacom);
 	if (error)
 		goto fail;
@@ -2509,9 +2502,6 @@ static void wacom_wireless_work(struct work_struct *work)
 
 		strscpy(wacom_wac->name, wacom_wac1->name,
 			sizeof(wacom_wac->name));
-		error = wacom_initialize_battery(wacom);
-		if (error)
-			goto fail;
 	}
 
 	return;
diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
index 9312d611db8e..dc0f7d9a992c 100644
--- a/drivers/hid/wacom_wac.c
+++ b/drivers/hid/wacom_wac.c
@@ -113,6 +113,11 @@ static void wacom_notify_battery(struct wacom_wac *wacom_wac,
 	bool bat_connected, bool ps_connected)
 {
 	struct wacom *wacom = container_of(wacom_wac, struct wacom, wacom_wac);
+	bool bat_initialized = wacom->battery.battery;
+	bool has_quirk = wacom_wac->features.quirks & WACOM_QUIRK_BATTERY;
+
+	if (bat_initialized != has_quirk)
+		wacom_schedule_work(wacom_wac, WACOM_WORKER_BATTERY);
 
 	__wacom_notify_battery(&wacom->battery, bat_status, bat_capacity,
 			       bat_charging, bat_connected, ps_connected);
@@ -1308,6 +1313,9 @@ static void wacom_intuos_pro2_bt_pen(struct wacom_wac *wacom)
 
 	struct input_dev *pen_input = wacom->pen_input;
 	unsigned char *data = wacom->data;
+	int number_of_valid_frames = 0;
+	int time_interval = 15000000;
+	ktime_t time_packet_received = ktime_get();
 	int i;
 
 	if (wacom->features.type == INTUOSP2_BT ||
@@ -1328,12 +1336,30 @@ static void wacom_intuos_pro2_bt_pen(struct wacom_wac *wacom)
 		wacom->id[0] |= (wacom->serial[0] >> 32) & 0xFFFFF;
 	}
 
+	/* number of valid frames */
 	for (i = 0; i < pen_frames; i++) {
 		unsigned char *frame = &data[i*pen_frame_len + 1];
 		bool valid = frame[0] & 0x80;
+
+		if (valid)
+			number_of_valid_frames++;
+	}
+
+	if (number_of_valid_frames) {
+		if (wacom->hid_data.time_delayed)
+			time_interval = ktime_get() - wacom->hid_data.time_delayed;
+		time_interval /= number_of_valid_frames;
+		wacom->hid_data.time_delayed = time_packet_received;
+	}
+
+	for (i = 0; i < number_of_valid_frames; i++) {
+		unsigned char *frame = &data[i*pen_frame_len + 1];
+		bool valid = frame[0] & 0x80;
 		bool prox = frame[0] & 0x40;
 		bool range = frame[0] & 0x20;
 		bool invert = frame[0] & 0x10;
+		int frames_number_reversed = number_of_valid_frames - i - 1;
+		int event_timestamp = time_packet_received - frames_number_reversed * time_interval;
 
 		if (!valid)
 			continue;
@@ -1346,6 +1372,7 @@ static void wacom_intuos_pro2_bt_pen(struct wacom_wac *wacom)
 			wacom->tool[0] = 0;
 			wacom->id[0] = 0;
 			wacom->serial[0] = 0;
+			wacom->hid_data.time_delayed = 0;
 			return;
 		}
 
@@ -1382,6 +1409,7 @@ static void wacom_intuos_pro2_bt_pen(struct wacom_wac *wacom)
 						 get_unaligned_le16(&frame[11]));
 			}
 		}
+
 		if (wacom->tool[0]) {
 			input_report_abs(pen_input, ABS_PRESSURE, get_unaligned_le16(&frame[5]));
 			if (wacom->features.type == INTUOSP2_BT ||
@@ -1405,6 +1433,9 @@ static void wacom_intuos_pro2_bt_pen(struct wacom_wac *wacom)
 
 		wacom->shared->stylus_in_proximity = prox;
 
+		/* add timestamp to unpack the frames */
+		input_set_timestamp(pen_input, event_timestamp);
+
 		input_sync(pen_input);
 	}
 }
@@ -1895,6 +1926,7 @@ static void wacom_map_usage(struct input_dev *input, struct hid_usage *usage,
 	int fmax = field->logical_maximum;
 	unsigned int equivalent_usage = wacom_equivalent_usage(usage->hid);
 	int resolution_code = code;
+	int resolution = hidinput_calc_abs_res(field, resolution_code);
 
 	if (equivalent_usage == HID_DG_TWIST) {
 		resolution_code = ABS_RZ;
@@ -1915,8 +1947,15 @@ static void wacom_map_usage(struct input_dev *input, struct hid_usage *usage,
 	switch (type) {
 	case EV_ABS:
 		input_set_abs_params(input, code, fmin, fmax, fuzz, 0);
-		input_abs_set_res(input, code,
-				  hidinput_calc_abs_res(field, resolution_code));
+
+		/* older tablet may miss physical usage */
+		if ((code == ABS_X || code == ABS_Y) && !resolution) {
+			resolution = WACOM_INTUOS_RES;
+			hid_warn(input,
+				 "Wacom usage (%d) missing resolution \n",
+				 code);
+		}
+		input_abs_set_res(input, code, resolution);
 		break;
 	case EV_KEY:
 	case EV_MSC:
@@ -1929,18 +1968,7 @@ static void wacom_map_usage(struct input_dev *input, struct hid_usage *usage,
 static void wacom_wac_battery_usage_mapping(struct hid_device *hdev,
 		struct hid_field *field, struct hid_usage *usage)
 {
-	struct wacom *wacom = hid_get_drvdata(hdev);
-	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
-	struct wacom_features *features = &wacom_wac->features;
-	unsigned equivalent_usage = wacom_equivalent_usage(usage->hid);
-
-	switch (equivalent_usage) {
-	case HID_DG_BATTERYSTRENGTH:
-	case WACOM_HID_WD_BATTERY_LEVEL:
-	case WACOM_HID_WD_BATTERY_CHARGING:
-		features->quirks |= WACOM_QUIRK_BATTERY;
-		break;
-	}
+	return;
 }
 
 static void wacom_wac_battery_event(struct hid_device *hdev, struct hid_field *field,
@@ -1961,18 +1989,21 @@ static void wacom_wac_battery_event(struct hid_device *hdev, struct hid_field *f
 			wacom_wac->hid_data.bat_connected = 1;
 			wacom_wac->hid_data.bat_status = WACOM_POWER_SUPPLY_STATUS_AUTO;
 		}
+		wacom_wac->features.quirks |= WACOM_QUIRK_BATTERY;
 		break;
 	case WACOM_HID_WD_BATTERY_LEVEL:
 		value = value * 100 / (field->logical_maximum - field->logical_minimum);
 		wacom_wac->hid_data.battery_capacity = value;
 		wacom_wac->hid_data.bat_connected = 1;
 		wacom_wac->hid_data.bat_status = WACOM_POWER_SUPPLY_STATUS_AUTO;
+		wacom_wac->features.quirks |= WACOM_QUIRK_BATTERY;
 		break;
 	case WACOM_HID_WD_BATTERY_CHARGING:
 		wacom_wac->hid_data.bat_charging = value;
 		wacom_wac->hid_data.ps_connected = value;
 		wacom_wac->hid_data.bat_connected = 1;
 		wacom_wac->hid_data.bat_status = WACOM_POWER_SUPPLY_STATUS_AUTO;
+		wacom_wac->features.quirks |= WACOM_QUIRK_BATTERY;
 		break;
 	}
 }
@@ -1988,18 +2019,15 @@ static void wacom_wac_battery_report(struct hid_device *hdev,
 {
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
-	struct wacom_features *features = &wacom_wac->features;
 
-	if (features->quirks & WACOM_QUIRK_BATTERY) {
-		int status = wacom_wac->hid_data.bat_status;
-		int capacity = wacom_wac->hid_data.battery_capacity;
-		bool charging = wacom_wac->hid_data.bat_charging;
-		bool connected = wacom_wac->hid_data.bat_connected;
-		bool powered = wacom_wac->hid_data.ps_connected;
+	int status = wacom_wac->hid_data.bat_status;
+	int capacity = wacom_wac->hid_data.battery_capacity;
+	bool charging = wacom_wac->hid_data.bat_charging;
+	bool connected = wacom_wac->hid_data.bat_connected;
+	bool powered = wacom_wac->hid_data.ps_connected;
 
-		wacom_notify_battery(wacom_wac, status, capacity, charging,
-				     connected, powered);
-	}
+	wacom_notify_battery(wacom_wac, status, capacity, charging,
+			     connected, powered);
 }
 
 static void wacom_wac_pad_usage_mapping(struct hid_device *hdev,
@@ -3365,19 +3393,13 @@ static int wacom_status_irq(struct wacom_wac *wacom_wac, size_t len)
 		int battery = (data[8] & 0x3f) * 100 / 31;
 		bool charging = !!(data[8] & 0x80);
 
+		features->quirks |= WACOM_QUIRK_BATTERY;
 		wacom_notify_battery(wacom_wac, WACOM_POWER_SUPPLY_STATUS_AUTO,
 				     battery, charging, battery || charging, 1);
-
-		if (!wacom->battery.battery &&
-		    !(features->quirks & WACOM_QUIRK_BATTERY)) {
-			features->quirks |= WACOM_QUIRK_BATTERY;
-			wacom_schedule_work(wacom_wac, WACOM_WORKER_BATTERY);
-		}
 	}
 	else if ((features->quirks & WACOM_QUIRK_BATTERY) &&
 		 wacom->battery.battery) {
 		features->quirks &= ~WACOM_QUIRK_BATTERY;
-		wacom_schedule_work(wacom_wac, WACOM_WORKER_BATTERY);
 		wacom_notify_battery(wacom_wac, POWER_SUPPLY_STATUS_UNKNOWN, 0, 0, 0, 0);
 	}
 	return 0;
diff --git a/drivers/hid/wacom_wac.h b/drivers/hid/wacom_wac.h
index 16f221388563..1a40bb8c5810 100644
--- a/drivers/hid/wacom_wac.h
+++ b/drivers/hid/wacom_wac.h
@@ -324,6 +324,7 @@ struct hid_data {
 	int ps_connected;
 	bool pad_input_event_flag;
 	unsigned short sequence_number;
+	int time_delayed;
 };
 
 struct wacom_remote_data {
diff --git a/drivers/hsi/hsi_core.c b/drivers/hsi/hsi_core.c
index 8fda8f1d064d..acbf82f755a8 100644
--- a/drivers/hsi/hsi_core.c
+++ b/drivers/hsi/hsi_core.c
@@ -207,7 +207,7 @@ static void hsi_add_client_from_dt(struct hsi_port *port,
 	if (!cl)
 		return;
 
-	err = of_modalias_node(client, name, sizeof(name));
+	err = of_alias_from_compatible(client, name, sizeof(name));
 	if (err)
 		goto err;
 
diff --git a/drivers/hte/hte-tegra194-test.c b/drivers/hte/hte-tegra194-test.c
index 5d776a185bd6..358d4a10c6a1 100644
--- a/drivers/hte/hte-tegra194-test.c
+++ b/drivers/hte/hte-tegra194-test.c
@@ -6,14 +6,14 @@
  */
 
 #include <linux/err.h>
-#include <linux/module.h>
-#include <linux/moduleparam.h>
+#include <linux/gpio/consumer.h>
+#include <linux/hte.h>
 #include <linux/interrupt.h>
-#include <linux/gpio.h>
-#include <linux/timer.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/timer.h>
 #include <linux/workqueue.h>
-#include <linux/hte.h>
 
 /*
  * This sample HTE GPIO test driver demonstrates HTE API usage by enabling
diff --git a/drivers/hte/hte.c b/drivers/hte/hte.c
index 7c3b4476f890..9f3221462e75 100644
--- a/drivers/hte/hte.c
+++ b/drivers/hte/hte.c
@@ -518,7 +518,7 @@ static struct hte_device *hte_of_get_dev(struct device *dev,
 
 	np = dev->of_node;
 
-	if (!of_find_property(np, "timestamp-names", NULL)) {
+	if (!of_property_present(np, "timestamp-names")) {
 		/* Let hte core construct it during request time */
 		desc->attr.name = NULL;
 	} else {
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index 309c78ce1d30..1c65a6dfb9fa 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -623,7 +623,7 @@ static const struct attribute_group vmbus_dev_group = {
 __ATTRIBUTE_GROUPS(vmbus_dev);
 
 /* Set up the attribute for /sys/bus/vmbus/hibernation */
-static ssize_t hibernation_show(struct bus_type *bus, char *buf)
+static ssize_t hibernation_show(const struct bus_type *bus, char *buf)
 {
 	return sprintf(buf, "%d\n", !!hv_is_hibernation_supported());
 }
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index 5b3b76477b0e..fc640201a2de 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -1929,6 +1929,16 @@ config SENSORS_STTS751
 	  This driver can also be built as a module. If so, the module
 	  will be called stts751.
 
+config SENSORS_SFCTEMP
+	tristate "Starfive JH71x0 temperature sensor"
+	depends on ARCH_STARFIVE || COMPILE_TEST
+	help
+	  If you say yes here you get support for temperature sensor
+	  on the Starfive JH71x0 SoCs.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called sfctemp.
+
 config SENSORS_SMM665
 	tristate "Summit Microelectronics SMM665"
 	depends on I2C
@@ -1976,7 +1986,7 @@ config SENSORS_ADS7871
 
 config SENSORS_AMC6821
 	tristate "Texas Instruments AMC6821"
-	depends on I2C 
+	depends on I2C
 	help
 	  If you say yes here you get support for the Texas Instruments
 	  AMC6821 hardware monitoring chips.
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 88712b5031c8..cd8c568c80a9 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -6,7 +6,7 @@
 obj-$(CONFIG_HWMON)		+= hwmon.o
 obj-$(CONFIG_HWMON_VID)		+= hwmon-vid.o
 
-# APCI drivers
+# ACPI drivers
 obj-$(CONFIG_SENSORS_ACPI_POWER) += acpi_power_meter.o
 obj-$(CONFIG_SENSORS_ATK0110)	+= asus_atk0110.o
 obj-$(CONFIG_SENSORS_ASUS_EC)	+= asus-ec-sensors.o
@@ -181,6 +181,7 @@ obj-$(CONFIG_SENSORS_SBRMI)	+= sbrmi.o
 obj-$(CONFIG_SENSORS_SCH56XX_COMMON)+= sch56xx-common.o
 obj-$(CONFIG_SENSORS_SCH5627)	+= sch5627.o
 obj-$(CONFIG_SENSORS_SCH5636)	+= sch5636.o
+obj-$(CONFIG_SENSORS_SFCTEMP)	+= sfctemp.o
 obj-$(CONFIG_SENSORS_SL28CPLD)	+= sl28cpld-hwmon.o
 obj-$(CONFIG_SENSORS_SHT15)	+= sht15.o
 obj-$(CONFIG_SENSORS_SHT21)	+= sht21.o
diff --git a/drivers/hwmon/adm1177.c b/drivers/hwmon/adm1177.c
index be17a26a84f1..bfe070a1b501 100644
--- a/drivers/hwmon/adm1177.c
+++ b/drivers/hwmon/adm1177.c
@@ -168,7 +168,7 @@ static umode_t adm1177_is_visible(const void *data,
 	return 0;
 }
 
-static const struct hwmon_channel_info *adm1177_info[] = {
+static const struct hwmon_channel_info * const adm1177_info[] = {
 	HWMON_CHANNEL_INFO(curr,
 			   HWMON_C_INPUT | HWMON_C_MAX_ALARM),
 	HWMON_CHANNEL_INFO(in,
diff --git a/drivers/hwmon/adm9240.c b/drivers/hwmon/adm9240.c
index 40e3558d3709..9eb973a38e4b 100644
--- a/drivers/hwmon/adm9240.c
+++ b/drivers/hwmon/adm9240.c
@@ -731,7 +731,7 @@ static const struct hwmon_ops adm9240_hwmon_ops = {
 	.write = adm9240_write,
 };
 
-static const struct hwmon_channel_info *adm9240_info[] = {
+static const struct hwmon_channel_info * const adm9240_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_ALARMS),
 	HWMON_CHANNEL_INFO(intrusion, HWMON_INTRUSION_ALARM),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/adt7411.c b/drivers/hwmon/adt7411.c
index bf5c5618f8d0..6ba84921614f 100644
--- a/drivers/hwmon/adt7411.c
+++ b/drivers/hwmon/adt7411.c
@@ -636,7 +636,7 @@ static int adt7411_init_device(struct adt7411_data *data)
 	return i2c_smbus_write_byte_data(data->client, ADT7411_REG_CFG1, val);
 }
 
-static const struct hwmon_channel_info *adt7411_info[] = {
+static const struct hwmon_channel_info * const adt7411_info[] = {
 	HWMON_CHANNEL_INFO(in,
 			   HWMON_I_INPUT | HWMON_I_MIN | HWMON_I_MAX | HWMON_I_ALARM,
 			   HWMON_I_INPUT | HWMON_I_MIN | HWMON_I_MAX | HWMON_I_ALARM,
diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c
index 927f8df05b7c..64f801b859ff 100644
--- a/drivers/hwmon/adt7470.c
+++ b/drivers/hwmon/adt7470.c
@@ -1187,7 +1187,7 @@ static const struct hwmon_ops adt7470_hwmon_ops = {
 	.write = adt7470_write,
 };
 
-static const struct hwmon_channel_info *adt7470_info[] = {
+static const struct hwmon_channel_info * const adt7470_info[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | HWMON_T_ALARM,
 			   HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | HWMON_T_ALARM,
diff --git a/drivers/hwmon/adt7475.c b/drivers/hwmon/adt7475.c
index 6e4c92b500b8..6a6ebcc896b1 100644
--- a/drivers/hwmon/adt7475.c
+++ b/drivers/hwmon/adt7475.c
@@ -1604,9 +1604,9 @@ static int adt7475_set_pwm_polarity(struct i2c_client *client)
 	int ret, i;
 	u8 val;
 
-	ret = of_property_read_u32_array(client->dev.of_node,
-					 "adi,pwm-active-state", states,
-					 ARRAY_SIZE(states));
+	ret = device_property_read_u32_array(&client->dev,
+					     "adi,pwm-active-state", states,
+					     ARRAY_SIZE(states));
 	if (ret)
 		return ret;
 
diff --git a/drivers/hwmon/adt7x10.c b/drivers/hwmon/adt7x10.c
index da67734edafd..6701920de17f 100644
--- a/drivers/hwmon/adt7x10.c
+++ b/drivers/hwmon/adt7x10.c
@@ -309,7 +309,7 @@ static int adt7x10_write(struct device *dev, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *adt7x10_info[] = {
+static const struct hwmon_channel_info * const adt7x10_info[] = {
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MIN |
 			   HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_MIN_HYST |
 			   HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM |
diff --git a/drivers/hwmon/aht10.c b/drivers/hwmon/aht10.c
index 9babd69d54a3..b8fe3f7248ba 100644
--- a/drivers/hwmon/aht10.c
+++ b/drivers/hwmon/aht10.c
@@ -270,7 +270,7 @@ static int aht10_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *aht10_info[] = {
+static const struct hwmon_channel_info * const aht10_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
 	HWMON_CHANNEL_INFO(humidity, HWMON_H_INPUT),
diff --git a/drivers/hwmon/aquacomputer_d5next.c b/drivers/hwmon/aquacomputer_d5next.c
index 12682a610ce7..a4fcd4ebf76c 100644
--- a/drivers/hwmon/aquacomputer_d5next.c
+++ b/drivers/hwmon/aquacomputer_d5next.c
@@ -29,12 +29,14 @@
 #define USB_PRODUCT_ID_FARBWERK360	0xf010
 #define USB_PRODUCT_ID_OCTO		0xf011
 #define USB_PRODUCT_ID_HIGHFLOWNEXT	0xf012
+#define USB_PRODUCT_ID_AQUASTREAMXT	0xf0b6
 #define USB_PRODUCT_ID_AQUASTREAMULT	0xf00b
 #define USB_PRODUCT_ID_POWERADJUST3	0xf0bd
 
 enum kinds {
 	d5next, farbwerk, farbwerk360, octo, quadro,
-	highflownext, aquaero, poweradjust3, aquastreamult
+	highflownext, aquaero, poweradjust3, aquastreamult,
+	aquastreamxt
 };
 
 static const char *const aqc_device_names[] = {
@@ -44,6 +46,7 @@ static const char *const aqc_device_names[] = {
 	[octo] = "octo",
 	[quadro] = "quadro",
 	[highflownext] = "highflownext",
+	[aquastreamxt] = "aquastreamxt",
 	[aquaero] = "aquaero",
 	[aquastreamult] = "aquastreamultimate",
 	[poweradjust3] = "poweradjust3"
@@ -56,6 +59,7 @@ static const char *const aqc_device_names[] = {
 #define SERIAL_PART_OFFSET		2
 
 #define CTRL_REPORT_ID			0x03
+#define AQUAERO_CTRL_REPORT_ID		0x0b
 
 /* The HID report that the official software always sends
  * after writing values, currently same for all devices
@@ -67,9 +71,23 @@ static u8 secondary_ctrl_report[] = {
 	0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x34, 0xC6
 };
 
+/* Secondary HID report values for Aquaero */
+#define AQUAERO_SECONDARY_CTRL_REPORT_ID	0x06
+#define AQUAERO_SECONDARY_CTRL_REPORT_SIZE	0x07
+
+static u8 aquaero_secondary_ctrl_report[] = {
+	0x06, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00
+};
+
 /* Report IDs for legacy devices */
+#define AQUASTREAMXT_STATUS_REPORT_ID	0x04
+
 #define POWERADJUST3_STATUS_REPORT_ID	0x03
 
+/* Data types for reading and writing control reports */
+#define AQC_8		0
+#define AQC_BE16	1
+
 /* Info, sensor sizes and offsets for most Aquacomputer devices */
 #define AQC_SERIAL_START		0x3
 #define AQC_FIRMWARE_VERSION		0xD
@@ -90,6 +108,10 @@ static u8 secondary_ctrl_report[] = {
 #define AQUAERO_NUM_VIRTUAL_SENSORS		8
 #define AQUAERO_NUM_CALC_VIRTUAL_SENSORS	4
 #define AQUAERO_NUM_FLOW_SENSORS		2
+#define AQUAERO_CTRL_REPORT_SIZE		0xa93
+#define AQUAERO_CTRL_PRESET_ID			0x5c
+#define AQUAERO_CTRL_PRESET_SIZE		0x02
+#define AQUAERO_CTRL_PRESET_START		0x55c
 
 /* Sensor report offsets for Aquaero fan controllers */
 #define AQUAERO_SENSOR_START			0x65
@@ -102,6 +124,13 @@ static u8 secondary_ctrl_report[] = {
 #define AQUAERO_FAN_SPEED_OFFSET		0x00
 static u16 aquaero_sensor_fan_offsets[] = { 0x167, 0x173, 0x17f, 0x18B };
 
+/* Control report offsets for the Aquaero fan controllers */
+#define AQUAERO_TEMP_CTRL_OFFSET	0xdb
+#define AQUAERO_FAN_CTRL_MIN_PWR_OFFSET	0x04
+#define AQUAERO_FAN_CTRL_MAX_PWR_OFFSET	0x06
+#define AQUAERO_FAN_CTRL_SRC_OFFSET	0x10
+static u16 aquaero_ctrl_fan_offsets[] = { 0x20c, 0x220, 0x234, 0x248 };
+
 /* Specs of the D5 Next pump */
 #define D5NEXT_NUM_FANS			2
 #define D5NEXT_NUM_SENSORS		1
@@ -207,6 +236,24 @@ static u16 quadro_ctrl_fan_offsets[] = { 0x37, 0x8c, 0xe1, 0x136 }; /* Fan speed
 #define HIGHFLOWNEXT_5V_VOLTAGE		97
 #define HIGHFLOWNEXT_5V_VOLTAGE_USB	99
 
+/* Specs of the Aquastream XT pump */
+#define AQUASTREAMXT_SERIAL_START		0x3a
+#define AQUASTREAMXT_FIRMWARE_VERSION		0x32
+#define AQUASTREAMXT_NUM_FANS			2
+#define AQUASTREAMXT_NUM_SENSORS		3
+#define AQUASTREAMXT_FAN_STOPPED		0x4
+#define AQUASTREAMXT_PUMP_CONVERSION_CONST	45000000
+#define AQUASTREAMXT_FAN_CONVERSION_CONST	5646000
+#define AQUASTREAMXT_SENSOR_REPORT_SIZE		0x42
+
+/* Sensor report offsets and info for Aquastream XT */
+#define AQUASTREAMXT_SENSOR_START		0xd
+#define AQUASTREAMXT_FAN_VOLTAGE_OFFSET		0x7
+#define AQUASTREAMXT_FAN_STATUS_OFFSET		0x1d
+#define AQUASTREAMXT_PUMP_VOLTAGE_OFFSET	0x9
+#define AQUASTREAMXT_PUMP_CURR_OFFSET		0xb
+static u16 aquastreamxt_sensor_fan_offsets[] = { 0x13, 0x1b };
+
 /* Specs of the Poweradjust 3 */
 #define POWERADJUST3_NUM_SENSORS	1
 #define POWERADJUST3_SENSOR_REPORT_SIZE	0x32
@@ -364,6 +411,13 @@ static const char *const label_highflownext_voltage[] = {
 	"+5V USB voltage"
 };
 
+/* Labels for Aquastream XT */
+static const char *const label_aquastreamxt_temp_sensors[] = {
+	"Fan IC temp",
+	"External sensor",
+	"Coolant temp"
+};
+
 /* Labels for Aquastream Ultimate */
 static const char *const label_aquastreamult_temp[] = {
 	"Coolant temp",
@@ -437,6 +491,10 @@ struct aqc_data {
 	const char *name;
 
 	int status_report_id;	/* Used for legacy devices, report is stored in buffer */
+	int ctrl_report_id;
+	int secondary_ctrl_report_id;
+	int secondary_ctrl_report_size;
+	u8 *secondary_ctrl_report;
 
 	int buffer_size;
 	u8 *buffer;
@@ -503,13 +561,29 @@ static int aqc_pwm_to_percent(long val)
 	return DIV_ROUND_CLOSEST(val * 100 * 100, 255);
 }
 
+/* Converts raw value for Aquastream XT pump speed to RPM */
+static int aqc_aquastreamxt_convert_pump_rpm(u16 val)
+{
+	if (val > 0)
+		return DIV_ROUND_CLOSEST(AQUASTREAMXT_PUMP_CONVERSION_CONST, val);
+	return 0;
+}
+
+/* Converts raw value for Aquastream XT fan speed to RPM */
+static int aqc_aquastreamxt_convert_fan_rpm(u16 val)
+{
+	if (val > 0)
+		return DIV_ROUND_CLOSEST(AQUASTREAMXT_FAN_CONVERSION_CONST, val);
+	return 0;
+}
+
 /* Expects the mutex to be locked */
 static int aqc_get_ctrl_data(struct aqc_data *priv)
 {
 	int ret;
 
 	memset(priv->buffer, 0x00, priv->buffer_size);
-	ret = hid_hw_raw_request(priv->hdev, CTRL_REPORT_ID, priv->buffer, priv->buffer_size,
+	ret = hid_hw_raw_request(priv->hdev, priv->ctrl_report_id, priv->buffer, priv->buffer_size,
 				 HID_FEATURE_REPORT, HID_REQ_GET_REPORT);
 	if (ret < 0)
 		ret = -ENODATA;
@@ -523,28 +597,32 @@ static int aqc_send_ctrl_data(struct aqc_data *priv)
 	int ret;
 	u16 checksum;
 
-	/* Init and xorout value for CRC-16/USB is 0xffff */
-	checksum = crc16(0xffff, priv->buffer + priv->checksum_start, priv->checksum_length);
-	checksum ^= 0xffff;
+	/* Checksum is not needed for Aquaero */
+	if (priv->kind != aquaero) {
+		/* Init and xorout value for CRC-16/USB is 0xffff */
+		checksum = crc16(0xffff, priv->buffer + priv->checksum_start,
+				 priv->checksum_length);
+		checksum ^= 0xffff;
 
-	/* Place the new checksum at the end of the report */
-	put_unaligned_be16(checksum, priv->buffer + priv->checksum_offset);
+		/* Place the new checksum at the end of the report */
+		put_unaligned_be16(checksum, priv->buffer + priv->checksum_offset);
+	}
 
 	/* Send the patched up report back to the device */
-	ret = hid_hw_raw_request(priv->hdev, CTRL_REPORT_ID, priv->buffer, priv->buffer_size,
+	ret = hid_hw_raw_request(priv->hdev, priv->ctrl_report_id, priv->buffer, priv->buffer_size,
 				 HID_FEATURE_REPORT, HID_REQ_SET_REPORT);
 	if (ret < 0)
 		return ret;
 
 	/* The official software sends this report after every change, so do it here as well */
-	ret = hid_hw_raw_request(priv->hdev, SECONDARY_CTRL_REPORT_ID, secondary_ctrl_report,
-				 SECONDARY_CTRL_REPORT_SIZE, HID_FEATURE_REPORT,
-				 HID_REQ_SET_REPORT);
+	ret = hid_hw_raw_request(priv->hdev, priv->secondary_ctrl_report_id,
+				 priv->secondary_ctrl_report, priv->secondary_ctrl_report_size,
+				 HID_FEATURE_REPORT, HID_REQ_SET_REPORT);
 	return ret;
 }
 
 /* Refreshes the control buffer and stores value at offset in val */
-static int aqc_get_ctrl_val(struct aqc_data *priv, int offset, long *val)
+static int aqc_get_ctrl_val(struct aqc_data *priv, int offset, long *val, int type)
 {
 	int ret;
 
@@ -554,16 +632,25 @@ static int aqc_get_ctrl_val(struct aqc_data *priv, int offset, long *val)
 	if (ret < 0)
 		goto unlock_and_return;
 
-	*val = (s16)get_unaligned_be16(priv->buffer + offset);
+	switch (type) {
+	case AQC_BE16:
+		*val = (s16)get_unaligned_be16(priv->buffer + offset);
+		break;
+	case AQC_8:
+		*val = priv->buffer[offset];
+		break;
+	default:
+		ret = -EINVAL;
+	}
 
 unlock_and_return:
 	mutex_unlock(&priv->mutex);
 	return ret;
 }
 
-static int aqc_set_ctrl_val(struct aqc_data *priv, int offset, long val)
+static int aqc_set_ctrl_vals(struct aqc_data *priv, int *offsets, long *vals, int *types, int len)
 {
-	int ret;
+	int ret, i;
 
 	mutex_lock(&priv->mutex);
 
@@ -571,7 +658,21 @@ static int aqc_set_ctrl_val(struct aqc_data *priv, int offset, long val)
 	if (ret < 0)
 		goto unlock_and_return;
 
-	put_unaligned_be16((s16)val, priv->buffer + offset);
+	for (i = 0; i < len; i++) {
+		switch (types[i]) {
+		case AQC_BE16:
+			put_unaligned_be16((s16)vals[i], priv->buffer + offsets[i]);
+			break;
+		case AQC_8:
+			priv->buffer[offsets[i]] = (u8)vals[i];
+			break;
+		default:
+			ret = -EINVAL;
+		}
+	}
+
+	if (ret < 0)
+		goto unlock_and_return;
 
 	ret = aqc_send_ctrl_data(priv);
 
@@ -580,6 +681,11 @@ unlock_and_return:
 	return ret;
 }
 
+static int aqc_set_ctrl_val(struct aqc_data *priv, int offset, long val, int type)
+{
+	return aqc_set_ctrl_vals(priv, &offset, &val, &type, 1);
+}
+
 static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u32 attr, int channel)
 {
 	const struct aqc_data *priv = data;
@@ -674,6 +780,8 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3
 			if (channel == 0)
 				return 0444;
 			break;
+		case aquastreamxt:
+			break;
 		default:
 			if (channel < priv->num_fans)
 				return 0444;
@@ -687,6 +795,11 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3
 			if (channel < 2)
 				return 0444;
 			break;
+		case aquastreamxt:
+			/* Special case to support pump current */
+			if (channel == 0)
+				return 0444;
+			break;
 		default:
 			if (channel < priv->num_fans)
 				return 0444;
@@ -739,6 +852,43 @@ static int aqc_legacy_read(struct aqc_data *priv)
 		priv->temp_input[i] = sensor_value * 10;
 	}
 
+	/* Special-case sensor readings */
+	switch (priv->kind) {
+	case aquastreamxt:
+		/* Info provided with every report */
+		priv->serial_number[0] = get_unaligned_le16(priv->buffer +
+							    priv->serial_number_start_offset);
+		priv->firmware_version =
+			get_unaligned_le16(priv->buffer + priv->firmware_version_offset);
+
+		/* Read pump speed in RPM */
+		sensor_value = get_unaligned_le16(priv->buffer + priv->fan_sensor_offsets[0]);
+		priv->speed_input[0] = aqc_aquastreamxt_convert_pump_rpm(sensor_value);
+
+		/* Read fan speed in RPM, if available */
+		sensor_value = get_unaligned_le16(priv->buffer + AQUASTREAMXT_FAN_STATUS_OFFSET);
+		if (sensor_value == AQUASTREAMXT_FAN_STOPPED) {
+			priv->speed_input[1] = 0;
+		} else {
+			sensor_value =
+				get_unaligned_le16(priv->buffer + priv->fan_sensor_offsets[1]);
+			priv->speed_input[1] = aqc_aquastreamxt_convert_fan_rpm(sensor_value);
+		}
+
+		/* Calculation derived from linear regression */
+		sensor_value = get_unaligned_le16(priv->buffer + AQUASTREAMXT_PUMP_CURR_OFFSET);
+		priv->current_input[0] = DIV_ROUND_CLOSEST(sensor_value * 176, 100) - 52;
+
+		sensor_value = get_unaligned_le16(priv->buffer + AQUASTREAMXT_PUMP_VOLTAGE_OFFSET);
+		priv->voltage_input[0] = DIV_ROUND_CLOSEST(sensor_value * 1000, 61);
+
+		sensor_value = get_unaligned_le16(priv->buffer + AQUASTREAMXT_FAN_VOLTAGE_OFFSET);
+		priv->voltage_input[1] = DIV_ROUND_CLOSEST(sensor_value * 1000, 63);
+		break;
+	default:
+		break;
+	}
+
 	priv->updated = jiffies;
 
 unlock_and_return:
@@ -775,7 +925,7 @@ static int aqc_read(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 		case hwmon_temp_offset:
 			ret =
 			    aqc_get_ctrl_val(priv, priv->temp_ctrl_offset +
-					     channel * AQC_SENSOR_SIZE, val);
+					     channel * AQC_SENSOR_SIZE, val, AQC_BE16);
 			if (ret < 0)
 				return ret;
 
@@ -791,7 +941,8 @@ static int aqc_read(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 			*val = priv->speed_input[channel];
 			break;
 		case hwmon_fan_pulses:
-			ret = aqc_get_ctrl_val(priv, priv->flow_pulses_ctrl_offset, val);
+			ret = aqc_get_ctrl_val(priv, priv->flow_pulses_ctrl_offset,
+					       val, AQC_BE16);
 			if (ret < 0)
 				return ret;
 			break;
@@ -803,12 +954,23 @@ static int aqc_read(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 		*val = priv->power_input[channel];
 		break;
 	case hwmon_pwm:
-		if (priv->fan_ctrl_offsets) {
-			ret = aqc_get_ctrl_val(priv, priv->fan_ctrl_offsets[channel], val);
+		switch (priv->kind) {
+		case aquaero:
+			ret = aqc_get_ctrl_val(priv,
+				AQUAERO_CTRL_PRESET_START + channel * AQUAERO_CTRL_PRESET_SIZE,
+				val, AQC_BE16);
+			if (ret < 0)
+				return ret;
+			*val = aqc_percent_to_pwm(*val);
+			break;
+		default:
+			ret = aqc_get_ctrl_val(priv, priv->fan_ctrl_offsets[channel],
+					       val, AQC_BE16);
 			if (ret < 0)
 				return ret;
 
 			*val = aqc_percent_to_pwm(ret);
+			break;
 		}
 		break;
 	case hwmon_in:
@@ -867,6 +1029,10 @@ static int aqc_write(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 		     long val)
 {
 	int ret, pwm_value;
+	/* Arrays for setting multiple values at once in the control report */
+	int ctrl_values_offsets[4];
+	long ctrl_values[4];
+	int ctrl_values_types[4];
 	struct aqc_data *priv = dev_get_drvdata(dev);
 
 	switch (type) {
@@ -877,7 +1043,7 @@ static int aqc_write(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 			val = clamp_val(val, -15000, 15000) / 10;
 			ret =
 			    aqc_set_ctrl_val(priv, priv->temp_ctrl_offset +
-					     channel * AQC_SENSOR_SIZE, val);
+					     channel * AQC_SENSOR_SIZE, val, AQC_BE16);
 			if (ret < 0)
 				return ret;
 			break;
@@ -889,7 +1055,8 @@ static int aqc_write(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 		switch (attr) {
 		case hwmon_fan_pulses:
 			val = clamp_val(val, 10, 1000);
-			ret = aqc_set_ctrl_val(priv, priv->flow_pulses_ctrl_offset, val);
+			ret = aqc_set_ctrl_val(priv, priv->flow_pulses_ctrl_offset,
+					       val, AQC_BE16);
 			if (ret < 0)
 				return ret;
 			break;
@@ -900,15 +1067,47 @@ static int aqc_write(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 	case hwmon_pwm:
 		switch (attr) {
 		case hwmon_pwm_input:
-			if (priv->fan_ctrl_offsets) {
-				pwm_value = aqc_pwm_to_percent(val);
-				if (pwm_value < 0)
-					return pwm_value;
+			pwm_value = aqc_pwm_to_percent(val);
+			if (pwm_value < 0)
+				return pwm_value;
 
+			switch (priv->kind) {
+			case aquaero:
+				/* Write pwm value to preset corresponding to the channel */
+				ctrl_values_offsets[0] = AQUAERO_CTRL_PRESET_START +
+				    channel * AQUAERO_CTRL_PRESET_SIZE;
+				ctrl_values[0] = pwm_value;
+				ctrl_values_types[0] = AQC_BE16;
+
+				/* Write preset number in fan control source */
+				ctrl_values_offsets[1] = priv->fan_ctrl_offsets[channel] +
+				    AQUAERO_FAN_CTRL_SRC_OFFSET;
+				ctrl_values[1] = AQUAERO_CTRL_PRESET_ID + channel;
+				ctrl_values_types[1] = AQC_BE16;
+
+				/* Set minimum power to 0 to allow the fan to turn off */
+				ctrl_values_offsets[2] = priv->fan_ctrl_offsets[channel] +
+				    AQUAERO_FAN_CTRL_MIN_PWR_OFFSET;
+				ctrl_values[2] = 0;
+				ctrl_values_types[2] = AQC_BE16;
+
+				/* Set maximum power to 255 to allow the fan to reach max speed */
+				ctrl_values_offsets[3] = priv->fan_ctrl_offsets[channel] +
+				    AQUAERO_FAN_CTRL_MAX_PWR_OFFSET;
+				ctrl_values[3] = aqc_pwm_to_percent(255);
+				ctrl_values_types[3] = AQC_BE16;
+
+				ret = aqc_set_ctrl_vals(priv, ctrl_values_offsets, ctrl_values,
+							ctrl_values_types, 4);
+				if (ret < 0)
+					return ret;
+				break;
+			default:
 				ret = aqc_set_ctrl_val(priv, priv->fan_ctrl_offsets[channel],
-						       pwm_value);
+						       pwm_value, AQC_BE16);
 				if (ret < 0)
 					return ret;
+				break;
 			}
 			break;
 		default:
@@ -929,16 +1128,16 @@ static const struct hwmon_ops aqc_hwmon_ops = {
 	.write = aqc_write
 };
 
-static const struct hwmon_channel_info *aqc_info[] = {
+static const struct hwmon_channel_info * const aqc_info[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_OFFSET,
 			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_OFFSET,
 			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_OFFSET,
 			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_OFFSET,
-			   HWMON_T_INPUT | HWMON_T_LABEL,
-			   HWMON_T_INPUT | HWMON_T_LABEL,
-			   HWMON_T_INPUT | HWMON_T_LABEL,
-			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_OFFSET,
+			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_OFFSET,
+			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_OFFSET,
+			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_OFFSET,
 			   HWMON_T_INPUT | HWMON_T_LABEL,
 			   HWMON_T_INPUT | HWMON_T_LABEL,
 			   HWMON_T_INPUT | HWMON_T_LABEL,
@@ -1231,6 +1430,7 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
 
 		priv->num_fans = AQUAERO_NUM_FANS;
 		priv->fan_sensor_offsets = aquaero_sensor_fan_offsets;
+		priv->fan_ctrl_offsets = aquaero_ctrl_fan_offsets;
 
 		priv->num_temp_sensors = AQUAERO_NUM_SENSORS;
 		priv->temp_sensor_start_offset = AQUAERO_SENSOR_START;
@@ -1241,6 +1441,9 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		priv->num_flow_sensors = AQUAERO_NUM_FLOW_SENSORS;
 		priv->flow_sensors_start_offset = AQUAERO_FLOW_SENSORS_START;
 
+		priv->buffer_size = AQUAERO_CTRL_REPORT_SIZE;
+		priv->temp_ctrl_offset = AQUAERO_TEMP_CTRL_OFFSET;
+
 		priv->temp_label = label_temp_sensors;
 		priv->virtual_temp_label = label_virtual_temp_sensors;
 		priv->calc_virt_temp_label = label_aquaero_calc_temp_sensors;
@@ -1368,6 +1571,21 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		priv->power_label = label_highflownext_power;
 		priv->voltage_label = label_highflownext_voltage;
 		break;
+	case USB_PRODUCT_ID_AQUASTREAMXT:
+		priv->kind = aquastreamxt;
+
+		priv->num_fans = AQUASTREAMXT_NUM_FANS;
+		priv->fan_sensor_offsets = aquastreamxt_sensor_fan_offsets;
+
+		priv->num_temp_sensors = AQUASTREAMXT_NUM_SENSORS;
+		priv->temp_sensor_start_offset = AQUASTREAMXT_SENSOR_START;
+		priv->buffer_size = AQUASTREAMXT_SENSOR_REPORT_SIZE;
+
+		priv->temp_label = label_aquastreamxt_temp_sensors;
+		priv->speed_label = label_d5next_speeds;
+		priv->voltage_label = label_d5next_voltages;
+		priv->current_label = label_d5next_current;
+		break;
 	case USB_PRODUCT_ID_AQUASTREAMULT:
 		priv->kind = aquastreamult;
 
@@ -1404,14 +1622,30 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		priv->firmware_version_offset = AQUAERO_FIRMWARE_VERSION;
 
 		priv->fan_structure = &aqc_aquaero_fan_structure;
+
+		priv->ctrl_report_id = AQUAERO_CTRL_REPORT_ID;
+		priv->secondary_ctrl_report_id = AQUAERO_SECONDARY_CTRL_REPORT_ID;
+		priv->secondary_ctrl_report_size = AQUAERO_SECONDARY_CTRL_REPORT_SIZE;
+		priv->secondary_ctrl_report = aquaero_secondary_ctrl_report;
 		break;
 	case poweradjust3:
 		priv->status_report_id = POWERADJUST3_STATUS_REPORT_ID;
 		break;
+	case aquastreamxt:
+		priv->serial_number_start_offset = AQUASTREAMXT_SERIAL_START;
+		priv->firmware_version_offset = AQUASTREAMXT_FIRMWARE_VERSION;
+
+		priv->status_report_id = AQUASTREAMXT_STATUS_REPORT_ID;
+		break;
 	default:
 		priv->serial_number_start_offset = AQC_SERIAL_START;
 		priv->firmware_version_offset = AQC_FIRMWARE_VERSION;
 
+		priv->ctrl_report_id = CTRL_REPORT_ID;
+		priv->secondary_ctrl_report_id = SECONDARY_CTRL_REPORT_ID;
+		priv->secondary_ctrl_report_size = SECONDARY_CTRL_REPORT_SIZE;
+		priv->secondary_ctrl_report = secondary_ctrl_report;
+
 		if (priv->kind == aquastreamult)
 			priv->fan_structure = &aqc_aquastreamult_fan_structure;
 		else
@@ -1473,6 +1707,7 @@ static const struct hid_device_id aqc_table[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_OCTO) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_QUADRO) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_HIGHFLOWNEXT) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_AQUASTREAMXT) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_AQUASTREAMULT) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_POWERADJUST3) },
 	{ }
diff --git a/drivers/hwmon/as370-hwmon.c b/drivers/hwmon/as370-hwmon.c
index 63b5b2d6e593..fffbf385a57f 100644
--- a/drivers/hwmon/as370-hwmon.c
+++ b/drivers/hwmon/as370-hwmon.c
@@ -76,7 +76,7 @@ as370_hwmon_is_visible(const void *data, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *as370_hwmon_info[] = {
+static const struct hwmon_channel_info * const as370_hwmon_info[] = {
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
 	NULL
 };
diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c
index 2768b7511684..e5be0cf472fc 100644
--- a/drivers/hwmon/asus-ec-sensors.c
+++ b/drivers/hwmon/asus-ec-sensors.c
@@ -303,6 +303,14 @@ static const struct ec_board_info board_info_pro_art_x570_creator_wifi = {
 	.family = family_amd_500_series,
 };
 
+static const struct ec_board_info board_info_pro_art_b550_creator = {
+	.sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB |
+		SENSOR_TEMP_T_SENSOR |
+		SENSOR_FAN_CPU_OPT,
+	.mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX,
+	.family = family_amd_500_series,
+};
+
 static const struct ec_board_info board_info_pro_ws_x570_ace = {
 	.sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM |
 		SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET |
@@ -400,6 +408,14 @@ static const struct ec_board_info board_info_strix_x570_i_gaming = {
 	.family = family_amd_500_series,
 };
 
+static const struct ec_board_info board_info_strix_z390_f_gaming = {
+	.sensors = SENSOR_TEMP_CHIPSET | SENSOR_TEMP_VRM |
+		SENSOR_TEMP_T_SENSOR |
+		SENSOR_FAN_CPU_OPT,
+	.mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX,
+	.family = family_intel_300_series,
+};
+
 static const struct ec_board_info board_info_strix_z690_a_gaming_wifi_d4 = {
 	.sensors = SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM,
 	.mutex_path = ASUS_HW_ACCESS_MUTEX_RMTW_ASMX,
@@ -435,6 +451,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_prime_x570_pro),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ProArt X570-CREATOR WIFI",
 					&board_info_pro_art_x570_creator_wifi),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ProArt B550-CREATOR",
+					&board_info_pro_art_b550_creator),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS X570-ACE",
 					&board_info_pro_ws_x570_ace),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII DARK HERO",
@@ -463,6 +481,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_strix_x570_f_gaming),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X570-I GAMING",
 					&board_info_strix_x570_i_gaming),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z390-F GAMING",
+					&board_info_strix_z390_f_gaming),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z690-A GAMING WIFI D4",
 					&board_info_strix_z690_a_gaming_wifi_d4),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG ZENITH II EXTREME",
diff --git a/drivers/hwmon/axi-fan-control.c b/drivers/hwmon/axi-fan-control.c
index 6724e0dd3088..5fd136baf1cd 100644
--- a/drivers/hwmon/axi-fan-control.c
+++ b/drivers/hwmon/axi-fan-control.c
@@ -394,7 +394,7 @@ static int axi_fan_control_init(struct axi_fan_control_data *ctl,
 	return ret;
 }
 
-static const struct hwmon_channel_info *axi_fan_control_info[] = {
+static const struct hwmon_channel_info * const axi_fan_control_info[] = {
 	HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT),
 	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_FAULT | HWMON_F_LABEL),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_LABEL),
diff --git a/drivers/hwmon/bt1-pvt.c b/drivers/hwmon/bt1-pvt.c
index 21ab172774ec..8d402a627306 100644
--- a/drivers/hwmon/bt1-pvt.c
+++ b/drivers/hwmon/bt1-pvt.c
@@ -379,7 +379,7 @@ static int pvt_read_alarm(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
 	return 0;
 }
 
-static const struct hwmon_channel_info *pvt_channel_info[] = {
+static const struct hwmon_channel_info * const pvt_channel_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp,
@@ -523,7 +523,7 @@ static int pvt_read_alarm(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
 	return -EOPNOTSUPP;
 }
 
-static const struct hwmon_channel_info *pvt_channel_info[] = {
+static const struct hwmon_channel_info * const pvt_channel_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c
index 30d77f451937..eba94f68585a 100644
--- a/drivers/hwmon/coretemp.c
+++ b/drivers/hwmon/coretemp.c
@@ -282,14 +282,8 @@ static int get_tjmax(struct temp_data *tdata, struct device *dev)
 			dev_warn(dev, "Unable to read TjMax from CPU %u\n", tdata->cpu);
 	} else {
 		val = (eax >> 16) & 0xff;
-		/*
-		 * If the TjMax is not plausible, an assumption
-		 * will be used
-		 */
-		if (val) {
-			dev_dbg(dev, "TjMax is %d degrees C\n", val);
+		if (val)
 			return val * 1000;
-		}
 	}
 
 	if (force_tjmax) {
diff --git a/drivers/hwmon/corsair-cpro.c b/drivers/hwmon/corsair-cpro.c
index fa6aa4fc8b52..463ab4296ede 100644
--- a/drivers/hwmon/corsair-cpro.c
+++ b/drivers/hwmon/corsair-cpro.c
@@ -385,7 +385,7 @@ static const struct hwmon_ops ccp_hwmon_ops = {
 	.write = ccp_write,
 };
 
-static const struct hwmon_channel_info *ccp_info[] = {
+static const struct hwmon_channel_info * const ccp_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/corsair-psu.c b/drivers/hwmon/corsair-psu.c
index 2210aa62e3d0..dc24c566d08b 100644
--- a/drivers/hwmon/corsair-psu.c
+++ b/drivers/hwmon/corsair-psu.c
@@ -571,7 +571,7 @@ static const struct hwmon_ops corsairpsu_hwmon_ops = {
 	.read_string	= corsairpsu_hwmon_ops_read_string,
 };
 
-static const struct hwmon_channel_info *corsairpsu_info[] = {
+static const struct hwmon_channel_info * const corsairpsu_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c
index 7ac778aedc68..44aaf9b9191d 100644
--- a/drivers/hwmon/dell-smm-hwmon.c
+++ b/drivers/hwmon/dell-smm-hwmon.c
@@ -920,7 +920,7 @@ static const struct hwmon_ops dell_smm_ops = {
 	.write = dell_smm_write,
 };
 
-static const struct hwmon_channel_info *dell_smm_info[] = {
+static const struct hwmon_channel_info * const dell_smm_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_LABEL,
diff --git a/drivers/hwmon/drivetemp.c b/drivers/hwmon/drivetemp.c
index 8e5759b42390..6bdd21aa005a 100644
--- a/drivers/hwmon/drivetemp.c
+++ b/drivers/hwmon/drivetemp.c
@@ -526,7 +526,7 @@ static umode_t drivetemp_is_visible(const void *data,
 	return 0;
 }
 
-static const struct hwmon_channel_info *drivetemp_info[] = {
+static const struct hwmon_channel_info * const drivetemp_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT |
@@ -550,7 +550,7 @@ static const struct hwmon_chip_info drivetemp_chip_info = {
  * The device argument points to sdev->sdev_dev. Its parent is
  * sdev->sdev_gendev, which we can use to get the scsi_device pointer.
  */
-static int drivetemp_add(struct device *dev, struct class_interface *intf)
+static int drivetemp_add(struct device *dev)
 {
 	struct scsi_device *sdev = to_scsi_device(dev->parent);
 	struct drivetemp_data *st;
@@ -585,7 +585,7 @@ abort:
 	return err;
 }
 
-static void drivetemp_remove(struct device *dev, struct class_interface *intf)
+static void drivetemp_remove(struct device *dev)
 {
 	struct drivetemp_data *st, *tmp;
 
diff --git a/drivers/hwmon/emc2305.c b/drivers/hwmon/emc2305.c
index f65467fbd86c..723f57518c9a 100644
--- a/drivers/hwmon/emc2305.c
+++ b/drivers/hwmon/emc2305.c
@@ -479,7 +479,7 @@ static const struct hwmon_ops emc2305_ops = {
 	.write = emc2305_write,
 };
 
-static const struct hwmon_channel_info *emc2305_info[] = {
+static const struct hwmon_channel_info * const emc2305_info[] = {
 	HWMON_CHANNEL_INFO(fan,
 			   HWMON_F_INPUT | HWMON_F_FAULT,
 			   HWMON_F_INPUT | HWMON_F_FAULT,
diff --git a/drivers/hwmon/ftsteutates.c b/drivers/hwmon/ftsteutates.c
index 25afd9167a34..f5a4d91a7e90 100644
--- a/drivers/hwmon/ftsteutates.c
+++ b/drivers/hwmon/ftsteutates.c
@@ -520,7 +520,7 @@ static const struct hwmon_ops fts_ops = {
 	.write = fts_write,
 };
 
-static const struct hwmon_channel_info *fts_info[] = {
+static const struct hwmon_channel_info * const fts_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_ALARM | HWMON_T_FAULT,
diff --git a/drivers/hwmon/g762.c b/drivers/hwmon/g762.c
index 64a0599b2da5..e2c3c34f06e8 100644
--- a/drivers/hwmon/g762.c
+++ b/drivers/hwmon/g762.c
@@ -620,7 +620,12 @@ static int g762_of_clock_enable(struct i2c_client *client)
 	data = i2c_get_clientdata(client);
 	data->clk = clk;
 
-	devm_add_action(&client->dev, g762_of_clock_disable, data);
+	ret = devm_add_action(&client->dev, g762_of_clock_disable, data);
+	if (ret) {
+		dev_err(&client->dev, "failed to add disable clock action\n");
+		goto clk_unprep;
+	}
+
 	return 0;
 
  clk_unprep:
diff --git a/drivers/hwmon/gpio-fan.c b/drivers/hwmon/gpio-fan.c
index e75db6f64e8c..d92c536be9af 100644
--- a/drivers/hwmon/gpio-fan.c
+++ b/drivers/hwmon/gpio-fan.c
@@ -586,7 +586,7 @@ static struct platform_driver gpio_fan_driver = {
 	.driver	= {
 		.name	= "gpio-fan",
 		.pm	= pm_sleep_ptr(&gpio_fan_pm),
-		.of_match_table = of_match_ptr(of_gpio_fan_match),
+		.of_match_table = of_gpio_fan_match,
 	},
 };
 
diff --git a/drivers/hwmon/gxp-fan-ctrl.c b/drivers/hwmon/gxp-fan-ctrl.c
index 0014b8b0fd41..2e05bc2f627a 100644
--- a/drivers/hwmon/gxp-fan-ctrl.c
+++ b/drivers/hwmon/gxp-fan-ctrl.c
@@ -168,7 +168,7 @@ static const struct hwmon_ops gxp_fan_ctrl_ops = {
 	.write = gxp_fan_ctrl_write,
 };
 
-static const struct hwmon_channel_info *gxp_fan_ctrl_info[] = {
+static const struct hwmon_channel_info * const gxp_fan_ctrl_info[] = {
 	HWMON_CHANNEL_INFO(fan,
 			   HWMON_F_FAULT | HWMON_F_ENABLE,
 			   HWMON_F_FAULT | HWMON_F_ENABLE,
diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c
index d193ed3cb35e..573b83b6c08c 100644
--- a/drivers/hwmon/hwmon.c
+++ b/drivers/hwmon/hwmon.c
@@ -138,7 +138,6 @@ static void hwmon_dev_release(struct device *dev)
 
 static struct class hwmon_class = {
 	.name = "hwmon",
-	.owner = THIS_MODULE,
 	.dev_groups = hwmon_dev_attr_groups,
 	.dev_release = hwmon_dev_release,
 };
@@ -154,7 +153,7 @@ static DEFINE_IDA(hwmon_ida);
 #ifdef CONFIG_THERMAL_OF
 static int hwmon_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct hwmon_thermal_data *tdata = tz->devdata;
+	struct hwmon_thermal_data *tdata = thermal_zone_device_priv(tz);
 	struct hwmon_device *hwdev = to_hwmon_device(tdata->dev);
 	int ret;
 	long t;
@@ -171,10 +170,10 @@ static int hwmon_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 
 static int hwmon_thermal_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct hwmon_thermal_data *tdata = tz->devdata;
+	struct hwmon_thermal_data *tdata = thermal_zone_device_priv(tz);
 	struct hwmon_device *hwdev = to_hwmon_device(tdata->dev);
 	const struct hwmon_chip_info *chip = hwdev->chip;
-	const struct hwmon_channel_info **info = chip->info;
+	const struct hwmon_channel_info * const *info = chip->info;
 	unsigned int i;
 	int err;
 
@@ -253,7 +252,7 @@ static int hwmon_thermal_register_sensors(struct device *dev)
 {
 	struct hwmon_device *hwdev = to_hwmon_device(dev);
 	const struct hwmon_chip_info *chip = hwdev->chip;
-	const struct hwmon_channel_info **info = chip->info;
+	const struct hwmon_channel_info * const *info = chip->info;
 	void *drvdata = dev_get_drvdata(dev);
 	int i;
 
diff --git a/drivers/hwmon/i5500_temp.c b/drivers/hwmon/i5500_temp.c
index 23b9f94fe0a9..7b00b38c7f7b 100644
--- a/drivers/hwmon/i5500_temp.c
+++ b/drivers/hwmon/i5500_temp.c
@@ -88,7 +88,7 @@ static const struct hwmon_ops i5500_ops = {
 	.read = i5500_read,
 };
 
-static const struct hwmon_channel_info *i5500_info[] = {
+static const struct hwmon_channel_info * const i5500_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MAX_HYST | HWMON_T_CRIT |
diff --git a/drivers/hwmon/ibmpowernv.c b/drivers/hwmon/ibmpowernv.c
index 8e3724728cce..594254d6a72d 100644
--- a/drivers/hwmon/ibmpowernv.c
+++ b/drivers/hwmon/ibmpowernv.c
@@ -456,9 +456,9 @@ static int populate_attr_groups(struct platform_device *pdev)
 		 */
 		if (!of_property_read_string(np, "label", &label))
 			sensor_groups[type].attr_count++;
-		if (of_find_property(np, "sensor-data-min", NULL))
+		if (of_property_present(np, "sensor-data-min"))
 			sensor_groups[type].attr_count++;
-		if (of_find_property(np, "sensor-data-max", NULL))
+		if (of_property_present(np, "sensor-data-max"))
 			sensor_groups[type].attr_count++;
 	}
 
diff --git a/drivers/hwmon/ina238.c b/drivers/hwmon/ina238.c
index 50eb9c5e132e..8af07bc0c50e 100644
--- a/drivers/hwmon/ina238.c
+++ b/drivers/hwmon/ina238.c
@@ -501,7 +501,7 @@ static umode_t ina238_is_visible(const void *drvdata,
 				HWMON_I_MAX | HWMON_I_MAX_ALARM | \
 				HWMON_I_MIN | HWMON_I_MIN_ALARM)
 
-static const struct hwmon_channel_info *ina238_info[] = {
+static const struct hwmon_channel_info * const ina238_info[] = {
 	HWMON_CHANNEL_INFO(in,
 			   /* 0: shunt voltage */
 			   INA238_HWMON_IN_CONFIG,
diff --git a/drivers/hwmon/ina2xx.c b/drivers/hwmon/ina2xx.c
index 00fc70305a89..fd50d9785ccb 100644
--- a/drivers/hwmon/ina2xx.c
+++ b/drivers/hwmon/ina2xx.c
@@ -656,6 +656,10 @@ static int ina2xx_probe(struct i2c_client *client)
 		return PTR_ERR(data->regmap);
 	}
 
+	ret = devm_regulator_get_enable(dev, "vs");
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable vs regulator\n");
+
 	ret = ina2xx_init(data);
 	if (ret < 0) {
 		dev_err(dev, "error configuring the device: %d\n", ret);
diff --git a/drivers/hwmon/ina3221.c b/drivers/hwmon/ina3221.c
index f3a4c5633b1e..2735e3782ffb 100644
--- a/drivers/hwmon/ina3221.c
+++ b/drivers/hwmon/ina3221.c
@@ -650,7 +650,7 @@ static umode_t ina3221_is_visible(const void *drvdata,
 				   HWMON_C_CRIT | HWMON_C_CRIT_ALARM | \
 				   HWMON_C_MAX | HWMON_C_MAX_ALARM)
 
-static const struct hwmon_channel_info *ina3221_info[] = {
+static const struct hwmon_channel_info * const ina3221_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_SAMPLES,
 			   HWMON_C_UPDATE_INTERVAL),
diff --git a/drivers/hwmon/intel-m10-bmc-hwmon.c b/drivers/hwmon/intel-m10-bmc-hwmon.c
index 2f0323c14bab..6512f4bec79a 100644
--- a/drivers/hwmon/intel-m10-bmc-hwmon.c
+++ b/drivers/hwmon/intel-m10-bmc-hwmon.c
@@ -24,7 +24,7 @@ struct m10bmc_sdata {
 
 struct m10bmc_hwmon_board_data {
 	const struct m10bmc_sdata *tables[hwmon_max];
-	const struct hwmon_channel_info **hinfo;
+	const struct hwmon_channel_info * const *hinfo;
 };
 
 struct m10bmc_hwmon {
@@ -67,7 +67,7 @@ static const struct m10bmc_sdata n3000bmc_power_tbl[] = {
 	{ 0x160, 0x0, 0x0, 0x0, 0x0, 1000, "Board Power" },
 };
 
-static const struct hwmon_channel_info *n3000bmc_hinfo[] = {
+static const struct hwmon_channel_info * const n3000bmc_hinfo[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MAX_HYST |
 			   HWMON_T_CRIT | HWMON_T_CRIT_HYST | HWMON_T_LABEL,
@@ -154,7 +154,7 @@ static const struct m10bmc_hwmon_board_data n3000bmc_hwmon_bdata = {
 	.hinfo = n3000bmc_hinfo,
 };
 
-static const struct hwmon_channel_info *d5005bmc_hinfo[] = {
+static const struct hwmon_channel_info * const d5005bmc_hinfo[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MAX_HYST |
 			   HWMON_T_CRIT | HWMON_T_CRIT_HYST | HWMON_T_LABEL,
@@ -280,7 +280,7 @@ static const struct m10bmc_sdata n5010bmc_curr_tbl[] = {
 	{ 0x1a0, 0x0, 0x0, 0x0, 0x0, 1, "CVL2 0.8V Current" },
 };
 
-static const struct hwmon_channel_info *n5010bmc_hinfo[] = {
+static const struct hwmon_channel_info * const n5010bmc_hinfo[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_CRIT | HWMON_T_LABEL,
 			   HWMON_T_INPUT | HWMON_T_CRIT | HWMON_T_LABEL,
@@ -432,7 +432,7 @@ static const struct m10bmc_sdata n6000bmc_power_tbl[] = {
 	{ 0x724, 0x0, 0x0, 0x0, 0x0, 1, "Board Power" },
 };
 
-static const struct hwmon_channel_info *n6000bmc_hinfo[] = {
+static const struct hwmon_channel_info * const n6000bmc_hinfo[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT |
 			   HWMON_T_LABEL,
diff --git a/drivers/hwmon/it87.c b/drivers/hwmon/it87.c
index e9614eb557d4..eb38f54ebeb6 100644
--- a/drivers/hwmon/it87.c
+++ b/drivers/hwmon/it87.c
@@ -162,8 +162,11 @@ static inline void superio_exit(int ioreg, bool noexit)
 #define IT8623E_DEVID 0x8623
 #define IT8628E_DEVID 0x8628
 #define IT87952E_DEVID 0x8695
-#define IT87_ACT_REG  0x30
-#define IT87_BASE_REG 0x60
+
+/* Logical device 4 (Environmental Monitor) registers */
+#define IT87_ACT_REG	0x30
+#define IT87_BASE_REG	0x60
+#define IT87_SPECIAL_CFG_REG	0xf3	/* special configuration register */
 
 /* Logical device 7 registers (IT8712F and later) */
 #define IT87_SIO_GPIO1_REG	0x25
@@ -284,6 +287,8 @@ struct it87_devices {
 	u32 features;
 	u8 peci_mask;
 	u8 old_peci_mask;
+	u8 smbus_bitmap;	/* SMBus enable bits in extra config register */
+	u8 ec_special_config;
 };
 
 #define FEAT_12MV_ADC		BIT(0)
@@ -469,6 +474,7 @@ static const struct it87_devices it87_devices[] = {
 		  | FEAT_FIVE_PWM | FEAT_IN7_INTERNAL | FEAT_PWM_FREQ2
 		  | FEAT_AVCC3 | FEAT_VIN3_5V,
 		.peci_mask = 0x07,
+		.smbus_bitmap = BIT(1) | BIT(2),
 	},
 	[it8628] = {
 		.name = "it8628",
@@ -533,6 +539,8 @@ struct it87_sio_data {
 	u8 skip_fan;
 	u8 skip_pwm;
 	u8 skip_temp;
+	u8 smbus_bitmap;
+	u8 ec_special_config;
 };
 
 /*
@@ -547,6 +555,9 @@ struct it87_data {
 	u8 peci_mask;
 	u8 old_peci_mask;
 
+	u8 smbus_bitmap;	/* !=0 if SMBus needs to be disabled */
+	u8 ec_special_config;	/* EC special config register restore value */
+
 	unsigned short addr;
 	const char *name;
 	struct mutex update_lock;
@@ -701,8 +712,42 @@ static const unsigned int pwm_freq[8] = {
 	750000,
 };
 
+static int smbus_disable(struct it87_data *data)
+{
+	int err;
+
+	if (data->smbus_bitmap) {
+		err = superio_enter(data->sioaddr);
+		if (err)
+			return err;
+		superio_select(data->sioaddr, PME);
+		superio_outb(data->sioaddr, IT87_SPECIAL_CFG_REG,
+			     data->ec_special_config & ~data->smbus_bitmap);
+		superio_exit(data->sioaddr, has_conf_noexit(data));
+	}
+	return 0;
+}
+
+static int smbus_enable(struct it87_data *data)
+{
+	int err;
+
+	if (data->smbus_bitmap) {
+		err = superio_enter(data->sioaddr);
+		if (err)
+			return err;
+
+		superio_select(data->sioaddr, PME);
+		superio_outb(data->sioaddr, IT87_SPECIAL_CFG_REG,
+			     data->ec_special_config);
+		superio_exit(data->sioaddr, has_conf_noexit(data));
+	}
+	return 0;
+}
+
 /*
  * Must be called with data->update_lock held, except during initialization.
+ * Must be called with SMBus accesses disabled.
  * We ignore the IT87 BUSY flag at this moment - it could lead to deadlocks,
  * would slow down the IT87 access and should not be necessary.
  */
@@ -714,6 +759,7 @@ static int it87_read_value(struct it87_data *data, u8 reg)
 
 /*
  * Must be called with data->update_lock held, except during initialization.
+ * Must be called with SMBus accesses disabled.
  * We ignore the IT87 BUSY flag at this moment - it could lead to deadlocks,
  * would slow down the IT87 access and should not be necessary.
  */
@@ -773,15 +819,39 @@ static void it87_update_pwm_ctrl(struct it87_data *data, int nr)
 	}
 }
 
+static int it87_lock(struct it87_data *data)
+{
+	int err;
+
+	mutex_lock(&data->update_lock);
+	err = smbus_disable(data);
+	if (err)
+		mutex_unlock(&data->update_lock);
+	return err;
+}
+
+static void it87_unlock(struct it87_data *data)
+{
+	smbus_enable(data);
+	mutex_unlock(&data->update_lock);
+}
+
 static struct it87_data *it87_update_device(struct device *dev)
 {
 	struct it87_data *data = dev_get_drvdata(dev);
+	struct it87_data *ret = data;
+	int err;
 	int i;
 
 	mutex_lock(&data->update_lock);
 
 	if (time_after(jiffies, data->last_updated + HZ + HZ / 2) ||
-	    !data->valid) {
+		       !data->valid) {
+		err = smbus_disable(data);
+		if (err) {
+			ret = ERR_PTR(err);
+			goto unlock;
+		}
 		if (update_vbat) {
 			/*
 			 * Cleared after each update, so reenable.  Value
@@ -884,11 +954,11 @@ static struct it87_data *it87_update_device(struct device *dev)
 		}
 		data->last_updated = jiffies;
 		data->valid = true;
+		smbus_enable(data);
 	}
-
+unlock:
 	mutex_unlock(&data->update_lock);
-
-	return data;
+	return ret;
 }
 
 static ssize_t show_in(struct device *dev, struct device_attribute *attr,
@@ -899,6 +969,9 @@ static ssize_t show_in(struct device *dev, struct device_attribute *attr,
 	int index = sattr->index;
 	int nr = sattr->nr;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%d\n", in_from_reg(data, nr, data->in[nr][index]));
 }
 
@@ -910,17 +983,21 @@ static ssize_t set_in(struct device *dev, struct device_attribute *attr,
 	int index = sattr->index;
 	int nr = sattr->nr;
 	unsigned long val;
+	int err;
 
 	if (kstrtoul(buf, 10, &val) < 0)
 		return -EINVAL;
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	data->in[nr][index] = in_to_reg(data, nr, val);
 	it87_write_value(data,
 			 index == 1 ? IT87_REG_VIN_MIN(nr)
 				    : IT87_REG_VIN_MAX(nr),
 			 data->in[nr][index]);
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 	return count;
 }
 
@@ -987,6 +1064,9 @@ static ssize_t show_temp(struct device *dev, struct device_attribute *attr,
 	int index = sattr->index;
 	struct it87_data *data = it87_update_device(dev);
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp[nr][index]));
 }
 
@@ -999,11 +1079,14 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *attr,
 	struct it87_data *data = dev_get_drvdata(dev);
 	long val;
 	u8 reg, regval;
+	int err;
 
 	if (kstrtol(buf, 10, &val) < 0)
 		return -EINVAL;
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
 
 	switch (index) {
 	default:
@@ -1026,7 +1109,7 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *attr,
 
 	data->temp[nr][index] = TEMP_TO_REG(val);
 	it87_write_value(data, reg, data->temp[nr][index]);
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 	return count;
 }
 
@@ -1061,8 +1144,13 @@ static ssize_t show_temp_type(struct device *dev, struct device_attribute *attr,
 	struct sensor_device_attribute *sensor_attr = to_sensor_dev_attr(attr);
 	int nr = sensor_attr->index;
 	struct it87_data *data = it87_update_device(dev);
-	u8 reg = data->sensor;	    /* In case value is updated while used */
-	u8 extra = data->extra;
+	u8 reg, extra;
+
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	reg = data->sensor;	/* In case value is updated while used */
+	extra = data->extra;
 
 	if ((has_temp_peci(data, nr) && (reg >> 6 == nr + 1)) ||
 	    (has_temp_old_peci(data, nr) && (extra & 0x80)))
@@ -1083,10 +1171,15 @@ static ssize_t set_temp_type(struct device *dev, struct device_attribute *attr,
 	struct it87_data *data = dev_get_drvdata(dev);
 	long val;
 	u8 reg, extra;
+	int err;
 
 	if (kstrtol(buf, 10, &val) < 0)
 		return -EINVAL;
 
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	reg = it87_read_value(data, IT87_REG_TEMP_ENABLE);
 	reg &= ~(1 << nr);
 	reg &= ~(8 << nr);
@@ -1109,17 +1202,19 @@ static ssize_t set_temp_type(struct device *dev, struct device_attribute *attr,
 		reg |= (nr + 1) << 6;
 	else if (has_temp_old_peci(data, nr) && val == 6)
 		extra |= 0x80;
-	else if (val != 0)
-		return -EINVAL;
+	else if (val != 0) {
+		count = -EINVAL;
+		goto unlock;
+	}
 
-	mutex_lock(&data->update_lock);
 	data->sensor = reg;
 	data->extra = extra;
 	it87_write_value(data, IT87_REG_TEMP_ENABLE, data->sensor);
 	if (has_temp_old_peci(data, nr))
 		it87_write_value(data, IT87_REG_TEMP_EXTRA, data->extra);
 	data->valid = false;	/* Force cache refresh */
-	mutex_unlock(&data->update_lock);
+unlock:
+	it87_unlock(data);
 	return count;
 }
 
@@ -1154,6 +1249,9 @@ static ssize_t show_fan(struct device *dev, struct device_attribute *attr,
 	int speed;
 	struct it87_data *data = it87_update_device(dev);
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	speed = has_16bit_fans(data) ?
 		FAN16_FROM_REG(data->fan[nr][index]) :
 		FAN_FROM_REG(data->fan[nr][index],
@@ -1168,6 +1266,9 @@ static ssize_t show_fan_div(struct device *dev, struct device_attribute *attr,
 	struct it87_data *data = it87_update_device(dev);
 	int nr = sensor_attr->index;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%lu\n", DIV_FROM_REG(data->fan_div[nr]));
 }
 
@@ -1178,6 +1279,9 @@ static ssize_t show_pwm_enable(struct device *dev,
 	struct it87_data *data = it87_update_device(dev);
 	int nr = sensor_attr->index;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%d\n", pwm_mode(data, nr));
 }
 
@@ -1188,6 +1292,9 @@ static ssize_t show_pwm(struct device *dev, struct device_attribute *attr,
 	struct it87_data *data = it87_update_device(dev);
 	int nr = sensor_attr->index;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%d\n",
 		       pwm_from_reg(data, data->pwm_duty[nr]));
 }
@@ -1201,6 +1308,9 @@ static ssize_t show_pwm_freq(struct device *dev, struct device_attribute *attr,
 	unsigned int freq;
 	int index;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	if (has_pwm_freq2(data) && nr == 1)
 		index = (data->extra >> 4) & 0x07;
 	else
@@ -1220,12 +1330,15 @@ static ssize_t set_fan(struct device *dev, struct device_attribute *attr,
 
 	struct it87_data *data = dev_get_drvdata(dev);
 	long val;
+	int err;
 	u8 reg;
 
 	if (kstrtol(buf, 10, &val) < 0)
 		return -EINVAL;
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
 
 	if (has_16bit_fans(data)) {
 		data->fan[nr][index] = FAN16_TO_REG(val);
@@ -1252,7 +1365,7 @@ static ssize_t set_fan(struct device *dev, struct device_attribute *attr,
 				 data->fan[nr][index]);
 	}
 
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 	return count;
 }
 
@@ -1263,13 +1376,16 @@ static ssize_t set_fan_div(struct device *dev, struct device_attribute *attr,
 	struct it87_data *data = dev_get_drvdata(dev);
 	int nr = sensor_attr->index;
 	unsigned long val;
-	int min;
+	int min, err;
 	u8 old;
 
 	if (kstrtoul(buf, 10, &val) < 0)
 		return -EINVAL;
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	old = it87_read_value(data, IT87_REG_FAN_DIV);
 
 	/* Save fan min limit */
@@ -1297,7 +1413,7 @@ static ssize_t set_fan_div(struct device *dev, struct device_attribute *attr,
 	data->fan[nr][1] = FAN_TO_REG(min, DIV_FROM_REG(data->fan_div[nr]));
 	it87_write_value(data, IT87_REG_FAN_MIN[nr], data->fan[nr][1]);
 
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 	return count;
 }
 
@@ -1338,6 +1454,7 @@ static ssize_t set_pwm_enable(struct device *dev, struct device_attribute *attr,
 	struct it87_data *data = dev_get_drvdata(dev);
 	int nr = sensor_attr->index;
 	long val;
+	int err;
 
 	if (kstrtol(buf, 10, &val) < 0 || val < 0 || val > 2)
 		return -EINVAL;
@@ -1348,7 +1465,9 @@ static ssize_t set_pwm_enable(struct device *dev, struct device_attribute *attr,
 			return -EINVAL;
 	}
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
 
 	if (val == 0) {
 		if (nr < 3 && data->type != it8603) {
@@ -1399,7 +1518,7 @@ static ssize_t set_pwm_enable(struct device *dev, struct device_attribute *attr,
 		}
 	}
 
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 	return count;
 }
 
@@ -1410,11 +1529,15 @@ static ssize_t set_pwm(struct device *dev, struct device_attribute *attr,
 	struct it87_data *data = dev_get_drvdata(dev);
 	int nr = sensor_attr->index;
 	long val;
+	int err;
 
 	if (kstrtol(buf, 10, &val) < 0 || val < 0 || val > 255)
 		return -EINVAL;
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	it87_update_pwm_ctrl(data, nr);
 	if (has_newer_autopwm(data)) {
 		/*
@@ -1422,8 +1545,8 @@ static ssize_t set_pwm(struct device *dev, struct device_attribute *attr,
 		 * is read-only so we can't write the value.
 		 */
 		if (data->pwm_ctrl[nr] & 0x80) {
-			mutex_unlock(&data->update_lock);
-			return -EBUSY;
+			count = -EBUSY;
+			goto unlock;
 		}
 		data->pwm_duty[nr] = pwm_to_reg(data, val);
 		it87_write_value(data, IT87_REG_PWM_DUTY[nr],
@@ -1440,7 +1563,8 @@ static ssize_t set_pwm(struct device *dev, struct device_attribute *attr,
 					 data->pwm_ctrl[nr]);
 		}
 	}
-	mutex_unlock(&data->update_lock);
+unlock:
+	it87_unlock(data);
 	return count;
 }
 
@@ -1451,6 +1575,7 @@ static ssize_t set_pwm_freq(struct device *dev, struct device_attribute *attr,
 	struct it87_data *data = dev_get_drvdata(dev);
 	int nr = sensor_attr->index;
 	unsigned long val;
+	int err;
 	int i;
 
 	if (kstrtoul(buf, 10, &val) < 0)
@@ -1465,7 +1590,10 @@ static ssize_t set_pwm_freq(struct device *dev, struct device_attribute *attr,
 			break;
 	}
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	if (nr == 0) {
 		data->fan_ctl = it87_read_value(data, IT87_REG_FAN_CTL) & 0x8f;
 		data->fan_ctl |= i << 4;
@@ -1475,7 +1603,7 @@ static ssize_t set_pwm_freq(struct device *dev, struct device_attribute *attr,
 		data->extra |= i << 4;
 		it87_write_value(data, IT87_REG_TEMP_EXTRA, data->extra);
 	}
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 
 	return count;
 }
@@ -1488,6 +1616,9 @@ static ssize_t show_pwm_temp_map(struct device *dev,
 	int nr = sensor_attr->index;
 	int map;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	map = data->pwm_temp_map[nr];
 	if (map >= 3)
 		map = 0;	/* Should never happen */
@@ -1505,6 +1636,7 @@ static ssize_t set_pwm_temp_map(struct device *dev,
 	struct it87_data *data = dev_get_drvdata(dev);
 	int nr = sensor_attr->index;
 	long val;
+	int err;
 	u8 reg;
 
 	if (kstrtol(buf, 10, &val) < 0)
@@ -1527,7 +1659,10 @@ static ssize_t set_pwm_temp_map(struct device *dev,
 		return -EINVAL;
 	}
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	it87_update_pwm_ctrl(data, nr);
 	data->pwm_temp_map[nr] = reg;
 	/*
@@ -1539,7 +1674,7 @@ static ssize_t set_pwm_temp_map(struct device *dev,
 						data->pwm_temp_map[nr];
 		it87_write_value(data, IT87_REG_PWM[nr], data->pwm_ctrl[nr]);
 	}
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 	return count;
 }
 
@@ -1552,6 +1687,9 @@ static ssize_t show_auto_pwm(struct device *dev, struct device_attribute *attr,
 	int nr = sensor_attr->nr;
 	int point = sensor_attr->index;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%d\n",
 		       pwm_from_reg(data, data->auto_pwm[nr][point]));
 }
@@ -1566,18 +1704,22 @@ static ssize_t set_auto_pwm(struct device *dev, struct device_attribute *attr,
 	int point = sensor_attr->index;
 	int regaddr;
 	long val;
+	int err;
 
 	if (kstrtol(buf, 10, &val) < 0 || val < 0 || val > 255)
 		return -EINVAL;
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	data->auto_pwm[nr][point] = pwm_to_reg(data, val);
 	if (has_newer_autopwm(data))
 		regaddr = IT87_REG_AUTO_TEMP(nr, 3);
 	else
 		regaddr = IT87_REG_AUTO_PWM(nr, point);
 	it87_write_value(data, regaddr, data->auto_pwm[nr][point]);
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 	return count;
 }
 
@@ -1588,6 +1730,9 @@ static ssize_t show_auto_pwm_slope(struct device *dev,
 	struct sensor_device_attribute *sensor_attr = to_sensor_dev_attr(attr);
 	int nr = sensor_attr->index;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%d\n", data->auto_pwm[nr][1] & 0x7f);
 }
 
@@ -1599,15 +1744,19 @@ static ssize_t set_auto_pwm_slope(struct device *dev,
 	struct sensor_device_attribute *sensor_attr = to_sensor_dev_attr(attr);
 	int nr = sensor_attr->index;
 	unsigned long val;
+	int err;
 
 	if (kstrtoul(buf, 10, &val) < 0 || val > 127)
 		return -EINVAL;
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	data->auto_pwm[nr][1] = (data->auto_pwm[nr][1] & 0x80) | val;
 	it87_write_value(data, IT87_REG_AUTO_TEMP(nr, 4),
 			 data->auto_pwm[nr][1]);
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 	return count;
 }
 
@@ -1621,6 +1770,9 @@ static ssize_t show_auto_temp(struct device *dev, struct device_attribute *attr,
 	int point = sensor_attr->index;
 	int reg;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	if (has_old_autopwm(data) || point)
 		reg = data->auto_temp[nr][point];
 	else
@@ -1639,11 +1791,15 @@ static ssize_t set_auto_temp(struct device *dev, struct device_attribute *attr,
 	int point = sensor_attr->index;
 	long val;
 	int reg;
+	int err;
 
 	if (kstrtol(buf, 10, &val) < 0 || val < -128000 || val > 127000)
 		return -EINVAL;
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	if (has_newer_autopwm(data) && !point) {
 		reg = data->auto_temp[nr][1] - TEMP_TO_REG(val);
 		reg = clamp_val(reg, 0, 0x1f) | (data->auto_temp[nr][0] & 0xe0);
@@ -1656,7 +1812,7 @@ static ssize_t set_auto_temp(struct device *dev, struct device_attribute *attr,
 			point--;
 		it87_write_value(data, IT87_REG_AUTO_TEMP(nr, point), reg);
 	}
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 	return count;
 }
 
@@ -1841,6 +1997,9 @@ static ssize_t alarms_show(struct device *dev, struct device_attribute *attr,
 {
 	struct it87_data *data = it87_update_device(dev);
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%u\n", data->alarms);
 }
 static DEVICE_ATTR_RO(alarms);
@@ -1851,6 +2010,9 @@ static ssize_t show_alarm(struct device *dev, struct device_attribute *attr,
 	struct it87_data *data = it87_update_device(dev);
 	int bitnr = to_sensor_dev_attr(attr)->index;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%u\n", (data->alarms >> bitnr) & 1);
 }
 
@@ -1859,13 +2021,16 @@ static ssize_t clear_intrusion(struct device *dev,
 			       size_t count)
 {
 	struct it87_data *data = dev_get_drvdata(dev);
-	int config;
+	int err, config;
 	long val;
 
 	if (kstrtol(buf, 10, &val) < 0 || val != 0)
 		return -EINVAL;
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	config = it87_read_value(data, IT87_REG_CONFIG);
 	if (config < 0) {
 		count = config;
@@ -1875,8 +2040,7 @@ static ssize_t clear_intrusion(struct device *dev,
 		/* Invalidate cache to force re-read */
 		data->valid = false;
 	}
-	mutex_unlock(&data->update_lock);
-
+	it87_unlock(data);
 	return count;
 }
 
@@ -1906,6 +2070,9 @@ static ssize_t show_beep(struct device *dev, struct device_attribute *attr,
 	struct it87_data *data = it87_update_device(dev);
 	int bitnr = to_sensor_dev_attr(attr)->index;
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%u\n", (data->beeps >> bitnr) & 1);
 }
 
@@ -1915,18 +2082,22 @@ static ssize_t set_beep(struct device *dev, struct device_attribute *attr,
 	int bitnr = to_sensor_dev_attr(attr)->index;
 	struct it87_data *data = dev_get_drvdata(dev);
 	long val;
+	int err;
 
 	if (kstrtol(buf, 10, &val) < 0 || (val != 0 && val != 1))
 		return -EINVAL;
 
-	mutex_lock(&data->update_lock);
+	err = it87_lock(data);
+	if (err)
+		return err;
+
 	data->beeps = it87_read_value(data, IT87_REG_BEEP_ENABLE);
 	if (val)
 		data->beeps |= BIT(bitnr);
 	else
 		data->beeps &= ~BIT(bitnr);
 	it87_write_value(data, IT87_REG_BEEP_ENABLE, data->beeps);
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 	return count;
 }
 
@@ -1979,6 +2150,9 @@ static ssize_t cpu0_vid_show(struct device *dev,
 {
 	struct it87_data *data = it87_update_device(dev);
 
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
 	return sprintf(buf, "%ld\n", (long)vid_from_reg(data->vid, data->vrm));
 }
 static DEVICE_ATTR_RO(cpu0_vid);
@@ -2004,7 +2178,7 @@ static ssize_t show_label(struct device *dev, struct device_attribute *attr,
 
 	if (has_vin3_5v(data) && nr == 0)
 		label = labels[0];
-	else if (has_12mv_adc(data) || has_10_9mv_adc(data))
+	else if (has_scaling(data))
 		label = labels_it8721[nr];
 	else
 		label = labels[nr];
@@ -2859,6 +3033,15 @@ static int __init it87_find(int sioaddr, unsigned short *address,
 	if (dmi_data)
 		sio_data->skip_pwm |= dmi_data->skip_pwm;
 
+	if (config->smbus_bitmap) {
+		u8 reg;
+
+		superio_select(sioaddr, PME);
+		reg = superio_inb(sioaddr, IT87_SPECIAL_CFG_REG);
+		sio_data->ec_special_config = reg;
+		sio_data->smbus_bitmap = reg & config->smbus_bitmap;
+	}
+
 exit:
 	superio_exit(sioaddr, config ? has_conf_noexit(config) : false);
 	return err;
@@ -3077,6 +3260,7 @@ static int it87_probe(struct platform_device *pdev)
 	struct it87_sio_data *sio_data = dev_get_platdata(dev);
 	int enable_pwm_interface;
 	struct device *hwmon_dev;
+	int err;
 
 	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
 	if (!devm_request_region(&pdev->dev, res->start, IT87_EC_EXTENT,
@@ -3094,6 +3278,8 @@ static int it87_probe(struct platform_device *pdev)
 	data->addr = res->start;
 	data->sioaddr = sio_data->sioaddr;
 	data->type = sio_data->type;
+	data->smbus_bitmap = sio_data->smbus_bitmap;
+	data->ec_special_config = sio_data->ec_special_config;
 	data->features = it87_devices[sio_data->type].features;
 	data->peci_mask = it87_devices[sio_data->type].peci_mask;
 	data->old_peci_mask = it87_devices[sio_data->type].old_peci_mask;
@@ -3120,15 +3306,21 @@ static int it87_probe(struct platform_device *pdev)
 		break;
 	}
 
-	/* Now, we do the remaining detection. */
-	if ((it87_read_value(data, IT87_REG_CONFIG) & 0x80) ||
-	    it87_read_value(data, IT87_REG_CHIPID) != 0x90)
-		return -ENODEV;
-
 	platform_set_drvdata(pdev, data);
 
 	mutex_init(&data->update_lock);
 
+	err = smbus_disable(data);
+	if (err)
+		return err;
+
+	/* Now, we do the remaining detection. */
+	if ((it87_read_value(data, IT87_REG_CONFIG) & 0x80) ||
+	    it87_read_value(data, IT87_REG_CHIPID) != 0x90) {
+		smbus_enable(data);
+		return -ENODEV;
+	}
+
 	/* Check PWM configuration */
 	enable_pwm_interface = it87_check_pwm(dev);
 	if (!enable_pwm_interface)
@@ -3189,6 +3381,8 @@ static int it87_probe(struct platform_device *pdev)
 	/* Initialize the IT87 chip */
 	it87_init_device(pdev);
 
+	smbus_enable(data);
+
 	if (!sio_data->skip_vid) {
 		data->has_vid = true;
 		data->vrm = vid_which_vrm();
@@ -3256,7 +3450,7 @@ static int it87_resume(struct device *dev)
 
 	it87_resume_sio(pdev);
 
-	mutex_lock(&data->update_lock);
+	it87_lock(data);
 
 	it87_check_pwm(dev);
 	it87_check_limit_regs(data);
@@ -3269,7 +3463,7 @@ static int it87_resume(struct device *dev)
 	/* force update */
 	data->valid = false;
 
-	mutex_unlock(&data->update_lock);
+	it87_unlock(data);
 
 	it87_update_device(dev);
 
diff --git a/drivers/hwmon/jc42.c b/drivers/hwmon/jc42.c
index 8523bf974310..4c60dc520b12 100644
--- a/drivers/hwmon/jc42.c
+++ b/drivers/hwmon/jc42.c
@@ -450,7 +450,7 @@ static int jc42_detect(struct i2c_client *client, struct i2c_board_info *info)
 	return -ENODEV;
 }
 
-static const struct hwmon_channel_info *jc42_info[] = {
+static const struct hwmon_channel_info * const jc42_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c
index 5a9d47a229e4..ba2f6a4f8c16 100644
--- a/drivers/hwmon/k10temp.c
+++ b/drivers/hwmon/k10temp.c
@@ -75,6 +75,7 @@ static DEFINE_MUTEX(nb_smu_ind_mutex);
 
 #define ZEN_CUR_TEMP_SHIFT			21
 #define ZEN_CUR_TEMP_RANGE_SEL_MASK		BIT(19)
+#define ZEN_CUR_TEMP_TJ_SEL_MASK		GENMASK(17, 16)
 
 struct k10temp_data {
 	struct pci_dev *pdev;
@@ -155,7 +156,8 @@ static long get_raw_temp(struct k10temp_data *data)
 
 	data->read_tempreg(data->pdev, &regval);
 	temp = (regval >> ZEN_CUR_TEMP_SHIFT) * 125;
-	if (regval & data->temp_adjust_mask)
+	if ((regval & data->temp_adjust_mask) ||
+	    (regval & ZEN_CUR_TEMP_TJ_SEL_MASK) == ZEN_CUR_TEMP_TJ_SEL_MASK)
 		temp -= 49000;
 	return temp;
 }
@@ -332,7 +334,7 @@ static bool has_erratum_319(struct pci_dev *pdev)
 	       (boot_cpu_data.x86_model == 4 && boot_cpu_data.x86_stepping <= 2);
 }
 
-static const struct hwmon_channel_info *k10temp_info[] = {
+static const struct hwmon_channel_info * const k10temp_info[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_MAX |
 			   HWMON_T_CRIT | HWMON_T_CRIT_HYST |
diff --git a/drivers/hwmon/k8temp.c b/drivers/hwmon/k8temp.c
index f73bd4eceb28..2b80ac410cd1 100644
--- a/drivers/hwmon/k8temp.c
+++ b/drivers/hwmon/k8temp.c
@@ -118,7 +118,7 @@ static const struct hwmon_ops k8temp_ops = {
 	.read = k8temp_read,
 };
 
-static const struct hwmon_channel_info *k8temp_info[] = {
+static const struct hwmon_channel_info * const k8temp_info[] = {
 	HWMON_CHANNEL_INFO(temp,
 		HWMON_T_INPUT, HWMON_T_INPUT, HWMON_T_INPUT, HWMON_T_INPUT),
 	NULL
diff --git a/drivers/hwmon/lan966x-hwmon.c b/drivers/hwmon/lan966x-hwmon.c
index f41df053ac31..f8658359a098 100644
--- a/drivers/hwmon/lan966x-hwmon.c
+++ b/drivers/hwmon/lan966x-hwmon.c
@@ -260,7 +260,7 @@ static umode_t lan966x_hwmon_is_visible(const void *data,
 	return mode;
 }
 
-static const struct hwmon_channel_info *lan966x_hwmon_info[] = {
+static const struct hwmon_channel_info * const lan966x_hwmon_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
 	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT),
diff --git a/drivers/hwmon/lm75.c b/drivers/hwmon/lm75.c
index bcc3adcb3af1..dbb99ea4a0ec 100644
--- a/drivers/hwmon/lm75.c
+++ b/drivers/hwmon/lm75.c
@@ -512,7 +512,7 @@ static umode_t lm75_is_visible(const void *data, enum hwmon_sensor_types type,
 	return 0;
 }
 
-static const struct hwmon_channel_info *lm75_info[] = {
+static const struct hwmon_channel_info * const lm75_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/lm83.c b/drivers/hwmon/lm83.c
index 616449f2cc50..bcaf31ec176e 100644
--- a/drivers/hwmon/lm83.c
+++ b/drivers/hwmon/lm83.c
@@ -337,7 +337,7 @@ static umode_t lm83_is_visible(const void *_data, enum hwmon_sensor_types type,
 	return 0;
 }
 
-static const struct hwmon_channel_info *lm83_info[] = {
+static const struct hwmon_channel_info * const lm83_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_ALARMS),
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT |
diff --git a/drivers/hwmon/lm95241.c b/drivers/hwmon/lm95241.c
index f1ed777a8735..647a90570bc6 100644
--- a/drivers/hwmon/lm95241.c
+++ b/drivers/hwmon/lm95241.c
@@ -409,7 +409,7 @@ static void lm95241_init_client(struct i2c_client *client,
 				  data->model);
 }
 
-static const struct hwmon_channel_info *lm95241_info[] = {
+static const struct hwmon_channel_info * const lm95241_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/lm95245.c b/drivers/hwmon/lm95245.c
index c433f0af2d31..4236d1e0544d 100644
--- a/drivers/hwmon/lm95245.c
+++ b/drivers/hwmon/lm95245.c
@@ -523,7 +523,7 @@ static const struct regmap_config lm95245_regmap_config = {
 	.use_single_write = true,
 };
 
-static const struct hwmon_channel_info *lm95245_info[] = {
+static const struct hwmon_channel_info * const lm95245_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/lochnagar-hwmon.c b/drivers/hwmon/lochnagar-hwmon.c
index 8b19adf2eeb0..6350904a8a8b 100644
--- a/drivers/hwmon/lochnagar-hwmon.c
+++ b/drivers/hwmon/lochnagar-hwmon.c
@@ -11,7 +11,6 @@
 #include <linux/delay.h>
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
-#include <linux/i2c.h>
 #include <linux/math64.h>
 #include <linux/mfd/lochnagar.h>
 #include <linux/mfd/lochnagar2_regs.h>
@@ -321,7 +320,7 @@ static const struct hwmon_ops lochnagar_ops = {
 	.write = lochnagar_write,
 };
 
-static const struct hwmon_channel_info *lochnagar_info[] = {
+static const struct hwmon_channel_info * const lochnagar_info[] = {
 	HWMON_CHANNEL_INFO(temp,  HWMON_T_INPUT),
 	HWMON_CHANNEL_INFO(in,    HWMON_I_INPUT | HWMON_I_LABEL,
 				  HWMON_I_INPUT | HWMON_I_LABEL,
diff --git a/drivers/hwmon/ltc2947-core.c b/drivers/hwmon/ltc2947-core.c
index 2dbbbac9de09..d2ff6e700770 100644
--- a/drivers/hwmon/ltc2947-core.c
+++ b/drivers/hwmon/ltc2947-core.c
@@ -901,7 +901,7 @@ static umode_t ltc2947_is_visible(const void *data,
 	}
 }
 
-static const struct hwmon_channel_info *ltc2947_info[] = {
+static const struct hwmon_channel_info * const ltc2947_info[] = {
 	HWMON_CHANNEL_INFO(in,
 			   HWMON_I_INPUT | HWMON_I_LOWEST | HWMON_I_HIGHEST |
 			   HWMON_I_MAX | HWMON_I_MIN | HWMON_I_RESET_HISTORY |
diff --git a/drivers/hwmon/ltc2992.c b/drivers/hwmon/ltc2992.c
index 69341de397cb..429a7f5837f0 100644
--- a/drivers/hwmon/ltc2992.c
+++ b/drivers/hwmon/ltc2992.c
@@ -812,7 +812,7 @@ static const struct hwmon_ops ltc2992_hwmon_ops = {
 	.write = ltc2992_write,
 };
 
-static const struct hwmon_channel_info *ltc2992_info[] = {
+static const struct hwmon_channel_info * const ltc2992_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_IN_RESET_HISTORY),
 	HWMON_CHANNEL_INFO(in,
diff --git a/drivers/hwmon/ltc4245.c b/drivers/hwmon/ltc4245.c
index 5088d28b3a7c..fb9bc8dbe99b 100644
--- a/drivers/hwmon/ltc4245.c
+++ b/drivers/hwmon/ltc4245.c
@@ -387,7 +387,7 @@ static umode_t ltc4245_is_visible(const void *_data,
 	}
 }
 
-static const struct hwmon_channel_info *ltc4245_info[] = {
+static const struct hwmon_channel_info * const ltc4245_info[] = {
 	HWMON_CHANNEL_INFO(in,
 			   HWMON_I_INPUT,
 			   HWMON_I_INPUT | HWMON_I_MIN_ALARM,
@@ -434,7 +434,7 @@ static bool ltc4245_use_extra_gpios(struct i2c_client *client)
 		return pdata->use_extra_gpios;
 
 	/* fallback on OF */
-	if (of_find_property(np, "ltc4245,use-extra-gpios", NULL))
+	if (of_property_read_bool(np, "ltc4245,use-extra-gpios"))
 		return true;
 
 	return false;
diff --git a/drivers/hwmon/ltq-cputemp.c b/drivers/hwmon/ltq-cputemp.c
index 019e770d4c0c..08e09a82acab 100644
--- a/drivers/hwmon/ltq-cputemp.c
+++ b/drivers/hwmon/ltq-cputemp.c
@@ -65,7 +65,7 @@ static umode_t ltq_is_visible(const void *_data, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *ltq_info[] = {
+static const struct hwmon_channel_info * const ltq_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/max127.c b/drivers/hwmon/max127.c
index 0e21e7e6bbd2..49de1d2be294 100644
--- a/drivers/hwmon/max127.c
+++ b/drivers/hwmon/max127.c
@@ -285,7 +285,7 @@ static const struct hwmon_ops max127_hwmon_ops = {
 	.write = max127_write,
 };
 
-static const struct hwmon_channel_info *max127_info[] = {
+static const struct hwmon_channel_info * const max127_info[] = {
 	HWMON_CHANNEL_INFO(in,
 			   HWMON_I_INPUT | HWMON_I_MIN | HWMON_I_MAX,
 			   HWMON_I_INPUT | HWMON_I_MIN | HWMON_I_MAX,
diff --git a/drivers/hwmon/max31730.c b/drivers/hwmon/max31730.c
index 746a767c9fc6..924333d89ce4 100644
--- a/drivers/hwmon/max31730.c
+++ b/drivers/hwmon/max31730.c
@@ -248,7 +248,7 @@ static umode_t max31730_is_visible(const void *data,
 	return 0;
 }
 
-static const struct hwmon_channel_info *max31730_info[] = {
+static const struct hwmon_channel_info * const max31730_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/max31760.c b/drivers/hwmon/max31760.c
index 06d5f39dc33d..4489110f109c 100644
--- a/drivers/hwmon/max31760.c
+++ b/drivers/hwmon/max31760.c
@@ -318,7 +318,7 @@ static int max31760_write(struct device *dev, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *max31760_info[] = {
+static const struct hwmon_channel_info * const max31760_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(fan,
diff --git a/drivers/hwmon/max31790.c b/drivers/hwmon/max31790.c
index 20bf5ffadefe..6caba6e8ee8f 100644
--- a/drivers/hwmon/max31790.c
+++ b/drivers/hwmon/max31790.c
@@ -445,7 +445,7 @@ static umode_t max31790_is_visible(const void *data,
 	}
 }
 
-static const struct hwmon_channel_info *max31790_info[] = {
+static const struct hwmon_channel_info * const max31790_info[] = {
 	HWMON_CHANNEL_INFO(fan,
 			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE,
 			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE,
diff --git a/drivers/hwmon/max6620.c b/drivers/hwmon/max6620.c
index 202b6438179d..0f277d945ea2 100644
--- a/drivers/hwmon/max6620.c
+++ b/drivers/hwmon/max6620.c
@@ -401,7 +401,7 @@ error:
 	return ret;
 }
 
-static const struct hwmon_channel_info *max6620_info[] = {
+static const struct hwmon_channel_info * const max6620_info[] = {
 	HWMON_CHANNEL_INFO(fan,
 			   HWMON_F_INPUT | HWMON_F_DIV | HWMON_F_TARGET | HWMON_F_ALARM,
 			   HWMON_F_INPUT | HWMON_F_DIV | HWMON_F_TARGET | HWMON_F_ALARM,
diff --git a/drivers/hwmon/max6621.c b/drivers/hwmon/max6621.c
index 7821132e17fa..0656eb1e7959 100644
--- a/drivers/hwmon/max6621.c
+++ b/drivers/hwmon/max6621.c
@@ -449,7 +449,7 @@ static const struct regmap_config max6621_regmap_config = {
 	.num_reg_defaults = ARRAY_SIZE(max6621_regmap_default),
 };
 
-static const struct hwmon_channel_info *max6621_info[] = {
+static const struct hwmon_channel_info * const max6621_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/max6650.c b/drivers/hwmon/max6650.c
index f8d4534ce172..19e2c762ed2d 100644
--- a/drivers/hwmon/max6650.c
+++ b/drivers/hwmon/max6650.c
@@ -737,7 +737,7 @@ static umode_t max6650_is_visible(const void *_data,
 	return 0;
 }
 
-static const struct hwmon_channel_info *max6650_info[] = {
+static const struct hwmon_channel_info * const max6650_info[] = {
 	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_DIV |
 			   HWMON_F_MIN_ALARM | HWMON_F_MAX_ALARM |
 			   HWMON_F_FAULT,
diff --git a/drivers/hwmon/mc34vr500.c b/drivers/hwmon/mc34vr500.c
index 6268e973049c..6a7a950a9332 100644
--- a/drivers/hwmon/mc34vr500.c
+++ b/drivers/hwmon/mc34vr500.c
@@ -138,7 +138,7 @@ static int mc34vr500_read(struct device *dev, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *mc34vr500_info[] = {
+static const struct hwmon_channel_info * const mc34vr500_info[] = {
 	HWMON_CHANNEL_INFO(in, HWMON_I_MIN_ALARM),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM
 			   | HWMON_T_EMERGENCY_ALARM),
diff --git a/drivers/hwmon/mcp3021.c b/drivers/hwmon/mcp3021.c
index e093b1998296..a5f7a294f33d 100644
--- a/drivers/hwmon/mcp3021.c
+++ b/drivers/hwmon/mcp3021.c
@@ -102,7 +102,7 @@ static umode_t mcp3021_is_visible(const void *_data,
 	return 0444;
 }
 
-static const struct hwmon_channel_info *mcp3021_info[] = {
+static const struct hwmon_channel_info * const mcp3021_info[] = {
 	HWMON_CHANNEL_INFO(in, HWMON_I_INPUT),
 	NULL
 };
diff --git a/drivers/hwmon/mlxreg-fan.c b/drivers/hwmon/mlxreg-fan.c
index 96017cc8da7e..c2a96468c9b4 100644
--- a/drivers/hwmon/mlxreg-fan.c
+++ b/drivers/hwmon/mlxreg-fan.c
@@ -285,7 +285,7 @@ static char *mlxreg_fan_name[] = {
 	"mlxreg_fan3",
 };
 
-static const struct hwmon_channel_info *mlxreg_fan_hwmon_info[] = {
+static const struct hwmon_channel_info * const mlxreg_fan_hwmon_info[] = {
 	HWMON_CHANNEL_INFO(fan,
 			   HWMON_F_INPUT | HWMON_F_FAULT,
 			   HWMON_F_INPUT | HWMON_F_FAULT,
diff --git a/drivers/hwmon/nct6775-platform.c b/drivers/hwmon/nct6775-platform.c
index 76c6b564d7fc..5782acfb4ee1 100644
--- a/drivers/hwmon/nct6775-platform.c
+++ b/drivers/hwmon/nct6775-platform.c
@@ -149,7 +149,7 @@ static int nct6775_asuswmi_evaluate_method(u32 method_id, u8 bank, u8 reg, u8 va
 		return -EIO;
 
 	if (retval)
-		*retval = (u32)result & 0xFFFFFFFF;
+		*retval = result;
 
 	return 0;
 #else
@@ -1052,33 +1052,117 @@ static int __init nct6775_find(int sioaddr, struct nct6775_sio_data *sio_data)
 static struct platform_device *pdev[2];
 
 static const char * const asus_wmi_boards[] = {
-	"PRO H410T",
-	"ProArt B550-CREATOR",
-	"ProArt X570-CREATOR WIFI",
-	"ProArt Z490-CREATOR 10G",
-	"Pro B550M-C",
-	"Pro WS X570-ACE",
+	"B360M-BASALT",
+	"B360M-D3H",
+	"EX-B360M-V",
+	"EX-B360M-V3",
+	"EX-B360M-V5",
+	"EX-B460M-V5",
+	"EX-H410M-V3",
+	"PRIME A520M-A",
+	"PRIME A520M-A II",
+	"PRIME A520M-E",
+	"PRIME A520M-K",
 	"PRIME B360-PLUS",
+	"PRIME B360M-A",
+	"PRIME B360M-C",
+	"PRIME B360M-D",
+	"PRIME B360M-K",
 	"PRIME B460-PLUS",
+	"PRIME B460I-PLUS",
+	"PRIME B460M-A",
+	"PRIME B460M-A R2.0",
+	"PRIME B460M-K",
 	"PRIME B550-PLUS",
+	"PRIME B550-PLUS AC-HES",
 	"PRIME B550M-A",
 	"PRIME B550M-A (WI-FI)",
+	"PRIME B550M-A AC",
+	"PRIME B550M-A WIFI II",
+	"PRIME B550M-K",
+	"PRIME H310-PLUS",
+	"PRIME H310I-PLUS",
+	"PRIME H310M-A",
+	"PRIME H310M-C",
+	"PRIME H310M-D",
+	"PRIME H310M-DASH",
+	"PRIME H310M-E",
+	"PRIME H310M-E/BR",
+	"PRIME H310M-F",
+	"PRIME H310M-K",
+	"PRIME H310T",
+	"PRIME H370-A",
+	"PRIME H370-PLUS",
+	"PRIME H370M-PLUS",
+	"PRIME H410I-PLUS",
+	"PRIME H410M-A",
+	"PRIME H410M-D",
+	"PRIME H410M-E",
+	"PRIME H410M-F",
+	"PRIME H410M-K",
+	"PRIME H410M-K R2.0",
 	"PRIME H410M-R",
+	"PRIME H470-PLUS",
+	"PRIME H470M-PLUS",
+	"PRIME H510M-K R2.0",
+	"PRIME Q370M-C",
 	"PRIME X570-P",
 	"PRIME X570-PRO",
+	"PRIME Z390-A",
+	"PRIME Z390-A/H10",
+	"PRIME Z390-P",
+	"PRIME Z390M-PLUS",
+	"PRIME Z490-A",
+	"PRIME Z490-P",
+	"PRIME Z490-V",
+	"PRIME Z490M-PLUS",
+	"PRO B460M-C",
+	"PRO H410M-C",
+	"PRO H410T",
+	"PRO Q470M-C",
+	"Pro A520M-C",
+	"Pro A520M-C II",
+	"Pro B550M-C",
+	"Pro WS X570-ACE",
+	"ProArt B550-CREATOR",
+	"ProArt X570-CREATOR WIFI",
+	"ProArt Z490-CREATOR 10G",
 	"ROG CROSSHAIR VIII DARK HERO",
 	"ROG CROSSHAIR VIII EXTREME",
 	"ROG CROSSHAIR VIII FORMULA",
 	"ROG CROSSHAIR VIII HERO",
 	"ROG CROSSHAIR VIII HERO (WI-FI)",
 	"ROG CROSSHAIR VIII IMPACT",
+	"ROG MAXIMUS XI APEX",
+	"ROG MAXIMUS XI CODE",
+	"ROG MAXIMUS XI EXTREME",
+	"ROG MAXIMUS XI FORMULA",
+	"ROG MAXIMUS XI GENE",
+	"ROG MAXIMUS XI HERO",
+	"ROG MAXIMUS XI HERO (WI-FI)",
+	"ROG MAXIMUS XII APEX",
+	"ROG MAXIMUS XII EXTREME",
+	"ROG MAXIMUS XII FORMULA",
+	"ROG MAXIMUS XII HERO (WI-FI)",
+	"ROG STRIX B360-F GAMING",
+	"ROG STRIX B360-G GAMING",
+	"ROG STRIX B360-H GAMING",
+	"ROG STRIX B360-H GAMING/OPTANE",
+	"ROG STRIX B360-I GAMING",
+	"ROG STRIX B460-F GAMING",
+	"ROG STRIX B460-G GAMING",
+	"ROG STRIX B460-H GAMING",
+	"ROG STRIX B460-I GAMING",
 	"ROG STRIX B550-A GAMING",
 	"ROG STRIX B550-E GAMING",
 	"ROG STRIX B550-F GAMING",
 	"ROG STRIX B550-F GAMING (WI-FI)",
 	"ROG STRIX B550-F GAMING WIFI II",
 	"ROG STRIX B550-I GAMING",
-	"ROG STRIX B550-XE GAMING (WI-FI)",
+	"ROG STRIX B550-XE GAMING WIFI",
+	"ROG STRIX H370-F GAMING",
+	"ROG STRIX H370-I GAMING",
+	"ROG STRIX H470-I GAMING",
 	"ROG STRIX X570-E GAMING",
 	"ROG STRIX X570-E GAMING WIFI II",
 	"ROG STRIX X570-F GAMING",
@@ -1094,65 +1178,248 @@ static const char * const asus_wmi_boards[] = {
 	"ROG STRIX Z490-G GAMING (WI-FI)",
 	"ROG STRIX Z490-H GAMING",
 	"ROG STRIX Z490-I GAMING",
+	"TUF B360-PLUS GAMING",
+	"TUF B360-PRO GAMING",
+	"TUF B360-PRO GAMING (WI-FI)",
+	"TUF B360M-E GAMING",
+	"TUF B360M-PLUS GAMING",
+	"TUF B360M-PLUS GAMING S",
+	"TUF B360M-PLUS GAMING/BR",
+	"TUF GAMING A520M-PLUS",
+	"TUF GAMING A520M-PLUS II",
+	"TUF GAMING A520M-PLUS WIFI",
+	"TUF GAMING B460-PLUS",
+	"TUF GAMING B460-PRO (WI-FI)",
+	"TUF GAMING B460M-PLUS",
+	"TUF GAMING B460M-PLUS (WI-FI)",
+	"TUF GAMING B460M-PRO",
+	"TUF GAMING B550-PLUS",
+	"TUF GAMING B550-PLUS (WI-FI)",
+	"TUF GAMING B550-PLUS WIFI II",
+	"TUF GAMING B550-PRO",
+	"TUF GAMING B550M ZAKU (WI-FI)",
 	"TUF GAMING B550M-E",
-	"TUF GAMING B550M-E (WI-FI)",
+	"TUF GAMING B550M-E WIFI",
 	"TUF GAMING B550M-PLUS",
 	"TUF GAMING B550M-PLUS (WI-FI)",
 	"TUF GAMING B550M-PLUS WIFI II",
-	"TUF GAMING B550-PLUS",
-	"TUF GAMING B550-PLUS WIFI II",
-	"TUF GAMING B550-PRO",
+	"TUF GAMING H470-PRO",
+	"TUF GAMING H470-PRO (WI-FI)",
 	"TUF GAMING X570-PLUS",
 	"TUF GAMING X570-PLUS (WI-FI)",
+	"TUF GAMING X570-PLUS_BR",
 	"TUF GAMING X570-PRO (WI-FI)",
+	"TUF GAMING X570-PRO WIFI II",
 	"TUF GAMING Z490-PLUS",
 	"TUF GAMING Z490-PLUS (WI-FI)",
+	"TUF H310-PLUS GAMING",
+	"TUF H310M-PLUS GAMING",
+	"TUF H310M-PLUS GAMING/BR",
+	"TUF H370-PRO GAMING",
+	"TUF H370-PRO GAMING (WI-FI)",
+	"TUF Z390-PLUS GAMING",
+	"TUF Z390-PLUS GAMING (WI-FI)",
+	"TUF Z390-PRO GAMING",
+	"TUF Z390M-PRO GAMING",
+	"TUF Z390M-PRO GAMING (WI-FI)",
+	"WS Z390 PRO",
+	"Z490-GUNDAM (WI-FI)",
 };
 
 static const char * const asus_msi_boards[] = {
+	"B560M-P",
+	"EX-B560M-V5",
+	"EX-B660M-V5 D4",
 	"EX-B660M-V5 PRO D4",
+	"EX-B760M-V5 D4",
+	"EX-H510M-V3",
+	"EX-H610M-V3 D4",
+	"PRIME A620M-A",
+	"PRIME B560-PLUS",
+	"PRIME B560-PLUS AC-HES",
+	"PRIME B560M-A",
+	"PRIME B560M-A AC",
+	"PRIME B560M-K",
 	"PRIME B650-PLUS",
 	"PRIME B650M-A",
 	"PRIME B650M-A AX",
+	"PRIME B650M-A AX II",
 	"PRIME B650M-A II",
 	"PRIME B650M-A WIFI",
 	"PRIME B650M-A WIFI II",
+	"PRIME B660-PLUS D4",
+	"PRIME B660M-A AC D4",
 	"PRIME B660M-A D4",
 	"PRIME B660M-A WIFI D4",
+	"PRIME B760-PLUS",
+	"PRIME B760-PLUS D4",
+	"PRIME B760M-A",
+	"PRIME B760M-A AX D4",
+	"PRIME B760M-A D4",
+	"PRIME B760M-A WIFI",
+	"PRIME B760M-A WIFI D4",
+	"PRIME B760M-AJ D4",
+	"PRIME B760M-K D4",
+	"PRIME H510M-A",
+	"PRIME H510M-A WIFI",
+	"PRIME H510M-D",
+	"PRIME H510M-E",
+	"PRIME H510M-F",
+	"PRIME H510M-K",
+	"PRIME H510M-R",
+	"PRIME H510T2/CSM",
+	"PRIME H570-PLUS",
+	"PRIME H570M-PLUS",
+	"PRIME H610I-PLUS D4",
+	"PRIME H610M-A D4",
+	"PRIME H610M-A WIFI D4",
+	"PRIME H610M-D D4",
+	"PRIME H610M-E D4",
+	"PRIME H610M-F D4",
+	"PRIME H610M-K D4",
+	"PRIME H610M-R D4",
+	"PRIME H670-PLUS D4",
+	"PRIME H770-PLUS D4",
 	"PRIME X670-P",
 	"PRIME X670-P WIFI",
 	"PRIME X670E-PRO WIFI",
-	"Pro B660M-C-D4",
+	"PRIME Z590-A",
+	"PRIME Z590-P",
+	"PRIME Z590-P WIFI",
+	"PRIME Z590-V",
+	"PRIME Z590M-PLUS",
+	"PRIME Z690-A",
+	"PRIME Z690-P",
+	"PRIME Z690-P D4",
+	"PRIME Z690-P WIFI",
+	"PRIME Z690-P WIFI D4",
+	"PRIME Z690M-PLUS D4",
+	"PRIME Z790-A WIFI",
+	"PRIME Z790-P",
+	"PRIME Z790-P D4",
+	"PRIME Z790-P WIFI",
+	"PRIME Z790-P WIFI D4",
+	"PRIME Z790M-PLUS",
+	"PRIME Z790M-PLUS D4",
+	"Pro B560M-C",
+	"Pro B560M-CT",
+	"Pro B660M-C",
+	"Pro B660M-C D4",
+	"Pro B760M-C",
+	"Pro B760M-CT",
+	"Pro H510M-C",
+	"Pro H510M-CT",
+	"Pro H610M-C",
+	"Pro H610M-C D4",
+	"Pro H610M-CT D4",
+	"Pro H610T D4",
+	"Pro Q670M-C",
+	"Pro WS W680-ACE",
+	"Pro WS W680-ACE IPMI",
+	"Pro WS W790-ACE",
+	"Pro WS W790E-SAGE SE",
+	"ProArt B650-CREATOR",
 	"ProArt B660-CREATOR D4",
+	"ProArt B760-CREATOR D4",
 	"ProArt X670E-CREATOR WIFI",
+	"ProArt Z690-CREATOR WIFI",
+	"ProArt Z790-CREATOR WIFI",
 	"ROG CROSSHAIR X670E EXTREME",
 	"ROG CROSSHAIR X670E GENE",
 	"ROG CROSSHAIR X670E HERO",
+	"ROG MAXIMUS XIII APEX",
+	"ROG MAXIMUS XIII EXTREME",
 	"ROG MAXIMUS XIII EXTREME GLACIAL",
+	"ROG MAXIMUS XIII HERO",
+	"ROG MAXIMUS Z690 APEX",
 	"ROG MAXIMUS Z690 EXTREME",
 	"ROG MAXIMUS Z690 EXTREME GLACIAL",
+	"ROG MAXIMUS Z690 FORMULA",
+	"ROG MAXIMUS Z690 HERO",
+	"ROG MAXIMUS Z690 HERO EVA",
+	"ROG MAXIMUS Z790 APEX",
+	"ROG MAXIMUS Z790 EXTREME",
+	"ROG MAXIMUS Z790 HERO",
+	"ROG STRIX B560-A GAMING WIFI",
+	"ROG STRIX B560-E GAMING WIFI",
+	"ROG STRIX B560-F GAMING WIFI",
+	"ROG STRIX B560-G GAMING WIFI",
+	"ROG STRIX B560-I GAMING WIFI",
 	"ROG STRIX B650-A GAMING WIFI",
 	"ROG STRIX B650E-E GAMING WIFI",
 	"ROG STRIX B650E-F GAMING WIFI",
 	"ROG STRIX B650E-I GAMING WIFI",
+	"ROG STRIX B660-A GAMING WIFI",
 	"ROG STRIX B660-A GAMING WIFI D4",
 	"ROG STRIX B660-F GAMING WIFI",
 	"ROG STRIX B660-G GAMING WIFI",
 	"ROG STRIX B660-I GAMING WIFI",
+	"ROG STRIX B760-A GAMING WIFI",
+	"ROG STRIX B760-A GAMING WIFI D4",
+	"ROG STRIX B760-F GAMING WIFI",
+	"ROG STRIX B760-G GAMING WIFI",
+	"ROG STRIX B760-G GAMING WIFI D4",
+	"ROG STRIX B760-I GAMING WIFI",
 	"ROG STRIX X670E-A GAMING WIFI",
 	"ROG STRIX X670E-E GAMING WIFI",
 	"ROG STRIX X670E-F GAMING WIFI",
 	"ROG STRIX X670E-I GAMING WIFI",
+	"ROG STRIX Z590-A GAMING WIFI",
 	"ROG STRIX Z590-A GAMING WIFI II",
+	"ROG STRIX Z590-E GAMING WIFI",
+	"ROG STRIX Z590-F GAMING WIFI",
+	"ROG STRIX Z590-I GAMING WIFI",
+	"ROG STRIX Z690-A GAMING WIFI",
 	"ROG STRIX Z690-A GAMING WIFI D4",
+	"ROG STRIX Z690-E GAMING WIFI",
+	"ROG STRIX Z690-F GAMING WIFI",
+	"ROG STRIX Z690-G GAMING WIFI",
+	"ROG STRIX Z690-I GAMING WIFI",
+	"ROG STRIX Z790-A GAMING WIFI",
+	"ROG STRIX Z790-A GAMING WIFI D4",
+	"ROG STRIX Z790-E GAMING WIFI",
+	"ROG STRIX Z790-F GAMING WIFI",
+	"ROG STRIX Z790-H GAMING WIFI",
+	"ROG STRIX Z790-I GAMING WIFI",
+	"TUF GAMING A620M-PLUS",
+	"TUF GAMING A620M-PLUS WIFI",
+	"TUF GAMING B560-PLUS WIFI",
+	"TUF GAMING B560M-E",
+	"TUF GAMING B560M-PLUS",
+	"TUF GAMING B560M-PLUS WIFI",
 	"TUF GAMING B650-PLUS",
 	"TUF GAMING B650-PLUS WIFI",
 	"TUF GAMING B650M-PLUS",
 	"TUF GAMING B650M-PLUS WIFI",
+	"TUF GAMING B660-PLUS WIFI D4",
+	"TUF GAMING B660M-E D4",
+	"TUF GAMING B660M-PLUS D4",
 	"TUF GAMING B660M-PLUS WIFI",
+	"TUF GAMING B660M-PLUS WIFI D4",
+	"TUF GAMING B760-PLUS WIFI",
+	"TUF GAMING B760-PLUS WIFI D4",
+	"TUF GAMING B760M-BTF WIFI D4",
+	"TUF GAMING B760M-E D4",
+	"TUF GAMING B760M-PLUS",
+	"TUF GAMING B760M-PLUS D4",
+	"TUF GAMING B760M-PLUS WIFI",
+	"TUF GAMING B760M-PLUS WIFI D4",
+	"TUF GAMING H570-PRO",
+	"TUF GAMING H570-PRO WIFI",
+	"TUF GAMING H670-PRO WIFI D4",
+	"TUF GAMING H770-PRO WIFI",
 	"TUF GAMING X670E-PLUS",
 	"TUF GAMING X670E-PLUS WIFI",
+	"TUF GAMING Z590-PLUS",
 	"TUF GAMING Z590-PLUS WIFI",
+	"TUF GAMING Z690-PLUS",
+	"TUF GAMING Z690-PLUS D4",
+	"TUF GAMING Z690-PLUS WIFI",
+	"TUF GAMING Z690-PLUS WIFI D4",
+	"TUF GAMING Z790-PLUS D4",
+	"TUF GAMING Z790-PLUS WIFI",
+	"TUF GAMING Z790-PLUS WIFI D4",
+	"Z590 WIFI GUNDAM EDITION",
 };
 
 #if IS_ENABLED(CONFIG_ACPI)
diff --git a/drivers/hwmon/nct7904.c b/drivers/hwmon/nct7904.c
index ecc5db0011a3..007bae4c7028 100644
--- a/drivers/hwmon/nct7904.c
+++ b/drivers/hwmon/nct7904.c
@@ -803,7 +803,7 @@ static int nct7904_detect(struct i2c_client *client,
 	return 0;
 }
 
-static const struct hwmon_channel_info *nct7904_info[] = {
+static const struct hwmon_channel_info * const nct7904_info[] = {
 	HWMON_CHANNEL_INFO(in,
 			   /* dummy, skipped in is_visible */
 			   HWMON_I_INPUT | HWMON_I_MIN | HWMON_I_MAX |
diff --git a/drivers/hwmon/npcm750-pwm-fan.c b/drivers/hwmon/npcm750-pwm-fan.c
index 11a28609da3c..10ed3f4335d4 100644
--- a/drivers/hwmon/npcm750-pwm-fan.c
+++ b/drivers/hwmon/npcm750-pwm-fan.c
@@ -629,7 +629,7 @@ static umode_t npcm7xx_is_visible(const void *data,
 	}
 }
 
-static const struct hwmon_channel_info *npcm7xx_info[] = {
+static const struct hwmon_channel_info * const npcm7xx_info[] = {
 	HWMON_CHANNEL_INFO(pwm,
 			   HWMON_PWM_INPUT,
 			   HWMON_PWM_INPUT,
diff --git a/drivers/hwmon/ntc_thermistor.c b/drivers/hwmon/ntc_thermistor.c
index 9c9e9f4ccb9e..ef75b63f5894 100644
--- a/drivers/hwmon/ntc_thermistor.c
+++ b/drivers/hwmon/ntc_thermistor.c
@@ -546,7 +546,7 @@ static umode_t ntc_is_visible(const void *data, enum hwmon_sensor_types type,
 	return 0;
 }
 
-static const struct hwmon_channel_info *ntc_info[] = {
+static const struct hwmon_channel_info * const ntc_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_TYPE),
 	NULL
diff --git a/drivers/hwmon/nzxt-kraken2.c b/drivers/hwmon/nzxt-kraken2.c
index 89f7ea4f42d4..428c77b5fce5 100644
--- a/drivers/hwmon/nzxt-kraken2.c
+++ b/drivers/hwmon/nzxt-kraken2.c
@@ -86,7 +86,7 @@ static const struct hwmon_ops kraken2_hwmon_ops = {
 	.read_string = kraken2_read_string,
 };
 
-static const struct hwmon_channel_info *kraken2_info[] = {
+static const struct hwmon_channel_info * const kraken2_info[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_LABEL),
 	HWMON_CHANNEL_INFO(fan,
diff --git a/drivers/hwmon/nzxt-smart2.c b/drivers/hwmon/nzxt-smart2.c
index 2b93ba89610a..7aa586eb74be 100644
--- a/drivers/hwmon/nzxt-smart2.c
+++ b/drivers/hwmon/nzxt-smart2.c
@@ -663,7 +663,7 @@ static const struct hwmon_ops nzxt_smart2_hwmon_ops = {
 	.write = nzxt_smart2_hwmon_write,
 };
 
-static const struct hwmon_channel_info *nzxt_smart2_channel_info[] = {
+static const struct hwmon_channel_info * const nzxt_smart2_channel_info[] = {
 	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_LABEL,
 			   HWMON_F_INPUT | HWMON_F_LABEL,
 			   HWMON_F_INPUT | HWMON_F_LABEL),
@@ -721,6 +721,11 @@ static int __maybe_unused nzxt_smart2_hid_reset_resume(struct hid_device *hdev)
 	return init_device(drvdata, drvdata->update_interval);
 }
 
+static void mutex_fini(void *lock)
+{
+	mutex_destroy(lock);
+}
+
 static int nzxt_smart2_hid_probe(struct hid_device *hdev,
 				 const struct hid_device_id *id)
 {
@@ -737,8 +742,9 @@ static int nzxt_smart2_hid_probe(struct hid_device *hdev,
 	init_waitqueue_head(&drvdata->wq);
 
 	mutex_init(&drvdata->mutex);
-	devm_add_action(&hdev->dev, (void (*)(void *))mutex_destroy,
-			&drvdata->mutex);
+	ret = devm_add_action_or_reset(&hdev->dev, mutex_fini, &drvdata->mutex);
+	if (ret)
+		return ret;
 
 	ret = hid_parse(hdev);
 	if (ret)
@@ -791,7 +797,8 @@ static const struct hid_device_id nzxt_smart2_hid_id_table[] = {
 	{ HID_USB_DEVICE(0x1e71, 0x2009) }, /* NZXT RGB & Fan Controller */
 	{ HID_USB_DEVICE(0x1e71, 0x200e) }, /* NZXT RGB & Fan Controller */
 	{ HID_USB_DEVICE(0x1e71, 0x2010) }, /* NZXT RGB & Fan Controller */
-	{ HID_USB_DEVICE(0x1e71, 0x2019) }, /* NZXT RGB & Fan Controller */
+	{ HID_USB_DEVICE(0x1e71, 0x2011) }, /* NZXT RGB & Fan Controller (6 RGB) */
+	{ HID_USB_DEVICE(0x1e71, 0x2019) }, /* NZXT RGB & Fan Controller (6 RGB) */
 	{},
 };
 
diff --git a/drivers/hwmon/oxp-sensors.c b/drivers/hwmon/oxp-sensors.c
index 36872b57912a..ae67207030e8 100644
--- a/drivers/hwmon/oxp-sensors.c
+++ b/drivers/hwmon/oxp-sensors.c
@@ -239,7 +239,7 @@ static int oxp_platform_write(struct device *dev, enum hwmon_sensor_types type,
 }
 
 /* Known sensors in the OXP EC controllers */
-static const struct hwmon_channel_info *oxp_platform_sensors[] = {
+static const struct hwmon_channel_info * const oxp_platform_sensors[] = {
 	HWMON_CHANNEL_INFO(fan,
 			   HWMON_F_INPUT),
 	HWMON_CHANNEL_INFO(pwm,
diff --git a/drivers/hwmon/peci/cputemp.c b/drivers/hwmon/peci/cputemp.c
index 87d56f0fc888..e5b65a382772 100644
--- a/drivers/hwmon/peci/cputemp.c
+++ b/drivers/hwmon/peci/cputemp.c
@@ -447,7 +447,7 @@ static const struct hwmon_ops peci_cputemp_ops = {
 	.read = cputemp_read,
 };
 
-static const struct hwmon_channel_info *peci_cputemp_info[] = {
+static const struct hwmon_channel_info * const peci_cputemp_info[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   /* Die temperature */
 			   HWMON_T_LABEL | HWMON_T_INPUT | HWMON_T_MAX |
diff --git a/drivers/hwmon/peci/dimmtemp.c b/drivers/hwmon/peci/dimmtemp.c
index 0a633bda3668..ed968401f93c 100644
--- a/drivers/hwmon/peci/dimmtemp.c
+++ b/drivers/hwmon/peci/dimmtemp.c
@@ -300,7 +300,7 @@ static int create_dimm_temp_label(struct peci_dimmtemp *priv, int chan)
 	return 0;
 }
 
-static const struct hwmon_channel_info *peci_dimmtemp_temp_info[] = {
+static const struct hwmon_channel_info * const peci_dimmtemp_temp_info[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   [0 ... DIMM_NUMS_MAX - 1] = HWMON_T_LABEL |
 				HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT),
diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig
index 59d9a7430499..270b6336b76d 100644
--- a/drivers/hwmon/pmbus/Kconfig
+++ b/drivers/hwmon/pmbus/Kconfig
@@ -27,6 +27,15 @@ config SENSORS_PMBUS
 	  This driver can also be built as a module. If so, the module will
 	  be called pmbus.
 
+config SENSORS_ACBEL_FSG032
+	tristate "ACBEL FSG032 Power Supply"
+	help
+	  If you say yes here you get hardware monitoring support for the ACBEL
+	  FSG032 Power Supply.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called acbel-fsg032.
+
 config SENSORS_ADM1266
 	tristate "Analog Devices ADM1266 Sequencer"
 	select CRC8
diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile
index 3ae019916267..84ee960a6c2d 100644
--- a/drivers/hwmon/pmbus/Makefile
+++ b/drivers/hwmon/pmbus/Makefile
@@ -5,6 +5,7 @@
 
 obj-$(CONFIG_PMBUS)		+= pmbus_core.o
 obj-$(CONFIG_SENSORS_PMBUS)	+= pmbus.o
+obj-$(CONFIG_SENSORS_ACBEL_FSG032) += acbel-fsg032.o
 obj-$(CONFIG_SENSORS_ADM1266)	+= adm1266.o
 obj-$(CONFIG_SENSORS_ADM1275)	+= adm1275.o
 obj-$(CONFIG_SENSORS_BEL_PFE)	+= bel-pfe.o
diff --git a/drivers/hwmon/pmbus/acbel-fsg032.c b/drivers/hwmon/pmbus/acbel-fsg032.c
new file mode 100644
index 000000000000..28a25f30e2cf
--- /dev/null
+++ b/drivers/hwmon/pmbus/acbel-fsg032.c
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2023 IBM Corp.
+ */
+
+#include <linux/device.h>
+#include <linux/fs.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/pmbus.h>
+#include <linux/hwmon-sysfs.h>
+#include "pmbus.h"
+
+static const struct i2c_device_id acbel_fsg032_id[] = {
+	{ "acbel_fsg032" },
+	{}
+};
+
+static struct pmbus_driver_info acbel_fsg032_info = {
+	.pages = 1,
+	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_IIN | PMBUS_HAVE_PIN |
+		   PMBUS_HAVE_VOUT | PMBUS_HAVE_IOUT | PMBUS_HAVE_POUT |
+		   PMBUS_HAVE_TEMP | PMBUS_HAVE_TEMP2 | PMBUS_HAVE_TEMP3 |
+		   PMBUS_HAVE_FAN12 | PMBUS_HAVE_STATUS_VOUT |
+		   PMBUS_HAVE_STATUS_IOUT | PMBUS_HAVE_STATUS_TEMP |
+		   PMBUS_HAVE_STATUS_INPUT | PMBUS_HAVE_STATUS_FAN12,
+};
+
+static int acbel_fsg032_probe(struct i2c_client *client)
+{
+	u8 buf[I2C_SMBUS_BLOCK_MAX + 1];
+	struct device *dev = &client->dev;
+	int rc;
+
+	rc = i2c_smbus_read_block_data(client, PMBUS_MFR_ID, buf);
+	if (rc < 0) {
+		dev_err(dev, "Failed to read PMBUS_MFR_ID\n");
+		return rc;
+	}
+	if (strncmp(buf, "ACBEL", 5)) {
+		buf[rc] = '\0';
+		dev_err(dev, "Manufacturer '%s' not supported\n", buf);
+		return -ENODEV;
+	}
+
+	rc = i2c_smbus_read_block_data(client, PMBUS_MFR_MODEL, buf);
+	if (rc < 0) {
+		dev_err(dev, "Failed to read PMBUS_MFR_MODEL\n");
+		return rc;
+	}
+
+	if (strncmp(buf, "FSG032", 6)) {
+		buf[rc] = '\0';
+		dev_err(dev, "Model '%s' not supported\n", buf);
+		return -ENODEV;
+	}
+
+	rc = pmbus_do_probe(client, &acbel_fsg032_info);
+	if (rc)
+		return rc;
+
+	return 0;
+}
+
+static const struct of_device_id acbel_fsg032_of_match[] = {
+	{ .compatible = "acbel,fsg032" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, acbel_fsg032_of_match);
+
+static struct i2c_driver acbel_fsg032_driver = {
+	.driver = {
+		.name = "acbel-fsg032",
+		.of_match_table = acbel_fsg032_of_match,
+	},
+	.probe_new = acbel_fsg032_probe,
+	.id_table = acbel_fsg032_id,
+};
+
+module_i2c_driver(acbel_fsg032_driver);
+
+MODULE_AUTHOR("Lakshmi Yadlapati");
+MODULE_DESCRIPTION("PMBus driver for AcBel Power System power supplies");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS(PMBUS);
diff --git a/drivers/hwmon/pmbus/fsp-3y.c b/drivers/hwmon/pmbus/fsp-3y.c
index aec294cc72d1..c7469d2cdedc 100644
--- a/drivers/hwmon/pmbus/fsp-3y.c
+++ b/drivers/hwmon/pmbus/fsp-3y.c
@@ -180,7 +180,6 @@ static struct pmbus_driver_info fsp3y_info[] = {
 			PMBUS_HAVE_FAN12,
 		.func[YM2151_PAGE_5VSB_LOG] =
 			PMBUS_HAVE_VOUT | PMBUS_HAVE_IOUT,
-			PMBUS_HAVE_IIN,
 		.read_word_data = fsp3y_read_word_data,
 		.read_byte_data = fsp3y_read_byte_data,
 	},
diff --git a/drivers/hwmon/pmbus/ibm-cffps.c b/drivers/hwmon/pmbus/ibm-cffps.c
index e3294a1a54bb..76e72e9acda7 100644
--- a/drivers/hwmon/pmbus/ibm-cffps.c
+++ b/drivers/hwmon/pmbus/ibm-cffps.c
@@ -18,12 +18,6 @@
 
 #include "pmbus.h"
 
-#define CFFPS_MFG_ID_CMD                        0x99
-#define CFFPS_FRU_CMD				0x9A
-#define CFFPS_PN_CMD				0x9B
-#define CFFPS_HEADER_CMD			0x9C
-#define CFFPS_SN_CMD				0x9E
-#define CFFPS_MAX_POWER_OUT_CMD			0xA7
 #define CFFPS_CCIN_CMD				0xBD
 #define CFFPS_FW_CMD				0xFA
 #define CFFPS1_FW_NUM_BYTES			4
@@ -32,7 +26,7 @@
 #define CFFPS_12VCS_VOUT_CMD			0xDE
 
 #define CFFPS_INPUT_HISTORY_CMD			0xD6
-#define CFFPS_INPUT_HISTORY_SIZE		100
+#define CFFPS_INPUT_HISTORY_SIZE		101
 
 #define CFFPS_CCIN_REVISION			GENMASK(7, 0)
 #define CFFPS_CCIN_REVISION_LEGACY		 0xde
@@ -57,13 +51,7 @@
 #define CFFPS_BLINK_RATE_MS			250
 
 enum {
-	CFFPS_DEBUGFS_INPUT_HISTORY = 0,
-	CFFPS_DEBUGFS_MFG_ID,
-	CFFPS_DEBUGFS_FRU,
-	CFFPS_DEBUGFS_PN,
-	CFFPS_DEBUGFS_HEADER,
-	CFFPS_DEBUGFS_SN,
-	CFFPS_DEBUGFS_MAX_POWER_OUT,
+	CFFPS_DEBUGFS_MAX_POWER_OUT = 0,
 	CFFPS_DEBUGFS_CCIN,
 	CFFPS_DEBUGFS_FW,
 	CFFPS_DEBUGFS_ON_OFF_CONFIG,
@@ -72,19 +60,11 @@ enum {
 
 enum versions { cffps1, cffps2, cffps_unknown };
 
-struct ibm_cffps_input_history {
-	struct mutex update_lock;
-	unsigned long last_update;
-
-	u8 byte_count;
-	u8 data[CFFPS_INPUT_HISTORY_SIZE];
-};
-
 struct ibm_cffps {
 	enum versions version;
 	struct i2c_client *client;
 
-	struct ibm_cffps_input_history input_history;
+	u8 input_history[CFFPS_INPUT_HISTORY_SIZE];
 
 	int debugfs_entries[CFFPS_DEBUGFS_NUM_ENTRIES];
 
@@ -93,118 +73,98 @@ struct ibm_cffps {
 	struct led_classdev led;
 };
 
-static const struct i2c_device_id ibm_cffps_id[];
-
 #define to_psu(x, y) container_of((x), struct ibm_cffps, debugfs_entries[(y)])
 
-static ssize_t ibm_cffps_read_input_history(struct ibm_cffps *psu,
-					    char __user *buf, size_t count,
-					    loff_t *ppos)
+static ssize_t ibm_cffps_debugfs_read_input_history(struct file *file, char __user *buf,
+						    size_t count, loff_t *ppos)
 {
 	int rc;
-	u8 msgbuf0[1] = { CFFPS_INPUT_HISTORY_CMD };
-	u8 msgbuf1[CFFPS_INPUT_HISTORY_SIZE + 1] = { 0 };
+	u8 cmd = CFFPS_INPUT_HISTORY_CMD;
+	struct ibm_cffps *psu = file->private_data;
 	struct i2c_msg msg[2] = {
 		{
 			.addr = psu->client->addr,
 			.flags = psu->client->flags,
 			.len = 1,
-			.buf = msgbuf0,
+			.buf = &cmd,
 		}, {
 			.addr = psu->client->addr,
 			.flags = psu->client->flags | I2C_M_RD,
-			.len = CFFPS_INPUT_HISTORY_SIZE + 1,
-			.buf = msgbuf1,
+			.len = CFFPS_INPUT_HISTORY_SIZE,
+			.buf = psu->input_history,
 		},
 	};
 
 	if (!*ppos) {
-		mutex_lock(&psu->input_history.update_lock);
-		if (time_after(jiffies, psu->input_history.last_update + HZ)) {
-			/*
-			 * Use a raw i2c transfer, since we need more bytes
-			 * than Linux I2C supports through smbus xfr (only 32).
-			 */
-			rc = i2c_transfer(psu->client->adapter, msg, 2);
-			if (rc < 0) {
-				mutex_unlock(&psu->input_history.update_lock);
-				return rc;
-			}
+		rc = pmbus_lock_interruptible(psu->client);
+		if (rc)
+			return rc;
 
-			psu->input_history.byte_count = msgbuf1[0];
-			memcpy(psu->input_history.data, &msgbuf1[1],
-			       CFFPS_INPUT_HISTORY_SIZE);
-			psu->input_history.last_update = jiffies;
+		rc = pmbus_set_page(psu->client, 0, 0xff);
+		if (rc) {
+			pmbus_unlock(psu->client);
+			return rc;
 		}
 
-		mutex_unlock(&psu->input_history.update_lock);
+		/*
+		 * Use a raw i2c transfer, since we need more bytes
+		 * than Linux I2C supports through smbus xfr (only 32).
+		 */
+		rc = i2c_transfer(psu->client->adapter, msg, 2);
+		pmbus_unlock(psu->client);
+		if (rc < 0)
+			return rc;
 	}
 
 	return simple_read_from_buffer(buf, count, ppos,
-				       psu->input_history.data,
-				       psu->input_history.byte_count);
+				       psu->input_history + 1,
+				       psu->input_history[0]);
 }
 
+static const struct file_operations ibm_cffps_input_history_fops = {
+	.llseek = noop_llseek,
+	.read = ibm_cffps_debugfs_read_input_history,
+	.open = simple_open,
+};
+
 static ssize_t ibm_cffps_debugfs_read(struct file *file, char __user *buf,
 				      size_t count, loff_t *ppos)
 {
-	u8 cmd;
 	int i, rc;
 	int *idxp = file->private_data;
 	int idx = *idxp;
 	struct ibm_cffps *psu = to_psu(idxp, idx);
 	char data[I2C_SMBUS_BLOCK_MAX + 2] = { 0 };
 
-	pmbus_set_page(psu->client, 0, 0xff);
+	rc = pmbus_lock_interruptible(psu->client);
+	if (rc)
+		return rc;
+
+	rc = pmbus_set_page(psu->client, 0, 0xff);
+	if (rc)
+		goto unlock;
 
 	switch (idx) {
-	case CFFPS_DEBUGFS_INPUT_HISTORY:
-		return ibm_cffps_read_input_history(psu, buf, count, ppos);
-	case CFFPS_DEBUGFS_MFG_ID:
-		cmd = CFFPS_MFG_ID_CMD;
-		break;
-	case CFFPS_DEBUGFS_FRU:
-		cmd = CFFPS_FRU_CMD;
-		break;
-	case CFFPS_DEBUGFS_PN:
-		cmd = CFFPS_PN_CMD;
-		break;
-	case CFFPS_DEBUGFS_HEADER:
-		cmd = CFFPS_HEADER_CMD;
-		break;
-	case CFFPS_DEBUGFS_SN:
-		cmd = CFFPS_SN_CMD;
-		break;
 	case CFFPS_DEBUGFS_MAX_POWER_OUT:
-		if (psu->version == cffps1) {
-			rc = i2c_smbus_read_word_swapped(psu->client,
-					CFFPS_MAX_POWER_OUT_CMD);
-		} else {
-			rc = i2c_smbus_read_word_data(psu->client,
-					CFFPS_MAX_POWER_OUT_CMD);
-		}
-
-		if (rc < 0)
-			return rc;
-
-		rc = snprintf(data, I2C_SMBUS_BLOCK_MAX, "%d", rc);
-		goto done;
+		if (psu->version == cffps1)
+			rc = i2c_smbus_read_word_swapped(psu->client, PMBUS_MFR_POUT_MAX);
+		else
+			rc = i2c_smbus_read_word_data(psu->client, PMBUS_MFR_POUT_MAX);
+		if (rc >= 0)
+			rc = snprintf(data, I2C_SMBUS_BLOCK_MAX, "%d", rc);
+		break;
 	case CFFPS_DEBUGFS_CCIN:
 		rc = i2c_smbus_read_word_swapped(psu->client, CFFPS_CCIN_CMD);
-		if (rc < 0)
-			return rc;
-
-		rc = snprintf(data, 5, "%04X", rc);
-		goto done;
+		if (rc >= 0)
+			rc = snprintf(data, 5, "%04X", rc);
+		break;
 	case CFFPS_DEBUGFS_FW:
 		switch (psu->version) {
 		case cffps1:
 			for (i = 0; i < CFFPS1_FW_NUM_BYTES; ++i) {
-				rc = i2c_smbus_read_byte_data(psu->client,
-							      CFFPS_FW_CMD +
-								i);
+				rc = i2c_smbus_read_byte_data(psu->client, CFFPS_FW_CMD + i);
 				if (rc < 0)
-					return rc;
+					goto unlock;
 
 				snprintf(&data[i * 2], 3, "%02X", rc);
 			}
@@ -213,11 +173,9 @@ static ssize_t ibm_cffps_debugfs_read(struct file *file, char __user *buf,
 			break;
 		case cffps2:
 			for (i = 0; i < CFFPS2_FW_NUM_WORDS; ++i) {
-				rc = i2c_smbus_read_word_data(psu->client,
-							      CFFPS_FW_CMD +
-								i);
+				rc = i2c_smbus_read_word_data(psu->client, CFFPS_FW_CMD + i);
 				if (rc < 0)
-					return rc;
+					goto unlock;
 
 				snprintf(&data[i * 4], 5, "%04X", rc);
 			}
@@ -225,26 +183,25 @@ static ssize_t ibm_cffps_debugfs_read(struct file *file, char __user *buf,
 			rc = i * 4;
 			break;
 		default:
-			return -EOPNOTSUPP;
+			rc = -EOPNOTSUPP;
+			break;
 		}
-		goto done;
+		break;
 	case CFFPS_DEBUGFS_ON_OFF_CONFIG:
-		rc = i2c_smbus_read_byte_data(psu->client,
-					      PMBUS_ON_OFF_CONFIG);
-		if (rc < 0)
-			return rc;
-
-		rc = snprintf(data, 3, "%02x", rc);
-		goto done;
+		rc = i2c_smbus_read_byte_data(psu->client, PMBUS_ON_OFF_CONFIG);
+		if (rc >= 0)
+			rc = snprintf(data, 3, "%02x", rc);
+		break;
 	default:
-		return -EINVAL;
+		rc = -EINVAL;
+		break;
 	}
 
-	rc = i2c_smbus_read_block_data(psu->client, cmd, data);
+unlock:
+	pmbus_unlock(psu->client);
 	if (rc < 0)
 		return rc;
 
-done:
 	data[rc] = '\n';
 	rc += 2;
 
@@ -263,14 +220,22 @@ static ssize_t ibm_cffps_debugfs_write(struct file *file,
 
 	switch (idx) {
 	case CFFPS_DEBUGFS_ON_OFF_CONFIG:
-		pmbus_set_page(psu->client, 0, 0xff);
-
 		rc = simple_write_to_buffer(&data, 1, ppos, buf, count);
 		if (rc <= 0)
 			return rc;
 
-		rc = i2c_smbus_write_byte_data(psu->client,
-					       PMBUS_ON_OFF_CONFIG, data);
+		rc = pmbus_lock_interruptible(psu->client);
+		if (rc)
+			return rc;
+
+		rc = pmbus_set_page(psu->client, 0, 0xff);
+		if (rc) {
+			pmbus_unlock(psu->client);
+			return rc;
+		}
+
+		rc = i2c_smbus_write_byte_data(psu->client, PMBUS_ON_OFF_CONFIG, data);
+		pmbus_unlock(psu->client);
 		if (rc)
 			return rc;
 
@@ -396,10 +361,19 @@ static int ibm_cffps_led_brightness_set(struct led_classdev *led_cdev,
 	dev_dbg(&psu->client->dev, "LED brightness set: %d. Command: %d.\n",
 		brightness, next_led_state);
 
-	pmbus_set_page(psu->client, 0, 0xff);
+	rc = pmbus_lock_interruptible(psu->client);
+	if (rc)
+		return rc;
+
+	rc = pmbus_set_page(psu->client, 0, 0xff);
+	if (rc) {
+		pmbus_unlock(psu->client);
+		return rc;
+	}
 
 	rc = i2c_smbus_write_byte_data(psu->client, CFFPS_SYS_CONFIG_CMD,
 				       next_led_state);
+	pmbus_unlock(psu->client);
 	if (rc < 0)
 		return rc;
 
@@ -418,10 +392,19 @@ static int ibm_cffps_led_blink_set(struct led_classdev *led_cdev,
 
 	dev_dbg(&psu->client->dev, "LED blink set.\n");
 
-	pmbus_set_page(psu->client, 0, 0xff);
+	rc = pmbus_lock_interruptible(psu->client);
+	if (rc)
+		return rc;
+
+	rc = pmbus_set_page(psu->client, 0, 0xff);
+	if (rc) {
+		pmbus_unlock(psu->client);
+		return rc;
+	}
 
 	rc = i2c_smbus_write_byte_data(psu->client, CFFPS_SYS_CONFIG_CMD,
 				       CFFPS_LED_BLINK);
+	pmbus_unlock(psu->client);
 	if (rc < 0)
 		return rc;
 
@@ -474,9 +457,11 @@ static struct pmbus_driver_info ibm_cffps_info[] = {
 			PMBUS_HAVE_STATUS_VOUT | PMBUS_HAVE_STATUS_IOUT |
 			PMBUS_HAVE_STATUS_INPUT | PMBUS_HAVE_STATUS_TEMP |
 			PMBUS_HAVE_STATUS_FAN12 | PMBUS_HAVE_VMON,
-		.func[1] = PMBUS_HAVE_VOUT | PMBUS_HAVE_IOUT |
-			PMBUS_HAVE_TEMP | PMBUS_HAVE_TEMP2 | PMBUS_HAVE_TEMP3 |
-			PMBUS_HAVE_STATUS_VOUT | PMBUS_HAVE_STATUS_IOUT,
+		.func[1] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_IOUT |
+			PMBUS_HAVE_PIN | PMBUS_HAVE_TEMP | PMBUS_HAVE_TEMP2 |
+			PMBUS_HAVE_TEMP3 | PMBUS_HAVE_STATUS_VOUT |
+			PMBUS_HAVE_STATUS_IOUT | PMBUS_HAVE_STATUS_INPUT |
+			PMBUS_HAVE_STATUS_TEMP,
 		.read_byte_data = ibm_cffps_read_byte_data,
 		.read_word_data = ibm_cffps_read_word_data,
 	},
@@ -486,12 +471,19 @@ static struct pmbus_platform_data ibm_cffps_pdata = {
 	.flags = PMBUS_SKIP_STATUS_CHECK | PMBUS_NO_CAPABILITY,
 };
 
+static const struct i2c_device_id ibm_cffps_id[] = {
+	{ "ibm_cffps1", cffps1 },
+	{ "ibm_cffps2", cffps2 },
+	{ "ibm_cffps", cffps_unknown },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, ibm_cffps_id);
+
 static int ibm_cffps_probe(struct i2c_client *client)
 {
 	int i, rc;
 	enum versions vs = cffps_unknown;
 	struct dentry *debugfs;
-	struct dentry *ibm_cffps_dir;
 	struct ibm_cffps *psu;
 	const void *md = of_device_get_match_data(&client->dev);
 	const struct i2c_device_id *id;
@@ -560,8 +552,6 @@ static int ibm_cffps_probe(struct i2c_client *client)
 
 	psu->version = vs;
 	psu->client = client;
-	mutex_init(&psu->input_history.update_lock);
-	psu->input_history.last_update = jiffies - HZ;
 
 	ibm_cffps_create_led_class(psu);
 
@@ -570,55 +560,29 @@ static int ibm_cffps_probe(struct i2c_client *client)
 	if (!debugfs)
 		return 0;
 
-	ibm_cffps_dir = debugfs_create_dir(client->name, debugfs);
-	if (!ibm_cffps_dir)
-		return 0;
-
 	for (i = 0; i < CFFPS_DEBUGFS_NUM_ENTRIES; ++i)
 		psu->debugfs_entries[i] = i;
 
-	debugfs_create_file("input_history", 0444, ibm_cffps_dir,
-			    &psu->debugfs_entries[CFFPS_DEBUGFS_INPUT_HISTORY],
-			    &ibm_cffps_fops);
-	debugfs_create_file("mfg_id", 0444, ibm_cffps_dir,
-			    &psu->debugfs_entries[CFFPS_DEBUGFS_MFG_ID],
-			    &ibm_cffps_fops);
-	debugfs_create_file("fru", 0444, ibm_cffps_dir,
-			    &psu->debugfs_entries[CFFPS_DEBUGFS_FRU],
-			    &ibm_cffps_fops);
-	debugfs_create_file("part_number", 0444, ibm_cffps_dir,
-			    &psu->debugfs_entries[CFFPS_DEBUGFS_PN],
-			    &ibm_cffps_fops);
-	debugfs_create_file("header", 0444, ibm_cffps_dir,
-			    &psu->debugfs_entries[CFFPS_DEBUGFS_HEADER],
-			    &ibm_cffps_fops);
-	debugfs_create_file("serial_number", 0444, ibm_cffps_dir,
-			    &psu->debugfs_entries[CFFPS_DEBUGFS_SN],
-			    &ibm_cffps_fops);
-	debugfs_create_file("max_power_out", 0444, ibm_cffps_dir,
+	debugfs_create_file("input_history", 0444, debugfs, psu, &ibm_cffps_input_history_fops);
+	debugfs_create_file("max_power_out", 0444, debugfs,
 			    &psu->debugfs_entries[CFFPS_DEBUGFS_MAX_POWER_OUT],
 			    &ibm_cffps_fops);
-	debugfs_create_file("ccin", 0444, ibm_cffps_dir,
+	debugfs_create_file("ccin", 0444, debugfs,
 			    &psu->debugfs_entries[CFFPS_DEBUGFS_CCIN],
 			    &ibm_cffps_fops);
-	debugfs_create_file("fw_version", 0444, ibm_cffps_dir,
+	debugfs_create_file("fw_version", 0444, debugfs,
 			    &psu->debugfs_entries[CFFPS_DEBUGFS_FW],
 			    &ibm_cffps_fops);
-	debugfs_create_file("on_off_config", 0644, ibm_cffps_dir,
+	debugfs_create_file("on_off_config", 0644, debugfs,
 			    &psu->debugfs_entries[CFFPS_DEBUGFS_ON_OFF_CONFIG],
 			    &ibm_cffps_fops);
 
+	/* For compatibility with users of the old naming scheme. */
+	debugfs_create_symlink(client->name, debugfs, ".");
+
 	return 0;
 }
 
-static const struct i2c_device_id ibm_cffps_id[] = {
-	{ "ibm_cffps1", cffps1 },
-	{ "ibm_cffps2", cffps2 },
-	{ "ibm_cffps", cffps_unknown },
-	{}
-};
-MODULE_DEVICE_TABLE(i2c, ibm_cffps_id);
-
 static const struct of_device_id ibm_cffps_of_match[] = {
 	{
 		.compatible = "ibm,cffps1",
diff --git a/drivers/hwmon/pmbus/pmbus.h b/drivers/hwmon/pmbus/pmbus.h
index 713ea7915425..b0832a4c690d 100644
--- a/drivers/hwmon/pmbus/pmbus.h
+++ b/drivers/hwmon/pmbus/pmbus.h
@@ -26,7 +26,7 @@ enum pmbus_regs {
 
 	PMBUS_CAPABILITY		= 0x19,
 	PMBUS_QUERY			= 0x1A,
-
+	PMBUS_SMBALERT_MASK		= 0x1B,
 	PMBUS_VOUT_MODE			= 0x20,
 	PMBUS_VOUT_COMMAND		= 0x21,
 	PMBUS_VOUT_TRIM			= 0x22,
@@ -505,6 +505,8 @@ int pmbus_get_fan_rate_device(struct i2c_client *client, int page, int id,
 			      enum pmbus_fan_mode mode);
 int pmbus_get_fan_rate_cached(struct i2c_client *client, int page, int id,
 			      enum pmbus_fan_mode mode);
+int pmbus_lock_interruptible(struct i2c_client *client);
+void pmbus_unlock(struct i2c_client *client);
 int pmbus_update_fan(struct i2c_client *client, int page, int id,
 		     u8 config, u8 mask, u16 command);
 struct dentry *pmbus_get_debugfs_dir(struct i2c_client *client);
diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c
index 95e95783972a..9d14954da94f 100644
--- a/drivers/hwmon/pmbus/pmbus_core.c
+++ b/drivers/hwmon/pmbus/pmbus_core.c
@@ -81,6 +81,7 @@ struct pmbus_label {
 struct pmbus_data {
 	struct device *dev;
 	struct device *hwmon_dev;
+	struct regulator_dev **rdevs;
 
 	u32 flags;		/* from platform data */
 
@@ -1272,7 +1273,7 @@ struct pmbus_thermal_data {
 
 static int pmbus_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct pmbus_thermal_data *tdata = tz->devdata;
+	struct pmbus_thermal_data *tdata = thermal_zone_device_priv(tz);
 	struct pmbus_sensor *sensor = tdata->sensor;
 	struct pmbus_data *pmbus_data = tdata->pmbus_data;
 	struct i2c_client *client = to_i2c_client(pmbus_data->dev);
@@ -2692,18 +2693,64 @@ static int pmbus_init_common(struct i2c_client *client, struct pmbus_data *data,
 	return 0;
 }
 
-#if IS_ENABLED(CONFIG_REGULATOR)
-static int pmbus_regulator_is_enabled(struct regulator_dev *rdev)
+/* A PMBus status flag and the corresponding REGULATOR_ERROR_* and REGULATOR_EVENTS_* flag */
+struct pmbus_status_assoc {
+	int pflag, rflag, eflag;
+};
+
+/* PMBus->regulator bit mappings for a PMBus status register */
+struct pmbus_status_category {
+	int func;
+	int reg;
+	const struct pmbus_status_assoc *bits; /* zero-terminated */
+};
+
+static const struct pmbus_status_category __maybe_unused pmbus_status_flag_map[] = {
+	{
+		.func = PMBUS_HAVE_STATUS_VOUT,
+		.reg = PMBUS_STATUS_VOUT,
+		.bits = (const struct pmbus_status_assoc[]) {
+			{ PB_VOLTAGE_UV_WARNING, REGULATOR_ERROR_UNDER_VOLTAGE_WARN,
+			REGULATOR_EVENT_UNDER_VOLTAGE_WARN },
+			{ PB_VOLTAGE_UV_FAULT,   REGULATOR_ERROR_UNDER_VOLTAGE,
+			REGULATOR_EVENT_UNDER_VOLTAGE },
+			{ PB_VOLTAGE_OV_WARNING, REGULATOR_ERROR_OVER_VOLTAGE_WARN,
+			REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+			{ PB_VOLTAGE_OV_FAULT,   REGULATOR_ERROR_REGULATION_OUT,
+			REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+			{ },
+		},
+	}, {
+		.func = PMBUS_HAVE_STATUS_IOUT,
+		.reg = PMBUS_STATUS_IOUT,
+		.bits = (const struct pmbus_status_assoc[]) {
+			{ PB_IOUT_OC_WARNING,   REGULATOR_ERROR_OVER_CURRENT_WARN,
+			REGULATOR_EVENT_OVER_CURRENT_WARN },
+			{ PB_IOUT_OC_FAULT,     REGULATOR_ERROR_OVER_CURRENT,
+			REGULATOR_EVENT_OVER_CURRENT },
+			{ PB_IOUT_OC_LV_FAULT,  REGULATOR_ERROR_OVER_CURRENT,
+			REGULATOR_EVENT_OVER_CURRENT },
+			{ },
+		},
+	}, {
+		.func = PMBUS_HAVE_STATUS_TEMP,
+		.reg = PMBUS_STATUS_TEMPERATURE,
+		.bits = (const struct pmbus_status_assoc[]) {
+			{ PB_TEMP_OT_WARNING,    REGULATOR_ERROR_OVER_TEMP_WARN,
+			REGULATOR_EVENT_OVER_TEMP_WARN },
+			{ PB_TEMP_OT_FAULT,      REGULATOR_ERROR_OVER_TEMP,
+			REGULATOR_EVENT_OVER_TEMP },
+			{ },
+		},
+	},
+};
+
+static int _pmbus_is_enabled(struct device *dev, u8 page)
 {
-	struct device *dev = rdev_get_dev(rdev);
 	struct i2c_client *client = to_i2c_client(dev->parent);
-	struct pmbus_data *data = i2c_get_clientdata(client);
-	u8 page = rdev_get_id(rdev);
 	int ret;
 
-	mutex_lock(&data->update_lock);
 	ret = _pmbus_read_byte_data(client, page, PMBUS_OPERATION);
-	mutex_unlock(&data->update_lock);
 
 	if (ret < 0)
 		return ret;
@@ -2711,106 +2758,77 @@ static int pmbus_regulator_is_enabled(struct regulator_dev *rdev)
 	return !!(ret & PB_OPERATION_CONTROL_ON);
 }
 
-static int _pmbus_regulator_on_off(struct regulator_dev *rdev, bool enable)
+static int __maybe_unused pmbus_is_enabled(struct device *dev, u8 page)
 {
-	struct device *dev = rdev_get_dev(rdev);
 	struct i2c_client *client = to_i2c_client(dev->parent);
 	struct pmbus_data *data = i2c_get_clientdata(client);
-	u8 page = rdev_get_id(rdev);
 	int ret;
 
 	mutex_lock(&data->update_lock);
-	ret = pmbus_update_byte_data(client, page, PMBUS_OPERATION,
-				     PB_OPERATION_CONTROL_ON,
-				     enable ? PB_OPERATION_CONTROL_ON : 0);
+	ret = _pmbus_is_enabled(dev, page);
 	mutex_unlock(&data->update_lock);
 
-	return ret;
+	return !!(ret & PB_OPERATION_CONTROL_ON);
 }
 
-static int pmbus_regulator_enable(struct regulator_dev *rdev)
-{
-	return _pmbus_regulator_on_off(rdev, 1);
-}
+#define to_dev_attr(_dev_attr) \
+	container_of(_dev_attr, struct device_attribute, attr)
 
-static int pmbus_regulator_disable(struct regulator_dev *rdev)
+static void pmbus_notify(struct pmbus_data *data, int page, int reg, int flags)
 {
-	return _pmbus_regulator_on_off(rdev, 0);
-}
-
-/* A PMBus status flag and the corresponding REGULATOR_ERROR_* flag */
-struct pmbus_regulator_status_assoc {
-	int pflag, rflag;
-};
+	int i;
 
-/* PMBus->regulator bit mappings for a PMBus status register */
-struct pmbus_regulator_status_category {
-	int func;
-	int reg;
-	const struct pmbus_regulator_status_assoc *bits; /* zero-terminated */
-};
+	for (i = 0; i < data->num_attributes; i++) {
+		struct device_attribute *da = to_dev_attr(data->group.attrs[i]);
+		struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+		int index = attr->index;
+		u16 smask = pb_index_to_mask(index);
+		u8 spage = pb_index_to_page(index);
+		u16 sreg = pb_index_to_reg(index);
+
+		if (reg == sreg && page == spage && (smask & flags)) {
+			dev_dbg(data->dev, "sysfs notify: %s", da->attr.name);
+			sysfs_notify(&data->dev->kobj, NULL, da->attr.name);
+			kobject_uevent(&data->dev->kobj, KOBJ_CHANGE);
+			flags &= ~smask;
+		}
 
-static const struct pmbus_regulator_status_category pmbus_regulator_flag_map[] = {
-	{
-		.func = PMBUS_HAVE_STATUS_VOUT,
-		.reg = PMBUS_STATUS_VOUT,
-		.bits = (const struct pmbus_regulator_status_assoc[]) {
-			{ PB_VOLTAGE_UV_WARNING, REGULATOR_ERROR_UNDER_VOLTAGE_WARN },
-			{ PB_VOLTAGE_UV_FAULT,   REGULATOR_ERROR_UNDER_VOLTAGE },
-			{ PB_VOLTAGE_OV_WARNING, REGULATOR_ERROR_OVER_VOLTAGE_WARN },
-			{ PB_VOLTAGE_OV_FAULT,   REGULATOR_ERROR_REGULATION_OUT },
-			{ },
-		},
-	}, {
-		.func = PMBUS_HAVE_STATUS_IOUT,
-		.reg = PMBUS_STATUS_IOUT,
-		.bits = (const struct pmbus_regulator_status_assoc[]) {
-			{ PB_IOUT_OC_WARNING,    REGULATOR_ERROR_OVER_CURRENT_WARN },
-			{ PB_IOUT_OC_FAULT,      REGULATOR_ERROR_OVER_CURRENT },
-			{ PB_IOUT_OC_LV_FAULT,   REGULATOR_ERROR_OVER_CURRENT },
-			{ },
-		},
-	}, {
-		.func = PMBUS_HAVE_STATUS_TEMP,
-		.reg = PMBUS_STATUS_TEMPERATURE,
-		.bits = (const struct pmbus_regulator_status_assoc[]) {
-			{ PB_TEMP_OT_WARNING,    REGULATOR_ERROR_OVER_TEMP_WARN },
-			{ PB_TEMP_OT_FAULT,      REGULATOR_ERROR_OVER_TEMP },
-			{ },
-		},
-	},
-};
+		if (!flags)
+			break;
+	}
+}
 
-static int pmbus_regulator_get_error_flags(struct regulator_dev *rdev, unsigned int *flags)
+static int _pmbus_get_flags(struct pmbus_data *data, u8 page, unsigned int *flags,
+			   unsigned int *event, bool notify)
 {
 	int i, status;
-	const struct pmbus_regulator_status_category *cat;
-	const struct pmbus_regulator_status_assoc *bit;
-	struct device *dev = rdev_get_dev(rdev);
-	struct i2c_client *client = to_i2c_client(dev->parent);
-	struct pmbus_data *data = i2c_get_clientdata(client);
-	u8 page = rdev_get_id(rdev);
+	const struct pmbus_status_category *cat;
+	const struct pmbus_status_assoc *bit;
+	struct device *dev = data->dev;
+	struct i2c_client *client = to_i2c_client(dev);
 	int func = data->info->func[page];
 
 	*flags = 0;
+	*event = 0;
 
-	mutex_lock(&data->update_lock);
-
-	for (i = 0; i < ARRAY_SIZE(pmbus_regulator_flag_map); i++) {
-		cat = &pmbus_regulator_flag_map[i];
+	for (i = 0; i < ARRAY_SIZE(pmbus_status_flag_map); i++) {
+		cat = &pmbus_status_flag_map[i];
 		if (!(func & cat->func))
 			continue;
 
 		status = _pmbus_read_byte_data(client, page, cat->reg);
-		if (status < 0) {
-			mutex_unlock(&data->update_lock);
+		if (status < 0)
 			return status;
-		}
 
-		for (bit = cat->bits; bit->pflag; bit++) {
-			if (status & bit->pflag)
+		for (bit = cat->bits; bit->pflag; bit++)
+			if (status & bit->pflag) {
 				*flags |= bit->rflag;
-		}
+				*event |= bit->eflag;
+			}
+
+		if (notify && status)
+			pmbus_notify(data, page, cat->reg, status);
+
 	}
 
 	/*
@@ -2823,25 +2841,32 @@ static int pmbus_regulator_get_error_flags(struct regulator_dev *rdev, unsigned
 	 * REGULATOR_ERROR_<foo>_WARN.
 	 */
 	status = pmbus_get_status(client, page, PMBUS_STATUS_WORD);
-	mutex_unlock(&data->update_lock);
 	if (status < 0)
 		return status;
 
-	if (pmbus_regulator_is_enabled(rdev)) {
-		if (status & PB_STATUS_OFF)
+	if (_pmbus_is_enabled(dev, page)) {
+		if (status & PB_STATUS_OFF) {
 			*flags |= REGULATOR_ERROR_FAIL;
+			*event |= REGULATOR_EVENT_FAIL;
+		}
 
-		if (status & PB_STATUS_POWER_GOOD_N)
+		if (status & PB_STATUS_POWER_GOOD_N) {
 			*flags |= REGULATOR_ERROR_REGULATION_OUT;
+			*event |= REGULATOR_EVENT_REGULATION_OUT;
+		}
 	}
 	/*
 	 * Unlike most other status bits, PB_STATUS_{IOUT_OC,VOUT_OV} are
 	 * defined strictly as fault indicators (not warnings).
 	 */
-	if (status & PB_STATUS_IOUT_OC)
+	if (status & PB_STATUS_IOUT_OC) {
 		*flags |= REGULATOR_ERROR_OVER_CURRENT;
-	if (status & PB_STATUS_VOUT_OV)
+		*event |= REGULATOR_EVENT_OVER_CURRENT;
+	}
+	if (status & PB_STATUS_VOUT_OV) {
 		*flags |= REGULATOR_ERROR_REGULATION_OUT;
+		*event |= REGULATOR_EVENT_FAIL;
+	}
 
 	/*
 	 * If we haven't discovered any thermal faults or warnings via
@@ -2849,12 +2874,70 @@ static int pmbus_regulator_get_error_flags(struct regulator_dev *rdev, unsigned
 	 * a (conservative) best-effort interpretation.
 	 */
 	if (!(*flags & (REGULATOR_ERROR_OVER_TEMP | REGULATOR_ERROR_OVER_TEMP_WARN)) &&
-	    (status & PB_STATUS_TEMPERATURE))
+	    (status & PB_STATUS_TEMPERATURE)) {
 		*flags |= REGULATOR_ERROR_OVER_TEMP_WARN;
+		*event |= REGULATOR_EVENT_OVER_TEMP_WARN;
+	}
+
 
 	return 0;
 }
 
+static int __maybe_unused pmbus_get_flags(struct pmbus_data *data, u8 page, unsigned int *flags,
+					  unsigned int *event, bool notify)
+{
+	int ret;
+
+	mutex_lock(&data->update_lock);
+	ret = _pmbus_get_flags(data, page, flags, event, notify);
+	mutex_unlock(&data->update_lock);
+
+	return ret;
+}
+
+#if IS_ENABLED(CONFIG_REGULATOR)
+static int pmbus_regulator_is_enabled(struct regulator_dev *rdev)
+{
+	return pmbus_is_enabled(rdev_get_dev(rdev), rdev_get_id(rdev));
+}
+
+static int _pmbus_regulator_on_off(struct regulator_dev *rdev, bool enable)
+{
+	struct device *dev = rdev_get_dev(rdev);
+	struct i2c_client *client = to_i2c_client(dev->parent);
+	struct pmbus_data *data = i2c_get_clientdata(client);
+	u8 page = rdev_get_id(rdev);
+	int ret;
+
+	mutex_lock(&data->update_lock);
+	ret = pmbus_update_byte_data(client, page, PMBUS_OPERATION,
+				     PB_OPERATION_CONTROL_ON,
+				     enable ? PB_OPERATION_CONTROL_ON : 0);
+	mutex_unlock(&data->update_lock);
+
+	return ret;
+}
+
+static int pmbus_regulator_enable(struct regulator_dev *rdev)
+{
+	return _pmbus_regulator_on_off(rdev, 1);
+}
+
+static int pmbus_regulator_disable(struct regulator_dev *rdev)
+{
+	return _pmbus_regulator_on_off(rdev, 0);
+}
+
+static int pmbus_regulator_get_error_flags(struct regulator_dev *rdev, unsigned int *flags)
+{
+	struct device *dev = rdev_get_dev(rdev);
+	struct i2c_client *client = to_i2c_client(dev->parent);
+	struct pmbus_data *data = i2c_get_clientdata(client);
+	int event;
+
+	return pmbus_get_flags(data, rdev_get_id(rdev), flags, &event, false);
+}
+
 static int pmbus_regulator_get_status(struct regulator_dev *rdev)
 {
 	struct device *dev = rdev_get_dev(rdev);
@@ -3050,9 +3133,13 @@ static int pmbus_regulator_register(struct pmbus_data *data)
 	struct device *dev = data->dev;
 	const struct pmbus_driver_info *info = data->info;
 	const struct pmbus_platform_data *pdata = dev_get_platdata(dev);
-	struct regulator_dev *rdev;
 	int i;
 
+	data->rdevs = devm_kzalloc(dev, sizeof(struct regulator_dev *) * info->num_regulators,
+				   GFP_KERNEL);
+	if (!data->rdevs)
+		return -ENOMEM;
+
 	for (i = 0; i < info->num_regulators; i++) {
 		struct regulator_config config = { };
 
@@ -3062,23 +3149,113 @@ static int pmbus_regulator_register(struct pmbus_data *data)
 		if (pdata && pdata->reg_init_data)
 			config.init_data = &pdata->reg_init_data[i];
 
-		rdev = devm_regulator_register(dev, &info->reg_desc[i],
-					       &config);
-		if (IS_ERR(rdev))
-			return dev_err_probe(dev, PTR_ERR(rdev),
+		data->rdevs[i] = devm_regulator_register(dev, &info->reg_desc[i],
+							 &config);
+		if (IS_ERR(data->rdevs[i]))
+			return dev_err_probe(dev, PTR_ERR(data->rdevs[i]),
 					     "Failed to register %s regulator\n",
 					     info->reg_desc[i].name);
 	}
 
 	return 0;
 }
+
+static int pmbus_regulator_notify(struct pmbus_data *data, int page, int event)
+{
+		int j;
+
+		for (j = 0; j < data->info->num_regulators; j++) {
+			if (page == rdev_get_id(data->rdevs[j])) {
+				regulator_notifier_call_chain(data->rdevs[j], event, NULL);
+				break;
+			}
+		}
+		return 0;
+}
 #else
 static int pmbus_regulator_register(struct pmbus_data *data)
 {
 	return 0;
 }
+
+static int pmbus_regulator_notify(struct pmbus_data *data, int page, int event)
+{
+		return 0;
+}
 #endif
 
+static int pmbus_write_smbalert_mask(struct i2c_client *client, u8 page, u8 reg, u8 val)
+{
+	return pmbus_write_word_data(client, page, PMBUS_SMBALERT_MASK, reg | (val << 8));
+}
+
+static irqreturn_t pmbus_fault_handler(int irq, void *pdata)
+{
+	struct pmbus_data *data = pdata;
+	struct i2c_client *client = to_i2c_client(data->dev);
+
+	int i, status, event;
+	mutex_lock(&data->update_lock);
+	for (i = 0; i < data->info->pages; i++) {
+		_pmbus_get_flags(data, i, &status, &event, true);
+
+		if (event)
+			pmbus_regulator_notify(data, i, event);
+	}
+
+	pmbus_clear_faults(client);
+	mutex_unlock(&data->update_lock);
+
+	return IRQ_HANDLED;
+}
+
+static int pmbus_irq_setup(struct i2c_client *client, struct pmbus_data *data)
+{
+	struct device *dev = &client->dev;
+	const struct pmbus_status_category *cat;
+	const struct pmbus_status_assoc *bit;
+	int i, j, err, func;
+	u8 mask;
+
+	static const u8 misc_status[] = {PMBUS_STATUS_CML, PMBUS_STATUS_OTHER,
+					 PMBUS_STATUS_MFR_SPECIFIC, PMBUS_STATUS_FAN_12,
+					 PMBUS_STATUS_FAN_34};
+
+	if (!client->irq)
+		return 0;
+
+	for (i = 0; i < data->info->pages; i++) {
+		func = data->info->func[i];
+
+		for (j = 0; j < ARRAY_SIZE(pmbus_status_flag_map); j++) {
+			cat = &pmbus_status_flag_map[j];
+			if (!(func & cat->func))
+				continue;
+			mask = 0;
+			for (bit = cat->bits; bit->pflag; bit++)
+				mask |= bit->pflag;
+
+			err = pmbus_write_smbalert_mask(client, i, cat->reg, ~mask);
+			if (err)
+				dev_dbg_once(dev, "Failed to set smbalert for reg 0x%02x\n",
+					     cat->reg);
+		}
+
+		for (j = 0; j < ARRAY_SIZE(misc_status); j++)
+			pmbus_write_smbalert_mask(client, i, misc_status[j], 0xff);
+	}
+
+	/* Register notifiers */
+	err = devm_request_threaded_irq(dev, client->irq, NULL, pmbus_fault_handler,
+					IRQF_ONESHOT, "pmbus-irq", data);
+	if (err) {
+		dev_err(dev, "failed to request an irq %d\n", err);
+		return err;
+	}
+
+	return 0;
+}
+
 static struct dentry *pmbus_debugfs_dir;	/* pmbus debugfs directory */
 
 #if IS_ENABLED(CONFIG_DEBUG_FS)
@@ -3086,8 +3263,13 @@ static int pmbus_debugfs_get(void *data, u64 *val)
 {
 	int rc;
 	struct pmbus_debugfs_entry *entry = data;
+	struct pmbus_data *pdata = i2c_get_clientdata(entry->client);
 
+	rc = mutex_lock_interruptible(&pdata->update_lock);
+	if (rc)
+		return rc;
 	rc = _pmbus_read_byte_data(entry->client, entry->page, entry->reg);
+	mutex_unlock(&pdata->update_lock);
 	if (rc < 0)
 		return rc;
 
@@ -3104,7 +3286,11 @@ static int pmbus_debugfs_get_status(void *data, u64 *val)
 	struct pmbus_debugfs_entry *entry = data;
 	struct pmbus_data *pdata = i2c_get_clientdata(entry->client);
 
+	rc = mutex_lock_interruptible(&pdata->update_lock);
+	if (rc)
+		return rc;
 	rc = pdata->read_status(entry->client, entry->page);
+	mutex_unlock(&pdata->update_lock);
 	if (rc < 0)
 		return rc;
 
@@ -3120,10 +3306,15 @@ static ssize_t pmbus_debugfs_mfr_read(struct file *file, char __user *buf,
 {
 	int rc;
 	struct pmbus_debugfs_entry *entry = file->private_data;
+	struct pmbus_data *pdata = i2c_get_clientdata(entry->client);
 	char data[I2C_SMBUS_BLOCK_MAX + 2] = { 0 };
 
+	rc = mutex_lock_interruptible(&pdata->update_lock);
+	if (rc)
+		return rc;
 	rc = pmbus_read_block_data(entry->client, entry->page, entry->reg,
 				   data);
+	mutex_unlock(&pdata->update_lock);
 	if (rc < 0)
 		return rc;
 
@@ -3441,6 +3632,10 @@ int pmbus_do_probe(struct i2c_client *client, struct pmbus_driver_info *info)
 	if (ret)
 		return ret;
 
+	ret = pmbus_irq_setup(client, data);
+	if (ret)
+		return ret;
+
 	ret = pmbus_init_debugfs(client, data);
 	if (ret)
 		dev_warn(dev, "Failed to register debugfs\n");
@@ -3457,6 +3652,22 @@ struct dentry *pmbus_get_debugfs_dir(struct i2c_client *client)
 }
 EXPORT_SYMBOL_NS_GPL(pmbus_get_debugfs_dir, PMBUS);
 
+int pmbus_lock_interruptible(struct i2c_client *client)
+{
+	struct pmbus_data *data = i2c_get_clientdata(client);
+
+	return mutex_lock_interruptible(&data->update_lock);
+}
+EXPORT_SYMBOL_NS_GPL(pmbus_lock_interruptible, PMBUS);
+
+void pmbus_unlock(struct i2c_client *client)
+{
+	struct pmbus_data *data = i2c_get_clientdata(client);
+
+	mutex_unlock(&data->update_lock);
+}
+EXPORT_SYMBOL_NS_GPL(pmbus_unlock, PMBUS);
+
 static int __init pmbus_core_init(void)
 {
 	pmbus_debugfs_dir = debugfs_create_dir("pmbus", NULL);
diff --git a/drivers/hwmon/powr1220.c b/drivers/hwmon/powr1220.c
index f77dc6db31ac..9cb0c2de5219 100644
--- a/drivers/hwmon/powr1220.c
+++ b/drivers/hwmon/powr1220.c
@@ -248,7 +248,7 @@ powr1220_read(struct device *dev, enum hwmon_sensor_types type, u32
 	return 0;
 }
 
-static const struct hwmon_channel_info *powr1220_info[] = {
+static const struct hwmon_channel_info * const powr1220_info[] = {
 	HWMON_CHANNEL_INFO(in,
 			   HWMON_I_INPUT | HWMON_I_HIGHEST | HWMON_I_LABEL,
 			   HWMON_I_INPUT | HWMON_I_HIGHEST | HWMON_I_LABEL,
diff --git a/drivers/hwmon/pwm-fan.c b/drivers/hwmon/pwm-fan.c
index 83a347ca35da..6e4516c2ab89 100644
--- a/drivers/hwmon/pwm-fan.c
+++ b/drivers/hwmon/pwm-fan.c
@@ -427,7 +427,7 @@ static int pwm_fan_of_get_cooling_data(struct device *dev,
 	struct device_node *np = dev->of_node;
 	int num, i, ret;
 
-	if (!of_find_property(np, "cooling-levels", NULL))
+	if (!of_property_present(np, "cooling-levels"))
 		return 0;
 
 	ret = of_property_count_u32_elems(np, "cooling-levels");
@@ -508,6 +508,14 @@ static int pwm_fan_probe(struct platform_device *pdev)
 	pwm_init_state(ctx->pwm, &ctx->pwm_state);
 
 	/*
+	 * PWM fans are controlled solely by the duty cycle of the PWM signal,
+	 * they do not care about the exact timing. Thus set usage_power to true
+	 * to allow less flexible hardware to work as a PWM source for fan
+	 * control.
+	 */
+	ctx->pwm_state.usage_power = true;
+
+	/*
 	 * set_pwm assumes that MAX_PWM * (period - 1) fits into an unsigned
 	 * long. Check this here to prevent the fan running at a too low
 	 * frequency.
diff --git a/drivers/hwmon/raspberrypi-hwmon.c b/drivers/hwmon/raspberrypi-hwmon.c
index 1650d3b4c26e..65cc52e47db0 100644
--- a/drivers/hwmon/raspberrypi-hwmon.c
+++ b/drivers/hwmon/raspberrypi-hwmon.c
@@ -87,7 +87,7 @@ static umode_t rpi_is_visible(const void *_data, enum hwmon_sensor_types type,
 	return 0444;
 }
 
-static const struct hwmon_channel_info *rpi_info[] = {
+static const struct hwmon_channel_info * const rpi_info[] = {
 	HWMON_CHANNEL_INFO(in,
 			   HWMON_I_LCRIT_ALARM),
 	NULL
diff --git a/drivers/hwmon/sbrmi.c b/drivers/hwmon/sbrmi.c
index 8ea5a4d3219f..529f0e766319 100644
--- a/drivers/hwmon/sbrmi.c
+++ b/drivers/hwmon/sbrmi.c
@@ -265,7 +265,7 @@ static umode_t sbrmi_is_visible(const void *data,
 	return 0;
 }
 
-static const struct hwmon_channel_info *sbrmi_info[] = {
+static const struct hwmon_channel_info * const sbrmi_info[] = {
 	HWMON_CHANNEL_INFO(power,
 			   HWMON_P_INPUT | HWMON_P_CAP | HWMON_P_CAP_MAX),
 	NULL
diff --git a/drivers/hwmon/sbtsi_temp.c b/drivers/hwmon/sbtsi_temp.c
index 4c37de846f93..7049d9464ac6 100644
--- a/drivers/hwmon/sbtsi_temp.c
+++ b/drivers/hwmon/sbtsi_temp.c
@@ -182,7 +182,7 @@ static umode_t sbtsi_is_visible(const void *data,
 	return 0;
 }
 
-static const struct hwmon_channel_info *sbtsi_info[] = {
+static const struct hwmon_channel_info * const sbtsi_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX),
 	NULL
diff --git a/drivers/hwmon/sch5627.c b/drivers/hwmon/sch5627.c
index 25fbbd4c9a2b..1bbda3b05532 100644
--- a/drivers/hwmon/sch5627.c
+++ b/drivers/hwmon/sch5627.c
@@ -379,7 +379,7 @@ static const struct hwmon_ops sch5627_ops = {
 	.write = sch5627_write,
 };
 
-static const struct hwmon_channel_info *sch5627_info[] = {
+static const struct hwmon_channel_info * const sch5627_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT | HWMON_T_FAULT,
diff --git a/drivers/hwmon/scmi-hwmon.c b/drivers/hwmon/scmi-hwmon.c
index e192f0c67146..364199b332c0 100644
--- a/drivers/hwmon/scmi-hwmon.c
+++ b/drivers/hwmon/scmi-hwmon.c
@@ -141,7 +141,7 @@ static int scmi_hwmon_thermal_get_temp(struct thermal_zone_device *tz,
 {
 	int ret;
 	long value;
-	struct scmi_thermal_sensor *th_sensor = tz->devdata;
+	struct scmi_thermal_sensor *th_sensor = thermal_zone_device_priv(tz);
 
 	ret = scmi_hwmon_read_scaled_value(th_sensor->ph, th_sensor->info,
 					   &value);
@@ -220,7 +220,7 @@ static int scmi_thermal_sensor_register(struct device *dev,
 			sensor->name);
 	} else {
 		dev_dbg(dev, "Sensor '%s' attached to thermal zone ID:%d\n",
-			sensor->name, tzd->id);
+			sensor->name, thermal_zone_device_id(tzd));
 	}
 
 	return 0;
diff --git a/drivers/hwmon/scpi-hwmon.c b/drivers/hwmon/scpi-hwmon.c
index 4d75385f7d5e..121e5e9f487f 100644
--- a/drivers/hwmon/scpi-hwmon.c
+++ b/drivers/hwmon/scpi-hwmon.c
@@ -64,7 +64,7 @@ static void scpi_scale_reading(u64 *value, struct sensor_data *sensor)
 
 static int scpi_read_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct scpi_thermal_zone *zone = tz->devdata;
+	struct scpi_thermal_zone *zone = thermal_zone_device_priv(tz);
 	struct scpi_sensors *scpi_sensors = zone->scpi_sensors;
 	struct scpi_ops *scpi_ops = scpi_sensors->scpi_ops;
 	struct sensor_data *sensor = &scpi_sensors->data[zone->sensor_id];
diff --git a/drivers/hwmon/sfctemp.c b/drivers/hwmon/sfctemp.c
new file mode 100644
index 000000000000..fb1da93383d7
--- /dev/null
+++ b/drivers/hwmon/sfctemp.c
@@ -0,0 +1,331 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2021 Emil Renner Berthing <kernel@esmil.dk>
+ * Copyright (C) 2021 Samin Guo <samin.guo@starfivetech.com>
+ */
+
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/hwmon.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+
+/*
+ * TempSensor reset. The RSTN can be de-asserted once the analog core has
+ * powered up. Trst(min 100ns)
+ * 0:reset  1:de-assert
+ */
+#define SFCTEMP_RSTN	BIT(0)
+
+/*
+ * TempSensor analog core power down. The analog core will be powered up
+ * Tpu(min 50us) after PD is de-asserted. RSTN should be held low until the
+ * analog core is powered up.
+ * 0:power up  1:power down
+ */
+#define SFCTEMP_PD	BIT(1)
+
+/*
+ * TempSensor start conversion enable.
+ * 0:disable  1:enable
+ */
+#define SFCTEMP_RUN	BIT(2)
+
+/*
+ * TempSensor conversion value output.
+ * Temp(C)=DOUT*Y/4094 - K
+ */
+#define SFCTEMP_DOUT_POS	16
+#define SFCTEMP_DOUT_MSK	GENMASK(27, 16)
+
+/* DOUT to Celcius conversion constants */
+#define SFCTEMP_Y1000	237500L
+#define SFCTEMP_Z	4094L
+#define SFCTEMP_K1000	81100L
+
+struct sfctemp {
+	/* serialize access to hardware register and enabled below */
+	struct mutex lock;
+	void __iomem *regs;
+	struct clk *clk_sense;
+	struct clk *clk_bus;
+	struct reset_control *rst_sense;
+	struct reset_control *rst_bus;
+	bool enabled;
+};
+
+static void sfctemp_power_up(struct sfctemp *sfctemp)
+{
+	/* make sure we're powered down first */
+	writel(SFCTEMP_PD, sfctemp->regs);
+	udelay(1);
+
+	writel(0, sfctemp->regs);
+	/* wait t_pu(50us) + t_rst(100ns) */
+	usleep_range(60, 200);
+
+	/* de-assert reset */
+	writel(SFCTEMP_RSTN, sfctemp->regs);
+	udelay(1); /* wait t_su(500ps) */
+}
+
+static void sfctemp_power_down(struct sfctemp *sfctemp)
+{
+	writel(SFCTEMP_PD, sfctemp->regs);
+}
+
+static void sfctemp_run(struct sfctemp *sfctemp)
+{
+	writel(SFCTEMP_RSTN | SFCTEMP_RUN, sfctemp->regs);
+	udelay(1);
+}
+
+static void sfctemp_stop(struct sfctemp *sfctemp)
+{
+	writel(SFCTEMP_RSTN, sfctemp->regs);
+}
+
+static int sfctemp_enable(struct sfctemp *sfctemp)
+{
+	int ret = 0;
+
+	mutex_lock(&sfctemp->lock);
+	if (sfctemp->enabled)
+		goto done;
+
+	ret = clk_prepare_enable(sfctemp->clk_bus);
+	if (ret)
+		goto err;
+	ret = reset_control_deassert(sfctemp->rst_bus);
+	if (ret)
+		goto err_disable_bus;
+
+	ret = clk_prepare_enable(sfctemp->clk_sense);
+	if (ret)
+		goto err_assert_bus;
+	ret = reset_control_deassert(sfctemp->rst_sense);
+	if (ret)
+		goto err_disable_sense;
+
+	sfctemp_power_up(sfctemp);
+	sfctemp_run(sfctemp);
+	sfctemp->enabled = true;
+done:
+	mutex_unlock(&sfctemp->lock);
+	return ret;
+
+err_disable_sense:
+	clk_disable_unprepare(sfctemp->clk_sense);
+err_assert_bus:
+	reset_control_assert(sfctemp->rst_bus);
+err_disable_bus:
+	clk_disable_unprepare(sfctemp->clk_bus);
+err:
+	mutex_unlock(&sfctemp->lock);
+	return ret;
+}
+
+static int sfctemp_disable(struct sfctemp *sfctemp)
+{
+	mutex_lock(&sfctemp->lock);
+	if (!sfctemp->enabled)
+		goto done;
+
+	sfctemp_stop(sfctemp);
+	sfctemp_power_down(sfctemp);
+	reset_control_assert(sfctemp->rst_sense);
+	clk_disable_unprepare(sfctemp->clk_sense);
+	reset_control_assert(sfctemp->rst_bus);
+	clk_disable_unprepare(sfctemp->clk_bus);
+	sfctemp->enabled = false;
+done:
+	mutex_unlock(&sfctemp->lock);
+	return 0;
+}
+
+static void sfctemp_disable_action(void *data)
+{
+	sfctemp_disable(data);
+}
+
+static int sfctemp_convert(struct sfctemp *sfctemp, long *val)
+{
+	int ret;
+
+	mutex_lock(&sfctemp->lock);
+	if (!sfctemp->enabled) {
+		ret = -ENODATA;
+		goto out;
+	}
+
+	/* calculate temperature in milli Celcius */
+	*val = (long)((readl(sfctemp->regs) & SFCTEMP_DOUT_MSK) >> SFCTEMP_DOUT_POS)
+		* SFCTEMP_Y1000 / SFCTEMP_Z - SFCTEMP_K1000;
+
+	ret = 0;
+out:
+	mutex_unlock(&sfctemp->lock);
+	return ret;
+}
+
+static umode_t sfctemp_is_visible(const void *data, enum hwmon_sensor_types type,
+				  u32 attr, int channel)
+{
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_enable:
+			return 0644;
+		case hwmon_temp_input:
+			return 0444;
+		default:
+			return 0;
+		}
+	default:
+		return 0;
+	}
+}
+
+static int sfctemp_read(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long *val)
+{
+	struct sfctemp *sfctemp = dev_get_drvdata(dev);
+
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_enable:
+			*val = sfctemp->enabled;
+			return 0;
+		case hwmon_temp_input:
+			return sfctemp_convert(sfctemp, val);
+		default:
+			return -EINVAL;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sfctemp_write(struct device *dev, enum hwmon_sensor_types type,
+			 u32 attr, int channel, long val)
+{
+	struct sfctemp *sfctemp = dev_get_drvdata(dev);
+
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_enable:
+			if (val == 0)
+				return sfctemp_disable(sfctemp);
+			if (val == 1)
+				return sfctemp_enable(sfctemp);
+			return -EINVAL;
+		default:
+			return -EINVAL;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct hwmon_channel_info *sfctemp_info[] = {
+	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
+	HWMON_CHANNEL_INFO(temp, HWMON_T_ENABLE | HWMON_T_INPUT),
+	NULL
+};
+
+static const struct hwmon_ops sfctemp_hwmon_ops = {
+	.is_visible = sfctemp_is_visible,
+	.read = sfctemp_read,
+	.write = sfctemp_write,
+};
+
+static const struct hwmon_chip_info sfctemp_chip_info = {
+	.ops = &sfctemp_hwmon_ops,
+	.info = sfctemp_info,
+};
+
+static int sfctemp_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device *hwmon_dev;
+	struct sfctemp *sfctemp;
+	int ret;
+
+	sfctemp = devm_kzalloc(dev, sizeof(*sfctemp), GFP_KERNEL);
+	if (!sfctemp)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, sfctemp);
+	mutex_init(&sfctemp->lock);
+
+	sfctemp->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(sfctemp->regs))
+		return PTR_ERR(sfctemp->regs);
+
+	sfctemp->clk_sense = devm_clk_get(dev, "sense");
+	if (IS_ERR(sfctemp->clk_sense))
+		return dev_err_probe(dev, PTR_ERR(sfctemp->clk_sense),
+				     "error getting sense clock\n");
+
+	sfctemp->clk_bus = devm_clk_get(dev, "bus");
+	if (IS_ERR(sfctemp->clk_bus))
+		return dev_err_probe(dev, PTR_ERR(sfctemp->clk_bus),
+				     "error getting bus clock\n");
+
+	sfctemp->rst_sense = devm_reset_control_get_exclusive(dev, "sense");
+	if (IS_ERR(sfctemp->rst_sense))
+		return dev_err_probe(dev, PTR_ERR(sfctemp->rst_sense),
+				     "error getting sense reset\n");
+
+	sfctemp->rst_bus = devm_reset_control_get_exclusive(dev, "bus");
+	if (IS_ERR(sfctemp->rst_bus))
+		return dev_err_probe(dev, PTR_ERR(sfctemp->rst_bus),
+				     "error getting busreset\n");
+
+	ret = reset_control_assert(sfctemp->rst_sense);
+	if (ret)
+		return dev_err_probe(dev, ret, "error asserting sense reset\n");
+
+	ret = reset_control_assert(sfctemp->rst_bus);
+	if (ret)
+		return dev_err_probe(dev, ret, "error asserting bus reset\n");
+
+	ret = devm_add_action(dev, sfctemp_disable_action, sfctemp);
+	if (ret)
+		return ret;
+
+	ret = sfctemp_enable(sfctemp);
+	if (ret)
+		return dev_err_probe(dev, ret, "error enabling temperature sensor\n");
+
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, "sfctemp", sfctemp,
+							 &sfctemp_chip_info, NULL);
+	return PTR_ERR_OR_ZERO(hwmon_dev);
+}
+
+static const struct of_device_id sfctemp_of_match[] = {
+	{ .compatible = "starfive,jh7100-temp" },
+	{ .compatible = "starfive,jh7110-temp" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, sfctemp_of_match);
+
+static struct platform_driver sfctemp_driver = {
+	.probe  = sfctemp_probe,
+	.driver = {
+		.name = "sfctemp",
+		.of_match_table = sfctemp_of_match,
+	},
+};
+module_platform_driver(sfctemp_driver);
+
+MODULE_AUTHOR("Emil Renner Berthing");
+MODULE_DESCRIPTION("StarFive JH71x0 temperature sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/sht4x.c b/drivers/hwmon/sht4x.c
index 13e042927bf8..5bbe09135ab9 100644
--- a/drivers/hwmon/sht4x.c
+++ b/drivers/hwmon/sht4x.c
@@ -214,7 +214,7 @@ static int sht4x_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *sht4x_info[] = {
+static const struct hwmon_channel_info * const sht4x_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
 	HWMON_CHANNEL_INFO(humidity, HWMON_H_INPUT),
diff --git a/drivers/hwmon/sl28cpld-hwmon.c b/drivers/hwmon/sl28cpld-hwmon.c
index 9ce4899a81a5..e020f25c9300 100644
--- a/drivers/hwmon/sl28cpld-hwmon.c
+++ b/drivers/hwmon/sl28cpld-hwmon.c
@@ -67,7 +67,7 @@ static int sl28cpld_hwmon_read(struct device *dev,
 	return 0;
 }
 
-static const struct hwmon_channel_info *sl28cpld_hwmon_info[] = {
+static const struct hwmon_channel_info * const sl28cpld_hwmon_info[] = {
 	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT),
 	NULL
 };
diff --git a/drivers/hwmon/smpro-hwmon.c b/drivers/hwmon/smpro-hwmon.c
index a76c49dd8438..d320adbd47f4 100644
--- a/drivers/hwmon/smpro-hwmon.c
+++ b/drivers/hwmon/smpro-hwmon.c
@@ -385,7 +385,7 @@ static umode_t smpro_is_visible(const void *data, enum hwmon_sensor_types type,
 	return 0444;
 }
 
-static const struct hwmon_channel_info *smpro_info[] = {
+static const struct hwmon_channel_info * const smpro_info[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_LABEL,
 			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_CRIT,
diff --git a/drivers/hwmon/sparx5-temp.c b/drivers/hwmon/sparx5-temp.c
index 04fd8505e5d6..d640904939cd 100644
--- a/drivers/hwmon/sparx5-temp.c
+++ b/drivers/hwmon/sparx5-temp.c
@@ -86,7 +86,7 @@ static umode_t s5_is_visible(const void *_data, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *s5_info[] = {
+static const struct hwmon_channel_info * const s5_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
 	NULL
diff --git a/drivers/hwmon/sy7636a-hwmon.c b/drivers/hwmon/sy7636a-hwmon.c
index 6dd9c2a0f0e0..ed110884786b 100644
--- a/drivers/hwmon/sy7636a-hwmon.c
+++ b/drivers/hwmon/sy7636a-hwmon.c
@@ -52,7 +52,7 @@ static const struct hwmon_ops sy7636a_hwmon_ops = {
 	.read = sy7636a_read,
 };
 
-static const struct hwmon_channel_info *sy7636a_info[] = {
+static const struct hwmon_channel_info * const sy7636a_info[] = {
 	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
 	NULL
diff --git a/drivers/hwmon/tmp102.c b/drivers/hwmon/tmp102.c
index 2bf496a62206..e271556efe0b 100644
--- a/drivers/hwmon/tmp102.c
+++ b/drivers/hwmon/tmp102.c
@@ -141,7 +141,7 @@ static umode_t tmp102_is_visible(const void *data, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *tmp102_info[] = {
+static const struct hwmon_channel_info * const tmp102_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/tmp103.c b/drivers/hwmon/tmp103.c
index 56d5cbf36a45..d257bb91fc69 100644
--- a/drivers/hwmon/tmp103.c
+++ b/drivers/hwmon/tmp103.c
@@ -119,7 +119,7 @@ static umode_t tmp103_is_visible(const void *data, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *tmp103_info[] = {
+static const struct hwmon_channel_info * const tmp103_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/tmp108.c b/drivers/hwmon/tmp108.c
index acb4ba750b09..43784c289a9e 100644
--- a/drivers/hwmon/tmp108.c
+++ b/drivers/hwmon/tmp108.c
@@ -272,7 +272,7 @@ static umode_t tmp108_is_visible(const void *data, enum hwmon_sensor_types type,
 	}
 }
 
-static const struct hwmon_channel_info *tmp108_info[] = {
+static const struct hwmon_channel_info * const tmp108_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/tmp464.c b/drivers/hwmon/tmp464.c
index 7814f39bd1a3..9213a493a590 100644
--- a/drivers/hwmon/tmp464.c
+++ b/drivers/hwmon/tmp464.c
@@ -589,7 +589,7 @@ static const struct hwmon_ops tmp464_ops = {
 	.write = tmp464_write,
 };
 
-static const struct hwmon_channel_info *tmp464_info[] = {
+static const struct hwmon_channel_info * const tmp464_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/tmp513.c b/drivers/hwmon/tmp513.c
index 7d5f7441aceb..0693eaee054f 100644
--- a/drivers/hwmon/tmp513.c
+++ b/drivers/hwmon/tmp513.c
@@ -491,7 +491,7 @@ static umode_t tmp51x_is_visible(const void *_data,
 	return 0;
 }
 
-static const struct hwmon_channel_info *tmp51x_info[] = {
+static const struct hwmon_channel_info * const tmp51x_info[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_CRIT | HWMON_T_CRIT_ALARM |
 			   HWMON_T_CRIT_HYST,
diff --git a/drivers/hwmon/tps23861.c b/drivers/hwmon/tps23861.c
index 68c77c493270..85a75f551148 100644
--- a/drivers/hwmon/tps23861.c
+++ b/drivers/hwmon/tps23861.c
@@ -341,7 +341,7 @@ static int tps23861_read_string(struct device *dev,
 	return 0;
 }
 
-static const struct hwmon_channel_info *tps23861_info[] = {
+static const struct hwmon_channel_info * const tps23861_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwmon/vt1211.c b/drivers/hwmon/vt1211.c
index 4a5e911d26eb..fcd4be7a5a85 100644
--- a/drivers/hwmon/vt1211.c
+++ b/drivers/hwmon/vt1211.c
@@ -195,12 +195,6 @@ struct vt1211_data {
 /* VT1211 logical device numbers */
 #define SIO_VT1211_LDN_HWMON	0x0b	/* HW monitor */
 
-static inline void superio_outb(int sio_cip, int reg, int val)
-{
-	outb(reg, sio_cip);
-	outb(val, sio_cip + 1);
-}
-
 static inline int superio_inb(int sio_cip, int reg)
 {
 	outb(reg, sio_cip);
diff --git a/drivers/hwmon/w83627ehf.c b/drivers/hwmon/w83627ehf.c
index 939d4c35e713..fe960c0a624f 100644
--- a/drivers/hwmon/w83627ehf.c
+++ b/drivers/hwmon/w83627ehf.c
@@ -1640,7 +1640,7 @@ static const struct hwmon_ops w83627ehf_ops = {
 	.write = w83627ehf_write,
 };
 
-static const struct hwmon_channel_info *w83627ehf_info[] = {
+static const struct hwmon_channel_info * const w83627ehf_info[] = {
 	HWMON_CHANNEL_INFO(fan,
 		HWMON_F_ALARM | HWMON_F_DIV | HWMON_F_INPUT | HWMON_F_MIN,
 		HWMON_F_ALARM | HWMON_F_DIV | HWMON_F_INPUT | HWMON_F_MIN,
diff --git a/drivers/hwmon/w83773g.c b/drivers/hwmon/w83773g.c
index 88d11dc5feb9..8dbcd05abd9a 100644
--- a/drivers/hwmon/w83773g.c
+++ b/drivers/hwmon/w83773g.c
@@ -233,7 +233,7 @@ static umode_t w83773_is_visible(const void *data, enum hwmon_sensor_types type,
 	return 0;
 }
 
-static const struct hwmon_channel_info *w83773_info[] = {
+static const struct hwmon_channel_info * const w83773_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
 	HWMON_CHANNEL_INFO(temp,
diff --git a/drivers/hwspinlock/hwspinlock_core.c b/drivers/hwspinlock/hwspinlock_core.c
index 22b8f2a70b3b..ada694ba9f95 100644
--- a/drivers/hwspinlock/hwspinlock_core.c
+++ b/drivers/hwspinlock/hwspinlock_core.c
@@ -949,6 +949,5 @@ struct hwspinlock *devm_hwspin_lock_request_specific(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_hwspin_lock_request_specific);
 
-MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("Hardware spinlock interface");
 MODULE_AUTHOR("Ohad Ben-Cohen <ohad@wizery.com>");
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c
index a48c97da8165..711f451b6946 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.c
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.c
@@ -901,6 +901,7 @@ int __init etm_perf_init(void)
 	etm_pmu.addr_filters_sync	= etm_addr_filters_sync;
 	etm_pmu.addr_filters_validate	= etm_addr_filters_validate;
 	etm_pmu.nr_addr_filters		= ETM_ADDR_CMP_MAX;
+	etm_pmu.module			= THIS_MODULE;
 
 	ret = perf_pmu_register(&etm_pmu, CORESIGHT_ETM_PMU_NAME, -1);
 	if (ret == 0)
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index 25eb4e8fd22f..87600b4aacb3 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -566,9 +566,12 @@ config I2C_DESIGNWARE_PLATFORM
 
 config I2C_DESIGNWARE_AMDPSP
 	bool "AMD PSP I2C semaphore support"
-	depends on X86_MSR
 	depends on ACPI
+	depends on CRYPTO_DEV_SP_PSP
+	depends on PCI
 	depends on I2C_DESIGNWARE_PLATFORM
+	depends on (I2C_DESIGNWARE_PLATFORM=y && CRYPTO_DEV_CCP_DD=y) || \
+		   (I2C_DESIGNWARE_PLATFORM=m && CRYPTO_DEV_CCP_DD)
 	help
 	  This driver enables managed host access to the selected I2C bus shared
 	  between AMD CPU and AMD PSP.
@@ -804,7 +807,7 @@ config I2C_MESON
 
 config I2C_MICROCHIP_CORE
 	tristate "Microchip FPGA I2C controller"
-	depends on SOC_MICROCHIP_POLARFIRE || COMPILE_TEST
+	depends on ARCH_MICROCHIP_POLARFIRE || COMPILE_TEST
 	depends on OF
 	help
 	  If you say yes to this option, support will be included for the
diff --git a/drivers/i2c/busses/i2c-brcmstb.c b/drivers/i2c/busses/i2c-brcmstb.c
index 69383be47905..ef942714642a 100644
--- a/drivers/i2c/busses/i2c-brcmstb.c
+++ b/drivers/i2c/busses/i2c-brcmstb.c
@@ -575,12 +575,10 @@ static void brcmstb_i2c_set_bsc_reg_defaults(struct brcmstb_i2c_dev *dev)
 static int bcm2711_release_bsc(struct brcmstb_i2c_dev *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev->device);
-	struct resource *iomem;
 	void __iomem *autoi2c;
 
 	/* Map hardware registers */
-	iomem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "auto-i2c");
-	autoi2c = devm_ioremap_resource(&pdev->dev, iomem);
+	autoi2c = devm_platform_ioremap_resource_byname(pdev, "auto-i2c");
 	if (IS_ERR(autoi2c))
 		return PTR_ERR(autoi2c);
 
diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c
index b5d22e7282c2..3a4edf7e75f9 100644
--- a/drivers/i2c/busses/i2c-cadence.c
+++ b/drivers/i2c/busses/i2c-cadence.c
@@ -16,6 +16,7 @@
 #include <linux/of.h>
 #include <linux/pm_runtime.h>
 #include <linux/pinctrl/consumer.h>
+#include <linux/reset.h>
 
 /* Register offsets for the I2C device. */
 #define CDNS_I2C_CR_OFFSET		0x00 /* Control Register, RW */
@@ -114,10 +115,10 @@
 /* timeout for pm runtime autosuspend */
 #define CNDS_I2C_PM_TIMEOUT		1000	/* ms */
 
-#define CDNS_I2C_FIFO_DEPTH		16
+#define CDNS_I2C_FIFO_DEPTH_DEFAULT	16
 #define CDNS_I2C_MAX_TRANSFER_SIZE	255
 /* Transfer size in multiples of data interrupt depth */
-#define CDNS_I2C_TRANSFER_SIZE	(CDNS_I2C_MAX_TRANSFER_SIZE - 3)
+#define CDNS_I2C_TRANSFER_SIZE(max)	((max) - 3)
 
 #define DRIVER_NAME		"cdns-i2c"
 
@@ -178,12 +179,15 @@ enum cdns_i2c_slave_state {
  * @bus_hold_flag:	Flag used in repeated start for clearing HOLD bit
  * @clk:		Pointer to struct clk
  * @clk_rate_change_nb:	Notifier block for clock rate changes
+ * @reset:		Reset control for the device
  * @quirks:		flag for broken hold bit usage in r1p10
  * @ctrl_reg:		Cached value of the control register.
  * @ctrl_reg_diva_divb: value of fields DIV_A and DIV_B from CR register
  * @slave:		Registered slave instance.
  * @dev_mode:		I2C operating role(master/slave).
  * @slave_state:	I2C Slave state(idle/read/write).
+ * @fifo_depth:		The depth of the transfer FIFO
+ * @transfer_size:	The maximum number of bytes in one transfer
  */
 struct cdns_i2c {
 	struct device		*dev;
@@ -202,6 +206,7 @@ struct cdns_i2c {
 	unsigned int bus_hold_flag;
 	struct clk *clk;
 	struct notifier_block clk_rate_change_nb;
+	struct reset_control *reset;
 	u32 quirks;
 	u32 ctrl_reg;
 	struct i2c_bus_recovery_info rinfo;
@@ -211,6 +216,8 @@ struct cdns_i2c {
 	enum cdns_i2c_mode dev_mode;
 	enum cdns_i2c_slave_state slave_state;
 #endif
+	u32 fifo_depth;
+	unsigned int transfer_size;
 };
 
 struct cdns_platform_data {
@@ -236,7 +243,7 @@ static void cdns_i2c_clear_bus_hold(struct cdns_i2c *id)
 static inline bool cdns_is_holdquirk(struct cdns_i2c *id, bool hold_wrkaround)
 {
 	return (hold_wrkaround &&
-		(id->curr_recv_count == CDNS_I2C_FIFO_DEPTH + 1));
+		(id->curr_recv_count == id->fifo_depth + 1));
 }
 
 #if IS_ENABLED(CONFIG_I2C_SLAVE)
@@ -431,7 +438,7 @@ static irqreturn_t cdns_i2c_master_isr(void *ptr)
 				 * if RX data left is less than or equal to
 				 * FIFO DEPTH unless repeated start is selected
 				 */
-				if (id->recv_count <= CDNS_I2C_FIFO_DEPTH &&
+				if (id->recv_count <= id->fifo_depth &&
 				    !id->bus_hold_flag)
 					cdns_i2c_clear_bus_hold(id);
 
@@ -456,22 +463,22 @@ static irqreturn_t cdns_i2c_master_isr(void *ptr)
 		if (cdns_is_holdquirk(id, updatetx)) {
 			/* wait while fifo is full */
 			while (cdns_i2c_readreg(CDNS_I2C_XFER_SIZE_OFFSET) !=
-			       (id->curr_recv_count - CDNS_I2C_FIFO_DEPTH))
+			       (id->curr_recv_count - id->fifo_depth))
 				;
 
 			/*
 			 * Check number of bytes to be received against maximum
 			 * transfer size and update register accordingly.
 			 */
-			if (((int)(id->recv_count) - CDNS_I2C_FIFO_DEPTH) >
-			    CDNS_I2C_TRANSFER_SIZE) {
-				cdns_i2c_writereg(CDNS_I2C_TRANSFER_SIZE,
+			if (((int)(id->recv_count) - id->fifo_depth) >
+			    id->transfer_size) {
+				cdns_i2c_writereg(id->transfer_size,
 						  CDNS_I2C_XFER_SIZE_OFFSET);
-				id->curr_recv_count = CDNS_I2C_TRANSFER_SIZE +
-						      CDNS_I2C_FIFO_DEPTH;
+				id->curr_recv_count = id->transfer_size +
+						      id->fifo_depth;
 			} else {
 				cdns_i2c_writereg(id->recv_count -
-						  CDNS_I2C_FIFO_DEPTH,
+						  id->fifo_depth,
 						  CDNS_I2C_XFER_SIZE_OFFSET);
 				id->curr_recv_count = id->recv_count;
 			}
@@ -494,7 +501,7 @@ static irqreturn_t cdns_i2c_master_isr(void *ptr)
 		 * space available in FIFO and fill with that many bytes.
 		 */
 		if (id->send_count) {
-			avail_bytes = CDNS_I2C_FIFO_DEPTH -
+			avail_bytes = id->fifo_depth -
 			    cdns_i2c_readreg(CDNS_I2C_XFER_SIZE_OFFSET);
 			if (id->send_count > avail_bytes)
 				bytes_to_send = avail_bytes;
@@ -588,7 +595,7 @@ static void cdns_i2c_mrecv(struct cdns_i2c *id)
 	 * Check for the message size against FIFO depth and set the
 	 * 'hold bus' bit if it is greater than FIFO depth.
 	 */
-	if (id->recv_count > CDNS_I2C_FIFO_DEPTH)
+	if (id->recv_count > id->fifo_depth)
 		ctrl_reg |= CDNS_I2C_CR_HOLD;
 
 	cdns_i2c_writereg(ctrl_reg, CDNS_I2C_CR_OFFSET);
@@ -603,17 +610,17 @@ static void cdns_i2c_mrecv(struct cdns_i2c *id)
 	 * receive if it is less than transfer size and transfer size if
 	 * it is more. Enable the interrupts.
 	 */
-	if (id->recv_count > CDNS_I2C_TRANSFER_SIZE) {
-		cdns_i2c_writereg(CDNS_I2C_TRANSFER_SIZE,
+	if (id->recv_count > id->transfer_size) {
+		cdns_i2c_writereg(id->transfer_size,
 				  CDNS_I2C_XFER_SIZE_OFFSET);
-		id->curr_recv_count = CDNS_I2C_TRANSFER_SIZE;
+		id->curr_recv_count = id->transfer_size;
 	} else {
 		cdns_i2c_writereg(id->recv_count, CDNS_I2C_XFER_SIZE_OFFSET);
 	}
 
 	/* Determine hold_clear based on number of bytes to receive and hold flag */
-	if (!id->bus_hold_flag && id->recv_count <= CDNS_I2C_FIFO_DEPTH) {
-		if (cdns_i2c_readreg(CDNS_I2C_CR_OFFSET) & CDNS_I2C_CR_HOLD) {
+	if (!id->bus_hold_flag && id->recv_count <= id->fifo_depth) {
+		if (ctrl_reg & CDNS_I2C_CR_HOLD) {
 			hold_clear = true;
 			if (id->quirks & CDNS_I2C_BROKEN_HOLD_BIT)
 				irq_save = true;
@@ -624,7 +631,7 @@ static void cdns_i2c_mrecv(struct cdns_i2c *id)
 	addr &= CDNS_I2C_ADDR_MASK;
 
 	if (hold_clear) {
-		ctrl_reg = cdns_i2c_readreg(CDNS_I2C_CR_OFFSET) & ~CDNS_I2C_CR_HOLD;
+		ctrl_reg &= ~CDNS_I2C_CR_HOLD;
 		/*
 		 * In case of Xilinx Zynq SOC, clear the HOLD bit before transfer size
 		 * register reaches '0'. This is an IP bug which causes transfer size
@@ -673,7 +680,7 @@ static void cdns_i2c_msend(struct cdns_i2c *id)
 	 * Check for the message size against FIFO depth and set the
 	 * 'hold bus' bit if it is greater than FIFO depth.
 	 */
-	if (id->send_count > CDNS_I2C_FIFO_DEPTH)
+	if (id->send_count > id->fifo_depth)
 		ctrl_reg |= CDNS_I2C_CR_HOLD;
 	cdns_i2c_writereg(ctrl_reg, CDNS_I2C_CR_OFFSET);
 
@@ -686,7 +693,7 @@ static void cdns_i2c_msend(struct cdns_i2c *id)
 	 * against the space available, and fill the FIFO accordingly.
 	 * Enable the interrupts.
 	 */
-	avail_bytes = CDNS_I2C_FIFO_DEPTH -
+	avail_bytes = id->fifo_depth -
 				cdns_i2c_readreg(CDNS_I2C_XFER_SIZE_OFFSET);
 
 	if (id->send_count > avail_bytes)
@@ -827,8 +834,10 @@ static int cdns_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
 #if IS_ENABLED(CONFIG_I2C_SLAVE)
 	/* Check i2c operating mode and switch if possible */
 	if (id->dev_mode == CDNS_I2C_MODE_SLAVE) {
-		if (id->slave_state != CDNS_I2C_SLAVE_STATE_IDLE)
-			return -EAGAIN;
+		if (id->slave_state != CDNS_I2C_SLAVE_STATE_IDLE) {
+			ret = -EAGAIN;
+			goto out;
+		}
 
 		/* Set mode to master */
 		cdns_i2c_set_mode(CDNS_I2C_MODE_MASTER, id);
@@ -1030,8 +1039,7 @@ static int cdns_i2c_calc_divs(unsigned long *f, unsigned long input_clk,
 		if (actual_fscl > fscl)
 			continue;
 
-		current_error = ((actual_fscl > fscl) ? (actual_fscl - fscl) :
-							(fscl - actual_fscl));
+		current_error = fscl - actual_fscl;
 
 		if (last_error > current_error) {
 			calc_div_a = div_a;
@@ -1227,6 +1235,37 @@ static const struct of_device_id cdns_i2c_of_match[] = {
 MODULE_DEVICE_TABLE(of, cdns_i2c_of_match);
 
 /**
+ * cdns_i2c_detect_transfer_size - Detect the maximum transfer size supported
+ * @id: Device private data structure
+ *
+ * Detect the maximum transfer size that is supported by this instance of the
+ * Cadence I2C controller.
+ */
+static void cdns_i2c_detect_transfer_size(struct cdns_i2c *id)
+{
+	u32 val;
+
+	/*
+	 * Writing to the transfer size register is only possible if these two bits
+	 * are set in the control register.
+	 */
+	cdns_i2c_writereg(CDNS_I2C_CR_MS | CDNS_I2C_CR_RW, CDNS_I2C_CR_OFFSET);
+
+	/*
+	 * The number of writable bits of the transfer size register can be between
+	 * 4 and 8. This is a controlled through a synthesis parameter of the IP
+	 * core and can vary from instance to instance. The unused MSBs always read
+	 * back as 0. Writing 0xff and then reading the value back will report the
+	 * maximum supported transfer size.
+	 */
+	cdns_i2c_writereg(CDNS_I2C_MAX_TRANSFER_SIZE, CDNS_I2C_XFER_SIZE_OFFSET);
+	val = cdns_i2c_readreg(CDNS_I2C_XFER_SIZE_OFFSET);
+	id->transfer_size = CDNS_I2C_TRANSFER_SIZE(val);
+	cdns_i2c_writereg(0, CDNS_I2C_XFER_SIZE_OFFSET);
+	cdns_i2c_writereg(0, CDNS_I2C_CR_OFFSET);
+}
+
+/**
  * cdns_i2c_probe - Platform registration call
  * @pdev:	Handle to the platform device structure
  *
@@ -1291,10 +1330,22 @@ static int cdns_i2c_probe(struct platform_device *pdev)
 		return dev_err_probe(&pdev->dev, PTR_ERR(id->clk),
 				     "input clock not found.\n");
 
+	id->reset = devm_reset_control_get_optional_shared(&pdev->dev, NULL);
+	if (IS_ERR(id->reset))
+		return dev_err_probe(&pdev->dev, PTR_ERR(id->reset),
+				     "Failed to request reset.\n");
+
 	ret = clk_prepare_enable(id->clk);
 	if (ret)
 		dev_err(&pdev->dev, "Unable to enable clock.\n");
 
+	ret = reset_control_deassert(id->reset);
+	if (ret) {
+		dev_err_probe(&pdev->dev, ret,
+			      "Failed to de-assert reset.\n");
+		goto err_clk_dis;
+	}
+
 	pm_runtime_set_autosuspend_delay(id->dev, CNDS_I2C_PM_TIMEOUT);
 	pm_runtime_use_autosuspend(id->dev);
 	pm_runtime_set_active(id->dev);
@@ -1317,32 +1368,39 @@ static int cdns_i2c_probe(struct platform_device *pdev)
 #endif
 	id->ctrl_reg = CDNS_I2C_CR_ACK_EN | CDNS_I2C_CR_NEA | CDNS_I2C_CR_MS;
 
+	id->fifo_depth = CDNS_I2C_FIFO_DEPTH_DEFAULT;
+	of_property_read_u32(pdev->dev.of_node, "fifo-depth", &id->fifo_depth);
+
+	cdns_i2c_detect_transfer_size(id);
+
 	ret = cdns_i2c_setclk(id->input_clk, id);
 	if (ret) {
 		dev_err(&pdev->dev, "invalid SCL clock: %u Hz\n", id->i2c_clk);
 		ret = -EINVAL;
-		goto err_clk_dis;
+		goto err_clk_notifier_unregister;
 	}
 
 	ret = devm_request_irq(&pdev->dev, irq, cdns_i2c_isr, 0,
 				 DRIVER_NAME, id);
 	if (ret) {
 		dev_err(&pdev->dev, "cannot get irq %d\n", irq);
-		goto err_clk_dis;
+		goto err_clk_notifier_unregister;
 	}
 	cdns_i2c_init(id);
 
 	ret = i2c_add_adapter(&id->adap);
 	if (ret < 0)
-		goto err_clk_dis;
+		goto err_clk_notifier_unregister;
 
 	dev_info(&pdev->dev, "%u kHz mmio %08lx irq %d\n",
 		 id->i2c_clk / 1000, (unsigned long)r_mem->start, irq);
 
 	return 0;
 
-err_clk_dis:
+err_clk_notifier_unregister:
 	clk_notifier_unregister(id->clk, &id->clk_rate_change_nb);
+	reset_control_assert(id->reset);
+err_clk_dis:
 	clk_disable_unprepare(id->clk);
 	pm_runtime_disable(&pdev->dev);
 	pm_runtime_set_suspended(&pdev->dev);
@@ -1367,6 +1425,7 @@ static int cdns_i2c_remove(struct platform_device *pdev)
 
 	i2c_del_adapter(&id->adap);
 	clk_notifier_unregister(id->clk, &id->clk_rate_change_nb);
+	reset_control_assert(id->reset);
 	clk_disable_unprepare(id->clk);
 
 	return 0;
diff --git a/drivers/i2c/busses/i2c-cros-ec-tunnel.c b/drivers/i2c/busses/i2c-cros-ec-tunnel.c
index 4e787dc709f9..8b3ff5bb14d8 100644
--- a/drivers/i2c/busses/i2c-cros-ec-tunnel.c
+++ b/drivers/i2c/busses/i2c-cros-ec-tunnel.c
@@ -292,13 +292,13 @@ static int ec_i2c_remove(struct platform_device *dev)
 	return 0;
 }
 
-static const struct of_device_id cros_ec_i2c_of_match[] = {
+static const struct of_device_id cros_ec_i2c_of_match[] __maybe_unused = {
 	{ .compatible = "google,cros-ec-i2c-tunnel" },
 	{},
 };
 MODULE_DEVICE_TABLE(of, cros_ec_i2c_of_match);
 
-static const struct acpi_device_id cros_ec_i2c_tunnel_acpi_id[] = {
+static const struct acpi_device_id cros_ec_i2c_tunnel_acpi_id[] __maybe_unused = {
 	{ "GOOG0012", 0 },
 	{ }
 };
diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c
index c836cf884185..9750310f2c96 100644
--- a/drivers/i2c/busses/i2c-davinci.c
+++ b/drivers/i2c/busses/i2c-davinci.c
@@ -764,11 +764,8 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 	int r, irq;
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq <= 0) {
-		if (!irq)
-			irq = -ENXIO;
+	if (irq < 0)
 		return dev_err_probe(&pdev->dev, irq, "can't get irq resource\n");
-	}
 
 	dev = devm_kzalloc(&pdev->dev, sizeof(struct davinci_i2c_dev),
 			GFP_KERNEL);
diff --git a/drivers/i2c/busses/i2c-designware-amdpsp.c b/drivers/i2c/busses/i2c-designware-amdpsp.c
index 8f36167bce62..63454b06e5da 100644
--- a/drivers/i2c/busses/i2c-designware-amdpsp.c
+++ b/drivers/i2c/busses/i2c-designware-amdpsp.c
@@ -1,41 +1,21 @@
 // SPDX-License-Identifier: GPL-2.0
 
-#include <linux/bitfield.h>
-#include <linux/bits.h>
 #include <linux/i2c.h>
-#include <linux/io-64-nonatomic-lo-hi.h>
-#include <linux/psp-sev.h>
-#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/psp-platform-access.h>
+#include <linux/psp.h>
 #include <linux/workqueue.h>
 
-#include <asm/msr.h>
-
 #include "i2c-designware-core.h"
 
-#define MSR_AMD_PSP_ADDR	0xc00110a2
-#define PSP_MBOX_OFFSET		0x10570
-#define PSP_CMD_TIMEOUT_US	(500 * USEC_PER_MSEC)
-
 #define PSP_I2C_RESERVATION_TIME_MS 100
 
-#define PSP_I2C_REQ_BUS_CMD		0x64
 #define PSP_I2C_REQ_RETRY_CNT		400
 #define PSP_I2C_REQ_RETRY_DELAY_US	(25 * USEC_PER_MSEC)
 #define PSP_I2C_REQ_STS_OK		0x0
 #define PSP_I2C_REQ_STS_BUS_BUSY	0x1
 #define PSP_I2C_REQ_STS_INV_PARAM	0x3
 
-#define PSP_MBOX_FIELDS_STS		GENMASK(15, 0)
-#define PSP_MBOX_FIELDS_CMD		GENMASK(23, 16)
-#define PSP_MBOX_FIELDS_RESERVED	GENMASK(29, 24)
-#define PSP_MBOX_FIELDS_RECOVERY	BIT(30)
-#define PSP_MBOX_FIELDS_READY		BIT(31)
-
-struct psp_req_buffer_hdr {
-	u32 total_size;
-	u32 status;
-};
-
 enum psp_i2c_req_type {
 	PSP_I2C_REQ_ACQUIRE,
 	PSP_I2C_REQ_RELEASE,
@@ -47,118 +27,13 @@ struct psp_i2c_req {
 	enum psp_i2c_req_type type;
 };
 
-struct psp_mbox {
-	u32 cmd_fields;
-	u64 i2c_req_addr;
-} __packed;
-
 static DEFINE_MUTEX(psp_i2c_access_mutex);
 static unsigned long psp_i2c_sem_acquired;
-static void __iomem *mbox_iomem;
 static u32 psp_i2c_access_count;
 static bool psp_i2c_mbox_fail;
 static struct device *psp_i2c_dev;
 
-/*
- * Implementation of PSP-x86 i2c-arbitration mailbox introduced for AMD Cezanne
- * family of SoCs.
- */
-
-static int psp_get_mbox_addr(unsigned long *mbox_addr)
-{
-	unsigned long long psp_mmio;
-
-	if (rdmsrl_safe(MSR_AMD_PSP_ADDR, &psp_mmio))
-		return -EIO;
-
-	*mbox_addr = (unsigned long)(psp_mmio + PSP_MBOX_OFFSET);
-
-	return 0;
-}
-
-static int psp_mbox_probe(void)
-{
-	unsigned long mbox_addr;
-	int ret;
-
-	ret = psp_get_mbox_addr(&mbox_addr);
-	if (ret)
-		return ret;
-
-	mbox_iomem = ioremap(mbox_addr, sizeof(struct psp_mbox));
-	if (!mbox_iomem)
-		return -ENOMEM;
-
-	return 0;
-}
-
-/* Recovery field should be equal 0 to start sending commands */
-static int psp_check_mbox_recovery(struct psp_mbox __iomem *mbox)
-{
-	u32 tmp;
-
-	tmp = readl(&mbox->cmd_fields);
-
-	return FIELD_GET(PSP_MBOX_FIELDS_RECOVERY, tmp);
-}
-
-static int psp_wait_cmd(struct psp_mbox __iomem *mbox)
-{
-	u32 tmp, expected;
-
-	/* Expect mbox_cmd to be cleared and ready bit to be set by PSP */
-	expected = FIELD_PREP(PSP_MBOX_FIELDS_READY, 1);
-
-	/*
-	 * Check for readiness of PSP mailbox in a tight loop in order to
-	 * process further as soon as command was consumed.
-	 */
-	return readl_poll_timeout(&mbox->cmd_fields, tmp, (tmp == expected),
-				  0, PSP_CMD_TIMEOUT_US);
-}
-
-/* Status equal to 0 means that PSP succeed processing command */
-static u32 psp_check_mbox_sts(struct psp_mbox __iomem *mbox)
-{
-	u32 cmd_reg;
-
-	cmd_reg = readl(&mbox->cmd_fields);
-
-	return FIELD_GET(PSP_MBOX_FIELDS_STS, cmd_reg);
-}
-
-static int psp_send_cmd(struct psp_i2c_req *req)
-{
-	struct psp_mbox __iomem *mbox = mbox_iomem;
-	phys_addr_t req_addr;
-	u32 cmd_reg;
-
-	if (psp_check_mbox_recovery(mbox))
-		return -EIO;
-
-	if (psp_wait_cmd(mbox))
-		return -EBUSY;
-
-	/*
-	 * Fill mailbox with address of command-response buffer, which will be
-	 * used for sending i2c requests as well as reading status returned by
-	 * PSP. Use physical address of buffer, since PSP will map this region.
-	 */
-	req_addr = __psp_pa((void *)req);
-	writeq(req_addr, &mbox->i2c_req_addr);
-
-	/* Write command register to trigger processing */
-	cmd_reg = FIELD_PREP(PSP_MBOX_FIELDS_CMD, PSP_I2C_REQ_BUS_CMD);
-	writel(cmd_reg, &mbox->cmd_fields);
-
-	if (psp_wait_cmd(mbox))
-		return -ETIMEDOUT;
-
-	if (psp_check_mbox_sts(mbox))
-		return -EIO;
-
-	return 0;
-}
+static int (*_psp_send_i2c_req)(struct psp_i2c_req *req);
 
 /* Helper to verify status returned by PSP */
 static int check_i2c_req_sts(struct psp_i2c_req *req)
@@ -179,22 +54,36 @@ static int check_i2c_req_sts(struct psp_i2c_req *req)
 	}
 }
 
-static int psp_send_check_i2c_req(struct psp_i2c_req *req)
+/*
+ * Errors in x86-PSP i2c-arbitration protocol may occur at two levels:
+ * 1. mailbox communication - PSP is not operational or some IO errors with
+ *    basic communication had happened.
+ * 2. i2c-requests - PSP refuses to grant i2c arbitration to x86 for too long.
+ *
+ * In order to distinguish between these in error handling code all mailbox
+ * communication errors on the first level (from CCP symbols) will be passed
+ * up and if -EIO is returned the second level will be checked.
+ */
+static int psp_send_i2c_req_cezanne(struct psp_i2c_req *req)
 {
-	/*
-	 * Errors in x86-PSP i2c-arbitration protocol may occur at two levels:
-	 * 1. mailbox communication - PSP is not operational or some IO errors
-	 * with basic communication had happened;
-	 * 2. i2c-requests - PSP refuses to grant i2c arbitration to x86 for too
-	 * long.
-	 * In order to distinguish between these two in error handling code, all
-	 * errors on the first level (returned by psp_send_cmd) are shadowed by
-	 * -EIO.
-	 */
-	if (psp_send_cmd(req))
-		return -EIO;
+	int ret;
+
+	ret = psp_send_platform_access_msg(PSP_I2C_REQ_BUS_CMD, (struct psp_request *)req);
+	if (ret == -EIO)
+		return check_i2c_req_sts(req);
 
-	return check_i2c_req_sts(req);
+	return ret;
+}
+
+static int psp_send_i2c_req_doorbell(struct psp_i2c_req *req)
+{
+	int ret;
+
+	ret = psp_ring_platform_doorbell(req->type, &req->hdr.status);
+	if (ret == -EIO)
+		return check_i2c_req_sts(req);
+
+	return ret;
 }
 
 static int psp_send_i2c_req(enum psp_i2c_req_type i2c_req_type)
@@ -208,11 +97,11 @@ static int psp_send_i2c_req(enum psp_i2c_req_type i2c_req_type)
 	if (!req)
 		return -ENOMEM;
 
-	req->hdr.total_size = sizeof(*req);
+	req->hdr.payload_size = sizeof(*req);
 	req->type = i2c_req_type;
 
 	start = jiffies;
-	ret = read_poll_timeout(psp_send_check_i2c_req, status,
+	ret = read_poll_timeout(_psp_send_i2c_req, status,
 				(status != -EBUSY),
 				PSP_I2C_REQ_RETRY_DELAY_US,
 				PSP_I2C_REQ_RETRY_CNT * PSP_I2C_REQ_RETRY_DELAY_US,
@@ -387,7 +276,10 @@ static const struct i2c_lock_operations i2c_dw_psp_lock_ops = {
 
 int i2c_dw_amdpsp_probe_lock_support(struct dw_i2c_dev *dev)
 {
-	int ret;
+	struct pci_dev *rdev;
+
+	if (!IS_REACHABLE(CONFIG_CRYPTO_DEV_CCP_DD))
+		return -ENODEV;
 
 	if (!dev)
 		return -ENODEV;
@@ -399,11 +291,18 @@ int i2c_dw_amdpsp_probe_lock_support(struct dw_i2c_dev *dev)
 	if (psp_i2c_dev)
 		return -EEXIST;
 
-	psp_i2c_dev = dev->dev;
+	/* Cezanne uses platform mailbox, Mendocino and later use doorbell */
+	rdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0));
+	if (rdev->device == 0x1630)
+		_psp_send_i2c_req = psp_send_i2c_req_cezanne;
+	else
+		_psp_send_i2c_req = psp_send_i2c_req_doorbell;
+	pci_dev_put(rdev);
 
-	ret = psp_mbox_probe();
-	if (ret)
-		return ret;
+	if (psp_check_platform_access_status())
+		return -EPROBE_DEFER;
+
+	psp_i2c_dev = dev->dev;
 
 	dev_info(psp_i2c_dev, "I2C bus managed by AMD PSP\n");
 
@@ -417,9 +316,3 @@ int i2c_dw_amdpsp_probe_lock_support(struct dw_i2c_dev *dev)
 
 	return 0;
 }
-
-/* Unmap area used as a mailbox with PSP */
-void i2c_dw_amdpsp_remove_lock_support(struct dw_i2c_dev *dev)
-{
-	iounmap(mbox_iomem);
-}
diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h
index 050d8c63ad3c..c5d87aae39c6 100644
--- a/drivers/i2c/busses/i2c-designware-core.h
+++ b/drivers/i2c/busses/i2c-designware-core.h
@@ -383,7 +383,6 @@ int i2c_dw_baytrail_probe_lock_support(struct dw_i2c_dev *dev);
 
 #if IS_ENABLED(CONFIG_I2C_DESIGNWARE_AMDPSP)
 int i2c_dw_amdpsp_probe_lock_support(struct dw_i2c_dev *dev);
-void i2c_dw_amdpsp_remove_lock_support(struct dw_i2c_dev *dev);
 #endif
 
 int i2c_dw_validate_speed(struct dw_i2c_dev *dev);
diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c
index 74182db03a88..89ad88c54754 100644
--- a/drivers/i2c/busses/i2c-designware-platdrv.c
+++ b/drivers/i2c/busses/i2c-designware-platdrv.c
@@ -214,7 +214,6 @@ static const struct i2c_dw_semaphore_callbacks i2c_dw_semaphore_cb_table[] = {
 #ifdef CONFIG_I2C_DESIGNWARE_AMDPSP
 	{
 		.probe = i2c_dw_amdpsp_probe_lock_support,
-		.remove = i2c_dw_amdpsp_remove_lock_support,
 	},
 #endif
 	{}
diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c
index cf5bacf3a488..42189a5f2905 100644
--- a/drivers/i2c/busses/i2c-imx.c
+++ b/drivers/i2c/busses/i2c-imx.c
@@ -1482,17 +1482,11 @@ static int i2c_imx_probe(struct platform_device *pdev)
 	ACPI_COMPANION_SET(&i2c_imx->adapter.dev, ACPI_COMPANION(&pdev->dev));
 
 	/* Get I2C clock */
-	i2c_imx->clk = devm_clk_get(&pdev->dev, NULL);
+	i2c_imx->clk = devm_clk_get_enabled(&pdev->dev, NULL);
 	if (IS_ERR(i2c_imx->clk))
 		return dev_err_probe(&pdev->dev, PTR_ERR(i2c_imx->clk),
 				     "can't get I2C clock\n");
 
-	ret = clk_prepare_enable(i2c_imx->clk);
-	if (ret) {
-		dev_err(&pdev->dev, "can't enable I2C clock, ret=%d\n", ret);
-		return ret;
-	}
-
 	/* Init queue */
 	init_waitqueue_head(&i2c_imx->queue);
 
@@ -1564,7 +1558,6 @@ rpm_disable:
 	pm_runtime_disable(&pdev->dev);
 	pm_runtime_set_suspended(&pdev->dev);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
-	clk_disable_unprepare(i2c_imx->clk);
 	return ret;
 }
 
@@ -1590,7 +1583,6 @@ static int i2c_imx_remove(struct platform_device *pdev)
 		imx_i2c_write_reg(0, i2c_imx, IMX_I2C_IFDR);
 		imx_i2c_write_reg(0, i2c_imx, IMX_I2C_I2CR);
 		imx_i2c_write_reg(0, i2c_imx, IMX_I2C_I2SR);
-		clk_disable(i2c_imx->clk);
 	}
 
 	clk_notifier_unregister(i2c_imx->clk, &i2c_imx->clk_change_nb);
@@ -1598,8 +1590,6 @@ static int i2c_imx_remove(struct platform_device *pdev)
 	if (irq >= 0)
 		free_irq(irq, i2c_imx);
 
-	clk_unprepare(i2c_imx->clk);
-
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
diff --git a/drivers/i2c/busses/i2c-mpc.c b/drivers/i2c/busses/i2c-mpc.c
index 81ac92bb4f6f..cfd074ee6d54 100644
--- a/drivers/i2c/busses/i2c-mpc.c
+++ b/drivers/i2c/busses/i2c-mpc.c
@@ -770,7 +770,6 @@ static const struct i2c_algorithm mpc_algo = {
 static struct i2c_adapter mpc_ops = {
 	.owner = THIS_MODULE,
 	.algo = &mpc_algo,
-	.timeout = HZ,
 };
 
 static struct i2c_bus_recovery_info fsl_i2c_recovery_info = {
@@ -781,11 +780,9 @@ static int fsl_i2c_probe(struct platform_device *op)
 {
 	const struct mpc_i2c_data *data;
 	struct mpc_i2c *i2c;
-	const u32 *prop;
-	u32 clock = MPC_I2C_CLOCK_LEGACY;
-	int result = 0;
-	int plen;
 	struct clk *clk;
+	int result;
+	u32 clock;
 	int err;
 
 	i2c = devm_kzalloc(&op->dev, sizeof(*i2c), GFP_KERNEL);
@@ -831,10 +828,10 @@ static int fsl_i2c_probe(struct platform_device *op)
 	if (of_property_read_bool(op->dev.of_node, "fsl,preserve-clocking")) {
 		clock = MPC_I2C_CLOCK_PRESERVE;
 	} else {
-		prop = of_get_property(op->dev.of_node, "clock-frequency",
-					&plen);
-		if (prop && plen == sizeof(u32))
-			clock = *prop;
+		result = of_property_read_u32(op->dev.of_node,
+					      "clock-frequency", &clock);
+		if (result)
+			clock = MPC_I2C_CLOCK_LEGACY;
 	}
 
 	data = device_get_match_data(&op->dev);
@@ -842,16 +839,30 @@ static int fsl_i2c_probe(struct platform_device *op)
 		data->setup(op->dev.of_node, i2c, clock);
 	} else {
 		/* Backwards compatibility */
-		if (of_get_property(op->dev.of_node, "dfsrr", NULL))
+		if (of_property_read_bool(op->dev.of_node, "dfsrr"))
 			mpc_i2c_setup_8xxx(op->dev.of_node, i2c, clock);
 	}
 
-	prop = of_get_property(op->dev.of_node, "fsl,timeout", &plen);
-	if (prop && plen == sizeof(u32)) {
-		mpc_ops.timeout = *prop * HZ / 1000000;
+	/*
+	 * "fsl,timeout" has been marked as deprecated and, to maintain
+	 * backward compatibility, we will only look for it if
+	 * "i2c-scl-clk-low-timeout-us" is not present.
+	 */
+	result = of_property_read_u32(op->dev.of_node,
+				      "i2c-scl-clk-low-timeout-us",
+				      &mpc_ops.timeout);
+	if (result == -EINVAL)
+		result = of_property_read_u32(op->dev.of_node,
+					      "fsl,timeout", &mpc_ops.timeout);
+
+	if (!result) {
+		mpc_ops.timeout *= HZ / 1000000;
 		if (mpc_ops.timeout < 5)
 			mpc_ops.timeout = 5;
+	} else {
+		mpc_ops.timeout = HZ;
 	}
+
 	dev_info(i2c->dev, "timeout %u us\n", mpc_ops.timeout * 1000000 / HZ);
 
 	if (of_property_read_bool(op->dev.of_node, "fsl,i2c-erratum-a004447"))
diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c
index 43dd966d5ef5..a43c4d77739a 100644
--- a/drivers/i2c/busses/i2c-mt65xx.c
+++ b/drivers/i2c/busses/i2c-mt65xx.c
@@ -431,6 +431,18 @@ static const struct mtk_i2c_compatible mt8168_compat = {
 	.max_dma_support = 33,
 };
 
+static const struct mtk_i2c_compatible mt7981_compat = {
+	.regs = mt_i2c_regs_v3,
+	.pmic_i2c = 0,
+	.dcm = 0,
+	.auto_restart = 1,
+	.aux_len_reg = 1,
+	.timing_adjust = 1,
+	.dma_sync = 1,
+	.ltiming_adjust = 1,
+	.max_dma_support = 33
+};
+
 static const struct mtk_i2c_compatible mt7986_compat = {
 	.quirks = &mt7622_i2c_quirks,
 	.regs = mt_i2c_regs_v1,
@@ -516,6 +528,7 @@ static const struct of_device_id mtk_i2c_of_match[] = {
 	{ .compatible = "mediatek,mt6577-i2c", .data = &mt6577_compat },
 	{ .compatible = "mediatek,mt6589-i2c", .data = &mt6589_compat },
 	{ .compatible = "mediatek,mt7622-i2c", .data = &mt7622_compat },
+	{ .compatible = "mediatek,mt7981-i2c", .data = &mt7981_compat },
 	{ .compatible = "mediatek,mt7986-i2c", .data = &mt7986_compat },
 	{ .compatible = "mediatek,mt8168-i2c", .data = &mt8168_compat },
 	{ .compatible = "mediatek,mt8173-i2c", .data = &mt8173_compat },
@@ -1546,7 +1559,7 @@ static struct platform_driver mtk_i2c_driver = {
 	.driver = {
 		.name = I2C_DRV_NAME,
 		.pm = &mtk_i2c_pm,
-		.of_match_table = of_match_ptr(mtk_i2c_of_match),
+		.of_match_table = mtk_i2c_of_match,
 	},
 };
 
diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c
index f9ae520aed22..2b4e2be51318 100644
--- a/drivers/i2c/busses/i2c-omap.c
+++ b/drivers/i2c/busses/i2c-omap.c
@@ -1525,14 +1525,17 @@ static int omap_i2c_remove(struct platform_device *pdev)
 	int ret;
 
 	i2c_del_adapter(&omap->adapter);
-	ret = pm_runtime_resume_and_get(&pdev->dev);
+
+	ret = pm_runtime_get_sync(&pdev->dev);
 	if (ret < 0)
-		return ret;
+		dev_err(omap->dev, "Failed to resume hardware, skip disable\n");
+	else
+		omap_i2c_write_reg(omap, OMAP_I2C_CON_REG, 0);
 
-	omap_i2c_write_reg(omap, OMAP_I2C_CON_REG, 0);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
+
 	return 0;
 }
 
diff --git a/drivers/i2c/busses/i2c-owl.c b/drivers/i2c/busses/i2c-owl.c
index 98882fe4e965..99ddd8894964 100644
--- a/drivers/i2c/busses/i2c-owl.c
+++ b/drivers/i2c/busses/i2c-owl.c
@@ -519,7 +519,7 @@ static struct platform_driver owl_i2c_driver = {
 	.probe		= owl_i2c_probe,
 	.driver		= {
 		.name	= "owl-i2c",
-		.of_match_table = of_match_ptr(owl_i2c_of_match),
+		.of_match_table = owl_i2c_of_match,
 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
diff --git a/drivers/i2c/busses/i2c-powermac.c b/drivers/i2c/busses/i2c-powermac.c
index 2e74747eec9c..ec706a3aba26 100644
--- a/drivers/i2c/busses/i2c-powermac.c
+++ b/drivers/i2c/busses/i2c-powermac.c
@@ -284,7 +284,7 @@ static bool i2c_powermac_get_type(struct i2c_adapter *adap,
 	 */
 
 	/* First try proper modalias */
-	if (of_modalias_node(node, tmp, sizeof(tmp)) >= 0) {
+	if (of_alias_from_compatible(node, tmp, sizeof(tmp)) >= 0) {
 		snprintf(type, type_size, "MAC,%s", tmp);
 		return true;
 	}
diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c
index b605b6e43cb9..f9fa5308556b 100644
--- a/drivers/i2c/busses/i2c-pxa.c
+++ b/drivers/i2c/busses/i2c-pxa.c
@@ -1261,10 +1261,8 @@ static int i2c_pxa_probe_dt(struct platform_device *pdev, struct pxa_i2c *i2c,
 	/* For device tree we always use the dynamic or alias-assigned ID */
 	i2c->adap.nr = -1;
 
-	if (of_get_property(np, "mrvl,i2c-polling", NULL))
-		i2c->use_pio = 1;
-	if (of_get_property(np, "mrvl,i2c-fast-mode", NULL))
-		i2c->fast_mode = 1;
+	i2c->use_pio = of_property_read_bool(np, "mrvl,i2c-polling");
+	i2c->fast_mode = of_property_read_bool(np, "mrvl,i2c-fast-mode");
 
 	*i2c_types = (enum pxa_i2c_types)(of_id->data);
 
diff --git a/drivers/i2c/busses/i2c-synquacer.c b/drivers/i2c/busses/i2c-synquacer.c
index e4026c5416b1..50d19cf99a03 100644
--- a/drivers/i2c/busses/i2c-synquacer.c
+++ b/drivers/i2c/busses/i2c-synquacer.c
@@ -629,7 +629,7 @@ static int synquacer_i2c_remove(struct platform_device *pdev)
 	return 0;
 };
 
-static const struct of_device_id synquacer_i2c_dt_ids[] = {
+static const struct of_device_id synquacer_i2c_dt_ids[] __maybe_unused = {
 	{ .compatible = "socionext,synquacer-i2c" },
 	{ /* sentinel */ }
 };
diff --git a/drivers/i2c/busses/i2c-xiic.c b/drivers/i2c/busses/i2c-xiic.c
index dbb792fc197e..8a3d9817cb41 100644
--- a/drivers/i2c/busses/i2c-xiic.c
+++ b/drivers/i2c/busses/i2c-xiic.c
@@ -1164,7 +1164,7 @@ static int xiic_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
 	err = xiic_start_xfer(i2c, msgs, num);
 	if (err < 0) {
 		dev_err(adap->dev.parent, "Error xiic_start_xfer\n");
-		return err;
+		goto out;
 	}
 
 	err = wait_for_completion_timeout(&i2c->completion, XIIC_XFER_TIMEOUT);
@@ -1178,6 +1178,8 @@ static int xiic_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
 		err = (i2c->state == STATE_DONE) ? num : -EIO;
 	}
 	mutex_unlock(&i2c->lock);
+
+out:
 	pm_runtime_mark_last_busy(i2c->dev);
 	pm_runtime_put_autosuspend(i2c->dev);
 	return err;
@@ -1199,11 +1201,11 @@ static const struct i2c_adapter xiic_adapter = {
 	.algo = &xiic_algorithm,
 };
 
+#if defined(CONFIG_OF)
 static const struct xiic_version_data xiic_2_00 = {
 	.quirks = DYNAMIC_MODE_READ_BROKEN_BIT,
 };
 
-#if defined(CONFIG_OF)
 static const struct of_device_id xiic_of_match[] = {
 	{ .compatible = "xlnx,xps-iic-2.00.a", .data = &xiic_2_00 },
 	{ .compatible = "xlnx,axi-iic-2.1", },
@@ -1233,8 +1235,7 @@ static int xiic_i2c_probe(struct platform_device *pdev)
 		i2c->quirks = data->quirks;
 	}
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->base = devm_ioremap_resource(&pdev->dev, res);
+	i2c->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(i2c->base))
 		return PTR_ERR(i2c->base);
 
diff --git a/drivers/i2c/i2c-core-of.c b/drivers/i2c/i2c-core-of.c
index 545436b7dd53..a6c407d36800 100644
--- a/drivers/i2c/i2c-core-of.c
+++ b/drivers/i2c/i2c-core-of.c
@@ -27,7 +27,7 @@ int of_i2c_get_board_info(struct device *dev, struct device_node *node,
 
 	memset(info, 0, sizeof(*info));
 
-	if (of_modalias_node(node, info->type, sizeof(info->type)) < 0) {
+	if (of_alias_from_compatible(node, info->type, sizeof(info->type)) < 0) {
 		dev_err(dev, "of_i2c: modalias failure on %pOF\n", node);
 		return -EINVAL;
 	}
@@ -55,7 +55,7 @@ int of_i2c_get_board_info(struct device *dev, struct device_node *node,
 	if (of_property_read_bool(node, "host-notify"))
 		info->flags |= I2C_CLIENT_HOST_NOTIFY;
 
-	if (of_get_property(node, "wakeup-source", NULL))
+	if (of_property_read_bool(node, "wakeup-source"))
 		info->flags |= I2C_CLIENT_WAKE;
 
 	return 0;
diff --git a/drivers/i2c/i2c-dev.c b/drivers/i2c/i2c-dev.c
index 95a0b63ac560..a01b59e3599b 100644
--- a/drivers/i2c/i2c-dev.c
+++ b/drivers/i2c/i2c-dev.c
@@ -751,7 +751,7 @@ static int __init i2c_dev_init(void)
 	if (res)
 		goto out;
 
-	i2c_dev_class = class_create(THIS_MODULE, "i2c-dev");
+	i2c_dev_class = class_create("i2c-dev");
 	if (IS_ERR(i2c_dev_class)) {
 		res = PTR_ERR(i2c_dev_class);
 		goto out_unreg_chrdev;
diff --git a/drivers/iio/Kconfig b/drivers/iio/Kconfig
index b190846c3dc2..52eb46ef84c1 100644
--- a/drivers/iio/Kconfig
+++ b/drivers/iio/Kconfig
@@ -30,6 +30,9 @@ config IIO_CONFIGFS
 	  (e.g. software triggers). For more info see
 	  Documentation/iio/iio_configfs.rst.
 
+config IIO_GTS_HELPER
+	tristate
+
 config IIO_TRIGGER
 	bool "Enable triggered sampling support"
 	help
diff --git a/drivers/iio/Makefile b/drivers/iio/Makefile
index 3be08cdadd7e..9622347a1c1b 100644
--- a/drivers/iio/Makefile
+++ b/drivers/iio/Makefile
@@ -9,6 +9,7 @@ industrialio-$(CONFIG_IIO_BUFFER) += industrialio-buffer.o
 industrialio-$(CONFIG_IIO_TRIGGER) += industrialio-trigger.o
 
 obj-$(CONFIG_IIO_CONFIGFS) += industrialio-configfs.o
+obj-$(CONFIG_IIO_GTS_HELPER) += industrialio-gts-helper.o
 obj-$(CONFIG_IIO_SW_DEVICE) += industrialio-sw-device.o
 obj-$(CONFIG_IIO_SW_TRIGGER) += industrialio-sw-trigger.o
 obj-$(CONFIG_IIO_TRIGGERED_EVENT) += industrialio-triggered-event.o
diff --git a/drivers/iio/accel/bma400_core.c b/drivers/iio/accel/bma400_core.c
index 623f37cbaf50..a68b845f5b4f 100644
--- a/drivers/iio/accel/bma400_core.c
+++ b/drivers/iio/accel/bma400_core.c
@@ -1688,7 +1688,7 @@ static irqreturn_t bma400_interrupt(int irq, void *private)
 
 	if (FIELD_GET(BMA400_INT_DRDY_MSK, le16_to_cpu(data->status))) {
 		mutex_unlock(&data->mutex);
-		iio_trigger_poll_chained(data->trig);
+		iio_trigger_poll_nested(data->trig);
 		return IRQ_HANDLED;
 	}
 
diff --git a/drivers/iio/accel/kionix-kx022a.c b/drivers/iio/accel/kionix-kx022a.c
index 1c3a72380fb8..f98393d74666 100644
--- a/drivers/iio/accel/kionix-kx022a.c
+++ b/drivers/iio/accel/kionix-kx022a.c
@@ -162,7 +162,6 @@ struct kx022a_data {
 	int inc_reg;
 	int ien_reg;
 
-	unsigned int g_range;
 	unsigned int state;
 	unsigned int odr_ns;
 
@@ -900,7 +899,7 @@ static irqreturn_t kx022a_irq_thread_handler(int irq, void *private)
 	mutex_lock(&data->mutex);
 
 	if (data->trigger_enabled) {
-		iio_trigger_poll_chained(data->trig);
+		iio_trigger_poll_nested(data->trig);
 		ret = IRQ_HANDLED;
 	}
 
diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c
index f97fb68e3a71..ea14e3aaa30a 100644
--- a/drivers/iio/accel/mma8452.c
+++ b/drivers/iio/accel/mma8452.c
@@ -1067,7 +1067,7 @@ static irqreturn_t mma8452_interrupt(int irq, void *p)
 		return IRQ_NONE;
 
 	if (src & MMA8452_INT_DRDY) {
-		iio_trigger_poll_chained(indio_dev->trig);
+		iio_trigger_poll_nested(indio_dev->trig);
 		ret = IRQ_HANDLED;
 	}
 
diff --git a/drivers/iio/accel/msa311.c b/drivers/iio/accel/msa311.c
index af94d3adf6d8..6690fa37da8f 100644
--- a/drivers/iio/accel/msa311.c
+++ b/drivers/iio/accel/msa311.c
@@ -951,7 +951,7 @@ static irqreturn_t msa311_irq_thread(int irq, void *p)
 	}
 
 	if (new_data_int_enabled)
-		iio_trigger_poll_chained(msa311->new_data_trig);
+		iio_trigger_poll_nested(msa311->new_data_trig);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/iio/accel/st_accel.h b/drivers/iio/accel/st_accel.h
index 56ed0c776d4a..e7525615712b 100644
--- a/drivers/iio/accel/st_accel.h
+++ b/drivers/iio/accel/st_accel.h
@@ -39,6 +39,7 @@
 #define LIS302DL_ACCEL_DEV_NAME		"lis302dl"
 #define LSM303C_ACCEL_DEV_NAME		"lsm303c_accel"
 #define SC7A20_ACCEL_DEV_NAME		"sc7a20"
+#define IIS328DQ_ACCEL_DEV_NAME		"iis328dq"
 
 
 #ifdef CONFIG_IIO_BUFFER
diff --git a/drivers/iio/accel/st_accel_core.c b/drivers/iio/accel/st_accel_core.c
index 6b8562f684d5..5f7d81b44b1d 100644
--- a/drivers/iio/accel/st_accel_core.c
+++ b/drivers/iio/accel/st_accel_core.c
@@ -517,6 +517,7 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = H3LIS331DL_ACCEL_DEV_NAME,
+			[1] = IIS328DQ_ACCEL_DEV_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_accel_12bit_channels,
 		.odr = {
diff --git a/drivers/iio/accel/st_accel_i2c.c b/drivers/iio/accel/st_accel_i2c.c
index 3f02fd5d5946..fb9e2d6f4210 100644
--- a/drivers/iio/accel/st_accel_i2c.c
+++ b/drivers/iio/accel/st_accel_i2c.c
@@ -119,6 +119,10 @@ static const struct of_device_id st_accel_of_match[] = {
 		.compatible = "silan,sc7a20",
 		.data = SC7A20_ACCEL_DEV_NAME,
 	},
+	{
+		.compatible = "st,iis328dq",
+		.data = IIS328DQ_ACCEL_DEV_NAME,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, st_accel_of_match);
@@ -157,6 +161,7 @@ static const struct i2c_device_id st_accel_id_table[] = {
 	{ LIS302DL_ACCEL_DEV_NAME },
 	{ LSM303C_ACCEL_DEV_NAME },
 	{ SC7A20_ACCEL_DEV_NAME },
+	{ IIS328DQ_ACCEL_DEV_NAME },
 	{},
 };
 MODULE_DEVICE_TABLE(i2c, st_accel_id_table);
diff --git a/drivers/iio/accel/st_accel_spi.c b/drivers/iio/accel/st_accel_spi.c
index 5740dc1820bd..f72a24f45322 100644
--- a/drivers/iio/accel/st_accel_spi.c
+++ b/drivers/iio/accel/st_accel_spi.c
@@ -100,6 +100,10 @@ static const struct of_device_id st_accel_of_match[] = {
 		.compatible = "st,lsm303c-accel",
 		.data = LSM303C_ACCEL_DEV_NAME,
 	},
+	{
+		.compatible = "st,iis328dq",
+		.data = IIS328DQ_ACCEL_DEV_NAME,
+	},
 	{}
 };
 MODULE_DEVICE_TABLE(of, st_accel_of_match);
@@ -157,6 +161,7 @@ static const struct spi_device_id st_accel_id_table[] = {
 	{ LIS3DE_ACCEL_DEV_NAME },
 	{ LIS302DL_ACCEL_DEV_NAME },
 	{ LSM303C_ACCEL_DEV_NAME },
+	{ IIS328DQ_ACCEL_DEV_NAME },
 	{},
 };
 MODULE_DEVICE_TABLE(spi, st_accel_id_table);
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index 45af2302be53..eb2b09ef5d5b 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -1229,6 +1229,16 @@ config TI_ADS7924
 	  This driver can also be built as a module. If so, the module will be
 	  called ti-ads7924.
 
+config TI_ADS1100
+	tristate "Texas Instruments ADS1100 and ADS1000 ADC"
+	depends on I2C
+	help
+	  If you say yes here you get support for Texas Instruments ADS1100 and
+	  ADS1000 ADC chips.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called ti-ads1100.
+
 config TI_ADS7950
 	tristate "Texas Instruments ADS7950 ADC driver"
 	depends on SPI && GPIOLIB
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index 36c18177322a..e07e4a3e6237 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -108,6 +108,7 @@ obj-$(CONFIG_TI_ADC108S102) += ti-adc108s102.o
 obj-$(CONFIG_TI_ADC128S052) += ti-adc128s052.o
 obj-$(CONFIG_TI_ADC161S626) += ti-adc161s626.o
 obj-$(CONFIG_TI_ADS1015) += ti-ads1015.o
+obj-$(CONFIG_TI_ADS1100) += ti-ads1100.o
 obj-$(CONFIG_TI_ADS7924) += ti-ads7924.o
 obj-$(CONFIG_TI_ADS7950) += ti-ads7950.o
 obj-$(CONFIG_TI_ADS8344) += ti-ads8344.o
diff --git a/drivers/iio/adc/ad7292.c b/drivers/iio/adc/ad7292.c
index a2f9fda25ff3..cccacec5db6d 100644
--- a/drivers/iio/adc/ad7292.c
+++ b/drivers/iio/adc/ad7292.c
@@ -8,6 +8,7 @@
 #include <linux/bitfield.h>
 #include <linux/device.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/regulator/consumer.h>
 #include <linux/spi/spi.h>
 
diff --git a/drivers/iio/adc/ad7606.c b/drivers/iio/adc/ad7606.c
index dd6b603f65ea..1928d9ae5bcf 100644
--- a/drivers/iio/adc/ad7606.c
+++ b/drivers/iio/adc/ad7606.c
@@ -477,7 +477,7 @@ static irqreturn_t ad7606_interrupt(int irq, void *dev_id)
 
 	if (iio_buffer_enabled(indio_dev)) {
 		gpiod_set_value(st->gpio_convst, 0);
-		iio_trigger_poll_chained(st->trig);
+		iio_trigger_poll_nested(st->trig);
 	} else {
 		complete(&st->completion);
 	}
diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c
index 50d02e5fc6fc..df67b63ccf69 100644
--- a/drivers/iio/adc/at91-sama5d2_adc.c
+++ b/drivers/iio/adc/at91-sama5d2_adc.c
@@ -1194,7 +1194,7 @@ static void at91_dma_buffer_done(void *data)
 {
 	struct iio_dev *indio_dev = data;
 
-	iio_trigger_poll_chained(indio_dev->trig);
+	iio_trigger_poll_nested(indio_dev->trig);
 }
 
 static int at91_adc_dma_start(struct iio_dev *indio_dev)
@@ -1409,7 +1409,7 @@ static struct iio_trigger *at91_adc_allocate_trigger(struct iio_dev *indio,
 	trig = devm_iio_trigger_alloc(&indio->dev, "%s-dev%d-%s", indio->name,
 				iio_device_id(indio), trigger_name);
 	if (!trig)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	trig->dev.parent = indio->dev.parent;
 	iio_trigger_set_drvdata(trig, indio);
@@ -2400,12 +2400,8 @@ static int at91_adc_probe(struct platform_device *pdev)
 	st->dma_st.phys_addr = res->start;
 
 	st->irq = platform_get_irq(pdev, 0);
-	if (st->irq <= 0) {
-		if (!st->irq)
-			st->irq = -ENXIO;
-
+	if (st->irq < 0)
 		return st->irq;
-	}
 
 	st->per_clk = devm_clk_get(&pdev->dev, "adc_clk");
 	if (IS_ERR(st->per_clk))
diff --git a/drivers/iio/adc/axp20x_adc.c b/drivers/iio/adc/axp20x_adc.c
index 53bf7d4899d2..75bda94dbce1 100644
--- a/drivers/iio/adc/axp20x_adc.c
+++ b/drivers/iio/adc/axp20x_adc.c
@@ -5,6 +5,7 @@
  *	Quentin Schulz <quentin.schulz@free-electrons.com>
  */
 
+#include <linux/bitfield.h>
 #include <linux/completion.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
@@ -22,20 +23,20 @@
 #include <linux/mfd/axp20x.h>
 
 #define AXP20X_ADC_EN1_MASK			GENMASK(7, 0)
-
 #define AXP20X_ADC_EN2_MASK			(GENMASK(3, 2) | BIT(7))
+
 #define AXP22X_ADC_EN1_MASK			(GENMASK(7, 5) | BIT(0))
 
 #define AXP20X_GPIO10_IN_RANGE_GPIO0		BIT(0)
 #define AXP20X_GPIO10_IN_RANGE_GPIO1		BIT(1)
-#define AXP20X_GPIO10_IN_RANGE_GPIO0_VAL(x)	((x) & BIT(0))
-#define AXP20X_GPIO10_IN_RANGE_GPIO1_VAL(x)	(((x) & BIT(0)) << 1)
 
 #define AXP20X_ADC_RATE_MASK			GENMASK(7, 6)
-#define AXP813_V_I_ADC_RATE_MASK		GENMASK(5, 4)
-#define AXP813_ADC_RATE_MASK			(AXP20X_ADC_RATE_MASK | AXP813_V_I_ADC_RATE_MASK)
 #define AXP20X_ADC_RATE_HZ(x)			((ilog2((x) / 25) << 6) & AXP20X_ADC_RATE_MASK)
+
 #define AXP22X_ADC_RATE_HZ(x)			((ilog2((x) / 100) << 6) & AXP20X_ADC_RATE_MASK)
+
+#define AXP813_V_I_ADC_RATE_MASK		GENMASK(5, 4)
+#define AXP813_ADC_RATE_MASK			(AXP20X_ADC_RATE_MASK | AXP813_V_I_ADC_RATE_MASK)
 #define AXP813_TS_GPIO0_ADC_RATE_HZ(x)		AXP20X_ADC_RATE_HZ(x)
 #define AXP813_V_I_ADC_RATE_HZ(x)		((ilog2((x) / 100) << 4) & AXP813_V_I_ADC_RATE_MASK)
 #define AXP813_ADC_RATE_HZ(x)			(AXP20X_ADC_RATE_HZ(x) | AXP813_V_I_ADC_RATE_HZ(x))
@@ -234,7 +235,7 @@ static int axp20x_adc_raw(struct iio_dev *indio_dev,
 			  struct iio_chan_spec const *chan, int *val)
 {
 	struct axp20x_adc_iio *info = iio_priv(indio_dev);
-	int size = 12;
+	int ret, size;
 
 	/*
 	 * N.B.:  Unlike the Chinese datasheets tell, the charging current is
@@ -246,10 +247,11 @@ static int axp20x_adc_raw(struct iio_dev *indio_dev,
 	else
 		size = 12;
 
-	*val = axp20x_read_variable_width(info->regmap, chan->address, size);
-	if (*val < 0)
-		return *val;
+	ret = axp20x_read_variable_width(info->regmap, chan->address, size);
+	if (ret < 0)
+		return ret;
 
+	*val = ret;
 	return IIO_VAL_INT;
 }
 
@@ -257,11 +259,13 @@ static int axp22x_adc_raw(struct iio_dev *indio_dev,
 			  struct iio_chan_spec const *chan, int *val)
 {
 	struct axp20x_adc_iio *info = iio_priv(indio_dev);
+	int ret;
 
-	*val = axp20x_read_variable_width(info->regmap, chan->address, 12);
-	if (*val < 0)
-		return *val;
+	ret = axp20x_read_variable_width(info->regmap, chan->address, 12);
+	if (ret < 0)
+		return ret;
 
+	*val = ret;
 	return IIO_VAL_INT;
 }
 
@@ -269,11 +273,13 @@ static int axp813_adc_raw(struct iio_dev *indio_dev,
 			  struct iio_chan_spec const *chan, int *val)
 {
 	struct axp20x_adc_iio *info = iio_priv(indio_dev);
+	int ret;
 
-	*val = axp20x_read_variable_width(info->regmap, chan->address, 12);
-	if (*val < 0)
-		return *val;
+	ret = axp20x_read_variable_width(info->regmap, chan->address, 12);
+	if (ret < 0)
+		return ret;
 
+	*val = ret;
 	return IIO_VAL_INT;
 }
 
@@ -443,27 +449,27 @@ static int axp20x_adc_offset_voltage(struct iio_dev *indio_dev, int channel,
 				     int *val)
 {
 	struct axp20x_adc_iio *info = iio_priv(indio_dev);
+	unsigned int regval;
 	int ret;
 
-	ret = regmap_read(info->regmap, AXP20X_GPIO10_IN_RANGE, val);
+	ret = regmap_read(info->regmap, AXP20X_GPIO10_IN_RANGE, &regval);
 	if (ret < 0)
 		return ret;
 
 	switch (channel) {
 	case AXP20X_GPIO0_V:
-		*val &= AXP20X_GPIO10_IN_RANGE_GPIO0;
+		regval = FIELD_GET(AXP20X_GPIO10_IN_RANGE_GPIO0, regval);
 		break;
 
 	case AXP20X_GPIO1_V:
-		*val &= AXP20X_GPIO10_IN_RANGE_GPIO1;
+		regval = FIELD_GET(AXP20X_GPIO10_IN_RANGE_GPIO1, regval);
 		break;
 
 	default:
 		return -EINVAL;
 	}
 
-	*val = *val ? 700000 : 0;
-
+	*val = regval ? 700000 : 0;
 	return IIO_VAL_INT;
 }
 
@@ -548,7 +554,7 @@ static int axp20x_write_raw(struct iio_dev *indio_dev,
 			    long mask)
 {
 	struct axp20x_adc_iio *info = iio_priv(indio_dev);
-	unsigned int reg, regval;
+	unsigned int regmask, regval;
 
 	/*
 	 * The AXP20X PMIC allows the user to choose between 0V and 0.7V offsets
@@ -560,25 +566,22 @@ static int axp20x_write_raw(struct iio_dev *indio_dev,
 	if (val != 0 && val != 700000)
 		return -EINVAL;
 
-	val = val ? 1 : 0;
-
 	switch (chan->channel) {
 	case AXP20X_GPIO0_V:
-		reg = AXP20X_GPIO10_IN_RANGE_GPIO0;
-		regval = AXP20X_GPIO10_IN_RANGE_GPIO0_VAL(val);
+		regmask = AXP20X_GPIO10_IN_RANGE_GPIO0;
+		regval = FIELD_PREP(AXP20X_GPIO10_IN_RANGE_GPIO0, !!val);
 		break;
 
 	case AXP20X_GPIO1_V:
-		reg = AXP20X_GPIO10_IN_RANGE_GPIO1;
-		regval = AXP20X_GPIO10_IN_RANGE_GPIO1_VAL(val);
+		regmask = AXP20X_GPIO10_IN_RANGE_GPIO1;
+		regval = FIELD_PREP(AXP20X_GPIO10_IN_RANGE_GPIO1, !!val);
 		break;
 
 	default:
 		return -EINVAL;
 	}
 
-	return regmap_update_bits(info->regmap, AXP20X_GPIO10_IN_RANGE, reg,
-				  regval);
+	return regmap_update_bits(info->regmap, AXP20X_GPIO10_IN_RANGE, regmask, regval);
 }
 
 static const struct iio_info axp20x_adc_iio_info = {
@@ -620,9 +623,9 @@ struct axp_data {
 	int				num_channels;
 	struct iio_chan_spec const	*channels;
 	unsigned long			adc_en1_mask;
+	unsigned long			adc_en2_mask;
 	int				(*adc_rate)(struct axp20x_adc_iio *info,
 						    int rate);
-	bool				adc_en2;
 	struct iio_map			*maps;
 };
 
@@ -631,8 +634,8 @@ static const struct axp_data axp20x_data = {
 	.num_channels = ARRAY_SIZE(axp20x_adc_channels),
 	.channels = axp20x_adc_channels,
 	.adc_en1_mask = AXP20X_ADC_EN1_MASK,
+	.adc_en2_mask = AXP20X_ADC_EN2_MASK,
 	.adc_rate = axp20x_adc_rate,
-	.adc_en2 = true,
 	.maps = axp20x_maps,
 };
 
@@ -642,7 +645,6 @@ static const struct axp_data axp22x_data = {
 	.channels = axp22x_adc_channels,
 	.adc_en1_mask = AXP22X_ADC_EN1_MASK,
 	.adc_rate = axp22x_adc_rate,
-	.adc_en2 = false,
 	.maps = axp22x_maps,
 };
 
@@ -652,7 +654,6 @@ static const struct axp_data axp813_data = {
 	.channels = axp813_adc_channels,
 	.adc_en1_mask = AXP22X_ADC_EN1_MASK,
 	.adc_rate = axp813_adc_rate,
-	.adc_en2 = false,
 	.maps = axp22x_maps,
 };
 
@@ -710,10 +711,10 @@ static int axp20x_probe(struct platform_device *pdev)
 	/* Enable the ADCs on IP */
 	regmap_write(info->regmap, AXP20X_ADC_EN1, info->data->adc_en1_mask);
 
-	if (info->data->adc_en2)
-		/* Enable GPIO0/1 and internal temperature ADCs */
+	if (info->data->adc_en2_mask)
 		regmap_update_bits(info->regmap, AXP20X_ADC_EN2,
-				   AXP20X_ADC_EN2_MASK, AXP20X_ADC_EN2_MASK);
+				   info->data->adc_en2_mask,
+				   info->data->adc_en2_mask);
 
 	/* Configure ADCs rate */
 	info->data->adc_rate(info, 100);
@@ -738,7 +739,7 @@ fail_register:
 fail_map:
 	regmap_write(info->regmap, AXP20X_ADC_EN1, 0);
 
-	if (info->data->adc_en2)
+	if (info->data->adc_en2_mask)
 		regmap_write(info->regmap, AXP20X_ADC_EN2, 0);
 
 	return ret;
@@ -754,7 +755,7 @@ static int axp20x_remove(struct platform_device *pdev)
 
 	regmap_write(info->regmap, AXP20X_ADC_EN1, 0);
 
-	if (info->data->adc_en2)
+	if (info->data->adc_en2_mask)
 		regmap_write(info->regmap, AXP20X_ADC_EN2, 0);
 
 	return 0;
diff --git a/drivers/iio/adc/max11410.c b/drivers/iio/adc/max11410.c
index f6895bc9fc4b..6af829349b4e 100644
--- a/drivers/iio/adc/max11410.c
+++ b/drivers/iio/adc/max11410.c
@@ -682,7 +682,7 @@ static irqreturn_t max11410_interrupt(int irq, void *dev_id)
 	struct max11410_state *st = iio_priv(indio_dev);
 
 	if (iio_buffer_enabled(indio_dev))
-		iio_trigger_poll_chained(st->trig);
+		iio_trigger_poll_nested(st->trig);
 	else
 		complete(&st->completion);
 
diff --git a/drivers/iio/adc/meson_saradc.c b/drivers/iio/adc/meson_saradc.c
index 85b6826cc10c..18937a262af6 100644
--- a/drivers/iio/adc/meson_saradc.c
+++ b/drivers/iio/adc/meson_saradc.c
@@ -957,14 +957,18 @@ err_lock:
 	return ret;
 }
 
-static int meson_sar_adc_hw_disable(struct iio_dev *indio_dev)
+static void meson_sar_adc_hw_disable(struct iio_dev *indio_dev)
 {
 	struct meson_sar_adc_priv *priv = iio_priv(indio_dev);
 	int ret;
 
+	/*
+	 * If taking the lock fails we have to assume that BL30 is broken. The
+	 * best we can do then is to release the resources anyhow.
+	 */
 	ret = meson_sar_adc_lock(indio_dev);
 	if (ret)
-		return ret;
+		dev_err(indio_dev->dev.parent, "Failed to lock ADC (%pE)\n", ERR_PTR(ret));
 
 	clk_disable_unprepare(priv->adc_clk);
 
@@ -977,9 +981,8 @@ static int meson_sar_adc_hw_disable(struct iio_dev *indio_dev)
 
 	regulator_disable(priv->vref);
 
-	meson_sar_adc_unlock(indio_dev);
-
-	return 0;
+	if (!ret)
+		meson_sar_adc_unlock(indio_dev);
 }
 
 static irqreturn_t meson_sar_adc_irq(int irq, void *data)
@@ -1283,14 +1286,18 @@ static int meson_sar_adc_remove(struct platform_device *pdev)
 
 	iio_device_unregister(indio_dev);
 
-	return meson_sar_adc_hw_disable(indio_dev);
+	meson_sar_adc_hw_disable(indio_dev);
+
+	return 0;
 }
 
 static int meson_sar_adc_suspend(struct device *dev)
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 
-	return meson_sar_adc_hw_disable(indio_dev);
+	meson_sar_adc_hw_disable(indio_dev);
+
+	return 0;
 }
 
 static int meson_sar_adc_resume(struct device *dev)
diff --git a/drivers/iio/adc/palmas_gpadc.c b/drivers/iio/adc/palmas_gpadc.c
index 849a697a467e..c1c439215aeb 100644
--- a/drivers/iio/adc/palmas_gpadc.c
+++ b/drivers/iio/adc/palmas_gpadc.c
@@ -20,6 +20,7 @@
 #include <linux/completion.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/iio/events.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/machine.h>
 #include <linux/iio/driver.h>
@@ -76,6 +77,17 @@ static struct palmas_gpadc_info palmas_gpadc_info[] = {
 	PALMAS_ADC_INFO(IN15, 0, 0, 0, 0, INVALID, INVALID, true),
 };
 
+struct palmas_adc_event {
+	bool enabled;
+	int channel;
+	enum iio_event_direction direction;
+};
+
+struct palmas_gpadc_thresholds {
+	int high;
+	int low;
+};
+
 /*
  * struct palmas_gpadc - the palmas_gpadc structure
  * @ch0_current:	channel 0 current source setting
@@ -111,14 +123,33 @@ struct palmas_gpadc {
 	int				irq_auto_1;
 	struct palmas_gpadc_info	*adc_info;
 	struct completion		conv_completion;
-	struct palmas_adc_wakeup_property wakeup1_data;
-	struct palmas_adc_wakeup_property wakeup2_data;
-	bool				wakeup1_enable;
-	bool				wakeup2_enable;
+	struct palmas_adc_event		event0;
+	struct palmas_adc_event		event1;
+	struct palmas_gpadc_thresholds	thresholds[PALMAS_ADC_CH_MAX];
 	int				auto_conversion_period;
 	struct mutex			lock;
 };
 
+static struct palmas_adc_event *palmas_gpadc_get_event(struct palmas_gpadc *adc,
+						       int adc_chan,
+						       enum iio_event_direction dir)
+{
+	if (adc_chan == adc->event0.channel && dir == adc->event0.direction)
+		return &adc->event0;
+
+	if (adc_chan == adc->event1.channel && dir == adc->event1.direction)
+		return &adc->event1;
+
+	return NULL;
+}
+
+static bool palmas_gpadc_channel_is_freerunning(struct palmas_gpadc *adc,
+						int adc_chan)
+{
+	return palmas_gpadc_get_event(adc, adc_chan, IIO_EV_DIR_RISING) ||
+		palmas_gpadc_get_event(adc, adc_chan, IIO_EV_DIR_FALLING);
+}
+
 /*
  * GPADC lock issue in AUTO mode.
  * Impact: In AUTO mode, GPADC conversion can be locked after disabling AUTO
@@ -188,11 +219,24 @@ static irqreturn_t palmas_gpadc_irq(int irq, void *data)
 
 static irqreturn_t palmas_gpadc_irq_auto(int irq, void *data)
 {
-	struct palmas_gpadc *adc = data;
+	struct iio_dev *indio_dev = data;
+	struct palmas_gpadc *adc = iio_priv(indio_dev);
+	struct palmas_adc_event *ev;
 
 	dev_dbg(adc->dev, "Threshold interrupt %d occurs\n", irq);
 	palmas_disable_auto_conversion(adc);
 
+	ev = (irq == adc->irq_auto_0) ? &adc->event0 : &adc->event1;
+	if (ev->channel != -1) {
+		enum iio_event_direction dir;
+		u64 code;
+
+		dir = ev->direction;
+		code = IIO_UNMOD_EVENT_CODE(IIO_VOLTAGE, ev->channel,
+					    IIO_EV_TYPE_THRESH, dir);
+		iio_push_event(indio_dev, code, iio_get_time_ns(indio_dev));
+	}
+
 	return IRQ_HANDLED;
 }
 
@@ -280,6 +324,9 @@ static int palmas_gpadc_read_prepare(struct palmas_gpadc *adc, int adc_chan)
 {
 	int ret;
 
+	if (palmas_gpadc_channel_is_freerunning(adc, adc_chan))
+		return 0; /* ADC already running */
+
 	ret = palmas_gpadc_enable(adc, adc_chan, true);
 	if (ret < 0)
 		return ret;
@@ -339,28 +386,43 @@ static int palmas_gpadc_start_conversion(struct palmas_gpadc *adc, int adc_chan)
 	unsigned int val;
 	int ret;
 
-	init_completion(&adc->conv_completion);
-	ret = palmas_update_bits(adc->palmas, PALMAS_GPADC_BASE,
-				PALMAS_GPADC_SW_SELECT,
-				PALMAS_GPADC_SW_SELECT_SW_START_CONV0,
-				PALMAS_GPADC_SW_SELECT_SW_START_CONV0);
-	if (ret < 0) {
-		dev_err(adc->dev, "SELECT_SW_START write failed: %d\n", ret);
-		return ret;
-	}
+	if (palmas_gpadc_channel_is_freerunning(adc, adc_chan)) {
+		int event = (adc_chan == adc->event0.channel) ? 0 : 1;
+		unsigned int reg = (event == 0) ?
+			PALMAS_GPADC_AUTO_CONV0_LSB :
+			PALMAS_GPADC_AUTO_CONV1_LSB;
 
-	ret = wait_for_completion_timeout(&adc->conv_completion,
-				PALMAS_ADC_CONVERSION_TIMEOUT);
-	if (ret == 0) {
-		dev_err(adc->dev, "conversion not completed\n");
-		return -ETIMEDOUT;
-	}
+		ret = palmas_bulk_read(adc->palmas, PALMAS_GPADC_BASE,
+					reg, &val, 2);
+		if (ret < 0) {
+			dev_err(adc->dev, "AUTO_CONV%x_LSB read failed: %d\n",
+				event, ret);
+			return ret;
+		}
+	} else {
+		init_completion(&adc->conv_completion);
+		ret = palmas_update_bits(adc->palmas, PALMAS_GPADC_BASE,
+					PALMAS_GPADC_SW_SELECT,
+					PALMAS_GPADC_SW_SELECT_SW_START_CONV0,
+					PALMAS_GPADC_SW_SELECT_SW_START_CONV0);
+		if (ret < 0) {
+			dev_err(adc->dev, "SELECT_SW_START write failed: %d\n", ret);
+			return ret;
+		}
 
-	ret = palmas_bulk_read(adc->palmas, PALMAS_GPADC_BASE,
-				PALMAS_GPADC_SW_CONV0_LSB, &val, 2);
-	if (ret < 0) {
-		dev_err(adc->dev, "SW_CONV0_LSB read failed: %d\n", ret);
-		return ret;
+		ret = wait_for_completion_timeout(&adc->conv_completion,
+					PALMAS_ADC_CONVERSION_TIMEOUT);
+		if (ret == 0) {
+			dev_err(adc->dev, "conversion not completed\n");
+			return -ETIMEDOUT;
+		}
+
+		ret = palmas_bulk_read(adc->palmas, PALMAS_GPADC_BASE,
+					PALMAS_GPADC_SW_CONV0_LSB, &val, 2);
+		if (ret < 0) {
+			dev_err(adc->dev, "SW_CONV0_LSB read failed: %d\n", ret);
+			return ret;
+		}
 	}
 
 	ret = val & 0xFFF;
@@ -386,6 +448,98 @@ static int palmas_gpadc_get_calibrated_code(struct palmas_gpadc *adc,
 	return val;
 }
 
+/*
+ * The high and low threshold values are calculated based on the advice given
+ * in TI Application Report SLIA087A, "Guide to Using the GPADC in PS65903x,
+ * TPS65917-Q1, TPS65919-Q1, and TPS65916 Devices". This document recommend
+ * taking ADC tolerances into account and is based on the device integral non-
+ * linearity (INL), offset error and gain error:
+ *
+ *   raw high threshold = (ideal threshold + INL) * gain error + offset error
+ *
+ * The gain error include both gain error, as specified in the datasheet, and
+ * the gain error drift. These paramenters vary depending on device and whether
+ * the the channel is calibrated (trimmed) or not.
+ */
+static int palmas_gpadc_threshold_with_tolerance(int val, const int INL,
+						 const int gain_error,
+						 const int offset_error)
+{
+	val = ((val + INL) * (1000 + gain_error)) / 1000 + offset_error;
+
+	return clamp(val, 0, 0xFFF);
+}
+
+/*
+ * The values below are taken from the datasheet of TWL6035, TWL6037.
+ * todo: get max INL, gain error, and offset error from OF.
+ */
+static int palmas_gpadc_get_high_threshold_raw(struct palmas_gpadc *adc,
+					       struct palmas_adc_event *ev)
+{
+	const int adc_chan = ev->channel;
+	int val = adc->thresholds[adc_chan].high;
+	/* integral nonlinearity, measured in LSB */
+	const int max_INL = 2;
+	/* measured in LSB */
+	int max_offset_error;
+	/* 0.2% when calibrated */
+	int max_gain_error = 2;
+
+	val = (val * 1000) / adc->adc_info[adc_chan].gain;
+
+	if (adc->adc_info[adc_chan].is_uncalibrated) {
+		/* 2% worse */
+		max_gain_error += 20;
+		max_offset_error = 36;
+	} else {
+		val = (val * adc->adc_info[adc_chan].gain_error +
+		       adc->adc_info[adc_chan].offset) /
+			1000;
+		max_offset_error = 2;
+	}
+
+	return palmas_gpadc_threshold_with_tolerance(val,
+						     max_INL,
+						     max_gain_error,
+						     max_offset_error);
+}
+
+/*
+ * The values below are taken from the datasheet of TWL6035, TWL6037.
+ * todo: get min INL, gain error, and offset error from OF.
+ */
+static int palmas_gpadc_get_low_threshold_raw(struct palmas_gpadc *adc,
+					      struct palmas_adc_event *ev)
+{
+	const int adc_chan = ev->channel;
+	int val = adc->thresholds[adc_chan].low;
+	/* integral nonlinearity, measured in LSB */
+	const int min_INL = -2;
+	/* measured in LSB */
+	int min_offset_error;
+	/* -0.6% when calibrated */
+	int min_gain_error = -6;
+
+	val = (val * 1000) / adc->adc_info[adc_chan].gain;
+
+        if (adc->adc_info[adc_chan].is_uncalibrated) {
+		/* 2% worse */
+		min_gain_error -= 20;
+		min_offset_error = -36;
+        } else {
+		val = (val * adc->adc_info[adc_chan].gain_error -
+		       adc->adc_info[adc_chan].offset) /
+			1000;
+		min_offset_error = -2;
+        }
+
+	return palmas_gpadc_threshold_with_tolerance(val,
+						     min_INL,
+						     min_gain_error,
+						     min_offset_error);
+}
+
 static int palmas_gpadc_read_raw(struct iio_dev *indio_dev,
 	struct iio_chan_spec const *chan, int *val, int *val2, long mask)
 {
@@ -432,8 +586,217 @@ out:
 	return ret;
 }
 
+static int palmas_gpadc_read_event_config(struct iio_dev *indio_dev,
+					  const struct iio_chan_spec *chan,
+					  enum iio_event_type type,
+					  enum iio_event_direction dir)
+{
+	struct palmas_gpadc *adc = iio_priv(indio_dev);
+	int adc_chan = chan->channel;
+	int ret = 0;
+
+	if (adc_chan > PALMAS_ADC_CH_MAX || type != IIO_EV_TYPE_THRESH)
+		return -EINVAL;
+
+	mutex_lock(&adc->lock);
+
+	if (palmas_gpadc_get_event(adc, adc_chan, dir))
+		ret = 1;
+
+	mutex_unlock(&adc->lock);
+
+	return ret;
+}
+
+static int palmas_adc_configure_events(struct palmas_gpadc *adc);
+static int palmas_adc_reset_events(struct palmas_gpadc *adc);
+
+static int palmas_gpadc_reconfigure_event_channels(struct palmas_gpadc *adc)
+{
+	return (adc->event0.enabled || adc->event1.enabled) ?
+		palmas_adc_configure_events(adc) :
+		palmas_adc_reset_events(adc);
+}
+
+static int palmas_gpadc_enable_event_config(struct palmas_gpadc *adc,
+					    const struct iio_chan_spec *chan,
+					    enum iio_event_direction dir)
+{
+	struct palmas_adc_event *ev;
+	int adc_chan = chan->channel;
+
+	if (palmas_gpadc_get_event(adc, adc_chan, dir))
+		/* already enabled */
+		return 0;
+
+	if (adc->event0.channel == -1) {
+		ev = &adc->event0;
+	} else if (adc->event1.channel == -1) {
+		/* event0 has to be the lowest channel */
+		if (adc_chan < adc->event0.channel) {
+			adc->event1 = adc->event0;
+			ev = &adc->event0;
+		} else {
+			ev = &adc->event1;
+		}
+	} else { /* both AUTO channels already in use */
+		dev_warn(adc->dev, "event0 - %d, event1 - %d\n",
+			 adc->event0.channel, adc->event1.channel);
+		return -EBUSY;
+	}
+
+	ev->enabled = true;
+	ev->channel = adc_chan;
+	ev->direction = dir;
+
+	return palmas_gpadc_reconfigure_event_channels(adc);
+}
+
+static int palmas_gpadc_disable_event_config(struct palmas_gpadc *adc,
+					     const struct iio_chan_spec *chan,
+					     enum iio_event_direction dir)
+{
+	int adc_chan = chan->channel;
+	struct palmas_adc_event *ev = palmas_gpadc_get_event(adc, adc_chan, dir);
+
+	if (!ev)
+		return 0;
+
+	if (ev == &adc->event0) {
+		adc->event0 = adc->event1;
+		ev = &adc->event1;
+	}
+
+	ev->enabled = false;
+	ev->channel = -1;
+	ev->direction = IIO_EV_DIR_NONE;
+
+	return palmas_gpadc_reconfigure_event_channels(adc);
+}
+
+static int palmas_gpadc_write_event_config(struct iio_dev *indio_dev,
+					   const struct iio_chan_spec *chan,
+					   enum iio_event_type type,
+					   enum iio_event_direction dir,
+					   int state)
+{
+	struct palmas_gpadc *adc = iio_priv(indio_dev);
+	int adc_chan = chan->channel;
+	int ret;
+
+	if (adc_chan > PALMAS_ADC_CH_MAX || type != IIO_EV_TYPE_THRESH)
+		return -EINVAL;
+
+	mutex_lock(&adc->lock);
+
+	if (state)
+		ret = palmas_gpadc_enable_event_config(adc, chan, dir);
+	else
+		ret = palmas_gpadc_disable_event_config(adc, chan, dir);
+
+	mutex_unlock(&adc->lock);
+
+	return ret;
+}
+
+static int palmas_gpadc_read_event_value(struct iio_dev *indio_dev,
+					 const struct iio_chan_spec *chan,
+					 enum iio_event_type type,
+					 enum iio_event_direction dir,
+					 enum iio_event_info info,
+					 int *val, int *val2)
+{
+	struct palmas_gpadc *adc = iio_priv(indio_dev);
+	int adc_chan = chan->channel;
+	int ret;
+
+	if (adc_chan > PALMAS_ADC_CH_MAX || type != IIO_EV_TYPE_THRESH)
+		return -EINVAL;
+
+	mutex_lock(&adc->lock);
+
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		*val = (dir == IIO_EV_DIR_RISING) ?
+			adc->thresholds[adc_chan].high :
+			adc->thresholds[adc_chan].low;
+		ret = IIO_VAL_INT;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	mutex_unlock(&adc->lock);
+
+	return ret;
+}
+
+static int palmas_gpadc_write_event_value(struct iio_dev *indio_dev,
+					  const struct iio_chan_spec *chan,
+					  enum iio_event_type type,
+					  enum iio_event_direction dir,
+					  enum iio_event_info info,
+					  int val, int val2)
+{
+	struct palmas_gpadc *adc = iio_priv(indio_dev);
+	int adc_chan = chan->channel;
+	int old;
+	int ret;
+
+	if (adc_chan > PALMAS_ADC_CH_MAX || type != IIO_EV_TYPE_THRESH)
+		return -EINVAL;
+
+	mutex_lock(&adc->lock);
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		if (val < 0 || val > 0xFFF) {
+			ret = -EINVAL;
+			goto out_unlock;
+		}
+		if (dir == IIO_EV_DIR_RISING) {
+			old = adc->thresholds[adc_chan].high;
+			adc->thresholds[adc_chan].high = val;
+		} else {
+			old = adc->thresholds[adc_chan].low;
+			adc->thresholds[adc_chan].low = val;
+		}
+		ret = 0;
+		break;
+	default:
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
+	if (val != old && palmas_gpadc_get_event(adc, adc_chan, dir))
+		ret = palmas_gpadc_reconfigure_event_channels(adc);
+
+out_unlock:
+	mutex_unlock(&adc->lock);
+
+	return ret;
+}
+
 static const struct iio_info palmas_gpadc_iio_info = {
 	.read_raw = palmas_gpadc_read_raw,
+	.read_event_config = palmas_gpadc_read_event_config,
+	.write_event_config = palmas_gpadc_write_event_config,
+	.read_event_value = palmas_gpadc_read_event_value,
+	.write_event_value = palmas_gpadc_write_event_value,
+};
+
+static const struct iio_event_spec palmas_gpadc_events[] = {
+	{
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_RISING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE) |
+				BIT(IIO_EV_INFO_ENABLE),
+	}, {
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_FALLING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE) |
+				BIT(IIO_EV_INFO_ENABLE),
+	},
 };
 
 #define PALMAS_ADC_CHAN_IIO(chan, _type, chan_info)	\
@@ -444,6 +807,8 @@ static const struct iio_info palmas_gpadc_iio_info = {
 			BIT(chan_info),			\
 	.indexed = 1,					\
 	.channel = PALMAS_ADC_CH_##chan,		\
+	.event_spec = palmas_gpadc_events,		\
+	.num_event_specs = ARRAY_SIZE(palmas_gpadc_events)	\
 }
 
 static const struct iio_chan_spec palmas_gpadc_iio_channel[] = {
@@ -493,6 +858,13 @@ static int palmas_gpadc_get_adc_dt_data(struct platform_device *pdev,
 	return 0;
 }
 
+static void palmas_gpadc_reset(void *data)
+{
+	struct palmas_gpadc *adc = data;
+	if (adc->event0.enabled || adc->event1.enabled)
+		palmas_adc_reset_events(adc);
+}
+
 static int palmas_gpadc_probe(struct platform_device *pdev)
 {
 	struct palmas_gpadc *adc;
@@ -532,53 +904,49 @@ static int palmas_gpadc_probe(struct platform_device *pdev)
 
 	adc->auto_conversion_period = gpadc_pdata->auto_conversion_period_ms;
 	adc->irq = palmas_irq_get_virq(adc->palmas, PALMAS_GPADC_EOC_SW_IRQ);
-	if (adc->irq < 0) {
-		dev_err(adc->dev,
-			"get virq failed: %d\n", adc->irq);
-		ret = adc->irq;
-		goto out;
-	}
-	ret = request_threaded_irq(adc->irq, NULL,
-		palmas_gpadc_irq,
-		IRQF_ONESHOT, dev_name(adc->dev),
-		adc);
-	if (ret < 0) {
-		dev_err(adc->dev,
-			"request irq %d failed: %d\n", adc->irq, ret);
-		goto out;
-	}
+	if (adc->irq < 0)
+		return dev_err_probe(adc->dev, adc->irq, "get virq failed\n");
 
-	if (gpadc_pdata->adc_wakeup1_data) {
-		memcpy(&adc->wakeup1_data, gpadc_pdata->adc_wakeup1_data,
-			sizeof(adc->wakeup1_data));
-		adc->wakeup1_enable = true;
-		adc->irq_auto_0 =  platform_get_irq(pdev, 1);
-		ret = request_threaded_irq(adc->irq_auto_0, NULL,
-				palmas_gpadc_irq_auto,
-				IRQF_ONESHOT,
-				"palmas-adc-auto-0", adc);
-		if (ret < 0) {
-			dev_err(adc->dev, "request auto0 irq %d failed: %d\n",
-				adc->irq_auto_0, ret);
-			goto out_irq_free;
-		}
-	}
+	ret = devm_request_threaded_irq(&pdev->dev, adc->irq, NULL,
+					palmas_gpadc_irq,
+					IRQF_ONESHOT, dev_name(adc->dev),
+					adc);
+	if (ret < 0)
+		return dev_err_probe(adc->dev, ret,
+				     "request irq %d failed\n", adc->irq);
 
-	if (gpadc_pdata->adc_wakeup2_data) {
-		memcpy(&adc->wakeup2_data, gpadc_pdata->adc_wakeup2_data,
-				sizeof(adc->wakeup2_data));
-		adc->wakeup2_enable = true;
-		adc->irq_auto_1 =  platform_get_irq(pdev, 2);
-		ret = request_threaded_irq(adc->irq_auto_1, NULL,
-				palmas_gpadc_irq_auto,
-				IRQF_ONESHOT,
-				"palmas-adc-auto-1", adc);
-		if (ret < 0) {
-			dev_err(adc->dev, "request auto1 irq %d failed: %d\n",
-				adc->irq_auto_1, ret);
-			goto out_irq_auto0_free;
-		}
-	}
+	adc->irq_auto_0 = platform_get_irq(pdev, 1);
+	if (adc->irq_auto_0 < 0)
+		return dev_err_probe(adc->dev, adc->irq_auto_0,
+				     "get auto0 irq failed\n");
+
+	ret = devm_request_threaded_irq(&pdev->dev, adc->irq_auto_0, NULL,
+					palmas_gpadc_irq_auto, IRQF_ONESHOT,
+					"palmas-adc-auto-0", indio_dev);
+	if (ret < 0)
+		return dev_err_probe(adc->dev, ret,
+				     "request auto0 irq %d failed\n",
+				     adc->irq_auto_0);
+
+	adc->irq_auto_1 = platform_get_irq(pdev, 2);
+	if (adc->irq_auto_1 < 0)
+		return dev_err_probe(adc->dev, adc->irq_auto_1,
+				     "get auto1 irq failed\n");
+
+	ret = devm_request_threaded_irq(&pdev->dev, adc->irq_auto_1, NULL,
+					palmas_gpadc_irq_auto, IRQF_ONESHOT,
+					"palmas-adc-auto-1", indio_dev);
+	if (ret < 0)
+		return dev_err_probe(adc->dev, ret,
+				     "request auto1 irq %d failed\n",
+				     adc->irq_auto_1);
+
+	adc->event0.enabled = false;
+	adc->event0.channel = -1;
+	adc->event0.direction = IIO_EV_DIR_NONE;
+	adc->event1.enabled = false;
+	adc->event1.channel = -1;
+	adc->event1.direction = IIO_EV_DIR_NONE;
 
 	/* set the current source 0 (value 0/5/15/20 uA => 0..3) */
 	if (gpadc_pdata->ch0_current <= 1)
@@ -608,11 +976,10 @@ static int palmas_gpadc_probe(struct platform_device *pdev)
 	indio_dev->channels = palmas_gpadc_iio_channel;
 	indio_dev->num_channels = ARRAY_SIZE(palmas_gpadc_iio_channel);
 
-	ret = iio_device_register(indio_dev);
-	if (ret < 0) {
-		dev_err(adc->dev, "iio_device_register() failed: %d\n", ret);
-		goto out_irq_auto1_free;
-	}
+	ret = devm_iio_device_register(&pdev->dev, indio_dev);
+	if (ret < 0)
+		return dev_err_probe(adc->dev, ret,
+				     "iio_device_register() failed\n");
 
 	device_set_wakeup_capable(&pdev->dev, 1);
 	for (i = 0; i < PALMAS_ADC_CH_MAX; i++) {
@@ -620,41 +987,14 @@ static int palmas_gpadc_probe(struct platform_device *pdev)
 			palmas_gpadc_calibrate(adc, i);
 	}
 
-	if (adc->wakeup1_enable || adc->wakeup2_enable)
-		device_wakeup_enable(&pdev->dev);
-
-	return 0;
-
-out_irq_auto1_free:
-	if (gpadc_pdata->adc_wakeup2_data)
-		free_irq(adc->irq_auto_1, adc);
-out_irq_auto0_free:
-	if (gpadc_pdata->adc_wakeup1_data)
-		free_irq(adc->irq_auto_0, adc);
-out_irq_free:
-	free_irq(adc->irq, adc);
-out:
-	return ret;
-}
-
-static int palmas_gpadc_remove(struct platform_device *pdev)
-{
-	struct iio_dev *indio_dev = dev_get_drvdata(&pdev->dev);
-	struct palmas_gpadc *adc = iio_priv(indio_dev);
-
-	if (adc->wakeup1_enable || adc->wakeup2_enable)
-		device_wakeup_disable(&pdev->dev);
-	iio_device_unregister(indio_dev);
-	free_irq(adc->irq, adc);
-	if (adc->wakeup1_enable)
-		free_irq(adc->irq_auto_0, adc);
-	if (adc->wakeup2_enable)
-		free_irq(adc->irq_auto_1, adc);
+	ret = devm_add_action(&pdev->dev, palmas_gpadc_reset, adc);
+	if (ret)
+		return ret;
 
 	return 0;
 }
 
-static int palmas_adc_wakeup_configure(struct palmas_gpadc *adc)
+static int palmas_adc_configure_events(struct palmas_gpadc *adc)
 {
 	int adc_period, conv;
 	int i;
@@ -680,17 +1020,23 @@ static int palmas_adc_wakeup_configure(struct palmas_gpadc *adc)
 	}
 
 	conv = 0;
-	if (adc->wakeup1_enable) {
+	if (adc->event0.enabled) {
+		struct palmas_adc_event *ev = &adc->event0;
 		int polarity;
 
-		ch0 = adc->wakeup1_data.adc_channel_number;
+		ch0 = ev->channel;
 		conv |= PALMAS_GPADC_AUTO_CTRL_AUTO_CONV0_EN;
-		if (adc->wakeup1_data.adc_high_threshold > 0) {
-			thres = adc->wakeup1_data.adc_high_threshold;
+		switch (ev->direction) {
+		case IIO_EV_DIR_RISING:
+			thres = palmas_gpadc_get_high_threshold_raw(adc, ev);
 			polarity = 0;
-		} else {
-			thres = adc->wakeup1_data.adc_low_threshold;
+			break;
+		case IIO_EV_DIR_FALLING:
+			thres = palmas_gpadc_get_low_threshold_raw(adc, ev);
 			polarity = PALMAS_GPADC_THRES_CONV0_MSB_THRES_CONV0_POL;
+			break;
+		default:
+			return -EINVAL;
 		}
 
 		ret = palmas_write(adc->palmas, PALMAS_GPADC_BASE,
@@ -711,17 +1057,23 @@ static int palmas_adc_wakeup_configure(struct palmas_gpadc *adc)
 		}
 	}
 
-	if (adc->wakeup2_enable) {
+	if (adc->event1.enabled) {
+		struct palmas_adc_event *ev = &adc->event1;
 		int polarity;
 
-		ch1 = adc->wakeup2_data.adc_channel_number;
+		ch1 = ev->channel;
 		conv |= PALMAS_GPADC_AUTO_CTRL_AUTO_CONV1_EN;
-		if (adc->wakeup2_data.adc_high_threshold > 0) {
-			thres = adc->wakeup2_data.adc_high_threshold;
+		switch (ev->direction) {
+		case IIO_EV_DIR_RISING:
+			thres = palmas_gpadc_get_high_threshold_raw(adc, ev);
 			polarity = 0;
-		} else {
-			thres = adc->wakeup2_data.adc_low_threshold;
+			break;
+		case IIO_EV_DIR_FALLING:
+			thres = palmas_gpadc_get_low_threshold_raw(adc, ev);
 			polarity = PALMAS_GPADC_THRES_CONV1_MSB_THRES_CONV1_POL;
+			break;
+		default:
+			return -EINVAL;
 		}
 
 		ret = palmas_write(adc->palmas, PALMAS_GPADC_BASE,
@@ -759,7 +1111,7 @@ static int palmas_adc_wakeup_configure(struct palmas_gpadc *adc)
 	return ret;
 }
 
-static int palmas_adc_wakeup_reset(struct palmas_gpadc *adc)
+static int palmas_adc_reset_events(struct palmas_gpadc *adc)
 {
 	int ret;
 
@@ -781,20 +1133,14 @@ static int palmas_gpadc_suspend(struct device *dev)
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct palmas_gpadc *adc = iio_priv(indio_dev);
-	int wakeup = adc->wakeup1_enable || adc->wakeup2_enable;
-	int ret;
 
-	if (!device_may_wakeup(dev) || !wakeup)
+	if (!device_may_wakeup(dev))
 		return 0;
 
-	ret = palmas_adc_wakeup_configure(adc);
-	if (ret < 0)
-		return ret;
-
-	if (adc->wakeup1_enable)
+	if (adc->event0.enabled)
 		enable_irq_wake(adc->irq_auto_0);
 
-	if (adc->wakeup2_enable)
+	if (adc->event1.enabled)
 		enable_irq_wake(adc->irq_auto_1);
 
 	return 0;
@@ -804,20 +1150,14 @@ static int palmas_gpadc_resume(struct device *dev)
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct palmas_gpadc *adc = iio_priv(indio_dev);
-	int wakeup = adc->wakeup1_enable || adc->wakeup2_enable;
-	int ret;
 
-	if (!device_may_wakeup(dev) || !wakeup)
+	if (!device_may_wakeup(dev))
 		return 0;
 
-	ret = palmas_adc_wakeup_reset(adc);
-	if (ret < 0)
-		return ret;
-
-	if (adc->wakeup1_enable)
+	if (adc->event0.enabled)
 		disable_irq_wake(adc->irq_auto_0);
 
-	if (adc->wakeup2_enable)
+	if (adc->event1.enabled)
 		disable_irq_wake(adc->irq_auto_1);
 
 	return 0;
@@ -834,7 +1174,6 @@ MODULE_DEVICE_TABLE(of, of_palmas_gpadc_match_tbl);
 
 static struct platform_driver palmas_gpadc_driver = {
 	.probe = palmas_gpadc_probe,
-	.remove = palmas_gpadc_remove,
 	.driver = {
 		.name = MOD_NAME,
 		.pm = pm_sleep_ptr(&palmas_pm_ops),
diff --git a/drivers/iio/adc/qcom-pm8xxx-xoadc.c b/drivers/iio/adc/qcom-pm8xxx-xoadc.c
index eb424496ee1d..64a3aeb6261c 100644
--- a/drivers/iio/adc/qcom-pm8xxx-xoadc.c
+++ b/drivers/iio/adc/qcom-pm8xxx-xoadc.c
@@ -758,7 +758,7 @@ static int pm8xxx_xoadc_parse_channel(struct device *dev,
 	/* Find the right channel setting */
 	chid = 0;
 	hwchan = &hw_channels[0];
-	while (hwchan && hwchan->datasheet_name) {
+	while (hwchan->datasheet_name) {
 		if (hwchan->pre_scale_mux == pre_scale_mux &&
 		    hwchan->amux_channel == amux_channel)
 			break;
diff --git a/drivers/iio/adc/rcar-gyroadc.c b/drivers/iio/adc/rcar-gyroadc.c
index 27d9e147b4b7..b8972f673c9d 100644
--- a/drivers/iio/adc/rcar-gyroadc.c
+++ b/drivers/iio/adc/rcar-gyroadc.c
@@ -283,7 +283,7 @@ static const struct of_device_id rcar_gyroadc_match[] = {
 
 MODULE_DEVICE_TABLE(of, rcar_gyroadc_match);
 
-static const struct of_device_id rcar_gyroadc_child_match[] = {
+static const struct of_device_id rcar_gyroadc_child_match[] __maybe_unused = {
 	/* Mode 1 ADCs */
 	{
 		.compatible	= "fujitsu,mb88101a",
diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c
index 45d4e79f8e55..1aadb2ad2cab 100644
--- a/drivers/iio/adc/stm32-adc.c
+++ b/drivers/iio/adc/stm32-adc.c
@@ -2588,7 +2588,7 @@ static const struct stm32_adc_cfg stm32f4_adc_cfg = {
 	.irq_clear = stm32f4_adc_irq_clear,
 };
 
-const unsigned int stm32_adc_min_ts_h7[] = { 0, 0, 0, 4300, 9000 };
+static const unsigned int stm32_adc_min_ts_h7[] = { 0, 0, 0, 4300, 9000 };
 static_assert(ARRAY_SIZE(stm32_adc_min_ts_h7) == STM32_ADC_INT_CH_NB);
 
 static const struct stm32_adc_cfg stm32h7_adc_cfg = {
@@ -2607,7 +2607,7 @@ static const struct stm32_adc_cfg stm32h7_adc_cfg = {
 	.ts_int_ch = stm32_adc_min_ts_h7,
 };
 
-const unsigned int stm32_adc_min_ts_mp1[] = { 100, 100, 100, 4300, 9800 };
+static const unsigned int stm32_adc_min_ts_mp1[] = { 100, 100, 100, 4300, 9800 };
 static_assert(ARRAY_SIZE(stm32_adc_min_ts_mp1) == STM32_ADC_INT_CH_NB);
 
 static const struct stm32_adc_cfg stm32mp1_adc_cfg = {
@@ -2627,7 +2627,7 @@ static const struct stm32_adc_cfg stm32mp1_adc_cfg = {
 	.ts_int_ch = stm32_adc_min_ts_mp1,
 };
 
-const unsigned int stm32_adc_min_ts_mp13[] = { 100, 0, 0, 4300, 9800 };
+static const unsigned int stm32_adc_min_ts_mp13[] = { 100, 0, 0, 4300, 9800 };
 static_assert(ARRAY_SIZE(stm32_adc_min_ts_mp13) == STM32_ADC_INT_CH_NB);
 
 static const struct stm32_adc_cfg stm32mp13_adc_cfg = {
diff --git a/drivers/iio/adc/sun4i-gpadc-iio.c b/drivers/iio/adc/sun4i-gpadc-iio.c
index a6ade70dedf8..a5322550c422 100644
--- a/drivers/iio/adc/sun4i-gpadc-iio.c
+++ b/drivers/iio/adc/sun4i-gpadc-iio.c
@@ -414,7 +414,7 @@ static int sun4i_gpadc_runtime_resume(struct device *dev)
 
 static int sun4i_gpadc_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct sun4i_gpadc_iio *info = tz->devdata;
+	struct sun4i_gpadc_iio *info = thermal_zone_device_priv(tz);
 	int val, scale, offset;
 
 	if (sun4i_gpadc_temp_read(info->indio_dev, &val))
diff --git a/drivers/iio/adc/ti-ads1100.c b/drivers/iio/adc/ti-ads1100.c
new file mode 100644
index 000000000000..6b5aebb82455
--- /dev/null
+++ b/drivers/iio/adc/ti-ads1100.c
@@ -0,0 +1,445 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ADS1100 - Texas Instruments Analog-to-Digital Converter
+ *
+ * Copyright (c) 2023, Topic Embedded Products
+ *
+ * Datasheet: https://www.ti.com/lit/gpn/ads1100
+ * IIO driver for ADS1100 and ADS1000 ADC 16-bit I2C
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/i2c.h>
+#include <linux/mutex.h>
+#include <linux/property.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
+#include <linux/units.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/types.h>
+
+/* The ADS1100 has a single byte config register */
+
+/* Conversion in progress bit */
+#define ADS1100_CFG_ST_BSY	BIT(7)
+/* Single conversion bit */
+#define ADS1100_CFG_SC		BIT(4)
+/* Data rate */
+#define ADS1100_DR_MASK		GENMASK(3, 2)
+/* Gain */
+#define ADS1100_PGA_MASK	GENMASK(1, 0)
+
+#define ADS1100_CONTINUOUS	0
+#define	ADS1100_SINGLESHOT	ADS1100_CFG_SC
+
+#define ADS1100_SLEEP_DELAY_MS	2000
+
+static const int ads1100_data_rate[] = { 128, 32, 16, 8 };
+static const int ads1100_data_rate_bits[] = { 12, 14, 15, 16 };
+
+struct ads1100_data {
+	struct i2c_client *client;
+	struct regulator *reg_vdd;
+	struct mutex lock;
+	int scale_avail[2 * 4]; /* 4 gain settings */
+	u8 config;
+	bool supports_data_rate; /* Only the ADS1100 can select the rate */
+};
+
+static const struct iio_chan_spec ads1100_channel = {
+	.type = IIO_VOLTAGE,
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+	.info_mask_shared_by_all =
+	    BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_SAMP_FREQ),
+	.info_mask_shared_by_all_available =
+	    BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_SAMP_FREQ),
+	.scan_type = {
+		      .sign = 's',
+		      .realbits = 16,
+		      .storagebits = 16,
+		      .endianness = IIO_CPU,
+		       },
+	.datasheet_name = "AIN",
+};
+
+static int ads1100_set_config_bits(struct ads1100_data *data, u8 mask, u8 value)
+{
+	int ret;
+	u8 config = (data->config & ~mask) | (value & mask);
+
+	if (data->config == config)
+		return 0;	/* Already done */
+
+	ret = i2c_master_send(data->client, &config, 1);
+	if (ret < 0)
+		return ret;
+
+	data->config = config;
+
+	return 0;
+};
+
+static int ads1100_data_bits(struct ads1100_data *data)
+{
+	return ads1100_data_rate_bits[FIELD_GET(ADS1100_DR_MASK, data->config)];
+}
+
+static int ads1100_get_adc_result(struct ads1100_data *data, int chan, int *val)
+{
+	int ret;
+	__be16 buffer;
+	s16 value;
+
+	if (chan != 0)
+		return -EINVAL;
+
+	ret = pm_runtime_resume_and_get(&data->client->dev);
+	if (ret < 0)
+		return ret;
+
+	ret = i2c_master_recv(data->client, (char *)&buffer, sizeof(buffer));
+
+	pm_runtime_mark_last_busy(&data->client->dev);
+	pm_runtime_put_autosuspend(&data->client->dev);
+
+	if (ret < 0) {
+		dev_err(&data->client->dev, "I2C read fail: %d\n", ret);
+		return ret;
+	}
+
+	/* Value is always 16-bit 2's complement */
+	value = be16_to_cpu(buffer);
+
+	/* Shift result to compensate for bit resolution vs. sample rate */
+	value <<= 16 - ads1100_data_bits(data);
+
+	*val = sign_extend32(value, 15);
+
+	return 0;
+}
+
+static int ads1100_set_scale(struct ads1100_data *data, int val, int val2)
+{
+	int microvolts;
+	int gain;
+
+	/* With Vdd between 2.7 and 5V, the scale is always below 1 */
+	if (val)
+		return -EINVAL;
+
+	if (!val2)
+		return -EINVAL;
+
+	microvolts = regulator_get_voltage(data->reg_vdd);
+	/*
+	 * val2 is in 'micro' units, n = val2 / 1000000
+	 * result must be millivolts, d = microvolts / 1000
+	 * the full-scale value is d/n, corresponds to 2^15,
+	 * hence the gain = (d / n) >> 15, factoring out the 1000 and moving the
+	 * bitshift so everything fits in 32-bits yields this formula.
+	 */
+	gain = DIV_ROUND_CLOSEST(microvolts, BIT(15)) * MILLI / val2;
+	if (gain < BIT(0) || gain > BIT(3))
+		return -EINVAL;
+
+	ads1100_set_config_bits(data, ADS1100_PGA_MASK, ffs(gain) - 1);
+
+	return 0;
+}
+
+static int ads1100_set_data_rate(struct ads1100_data *data, int chan, int rate)
+{
+	unsigned int i;
+	unsigned int size;
+
+	size = data->supports_data_rate ? ARRAY_SIZE(ads1100_data_rate) : 1;
+	for (i = 0; i < size; i++) {
+		if (ads1100_data_rate[i] == rate)
+			return ads1100_set_config_bits(data, ADS1100_DR_MASK,
+						       FIELD_PREP(ADS1100_DR_MASK, i));
+	}
+
+	return -EINVAL;
+}
+
+static int ads1100_get_vdd_millivolts(struct ads1100_data *data)
+{
+	return regulator_get_voltage(data->reg_vdd) / (MICRO / MILLI);
+}
+
+static void ads1100_calc_scale_avail(struct ads1100_data *data)
+{
+	int millivolts = ads1100_get_vdd_millivolts(data);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(data->scale_avail) / 2; i++) {
+		data->scale_avail[i * 2 + 0] = millivolts;
+		data->scale_avail[i * 2 + 1] = 15 + i;
+	}
+}
+
+static int ads1100_read_avail(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      const int **vals, int *type, int *length,
+			      long mask)
+{
+	struct ads1100_data *data = iio_priv(indio_dev);
+
+	if (chan->type != IIO_VOLTAGE)
+		return -EINVAL;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*type = IIO_VAL_INT;
+		*vals = ads1100_data_rate;
+		if (data->supports_data_rate)
+			*length = ARRAY_SIZE(ads1100_data_rate);
+		else
+			*length = 1;
+		return IIO_AVAIL_LIST;
+	case IIO_CHAN_INFO_SCALE:
+		*type = IIO_VAL_FRACTIONAL_LOG2;
+		*vals = data->scale_avail;
+		*length = ARRAY_SIZE(data->scale_avail);
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ads1100_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan, int *val,
+			    int *val2, long mask)
+{
+	int ret;
+	struct ads1100_data *data = iio_priv(indio_dev);
+
+	mutex_lock(&data->lock);
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			break;
+
+		ret = ads1100_get_adc_result(data, chan->address, val);
+		if (ret >= 0)
+			ret = IIO_VAL_INT;
+		iio_device_release_direct_mode(indio_dev);
+		break;
+	case IIO_CHAN_INFO_SCALE:
+		/* full-scale is the supply voltage in millivolts */
+		*val = ads1100_get_vdd_millivolts(data);
+		*val2 = 15 + FIELD_GET(ADS1100_PGA_MASK, data->config);
+		ret = IIO_VAL_FRACTIONAL_LOG2;
+		break;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = ads1100_data_rate[FIELD_GET(ADS1100_DR_MASK,
+						   data->config)];
+		ret = IIO_VAL_INT;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
+static int ads1100_write_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan, int val,
+			     int val2, long mask)
+{
+	struct ads1100_data *data = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&data->lock);
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		ret = ads1100_set_scale(data, val, val2);
+		break;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		ret = ads1100_set_data_rate(data, chan->address, val);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
+static const struct iio_info ads1100_info = {
+	.read_avail = ads1100_read_avail,
+	.read_raw = ads1100_read_raw,
+	.write_raw = ads1100_write_raw,
+};
+
+static int ads1100_setup(struct ads1100_data *data)
+{
+	int ret;
+	u8 buffer[3];
+
+	/* Setup continuous sampling mode at 8sps */
+	buffer[0] = ADS1100_DR_MASK | ADS1100_CONTINUOUS;
+	ret = i2c_master_send(data->client, buffer, 1);
+	if (ret < 0)
+		return ret;
+
+	ret = i2c_master_recv(data->client, buffer, sizeof(buffer));
+	if (ret < 0)
+		return ret;
+
+	/* Config register returned in third byte, strip away the busy status */
+	data->config = buffer[2] & ~ADS1100_CFG_ST_BSY;
+
+	/* Detect the sample rate capability by checking the DR bits */
+	data->supports_data_rate = FIELD_GET(ADS1100_DR_MASK, buffer[2]) != 0;
+
+	return 0;
+}
+
+static void ads1100_reg_disable(void *reg)
+{
+	regulator_disable(reg);
+}
+
+static void ads1100_disable_continuous(void *data)
+{
+	ads1100_set_config_bits(data, ADS1100_CFG_SC, ADS1100_SINGLESHOT);
+}
+
+static int ads1100_probe(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev;
+	struct ads1100_data *data;
+	struct device *dev = &client->dev;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	dev_set_drvdata(dev, data);
+	data->client = client;
+	mutex_init(&data->lock);
+
+	indio_dev->name = "ads1100";
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = &ads1100_channel;
+	indio_dev->num_channels = 1;
+	indio_dev->info = &ads1100_info;
+
+	data->reg_vdd = devm_regulator_get(dev, "vdd");
+	if (IS_ERR(data->reg_vdd))
+		return dev_err_probe(dev, PTR_ERR(data->reg_vdd),
+				     "Failed to get vdd regulator\n");
+
+	ret = regulator_enable(data->reg_vdd);
+	if (ret < 0)
+		return dev_err_probe(dev, ret,
+				     "Failed to enable vdd regulator\n");
+
+	ret = devm_add_action_or_reset(dev, ads1100_reg_disable, data->reg_vdd);
+	if (ret)
+		return ret;
+
+	ret = ads1100_setup(data);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to communicate with device\n");
+
+	ret = devm_add_action_or_reset(dev, ads1100_disable_continuous, data);
+	if (ret)
+		return ret;
+
+	ads1100_calc_scale_avail(data);
+
+	pm_runtime_set_autosuspend_delay(dev, ADS1100_SLEEP_DELAY_MS);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_active(dev);
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to enable pm_runtime\n");
+
+	ret = devm_iio_device_register(dev, indio_dev);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to register IIO device\n");
+
+	return 0;
+}
+
+static int ads1100_runtime_suspend(struct device *dev)
+{
+	struct ads1100_data *data = dev_get_drvdata(dev);
+
+	ads1100_set_config_bits(data, ADS1100_CFG_SC, ADS1100_SINGLESHOT);
+	regulator_disable(data->reg_vdd);
+
+	return 0;
+}
+
+static int ads1100_runtime_resume(struct device *dev)
+{
+	struct ads1100_data *data = dev_get_drvdata(dev);
+	int ret;
+
+	ret = regulator_enable(data->reg_vdd);
+	if (ret) {
+		dev_err(&data->client->dev, "Failed to enable Vdd\n");
+		return ret;
+	}
+
+	/*
+	 * We'll always change the mode bit in the config register, so there is
+	 * no need here to "force" a write to the config register. If the device
+	 * has been power-cycled, we'll re-write its config register now.
+	 */
+	return ads1100_set_config_bits(data, ADS1100_CFG_SC,
+				       ADS1100_CONTINUOUS);
+}
+
+static DEFINE_RUNTIME_DEV_PM_OPS(ads1100_pm_ops,
+				 ads1100_runtime_suspend,
+				 ads1100_runtime_resume,
+				 NULL);
+
+static const struct i2c_device_id ads1100_id[] = {
+	{ "ads1100" },
+	{ "ads1000" },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(i2c, ads1100_id);
+
+static const struct of_device_id ads1100_of_match[] = {
+	{.compatible = "ti,ads1100" },
+	{.compatible = "ti,ads1000" },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(of, ads1100_of_match);
+
+static struct i2c_driver ads1100_driver = {
+	.driver = {
+		   .name = "ads1100",
+		   .of_match_table = ads1100_of_match,
+		   .pm = pm_ptr(&ads1100_pm_ops),
+	},
+	.probe_new = ads1100_probe,
+	.id_table = ads1100_id,
+};
+
+module_i2c_driver(ads1100_driver);
+
+MODULE_AUTHOR("Mike Looijmans <mike.looijmans@topic.nl>");
+MODULE_DESCRIPTION("Texas Instruments ADS1100 ADC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/addac/Kconfig b/drivers/iio/addac/Kconfig
index 2843fcb70e24..877f9124803c 100644
--- a/drivers/iio/addac/Kconfig
+++ b/drivers/iio/addac/Kconfig
@@ -35,7 +35,9 @@ config STX104
 	tristate "Apex Embedded Systems STX104 driver"
 	depends on PC104 && X86
 	select ISA_BUS_API
+	select REGMAP_MMIO
 	select GPIOLIB
+	select GPIO_REGMAP
 	help
 	  Say yes here to build support for the Apex Embedded Systems STX104
 	  integrated analog PC/104 card.
diff --git a/drivers/iio/addac/ad74413r.c b/drivers/iio/addac/ad74413r.c
index f32c8c2fb26d..07e9f6ae16a8 100644
--- a/drivers/iio/addac/ad74413r.c
+++ b/drivers/iio/addac/ad74413r.c
@@ -39,6 +39,7 @@ struct ad74413r_chip_info {
 
 struct ad74413r_channel_config {
 	u32		func;
+	u32		drive_strength;
 	bool		gpo_comparator;
 	bool		initialized;
 };
@@ -99,6 +100,7 @@ struct ad74413r_state {
 #define AD74413R_REG_ADC_CONFIG_X(x)		(0x05 + (x))
 #define AD74413R_ADC_CONFIG_RANGE_MASK		GENMASK(7, 5)
 #define AD74413R_ADC_CONFIG_REJECTION_MASK	GENMASK(4, 3)
+#define AD74413R_ADC_CONFIG_CH_200K_TO_GND	BIT(2)
 #define AD74413R_ADC_RANGE_10V			0b000
 #define AD74413R_ADC_RANGE_2P5V_EXT_POW		0b001
 #define AD74413R_ADC_RANGE_2P5V_INT_POW		0b010
@@ -111,6 +113,7 @@ struct ad74413r_state {
 #define AD74413R_REG_DIN_CONFIG_X(x)	(0x09 + (x))
 #define AD74413R_DIN_DEBOUNCE_MASK	GENMASK(4, 0)
 #define AD74413R_DIN_DEBOUNCE_LEN	BIT(5)
+#define AD74413R_DIN_SINK_MASK		GENMASK(9, 6)
 
 #define AD74413R_REG_DAC_CODE_X(x)	(0x16 + (x))
 #define AD74413R_DAC_CODE_MAX		GENMASK(12, 0)
@@ -261,6 +264,18 @@ static int ad74413r_set_comp_debounce(struct ad74413r_state *st,
 				  val);
 }
 
+static int ad74413r_set_comp_drive_strength(struct ad74413r_state *st,
+					    unsigned int offset,
+					    unsigned int strength)
+{
+	strength = min(strength, 1800U);
+
+	return regmap_update_bits(st->regmap, AD74413R_REG_DIN_CONFIG_X(offset),
+				  AD74413R_DIN_SINK_MASK,
+				  FIELD_PREP(AD74413R_DIN_SINK_MASK, strength / 120));
+}
+
+
 static void ad74413r_gpio_set(struct gpio_chip *chip,
 			      unsigned int offset, int val)
 {
@@ -424,9 +439,20 @@ static int ad74413r_set_channel_dac_code(struct ad74413r_state *st,
 static int ad74413r_set_channel_function(struct ad74413r_state *st,
 					 unsigned int channel, u8 func)
 {
-	return regmap_update_bits(st->regmap,
+	int ret;
+
+	ret = regmap_update_bits(st->regmap,
 				  AD74413R_REG_CH_FUNC_SETUP_X(channel),
 				  AD74413R_CH_FUNC_SETUP_MASK, func);
+	if (ret)
+		return ret;
+
+	if (func == CH_FUNC_CURRENT_INPUT_LOOP_POWER)
+		ret = regmap_set_bits(st->regmap,
+				      AD74413R_REG_ADC_CONFIG_X(channel),
+				      AD74413R_ADC_CONFIG_CH_200K_TO_GND);
+
+	return ret;
 }
 
 static int ad74413r_set_adc_conv_seq(struct ad74413r_state *st,
@@ -1112,6 +1138,11 @@ static struct iio_chan_spec ad74413r_current_input_channels[] = {
 	AD74413R_ADC_CURRENT_CHANNEL,
 };
 
+static struct iio_chan_spec ad74413r_current_input_loop_channels[] = {
+	AD74413R_DAC_CHANNEL(IIO_CURRENT, BIT(IIO_CHAN_INFO_SCALE)),
+	AD74413R_ADC_CURRENT_CHANNEL,
+};
+
 static struct iio_chan_spec ad74413r_resistance_input_channels[] = {
 	AD74413R_ADC_CHANNEL(IIO_RESISTANCE, BIT(IIO_CHAN_INFO_PROCESSED)),
 };
@@ -1135,7 +1166,7 @@ static const struct ad74413r_channels ad74413r_channels_map[] = {
 	[CH_FUNC_CURRENT_OUTPUT] = AD74413R_CHANNELS(current_output),
 	[CH_FUNC_VOLTAGE_INPUT] = AD74413R_CHANNELS(voltage_input),
 	[CH_FUNC_CURRENT_INPUT_EXT_POWER] = AD74413R_CHANNELS(current_input),
-	[CH_FUNC_CURRENT_INPUT_LOOP_POWER] = AD74413R_CHANNELS(current_input),
+	[CH_FUNC_CURRENT_INPUT_LOOP_POWER] = AD74413R_CHANNELS(current_input_loop),
 	[CH_FUNC_RESISTANCE_INPUT] = AD74413R_CHANNELS(resistance_input),
 	[CH_FUNC_DIGITAL_INPUT_LOGIC] = AD74413R_CHANNELS(digital_input),
 	[CH_FUNC_DIGITAL_INPUT_LOOP_POWER] = AD74413R_CHANNELS(digital_input),
@@ -1190,6 +1221,9 @@ static int ad74413r_parse_channel_config(struct iio_dev *indio_dev,
 	config->gpo_comparator = fwnode_property_read_bool(channel_node,
 		"adi,gpo-comparator");
 
+	fwnode_property_read_u32(channel_node, "drive-strength-microamp",
+				 &config->drive_strength);
+
 	if (!config->gpo_comparator)
 		st->num_gpo_gpios++;
 
@@ -1269,6 +1303,7 @@ static int ad74413r_setup_gpios(struct ad74413r_state *st)
 	unsigned int gpo_gpio_i = 0;
 	unsigned int i;
 	u8 gpo_config;
+	u32 strength;
 	int ret;
 
 	for (i = 0; i < AD74413R_CHANNEL_MAX; i++) {
@@ -1285,6 +1320,11 @@ static int ad74413r_setup_gpios(struct ad74413r_state *st)
 		    config->func == CH_FUNC_DIGITAL_INPUT_LOOP_POWER)
 			st->comp_gpio_offsets[comp_gpio_i++] = i;
 
+		strength = config->drive_strength;
+		ret = ad74413r_set_comp_drive_strength(st, i, strength);
+		if (ret)
+			return ret;
+
 		ret = ad74413r_set_gpo_config(st, i, gpo_config);
 		if (ret)
 			return ret;
diff --git a/drivers/iio/addac/stx104.c b/drivers/iio/addac/stx104.c
index 48a91a95e597..d1f7ce033b46 100644
--- a/drivers/iio/addac/stx104.c
+++ b/drivers/iio/addac/stx104.c
@@ -3,19 +3,20 @@
  * IIO driver for the Apex Embedded Systems STX104
  * Copyright (C) 2016 William Breathitt Gray
  */
-#include <linux/bitops.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
 #include <linux/device.h>
-#include <linux/errno.h>
-#include <linux/gpio/driver.h>
+#include <linux/err.h>
+#include <linux/gpio/regmap.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/types.h>
-#include <linux/io.h>
-#include <linux/ioport.h>
 #include <linux/isa.h>
 #include <linux/kernel.h>
+#include <linux/limits.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
-#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/regmap.h>
 #include <linux/types.h>
 
 #define STX104_OUT_CHAN(chan) {				\
@@ -45,101 +46,211 @@ static unsigned int num_stx104;
 module_param_hw_array(base, uint, ioport, &num_stx104, 0);
 MODULE_PARM_DESC(base, "Apex Embedded Systems STX104 base addresses");
 
-/**
- * struct stx104_reg - device register structure
- * @ssr_ad:	Software Strobe Register and ADC Data
- * @achan:	ADC Channel
- * @dio:	Digital I/O
- * @dac:	DAC Channels
- * @cir_asr:	Clear Interrupts and ADC Status
- * @acr:	ADC Control
- * @pccr_fsh:	Pacer Clock Control and FIFO Status MSB
- * @acfg:	ADC Configuration
- */
-struct stx104_reg {
-	u16 ssr_ad;
-	u8 achan;
-	u8 dio;
-	u16 dac[2];
-	u8 cir_asr;
-	u8 acr;
-	u8 pccr_fsh;
-	u8 acfg;
-};
+#define STX104_AIO_BASE 0x0
+#define STX104_SOFTWARE_STROBE STX104_AIO_BASE
+#define STX104_ADC_DATA STX104_AIO_BASE
+#define STX104_ADC_CHANNEL (STX104_AIO_BASE + 0x2)
+#define STX104_DIO_REG (STX104_AIO_BASE + 0x3)
+#define STX104_DAC_BASE (STX104_AIO_BASE + 0x4)
+#define STX104_ADC_STATUS (STX104_AIO_BASE + 0x8)
+#define STX104_ADC_CONTROL (STX104_AIO_BASE + 0x9)
+#define STX104_ADC_CONFIGURATION (STX104_AIO_BASE + 0x11)
+
+#define STX104_AIO_DATA_STRIDE 2
+#define STX104_DAC_OFFSET(_channel) (STX104_DAC_BASE + STX104_AIO_DATA_STRIDE * (_channel))
+
+/* ADC Channel */
+#define STX104_FC GENMASK(3, 0)
+#define STX104_LC GENMASK(7, 4)
+#define STX104_SINGLE_CHANNEL(_channel) \
+	(u8_encode_bits(_channel, STX104_FC) | u8_encode_bits(_channel, STX104_LC))
+
+/* ADC Status */
+#define STX104_SD BIT(5)
+#define STX104_CNV BIT(7)
+#define STX104_DIFFERENTIAL 1
+
+/* ADC Control */
+#define STX104_ALSS GENMASK(1, 0)
+#define STX104_SOFTWARE_TRIGGER u8_encode_bits(0x0, STX104_ALSS)
+
+/* ADC Configuration */
+#define STX104_GAIN GENMASK(1, 0)
+#define STX104_ADBU BIT(2)
+#define STX104_BIPOLAR 0
+#define STX104_GAIN_X1 0
+#define STX104_GAIN_X2 1
+#define STX104_GAIN_X4 2
+#define STX104_GAIN_X8 3
 
 /**
  * struct stx104_iio - IIO device private data structure
- * @chan_out_states:	channels' output states
- * @reg:		I/O address offset for the device registers
+ * @lock: synchronization lock to prevent I/O race conditions
+ * @aio_data_map: Regmap for analog I/O data
+ * @aio_ctl_map: Regmap for analog I/O control
  */
 struct stx104_iio {
-	unsigned int chan_out_states[STX104_NUM_OUT_CHAN];
-	struct stx104_reg __iomem *reg;
+	struct mutex lock;
+	struct regmap *aio_data_map;
+	struct regmap *aio_ctl_map;
 };
 
-/**
- * struct stx104_gpio - GPIO device private data structure
- * @chip:	instance of the gpio_chip
- * @lock:	synchronization lock to prevent I/O race conditions
- * @base:	base port address of the GPIO device
- * @out_state:	output bits state
- */
-struct stx104_gpio {
-	struct gpio_chip chip;
-	spinlock_t lock;
-	u8 __iomem *base;
-	unsigned int out_state;
+static const struct regmap_range aio_ctl_wr_ranges[] = {
+	regmap_reg_range(0x0, 0x0), regmap_reg_range(0x2, 0x2), regmap_reg_range(0x9, 0x9),
+	regmap_reg_range(0x11, 0x11),
+};
+static const struct regmap_range aio_ctl_rd_ranges[] = {
+	regmap_reg_range(0x2, 0x2), regmap_reg_range(0x8, 0x9), regmap_reg_range(0x11, 0x11),
+};
+static const struct regmap_range aio_ctl_volatile_ranges[] = {
+	regmap_reg_range(0x8, 0x8),
+};
+static const struct regmap_access_table aio_ctl_wr_table = {
+	.yes_ranges = aio_ctl_wr_ranges,
+	.n_yes_ranges = ARRAY_SIZE(aio_ctl_wr_ranges),
+};
+static const struct regmap_access_table aio_ctl_rd_table = {
+	.yes_ranges = aio_ctl_rd_ranges,
+	.n_yes_ranges = ARRAY_SIZE(aio_ctl_rd_ranges),
+};
+static const struct regmap_access_table aio_ctl_volatile_table = {
+	.yes_ranges = aio_ctl_volatile_ranges,
+	.n_yes_ranges = ARRAY_SIZE(aio_ctl_volatile_ranges),
+};
+
+static const struct regmap_config aio_ctl_regmap_config = {
+	.name = "aio_ctl",
+	.reg_bits = 8,
+	.reg_stride = 1,
+	.reg_base = STX104_AIO_BASE,
+	.val_bits = 8,
+	.io_port = true,
+	.wr_table = &aio_ctl_wr_table,
+	.rd_table = &aio_ctl_rd_table,
+	.volatile_table = &aio_ctl_volatile_table,
+	.cache_type = REGCACHE_FLAT,
+};
+
+static const struct regmap_range aio_data_wr_ranges[] = {
+	regmap_reg_range(0x4, 0x6),
+};
+static const struct regmap_range aio_data_rd_ranges[] = {
+	regmap_reg_range(0x0, 0x0),
+};
+static const struct regmap_access_table aio_data_wr_table = {
+	.yes_ranges = aio_data_wr_ranges,
+	.n_yes_ranges = ARRAY_SIZE(aio_data_wr_ranges),
+};
+static const struct regmap_access_table aio_data_rd_table = {
+	.yes_ranges = aio_data_rd_ranges,
+	.n_yes_ranges = ARRAY_SIZE(aio_data_rd_ranges),
+};
+
+static const struct regmap_config aio_data_regmap_config = {
+	.name = "aio_data",
+	.reg_bits = 16,
+	.reg_stride = STX104_AIO_DATA_STRIDE,
+	.reg_base = STX104_AIO_BASE,
+	.val_bits = 16,
+	.io_port = true,
+	.wr_table = &aio_data_wr_table,
+	.rd_table = &aio_data_rd_table,
+	.volatile_table = &aio_data_rd_table,
+	.cache_type = REGCACHE_FLAT,
+};
+
+static const struct regmap_config dio_regmap_config = {
+	.name = "dio",
+	.reg_bits = 8,
+	.reg_stride = 1,
+	.reg_base = STX104_DIO_REG,
+	.val_bits = 8,
+	.io_port = true,
 };
 
 static int stx104_read_raw(struct iio_dev *indio_dev,
 	struct iio_chan_spec const *chan, int *val, int *val2, long mask)
 {
 	struct stx104_iio *const priv = iio_priv(indio_dev);
-	struct stx104_reg __iomem *const reg = priv->reg;
+	int err;
 	unsigned int adc_config;
-	int adbu;
-	int gain;
+	unsigned int value;
+	unsigned int adc_status;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_HARDWAREGAIN:
-		/* get gain configuration */
-		adc_config = ioread8(&reg->acfg);
-		gain = adc_config & 0x3;
+		err = regmap_read(priv->aio_ctl_map, STX104_ADC_CONFIGURATION, &adc_config);
+		if (err)
+			return err;
 
-		*val = 1 << gain;
+		*val = BIT(u8_get_bits(adc_config, STX104_GAIN));
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_RAW:
 		if (chan->output) {
-			*val = priv->chan_out_states[chan->channel];
+			err = regmap_read(priv->aio_data_map, STX104_DAC_OFFSET(chan->channel),
+					  &value);
+			if (err)
+				return err;
+			*val = value;
 			return IIO_VAL_INT;
 		}
 
+		mutex_lock(&priv->lock);
+
 		/* select ADC channel */
-		iowrite8(chan->channel | (chan->channel << 4), &reg->achan);
+		err = regmap_write(priv->aio_ctl_map, STX104_ADC_CHANNEL,
+				   STX104_SINGLE_CHANNEL(chan->channel));
+		if (err) {
+			mutex_unlock(&priv->lock);
+			return err;
+		}
 
-		/* trigger ADC sample capture by writing to the 8-bit
-		 * Software Strobe Register and wait for completion
+		/*
+		 * Trigger ADC sample capture by writing to the 8-bit Software Strobe Register and
+		 * wait for completion; the conversion time range is 5 microseconds to 53.68 seconds
+		 * in steps of 25 nanoseconds. The actual Analog Input Frame Timer time interval is
+		 * calculated as:
+		 * ai_time_frame_ns = ( AIFT + 1 ) * ( 25 nanoseconds ).
+		 * Where 0 <= AIFT <= 2147483648.
 		 */
-		iowrite8(0, &reg->ssr_ad);
-		while (ioread8(&reg->cir_asr) & BIT(7));
+		err = regmap_write(priv->aio_ctl_map, STX104_SOFTWARE_STROBE, 0);
+		if (err) {
+			mutex_unlock(&priv->lock);
+			return err;
+		}
+		err = regmap_read_poll_timeout(priv->aio_ctl_map, STX104_ADC_STATUS, adc_status,
+					       !u8_get_bits(adc_status, STX104_CNV), 0, 53687092);
+		if (err) {
+			mutex_unlock(&priv->lock);
+			return err;
+		}
+
+		err = regmap_read(priv->aio_data_map, STX104_ADC_DATA, &value);
+		if (err) {
+			mutex_unlock(&priv->lock);
+			return err;
+		}
+		*val = value;
 
-		*val = ioread16(&reg->ssr_ad);
+		mutex_unlock(&priv->lock);
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_OFFSET:
 		/* get ADC bipolar/unipolar configuration */
-		adc_config = ioread8(&reg->acfg);
-		adbu = !(adc_config & BIT(2));
+		err = regmap_read(priv->aio_ctl_map, STX104_ADC_CONFIGURATION, &adc_config);
+		if (err)
+			return err;
 
-		*val = -32768 * adbu;
+		*val = (u8_get_bits(adc_config, STX104_ADBU) == STX104_BIPOLAR) ? -32768 : 0;
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_SCALE:
 		/* get ADC bipolar/unipolar and gain configuration */
-		adc_config = ioread8(&reg->acfg);
-		adbu = !(adc_config & BIT(2));
-		gain = adc_config & 0x3;
+		err = regmap_read(priv->aio_ctl_map, STX104_ADC_CONFIGURATION, &adc_config);
+		if (err)
+			return err;
 
 		*val = 5;
-		*val2 = 15 - adbu + gain;
+		*val2 = (u8_get_bits(adc_config, STX104_ADBU) == STX104_BIPOLAR) ? 14 : 15;
+		*val2 += u8_get_bits(adc_config, STX104_GAIN);
 		return IIO_VAL_FRACTIONAL_LOG2;
 	}
 
@@ -150,40 +261,37 @@ static int stx104_write_raw(struct iio_dev *indio_dev,
 	struct iio_chan_spec const *chan, int val, int val2, long mask)
 {
 	struct stx104_iio *const priv = iio_priv(indio_dev);
+	u8 gain;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_HARDWAREGAIN:
 		/* Only four gain states (x1, x2, x4, x8) */
 		switch (val) {
 		case 1:
-			iowrite8(0, &priv->reg->acfg);
+			gain = STX104_GAIN_X1;
 			break;
 		case 2:
-			iowrite8(1, &priv->reg->acfg);
+			gain = STX104_GAIN_X2;
 			break;
 		case 4:
-			iowrite8(2, &priv->reg->acfg);
+			gain = STX104_GAIN_X4;
 			break;
 		case 8:
-			iowrite8(3, &priv->reg->acfg);
+			gain = STX104_GAIN_X8;
 			break;
 		default:
 			return -EINVAL;
 		}
 
-		return 0;
+		return regmap_write(priv->aio_ctl_map, STX104_ADC_CONFIGURATION, gain);
 	case IIO_CHAN_INFO_RAW:
-		if (chan->output) {
-			/* DAC can only accept up to a 16-bit value */
-			if ((unsigned int)val > 65535)
-				return -EINVAL;
+		if (!chan->output)
+			return -EINVAL;
 
-			priv->chan_out_states[chan->channel] = val;
-			iowrite16(val, &priv->reg->dac[chan->channel]);
+		if (val < 0 || val > U16_MAX)
+			return -EINVAL;
 
-			return 0;
-		}
-		return -EINVAL;
+		return regmap_write(priv->aio_data_map, STX104_DAC_OFFSET(chan->channel), val);
 	}
 
 	return -EINVAL;
@@ -212,119 +320,66 @@ static const struct iio_chan_spec stx104_channels_diff[] = {
 	STX104_IN_CHAN(6, 1), STX104_IN_CHAN(7, 1)
 };
 
-static int stx104_gpio_get_direction(struct gpio_chip *chip,
-	unsigned int offset)
-{
-	/* GPIO 0-3 are input only, while the rest are output only */
-	if (offset < 4)
-		return 1;
-
-	return 0;
-}
-
-static int stx104_gpio_direction_input(struct gpio_chip *chip,
-	unsigned int offset)
+static int stx104_reg_mask_xlate(struct gpio_regmap *const gpio, const unsigned int base,
+				 unsigned int offset, unsigned int *const reg,
+				 unsigned int *const mask)
 {
+	/* Output lines are located at same register bit offsets as input lines */
 	if (offset >= 4)
-		return -EINVAL;
-
-	return 0;
-}
+		offset -= 4;
 
-static int stx104_gpio_direction_output(struct gpio_chip *chip,
-	unsigned int offset, int value)
-{
-	if (offset < 4)
-		return -EINVAL;
+	*reg = base;
+	*mask = BIT(offset);
 
-	chip->set(chip, offset, value);
 	return 0;
 }
 
-static int stx104_gpio_get(struct gpio_chip *chip, unsigned int offset)
-{
-	struct stx104_gpio *const stx104gpio = gpiochip_get_data(chip);
-
-	if (offset >= 4)
-		return -EINVAL;
-
-	return !!(ioread8(stx104gpio->base) & BIT(offset));
-}
-
-static int stx104_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask,
-	unsigned long *bits)
-{
-	struct stx104_gpio *const stx104gpio = gpiochip_get_data(chip);
-
-	*bits = ioread8(stx104gpio->base);
-
-	return 0;
-}
-
-static void stx104_gpio_set(struct gpio_chip *chip, unsigned int offset,
-	int value)
-{
-	struct stx104_gpio *const stx104gpio = gpiochip_get_data(chip);
-	const unsigned int mask = BIT(offset) >> 4;
-	unsigned long flags;
-
-	if (offset < 4)
-		return;
-
-	spin_lock_irqsave(&stx104gpio->lock, flags);
-
-	if (value)
-		stx104gpio->out_state |= mask;
-	else
-		stx104gpio->out_state &= ~mask;
-
-	iowrite8(stx104gpio->out_state, stx104gpio->base);
-
-	spin_unlock_irqrestore(&stx104gpio->lock, flags);
-}
-
 #define STX104_NGPIO 8
 static const char *stx104_names[STX104_NGPIO] = {
 	"DIN0", "DIN1", "DIN2", "DIN3", "DOUT0", "DOUT1", "DOUT2", "DOUT3"
 };
 
-static void stx104_gpio_set_multiple(struct gpio_chip *chip,
-	unsigned long *mask, unsigned long *bits)
+static int stx104_init_hw(struct stx104_iio *const priv)
 {
-	struct stx104_gpio *const stx104gpio = gpiochip_get_data(chip);
-	unsigned long flags;
-
-	/* verify masked GPIO are output */
-	if (!(*mask & 0xF0))
-		return;
+	int err;
 
-	*mask >>= 4;
-	*bits >>= 4;
+	/* configure device for software trigger operation */
+	err = regmap_write(priv->aio_ctl_map, STX104_ADC_CONTROL, STX104_SOFTWARE_TRIGGER);
+	if (err)
+		return err;
 
-	spin_lock_irqsave(&stx104gpio->lock, flags);
+	/* initialize gain setting to x1 */
+	err = regmap_write(priv->aio_ctl_map, STX104_ADC_CONFIGURATION, STX104_GAIN_X1);
+	if (err)
+		return err;
 
-	stx104gpio->out_state &= ~*mask;
-	stx104gpio->out_state |= *mask & *bits;
-	iowrite8(stx104gpio->out_state, stx104gpio->base);
+	/* initialize DAC outputs to 0V */
+	err = regmap_write(priv->aio_data_map, STX104_DAC_BASE, 0);
+	if (err)
+		return err;
+	err = regmap_write(priv->aio_data_map, STX104_DAC_BASE + STX104_AIO_DATA_STRIDE, 0);
+	if (err)
+		return err;
 
-	spin_unlock_irqrestore(&stx104gpio->lock, flags);
+	return 0;
 }
 
 static int stx104_probe(struct device *dev, unsigned int id)
 {
 	struct iio_dev *indio_dev;
 	struct stx104_iio *priv;
-	struct stx104_gpio *stx104gpio;
+	struct gpio_regmap_config gpio_config;
+	void __iomem *stx104_base;
+	struct regmap *aio_ctl_map;
+	struct regmap *aio_data_map;
+	struct regmap *dio_map;
 	int err;
+	unsigned int adc_status;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*priv));
 	if (!indio_dev)
 		return -ENOMEM;
 
-	stx104gpio = devm_kzalloc(dev, sizeof(*stx104gpio), GFP_KERNEL);
-	if (!stx104gpio)
-		return -ENOMEM;
-
 	if (!devm_request_region(dev, base[id], STX104_EXTENT,
 		dev_name(dev))) {
 		dev_err(dev, "Unable to lock port addresses (0x%X-0x%X)\n",
@@ -332,16 +387,37 @@ static int stx104_probe(struct device *dev, unsigned int id)
 		return -EBUSY;
 	}
 
-	priv = iio_priv(indio_dev);
-	priv->reg = devm_ioport_map(dev, base[id], STX104_EXTENT);
-	if (!priv->reg)
+	stx104_base = devm_ioport_map(dev, base[id], STX104_EXTENT);
+	if (!stx104_base)
 		return -ENOMEM;
 
+	aio_ctl_map = devm_regmap_init_mmio(dev, stx104_base, &aio_ctl_regmap_config);
+	if (IS_ERR(aio_ctl_map))
+		return dev_err_probe(dev, PTR_ERR(aio_ctl_map),
+				     "Unable to initialize aio_ctl register map\n");
+
+	aio_data_map = devm_regmap_init_mmio(dev, stx104_base, &aio_data_regmap_config);
+	if (IS_ERR(aio_data_map))
+		return dev_err_probe(dev, PTR_ERR(aio_data_map),
+				     "Unable to initialize aio_data register map\n");
+
+	dio_map = devm_regmap_init_mmio(dev, stx104_base, &dio_regmap_config);
+	if (IS_ERR(dio_map))
+		return dev_err_probe(dev, PTR_ERR(dio_map),
+				     "Unable to initialize dio register map\n");
+
+	priv = iio_priv(indio_dev);
+	priv->aio_ctl_map = aio_ctl_map;
+	priv->aio_data_map = aio_data_map;
+
 	indio_dev->info = &stx104_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	/* determine if differential inputs */
-	if (ioread8(&priv->reg->cir_asr) & BIT(5)) {
+	err = regmap_read(aio_ctl_map, STX104_ADC_STATUS, &adc_status);
+	if (err)
+		return err;
+
+	if (u8_get_bits(adc_status, STX104_SD) == STX104_DIFFERENTIAL) {
 		indio_dev->num_channels = ARRAY_SIZE(stx104_channels_diff);
 		indio_dev->channels = stx104_channels_diff;
 	} else {
@@ -351,41 +427,29 @@ static int stx104_probe(struct device *dev, unsigned int id)
 
 	indio_dev->name = dev_name(dev);
 
-	/* configure device for software trigger operation */
-	iowrite8(0, &priv->reg->acr);
+	mutex_init(&priv->lock);
 
-	/* initialize gain setting to x1 */
-	iowrite8(0, &priv->reg->acfg);
-
-	/* initialize DAC output to 0V */
-	iowrite16(0, &priv->reg->dac[0]);
-	iowrite16(0, &priv->reg->dac[1]);
-
-	stx104gpio->chip.label = dev_name(dev);
-	stx104gpio->chip.parent = dev;
-	stx104gpio->chip.owner = THIS_MODULE;
-	stx104gpio->chip.base = -1;
-	stx104gpio->chip.ngpio = STX104_NGPIO;
-	stx104gpio->chip.names = stx104_names;
-	stx104gpio->chip.get_direction = stx104_gpio_get_direction;
-	stx104gpio->chip.direction_input = stx104_gpio_direction_input;
-	stx104gpio->chip.direction_output = stx104_gpio_direction_output;
-	stx104gpio->chip.get = stx104_gpio_get;
-	stx104gpio->chip.get_multiple = stx104_gpio_get_multiple;
-	stx104gpio->chip.set = stx104_gpio_set;
-	stx104gpio->chip.set_multiple = stx104_gpio_set_multiple;
-	stx104gpio->base = &priv->reg->dio;
-	stx104gpio->out_state = 0x0;
-
-	spin_lock_init(&stx104gpio->lock);
-
-	err = devm_gpiochip_add_data(dev, &stx104gpio->chip, stx104gpio);
-	if (err) {
-		dev_err(dev, "GPIO registering failed (%d)\n", err);
+	err = stx104_init_hw(priv);
+	if (err)
+		return err;
+
+	err = devm_iio_device_register(dev, indio_dev);
+	if (err)
 		return err;
-	}
 
-	return devm_iio_device_register(dev, indio_dev);
+	gpio_config = (struct gpio_regmap_config) {
+		.parent = dev,
+		.regmap = dio_map,
+		.ngpio = STX104_NGPIO,
+		.names = stx104_names,
+		.reg_dat_base = GPIO_REGMAP_ADDR(STX104_DIO_REG),
+		.reg_set_base = GPIO_REGMAP_ADDR(STX104_DIO_REG),
+		.ngpio_per_reg = STX104_NGPIO,
+		.reg_mask_xlate = stx104_reg_mask_xlate,
+		.drvdata = dio_map,
+	};
+
+	return PTR_ERR_OR_ZERO(devm_gpio_regmap_register(dev, &gpio_config));
 }
 
 static struct isa_driver stx104_driver = {
diff --git a/drivers/iio/chemical/sps30_i2c.c b/drivers/iio/chemical/sps30_i2c.c
index 2aed483a2fde..0cb5d9b65d62 100644
--- a/drivers/iio/chemical/sps30_i2c.c
+++ b/drivers/iio/chemical/sps30_i2c.c
@@ -68,10 +68,10 @@ static int sps30_i2c_command(struct sps30_state *state, u16 cmd, void *arg, size
 	/*
 	 * Internally sensor stores measurements in a following manner:
 	 *
-	 * PM1: upper two bytes, crc8, lower two bytes, crc8
+	 * PM1:   upper two bytes, crc8, lower two bytes, crc8
 	 * PM2P5: upper two bytes, crc8, lower two bytes, crc8
-	 * PM4: upper two bytes, crc8, lower two bytes, crc8
-	 * PM10: upper two bytes, crc8, lower two bytes, crc8
+	 * PM4:   upper two bytes, crc8, lower two bytes, crc8
+	 * PM10:  upper two bytes, crc8, lower two bytes, crc8
 	 *
 	 * What follows next are number concentration measurements and
 	 * typical particle size measurement which we omit.
diff --git a/drivers/iio/common/st_sensors/st_sensors_trigger.c b/drivers/iio/common/st_sensors/st_sensors_trigger.c
index 899b640c0a70..a0df9250a69f 100644
--- a/drivers/iio/common/st_sensors/st_sensors_trigger.c
+++ b/drivers/iio/common/st_sensors/st_sensors_trigger.c
@@ -85,7 +85,7 @@ static irqreturn_t st_sensors_irq_thread(int irq, void *p)
 	 */
 	if (sdata->hw_irq_trigger &&
 	    st_sensors_new_samples_available(indio_dev, sdata)) {
-		iio_trigger_poll_chained(p);
+		iio_trigger_poll_nested(p);
 	} else {
 		dev_dbg(indio_dev->dev.parent, "spurious IRQ\n");
 		return IRQ_NONE;
@@ -110,7 +110,7 @@ static irqreturn_t st_sensors_irq_thread(int irq, void *p)
 		dev_dbg(indio_dev->dev.parent,
 			"more samples came in during polling\n");
 		sdata->hw_timestamp = iio_get_time_ns(indio_dev);
-		iio_trigger_poll_chained(p);
+		iio_trigger_poll_nested(p);
 	}
 
 	return IRQ_HANDLED;
diff --git a/drivers/iio/dac/Kconfig b/drivers/iio/dac/Kconfig
index d3f90cf86143..3acd9c3f388e 100644
--- a/drivers/iio/dac/Kconfig
+++ b/drivers/iio/dac/Kconfig
@@ -277,6 +277,7 @@ config CIO_DAC
 	tristate "Measurement Computing CIO-DAC IIO driver"
 	depends on X86 && (ISA_BUS || PC104)
 	select ISA_BUS_API
+	select REGMAP_MMIO
 	help
 	  Say yes here to build support for the Measurement Computing CIO-DAC
 	  analog output device family (CIO-DAC16, CIO-DAC08, PC104-DAC06). The
diff --git a/drivers/iio/dac/ad5592r-base.c b/drivers/iio/dac/ad5592r-base.c
index 7a9b5fc1e579..076bc9ecfb49 100644
--- a/drivers/iio/dac/ad5592r-base.c
+++ b/drivers/iio/dac/ad5592r-base.c
@@ -124,6 +124,10 @@ static int ad5592r_gpio_request(struct gpio_chip *chip, unsigned offset)
 	return 0;
 }
 
+static const char * const ad5592r_gpio_names[] = {
+	"GPIO0", "GPIO1", "GPIO2", "GPIO3", "GPIO4", "GPIO5", "GPIO6", "GPIO7",
+};
+
 static int ad5592r_gpio_init(struct ad5592r_state *st)
 {
 	if (!st->gpio_map)
@@ -140,6 +144,7 @@ static int ad5592r_gpio_init(struct ad5592r_state *st)
 	st->gpiochip.set = ad5592r_gpio_set;
 	st->gpiochip.request = ad5592r_gpio_request;
 	st->gpiochip.owner = THIS_MODULE;
+	st->gpiochip.names = ad5592r_gpio_names;
 
 	mutex_init(&st->gpio_lock);
 
diff --git a/drivers/iio/dac/ad5755.c b/drivers/iio/dac/ad5755.c
index beadfa938d2d..404865e35460 100644
--- a/drivers/iio/dac/ad5755.c
+++ b/drivers/iio/dac/ad5755.c
@@ -802,6 +802,7 @@ static struct ad5755_platform_data *ad5755_parse_fw(struct device *dev)
 	return pdata;
 
  error_out:
+	fwnode_handle_put(pp);
 	devm_kfree(dev, pdata);
 	return NULL;
 }
diff --git a/drivers/iio/dac/cio-dac.c b/drivers/iio/dac/cio-dac.c
index 18a64f72fc18..069904d00c2e 100644
--- a/drivers/iio/dac/cio-dac.c
+++ b/drivers/iio/dac/cio-dac.c
@@ -4,18 +4,17 @@
  * Copyright (C) 2016 William Breathitt Gray
  *
  * This driver supports the following Measurement Computing devices: CIO-DAC16,
- * CIO-DAC06, and PC104-DAC06.
+ * CIO-DAC08, and PC104-DAC06.
  */
-#include <linux/bitops.h>
+#include <linux/bits.h>
 #include <linux/device.h>
-#include <linux/errno.h>
+#include <linux/err.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/types.h>
-#include <linux/io.h>
-#include <linux/ioport.h>
 #include <linux/isa.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
+#include <linux/regmap.h>
 #include <linux/types.h>
 
 #define CIO_DAC_NUM_CHAN 16
@@ -35,25 +34,51 @@ static unsigned int num_cio_dac;
 module_param_hw_array(base, uint, ioport, &num_cio_dac, 0);
 MODULE_PARM_DESC(base, "Measurement Computing CIO-DAC base addresses");
 
+#define CIO_DAC_BASE 0x00
+#define CIO_DAC_CHANNEL_STRIDE 2
+
+static bool cio_dac_precious_reg(struct device *dev, unsigned int reg)
+{
+	/*
+	 * All registers are considered precious; if the XFER jumper is set on
+	 * the device, then no update occurs until a DAC register is read.
+	 */
+	return true;
+}
+
+static const struct regmap_config cio_dac_regmap_config = {
+	.reg_bits = 16,
+	.reg_stride = 2,
+	.val_bits = 16,
+	.io_port = true,
+	.max_register = 0x1F,
+	.precious_reg = cio_dac_precious_reg,
+};
+
 /**
  * struct cio_dac_iio - IIO device private data structure
- * @chan_out_states:	channels' output states
- * @base:		base memory address of the DAC device
+ * @map: Regmap for the device
  */
 struct cio_dac_iio {
-	int chan_out_states[CIO_DAC_NUM_CHAN];
-	u16 __iomem *base;
+	struct regmap *map;
 };
 
 static int cio_dac_read_raw(struct iio_dev *indio_dev,
 	struct iio_chan_spec const *chan, int *val, int *val2, long mask)
 {
 	struct cio_dac_iio *const priv = iio_priv(indio_dev);
+	const unsigned int offset = chan->channel * CIO_DAC_CHANNEL_STRIDE;
+	int err;
+	unsigned int dac_val;
 
 	if (mask != IIO_CHAN_INFO_RAW)
 		return -EINVAL;
 
-	*val = priv->chan_out_states[chan->channel];
+	err = regmap_read(priv->map, CIO_DAC_BASE + offset, &dac_val);
+	if (err)
+		return err;
+
+	*val = dac_val;
 
 	return IIO_VAL_INT;
 }
@@ -62,6 +87,7 @@ static int cio_dac_write_raw(struct iio_dev *indio_dev,
 	struct iio_chan_spec const *chan, int val, int val2, long mask)
 {
 	struct cio_dac_iio *const priv = iio_priv(indio_dev);
+	const unsigned int offset = chan->channel * CIO_DAC_CHANNEL_STRIDE;
 
 	if (mask != IIO_CHAN_INFO_RAW)
 		return -EINVAL;
@@ -70,10 +96,7 @@ static int cio_dac_write_raw(struct iio_dev *indio_dev,
 	if ((unsigned int)val > 4095)
 		return -EINVAL;
 
-	priv->chan_out_states[chan->channel] = val;
-	iowrite16(val, priv->base + chan->channel);
-
-	return 0;
+	return regmap_write(priv->map, CIO_DAC_BASE + offset, val);
 }
 
 static const struct iio_info cio_dac_info = {
@@ -92,7 +115,7 @@ static int cio_dac_probe(struct device *dev, unsigned int id)
 {
 	struct iio_dev *indio_dev;
 	struct cio_dac_iio *priv;
-	unsigned int i;
+	void __iomem *regs;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*priv));
 	if (!indio_dev)
@@ -105,21 +128,22 @@ static int cio_dac_probe(struct device *dev, unsigned int id)
 		return -EBUSY;
 	}
 
-	priv = iio_priv(indio_dev);
-	priv->base = devm_ioport_map(dev, base[id], CIO_DAC_EXTENT);
-	if (!priv->base)
+	regs = devm_ioport_map(dev, base[id], CIO_DAC_EXTENT);
+	if (!regs)
 		return -ENOMEM;
 
+	priv = iio_priv(indio_dev);
+	priv->map = devm_regmap_init_mmio(dev, regs, &cio_dac_regmap_config);
+	if (IS_ERR(priv->map))
+		return dev_err_probe(dev, PTR_ERR(priv->map),
+				     "Unable to initialize register map\n");
+
 	indio_dev->info = &cio_dac_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->channels = cio_dac_channels;
 	indio_dev->num_channels = CIO_DAC_NUM_CHAN;
 	indio_dev->name = dev_name(dev);
 
-	/* initialize DAC outputs to 0V */
-	for (i = 0; i < CIO_DAC_NUM_CHAN; i++)
-		iowrite16(0, priv->base + i);
-
 	return devm_iio_device_register(dev, indio_dev);
 }
 
diff --git a/drivers/iio/dac/max5522.c b/drivers/iio/dac/max5522.c
index 00ba4e98fb9c..05034a306597 100644
--- a/drivers/iio/dac/max5522.c
+++ b/drivers/iio/dac/max5522.c
@@ -52,7 +52,7 @@ struct max5522_state {
 	} \
 }
 
-const struct iio_chan_spec max5522_channels[] = {
+static const struct iio_chan_spec max5522_channels[] = {
 	MAX5522_CHANNEL(0),
 	MAX5522_CHANNEL(1),
 };
diff --git a/drivers/iio/frequency/admv1013.c b/drivers/iio/frequency/admv1013.c
index ed8167271358..9bf8337806fc 100644
--- a/drivers/iio/frequency/admv1013.c
+++ b/drivers/iio/frequency/admv1013.c
@@ -490,11 +490,6 @@ static int admv1013_init(struct admv1013_state *st)
 					  st->input_mode);
 }
 
-static void admv1013_clk_disable(void *data)
-{
-	clk_disable_unprepare(data);
-}
-
 static void admv1013_reg_disable(void *data)
 {
 	regulator_disable(data);
@@ -559,11 +554,6 @@ static int admv1013_properties_parse(struct admv1013_state *st)
 		return dev_err_probe(&spi->dev, PTR_ERR(st->reg),
 				     "failed to get the common-mode voltage\n");
 
-	st->clkin = devm_clk_get(&spi->dev, "lo_in");
-	if (IS_ERR(st->clkin))
-		return dev_err_probe(&spi->dev, PTR_ERR(st->clkin),
-				     "failed to get the LO input clock\n");
-
 	return 0;
 }
 
@@ -601,13 +591,10 @@ static int admv1013_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = clk_prepare_enable(st->clkin);
-	if (ret)
-		return ret;
-
-	ret = devm_add_action_or_reset(&spi->dev, admv1013_clk_disable, st->clkin);
-	if (ret)
-		return ret;
+	st->clkin = devm_clk_get_enabled(&spi->dev, "lo_in");
+	if (IS_ERR(st->clkin))
+		return dev_err_probe(&spi->dev, PTR_ERR(st->clkin),
+				     "failed to get the LO input clock\n");
 
 	st->nb.notifier_call = admv1013_freq_change;
 	ret = devm_clk_notifier_register(&spi->dev, st->clkin, &st->nb);
diff --git a/drivers/iio/gyro/fxas21002c_core.c b/drivers/iio/gyro/fxas21002c_core.c
index 3ea1d4613080..c28d17ca6f5e 100644
--- a/drivers/iio/gyro/fxas21002c_core.c
+++ b/drivers/iio/gyro/fxas21002c_core.c
@@ -813,7 +813,7 @@ static irqreturn_t fxas21002c_data_rdy_thread(int irq, void *private)
 	if (!data_ready)
 		return IRQ_NONE;
 
-	iio_trigger_poll_chained(data->dready_trig);
+	iio_trigger_poll_nested(data->dready_trig);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c
index 6a6d84a3deda..a791ba3a693a 100644
--- a/drivers/iio/gyro/mpu3050-core.c
+++ b/drivers/iio/gyro/mpu3050-core.c
@@ -939,7 +939,7 @@ static irqreturn_t mpu3050_irq_thread(int irq, void *p)
 	if (!(val & MPU3050_INT_STATUS_RAW_RDY))
 		return IRQ_NONE;
 
-	iio_trigger_poll_chained(p);
+	iio_trigger_poll_nested(p);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/iio/humidity/hts221_buffer.c b/drivers/iio/humidity/hts221_buffer.c
index 2a4107a79662..11ef38994a95 100644
--- a/drivers/iio/humidity/hts221_buffer.c
+++ b/drivers/iio/humidity/hts221_buffer.c
@@ -68,7 +68,7 @@ static irqreturn_t hts221_trigger_handler_thread(int irq, void *private)
 	if (!(status & HTS221_RH_DRDY_MASK))
 		return IRQ_NONE;
 
-	iio_trigger_poll_chained(hw->trig);
+	iio_trigger_poll_nested(hw->trig);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/iio/imu/adis16400.c b/drivers/iio/imu/adis16400.c
index c02fc35dceb4..3eda32e12a53 100644
--- a/drivers/iio/imu/adis16400.c
+++ b/drivers/iio/imu/adis16400.c
@@ -466,7 +466,7 @@ static int adis16400_initial_setup(struct iio_dev *indio_dev)
 
 		dev_info(&indio_dev->dev, "%s: prod_id 0x%04x at CS%d (irq %d)\n",
 			indio_dev->name, prod_id,
-			st->adis.spi->chip_select, st->adis.spi->irq);
+			spi_get_chipselect(st->adis.spi, 0), st->adis.spi->irq);
 	}
 	/* use high spi speed if possible */
 	if (st->variant->flags & ADIS16400_HAS_SLOW_MODE) {
diff --git a/drivers/iio/imu/adis16475.c b/drivers/iio/imu/adis16475.c
index aec55f7e1f26..3abffb01ba31 100644
--- a/drivers/iio/imu/adis16475.c
+++ b/drivers/iio/imu/adis16475.c
@@ -326,11 +326,11 @@ static int adis16475_set_freq(struct adis16475 *st, const u32 freq)
 
 		/*
 		 * This is not an hard requirement but it's not advised to run the IMU
-		 * with a sample rate lower than 4000Hz due to possible undersampling
+		 * with a sample rate lower than 1900Hz due to possible undersampling
 		 * issues. However, there are users that might really want to take the risk.
 		 * Hence, we provide a module parameter for them. If set, we allow sample
-		 * rates lower than 4KHz. By default, we won't allow this and we just roundup
-		 * the rate to the next multiple of the input clock bigger than 4KHz. This
+		 * rates lower than 1.9KHz. By default, we won't allow this and we just roundup
+		 * the rate to the next multiple of the input clock bigger than 1.9KHz. This
 		 * is done like this as in some cases (when DEC_RATE is 0) might give
 		 * us the closest value to the one desired by the user...
 		 */
diff --git a/drivers/iio/imu/st_lsm6dsx/Kconfig b/drivers/iio/imu/st_lsm6dsx/Kconfig
index 8c16cdacf2f2..5865a295a4df 100644
--- a/drivers/iio/imu/st_lsm6dsx/Kconfig
+++ b/drivers/iio/imu/st_lsm6dsx/Kconfig
@@ -14,8 +14,8 @@ config IIO_ST_LSM6DSX
 	  sensor. Supported devices: lsm6ds3, lsm6ds3h, lsm6dsl, lsm6dsm,
 	  ism330dlc, lsm6dso, lsm6dsox, asm330lhh, asm330lhhx, lsm6dsr,
 	  lsm6ds3tr-c, ism330dhcx, lsm6dsrx, lsm6ds0, lsm6dsop, lsm6dstx,
-	  lsm6dsv, lsm6dsv16x, lsm6dso16is, ism330is, lsm6dst and the
-	  accelerometer/gyroscope of lsm9ds1.
+	  lsm6dsv, lsm6dsv16x, lsm6dso16is, ism330is, asm330lhb, lsm6dst
+	  and the accelerometer/gyroscope of lsm9ds1.
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called st_lsm6dsx.
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
index 499fcf8875b4..c19237717e81 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
@@ -37,9 +37,10 @@
 #define ST_LSM6DSV16X_DEV_NAME	"lsm6dsv16x"
 #define ST_LSM6DSO16IS_DEV_NAME	"lsm6dso16is"
 #define ST_ISM330IS_DEV_NAME	"ism330is"
+#define ST_ASM330LHB_DEV_NAME	"asm330lhb"
 
 enum st_lsm6dsx_hw_id {
-	ST_LSM6DS3_ID,
+	ST_LSM6DS3_ID = 1,
 	ST_LSM6DS3H_ID,
 	ST_LSM6DSL_ID,
 	ST_LSM6DSM_ID,
@@ -61,6 +62,7 @@ enum st_lsm6dsx_hw_id {
 	ST_LSM6DSV16X_ID,
 	ST_LSM6DSO16IS_ID,
 	ST_ISM330IS_ID,
+	ST_ASM330LHB_ID,
 	ST_LSM6DSX_MAX_ID,
 };
 
@@ -137,6 +139,13 @@ struct st_lsm6dsx_odr_table_entry {
 	int odr_len;
 };
 
+struct st_lsm6dsx_samples_to_discard {
+	struct {
+		u32 milli_hz;
+		u16 samples;
+	} val[ST_LSM6DSX_ODR_LIST_SIZE];
+};
+
 struct st_lsm6dsx_fs {
 	u32 gain;
 	u8 val;
@@ -291,6 +300,7 @@ struct st_lsm6dsx_ext_dev_settings {
  * @irq_config: interrupts related registers.
  * @drdy_mask: register info for data-ready mask (addr + mask).
  * @odr_table: Hw sensors odr table (Hz + val).
+ * @samples_to_discard: Number of samples to discard for filters settling time.
  * @fs_table: Hw sensors gain table (gain + val).
  * @decimator: List of decimator register info (addr + mask).
  * @batch: List of FIFO batching register info (addr + mask).
@@ -323,6 +333,7 @@ struct st_lsm6dsx_settings {
 	} irq_config;
 	struct st_lsm6dsx_reg drdy_mask;
 	struct st_lsm6dsx_odr_table_entry odr_table[2];
+	struct st_lsm6dsx_samples_to_discard samples_to_discard[2];
 	struct st_lsm6dsx_fs_table_entry fs_table[2];
 	struct st_lsm6dsx_reg decimator[ST_LSM6DSX_MAX_ID];
 	struct st_lsm6dsx_reg batch[ST_LSM6DSX_MAX_ID];
@@ -353,6 +364,7 @@ enum st_lsm6dsx_fifo_mode {
  * @hw: Pointer to instance of struct st_lsm6dsx_hw.
  * @gain: Configured sensor sensitivity.
  * @odr: Output data rate of the sensor [Hz].
+ * @samples_to_discard: Number of samples to discard for filters settling time.
  * @watermark: Sensor watermark level.
  * @decimator: Sensor decimation factor.
  * @sip: Number of samples in a given pattern.
@@ -367,6 +379,7 @@ struct st_lsm6dsx_sensor {
 	u32 gain;
 	u32 odr;
 
+	u16 samples_to_discard;
 	u16 watermark;
 	u8 decimator;
 	u8 sip;
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
index 7dd5205aea5b..066fe561c5e8 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
@@ -15,7 +15,7 @@
  * value of the decimation factor and ODR set for each FIFO data set.
  *
  * LSM6DSO/LSM6DSOX/ASM330LHH/ASM330LHHX/LSM6DSR/LSM6DSRX/ISM330DHCX/
- * LSM6DST/LSM6DSOP/LSM6DSTX/LSM6DSV:
+ * LSM6DST/LSM6DSOP/LSM6DSTX/LSM6DSV/ASM330LHB:
  * The FIFO buffer can be configured to store data from gyroscope and
  * accelerometer. Each sample is queued with a tag (1B) indicating data
  * source (gyroscope, accelerometer, hw timer).
@@ -457,17 +457,31 @@ int st_lsm6dsx_read_fifo(struct st_lsm6dsx_hw *hw)
 			}
 
 			if (gyro_sip > 0 && !(sip % gyro_sensor->decimator)) {
-				iio_push_to_buffers_with_timestamp(
-					hw->iio_devs[ST_LSM6DSX_ID_GYRO],
-					&hw->scan[ST_LSM6DSX_ID_GYRO],
-					gyro_sensor->ts_ref + ts);
+				/*
+				 * We need to discards gyro samples during
+				 * filters settling time
+				 */
+				if (gyro_sensor->samples_to_discard > 0)
+					gyro_sensor->samples_to_discard--;
+				else
+					iio_push_to_buffers_with_timestamp(
+						hw->iio_devs[ST_LSM6DSX_ID_GYRO],
+						&hw->scan[ST_LSM6DSX_ID_GYRO],
+						gyro_sensor->ts_ref + ts);
 				gyro_sip--;
 			}
 			if (acc_sip > 0 && !(sip % acc_sensor->decimator)) {
-				iio_push_to_buffers_with_timestamp(
-					hw->iio_devs[ST_LSM6DSX_ID_ACC],
-					&hw->scan[ST_LSM6DSX_ID_ACC],
-					acc_sensor->ts_ref + ts);
+				/*
+				 * We need to discards accel samples during
+				 * filters settling time
+				 */
+				if (acc_sensor->samples_to_discard > 0)
+					acc_sensor->samples_to_discard--;
+				else
+					iio_push_to_buffers_with_timestamp(
+						hw->iio_devs[ST_LSM6DSX_ID_ACC],
+						&hw->scan[ST_LSM6DSX_ID_ACC],
+						acc_sensor->ts_ref + ts);
 				acc_sip--;
 			}
 			if (ext_sip > 0 && !(sip % ext_sensor->decimator)) {
@@ -654,6 +668,30 @@ int st_lsm6dsx_flush_fifo(struct st_lsm6dsx_hw *hw)
 	return err;
 }
 
+static void
+st_lsm6dsx_update_samples_to_discard(struct st_lsm6dsx_sensor *sensor)
+{
+	const struct st_lsm6dsx_samples_to_discard *data;
+	struct st_lsm6dsx_hw *hw = sensor->hw;
+	int i;
+
+	if (sensor->id != ST_LSM6DSX_ID_GYRO &&
+	    sensor->id != ST_LSM6DSX_ID_ACC)
+		return;
+
+	/* check if drdy mask is supported in hw */
+	if (hw->settings->drdy_mask.addr)
+		return;
+
+	data = &hw->settings->samples_to_discard[sensor->id];
+	for (i = 0; i < ST_LSM6DSX_ODR_LIST_SIZE; i++) {
+		if (data->val[i].milli_hz == sensor->odr) {
+			sensor->samples_to_discard = data->val[i].samples;
+			return;
+		}
+	}
+}
+
 int st_lsm6dsx_update_fifo(struct st_lsm6dsx_sensor *sensor, bool enable)
 {
 	struct st_lsm6dsx_hw *hw = sensor->hw;
@@ -673,6 +711,9 @@ int st_lsm6dsx_update_fifo(struct st_lsm6dsx_sensor *sensor, bool enable)
 			goto out;
 	}
 
+	if (enable)
+		st_lsm6dsx_update_samples_to_discard(sensor);
+
 	err = st_lsm6dsx_device_set_enable(sensor, enable);
 	if (err < 0)
 		goto out;
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
index 3f6060c64f32..6a18b363cf73 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
@@ -56,6 +56,7 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/acpi.h>
 #include <linux/delay.h>
 #include <linux/iio/events.h>
 #include <linux/iio/iio.h>
@@ -634,6 +635,24 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 				.fs_len = 4,
 			},
 		},
+		.samples_to_discard = {
+			[ST_LSM6DSX_ID_ACC] = {
+				.val[0] = {  12500, 1 },
+				.val[1] = {  26000, 1 },
+				.val[2] = {  52000, 1 },
+				.val[3] = { 104000, 2 },
+				.val[4] = { 208000, 2 },
+				.val[5] = { 416000, 2 },
+			},
+			[ST_LSM6DSX_ID_GYRO] = {
+				.val[0] = {  12500,  2 },
+				.val[1] = {  26000,  5 },
+				.val[2] = {  52000,  7 },
+				.val[3] = { 104000, 12 },
+				.val[4] = { 208000, 20 },
+				.val[5] = { 416000, 36 },
+			},
+		},
 		.irq_config = {
 			.irq1 = {
 				.addr = 0x0d,
@@ -1014,6 +1033,10 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 				.hw_id = ST_LSM6DSOP_ID,
 				.name = ST_LSM6DSOP_DEV_NAME,
 				.wai = 0x6c,
+			}, {
+				.hw_id = ST_ASM330LHB_ID,
+				.name = ST_ASM330LHB_DEV_NAME,
+				.wai = 0x6b,
 			},
 		},
 		.channels = {
@@ -2602,6 +2625,73 @@ static int st_lsm6dsx_init_regulators(struct device *dev)
 	return 0;
 }
 
+#ifdef CONFIG_ACPI
+
+static int lsm6dsx_get_acpi_mount_matrix(struct device *dev,
+					 struct iio_mount_matrix *orientation)
+{
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	struct acpi_device *adev = ACPI_COMPANION(dev);
+	union acpi_object *obj, *elements;
+	acpi_status status;
+	int i, j, val[3];
+	char *str;
+
+	if (!has_acpi_companion(dev))
+		return -EINVAL;
+
+	if (!acpi_has_method(adev->handle, "ROTM"))
+		return -EINVAL;
+
+	status = acpi_evaluate_object(adev->handle, "ROTM", NULL, &buffer);
+	if (ACPI_FAILURE(status)) {
+		dev_warn(dev, "Failed to get ACPI mount matrix: %d\n", status);
+		return -EINVAL;
+	}
+
+	obj = buffer.pointer;
+	if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count != 3)
+		goto unknown_format;
+
+	elements = obj->package.elements;
+	for (i = 0; i < 3; i++) {
+		if (elements[i].type != ACPI_TYPE_STRING)
+			goto unknown_format;
+
+		str = elements[i].string.pointer;
+		if (sscanf(str, "%d %d %d", &val[0], &val[1], &val[2]) != 3)
+			goto unknown_format;
+
+		for (j = 0; j < 3; j++) {
+			switch (val[j]) {
+			case -1: str = "-1"; break;
+			case 0:  str = "0";  break;
+			case 1:  str = "1";  break;
+			default: goto unknown_format;
+			}
+			orientation->rotation[i * 3 + j] = str;
+		}
+	}
+
+	kfree(buffer.pointer);
+	return 0;
+
+unknown_format:
+	dev_warn(dev, "Unknown ACPI mount matrix format, ignoring\n");
+	kfree(buffer.pointer);
+	return -EINVAL;
+}
+
+#else
+
+static int lsm6dsx_get_acpi_mount_matrix(struct device *dev,
+					  struct iio_mount_matrix *orientation)
+{
+	return false;
+}
+
+#endif
+
 int st_lsm6dsx_probe(struct device *dev, int irq, int hw_id,
 		     struct regmap *regmap)
 {
@@ -2676,9 +2766,12 @@ int st_lsm6dsx_probe(struct device *dev, int irq, int hw_id,
 			return err;
 	}
 
-	err = iio_read_mount_matrix(hw->dev, &hw->orientation);
-	if (err)
-		return err;
+	err = lsm6dsx_get_acpi_mount_matrix(hw->dev, &hw->orientation);
+	if (err) {
+		err = iio_read_mount_matrix(hw->dev, &hw->orientation);
+		if (err)
+			return err;
+	}
 
 	for (i = 0; i < ST_LSM6DSX_ID_MAX; i++) {
 		if (!hw->iio_devs[i])
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c
index df5f60925260..020717f92363 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c
@@ -23,10 +23,15 @@ static const struct regmap_config st_lsm6dsx_i2c_regmap_config = {
 
 static int st_lsm6dsx_i2c_probe(struct i2c_client *client)
 {
-	const struct i2c_device_id *id = i2c_client_get_device_id(client);
-	int hw_id = id->driver_data;
+	int hw_id;
 	struct regmap *regmap;
 
+	hw_id = (kernel_ulong_t)device_get_match_data(&client->dev);
+	if (!hw_id)
+		hw_id = i2c_client_get_device_id(client)->driver_data;
+	if (!hw_id)
+		return -EINVAL;
+
 	regmap = devm_regmap_init_i2c(client, &st_lsm6dsx_i2c_regmap_config);
 	if (IS_ERR(regmap)) {
 		dev_err(&client->dev, "Failed to register i2c regmap %ld\n", PTR_ERR(regmap));
@@ -125,10 +130,20 @@ static const struct of_device_id st_lsm6dsx_i2c_of_match[] = {
 		.compatible = "st,ism330is",
 		.data = (void *)ST_ISM330IS_ID,
 	},
+	{
+		.compatible = "st,asm330lhb",
+		.data = (void *)ST_ASM330LHB_ID,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, st_lsm6dsx_i2c_of_match);
 
+static const struct acpi_device_id st_lsm6dsx_i2c_acpi_match[] = {
+	{ "SMO8B30", ST_LSM6DS3TRC_ID, },
+	{}
+};
+MODULE_DEVICE_TABLE(acpi, st_lsm6dsx_i2c_acpi_match);
+
 static const struct i2c_device_id st_lsm6dsx_i2c_id_table[] = {
 	{ ST_LSM6DS3_DEV_NAME, ST_LSM6DS3_ID },
 	{ ST_LSM6DS3H_DEV_NAME, ST_LSM6DS3H_ID },
@@ -152,6 +167,7 @@ static const struct i2c_device_id st_lsm6dsx_i2c_id_table[] = {
 	{ ST_LSM6DSV16X_DEV_NAME, ST_LSM6DSV16X_ID },
 	{ ST_LSM6DSO16IS_DEV_NAME, ST_LSM6DSO16IS_ID },
 	{ ST_ISM330IS_DEV_NAME, ST_ISM330IS_ID },
+	{ ST_ASM330LHB_DEV_NAME, ST_ASM330LHB_ID },
 	{},
 };
 MODULE_DEVICE_TABLE(i2c, st_lsm6dsx_i2c_id_table);
@@ -161,6 +177,7 @@ static struct i2c_driver st_lsm6dsx_driver = {
 		.name = "st_lsm6dsx_i2c",
 		.pm = pm_sleep_ptr(&st_lsm6dsx_pm_ops),
 		.of_match_table = st_lsm6dsx_i2c_of_match,
+		.acpi_match_table = st_lsm6dsx_i2c_acpi_match,
 	},
 	.probe_new = st_lsm6dsx_i2c_probe,
 	.id_table = st_lsm6dsx_i2c_id_table,
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
index 974584bda875..f56c170c41a9 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
@@ -125,6 +125,10 @@ static const struct of_device_id st_lsm6dsx_spi_of_match[] = {
 		.compatible = "st,ism330is",
 		.data = (void *)ST_ISM330IS_ID,
 	},
+	{
+		.compatible = "st,asm330lhb",
+		.data = (void *)ST_ASM330LHB_ID,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, st_lsm6dsx_spi_of_match);
@@ -152,6 +156,7 @@ static const struct spi_device_id st_lsm6dsx_spi_id_table[] = {
 	{ ST_LSM6DSV16X_DEV_NAME, ST_LSM6DSV16X_ID },
 	{ ST_LSM6DSO16IS_DEV_NAME, ST_LSM6DSO16IS_ID },
 	{ ST_ISM330IS_DEV_NAME, ST_ISM330IS_ID },
+	{ ST_ASM330LHB_DEV_NAME, ST_ASM330LHB_ID },
 	{},
 };
 MODULE_DEVICE_TABLE(spi, st_lsm6dsx_spi_id_table);
diff --git a/drivers/iio/industrialio-gts-helper.c b/drivers/iio/industrialio-gts-helper.c
new file mode 100644
index 000000000000..8bb68975b259
--- /dev/null
+++ b/drivers/iio/industrialio-gts-helper.c
@@ -0,0 +1,1077 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* gain-time-scale conversion helpers for IIO light sensors
+ *
+ * Copyright (c) 2023 Matti Vaittinen <mazziesaccount@gmail.com>
+ */
+
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/export.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/overflow.h>
+#include <linux/slab.h>
+#include <linux/sort.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+#include <linux/iio/iio-gts-helper.h>
+#include <linux/iio/types.h>
+
+/**
+ * iio_gts_get_gain - Convert scale to total gain
+ *
+ * Internal helper for converting scale to total gain.
+ *
+ * @max:	Maximum linearized scale. As an example, when scale is created
+ *		in magnitude of NANOs and max scale is 64.1 - The linearized
+ *		scale is 64 100 000 000.
+ * @scale:	Linearized scale to compute the gain for.
+ *
+ * Return:	(floored) gain corresponding to the scale. -EINVAL if scale
+ *		is invalid.
+ */
+static int iio_gts_get_gain(const u64 max, const u64 scale)
+{
+	u64 full = max;
+	int tmp = 1;
+
+	if (scale > full || !scale)
+		return -EINVAL;
+
+	if (U64_MAX - full < scale) {
+		/* Risk of overflow */
+		if (full - scale < scale)
+			return 1;
+
+		full -= scale;
+		tmp++;
+	}
+
+	while (full > scale * (u64)tmp)
+		tmp++;
+
+	return tmp;
+}
+
+/**
+ * gain_get_scale_fraction - get the gain or time based on scale and known one
+ *
+ * @max:	Maximum linearized scale. As an example, when scale is created
+ *		in magnitude of NANOs and max scale is 64.1 - The linearized
+ *		scale is 64 100 000 000.
+ * @scale:	Linearized scale to compute the gain/time for.
+ * @known:	Either integration time or gain depending on which one is known
+ * @unknown:	Pointer to variable where the computed gain/time is stored
+ *
+ * Internal helper for computing unknown fraction of total gain.
+ * Compute either gain or time based on scale and either the gain or time
+ * depending on which one is known.
+ *
+ * Return:	0 on success.
+ */
+static int gain_get_scale_fraction(const u64 max, u64 scale, int known,
+				   int *unknown)
+{
+	int tot_gain;
+
+	tot_gain = iio_gts_get_gain(max, scale);
+	if (tot_gain < 0)
+		return tot_gain;
+
+	*unknown = tot_gain / known;
+
+	/* We require total gain to be exact multiple of known * unknown */
+	if (!*unknown || *unknown * known != tot_gain)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int iio_gts_delinearize(u64 lin_scale, unsigned long scaler,
+			       int *scale_whole, int *scale_nano)
+{
+	int frac;
+
+	if (scaler > NANO)
+		return -EOVERFLOW;
+
+	if (!scaler)
+		return -EINVAL;
+
+	frac = do_div(lin_scale, scaler);
+
+	*scale_whole = lin_scale;
+	*scale_nano = frac * (NANO / scaler);
+
+	return 0;
+}
+
+static int iio_gts_linearize(int scale_whole, int scale_nano,
+			     unsigned long scaler, u64 *lin_scale)
+{
+	/*
+	 * Expect scale to be (mostly) NANO or MICRO. Divide divider instead of
+	 * multiplication followed by division to avoid overflow.
+	 */
+	if (scaler > NANO || !scaler)
+		return -EINVAL;
+
+	*lin_scale = (u64)scale_whole * (u64)scaler +
+		     (u64)(scale_nano / (NANO / scaler));
+
+	return 0;
+}
+
+/**
+ * iio_gts_total_gain_to_scale - convert gain to scale
+ * @gts:	Gain time scale descriptor
+ * @total_gain:	the gain to be converted
+ * @scale_int:	Pointer to integral part of the scale (typically val1)
+ * @scale_nano:	Pointer to fractional part of the scale (nano or ppb)
+ *
+ * Convert the total gain value to scale. NOTE: This does not separate gain
+ * generated by HW-gain or integration time. It is up to caller to decide what
+ * part of the total gain is due to integration time and what due to HW-gain.
+ *
+ * Return: 0 on success. Negative errno on failure.
+ */
+int iio_gts_total_gain_to_scale(struct iio_gts *gts, int total_gain,
+				int *scale_int, int *scale_nano)
+{
+	u64 tmp;
+
+	tmp = gts->max_scale;
+
+	do_div(tmp, total_gain);
+
+	return iio_gts_delinearize(tmp, NANO, scale_int, scale_nano);
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_total_gain_to_scale, IIO_GTS_HELPER);
+
+/**
+ * iio_gts_purge_avail_scale_table - free-up the available scale tables
+ * @gts:	Gain time scale descriptor
+ *
+ * Free the space reserved by iio_gts_build_avail_scale_table().
+ */
+static void iio_gts_purge_avail_scale_table(struct iio_gts *gts)
+{
+	int i;
+
+	if (gts->per_time_avail_scale_tables) {
+		for (i = 0; i < gts->num_itime; i++)
+			kfree(gts->per_time_avail_scale_tables[i]);
+
+		kfree(gts->per_time_avail_scale_tables);
+		gts->per_time_avail_scale_tables = NULL;
+	}
+
+	kfree(gts->avail_all_scales_table);
+	gts->avail_all_scales_table = NULL;
+
+	gts->num_avail_all_scales = 0;
+}
+
+static int iio_gts_gain_cmp(const void *a, const void *b)
+{
+	return *(int *)a - *(int *)b;
+}
+
+static int gain_to_scaletables(struct iio_gts *gts, int **gains, int **scales)
+{
+	int ret, i, j, new_idx, time_idx;
+	int *all_gains;
+	size_t gain_bytes;
+
+	for (i = 0; i < gts->num_itime; i++) {
+		/*
+		 * Sort the tables for nice output and for easier finding of
+		 * unique values.
+		 */
+		sort(gains[i], gts->num_hwgain, sizeof(int), iio_gts_gain_cmp,
+		     NULL);
+
+		/* Convert gains to scales */
+		for (j = 0; j < gts->num_hwgain; j++) {
+			ret = iio_gts_total_gain_to_scale(gts, gains[i][j],
+							  &scales[i][2 * j],
+							  &scales[i][2 * j + 1]);
+			if (ret)
+				return ret;
+		}
+	}
+
+	gain_bytes = array_size(gts->num_hwgain, sizeof(int));
+	all_gains = kcalloc(gts->num_itime, gain_bytes, GFP_KERNEL);
+	if (!all_gains)
+		return -ENOMEM;
+
+	/*
+	 * We assume all the gains for same integration time were unique.
+	 * It is likely the first time table had greatest time multiplier as
+	 * the times are in the order of preference and greater times are
+	 * usually preferred. Hence we start from the last table which is likely
+	 * to have the smallest total gains.
+	 */
+	time_idx = gts->num_itime - 1;
+	memcpy(all_gains, gains[time_idx], gain_bytes);
+	new_idx = gts->num_hwgain;
+
+	while (time_idx--) {
+		for (j = 0; j < gts->num_hwgain; j++) {
+			int candidate = gains[time_idx][j];
+			int chk;
+
+			if (candidate > all_gains[new_idx - 1]) {
+				all_gains[new_idx] = candidate;
+				new_idx++;
+
+				continue;
+			}
+			for (chk = 0; chk < new_idx; chk++)
+				if (candidate <= all_gains[chk])
+					break;
+
+			if (candidate == all_gains[chk])
+				continue;
+
+			memmove(&all_gains[chk + 1], &all_gains[chk],
+				(new_idx - chk) * sizeof(int));
+			all_gains[chk] = candidate;
+			new_idx++;
+		}
+	}
+
+	gts->avail_all_scales_table = kcalloc(new_idx, 2 * sizeof(int),
+					      GFP_KERNEL);
+	if (!gts->avail_all_scales_table) {
+		ret = -ENOMEM;
+		goto free_out;
+	}
+	gts->num_avail_all_scales = new_idx;
+
+	for (i = 0; i < gts->num_avail_all_scales; i++) {
+		ret = iio_gts_total_gain_to_scale(gts, all_gains[i],
+					&gts->avail_all_scales_table[i * 2],
+					&gts->avail_all_scales_table[i * 2 + 1]);
+
+		if (ret) {
+			kfree(gts->avail_all_scales_table);
+			gts->num_avail_all_scales = 0;
+			goto free_out;
+		}
+	}
+
+free_out:
+	kfree(all_gains);
+
+	return ret;
+}
+
+/**
+ * iio_gts_build_avail_scale_table - create tables of available scales
+ * @gts:	Gain time scale descriptor
+ *
+ * Build the tables which can represent the available scales based on the
+ * originally given gain and time tables. When both time and gain tables are
+ * given this results:
+ * 1. A set of tables representing available scales for each supported
+ *    integration time.
+ * 2. A single table listing all the unique scales that any combination of
+ *    supported gains and times can provide.
+ *
+ * NOTE: Space allocated for the tables must be freed using
+ * iio_gts_purge_avail_scale_table() when the tables are no longer needed.
+ *
+ * Return: 0 on success.
+ */
+static int iio_gts_build_avail_scale_table(struct iio_gts *gts)
+{
+	int **per_time_gains, **per_time_scales, i, j, ret = -ENOMEM;
+
+	per_time_gains = kcalloc(gts->num_itime, sizeof(*per_time_gains), GFP_KERNEL);
+	if (!per_time_gains)
+		return ret;
+
+	per_time_scales = kcalloc(gts->num_itime, sizeof(*per_time_scales), GFP_KERNEL);
+	if (!per_time_scales)
+		goto free_gains;
+
+	for (i = 0; i < gts->num_itime; i++) {
+		per_time_scales[i] = kcalloc(gts->num_hwgain, 2 * sizeof(int),
+					     GFP_KERNEL);
+		if (!per_time_scales[i])
+			goto err_free_out;
+
+		per_time_gains[i] = kcalloc(gts->num_hwgain, sizeof(int),
+					    GFP_KERNEL);
+		if (!per_time_gains[i]) {
+			kfree(per_time_scales[i]);
+			goto err_free_out;
+		}
+
+		for (j = 0; j < gts->num_hwgain; j++)
+			per_time_gains[i][j] = gts->hwgain_table[j].gain *
+					       gts->itime_table[i].mul;
+	}
+
+	ret = gain_to_scaletables(gts, per_time_gains, per_time_scales);
+	if (ret)
+		goto err_free_out;
+
+	kfree(per_time_gains);
+	gts->per_time_avail_scale_tables = per_time_scales;
+
+	return 0;
+
+err_free_out:
+	for (i--; i; i--) {
+		kfree(per_time_scales[i]);
+		kfree(per_time_gains[i]);
+	}
+	kfree(per_time_scales);
+free_gains:
+	kfree(per_time_gains);
+
+	return ret;
+}
+
+/**
+ * iio_gts_build_avail_time_table - build table of available integration times
+ * @gts:	Gain time scale descriptor
+ *
+ * Build the table which can represent the available times to be returned
+ * to users using the read_avail-callback.
+ *
+ * NOTE: Space allocated for the tables must be freed using
+ * iio_gts_purge_avail_time_table() when the tables are no longer needed.
+ *
+ * Return: 0 on success.
+ */
+static int iio_gts_build_avail_time_table(struct iio_gts *gts)
+{
+	int *times, i, j, idx = 0;
+
+	if (!gts->num_itime)
+		return 0;
+
+	times = kcalloc(gts->num_itime, sizeof(int), GFP_KERNEL);
+	if (!times)
+		return -ENOMEM;
+
+	/* Sort times from all tables to one and remove duplicates */
+	for (i = gts->num_itime - 1; i >= 0; i--) {
+		int new = gts->itime_table[i].time_us;
+
+		if (times[idx] < new) {
+			times[idx++] = new;
+			continue;
+		}
+
+		for (j = 0; j <= idx; j++) {
+			if (times[j] > new) {
+				memmove(&times[j + 1], &times[j],
+					(idx - j) * sizeof(int));
+				times[j] = new;
+				idx++;
+			}
+		}
+	}
+	gts->avail_time_tables = times;
+	/*
+	 * This is just to survive a unlikely corner-case where times in the
+	 * given time table were not unique. Else we could just trust the
+	 * gts->num_itime.
+	 */
+	gts->num_avail_time_tables = idx;
+
+	return 0;
+}
+
+/**
+ * iio_gts_purge_avail_time_table - free-up the available integration time table
+ * @gts:	Gain time scale descriptor
+ *
+ * Free the space reserved by iio_gts_build_avail_time_table().
+ */
+static void iio_gts_purge_avail_time_table(struct iio_gts *gts)
+{
+	if (gts->num_avail_time_tables) {
+		kfree(gts->avail_time_tables);
+		gts->avail_time_tables = NULL;
+		gts->num_avail_time_tables = 0;
+	}
+}
+
+/**
+ * iio_gts_build_avail_tables - create tables of available scales and int times
+ * @gts:	Gain time scale descriptor
+ *
+ * Build the tables which can represent the available scales and available
+ * integration times. Availability tables are built based on the originally
+ * given gain and given time tables.
+ *
+ * When both time and gain tables are
+ * given this results:
+ * 1. A set of sorted tables representing available scales for each supported
+ *    integration time.
+ * 2. A single sorted table listing all the unique scales that any combination
+ *    of supported gains and times can provide.
+ * 3. A sorted table of supported integration times
+ *
+ * After these tables are built one can use the iio_gts_all_avail_scales(),
+ * iio_gts_avail_scales_for_time() and iio_gts_avail_times() helpers to
+ * implement the read_avail operations.
+ *
+ * NOTE: Space allocated for the tables must be freed using
+ * iio_gts_purge_avail_tables() when the tables are no longer needed.
+ *
+ * Return: 0 on success.
+ */
+static int iio_gts_build_avail_tables(struct iio_gts *gts)
+{
+	int ret;
+
+	ret = iio_gts_build_avail_scale_table(gts);
+	if (ret)
+		return ret;
+
+	ret = iio_gts_build_avail_time_table(gts);
+	if (ret)
+		iio_gts_purge_avail_scale_table(gts);
+
+	return ret;
+}
+
+/**
+ * iio_gts_purge_avail_tables - free-up the availability tables
+ * @gts:	Gain time scale descriptor
+ *
+ * Free the space reserved by iio_gts_build_avail_tables(). Frees both the
+ * integration time and scale tables.
+ */
+static void iio_gts_purge_avail_tables(struct iio_gts *gts)
+{
+	iio_gts_purge_avail_time_table(gts);
+	iio_gts_purge_avail_scale_table(gts);
+}
+
+static void devm_iio_gts_avail_all_drop(void *res)
+{
+	iio_gts_purge_avail_tables(res);
+}
+
+/**
+ * devm_iio_gts_build_avail_tables - manged add availability tables
+ * @dev:	Pointer to the device whose lifetime tables are bound
+ * @gts:	Gain time scale descriptor
+ *
+ * Build the tables which can represent the available scales and available
+ * integration times. Availability tables are built based on the originally
+ * given gain and given time tables.
+ *
+ * When both time and gain tables are given this results:
+ * 1. A set of sorted tables representing available scales for each supported
+ *    integration time.
+ * 2. A single sorted table listing all the unique scales that any combination
+ *    of supported gains and times can provide.
+ * 3. A sorted table of supported integration times
+ *
+ * After these tables are built one can use the iio_gts_all_avail_scales(),
+ * iio_gts_avail_scales_for_time() and iio_gts_avail_times() helpers to
+ * implement the read_avail operations.
+ *
+ * The tables are automatically released upon device detach.
+ *
+ * Return: 0 on success.
+ */
+static int devm_iio_gts_build_avail_tables(struct device *dev,
+					   struct iio_gts *gts)
+{
+	int ret;
+
+	ret = iio_gts_build_avail_tables(gts);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(dev, devm_iio_gts_avail_all_drop, gts);
+}
+
+static int sanity_check_time(const struct iio_itime_sel_mul *t)
+{
+	if (t->sel < 0 || t->time_us < 0 || t->mul <= 0)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int sanity_check_gain(const struct iio_gain_sel_pair *g)
+{
+	if (g->sel < 0 || g->gain <= 0)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int iio_gts_sanity_check(struct iio_gts *gts)
+{
+	int g, t, ret;
+
+	if (!gts->num_hwgain && !gts->num_itime)
+		return -EINVAL;
+
+	for (t = 0; t < gts->num_itime; t++) {
+		ret = sanity_check_time(&gts->itime_table[t]);
+		if (ret)
+			return ret;
+	}
+
+	for (g = 0; g < gts->num_hwgain; g++) {
+		ret = sanity_check_gain(&gts->hwgain_table[g]);
+		if (ret)
+			return ret;
+	}
+
+	for (g = 0; g < gts->num_hwgain; g++) {
+		for (t = 0; t < gts->num_itime; t++) {
+			int gain, mul, res;
+
+			gain = gts->hwgain_table[g].gain;
+			mul = gts->itime_table[t].mul;
+
+			if (check_mul_overflow(gain, mul, &res))
+				return -EOVERFLOW;
+		}
+	}
+
+	return 0;
+}
+
+static int iio_init_iio_gts(int max_scale_int, int max_scale_nano,
+			const struct iio_gain_sel_pair *gain_tbl, int num_gain,
+			const struct iio_itime_sel_mul *tim_tbl, int num_times,
+			struct iio_gts *gts)
+{
+	int ret;
+
+	memset(gts, 0, sizeof(*gts));
+
+	ret = iio_gts_linearize(max_scale_int, max_scale_nano, NANO,
+				   &gts->max_scale);
+	if (ret)
+		return ret;
+
+	gts->hwgain_table = gain_tbl;
+	gts->num_hwgain = num_gain;
+	gts->itime_table = tim_tbl;
+	gts->num_itime = num_times;
+
+	return iio_gts_sanity_check(gts);
+}
+
+/**
+ * devm_iio_init_iio_gts - Initialize the gain-time-scale helper
+ * @dev:		Pointer to the device whose lifetime gts resources are
+ *			bound
+ * @max_scale_int:	integer part of the maximum scale value
+ * @max_scale_nano:	fraction part of the maximum scale value
+ * @gain_tbl:		table describing supported gains
+ * @num_gain:		number of gains in the gain table
+ * @tim_tbl:		table describing supported integration times. Provide
+ *			the integration time table sorted so that the preferred
+ *			integration time is in the first array index. The search
+ *			functions like the
+ *			iio_gts_find_time_and_gain_sel_for_scale() start search
+ *			from first provided time.
+ * @num_times:		number of times in the time table
+ * @gts:		pointer to the helper struct
+ *
+ * Initialize the gain-time-scale helper for use. Note, gains, times, selectors
+ * and multipliers must be positive. Negative values are reserved for error
+ * checking. The total gain (maximum gain * maximum time multiplier) must not
+ * overflow int. The allocated resources will be released upon device detach.
+ *
+ * Return: 0 on success.
+ */
+int devm_iio_init_iio_gts(struct device *dev, int max_scale_int, int max_scale_nano,
+			  const struct iio_gain_sel_pair *gain_tbl, int num_gain,
+			  const struct iio_itime_sel_mul *tim_tbl, int num_times,
+			  struct iio_gts *gts)
+{
+	int ret;
+
+	ret = iio_init_iio_gts(max_scale_int, max_scale_nano, gain_tbl,
+			       num_gain, tim_tbl, num_times, gts);
+	if (ret)
+		return ret;
+
+	return devm_iio_gts_build_avail_tables(dev, gts);
+}
+EXPORT_SYMBOL_NS_GPL(devm_iio_init_iio_gts, IIO_GTS_HELPER);
+
+/**
+ * iio_gts_all_avail_scales - helper for listing all available scales
+ * @gts:	Gain time scale descriptor
+ * @vals:	Returned array of supported scales
+ * @type:	Type of returned scale values
+ * @length:	Amount of returned values in array
+ *
+ * Return: a value suitable to be returned from read_avail or a negative error.
+ */
+int iio_gts_all_avail_scales(struct iio_gts *gts, const int **vals, int *type,
+			     int *length)
+{
+	if (!gts->num_avail_all_scales)
+		return -EINVAL;
+
+	*vals = gts->avail_all_scales_table;
+	*type = IIO_VAL_INT_PLUS_NANO;
+	*length = gts->num_avail_all_scales * 2;
+
+	return IIO_AVAIL_LIST;
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_all_avail_scales, IIO_GTS_HELPER);
+
+/**
+ * iio_gts_avail_scales_for_time - list scales for integration time
+ * @gts:	Gain time scale descriptor
+ * @time:	Integration time for which the scales are listed
+ * @vals:	Returned array of supported scales
+ * @type:	Type of returned scale values
+ * @length:	Amount of returned values in array
+ *
+ * Drivers which do not allow scale setting to change integration time can
+ * use this helper to list only the scales which are valid for given integration
+ * time.
+ *
+ * Return: a value suitable to be returned from read_avail or a negative error.
+ */
+int iio_gts_avail_scales_for_time(struct iio_gts *gts, int time,
+				  const int **vals, int *type, int *length)
+{
+	int i;
+
+	for (i = 0; i < gts->num_itime; i++)
+		if (gts->itime_table[i].time_us == time)
+			break;
+
+	if (i == gts->num_itime)
+		return -EINVAL;
+
+	*vals = gts->per_time_avail_scale_tables[i];
+	*type = IIO_VAL_INT_PLUS_NANO;
+	*length = gts->num_hwgain * 2;
+
+	return IIO_AVAIL_LIST;
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_avail_scales_for_time, IIO_GTS_HELPER);
+
+/**
+ * iio_gts_avail_times - helper for listing available integration times
+ * @gts:	Gain time scale descriptor
+ * @vals:	Returned array of supported times
+ * @type:	Type of returned scale values
+ * @length:	Amount of returned values in array
+ *
+ * Return: a value suitable to be returned from read_avail or a negative error.
+ */
+int iio_gts_avail_times(struct iio_gts *gts,  const int **vals, int *type,
+			int *length)
+{
+	if (!gts->num_avail_time_tables)
+		return -EINVAL;
+
+	*vals = gts->avail_time_tables;
+	*type = IIO_VAL_INT;
+	*length = gts->num_avail_time_tables;
+
+	return IIO_AVAIL_LIST;
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_avail_times, IIO_GTS_HELPER);
+
+/**
+ * iio_gts_find_sel_by_gain - find selector corresponding to a HW-gain
+ * @gts:	Gain time scale descriptor
+ * @gain:	HW-gain for which matching selector is searched for
+ *
+ * Return:	a selector matching given HW-gain or -EINVAL if selector was
+ *		not found.
+ */
+int iio_gts_find_sel_by_gain(struct iio_gts *gts, int gain)
+{
+	int i;
+
+	for (i = 0; i < gts->num_hwgain; i++)
+		if (gts->hwgain_table[i].gain == gain)
+			return gts->hwgain_table[i].sel;
+
+	return -EINVAL;
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_find_sel_by_gain, IIO_GTS_HELPER);
+
+/**
+ * iio_gts_find_gain_by_sel - find HW-gain corresponding to a selector
+ * @gts:	Gain time scale descriptor
+ * @sel:	selector for which matching HW-gain is searched for
+ *
+ * Return:	a HW-gain matching given selector or -EINVAL if HW-gain was not
+ *		found.
+ */
+int iio_gts_find_gain_by_sel(struct iio_gts *gts, int sel)
+{
+	int i;
+
+	for (i = 0; i < gts->num_hwgain; i++)
+		if (gts->hwgain_table[i].sel == sel)
+			return gts->hwgain_table[i].gain;
+
+	return -EINVAL;
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_find_gain_by_sel, IIO_GTS_HELPER);
+
+/**
+ * iio_gts_get_min_gain - find smallest valid HW-gain
+ * @gts:	Gain time scale descriptor
+ *
+ * Return:	The smallest HW-gain -EINVAL if no HW-gains were in the tables.
+ */
+int iio_gts_get_min_gain(struct iio_gts *gts)
+{
+	int i, min = -EINVAL;
+
+	for (i = 0; i < gts->num_hwgain; i++) {
+		int gain = gts->hwgain_table[i].gain;
+
+		if (min == -EINVAL)
+			min = gain;
+		else
+			min = min(min, gain);
+	}
+
+	return min;
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_get_min_gain, IIO_GTS_HELPER);
+
+/**
+ * iio_find_closest_gain_low - Find the closest lower matching gain
+ * @gts:	Gain time scale descriptor
+ * @gain:	HW-gain for which the closest match is searched
+ * @in_range:	indicate if the @gain was actually in the range of
+ *		supported gains.
+ *
+ * Search for closest supported gain that is lower than or equal to the
+ * gain given as a parameter. This is usable for drivers which do not require
+ * user to request exact matching gain but rather for rounding to a supported
+ * gain value which is equal or lower (setting lower gain is typical for
+ * avoiding saturation)
+ *
+ * Return:	The closest matching supported gain or -EINVAL if @gain
+ *		was smaller than the smallest supported gain.
+ */
+int iio_find_closest_gain_low(struct iio_gts *gts, int gain, bool *in_range)
+{
+	int i, diff = 0;
+	int best = -1;
+
+	*in_range = false;
+
+	for (i = 0; i < gts->num_hwgain; i++) {
+		if (gain == gts->hwgain_table[i].gain) {
+			*in_range = true;
+			return gain;
+		}
+
+		if (gain > gts->hwgain_table[i].gain) {
+			if (!diff) {
+				diff = gain - gts->hwgain_table[i].gain;
+				best = i;
+			} else {
+				int tmp = gain - gts->hwgain_table[i].gain;
+
+				if (tmp < diff) {
+					diff = tmp;
+					best = i;
+				}
+			}
+		} else {
+			/*
+			 * We found valid HW-gain which is greater than
+			 * reference. So, unless we return a failure below we
+			 * will have found an in-range gain
+			 */
+			*in_range = true;
+		}
+	}
+	/* The requested gain was smaller than anything we support */
+	if (!diff) {
+		*in_range = false;
+
+		return -EINVAL;
+	}
+
+	return gts->hwgain_table[best].gain;
+}
+EXPORT_SYMBOL_NS_GPL(iio_find_closest_gain_low, IIO_GTS_HELPER);
+
+static int iio_gts_get_int_time_gain_multiplier_by_sel(struct iio_gts *gts,
+						       int sel)
+{
+	const struct iio_itime_sel_mul *time;
+
+	time = iio_gts_find_itime_by_sel(gts, sel);
+	if (!time)
+		return -EINVAL;
+
+	return time->mul;
+}
+
+/**
+ * iio_gts_find_gain_for_scale_using_time - Find gain by time and scale
+ * @gts:	Gain time scale descriptor
+ * @time_sel:	Integration time selector corresponding to the time gain is
+ *		searched for
+ * @scale_int:	Integral part of the scale (typically val1)
+ * @scale_nano:	Fractional part of the scale (nano or ppb)
+ * @gain:	Pointer to value where gain is stored.
+ *
+ * In some cases the light sensors may want to find a gain setting which
+ * corresponds given scale and integration time. Sensors which fill the
+ * gain and time tables may use this helper to retrieve the gain.
+ *
+ * Return:	0 on success. -EINVAL if gain matching the parameters is not
+ *		found.
+ */
+static int iio_gts_find_gain_for_scale_using_time(struct iio_gts *gts, int time_sel,
+						  int scale_int, int scale_nano,
+						  int *gain)
+{
+	u64 scale_linear;
+	int ret, mul;
+
+	ret = iio_gts_linearize(scale_int, scale_nano, NANO, &scale_linear);
+	if (ret)
+		return ret;
+
+	ret = iio_gts_get_int_time_gain_multiplier_by_sel(gts, time_sel);
+	if (ret < 0)
+		return ret;
+
+	mul = ret;
+
+	ret = gain_get_scale_fraction(gts->max_scale, scale_linear, mul, gain);
+	if (ret)
+		return ret;
+
+	if (!iio_gts_valid_gain(gts, *gain))
+		return -EINVAL;
+
+	return 0;
+}
+
+/**
+ * iio_gts_find_gain_sel_for_scale_using_time - Fetch gain selector.
+ * @gts:	Gain time scale descriptor
+ * @time_sel:	Integration time selector corresponding to the time gain is
+ *		searched for
+ * @scale_int:	Integral part of the scale (typically val1)
+ * @scale_nano:	Fractional part of the scale (nano or ppb)
+ * @gain_sel:	Pointer to value where gain selector is stored.
+ *
+ * See iio_gts_find_gain_for_scale_using_time() for more information
+ */
+int iio_gts_find_gain_sel_for_scale_using_time(struct iio_gts *gts, int time_sel,
+					       int scale_int, int scale_nano,
+					       int *gain_sel)
+{
+	int gain, ret;
+
+	ret = iio_gts_find_gain_for_scale_using_time(gts, time_sel, scale_int,
+						     scale_nano, &gain);
+	if (ret)
+		return ret;
+
+	ret = iio_gts_find_sel_by_gain(gts, gain);
+	if (ret < 0)
+		return ret;
+
+	*gain_sel = ret;
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_find_gain_sel_for_scale_using_time, IIO_GTS_HELPER);
+
+static int iio_gts_get_total_gain(struct iio_gts *gts, int gain, int time)
+{
+	const struct iio_itime_sel_mul *itime;
+
+	if (!iio_gts_valid_gain(gts, gain))
+		return -EINVAL;
+
+	if (!gts->num_itime)
+		return gain;
+
+	itime = iio_gts_find_itime_by_time(gts, time);
+	if (!itime)
+		return -EINVAL;
+
+	return gain * itime->mul;
+}
+
+static int iio_gts_get_scale_linear(struct iio_gts *gts, int gain, int time,
+				    u64 *scale)
+{
+	int total_gain;
+	u64 tmp;
+
+	total_gain = iio_gts_get_total_gain(gts, gain, time);
+	if (total_gain < 0)
+		return total_gain;
+
+	tmp = gts->max_scale;
+
+	do_div(tmp, total_gain);
+
+	*scale = tmp;
+
+	return 0;
+}
+
+/**
+ * iio_gts_get_scale - get scale based on integration time and HW-gain
+ * @gts:	Gain time scale descriptor
+ * @gain:	HW-gain for which the scale is computed
+ * @time:	Integration time for which the scale is computed
+ * @scale_int:	Integral part of the scale (typically val1)
+ * @scale_nano:	Fractional part of the scale (nano or ppb)
+ *
+ * Compute scale matching the integration time and HW-gain given as parameter.
+ *
+ * Return: 0 on success.
+ */
+int iio_gts_get_scale(struct iio_gts *gts, int gain, int time, int *scale_int,
+		      int *scale_nano)
+{
+	u64 lin_scale;
+	int ret;
+
+	ret = iio_gts_get_scale_linear(gts, gain, time, &lin_scale);
+	if (ret)
+		return ret;
+
+	return iio_gts_delinearize(lin_scale, NANO, scale_int, scale_nano);
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_get_scale, IIO_GTS_HELPER);
+
+/**
+ * iio_gts_find_new_gain_sel_by_old_gain_time - compensate for time change
+ * @gts:		Gain time scale descriptor
+ * @old_gain:		Previously set gain
+ * @old_time_sel:	Selector corresponding previously set time
+ * @new_time_sel:	Selector corresponding new time to be set
+ * @new_gain:		Pointer to value where new gain is to be written
+ *
+ * We may want to mitigate the scale change caused by setting a new integration
+ * time (for a light sensor) by also updating the (HW)gain. This helper computes
+ * new gain value to maintain the scale with new integration time.
+ *
+ * Return: 0 if an exactly matching supported new gain was found. When a
+ * non-zero value is returned, the @new_gain will be set to a negative or
+ * positive value. The negative value means that no gain could be computed.
+ * Positive value will be the "best possible new gain there could be". There
+ * can be two reasons why finding the "best possible" new gain is not deemed
+ * successful. 1) This new value cannot be supported by the hardware. 2) The new
+ * gain required to maintain the scale would not be an integer. In this case,
+ * the "best possible" new gain will be a floored optimal gain, which may or
+ * may not be supported by the hardware.
+ */
+int iio_gts_find_new_gain_sel_by_old_gain_time(struct iio_gts *gts,
+					       int old_gain, int old_time_sel,
+					       int new_time_sel, int *new_gain)
+{
+	const struct iio_itime_sel_mul *itime_old, *itime_new;
+	u64 scale;
+	int ret;
+
+	*new_gain = -1;
+
+	itime_old = iio_gts_find_itime_by_sel(gts, old_time_sel);
+	if (!itime_old)
+		return -EINVAL;
+
+	itime_new = iio_gts_find_itime_by_sel(gts, new_time_sel);
+	if (!itime_new)
+		return -EINVAL;
+
+	ret = iio_gts_get_scale_linear(gts, old_gain, itime_old->time_us,
+				       &scale);
+	if (ret)
+		return ret;
+
+	ret = gain_get_scale_fraction(gts->max_scale, scale, itime_new->mul,
+				      new_gain);
+	if (ret)
+		return ret;
+
+	if (!iio_gts_valid_gain(gts, *new_gain))
+		return -EINVAL;
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_find_new_gain_sel_by_old_gain_time, IIO_GTS_HELPER);
+
+/**
+ * iio_gts_find_new_gain_by_old_gain_time - compensate for time change
+ * @gts:		Gain time scale descriptor
+ * @old_gain:		Previously set gain
+ * @old_time:		Selector corresponding previously set time
+ * @new_time:		Selector corresponding new time to be set
+ * @new_gain:		Pointer to value where new gain is to be written
+ *
+ * We may want to mitigate the scale change caused by setting a new integration
+ * time (for a light sensor) by also updating the (HW)gain. This helper computes
+ * new gain value to maintain the scale with new integration time.
+ *
+ * Return: 0 if an exactly matching supported new gain was found. When a
+ * non-zero value is returned, the @new_gain will be set to a negative or
+ * positive value. The negative value means that no gain could be computed.
+ * Positive value will be the "best possible new gain there could be". There
+ * can be two reasons why finding the "best possible" new gain is not deemed
+ * successful. 1) This new value cannot be supported by the hardware. 2) The new
+ * gain required to maintain the scale would not be an integer. In this case,
+ * the "best possible" new gain will be a floored optimal gain, which may or
+ * may not be supported by the hardware.
+ */
+int iio_gts_find_new_gain_by_old_gain_time(struct iio_gts *gts, int old_gain,
+					   int old_time, int new_time,
+					   int *new_gain)
+{
+	const struct iio_itime_sel_mul *itime_new;
+	u64 scale;
+	int ret;
+
+	*new_gain = -1;
+
+	itime_new = iio_gts_find_itime_by_time(gts, new_time);
+	if (!itime_new)
+		return -EINVAL;
+
+	ret = iio_gts_get_scale_linear(gts, old_gain, old_time, &scale);
+	if (ret)
+		return ret;
+
+	ret = gain_get_scale_fraction(gts->max_scale, scale, itime_new->mul,
+				      new_gain);
+	if (ret)
+		return ret;
+
+	if (!iio_gts_valid_gain(gts, *new_gain))
+		return -EINVAL;
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(iio_gts_find_new_gain_by_old_gain_time, IIO_GTS_HELPER);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Matti Vaittinen <mazziesaccount@gmail.com>");
+MODULE_DESCRIPTION("IIO light sensor gain-time-scale helpers");
diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c
index a2f3cc2f65ef..784dc1e00310 100644
--- a/drivers/iio/industrialio-trigger.c
+++ b/drivers/iio/industrialio-trigger.c
@@ -192,6 +192,12 @@ static void iio_trigger_notify_done_atomic(struct iio_trigger *trig)
 		schedule_work(&trig->reenable_work);
 }
 
+/**
+ * iio_trigger_poll() - Call the IRQ trigger handler of the consumers
+ * @trig: trigger which occurred
+ *
+ * This function should only be called from a hard IRQ context.
+ */
 void iio_trigger_poll(struct iio_trigger *trig)
 {
 	int i;
@@ -216,7 +222,14 @@ irqreturn_t iio_trigger_generic_data_rdy_poll(int irq, void *private)
 }
 EXPORT_SYMBOL(iio_trigger_generic_data_rdy_poll);
 
-void iio_trigger_poll_chained(struct iio_trigger *trig)
+/**
+ * iio_trigger_poll_nested() - Call the threaded trigger handler of the
+ * consumers
+ * @trig: trigger which occurred
+ *
+ * This function should only be called from a kernel thread context.
+ */
+void iio_trigger_poll_nested(struct iio_trigger *trig)
 {
 	int i;
 
@@ -231,7 +244,7 @@ void iio_trigger_poll_chained(struct iio_trigger *trig)
 		}
 	}
 }
-EXPORT_SYMBOL(iio_trigger_poll_chained);
+EXPORT_SYMBOL(iio_trigger_poll_nested);
 
 void iio_trigger_notify_done(struct iio_trigger *trig)
 {
diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig
index 0d4447df7200..6fa31fcd71a1 100644
--- a/drivers/iio/light/Kconfig
+++ b/drivers/iio/light/Kconfig
@@ -289,6 +289,20 @@ config JSA1212
 	  To compile this driver as a module, choose M here:
 	  the module will be called jsa1212.
 
+config ROHM_BU27034
+	tristate "ROHM BU27034 ambient light sensor"
+	depends on I2C
+	select REGMAP_I2C
+	select IIO_GTS_HELPER
+	select IIO_BUFFER
+	select IIO_KFIFO_BUF
+	help
+	  Enable support for the ROHM BU27034 ambient light sensor. ROHM BU27034
+	  is an ambient light sesnor with 3 channels and 3 photo diodes capable
+	  of detecting a very wide range of illuminance.
+	  Typical application is adjusting LCD and backlight power of TVs and
+	  mobile phones.
+
 config RPR0521
 	tristate "ROHM RPR0521 ALS and proximity sensor driver"
 	depends on I2C
diff --git a/drivers/iio/light/Makefile b/drivers/iio/light/Makefile
index d74d2b5ff14c..985f6feaccd4 100644
--- a/drivers/iio/light/Makefile
+++ b/drivers/iio/light/Makefile
@@ -38,6 +38,7 @@ obj-$(CONFIG_MAX44009)		+= max44009.o
 obj-$(CONFIG_NOA1305)		+= noa1305.o
 obj-$(CONFIG_OPT3001)		+= opt3001.o
 obj-$(CONFIG_PA12203001)	+= pa12203001.o
+obj-$(CONFIG_ROHM_BU27034)	+= rohm-bu27034.o
 obj-$(CONFIG_RPR0521)		+= rpr0521.o
 obj-$(CONFIG_SI1133)		+= si1133.o
 obj-$(CONFIG_SI1145)		+= si1145.o
diff --git a/drivers/iio/light/acpi-als.c b/drivers/iio/light/acpi-als.c
index e1ff6f524f4b..2d91caf24dd0 100644
--- a/drivers/iio/light/acpi-als.c
+++ b/drivers/iio/light/acpi-als.c
@@ -108,7 +108,7 @@ static void acpi_als_notify(struct acpi_device *device, u32 event)
 	if (iio_buffer_enabled(indio_dev) && iio_trigger_using_own(indio_dev)) {
 		switch (event) {
 		case ACPI_ALS_NOTIFY_ILLUMINANCE:
-			iio_trigger_poll_chained(als->trig);
+			iio_trigger_poll_nested(als->trig);
 			break;
 		default:
 			/* Unhandled event */
diff --git a/drivers/iio/light/max44009.c b/drivers/iio/light/max44009.c
index 3dadace09fe2..176dcad6e8e8 100644
--- a/drivers/iio/light/max44009.c
+++ b/drivers/iio/light/max44009.c
@@ -527,6 +527,12 @@ static int max44009_probe(struct i2c_client *client)
 	return devm_iio_device_register(&client->dev, indio_dev);
 }
 
+static const struct of_device_id max44009_of_match[] = {
+	{ .compatible = "maxim,max44009" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, max44009_of_match);
+
 static const struct i2c_device_id max44009_id[] = {
 	{ "max44009", 0 },
 	{ }
@@ -536,18 +542,13 @@ MODULE_DEVICE_TABLE(i2c, max44009_id);
 static struct i2c_driver max44009_driver = {
 	.driver = {
 		.name = MAX44009_DRV_NAME,
+		.of_match_table = max44009_of_match,
 	},
 	.probe_new = max44009_probe,
 	.id_table = max44009_id,
 };
 module_i2c_driver(max44009_driver);
 
-static const struct of_device_id max44009_of_match[] = {
-	{ .compatible = "maxim,max44009" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, max44009_of_match);
-
 MODULE_AUTHOR("Robert Eshleman <bobbyeshleman@gmail.com>");
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("MAX44009 ambient light sensor driver");
diff --git a/drivers/iio/light/rohm-bu27034.c b/drivers/iio/light/rohm-bu27034.c
new file mode 100644
index 000000000000..e486dcf35eba
--- /dev/null
+++ b/drivers/iio/light/rohm-bu27034.c
@@ -0,0 +1,1497 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * BU27034 ROHM Ambient Light Sensor
+ *
+ * Copyright (c) 2023, ROHM Semiconductor.
+ * https://fscdn.rohm.com/en/products/databook/datasheet/ic/sensor/light/bu27034nuc-e.pdf
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/device.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/units.h>
+
+#include <linux/iio/buffer.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/iio-gts-helper.h>
+#include <linux/iio/kfifo_buf.h>
+
+#define BU27034_REG_SYSTEM_CONTROL	0x40
+#define BU27034_MASK_SW_RESET		BIT(7)
+#define BU27034_MASK_PART_ID		GENMASK(5, 0)
+#define BU27034_ID			0x19
+#define BU27034_REG_MODE_CONTROL1	0x41
+#define BU27034_MASK_MEAS_MODE		GENMASK(2, 0)
+
+#define BU27034_REG_MODE_CONTROL2	0x42
+#define BU27034_MASK_D01_GAIN		GENMASK(7, 3)
+#define BU27034_MASK_D2_GAIN_HI		GENMASK(7, 6)
+#define BU27034_MASK_D2_GAIN_LO		GENMASK(2, 0)
+
+#define BU27034_REG_MODE_CONTROL3	0x43
+#define BU27034_REG_MODE_CONTROL4	0x44
+#define BU27034_MASK_MEAS_EN		BIT(0)
+#define BU27034_MASK_VALID		BIT(7)
+#define BU27034_REG_DATA0_LO		0x50
+#define BU27034_REG_DATA1_LO		0x52
+#define BU27034_REG_DATA2_LO		0x54
+#define BU27034_REG_DATA2_HI		0x55
+#define BU27034_REG_MANUFACTURER_ID	0x92
+#define BU27034_REG_MAX BU27034_REG_MANUFACTURER_ID
+
+/*
+ * The BU27034 does not have interrupt to trigger the data read when a
+ * measurement has finished. Hence we poll the VALID bit in a thread. We will
+ * try to wake the thread BU27034_MEAS_WAIT_PREMATURE_MS milliseconds before
+ * the expected sampling time to prevent the drifting.
+ *
+ * If we constantly wake up a bit too late we would eventually skip a sample.
+ * And because the sleep can't wake up _exactly_ at given time this would be
+ * inevitable even if the sensor clock would be perfectly phase-locked to CPU
+ * clock - which we can't say is the case.
+ *
+ * This is still fragile. No matter how big advance do we have, we will still
+ * risk of losing a sample because things can in a rainy-day scenario be
+ * delayed a lot. Yet, more we reserve the time for polling, more we also lose
+ * the performance by spending cycles polling the register. So, selecting this
+ * value is a balancing dance between severity of wasting CPU time and severity
+ * of losing samples.
+ *
+ * In most cases losing the samples is not _that_ crucial because light levels
+ * tend to change slowly.
+ *
+ * Other option that was pointed to me would be always sleeping 1/2 of the
+ * measurement time, checking the VALID bit and just sleeping again if the bit
+ * was not set. That should be pretty tolerant against missing samples due to
+ * the scheduling delays while also not wasting much of cycles for polling.
+ * Downside is that the time-stamps would be very inaccurate as the wake-up
+ * would not really be tied to the sensor toggling the valid bit. This would also
+ * result 'jumps' in the time-stamps when the delay drifted so that wake-up was
+ * performed during the consecutive wake-ups (Or, when sensor and CPU clocks
+ * were very different and scheduling the wake-ups was very close to given
+ * timeout - and when the time-outs were very close to the actual sensor
+ * sampling, Eg. once in a blue moon, two consecutive time-outs would occur
+ * without having a sample ready).
+ */
+#define BU27034_MEAS_WAIT_PREMATURE_MS	5
+#define BU27034_DATA_WAIT_TIME_US	1000
+#define BU27034_TOTAL_DATA_WAIT_TIME_US (BU27034_MEAS_WAIT_PREMATURE_MS * 1000)
+
+#define BU27034_RETRY_LIMIT 18
+
+enum {
+	BU27034_CHAN_ALS,
+	BU27034_CHAN_DATA0,
+	BU27034_CHAN_DATA1,
+	BU27034_CHAN_DATA2,
+	BU27034_NUM_CHANS
+};
+
+static const unsigned long bu27034_scan_masks[] = {
+	GENMASK(BU27034_CHAN_DATA2, BU27034_CHAN_ALS), 0
+};
+
+/*
+ * Available scales with gain 1x - 4096x, timings 55, 100, 200, 400 mS
+ * Time impacts to gain: 1x, 2x, 4x, 8x.
+ *
+ * => Max total gain is HWGAIN * gain by integration time (8 * 4096) = 32768
+ *
+ * Using NANO precision for scale we must use scale 64x corresponding gain 1x
+ * to avoid precision loss. (32x would result scale 976 562.5(nanos).
+ */
+#define BU27034_SCALE_1X	64
+
+/* See the data sheet for the "Gain Setting" table */
+#define BU27034_GSEL_1X		0x00 /* 00000 */
+#define BU27034_GSEL_4X		0x08 /* 01000 */
+#define BU27034_GSEL_16X	0x0a /* 01010 */
+#define BU27034_GSEL_32X	0x0b /* 01011 */
+#define BU27034_GSEL_64X	0x0c /* 01100 */
+#define BU27034_GSEL_256X	0x18 /* 11000 */
+#define BU27034_GSEL_512X	0x19 /* 11001 */
+#define BU27034_GSEL_1024X	0x1a /* 11010 */
+#define BU27034_GSEL_2048X	0x1b /* 11011 */
+#define BU27034_GSEL_4096X	0x1c /* 11100 */
+
+/* Available gain settings */
+static const struct iio_gain_sel_pair bu27034_gains[] = {
+	GAIN_SCALE_GAIN(1, BU27034_GSEL_1X),
+	GAIN_SCALE_GAIN(4, BU27034_GSEL_4X),
+	GAIN_SCALE_GAIN(16, BU27034_GSEL_16X),
+	GAIN_SCALE_GAIN(32, BU27034_GSEL_32X),
+	GAIN_SCALE_GAIN(64, BU27034_GSEL_64X),
+	GAIN_SCALE_GAIN(256, BU27034_GSEL_256X),
+	GAIN_SCALE_GAIN(512, BU27034_GSEL_512X),
+	GAIN_SCALE_GAIN(1024, BU27034_GSEL_1024X),
+	GAIN_SCALE_GAIN(2048, BU27034_GSEL_2048X),
+	GAIN_SCALE_GAIN(4096, BU27034_GSEL_4096X),
+};
+
+/*
+ * The IC has 5 modes for sampling time. 5 mS mode is exceptional as it limits
+ * the data collection to data0-channel only and cuts the supported range to
+ * 10 bit. It is not supported by the driver.
+ *
+ * "normal" modes are 55, 100, 200 and 400 mS modes - which do have direct
+ * multiplying impact to the register values (similar to gain).
+ *
+ * This means that if meas-mode is changed for example from 400 => 200,
+ * the scale is doubled. Eg, time impact to total gain is x1, x2, x4, x8.
+ */
+#define BU27034_MEAS_MODE_100MS		0
+#define BU27034_MEAS_MODE_55MS		1
+#define BU27034_MEAS_MODE_200MS		2
+#define BU27034_MEAS_MODE_400MS		4
+
+static const struct iio_itime_sel_mul bu27034_itimes[] = {
+	GAIN_SCALE_ITIME_US(400000, BU27034_MEAS_MODE_400MS, 8),
+	GAIN_SCALE_ITIME_US(200000, BU27034_MEAS_MODE_200MS, 4),
+	GAIN_SCALE_ITIME_US(100000, BU27034_MEAS_MODE_100MS, 2),
+	GAIN_SCALE_ITIME_US(55000, BU27034_MEAS_MODE_55MS, 1),
+};
+
+#define BU27034_CHAN_DATA(_name, _ch2)					\
+{									\
+	.type = IIO_INTENSITY,						\
+	.channel = BU27034_CHAN_##_name,				\
+	.channel2 = (_ch2),						\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |			\
+			      BIT(IIO_CHAN_INFO_SCALE),			\
+	.info_mask_separate_available = BIT(IIO_CHAN_INFO_SCALE),	\
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_INT_TIME),		\
+	.info_mask_shared_by_all_available =				\
+					BIT(IIO_CHAN_INFO_INT_TIME),	\
+	.address = BU27034_REG_##_name##_LO,				\
+	.scan_index = BU27034_CHAN_##_name,				\
+	.scan_type = {							\
+		.sign = 'u',						\
+		.realbits = 16,						\
+		.storagebits = 16,					\
+		.endianness = IIO_LE,					\
+	},								\
+	.indexed = 1,							\
+}
+
+static const struct iio_chan_spec bu27034_channels[] = {
+	{
+		.type = IIO_LIGHT,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.channel = BU27034_CHAN_ALS,
+		.scan_index = BU27034_CHAN_ALS,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 32,
+			.storagebits = 32,
+			.endianness = IIO_CPU,
+		},
+	},
+	/*
+	 * The BU27034 DATA0 and DATA1 channels are both on the visible light
+	 * area (mostly). The data0 sensitivity peaks at 500nm, DATA1 at 600nm.
+	 * These wave lengths are pretty much on the border of colours making
+	 * these a poor candidates for R/G/B standardization. Hence they're both
+	 * marked as clear channels
+	 */
+	BU27034_CHAN_DATA(DATA0, IIO_MOD_LIGHT_CLEAR),
+	BU27034_CHAN_DATA(DATA1, IIO_MOD_LIGHT_CLEAR),
+	BU27034_CHAN_DATA(DATA2, IIO_MOD_LIGHT_IR),
+	IIO_CHAN_SOFT_TIMESTAMP(4),
+};
+
+struct bu27034_data {
+	struct regmap *regmap;
+	struct device *dev;
+	/*
+	 * Protect gain and time during scale adjustment and data reading.
+	 * Protect measurement enabling/disabling.
+	 */
+	struct mutex mutex;
+	struct iio_gts gts;
+	struct task_struct *task;
+	__le16 raw[3];
+	struct {
+		u32 mlux;
+		__le16 channels[3];
+		s64 ts __aligned(8);
+	} scan;
+};
+
+struct bu27034_result {
+	u16 ch0;
+	u16 ch1;
+	u16 ch2;
+};
+
+static const struct regmap_range bu27034_volatile_ranges[] = {
+	{
+		.range_min = BU27034_REG_MODE_CONTROL4,
+		.range_max = BU27034_REG_MODE_CONTROL4,
+	}, {
+		.range_min = BU27034_REG_DATA0_LO,
+		.range_max = BU27034_REG_DATA2_HI,
+	},
+};
+
+static const struct regmap_access_table bu27034_volatile_regs = {
+	.yes_ranges = &bu27034_volatile_ranges[0],
+	.n_yes_ranges = ARRAY_SIZE(bu27034_volatile_ranges),
+};
+
+static const struct regmap_range bu27034_read_only_ranges[] = {
+	{
+		.range_min = BU27034_REG_DATA0_LO,
+		.range_max = BU27034_REG_DATA2_HI,
+	}, {
+		.range_min = BU27034_REG_MANUFACTURER_ID,
+		.range_max = BU27034_REG_MANUFACTURER_ID,
+	}
+};
+
+static const struct regmap_access_table bu27034_ro_regs = {
+	.no_ranges = &bu27034_read_only_ranges[0],
+	.n_no_ranges = ARRAY_SIZE(bu27034_read_only_ranges),
+};
+
+static const struct regmap_config bu27034_regmap = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = BU27034_REG_MAX,
+	.cache_type = REGCACHE_RBTREE,
+	.volatile_table = &bu27034_volatile_regs,
+	.wr_table = &bu27034_ro_regs,
+};
+
+struct bu27034_gain_check {
+	int old_gain;
+	int new_gain;
+	int chan;
+};
+
+static int bu27034_get_gain_sel(struct bu27034_data *data, int chan)
+{
+	int ret, val;
+
+	switch (chan) {
+	case BU27034_CHAN_DATA0:
+	case BU27034_CHAN_DATA1:
+	{
+		int reg[] = {
+			[BU27034_CHAN_DATA0] = BU27034_REG_MODE_CONTROL2,
+			[BU27034_CHAN_DATA1] = BU27034_REG_MODE_CONTROL3,
+		};
+		ret = regmap_read(data->regmap, reg[chan], &val);
+		if (ret)
+			return ret;
+
+		return FIELD_GET(BU27034_MASK_D01_GAIN, val);
+	}
+	case BU27034_CHAN_DATA2:
+	{
+		int d2_lo_bits = fls(BU27034_MASK_D2_GAIN_LO);
+
+		ret = regmap_read(data->regmap, BU27034_REG_MODE_CONTROL2, &val);
+		if (ret)
+			return ret;
+
+		/*
+		 * The data2 channel gain is composed by 5 non continuous bits
+		 * [7:6], [2:0]. Thus when we combine the 5-bit 'selector'
+		 * from register value we must right shift the high bits by 3.
+		 */
+		return FIELD_GET(BU27034_MASK_D2_GAIN_HI, val) << d2_lo_bits |
+		       FIELD_GET(BU27034_MASK_D2_GAIN_LO, val);
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int bu27034_get_gain(struct bu27034_data *data, int chan, int *gain)
+{
+	int ret, sel;
+
+	ret = bu27034_get_gain_sel(data, chan);
+	if (ret < 0)
+		return ret;
+
+	sel = ret;
+
+	ret = iio_gts_find_gain_by_sel(&data->gts, sel);
+	if (ret < 0) {
+		dev_err(data->dev, "chan %u: unknown gain value 0x%x\n", chan,
+			sel);
+
+		return ret;
+	}
+
+	*gain = ret;
+
+	return 0;
+}
+
+static int bu27034_get_int_time(struct bu27034_data *data)
+{
+	int ret, sel;
+
+	ret = regmap_read(data->regmap, BU27034_REG_MODE_CONTROL1, &sel);
+	if (ret)
+		return ret;
+
+	return iio_gts_find_int_time_by_sel(&data->gts,
+					    sel & BU27034_MASK_MEAS_MODE);
+}
+
+static int _bu27034_get_scale(struct bu27034_data *data, int channel, int *val,
+			      int *val2)
+{
+	int gain, ret;
+
+	ret = bu27034_get_gain(data, channel, &gain);
+	if (ret)
+		return ret;
+
+	ret = bu27034_get_int_time(data);
+	if (ret < 0)
+		return ret;
+
+	return iio_gts_get_scale(&data->gts, gain, ret, val, val2);
+}
+
+static int bu27034_get_scale(struct bu27034_data *data, int channel, int *val,
+			      int *val2)
+{
+	int ret;
+
+	if (channel == BU27034_CHAN_ALS) {
+		*val = 0;
+		*val2 = 1000;
+		return IIO_VAL_INT_PLUS_MICRO;
+	}
+
+	mutex_lock(&data->mutex);
+	ret = _bu27034_get_scale(data, channel, val, val2);
+	mutex_unlock(&data->mutex);
+	if (ret)
+		return ret;
+
+	return IIO_VAL_INT_PLUS_NANO;
+}
+
+/* Caller should hold the lock to protect lux reading */
+static int bu27034_write_gain_sel(struct bu27034_data *data, int chan, int sel)
+{
+	static const int reg[] = {
+		[BU27034_CHAN_DATA0] = BU27034_REG_MODE_CONTROL2,
+		[BU27034_CHAN_DATA1] = BU27034_REG_MODE_CONTROL3,
+	};
+	int mask, val;
+
+	if (chan != BU27034_CHAN_DATA0 && chan != BU27034_CHAN_DATA1)
+		return -EINVAL;
+
+	val = FIELD_PREP(BU27034_MASK_D01_GAIN, sel);
+
+	mask = BU27034_MASK_D01_GAIN;
+
+	if (chan == BU27034_CHAN_DATA0) {
+		/*
+		 * We keep the same gain for channel 2 as we set for channel 0
+		 * We can't allow them to be individually controlled because
+		 * setting one will impact also the other. Also, if we don't
+		 * always update both gains we may result unsupported bit
+		 * combinations.
+		 *
+		 * This is not nice but this is yet another place where the
+		 * user space must be prepared to surprizes. Namely, see chan 2
+		 * gain changed when chan 0 gain is changed.
+		 *
+		 * This is not fatal for most users though. I don't expect the
+		 * channel 2 to be used in any generic cases - the intensity
+		 * values provided by the sensor for IR area are not openly
+		 * documented. Also, channel 2 is not used for visible light.
+		 *
+		 * So, if there is application which is written to utilize the
+		 * channel 2 - then it is probably specifically targeted to this
+		 * sensor and knows how to utilize those values. It is safe to
+		 * hope such user can also cope with the gain changes.
+		 */
+		mask |=  BU27034_MASK_D2_GAIN_LO;
+
+		/*
+		 * The D2 gain bits are directly the lowest bits of selector.
+		 * Just do add those bits to the value
+		 */
+		val |= sel & BU27034_MASK_D2_GAIN_LO;
+	}
+
+	return regmap_update_bits(data->regmap, reg[chan], mask, val);
+}
+
+static int bu27034_set_gain(struct bu27034_data *data, int chan, int gain)
+{
+	int ret;
+
+	/*
+	 * We don't allow setting channel 2 gain as it messes up the
+	 * gain for channel 0 - which shares the high bits
+	 */
+	if (chan != BU27034_CHAN_DATA0 && chan != BU27034_CHAN_DATA1)
+		return -EINVAL;
+
+	ret = iio_gts_find_sel_by_gain(&data->gts, gain);
+	if (ret < 0)
+		return ret;
+
+	return bu27034_write_gain_sel(data, chan, ret);
+}
+
+/* Caller should hold the lock to protect data->int_time */
+static int bu27034_set_int_time(struct bu27034_data *data, int time)
+{
+	int ret;
+
+	ret = iio_gts_find_sel_by_int_time(&data->gts, time);
+	if (ret < 0)
+		return ret;
+
+	return regmap_update_bits(data->regmap, BU27034_REG_MODE_CONTROL1,
+				 BU27034_MASK_MEAS_MODE, ret);
+}
+
+/*
+ * We try to change the time in such way that the scale is maintained for
+ * given channels by adjusting gain so that it compensates the time change.
+ */
+static int bu27034_try_set_int_time(struct bu27034_data *data, int time_us)
+{
+	struct bu27034_gain_check gains[] = {
+		{ .chan = BU27034_CHAN_DATA0 },
+		{ .chan = BU27034_CHAN_DATA1 },
+	};
+	int numg = ARRAY_SIZE(gains);
+	int ret, int_time_old, i;
+
+	mutex_lock(&data->mutex);
+	ret = bu27034_get_int_time(data);
+	if (ret < 0)
+		goto unlock_out;
+
+	int_time_old = ret;
+
+	if (!iio_gts_valid_time(&data->gts, time_us)) {
+		dev_err(data->dev, "Unsupported integration time %u\n",
+			time_us);
+		ret = -EINVAL;
+
+		goto unlock_out;
+	}
+
+	if (time_us == int_time_old) {
+		ret = 0;
+		goto unlock_out;
+	}
+
+	for (i = 0; i < numg; i++) {
+		ret = bu27034_get_gain(data, gains[i].chan, &gains[i].old_gain);
+		if (ret)
+			goto unlock_out;
+
+		ret = iio_gts_find_new_gain_by_old_gain_time(&data->gts,
+							     gains[i].old_gain,
+							     int_time_old, time_us,
+							     &gains[i].new_gain);
+		if (ret) {
+			int scale1, scale2;
+			bool ok;
+
+			_bu27034_get_scale(data, gains[i].chan, &scale1, &scale2);
+			dev_dbg(data->dev,
+				"chan %u, can't support time %u with scale %u %u\n",
+				gains[i].chan, time_us, scale1, scale2);
+
+			if (gains[i].new_gain < 0)
+				goto unlock_out;
+
+			/*
+			 * If caller requests for integration time change and we
+			 * can't support the scale - then the caller should be
+			 * prepared to 'pick up the pieces and deal with the
+			 * fact that the scale changed'.
+			 */
+			ret = iio_find_closest_gain_low(&data->gts,
+							gains[i].new_gain, &ok);
+
+			if (!ok)
+				dev_dbg(data->dev,
+					"optimal gain out of range for chan %u\n",
+					gains[i].chan);
+
+			if (ret < 0) {
+				dev_dbg(data->dev,
+					 "Total gain increase. Risk of saturation");
+				ret = iio_gts_get_min_gain(&data->gts);
+				if (ret < 0)
+					goto unlock_out;
+			}
+			dev_dbg(data->dev, "chan %u scale changed\n",
+				 gains[i].chan);
+			gains[i].new_gain = ret;
+			dev_dbg(data->dev, "chan %u new gain %u\n",
+				gains[i].chan, gains[i].new_gain);
+		}
+	}
+
+	for (i = 0; i < numg; i++) {
+		ret = bu27034_set_gain(data, gains[i].chan, gains[i].new_gain);
+		if (ret)
+			goto unlock_out;
+	}
+
+	ret = bu27034_set_int_time(data, time_us);
+
+unlock_out:
+	mutex_unlock(&data->mutex);
+
+	return ret;
+}
+
+static int bu27034_set_scale(struct bu27034_data *data, int chan,
+			    int val, int val2)
+{
+	int ret, time_sel, gain_sel, i;
+	bool found = false;
+
+	if (chan == BU27034_CHAN_DATA2)
+		return -EINVAL;
+
+	if (chan == BU27034_CHAN_ALS) {
+		if (val == 0 && val2 == 1000)
+			return 0;
+
+		return -EINVAL;
+	}
+
+	mutex_lock(&data->mutex);
+	ret = regmap_read(data->regmap, BU27034_REG_MODE_CONTROL1, &time_sel);
+	if (ret)
+		goto unlock_out;
+
+	ret = iio_gts_find_gain_sel_for_scale_using_time(&data->gts, time_sel,
+						val, val2 * 1000, &gain_sel);
+	if (ret) {
+		/*
+		 * Could not support scale with given time. Need to change time.
+		 * We still want to maintain the scale for all channels
+		 */
+		struct bu27034_gain_check gain;
+		int new_time_sel;
+
+		/*
+		 * Populate information for the other channel which should also
+		 * maintain the scale. (Due to the HW limitations the chan2
+		 * gets the same gain as chan0, so we only need to explicitly
+		 * set the chan 0 and 1).
+		 */
+		if (chan == BU27034_CHAN_DATA0)
+			gain.chan = BU27034_CHAN_DATA1;
+		else if (chan == BU27034_CHAN_DATA1)
+			gain.chan = BU27034_CHAN_DATA0;
+
+		ret = bu27034_get_gain(data, gain.chan, &gain.old_gain);
+		if (ret)
+			goto unlock_out;
+
+		/*
+		 * Iterate through all the times to see if we find one which
+		 * can support requested scale for requested channel, while
+		 * maintaining the scale for other channels
+		 */
+		for (i = 0; i < data->gts.num_itime; i++) {
+			new_time_sel = data->gts.itime_table[i].sel;
+
+			if (new_time_sel == time_sel)
+				continue;
+
+			/* Can we provide requested scale with this time? */
+			ret = iio_gts_find_gain_sel_for_scale_using_time(
+				&data->gts, new_time_sel, val, val2 * 1000,
+				&gain_sel);
+			if (ret)
+				continue;
+
+			/* Can the other channel(s) maintain scale? */
+			ret = iio_gts_find_new_gain_sel_by_old_gain_time(
+				&data->gts, gain.old_gain, time_sel,
+				new_time_sel, &gain.new_gain);
+			if (!ret) {
+				/* Yes - we found suitable time */
+				found = true;
+				break;
+			}
+		}
+		if (!found) {
+			dev_dbg(data->dev,
+				"Can't set scale maintaining other channels\n");
+			ret = -EINVAL;
+
+			goto unlock_out;
+		}
+
+		ret = bu27034_set_gain(data, gain.chan, gain.new_gain);
+		if (ret)
+			goto unlock_out;
+
+		ret = regmap_update_bits(data->regmap, BU27034_REG_MODE_CONTROL1,
+				  BU27034_MASK_MEAS_MODE, new_time_sel);
+		if (ret)
+			goto unlock_out;
+	}
+
+	ret = bu27034_write_gain_sel(data, chan, gain_sel);
+unlock_out:
+	mutex_unlock(&data->mutex);
+
+	return ret;
+}
+
+/*
+ * for (D1/D0 < 0.87):
+ * lx = 0.004521097 * D1 - 0.002663996 * D0 +
+ *	0.00012213 * D1 * D1 / D0
+ *
+ * =>	115.7400832 * ch1 / gain1 / mt -
+ *	68.1982976 * ch0 / gain0 / mt +
+ *	0.00012213 * 25600 * (ch1 / gain1 / mt) * 25600 *
+ *	(ch1 /gain1 / mt) / (25600 * ch0 / gain0 / mt)
+ *
+ * A =	0.00012213 * 25600 * (ch1 /gain1 / mt) * 25600 *
+ *	(ch1 /gain1 / mt) / (25600 * ch0 / gain0 / mt)
+ * =>	0.00012213 * 25600 * (ch1 /gain1 / mt) *
+ *	(ch1 /gain1 / mt) / (ch0 / gain0 / mt)
+ * =>	0.00012213 * 25600 * (ch1 / gain1) * (ch1 /gain1 / mt) /
+ *	(ch0 / gain0)
+ * =>	0.00012213 * 25600 * (ch1 / gain1) * (ch1 /gain1 / mt) *
+ *	gain0 / ch0
+ * =>	3.126528 * ch1 * ch1 * gain0 / gain1 / gain1 / mt /ch0
+ *
+ * lx = (115.7400832 * ch1 / gain1 - 68.1982976 * ch0 / gain0) /
+ *	mt + A
+ * =>	(115.7400832 * ch1 / gain1 - 68.1982976 * ch0 / gain0) /
+ *	mt + 3.126528 * ch1 * ch1 * gain0 / gain1 / gain1 / mt /
+ *	ch0
+ *
+ * =>	(115.7400832 * ch1 / gain1 - 68.1982976 * ch0 / gain0 +
+ *	  3.126528 * ch1 * ch1 * gain0 / gain1 / gain1 / ch0) /
+ *	  mt
+ *
+ * For (0.87 <= D1/D0 < 1.00)
+ * lx = (0.001331* D0 + 0.0000354 * D1) * ((D1/D0 – 0.87) * (0.385) + 1)
+ * =>	(0.001331 * 256 * 100 * ch0 / gain0 / mt + 0.0000354 * 256 *
+ *	100 * ch1 / gain1 / mt) * ((D1/D0 -  0.87) * (0.385) + 1)
+ * =>	(34.0736 * ch0 / gain0 / mt + 0.90624 * ch1 / gain1 / mt) *
+ *	((D1/D0 -  0.87) * (0.385) + 1)
+ * =>	(34.0736 * ch0 / gain0 / mt + 0.90624 * ch1 / gain1 / mt) *
+ *	(0.385 * D1/D0 - 0.66505)
+ * =>	(34.0736 * ch0 / gain0 / mt + 0.90624 * ch1 / gain1 / mt) *
+ *	(0.385 * 256 * 100 * ch1 / gain1 / mt / (256 * 100 * ch0 / gain0 / mt) - 0.66505)
+ * =>	(34.0736 * ch0 / gain0 / mt + 0.90624 * ch1 / gain1 / mt) *
+ *	(9856 * ch1 / gain1 / mt / (25600 * ch0 / gain0 / mt) + 0.66505)
+ * =>	13.118336 * ch1 / (gain1 * mt)
+ *	+ 22.66064768 * ch0 / (gain0 * mt)
+ *	+ 8931.90144 * ch1 * ch1 * gain0 /
+ *	  (25600 * ch0 * gain1 * gain1 * mt)
+ *	+ 0.602694912 * ch1 / (gain1 * mt)
+ *
+ * =>	[0.3489024 * ch1 * ch1 * gain0 / (ch0 * gain1 * gain1)
+ *	 + 22.66064768 * ch0 / gain0
+ *	 + 13.721030912 * ch1 / gain1
+ *	] / mt
+ *
+ * For (D1/D0 >= 1.00)
+ *
+ * lx	= (0.001331* D0 + 0.0000354 * D1) * ((D1/D0 – 2.0) * (-0.05) + 1)
+ *	=> (0.001331* D0 + 0.0000354 * D1) * (-0.05D1/D0 + 1.1)
+ *	=> (0.001331 * 256 * 100 * ch0 / gain0 / mt + 0.0000354 * 256 *
+ *	   100 * ch1 / gain1 / mt) * (-0.05D1/D0 + 1.1)
+ *	=> (34.0736 * ch0 / gain0 / mt + 0.90624 * ch1 / gain1 / mt) *
+ *	   (-0.05 * 256 * 100 * ch1 / gain1 / mt / (256 * 100 * ch0 / gain0 / mt) + 1.1)
+ *	=> (34.0736 * ch0 / gain0 / mt + 0.90624 * ch1 / gain1 / mt) *
+ *	   (-1280 * ch1 / (gain1 * mt * 25600 * ch0 / gain0 / mt) + 1.1)
+ *	=> (34.0736 * ch0 * -1280 * ch1 * gain0 * mt /( gain0 * mt * gain1 * mt * 25600 * ch0)
+ *	    + 34.0736 * 1.1 * ch0 / (gain0 * mt)
+ *	    + 0.90624 * ch1 * -1280 * ch1 *gain0 * mt / (gain1 * mt *gain1 * mt * 25600 * ch0)
+ *	    + 1.1 * 0.90624 * ch1 / (gain1 * mt)
+ *	=> -43614.208 * ch1 / (gain1 * mt * 25600)
+ *	    + 37.48096  ch0 / (gain0 * mt)
+ *	    - 1159.9872 * ch1 * ch1 * gain0 / (gain1 * gain1 * mt * 25600 * ch0)
+ *	    + 0.996864 ch1 / (gain1 * mt)
+ *	=> [
+ *		- 0.045312 * ch1 * ch1 * gain0 / (gain1 * gain1 * ch0)
+ *		- 0.706816 * ch1 / gain1
+ *		+ 37.48096  ch0 /gain0
+ *	   ] * mt
+ *
+ *
+ * So, the first case (D1/D0 < 0.87) can be computed to a form:
+ *
+ * lx = (3.126528 * ch1 * ch1 * gain0 / (ch0 * gain1 * gain1) +
+ *	 115.7400832 * ch1 / gain1 +
+ *	-68.1982976 * ch0 / gain0
+ *	 / mt
+ *
+ * Second case (0.87 <= D1/D0 < 1.00) goes to form:
+ *
+ *	=> [0.3489024 * ch1 * ch1 * gain0 / (ch0 * gain1 * gain1) +
+ *	    13.721030912 * ch1 / gain1 +
+ *	    22.66064768 * ch0 / gain0
+ *	   ] / mt
+ *
+ * Third case (D1/D0 >= 1.00) goes to form:
+ *	=> [-0.045312 * ch1 * ch1 * gain0 / (ch0 * gain1 * gain1) +
+ *	    -0.706816 * ch1 / gain1 +
+ *	    37.48096  ch0 /(gain0
+ *	   ] / mt
+ *
+ * This can be unified to format:
+ * lx = [
+ *	 A * ch1 * ch1 * gain0 / (ch0 * gain1 * gain1) +
+ *	 B * ch1 / gain1 +
+ *	 C * ch0 / gain0
+ *	] / mt
+ *
+ * For case 1:
+ * A = 3.126528,
+ * B = 115.7400832
+ * C = -68.1982976
+ *
+ * For case 2:
+ * A = 0.3489024
+ * B = 13.721030912
+ * C = 22.66064768
+ *
+ * For case 3:
+ * A = -0.045312
+ * B = -0.706816
+ * C = 37.48096
+ */
+
+struct bu27034_lx_coeff {
+	unsigned int A;
+	unsigned int B;
+	unsigned int C;
+	/* Indicate which of the coefficients above are negative */
+	bool is_neg[3];
+};
+
+static inline u64 gain_mul_div_helper(u64 val, unsigned int gain,
+				      unsigned int div)
+{
+	/*
+	 * Max gain for a channel is 4096. The max u64 (0xffffffffffffffffULL)
+	 * divided by 4096 is 0xFFFFFFFFFFFFF (GENMASK_ULL(51, 0)) (floored).
+	 * Thus, the 0xFFFFFFFFFFFFF is the largest value we can safely multiply
+	 * with the gain, no matter what gain is set.
+	 *
+	 * So, multiplication with max gain may overflow if val is greater than
+	 * 0xFFFFFFFFFFFFF (52 bits set)..
+	 *
+	 * If this is the case we divide first.
+	 */
+	if (val < GENMASK_ULL(51, 0)) {
+		val *= gain;
+		do_div(val, div);
+	} else {
+		do_div(val, div);
+		val *= gain;
+	}
+
+	return val;
+}
+
+static u64 bu27034_fixp_calc_t1_64bit(unsigned int coeff, unsigned int ch0,
+				      unsigned int ch1, unsigned int gain0,
+				      unsigned int gain1)
+{
+	unsigned int helper;
+	u64 helper64;
+
+	helper64 = (u64)coeff * (u64)ch1 * (u64)ch1;
+
+	helper = gain1 * gain1;
+	if (helper > ch0) {
+		do_div(helper64, helper);
+
+		return gain_mul_div_helper(helper64, gain0, ch0);
+	}
+
+	do_div(helper64, ch0);
+
+	return gain_mul_div_helper(helper64, gain0, helper);
+
+}
+
+static u64 bu27034_fixp_calc_t1(unsigned int coeff, unsigned int ch0,
+				unsigned int ch1, unsigned int gain0,
+				unsigned int gain1)
+{
+	unsigned int helper, tmp;
+
+	/*
+	 * Here we could overflow even the 64bit value. Hence we
+	 * multiply with gain0 only after the divisions - even though
+	 * it may result loss of accuracy
+	 */
+	helper = coeff * ch1 * ch1;
+	tmp = helper * gain0;
+
+	helper = ch1 * ch1;
+
+	if (check_mul_overflow(helper, coeff, &helper))
+		return bu27034_fixp_calc_t1_64bit(coeff, ch0, ch1, gain0, gain1);
+
+	if (check_mul_overflow(helper, gain0, &tmp))
+		return bu27034_fixp_calc_t1_64bit(coeff, ch0, ch1, gain0, gain1);
+
+	return tmp / (gain1 * gain1) / ch0;
+
+}
+
+static u64 bu27034_fixp_calc_t23(unsigned int coeff, unsigned int ch,
+				 unsigned int gain)
+{
+	unsigned int helper;
+	u64 helper64;
+
+	if (!check_mul_overflow(coeff, ch, &helper))
+		return helper / gain;
+
+	helper64 = (u64)coeff * (u64)ch;
+	do_div(helper64, gain);
+
+	return helper64;
+}
+
+static int bu27034_fixp_calc_lx(unsigned int ch0, unsigned int ch1,
+				unsigned int gain0, unsigned int gain1,
+				unsigned int meastime, int coeff_idx)
+{
+	static const struct bu27034_lx_coeff coeff[] = {
+		{
+			.A = 31265280,		/* 3.126528 */
+			.B = 1157400832,	/*115.7400832 */
+			.C = 681982976,		/* -68.1982976 */
+			.is_neg = {false, false, true},
+		}, {
+			.A = 3489024,		/* 0.3489024 */
+			.B = 137210309,		/* 13.721030912 */
+			.C = 226606476,		/* 22.66064768 */
+			/* All terms positive */
+		}, {
+			.A = 453120,		/* -0.045312 */
+			.B = 7068160,		/* -0.706816 */
+			.C = 374809600,		/* 37.48096 */
+			.is_neg = {true, true, false},
+		}
+	};
+	const struct bu27034_lx_coeff *c = &coeff[coeff_idx];
+	u64 res = 0, terms[3];
+	int i;
+
+	if (coeff_idx >= ARRAY_SIZE(coeff))
+		return -EINVAL;
+
+	terms[0] = bu27034_fixp_calc_t1(c->A, ch0, ch1, gain0, gain1);
+	terms[1] = bu27034_fixp_calc_t23(c->B, ch1, gain1);
+	terms[2] = bu27034_fixp_calc_t23(c->C, ch0, gain0);
+
+	/* First, add positive terms */
+	for (i = 0; i < 3; i++)
+		if (!c->is_neg[i])
+			res += terms[i];
+
+	/* No positive term => zero lux */
+	if (!res)
+		return 0;
+
+	/* Then, subtract negative terms (if any) */
+	for (i = 0; i < 3; i++)
+		if (c->is_neg[i]) {
+			/*
+			 * If the negative term is greater than positive - then
+			 * the darkness has taken over and we are all doomed! Eh,
+			 * I mean, then we can just return 0 lx and go out
+			 */
+			if (terms[i] >= res)
+				return 0;
+
+			res -= terms[i];
+		}
+
+	meastime *= 10;
+	do_div(res, meastime);
+
+	return (int) res;
+}
+
+static bool bu27034_has_valid_sample(struct bu27034_data *data)
+{
+	int ret, val;
+
+	ret = regmap_read(data->regmap, BU27034_REG_MODE_CONTROL4, &val);
+	if (ret) {
+		dev_err(data->dev, "Read failed %d\n", ret);
+
+		return false;
+	}
+
+	return val & BU27034_MASK_VALID;
+}
+
+/*
+ * Reading the register where VALID bit is clears this bit. (So does changing
+ * any gain / integration time configuration registers) The bit gets
+ * set when we have acquired new data. We use this bit to indicate data
+ * validity.
+ */
+static void bu27034_invalidate_read_data(struct bu27034_data *data)
+{
+	bu27034_has_valid_sample(data);
+}
+
+static int bu27034_read_result(struct bu27034_data *data, int chan, int *res)
+{
+	int reg[] = {
+		[BU27034_CHAN_DATA0] = BU27034_REG_DATA0_LO,
+		[BU27034_CHAN_DATA1] = BU27034_REG_DATA1_LO,
+		[BU27034_CHAN_DATA2] = BU27034_REG_DATA2_LO,
+	};
+	int valid, ret;
+	__le16 val;
+
+	ret = regmap_read_poll_timeout(data->regmap, BU27034_REG_MODE_CONTROL4,
+				       valid, (valid & BU27034_MASK_VALID),
+				       BU27034_DATA_WAIT_TIME_US, 0);
+	if (ret)
+		return ret;
+
+	ret = regmap_bulk_read(data->regmap, reg[chan], &val, sizeof(val));
+	if (ret)
+		return ret;
+
+	*res = le16_to_cpu(val);
+
+	return 0;
+}
+
+static int bu27034_get_result_unlocked(struct bu27034_data *data, __le16 *res,
+				       int size)
+{
+	int ret = 0, retry_cnt = 0;
+
+retry:
+	/* Get new value from sensor if data is ready */
+	if (bu27034_has_valid_sample(data)) {
+		ret = regmap_bulk_read(data->regmap, BU27034_REG_DATA0_LO,
+				       res, size);
+		if (ret)
+			return ret;
+
+		bu27034_invalidate_read_data(data);
+	} else {
+		/* No new data in sensor. Wait and retry */
+		retry_cnt++;
+
+		if (retry_cnt > BU27034_RETRY_LIMIT) {
+			dev_err(data->dev, "No data from sensor\n");
+
+			return -ETIMEDOUT;
+		}
+
+		msleep(25);
+
+		goto retry;
+	}
+
+	return ret;
+}
+
+static int bu27034_meas_set(struct bu27034_data *data, bool en)
+{
+	if (en)
+		return regmap_set_bits(data->regmap, BU27034_REG_MODE_CONTROL4,
+				       BU27034_MASK_MEAS_EN);
+
+	return regmap_clear_bits(data->regmap, BU27034_REG_MODE_CONTROL4,
+				 BU27034_MASK_MEAS_EN);
+}
+
+static int bu27034_get_single_result(struct bu27034_data *data, int chan,
+				     int *val)
+{
+	int ret;
+
+	if (chan < BU27034_CHAN_DATA0 || chan > BU27034_CHAN_DATA2)
+		return -EINVAL;
+
+	ret = bu27034_meas_set(data, true);
+	if (ret)
+		return ret;
+
+	ret = bu27034_get_int_time(data);
+	if (ret < 0)
+		return ret;
+
+	msleep(ret / 1000);
+
+	return bu27034_read_result(data, chan, val);
+}
+
+/*
+ * The formula given by vendor for computing luxes out of data0 and data1
+ * (in open air) is as follows:
+ *
+ * Let's mark:
+ * D0 = data0/ch0_gain/meas_time_ms * 25600
+ * D1 = data1/ch1_gain/meas_time_ms * 25600
+ *
+ * Then:
+ * if (D1/D0 < 0.87)
+ *	lx = (0.001331 * D0 + 0.0000354 * D1) * ((D1 / D0 - 0.87) * 3.45 + 1)
+ * else if (D1/D0 < 1)
+ *	lx = (0.001331 * D0 + 0.0000354 * D1) * ((D1 / D0 - 0.87) * 0.385 + 1)
+ * else
+ *	lx = (0.001331 * D0 + 0.0000354 * D1) * ((D1 / D0 - 2) * -0.05 + 1)
+ *
+ * We use it here. Users who have for example some colored lens
+ * need to modify the calculation but I hope this gives a starting point for
+ * those working with such devices.
+ */
+
+static int bu27034_calc_mlux(struct bu27034_data *data, __le16 *res, int *val)
+{
+	unsigned int gain0, gain1, meastime;
+	unsigned int d1_d0_ratio_scaled;
+	u16 ch0, ch1;
+	u64 helper64;
+	int ret;
+
+	/*
+	 * We return 0 lux if calculation fails. This should be reasonably
+	 * easy to spot from the buffers especially if raw-data channels show
+	 * valid values
+	 */
+	*val = 0;
+
+	ch0 = max_t(u16, 1, le16_to_cpu(res[0]));
+	ch1 = max_t(u16, 1, le16_to_cpu(res[1]));
+
+	ret = bu27034_get_gain(data, BU27034_CHAN_DATA0, &gain0);
+	if (ret)
+		return ret;
+
+	ret = bu27034_get_gain(data, BU27034_CHAN_DATA1, &gain1);
+	if (ret)
+		return ret;
+
+	ret = bu27034_get_int_time(data);
+	if (ret < 0)
+		return ret;
+
+	meastime = ret;
+
+	d1_d0_ratio_scaled = (unsigned int)ch1 * (unsigned int)gain0 * 100;
+	helper64 = (u64)ch1 * (u64)gain0 * 100LLU;
+
+	if (helper64 != d1_d0_ratio_scaled) {
+		unsigned int div = (unsigned int)ch0 * gain1;
+
+		do_div(helper64, div);
+		d1_d0_ratio_scaled = helper64;
+	} else {
+		d1_d0_ratio_scaled /= ch0 * gain1;
+	}
+
+	if (d1_d0_ratio_scaled < 87)
+		ret = bu27034_fixp_calc_lx(ch0, ch1, gain0, gain1, meastime, 0);
+	else if (d1_d0_ratio_scaled < 100)
+		ret = bu27034_fixp_calc_lx(ch0, ch1, gain0, gain1, meastime, 1);
+	else
+		ret = bu27034_fixp_calc_lx(ch0, ch1, gain0, gain1, meastime, 2);
+
+	if (ret < 0)
+		return ret;
+
+	*val = ret;
+
+	return 0;
+
+}
+
+static int bu27034_get_mlux(struct bu27034_data *data, int chan, int *val)
+{
+	__le16 res[3];
+	int ret;
+
+	ret = bu27034_meas_set(data, true);
+	if (ret)
+		return ret;
+
+	ret = bu27034_get_result_unlocked(data, &res[0], sizeof(res));
+	if (ret)
+		return ret;
+
+	ret = bu27034_calc_mlux(data, res, val);
+	if (ret)
+		return ret;
+
+	ret = bu27034_meas_set(data, false);
+	if (ret)
+		dev_err(data->dev, "failed to disable measurement\n");
+
+	return 0;
+}
+
+static int bu27034_read_raw(struct iio_dev *idev,
+			   struct iio_chan_spec const *chan,
+			   int *val, int *val2, long mask)
+{
+	struct bu27034_data *data = iio_priv(idev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_INT_TIME:
+		*val = bu27034_get_int_time(data);
+		if (*val < 0)
+			return *val;
+
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SCALE:
+		return bu27034_get_scale(data, chan->channel, val, val2);
+
+	case IIO_CHAN_INFO_RAW:
+	{
+		int (*result_get)(struct bu27034_data *data, int chan, int *val);
+
+		if (chan->type == IIO_INTENSITY)
+			result_get = bu27034_get_single_result;
+		else if (chan->type == IIO_LIGHT)
+			result_get = bu27034_get_mlux;
+		else
+			return -EINVAL;
+
+		/* Don't mess with measurement enabling while buffering */
+		ret = iio_device_claim_direct_mode(idev);
+		if (ret)
+			return ret;
+
+		mutex_lock(&data->mutex);
+		/*
+		 * Reading one channel at a time is inefficient but we
+		 * don't care here. Buffered version should be used if
+		 * performance is an issue.
+		 */
+		ret = result_get(data, chan->channel, val);
+
+		mutex_unlock(&data->mutex);
+		iio_device_release_direct_mode(idev);
+
+		if (ret)
+			return ret;
+
+		return IIO_VAL_INT;
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int bu27034_write_raw(struct iio_dev *idev,
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
+{
+	struct bu27034_data *data = iio_priv(idev);
+	int ret;
+
+	ret = iio_device_claim_direct_mode(idev);
+	if (ret)
+		return ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		ret = bu27034_set_scale(data, chan->channel, val, val2);
+		break;
+	case IIO_CHAN_INFO_INT_TIME:
+		ret = bu27034_try_set_int_time(data, val);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	iio_device_release_direct_mode(idev);
+
+	return ret;
+}
+
+static int bu27034_read_avail(struct iio_dev *idev,
+			      struct iio_chan_spec const *chan, const int **vals,
+			      int *type, int *length, long mask)
+{
+	struct bu27034_data *data = iio_priv(idev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_INT_TIME:
+		return iio_gts_avail_times(&data->gts, vals, type, length);
+	case IIO_CHAN_INFO_SCALE:
+		return iio_gts_all_avail_scales(&data->gts, vals, type, length);
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info bu27034_info = {
+	.read_raw = &bu27034_read_raw,
+	.write_raw = &bu27034_write_raw,
+	.read_avail = &bu27034_read_avail,
+};
+
+static int bu27034_chip_init(struct bu27034_data *data)
+{
+	int ret, sel;
+
+	/* Reset */
+	ret = regmap_update_bits(data->regmap, BU27034_REG_SYSTEM_CONTROL,
+			   BU27034_MASK_SW_RESET, BU27034_MASK_SW_RESET);
+	if (ret)
+		return dev_err_probe(data->dev, ret, "Sensor reset failed\n");
+
+	msleep(1);
+	/*
+	 * Read integration time here to ensure it is in regmap cache. We do
+	 * this to speed-up the int-time acquisition in the start of the buffer
+	 * handling thread where longer delays could make it more likely we end
+	 * up skipping a sample, and where the longer delays make timestamps
+	 * less accurate.
+	 */
+	ret = regmap_read(data->regmap, BU27034_REG_MODE_CONTROL1, &sel);
+	if (ret)
+		dev_err(data->dev, "reading integration time failed\n");
+
+	return 0;
+}
+
+static int bu27034_wait_for_data(struct bu27034_data *data)
+{
+	int ret, val;
+
+	ret = regmap_read_poll_timeout(data->regmap, BU27034_REG_MODE_CONTROL4,
+				       val, val & BU27034_MASK_VALID,
+				       BU27034_DATA_WAIT_TIME_US,
+				       BU27034_TOTAL_DATA_WAIT_TIME_US);
+	if (ret) {
+		dev_err(data->dev, "data polling %s\n",
+			!(val & BU27034_MASK_VALID) ? "timeout" : "fail");
+
+		return ret;
+	}
+
+	ret = regmap_bulk_read(data->regmap, BU27034_REG_DATA0_LO,
+			       &data->scan.channels[0],
+			       sizeof(data->scan.channels));
+	if (ret)
+		return ret;
+
+	bu27034_invalidate_read_data(data);
+
+	return 0;
+}
+
+static int bu27034_buffer_thread(void *arg)
+{
+	struct iio_dev *idev = arg;
+	struct bu27034_data *data;
+	int wait_ms;
+
+	data = iio_priv(idev);
+
+	wait_ms = bu27034_get_int_time(data);
+	wait_ms /= 1000;
+
+	wait_ms -= BU27034_MEAS_WAIT_PREMATURE_MS;
+
+	while (!kthread_should_stop()) {
+		int ret;
+		int64_t tstamp;
+
+		msleep(wait_ms);
+		ret = bu27034_wait_for_data(data);
+		if (ret)
+			continue;
+
+		tstamp = iio_get_time_ns(idev);
+
+		if (test_bit(BU27034_CHAN_ALS, idev->active_scan_mask)) {
+			int mlux;
+
+			ret = bu27034_calc_mlux(data, &data->scan.channels[0],
+					       &mlux);
+			if (ret)
+				dev_err(data->dev, "failed to calculate lux\n");
+
+			/*
+			 * The maximum Milli lux value we get with gain 1x time
+			 * 55mS data ch0 = 0xffff ch1 = 0xffff fits in 26 bits
+			 * so there should be no problem returning int from
+			 * computations and casting it to u32
+			 */
+			data->scan.mlux = (u32)mlux;
+		}
+		iio_push_to_buffers_with_timestamp(idev, &data->scan, tstamp);
+	}
+
+	return 0;
+}
+
+static int bu27034_buffer_enable(struct iio_dev *idev)
+{
+	struct bu27034_data *data = iio_priv(idev);
+	struct task_struct *task;
+	int ret;
+
+	mutex_lock(&data->mutex);
+	ret = bu27034_meas_set(data, true);
+	if (ret)
+		goto unlock_out;
+
+	task = kthread_run(bu27034_buffer_thread, idev,
+				 "bu27034-buffering-%u",
+				 iio_device_id(idev));
+	if (IS_ERR(task)) {
+		ret = PTR_ERR(task);
+		goto unlock_out;
+	}
+
+	data->task = task;
+
+unlock_out:
+	mutex_unlock(&data->mutex);
+
+	return ret;
+}
+
+static int bu27034_buffer_disable(struct iio_dev *idev)
+{
+	struct bu27034_data *data = iio_priv(idev);
+	int ret;
+
+	mutex_lock(&data->mutex);
+	if (data->task) {
+		kthread_stop(data->task);
+		data->task = NULL;
+	}
+
+	ret = bu27034_meas_set(data, false);
+	mutex_unlock(&data->mutex);
+
+	return ret;
+}
+
+static const struct iio_buffer_setup_ops bu27034_buffer_ops = {
+	.postenable = &bu27034_buffer_enable,
+	.predisable = &bu27034_buffer_disable,
+};
+
+static int bu27034_probe(struct i2c_client *i2c)
+{
+	struct device *dev = &i2c->dev;
+	struct bu27034_data *data;
+	struct regmap *regmap;
+	struct iio_dev *idev;
+	unsigned int part_id, reg;
+	int ret;
+
+	regmap = devm_regmap_init_i2c(i2c, &bu27034_regmap);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap),
+				     "Failed to initialize Regmap\n");
+
+	idev = devm_iio_device_alloc(dev, sizeof(*data));
+	if (!idev)
+		return -ENOMEM;
+
+	ret = devm_regulator_get_enable(dev, "vdd");
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get regulator\n");
+
+	data = iio_priv(idev);
+
+	ret = regmap_read(regmap, BU27034_REG_SYSTEM_CONTROL, &reg);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to access sensor\n");
+
+	part_id = FIELD_GET(BU27034_MASK_PART_ID, reg);
+
+	if (part_id != BU27034_ID)
+		dev_warn(dev, "unknown device 0x%x\n", part_id);
+
+	ret = devm_iio_init_iio_gts(dev, BU27034_SCALE_1X, 0, bu27034_gains,
+				    ARRAY_SIZE(bu27034_gains), bu27034_itimes,
+				    ARRAY_SIZE(bu27034_itimes), &data->gts);
+	if (ret)
+		return ret;
+
+	mutex_init(&data->mutex);
+	data->regmap = regmap;
+	data->dev = dev;
+
+	idev->channels = bu27034_channels;
+	idev->num_channels = ARRAY_SIZE(bu27034_channels);
+	idev->name = "bu27034";
+	idev->info = &bu27034_info;
+
+	idev->modes = INDIO_DIRECT_MODE | INDIO_BUFFER_SOFTWARE;
+	idev->available_scan_masks = bu27034_scan_masks;
+
+	ret = bu27034_chip_init(data);
+	if (ret)
+		return ret;
+
+	ret = devm_iio_kfifo_buffer_setup(dev, idev, &bu27034_buffer_ops);
+	if (ret)
+		return dev_err_probe(dev, ret, "buffer setup failed\n");
+
+	ret = devm_iio_device_register(dev, idev);
+	if (ret < 0)
+		return dev_err_probe(dev, ret,
+				     "Unable to register iio device\n");
+
+	return ret;
+}
+
+static const struct of_device_id bu27034_of_match[] = {
+	{ .compatible = "rohm,bu27034" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, bu27034_of_match);
+
+static struct i2c_driver bu27034_i2c_driver = {
+	.driver = {
+		.name = "bu27034-als",
+		.of_match_table = bu27034_of_match,
+	},
+	.probe_new = bu27034_probe,
+};
+module_i2c_driver(bu27034_i2c_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>");
+MODULE_DESCRIPTION("ROHM BU27034 ambient light sensor driver");
+MODULE_IMPORT_NS(IIO_GTS_HELPER);
diff --git a/drivers/iio/light/rpr0521.c b/drivers/iio/light/rpr0521.c
index 668e444f6049..9d0218b7426e 100644
--- a/drivers/iio/light/rpr0521.c
+++ b/drivers/iio/light/rpr0521.c
@@ -431,7 +431,7 @@ static irqreturn_t rpr0521_drdy_irq_thread(int irq, void *private)
 	struct rpr0521_data *data = iio_priv(indio_dev);
 
 	if (rpr0521_is_triggered(data)) {
-		iio_trigger_poll_chained(data->drdy_trigger0);
+		iio_trigger_poll_nested(data->drdy_trigger0);
 		return IRQ_HANDLED;
 	}
 
diff --git a/drivers/iio/light/st_uvis25_core.c b/drivers/iio/light/st_uvis25_core.c
index c737d3e193ae..50f95c5d2060 100644
--- a/drivers/iio/light/st_uvis25_core.c
+++ b/drivers/iio/light/st_uvis25_core.c
@@ -161,7 +161,7 @@ static irqreturn_t st_uvis25_trigger_handler_thread(int irq, void *private)
 	if (!(status & ST_UVIS25_REG_UV_DA_MASK))
 		return IRQ_NONE;
 
-	iio_trigger_poll_chained(hw->trig);
+	iio_trigger_poll_nested(hw->trig);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/iio/light/tsl2772.c b/drivers/iio/light/tsl2772.c
index ad50baa0202c..e823c145f679 100644
--- a/drivers/iio/light/tsl2772.c
+++ b/drivers/iio/light/tsl2772.c
@@ -601,6 +601,7 @@ static int tsl2772_read_prox_diodes(struct tsl2772_chip *chip)
 			return -EINVAL;
 		}
 	}
+	chip->settings.prox_diode = prox_diode_mask;
 
 	return 0;
 }
diff --git a/drivers/iio/light/vcnl4000.c b/drivers/iio/light/vcnl4000.c
index 5c44a36ab5b3..56d3963d3d66 100644
--- a/drivers/iio/light/vcnl4000.c
+++ b/drivers/iio/light/vcnl4000.c
@@ -1077,7 +1077,7 @@ static irqreturn_t vcnl4010_irq_thread(int irq, void *p)
 	}
 
 	if (isr & VCNL4010_INT_DRDY && iio_buffer_enabled(indio_dev))
-		iio_trigger_poll_chained(indio_dev->trig);
+		iio_trigger_poll_nested(indio_dev->trig);
 
 end:
 	return IRQ_HANDLED;
diff --git a/drivers/iio/light/vcnl4035.c b/drivers/iio/light/vcnl4035.c
index 84148b944000..14e29330e972 100644
--- a/drivers/iio/light/vcnl4035.c
+++ b/drivers/iio/light/vcnl4035.c
@@ -89,7 +89,7 @@ static irqreturn_t vcnl4035_drdy_irq_thread(int irq, void *private)
 							IIO_EV_TYPE_THRESH,
 							IIO_EV_DIR_EITHER),
 				iio_get_time_ns(indio_dev));
-		iio_trigger_poll_chained(data->drdy_trigger0);
+		iio_trigger_poll_nested(data->drdy_trigger0);
 		return IRQ_HANDLED;
 	}
 
diff --git a/drivers/iio/potentiostat/lmp91000.c b/drivers/iio/potentiostat/lmp91000.c
index b82f093f1e6a..0083e858c21e 100644
--- a/drivers/iio/potentiostat/lmp91000.c
+++ b/drivers/iio/potentiostat/lmp91000.c
@@ -118,7 +118,7 @@ static int lmp91000_read(struct lmp91000_data *data, int channel, int *val)
 
 	data->chan_select = channel != LMP91000_REG_MODECN_3LEAD;
 
-	iio_trigger_poll_chained(data->trig);
+	iio_trigger_poll_nested(data->trig);
 
 	ret = wait_for_completion_timeout(&data->completion, HZ);
 	reinit_completion(&data->completion);
diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig
index c9453389e4f7..02b97e89de50 100644
--- a/drivers/iio/pressure/Kconfig
+++ b/drivers/iio/pressure/Kconfig
@@ -17,14 +17,14 @@ config ABP060MG
 	  will be called abp060mg.
 
 config BMP280
-	tristate "Bosch Sensortec BMP180/BMP280/BMP380 pressure sensor I2C driver"
+	tristate "Bosch Sensortec BMP180/BMP280/BMP380/BMP580 pressure sensor driver"
 	depends on (I2C || SPI_MASTER)
 	select REGMAP
 	select BMP280_I2C if (I2C)
 	select BMP280_SPI if (SPI_MASTER)
 	help
-	  Say yes here to build support for Bosch Sensortec BMP180, BMP280 and
-	  BMP380 pressure and temperature sensors. Also supports the BME280 with
+	  Say yes here to build support for Bosch Sensortec BMP180, BMP280, BMP380
+	  and BMP580 pressure and temperature sensors. Also supports the BME280 with
 	  an additional humidity sensor channel.
 
 	  To compile this driver as a module, choose M here: the core module
diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c
index c0aff78489b4..6089f3f9d8f4 100644
--- a/drivers/iio/pressure/bmp280-core.c
+++ b/drivers/iio/pressure/bmp280-core.c
@@ -13,6 +13,7 @@
  * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf
  * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bme280-ds002.pdf
  * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp388-ds001.pdf
+ * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp581-ds004.pdf
  *
  * Notice:
  * The link to the bmp180 datasheet points to an outdated version missing these changes:
@@ -27,6 +28,7 @@
 #include <linux/bitfield.h>
 #include <linux/device.h>
 #include <linux/module.h>
+#include <linux/nvmem-provider.h>
 #include <linux/regmap.h>
 #include <linux/delay.h>
 #include <linux/iio/iio.h>
@@ -49,65 +51,6 @@
  */
 enum { AC1, AC2, AC3, AC4, AC5, AC6, B1, B2, MB, MC, MD };
 
-struct bmp180_calib {
-	s16 AC1;
-	s16 AC2;
-	s16 AC3;
-	u16 AC4;
-	u16 AC5;
-	u16 AC6;
-	s16 B1;
-	s16 B2;
-	s16 MB;
-	s16 MC;
-	s16 MD;
-};
-
-/* See datasheet Section 4.2.2. */
-struct bmp280_calib {
-	u16 T1;
-	s16 T2;
-	s16 T3;
-	u16 P1;
-	s16 P2;
-	s16 P3;
-	s16 P4;
-	s16 P5;
-	s16 P6;
-	s16 P7;
-	s16 P8;
-	s16 P9;
-	u8  H1;
-	s16 H2;
-	u8  H3;
-	s16 H4;
-	s16 H5;
-	s8  H6;
-};
-
-/* See datasheet Section 3.11.1. */
-struct bmp380_calib {
-	u16 T1;
-	u16 T2;
-	s8  T3;
-	s16 P1;
-	s16 P2;
-	s8  P3;
-	s8  P4;
-	u16 P5;
-	u16 P6;
-	s8  P7;
-	s8  P8;
-	s16 P9;
-	s8  P10;
-	s8  P11;
-};
-
-static const char *const bmp280_supply_names[] = {
-	"vddd", "vdda"
-};
-
-#define BMP280_NUM_SUPPLIES ARRAY_SIZE(bmp280_supply_names)
 
 enum bmp380_odr {
 	BMP380_ODR_200HZ,
@@ -130,92 +73,39 @@ enum bmp380_odr {
 	BMP380_ODR_0_0015HZ,
 };
 
-struct bmp280_data {
-	struct device *dev;
-	struct mutex lock;
-	struct regmap *regmap;
-	struct completion done;
-	bool use_eoc;
-	const struct bmp280_chip_info *chip_info;
-	union {
-		struct bmp180_calib bmp180;
-		struct bmp280_calib bmp280;
-		struct bmp380_calib bmp380;
-	} calib;
-	struct regulator_bulk_data supplies[BMP280_NUM_SUPPLIES];
-	unsigned int start_up_time; /* in microseconds */
-
-	/* log of base 2 of oversampling rate */
-	u8 oversampling_press;
-	u8 oversampling_temp;
-	u8 oversampling_humid;
-	u8 iir_filter_coeff;
-
-	/*
-	 * BMP380 devices introduce sampling frequency configuration. See
-	 * datasheet sections 3.3.3. and 4.3.19 for more details.
-	 *
-	 * BMx280 devices allowed indirect configuration of sampling frequency
-	 * changing the t_standby duration between measurements, as detailed on
-	 * section 3.6.3 of the datasheet.
-	 */
-	int sampling_freq;
-
-	/*
-	 * Carryover value from temperature conversion, used in pressure
-	 * calculation.
-	 */
-	s32 t_fine;
-
-	/*
-	 * DMA (thus cache coherency maintenance) may require the
-	 * transfer buffers to live in their own cache lines.
-	 */
-	union {
-		/* Sensor data buffer */
-		u8 buf[3];
-		/* Calibration data buffers */
-		__le16 bmp280_cal_buf[BMP280_CONTIGUOUS_CALIB_REGS / 2];
-		__be16 bmp180_cal_buf[BMP180_REG_CALIB_COUNT / 2];
-		u8 bmp380_cal_buf[BMP380_CALIB_REG_COUNT];
-		/* Miscellaneous, endianess-aware data buffers */
-		__le16 le16;
-		__be16 be16;
-	} __aligned(IIO_DMA_MINALIGN);
-};
-
-struct bmp280_chip_info {
-	unsigned int id_reg;
-
-	const struct iio_chan_spec *channels;
-	int num_channels;
-	unsigned int start_up_time;
-
-	const int *oversampling_temp_avail;
-	int num_oversampling_temp_avail;
-	int oversampling_temp_default;
-
-	const int *oversampling_press_avail;
-	int num_oversampling_press_avail;
-	int oversampling_press_default;
-
-	const int *oversampling_humid_avail;
-	int num_oversampling_humid_avail;
-	int oversampling_humid_default;
-
-	const int *iir_filter_coeffs_avail;
-	int num_iir_filter_coeffs_avail;
-	int iir_filter_coeff_default;
-
-	const int (*sampling_freq_avail)[2];
-	int num_sampling_freq_avail;
-	int sampling_freq_default;
-
-	int (*chip_config)(struct bmp280_data *);
-	int (*read_temp)(struct bmp280_data *, int *);
-	int (*read_press)(struct bmp280_data *, int *, int *);
-	int (*read_humid)(struct bmp280_data *, int *, int *);
-	int (*read_calib)(struct bmp280_data *);
+enum bmp580_odr {
+	BMP580_ODR_240HZ,
+	BMP580_ODR_218HZ,
+	BMP580_ODR_199HZ,
+	BMP580_ODR_179HZ,
+	BMP580_ODR_160HZ,
+	BMP580_ODR_149HZ,
+	BMP580_ODR_140HZ,
+	BMP580_ODR_129HZ,
+	BMP580_ODR_120HZ,
+	BMP580_ODR_110HZ,
+	BMP580_ODR_100HZ,
+	BMP580_ODR_89HZ,
+	BMP580_ODR_80HZ,
+	BMP580_ODR_70HZ,
+	BMP580_ODR_60HZ,
+	BMP580_ODR_50HZ,
+	BMP580_ODR_45HZ,
+	BMP580_ODR_40HZ,
+	BMP580_ODR_35HZ,
+	BMP580_ODR_30HZ,
+	BMP580_ODR_25HZ,
+	BMP580_ODR_20HZ,
+	BMP580_ODR_15HZ,
+	BMP580_ODR_10HZ,
+	BMP580_ODR_5HZ,
+	BMP580_ODR_4HZ,
+	BMP580_ODR_3HZ,
+	BMP580_ODR_2HZ,
+	BMP580_ODR_1HZ,
+	BMP580_ODR_0_5HZ,
+	BMP580_ODR_0_25HZ,
+	BMP580_ODR_0_125HZ,
 };
 
 /*
@@ -473,7 +363,7 @@ static u32 bmp280_compensate_press(struct bmp280_data *data,
 }
 
 static int bmp280_read_temp(struct bmp280_data *data,
-			    int *val)
+			    int *val, int *val2)
 {
 	s32 adc_temp, comp_temp;
 	int ret;
@@ -513,7 +403,7 @@ static int bmp280_read_press(struct bmp280_data *data,
 	int ret;
 
 	/* Read and compensate temperature so we get a reading of t_fine. */
-	ret = bmp280_read_temp(data, NULL);
+	ret = bmp280_read_temp(data, NULL, NULL);
 	if (ret < 0)
 		return ret;
 
@@ -545,7 +435,7 @@ static int bmp280_read_humid(struct bmp280_data *data, int *val, int *val2)
 	int ret;
 
 	/* Read and compensate temperature so we get a reading of t_fine. */
-	ret = bmp280_read_temp(data, NULL);
+	ret = bmp280_read_temp(data, NULL, NULL);
 	if (ret < 0)
 		return ret;
 
@@ -589,7 +479,7 @@ static int bmp280_read_raw(struct iio_dev *indio_dev,
 			ret = data->chip_info->read_press(data, val, val2);
 			break;
 		case IIO_TEMP:
-			ret = data->chip_info->read_temp(data, val);
+			ret = data->chip_info->read_temp(data, val, val2);
 			break;
 		default:
 			ret = -EINVAL;
@@ -905,8 +795,10 @@ static int bmp280_chip_config(struct bmp280_data *data)
 
 static const int bmp280_oversampling_avail[] = { 1, 2, 4, 8, 16 };
 
-static const struct bmp280_chip_info bmp280_chip_info = {
+const struct bmp280_chip_info bmp280_chip_info = {
 	.id_reg = BMP280_REG_ID,
+	.chip_id = BMP280_CHIP_ID,
+	.regmap_config = &bmp280_regmap_config,
 	.start_up_time = 2000,
 	.channels = bmp280_channels,
 	.num_channels = 2,
@@ -934,6 +826,7 @@ static const struct bmp280_chip_info bmp280_chip_info = {
 	.read_press = bmp280_read_press,
 	.read_calib = bmp280_read_calib,
 };
+EXPORT_SYMBOL_NS(bmp280_chip_info, IIO_BMP280);
 
 static int bme280_chip_config(struct bmp280_data *data)
 {
@@ -953,8 +846,10 @@ static int bme280_chip_config(struct bmp280_data *data)
 	return bmp280_chip_config(data);
 }
 
-static const struct bmp280_chip_info bme280_chip_info = {
+const struct bmp280_chip_info bme280_chip_info = {
 	.id_reg = BMP280_REG_ID,
+	.chip_id = BME280_CHIP_ID,
+	.regmap_config = &bmp280_regmap_config,
 	.start_up_time = 2000,
 	.channels = bmp280_channels,
 	.num_channels = 3,
@@ -977,6 +872,7 @@ static const struct bmp280_chip_info bme280_chip_info = {
 	.read_humid = bmp280_read_humid,
 	.read_calib = bme280_read_calib,
 };
+EXPORT_SYMBOL_NS(bme280_chip_info, IIO_BMP280);
 
 /*
  * Helper function to send a command to BMP3XX sensors.
@@ -1095,7 +991,7 @@ static u32 bmp380_compensate_press(struct bmp280_data *data, u32 adc_press)
 	return comp_press;
 }
 
-static int bmp380_read_temp(struct bmp280_data *data, int *val)
+static int bmp380_read_temp(struct bmp280_data *data, int *val, int *val2)
 {
 	s32 comp_temp;
 	u32 adc_temp;
@@ -1135,7 +1031,7 @@ static int bmp380_read_press(struct bmp280_data *data, int *val, int *val2)
 	int ret;
 
 	/* Read and compensate for temperature so we get a reading of t_fine */
-	ret = bmp380_read_temp(data, NULL);
+	ret = bmp380_read_temp(data, NULL, NULL);
 	if (ret)
 		return ret;
 
@@ -1217,6 +1113,12 @@ static const int bmp380_odr_table[][2] = {
 	[BMP380_ODR_0_0015HZ]	= {0, 1526},
 };
 
+static int bmp380_preinit(struct bmp280_data *data)
+{
+	/* BMP3xx requires soft-reset as part of initialization */
+	return bmp380_cmd(data, BMP380_CMD_SOFT_RESET);
+}
+
 static int bmp380_chip_config(struct bmp280_data *data)
 {
 	bool change = false, aux;
@@ -1319,8 +1221,10 @@ static int bmp380_chip_config(struct bmp280_data *data)
 static const int bmp380_oversampling_avail[] = { 1, 2, 4, 8, 16, 32 };
 static const int bmp380_iir_filter_coeffs_avail[] = { 1, 2, 4, 8, 16, 32, 64, 128};
 
-static const struct bmp280_chip_info bmp380_chip_info = {
+const struct bmp280_chip_info bmp380_chip_info = {
 	.id_reg = BMP380_REG_ID,
+	.chip_id = BMP380_CHIP_ID,
+	.regmap_config = &bmp380_regmap_config,
 	.start_up_time = 2000,
 	.channels = bmp380_channels,
 	.num_channels = 2,
@@ -1345,7 +1249,508 @@ static const struct bmp280_chip_info bmp380_chip_info = {
 	.read_temp = bmp380_read_temp,
 	.read_press = bmp380_read_press,
 	.read_calib = bmp380_read_calib,
+	.preinit = bmp380_preinit,
+};
+EXPORT_SYMBOL_NS(bmp380_chip_info, IIO_BMP280);
+
+static int bmp580_soft_reset(struct bmp280_data *data)
+{
+	unsigned int reg;
+	int ret;
+
+	ret = regmap_write(data->regmap, BMP580_REG_CMD, BMP580_CMD_SOFT_RESET);
+	if (ret) {
+		dev_err(data->dev, "failed to send reset command to device\n");
+		return ret;
+	}
+	usleep_range(2000, 2500);
+
+	/* Dummy read of chip_id */
+	ret = regmap_read(data->regmap, BMP580_REG_CHIP_ID, &reg);
+	if (ret) {
+		dev_err(data->dev, "failed to reestablish comms after reset\n");
+		return ret;
+	}
+
+	ret = regmap_read(data->regmap, BMP580_REG_INT_STATUS, &reg);
+	if (ret) {
+		dev_err(data->dev, "error reading interrupt status register\n");
+		return ret;
+	}
+	if (!(reg & BMP580_INT_STATUS_POR_MASK)) {
+		dev_err(data->dev, "error resetting sensor\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * bmp580_nvm_operation() - Helper function to commit NVM memory operations
+ * @data: sensor data struct
+ * @is_write: flag to signal write operation
+ */
+static int bmp580_nvm_operation(struct bmp280_data *data, bool is_write)
+{
+	unsigned long timeout, poll;
+	unsigned int reg;
+	int ret;
+
+	/* Check NVM ready flag */
+	ret = regmap_read(data->regmap, BMP580_REG_STATUS, &reg);
+	if (ret) {
+		dev_err(data->dev, "failed to check nvm status\n");
+		return ret;
+	}
+	if (!(reg & BMP580_STATUS_NVM_RDY_MASK)) {
+		dev_err(data->dev, "sensor's nvm is not ready\n");
+		return -EIO;
+	}
+
+	/* Start NVM operation sequence */
+	ret = regmap_write(data->regmap, BMP580_REG_CMD, BMP580_CMD_NVM_OP_SEQ_0);
+	if (ret) {
+		dev_err(data->dev, "failed to send nvm operation's first sequence\n");
+		return ret;
+	}
+	if (is_write) {
+		/* Send NVM write sequence */
+		ret = regmap_write(data->regmap, BMP580_REG_CMD,
+				   BMP580_CMD_NVM_WRITE_SEQ_1);
+		if (ret) {
+			dev_err(data->dev, "failed to send nvm write sequence\n");
+			return ret;
+		}
+		/* Datasheet says on 4.8.1.2 it takes approximately 10ms */
+		poll = 2000;
+		timeout = 12000;
+	} else {
+		/* Send NVM read sequence */
+		ret = regmap_write(data->regmap, BMP580_REG_CMD,
+				   BMP580_CMD_NVM_READ_SEQ_1);
+		if (ret) {
+			dev_err(data->dev, "failed to send nvm read sequence\n");
+			return ret;
+		}
+		/* Datasheet says on 4.8.1.1 it takes approximately 200us */
+		poll = 50;
+		timeout = 400;
+	}
+	if (ret) {
+		dev_err(data->dev, "failed to write command sequence\n");
+		return -EIO;
+	}
+
+	/* Wait until NVM is ready again */
+	ret = regmap_read_poll_timeout(data->regmap, BMP580_REG_STATUS, reg,
+				       (reg & BMP580_STATUS_NVM_RDY_MASK),
+				       poll, timeout);
+	if (ret) {
+		dev_err(data->dev, "error checking nvm operation status\n");
+		return ret;
+	}
+
+	/* Check NVM error flags */
+	if ((reg & BMP580_STATUS_NVM_ERR_MASK) || (reg & BMP580_STATUS_NVM_CMD_ERR_MASK)) {
+		dev_err(data->dev, "error processing nvm operation\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/*
+ * Contrary to previous sensors families, compensation algorithm is builtin.
+ * We are only required to read the register raw data and adapt the ranges
+ * for what is expected on IIO ABI.
+ */
+
+static int bmp580_read_temp(struct bmp280_data *data, int *val, int *val2)
+{
+	s32 raw_temp;
+	int ret;
+
+	ret = regmap_bulk_read(data->regmap, BMP580_REG_TEMP_XLSB, data->buf,
+			       sizeof(data->buf));
+	if (ret) {
+		dev_err(data->dev, "failed to read temperature\n");
+		return ret;
+	}
+
+	raw_temp = get_unaligned_le24(data->buf);
+	if (raw_temp == BMP580_TEMP_SKIPPED) {
+		dev_err(data->dev, "reading temperature skipped\n");
+		return -EIO;
+	}
+
+	/*
+	 * Temperature is returned in Celsius degrees in fractional
+	 * form down 2^16. We reescale by x1000 to return milli Celsius
+	 * to respect IIO ABI.
+	 */
+	*val = raw_temp * 1000;
+	*val2 = 16;
+	return IIO_VAL_FRACTIONAL_LOG2;
+}
+
+static int bmp580_read_press(struct bmp280_data *data, int *val, int *val2)
+{
+	u32 raw_press;
+	int ret;
+
+	ret = regmap_bulk_read(data->regmap, BMP580_REG_PRESS_XLSB, data->buf,
+			       sizeof(data->buf));
+	if (ret) {
+		dev_err(data->dev, "failed to read pressure\n");
+		return ret;
+	}
+
+	raw_press = get_unaligned_le24(data->buf);
+	if (raw_press == BMP580_PRESS_SKIPPED) {
+		dev_err(data->dev, "reading pressure skipped\n");
+		return -EIO;
+	}
+	/*
+	 * Pressure is returned in Pascals in fractional form down 2^16.
+	 * We reescale /1000 to convert to kilopascal to respect IIO ABI.
+	 */
+	*val = raw_press;
+	*val2 = 64000; /* 2^6 * 1000 */
+	return IIO_VAL_FRACTIONAL;
+}
+
+static const int bmp580_odr_table[][2] = {
+	[BMP580_ODR_240HZ] =	{240, 0},
+	[BMP580_ODR_218HZ] =	{218, 0},
+	[BMP580_ODR_199HZ] =	{199, 0},
+	[BMP580_ODR_179HZ] =	{179, 0},
+	[BMP580_ODR_160HZ] =	{160, 0},
+	[BMP580_ODR_149HZ] =	{149, 0},
+	[BMP580_ODR_140HZ] =	{140, 0},
+	[BMP580_ODR_129HZ] =	{129, 0},
+	[BMP580_ODR_120HZ] =	{120, 0},
+	[BMP580_ODR_110HZ] =	{110, 0},
+	[BMP580_ODR_100HZ] =	{100, 0},
+	[BMP580_ODR_89HZ] =	{89, 0},
+	[BMP580_ODR_80HZ] =	{80, 0},
+	[BMP580_ODR_70HZ] =	{70, 0},
+	[BMP580_ODR_60HZ] =	{60, 0},
+	[BMP580_ODR_50HZ] =	{50, 0},
+	[BMP580_ODR_45HZ] =	{45, 0},
+	[BMP580_ODR_40HZ] =	{40, 0},
+	[BMP580_ODR_35HZ] =	{35, 0},
+	[BMP580_ODR_30HZ] =	{30, 0},
+	[BMP580_ODR_25HZ] =	{25, 0},
+	[BMP580_ODR_20HZ] =	{20, 0},
+	[BMP580_ODR_15HZ] =	{15, 0},
+	[BMP580_ODR_10HZ] =	{10, 0},
+	[BMP580_ODR_5HZ] =	{5, 0},
+	[BMP580_ODR_4HZ] =	{4, 0},
+	[BMP580_ODR_3HZ] =	{3, 0},
+	[BMP580_ODR_2HZ] =	{2, 0},
+	[BMP580_ODR_1HZ] =	{1, 0},
+	[BMP580_ODR_0_5HZ] =	{0, 500000},
+	[BMP580_ODR_0_25HZ] =	{0, 250000},
+	[BMP580_ODR_0_125HZ] =	{0, 125000},
+};
+
+static const int bmp580_nvmem_addrs[] = { 0x20, 0x21, 0x22 };
+
+static int bmp580_nvmem_read(void *priv, unsigned int offset, void *val,
+			     size_t bytes)
+{
+	struct bmp280_data *data = priv;
+	u16 *dst = val;
+	int ret, addr;
+
+	pm_runtime_get_sync(data->dev);
+	mutex_lock(&data->lock);
+
+	/* Set sensor in standby mode */
+	ret = regmap_update_bits(data->regmap, BMP580_REG_ODR_CONFIG,
+				 BMP580_MODE_MASK | BMP580_ODR_DEEPSLEEP_DIS,
+				 BMP580_ODR_DEEPSLEEP_DIS |
+				 FIELD_PREP(BMP580_MODE_MASK, BMP580_MODE_SLEEP));
+	if (ret) {
+		dev_err(data->dev, "failed to change sensor to standby mode\n");
+		goto exit;
+	}
+	/* Wait standby transition time */
+	usleep_range(2500, 3000);
+
+	while (bytes >= sizeof(*dst)) {
+		addr = bmp580_nvmem_addrs[offset / sizeof(*dst)];
+
+		ret = regmap_write(data->regmap, BMP580_REG_NVM_ADDR,
+				   FIELD_PREP(BMP580_NVM_ROW_ADDR_MASK, addr));
+		if (ret) {
+			dev_err(data->dev, "error writing nvm address\n");
+			goto exit;
+		}
+
+		ret = bmp580_nvm_operation(data, false);
+		if (ret)
+			goto exit;
+
+		ret = regmap_bulk_read(data->regmap, BMP580_REG_NVM_DATA_LSB, &data->le16,
+				       sizeof(data->le16));
+		if (ret) {
+			dev_err(data->dev, "error reading nvm data regs\n");
+			goto exit;
+		}
+
+		*dst++ = le16_to_cpu(data->le16);
+		bytes -= sizeof(*dst);
+		offset += sizeof(*dst);
+	}
+exit:
+	/* Restore chip config */
+	data->chip_info->chip_config(data);
+	mutex_unlock(&data->lock);
+	pm_runtime_mark_last_busy(data->dev);
+	pm_runtime_put_autosuspend(data->dev);
+	return ret;
+}
+
+static int bmp580_nvmem_write(void *priv, unsigned int offset, void *val,
+			      size_t bytes)
+{
+	struct bmp280_data *data = priv;
+	u16 *buf = val;
+	int ret, addr;
+
+	pm_runtime_get_sync(data->dev);
+	mutex_lock(&data->lock);
+
+	/* Set sensor in standby mode */
+	ret = regmap_update_bits(data->regmap, BMP580_REG_ODR_CONFIG,
+				 BMP580_MODE_MASK | BMP580_ODR_DEEPSLEEP_DIS,
+				 BMP580_ODR_DEEPSLEEP_DIS |
+				 FIELD_PREP(BMP580_MODE_MASK, BMP580_MODE_SLEEP));
+	if (ret) {
+		dev_err(data->dev, "failed to change sensor to standby mode\n");
+		goto exit;
+	}
+	/* Wait standby transition time */
+	usleep_range(2500, 3000);
+
+	while (bytes >= sizeof(*buf)) {
+		addr = bmp580_nvmem_addrs[offset / sizeof(*buf)];
+
+		ret = regmap_write(data->regmap, BMP580_REG_NVM_ADDR, BMP580_NVM_PROG_EN |
+				   FIELD_PREP(BMP580_NVM_ROW_ADDR_MASK, addr));
+		if (ret) {
+			dev_err(data->dev, "error writing nvm address\n");
+			goto exit;
+		}
+		data->le16 = cpu_to_le16(*buf++);
+
+		ret = regmap_bulk_write(data->regmap, BMP580_REG_NVM_DATA_LSB, &data->le16,
+					sizeof(data->le16));
+		if (ret) {
+			dev_err(data->dev, "error writing LSB NVM data regs\n");
+			goto exit;
+		}
+
+		ret = bmp580_nvm_operation(data, true);
+		if (ret)
+			goto exit;
+
+		/* Disable programming mode bit */
+		ret = regmap_update_bits(data->regmap, BMP580_REG_NVM_ADDR,
+					 BMP580_NVM_PROG_EN, 0);
+		if (ret) {
+			dev_err(data->dev, "error resetting nvm write\n");
+			goto exit;
+		}
+
+		bytes -= sizeof(*buf);
+		offset += sizeof(*buf);
+	}
+exit:
+	/* Restore chip config */
+	data->chip_info->chip_config(data);
+	mutex_unlock(&data->lock);
+	pm_runtime_mark_last_busy(data->dev);
+	pm_runtime_put_autosuspend(data->dev);
+	return ret;
+}
+
+static int bmp580_preinit(struct bmp280_data *data)
+{
+	struct nvmem_config config = {
+		.dev = data->dev,
+		.priv = data,
+		.name = "bmp580_nvmem",
+		.word_size = sizeof(u16),
+		.stride = sizeof(u16),
+		.size = 3 * sizeof(u16),
+		.reg_read = bmp580_nvmem_read,
+		.reg_write = bmp580_nvmem_write,
+	};
+	unsigned int reg;
+	int ret;
+
+	/* Issue soft-reset command */
+	ret = bmp580_soft_reset(data);
+	if (ret)
+		return ret;
+
+	/* Post powerup sequence */
+	ret = regmap_read(data->regmap, BMP580_REG_CHIP_ID, &reg);
+	if (ret)
+		return ret;
+
+	/* Print warn message if we don't know the chip id */
+	if (reg != BMP580_CHIP_ID && reg != BMP580_CHIP_ID_ALT)
+		dev_warn(data->dev, "preinit: unexpected chip_id\n");
+
+	ret = regmap_read(data->regmap, BMP580_REG_STATUS, &reg);
+	if (ret)
+		return ret;
+
+	/* Check nvm status */
+	if (!(reg & BMP580_STATUS_NVM_RDY_MASK) || (reg & BMP580_STATUS_NVM_ERR_MASK)) {
+		dev_err(data->dev, "preinit: nvm error on powerup sequence\n");
+		return -EIO;
+	}
+
+	/* Register nvmem device */
+	return PTR_ERR_OR_ZERO(devm_nvmem_register(config.dev, &config));
+}
+
+static int bmp580_chip_config(struct bmp280_data *data)
+{
+	bool change = false, aux;
+	unsigned int tmp;
+	u8 reg_val;
+	int ret;
+
+	/* Sets sensor in standby mode */
+	ret = regmap_update_bits(data->regmap, BMP580_REG_ODR_CONFIG,
+				 BMP580_MODE_MASK | BMP580_ODR_DEEPSLEEP_DIS,
+				 BMP580_ODR_DEEPSLEEP_DIS |
+				 FIELD_PREP(BMP580_MODE_MASK, BMP580_MODE_SLEEP));
+	if (ret) {
+		dev_err(data->dev, "failed to change sensor to standby mode\n");
+		return ret;
+	}
+	/* From datasheet's table 4: electrical characteristics */
+	usleep_range(2500, 3000);
+
+	/* Set default DSP mode settings */
+	reg_val = FIELD_PREP(BMP580_DSP_COMP_MASK, BMP580_DSP_PRESS_TEMP_COMP_EN) |
+		  BMP580_DSP_SHDW_IIR_TEMP_EN | BMP580_DSP_SHDW_IIR_PRESS_EN;
+
+	ret = regmap_update_bits(data->regmap, BMP580_REG_DSP_CONFIG,
+				 BMP580_DSP_COMP_MASK |
+				 BMP580_DSP_SHDW_IIR_TEMP_EN |
+				 BMP580_DSP_SHDW_IIR_PRESS_EN, reg_val);
+
+	/* Configure oversampling */
+	reg_val = FIELD_PREP(BMP580_OSR_TEMP_MASK, data->oversampling_temp) |
+		  FIELD_PREP(BMP580_OSR_PRESS_MASK, data->oversampling_press) |
+		  BMP580_OSR_PRESS_EN;
+
+	ret = regmap_update_bits_check(data->regmap, BMP580_REG_OSR_CONFIG,
+				       BMP580_OSR_TEMP_MASK | BMP580_OSR_PRESS_MASK |
+				       BMP580_OSR_PRESS_EN,
+				       reg_val, &aux);
+	if (ret) {
+		dev_err(data->dev, "failed to write oversampling register\n");
+		return ret;
+	}
+	change = change || aux;
+
+	/* Configure output data rate */
+	ret = regmap_update_bits_check(data->regmap, BMP580_REG_ODR_CONFIG, BMP580_ODR_MASK,
+				       FIELD_PREP(BMP580_ODR_MASK, data->sampling_freq),
+				       &aux);
+	if (ret) {
+		dev_err(data->dev, "failed to write ODR configuration register\n");
+		return ret;
+	}
+	change = change || aux;
+
+	/* Set filter data */
+	reg_val = FIELD_PREP(BMP580_DSP_IIR_PRESS_MASK, data->iir_filter_coeff) |
+		  FIELD_PREP(BMP580_DSP_IIR_TEMP_MASK, data->iir_filter_coeff);
+
+	ret = regmap_update_bits_check(data->regmap, BMP580_REG_DSP_IIR,
+				       BMP580_DSP_IIR_PRESS_MASK |
+				       BMP580_DSP_IIR_TEMP_MASK,
+				       reg_val, &aux);
+	if (ret) {
+		dev_err(data->dev, "failed to write config register\n");
+		return ret;
+	}
+	change = change || aux;
+
+	/* Restore sensor to normal operation mode */
+	ret = regmap_write_bits(data->regmap, BMP580_REG_ODR_CONFIG,
+				BMP580_MODE_MASK,
+				FIELD_PREP(BMP580_MODE_MASK, BMP580_MODE_NORMAL));
+	if (ret) {
+		dev_err(data->dev, "failed to set normal mode\n");
+		return ret;
+	}
+	/* From datasheet's table 4: electrical characteristics */
+	usleep_range(3000, 3500);
+
+	if (change) {
+		/*
+		 * Check if ODR and OSR settings are valid or we are
+		 * operating in a degraded mode.
+		 */
+		ret = regmap_read(data->regmap, BMP580_REG_EFF_OSR, &tmp);
+		if (ret) {
+			dev_err(data->dev, "error reading effective OSR register\n");
+			return ret;
+		}
+		if (!(tmp & BMP580_EFF_OSR_VALID_ODR)) {
+			dev_warn(data->dev, "OSR and ODR incompatible settings detected\n");
+			/* Set current OSR settings from data on effective OSR */
+			data->oversampling_temp = FIELD_GET(BMP580_EFF_OSR_TEMP_MASK, tmp);
+			data->oversampling_press = FIELD_GET(BMP580_EFF_OSR_PRESS_MASK, tmp);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static const int bmp580_oversampling_avail[] = { 1, 2, 4, 8, 16, 32, 64, 128 };
+
+const struct bmp280_chip_info bmp580_chip_info = {
+	.id_reg = BMP580_REG_CHIP_ID,
+	.chip_id = BMP580_CHIP_ID,
+	.regmap_config = &bmp580_regmap_config,
+	.start_up_time = 2000,
+	.channels = bmp380_channels,
+	.num_channels = 2,
+
+	.oversampling_temp_avail = bmp580_oversampling_avail,
+	.num_oversampling_temp_avail = ARRAY_SIZE(bmp580_oversampling_avail),
+	.oversampling_temp_default = ilog2(1),
+
+	.oversampling_press_avail = bmp580_oversampling_avail,
+	.num_oversampling_press_avail = ARRAY_SIZE(bmp580_oversampling_avail),
+	.oversampling_press_default = ilog2(4),
+
+	.sampling_freq_avail = bmp580_odr_table,
+	.num_sampling_freq_avail = ARRAY_SIZE(bmp580_odr_table) * 2,
+	.sampling_freq_default = BMP580_ODR_50HZ,
+
+	.iir_filter_coeffs_avail = bmp380_iir_filter_coeffs_avail,
+	.num_iir_filter_coeffs_avail = ARRAY_SIZE(bmp380_iir_filter_coeffs_avail),
+	.iir_filter_coeff_default = 2,
+
+	.chip_config = bmp580_chip_config,
+	.read_temp = bmp580_read_temp,
+	.read_press = bmp580_read_press,
+	.preinit = bmp580_preinit,
 };
+EXPORT_SYMBOL_NS(bmp580_chip_info, IIO_BMP280);
 
 static int bmp180_measure(struct bmp280_data *data, u8 ctrl_meas)
 {
@@ -1467,7 +1872,7 @@ static s32 bmp180_compensate_temp(struct bmp280_data *data, s32 adc_temp)
 	return (data->t_fine + 8) >> 4;
 }
 
-static int bmp180_read_temp(struct bmp280_data *data, int *val)
+static int bmp180_read_temp(struct bmp280_data *data, int *val, int *val2)
 {
 	s32 adc_temp, comp_temp;
 	int ret;
@@ -1555,7 +1960,7 @@ static int bmp180_read_press(struct bmp280_data *data,
 	int ret;
 
 	/* Read and compensate temperature so we get a reading of t_fine. */
-	ret = bmp180_read_temp(data, NULL);
+	ret = bmp180_read_temp(data, NULL, NULL);
 	if (ret)
 		return ret;
 
@@ -1579,8 +1984,10 @@ static int bmp180_chip_config(struct bmp280_data *data)
 static const int bmp180_oversampling_temp_avail[] = { 1 };
 static const int bmp180_oversampling_press_avail[] = { 1, 2, 4, 8 };
 
-static const struct bmp280_chip_info bmp180_chip_info = {
+const struct bmp280_chip_info bmp180_chip_info = {
 	.id_reg = BMP280_REG_ID,
+	.chip_id = BMP180_CHIP_ID,
+	.regmap_config = &bmp180_regmap_config,
 	.start_up_time = 2000,
 	.channels = bmp280_channels,
 	.num_channels = 2,
@@ -1600,6 +2007,7 @@ static const struct bmp280_chip_info bmp180_chip_info = {
 	.read_press = bmp180_read_press,
 	.read_calib = bmp180_read_calib,
 };
+EXPORT_SYMBOL_NS(bmp180_chip_info, IIO_BMP280);
 
 static irqreturn_t bmp085_eoc_irq(int irq, void *d)
 {
@@ -1661,11 +2069,10 @@ static void bmp280_regulators_disable(void *data)
 
 int bmp280_common_probe(struct device *dev,
 			struct regmap *regmap,
-			unsigned int chip,
+			const struct bmp280_chip_info *chip_info,
 			const char *name,
 			int irq)
 {
-	const struct bmp280_chip_info *chip_info;
 	struct iio_dev *indio_dev;
 	struct bmp280_data *data;
 	struct gpio_desc *gpiod;
@@ -1684,22 +2091,6 @@ int bmp280_common_probe(struct device *dev,
 	indio_dev->info = &bmp280_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	switch (chip) {
-	case BMP180_CHIP_ID:
-		chip_info = &bmp180_chip_info;
-		break;
-	case BMP280_CHIP_ID:
-		chip_info = &bmp280_chip_info;
-		break;
-	case BME280_CHIP_ID:
-		chip_info = &bme280_chip_info;
-		break;
-	case BMP380_CHIP_ID:
-		chip_info = &bmp380_chip_info;
-		break;
-	default:
-		return -EINVAL;
-	}
 	data->chip_info = chip_info;
 
 	/* Apply initial values from chip info structure */
@@ -1751,17 +2142,17 @@ int bmp280_common_probe(struct device *dev,
 	ret = regmap_read(regmap, data->chip_info->id_reg, &chip_id);
 	if (ret < 0)
 		return ret;
-	if (chip_id != chip) {
+	if (chip_id != data->chip_info->chip_id) {
 		dev_err(dev, "bad chip id: expected %x got %x\n",
-			chip, chip_id);
+			data->chip_info->chip_id, chip_id);
 		return -EINVAL;
 	}
 
-	/* BMP3xx requires soft-reset as part of initialization */
-	if (chip_id == BMP380_CHIP_ID) {
-		ret = bmp380_cmd(data, BMP380_CMD_SOFT_RESET);
-		if (ret < 0)
-			return ret;
+	if (data->chip_info->preinit) {
+		ret = data->chip_info->preinit(data);
+		if (ret)
+			return dev_err_probe(data->dev, ret,
+					     "error running preinit tasks\n");
 	}
 
 	ret = data->chip_info->chip_config(data);
@@ -1776,10 +2167,12 @@ int bmp280_common_probe(struct device *dev,
 	 * time once. They will not change.
 	 */
 
-	ret = data->chip_info->read_calib(data);
-	if (ret < 0)
-		return dev_err_probe(data->dev, ret,
-				     "failed to read calibration coefficients\n");
+	if (data->chip_info->read_calib) {
+		ret = data->chip_info->read_calib(data);
+		if (ret < 0)
+			return dev_err_probe(data->dev, ret,
+					     "failed to read calibration coefficients\n");
+	}
 
 	/*
 	 * Attempt to grab an optional EOC IRQ - only the BMP085 has this
diff --git a/drivers/iio/pressure/bmp280-i2c.c b/drivers/iio/pressure/bmp280-i2c.c
index 14eab086d24a..567b945e6427 100644
--- a/drivers/iio/pressure/bmp280-i2c.c
+++ b/drivers/iio/pressure/bmp280-i2c.c
@@ -8,25 +8,14 @@
 static int bmp280_i2c_probe(struct i2c_client *client)
 {
 	struct regmap *regmap;
-	const struct regmap_config *regmap_config;
+	const struct bmp280_chip_info *chip_info;
 	const struct i2c_device_id *id = i2c_client_get_device_id(client);
 
-	switch (id->driver_data) {
-	case BMP180_CHIP_ID:
-		regmap_config = &bmp180_regmap_config;
-		break;
-	case BMP280_CHIP_ID:
-	case BME280_CHIP_ID:
-		regmap_config = &bmp280_regmap_config;
-		break;
-	case BMP380_CHIP_ID:
-		regmap_config = &bmp380_regmap_config;
-		break;
-	default:
-		return -EINVAL;
-	}
+	chip_info = device_get_match_data(&client->dev);
+	if (!chip_info)
+		chip_info = (const struct bmp280_chip_info *) id->driver_data;
 
-	regmap = devm_regmap_init_i2c(client, regmap_config);
+	regmap = devm_regmap_init_i2c(client, chip_info->regmap_config);
 	if (IS_ERR(regmap)) {
 		dev_err(&client->dev, "failed to allocate register map\n");
 		return PTR_ERR(regmap);
@@ -34,27 +23,29 @@ static int bmp280_i2c_probe(struct i2c_client *client)
 
 	return bmp280_common_probe(&client->dev,
 				   regmap,
-				   id->driver_data,
+				   chip_info,
 				   id->name,
 				   client->irq);
 }
 
 static const struct of_device_id bmp280_of_i2c_match[] = {
-	{ .compatible = "bosch,bmp085", .data = (void *)BMP180_CHIP_ID },
-	{ .compatible = "bosch,bmp180", .data = (void *)BMP180_CHIP_ID },
-	{ .compatible = "bosch,bmp280", .data = (void *)BMP280_CHIP_ID },
-	{ .compatible = "bosch,bme280", .data = (void *)BME280_CHIP_ID },
-	{ .compatible = "bosch,bmp380", .data = (void *)BMP380_CHIP_ID },
+	{ .compatible = "bosch,bmp085", .data = &bmp180_chip_info },
+	{ .compatible = "bosch,bmp180", .data = &bmp180_chip_info },
+	{ .compatible = "bosch,bmp280", .data = &bmp280_chip_info },
+	{ .compatible = "bosch,bme280", .data = &bme280_chip_info },
+	{ .compatible = "bosch,bmp380", .data = &bmp380_chip_info },
+	{ .compatible = "bosch,bmp580", .data = &bmp580_chip_info },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, bmp280_of_i2c_match);
 
 static const struct i2c_device_id bmp280_i2c_id[] = {
-	{"bmp085", BMP180_CHIP_ID },
-	{"bmp180", BMP180_CHIP_ID },
-	{"bmp280", BMP280_CHIP_ID },
-	{"bme280", BME280_CHIP_ID },
-	{"bmp380", BMP380_CHIP_ID },
+	{"bmp085", (kernel_ulong_t)&bmp180_chip_info },
+	{"bmp180", (kernel_ulong_t)&bmp180_chip_info },
+	{"bmp280", (kernel_ulong_t)&bmp280_chip_info },
+	{"bme280", (kernel_ulong_t)&bme280_chip_info },
+	{"bmp380", (kernel_ulong_t)&bmp380_chip_info },
+	{"bmp580", (kernel_ulong_t)&bmp580_chip_info },
 	{ },
 };
 MODULE_DEVICE_TABLE(i2c, bmp280_i2c_id);
diff --git a/drivers/iio/pressure/bmp280-regmap.c b/drivers/iio/pressure/bmp280-regmap.c
index c98c67970265..3ee56720428c 100644
--- a/drivers/iio/pressure/bmp280-regmap.c
+++ b/drivers/iio/pressure/bmp280-regmap.c
@@ -115,6 +115,54 @@ static bool bmp380_is_volatile_reg(struct device *dev, unsigned int reg)
 	}
 }
 
+static bool bmp580_is_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case BMP580_REG_NVM_DATA_MSB:
+	case BMP580_REG_NVM_DATA_LSB:
+	case BMP580_REG_NVM_ADDR:
+	case BMP580_REG_ODR_CONFIG:
+	case BMP580_REG_OSR_CONFIG:
+	case BMP580_REG_INT_SOURCE:
+	case BMP580_REG_INT_CONFIG:
+	case BMP580_REG_OOR_THR_MSB:
+	case BMP580_REG_OOR_THR_LSB:
+	case BMP580_REG_OOR_CONFIG:
+	case BMP580_REG_OOR_RANGE:
+	case BMP580_REG_IF_CONFIG:
+	case BMP580_REG_FIFO_CONFIG:
+	case BMP580_REG_FIFO_SEL:
+	case BMP580_REG_DSP_CONFIG:
+	case BMP580_REG_DSP_IIR:
+	case BMP580_REG_CMD:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool bmp580_is_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case BMP580_REG_NVM_DATA_MSB:
+	case BMP580_REG_NVM_DATA_LSB:
+	case BMP580_REG_FIFO_COUNT:
+	case BMP580_REG_INT_STATUS:
+	case BMP580_REG_PRESS_XLSB:
+	case BMP580_REG_PRESS_LSB:
+	case BMP580_REG_PRESS_MSB:
+	case BMP580_REG_FIFO_DATA:
+	case BMP580_REG_TEMP_XLSB:
+	case BMP580_REG_TEMP_LSB:
+	case BMP580_REG_TEMP_MSB:
+	case BMP580_REG_EFF_OSR:
+	case BMP580_REG_STATUS:
+		return true;
+	default:
+		return false;
+	}
+}
+
 const struct regmap_config bmp280_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
@@ -138,3 +186,15 @@ const struct regmap_config bmp380_regmap_config = {
 	.volatile_reg = bmp380_is_volatile_reg,
 };
 EXPORT_SYMBOL_NS(bmp380_regmap_config, IIO_BMP280);
+
+const struct regmap_config bmp580_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+
+	.max_register = BMP580_REG_CMD,
+	.cache_type = REGCACHE_RBTREE,
+
+	.writeable_reg = bmp580_is_writeable_reg,
+	.volatile_reg = bmp580_is_volatile_reg,
+};
+EXPORT_SYMBOL_NS(bmp580_regmap_config, IIO_BMP280);
diff --git a/drivers/iio/pressure/bmp280-spi.c b/drivers/iio/pressure/bmp280-spi.c
index 011c68e07ebf..1dff9bb7c4e9 100644
--- a/drivers/iio/pressure/bmp280-spi.c
+++ b/drivers/iio/pressure/bmp280-spi.c
@@ -47,8 +47,8 @@ static struct regmap_bus bmp280_regmap_bus = {
 static int bmp280_spi_probe(struct spi_device *spi)
 {
 	const struct spi_device_id *id = spi_get_device_id(spi);
+	const struct bmp280_chip_info *chip_info;
 	struct regmap *regmap;
-	const struct regmap_config *regmap_config;
 	int ret;
 
 	spi->bits_per_word = 8;
@@ -58,25 +58,14 @@ static int bmp280_spi_probe(struct spi_device *spi)
 		return ret;
 	}
 
-	switch (id->driver_data) {
-	case BMP180_CHIP_ID:
-		regmap_config = &bmp180_regmap_config;
-		break;
-	case BMP280_CHIP_ID:
-	case BME280_CHIP_ID:
-		regmap_config = &bmp280_regmap_config;
-		break;
-	case BMP380_CHIP_ID:
-		regmap_config = &bmp380_regmap_config;
-		break;
-	default:
-		return -EINVAL;
-	}
+	chip_info = device_get_match_data(&spi->dev);
+	if (!chip_info)
+		chip_info = (const struct bmp280_chip_info *) id->driver_data;
 
 	regmap = devm_regmap_init(&spi->dev,
 				  &bmp280_regmap_bus,
 				  &spi->dev,
-				  regmap_config);
+				  chip_info->regmap_config);
 	if (IS_ERR(regmap)) {
 		dev_err(&spi->dev, "failed to allocate register map\n");
 		return PTR_ERR(regmap);
@@ -84,28 +73,30 @@ static int bmp280_spi_probe(struct spi_device *spi)
 
 	return bmp280_common_probe(&spi->dev,
 				   regmap,
-				   id->driver_data,
+				   chip_info,
 				   id->name,
 				   spi->irq);
 }
 
 static const struct of_device_id bmp280_of_spi_match[] = {
-	{ .compatible = "bosch,bmp085", },
-	{ .compatible = "bosch,bmp180", },
-	{ .compatible = "bosch,bmp181", },
-	{ .compatible = "bosch,bmp280", },
-	{ .compatible = "bosch,bme280", },
-	{ .compatible = "bosch,bmp380", },
+	{ .compatible = "bosch,bmp085", .data = &bmp180_chip_info },
+	{ .compatible = "bosch,bmp180", .data = &bmp180_chip_info },
+	{ .compatible = "bosch,bmp181", .data = &bmp180_chip_info },
+	{ .compatible = "bosch,bmp280", .data = &bmp280_chip_info },
+	{ .compatible = "bosch,bme280", .data = &bmp280_chip_info },
+	{ .compatible = "bosch,bmp380", .data = &bmp380_chip_info },
+	{ .compatible = "bosch,bmp580", .data = &bmp580_chip_info },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, bmp280_of_spi_match);
 
 static const struct spi_device_id bmp280_spi_id[] = {
-	{ "bmp180", BMP180_CHIP_ID },
-	{ "bmp181", BMP180_CHIP_ID },
-	{ "bmp280", BMP280_CHIP_ID },
-	{ "bme280", BME280_CHIP_ID },
-	{ "bmp380", BMP380_CHIP_ID },
+	{ "bmp180", (kernel_ulong_t)&bmp180_chip_info },
+	{ "bmp181", (kernel_ulong_t)&bmp180_chip_info },
+	{ "bmp280", (kernel_ulong_t)&bmp280_chip_info },
+	{ "bme280", (kernel_ulong_t)&bmp280_chip_info },
+	{ "bmp380", (kernel_ulong_t)&bmp380_chip_info },
+	{ "bmp580", (kernel_ulong_t)&bmp580_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bmp280_spi_id);
diff --git a/drivers/iio/pressure/bmp280.h b/drivers/iio/pressure/bmp280.h
index c791325c7416..5c0563ce7572 100644
--- a/drivers/iio/pressure/bmp280.h
+++ b/drivers/iio/pressure/bmp280.h
@@ -1,7 +1,114 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #include <linux/bitops.h>
 #include <linux/device.h>
+#include <linux/iio/iio.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+
+
+/* BMP580 specific registers */
+#define BMP580_REG_CMD			0x7E
+#define BMP580_REG_EFF_OSR		0x38
+#define BMP580_REG_ODR_CONFIG		0x37
+#define BMP580_REG_OSR_CONFIG		0x36
+#define BMP580_REG_IF_CONFIG		0x13
+#define BMP580_REG_REV_ID		0x02
+#define BMP580_REG_CHIP_ID		0x01
+/* OOR allows to configure a pressure alarm */
+#define BMP580_REG_OOR_CONFIG		0x35
+#define BMP580_REG_OOR_RANGE		0x34
+#define BMP580_REG_OOR_THR_MSB		0x33
+#define BMP580_REG_OOR_THR_LSB		0x32
+/* DSP registers (IIR filters) */
+#define BMP580_REG_DSP_IIR		0x31
+#define BMP580_REG_DSP_CONFIG		0x30
+/* NVM access registers */
+#define BMP580_REG_NVM_DATA_MSB		0x2D
+#define BMP580_REG_NVM_DATA_LSB		0x2C
+#define BMP580_REG_NVM_ADDR		0x2B
+/* Status registers */
+#define BMP580_REG_STATUS		0x28
+#define BMP580_REG_INT_STATUS		0x27
+#define BMP580_REG_CHIP_STATUS		0x11
+/* Data registers */
+#define BMP580_REG_FIFO_DATA		0x29
+#define BMP580_REG_PRESS_MSB		0x22
+#define BMP580_REG_PRESS_LSB		0x21
+#define BMP580_REG_PRESS_XLSB		0x20
+#define BMP580_REG_TEMP_MSB		0x1F
+#define BMP580_REG_TEMP_LSB		0x1E
+#define BMP580_REG_TEMP_XLSB		0x1D
+/* FIFO config registers */
+#define BMP580_REG_FIFO_SEL		0x18
+#define BMP580_REG_FIFO_COUNT		0x17
+#define BMP580_REG_FIFO_CONFIG		0x16
+/* Interruptions config registers */
+#define BMP580_REG_INT_SOURCE		0x15
+#define BMP580_REG_INT_CONFIG		0x14
+
+#define BMP580_CMD_NOOP			0x00
+#define BMP580_CMD_EXTMODE_SEQ_0	0x73
+#define BMP580_CMD_EXTMODE_SEQ_1	0xB4
+#define BMP580_CMD_EXTMODE_SEQ_2	0x69
+#define BMP580_CMD_NVM_OP_SEQ_0		0x5D
+#define BMP580_CMD_NVM_READ_SEQ_1	0xA5
+#define BMP580_CMD_NVM_WRITE_SEQ_1	0xA0
+#define BMP580_CMD_SOFT_RESET		0xB6
+
+#define BMP580_INT_STATUS_POR_MASK	BIT(4)
+
+#define BMP580_STATUS_CORE_RDY_MASK	BIT(0)
+#define BMP580_STATUS_NVM_RDY_MASK	BIT(1)
+#define BMP580_STATUS_NVM_ERR_MASK	BIT(2)
+#define BMP580_STATUS_NVM_CMD_ERR_MASK	BIT(3)
+
+#define BMP580_OSR_PRESS_MASK		GENMASK(5, 3)
+#define BMP580_OSR_TEMP_MASK		GENMASK(2, 0)
+#define BMP580_OSR_PRESS_EN		BIT(6)
+#define BMP580_EFF_OSR_PRESS_MASK	GENMASK(5, 3)
+#define BMP580_EFF_OSR_TEMP_MASK	GENMASK(2, 0)
+#define BMP580_EFF_OSR_VALID_ODR	BIT(7)
+
+#define BMP580_ODR_MASK			GENMASK(6, 2)
+#define BMP580_MODE_MASK		GENMASK(1, 0)
+#define BMP580_MODE_SLEEP		0
+#define BMP580_MODE_NORMAL		1
+#define BMP580_MODE_FORCED		2
+#define BMP580_MODE_CONTINOUS		3
+#define BMP580_ODR_DEEPSLEEP_DIS	BIT(7)
+
+#define BMP580_DSP_COMP_MASK		GENMASK(1, 0)
+#define BMP580_DSP_COMP_DIS		0
+#define BMP580_DSP_TEMP_COMP_EN		1
+/*
+ * In section 7.27 of datasheet, modes 2 and 3 are technically the same.
+ * Pressure compensation means also enabling temperature compensation
+ */
+#define BMP580_DSP_PRESS_COMP_EN	2
+#define BMP580_DSP_PRESS_TEMP_COMP_EN	3
+#define BMP580_DSP_IIR_FORCED_FLUSH	BIT(2)
+#define BMP580_DSP_SHDW_IIR_TEMP_EN	BIT(3)
+#define BMP580_DSP_FIFO_IIR_TEMP_EN	BIT(4)
+#define BMP580_DSP_SHDW_IIR_PRESS_EN	BIT(5)
+#define BMP580_DSP_FIFO_IIR_PRESS_EN	BIT(6)
+#define BMP580_DSP_OOR_IIR_PRESS_EN	BIT(7)
+
+#define BMP580_DSP_IIR_PRESS_MASK	GENMASK(5, 3)
+#define BMP580_DSP_IIR_TEMP_MASK	GENMASK(2, 0)
+#define BMP580_FILTER_OFF		0
+#define BMP580_FILTER_1X		1
+#define BMP580_FILTER_3X		2
+#define BMP580_FILTER_7X		3
+#define BMP580_FILTER_15X		4
+#define BMP580_FILTER_31X		5
+#define BMP580_FILTER_63X		6
+#define BMP580_FILTER_127X		7
+
+#define BMP580_NVM_ROW_ADDR_MASK	GENMASK(5, 0)
+#define BMP580_NVM_PROG_EN		BIT(6)
+
+#define BMP580_TEMP_SKIPPED		0x7f7f7f
+#define BMP580_PRESS_SKIPPED		0x7f7f7f
 
 /* BMP380 specific registers */
 #define BMP380_REG_CMD			0x7E
@@ -181,6 +288,8 @@
 #define BMP280_REG_ID			0xD0
 
 #define BMP380_CHIP_ID			0x50
+#define BMP580_CHIP_ID			0x50
+#define BMP580_CHIP_ID_ALT		0x51
 #define BMP180_CHIP_ID			0x55
 #define BMP280_CHIP_ID			0x58
 #define BME280_CHIP_ID			0x60
@@ -191,15 +300,177 @@
 #define BMP280_PRESS_SKIPPED		0x80000
 #define BMP280_HUMIDITY_SKIPPED		0x8000
 
+/* Core exported structs */
+
+static const char *const bmp280_supply_names[] = {
+	"vddd", "vdda"
+};
+
+#define BMP280_NUM_SUPPLIES ARRAY_SIZE(bmp280_supply_names)
+
+struct bmp180_calib {
+	s16 AC1;
+	s16 AC2;
+	s16 AC3;
+	u16 AC4;
+	u16 AC5;
+	u16 AC6;
+	s16 B1;
+	s16 B2;
+	s16 MB;
+	s16 MC;
+	s16 MD;
+};
+
+/* See datasheet Section 4.2.2. */
+struct bmp280_calib {
+	u16 T1;
+	s16 T2;
+	s16 T3;
+	u16 P1;
+	s16 P2;
+	s16 P3;
+	s16 P4;
+	s16 P5;
+	s16 P6;
+	s16 P7;
+	s16 P8;
+	s16 P9;
+	u8  H1;
+	s16 H2;
+	u8  H3;
+	s16 H4;
+	s16 H5;
+	s8  H6;
+};
+
+/* See datasheet Section 3.11.1. */
+struct bmp380_calib {
+	u16 T1;
+	u16 T2;
+	s8  T3;
+	s16 P1;
+	s16 P2;
+	s8  P3;
+	s8  P4;
+	u16 P5;
+	u16 P6;
+	s8  P7;
+	s8  P8;
+	s16 P9;
+	s8  P10;
+	s8  P11;
+};
+
+struct bmp280_data {
+	struct device *dev;
+	struct mutex lock;
+	struct regmap *regmap;
+	struct completion done;
+	bool use_eoc;
+	const struct bmp280_chip_info *chip_info;
+	union {
+		struct bmp180_calib bmp180;
+		struct bmp280_calib bmp280;
+		struct bmp380_calib bmp380;
+	} calib;
+	struct regulator_bulk_data supplies[BMP280_NUM_SUPPLIES];
+	unsigned int start_up_time; /* in microseconds */
+
+	/* log of base 2 of oversampling rate */
+	u8 oversampling_press;
+	u8 oversampling_temp;
+	u8 oversampling_humid;
+	u8 iir_filter_coeff;
+
+	/*
+	 * BMP380 devices introduce sampling frequency configuration. See
+	 * datasheet sections 3.3.3. and 4.3.19 for more details.
+	 *
+	 * BMx280 devices allowed indirect configuration of sampling frequency
+	 * changing the t_standby duration between measurements, as detailed on
+	 * section 3.6.3 of the datasheet.
+	 */
+	int sampling_freq;
+
+	/*
+	 * Carryover value from temperature conversion, used in pressure
+	 * calculation.
+	 */
+	s32 t_fine;
+
+	/*
+	 * DMA (thus cache coherency maintenance) may require the
+	 * transfer buffers to live in their own cache lines.
+	 */
+	union {
+		/* Sensor data buffer */
+		u8 buf[3];
+		/* Calibration data buffers */
+		__le16 bmp280_cal_buf[BMP280_CONTIGUOUS_CALIB_REGS / 2];
+		__be16 bmp180_cal_buf[BMP180_REG_CALIB_COUNT / 2];
+		u8 bmp380_cal_buf[BMP380_CALIB_REG_COUNT];
+		/* Miscellaneous, endianess-aware data buffers */
+		__le16 le16;
+		__be16 be16;
+	} __aligned(IIO_DMA_MINALIGN);
+};
+
+struct bmp280_chip_info {
+	unsigned int id_reg;
+	const unsigned int chip_id;
+
+	const struct regmap_config *regmap_config;
+
+	const struct iio_chan_spec *channels;
+	int num_channels;
+	unsigned int start_up_time;
+
+	const int *oversampling_temp_avail;
+	int num_oversampling_temp_avail;
+	int oversampling_temp_default;
+
+	const int *oversampling_press_avail;
+	int num_oversampling_press_avail;
+	int oversampling_press_default;
+
+	const int *oversampling_humid_avail;
+	int num_oversampling_humid_avail;
+	int oversampling_humid_default;
+
+	const int *iir_filter_coeffs_avail;
+	int num_iir_filter_coeffs_avail;
+	int iir_filter_coeff_default;
+
+	const int (*sampling_freq_avail)[2];
+	int num_sampling_freq_avail;
+	int sampling_freq_default;
+
+	int (*chip_config)(struct bmp280_data *);
+	int (*read_temp)(struct bmp280_data *, int *, int *);
+	int (*read_press)(struct bmp280_data *, int *, int *);
+	int (*read_humid)(struct bmp280_data *, int *, int *);
+	int (*read_calib)(struct bmp280_data *);
+	int (*preinit)(struct bmp280_data *);
+};
+
+/* Chip infos for each variant */
+extern const struct bmp280_chip_info bmp180_chip_info;
+extern const struct bmp280_chip_info bmp280_chip_info;
+extern const struct bmp280_chip_info bme280_chip_info;
+extern const struct bmp280_chip_info bmp380_chip_info;
+extern const struct bmp280_chip_info bmp580_chip_info;
+
 /* Regmap configurations */
 extern const struct regmap_config bmp180_regmap_config;
 extern const struct regmap_config bmp280_regmap_config;
 extern const struct regmap_config bmp380_regmap_config;
+extern const struct regmap_config bmp580_regmap_config;
 
 /* Probe called from different transports */
 int bmp280_common_probe(struct device *dev,
 			struct regmap *regmap,
-			unsigned int chip,
+			const struct bmp280_chip_info *,
 			const char *name,
 			int irq);
 
diff --git a/drivers/iio/pressure/zpa2326.c b/drivers/iio/pressure/zpa2326.c
index 67119a9b95fc..421e059d1f19 100644
--- a/drivers/iio/pressure/zpa2326.c
+++ b/drivers/iio/pressure/zpa2326.c
@@ -829,7 +829,7 @@ static irqreturn_t zpa2326_handle_threaded_irq(int irq, void *data)
 	}
 
 	/* New sample available: dispatch internal trigger consumers. */
-	iio_trigger_poll_chained(priv->trigger);
+	iio_trigger_poll_nested(priv->trigger);
 
 	if (cont)
 		/*
diff --git a/drivers/iio/proximity/as3935.c b/drivers/iio/proximity/as3935.c
index ebc95cf8f5f4..96fa97451cbf 100644
--- a/drivers/iio/proximity/as3935.c
+++ b/drivers/iio/proximity/as3935.c
@@ -257,7 +257,7 @@ static void as3935_event_work(struct work_struct *work)
 
 	switch (val) {
 	case AS3935_EVENT_INT:
-		iio_trigger_poll_chained(st->trig);
+		iio_trigger_poll_nested(st->trig);
 		break;
 	case AS3935_DISTURB_INT:
 	case AS3935_NOISE_INT:
diff --git a/drivers/iio/proximity/sx9324.c b/drivers/iio/proximity/sx9324.c
index 977cf17cec52..9a40ca32bb1c 100644
--- a/drivers/iio/proximity/sx9324.c
+++ b/drivers/iio/proximity/sx9324.c
@@ -783,73 +783,75 @@ static int sx9324_write_raw(struct iio_dev *indio_dev,
 
 static const struct sx_common_reg_default sx9324_default_regs[] = {
 	{ SX9324_REG_IRQ_MSK, 0x00 },
-	{ SX9324_REG_IRQ_CFG0, 0x00 },
-	{ SX9324_REG_IRQ_CFG1, SX9324_REG_IRQ_CFG1_FAILCOND },
-	{ SX9324_REG_IRQ_CFG2, 0x00 },
-	{ SX9324_REG_GNRL_CTRL0, SX9324_REG_GNRL_CTRL0_SCANPERIOD_100MS },
+	{ SX9324_REG_IRQ_CFG0, 0x00, "irq_cfg0" },
+	{ SX9324_REG_IRQ_CFG1, SX9324_REG_IRQ_CFG1_FAILCOND, "irq_cfg1" },
+	{ SX9324_REG_IRQ_CFG2, 0x00, "irq_cfg2" },
+	{ SX9324_REG_GNRL_CTRL0, SX9324_REG_GNRL_CTRL0_SCANPERIOD_100MS, "gnrl_ctrl0" },
 	/*
 	 * The lower 4 bits should not be set as it enable sensors measurements.
 	 * Turning the detection on before the configuration values are set to
 	 * good values can cause the device to return erroneous readings.
 	 */
-	{ SX9324_REG_GNRL_CTRL1, SX9324_REG_GNRL_CTRL1_PAUSECTRL },
+	{ SX9324_REG_GNRL_CTRL1, SX9324_REG_GNRL_CTRL1_PAUSECTRL, "gnrl_ctrl1" },
 
-	{ SX9324_REG_AFE_CTRL0, SX9324_REG_AFE_CTRL0_RINT_LOWEST },
-	{ SX9324_REG_AFE_CTRL3, 0x00 },
+	{ SX9324_REG_AFE_CTRL0, SX9324_REG_AFE_CTRL0_RINT_LOWEST, "afe_ctrl0" },
+	{ SX9324_REG_AFE_CTRL3, 0x00, "afe_ctrl3" },
 	{ SX9324_REG_AFE_CTRL4, SX9324_REG_AFE_CTRL4_FREQ_83_33HZ |
-		SX9324_REG_AFE_CTRL4_RES_100 },
-	{ SX9324_REG_AFE_CTRL6, 0x00 },
+		SX9324_REG_AFE_CTRL4_RES_100, "afe_ctrl4" },
+	{ SX9324_REG_AFE_CTRL6, 0x00, "afe_ctrl6" },
 	{ SX9324_REG_AFE_CTRL7, SX9324_REG_AFE_CTRL4_FREQ_83_33HZ |
-		SX9324_REG_AFE_CTRL4_RES_100 },
+		SX9324_REG_AFE_CTRL4_RES_100, "afe_ctrl7" },
 
 	/* TODO(gwendal): PHx use chip default or all grounded? */
-	{ SX9324_REG_AFE_PH0, 0x29 },
-	{ SX9324_REG_AFE_PH1, 0x26 },
-	{ SX9324_REG_AFE_PH2, 0x1a },
-	{ SX9324_REG_AFE_PH3, 0x16 },
+	{ SX9324_REG_AFE_PH0, 0x29, "afe_ph0" },
+	{ SX9324_REG_AFE_PH1, 0x26, "afe_ph1" },
+	{ SX9324_REG_AFE_PH2, 0x1a, "afe_ph2" },
+	{ SX9324_REG_AFE_PH3, 0x16, "afe_ph3" },
 
 	{ SX9324_REG_AFE_CTRL8, SX9324_REG_AFE_CTRL8_RESERVED |
-		SX9324_REG_AFE_CTRL8_RESFILTIN_4KOHM },
-	{ SX9324_REG_AFE_CTRL9, SX9324_REG_AFE_CTRL9_AGAIN_1 },
+		SX9324_REG_AFE_CTRL8_RESFILTIN_4KOHM, "afe_ctrl8" },
+	{ SX9324_REG_AFE_CTRL9, SX9324_REG_AFE_CTRL9_AGAIN_1, "afe_ctrl9" },
 
 	{ SX9324_REG_PROX_CTRL0,
 		SX9324_REG_PROX_CTRL0_GAIN_1 << SX9324_REG_PROX_CTRL0_GAIN_SHIFT |
-		SX9324_REG_PROX_CTRL0_RAWFILT_1P50 },
+		SX9324_REG_PROX_CTRL0_RAWFILT_1P50, "prox_ctrl0" },
 	{ SX9324_REG_PROX_CTRL1,
 		SX9324_REG_PROX_CTRL0_GAIN_1 << SX9324_REG_PROX_CTRL0_GAIN_SHIFT |
-		SX9324_REG_PROX_CTRL0_RAWFILT_1P50 },
-	{ SX9324_REG_PROX_CTRL2, SX9324_REG_PROX_CTRL2_AVGNEG_THRESH_16K },
+		SX9324_REG_PROX_CTRL0_RAWFILT_1P50, "prox_ctrl1" },
+	{ SX9324_REG_PROX_CTRL2, SX9324_REG_PROX_CTRL2_AVGNEG_THRESH_16K, "prox_ctrl2" },
 	{ SX9324_REG_PROX_CTRL3, SX9324_REG_PROX_CTRL3_AVGDEB_2SAMPLES |
-		SX9324_REG_PROX_CTRL3_AVGPOS_THRESH_16K },
+		SX9324_REG_PROX_CTRL3_AVGPOS_THRESH_16K, "prox_ctrl3" },
 	{ SX9324_REG_PROX_CTRL4, SX9324_REG_PROX_CTRL4_AVGNEG_FILT_2 |
-		SX9324_REG_PROX_CTRL4_AVGPOS_FILT_256 },
-	{ SX9324_REG_PROX_CTRL5, 0x00 },
-	{ SX9324_REG_PROX_CTRL6, SX9324_REG_PROX_CTRL6_PROXTHRESH_32 },
-	{ SX9324_REG_PROX_CTRL7, SX9324_REG_PROX_CTRL6_PROXTHRESH_32 },
-	{ SX9324_REG_ADV_CTRL0, 0x00 },
-	{ SX9324_REG_ADV_CTRL1, 0x00 },
-	{ SX9324_REG_ADV_CTRL2, 0x00 },
-	{ SX9324_REG_ADV_CTRL3, 0x00 },
-	{ SX9324_REG_ADV_CTRL4, 0x00 },
+		SX9324_REG_PROX_CTRL4_AVGPOS_FILT_256, "prox_ctrl4" },
+	{ SX9324_REG_PROX_CTRL5, 0x00, "prox_ctrl5" },
+	{ SX9324_REG_PROX_CTRL6, SX9324_REG_PROX_CTRL6_PROXTHRESH_32, "prox_ctrl6" },
+	{ SX9324_REG_PROX_CTRL7, SX9324_REG_PROX_CTRL6_PROXTHRESH_32, "prox_ctrl7" },
+	{ SX9324_REG_ADV_CTRL0, 0x00, "adv_ctrl0" },
+	{ SX9324_REG_ADV_CTRL1, 0x00, "adv_ctrl1" },
+	{ SX9324_REG_ADV_CTRL2, 0x00, "adv_ctrl2" },
+	{ SX9324_REG_ADV_CTRL3, 0x00, "adv_ctrl3" },
+	{ SX9324_REG_ADV_CTRL4, 0x00, "adv_ctrl4" },
 	{ SX9324_REG_ADV_CTRL5, SX9324_REG_ADV_CTRL5_STARTUP_SENSOR_1 |
-		SX9324_REG_ADV_CTRL5_STARTUP_METHOD_1 },
-	{ SX9324_REG_ADV_CTRL6, 0x00 },
-	{ SX9324_REG_ADV_CTRL7, 0x00 },
-	{ SX9324_REG_ADV_CTRL8, 0x00 },
-	{ SX9324_REG_ADV_CTRL9, 0x00 },
+		SX9324_REG_ADV_CTRL5_STARTUP_METHOD_1, "adv_ctrl5" },
+	{ SX9324_REG_ADV_CTRL6, 0x00, "adv_ctrl6" },
+	{ SX9324_REG_ADV_CTRL7, 0x00, "adv_ctrl7" },
+	{ SX9324_REG_ADV_CTRL8, 0x00, "adv_ctrl8" },
+	{ SX9324_REG_ADV_CTRL9, 0x00, "adv_ctrl9" },
 	/* Body/Table threshold */
-	{ SX9324_REG_ADV_CTRL10, 0x00 },
-	{ SX9324_REG_ADV_CTRL11, 0x00 },
-	{ SX9324_REG_ADV_CTRL12, 0x00 },
+	{ SX9324_REG_ADV_CTRL10, 0x00, "adv_ctrl10" },
+	{ SX9324_REG_ADV_CTRL11, 0x00, "adv_ctrl11" },
+	{ SX9324_REG_ADV_CTRL12, 0x00, "adv_ctrl12" },
 	/* TODO(gwendal): SAR currenly disabled */
-	{ SX9324_REG_ADV_CTRL13, 0x00 },
-	{ SX9324_REG_ADV_CTRL14, 0x00 },
-	{ SX9324_REG_ADV_CTRL15, 0x00 },
-	{ SX9324_REG_ADV_CTRL16, 0x00 },
-	{ SX9324_REG_ADV_CTRL17, 0x00 },
-	{ SX9324_REG_ADV_CTRL18, 0x00 },
-	{ SX9324_REG_ADV_CTRL19, SX9324_REG_ADV_CTRL19_HIGHT_FAILURE_THRESH_SATURATION },
-	{ SX9324_REG_ADV_CTRL20, SX9324_REG_ADV_CTRL19_HIGHT_FAILURE_THRESH_SATURATION },
+	{ SX9324_REG_ADV_CTRL13, 0x00, "adv_ctrl13" },
+	{ SX9324_REG_ADV_CTRL14, 0x00, "adv_ctrl14" },
+	{ SX9324_REG_ADV_CTRL15, 0x00, "adv_ctrl15" },
+	{ SX9324_REG_ADV_CTRL16, 0x00, "adv_ctrl16" },
+	{ SX9324_REG_ADV_CTRL17, 0x00, "adv_ctrl17" },
+	{ SX9324_REG_ADV_CTRL18, 0x00, "adv_ctrl18" },
+	{ SX9324_REG_ADV_CTRL19,
+		SX9324_REG_ADV_CTRL19_HIGHT_FAILURE_THRESH_SATURATION, "adv_ctrl19" },
+	{ SX9324_REG_ADV_CTRL20,
+		SX9324_REG_ADV_CTRL19_HIGHT_FAILURE_THRESH_SATURATION, "adv_ctrl20" },
 };
 
 /* Activate all channels and perform an initial compensation. */
@@ -889,13 +891,15 @@ sx9324_get_default_reg(struct device *dev, int idx,
 	const char *res;
 
 	memcpy(reg_def, &sx9324_default_regs[idx], sizeof(*reg_def));
+
+	sx_common_get_raw_register_config(dev, reg_def);
 	switch (reg_def->reg) {
 	case SX9324_REG_AFE_PH0:
 	case SX9324_REG_AFE_PH1:
 	case SX9324_REG_AFE_PH2:
 	case SX9324_REG_AFE_PH3:
 		ph = reg_def->reg - SX9324_REG_AFE_PH0;
-		scnprintf(prop, ARRAY_SIZE(prop), "semtech,ph%d-pin", ph);
+		snprintf(prop, ARRAY_SIZE(prop), "semtech,ph%d-pin", ph);
 
 		count = device_property_count_u32(dev, prop);
 		if (count != ARRAY_SIZE(pin_defs))
diff --git a/drivers/iio/proximity/sx9360.c b/drivers/iio/proximity/sx9360.c
index 6e19d22e6a01..a50d9176411a 100644
--- a/drivers/iio/proximity/sx9360.c
+++ b/drivers/iio/proximity/sx9360.c
@@ -663,37 +663,37 @@ static int sx9360_write_raw(struct iio_dev *indio_dev,
 
 static const struct sx_common_reg_default sx9360_default_regs[] = {
 	{ SX9360_REG_IRQ_MSK, 0x00 },
-	{ SX9360_REG_IRQ_CFG, 0x00 },
+	{ SX9360_REG_IRQ_CFG, 0x00, "irq_cfg" },
 	/*
 	 * The lower 2 bits should not be set as it enable sensors measurements.
 	 * Turning the detection on before the configuration values are set to
 	 * good values can cause the device to return erroneous readings.
 	 */
-	{ SX9360_REG_GNRL_CTRL0, 0x00 },
-	{ SX9360_REG_GNRL_CTRL1, 0x00 },
-	{ SX9360_REG_GNRL_CTRL2, SX9360_REG_GNRL_CTRL2_PERIOD_102MS },
+	{ SX9360_REG_GNRL_CTRL0, 0x00, "gnrl_ctrl0" },
+	{ SX9360_REG_GNRL_CTRL1, 0x00, "gnrl_ctrl1" },
+	{ SX9360_REG_GNRL_CTRL2, SX9360_REG_GNRL_CTRL2_PERIOD_102MS, "gnrl_ctrl2" },
 
-	{ SX9360_REG_AFE_CTRL1, SX9360_REG_AFE_CTRL1_RESFILTIN_0OHMS },
+	{ SX9360_REG_AFE_CTRL1, SX9360_REG_AFE_CTRL1_RESFILTIN_0OHMS, "afe_ctrl0" },
 	{ SX9360_REG_AFE_PARAM0_PHR, SX9360_REG_AFE_PARAM0_RSVD |
-		SX9360_REG_AFE_PARAM0_RESOLUTION_128 },
+		SX9360_REG_AFE_PARAM0_RESOLUTION_128, "afe_param0_phr" },
 	{ SX9360_REG_AFE_PARAM1_PHR, SX9360_REG_AFE_PARAM1_AGAIN_PHM_6PF |
-		SX9360_REG_AFE_PARAM1_FREQ_83_33HZ },
+		SX9360_REG_AFE_PARAM1_FREQ_83_33HZ, "afe_param1_phr" },
 	{ SX9360_REG_AFE_PARAM0_PHM, SX9360_REG_AFE_PARAM0_RSVD |
-		SX9360_REG_AFE_PARAM0_RESOLUTION_128 },
+		SX9360_REG_AFE_PARAM0_RESOLUTION_128, "afe_param0_phm" },
 	{ SX9360_REG_AFE_PARAM1_PHM, SX9360_REG_AFE_PARAM1_AGAIN_PHM_6PF |
-		SX9360_REG_AFE_PARAM1_FREQ_83_33HZ },
+		SX9360_REG_AFE_PARAM1_FREQ_83_33HZ, "afe_param1_phm" },
 
 	{ SX9360_REG_PROX_CTRL0_PHR, SX9360_REG_PROX_CTRL0_GAIN_1 |
-		SX9360_REG_PROX_CTRL0_RAWFILT_1P50 },
+		SX9360_REG_PROX_CTRL0_RAWFILT_1P50, "prox_ctrl0_phr" },
 	{ SX9360_REG_PROX_CTRL0_PHM, SX9360_REG_PROX_CTRL0_GAIN_1 |
-		SX9360_REG_PROX_CTRL0_RAWFILT_1P50 },
-	{ SX9360_REG_PROX_CTRL1, SX9360_REG_PROX_CTRL1_AVGNEG_THRESH_16K },
+		SX9360_REG_PROX_CTRL0_RAWFILT_1P50, "prox_ctrl0_phm" },
+	{ SX9360_REG_PROX_CTRL1, SX9360_REG_PROX_CTRL1_AVGNEG_THRESH_16K, "prox_ctrl1" },
 	{ SX9360_REG_PROX_CTRL2, SX9360_REG_PROX_CTRL2_AVGDEB_2SAMPLES |
-		SX9360_REG_PROX_CTRL2_AVGPOS_THRESH_16K },
+		SX9360_REG_PROX_CTRL2_AVGPOS_THRESH_16K, "prox_ctrl2" },
 	{ SX9360_REG_PROX_CTRL3, SX9360_REG_PROX_CTRL3_AVGNEG_FILT_2 |
-		SX9360_REG_PROX_CTRL3_AVGPOS_FILT_256 },
-	{ SX9360_REG_PROX_CTRL4, 0x00 },
-	{ SX9360_REG_PROX_CTRL5, SX9360_REG_PROX_CTRL5_PROXTHRESH_32 },
+		SX9360_REG_PROX_CTRL3_AVGPOS_FILT_256, "prox_ctrl3" },
+	{ SX9360_REG_PROX_CTRL4, 0x00, "prox_ctrl4" },
+	{ SX9360_REG_PROX_CTRL5, SX9360_REG_PROX_CTRL5_PROXTHRESH_32, "prox_ctrl5" },
 };
 
 /* Activate all channels and perform an initial compensation. */
diff --git a/drivers/iio/proximity/sx9500.c b/drivers/iio/proximity/sx9500.c
index 8794e75e5bf9..9b2cfcade6a4 100644
--- a/drivers/iio/proximity/sx9500.c
+++ b/drivers/iio/proximity/sx9500.c
@@ -1051,8 +1051,8 @@ MODULE_DEVICE_TABLE(i2c, sx9500_id);
 static struct i2c_driver sx9500_driver = {
 	.driver = {
 		.name	= SX9500_DRIVER_NAME,
-		.acpi_match_table = ACPI_PTR(sx9500_acpi_match),
-		.of_match_table = of_match_ptr(sx9500_of_match),
+		.acpi_match_table = sx9500_acpi_match,
+		.of_match_table = sx9500_of_match,
 		.pm = pm_sleep_ptr(&sx9500_pm_ops),
 	},
 	.probe_new	= sx9500_probe,
diff --git a/drivers/iio/proximity/sx_common.c b/drivers/iio/proximity/sx_common.c
index eba9256730ec..fe07d1444ac3 100644
--- a/drivers/iio/proximity/sx_common.c
+++ b/drivers/iio/proximity/sx_common.c
@@ -424,6 +424,27 @@ static const struct iio_buffer_setup_ops sx_common_buffer_setup_ops = {
 	.postdisable = sx_common_buffer_postdisable,
 };
 
+void sx_common_get_raw_register_config(struct device *dev,
+				       struct sx_common_reg_default *reg_def)
+{
+#ifdef CONFIG_ACPI
+	struct acpi_device *adev = ACPI_COMPANION(dev);
+	u32 raw = 0, ret;
+	char prop[80];
+
+	if (!reg_def->property || !adev)
+		return;
+
+	snprintf(prop, ARRAY_SIZE(prop), "%s,reg_%s", acpi_device_hid(adev), reg_def->property);
+	ret = device_property_read_u32(dev, prop, &raw);
+	if (ret)
+		return;
+
+	reg_def->def = raw;
+#endif
+}
+EXPORT_SYMBOL_NS_GPL(sx_common_get_raw_register_config, SEMTECH_PROX);
+
 #define SX_COMMON_SOFT_RESET				0xde
 
 static int sx_common_init_device(struct device *dev, struct iio_dev *indio_dev)
diff --git a/drivers/iio/proximity/sx_common.h b/drivers/iio/proximity/sx_common.h
index 49d4517103b0..101b90e52ff2 100644
--- a/drivers/iio/proximity/sx_common.h
+++ b/drivers/iio/proximity/sx_common.h
@@ -8,6 +8,7 @@
 #ifndef IIO_SX_COMMON_H
 #define IIO_SX_COMMON_H
 
+#include <linux/acpi.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/types.h>
 #include <linux/regulator/consumer.h>
@@ -26,6 +27,7 @@ static_assert(SX_COMMON_MAX_NUM_CHANNELS < BITS_PER_LONG);
 struct sx_common_reg_default {
 	u8 reg;
 	u8 def;
+	const char *property;
 };
 
 /**
@@ -101,7 +103,6 @@ struct sx_common_chip_info {
  * @client:		I2C client structure.
  * @trig:		IIO trigger object.
  * @regmap:		Register map.
- * @num_default_regs:	Number of default registers to set at init.
  * @chan_prox_stat:	Last reading of the proximity status for each channel.
  *			We only send an event to user space when this changes.
  * @trigger_enabled:	True when the device trigger is enabled.
@@ -149,6 +150,9 @@ int sx_common_probe(struct i2c_client *client,
 		    const struct sx_common_chip_info *chip_info,
 		    const struct regmap_config *regmap_config);
 
+void sx_common_get_raw_register_config(struct device *dev,
+				       struct sx_common_reg_default *reg_def);
+
 /* 3 is the number of events defined by a single phase. */
 extern const struct iio_event_spec sx_common_events[3];
 
diff --git a/drivers/iio/temperature/tmp117.c b/drivers/iio/temperature/tmp117.c
index f9b8f2b570f6..638e3a5bd6b8 100644
--- a/drivers/iio/temperature/tmp117.c
+++ b/drivers/iio/temperature/tmp117.c
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/limits.h>
+#include <linux/property.h>
 
 #include <linux/iio/iio.h>
 
@@ -31,17 +32,19 @@
 #define TMP117_REG_DEVICE_ID		0xF
 
 #define TMP117_RESOLUTION_10UC		78125
-#define TMP117_DEVICE_ID		0x0117
 #define MICRODEGREE_PER_10MILLIDEGREE	10000
 
+#define TMP116_DEVICE_ID		0x1116
+#define TMP117_DEVICE_ID		0x0117
+
 struct tmp117_data {
 	struct i2c_client *client;
 	s16 calibbias;
 };
 
 static int tmp117_read_raw(struct iio_dev *indio_dev,
-		struct iio_chan_spec const *channel, int *val,
-		int *val2, long mask)
+			   struct iio_chan_spec const *channel, int *val,
+			   int *val2, long mask)
 {
 	struct tmp117_data *data = iio_priv(indio_dev);
 	s32 ret;
@@ -49,7 +52,7 @@ static int tmp117_read_raw(struct iio_dev *indio_dev,
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
 		ret = i2c_smbus_read_word_swapped(data->client,
-						TMP117_REG_TEMP);
+						  TMP117_REG_TEMP);
 		if (ret < 0)
 			return ret;
 		*val = sign_extend32(ret, 15);
@@ -57,7 +60,7 @@ static int tmp117_read_raw(struct iio_dev *indio_dev,
 
 	case IIO_CHAN_INFO_CALIBBIAS:
 		ret = i2c_smbus_read_word_swapped(data->client,
-					TMP117_REG_TEMP_OFFSET);
+						  TMP117_REG_TEMP_OFFSET);
 		if (ret < 0)
 			return ret;
 		*val = sign_extend32(ret, 15);
@@ -79,9 +82,8 @@ static int tmp117_read_raw(struct iio_dev *indio_dev,
 	}
 }
 
-static int tmp117_write_raw(struct iio_dev *indio_dev,
-		struct iio_chan_spec const *channel, int val,
-		int val2, long mask)
+static int tmp117_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec
+			    const *channel, int val, int val2, long mask)
 {
 	struct tmp117_data *data = iio_priv(indio_dev);
 	s16 off;
@@ -104,7 +106,16 @@ static const struct iio_chan_spec tmp117_channels[] = {
 	{
 		.type = IIO_TEMP,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
-			BIT(IIO_CHAN_INFO_CALIBBIAS) | BIT(IIO_CHAN_INFO_SCALE),
+				      BIT(IIO_CHAN_INFO_CALIBBIAS) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+	},
+};
+
+static const struct iio_chan_spec tmp116_channels[] = {
+	{
+		.type = IIO_TEMP,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE),
 	},
 };
 
@@ -115,23 +126,41 @@ static const struct iio_info tmp117_info = {
 
 static int tmp117_identify(struct i2c_client *client)
 {
+	const struct i2c_device_id *id;
+	unsigned long match_data;
 	int dev_id;
 
 	dev_id = i2c_smbus_read_word_swapped(client, TMP117_REG_DEVICE_ID);
 	if (dev_id < 0)
 		return dev_id;
-	if (dev_id != TMP117_DEVICE_ID) {
-		dev_err(&client->dev, "TMP117 not found\n");
-		return -ENODEV;
+
+	switch (dev_id) {
+	case TMP116_DEVICE_ID:
+	case TMP117_DEVICE_ID:
+		return dev_id;
 	}
-	return 0;
+
+	dev_info(&client->dev, "Unknown device id (0x%x), use fallback compatible\n",
+		 dev_id);
+
+	match_data = (uintptr_t)device_get_match_data(&client->dev);
+	if (match_data)
+		return match_data;
+
+	id = i2c_client_get_device_id(client);
+	if (id)
+		return id->driver_data;
+
+	dev_err(&client->dev, "Failed to identify unsupported device\n");
+
+	return -ENODEV;
 }
 
 static int tmp117_probe(struct i2c_client *client)
 {
 	struct tmp117_data *data;
 	struct iio_dev *indio_dev;
-	int ret;
+	int ret, dev_id;
 
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_WORD_DATA))
 		return -EOPNOTSUPP;
@@ -140,6 +169,8 @@ static int tmp117_probe(struct i2c_client *client)
 	if (ret < 0)
 		return ret;
 
+	dev_id = ret;
+
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
 	if (!indio_dev)
 		return -ENOMEM;
@@ -148,24 +179,35 @@ static int tmp117_probe(struct i2c_client *client)
 	data->client = client;
 	data->calibbias = 0;
 
-	indio_dev->name = "tmp117";
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->info = &tmp117_info;
 
-	indio_dev->channels = tmp117_channels;
-	indio_dev->num_channels = ARRAY_SIZE(tmp117_channels);
+	switch (dev_id) {
+	case TMP116_DEVICE_ID:
+		indio_dev->channels = tmp116_channels;
+		indio_dev->num_channels = ARRAY_SIZE(tmp116_channels);
+		indio_dev->name = "tmp116";
+		break;
+	case TMP117_DEVICE_ID:
+		indio_dev->channels = tmp117_channels;
+		indio_dev->num_channels = ARRAY_SIZE(tmp117_channels);
+		indio_dev->name = "tmp117";
+		break;
+	}
 
 	return devm_iio_device_register(&client->dev, indio_dev);
 }
 
 static const struct of_device_id tmp117_of_match[] = {
-	{ .compatible = "ti,tmp117", },
+	{ .compatible = "ti,tmp116", .data = (void *)TMP116_DEVICE_ID },
+	{ .compatible = "ti,tmp117", .data = (void *)TMP117_DEVICE_ID },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, tmp117_of_match);
 
 static const struct i2c_device_id tmp117_id[] = {
-	{ "tmp117", 0 },
+	{ "tmp116", TMP116_DEVICE_ID },
+	{ "tmp117", TMP117_DEVICE_ID },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tmp117_id);
diff --git a/drivers/iio/trigger/iio-trig-loop.c b/drivers/iio/trigger/iio-trig-loop.c
index 96ec06bbe546..7aaed0611899 100644
--- a/drivers/iio/trigger/iio-trig-loop.c
+++ b/drivers/iio/trigger/iio-trig-loop.c
@@ -46,7 +46,7 @@ static int iio_loop_thread(void *data)
 	set_freezable();
 
 	do {
-		iio_trigger_poll_chained(trig);
+		iio_trigger_poll_nested(trig);
 	} while (likely(!kthread_freezable_should_stop(NULL)));
 
 	return 0;
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index f83954180a33..0e9e04f8c685 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -1229,8 +1229,8 @@ static char *umad_devnode(const struct device *dev, umode_t *mode)
 	return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
 }
 
-static ssize_t abi_version_show(struct class *class,
-				struct class_attribute *attr, char *buf)
+static ssize_t abi_version_show(const struct class *class,
+				const struct class_attribute *attr, char *buf)
 {
 	return sysfs_emit(buf, "%d\n", IB_USER_MAD_ABI_VERSION);
 }
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index bdb179a09d77..fbace69672ca 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -1264,7 +1264,7 @@ static int __init ib_uverbs_init(void)
 		goto out_alloc;
 	}
 
-	uverbs_class = class_create(THIS_MODULE, "infiniband_verbs");
+	uverbs_class = class_create("infiniband_verbs");
 	if (IS_ERR(uverbs_class)) {
 		ret = PTR_ERR(uverbs_class);
 		pr_err("user_verbs: couldn't create class infiniband_verbs\n");
diff --git a/drivers/infiniband/hw/hfi1/device.c b/drivers/infiniband/hw/hfi1/device.c
index 1f4496032170..05be0d119f79 100644
--- a/drivers/infiniband/hw/hfi1/device.c
+++ b/drivers/infiniband/hw/hfi1/device.c
@@ -102,7 +102,7 @@ int __init dev_init(void)
 		goto done;
 	}
 
-	class = class_create(THIS_MODULE, class_name());
+	class = class_create(class_name());
 	if (IS_ERR(class)) {
 		ret = PTR_ERR(class);
 		pr_err("Could not create device class (err %d)\n", -ret);
@@ -111,7 +111,7 @@ int __init dev_init(void)
 	}
 	class->devnode = hfi1_devnode;
 
-	user_class = class_create(THIS_MODULE, class_name_user());
+	user_class = class_create(class_name_user());
 	if (IS_ERR(user_class)) {
 		ret = PTR_ERR(user_class);
 		pr_err("Could not create device class for user accessible files (err %d)\n",
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index b1d6ca7e9708..f3d6ce45c397 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -267,6 +267,8 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 
 	if (!HFI1_CAP_IS_KSET(SDMA))
 		return -EINVAL;
+	if (!from->user_backed)
+		return -EINVAL;
 	idx = srcu_read_lock(&fd->pq_srcu);
 	pq = srcu_dereference(fd->pq, &fd->pq_srcu);
 	if (!cq || !pq) {
@@ -274,11 +276,6 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 		return -EIO;
 	}
 
-	if (!iter_is_iovec(from) || !dim) {
-		srcu_read_unlock(&fd->pq_srcu, idx);
-		return -EINVAL;
-	}
-
 	trace_hfi1_sdma_request(fd->dd, fd->uctxt->ctxt, fd->subctxt, dim);
 
 	if (atomic_read(&pq->n_reqs) == pq->n_max_reqs) {
@@ -287,11 +284,12 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 	}
 
 	while (dim) {
+		const struct iovec *iov = iter_iov(from);
 		int ret;
 		unsigned long count = 0;
 
 		ret = hfi1_user_sdma_process_request(
-			fd, (struct iovec *)(from->iov + done),
+			fd, (struct iovec *)(iov + done),
 			dim, &count);
 		if (ret) {
 			reqs = ret;
diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c
index 80fe92a21f96..f22352f2b369 100644
--- a/drivers/infiniband/hw/qib/qib_file_ops.c
+++ b/drivers/infiniband/hw/qib/qib_file_ops.c
@@ -2245,10 +2245,10 @@ static ssize_t qib_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	struct qib_ctxtdata *rcd = ctxt_fp(iocb->ki_filp);
 	struct qib_user_sdma_queue *pq = fp->pq;
 
-	if (!iter_is_iovec(from) || !from->nr_segs || !pq)
+	if (!from->user_backed || !from->nr_segs || !pq)
 		return -EINVAL;
 
-	return qib_user_sdma_writev(rcd, pq, from->iov, from->nr_segs);
+	return qib_user_sdma_writev(rcd, pq, iter_iov(from), from->nr_segs);
 }
 
 static struct class *qib_class;
@@ -2326,7 +2326,7 @@ int __init qib_dev_init(void)
 		goto done;
 	}
 
-	qib_class = class_create(THIS_MODULE, "ipath");
+	qib_class = class_create("ipath");
 	if (IS_ERR(qib_class)) {
 		ret = PTR_ERR(qib_class);
 		pr_err("Could not create device class (err %d)\n", -ret);
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 3acab569fbb9..9b4c0389d2c0 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -97,7 +97,7 @@ static void cacheless_memcpy(void *dst, void *src, size_t n)
 	 * there are no security issues.  The extra fault recovery machinery
 	 * is not invoked.
 	 */
-	__copy_user_nocache(dst, (void __user *)src, n, 0);
+	__copy_user_nocache(dst, (void __user *)src, n);
 }
 
 void rvt_wss_exit(struct rvt_dev_info *rdi)
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 6b7603765383..bb9aaff92ca3 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -78,7 +78,7 @@ MODULE_DESCRIPTION("iSER (iSCSI Extensions for RDMA) Datamover");
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_AUTHOR("Alex Nezhinsky, Dan Bar Dov, Or Gerlitz");
 
-static struct scsi_host_template iscsi_iser_sht;
+static const struct scsi_host_template iscsi_iser_sht;
 static struct iscsi_transport iscsi_iser_transport;
 static struct scsi_transport_template *iscsi_iser_scsi_transport;
 static struct workqueue_struct *release_wq;
@@ -956,7 +956,7 @@ static umode_t iser_attr_is_visible(int param_type, int param)
 	return 0;
 }
 
-static struct scsi_host_template iscsi_iser_sht = {
+static const struct scsi_host_template iscsi_iser_sht = {
 	.module                 = THIS_MODULE,
 	.name                   = "iSCSI Initiator over iSER",
 	.queuecommand           = iscsi_queuecommand,
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 75404885cf98..f290cd49698e 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -2506,8 +2506,8 @@ isert_wait4cmds(struct iscsit_conn *conn)
 	isert_info("iscsit_conn %p\n", conn);
 
 	if (conn->sess) {
-		target_stop_session(conn->sess->se_sess);
-		target_wait_for_sess_cmds(conn->sess->se_sess);
+		target_stop_cmd_counter(conn->cmd_cnt);
+		target_wait_for_cmds(conn->cmd_cnt);
 	}
 }
 
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index 80abf45a197a..edb2e3a25880 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -3163,7 +3163,7 @@ static int __init rtrs_client_init(void)
 {
 	rtrs_rdma_dev_pd_init(0, &dev_pd);
 
-	rtrs_clt_dev_class = class_create(THIS_MODULE, "rtrs-client");
+	rtrs_clt_dev_class = class_create("rtrs-client");
 	if (IS_ERR(rtrs_clt_dev_class)) {
 		pr_err("Failed to create rtrs-client dev class\n");
 		return PTR_ERR(rtrs_clt_dev_class);
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
index d1703e2c0b82..c38901e2c8f4 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
@@ -2253,7 +2253,7 @@ static int __init rtrs_server_init(void)
 		       err);
 		return err;
 	}
-	rtrs_dev_class = class_create(THIS_MODULE, "rtrs-server");
+	rtrs_dev_class = class_create("rtrs-server");
 	if (IS_ERR(rtrs_dev_class)) {
 		err = PTR_ERR(rtrs_dev_class);
 		goto out_err;
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index df21b30b7735..0e513a7e5ac8 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -62,11 +62,6 @@ MODULE_AUTHOR("Roland Dreier");
 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
 MODULE_LICENSE("Dual BSD/GPL");
 
-#if !defined(CONFIG_DYNAMIC_DEBUG)
-#define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)
-#define DYNAMIC_DEBUG_BRANCH(descriptor) false
-#endif
-
 static unsigned int srp_sg_tablesize;
 static unsigned int cmd_sg_entries;
 static unsigned int indirect_sg_entries;
@@ -3077,7 +3072,7 @@ static struct attribute *srp_host_attrs[] = {
 
 ATTRIBUTE_GROUPS(srp_host);
 
-static struct scsi_host_template srp_template = {
+static const struct scsi_host_template srp_template = {
 	.module				= THIS_MODULE,
 	.name				= "InfiniBand SRP initiator",
 	.proc_name			= DRV_NAME,
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 3c3fae738c3e..3e9343fd2d8e 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -3300,11 +3300,6 @@ static int srpt_check_true(struct se_portal_group *se_tpg)
 	return 1;
 }
 
-static int srpt_check_false(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
 static struct srpt_port *srpt_tpg_to_sport(struct se_portal_group *tpg)
 {
 	return tpg->se_tpg_wwn->priv;
@@ -3334,11 +3329,6 @@ static u16 srpt_get_tag(struct se_portal_group *tpg)
 	return 1;
 }
 
-static u32 srpt_tpg_get_inst_index(struct se_portal_group *se_tpg)
-{
-	return 1;
-}
-
 static void srpt_release_cmd(struct se_cmd *se_cmd)
 {
 	struct srpt_send_ioctx *ioctx = container_of(se_cmd,
@@ -3378,24 +3368,6 @@ static void srpt_close_session(struct se_session *se_sess)
 	srpt_disconnect_ch_sync(ch);
 }
 
-/**
- * srpt_sess_get_index - return the value of scsiAttIntrPortIndex (SCSI-MIB)
- * @se_sess: SCSI target session.
- *
- * A quote from RFC 4455 (SCSI-MIB) about this MIB object:
- * This object represents an arbitrary integer used to uniquely identify a
- * particular attached remote initiator port to a particular SCSI target port
- * within a particular SCSI target device within a particular SCSI instance.
- */
-static u32 srpt_sess_get_index(struct se_session *se_sess)
-{
-	return 0;
-}
-
-static void srpt_set_default_node_attrs(struct se_node_acl *nacl)
-{
-}
-
 /* Note: only used from inside debug printk's by the TCM core. */
 static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd)
 {
@@ -3866,18 +3838,13 @@ static const struct target_core_fabric_ops srpt_template = {
 	.fabric_name			= "srpt",
 	.tpg_get_wwn			= srpt_get_fabric_wwn,
 	.tpg_get_tag			= srpt_get_tag,
-	.tpg_check_demo_mode		= srpt_check_false,
 	.tpg_check_demo_mode_cache	= srpt_check_true,
 	.tpg_check_demo_mode_write_protect = srpt_check_true,
-	.tpg_check_prod_mode_write_protect = srpt_check_false,
-	.tpg_get_inst_index		= srpt_tpg_get_inst_index,
 	.release_cmd			= srpt_release_cmd,
 	.check_stop_free		= srpt_check_stop_free,
 	.close_session			= srpt_close_session,
-	.sess_get_index			= srpt_sess_get_index,
 	.sess_get_initiator_sid		= NULL,
 	.write_pending			= srpt_write_pending,
-	.set_default_node_attributes	= srpt_set_default_node_attrs,
 	.get_cmd_state			= srpt_get_tcm_cmd_state,
 	.queue_data_in			= srpt_queue_data_in,
 	.queue_status			= srpt_queue_status,
diff --git a/drivers/input/tablet/pegasus_notetaker.c b/drivers/input/tablet/pegasus_notetaker.c
index d836d3dcc6a2..a68da2988f9c 100644
--- a/drivers/input/tablet/pegasus_notetaker.c
+++ b/drivers/input/tablet/pegasus_notetaker.c
@@ -296,6 +296,12 @@ static int pegasus_probe(struct usb_interface *intf,
 	pegasus->intf = intf;
 
 	pipe = usb_rcvintpipe(dev, endpoint->bEndpointAddress);
+	/* Sanity check that pipe's type matches endpoint's type */
+	if (usb_pipe_type_check(dev, pipe)) {
+		error = -EINVAL;
+		goto err_free_mem;
+	}
+
 	pegasus->data_len = usb_maxpacket(dev, pipe);
 
 	pegasus->data = usb_alloc_coherent(dev, pegasus->data_len, GFP_KERNEL,
diff --git a/drivers/input/touchscreen/ads7846.c b/drivers/input/touchscreen/ads7846.c
index 17f11bce8113..bb1058b1e7fd 100644
--- a/drivers/input/touchscreen/ads7846.c
+++ b/drivers/input/touchscreen/ads7846.c
@@ -27,6 +27,7 @@
 #include <linux/of.h>
 #include <linux/of_gpio.h>
 #include <linux/of_device.h>
+#include <linux/gpio/consumer.h>
 #include <linux/gpio.h>
 #include <linux/spi/spi.h>
 #include <linux/spi/ads7846.h>
@@ -1012,8 +1013,8 @@ static int ads7846_setup_pendown(struct spi_device *spi,
 		ts->gpio_pendown = pdata->gpio_pendown;
 
 		if (pdata->gpio_pendown_debounce)
-			gpio_set_debounce(pdata->gpio_pendown,
-					  pdata->gpio_pendown_debounce);
+			gpiod_set_debounce(gpio_to_desc(ts->gpio_pendown),
+					   pdata->gpio_pendown_debounce);
 	} else {
 		dev_err(&spi->dev, "no get_pendown_state nor gpio_pendown?\n");
 		return -EINVAL;
diff --git a/drivers/input/touchscreen/cyttsp5.c b/drivers/input/touchscreen/cyttsp5.c
index 16caffa35dd9..30102cb80fac 100644
--- a/drivers/input/touchscreen/cyttsp5.c
+++ b/drivers/input/touchscreen/cyttsp5.c
@@ -111,6 +111,7 @@ struct cyttsp5_sensing_conf_data_dev {
 	__le16 max_z;
 	u8 origin_x;
 	u8 origin_y;
+	u8 panel_id;
 	u8 btn;
 	u8 scan_mode;
 	u8 max_num_of_tch_per_refresh_cycle;
diff --git a/drivers/input/touchscreen/sun4i-ts.c b/drivers/input/touchscreen/sun4i-ts.c
index 73eb8f80be6e..1117fba30020 100644
--- a/drivers/input/touchscreen/sun4i-ts.c
+++ b/drivers/input/touchscreen/sun4i-ts.c
@@ -194,7 +194,7 @@ static int sun4i_get_temp(const struct sun4i_ts_data *ts, int *temp)
 
 static int sun4i_get_tz_temp(struct thermal_zone_device *tz, int *temp)
 {
-	return sun4i_get_temp(tz->devdata, temp);
+	return sun4i_get_temp(thermal_zone_device_priv(tz), temp);
 }
 
 static const struct thermal_zone_device_ops sun4i_ts_tz_ops = {
diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c
index 7a24c1444ace..ec46bcb16d5e 100644
--- a/drivers/interconnect/core.c
+++ b/drivers/interconnect/core.c
@@ -13,7 +13,6 @@
 #include <linux/interconnect.h>
 #include <linux/interconnect-provider.h>
 #include <linux/list.h>
-#include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/of.h>
@@ -451,7 +450,7 @@ struct icc_path *of_icc_get_by_index(struct device *dev, int idx)
 	 * When the consumer DT node do not have "interconnects" property
 	 * return a NULL path to skip setting constraints.
 	 */
-	if (!of_find_property(np, "interconnects", NULL))
+	if (!of_property_present(np, "interconnects"))
 		return NULL;
 
 	/*
@@ -544,7 +543,7 @@ struct icc_path *of_icc_get(struct device *dev, const char *name)
 	 * When the consumer DT node do not have "interconnects" property
 	 * return a NULL path to skip setting constraints.
 	 */
-	if (!of_find_property(np, "interconnects", NULL))
+	if (!of_property_present(np, "interconnects"))
 		return NULL;
 
 	/*
@@ -911,52 +910,6 @@ out:
 EXPORT_SYMBOL_GPL(icc_link_create);
 
 /**
- * icc_link_destroy() - destroy a link between two nodes
- * @src: pointer to source node
- * @dst: pointer to destination node
- *
- * Return: 0 on success, or an error code otherwise
- */
-int icc_link_destroy(struct icc_node *src, struct icc_node *dst)
-{
-	struct icc_node **new;
-	size_t slot;
-	int ret = 0;
-
-	if (IS_ERR_OR_NULL(src))
-		return -EINVAL;
-
-	if (IS_ERR_OR_NULL(dst))
-		return -EINVAL;
-
-	mutex_lock(&icc_lock);
-
-	for (slot = 0; slot < src->num_links; slot++)
-		if (src->links[slot] == dst)
-			break;
-
-	if (WARN_ON(slot == src->num_links)) {
-		ret = -ENXIO;
-		goto out;
-	}
-
-	src->links[slot] = src->links[--src->num_links];
-
-	new = krealloc(src->links, src->num_links * sizeof(*src->links),
-		       GFP_KERNEL);
-	if (new)
-		src->links = new;
-	else
-		ret = -ENOMEM;
-
-out:
-	mutex_unlock(&icc_lock);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(icc_link_destroy);
-
-/**
  * icc_node_add() - add interconnect node to interconnect provider
  * @node: pointer to the interconnect node
  * @provider: pointer to the interconnect provider
@@ -981,14 +934,17 @@ void icc_node_add(struct icc_node *node, struct icc_provider *provider)
 	node->avg_bw = node->init_avg;
 	node->peak_bw = node->init_peak;
 
-	if (provider->pre_aggregate)
-		provider->pre_aggregate(node);
+	if (node->avg_bw || node->peak_bw) {
+		if (provider->pre_aggregate)
+			provider->pre_aggregate(node);
 
-	if (provider->aggregate)
-		provider->aggregate(node, 0, node->init_avg, node->init_peak,
-				    &node->avg_bw, &node->peak_bw);
+		if (provider->aggregate)
+			provider->aggregate(node, 0, node->init_avg, node->init_peak,
+					    &node->avg_bw, &node->peak_bw);
+		if (provider->set)
+			provider->set(node, node);
+	}
 
-	provider->set(node, node);
 	node->avg_bw = 0;
 	node->peak_bw = 0;
 
@@ -1081,22 +1037,6 @@ void icc_provider_deregister(struct icc_provider *provider)
 }
 EXPORT_SYMBOL_GPL(icc_provider_deregister);
 
-int icc_provider_add(struct icc_provider *provider)
-{
-	icc_provider_init(provider);
-
-	return icc_provider_register(provider);
-}
-EXPORT_SYMBOL_GPL(icc_provider_add);
-
-void icc_provider_del(struct icc_provider *provider)
-{
-	WARN_ON(!list_empty(&provider->nodes));
-
-	icc_provider_deregister(provider);
-}
-EXPORT_SYMBOL_GPL(icc_provider_del);
-
 static const struct of_device_id __maybe_unused ignore_list[] = {
 	{ .compatible = "qcom,sc7180-ipa-virt" },
 	{ .compatible = "qcom,sc8180x-ipa-virt" },
@@ -1165,7 +1105,3 @@ static int __init icc_init(void)
 }
 
 device_initcall(icc_init);
-
-MODULE_AUTHOR("Georgi Djakov <georgi.djakov@linaro.org>");
-MODULE_DESCRIPTION("Interconnect Driver Core");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/interconnect/qcom/Kconfig b/drivers/interconnect/qcom/Kconfig
index 92d65c7bda23..825b647d9169 100644
--- a/drivers/interconnect/qcom/Kconfig
+++ b/drivers/interconnect/qcom/Kconfig
@@ -83,7 +83,7 @@ config INTERCONNECT_QCOM_RPMH_POSSIBLE
 	default INTERCONNECT_QCOM
 	depends on QCOM_RPMH || (COMPILE_TEST && !QCOM_RPMH)
 	depends on QCOM_COMMAND_DB || (COMPILE_TEST && !QCOM_COMMAND_DB)
-	depends on OF || COMPILE_TEST
+	depends on OF
 	help
 	  Compile-testing RPMH drivers is possible on other platforms,
 	  but in order to avoid link failures, drivers must not be built-in
diff --git a/drivers/interconnect/qcom/icc-rpm.c b/drivers/interconnect/qcom/icc-rpm.c
index 4180a06681b2..5341fa169dbf 100644
--- a/drivers/interconnect/qcom/icc-rpm.c
+++ b/drivers/interconnect/qcom/icc-rpm.c
@@ -11,7 +11,6 @@
 #include <linux/of_device.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
-#include <linux/pm_domain.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
 
@@ -48,6 +47,9 @@
 #define NOC_QOS_MODEn_ADDR(n)		(0xc + (n * 0x1000))
 #define NOC_QOS_MODEn_MASK		0x3
 
+#define NOC_QOS_MODE_FIXED_VAL		0x0
+#define NOC_QOS_MODE_BYPASS_VAL		0x2
+
 static int qcom_icc_set_qnoc_qos(struct icc_node *src, u64 max_bw)
 {
 	struct icc_provider *provider = src->provider;
@@ -153,7 +155,7 @@ static int qcom_icc_set_noc_qos(struct icc_node *src, u64 max_bw)
 	struct qcom_icc_provider *qp;
 	struct qcom_icc_node *qn;
 	struct icc_provider *provider;
-	u32 mode = NOC_QOS_MODE_BYPASS;
+	u32 mode = NOC_QOS_MODE_BYPASS_VAL;
 	int rc = 0;
 
 	qn = src->data;
@@ -167,18 +169,17 @@ static int qcom_icc_set_noc_qos(struct icc_node *src, u64 max_bw)
 		return 0;
 	}
 
-	if (qn->qos.qos_mode != NOC_QOS_MODE_INVALID)
-		mode = qn->qos.qos_mode;
-
-	if (mode == NOC_QOS_MODE_FIXED) {
-		dev_dbg(src->provider->dev, "NoC QoS: %s: Set Fixed mode\n",
-			qn->name);
+	if (qn->qos.qos_mode == NOC_QOS_MODE_FIXED) {
+		dev_dbg(src->provider->dev, "NoC QoS: %s: Set Fixed mode\n", qn->name);
+		mode = NOC_QOS_MODE_FIXED_VAL;
 		rc = qcom_icc_noc_set_qos_priority(qp, &qn->qos);
 		if (rc)
 			return rc;
-	} else if (mode == NOC_QOS_MODE_BYPASS) {
-		dev_dbg(src->provider->dev, "NoC QoS: %s: Set Bypass mode\n",
-			qn->name);
+	} else if (qn->qos.qos_mode == NOC_QOS_MODE_BYPASS) {
+		dev_dbg(src->provider->dev, "NoC QoS: %s: Set Bypass mode\n", qn->name);
+		mode = NOC_QOS_MODE_BYPASS_VAL;
+	} else {
+		/* How did we get here? */
 	}
 
 	return regmap_update_bits(qp->regmap,
@@ -244,7 +245,7 @@ static int __qcom_icc_set(struct icc_node *n, struct qcom_icc_node *qn,
 		ret = qcom_icc_rpm_set(qn->mas_rpm_id, qn->slv_rpm_id, sum_bw);
 		if (ret)
 			return ret;
-	} else if (qn->qos.qos_mode != -1) {
+	} else if (qn->qos.qos_mode != NOC_QOS_MODE_INVALID) {
 		/* set bandwidth directly from the AP */
 		ret = qcom_icc_qos_set(n, sum_bw);
 		if (ret)
@@ -315,6 +316,7 @@ static void qcom_icc_bus_aggregate(struct icc_provider *provider,
 {
 	struct icc_node *node;
 	struct qcom_icc_node *qn;
+	u64 sum_avg[QCOM_ICC_NUM_BUCKETS];
 	int i;
 
 	/* Initialise aggregate values */
@@ -332,7 +334,11 @@ static void qcom_icc_bus_aggregate(struct icc_provider *provider,
 	list_for_each_entry(node, &provider->nodes, node_list) {
 		qn = node->data;
 		for (i = 0; i < QCOM_ICC_NUM_BUCKETS; i++) {
-			agg_avg[i] += qn->sum_avg[i];
+			if (qn->channels)
+				sum_avg[i] = div_u64(qn->sum_avg[i], qn->channels);
+			else
+				sum_avg[i] = qn->sum_avg[i];
+			agg_avg[i] += sum_avg[i];
 			agg_peak[i] = max_t(u64, agg_peak[i], qn->max_peak[i]);
 		}
 	}
@@ -496,12 +502,6 @@ regmap_done:
 	if (ret)
 		return ret;
 
-	if (desc->has_bus_pd) {
-		ret = dev_pm_domain_attach(dev, true);
-		if (ret)
-			return ret;
-	}
-
 	provider = &qp->provider;
 	provider->dev = dev;
 	provider->set = qcom_icc_set;
diff --git a/drivers/interconnect/qcom/icc-rpm.h b/drivers/interconnect/qcom/icc-rpm.h
index a49af844ab13..22bdb1e4bb12 100644
--- a/drivers/interconnect/qcom/icc-rpm.h
+++ b/drivers/interconnect/qcom/icc-rpm.h
@@ -23,12 +23,12 @@ enum qcom_icc_type {
 /**
  * struct qcom_icc_provider - Qualcomm specific interconnect provider
  * @provider: generic interconnect provider
- * @bus_clks: the clk_bulk_data table of bus clocks
  * @num_clks: the total number of clk_bulk_data entries
  * @type: the ICC provider type
- * @qos_offset: offset to QoS registers
  * @regmap: regmap for QoS registers read/write access
+ * @qos_offset: offset to QoS registers
  * @bus_clk_rate: bus clock rate in Hz
+ * @bus_clks: the clk_bulk_data table of bus clocks
  */
 struct qcom_icc_provider {
 	struct icc_provider provider;
@@ -66,6 +66,7 @@ struct qcom_icc_qos {
  * @id: a unique node identifier
  * @links: an array of nodes where we can go next while traversing
  * @num_links: the total number of @links
+ * @channels: number of channels at this node (e.g. DDR channels)
  * @buswidth: width of the interconnect between a node and the bus (bytes)
  * @sum_avg: current sum aggregate value of all avg bw requests
  * @max_peak: current max aggregate value of all peak bw requests
@@ -78,6 +79,7 @@ struct qcom_icc_node {
 	u16 id;
 	const u16 *links;
 	u16 num_links;
+	u16 channels;
 	u16 buswidth;
 	u64 sum_avg[QCOM_ICC_NUM_BUCKETS];
 	u64 max_peak[QCOM_ICC_NUM_BUCKETS];
@@ -91,16 +93,17 @@ struct qcom_icc_desc {
 	size_t num_nodes;
 	const char * const *clocks;
 	size_t num_clocks;
-	bool has_bus_pd;
 	enum qcom_icc_type type;
 	const struct regmap_config *regmap_cfg;
 	unsigned int qos_offset;
 };
 
-/* Valid for both NoC and BIMC */
-#define NOC_QOS_MODE_INVALID		-1
-#define NOC_QOS_MODE_FIXED		0x0
-#define NOC_QOS_MODE_BYPASS		0x2
+/* Valid for all bus types */
+enum qos_mode {
+	NOC_QOS_MODE_INVALID = 0,
+	NOC_QOS_MODE_FIXED,
+	NOC_QOS_MODE_BYPASS,
+};
 
 int qnoc_probe(struct platform_device *pdev);
 int qnoc_remove(struct platform_device *pdev);
diff --git a/drivers/interconnect/qcom/msm8996.c b/drivers/interconnect/qcom/msm8996.c
index 25a1a32bc611..14efd2761b7a 100644
--- a/drivers/interconnect/qcom/msm8996.c
+++ b/drivers/interconnect/qcom/msm8996.c
@@ -1823,7 +1823,6 @@ static const struct qcom_icc_desc msm8996_a0noc = {
 	.num_nodes = ARRAY_SIZE(a0noc_nodes),
 	.clocks = bus_a0noc_clocks,
 	.num_clocks = ARRAY_SIZE(bus_a0noc_clocks),
-	.has_bus_pd = true,
 	.regmap_cfg = &msm8996_a0noc_regmap_config
 };
 
diff --git a/drivers/interconnect/qcom/osm-l3.c b/drivers/interconnect/qcom/osm-l3.c
index 1bafb54f1432..a1f4f918b911 100644
--- a/drivers/interconnect/qcom/osm-l3.c
+++ b/drivers/interconnect/qcom/osm-l3.c
@@ -14,13 +14,6 @@
 
 #include <dt-bindings/interconnect/qcom,osm-l3.h>
 
-#include "sc7180.h"
-#include "sc7280.h"
-#include "sc8180x.h"
-#include "sdm845.h"
-#include "sm8150.h"
-#include "sm8250.h"
-
 #define LUT_MAX_ENTRIES			40U
 #define LUT_SRC				GENMASK(31, 30)
 #define LUT_L_VAL			GENMASK(7, 0)
diff --git a/drivers/interconnect/qcom/sc7180.h b/drivers/interconnect/qcom/sc7180.h
index 7a2b3eb00923..2b718922c109 100644
--- a/drivers/interconnect/qcom/sc7180.h
+++ b/drivers/interconnect/qcom/sc7180.h
@@ -145,7 +145,5 @@
 #define SC7180_SLAVE_SERVICE_SNOC			134
 #define SC7180_SLAVE_QDSS_STM				135
 #define SC7180_SLAVE_TCU				136
-#define SC7180_MASTER_OSM_L3_APPS			137
-#define SC7180_SLAVE_OSM_L3				138
 
 #endif
diff --git a/drivers/interconnect/qcom/sc7280.h b/drivers/interconnect/qcom/sc7280.h
index 1fb9839b2c14..175e400305c5 100644
--- a/drivers/interconnect/qcom/sc7280.h
+++ b/drivers/interconnect/qcom/sc7280.h
@@ -150,7 +150,5 @@
 #define SC7280_SLAVE_PCIE_1			139
 #define SC7280_SLAVE_QDSS_STM			140
 #define SC7280_SLAVE_TCU			141
-#define SC7280_MASTER_EPSS_L3_APPS		142
-#define SC7280_SLAVE_EPSS_L3			143
 
 #endif
diff --git a/drivers/interconnect/qcom/sc8180x.h b/drivers/interconnect/qcom/sc8180x.h
index c138dcd350f1..f8d90598335a 100644
--- a/drivers/interconnect/qcom/sc8180x.h
+++ b/drivers/interconnect/qcom/sc8180x.h
@@ -168,8 +168,6 @@
 #define SC8180X_SLAVE_EBI_CH0_DISPLAY		158
 #define SC8180X_SLAVE_MNOC_SF_MEM_NOC_DISPLAY	159
 #define SC8180X_SLAVE_MNOC_HF_MEM_NOC_DISPLAY	160
-#define SC8180X_MASTER_OSM_L3_APPS		161
-#define SC8180X_SLAVE_OSM_L3			162
 
 #define SC8180X_MASTER_QUP_CORE_0		163
 #define SC8180X_MASTER_QUP_CORE_1		164
diff --git a/drivers/interconnect/qcom/sdm845.h b/drivers/interconnect/qcom/sdm845.h
index 776e9c2acb27..bc7e425ce985 100644
--- a/drivers/interconnect/qcom/sdm845.h
+++ b/drivers/interconnect/qcom/sdm845.h
@@ -136,7 +136,5 @@
 #define SDM845_SLAVE_SERVICE_SNOC			128
 #define SDM845_SLAVE_QDSS_STM				129
 #define SDM845_SLAVE_TCU				130
-#define SDM845_MASTER_OSM_L3_APPS			131
-#define SDM845_SLAVE_OSM_L3				132
 
 #endif /* __DRIVERS_INTERCONNECT_QCOM_SDM845_H__ */
diff --git a/drivers/interconnect/qcom/sm8150.h b/drivers/interconnect/qcom/sm8150.h
index 023161681fb8..1d587c94eb06 100644
--- a/drivers/interconnect/qcom/sm8150.h
+++ b/drivers/interconnect/qcom/sm8150.h
@@ -148,7 +148,5 @@
 #define SM8150_SLAVE_VSENSE_CTRL_CFG		137
 #define SM8150_SNOC_CNOC_MAS			138
 #define SM8150_SNOC_CNOC_SLV			139
-#define SM8150_MASTER_OSM_L3_APPS		140
-#define SM8150_SLAVE_OSM_L3			141
 
 #endif
diff --git a/drivers/interconnect/qcom/sm8250.h b/drivers/interconnect/qcom/sm8250.h
index e3fc56bc7ca0..209ab195f21f 100644
--- a/drivers/interconnect/qcom/sm8250.h
+++ b/drivers/interconnect/qcom/sm8250.h
@@ -158,7 +158,5 @@
 #define SM8250_SLAVE_VSENSE_CTRL_CFG		147
 #define SM8250_SNOC_CNOC_MAS			148
 #define SM8250_SNOC_CNOC_SLV			149
-#define SM8250_MASTER_EPSS_L3_APPS		150
-#define SM8250_SLAVE_EPSS_L3			151
 
 #endif
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 10db680acaed..807c98de40d4 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -28,6 +28,7 @@
 #include <linux/fsl/mc.h>
 #include <linux/module.h>
 #include <linux/cc_platform.h>
+#include <linux/cdx/cdx_bus.h>
 #include <trace/events/iommu.h>
 #include <linux/sched/mm.h>
 #include <linux/msi.h>
@@ -89,7 +90,7 @@ static int iommu_bus_notifier(struct notifier_block *nb,
 			      unsigned long action, void *data);
 static int iommu_alloc_default_domain(struct iommu_group *group,
 				      struct device *dev);
-static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
+static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus,
 						 unsigned type);
 static int __iommu_attach_device(struct iommu_domain *domain,
 				 struct device *dev);
@@ -129,6 +130,9 @@ static struct bus_type * const iommu_buses[] = {
 #ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS
 	&host1x_context_device_bus_type,
 #endif
+#ifdef CONFIG_CDX_BUS
+	&cdx_bus_type,
+#endif
 };
 
 /*
@@ -1631,7 +1635,7 @@ static int iommu_get_def_domain_type(struct device *dev)
 	return 0;
 }
 
-static int iommu_group_alloc_default_domain(struct bus_type *bus,
+static int iommu_group_alloc_default_domain(const struct bus_type *bus,
 					    struct iommu_group *group,
 					    unsigned int type)
 {
@@ -1777,7 +1781,7 @@ static int probe_get_default_domain_type(struct device *dev, void *data)
 	return 0;
 }
 
-static void probe_alloc_default_domain(struct bus_type *bus,
+static void probe_alloc_default_domain(const struct bus_type *bus,
 				       struct iommu_group *group)
 {
 	struct __group_domain_type gtype;
@@ -1832,7 +1836,7 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group)
 					  iommu_do_create_direct_mappings);
 }
 
-int bus_iommu_probe(struct bus_type *bus)
+int bus_iommu_probe(const struct bus_type *bus)
 {
 	struct iommu_group *group, *next;
 	LIST_HEAD(group_list);
@@ -1876,7 +1880,7 @@ int bus_iommu_probe(struct bus_type *bus)
 	return ret;
 }
 
-bool iommu_present(struct bus_type *bus)
+bool iommu_present(const struct bus_type *bus)
 {
 	return bus->iommu_ops != NULL;
 }
@@ -1951,7 +1955,7 @@ void iommu_set_fault_handler(struct iommu_domain *domain,
 }
 EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
 
-static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
+static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus,
 						 unsigned type)
 {
 	struct iommu_domain *domain;
@@ -1976,7 +1980,7 @@ static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
 	return domain;
 }
 
-struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
+struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
 {
 	return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED);
 }
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
index a0c66f47a65a..4f9b2142274c 100644
--- a/drivers/iommu/iommufd/device.c
+++ b/drivers/iommu/iommufd/device.c
@@ -15,23 +15,6 @@ MODULE_PARM_DESC(
 	"Allow IOMMUFD to bind to devices even if the platform cannot isolate "
 	"the MSI interrupt window. Enabling this is a security weakness.");
 
-/*
- * A iommufd_device object represents the binding relationship between a
- * consuming driver and the iommufd. These objects are created/destroyed by
- * external drivers, not by userspace.
- */
-struct iommufd_device {
-	struct iommufd_object obj;
-	struct iommufd_ctx *ictx;
-	struct iommufd_hw_pagetable *hwpt;
-	/* Head at iommufd_hw_pagetable::devices */
-	struct list_head devices_item;
-	/* always the physical device */
-	struct device *dev;
-	struct iommu_group *group;
-	bool enforce_cache_coherency;
-};
-
 void iommufd_device_destroy(struct iommufd_object *obj)
 {
 	struct iommufd_device *idev =
@@ -39,7 +22,8 @@ void iommufd_device_destroy(struct iommufd_object *obj)
 
 	iommu_device_release_dma_owner(idev->dev);
 	iommu_group_put(idev->group);
-	iommufd_ctx_put(idev->ictx);
+	if (!iommufd_selftest_is_mock_dev(idev->dev))
+		iommufd_ctx_put(idev->ictx);
 }
 
 /**
@@ -86,7 +70,8 @@ struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
 		goto out_release_owner;
 	}
 	idev->ictx = ictx;
-	iommufd_ctx_get(ictx);
+	if (!iommufd_selftest_is_mock_dev(dev))
+		iommufd_ctx_get(ictx);
 	idev->dev = dev;
 	idev->enforce_cache_coherency =
 		device_iommu_capable(dev, IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
@@ -168,7 +153,8 @@ static int iommufd_device_setup_msi(struct iommufd_device *idev,
 	 * operation from the device (eg a simple DMA) cannot trigger an
 	 * interrupt outside this iommufd context.
 	 */
-	if (!iommu_group_has_isolated_msi(idev->group)) {
+	if (!iommufd_selftest_is_mock_dev(idev->dev) &&
+	    !iommu_group_has_isolated_msi(idev->group)) {
 		if (!allow_unsafe_interrupts)
 			return -EPERM;
 
@@ -186,19 +172,24 @@ static bool iommufd_hw_pagetable_has_group(struct iommufd_hw_pagetable *hwpt,
 {
 	struct iommufd_device *cur_dev;
 
+	lockdep_assert_held(&hwpt->devices_lock);
+
 	list_for_each_entry(cur_dev, &hwpt->devices, devices_item)
 		if (cur_dev->group == group)
 			return true;
 	return false;
 }
 
-static int iommufd_device_do_attach(struct iommufd_device *idev,
-				    struct iommufd_hw_pagetable *hwpt)
+int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
+				struct iommufd_device *idev)
 {
 	phys_addr_t sw_msi_start = PHYS_ADDR_MAX;
 	int rc;
 
-	mutex_lock(&hwpt->devices_lock);
+	lockdep_assert_held(&hwpt->devices_lock);
+
+	if (WARN_ON(idev->hwpt))
+		return -EINVAL;
 
 	/*
 	 * Try to upgrade the domain we have, it is an iommu driver bug to
@@ -213,19 +204,18 @@ static int iommufd_device_do_attach(struct iommufd_device *idev,
 					hwpt->domain);
 		if (!hwpt->enforce_cache_coherency) {
 			WARN_ON(list_empty(&hwpt->devices));
-			rc = -EINVAL;
-			goto out_unlock;
+			return -EINVAL;
 		}
 	}
 
 	rc = iopt_table_enforce_group_resv_regions(&hwpt->ioas->iopt, idev->dev,
 						   idev->group, &sw_msi_start);
 	if (rc)
-		goto out_unlock;
+		return rc;
 
 	rc = iommufd_device_setup_msi(idev, hwpt, sw_msi_start);
 	if (rc)
-		goto out_iova;
+		goto err_unresv;
 
 	/*
 	 * FIXME: Hack around missing a device-centric iommu api, only attach to
@@ -234,26 +224,35 @@ static int iommufd_device_do_attach(struct iommufd_device *idev,
 	if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) {
 		rc = iommu_attach_group(hwpt->domain, idev->group);
 		if (rc)
-			goto out_iova;
-
-		if (list_empty(&hwpt->devices)) {
-			rc = iopt_table_add_domain(&hwpt->ioas->iopt,
-						   hwpt->domain);
-			if (rc)
-				goto out_detach;
-		}
+			goto err_unresv;
 	}
+	return 0;
+err_unresv:
+	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
+	return rc;
+}
+
+void iommufd_hw_pagetable_detach(struct iommufd_hw_pagetable *hwpt,
+				 struct iommufd_device *idev)
+{
+	if (!iommufd_hw_pagetable_has_group(hwpt, idev->group))
+		iommu_detach_group(hwpt->domain, idev->group);
+	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
+}
+
+static int iommufd_device_do_attach(struct iommufd_device *idev,
+				    struct iommufd_hw_pagetable *hwpt)
+{
+	int rc;
+
+	mutex_lock(&hwpt->devices_lock);
+	rc = iommufd_hw_pagetable_attach(hwpt, idev);
+	if (rc)
+		goto out_unlock;
 
 	idev->hwpt = hwpt;
 	refcount_inc(&hwpt->obj.users);
 	list_add(&idev->devices_item, &hwpt->devices);
-	mutex_unlock(&hwpt->devices_lock);
-	return 0;
-
-out_detach:
-	iommu_detach_group(hwpt->domain, idev->group);
-out_iova:
-	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
 out_unlock:
 	mutex_unlock(&hwpt->devices_lock);
 	return rc;
@@ -280,7 +279,10 @@ static int iommufd_device_auto_get_domain(struct iommufd_device *idev,
 		if (!hwpt->auto_domain)
 			continue;
 
+		if (!iommufd_lock_obj(&hwpt->obj))
+			continue;
 		rc = iommufd_device_do_attach(idev, hwpt);
+		iommufd_put_object(&hwpt->obj);
 
 		/*
 		 * -EINVAL means the domain is incompatible with the device.
@@ -292,24 +294,16 @@ static int iommufd_device_auto_get_domain(struct iommufd_device *idev,
 		goto out_unlock;
 	}
 
-	hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev->dev);
+	hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev, true);
 	if (IS_ERR(hwpt)) {
 		rc = PTR_ERR(hwpt);
 		goto out_unlock;
 	}
 	hwpt->auto_domain = true;
 
-	rc = iommufd_device_do_attach(idev, hwpt);
-	if (rc)
-		goto out_abort;
-	list_add_tail(&hwpt->hwpt_item, &ioas->hwpt_list);
-
 	mutex_unlock(&ioas->mutex);
 	iommufd_object_finalize(idev->ictx, &hwpt->obj);
 	return 0;
-
-out_abort:
-	iommufd_object_abort_and_destroy(idev->ictx, &hwpt->obj);
 out_unlock:
 	mutex_unlock(&ioas->mutex);
 	return rc;
@@ -381,28 +375,17 @@ void iommufd_device_detach(struct iommufd_device *idev)
 {
 	struct iommufd_hw_pagetable *hwpt = idev->hwpt;
 
-	mutex_lock(&hwpt->ioas->mutex);
 	mutex_lock(&hwpt->devices_lock);
 	list_del(&idev->devices_item);
-	if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) {
-		if (list_empty(&hwpt->devices)) {
-			iopt_table_remove_domain(&hwpt->ioas->iopt,
-						 hwpt->domain);
-			list_del(&hwpt->hwpt_item);
-		}
-		iommu_detach_group(hwpt->domain, idev->group);
-	}
-	iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev);
+	idev->hwpt = NULL;
+	iommufd_hw_pagetable_detach(hwpt, idev);
 	mutex_unlock(&hwpt->devices_lock);
-	mutex_unlock(&hwpt->ioas->mutex);
 
 	if (hwpt->auto_domain)
 		iommufd_object_destroy_user(idev->ictx, &hwpt->obj);
 	else
 		refcount_dec(&hwpt->obj.users);
 
-	idev->hwpt = NULL;
-
 	refcount_dec(&idev->obj.users);
 }
 EXPORT_SYMBOL_NS_GPL(iommufd_device_detach, IOMMUFD);
@@ -412,17 +395,20 @@ void iommufd_access_destroy_object(struct iommufd_object *obj)
 	struct iommufd_access *access =
 		container_of(obj, struct iommufd_access, obj);
 
-	iopt_remove_access(&access->ioas->iopt, access);
+	if (access->ioas) {
+		iopt_remove_access(&access->ioas->iopt, access);
+		refcount_dec(&access->ioas->obj.users);
+		access->ioas = NULL;
+	}
 	iommufd_ctx_put(access->ictx);
-	refcount_dec(&access->ioas->obj.users);
 }
 
 /**
  * iommufd_access_create - Create an iommufd_access
  * @ictx: iommufd file descriptor
- * @ioas_id: ID for a IOMMUFD_OBJ_IOAS
  * @ops: Driver's ops to associate with the access
  * @data: Opaque data to pass into ops functions
+ * @id: Output ID number to return to userspace for this access
  *
  * An iommufd_access allows a driver to read/write to the IOAS without using
  * DMA. The underlying CPU memory can be accessed using the
@@ -431,12 +417,10 @@ void iommufd_access_destroy_object(struct iommufd_object *obj)
  * The provided ops are required to use iommufd_access_pin_pages().
  */
 struct iommufd_access *
-iommufd_access_create(struct iommufd_ctx *ictx, u32 ioas_id,
-		      const struct iommufd_access_ops *ops, void *data)
+iommufd_access_create(struct iommufd_ctx *ictx,
+		      const struct iommufd_access_ops *ops, void *data, u32 *id)
 {
 	struct iommufd_access *access;
-	struct iommufd_object *obj;
-	int rc;
 
 	/*
 	 * There is no uAPI for the access object, but to keep things symmetric
@@ -449,33 +433,18 @@ iommufd_access_create(struct iommufd_ctx *ictx, u32 ioas_id,
 	access->data = data;
 	access->ops = ops;
 
-	obj = iommufd_get_object(ictx, ioas_id, IOMMUFD_OBJ_IOAS);
-	if (IS_ERR(obj)) {
-		rc = PTR_ERR(obj);
-		goto out_abort;
-	}
-	access->ioas = container_of(obj, struct iommufd_ioas, obj);
-	iommufd_ref_to_users(obj);
-
 	if (ops->needs_pin_pages)
 		access->iova_alignment = PAGE_SIZE;
 	else
 		access->iova_alignment = 1;
-	rc = iopt_add_access(&access->ioas->iopt, access);
-	if (rc)
-		goto out_put_ioas;
 
 	/* The calling driver is a user until iommufd_access_destroy() */
 	refcount_inc(&access->obj.users);
 	access->ictx = ictx;
 	iommufd_ctx_get(ictx);
 	iommufd_object_finalize(ictx, &access->obj);
+	*id = access->obj.id;
 	return access;
-out_put_ioas:
-	refcount_dec(&access->ioas->obj.users);
-out_abort:
-	iommufd_object_abort(ictx, &access->obj);
-	return ERR_PTR(rc);
 }
 EXPORT_SYMBOL_NS_GPL(iommufd_access_create, IOMMUFD);
 
@@ -494,6 +463,30 @@ void iommufd_access_destroy(struct iommufd_access *access)
 }
 EXPORT_SYMBOL_NS_GPL(iommufd_access_destroy, IOMMUFD);
 
+int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id)
+{
+	struct iommufd_ioas *new_ioas;
+	int rc = 0;
+
+	if (access->ioas)
+		return -EINVAL;
+
+	new_ioas = iommufd_get_ioas(access->ictx, ioas_id);
+	if (IS_ERR(new_ioas))
+		return PTR_ERR(new_ioas);
+
+	rc = iopt_add_access(&new_ioas->iopt, access);
+	if (rc) {
+		iommufd_put_object(&new_ioas->obj);
+		return rc;
+	}
+	iommufd_ref_to_users(&new_ioas->obj);
+
+	access->ioas = new_ioas;
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(iommufd_access_attach, IOMMUFD);
+
 /**
  * iommufd_access_notify_unmap - Notify users of an iopt to stop using it
  * @iopt: iopt to work on
@@ -726,41 +719,3 @@ err_out:
 	return rc;
 }
 EXPORT_SYMBOL_NS_GPL(iommufd_access_rw, IOMMUFD);
-
-#ifdef CONFIG_IOMMUFD_TEST
-/*
- * Creating a real iommufd_device is too hard, bypass creating a iommufd_device
- * and go directly to attaching a domain.
- */
-struct iommufd_hw_pagetable *
-iommufd_device_selftest_attach(struct iommufd_ctx *ictx,
-			       struct iommufd_ioas *ioas,
-			       struct device *mock_dev)
-{
-	struct iommufd_hw_pagetable *hwpt;
-	int rc;
-
-	hwpt = iommufd_hw_pagetable_alloc(ictx, ioas, mock_dev);
-	if (IS_ERR(hwpt))
-		return hwpt;
-
-	rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain);
-	if (rc)
-		goto out_hwpt;
-
-	refcount_inc(&hwpt->obj.users);
-	iommufd_object_finalize(ictx, &hwpt->obj);
-	return hwpt;
-
-out_hwpt:
-	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
-	return ERR_PTR(rc);
-}
-
-void iommufd_device_selftest_detach(struct iommufd_ctx *ictx,
-				    struct iommufd_hw_pagetable *hwpt)
-{
-	iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain);
-	refcount_dec(&hwpt->obj.users);
-}
-#endif
diff --git a/drivers/iommu/iommufd/hw_pagetable.c b/drivers/iommu/iommufd/hw_pagetable.c
index 43d473989a06..6cdb6749d359 100644
--- a/drivers/iommu/iommufd/hw_pagetable.c
+++ b/drivers/iommu/iommufd/hw_pagetable.c
@@ -13,7 +13,17 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj)
 
 	WARN_ON(!list_empty(&hwpt->devices));
 
-	iommu_domain_free(hwpt->domain);
+	if (!list_empty(&hwpt->hwpt_item)) {
+		mutex_lock(&hwpt->ioas->mutex);
+		list_del(&hwpt->hwpt_item);
+		mutex_unlock(&hwpt->ioas->mutex);
+
+		iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain);
+	}
+
+	if (hwpt->domain)
+		iommu_domain_free(hwpt->domain);
+
 	refcount_dec(&hwpt->ioas->obj.users);
 	mutex_destroy(&hwpt->devices_lock);
 }
@@ -22,36 +32,74 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj)
  * iommufd_hw_pagetable_alloc() - Get an iommu_domain for a device
  * @ictx: iommufd context
  * @ioas: IOAS to associate the domain with
- * @dev: Device to get an iommu_domain for
+ * @idev: Device to get an iommu_domain for
+ * @immediate_attach: True if idev should be attached to the hwpt
  *
- * Allocate a new iommu_domain and return it as a hw_pagetable.
+ * Allocate a new iommu_domain and return it as a hw_pagetable. The HWPT
+ * will be linked to the given ioas and upon return the underlying iommu_domain
+ * is fully popoulated.
  */
 struct iommufd_hw_pagetable *
 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
-			   struct device *dev)
+			   struct iommufd_device *idev, bool immediate_attach)
 {
 	struct iommufd_hw_pagetable *hwpt;
 	int rc;
 
+	lockdep_assert_held(&ioas->mutex);
+
 	hwpt = iommufd_object_alloc(ictx, hwpt, IOMMUFD_OBJ_HW_PAGETABLE);
 	if (IS_ERR(hwpt))
 		return hwpt;
 
-	hwpt->domain = iommu_domain_alloc(dev->bus);
-	if (!hwpt->domain) {
-		rc = -ENOMEM;
-		goto out_abort;
-	}
-
 	INIT_LIST_HEAD(&hwpt->devices);
 	INIT_LIST_HEAD(&hwpt->hwpt_item);
 	mutex_init(&hwpt->devices_lock);
 	/* Pairs with iommufd_hw_pagetable_destroy() */
 	refcount_inc(&ioas->obj.users);
 	hwpt->ioas = ioas;
+
+	hwpt->domain = iommu_domain_alloc(idev->dev->bus);
+	if (!hwpt->domain) {
+		rc = -ENOMEM;
+		goto out_abort;
+	}
+
+	mutex_lock(&hwpt->devices_lock);
+
+	/*
+	 * immediate_attach exists only to accommodate iommu drivers that cannot
+	 * directly allocate a domain. These drivers do not finish creating the
+	 * domain until attach is completed. Thus we must have this call
+	 * sequence. Once those drivers are fixed this should be removed.
+	 */
+	if (immediate_attach) {
+		rc = iommufd_hw_pagetable_attach(hwpt, idev);
+		if (rc)
+			goto out_unlock;
+	}
+
+	rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain);
+	if (rc)
+		goto out_detach;
+	list_add_tail(&hwpt->hwpt_item, &hwpt->ioas->hwpt_list);
+
+	if (immediate_attach) {
+		/* See iommufd_device_do_attach() */
+		refcount_inc(&hwpt->obj.users);
+		idev->hwpt = hwpt;
+		list_add(&idev->devices_item, &hwpt->devices);
+	}
+
+	mutex_unlock(&hwpt->devices_lock);
 	return hwpt;
 
+out_detach:
+	if (immediate_attach)
+		iommufd_hw_pagetable_detach(hwpt, idev);
+out_unlock:
+	mutex_unlock(&hwpt->devices_lock);
 out_abort:
-	iommufd_object_abort(ictx, &hwpt->obj);
+	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
 	return ERR_PTR(rc);
 }
diff --git a/drivers/iommu/iommufd/ioas.c b/drivers/iommu/iommufd/ioas.c
index 31577e9d434f..d5624577f79f 100644
--- a/drivers/iommu/iommufd/ioas.c
+++ b/drivers/iommu/iommufd/ioas.c
@@ -71,7 +71,7 @@ int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd)
 	if (cmd->__reserved)
 		return -EOPNOTSUPP;
 
-	ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
+	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
 	if (IS_ERR(ioas))
 		return PTR_ERR(ioas);
 
@@ -151,7 +151,7 @@ int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd)
 	if (cmd->__reserved)
 		return -EOPNOTSUPP;
 
-	ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
+	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
 	if (IS_ERR(ioas))
 		return PTR_ERR(ioas);
 	iopt = &ioas->iopt;
@@ -213,7 +213,7 @@ int iommufd_ioas_map(struct iommufd_ucmd *ucmd)
 	if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX)
 		return -EOVERFLOW;
 
-	ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
+	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
 	if (IS_ERR(ioas))
 		return PTR_ERR(ioas);
 
@@ -253,7 +253,7 @@ int iommufd_ioas_copy(struct iommufd_ucmd *ucmd)
 	    cmd->dst_iova >= ULONG_MAX)
 		return -EOVERFLOW;
 
-	src_ioas = iommufd_get_ioas(ucmd, cmd->src_ioas_id);
+	src_ioas = iommufd_get_ioas(ucmd->ictx, cmd->src_ioas_id);
 	if (IS_ERR(src_ioas))
 		return PTR_ERR(src_ioas);
 	rc = iopt_get_pages(&src_ioas->iopt, cmd->src_iova, cmd->length,
@@ -262,7 +262,7 @@ int iommufd_ioas_copy(struct iommufd_ucmd *ucmd)
 	if (rc)
 		return rc;
 
-	dst_ioas = iommufd_get_ioas(ucmd, cmd->dst_ioas_id);
+	dst_ioas = iommufd_get_ioas(ucmd->ictx, cmd->dst_ioas_id);
 	if (IS_ERR(dst_ioas)) {
 		rc = PTR_ERR(dst_ioas);
 		goto out_pages;
@@ -292,7 +292,7 @@ int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd)
 	unsigned long unmapped = 0;
 	int rc;
 
-	ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
+	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
 	if (IS_ERR(ioas))
 		return PTR_ERR(ioas);
 
@@ -381,7 +381,7 @@ int iommufd_ioas_option(struct iommufd_ucmd *ucmd)
 	if (cmd->__reserved)
 		return -EOPNOTSUPP;
 
-	ioas = iommufd_get_ioas(ucmd, cmd->object_id);
+	ioas = iommufd_get_ioas(ucmd->ictx, cmd->object_id);
 	if (IS_ERR(ioas))
 		return PTR_ERR(ioas);
 
diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h
index 9d7f71510ca1..b38e67d1988b 100644
--- a/drivers/iommu/iommufd/iommufd_private.h
+++ b/drivers/iommu/iommufd/iommufd_private.h
@@ -12,6 +12,7 @@
 struct iommu_domain;
 struct iommu_group;
 struct iommu_option;
+struct iommufd_device;
 
 struct iommufd_ctx {
 	struct file *file;
@@ -211,10 +212,10 @@ struct iommufd_ioas {
 	struct list_head hwpt_list;
 };
 
-static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ucmd *ucmd,
+static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ctx *ictx,
 						    u32 id)
 {
-	return container_of(iommufd_get_object(ucmd->ictx, id,
+	return container_of(iommufd_get_object(ictx, id,
 					       IOMMUFD_OBJ_IOAS),
 			    struct iommufd_ioas, obj);
 }
@@ -254,9 +255,30 @@ struct iommufd_hw_pagetable {
 
 struct iommufd_hw_pagetable *
 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
-			   struct device *dev);
+			   struct iommufd_device *idev, bool immediate_attach);
+int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
+				struct iommufd_device *idev);
+void iommufd_hw_pagetable_detach(struct iommufd_hw_pagetable *hwpt,
+				 struct iommufd_device *idev);
 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj);
 
+/*
+ * A iommufd_device object represents the binding relationship between a
+ * consuming driver and the iommufd. These objects are created/destroyed by
+ * external drivers, not by userspace.
+ */
+struct iommufd_device {
+	struct iommufd_object obj;
+	struct iommufd_ctx *ictx;
+	struct iommufd_hw_pagetable *hwpt;
+	/* Head at iommufd_hw_pagetable::devices */
+	struct list_head devices_item;
+	/* always the physical device */
+	struct device *dev;
+	struct iommu_group *group;
+	bool enforce_cache_coherency;
+};
+
 void iommufd_device_destroy(struct iommufd_object *obj);
 
 struct iommufd_access {
@@ -275,12 +297,6 @@ void iopt_remove_access(struct io_pagetable *iopt,
 void iommufd_access_destroy_object(struct iommufd_object *obj);
 
 #ifdef CONFIG_IOMMUFD_TEST
-struct iommufd_hw_pagetable *
-iommufd_device_selftest_attach(struct iommufd_ctx *ictx,
-			       struct iommufd_ioas *ioas,
-			       struct device *mock_dev);
-void iommufd_device_selftest_detach(struct iommufd_ctx *ictx,
-				    struct iommufd_hw_pagetable *hwpt);
 int iommufd_test(struct iommufd_ucmd *ucmd);
 void iommufd_selftest_destroy(struct iommufd_object *obj);
 extern size_t iommufd_test_memory_limit;
@@ -289,6 +305,7 @@ void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
 bool iommufd_should_fail(void);
 void __init iommufd_test_init(void);
 void iommufd_test_exit(void);
+bool iommufd_selftest_is_mock_dev(struct device *dev);
 #else
 static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
 						 unsigned int ioas_id,
@@ -305,5 +322,9 @@ static inline void __init iommufd_test_init(void)
 static inline void iommufd_test_exit(void)
 {
 }
+static inline bool iommufd_selftest_is_mock_dev(struct device *dev)
+{
+	return false;
+}
 #endif
 #endif
diff --git a/drivers/iommu/iommufd/iommufd_test.h b/drivers/iommu/iommufd/iommufd_test.h
index 1d96a8f466fd..b3d69cca7729 100644
--- a/drivers/iommu/iommufd/iommufd_test.h
+++ b/drivers/iommu/iommufd/iommufd_test.h
@@ -49,7 +49,7 @@ struct iommu_test_cmd {
 			__aligned_u64 length;
 		} add_reserved;
 		struct {
-			__u32 out_device_id;
+			__u32 out_stdev_id;
 			__u32 out_hwpt_id;
 		} mock_domain;
 		struct {
diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c
index cfb5fe9a5e0e..74c2076105d4 100644
--- a/drivers/iommu/iommufd/selftest.c
+++ b/drivers/iommu/iommufd/selftest.c
@@ -75,7 +75,7 @@ void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
 		return;
 	*flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ;
 
-	ioas = iommufd_get_ioas(ucmd, ioas_id);
+	ioas = iommufd_get_ioas(ucmd->ictx, ioas_id);
 	if (IS_ERR(ioas))
 		return;
 	*iova = iommufd_test_syz_conv_iova(&ioas->iopt, iova);
@@ -91,23 +91,50 @@ enum selftest_obj_type {
 	TYPE_IDEV,
 };
 
+struct mock_dev {
+	struct device dev;
+};
+
 struct selftest_obj {
 	struct iommufd_object obj;
 	enum selftest_obj_type type;
 
 	union {
 		struct {
-			struct iommufd_hw_pagetable *hwpt;
+			struct iommufd_device *idev;
 			struct iommufd_ctx *ictx;
-			struct device mock_dev;
+			struct mock_dev *mock_dev;
 		} idev;
 	};
 };
 
+static void mock_domain_blocking_free(struct iommu_domain *domain)
+{
+}
+
+static int mock_domain_nop_attach(struct iommu_domain *domain,
+				  struct device *dev)
+{
+	return 0;
+}
+
+static const struct iommu_domain_ops mock_blocking_ops = {
+	.free = mock_domain_blocking_free,
+	.attach_dev = mock_domain_nop_attach,
+};
+
+static struct iommu_domain mock_blocking_domain = {
+	.type = IOMMU_DOMAIN_BLOCKED,
+	.ops = &mock_blocking_ops,
+};
+
 static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type)
 {
 	struct mock_iommu_domain *mock;
 
+	if (iommu_domain_type == IOMMU_DOMAIN_BLOCKED)
+		return &mock_blocking_domain;
+
 	if (WARN_ON(iommu_domain_type != IOMMU_DOMAIN_UNMANAGED))
 		return NULL;
 
@@ -236,19 +263,39 @@ static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain,
 	return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE;
 }
 
+static bool mock_domain_capable(struct device *dev, enum iommu_cap cap)
+{
+	return cap == IOMMU_CAP_CACHE_COHERENCY;
+}
+
+static void mock_domain_set_plaform_dma_ops(struct device *dev)
+{
+	/*
+	 * mock doesn't setup default domains because we can't hook into the
+	 * normal probe path
+	 */
+}
+
 static const struct iommu_ops mock_ops = {
 	.owner = THIS_MODULE,
 	.pgsize_bitmap = MOCK_IO_PAGE_SIZE,
 	.domain_alloc = mock_domain_alloc,
+	.capable = mock_domain_capable,
+	.set_platform_dma_ops = mock_domain_set_plaform_dma_ops,
 	.default_domain_ops =
 		&(struct iommu_domain_ops){
 			.free = mock_domain_free,
+			.attach_dev = mock_domain_nop_attach,
 			.map_pages = mock_domain_map_pages,
 			.unmap_pages = mock_domain_unmap_pages,
 			.iova_to_phys = mock_domain_iova_to_phys,
 		},
 };
 
+static struct iommu_device mock_iommu_device = {
+	.ops = &mock_ops,
+};
+
 static inline struct iommufd_hw_pagetable *
 get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id,
 		 struct mock_iommu_domain **mock)
@@ -269,48 +316,142 @@ get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id,
 	return hwpt;
 }
 
+static struct bus_type iommufd_mock_bus_type = {
+	.name = "iommufd_mock",
+	.iommu_ops = &mock_ops,
+};
+
+static void mock_dev_release(struct device *dev)
+{
+	struct mock_dev *mdev = container_of(dev, struct mock_dev, dev);
+
+	kfree(mdev);
+}
+
+static struct mock_dev *mock_dev_create(void)
+{
+	struct iommu_group *iommu_group;
+	struct dev_iommu *dev_iommu;
+	struct mock_dev *mdev;
+	int rc;
+
+	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
+	if (!mdev)
+		return ERR_PTR(-ENOMEM);
+
+	device_initialize(&mdev->dev);
+	mdev->dev.release = mock_dev_release;
+	mdev->dev.bus = &iommufd_mock_bus_type;
+
+	iommu_group = iommu_group_alloc();
+	if (IS_ERR(iommu_group)) {
+		rc = PTR_ERR(iommu_group);
+		goto err_put;
+	}
+
+	rc = dev_set_name(&mdev->dev, "iommufd_mock%u",
+			  iommu_group_id(iommu_group));
+	if (rc)
+		goto err_group;
+
+	/*
+	 * The iommu core has no way to associate a single device with an iommu
+	 * driver (heck currently it can't even support two iommu_drivers
+	 * registering). Hack it together with an open coded dev_iommu_get().
+	 * Notice that the normal notifier triggered iommu release process also
+	 * does not work here because this bus is not in iommu_buses.
+	 */
+	mdev->dev.iommu = kzalloc(sizeof(*dev_iommu), GFP_KERNEL);
+	if (!mdev->dev.iommu) {
+		rc = -ENOMEM;
+		goto err_group;
+	}
+	mutex_init(&mdev->dev.iommu->lock);
+	mdev->dev.iommu->iommu_dev = &mock_iommu_device;
+
+	rc = device_add(&mdev->dev);
+	if (rc)
+		goto err_dev_iommu;
+
+	rc = iommu_group_add_device(iommu_group, &mdev->dev);
+	if (rc)
+		goto err_del;
+	iommu_group_put(iommu_group);
+	return mdev;
+
+err_del:
+	device_del(&mdev->dev);
+err_dev_iommu:
+	kfree(mdev->dev.iommu);
+	mdev->dev.iommu = NULL;
+err_group:
+	iommu_group_put(iommu_group);
+err_put:
+	put_device(&mdev->dev);
+	return ERR_PTR(rc);
+}
+
+static void mock_dev_destroy(struct mock_dev *mdev)
+{
+	iommu_group_remove_device(&mdev->dev);
+	device_del(&mdev->dev);
+	kfree(mdev->dev.iommu);
+	mdev->dev.iommu = NULL;
+	put_device(&mdev->dev);
+}
+
+bool iommufd_selftest_is_mock_dev(struct device *dev)
+{
+	return dev->release == mock_dev_release;
+}
+
 /* Create an hw_pagetable with the mock domain so we can test the domain ops */
 static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd,
 				    struct iommu_test_cmd *cmd)
 {
-	static struct bus_type mock_bus = { .iommu_ops = &mock_ops };
-	struct iommufd_hw_pagetable *hwpt;
+	struct iommufd_device *idev;
 	struct selftest_obj *sobj;
-	struct iommufd_ioas *ioas;
+	u32 pt_id = cmd->id;
+	u32 idev_id;
 	int rc;
 
-	ioas = iommufd_get_ioas(ucmd, cmd->id);
-	if (IS_ERR(ioas))
-		return PTR_ERR(ioas);
-
 	sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST);
-	if (IS_ERR(sobj)) {
-		rc = PTR_ERR(sobj);
-		goto out_ioas;
-	}
+	if (IS_ERR(sobj))
+		return PTR_ERR(sobj);
+
 	sobj->idev.ictx = ucmd->ictx;
 	sobj->type = TYPE_IDEV;
-	sobj->idev.mock_dev.bus = &mock_bus;
 
-	hwpt = iommufd_device_selftest_attach(ucmd->ictx, ioas,
-					      &sobj->idev.mock_dev);
-	if (IS_ERR(hwpt)) {
-		rc = PTR_ERR(hwpt);
+	sobj->idev.mock_dev = mock_dev_create();
+	if (IS_ERR(sobj->idev.mock_dev)) {
+		rc = PTR_ERR(sobj->idev.mock_dev);
 		goto out_sobj;
 	}
-	sobj->idev.hwpt = hwpt;
 
-	/* Userspace must destroy both of these IDs to destroy the object */
-	cmd->mock_domain.out_hwpt_id = hwpt->obj.id;
-	cmd->mock_domain.out_device_id = sobj->obj.id;
+	idev = iommufd_device_bind(ucmd->ictx, &sobj->idev.mock_dev->dev,
+				   &idev_id);
+	if (IS_ERR(idev)) {
+		rc = PTR_ERR(idev);
+		goto out_mdev;
+	}
+	sobj->idev.idev = idev;
+
+	rc = iommufd_device_attach(idev, &pt_id);
+	if (rc)
+		goto out_unbind;
+
+	/* Userspace must destroy the device_id to destroy the object */
+	cmd->mock_domain.out_hwpt_id = pt_id;
+	cmd->mock_domain.out_stdev_id = sobj->obj.id;
 	iommufd_object_finalize(ucmd->ictx, &sobj->obj);
-	iommufd_put_object(&ioas->obj);
 	return iommufd_ucmd_respond(ucmd, sizeof(*cmd));
 
+out_unbind:
+	iommufd_device_unbind(idev);
+out_mdev:
+	mock_dev_destroy(sobj->idev.mock_dev);
 out_sobj:
 	iommufd_object_abort(ucmd->ictx, &sobj->obj);
-out_ioas:
-	iommufd_put_object(&ioas->obj);
 	return rc;
 }
 
@@ -322,7 +463,7 @@ static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd,
 	struct iommufd_ioas *ioas;
 	int rc;
 
-	ioas = iommufd_get_ioas(ucmd, mockpt_id);
+	ioas = iommufd_get_ioas(ucmd->ictx, mockpt_id);
 	if (IS_ERR(ioas))
 		return PTR_ERR(ioas);
 	down_write(&ioas->iopt.iova_rwsem);
@@ -339,10 +480,12 @@ static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd,
 {
 	struct iommufd_hw_pagetable *hwpt;
 	struct mock_iommu_domain *mock;
+	uintptr_t end;
 	int rc;
 
 	if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE ||
-	    (uintptr_t)uptr % MOCK_IO_PAGE_SIZE)
+	    (uintptr_t)uptr % MOCK_IO_PAGE_SIZE ||
+	    check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end))
 		return -EINVAL;
 
 	hwpt = get_md_pagetable(ucmd, mockpt_id, &mock);
@@ -390,7 +533,10 @@ static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd,
 				      void __user *uptr, size_t length,
 				      unsigned int refs)
 {
-	if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE)
+	uintptr_t end;
+
+	if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE ||
+	    check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end))
 		return -EINVAL;
 
 	for (; length; length -= PAGE_SIZE) {
@@ -554,6 +700,7 @@ static int iommufd_test_create_access(struct iommufd_ucmd *ucmd,
 	struct iommu_test_cmd *cmd = ucmd->cmd;
 	struct selftest_access *staccess;
 	struct iommufd_access *access;
+	u32 id;
 	int fdno;
 	int rc;
 
@@ -571,15 +718,18 @@ static int iommufd_test_create_access(struct iommufd_ucmd *ucmd,
 	}
 
 	access = iommufd_access_create(
-		ucmd->ictx, ioas_id,
+		ucmd->ictx,
 		(flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ?
 			&selftest_access_ops_pin :
 			&selftest_access_ops,
-		staccess);
+		staccess, &id);
 	if (IS_ERR(access)) {
 		rc = PTR_ERR(access);
 		goto out_put_fdno;
 	}
+	rc = iommufd_access_attach(access, ioas_id);
+	if (rc)
+		goto out_destroy;
 	cmd->create_access.out_access_fd = fdno;
 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
 	if (rc)
@@ -780,8 +930,9 @@ void iommufd_selftest_destroy(struct iommufd_object *obj)
 
 	switch (sobj->type) {
 	case TYPE_IDEV:
-		iommufd_device_selftest_detach(sobj->idev.ictx,
-					       sobj->idev.hwpt);
+		iommufd_device_detach(sobj->idev.idev);
+		iommufd_device_unbind(sobj->idev.idev);
+		mock_dev_destroy(sobj->idev.mock_dev);
 		break;
 	}
 }
@@ -845,9 +996,11 @@ void __init iommufd_test_init(void)
 {
 	dbgfs_root =
 		fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd);
+	WARN_ON(bus_register(&iommufd_mock_bus_type));
 }
 
 void iommufd_test_exit(void)
 {
 	debugfs_remove_recursive(dbgfs_root);
+	bus_unregister(&iommufd_mock_bus_type);
 }
diff --git a/drivers/iommu/iommufd/vfio_compat.c b/drivers/iommu/iommufd/vfio_compat.c
index 514494a0025b..fe02517c73cc 100644
--- a/drivers/iommu/iommufd/vfio_compat.c
+++ b/drivers/iommu/iommufd/vfio_compat.c
@@ -137,7 +137,7 @@ int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd)
 		return iommufd_ucmd_respond(ucmd, sizeof(*cmd));
 
 	case IOMMU_VFIO_IOAS_SET:
-		ioas = iommufd_get_ioas(ucmd, cmd->ioas_id);
+		ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
 		if (IS_ERR(ioas))
 			return PTR_ERR(ioas);
 		xa_lock(&ucmd->ictx->objects);
diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c
index 2d993d0cea7d..74c5cb93e900 100644
--- a/drivers/iommu/sun50i-iommu.c
+++ b/drivers/iommu/sun50i-iommu.c
@@ -1076,4 +1076,3 @@ builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe);
 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");
 MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>");
 MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>");
-MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index 7dc990eb2c9b..09e422da482f 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -7,6 +7,7 @@ config IRQCHIP
 
 config ARM_GIC
 	bool
+	depends on OF
 	select IRQ_DOMAIN_HIERARCHY
 	select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP
 
@@ -35,6 +36,7 @@ config ARM_GIC_V3
 	select IRQ_DOMAIN_HIERARCHY
 	select PARTITION_PERCPU
 	select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP
+	select HAVE_ARM_SMCCC_DISCOVERY
 
 config ARM_GIC_V3_ITS
 	bool
@@ -535,6 +537,7 @@ config TI_PRUSS_INTC
 config RISCV_INTC
 	bool
 	depends on RISCV
+	select IRQ_DOMAIN_HIERARCHY
 
 config SIFIVE_PLIC
 	bool
diff --git a/drivers/irqchip/irq-al-fic.c b/drivers/irqchip/irq-al-fic.c
index 886de028a901..dfb761e86c9c 100644
--- a/drivers/irqchip/irq-al-fic.c
+++ b/drivers/irqchip/irq-al-fic.c
@@ -26,7 +26,6 @@
 
 MODULE_AUTHOR("Talel Shenhar");
 MODULE_DESCRIPTION("Amazon's Annapurna Labs Interrupt Controller Driver");
-MODULE_LICENSE("GPL v2");
 
 enum al_fic_state {
 	AL_FIC_UNCONFIGURED = 0,
diff --git a/drivers/irqchip/irq-bcm6345-l1.c b/drivers/irqchip/irq-bcm6345-l1.c
index 6899e37810a8..fa113cb2529a 100644
--- a/drivers/irqchip/irq-bcm6345-l1.c
+++ b/drivers/irqchip/irq-bcm6345-l1.c
@@ -257,6 +257,9 @@ static int __init bcm6345_l1_init_one(struct device_node *dn,
 	if (!cpu->map_base)
 		return -ENOMEM;
 
+	if (!request_mem_region(res.start, sz, res.name))
+		pr_err("failed to request intc memory");
+
 	for (i = 0; i < n_words; i++) {
 		cpu->enable_cache[i] = 0;
 		__raw_writel(0, cpu->map_base + reg_enable(intc, i));
@@ -335,8 +338,7 @@ static int __init bcm6345_l1_of_init(struct device_node *dn,
 	for_each_cpu(idx, &intc->cpumask) {
 		struct bcm6345_l1_cpu *cpu = intc->cpus[idx];
 
-		pr_info("  CPU%u at MMIO 0x%p (irq = %d)\n", idx,
-				cpu->map_base, cpu->parent_irq);
+		pr_info("  CPU%u (irq = %d)\n", idx, cpu->parent_irq);
 	}
 
 	return 0;
diff --git a/drivers/irqchip/irq-csky-apb-intc.c b/drivers/irqchip/irq-csky-apb-intc.c
index 42d8a2438ebc..6710691e4c25 100644
--- a/drivers/irqchip/irq-csky-apb-intc.c
+++ b/drivers/irqchip/irq-csky-apb-intc.c
@@ -68,7 +68,7 @@ static void __init ck_set_gc(struct device_node *node, void __iomem *reg_base,
 	gc->chip_types[0].chip.irq_mask = irq_gc_mask_clr_bit;
 	gc->chip_types[0].chip.irq_unmask = irq_gc_mask_set_bit;
 
-	if (of_find_property(node, "csky,support-pulse-signal", NULL))
+	if (of_property_read_bool(node, "csky,support-pulse-signal"))
 		gc->chip_types[0].chip.irq_unmask = irq_ck_mask_set_bit;
 }
 
diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c
index f1e75b35a52a..f2ff4387870d 100644
--- a/drivers/irqchip/irq-gic-v2m.c
+++ b/drivers/irqchip/irq-gic-v2m.c
@@ -421,7 +421,7 @@ static int __init gicv2m_of_init(struct fwnode_handle *parent_handle,
 		u32 spi_start = 0, nr_spis = 0;
 		struct resource res;
 
-		if (!of_find_property(child, "msi-controller", NULL))
+		if (!of_property_read_bool(child, "msi-controller"))
 			continue;
 
 		ret = of_address_to_resource(child, 0, &res);
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 586271b8aa39..fa4641a5dfd8 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -42,9 +42,11 @@
 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING		(1ULL << 0)
 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375	(1ULL << 1)
 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144	(1ULL << 2)
+#define ITS_FLAGS_FORCE_NON_SHAREABLE		(1ULL << 3)
 
 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING	(1 << 0)
 #define RDIST_FLAGS_RD_TABLES_PREALLOCATED	(1 << 1)
+#define RDIST_FLAGS_FORCE_NON_SHAREABLE		(1 << 2)
 
 #define RD_LOCAL_LPI_ENABLED                    BIT(0)
 #define RD_LOCAL_PENDTABLE_PREALLOCATED         BIT(1)
@@ -2359,6 +2361,9 @@ retry_baser:
 	its_write_baser(its, baser, val);
 	tmp = baser->val;
 
+	if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)
+		tmp &= ~GITS_BASER_SHAREABILITY_MASK;
+
 	if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
 		/*
 		 * Shareability didn't stick. Just use
@@ -3096,6 +3101,9 @@ static void its_cpu_init_lpis(void)
 	gicr_write_propbaser(val, rbase + GICR_PROPBASER);
 	tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
 
+	if (gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE)
+		tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
+
 	if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
 		if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
 			/*
@@ -3120,6 +3128,9 @@ static void its_cpu_init_lpis(void)
 	gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
 	tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
 
+	if (gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE)
+		tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
+
 	if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
 		/*
 		 * The HW reports non-shareable, we must remove the
@@ -4710,6 +4721,19 @@ static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
 	return true;
 }
 
+static bool __maybe_unused its_enable_rk3588001(void *data)
+{
+	struct its_node *its = data;
+
+	if (!of_machine_is_compatible("rockchip,rk3588"))
+		return false;
+
+	its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
+	gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
+
+	return true;
+}
+
 static const struct gic_quirk its_quirks[] = {
 #ifdef CONFIG_CAVIUM_ERRATUM_22375
 	{
@@ -4756,6 +4780,14 @@ static const struct gic_quirk its_quirks[] = {
 		.init	= its_enable_quirk_hip07_161600802,
 	},
 #endif
+#ifdef CONFIG_ROCKCHIP_ERRATUM_3588001
+	{
+		.desc   = "ITS: Rockchip erratum RK3588001",
+		.iidr   = 0x0201743b,
+		.mask   = 0xffffffff,
+		.init   = its_enable_rk3588001,
+	},
+#endif
 	{
 	}
 };
@@ -5096,6 +5128,9 @@ static int __init its_probe_one(struct resource *res,
 	gits_write_cbaser(baser, its->base + GITS_CBASER);
 	tmp = gits_read_cbaser(its->base + GITS_CBASER);
 
+	if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)
+		tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
+
 	if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
 		if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
 			/*
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index fd134e1f481a..6fcee221f201 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -24,6 +24,9 @@
 #include <linux/irqchip/arm-gic-common.h>
 #include <linux/irqchip/arm-gic-v3.h>
 #include <linux/irqchip/irq-partition-percpu.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/arm-smccc.h>
 
 #include <asm/cputype.h>
 #include <asm/exception.h>
@@ -47,6 +50,7 @@ struct redist_region {
 
 struct gic_chip_data {
 	struct fwnode_handle	*fwnode;
+	phys_addr_t		dist_phys_base;
 	void __iomem		*dist_base;
 	struct redist_region	*redist_regions;
 	struct rdists		rdists;
@@ -59,6 +63,10 @@ struct gic_chip_data {
 	struct partition_desc	**ppi_descs;
 };
 
+#define T241_CHIPS_MAX		4
+static void __iomem *t241_dist_base_alias[T241_CHIPS_MAX] __read_mostly;
+static DEFINE_STATIC_KEY_FALSE(gic_nvidia_t241_erratum);
+
 static struct gic_chip_data gic_data __read_mostly;
 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
 
@@ -179,6 +187,39 @@ static inline bool gic_irq_in_rdist(struct irq_data *d)
 	}
 }
 
+static inline void __iomem *gic_dist_base_alias(struct irq_data *d)
+{
+	if (static_branch_unlikely(&gic_nvidia_t241_erratum)) {
+		irq_hw_number_t hwirq = irqd_to_hwirq(d);
+		u32 chip;
+
+		/*
+		 * For the erratum T241-FABRIC-4, read accesses to GICD_In{E}
+		 * registers are directed to the chip that owns the SPI. The
+		 * the alias region can also be used for writes to the
+		 * GICD_In{E} except GICD_ICENABLERn. Each chip has support
+		 * for 320 {E}SPIs. Mappings for all 4 chips:
+		 *    Chip0 = 32-351
+		 *    Chip1 = 352-671
+		 *    Chip2 = 672-991
+		 *    Chip3 = 4096-4415
+		 */
+		switch (__get_intid_range(hwirq)) {
+		case SPI_RANGE:
+			chip = (hwirq - 32) / 320;
+			break;
+		case ESPI_RANGE:
+			chip = 3;
+			break;
+		default:
+			unreachable();
+		}
+		return t241_dist_base_alias[chip];
+	}
+
+	return gic_data.dist_base;
+}
+
 static inline void __iomem *gic_dist_base(struct irq_data *d)
 {
 	switch (get_intid_range(d)) {
@@ -337,7 +378,7 @@ static int gic_peek_irq(struct irq_data *d, u32 offset)
 	if (gic_irq_in_rdist(d))
 		base = gic_data_rdist_sgi_base();
 	else
-		base = gic_data.dist_base;
+		base = gic_dist_base_alias(d);
 
 	return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
 }
@@ -588,7 +629,7 @@ static int gic_set_type(struct irq_data *d, unsigned int type)
 	if (gic_irq_in_rdist(d))
 		base = gic_data_rdist_sgi_base();
 	else
-		base = gic_data.dist_base;
+		base = gic_dist_base_alias(d);
 
 	offset = convert_offset_index(d, GICD_ICFGR, &index);
 
@@ -1708,6 +1749,43 @@ static bool gic_enable_quirk_hip06_07(void *data)
 	return false;
 }
 
+#define T241_CHIPN_MASK		GENMASK_ULL(45, 44)
+#define T241_CHIP_GICDA_OFFSET	0x1580000
+#define SMCCC_SOC_ID_T241	0x036b0241
+
+static bool gic_enable_quirk_nvidia_t241(void *data)
+{
+	s32 soc_id = arm_smccc_get_soc_id_version();
+	unsigned long chip_bmask = 0;
+	phys_addr_t phys;
+	u32 i;
+
+	/* Check JEP106 code for NVIDIA T241 chip (036b:0241) */
+	if ((soc_id < 0) || (soc_id != SMCCC_SOC_ID_T241))
+		return false;
+
+	/* Find the chips based on GICR regions PHYS addr */
+	for (i = 0; i < gic_data.nr_redist_regions; i++) {
+		chip_bmask |= BIT(FIELD_GET(T241_CHIPN_MASK,
+				  (u64)gic_data.redist_regions[i].phys_base));
+	}
+
+	if (hweight32(chip_bmask) < 3)
+		return false;
+
+	/* Setup GICD alias regions */
+	for (i = 0; i < ARRAY_SIZE(t241_dist_base_alias); i++) {
+		if (chip_bmask & BIT(i)) {
+			phys = gic_data.dist_phys_base + T241_CHIP_GICDA_OFFSET;
+			phys |= FIELD_PREP(T241_CHIPN_MASK, i);
+			t241_dist_base_alias[i] = ioremap(phys, SZ_64K);
+			WARN_ON_ONCE(!t241_dist_base_alias[i]);
+		}
+	}
+	static_branch_enable(&gic_nvidia_t241_erratum);
+	return true;
+}
+
 static const struct gic_quirk gic_quirks[] = {
 	{
 		.desc	= "GICv3: Qualcomm MSM8996 broken firmware",
@@ -1740,6 +1818,12 @@ static const struct gic_quirk gic_quirks[] = {
 		.init	= gic_enable_quirk_cavium_38539,
 	},
 	{
+		.desc	= "GICv3: NVIDIA erratum T241-FABRIC-4",
+		.iidr	= 0x0402043b,
+		.mask	= 0xffffffff,
+		.init	= gic_enable_quirk_nvidia_t241,
+	},
+	{
 	}
 };
 
@@ -1798,7 +1882,8 @@ static void gic_enable_nmi_support(void)
 		gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
 }
 
-static int __init gic_init_bases(void __iomem *dist_base,
+static int __init gic_init_bases(phys_addr_t dist_phys_base,
+				 void __iomem *dist_base,
 				 struct redist_region *rdist_regs,
 				 u32 nr_redist_regions,
 				 u64 redist_stride,
@@ -1814,6 +1899,7 @@ static int __init gic_init_bases(void __iomem *dist_base,
 		pr_info("GIC: Using split EOI/Deactivate mode\n");
 
 	gic_data.fwnode = handle;
+	gic_data.dist_phys_base = dist_phys_base;
 	gic_data.dist_base = dist_base;
 	gic_data.redist_regions = rdist_regs;
 	gic_data.nr_redist_regions = nr_redist_regions;
@@ -1841,10 +1927,13 @@ static int __init gic_init_bases(void __iomem *dist_base,
 	gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
 						 &gic_data);
 	gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
-	gic_data.rdists.has_rvpeid = true;
-	gic_data.rdists.has_vlpis = true;
-	gic_data.rdists.has_direct_lpi = true;
-	gic_data.rdists.has_vpend_valid_dirty = true;
+	if (!static_branch_unlikely(&gic_nvidia_t241_erratum)) {
+		/* Disable GICv4.x features for the erratum T241-FABRIC-4 */
+		gic_data.rdists.has_rvpeid = true;
+		gic_data.rdists.has_vlpis = true;
+		gic_data.rdists.has_direct_lpi = true;
+		gic_data.rdists.has_vpend_valid_dirty = true;
+	}
 
 	if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
 		err = -ENOMEM;
@@ -2050,6 +2139,7 @@ static void __iomem *gic_of_iomap(struct device_node *node, int idx,
 
 static int __init gic_of_init(struct device_node *node, struct device_node *parent)
 {
+	phys_addr_t dist_phys_base;
 	void __iomem *dist_base;
 	struct redist_region *rdist_regs;
 	struct resource res;
@@ -2063,6 +2153,8 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare
 		return PTR_ERR(dist_base);
 	}
 
+	dist_phys_base = res.start;
+
 	err = gic_validate_dist_version(dist_base);
 	if (err) {
 		pr_err("%pOF: no distributor detected, giving up\n", node);
@@ -2094,8 +2186,8 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare
 
 	gic_enable_of_quirks(node, gic_quirks, &gic_data);
 
-	err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions,
-			     redist_stride, &node->fwnode);
+	err = gic_init_bases(dist_phys_base, dist_base, rdist_regs,
+			     nr_redist_regions, redist_stride, &node->fwnode);
 	if (err)
 		goto out_unmap_rdist;
 
@@ -2411,8 +2503,9 @@ gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end)
 		goto out_redist_unmap;
 	}
 
-	err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs,
-			     acpi_data.nr_redist_regions, 0, gsi_domain_handle);
+	err = gic_init_bases(dist->base_address, acpi_data.dist_base,
+			     acpi_data.redist_regs, acpi_data.nr_redist_regions,
+			     0, gsi_domain_handle);
 	if (err)
 		goto out_fwhandle_free;
 
diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c
index 95e3d2a71db6..412196a7dad5 100644
--- a/drivers/irqchip/irq-gic.c
+++ b/drivers/irqchip/irq-gic.c
@@ -1081,10 +1081,6 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
 	return 0;
 }
 
-static void gic_irq_domain_unmap(struct irq_domain *d, unsigned int irq)
-{
-}
-
 static int gic_irq_domain_translate(struct irq_domain *d,
 				    struct irq_fwspec *fwspec,
 				    unsigned long *hwirq,
@@ -1167,11 +1163,6 @@ static const struct irq_domain_ops gic_irq_domain_hierarchy_ops = {
 	.free = irq_domain_free_irqs_top,
 };
 
-static const struct irq_domain_ops gic_irq_domain_ops = {
-	.map = gic_irq_domain_map,
-	.unmap = gic_irq_domain_unmap,
-};
-
 static int gic_init_bases(struct gic_chip_data *gic,
 			  struct fwnode_handle *handle)
 {
@@ -1219,30 +1210,9 @@ static int gic_init_bases(struct gic_chip_data *gic,
 		gic_irqs = 1020;
 	gic->gic_irqs = gic_irqs;
 
-	if (handle) {		/* DT/ACPI */
-		gic->domain = irq_domain_create_linear(handle, gic_irqs,
-						       &gic_irq_domain_hierarchy_ops,
-						       gic);
-	} else {		/* Legacy support */
-		/*
-		 * For primary GICs, skip over SGIs.
-		 * No secondary GIC support whatsoever.
-		 */
-		int irq_base;
-
-		gic_irqs -= 16; /* calculate # of irqs to allocate */
-
-		irq_base = irq_alloc_descs(16, 16, gic_irqs,
-					   numa_node_id());
-		if (irq_base < 0) {
-			WARN(1, "Cannot allocate irq_descs @ IRQ16, assuming pre-allocated\n");
-			irq_base = 16;
-		}
-
-		gic->domain = irq_domain_add_legacy(NULL, gic_irqs, irq_base,
-						    16, &gic_irq_domain_ops, gic);
-	}
-
+	gic->domain = irq_domain_create_linear(handle, gic_irqs,
+					       &gic_irq_domain_hierarchy_ops,
+					       gic);
 	if (WARN_ON(!gic->domain)) {
 		ret = -ENODEV;
 		goto error;
@@ -1297,23 +1267,6 @@ static int __init __gic_init_bases(struct gic_chip_data *gic,
 	return ret;
 }
 
-void __init gic_init(void __iomem *dist_base, void __iomem *cpu_base)
-{
-	struct gic_chip_data *gic;
-
-	/*
-	 * Non-DT/ACPI systems won't run a hypervisor, so let's not
-	 * bother with these...
-	 */
-	static_branch_disable(&supports_deactivate_key);
-
-	gic = &gic_data[0];
-	gic->raw_dist_base = dist_base;
-	gic->raw_cpu_base = cpu_base;
-
-	__gic_init_bases(gic, NULL);
-}
-
 static void gic_teardown(struct gic_chip_data *gic)
 {
 	if (WARN_ON(!gic))
@@ -1325,7 +1278,6 @@ static void gic_teardown(struct gic_chip_data *gic)
 		iounmap(gic->raw_cpu_base);
 }
 
-#ifdef CONFIG_OF
 static int gic_cnt __initdata;
 static bool gicv2_force_probe;
 
@@ -1570,12 +1522,6 @@ IRQCHIP_DECLARE(cortex_a7_gic, "arm,cortex-a7-gic", gic_of_init);
 IRQCHIP_DECLARE(msm_8660_qgic, "qcom,msm-8660-qgic", gic_of_init);
 IRQCHIP_DECLARE(msm_qgic2, "qcom,msm-qgic2", gic_of_init);
 IRQCHIP_DECLARE(pl390, "arm,pl390", gic_of_init);
-#else
-int gic_of_init_child(struct device *dev, struct gic_chip_data **gic, int irq)
-{
-	return -ENOTSUPP;
-}
-#endif
 
 #ifdef CONFIG_ACPI
 static struct
diff --git a/drivers/irqchip/irq-loongson-eiointc.c b/drivers/irqchip/irq-loongson-eiointc.c
index d15fd38c1756..71ef19f77a5a 100644
--- a/drivers/irqchip/irq-loongson-eiointc.c
+++ b/drivers/irqchip/irq-loongson-eiointc.c
@@ -7,16 +7,13 @@
 
 #define pr_fmt(fmt) "eiointc: " fmt
 
+#include <linux/cpuhotplug.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irqchip.h>
 #include <linux/irqdomain.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/kernel.h>
-#include <linux/platform_device.h>
-#include <linux/of_address.h>
-#include <linux/of_irq.h>
-#include <linux/of_platform.h>
 #include <linux/syscore_ops.h>
 
 #define EIOINTC_REG_NODEMAP	0x14a0
@@ -280,9 +277,6 @@ static void acpi_set_vec_parent(int node, struct irq_domain *parent, struct acpi
 {
 	int i;
 
-	if (cpu_has_flatmode)
-		node = cpu_to_node(node * CORES_PER_EIO_NODE);
-
 	for (i = 0; i < MAX_IO_PICS; i++) {
 		if (node == vec_group[i].node) {
 			vec_group[i].parent = parent;
@@ -343,19 +337,27 @@ static int __init pch_pic_parse_madt(union acpi_subtable_headers *header,
 	if (parent)
 		return pch_pic_acpi_init(parent, pchpic_entry);
 
-	return -EINVAL;
+	return 0;
 }
 
 static int __init pch_msi_parse_madt(union acpi_subtable_headers *header,
 					const unsigned long end)
 {
+	struct irq_domain *parent;
 	struct acpi_madt_msi_pic *pchmsi_entry = (struct acpi_madt_msi_pic *)header;
-	struct irq_domain *parent = acpi_get_vec_parent(eiointc_priv[nr_pics - 1]->node, msi_group);
+	int node;
+
+	if (cpu_has_flatmode)
+		node = cpu_to_node(eiointc_priv[nr_pics - 1]->node * CORES_PER_EIO_NODE);
+	else
+		node = eiointc_priv[nr_pics - 1]->node;
+
+	parent = acpi_get_vec_parent(node, msi_group);
 
 	if (parent)
 		return pch_msi_acpi_init(parent, pchmsi_entry);
 
-	return -EINVAL;
+	return 0;
 }
 
 static int __init acpi_cascade_irqdomain_init(void)
@@ -379,6 +381,7 @@ int __init eiointc_acpi_init(struct irq_domain *parent,
 	int i, ret, parent_irq;
 	unsigned long node_map;
 	struct eiointc_priv *priv;
+	int node;
 
 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -416,13 +419,19 @@ int __init eiointc_acpi_init(struct irq_domain *parent,
 	parent_irq = irq_create_mapping(parent, acpi_eiointc->cascade);
 	irq_set_chained_handler_and_data(parent_irq, eiointc_irq_dispatch, priv);
 
-	register_syscore_ops(&eiointc_syscore_ops);
-	cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_LOONGARCH_STARTING,
+	if (nr_pics == 1) {
+		register_syscore_ops(&eiointc_syscore_ops);
+		cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_LOONGARCH_STARTING,
 				  "irqchip/loongarch/intc:starting",
 				  eiointc_router_init, NULL);
+	}
 
-	acpi_set_vec_parent(acpi_eiointc->node, priv->eiointc_domain, pch_group);
-	acpi_set_vec_parent(acpi_eiointc->node, priv->eiointc_domain, msi_group);
+	if (cpu_has_flatmode)
+		node = cpu_to_node(acpi_eiointc->node * CORES_PER_EIO_NODE);
+	else
+		node = acpi_eiointc->node;
+	acpi_set_vec_parent(node, priv->eiointc_domain, pch_group);
+	acpi_set_vec_parent(node, priv->eiointc_domain, msi_group);
 	ret = acpi_cascade_irqdomain_init();
 
 	return ret;
diff --git a/drivers/irqchip/irq-loongson-pch-pic.c b/drivers/irqchip/irq-loongson-pch-pic.c
index 437f1af693d0..e5fe4d50be05 100644
--- a/drivers/irqchip/irq-loongson-pch-pic.c
+++ b/drivers/irqchip/irq-loongson-pch-pic.c
@@ -311,7 +311,8 @@ static int pch_pic_init(phys_addr_t addr, unsigned long size, int vec_base,
 	pch_pic_handle[nr_pics] = domain_handle;
 	pch_pic_priv[nr_pics++] = priv;
 
-	register_syscore_ops(&pch_pic_syscore_ops);
+	if (nr_pics == 1)
+		register_syscore_ops(&pch_pic_syscore_ops);
 
 	return 0;
 
@@ -403,6 +404,9 @@ int __init pch_pic_acpi_init(struct irq_domain *parent,
 	int ret, vec_base;
 	struct fwnode_handle *domain_handle;
 
+	if (find_pch_pic(acpi_pchpic->gsi_base) >= 0)
+		return 0;
+
 	vec_base = acpi_pchpic->gsi_base - GSI_MIN_PCH_IRQ;
 
 	domain_handle = irq_domain_alloc_fwnode(&acpi_pchpic->address);
diff --git a/drivers/irqchip/irq-ls-scfg-msi.c b/drivers/irqchip/irq-ls-scfg-msi.c
index 527c90e0920e..f5ba3f9f8415 100644
--- a/drivers/irqchip/irq-ls-scfg-msi.c
+++ b/drivers/irqchip/irq-ls-scfg-msi.c
@@ -430,4 +430,3 @@ module_platform_driver(ls_scfg_msi_driver);
 
 MODULE_AUTHOR("Minghuan Lian <Minghuan.Lian@nxp.com>");
 MODULE_DESCRIPTION("Freescale Layerscape SCFG MSI controller driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/irqchip/irq-mbigen.c b/drivers/irqchip/irq-mbigen.c
index f3faf5c99770..eada5e0e3eb9 100644
--- a/drivers/irqchip/irq-mbigen.c
+++ b/drivers/irqchip/irq-mbigen.c
@@ -245,11 +245,14 @@ static int mbigen_of_create_domain(struct platform_device *pdev,
 		if (!of_property_read_bool(np, "interrupt-controller"))
 			continue;
 
-		parent = platform_bus_type.dev_root;
-		child = of_platform_device_create(np, NULL, parent);
-		if (!child) {
-			of_node_put(np);
-			return -ENOMEM;
+		parent = bus_get_dev_root(&platform_bus_type);
+		if (parent) {
+			child = of_platform_device_create(np, NULL, parent);
+			put_device(parent);
+			if (!child) {
+				of_node_put(np);
+				return -ENOMEM;
+			}
 		}
 
 		if (of_property_read_u32(child->dev.of_node, "num-pins",
@@ -389,5 +392,4 @@ module_platform_driver(mbigen_platform_driver);
 
 MODULE_AUTHOR("Jun Ma <majun258@huawei.com>");
 MODULE_AUTHOR("Yun Wu <wuyun.wu@huawei.com>");
-MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("HiSilicon MBI Generator driver");
diff --git a/drivers/irqchip/irq-mchp-eic.c b/drivers/irqchip/irq-mchp-eic.c
index c726a19837d2..5dcd94c000a2 100644
--- a/drivers/irqchip/irq-mchp-eic.c
+++ b/drivers/irqchip/irq-mchp-eic.c
@@ -276,5 +276,4 @@ IRQCHIP_MATCH("microchip,sama7g5-eic", mchp_eic_init)
 IRQCHIP_PLATFORM_DRIVER_END(mchp_eic)
 
 MODULE_DESCRIPTION("Microchip External Interrupt Controller");
-MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Claudiu Beznea <claudiu.beznea@microchip.com>");
diff --git a/drivers/irqchip/irq-renesas-intc-irqpin.c b/drivers/irqchip/irq-renesas-intc-irqpin.c
index e83756aca14e..26e4c17a7bf2 100644
--- a/drivers/irqchip/irq-renesas-intc-irqpin.c
+++ b/drivers/irqchip/irq-renesas-intc-irqpin.c
@@ -608,4 +608,3 @@ module_exit(intc_irqpin_exit);
 
 MODULE_AUTHOR("Magnus Damm");
 MODULE_DESCRIPTION("Renesas INTC External IRQ Pin Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/irqchip/irq-renesas-irqc.c b/drivers/irqchip/irq-renesas-irqc.c
index 1ee5e9941f67..49b446b396f9 100644
--- a/drivers/irqchip/irq-renesas-irqc.c
+++ b/drivers/irqchip/irq-renesas-irqc.c
@@ -270,4 +270,3 @@ module_exit(irqc_exit);
 
 MODULE_AUTHOR("Magnus Damm");
 MODULE_DESCRIPTION("Renesas IRQC Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/irqchip/irq-renesas-rza1.c b/drivers/irqchip/irq-renesas-rza1.c
index 72c06e883d1c..e4c99c2e0373 100644
--- a/drivers/irqchip/irq-renesas-rza1.c
+++ b/drivers/irqchip/irq-renesas-rza1.c
@@ -281,4 +281,3 @@ module_exit(rza1_irqc_exit);
 
 MODULE_AUTHOR("Geert Uytterhoeven <geert+renesas@glider.be>");
 MODULE_DESCRIPTION("Renesas RZ/A1 IRQC Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/irqchip/irq-renesas-rzg2l.c b/drivers/irqchip/irq-renesas-rzg2l.c
index 25fd8ee66565..4bbfa2b0a4df 100644
--- a/drivers/irqchip/irq-renesas-rzg2l.c
+++ b/drivers/irqchip/irq-renesas-rzg2l.c
@@ -390,4 +390,3 @@ IRQCHIP_MATCH("renesas,rzg2l-irqc", rzg2l_irqc_init)
 IRQCHIP_PLATFORM_DRIVER_END(rzg2l_irqc)
 MODULE_AUTHOR("Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>");
 MODULE_DESCRIPTION("Renesas RZ/G2L IRQC Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/irqchip/irq-riscv-intc.c b/drivers/irqchip/irq-riscv-intc.c
index 499e5f81b3fe..f229e3e66387 100644
--- a/drivers/irqchip/irq-riscv-intc.c
+++ b/drivers/irqchip/irq-riscv-intc.c
@@ -26,20 +26,7 @@ static asmlinkage void riscv_intc_irq(struct pt_regs *regs)
 	if (unlikely(cause >= BITS_PER_LONG))
 		panic("unexpected interrupt cause");
 
-	switch (cause) {
-#ifdef CONFIG_SMP
-	case RV_IRQ_SOFT:
-		/*
-		 * We only use software interrupts to pass IPIs, so if a
-		 * non-SMP system gets one, then we don't know what to do.
-		 */
-		handle_IPI(regs);
-		break;
-#endif
-	default:
-		generic_handle_domain_irq(intc_domain, cause);
-		break;
-	}
+	generic_handle_domain_irq(intc_domain, cause);
 }
 
 /*
@@ -59,22 +46,27 @@ static void riscv_intc_irq_unmask(struct irq_data *d)
 	csr_set(CSR_IE, BIT(d->hwirq));
 }
 
-static int riscv_intc_cpu_starting(unsigned int cpu)
-{
-	csr_set(CSR_IE, BIT(RV_IRQ_SOFT));
-	return 0;
-}
-
-static int riscv_intc_cpu_dying(unsigned int cpu)
+static void riscv_intc_irq_eoi(struct irq_data *d)
 {
-	csr_clear(CSR_IE, BIT(RV_IRQ_SOFT));
-	return 0;
+	/*
+	 * The RISC-V INTC driver uses handle_percpu_devid_irq() flow
+	 * for the per-HART local interrupts and child irqchip drivers
+	 * (such as PLIC, SBI IPI, CLINT, APLIC, IMSIC, etc) implement
+	 * chained handlers for the per-HART local interrupts.
+	 *
+	 * In the absence of irq_eoi(), the chained_irq_enter() and
+	 * chained_irq_exit() functions (used by child irqchip drivers)
+	 * will do unnecessary mask/unmask of per-HART local interrupts
+	 * at the time of handling interrupts. To avoid this, we provide
+	 * an empty irq_eoi() callback for RISC-V INTC irqchip.
+	 */
 }
 
 static struct irq_chip riscv_intc_chip = {
 	.name = "RISC-V INTC",
 	.irq_mask = riscv_intc_irq_mask,
 	.irq_unmask = riscv_intc_irq_unmask,
+	.irq_eoi = riscv_intc_irq_eoi,
 };
 
 static int riscv_intc_domain_map(struct irq_domain *d, unsigned int irq,
@@ -87,11 +79,39 @@ static int riscv_intc_domain_map(struct irq_domain *d, unsigned int irq,
 	return 0;
 }
 
+static int riscv_intc_domain_alloc(struct irq_domain *domain,
+				   unsigned int virq, unsigned int nr_irqs,
+				   void *arg)
+{
+	int i, ret;
+	irq_hw_number_t hwirq;
+	unsigned int type = IRQ_TYPE_NONE;
+	struct irq_fwspec *fwspec = arg;
+
+	ret = irq_domain_translate_onecell(domain, fwspec, &hwirq, &type);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < nr_irqs; i++) {
+		ret = riscv_intc_domain_map(domain, virq + i, hwirq + i);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static const struct irq_domain_ops riscv_intc_domain_ops = {
 	.map	= riscv_intc_domain_map,
 	.xlate	= irq_domain_xlate_onecell,
+	.alloc	= riscv_intc_domain_alloc
 };
 
+static struct fwnode_handle *riscv_intc_hwnode(void)
+{
+	return intc_domain->fwnode;
+}
+
 static int __init riscv_intc_init(struct device_node *node,
 				  struct device_node *parent)
 {
@@ -126,10 +146,7 @@ static int __init riscv_intc_init(struct device_node *node,
 		return rc;
 	}
 
-	cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_STARTING,
-			  "irqchip/riscv/intc:starting",
-			  riscv_intc_cpu_starting,
-			  riscv_intc_cpu_dying);
+	riscv_set_intc_hwnode_fn(riscv_intc_hwnode);
 
 	pr_info("%d local interrupts mapped\n", BITS_PER_LONG);
 
diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c
index ff47bd0dec45..e1484905b7bd 100644
--- a/drivers/irqchip/irq-sifive-plic.c
+++ b/drivers/irqchip/irq-sifive-plic.c
@@ -17,6 +17,7 @@
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
+#include <linux/syscore_ops.h>
 #include <asm/smp.h>
 
 /*
@@ -67,6 +68,8 @@ struct plic_priv {
 	struct irq_domain *irqdomain;
 	void __iomem *regs;
 	unsigned long plic_quirks;
+	unsigned int nr_irqs;
+	unsigned long *prio_save;
 };
 
 struct plic_handler {
@@ -78,6 +81,7 @@ struct plic_handler {
 	 */
 	raw_spinlock_t		enable_lock;
 	void __iomem		*enable_base;
+	u32			*enable_save;
 	struct plic_priv	*priv;
 };
 static int plic_parent_irq __ro_after_init;
@@ -229,6 +233,71 @@ static int plic_irq_set_type(struct irq_data *d, unsigned int type)
 	return IRQ_SET_MASK_OK;
 }
 
+static int plic_irq_suspend(void)
+{
+	unsigned int i, cpu;
+	u32 __iomem *reg;
+	struct plic_priv *priv;
+
+	priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv;
+
+	for (i = 0; i < priv->nr_irqs; i++)
+		if (readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID))
+			__set_bit(i, priv->prio_save);
+		else
+			__clear_bit(i, priv->prio_save);
+
+	for_each_cpu(cpu, cpu_present_mask) {
+		struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
+
+		if (!handler->present)
+			continue;
+
+		raw_spin_lock(&handler->enable_lock);
+		for (i = 0; i < DIV_ROUND_UP(priv->nr_irqs, 32); i++) {
+			reg = handler->enable_base + i * sizeof(u32);
+			handler->enable_save[i] = readl(reg);
+		}
+		raw_spin_unlock(&handler->enable_lock);
+	}
+
+	return 0;
+}
+
+static void plic_irq_resume(void)
+{
+	unsigned int i, index, cpu;
+	u32 __iomem *reg;
+	struct plic_priv *priv;
+
+	priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv;
+
+	for (i = 0; i < priv->nr_irqs; i++) {
+		index = BIT_WORD(i);
+		writel((priv->prio_save[index] & BIT_MASK(i)) ? 1 : 0,
+		       priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID);
+	}
+
+	for_each_cpu(cpu, cpu_present_mask) {
+		struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
+
+		if (!handler->present)
+			continue;
+
+		raw_spin_lock(&handler->enable_lock);
+		for (i = 0; i < DIV_ROUND_UP(priv->nr_irqs, 32); i++) {
+			reg = handler->enable_base + i * sizeof(u32);
+			writel(handler->enable_save[i], reg);
+		}
+		raw_spin_unlock(&handler->enable_lock);
+	}
+}
+
+static struct syscore_ops plic_irq_syscore_ops = {
+	.suspend	= plic_irq_suspend,
+	.resume		= plic_irq_resume,
+};
+
 static int plic_irqdomain_map(struct irq_domain *d, unsigned int irq,
 			      irq_hw_number_t hwirq)
 {
@@ -345,6 +414,7 @@ static int __init __plic_init(struct device_node *node,
 	u32 nr_irqs;
 	struct plic_priv *priv;
 	struct plic_handler *handler;
+	unsigned int cpu;
 
 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -363,15 +433,21 @@ static int __init __plic_init(struct device_node *node,
 	if (WARN_ON(!nr_irqs))
 		goto out_iounmap;
 
+	priv->nr_irqs = nr_irqs;
+
+	priv->prio_save = bitmap_alloc(nr_irqs, GFP_KERNEL);
+	if (!priv->prio_save)
+		goto out_free_priority_reg;
+
 	nr_contexts = of_irq_count(node);
 	if (WARN_ON(!nr_contexts))
-		goto out_iounmap;
+		goto out_free_priority_reg;
 
 	error = -ENOMEM;
 	priv->irqdomain = irq_domain_add_linear(node, nr_irqs + 1,
 			&plic_irqdomain_ops, priv);
 	if (WARN_ON(!priv->irqdomain))
-		goto out_iounmap;
+		goto out_free_priority_reg;
 
 	for (i = 0; i < nr_contexts; i++) {
 		struct of_phandle_args parent;
@@ -441,6 +517,11 @@ static int __init __plic_init(struct device_node *node,
 		handler->enable_base = priv->regs + CONTEXT_ENABLE_BASE +
 			i * CONTEXT_ENABLE_SIZE;
 		handler->priv = priv;
+
+		handler->enable_save =  kcalloc(DIV_ROUND_UP(nr_irqs, 32),
+						sizeof(*handler->enable_save), GFP_KERNEL);
+		if (!handler->enable_save)
+			goto out_free_enable_reg;
 done:
 		for (hwirq = 1; hwirq <= nr_irqs; hwirq++) {
 			plic_toggle(handler, hwirq, 0);
@@ -461,11 +542,19 @@ done:
 				  plic_starting_cpu, plic_dying_cpu);
 		plic_cpuhp_setup_done = true;
 	}
+	register_syscore_ops(&plic_irq_syscore_ops);
 
 	pr_info("%pOFP: mapped %d interrupts with %d handlers for"
 		" %d contexts.\n", node, nr_irqs, nr_handlers, nr_contexts);
 	return 0;
 
+out_free_enable_reg:
+	for_each_cpu(cpu, cpu_present_mask) {
+		handler = per_cpu_ptr(&plic_handlers, cpu);
+		kfree(handler->enable_save);
+	}
+out_free_priority_reg:
+	kfree(priv->prio_save);
 out_iounmap:
 	iounmap(priv->regs);
 out_free_priv:
diff --git a/drivers/irqchip/irq-sl28cpld.c b/drivers/irqchip/irq-sl28cpld.c
index f2172240172c..e50f9eaba4cd 100644
--- a/drivers/irqchip/irq-sl28cpld.c
+++ b/drivers/irqchip/irq-sl28cpld.c
@@ -92,4 +92,3 @@ module_platform_driver(sl28cpld_intc_driver);
 
 MODULE_DESCRIPTION("sl28cpld Interrupt Controller Driver");
 MODULE_AUTHOR("Michael Walle <michael@walle.cc>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/irqchip/irq-st.c b/drivers/irqchip/irq-st.c
index 1b83512b29c6..819a12297b58 100644
--- a/drivers/irqchip/irq-st.c
+++ b/drivers/irqchip/irq-st.c
@@ -15,10 +15,7 @@
 #include <linux/regmap.h>
 #include <linux/slab.h>
 
-#define STIH415_SYSCFG_642		0x0a8
-#define STIH416_SYSCFG_7543		0x87c
 #define STIH407_SYSCFG_5102		0x198
-#define STID127_SYSCFG_734		0x088
 
 #define ST_A9_IRQ_MASK			0x001FFFFF
 #define ST_A9_IRQ_MAX_CHANS		2
@@ -45,21 +42,9 @@ struct st_irq_syscfg {
 
 static const struct of_device_id st_irq_syscfg_match[] = {
 	{
-		.compatible = "st,stih415-irq-syscfg",
-		.data = (void *)STIH415_SYSCFG_642,
-	},
-	{
-		.compatible = "st,stih416-irq-syscfg",
-		.data = (void *)STIH416_SYSCFG_7543,
-	},
-	{
 		.compatible = "st,stih407-irq-syscfg",
 		.data = (void *)STIH407_SYSCFG_5102,
 	},
-	{
-		.compatible = "st,stid127-irq-syscfg",
-		.data = (void *)STID127_SYSCFG_734,
-	},
 	{}
 };
 
diff --git a/drivers/irqchip/irq-ti-sci-inta.c b/drivers/irqchip/irq-ti-sci-inta.c
index a6ecc53d055c..7133f9fa6fd9 100644
--- a/drivers/irqchip/irq-ti-sci-inta.c
+++ b/drivers/irqchip/irq-ti-sci-inta.c
@@ -743,4 +743,3 @@ module_platform_driver(ti_sci_inta_irq_domain_driver);
 
 MODULE_AUTHOR("Lokesh Vutla <lokeshvutla@ti.com>");
 MODULE_DESCRIPTION("K3 Interrupt Aggregator driver over TI SCI protocol");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/irqchip/irq-ti-sci-intr.c b/drivers/irqchip/irq-ti-sci-intr.c
index 020ddf29efb8..1186f1e431a3 100644
--- a/drivers/irqchip/irq-ti-sci-intr.c
+++ b/drivers/irqchip/irq-ti-sci-intr.c
@@ -303,4 +303,3 @@ module_platform_driver(ti_sci_intr_irq_domain_driver);
 
 MODULE_AUTHOR("Lokesh Vutla <lokeshvutla@ticom>");
 MODULE_DESCRIPTION("K3 Interrupt Router driver over TI SCI protocol");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/isdn/capi/capi.c b/drivers/isdn/capi/capi.c
index 0f00be62438d..45a4043c5042 100644
--- a/drivers/isdn/capi/capi.c
+++ b/drivers/isdn/capi/capi.c
@@ -1393,7 +1393,7 @@ static int __init capi_init(void)
 		kcapi_exit();
 		return major_ret;
 	}
-	capi_class = class_create(THIS_MODULE, "capi");
+	capi_class = class_create("capi");
 	if (IS_ERR(capi_class)) {
 		unregister_chrdev(capi_major, "capi20");
 		kcapi_exit();
diff --git a/drivers/isdn/hardware/mISDN/hfcmulti.c b/drivers/isdn/hardware/mISDN/hfcmulti.c
index e840609c50eb..2e5cb9dde3ec 100644
--- a/drivers/isdn/hardware/mISDN/hfcmulti.c
+++ b/drivers/isdn/hardware/mISDN/hfcmulti.c
@@ -639,23 +639,6 @@ cpld_write_reg(struct hfc_multi *hc, unsigned char reg, unsigned char val)
 	return;
 }
 
-static inline unsigned char
-cpld_read_reg(struct hfc_multi *hc, unsigned char reg)
-{
-	unsigned char bytein;
-
-	cpld_set_reg(hc, reg);
-
-	/* Do data pin read low byte */
-	HFC_outb(hc, R_GPIO_OUT1, reg);
-
-	enablepcibridge(hc);
-	bytein = readpcibridge(hc, 1);
-	disablepcibridge(hc);
-
-	return bytein;
-}
-
 static inline void
 vpm_write_address(struct hfc_multi *hc, unsigned short addr)
 {
@@ -663,20 +646,6 @@ vpm_write_address(struct hfc_multi *hc, unsigned short addr)
 	cpld_write_reg(hc, 1, 0x01 & (addr >> 8));
 }
 
-static inline unsigned short
-vpm_read_address(struct hfc_multi *c)
-{
-	unsigned short addr;
-	unsigned short highbit;
-
-	addr = cpld_read_reg(c, 0);
-	highbit = cpld_read_reg(c, 1);
-
-	addr = addr | (highbit << 8);
-
-	return addr & 0x1ff;
-}
-
 static inline unsigned char
 vpm_in(struct hfc_multi *c, int which, unsigned short addr)
 {
diff --git a/drivers/isdn/hardware/mISDN/netjet.c b/drivers/isdn/hardware/mISDN/netjet.c
index f8447135a902..566c790a9481 100644
--- a/drivers/isdn/hardware/mISDN/netjet.c
+++ b/drivers/isdn/hardware/mISDN/netjet.c
@@ -970,7 +970,6 @@ nj_release(struct tiger_hw *card)
 	write_lock_irqsave(&card_lock, flags);
 	list_del(&card->list);
 	write_unlock_irqrestore(&card_lock, flags);
-	pci_clear_master(card->pdev);
 	pci_disable_device(card->pdev);
 	pci_set_drvdata(card->pdev, NULL);
 	kfree(card);
diff --git a/drivers/isdn/mISDN/core.c b/drivers/isdn/mISDN/core.c
index 9120be590325..ab8513a7acd5 100644
--- a/drivers/isdn/mISDN/core.c
+++ b/drivers/isdn/mISDN/core.c
@@ -152,18 +152,11 @@ static int mISDN_uevent(const struct device *dev, struct kobj_uevent_env *env)
 	return 0;
 }
 
-static void mISDN_class_release(struct class *cls)
-{
-	/* do nothing, it's static */
-}
-
 static struct class mISDN_class = {
 	.name = "mISDN",
-	.owner = THIS_MODULE,
 	.dev_uevent = mISDN_uevent,
 	.dev_groups = mISDN_groups,
 	.dev_release = mISDN_dev_release,
-	.class_release = mISDN_class_release,
 };
 
 static int
diff --git a/drivers/isdn/mISDN/dsp_pipeline.c b/drivers/isdn/mISDN/dsp_pipeline.c
index cfbcd9e973c2..09b72f14d4b7 100644
--- a/drivers/isdn/mISDN/dsp_pipeline.c
+++ b/drivers/isdn/mISDN/dsp_pipeline.c
@@ -131,7 +131,7 @@ EXPORT_SYMBOL(mISDN_dsp_element_unregister);
 
 int dsp_pipeline_module_init(void)
 {
-	elements_class = class_create(THIS_MODULE, "dsp_pipeline");
+	elements_class = class_create("dsp_pipeline");
 	if (IS_ERR(elements_class))
 		return PTR_ERR(elements_class);
 
diff --git a/drivers/leds/led-class.c b/drivers/leds/led-class.c
index a6b3adcd044a..9255bc11f99d 100644
--- a/drivers/leds/led-class.c
+++ b/drivers/leds/led-class.c
@@ -626,7 +626,7 @@ EXPORT_SYMBOL_GPL(devm_led_classdev_unregister);
 
 static int __init leds_init(void)
 {
-	leds_class = class_create(THIS_MODULE, "leds");
+	leds_class = class_create("leds");
 	if (IS_ERR(leds_class))
 		return PTR_ERR(leds_class);
 	leds_class->pm = &leds_class_dev_pm_ops;
diff --git a/drivers/macintosh/adb.c b/drivers/macintosh/adb.c
index 23bd0c77ac1a..057b0221f695 100644
--- a/drivers/macintosh/adb.c
+++ b/drivers/macintosh/adb.c
@@ -80,7 +80,7 @@ static struct adb_driver *adb_controller;
 BLOCKING_NOTIFIER_HEAD(adb_client_list);
 static int adb_got_sleep;
 static int adb_inited;
-static DEFINE_SEMAPHORE(adb_probe_mutex);
+static DEFINE_SEMAPHORE(adb_probe_mutex, 1);
 static int sleepy_trackpad;
 static int autopoll_devs;
 int __adb_probe_sync;
@@ -888,7 +888,7 @@ adbdev_init(void)
 		return;
 	}
 
-	adb_dev_class = class_create(THIS_MODULE, "adb");
+	adb_dev_class = class_create("adb");
 	if (IS_ERR(adb_dev_class))
 		return;
 	device_create(adb_dev_class, NULL, MKDEV(ADB_MAJOR, 0), NULL, "adb");
diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig
index 1495965bc394..af6b0f5b491d 100644
--- a/drivers/mailbox/Kconfig
+++ b/drivers/mailbox/Kconfig
@@ -259,7 +259,7 @@ config MTK_CMDQ_MBOX
 	  during the vblank.
 
 config ZYNQMP_IPI_MBOX
-	bool "Xilinx ZynqMP IPI Mailbox"
+	tristate "Xilinx ZynqMP IPI Mailbox"
 	depends on ARCH_ZYNQMP && OF
 	help
 	  Say yes here to add support for Xilinx IPI mailbox driver.
diff --git a/drivers/mailbox/mailbox-mpfs.c b/drivers/mailbox/mailbox-mpfs.c
index 853901acaeec..162df49654fb 100644
--- a/drivers/mailbox/mailbox-mpfs.c
+++ b/drivers/mailbox/mailbox-mpfs.c
@@ -39,7 +39,7 @@
 #define SCB_CTRL_NOTIFY_MASK BIT(SCB_CTRL_NOTIFY)
 
 #define SCB_CTRL_POS (16)
-#define SCB_CTRL_MASK GENMASK_ULL(SCB_CTRL_POS + SCB_MASK_WIDTH, SCB_CTRL_POS)
+#define SCB_CTRL_MASK GENMASK(SCB_CTRL_POS + SCB_MASK_WIDTH - 1, SCB_CTRL_POS)
 
 /* SCBCTRL service status register */
 
@@ -79,6 +79,27 @@ static bool mpfs_mbox_busy(struct mpfs_mbox *mbox)
 	return status & SCB_STATUS_BUSY_MASK;
 }
 
+static bool mpfs_mbox_last_tx_done(struct mbox_chan *chan)
+{
+	struct mpfs_mbox *mbox = (struct mpfs_mbox *)chan->con_priv;
+	struct mpfs_mss_response *response = mbox->response;
+	u32 val;
+
+	if (mpfs_mbox_busy(mbox))
+		return false;
+
+	/*
+	 * The service status is stored in bits 31:16 of the SERVICES_SR
+	 * register & is only valid when the system controller is not busy.
+	 * Failed services are intended to generated interrupts, but in reality
+	 * this does not happen, so the status must be checked here.
+	 */
+	val = readl_relaxed(mbox->ctrl_base + SERVICES_SR_OFFSET);
+	response->resp_status = (val & SCB_STATUS_MASK) >> SCB_STATUS_POS;
+
+	return true;
+}
+
 static int mpfs_mbox_send_data(struct mbox_chan *chan, void *data)
 {
 	struct mpfs_mbox *mbox = (struct mpfs_mbox *)chan->con_priv;
@@ -118,6 +139,7 @@ static int mpfs_mbox_send_data(struct mbox_chan *chan, void *data)
 	}
 
 	opt_sel = ((msg->mbox_offset << 7u) | (msg->cmd_opcode & 0x7fu));
+
 	tx_trigger = (opt_sel << SCB_CTRL_POS) & SCB_CTRL_MASK;
 	tx_trigger |= SCB_CTRL_REQ_MASK | SCB_STATUS_NOTIFY_MASK;
 	writel_relaxed(tx_trigger, mbox->ctrl_base + SERVICES_CR_OFFSET);
@@ -130,7 +152,7 @@ static void mpfs_mbox_rx_data(struct mbox_chan *chan)
 	struct mpfs_mbox *mbox = (struct mpfs_mbox *)chan->con_priv;
 	struct mpfs_mss_response *response = mbox->response;
 	u16 num_words = ALIGN((response->resp_size), (4)) / 4U;
-	u32 i, status;
+	u32 i;
 
 	if (!response->resp_msg) {
 		dev_err(mbox->dev, "failed to assign memory for response %d\n", -ENOMEM);
@@ -138,8 +160,6 @@ static void mpfs_mbox_rx_data(struct mbox_chan *chan)
 	}
 
 	/*
-	 * The status is stored in bits 31:16 of the SERVICES_SR register.
-	 * It is only valid when BUSY == 0.
 	 * We should *never* get an interrupt while the controller is
 	 * still in the busy state. If we do, something has gone badly
 	 * wrong & the content of the mailbox would not be valid.
@@ -150,24 +170,10 @@ static void mpfs_mbox_rx_data(struct mbox_chan *chan)
 		return;
 	}
 
-	status = readl_relaxed(mbox->ctrl_base + SERVICES_SR_OFFSET);
-
-	/*
-	 * If the status of the individual servers is non-zero, the service has
-	 * failed. The contents of the mailbox at this point are not be valid,
-	 * so don't bother reading them. Set the status so that the driver
-	 * implementing the service can handle the result.
-	 */
-	response->resp_status = (status & SCB_STATUS_MASK) >> SCB_STATUS_POS;
-	if (response->resp_status)
-		return;
-
-	if (!mpfs_mbox_busy(mbox)) {
-		for (i = 0; i < num_words; i++) {
-			response->resp_msg[i] =
-				readl_relaxed(mbox->mbox_base
-					      + mbox->resp_offset + i * 0x4);
-		}
+	for (i = 0; i < num_words; i++) {
+		response->resp_msg[i] =
+			readl_relaxed(mbox->mbox_base
+				      + mbox->resp_offset + i * 0x4);
 	}
 
 	mbox_chan_received_data(chan, response);
@@ -182,7 +188,6 @@ static irqreturn_t mpfs_mbox_inbox_isr(int irq, void *data)
 
 	mpfs_mbox_rx_data(chan);
 
-	mbox_chan_txdone(chan, 0);
 	return IRQ_HANDLED;
 }
 
@@ -212,6 +217,7 @@ static const struct mbox_chan_ops mpfs_mbox_ops = {
 	.send_data = mpfs_mbox_send_data,
 	.startup = mpfs_mbox_startup,
 	.shutdown = mpfs_mbox_shutdown,
+	.last_tx_done = mpfs_mbox_last_tx_done,
 };
 
 static int mpfs_mbox_probe(struct platform_device *pdev)
@@ -247,7 +253,8 @@ static int mpfs_mbox_probe(struct platform_device *pdev)
 	mbox->controller.num_chans = 1;
 	mbox->controller.chans = mbox->chans;
 	mbox->controller.ops = &mpfs_mbox_ops;
-	mbox->controller.txdone_irq = true;
+	mbox->controller.txdone_poll = true;
+	mbox->controller.txpoll_period = 10u;
 
 	ret = devm_mbox_controller_register(&pdev->dev, &mbox->controller);
 	if (ret) {
diff --git a/drivers/mailbox/rockchip-mailbox.c b/drivers/mailbox/rockchip-mailbox.c
index e02d3c9e3693..28d7bfee2641 100644
--- a/drivers/mailbox/rockchip-mailbox.c
+++ b/drivers/mailbox/rockchip-mailbox.c
@@ -254,7 +254,6 @@ static struct platform_driver rockchip_mbox_driver = {
 
 module_platform_driver(rockchip_mbox_driver);
 
-MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("Rockchip mailbox: communicate between CPU cores and MCU");
 MODULE_AUTHOR("Addy Ke <addy.ke@rock-chips.com>");
 MODULE_AUTHOR("Caesar Wang <wxt@rock-chips.com>");
diff --git a/drivers/mailbox/zynqmp-ipi-mailbox.c b/drivers/mailbox/zynqmp-ipi-mailbox.c
index a4c8d23c76e2..d097f45b0e5f 100644
--- a/drivers/mailbox/zynqmp-ipi-mailbox.c
+++ b/drivers/mailbox/zynqmp-ipi-mailbox.c
@@ -152,7 +152,7 @@ static irqreturn_t zynqmp_ipi_interrupt(int irq, void *data)
 	struct zynqmp_ipi_message *msg;
 	u64 arg0, arg3;
 	struct arm_smccc_res res;
-	int ret, i;
+	int ret, i, status = IRQ_NONE;
 
 	(void)irq;
 	arg0 = SMC_IPI_MAILBOX_STATUS_ENQUIRY;
@@ -170,11 +170,11 @@ static irqreturn_t zynqmp_ipi_interrupt(int irq, void *data)
 				memcpy_fromio(msg->data, mchan->req_buf,
 					      msg->len);
 				mbox_chan_received_data(chan, (void *)msg);
-				return IRQ_HANDLED;
+				status = IRQ_HANDLED;
 			}
 		}
 	}
-	return IRQ_NONE;
+	return status;
 }
 
 /**
@@ -634,7 +634,12 @@ static int zynqmp_ipi_probe(struct platform_device *pdev)
 	struct zynqmp_ipi_mbox *mbox;
 	int num_mboxes, ret = -EINVAL;
 
-	num_mboxes = of_get_child_count(np);
+	num_mboxes = of_get_available_child_count(np);
+	if (num_mboxes == 0) {
+		dev_err(dev, "mailbox nodes not available\n");
+		return -EINVAL;
+	}
+
 	pdata = devm_kzalloc(dev, struct_size(pdata, ipi_mboxes, num_mboxes),
 			     GFP_KERNEL);
 	if (!pdata)
diff --git a/drivers/mcb/mcb-lpc.c b/drivers/mcb/mcb-lpc.c
index 53decd89876e..a851e0236464 100644
--- a/drivers/mcb/mcb-lpc.c
+++ b/drivers/mcb/mcb-lpc.c
@@ -23,7 +23,7 @@ static int mcb_lpc_probe(struct platform_device *pdev)
 {
 	struct resource *res;
 	struct priv *priv;
-	int ret = 0;
+	int ret = 0, table_size;
 
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -58,16 +58,43 @@ static int mcb_lpc_probe(struct platform_device *pdev)
 
 	ret = chameleon_parse_cells(priv->bus, priv->mem->start, priv->base);
 	if (ret < 0) {
-		mcb_release_bus(priv->bus);
-		return ret;
+		goto out_mcb_bus;
 	}
 
-	dev_dbg(&pdev->dev, "Found %d cells\n", ret);
+	table_size = ret;
+
+	if (table_size < CHAM_HEADER_SIZE) {
+		/* Release the previous resources */
+		devm_iounmap(&pdev->dev, priv->base);
+		devm_release_mem_region(&pdev->dev, priv->mem->start, resource_size(priv->mem));
+
+		/* Then, allocate it again with the actual chameleon table size */
+		res = devm_request_mem_region(&pdev->dev, priv->mem->start,
+					      table_size,
+					      KBUILD_MODNAME);
+		if (!res) {
+			dev_err(&pdev->dev, "Failed to request PCI memory\n");
+			ret = -EBUSY;
+			goto out_mcb_bus;
+		}
+
+		priv->base = devm_ioremap(&pdev->dev, priv->mem->start, table_size);
+		if (!priv->base) {
+			dev_err(&pdev->dev, "Cannot ioremap\n");
+			ret = -ENOMEM;
+			goto out_mcb_bus;
+		}
+
+		platform_set_drvdata(pdev, priv);
+	}
 
 	mcb_bus_add_devices(priv->bus);
 
 	return 0;
 
+out_mcb_bus:
+	mcb_release_bus(priv->bus);
+	return ret;
 }
 
 static int mcb_lpc_remove(struct platform_device *pdev)
diff --git a/drivers/mcb/mcb-parse.c b/drivers/mcb/mcb-parse.c
index aa6938da0db8..2aef990f379f 100644
--- a/drivers/mcb/mcb-parse.c
+++ b/drivers/mcb/mcb-parse.c
@@ -130,7 +130,7 @@ static void chameleon_parse_bar(void __iomem *base,
 	}
 }
 
-static int chameleon_get_bar(char __iomem **base, phys_addr_t mapbase,
+static int chameleon_get_bar(void __iomem **base, phys_addr_t mapbase,
 			     struct chameleon_bar **cb)
 {
 	struct chameleon_bar *c;
@@ -179,12 +179,13 @@ int chameleon_parse_cells(struct mcb_bus *bus, phys_addr_t mapbase,
 {
 	struct chameleon_fpga_header *header;
 	struct chameleon_bar *cb;
-	char __iomem *p = base;
+	void __iomem *p = base;
 	int num_cells = 0;
 	uint32_t dtype;
 	int bar_count;
 	int ret;
 	u32 hsize;
+	u32 table_size;
 
 	hsize = sizeof(struct chameleon_fpga_header);
 
@@ -239,12 +240,16 @@ int chameleon_parse_cells(struct mcb_bus *bus, phys_addr_t mapbase,
 		num_cells++;
 	}
 
-	if (num_cells == 0)
-		num_cells = -EINVAL;
+	if (num_cells == 0) {
+		ret = -EINVAL;
+		goto free_bar;
+	}
 
+	table_size = p - base;
+	pr_debug("%d cell(s) found. Chameleon table size: 0x%04x bytes\n", num_cells, table_size);
 	kfree(cb);
 	kfree(header);
-	return num_cells;
+	return table_size;
 
 free_bar:
 	kfree(cb);
diff --git a/drivers/mcb/mcb-pci.c b/drivers/mcb/mcb-pci.c
index dc88232d9af8..53d9202ff9a7 100644
--- a/drivers/mcb/mcb-pci.c
+++ b/drivers/mcb/mcb-pci.c
@@ -31,7 +31,7 @@ static int mcb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	struct resource *res;
 	struct priv *priv;
-	int ret;
+	int ret, table_size;
 	unsigned long flags;
 
 	priv = devm_kzalloc(&pdev->dev, sizeof(struct priv), GFP_KERNEL);
@@ -90,7 +90,30 @@ static int mcb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (ret < 0)
 		goto out_mcb_bus;
 
-	dev_dbg(&pdev->dev, "Found %d cells\n", ret);
+	table_size = ret;
+
+	if (table_size < CHAM_HEADER_SIZE) {
+		/* Release the previous resources */
+		devm_iounmap(&pdev->dev, priv->base);
+		devm_release_mem_region(&pdev->dev, priv->mapbase, CHAM_HEADER_SIZE);
+
+		/* Then, allocate it again with the actual chameleon table size */
+		res = devm_request_mem_region(&pdev->dev, priv->mapbase,
+						table_size,
+						KBUILD_MODNAME);
+		if (!res) {
+			dev_err(&pdev->dev, "Failed to request PCI memory\n");
+			ret = -EBUSY;
+			goto out_mcb_bus;
+		}
+
+		priv->base = devm_ioremap(&pdev->dev, priv->mapbase, table_size);
+		if (!priv->base) {
+			dev_err(&pdev->dev, "Cannot ioremap\n");
+			ret = -ENOMEM;
+			goto out_mcb_bus;
+		}
+	}
 
 	mcb_bus_add_devices(priv->bus);
 
diff --git a/drivers/md/dm-bio-prison-v1.c b/drivers/md/dm-bio-prison-v1.c
index c4c05d5d8909..92afdca760ae 100644
--- a/drivers/md/dm-bio-prison-v1.c
+++ b/drivers/md/dm-bio-prison-v1.c
@@ -18,10 +18,15 @@
 
 #define MIN_CELLS 1024
 
-struct dm_bio_prison {
+struct prison_region {
 	spinlock_t lock;
-	struct rb_root cells;
+	struct rb_root cell;
+} ____cacheline_aligned_in_smp;
+
+struct dm_bio_prison {
 	mempool_t cell_pool;
+	unsigned int num_locks;
+	struct prison_region regions[];
 };
 
 static struct kmem_cache *_cell_cache;
@@ -34,13 +39,20 @@ static struct kmem_cache *_cell_cache;
  */
 struct dm_bio_prison *dm_bio_prison_create(void)
 {
-	struct dm_bio_prison *prison = kzalloc(sizeof(*prison), GFP_KERNEL);
 	int ret;
+	unsigned int i, num_locks;
+	struct dm_bio_prison *prison;
 
+	num_locks = dm_num_hash_locks();
+	prison = kzalloc(struct_size(prison, regions, num_locks), GFP_KERNEL);
 	if (!prison)
 		return NULL;
+	prison->num_locks = num_locks;
 
-	spin_lock_init(&prison->lock);
+	for (i = 0; i < prison->num_locks; i++) {
+		spin_lock_init(&prison->regions[i].lock);
+		prison->regions[i].cell = RB_ROOT;
+	}
 
 	ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
 	if (ret) {
@@ -48,8 +60,6 @@ struct dm_bio_prison *dm_bio_prison_create(void)
 		return NULL;
 	}
 
-	prison->cells = RB_ROOT;
-
 	return prison;
 }
 EXPORT_SYMBOL_GPL(dm_bio_prison_create);
@@ -107,14 +117,32 @@ static int cmp_keys(struct dm_cell_key *lhs,
 	return 0;
 }
 
-static int __bio_detain(struct dm_bio_prison *prison,
+static inline unsigned int lock_nr(struct dm_cell_key *key, unsigned int num_locks)
+{
+	return dm_hash_locks_index((key->block_begin >> BIO_PRISON_MAX_RANGE_SHIFT),
+				   num_locks);
+}
+
+bool dm_cell_key_has_valid_range(struct dm_cell_key *key)
+{
+	if (WARN_ON_ONCE(key->block_end - key->block_begin > BIO_PRISON_MAX_RANGE))
+		return false;
+	if (WARN_ON_ONCE((key->block_begin >> BIO_PRISON_MAX_RANGE_SHIFT) !=
+			 (key->block_end - 1) >> BIO_PRISON_MAX_RANGE_SHIFT))
+		return false;
+
+	return true;
+}
+EXPORT_SYMBOL(dm_cell_key_has_valid_range);
+
+static int __bio_detain(struct rb_root *root,
 			struct dm_cell_key *key,
 			struct bio *inmate,
 			struct dm_bio_prison_cell *cell_prealloc,
 			struct dm_bio_prison_cell **cell_result)
 {
 	int r;
-	struct rb_node **new = &prison->cells.rb_node, *parent = NULL;
+	struct rb_node **new = &root->rb_node, *parent = NULL;
 
 	while (*new) {
 		struct dm_bio_prison_cell *cell =
@@ -139,7 +167,7 @@ static int __bio_detain(struct dm_bio_prison *prison,
 	*cell_result = cell_prealloc;
 
 	rb_link_node(&cell_prealloc->node, parent, new);
-	rb_insert_color(&cell_prealloc->node, &prison->cells);
+	rb_insert_color(&cell_prealloc->node, root);
 
 	return 0;
 }
@@ -151,10 +179,11 @@ static int bio_detain(struct dm_bio_prison *prison,
 		      struct dm_bio_prison_cell **cell_result)
 {
 	int r;
+	unsigned l = lock_nr(key, prison->num_locks);
 
-	spin_lock_irq(&prison->lock);
-	r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
-	spin_unlock_irq(&prison->lock);
+	spin_lock_irq(&prison->regions[l].lock);
+	r = __bio_detain(&prison->regions[l].cell, key, inmate, cell_prealloc, cell_result);
+	spin_unlock_irq(&prison->regions[l].lock);
 
 	return r;
 }
@@ -181,11 +210,11 @@ EXPORT_SYMBOL_GPL(dm_get_cell);
 /*
  * @inmates must have been initialised prior to this call
  */
-static void __cell_release(struct dm_bio_prison *prison,
+static void __cell_release(struct rb_root *root,
 			   struct dm_bio_prison_cell *cell,
 			   struct bio_list *inmates)
 {
-	rb_erase(&cell->node, &prison->cells);
+	rb_erase(&cell->node, root);
 
 	if (inmates) {
 		if (cell->holder)
@@ -198,20 +227,22 @@ void dm_cell_release(struct dm_bio_prison *prison,
 		     struct dm_bio_prison_cell *cell,
 		     struct bio_list *bios)
 {
-	spin_lock_irq(&prison->lock);
-	__cell_release(prison, cell, bios);
-	spin_unlock_irq(&prison->lock);
+	unsigned l = lock_nr(&cell->key, prison->num_locks);
+
+	spin_lock_irq(&prison->regions[l].lock);
+	__cell_release(&prison->regions[l].cell, cell, bios);
+	spin_unlock_irq(&prison->regions[l].lock);
 }
 EXPORT_SYMBOL_GPL(dm_cell_release);
 
 /*
  * Sometimes we don't want the holder, just the additional bios.
  */
-static void __cell_release_no_holder(struct dm_bio_prison *prison,
+static void __cell_release_no_holder(struct rb_root *root,
 				     struct dm_bio_prison_cell *cell,
 				     struct bio_list *inmates)
 {
-	rb_erase(&cell->node, &prison->cells);
+	rb_erase(&cell->node, root);
 	bio_list_merge(inmates, &cell->bios);
 }
 
@@ -219,11 +250,12 @@ void dm_cell_release_no_holder(struct dm_bio_prison *prison,
 			       struct dm_bio_prison_cell *cell,
 			       struct bio_list *inmates)
 {
+	unsigned l = lock_nr(&cell->key, prison->num_locks);
 	unsigned long flags;
 
-	spin_lock_irqsave(&prison->lock, flags);
-	__cell_release_no_holder(prison, cell, inmates);
-	spin_unlock_irqrestore(&prison->lock, flags);
+	spin_lock_irqsave(&prison->regions[l].lock, flags);
+	__cell_release_no_holder(&prison->regions[l].cell, cell, inmates);
+	spin_unlock_irqrestore(&prison->regions[l].lock, flags);
 }
 EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);
 
@@ -248,18 +280,19 @@ void dm_cell_visit_release(struct dm_bio_prison *prison,
 			   void *context,
 			   struct dm_bio_prison_cell *cell)
 {
-	spin_lock_irq(&prison->lock);
+	unsigned l = lock_nr(&cell->key, prison->num_locks);
+	spin_lock_irq(&prison->regions[l].lock);
 	visit_fn(context, cell);
-	rb_erase(&cell->node, &prison->cells);
-	spin_unlock_irq(&prison->lock);
+	rb_erase(&cell->node, &prison->regions[l].cell);
+	spin_unlock_irq(&prison->regions[l].lock);
 }
 EXPORT_SYMBOL_GPL(dm_cell_visit_release);
 
-static int __promote_or_release(struct dm_bio_prison *prison,
+static int __promote_or_release(struct rb_root *root,
 				struct dm_bio_prison_cell *cell)
 {
 	if (bio_list_empty(&cell->bios)) {
-		rb_erase(&cell->node, &prison->cells);
+		rb_erase(&cell->node, root);
 		return 1;
 	}
 
@@ -271,10 +304,11 @@ int dm_cell_promote_or_release(struct dm_bio_prison *prison,
 			       struct dm_bio_prison_cell *cell)
 {
 	int r;
+	unsigned l = lock_nr(&cell->key, prison->num_locks);
 
-	spin_lock_irq(&prison->lock);
-	r = __promote_or_release(prison, cell);
-	spin_unlock_irq(&prison->lock);
+	spin_lock_irq(&prison->regions[l].lock);
+	r = __promote_or_release(&prison->regions[l].cell, cell);
+	spin_unlock_irq(&prison->regions[l].lock);
 
 	return r;
 }
diff --git a/drivers/md/dm-bio-prison-v1.h b/drivers/md/dm-bio-prison-v1.h
index dfbf1e94cb75..2a097ed0d85e 100644
--- a/drivers/md/dm-bio-prison-v1.h
+++ b/drivers/md/dm-bio-prison-v1.h
@@ -35,6 +35,16 @@ struct dm_cell_key {
 };
 
 /*
+ * The range of a key (block_end - block_begin) must not
+ * exceed BIO_PRISON_MAX_RANGE.  Also the range must not
+ * cross a similarly sized boundary.
+ *
+ * Must be a power of 2.
+ */
+#define BIO_PRISON_MAX_RANGE 1024
+#define BIO_PRISON_MAX_RANGE_SHIFT 10
+
+/*
  * Treat this as opaque, only in header so callers can manage allocation
  * themselves.
  */
@@ -74,6 +84,11 @@ int dm_get_cell(struct dm_bio_prison *prison,
 		struct dm_bio_prison_cell **cell_result);
 
 /*
+ * Returns false if key is beyond BIO_PRISON_MAX_RANGE or spans a boundary.
+ */
+bool dm_cell_key_has_valid_range(struct dm_cell_key *key);
+
+/*
  * An atomic op that combines retrieving or creating a cell, and adding a
  * bio to it.
  *
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index cf077f9b30c3..8a448185662c 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -21,6 +21,8 @@
 #include <linux/stacktrace.h>
 #include <linux/jump_label.h>
 
+#include "dm.h"
+
 #define DM_MSG_PREFIX "bufio"
 
 /*
@@ -66,57 +68,241 @@
 #define LIST_DIRTY	1
 #define LIST_SIZE	2
 
+/*--------------------------------------------------------------*/
+
 /*
- * Linking of buffers:
- *	All buffers are linked to buffer_tree with their node field.
- *
- *	Clean buffers that are not being written (B_WRITING not set)
- *	are linked to lru[LIST_CLEAN] with their lru_list field.
- *
- *	Dirty and clean buffers that are being written are linked to
- *	lru[LIST_DIRTY] with their lru_list field. When the write
- *	finishes, the buffer cannot be relinked immediately (because we
- *	are in an interrupt context and relinking requires process
- *	context), so some clean-not-writing buffers can be held on
- *	dirty_lru too.  They are later added to lru in the process
- *	context.
+ * Rather than use an LRU list, we use a clock algorithm where entries
+ * are held in a circular list.  When an entry is 'hit' a reference bit
+ * is set.  The least recently used entry is approximated by running a
+ * cursor around the list selecting unreferenced entries. Referenced
+ * entries have their reference bit cleared as the cursor passes them.
  */
-struct dm_bufio_client {
-	struct mutex lock;
-	spinlock_t spinlock;
-	bool no_sleep;
+struct lru_entry {
+	struct list_head list;
+	atomic_t referenced;
+};
 
-	struct list_head lru[LIST_SIZE];
-	unsigned long n_buffers[LIST_SIZE];
+struct lru_iter {
+	struct lru *lru;
+	struct list_head list;
+	struct lru_entry *stop;
+	struct lru_entry *e;
+};
 
-	struct block_device *bdev;
-	unsigned int block_size;
-	s8 sectors_per_block_bits;
-	void (*alloc_callback)(struct dm_buffer *buf);
-	void (*write_callback)(struct dm_buffer *buf);
-	struct kmem_cache *slab_buffer;
-	struct kmem_cache *slab_cache;
-	struct dm_io_client *dm_io;
+struct lru {
+	struct list_head *cursor;
+	unsigned long count;
 
-	struct list_head reserved_buffers;
-	unsigned int need_reserved_buffers;
+	struct list_head iterators;
+};
 
-	unsigned int minimum_buffers;
+/*--------------*/
 
-	struct rb_root buffer_tree;
-	wait_queue_head_t free_buffer_wait;
+static void lru_init(struct lru *lru)
+{
+	lru->cursor = NULL;
+	lru->count = 0;
+	INIT_LIST_HEAD(&lru->iterators);
+}
 
-	sector_t start;
+static void lru_destroy(struct lru *lru)
+{
+	WARN_ON_ONCE(lru->cursor);
+	WARN_ON_ONCE(!list_empty(&lru->iterators));
+}
 
-	int async_write_error;
+/*
+ * Insert a new entry into the lru.
+ */
+static void lru_insert(struct lru *lru, struct lru_entry *le)
+{
+	/*
+	 * Don't be tempted to set to 1, makes the lru aspect
+	 * perform poorly.
+	 */
+	atomic_set(&le->referenced, 0);
 
-	struct list_head client_list;
+	if (lru->cursor) {
+		list_add_tail(&le->list, lru->cursor);
+	} else {
+		INIT_LIST_HEAD(&le->list);
+		lru->cursor = &le->list;
+	}
+	lru->count++;
+}
 
-	struct shrinker shrinker;
-	struct work_struct shrink_work;
-	atomic_long_t need_shrink;
+/*--------------*/
+
+/*
+ * Convert a list_head pointer to an lru_entry pointer.
+ */
+static inline struct lru_entry *to_le(struct list_head *l)
+{
+	return container_of(l, struct lru_entry, list);
+}
+
+/*
+ * Initialize an lru_iter and add it to the list of cursors in the lru.
+ */
+static void lru_iter_begin(struct lru *lru, struct lru_iter *it)
+{
+	it->lru = lru;
+	it->stop = lru->cursor ? to_le(lru->cursor->prev) : NULL;
+	it->e = lru->cursor ? to_le(lru->cursor) : NULL;
+	list_add(&it->list, &lru->iterators);
+}
+
+/*
+ * Remove an lru_iter from the list of cursors in the lru.
+ */
+static inline void lru_iter_end(struct lru_iter *it)
+{
+	list_del(&it->list);
+}
+
+/* Predicate function type to be used with lru_iter_next */
+typedef bool (*iter_predicate)(struct lru_entry *le, void *context);
+
+/*
+ * Advance the cursor to the next entry that passes the
+ * predicate, and return that entry.  Returns NULL if the
+ * iteration is complete.
+ */
+static struct lru_entry *lru_iter_next(struct lru_iter *it,
+				       iter_predicate pred, void *context)
+{
+	struct lru_entry *e;
+
+	while (it->e) {
+		e = it->e;
+
+		/* advance the cursor */
+		if (it->e == it->stop)
+			it->e = NULL;
+		else
+			it->e = to_le(it->e->list.next);
+
+		if (pred(e, context))
+			return e;
+	}
+
+	return NULL;
+}
+
+/*
+ * Invalidate a specific lru_entry and update all cursors in
+ * the lru accordingly.
+ */
+static void lru_iter_invalidate(struct lru *lru, struct lru_entry *e)
+{
+	struct lru_iter *it;
+
+	list_for_each_entry(it, &lru->iterators, list) {
+		/* Move c->e forwards if necc. */
+		if (it->e == e) {
+			it->e = to_le(it->e->list.next);
+			if (it->e == e)
+				it->e = NULL;
+		}
+
+		/* Move it->stop backwards if necc. */
+		if (it->stop == e) {
+			it->stop = to_le(it->stop->list.prev);
+			if (it->stop == e)
+				it->stop = NULL;
+		}
+	}
+}
+
+/*--------------*/
+
+/*
+ * Remove a specific entry from the lru.
+ */
+static void lru_remove(struct lru *lru, struct lru_entry *le)
+{
+	lru_iter_invalidate(lru, le);
+	if (lru->count == 1) {
+		lru->cursor = NULL;
+	} else {
+		if (lru->cursor == &le->list)
+			lru->cursor = lru->cursor->next;
+		list_del(&le->list);
+	}
+	lru->count--;
+}
+
+/*
+ * Mark as referenced.
+ */
+static inline void lru_reference(struct lru_entry *le)
+{
+	atomic_set(&le->referenced, 1);
+}
+
+/*--------------*/
+
+/*
+ * Remove the least recently used entry (approx), that passes the predicate.
+ * Returns NULL on failure.
+ */
+enum evict_result {
+	ER_EVICT,
+	ER_DONT_EVICT,
+	ER_STOP, /* stop looking for something to evict */
 };
 
+typedef enum evict_result (*le_predicate)(struct lru_entry *le, void *context);
+
+static struct lru_entry *lru_evict(struct lru *lru, le_predicate pred, void *context)
+{
+	unsigned long tested = 0;
+	struct list_head *h = lru->cursor;
+	struct lru_entry *le;
+
+	if (!h)
+		return NULL;
+	/*
+	 * In the worst case we have to loop around twice. Once to clear
+	 * the reference flags, and then again to discover the predicate
+	 * fails for all entries.
+	 */
+	while (tested < lru->count) {
+		le = container_of(h, struct lru_entry, list);
+
+		if (atomic_read(&le->referenced)) {
+			atomic_set(&le->referenced, 0);
+		} else {
+			tested++;
+			switch (pred(le, context)) {
+			case ER_EVICT:
+				/*
+				 * Adjust the cursor, so we start the next
+				 * search from here.
+				 */
+				lru->cursor = le->list.next;
+				lru_remove(lru, le);
+				return le;
+
+			case ER_DONT_EVICT:
+				break;
+
+			case ER_STOP:
+				lru->cursor = le->list.next;
+				return NULL;
+			}
+		}
+
+		h = h->next;
+
+		cond_resched();
+	}
+
+	return NULL;
+}
+
+/*--------------------------------------------------------------*/
+
 /*
  * Buffer state bits.
  */
@@ -137,26 +323,37 @@ enum data_mode {
 };
 
 struct dm_buffer {
+	/* protected by the locks in dm_buffer_cache */
 	struct rb_node node;
-	struct list_head lru_list;
-	struct list_head global_list;
+
+	/* immutable, so don't need protecting */
 	sector_t block;
 	void *data;
 	unsigned char data_mode;		/* DATA_MODE_* */
+
+	/*
+	 * These two fields are used in isolation, so do not need
+	 * a surrounding lock.
+	 */
+	atomic_t hold_count;
+	unsigned long last_accessed;
+
+	/*
+	 * Everything else is protected by the mutex in
+	 * dm_bufio_client
+	 */
+	unsigned long state;
+	struct lru_entry lru;
 	unsigned char list_mode;		/* LIST_* */
 	blk_status_t read_error;
 	blk_status_t write_error;
-	unsigned int accessed;
-	unsigned int hold_count;
-	unsigned long state;
-	unsigned long last_accessed;
 	unsigned int dirty_start;
 	unsigned int dirty_end;
 	unsigned int write_start;
 	unsigned int write_end;
-	struct dm_bufio_client *c;
 	struct list_head write_list;
-	void (*end_io)(struct dm_buffer *buf, blk_status_t stat);
+	struct dm_bufio_client *c;
+	void (*end_io)(struct dm_buffer *b, blk_status_t bs);
 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
 #define MAX_STACK 10
 	unsigned int stack_len;
@@ -164,126 +361,507 @@ struct dm_buffer {
 #endif
 };
 
-static DEFINE_STATIC_KEY_FALSE(no_sleep_enabled);
+/*--------------------------------------------------------------*/
 
-/*----------------------------------------------------------------*/
+/*
+ * The buffer cache manages buffers, particularly:
+ *  - inc/dec of holder count
+ *  - setting the last_accessed field
+ *  - maintains clean/dirty state along with lru
+ *  - selecting buffers that match predicates
+ *
+ * It does *not* handle:
+ *  - allocation/freeing of buffers.
+ *  - IO
+ *  - Eviction or cache sizing.
+ *
+ * cache_get() and cache_put() are threadsafe, you do not need to
+ * protect these calls with a surrounding mutex.  All the other
+ * methods are not threadsafe; they do use locking primitives, but
+ * only enough to ensure get/put are threadsafe.
+ */
 
-#define dm_bufio_in_request()	(!!current->bio_list)
+struct buffer_tree {
+	struct rw_semaphore lock;
+	struct rb_root root;
+} ____cacheline_aligned_in_smp;
 
-static void dm_bufio_lock(struct dm_bufio_client *c)
+struct dm_buffer_cache {
+	struct lru lru[LIST_SIZE];
+	/*
+	 * We spread entries across multiple trees to reduce contention
+	 * on the locks.
+	 */
+	unsigned int num_locks;
+	struct buffer_tree trees[];
+};
+
+static inline unsigned int cache_index(sector_t block, unsigned int num_locks)
 {
-	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
-		spin_lock_bh(&c->spinlock);
-	else
-		mutex_lock_nested(&c->lock, dm_bufio_in_request());
+	return dm_hash_locks_index(block, num_locks);
 }
 
-static int dm_bufio_trylock(struct dm_bufio_client *c)
+static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block)
 {
-	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
-		return spin_trylock_bh(&c->spinlock);
+	down_read(&bc->trees[cache_index(block, bc->num_locks)].lock);
+}
+
+static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t block)
+{
+	up_read(&bc->trees[cache_index(block, bc->num_locks)].lock);
+}
+
+static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t block)
+{
+	down_write(&bc->trees[cache_index(block, bc->num_locks)].lock);
+}
+
+static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t block)
+{
+	up_write(&bc->trees[cache_index(block, bc->num_locks)].lock);
+}
+
+/*
+ * Sometimes we want to repeatedly get and drop locks as part of an iteration.
+ * This struct helps avoid redundant drop and gets of the same lock.
+ */
+struct lock_history {
+	struct dm_buffer_cache *cache;
+	bool write;
+	unsigned int previous;
+	unsigned int no_previous;
+};
+
+static void lh_init(struct lock_history *lh, struct dm_buffer_cache *cache, bool write)
+{
+	lh->cache = cache;
+	lh->write = write;
+	lh->no_previous = cache->num_locks;
+	lh->previous = lh->no_previous;
+}
+
+static void __lh_lock(struct lock_history *lh, unsigned int index)
+{
+	if (lh->write)
+		down_write(&lh->cache->trees[index].lock);
 	else
-		return mutex_trylock(&c->lock);
+		down_read(&lh->cache->trees[index].lock);
 }
 
-static void dm_bufio_unlock(struct dm_bufio_client *c)
+static void __lh_unlock(struct lock_history *lh, unsigned int index)
 {
-	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
-		spin_unlock_bh(&c->spinlock);
+	if (lh->write)
+		up_write(&lh->cache->trees[index].lock);
 	else
-		mutex_unlock(&c->lock);
+		up_read(&lh->cache->trees[index].lock);
 }
 
-/*----------------------------------------------------------------*/
+/*
+ * Make sure you call this since it will unlock the final lock.
+ */
+static void lh_exit(struct lock_history *lh)
+{
+	if (lh->previous != lh->no_previous) {
+		__lh_unlock(lh, lh->previous);
+		lh->previous = lh->no_previous;
+	}
+}
 
 /*
- * Default cache size: available memory divided by the ratio.
+ * Named 'next' because there is no corresponding
+ * 'up/unlock' call since it's done automatically.
  */
-static unsigned long dm_bufio_default_cache_size;
+static void lh_next(struct lock_history *lh, sector_t b)
+{
+	unsigned int index = cache_index(b, lh->no_previous); /* no_previous is num_locks */
+
+	if (lh->previous != lh->no_previous) {
+		if (lh->previous != index) {
+			__lh_unlock(lh, lh->previous);
+			__lh_lock(lh, index);
+			lh->previous = index;
+		}
+	} else {
+		__lh_lock(lh, index);
+		lh->previous = index;
+	}
+}
+
+static inline struct dm_buffer *le_to_buffer(struct lru_entry *le)
+{
+	return container_of(le, struct dm_buffer, lru);
+}
+
+static struct dm_buffer *list_to_buffer(struct list_head *l)
+{
+	struct lru_entry *le = list_entry(l, struct lru_entry, list);
+
+	if (!le)
+		return NULL;
+
+	return le_to_buffer(le);
+}
+
+static void cache_init(struct dm_buffer_cache *bc, unsigned int num_locks)
+{
+	unsigned int i;
+
+	bc->num_locks = num_locks;
+
+	for (i = 0; i < bc->num_locks; i++) {
+		init_rwsem(&bc->trees[i].lock);
+		bc->trees[i].root = RB_ROOT;
+	}
+
+	lru_init(&bc->lru[LIST_CLEAN]);
+	lru_init(&bc->lru[LIST_DIRTY]);
+}
+
+static void cache_destroy(struct dm_buffer_cache *bc)
+{
+	unsigned int i;
+
+	for (i = 0; i < bc->num_locks; i++)
+		WARN_ON_ONCE(!RB_EMPTY_ROOT(&bc->trees[i].root));
+
+	lru_destroy(&bc->lru[LIST_CLEAN]);
+	lru_destroy(&bc->lru[LIST_DIRTY]);
+}
+
+/*--------------*/
 
 /*
- * Total cache size set by the user.
+ * not threadsafe, or racey depending how you look at it
  */
-static unsigned long dm_bufio_cache_size;
+static inline unsigned long cache_count(struct dm_buffer_cache *bc, int list_mode)
+{
+	return bc->lru[list_mode].count;
+}
+
+static inline unsigned long cache_total(struct dm_buffer_cache *bc)
+{
+	return cache_count(bc, LIST_CLEAN) + cache_count(bc, LIST_DIRTY);
+}
+
+/*--------------*/
 
 /*
- * A copy of dm_bufio_cache_size because dm_bufio_cache_size can change
- * at any time.  If it disagrees, the user has changed cache size.
+ * Gets a specific buffer, indexed by block.
+ * If the buffer is found then its holder count will be incremented and
+ * lru_reference will be called.
+ *
+ * threadsafe
  */
-static unsigned long dm_bufio_cache_size_latch;
+static struct dm_buffer *__cache_get(const struct rb_root *root, sector_t block)
+{
+	struct rb_node *n = root->rb_node;
+	struct dm_buffer *b;
 
-static DEFINE_SPINLOCK(global_spinlock);
+	while (n) {
+		b = container_of(n, struct dm_buffer, node);
+
+		if (b->block == block)
+			return b;
+
+		n = block < b->block ? n->rb_left : n->rb_right;
+	}
+
+	return NULL;
+}
+
+static void __cache_inc_buffer(struct dm_buffer *b)
+{
+	atomic_inc(&b->hold_count);
+	WRITE_ONCE(b->last_accessed, jiffies);
+}
+
+static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, sector_t block)
+{
+	struct dm_buffer *b;
+
+	cache_read_lock(bc, block);
+	b = __cache_get(&bc->trees[cache_index(block, bc->num_locks)].root, block);
+	if (b) {
+		lru_reference(&b->lru);
+		__cache_inc_buffer(b);
+	}
+	cache_read_unlock(bc, block);
 
-static LIST_HEAD(global_queue);
+	return b;
+}
 
-static unsigned long global_num;
+/*--------------*/
 
 /*
- * Buffers are freed after this timeout
+ * Returns true if the hold count hits zero.
+ * threadsafe
  */
-static unsigned int dm_bufio_max_age = DM_BUFIO_DEFAULT_AGE_SECS;
-static unsigned long dm_bufio_retain_bytes = DM_BUFIO_DEFAULT_RETAIN_BYTES;
+static bool cache_put(struct dm_buffer_cache *bc, struct dm_buffer *b)
+{
+	bool r;
 
-static unsigned long dm_bufio_peak_allocated;
-static unsigned long dm_bufio_allocated_kmem_cache;
-static unsigned long dm_bufio_allocated_get_free_pages;
-static unsigned long dm_bufio_allocated_vmalloc;
-static unsigned long dm_bufio_current_allocated;
+	cache_read_lock(bc, b->block);
+	BUG_ON(!atomic_read(&b->hold_count));
+	r = atomic_dec_and_test(&b->hold_count);
+	cache_read_unlock(bc, b->block);
 
-/*----------------------------------------------------------------*/
+	return r;
+}
+
+/*--------------*/
+
+typedef enum evict_result (*b_predicate)(struct dm_buffer *, void *);
 
 /*
- * The current number of clients.
+ * Evicts a buffer based on a predicate.  The oldest buffer that
+ * matches the predicate will be selected.  In addition to the
+ * predicate the hold_count of the selected buffer will be zero.
  */
-static int dm_bufio_client_count;
+struct evict_wrapper {
+	struct lock_history *lh;
+	b_predicate pred;
+	void *context;
+};
 
 /*
- * The list of all clients.
+ * Wraps the buffer predicate turning it into an lru predicate.  Adds
+ * extra test for hold_count.
  */
-static LIST_HEAD(dm_bufio_all_clients);
+static enum evict_result __evict_pred(struct lru_entry *le, void *context)
+{
+	struct evict_wrapper *w = context;
+	struct dm_buffer *b = le_to_buffer(le);
+
+	lh_next(w->lh, b->block);
+
+	if (atomic_read(&b->hold_count))
+		return ER_DONT_EVICT;
+
+	return w->pred(b, w->context);
+}
+
+static struct dm_buffer *__cache_evict(struct dm_buffer_cache *bc, int list_mode,
+				       b_predicate pred, void *context,
+				       struct lock_history *lh)
+{
+	struct evict_wrapper w = {.lh = lh, .pred = pred, .context = context};
+	struct lru_entry *le;
+	struct dm_buffer *b;
+
+	le = lru_evict(&bc->lru[list_mode], __evict_pred, &w);
+	if (!le)
+		return NULL;
+
+	b = le_to_buffer(le);
+	/* __evict_pred will have locked the appropriate tree. */
+	rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root);
+
+	return b;
+}
+
+static struct dm_buffer *cache_evict(struct dm_buffer_cache *bc, int list_mode,
+				     b_predicate pred, void *context)
+{
+	struct dm_buffer *b;
+	struct lock_history lh;
+
+	lh_init(&lh, bc, true);
+	b = __cache_evict(bc, list_mode, pred, context, &lh);
+	lh_exit(&lh);
+
+	return b;
+}
+
+/*--------------*/
 
 /*
- * This mutex protects dm_bufio_cache_size_latch and dm_bufio_client_count
+ * Mark a buffer as clean or dirty. Not threadsafe.
  */
-static DEFINE_MUTEX(dm_bufio_clients_lock);
+static void cache_mark(struct dm_buffer_cache *bc, struct dm_buffer *b, int list_mode)
+{
+	cache_write_lock(bc, b->block);
+	if (list_mode != b->list_mode) {
+		lru_remove(&bc->lru[b->list_mode], &b->lru);
+		b->list_mode = list_mode;
+		lru_insert(&bc->lru[b->list_mode], &b->lru);
+	}
+	cache_write_unlock(bc, b->block);
+}
 
-static struct workqueue_struct *dm_bufio_wq;
-static struct delayed_work dm_bufio_cleanup_old_work;
-static struct work_struct dm_bufio_replacement_work;
+/*--------------*/
 
+/*
+ * Runs through the lru associated with 'old_mode', if the predicate matches then
+ * it moves them to 'new_mode'.  Not threadsafe.
+ */
+static void __cache_mark_many(struct dm_buffer_cache *bc, int old_mode, int new_mode,
+			      b_predicate pred, void *context, struct lock_history *lh)
+{
+	struct lru_entry *le;
+	struct dm_buffer *b;
+	struct evict_wrapper w = {.lh = lh, .pred = pred, .context = context};
 
-#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
-static void buffer_record_stack(struct dm_buffer *b)
+	while (true) {
+		le = lru_evict(&bc->lru[old_mode], __evict_pred, &w);
+		if (!le)
+			break;
+
+		b = le_to_buffer(le);
+		b->list_mode = new_mode;
+		lru_insert(&bc->lru[b->list_mode], &b->lru);
+	}
+}
+
+static void cache_mark_many(struct dm_buffer_cache *bc, int old_mode, int new_mode,
+			    b_predicate pred, void *context)
 {
-	b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2);
+	struct lock_history lh;
+
+	lh_init(&lh, bc, true);
+	__cache_mark_many(bc, old_mode, new_mode, pred, context, &lh);
+	lh_exit(&lh);
 }
-#endif
+
+/*--------------*/
+
+/*
+ * Iterates through all clean or dirty entries calling a function for each
+ * entry.  The callback may terminate the iteration early.  Not threadsafe.
+ */
 
 /*
- *----------------------------------------------------------------
- * A red/black tree acts as an index for all the buffers.
- *----------------------------------------------------------------
+ * Iterator functions should return one of these actions to indicate
+ * how the iteration should proceed.
  */
-static struct dm_buffer *__find(struct dm_bufio_client *c, sector_t block)
+enum it_action {
+	IT_NEXT,
+	IT_COMPLETE,
+};
+
+typedef enum it_action (*iter_fn)(struct dm_buffer *b, void *context);
+
+static void __cache_iterate(struct dm_buffer_cache *bc, int list_mode,
+			    iter_fn fn, void *context, struct lock_history *lh)
 {
-	struct rb_node *n = c->buffer_tree.rb_node;
-	struct dm_buffer *b;
+	struct lru *lru = &bc->lru[list_mode];
+	struct lru_entry *le, *first;
 
-	while (n) {
-		b = container_of(n, struct dm_buffer, node);
+	if (!lru->cursor)
+		return;
 
-		if (b->block == block)
-			return b;
+	first = le = to_le(lru->cursor);
+	do {
+		struct dm_buffer *b = le_to_buffer(le);
 
-		n = block < b->block ? n->rb_left : n->rb_right;
+		lh_next(lh, b->block);
+
+		switch (fn(b, context)) {
+		case IT_NEXT:
+			break;
+
+		case IT_COMPLETE:
+			return;
+		}
+		cond_resched();
+
+		le = to_le(le->list.next);
+	} while (le != first);
+}
+
+static void cache_iterate(struct dm_buffer_cache *bc, int list_mode,
+			  iter_fn fn, void *context)
+{
+	struct lock_history lh;
+
+	lh_init(&lh, bc, false);
+	__cache_iterate(bc, list_mode, fn, context, &lh);
+	lh_exit(&lh);
+}
+
+/*--------------*/
+
+/*
+ * Passes ownership of the buffer to the cache. Returns false if the
+ * buffer was already present (in which case ownership does not pass).
+ * eg, a race with another thread.
+ *
+ * Holder count should be 1 on insertion.
+ *
+ * Not threadsafe.
+ */
+static bool __cache_insert(struct rb_root *root, struct dm_buffer *b)
+{
+	struct rb_node **new = &root->rb_node, *parent = NULL;
+	struct dm_buffer *found;
+
+	while (*new) {
+		found = container_of(*new, struct dm_buffer, node);
+
+		if (found->block == b->block)
+			return false;
+
+		parent = *new;
+		new = b->block < found->block ?
+			&found->node.rb_left : &found->node.rb_right;
 	}
 
-	return NULL;
+	rb_link_node(&b->node, parent, new);
+	rb_insert_color(&b->node, root);
+
+	return true;
+}
+
+static bool cache_insert(struct dm_buffer_cache *bc, struct dm_buffer *b)
+{
+	bool r;
+
+	if (WARN_ON_ONCE(b->list_mode >= LIST_SIZE))
+		return false;
+
+	cache_write_lock(bc, b->block);
+	BUG_ON(atomic_read(&b->hold_count) != 1);
+	r = __cache_insert(&bc->trees[cache_index(b->block, bc->num_locks)].root, b);
+	if (r)
+		lru_insert(&bc->lru[b->list_mode], &b->lru);
+	cache_write_unlock(bc, b->block);
+
+	return r;
 }
 
-static struct dm_buffer *__find_next(struct dm_bufio_client *c, sector_t block)
+/*--------------*/
+
+/*
+ * Removes buffer from cache, ownership of the buffer passes back to the caller.
+ * Fails if the hold_count is not one (ie. the caller holds the only reference).
+ *
+ * Not threadsafe.
+ */
+static bool cache_remove(struct dm_buffer_cache *bc, struct dm_buffer *b)
 {
-	struct rb_node *n = c->buffer_tree.rb_node;
+	bool r;
+
+	cache_write_lock(bc, b->block);
+
+	if (atomic_read(&b->hold_count) != 1) {
+		r = false;
+	} else {
+		r = true;
+		rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root);
+		lru_remove(&bc->lru[b->list_mode], &b->lru);
+	}
+
+	cache_write_unlock(bc, b->block);
+
+	return r;
+}
+
+/*--------------*/
+
+typedef void (*b_release)(struct dm_buffer *);
+
+static struct dm_buffer *__find_next(struct rb_root *root, sector_t block)
+{
+	struct rb_node *n = root->rb_node;
 	struct dm_buffer *b;
 	struct dm_buffer *best = NULL;
 
@@ -304,35 +882,188 @@ static struct dm_buffer *__find_next(struct dm_bufio_client *c, sector_t block)
 	return best;
 }
 
-static void __insert(struct dm_bufio_client *c, struct dm_buffer *b)
+static void __remove_range(struct dm_buffer_cache *bc,
+			   struct rb_root *root,
+			   sector_t begin, sector_t end,
+			   b_predicate pred, b_release release)
 {
-	struct rb_node **new = &c->buffer_tree.rb_node, *parent = NULL;
-	struct dm_buffer *found;
+	struct dm_buffer *b;
 
-	while (*new) {
-		found = container_of(*new, struct dm_buffer, node);
+	while (true) {
+		cond_resched();
 
-		if (found->block == b->block) {
-			BUG_ON(found != b);
-			return;
+		b = __find_next(root, begin);
+		if (!b || (b->block >= end))
+			break;
+
+		begin = b->block + 1;
+
+		if (atomic_read(&b->hold_count))
+			continue;
+
+		if (pred(b, NULL) == ER_EVICT) {
+			rb_erase(&b->node, root);
+			lru_remove(&bc->lru[b->list_mode], &b->lru);
+			release(b);
 		}
+	}
+}
 
-		parent = *new;
-		new = b->block < found->block ?
-			&found->node.rb_left : &found->node.rb_right;
+static void cache_remove_range(struct dm_buffer_cache *bc,
+			       sector_t begin, sector_t end,
+			       b_predicate pred, b_release release)
+{
+	unsigned int i;
+
+	for (i = 0; i < bc->num_locks; i++) {
+		down_write(&bc->trees[i].lock);
+		__remove_range(bc, &bc->trees[i].root, begin, end, pred, release);
+		up_write(&bc->trees[i].lock);
 	}
+}
 
-	rb_link_node(&b->node, parent, new);
-	rb_insert_color(&b->node, &c->buffer_tree);
+/*----------------------------------------------------------------*/
+
+/*
+ * Linking of buffers:
+ *	All buffers are linked to buffer_cache with their node field.
+ *
+ *	Clean buffers that are not being written (B_WRITING not set)
+ *	are linked to lru[LIST_CLEAN] with their lru_list field.
+ *
+ *	Dirty and clean buffers that are being written are linked to
+ *	lru[LIST_DIRTY] with their lru_list field. When the write
+ *	finishes, the buffer cannot be relinked immediately (because we
+ *	are in an interrupt context and relinking requires process
+ *	context), so some clean-not-writing buffers can be held on
+ *	dirty_lru too.  They are later added to lru in the process
+ *	context.
+ */
+struct dm_bufio_client {
+	struct block_device *bdev;
+	unsigned int block_size;
+	s8 sectors_per_block_bits;
+
+	bool no_sleep;
+	struct mutex lock;
+	spinlock_t spinlock;
+
+	int async_write_error;
+
+	void (*alloc_callback)(struct dm_buffer *buf);
+	void (*write_callback)(struct dm_buffer *buf);
+	struct kmem_cache *slab_buffer;
+	struct kmem_cache *slab_cache;
+	struct dm_io_client *dm_io;
+
+	struct list_head reserved_buffers;
+	unsigned int need_reserved_buffers;
+
+	unsigned int minimum_buffers;
+
+	sector_t start;
+
+	struct shrinker shrinker;
+	struct work_struct shrink_work;
+	atomic_long_t need_shrink;
+
+	wait_queue_head_t free_buffer_wait;
+
+	struct list_head client_list;
+
+	/*
+	 * Used by global_cleanup to sort the clients list.
+	 */
+	unsigned long oldest_buffer;
+
+	struct dm_buffer_cache cache; /* must be last member */
+};
+
+static DEFINE_STATIC_KEY_FALSE(no_sleep_enabled);
+
+/*----------------------------------------------------------------*/
+
+#define dm_bufio_in_request()	(!!current->bio_list)
+
+static void dm_bufio_lock(struct dm_bufio_client *c)
+{
+	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
+		spin_lock_bh(&c->spinlock);
+	else
+		mutex_lock_nested(&c->lock, dm_bufio_in_request());
 }
 
-static void __remove(struct dm_bufio_client *c, struct dm_buffer *b)
+static void dm_bufio_unlock(struct dm_bufio_client *c)
 {
-	rb_erase(&b->node, &c->buffer_tree);
+	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
+		spin_unlock_bh(&c->spinlock);
+	else
+		mutex_unlock(&c->lock);
 }
 
 /*----------------------------------------------------------------*/
 
+/*
+ * Default cache size: available memory divided by the ratio.
+ */
+static unsigned long dm_bufio_default_cache_size;
+
+/*
+ * Total cache size set by the user.
+ */
+static unsigned long dm_bufio_cache_size;
+
+/*
+ * A copy of dm_bufio_cache_size because dm_bufio_cache_size can change
+ * at any time.  If it disagrees, the user has changed cache size.
+ */
+static unsigned long dm_bufio_cache_size_latch;
+
+static DEFINE_SPINLOCK(global_spinlock);
+
+/*
+ * Buffers are freed after this timeout
+ */
+static unsigned int dm_bufio_max_age = DM_BUFIO_DEFAULT_AGE_SECS;
+static unsigned long dm_bufio_retain_bytes = DM_BUFIO_DEFAULT_RETAIN_BYTES;
+
+static unsigned long dm_bufio_peak_allocated;
+static unsigned long dm_bufio_allocated_kmem_cache;
+static unsigned long dm_bufio_allocated_get_free_pages;
+static unsigned long dm_bufio_allocated_vmalloc;
+static unsigned long dm_bufio_current_allocated;
+
+/*----------------------------------------------------------------*/
+
+/*
+ * The current number of clients.
+ */
+static int dm_bufio_client_count;
+
+/*
+ * The list of all clients.
+ */
+static LIST_HEAD(dm_bufio_all_clients);
+
+/*
+ * This mutex protects dm_bufio_cache_size_latch and dm_bufio_client_count
+ */
+static DEFINE_MUTEX(dm_bufio_clients_lock);
+
+static struct workqueue_struct *dm_bufio_wq;
+static struct delayed_work dm_bufio_cleanup_old_work;
+static struct work_struct dm_bufio_replacement_work;
+
+
+#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
+static void buffer_record_stack(struct dm_buffer *b)
+{
+	b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2);
+}
+#endif
+
+/*----------------------------------------------------------------*/
+
 static void adjust_total_allocated(struct dm_buffer *b, bool unlink)
 {
 	unsigned char data_mode;
@@ -358,16 +1089,9 @@ static void adjust_total_allocated(struct dm_buffer *b, bool unlink)
 	if (dm_bufio_current_allocated > dm_bufio_peak_allocated)
 		dm_bufio_peak_allocated = dm_bufio_current_allocated;
 
-	b->accessed = 1;
-
 	if (!unlink) {
-		list_add(&b->global_list, &global_queue);
-		global_num++;
 		if (dm_bufio_current_allocated > dm_bufio_cache_size)
 			queue_work(dm_bufio_wq, &dm_bufio_replacement_work);
-	} else {
-		list_del(&b->global_list);
-		global_num--;
 	}
 
 	spin_unlock(&global_spinlock);
@@ -378,8 +1102,10 @@ static void adjust_total_allocated(struct dm_buffer *b, bool unlink)
  */
 static void __cache_size_refresh(void)
 {
-	BUG_ON(!mutex_is_locked(&dm_bufio_clients_lock));
-	BUG_ON(dm_bufio_client_count < 0);
+	if (WARN_ON(!mutex_is_locked(&dm_bufio_clients_lock)))
+		return;
+	if (WARN_ON(dm_bufio_client_count < 0))
+		return;
 
 	dm_bufio_cache_size_latch = READ_ONCE(dm_bufio_cache_size);
 
@@ -495,6 +1221,7 @@ static struct dm_buffer *alloc_buffer(struct dm_bufio_client *c, gfp_t gfp_mask)
 		kmem_cache_free(c->slab_buffer, b);
 		return NULL;
 	}
+	adjust_total_allocated(b, false);
 
 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
 	b->stack_len = 0;
@@ -509,62 +1236,12 @@ static void free_buffer(struct dm_buffer *b)
 {
 	struct dm_bufio_client *c = b->c;
 
+	adjust_total_allocated(b, true);
 	free_buffer_data(c, b->data, b->data_mode);
 	kmem_cache_free(c->slab_buffer, b);
 }
 
 /*
- * Link buffer to the buffer tree and clean or dirty queue.
- */
-static void __link_buffer(struct dm_buffer *b, sector_t block, int dirty)
-{
-	struct dm_bufio_client *c = b->c;
-
-	c->n_buffers[dirty]++;
-	b->block = block;
-	b->list_mode = dirty;
-	list_add(&b->lru_list, &c->lru[dirty]);
-	__insert(b->c, b);
-	b->last_accessed = jiffies;
-
-	adjust_total_allocated(b, false);
-}
-
-/*
- * Unlink buffer from the buffer tree and dirty or clean queue.
- */
-static void __unlink_buffer(struct dm_buffer *b)
-{
-	struct dm_bufio_client *c = b->c;
-
-	BUG_ON(!c->n_buffers[b->list_mode]);
-
-	c->n_buffers[b->list_mode]--;
-	__remove(b->c, b);
-	list_del(&b->lru_list);
-
-	adjust_total_allocated(b, true);
-}
-
-/*
- * Place the buffer to the head of dirty or clean LRU queue.
- */
-static void __relink_lru(struct dm_buffer *b, int dirty)
-{
-	struct dm_bufio_client *c = b->c;
-
-	b->accessed = 1;
-
-	BUG_ON(!c->n_buffers[b->list_mode]);
-
-	c->n_buffers[b->list_mode]--;
-	c->n_buffers[dirty]++;
-	b->list_mode = dirty;
-	list_move(&b->lru_list, &c->lru[dirty]);
-	b->last_accessed = jiffies;
-}
-
-/*
  *--------------------------------------------------------------------------
  * Submit I/O on the buffer.
  *
@@ -639,19 +1316,14 @@ static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector,
 {
 	struct bio *bio;
 	char *ptr;
-	unsigned int vec_size, len;
+	unsigned int len;
 
-	vec_size = b->c->block_size >> PAGE_SHIFT;
-	if (unlikely(b->c->sectors_per_block_bits < PAGE_SHIFT - SECTOR_SHIFT))
-		vec_size += 2;
-
-	bio = bio_kmalloc(vec_size, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN);
+	bio = bio_kmalloc(1, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN);
 	if (!bio) {
-dmio:
 		use_dmio(b, op, sector, n_sectors, offset);
 		return;
 	}
-	bio_init(bio, b->c->bdev, bio->bi_inline_vecs, vec_size, op);
+	bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op);
 	bio->bi_iter.bi_sector = sector;
 	bio->bi_end_io = bio_complete;
 	bio->bi_private = b;
@@ -659,18 +1331,7 @@ dmio:
 	ptr = (char *)b->data + offset;
 	len = n_sectors << SECTOR_SHIFT;
 
-	do {
-		unsigned int this_step = min((unsigned int)(PAGE_SIZE - offset_in_page(ptr)), len);
-
-		if (!bio_add_page(bio, virt_to_page(ptr), this_step,
-				  offset_in_page(ptr))) {
-			bio_put(bio);
-			goto dmio;
-		}
-
-		len -= this_step;
-		ptr += this_step;
-	} while (len > 0);
+	__bio_add_page(bio, virt_to_page(ptr), len, offset_in_page(ptr));
 
 	submit_bio(bio);
 }
@@ -803,7 +1464,7 @@ static void __flush_write_list(struct list_head *write_list)
  */
 static void __make_buffer_clean(struct dm_buffer *b)
 {
-	BUG_ON(b->hold_count);
+	BUG_ON(atomic_read(&b->hold_count));
 
 	/* smp_load_acquire() pairs with read_endio()'s smp_mb__before_atomic() */
 	if (!smp_load_acquire(&b->state))	/* fast case */
@@ -814,6 +1475,36 @@ static void __make_buffer_clean(struct dm_buffer *b)
 	wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
 }
 
+static enum evict_result is_clean(struct dm_buffer *b, void *context)
+{
+	struct dm_bufio_client *c = context;
+
+	/* These should never happen */
+	if (WARN_ON_ONCE(test_bit(B_WRITING, &b->state)))
+		return ER_DONT_EVICT;
+	if (WARN_ON_ONCE(test_bit(B_DIRTY, &b->state)))
+		return ER_DONT_EVICT;
+	if (WARN_ON_ONCE(b->list_mode != LIST_CLEAN))
+		return ER_DONT_EVICT;
+
+	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep &&
+	    unlikely(test_bit(B_READING, &b->state)))
+		return ER_DONT_EVICT;
+
+	return ER_EVICT;
+}
+
+static enum evict_result is_dirty(struct dm_buffer *b, void *context)
+{
+	/* These should never happen */
+	if (WARN_ON_ONCE(test_bit(B_READING, &b->state)))
+		return ER_DONT_EVICT;
+	if (WARN_ON_ONCE(b->list_mode != LIST_DIRTY))
+		return ER_DONT_EVICT;
+
+	return ER_EVICT;
+}
+
 /*
  * Find some buffer that is not held by anybody, clean it, unlink it and
  * return it.
@@ -822,34 +1513,20 @@ static struct dm_buffer *__get_unclaimed_buffer(struct dm_bufio_client *c)
 {
 	struct dm_buffer *b;
 
-	list_for_each_entry_reverse(b, &c->lru[LIST_CLEAN], lru_list) {
-		BUG_ON(test_bit(B_WRITING, &b->state));
-		BUG_ON(test_bit(B_DIRTY, &b->state));
-
-		if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep &&
-		    unlikely(test_bit_acquire(B_READING, &b->state)))
-			continue;
-
-		if (!b->hold_count) {
-			__make_buffer_clean(b);
-			__unlink_buffer(b);
-			return b;
-		}
-		cond_resched();
+	b = cache_evict(&c->cache, LIST_CLEAN, is_clean, c);
+	if (b) {
+		/* this also waits for pending reads */
+		__make_buffer_clean(b);
+		return b;
 	}
 
 	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
 		return NULL;
 
-	list_for_each_entry_reverse(b, &c->lru[LIST_DIRTY], lru_list) {
-		BUG_ON(test_bit(B_READING, &b->state));
-
-		if (!b->hold_count) {
-			__make_buffer_clean(b);
-			__unlink_buffer(b);
-			return b;
-		}
-		cond_resched();
+	b = cache_evict(&c->cache, LIST_DIRTY, is_dirty, NULL);
+	if (b) {
+		__make_buffer_clean(b);
+		return b;
 	}
 
 	return NULL;
@@ -870,7 +1547,12 @@ static void __wait_for_free_buffer(struct dm_bufio_client *c)
 	set_current_state(TASK_UNINTERRUPTIBLE);
 	dm_bufio_unlock(c);
 
-	io_schedule();
+	/*
+	 * It's possible to miss a wake up event since we don't always
+	 * hold c->lock when wake_up is called.  So we have a timeout here,
+	 * just in case.
+	 */
+	io_schedule_timeout(5 * HZ);
 
 	remove_wait_queue(&c->free_buffer_wait, &wait);
 
@@ -928,9 +1610,8 @@ static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client
 		}
 
 		if (!list_empty(&c->reserved_buffers)) {
-			b = list_entry(c->reserved_buffers.next,
-				       struct dm_buffer, lru_list);
-			list_del(&b->lru_list);
+			b = list_to_buffer(c->reserved_buffers.next);
+			list_del(&b->lru.list);
 			c->need_reserved_buffers++;
 
 			return b;
@@ -964,36 +1645,61 @@ static void __free_buffer_wake(struct dm_buffer *b)
 {
 	struct dm_bufio_client *c = b->c;
 
+	b->block = -1;
 	if (!c->need_reserved_buffers)
 		free_buffer(b);
 	else {
-		list_add(&b->lru_list, &c->reserved_buffers);
+		list_add(&b->lru.list, &c->reserved_buffers);
 		c->need_reserved_buffers--;
 	}
 
-	wake_up(&c->free_buffer_wait);
+	/*
+	 * We hold the bufio lock here, so no one can add entries to the
+	 * wait queue anyway.
+	 */
+	if (unlikely(waitqueue_active(&c->free_buffer_wait)))
+		wake_up(&c->free_buffer_wait);
 }
 
-static void __write_dirty_buffers_async(struct dm_bufio_client *c, int no_wait,
-					struct list_head *write_list)
+static enum evict_result cleaned(struct dm_buffer *b, void *context)
 {
-	struct dm_buffer *b, *tmp;
+	if (WARN_ON_ONCE(test_bit(B_READING, &b->state)))
+		return ER_DONT_EVICT; /* should never happen */
 
-	list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) {
-		BUG_ON(test_bit(B_READING, &b->state));
+	if (test_bit(B_DIRTY, &b->state) || test_bit(B_WRITING, &b->state))
+		return ER_DONT_EVICT;
+	else
+		return ER_EVICT;
+}
 
-		if (!test_bit(B_DIRTY, &b->state) &&
-		    !test_bit(B_WRITING, &b->state)) {
-			__relink_lru(b, LIST_CLEAN);
-			continue;
-		}
+static void __move_clean_buffers(struct dm_bufio_client *c)
+{
+	cache_mark_many(&c->cache, LIST_DIRTY, LIST_CLEAN, cleaned, NULL);
+}
 
-		if (no_wait && test_bit(B_WRITING, &b->state))
-			return;
+struct write_context {
+	int no_wait;
+	struct list_head *write_list;
+};
 
-		__write_dirty_buffer(b, write_list);
-		cond_resched();
-	}
+static enum it_action write_one(struct dm_buffer *b, void *context)
+{
+	struct write_context *wc = context;
+
+	if (wc->no_wait && test_bit(B_WRITING, &b->state))
+		return IT_COMPLETE;
+
+	__write_dirty_buffer(b, wc->write_list);
+	return IT_NEXT;
+}
+
+static void __write_dirty_buffers_async(struct dm_bufio_client *c, int no_wait,
+					struct list_head *write_list)
+{
+	struct write_context wc = {.no_wait = no_wait, .write_list = write_list};
+
+	__move_clean_buffers(c);
+	cache_iterate(&c->cache, LIST_DIRTY, write_one, &wc);
 }
 
 /*
@@ -1004,7 +1710,8 @@ static void __write_dirty_buffers_async(struct dm_bufio_client *c, int no_wait,
 static void __check_watermark(struct dm_bufio_client *c,
 			      struct list_head *write_list)
 {
-	if (c->n_buffers[LIST_DIRTY] > c->n_buffers[LIST_CLEAN] * DM_BUFIO_WRITEBACK_RATIO)
+	if (cache_count(&c->cache, LIST_DIRTY) >
+	    cache_count(&c->cache, LIST_CLEAN) * DM_BUFIO_WRITEBACK_RATIO)
 		__write_dirty_buffers_async(c, 1, write_list);
 }
 
@@ -1014,6 +1721,21 @@ static void __check_watermark(struct dm_bufio_client *c,
  *--------------------------------------------------------------
  */
 
+static void cache_put_and_wake(struct dm_bufio_client *c, struct dm_buffer *b)
+{
+	/*
+	 * Relying on waitqueue_active() is racey, but we sleep
+	 * with schedule_timeout anyway.
+	 */
+	if (cache_put(&c->cache, b) &&
+	    unlikely(waitqueue_active(&c->free_buffer_wait)))
+		wake_up(&c->free_buffer_wait);
+}
+
+/*
+ * This assumes you have already checked the cache to see if the buffer
+ * is already present (it will recheck after dropping the lock for allocation).
+ */
 static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
 				     enum new_flag nf, int *need_submit,
 				     struct list_head *write_list)
@@ -1022,11 +1744,8 @@ static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
 
 	*need_submit = 0;
 
-	b = __find(c, block);
-	if (b)
-		goto found_buffer;
-
-	if (nf == NF_GET)
+	/* This can't be called with NF_GET */
+	if (WARN_ON_ONCE(nf == NF_GET))
 		return NULL;
 
 	new_b = __alloc_buffer_wait(c, nf);
@@ -1037,7 +1756,7 @@ static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
 	 * We've had a period where the mutex was unlocked, so need to
 	 * recheck the buffer tree.
 	 */
-	b = __find(c, block);
+	b = cache_get(&c->cache, block);
 	if (b) {
 		__free_buffer_wake(new_b);
 		goto found_buffer;
@@ -1046,24 +1765,35 @@ static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
 	__check_watermark(c, write_list);
 
 	b = new_b;
-	b->hold_count = 1;
+	atomic_set(&b->hold_count, 1);
+	WRITE_ONCE(b->last_accessed, jiffies);
+	b->block = block;
 	b->read_error = 0;
 	b->write_error = 0;
-	__link_buffer(b, block, LIST_CLEAN);
+	b->list_mode = LIST_CLEAN;
 
-	if (nf == NF_FRESH) {
+	if (nf == NF_FRESH)
 		b->state = 0;
-		return b;
+	else {
+		b->state = 1 << B_READING;
+		*need_submit = 1;
 	}
 
-	b->state = 1 << B_READING;
-	*need_submit = 1;
+	/*
+	 * We mustn't insert into the cache until the B_READING state
+	 * is set.  Otherwise another thread could get it and use
+	 * it before it had been read.
+	 */
+	cache_insert(&c->cache, b);
 
 	return b;
 
 found_buffer:
-	if (nf == NF_PREFETCH)
+	if (nf == NF_PREFETCH) {
+		cache_put_and_wake(c, b);
 		return NULL;
+	}
+
 	/*
 	 * Note: it is essential that we don't wait for the buffer to be
 	 * read if dm_bufio_get function is used. Both dm_bufio_get and
@@ -1071,12 +1801,11 @@ found_buffer:
 	 * If the user called both dm_bufio_prefetch and dm_bufio_get on
 	 * the same buffer, it would deadlock if we waited.
 	 */
-	if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state)))
+	if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) {
+		cache_put_and_wake(c, b);
 		return NULL;
+	}
 
-	b->hold_count++;
-	__relink_lru(b, test_bit(B_DIRTY, &b->state) ||
-		     test_bit(B_WRITING, &b->state));
 	return b;
 }
 
@@ -1106,18 +1835,50 @@ static void read_endio(struct dm_buffer *b, blk_status_t status)
 static void *new_read(struct dm_bufio_client *c, sector_t block,
 		      enum new_flag nf, struct dm_buffer **bp)
 {
-	int need_submit;
+	int need_submit = 0;
 	struct dm_buffer *b;
 
 	LIST_HEAD(write_list);
 
-	dm_bufio_lock(c);
-	b = __bufio_new(c, block, nf, &need_submit, &write_list);
+	*bp = NULL;
+
+	/*
+	 * Fast path, hopefully the block is already in the cache.  No need
+	 * to get the client lock for this.
+	 */
+	b = cache_get(&c->cache, block);
+	if (b) {
+		if (nf == NF_PREFETCH) {
+			cache_put_and_wake(c, b);
+			return NULL;
+		}
+
+		/*
+		 * Note: it is essential that we don't wait for the buffer to be
+		 * read if dm_bufio_get function is used. Both dm_bufio_get and
+		 * dm_bufio_prefetch can be used in the driver request routine.
+		 * If the user called both dm_bufio_prefetch and dm_bufio_get on
+		 * the same buffer, it would deadlock if we waited.
+		 */
+		if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) {
+			cache_put_and_wake(c, b);
+			return NULL;
+		}
+	}
+
+	if (!b) {
+		if (nf == NF_GET)
+			return NULL;
+
+		dm_bufio_lock(c);
+		b = __bufio_new(c, block, nf, &need_submit, &write_list);
+		dm_bufio_unlock(c);
+	}
+
 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
-	if (b && b->hold_count == 1)
+	if (b && (atomic_read(&b->hold_count) == 1))
 		buffer_record_stack(b);
 #endif
-	dm_bufio_unlock(c);
 
 	__flush_write_list(&write_list);
 
@@ -1152,7 +1913,8 @@ EXPORT_SYMBOL_GPL(dm_bufio_get);
 void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
 		    struct dm_buffer **bp)
 {
-	BUG_ON(dm_bufio_in_request());
+	if (WARN_ON_ONCE(dm_bufio_in_request()))
+		return ERR_PTR(-EINVAL);
 
 	return new_read(c, block, NF_READ, bp);
 }
@@ -1161,7 +1923,8 @@ EXPORT_SYMBOL_GPL(dm_bufio_read);
 void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
 		   struct dm_buffer **bp)
 {
-	BUG_ON(dm_bufio_in_request());
+	if (WARN_ON_ONCE(dm_bufio_in_request()))
+		return ERR_PTR(-EINVAL);
 
 	return new_read(c, block, NF_FRESH, bp);
 }
@@ -1174,15 +1937,23 @@ void dm_bufio_prefetch(struct dm_bufio_client *c,
 
 	LIST_HEAD(write_list);
 
-	BUG_ON(dm_bufio_in_request());
+	if (WARN_ON_ONCE(dm_bufio_in_request()))
+		return; /* should never happen */
 
 	blk_start_plug(&plug);
-	dm_bufio_lock(c);
 
 	for (; n_blocks--; block++) {
 		int need_submit;
 		struct dm_buffer *b;
 
+		b = cache_get(&c->cache, block);
+		if (b) {
+			/* already in cache */
+			cache_put_and_wake(c, b);
+			continue;
+		}
+
+		dm_bufio_lock(c);
 		b = __bufio_new(c, block, NF_PREFETCH, &need_submit,
 				&write_list);
 		if (unlikely(!list_empty(&write_list))) {
@@ -1205,10 +1976,9 @@ void dm_bufio_prefetch(struct dm_bufio_client *c,
 				goto flush_plug;
 			dm_bufio_lock(c);
 		}
+		dm_bufio_unlock(c);
 	}
 
-	dm_bufio_unlock(c);
-
 flush_plug:
 	blk_finish_plug(&plug);
 }
@@ -1218,29 +1988,28 @@ void dm_bufio_release(struct dm_buffer *b)
 {
 	struct dm_bufio_client *c = b->c;
 
-	dm_bufio_lock(c);
-
-	BUG_ON(!b->hold_count);
-
-	b->hold_count--;
-	if (!b->hold_count) {
-		wake_up(&c->free_buffer_wait);
+	/*
+	 * If there were errors on the buffer, and the buffer is not
+	 * to be written, free the buffer. There is no point in caching
+	 * invalid buffer.
+	 */
+	if ((b->read_error || b->write_error) &&
+	    !test_bit_acquire(B_READING, &b->state) &&
+	    !test_bit(B_WRITING, &b->state) &&
+	    !test_bit(B_DIRTY, &b->state)) {
+		dm_bufio_lock(c);
 
-		/*
-		 * If there were errors on the buffer, and the buffer is not
-		 * to be written, free the buffer. There is no point in caching
-		 * invalid buffer.
-		 */
-		if ((b->read_error || b->write_error) &&
-		    !test_bit_acquire(B_READING, &b->state) &&
-		    !test_bit(B_WRITING, &b->state) &&
-		    !test_bit(B_DIRTY, &b->state)) {
-			__unlink_buffer(b);
+		/* cache remove can fail if there are other holders */
+		if (cache_remove(&c->cache, b)) {
 			__free_buffer_wake(b);
+			dm_bufio_unlock(c);
+			return;
 		}
+
+		dm_bufio_unlock(c);
 	}
 
-	dm_bufio_unlock(c);
+	cache_put_and_wake(c, b);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_release);
 
@@ -1259,7 +2028,7 @@ void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b,
 	if (!test_and_set_bit(B_DIRTY, &b->state)) {
 		b->dirty_start = start;
 		b->dirty_end = end;
-		__relink_lru(b, LIST_DIRTY);
+		cache_mark(&c->cache, b, LIST_DIRTY);
 	} else {
 		if (start < b->dirty_start)
 			b->dirty_start = start;
@@ -1281,7 +2050,8 @@ void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c)
 {
 	LIST_HEAD(write_list);
 
-	BUG_ON(dm_bufio_in_request());
+	if (WARN_ON_ONCE(dm_bufio_in_request()))
+		return; /* should never happen */
 
 	dm_bufio_lock(c);
 	__write_dirty_buffers_async(c, 0, &write_list);
@@ -1297,11 +2067,19 @@ EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers_async);
  *
  * Finally, we flush hardware disk cache.
  */
+static bool is_writing(struct lru_entry *e, void *context)
+{
+	struct dm_buffer *b = le_to_buffer(e);
+
+	return test_bit(B_WRITING, &b->state);
+}
+
 int dm_bufio_write_dirty_buffers(struct dm_bufio_client *c)
 {
 	int a, f;
-	unsigned long buffers_processed = 0;
-	struct dm_buffer *b, *tmp;
+	unsigned long nr_buffers;
+	struct lru_entry *e;
+	struct lru_iter it;
 
 	LIST_HEAD(write_list);
 
@@ -1311,52 +2089,32 @@ int dm_bufio_write_dirty_buffers(struct dm_bufio_client *c)
 	__flush_write_list(&write_list);
 	dm_bufio_lock(c);
 
-again:
-	list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) {
-		int dropped_lock = 0;
-
-		if (buffers_processed < c->n_buffers[LIST_DIRTY])
-			buffers_processed++;
+	nr_buffers = cache_count(&c->cache, LIST_DIRTY);
+	lru_iter_begin(&c->cache.lru[LIST_DIRTY], &it);
+	while ((e = lru_iter_next(&it, is_writing, c))) {
+		struct dm_buffer *b = le_to_buffer(e);
+		__cache_inc_buffer(b);
 
 		BUG_ON(test_bit(B_READING, &b->state));
 
-		if (test_bit(B_WRITING, &b->state)) {
-			if (buffers_processed < c->n_buffers[LIST_DIRTY]) {
-				dropped_lock = 1;
-				b->hold_count++;
-				dm_bufio_unlock(c);
-				wait_on_bit_io(&b->state, B_WRITING,
-					       TASK_UNINTERRUPTIBLE);
-				dm_bufio_lock(c);
-				b->hold_count--;
-			} else
-				wait_on_bit_io(&b->state, B_WRITING,
-					       TASK_UNINTERRUPTIBLE);
+		if (nr_buffers) {
+			nr_buffers--;
+			dm_bufio_unlock(c);
+			wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
+			dm_bufio_lock(c);
+		} else {
+			wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
 		}
 
-		if (!test_bit(B_DIRTY, &b->state) &&
-		    !test_bit(B_WRITING, &b->state))
-			__relink_lru(b, LIST_CLEAN);
+		if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state))
+			cache_mark(&c->cache, b, LIST_CLEAN);
 
-		cond_resched();
+		cache_put_and_wake(c, b);
 
-		/*
-		 * If we dropped the lock, the list is no longer consistent,
-		 * so we must restart the search.
-		 *
-		 * In the most common case, the buffer just processed is
-		 * relinked to the clean list, so we won't loop scanning the
-		 * same buffer again and again.
-		 *
-		 * This may livelock if there is another thread simultaneously
-		 * dirtying buffers, so we count the number of buffers walked
-		 * and if it exceeds the total number of buffers, it means that
-		 * someone is doing some writes simultaneously with us.  In
-		 * this case, stop, dropping the lock.
-		 */
-		if (dropped_lock)
-			goto again;
+		cond_resched();
 	}
+	lru_iter_end(&it);
+
 	wake_up(&c->free_buffer_wait);
 	dm_bufio_unlock(c);
 
@@ -1386,7 +2144,8 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c)
 		.count = 0,
 	};
 
-	BUG_ON(dm_bufio_in_request());
+	if (WARN_ON_ONCE(dm_bufio_in_request()))
+		return -EINVAL;
 
 	return dm_io(&io_req, 1, &io_reg, NULL);
 }
@@ -1409,95 +2168,30 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c
 		.count = block_to_sector(c, count),
 	};
 
-	BUG_ON(dm_bufio_in_request());
+	if (WARN_ON_ONCE(dm_bufio_in_request()))
+		return -EINVAL; /* discards are optional */
 
 	return dm_io(&io_req, 1, &io_reg, NULL);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_issue_discard);
 
-/*
- * We first delete any other buffer that may be at that new location.
- *
- * Then, we write the buffer to the original location if it was dirty.
- *
- * Then, if we are the only one who is holding the buffer, relink the buffer
- * in the buffer tree for the new location.
- *
- * If there was someone else holding the buffer, we write it to the new
- * location but not relink it, because that other user needs to have the buffer
- * at the same place.
- */
-void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block)
+static bool forget_buffer(struct dm_bufio_client *c, sector_t block)
 {
-	struct dm_bufio_client *c = b->c;
-	struct dm_buffer *new;
-
-	BUG_ON(dm_bufio_in_request());
-
-	dm_bufio_lock(c);
+	struct dm_buffer *b;
 
-retry:
-	new = __find(c, new_block);
-	if (new) {
-		if (new->hold_count) {
-			__wait_for_free_buffer(c);
-			goto retry;
+	b = cache_get(&c->cache, block);
+	if (b) {
+		if (likely(!smp_load_acquire(&b->state))) {
+			if (cache_remove(&c->cache, b))
+				__free_buffer_wake(b);
+			else
+				cache_put_and_wake(c, b);
+		} else {
+			cache_put_and_wake(c, b);
 		}
-
-		/*
-		 * FIXME: Is there any point waiting for a write that's going
-		 * to be overwritten in a bit?
-		 */
-		__make_buffer_clean(new);
-		__unlink_buffer(new);
-		__free_buffer_wake(new);
 	}
 
-	BUG_ON(!b->hold_count);
-	BUG_ON(test_bit(B_READING, &b->state));
-
-	__write_dirty_buffer(b, NULL);
-	if (b->hold_count == 1) {
-		wait_on_bit_io(&b->state, B_WRITING,
-			       TASK_UNINTERRUPTIBLE);
-		set_bit(B_DIRTY, &b->state);
-		b->dirty_start = 0;
-		b->dirty_end = c->block_size;
-		__unlink_buffer(b);
-		__link_buffer(b, new_block, LIST_DIRTY);
-	} else {
-		sector_t old_block;
-
-		wait_on_bit_lock_io(&b->state, B_WRITING,
-				    TASK_UNINTERRUPTIBLE);
-		/*
-		 * Relink buffer to "new_block" so that write_callback
-		 * sees "new_block" as a block number.
-		 * After the write, link the buffer back to old_block.
-		 * All this must be done in bufio lock, so that block number
-		 * change isn't visible to other threads.
-		 */
-		old_block = b->block;
-		__unlink_buffer(b);
-		__link_buffer(b, new_block, b->list_mode);
-		submit_io(b, REQ_OP_WRITE, write_endio);
-		wait_on_bit_io(&b->state, B_WRITING,
-			       TASK_UNINTERRUPTIBLE);
-		__unlink_buffer(b);
-		__link_buffer(b, old_block, b->list_mode);
-	}
-
-	dm_bufio_unlock(c);
-	dm_bufio_release(b);
-}
-EXPORT_SYMBOL_GPL(dm_bufio_release_move);
-
-static void forget_buffer_locked(struct dm_buffer *b)
-{
-	if (likely(!b->hold_count) && likely(!smp_load_acquire(&b->state))) {
-		__unlink_buffer(b);
-		__free_buffer_wake(b);
-	}
+	return b ? true : false;
 }
 
 /*
@@ -1508,38 +2202,22 @@ static void forget_buffer_locked(struct dm_buffer *b)
  */
 void dm_bufio_forget(struct dm_bufio_client *c, sector_t block)
 {
-	struct dm_buffer *b;
-
 	dm_bufio_lock(c);
-
-	b = __find(c, block);
-	if (b)
-		forget_buffer_locked(b);
-
+	forget_buffer(c, block);
 	dm_bufio_unlock(c);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_forget);
 
-void dm_bufio_forget_buffers(struct dm_bufio_client *c, sector_t block, sector_t n_blocks)
+static enum evict_result idle(struct dm_buffer *b, void *context)
 {
-	struct dm_buffer *b;
-	sector_t end_block = block + n_blocks;
-
-	while (block < end_block) {
-		dm_bufio_lock(c);
-
-		b = __find_next(c, block);
-		if (b) {
-			block = b->block + 1;
-			forget_buffer_locked(b);
-		}
-
-		dm_bufio_unlock(c);
-
-		if (!b)
-			break;
-	}
+	return b->state ? ER_DONT_EVICT : ER_EVICT;
+}
 
+void dm_bufio_forget_buffers(struct dm_bufio_client *c, sector_t block, sector_t n_blocks)
+{
+	dm_bufio_lock(c);
+	cache_remove_range(&c->cache, block, block + n_blocks, idle, __free_buffer_wake);
+	dm_bufio_unlock(c);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_forget_buffers);
 
@@ -1601,13 +2279,29 @@ struct dm_bufio_client *dm_bufio_get_client(struct dm_buffer *b)
 }
 EXPORT_SYMBOL_GPL(dm_bufio_get_client);
 
+static enum it_action warn_leak(struct dm_buffer *b, void *context)
+{
+	bool *warned = context;
+
+	WARN_ON(!(*warned));
+	*warned = true;
+	DMERR("leaked buffer %llx, hold count %u, list %d",
+	      (unsigned long long)b->block, atomic_read(&b->hold_count), b->list_mode);
+#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
+	stack_trace_print(b->stack_entries, b->stack_len, 1);
+	/* mark unclaimed to avoid WARN_ON at end of drop_buffers() */
+	atomic_set(&b->hold_count, 0);
+#endif
+	return IT_NEXT;
+}
+
 static void drop_buffers(struct dm_bufio_client *c)
 {
-	struct dm_buffer *b;
 	int i;
-	bool warned = false;
+	struct dm_buffer *b;
 
-	BUG_ON(dm_bufio_in_request());
+	if (WARN_ON(dm_bufio_in_request()))
+		return; /* should never happen */
 
 	/*
 	 * An optimization so that the buffers are not written one-by-one.
@@ -1619,18 +2313,11 @@ static void drop_buffers(struct dm_bufio_client *c)
 	while ((b = __get_unclaimed_buffer(c)))
 		__free_buffer_wake(b);
 
-	for (i = 0; i < LIST_SIZE; i++)
-		list_for_each_entry(b, &c->lru[i], lru_list) {
-			WARN_ON(!warned);
-			warned = true;
-			DMERR("leaked buffer %llx, hold count %u, list %d",
-			      (unsigned long long)b->block, b->hold_count, i);
-#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
-			stack_trace_print(b->stack_entries, b->stack_len, 1);
-			/* mark unclaimed to avoid BUG_ON below */
-			b->hold_count = 0;
-#endif
-		}
+	for (i = 0; i < LIST_SIZE; i++) {
+		bool warned = false;
+
+		cache_iterate(&c->cache, i, warn_leak, &warned);
+	}
 
 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
 	while ((b = __get_unclaimed_buffer(c)))
@@ -1638,39 +2325,11 @@ static void drop_buffers(struct dm_bufio_client *c)
 #endif
 
 	for (i = 0; i < LIST_SIZE; i++)
-		BUG_ON(!list_empty(&c->lru[i]));
+		WARN_ON(cache_count(&c->cache, i));
 
 	dm_bufio_unlock(c);
 }
 
-/*
- * We may not be able to evict this buffer if IO pending or the client
- * is still using it.  Caller is expected to know buffer is too old.
- *
- * And if GFP_NOFS is used, we must not do any I/O because we hold
- * dm_bufio_clients_lock and we would risk deadlock if the I/O gets
- * rerouted to different bufio client.
- */
-static bool __try_evict_buffer(struct dm_buffer *b, gfp_t gfp)
-{
-	if (!(gfp & __GFP_FS) ||
-	    (static_branch_unlikely(&no_sleep_enabled) && b->c->no_sleep)) {
-		if (test_bit_acquire(B_READING, &b->state) ||
-		    test_bit(B_WRITING, &b->state) ||
-		    test_bit(B_DIRTY, &b->state))
-			return false;
-	}
-
-	if (b->hold_count)
-		return false;
-
-	__make_buffer_clean(b);
-	__unlink_buffer(b);
-	__free_buffer_wake(b);
-
-	return true;
-}
-
 static unsigned long get_retain_buffers(struct dm_bufio_client *c)
 {
 	unsigned long retain_bytes = READ_ONCE(dm_bufio_retain_bytes);
@@ -1686,22 +2345,28 @@ static unsigned long get_retain_buffers(struct dm_bufio_client *c)
 static void __scan(struct dm_bufio_client *c)
 {
 	int l;
-	struct dm_buffer *b, *tmp;
+	struct dm_buffer *b;
 	unsigned long freed = 0;
-	unsigned long count = c->n_buffers[LIST_CLEAN] +
-			      c->n_buffers[LIST_DIRTY];
 	unsigned long retain_target = get_retain_buffers(c);
+	unsigned long count = cache_total(&c->cache);
 
 	for (l = 0; l < LIST_SIZE; l++) {
-		list_for_each_entry_safe_reverse(b, tmp, &c->lru[l], lru_list) {
+		while (true) {
 			if (count - freed <= retain_target)
 				atomic_long_set(&c->need_shrink, 0);
 			if (!atomic_long_read(&c->need_shrink))
-				return;
-			if (__try_evict_buffer(b, GFP_KERNEL)) {
-				atomic_long_dec(&c->need_shrink);
-				freed++;
-			}
+				break;
+
+			b = cache_evict(&c->cache, l,
+					l == LIST_CLEAN ? is_clean : is_dirty, c);
+			if (!b)
+				break;
+
+			__make_buffer_clean(b);
+			__free_buffer_wake(b);
+
+			atomic_long_dec(&c->need_shrink);
+			freed++;
 			cond_resched();
 		}
 	}
@@ -1730,8 +2395,7 @@ static unsigned long dm_bufio_shrink_scan(struct shrinker *shrink, struct shrink
 static unsigned long dm_bufio_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
 {
 	struct dm_bufio_client *c = container_of(shrink, struct dm_bufio_client, shrinker);
-	unsigned long count = READ_ONCE(c->n_buffers[LIST_CLEAN]) +
-			      READ_ONCE(c->n_buffers[LIST_DIRTY]);
+	unsigned long count = cache_total(&c->cache);
 	unsigned long retain_target = get_retain_buffers(c);
 	unsigned long queued_for_cleanup = atomic_long_read(&c->need_shrink);
 
@@ -1758,8 +2422,8 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign
 					       unsigned int flags)
 {
 	int r;
+	unsigned int num_locks;
 	struct dm_bufio_client *c;
-	unsigned int i;
 	char slab_name[27];
 
 	if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) {
@@ -1768,12 +2432,13 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign
 		goto bad_client;
 	}
 
-	c = kzalloc(sizeof(*c), GFP_KERNEL);
+	num_locks = dm_num_hash_locks();
+	c = kzalloc(sizeof(*c) + (num_locks * sizeof(struct buffer_tree)), GFP_KERNEL);
 	if (!c) {
 		r = -ENOMEM;
 		goto bad_client;
 	}
-	c->buffer_tree = RB_ROOT;
+	cache_init(&c->cache, num_locks);
 
 	c->bdev = bdev;
 	c->block_size = block_size;
@@ -1790,11 +2455,6 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign
 		static_branch_inc(&no_sleep_enabled);
 	}
 
-	for (i = 0; i < LIST_SIZE; i++) {
-		INIT_LIST_HEAD(&c->lru[i]);
-		c->n_buffers[i] = 0;
-	}
-
 	mutex_init(&c->lock);
 	spin_lock_init(&c->spinlock);
 	INIT_LIST_HEAD(&c->reserved_buffers);
@@ -1866,9 +2526,9 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign
 
 bad:
 	while (!list_empty(&c->reserved_buffers)) {
-		struct dm_buffer *b = list_entry(c->reserved_buffers.next,
-						 struct dm_buffer, lru_list);
-		list_del(&b->lru_list);
+		struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next);
+
+		list_del(&b->lru.list);
 		free_buffer(b);
 	}
 	kmem_cache_destroy(c->slab_cache);
@@ -1905,23 +2565,23 @@ void dm_bufio_client_destroy(struct dm_bufio_client *c)
 
 	mutex_unlock(&dm_bufio_clients_lock);
 
-	BUG_ON(!RB_EMPTY_ROOT(&c->buffer_tree));
-	BUG_ON(c->need_reserved_buffers);
+	WARN_ON(c->need_reserved_buffers);
 
 	while (!list_empty(&c->reserved_buffers)) {
-		struct dm_buffer *b = list_entry(c->reserved_buffers.next,
-						 struct dm_buffer, lru_list);
-		list_del(&b->lru_list);
+		struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next);
+
+		list_del(&b->lru.list);
 		free_buffer(b);
 	}
 
 	for (i = 0; i < LIST_SIZE; i++)
-		if (c->n_buffers[i])
-			DMERR("leaked buffer count %d: %ld", i, c->n_buffers[i]);
+		if (cache_count(&c->cache, i))
+			DMERR("leaked buffer count %d: %lu", i, cache_count(&c->cache, i));
 
 	for (i = 0; i < LIST_SIZE; i++)
-		BUG_ON(c->n_buffers[i]);
+		WARN_ON(cache_count(&c->cache, i));
 
+	cache_destroy(&c->cache);
 	kmem_cache_destroy(c->slab_cache);
 	kmem_cache_destroy(c->slab_buffer);
 	dm_io_client_destroy(c->dm_io);
@@ -1938,6 +2598,8 @@ void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start)
 }
 EXPORT_SYMBOL_GPL(dm_bufio_set_sector_offset);
 
+/*--------------------------------------------------------------*/
+
 static unsigned int get_max_age_hz(void)
 {
 	unsigned int max_age = READ_ONCE(dm_bufio_max_age);
@@ -1950,13 +2612,74 @@ static unsigned int get_max_age_hz(void)
 
 static bool older_than(struct dm_buffer *b, unsigned long age_hz)
 {
-	return time_after_eq(jiffies, b->last_accessed + age_hz);
+	return time_after_eq(jiffies, READ_ONCE(b->last_accessed) + age_hz);
+}
+
+struct evict_params {
+	gfp_t gfp;
+	unsigned long age_hz;
+
+	/*
+	 * This gets updated with the largest last_accessed (ie. most
+	 * recently used) of the evicted buffers.  It will not be reinitialised
+	 * by __evict_many(), so you can use it across multiple invocations.
+	 */
+	unsigned long last_accessed;
+};
+
+/*
+ * We may not be able to evict this buffer if IO pending or the client
+ * is still using it.
+ *
+ * And if GFP_NOFS is used, we must not do any I/O because we hold
+ * dm_bufio_clients_lock and we would risk deadlock if the I/O gets
+ * rerouted to different bufio client.
+ */
+static enum evict_result select_for_evict(struct dm_buffer *b, void *context)
+{
+	struct evict_params *params = context;
+
+	if (!(params->gfp & __GFP_FS) ||
+	    (static_branch_unlikely(&no_sleep_enabled) && b->c->no_sleep)) {
+		if (test_bit_acquire(B_READING, &b->state) ||
+		    test_bit(B_WRITING, &b->state) ||
+		    test_bit(B_DIRTY, &b->state))
+			return ER_DONT_EVICT;
+	}
+
+	return older_than(b, params->age_hz) ? ER_EVICT : ER_STOP;
 }
 
-static void __evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
+static unsigned long __evict_many(struct dm_bufio_client *c,
+				  struct evict_params *params,
+				  int list_mode, unsigned long max_count)
 {
-	struct dm_buffer *b, *tmp;
-	unsigned long retain_target = get_retain_buffers(c);
+	unsigned long count;
+	unsigned long last_accessed;
+	struct dm_buffer *b;
+
+	for (count = 0; count < max_count; count++) {
+		b = cache_evict(&c->cache, list_mode, select_for_evict, params);
+		if (!b)
+			break;
+
+		last_accessed = READ_ONCE(b->last_accessed);
+		if (time_after_eq(params->last_accessed, last_accessed))
+			params->last_accessed = last_accessed;
+
+		__make_buffer_clean(b);
+		__free_buffer_wake(b);
+
+		cond_resched();
+	}
+
+	return count;
+}
+
+static void evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
+{
+	struct evict_params params = {.gfp = 0, .age_hz = age_hz, .last_accessed = 0};
+	unsigned long retain = get_retain_buffers(c);
 	unsigned long count;
 	LIST_HEAD(write_list);
 
@@ -1969,112 +2692,135 @@ static void __evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
 		dm_bufio_lock(c);
 	}
 
-	count = c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY];
-	list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_CLEAN], lru_list) {
-		if (count <= retain_target)
-			break;
-
-		if (!older_than(b, age_hz))
-			break;
-
-		if (__try_evict_buffer(b, 0))
-			count--;
-
-		cond_resched();
-	}
+	count = cache_total(&c->cache);
+	if (count > retain)
+		__evict_many(c, &params, LIST_CLEAN, count - retain);
 
 	dm_bufio_unlock(c);
 }
 
-static void do_global_cleanup(struct work_struct *w)
+static void cleanup_old_buffers(void)
 {
-	struct dm_bufio_client *locked_client = NULL;
-	struct dm_bufio_client *current_client;
-	struct dm_buffer *b;
-	unsigned int spinlock_hold_count;
-	unsigned long threshold = dm_bufio_cache_size -
-		dm_bufio_cache_size / DM_BUFIO_LOW_WATERMARK_RATIO;
-	unsigned long loops = global_num * 2;
+	unsigned long max_age_hz = get_max_age_hz();
+	struct dm_bufio_client *c;
 
 	mutex_lock(&dm_bufio_clients_lock);
 
-	while (1) {
-		cond_resched();
+	__cache_size_refresh();
 
-		spin_lock(&global_spinlock);
-		if (unlikely(dm_bufio_current_allocated <= threshold))
-			break;
+	list_for_each_entry(c, &dm_bufio_all_clients, client_list)
+		evict_old_buffers(c, max_age_hz);
 
-		spinlock_hold_count = 0;
-get_next:
-		if (!loops--)
-			break;
-		if (unlikely(list_empty(&global_queue)))
-			break;
-		b = list_entry(global_queue.prev, struct dm_buffer, global_list);
-
-		if (b->accessed) {
-			b->accessed = 0;
-			list_move(&b->global_list, &global_queue);
-			if (likely(++spinlock_hold_count < 16))
-				goto get_next;
-			spin_unlock(&global_spinlock);
-			continue;
-		}
+	mutex_unlock(&dm_bufio_clients_lock);
+}
 
-		current_client = b->c;
-		if (unlikely(current_client != locked_client)) {
-			if (locked_client)
-				dm_bufio_unlock(locked_client);
+static void work_fn(struct work_struct *w)
+{
+	cleanup_old_buffers();
 
-			if (!dm_bufio_trylock(current_client)) {
-				spin_unlock(&global_spinlock);
-				dm_bufio_lock(current_client);
-				locked_client = current_client;
-				continue;
-			}
+	queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
+			   DM_BUFIO_WORK_TIMER_SECS * HZ);
+}
 
-			locked_client = current_client;
-		}
+/*--------------------------------------------------------------*/
 
-		spin_unlock(&global_spinlock);
+/*
+ * Global cleanup tries to evict the oldest buffers from across _all_
+ * the clients.  It does this by repeatedly evicting a few buffers from
+ * the client that holds the oldest buffer.  It's approximate, but hopefully
+ * good enough.
+ */
+static struct dm_bufio_client *__pop_client(void)
+{
+	struct list_head *h;
 
-		if (unlikely(!__try_evict_buffer(b, GFP_KERNEL))) {
-			spin_lock(&global_spinlock);
-			list_move(&b->global_list, &global_queue);
-			spin_unlock(&global_spinlock);
-		}
+	if (list_empty(&dm_bufio_all_clients))
+		return NULL;
+
+	h = dm_bufio_all_clients.next;
+	list_del(h);
+	return container_of(h, struct dm_bufio_client, client_list);
+}
+
+/*
+ * Inserts the client in the global client list based on its
+ * 'oldest_buffer' field.
+ */
+static void __insert_client(struct dm_bufio_client *new_client)
+{
+	struct dm_bufio_client *c;
+	struct list_head *h = dm_bufio_all_clients.next;
+
+	while (h != &dm_bufio_all_clients) {
+		c = container_of(h, struct dm_bufio_client, client_list);
+		if (time_after_eq(c->oldest_buffer, new_client->oldest_buffer))
+			break;
+		h = h->next;
 	}
 
-	spin_unlock(&global_spinlock);
+	list_add_tail(&new_client->client_list, h);
+}
 
-	if (locked_client)
-		dm_bufio_unlock(locked_client);
+static unsigned long __evict_a_few(unsigned long nr_buffers)
+{
+	unsigned long count;
+	struct dm_bufio_client *c;
+	struct evict_params params = {
+		.gfp = GFP_KERNEL,
+		.age_hz = 0,
+		/* set to jiffies in case there are no buffers in this client */
+		.last_accessed = jiffies
+	};
 
-	mutex_unlock(&dm_bufio_clients_lock);
+	c = __pop_client();
+	if (!c)
+		return 0;
+
+	dm_bufio_lock(c);
+	count = __evict_many(c, &params, LIST_CLEAN, nr_buffers);
+	dm_bufio_unlock(c);
+
+	if (count)
+		c->oldest_buffer = params.last_accessed;
+	__insert_client(c);
+
+	return count;
 }
 
-static void cleanup_old_buffers(void)
+static void check_watermarks(void)
 {
-	unsigned long max_age_hz = get_max_age_hz();
+	LIST_HEAD(write_list);
 	struct dm_bufio_client *c;
 
 	mutex_lock(&dm_bufio_clients_lock);
+	list_for_each_entry(c, &dm_bufio_all_clients, client_list) {
+		dm_bufio_lock(c);
+		__check_watermark(c, &write_list);
+		dm_bufio_unlock(c);
+	}
+	mutex_unlock(&dm_bufio_clients_lock);
 
-	__cache_size_refresh();
+	__flush_write_list(&write_list);
+}
 
-	list_for_each_entry(c, &dm_bufio_all_clients, client_list)
-		__evict_old_buffers(c, max_age_hz);
+static void evict_old(void)
+{
+	unsigned long threshold = dm_bufio_cache_size -
+		dm_bufio_cache_size / DM_BUFIO_LOW_WATERMARK_RATIO;
 
+	mutex_lock(&dm_bufio_clients_lock);
+	while (dm_bufio_current_allocated > threshold) {
+		if (!__evict_a_few(64))
+			break;
+		cond_resched();
+	}
 	mutex_unlock(&dm_bufio_clients_lock);
 }
 
-static void work_fn(struct work_struct *w)
+static void do_global_cleanup(struct work_struct *w)
 {
-	cleanup_old_buffers();
-
-	queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
-			   DM_BUFIO_WORK_TIMER_SECS * HZ);
+	check_watermarks();
+	evict_old();
 }
 
 /*
@@ -2159,7 +2905,7 @@ static void __exit dm_bufio_exit(void)
 		bug = 1;
 	}
 
-	BUG_ON(bug);
+	WARN_ON(bug); /* leaks are not worth crashing the system */
 }
 
 module_init(dm_bufio_init)
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index dbbcfa580078..872896218550 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -3459,7 +3459,6 @@ static int __init dm_cache_init(void)
 
 	r = dm_register_target(&cache_target);
 	if (r) {
-		DMERR("cache target registration failed: %d", r);
 		kmem_cache_destroy(migration_cache);
 		return r;
 	}
diff --git a/drivers/md/dm-clone-target.c b/drivers/md/dm-clone-target.c
index f38a27604c7a..f467cdb5a022 100644
--- a/drivers/md/dm-clone-target.c
+++ b/drivers/md/dm-clone-target.c
@@ -2204,7 +2204,7 @@ static int __init dm_clone_init(void)
 
 	r = dm_register_target(&clone_target);
 	if (r < 0) {
-		DMERR("Failed to register clone target");
+		kmem_cache_destroy(_hydration_cache);
 		return r;
 	}
 
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 3ba53dc3cc3f..8b47b913ee83 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -3659,25 +3659,7 @@ static struct target_type crypt_target = {
 	.iterate_devices = crypt_iterate_devices,
 	.io_hints = crypt_io_hints,
 };
-
-static int __init dm_crypt_init(void)
-{
-	int r;
-
-	r = dm_register_target(&crypt_target);
-	if (r < 0)
-		DMERR("register failed %d", r);
-
-	return r;
-}
-
-static void __exit dm_crypt_exit(void)
-{
-	dm_unregister_target(&crypt_target);
-}
-
-module_init(dm_crypt_init);
-module_exit(dm_crypt_exit);
+module_dm(crypt);
 
 MODULE_AUTHOR("Jana Saout <jana@saout.de>");
 MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
diff --git a/drivers/md/dm-delay.c b/drivers/md/dm-delay.c
index a425046f88c7..7433525e5985 100644
--- a/drivers/md/dm-delay.c
+++ b/drivers/md/dm-delay.c
@@ -367,31 +367,7 @@ static struct target_type delay_target = {
 	.status	     = delay_status,
 	.iterate_devices = delay_iterate_devices,
 };
-
-static int __init dm_delay_init(void)
-{
-	int r;
-
-	r = dm_register_target(&delay_target);
-	if (r < 0) {
-		DMERR("register failed %d", r);
-		goto bad_register;
-	}
-
-	return 0;
-
-bad_register:
-	return r;
-}
-
-static void __exit dm_delay_exit(void)
-{
-	dm_unregister_target(&delay_target);
-}
-
-/* Module hooks */
-module_init(dm_delay_init);
-module_exit(dm_delay_exit);
+module_dm(delay);
 
 MODULE_DESCRIPTION(DM_NAME " delay target");
 MODULE_AUTHOR("Heinz Mauelshagen <mauelshagen@redhat.com>");
diff --git a/drivers/md/dm-dust.c b/drivers/md/dm-dust.c
index 7ae9936752de..12a377e06d02 100644
--- a/drivers/md/dm-dust.c
+++ b/drivers/md/dm-dust.c
@@ -570,24 +570,7 @@ static struct target_type dust_target = {
 	.status = dust_status,
 	.prepare_ioctl = dust_prepare_ioctl,
 };
-
-static int __init dm_dust_init(void)
-{
-	int r = dm_register_target(&dust_target);
-
-	if (r < 0)
-		DMERR("dm_register_target failed %d", r);
-
-	return r;
-}
-
-static void __exit dm_dust_exit(void)
-{
-	dm_unregister_target(&dust_target);
-}
-
-module_init(dm_dust_init);
-module_exit(dm_dust_exit);
+module_dm(dust);
 
 MODULE_DESCRIPTION(DM_NAME " dust test target");
 MODULE_AUTHOR("Bryan Gurney <dm-devel@redhat.com>");
diff --git a/drivers/md/dm-ebs-target.c b/drivers/md/dm-ebs-target.c
index b1068a68bc46..435b45201f4d 100644
--- a/drivers/md/dm-ebs-target.c
+++ b/drivers/md/dm-ebs-target.c
@@ -452,24 +452,7 @@ static struct target_type ebs_target = {
 	.prepare_ioctl	 = ebs_prepare_ioctl,
 	.iterate_devices = ebs_iterate_devices,
 };
-
-static int __init dm_ebs_init(void)
-{
-	int r = dm_register_target(&ebs_target);
-
-	if (r < 0)
-		DMERR("register failed %d", r);
-
-	return r;
-}
-
-static void dm_ebs_exit(void)
-{
-	dm_unregister_target(&ebs_target);
-}
-
-module_init(dm_ebs_init);
-module_exit(dm_ebs_exit);
+module_dm(ebs);
 
 MODULE_AUTHOR("Heinz Mauelshagen <dm-devel@redhat.com>");
 MODULE_DESCRIPTION(DM_NAME " emulated block size target");
diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c
index c2e7780cdd2d..0d70914217ee 100644
--- a/drivers/md/dm-era-target.c
+++ b/drivers/md/dm-era-target.c
@@ -1753,27 +1753,7 @@ static struct target_type era_target = {
 	.iterate_devices = era_iterate_devices,
 	.io_hints = era_io_hints
 };
-
-static int __init dm_era_init(void)
-{
-	int r;
-
-	r = dm_register_target(&era_target);
-	if (r) {
-		DMERR("era target registration failed: %d", r);
-		return r;
-	}
-
-	return 0;
-}
-
-static void __exit dm_era_exit(void)
-{
-	dm_unregister_target(&era_target);
-}
-
-module_init(dm_era_init);
-module_exit(dm_era_exit);
+module_dm(era);
 
 MODULE_DESCRIPTION(DM_NAME " era target");
 MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c
index 5b7556d2a9d9..bd80bcafbe50 100644
--- a/drivers/md/dm-flakey.c
+++ b/drivers/md/dm-flakey.c
@@ -37,6 +37,7 @@ struct flakey_c {
 };
 
 enum feature_flag_bits {
+	ERROR_READS,
 	DROP_WRITES,
 	ERROR_WRITES
 };
@@ -53,7 +54,7 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
 	const char *arg_name;
 
 	static const struct dm_arg _args[] = {
-		{0, 6, "Invalid number of feature args"},
+		{0, 7, "Invalid number of feature args"},
 		{1, UINT_MAX, "Invalid corrupt bio byte"},
 		{0, 255, "Invalid corrupt value to write into bio byte (0-255)"},
 		{0, UINT_MAX, "Invalid corrupt bio flags mask"},
@@ -77,6 +78,17 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
 		}
 
 		/*
+		 * error_reads
+		 */
+		if (!strcasecmp(arg_name, "error_reads")) {
+			if (test_and_set_bit(ERROR_READS, &fc->flags)) {
+				ti->error = "Feature error_reads duplicated";
+				return -EINVAL;
+			}
+			continue;
+		}
+
+		/*
 		 * drop_writes
 		 */
 		if (!strcasecmp(arg_name, "drop_writes")) {
@@ -125,9 +137,9 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
 			 * Direction r or w?
 			 */
 			arg_name = dm_shift_arg(as);
-			if (!strcasecmp(arg_name, "w"))
+			if (arg_name && !strcasecmp(arg_name, "w"))
 				fc->corrupt_bio_rw = WRITE;
-			else if (!strcasecmp(arg_name, "r"))
+			else if (arg_name && !strcasecmp(arg_name, "r"))
 				fc->corrupt_bio_rw = READ;
 			else {
 				ti->error = "Invalid corrupt bio direction (r or w)";
@@ -171,6 +183,12 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
 		return -EINVAL;
 	}
 
+	if (!fc->corrupt_bio_byte && !test_bit(ERROR_READS, &fc->flags) &&
+	    !test_bit(DROP_WRITES, &fc->flags) && !test_bit(ERROR_WRITES, &fc->flags)) {
+		set_bit(ERROR_WRITES, &fc->flags);
+		set_bit(ERROR_READS, &fc->flags);
+	}
+
 	return 0;
 }
 
@@ -346,8 +364,7 @@ static int flakey_map(struct dm_target *ti, struct bio *bio)
 		 * Otherwise, flakey_end_io() will decide if the reads should be modified.
 		 */
 		if (bio_data_dir(bio) == READ) {
-			if (!fc->corrupt_bio_byte && !test_bit(DROP_WRITES, &fc->flags) &&
-			    !test_bit(ERROR_WRITES, &fc->flags))
+			if (test_bit(ERROR_READS, &fc->flags))
 				return DM_MAPIO_KILL;
 			goto map_bio;
 		}
@@ -373,11 +390,6 @@ static int flakey_map(struct dm_target *ti, struct bio *bio)
 			}
 			goto map_bio;
 		}
-
-		/*
-		 * By default, error all I/O.
-		 */
-		return DM_MAPIO_KILL;
 	}
 
 map_bio:
@@ -404,8 +416,8 @@ static int flakey_end_io(struct dm_target *ti, struct bio *bio,
 				 */
 				corrupt_bio_data(bio, fc);
 			}
-		} else if (!test_bit(DROP_WRITES, &fc->flags) &&
-			   !test_bit(ERROR_WRITES, &fc->flags)) {
+		}
+		if (test_bit(ERROR_READS, &fc->flags)) {
 			/*
 			 * Error read during the down_interval if drop_writes
 			 * and error_writes were not configured.
@@ -422,7 +434,7 @@ static void flakey_status(struct dm_target *ti, status_type_t type,
 {
 	unsigned int sz = 0;
 	struct flakey_c *fc = ti->private;
-	unsigned int drop_writes, error_writes;
+	unsigned int error_reads, drop_writes, error_writes;
 
 	switch (type) {
 	case STATUSTYPE_INFO:
@@ -430,21 +442,24 @@ static void flakey_status(struct dm_target *ti, status_type_t type,
 		break;
 
 	case STATUSTYPE_TABLE:
-		DMEMIT("%s %llu %u %u ", fc->dev->name,
+		DMEMIT("%s %llu %u %u", fc->dev->name,
 		       (unsigned long long)fc->start, fc->up_interval,
 		       fc->down_interval);
 
+		error_reads = test_bit(ERROR_READS, &fc->flags);
 		drop_writes = test_bit(DROP_WRITES, &fc->flags);
 		error_writes = test_bit(ERROR_WRITES, &fc->flags);
-		DMEMIT("%u ", drop_writes + error_writes + (fc->corrupt_bio_byte > 0) * 5);
+		DMEMIT(" %u", error_reads + drop_writes + error_writes + (fc->corrupt_bio_byte > 0) * 5);
 
+		if (error_reads)
+			DMEMIT(" error_reads");
 		if (drop_writes)
-			DMEMIT("drop_writes ");
+			DMEMIT(" drop_writes");
 		else if (error_writes)
-			DMEMIT("error_writes ");
+			DMEMIT(" error_writes");
 
 		if (fc->corrupt_bio_byte)
-			DMEMIT("corrupt_bio_byte %u %c %u %u ",
+			DMEMIT(" corrupt_bio_byte %u %c %u %u",
 			       fc->corrupt_bio_byte,
 			       (fc->corrupt_bio_rw == WRITE) ? 'w' : 'r',
 			       fc->corrupt_bio_value, fc->corrupt_bio_flags);
@@ -506,25 +521,7 @@ static struct target_type flakey_target = {
 	.prepare_ioctl = flakey_prepare_ioctl,
 	.iterate_devices = flakey_iterate_devices,
 };
-
-static int __init dm_flakey_init(void)
-{
-	int r = dm_register_target(&flakey_target);
-
-	if (r < 0)
-		DMERR("register failed %d", r);
-
-	return r;
-}
-
-static void __exit dm_flakey_exit(void)
-{
-	dm_unregister_target(&flakey_target);
-}
-
-/* Module hooks */
-module_init(dm_flakey_init);
-module_exit(dm_flakey_exit);
+module_dm(flakey);
 
 MODULE_DESCRIPTION(DM_NAME " flakey target");
 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index b0d5057fbdd9..31838b13ea54 100644
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -3118,7 +3118,7 @@ static int dm_integrity_reboot(struct notifier_block *n, unsigned long code, voi
 
 static void dm_integrity_postsuspend(struct dm_target *ti)
 {
-	struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
+	struct dm_integrity_c *ic = ti->private;
 	int r;
 
 	WARN_ON(unregister_reboot_notifier(&ic->reboot_notifier));
@@ -3167,7 +3167,7 @@ static void dm_integrity_postsuspend(struct dm_target *ti)
 
 static void dm_integrity_resume(struct dm_target *ti)
 {
-	struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
+	struct dm_integrity_c *ic = ti->private;
 	__u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors);
 	int r;
 
@@ -3290,7 +3290,7 @@ static void dm_integrity_resume(struct dm_target *ti)
 static void dm_integrity_status(struct dm_target *ti, status_type_t type,
 				unsigned int status_flags, char *result, unsigned int maxlen)
 {
-	struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
+	struct dm_integrity_c *ic = ti->private;
 	unsigned int arg_count;
 	size_t sz = 0;
 
@@ -4703,11 +4703,12 @@ static int __init dm_integrity_init(void)
 	}
 
 	r = dm_register_target(&integrity_target);
+	if (r < 0) {
+		kmem_cache_destroy(journal_io_cache);
+		return r;
+	}
 
-	if (r < 0)
-		DMERR("register failed %d", r);
-
-	return r;
+	return 0;
 }
 
 static void __exit dm_integrity_exit(void)
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index dc2df76999b0..f053ce245814 100644
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -187,7 +187,7 @@ static void list_get_page(struct dpages *dp,
 		  struct page **p, unsigned long *len, unsigned int *offset)
 {
 	unsigned int o = dp->context_u;
-	struct page_list *pl = (struct page_list *) dp->context_ptr;
+	struct page_list *pl = dp->context_ptr;
 
 	*p = pl->page;
 	*len = PAGE_SIZE - o;
@@ -196,7 +196,7 @@ static void list_get_page(struct dpages *dp,
 
 static void list_next_page(struct dpages *dp)
 {
-	struct page_list *pl = (struct page_list *) dp->context_ptr;
+	struct page_list *pl = dp->context_ptr;
 
 	dp->context_ptr = pl->next;
 	dp->context_u = 0;
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index 50a1259294d1..cc77cf3d4109 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -1168,10 +1168,13 @@ static int do_resume(struct dm_ioctl *param)
 	/* Do we need to load a new map ? */
 	if (new_map) {
 		sector_t old_size, new_size;
+		int srcu_idx;
 
 		/* Suspend if it isn't already suspended */
-		if (param->flags & DM_SKIP_LOCKFS_FLAG)
+		old_map = dm_get_live_table(md, &srcu_idx);
+		if ((param->flags & DM_SKIP_LOCKFS_FLAG) || !old_map)
 			suspend_flags &= ~DM_SUSPEND_LOCKFS_FLAG;
+		dm_put_live_table(md, srcu_idx);
 		if (param->flags & DM_NOFLUSH_FLAG)
 			suspend_flags |= DM_SUSPEND_NOFLUSH_FLAG;
 		if (!dm_suspended_md(md))
@@ -1556,11 +1559,12 @@ static int table_clear(struct file *filp, struct dm_ioctl *param, size_t param_s
 		has_new_map = true;
 	}
 
-	param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
-
-	__dev_status(hc->md, param);
 	md = hc->md;
 	up_write(&_hash_lock);
+
+	param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
+	__dev_status(md, param);
+
 	if (old_map) {
 		dm_sync_table(md);
 		dm_table_destroy(old_map);
diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index a158c6e5fbd7..d01807c50f20 100644
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -519,7 +519,7 @@ static int run_complete_job(struct kcopyd_job *job)
 
 static void complete_io(unsigned long error, void *context)
 {
-	struct kcopyd_job *job = (struct kcopyd_job *) context;
+	struct kcopyd_job *job = context;
 	struct dm_kcopyd_client *kc = job->kc;
 
 	io_job_finish(kc->throttle);
@@ -696,7 +696,7 @@ static void segment_complete(int read_err, unsigned long write_err,
 	/* FIXME: tidy this function */
 	sector_t progress = 0;
 	sector_t count = 0;
-	struct kcopyd_job *sub_job = (struct kcopyd_job *) context;
+	struct kcopyd_job *sub_job = context;
 	struct kcopyd_job *job = sub_job->master_job;
 	struct dm_kcopyd_client *kc = job->kc;
 
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c
index 3e622dcc9dbd..f4448d520ee9 100644
--- a/drivers/md/dm-linear.c
+++ b/drivers/md/dm-linear.c
@@ -72,7 +72,7 @@ bad:
 
 static void linear_dtr(struct dm_target *ti)
 {
-	struct linear_c *lc = (struct linear_c *) ti->private;
+	struct linear_c *lc = ti->private;
 
 	dm_put_device(ti, lc->dev);
 	kfree(lc);
@@ -98,7 +98,7 @@ static int linear_map(struct dm_target *ti, struct bio *bio)
 static void linear_status(struct dm_target *ti, status_type_t type,
 			  unsigned int status_flags, char *result, unsigned int maxlen)
 {
-	struct linear_c *lc = (struct linear_c *) ti->private;
+	struct linear_c *lc = ti->private;
 	size_t sz = 0;
 
 	switch (type) {
@@ -120,7 +120,7 @@ static void linear_status(struct dm_target *ti, status_type_t type,
 
 static int linear_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)
 {
-	struct linear_c *lc = (struct linear_c *) ti->private;
+	struct linear_c *lc = ti->private;
 	struct dm_dev *dev = lc->dev;
 
 	*bdev = dev->bdev;
diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c
index cbd0f81f4a35..f17a6cf2284e 100644
--- a/drivers/md/dm-log-writes.c
+++ b/drivers/md/dm-log-writes.c
@@ -429,7 +429,7 @@ static inline sector_t logdev_last_sector(struct log_writes_c *lc)
 
 static int log_writes_kthread(void *arg)
 {
-	struct log_writes_c *lc = (struct log_writes_c *)arg;
+	struct log_writes_c *lc = arg;
 	sector_t sector = 0;
 
 	while (!kthread_should_stop()) {
@@ -937,24 +937,7 @@ static struct target_type log_writes_target = {
 	.dax_zero_page_range = log_writes_dax_zero_page_range,
 	.dax_recovery_write = log_writes_dax_recovery_write,
 };
-
-static int __init dm_log_writes_init(void)
-{
-	int r = dm_register_target(&log_writes_target);
-
-	if (r < 0)
-		DMERR("register failed %d", r);
-
-	return r;
-}
-
-static void __exit dm_log_writes_exit(void)
-{
-	dm_unregister_target(&log_writes_target);
-}
-
-module_init(dm_log_writes_init);
-module_exit(dm_log_writes_exit);
+module_dm(log_writes);
 
 MODULE_DESCRIPTION(DM_NAME " log writes target");
 MODULE_AUTHOR("Josef Bacik <jbacik@fb.com>");
diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c
index afd94d2e7295..f9f84236dfcd 100644
--- a/drivers/md/dm-log.c
+++ b/drivers/md/dm-log.c
@@ -530,7 +530,7 @@ static void destroy_log_context(struct log_c *lc)
 
 static void core_dtr(struct dm_dirty_log *log)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	vfree(lc->clean_bits);
 	destroy_log_context(lc);
@@ -569,7 +569,7 @@ static int disk_ctr(struct dm_dirty_log *log, struct dm_target *ti,
 
 static void disk_dtr(struct dm_dirty_log *log)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	dm_put_device(lc->ti, lc->log_dev);
 	vfree(lc->disk_header);
@@ -590,7 +590,7 @@ static int disk_resume(struct dm_dirty_log *log)
 {
 	int r;
 	unsigned int i;
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 	size_t size = lc->bitset_uint32_count * sizeof(uint32_t);
 
 	/* read the disk header */
@@ -652,14 +652,14 @@ static int disk_resume(struct dm_dirty_log *log)
 
 static uint32_t core_get_region_size(struct dm_dirty_log *log)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	return lc->region_size;
 }
 
 static int core_resume(struct dm_dirty_log *log)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	lc->sync_search = 0;
 	return 0;
@@ -667,14 +667,14 @@ static int core_resume(struct dm_dirty_log *log)
 
 static int core_is_clean(struct dm_dirty_log *log, region_t region)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	return log_test_bit(lc->clean_bits, region);
 }
 
 static int core_in_sync(struct dm_dirty_log *log, region_t region, int block)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	return log_test_bit(lc->sync_bits, region);
 }
@@ -727,14 +727,14 @@ static int disk_flush(struct dm_dirty_log *log)
 
 static void core_mark_region(struct dm_dirty_log *log, region_t region)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	log_clear_bit(lc, lc->clean_bits, region);
 }
 
 static void core_clear_region(struct dm_dirty_log *log, region_t region)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	if (likely(!lc->flush_failed))
 		log_set_bit(lc, lc->clean_bits, region);
@@ -742,7 +742,7 @@ static void core_clear_region(struct dm_dirty_log *log, region_t region)
 
 static int core_get_resync_work(struct dm_dirty_log *log, region_t *region)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	if (lc->sync_search >= lc->region_count)
 		return 0;
@@ -765,7 +765,7 @@ static int core_get_resync_work(struct dm_dirty_log *log, region_t *region)
 static void core_set_region_sync(struct dm_dirty_log *log, region_t region,
 				 int in_sync)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	log_clear_bit(lc, lc->recovering_bits, region);
 	if (in_sync) {
@@ -779,7 +779,7 @@ static void core_set_region_sync(struct dm_dirty_log *log, region_t region,
 
 static region_t core_get_sync_count(struct dm_dirty_log *log)
 {
-	struct log_c *lc = (struct log_c *) log->context;
+	struct log_c *lc = log->context;
 
 	return lc->sync_count;
 }
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index 61ab1a8d2c9c..bea3cda9938e 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -2235,11 +2235,8 @@ static int __init dm_multipath_init(void)
 	}
 
 	r = dm_register_target(&multipath_target);
-	if (r < 0) {
-		DMERR("request-based register failed %d", r);
-		r = -EINVAL;
+	if (r < 0)
 		goto bad_register_target;
-	}
 
 	return 0;
 
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c
index 60632b409b80..c8821fcb8299 100644
--- a/drivers/md/dm-raid.c
+++ b/drivers/md/dm-raid.c
@@ -2209,7 +2209,6 @@ static int super_load(struct md_rdev *rdev, struct md_rdev *refdev)
 static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev)
 {
 	int role;
-	unsigned int d;
 	struct mddev *mddev = &rs->md;
 	uint64_t events_sb;
 	uint64_t failed_devices[DISKS_ARRAY_ELEMS];
@@ -2324,7 +2323,6 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev)
 	 *    to provide capacity for redundancy or during reshape
 	 *    to add capacity to grow the raid set.
 	 */
-	d = 0;
 	rdev_for_each(r, mddev) {
 		if (test_bit(Journal, &rdev->flags))
 			continue;
@@ -2340,8 +2338,6 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev)
 			if (test_bit(FirstUse, &r->flags))
 				rebuild_and_new++;
 		}
-
-		d++;
 	}
 
 	if (new_devs == rs->raid_disks || !rebuilds) {
@@ -4081,23 +4077,7 @@ static struct target_type raid_target = {
 	.preresume = raid_preresume,
 	.resume = raid_resume,
 };
-
-static int __init dm_raid_init(void)
-{
-	DMINFO("Loading target version %u.%u.%u",
-	       raid_target.version[0],
-	       raid_target.version[1],
-	       raid_target.version[2]);
-	return dm_register_target(&raid_target);
-}
-
-static void __exit dm_raid_exit(void)
-{
-	dm_unregister_target(&raid_target);
-}
-
-module_init(dm_raid_init);
-module_exit(dm_raid_exit);
+module_dm(raid);
 
 module_param(devices_handle_discard_safely, bool, 0644);
 MODULE_PARM_DESC(devices_handle_discard_safely,
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index bc417a5e5b89..ddcb2bc4a617 100644
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -604,7 +604,7 @@ static void do_reads(struct mirror_set *ms, struct bio_list *reads)
 static void write_callback(unsigned long error, void *context)
 {
 	unsigned int i;
-	struct bio *bio = (struct bio *) context;
+	struct bio *bio = context;
 	struct mirror_set *ms;
 	int should_wake = 0;
 	unsigned long flags;
@@ -1180,7 +1180,7 @@ err_free_context:
 
 static void mirror_dtr(struct dm_target *ti)
 {
-	struct mirror_set *ms = (struct mirror_set *) ti->private;
+	struct mirror_set *ms = ti->private;
 
 	del_timer_sync(&ms->timer);
 	flush_workqueue(ms->kmirrord_wq);
@@ -1246,7 +1246,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio,
 		blk_status_t *error)
 {
 	int rw = bio_data_dir(bio);
-	struct mirror_set *ms = (struct mirror_set *) ti->private;
+	struct mirror_set *ms = ti->private;
 	struct mirror *m = NULL;
 	struct dm_bio_details *bd = NULL;
 	struct dm_raid1_bio_record *bio_record =
@@ -1311,7 +1311,7 @@ out:
 
 static void mirror_presuspend(struct dm_target *ti)
 {
-	struct mirror_set *ms = (struct mirror_set *) ti->private;
+	struct mirror_set *ms = ti->private;
 	struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
 
 	struct bio_list holds;
@@ -1407,7 +1407,7 @@ static void mirror_status(struct dm_target *ti, status_type_t type,
 {
 	unsigned int m, sz = 0;
 	int num_feature_args = 0;
-	struct mirror_set *ms = (struct mirror_set *) ti->private;
+	struct mirror_set *ms = ti->private;
 	struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
 	char buffer[MAX_NR_MIRRORS + 1];
 
@@ -1498,23 +1498,21 @@ static struct target_type mirror_target = {
 
 static int __init dm_mirror_init(void)
 {
-	int r = -ENOMEM;
+	int r;
 
 	dm_raid1_wq = alloc_workqueue("dm_raid1_wq", 0, 0);
-	if (!dm_raid1_wq)
-		goto bad_target;
+	if (!dm_raid1_wq) {
+		DMERR("Failed to alloc workqueue");
+		return -ENOMEM;
+	}
 
 	r = dm_register_target(&mirror_target);
 	if (r < 0) {
 		destroy_workqueue(dm_raid1_wq);
-		goto bad_target;
+		return r;
 	}
 
 	return 0;
-
-bad_target:
-	DMERR("Failed to register mirror target");
-	return r;
 }
 
 static void __exit dm_mirror_exit(void)
diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c
index f14e5df27874..15649921f2a9 100644
--- a/drivers/md/dm-snap-persistent.c
+++ b/drivers/md/dm-snap-persistent.c
@@ -567,7 +567,7 @@ ret_destroy_bufio:
 
 static struct pstore *get_info(struct dm_exception_store *store)
 {
-	return (struct pstore *) store->context;
+	return store->context;
 }
 
 static void persistent_usage(struct dm_exception_store *store,
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index f766c21408f1..9c49f53760d0 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -2815,22 +2815,16 @@ static int __init dm_snapshot_init(void)
 	}
 
 	r = dm_register_target(&snapshot_target);
-	if (r < 0) {
-		DMERR("snapshot target register failed %d", r);
+	if (r < 0)
 		goto bad_register_snapshot_target;
-	}
 
 	r = dm_register_target(&origin_target);
-	if (r < 0) {
-		DMERR("Origin target register failed %d", r);
+	if (r < 0)
 		goto bad_register_origin_target;
-	}
 
 	r = dm_register_target(&merge_target);
-	if (r < 0) {
-		DMERR("Merge target register failed %d", r);
+	if (r < 0)
 		goto bad_register_merge_target;
-	}
 
 	return 0;
 
diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c
index 8d6951157106..e2854a3cbd28 100644
--- a/drivers/md/dm-stripe.c
+++ b/drivers/md/dm-stripe.c
@@ -189,7 +189,7 @@ static int stripe_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 static void stripe_dtr(struct dm_target *ti)
 {
 	unsigned int i;
-	struct stripe_c *sc = (struct stripe_c *) ti->private;
+	struct stripe_c *sc = ti->private;
 
 	for (i = 0; i < sc->stripes; i++)
 		dm_put_device(ti, sc->stripe[i].dev);
@@ -360,7 +360,7 @@ static size_t stripe_dax_recovery_write(struct dm_target *ti, pgoff_t pgoff,
 static void stripe_status(struct dm_target *ti, status_type_t type,
 			  unsigned int status_flags, char *result, unsigned int maxlen)
 {
-	struct stripe_c *sc = (struct stripe_c *) ti->private;
+	struct stripe_c *sc = ti->private;
 	unsigned int sz = 0;
 	unsigned int i;
 
diff --git a/drivers/md/dm-switch.c b/drivers/md/dm-switch.c
index ee2432927e90..dfd9fb52a6f3 100644
--- a/drivers/md/dm-switch.c
+++ b/drivers/md/dm-switch.c
@@ -565,25 +565,7 @@ static struct target_type switch_target = {
 	.prepare_ioctl = switch_prepare_ioctl,
 	.iterate_devices = switch_iterate_devices,
 };
-
-static int __init dm_switch_init(void)
-{
-	int r;
-
-	r = dm_register_target(&switch_target);
-	if (r < 0)
-		DMERR("dm_register_target() failed %d", r);
-
-	return r;
-}
-
-static void __exit dm_switch_exit(void)
-{
-	dm_unregister_target(&switch_target);
-}
-
-module_init(dm_switch_init);
-module_exit(dm_switch_exit);
+module_dm(switch);
 
 MODULE_DESCRIPTION(DM_NAME " dynamic path switching target");
 MODULE_AUTHOR("Kevin D. O'Kelley <Kevin_OKelley@dell.com>");
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index 2055a758541d..1398f1d6e83e 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -1202,21 +1202,12 @@ struct dm_crypto_profile {
 	struct mapped_device *md;
 };
 
-struct dm_keyslot_evict_args {
-	const struct blk_crypto_key *key;
-	int err;
-};
-
 static int dm_keyslot_evict_callback(struct dm_target *ti, struct dm_dev *dev,
 				     sector_t start, sector_t len, void *data)
 {
-	struct dm_keyslot_evict_args *args = data;
-	int err;
+	const struct blk_crypto_key *key = data;
 
-	err = blk_crypto_evict_key(dev->bdev, args->key);
-	if (!args->err)
-		args->err = err;
-	/* Always try to evict the key from all devices. */
+	blk_crypto_evict_key(dev->bdev, key);
 	return 0;
 }
 
@@ -1229,7 +1220,6 @@ static int dm_keyslot_evict(struct blk_crypto_profile *profile,
 {
 	struct mapped_device *md =
 		container_of(profile, struct dm_crypto_profile, profile)->md;
-	struct dm_keyslot_evict_args args = { key };
 	struct dm_table *t;
 	int srcu_idx;
 
@@ -1242,11 +1232,12 @@ static int dm_keyslot_evict(struct blk_crypto_profile *profile,
 
 		if (!ti->type->iterate_devices)
 			continue;
-		ti->type->iterate_devices(ti, dm_keyslot_evict_callback, &args);
+		ti->type->iterate_devices(ti, dm_keyslot_evict_callback,
+					  (void *)key);
 	}
 
 	dm_put_live_table(md, srcu_idx);
-	return args.err;
+	return 0;
 }
 
 static int
@@ -1670,8 +1661,12 @@ int dm_calculate_queue_limits(struct dm_table *t,
 
 		blk_set_stacking_limits(&ti_limits);
 
-		if (!ti->type->iterate_devices)
+		if (!ti->type->iterate_devices) {
+			/* Set I/O hints portion of queue limits */
+			if (ti->type->io_hints)
+				ti->type->io_hints(ti, &ti_limits);
 			goto combine_limits;
+		}
 
 		/*
 		 * Combine queue limits of all the devices this target uses.
diff --git a/drivers/md/dm-target.c b/drivers/md/dm-target.c
index 26ea22b1a0d7..27e2992ff249 100644
--- a/drivers/md/dm-target.c
+++ b/drivers/md/dm-target.c
@@ -85,12 +85,15 @@ int dm_register_target(struct target_type *tt)
 	int rv = 0;
 
 	down_write(&_lock);
-	if (__find_target_type(tt->name))
+	if (__find_target_type(tt->name)) {
+		DMERR("%s: '%s' target already registered",
+		      __func__, tt->name);
 		rv = -EEXIST;
-	else
+	} else {
 		list_add(&tt->list, &_targets);
-
+	}
 	up_write(&_lock);
+
 	return rv;
 }
 EXPORT_SYMBOL(dm_register_target);
@@ -119,6 +122,7 @@ static int io_err_ctr(struct dm_target *tt, unsigned int argc, char **args)
 	 * Return error for discards instead of -EOPNOTSUPP
 	 */
 	tt->num_discard_bios = 1;
+	tt->discards_supported = true;
 
 	return 0;
 }
@@ -145,6 +149,13 @@ static void io_err_release_clone_rq(struct request *clone,
 {
 }
 
+static void io_err_io_hints(struct dm_target *ti, struct queue_limits *limits)
+{
+	limits->max_discard_sectors = UINT_MAX;
+	limits->max_hw_discard_sectors = UINT_MAX;
+	limits->discard_granularity = 512;
+}
+
 static long io_err_dax_direct_access(struct dm_target *ti, pgoff_t pgoff,
 		long nr_pages, enum dax_access_mode mode, void **kaddr,
 		pfn_t *pfn)
@@ -154,13 +165,14 @@ static long io_err_dax_direct_access(struct dm_target *ti, pgoff_t pgoff,
 
 static struct target_type error_target = {
 	.name = "error",
-	.version = {1, 5, 0},
+	.version = {1, 6, 0},
 	.features = DM_TARGET_WILDCARD,
 	.ctr  = io_err_ctr,
 	.dtr  = io_err_dtr,
 	.map  = io_err_map,
 	.clone_and_map_rq = io_err_clone_and_map_rq,
 	.release_clone_rq = io_err_release_clone_rq,
+	.io_hints = io_err_io_hints,
 	.direct_access = io_err_dax_direct_access,
 };
 
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index 13d4677baafd..2b13c949bd72 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -118,25 +118,27 @@ enum lock_space {
 	PHYSICAL
 };
 
-static void build_key(struct dm_thin_device *td, enum lock_space ls,
+static bool build_key(struct dm_thin_device *td, enum lock_space ls,
 		      dm_block_t b, dm_block_t e, struct dm_cell_key *key)
 {
 	key->virtual = (ls == VIRTUAL);
 	key->dev = dm_thin_dev_id(td);
 	key->block_begin = b;
 	key->block_end = e;
+
+	return dm_cell_key_has_valid_range(key);
 }
 
 static void build_data_key(struct dm_thin_device *td, dm_block_t b,
 			   struct dm_cell_key *key)
 {
-	build_key(td, PHYSICAL, b, b + 1llu, key);
+	(void) build_key(td, PHYSICAL, b, b + 1llu, key);
 }
 
 static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
 			      struct dm_cell_key *key)
 {
-	build_key(td, VIRTUAL, b, b + 1llu, key);
+	(void) build_key(td, VIRTUAL, b, b + 1llu, key);
 }
 
 /*----------------------------------------------------------------*/
@@ -883,15 +885,17 @@ static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *c
 {
 	struct pool *pool = tc->pool;
 	unsigned long flags;
-	int has_work;
+	struct bio_list bios;
 
-	spin_lock_irqsave(&tc->lock, flags);
-	cell_release_no_holder(pool, cell, &tc->deferred_bio_list);
-	has_work = !bio_list_empty(&tc->deferred_bio_list);
-	spin_unlock_irqrestore(&tc->lock, flags);
+	bio_list_init(&bios);
+	cell_release_no_holder(pool, cell, &bios);
 
-	if (has_work)
+	if (!bio_list_empty(&bios)) {
+		spin_lock_irqsave(&tc->lock, flags);
+		bio_list_merge(&tc->deferred_bio_list, &bios);
+		spin_unlock_irqrestore(&tc->lock, flags);
 		wake_worker(pool);
+	}
 }
 
 static void thin_defer_bio(struct thin_c *tc, struct bio *bio);
@@ -1672,54 +1676,70 @@ static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t
 	struct dm_cell_key data_key;
 	struct dm_bio_prison_cell *data_cell;
 	struct dm_thin_new_mapping *m;
-	dm_block_t virt_begin, virt_end, data_begin;
+	dm_block_t virt_begin, virt_end, data_begin, data_end;
+	dm_block_t len, next_boundary;
 
 	while (begin != end) {
-		r = ensure_next_mapping(pool);
-		if (r)
-			/* we did our best */
-			return;
-
 		r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end,
 					      &data_begin, &maybe_shared);
-		if (r)
+		if (r) {
 			/*
 			 * Silently fail, letting any mappings we've
 			 * created complete.
 			 */
 			break;
-
-		build_key(tc->td, PHYSICAL, data_begin, data_begin + (virt_end - virt_begin), &data_key);
-		if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) {
-			/* contention, we'll give up with this range */
-			begin = virt_end;
-			continue;
 		}
 
-		/*
-		 * IO may still be going to the destination block.  We must
-		 * quiesce before we can do the removal.
-		 */
-		m = get_next_mapping(pool);
-		m->tc = tc;
-		m->maybe_shared = maybe_shared;
-		m->virt_begin = virt_begin;
-		m->virt_end = virt_end;
-		m->data_block = data_begin;
-		m->cell = data_cell;
-		m->bio = bio;
+		data_end = data_begin + (virt_end - virt_begin);
 
 		/*
-		 * The parent bio must not complete before sub discard bios are
-		 * chained to it (see end_discard's bio_chain)!
-		 *
-		 * This per-mapping bi_remaining increment is paired with
-		 * the implicit decrement that occurs via bio_endio() in
-		 * end_discard().
+		 * Make sure the data region obeys the bio prison restrictions.
 		 */
-		bio_inc_remaining(bio);
-		if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
-			pool->process_prepared_discard(m);
+		while (data_begin < data_end) {
+			r = ensure_next_mapping(pool);
+			if (r)
+				return; /* we did our best */
+
+			next_boundary = ((data_begin >> BIO_PRISON_MAX_RANGE_SHIFT) + 1)
+				<< BIO_PRISON_MAX_RANGE_SHIFT;
+			len = min_t(sector_t, data_end - data_begin, next_boundary - data_begin);
+
+			/* This key is certainly within range given the above splitting */
+			(void) build_key(tc->td, PHYSICAL, data_begin, data_begin + len, &data_key);
+			if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) {
+				/* contention, we'll give up with this range */
+				data_begin += len;
+				continue;
+			}
+
+			/*
+			 * IO may still be going to the destination block.  We must
+			 * quiesce before we can do the removal.
+			 */
+			m = get_next_mapping(pool);
+			m->tc = tc;
+			m->maybe_shared = maybe_shared;
+			m->virt_begin = virt_begin;
+			m->virt_end = virt_begin + len;
+			m->data_block = data_begin;
+			m->cell = data_cell;
+			m->bio = bio;
+
+			/*
+			 * The parent bio must not complete before sub discard bios are
+			 * chained to it (see end_discard's bio_chain)!
+			 *
+			 * This per-mapping bi_remaining increment is paired with
+			 * the implicit decrement that occurs via bio_endio() in
+			 * end_discard().
+			 */
+			bio_inc_remaining(bio);
+			if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
+				pool->process_prepared_discard(m);
+
+			virt_begin += len;
+			data_begin += len;
+		}
 
 		begin = virt_end;
 	}
@@ -1761,8 +1781,13 @@ static void process_discard_bio(struct thin_c *tc, struct bio *bio)
 		return;
 	}
 
-	build_key(tc->td, VIRTUAL, begin, end, &virt_key);
-	if (bio_detain(tc->pool, &virt_key, bio, &virt_cell))
+	if (unlikely(!build_key(tc->td, VIRTUAL, begin, end, &virt_key))) {
+		DMERR_LIMIT("Discard doesn't respect bio prison limits");
+		bio_endio(bio);
+		return;
+	}
+
+	if (bio_detain(tc->pool, &virt_key, bio, &virt_cell)) {
 		/*
 		 * Potential starvation issue: We're relying on the
 		 * fs/application being well behaved, and not trying to
@@ -1771,6 +1796,7 @@ static void process_discard_bio(struct thin_c *tc, struct bio *bio)
 		 * cell will never be granted.
 		 */
 		return;
+	}
 
 	tc->pool->process_discard_cell(tc, virt_cell);
 }
@@ -3378,13 +3404,13 @@ static int pool_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	 */
 	if (pf.discard_enabled && pf.discard_passdown) {
 		ti->num_discard_bios = 1;
-
 		/*
 		 * Setting 'discards_supported' circumvents the normal
 		 * stacking of discard limits (this keeps the pool and
 		 * thin devices' discard limits consistent).
 		 */
 		ti->discards_supported = true;
+		ti->max_discard_granularity = true;
 	}
 	ti->private = pt;
 
@@ -4094,7 +4120,7 @@ static struct target_type pool_target = {
 	.name = "thin-pool",
 	.features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
 		    DM_TARGET_IMMUTABLE,
-	.version = {1, 22, 0},
+	.version = {1, 23, 0},
 	.module = THIS_MODULE,
 	.ctr = pool_ctr,
 	.dtr = pool_dtr,
@@ -4259,6 +4285,7 @@ static int thin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	if (tc->pool->pf.discard_enabled) {
 		ti->discards_supported = true;
 		ti->num_discard_bios = 1;
+		ti->max_discard_granularity = true;
 	}
 
 	mutex_unlock(&dm_thin_pool_table.mutex);
@@ -4474,12 +4501,12 @@ static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
 		return;
 
 	limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
-	limits->max_discard_sectors = 2048 * 1024 * 16; /* 16G */
+	limits->max_discard_sectors = pool->sectors_per_block * BIO_PRISON_MAX_RANGE;
 }
 
 static struct target_type thin_target = {
 	.name = "thin",
-	.version = {1, 22, 0},
+	.version = {1, 23, 0},
 	.module	= THIS_MODULE,
 	.ctr = thin_ctr,
 	.dtr = thin_dtr,
diff --git a/drivers/md/dm-unstripe.c b/drivers/md/dm-unstripe.c
index e7b7d5983a16..48587c16c445 100644
--- a/drivers/md/dm-unstripe.c
+++ b/drivers/md/dm-unstripe.c
@@ -192,19 +192,7 @@ static struct target_type unstripe_target = {
 	.iterate_devices = unstripe_iterate_devices,
 	.io_hints = unstripe_io_hints,
 };
-
-static int __init dm_unstripe_init(void)
-{
-	return dm_register_target(&unstripe_target);
-}
-
-static void __exit dm_unstripe_exit(void)
-{
-	dm_unregister_target(&unstripe_target);
-}
-
-module_init(dm_unstripe_init);
-module_exit(dm_unstripe_exit);
+module_dm(unstripe);
 
 MODULE_DESCRIPTION(DM_NAME " unstriped target");
 MODULE_ALIAS("dm-unstriped");
diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c
index 962fc32c947c..a9ee2faa75a2 100644
--- a/drivers/md/dm-verity-fec.c
+++ b/drivers/md/dm-verity-fec.c
@@ -567,14 +567,14 @@ out:
 
 static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data)
 {
-	struct dm_verity *v = (struct dm_verity *)pool_data;
+	struct dm_verity *v = pool_data;
 
 	return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask);
 }
 
 static void fec_rs_free(void *element, void *pool_data)
 {
-	struct rs_control *rs = (struct rs_control *)element;
+	struct rs_control *rs = element;
 
 	if (rs)
 		free_rs(rs);
diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index ade83ef3b439..e35c16e06d06 100644
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -16,6 +16,7 @@
 #include "dm-verity.h"
 #include "dm-verity-fec.h"
 #include "dm-verity-verify-sig.h"
+#include "dm-audit.h"
 #include <linux/module.h>
 #include <linux/reboot.h>
 #include <linux/scatterlist.h>
@@ -248,8 +249,10 @@ static int verity_handle_err(struct dm_verity *v, enum verity_block_type type,
 	DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name,
 		    type_str, block);
 
-	if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS)
+	if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) {
 		DMERR("%s: reached maximum errors", v->data_dev->name);
+		dm_audit_log_target(DM_MSG_PREFIX, "max-corrupted-errors", v->ti, 0);
+	}
 
 	snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu",
 		DM_VERITY_ENV_VAR_NAME, type, block);
@@ -340,6 +343,11 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
 		else if (verity_handle_err(v,
 					   DM_VERITY_BLOCK_TYPE_METADATA,
 					   hash_block)) {
+			struct bio *bio =
+				dm_bio_from_per_bio_data(io,
+							 v->ti->per_io_data_size);
+			dm_audit_log_bio(DM_MSG_PREFIX, "verify-metadata", bio,
+					 block, 0);
 			r = -EIO;
 			goto release_ret_r;
 		}
@@ -523,7 +531,7 @@ static int verity_verify_io(struct dm_verity_io *io)
 		sector_t cur_block = io->block + b;
 		struct ahash_request *req = verity_io_hash_req(v, io);
 
-		if (v->validated_blocks &&
+		if (v->validated_blocks && bio->bi_status == BLK_STS_OK &&
 		    likely(test_bit(cur_block, v->validated_blocks))) {
 			verity_bv_skip_block(v, io, iter);
 			continue;
@@ -590,8 +598,11 @@ static int verity_verify_io(struct dm_verity_io *io)
 				return -EIO;
 			}
 			if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA,
-					      cur_block))
+					      cur_block)) {
+				dm_audit_log_bio(DM_MSG_PREFIX, "verify-data",
+						 bio, cur_block, 0);
 				return -EIO;
+			}
 		}
 	}
 
@@ -975,6 +986,8 @@ static void verity_dtr(struct dm_target *ti)
 		static_branch_dec(&use_tasklet_enabled);
 
 	kfree(v);
+
+	dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1);
 }
 
 static int verity_alloc_most_once(struct dm_verity *v)
@@ -1429,11 +1442,14 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 
 	verity_verify_sig_opts_cleanup(&verify_args);
 
+	dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1);
+
 	return 0;
 
 bad:
 
 	verity_verify_sig_opts_cleanup(&verify_args);
+	dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0);
 	verity_dtr(ti);
 
 	return r;
@@ -1498,25 +1514,7 @@ static struct target_type verity_target = {
 	.iterate_devices = verity_iterate_devices,
 	.io_hints	= verity_io_hints,
 };
-
-static int __init dm_verity_init(void)
-{
-	int r;
-
-	r = dm_register_target(&verity_target);
-	if (r < 0)
-		DMERR("register failed %d", r);
-
-	return r;
-}
-
-static void __exit dm_verity_exit(void)
-{
-	dm_unregister_target(&verity_target);
-}
-
-module_init(dm_verity_init);
-module_exit(dm_verity_exit);
+module_dm(verity);
 
 MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
 MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c
index 3aa5874f0aef..074cb785eafc 100644
--- a/drivers/md/dm-writecache.c
+++ b/drivers/md/dm-writecache.c
@@ -2773,27 +2773,7 @@ static struct target_type writecache_target = {
 	.iterate_devices	= writecache_iterate_devices,
 	.io_hints		= writecache_io_hints,
 };
-
-static int __init dm_writecache_init(void)
-{
-	int r;
-
-	r = dm_register_target(&writecache_target);
-	if (r < 0) {
-		DMERR("register failed %d", r);
-		return r;
-	}
-
-	return 0;
-}
-
-static void __exit dm_writecache_exit(void)
-{
-	dm_unregister_target(&writecache_target);
-}
-
-module_init(dm_writecache_init);
-module_exit(dm_writecache_exit);
+module_dm(writecache);
 
 MODULE_DESCRIPTION(DM_NAME " writecache target");
 MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");
diff --git a/drivers/md/dm-zero.c b/drivers/md/dm-zero.c
index 2601cd520384..3b13e6eb1aa4 100644
--- a/drivers/md/dm-zero.c
+++ b/drivers/md/dm-zero.c
@@ -27,6 +27,7 @@ static int zero_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	 * Silently drop discards, avoiding -EOPNOTSUPP.
 	 */
 	ti->num_discard_bios = 1;
+	ti->discards_supported = true;
 
 	return 0;
 }
@@ -45,6 +46,7 @@ static int zero_map(struct dm_target *ti, struct bio *bio)
 		zero_fill_bio(bio);
 		break;
 	case REQ_OP_WRITE:
+	case REQ_OP_DISCARD:
 		/* writes get silently dropped */
 		break;
 	default:
@@ -57,32 +59,23 @@ static int zero_map(struct dm_target *ti, struct bio *bio)
 	return DM_MAPIO_SUBMITTED;
 }
 
+static void zero_io_hints(struct dm_target *ti, struct queue_limits *limits)
+{
+	limits->max_discard_sectors = UINT_MAX;
+	limits->max_hw_discard_sectors = UINT_MAX;
+	limits->discard_granularity = 512;
+}
+
 static struct target_type zero_target = {
 	.name   = "zero",
-	.version = {1, 1, 0},
+	.version = {1, 2, 0},
 	.features = DM_TARGET_NOWAIT,
 	.module = THIS_MODULE,
 	.ctr    = zero_ctr,
 	.map    = zero_map,
+	.io_hints = zero_io_hints,
 };
-
-static int __init dm_zero_init(void)
-{
-	int r = dm_register_target(&zero_target);
-
-	if (r < 0)
-		DMERR("register failed %d", r);
-
-	return r;
-}
-
-static void __exit dm_zero_exit(void)
-{
-	dm_unregister_target(&zero_target);
-}
-
-module_init(dm_zero_init)
-module_exit(dm_zero_exit)
+module_dm(zero);
 
 MODULE_AUTHOR("Jana Saout <jana@saout.de>");
 MODULE_DESCRIPTION(DM_NAME " dummy target returning zeros");
diff --git a/drivers/md/dm-zoned-metadata.c b/drivers/md/dm-zoned-metadata.c
index cf9402064aba..8f0896a6990b 100644
--- a/drivers/md/dm-zoned-metadata.c
+++ b/drivers/md/dm-zoned-metadata.c
@@ -1701,7 +1701,7 @@ static int dmz_load_mapping(struct dmz_metadata *zmd)
 			if (IS_ERR(dmap_mblk))
 				return PTR_ERR(dmap_mblk);
 			zmd->map_mblk[i] = dmap_mblk;
-			dmap = (struct dmz_map *) dmap_mblk->data;
+			dmap = dmap_mblk->data;
 			i++;
 			e = 0;
 		}
@@ -1832,7 +1832,7 @@ static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk,
 				  unsigned int dzone_id, unsigned int bzone_id)
 {
 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
-	struct dmz_map *dmap = (struct dmz_map *) dmap_mblk->data;
+	struct dmz_map *dmap = dmap_mblk->data;
 	int map_idx = chunk & DMZ_MAP_ENTRIES_MASK;
 
 	dmap[map_idx].dzone_id = cpu_to_le32(dzone_id);
@@ -2045,7 +2045,7 @@ struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd,
 				      unsigned int chunk, enum req_op op)
 {
 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
-	struct dmz_map *dmap = (struct dmz_map *) dmap_mblk->data;
+	struct dmz_map *dmap = dmap_mblk->data;
 	int dmap_idx = chunk & DMZ_MAP_ENTRIES_MASK;
 	unsigned int dzone_id;
 	struct dm_zone *dzone = NULL;
diff --git a/drivers/md/dm-zoned-target.c b/drivers/md/dm-zoned-target.c
index ad4764dcd013..ad8e670a2f9b 100644
--- a/drivers/md/dm-zoned-target.c
+++ b/drivers/md/dm-zoned-target.c
@@ -1138,7 +1138,7 @@ static int dmz_message(struct dm_target *ti, unsigned int argc, char **argv,
 	return r;
 }
 
-static struct target_type dmz_type = {
+static struct target_type zoned_target = {
 	.name		 = "zoned",
 	.version	 = {2, 0, 0},
 	.features	 = DM_TARGET_SINGLETON | DM_TARGET_MIXED_ZONED_MODEL,
@@ -1154,19 +1154,7 @@ static struct target_type dmz_type = {
 	.status		 = dmz_status,
 	.message	 = dmz_message,
 };
-
-static int __init dmz_init(void)
-{
-	return dm_register_target(&dmz_type);
-}
-
-static void __exit dmz_exit(void)
-{
-	dm_unregister_target(&dmz_type);
-}
-
-module_init(dmz_init);
-module_exit(dmz_exit);
+module_dm(zoned);
 
 MODULE_DESCRIPTION(DM_NAME " target for zoned block devices");
 MODULE_AUTHOR("Damien Le Moal <damien.lemoal@wdc.com>");
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index dfde0088147a..3b694ba3a106 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1072,6 +1072,15 @@ static void dm_io_dec_pending(struct dm_io *io, blk_status_t error)
 	__dm_io_dec_pending(io);
 }
 
+/*
+ * The queue_limits are only valid as long as you have a reference
+ * count on 'md'. But _not_ imposing verification to avoid atomic_read(),
+ */
+static inline struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
+{
+	return &md->queue->limits;
+}
+
 void disable_discard(struct mapped_device *md)
 {
 	struct queue_limits *limits = dm_get_queue_limits(md);
@@ -1162,7 +1171,8 @@ static inline sector_t max_io_len_target_boundary(struct dm_target *ti,
 	return ti->len - target_offset;
 }
 
-static sector_t max_io_len(struct dm_target *ti, sector_t sector)
+static sector_t __max_io_len(struct dm_target *ti, sector_t sector,
+			     unsigned int max_granularity)
 {
 	sector_t target_offset = dm_target_offset(ti, sector);
 	sector_t len = max_io_len_target_boundary(ti, target_offset);
@@ -1173,11 +1183,16 @@ static sector_t max_io_len(struct dm_target *ti, sector_t sector)
 	 *   explains why stacked chunk_sectors based splitting via
 	 *   bio_split_to_limits() isn't possible here.
 	 */
-	if (!ti->max_io_len)
+	if (!max_granularity)
 		return len;
 	return min_t(sector_t, len,
 		min(queue_max_sectors(ti->table->md->queue),
-		    blk_chunk_sectors_left(target_offset, ti->max_io_len)));
+		    blk_chunk_sectors_left(target_offset, max_granularity)));
+}
+
+static inline sector_t max_io_len(struct dm_target *ti, sector_t sector)
+{
+	return __max_io_len(ti, sector, ti->max_io_len);
 }
 
 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
@@ -1565,12 +1580,13 @@ static void __send_empty_flush(struct clone_info *ci)
 }
 
 static void __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
-					unsigned int num_bios)
+					unsigned int num_bios,
+					unsigned int max_granularity)
 {
 	unsigned int len, bios;
 
 	len = min_t(sector_t, ci->sector_count,
-		    max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));
+		    __max_io_len(ti, ci->sector, max_granularity));
 
 	atomic_add(num_bios, &ci->io->io_count);
 	bios = __send_duplicate_bios(ci, ti, num_bios, &len);
@@ -1606,16 +1622,24 @@ static blk_status_t __process_abnormal_io(struct clone_info *ci,
 					  struct dm_target *ti)
 {
 	unsigned int num_bios = 0;
+	unsigned int max_granularity = 0;
+	struct queue_limits *limits = dm_get_queue_limits(ti->table->md);
 
 	switch (bio_op(ci->bio)) {
 	case REQ_OP_DISCARD:
 		num_bios = ti->num_discard_bios;
+		if (ti->max_discard_granularity)
+			max_granularity = limits->max_discard_sectors;
 		break;
 	case REQ_OP_SECURE_ERASE:
 		num_bios = ti->num_secure_erase_bios;
+		if (ti->max_secure_erase_granularity)
+			max_granularity = limits->max_secure_erase_sectors;
 		break;
 	case REQ_OP_WRITE_ZEROES:
 		num_bios = ti->num_write_zeroes_bios;
+		if (ti->max_write_zeroes_granularity)
+			max_granularity = limits->max_write_zeroes_sectors;
 		break;
 	default:
 		break;
@@ -1630,7 +1654,7 @@ static blk_status_t __process_abnormal_io(struct clone_info *ci,
 	if (unlikely(!num_bios))
 		return BLK_STS_NOTSUPP;
 
-	__send_changing_extent_only(ci, ti, num_bios);
+	__send_changing_extent_only(ci, ti, num_bios, max_granularity);
 	return BLK_STS_OK;
 }
 
@@ -2297,17 +2321,6 @@ struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
 }
 
 /*
- * The queue_limits are only valid as long as you have a reference
- * count on 'md'.
- */
-struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
-{
-	BUG_ON(!atomic_read(&md->holders));
-	return &md->queue->limits;
-}
-EXPORT_SYMBOL_GPL(dm_get_queue_limits);
-
-/*
  * Setup the DM device's queue based on md's type
  */
 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
index 22eaed188907..a856e0aee73b 100644
--- a/drivers/md/dm.h
+++ b/drivers/md/dm.h
@@ -20,6 +20,7 @@
 #include <linux/completion.h>
 #include <linux/kobject.h>
 #include <linux/refcount.h>
+#include <linux/log2.h>
 
 #include "dm-stats.h"
 
@@ -228,4 +229,25 @@ void dm_free_md_mempools(struct dm_md_mempools *pools);
  */
 unsigned int dm_get_reserved_bio_based_ios(void);
 
+#define DM_HASH_LOCKS_MAX 64
+
+static inline unsigned int dm_num_hash_locks(void)
+{
+	unsigned int num_locks = roundup_pow_of_two(num_online_cpus()) << 1;
+
+	return min_t(unsigned int, num_locks, DM_HASH_LOCKS_MAX);
+}
+
+#define DM_HASH_LOCKS_MULT  4294967291ULL
+#define DM_HASH_LOCKS_SHIFT 6
+
+static inline unsigned int dm_hash_locks_index(sector_t block,
+					       unsigned int num_locks)
+{
+	sector_t h1 = (block * DM_HASH_LOCKS_MULT) >> DM_HASH_LOCKS_SHIFT;
+	sector_t h2 = h1 >> DM_HASH_LOCKS_SHIFT;
+
+	return (h1 ^ h2) & (num_locks - 1);
+}
+
 #endif
diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c
index e7cc6ba1b657..920bb68156d2 100644
--- a/drivers/md/md-bitmap.c
+++ b/drivers/md/md-bitmap.c
@@ -209,76 +209,99 @@ static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mdde
 	return NULL;
 }
 
-static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
+static unsigned int optimal_io_size(struct block_device *bdev,
+				    unsigned int last_page_size,
+				    unsigned int io_size)
+{
+	if (bdev_io_opt(bdev) > bdev_logical_block_size(bdev))
+		return roundup(last_page_size, bdev_io_opt(bdev));
+	return io_size;
+}
+
+static unsigned int bitmap_io_size(unsigned int io_size, unsigned int opt_size,
+				   sector_t start, sector_t boundary)
+{
+	if (io_size != opt_size &&
+	    start + opt_size / SECTOR_SIZE <= boundary)
+		return opt_size;
+	if (start + io_size / SECTOR_SIZE <= boundary)
+		return io_size;
+
+	/* Overflows boundary */
+	return 0;
+}
+
+static int __write_sb_page(struct md_rdev *rdev, struct bitmap *bitmap,
+			   struct page *page)
 {
-	struct md_rdev *rdev;
 	struct block_device *bdev;
 	struct mddev *mddev = bitmap->mddev;
 	struct bitmap_storage *store = &bitmap->storage;
+	sector_t offset = mddev->bitmap_info.offset;
+	sector_t ps, sboff, doff;
+	unsigned int size = PAGE_SIZE;
+	unsigned int opt_size = PAGE_SIZE;
+
+	bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
+	if (page->index == store->file_pages - 1) {
+		unsigned int last_page_size = store->bytes & (PAGE_SIZE - 1);
+
+		if (last_page_size == 0)
+			last_page_size = PAGE_SIZE;
+		size = roundup(last_page_size, bdev_logical_block_size(bdev));
+		opt_size = optimal_io_size(bdev, last_page_size, size);
+	}
+
+	ps = page->index * PAGE_SIZE / SECTOR_SIZE;
+	sboff = rdev->sb_start + offset;
+	doff = rdev->data_offset;
+
+	/* Just make sure we aren't corrupting data or metadata */
+	if (mddev->external) {
+		/* Bitmap could be anywhere. */
+		if (sboff + ps > doff &&
+		    sboff < (doff + mddev->dev_sectors + PAGE_SIZE / SECTOR_SIZE))
+			return -EINVAL;
+	} else if (offset < 0) {
+		/* DATA  BITMAP METADATA  */
+		size = bitmap_io_size(size, opt_size, offset + ps, 0);
+		if (size == 0)
+			/* bitmap runs in to metadata */
+			return -EINVAL;
+
+		if (doff + mddev->dev_sectors > sboff)
+			/* data runs in to bitmap */
+			return -EINVAL;
+	} else if (rdev->sb_start < rdev->data_offset) {
+		/* METADATA BITMAP DATA */
+		size = bitmap_io_size(size, opt_size, sboff + ps, doff);
+		if (size == 0)
+			/* bitmap runs in to data */
+			return -EINVAL;
+	} else {
+		/* DATA METADATA BITMAP - no problems */
+	}
 
-restart:
-	rdev = NULL;
-	while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
-		int size = PAGE_SIZE;
-		loff_t offset = mddev->bitmap_info.offset;
+	md_super_write(mddev, rdev, sboff + ps, (int) size, page);
+	return 0;
+}
 
-		bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
+static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
+{
+	struct md_rdev *rdev;
+	struct mddev *mddev = bitmap->mddev;
+	int ret;
 
-		if (page->index == store->file_pages-1) {
-			int last_page_size = store->bytes & (PAGE_SIZE-1);
-			if (last_page_size == 0)
-				last_page_size = PAGE_SIZE;
-			size = roundup(last_page_size,
-				       bdev_logical_block_size(bdev));
-		}
-		/* Just make sure we aren't corrupting data or
-		 * metadata
-		 */
-		if (mddev->external) {
-			/* Bitmap could be anywhere. */
-			if (rdev->sb_start + offset + (page->index
-						       * (PAGE_SIZE/512))
-			    > rdev->data_offset
-			    &&
-			    rdev->sb_start + offset
-			    < (rdev->data_offset + mddev->dev_sectors
-			     + (PAGE_SIZE/512)))
-				goto bad_alignment;
-		} else if (offset < 0) {
-			/* DATA  BITMAP METADATA  */
-			if (offset
-			    + (long)(page->index * (PAGE_SIZE/512))
-			    + size/512 > 0)
-				/* bitmap runs in to metadata */
-				goto bad_alignment;
-			if (rdev->data_offset + mddev->dev_sectors
-			    > rdev->sb_start + offset)
-				/* data runs in to bitmap */
-				goto bad_alignment;
-		} else if (rdev->sb_start < rdev->data_offset) {
-			/* METADATA BITMAP DATA */
-			if (rdev->sb_start
-			    + offset
-			    + page->index*(PAGE_SIZE/512) + size/512
-			    > rdev->data_offset)
-				/* bitmap runs in to data */
-				goto bad_alignment;
-		} else {
-			/* DATA METADATA BITMAP - no problems */
+	do {
+		rdev = NULL;
+		while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
+			ret = __write_sb_page(rdev, bitmap, page);
+			if (ret)
+				return ret;
 		}
-		md_super_write(mddev, rdev,
-			       rdev->sb_start + offset
-			       + page->index * (PAGE_SIZE/512),
-			       size,
-			       page);
-	}
+	} while (wait && md_super_wait(mddev) < 0);
 
-	if (wait && md_super_wait(mddev) < 0)
-		goto restart;
 	return 0;
-
- bad_alignment:
-	return -EINVAL;
 }
 
 static void md_bitmap_file_kick(struct bitmap *bitmap);
diff --git a/drivers/md/md-linear.c b/drivers/md/md-linear.c
index 6e7797b4e738..4eb72b9dd933 100644
--- a/drivers/md/md-linear.c
+++ b/drivers/md/md-linear.c
@@ -223,7 +223,8 @@ static bool linear_make_request(struct mddev *mddev, struct bio *bio)
 		     bio_sector < start_sector))
 		goto out_of_bounds;
 
-	if (unlikely(is_mddev_broken(tmp_dev->rdev, "linear"))) {
+	if (unlikely(is_rdev_broken(tmp_dev->rdev))) {
+		md_error(mddev, tmp_dev->rdev);
 		bio_io_error(bio);
 		return true;
 	}
@@ -270,6 +271,16 @@ static void linear_status (struct seq_file *seq, struct mddev *mddev)
 	seq_printf(seq, " %dk rounding", mddev->chunk_sectors / 2);
 }
 
+static void linear_error(struct mddev *mddev, struct md_rdev *rdev)
+{
+	if (!test_and_set_bit(MD_BROKEN, &mddev->flags)) {
+		char *md_name = mdname(mddev);
+
+		pr_crit("md/linear%s: Disk failure on %pg detected, failing array.\n",
+			md_name, rdev->bdev);
+	}
+}
+
 static void linear_quiesce(struct mddev *mddev, int state)
 {
 }
@@ -286,6 +297,7 @@ static struct md_personality linear_personality =
 	.hot_add_disk	= linear_add,
 	.size		= linear_size,
 	.quiesce	= linear_quiesce,
+	.error_handler	= linear_error,
 };
 
 static int __init linear_init (void)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 13321dbb5fbc..8e344b4b3444 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -78,7 +78,7 @@
 static LIST_HEAD(pers_list);
 static DEFINE_SPINLOCK(pers_lock);
 
-static struct kobj_type md_ktype;
+static const struct kobj_type md_ktype;
 
 struct md_cluster_operations *md_cluster_ops;
 EXPORT_SYMBOL(md_cluster_ops);
@@ -322,26 +322,6 @@ static struct ctl_table raid_table[] = {
 	{ }
 };
 
-static struct ctl_table raid_dir_table[] = {
-	{
-		.procname	= "raid",
-		.maxlen		= 0,
-		.mode		= S_IRUGO|S_IXUGO,
-		.child		= raid_table,
-	},
-	{ }
-};
-
-static struct ctl_table raid_root_table[] = {
-	{
-		.procname	= "dev",
-		.maxlen		= 0,
-		.mode		= 0555,
-		.child		= raid_dir_table,
-	},
-	{  }
-};
-
 static int start_readonly;
 
 /*
@@ -3600,7 +3580,7 @@ static const struct sysfs_ops rdev_sysfs_ops = {
 	.show		= rdev_attr_show,
 	.store		= rdev_attr_store,
 };
-static struct kobj_type rdev_ktype = {
+static const struct kobj_type rdev_ktype = {
 	.release	= rdev_free,
 	.sysfs_ops	= &rdev_sysfs_ops,
 	.default_groups	= rdev_default_groups,
@@ -5558,7 +5538,7 @@ static const struct sysfs_ops md_sysfs_ops = {
 	.show	= md_attr_show,
 	.store	= md_attr_store,
 };
-static struct kobj_type md_ktype = {
+static const struct kobj_type md_ktype = {
 	.release	= md_kobj_release,
 	.sysfs_ops	= &md_sysfs_ops,
 	.default_groups	= md_attr_groups,
@@ -7974,6 +7954,9 @@ void md_error(struct mddev *mddev, struct md_rdev *rdev)
 		return;
 	mddev->pers->error_handler(mddev, rdev);
 
+	if (mddev->pers->level == 0 || mddev->pers->level == LEVEL_LINEAR)
+		return;
+
 	if (mddev->degraded && !test_bit(MD_BROKEN, &mddev->flags))
 		set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
 	sysfs_notify_dirent_safe(rdev->sysfs_state);
@@ -8029,16 +8012,16 @@ static int status_resync(struct seq_file *seq, struct mddev *mddev)
 	} else if (resync > max_sectors) {
 		resync = max_sectors;
 	} else {
-		resync -= atomic_read(&mddev->recovery_active);
-		if (resync < MD_RESYNC_ACTIVE) {
-			/*
-			 * Resync has started, but the subtraction has
-			 * yielded one of the special values. Force it
-			 * to active to ensure the status reports an
-			 * active resync.
-			 */
+		res = atomic_read(&mddev->recovery_active);
+		/*
+		 * Resync has started, but the subtraction has overflowed or
+		 * yielded one of the special values. Force it to active to
+		 * ensure the status reports an active resync.
+		 */
+		if (resync < res || resync - res < MD_RESYNC_ACTIVE)
 			resync = MD_RESYNC_ACTIVE;
-		}
+		else
+			resync -= res;
 	}
 
 	if (resync == MD_RESYNC_NONE) {
@@ -9650,7 +9633,7 @@ static int __init md_init(void)
 	mdp_major = ret;
 
 	register_reboot_notifier(&md_notifier);
-	raid_table_header = register_sysctl_table(raid_root_table);
+	raid_table_header = register_sysctl("dev/raid", raid_table);
 
 	md_geninit();
 	return 0;
diff --git a/drivers/md/md.h b/drivers/md/md.h
index e148e3c83b0d..fd8f260ed5f8 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -790,15 +790,9 @@ extern void mddev_destroy_serial_pool(struct mddev *mddev, struct md_rdev *rdev,
 struct md_rdev *md_find_rdev_nr_rcu(struct mddev *mddev, int nr);
 struct md_rdev *md_find_rdev_rcu(struct mddev *mddev, dev_t dev);
 
-static inline bool is_mddev_broken(struct md_rdev *rdev, const char *md_type)
+static inline bool is_rdev_broken(struct md_rdev *rdev)
 {
-	if (!disk_live(rdev->bdev->bd_disk)) {
-		if (!test_and_set_bit(MD_BROKEN, &rdev->mddev->flags))
-			pr_warn("md: %s: %s array has a missing/failed member\n",
-				mdname(rdev->mddev), md_type);
-		return true;
-	}
-	return false;
+	return !disk_live(rdev->bdev->bd_disk);
 }
 
 static inline void rdev_dec_pending(struct md_rdev *rdev, struct mddev *mddev)
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index b536befd8898..f8ee9a95e25d 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -569,8 +569,9 @@ static bool raid0_make_request(struct mddev *mddev, struct bio *bio)
 		return true;
 	}
 
-	if (unlikely(is_mddev_broken(tmp_dev, "raid0"))) {
+	if (unlikely(is_rdev_broken(tmp_dev))) {
 		bio_io_error(bio);
+		md_error(mddev, tmp_dev);
 		return true;
 	}
 
@@ -592,6 +593,16 @@ static void raid0_status(struct seq_file *seq, struct mddev *mddev)
 	return;
 }
 
+static void raid0_error(struct mddev *mddev, struct md_rdev *rdev)
+{
+	if (!test_and_set_bit(MD_BROKEN, &mddev->flags)) {
+		char *md_name = mdname(mddev);
+
+		pr_crit("md/raid0%s: Disk failure on %pg detected, failing array.\n",
+			md_name, rdev->bdev);
+	}
+}
+
 static void *raid0_takeover_raid45(struct mddev *mddev)
 {
 	struct md_rdev *rdev;
@@ -767,6 +778,7 @@ static struct md_personality raid0_personality=
 	.size		= raid0_size,
 	.takeover	= raid0_takeover,
 	.quiesce	= raid0_quiesce,
+	.error_handler	= raid0_error,
 };
 
 static int __init raid0_init (void)
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 6c66357f92f5..4fcfcb350d2b 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -952,7 +952,9 @@ static void flush_pending_writes(struct r10conf *conf)
 static void raise_barrier(struct r10conf *conf, int force)
 {
 	write_seqlock_irq(&conf->resync_lock);
-	BUG_ON(force && !conf->barrier);
+
+	if (WARN_ON_ONCE(force && !conf->barrier))
+		force = false;
 
 	/* Wait until no block IO is waiting (unless 'force') */
 	wait_event_barrier(conf, force || !conf->nr_waiting);
@@ -995,11 +997,15 @@ static bool stop_waiting_barrier(struct r10conf *conf)
 	    (!bio_list_empty(&bio_list[0]) || !bio_list_empty(&bio_list[1])))
 		return true;
 
-	/* move on if recovery thread is blocked by us */
-	if (conf->mddev->thread->tsk == current &&
-	    test_bit(MD_RECOVERY_RUNNING, &conf->mddev->recovery) &&
-	    conf->nr_queued > 0)
+	/*
+	 * move on if io is issued from raid10d(), nr_pending is not released
+	 * from original io(see handle_read_error()). All raise barrier is
+	 * blocked until this io is done.
+	 */
+	if (conf->mddev->thread->tsk == current) {
+		WARN_ON_ONCE(atomic_read(&conf->nr_pending) == 0);
 		return true;
+	}
 
 	return false;
 }
@@ -1244,7 +1250,8 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio,
 	}
 	slot = r10_bio->read_slot;
 
-	if (blk_queue_io_stat(bio->bi_bdev->bd_disk->queue))
+	if (!r10_bio->start_time &&
+	    blk_queue_io_stat(bio->bi_bdev->bd_disk->queue))
 		r10_bio->start_time = bio_start_io_acct(bio);
 	read_bio = bio_alloc_clone(rdev->bdev, bio, gfp, &mddev->bio_set);
 
@@ -1574,6 +1581,7 @@ static void __make_request(struct mddev *mddev, struct bio *bio, int sectors)
 	r10_bio->sector = bio->bi_iter.bi_sector;
 	r10_bio->state = 0;
 	r10_bio->read_slot = -1;
+	r10_bio->start_time = 0;
 	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) *
 			conf->geo.raid_disks);
 
@@ -1626,7 +1634,7 @@ static void raid10_end_discard_request(struct bio *bio)
 		/*
 		 * raid10_remove_disk uses smp_mb to make sure rdev is set to
 		 * replacement before setting replacement to NULL. It can read
-		 * rdev first without barrier protect even replacment is NULL
+		 * rdev first without barrier protect even replacement is NULL
 		 */
 		smp_rmb();
 		rdev = conf->mirrors[dev].rdev;
@@ -2609,11 +2617,22 @@ static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
 {
 	struct r10conf *conf = mddev->private;
 	int d;
-	struct bio *wbio, *wbio2;
+	struct bio *wbio = r10_bio->devs[1].bio;
+	struct bio *wbio2 = r10_bio->devs[1].repl_bio;
+
+	/* Need to test wbio2->bi_end_io before we call
+	 * submit_bio_noacct as if the former is NULL,
+	 * the latter is free to free wbio2.
+	 */
+	if (wbio2 && !wbio2->bi_end_io)
+		wbio2 = NULL;
 
 	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
 		fix_recovery_read_error(r10_bio);
-		end_sync_request(r10_bio);
+		if (wbio->bi_end_io)
+			end_sync_request(r10_bio);
+		if (wbio2)
+			end_sync_request(r10_bio);
 		return;
 	}
 
@@ -2622,14 +2641,6 @@ static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
 	 * and submit the write request
 	 */
 	d = r10_bio->devs[1].devnum;
-	wbio = r10_bio->devs[1].bio;
-	wbio2 = r10_bio->devs[1].repl_bio;
-	/* Need to test wbio2->bi_end_io before we call
-	 * submit_bio_noacct as if the former is NULL,
-	 * the latter is free to free wbio2.
-	 */
-	if (wbio2 && !wbio2->bi_end_io)
-		wbio2 = NULL;
 	if (wbio->bi_end_io) {
 		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
 		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio));
@@ -2978,9 +2989,13 @@ static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
 		md_error(mddev, rdev);
 
 	rdev_dec_pending(rdev, mddev);
-	allow_barrier(conf);
 	r10_bio->state = 0;
 	raid10_read_request(mddev, r10_bio->master_bio, r10_bio);
+	/*
+	 * allow_barrier after re-submit to ensure no sync io
+	 * can be issued while regular io pending.
+	 */
+	allow_barrier(conf);
 }
 
 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
@@ -3289,10 +3304,6 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 	sector_t chunk_mask = conf->geo.chunk_mask;
 	int page_idx = 0;
 
-	if (!mempool_initialized(&conf->r10buf_pool))
-		if (init_resync(conf))
-			return 0;
-
 	/*
 	 * Allow skipping a full rebuild for incremental assembly
 	 * of a clean array, like RAID1 does.
@@ -3308,6 +3319,10 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 		return mddev->dev_sectors - sector_nr;
 	}
 
+	if (!mempool_initialized(&conf->r10buf_pool))
+		if (init_resync(conf))
+			return 0;
+
  skipped:
 	max_sector = mddev->dev_sectors;
 	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
@@ -4004,6 +4019,20 @@ static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
 	return nc*fc;
 }
 
+static void raid10_free_conf(struct r10conf *conf)
+{
+	if (!conf)
+		return;
+
+	mempool_exit(&conf->r10bio_pool);
+	kfree(conf->mirrors);
+	kfree(conf->mirrors_old);
+	kfree(conf->mirrors_new);
+	safe_put_page(conf->tmppage);
+	bioset_exit(&conf->bio_split);
+	kfree(conf);
+}
+
 static struct r10conf *setup_conf(struct mddev *mddev)
 {
 	struct r10conf *conf = NULL;
@@ -4086,13 +4115,7 @@ static struct r10conf *setup_conf(struct mddev *mddev)
 	return conf;
 
  out:
-	if (conf) {
-		mempool_exit(&conf->r10bio_pool);
-		kfree(conf->mirrors);
-		safe_put_page(conf->tmppage);
-		bioset_exit(&conf->bio_split);
-		kfree(conf);
-	}
+	raid10_free_conf(conf);
 	return ERR_PTR(err);
 }
 
@@ -4129,6 +4152,9 @@ static int raid10_run(struct mddev *mddev)
 	if (!conf)
 		goto out;
 
+	mddev->thread = conf->thread;
+	conf->thread = NULL;
+
 	if (mddev_is_clustered(conf->mddev)) {
 		int fc, fo;
 
@@ -4141,9 +4167,6 @@ static int raid10_run(struct mddev *mddev)
 		}
 	}
 
-	mddev->thread = conf->thread;
-	conf->thread = NULL;
-
 	if (mddev->queue) {
 		blk_queue_max_write_zeroes_sectors(mddev->queue, 0);
 		blk_queue_io_min(mddev->queue, mddev->chunk_sectors << 9);
@@ -4283,10 +4306,7 @@ static int raid10_run(struct mddev *mddev)
 
 out_free_conf:
 	md_unregister_thread(&mddev->thread);
-	mempool_exit(&conf->r10bio_pool);
-	safe_put_page(conf->tmppage);
-	kfree(conf->mirrors);
-	kfree(conf);
+	raid10_free_conf(conf);
 	mddev->private = NULL;
 out:
 	return -EIO;
@@ -4294,15 +4314,7 @@ out:
 
 static void raid10_free(struct mddev *mddev, void *priv)
 {
-	struct r10conf *conf = priv;
-
-	mempool_exit(&conf->r10bio_pool);
-	safe_put_page(conf->tmppage);
-	kfree(conf->mirrors);
-	kfree(conf->mirrors_old);
-	kfree(conf->mirrors_new);
-	bioset_exit(&conf->bio_split);
-	kfree(conf);
+	raid10_free_conf(priv);
 }
 
 static void raid10_quiesce(struct mddev *mddev, int quiesce)
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 7b820b81d8c2..812a12e3e41a 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -7716,7 +7716,6 @@ static void raid5_set_io_opt(struct r5conf *conf)
 static int raid5_run(struct mddev *mddev)
 {
 	struct r5conf *conf;
-	int working_disks = 0;
 	int dirty_parity_disks = 0;
 	struct md_rdev *rdev;
 	struct md_rdev *journal_dev = NULL;
@@ -7912,10 +7911,8 @@ static int raid5_run(struct mddev *mddev)
 			pr_warn("md: cannot handle concurrent replacement and reshape.\n");
 			goto abort;
 		}
-		if (test_bit(In_sync, &rdev->flags)) {
-			working_disks++;
+		if (test_bit(In_sync, &rdev->flags))
 			continue;
-		}
 		/* This disc is not fully in-sync.  However if it
 		 * just stored parity (beyond the recovery_offset),
 		 * when we don't need to be concerned about the
diff --git a/drivers/media/cec/core/cec-adap.c b/drivers/media/cec/core/cec-adap.c
index 4f5ab3cae8a7..769ea6b2e1d0 100644
--- a/drivers/media/cec/core/cec-adap.c
+++ b/drivers/media/cec/core/cec-adap.c
@@ -1052,6 +1052,7 @@ void cec_received_msg_ts(struct cec_adapter *adap,
 	u8 cmd = msg->msg[1];
 	bool is_reply = false;
 	bool valid_la = true;
+	bool monitor_valid_la = true;
 	u8 min_len = 0;
 
 	if (WARN_ON(!msg->len || msg->len > CEC_MAX_MSG_SIZE))
@@ -1093,8 +1094,10 @@ void cec_received_msg_ts(struct cec_adapter *adap,
 	adap->last_initiator = 0xff;
 
 	/* Check if this message was for us (directed or broadcast). */
-	if (!cec_msg_is_broadcast(msg))
+	if (!cec_msg_is_broadcast(msg)) {
 		valid_la = cec_has_log_addr(adap, msg_dest);
+		monitor_valid_la = valid_la;
+	}
 
 	/*
 	 * Check if the length is not too short or if the message is a
@@ -1227,7 +1230,7 @@ void cec_received_msg_ts(struct cec_adapter *adap,
 	mutex_unlock(&adap->lock);
 
 	/* Pass the message on to any monitoring filehandles */
-	cec_queue_msg_monitor(adap, msg, valid_la);
+	cec_queue_msg_monitor(adap, msg, monitor_valid_la);
 
 	/* We're done if it is not for us or a poll message */
 	if (!valid_la || msg->len <= 1)
diff --git a/drivers/media/cec/platform/cec-gpio/cec-gpio.c b/drivers/media/cec/platform/cec-gpio/cec-gpio.c
index c8c4efc83f5f..ff34490fd869 100644
--- a/drivers/media/cec/platform/cec-gpio/cec-gpio.c
+++ b/drivers/media/cec/platform/cec-gpio/cec-gpio.c
@@ -269,13 +269,12 @@ del_adap:
 	return ret;
 }
 
-static int cec_gpio_remove(struct platform_device *pdev)
+static void cec_gpio_remove(struct platform_device *pdev)
 {
 	struct cec_gpio *cec = platform_get_drvdata(pdev);
 
 	cec_notifier_cec_adap_unregister(cec->notifier, cec->adap);
 	cec_unregister_adapter(cec->adap);
-	return 0;
 }
 
 static const struct of_device_id cec_gpio_match[] = {
@@ -288,7 +287,7 @@ MODULE_DEVICE_TABLE(of, cec_gpio_match);
 
 static struct platform_driver cec_gpio_pdrv = {
 	.probe	= cec_gpio_probe,
-	.remove = cec_gpio_remove,
+	.remove_new = cec_gpio_remove,
 	.driver = {
 		.name		= "cec-gpio",
 		.of_match_table	= cec_gpio_match,
diff --git a/drivers/media/cec/platform/cros-ec/cros-ec-cec.c b/drivers/media/cec/platform/cros-ec/cros-ec-cec.c
index 6ebedc71d67d..c17faf002877 100644
--- a/drivers/media/cec/platform/cros-ec/cros-ec-cec.c
+++ b/drivers/media/cec/platform/cros-ec/cros-ec-cec.c
@@ -225,6 +225,12 @@ static const struct cec_dmi_match cec_dmi_match_table[] = {
 	{ "Google", "Kinox", "0000:00:02.0", "Port B" },
 	/* Google Kuldax */
 	{ "Google", "Kuldax", "0000:00:02.0", "Port B" },
+	/* Google Aurash */
+	{ "Google", "Aurash", "0000:00:02.0", "Port B" },
+	/* Google Gladios */
+	{ "Google", "Gladios", "0000:00:02.0", "Port B" },
+	/* Google Lisbon */
+	{ "Google", "Lisbon", "0000:00:02.0", "Port B" },
 };
 
 static struct device *cros_ec_cec_find_hdmi_dev(struct device *dev,
@@ -326,31 +332,31 @@ out_probe_adapter:
 	return ret;
 }
 
-static int cros_ec_cec_remove(struct platform_device *pdev)
+static void cros_ec_cec_remove(struct platform_device *pdev)
 {
 	struct cros_ec_cec *cros_ec_cec = platform_get_drvdata(pdev);
 	struct device *dev = &pdev->dev;
 	int ret;
 
+	/*
+	 * blocking_notifier_chain_unregister() only fails if the notifier isn't
+	 * in the list. We know it was added to it by .probe(), so there should
+	 * be no need for error checking. Be cautious and still check.
+	 */
 	ret = blocking_notifier_chain_unregister(
 			&cros_ec_cec->cros_ec->event_notifier,
 			&cros_ec_cec->notifier);
-
-	if (ret) {
+	if (ret)
 		dev_err(dev, "failed to unregister notifier\n");
-		return ret;
-	}
 
 	cec_notifier_cec_adap_unregister(cros_ec_cec->notify,
 					 cros_ec_cec->adap);
 	cec_unregister_adapter(cros_ec_cec->adap);
-
-	return 0;
 }
 
 static struct platform_driver cros_ec_cec_driver = {
 	.probe = cros_ec_cec_probe,
-	.remove  = cros_ec_cec_remove,
+	.remove_new = cros_ec_cec_remove,
 	.driver = {
 		.name = DRV_NAME,
 		.pm = &cros_ec_cec_pm_ops,
diff --git a/drivers/media/cec/platform/meson/ao-cec-g12a.c b/drivers/media/cec/platform/meson/ao-cec-g12a.c
index 68fe6d6a8178..51294b9b6cd5 100644
--- a/drivers/media/cec/platform/meson/ao-cec-g12a.c
+++ b/drivers/media/cec/platform/meson/ao-cec-g12a.c
@@ -744,7 +744,7 @@ out_probe_adapter:
 	return ret;
 }
 
-static int meson_ao_cec_g12a_remove(struct platform_device *pdev)
+static void meson_ao_cec_g12a_remove(struct platform_device *pdev)
 {
 	struct meson_ao_cec_g12a_device *ao_cec = platform_get_drvdata(pdev);
 
@@ -753,8 +753,6 @@ static int meson_ao_cec_g12a_remove(struct platform_device *pdev)
 	cec_notifier_cec_adap_unregister(ao_cec->notify, ao_cec->adap);
 
 	cec_unregister_adapter(ao_cec->adap);
-
-	return 0;
 }
 
 static const struct meson_ao_cec_g12a_data ao_cec_g12a_data = {
@@ -780,7 +778,7 @@ MODULE_DEVICE_TABLE(of, meson_ao_cec_g12a_of_match);
 
 static struct platform_driver meson_ao_cec_g12a_driver = {
 	.probe   = meson_ao_cec_g12a_probe,
-	.remove  = meson_ao_cec_g12a_remove,
+	.remove_new = meson_ao_cec_g12a_remove,
 	.driver  = {
 		.name = "meson-ao-cec-g12a",
 		.of_match_table = of_match_ptr(meson_ao_cec_g12a_of_match),
diff --git a/drivers/media/cec/platform/meson/ao-cec.c b/drivers/media/cec/platform/meson/ao-cec.c
index 6b440f0635d9..f6f51a34f7bd 100644
--- a/drivers/media/cec/platform/meson/ao-cec.c
+++ b/drivers/media/cec/platform/meson/ao-cec.c
@@ -696,7 +696,7 @@ out_probe_adapter:
 	return ret;
 }
 
-static int meson_ao_cec_remove(struct platform_device *pdev)
+static void meson_ao_cec_remove(struct platform_device *pdev)
 {
 	struct meson_ao_cec_device *ao_cec = platform_get_drvdata(pdev);
 
@@ -704,8 +704,6 @@ static int meson_ao_cec_remove(struct platform_device *pdev)
 
 	cec_notifier_cec_adap_unregister(ao_cec->notify, ao_cec->adap);
 	cec_unregister_adapter(ao_cec->adap);
-
-	return 0;
 }
 
 static const struct of_device_id meson_ao_cec_of_match[] = {
@@ -716,7 +714,7 @@ MODULE_DEVICE_TABLE(of, meson_ao_cec_of_match);
 
 static struct platform_driver meson_ao_cec_driver = {
 	.probe   = meson_ao_cec_probe,
-	.remove  = meson_ao_cec_remove,
+	.remove_new = meson_ao_cec_remove,
 	.driver  = {
 		.name = "meson-ao-cec",
 		.of_match_table = of_match_ptr(meson_ao_cec_of_match),
diff --git a/drivers/media/cec/platform/s5p/s5p_cec.c b/drivers/media/cec/platform/s5p/s5p_cec.c
index 0a30e7acdc10..51ab4a80aafe 100644
--- a/drivers/media/cec/platform/s5p/s5p_cec.c
+++ b/drivers/media/cec/platform/s5p/s5p_cec.c
@@ -249,14 +249,13 @@ err_delete_adapter:
 	return ret;
 }
 
-static int s5p_cec_remove(struct platform_device *pdev)
+static void s5p_cec_remove(struct platform_device *pdev)
 {
 	struct s5p_cec_dev *cec = platform_get_drvdata(pdev);
 
 	cec_notifier_cec_adap_unregister(cec->notifier, cec->adap);
 	cec_unregister_adapter(cec->adap);
 	pm_runtime_disable(&pdev->dev);
-	return 0;
 }
 
 static int __maybe_unused s5p_cec_runtime_suspend(struct device *dev)
@@ -295,7 +294,7 @@ MODULE_DEVICE_TABLE(of, s5p_cec_match);
 
 static struct platform_driver s5p_cec_pdrv = {
 	.probe	= s5p_cec_probe,
-	.remove	= s5p_cec_remove,
+	.remove_new = s5p_cec_remove,
 	.driver	= {
 		.name		= CEC_NAME,
 		.of_match_table	= s5p_cec_match,
diff --git a/drivers/media/cec/platform/seco/seco-cec.c b/drivers/media/cec/platform/seco/seco-cec.c
index 580905e3d066..5d4c5a2cae09 100644
--- a/drivers/media/cec/platform/seco/seco-cec.c
+++ b/drivers/media/cec/platform/seco/seco-cec.c
@@ -668,7 +668,7 @@ err:
 	return ret;
 }
 
-static int secocec_remove(struct platform_device *pdev)
+static void secocec_remove(struct platform_device *pdev)
 {
 	struct secocec_data *secocec = platform_get_drvdata(pdev);
 	u16 val;
@@ -686,8 +686,6 @@ static int secocec_remove(struct platform_device *pdev)
 	release_region(BRA_SMB_BASE_ADDR, 7);
 
 	dev_dbg(&pdev->dev, "CEC device removed\n");
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -780,7 +778,7 @@ static struct platform_driver secocec_driver = {
 		   .pm = SECOCEC_PM_OPS,
 	},
 	.probe = secocec_probe,
-	.remove = secocec_remove,
+	.remove_new = secocec_remove,
 };
 
 module_platform_driver(secocec_driver);
diff --git a/drivers/media/cec/platform/sti/stih-cec.c b/drivers/media/cec/platform/sti/stih-cec.c
index 4edbdd09535d..a20fc5c0c88d 100644
--- a/drivers/media/cec/platform/sti/stih-cec.c
+++ b/drivers/media/cec/platform/sti/stih-cec.c
@@ -364,14 +364,12 @@ err_delete_adapter:
 	return ret;
 }
 
-static int stih_cec_remove(struct platform_device *pdev)
+static void stih_cec_remove(struct platform_device *pdev)
 {
 	struct stih_cec *cec = platform_get_drvdata(pdev);
 
 	cec_notifier_cec_adap_unregister(cec->notifier, cec->adap);
 	cec_unregister_adapter(cec->adap);
-
-	return 0;
 }
 
 static const struct of_device_id stih_cec_match[] = {
@@ -384,7 +382,7 @@ MODULE_DEVICE_TABLE(of, stih_cec_match);
 
 static struct platform_driver stih_cec_pdrv = {
 	.probe	= stih_cec_probe,
-	.remove = stih_cec_remove,
+	.remove_new = stih_cec_remove,
 	.driver = {
 		.name		= CEC_NAME,
 		.of_match_table	= stih_cec_match,
diff --git a/drivers/media/cec/platform/stm32/stm32-cec.c b/drivers/media/cec/platform/stm32/stm32-cec.c
index 7b2db46a5722..ada3d153362a 100644
--- a/drivers/media/cec/platform/stm32/stm32-cec.c
+++ b/drivers/media/cec/platform/stm32/stm32-cec.c
@@ -344,7 +344,7 @@ err_unprepare_cec_clk:
 	return ret;
 }
 
-static int stm32_cec_remove(struct platform_device *pdev)
+static void stm32_cec_remove(struct platform_device *pdev)
 {
 	struct stm32_cec *cec = platform_get_drvdata(pdev);
 
@@ -352,8 +352,6 @@ static int stm32_cec_remove(struct platform_device *pdev)
 	clk_unprepare(cec->clk_hdmi_cec);
 
 	cec_unregister_adapter(cec->adap);
-
-	return 0;
 }
 
 static const struct of_device_id stm32_cec_of_match[] = {
@@ -364,7 +362,7 @@ MODULE_DEVICE_TABLE(of, stm32_cec_of_match);
 
 static struct platform_driver stm32_cec_driver = {
 	.probe  = stm32_cec_probe,
-	.remove = stm32_cec_remove,
+	.remove_new = stm32_cec_remove,
 	.driver = {
 		.name		= CEC_NAME,
 		.of_match_table = stm32_cec_of_match,
diff --git a/drivers/media/cec/platform/tegra/tegra_cec.c b/drivers/media/cec/platform/tegra/tegra_cec.c
index 5e907395ca2e..04dc06e3c42a 100644
--- a/drivers/media/cec/platform/tegra/tegra_cec.c
+++ b/drivers/media/cec/platform/tegra/tegra_cec.c
@@ -421,7 +421,7 @@ err_clk:
 	return ret;
 }
 
-static int tegra_cec_remove(struct platform_device *pdev)
+static void tegra_cec_remove(struct platform_device *pdev)
 {
 	struct tegra_cec *cec = platform_get_drvdata(pdev);
 
@@ -429,8 +429,6 @@ static int tegra_cec_remove(struct platform_device *pdev)
 
 	cec_notifier_cec_adap_unregister(cec->notifier, cec->adap);
 	cec_unregister_adapter(cec->adap);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -467,7 +465,7 @@ static struct platform_driver tegra_cec_driver = {
 		.of_match_table = of_match_ptr(tegra_cec_of_match),
 	},
 	.probe = tegra_cec_probe,
-	.remove = tegra_cec_remove,
+	.remove_new = tegra_cec_remove,
 
 #ifdef CONFIG_PM
 	.suspend = tegra_cec_suspend,
diff --git a/drivers/media/common/btcx-risc.h b/drivers/media/common/btcx-risc.h
deleted file mode 100644
index 76dc16f402b9..000000000000
--- a/drivers/media/common/btcx-risc.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- */
-struct btcx_riscmem {
-	unsigned int   size;
-	__le32         *cpu;
-	__le32         *jmp;
-	dma_addr_t     dma;
-};
-
-struct btcx_skiplist {
-	int start;
-	int end;
-};
-
-int  btcx_riscmem_alloc(struct pci_dev *pci,
-			struct btcx_riscmem *risc,
-			unsigned int size);
-void btcx_riscmem_free(struct pci_dev *pci,
-		       struct btcx_riscmem *risc);
-
-int btcx_screen_clips(int swidth, int sheight, struct v4l2_rect *win,
-		      struct v4l2_clip *clips, unsigned int n);
-int btcx_align(struct v4l2_rect *win, struct v4l2_clip *clips,
-	       unsigned int n, int mask);
-void btcx_sort_clips(struct v4l2_clip *clips, unsigned int nclips);
-void btcx_calc_skips(int line, int width, int *maxy,
-		     struct btcx_skiplist *skips, unsigned int *nskips,
-		     const struct v4l2_clip *clips, unsigned int nclips);
diff --git a/drivers/media/common/saa7146/Kconfig b/drivers/media/common/saa7146/Kconfig
index a0aa155e5d85..dfec86e50dff 100644
--- a/drivers/media/common/saa7146/Kconfig
+++ b/drivers/media/common/saa7146/Kconfig
@@ -6,5 +6,5 @@ config VIDEO_SAA7146
 config VIDEO_SAA7146_VV
 	tristate
 	depends on VIDEO_DEV
-	select VIDEOBUF_DMA_SG
+	select VIDEOBUF2_DMA_SG
 	select VIDEO_SAA7146
diff --git a/drivers/media/common/saa7146/saa7146_core.c b/drivers/media/common/saa7146/saa7146_core.c
index e50fa0ff7c5d..bcb957883044 100644
--- a/drivers/media/common/saa7146/saa7146_core.c
+++ b/drivers/media/common/saa7146/saa7146_core.c
@@ -37,7 +37,8 @@ void saa7146_setgpio(struct saa7146_dev *dev, int port, u32 data)
 {
 	u32 value = 0;
 
-	BUG_ON(port > 3);
+	if (WARN_ON(port > 3))
+		return;
 
 	value = saa7146_read(dev, GPIO_CTRL);
 	value &= ~(0xff << (8*port));
@@ -148,7 +149,8 @@ static struct scatterlist* vmalloc_to_sg(unsigned char *virt, int nr_pages)
 		pg = vmalloc_to_page(virt);
 		if (NULL == pg)
 			goto err;
-		BUG_ON(PageHighMem(pg));
+		if (WARN_ON(PageHighMem(pg)))
+			return NULL;
 		sg_set_page(&sglist[i], pg, PAGE_SIZE, 0);
 	}
 	return sglist;
@@ -233,46 +235,32 @@ int saa7146_pgtable_alloc(struct pci_dev *pci, struct saa7146_pgtable *pt)
 }
 
 int saa7146_pgtable_build_single(struct pci_dev *pci, struct saa7146_pgtable *pt,
-	struct scatterlist *list, int sglen  )
+				 struct scatterlist *list, int sglen)
 {
+	struct sg_dma_page_iter dma_iter;
 	__le32 *ptr, fill;
 	int nr_pages = 0;
-	int i,p;
+	int i;
 
-	BUG_ON(0 == sglen);
-	BUG_ON(list->offset > PAGE_SIZE);
+	if (WARN_ON(!sglen) ||
+	    WARN_ON(list->offset > PAGE_SIZE))
+		return -EIO;
 
 	/* if we have a user buffer, the first page may not be
 	   aligned to a page boundary. */
 	pt->offset = list->offset;
 
 	ptr = pt->cpu;
-	for (i = 0; i < sglen; i++, list++) {
-/*
-		pr_debug("i:%d, adr:0x%08x, len:%d, offset:%d\n",
-			 i, sg_dma_address(list), sg_dma_len(list),
-			 list->offset);
-*/
-		for (p = 0; p * 4096 < sg_dma_len(list); p++, ptr++) {
-			*ptr = cpu_to_le32(sg_dma_address(list) + p * 4096);
-			nr_pages++;
-		}
+	for_each_sg_dma_page(list, &dma_iter, sglen, 0) {
+		*ptr++ = cpu_to_le32(sg_page_iter_dma_address(&dma_iter));
+		nr_pages++;
 	}
 
 
 	/* safety; fill the page table up with the last valid page */
 	fill = *(ptr-1);
-	for(i=nr_pages;i<1024;i++) {
+	for (i = nr_pages; i < 1024; i++)
 		*ptr++ = fill;
-	}
-
-/*
-	ptr = pt->cpu;
-	pr_debug("offset: %d\n", pt->offset);
-	for(i=0;i<5;i++) {
-		pr_debug("ptr1 %d: 0x%08x\n", i, ptr[i]);
-	}
-*/
 	return 0;
 }
 
diff --git a/drivers/media/common/saa7146/saa7146_fops.c b/drivers/media/common/saa7146/saa7146_fops.c
index e9a15de6126e..79214459387a 100644
--- a/drivers/media/common/saa7146/saa7146_fops.c
+++ b/drivers/media/common/saa7146/saa7146_fops.c
@@ -7,12 +7,11 @@
 /****************************************************************************/
 /* resource management functions, shamelessly stolen from saa7134 driver */
 
-int saa7146_res_get(struct saa7146_fh *fh, unsigned int bit)
+int saa7146_res_get(struct saa7146_dev *dev, unsigned int bit)
 {
-	struct saa7146_dev *dev = fh->dev;
 	struct saa7146_vv *vv = dev->vv_data;
 
-	if (fh->resources & bit) {
+	if (vv->resources & bit) {
 		DEB_D("already allocated! want: 0x%02x, cur:0x%02x\n",
 		      bit, vv->resources);
 		/* have it already allocated */
@@ -27,42 +26,23 @@ int saa7146_res_get(struct saa7146_fh *fh, unsigned int bit)
 		return 0;
 	}
 	/* it's free, grab it */
-	fh->resources |= bit;
 	vv->resources |= bit;
 	DEB_D("res: get 0x%02x, cur:0x%02x\n", bit, vv->resources);
 	return 1;
 }
 
-void saa7146_res_free(struct saa7146_fh *fh, unsigned int bits)
+void saa7146_res_free(struct saa7146_dev *dev, unsigned int bits)
 {
-	struct saa7146_dev *dev = fh->dev;
 	struct saa7146_vv *vv = dev->vv_data;
 
-	BUG_ON((fh->resources & bits) != bits);
+	WARN_ON((vv->resources & bits) != bits);
 
-	fh->resources &= ~bits;
 	vv->resources &= ~bits;
 	DEB_D("res: put 0x%02x, cur:0x%02x\n", bits, vv->resources);
 }
 
 
 /********************************************************************************/
-/* common dma functions */
-
-void saa7146_dma_free(struct saa7146_dev *dev,struct videobuf_queue *q,
-						struct saa7146_buf *buf)
-{
-	struct videobuf_dmabuf *dma=videobuf_to_dma(&buf->vb);
-	DEB_EE("dev:%p, buf:%p\n", dev, buf);
-
-	videobuf_waiton(q, &buf->vb, 0, 0);
-	videobuf_dma_unmap(q->dev, dma);
-	videobuf_dma_free(dma);
-	buf->vb.state = VIDEOBUF_NEEDS_INIT;
-}
-
-
-/********************************************************************************/
 /* common buffer functions */
 
 int saa7146_buffer_queue(struct saa7146_dev *dev,
@@ -72,15 +52,15 @@ int saa7146_buffer_queue(struct saa7146_dev *dev,
 	assert_spin_locked(&dev->slock);
 	DEB_EE("dev:%p, dmaq:%p, buf:%p\n", dev, q, buf);
 
-	BUG_ON(!q);
+	if (WARN_ON(!q))
+		return -EIO;
 
 	if (NULL == q->curr) {
 		q->curr = buf;
 		DEB_D("immediately activating buffer %p\n", buf);
 		buf->activate(dev,buf,NULL);
 	} else {
-		list_add_tail(&buf->vb.queue,&q->queue);
-		buf->vb.state = VIDEOBUF_QUEUED;
+		list_add_tail(&buf->list, &q->queue);
 		DEB_D("adding buffer %p to queue. (active buffer present)\n",
 		      buf);
 	}
@@ -91,21 +71,31 @@ void saa7146_buffer_finish(struct saa7146_dev *dev,
 			   struct saa7146_dmaqueue *q,
 			   int state)
 {
+	struct saa7146_vv *vv = dev->vv_data;
+	struct saa7146_buf *buf = q->curr;
+
 	assert_spin_locked(&dev->slock);
 	DEB_EE("dev:%p, dmaq:%p, state:%d\n", dev, q, state);
 	DEB_EE("q->curr:%p\n", q->curr);
 
 	/* finish current buffer */
-	if (NULL == q->curr) {
+	if (!buf) {
 		DEB_D("aiii. no current buffer\n");
 		return;
 	}
 
-	q->curr->vb.state = state;
-	q->curr->vb.ts = ktime_get_ns();
-	wake_up(&q->curr->vb.done);
-
 	q->curr = NULL;
+	buf->vb.vb2_buf.timestamp = ktime_get_ns();
+	if (vv->video_fmt.field == V4L2_FIELD_ALTERNATE)
+		buf->vb.field = vv->last_field;
+	else if (vv->video_fmt.field == V4L2_FIELD_ANY)
+		buf->vb.field = (vv->video_fmt.height > vv->standard->v_max_out / 2)
+			? V4L2_FIELD_INTERLACED
+			: V4L2_FIELD_BOTTOM;
+	else
+		buf->vb.field = vv->video_fmt.field;
+	buf->vb.sequence = vv->seqnr++;
+	vb2_buffer_done(&buf->vb.vb2_buf, state);
 }
 
 void saa7146_buffer_next(struct saa7146_dev *dev,
@@ -113,17 +103,18 @@ void saa7146_buffer_next(struct saa7146_dev *dev,
 {
 	struct saa7146_buf *buf,*next = NULL;
 
-	BUG_ON(!q);
+	if (WARN_ON(!q))
+		return;
 
 	DEB_INT("dev:%p, dmaq:%p, vbi:%d\n", dev, q, vbi);
 
 	assert_spin_locked(&dev->slock);
 	if (!list_empty(&q->queue)) {
 		/* activate next one from queue */
-		buf = list_entry(q->queue.next,struct saa7146_buf,vb.queue);
-		list_del(&buf->vb.queue);
+		buf = list_entry(q->queue.next, struct saa7146_buf, list);
+		list_del(&buf->list);
 		if (!list_empty(&q->queue))
-			next = list_entry(q->queue.next,struct saa7146_buf, vb.queue);
+			next = list_entry(q->queue.next, struct saa7146_buf, list);
 		q->curr = buf;
 		DEB_INT("next buffer: buf:%p, prev:%p, next:%p\n",
 			buf, q->queue.prev, q->queue.next);
@@ -171,7 +162,7 @@ void saa7146_buffer_timeout(struct timer_list *t)
 	spin_lock_irqsave(&dev->slock,flags);
 	if (q->curr) {
 		DEB_D("timeout on %p\n", q->curr);
-		saa7146_buffer_finish(dev,q,VIDEOBUF_ERROR);
+		saa7146_buffer_finish(dev, q, VB2_BUF_STATE_ERROR);
 	}
 
 	/* we don't restart the transfer here like other drivers do. when
@@ -180,256 +171,39 @@ void saa7146_buffer_timeout(struct timer_list *t)
 	   we mess up our capture logic. if a timeout occurs on another buffer,
 	   then something is seriously broken before, so no need to buffer the
 	   next capture IMHO... */
-/*
-	saa7146_buffer_next(dev,q);
-*/
+
+	saa7146_buffer_next(dev, q, 0);
+
 	spin_unlock_irqrestore(&dev->slock,flags);
 }
 
 /********************************************************************************/
 /* file operations */
 
-static int fops_open(struct file *file)
+static ssize_t fops_write(struct file *file, const char __user *data, size_t count, loff_t *ppos)
 {
 	struct video_device *vdev = video_devdata(file);
 	struct saa7146_dev *dev = video_drvdata(file);
-	struct saa7146_fh *fh = NULL;
-	int result = 0;
-
-	DEB_EE("file:%p, dev:%s\n", file, video_device_node_name(vdev));
-
-	if (mutex_lock_interruptible(vdev->lock))
-		return -ERESTARTSYS;
-
-	DEB_D("using: %p\n", dev);
-
-	/* check if an extension is registered */
-	if( NULL == dev->ext ) {
-		DEB_S("no extension registered for this device\n");
-		result = -ENODEV;
-		goto out;
-	}
-
-	/* allocate per open data */
-	fh = kzalloc(sizeof(*fh),GFP_KERNEL);
-	if (NULL == fh) {
-		DEB_S("cannot allocate memory for per open data\n");
-		result = -ENOMEM;
-		goto out;
-	}
-
-	v4l2_fh_init(&fh->fh, vdev);
-
-	file->private_data = &fh->fh;
-	fh->dev = dev;
-
-	if (vdev->vfl_type == VFL_TYPE_VBI) {
-		DEB_S("initializing vbi...\n");
-		if (dev->ext_vv_data->capabilities & V4L2_CAP_VBI_CAPTURE)
-			result = saa7146_vbi_uops.open(dev,file);
-		if (dev->ext_vv_data->vbi_fops.open)
-			dev->ext_vv_data->vbi_fops.open(file);
-	} else {
-		DEB_S("initializing video...\n");
-		result = saa7146_video_uops.open(dev,file);
-	}
-
-	if (0 != result) {
-		goto out;
-	}
-
-	if( 0 == try_module_get(dev->ext->module)) {
-		result = -EINVAL;
-		goto out;
-	}
-
-	result = 0;
-	v4l2_fh_add(&fh->fh);
-out:
-	if (fh && result != 0) {
-		kfree(fh);
-		file->private_data = NULL;
-	}
-	mutex_unlock(vdev->lock);
-	return result;
-}
-
-static int fops_release(struct file *file)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct saa7146_fh  *fh  = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-
-	DEB_EE("file:%p\n", file);
-
-	mutex_lock(vdev->lock);
-
-	if (vdev->vfl_type == VFL_TYPE_VBI) {
-		if (dev->ext_vv_data->capabilities & V4L2_CAP_VBI_CAPTURE)
-			saa7146_vbi_uops.release(dev,file);
-		if (dev->ext_vv_data->vbi_fops.release)
-			dev->ext_vv_data->vbi_fops.release(file);
-	} else {
-		saa7146_video_uops.release(dev,file);
-	}
-
-	v4l2_fh_del(&fh->fh);
-	v4l2_fh_exit(&fh->fh);
-	module_put(dev->ext->module);
-	file->private_data = NULL;
-	kfree(fh);
-
-	mutex_unlock(vdev->lock);
-
-	return 0;
-}
-
-static int fops_mmap(struct file *file, struct vm_area_struct * vma)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct saa7146_fh *fh = file->private_data;
-	struct videobuf_queue *q;
-	int res;
-
-	switch (vdev->vfl_type) {
-	case VFL_TYPE_VIDEO: {
-		DEB_EE("V4L2_BUF_TYPE_VIDEO_CAPTURE: file:%p, vma:%p\n",
-		       file, vma);
-		q = &fh->video_q;
-		break;
-		}
-	case VFL_TYPE_VBI: {
-		DEB_EE("V4L2_BUF_TYPE_VBI_CAPTURE: file:%p, vma:%p\n",
-		       file, vma);
-		if (fh->dev->ext_vv_data->capabilities & V4L2_CAP_SLICED_VBI_OUTPUT)
-			return -ENODEV;
-		q = &fh->vbi_q;
-		break;
-		}
-	default:
-		BUG();
-	}
+	int ret;
 
+	if (vdev->vfl_type != VFL_TYPE_VBI || !dev->ext_vv_data->vbi_fops.write)
+		return -EINVAL;
 	if (mutex_lock_interruptible(vdev->lock))
 		return -ERESTARTSYS;
-	res = videobuf_mmap_mapper(q, vma);
+	ret = dev->ext_vv_data->vbi_fops.write(file, data, count, ppos);
 	mutex_unlock(vdev->lock);
-	return res;
-}
-
-static __poll_t __fops_poll(struct file *file, struct poll_table_struct *wait)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct saa7146_fh *fh = file->private_data;
-	struct videobuf_buffer *buf = NULL;
-	struct videobuf_queue *q;
-	__poll_t res = v4l2_ctrl_poll(file, wait);
-
-	DEB_EE("file:%p, poll:%p\n", file, wait);
-
-	if (vdev->vfl_type == VFL_TYPE_VBI) {
-		if (fh->dev->ext_vv_data->capabilities & V4L2_CAP_SLICED_VBI_OUTPUT)
-			return res | EPOLLOUT | EPOLLWRNORM;
-		if( 0 == fh->vbi_q.streaming )
-			return res | videobuf_poll_stream(file, &fh->vbi_q, wait);
-		q = &fh->vbi_q;
-	} else {
-		DEB_D("using video queue\n");
-		q = &fh->video_q;
-	}
-
-	if (!list_empty(&q->stream))
-		buf = list_entry(q->stream.next, struct videobuf_buffer, stream);
-
-	if (!buf) {
-		DEB_D("buf == NULL!\n");
-		return res | EPOLLERR;
-	}
-
-	poll_wait(file, &buf->done, wait);
-	if (buf->state == VIDEOBUF_DONE || buf->state == VIDEOBUF_ERROR) {
-		DEB_D("poll succeeded!\n");
-		return res | EPOLLIN | EPOLLRDNORM;
-	}
-
-	DEB_D("nothing to poll for, buf->state:%d\n", buf->state);
-	return res;
-}
-
-static __poll_t fops_poll(struct file *file, struct poll_table_struct *wait)
-{
-	struct video_device *vdev = video_devdata(file);
-	__poll_t res;
-
-	mutex_lock(vdev->lock);
-	res = __fops_poll(file, wait);
-	mutex_unlock(vdev->lock);
-	return res;
-}
-
-static ssize_t fops_read(struct file *file, char __user *data, size_t count, loff_t *ppos)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct saa7146_fh *fh = file->private_data;
-	int ret;
-
-	switch (vdev->vfl_type) {
-	case VFL_TYPE_VIDEO:
-/*
-		DEB_EE("V4L2_BUF_TYPE_VIDEO_CAPTURE: file:%p, data:%p, count:%lun",
-		       file, data, (unsigned long)count);
-*/
-		return saa7146_video_uops.read(file,data,count,ppos);
-	case VFL_TYPE_VBI:
-/*
-		DEB_EE("V4L2_BUF_TYPE_VBI_CAPTURE: file:%p, data:%p, count:%lu\n",
-		       file, data, (unsigned long)count);
-*/
-		if (fh->dev->ext_vv_data->capabilities & V4L2_CAP_VBI_CAPTURE) {
-			if (mutex_lock_interruptible(vdev->lock))
-				return -ERESTARTSYS;
-			ret = saa7146_vbi_uops.read(file, data, count, ppos);
-			mutex_unlock(vdev->lock);
-			return ret;
-		}
-		return -EINVAL;
-	default:
-		BUG();
-	}
-}
-
-static ssize_t fops_write(struct file *file, const char __user *data, size_t count, loff_t *ppos)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct saa7146_fh *fh = file->private_data;
-	int ret;
-
-	switch (vdev->vfl_type) {
-	case VFL_TYPE_VIDEO:
-		return -EINVAL;
-	case VFL_TYPE_VBI:
-		if (fh->dev->ext_vv_data->vbi_fops.write) {
-			if (mutex_lock_interruptible(vdev->lock))
-				return -ERESTARTSYS;
-			ret = fh->dev->ext_vv_data->vbi_fops.write(file, data, count, ppos);
-			mutex_unlock(vdev->lock);
-			return ret;
-		}
-		return -EINVAL;
-	default:
-		BUG();
-	}
+	return ret;
 }
 
 static const struct v4l2_file_operations video_fops =
 {
 	.owner		= THIS_MODULE,
-	.open		= fops_open,
-	.release	= fops_release,
-	.read		= fops_read,
+	.open		= v4l2_fh_open,
+	.release	= vb2_fop_release,
+	.read		= vb2_fop_read,
 	.write		= fops_write,
-	.poll		= fops_poll,
-	.mmap		= fops_mmap,
+	.poll		= vb2_fop_poll,
+	.mmap		= vb2_fop_mmap,
 	.unlocked_ioctl	= video_ioctl2,
 };
 
@@ -516,33 +290,15 @@ int saa7146_vv_init(struct saa7146_dev* dev, struct saa7146_ext_vv *ext_vv)
 	   configuration data) */
 	dev->ext_vv_data = ext_vv;
 
-	vv->d_clipping.cpu_addr =
-		dma_alloc_coherent(&dev->pci->dev, SAA7146_CLIPPING_MEM,
-				   &vv->d_clipping.dma_handle, GFP_KERNEL);
-	if( NULL == vv->d_clipping.cpu_addr ) {
-		ERR("out of memory. aborting.\n");
-		kfree(vv);
-		v4l2_ctrl_handler_free(hdl);
-		v4l2_device_unregister(&dev->v4l2_dev);
-		return -ENOMEM;
-	}
-
 	saa7146_video_uops.init(dev,vv);
 	if (dev->ext_vv_data->capabilities & V4L2_CAP_VBI_CAPTURE)
 		saa7146_vbi_uops.init(dev,vv);
 
-	vv->ov_fb.fmt.width = vv->standard->h_max_out;
-	vv->ov_fb.fmt.height = vv->standard->v_max_out;
-	vv->ov_fb.fmt.pixelformat = V4L2_PIX_FMT_RGB565;
-	vv->ov_fb.fmt.bytesperline = 2 * vv->ov_fb.fmt.width;
-	vv->ov_fb.fmt.sizeimage = vv->ov_fb.fmt.bytesperline * vv->ov_fb.fmt.height;
-	vv->ov_fb.fmt.colorspace = V4L2_COLORSPACE_SRGB;
-
 	fmt = &vv->video_fmt;
 	fmt->width = 384;
 	fmt->height = 288;
 	fmt->pixelformat = V4L2_PIX_FMT_BGR24;
-	fmt->field = V4L2_FIELD_ANY;
+	fmt->field = V4L2_FIELD_INTERLACED;
 	fmt->colorspace = V4L2_COLORSPACE_SMPTE170M;
 	fmt->bytesperline = 3 * fmt->width;
 	fmt->sizeimage = fmt->bytesperline * fmt->height;
@@ -561,8 +317,6 @@ int saa7146_vv_init(struct saa7146_dev* dev, struct saa7146_ext_vv *ext_vv)
 
 	timer_setup(&vv->vbi_read_timeout, NULL, 0);
 
-	vv->ov_fb.capability = V4L2_FBUF_CAP_LIST_CLIPPING;
-	vv->ov_fb.flags = V4L2_FBUF_FLAG_PRIMARY;
 	dev->vv_data = vv;
 	dev->vv_callback = &vv_callback;
 
@@ -577,8 +331,6 @@ int saa7146_vv_release(struct saa7146_dev* dev)
 	DEB_EE("dev:%p\n", dev);
 
 	v4l2_device_unregister(&dev->v4l2_dev);
-	dma_free_coherent(&dev->pci->dev, SAA7146_CLIPPING_MEM,
-			  vv->d_clipping.cpu_addr, vv->d_clipping.dma_handle);
 	v4l2_ctrl_handler_free(&dev->ctrl_handler);
 	kfree(vv);
 	dev->vv_data = NULL;
@@ -591,16 +343,20 @@ EXPORT_SYMBOL_GPL(saa7146_vv_release);
 int saa7146_register_device(struct video_device *vfd, struct saa7146_dev *dev,
 			    char *name, int type)
 {
+	struct vb2_queue *q;
 	int err;
 	int i;
 
 	DEB_EE("dev:%p, name:'%s', type:%d\n", dev, name, type);
 
 	vfd->fops = &video_fops;
-	if (type == VFL_TYPE_VIDEO)
+	if (type == VFL_TYPE_VIDEO) {
 		vfd->ioctl_ops = &dev->ext_vv_data->vid_ops;
-	else
+		q = &dev->vv_data->video_dmaq.q;
+	} else {
 		vfd->ioctl_ops = &dev->ext_vv_data->vbi_ops;
+		q = &dev->vv_data->vbi_dmaq.q;
+	}
 	vfd->release = video_device_release_empty;
 	vfd->lock = &dev->v4l2_lock;
 	vfd->v4l2_dev = &dev->v4l2_dev;
@@ -608,15 +364,36 @@ int saa7146_register_device(struct video_device *vfd, struct saa7146_dev *dev,
 	for (i = 0; i < dev->ext_vv_data->num_stds; i++)
 		vfd->tvnorms |= dev->ext_vv_data->stds[i].id;
 	strscpy(vfd->name, name, sizeof(vfd->name));
-	vfd->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_OVERLAY |
+	vfd->device_caps = V4L2_CAP_VIDEO_CAPTURE |
 			   V4L2_CAP_READWRITE | V4L2_CAP_STREAMING;
 	vfd->device_caps |= dev->ext_vv_data->capabilities;
-	if (type == VFL_TYPE_VIDEO)
+	if (type == VFL_TYPE_VIDEO) {
 		vfd->device_caps &=
 			~(V4L2_CAP_VBI_CAPTURE | V4L2_CAP_SLICED_VBI_OUTPUT);
-	else
-		vfd->device_caps &=
-			~(V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_OVERLAY | V4L2_CAP_AUDIO);
+	} else if (vfd->device_caps & V4L2_CAP_SLICED_VBI_OUTPUT) {
+		vfd->vfl_dir = VFL_DIR_TX;
+		vfd->device_caps &= ~(V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING |
+				      V4L2_CAP_AUDIO | V4L2_CAP_TUNER);
+	} else {
+		vfd->device_caps &= ~V4L2_CAP_VIDEO_CAPTURE;
+	}
+
+	q->type = type == VFL_TYPE_VIDEO ? V4L2_BUF_TYPE_VIDEO_CAPTURE : V4L2_BUF_TYPE_VBI_CAPTURE;
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->io_modes = VB2_MMAP | VB2_READ | VB2_DMABUF;
+	q->ops = type == VFL_TYPE_VIDEO ? &video_qops : &vbi_qops;
+	q->mem_ops = &vb2_dma_sg_memops;
+	q->drv_priv = dev;
+	q->gfp_flags = __GFP_DMA32;
+	q->buf_struct_size = sizeof(struct saa7146_buf);
+	q->lock = &dev->v4l2_lock;
+	q->min_buffers_needed = 2;
+	q->dev = &dev->pci->dev;
+	err = vb2_queue_init(q);
+	if (err)
+		return err;
+	vfd->queue = q;
+
 	video_set_drvdata(vfd, dev);
 
 	err = video_register_device(vfd, type, -1);
diff --git a/drivers/media/common/saa7146/saa7146_hlp.c b/drivers/media/common/saa7146/saa7146_hlp.c
index 6c9946a402ee..7569d8cdd4d8 100644
--- a/drivers/media/common/saa7146/saa7146_hlp.c
+++ b/drivers/media/common/saa7146/saa7146_hlp.c
@@ -306,221 +306,6 @@ static int calculate_v_scale_registers(struct saa7146_dev *dev, enum v4l2_field
 }
 
 /* simple bubble-sort algorithm with duplicate elimination */
-static int sort_and_eliminate(u32* values, int* count)
-{
-	int low = 0, high = 0, top = 0;
-	int cur = 0, next = 0;
-
-	/* sanity checks */
-	if( (0 > *count) || (NULL == values) ) {
-		return -EINVAL;
-	}
-
-	/* bubble sort the first @count items of the array @values */
-	for( top = *count; top > 0; top--) {
-		for( low = 0, high = 1; high < top; low++, high++) {
-			if( values[low] > values[high] )
-				swap(values[low], values[high]);
-		}
-	}
-
-	/* remove duplicate items */
-	for( cur = 0, next = 1; next < *count; next++) {
-		if( values[cur] != values[next])
-			values[++cur] = values[next];
-	}
-
-	*count = cur + 1;
-
-	return 0;
-}
-
-static void calculate_clipping_registers_rect(struct saa7146_dev *dev, struct saa7146_fh *fh,
-	struct saa7146_video_dma *vdma2, u32* clip_format, u32* arbtr_ctrl, enum v4l2_field field)
-{
-	struct saa7146_vv *vv = dev->vv_data;
-	__le32 *clipping = vv->d_clipping.cpu_addr;
-
-	int width = vv->ov.win.w.width;
-	int height =  vv->ov.win.w.height;
-	int clipcount = vv->ov.nclips;
-
-	u32 line_list[32];
-	u32 pixel_list[32];
-	int numdwords = 0;
-
-	int i = 0, j = 0;
-	int cnt_line = 0, cnt_pixel = 0;
-
-	int x[32], y[32], w[32], h[32];
-
-	/* clear out memory */
-	memset(&line_list[0],  0x00, sizeof(u32)*32);
-	memset(&pixel_list[0], 0x00, sizeof(u32)*32);
-	memset(clipping,  0x00, SAA7146_CLIPPING_MEM);
-
-	/* fill the line and pixel-lists */
-	for(i = 0; i < clipcount; i++) {
-		int l = 0, r = 0, t = 0, b = 0;
-
-		x[i] = vv->ov.clips[i].c.left;
-		y[i] = vv->ov.clips[i].c.top;
-		w[i] = vv->ov.clips[i].c.width;
-		h[i] = vv->ov.clips[i].c.height;
-
-		if( w[i] < 0) {
-			x[i] += w[i]; w[i] = -w[i];
-		}
-		if( h[i] < 0) {
-			y[i] += h[i]; h[i] = -h[i];
-		}
-		if( x[i] < 0) {
-			w[i] += x[i]; x[i] = 0;
-		}
-		if( y[i] < 0) {
-			h[i] += y[i]; y[i] = 0;
-		}
-		if( 0 != vv->vflip ) {
-			y[i] = height - y[i] - h[i];
-		}
-
-		l = x[i];
-		r = x[i]+w[i];
-		t = y[i];
-		b = y[i]+h[i];
-
-		/* insert left/right coordinates */
-		pixel_list[ 2*i   ] = min_t(int, l, width);
-		pixel_list[(2*i)+1] = min_t(int, r, width);
-		/* insert top/bottom coordinates */
-		line_list[ 2*i   ] = min_t(int, t, height);
-		line_list[(2*i)+1] = min_t(int, b, height);
-	}
-
-	/* sort and eliminate lists */
-	cnt_line = cnt_pixel = 2*clipcount;
-	sort_and_eliminate( &pixel_list[0], &cnt_pixel );
-	sort_and_eliminate( &line_list[0], &cnt_line );
-
-	/* calculate the number of used u32s */
-	numdwords = max_t(int, (cnt_line+1), (cnt_pixel+1))*2;
-	numdwords = max_t(int, 4, numdwords);
-	numdwords = min_t(int, 64, numdwords);
-
-	/* fill up cliptable */
-	for(i = 0; i < cnt_pixel; i++) {
-		clipping[2*i] |= cpu_to_le32(pixel_list[i] << 16);
-	}
-	for(i = 0; i < cnt_line; i++) {
-		clipping[(2*i)+1] |= cpu_to_le32(line_list[i] << 16);
-	}
-
-	/* fill up cliptable with the display infos */
-	for(j = 0; j < clipcount; j++) {
-
-		for(i = 0; i < cnt_pixel; i++) {
-
-			if( x[j] < 0)
-				x[j] = 0;
-
-			if( pixel_list[i] < (x[j] + w[j])) {
-
-				if ( pixel_list[i] >= x[j] ) {
-					clipping[2*i] |= cpu_to_le32(1 << j);
-				}
-			}
-		}
-		for(i = 0; i < cnt_line; i++) {
-
-			if( y[j] < 0)
-				y[j] = 0;
-
-			if( line_list[i] < (y[j] + h[j]) ) {
-
-				if( line_list[i] >= y[j] ) {
-					clipping[(2*i)+1] |= cpu_to_le32(1 << j);
-				}
-			}
-		}
-	}
-
-	/* adjust arbitration control register */
-	*arbtr_ctrl &= 0xffff00ff;
-	*arbtr_ctrl |= 0x00001c00;
-
-	vdma2->base_even	= vv->d_clipping.dma_handle;
-	vdma2->base_odd		= vv->d_clipping.dma_handle;
-	vdma2->prot_addr	= vv->d_clipping.dma_handle+((sizeof(u32))*(numdwords));
-	vdma2->base_page	= 0x04;
-	vdma2->pitch		= 0x00;
-	vdma2->num_line_byte	= (0 << 16 | (sizeof(u32))*(numdwords-1) );
-
-	/* set clipping-mode. this depends on the field(s) used */
-	*clip_format &= 0xfffffff7;
-	if (V4L2_FIELD_HAS_BOTH(field)) {
-		*clip_format |= 0x00000008;
-	} else {
-		*clip_format |= 0x00000000;
-	}
-}
-
-/* disable clipping */
-static void saa7146_disable_clipping(struct saa7146_dev *dev)
-{
-	u32 clip_format	= saa7146_read(dev, CLIP_FORMAT_CTRL);
-
-	/* mask out relevant bits (=lower word)*/
-	clip_format &= MASK_W1;
-
-	/* upload clipping-registers*/
-	saa7146_write(dev, CLIP_FORMAT_CTRL,clip_format);
-	saa7146_write(dev, MC2, (MASK_05 | MASK_21));
-
-	/* disable video dma2 */
-	saa7146_write(dev, MC1, MASK_21);
-}
-
-static void saa7146_set_clipping_rect(struct saa7146_fh *fh)
-{
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	enum v4l2_field field = vv->ov.win.field;
-	struct	saa7146_video_dma vdma2;
-	u32 clip_format;
-	u32 arbtr_ctrl;
-
-	/* check clipcount, disable clipping if clipcount == 0*/
-	if (vv->ov.nclips == 0) {
-		saa7146_disable_clipping(dev);
-		return;
-	}
-
-	clip_format = saa7146_read(dev, CLIP_FORMAT_CTRL);
-	arbtr_ctrl = saa7146_read(dev, PCI_BT_V1);
-
-	calculate_clipping_registers_rect(dev, fh, &vdma2, &clip_format, &arbtr_ctrl, field);
-
-	/* set clipping format */
-	clip_format &= 0xffff0008;
-	clip_format |= (SAA7146_CLIPPING_RECT << 4);
-
-	/* prepare video dma2 */
-	saa7146_write(dev, BASE_EVEN2,		vdma2.base_even);
-	saa7146_write(dev, BASE_ODD2,		vdma2.base_odd);
-	saa7146_write(dev, PROT_ADDR2,		vdma2.prot_addr);
-	saa7146_write(dev, BASE_PAGE2,		vdma2.base_page);
-	saa7146_write(dev, PITCH2,		vdma2.pitch);
-	saa7146_write(dev, NUM_LINE_BYTE2,	vdma2.num_line_byte);
-
-	/* prepare the rest */
-	saa7146_write(dev, CLIP_FORMAT_CTRL,clip_format);
-	saa7146_write(dev, PCI_BT_V1, arbtr_ctrl);
-
-	/* upload clip_control-register, clipping-registers, enable video dma2 */
-	saa7146_write(dev, MC2, (MASK_05 | MASK_21 | MASK_03 | MASK_19));
-	saa7146_write(dev, MC1, (MASK_05 | MASK_21));
-}
-
 static void saa7146_set_window(struct saa7146_dev *dev, int width, int height, enum v4l2_field field)
 {
 	struct saa7146_vv *vv = dev->vv_data;
@@ -556,62 +341,6 @@ static void saa7146_set_window(struct saa7146_dev *dev, int width, int height, e
 	saa7146_write(dev, MC2, (MASK_05 | MASK_06 | MASK_21 | MASK_22) );
 }
 
-/* calculate the new memory offsets for a desired position */
-static void saa7146_set_position(struct saa7146_dev *dev, int w_x, int w_y, int w_height, enum v4l2_field field, u32 pixelformat)
-{
-	struct saa7146_vv *vv = dev->vv_data;
-	struct saa7146_format *sfmt = saa7146_format_by_fourcc(dev, pixelformat);
-
-	int b_depth = vv->ov_fmt->depth;
-	int b_bpl = vv->ov_fb.fmt.bytesperline;
-	/* The unsigned long cast is to remove a 64-bit compile warning since
-	   it looks like a 64-bit address is cast to a 32-bit value, even
-	   though the base pointer is really a 32-bit physical address that
-	   goes into a 32-bit DMA register.
-	   FIXME: might not work on some 64-bit platforms, but see the FIXME
-	   in struct v4l2_framebuffer (videodev2.h) for that.
-	 */
-	u32 base = (u32)(unsigned long)vv->ov_fb.base;
-
-	struct	saa7146_video_dma vdma1;
-
-	/* calculate memory offsets for picture, look if we shall top-down-flip */
-	vdma1.pitch	= 2*b_bpl;
-	if ( 0 == vv->vflip ) {
-		vdma1.base_even = base + (w_y * (vdma1.pitch/2)) + (w_x * (b_depth / 8));
-		vdma1.base_odd  = vdma1.base_even + (vdma1.pitch / 2);
-		vdma1.prot_addr = vdma1.base_even + (w_height * (vdma1.pitch / 2));
-	}
-	else {
-		vdma1.base_even = base + ((w_y+w_height) * (vdma1.pitch/2)) + (w_x * (b_depth / 8));
-		vdma1.base_odd  = vdma1.base_even - (vdma1.pitch / 2);
-		vdma1.prot_addr = vdma1.base_odd - (w_height * (vdma1.pitch / 2));
-	}
-
-	if (V4L2_FIELD_HAS_BOTH(field)) {
-	} else if (field == V4L2_FIELD_ALTERNATE) {
-		/* fixme */
-		vdma1.base_odd = vdma1.prot_addr;
-		vdma1.pitch /= 2;
-	} else if (field == V4L2_FIELD_TOP) {
-		vdma1.base_odd = vdma1.prot_addr;
-		vdma1.pitch /= 2;
-	} else if (field == V4L2_FIELD_BOTTOM) {
-		vdma1.base_odd = vdma1.base_even;
-		vdma1.base_even = vdma1.prot_addr;
-		vdma1.pitch /= 2;
-	}
-
-	if ( 0 != vv->vflip ) {
-		vdma1.pitch *= -1;
-	}
-
-	vdma1.base_page = sfmt->swap;
-	vdma1.num_line_byte = (vv->standard->v_field<<16)+vv->standard->h_pixels;
-
-	saa7146_write_out_dma(dev, 1, &vdma1);
-}
-
 static void saa7146_set_output_format(struct saa7146_dev *dev, unsigned long palette)
 {
 	u32 clip_format = saa7146_read(dev, CLIP_FORMAT_CTRL);
@@ -645,30 +374,6 @@ void saa7146_set_hps_source_and_sync(struct saa7146_dev *dev, int source, int sy
 }
 EXPORT_SYMBOL_GPL(saa7146_set_hps_source_and_sync);
 
-int saa7146_enable_overlay(struct saa7146_fh *fh)
-{
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-
-	saa7146_set_window(dev, vv->ov.win.w.width, vv->ov.win.w.height, vv->ov.win.field);
-	saa7146_set_position(dev, vv->ov.win.w.left, vv->ov.win.w.top, vv->ov.win.w.height, vv->ov.win.field, vv->ov_fmt->pixelformat);
-	saa7146_set_output_format(dev, vv->ov_fmt->trans);
-	saa7146_set_clipping_rect(fh);
-
-	/* enable video dma1 */
-	saa7146_write(dev, MC1, (MASK_06 | MASK_22));
-	return 0;
-}
-
-void saa7146_disable_overlay(struct saa7146_fh *fh)
-{
-	struct saa7146_dev *dev = fh->dev;
-
-	/* disable clipping + video dma1 */
-	saa7146_disable_clipping(dev);
-	saa7146_write(dev, MC1, MASK_22);
-}
-
 void saa7146_write_out_dma(struct saa7146_dev* dev, int which, struct saa7146_video_dma* vdma)
 {
 	int where = 0;
@@ -702,14 +407,14 @@ void saa7146_write_out_dma(struct saa7146_dev* dev, int which, struct saa7146_vi
 static int calculate_video_dma_grab_packed(struct saa7146_dev* dev, struct saa7146_buf *buf)
 {
 	struct saa7146_vv *vv = dev->vv_data;
+	struct v4l2_pix_format *pix = &vv->video_fmt;
 	struct saa7146_video_dma vdma1;
+	struct saa7146_format *sfmt = saa7146_format_by_fourcc(dev, pix->pixelformat);
 
-	struct saa7146_format *sfmt = saa7146_format_by_fourcc(dev,buf->fmt->pixelformat);
-
-	int width = buf->fmt->width;
-	int height = buf->fmt->height;
-	int bytesperline = buf->fmt->bytesperline;
-	enum v4l2_field field = buf->fmt->field;
+	int width = pix->width;
+	int height = pix->height;
+	int bytesperline = pix->bytesperline;
+	enum v4l2_field field = pix->field;
 
 	int depth = sfmt->depth;
 
@@ -764,8 +469,9 @@ static int calculate_video_dma_grab_packed(struct saa7146_dev* dev, struct saa71
 
 static int calc_planar_422(struct saa7146_vv *vv, struct saa7146_buf *buf, struct saa7146_video_dma *vdma2, struct saa7146_video_dma *vdma3)
 {
-	int height = buf->fmt->height;
-	int width = buf->fmt->width;
+	struct v4l2_pix_format *pix = &vv->video_fmt;
+	int height = pix->height;
+	int width = pix->width;
 
 	vdma2->pitch	= width;
 	vdma3->pitch	= width;
@@ -795,8 +501,9 @@ static int calc_planar_422(struct saa7146_vv *vv, struct saa7146_buf *buf, struc
 
 static int calc_planar_420(struct saa7146_vv *vv, struct saa7146_buf *buf, struct saa7146_video_dma *vdma2, struct saa7146_video_dma *vdma3)
 {
-	int height = buf->fmt->height;
-	int width = buf->fmt->width;
+	struct v4l2_pix_format *pix = &vv->video_fmt;
+	int height = pix->height;
+	int width = pix->width;
 
 	vdma2->pitch	= width/2;
 	vdma3->pitch	= width/2;
@@ -825,19 +532,20 @@ static int calc_planar_420(struct saa7146_vv *vv, struct saa7146_buf *buf, struc
 static int calculate_video_dma_grab_planar(struct saa7146_dev* dev, struct saa7146_buf *buf)
 {
 	struct saa7146_vv *vv = dev->vv_data;
+	struct v4l2_pix_format *pix = &vv->video_fmt;
 	struct saa7146_video_dma vdma1;
 	struct saa7146_video_dma vdma2;
 	struct saa7146_video_dma vdma3;
+	struct saa7146_format *sfmt = saa7146_format_by_fourcc(dev, pix->pixelformat);
 
-	struct saa7146_format *sfmt = saa7146_format_by_fourcc(dev,buf->fmt->pixelformat);
+	int width = pix->width;
+	int height = pix->height;
+	enum v4l2_field field = pix->field;
 
-	int width = buf->fmt->width;
-	int height = buf->fmt->height;
-	enum v4l2_field field = buf->fmt->field;
-
-	BUG_ON(0 == buf->pt[0].dma);
-	BUG_ON(0 == buf->pt[1].dma);
-	BUG_ON(0 == buf->pt[2].dma);
+	if (WARN_ON(!buf->pt[0].dma) ||
+	    WARN_ON(!buf->pt[1].dma) ||
+	    WARN_ON(!buf->pt[2].dma))
+		return -1;
 
 	DEB_CAP("[size=%dx%d,fields=%s]\n",
 		width, height, v4l2_field_names[field]);
@@ -994,10 +702,27 @@ static void program_capture_engine(struct saa7146_dev *dev, int planar)
 	WRITE_RPS0(CMD_STOP);
 }
 
+/* disable clipping */
+static void saa7146_disable_clipping(struct saa7146_dev *dev)
+{
+	u32 clip_format	= saa7146_read(dev, CLIP_FORMAT_CTRL);
+
+	/* mask out relevant bits (=lower word)*/
+	clip_format &= MASK_W1;
+
+	/* upload clipping-registers*/
+	saa7146_write(dev, CLIP_FORMAT_CTRL, clip_format);
+	saa7146_write(dev, MC2, (MASK_05 | MASK_21));
+
+	/* disable video dma2 */
+	saa7146_write(dev, MC1, MASK_21);
+}
+
 void saa7146_set_capture(struct saa7146_dev *dev, struct saa7146_buf *buf, struct saa7146_buf *next)
 {
-	struct saa7146_format *sfmt = saa7146_format_by_fourcc(dev,buf->fmt->pixelformat);
 	struct saa7146_vv *vv = dev->vv_data;
+	struct v4l2_pix_format *pix = &vv->video_fmt;
+	struct saa7146_format *sfmt = saa7146_format_by_fourcc(dev, pix->pixelformat);
 	u32 vdma1_prot_addr;
 
 	DEB_CAP("buf:%p, next:%p\n", buf, next);
@@ -1009,7 +734,7 @@ void saa7146_set_capture(struct saa7146_dev *dev, struct saa7146_buf *buf, struc
 		saa7146_write(dev, MC2, MASK_27 );
 	}
 
-	saa7146_set_window(dev, buf->fmt->width, buf->fmt->height, buf->fmt->field);
+	saa7146_set_window(dev, pix->width, pix->height, pix->field);
 	saa7146_set_output_format(dev, sfmt->trans);
 	saa7146_disable_clipping(dev);
 
diff --git a/drivers/media/common/saa7146/saa7146_vbi.c b/drivers/media/common/saa7146/saa7146_vbi.c
index bd442b984423..bb7d81f7eba6 100644
--- a/drivers/media/common/saa7146/saa7146_vbi.c
+++ b/drivers/media/common/saa7146/saa7146_vbi.c
@@ -207,7 +207,6 @@ static int buffer_activate(struct saa7146_dev *dev,
 			   struct saa7146_buf *next)
 {
 	struct saa7146_vv *vv = dev->vv_data;
-	buf->vb.state = VIDEOBUF_ACTIVE;
 
 	DEB_VBI("dev:%p, buf:%p, next:%p\n", dev, buf, next);
 	saa7146_set_vbi_capture(dev,buf,next);
@@ -216,114 +215,101 @@ static int buffer_activate(struct saa7146_dev *dev,
 	return 0;
 }
 
-static int buffer_prepare(struct videobuf_queue *q, struct videobuf_buffer *vb,enum v4l2_field field)
-{
-	struct file *file = q->priv_data;
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_buf *buf = (struct saa7146_buf *)vb;
+/* ------------------------------------------------------------------ */
 
-	int err = 0;
-	int lines, llength, size;
+static int queue_setup(struct vb2_queue *q,
+		       unsigned int *num_buffers, unsigned int *num_planes,
+		       unsigned int sizes[], struct device *alloc_devs[])
+{
+	unsigned int size = 16 * 2 * vbi_pixel_to_capture;
 
-	lines   = 16 * 2 ; /* 2 fields */
-	llength = vbi_pixel_to_capture;
-	size = lines * llength;
+	if (*num_planes)
+		return sizes[0] < size ? -EINVAL : 0;
+	*num_planes = 1;
+	sizes[0] = size;
 
-	DEB_VBI("vb:%p\n", vb);
+	return 0;
+}
 
-	if (0 != buf->vb.baddr  &&  buf->vb.bsize < size) {
-		DEB_VBI("size mismatch\n");
-		return -EINVAL;
-	}
+static void buf_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct vb2_queue *vq = vb->vb2_queue;
+	struct saa7146_dev *dev = vb2_get_drv_priv(vq);
+	struct saa7146_buf *buf = container_of(vbuf, struct saa7146_buf, vb);
+	unsigned long flags;
 
-	if (buf->vb.size != size)
-		saa7146_dma_free(dev,q,buf);
+	spin_lock_irqsave(&dev->slock, flags);
 
-	if (VIDEOBUF_NEEDS_INIT == buf->vb.state) {
-		struct videobuf_dmabuf *dma=videobuf_to_dma(&buf->vb);
+	saa7146_buffer_queue(dev, &dev->vv_data->vbi_dmaq, buf);
+	spin_unlock_irqrestore(&dev->slock, flags);
+}
 
-		buf->vb.width  = llength;
-		buf->vb.height = lines;
-		buf->vb.size   = size;
-		buf->vb.field  = field;	// FIXME: check this
+static int buf_init(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct saa7146_buf *buf = container_of(vbuf, struct saa7146_buf, vb);
+	struct sg_table *sgt = vb2_dma_sg_plane_desc(&buf->vb.vb2_buf, 0);
+	struct scatterlist *list = sgt->sgl;
+	int length = sgt->nents;
+	struct vb2_queue *vq = vb->vb2_queue;
+	struct saa7146_dev *dev = vb2_get_drv_priv(vq);
+	int ret;
 
-		saa7146_pgtable_free(dev->pci, &buf->pt[2]);
-		saa7146_pgtable_alloc(dev->pci, &buf->pt[2]);
-
-		err = videobuf_iolock(q,&buf->vb, NULL);
-		if (err)
-			goto oops;
-		err = saa7146_pgtable_build_single(dev->pci, &buf->pt[2],
-						 dma->sglist, dma->sglen);
-		if (0 != err)
-			return err;
-	}
-	buf->vb.state = VIDEOBUF_PREPARED;
 	buf->activate = buffer_activate;
 
-	return 0;
-
- oops:
-	DEB_VBI("error out\n");
-	saa7146_dma_free(dev,q,buf);
+	saa7146_pgtable_alloc(dev->pci, &buf->pt[2]);
 
-	return err;
+	ret = saa7146_pgtable_build_single(dev->pci, &buf->pt[2],
+					   list, length);
+	if (ret)
+		saa7146_pgtable_free(dev->pci, &buf->pt[2]);
+	return ret;
 }
 
-static int buffer_setup(struct videobuf_queue *q, unsigned int *count, unsigned int *size)
+static int buf_prepare(struct vb2_buffer *vb)
 {
-	int llength,lines;
-
-	lines   = 16 * 2 ; /* 2 fields */
-	llength = vbi_pixel_to_capture;
-
-	*size = lines * llength;
-	*count = 2;
-
-	DEB_VBI("count:%d, size:%d\n", *count, *size);
+	unsigned int size = 16 * 2 * vbi_pixel_to_capture;
 
+	if (vb2_plane_size(vb, 0) < size)
+		return -EINVAL;
+	vb2_set_plane_payload(vb, 0, size);
 	return 0;
 }
 
-static void buffer_queue(struct videobuf_queue *q, struct videobuf_buffer *vb)
+static void buf_cleanup(struct vb2_buffer *vb)
 {
-	struct file *file = q->priv_data;
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	struct saa7146_buf *buf = (struct saa7146_buf *)vb;
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct saa7146_buf *buf = container_of(vbuf, struct saa7146_buf, vb);
+	struct vb2_queue *vq = vb->vb2_queue;
+	struct saa7146_dev *dev = vb2_get_drv_priv(vq);
 
-	DEB_VBI("vb:%p\n", vb);
-	saa7146_buffer_queue(dev, &vv->vbi_dmaq, buf);
+	saa7146_pgtable_free(dev->pci, &buf->pt[2]);
 }
 
-static void buffer_release(struct videobuf_queue *q, struct videobuf_buffer *vb)
+static void return_buffers(struct vb2_queue *q, int state)
 {
-	struct file *file = q->priv_data;
-	struct saa7146_fh *fh   = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_buf *buf = (struct saa7146_buf *)vb;
-
-	DEB_VBI("vb:%p\n", vb);
-	saa7146_dma_free(dev,q,buf);
+	struct saa7146_dev *dev = vb2_get_drv_priv(q);
+	struct saa7146_dmaqueue *dq = &dev->vv_data->vbi_dmaq;
+	struct saa7146_buf *buf;
+
+	if (dq->curr) {
+		buf = dq->curr;
+		dq->curr = NULL;
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+	}
+	while (!list_empty(&dq->queue)) {
+		buf = list_entry(dq->queue.next, struct saa7146_buf, list);
+		list_del(&buf->list);
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+	}
 }
 
-static const struct videobuf_queue_ops vbi_qops = {
-	.buf_setup    = buffer_setup,
-	.buf_prepare  = buffer_prepare,
-	.buf_queue    = buffer_queue,
-	.buf_release  = buffer_release,
-};
-
-/* ------------------------------------------------------------------ */
-
-static void vbi_stop(struct saa7146_fh *fh, struct file *file)
+static void vbi_stop(struct saa7146_dev *dev)
 {
-	struct saa7146_dev *dev = fh->dev;
 	struct saa7146_vv *vv = dev->vv_data;
 	unsigned long flags;
-	DEB_VBI("dev:%p, fh:%p\n", dev, fh);
+	DEB_VBI("dev:%p\n", dev);
 
 	spin_lock_irqsave(&dev->slock,flags);
 
@@ -336,13 +322,6 @@ static void vbi_stop(struct saa7146_fh *fh, struct file *file)
 	/* shut down dma 3 transfers */
 	saa7146_write(dev, MC1, MASK_20);
 
-	if (vv->vbi_dmaq.curr)
-		saa7146_buffer_finish(dev, &vv->vbi_dmaq, VIDEOBUF_DONE);
-
-	videobuf_queue_cancel(&fh->vbi_q);
-
-	vv->vbi_streaming = NULL;
-
 	del_timer(&vv->vbi_dmaq.timeout);
 	del_timer(&vv->vbi_read_timeout);
 
@@ -352,38 +331,22 @@ static void vbi_stop(struct saa7146_fh *fh, struct file *file)
 static void vbi_read_timeout(struct timer_list *t)
 {
 	struct saa7146_vv *vv = from_timer(vv, t, vbi_read_timeout);
-	struct file *file = vv->vbi_read_timeout_file;
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-
-	DEB_VBI("dev:%p, fh:%p\n", dev, fh);
+	struct saa7146_dev *dev = vv->vbi_dmaq.dev;
 
-	vbi_stop(fh, file);
-}
-
-static void vbi_init(struct saa7146_dev *dev, struct saa7146_vv *vv)
-{
 	DEB_VBI("dev:%p\n", dev);
 
-	INIT_LIST_HEAD(&vv->vbi_dmaq.queue);
-
-	timer_setup(&vv->vbi_dmaq.timeout, saa7146_buffer_timeout, 0);
-	vv->vbi_dmaq.dev              = dev;
-
-	init_waitqueue_head(&vv->vbi_wq);
+	vbi_stop(dev);
 }
 
-static int vbi_open(struct saa7146_dev *dev, struct file *file)
+static int vbi_begin(struct saa7146_dev *dev)
 {
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_vv *vv = fh->dev->vv_data;
-
+	struct saa7146_vv *vv = dev->vv_data;
 	u32 arbtr_ctrl	= saa7146_read(dev, PCI_BT_V1);
 	int ret = 0;
 
-	DEB_VBI("dev:%p, fh:%p\n", dev, fh);
+	DEB_VBI("dev:%p\n", dev);
 
-	ret = saa7146_res_get(fh, RESOURCE_DMA3_BRS);
+	ret = saa7146_res_get(dev, RESOURCE_DMA3_BRS);
 	if (0 == ret) {
 		DEB_S("cannot get vbi RESOURCE_DMA3_BRS resource\n");
 		return -EBUSY;
@@ -395,15 +358,7 @@ static int vbi_open(struct saa7146_dev *dev, struct file *file)
 	saa7146_write(dev, PCI_BT_V1, arbtr_ctrl);
 	saa7146_write(dev, MC2, (MASK_04|MASK_20));
 
-	videobuf_queue_sg_init(&fh->vbi_q, &vbi_qops,
-			    &dev->pci->dev, &dev->slock,
-			    V4L2_BUF_TYPE_VBI_CAPTURE,
-			    V4L2_FIELD_SEQ_TB, // FIXME: does this really work?
-			    sizeof(struct saa7146_buf),
-			    file, &dev->v4l2_lock);
-
 	vv->vbi_read_timeout.function = vbi_read_timeout;
-	vv->vbi_read_timeout_file = file;
 
 	/* initialize the brs */
 	if ( 0 != (SAA7146_USE_PORT_B_FOR_VBI & dev->ext_vv_data->flags)) {
@@ -422,16 +377,52 @@ static int vbi_open(struct saa7146_dev *dev, struct file *file)
 	return 0;
 }
 
-static void vbi_close(struct saa7146_dev *dev, struct file *file)
+static int start_streaming(struct vb2_queue *q, unsigned int count)
 {
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_vv *vv = dev->vv_data;
-	DEB_VBI("dev:%p, fh:%p\n", dev, fh);
+	struct saa7146_dev *dev = vb2_get_drv_priv(q);
+	int ret;
+
+	if (!vb2_is_streaming(&dev->vv_data->vbi_dmaq.q))
+		dev->vv_data->seqnr = 0;
+	ret = vbi_begin(dev);
+	if (ret)
+		return_buffers(q, VB2_BUF_STATE_QUEUED);
+	return ret;
+}
 
-	if( fh == vv->vbi_streaming ) {
-		vbi_stop(fh, file);
-	}
-	saa7146_res_free(fh, RESOURCE_DMA3_BRS);
+static void stop_streaming(struct vb2_queue *q)
+{
+	struct saa7146_dev *dev = vb2_get_drv_priv(q);
+
+	vbi_stop(dev);
+	return_buffers(q, VB2_BUF_STATE_ERROR);
+	saa7146_res_free(dev, RESOURCE_DMA3_BRS);
+}
+
+const struct vb2_ops vbi_qops = {
+	.queue_setup	= queue_setup,
+	.buf_queue	= buf_queue,
+	.buf_init	= buf_init,
+	.buf_prepare	= buf_prepare,
+	.buf_cleanup	= buf_cleanup,
+	.start_streaming = start_streaming,
+	.stop_streaming = stop_streaming,
+	.wait_prepare	= vb2_ops_wait_prepare,
+	.wait_finish	= vb2_ops_wait_finish,
+};
+
+/* ------------------------------------------------------------------ */
+
+static void vbi_init(struct saa7146_dev *dev, struct saa7146_vv *vv)
+{
+	DEB_VBI("dev:%p\n", dev);
+
+	INIT_LIST_HEAD(&vv->vbi_dmaq.queue);
+
+	timer_setup(&vv->vbi_dmaq.timeout, saa7146_buffer_timeout, 0);
+	vv->vbi_dmaq.dev              = dev;
+
+	init_waitqueue_head(&vv->vbi_wq);
 }
 
 static void vbi_irq_done(struct saa7146_dev *dev, unsigned long status)
@@ -441,10 +432,7 @@ static void vbi_irq_done(struct saa7146_dev *dev, unsigned long status)
 
 	if (vv->vbi_dmaq.curr) {
 		DEB_VBI("dev:%p, curr:%p\n", dev, vv->vbi_dmaq.curr);
-		/* this must be += 2, one count for each field */
-		vv->vbi_fieldcount+=2;
-		vv->vbi_dmaq.curr->vb.field_count = vv->vbi_fieldcount;
-		saa7146_buffer_finish(dev, &vv->vbi_dmaq, VIDEOBUF_DONE);
+		saa7146_buffer_finish(dev, &vv->vbi_dmaq, VB2_BUF_STATE_DONE);
 	} else {
 		DEB_VBI("dev:%p\n", dev);
 	}
@@ -453,46 +441,7 @@ static void vbi_irq_done(struct saa7146_dev *dev, unsigned long status)
 	spin_unlock(&dev->slock);
 }
 
-static ssize_t vbi_read(struct file *file, char __user *data, size_t count, loff_t *ppos)
-{
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	ssize_t ret = 0;
-
-	DEB_VBI("dev:%p, fh:%p\n", dev, fh);
-
-	if( NULL == vv->vbi_streaming ) {
-		// fixme: check if dma3 is available
-		// fixme: activate vbi engine here if necessary. (really?)
-		vv->vbi_streaming = fh;
-	}
-
-	if( fh != vv->vbi_streaming ) {
-		DEB_VBI("open %p is already using vbi capture\n",
-			vv->vbi_streaming);
-		return -EBUSY;
-	}
-
-	mod_timer(&vv->vbi_read_timeout, jiffies+BUFFER_TIMEOUT);
-	ret = videobuf_read_stream(&fh->vbi_q, data, count, ppos, 1,
-				   file->f_flags & O_NONBLOCK);
-/*
-	printk("BASE_ODD3:      0x%08x\n", saa7146_read(dev, BASE_ODD3));
-	printk("BASE_EVEN3:     0x%08x\n", saa7146_read(dev, BASE_EVEN3));
-	printk("PROT_ADDR3:     0x%08x\n", saa7146_read(dev, PROT_ADDR3));
-	printk("PITCH3:         0x%08x\n", saa7146_read(dev, PITCH3));
-	printk("BASE_PAGE3:     0x%08x\n", saa7146_read(dev, BASE_PAGE3));
-	printk("NUM_LINE_BYTE3: 0x%08x\n", saa7146_read(dev, NUM_LINE_BYTE3));
-	printk("BRS_CTRL:       0x%08x\n", saa7146_read(dev, BRS_CTRL));
-*/
-	return ret;
-}
-
 const struct saa7146_use_ops saa7146_vbi_uops = {
 	.init		= vbi_init,
-	.open		= vbi_open,
-	.release	= vbi_close,
 	.irq_done	= vbi_irq_done,
-	.read		= vbi_read,
 };
diff --git a/drivers/media/common/saa7146/saa7146_video.c b/drivers/media/common/saa7146/saa7146_video.c
index 2296765079a4..040489e15ea0 100644
--- a/drivers/media/common/saa7146/saa7146_video.c
+++ b/drivers/media/common/saa7146/saa7146_video.c
@@ -6,17 +6,6 @@
 #include <linux/module.h>
 #include <linux/kernel.h>
 
-static int max_memory = 32;
-
-module_param(max_memory, int, 0644);
-MODULE_PARM_DESC(max_memory, "maximum memory usage for capture buffers (default: 32Mb)");
-
-#define IS_CAPTURE_ACTIVE(fh) \
-	(((vv->video_status & STATUS_CAPTURE) != 0) && (vv->video_fh == fh))
-
-#define IS_OVERLAY_ACTIVE(fh) \
-	(((vv->video_status & STATUS_OVERLAY) != 0) && (vv->video_fh == fh))
-
 /* format descriptions for capture and preview */
 static struct saa7146_format formats[] = {
 	{
@@ -54,7 +43,7 @@ static struct saa7146_format formats[] = {
 		.pixelformat	= V4L2_PIX_FMT_YUV422P,
 		.trans		= YUV422_DECOMPOSED,
 		.depth		= 16,
-		.flags		= FORMAT_BYTE_SWAP|FORMAT_IS_PLANAR,
+		.flags		= FORMAT_IS_PLANAR,
 	}, {
 		.pixelformat	= V4L2_PIX_FMT_YVU420,
 		.trans		= YUV420_DECOMPOSED,
@@ -91,115 +80,18 @@ struct saa7146_format* saa7146_format_by_fourcc(struct saa7146_dev *dev, int fou
 	return NULL;
 }
 
-static int vidioc_try_fmt_vid_overlay(struct file *file, void *fh, struct v4l2_format *f);
-
-int saa7146_start_preview(struct saa7146_fh *fh)
-{
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	struct v4l2_format fmt;
-	int ret = 0, err = 0;
-
-	DEB_EE("dev:%p, fh:%p\n", dev, fh);
-
-	/* check if we have overlay information */
-	if (vv->ov.fh == NULL) {
-		DEB_D("no overlay data available. try S_FMT first.\n");
-		return -EAGAIN;
-	}
-
-	/* check if streaming capture is running */
-	if (IS_CAPTURE_ACTIVE(fh) != 0) {
-		DEB_D("streaming capture is active\n");
-		return -EBUSY;
-	}
-
-	/* check if overlay is running */
-	if (IS_OVERLAY_ACTIVE(fh) != 0) {
-		if (vv->video_fh == fh) {
-			DEB_D("overlay is already active\n");
-			return 0;
-		}
-		DEB_D("overlay is already active in another open\n");
-		return -EBUSY;
-	}
-
-	if (0 == saa7146_res_get(fh, RESOURCE_DMA1_HPS|RESOURCE_DMA2_CLP)) {
-		DEB_D("cannot get necessary overlay resources\n");
-		return -EBUSY;
-	}
-
-	fmt.fmt.win = vv->ov.win;
-	err = vidioc_try_fmt_vid_overlay(NULL, fh, &fmt);
-	if (0 != err) {
-		saa7146_res_free(vv->video_fh, RESOURCE_DMA1_HPS|RESOURCE_DMA2_CLP);
-		return -EBUSY;
-	}
-	vv->ov.win = fmt.fmt.win;
-
-	DEB_D("%dx%d+%d+%d 0x%08x field=%s\n",
-	      vv->ov.win.w.width, vv->ov.win.w.height,
-	      vv->ov.win.w.left, vv->ov.win.w.top,
-	      vv->ov_fmt->pixelformat, v4l2_field_names[vv->ov.win.field]);
-
-	if (0 != (ret = saa7146_enable_overlay(fh))) {
-		DEB_D("enabling overlay failed: %d\n", ret);
-		saa7146_res_free(vv->video_fh, RESOURCE_DMA1_HPS|RESOURCE_DMA2_CLP);
-		return ret;
-	}
-
-	vv->video_status = STATUS_OVERLAY;
-	vv->video_fh = fh;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(saa7146_start_preview);
-
-int saa7146_stop_preview(struct saa7146_fh *fh)
-{
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-
-	DEB_EE("dev:%p, fh:%p\n", dev, fh);
-
-	/* check if streaming capture is running */
-	if (IS_CAPTURE_ACTIVE(fh) != 0) {
-		DEB_D("streaming capture is active\n");
-		return -EBUSY;
-	}
-
-	/* check if overlay is running at all */
-	if ((vv->video_status & STATUS_OVERLAY) == 0) {
-		DEB_D("no active overlay\n");
-		return 0;
-	}
-
-	if (vv->video_fh != fh) {
-		DEB_D("overlay is active, but in another open\n");
-		return -EBUSY;
-	}
-
-	vv->video_status = 0;
-	vv->video_fh = NULL;
-
-	saa7146_disable_overlay(fh);
-
-	saa7146_res_free(fh, RESOURCE_DMA1_HPS|RESOURCE_DMA2_CLP);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(saa7146_stop_preview);
-
 /********************************************************************************/
 /* common pagetable functions */
 
 static int saa7146_pgtable_build(struct saa7146_dev *dev, struct saa7146_buf *buf)
 {
+	struct saa7146_vv *vv = dev->vv_data;
 	struct pci_dev *pci = dev->pci;
-	struct videobuf_dmabuf *dma=videobuf_to_dma(&buf->vb);
-	struct scatterlist *list = dma->sglist;
-	int length = dma->sglen;
-	struct saa7146_format *sfmt = saa7146_format_by_fourcc(dev,buf->fmt->pixelformat);
+	struct sg_table *sgt = vb2_dma_sg_plane_desc(&buf->vb.vb2_buf, 0);
+	struct scatterlist *list = sgt->sgl;
+	int length = sgt->nents;
+	struct v4l2_pix_format *pix = &vv->video_fmt;
+	struct saa7146_format *sfmt = saa7146_format_by_fourcc(dev, pix->pixelformat);
 
 	DEB_EE("dev:%p, buf:%p, sg_len:%d\n", dev, buf, length);
 
@@ -207,31 +99,32 @@ static int saa7146_pgtable_build(struct saa7146_dev *dev, struct saa7146_buf *bu
 		struct saa7146_pgtable *pt1 = &buf->pt[0];
 		struct saa7146_pgtable *pt2 = &buf->pt[1];
 		struct saa7146_pgtable *pt3 = &buf->pt[2];
+		struct sg_dma_page_iter dma_iter;
 		__le32  *ptr1, *ptr2, *ptr3;
 		__le32 fill;
 
-		int size = buf->fmt->width*buf->fmt->height;
-		int i,p,m1,m2,m3,o1,o2;
+		int size = pix->width * pix->height;
+		int i, m1, m2, m3, o1, o2;
 
 		switch( sfmt->depth ) {
 			case 12: {
 				/* create some offsets inside the page table */
-				m1 = ((size+PAGE_SIZE)/PAGE_SIZE)-1;
-				m2 = ((size+(size/4)+PAGE_SIZE)/PAGE_SIZE)-1;
-				m3 = ((size+(size/2)+PAGE_SIZE)/PAGE_SIZE)-1;
-				o1 = size%PAGE_SIZE;
-				o2 = (size+(size/4))%PAGE_SIZE;
+				m1 = ((size + PAGE_SIZE) / PAGE_SIZE) - 1;
+				m2 = ((size + (size / 4) + PAGE_SIZE) / PAGE_SIZE) - 1;
+				m3 = ((size + (size / 2) + PAGE_SIZE) / PAGE_SIZE) - 1;
+				o1 = size % PAGE_SIZE;
+				o2 = (size + (size / 4)) % PAGE_SIZE;
 				DEB_CAP("size:%d, m1:%d, m2:%d, m3:%d, o1:%d, o2:%d\n",
 					size, m1, m2, m3, o1, o2);
 				break;
 			}
 			case 16: {
 				/* create some offsets inside the page table */
-				m1 = ((size+PAGE_SIZE)/PAGE_SIZE)-1;
-				m2 = ((size+(size/2)+PAGE_SIZE)/PAGE_SIZE)-1;
-				m3 = ((2*size+PAGE_SIZE)/PAGE_SIZE)-1;
-				o1 = size%PAGE_SIZE;
-				o2 = (size+(size/2))%PAGE_SIZE;
+				m1 = ((size + PAGE_SIZE) / PAGE_SIZE) - 1;
+				m2 = ((size + (size / 2) + PAGE_SIZE) / PAGE_SIZE) - 1;
+				m3 = ((2 * size + PAGE_SIZE) / PAGE_SIZE) - 1;
+				o1 = size % PAGE_SIZE;
+				o2 = (size + (size / 2)) % PAGE_SIZE;
 				DEB_CAP("size:%d, m1:%d, m2:%d, m3:%d, o1:%d, o2:%d\n",
 					size, m1, m2, m3, o1, o2);
 				break;
@@ -245,64 +138,37 @@ static int saa7146_pgtable_build(struct saa7146_dev *dev, struct saa7146_buf *bu
 		ptr2 = pt2->cpu;
 		ptr3 = pt3->cpu;
 
-		/* walk all pages, copy all page addresses to ptr1 */
-		for (i = 0; i < length; i++, list++) {
-			for (p = 0; p * 4096 < sg_dma_len(list); p++, ptr1++)
-				*ptr1 = cpu_to_le32(sg_dma_address(list) - list->offset);
-		}
-/*
-		ptr1 = pt1->cpu;
-		for(j=0;j<40;j++) {
-			printk("ptr1 %d: 0x%08x\n",j,ptr1[j]);
-		}
-*/
+		for_each_sg_dma_page(list, &dma_iter, length, 0)
+			*ptr1++ = cpu_to_le32(sg_page_iter_dma_address(&dma_iter) - list->offset);
 
 		/* if we have a user buffer, the first page may not be
 		   aligned to a page boundary. */
-		pt1->offset = dma->sglist->offset;
-		pt2->offset = pt1->offset+o1;
-		pt3->offset = pt1->offset+o2;
+		pt1->offset = sgt->sgl->offset;
+		pt2->offset = pt1->offset + o1;
+		pt3->offset = pt1->offset + o2;
 
 		/* create video-dma2 page table */
 		ptr1 = pt1->cpu;
-		for(i = m1; i <= m2 ; i++, ptr2++) {
+		for (i = m1; i <= m2; i++, ptr2++)
 			*ptr2 = ptr1[i];
-		}
-		fill = *(ptr2-1);
-		for(;i<1024;i++,ptr2++) {
+		fill = *(ptr2 - 1);
+		for (; i < 1024; i++, ptr2++)
 			*ptr2 = fill;
-		}
 		/* create video-dma3 page table */
 		ptr1 = pt1->cpu;
-		for(i = m2; i <= m3; i++,ptr3++) {
+		for (i = m2; i <= m3; i++, ptr3++)
 			*ptr3 = ptr1[i];
-		}
-		fill = *(ptr3-1);
-		for(;i<1024;i++,ptr3++) {
+		fill = *(ptr3 - 1);
+		for (; i < 1024; i++, ptr3++)
 			*ptr3 = fill;
-		}
 		/* finally: finish up video-dma1 page table */
-		ptr1 = pt1->cpu+m1;
+		ptr1 = pt1->cpu + m1;
 		fill = pt1->cpu[m1];
-		for(i=m1;i<1024;i++,ptr1++) {
+		for (i = m1; i < 1024; i++, ptr1++)
 			*ptr1 = fill;
-		}
-/*
-		ptr1 = pt1->cpu;
-		ptr2 = pt2->cpu;
-		ptr3 = pt3->cpu;
-		for(j=0;j<40;j++) {
-			printk("ptr1 %d: 0x%08x\n",j,ptr1[j]);
-		}
-		for(j=0;j<40;j++) {
-			printk("ptr2 %d: 0x%08x\n",j,ptr2[j]);
-		}
-		for(j=0;j<40;j++) {
-			printk("ptr3 %d: 0x%08x\n",j,ptr3[j]);
-		}
-*/
 	} else {
 		struct saa7146_pgtable *pt = &buf->pt[0];
+
 		return saa7146_pgtable_build_single(pci, pt, list, length);
 	}
 
@@ -313,34 +179,14 @@ static int saa7146_pgtable_build(struct saa7146_dev *dev, struct saa7146_buf *bu
 /********************************************************************************/
 /* file operations */
 
-static int video_begin(struct saa7146_fh *fh)
+static int video_begin(struct saa7146_dev *dev)
 {
-	struct saa7146_dev *dev = fh->dev;
 	struct saa7146_vv *vv = dev->vv_data;
 	struct saa7146_format *fmt = NULL;
 	unsigned int resource;
-	int ret = 0, err = 0;
-
-	DEB_EE("dev:%p, fh:%p\n", dev, fh);
-
-	if ((vv->video_status & STATUS_CAPTURE) != 0) {
-		if (vv->video_fh == fh) {
-			DEB_S("already capturing\n");
-			return 0;
-		}
-		DEB_S("already capturing in another open\n");
-		return -EBUSY;
-	}
+	int ret = 0;
 
-	if ((vv->video_status & STATUS_OVERLAY) != 0) {
-		DEB_S("warning: suspending overlay video for streaming capture\n");
-		vv->ov_suspend = vv->video_fh;
-		err = saa7146_stop_preview(vv->video_fh); /* side effect: video_status is now 0, video_fh is NULL */
-		if (0 != err) {
-			DEB_D("suspending video failed. aborting\n");
-			return err;
-		}
-	}
+	DEB_EE("dev:%p\n", dev);
 
 	fmt = saa7146_format_by_fourcc(dev, vv->video_fmt.pixelformat);
 	/* we need to have a valid format set here */
@@ -353,13 +199,9 @@ static int video_begin(struct saa7146_fh *fh)
 		resource = RESOURCE_DMA1_HPS;
 	}
 
-	ret = saa7146_res_get(fh, resource);
+	ret = saa7146_res_get(dev, resource);
 	if (0 == ret) {
 		DEB_S("cannot get capture resource %d\n", resource);
-		if (vv->ov_suspend != NULL) {
-			saa7146_start_preview(vv->ov_suspend);
-			vv->ov_suspend = NULL;
-		}
 		return -EBUSY;
 	}
 
@@ -369,37 +211,22 @@ static int video_begin(struct saa7146_fh *fh)
 	/* enable rps0 irqs */
 	SAA7146_IER_ENABLE(dev, MASK_27);
 
-	vv->video_fh = fh;
-	vv->video_status = STATUS_CAPTURE;
-
 	return 0;
 }
 
-static int video_end(struct saa7146_fh *fh, struct file *file)
+static void video_end(struct saa7146_dev *dev)
 {
-	struct saa7146_dev *dev = fh->dev;
 	struct saa7146_vv *vv = dev->vv_data;
-	struct saa7146_dmaqueue *q = &vv->video_dmaq;
 	struct saa7146_format *fmt = NULL;
 	unsigned long flags;
 	unsigned int resource;
 	u32 dmas = 0;
-	DEB_EE("dev:%p, fh:%p\n", dev, fh);
-
-	if ((vv->video_status & STATUS_CAPTURE) != STATUS_CAPTURE) {
-		DEB_S("not capturing\n");
-		return 0;
-	}
-
-	if (vv->video_fh != fh) {
-		DEB_S("capturing, but in another open\n");
-		return -EBUSY;
-	}
+	DEB_EE("dev:%p\n", dev);
 
 	fmt = saa7146_format_by_fourcc(dev, vv->video_fmt.pixelformat);
 	/* we need to have a valid format set here */
 	if (!fmt)
-		return -EINVAL;
+		return;
 
 	if (0 != (fmt->flags & FORMAT_IS_PLANAR)) {
 		resource = RESOURCE_DMA1_HPS|RESOURCE_DMA2_CLP|RESOURCE_DMA3_BRS;
@@ -419,88 +246,24 @@ static int video_end(struct saa7146_fh *fh, struct file *file)
 	/* shut down all used video dma transfers */
 	saa7146_write(dev, MC1, dmas);
 
-	if (q->curr)
-		saa7146_buffer_finish(dev, q, VIDEOBUF_DONE);
-
 	spin_unlock_irqrestore(&dev->slock, flags);
 
-	vv->video_fh = NULL;
-	vv->video_status = 0;
-
-	saa7146_res_free(fh, resource);
-
-	if (vv->ov_suspend != NULL) {
-		saa7146_start_preview(vv->ov_suspend);
-		vv->ov_suspend = NULL;
-	}
-
-	return 0;
+	saa7146_res_free(dev, resource);
 }
 
 static int vidioc_querycap(struct file *file, void *fh, struct v4l2_capability *cap)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 
 	strscpy((char *)cap->driver, "saa7146 v4l2", sizeof(cap->driver));
 	strscpy((char *)cap->card, dev->ext->name, sizeof(cap->card));
-	cap->capabilities = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_OVERLAY |
+	cap->capabilities = V4L2_CAP_VIDEO_CAPTURE |
 			    V4L2_CAP_READWRITE | V4L2_CAP_STREAMING |
 			    V4L2_CAP_DEVICE_CAPS;
 	cap->capabilities |= dev->ext_vv_data->capabilities;
 	return 0;
 }
 
-static int vidioc_g_fbuf(struct file *file, void *fh, struct v4l2_framebuffer *fb)
-{
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-
-	*fb = vv->ov_fb;
-	fb->capability = V4L2_FBUF_CAP_LIST_CLIPPING;
-	fb->flags = V4L2_FBUF_FLAG_PRIMARY;
-	return 0;
-}
-
-static int vidioc_s_fbuf(struct file *file, void *fh, const struct v4l2_framebuffer *fb)
-{
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	struct saa7146_format *fmt;
-
-	DEB_EE("VIDIOC_S_FBUF\n");
-
-	if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RAWIO))
-		return -EPERM;
-
-	/* check args */
-	fmt = saa7146_format_by_fourcc(dev, fb->fmt.pixelformat);
-	if (NULL == fmt)
-		return -EINVAL;
-
-	/* planar formats are not allowed for overlay video, clipping and video dma would clash */
-	if (fmt->flags & FORMAT_IS_PLANAR)
-		DEB_S("planar pixelformat '%4.4s' not allowed for overlay\n",
-		      (char *)&fmt->pixelformat);
-
-	/* check if overlay is running */
-	if (IS_OVERLAY_ACTIVE(fh) != 0) {
-		if (vv->video_fh != fh) {
-			DEB_D("refusing to change framebuffer information while overlay is active in another open\n");
-			return -EBUSY;
-		}
-	}
-
-	/* ok, accept it */
-	vv->ov_fb = *fb;
-	vv->ov_fmt = fmt;
-
-	if (vv->ov_fb.fmt.bytesperline < vv->ov_fb.fmt.width) {
-		vv->ov_fb.fmt.bytesperline = vv->ov_fb.fmt.width * fmt->depth / 8;
-		DEB_D("setting bytesperline to %d\n", vv->ov_fb.fmt.bytesperline);
-	}
-	return 0;
-}
-
 static int vidioc_enum_fmt_vid_cap(struct file *file, void *fh, struct v4l2_fmtdesc *f)
 {
 	if (f->index >= ARRAY_SIZE(formats))
@@ -543,13 +306,13 @@ int saa7146_s_ctrl(struct v4l2_ctrl *ctrl)
 
 	case V4L2_CID_HFLIP:
 		/* fixme: we can support changing VFLIP and HFLIP here... */
-		if ((vv->video_status & STATUS_CAPTURE))
+		if (vb2_is_busy(&vv->video_dmaq.q))
 			return -EBUSY;
 		vv->hflip = ctrl->val;
 		break;
 
 	case V4L2_CID_VFLIP:
-		if ((vv->video_status & STATUS_CAPTURE))
+		if (vb2_is_busy(&vv->video_dmaq.q))
 			return -EBUSY;
 		vv->vflip = ctrl->val;
 		break;
@@ -557,20 +320,13 @@ int saa7146_s_ctrl(struct v4l2_ctrl *ctrl)
 	default:
 		return -EINVAL;
 	}
-
-	if ((vv->video_status & STATUS_OVERLAY) != 0) { /* CHECK: && (vv->video_fh == fh)) */
-		struct saa7146_fh *fh = vv->video_fh;
-
-		saa7146_stop_preview(fh);
-		saa7146_start_preview(fh);
-	}
 	return 0;
 }
 
 static int vidioc_g_parm(struct file *file, void *fh,
 		struct v4l2_streamparm *parm)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct saa7146_vv *vv = dev->vv_data;
 
 	if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
@@ -583,25 +339,16 @@ static int vidioc_g_parm(struct file *file, void *fh,
 
 static int vidioc_g_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct saa7146_vv *vv = dev->vv_data;
 
 	f->fmt.pix = vv->video_fmt;
 	return 0;
 }
 
-static int vidioc_g_fmt_vid_overlay(struct file *file, void *fh, struct v4l2_format *f)
-{
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-
-	f->fmt.win = vv->ov.win;
-	return 0;
-}
-
 static int vidioc_g_fmt_vbi_cap(struct file *file, void *fh, struct v4l2_format *f)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct saa7146_vv *vv = dev->vv_data;
 
 	f->fmt.vbi = vv->vbi_fmt;
@@ -610,7 +357,7 @@ static int vidioc_g_fmt_vbi_cap(struct file *file, void *fh, struct v4l2_format
 
 static int vidioc_try_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct saa7146_vv *vv = dev->vv_data;
 	struct saa7146_format *fmt;
 	enum v4l2_field field;
@@ -634,24 +381,21 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *fh, struct v4l2_forma
 	}
 	switch (field) {
 	case V4L2_FIELD_ALTERNATE:
-		vv->last_field = V4L2_FIELD_TOP;
-		maxh = maxh / 2;
-		break;
 	case V4L2_FIELD_TOP:
 	case V4L2_FIELD_BOTTOM:
-		vv->last_field = V4L2_FIELD_INTERLACED;
 		maxh = maxh / 2;
 		break;
-	case V4L2_FIELD_INTERLACED:
-		vv->last_field = V4L2_FIELD_INTERLACED;
-		break;
 	default:
-		DEB_D("no known field mode '%d'\n", field);
-		return -EINVAL;
+		field = V4L2_FIELD_INTERLACED;
+		break;
 	}
 
 	f->fmt.pix.field = field;
 	f->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
+	if (f->fmt.pix.width < 48)
+		f->fmt.pix.width = 48;
+	if (f->fmt.pix.height < 32)
+		f->fmt.pix.height = 32;
 	if (f->fmt.pix.width > maxw)
 		f->fmt.pix.width = maxw;
 	if (f->fmt.pix.height > maxh)
@@ -673,169 +417,65 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *fh, struct v4l2_forma
 	return 0;
 }
 
-
-static int vidioc_try_fmt_vid_overlay(struct file *file, void *fh, struct v4l2_format *f)
+static int vidioc_s_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	struct v4l2_window *win = &f->fmt.win;
-	enum v4l2_field field;
-	int maxw, maxh;
-
-	DEB_EE("dev:%p\n", dev);
-
-	if (NULL == vv->ov_fb.base) {
-		DEB_D("no fb base set\n");
-		return -EINVAL;
-	}
-	if (NULL == vv->ov_fmt) {
-		DEB_D("no fb fmt set\n");
-		return -EINVAL;
-	}
-	if (win->w.width < 48 || win->w.height < 32) {
-		DEB_D("min width/height. (%d,%d)\n",
-		      win->w.width, win->w.height);
-		return -EINVAL;
-	}
-	if (win->clipcount > 16) {
-		DEB_D("clipcount too big\n");
-		return -EINVAL;
-	}
-
-	field = win->field;
-	maxw  = vv->standard->h_max_out;
-	maxh  = vv->standard->v_max_out;
-
-	if (V4L2_FIELD_ANY == field) {
-		field = (win->w.height > maxh / 2)
-			? V4L2_FIELD_INTERLACED
-			: V4L2_FIELD_TOP;
-		}
-	switch (field) {
-	case V4L2_FIELD_TOP:
-	case V4L2_FIELD_BOTTOM:
-	case V4L2_FIELD_ALTERNATE:
-		maxh = maxh / 2;
-		break;
-	case V4L2_FIELD_INTERLACED:
-		break;
-	default:
-		DEB_D("no known field mode '%d'\n", field);
-		return -EINVAL;
-	}
-
-	win->field = field;
-	if (win->w.width > maxw)
-		win->w.width = maxw;
-	if (win->w.height > maxh)
-		win->w.height = maxh;
-
-	return 0;
-}
-
-static int vidioc_s_fmt_vid_cap(struct file *file, void *__fh, struct v4l2_format *f)
-{
-	struct saa7146_fh *fh = __fh;
-	struct saa7146_dev *dev = fh->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct saa7146_vv *vv = dev->vv_data;
 	int err;
 
-	DEB_EE("V4L2_BUF_TYPE_VIDEO_CAPTURE: dev:%p, fh:%p\n", dev, fh);
-	if (IS_CAPTURE_ACTIVE(fh) != 0) {
+	DEB_EE("V4L2_BUF_TYPE_VIDEO_CAPTURE: dev:%p\n", dev);
+	if (vb2_is_busy(&vv->video_dmaq.q)) {
 		DEB_EE("streaming capture is active\n");
 		return -EBUSY;
 	}
 	err = vidioc_try_fmt_vid_cap(file, fh, f);
 	if (0 != err)
 		return err;
+	switch (f->fmt.pix.field) {
+	case V4L2_FIELD_ALTERNATE:
+		vv->last_field = V4L2_FIELD_TOP;
+		break;
+	default:
+		vv->last_field = V4L2_FIELD_INTERLACED;
+		break;
+	}
 	vv->video_fmt = f->fmt.pix;
 	DEB_EE("set to pixelformat '%4.4s'\n",
 	       (char *)&vv->video_fmt.pixelformat);
 	return 0;
 }
 
-static int vidioc_s_fmt_vid_overlay(struct file *file, void *__fh, struct v4l2_format *f)
-{
-	struct saa7146_fh *fh = __fh;
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	int err;
-
-	DEB_EE("V4L2_BUF_TYPE_VIDEO_OVERLAY: dev:%p, fh:%p\n", dev, fh);
-	err = vidioc_try_fmt_vid_overlay(file, fh, f);
-	if (0 != err)
-		return err;
-	vv->ov.win    = f->fmt.win;
-	vv->ov.nclips = f->fmt.win.clipcount;
-	if (vv->ov.nclips > 16)
-		vv->ov.nclips = 16;
-	memcpy(vv->ov.clips, f->fmt.win.clips,
-	       sizeof(struct v4l2_clip) * vv->ov.nclips);
-
-	/* vv->ov.fh is used to indicate that we have valid overlay information, too */
-	vv->ov.fh = fh;
-
-	/* check if our current overlay is active */
-	if (IS_OVERLAY_ACTIVE(fh) != 0) {
-		saa7146_stop_preview(fh);
-		saa7146_start_preview(fh);
-	}
-	return 0;
-}
-
 static int vidioc_g_std(struct file *file, void *fh, v4l2_std_id *norm)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct saa7146_vv *vv = dev->vv_data;
 
 	*norm = vv->standard->id;
 	return 0;
 }
 
-	/* the saa7146 supfhrts (used in conjunction with the saa7111a for example)
-	   PAL / NTSC / SECAM. if your hardware does not (or does more)
-	   -- override this function in your extension */
-/*
-	case VIDIOC_ENUMSTD:
-	{
-		struct v4l2_standard *e = arg;
-		if (e->index < 0 )
-			return -EINVAL;
-		if( e->index < dev->ext_vv_data->num_stds ) {
-			DEB_EE("VIDIOC_ENUMSTD: index:%d\n", e->index);
-			v4l2_video_std_construct(e, dev->ext_vv_data->stds[e->index].id, dev->ext_vv_data->stds[e->index].name);
-			return 0;
-		}
-		return -EINVAL;
-	}
-	*/
-
 static int vidioc_s_std(struct file *file, void *fh, v4l2_std_id id)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct saa7146_vv *vv = dev->vv_data;
 	int found = 0;
-	int err, i;
+	int i;
 
 	DEB_EE("VIDIOC_S_STD\n");
 
-	if ((vv->video_status & STATUS_CAPTURE) == STATUS_CAPTURE) {
+	for (i = 0; i < dev->ext_vv_data->num_stds; i++)
+		if (id & dev->ext_vv_data->stds[i].id)
+			break;
+
+	if (i != dev->ext_vv_data->num_stds &&
+	    vv->standard == &dev->ext_vv_data->stds[i])
+		return 0;
+
+	if (vb2_is_busy(&vv->video_dmaq.q) || vb2_is_busy(&vv->vbi_dmaq.q)) {
 		DEB_D("cannot change video standard while streaming capture is active\n");
 		return -EBUSY;
 	}
 
-	if ((vv->video_status & STATUS_OVERLAY) != 0) {
-		vv->ov_suspend = vv->video_fh;
-		err = saa7146_stop_preview(vv->video_fh); /* side effect: video_status is now 0, video_fh is NULL */
-		if (0 != err) {
-			DEB_D("suspending video failed. aborting\n");
-			return err;
-		}
-	}
-
-	for (i = 0; i < dev->ext_vv_data->num_stds; i++)
-		if (id & dev->ext_vv_data->stds[i].id)
-			break;
 	if (i != dev->ext_vv_data->num_stds) {
 		vv->standard = &dev->ext_vv_data->stds[i];
 		if (NULL != dev->ext_vv_data->std_callback)
@@ -843,11 +483,6 @@ static int vidioc_s_std(struct file *file, void *fh, v4l2_std_id id)
 		found = 1;
 	}
 
-	if (vv->ov_suspend != NULL) {
-		saa7146_start_preview(vv->ov_suspend);
-		vv->ov_suspend = NULL;
-	}
-
 	if (!found) {
 		DEB_EE("VIDIOC_S_STD: standard not found\n");
 		return -EINVAL;
@@ -857,138 +492,22 @@ static int vidioc_s_std(struct file *file, void *fh, v4l2_std_id id)
 	return 0;
 }
 
-static int vidioc_overlay(struct file *file, void *fh, unsigned int on)
-{
-	int err;
-
-	DEB_D("VIDIOC_OVERLAY on:%d\n", on);
-	if (on)
-		err = saa7146_start_preview(fh);
-	else
-		err = saa7146_stop_preview(fh);
-	return err;
-}
-
-static int vidioc_reqbufs(struct file *file, void *__fh, struct v4l2_requestbuffers *b)
-{
-	struct saa7146_fh *fh = __fh;
-
-	if (b->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return videobuf_reqbufs(&fh->video_q, b);
-	if (b->type == V4L2_BUF_TYPE_VBI_CAPTURE)
-		return videobuf_reqbufs(&fh->vbi_q, b);
-	return -EINVAL;
-}
-
-static int vidioc_querybuf(struct file *file, void *__fh, struct v4l2_buffer *buf)
-{
-	struct saa7146_fh *fh = __fh;
-
-	if (buf->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return videobuf_querybuf(&fh->video_q, buf);
-	if (buf->type == V4L2_BUF_TYPE_VBI_CAPTURE)
-		return videobuf_querybuf(&fh->vbi_q, buf);
-	return -EINVAL;
-}
-
-static int vidioc_qbuf(struct file *file, void *__fh, struct v4l2_buffer *buf)
-{
-	struct saa7146_fh *fh = __fh;
-
-	if (buf->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return videobuf_qbuf(&fh->video_q, buf);
-	if (buf->type == V4L2_BUF_TYPE_VBI_CAPTURE)
-		return videobuf_qbuf(&fh->vbi_q, buf);
-	return -EINVAL;
-}
-
-static int vidioc_dqbuf(struct file *file, void *__fh, struct v4l2_buffer *buf)
-{
-	struct saa7146_fh *fh = __fh;
-
-	if (buf->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return videobuf_dqbuf(&fh->video_q, buf, file->f_flags & O_NONBLOCK);
-	if (buf->type == V4L2_BUF_TYPE_VBI_CAPTURE)
-		return videobuf_dqbuf(&fh->vbi_q, buf, file->f_flags & O_NONBLOCK);
-	return -EINVAL;
-}
-
-static int vidioc_streamon(struct file *file, void *__fh, enum v4l2_buf_type type)
-{
-	struct saa7146_fh *fh = __fh;
-	int err;
-
-	DEB_D("VIDIOC_STREAMON, type:%d\n", type);
-
-	err = video_begin(fh);
-	if (err)
-		return err;
-	if (type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return videobuf_streamon(&fh->video_q);
-	if (type == V4L2_BUF_TYPE_VBI_CAPTURE)
-		return videobuf_streamon(&fh->vbi_q);
-	return -EINVAL;
-}
-
-static int vidioc_streamoff(struct file *file, void *__fh, enum v4l2_buf_type type)
-{
-	struct saa7146_fh *fh = __fh;
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	int err;
-
-	DEB_D("VIDIOC_STREAMOFF, type:%d\n", type);
-
-	/* ugly: we need to copy some checks from video_end(),
-	   because videobuf_streamoff() relies on the capture running.
-	   check and fix this */
-	if ((vv->video_status & STATUS_CAPTURE) != STATUS_CAPTURE) {
-		DEB_S("not capturing\n");
-		return 0;
-	}
-
-	if (vv->video_fh != fh) {
-		DEB_S("capturing, but in another open\n");
-		return -EBUSY;
-	}
-
-	err = -EINVAL;
-	if (type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		err = videobuf_streamoff(&fh->video_q);
-	else if (type == V4L2_BUF_TYPE_VBI_CAPTURE)
-		err = videobuf_streamoff(&fh->vbi_q);
-	if (0 != err) {
-		DEB_D("warning: videobuf_streamoff() failed\n");
-		video_end(fh, file);
-	} else {
-		err = video_end(fh, file);
-	}
-	return err;
-}
-
 const struct v4l2_ioctl_ops saa7146_video_ioctl_ops = {
 	.vidioc_querycap             = vidioc_querycap,
 	.vidioc_enum_fmt_vid_cap     = vidioc_enum_fmt_vid_cap,
-	.vidioc_enum_fmt_vid_overlay = vidioc_enum_fmt_vid_cap,
 	.vidioc_g_fmt_vid_cap        = vidioc_g_fmt_vid_cap,
 	.vidioc_try_fmt_vid_cap      = vidioc_try_fmt_vid_cap,
 	.vidioc_s_fmt_vid_cap        = vidioc_s_fmt_vid_cap,
-	.vidioc_g_fmt_vid_overlay    = vidioc_g_fmt_vid_overlay,
-	.vidioc_try_fmt_vid_overlay  = vidioc_try_fmt_vid_overlay,
-	.vidioc_s_fmt_vid_overlay    = vidioc_s_fmt_vid_overlay,
-
-	.vidioc_overlay		     = vidioc_overlay,
-	.vidioc_g_fbuf		     = vidioc_g_fbuf,
-	.vidioc_s_fbuf		     = vidioc_s_fbuf,
-	.vidioc_reqbufs              = vidioc_reqbufs,
-	.vidioc_querybuf             = vidioc_querybuf,
-	.vidioc_qbuf                 = vidioc_qbuf,
-	.vidioc_dqbuf                = vidioc_dqbuf,
 	.vidioc_g_std                = vidioc_g_std,
 	.vidioc_s_std                = vidioc_s_std,
-	.vidioc_streamon             = vidioc_streamon,
-	.vidioc_streamoff            = vidioc_streamoff,
 	.vidioc_g_parm		     = vidioc_g_parm,
+	.vidioc_reqbufs		     = vb2_ioctl_reqbufs,
+	.vidioc_create_bufs	     = vb2_ioctl_create_bufs,
+	.vidioc_querybuf	     = vb2_ioctl_querybuf,
+	.vidioc_qbuf		     = vb2_ioctl_qbuf,
+	.vidioc_dqbuf		     = vb2_ioctl_dqbuf,
+	.vidioc_streamon	     = vb2_ioctl_streamon,
+	.vidioc_streamoff	     = vb2_ioctl_streamoff,
 	.vidioc_subscribe_event      = v4l2_ctrl_subscribe_event,
 	.vidioc_unsubscribe_event    = v4l2_event_unsubscribe,
 };
@@ -996,16 +515,18 @@ const struct v4l2_ioctl_ops saa7146_video_ioctl_ops = {
 const struct v4l2_ioctl_ops saa7146_vbi_ioctl_ops = {
 	.vidioc_querycap             = vidioc_querycap,
 	.vidioc_g_fmt_vbi_cap        = vidioc_g_fmt_vbi_cap,
-
-	.vidioc_reqbufs              = vidioc_reqbufs,
-	.vidioc_querybuf             = vidioc_querybuf,
-	.vidioc_qbuf                 = vidioc_qbuf,
-	.vidioc_dqbuf                = vidioc_dqbuf,
+	.vidioc_try_fmt_vbi_cap      = vidioc_g_fmt_vbi_cap,
+	.vidioc_s_fmt_vbi_cap        = vidioc_g_fmt_vbi_cap,
 	.vidioc_g_std                = vidioc_g_std,
 	.vidioc_s_std                = vidioc_s_std,
-	.vidioc_streamon             = vidioc_streamon,
-	.vidioc_streamoff            = vidioc_streamoff,
 	.vidioc_g_parm		     = vidioc_g_parm,
+	.vidioc_reqbufs		     = vb2_ioctl_reqbufs,
+	.vidioc_create_bufs	     = vb2_ioctl_create_bufs,
+	.vidioc_querybuf	     = vb2_ioctl_querybuf,
+	.vidioc_qbuf		     = vb2_ioctl_qbuf,
+	.vidioc_dqbuf		     = vb2_ioctl_dqbuf,
+	.vidioc_streamon	     = vb2_ioctl_streamon,
+	.vidioc_streamoff	     = vb2_ioctl_streamoff,
 	.vidioc_subscribe_event      = v4l2_ctrl_subscribe_event,
 	.vidioc_unsubscribe_event    = v4l2_event_unsubscribe,
 };
@@ -1019,7 +540,6 @@ static int buffer_activate (struct saa7146_dev *dev,
 {
 	struct saa7146_vv *vv = dev->vv_data;
 
-	buf->vb.state = VIDEOBUF_ACTIVE;
 	saa7146_set_capture(dev,buf,next);
 
 	mod_timer(&vv->video_dmaq.timeout, jiffies+BUFFER_TIMEOUT);
@@ -1033,140 +553,136 @@ static void release_all_pagetables(struct saa7146_dev *dev, struct saa7146_buf *
 	saa7146_pgtable_free(dev->pci, &buf->pt[2]);
 }
 
-static int buffer_prepare(struct videobuf_queue *q,
-			  struct videobuf_buffer *vb, enum v4l2_field field)
+static int queue_setup(struct vb2_queue *q,
+		       unsigned int *num_buffers, unsigned int *num_planes,
+		       unsigned int sizes[], struct device *alloc_devs[])
 {
-	struct file *file = q->priv_data;
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	struct saa7146_buf *buf = (struct saa7146_buf *)vb;
-	int size,err = 0;
-
-	DEB_CAP("vbuf:%p\n", vb);
-
-	/* sanity checks */
-	if (vv->video_fmt.width  < 48 ||
-	    vv->video_fmt.height < 32 ||
-	    vv->video_fmt.width  > vv->standard->h_max_out ||
-	    vv->video_fmt.height > vv->standard->v_max_out) {
-		DEB_D("w (%d) / h (%d) out of bounds\n",
-		      vv->video_fmt.width, vv->video_fmt.height);
-		return -EINVAL;
-	}
+	struct saa7146_dev *dev = vb2_get_drv_priv(q);
+	unsigned int size = dev->vv_data->video_fmt.sizeimage;
 
-	size = vv->video_fmt.sizeimage;
-	if (0 != buf->vb.baddr && buf->vb.bsize < size) {
-		DEB_D("size mismatch\n");
-		return -EINVAL;
-	}
+	if (*num_planes)
+		return sizes[0] < size ? -EINVAL : 0;
+	*num_planes = 1;
+	sizes[0] = size;
 
-	DEB_CAP("buffer_prepare [size=%dx%d,bytes=%d,fields=%s]\n",
-		vv->video_fmt.width, vv->video_fmt.height,
-		size, v4l2_field_names[vv->video_fmt.field]);
-	if (buf->vb.width  != vv->video_fmt.width  ||
-	    buf->vb.bytesperline != vv->video_fmt.bytesperline ||
-	    buf->vb.height != vv->video_fmt.height ||
-	    buf->vb.size   != size ||
-	    buf->vb.field  != field      ||
-	    buf->vb.field  != vv->video_fmt.field  ||
-	    buf->fmt       != &vv->video_fmt) {
-		saa7146_dma_free(dev,q,buf);
-	}
+	return 0;
+}
 
-	if (VIDEOBUF_NEEDS_INIT == buf->vb.state) {
-		struct saa7146_format *sfmt;
+static void buf_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct vb2_queue *vq = vb->vb2_queue;
+	struct saa7146_dev *dev = vb2_get_drv_priv(vq);
+	struct saa7146_buf *buf = container_of(vbuf, struct saa7146_buf, vb);
+	unsigned long flags;
 
-		buf->vb.bytesperline  = vv->video_fmt.bytesperline;
-		buf->vb.width  = vv->video_fmt.width;
-		buf->vb.height = vv->video_fmt.height;
-		buf->vb.size   = size;
-		buf->vb.field  = field;
-		buf->fmt       = &vv->video_fmt;
-		buf->vb.field  = vv->video_fmt.field;
+	spin_lock_irqsave(&dev->slock, flags);
 
-		sfmt = saa7146_format_by_fourcc(dev,buf->fmt->pixelformat);
+	saa7146_buffer_queue(dev, &dev->vv_data->video_dmaq, buf);
+	spin_unlock_irqrestore(&dev->slock, flags);
+}
 
-		release_all_pagetables(dev, buf);
-		if( 0 != IS_PLANAR(sfmt->trans)) {
-			saa7146_pgtable_alloc(dev->pci, &buf->pt[0]);
-			saa7146_pgtable_alloc(dev->pci, &buf->pt[1]);
-			saa7146_pgtable_alloc(dev->pci, &buf->pt[2]);
-		} else {
-			saa7146_pgtable_alloc(dev->pci, &buf->pt[0]);
-		}
+static int buf_init(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct saa7146_buf *buf = container_of(vbuf, struct saa7146_buf, vb);
+	struct vb2_queue *vq = vb->vb2_queue;
+	struct saa7146_dev *dev = vb2_get_drv_priv(vq);
+	struct saa7146_vv *vv = dev->vv_data;
+	struct saa7146_format *sfmt;
+	int ret;
 
-		err = videobuf_iolock(q,&buf->vb, &vv->ov_fb);
-		if (err)
-			goto oops;
-		err = saa7146_pgtable_build(dev,buf);
-		if (err)
-			goto oops;
-	}
-	buf->vb.state = VIDEOBUF_PREPARED;
 	buf->activate = buffer_activate;
+	sfmt = saa7146_format_by_fourcc(dev, vv->video_fmt.pixelformat);
 
-	return 0;
-
- oops:
-	DEB_D("error out\n");
-	saa7146_dma_free(dev,q,buf);
+	if (IS_PLANAR(sfmt->trans)) {
+		saa7146_pgtable_alloc(dev->pci, &buf->pt[0]);
+		saa7146_pgtable_alloc(dev->pci, &buf->pt[1]);
+		saa7146_pgtable_alloc(dev->pci, &buf->pt[2]);
+	} else {
+		saa7146_pgtable_alloc(dev->pci, &buf->pt[0]);
+	}
 
-	return err;
+	ret = saa7146_pgtable_build(dev, buf);
+	if (ret)
+		release_all_pagetables(dev, buf);
+	return ret;
 }
 
-static int buffer_setup(struct videobuf_queue *q, unsigned int *count, unsigned int *size)
+static int buf_prepare(struct vb2_buffer *vb)
 {
-	struct file *file = q->priv_data;
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_vv *vv = fh->dev->vv_data;
-
-	if (0 == *count || *count > MAX_SAA7146_CAPTURE_BUFFERS)
-		*count = MAX_SAA7146_CAPTURE_BUFFERS;
-
-	*size = vv->video_fmt.sizeimage;
+	struct vb2_queue *vq = vb->vb2_queue;
+	struct saa7146_dev *dev = vb2_get_drv_priv(vq);
+	struct saa7146_vv *vv = dev->vv_data;
+	unsigned int size = vv->video_fmt.sizeimage;
 
-	/* check if we exceed the "max_memory" parameter */
-	if( (*count * *size) > (max_memory*1048576) ) {
-		*count = (max_memory*1048576) / *size;
-	}
+	if (vb2_plane_size(vb, 0) < size)
+		return -EINVAL;
+	vb2_set_plane_payload(vb, 0, size);
+	return 0;
+}
 
-	DEB_CAP("%d buffers, %d bytes each\n", *count, *size);
+static void buf_cleanup(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct saa7146_buf *buf = container_of(vbuf, struct saa7146_buf, vb);
+	struct vb2_queue *vq = vb->vb2_queue;
+	struct saa7146_dev *dev = vb2_get_drv_priv(vq);
 
-	return 0;
+	release_all_pagetables(dev, buf);
 }
 
-static void buffer_queue(struct videobuf_queue *q, struct videobuf_buffer *vb)
+static void return_buffers(struct vb2_queue *q, int state)
 {
-	struct file *file = q->priv_data;
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	struct saa7146_buf *buf = (struct saa7146_buf *)vb;
+	struct saa7146_dev *dev = vb2_get_drv_priv(q);
+	struct saa7146_dmaqueue *dq = &dev->vv_data->video_dmaq;
+	struct saa7146_buf *buf;
 
-	DEB_CAP("vbuf:%p\n", vb);
-	saa7146_buffer_queue(fh->dev, &vv->video_dmaq, buf);
+	if (dq->curr) {
+		buf = dq->curr;
+		dq->curr = NULL;
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+	}
+	while (!list_empty(&dq->queue)) {
+		buf = list_entry(dq->queue.next, struct saa7146_buf, list);
+		list_del(&buf->list);
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+	}
 }
 
-static void buffer_release(struct videobuf_queue *q, struct videobuf_buffer *vb)
+static int start_streaming(struct vb2_queue *q, unsigned int count)
 {
-	struct file *file = q->priv_data;
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_buf *buf = (struct saa7146_buf *)vb;
+	struct saa7146_dev *dev = vb2_get_drv_priv(q);
+	int ret;
 
-	DEB_CAP("vbuf:%p\n", vb);
+	if (!vb2_is_streaming(&dev->vv_data->video_dmaq.q))
+		dev->vv_data->seqnr = 0;
+	ret = video_begin(dev);
+	if (ret)
+		return_buffers(q, VB2_BUF_STATE_QUEUED);
+	return ret;
+}
 
-	saa7146_dma_free(dev,q,buf);
+static void stop_streaming(struct vb2_queue *q)
+{
+	struct saa7146_dev *dev = vb2_get_drv_priv(q);
+	struct saa7146_dmaqueue *dq = &dev->vv_data->video_dmaq;
 
-	release_all_pagetables(dev, buf);
+	del_timer(&dq->timeout);
+	video_end(dev);
+	return_buffers(q, VB2_BUF_STATE_ERROR);
 }
 
-static const struct videobuf_queue_ops video_qops = {
-	.buf_setup    = buffer_setup,
-	.buf_prepare  = buffer_prepare,
-	.buf_queue    = buffer_queue,
-	.buf_release  = buffer_release,
+const struct vb2_ops video_qops = {
+	.queue_setup	= queue_setup,
+	.buf_queue	= buf_queue,
+	.buf_init	= buf_init,
+	.buf_prepare	= buf_prepare,
+	.buf_cleanup	= buf_cleanup,
+	.start_streaming = start_streaming,
+	.stop_streaming = stop_streaming,
+	.wait_prepare	= vb2_ops_wait_prepare,
+	.wait_finish	= vb2_ops_wait_finish,
 };
 
 /********************************************************************************/
@@ -1187,38 +703,6 @@ static void video_init(struct saa7146_dev *dev, struct saa7146_vv *vv)
 	vv->current_hps_sync = SAA7146_HPS_SYNC_PORT_A;
 }
 
-
-static int video_open(struct saa7146_dev *dev, struct file *file)
-{
-	struct saa7146_fh *fh = file->private_data;
-
-	videobuf_queue_sg_init(&fh->video_q, &video_qops,
-			    &dev->pci->dev, &dev->slock,
-			    V4L2_BUF_TYPE_VIDEO_CAPTURE,
-			    V4L2_FIELD_INTERLACED,
-			    sizeof(struct saa7146_buf),
-			    file, &dev->v4l2_lock);
-
-	return 0;
-}
-
-
-static void video_close(struct saa7146_dev *dev, struct file *file)
-{
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_vv *vv = dev->vv_data;
-	struct videobuf_queue *q = &fh->video_q;
-
-	if (IS_CAPTURE_ACTIVE(fh) != 0)
-		video_end(fh, file);
-	else if (IS_OVERLAY_ACTIVE(fh) != 0)
-		saa7146_stop_preview(fh);
-
-	videobuf_stop(q);
-	/* hmm, why is this function declared void? */
-}
-
-
 static void video_irq_done(struct saa7146_dev *dev, unsigned long st)
 {
 	struct saa7146_vv *vv = dev->vv_data;
@@ -1228,59 +712,14 @@ static void video_irq_done(struct saa7146_dev *dev, unsigned long st)
 	DEB_CAP("called\n");
 
 	/* only finish the buffer if we have one... */
-	if( NULL != q->curr ) {
-		saa7146_buffer_finish(dev,q,VIDEOBUF_DONE);
-	}
+	if (q->curr)
+		saa7146_buffer_finish(dev, q, VB2_BUF_STATE_DONE);
 	saa7146_buffer_next(dev,q,0);
 
 	spin_unlock(&dev->slock);
 }
 
-static ssize_t video_read(struct file *file, char __user *data, size_t count, loff_t *ppos)
-{
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
-	ssize_t ret = 0;
-
-	DEB_EE("called\n");
-
-	if ((vv->video_status & STATUS_CAPTURE) != 0) {
-		/* fixme: should we allow read() captures while streaming capture? */
-		if (vv->video_fh == fh) {
-			DEB_S("already capturing\n");
-			return -EBUSY;
-		}
-		DEB_S("already capturing in another open\n");
-		return -EBUSY;
-	}
-
-	ret = video_begin(fh);
-	if( 0 != ret) {
-		goto out;
-	}
-
-	ret = videobuf_read_one(&fh->video_q , data, count, ppos,
-				file->f_flags & O_NONBLOCK);
-	if (ret != 0) {
-		video_end(fh, file);
-	} else {
-		ret = video_end(fh, file);
-	}
-out:
-	/* restart overlay if it was active before */
-	if (vv->ov_suspend != NULL) {
-		saa7146_start_preview(vv->ov_suspend);
-		vv->ov_suspend = NULL;
-	}
-
-	return ret;
-}
-
 const struct saa7146_use_ops saa7146_video_uops = {
 	.init = video_init,
-	.open = video_open,
-	.release = video_close,
 	.irq_done = video_irq_done,
-	.read = video_read,
 };
diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c
index 1f5d235a8441..c7a54d82a55e 100644
--- a/drivers/media/common/videobuf2/videobuf2-v4l2.c
+++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c
@@ -1171,10 +1171,11 @@ ssize_t vb2_fop_read(struct file *file, char __user *buf,
 		return -ERESTARTSYS;
 	if (vb2_queue_is_busy(vdev->queue, file))
 		goto exit;
+	vdev->queue->owner = file->private_data;
 	err = vb2_read(vdev->queue, buf, count, ppos,
 		       file->f_flags & O_NONBLOCK);
-	if (vdev->queue->fileio)
-		vdev->queue->owner = file->private_data;
+	if (!vdev->queue->fileio)
+		vdev->queue->owner = NULL;
 exit:
 	if (lock)
 		mutex_unlock(lock);
diff --git a/drivers/media/dvb-core/dvbdev.c b/drivers/media/dvb-core/dvbdev.c
index 0ed087caf7f3..e9b3ce09e534 100644
--- a/drivers/media/dvb-core/dvbdev.c
+++ b/drivers/media/dvb-core/dvbdev.c
@@ -1063,7 +1063,7 @@ static int __init init_dvbdev(void)
 		goto error;
 	}
 
-	dvb_class = class_create(THIS_MODULE, "dvb");
+	dvb_class = class_create("dvb");
 	if (IS_ERR(dvb_class)) {
 		retval = PTR_ERR(dvb_class);
 		goto error;
diff --git a/drivers/media/dvb-frontends/drx39xyj/drxj.c b/drivers/media/dvb-frontends/drx39xyj/drxj.c
index 6bf6559b127f..68f4e8b5a0ab 100644
--- a/drivers/media/dvb-frontends/drx39xyj/drxj.c
+++ b/drivers/media/dvb-frontends/drx39xyj/drxj.c
@@ -11070,7 +11070,7 @@ ctrl_power_mode(struct drx_demod_instance *demod, enum drx_power_mode *mode)
 		sio_cc_pwd_mode = SIO_CC_PWD_MODE_LEVEL_OSC;
 		break;
 	default:
-		/* Unknow sleep mode */
+		/* Unknown sleep mode */
 		return -EINVAL;
 	}
 
diff --git a/drivers/media/dvb-frontends/drxk_hard.c b/drivers/media/dvb-frontends/drxk_hard.c
index 9807f5411996..3301ef75d441 100644
--- a/drivers/media/dvb-frontends/drxk_hard.c
+++ b/drivers/media/dvb-frontends/drxk_hard.c
@@ -1585,7 +1585,7 @@ static int ctrl_power_mode(struct drxk_state *state, enum drx_power_mode *mode)
 		sio_cc_pwd_mode = SIO_CC_PWD_MODE_LEVEL_OSC;
 		break;
 	default:
-		/* Unknow sleep mode */
+		/* Unknown sleep mode */
 		return -EINVAL;
 	}
 
diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.c b/drivers/media/dvb-frontends/rtl2832_sdr.c
index 05f71d169726..02c619e51641 100644
--- a/drivers/media/dvb-frontends/rtl2832_sdr.c
+++ b/drivers/media/dvb-frontends/rtl2832_sdr.c
@@ -1463,7 +1463,7 @@ err:
 	return ret;
 }
 
-static int rtl2832_sdr_remove(struct platform_device *pdev)
+static void rtl2832_sdr_remove(struct platform_device *pdev)
 {
 	struct rtl2832_sdr_dev *dev = platform_get_drvdata(pdev);
 
@@ -1479,8 +1479,6 @@ static int rtl2832_sdr_remove(struct platform_device *pdev)
 	mutex_unlock(&dev->vb_queue_lock);
 	v4l2_device_put(&dev->v4l2_dev);
 	module_put(pdev->dev.parent->driver->owner);
-
-	return 0;
 }
 
 static struct platform_driver rtl2832_sdr_driver = {
@@ -1488,7 +1486,7 @@ static struct platform_driver rtl2832_sdr_driver = {
 		.name   = "rtl2832_sdr",
 	},
 	.probe          = rtl2832_sdr_probe,
-	.remove         = rtl2832_sdr_remove,
+	.remove_new     = rtl2832_sdr_remove,
 };
 module_platform_driver(rtl2832_sdr_driver);
 
diff --git a/drivers/media/dvb-frontends/zd1301_demod.c b/drivers/media/dvb-frontends/zd1301_demod.c
index bbabe6a2d4f4..17f6e373c13d 100644
--- a/drivers/media/dvb-frontends/zd1301_demod.c
+++ b/drivers/media/dvb-frontends/zd1301_demod.c
@@ -515,7 +515,7 @@ err:
 	return ret;
 }
 
-static int zd1301_demod_remove(struct platform_device *pdev)
+static void zd1301_demod_remove(struct platform_device *pdev)
 {
 	struct zd1301_demod_dev *dev = platform_get_drvdata(pdev);
 
@@ -523,8 +523,6 @@ static int zd1301_demod_remove(struct platform_device *pdev)
 
 	i2c_del_adapter(&dev->adapter);
 	kfree(dev);
-
-	return 0;
 }
 
 static struct platform_driver zd1301_demod_driver = {
@@ -533,7 +531,7 @@ static struct platform_driver zd1301_demod_driver = {
 		.suppress_bind_attrs = true,
 	},
 	.probe          = zd1301_demod_probe,
-	.remove         = zd1301_demod_remove,
+	.remove_new     = zd1301_demod_remove,
 };
 module_platform_driver(zd1301_demod_driver);
 
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index c3d5952ca27e..256d55bb2b1d 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -267,16 +267,6 @@ config VIDEO_MT9M001
 	  This driver supports MT9M001 cameras from Micron, monochrome
 	  and colour models.
 
-config VIDEO_MT9M032
-	tristate "MT9M032 camera sensor support"
-	depends on I2C && VIDEO_DEV
-	select MEDIA_CONTROLLER
-	select VIDEO_V4L2_SUBDEV_API
-	select VIDEO_APTINA_PLL
-	help
-	  This driver supports MT9M032 camera sensors from Aptina, monochrome
-	  models only.
-
 config VIDEO_MT9M111
 	tristate "mt9m111, mt9m112 and mt9m131 support"
 	depends on I2C && VIDEO_DEV
@@ -296,15 +286,6 @@ config VIDEO_MT9P031
 	  This is a Video4Linux2 sensor driver for the Aptina
 	  (Micron) mt9p031 5 Mpixel camera.
 
-config VIDEO_MT9T001
-	tristate "Aptina MT9T001 support"
-	depends on I2C && VIDEO_DEV
-	select MEDIA_CONTROLLER
-	select VIDEO_V4L2_SUBDEV_API
-	help
-	  This is a Video4Linux2 sensor driver for the Aptina
-	  (Micron) mt0t001 3 Mpixel camera.
-
 config VIDEO_MT9T112
 	tristate "Aptina MT9T111/MT9T112 support"
 	depends on I2C && VIDEO_DEV
@@ -344,14 +325,6 @@ config VIDEO_MT9V111
 	  To compile this driver as a module, choose M here: the
 	  module will be called mt9v111.
 
-config VIDEO_NOON010PC30
-	tristate "Siliconfile NOON010PC30 sensor support"
-	depends on I2C && VIDEO_DEV
-	select MEDIA_CONTROLLER
-	select VIDEO_V4L2_SUBDEV_API
-	help
-	  This driver supports NOON010PC30 CIF camera from Siliconfile
-
 config VIDEO_OG01A1B
 	tristate "OmniVision OG01A1B sensor support"
 	depends on I2C && VIDEO_DEV
@@ -462,6 +435,7 @@ config VIDEO_OV2685
 	tristate "OmniVision OV2685 sensor support"
 	depends on VIDEO_DEV && I2C
 	select MEDIA_CONTROLLER
+	select VIDEO_V4L2_SUBDEV_API
 	select V4L2_FWNODE
 	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
@@ -810,21 +784,6 @@ config VIDEO_S5K6A3
 	  This is a V4L2 sensor driver for Samsung S5K6A3 raw
 	  camera sensor.
 
-config VIDEO_S5K6AA
-	tristate "Samsung S5K6AAFX sensor support"
-	depends on I2C && VIDEO_DEV
-	select MEDIA_CONTROLLER
-	select VIDEO_V4L2_SUBDEV_API
-	help
-	  This is a V4L2 sensor driver for Samsung S5K6AA(FX) 1.3M
-	  camera sensor with an embedded SoC image signal processor.
-
-config VIDEO_SR030PC30
-	tristate "Siliconfile SR030PC30 sensor support"
-	depends on I2C && VIDEO_DEV
-	help
-	  This driver supports SR030PC30 VGA camera from Siliconfile
-
 config VIDEO_ST_VGXY61
 	tristate "ST VGXY61 sensor support"
 	depends on OF && GPIOLIB && VIDEO_DEV && I2C
@@ -835,19 +794,8 @@ config VIDEO_ST_VGXY61
 	  This is a Video4Linux2 sensor driver for the ST VGXY61
 	  camera sensor.
 
-config VIDEO_VS6624
-	tristate "ST VS6624 sensor support"
-	depends on VIDEO_DEV && I2C
-	help
-	  This is a Video4Linux2 sensor driver for the ST VS6624
-	  camera.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called vs6624.
-
 source "drivers/media/i2c/ccs/Kconfig"
 source "drivers/media/i2c/et8ek8/Kconfig"
-source "drivers/media/i2c/m5mols/Kconfig"
 
 endmenu
 
@@ -1453,20 +1401,6 @@ endmenu
 menu "Video encoders"
 	visible if !MEDIA_HIDE_ANCILLARY_SUBDRV
 
-config VIDEO_AD9389B
-	tristate "Analog Devices AD9389B encoder"
-	depends on VIDEO_DEV && I2C
-	select MEDIA_CONTROLLER
-	select VIDEO_V4L2_SUBDEV_API
-
-	help
-	  Support for the Analog Devices AD9389B video encoder.
-
-	  This is a Analog Devices HDMI transmitter.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ad9389b.
-
 config VIDEO_ADV7170
 	tristate "Analog Devices ADV7170 video encoder"
 	depends on VIDEO_DEV && I2C
diff --git a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile
index 4f5e9d9cee85..b44dacf935f4 100644
--- a/drivers/media/i2c/Makefile
+++ b/drivers/media/i2c/Makefile
@@ -4,7 +4,6 @@ msp3400-objs	:=	msp3400-driver.o msp3400-kthreads.o
 
 obj-$(CONFIG_SDR_MAX2175) += max2175.o
 obj-$(CONFIG_VIDEO_AD5820) += ad5820.o
-obj-$(CONFIG_VIDEO_AD9389B) += ad9389b.o
 obj-$(CONFIG_VIDEO_ADP1653) += adp1653.o
 obj-$(CONFIG_VIDEO_ADV7170) += adv7170.o
 obj-$(CONFIG_VIDEO_ADV7175) += adv7175.o
@@ -56,21 +55,17 @@ obj-$(CONFIG_VIDEO_KS0127) += ks0127.o
 obj-$(CONFIG_VIDEO_LM3560) += lm3560.o
 obj-$(CONFIG_VIDEO_LM3646) += lm3646.o
 obj-$(CONFIG_VIDEO_M52790) += m52790.o
-obj-$(CONFIG_VIDEO_M5MOLS) += m5mols/
 obj-$(CONFIG_VIDEO_MAX9271_LIB) += max9271.o
 obj-$(CONFIG_VIDEO_MAX9286) += max9286.o
 obj-$(CONFIG_VIDEO_ML86V7667) += ml86v7667.o
 obj-$(CONFIG_VIDEO_MSP3400) += msp3400.o
 obj-$(CONFIG_VIDEO_MT9M001) += mt9m001.o
-obj-$(CONFIG_VIDEO_MT9M032) += mt9m032.o
 obj-$(CONFIG_VIDEO_MT9M111) += mt9m111.o
 obj-$(CONFIG_VIDEO_MT9P031) += mt9p031.o
-obj-$(CONFIG_VIDEO_MT9T001) += mt9t001.o
 obj-$(CONFIG_VIDEO_MT9T112) += mt9t112.o
 obj-$(CONFIG_VIDEO_MT9V011) += mt9v011.o
 obj-$(CONFIG_VIDEO_MT9V032) += mt9v032.o
 obj-$(CONFIG_VIDEO_MT9V111) += mt9v111.o
-obj-$(CONFIG_VIDEO_NOON010PC30) += noon010pc30.o
 obj-$(CONFIG_VIDEO_OG01A1B) += og01a1b.o
 obj-$(CONFIG_VIDEO_OV02A10) += ov02a10.o
 obj-$(CONFIG_VIDEO_OV08D10) += ov08d10.o
@@ -110,7 +105,6 @@ obj-$(CONFIG_VIDEO_RJ54N1) += rj54n1cb0c.o
 obj-$(CONFIG_VIDEO_S5C73M3) += s5c73m3/
 obj-$(CONFIG_VIDEO_S5K5BAF) += s5k5baf.o
 obj-$(CONFIG_VIDEO_S5K6A3) += s5k6a3.o
-obj-$(CONFIG_VIDEO_S5K6AA) += s5k6aa.o
 obj-$(CONFIG_VIDEO_SAA6588) += saa6588.o
 obj-$(CONFIG_VIDEO_SAA6752HS) += saa6752hs.o
 obj-$(CONFIG_VIDEO_SAA7110) += saa7110.o
@@ -119,7 +113,6 @@ obj-$(CONFIG_VIDEO_SAA7127) += saa7127.o
 obj-$(CONFIG_VIDEO_SAA717X) += saa717x.o
 obj-$(CONFIG_VIDEO_SAA7185) += saa7185.o
 obj-$(CONFIG_VIDEO_SONY_BTF_MPX) += sony-btf-mpx.o
-obj-$(CONFIG_VIDEO_SR030PC30) += sr030pc30.o
 obj-$(CONFIG_VIDEO_ST_MIPID02) += st-mipid02.o
 obj-$(CONFIG_VIDEO_ST_VGXY61) += st-vgxy61.o
 obj-$(CONFIG_VIDEO_TC358743) += tc358743.o
@@ -145,6 +138,5 @@ obj-$(CONFIG_VIDEO_UPD64031A) += upd64031a.o
 obj-$(CONFIG_VIDEO_UPD64083) += upd64083.o
 obj-$(CONFIG_VIDEO_VP27SMPX) += vp27smpx.o
 obj-$(CONFIG_VIDEO_VPX3220) += vpx3220.o
-obj-$(CONFIG_VIDEO_VS6624) += vs6624.o
 obj-$(CONFIG_VIDEO_WM8739) += wm8739.o
 obj-$(CONFIG_VIDEO_WM8775) += wm8775.o
diff --git a/drivers/media/i2c/ad9389b.c b/drivers/media/i2c/ad9389b.c
deleted file mode 100644
index ad17097a2d25..000000000000
--- a/drivers/media/i2c/ad9389b.c
+++ /dev/null
@@ -1,1215 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Analog Devices AD9389B/AD9889B video encoder driver
- *
- * Copyright 2012 Cisco Systems, Inc. and/or its affiliates. All rights reserved.
- */
-
-/*
- * References (c = chapter, p = page):
- * REF_01 - Analog Devices, Programming Guide, AD9889B/AD9389B,
- * HDMI Transitter, Rev. A, October 2010
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/i2c.h>
-#include <linux/delay.h>
-#include <linux/videodev2.h>
-#include <linux/workqueue.h>
-#include <linux/v4l2-dv-timings.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-common.h>
-#include <media/v4l2-dv-timings.h>
-#include <media/v4l2-ctrls.h>
-#include <media/i2c/ad9389b.h>
-
-static int debug;
-module_param(debug, int, 0644);
-MODULE_PARM_DESC(debug, "debug level (0-2)");
-
-MODULE_DESCRIPTION("Analog Devices AD9389B/AD9889B video encoder driver");
-MODULE_AUTHOR("Hans Verkuil <hans.verkuil@cisco.com>");
-MODULE_AUTHOR("Martin Bugge <marbugge@cisco.com>");
-MODULE_LICENSE("GPL");
-
-#define MASK_AD9389B_EDID_RDY_INT   0x04
-#define MASK_AD9389B_MSEN_INT       0x40
-#define MASK_AD9389B_HPD_INT        0x80
-
-#define MASK_AD9389B_HPD_DETECT     0x40
-#define MASK_AD9389B_MSEN_DETECT    0x20
-#define MASK_AD9389B_EDID_RDY       0x10
-
-#define EDID_MAX_RETRIES (8)
-#define EDID_DELAY 250
-#define EDID_MAX_SEGM 8
-
-/*
-**********************************************************************
-*
-*  Arrays with configuration parameters for the AD9389B
-*
-**********************************************************************
-*/
-
-struct ad9389b_state_edid {
-	/* total number of blocks */
-	u32 blocks;
-	/* Number of segments read */
-	u32 segments;
-	u8 data[EDID_MAX_SEGM * 256];
-	/* Number of EDID read retries left */
-	unsigned read_retries;
-};
-
-struct ad9389b_state {
-	struct ad9389b_platform_data pdata;
-	struct v4l2_subdev sd;
-	struct media_pad pad;
-	struct v4l2_ctrl_handler hdl;
-	int chip_revision;
-	/* Is the ad9389b powered on? */
-	bool power_on;
-	/* Did we receive hotplug and rx-sense signals? */
-	bool have_monitor;
-	/* timings from s_dv_timings */
-	struct v4l2_dv_timings dv_timings;
-	/* controls */
-	struct v4l2_ctrl *hdmi_mode_ctrl;
-	struct v4l2_ctrl *hotplug_ctrl;
-	struct v4l2_ctrl *rx_sense_ctrl;
-	struct v4l2_ctrl *have_edid0_ctrl;
-	struct v4l2_ctrl *rgb_quantization_range_ctrl;
-	struct i2c_client *edid_i2c_client;
-	struct ad9389b_state_edid edid;
-	/* Running counter of the number of detected EDIDs (for debugging) */
-	unsigned edid_detect_counter;
-	struct delayed_work edid_handler; /* work entry */
-};
-
-static void ad9389b_check_monitor_present_status(struct v4l2_subdev *sd);
-static bool ad9389b_check_edid_status(struct v4l2_subdev *sd);
-static void ad9389b_setup(struct v4l2_subdev *sd);
-static int ad9389b_s_i2s_clock_freq(struct v4l2_subdev *sd, u32 freq);
-static int ad9389b_s_clock_freq(struct v4l2_subdev *sd, u32 freq);
-
-static inline struct ad9389b_state *get_ad9389b_state(struct v4l2_subdev *sd)
-{
-	return container_of(sd, struct ad9389b_state, sd);
-}
-
-static inline struct v4l2_subdev *to_sd(struct v4l2_ctrl *ctrl)
-{
-	return &container_of(ctrl->handler, struct ad9389b_state, hdl)->sd;
-}
-
-/* ------------------------ I2C ----------------------------------------------- */
-
-static int ad9389b_rd(struct v4l2_subdev *sd, u8 reg)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-
-	return i2c_smbus_read_byte_data(client, reg);
-}
-
-static int ad9389b_wr(struct v4l2_subdev *sd, u8 reg, u8 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-	int i;
-
-	for (i = 0; i < 3; i++) {
-		ret = i2c_smbus_write_byte_data(client, reg, val);
-		if (ret == 0)
-			return 0;
-	}
-	v4l2_err(sd, "%s: failed reg 0x%x, val 0x%x\n", __func__, reg, val);
-	return ret;
-}
-
-/* To set specific bits in the register, a clear-mask is given (to be AND-ed),
-   and then the value-mask (to be OR-ed). */
-static inline void ad9389b_wr_and_or(struct v4l2_subdev *sd, u8 reg,
-				     u8 clr_mask, u8 val_mask)
-{
-	ad9389b_wr(sd, reg, (ad9389b_rd(sd, reg) & clr_mask) | val_mask);
-}
-
-static void ad9389b_edid_rd(struct v4l2_subdev *sd, u16 len, u8 *buf)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-	int i;
-
-	v4l2_dbg(1, debug, sd, "%s:\n", __func__);
-
-	for (i = 0; i < len; i++)
-		buf[i] = i2c_smbus_read_byte_data(state->edid_i2c_client, i);
-}
-
-static inline bool ad9389b_have_hotplug(struct v4l2_subdev *sd)
-{
-	return ad9389b_rd(sd, 0x42) & MASK_AD9389B_HPD_DETECT;
-}
-
-static inline bool ad9389b_have_rx_sense(struct v4l2_subdev *sd)
-{
-	return ad9389b_rd(sd, 0x42) & MASK_AD9389B_MSEN_DETECT;
-}
-
-static void ad9389b_csc_conversion_mode(struct v4l2_subdev *sd, u8 mode)
-{
-	ad9389b_wr_and_or(sd, 0x17, 0xe7, (mode & 0x3)<<3);
-	ad9389b_wr_and_or(sd, 0x18, 0x9f, (mode & 0x3)<<5);
-}
-
-static void ad9389b_csc_coeff(struct v4l2_subdev *sd,
-			      u16 A1, u16 A2, u16 A3, u16 A4,
-			      u16 B1, u16 B2, u16 B3, u16 B4,
-			      u16 C1, u16 C2, u16 C3, u16 C4)
-{
-	/* A */
-	ad9389b_wr_and_or(sd, 0x18, 0xe0, A1>>8);
-	ad9389b_wr(sd, 0x19, A1);
-	ad9389b_wr_and_or(sd, 0x1A, 0xe0, A2>>8);
-	ad9389b_wr(sd, 0x1B, A2);
-	ad9389b_wr_and_or(sd, 0x1c, 0xe0, A3>>8);
-	ad9389b_wr(sd, 0x1d, A3);
-	ad9389b_wr_and_or(sd, 0x1e, 0xe0, A4>>8);
-	ad9389b_wr(sd, 0x1f, A4);
-
-	/* B */
-	ad9389b_wr_and_or(sd, 0x20, 0xe0, B1>>8);
-	ad9389b_wr(sd, 0x21, B1);
-	ad9389b_wr_and_or(sd, 0x22, 0xe0, B2>>8);
-	ad9389b_wr(sd, 0x23, B2);
-	ad9389b_wr_and_or(sd, 0x24, 0xe0, B3>>8);
-	ad9389b_wr(sd, 0x25, B3);
-	ad9389b_wr_and_or(sd, 0x26, 0xe0, B4>>8);
-	ad9389b_wr(sd, 0x27, B4);
-
-	/* C */
-	ad9389b_wr_and_or(sd, 0x28, 0xe0, C1>>8);
-	ad9389b_wr(sd, 0x29, C1);
-	ad9389b_wr_and_or(sd, 0x2A, 0xe0, C2>>8);
-	ad9389b_wr(sd, 0x2B, C2);
-	ad9389b_wr_and_or(sd, 0x2C, 0xe0, C3>>8);
-	ad9389b_wr(sd, 0x2D, C3);
-	ad9389b_wr_and_or(sd, 0x2E, 0xe0, C4>>8);
-	ad9389b_wr(sd, 0x2F, C4);
-}
-
-static void ad9389b_csc_rgb_full2limit(struct v4l2_subdev *sd, bool enable)
-{
-	if (enable) {
-		u8 csc_mode = 0;
-
-		ad9389b_csc_conversion_mode(sd, csc_mode);
-		ad9389b_csc_coeff(sd,
-				  4096-564, 0, 0, 256,
-				  0, 4096-564, 0, 256,
-				  0, 0, 4096-564, 256);
-		/* enable CSC */
-		ad9389b_wr_and_or(sd, 0x3b, 0xfe, 0x1);
-		/* AVI infoframe: Limited range RGB (16-235) */
-		ad9389b_wr_and_or(sd, 0xcd, 0xf9, 0x02);
-	} else {
-		/* disable CSC */
-		ad9389b_wr_and_or(sd, 0x3b, 0xfe, 0x0);
-		/* AVI infoframe: Full range RGB (0-255) */
-		ad9389b_wr_and_or(sd, 0xcd, 0xf9, 0x04);
-	}
-}
-
-static void ad9389b_set_IT_content_AVI_InfoFrame(struct v4l2_subdev *sd)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-
-	if (state->dv_timings.bt.flags & V4L2_DV_FL_IS_CE_VIDEO) {
-		/* CE format, not IT  */
-		ad9389b_wr_and_or(sd, 0xcd, 0xbf, 0x00);
-	} else {
-		/* IT format */
-		ad9389b_wr_and_or(sd, 0xcd, 0xbf, 0x40);
-	}
-}
-
-static int ad9389b_set_rgb_quantization_mode(struct v4l2_subdev *sd, struct v4l2_ctrl *ctrl)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-
-	switch (ctrl->val) {
-	case V4L2_DV_RGB_RANGE_AUTO:
-		/* automatic */
-		if (state->dv_timings.bt.flags & V4L2_DV_FL_IS_CE_VIDEO) {
-			/* CE format, RGB limited range (16-235) */
-			ad9389b_csc_rgb_full2limit(sd, true);
-		} else {
-			/* not CE format, RGB full range (0-255) */
-			ad9389b_csc_rgb_full2limit(sd, false);
-		}
-		break;
-	case V4L2_DV_RGB_RANGE_LIMITED:
-		/* RGB limited range (16-235) */
-		ad9389b_csc_rgb_full2limit(sd, true);
-		break;
-	case V4L2_DV_RGB_RANGE_FULL:
-		/* RGB full range (0-255) */
-		ad9389b_csc_rgb_full2limit(sd, false);
-		break;
-	default:
-		return -EINVAL;
-	}
-	return 0;
-}
-
-static void ad9389b_set_manual_pll_gear(struct v4l2_subdev *sd, u32 pixelclock)
-{
-	u8 gear;
-
-	/* Workaround for TMDS PLL problem
-	 * The TMDS PLL in AD9389b change gear when the chip is heated above a
-	 * certain temperature. The output is disabled when the PLL change gear
-	 * so the monitor has to lock on the signal again. A workaround for
-	 * this is to use the manual PLL gears. This is a solution from Analog
-	 * Devices that is not documented in the datasheets.
-	 * 0x98 [7] = enable manual gearing. 0x98 [6:4] = gear
-	 *
-	 * The pixel frequency ranges are based on readout of the gear the
-	 * automatic gearing selects for different pixel clocks
-	 * (read from 0x9e [3:1]).
-	 */
-
-	if (pixelclock > 140000000)
-		gear = 0xc0; /* 4th gear */
-	else if (pixelclock > 117000000)
-		gear = 0xb0; /* 3rd gear */
-	else if (pixelclock > 87000000)
-		gear = 0xa0; /* 2nd gear */
-	else if (pixelclock > 60000000)
-		gear = 0x90; /* 1st gear */
-	else
-		gear = 0x80; /* 0th gear */
-
-	ad9389b_wr_and_or(sd, 0x98, 0x0f, gear);
-}
-
-/* ------------------------------ CTRL OPS ------------------------------ */
-
-static int ad9389b_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct v4l2_subdev *sd = to_sd(ctrl);
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-
-	v4l2_dbg(1, debug, sd,
-		 "%s: ctrl id: %d, ctrl->val %d\n", __func__, ctrl->id, ctrl->val);
-
-	if (state->hdmi_mode_ctrl == ctrl) {
-		/* Set HDMI or DVI-D */
-		ad9389b_wr_and_or(sd, 0xaf, 0xfd,
-				  ctrl->val == V4L2_DV_TX_MODE_HDMI ? 0x02 : 0x00);
-		return 0;
-	}
-	if (state->rgb_quantization_range_ctrl == ctrl)
-		return ad9389b_set_rgb_quantization_mode(sd, ctrl);
-	return -EINVAL;
-}
-
-static const struct v4l2_ctrl_ops ad9389b_ctrl_ops = {
-	.s_ctrl = ad9389b_s_ctrl,
-};
-
-/* ---------------------------- CORE OPS ------------------------------------------- */
-
-#ifdef CONFIG_VIDEO_ADV_DEBUG
-static int ad9389b_g_register(struct v4l2_subdev *sd, struct v4l2_dbg_register *reg)
-{
-	reg->val = ad9389b_rd(sd, reg->reg & 0xff);
-	reg->size = 1;
-	return 0;
-}
-
-static int ad9389b_s_register(struct v4l2_subdev *sd, const struct v4l2_dbg_register *reg)
-{
-	ad9389b_wr(sd, reg->reg & 0xff, reg->val & 0xff);
-	return 0;
-}
-#endif
-
-static int ad9389b_log_status(struct v4l2_subdev *sd)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-	struct ad9389b_state_edid *edid = &state->edid;
-
-	static const char * const states[] = {
-		"in reset",
-		"reading EDID",
-		"idle",
-		"initializing HDCP",
-		"HDCP enabled",
-		"initializing HDCP repeater",
-		"6", "7", "8", "9", "A", "B", "C", "D", "E", "F"
-	};
-	static const char * const errors[] = {
-		"no error",
-		"bad receiver BKSV",
-		"Ri mismatch",
-		"Pj mismatch",
-		"i2c error",
-		"timed out",
-		"max repeater cascade exceeded",
-		"hash check failed",
-		"too many devices",
-		"9", "A", "B", "C", "D", "E", "F"
-	};
-
-	u8 manual_gear;
-
-	v4l2_info(sd, "chip revision %d\n", state->chip_revision);
-	v4l2_info(sd, "power %s\n", state->power_on ? "on" : "off");
-	v4l2_info(sd, "%s hotplug, %s Rx Sense, %s EDID (%d block(s))\n",
-		  (ad9389b_rd(sd, 0x42) & MASK_AD9389B_HPD_DETECT) ?
-		  "detected" : "no",
-		  (ad9389b_rd(sd, 0x42) & MASK_AD9389B_MSEN_DETECT) ?
-		  "detected" : "no",
-		  edid->segments ? "found" : "no", edid->blocks);
-	v4l2_info(sd, "%s output %s\n",
-		  (ad9389b_rd(sd, 0xaf) & 0x02) ?
-		  "HDMI" : "DVI-D",
-		  (ad9389b_rd(sd, 0xa1) & 0x3c) ?
-		  "disabled" : "enabled");
-	v4l2_info(sd, "ad9389b: %s\n", (ad9389b_rd(sd, 0xb8) & 0x40) ?
-		  "encrypted" : "no encryption");
-	v4l2_info(sd, "state: %s, error: %s, detect count: %u, msk/irq: %02x/%02x\n",
-		  states[ad9389b_rd(sd, 0xc8) & 0xf],
-		  errors[ad9389b_rd(sd, 0xc8) >> 4],
-		  state->edid_detect_counter,
-		  ad9389b_rd(sd, 0x94), ad9389b_rd(sd, 0x96));
-	manual_gear = ad9389b_rd(sd, 0x98) & 0x80;
-	v4l2_info(sd, "ad9389b: RGB quantization: %s range\n",
-		  ad9389b_rd(sd, 0x3b) & 0x01 ? "limited" : "full");
-	v4l2_info(sd, "ad9389b: %s gear %d\n",
-		  manual_gear ? "manual" : "automatic",
-		  manual_gear ? ((ad9389b_rd(sd, 0x98) & 0x70) >> 4) :
-		  ((ad9389b_rd(sd, 0x9e) & 0x0e) >> 1));
-	if (ad9389b_rd(sd, 0xaf) & 0x02) {
-		/* HDMI only */
-		u8 manual_cts = ad9389b_rd(sd, 0x0a) & 0x80;
-		u32 N = (ad9389b_rd(sd, 0x01) & 0xf) << 16 |
-			ad9389b_rd(sd, 0x02) << 8 |
-			ad9389b_rd(sd, 0x03);
-		u8 vic_detect = ad9389b_rd(sd, 0x3e) >> 2;
-		u8 vic_sent = ad9389b_rd(sd, 0x3d) & 0x3f;
-		u32 CTS;
-
-		if (manual_cts)
-			CTS = (ad9389b_rd(sd, 0x07) & 0xf) << 16 |
-			      ad9389b_rd(sd, 0x08) << 8 |
-			      ad9389b_rd(sd, 0x09);
-		else
-			CTS = (ad9389b_rd(sd, 0x04) & 0xf) << 16 |
-			      ad9389b_rd(sd, 0x05) << 8 |
-			      ad9389b_rd(sd, 0x06);
-		N = (ad9389b_rd(sd, 0x01) & 0xf) << 16 |
-		    ad9389b_rd(sd, 0x02) << 8 |
-		    ad9389b_rd(sd, 0x03);
-
-		v4l2_info(sd, "ad9389b: CTS %s mode: N %d, CTS %d\n",
-			  manual_cts ? "manual" : "automatic", N, CTS);
-
-		v4l2_info(sd, "ad9389b: VIC: detected %d, sent %d\n",
-			  vic_detect, vic_sent);
-	}
-	if (state->dv_timings.type == V4L2_DV_BT_656_1120)
-		v4l2_print_dv_timings(sd->name, "timings: ",
-				&state->dv_timings, false);
-	else
-		v4l2_info(sd, "no timings set\n");
-	return 0;
-}
-
-/* Power up/down ad9389b */
-static int ad9389b_s_power(struct v4l2_subdev *sd, int on)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-	struct ad9389b_platform_data *pdata = &state->pdata;
-	const int retries = 20;
-	int i;
-
-	v4l2_dbg(1, debug, sd, "%s: power %s\n", __func__, on ? "on" : "off");
-
-	state->power_on = on;
-
-	if (!on) {
-		/* Power down */
-		ad9389b_wr_and_or(sd, 0x41, 0xbf, 0x40);
-		return true;
-	}
-
-	/* Power up */
-	/* The ad9389b does not always come up immediately.
-	   Retry multiple times. */
-	for (i = 0; i < retries; i++) {
-		ad9389b_wr_and_or(sd, 0x41, 0xbf, 0x0);
-		if ((ad9389b_rd(sd, 0x41) & 0x40) == 0)
-			break;
-		ad9389b_wr_and_or(sd, 0x41, 0xbf, 0x40);
-		msleep(10);
-	}
-	if (i == retries) {
-		v4l2_dbg(1, debug, sd, "failed to powerup the ad9389b\n");
-		ad9389b_s_power(sd, 0);
-		return false;
-	}
-	if (i > 1)
-		v4l2_dbg(1, debug, sd,
-			 "needed %d retries to powerup the ad9389b\n", i);
-
-	/* Select chip: AD9389B */
-	ad9389b_wr_and_or(sd, 0xba, 0xef, 0x10);
-
-	/* Reserved registers that must be set according to REF_01 p. 11*/
-	ad9389b_wr_and_or(sd, 0x98, 0xf0, 0x07);
-	ad9389b_wr(sd, 0x9c, 0x38);
-	ad9389b_wr_and_or(sd, 0x9d, 0xfc, 0x01);
-
-	/* Differential output drive strength */
-	if (pdata->diff_data_drive_strength > 0)
-		ad9389b_wr(sd, 0xa2, pdata->diff_data_drive_strength);
-	else
-		ad9389b_wr(sd, 0xa2, 0x87);
-
-	if (pdata->diff_clk_drive_strength > 0)
-		ad9389b_wr(sd, 0xa3, pdata->diff_clk_drive_strength);
-	else
-		ad9389b_wr(sd, 0xa3, 0x87);
-
-	ad9389b_wr(sd, 0x0a, 0x01);
-	ad9389b_wr(sd, 0xbb, 0xff);
-
-	/* Set number of attempts to read the EDID */
-	ad9389b_wr(sd, 0xc9, 0xf);
-	return true;
-}
-
-/* Enable interrupts */
-static void ad9389b_set_isr(struct v4l2_subdev *sd, bool enable)
-{
-	u8 irqs = MASK_AD9389B_HPD_INT | MASK_AD9389B_MSEN_INT;
-	u8 irqs_rd;
-	int retries = 100;
-
-	/* The datasheet says that the EDID ready interrupt should be
-	   disabled if there is no hotplug. */
-	if (!enable)
-		irqs = 0;
-	else if (ad9389b_have_hotplug(sd))
-		irqs |= MASK_AD9389B_EDID_RDY_INT;
-
-	/*
-	 * This i2c write can fail (approx. 1 in 1000 writes). But it
-	 * is essential that this register is correct, so retry it
-	 * multiple times.
-	 *
-	 * Note that the i2c write does not report an error, but the readback
-	 * clearly shows the wrong value.
-	 */
-	do {
-		ad9389b_wr(sd, 0x94, irqs);
-		irqs_rd = ad9389b_rd(sd, 0x94);
-	} while (retries-- && irqs_rd != irqs);
-
-	if (irqs_rd != irqs)
-		v4l2_err(sd, "Could not set interrupts: hw failure?\n");
-}
-
-/* Interrupt handler */
-static int ad9389b_isr(struct v4l2_subdev *sd, u32 status, bool *handled)
-{
-	u8 irq_status;
-
-	/* disable interrupts to prevent a race condition */
-	ad9389b_set_isr(sd, false);
-	irq_status = ad9389b_rd(sd, 0x96);
-	/* clear detected interrupts */
-	ad9389b_wr(sd, 0x96, irq_status);
-	/* enable interrupts */
-	ad9389b_set_isr(sd, true);
-
-	v4l2_dbg(1, debug, sd, "%s: irq_status 0x%x\n", __func__, irq_status);
-
-	if (irq_status & (MASK_AD9389B_HPD_INT))
-		ad9389b_check_monitor_present_status(sd);
-	if (irq_status & MASK_AD9389B_EDID_RDY_INT)
-		ad9389b_check_edid_status(sd);
-
-	*handled = true;
-	return 0;
-}
-
-static const struct v4l2_subdev_core_ops ad9389b_core_ops = {
-	.log_status = ad9389b_log_status,
-#ifdef CONFIG_VIDEO_ADV_DEBUG
-	.g_register = ad9389b_g_register,
-	.s_register = ad9389b_s_register,
-#endif
-	.s_power = ad9389b_s_power,
-	.interrupt_service_routine = ad9389b_isr,
-};
-
-/* ------------------------------ VIDEO OPS ------------------------------ */
-
-/* Enable/disable ad9389b output */
-static int ad9389b_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	v4l2_dbg(1, debug, sd, "%s: %sable\n", __func__, (enable ? "en" : "dis"));
-
-	ad9389b_wr_and_or(sd, 0xa1, ~0x3c, (enable ? 0 : 0x3c));
-	if (enable) {
-		ad9389b_check_monitor_present_status(sd);
-	} else {
-		ad9389b_s_power(sd, 0);
-	}
-	return 0;
-}
-
-static const struct v4l2_dv_timings_cap ad9389b_timings_cap = {
-	.type = V4L2_DV_BT_656_1120,
-	/* keep this initialization for compatibility with GCC < 4.4.6 */
-	.reserved = { 0 },
-	V4L2_INIT_BT_TIMINGS(640, 1920, 350, 1200, 25000000, 170000000,
-		V4L2_DV_BT_STD_CEA861 | V4L2_DV_BT_STD_DMT |
-			V4L2_DV_BT_STD_GTF | V4L2_DV_BT_STD_CVT,
-		V4L2_DV_BT_CAP_PROGRESSIVE | V4L2_DV_BT_CAP_REDUCED_BLANKING |
-		V4L2_DV_BT_CAP_CUSTOM)
-};
-
-static int ad9389b_s_dv_timings(struct v4l2_subdev *sd,
-				struct v4l2_dv_timings *timings)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-
-	v4l2_dbg(1, debug, sd, "%s:\n", __func__);
-
-	/* quick sanity check */
-	if (!v4l2_valid_dv_timings(timings, &ad9389b_timings_cap, NULL, NULL))
-		return -EINVAL;
-
-	/* Fill the optional fields .standards and .flags in struct v4l2_dv_timings
-	   if the format is one of the CEA or DMT timings. */
-	v4l2_find_dv_timings_cap(timings, &ad9389b_timings_cap, 0, NULL, NULL);
-
-	timings->bt.flags &= ~V4L2_DV_FL_REDUCED_FPS;
-
-	/* save timings */
-	state->dv_timings = *timings;
-
-	/* update quantization range based on new dv_timings */
-	ad9389b_set_rgb_quantization_mode(sd, state->rgb_quantization_range_ctrl);
-
-	/* update PLL gear based on new dv_timings */
-	if (state->pdata.tmds_pll_gear == AD9389B_TMDS_PLL_GEAR_SEMI_AUTOMATIC)
-		ad9389b_set_manual_pll_gear(sd, (u32)timings->bt.pixelclock);
-
-	/* update AVI infoframe */
-	ad9389b_set_IT_content_AVI_InfoFrame(sd);
-
-	return 0;
-}
-
-static int ad9389b_g_dv_timings(struct v4l2_subdev *sd,
-				struct v4l2_dv_timings *timings)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-
-	v4l2_dbg(1, debug, sd, "%s:\n", __func__);
-
-	if (!timings)
-		return -EINVAL;
-
-	*timings = state->dv_timings;
-
-	return 0;
-}
-
-static int ad9389b_enum_dv_timings(struct v4l2_subdev *sd,
-				   struct v4l2_enum_dv_timings *timings)
-{
-	if (timings->pad != 0)
-		return -EINVAL;
-
-	return v4l2_enum_dv_timings_cap(timings, &ad9389b_timings_cap,
-			NULL, NULL);
-}
-
-static int ad9389b_dv_timings_cap(struct v4l2_subdev *sd,
-				  struct v4l2_dv_timings_cap *cap)
-{
-	if (cap->pad != 0)
-		return -EINVAL;
-
-	*cap = ad9389b_timings_cap;
-	return 0;
-}
-
-static const struct v4l2_subdev_video_ops ad9389b_video_ops = {
-	.s_stream = ad9389b_s_stream,
-	.s_dv_timings = ad9389b_s_dv_timings,
-	.g_dv_timings = ad9389b_g_dv_timings,
-};
-
-/* ------------------------------ PAD OPS ------------------------------ */
-
-static int ad9389b_get_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-
-	if (edid->pad != 0)
-		return -EINVAL;
-	if (edid->blocks == 0 || edid->blocks > 256)
-		return -EINVAL;
-	if (!state->edid.segments) {
-		v4l2_dbg(1, debug, sd, "EDID segment 0 not found\n");
-		return -ENODATA;
-	}
-	if (edid->start_block >= state->edid.segments * 2)
-		return -E2BIG;
-	if (edid->blocks + edid->start_block >= state->edid.segments * 2)
-		edid->blocks = state->edid.segments * 2 - edid->start_block;
-	memcpy(edid->edid, &state->edid.data[edid->start_block * 128],
-	       128 * edid->blocks);
-	return 0;
-}
-
-static const struct v4l2_subdev_pad_ops ad9389b_pad_ops = {
-	.get_edid = ad9389b_get_edid,
-	.enum_dv_timings = ad9389b_enum_dv_timings,
-	.dv_timings_cap = ad9389b_dv_timings_cap,
-};
-
-/* ------------------------------ AUDIO OPS ------------------------------ */
-
-static int ad9389b_s_audio_stream(struct v4l2_subdev *sd, int enable)
-{
-	v4l2_dbg(1, debug, sd, "%s: %sable\n", __func__, (enable ? "en" : "dis"));
-
-	if (enable)
-		ad9389b_wr_and_or(sd, 0x45, 0x3f, 0x80);
-	else
-		ad9389b_wr_and_or(sd, 0x45, 0x3f, 0x40);
-
-	return 0;
-}
-
-static int ad9389b_s_clock_freq(struct v4l2_subdev *sd, u32 freq)
-{
-	u32 N;
-
-	switch (freq) {
-	case 32000:  N = 4096;  break;
-	case 44100:  N = 6272;  break;
-	case 48000:  N = 6144;  break;
-	case 88200:  N = 12544; break;
-	case 96000:  N = 12288; break;
-	case 176400: N = 25088; break;
-	case 192000: N = 24576; break;
-	default:
-	     return -EINVAL;
-	}
-
-	/* Set N (used with CTS to regenerate the audio clock) */
-	ad9389b_wr(sd, 0x01, (N >> 16) & 0xf);
-	ad9389b_wr(sd, 0x02, (N >> 8) & 0xff);
-	ad9389b_wr(sd, 0x03, N & 0xff);
-
-	return 0;
-}
-
-static int ad9389b_s_i2s_clock_freq(struct v4l2_subdev *sd, u32 freq)
-{
-	u32 i2s_sf;
-
-	switch (freq) {
-	case 32000:  i2s_sf = 0x30; break;
-	case 44100:  i2s_sf = 0x00; break;
-	case 48000:  i2s_sf = 0x20; break;
-	case 88200:  i2s_sf = 0x80; break;
-	case 96000:  i2s_sf = 0xa0; break;
-	case 176400: i2s_sf = 0xc0; break;
-	case 192000: i2s_sf = 0xe0; break;
-	default:
-	     return -EINVAL;
-	}
-
-	/* Set sampling frequency for I2S audio to 48 kHz */
-	ad9389b_wr_and_or(sd, 0x15, 0xf, i2s_sf);
-
-	return 0;
-}
-
-static int ad9389b_s_routing(struct v4l2_subdev *sd, u32 input, u32 output, u32 config)
-{
-	/* TODO based on input/output/config */
-	/* TODO See datasheet "Programmers guide" p. 39-40 */
-
-	/* Only 2 channels in use for application */
-	ad9389b_wr_and_or(sd, 0x50, 0x1f, 0x20);
-	/* Speaker mapping */
-	ad9389b_wr(sd, 0x51, 0x00);
-
-	/* TODO Where should this be placed? */
-	/* 16 bit audio word length */
-	ad9389b_wr_and_or(sd, 0x14, 0xf0, 0x02);
-
-	return 0;
-}
-
-static const struct v4l2_subdev_audio_ops ad9389b_audio_ops = {
-	.s_stream = ad9389b_s_audio_stream,
-	.s_clock_freq = ad9389b_s_clock_freq,
-	.s_i2s_clock_freq = ad9389b_s_i2s_clock_freq,
-	.s_routing = ad9389b_s_routing,
-};
-
-/* --------------------- SUBDEV OPS --------------------------------------- */
-
-static const struct v4l2_subdev_ops ad9389b_ops = {
-	.core  = &ad9389b_core_ops,
-	.video = &ad9389b_video_ops,
-	.audio = &ad9389b_audio_ops,
-	.pad = &ad9389b_pad_ops,
-};
-
-/* ----------------------------------------------------------------------- */
-static void ad9389b_dbg_dump_edid(int lvl, int debug, struct v4l2_subdev *sd,
-				  int segment, u8 *buf)
-{
-	int i, j;
-
-	if (debug < lvl)
-		return;
-
-	v4l2_dbg(lvl, debug, sd, "edid segment %d\n", segment);
-	for (i = 0; i < 256; i += 16) {
-		u8 b[128];
-		u8 *bp = b;
-
-		if (i == 128)
-			v4l2_dbg(lvl, debug, sd, "\n");
-		for (j = i; j < i + 16; j++) {
-			sprintf(bp, "0x%02x, ", buf[j]);
-			bp += 6;
-		}
-		bp[0] = '\0';
-		v4l2_dbg(lvl, debug, sd, "%s\n", b);
-	}
-}
-
-static void ad9389b_edid_handler(struct work_struct *work)
-{
-	struct delayed_work *dwork = to_delayed_work(work);
-	struct ad9389b_state *state =
-		container_of(dwork, struct ad9389b_state, edid_handler);
-	struct v4l2_subdev *sd = &state->sd;
-	struct ad9389b_edid_detect ed;
-
-	v4l2_dbg(1, debug, sd, "%s:\n", __func__);
-
-	if (ad9389b_check_edid_status(sd)) {
-		/* Return if we received the EDID. */
-		return;
-	}
-
-	if (ad9389b_have_hotplug(sd)) {
-		/* We must retry reading the EDID several times, it is possible
-		 * that initially the EDID couldn't be read due to i2c errors
-		 * (DVI connectors are particularly prone to this problem). */
-		if (state->edid.read_retries) {
-			state->edid.read_retries--;
-			v4l2_dbg(1, debug, sd, "%s: edid read failed\n", __func__);
-			ad9389b_s_power(sd, false);
-			ad9389b_s_power(sd, true);
-			schedule_delayed_work(&state->edid_handler, EDID_DELAY);
-			return;
-		}
-	}
-
-	/* We failed to read the EDID, so send an event for this. */
-	ed.present = false;
-	ed.segment = ad9389b_rd(sd, 0xc4);
-	v4l2_subdev_notify(sd, AD9389B_EDID_DETECT, (void *)&ed);
-	v4l2_dbg(1, debug, sd, "%s: no edid found\n", __func__);
-}
-
-static void ad9389b_audio_setup(struct v4l2_subdev *sd)
-{
-	v4l2_dbg(1, debug, sd, "%s\n", __func__);
-
-	ad9389b_s_i2s_clock_freq(sd, 48000);
-	ad9389b_s_clock_freq(sd, 48000);
-	ad9389b_s_routing(sd, 0, 0, 0);
-}
-
-/* Initial setup of AD9389b */
-
-/* Configure hdmi transmitter. */
-static void ad9389b_setup(struct v4l2_subdev *sd)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-
-	v4l2_dbg(1, debug, sd, "%s\n", __func__);
-
-	/* Input format: RGB 4:4:4 */
-	ad9389b_wr_and_or(sd, 0x15, 0xf1, 0x0);
-	/* Output format: RGB 4:4:4 */
-	ad9389b_wr_and_or(sd, 0x16, 0x3f, 0x0);
-	/* 1st order interpolation 4:2:2 -> 4:4:4 up conversion,
-	   Aspect ratio: 16:9 */
-	ad9389b_wr_and_or(sd, 0x17, 0xf9, 0x06);
-	/* Output format: RGB 4:4:4, Active Format Information is valid. */
-	ad9389b_wr_and_or(sd, 0x45, 0xc7, 0x08);
-	/* Underscanned */
-	ad9389b_wr_and_or(sd, 0x46, 0x3f, 0x80);
-	/* Setup video format */
-	ad9389b_wr(sd, 0x3c, 0x0);
-	/* Active format aspect ratio: same as picure. */
-	ad9389b_wr(sd, 0x47, 0x80);
-	/* No encryption */
-	ad9389b_wr_and_or(sd, 0xaf, 0xef, 0x0);
-	/* Positive clk edge capture for input video clock */
-	ad9389b_wr_and_or(sd, 0xba, 0x1f, 0x60);
-
-	ad9389b_audio_setup(sd);
-
-	v4l2_ctrl_handler_setup(&state->hdl);
-
-	ad9389b_set_IT_content_AVI_InfoFrame(sd);
-}
-
-static void ad9389b_notify_monitor_detect(struct v4l2_subdev *sd)
-{
-	struct ad9389b_monitor_detect mdt;
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-
-	mdt.present = state->have_monitor;
-	v4l2_subdev_notify(sd, AD9389B_MONITOR_DETECT, (void *)&mdt);
-}
-
-static void ad9389b_update_monitor_present_status(struct v4l2_subdev *sd)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-	/* read hotplug and rx-sense state */
-	u8 status = ad9389b_rd(sd, 0x42);
-
-	v4l2_dbg(1, debug, sd, "%s: status: 0x%x%s%s\n",
-		 __func__,
-		 status,
-		 status & MASK_AD9389B_HPD_DETECT ? ", hotplug" : "",
-		 status & MASK_AD9389B_MSEN_DETECT ? ", rx-sense" : "");
-
-	if (status & MASK_AD9389B_HPD_DETECT) {
-		v4l2_dbg(1, debug, sd, "%s: hotplug detected\n", __func__);
-		state->have_monitor = true;
-		if (!ad9389b_s_power(sd, true)) {
-			v4l2_dbg(1, debug, sd,
-				 "%s: monitor detected, powerup failed\n", __func__);
-			return;
-		}
-		ad9389b_setup(sd);
-		ad9389b_notify_monitor_detect(sd);
-		state->edid.read_retries = EDID_MAX_RETRIES;
-		schedule_delayed_work(&state->edid_handler, EDID_DELAY);
-	} else if (!(status & MASK_AD9389B_HPD_DETECT)) {
-		v4l2_dbg(1, debug, sd, "%s: hotplug not detected\n", __func__);
-		state->have_monitor = false;
-		ad9389b_notify_monitor_detect(sd);
-		ad9389b_s_power(sd, false);
-		memset(&state->edid, 0, sizeof(struct ad9389b_state_edid));
-	}
-
-	/* update read only ctrls */
-	v4l2_ctrl_s_ctrl(state->hotplug_ctrl, ad9389b_have_hotplug(sd) ? 0x1 : 0x0);
-	v4l2_ctrl_s_ctrl(state->rx_sense_ctrl, ad9389b_have_rx_sense(sd) ? 0x1 : 0x0);
-	v4l2_ctrl_s_ctrl(state->have_edid0_ctrl, state->edid.segments ? 0x1 : 0x0);
-
-	/* update with setting from ctrls */
-	ad9389b_s_ctrl(state->rgb_quantization_range_ctrl);
-	ad9389b_s_ctrl(state->hdmi_mode_ctrl);
-}
-
-static void ad9389b_check_monitor_present_status(struct v4l2_subdev *sd)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-	int retry = 0;
-
-	ad9389b_update_monitor_present_status(sd);
-
-	/*
-	 * Rapid toggling of the hotplug may leave the chip powered off,
-	 * even if we think it is on. In that case reset and power up again.
-	 */
-	while (state->power_on && (ad9389b_rd(sd, 0x41) & 0x40)) {
-		if (++retry > 5) {
-			v4l2_err(sd, "retried %d times, give up\n", retry);
-			return;
-		}
-		v4l2_dbg(1, debug, sd, "%s: reset and re-check status (%d)\n", __func__, retry);
-		ad9389b_notify_monitor_detect(sd);
-		cancel_delayed_work_sync(&state->edid_handler);
-		memset(&state->edid, 0, sizeof(struct ad9389b_state_edid));
-		ad9389b_s_power(sd, false);
-		ad9389b_update_monitor_present_status(sd);
-	}
-}
-
-static bool edid_block_verify_crc(u8 *edid_block)
-{
-	u8 sum = 0;
-	int i;
-
-	for (i = 0; i < 128; i++)
-		sum += edid_block[i];
-	return sum == 0;
-}
-
-static bool edid_verify_crc(struct v4l2_subdev *sd, u32 segment)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-	u32 blocks = state->edid.blocks;
-	u8 *data = state->edid.data;
-
-	if (edid_block_verify_crc(&data[segment * 256])) {
-		if ((segment + 1) * 2 <= blocks)
-			return edid_block_verify_crc(&data[segment * 256 + 128]);
-		return true;
-	}
-	return false;
-}
-
-static bool edid_verify_header(struct v4l2_subdev *sd, u32 segment)
-{
-	static const u8 hdmi_header[] = {
-		0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00
-	};
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-	u8 *data = state->edid.data;
-	int i;
-
-	if (segment)
-		return true;
-
-	for (i = 0; i < ARRAY_SIZE(hdmi_header); i++)
-		if (data[i] != hdmi_header[i])
-			return false;
-
-	return true;
-}
-
-static bool ad9389b_check_edid_status(struct v4l2_subdev *sd)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-	struct ad9389b_edid_detect ed;
-	int segment;
-	u8 edidRdy = ad9389b_rd(sd, 0xc5);
-
-	v4l2_dbg(1, debug, sd, "%s: edid ready (retries: %d)\n",
-		 __func__, EDID_MAX_RETRIES - state->edid.read_retries);
-
-	if (!(edidRdy & MASK_AD9389B_EDID_RDY))
-		return false;
-
-	segment = ad9389b_rd(sd, 0xc4);
-	if (segment >= EDID_MAX_SEGM) {
-		v4l2_err(sd, "edid segment number too big\n");
-		return false;
-	}
-	v4l2_dbg(1, debug, sd, "%s: got segment %d\n", __func__, segment);
-	ad9389b_edid_rd(sd, 256, &state->edid.data[segment * 256]);
-	ad9389b_dbg_dump_edid(2, debug, sd, segment,
-			      &state->edid.data[segment * 256]);
-	if (segment == 0) {
-		state->edid.blocks = state->edid.data[0x7e] + 1;
-		v4l2_dbg(1, debug, sd, "%s: %d blocks in total\n",
-			 __func__, state->edid.blocks);
-	}
-	if (!edid_verify_crc(sd, segment) ||
-	    !edid_verify_header(sd, segment)) {
-		/* edid crc error, force reread of edid segment */
-		v4l2_err(sd, "%s: edid crc or header error\n", __func__);
-		ad9389b_s_power(sd, false);
-		ad9389b_s_power(sd, true);
-		return false;
-	}
-	/* one more segment read ok */
-	state->edid.segments = segment + 1;
-	if (((state->edid.data[0x7e] >> 1) + 1) > state->edid.segments) {
-		/* Request next EDID segment */
-		v4l2_dbg(1, debug, sd, "%s: request segment %d\n",
-			 __func__, state->edid.segments);
-		ad9389b_wr(sd, 0xc9, 0xf);
-		ad9389b_wr(sd, 0xc4, state->edid.segments);
-		state->edid.read_retries = EDID_MAX_RETRIES;
-		schedule_delayed_work(&state->edid_handler, EDID_DELAY);
-		return false;
-	}
-
-	/* report when we have all segments but report only for segment 0 */
-	ed.present = true;
-	ed.segment = 0;
-	v4l2_subdev_notify(sd, AD9389B_EDID_DETECT, (void *)&ed);
-	state->edid_detect_counter++;
-	v4l2_ctrl_s_ctrl(state->have_edid0_ctrl, state->edid.segments ? 0x1 : 0x0);
-	return ed.present;
-}
-
-/* ----------------------------------------------------------------------- */
-
-static void ad9389b_init_setup(struct v4l2_subdev *sd)
-{
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-	struct ad9389b_state_edid *edid = &state->edid;
-
-	v4l2_dbg(1, debug, sd, "%s\n", __func__);
-
-	/* clear all interrupts */
-	ad9389b_wr(sd, 0x96, 0xff);
-
-	memset(edid, 0, sizeof(struct ad9389b_state_edid));
-	state->have_monitor = false;
-	ad9389b_set_isr(sd, false);
-}
-
-static int ad9389b_probe(struct i2c_client *client)
-{
-	const struct v4l2_dv_timings dv1080p60 = V4L2_DV_BT_CEA_1920X1080P60;
-	struct ad9389b_state *state;
-	struct ad9389b_platform_data *pdata = client->dev.platform_data;
-	struct v4l2_ctrl_handler *hdl;
-	struct v4l2_subdev *sd;
-	int err = -EIO;
-
-	/* Check if the adapter supports the needed features */
-	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA))
-		return -EIO;
-
-	v4l_dbg(1, debug, client, "detecting ad9389b client on address 0x%x\n",
-		client->addr << 1);
-
-	state = devm_kzalloc(&client->dev, sizeof(*state), GFP_KERNEL);
-	if (!state)
-		return -ENOMEM;
-
-	/* Platform data */
-	if (pdata == NULL) {
-		v4l_err(client, "No platform data!\n");
-		return -ENODEV;
-	}
-	memcpy(&state->pdata, pdata, sizeof(state->pdata));
-
-	sd = &state->sd;
-	v4l2_i2c_subdev_init(sd, client, &ad9389b_ops);
-	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-
-	hdl = &state->hdl;
-	v4l2_ctrl_handler_init(hdl, 5);
-
-	state->hdmi_mode_ctrl = v4l2_ctrl_new_std_menu(hdl, &ad9389b_ctrl_ops,
-			V4L2_CID_DV_TX_MODE, V4L2_DV_TX_MODE_HDMI,
-			0, V4L2_DV_TX_MODE_DVI_D);
-	state->hotplug_ctrl = v4l2_ctrl_new_std(hdl, NULL,
-			V4L2_CID_DV_TX_HOTPLUG, 0, 1, 0, 0);
-	state->rx_sense_ctrl = v4l2_ctrl_new_std(hdl, NULL,
-			V4L2_CID_DV_TX_RXSENSE, 0, 1, 0, 0);
-	state->have_edid0_ctrl = v4l2_ctrl_new_std(hdl, NULL,
-			V4L2_CID_DV_TX_EDID_PRESENT, 0, 1, 0, 0);
-	state->rgb_quantization_range_ctrl =
-		v4l2_ctrl_new_std_menu(hdl, &ad9389b_ctrl_ops,
-			V4L2_CID_DV_TX_RGB_RANGE, V4L2_DV_RGB_RANGE_FULL,
-			0, V4L2_DV_RGB_RANGE_AUTO);
-	sd->ctrl_handler = hdl;
-	if (hdl->error) {
-		err = hdl->error;
-
-		goto err_hdl;
-	}
-	state->pad.flags = MEDIA_PAD_FL_SINK;
-	sd->entity.function = MEDIA_ENT_F_DV_ENCODER;
-	err = media_entity_pads_init(&sd->entity, 1, &state->pad);
-	if (err)
-		goto err_hdl;
-
-	state->chip_revision = ad9389b_rd(sd, 0x0);
-	if (state->chip_revision != 2) {
-		v4l2_err(sd, "chip_revision %d != 2\n", state->chip_revision);
-		err = -EIO;
-		goto err_entity;
-	}
-	v4l2_dbg(1, debug, sd, "reg 0x41 0x%x, chip version (reg 0x00) 0x%x\n",
-		 ad9389b_rd(sd, 0x41), state->chip_revision);
-
-	state->edid_i2c_client = i2c_new_dummy_device(client->adapter, (0x7e >> 1));
-	if (IS_ERR(state->edid_i2c_client)) {
-		v4l2_err(sd, "failed to register edid i2c client\n");
-		err = PTR_ERR(state->edid_i2c_client);
-		goto err_entity;
-	}
-
-	INIT_DELAYED_WORK(&state->edid_handler, ad9389b_edid_handler);
-	state->dv_timings = dv1080p60;
-
-	ad9389b_init_setup(sd);
-	ad9389b_set_isr(sd, true);
-
-	v4l2_info(sd, "%s found @ 0x%x (%s)\n", client->name,
-		  client->addr << 1, client->adapter->name);
-	return 0;
-
-err_entity:
-	media_entity_cleanup(&sd->entity);
-err_hdl:
-	v4l2_ctrl_handler_free(&state->hdl);
-	return err;
-}
-
-/* ----------------------------------------------------------------------- */
-
-static void ad9389b_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct ad9389b_state *state = get_ad9389b_state(sd);
-
-	state->chip_revision = -1;
-
-	v4l2_dbg(1, debug, sd, "%s removed @ 0x%x (%s)\n", client->name,
-		 client->addr << 1, client->adapter->name);
-
-	ad9389b_s_stream(sd, false);
-	ad9389b_s_audio_stream(sd, false);
-	ad9389b_init_setup(sd);
-	cancel_delayed_work_sync(&state->edid_handler);
-	i2c_unregister_device(state->edid_i2c_client);
-	v4l2_device_unregister_subdev(sd);
-	media_entity_cleanup(&sd->entity);
-	v4l2_ctrl_handler_free(sd->ctrl_handler);
-}
-
-/* ----------------------------------------------------------------------- */
-
-static const struct i2c_device_id ad9389b_id[] = {
-	{ "ad9389b", 0 },
-	{ "ad9889b", 0 },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, ad9389b_id);
-
-static struct i2c_driver ad9389b_driver = {
-	.driver = {
-		.name = "ad9389b",
-	},
-	.probe_new = ad9389b_probe,
-	.remove = ad9389b_remove,
-	.id_table = ad9389b_id,
-};
-
-module_i2c_driver(ad9389b_driver);
diff --git a/drivers/media/i2c/adv748x/adv748x-hdmi.c b/drivers/media/i2c/adv748x/adv748x-hdmi.c
index 52fa7bd75660..400d71c2745c 100644
--- a/drivers/media/i2c/adv748x/adv748x-hdmi.c
+++ b/drivers/media/i2c/adv748x/adv748x-hdmi.c
@@ -176,9 +176,9 @@ static int adv748x_hdmi_set_video_timings(struct adv748x_state *state,
 	unsigned int i;
 
 	for (i = 0; i < ARRAY_SIZE(adv748x_hdmi_video_standards); i++) {
-		if (!v4l2_match_dv_timings(timings, &stds[i].timings, 250000,
-					   false))
-			continue;
+		if (v4l2_match_dv_timings(timings, &stds[i].timings, 250000,
+					  false))
+			break;
 	}
 
 	if (i >= ARRAY_SIZE(adv748x_hdmi_video_standards))
@@ -283,6 +283,16 @@ static int adv748x_hdmi_query_dv_timings(struct v4l2_subdev *sd,
 
 	memset(timings, 0, sizeof(struct v4l2_dv_timings));
 
+	/*
+	 * If the pattern generator is enabled the device shall not be queried
+	 * for timings. Instead the timings programmed shall be reported as they
+	 * are the ones being used to generate the pattern.
+	 */
+	if (cp_read(state, ADV748X_CP_PAT_GEN) & ADV748X_CP_PAT_GEN_EN) {
+		*timings = hdmi->timings;
+		return 0;
+	}
+
 	if (!adv748x_hdmi_has_signal(state))
 		return -ENOLINK;
 
@@ -721,11 +731,10 @@ static int adv748x_hdmi_init_controls(struct adv748x_hdmi *hdmi)
 int adv748x_hdmi_init(struct adv748x_hdmi *hdmi)
 {
 	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
-	static const struct v4l2_dv_timings cea1280x720 =
-		V4L2_DV_BT_CEA_1280X720P30;
+	struct v4l2_dv_timings cea1280x720 = V4L2_DV_BT_CEA_1280X720P30;
 	int ret;
 
-	hdmi->timings = cea1280x720;
+	adv748x_hdmi_s_dv_timings(&hdmi->sd, &cea1280x720);
 
 	/* Initialise a default 16:9 aspect ratio */
 	hdmi->aspect_ratio.numerator = 16;
diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c
index 9d218962d7c8..3d0898c4175e 100644
--- a/drivers/media/i2c/adv7604.c
+++ b/drivers/media/i2c/adv7604.c
@@ -1805,6 +1805,9 @@ static void select_input(struct v4l2_subdev *sd)
 		v4l2_dbg(2, debug, sd, "%s: Unknown port %d selected\n",
 				__func__, state->selected_input);
 	}
+
+	/* Enable video adjustment (contrast, saturation, brightness and hue) */
+	cp_write_clr_set(sd, 0x3e, 0x80, 0x80);
 }
 
 static int adv76xx_s_routing(struct v4l2_subdev *sd,
@@ -3545,7 +3548,7 @@ static int adv76xx_probe(struct i2c_client *client)
 	v4l2_ctrl_new_std(hdl, &adv76xx_ctrl_ops,
 			V4L2_CID_SATURATION, 0, 255, 1, 128);
 	v4l2_ctrl_new_std(hdl, &adv76xx_ctrl_ops,
-			V4L2_CID_HUE, 0, 128, 1, 0);
+			V4L2_CID_HUE, 0, 255, 1, 0);
 	ctrl = v4l2_ctrl_new_std_menu(hdl, &adv76xx_ctrl_ops,
 			V4L2_CID_DV_RX_IT_CONTENT_TYPE, V4L2_DV_IT_CONTENT_TYPE_NO_ITC,
 			0, V4L2_DV_IT_CONTENT_TYPE_NO_ITC);
diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c
index 4a14d7e5d9f2..559a415fd827 100644
--- a/drivers/media/i2c/ccs/ccs-core.c
+++ b/drivers/media/i2c/ccs/ccs-core.c
@@ -569,8 +569,6 @@ static u32 ccs_pixel_order(struct ccs_sensor *sensor)
 			flip |= CCS_IMAGE_ORIENTATION_VERTICAL_FLIP;
 	}
 
-	flip ^= sensor->hvflip_inv_mask;
-
 	dev_dbg(&client->dev, "flip %u\n", flip);
 	return sensor->default_pixel_order ^ flip;
 }
@@ -632,8 +630,6 @@ static int ccs_set_ctrl(struct v4l2_ctrl *ctrl)
 		if (sensor->vflip->val)
 			orient |= CCS_IMAGE_ORIENTATION_VERTICAL_FLIP;
 
-		orient ^= sensor->hvflip_inv_mask;
-
 		ccs_update_mbus_formats(sensor);
 
 		break;
@@ -800,14 +796,24 @@ static const struct v4l2_ctrl_ops ccs_ctrl_ops = {
 static int ccs_init_controls(struct ccs_sensor *sensor)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(&sensor->src->sd);
+	struct v4l2_fwnode_device_properties props;
 	int rval;
 
-	rval = v4l2_ctrl_handler_init(&sensor->pixel_array->ctrl_handler, 17);
+	rval = v4l2_ctrl_handler_init(&sensor->pixel_array->ctrl_handler, 19);
 	if (rval)
 		return rval;
 
 	sensor->pixel_array->ctrl_handler.lock = &sensor->mutex;
 
+	rval = v4l2_fwnode_device_parse(&client->dev, &props);
+	if (rval)
+		return rval;
+
+	rval = v4l2_ctrl_new_fwnode_properties(&sensor->pixel_array->ctrl_handler,
+					       &ccs_ctrl_ops, &props);
+	if (rval)
+		return rval;
+
 	switch (CCS_LIM(sensor, ANALOG_GAIN_CAPABILITY)) {
 	case CCS_ANALOG_GAIN_CAPABILITY_GLOBAL: {
 		struct {
@@ -2828,6 +2834,10 @@ static int ccs_identify_module(struct ccs_sensor *sensor)
 		rval = ccs_read_addr_8only(sensor,
 					   CCS_R_SENSOR_REVISION_NUMBER,
 					   &minfo->sensor_revision_number);
+	if (!rval && !minfo->sensor_revision_number)
+		rval = ccs_read_addr_8only(sensor,
+					   CCS_R_SENSOR_REVISION_NUMBER_16,
+					   &minfo->sensor_revision_number);
 	if (!rval)
 		rval = ccs_read_addr_8only(sensor,
 					   CCS_R_SENSOR_FIRMWARE_VERSION,
@@ -2870,7 +2880,7 @@ static int ccs_identify_module(struct ccs_sensor *sensor)
 			minfo->sensor_model_id);
 
 	dev_dbg(&client->dev,
-		"sensor revision 0x%2.2x firmware version 0x%2.2x\n",
+		"sensor revision 0x%4.4x firmware version 0x%2.2x\n",
 		minfo->sensor_revision_number, minfo->sensor_firmware_version);
 
 	if (minfo->ccs_version) {
@@ -2884,19 +2894,18 @@ static int ccs_identify_module(struct ccs_sensor *sensor)
 			"smia version %2.2d smiapp version %2.2d\n",
 			minfo->smia_version, minfo->smiapp_version);
 		minfo->name = SMIAPP_NAME;
-	}
-
-	/*
-	 * Some modules have bad data in the lvalues below. Hope the
-	 * rvalues have better stuff. The lvalues are module
-	 * parameters whereas the rvalues are sensor parameters.
-	 */
-	if (minfo->sensor_smia_manufacturer_id &&
-	    !minfo->smia_manufacturer_id && !minfo->model_id) {
-		minfo->smia_manufacturer_id =
-			minfo->sensor_smia_manufacturer_id;
-		minfo->model_id = minfo->sensor_model_id;
-		minfo->revision_number = minfo->sensor_revision_number;
+		/*
+		 * Some modules have bad data in the lvalues below. Hope the
+		 * rvalues have better stuff. The lvalues are module
+		 * parameters whereas the rvalues are sensor parameters.
+		 */
+		if (minfo->sensor_smia_manufacturer_id &&
+		    !minfo->smia_manufacturer_id && !minfo->model_id) {
+			minfo->smia_manufacturer_id =
+				minfo->sensor_smia_manufacturer_id;
+			minfo->model_id = minfo->sensor_model_id;
+			minfo->revision_number = minfo->sensor_revision_number;
+		}
 	}
 
 	for (i = 0; i < ARRAY_SIZE(ccs_module_idents); i++) {
@@ -3185,7 +3194,6 @@ static int ccs_get_hwconfig(struct ccs_sensor *sensor, struct device *dev)
 	struct v4l2_fwnode_endpoint bus_cfg = { .bus_type = V4L2_MBUS_UNKNOWN };
 	struct fwnode_handle *ep;
 	struct fwnode_handle *fwnode = dev_fwnode(dev);
-	u32 rotation;
 	unsigned int i;
 	int rval;
 
@@ -3224,22 +3232,6 @@ static int ccs_get_hwconfig(struct ccs_sensor *sensor, struct device *dev)
 		goto out_err;
 	}
 
-	rval = fwnode_property_read_u32(fwnode, "rotation", &rotation);
-	if (!rval) {
-		switch (rotation) {
-		case 180:
-			hwcfg->module_board_orient =
-				CCS_MODULE_BOARD_ORIENT_180;
-			fallthrough;
-		case 0:
-			break;
-		default:
-			dev_err(dev, "invalid rotation %u\n", rotation);
-			rval = -EINVAL;
-			goto out_err;
-		}
-	}
-
 	rval = fwnode_property_read_u32(dev_fwnode(dev), "clock-frequency",
 					&hwcfg->ext_clk);
 	if (rval)
@@ -3279,8 +3271,47 @@ out_err:
 	return rval;
 }
 
+static int ccs_firmware_name(struct i2c_client *client,
+			     struct ccs_sensor *sensor, char *filename,
+			     size_t filename_size, bool is_module)
+{
+	const struct ccs_device *ccsdev = device_get_match_data(&client->dev);
+	bool is_ccs = !(ccsdev->flags & CCS_DEVICE_FLAG_IS_SMIA);
+	bool is_smiapp = sensor->minfo.smiapp_version;
+	u16 manufacturer_id;
+	u16 model_id;
+	u16 revision_number;
+
+	/*
+	 * Old SMIA is module-agnostic. Its sensor identification is based on
+	 * what now are those of the module.
+	 */
+	if (is_module || (!is_ccs && !is_smiapp)) {
+		manufacturer_id = is_ccs ?
+			sensor->minfo.mipi_manufacturer_id :
+			sensor->minfo.smia_manufacturer_id;
+		model_id = sensor->minfo.model_id;
+		revision_number = sensor->minfo.revision_number;
+	} else {
+		manufacturer_id = is_ccs ?
+			sensor->minfo.sensor_mipi_manufacturer_id :
+			sensor->minfo.sensor_smia_manufacturer_id;
+		model_id = sensor->minfo.sensor_model_id;
+		revision_number = sensor->minfo.sensor_revision_number;
+	}
+
+	return snprintf(filename, filename_size,
+			"ccs/%s-%s-%0*x-%4.4x-%0*x.fw",
+			is_ccs ? "ccs" : is_smiapp ? "smiapp" : "smia",
+			is_module || (!is_ccs && !is_smiapp) ?
+				"module" : "sensor",
+			is_ccs ? 4 : 2, manufacturer_id, model_id,
+			!is_ccs && !is_module ? 2 : 4, revision_number);
+}
+
 static int ccs_probe(struct i2c_client *client)
 {
+	const struct ccs_device *ccsdev = device_get_match_data(&client->dev);
 	struct ccs_sensor *sensor;
 	const struct firmware *fw;
 	char filename[40];
@@ -3389,11 +3420,8 @@ static int ccs_probe(struct i2c_client *client)
 		goto out_power_off;
 	}
 
-	rval = snprintf(filename, sizeof(filename),
-			"ccs/ccs-sensor-%4.4x-%4.4x-%4.4x.fw",
-			sensor->minfo.sensor_mipi_manufacturer_id,
-			sensor->minfo.sensor_model_id,
-			sensor->minfo.sensor_revision_number);
+	rval = ccs_firmware_name(client, sensor, filename, sizeof(filename),
+				 false);
 	if (rval >= sizeof(filename)) {
 		rval = -ENOMEM;
 		goto out_power_off;
@@ -3406,21 +3434,21 @@ static int ccs_probe(struct i2c_client *client)
 		release_firmware(fw);
 	}
 
-	rval = snprintf(filename, sizeof(filename),
-			"ccs/ccs-module-%4.4x-%4.4x-%4.4x.fw",
-			sensor->minfo.mipi_manufacturer_id,
-			sensor->minfo.model_id,
-			sensor->minfo.revision_number);
-	if (rval >= sizeof(filename)) {
-		rval = -ENOMEM;
-		goto out_release_sdata;
-	}
+	if (!(ccsdev->flags & CCS_DEVICE_FLAG_IS_SMIA) ||
+	    sensor->minfo.smiapp_version) {
+		rval = ccs_firmware_name(client, sensor, filename,
+					 sizeof(filename), true);
+		if (rval >= sizeof(filename)) {
+			rval = -ENOMEM;
+			goto out_release_sdata;
+		}
 
-	rval = request_firmware(&fw, filename, &client->dev);
-	if (!rval) {
-		ccs_data_parse(&sensor->mdata, fw->data, fw->size, &client->dev,
-			       true);
-		release_firmware(fw);
+		rval = request_firmware(&fw, filename, &client->dev);
+		if (!rval) {
+			ccs_data_parse(&sensor->mdata, fw->data, fw->size,
+				       &client->dev, true);
+			release_firmware(fw);
+		}
 	}
 
 	rval = ccs_read_all_limits(sensor);
@@ -3437,25 +3465,6 @@ static int ccs_probe(struct i2c_client *client)
 	if (rval < 0)
 		goto out_free_ccs_limits;
 
-	/*
-	 * Handle Sensor Module orientation on the board.
-	 *
-	 * The application of H-FLIP and V-FLIP on the sensor is modified by
-	 * the sensor orientation on the board.
-	 *
-	 * For CCS_BOARD_SENSOR_ORIENT_180 the default behaviour is to set
-	 * both H-FLIP and V-FLIP for normal operation which also implies
-	 * that a set/unset operation for user space HFLIP and VFLIP v4l2
-	 * controls will need to be internally inverted.
-	 *
-	 * Rotation also changes the bayer pattern.
-	 */
-	if (sensor->hwcfg.module_board_orient ==
-	    CCS_MODULE_BOARD_ORIENT_180)
-		sensor->hvflip_inv_mask =
-			CCS_IMAGE_ORIENTATION_HORIZONTAL_MIRROR |
-			CCS_IMAGE_ORIENTATION_VERTICAL_FLIP;
-
 	rval = ccs_call_quirk(sensor, limits);
 	if (rval) {
 		dev_err(&client->dev, "limits quirks failed\n");
diff --git a/drivers/media/i2c/ccs/ccs.h b/drivers/media/i2c/ccs/ccs.h
index 6beac375cc48..a94c796cea48 100644
--- a/drivers/media/i2c/ccs/ccs.h
+++ b/drivers/media/i2c/ccs/ccs.h
@@ -57,17 +57,6 @@
 #define CCS_LIM_AT(sensor, limit, offset)	\
 	ccs_get_limit(sensor, CCS_L_##limit, CCS_L_##limit##_OFFSET(offset))
 
-/*
- * Sometimes due to board layout considerations the camera module can be
- * mounted rotated. The typical rotation used is 180 degrees which can be
- * corrected by giving a default H-FLIP and V-FLIP in the sensor readout.
- * FIXME: rotation also changes the bayer pattern.
- */
-enum ccs_module_board_orient {
-	CCS_MODULE_BOARD_ORIENT_0 = 0,
-	CCS_MODULE_BOARD_ORIENT_180,
-};
-
 struct ccs_flash_strobe_parms {
 	u8 mode;
 	u32 strobe_width_high_us;
@@ -90,8 +79,6 @@ struct ccs_hwconfig {
 	u32 csi_signalling_mode;	/* CCS_CSI_SIGNALLING_MODE_* */
 	u64 *op_sys_clock;
 
-	enum ccs_module_board_orient module_board_orient;
-
 	struct ccs_flash_strobe_parms *strobe_setup;
 };
 
@@ -243,7 +230,6 @@ struct ccs_sensor {
 	u8 scale_m;
 	u8 scaling_mode;
 
-	u8 hvflip_inv_mask; /* H/VFLIP inversion due to sensor orientation */
 	u8 frame_skip;
 	u16 embedded_start; /* embedded data start line */
 	u16 embedded_end;
diff --git a/drivers/media/i2c/hi556.c b/drivers/media/i2c/hi556.c
index e422ac7609b5..7daefab35cf0 100644
--- a/drivers/media/i2c/hi556.c
+++ b/drivers/media/i2c/hi556.c
@@ -63,6 +63,14 @@
 #define HI556_REG_ISP_TPG_EN		0x01
 #define HI556_REG_TEST_PATTERN		0x0201
 
+/* HI556 native and active pixel array size. */
+#define HI556_NATIVE_WIDTH		2592U
+#define HI556_NATIVE_HEIGHT		1944U
+#define HI556_PIXEL_ARRAY_LEFT		0U
+#define HI556_PIXEL_ARRAY_TOP		0U
+#define HI556_PIXEL_ARRAY_WIDTH	2592U
+#define HI556_PIXEL_ARRAY_HEIGHT	1944U
+
 enum {
 	HI556_LINK_FREQ_437MHZ_INDEX,
 };
@@ -88,6 +96,9 @@ struct hi556_mode {
 	/* Frame height in pixels */
 	u32 height;
 
+	/* Analog crop rectangle. */
+	struct v4l2_rect crop;
+
 	/* Horizontal timining size */
 	u32 llp;
 
@@ -378,6 +389,49 @@ static const struct hi556_reg mode_2592x1944_regs[] = {
 	{0x0958, 0xbb80},
 };
 
+static const struct hi556_reg mode_2592x1444_regs[] = {
+	{0x0a00, 0x0000},
+	{0x0b0a, 0x8252},
+	{0x0f30, 0xe545},
+	{0x0f32, 0x7067},
+	{0x004a, 0x0100},
+	{0x004c, 0x0000},
+	{0x000c, 0x0022},
+	{0x0008, 0x0b00},
+	{0x005a, 0x0202},
+	{0x0012, 0x000e},
+	{0x0018, 0x0a33},
+	{0x0022, 0x0008},
+	{0x0028, 0x0017},
+	{0x0024, 0x0122},
+	{0x002a, 0x0127},
+	{0x0026, 0x012a},
+	{0x002c, 0x06cf},
+	{0x002e, 0x1111},
+	{0x0030, 0x1111},
+	{0x0032, 0x1111},
+	{0x0006, 0x0821},
+	{0x0a22, 0x0000},
+	{0x0a12, 0x0a20},
+	{0x0a14, 0x05a4},
+	{0x003e, 0x0000},
+	{0x0074, 0x081f},
+	{0x0070, 0x040f},
+	{0x0804, 0x0300},
+	{0x0806, 0x0100},
+	{0x0a04, 0x014a},
+	{0x090c, 0x0fdc},
+	{0x090e, 0x002d},
+	{0x0902, 0x4319},
+	{0x0914, 0xc10a},
+	{0x0916, 0x071f},
+	{0x0918, 0x0408},
+	{0x091a, 0x0c0d},
+	{0x091c, 0x0f09},
+	{0x091e, 0x0a00},
+	{0x0958, 0xbb80},
+};
+
 static const struct hi556_reg mode_1296x972_regs[] = {
 	{0x0a00, 0x0000},
 	{0x0b0a, 0x8259},
@@ -450,8 +504,14 @@ static const struct hi556_link_freq_config link_freq_configs[] = {
 
 static const struct hi556_mode supported_modes[] = {
 	{
-		.width = 2592,
-		.height = 1944,
+		.width = HI556_PIXEL_ARRAY_WIDTH,
+		.height = HI556_PIXEL_ARRAY_HEIGHT,
+		.crop = {
+			.left = HI556_PIXEL_ARRAY_LEFT,
+			.top = HI556_PIXEL_ARRAY_TOP,
+			.width = HI556_PIXEL_ARRAY_WIDTH,
+			.height = HI556_PIXEL_ARRAY_HEIGHT
+		},
 		.fll_def = HI556_FLL_30FPS,
 		.fll_min = HI556_FLL_30FPS_MIN,
 		.llp = 0x0b00,
@@ -462,8 +522,32 @@ static const struct hi556_mode supported_modes[] = {
 		.link_freq_index = HI556_LINK_FREQ_437MHZ_INDEX,
 	},
 	{
+		.width = HI556_PIXEL_ARRAY_WIDTH,
+		.height = 1444,
+		.crop = {
+			.left = HI556_PIXEL_ARRAY_LEFT,
+			.top = 250,
+			.width = HI556_PIXEL_ARRAY_WIDTH,
+			.height = 1444
+		},
+		.fll_def = 0x821,
+		.fll_min = 0x821,
+		.llp = 0x0b00,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_2592x1444_regs),
+			.regs = mode_2592x1444_regs,
+		},
+		.link_freq_index = HI556_LINK_FREQ_437MHZ_INDEX,
+	},
+	{
 		.width = 1296,
 		.height = 972,
+		.crop = {
+			.left = HI556_PIXEL_ARRAY_LEFT,
+			.top = HI556_PIXEL_ARRAY_TOP,
+			.width = HI556_PIXEL_ARRAY_WIDTH,
+			.height = HI556_PIXEL_ARRAY_HEIGHT
+		},
 		.fll_def = HI556_FLL_30FPS,
 		.fll_min = HI556_FLL_30FPS_MIN,
 		.llp = 0x0b00,
@@ -785,6 +869,58 @@ static int hi556_identify_module(struct hi556 *hi556)
 	return 0;
 }
 
+static const struct v4l2_rect *
+__hi556_get_pad_crop(struct hi556 *hi556,
+		     struct v4l2_subdev_state *sd_state,
+		     unsigned int pad, enum v4l2_subdev_format_whence which)
+{
+	switch (which) {
+	case V4L2_SUBDEV_FORMAT_TRY:
+		return v4l2_subdev_get_try_crop(&hi556->sd, sd_state, pad);
+	case V4L2_SUBDEV_FORMAT_ACTIVE:
+		return &hi556->cur_mode->crop;
+	}
+
+	return NULL;
+}
+
+static int hi556_get_selection(struct v4l2_subdev *sd,
+			       struct v4l2_subdev_state *sd_state,
+			       struct v4l2_subdev_selection *sel)
+{
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP: {
+		struct hi556 *hi556 = to_hi556(sd);
+
+		mutex_lock(&hi556->mutex);
+		sel->r = *__hi556_get_pad_crop(hi556, sd_state, sel->pad,
+						sel->which);
+		mutex_unlock(&hi556->mutex);
+
+		return 0;
+	}
+
+	case V4L2_SEL_TGT_NATIVE_SIZE:
+		sel->r.top = 0;
+		sel->r.left = 0;
+		sel->r.width = HI556_NATIVE_WIDTH;
+		sel->r.height = HI556_NATIVE_HEIGHT;
+
+		return 0;
+
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		sel->r.top = HI556_PIXEL_ARRAY_TOP;
+		sel->r.left = HI556_PIXEL_ARRAY_LEFT;
+		sel->r.width = HI556_PIXEL_ARRAY_WIDTH;
+		sel->r.height = HI556_PIXEL_ARRAY_HEIGHT;
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
 static int hi556_start_streaming(struct hi556 *hi556)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(&hi556->sd);
@@ -1000,10 +1136,19 @@ static int hi556_enum_frame_size(struct v4l2_subdev *sd,
 static int hi556_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 {
 	struct hi556 *hi556 = to_hi556(sd);
+	struct v4l2_rect *try_crop;
 
 	mutex_lock(&hi556->mutex);
 	hi556_assign_pad_format(&supported_modes[0],
 				v4l2_subdev_get_try_format(sd, fh->state, 0));
+
+	/* Initialize try_crop rectangle. */
+	try_crop = v4l2_subdev_get_try_crop(sd, fh->state, 0);
+	try_crop->top = HI556_PIXEL_ARRAY_TOP;
+	try_crop->left = HI556_PIXEL_ARRAY_LEFT;
+	try_crop->width = HI556_PIXEL_ARRAY_WIDTH;
+	try_crop->height = HI556_PIXEL_ARRAY_HEIGHT;
+
 	mutex_unlock(&hi556->mutex);
 
 	return 0;
@@ -1016,6 +1161,7 @@ static const struct v4l2_subdev_video_ops hi556_video_ops = {
 static const struct v4l2_subdev_pad_ops hi556_pad_ops = {
 	.set_fmt = hi556_set_format,
 	.get_fmt = hi556_get_format,
+	.get_selection = hi556_get_selection,
 	.enum_mbus_code = hi556_enum_mbus_code,
 	.enum_frame_size = hi556_enum_frame_size,
 };
diff --git a/drivers/media/i2c/hi846.c b/drivers/media/i2c/hi846.c
index 7c61873b7198..306dc35e925f 100644
--- a/drivers/media/i2c/hi846.c
+++ b/drivers/media/i2c/hi846.c
@@ -1472,21 +1472,26 @@ static int hi846_init_controls(struct hi846 *hi846)
 	if (ctrl_hdlr->error) {
 		dev_err(&client->dev, "v4l ctrl handler error: %d\n",
 			ctrl_hdlr->error);
-		return ctrl_hdlr->error;
+		ret = ctrl_hdlr->error;
+		goto error;
 	}
 
 	ret = v4l2_fwnode_device_parse(&client->dev, &props);
 	if (ret)
-		return ret;
+		goto error;
 
 	ret = v4l2_ctrl_new_fwnode_properties(ctrl_hdlr, &hi846_ctrl_ops,
 					      &props);
 	if (ret)
-		return ret;
+		goto error;
 
 	hi846->sd.ctrl_handler = ctrl_hdlr;
 
 	return 0;
+
+error:
+	v4l2_ctrl_handler_free(ctrl_hdlr);
+	return ret;
 }
 
 static int hi846_set_video_mode(struct hi846 *hi846, int fps)
diff --git a/drivers/media/i2c/imx258.c b/drivers/media/i2c/imx258.c
index eab5fc1ee2f7..85d73b186111 100644
--- a/drivers/media/i2c/imx258.c
+++ b/drivers/media/i2c/imx258.c
@@ -9,6 +9,7 @@
 #include <linux/pm_runtime.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
 #include <asm/unaligned.h>
 
 #define IMX258_REG_VALUE_08BIT		1
@@ -1148,7 +1149,9 @@ static const struct v4l2_subdev_internal_ops imx258_internal_ops = {
 static int imx258_init_controls(struct imx258 *imx258)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(&imx258->sd);
+	struct v4l2_fwnode_device_properties props;
 	struct v4l2_ctrl_handler *ctrl_hdlr;
+	struct v4l2_ctrl *vflip, *hflip;
 	s64 vblank_def;
 	s64 vblank_min;
 	s64 pixel_rate_min;
@@ -1156,7 +1159,7 @@ static int imx258_init_controls(struct imx258 *imx258)
 	int ret;
 
 	ctrl_hdlr = &imx258->ctrl_handler;
-	ret = v4l2_ctrl_handler_init(ctrl_hdlr, 8);
+	ret = v4l2_ctrl_handler_init(ctrl_hdlr, 13);
 	if (ret)
 		return ret;
 
@@ -1172,6 +1175,17 @@ static int imx258_init_controls(struct imx258 *imx258)
 	if (imx258->link_freq)
 		imx258->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
+	/* The driver only supports one bayer order and flips by default. */
+	hflip = v4l2_ctrl_new_std(ctrl_hdlr, &imx258_ctrl_ops,
+				  V4L2_CID_HFLIP, 1, 1, 1, 1);
+	if (hflip)
+		hflip->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	vflip = v4l2_ctrl_new_std(ctrl_hdlr, &imx258_ctrl_ops,
+				  V4L2_CID_VFLIP, 1, 1, 1, 1);
+	if (vflip)
+		vflip->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
 	pixel_rate_max = link_freq_to_pixel_rate(link_freq_menu_items[0]);
 	pixel_rate_min = link_freq_to_pixel_rate(link_freq_menu_items[1]);
 	/* By default, PIXEL_RATE is read only */
@@ -1232,6 +1246,15 @@ static int imx258_init_controls(struct imx258 *imx258)
 		goto error;
 	}
 
+	ret = v4l2_fwnode_device_parse(&client->dev, &props);
+	if (ret)
+		goto error;
+
+	ret = v4l2_ctrl_new_fwnode_properties(ctrl_hdlr, &imx258_ctrl_ops,
+					      &props);
+	if (ret)
+		goto error;
+
 	imx258->sd.ctrl_handler = ctrl_hdlr;
 
 	return 0;
@@ -1276,14 +1299,6 @@ static int imx258_probe(struct i2c_client *client)
 		return -EINVAL;
 	}
 
-	/*
-	 * Check that the device is mounted upside down. The driver only
-	 * supports a single pixel order right now.
-	 */
-	ret = device_property_read_u32(&client->dev, "rotation", &val);
-	if (ret || val != 180)
-		return -EINVAL;
-
 	/* Initialize subdev */
 	v4l2_i2c_subdev_init(&imx258->sd, client, &imx258_subdev_ops);
 
diff --git a/drivers/media/i2c/imx290.c b/drivers/media/i2c/imx290.c
index 48ae2e0adf9e..5ea25b7acc55 100644
--- a/drivers/media/i2c/imx290.c
+++ b/drivers/media/i2c/imx290.c
@@ -13,12 +13,17 @@
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
+
+#include <asm/unaligned.h>
+
 #include <media/media-entity.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
 #include <media/v4l2-fwnode.h>
 #include <media/v4l2-subdev.h>
 
@@ -44,7 +49,9 @@
 #define IMX290_BLKLEVEL					IMX290_REG_16BIT(0x300a)
 #define IMX290_GAIN					IMX290_REG_8BIT(0x3014)
 #define IMX290_VMAX					IMX290_REG_24BIT(0x3018)
+#define IMX290_VMAX_MAX					0x3ffff
 #define IMX290_HMAX					IMX290_REG_16BIT(0x301c)
+#define IMX290_HMAX_MAX					0xffff
 #define IMX290_SHS1					IMX290_REG_24BIT(0x3020)
 #define IMX290_WINWV_OB					IMX290_REG_8BIT(0x303a)
 #define IMX290_WINPV					IMX290_REG_16BIT(0x303c)
@@ -98,13 +105,16 @@
 #define IMX290_TCLKPREPARE				IMX290_REG_16BIT(0x3452)
 #define IMX290_TLPX					IMX290_REG_16BIT(0x3454)
 #define IMX290_X_OUT_SIZE				IMX290_REG_16BIT(0x3472)
+#define IMX290_INCKSEL7					IMX290_REG_8BIT(0x3480)
 
 #define IMX290_PGCTRL_REGEN				BIT(0)
 #define IMX290_PGCTRL_THRU				BIT(1)
 #define IMX290_PGCTRL_MODE(n)				((n) << 4)
 
-#define IMX290_VMAX_DEFAULT				1125
+/* Number of lines by which exposure must be less than VMAX */
+#define IMX290_EXPOSURE_OFFSET				2
 
+#define IMX290_PIXEL_RATE				148500000
 
 /*
  * The IMX290 pixel array is organized as follows:
@@ -157,26 +167,81 @@
 
 #define IMX290_NUM_SUPPLIES				3
 
+enum imx290_colour_variant {
+	IMX290_VARIANT_COLOUR,
+	IMX290_VARIANT_MONO,
+	IMX290_VARIANT_MAX
+};
+
+enum imx290_model {
+	IMX290_MODEL_IMX290LQR,
+	IMX290_MODEL_IMX290LLR,
+	IMX290_MODEL_IMX327LQR,
+};
+
+struct imx290_model_info {
+	enum imx290_colour_variant colour_variant;
+	const struct imx290_regval *init_regs;
+	size_t init_regs_num;
+	const char *name;
+};
+
+enum imx290_clk_freq {
+	IMX290_CLK_37_125,
+	IMX290_CLK_74_25,
+	IMX290_NUM_CLK
+};
+
 struct imx290_regval {
 	u32 reg;
 	u32 val;
 };
 
+/*
+ * Clock configuration for registers INCKSEL1 to INCKSEL6.
+ */
+struct imx290_clk_cfg {
+	u8 incksel1;
+	u8 incksel2;
+	u8 incksel3;
+	u8 incksel4;
+	u8 incksel5;
+	u8 incksel6;
+};
+
 struct imx290_mode {
 	u32 width;
 	u32 height;
-	u32 hmax;
+	u32 hmax_min;
+	u32 vmax_min;
 	u8 link_freq_index;
+	u8 ctrl_07;
 
 	const struct imx290_regval *data;
 	u32 data_size;
+
+	const struct imx290_clk_cfg *clk_cfg;
+};
+
+struct imx290_csi_cfg {
+	u16 repetition;
+	u16 tclkpost;
+	u16 thszero;
+	u16 thsprepare;
+	u16 tclktrail;
+	u16 thstrail;
+	u16 tclkzero;
+	u16 tclkprepare;
+	u16 tlpx;
 };
 
 struct imx290 {
 	struct device *dev;
 	struct clk *xclk;
 	struct regmap *regmap;
+	enum imx290_clk_freq xclk_idx;
 	u8 nlanes;
+	const struct imx290_model_info *model;
 
 	struct v4l2_subdev sd;
 	struct media_pad pad;
@@ -188,9 +253,13 @@ struct imx290 {
 
 	struct v4l2_ctrl_handler ctrls;
 	struct v4l2_ctrl *link_freq;
-	struct v4l2_ctrl *pixel_rate;
 	struct v4l2_ctrl *hblank;
 	struct v4l2_ctrl *vblank;
+	struct v4l2_ctrl *exposure;
+	struct {
+		struct v4l2_ctrl *hflip;
+		struct v4l2_ctrl *vflip;
+	};
 };
 
 static inline struct imx290 *to_imx290(struct v4l2_subdev *_sd)
@@ -203,9 +272,6 @@ static inline struct imx290 *to_imx290(struct v4l2_subdev *_sd)
  */
 
 static const struct imx290_regval imx290_global_init_settings[] = {
-	{ IMX290_CTRL_07, IMX290_WINMODE_1080P },
-	{ IMX290_VMAX, IMX290_VMAX_DEFAULT },
-	{ IMX290_EXTCK_FREQ, 0x2520 },
 	{ IMX290_WINWV_OB, 12 },
 	{ IMX290_WINPH, 0 },
 	{ IMX290_WINPV, 0 },
@@ -213,10 +279,14 @@ static const struct imx290_regval imx290_global_init_settings[] = {
 	{ IMX290_WINWV, 1097 },
 	{ IMX290_XSOUTSEL, IMX290_XSOUTSEL_XVSOUTSEL_VSYNC |
 			   IMX290_XSOUTSEL_XHSOUTSEL_HSYNC },
-	{ IMX290_REG_8BIT(0x300f), 0x00 },
-	{ IMX290_REG_8BIT(0x3010), 0x21 },
+	{ IMX290_REG_8BIT(0x3011), 0x02 },
 	{ IMX290_REG_8BIT(0x3012), 0x64 },
 	{ IMX290_REG_8BIT(0x3013), 0x00 },
+};
+
+static const struct imx290_regval imx290_global_init_settings_290[] = {
+	{ IMX290_REG_8BIT(0x300f), 0x00 },
+	{ IMX290_REG_8BIT(0x3010), 0x21 },
 	{ IMX290_REG_8BIT(0x3016), 0x09 },
 	{ IMX290_REG_8BIT(0x3070), 0x02 },
 	{ IMX290_REG_8BIT(0x3071), 0x11 },
@@ -255,57 +325,40 @@ static const struct imx290_regval imx290_global_init_settings[] = {
 	{ IMX290_REG_8BIT(0x33b0), 0x50 },
 	{ IMX290_REG_8BIT(0x33b2), 0x1a },
 	{ IMX290_REG_8BIT(0x33b3), 0x04 },
-	{ IMX290_REG_8BIT(0x3480), 0x49 },
+};
+
+#define IMX290_NUM_CLK_REGS	2
+static const struct imx290_regval xclk_regs[][IMX290_NUM_CLK_REGS] = {
+	[IMX290_CLK_37_125] = {
+		{ IMX290_EXTCK_FREQ, (37125 * 256) / 1000 },
+		{ IMX290_INCKSEL7, 0x49 },
+	},
+	[IMX290_CLK_74_25] = {
+		{ IMX290_EXTCK_FREQ, (74250 * 256) / 1000 },
+		{ IMX290_INCKSEL7, 0x92 },
+	},
+};
+
+static const struct imx290_regval imx290_global_init_settings_327[] = {
+	{ IMX290_REG_8BIT(0x309e), 0x4A },
+	{ IMX290_REG_8BIT(0x309f), 0x4A },
+	{ IMX290_REG_8BIT(0x313b), 0x61 },
 };
 
 static const struct imx290_regval imx290_1080p_settings[] = {
 	/* mode settings */
-	{ IMX290_CTRL_07, IMX290_WINMODE_1080P },
 	{ IMX290_WINWV_OB, 12 },
 	{ IMX290_OPB_SIZE_V, 10 },
 	{ IMX290_X_OUT_SIZE, 1920 },
 	{ IMX290_Y_OUT_SIZE, 1080 },
-	{ IMX290_INCKSEL1, 0x18 },
-	{ IMX290_INCKSEL2, 0x03 },
-	{ IMX290_INCKSEL3, 0x20 },
-	{ IMX290_INCKSEL4, 0x01 },
-	{ IMX290_INCKSEL5, 0x1a },
-	{ IMX290_INCKSEL6, 0x1a },
-	/* data rate settings */
-	{ IMX290_REPETITION, 0x10 },
-	{ IMX290_TCLKPOST, 87 },
-	{ IMX290_THSZERO, 55 },
-	{ IMX290_THSPREPARE, 31 },
-	{ IMX290_TCLKTRAIL, 31 },
-	{ IMX290_THSTRAIL, 31 },
-	{ IMX290_TCLKZERO, 119 },
-	{ IMX290_TCLKPREPARE, 31 },
-	{ IMX290_TLPX, 23 },
 };
 
 static const struct imx290_regval imx290_720p_settings[] = {
 	/* mode settings */
-	{ IMX290_CTRL_07, IMX290_WINMODE_720P },
 	{ IMX290_WINWV_OB, 6 },
 	{ IMX290_OPB_SIZE_V, 4 },
 	{ IMX290_X_OUT_SIZE, 1280 },
 	{ IMX290_Y_OUT_SIZE, 720 },
-	{ IMX290_INCKSEL1, 0x20 },
-	{ IMX290_INCKSEL2, 0x00 },
-	{ IMX290_INCKSEL3, 0x20 },
-	{ IMX290_INCKSEL4, 0x01 },
-	{ IMX290_INCKSEL5, 0x1a },
-	{ IMX290_INCKSEL6, 0x1a },
-	/* data rate settings */
-	{ IMX290_REPETITION, 0x10 },
-	{ IMX290_TCLKPOST, 79 },
-	{ IMX290_THSZERO, 47 },
-	{ IMX290_THSPREPARE, 23 },
-	{ IMX290_TCLKTRAIL, 23 },
-	{ IMX290_THSTRAIL, 23 },
-	{ IMX290_TCLKZERO, 87 },
-	{ IMX290_TCLKPREPARE, 23 },
-	{ IMX290_TLPX, 23 },
 };
 
 static const struct imx290_regval imx290_10bit_settings[] = {
@@ -326,6 +379,58 @@ static const struct imx290_regval imx290_12bit_settings[] = {
 	{ IMX290_CSI_DT_FMT, IMX290_CSI_DT_FMT_RAW12 },
 };
 
+static const struct imx290_csi_cfg imx290_csi_222_75mhz = {
+	/* 222.75MHz or 445.5Mbit/s per lane */
+	.repetition = 0x10,
+	.tclkpost = 87,
+	.thszero = 55,
+	.thsprepare = 31,
+	.tclktrail = 31,
+	.thstrail = 31,
+	.tclkzero = 119,
+	.tclkprepare = 31,
+	.tlpx = 23,
+};
+
+static const struct imx290_csi_cfg imx290_csi_445_5mhz = {
+	/* 445.5MHz or 891Mbit/s per lane */
+	.repetition = 0x00,
+	.tclkpost = 119,
+	.thszero = 103,
+	.thsprepare = 71,
+	.tclktrail = 55,
+	.thstrail = 63,
+	.tclkzero = 255,
+	.tclkprepare = 63,
+	.tlpx = 55,
+};
+
+static const struct imx290_csi_cfg imx290_csi_148_5mhz = {
+	/* 148.5MHz or 297Mbit/s per lane */
+	.repetition = 0x10,
+	.tclkpost = 79,
+	.thszero = 47,
+	.thsprepare = 23,
+	.tclktrail = 23,
+	.thstrail = 23,
+	.tclkzero = 87,
+	.tclkprepare = 23,
+	.tlpx = 23,
+};
+
+static const struct imx290_csi_cfg imx290_csi_297mhz = {
+	/* 297MHz or 594Mbit/s per lane */
+	.repetition = 0x00,
+	.tclkpost = 103,
+	.thszero = 87,
+	.thsprepare = 47,
+	.tclktrail = 39,
+	.thstrail = 47,
+	.tclkzero = 191,
+	.tclkprepare = 47,
+	.tlpx = 39,
+};
+
 /* supported link frequencies */
 #define FREQ_INDEX_1080P	0
 #define FREQ_INDEX_720P		1
@@ -333,6 +438,7 @@ static const s64 imx290_link_freq_2lanes[] = {
 	[FREQ_INDEX_1080P] = 445500000,
 	[FREQ_INDEX_720P] = 297000000,
 };
+
 static const s64 imx290_link_freq_4lanes[] = {
 	[FREQ_INDEX_1080P] = 222750000,
 	[FREQ_INDEX_720P] = 148500000,
@@ -358,23 +464,71 @@ static inline int imx290_link_freqs_num(const struct imx290 *imx290)
 		return ARRAY_SIZE(imx290_link_freq_4lanes);
 }
 
+static const struct imx290_clk_cfg imx290_1080p_clock_config[] = {
+	[IMX290_CLK_37_125] = {
+		/* 37.125MHz clock config */
+		.incksel1 = 0x18,
+		.incksel2 = 0x03,
+		.incksel3 = 0x20,
+		.incksel4 = 0x01,
+		.incksel5 = 0x1a,
+		.incksel6 = 0x1a,
+	},
+	[IMX290_CLK_74_25] = {
+		/* 74.25MHz clock config */
+		.incksel1 = 0x0c,
+		.incksel2 = 0x03,
+		.incksel3 = 0x10,
+		.incksel4 = 0x01,
+		.incksel5 = 0x1b,
+		.incksel6 = 0x1b,
+	},
+};
+
+static const struct imx290_clk_cfg imx290_720p_clock_config[] = {
+	[IMX290_CLK_37_125] = {
+		/* 37.125MHz clock config */
+		.incksel1 = 0x20,
+		.incksel2 = 0x00,
+		.incksel3 = 0x20,
+		.incksel4 = 0x01,
+		.incksel5 = 0x1a,
+		.incksel6 = 0x1a,
+	},
+	[IMX290_CLK_74_25] = {
+		/* 74.25MHz clock config */
+		.incksel1 = 0x10,
+		.incksel2 = 0x00,
+		.incksel3 = 0x10,
+		.incksel4 = 0x01,
+		.incksel5 = 0x1b,
+		.incksel6 = 0x1b,
+	},
+};
+
 /* Mode configs */
 static const struct imx290_mode imx290_modes_2lanes[] = {
 	{
 		.width = 1920,
 		.height = 1080,
-		.hmax = 4400,
+		.hmax_min = 2200,
+		.vmax_min = 1125,
 		.link_freq_index = FREQ_INDEX_1080P,
+		.ctrl_07 = IMX290_WINMODE_1080P,
 		.data = imx290_1080p_settings,
 		.data_size = ARRAY_SIZE(imx290_1080p_settings),
+		.clk_cfg = imx290_1080p_clock_config,
 	},
 	{
 		.width = 1280,
 		.height = 720,
-		.hmax = 6600,
+		.hmax_min = 3300,
+		.vmax_min = 750,
 		.link_freq_index = FREQ_INDEX_720P,
+		.ctrl_07 = IMX290_WINMODE_720P,
 		.data = imx290_720p_settings,
 		.data_size = ARRAY_SIZE(imx290_720p_settings),
+		.clk_cfg = imx290_720p_clock_config,
 	},
 };
 
@@ -382,18 +536,24 @@ static const struct imx290_mode imx290_modes_4lanes[] = {
 	{
 		.width = 1920,
 		.height = 1080,
-		.hmax = 2200,
+		.hmax_min = 2200,
+		.vmax_min = 1125,
 		.link_freq_index = FREQ_INDEX_1080P,
+		.ctrl_07 = IMX290_WINMODE_1080P,
 		.data = imx290_1080p_settings,
 		.data_size = ARRAY_SIZE(imx290_1080p_settings),
+		.clk_cfg = imx290_1080p_clock_config,
 	},
 	{
 		.width = 1280,
 		.height = 720,
-		.hmax = 3300,
+		.hmax_min = 3300,
+		.vmax_min = 750,
 		.link_freq_index = FREQ_INDEX_720P,
+		.ctrl_07 = IMX290_WINMODE_720P,
 		.data = imx290_720p_settings,
 		.data_size = ARRAY_SIZE(imx290_720p_settings),
+		.clk_cfg = imx290_720p_clock_config,
 	},
 };
 
@@ -414,7 +574,7 @@ static inline int imx290_modes_num(const struct imx290 *imx290)
 }
 
 struct imx290_format_info {
-	u32 code;
+	u32 code[IMX290_VARIANT_MAX];
 	u8 bpp;
 	const struct imx290_regval *regs;
 	unsigned int num_regs;
@@ -422,26 +582,33 @@ struct imx290_format_info {
 
 static const struct imx290_format_info imx290_formats[] = {
 	{
-		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.code = {
+			[IMX290_VARIANT_COLOUR] = MEDIA_BUS_FMT_SRGGB10_1X10,
+			[IMX290_VARIANT_MONO] = MEDIA_BUS_FMT_Y10_1X10
+		},
 		.bpp = 10,
 		.regs = imx290_10bit_settings,
 		.num_regs = ARRAY_SIZE(imx290_10bit_settings),
 	}, {
-		.code = MEDIA_BUS_FMT_SRGGB12_1X12,
+		.code = {
+			[IMX290_VARIANT_COLOUR] = MEDIA_BUS_FMT_SRGGB12_1X12,
+			[IMX290_VARIANT_MONO] = MEDIA_BUS_FMT_Y12_1X12
+		},
 		.bpp = 12,
 		.regs = imx290_12bit_settings,
 		.num_regs = ARRAY_SIZE(imx290_12bit_settings),
 	}
 };
 
-static const struct imx290_format_info *imx290_format_info(u32 code)
+static const struct imx290_format_info *
+imx290_format_info(const struct imx290 *imx290, u32 code)
 {
 	unsigned int i;
 
 	for (i = 0; i < ARRAY_SIZE(imx290_formats); ++i) {
 		const struct imx290_format_info *info = &imx290_formats[i];
 
-		if (info->code == code)
+		if (info->code[imx290->model->colour_variant] == code)
 			return info;
 	}
 
@@ -461,28 +628,30 @@ static int __always_unused imx290_read(struct imx290 *imx290, u32 addr, u32 *val
 			      data, (addr >> IMX290_REG_SIZE_SHIFT) & 3);
 	if (ret < 0) {
 		dev_err(imx290->dev, "%u-bit read from 0x%04x failed: %d\n",
-			 ((addr >> IMX290_REG_SIZE_SHIFT) & 3) * 8,
+			((addr >> IMX290_REG_SIZE_SHIFT) & 3) * 8,
 			 addr & IMX290_REG_ADDR_MASK, ret);
 		return ret;
 	}
 
-	*value = (data[2] << 16) | (data[1] << 8) | data[0];
+	*value = get_unaligned_le24(data);
 	return 0;
 }
 
 static int imx290_write(struct imx290 *imx290, u32 addr, u32 value, int *err)
 {
-	u8 data[3] = { value & 0xff, (value >> 8) & 0xff, value >> 16 };
+	u8 data[3];
 	int ret;
 
 	if (err && *err)
 		return *err;
 
+	put_unaligned_le24(value, data);
+
 	ret = regmap_raw_write(imx290->regmap, addr & IMX290_REG_ADDR_MASK,
 			       data, (addr >> IMX290_REG_SIZE_SHIFT) & 3);
 	if (ret < 0) {
 		dev_err(imx290->dev, "%u-bit write to 0x%04x failed: %d\n",
-			 ((addr >> IMX290_REG_SIZE_SHIFT) & 3) * 8,
+			((addr >> IMX290_REG_SIZE_SHIFT) & 3) * 8,
 			 addr & IMX290_REG_ADDR_MASK, ret);
 		if (err)
 			*err = ret;
@@ -510,24 +679,33 @@ static int imx290_set_register_array(struct imx290 *imx290,
 	return 0;
 }
 
+static int imx290_set_clock(struct imx290 *imx290)
+{
+	const struct imx290_mode *mode = imx290->current_mode;
+	enum imx290_clk_freq clk_idx = imx290->xclk_idx;
+	const struct imx290_clk_cfg *clk_cfg = &mode->clk_cfg[clk_idx];
+	int ret;
+
+	ret = imx290_set_register_array(imx290, xclk_regs[clk_idx],
+					IMX290_NUM_CLK_REGS);
+
+	imx290_write(imx290, IMX290_INCKSEL1, clk_cfg->incksel1, &ret);
+	imx290_write(imx290, IMX290_INCKSEL2, clk_cfg->incksel2, &ret);
+	imx290_write(imx290, IMX290_INCKSEL3, clk_cfg->incksel3, &ret);
+	imx290_write(imx290, IMX290_INCKSEL4, clk_cfg->incksel4, &ret);
+	imx290_write(imx290, IMX290_INCKSEL5, clk_cfg->incksel5, &ret);
+	imx290_write(imx290, IMX290_INCKSEL6, clk_cfg->incksel6, &ret);
+
+	return ret;
+}
+
 static int imx290_set_data_lanes(struct imx290 *imx290)
 {
 	int ret = 0;
-	u32 frsel;
-
-	switch (imx290->nlanes) {
-	case 2:
-	default:
-		frsel = 0x02;
-		break;
-	case 4:
-		frsel = 0x01;
-		break;
-	}
 
 	imx290_write(imx290, IMX290_PHY_LANE_NUM, imx290->nlanes - 1, &ret);
 	imx290_write(imx290, IMX290_CSI_LANE_MODE, imx290->nlanes - 1, &ret);
-	imx290_write(imx290, IMX290_FR_FDG_SEL, frsel, &ret);
+	imx290_write(imx290, IMX290_FR_FDG_SEL, 0x01, &ret);
 
 	return ret;
 }
@@ -536,19 +714,55 @@ static int imx290_set_black_level(struct imx290 *imx290,
 				  const struct v4l2_mbus_framefmt *format,
 				  unsigned int black_level, int *err)
 {
-	unsigned int bpp = imx290_format_info(format->code)->bpp;
+	unsigned int bpp = imx290_format_info(imx290, format->code)->bpp;
 
 	return imx290_write(imx290, IMX290_BLKLEVEL,
 			    black_level >> (16 - bpp), err);
 }
 
+static int imx290_set_csi_config(struct imx290 *imx290)
+{
+	const s64 *link_freqs = imx290_link_freqs_ptr(imx290);
+	const struct imx290_csi_cfg *csi_cfg;
+	int ret = 0;
+
+	switch (link_freqs[imx290->current_mode->link_freq_index]) {
+	case 445500000:
+		csi_cfg = &imx290_csi_445_5mhz;
+		break;
+	case 297000000:
+		csi_cfg = &imx290_csi_297mhz;
+		break;
+	case 222750000:
+		csi_cfg = &imx290_csi_222_75mhz;
+		break;
+	case 148500000:
+		csi_cfg = &imx290_csi_148_5mhz;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	imx290_write(imx290, IMX290_REPETITION, csi_cfg->repetition, &ret);
+	imx290_write(imx290, IMX290_TCLKPOST, csi_cfg->tclkpost, &ret);
+	imx290_write(imx290, IMX290_THSZERO, csi_cfg->thszero, &ret);
+	imx290_write(imx290, IMX290_THSPREPARE, csi_cfg->thsprepare, &ret);
+	imx290_write(imx290, IMX290_TCLKTRAIL, csi_cfg->tclktrail, &ret);
+	imx290_write(imx290, IMX290_THSTRAIL, csi_cfg->thstrail, &ret);
+	imx290_write(imx290, IMX290_TCLKZERO, csi_cfg->tclkzero, &ret);
+	imx290_write(imx290, IMX290_TCLKPREPARE, csi_cfg->tclkprepare, &ret);
+	imx290_write(imx290, IMX290_TLPX, csi_cfg->tlpx, &ret);
+
+	return ret;
+}
+
 static int imx290_setup_format(struct imx290 *imx290,
 			       const struct v4l2_mbus_framefmt *format)
 {
 	const struct imx290_format_info *info;
 	int ret;
 
-	info = imx290_format_info(format->code);
+	info = imx290_format_info(imx290, format->code);
 
 	ret = imx290_set_register_array(imx290, info->regs, info->num_regs);
 	if (ret < 0) {
@@ -563,6 +777,16 @@ static int imx290_setup_format(struct imx290 *imx290,
 /* ----------------------------------------------------------------------------
  * Controls
  */
+static void imx290_exposure_update(struct imx290 *imx290,
+				   const struct imx290_mode *mode)
+{
+	unsigned int exposure_max;
+
+	exposure_max = imx290->vblank->val + mode->height -
+		       IMX290_EXPOSURE_OFFSET;
+	__v4l2_ctrl_modify_range(imx290->exposure, 1, exposure_max, 1,
+				 exposure_max);
+}
 
 static int imx290_set_ctrl(struct v4l2_ctrl *ctrl)
 {
@@ -570,7 +794,7 @@ static int imx290_set_ctrl(struct v4l2_ctrl *ctrl)
 					     struct imx290, ctrls);
 	const struct v4l2_mbus_framefmt *format;
 	struct v4l2_subdev_state *state;
-	int ret = 0;
+	int ret = 0, vmax;
 
 	/*
 	 * Return immediately for controls that don't need to be applied to the
@@ -579,6 +803,11 @@ static int imx290_set_ctrl(struct v4l2_ctrl *ctrl)
 	if (ctrl->flags & V4L2_CTRL_FLAG_READ_ONLY)
 		return 0;
 
+	if (ctrl->id == V4L2_CID_VBLANK) {
+		/* Changing vblank changes the allowed range for exposure. */
+		imx290_exposure_update(imx290, imx290->current_mode);
+	}
+
 	/* V4L2 controls values will be applied only when power is already up */
 	if (!pm_runtime_get_if_in_use(imx290->dev))
 		return 0;
@@ -591,9 +820,23 @@ static int imx290_set_ctrl(struct v4l2_ctrl *ctrl)
 		ret = imx290_write(imx290, IMX290_GAIN, ctrl->val, NULL);
 		break;
 
+	case V4L2_CID_VBLANK:
+		ret = imx290_write(imx290, IMX290_VMAX,
+				   ctrl->val + imx290->current_mode->height,
+				   NULL);
+		/*
+		 * Due to the way that exposure is programmed in this sensor in
+		 * relation to VMAX, we have to reprogramme it whenever VMAX is
+		 * changed.
+		 * Update ctrl so that the V4L2_CID_EXPOSURE case can refer to
+		 * it.
+		 */
+		ctrl = imx290->exposure;
+		fallthrough;
 	case V4L2_CID_EXPOSURE:
+		vmax = imx290->vblank->val + imx290->current_mode->height;
 		ret = imx290_write(imx290, IMX290_SHS1,
-				   IMX290_VMAX_DEFAULT - ctrl->val - 1, NULL);
+				   vmax - ctrl->val - 1, NULL);
 		break;
 
 	case V4L2_CID_TEST_PATTERN:
@@ -612,6 +855,26 @@ static int imx290_set_ctrl(struct v4l2_ctrl *ctrl)
 		}
 		break;
 
+	case V4L2_CID_HBLANK:
+		ret = imx290_write(imx290, IMX290_HMAX,
+				   ctrl->val + imx290->current_mode->width,
+				   NULL);
+		break;
+
+	case V4L2_CID_HFLIP:
+	case V4L2_CID_VFLIP:
+	{
+		u32 reg;
+
+		reg = imx290->current_mode->ctrl_07;
+		if (imx290->hflip->val)
+			reg |= IMX290_HREVERSE;
+		if (imx290->vflip->val)
+			reg |= IMX290_VREVERSE;
+		ret = imx290_write(imx290, IMX290_CTRL_07, reg, NULL);
+		break;
+	}
+
 	default:
 		ret = -EINVAL;
 		break;
@@ -642,20 +905,17 @@ static void imx290_ctrl_update(struct imx290 *imx290,
 			       const struct v4l2_mbus_framefmt *format,
 			       const struct imx290_mode *mode)
 {
-	unsigned int hblank = mode->hmax - mode->width;
-	unsigned int vblank = IMX290_VMAX_DEFAULT - mode->height;
-	s64 link_freq = imx290_link_freqs_ptr(imx290)[mode->link_freq_index];
-	u64 pixel_rate;
-
-	/* pixel rate = link_freq * 2 * nr_of_lanes / bits_per_sample */
-	pixel_rate = link_freq * 2 * imx290->nlanes;
-	do_div(pixel_rate, imx290_format_info(format->code)->bpp);
+	unsigned int hblank_min = mode->hmax_min - mode->width;
+	unsigned int hblank_max = IMX290_HMAX_MAX - mode->width;
+	unsigned int vblank_min = mode->vmax_min - mode->height;
+	unsigned int vblank_max = IMX290_VMAX_MAX - mode->height;
 
 	__v4l2_ctrl_s_ctrl(imx290->link_freq, mode->link_freq_index);
-	__v4l2_ctrl_s_ctrl_int64(imx290->pixel_rate, pixel_rate);
 
-	__v4l2_ctrl_modify_range(imx290->hblank, hblank, hblank, 1, hblank);
-	__v4l2_ctrl_modify_range(imx290->vblank, vblank, vblank, 1, vblank);
+	__v4l2_ctrl_modify_range(imx290->hblank, hblank_min, hblank_max, 1,
+				 hblank_min);
+	__v4l2_ctrl_modify_range(imx290->vblank, vblank_min, vblank_max, 1,
+				 vblank_min);
 }
 
 static int imx290_ctrl_init(struct imx290 *imx290)
@@ -667,7 +927,7 @@ static int imx290_ctrl_init(struct imx290 *imx290)
 	if (ret < 0)
 		return ret;
 
-	v4l2_ctrl_handler_init(&imx290->ctrls, 9);
+	v4l2_ctrl_handler_init(&imx290->ctrls, 11);
 
 	/*
 	 * The sensor has an analog gain and a digital gain, both controlled
@@ -685,9 +945,13 @@ static int imx290_ctrl_init(struct imx290 *imx290)
 	v4l2_ctrl_new_std(&imx290->ctrls, &imx290_ctrl_ops,
 			  V4L2_CID_ANALOGUE_GAIN, 0, 100, 1, 0);
 
-	v4l2_ctrl_new_std(&imx290->ctrls, &imx290_ctrl_ops,
-			  V4L2_CID_EXPOSURE, 1, IMX290_VMAX_DEFAULT - 2, 1,
-			  IMX290_VMAX_DEFAULT - 2);
+	/*
+	 * Correct range will be determined through imx290_ctrl_update setting
+	 * V4L2_CID_VBLANK.
+	 */
+	imx290->exposure = v4l2_ctrl_new_std(&imx290->ctrls, &imx290_ctrl_ops,
+					     V4L2_CID_EXPOSURE, 1, 65535, 1,
+					     65535);
 
 	/*
 	 * Set the link frequency, pixel rate, horizontal blanking and vertical
@@ -702,24 +966,29 @@ static int imx290_ctrl_init(struct imx290 *imx290)
 	if (imx290->link_freq)
 		imx290->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
-	imx290->pixel_rate = v4l2_ctrl_new_std(&imx290->ctrls, &imx290_ctrl_ops,
-					       V4L2_CID_PIXEL_RATE,
-					       1, INT_MAX, 1, 1);
+	v4l2_ctrl_new_std(&imx290->ctrls, &imx290_ctrl_ops, V4L2_CID_PIXEL_RATE,
+			  IMX290_PIXEL_RATE, IMX290_PIXEL_RATE, 1,
+			  IMX290_PIXEL_RATE);
 
 	v4l2_ctrl_new_std_menu_items(&imx290->ctrls, &imx290_ctrl_ops,
 				     V4L2_CID_TEST_PATTERN,
 				     ARRAY_SIZE(imx290_test_pattern_menu) - 1,
 				     0, 0, imx290_test_pattern_menu);
 
+	/*
+	 * Actual range will be set from imx290_ctrl_update later in the probe.
+	 */
 	imx290->hblank = v4l2_ctrl_new_std(&imx290->ctrls, &imx290_ctrl_ops,
 					   V4L2_CID_HBLANK, 1, 1, 1, 1);
-	if (imx290->hblank)
-		imx290->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
 	imx290->vblank = v4l2_ctrl_new_std(&imx290->ctrls, &imx290_ctrl_ops,
 					   V4L2_CID_VBLANK, 1, 1, 1, 1);
-	if (imx290->vblank)
-		imx290->vblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	imx290->hflip = v4l2_ctrl_new_std(&imx290->ctrls, &imx290_ctrl_ops,
+					  V4L2_CID_HFLIP, 0, 1, 1, 0);
+	imx290->vflip = v4l2_ctrl_new_std(&imx290->ctrls, &imx290_ctrl_ops,
+					  V4L2_CID_VFLIP, 0, 1, 1, 0);
+	v4l2_ctrl_cluster(2, &imx290->hflip);
 
 	v4l2_ctrl_new_fwnode_properties(&imx290->ctrls, &imx290_ctrl_ops,
 					&props);
@@ -748,17 +1017,37 @@ static int imx290_start_streaming(struct imx290 *imx290,
 
 	/* Set init register settings */
 	ret = imx290_set_register_array(imx290, imx290_global_init_settings,
-					ARRAY_SIZE(
-						imx290_global_init_settings));
+					ARRAY_SIZE(imx290_global_init_settings));
 	if (ret < 0) {
 		dev_err(imx290->dev, "Could not set init registers\n");
 		return ret;
 	}
 
+	/* Set mdel specific init register settings */
+	ret = imx290_set_register_array(imx290, imx290->model->init_regs,
+					imx290->model->init_regs_num);
+	if (ret < 0) {
+		dev_err(imx290->dev, "Could not set model specific init registers\n");
+		return ret;
+	}
+
+	/* Set clock parameters based on mode and xclk */
+	ret = imx290_set_clock(imx290);
+	if (ret < 0) {
+		dev_err(imx290->dev, "Could not set clocks - %d\n", ret);
+		return ret;
+	}
+
 	/* Set data lane count */
 	ret = imx290_set_data_lanes(imx290);
 	if (ret < 0) {
-		dev_err(imx290->dev, "Could not set data lanes\n");
+		dev_err(imx290->dev, "Could not set data lanes - %d\n", ret);
+		return ret;
+	}
+
+	ret = imx290_set_csi_config(imx290);
+	if (ret < 0) {
+		dev_err(imx290->dev, "Could not set csi cfg - %d\n", ret);
 		return ret;
 	}
 
@@ -766,7 +1055,7 @@ static int imx290_start_streaming(struct imx290 *imx290,
 	format = v4l2_subdev_get_pad_format(&imx290->sd, state, 0);
 	ret = imx290_setup_format(imx290, format);
 	if (ret < 0) {
-		dev_err(imx290->dev, "Could not set frame format\n");
+		dev_err(imx290->dev, "Could not set frame format - %d\n", ret);
 		return ret;
 	}
 
@@ -774,19 +1063,14 @@ static int imx290_start_streaming(struct imx290 *imx290,
 	ret = imx290_set_register_array(imx290, imx290->current_mode->data,
 					imx290->current_mode->data_size);
 	if (ret < 0) {
-		dev_err(imx290->dev, "Could not set current mode\n");
+		dev_err(imx290->dev, "Could not set current mode - %d\n", ret);
 		return ret;
 	}
 
-	ret = imx290_write(imx290, IMX290_HMAX, imx290->current_mode->hmax,
-			   NULL);
-	if (ret)
-		return ret;
-
 	/* Apply customized values from user */
 	ret = __v4l2_ctrl_handler_setup(imx290->sd.ctrl_handler);
 	if (ret) {
-		dev_err(imx290->dev, "Could not sync v4l2 controls\n");
+		dev_err(imx290->dev, "Could not sync v4l2 controls - %d\n", ret);
 		return ret;
 	}
 
@@ -835,6 +1119,13 @@ static int imx290_set_stream(struct v4l2_subdev *sd, int enable)
 		pm_runtime_put_autosuspend(imx290->dev);
 	}
 
+	/*
+	 * vflip and hflip should not be changed during streaming as the sensor
+	 * will produce an invalid frame.
+	 */
+	__v4l2_ctrl_grab(imx290->vflip, enable);
+	__v4l2_ctrl_grab(imx290->hflip, enable);
+
 unlock:
 	v4l2_subdev_unlock_state(state);
 	return ret;
@@ -844,10 +1135,12 @@ static int imx290_enum_mbus_code(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_mbus_code_enum *code)
 {
+	const struct imx290 *imx290 = to_imx290(sd);
+
 	if (code->index >= ARRAY_SIZE(imx290_formats))
 		return -EINVAL;
 
-	code->code = imx290_formats[code->index].code;
+	code->code = imx290_formats[code->index].code[imx290->model->colour_variant];
 
 	return 0;
 }
@@ -859,7 +1152,7 @@ static int imx290_enum_frame_size(struct v4l2_subdev *sd,
 	const struct imx290 *imx290 = to_imx290(sd);
 	const struct imx290_mode *imx290_modes = imx290_modes_ptr(imx290);
 
-	if (!imx290_format_info(fse->code))
+	if (!imx290_format_info(imx290, fse->code))
 		return -EINVAL;
 
 	if (fse->index >= imx290_modes_num(imx290))
@@ -888,10 +1181,14 @@ static int imx290_set_fmt(struct v4l2_subdev *sd,
 	fmt->format.width = mode->width;
 	fmt->format.height = mode->height;
 
-	if (!imx290_format_info(fmt->format.code))
-		fmt->format.code = imx290_formats[0].code;
+	if (!imx290_format_info(imx290, fmt->format.code))
+		fmt->format.code = imx290_formats[0].code[imx290->model->colour_variant];
 
 	fmt->format.field = V4L2_FIELD_NONE;
+	fmt->format.colorspace = V4L2_COLORSPACE_RAW;
+	fmt->format.ycbcr_enc = V4L2_YCBCR_ENC_601;
+	fmt->format.quantization = V4L2_QUANTIZATION_FULL_RANGE;
+	fmt->format.xfer_func = V4L2_XFER_FUNC_NONE;
 
 	format = v4l2_subdev_get_pad_format(sd, sd_state, 0);
 
@@ -899,6 +1196,7 @@ static int imx290_set_fmt(struct v4l2_subdev *sd,
 		imx290->current_mode = mode;
 
 		imx290_ctrl_update(imx290, &fmt->format, mode);
+		imx290_exposure_update(imx290, mode);
 	}
 
 	*format = fmt->format;
@@ -910,16 +1208,23 @@ static int imx290_get_selection(struct v4l2_subdev *sd,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
+	struct imx290 *imx290 = to_imx290(sd);
 	struct v4l2_mbus_framefmt *format;
 
 	switch (sel->target) {
 	case V4L2_SEL_TGT_CROP: {
 		format = v4l2_subdev_get_pad_format(sd, sd_state, 0);
 
+		/*
+		 * The sensor moves the readout by 1 pixel based on flips to
+		 * keep the Bayer order the same.
+		 */
 		sel->r.top = IMX920_PIXEL_ARRAY_MARGIN_TOP
-			   + (IMX290_PIXEL_ARRAY_RECORDING_HEIGHT - format->height) / 2;
+			   + (IMX290_PIXEL_ARRAY_RECORDING_HEIGHT - format->height) / 2
+			   + imx290->vflip->val;
 		sel->r.left = IMX920_PIXEL_ARRAY_MARGIN_LEFT
-			    + (IMX290_PIXEL_ARRAY_RECORDING_WIDTH - format->width) / 2;
+			    + (IMX290_PIXEL_ARRAY_RECORDING_WIDTH - format->width) / 2
+			    + imx290->hflip->val;
 		sel->r.width = format->width;
 		sel->r.height = format->height;
 
@@ -964,6 +1269,11 @@ static int imx290_entity_init_cfg(struct v4l2_subdev *subdev,
 	return 0;
 }
 
+static const struct v4l2_subdev_core_ops imx290_core_ops = {
+	.subscribe_event = v4l2_ctrl_subdev_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
+};
+
 static const struct v4l2_subdev_video_ops imx290_video_ops = {
 	.s_stream = imx290_set_stream,
 };
@@ -978,6 +1288,7 @@ static const struct v4l2_subdev_pad_ops imx290_pad_ops = {
 };
 
 static const struct v4l2_subdev_ops imx290_subdev_ops = {
+	.core = &imx290_core_ops,
 	.video = &imx290_video_ops,
 	.pad = &imx290_pad_ops,
 };
@@ -996,7 +1307,8 @@ static int imx290_subdev_init(struct imx290 *imx290)
 	imx290->current_mode = &imx290_modes_ptr(imx290)[0];
 
 	v4l2_i2c_subdev_init(&imx290->sd, client, &imx290_subdev_ops);
-	imx290->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	imx290->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE |
+			    V4L2_SUBDEV_FL_HAS_EVENTS;
 	imx290->sd.dev = imx290->dev;
 	imx290->sd.entity.ops = &imx290_subdev_entity_ops;
 	imx290->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
@@ -1132,15 +1444,22 @@ static int imx290_init_clk(struct imx290 *imx290)
 	u32 xclk_freq;
 	int ret;
 
-	ret = fwnode_property_read_u32(dev_fwnode(imx290->dev),
-				       "clock-frequency", &xclk_freq);
+	ret = device_property_read_u32(imx290->dev, "clock-frequency",
+				       &xclk_freq);
 	if (ret) {
 		dev_err(imx290->dev, "Could not get xclk frequency\n");
 		return ret;
 	}
 
-	/* external clock must be 37.125 MHz */
-	if (xclk_freq != 37125000) {
+	/* external clock must be 37.125 MHz or 74.25MHz */
+	switch (xclk_freq) {
+	case 37125000:
+		imx290->xclk_idx = IMX290_CLK_37_125;
+		break;
+	case 74250000:
+		imx290->xclk_idx = IMX290_CLK_74_25;
+		break;
+	default:
 		dev_err(imx290->dev, "External clock frequency %u is not supported\n",
 			xclk_freq);
 		return -EINVAL;
@@ -1177,6 +1496,27 @@ static s64 imx290_check_link_freqs(const struct imx290 *imx290,
 	return 0;
 }
 
+static const struct imx290_model_info imx290_models[] = {
+	[IMX290_MODEL_IMX290LQR] = {
+		.colour_variant = IMX290_VARIANT_COLOUR,
+		.init_regs = imx290_global_init_settings_290,
+		.init_regs_num = ARRAY_SIZE(imx290_global_init_settings_290),
+		.name = "imx290",
+	},
+	[IMX290_MODEL_IMX290LLR] = {
+		.colour_variant = IMX290_VARIANT_MONO,
+		.init_regs = imx290_global_init_settings_290,
+		.init_regs_num = ARRAY_SIZE(imx290_global_init_settings_290),
+		.name = "imx290",
+	},
+	[IMX290_MODEL_IMX327LQR] = {
+		.colour_variant = IMX290_VARIANT_COLOUR,
+		.init_regs = imx290_global_init_settings_327,
+		.init_regs_num = ARRAY_SIZE(imx290_global_init_settings_327),
+		.name = "imx327",
+	},
+};
+
 static int imx290_parse_dt(struct imx290 *imx290)
 {
 	/* Only CSI2 is supported for now: */
@@ -1187,6 +1527,8 @@ static int imx290_parse_dt(struct imx290 *imx290)
 	int ret;
 	s64 fq;
 
+	imx290->model = of_device_get_match_data(imx290->dev);
+
 	endpoint = fwnode_graph_get_next_endpoint(dev_fwnode(imx290->dev), NULL);
 	if (!endpoint) {
 		dev_err(imx290->dev, "Endpoint node not found\n");
@@ -1260,7 +1602,7 @@ static int imx290_probe(struct i2c_client *client)
 	imx290->xclk = devm_clk_get(dev, "xclk");
 	if (IS_ERR(imx290->xclk))
 		return dev_err_probe(dev, PTR_ERR(imx290->xclk),
-				     "Could not get xclk");
+				     "Could not get xclk\n");
 
 	ret = imx290_get_regulators(dev, imx290);
 	if (ret < 0)
@@ -1304,6 +1646,9 @@ static int imx290_probe(struct i2c_client *client)
 	if (ret)
 		goto err_pm;
 
+	v4l2_i2c_subdev_set_name(&imx290->sd, client,
+				 imx290->model->name, NULL);
+
 	/*
 	 * Finally, register the V4L2 subdev. This must be done after
 	 * initializing everything as the subdev can be used immediately after
@@ -1352,8 +1697,21 @@ static void imx290_remove(struct i2c_client *client)
 }
 
 static const struct of_device_id imx290_of_match[] = {
-	{ .compatible = "sony,imx290" },
-	{ /* sentinel */ }
+	{
+		/* Deprecated - synonym for "sony,imx290lqr" */
+		.compatible = "sony,imx290",
+		.data = &imx290_models[IMX290_MODEL_IMX290LQR],
+	}, {
+		.compatible = "sony,imx290lqr",
+		.data = &imx290_models[IMX290_MODEL_IMX290LQR],
+	}, {
+		.compatible = "sony,imx290llr",
+		.data = &imx290_models[IMX290_MODEL_IMX290LLR],
+	}, {
+		.compatible = "sony,imx327lqr",
+		.data = &imx290_models[IMX290_MODEL_IMX327LQR],
+	},
+	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, imx290_of_match);
 
diff --git a/drivers/media/i2c/imx296.c b/drivers/media/i2c/imx296.c
index 3c12b6edeac9..4f22c0515ef8 100644
--- a/drivers/media/i2c/imx296.c
+++ b/drivers/media/i2c/imx296.c
@@ -685,15 +685,6 @@ static int imx296_enum_frame_size(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int imx296_get_format(struct v4l2_subdev *sd,
-			     struct v4l2_subdev_state *state,
-			     struct v4l2_subdev_format *fmt)
-{
-	fmt->format = *v4l2_subdev_get_pad_format(sd, state, fmt->pad);
-
-	return 0;
-}
-
 static int imx296_set_format(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *fmt)
@@ -845,7 +836,7 @@ static const struct v4l2_subdev_video_ops imx296_subdev_video_ops = {
 static const struct v4l2_subdev_pad_ops imx296_subdev_pad_ops = {
 	.enum_mbus_code = imx296_enum_mbus_code,
 	.enum_frame_size = imx296_enum_frame_size,
-	.get_fmt = imx296_get_format,
+	.get_fmt = v4l2_subdev_get_fmt,
 	.set_fmt = imx296_set_format,
 	.get_selection = imx296_get_selection,
 	.set_selection = imx296_set_selection,
diff --git a/drivers/media/i2c/imx334.c b/drivers/media/i2c/imx334.c
index 7b0a9086447d..309c706114d2 100644
--- a/drivers/media/i2c/imx334.c
+++ b/drivers/media/i2c/imx334.c
@@ -79,7 +79,6 @@ struct imx334_reg_list {
  * struct imx334_mode - imx334 sensor mode structure
  * @width: Frame width
  * @height: Frame height
- * @code: Format code
  * @hblank: Horizontal blanking in lines
  * @vblank: Vertical blanking in lines
  * @vblank_min: Minimal vertical blanking in lines
@@ -91,7 +90,6 @@ struct imx334_reg_list {
 struct imx334_mode {
 	u32 width;
 	u32 height;
-	u32 code;
 	u32 hblank;
 	u32 vblank;
 	u32 vblank_min;
@@ -119,6 +117,7 @@ struct imx334_mode {
  * @vblank: Vertical blanking in lines
  * @cur_mode: Pointer to current selected sensor mode
  * @mutex: Mutex for serializing sensor controls
+ * @cur_code: current selected format code
  * @streaming: Flag indicating streaming state
  */
 struct imx334 {
@@ -140,6 +139,7 @@ struct imx334 {
 	u32 vblank;
 	const struct imx334_mode *cur_mode;
 	struct mutex mutex;
+	u32 cur_code;
 	bool streaming;
 };
 
@@ -147,7 +147,170 @@ static const s64 link_freq[] = {
 	IMX334_LINK_FREQ,
 };
 
-/* Sensor mode registers */
+/* Sensor mode registers for 1920x1080@30fps */
+static const struct imx334_reg mode_1920x1080_regs[] = {
+	{0x3000, 0x01},
+	{0x3018, 0x04},
+	{0x3030, 0xca},
+	{0x3031, 0x08},
+	{0x3032, 0x00},
+	{0x3034, 0x4c},
+	{0x3035, 0x04},
+	{0x302c, 0xf0},
+	{0x302d, 0x03},
+	{0x302e, 0x80},
+	{0x302f, 0x07},
+	{0x3074, 0xcc},
+	{0x3075, 0x02},
+	{0x308e, 0xcd},
+	{0x308f, 0x02},
+	{0x3076, 0x38},
+	{0x3077, 0x04},
+	{0x3090, 0x38},
+	{0x3091, 0x04},
+	{0x3308, 0x38},
+	{0x3309, 0x04},
+	{0x30C6, 0x00},
+	{0x30c7, 0x00},
+	{0x30ce, 0x00},
+	{0x30cf, 0x00},
+	{0x30d8, 0x18},
+	{0x30d9, 0x0a},
+	{0x304c, 0x00},
+	{0x304e, 0x00},
+	{0x304f, 0x00},
+	{0x3050, 0x00},
+	{0x30b6, 0x00},
+	{0x30b7, 0x00},
+	{0x3116, 0x08},
+	{0x3117, 0x00},
+	{0x31a0, 0x20},
+	{0x31a1, 0x0f},
+	{0x300c, 0x3b},
+	{0x300d, 0x29},
+	{0x314c, 0x29},
+	{0x314d, 0x01},
+	{0x315a, 0x06},
+	{0x3168, 0xa0},
+	{0x316a, 0x7e},
+	{0x319e, 0x02},
+	{0x3199, 0x00},
+	{0x319d, 0x00},
+	{0x31dd, 0x03},
+	{0x3300, 0x00},
+	{0x341c, 0xff},
+	{0x341d, 0x01},
+	{0x3a01, 0x03},
+	{0x3a18, 0x7f},
+	{0x3a19, 0x00},
+	{0x3a1a, 0x37},
+	{0x3a1b, 0x00},
+	{0x3a1c, 0x37},
+	{0x3a1d, 0x00},
+	{0x3a1e, 0xf7},
+	{0x3a1f, 0x00},
+	{0x3a20, 0x3f},
+	{0x3a21, 0x00},
+	{0x3a20, 0x6f},
+	{0x3a21, 0x00},
+	{0x3a20, 0x3f},
+	{0x3a21, 0x00},
+	{0x3a20, 0x5f},
+	{0x3a21, 0x00},
+	{0x3a20, 0x2f},
+	{0x3a21, 0x00},
+	{0x3078, 0x02},
+	{0x3079, 0x00},
+	{0x307a, 0x00},
+	{0x307b, 0x00},
+	{0x3080, 0x02},
+	{0x3081, 0x00},
+	{0x3082, 0x00},
+	{0x3083, 0x00},
+	{0x3088, 0x02},
+	{0x3094, 0x00},
+	{0x3095, 0x00},
+	{0x3096, 0x00},
+	{0x309b, 0x02},
+	{0x309c, 0x00},
+	{0x309d, 0x00},
+	{0x309e, 0x00},
+	{0x30a4, 0x00},
+	{0x30a5, 0x00},
+	{0x3288, 0x21},
+	{0x328a, 0x02},
+	{0x3414, 0x05},
+	{0x3416, 0x18},
+	{0x35Ac, 0x0e},
+	{0x3648, 0x01},
+	{0x364a, 0x04},
+	{0x364c, 0x04},
+	{0x3678, 0x01},
+	{0x367c, 0x31},
+	{0x367e, 0x31},
+	{0x3708, 0x02},
+	{0x3714, 0x01},
+	{0x3715, 0x02},
+	{0x3716, 0x02},
+	{0x3717, 0x02},
+	{0x371c, 0x3d},
+	{0x371d, 0x3f},
+	{0x372c, 0x00},
+	{0x372d, 0x00},
+	{0x372e, 0x46},
+	{0x372f, 0x00},
+	{0x3730, 0x89},
+	{0x3731, 0x00},
+	{0x3732, 0x08},
+	{0x3733, 0x01},
+	{0x3734, 0xfe},
+	{0x3735, 0x05},
+	{0x375d, 0x00},
+	{0x375e, 0x00},
+	{0x375f, 0x61},
+	{0x3760, 0x06},
+	{0x3768, 0x1b},
+	{0x3769, 0x1b},
+	{0x376a, 0x1a},
+	{0x376b, 0x19},
+	{0x376c, 0x18},
+	{0x376d, 0x14},
+	{0x376e, 0x0f},
+	{0x3776, 0x00},
+	{0x3777, 0x00},
+	{0x3778, 0x46},
+	{0x3779, 0x00},
+	{0x377a, 0x08},
+	{0x377b, 0x01},
+	{0x377c, 0x45},
+	{0x377d, 0x01},
+	{0x377e, 0x23},
+	{0x377f, 0x02},
+	{0x3780, 0xd9},
+	{0x3781, 0x03},
+	{0x3782, 0xf5},
+	{0x3783, 0x06},
+	{0x3784, 0xa5},
+	{0x3788, 0x0f},
+	{0x378a, 0xd9},
+	{0x378b, 0x03},
+	{0x378c, 0xeb},
+	{0x378d, 0x05},
+	{0x378e, 0x87},
+	{0x378f, 0x06},
+	{0x3790, 0xf5},
+	{0x3792, 0x43},
+	{0x3794, 0x7a},
+	{0x3796, 0xa1},
+	{0x37b0, 0x37},
+	{0x3e04, 0x0e},
+	{0x30e8, 0x50},
+	{0x30e9, 0x00},
+	{0x3e04, 0x0e},
+	{0x3002, 0x00},
+};
+
+/* Sensor mode registers for 3840x2160@30fps */
 static const struct imx334_reg mode_3840x2160_regs[] = {
 	{0x3000, 0x01},
 	{0x3002, 0x00},
@@ -166,6 +329,7 @@ static const struct imx334_reg mode_3840x2160_regs[] = {
 	{0x3288, 0x21},
 	{0x328a, 0x02},
 	{0x302c, 0x3c},
+	{0x302d, 0x00},
 	{0x302e, 0x00},
 	{0x302f, 0x0f},
 	{0x3076, 0x70},
@@ -240,23 +404,75 @@ static const struct imx334_reg mode_3840x2160_regs[] = {
 	{0x3794, 0x7a},
 	{0x3796, 0xa1},
 	{0x3e04, 0x0e},
+	{0x319e, 0x00},
 	{0x3a00, 0x01},
+	{0x3a18, 0xbf},
+	{0x3a19, 0x00},
+	{0x3a1a, 0x67},
+	{0x3a1b, 0x00},
+	{0x3a1c, 0x6f},
+	{0x3a1d, 0x00},
+	{0x3a1e, 0xd7},
+	{0x3a1f, 0x01},
+	{0x3a20, 0x6f},
+	{0x3a21, 0x00},
+	{0x3a22, 0xcf},
+	{0x3a23, 0x00},
+	{0x3a24, 0x6f},
+	{0x3a25, 0x00},
+	{0x3a26, 0xb7},
+	{0x3a27, 0x00},
+	{0x3a28, 0x5f},
+	{0x3a29, 0x00},
+};
+
+static const struct imx334_reg raw10_framefmt_regs[] = {
+	{0x3050, 0x00},
+	{0x319d, 0x00},
+	{0x341c, 0xff},
+	{0x341d, 0x01},
+};
+
+static const struct imx334_reg raw12_framefmt_regs[] = {
+	{0x3050, 0x01},
+	{0x319d, 0x01},
+	{0x341c, 0x47},
+	{0x341d, 0x00},
+};
+
+static const u32 imx334_mbus_codes[] = {
+	MEDIA_BUS_FMT_SRGGB12_1X12,
+	MEDIA_BUS_FMT_SRGGB10_1X10,
 };
 
 /* Supported sensor mode configurations */
-static const struct imx334_mode supported_mode = {
-	.width = 3840,
-	.height = 2160,
-	.hblank = 560,
-	.vblank = 2340,
-	.vblank_min = 90,
-	.vblank_max = 132840,
-	.pclk = 594000000,
-	.link_freq_idx = 0,
-	.code = MEDIA_BUS_FMT_SRGGB12_1X12,
-	.reg_list = {
-		.num_of_regs = ARRAY_SIZE(mode_3840x2160_regs),
-		.regs = mode_3840x2160_regs,
+static const struct imx334_mode supported_modes[] = {
+	{
+		.width = 3840,
+		.height = 2160,
+		.hblank = 560,
+		.vblank = 2340,
+		.vblank_min = 90,
+		.vblank_max = 132840,
+		.pclk = 594000000,
+		.link_freq_idx = 0,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_3840x2160_regs),
+			.regs = mode_3840x2160_regs,
+		},
+	}, {
+		.width = 1920,
+		.height = 1080,
+		.hblank = 2480,
+		.vblank = 1170,
+		.vblank_min = 45,
+		.vblank_max = 132840,
+		.pclk = 297000000,
+		.link_freq_idx = 0,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_1920x1080_regs),
+			.regs = mode_1920x1080_regs,
+		},
 	},
 };
 
@@ -382,7 +598,8 @@ static int imx334_update_controls(struct imx334 *imx334,
 	if (ret)
 		return ret;
 
-	ret = __v4l2_ctrl_s_ctrl(imx334->hblank_ctrl, mode->hblank);
+	ret = __v4l2_ctrl_modify_range(imx334->hblank_ctrl, mode->hblank,
+				       mode->hblank, 1, mode->hblank);
 	if (ret)
 		return ret;
 
@@ -481,6 +698,9 @@ static int imx334_set_ctrl(struct v4l2_ctrl *ctrl)
 		pm_runtime_put(imx334->dev);
 
 		break;
+	case V4L2_CID_HBLANK:
+		ret = 0;
+		break;
 	default:
 		dev_err(imx334->dev, "Invalid control %d", ctrl->id);
 		ret = -EINVAL;
@@ -494,6 +714,18 @@ static const struct v4l2_ctrl_ops imx334_ctrl_ops = {
 	.s_ctrl = imx334_set_ctrl,
 };
 
+static int imx334_get_format_code(struct imx334 *imx334, u32 code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(imx334_mbus_codes); i++) {
+		if (imx334_mbus_codes[i] == code)
+			return imx334_mbus_codes[i];
+	}
+
+	return imx334_mbus_codes[0];
+}
+
 /**
  * imx334_enum_mbus_code() - Enumerate V4L2 sub-device mbus codes
  * @sd: pointer to imx334 V4L2 sub-device structure
@@ -506,10 +738,10 @@ static int imx334_enum_mbus_code(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_mbus_code_enum *code)
 {
-	if (code->index > 0)
+	if (code->index >= ARRAY_SIZE(imx334_mbus_codes))
 		return -EINVAL;
 
-	code->code = supported_mode.code;
+	code->code = imx334_mbus_codes[code->index];
 
 	return 0;
 }
@@ -526,15 +758,20 @@ static int imx334_enum_frame_size(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_frame_size_enum *fsize)
 {
-	if (fsize->index > 0)
+	struct imx334 *imx334 = to_imx334(sd);
+	u32 code;
+
+	if (fsize->index >= ARRAY_SIZE(supported_modes))
 		return -EINVAL;
 
-	if (fsize->code != supported_mode.code)
+	code = imx334_get_format_code(imx334, fsize->code);
+
+	if (fsize->code != code)
 		return -EINVAL;
 
-	fsize->min_width = supported_mode.width;
+	fsize->min_width = supported_modes[fsize->index].width;
 	fsize->max_width = fsize->min_width;
-	fsize->min_height = supported_mode.height;
+	fsize->min_height = supported_modes[fsize->index].height;
 	fsize->max_height = fsize->min_height;
 
 	return 0;
@@ -553,7 +790,6 @@ static void imx334_fill_pad_format(struct imx334 *imx334,
 {
 	fmt->format.width = mode->width;
 	fmt->format.height = mode->height;
-	fmt->format.code = mode->code;
 	fmt->format.field = V4L2_FIELD_NONE;
 	fmt->format.colorspace = V4L2_COLORSPACE_RAW;
 	fmt->format.ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
@@ -583,6 +819,7 @@ static int imx334_get_pad_format(struct v4l2_subdev *sd,
 		framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad);
 		fmt->format = *framefmt;
 	} else {
+		fmt->format.code = imx334->cur_code;
 		imx334_fill_pad_format(imx334, imx334->cur_mode, fmt);
 	}
 
@@ -609,15 +846,21 @@ static int imx334_set_pad_format(struct v4l2_subdev *sd,
 
 	mutex_lock(&imx334->mutex);
 
-	mode = &supported_mode;
+	mode = v4l2_find_nearest_size(supported_modes,
+				      ARRAY_SIZE(supported_modes),
+				      width, height,
+				      fmt->format.width, fmt->format.height);
+
 	imx334_fill_pad_format(imx334, mode, fmt);
+	fmt->format.code = imx334_get_format_code(imx334, fmt->format.code);
 
 	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
 		struct v4l2_mbus_framefmt *framefmt;
 
 		framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad);
 		*framefmt = fmt->format;
-	} else {
+	} else if (imx334->cur_mode != mode || imx334->cur_code != fmt->format.code) {
+		imx334->cur_code = fmt->format.code;
 		ret = imx334_update_controls(imx334, mode);
 		if (!ret)
 			imx334->cur_mode = mode;
@@ -642,11 +885,26 @@ static int imx334_init_pad_cfg(struct v4l2_subdev *sd,
 	struct v4l2_subdev_format fmt = { 0 };
 
 	fmt.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE;
-	imx334_fill_pad_format(imx334, &supported_mode, &fmt);
+	imx334_fill_pad_format(imx334, &supported_modes[0], &fmt);
 
 	return imx334_set_pad_format(sd, sd_state, &fmt);
 }
 
+static int imx334_set_framefmt(struct imx334 *imx334)
+{
+	switch (imx334->cur_code) {
+	case MEDIA_BUS_FMT_SRGGB10_1X10:
+		return imx334_write_regs(imx334, raw10_framefmt_regs,
+					 ARRAY_SIZE(raw10_framefmt_regs));
+
+	case MEDIA_BUS_FMT_SRGGB12_1X12:
+		return imx334_write_regs(imx334, raw12_framefmt_regs,
+					 ARRAY_SIZE(raw12_framefmt_regs));
+	}
+
+	return -EINVAL;
+}
+
 /**
  * imx334_start_streaming() - Start sensor stream
  * @imx334: pointer to imx334 device
@@ -667,6 +925,13 @@ static int imx334_start_streaming(struct imx334 *imx334)
 		return ret;
 	}
 
+	ret = imx334_set_framefmt(imx334);
+	if (ret) {
+		dev_err(imx334->dev, "%s failed to set frame format: %d\n",
+			__func__, ret);
+		return ret;
+	}
+
 	/* Setup handler will write actual exposure and gain */
 	ret =  __v4l2_ctrl_handler_setup(imx334->sd.ctrl_handler);
 	if (ret) {
@@ -1037,7 +1302,8 @@ static int imx334_probe(struct i2c_client *client)
 	}
 
 	/* Set default mode to max resolution */
-	imx334->cur_mode = &supported_mode;
+	imx334->cur_mode = &supported_modes[0];
+	imx334->cur_code = imx334_mbus_codes[0];
 	imx334->vblank = imx334->cur_mode->vblank;
 
 	ret = imx334_init_controls(imx334);
diff --git a/drivers/media/i2c/m5mols/Kconfig b/drivers/media/i2c/m5mols/Kconfig
deleted file mode 100644
index 7f0af32f4376..000000000000
--- a/drivers/media/i2c/m5mols/Kconfig
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config VIDEO_M5MOLS
-	tristate "Fujitsu M-5MOLS 8MP sensor support"
-	depends on I2C && VIDEO_DEV
-	select MEDIA_CONTROLLER
-	select VIDEO_V4L2_SUBDEV_API
-	help
-	  This driver supports Fujitsu M-5MOLS camera sensor with ISP
diff --git a/drivers/media/i2c/m5mols/Makefile b/drivers/media/i2c/m5mols/Makefile
deleted file mode 100644
index 13fa8ec29ac0..000000000000
--- a/drivers/media/i2c/m5mols/Makefile
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-m5mols-objs	:= m5mols_core.o m5mols_controls.o m5mols_capture.o
-
-obj-$(CONFIG_VIDEO_M5MOLS)		+= m5mols.o
diff --git a/drivers/media/i2c/m5mols/m5mols.h b/drivers/media/i2c/m5mols/m5mols.h
deleted file mode 100644
index d8545d2280af..000000000000
--- a/drivers/media/i2c/m5mols/m5mols.h
+++ /dev/null
@@ -1,349 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Header for M-5MOLS 8M Pixel camera sensor with ISP
- *
- * Copyright (C) 2011 Samsung Electronics Co., Ltd.
- * Author: HeungJun Kim <riverful.kim@samsung.com>
- *
- * Copyright (C) 2009 Samsung Electronics Co., Ltd.
- * Author: Dongsoo Nathaniel Kim <dongsoo45.kim@samsung.com>
- */
-
-#ifndef M5MOLS_H
-#define M5MOLS_H
-
-#include <linux/sizes.h>
-#include <linux/gpio/consumer.h>
-#include <media/v4l2-subdev.h>
-#include "m5mols_reg.h"
-
-
-/* An amount of data transmitted in addition to the value
- * determined by CAPP_JPEG_SIZE_MAX register.
- */
-#define M5MOLS_JPEG_TAGS_SIZE		0x20000
-#define M5MOLS_MAIN_JPEG_SIZE_MAX	(5 * SZ_1M)
-
-extern int m5mols_debug;
-
-enum m5mols_restype {
-	M5MOLS_RESTYPE_MONITOR,
-	M5MOLS_RESTYPE_CAPTURE,
-	M5MOLS_RESTYPE_MAX,
-};
-
-/**
- * struct m5mols_resolution - structure for the resolution
- * @type: resolution type according to the pixel code
- * @width: width of the resolution
- * @height: height of the resolution
- * @reg: resolution preset register value
- */
-struct m5mols_resolution {
-	u8 reg;
-	enum m5mols_restype type;
-	u16 width;
-	u16 height;
-};
-
-/**
- * struct m5mols_exif - structure for the EXIF information of M-5MOLS
- * @exposure_time: exposure time register value
- * @shutter_speed: speed of the shutter register value
- * @aperture: aperture register value
- * @brightness: brightness register value
- * @exposure_bias: it calls also EV bias
- * @iso_speed: ISO register value
- * @flash: status register value of the flash
- * @sdr: status register value of the Subject Distance Range
- * @qval: not written exact meaning in document
- */
-struct m5mols_exif {
-	u32 exposure_time;
-	u32 shutter_speed;
-	u32 aperture;
-	u32 brightness;
-	u32 exposure_bias;
-	u16 iso_speed;
-	u16 flash;
-	u16 sdr;
-	u16 qval;
-};
-
-/**
- * struct m5mols_capture - Structure for the capture capability
- * @exif: EXIF information
- * @buf_size: internal JPEG frame buffer size, in bytes
- * @main: size in bytes of the main image
- * @thumb: size in bytes of the thumb image, if it was accompanied
- * @total: total size in bytes of the produced image
- */
-struct m5mols_capture {
-	struct m5mols_exif exif;
-	unsigned int buf_size;
-	u32 main;
-	u32 thumb;
-	u32 total;
-};
-
-/**
- * struct m5mols_scenemode - structure for the scenemode capability
- * @metering: metering light register value
- * @ev_bias: EV bias register value
- * @wb_mode: mode which means the WhiteBalance is Auto or Manual
- * @wb_preset: whitebalance preset register value in the Manual mode
- * @chroma_en: register value whether the Chroma capability is enabled or not
- * @chroma_lvl: chroma's level register value
- * @edge_en: register value Whether the Edge capability is enabled or not
- * @edge_lvl: edge's level register value
- * @af_range: Auto Focus's range
- * @fd_mode: Face Detection mode
- * @mcc: Multi-axis Color Conversion which means emotion color
- * @light: status of the Light
- * @flash: status of the Flash
- * @tone: Tone color which means Contrast
- * @iso: ISO register value
- * @capt_mode: Mode of the Image Stabilization while the camera capturing
- * @wdr: Wide Dynamic Range register value
- *
- * The each value according to each scenemode is recommended in the documents.
- */
-struct m5mols_scenemode {
-	u8 metering;
-	u8 ev_bias;
-	u8 wb_mode;
-	u8 wb_preset;
-	u8 chroma_en;
-	u8 chroma_lvl;
-	u8 edge_en;
-	u8 edge_lvl;
-	u8 af_range;
-	u8 fd_mode;
-	u8 mcc;
-	u8 light;
-	u8 flash;
-	u8 tone;
-	u8 iso;
-	u8 capt_mode;
-	u8 wdr;
-};
-
-#define VERSION_STRING_SIZE	22
-
-/**
- * struct m5mols_version - firmware version information
- * @customer:	customer information
- * @project:	version of project information according to customer
- * @fw:		firmware revision
- * @hw:		hardware revision
- * @param:	version of the parameter
- * @awb:	Auto WhiteBalance algorithm version
- * @str:	information about manufacturer and packaging vendor
- * @af:		Auto Focus version
- *
- * The register offset starts the customer version at 0x0, and it ends
- * the awb version at 0x09. The customer, project information occupies 1 bytes
- * each. And also the fw, hw, param, awb each requires 2 bytes. The str is
- * unique string associated with firmware's version. It includes information
- * about manufacturer and the vendor of the sensor's packaging. The least
- * significant 2 bytes of the string indicate packaging manufacturer.
- */
-struct m5mols_version {
-	u8	customer;
-	u8	project;
-	u16	fw;
-	u16	hw;
-	u16	param;
-	u16	awb;
-	u8	str[VERSION_STRING_SIZE];
-	u8	af;
-};
-
-/**
- * struct m5mols_info - M-5MOLS driver data structure
- * @pdata: platform data
- * @sd: v4l-subdev instance
- * @pad: media pad
- * @irq_waitq: waitqueue for the capture
- * @irq_done: set to 1 in the interrupt handler
- * @handle: control handler
- * @auto_exposure: auto/manual exposure control
- * @exposure_bias: exposure compensation control
- * @exposure: manual exposure control
- * @metering: exposure metering control
- * @auto_iso: auto/manual ISO sensitivity control
- * @iso: manual ISO sensitivity control
- * @auto_wb: auto white balance control
- * @lock_3a: 3A lock control
- * @colorfx: color effect control
- * @saturation: saturation control
- * @zoom: zoom control
- * @wdr: wide dynamic range control
- * @stabilization: image stabilization control
- * @jpeg_quality: JPEG compression quality control
- * @set_power: optional power callback to the board code
- * @reset: GPIO driving the reset pin of M-5MOLS
- * @lock: mutex protecting the structure fields below
- * @ffmt: current fmt according to resolution type
- * @res_type: current resolution type
- * @ver: information of the version
- * @cap: the capture mode attributes
- * @isp_ready: 1 when the ISP controller has completed booting
- * @power: current sensor's power status
- * @ctrl_sync: 1 when the control handler state is restored in H/W
- * @resolution:	register value for current resolution
- * @mode: register value for current operation mode
- */
-struct m5mols_info {
-	const struct m5mols_platform_data *pdata;
-	struct v4l2_subdev sd;
-	struct media_pad pad;
-
-	wait_queue_head_t irq_waitq;
-	atomic_t irq_done;
-
-	struct v4l2_ctrl_handler handle;
-	struct {
-		/* exposure/exposure bias/auto exposure cluster */
-		struct v4l2_ctrl *auto_exposure;
-		struct v4l2_ctrl *exposure_bias;
-		struct v4l2_ctrl *exposure;
-		struct v4l2_ctrl *metering;
-	};
-	struct {
-		/* iso/auto iso cluster */
-		struct v4l2_ctrl *auto_iso;
-		struct v4l2_ctrl *iso;
-	};
-	struct v4l2_ctrl *auto_wb;
-
-	struct v4l2_ctrl *lock_3a;
-	struct v4l2_ctrl *colorfx;
-	struct v4l2_ctrl *saturation;
-	struct v4l2_ctrl *zoom;
-	struct v4l2_ctrl *wdr;
-	struct v4l2_ctrl *stabilization;
-	struct v4l2_ctrl *jpeg_quality;
-
-	int (*set_power)(struct device *dev, int on);
-	struct gpio_desc *reset;
-
-	struct mutex lock;
-
-	struct v4l2_mbus_framefmt ffmt[M5MOLS_RESTYPE_MAX];
-	int res_type;
-
-	struct m5mols_version ver;
-	struct m5mols_capture cap;
-
-	unsigned int isp_ready:1;
-	unsigned int power:1;
-	unsigned int ctrl_sync:1;
-
-	u8 resolution;
-	u8 mode;
-};
-
-#define is_available_af(__info)	(__info->ver.af)
-#define is_code(__code, __type) (__code == m5mols_default_ffmt[__type].code)
-#define is_manufacturer(__info, __manufacturer)	\
-				(__info->ver.str[0] == __manufacturer[0] && \
-				 __info->ver.str[1] == __manufacturer[1])
-/*
- * I2C operation of the M-5MOLS
- *
- * The I2C read operation of the M-5MOLS requires 2 messages. The first
- * message sends the information about the command, command category, and total
- * message size. The second message is used to retrieve the data specified in
- * the first message
- *
- *   1st message                                2nd message
- *   +-------+---+----------+-----+-------+     +------+------+------+------+
- *   | size1 | R | category | cmd | size2 |     | d[0] | d[1] | d[2] | d[3] |
- *   +-------+---+----------+-----+-------+     +------+------+------+------+
- *   - size1: message data size(5 in this case)
- *   - size2: desired buffer size of the 2nd message
- *   - d[0..3]: according to size2
- *
- * The I2C write operation needs just one message. The message includes
- * category, command, total size, and desired data.
- *
- *   1st message
- *   +-------+---+----------+-----+------+------+------+------+
- *   | size1 | W | category | cmd | d[0] | d[1] | d[2] | d[3] |
- *   +-------+---+----------+-----+------+------+------+------+
- *   - d[0..3]: according to size1
- */
-int m5mols_read_u8(struct v4l2_subdev *sd, u32 reg_comb, u8 *val);
-int m5mols_read_u16(struct v4l2_subdev *sd, u32 reg_comb, u16 *val);
-int m5mols_read_u32(struct v4l2_subdev *sd, u32 reg_comb, u32 *val);
-int m5mols_write(struct v4l2_subdev *sd, u32 reg_comb, u32 val);
-
-int m5mols_busy_wait(struct v4l2_subdev *sd, u32 reg, u32 value, u32 mask,
-		     int timeout);
-
-/* Mask value for busy waiting until M-5MOLS I2C interface is initialized */
-#define M5MOLS_I2C_RDY_WAIT_FL		(1 << 16)
-/* ISP state transition timeout, in ms */
-#define M5MOLS_MODE_CHANGE_TIMEOUT	200
-#define M5MOLS_BUSY_WAIT_DEF_TIMEOUT	250
-
-/*
- * Mode operation of the M-5MOLS
- *
- * Changing the mode of the M-5MOLS is needed right executing order.
- * There are three modes(PARAMETER, MONITOR, CAPTURE) which can be changed
- * by user. There are various categories associated with each mode.
- *
- * +============================================================+
- * | mode	| category					|
- * +============================================================+
- * | FLASH	| FLASH(only after Stand-by or Power-on)	|
- * | SYSTEM	| SYSTEM(only after sensor arm-booting)		|
- * | PARAMETER	| PARAMETER					|
- * | MONITOR	| MONITOR(preview), Auto Focus, Face Detection	|
- * | CAPTURE	| Single CAPTURE, Preview(recording)		|
- * +============================================================+
- *
- * The available executing order between each modes are as follows:
- *   PARAMETER <---> MONITOR <---> CAPTURE
- */
-int m5mols_set_mode(struct m5mols_info *info, u8 mode);
-
-int m5mols_enable_interrupt(struct v4l2_subdev *sd, u8 reg);
-int m5mols_wait_interrupt(struct v4l2_subdev *sd, u8 condition, u32 timeout);
-int m5mols_restore_controls(struct m5mols_info *info);
-int m5mols_start_capture(struct m5mols_info *info);
-int m5mols_do_scenemode(struct m5mols_info *info, u8 mode);
-int m5mols_lock_3a(struct m5mols_info *info, bool lock);
-int m5mols_set_ctrl(struct v4l2_ctrl *ctrl);
-int m5mols_init_controls(struct v4l2_subdev *sd);
-
-/* The firmware function */
-int m5mols_update_fw(struct v4l2_subdev *sd,
-		     int (*set_power)(struct m5mols_info *, bool));
-
-static inline struct m5mols_info *to_m5mols(struct v4l2_subdev *subdev)
-{
-	return container_of(subdev, struct m5mols_info, sd);
-}
-
-static inline struct v4l2_subdev *to_sd(struct v4l2_ctrl *ctrl)
-{
-	struct m5mols_info *info = container_of(ctrl->handler,
-						struct m5mols_info, handle);
-	return &info->sd;
-}
-
-static inline void m5mols_set_ctrl_mode(struct v4l2_ctrl *ctrl,
-					unsigned int mode)
-{
-	ctrl->priv = (void *)(uintptr_t)mode;
-}
-
-static inline unsigned int m5mols_get_ctrl_mode(struct v4l2_ctrl *ctrl)
-{
-	return (unsigned int)(uintptr_t)ctrl->priv;
-}
-
-#endif	/* M5MOLS_H */
diff --git a/drivers/media/i2c/m5mols/m5mols_capture.c b/drivers/media/i2c/m5mols/m5mols_capture.c
deleted file mode 100644
index 275c5b2539fd..000000000000
--- a/drivers/media/i2c/m5mols/m5mols_capture.c
+++ /dev/null
@@ -1,158 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-/*
- * The Capture code for Fujitsu M-5MOLS ISP
- *
- * Copyright (C) 2011 Samsung Electronics Co., Ltd.
- * Author: HeungJun Kim <riverful.kim@samsung.com>
- *
- * Copyright (C) 2009 Samsung Electronics Co., Ltd.
- * Author: Dongsoo Nathaniel Kim <dongsoo45.kim@samsung.com>
- */
-
-#include <linux/i2c.h>
-#include <linux/slab.h>
-#include <linux/irq.h>
-#include <linux/interrupt.h>
-#include <linux/delay.h>
-#include <linux/regulator/consumer.h>
-#include <linux/videodev2.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-subdev.h>
-#include <media/i2c/m5mols.h>
-#include <media/drv-intf/exynos-fimc.h>
-
-#include "m5mols.h"
-#include "m5mols_reg.h"
-
-/**
- * m5mols_read_rational - I2C read of a rational number
- * @sd: sub-device, as pointed by struct v4l2_subdev
- * @addr_num: numerator register
- * @addr_den: denominator register
- * @val: place to store the division result
- *
- * Read numerator and denominator from registers @addr_num and @addr_den
- * respectively and return the division result in @val.
- */
-static int m5mols_read_rational(struct v4l2_subdev *sd, u32 addr_num,
-				u32 addr_den, u32 *val)
-{
-	u32 num, den;
-
-	int ret = m5mols_read_u32(sd, addr_num, &num);
-	if (!ret)
-		ret = m5mols_read_u32(sd, addr_den, &den);
-	if (ret)
-		return ret;
-	*val = den == 0 ? 0 : num / den;
-	return ret;
-}
-
-/**
- * m5mols_capture_info - Gather captured image information
- * @info: M-5MOLS driver data structure
- *
- * For now it gathers only EXIF information and file size.
- */
-static int m5mols_capture_info(struct m5mols_info *info)
-{
-	struct m5mols_exif *exif = &info->cap.exif;
-	struct v4l2_subdev *sd = &info->sd;
-	int ret;
-
-	ret = m5mols_read_rational(sd, EXIF_INFO_EXPTIME_NU,
-				   EXIF_INFO_EXPTIME_DE, &exif->exposure_time);
-	if (ret)
-		return ret;
-	ret = m5mols_read_rational(sd, EXIF_INFO_TV_NU, EXIF_INFO_TV_DE,
-				   &exif->shutter_speed);
-	if (ret)
-		return ret;
-	ret = m5mols_read_rational(sd, EXIF_INFO_AV_NU, EXIF_INFO_AV_DE,
-				   &exif->aperture);
-	if (ret)
-		return ret;
-	ret = m5mols_read_rational(sd, EXIF_INFO_BV_NU, EXIF_INFO_BV_DE,
-				   &exif->brightness);
-	if (ret)
-		return ret;
-	ret = m5mols_read_rational(sd, EXIF_INFO_EBV_NU, EXIF_INFO_EBV_DE,
-				   &exif->exposure_bias);
-	if (ret)
-		return ret;
-
-	ret = m5mols_read_u16(sd, EXIF_INFO_ISO, &exif->iso_speed);
-	if (!ret)
-		ret = m5mols_read_u16(sd, EXIF_INFO_FLASH, &exif->flash);
-	if (!ret)
-		ret = m5mols_read_u16(sd, EXIF_INFO_SDR, &exif->sdr);
-	if (!ret)
-		ret = m5mols_read_u16(sd, EXIF_INFO_QVAL, &exif->qval);
-	if (ret)
-		return ret;
-
-	if (!ret)
-		ret = m5mols_read_u32(sd, CAPC_IMAGE_SIZE, &info->cap.main);
-	if (!ret)
-		ret = m5mols_read_u32(sd, CAPC_THUMB_SIZE, &info->cap.thumb);
-	if (!ret)
-		info->cap.total = info->cap.main + info->cap.thumb;
-
-	return ret;
-}
-
-int m5mols_start_capture(struct m5mols_info *info)
-{
-	unsigned int framesize = info->cap.buf_size - M5MOLS_JPEG_TAGS_SIZE;
-	struct v4l2_subdev *sd = &info->sd;
-	int ret;
-
-	/*
-	 * Synchronize the controls, set the capture frame resolution and color
-	 * format. The frame capture is initiated during switching from Monitor
-	 * to Capture mode.
-	 */
-	ret = m5mols_set_mode(info, REG_MONITOR);
-	if (!ret)
-		ret = m5mols_restore_controls(info);
-	if (!ret)
-		ret = m5mols_write(sd, CAPP_YUVOUT_MAIN, REG_JPEG);
-	if (!ret)
-		ret = m5mols_write(sd, CAPP_MAIN_IMAGE_SIZE, info->resolution);
-	if (!ret)
-		ret = m5mols_write(sd, CAPP_JPEG_SIZE_MAX, framesize);
-	if (!ret)
-		ret = m5mols_set_mode(info, REG_CAPTURE);
-	if (!ret)
-		/* Wait until a frame is captured to ISP internal memory */
-		ret = m5mols_wait_interrupt(sd, REG_INT_CAPTURE, 2000);
-	if (ret)
-		return ret;
-
-	/*
-	 * Initiate the captured data transfer to a MIPI-CSI receiver.
-	 */
-	ret = m5mols_write(sd, CAPC_SEL_FRAME, 1);
-	if (!ret)
-		ret = m5mols_write(sd, CAPC_START, REG_CAP_START_MAIN);
-	if (!ret) {
-		bool captured = false;
-		unsigned int size;
-
-		/* Wait for the capture completion interrupt */
-		ret = m5mols_wait_interrupt(sd, REG_INT_CAPTURE, 2000);
-		if (!ret) {
-			captured = true;
-			ret = m5mols_capture_info(info);
-		}
-		size = captured ? info->cap.main : 0;
-		v4l2_dbg(1, m5mols_debug, sd, "%s: size: %d, thumb.: %d B\n",
-			 __func__, size, info->cap.thumb);
-
-		v4l2_subdev_notify(sd, S5P_FIMC_TX_END_NOTIFY, &size);
-	}
-
-	return ret;
-}
diff --git a/drivers/media/i2c/m5mols/m5mols_controls.c b/drivers/media/i2c/m5mols/m5mols_controls.c
deleted file mode 100644
index b45e0e08b6c8..000000000000
--- a/drivers/media/i2c/m5mols/m5mols_controls.c
+++ /dev/null
@@ -1,625 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Controls for M-5MOLS 8M Pixel camera sensor with ISP
- *
- * Copyright (C) 2011 Samsung Electronics Co., Ltd.
- * Author: HeungJun Kim <riverful.kim@samsung.com>
- *
- * Copyright (C) 2009 Samsung Electronics Co., Ltd.
- * Author: Dongsoo Nathaniel Kim <dongsoo45.kim@samsung.com>
- */
-
-#include <linux/i2c.h>
-#include <linux/delay.h>
-#include <linux/videodev2.h>
-#include <media/v4l2-ctrls.h>
-
-#include "m5mols.h"
-#include "m5mols_reg.h"
-
-static struct m5mols_scenemode m5mols_default_scenemode[] = {
-	[REG_SCENE_NORMAL] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 3, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_NORMAL, REG_LIGHT_OFF, REG_FLASH_OFF,
-		5, REG_ISO_AUTO, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_PORTRAIT] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 3, REG_EDGE_ON, 4,
-		REG_AF_NORMAL, BIT_FD_EN | BIT_FD_DRAW_FACE_FRAME,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_AUTO, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_LANDSCAPE] = {
-		REG_AE_ALL, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 4, REG_EDGE_ON, 6,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_AUTO, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_SPORTS] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 3, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_AUTO, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_PARTY_INDOOR] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 4, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_200, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_BEACH_SNOW] = {
-		REG_AE_CENTER, REG_AE_INDEX_10_POS, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 4, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_50, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_SUNSET] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_PRESET,
-		REG_AWB_DAYLIGHT,
-		REG_CHROMA_ON, 3, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_AUTO, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_DAWN_DUSK] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_PRESET,
-		REG_AWB_FLUORESCENT_1,
-		REG_CHROMA_ON, 3, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_AUTO, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_FALL] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 5, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_AUTO, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_NIGHT] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 3, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_AUTO, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_AGAINST_LIGHT] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 3, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_AUTO, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_FIRE] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 3, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_50, REG_CAP_NONE, REG_WDR_OFF,
-	},
-	[REG_SCENE_TEXT] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 3, REG_EDGE_ON, 7,
-		REG_AF_MACRO, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_AUTO, REG_CAP_ANTI_SHAKE, REG_WDR_ON,
-	},
-	[REG_SCENE_CANDLE] = {
-		REG_AE_CENTER, REG_AE_INDEX_00, REG_AWB_AUTO, 0,
-		REG_CHROMA_ON, 3, REG_EDGE_ON, 5,
-		REG_AF_NORMAL, REG_FD_OFF,
-		REG_MCC_OFF, REG_LIGHT_OFF, REG_FLASH_OFF,
-		6, REG_ISO_AUTO, REG_CAP_NONE, REG_WDR_OFF,
-	},
-};
-
-/**
- * m5mols_do_scenemode() - Change current scenemode
- * @info: M-5MOLS driver data structure
- * @mode:	Desired mode of the scenemode
- *
- * WARNING: The execution order is important. Do not change the order.
- */
-int m5mols_do_scenemode(struct m5mols_info *info, u8 mode)
-{
-	struct v4l2_subdev *sd = &info->sd;
-	struct m5mols_scenemode scenemode = m5mols_default_scenemode[mode];
-	int ret;
-
-	if (mode > REG_SCENE_CANDLE)
-		return -EINVAL;
-
-	ret = v4l2_ctrl_s_ctrl(info->lock_3a, 0);
-	if (!ret)
-		ret = m5mols_write(sd, AE_EV_PRESET_MONITOR, mode);
-	if (!ret)
-		ret = m5mols_write(sd, AE_EV_PRESET_CAPTURE, mode);
-	if (!ret)
-		ret = m5mols_write(sd, AE_MODE, scenemode.metering);
-	if (!ret)
-		ret = m5mols_write(sd, AE_INDEX, scenemode.ev_bias);
-	if (!ret)
-		ret = m5mols_write(sd, AWB_MODE, scenemode.wb_mode);
-	if (!ret)
-		ret = m5mols_write(sd, AWB_MANUAL, scenemode.wb_preset);
-	if (!ret)
-		ret = m5mols_write(sd, MON_CHROMA_EN, scenemode.chroma_en);
-	if (!ret)
-		ret = m5mols_write(sd, MON_CHROMA_LVL, scenemode.chroma_lvl);
-	if (!ret)
-		ret = m5mols_write(sd, MON_EDGE_EN, scenemode.edge_en);
-	if (!ret)
-		ret = m5mols_write(sd, MON_EDGE_LVL, scenemode.edge_lvl);
-	if (!ret && is_available_af(info))
-		ret = m5mols_write(sd, AF_MODE, scenemode.af_range);
-	if (!ret && is_available_af(info))
-		ret = m5mols_write(sd, FD_CTL, scenemode.fd_mode);
-	if (!ret)
-		ret = m5mols_write(sd, MON_TONE_CTL, scenemode.tone);
-	if (!ret)
-		ret = m5mols_write(sd, AE_ISO, scenemode.iso);
-	if (!ret)
-		ret = m5mols_set_mode(info, REG_CAPTURE);
-	if (!ret)
-		ret = m5mols_write(sd, CAPP_WDR_EN, scenemode.wdr);
-	if (!ret)
-		ret = m5mols_write(sd, CAPP_MCC_MODE, scenemode.mcc);
-	if (!ret)
-		ret = m5mols_write(sd, CAPP_LIGHT_CTRL, scenemode.light);
-	if (!ret)
-		ret = m5mols_write(sd, CAPP_FLASH_CTRL, scenemode.flash);
-	if (!ret)
-		ret = m5mols_write(sd, CAPC_MODE, scenemode.capt_mode);
-	if (!ret)
-		ret = m5mols_set_mode(info, REG_MONITOR);
-
-	return ret;
-}
-
-static int m5mols_3a_lock(struct m5mols_info *info, struct v4l2_ctrl *ctrl)
-{
-	bool af_lock = ctrl->val & V4L2_LOCK_FOCUS;
-	int ret = 0;
-
-	if ((ctrl->val ^ ctrl->cur.val) & V4L2_LOCK_EXPOSURE) {
-		bool ae_lock = ctrl->val & V4L2_LOCK_EXPOSURE;
-
-		ret = m5mols_write(&info->sd, AE_LOCK, ae_lock ?
-				   REG_AE_LOCK : REG_AE_UNLOCK);
-		if (ret)
-			return ret;
-	}
-
-	if (((ctrl->val ^ ctrl->cur.val) & V4L2_LOCK_WHITE_BALANCE)
-	    && info->auto_wb->val) {
-		bool awb_lock = ctrl->val & V4L2_LOCK_WHITE_BALANCE;
-
-		ret = m5mols_write(&info->sd, AWB_LOCK, awb_lock ?
-				   REG_AWB_LOCK : REG_AWB_UNLOCK);
-		if (ret)
-			return ret;
-	}
-
-	if (!info->ver.af || !af_lock)
-		return ret;
-
-	if ((ctrl->val ^ ctrl->cur.val) & V4L2_LOCK_FOCUS)
-		ret = m5mols_write(&info->sd, AF_EXECUTE, REG_AF_STOP);
-
-	return ret;
-}
-
-static int m5mols_set_metering_mode(struct m5mols_info *info, int mode)
-{
-	unsigned int metering;
-
-	switch (mode) {
-	case V4L2_EXPOSURE_METERING_CENTER_WEIGHTED:
-		metering = REG_AE_CENTER;
-		break;
-	case V4L2_EXPOSURE_METERING_SPOT:
-		metering = REG_AE_SPOT;
-		break;
-	default:
-		metering = REG_AE_ALL;
-		break;
-	}
-
-	return m5mols_write(&info->sd, AE_MODE, metering);
-}
-
-static int m5mols_set_exposure(struct m5mols_info *info, int exposure)
-{
-	struct v4l2_subdev *sd = &info->sd;
-	int ret = 0;
-
-	if (exposure == V4L2_EXPOSURE_AUTO) {
-		/* Unlock auto exposure */
-		info->lock_3a->val &= ~V4L2_LOCK_EXPOSURE;
-		m5mols_3a_lock(info, info->lock_3a);
-
-		ret = m5mols_set_metering_mode(info, info->metering->val);
-		if (ret < 0)
-			return ret;
-
-		v4l2_dbg(1, m5mols_debug, sd,
-			 "%s: exposure bias: %#x, metering: %#x\n",
-			 __func__, info->exposure_bias->val,
-			 info->metering->val);
-
-		return m5mols_write(sd, AE_INDEX, info->exposure_bias->val);
-	}
-
-	if (exposure == V4L2_EXPOSURE_MANUAL) {
-		ret = m5mols_write(sd, AE_MODE, REG_AE_OFF);
-		if (ret == 0)
-			ret = m5mols_write(sd, AE_MAN_GAIN_MON,
-					   info->exposure->val);
-		if (ret == 0)
-			ret = m5mols_write(sd, AE_MAN_GAIN_CAP,
-					   info->exposure->val);
-
-		v4l2_dbg(1, m5mols_debug, sd, "%s: exposure: %#x\n",
-			 __func__, info->exposure->val);
-	}
-
-	return ret;
-}
-
-static int m5mols_set_white_balance(struct m5mols_info *info, int val)
-{
-	static const unsigned short wb[][2] = {
-		{ V4L2_WHITE_BALANCE_INCANDESCENT,  REG_AWB_INCANDESCENT },
-		{ V4L2_WHITE_BALANCE_FLUORESCENT,   REG_AWB_FLUORESCENT_1 },
-		{ V4L2_WHITE_BALANCE_FLUORESCENT_H, REG_AWB_FLUORESCENT_2 },
-		{ V4L2_WHITE_BALANCE_HORIZON,       REG_AWB_HORIZON },
-		{ V4L2_WHITE_BALANCE_DAYLIGHT,      REG_AWB_DAYLIGHT },
-		{ V4L2_WHITE_BALANCE_FLASH,         REG_AWB_LEDLIGHT },
-		{ V4L2_WHITE_BALANCE_CLOUDY,        REG_AWB_CLOUDY },
-		{ V4L2_WHITE_BALANCE_SHADE,         REG_AWB_SHADE },
-		{ V4L2_WHITE_BALANCE_AUTO,          REG_AWB_AUTO },
-	};
-	int i;
-	struct v4l2_subdev *sd = &info->sd;
-	int ret = -EINVAL;
-
-	for (i = 0; i < ARRAY_SIZE(wb); i++) {
-		int awb;
-		if (wb[i][0] != val)
-			continue;
-
-		v4l2_dbg(1, m5mols_debug, sd,
-			 "Setting white balance to: %#x\n", wb[i][0]);
-
-		awb = wb[i][0] == V4L2_WHITE_BALANCE_AUTO;
-		ret = m5mols_write(sd, AWB_MODE, awb ? REG_AWB_AUTO :
-						 REG_AWB_PRESET);
-		if (ret < 0)
-			return ret;
-
-		if (!awb)
-			ret = m5mols_write(sd, AWB_MANUAL, wb[i][1]);
-	}
-
-	return ret;
-}
-
-static int m5mols_set_saturation(struct m5mols_info *info, int val)
-{
-	int ret = m5mols_write(&info->sd, MON_CHROMA_LVL, val);
-	if (ret < 0)
-		return ret;
-
-	return m5mols_write(&info->sd, MON_CHROMA_EN, REG_CHROMA_ON);
-}
-
-static int m5mols_set_color_effect(struct m5mols_info *info, int val)
-{
-	unsigned int m_effect = REG_COLOR_EFFECT_OFF;
-	unsigned int p_effect = REG_EFFECT_OFF;
-	unsigned int cfix_r = 0, cfix_b = 0;
-	struct v4l2_subdev *sd = &info->sd;
-	int ret = 0;
-
-	switch (val) {
-	case V4L2_COLORFX_BW:
-		m_effect = REG_COLOR_EFFECT_ON;
-		break;
-	case V4L2_COLORFX_NEGATIVE:
-		p_effect = REG_EFFECT_NEGA;
-		break;
-	case V4L2_COLORFX_EMBOSS:
-		p_effect = REG_EFFECT_EMBOSS;
-		break;
-	case V4L2_COLORFX_SEPIA:
-		m_effect = REG_COLOR_EFFECT_ON;
-		cfix_r = REG_CFIXR_SEPIA;
-		cfix_b = REG_CFIXB_SEPIA;
-		break;
-	}
-
-	ret = m5mols_write(sd, PARM_EFFECT, p_effect);
-	if (!ret)
-		ret = m5mols_write(sd, MON_EFFECT, m_effect);
-
-	if (ret == 0 && m_effect == REG_COLOR_EFFECT_ON) {
-		ret = m5mols_write(sd, MON_CFIXR, cfix_r);
-		if (!ret)
-			ret = m5mols_write(sd, MON_CFIXB, cfix_b);
-	}
-
-	v4l2_dbg(1, m5mols_debug, sd,
-		 "p_effect: %#x, m_effect: %#x, r: %#x, b: %#x (%d)\n",
-		 p_effect, m_effect, cfix_r, cfix_b, ret);
-
-	return ret;
-}
-
-static int m5mols_set_iso(struct m5mols_info *info, int auto_iso)
-{
-	u32 iso = auto_iso ? 0 : info->iso->val + 1;
-
-	return m5mols_write(&info->sd, AE_ISO, iso);
-}
-
-static int m5mols_set_wdr(struct m5mols_info *info, int wdr)
-{
-	int ret;
-
-	ret = m5mols_write(&info->sd, MON_TONE_CTL, wdr ? 9 : 5);
-	if (ret < 0)
-		return ret;
-
-	ret = m5mols_set_mode(info, REG_CAPTURE);
-	if (ret < 0)
-		return ret;
-
-	return m5mols_write(&info->sd, CAPP_WDR_EN, wdr);
-}
-
-static int m5mols_set_stabilization(struct m5mols_info *info, int val)
-{
-	struct v4l2_subdev *sd = &info->sd;
-	unsigned int evp = val ? 0xe : 0x0;
-	int ret;
-
-	ret = m5mols_write(sd, AE_EV_PRESET_MONITOR, evp);
-	if (ret < 0)
-		return ret;
-
-	return m5mols_write(sd, AE_EV_PRESET_CAPTURE, evp);
-}
-
-static int m5mols_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct v4l2_subdev *sd = to_sd(ctrl);
-	struct m5mols_info *info = to_m5mols(sd);
-	int ret = 0;
-	u8 status = REG_ISO_AUTO;
-
-	v4l2_dbg(1, m5mols_debug, sd, "%s: ctrl: %s (%d)\n",
-		 __func__, ctrl->name, info->isp_ready);
-
-	if (!info->isp_ready)
-		return -EBUSY;
-
-	switch (ctrl->id) {
-	case V4L2_CID_ISO_SENSITIVITY_AUTO:
-		ret = m5mols_read_u8(sd, AE_ISO, &status);
-		if (ret == 0)
-			ctrl->val = !status;
-		if (status != REG_ISO_AUTO)
-			info->iso->val = status - 1;
-		break;
-
-	case V4L2_CID_3A_LOCK:
-		ctrl->val &= ~0x7;
-
-		ret = m5mols_read_u8(sd, AE_LOCK, &status);
-		if (ret)
-			return ret;
-		if (status)
-			info->lock_3a->val |= V4L2_LOCK_EXPOSURE;
-
-		ret = m5mols_read_u8(sd, AWB_LOCK, &status);
-		if (ret)
-			return ret;
-		if (status)
-			info->lock_3a->val |= V4L2_LOCK_EXPOSURE;
-
-		ret = m5mols_read_u8(sd, AF_EXECUTE, &status);
-		if (!status)
-			info->lock_3a->val |= V4L2_LOCK_EXPOSURE;
-		break;
-	}
-
-	return ret;
-}
-
-static int m5mols_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	unsigned int ctrl_mode = m5mols_get_ctrl_mode(ctrl);
-	struct v4l2_subdev *sd = to_sd(ctrl);
-	struct m5mols_info *info = to_m5mols(sd);
-	int last_mode = info->mode;
-	int ret = 0;
-
-	/*
-	 * If needed, defer restoring the controls until
-	 * the device is fully initialized.
-	 */
-	if (!info->isp_ready) {
-		info->ctrl_sync = 0;
-		return 0;
-	}
-
-	v4l2_dbg(1, m5mols_debug, sd, "%s: %s, val: %d, priv: %p\n",
-		 __func__, ctrl->name, ctrl->val, ctrl->priv);
-
-	if (ctrl_mode && ctrl_mode != info->mode) {
-		ret = m5mols_set_mode(info, ctrl_mode);
-		if (ret < 0)
-			return ret;
-	}
-
-	switch (ctrl->id) {
-	case V4L2_CID_3A_LOCK:
-		ret = m5mols_3a_lock(info, ctrl);
-		break;
-
-	case V4L2_CID_ZOOM_ABSOLUTE:
-		ret = m5mols_write(sd, MON_ZOOM, ctrl->val);
-		break;
-
-	case V4L2_CID_EXPOSURE_AUTO:
-		ret = m5mols_set_exposure(info, ctrl->val);
-		break;
-
-	case V4L2_CID_ISO_SENSITIVITY:
-		ret = m5mols_set_iso(info, ctrl->val);
-		break;
-
-	case V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE:
-		ret = m5mols_set_white_balance(info, ctrl->val);
-		break;
-
-	case V4L2_CID_SATURATION:
-		ret = m5mols_set_saturation(info, ctrl->val);
-		break;
-
-	case V4L2_CID_COLORFX:
-		ret = m5mols_set_color_effect(info, ctrl->val);
-		break;
-
-	case V4L2_CID_WIDE_DYNAMIC_RANGE:
-		ret = m5mols_set_wdr(info, ctrl->val);
-		break;
-
-	case V4L2_CID_IMAGE_STABILIZATION:
-		ret = m5mols_set_stabilization(info, ctrl->val);
-		break;
-
-	case V4L2_CID_JPEG_COMPRESSION_QUALITY:
-		ret = m5mols_write(sd, CAPP_JPEG_RATIO, ctrl->val);
-		break;
-	}
-
-	if (ret == 0 && info->mode != last_mode)
-		ret = m5mols_set_mode(info, last_mode);
-
-	return ret;
-}
-
-static const struct v4l2_ctrl_ops m5mols_ctrl_ops = {
-	.g_volatile_ctrl	= m5mols_g_volatile_ctrl,
-	.s_ctrl			= m5mols_s_ctrl,
-};
-
-/* Supported manual ISO values */
-static const s64 iso_qmenu[] = {
-	/* AE_ISO: 0x01...0x07 (ISO: 50...3200) */
-	50000, 100000, 200000, 400000, 800000, 1600000, 3200000
-};
-
-/* Supported Exposure Bias values, -2.0EV...+2.0EV */
-static const s64 ev_bias_qmenu[] = {
-	/* AE_INDEX: 0x00...0x08 */
-	-2000, -1500, -1000, -500, 0, 500, 1000, 1500, 2000
-};
-
-int m5mols_init_controls(struct v4l2_subdev *sd)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-	u16 exposure_max;
-	u16 zoom_step;
-	int ret;
-
-	/* Determine the firmware dependent control range and step values */
-	ret = m5mols_read_u16(sd, AE_MAX_GAIN_MON, &exposure_max);
-	if (ret < 0)
-		return ret;
-
-	zoom_step = is_manufacturer(info, REG_SAMSUNG_OPTICS) ? 31 : 1;
-	v4l2_ctrl_handler_init(&info->handle, 20);
-
-	info->auto_wb = v4l2_ctrl_new_std_menu(&info->handle,
-			&m5mols_ctrl_ops, V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE,
-			9, ~0x3fe, V4L2_WHITE_BALANCE_AUTO);
-
-	/* Exposure control cluster */
-	info->auto_exposure = v4l2_ctrl_new_std_menu(&info->handle,
-			&m5mols_ctrl_ops, V4L2_CID_EXPOSURE_AUTO,
-			1, ~0x03, V4L2_EXPOSURE_AUTO);
-
-	info->exposure = v4l2_ctrl_new_std(&info->handle,
-			&m5mols_ctrl_ops, V4L2_CID_EXPOSURE,
-			0, exposure_max, 1, exposure_max / 2);
-
-	info->exposure_bias = v4l2_ctrl_new_int_menu(&info->handle,
-			&m5mols_ctrl_ops, V4L2_CID_AUTO_EXPOSURE_BIAS,
-			ARRAY_SIZE(ev_bias_qmenu) - 1,
-			ARRAY_SIZE(ev_bias_qmenu)/2 - 1,
-			ev_bias_qmenu);
-
-	info->metering = v4l2_ctrl_new_std_menu(&info->handle,
-			&m5mols_ctrl_ops, V4L2_CID_EXPOSURE_METERING,
-			2, ~0x7, V4L2_EXPOSURE_METERING_AVERAGE);
-
-	/* ISO control cluster */
-	info->auto_iso = v4l2_ctrl_new_std_menu(&info->handle, &m5mols_ctrl_ops,
-			V4L2_CID_ISO_SENSITIVITY_AUTO, 1, ~0x03, 1);
-
-	info->iso = v4l2_ctrl_new_int_menu(&info->handle, &m5mols_ctrl_ops,
-			V4L2_CID_ISO_SENSITIVITY, ARRAY_SIZE(iso_qmenu) - 1,
-			ARRAY_SIZE(iso_qmenu)/2 - 1, iso_qmenu);
-
-	info->saturation = v4l2_ctrl_new_std(&info->handle, &m5mols_ctrl_ops,
-			V4L2_CID_SATURATION, 1, 5, 1, 3);
-
-	info->zoom = v4l2_ctrl_new_std(&info->handle, &m5mols_ctrl_ops,
-			V4L2_CID_ZOOM_ABSOLUTE, 1, 70, zoom_step, 1);
-
-	info->colorfx = v4l2_ctrl_new_std_menu(&info->handle, &m5mols_ctrl_ops,
-			V4L2_CID_COLORFX, 4, 0, V4L2_COLORFX_NONE);
-
-	info->wdr = v4l2_ctrl_new_std(&info->handle, &m5mols_ctrl_ops,
-			V4L2_CID_WIDE_DYNAMIC_RANGE, 0, 1, 1, 0);
-
-	info->stabilization = v4l2_ctrl_new_std(&info->handle, &m5mols_ctrl_ops,
-			V4L2_CID_IMAGE_STABILIZATION, 0, 1, 1, 0);
-
-	info->jpeg_quality = v4l2_ctrl_new_std(&info->handle, &m5mols_ctrl_ops,
-			V4L2_CID_JPEG_COMPRESSION_QUALITY, 1, 100, 1, 80);
-
-	info->lock_3a = v4l2_ctrl_new_std(&info->handle, &m5mols_ctrl_ops,
-			V4L2_CID_3A_LOCK, 0, 0x7, 0, 0);
-
-	if (info->handle.error) {
-		int ret = info->handle.error;
-		v4l2_err(sd, "Failed to initialize controls: %d\n", ret);
-		v4l2_ctrl_handler_free(&info->handle);
-		return ret;
-	}
-
-	v4l2_ctrl_auto_cluster(4, &info->auto_exposure, 1, false);
-	info->auto_iso->flags |= V4L2_CTRL_FLAG_VOLATILE |
-				V4L2_CTRL_FLAG_UPDATE;
-	v4l2_ctrl_auto_cluster(2, &info->auto_iso, 0, false);
-
-	info->lock_3a->flags |= V4L2_CTRL_FLAG_VOLATILE;
-
-	m5mols_set_ctrl_mode(info->auto_exposure, REG_PARAMETER);
-	m5mols_set_ctrl_mode(info->auto_wb, REG_PARAMETER);
-	m5mols_set_ctrl_mode(info->colorfx, REG_MONITOR);
-
-	sd->ctrl_handler = &info->handle;
-
-	return 0;
-}
diff --git a/drivers/media/i2c/m5mols/m5mols_core.c b/drivers/media/i2c/m5mols/m5mols_core.c
deleted file mode 100644
index 5c2336f318d9..000000000000
--- a/drivers/media/i2c/m5mols/m5mols_core.c
+++ /dev/null
@@ -1,1051 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for M-5MOLS 8M Pixel camera sensor with ISP
- *
- * Copyright (C) 2011 Samsung Electronics Co., Ltd.
- * Author: HeungJun Kim <riverful.kim@samsung.com>
- *
- * Copyright (C) 2009 Samsung Electronics Co., Ltd.
- * Author: Dongsoo Nathaniel Kim <dongsoo45.kim@samsung.com>
- */
-
-#include <linux/i2c.h>
-#include <linux/slab.h>
-#include <linux/irq.h>
-#include <linux/interrupt.h>
-#include <linux/delay.h>
-#include <linux/gpio/consumer.h>
-#include <linux/regulator/consumer.h>
-#include <linux/videodev2.h>
-#include <linux/module.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-subdev.h>
-#include <media/i2c/m5mols.h>
-
-#include "m5mols.h"
-#include "m5mols_reg.h"
-
-int m5mols_debug;
-module_param(m5mols_debug, int, 0644);
-
-#define MODULE_NAME		"M5MOLS"
-#define M5MOLS_I2C_CHECK_RETRY	500
-
-/* The regulator consumer names for external voltage regulators */
-static struct regulator_bulk_data supplies[] = {
-	{
-		.supply = "core",	/* ARM core power, 1.2V */
-	}, {
-		.supply	= "dig_18",	/* digital power 1, 1.8V */
-	}, {
-		.supply	= "d_sensor",	/* sensor power 1, 1.8V */
-	}, {
-		.supply	= "dig_28",	/* digital power 2, 2.8V */
-	}, {
-		.supply	= "a_sensor",	/* analog power */
-	}, {
-		.supply	= "dig_12",	/* digital power 3, 1.2V */
-	},
-};
-
-static struct v4l2_mbus_framefmt m5mols_default_ffmt[M5MOLS_RESTYPE_MAX] = {
-	[M5MOLS_RESTYPE_MONITOR] = {
-		.width		= 1920,
-		.height		= 1080,
-		.code		= MEDIA_BUS_FMT_VYUY8_2X8,
-		.field		= V4L2_FIELD_NONE,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-	},
-	[M5MOLS_RESTYPE_CAPTURE] = {
-		.width		= 1920,
-		.height		= 1080,
-		.code		= MEDIA_BUS_FMT_JPEG_1X8,
-		.field		= V4L2_FIELD_NONE,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-	},
-};
-#define SIZE_DEFAULT_FFMT	ARRAY_SIZE(m5mols_default_ffmt)
-
-static const struct m5mols_resolution m5mols_reg_res[] = {
-	{ 0x01, M5MOLS_RESTYPE_MONITOR, 128, 96 },	/* SUB-QCIF */
-	{ 0x03, M5MOLS_RESTYPE_MONITOR, 160, 120 },	/* QQVGA */
-	{ 0x05, M5MOLS_RESTYPE_MONITOR, 176, 144 },	/* QCIF */
-	{ 0x06, M5MOLS_RESTYPE_MONITOR, 176, 176 },
-	{ 0x08, M5MOLS_RESTYPE_MONITOR, 240, 320 },	/* QVGA */
-	{ 0x09, M5MOLS_RESTYPE_MONITOR, 320, 240 },	/* QVGA */
-	{ 0x0c, M5MOLS_RESTYPE_MONITOR, 240, 400 },	/* WQVGA */
-	{ 0x0d, M5MOLS_RESTYPE_MONITOR, 400, 240 },	/* WQVGA */
-	{ 0x0e, M5MOLS_RESTYPE_MONITOR, 352, 288 },	/* CIF */
-	{ 0x13, M5MOLS_RESTYPE_MONITOR, 480, 360 },
-	{ 0x15, M5MOLS_RESTYPE_MONITOR, 640, 360 },	/* qHD */
-	{ 0x17, M5MOLS_RESTYPE_MONITOR, 640, 480 },	/* VGA */
-	{ 0x18, M5MOLS_RESTYPE_MONITOR, 720, 480 },
-	{ 0x1a, M5MOLS_RESTYPE_MONITOR, 800, 480 },	/* WVGA */
-	{ 0x1f, M5MOLS_RESTYPE_MONITOR, 800, 600 },	/* SVGA */
-	{ 0x21, M5MOLS_RESTYPE_MONITOR, 1280, 720 },	/* HD */
-	{ 0x25, M5MOLS_RESTYPE_MONITOR, 1920, 1080 },	/* 1080p */
-	{ 0x29, M5MOLS_RESTYPE_MONITOR, 3264, 2448 },	/* 2.63fps 8M */
-	{ 0x39, M5MOLS_RESTYPE_MONITOR, 800, 602 },	/* AHS_MON debug */
-
-	{ 0x02, M5MOLS_RESTYPE_CAPTURE, 320, 240 },	/* QVGA */
-	{ 0x04, M5MOLS_RESTYPE_CAPTURE, 400, 240 },	/* WQVGA */
-	{ 0x07, M5MOLS_RESTYPE_CAPTURE, 480, 360 },
-	{ 0x08, M5MOLS_RESTYPE_CAPTURE, 640, 360 },	/* qHD */
-	{ 0x09, M5MOLS_RESTYPE_CAPTURE, 640, 480 },	/* VGA */
-	{ 0x0a, M5MOLS_RESTYPE_CAPTURE, 800, 480 },	/* WVGA */
-	{ 0x10, M5MOLS_RESTYPE_CAPTURE, 1280, 720 },	/* HD */
-	{ 0x14, M5MOLS_RESTYPE_CAPTURE, 1280, 960 },	/* 1M */
-	{ 0x17, M5MOLS_RESTYPE_CAPTURE, 1600, 1200 },	/* 2M */
-	{ 0x19, M5MOLS_RESTYPE_CAPTURE, 1920, 1080 },	/* Full-HD */
-	{ 0x1a, M5MOLS_RESTYPE_CAPTURE, 2048, 1152 },	/* 3Mega */
-	{ 0x1b, M5MOLS_RESTYPE_CAPTURE, 2048, 1536 },
-	{ 0x1c, M5MOLS_RESTYPE_CAPTURE, 2560, 1440 },	/* 4Mega */
-	{ 0x1d, M5MOLS_RESTYPE_CAPTURE, 2560, 1536 },
-	{ 0x1f, M5MOLS_RESTYPE_CAPTURE, 2560, 1920 },	/* 5Mega */
-	{ 0x21, M5MOLS_RESTYPE_CAPTURE, 3264, 1836 },	/* 6Mega */
-	{ 0x22, M5MOLS_RESTYPE_CAPTURE, 3264, 1960 },
-	{ 0x25, M5MOLS_RESTYPE_CAPTURE, 3264, 2448 },	/* 8Mega */
-};
-
-/**
- * m5mols_swap_byte - an byte array to integer conversion function
- * @data: byte array
- * @length: size in bytes of I2C packet defined in the M-5MOLS datasheet
- *
- * Convert I2C data byte array with performing any required byte
- * reordering to assure proper values for each data type, regardless
- * of the architecture endianness.
- */
-static u32 m5mols_swap_byte(u8 *data, u8 length)
-{
-	if (length == 1)
-		return *data;
-	else if (length == 2)
-		return be16_to_cpu(*((__be16 *)data));
-	else
-		return be32_to_cpu(*((__be32 *)data));
-}
-
-/**
- * m5mols_read -  I2C read function
- * @sd: sub-device, as pointed by struct v4l2_subdev
- * @size: desired size of I2C packet
- * @reg: combination of size, category and command for the I2C packet
- * @val: read value
- *
- * Returns 0 on success, or else negative errno.
- */
-static int m5mols_read(struct v4l2_subdev *sd, u32 size, u32 reg, u32 *val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct m5mols_info *info = to_m5mols(sd);
-	u8 rbuf[M5MOLS_I2C_MAX_SIZE + 1];
-	u8 category = I2C_CATEGORY(reg);
-	u8 cmd = I2C_COMMAND(reg);
-	struct i2c_msg msg[2];
-	u8 wbuf[5];
-	int ret;
-
-	if (!client->adapter)
-		return -ENODEV;
-
-	msg[0].addr = client->addr;
-	msg[0].flags = 0;
-	msg[0].len = 5;
-	msg[0].buf = wbuf;
-	wbuf[0] = 5;
-	wbuf[1] = M5MOLS_BYTE_READ;
-	wbuf[2] = category;
-	wbuf[3] = cmd;
-	wbuf[4] = size;
-
-	msg[1].addr = client->addr;
-	msg[1].flags = I2C_M_RD;
-	msg[1].len = size + 1;
-	msg[1].buf = rbuf;
-
-	/* minimum stabilization time */
-	usleep_range(200, 300);
-
-	ret = i2c_transfer(client->adapter, msg, 2);
-
-	if (ret == 2) {
-		*val = m5mols_swap_byte(&rbuf[1], size);
-		return 0;
-	}
-
-	if (info->isp_ready)
-		v4l2_err(sd, "read failed: size:%d cat:%02x cmd:%02x. %d\n",
-			 size, category, cmd, ret);
-
-	return ret < 0 ? ret : -EIO;
-}
-
-int m5mols_read_u8(struct v4l2_subdev *sd, u32 reg, u8 *val)
-{
-	u32 val_32;
-	int ret;
-
-	if (I2C_SIZE(reg) != 1) {
-		v4l2_err(sd, "Wrong data size\n");
-		return -EINVAL;
-	}
-
-	ret = m5mols_read(sd, I2C_SIZE(reg), reg, &val_32);
-	if (ret)
-		return ret;
-
-	*val = (u8)val_32;
-	return ret;
-}
-
-int m5mols_read_u16(struct v4l2_subdev *sd, u32 reg, u16 *val)
-{
-	u32 val_32;
-	int ret;
-
-	if (I2C_SIZE(reg) != 2) {
-		v4l2_err(sd, "Wrong data size\n");
-		return -EINVAL;
-	}
-
-	ret = m5mols_read(sd, I2C_SIZE(reg), reg, &val_32);
-	if (ret)
-		return ret;
-
-	*val = (u16)val_32;
-	return ret;
-}
-
-int m5mols_read_u32(struct v4l2_subdev *sd, u32 reg, u32 *val)
-{
-	if (I2C_SIZE(reg) != 4) {
-		v4l2_err(sd, "Wrong data size\n");
-		return -EINVAL;
-	}
-
-	return m5mols_read(sd, I2C_SIZE(reg), reg, val);
-}
-
-/**
- * m5mols_write - I2C command write function
- * @sd: sub-device, as pointed by struct v4l2_subdev
- * @reg: combination of size, category and command for the I2C packet
- * @val: value to write
- *
- * Returns 0 on success, or else negative errno.
- */
-int m5mols_write(struct v4l2_subdev *sd, u32 reg, u32 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct m5mols_info *info = to_m5mols(sd);
-	u8 wbuf[M5MOLS_I2C_MAX_SIZE + 4];
-	u8 category = I2C_CATEGORY(reg);
-	u8 cmd = I2C_COMMAND(reg);
-	u8 size	= I2C_SIZE(reg);
-	u32 *buf = (u32 *)&wbuf[4];
-	struct i2c_msg msg[1];
-	int ret;
-
-	if (!client->adapter)
-		return -ENODEV;
-
-	if (size != 1 && size != 2 && size != 4) {
-		v4l2_err(sd, "Wrong data size\n");
-		return -EINVAL;
-	}
-
-	msg->addr = client->addr;
-	msg->flags = 0;
-	msg->len = (u16)size + 4;
-	msg->buf = wbuf;
-	wbuf[0] = size + 4;
-	wbuf[1] = M5MOLS_BYTE_WRITE;
-	wbuf[2] = category;
-	wbuf[3] = cmd;
-
-	*buf = m5mols_swap_byte((u8 *)&val, size);
-
-	/* minimum stabilization time */
-	usleep_range(200, 300);
-
-	ret = i2c_transfer(client->adapter, msg, 1);
-	if (ret == 1)
-		return 0;
-
-	if (info->isp_ready)
-		v4l2_err(sd, "write failed: cat:%02x cmd:%02x ret:%d\n",
-			 category, cmd, ret);
-
-	return ret < 0 ? ret : -EIO;
-}
-
-/**
- * m5mols_busy_wait - Busy waiting with I2C register polling
- * @sd: sub-device, as pointed by struct v4l2_subdev
- * @reg: the I2C_REG() address of an 8-bit status register to check
- * @value: expected status register value
- * @mask: bit mask for the read status register value
- * @timeout: timeout in milliseconds, or -1 for default timeout
- *
- * The @reg register value is ORed with @mask before comparing with @value.
- *
- * Return: 0 if the requested condition became true within less than
- *         @timeout ms, or else negative errno.
- */
-int m5mols_busy_wait(struct v4l2_subdev *sd, u32 reg, u32 value, u32 mask,
-		     int timeout)
-{
-	int ms = timeout < 0 ? M5MOLS_BUSY_WAIT_DEF_TIMEOUT : timeout;
-	unsigned long end = jiffies + msecs_to_jiffies(ms);
-	u8 status;
-
-	do {
-		int ret = m5mols_read_u8(sd, reg, &status);
-
-		if (ret < 0 && !(mask & M5MOLS_I2C_RDY_WAIT_FL))
-			return ret;
-		if (!ret && (status & mask & 0xff) == (value & 0xff))
-			return 0;
-		usleep_range(100, 250);
-	} while (ms > 0 && time_is_after_jiffies(end));
-
-	return -EBUSY;
-}
-
-/**
- * m5mols_enable_interrupt - Clear interrupt pending bits and unmask interrupts
- * @sd: sub-device, as pointed by struct v4l2_subdev
- * @reg: combination of size, category and command for the I2C packet
- *
- * Before writing desired interrupt value the INT_FACTOR register should
- * be read to clear pending interrupts.
- */
-int m5mols_enable_interrupt(struct v4l2_subdev *sd, u8 reg)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-	u8 mask = is_available_af(info) ? REG_INT_AF : 0;
-	u8 dummy;
-	int ret;
-
-	ret = m5mols_read_u8(sd, SYSTEM_INT_FACTOR, &dummy);
-	if (!ret)
-		ret = m5mols_write(sd, SYSTEM_INT_ENABLE, reg & ~mask);
-	return ret;
-}
-
-int m5mols_wait_interrupt(struct v4l2_subdev *sd, u8 irq_mask, u32 timeout)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-
-	int ret = wait_event_interruptible_timeout(info->irq_waitq,
-				atomic_add_unless(&info->irq_done, -1, 0),
-				msecs_to_jiffies(timeout));
-	if (ret <= 0)
-		return ret ? ret : -ETIMEDOUT;
-
-	return m5mols_busy_wait(sd, SYSTEM_INT_FACTOR, irq_mask,
-				M5MOLS_I2C_RDY_WAIT_FL | irq_mask, -1);
-}
-
-/**
- * m5mols_reg_mode - Write the mode and check busy status
- * @sd: sub-device, as pointed by struct v4l2_subdev
- * @mode: the required operation mode
- *
- * It always accompanies a little delay changing the M-5MOLS mode, so it is
- * needed checking current busy status to guarantee right mode.
- */
-static int m5mols_reg_mode(struct v4l2_subdev *sd, u8 mode)
-{
-	int ret = m5mols_write(sd, SYSTEM_SYSMODE, mode);
-	if (ret < 0)
-		return ret;
-	return m5mols_busy_wait(sd, SYSTEM_SYSMODE, mode, 0xff,
-				M5MOLS_MODE_CHANGE_TIMEOUT);
-}
-
-/**
- * m5mols_set_mode - set the M-5MOLS controller mode
- * @info: M-5MOLS driver data structure
- * @mode: the required operation mode
- *
- * The commands of M-5MOLS are grouped into specific modes. Each functionality
- * can be guaranteed only when the sensor is operating in mode which a command
- * belongs to.
- */
-int m5mols_set_mode(struct m5mols_info *info, u8 mode)
-{
-	struct v4l2_subdev *sd = &info->sd;
-	int ret = -EINVAL;
-	u8 reg;
-
-	if (mode < REG_PARAMETER || mode > REG_CAPTURE)
-		return ret;
-
-	ret = m5mols_read_u8(sd, SYSTEM_SYSMODE, &reg);
-	if (ret || reg == mode)
-		return ret;
-
-	switch (reg) {
-	case REG_PARAMETER:
-		ret = m5mols_reg_mode(sd, REG_MONITOR);
-		if (mode == REG_MONITOR)
-			break;
-		if (!ret)
-			ret = m5mols_reg_mode(sd, REG_CAPTURE);
-		break;
-
-	case REG_MONITOR:
-		if (mode == REG_PARAMETER) {
-			ret = m5mols_reg_mode(sd, REG_PARAMETER);
-			break;
-		}
-
-		ret = m5mols_reg_mode(sd, REG_CAPTURE);
-		break;
-
-	case REG_CAPTURE:
-		ret = m5mols_reg_mode(sd, REG_MONITOR);
-		if (mode == REG_MONITOR)
-			break;
-		if (!ret)
-			ret = m5mols_reg_mode(sd, REG_PARAMETER);
-		break;
-
-	default:
-		v4l2_warn(sd, "Wrong mode: %d\n", mode);
-	}
-
-	if (!ret)
-		info->mode = mode;
-
-	return ret;
-}
-
-/**
- * m5mols_get_version - retrieve full revisions information of M-5MOLS
- * @sd: sub-device, as pointed by struct v4l2_subdev
- *
- * The version information includes revisions of hardware and firmware,
- * AutoFocus alghorithm version and the version string.
- */
-static int m5mols_get_version(struct v4l2_subdev *sd)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-	struct m5mols_version *ver = &info->ver;
-	u8 *str = ver->str;
-	int i;
-	int ret;
-
-	ret = m5mols_read_u8(sd, SYSTEM_VER_CUSTOMER, &ver->customer);
-	if (!ret)
-		ret = m5mols_read_u8(sd, SYSTEM_VER_PROJECT, &ver->project);
-	if (!ret)
-		ret = m5mols_read_u16(sd, SYSTEM_VER_FIRMWARE, &ver->fw);
-	if (!ret)
-		ret = m5mols_read_u16(sd, SYSTEM_VER_HARDWARE, &ver->hw);
-	if (!ret)
-		ret = m5mols_read_u16(sd, SYSTEM_VER_PARAMETER, &ver->param);
-	if (!ret)
-		ret = m5mols_read_u16(sd, SYSTEM_VER_AWB, &ver->awb);
-	if (!ret)
-		ret = m5mols_read_u8(sd, AF_VERSION, &ver->af);
-	if (ret)
-		return ret;
-
-	for (i = 0; i < VERSION_STRING_SIZE; i++) {
-		ret = m5mols_read_u8(sd, SYSTEM_VER_STRING, &str[i]);
-		if (ret)
-			return ret;
-	}
-
-	v4l2_info(sd, "Manufacturer\t[%s]\n",
-			is_manufacturer(info, REG_SAMSUNG_ELECTRO) ?
-			"Samsung Electro-Mechanics" :
-			is_manufacturer(info, REG_SAMSUNG_OPTICS) ?
-			"Samsung Fiber-Optics" :
-			is_manufacturer(info, REG_SAMSUNG_TECHWIN) ?
-			"Samsung Techwin" : "None");
-	v4l2_info(sd, "Customer/Project\t[0x%02x/0x%02x]\n",
-			info->ver.customer, info->ver.project);
-
-	if (!is_available_af(info))
-		v4l2_info(sd, "No support Auto Focus on this firmware\n");
-
-	return ret;
-}
-
-/**
- * __find_restype - Lookup M-5MOLS resolution type according to pixel code
- * @code: pixel code
- */
-static enum m5mols_restype __find_restype(u32 code)
-{
-	enum m5mols_restype type = M5MOLS_RESTYPE_MONITOR;
-
-	do {
-		if (code == m5mols_default_ffmt[type].code)
-			return type;
-	} while (++type != SIZE_DEFAULT_FFMT);
-
-	return 0;
-}
-
-/**
- * __find_resolution - Lookup preset and type of M-5MOLS's resolution
- * @sd: sub-device, as pointed by struct v4l2_subdev
- * @mf: pixel format to find/negotiate the resolution preset for
- * @type: M-5MOLS resolution type
- * @resolution:	M-5MOLS resolution preset register value
- *
- * Find nearest resolution matching resolution preset and adjust mf
- * to supported values.
- */
-static int __find_resolution(struct v4l2_subdev *sd,
-			     struct v4l2_mbus_framefmt *mf,
-			     enum m5mols_restype *type,
-			     u32 *resolution)
-{
-	const struct m5mols_resolution *fsize = &m5mols_reg_res[0];
-	const struct m5mols_resolution *match = NULL;
-	enum m5mols_restype stype = __find_restype(mf->code);
-	int i = ARRAY_SIZE(m5mols_reg_res);
-	unsigned int min_err = ~0;
-
-	while (i--) {
-		int err;
-		if (stype == fsize->type) {
-			err = abs(fsize->width - mf->width)
-				+ abs(fsize->height - mf->height);
-
-			if (err < min_err) {
-				min_err = err;
-				match = fsize;
-			}
-		}
-		fsize++;
-	}
-	if (match) {
-		mf->width  = match->width;
-		mf->height = match->height;
-		*resolution = match->reg;
-		*type = stype;
-		return 0;
-	}
-
-	return -EINVAL;
-}
-
-static struct v4l2_mbus_framefmt *__find_format(struct m5mols_info *info,
-				struct v4l2_subdev_state *sd_state,
-				enum v4l2_subdev_format_whence which,
-				enum m5mols_restype type)
-{
-	if (which == V4L2_SUBDEV_FORMAT_TRY)
-		return sd_state ? v4l2_subdev_get_try_format(&info->sd,
-							     sd_state, 0) : NULL;
-
-	return &info->ffmt[type];
-}
-
-static int m5mols_get_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_state *sd_state,
-			  struct v4l2_subdev_format *fmt)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
-
-	mutex_lock(&info->lock);
-
-	format = __find_format(info, sd_state, fmt->which, info->res_type);
-	if (format)
-		fmt->format = *format;
-	else
-		ret = -EINVAL;
-
-	mutex_unlock(&info->lock);
-	return ret;
-}
-
-static int m5mols_set_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_state *sd_state,
-			  struct v4l2_subdev_format *fmt)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-	struct v4l2_mbus_framefmt *format = &fmt->format;
-	struct v4l2_mbus_framefmt *sfmt;
-	enum m5mols_restype type;
-	u32 resolution = 0;
-	int ret;
-
-	ret = __find_resolution(sd, format, &type, &resolution);
-	if (ret < 0)
-		return ret;
-
-	sfmt = __find_format(info, sd_state, fmt->which, type);
-	if (!sfmt)
-		return 0;
-
-	mutex_lock(&info->lock);
-
-	format->code = m5mols_default_ffmt[type].code;
-	format->colorspace = V4L2_COLORSPACE_JPEG;
-	format->field = V4L2_FIELD_NONE;
-
-	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
-		*sfmt = *format;
-		info->resolution = resolution;
-		info->res_type = type;
-	}
-
-	mutex_unlock(&info->lock);
-	return ret;
-}
-
-static int m5mols_get_frame_desc(struct v4l2_subdev *sd, unsigned int pad,
-				 struct v4l2_mbus_frame_desc *fd)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-
-	if (pad != 0 || fd == NULL)
-		return -EINVAL;
-
-	mutex_lock(&info->lock);
-	/*
-	 * .get_frame_desc is only used for compressed formats,
-	 * thus we always return the capture frame parameters here.
-	 */
-	fd->entry[0].length = info->cap.buf_size;
-	fd->entry[0].pixelcode = info->ffmt[M5MOLS_RESTYPE_CAPTURE].code;
-	mutex_unlock(&info->lock);
-
-	fd->entry[0].flags = V4L2_MBUS_FRAME_DESC_FL_LEN_MAX;
-	fd->num_entries = 1;
-
-	return 0;
-}
-
-static int m5mols_set_frame_desc(struct v4l2_subdev *sd, unsigned int pad,
-				 struct v4l2_mbus_frame_desc *fd)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-	struct v4l2_mbus_framefmt *mf = &info->ffmt[M5MOLS_RESTYPE_CAPTURE];
-
-	if (pad != 0 || fd == NULL)
-		return -EINVAL;
-
-	fd->entry[0].flags = V4L2_MBUS_FRAME_DESC_FL_LEN_MAX;
-	fd->num_entries = 1;
-	fd->entry[0].length = clamp_t(u32, fd->entry[0].length,
-				      mf->width * mf->height,
-				      M5MOLS_MAIN_JPEG_SIZE_MAX);
-	mutex_lock(&info->lock);
-	info->cap.buf_size = fd->entry[0].length;
-	mutex_unlock(&info->lock);
-
-	return 0;
-}
-
-
-static int m5mols_enum_mbus_code(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_state *sd_state,
-				 struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (!code || code->index >= SIZE_DEFAULT_FFMT)
-		return -EINVAL;
-
-	code->code = m5mols_default_ffmt[code->index].code;
-
-	return 0;
-}
-
-static const struct v4l2_subdev_pad_ops m5mols_pad_ops = {
-	.enum_mbus_code	= m5mols_enum_mbus_code,
-	.get_fmt	= m5mols_get_fmt,
-	.set_fmt	= m5mols_set_fmt,
-	.get_frame_desc	= m5mols_get_frame_desc,
-	.set_frame_desc	= m5mols_set_frame_desc,
-};
-
-/**
- * m5mols_restore_controls - Apply current control values to the registers
- * @info: M-5MOLS driver data structure
- *
- * m5mols_do_scenemode() handles all parameters for which there is yet no
- * individual control. It should be replaced at some point by setting each
- * control individually, in required register set up order.
- */
-int m5mols_restore_controls(struct m5mols_info *info)
-{
-	int ret;
-
-	if (info->ctrl_sync)
-		return 0;
-
-	ret = m5mols_do_scenemode(info, REG_SCENE_NORMAL);
-	if (ret)
-		return ret;
-
-	ret = v4l2_ctrl_handler_setup(&info->handle);
-	info->ctrl_sync = !ret;
-
-	return ret;
-}
-
-/**
- * m5mols_start_monitor - Start the monitor mode
- * @info: M-5MOLS driver data structure
- *
- * Before applying the controls setup the resolution and frame rate
- * in PARAMETER mode, and then switch over to MONITOR mode.
- */
-static int m5mols_start_monitor(struct m5mols_info *info)
-{
-	struct v4l2_subdev *sd = &info->sd;
-	int ret;
-
-	ret = m5mols_set_mode(info, REG_PARAMETER);
-	if (!ret)
-		ret = m5mols_write(sd, PARM_MON_SIZE, info->resolution);
-	if (!ret)
-		ret = m5mols_write(sd, PARM_MON_FPS, REG_FPS_30);
-	if (!ret)
-		ret = m5mols_set_mode(info, REG_MONITOR);
-	if (!ret)
-		ret = m5mols_restore_controls(info);
-
-	return ret;
-}
-
-static int m5mols_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-	u32 code;
-	int ret;
-
-	mutex_lock(&info->lock);
-	code = info->ffmt[info->res_type].code;
-
-	if (enable) {
-		if (is_code(code, M5MOLS_RESTYPE_MONITOR))
-			ret = m5mols_start_monitor(info);
-		else if (is_code(code, M5MOLS_RESTYPE_CAPTURE))
-			ret = m5mols_start_capture(info);
-		else
-			ret = -EINVAL;
-	} else {
-		ret = m5mols_set_mode(info, REG_PARAMETER);
-	}
-
-	mutex_unlock(&info->lock);
-	return ret;
-}
-
-static const struct v4l2_subdev_video_ops m5mols_video_ops = {
-	.s_stream	= m5mols_s_stream,
-};
-
-static int m5mols_sensor_power(struct m5mols_info *info, bool enable)
-{
-	struct v4l2_subdev *sd = &info->sd;
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	if (info->power == enable)
-		return 0;
-
-	if (enable) {
-		if (info->set_power) {
-			ret = info->set_power(&client->dev, 1);
-			if (ret)
-				return ret;
-		}
-
-		ret = regulator_bulk_enable(ARRAY_SIZE(supplies), supplies);
-		if (ret) {
-			if (info->set_power)
-				info->set_power(&client->dev, 0);
-			return ret;
-		}
-
-		gpiod_set_value(info->reset, 0);
-		info->power = 1;
-
-		return ret;
-	}
-
-	ret = regulator_bulk_disable(ARRAY_SIZE(supplies), supplies);
-	if (ret)
-		return ret;
-
-	if (info->set_power)
-		info->set_power(&client->dev, 0);
-
-	gpiod_set_value(info->reset, 1);
-
-	info->isp_ready = 0;
-	info->power = 0;
-
-	return ret;
-}
-
-/* m5mols_update_fw - optional firmware update routine */
-int __attribute__ ((weak)) m5mols_update_fw(struct v4l2_subdev *sd,
-		int (*set_power)(struct m5mols_info *, bool))
-{
-	return 0;
-}
-
-/**
- * m5mols_fw_start - M-5MOLS internal ARM controller initialization
- * @sd: sub-device, as pointed by struct v4l2_subdev
- *
- * Execute the M-5MOLS internal ARM controller initialization sequence.
- * This function should be called after the supply voltage has been
- * applied and before any requests to the device are made.
- */
-static int m5mols_fw_start(struct v4l2_subdev *sd)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-	int ret;
-
-	atomic_set(&info->irq_done, 0);
-	/* Wait until I2C slave is initialized in Flash Writer mode */
-	ret = m5mols_busy_wait(sd, FLASH_CAM_START, REG_IN_FLASH_MODE,
-			       M5MOLS_I2C_RDY_WAIT_FL | 0xff, -1);
-	if (!ret)
-		ret = m5mols_write(sd, FLASH_CAM_START, REG_START_ARM_BOOT);
-	if (!ret)
-		ret = m5mols_wait_interrupt(sd, REG_INT_MODE, 2000);
-	if (ret < 0)
-		return ret;
-
-	info->isp_ready = 1;
-
-	ret = m5mols_get_version(sd);
-	if (!ret)
-		ret = m5mols_update_fw(sd, m5mols_sensor_power);
-	if (ret)
-		return ret;
-
-	v4l2_dbg(1, m5mols_debug, sd, "Success ARM Booting\n");
-
-	ret = m5mols_write(sd, PARM_INTERFACE, REG_INTERFACE_MIPI);
-	if (!ret)
-		ret = m5mols_enable_interrupt(sd,
-				REG_INT_AF | REG_INT_CAPTURE);
-
-	return ret;
-}
-
-/* Execute the lens soft-landing algorithm */
-static int m5mols_auto_focus_stop(struct m5mols_info *info)
-{
-	int ret;
-
-	ret = m5mols_write(&info->sd, AF_EXECUTE, REG_AF_STOP);
-	if (!ret)
-		ret = m5mols_write(&info->sd, AF_MODE, REG_AF_POWEROFF);
-	if (!ret)
-		ret = m5mols_busy_wait(&info->sd, SYSTEM_STATUS, REG_AF_IDLE,
-				       0xff, -1);
-	return ret;
-}
-
-/**
- * m5mols_s_power - Main sensor power control function
- * @sd: sub-device, as pointed by struct v4l2_subdev
- * @on: if true, powers on the device; powers off otherwise.
- *
- * To prevent breaking the lens when the sensor is powered off the Soft-Landing
- * algorithm is called where available. The Soft-Landing algorithm availability
- * dependends on the firmware provider.
- */
-static int m5mols_s_power(struct v4l2_subdev *sd, int on)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-	int ret;
-
-	mutex_lock(&info->lock);
-
-	if (on) {
-		ret = m5mols_sensor_power(info, true);
-		if (!ret)
-			ret = m5mols_fw_start(sd);
-	} else {
-		if (is_manufacturer(info, REG_SAMSUNG_TECHWIN)) {
-			ret = m5mols_set_mode(info, REG_MONITOR);
-			if (!ret)
-				ret = m5mols_auto_focus_stop(info);
-			if (ret < 0)
-				v4l2_warn(sd, "Soft landing lens failed\n");
-		}
-		ret = m5mols_sensor_power(info, false);
-
-		info->ctrl_sync = 0;
-	}
-
-	mutex_unlock(&info->lock);
-	return ret;
-}
-
-static int m5mols_log_status(struct v4l2_subdev *sd)
-{
-	struct m5mols_info *info = to_m5mols(sd);
-
-	v4l2_ctrl_handler_log_status(&info->handle, sd->name);
-
-	return 0;
-}
-
-static const struct v4l2_subdev_core_ops m5mols_core_ops = {
-	.s_power	= m5mols_s_power,
-	.log_status	= m5mols_log_status,
-};
-
-/*
- * V4L2 subdev internal operations
- */
-static int m5mols_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
-{
-	struct v4l2_mbus_framefmt *format = v4l2_subdev_get_try_format(sd,
-								       fh->state,
-								       0);
-
-	*format = m5mols_default_ffmt[0];
-	return 0;
-}
-
-static const struct v4l2_subdev_internal_ops m5mols_subdev_internal_ops = {
-	.open		= m5mols_open,
-};
-
-static const struct v4l2_subdev_ops m5mols_ops = {
-	.core		= &m5mols_core_ops,
-	.pad		= &m5mols_pad_ops,
-	.video		= &m5mols_video_ops,
-};
-
-static irqreturn_t m5mols_irq_handler(int irq, void *data)
-{
-	struct m5mols_info *info = to_m5mols(data);
-
-	atomic_set(&info->irq_done, 1);
-	wake_up_interruptible(&info->irq_waitq);
-
-	return IRQ_HANDLED;
-}
-
-static int m5mols_probe(struct i2c_client *client)
-{
-	const struct m5mols_platform_data *pdata = client->dev.platform_data;
-	struct m5mols_info *info;
-	struct v4l2_subdev *sd;
-	int ret;
-
-	if (pdata == NULL) {
-		dev_err(&client->dev, "No platform data\n");
-		return -EINVAL;
-	}
-
-	if (!client->irq) {
-		dev_err(&client->dev, "Interrupt not assigned\n");
-		return -EINVAL;
-	}
-
-	info = devm_kzalloc(&client->dev, sizeof(*info), GFP_KERNEL);
-	if (!info)
-		return -ENOMEM;
-
-	/* This asserts reset, descriptor shall have polarity specified */
-	info->reset = devm_gpiod_get(&client->dev, "reset", GPIOD_OUT_HIGH);
-	if (IS_ERR(info->reset))
-		return PTR_ERR(info->reset);
-	/* Notice: the "N" in M5MOLS_NRST implies active low */
-	gpiod_set_consumer_name(info->reset, "M5MOLS_NRST");
-
-	info->pdata = pdata;
-	info->set_power	= pdata->set_power;
-
-	ret = devm_regulator_bulk_get(&client->dev, ARRAY_SIZE(supplies),
-				      supplies);
-	if (ret) {
-		dev_err(&client->dev, "Failed to get regulators: %d\n", ret);
-		return ret;
-	}
-
-	sd = &info->sd;
-	v4l2_i2c_subdev_init(sd, client, &m5mols_ops);
-	/* Static name; NEVER use in new drivers! */
-	strscpy(sd->name, MODULE_NAME, sizeof(sd->name));
-	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-
-	sd->internal_ops = &m5mols_subdev_internal_ops;
-	info->pad.flags = MEDIA_PAD_FL_SOURCE;
-	ret = media_entity_pads_init(&sd->entity, 1, &info->pad);
-	if (ret < 0)
-		return ret;
-	sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
-
-	init_waitqueue_head(&info->irq_waitq);
-	mutex_init(&info->lock);
-
-	ret = devm_request_irq(&client->dev, client->irq, m5mols_irq_handler,
-			       IRQF_TRIGGER_RISING, MODULE_NAME, sd);
-	if (ret) {
-		dev_err(&client->dev, "Interrupt request failed: %d\n", ret);
-		goto error;
-	}
-	info->res_type = M5MOLS_RESTYPE_MONITOR;
-	info->ffmt[0] = m5mols_default_ffmt[0];
-	info->ffmt[1] =	m5mols_default_ffmt[1];
-
-	ret = m5mols_sensor_power(info, true);
-	if (ret)
-		goto error;
-
-	ret = m5mols_fw_start(sd);
-	if (!ret)
-		ret = m5mols_init_controls(sd);
-
-	ret = m5mols_sensor_power(info, false);
-	if (!ret)
-		return 0;
-error:
-	media_entity_cleanup(&sd->entity);
-	return ret;
-}
-
-static void m5mols_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-
-	v4l2_device_unregister_subdev(sd);
-	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	media_entity_cleanup(&sd->entity);
-}
-
-static const struct i2c_device_id m5mols_id[] = {
-	{ MODULE_NAME, 0 },
-	{ },
-};
-MODULE_DEVICE_TABLE(i2c, m5mols_id);
-
-static struct i2c_driver m5mols_i2c_driver = {
-	.driver = {
-		.name	= MODULE_NAME,
-	},
-	.probe_new	= m5mols_probe,
-	.remove		= m5mols_remove,
-	.id_table	= m5mols_id,
-};
-
-module_i2c_driver(m5mols_i2c_driver);
-
-MODULE_AUTHOR("HeungJun Kim <riverful.kim@samsung.com>");
-MODULE_AUTHOR("Dongsoo Kim <dongsoo45.kim@samsung.com>");
-MODULE_DESCRIPTION("Fujitsu M-5MOLS 8M Pixel camera driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/m5mols/m5mols_reg.h b/drivers/media/i2c/m5mols/m5mols_reg.h
deleted file mode 100644
index 947ee33812d3..000000000000
--- a/drivers/media/i2c/m5mols/m5mols_reg.h
+++ /dev/null
@@ -1,359 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Register map for M-5MOLS 8M Pixel camera sensor with ISP
- *
- * Copyright (C) 2011 Samsung Electronics Co., Ltd.
- * Author: HeungJun Kim <riverful.kim@samsung.com>
- *
- * Copyright (C) 2009 Samsung Electronics Co., Ltd.
- * Author: Dongsoo Nathaniel Kim <dongsoo45.kim@samsung.com>
- */
-
-#ifndef M5MOLS_REG_H
-#define M5MOLS_REG_H
-
-#define M5MOLS_I2C_MAX_SIZE	4
-#define M5MOLS_BYTE_READ	0x01
-#define M5MOLS_BYTE_WRITE	0x02
-
-#define I2C_CATEGORY(__cat)		((__cat >> 16) & 0xff)
-#define I2C_COMMAND(__comm)		((__comm >> 8) & 0xff)
-#define I2C_SIZE(__reg_s)		((__reg_s) & 0xff)
-#define I2C_REG(__cat, __cmd, __reg_s)	((__cat << 16) | (__cmd << 8) | __reg_s)
-
-/*
- * Category section register
- *
- * The category means set including relevant command of M-5MOLS.
- */
-#define CAT_SYSTEM		0x00
-#define CAT_PARAM		0x01
-#define CAT_MONITOR		0x02
-#define CAT_AE			0x03
-#define CAT_WB			0x06
-#define CAT_EXIF		0x07
-#define CAT_FD			0x09
-#define CAT_LENS		0x0a
-#define CAT_CAPT_PARM		0x0b
-#define CAT_CAPT_CTRL		0x0c
-#define CAT_FLASH		0x0f	/* related to FW, revisions, booting */
-
-/*
- * Category 0 - SYSTEM mode
- *
- * The SYSTEM mode in the M-5MOLS means area available to handle with the whole
- * & all-round system of sensor. It deals with version/interrupt/setting mode &
- * even sensor's status. Especially, the M-5MOLS sensor with ISP varies by
- * packaging & manufacturer, even the customer and project code. And the
- * function details may vary among them. The version information helps to
- * determine what methods shall be used in the driver.
- *
- * There is many registers between customer version address and awb one. For
- * more specific contents, see definition if file m5mols.h.
- */
-#define SYSTEM_VER_CUSTOMER	I2C_REG(CAT_SYSTEM, 0x00, 1)
-#define SYSTEM_VER_PROJECT	I2C_REG(CAT_SYSTEM, 0x01, 1)
-#define SYSTEM_VER_FIRMWARE	I2C_REG(CAT_SYSTEM, 0x02, 2)
-#define SYSTEM_VER_HARDWARE	I2C_REG(CAT_SYSTEM, 0x04, 2)
-#define SYSTEM_VER_PARAMETER	I2C_REG(CAT_SYSTEM, 0x06, 2)
-#define SYSTEM_VER_AWB		I2C_REG(CAT_SYSTEM, 0x08, 2)
-
-#define SYSTEM_SYSMODE		I2C_REG(CAT_SYSTEM, 0x0b, 1)
-#define REG_SYSINIT		0x00	/* SYSTEM mode */
-#define REG_PARAMETER		0x01	/* PARAMETER mode */
-#define REG_MONITOR		0x02	/* MONITOR mode */
-#define REG_CAPTURE		0x03	/* CAPTURE mode */
-
-#define SYSTEM_CMD(__cmd)	I2C_REG(CAT_SYSTEM, cmd, 1)
-#define SYSTEM_VER_STRING	I2C_REG(CAT_SYSTEM, 0x0a, 1)
-#define REG_SAMSUNG_ELECTRO	"SE"	/* Samsung Electro-Mechanics */
-#define REG_SAMSUNG_OPTICS	"OP"	/* Samsung Fiber-Optics */
-#define REG_SAMSUNG_TECHWIN	"TB"	/* Samsung Techwin */
-/* SYSTEM mode status */
-#define SYSTEM_STATUS	I2C_REG(CAT_SYSTEM, 0x0c, 1)
-
-/* Interrupt pending register */
-#define SYSTEM_INT_FACTOR	I2C_REG(CAT_SYSTEM, 0x10, 1)
-/* interrupt enable register */
-#define SYSTEM_INT_ENABLE	I2C_REG(CAT_SYSTEM, 0x11, 1)
-#define REG_INT_MODE		(1 << 0)
-#define REG_INT_AF		(1 << 1)
-#define REG_INT_ZOOM		(1 << 2)
-#define REG_INT_CAPTURE		(1 << 3)
-#define REG_INT_FRAMESYNC	(1 << 4)
-#define REG_INT_FD		(1 << 5)
-#define REG_INT_LENS_INIT	(1 << 6)
-#define REG_INT_SOUND		(1 << 7)
-#define REG_INT_MASK		0x0f
-
-/*
- * category 1 - PARAMETER mode
- *
- * This category supports function of camera features of M-5MOLS. It means we
- * can handle with preview(MONITOR) resolution size/frame per second/interface
- * between the sensor and the Application Processor/even the image effect.
- */
-
-/* Resolution in the MONITOR mode */
-#define PARM_MON_SIZE		I2C_REG(CAT_PARAM, 0x01, 1)
-
-/* Frame rate */
-#define PARM_MON_FPS		I2C_REG(CAT_PARAM, 0x02, 1)
-#define REG_FPS_30		0x02
-
-/* Video bus between the sensor and a host processor */
-#define PARM_INTERFACE		I2C_REG(CAT_PARAM, 0x00, 1)
-#define REG_INTERFACE_MIPI	0x02
-
-/* Image effects */
-#define PARM_EFFECT		I2C_REG(CAT_PARAM, 0x0b, 1)
-#define REG_EFFECT_OFF		0x00
-#define REG_EFFECT_NEGA		0x01
-#define REG_EFFECT_EMBOSS	0x06
-#define REG_EFFECT_OUTLINE	0x07
-#define REG_EFFECT_WATERCOLOR	0x08
-
-/*
- * Category 2 - MONITOR mode
- *
- * The MONITOR mode is same as preview mode as we said. The M-5MOLS has another
- * mode named "Preview", but this preview mode is used at the case specific
- * vider-recording mode. This mmode supports only YUYV format. On the other
- * hand, the JPEG & RAW formats is supports by CAPTURE mode. And, there are
- * another options like zoom/color effect(different with effect in PARAMETER
- * mode)/anti hand shaking algorithm.
- */
-
-/* Target digital zoom position */
-#define MON_ZOOM		I2C_REG(CAT_MONITOR, 0x01, 1)
-
-/* CR value for color effect */
-#define MON_CFIXR		I2C_REG(CAT_MONITOR, 0x0a, 1)
-/* CB value for color effect */
-#define MON_CFIXB		I2C_REG(CAT_MONITOR, 0x09, 1)
-#define REG_CFIXB_SEPIA		0xd8
-#define REG_CFIXR_SEPIA		0x18
-
-#define MON_EFFECT		I2C_REG(CAT_MONITOR, 0x0b, 1)
-#define REG_COLOR_EFFECT_OFF	0x00
-#define REG_COLOR_EFFECT_ON	0x01
-
-/* Chroma enable */
-#define MON_CHROMA_EN		I2C_REG(CAT_MONITOR, 0x10, 1)
-/* Chroma level */
-#define MON_CHROMA_LVL		I2C_REG(CAT_MONITOR, 0x0f, 1)
-#define REG_CHROMA_OFF		0x00
-#define REG_CHROMA_ON		0x01
-
-/* Sharpness on/off */
-#define MON_EDGE_EN		I2C_REG(CAT_MONITOR, 0x12, 1)
-/* Sharpness level */
-#define MON_EDGE_LVL		I2C_REG(CAT_MONITOR, 0x11, 1)
-#define REG_EDGE_OFF		0x00
-#define REG_EDGE_ON		0x01
-
-/* Set color tone (contrast) */
-#define MON_TONE_CTL		I2C_REG(CAT_MONITOR, 0x25, 1)
-
-/*
- * Category 3 - Auto Exposure
- *
- * The M-5MOLS exposure capbility is detailed as which is similar to digital
- * camera. This category supports AE locking/various AE mode(range of exposure)
- * /ISO/flickering/EV bias/shutter/meteoring, and anything else. And the
- * maximum/minimum exposure gain value depending on M-5MOLS firmware, may be
- * different. So, this category also provide getting the max/min values. And,
- * each MONITOR and CAPTURE mode has each gain/shutter/max exposure values.
- */
-
-/* Auto Exposure locking */
-#define AE_LOCK			I2C_REG(CAT_AE, 0x00, 1)
-#define REG_AE_UNLOCK		0x00
-#define REG_AE_LOCK		0x01
-
-/* Auto Exposure algorithm mode */
-#define AE_MODE			I2C_REG(CAT_AE, 0x01, 1)
-#define REG_AE_OFF		0x00	/* AE off */
-#define REG_AE_ALL		0x01	/* calc AE in all block integral */
-#define REG_AE_CENTER		0x03	/* calc AE in center weighted */
-#define REG_AE_SPOT		0x06	/* calc AE in specific spot */
-
-#define AE_ISO			I2C_REG(CAT_AE, 0x05, 1)
-#define REG_ISO_AUTO		0x00
-#define REG_ISO_50		0x01
-#define REG_ISO_100		0x02
-#define REG_ISO_200		0x03
-#define REG_ISO_400		0x04
-#define REG_ISO_800		0x05
-
-/* EV (scenemode) preset for MONITOR */
-#define AE_EV_PRESET_MONITOR	I2C_REG(CAT_AE, 0x0a, 1)
-/* EV (scenemode) preset for CAPTURE */
-#define AE_EV_PRESET_CAPTURE	I2C_REG(CAT_AE, 0x0b, 1)
-#define REG_SCENE_NORMAL	0x00
-#define REG_SCENE_PORTRAIT	0x01
-#define REG_SCENE_LANDSCAPE	0x02
-#define REG_SCENE_SPORTS	0x03
-#define REG_SCENE_PARTY_INDOOR	0x04
-#define REG_SCENE_BEACH_SNOW	0x05
-#define REG_SCENE_SUNSET	0x06
-#define REG_SCENE_DAWN_DUSK	0x07
-#define REG_SCENE_FALL		0x08
-#define REG_SCENE_NIGHT		0x09
-#define REG_SCENE_AGAINST_LIGHT	0x0a
-#define REG_SCENE_FIRE		0x0b
-#define REG_SCENE_TEXT		0x0c
-#define REG_SCENE_CANDLE	0x0d
-
-/* Manual gain in MONITOR mode */
-#define AE_MAN_GAIN_MON		I2C_REG(CAT_AE, 0x12, 2)
-/* Maximum gain in MONITOR mode */
-#define AE_MAX_GAIN_MON		I2C_REG(CAT_AE, 0x1a, 2)
-/* Manual gain in CAPTURE mode */
-#define AE_MAN_GAIN_CAP		I2C_REG(CAT_AE, 0x26, 2)
-
-#define AE_INDEX		I2C_REG(CAT_AE, 0x38, 1)
-#define REG_AE_INDEX_20_NEG	0x00
-#define REG_AE_INDEX_15_NEG	0x01
-#define REG_AE_INDEX_10_NEG	0x02
-#define REG_AE_INDEX_05_NEG	0x03
-#define REG_AE_INDEX_00		0x04
-#define REG_AE_INDEX_05_POS	0x05
-#define REG_AE_INDEX_10_POS	0x06
-#define REG_AE_INDEX_15_POS	0x07
-#define REG_AE_INDEX_20_POS	0x08
-
-/*
- * Category 6 - White Balance
- */
-
-/* Auto Whitebalance locking */
-#define AWB_LOCK		I2C_REG(CAT_WB, 0x00, 1)
-#define REG_AWB_UNLOCK		0x00
-#define REG_AWB_LOCK		0x01
-
-#define AWB_MODE		I2C_REG(CAT_WB, 0x02, 1)
-#define REG_AWB_AUTO		0x01	/* AWB off */
-#define REG_AWB_PRESET		0x02	/* AWB preset */
-
-/* Manual WB (preset) */
-#define AWB_MANUAL		I2C_REG(CAT_WB, 0x03, 1)
-#define REG_AWB_INCANDESCENT	0x01
-#define REG_AWB_FLUORESCENT_1	0x02
-#define REG_AWB_FLUORESCENT_2	0x03
-#define REG_AWB_DAYLIGHT	0x04
-#define REG_AWB_CLOUDY		0x05
-#define REG_AWB_SHADE		0x06
-#define REG_AWB_HORIZON		0x07
-#define REG_AWB_LEDLIGHT	0x09
-
-/*
- * Category 7 - EXIF information
- */
-#define EXIF_INFO_EXPTIME_NU	I2C_REG(CAT_EXIF, 0x00, 4)
-#define EXIF_INFO_EXPTIME_DE	I2C_REG(CAT_EXIF, 0x04, 4)
-#define EXIF_INFO_TV_NU		I2C_REG(CAT_EXIF, 0x08, 4)
-#define EXIF_INFO_TV_DE		I2C_REG(CAT_EXIF, 0x0c, 4)
-#define EXIF_INFO_AV_NU		I2C_REG(CAT_EXIF, 0x10, 4)
-#define EXIF_INFO_AV_DE		I2C_REG(CAT_EXIF, 0x14, 4)
-#define EXIF_INFO_BV_NU		I2C_REG(CAT_EXIF, 0x18, 4)
-#define EXIF_INFO_BV_DE		I2C_REG(CAT_EXIF, 0x1c, 4)
-#define EXIF_INFO_EBV_NU	I2C_REG(CAT_EXIF, 0x20, 4)
-#define EXIF_INFO_EBV_DE	I2C_REG(CAT_EXIF, 0x24, 4)
-#define EXIF_INFO_ISO		I2C_REG(CAT_EXIF, 0x28, 2)
-#define EXIF_INFO_FLASH		I2C_REG(CAT_EXIF, 0x2a, 2)
-#define EXIF_INFO_SDR		I2C_REG(CAT_EXIF, 0x2c, 2)
-#define EXIF_INFO_QVAL		I2C_REG(CAT_EXIF, 0x2e, 2)
-
-/*
- * Category 9 - Face Detection
- */
-#define FD_CTL			I2C_REG(CAT_FD, 0x00, 1)
-#define BIT_FD_EN		0
-#define BIT_FD_DRAW_FACE_FRAME	4
-#define BIT_FD_DRAW_SMILE_LVL	6
-#define REG_FD(shift)		(1 << shift)
-#define REG_FD_OFF		0x0
-
-/*
- * Category A - Lens Parameter
- */
-#define AF_MODE			I2C_REG(CAT_LENS, 0x01, 1)
-#define REG_AF_NORMAL		0x00	/* Normal AF, one time */
-#define REG_AF_MACRO		0x01	/* Macro AF, one time */
-#define REG_AF_POWEROFF		0x07
-
-#define AF_EXECUTE		I2C_REG(CAT_LENS, 0x02, 1)
-#define REG_AF_STOP		0x00
-#define REG_AF_EXE_AUTO		0x01
-#define REG_AF_EXE_CAF		0x02
-
-#define AF_STATUS		I2C_REG(CAT_LENS, 0x03, 1)
-#define REG_AF_FAIL		0x00
-#define REG_AF_SUCCESS		0x02
-#define REG_AF_IDLE		0x04
-#define REG_AF_BUSY		0x05
-
-#define AF_VERSION		I2C_REG(CAT_LENS, 0x0a, 1)
-
-/*
- * Category B - CAPTURE Parameter
- */
-#define CAPP_YUVOUT_MAIN	I2C_REG(CAT_CAPT_PARM, 0x00, 1)
-#define REG_YUV422		0x00
-#define REG_BAYER10		0x05
-#define REG_BAYER8		0x06
-#define REG_JPEG		0x10
-
-#define CAPP_MAIN_IMAGE_SIZE	I2C_REG(CAT_CAPT_PARM, 0x01, 1)
-#define CAPP_JPEG_SIZE_MAX	I2C_REG(CAT_CAPT_PARM, 0x0f, 4)
-#define CAPP_JPEG_RATIO		I2C_REG(CAT_CAPT_PARM, 0x17, 1)
-
-#define CAPP_MCC_MODE		I2C_REG(CAT_CAPT_PARM, 0x1d, 1)
-#define REG_MCC_OFF		0x00
-#define REG_MCC_NORMAL		0x01
-
-#define CAPP_WDR_EN		I2C_REG(CAT_CAPT_PARM, 0x2c, 1)
-#define REG_WDR_OFF		0x00
-#define REG_WDR_ON		0x01
-#define REG_WDR_AUTO		0x02
-
-#define CAPP_LIGHT_CTRL		I2C_REG(CAT_CAPT_PARM, 0x40, 1)
-#define REG_LIGHT_OFF		0x00
-#define REG_LIGHT_ON		0x01
-#define REG_LIGHT_AUTO		0x02
-
-#define CAPP_FLASH_CTRL		I2C_REG(CAT_CAPT_PARM, 0x41, 1)
-#define REG_FLASH_OFF		0x00
-#define REG_FLASH_ON		0x01
-#define REG_FLASH_AUTO		0x02
-
-/*
- * Category C - CAPTURE Control
- */
-#define CAPC_MODE		I2C_REG(CAT_CAPT_CTRL, 0x00, 1)
-#define REG_CAP_NONE		0x00
-#define REG_CAP_ANTI_SHAKE	0x02
-
-/* Select single- or multi-shot capture */
-#define CAPC_SEL_FRAME		I2C_REG(CAT_CAPT_CTRL, 0x06, 1)
-
-#define CAPC_START		I2C_REG(CAT_CAPT_CTRL, 0x09, 1)
-#define REG_CAP_START_MAIN	0x01
-#define REG_CAP_START_THUMB	0x03
-
-#define CAPC_IMAGE_SIZE		I2C_REG(CAT_CAPT_CTRL, 0x0d, 4)
-#define CAPC_THUMB_SIZE		I2C_REG(CAT_CAPT_CTRL, 0x11, 4)
-
-/*
- * Category F - Flash
- *
- * This mode provides functions about internal flash stuff and system startup.
- */
-
-/* Starts internal ARM core booting after power-up */
-#define FLASH_CAM_START		I2C_REG(CAT_FLASH, 0x12, 1)
-#define REG_START_ARM_BOOT	0x01	/* write value */
-#define REG_IN_FLASH_MODE	0x00	/* read value */
-
-#endif	/* M5MOLS_REG_H */
diff --git a/drivers/media/i2c/max9286.c b/drivers/media/i2c/max9286.c
index 701038d6d19b..13a986b88588 100644
--- a/drivers/media/i2c/max9286.c
+++ b/drivers/media/i2c/max9286.c
@@ -1122,6 +1122,7 @@ err_async:
 static void max9286_v4l2_unregister(struct max9286_priv *priv)
 {
 	fwnode_handle_put(priv->sd.fwnode);
+	v4l2_ctrl_handler_free(&priv->ctrls);
 	v4l2_async_unregister_subdev(&priv->sd);
 	max9286_v4l2_notifier_unregister(priv);
 }
diff --git a/drivers/media/i2c/mt9m032.c b/drivers/media/i2c/mt9m032.c
deleted file mode 100644
index 958cfdd73d57..000000000000
--- a/drivers/media/i2c/mt9m032.c
+++ /dev/null
@@ -1,891 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Driver for MT9M032 CMOS Image Sensor from Micron
- *
- * Copyright (C) 2010-2011 Lund Engineering
- * Contact: Gil Lund <gwlund@lundeng.com>
- * Author: Martin Hostettler <martin@neutronstar.dyndns.org>
- */
-
-#include <linux/delay.h>
-#include <linux/i2c.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/math64.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/slab.h>
-#include <linux/v4l2-mediabus.h>
-
-#include <media/media-entity.h>
-#include <media/i2c/mt9m032.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-subdev.h>
-
-#include "aptina-pll.h"
-
-/*
- * width and height include active boundary and black parts
- *
- * column    0-  15 active boundary
- * column   16-1455 image
- * column 1456-1471 active boundary
- * column 1472-1599 black
- *
- * row       0-  51 black
- * row      53-  59 active boundary
- * row      60-1139 image
- * row    1140-1147 active boundary
- * row    1148-1151 black
- */
-
-#define MT9M032_PIXEL_ARRAY_WIDTH			1600
-#define MT9M032_PIXEL_ARRAY_HEIGHT			1152
-
-#define MT9M032_CHIP_VERSION				0x00
-#define		MT9M032_CHIP_VERSION_VALUE		0x1402
-#define MT9M032_ROW_START				0x01
-#define		MT9M032_ROW_START_MIN			0
-#define		MT9M032_ROW_START_MAX			1152
-#define		MT9M032_ROW_START_DEF			60
-#define MT9M032_COLUMN_START				0x02
-#define		MT9M032_COLUMN_START_MIN		0
-#define		MT9M032_COLUMN_START_MAX		1600
-#define		MT9M032_COLUMN_START_DEF		16
-#define MT9M032_ROW_SIZE				0x03
-#define		MT9M032_ROW_SIZE_MIN			32
-#define		MT9M032_ROW_SIZE_MAX			1152
-#define		MT9M032_ROW_SIZE_DEF			1080
-#define MT9M032_COLUMN_SIZE				0x04
-#define		MT9M032_COLUMN_SIZE_MIN			32
-#define		MT9M032_COLUMN_SIZE_MAX			1600
-#define		MT9M032_COLUMN_SIZE_DEF			1440
-#define MT9M032_HBLANK					0x05
-#define MT9M032_VBLANK					0x06
-#define		MT9M032_VBLANK_MAX			0x7ff
-#define MT9M032_SHUTTER_WIDTH_HIGH			0x08
-#define MT9M032_SHUTTER_WIDTH_LOW			0x09
-#define		MT9M032_SHUTTER_WIDTH_MIN		1
-#define		MT9M032_SHUTTER_WIDTH_MAX		1048575
-#define		MT9M032_SHUTTER_WIDTH_DEF		1943
-#define MT9M032_PIX_CLK_CTRL				0x0a
-#define		MT9M032_PIX_CLK_CTRL_INV_PIXCLK		0x8000
-#define MT9M032_RESTART					0x0b
-#define MT9M032_RESET					0x0d
-#define MT9M032_PLL_CONFIG1				0x11
-#define		MT9M032_PLL_CONFIG1_PREDIV_MASK		0x3f
-#define		MT9M032_PLL_CONFIG1_MUL_SHIFT		8
-#define MT9M032_READ_MODE1				0x1e
-#define		MT9M032_READ_MODE1_OUTPUT_BAD_FRAMES	(1 << 13)
-#define		MT9M032_READ_MODE1_MAINTAIN_FRAME_RATE	(1 << 12)
-#define		MT9M032_READ_MODE1_XOR_LINE_VALID	(1 << 11)
-#define		MT9M032_READ_MODE1_CONT_LINE_VALID	(1 << 10)
-#define		MT9M032_READ_MODE1_INVERT_TRIGGER	(1 << 9)
-#define		MT9M032_READ_MODE1_SNAPSHOT		(1 << 8)
-#define		MT9M032_READ_MODE1_GLOBAL_RESET		(1 << 7)
-#define		MT9M032_READ_MODE1_BULB_EXPOSURE	(1 << 6)
-#define		MT9M032_READ_MODE1_INVERT_STROBE	(1 << 5)
-#define		MT9M032_READ_MODE1_STROBE_ENABLE	(1 << 4)
-#define		MT9M032_READ_MODE1_STROBE_START_TRIG1	(0 << 2)
-#define		MT9M032_READ_MODE1_STROBE_START_EXP	(1 << 2)
-#define		MT9M032_READ_MODE1_STROBE_START_SHUTTER	(2 << 2)
-#define		MT9M032_READ_MODE1_STROBE_START_TRIG2	(3 << 2)
-#define		MT9M032_READ_MODE1_STROBE_END_TRIG1	(0 << 0)
-#define		MT9M032_READ_MODE1_STROBE_END_EXP	(1 << 0)
-#define		MT9M032_READ_MODE1_STROBE_END_SHUTTER	(2 << 0)
-#define		MT9M032_READ_MODE1_STROBE_END_TRIG2	(3 << 0)
-#define MT9M032_READ_MODE2				0x20
-#define		MT9M032_READ_MODE2_VFLIP_SHIFT		15
-#define		MT9M032_READ_MODE2_HFLIP_SHIFT		14
-#define		MT9M032_READ_MODE2_ROW_BLC		0x40
-#define MT9M032_GAIN_GREEN1				0x2b
-#define MT9M032_GAIN_BLUE				0x2c
-#define MT9M032_GAIN_RED				0x2d
-#define MT9M032_GAIN_GREEN2				0x2e
-
-/* write only */
-#define MT9M032_GAIN_ALL				0x35
-#define		MT9M032_GAIN_DIGITAL_MASK		0x7f
-#define		MT9M032_GAIN_DIGITAL_SHIFT		8
-#define		MT9M032_GAIN_AMUL_SHIFT			6
-#define		MT9M032_GAIN_ANALOG_MASK		0x3f
-#define MT9M032_FORMATTER1				0x9e
-#define		MT9M032_FORMATTER1_PLL_P1_6		(1 << 8)
-#define		MT9M032_FORMATTER1_PARALLEL		(1 << 12)
-#define MT9M032_FORMATTER2				0x9f
-#define		MT9M032_FORMATTER2_DOUT_EN		0x1000
-#define		MT9M032_FORMATTER2_PIXCLK_EN		0x2000
-
-/*
- * The available MT9M032 datasheet is missing documentation for register 0x10
- * MT9P031 seems to be close enough, so use constants from that datasheet for
- * now.
- * But keep the name MT9P031 to remind us, that this isn't really confirmed
- * for this sensor.
- */
-#define MT9P031_PLL_CONTROL				0x10
-#define		MT9P031_PLL_CONTROL_PWROFF		0x0050
-#define		MT9P031_PLL_CONTROL_PWRON		0x0051
-#define		MT9P031_PLL_CONTROL_USEPLL		0x0052
-
-struct mt9m032 {
-	struct v4l2_subdev subdev;
-	struct media_pad pad;
-	struct mt9m032_platform_data *pdata;
-
-	unsigned int pix_clock;
-
-	struct v4l2_ctrl_handler ctrls;
-	struct {
-		struct v4l2_ctrl *hflip;
-		struct v4l2_ctrl *vflip;
-	};
-
-	struct mutex lock; /* Protects streaming, format, interval and crop */
-
-	bool streaming;
-
-	struct v4l2_mbus_framefmt format;
-	struct v4l2_rect crop;
-	struct v4l2_fract frame_interval;
-};
-
-#define to_mt9m032(sd)	container_of(sd, struct mt9m032, subdev)
-#define to_dev(sensor) \
-	(&((struct i2c_client *)v4l2_get_subdevdata(&(sensor)->subdev))->dev)
-
-static int mt9m032_read(struct i2c_client *client, u8 reg)
-{
-	return i2c_smbus_read_word_swapped(client, reg);
-}
-
-static int mt9m032_write(struct i2c_client *client, u8 reg, const u16 data)
-{
-	return i2c_smbus_write_word_swapped(client, reg, data);
-}
-
-static u32 mt9m032_row_time(struct mt9m032 *sensor, unsigned int width)
-{
-	unsigned int effective_width;
-	u32 ns;
-
-	effective_width = width + 716; /* empirical value */
-	ns = div_u64(1000000000ULL * effective_width, sensor->pix_clock);
-	dev_dbg(to_dev(sensor),	"MT9M032 line time: %u ns\n", ns);
-	return ns;
-}
-
-static int mt9m032_update_timing(struct mt9m032 *sensor,
-				 struct v4l2_fract *interval)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&sensor->subdev);
-	struct v4l2_rect *crop = &sensor->crop;
-	unsigned int min_vblank;
-	unsigned int vblank;
-	u32 row_time;
-
-	if (!interval)
-		interval = &sensor->frame_interval;
-
-	row_time = mt9m032_row_time(sensor, crop->width);
-
-	vblank = div_u64(1000000000ULL * interval->numerator,
-			 (u64)row_time * interval->denominator)
-	       - crop->height;
-
-	if (vblank > MT9M032_VBLANK_MAX) {
-		/* hardware limits to 11 bit values */
-		interval->denominator = 1000;
-		interval->numerator =
-			div_u64((crop->height + MT9M032_VBLANK_MAX) *
-				(u64)row_time * interval->denominator,
-				1000000000ULL);
-		vblank = div_u64(1000000000ULL * interval->numerator,
-				 (u64)row_time * interval->denominator)
-		       - crop->height;
-	}
-	/* enforce minimal 1.6ms blanking time. */
-	min_vblank = 1600000 / row_time;
-	vblank = clamp_t(unsigned int, vblank, min_vblank, MT9M032_VBLANK_MAX);
-
-	return mt9m032_write(client, MT9M032_VBLANK, vblank);
-}
-
-static int mt9m032_update_geom_timing(struct mt9m032 *sensor)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&sensor->subdev);
-	int ret;
-
-	ret = mt9m032_write(client, MT9M032_COLUMN_SIZE,
-			    sensor->crop.width - 1);
-	if (!ret)
-		ret = mt9m032_write(client, MT9M032_ROW_SIZE,
-				    sensor->crop.height - 1);
-	if (!ret)
-		ret = mt9m032_write(client, MT9M032_COLUMN_START,
-				    sensor->crop.left);
-	if (!ret)
-		ret = mt9m032_write(client, MT9M032_ROW_START,
-				    sensor->crop.top);
-	if (!ret)
-		ret = mt9m032_update_timing(sensor, NULL);
-	return ret;
-}
-
-static int update_formatter2(struct mt9m032 *sensor, bool streaming)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&sensor->subdev);
-	u16 reg_val =   MT9M032_FORMATTER2_DOUT_EN
-		      | 0x0070;  /* parts reserved! */
-				 /* possibly for changing to 14-bit mode */
-
-	if (streaming)
-		reg_val |= MT9M032_FORMATTER2_PIXCLK_EN;   /* pixclock enable */
-
-	return mt9m032_write(client, MT9M032_FORMATTER2, reg_val);
-}
-
-static int mt9m032_setup_pll(struct mt9m032 *sensor)
-{
-	static const struct aptina_pll_limits limits = {
-		.ext_clock_min = 8000000,
-		.ext_clock_max = 16500000,
-		.int_clock_min = 2000000,
-		.int_clock_max = 24000000,
-		.out_clock_min = 322000000,
-		.out_clock_max = 693000000,
-		.pix_clock_max = 99000000,
-		.n_min = 1,
-		.n_max = 64,
-		.m_min = 16,
-		.m_max = 255,
-		.p1_min = 6,
-		.p1_max = 7,
-	};
-
-	struct i2c_client *client = v4l2_get_subdevdata(&sensor->subdev);
-	struct mt9m032_platform_data *pdata = sensor->pdata;
-	struct aptina_pll pll;
-	u16 reg_val;
-	int ret;
-
-	pll.ext_clock = pdata->ext_clock;
-	pll.pix_clock = pdata->pix_clock;
-
-	ret = aptina_pll_calculate(&client->dev, &limits, &pll);
-	if (ret < 0)
-		return ret;
-
-	sensor->pix_clock = pdata->pix_clock;
-
-	ret = mt9m032_write(client, MT9M032_PLL_CONFIG1,
-			    (pll.m << MT9M032_PLL_CONFIG1_MUL_SHIFT) |
-			    ((pll.n - 1) & MT9M032_PLL_CONFIG1_PREDIV_MASK));
-	if (!ret)
-		ret = mt9m032_write(client, MT9P031_PLL_CONTROL,
-				    MT9P031_PLL_CONTROL_PWRON |
-				    MT9P031_PLL_CONTROL_USEPLL);
-	if (!ret)		/* more reserved, Continuous, Master Mode */
-		ret = mt9m032_write(client, MT9M032_READ_MODE1, 0x8000 |
-				    MT9M032_READ_MODE1_STROBE_START_EXP |
-				    MT9M032_READ_MODE1_STROBE_END_SHUTTER);
-	if (!ret) {
-		reg_val = (pll.p1 == 6 ? MT9M032_FORMATTER1_PLL_P1_6 : 0)
-			| MT9M032_FORMATTER1_PARALLEL | 0x001e; /* 14-bit */
-		ret = mt9m032_write(client, MT9M032_FORMATTER1, reg_val);
-	}
-
-	return ret;
-}
-
-/* -----------------------------------------------------------------------------
- * Subdev pad operations
- */
-
-static int mt9m032_enum_mbus_code(struct v4l2_subdev *subdev,
-				  struct v4l2_subdev_state *sd_state,
-				  struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index != 0)
-		return -EINVAL;
-
-	code->code = MEDIA_BUS_FMT_Y8_1X8;
-	return 0;
-}
-
-static int mt9m032_enum_frame_size(struct v4l2_subdev *subdev,
-				   struct v4l2_subdev_state *sd_state,
-				   struct v4l2_subdev_frame_size_enum *fse)
-{
-	if (fse->index != 0 || fse->code != MEDIA_BUS_FMT_Y8_1X8)
-		return -EINVAL;
-
-	fse->min_width = MT9M032_COLUMN_SIZE_DEF;
-	fse->max_width = MT9M032_COLUMN_SIZE_DEF;
-	fse->min_height = MT9M032_ROW_SIZE_DEF;
-	fse->max_height = MT9M032_ROW_SIZE_DEF;
-
-	return 0;
-}
-
-/**
- * __mt9m032_get_pad_crop() - get crop rect
- * @sensor: pointer to the sensor struct
- * @sd_state: v4l2_subdev_state for getting the try crop rect from
- * @which: select try or active crop rect
- *
- * Returns a pointer the current active or fh relative try crop rect
- */
-static struct v4l2_rect *
-__mt9m032_get_pad_crop(struct mt9m032 *sensor,
-		       struct v4l2_subdev_state *sd_state,
-		       enum v4l2_subdev_format_whence which)
-{
-	switch (which) {
-	case V4L2_SUBDEV_FORMAT_TRY:
-		return v4l2_subdev_get_try_crop(&sensor->subdev, sd_state, 0);
-	case V4L2_SUBDEV_FORMAT_ACTIVE:
-		return &sensor->crop;
-	default:
-		return NULL;
-	}
-}
-
-/**
- * __mt9m032_get_pad_format() - get format
- * @sensor: pointer to the sensor struct
- * @sd_state: v4l2_subdev_state for getting the try format from
- * @which: select try or active format
- *
- * Returns a pointer the current active or fh relative try format
- */
-static struct v4l2_mbus_framefmt *
-__mt9m032_get_pad_format(struct mt9m032 *sensor,
-			 struct v4l2_subdev_state *sd_state,
-			 enum v4l2_subdev_format_whence which)
-{
-	switch (which) {
-	case V4L2_SUBDEV_FORMAT_TRY:
-		return v4l2_subdev_get_try_format(&sensor->subdev, sd_state,
-						  0);
-	case V4L2_SUBDEV_FORMAT_ACTIVE:
-		return &sensor->format;
-	default:
-		return NULL;
-	}
-}
-
-static int mt9m032_get_pad_format(struct v4l2_subdev *subdev,
-				  struct v4l2_subdev_state *sd_state,
-				  struct v4l2_subdev_format *fmt)
-{
-	struct mt9m032 *sensor = to_mt9m032(subdev);
-
-	mutex_lock(&sensor->lock);
-	fmt->format = *__mt9m032_get_pad_format(sensor, sd_state, fmt->which);
-	mutex_unlock(&sensor->lock);
-
-	return 0;
-}
-
-static int mt9m032_set_pad_format(struct v4l2_subdev *subdev,
-				  struct v4l2_subdev_state *sd_state,
-				  struct v4l2_subdev_format *fmt)
-{
-	struct mt9m032 *sensor = to_mt9m032(subdev);
-	int ret;
-
-	mutex_lock(&sensor->lock);
-
-	if (sensor->streaming && fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
-		ret = -EBUSY;
-		goto done;
-	}
-
-	/* Scaling is not supported, the format is thus fixed. */
-	fmt->format = *__mt9m032_get_pad_format(sensor, sd_state, fmt->which);
-	ret = 0;
-
-done:
-	mutex_unlock(&sensor->lock);
-	return ret;
-}
-
-static int mt9m032_get_pad_selection(struct v4l2_subdev *subdev,
-				     struct v4l2_subdev_state *sd_state,
-				     struct v4l2_subdev_selection *sel)
-{
-	struct mt9m032 *sensor = to_mt9m032(subdev);
-
-	if (sel->target != V4L2_SEL_TGT_CROP)
-		return -EINVAL;
-
-	mutex_lock(&sensor->lock);
-	sel->r = *__mt9m032_get_pad_crop(sensor, sd_state, sel->which);
-	mutex_unlock(&sensor->lock);
-
-	return 0;
-}
-
-static int mt9m032_set_pad_selection(struct v4l2_subdev *subdev,
-				     struct v4l2_subdev_state *sd_state,
-				     struct v4l2_subdev_selection *sel)
-{
-	struct mt9m032 *sensor = to_mt9m032(subdev);
-	struct v4l2_mbus_framefmt *format;
-	struct v4l2_rect *__crop;
-	struct v4l2_rect rect;
-	int ret = 0;
-
-	if (sel->target != V4L2_SEL_TGT_CROP)
-		return -EINVAL;
-
-	mutex_lock(&sensor->lock);
-
-	if (sensor->streaming && sel->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
-		ret = -EBUSY;
-		goto done;
-	}
-
-	/* Clamp the crop rectangle boundaries and align them to a multiple of 2
-	 * pixels to ensure a GRBG Bayer pattern.
-	 */
-	rect.left = clamp(ALIGN(sel->r.left, 2), MT9M032_COLUMN_START_MIN,
-			  MT9M032_COLUMN_START_MAX);
-	rect.top = clamp(ALIGN(sel->r.top, 2), MT9M032_ROW_START_MIN,
-			 MT9M032_ROW_START_MAX);
-	rect.width = clamp_t(unsigned int, ALIGN(sel->r.width, 2),
-			     MT9M032_COLUMN_SIZE_MIN, MT9M032_COLUMN_SIZE_MAX);
-	rect.height = clamp_t(unsigned int, ALIGN(sel->r.height, 2),
-			      MT9M032_ROW_SIZE_MIN, MT9M032_ROW_SIZE_MAX);
-
-	rect.width = min_t(unsigned int, rect.width,
-			   MT9M032_PIXEL_ARRAY_WIDTH - rect.left);
-	rect.height = min_t(unsigned int, rect.height,
-			    MT9M032_PIXEL_ARRAY_HEIGHT - rect.top);
-
-	__crop = __mt9m032_get_pad_crop(sensor, sd_state, sel->which);
-
-	if (rect.width != __crop->width || rect.height != __crop->height) {
-		/* Reset the output image size if the crop rectangle size has
-		 * been modified.
-		 */
-		format = __mt9m032_get_pad_format(sensor, sd_state,
-						  sel->which);
-		format->width = rect.width;
-		format->height = rect.height;
-	}
-
-	*__crop = rect;
-	sel->r = rect;
-
-	if (sel->which == V4L2_SUBDEV_FORMAT_ACTIVE)
-		ret = mt9m032_update_geom_timing(sensor);
-
-done:
-	mutex_unlock(&sensor->lock);
-	return ret;
-}
-
-static int mt9m032_get_frame_interval(struct v4l2_subdev *subdev,
-				      struct v4l2_subdev_frame_interval *fi)
-{
-	struct mt9m032 *sensor = to_mt9m032(subdev);
-
-	mutex_lock(&sensor->lock);
-	memset(fi, 0, sizeof(*fi));
-	fi->interval = sensor->frame_interval;
-	mutex_unlock(&sensor->lock);
-
-	return 0;
-}
-
-static int mt9m032_set_frame_interval(struct v4l2_subdev *subdev,
-				      struct v4l2_subdev_frame_interval *fi)
-{
-	struct mt9m032 *sensor = to_mt9m032(subdev);
-	int ret;
-
-	mutex_lock(&sensor->lock);
-
-	if (sensor->streaming) {
-		ret = -EBUSY;
-		goto done;
-	}
-
-	/* Avoid divisions by 0. */
-	if (fi->interval.denominator == 0)
-		fi->interval.denominator = 1;
-
-	ret = mt9m032_update_timing(sensor, &fi->interval);
-	if (!ret)
-		sensor->frame_interval = fi->interval;
-
-done:
-	mutex_unlock(&sensor->lock);
-	return ret;
-}
-
-static int mt9m032_s_stream(struct v4l2_subdev *subdev, int streaming)
-{
-	struct mt9m032 *sensor = to_mt9m032(subdev);
-	int ret;
-
-	mutex_lock(&sensor->lock);
-	ret = update_formatter2(sensor, streaming);
-	if (!ret)
-		sensor->streaming = streaming;
-	mutex_unlock(&sensor->lock);
-
-	return ret;
-}
-
-/* -----------------------------------------------------------------------------
- * V4L2 subdev core operations
- */
-
-#ifdef CONFIG_VIDEO_ADV_DEBUG
-static int mt9m032_g_register(struct v4l2_subdev *sd,
-			      struct v4l2_dbg_register *reg)
-{
-	struct mt9m032 *sensor = to_mt9m032(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(&sensor->subdev);
-	int val;
-
-	if (reg->reg > 0xff)
-		return -EINVAL;
-
-	val = mt9m032_read(client, reg->reg);
-	if (val < 0)
-		return -EIO;
-
-	reg->size = 2;
-	reg->val = val;
-
-	return 0;
-}
-
-static int mt9m032_s_register(struct v4l2_subdev *sd,
-			      const struct v4l2_dbg_register *reg)
-{
-	struct mt9m032 *sensor = to_mt9m032(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(&sensor->subdev);
-
-	if (reg->reg > 0xff)
-		return -EINVAL;
-
-	return mt9m032_write(client, reg->reg, reg->val);
-}
-#endif
-
-/* -----------------------------------------------------------------------------
- * V4L2 subdev control operations
- */
-
-static int update_read_mode2(struct mt9m032 *sensor, bool vflip, bool hflip)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&sensor->subdev);
-	int reg_val = (vflip << MT9M032_READ_MODE2_VFLIP_SHIFT)
-		    | (hflip << MT9M032_READ_MODE2_HFLIP_SHIFT)
-		    | MT9M032_READ_MODE2_ROW_BLC
-		    | 0x0007;
-
-	return mt9m032_write(client, MT9M032_READ_MODE2, reg_val);
-}
-
-static int mt9m032_set_gain(struct mt9m032 *sensor, s32 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&sensor->subdev);
-	int digital_gain_val;	/* in 1/8th (0..127) */
-	int analog_mul;		/* 0 or 1 */
-	int analog_gain_val;	/* in 1/16th. (0..63) */
-	u16 reg_val;
-
-	digital_gain_val = 51; /* from setup example */
-
-	if (val < 63) {
-		analog_mul = 0;
-		analog_gain_val = val;
-	} else {
-		analog_mul = 1;
-		analog_gain_val = val / 2;
-	}
-
-	/* a_gain = (1 + analog_mul) + (analog_gain_val + 1) / 16 */
-	/* overall_gain = a_gain * (1 + digital_gain_val / 8) */
-
-	reg_val = ((digital_gain_val & MT9M032_GAIN_DIGITAL_MASK)
-		   << MT9M032_GAIN_DIGITAL_SHIFT)
-		| ((analog_mul & 1) << MT9M032_GAIN_AMUL_SHIFT)
-		| (analog_gain_val & MT9M032_GAIN_ANALOG_MASK);
-
-	return mt9m032_write(client, MT9M032_GAIN_ALL, reg_val);
-}
-
-static int mt9m032_try_ctrl(struct v4l2_ctrl *ctrl)
-{
-	if (ctrl->id == V4L2_CID_GAIN && ctrl->val >= 63) {
-		/* round because of multiplier used for values >= 63 */
-		ctrl->val &= ~1;
-	}
-
-	return 0;
-}
-
-static int mt9m032_set_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct mt9m032 *sensor =
-		container_of(ctrl->handler, struct mt9m032, ctrls);
-	struct i2c_client *client = v4l2_get_subdevdata(&sensor->subdev);
-	int ret;
-
-	switch (ctrl->id) {
-	case V4L2_CID_GAIN:
-		return mt9m032_set_gain(sensor, ctrl->val);
-
-	case V4L2_CID_HFLIP:
-	/* case V4L2_CID_VFLIP: -- In the same cluster */
-		return update_read_mode2(sensor, sensor->vflip->val,
-					 sensor->hflip->val);
-
-	case V4L2_CID_EXPOSURE:
-		ret = mt9m032_write(client, MT9M032_SHUTTER_WIDTH_HIGH,
-				    (ctrl->val >> 16) & 0xffff);
-		if (ret < 0)
-			return ret;
-
-		return mt9m032_write(client, MT9M032_SHUTTER_WIDTH_LOW,
-				     ctrl->val & 0xffff);
-	}
-
-	return 0;
-}
-
-static const struct v4l2_ctrl_ops mt9m032_ctrl_ops = {
-	.s_ctrl = mt9m032_set_ctrl,
-	.try_ctrl = mt9m032_try_ctrl,
-};
-
-/* -------------------------------------------------------------------------- */
-
-static const struct v4l2_subdev_core_ops mt9m032_core_ops = {
-#ifdef CONFIG_VIDEO_ADV_DEBUG
-	.g_register = mt9m032_g_register,
-	.s_register = mt9m032_s_register,
-#endif
-};
-
-static const struct v4l2_subdev_video_ops mt9m032_video_ops = {
-	.s_stream = mt9m032_s_stream,
-	.g_frame_interval = mt9m032_get_frame_interval,
-	.s_frame_interval = mt9m032_set_frame_interval,
-};
-
-static const struct v4l2_subdev_pad_ops mt9m032_pad_ops = {
-	.enum_mbus_code = mt9m032_enum_mbus_code,
-	.enum_frame_size = mt9m032_enum_frame_size,
-	.get_fmt = mt9m032_get_pad_format,
-	.set_fmt = mt9m032_set_pad_format,
-	.set_selection = mt9m032_set_pad_selection,
-	.get_selection = mt9m032_get_pad_selection,
-};
-
-static const struct v4l2_subdev_ops mt9m032_ops = {
-	.core = &mt9m032_core_ops,
-	.video = &mt9m032_video_ops,
-	.pad = &mt9m032_pad_ops,
-};
-
-/* -----------------------------------------------------------------------------
- * Driver initialization and probing
- */
-
-static int mt9m032_probe(struct i2c_client *client)
-{
-	struct mt9m032_platform_data *pdata = client->dev.platform_data;
-	struct i2c_adapter *adapter = client->adapter;
-	struct mt9m032 *sensor;
-	int chip_version;
-	int ret;
-
-	if (pdata == NULL) {
-		dev_err(&client->dev, "No platform data\n");
-		return -EINVAL;
-	}
-
-	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_WORD_DATA)) {
-		dev_warn(&client->dev,
-			 "I2C-Adapter doesn't support I2C_FUNC_SMBUS_WORD\n");
-		return -EIO;
-	}
-
-	if (!client->dev.platform_data)
-		return -ENODEV;
-
-	sensor = devm_kzalloc(&client->dev, sizeof(*sensor), GFP_KERNEL);
-	if (sensor == NULL)
-		return -ENOMEM;
-
-	mutex_init(&sensor->lock);
-
-	sensor->pdata = pdata;
-
-	v4l2_i2c_subdev_init(&sensor->subdev, client, &mt9m032_ops);
-	sensor->subdev.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-
-	chip_version = mt9m032_read(client, MT9M032_CHIP_VERSION);
-	if (chip_version != MT9M032_CHIP_VERSION_VALUE) {
-		dev_err(&client->dev, "MT9M032 not detected, wrong version "
-			"0x%04x\n", chip_version);
-		ret = -ENODEV;
-		goto error_sensor;
-	}
-
-	dev_info(&client->dev, "MT9M032 detected at address 0x%02x\n",
-		 client->addr);
-
-	sensor->frame_interval.numerator = 1;
-	sensor->frame_interval.denominator = 30;
-
-	sensor->crop.left = MT9M032_COLUMN_START_DEF;
-	sensor->crop.top = MT9M032_ROW_START_DEF;
-	sensor->crop.width = MT9M032_COLUMN_SIZE_DEF;
-	sensor->crop.height = MT9M032_ROW_SIZE_DEF;
-
-	sensor->format.width = sensor->crop.width;
-	sensor->format.height = sensor->crop.height;
-	sensor->format.code = MEDIA_BUS_FMT_Y8_1X8;
-	sensor->format.field = V4L2_FIELD_NONE;
-	sensor->format.colorspace = V4L2_COLORSPACE_SRGB;
-
-	v4l2_ctrl_handler_init(&sensor->ctrls, 5);
-
-	v4l2_ctrl_new_std(&sensor->ctrls, &mt9m032_ctrl_ops,
-			  V4L2_CID_GAIN, 0, 127, 1, 64);
-
-	sensor->hflip = v4l2_ctrl_new_std(&sensor->ctrls,
-					  &mt9m032_ctrl_ops,
-					  V4L2_CID_HFLIP, 0, 1, 1, 0);
-	sensor->vflip = v4l2_ctrl_new_std(&sensor->ctrls,
-					  &mt9m032_ctrl_ops,
-					  V4L2_CID_VFLIP, 0, 1, 1, 0);
-
-	v4l2_ctrl_new_std(&sensor->ctrls, &mt9m032_ctrl_ops,
-			  V4L2_CID_EXPOSURE, MT9M032_SHUTTER_WIDTH_MIN,
-			  MT9M032_SHUTTER_WIDTH_MAX, 1,
-			  MT9M032_SHUTTER_WIDTH_DEF);
-	v4l2_ctrl_new_std(&sensor->ctrls, &mt9m032_ctrl_ops,
-			  V4L2_CID_PIXEL_RATE, pdata->pix_clock,
-			  pdata->pix_clock, 1, pdata->pix_clock);
-
-	if (sensor->ctrls.error) {
-		ret = sensor->ctrls.error;
-		dev_err(&client->dev, "control initialization error %d\n", ret);
-		goto error_ctrl;
-	}
-
-	v4l2_ctrl_cluster(2, &sensor->hflip);
-
-	sensor->subdev.ctrl_handler = &sensor->ctrls;
-	sensor->subdev.entity.function = MEDIA_ENT_F_CAM_SENSOR;
-	sensor->pad.flags = MEDIA_PAD_FL_SOURCE;
-	ret = media_entity_pads_init(&sensor->subdev.entity, 1, &sensor->pad);
-	if (ret < 0)
-		goto error_ctrl;
-
-	ret = mt9m032_write(client, MT9M032_RESET, 1);	/* reset on */
-	if (ret < 0)
-		goto error_entity;
-	ret = mt9m032_write(client, MT9M032_RESET, 0);	/* reset off */
-	if (ret < 0)
-		goto error_entity;
-
-	ret = mt9m032_setup_pll(sensor);
-	if (ret < 0)
-		goto error_entity;
-	usleep_range(10000, 11000);
-
-	ret = v4l2_ctrl_handler_setup(&sensor->ctrls);
-	if (ret < 0)
-		goto error_entity;
-
-	/* SIZE */
-	ret = mt9m032_update_geom_timing(sensor);
-	if (ret < 0)
-		goto error_entity;
-
-	ret = mt9m032_write(client, 0x41, 0x0000);	/* reserved !!! */
-	if (ret < 0)
-		goto error_entity;
-	ret = mt9m032_write(client, 0x42, 0x0003);	/* reserved !!! */
-	if (ret < 0)
-		goto error_entity;
-	ret = mt9m032_write(client, 0x43, 0x0003);	/* reserved !!! */
-	if (ret < 0)
-		goto error_entity;
-	ret = mt9m032_write(client, 0x7f, 0x0000);	/* reserved !!! */
-	if (ret < 0)
-		goto error_entity;
-	if (sensor->pdata->invert_pixclock) {
-		ret = mt9m032_write(client, MT9M032_PIX_CLK_CTRL,
-				    MT9M032_PIX_CLK_CTRL_INV_PIXCLK);
-		if (ret < 0)
-			goto error_entity;
-	}
-
-	ret = mt9m032_write(client, MT9M032_RESTART, 1); /* Restart on */
-	if (ret < 0)
-		goto error_entity;
-	msleep(100);
-	ret = mt9m032_write(client, MT9M032_RESTART, 0); /* Restart off */
-	if (ret < 0)
-		goto error_entity;
-	msleep(100);
-	ret = update_formatter2(sensor, false);
-	if (ret < 0)
-		goto error_entity;
-
-	return ret;
-
-error_entity:
-	media_entity_cleanup(&sensor->subdev.entity);
-error_ctrl:
-	v4l2_ctrl_handler_free(&sensor->ctrls);
-error_sensor:
-	mutex_destroy(&sensor->lock);
-	return ret;
-}
-
-static void mt9m032_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
-	struct mt9m032 *sensor = to_mt9m032(subdev);
-
-	v4l2_device_unregister_subdev(subdev);
-	v4l2_ctrl_handler_free(&sensor->ctrls);
-	media_entity_cleanup(&subdev->entity);
-	mutex_destroy(&sensor->lock);
-}
-
-static const struct i2c_device_id mt9m032_id_table[] = {
-	{ MT9M032_NAME, 0 },
-	{ }
-};
-
-MODULE_DEVICE_TABLE(i2c, mt9m032_id_table);
-
-static struct i2c_driver mt9m032_i2c_driver = {
-	.driver = {
-		.name = MT9M032_NAME,
-	},
-	.probe_new = mt9m032_probe,
-	.remove = mt9m032_remove,
-	.id_table = mt9m032_id_table,
-};
-
-module_i2c_driver(mt9m032_i2c_driver);
-
-MODULE_AUTHOR("Martin Hostettler <martin@neutronstar.dyndns.org>");
-MODULE_DESCRIPTION("MT9M032 camera sensor driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/i2c/mt9t001.c b/drivers/media/i2c/mt9t001.c
deleted file mode 100644
index c635ed11388a..000000000000
--- a/drivers/media/i2c/mt9t001.c
+++ /dev/null
@@ -1,992 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Driver for MT9T001 CMOS Image Sensor from Aptina (Micron)
- *
- * Copyright (C) 2010-2011, Laurent Pinchart <laurent.pinchart@ideasonboard.com>
- *
- * Based on the MT9M001 driver,
- *
- * Copyright (C) 2008, Guennadi Liakhovetski <kernel@pengutronix.de>
- */
-
-#include <linux/clk.h>
-#include <linux/i2c.h>
-#include <linux/log2.h>
-#include <linux/module.h>
-#include <linux/regulator/consumer.h>
-#include <linux/slab.h>
-#include <linux/videodev2.h>
-#include <linux/v4l2-mediabus.h>
-
-#include <media/i2c/mt9t001.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-subdev.h>
-
-#define MT9T001_PIXEL_ARRAY_HEIGHT			1568
-#define MT9T001_PIXEL_ARRAY_WIDTH			2112
-
-#define MT9T001_CHIP_VERSION				0x00
-#define		MT9T001_CHIP_ID				0x1621
-#define MT9T001_ROW_START				0x01
-#define		MT9T001_ROW_START_MIN			0
-#define		MT9T001_ROW_START_DEF			20
-#define		MT9T001_ROW_START_MAX			1534
-#define MT9T001_COLUMN_START				0x02
-#define		MT9T001_COLUMN_START_MIN		0
-#define		MT9T001_COLUMN_START_DEF		32
-#define		MT9T001_COLUMN_START_MAX		2046
-#define MT9T001_WINDOW_HEIGHT				0x03
-#define		MT9T001_WINDOW_HEIGHT_MIN		1
-#define		MT9T001_WINDOW_HEIGHT_DEF		1535
-#define		MT9T001_WINDOW_HEIGHT_MAX		1567
-#define MT9T001_WINDOW_WIDTH				0x04
-#define		MT9T001_WINDOW_WIDTH_MIN		1
-#define		MT9T001_WINDOW_WIDTH_DEF		2047
-#define		MT9T001_WINDOW_WIDTH_MAX		2111
-#define MT9T001_HORIZONTAL_BLANKING			0x05
-#define		MT9T001_HORIZONTAL_BLANKING_MIN		21
-#define		MT9T001_HORIZONTAL_BLANKING_MAX		1023
-#define MT9T001_VERTICAL_BLANKING			0x06
-#define		MT9T001_VERTICAL_BLANKING_MIN		3
-#define		MT9T001_VERTICAL_BLANKING_MAX		1023
-#define MT9T001_OUTPUT_CONTROL				0x07
-#define		MT9T001_OUTPUT_CONTROL_SYNC		(1 << 0)
-#define		MT9T001_OUTPUT_CONTROL_CHIP_ENABLE	(1 << 1)
-#define		MT9T001_OUTPUT_CONTROL_TEST_DATA	(1 << 6)
-#define		MT9T001_OUTPUT_CONTROL_DEF		0x0002
-#define MT9T001_SHUTTER_WIDTH_HIGH			0x08
-#define MT9T001_SHUTTER_WIDTH_LOW			0x09
-#define		MT9T001_SHUTTER_WIDTH_MIN		1
-#define		MT9T001_SHUTTER_WIDTH_DEF		1561
-#define		MT9T001_SHUTTER_WIDTH_MAX		(1024 * 1024)
-#define MT9T001_PIXEL_CLOCK				0x0a
-#define		MT9T001_PIXEL_CLOCK_INVERT		(1 << 15)
-#define		MT9T001_PIXEL_CLOCK_SHIFT_MASK		(7 << 8)
-#define		MT9T001_PIXEL_CLOCK_SHIFT_SHIFT		8
-#define		MT9T001_PIXEL_CLOCK_DIVIDE_MASK		(0x7f << 0)
-#define MT9T001_FRAME_RESTART				0x0b
-#define MT9T001_SHUTTER_DELAY				0x0c
-#define		MT9T001_SHUTTER_DELAY_MAX		2047
-#define MT9T001_RESET					0x0d
-#define MT9T001_READ_MODE1				0x1e
-#define		MT9T001_READ_MODE_SNAPSHOT		(1 << 8)
-#define		MT9T001_READ_MODE_STROBE_ENABLE		(1 << 9)
-#define		MT9T001_READ_MODE_STROBE_WIDTH		(1 << 10)
-#define		MT9T001_READ_MODE_STROBE_OVERRIDE	(1 << 11)
-#define MT9T001_READ_MODE2				0x20
-#define		MT9T001_READ_MODE_BAD_FRAMES		(1 << 0)
-#define		MT9T001_READ_MODE_LINE_VALID_CONTINUOUS	(1 << 9)
-#define		MT9T001_READ_MODE_LINE_VALID_FRAME	(1 << 10)
-#define MT9T001_READ_MODE3				0x21
-#define		MT9T001_READ_MODE_GLOBAL_RESET		(1 << 0)
-#define		MT9T001_READ_MODE_GHST_CTL		(1 << 1)
-#define MT9T001_ROW_ADDRESS_MODE			0x22
-#define		MT9T001_ROW_SKIP_MASK			(7 << 0)
-#define		MT9T001_ROW_BIN_MASK			(3 << 3)
-#define		MT9T001_ROW_BIN_SHIFT			3
-#define MT9T001_COLUMN_ADDRESS_MODE			0x23
-#define		MT9T001_COLUMN_SKIP_MASK		(7 << 0)
-#define		MT9T001_COLUMN_BIN_MASK			(3 << 3)
-#define		MT9T001_COLUMN_BIN_SHIFT		3
-#define MT9T001_GREEN1_GAIN				0x2b
-#define MT9T001_BLUE_GAIN				0x2c
-#define MT9T001_RED_GAIN				0x2d
-#define MT9T001_GREEN2_GAIN				0x2e
-#define MT9T001_TEST_DATA				0x32
-#define MT9T001_GLOBAL_GAIN				0x35
-#define		MT9T001_GLOBAL_GAIN_MIN			8
-#define		MT9T001_GLOBAL_GAIN_MAX			1024
-#define MT9T001_BLACK_LEVEL				0x49
-#define MT9T001_ROW_BLACK_DEFAULT_OFFSET		0x4b
-#define MT9T001_BLC_DELTA_THRESHOLDS			0x5d
-#define MT9T001_CAL_THRESHOLDS				0x5f
-#define MT9T001_GREEN1_OFFSET				0x60
-#define MT9T001_GREEN2_OFFSET				0x61
-#define MT9T001_BLACK_LEVEL_CALIBRATION			0x62
-#define		MT9T001_BLACK_LEVEL_OVERRIDE		(1 << 0)
-#define		MT9T001_BLACK_LEVEL_DISABLE_OFFSET	(1 << 1)
-#define		MT9T001_BLACK_LEVEL_RECALCULATE		(1 << 12)
-#define		MT9T001_BLACK_LEVEL_LOCK_RED_BLUE	(1 << 13)
-#define		MT9T001_BLACK_LEVEL_LOCK_GREEN		(1 << 14)
-#define MT9T001_RED_OFFSET				0x63
-#define MT9T001_BLUE_OFFSET				0x64
-
-struct mt9t001 {
-	struct v4l2_subdev subdev;
-	struct media_pad pad;
-
-	struct clk *clk;
-	struct regulator_bulk_data regulators[2];
-
-	struct mutex power_lock; /* lock to protect power_count */
-	int power_count;
-
-	struct v4l2_mbus_framefmt format;
-	struct v4l2_rect crop;
-
-	struct v4l2_ctrl_handler ctrls;
-	struct v4l2_ctrl *gains[4];
-
-	u16 output_control;
-	u16 black_level;
-};
-
-static inline struct mt9t001 *to_mt9t001(struct v4l2_subdev *sd)
-{
-	return container_of(sd, struct mt9t001, subdev);
-}
-
-static int mt9t001_read(struct i2c_client *client, u8 reg)
-{
-	return i2c_smbus_read_word_swapped(client, reg);
-}
-
-static int mt9t001_write(struct i2c_client *client, u8 reg, u16 data)
-{
-	return i2c_smbus_write_word_swapped(client, reg, data);
-}
-
-static int mt9t001_set_output_control(struct mt9t001 *mt9t001, u16 clear,
-				      u16 set)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&mt9t001->subdev);
-	u16 value = (mt9t001->output_control & ~clear) | set;
-	int ret;
-
-	if (value == mt9t001->output_control)
-		return 0;
-
-	ret = mt9t001_write(client, MT9T001_OUTPUT_CONTROL, value);
-	if (ret < 0)
-		return ret;
-
-	mt9t001->output_control = value;
-	return 0;
-}
-
-static int mt9t001_reset(struct mt9t001 *mt9t001)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&mt9t001->subdev);
-	int ret;
-
-	/* Reset the chip and stop data read out */
-	ret = mt9t001_write(client, MT9T001_RESET, 1);
-	if (ret < 0)
-		return ret;
-
-	ret = mt9t001_write(client, MT9T001_RESET, 0);
-	if (ret < 0)
-		return ret;
-
-	mt9t001->output_control = MT9T001_OUTPUT_CONTROL_DEF;
-
-	return mt9t001_set_output_control(mt9t001,
-					  MT9T001_OUTPUT_CONTROL_CHIP_ENABLE,
-					  0);
-}
-
-static int mt9t001_power_on(struct mt9t001 *mt9t001)
-{
-	int ret;
-
-	/* Bring up the supplies */
-	ret = regulator_bulk_enable(ARRAY_SIZE(mt9t001->regulators),
-				   mt9t001->regulators);
-	if (ret < 0)
-		return ret;
-
-	/* Enable clock */
-	ret = clk_prepare_enable(mt9t001->clk);
-	if (ret < 0)
-		regulator_bulk_disable(ARRAY_SIZE(mt9t001->regulators),
-				       mt9t001->regulators);
-
-	return ret;
-}
-
-static void mt9t001_power_off(struct mt9t001 *mt9t001)
-{
-	regulator_bulk_disable(ARRAY_SIZE(mt9t001->regulators),
-			       mt9t001->regulators);
-
-	clk_disable_unprepare(mt9t001->clk);
-}
-
-static int __mt9t001_set_power(struct mt9t001 *mt9t001, bool on)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&mt9t001->subdev);
-	int ret;
-
-	if (!on) {
-		mt9t001_power_off(mt9t001);
-		return 0;
-	}
-
-	ret = mt9t001_power_on(mt9t001);
-	if (ret < 0)
-		return ret;
-
-	ret = mt9t001_reset(mt9t001);
-	if (ret < 0) {
-		dev_err(&client->dev, "Failed to reset the camera\n");
-		goto e_power;
-	}
-
-	ret = v4l2_ctrl_handler_setup(&mt9t001->ctrls);
-	if (ret < 0) {
-		dev_err(&client->dev, "Failed to set up control handlers\n");
-		goto e_power;
-	}
-
-	return 0;
-
-e_power:
-	mt9t001_power_off(mt9t001);
-
-	return ret;
-}
-
-/* -----------------------------------------------------------------------------
- * V4L2 subdev video operations
- */
-
-static struct v4l2_mbus_framefmt *
-__mt9t001_get_pad_format(struct mt9t001 *mt9t001,
-			 struct v4l2_subdev_state *sd_state,
-			 unsigned int pad, enum v4l2_subdev_format_whence which)
-{
-	switch (which) {
-	case V4L2_SUBDEV_FORMAT_TRY:
-		return v4l2_subdev_get_try_format(&mt9t001->subdev, sd_state,
-						  pad);
-	case V4L2_SUBDEV_FORMAT_ACTIVE:
-		return &mt9t001->format;
-	default:
-		return NULL;
-	}
-}
-
-static struct v4l2_rect *
-__mt9t001_get_pad_crop(struct mt9t001 *mt9t001,
-		       struct v4l2_subdev_state *sd_state,
-		       unsigned int pad, enum v4l2_subdev_format_whence which)
-{
-	switch (which) {
-	case V4L2_SUBDEV_FORMAT_TRY:
-		return v4l2_subdev_get_try_crop(&mt9t001->subdev, sd_state,
-						pad);
-	case V4L2_SUBDEV_FORMAT_ACTIVE:
-		return &mt9t001->crop;
-	default:
-		return NULL;
-	}
-}
-
-static int mt9t001_s_stream(struct v4l2_subdev *subdev, int enable)
-{
-	const u16 mode = MT9T001_OUTPUT_CONTROL_CHIP_ENABLE;
-	struct i2c_client *client = v4l2_get_subdevdata(subdev);
-	struct mt9t001_platform_data *pdata = client->dev.platform_data;
-	struct mt9t001 *mt9t001 = to_mt9t001(subdev);
-	struct v4l2_mbus_framefmt *format = &mt9t001->format;
-	struct v4l2_rect *crop = &mt9t001->crop;
-	unsigned int hratio;
-	unsigned int vratio;
-	int ret;
-
-	if (!enable)
-		return mt9t001_set_output_control(mt9t001, mode, 0);
-
-	/* Configure the pixel clock polarity */
-	if (pdata->clk_pol) {
-		ret  = mt9t001_write(client, MT9T001_PIXEL_CLOCK,
-				     MT9T001_PIXEL_CLOCK_INVERT);
-		if (ret < 0)
-			return ret;
-	}
-
-	/* Configure the window size and row/column bin */
-	hratio = DIV_ROUND_CLOSEST(crop->width, format->width);
-	vratio = DIV_ROUND_CLOSEST(crop->height, format->height);
-
-	ret = mt9t001_write(client, MT9T001_ROW_ADDRESS_MODE, hratio - 1);
-	if (ret < 0)
-		return ret;
-
-	ret = mt9t001_write(client, MT9T001_COLUMN_ADDRESS_MODE, vratio - 1);
-	if (ret < 0)
-		return ret;
-
-	ret = mt9t001_write(client, MT9T001_COLUMN_START, crop->left);
-	if (ret < 0)
-		return ret;
-
-	ret = mt9t001_write(client, MT9T001_ROW_START, crop->top);
-	if (ret < 0)
-		return ret;
-
-	ret = mt9t001_write(client, MT9T001_WINDOW_WIDTH, crop->width - 1);
-	if (ret < 0)
-		return ret;
-
-	ret = mt9t001_write(client, MT9T001_WINDOW_HEIGHT, crop->height - 1);
-	if (ret < 0)
-		return ret;
-
-	/* Switch to master "normal" mode */
-	return mt9t001_set_output_control(mt9t001, 0, mode);
-}
-
-static int mt9t001_enum_mbus_code(struct v4l2_subdev *subdev,
-				  struct v4l2_subdev_state *sd_state,
-				  struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index > 0)
-		return -EINVAL;
-
-	code->code = MEDIA_BUS_FMT_SGRBG10_1X10;
-	return 0;
-}
-
-static int mt9t001_enum_frame_size(struct v4l2_subdev *subdev,
-				   struct v4l2_subdev_state *sd_state,
-				   struct v4l2_subdev_frame_size_enum *fse)
-{
-	if (fse->index >= 8 || fse->code != MEDIA_BUS_FMT_SGRBG10_1X10)
-		return -EINVAL;
-
-	fse->min_width = (MT9T001_WINDOW_WIDTH_DEF + 1) / fse->index;
-	fse->max_width = fse->min_width;
-	fse->min_height = (MT9T001_WINDOW_HEIGHT_DEF + 1) / fse->index;
-	fse->max_height = fse->min_height;
-
-	return 0;
-}
-
-static int mt9t001_get_format(struct v4l2_subdev *subdev,
-			      struct v4l2_subdev_state *sd_state,
-			      struct v4l2_subdev_format *format)
-{
-	struct mt9t001 *mt9t001 = to_mt9t001(subdev);
-
-	format->format = *__mt9t001_get_pad_format(mt9t001, sd_state,
-						   format->pad,
-						   format->which);
-	return 0;
-}
-
-static int mt9t001_set_format(struct v4l2_subdev *subdev,
-			      struct v4l2_subdev_state *sd_state,
-			      struct v4l2_subdev_format *format)
-{
-	struct mt9t001 *mt9t001 = to_mt9t001(subdev);
-	struct v4l2_mbus_framefmt *__format;
-	struct v4l2_rect *__crop;
-	unsigned int width;
-	unsigned int height;
-	unsigned int hratio;
-	unsigned int vratio;
-
-	__crop = __mt9t001_get_pad_crop(mt9t001, sd_state, format->pad,
-					format->which);
-
-	/* Clamp the width and height to avoid dividing by zero. */
-	width = clamp_t(unsigned int, ALIGN(format->format.width, 2),
-			max_t(unsigned int, __crop->width / 8,
-			      MT9T001_WINDOW_HEIGHT_MIN + 1),
-			__crop->width);
-	height = clamp_t(unsigned int, ALIGN(format->format.height, 2),
-			 max_t(unsigned int, __crop->height / 8,
-			       MT9T001_WINDOW_HEIGHT_MIN + 1),
-			 __crop->height);
-
-	hratio = DIV_ROUND_CLOSEST(__crop->width, width);
-	vratio = DIV_ROUND_CLOSEST(__crop->height, height);
-
-	__format = __mt9t001_get_pad_format(mt9t001, sd_state, format->pad,
-					    format->which);
-	__format->width = __crop->width / hratio;
-	__format->height = __crop->height / vratio;
-
-	format->format = *__format;
-
-	return 0;
-}
-
-static int mt9t001_get_selection(struct v4l2_subdev *subdev,
-				 struct v4l2_subdev_state *sd_state,
-				 struct v4l2_subdev_selection *sel)
-{
-	struct mt9t001 *mt9t001 = to_mt9t001(subdev);
-
-	if (sel->target != V4L2_SEL_TGT_CROP)
-		return -EINVAL;
-
-	sel->r = *__mt9t001_get_pad_crop(mt9t001, sd_state, sel->pad,
-					 sel->which);
-	return 0;
-}
-
-static int mt9t001_set_selection(struct v4l2_subdev *subdev,
-				 struct v4l2_subdev_state *sd_state,
-				 struct v4l2_subdev_selection *sel)
-{
-	struct mt9t001 *mt9t001 = to_mt9t001(subdev);
-	struct v4l2_mbus_framefmt *__format;
-	struct v4l2_rect *__crop;
-	struct v4l2_rect rect;
-
-	if (sel->target != V4L2_SEL_TGT_CROP)
-		return -EINVAL;
-
-	/* Clamp the crop rectangle boundaries and align them to a multiple of 2
-	 * pixels.
-	 */
-	rect.left = clamp(ALIGN(sel->r.left, 2),
-			  MT9T001_COLUMN_START_MIN,
-			  MT9T001_COLUMN_START_MAX);
-	rect.top = clamp(ALIGN(sel->r.top, 2),
-			 MT9T001_ROW_START_MIN,
-			 MT9T001_ROW_START_MAX);
-	rect.width = clamp_t(unsigned int, ALIGN(sel->r.width, 2),
-			     MT9T001_WINDOW_WIDTH_MIN + 1,
-			     MT9T001_WINDOW_WIDTH_MAX + 1);
-	rect.height = clamp_t(unsigned int, ALIGN(sel->r.height, 2),
-			      MT9T001_WINDOW_HEIGHT_MIN + 1,
-			      MT9T001_WINDOW_HEIGHT_MAX + 1);
-
-	rect.width = min_t(unsigned int, rect.width,
-			   MT9T001_PIXEL_ARRAY_WIDTH - rect.left);
-	rect.height = min_t(unsigned int, rect.height,
-			    MT9T001_PIXEL_ARRAY_HEIGHT - rect.top);
-
-	__crop = __mt9t001_get_pad_crop(mt9t001, sd_state, sel->pad,
-					sel->which);
-
-	if (rect.width != __crop->width || rect.height != __crop->height) {
-		/* Reset the output image size if the crop rectangle size has
-		 * been modified.
-		 */
-		__format = __mt9t001_get_pad_format(mt9t001, sd_state,
-						    sel->pad,
-						    sel->which);
-		__format->width = rect.width;
-		__format->height = rect.height;
-	}
-
-	*__crop = rect;
-	sel->r = rect;
-
-	return 0;
-}
-
-/* -----------------------------------------------------------------------------
- * V4L2 subdev control operations
- */
-
-#define V4L2_CID_TEST_PATTERN_COLOR	(V4L2_CID_USER_BASE | 0x1001)
-#define V4L2_CID_BLACK_LEVEL_AUTO	(V4L2_CID_USER_BASE | 0x1002)
-#define V4L2_CID_BLACK_LEVEL_OFFSET	(V4L2_CID_USER_BASE | 0x1003)
-#define V4L2_CID_BLACK_LEVEL_CALIBRATE	(V4L2_CID_USER_BASE | 0x1004)
-
-#define V4L2_CID_GAIN_RED		(V4L2_CTRL_CLASS_CAMERA | 0x1001)
-#define V4L2_CID_GAIN_GREEN_RED		(V4L2_CTRL_CLASS_CAMERA | 0x1002)
-#define V4L2_CID_GAIN_GREEN_BLUE	(V4L2_CTRL_CLASS_CAMERA | 0x1003)
-#define V4L2_CID_GAIN_BLUE		(V4L2_CTRL_CLASS_CAMERA | 0x1004)
-
-static u16 mt9t001_gain_value(s32 *gain)
-{
-	/* Gain is controlled by 2 analog stages and a digital stage. Valid
-	 * values for the 3 stages are
-	 *
-	 * Stage		Min	Max	Step
-	 * ------------------------------------------
-	 * First analog stage	x1	x2	1
-	 * Second analog stage	x1	x4	0.125
-	 * Digital stage	x1	x16	0.125
-	 *
-	 * To minimize noise, the gain stages should be used in the second
-	 * analog stage, first analog stage, digital stage order. Gain from a
-	 * previous stage should be pushed to its maximum value before the next
-	 * stage is used.
-	 */
-	if (*gain <= 32)
-		return *gain;
-
-	if (*gain <= 64) {
-		*gain &= ~1;
-		return (1 << 6) | (*gain >> 1);
-	}
-
-	*gain &= ~7;
-	return ((*gain - 64) << 5) | (1 << 6) | 32;
-}
-
-static int mt9t001_ctrl_freeze(struct mt9t001 *mt9t001, bool freeze)
-{
-	return mt9t001_set_output_control(mt9t001,
-		freeze ? 0 : MT9T001_OUTPUT_CONTROL_SYNC,
-		freeze ? MT9T001_OUTPUT_CONTROL_SYNC : 0);
-}
-
-static int mt9t001_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	static const u8 gains[4] = {
-		MT9T001_RED_GAIN, MT9T001_GREEN1_GAIN,
-		MT9T001_GREEN2_GAIN, MT9T001_BLUE_GAIN
-	};
-
-	struct mt9t001 *mt9t001 =
-			container_of(ctrl->handler, struct mt9t001, ctrls);
-	struct i2c_client *client = v4l2_get_subdevdata(&mt9t001->subdev);
-	unsigned int count;
-	unsigned int i;
-	u16 value;
-	int ret;
-
-	switch (ctrl->id) {
-	case V4L2_CID_GAIN_RED:
-	case V4L2_CID_GAIN_GREEN_RED:
-	case V4L2_CID_GAIN_GREEN_BLUE:
-	case V4L2_CID_GAIN_BLUE:
-
-		/* Disable control updates if more than one control has changed
-		 * in the cluster.
-		 */
-		for (i = 0, count = 0; i < 4; ++i) {
-			struct v4l2_ctrl *gain = mt9t001->gains[i];
-
-			if (gain->val != gain->cur.val)
-				count++;
-		}
-
-		if (count > 1) {
-			ret = mt9t001_ctrl_freeze(mt9t001, true);
-			if (ret < 0)
-				return ret;
-		}
-
-		/* Update the gain controls. */
-		for (i = 0; i < 4; ++i) {
-			struct v4l2_ctrl *gain = mt9t001->gains[i];
-
-			if (gain->val == gain->cur.val)
-				continue;
-
-			value = mt9t001_gain_value(&gain->val);
-			ret = mt9t001_write(client, gains[i], value);
-			if (ret < 0) {
-				mt9t001_ctrl_freeze(mt9t001, false);
-				return ret;
-			}
-		}
-
-		/* Enable control updates. */
-		if (count > 1) {
-			ret = mt9t001_ctrl_freeze(mt9t001, false);
-			if (ret < 0)
-				return ret;
-		}
-
-		break;
-
-	case V4L2_CID_EXPOSURE:
-		ret = mt9t001_write(client, MT9T001_SHUTTER_WIDTH_LOW,
-				    ctrl->val & 0xffff);
-		if (ret < 0)
-			return ret;
-
-		return mt9t001_write(client, MT9T001_SHUTTER_WIDTH_HIGH,
-				     ctrl->val >> 16);
-
-	case V4L2_CID_TEST_PATTERN:
-		return mt9t001_set_output_control(mt9t001,
-			ctrl->val ? 0 : MT9T001_OUTPUT_CONTROL_TEST_DATA,
-			ctrl->val ? MT9T001_OUTPUT_CONTROL_TEST_DATA : 0);
-
-	case V4L2_CID_TEST_PATTERN_COLOR:
-		return mt9t001_write(client, MT9T001_TEST_DATA, ctrl->val << 2);
-
-	case V4L2_CID_BLACK_LEVEL_AUTO:
-		value = ctrl->val ? 0 : MT9T001_BLACK_LEVEL_OVERRIDE;
-		ret = mt9t001_write(client, MT9T001_BLACK_LEVEL_CALIBRATION,
-				    value);
-		if (ret < 0)
-			return ret;
-
-		mt9t001->black_level = value;
-		break;
-
-	case V4L2_CID_BLACK_LEVEL_OFFSET:
-		ret = mt9t001_write(client, MT9T001_GREEN1_OFFSET, ctrl->val);
-		if (ret < 0)
-			return ret;
-
-		ret = mt9t001_write(client, MT9T001_GREEN2_OFFSET, ctrl->val);
-		if (ret < 0)
-			return ret;
-
-		ret = mt9t001_write(client, MT9T001_RED_OFFSET, ctrl->val);
-		if (ret < 0)
-			return ret;
-
-		return mt9t001_write(client, MT9T001_BLUE_OFFSET, ctrl->val);
-
-	case V4L2_CID_BLACK_LEVEL_CALIBRATE:
-		return mt9t001_write(client, MT9T001_BLACK_LEVEL_CALIBRATION,
-				     MT9T001_BLACK_LEVEL_RECALCULATE |
-				     mt9t001->black_level);
-	}
-
-	return 0;
-}
-
-static const struct v4l2_ctrl_ops mt9t001_ctrl_ops = {
-	.s_ctrl = mt9t001_s_ctrl,
-};
-
-static const char * const mt9t001_test_pattern_menu[] = {
-	"Disabled",
-	"Enabled",
-};
-
-static const struct v4l2_ctrl_config mt9t001_ctrls[] = {
-	{
-		.ops		= &mt9t001_ctrl_ops,
-		.id		= V4L2_CID_TEST_PATTERN_COLOR,
-		.type		= V4L2_CTRL_TYPE_INTEGER,
-		.name		= "Test Pattern Color",
-		.min		= 0,
-		.max		= 1023,
-		.step		= 1,
-		.def		= 0,
-		.flags		= 0,
-	}, {
-		.ops		= &mt9t001_ctrl_ops,
-		.id		= V4L2_CID_BLACK_LEVEL_AUTO,
-		.type		= V4L2_CTRL_TYPE_BOOLEAN,
-		.name		= "Black Level, Auto",
-		.min		= 0,
-		.max		= 1,
-		.step		= 1,
-		.def		= 1,
-		.flags		= 0,
-	}, {
-		.ops		= &mt9t001_ctrl_ops,
-		.id		= V4L2_CID_BLACK_LEVEL_OFFSET,
-		.type		= V4L2_CTRL_TYPE_INTEGER,
-		.name		= "Black Level, Offset",
-		.min		= -256,
-		.max		= 255,
-		.step		= 1,
-		.def		= 32,
-		.flags		= 0,
-	}, {
-		.ops		= &mt9t001_ctrl_ops,
-		.id		= V4L2_CID_BLACK_LEVEL_CALIBRATE,
-		.type		= V4L2_CTRL_TYPE_BUTTON,
-		.name		= "Black Level, Calibrate",
-		.min		= 0,
-		.max		= 0,
-		.step		= 0,
-		.def		= 0,
-		.flags		= V4L2_CTRL_FLAG_WRITE_ONLY,
-	},
-};
-
-static const struct v4l2_ctrl_config mt9t001_gains[] = {
-	{
-		.ops		= &mt9t001_ctrl_ops,
-		.id		= V4L2_CID_GAIN_RED,
-		.type		= V4L2_CTRL_TYPE_INTEGER,
-		.name		= "Gain, Red",
-		.min		= MT9T001_GLOBAL_GAIN_MIN,
-		.max		= MT9T001_GLOBAL_GAIN_MAX,
-		.step		= 1,
-		.def		= MT9T001_GLOBAL_GAIN_MIN,
-		.flags		= 0,
-	}, {
-		.ops		= &mt9t001_ctrl_ops,
-		.id		= V4L2_CID_GAIN_GREEN_RED,
-		.type		= V4L2_CTRL_TYPE_INTEGER,
-		.name		= "Gain, Green (R)",
-		.min		= MT9T001_GLOBAL_GAIN_MIN,
-		.max		= MT9T001_GLOBAL_GAIN_MAX,
-		.step		= 1,
-		.def		= MT9T001_GLOBAL_GAIN_MIN,
-		.flags		= 0,
-	}, {
-		.ops		= &mt9t001_ctrl_ops,
-		.id		= V4L2_CID_GAIN_GREEN_BLUE,
-		.type		= V4L2_CTRL_TYPE_INTEGER,
-		.name		= "Gain, Green (B)",
-		.min		= MT9T001_GLOBAL_GAIN_MIN,
-		.max		= MT9T001_GLOBAL_GAIN_MAX,
-		.step		= 1,
-		.def		= MT9T001_GLOBAL_GAIN_MIN,
-		.flags		= 0,
-	}, {
-		.ops		= &mt9t001_ctrl_ops,
-		.id		= V4L2_CID_GAIN_BLUE,
-		.type		= V4L2_CTRL_TYPE_INTEGER,
-		.name		= "Gain, Blue",
-		.min		= MT9T001_GLOBAL_GAIN_MIN,
-		.max		= MT9T001_GLOBAL_GAIN_MAX,
-		.step		= 1,
-		.def		= MT9T001_GLOBAL_GAIN_MIN,
-		.flags		= 0,
-	},
-};
-
-/* -----------------------------------------------------------------------------
- * V4L2 subdev core operations
- */
-
-static int mt9t001_set_power(struct v4l2_subdev *subdev, int on)
-{
-	struct mt9t001 *mt9t001 = to_mt9t001(subdev);
-	int ret = 0;
-
-	mutex_lock(&mt9t001->power_lock);
-
-	/* If the power count is modified from 0 to != 0 or from != 0 to 0,
-	 * update the power state.
-	 */
-	if (mt9t001->power_count == !on) {
-		ret = __mt9t001_set_power(mt9t001, !!on);
-		if (ret < 0)
-			goto out;
-	}
-
-	/* Update the power count. */
-	mt9t001->power_count += on ? 1 : -1;
-	WARN_ON(mt9t001->power_count < 0);
-
-out:
-	mutex_unlock(&mt9t001->power_lock);
-	return ret;
-}
-
-/* -----------------------------------------------------------------------------
- * V4L2 subdev internal operations
- */
-
-static int mt9t001_registered(struct v4l2_subdev *subdev)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(subdev);
-	struct mt9t001 *mt9t001 = to_mt9t001(subdev);
-	s32 data;
-	int ret;
-
-	ret = mt9t001_power_on(mt9t001);
-	if (ret < 0) {
-		dev_err(&client->dev, "MT9T001 power up failed\n");
-		return ret;
-	}
-
-	/* Read out the chip version register */
-	data = mt9t001_read(client, MT9T001_CHIP_VERSION);
-	mt9t001_power_off(mt9t001);
-
-	if (data != MT9T001_CHIP_ID) {
-		dev_err(&client->dev,
-			"MT9T001 not detected, wrong version 0x%04x\n", data);
-		return -ENODEV;
-	}
-
-	dev_info(&client->dev, "MT9T001 detected at address 0x%02x\n",
-		 client->addr);
-
-	return 0;
-}
-
-static int mt9t001_open(struct v4l2_subdev *subdev, struct v4l2_subdev_fh *fh)
-{
-	struct v4l2_mbus_framefmt *format;
-	struct v4l2_rect *crop;
-
-	crop = v4l2_subdev_get_try_crop(subdev, fh->state, 0);
-	crop->left = MT9T001_COLUMN_START_DEF;
-	crop->top = MT9T001_ROW_START_DEF;
-	crop->width = MT9T001_WINDOW_WIDTH_DEF + 1;
-	crop->height = MT9T001_WINDOW_HEIGHT_DEF + 1;
-
-	format = v4l2_subdev_get_try_format(subdev, fh->state, 0);
-	format->code = MEDIA_BUS_FMT_SGRBG10_1X10;
-	format->width = MT9T001_WINDOW_WIDTH_DEF + 1;
-	format->height = MT9T001_WINDOW_HEIGHT_DEF + 1;
-	format->field = V4L2_FIELD_NONE;
-	format->colorspace = V4L2_COLORSPACE_SRGB;
-
-	return mt9t001_set_power(subdev, 1);
-}
-
-static int mt9t001_close(struct v4l2_subdev *subdev, struct v4l2_subdev_fh *fh)
-{
-	return mt9t001_set_power(subdev, 0);
-}
-
-static const struct v4l2_subdev_core_ops mt9t001_subdev_core_ops = {
-	.s_power = mt9t001_set_power,
-};
-
-static const struct v4l2_subdev_video_ops mt9t001_subdev_video_ops = {
-	.s_stream = mt9t001_s_stream,
-};
-
-static const struct v4l2_subdev_pad_ops mt9t001_subdev_pad_ops = {
-	.enum_mbus_code = mt9t001_enum_mbus_code,
-	.enum_frame_size = mt9t001_enum_frame_size,
-	.get_fmt = mt9t001_get_format,
-	.set_fmt = mt9t001_set_format,
-	.get_selection = mt9t001_get_selection,
-	.set_selection = mt9t001_set_selection,
-};
-
-static const struct v4l2_subdev_ops mt9t001_subdev_ops = {
-	.core = &mt9t001_subdev_core_ops,
-	.video = &mt9t001_subdev_video_ops,
-	.pad = &mt9t001_subdev_pad_ops,
-};
-
-static const struct v4l2_subdev_internal_ops mt9t001_subdev_internal_ops = {
-	.registered = mt9t001_registered,
-	.open = mt9t001_open,
-	.close = mt9t001_close,
-};
-
-static int mt9t001_probe(struct i2c_client *client)
-{
-	struct mt9t001_platform_data *pdata = client->dev.platform_data;
-	struct mt9t001 *mt9t001;
-	unsigned int i;
-	int ret;
-
-	if (pdata == NULL) {
-		dev_err(&client->dev, "No platform data\n");
-		return -EINVAL;
-	}
-
-	if (!i2c_check_functionality(client->adapter,
-				     I2C_FUNC_SMBUS_WORD_DATA)) {
-		dev_warn(&client->adapter->dev,
-			 "I2C-Adapter doesn't support I2C_FUNC_SMBUS_WORD\n");
-		return -EIO;
-	}
-
-	mt9t001 = devm_kzalloc(&client->dev, sizeof(*mt9t001), GFP_KERNEL);
-	if (!mt9t001)
-		return -ENOMEM;
-
-	mutex_init(&mt9t001->power_lock);
-	mt9t001->output_control = MT9T001_OUTPUT_CONTROL_DEF;
-
-	mt9t001->regulators[0].supply = "vdd";
-	mt9t001->regulators[1].supply = "vaa";
-
-	ret = devm_regulator_bulk_get(&client->dev, 2, mt9t001->regulators);
-	if (ret < 0) {
-		dev_err(&client->dev, "Unable to get regulators\n");
-		return ret;
-	}
-
-	mt9t001->clk = devm_clk_get(&client->dev, NULL);
-	if (IS_ERR(mt9t001->clk)) {
-		dev_err(&client->dev, "Unable to get clock\n");
-		return PTR_ERR(mt9t001->clk);
-	}
-
-	v4l2_ctrl_handler_init(&mt9t001->ctrls, ARRAY_SIZE(mt9t001_ctrls) +
-						ARRAY_SIZE(mt9t001_gains) + 4);
-
-	v4l2_ctrl_new_std(&mt9t001->ctrls, &mt9t001_ctrl_ops,
-			  V4L2_CID_EXPOSURE, MT9T001_SHUTTER_WIDTH_MIN,
-			  MT9T001_SHUTTER_WIDTH_MAX, 1,
-			  MT9T001_SHUTTER_WIDTH_DEF);
-	v4l2_ctrl_new_std(&mt9t001->ctrls, &mt9t001_ctrl_ops,
-			  V4L2_CID_BLACK_LEVEL, 1, 1, 1, 1);
-	v4l2_ctrl_new_std(&mt9t001->ctrls, &mt9t001_ctrl_ops,
-			  V4L2_CID_PIXEL_RATE, pdata->ext_clk, pdata->ext_clk,
-			  1, pdata->ext_clk);
-	v4l2_ctrl_new_std_menu_items(&mt9t001->ctrls, &mt9t001_ctrl_ops,
-			V4L2_CID_TEST_PATTERN,
-			ARRAY_SIZE(mt9t001_test_pattern_menu) - 1, 0,
-			0, mt9t001_test_pattern_menu);
-
-	for (i = 0; i < ARRAY_SIZE(mt9t001_ctrls); ++i)
-		v4l2_ctrl_new_custom(&mt9t001->ctrls, &mt9t001_ctrls[i], NULL);
-
-	for (i = 0; i < ARRAY_SIZE(mt9t001_gains); ++i)
-		mt9t001->gains[i] = v4l2_ctrl_new_custom(&mt9t001->ctrls,
-			&mt9t001_gains[i], NULL);
-
-	v4l2_ctrl_cluster(ARRAY_SIZE(mt9t001_gains), mt9t001->gains);
-
-	mt9t001->subdev.ctrl_handler = &mt9t001->ctrls;
-
-	if (mt9t001->ctrls.error) {
-		printk(KERN_INFO "%s: control initialization error %d\n",
-		       __func__, mt9t001->ctrls.error);
-		ret = -EINVAL;
-		goto done;
-	}
-
-	mt9t001->crop.left = MT9T001_COLUMN_START_DEF;
-	mt9t001->crop.top = MT9T001_ROW_START_DEF;
-	mt9t001->crop.width = MT9T001_WINDOW_WIDTH_DEF + 1;
-	mt9t001->crop.height = MT9T001_WINDOW_HEIGHT_DEF + 1;
-
-	mt9t001->format.code = MEDIA_BUS_FMT_SGRBG10_1X10;
-	mt9t001->format.width = MT9T001_WINDOW_WIDTH_DEF + 1;
-	mt9t001->format.height = MT9T001_WINDOW_HEIGHT_DEF + 1;
-	mt9t001->format.field = V4L2_FIELD_NONE;
-	mt9t001->format.colorspace = V4L2_COLORSPACE_SRGB;
-
-	v4l2_i2c_subdev_init(&mt9t001->subdev, client, &mt9t001_subdev_ops);
-	mt9t001->subdev.internal_ops = &mt9t001_subdev_internal_ops;
-	mt9t001->subdev.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-
-	mt9t001->subdev.entity.function = MEDIA_ENT_F_CAM_SENSOR;
-	mt9t001->pad.flags = MEDIA_PAD_FL_SOURCE;
-	ret = media_entity_pads_init(&mt9t001->subdev.entity, 1, &mt9t001->pad);
-
-done:
-	if (ret < 0) {
-		v4l2_ctrl_handler_free(&mt9t001->ctrls);
-		media_entity_cleanup(&mt9t001->subdev.entity);
-	}
-
-	return ret;
-}
-
-static void mt9t001_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
-	struct mt9t001 *mt9t001 = to_mt9t001(subdev);
-
-	v4l2_ctrl_handler_free(&mt9t001->ctrls);
-	v4l2_device_unregister_subdev(subdev);
-	media_entity_cleanup(&subdev->entity);
-}
-
-static const struct i2c_device_id mt9t001_id[] = {
-	{ "mt9t001", 0 },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, mt9t001_id);
-
-static struct i2c_driver mt9t001_driver = {
-	.driver = {
-		.name = "mt9t001",
-	},
-	.probe_new	= mt9t001_probe,
-	.remove		= mt9t001_remove,
-	.id_table	= mt9t001_id,
-};
-
-module_i2c_driver(mt9t001_driver);
-
-MODULE_DESCRIPTION("Aptina (Micron) MT9T001 Camera driver");
-MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/noon010pc30.c b/drivers/media/i2c/noon010pc30.c
deleted file mode 100644
index 144bef2835f7..000000000000
--- a/drivers/media/i2c/noon010pc30.c
+++ /dev/null
@@ -1,821 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for SiliconFile NOON010PC30 CIF (1/11") Image Sensor with ISP
- *
- * Copyright (C) 2010 - 2011 Samsung Electronics Co., Ltd.
- * Contact: Sylwester Nawrocki, <s.nawrocki@samsung.com>
- *
- * Initial register configuration based on a driver authored by
- * HeungJun Kim <riverful.kim@samsung.com>.
- */
-
-#include <linux/delay.h>
-#include <linux/gpio/consumer.h>
-#include <linux/i2c.h>
-#include <linux/slab.h>
-#include <linux/regulator/consumer.h>
-#include <media/i2c/noon010pc30.h>
-#include <linux/videodev2.h>
-#include <linux/module.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-mediabus.h>
-#include <media/v4l2-subdev.h>
-
-static int debug;
-module_param(debug, int, 0644);
-MODULE_PARM_DESC(debug, "Enable module debug trace. Set to 1 to enable.");
-
-#define MODULE_NAME		"NOON010PC30"
-
-/*
- * Register offsets within a page
- * b15..b8 - page id, b7..b0 - register address
- */
-#define POWER_CTRL_REG		0x0001
-#define PAGEMODE_REG		0x03
-#define DEVICE_ID_REG		0x0004
-#define NOON010PC30_ID		0x86
-#define VDO_CTL_REG(n)		(0x0010 + (n))
-#define SYNC_CTL_REG		0x0012
-/* Window size and position */
-#define WIN_ROWH_REG		0x0013
-#define WIN_ROWL_REG		0x0014
-#define WIN_COLH_REG		0x0015
-#define WIN_COLL_REG		0x0016
-#define WIN_HEIGHTH_REG		0x0017
-#define WIN_HEIGHTL_REG		0x0018
-#define WIN_WIDTHH_REG		0x0019
-#define WIN_WIDTHL_REG		0x001A
-#define HBLANKH_REG		0x001B
-#define HBLANKL_REG		0x001C
-#define VSYNCH_REG		0x001D
-#define VSYNCL_REG		0x001E
-/* VSYNC control */
-#define VS_CTL_REG(n)		(0x00A1 + (n))
-/* page 1 */
-#define ISP_CTL_REG(n)		(0x0110 + (n))
-#define YOFS_REG		0x0119
-#define DARK_YOFS_REG		0x011A
-#define SAT_CTL_REG		0x0120
-#define BSAT_REG		0x0121
-#define RSAT_REG		0x0122
-/* Color correction */
-#define CMC_CTL_REG		0x0130
-#define CMC_OFSGH_REG		0x0133
-#define CMC_OFSGL_REG		0x0135
-#define CMC_SIGN_REG		0x0136
-#define CMC_GOFS_REG		0x0137
-#define CMC_COEF_REG(n)		(0x0138 + (n))
-#define CMC_OFS_REG(n)		(0x0141 + (n))
-/* Gamma correction */
-#define GMA_CTL_REG		0x0160
-#define GMA_COEF_REG(n)		(0x0161 + (n))
-/* Lens Shading */
-#define LENS_CTRL_REG		0x01D0
-#define LENS_XCEN_REG		0x01D1
-#define LENS_YCEN_REG		0x01D2
-#define LENS_RC_REG		0x01D3
-#define LENS_GC_REG		0x01D4
-#define LENS_BC_REG		0x01D5
-#define L_AGON_REG		0x01D6
-#define L_AGOFF_REG		0x01D7
-/* Page 3 - Auto Exposure */
-#define AE_CTL_REG(n)		(0x0310 + (n))
-#define AE_CTL9_REG		0x032C
-#define AE_CTL10_REG		0x032D
-#define AE_YLVL_REG		0x031C
-#define AE_YTH_REG(n)		(0x031D + (n))
-#define AE_WGT_REG		0x0326
-#define EXP_TIMEH_REG		0x0333
-#define EXP_TIMEM_REG		0x0334
-#define EXP_TIMEL_REG		0x0335
-#define EXP_MMINH_REG		0x0336
-#define EXP_MMINL_REG		0x0337
-#define EXP_MMAXH_REG		0x0338
-#define EXP_MMAXM_REG		0x0339
-#define EXP_MMAXL_REG		0x033A
-/* Page 4 - Auto White Balance */
-#define AWB_CTL_REG(n)		(0x0410 + (n))
-#define AWB_ENABE		0x80
-#define AWB_WGHT_REG		0x0419
-#define BGAIN_PAR_REG(n)	(0x044F + (n))
-/* Manual white balance, when AWB_CTL2[0]=1 */
-#define MWB_RGAIN_REG		0x0466
-#define MWB_BGAIN_REG		0x0467
-
-/* The token to mark an array end */
-#define REG_TERM		0xFFFF
-
-struct noon010_format {
-	u32 code;
-	enum v4l2_colorspace colorspace;
-	u16 ispctl1_reg;
-};
-
-struct noon010_frmsize {
-	u16 width;
-	u16 height;
-	int vid_ctl1;
-};
-
-static const char * const noon010_supply_name[] = {
-	"vdd_core", "vddio", "vdda"
-};
-
-#define NOON010_NUM_SUPPLIES ARRAY_SIZE(noon010_supply_name)
-
-struct noon010_info {
-	struct v4l2_subdev sd;
-	struct media_pad pad;
-	struct v4l2_ctrl_handler hdl;
-	struct regulator_bulk_data supply[NOON010_NUM_SUPPLIES];
-	struct gpio_desc *reset;
-	struct gpio_desc *stby;
-
-	/* Protects the struct members below */
-	struct mutex lock;
-
-	const struct noon010_format *curr_fmt;
-	const struct noon010_frmsize *curr_win;
-	unsigned int apply_new_cfg:1;
-	unsigned int streaming:1;
-	unsigned int hflip:1;
-	unsigned int vflip:1;
-	unsigned int power:1;
-	u8 i2c_reg_page;
-};
-
-struct i2c_regval {
-	u16 addr;
-	u16 val;
-};
-
-/* Supported resolutions. */
-static const struct noon010_frmsize noon010_sizes[] = {
-	{
-		.width		= 352,
-		.height		= 288,
-		.vid_ctl1	= 0,
-	}, {
-		.width		= 176,
-		.height		= 144,
-		.vid_ctl1	= 0x10,
-	}, {
-		.width		= 88,
-		.height		= 72,
-		.vid_ctl1	= 0x20,
-	},
-};
-
-/* Supported pixel formats. */
-static const struct noon010_format noon010_formats[] = {
-	{
-		.code		= MEDIA_BUS_FMT_YUYV8_2X8,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-		.ispctl1_reg	= 0x03,
-	}, {
-		.code		= MEDIA_BUS_FMT_YVYU8_2X8,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-		.ispctl1_reg	= 0x02,
-	}, {
-		.code		= MEDIA_BUS_FMT_VYUY8_2X8,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-		.ispctl1_reg	= 0,
-	}, {
-		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-		.ispctl1_reg	= 0x01,
-	}, {
-		.code		= MEDIA_BUS_FMT_RGB565_2X8_BE,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-		.ispctl1_reg	= 0x40,
-	},
-};
-
-static const struct i2c_regval noon010_base_regs[] = {
-	{ WIN_COLL_REG,		0x06 },	{ HBLANKL_REG,		0x7C },
-	/* Color corection and saturation */
-	{ ISP_CTL_REG(0),	0x30 }, { ISP_CTL_REG(2),	0x30 },
-	{ YOFS_REG,		0x80 }, { DARK_YOFS_REG,	0x04 },
-	{ SAT_CTL_REG,		0x1F }, { BSAT_REG,		0x90 },
-	{ CMC_CTL_REG,		0x0F }, { CMC_OFSGH_REG,	0x3C },
-	{ CMC_OFSGL_REG,	0x2C }, { CMC_SIGN_REG,		0x3F },
-	{ CMC_COEF_REG(0),	0x79 }, { CMC_OFS_REG(0),	0x00 },
-	{ CMC_COEF_REG(1),	0x39 }, { CMC_OFS_REG(1),	0x00 },
-	{ CMC_COEF_REG(2),	0x00 }, { CMC_OFS_REG(2),	0x00 },
-	{ CMC_COEF_REG(3),	0x11 }, { CMC_OFS_REG(3),	0x8B },
-	{ CMC_COEF_REG(4),	0x65 }, { CMC_OFS_REG(4),	0x07 },
-	{ CMC_COEF_REG(5),	0x14 }, { CMC_OFS_REG(5),	0x04 },
-	{ CMC_COEF_REG(6),	0x01 }, { CMC_OFS_REG(6),	0x9C },
-	{ CMC_COEF_REG(7),	0x33 }, { CMC_OFS_REG(7),	0x89 },
-	{ CMC_COEF_REG(8),	0x74 }, { CMC_OFS_REG(8),	0x25 },
-	/* Automatic white balance */
-	{ AWB_CTL_REG(0),	0x78 }, { AWB_CTL_REG(1),	0x2E },
-	{ AWB_CTL_REG(2),	0x20 }, { AWB_CTL_REG(3),	0x85 },
-	/* Auto exposure */
-	{ AE_CTL_REG(0),	0xDC }, { AE_CTL_REG(1),	0x81 },
-	{ AE_CTL_REG(2),	0x30 }, { AE_CTL_REG(3),	0xA5 },
-	{ AE_CTL_REG(4),	0x40 }, { AE_CTL_REG(5),	0x51 },
-	{ AE_CTL_REG(6),	0x33 }, { AE_CTL_REG(7),	0x7E },
-	{ AE_CTL9_REG,		0x00 }, { AE_CTL10_REG,		0x02 },
-	{ AE_YLVL_REG,		0x44 },	{ AE_YTH_REG(0),	0x34 },
-	{ AE_YTH_REG(1),	0x30 },	{ AE_WGT_REG,		0xD5 },
-	/* Lens shading compensation */
-	{ LENS_CTRL_REG,	0x01 }, { LENS_XCEN_REG,	0x80 },
-	{ LENS_YCEN_REG,	0x70 }, { LENS_RC_REG,		0x53 },
-	{ LENS_GC_REG,		0x40 }, { LENS_BC_REG,		0x3E },
-	{ REG_TERM,		0 },
-};
-
-static inline struct noon010_info *to_noon010(struct v4l2_subdev *sd)
-{
-	return container_of(sd, struct noon010_info, sd);
-}
-
-static inline struct v4l2_subdev *to_sd(struct v4l2_ctrl *ctrl)
-{
-	return &container_of(ctrl->handler, struct noon010_info, hdl)->sd;
-}
-
-static inline int set_i2c_page(struct noon010_info *info,
-			       struct i2c_client *client, unsigned int reg)
-{
-	u32 page = reg >> 8 & 0xFF;
-	int ret = 0;
-
-	if (info->i2c_reg_page != page && (reg & 0xFF) != 0x03) {
-		ret = i2c_smbus_write_byte_data(client, PAGEMODE_REG, page);
-		if (!ret)
-			info->i2c_reg_page = page;
-	}
-	return ret;
-}
-
-static int cam_i2c_read(struct v4l2_subdev *sd, u32 reg_addr)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct noon010_info *info = to_noon010(sd);
-	int ret = set_i2c_page(info, client, reg_addr);
-
-	if (ret)
-		return ret;
-	return i2c_smbus_read_byte_data(client, reg_addr & 0xFF);
-}
-
-static int cam_i2c_write(struct v4l2_subdev *sd, u32 reg_addr, u32 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct noon010_info *info = to_noon010(sd);
-	int ret = set_i2c_page(info, client, reg_addr);
-
-	if (ret)
-		return ret;
-	return i2c_smbus_write_byte_data(client, reg_addr & 0xFF, val);
-}
-
-static inline int noon010_bulk_write_reg(struct v4l2_subdev *sd,
-					 const struct i2c_regval *msg)
-{
-	while (msg->addr != REG_TERM) {
-		int ret = cam_i2c_write(sd, msg->addr, msg->val);
-
-		if (ret)
-			return ret;
-		msg++;
-	}
-	return 0;
-}
-
-/* Device reset and sleep mode control */
-static int noon010_power_ctrl(struct v4l2_subdev *sd, bool reset, bool sleep)
-{
-	struct noon010_info *info = to_noon010(sd);
-	u8 reg = sleep ? 0xF1 : 0xF0;
-	int ret = 0;
-
-	if (reset) {
-		ret = cam_i2c_write(sd, POWER_CTRL_REG, reg | 0x02);
-		udelay(20);
-	}
-	if (!ret) {
-		ret = cam_i2c_write(sd, POWER_CTRL_REG, reg);
-		if (reset && !ret)
-			info->i2c_reg_page = -1;
-	}
-	return ret;
-}
-
-/* Automatic white balance control */
-static int noon010_enable_autowhitebalance(struct v4l2_subdev *sd, int on)
-{
-	int ret;
-
-	ret = cam_i2c_write(sd, AWB_CTL_REG(1), on ? 0x2E : 0x2F);
-	if (!ret)
-		ret = cam_i2c_write(sd, AWB_CTL_REG(0), on ? 0xFB : 0x7B);
-	return ret;
-}
-
-/* Called with struct noon010_info.lock mutex held */
-static int noon010_set_flip(struct v4l2_subdev *sd, int hflip, int vflip)
-{
-	struct noon010_info *info = to_noon010(sd);
-	int reg, ret;
-
-	reg = cam_i2c_read(sd, VDO_CTL_REG(1));
-	if (reg < 0)
-		return reg;
-
-	reg &= 0x7C;
-	if (hflip)
-		reg |= 0x01;
-	if (vflip)
-		reg |= 0x02;
-
-	ret = cam_i2c_write(sd, VDO_CTL_REG(1), reg | 0x80);
-	if (!ret) {
-		info->hflip = hflip;
-		info->vflip = vflip;
-	}
-	return ret;
-}
-
-/* Configure resolution and color format */
-static int noon010_set_params(struct v4l2_subdev *sd)
-{
-	struct noon010_info *info = to_noon010(sd);
-
-	int ret = cam_i2c_write(sd, VDO_CTL_REG(0),
-				info->curr_win->vid_ctl1);
-	if (ret)
-		return ret;
-	return cam_i2c_write(sd, ISP_CTL_REG(0),
-			     info->curr_fmt->ispctl1_reg);
-}
-
-/* Find nearest matching image pixel size. */
-static int noon010_try_frame_size(struct v4l2_mbus_framefmt *mf,
-				  const struct noon010_frmsize **size)
-{
-	unsigned int min_err = ~0;
-	int i = ARRAY_SIZE(noon010_sizes);
-	const struct noon010_frmsize *fsize = &noon010_sizes[0],
-		*match = NULL;
-
-	while (i--) {
-		int err = abs(fsize->width - mf->width)
-				+ abs(fsize->height - mf->height);
-
-		if (err < min_err) {
-			min_err = err;
-			match = fsize;
-		}
-		fsize++;
-	}
-	if (match) {
-		mf->width  = match->width;
-		mf->height = match->height;
-		if (size)
-			*size = match;
-		return 0;
-	}
-	return -EINVAL;
-}
-
-/* Called with info.lock mutex held */
-static int power_enable(struct noon010_info *info)
-{
-	int ret;
-
-	if (info->power) {
-		v4l2_info(&info->sd, "%s: sensor is already on\n", __func__);
-		return 0;
-	}
-
-	/* Assert standby: line should be flagged active low in descriptor */
-	if (info->stby)
-		gpiod_set_value(info->stby, 1);
-
-	/* Assert reset: line should be flagged active low in descriptor */
-	if (info->reset)
-		gpiod_set_value(info->reset, 1);
-
-	ret = regulator_bulk_enable(NOON010_NUM_SUPPLIES, info->supply);
-	if (ret)
-		return ret;
-
-	/* De-assert reset and standby */
-	if (info->reset) {
-		msleep(50);
-		gpiod_set_value(info->reset, 0);
-	}
-	if (info->stby) {
-		udelay(1000);
-		gpiod_set_value(info->stby, 0);
-	}
-	/* Cycle reset: assert and deassert */
-	if (info->reset) {
-		udelay(1000);
-		gpiod_set_value(info->reset, 1);
-		msleep(100);
-		gpiod_set_value(info->reset, 0);
-		msleep(20);
-	}
-	info->power = 1;
-
-	v4l2_dbg(1, debug, &info->sd,  "%s: sensor is on\n", __func__);
-	return 0;
-}
-
-/* Called with info.lock mutex held */
-static int power_disable(struct noon010_info *info)
-{
-	int ret;
-
-	if (!info->power) {
-		v4l2_info(&info->sd, "%s: sensor is already off\n", __func__);
-		return 0;
-	}
-
-	ret = regulator_bulk_disable(NOON010_NUM_SUPPLIES, info->supply);
-	if (ret)
-		return ret;
-
-	/* Assert standby and reset */
-	if (info->stby)
-		gpiod_set_value(info->stby, 1);
-
-	if (info->reset)
-		gpiod_set_value(info->reset, 1);
-
-	info->power = 0;
-
-	v4l2_dbg(1, debug, &info->sd,  "%s: sensor is off\n", __func__);
-
-	return 0;
-}
-
-static int noon010_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct v4l2_subdev *sd = to_sd(ctrl);
-	struct noon010_info *info = to_noon010(sd);
-	int ret = 0;
-
-	v4l2_dbg(1, debug, sd, "%s: ctrl_id: %d, value: %d\n",
-		 __func__, ctrl->id, ctrl->val);
-
-	mutex_lock(&info->lock);
-	/*
-	 * If the device is not powered up by the host driver do
-	 * not apply any controls to H/W at this time. Instead
-	 * the controls will be restored right after power-up.
-	 */
-	if (!info->power)
-		goto unlock;
-
-	switch (ctrl->id) {
-	case V4L2_CID_AUTO_WHITE_BALANCE:
-		ret = noon010_enable_autowhitebalance(sd, ctrl->val);
-		break;
-	case V4L2_CID_BLUE_BALANCE:
-		ret = cam_i2c_write(sd, MWB_BGAIN_REG, ctrl->val);
-		break;
-	case V4L2_CID_RED_BALANCE:
-		ret =  cam_i2c_write(sd, MWB_RGAIN_REG, ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-unlock:
-	mutex_unlock(&info->lock);
-	return ret;
-}
-
-static int noon010_enum_mbus_code(struct v4l2_subdev *sd,
-				  struct v4l2_subdev_state *sd_state,
-				  struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index >= ARRAY_SIZE(noon010_formats))
-		return -EINVAL;
-
-	code->code = noon010_formats[code->index].code;
-	return 0;
-}
-
-static int noon010_get_fmt(struct v4l2_subdev *sd,
-			   struct v4l2_subdev_state *sd_state,
-			   struct v4l2_subdev_format *fmt)
-{
-	struct noon010_info *info = to_noon010(sd);
-	struct v4l2_mbus_framefmt *mf;
-
-	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
-		if (sd_state) {
-			mf = v4l2_subdev_get_try_format(sd, sd_state, 0);
-			fmt->format = *mf;
-		}
-		return 0;
-	}
-	mf = &fmt->format;
-
-	mutex_lock(&info->lock);
-	mf->width = info->curr_win->width;
-	mf->height = info->curr_win->height;
-	mf->code = info->curr_fmt->code;
-	mf->colorspace = info->curr_fmt->colorspace;
-	mf->field = V4L2_FIELD_NONE;
-
-	mutex_unlock(&info->lock);
-	return 0;
-}
-
-/* Return nearest media bus frame format. */
-static const struct noon010_format *noon010_try_fmt(struct v4l2_subdev *sd,
-					    struct v4l2_mbus_framefmt *mf)
-{
-	int i = ARRAY_SIZE(noon010_formats);
-
-	while (--i)
-		if (mf->code == noon010_formats[i].code)
-			break;
-	mf->code = noon010_formats[i].code;
-
-	return &noon010_formats[i];
-}
-
-static int noon010_set_fmt(struct v4l2_subdev *sd,
-			   struct v4l2_subdev_state *sd_state,
-			   struct v4l2_subdev_format *fmt)
-{
-	struct noon010_info *info = to_noon010(sd);
-	const struct noon010_frmsize *size = NULL;
-	const struct noon010_format *nf;
-	struct v4l2_mbus_framefmt *mf;
-	int ret = 0;
-
-	nf = noon010_try_fmt(sd, &fmt->format);
-	noon010_try_frame_size(&fmt->format, &size);
-	fmt->format.colorspace = V4L2_COLORSPACE_JPEG;
-	fmt->format.field = V4L2_FIELD_NONE;
-
-	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
-		if (sd_state) {
-			mf = v4l2_subdev_get_try_format(sd, sd_state, 0);
-			*mf = fmt->format;
-		}
-		return 0;
-	}
-	mutex_lock(&info->lock);
-	if (!info->streaming) {
-		info->apply_new_cfg = 1;
-		info->curr_fmt = nf;
-		info->curr_win = size;
-	} else {
-		ret = -EBUSY;
-	}
-	mutex_unlock(&info->lock);
-	return ret;
-}
-
-/* Called with struct noon010_info.lock mutex held */
-static int noon010_base_config(struct v4l2_subdev *sd)
-{
-	int ret = noon010_bulk_write_reg(sd, noon010_base_regs);
-	if (!ret)
-		ret = noon010_set_params(sd);
-	if (!ret)
-		ret = noon010_set_flip(sd, 1, 0);
-
-	return ret;
-}
-
-static int noon010_s_power(struct v4l2_subdev *sd, int on)
-{
-	struct noon010_info *info = to_noon010(sd);
-	int ret;
-
-	mutex_lock(&info->lock);
-	if (on) {
-		ret = power_enable(info);
-		if (!ret)
-			ret = noon010_base_config(sd);
-	} else {
-		noon010_power_ctrl(sd, false, true);
-		ret = power_disable(info);
-	}
-	mutex_unlock(&info->lock);
-
-	/* Restore the controls state */
-	if (!ret && on)
-		ret = v4l2_ctrl_handler_setup(&info->hdl);
-
-	return ret;
-}
-
-static int noon010_s_stream(struct v4l2_subdev *sd, int on)
-{
-	struct noon010_info *info = to_noon010(sd);
-	int ret = 0;
-
-	mutex_lock(&info->lock);
-	if (!info->streaming != !on) {
-		ret = noon010_power_ctrl(sd, false, !on);
-		if (!ret)
-			info->streaming = on;
-	}
-	if (!ret && on && info->apply_new_cfg) {
-		ret = noon010_set_params(sd);
-		if (!ret)
-			info->apply_new_cfg = 0;
-	}
-	mutex_unlock(&info->lock);
-	return ret;
-}
-
-static int noon010_log_status(struct v4l2_subdev *sd)
-{
-	struct noon010_info *info = to_noon010(sd);
-
-	v4l2_ctrl_handler_log_status(&info->hdl, sd->name);
-	return 0;
-}
-
-static int noon010_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
-{
-	struct v4l2_mbus_framefmt *mf = v4l2_subdev_get_try_format(sd,
-								   fh->state,
-								   0);
-
-	mf->width = noon010_sizes[0].width;
-	mf->height = noon010_sizes[0].height;
-	mf->code = noon010_formats[0].code;
-	mf->colorspace = V4L2_COLORSPACE_JPEG;
-	mf->field = V4L2_FIELD_NONE;
-	return 0;
-}
-
-static const struct v4l2_subdev_internal_ops noon010_subdev_internal_ops = {
-	.open = noon010_open,
-};
-
-static const struct v4l2_ctrl_ops noon010_ctrl_ops = {
-	.s_ctrl = noon010_s_ctrl,
-};
-
-static const struct v4l2_subdev_core_ops noon010_core_ops = {
-	.s_power	= noon010_s_power,
-	.log_status	= noon010_log_status,
-};
-
-static const struct v4l2_subdev_pad_ops noon010_pad_ops = {
-	.enum_mbus_code	= noon010_enum_mbus_code,
-	.get_fmt	= noon010_get_fmt,
-	.set_fmt	= noon010_set_fmt,
-};
-
-static const struct v4l2_subdev_video_ops noon010_video_ops = {
-	.s_stream	= noon010_s_stream,
-};
-
-static const struct v4l2_subdev_ops noon010_ops = {
-	.core	= &noon010_core_ops,
-	.pad	= &noon010_pad_ops,
-	.video	= &noon010_video_ops,
-};
-
-/* Return 0 if NOON010PC30L sensor type was detected or -ENODEV otherwise. */
-static int noon010_detect(struct i2c_client *client, struct noon010_info *info)
-{
-	int ret;
-
-	ret = power_enable(info);
-	if (ret)
-		return ret;
-
-	ret = i2c_smbus_read_byte_data(client, DEVICE_ID_REG);
-	if (ret < 0)
-		dev_err(&client->dev, "I2C read failed: 0x%X\n", ret);
-
-	power_disable(info);
-
-	return ret == NOON010PC30_ID ? 0 : -ENODEV;
-}
-
-static int noon010_probe(struct i2c_client *client)
-{
-	struct noon010_info *info;
-	struct v4l2_subdev *sd;
-	const struct noon010pc30_platform_data *pdata
-		= client->dev.platform_data;
-	int ret;
-	int i;
-
-	if (!pdata) {
-		dev_err(&client->dev, "No platform data!\n");
-		return -EIO;
-	}
-
-	info = devm_kzalloc(&client->dev, sizeof(*info), GFP_KERNEL);
-	if (!info)
-		return -ENOMEM;
-
-	mutex_init(&info->lock);
-	sd = &info->sd;
-	v4l2_i2c_subdev_init(sd, client, &noon010_ops);
-	/* Static name; NEVER use in new drivers! */
-	strscpy(sd->name, MODULE_NAME, sizeof(sd->name));
-
-	sd->internal_ops = &noon010_subdev_internal_ops;
-	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-
-	v4l2_ctrl_handler_init(&info->hdl, 3);
-
-	v4l2_ctrl_new_std(&info->hdl, &noon010_ctrl_ops,
-			  V4L2_CID_AUTO_WHITE_BALANCE, 0, 1, 1, 1);
-	v4l2_ctrl_new_std(&info->hdl, &noon010_ctrl_ops,
-			  V4L2_CID_RED_BALANCE, 0, 127, 1, 64);
-	v4l2_ctrl_new_std(&info->hdl, &noon010_ctrl_ops,
-			  V4L2_CID_BLUE_BALANCE, 0, 127, 1, 64);
-
-	sd->ctrl_handler = &info->hdl;
-
-	ret = info->hdl.error;
-	if (ret)
-		goto np_err;
-
-	info->i2c_reg_page	= -1;
-	info->curr_fmt		= &noon010_formats[0];
-	info->curr_win		= &noon010_sizes[0];
-
-	/* Request reset asserted so we get put into reset */
-	info->reset = devm_gpiod_get(&client->dev, "reset", GPIOD_OUT_HIGH);
-	if (IS_ERR(info->reset)) {
-		ret = PTR_ERR(info->reset);
-		goto np_err;
-	}
-	gpiod_set_consumer_name(info->reset, "NOON010PC30 NRST");
-
-	/* Request standby asserted so we get put into standby */
-	info->stby = devm_gpiod_get(&client->dev, "standby", GPIOD_OUT_HIGH);
-	if (IS_ERR(info->stby)) {
-		ret = PTR_ERR(info->stby);
-		goto np_err;
-	}
-	gpiod_set_consumer_name(info->reset, "NOON010PC30 STBY");
-
-	for (i = 0; i < NOON010_NUM_SUPPLIES; i++)
-		info->supply[i].supply = noon010_supply_name[i];
-
-	ret = devm_regulator_bulk_get(&client->dev, NOON010_NUM_SUPPLIES,
-				 info->supply);
-	if (ret)
-		goto np_err;
-
-	info->pad.flags = MEDIA_PAD_FL_SOURCE;
-	sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
-	ret = media_entity_pads_init(&sd->entity, 1, &info->pad);
-	if (ret < 0)
-		goto np_err;
-
-	ret = noon010_detect(client, info);
-	if (!ret)
-		return 0;
-
-np_err:
-	v4l2_ctrl_handler_free(&info->hdl);
-	v4l2_device_unregister_subdev(sd);
-	return ret;
-}
-
-static void noon010_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct noon010_info *info = to_noon010(sd);
-
-	v4l2_device_unregister_subdev(sd);
-	v4l2_ctrl_handler_free(&info->hdl);
-	media_entity_cleanup(&sd->entity);
-}
-
-static const struct i2c_device_id noon010_id[] = {
-	{ MODULE_NAME, 0 },
-	{ },
-};
-MODULE_DEVICE_TABLE(i2c, noon010_id);
-
-
-static struct i2c_driver noon010_i2c_driver = {
-	.driver = {
-		.name = MODULE_NAME
-	},
-	.probe_new	= noon010_probe,
-	.remove		= noon010_remove,
-	.id_table	= noon010_id,
-};
-
-module_i2c_driver(noon010_i2c_driver);
-
-MODULE_DESCRIPTION("Siliconfile NOON010PC30 camera driver");
-MODULE_AUTHOR("Sylwester Nawrocki <s.nawrocki@samsung.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/ov13b10.c b/drivers/media/i2c/ov13b10.c
index 549e5d93e568..c1430044fb1e 100644
--- a/drivers/media/i2c/ov13b10.c
+++ b/drivers/media/i2c/ov13b10.c
@@ -243,7 +243,6 @@ static const struct ov13b10_reg mipi_data_rate_1120mbps[] = {
 	{0x5047, 0xa4},
 	{0x5048, 0x20},
 	{0x5049, 0xa4},
-	{0x0100, 0x01},
 };
 
 static const struct ov13b10_reg mode_4208x3120_regs[] = {
@@ -589,6 +588,9 @@ struct ov13b10 {
 
 	/* Streaming on/off */
 	bool streaming;
+
+	/* True if the device has been identified */
+	bool identified;
 };
 
 #define to_ov13b10(_sd)	container_of(_sd, struct ov13b10, sd)
@@ -1023,12 +1025,42 @@ ov13b10_set_pad_format(struct v4l2_subdev *sd,
 	return 0;
 }
 
+/* Verify chip ID */
+static int ov13b10_identify_module(struct ov13b10 *ov13b)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&ov13b->sd);
+	int ret;
+	u32 val;
+
+	if (ov13b->identified)
+		return 0;
+
+	ret = ov13b10_read_reg(ov13b, OV13B10_REG_CHIP_ID,
+			       OV13B10_REG_VALUE_24BIT, &val);
+	if (ret)
+		return ret;
+
+	if (val != OV13B10_CHIP_ID) {
+		dev_err(&client->dev, "chip id mismatch: %x!=%x\n",
+			OV13B10_CHIP_ID, val);
+		return -EIO;
+	}
+
+	ov13b->identified = true;
+
+	return 0;
+}
+
 static int ov13b10_start_streaming(struct ov13b10 *ov13b)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(&ov13b->sd);
 	const struct ov13b10_reg_list *reg_list;
 	int ret, link_freq_index;
 
+	ret = ov13b10_identify_module(ov13b);
+	if (ret)
+		return ret;
+
 	/* Get out of from software reset */
 	ret = ov13b10_write_reg(ov13b, OV13B10_REG_SOFTWARE_RST,
 				OV13B10_REG_VALUE_08BIT, OV13B10_SOFTWARE_RST);
@@ -1144,27 +1176,6 @@ error:
 	return ret;
 }
 
-/* Verify chip ID */
-static int ov13b10_identify_module(struct ov13b10 *ov13b)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&ov13b->sd);
-	int ret;
-	u32 val;
-
-	ret = ov13b10_read_reg(ov13b, OV13B10_REG_CHIP_ID,
-			       OV13B10_REG_VALUE_24BIT, &val);
-	if (ret)
-		return ret;
-
-	if (val != OV13B10_CHIP_ID) {
-		dev_err(&client->dev, "chip id mismatch: %x!=%x\n",
-			OV13B10_CHIP_ID, val);
-		return -EIO;
-	}
-
-	return 0;
-}
-
 static const struct v4l2_subdev_video_ops ov13b10_video_ops = {
 	.s_stream = ov13b10_set_stream,
 };
@@ -1379,6 +1390,7 @@ out_err:
 static int ov13b10_probe(struct i2c_client *client)
 {
 	struct ov13b10 *ov13b;
+	bool full_power;
 	int ret;
 
 	/* Check HW config */
@@ -1395,11 +1407,14 @@ static int ov13b10_probe(struct i2c_client *client)
 	/* Initialize subdev */
 	v4l2_i2c_subdev_init(&ov13b->sd, client, &ov13b10_subdev_ops);
 
-	/* Check module identity */
-	ret = ov13b10_identify_module(ov13b);
-	if (ret) {
-		dev_err(&client->dev, "failed to find sensor: %d\n", ret);
-		return ret;
+	full_power = acpi_dev_state_d0(&client->dev);
+	if (full_power) {
+		/* Check module identity */
+		ret = ov13b10_identify_module(ov13b);
+		if (ret) {
+			dev_err(&client->dev, "failed to find sensor: %d\n", ret);
+			return ret;
+		}
 	}
 
 	/* Set default mode to max resolution */
@@ -1431,7 +1446,10 @@ static int ov13b10_probe(struct i2c_client *client)
 	 * Device is already turned on by i2c-core with ACPI domain PM.
 	 * Enable runtime PM and turn off the device.
 	 */
-	pm_runtime_set_active(&client->dev);
+
+	/* Set the device's state to active if it's in D0 state. */
+	if (full_power)
+		pm_runtime_set_active(&client->dev);
 	pm_runtime_enable(&client->dev);
 	pm_runtime_idle(&client->dev);
 
@@ -1480,6 +1498,7 @@ static struct i2c_driver ov13b10_i2c_driver = {
 	},
 	.probe_new = ov13b10_probe,
 	.remove = ov13b10_remove,
+	.flags = I2C_DRV_ACPI_WAIVE_D0_PROBE,
 };
 
 module_i2c_driver(ov13b10_i2c_driver);
diff --git a/drivers/media/i2c/ov2685.c b/drivers/media/i2c/ov2685.c
index 1c80b121e7d6..f119a93e7c64 100644
--- a/drivers/media/i2c/ov2685.c
+++ b/drivers/media/i2c/ov2685.c
@@ -17,6 +17,7 @@
 #include <media/media-entity.h>
 #include <media/v4l2-async.h>
 #include <media/v4l2-ctrls.h>
+#include <media/v4l2-fwnode.h>
 #include <media/v4l2-subdev.h>
 
 #define CHIP_ID				0x2685
@@ -55,6 +56,9 @@
 #define OV2685_REG_VALUE_16BIT		2
 #define OV2685_REG_VALUE_24BIT		3
 
+#define OV2685_NATIVE_WIDTH		1616
+#define OV2685_NATIVE_HEIGHT		1216
+
 #define OV2685_LANES			1
 #define OV2685_BITS_PER_SAMPLE		10
 
@@ -77,6 +81,7 @@ struct ov2685_mode {
 	u32 exp_def;
 	u32 hts_def;
 	u32 vts_def;
+	const struct v4l2_rect *analog_crop;
 	const struct regval *reg_list;
 };
 
@@ -230,6 +235,13 @@ static const int ov2685_test_pattern_val[] = {
 	OV2685_TEST_PATTERN_COLOR_SQUARE,
 };
 
+static const struct v4l2_rect ov2685_analog_crop = {
+	.left	= 8,
+	.top	= 8,
+	.width	= 1600,
+	.height	= 1200,
+};
+
 static const struct ov2685_mode supported_modes[] = {
 	{
 		.width = 1600,
@@ -237,6 +249,7 @@ static const struct ov2685_mode supported_modes[] = {
 		.exp_def = 0x04ee,
 		.hts_def = 0x06a4,
 		.vts_def = 0x050e,
+		.analog_crop = &ov2685_analog_crop,
 		.reg_list = ov2685_1600x1200_regs,
 	},
 };
@@ -383,6 +396,53 @@ static int ov2685_enum_frame_sizes(struct v4l2_subdev *sd,
 	return 0;
 }
 
+static const struct v4l2_rect *
+__ov2685_get_pad_crop(struct ov2685 *ov2685,
+		      struct v4l2_subdev_state *state, unsigned int pad,
+		      enum v4l2_subdev_format_whence which)
+{
+	const struct ov2685_mode *mode = ov2685->cur_mode;
+
+	switch (which) {
+	case V4L2_SUBDEV_FORMAT_TRY:
+		return v4l2_subdev_get_try_crop(&ov2685->subdev, state, pad);
+	case V4L2_SUBDEV_FORMAT_ACTIVE:
+		return mode->analog_crop;
+	}
+
+	return NULL;
+}
+
+static int ov2685_get_selection(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *sd_state,
+				struct v4l2_subdev_selection *sel)
+{
+	struct ov2685 *ov2685 = to_ov2685(sd);
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		mutex_lock(&ov2685->mutex);
+		sel->r = *__ov2685_get_pad_crop(ov2685, sd_state, sel->pad,
+				sel->which);
+		mutex_unlock(&ov2685->mutex);
+		break;
+	case V4L2_SEL_TGT_NATIVE_SIZE:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		sel->r.top = 0;
+		sel->r.left = 0;
+		sel->r.width = OV2685_NATIVE_WIDTH;
+		sel->r.height = OV2685_NATIVE_HEIGHT;
+		break;
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r = ov2685_analog_crop;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 /* Calculate the delay in us by clock rate and clock cycles */
 static inline u32 ov2685_cal_delay(u32 cycles)
 {
@@ -419,8 +479,10 @@ static int __ov2685_power_on(struct ov2685 *ov2685)
 	 * writing register before .s_stream() as a workaround
 	 */
 	ret = ov2685_write_array(ov2685->client, ov2685->cur_mode->reg_list);
-	if (ret)
+	if (ret) {
+		dev_err(dev, "Failed to set regs for power on\n");
 		goto disable_supplies;
+	}
 
 	return 0;
 
@@ -589,6 +651,8 @@ static const struct v4l2_subdev_pad_ops ov2685_pad_ops = {
 	.enum_frame_size = ov2685_enum_frame_sizes,
 	.get_fmt = ov2685_get_fmt,
 	.set_fmt = ov2685_set_fmt,
+	.get_selection = ov2685_get_selection,
+	.set_selection = ov2685_get_selection,
 };
 
 static const struct v4l2_subdev_ops ov2685_subdev_ops = {
@@ -611,13 +675,14 @@ static int ov2685_initialize_controls(struct ov2685 *ov2685)
 	const struct ov2685_mode *mode;
 	struct v4l2_ctrl_handler *handler;
 	struct v4l2_ctrl *ctrl;
+	struct v4l2_fwnode_device_properties props;
 	u64 exposure_max;
 	u32 pixel_rate, h_blank;
 	int ret;
 
 	handler = &ov2685->ctrl_handler;
 	mode = ov2685->cur_mode;
-	ret = v4l2_ctrl_handler_init(handler, 8);
+	ret = v4l2_ctrl_handler_init(handler, 10);
 	if (ret)
 		return ret;
 	handler->lock = &ov2685->mutex;
@@ -659,6 +724,15 @@ static int ov2685_initialize_controls(struct ov2685 *ov2685)
 				ARRAY_SIZE(ov2685_test_pattern_menu) - 1,
 				0, 0, ov2685_test_pattern_menu);
 
+	/* set properties from fwnode (e.g. rotation, orientation) */
+	ret = v4l2_fwnode_device_parse(&ov2685->client->dev, &props);
+	if (ret)
+		goto err_free_handler;
+
+	ret = v4l2_ctrl_new_fwnode_properties(handler, &ov2685_ctrl_ops, &props);
+	if (ret)
+		goto err_free_handler;
+
 	if (handler->error) {
 		ret = handler->error;
 		dev_err(&ov2685->client->dev,
@@ -733,7 +807,7 @@ static int ov2685_probe(struct i2c_client *client)
 	if (clk_get_rate(ov2685->xvclk) != OV2685_XVCLK_FREQ)
 		dev_warn(dev, "xvclk mismatched, modes are based on 24MHz\n");
 
-	ov2685->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	ov2685->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
 	if (IS_ERR(ov2685->reset_gpio)) {
 		dev_err(dev, "Failed to get reset-gpios\n");
 		return -EINVAL;
diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c
index 847a7bbb69c5..233576ee9503 100644
--- a/drivers/media/i2c/ov5647.c
+++ b/drivers/media/i2c/ov5647.c
@@ -58,6 +58,7 @@
 #define OV5647_REG_MIPI_CTRL00		0x4800
 #define OV5647_REG_MIPI_CTRL14		0x4814
 #define OV5647_REG_AWB			0x5001
+#define OV5647_REG_ISPCTRL3D		0x503d
 
 #define REG_TERM 0xfffe
 #define VAL_TERM 0xfe
@@ -116,6 +117,20 @@ static inline struct ov5647 *to_sensor(struct v4l2_subdev *sd)
 	return container_of(sd, struct ov5647, sd);
 }
 
+static const char * const ov5647_test_pattern_menu[] = {
+	"Disabled",
+	"Color Bars",
+	"Color Squares",
+	"Random Data",
+};
+
+static const u8 ov5647_test_pattern_val[] = {
+	0x00,	/* Disabled */
+	0x80,	/* Color Bars */
+	0x82,	/* Color Squares */
+	0x81,	/* Random Data */
+};
+
 static const struct regval_list sensor_oe_disable_regs[] = {
 	{0x3000, 0x00},
 	{0x3001, 0x00},
@@ -614,23 +629,29 @@ static int ov5647_write(struct v4l2_subdev *sd, u16 reg, u8 val)
 
 static int ov5647_read(struct v4l2_subdev *sd, u16 reg, u8 *val)
 {
-	unsigned char data_w[2] = { reg >> 8, reg & 0xff };
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	u8 buf[2] = { reg >> 8, reg & 0xff };
+	struct i2c_msg msg[2];
 	int ret;
 
-	ret = i2c_master_send(client, data_w, 2);
-	if (ret < 0) {
-		dev_dbg(&client->dev, "%s: i2c write error, reg: %x\n",
-			__func__, reg);
-		return ret;
+	msg[0].addr = client->addr;
+	msg[0].flags = client->flags;
+	msg[0].buf = buf;
+	msg[0].len = sizeof(buf);
+
+	msg[1].addr = client->addr;
+	msg[1].flags = client->flags | I2C_M_RD;
+	msg[1].buf = buf;
+	msg[1].len = 1;
+
+	ret = i2c_transfer(client->adapter, msg, 2);
+	if (ret != 2) {
+		dev_err(&client->dev, "%s: i2c read error, reg: %x = %d\n",
+			__func__, reg, ret);
+		return ret >= 0 ? -EINVAL : ret;
 	}
 
-	ret = i2c_master_recv(client, val, 1);
-	if (ret < 0) {
-		dev_dbg(&client->dev, "%s: i2c read error, reg: %x\n",
-				__func__, reg);
-		return ret;
-	}
+	*val = buf[0];
 
 	return 0;
 }
@@ -1242,6 +1263,10 @@ static int ov5647_s_ctrl(struct v4l2_ctrl *ctrl)
 		ret = ov5647_write16(sd, OV5647_REG_VTS_HI,
 				     sensor->mode->format.height + ctrl->val);
 		break;
+	case V4L2_CID_TEST_PATTERN:
+		ret = ov5647_write(sd, OV5647_REG_ISPCTRL3D,
+				   ov5647_test_pattern_val[ctrl->val]);
+		break;
 
 	/* Read-only, but we adjust it based on mode. */
 	case V4L2_CID_PIXEL_RATE:
@@ -1270,7 +1295,7 @@ static int ov5647_init_controls(struct ov5647 *sensor)
 	struct i2c_client *client = v4l2_get_subdevdata(&sensor->sd);
 	int hblank, exposure_max, exposure_def;
 
-	v4l2_ctrl_handler_init(&sensor->ctrls, 8);
+	v4l2_ctrl_handler_init(&sensor->ctrls, 9);
 
 	v4l2_ctrl_new_std(&sensor->ctrls, &ov5647_ctrl_ops,
 			  V4L2_CID_AUTOGAIN, 0, 1, 1, 0);
@@ -1314,6 +1339,11 @@ static int ov5647_init_controls(struct ov5647 *sensor)
 					   sensor->mode->vts -
 					   sensor->mode->format.height);
 
+	v4l2_ctrl_new_std_menu_items(&sensor->ctrls, &ov5647_ctrl_ops,
+				     V4L2_CID_TEST_PATTERN,
+				     ARRAY_SIZE(ov5647_test_pattern_menu) - 1,
+				     0, 0, ov5647_test_pattern_menu);
+
 	if (sensor->ctrls.error)
 		goto handler_free;
 
diff --git a/drivers/media/i2c/ov5670.c b/drivers/media/i2c/ov5670.c
index f79d908f4531..c026610d0f31 100644
--- a/drivers/media/i2c/ov5670.c
+++ b/drivers/media/i2c/ov5670.c
@@ -29,6 +29,12 @@
 #define OV5670_REG_SOFTWARE_RST		0x0103
 #define OV5670_SOFTWARE_RST		0x01
 
+#define OV5670_MIPI_SC_CTRL0_REG		0x3018
+#define OV5670_MIPI_SC_CTRL0_LANES(v)		((((v) - 1) << 5) & \
+						 GENMASK(7, 5))
+#define OV5670_MIPI_SC_CTRL0_MIPI_EN		BIT(4)
+#define OV5670_MIPI_SC_CTRL0_RESERVED		BIT(1)
+
 /* vertical-timings from sensor */
 #define OV5670_REG_VTS			0x380e
 #define OV5670_VTS_30FPS		0x0808 /* default for 30 fps */
@@ -92,7 +98,6 @@ struct ov5670_reg_list {
 };
 
 struct ov5670_link_freq_config {
-	u32 pixel_rate;
 	const struct ov5670_reg_list reg_list;
 };
 
@@ -163,7 +168,6 @@ static const struct ov5670_reg mode_2592x1944_regs[] = {
 	{0x3005, 0xf0},
 	{0x3007, 0x00},
 	{0x3015, 0x0f},
-	{0x3018, 0x32},
 	{0x301a, 0xf0},
 	{0x301b, 0xf0},
 	{0x301c, 0xf0},
@@ -429,7 +433,6 @@ static const struct ov5670_reg mode_1296x972_regs[] = {
 	{0x3005, 0xf0},
 	{0x3007, 0x00},
 	{0x3015, 0x0f},
-	{0x3018, 0x32},
 	{0x301a, 0xf0},
 	{0x301b, 0xf0},
 	{0x301c, 0xf0},
@@ -695,7 +698,6 @@ static const struct ov5670_reg mode_648x486_regs[] = {
 	{0x3005, 0xf0},
 	{0x3007, 0x00},
 	{0x3015, 0x0f},
-	{0x3018, 0x32},
 	{0x301a, 0xf0},
 	{0x301b, 0xf0},
 	{0x301c, 0xf0},
@@ -961,7 +963,6 @@ static const struct ov5670_reg mode_2560x1440_regs[] = {
 	{0x3005, 0xf0},
 	{0x3007, 0x00},
 	{0x3015, 0x0f},
-	{0x3018, 0x32},
 	{0x301a, 0xf0},
 	{0x301b, 0xf0},
 	{0x301c, 0xf0},
@@ -1226,7 +1227,6 @@ static const struct ov5670_reg mode_1280x720_regs[] = {
 	{0x3005, 0xf0},
 	{0x3007, 0x00},
 	{0x3015, 0x0f},
-	{0x3018, 0x32},
 	{0x301a, 0xf0},
 	{0x301b, 0xf0},
 	{0x301c, 0xf0},
@@ -1492,7 +1492,6 @@ static const struct ov5670_reg mode_640x360_regs[] = {
 	{0x3005, 0xf0},
 	{0x3007, 0x00},
 	{0x3015, 0x0f},
-	{0x3018, 0x32},
 	{0x301a, 0xf0},
 	{0x301b, 0xf0},
 	{0x301c, 0xf0},
@@ -1762,8 +1761,6 @@ static const char * const ov5670_test_pattern_menu[] = {
 #define OV5670_LINK_FREQ_422MHZ_INDEX	0
 static const struct ov5670_link_freq_config link_freq_configs[] = {
 	{
-		/* pixel_rate = link_freq * 2 * nr_of_lanes / bits_per_sample */
-		.pixel_rate = (OV5670_LINK_FREQ_422MHZ * 2 * 2) / 10,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mipi_data_rate_840mbps),
 			.regs = mipi_data_rate_840mbps,
@@ -1859,6 +1856,7 @@ static const struct ov5670_mode supported_modes[] = {
 struct ov5670 {
 	struct v4l2_subdev sd;
 	struct media_pad pad;
+	struct v4l2_fwnode_endpoint endpoint;
 
 	struct v4l2_ctrl_handler ctrl_handler;
 	/* V4L2 Controls */
@@ -2101,9 +2099,13 @@ static const struct v4l2_ctrl_ops ov5670_ctrl_ops = {
 /* Initialize control handlers */
 static int ov5670_init_controls(struct ov5670 *ov5670)
 {
+	struct v4l2_mbus_config_mipi_csi2 *bus_mipi_csi2 =
+		&ov5670->endpoint.bus.mipi_csi2;
 	struct i2c_client *client = v4l2_get_subdevdata(&ov5670->sd);
 	struct v4l2_fwnode_device_properties props;
 	struct v4l2_ctrl_handler *ctrl_hdlr;
+	unsigned int lanes_count;
+	s64 mipi_pixel_rate;
 	s64 vblank_max;
 	s64 vblank_def;
 	s64 vblank_min;
@@ -2124,12 +2126,15 @@ static int ov5670_init_controls(struct ov5670 *ov5670)
 		ov5670->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
 	/* By default, V4L2_CID_PIXEL_RATE is read only */
+	lanes_count = bus_mipi_csi2->num_data_lanes;
+	mipi_pixel_rate = OV5670_LINK_FREQ_422MHZ * 2 * lanes_count / 10;
+
 	ov5670->pixel_rate = v4l2_ctrl_new_std(ctrl_hdlr, &ov5670_ctrl_ops,
 					       V4L2_CID_PIXEL_RATE,
-					       link_freq_configs[0].pixel_rate,
-					       link_freq_configs[0].pixel_rate,
+					       mipi_pixel_rate,
+					       mipi_pixel_rate,
 					       1,
-					       link_freq_configs[0].pixel_rate);
+					       mipi_pixel_rate);
 
 	vblank_max = OV5670_VTS_MAX - ov5670->cur_mode->height;
 	vblank_def = ov5670->cur_mode->vts_def - ov5670->cur_mode->height;
@@ -2288,8 +2293,13 @@ static int ov5670_set_pad_format(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_format *fmt)
 {
 	struct ov5670 *ov5670 = to_ov5670(sd);
+	struct v4l2_mbus_config_mipi_csi2 *bus_mipi_csi2 =
+		&ov5670->endpoint.bus.mipi_csi2;
 	const struct ov5670_mode *mode;
+	unsigned int lanes_count;
+	s64 mipi_pixel_rate;
 	s32 vblank_def;
+	s64 link_freq;
 	s32 h_blank;
 
 	mutex_lock(&ov5670->mutex);
@@ -2306,9 +2316,14 @@ static int ov5670_set_pad_format(struct v4l2_subdev *sd,
 	} else {
 		ov5670->cur_mode = mode;
 		__v4l2_ctrl_s_ctrl(ov5670->link_freq, mode->link_freq_index);
+
+		lanes_count = bus_mipi_csi2->num_data_lanes;
+		link_freq = link_freq_menu_items[mode->link_freq_index];
+		/* pixel_rate = link_freq * 2 * nr_of_lanes / bits_per_sample */
+		mipi_pixel_rate = div_s64(link_freq * 2 * lanes_count, 10);
 		__v4l2_ctrl_s_ctrl_int64(
 			ov5670->pixel_rate,
-			link_freq_configs[mode->link_freq_index].pixel_rate);
+			mipi_pixel_rate);
 		/* Update limits and set FPS to default */
 		vblank_def = ov5670->cur_mode->vts_def -
 			     ov5670->cur_mode->height;
@@ -2361,6 +2376,19 @@ static int ov5670_identify_module(struct ov5670 *ov5670)
 	return 0;
 }
 
+static int ov5670_mipi_configure(struct ov5670 *ov5670)
+{
+	struct v4l2_mbus_config_mipi_csi2 *bus_mipi_csi2 =
+		&ov5670->endpoint.bus.mipi_csi2;
+	unsigned int lanes_count = bus_mipi_csi2->num_data_lanes;
+
+	return ov5670_write_reg(ov5670, OV5670_MIPI_SC_CTRL0_REG,
+				OV5670_REG_VALUE_08BIT,
+				OV5670_MIPI_SC_CTRL0_LANES(lanes_count) |
+				OV5670_MIPI_SC_CTRL0_MIPI_EN |
+				OV5670_MIPI_SC_CTRL0_RESERVED);
+}
+
 /* Prepare streaming by writing default values and customized values */
 static int ov5670_start_streaming(struct ov5670 *ov5670)
 {
@@ -2399,6 +2427,12 @@ static int ov5670_start_streaming(struct ov5670 *ov5670)
 		return ret;
 	}
 
+	ret = ov5670_mipi_configure(ov5670);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to configure MIPI\n", __func__);
+		return ret;
+	}
+
 	ret = __v4l2_ctrl_handler_setup(ov5670->sd.ctrl_handler);
 	if (ret)
 		return ret;
@@ -2647,23 +2681,20 @@ static int ov5670_gpio_probe(struct ov5670 *ov5670)
 
 static int ov5670_probe(struct i2c_client *client)
 {
+	struct fwnode_handle *handle;
 	struct ov5670 *ov5670;
-	const char *err_msg;
 	u32 input_clk = 0;
 	bool full_power;
 	int ret;
 
 	ov5670 = devm_kzalloc(&client->dev, sizeof(*ov5670), GFP_KERNEL);
-	if (!ov5670) {
-		ret = -ENOMEM;
-		err_msg = "devm_kzalloc() error";
-		goto error_print;
-	}
+	if (!ov5670)
+		return -ENOMEM;
 
-	ov5670->xvclk = devm_clk_get(&client->dev, NULL);
+	ov5670->xvclk = devm_clk_get_optional(&client->dev, NULL);
 	if (!IS_ERR_OR_NULL(ov5670->xvclk))
 		input_clk = clk_get_rate(ov5670->xvclk);
-	else if (PTR_ERR(ov5670->xvclk) == -ENOENT)
+	else if (!ov5670->xvclk || PTR_ERR(ov5670->xvclk) == -ENOENT)
 		device_property_read_u32(&client->dev, "clock-frequency",
 					 &input_clk);
 	else
@@ -2680,29 +2711,38 @@ static int ov5670_probe(struct i2c_client *client)
 	v4l2_i2c_subdev_init(&ov5670->sd, client, &ov5670_subdev_ops);
 
 	ret = ov5670_regulators_probe(ov5670);
-	if (ret) {
-		err_msg = "Regulators probe failed";
-		goto error_print;
-	}
+	if (ret)
+		return dev_err_probe(&client->dev, ret, "Regulators probe failed\n");
 
 	ret = ov5670_gpio_probe(ov5670);
-	if (ret) {
-		err_msg = "GPIO probe failed";
-		goto error_print;
-	}
+	if (ret)
+		return dev_err_probe(&client->dev, ret, "GPIO probe failed\n");
+
+	/* Graph Endpoint */
+	handle = fwnode_graph_get_next_endpoint(dev_fwnode(&client->dev), NULL);
+	if (!handle)
+		return dev_err_probe(&client->dev, -ENXIO, "Endpoint for node get failed\n");
+
+	ov5670->endpoint.bus_type = V4L2_MBUS_CSI2_DPHY;
+	ov5670->endpoint.bus.mipi_csi2.num_data_lanes = 2;
+
+	ret = v4l2_fwnode_endpoint_alloc_parse(handle, &ov5670->endpoint);
+	fwnode_handle_put(handle);
+	if (ret)
+		return dev_err_probe(&client->dev, ret, "Endpoint parse failed\n");
 
 	full_power = acpi_dev_state_d0(&client->dev);
 	if (full_power) {
 		ret = ov5670_runtime_resume(&client->dev);
 		if (ret) {
-			err_msg = "Power up failed";
-			goto error_print;
+			dev_err_probe(&client->dev, ret, "Power up failed\n");
+			goto error_endpoint;
 		}
 
 		/* Check module identity */
 		ret = ov5670_identify_module(ov5670);
 		if (ret) {
-			err_msg = "ov5670_identify_module() error";
+			dev_err_probe(&client->dev, ret, "ov5670_identify_module() error\n");
 			goto error_power_off;
 		}
 	}
@@ -2714,7 +2754,7 @@ static int ov5670_probe(struct i2c_client *client)
 
 	ret = ov5670_init_controls(ov5670);
 	if (ret) {
-		err_msg = "ov5670_init_controls() error";
+		dev_err_probe(&client->dev, ret, "ov5670_init_controls() error\n");
 		goto error_mutex_destroy;
 	}
 
@@ -2727,7 +2767,7 @@ static int ov5670_probe(struct i2c_client *client)
 	ov5670->pad.flags = MEDIA_PAD_FL_SOURCE;
 	ret = media_entity_pads_init(&ov5670->sd.entity, 1, &ov5670->pad);
 	if (ret) {
-		err_msg = "media_entity_pads_init() error";
+		dev_err_probe(&client->dev, ret, "media_entity_pads_init() error\n");
 		goto error_handler_free;
 	}
 
@@ -2741,7 +2781,7 @@ static int ov5670_probe(struct i2c_client *client)
 	/* Async register for subdev */
 	ret = v4l2_async_register_subdev_sensor(&ov5670->sd);
 	if (ret < 0) {
-		err_msg = "v4l2_async_register_subdev() error";
+		dev_err_probe(&client->dev, ret, "v4l2_async_register_subdev() error\n");
 		goto error_pm_disable;
 	}
 
@@ -2764,8 +2804,8 @@ error_power_off:
 	if (full_power)
 		ov5670_runtime_suspend(&client->dev);
 
-error_print:
-	dev_err(&client->dev, "%s: %s %d\n", __func__, err_msg, ret);
+error_endpoint:
+	v4l2_fwnode_endpoint_free(&ov5670->endpoint);
 
 	return ret;
 }
@@ -2782,6 +2822,8 @@ static void ov5670_remove(struct i2c_client *client)
 
 	pm_runtime_disable(&client->dev);
 	ov5670_runtime_suspend(&client->dev);
+
+	v4l2_fwnode_endpoint_free(&ov5670->endpoint);
 }
 
 static const struct dev_pm_ops ov5670_pm_ops = {
diff --git a/drivers/media/i2c/ov7670.c b/drivers/media/i2c/ov7670.c
index b1bb0833571e..ecbded4f0765 100644
--- a/drivers/media/i2c/ov7670.c
+++ b/drivers/media/i2c/ov7670.c
@@ -1894,14 +1894,9 @@ static int ov7670_probe(struct i2c_client *client)
 			info->pclk_hb_disable = true;
 	}
 
-	info->clk = devm_clk_get(&client->dev, "xclk"); /* optional */
-	if (IS_ERR(info->clk)) {
-		ret = PTR_ERR(info->clk);
-		if (ret == -ENOENT)
-			info->clk = NULL;
-		else
-			return ret;
-	}
+	info->clk = devm_clk_get_optional(&client->dev, "xclk");
+	if (IS_ERR(info->clk))
+		return PTR_ERR(info->clk);
 
 	ret = ov7670_init_gpio(client, info);
 	if (ret)
diff --git a/drivers/media/i2c/ov8856.c b/drivers/media/i2c/ov8856.c
index cf8384e09413..b5c7881383ca 100644
--- a/drivers/media/i2c/ov8856.c
+++ b/drivers/media/i2c/ov8856.c
@@ -1709,46 +1709,6 @@ static int ov8856_identify_module(struct ov8856 *ov8856)
 		return -ENXIO;
 	}
 
-	ret = ov8856_write_reg(ov8856, OV8856_REG_MODE_SELECT,
-			       OV8856_REG_VALUE_08BIT, OV8856_MODE_STREAMING);
-	if (ret)
-		return ret;
-
-	ret = ov8856_write_reg(ov8856, OV8856_OTP_MODE_CTRL,
-			       OV8856_REG_VALUE_08BIT, OV8856_OTP_MODE_AUTO);
-	if (ret) {
-		dev_err(&client->dev, "failed to set otp mode");
-		return ret;
-	}
-
-	ret = ov8856_write_reg(ov8856, OV8856_OTP_LOAD_CTRL,
-			       OV8856_REG_VALUE_08BIT,
-			       OV8856_OTP_LOAD_CTRL_ENABLE);
-	if (ret) {
-		dev_err(&client->dev, "failed to enable load control");
-		return ret;
-	}
-
-	ret = ov8856_read_reg(ov8856, OV8856_MODULE_REVISION,
-			      OV8856_REG_VALUE_08BIT, &val);
-	if (ret) {
-		dev_err(&client->dev, "failed to read module revision");
-		return ret;
-	}
-
-	dev_info(&client->dev, "OV8856 revision %x (%s) at address 0x%02x\n",
-		 val,
-		 val == OV8856_2A_MODULE ? "2A" :
-		 val == OV8856_1B_MODULE ? "1B" : "unknown revision",
-		 client->addr);
-
-	ret = ov8856_write_reg(ov8856, OV8856_REG_MODE_SELECT,
-			       OV8856_REG_VALUE_08BIT, OV8856_MODE_STANDBY);
-	if (ret) {
-		dev_err(&client->dev, "failed to exit streaming mode");
-		return ret;
-	}
-
 	ov8856->identified = true;
 
 	return 0;
diff --git a/drivers/media/i2c/s5k6aa.c b/drivers/media/i2c/s5k6aa.c
deleted file mode 100644
index 5996153371fc..000000000000
--- a/drivers/media/i2c/s5k6aa.c
+++ /dev/null
@@ -1,1652 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for Samsung S5K6AAFX SXGA 1/6" 1.3M CMOS Image Sensor
- * with embedded SoC ISP.
- *
- * Copyright (C) 2011, Samsung Electronics Co., Ltd.
- * Sylwester Nawrocki <s.nawrocki@samsung.com>
- *
- * Based on a driver authored by Dongsoo Nathaniel Kim.
- * Copyright (C) 2009, Dongsoo Nathaniel Kim <dongsoo45.kim@samsung.com>
- */
-
-#include <linux/clk.h>
-#include <linux/delay.h>
-#include <linux/gpio.h>
-#include <linux/i2c.h>
-#include <linux/media.h>
-#include <linux/module.h>
-#include <linux/regulator/consumer.h>
-#include <linux/slab.h>
-
-#include <media/media-entity.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-subdev.h>
-#include <media/v4l2-mediabus.h>
-#include <media/i2c/s5k6aa.h>
-
-static int debug;
-module_param(debug, int, 0644);
-
-#define DRIVER_NAME			"S5K6AA"
-
-/* The token to indicate array termination */
-#define S5K6AA_TERM			0xffff
-#define S5K6AA_OUT_WIDTH_DEF		640
-#define S5K6AA_OUT_HEIGHT_DEF		480
-#define S5K6AA_WIN_WIDTH_MAX		1280
-#define S5K6AA_WIN_HEIGHT_MAX		1024
-#define S5K6AA_WIN_WIDTH_MIN		8
-#define S5K6AA_WIN_HEIGHT_MIN		8
-
-/*
- * H/W register Interface (0xD0000000 - 0xD0000FFF)
- */
-#define AHB_MSB_ADDR_PTR		0xfcfc
-#define GEN_REG_OFFSH			0xd000
-#define REG_CMDWR_ADDRH			0x0028
-#define REG_CMDWR_ADDRL			0x002a
-#define REG_CMDRD_ADDRH			0x002c
-#define REG_CMDRD_ADDRL			0x002e
-#define REG_CMDBUF0_ADDR		0x0f12
-#define REG_CMDBUF1_ADDR		0x0f10
-
-/*
- * Host S/W Register interface (0x70000000 - 0x70002000)
- * The value of the two most significant address bytes is 0x7000,
- * (HOST_SWIF_OFFS_H). The register addresses below specify 2 LSBs.
- */
-#define HOST_SWIF_OFFSH			0x7000
-
-/* Initialization parameters */
-/* Master clock frequency in KHz */
-#define REG_I_INCLK_FREQ_L		0x01b8
-#define REG_I_INCLK_FREQ_H		0x01ba
-#define  MIN_MCLK_FREQ_KHZ		6000U
-#define  MAX_MCLK_FREQ_KHZ		27000U
-#define REG_I_USE_NPVI_CLOCKS		0x01c6
-#define REG_I_USE_NMIPI_CLOCKS		0x01c8
-
-/* Clock configurations, n = 0..2. REG_I_* frequency unit is 4 kHz. */
-#define REG_I_OPCLK_4KHZ(n)		((n) * 6 + 0x01cc)
-#define REG_I_MIN_OUTRATE_4KHZ(n)	((n) * 6 + 0x01ce)
-#define REG_I_MAX_OUTRATE_4KHZ(n)	((n) * 6 + 0x01d0)
-#define  SYS_PLL_OUT_FREQ		(48000000 / 4000)
-#define  PCLK_FREQ_MIN			(24000000 / 4000)
-#define  PCLK_FREQ_MAX			(48000000 / 4000)
-#define REG_I_INIT_PARAMS_UPDATED	0x01e0
-#define REG_I_ERROR_INFO		0x01e2
-
-/* General purpose parameters */
-#define REG_USER_BRIGHTNESS		0x01e4
-#define REG_USER_CONTRAST		0x01e6
-#define REG_USER_SATURATION		0x01e8
-#define REG_USER_SHARPBLUR		0x01ea
-
-#define REG_G_SPEC_EFFECTS		0x01ee
-#define REG_G_ENABLE_PREV		0x01f0
-#define REG_G_ENABLE_PREV_CHG		0x01f2
-#define REG_G_NEW_CFG_SYNC		0x01f8
-#define REG_G_PREVZOOM_IN_WIDTH		0x020a
-#define REG_G_PREVZOOM_IN_HEIGHT	0x020c
-#define REG_G_PREVZOOM_IN_XOFFS		0x020e
-#define REG_G_PREVZOOM_IN_YOFFS		0x0210
-#define REG_G_INPUTS_CHANGE_REQ		0x021a
-#define REG_G_ACTIVE_PREV_CFG		0x021c
-#define REG_G_PREV_CFG_CHG		0x021e
-#define REG_G_PREV_OPEN_AFTER_CH	0x0220
-#define REG_G_PREV_CFG_ERROR		0x0222
-
-/* Preview control section. n = 0...4. */
-#define PREG(n, x)			((n) * 0x26 + x)
-#define REG_P_OUT_WIDTH(n)		PREG(n, 0x0242)
-#define REG_P_OUT_HEIGHT(n)		PREG(n, 0x0244)
-#define REG_P_FMT(n)			PREG(n, 0x0246)
-#define REG_P_MAX_OUT_RATE(n)		PREG(n, 0x0248)
-#define REG_P_MIN_OUT_RATE(n)		PREG(n, 0x024a)
-#define REG_P_PVI_MASK(n)		PREG(n, 0x024c)
-#define REG_P_CLK_INDEX(n)		PREG(n, 0x024e)
-#define REG_P_FR_RATE_TYPE(n)		PREG(n, 0x0250)
-#define  FR_RATE_DYNAMIC		0
-#define  FR_RATE_FIXED			1
-#define  FR_RATE_FIXED_ACCURATE		2
-#define REG_P_FR_RATE_Q_TYPE(n)		PREG(n, 0x0252)
-#define  FR_RATE_Q_BEST_FRRATE		1 /* Binning enabled */
-#define  FR_RATE_Q_BEST_QUALITY		2 /* Binning disabled */
-/* Frame period in 0.1 ms units */
-#define REG_P_MAX_FR_TIME(n)		PREG(n, 0x0254)
-#define REG_P_MIN_FR_TIME(n)		PREG(n, 0x0256)
-/* Conversion to REG_P_[MAX/MIN]_FR_TIME value; __t: time in us */
-#define  US_TO_FR_TIME(__t)		((__t) / 100)
-#define  S5K6AA_MIN_FR_TIME		33300  /* us */
-#define  S5K6AA_MAX_FR_TIME		650000 /* us */
-#define  S5K6AA_MAX_HIGHRES_FR_TIME	666    /* x100 us */
-/* The below 5 registers are for "device correction" values */
-#define REG_P_COLORTEMP(n)		PREG(n, 0x025e)
-#define REG_P_PREV_MIRROR(n)		PREG(n, 0x0262)
-
-/* Extended image property controls */
-/* Exposure time in 10 us units */
-#define REG_SF_USR_EXPOSURE_L		0x03c6
-#define REG_SF_USR_EXPOSURE_H		0x03c8
-#define REG_SF_USR_EXPOSURE_CHG		0x03ca
-#define REG_SF_USR_TOT_GAIN		0x03cc
-#define REG_SF_USR_TOT_GAIN_CHG		0x03ce
-#define REG_SF_RGAIN			0x03d0
-#define REG_SF_RGAIN_CHG		0x03d2
-#define REG_SF_GGAIN			0x03d4
-#define REG_SF_GGAIN_CHG		0x03d6
-#define REG_SF_BGAIN			0x03d8
-#define REG_SF_BGAIN_CHG		0x03da
-#define REG_SF_FLICKER_QUANT		0x03dc
-#define REG_SF_FLICKER_QUANT_CHG	0x03de
-
-/* Output interface (parallel/MIPI) setup */
-#define REG_OIF_EN_MIPI_LANES		0x03fa
-#define REG_OIF_EN_PACKETS		0x03fc
-#define REG_OIF_CFG_CHG			0x03fe
-
-/* Auto-algorithms enable mask */
-#define REG_DBG_AUTOALG_EN		0x0400
-#define  AALG_ALL_EN_MASK		(1 << 0)
-#define  AALG_AE_EN_MASK		(1 << 1)
-#define  AALG_DIVLEI_EN_MASK		(1 << 2)
-#define  AALG_WB_EN_MASK		(1 << 3)
-#define  AALG_FLICKER_EN_MASK		(1 << 5)
-#define  AALG_FIT_EN_MASK		(1 << 6)
-#define  AALG_WRHW_EN_MASK		(1 << 7)
-
-/* Firmware revision information */
-#define REG_FW_APIVER			0x012e
-#define  S5K6AAFX_FW_APIVER		0x0001
-#define REG_FW_REVISION			0x0130
-
-/* For now we use only one user configuration register set */
-#define S5K6AA_MAX_PRESETS		1
-
-static const char * const s5k6aa_supply_names[] = {
-	"vdd_core",	/* Digital core supply 1.5V (1.4V to 1.6V) */
-	"vdda",		/* Analog power supply 2.8V (2.6V to 3.0V) */
-	"vdd_reg",	/* Regulator input power 1.8V (1.7V to 1.9V)
-			   or 2.8V (2.6V to 3.0) */
-	"vddio",	/* I/O supply 1.8V (1.65V to 1.95V)
-			   or 2.8V (2.5V to 3.1V) */
-};
-#define S5K6AA_NUM_SUPPLIES ARRAY_SIZE(s5k6aa_supply_names)
-
-enum s5k6aa_gpio_id {
-	STBY,
-	RSET,
-	GPIO_NUM,
-};
-
-struct s5k6aa_regval {
-	u16 addr;
-	u16 val;
-};
-
-struct s5k6aa_pixfmt {
-	u32 code;
-	u32 colorspace;
-	/* REG_P_FMT(x) register value */
-	u16 reg_p_fmt;
-};
-
-struct s5k6aa_preset {
-	/* output pixel format and resolution */
-	struct v4l2_mbus_framefmt mbus_fmt;
-	u8 clk_id;
-	u8 index;
-};
-
-struct s5k6aa_ctrls {
-	struct v4l2_ctrl_handler handler;
-	/* Auto / manual white balance cluster */
-	struct v4l2_ctrl *awb;
-	struct v4l2_ctrl *gain_red;
-	struct v4l2_ctrl *gain_blue;
-	struct v4l2_ctrl *gain_green;
-	/* Mirror cluster */
-	struct v4l2_ctrl *hflip;
-	struct v4l2_ctrl *vflip;
-	/* Auto exposure / manual exposure and gain cluster */
-	struct v4l2_ctrl *auto_exp;
-	struct v4l2_ctrl *exposure;
-	struct v4l2_ctrl *gain;
-};
-
-struct s5k6aa_interval {
-	u16 reg_fr_time;
-	struct v4l2_fract interval;
-	/* Maximum rectangle for the interval */
-	struct v4l2_frmsize_discrete size;
-};
-
-struct s5k6aa {
-	struct v4l2_subdev sd;
-	struct media_pad pad;
-
-	enum v4l2_mbus_type bus_type;
-	u8 mipi_lanes;
-
-	int (*s_power)(int enable);
-	struct regulator_bulk_data supplies[S5K6AA_NUM_SUPPLIES];
-	struct s5k6aa_gpio gpio[GPIO_NUM];
-
-	/* external master clock frequency */
-	unsigned long mclk_frequency;
-	/* ISP internal master clock frequency */
-	u16 clk_fop;
-	/* output pixel clock frequency range */
-	u16 pclk_fmin;
-	u16 pclk_fmax;
-
-	unsigned int inv_hflip:1;
-	unsigned int inv_vflip:1;
-
-	/* protects the struct members below */
-	struct mutex lock;
-
-	/* sensor matrix scan window */
-	struct v4l2_rect ccd_rect;
-
-	struct s5k6aa_ctrls ctrls;
-	struct s5k6aa_preset presets[S5K6AA_MAX_PRESETS];
-	struct s5k6aa_preset *preset;
-	const struct s5k6aa_interval *fiv;
-
-	unsigned int streaming:1;
-	unsigned int apply_cfg:1;
-	unsigned int apply_crop:1;
-	unsigned int power;
-};
-
-static struct s5k6aa_regval s5k6aa_analog_config[] = {
-	/* Analog settings */
-	{ 0x112a, 0x0000 }, { 0x1132, 0x0000 },
-	{ 0x113e, 0x0000 }, { 0x115c, 0x0000 },
-	{ 0x1164, 0x0000 }, { 0x1174, 0x0000 },
-	{ 0x1178, 0x0000 }, { 0x077a, 0x0000 },
-	{ 0x077c, 0x0000 }, { 0x077e, 0x0000 },
-	{ 0x0780, 0x0000 }, { 0x0782, 0x0000 },
-	{ 0x0784, 0x0000 }, { 0x0786, 0x0000 },
-	{ 0x0788, 0x0000 }, { 0x07a2, 0x0000 },
-	{ 0x07a4, 0x0000 }, { 0x07a6, 0x0000 },
-	{ 0x07a8, 0x0000 }, { 0x07b6, 0x0000 },
-	{ 0x07b8, 0x0002 }, { 0x07ba, 0x0004 },
-	{ 0x07bc, 0x0004 }, { 0x07be, 0x0005 },
-	{ 0x07c0, 0x0005 }, { S5K6AA_TERM, 0 },
-};
-
-/* TODO: Add RGB888 and Bayer format */
-static const struct s5k6aa_pixfmt s5k6aa_formats[] = {
-	{ MEDIA_BUS_FMT_YUYV8_2X8,	V4L2_COLORSPACE_JPEG,	5 },
-	/* range 16-240 */
-	{ MEDIA_BUS_FMT_YUYV8_2X8,	V4L2_COLORSPACE_REC709,	6 },
-	{ MEDIA_BUS_FMT_RGB565_2X8_BE,	V4L2_COLORSPACE_JPEG,	0 },
-};
-
-static const struct s5k6aa_interval s5k6aa_intervals[] = {
-	{ 1000, {10000, 1000000}, {1280, 1024} }, /* 10 fps */
-	{ 666,  {15000, 1000000}, {1280, 1024} }, /* 15 fps */
-	{ 500,  {20000, 1000000}, {1280, 720} },  /* 20 fps */
-	{ 400,  {25000, 1000000}, {640, 480} },   /* 25 fps */
-	{ 333,  {33300, 1000000}, {640, 480} },   /* 30 fps */
-};
-
-#define S5K6AA_INTERVAL_DEF_INDEX 1 /* 15 fps */
-
-static inline struct v4l2_subdev *ctrl_to_sd(struct v4l2_ctrl *ctrl)
-{
-	return &container_of(ctrl->handler, struct s5k6aa, ctrls.handler)->sd;
-}
-
-static inline struct s5k6aa *to_s5k6aa(struct v4l2_subdev *sd)
-{
-	return container_of(sd, struct s5k6aa, sd);
-}
-
-/* Set initial values for all preview presets */
-static void s5k6aa_presets_data_init(struct s5k6aa *s5k6aa)
-{
-	struct s5k6aa_preset *preset = &s5k6aa->presets[0];
-	int i;
-
-	for (i = 0; i < S5K6AA_MAX_PRESETS; i++) {
-		preset->mbus_fmt.width	= S5K6AA_OUT_WIDTH_DEF;
-		preset->mbus_fmt.height	= S5K6AA_OUT_HEIGHT_DEF;
-		preset->mbus_fmt.code	= s5k6aa_formats[0].code;
-		preset->index		= i;
-		preset->clk_id		= 0;
-		preset++;
-	}
-
-	s5k6aa->fiv = &s5k6aa_intervals[S5K6AA_INTERVAL_DEF_INDEX];
-	s5k6aa->preset = &s5k6aa->presets[0];
-}
-
-static int s5k6aa_i2c_read(struct i2c_client *client, u16 addr, u16 *val)
-{
-	u8 wbuf[2] = {addr >> 8, addr & 0xFF};
-	struct i2c_msg msg[2];
-	u8 rbuf[2];
-	int ret;
-
-	msg[0].addr = client->addr;
-	msg[0].flags = 0;
-	msg[0].len = 2;
-	msg[0].buf = wbuf;
-
-	msg[1].addr = client->addr;
-	msg[1].flags = I2C_M_RD;
-	msg[1].len = 2;
-	msg[1].buf = rbuf;
-
-	ret = i2c_transfer(client->adapter, msg, 2);
-	*val = be16_to_cpu(*((__be16 *)rbuf));
-
-	v4l2_dbg(3, debug, client, "i2c_read: 0x%04X : 0x%04x\n", addr, *val);
-
-	return ret == 2 ? 0 : ret;
-}
-
-static int s5k6aa_i2c_write(struct i2c_client *client, u16 addr, u16 val)
-{
-	u8 buf[4] = {addr >> 8, addr & 0xFF, val >> 8, val & 0xFF};
-
-	int ret = i2c_master_send(client, buf, 4);
-	v4l2_dbg(3, debug, client, "i2c_write: 0x%04X : 0x%04x\n", addr, val);
-
-	return ret == 4 ? 0 : ret;
-}
-
-/* The command register write, assumes Command_Wr_addH = 0x7000. */
-static int s5k6aa_write(struct i2c_client *c, u16 addr, u16 val)
-{
-	int ret = s5k6aa_i2c_write(c, REG_CMDWR_ADDRL, addr);
-	if (ret)
-		return ret;
-	return s5k6aa_i2c_write(c, REG_CMDBUF0_ADDR, val);
-}
-
-/* The command register read, assumes Command_Rd_addH = 0x7000. */
-static int s5k6aa_read(struct i2c_client *client, u16 addr, u16 *val)
-{
-	int ret = s5k6aa_i2c_write(client, REG_CMDRD_ADDRL, addr);
-	if (ret)
-		return ret;
-	return s5k6aa_i2c_read(client, REG_CMDBUF0_ADDR, val);
-}
-
-static int s5k6aa_write_array(struct v4l2_subdev *sd,
-			      const struct s5k6aa_regval *msg)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u16 addr_incr = 0;
-	int ret = 0;
-
-	while (msg->addr != S5K6AA_TERM) {
-		if (addr_incr != 2)
-			ret = s5k6aa_i2c_write(client, REG_CMDWR_ADDRL,
-					       msg->addr);
-		if (ret)
-			break;
-		ret = s5k6aa_i2c_write(client, REG_CMDBUF0_ADDR, msg->val);
-		if (ret)
-			break;
-		/* Assume that msg->addr is always less than 0xfffc */
-		addr_incr = (msg + 1)->addr - msg->addr;
-		msg++;
-	}
-
-	return ret;
-}
-
-/* Configure the AHB high address bytes for GTG registers access */
-static int s5k6aa_set_ahb_address(struct i2c_client *client)
-{
-	int ret = s5k6aa_i2c_write(client, AHB_MSB_ADDR_PTR, GEN_REG_OFFSH);
-	if (ret)
-		return ret;
-	ret = s5k6aa_i2c_write(client, REG_CMDRD_ADDRH, HOST_SWIF_OFFSH);
-	if (ret)
-		return ret;
-	return s5k6aa_i2c_write(client, REG_CMDWR_ADDRH, HOST_SWIF_OFFSH);
-}
-
-/**
- * s5k6aa_configure_pixel_clocks - apply ISP main clock/PLL configuration
- * @s5k6aa: pointer to &struct s5k6aa describing the device
- *
- * Configure the internal ISP PLL for the required output frequency.
- * Locking: called with s5k6aa.lock mutex held.
- */
-static int s5k6aa_configure_pixel_clocks(struct s5k6aa *s5k6aa)
-{
-	struct i2c_client *c = v4l2_get_subdevdata(&s5k6aa->sd);
-	unsigned long fmclk = s5k6aa->mclk_frequency / 1000;
-	u16 status;
-	int ret;
-
-	if (WARN(fmclk < MIN_MCLK_FREQ_KHZ || fmclk > MAX_MCLK_FREQ_KHZ,
-		 "Invalid clock frequency: %ld\n", fmclk))
-		return -EINVAL;
-
-	s5k6aa->pclk_fmin = PCLK_FREQ_MIN;
-	s5k6aa->pclk_fmax = PCLK_FREQ_MAX;
-	s5k6aa->clk_fop = SYS_PLL_OUT_FREQ;
-
-	/* External input clock frequency in kHz */
-	ret = s5k6aa_write(c, REG_I_INCLK_FREQ_H, fmclk >> 16);
-	if (!ret)
-		ret = s5k6aa_write(c, REG_I_INCLK_FREQ_L, fmclk & 0xFFFF);
-	if (!ret)
-		ret = s5k6aa_write(c, REG_I_USE_NPVI_CLOCKS, 1);
-	/* Internal PLL frequency */
-	if (!ret)
-		ret = s5k6aa_write(c, REG_I_OPCLK_4KHZ(0), s5k6aa->clk_fop);
-	if (!ret)
-		ret = s5k6aa_write(c, REG_I_MIN_OUTRATE_4KHZ(0),
-				   s5k6aa->pclk_fmin);
-	if (!ret)
-		ret = s5k6aa_write(c, REG_I_MAX_OUTRATE_4KHZ(0),
-				   s5k6aa->pclk_fmax);
-	if (!ret)
-		ret = s5k6aa_write(c, REG_I_INIT_PARAMS_UPDATED, 1);
-	if (!ret)
-		ret = s5k6aa_read(c, REG_I_ERROR_INFO, &status);
-
-	return ret ? ret : (status ? -EINVAL : 0);
-}
-
-/* Set horizontal and vertical image flipping */
-static int s5k6aa_set_mirror(struct s5k6aa *s5k6aa, int horiz_flip)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&s5k6aa->sd);
-	int index = s5k6aa->preset->index;
-
-	unsigned int vflip = s5k6aa->ctrls.vflip->val ^ s5k6aa->inv_vflip;
-	unsigned int flip = (horiz_flip ^ s5k6aa->inv_hflip) | (vflip << 1);
-
-	return s5k6aa_write(client, REG_P_PREV_MIRROR(index), flip);
-}
-
-/* Configure auto/manual white balance and R/G/B gains */
-static int s5k6aa_set_awb(struct s5k6aa *s5k6aa, int awb)
-{
-	struct i2c_client *c = v4l2_get_subdevdata(&s5k6aa->sd);
-	struct s5k6aa_ctrls *ctrls = &s5k6aa->ctrls;
-	u16 reg;
-
-	int ret = s5k6aa_read(c, REG_DBG_AUTOALG_EN, &reg);
-
-	if (!ret && !awb) {
-		ret = s5k6aa_write(c, REG_SF_RGAIN, ctrls->gain_red->val);
-		if (!ret)
-			ret = s5k6aa_write(c, REG_SF_RGAIN_CHG, 1);
-		if (ret)
-			return ret;
-
-		ret = s5k6aa_write(c, REG_SF_GGAIN, ctrls->gain_green->val);
-		if (!ret)
-			ret = s5k6aa_write(c, REG_SF_GGAIN_CHG, 1);
-		if (ret)
-			return ret;
-
-		ret = s5k6aa_write(c, REG_SF_BGAIN, ctrls->gain_blue->val);
-		if (!ret)
-			ret = s5k6aa_write(c, REG_SF_BGAIN_CHG, 1);
-	}
-	if (!ret) {
-		reg = awb ? reg | AALG_WB_EN_MASK : reg & ~AALG_WB_EN_MASK;
-		ret = s5k6aa_write(c, REG_DBG_AUTOALG_EN, reg);
-	}
-
-	return ret;
-}
-
-/* Program FW with exposure time, 'exposure' in us units */
-static int s5k6aa_set_user_exposure(struct i2c_client *client, int exposure)
-{
-	unsigned int time = exposure / 10;
-
-	int ret = s5k6aa_write(client, REG_SF_USR_EXPOSURE_L, time & 0xffff);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_SF_USR_EXPOSURE_H, time >> 16);
-	if (ret)
-		return ret;
-	return s5k6aa_write(client, REG_SF_USR_EXPOSURE_CHG, 1);
-}
-
-static int s5k6aa_set_user_gain(struct i2c_client *client, int gain)
-{
-	int ret = s5k6aa_write(client, REG_SF_USR_TOT_GAIN, gain);
-	if (ret)
-		return ret;
-	return s5k6aa_write(client, REG_SF_USR_TOT_GAIN_CHG, 1);
-}
-
-/* Set auto/manual exposure and total gain */
-static int s5k6aa_set_auto_exposure(struct s5k6aa *s5k6aa, int value)
-{
-	struct i2c_client *c = v4l2_get_subdevdata(&s5k6aa->sd);
-	unsigned int exp_time = s5k6aa->ctrls.exposure->val;
-	u16 auto_alg;
-
-	int ret = s5k6aa_read(c, REG_DBG_AUTOALG_EN, &auto_alg);
-	if (ret)
-		return ret;
-
-	v4l2_dbg(1, debug, c, "man_exp: %d, auto_exp: %d, a_alg: 0x%x\n",
-		 exp_time, value, auto_alg);
-
-	if (value == V4L2_EXPOSURE_AUTO) {
-		auto_alg |= AALG_AE_EN_MASK | AALG_DIVLEI_EN_MASK;
-	} else {
-		ret = s5k6aa_set_user_exposure(c, exp_time);
-		if (ret)
-			return ret;
-		ret = s5k6aa_set_user_gain(c, s5k6aa->ctrls.gain->val);
-		if (ret)
-			return ret;
-		auto_alg &= ~(AALG_AE_EN_MASK | AALG_DIVLEI_EN_MASK);
-	}
-
-	return s5k6aa_write(c, REG_DBG_AUTOALG_EN, auto_alg);
-}
-
-static int s5k6aa_set_anti_flicker(struct s5k6aa *s5k6aa, int value)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&s5k6aa->sd);
-	u16 auto_alg;
-	int ret;
-
-	ret = s5k6aa_read(client, REG_DBG_AUTOALG_EN, &auto_alg);
-	if (ret)
-		return ret;
-
-	if (value == V4L2_CID_POWER_LINE_FREQUENCY_AUTO) {
-		auto_alg |= AALG_FLICKER_EN_MASK;
-	} else {
-		auto_alg &= ~AALG_FLICKER_EN_MASK;
-		/* The V4L2_CID_LINE_FREQUENCY control values match
-		 * the register values */
-		ret = s5k6aa_write(client, REG_SF_FLICKER_QUANT, value);
-		if (ret)
-			return ret;
-		ret = s5k6aa_write(client, REG_SF_FLICKER_QUANT_CHG, 1);
-		if (ret)
-			return ret;
-	}
-
-	return s5k6aa_write(client, REG_DBG_AUTOALG_EN, auto_alg);
-}
-
-static int s5k6aa_set_colorfx(struct s5k6aa *s5k6aa, int val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&s5k6aa->sd);
-	static const struct v4l2_control colorfx[] = {
-		{ V4L2_COLORFX_NONE,	 0 },
-		{ V4L2_COLORFX_BW,	 1 },
-		{ V4L2_COLORFX_NEGATIVE, 2 },
-		{ V4L2_COLORFX_SEPIA,	 3 },
-		{ V4L2_COLORFX_SKY_BLUE, 4 },
-		{ V4L2_COLORFX_SKETCH,	 5 },
-	};
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(colorfx); i++) {
-		if (colorfx[i].id == val)
-			return s5k6aa_write(client, REG_G_SPEC_EFFECTS,
-					    colorfx[i].value);
-	}
-	return -EINVAL;
-}
-
-static int s5k6aa_preview_config_status(struct i2c_client *client)
-{
-	u16 error = 0;
-	int ret = s5k6aa_read(client, REG_G_PREV_CFG_ERROR, &error);
-
-	v4l2_dbg(1, debug, client, "error: 0x%x (%d)\n", error, ret);
-	return ret ? ret : (error ? -EINVAL : 0);
-}
-
-static int s5k6aa_get_pixfmt_index(struct s5k6aa *s5k6aa,
-				   struct v4l2_mbus_framefmt *mf)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(s5k6aa_formats); i++)
-		if (mf->colorspace == s5k6aa_formats[i].colorspace &&
-		    mf->code == s5k6aa_formats[i].code)
-			return i;
-	return 0;
-}
-
-static int s5k6aa_set_output_framefmt(struct s5k6aa *s5k6aa,
-				      struct s5k6aa_preset *preset)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&s5k6aa->sd);
-	int fmt_index = s5k6aa_get_pixfmt_index(s5k6aa, &preset->mbus_fmt);
-	int ret;
-
-	ret = s5k6aa_write(client, REG_P_OUT_WIDTH(preset->index),
-			   preset->mbus_fmt.width);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_P_OUT_HEIGHT(preset->index),
-				   preset->mbus_fmt.height);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_P_FMT(preset->index),
-				   s5k6aa_formats[fmt_index].reg_p_fmt);
-	return ret;
-}
-
-static int s5k6aa_set_input_params(struct s5k6aa *s5k6aa)
-{
-	struct i2c_client *c = v4l2_get_subdevdata(&s5k6aa->sd);
-	struct v4l2_rect *r = &s5k6aa->ccd_rect;
-	int ret;
-
-	ret = s5k6aa_write(c, REG_G_PREVZOOM_IN_WIDTH, r->width);
-	if (!ret)
-		ret = s5k6aa_write(c, REG_G_PREVZOOM_IN_HEIGHT, r->height);
-	if (!ret)
-		ret = s5k6aa_write(c, REG_G_PREVZOOM_IN_XOFFS, r->left);
-	if (!ret)
-		ret = s5k6aa_write(c, REG_G_PREVZOOM_IN_YOFFS, r->top);
-	if (!ret)
-		ret = s5k6aa_write(c, REG_G_INPUTS_CHANGE_REQ, 1);
-	if (!ret)
-		s5k6aa->apply_crop = 0;
-
-	return ret;
-}
-
-/**
- * s5k6aa_configure_video_bus - configure the video output interface
- * @s5k6aa: pointer to &struct s5k6aa describing the device
- * @bus_type: video bus type: parallel or MIPI-CSI
- * @nlanes: number of MIPI lanes to be used (MIPI-CSI only)
- *
- * Note: Only parallel bus operation has been tested.
- */
-static int s5k6aa_configure_video_bus(struct s5k6aa *s5k6aa,
-				      enum v4l2_mbus_type bus_type, int nlanes)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&s5k6aa->sd);
-	u16 cfg = 0;
-	int ret;
-
-	/*
-	 * TODO: The sensor is supposed to support BT.601 and BT.656
-	 * but there is nothing indicating how to switch between both
-	 * in the datasheet. For now default BT.601 interface is assumed.
-	 */
-	if (bus_type == V4L2_MBUS_CSI2_DPHY)
-		cfg = nlanes;
-	else if (bus_type != V4L2_MBUS_PARALLEL)
-		return -EINVAL;
-
-	ret = s5k6aa_write(client, REG_OIF_EN_MIPI_LANES, cfg);
-	if (ret)
-		return ret;
-	return s5k6aa_write(client, REG_OIF_CFG_CHG, 1);
-}
-
-/* This function should be called when switching to new user configuration set*/
-static int s5k6aa_new_config_sync(struct i2c_client *client, int timeout,
-				  int cid)
-{
-	unsigned long end = jiffies + msecs_to_jiffies(timeout);
-	u16 reg = 1;
-	int ret;
-
-	ret = s5k6aa_write(client, REG_G_ACTIVE_PREV_CFG, cid);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_G_PREV_CFG_CHG, 1);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_G_NEW_CFG_SYNC, 1);
-	if (timeout == 0)
-		return ret;
-
-	while (ret >= 0 && time_is_after_jiffies(end)) {
-		ret = s5k6aa_read(client, REG_G_NEW_CFG_SYNC, &reg);
-		if (!reg)
-			return 0;
-		usleep_range(1000, 5000);
-	}
-	return ret ? ret : -ETIMEDOUT;
-}
-
-/**
- * s5k6aa_set_prev_config - write user preview register set
- * @s5k6aa: pointer to &struct s5k6aa describing the device
- * @preset: s5kaa preset to be applied
- *
- * Configure output resolution and color format, pixel clock
- * frequency range, device frame rate type and frame period range.
- */
-static int s5k6aa_set_prev_config(struct s5k6aa *s5k6aa,
-				  struct s5k6aa_preset *preset)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&s5k6aa->sd);
-	int idx = preset->index;
-	u16 frame_rate_q;
-	int ret;
-
-	if (s5k6aa->fiv->reg_fr_time >= S5K6AA_MAX_HIGHRES_FR_TIME)
-		frame_rate_q = FR_RATE_Q_BEST_FRRATE;
-	else
-		frame_rate_q = FR_RATE_Q_BEST_QUALITY;
-
-	ret = s5k6aa_set_output_framefmt(s5k6aa, preset);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_P_MAX_OUT_RATE(idx),
-				   s5k6aa->pclk_fmax);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_P_MIN_OUT_RATE(idx),
-				   s5k6aa->pclk_fmin);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_P_CLK_INDEX(idx),
-				   preset->clk_id);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_P_FR_RATE_TYPE(idx),
-				   FR_RATE_DYNAMIC);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_P_FR_RATE_Q_TYPE(idx),
-				   frame_rate_q);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_P_MAX_FR_TIME(idx),
-				   s5k6aa->fiv->reg_fr_time + 33);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_P_MIN_FR_TIME(idx),
-				   s5k6aa->fiv->reg_fr_time - 33);
-	if (!ret)
-		ret = s5k6aa_new_config_sync(client, 250, idx);
-	if (!ret)
-		ret = s5k6aa_preview_config_status(client);
-	if (!ret)
-		s5k6aa->apply_cfg = 0;
-
-	v4l2_dbg(1, debug, client, "Frame interval: %d +/- 3.3ms. (%d)\n",
-		 s5k6aa->fiv->reg_fr_time, ret);
-	return ret;
-}
-
-/**
- * s5k6aa_initialize_isp - basic ISP MCU initialization
- * @sd: pointer to V4L2 sub-device descriptor
- *
- * Configure AHB addresses for registers read/write; configure PLLs for
- * required output pixel clock. The ISP power supply needs to be already
- * enabled, with an optional H/W reset.
- * Locking: called with s5k6aa.lock mutex held.
- */
-static int s5k6aa_initialize_isp(struct v4l2_subdev *sd)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	int ret;
-
-	s5k6aa->apply_crop = 1;
-	s5k6aa->apply_cfg = 1;
-	msleep(100);
-
-	ret = s5k6aa_set_ahb_address(client);
-	if (ret)
-		return ret;
-	ret = s5k6aa_configure_video_bus(s5k6aa, s5k6aa->bus_type,
-					 s5k6aa->mipi_lanes);
-	if (ret)
-		return ret;
-	ret = s5k6aa_write_array(sd, s5k6aa_analog_config);
-	if (ret)
-		return ret;
-	msleep(20);
-
-	return s5k6aa_configure_pixel_clocks(s5k6aa);
-}
-
-static int s5k6aa_gpio_set_value(struct s5k6aa *priv, int id, u32 val)
-{
-	if (!gpio_is_valid(priv->gpio[id].gpio))
-		return 0;
-	gpio_set_value(priv->gpio[id].gpio, !!val);
-	return 1;
-}
-
-static int s5k6aa_gpio_assert(struct s5k6aa *priv, int id)
-{
-	return s5k6aa_gpio_set_value(priv, id, priv->gpio[id].level);
-}
-
-static int s5k6aa_gpio_deassert(struct s5k6aa *priv, int id)
-{
-	return s5k6aa_gpio_set_value(priv, id, !priv->gpio[id].level);
-}
-
-static int __s5k6aa_power_on(struct s5k6aa *s5k6aa)
-{
-	int ret;
-
-	ret = regulator_bulk_enable(S5K6AA_NUM_SUPPLIES, s5k6aa->supplies);
-	if (ret)
-		return ret;
-	if (s5k6aa_gpio_deassert(s5k6aa, STBY))
-		usleep_range(150, 200);
-
-	if (s5k6aa->s_power)
-		ret = s5k6aa->s_power(1);
-	usleep_range(4000, 5000);
-
-	if (s5k6aa_gpio_deassert(s5k6aa, RSET))
-		msleep(20);
-
-	return ret;
-}
-
-static int __s5k6aa_power_off(struct s5k6aa *s5k6aa)
-{
-	int ret;
-
-	if (s5k6aa_gpio_assert(s5k6aa, RSET))
-		usleep_range(100, 150);
-
-	if (s5k6aa->s_power) {
-		ret = s5k6aa->s_power(0);
-		if (ret)
-			return ret;
-	}
-	if (s5k6aa_gpio_assert(s5k6aa, STBY))
-		usleep_range(50, 100);
-	s5k6aa->streaming = 0;
-
-	return regulator_bulk_disable(S5K6AA_NUM_SUPPLIES, s5k6aa->supplies);
-}
-
-/*
- * V4L2 subdev core and video operations
- */
-static int s5k6aa_set_power(struct v4l2_subdev *sd, int on)
-{
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	int ret = 0;
-
-	mutex_lock(&s5k6aa->lock);
-
-	if (s5k6aa->power == !on) {
-		if (on) {
-			ret = __s5k6aa_power_on(s5k6aa);
-			if (!ret)
-				ret = s5k6aa_initialize_isp(sd);
-		} else {
-			ret = __s5k6aa_power_off(s5k6aa);
-		}
-
-		if (!ret)
-			s5k6aa->power += on ? 1 : -1;
-	}
-
-	mutex_unlock(&s5k6aa->lock);
-
-	if (!on || ret || s5k6aa->power != 1)
-		return ret;
-
-	return v4l2_ctrl_handler_setup(sd->ctrl_handler);
-}
-
-static int __s5k6aa_stream(struct s5k6aa *s5k6aa, int enable)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&s5k6aa->sd);
-	int ret = 0;
-
-	ret = s5k6aa_write(client, REG_G_ENABLE_PREV, enable);
-	if (!ret)
-		ret = s5k6aa_write(client, REG_G_ENABLE_PREV_CHG, 1);
-	if (!ret)
-		s5k6aa->streaming = enable;
-
-	return ret;
-}
-
-static int s5k6aa_s_stream(struct v4l2_subdev *sd, int on)
-{
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	int ret = 0;
-
-	mutex_lock(&s5k6aa->lock);
-
-	if (s5k6aa->streaming == !on) {
-		if (!ret && s5k6aa->apply_cfg)
-			ret = s5k6aa_set_prev_config(s5k6aa, s5k6aa->preset);
-		if (s5k6aa->apply_crop)
-			ret = s5k6aa_set_input_params(s5k6aa);
-		if (!ret)
-			ret = __s5k6aa_stream(s5k6aa, !!on);
-	}
-	mutex_unlock(&s5k6aa->lock);
-
-	return ret;
-}
-
-static int s5k6aa_g_frame_interval(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_frame_interval *fi)
-{
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-
-	mutex_lock(&s5k6aa->lock);
-	fi->interval = s5k6aa->fiv->interval;
-	mutex_unlock(&s5k6aa->lock);
-
-	return 0;
-}
-
-static int __s5k6aa_set_frame_interval(struct s5k6aa *s5k6aa,
-				       struct v4l2_subdev_frame_interval *fi)
-{
-	struct v4l2_mbus_framefmt *mbus_fmt = &s5k6aa->preset->mbus_fmt;
-	const struct s5k6aa_interval *fiv = &s5k6aa_intervals[0];
-	unsigned int err, min_err = UINT_MAX;
-	unsigned int i, fr_time;
-
-	if (fi->interval.denominator == 0)
-		return -EINVAL;
-
-	fr_time = fi->interval.numerator * 10000 / fi->interval.denominator;
-
-	for (i = 0; i < ARRAY_SIZE(s5k6aa_intervals); i++) {
-		const struct s5k6aa_interval *iv = &s5k6aa_intervals[i];
-
-		if (mbus_fmt->width > iv->size.width ||
-		    mbus_fmt->height > iv->size.height)
-			continue;
-
-		err = abs(iv->reg_fr_time - fr_time);
-		if (err < min_err) {
-			fiv = iv;
-			min_err = err;
-		}
-	}
-	s5k6aa->fiv = fiv;
-
-	v4l2_dbg(1, debug, &s5k6aa->sd, "Changed frame interval to %d us\n",
-		 fiv->reg_fr_time * 100);
-	return 0;
-}
-
-static int s5k6aa_s_frame_interval(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_frame_interval *fi)
-{
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	int ret;
-
-	v4l2_dbg(1, debug, sd, "Setting %d/%d frame interval\n",
-		 fi->interval.numerator, fi->interval.denominator);
-
-	mutex_lock(&s5k6aa->lock);
-	ret = __s5k6aa_set_frame_interval(s5k6aa, fi);
-	s5k6aa->apply_cfg = 1;
-
-	mutex_unlock(&s5k6aa->lock);
-	return ret;
-}
-
-/*
- * V4L2 subdev pad level and video operations
- */
-static int s5k6aa_enum_frame_interval(struct v4l2_subdev *sd,
-			      struct v4l2_subdev_state *sd_state,
-			      struct v4l2_subdev_frame_interval_enum *fie)
-{
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	const struct s5k6aa_interval *fi;
-	int ret = 0;
-
-	if (fie->index >= ARRAY_SIZE(s5k6aa_intervals))
-		return -EINVAL;
-
-	v4l_bound_align_image(&fie->width, S5K6AA_WIN_WIDTH_MIN,
-			      S5K6AA_WIN_WIDTH_MAX, 1,
-			      &fie->height, S5K6AA_WIN_HEIGHT_MIN,
-			      S5K6AA_WIN_HEIGHT_MAX, 1, 0);
-
-	mutex_lock(&s5k6aa->lock);
-	fi = &s5k6aa_intervals[fie->index];
-	if (fie->width > fi->size.width || fie->height > fi->size.height)
-		ret = -EINVAL;
-	else
-		fie->interval = fi->interval;
-	mutex_unlock(&s5k6aa->lock);
-
-	return ret;
-}
-
-static int s5k6aa_enum_mbus_code(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_state *sd_state,
-				 struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index >= ARRAY_SIZE(s5k6aa_formats))
-		return -EINVAL;
-
-	code->code = s5k6aa_formats[code->index].code;
-	return 0;
-}
-
-static int s5k6aa_enum_frame_size(struct v4l2_subdev *sd,
-				  struct v4l2_subdev_state *sd_state,
-				  struct v4l2_subdev_frame_size_enum *fse)
-{
-	int i = ARRAY_SIZE(s5k6aa_formats);
-
-	if (fse->index > 0)
-		return -EINVAL;
-
-	while (--i)
-		if (fse->code == s5k6aa_formats[i].code)
-			break;
-
-	fse->code = s5k6aa_formats[i].code;
-	fse->min_width  = S5K6AA_WIN_WIDTH_MIN;
-	fse->max_width  = S5K6AA_WIN_WIDTH_MAX;
-	fse->max_height = S5K6AA_WIN_HEIGHT_MIN;
-	fse->min_height = S5K6AA_WIN_HEIGHT_MAX;
-
-	return 0;
-}
-
-static struct v4l2_rect *
-__s5k6aa_get_crop_rect(struct s5k6aa *s5k6aa,
-		       struct v4l2_subdev_state *sd_state,
-		       enum v4l2_subdev_format_whence which)
-{
-	if (which == V4L2_SUBDEV_FORMAT_ACTIVE)
-		return &s5k6aa->ccd_rect;
-
-	WARN_ON(which != V4L2_SUBDEV_FORMAT_TRY);
-	return v4l2_subdev_get_try_crop(&s5k6aa->sd, sd_state, 0);
-}
-
-static void s5k6aa_try_format(struct s5k6aa *s5k6aa,
-			      struct v4l2_mbus_framefmt *mf)
-{
-	unsigned int index;
-
-	v4l_bound_align_image(&mf->width, S5K6AA_WIN_WIDTH_MIN,
-			      S5K6AA_WIN_WIDTH_MAX, 1,
-			      &mf->height, S5K6AA_WIN_HEIGHT_MIN,
-			      S5K6AA_WIN_HEIGHT_MAX, 1, 0);
-
-	if (mf->colorspace != V4L2_COLORSPACE_JPEG &&
-	    mf->colorspace != V4L2_COLORSPACE_REC709)
-		mf->colorspace = V4L2_COLORSPACE_JPEG;
-
-	index = s5k6aa_get_pixfmt_index(s5k6aa, mf);
-
-	mf->colorspace	= s5k6aa_formats[index].colorspace;
-	mf->code	= s5k6aa_formats[index].code;
-	mf->field	= V4L2_FIELD_NONE;
-}
-
-static int s5k6aa_get_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_state *sd_state,
-			  struct v4l2_subdev_format *fmt)
-{
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	struct v4l2_mbus_framefmt *mf;
-
-	memset(fmt->reserved, 0, sizeof(fmt->reserved));
-
-	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
-		mf = v4l2_subdev_get_try_format(sd, sd_state, 0);
-		fmt->format = *mf;
-		return 0;
-	}
-
-	mutex_lock(&s5k6aa->lock);
-	fmt->format = s5k6aa->preset->mbus_fmt;
-	mutex_unlock(&s5k6aa->lock);
-
-	return 0;
-}
-
-static int s5k6aa_set_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_state *sd_state,
-			  struct v4l2_subdev_format *fmt)
-{
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	struct s5k6aa_preset *preset = s5k6aa->preset;
-	struct v4l2_mbus_framefmt *mf;
-	struct v4l2_rect *crop;
-	int ret = 0;
-
-	mutex_lock(&s5k6aa->lock);
-	s5k6aa_try_format(s5k6aa, &fmt->format);
-
-	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
-		mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad);
-		crop = v4l2_subdev_get_try_crop(sd, sd_state, 0);
-	} else {
-		if (s5k6aa->streaming) {
-			ret = -EBUSY;
-		} else {
-			mf = &preset->mbus_fmt;
-			crop = &s5k6aa->ccd_rect;
-			s5k6aa->apply_cfg = 1;
-		}
-	}
-
-	if (ret == 0) {
-		struct v4l2_subdev_frame_interval fiv = {
-			.interval = {0, 1}
-		};
-
-		*mf = fmt->format;
-		/*
-		 * Make sure the crop window is valid, i.e. its size is
-		 * greater than the output window, as the ISP supports
-		 * only down-scaling.
-		 */
-		crop->width = clamp_t(unsigned int, crop->width, mf->width,
-				      S5K6AA_WIN_WIDTH_MAX);
-		crop->height = clamp_t(unsigned int, crop->height, mf->height,
-				       S5K6AA_WIN_HEIGHT_MAX);
-		crop->left = clamp_t(unsigned int, crop->left, 0,
-				     S5K6AA_WIN_WIDTH_MAX - crop->width);
-		crop->top  = clamp_t(unsigned int, crop->top, 0,
-				     S5K6AA_WIN_HEIGHT_MAX - crop->height);
-
-		/* Reset to minimum possible frame interval */
-		ret = __s5k6aa_set_frame_interval(s5k6aa, &fiv);
-	}
-	mutex_unlock(&s5k6aa->lock);
-
-	return ret;
-}
-
-static int s5k6aa_get_selection(struct v4l2_subdev *sd,
-				struct v4l2_subdev_state *sd_state,
-				struct v4l2_subdev_selection *sel)
-{
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	struct v4l2_rect *rect;
-
-	if (sel->target != V4L2_SEL_TGT_CROP)
-		return -EINVAL;
-
-	memset(sel->reserved, 0, sizeof(sel->reserved));
-
-	mutex_lock(&s5k6aa->lock);
-	rect = __s5k6aa_get_crop_rect(s5k6aa, sd_state, sel->which);
-	sel->r = *rect;
-	mutex_unlock(&s5k6aa->lock);
-
-	v4l2_dbg(1, debug, sd, "Current crop rectangle: (%d,%d)/%dx%d\n",
-		 rect->left, rect->top, rect->width, rect->height);
-
-	return 0;
-}
-
-static int s5k6aa_set_selection(struct v4l2_subdev *sd,
-				struct v4l2_subdev_state *sd_state,
-				struct v4l2_subdev_selection *sel)
-{
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	struct v4l2_mbus_framefmt *mf;
-	unsigned int max_x, max_y;
-	struct v4l2_rect *crop_r;
-
-	if (sel->target != V4L2_SEL_TGT_CROP)
-		return -EINVAL;
-
-	mutex_lock(&s5k6aa->lock);
-	crop_r = __s5k6aa_get_crop_rect(s5k6aa, sd_state, sel->which);
-
-	if (sel->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
-		mf = &s5k6aa->preset->mbus_fmt;
-		s5k6aa->apply_crop = 1;
-	} else {
-		mf = v4l2_subdev_get_try_format(sd, sd_state, 0);
-	}
-	v4l_bound_align_image(&sel->r.width, mf->width,
-			      S5K6AA_WIN_WIDTH_MAX, 1,
-			      &sel->r.height, mf->height,
-			      S5K6AA_WIN_HEIGHT_MAX, 1, 0);
-
-	max_x = (S5K6AA_WIN_WIDTH_MAX - sel->r.width) & ~1;
-	max_y = (S5K6AA_WIN_HEIGHT_MAX - sel->r.height) & ~1;
-
-	sel->r.left = clamp_t(unsigned int, sel->r.left, 0, max_x);
-	sel->r.top  = clamp_t(unsigned int, sel->r.top, 0, max_y);
-
-	*crop_r = sel->r;
-
-	mutex_unlock(&s5k6aa->lock);
-
-	v4l2_dbg(1, debug, sd, "Set crop rectangle: (%d,%d)/%dx%d\n",
-		 crop_r->left, crop_r->top, crop_r->width, crop_r->height);
-
-	return 0;
-}
-
-static const struct v4l2_subdev_pad_ops s5k6aa_pad_ops = {
-	.enum_mbus_code		= s5k6aa_enum_mbus_code,
-	.enum_frame_size	= s5k6aa_enum_frame_size,
-	.enum_frame_interval	= s5k6aa_enum_frame_interval,
-	.get_fmt		= s5k6aa_get_fmt,
-	.set_fmt		= s5k6aa_set_fmt,
-	.get_selection		= s5k6aa_get_selection,
-	.set_selection		= s5k6aa_set_selection,
-};
-
-static const struct v4l2_subdev_video_ops s5k6aa_video_ops = {
-	.g_frame_interval	= s5k6aa_g_frame_interval,
-	.s_frame_interval	= s5k6aa_s_frame_interval,
-	.s_stream		= s5k6aa_s_stream,
-};
-
-/*
- * V4L2 subdev controls
- */
-
-static int s5k6aa_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct v4l2_subdev *sd = ctrl_to_sd(ctrl);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	int idx, err = 0;
-
-	v4l2_dbg(1, debug, sd, "ctrl: 0x%x, value: %d\n", ctrl->id, ctrl->val);
-
-	mutex_lock(&s5k6aa->lock);
-	/*
-	 * If the device is not powered up by the host driver do
-	 * not apply any controls to H/W at this time. Instead
-	 * the controls will be restored right after power-up.
-	 */
-	if (s5k6aa->power == 0)
-		goto unlock;
-	idx = s5k6aa->preset->index;
-
-	switch (ctrl->id) {
-	case V4L2_CID_AUTO_WHITE_BALANCE:
-		err = s5k6aa_set_awb(s5k6aa, ctrl->val);
-		break;
-
-	case V4L2_CID_BRIGHTNESS:
-		err = s5k6aa_write(client, REG_USER_BRIGHTNESS, ctrl->val);
-		break;
-
-	case V4L2_CID_COLORFX:
-		err = s5k6aa_set_colorfx(s5k6aa, ctrl->val);
-		break;
-
-	case V4L2_CID_CONTRAST:
-		err = s5k6aa_write(client, REG_USER_CONTRAST, ctrl->val);
-		break;
-
-	case V4L2_CID_EXPOSURE_AUTO:
-		err = s5k6aa_set_auto_exposure(s5k6aa, ctrl->val);
-		break;
-
-	case V4L2_CID_HFLIP:
-		err = s5k6aa_set_mirror(s5k6aa, ctrl->val);
-		if (err)
-			break;
-		err = s5k6aa_write(client, REG_G_PREV_CFG_CHG, 1);
-		break;
-
-	case V4L2_CID_POWER_LINE_FREQUENCY:
-		err = s5k6aa_set_anti_flicker(s5k6aa, ctrl->val);
-		break;
-
-	case V4L2_CID_SATURATION:
-		err = s5k6aa_write(client, REG_USER_SATURATION, ctrl->val);
-		break;
-
-	case V4L2_CID_SHARPNESS:
-		err = s5k6aa_write(client, REG_USER_SHARPBLUR, ctrl->val);
-		break;
-
-	case V4L2_CID_WHITE_BALANCE_TEMPERATURE:
-		err = s5k6aa_write(client, REG_P_COLORTEMP(idx), ctrl->val);
-		if (err)
-			break;
-		err = s5k6aa_write(client, REG_G_PREV_CFG_CHG, 1);
-		break;
-	}
-unlock:
-	mutex_unlock(&s5k6aa->lock);
-	return err;
-}
-
-static const struct v4l2_ctrl_ops s5k6aa_ctrl_ops = {
-	.s_ctrl	= s5k6aa_s_ctrl,
-};
-
-static int s5k6aa_log_status(struct v4l2_subdev *sd)
-{
-	v4l2_ctrl_handler_log_status(sd->ctrl_handler, sd->name);
-	return 0;
-}
-
-#define V4L2_CID_RED_GAIN	(V4L2_CTRL_CLASS_CAMERA | 0x1001)
-#define V4L2_CID_GREEN_GAIN	(V4L2_CTRL_CLASS_CAMERA | 0x1002)
-#define V4L2_CID_BLUE_GAIN	(V4L2_CTRL_CLASS_CAMERA | 0x1003)
-
-static const struct v4l2_ctrl_config s5k6aa_ctrls[] = {
-	{
-		.ops	= &s5k6aa_ctrl_ops,
-		.id	= V4L2_CID_RED_GAIN,
-		.type	= V4L2_CTRL_TYPE_INTEGER,
-		.name	= "Gain, Red",
-		.min	= 0,
-		.max	= 256,
-		.def	= 127,
-		.step	= 1,
-	}, {
-		.ops	= &s5k6aa_ctrl_ops,
-		.id	= V4L2_CID_GREEN_GAIN,
-		.type	= V4L2_CTRL_TYPE_INTEGER,
-		.name	= "Gain, Green",
-		.min	= 0,
-		.max	= 256,
-		.def	= 127,
-		.step	= 1,
-	}, {
-		.ops	= &s5k6aa_ctrl_ops,
-		.id	= V4L2_CID_BLUE_GAIN,
-		.type	= V4L2_CTRL_TYPE_INTEGER,
-		.name	= "Gain, Blue",
-		.min	= 0,
-		.max	= 256,
-		.def	= 127,
-		.step	= 1,
-	},
-};
-
-static int s5k6aa_initialize_ctrls(struct s5k6aa *s5k6aa)
-{
-	const struct v4l2_ctrl_ops *ops = &s5k6aa_ctrl_ops;
-	struct s5k6aa_ctrls *ctrls = &s5k6aa->ctrls;
-	struct v4l2_ctrl_handler *hdl = &ctrls->handler;
-
-	int ret = v4l2_ctrl_handler_init(hdl, 16);
-	if (ret)
-		return ret;
-	/* Auto white balance cluster */
-	ctrls->awb = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_AUTO_WHITE_BALANCE,
-				       0, 1, 1, 1);
-	ctrls->gain_red = v4l2_ctrl_new_custom(hdl, &s5k6aa_ctrls[0], NULL);
-	ctrls->gain_green = v4l2_ctrl_new_custom(hdl, &s5k6aa_ctrls[1], NULL);
-	ctrls->gain_blue = v4l2_ctrl_new_custom(hdl, &s5k6aa_ctrls[2], NULL);
-	v4l2_ctrl_auto_cluster(4, &ctrls->awb, 0, false);
-
-	ctrls->hflip = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_HFLIP, 0, 1, 1, 0);
-	ctrls->vflip = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_VFLIP, 0, 1, 1, 0);
-	v4l2_ctrl_cluster(2, &ctrls->hflip);
-
-	ctrls->auto_exp = v4l2_ctrl_new_std_menu(hdl, ops,
-				V4L2_CID_EXPOSURE_AUTO,
-				V4L2_EXPOSURE_MANUAL, 0, V4L2_EXPOSURE_AUTO);
-	/* Exposure time: x 1 us */
-	ctrls->exposure = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_EXPOSURE,
-					    0, 6000000U, 1, 100000U);
-	/* Total gain: 256 <=> 1x */
-	ctrls->gain = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_GAIN,
-					0, 256, 1, 256);
-	v4l2_ctrl_auto_cluster(3, &ctrls->auto_exp, 0, false);
-
-	v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_POWER_LINE_FREQUENCY,
-			       V4L2_CID_POWER_LINE_FREQUENCY_AUTO, 0,
-			       V4L2_CID_POWER_LINE_FREQUENCY_AUTO);
-
-	v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_COLORFX,
-			       V4L2_COLORFX_SKY_BLUE, ~0x6f, V4L2_COLORFX_NONE);
-
-	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_WHITE_BALANCE_TEMPERATURE,
-			  0, 256, 1, 0);
-
-	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_SATURATION, -127, 127, 1, 0);
-	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_BRIGHTNESS, -127, 127, 1, 0);
-	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_CONTRAST, -127, 127, 1, 0);
-	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_SHARPNESS, -127, 127, 1, 0);
-
-	if (hdl->error) {
-		ret = hdl->error;
-		v4l2_ctrl_handler_free(hdl);
-		return ret;
-	}
-
-	s5k6aa->sd.ctrl_handler = hdl;
-	return 0;
-}
-
-/*
- * V4L2 subdev internal operations
- */
-static int s5k6aa_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
-{
-	struct v4l2_mbus_framefmt *format = v4l2_subdev_get_try_format(sd,
-								       fh->state,
-								       0);
-	struct v4l2_rect *crop = v4l2_subdev_get_try_crop(sd, fh->state, 0);
-
-	format->colorspace = s5k6aa_formats[0].colorspace;
-	format->code = s5k6aa_formats[0].code;
-	format->width = S5K6AA_OUT_WIDTH_DEF;
-	format->height = S5K6AA_OUT_HEIGHT_DEF;
-	format->field = V4L2_FIELD_NONE;
-
-	crop->width = S5K6AA_WIN_WIDTH_MAX;
-	crop->height = S5K6AA_WIN_HEIGHT_MAX;
-	crop->left = 0;
-	crop->top = 0;
-
-	return 0;
-}
-
-static int s5k6aa_check_fw_revision(struct s5k6aa *s5k6aa)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&s5k6aa->sd);
-	u16 api_ver = 0, fw_rev = 0;
-
-	int ret = s5k6aa_set_ahb_address(client);
-
-	if (!ret)
-		ret = s5k6aa_read(client, REG_FW_APIVER, &api_ver);
-	if (!ret)
-		ret = s5k6aa_read(client, REG_FW_REVISION, &fw_rev);
-	if (ret) {
-		v4l2_err(&s5k6aa->sd, "FW revision check failed!\n");
-		return ret;
-	}
-
-	v4l2_info(&s5k6aa->sd, "FW API ver.: 0x%X, FW rev.: 0x%X\n",
-		  api_ver, fw_rev);
-
-	return api_ver == S5K6AAFX_FW_APIVER ? 0 : -ENODEV;
-}
-
-static int s5k6aa_registered(struct v4l2_subdev *sd)
-{
-	struct s5k6aa *s5k6aa = to_s5k6aa(sd);
-	int ret;
-
-	mutex_lock(&s5k6aa->lock);
-	ret = __s5k6aa_power_on(s5k6aa);
-	if (!ret) {
-		msleep(100);
-		ret = s5k6aa_check_fw_revision(s5k6aa);
-		__s5k6aa_power_off(s5k6aa);
-	}
-	mutex_unlock(&s5k6aa->lock);
-
-	return ret;
-}
-
-static const struct v4l2_subdev_internal_ops s5k6aa_subdev_internal_ops = {
-	.registered = s5k6aa_registered,
-	.open = s5k6aa_open,
-};
-
-static const struct v4l2_subdev_core_ops s5k6aa_core_ops = {
-	.s_power = s5k6aa_set_power,
-	.log_status = s5k6aa_log_status,
-};
-
-static const struct v4l2_subdev_ops s5k6aa_subdev_ops = {
-	.core = &s5k6aa_core_ops,
-	.pad = &s5k6aa_pad_ops,
-	.video = &s5k6aa_video_ops,
-};
-
-/*
- * GPIO setup
- */
-
-static int s5k6aa_configure_gpios(struct s5k6aa *s5k6aa,
-				  const struct s5k6aa_platform_data *pdata)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&s5k6aa->sd);
-	const struct s5k6aa_gpio *gpio;
-	unsigned long flags;
-	int ret;
-
-	s5k6aa->gpio[STBY].gpio = -EINVAL;
-	s5k6aa->gpio[RSET].gpio  = -EINVAL;
-
-	gpio = &pdata->gpio_stby;
-	if (gpio_is_valid(gpio->gpio)) {
-		flags = (gpio->level ? GPIOF_OUT_INIT_HIGH : GPIOF_OUT_INIT_LOW)
-		      | GPIOF_EXPORT;
-		ret = devm_gpio_request_one(&client->dev, gpio->gpio, flags,
-					    "S5K6AA_STBY");
-		if (ret < 0)
-			return ret;
-
-		s5k6aa->gpio[STBY] = *gpio;
-	}
-
-	gpio = &pdata->gpio_reset;
-	if (gpio_is_valid(gpio->gpio)) {
-		flags = (gpio->level ? GPIOF_OUT_INIT_HIGH : GPIOF_OUT_INIT_LOW)
-		      | GPIOF_EXPORT;
-		ret = devm_gpio_request_one(&client->dev, gpio->gpio, flags,
-					    "S5K6AA_RST");
-		if (ret < 0)
-			return ret;
-
-		s5k6aa->gpio[RSET] = *gpio;
-	}
-
-	return 0;
-}
-
-static int s5k6aa_probe(struct i2c_client *client)
-{
-	const struct s5k6aa_platform_data *pdata = client->dev.platform_data;
-	struct v4l2_subdev *sd;
-	struct s5k6aa *s5k6aa;
-	int i, ret;
-
-	if (pdata == NULL) {
-		dev_err(&client->dev, "Platform data not specified\n");
-		return -EINVAL;
-	}
-
-	if (pdata->mclk_frequency == 0) {
-		dev_err(&client->dev, "MCLK frequency not specified\n");
-		return -EINVAL;
-	}
-
-	s5k6aa = devm_kzalloc(&client->dev, sizeof(*s5k6aa), GFP_KERNEL);
-	if (!s5k6aa)
-		return -ENOMEM;
-
-	mutex_init(&s5k6aa->lock);
-
-	s5k6aa->mclk_frequency = pdata->mclk_frequency;
-	s5k6aa->bus_type = pdata->bus_type;
-	s5k6aa->mipi_lanes = pdata->nlanes;
-	s5k6aa->s_power	= pdata->set_power;
-	s5k6aa->inv_hflip = pdata->horiz_flip;
-	s5k6aa->inv_vflip = pdata->vert_flip;
-
-	sd = &s5k6aa->sd;
-	v4l2_i2c_subdev_init(sd, client, &s5k6aa_subdev_ops);
-	/* Static name; NEVER use in new drivers! */
-	strscpy(sd->name, DRIVER_NAME, sizeof(sd->name));
-
-	sd->internal_ops = &s5k6aa_subdev_internal_ops;
-	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-
-	s5k6aa->pad.flags = MEDIA_PAD_FL_SOURCE;
-	sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
-	ret = media_entity_pads_init(&sd->entity, 1, &s5k6aa->pad);
-	if (ret)
-		return ret;
-
-	ret = s5k6aa_configure_gpios(s5k6aa, pdata);
-	if (ret)
-		goto out_err;
-
-	for (i = 0; i < S5K6AA_NUM_SUPPLIES; i++)
-		s5k6aa->supplies[i].supply = s5k6aa_supply_names[i];
-
-	ret = devm_regulator_bulk_get(&client->dev, S5K6AA_NUM_SUPPLIES,
-				 s5k6aa->supplies);
-	if (ret) {
-		dev_err(&client->dev, "Failed to get regulators\n");
-		goto out_err;
-	}
-
-	ret = s5k6aa_initialize_ctrls(s5k6aa);
-	if (ret)
-		goto out_err;
-
-	s5k6aa_presets_data_init(s5k6aa);
-
-	s5k6aa->ccd_rect.width = S5K6AA_WIN_WIDTH_MAX;
-	s5k6aa->ccd_rect.height	= S5K6AA_WIN_HEIGHT_MAX;
-	s5k6aa->ccd_rect.left = 0;
-	s5k6aa->ccd_rect.top = 0;
-
-	return 0;
-
-out_err:
-	media_entity_cleanup(&s5k6aa->sd.entity);
-	return ret;
-}
-
-static void s5k6aa_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-
-	v4l2_device_unregister_subdev(sd);
-	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	media_entity_cleanup(&sd->entity);
-}
-
-static const struct i2c_device_id s5k6aa_id[] = {
-	{ DRIVER_NAME, 0 },
-	{ },
-};
-MODULE_DEVICE_TABLE(i2c, s5k6aa_id);
-
-
-static struct i2c_driver s5k6aa_i2c_driver = {
-	.driver = {
-		.name = DRIVER_NAME
-	},
-	.probe_new	= s5k6aa_probe,
-	.remove		= s5k6aa_remove,
-	.id_table	= s5k6aa_id,
-};
-
-module_i2c_driver(s5k6aa_i2c_driver);
-
-MODULE_DESCRIPTION("Samsung S5K6AA(FX) SXGA camera driver");
-MODULE_AUTHOR("Sylwester Nawrocki <s.nawrocki@samsung.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/sr030pc30.c b/drivers/media/i2c/sr030pc30.c
deleted file mode 100644
index a83c8bf1c5dd..000000000000
--- a/drivers/media/i2c/sr030pc30.c
+++ /dev/null
@@ -1,762 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for SiliconFile SR030PC30 VGA (1/10-Inch) Image Sensor with ISP
- *
- * Copyright (C) 2010 Samsung Electronics Co., Ltd
- * Author: Sylwester Nawrocki, s.nawrocki@samsung.com
- *
- * Based on original driver authored by Dongsoo Nathaniel Kim
- * and HeungJun Kim <riverful.kim@samsung.com>.
- *
- * Based on mt9v011 Micron Digital Image Sensor driver
- * Copyright (c) 2009 Mauro Carvalho Chehab
- */
-
-#include <linux/i2c.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/module.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-subdev.h>
-#include <media/v4l2-mediabus.h>
-#include <media/v4l2-ctrls.h>
-#include <media/i2c/sr030pc30.h>
-
-static int debug;
-module_param(debug, int, 0644);
-
-#define MODULE_NAME	"SR030PC30"
-
-/*
- * Register offsets within a page
- * b15..b8 - page id, b7..b0 - register address
- */
-#define POWER_CTRL_REG		0x0001
-#define PAGEMODE_REG		0x03
-#define DEVICE_ID_REG		0x0004
-#define NOON010PC30_ID		0x86
-#define SR030PC30_ID		0x8C
-#define VDO_CTL1_REG		0x0010
-#define SUBSAMPL_NONE_VGA	0
-#define SUBSAMPL_QVGA		0x10
-#define SUBSAMPL_QQVGA		0x20
-#define VDO_CTL2_REG		0x0011
-#define SYNC_CTL_REG		0x0012
-#define WIN_ROWH_REG		0x0020
-#define WIN_ROWL_REG		0x0021
-#define WIN_COLH_REG		0x0022
-#define WIN_COLL_REG		0x0023
-#define WIN_HEIGHTH_REG		0x0024
-#define WIN_HEIGHTL_REG		0x0025
-#define WIN_WIDTHH_REG		0x0026
-#define WIN_WIDTHL_REG		0x0027
-#define HBLANKH_REG		0x0040
-#define HBLANKL_REG		0x0041
-#define VSYNCH_REG		0x0042
-#define VSYNCL_REG		0x0043
-/* page 10 */
-#define ISP_CTL_REG(n)		(0x1010 + (n))
-#define YOFS_REG		0x1040
-#define DARK_YOFS_REG		0x1041
-#define AG_ABRTH_REG		0x1050
-#define SAT_CTL_REG		0x1060
-#define BSAT_REG		0x1061
-#define RSAT_REG		0x1062
-#define AG_SAT_TH_REG		0x1063
-/* page 11 */
-#define ZLPF_CTRL_REG		0x1110
-#define ZLPF_CTRL2_REG		0x1112
-#define ZLPF_AGH_THR_REG	0x1121
-#define ZLPF_THR_REG		0x1160
-#define ZLPF_DYN_THR_REG	0x1160
-/* page 12 */
-#define YCLPF_CTL1_REG		0x1240
-#define YCLPF_CTL2_REG		0x1241
-#define YCLPF_THR_REG		0x1250
-#define BLPF_CTL_REG		0x1270
-#define BLPF_THR1_REG		0x1274
-#define BLPF_THR2_REG		0x1275
-/* page 14 - Lens Shading Compensation */
-#define LENS_CTRL_REG		0x1410
-#define LENS_XCEN_REG		0x1420
-#define LENS_YCEN_REG		0x1421
-#define LENS_R_COMP_REG		0x1422
-#define LENS_G_COMP_REG		0x1423
-#define LENS_B_COMP_REG		0x1424
-/* page 15 - Color correction */
-#define CMC_CTL_REG		0x1510
-#define CMC_OFSGH_REG		0x1514
-#define CMC_OFSGL_REG		0x1516
-#define CMC_SIGN_REG		0x1517
-/* Color correction coefficients */
-#define CMC_COEF_REG(n)		(0x1530 + (n))
-/* Color correction offset coefficients */
-#define CMC_OFS_REG(n)		(0x1540 + (n))
-/* page 16 - Gamma correction */
-#define GMA_CTL_REG		0x1610
-/* Gamma correction coefficients 0.14 */
-#define GMA_COEF_REG(n)		(0x1630 + (n))
-/* page 20 - Auto Exposure */
-#define AE_CTL1_REG		0x2010
-#define AE_CTL2_REG		0x2011
-#define AE_FRM_CTL_REG		0x2020
-#define AE_FINE_CTL_REG(n)	(0x2028 + (n))
-#define EXP_TIMEH_REG		0x2083
-#define EXP_TIMEM_REG		0x2084
-#define EXP_TIMEL_REG		0x2085
-#define EXP_MMINH_REG		0x2086
-#define EXP_MMINL_REG		0x2087
-#define EXP_MMAXH_REG		0x2088
-#define EXP_MMAXM_REG		0x2089
-#define EXP_MMAXL_REG		0x208A
-/* page 22 - Auto White Balance */
-#define AWB_CTL1_REG		0x2210
-#define AWB_ENABLE		0x80
-#define AWB_CTL2_REG		0x2211
-#define MWB_ENABLE		0x01
-/* RGB gain control (manual WB) when AWB_CTL1[7]=0 */
-#define AWB_RGAIN_REG		0x2280
-#define AWB_GGAIN_REG		0x2281
-#define AWB_BGAIN_REG		0x2282
-#define AWB_RMAX_REG		0x2283
-#define AWB_RMIN_REG		0x2284
-#define AWB_BMAX_REG		0x2285
-#define AWB_BMIN_REG		0x2286
-/* R, B gain range in bright light conditions */
-#define AWB_RMAXB_REG		0x2287
-#define AWB_RMINB_REG		0x2288
-#define AWB_BMAXB_REG		0x2289
-#define AWB_BMINB_REG		0x228A
-/* manual white balance, when AWB_CTL2[0]=1 */
-#define MWB_RGAIN_REG		0x22B2
-#define MWB_BGAIN_REG		0x22B3
-/* the token to mark an array end */
-#define REG_TERM		0xFFFF
-
-/* Minimum and maximum exposure time in ms */
-#define EXPOS_MIN_MS		1
-#define EXPOS_MAX_MS		125
-
-struct sr030pc30_info {
-	struct v4l2_subdev sd;
-	struct v4l2_ctrl_handler hdl;
-	const struct sr030pc30_platform_data *pdata;
-	const struct sr030pc30_format *curr_fmt;
-	const struct sr030pc30_frmsize *curr_win;
-	unsigned int hflip:1;
-	unsigned int vflip:1;
-	unsigned int sleep:1;
-	struct {
-		/* auto whitebalance control cluster */
-		struct v4l2_ctrl *awb;
-		struct v4l2_ctrl *red;
-		struct v4l2_ctrl *blue;
-	};
-	struct {
-		/* auto exposure control cluster */
-		struct v4l2_ctrl *autoexp;
-		struct v4l2_ctrl *exp;
-	};
-	u8 i2c_reg_page;
-};
-
-struct sr030pc30_format {
-	u32 code;
-	enum v4l2_colorspace colorspace;
-	u16 ispctl1_reg;
-};
-
-struct sr030pc30_frmsize {
-	u16 width;
-	u16 height;
-	int vid_ctl1;
-};
-
-struct i2c_regval {
-	u16 addr;
-	u16 val;
-};
-
-/* supported resolutions */
-static const struct sr030pc30_frmsize sr030pc30_sizes[] = {
-	{
-		.width		= 640,
-		.height		= 480,
-		.vid_ctl1	= SUBSAMPL_NONE_VGA,
-	}, {
-		.width		= 320,
-		.height		= 240,
-		.vid_ctl1	= SUBSAMPL_QVGA,
-	}, {
-		.width		= 160,
-		.height		= 120,
-		.vid_ctl1	= SUBSAMPL_QQVGA,
-	},
-};
-
-/* supported pixel formats */
-static const struct sr030pc30_format sr030pc30_formats[] = {
-	{
-		.code		= MEDIA_BUS_FMT_YUYV8_2X8,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-		.ispctl1_reg	= 0x03,
-	}, {
-		.code		= MEDIA_BUS_FMT_YVYU8_2X8,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-		.ispctl1_reg	= 0x02,
-	}, {
-		.code		= MEDIA_BUS_FMT_VYUY8_2X8,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-		.ispctl1_reg	= 0,
-	}, {
-		.code		= MEDIA_BUS_FMT_UYVY8_2X8,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-		.ispctl1_reg	= 0x01,
-	}, {
-		.code		= MEDIA_BUS_FMT_RGB565_2X8_BE,
-		.colorspace	= V4L2_COLORSPACE_JPEG,
-		.ispctl1_reg	= 0x40,
-	},
-};
-
-static const struct i2c_regval sr030pc30_base_regs[] = {
-	/* Window size and position within pixel matrix */
-	{ WIN_ROWH_REG,		0x00 }, { WIN_ROWL_REG,		0x06 },
-	{ WIN_COLH_REG,		0x00 },	{ WIN_COLL_REG,		0x06 },
-	{ WIN_HEIGHTH_REG,	0x01 }, { WIN_HEIGHTL_REG,	0xE0 },
-	{ WIN_WIDTHH_REG,	0x02 }, { WIN_WIDTHL_REG,	0x80 },
-	{ HBLANKH_REG,		0x01 }, { HBLANKL_REG,		0x50 },
-	{ VSYNCH_REG,		0x00 }, { VSYNCL_REG,		0x14 },
-	{ SYNC_CTL_REG,		0 },
-	/* Color corection and saturation */
-	{ ISP_CTL_REG(0),	0x30 }, { YOFS_REG,		0x80 },
-	{ DARK_YOFS_REG,	0x04 }, { AG_ABRTH_REG,		0x78 },
-	{ SAT_CTL_REG,		0x1F }, { BSAT_REG,		0x90 },
-	{ AG_SAT_TH_REG,	0xF0 }, { 0x1064,		0x80 },
-	{ CMC_CTL_REG,		0x03 }, { CMC_OFSGH_REG,	0x3C },
-	{ CMC_OFSGL_REG,	0x2C }, { CMC_SIGN_REG,		0x2F },
-	{ CMC_COEF_REG(0),	0xCB }, { CMC_OFS_REG(0),	0x87 },
-	{ CMC_COEF_REG(1),	0x61 }, { CMC_OFS_REG(1),	0x18 },
-	{ CMC_COEF_REG(2),	0x16 }, { CMC_OFS_REG(2),	0x91 },
-	{ CMC_COEF_REG(3),	0x23 }, { CMC_OFS_REG(3),	0x94 },
-	{ CMC_COEF_REG(4),	0xCE }, { CMC_OFS_REG(4),	0x9f },
-	{ CMC_COEF_REG(5),	0x2B }, { CMC_OFS_REG(5),	0x33 },
-	{ CMC_COEF_REG(6),	0x01 }, { CMC_OFS_REG(6),	0x00 },
-	{ CMC_COEF_REG(7),	0x34 }, { CMC_OFS_REG(7),	0x94 },
-	{ CMC_COEF_REG(8),	0x75 }, { CMC_OFS_REG(8),	0x14 },
-	/* Color corection coefficients */
-	{ GMA_CTL_REG,		0x03 },	{ GMA_COEF_REG(0),	0x00 },
-	{ GMA_COEF_REG(1),	0x19 },	{ GMA_COEF_REG(2),	0x26 },
-	{ GMA_COEF_REG(3),	0x3B },	{ GMA_COEF_REG(4),	0x5D },
-	{ GMA_COEF_REG(5),	0x79 }, { GMA_COEF_REG(6),	0x8E },
-	{ GMA_COEF_REG(7),	0x9F },	{ GMA_COEF_REG(8),	0xAF },
-	{ GMA_COEF_REG(9),	0xBD },	{ GMA_COEF_REG(10),	0xCA },
-	{ GMA_COEF_REG(11),	0xDD }, { GMA_COEF_REG(12),	0xEC },
-	{ GMA_COEF_REG(13),	0xF7 },	{ GMA_COEF_REG(14),	0xFF },
-	/* Noise reduction, Z-LPF, YC-LPF and BLPF filters setup */
-	{ ZLPF_CTRL_REG,	0x99 }, { ZLPF_CTRL2_REG,	0x0E },
-	{ ZLPF_AGH_THR_REG,	0x29 }, { ZLPF_THR_REG,		0x0F },
-	{ ZLPF_DYN_THR_REG,	0x63 }, { YCLPF_CTL1_REG,	0x23 },
-	{ YCLPF_CTL2_REG,	0x3B }, { YCLPF_THR_REG,	0x05 },
-	{ BLPF_CTL_REG,		0x1D }, { BLPF_THR1_REG,	0x05 },
-	{ BLPF_THR2_REG,	0x04 },
-	/* Automatic white balance */
-	{ AWB_CTL1_REG,		0xFB }, { AWB_CTL2_REG,		0x26 },
-	{ AWB_RMAX_REG,		0x54 }, { AWB_RMIN_REG,		0x2B },
-	{ AWB_BMAX_REG,		0x57 }, { AWB_BMIN_REG,		0x29 },
-	{ AWB_RMAXB_REG,	0x50 }, { AWB_RMINB_REG,	0x43 },
-	{ AWB_BMAXB_REG,	0x30 }, { AWB_BMINB_REG,	0x22 },
-	/* Auto exposure */
-	{ AE_CTL1_REG,		0x8C }, { AE_CTL2_REG,		0x04 },
-	{ AE_FRM_CTL_REG,	0x01 }, { AE_FINE_CTL_REG(0),	0x3F },
-	{ AE_FINE_CTL_REG(1),	0xA3 }, { AE_FINE_CTL_REG(3),	0x34 },
-	/* Lens shading compensation */
-	{ LENS_CTRL_REG,	0x01 }, { LENS_XCEN_REG,	0x80 },
-	{ LENS_YCEN_REG,	0x70 }, { LENS_R_COMP_REG,	0x53 },
-	{ LENS_G_COMP_REG,	0x40 }, { LENS_B_COMP_REG,	0x3e },
-	{ REG_TERM,		0 },
-};
-
-static inline struct sr030pc30_info *to_sr030pc30(struct v4l2_subdev *sd)
-{
-	return container_of(sd, struct sr030pc30_info, sd);
-}
-
-static inline int set_i2c_page(struct sr030pc30_info *info,
-			       struct i2c_client *client, unsigned int reg)
-{
-	int ret = 0;
-	u32 page = reg >> 8 & 0xFF;
-
-	if (info->i2c_reg_page != page && (reg & 0xFF) != 0x03) {
-		ret = i2c_smbus_write_byte_data(client, PAGEMODE_REG, page);
-		if (!ret)
-			info->i2c_reg_page = page;
-	}
-	return ret;
-}
-
-static int cam_i2c_read(struct v4l2_subdev *sd, u32 reg_addr)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct sr030pc30_info *info = to_sr030pc30(sd);
-
-	int ret = set_i2c_page(info, client, reg_addr);
-	if (!ret)
-		ret = i2c_smbus_read_byte_data(client, reg_addr & 0xFF);
-	return ret;
-}
-
-static int cam_i2c_write(struct v4l2_subdev *sd, u32 reg_addr, u32 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct sr030pc30_info *info = to_sr030pc30(sd);
-
-	int ret = set_i2c_page(info, client, reg_addr);
-	if (!ret)
-		ret = i2c_smbus_write_byte_data(
-			client, reg_addr & 0xFF, val);
-	return ret;
-}
-
-static inline int sr030pc30_bulk_write_reg(struct v4l2_subdev *sd,
-				const struct i2c_regval *msg)
-{
-	while (msg->addr != REG_TERM) {
-		int ret = cam_i2c_write(sd, msg->addr, msg->val);
-		if (ret)
-			return ret;
-		msg++;
-	}
-	return 0;
-}
-
-/* Device reset and sleep mode control */
-static int sr030pc30_pwr_ctrl(struct v4l2_subdev *sd,
-				     bool reset, bool sleep)
-{
-	struct sr030pc30_info *info = to_sr030pc30(sd);
-	u8 reg = sleep ? 0xF1 : 0xF0;
-	int ret = 0;
-
-	if (reset)
-		ret = cam_i2c_write(sd, POWER_CTRL_REG, reg | 0x02);
-	if (!ret) {
-		ret = cam_i2c_write(sd, POWER_CTRL_REG, reg);
-		if (!ret) {
-			info->sleep = sleep;
-			if (reset)
-				info->i2c_reg_page = -1;
-		}
-	}
-	return ret;
-}
-
-static int sr030pc30_set_flip(struct v4l2_subdev *sd)
-{
-	struct sr030pc30_info *info = to_sr030pc30(sd);
-
-	s32 reg = cam_i2c_read(sd, VDO_CTL2_REG);
-	if (reg < 0)
-		return reg;
-
-	reg &= 0x7C;
-	if (info->hflip)
-		reg |= 0x01;
-	if (info->vflip)
-		reg |= 0x02;
-	return cam_i2c_write(sd, VDO_CTL2_REG, reg | 0x80);
-}
-
-/* Configure resolution, color format and image flip */
-static int sr030pc30_set_params(struct v4l2_subdev *sd)
-{
-	struct sr030pc30_info *info = to_sr030pc30(sd);
-	int ret;
-
-	if (!info->curr_win)
-		return -EINVAL;
-
-	/* Configure the resolution through subsampling */
-	ret = cam_i2c_write(sd, VDO_CTL1_REG,
-			    info->curr_win->vid_ctl1);
-
-	if (!ret && info->curr_fmt)
-		ret = cam_i2c_write(sd, ISP_CTL_REG(0),
-				info->curr_fmt->ispctl1_reg);
-	if (!ret)
-		ret = sr030pc30_set_flip(sd);
-
-	return ret;
-}
-
-/* Find nearest matching image pixel size. */
-static int sr030pc30_try_frame_size(struct v4l2_mbus_framefmt *mf)
-{
-	unsigned int min_err = ~0;
-	int i = ARRAY_SIZE(sr030pc30_sizes);
-	const struct sr030pc30_frmsize *fsize = &sr030pc30_sizes[0],
-					*match = NULL;
-	while (i--) {
-		int err = abs(fsize->width - mf->width)
-				+ abs(fsize->height - mf->height);
-		if (err < min_err) {
-			min_err = err;
-			match = fsize;
-		}
-		fsize++;
-	}
-	if (match) {
-		mf->width  = match->width;
-		mf->height = match->height;
-		return 0;
-	}
-	return -EINVAL;
-}
-
-static int sr030pc30_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct sr030pc30_info *info =
-		container_of(ctrl->handler, struct sr030pc30_info, hdl);
-	struct v4l2_subdev *sd = &info->sd;
-	int ret = 0;
-
-	v4l2_dbg(1, debug, sd, "%s: ctrl_id: %d, value: %d\n",
-			 __func__, ctrl->id, ctrl->val);
-
-	switch (ctrl->id) {
-	case V4L2_CID_AUTO_WHITE_BALANCE:
-		if (ctrl->is_new) {
-			ret = cam_i2c_write(sd, AWB_CTL2_REG,
-					ctrl->val ? 0x2E : 0x2F);
-			if (!ret)
-				ret = cam_i2c_write(sd, AWB_CTL1_REG,
-						ctrl->val ? 0xFB : 0x7B);
-		}
-		if (!ret && info->blue->is_new)
-			ret = cam_i2c_write(sd, MWB_BGAIN_REG, info->blue->val);
-		if (!ret && info->red->is_new)
-			ret = cam_i2c_write(sd, MWB_RGAIN_REG, info->red->val);
-		return ret;
-
-	case V4L2_CID_EXPOSURE_AUTO:
-		/* auto anti-flicker is also enabled here */
-		if (ctrl->is_new)
-			ret = cam_i2c_write(sd, AE_CTL1_REG,
-				ctrl->val == V4L2_EXPOSURE_AUTO ? 0xDC : 0x0C);
-		if (info->exp->is_new) {
-			unsigned long expos = info->exp->val;
-
-			expos = expos * info->pdata->clk_rate / (8 * 1000);
-
-			if (!ret)
-				ret = cam_i2c_write(sd, EXP_TIMEH_REG,
-						expos >> 16 & 0xFF);
-			if (!ret)
-				ret = cam_i2c_write(sd, EXP_TIMEM_REG,
-						expos >> 8 & 0xFF);
-			if (!ret)
-				ret = cam_i2c_write(sd, EXP_TIMEL_REG,
-						expos & 0xFF);
-		}
-		return ret;
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int sr030pc30_enum_mbus_code(struct v4l2_subdev *sd,
-		struct v4l2_subdev_state *sd_state,
-		struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (!code || code->pad ||
-	    code->index >= ARRAY_SIZE(sr030pc30_formats))
-		return -EINVAL;
-
-	code->code = sr030pc30_formats[code->index].code;
-	return 0;
-}
-
-static int sr030pc30_get_fmt(struct v4l2_subdev *sd,
-		struct v4l2_subdev_state *sd_state,
-		struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *mf;
-	struct sr030pc30_info *info = to_sr030pc30(sd);
-
-	if (!format || format->pad)
-		return -EINVAL;
-
-	mf = &format->format;
-
-	if (!info->curr_win || !info->curr_fmt)
-		return -EINVAL;
-
-	mf->width	= info->curr_win->width;
-	mf->height	= info->curr_win->height;
-	mf->code	= info->curr_fmt->code;
-	mf->colorspace	= info->curr_fmt->colorspace;
-	mf->field	= V4L2_FIELD_NONE;
-
-	return 0;
-}
-
-/* Return nearest media bus frame format. */
-static const struct sr030pc30_format *try_fmt(struct v4l2_subdev *sd,
-					      struct v4l2_mbus_framefmt *mf)
-{
-	int i;
-
-	sr030pc30_try_frame_size(mf);
-
-	for (i = 0; i < ARRAY_SIZE(sr030pc30_formats); i++) {
-		if (mf->code == sr030pc30_formats[i].code)
-			break;
-	}
-	if (i == ARRAY_SIZE(sr030pc30_formats))
-		i = 0;
-
-	mf->code = sr030pc30_formats[i].code;
-
-	return &sr030pc30_formats[i];
-}
-
-/* Return nearest media bus frame format. */
-static int sr030pc30_set_fmt(struct v4l2_subdev *sd,
-		struct v4l2_subdev_state *sd_state,
-		struct v4l2_subdev_format *format)
-{
-	struct sr030pc30_info *info = sd ? to_sr030pc30(sd) : NULL;
-	const struct sr030pc30_format *fmt;
-	struct v4l2_mbus_framefmt *mf;
-
-	if (!sd || !format)
-		return -EINVAL;
-
-	mf = &format->format;
-	if (format->pad)
-		return -EINVAL;
-
-	fmt = try_fmt(sd, mf);
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		sd_state->pads->try_fmt = *mf;
-		return 0;
-	}
-
-	info->curr_fmt = fmt;
-
-	return sr030pc30_set_params(sd);
-}
-
-static int sr030pc30_base_config(struct v4l2_subdev *sd)
-{
-	struct sr030pc30_info *info = to_sr030pc30(sd);
-	int ret;
-	unsigned long expmin, expmax;
-
-	ret = sr030pc30_bulk_write_reg(sd, sr030pc30_base_regs);
-	if (!ret) {
-		info->curr_fmt = &sr030pc30_formats[0];
-		info->curr_win = &sr030pc30_sizes[0];
-		ret = sr030pc30_set_params(sd);
-	}
-	if (!ret)
-		ret = sr030pc30_pwr_ctrl(sd, false, false);
-
-	if (ret)
-		return ret;
-
-	expmin = EXPOS_MIN_MS * info->pdata->clk_rate / (8 * 1000);
-	expmax = EXPOS_MAX_MS * info->pdata->clk_rate / (8 * 1000);
-
-	v4l2_dbg(1, debug, sd, "%s: expmin= %lx, expmax= %lx", __func__,
-		 expmin, expmax);
-
-	/* Setting up manual exposure time range */
-	ret = cam_i2c_write(sd, EXP_MMINH_REG, expmin >> 8 & 0xFF);
-	if (!ret)
-		ret = cam_i2c_write(sd, EXP_MMINL_REG, expmin & 0xFF);
-	if (!ret)
-		ret = cam_i2c_write(sd, EXP_MMAXH_REG, expmax >> 16 & 0xFF);
-	if (!ret)
-		ret = cam_i2c_write(sd, EXP_MMAXM_REG, expmax >> 8 & 0xFF);
-	if (!ret)
-		ret = cam_i2c_write(sd, EXP_MMAXL_REG, expmax & 0xFF);
-
-	return ret;
-}
-
-static int sr030pc30_s_power(struct v4l2_subdev *sd, int on)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct sr030pc30_info *info = to_sr030pc30(sd);
-	const struct sr030pc30_platform_data *pdata = info->pdata;
-	int ret;
-
-	if (pdata == NULL) {
-		WARN(1, "No platform data!\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * Put sensor into power sleep mode before switching off
-	 * power and disabling MCLK.
-	 */
-	if (!on)
-		sr030pc30_pwr_ctrl(sd, false, true);
-
-	/* set_power controls sensor's power and clock */
-	if (pdata->set_power) {
-		ret = pdata->set_power(&client->dev, on);
-		if (ret)
-			return ret;
-	}
-
-	if (on) {
-		ret = sr030pc30_base_config(sd);
-	} else {
-		ret = 0;
-		info->curr_win = NULL;
-		info->curr_fmt = NULL;
-	}
-
-	return ret;
-}
-
-static const struct v4l2_ctrl_ops sr030pc30_ctrl_ops = {
-	.s_ctrl = sr030pc30_s_ctrl,
-};
-
-static const struct v4l2_subdev_core_ops sr030pc30_core_ops = {
-	.s_power	= sr030pc30_s_power,
-};
-
-static const struct v4l2_subdev_pad_ops sr030pc30_pad_ops = {
-	.enum_mbus_code = sr030pc30_enum_mbus_code,
-	.get_fmt	= sr030pc30_get_fmt,
-	.set_fmt	= sr030pc30_set_fmt,
-};
-
-static const struct v4l2_subdev_ops sr030pc30_ops = {
-	.core	= &sr030pc30_core_ops,
-	.pad	= &sr030pc30_pad_ops,
-};
-
-/*
- * Detect sensor type. Return 0 if SR030PC30 was detected
- * or -ENODEV otherwise.
- */
-static int sr030pc30_detect(struct i2c_client *client)
-{
-	const struct sr030pc30_platform_data *pdata
-		= client->dev.platform_data;
-	int ret;
-
-	/* Enable sensor's power and clock */
-	if (pdata->set_power) {
-		ret = pdata->set_power(&client->dev, 1);
-		if (ret)
-			return ret;
-	}
-
-	ret = i2c_smbus_read_byte_data(client, DEVICE_ID_REG);
-
-	if (pdata->set_power)
-		pdata->set_power(&client->dev, 0);
-
-	if (ret < 0) {
-		dev_err(&client->dev, "%s: I2C read failed\n", __func__);
-		return ret;
-	}
-
-	return ret == SR030PC30_ID ? 0 : -ENODEV;
-}
-
-
-static int sr030pc30_probe(struct i2c_client *client)
-{
-	struct sr030pc30_info *info;
-	struct v4l2_subdev *sd;
-	struct v4l2_ctrl_handler *hdl;
-	const struct sr030pc30_platform_data *pdata
-		= client->dev.platform_data;
-	int ret;
-
-	if (!pdata) {
-		dev_err(&client->dev, "No platform data!");
-		return -EIO;
-	}
-
-	ret = sr030pc30_detect(client);
-	if (ret)
-		return ret;
-
-	info = devm_kzalloc(&client->dev, sizeof(*info), GFP_KERNEL);
-	if (!info)
-		return -ENOMEM;
-
-	sd = &info->sd;
-	info->pdata = client->dev.platform_data;
-
-	v4l2_i2c_subdev_init(sd, client, &sr030pc30_ops);
-
-	hdl = &info->hdl;
-	v4l2_ctrl_handler_init(hdl, 6);
-	info->awb = v4l2_ctrl_new_std(hdl, &sr030pc30_ctrl_ops,
-			V4L2_CID_AUTO_WHITE_BALANCE, 0, 1, 1, 1);
-	info->red = v4l2_ctrl_new_std(hdl, &sr030pc30_ctrl_ops,
-			V4L2_CID_RED_BALANCE, 0, 127, 1, 64);
-	info->blue = v4l2_ctrl_new_std(hdl, &sr030pc30_ctrl_ops,
-			V4L2_CID_BLUE_BALANCE, 0, 127, 1, 64);
-	info->autoexp = v4l2_ctrl_new_std(hdl, &sr030pc30_ctrl_ops,
-			V4L2_CID_EXPOSURE_AUTO, 0, 1, 1, 1);
-	info->exp = v4l2_ctrl_new_std(hdl, &sr030pc30_ctrl_ops,
-			V4L2_CID_EXPOSURE, EXPOS_MIN_MS, EXPOS_MAX_MS, 1, 30);
-	sd->ctrl_handler = hdl;
-	if (hdl->error) {
-		int err = hdl->error;
-
-		v4l2_ctrl_handler_free(hdl);
-		return err;
-	}
-	v4l2_ctrl_auto_cluster(3, &info->awb, 0, false);
-	v4l2_ctrl_auto_cluster(2, &info->autoexp, V4L2_EXPOSURE_MANUAL, false);
-	v4l2_ctrl_handler_setup(hdl);
-
-	info->i2c_reg_page	= -1;
-	info->hflip		= 1;
-
-	return 0;
-}
-
-static void sr030pc30_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-
-	v4l2_device_unregister_subdev(sd);
-	v4l2_ctrl_handler_free(sd->ctrl_handler);
-}
-
-static const struct i2c_device_id sr030pc30_id[] = {
-	{ MODULE_NAME, 0 },
-	{ },
-};
-MODULE_DEVICE_TABLE(i2c, sr030pc30_id);
-
-
-static struct i2c_driver sr030pc30_i2c_driver = {
-	.driver = {
-		.name = MODULE_NAME
-	},
-	.probe_new	= sr030pc30_probe,
-	.remove		= sr030pc30_remove,
-	.id_table	= sr030pc30_id,
-};
-
-module_i2c_driver(sr030pc30_i2c_driver);
-
-MODULE_DESCRIPTION("Siliconfile SR030PC30 camera driver");
-MODULE_AUTHOR("Sylwester Nawrocki <s.nawrocki@samsung.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/st-vgxy61.c b/drivers/media/i2c/st-vgxy61.c
index 5dcabee6677d..adbd093ad190 100644
--- a/drivers/media/i2c/st-vgxy61.c
+++ b/drivers/media/i2c/st-vgxy61.c
@@ -1334,7 +1334,6 @@ static int vgxy61_init_cfg(struct v4l2_subdev *sd,
 	struct vgxy61_dev *sensor = to_vgxy61_dev(sd);
 	struct v4l2_subdev_format fmt = { 0 };
 
-	sensor->current_mode = sensor->default_mode;
 	vgxy61_fill_framefmt(sensor, sensor->current_mode, &fmt.format,
 			     VGXY61_MEDIA_BUS_FMT_DEF);
 
@@ -1549,7 +1548,7 @@ static int vgxy61_tx_from_ep(struct vgxy61_dev *sensor,
 	sensor->nb_of_lane = l_nb;
 
 	dev_dbg(&client->dev, "tx uses %d lanes", l_nb);
-	for (i = 0; i < 5; i++) {
+	for (i = 0; i < VGXY61_NB_POLARITIES; i++) {
 		dev_dbg(&client->dev, "log2phy[%d] = %d\n", i, log2phy[i]);
 		dev_dbg(&client->dev, "phy2log[%d] = %d\n", i, phy2log[i]);
 		dev_dbg(&client->dev, "polarity[%d] = %d\n", i, polarities[i]);
@@ -1735,6 +1734,12 @@ static int vgxy61_power_on(struct device *dev)
 		}
 	}
 
+	ret = vgxy61_detect(sensor);
+	if (ret) {
+		dev_err(&client->dev, "sensor detect failed %d\n", ret);
+		goto disable_clock;
+	}
+
 	ret = vgxy61_patch(sensor);
 	if (ret) {
 		dev_err(&client->dev, "sensor patch failed %d\n", ret);
@@ -1861,21 +1866,15 @@ static int vgxy61_probe(struct i2c_client *client)
 	if (ret)
 		return ret;
 
-	ret = vgxy61_detect(sensor);
-	if (ret) {
-		dev_err(&client->dev, "sensor detect failed %d\n", ret);
-		return ret;
-	}
-
 	vgxy61_fill_sensor_param(sensor);
 	vgxy61_fill_framefmt(sensor, sensor->current_mode, &sensor->fmt,
 			     VGXY61_MEDIA_BUS_FMT_DEF);
 
+	mutex_init(&sensor->lock);
+
 	ret = vgxy61_update_hdr(sensor, sensor->hdr);
 	if (ret)
-		return ret;
-
-	mutex_init(&sensor->lock);
+		goto error_power_off;
 
 	ret = vgxy61_init_controls(sensor);
 	if (ret) {
@@ -1914,8 +1913,8 @@ error_pm_runtime:
 	media_entity_cleanup(&sensor->sd.entity);
 error_handler_free:
 	v4l2_ctrl_handler_free(sensor->sd.ctrl_handler);
-	mutex_destroy(&sensor->lock);
 error_power_off:
+	mutex_destroy(&sensor->lock);
 	vgxy61_power_off(dev);
 
 	return ret;
diff --git a/drivers/media/i2c/tc358746.c b/drivers/media/i2c/tc358746.c
index 4063754a6732..ec1a193ba161 100644
--- a/drivers/media/i2c/tc358746.c
+++ b/drivers/media/i2c/tc358746.c
@@ -854,11 +854,11 @@ static unsigned long tc358746_find_pll_settings(struct tc358746 *tc358746,
 			m_best = mul;
 			min_delta = delta;
 			best_freq = tmp;
-		};
+		}
 
 		if (delta == 0)
 			break;
-	};
+	}
 
 	if (!best_freq) {
 		dev_err(dev, "Failed find PLL frequency\n");
diff --git a/drivers/media/i2c/vs6624.c b/drivers/media/i2c/vs6624.c
deleted file mode 100644
index d35c5ec148f4..000000000000
--- a/drivers/media/i2c/vs6624.c
+++ /dev/null
@@ -1,854 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * vs6624.c ST VS6624 CMOS image sensor driver
- *
- * Copyright (c) 2011 Analog Devices Inc.
- */
-
-#include <linux/delay.h>
-#include <linux/errno.h>
-#include <linux/gpio.h>
-#include <linux/i2c.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/types.h>
-#include <linux/videodev2.h>
-
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-mediabus.h>
-#include <media/v4l2-image-sizes.h>
-
-#include "vs6624_regs.h"
-
-#define MAX_FRAME_RATE  30
-
-struct vs6624 {
-	struct v4l2_subdev sd;
-	struct v4l2_ctrl_handler hdl;
-	struct v4l2_fract frame_rate;
-	struct v4l2_mbus_framefmt fmt;
-	unsigned ce_pin;
-};
-
-static const struct vs6624_format {
-	u32 mbus_code;
-	enum v4l2_colorspace colorspace;
-} vs6624_formats[] = {
-	{
-		.mbus_code      = MEDIA_BUS_FMT_UYVY8_2X8,
-		.colorspace     = V4L2_COLORSPACE_JPEG,
-	},
-	{
-		.mbus_code      = MEDIA_BUS_FMT_YUYV8_2X8,
-		.colorspace     = V4L2_COLORSPACE_JPEG,
-	},
-	{
-		.mbus_code      = MEDIA_BUS_FMT_RGB565_2X8_LE,
-		.colorspace     = V4L2_COLORSPACE_SRGB,
-	},
-};
-
-static const struct v4l2_mbus_framefmt vs6624_default_fmt = {
-	.width = VGA_WIDTH,
-	.height = VGA_HEIGHT,
-	.code = MEDIA_BUS_FMT_UYVY8_2X8,
-	.field = V4L2_FIELD_NONE,
-	.colorspace = V4L2_COLORSPACE_JPEG,
-};
-
-static const u16 vs6624_p1[] = {
-	0x8104, 0x03,
-	0x8105, 0x01,
-	0xc900, 0x03,
-	0xc904, 0x47,
-	0xc905, 0x10,
-	0xc906, 0x80,
-	0xc907, 0x3a,
-	0x903a, 0x02,
-	0x903b, 0x47,
-	0x903c, 0x15,
-	0xc908, 0x31,
-	0xc909, 0xdc,
-	0xc90a, 0x80,
-	0xc90b, 0x44,
-	0x9044, 0x02,
-	0x9045, 0x31,
-	0x9046, 0xe2,
-	0xc90c, 0x07,
-	0xc90d, 0xe0,
-	0xc90e, 0x80,
-	0xc90f, 0x47,
-	0x9047, 0x90,
-	0x9048, 0x83,
-	0x9049, 0x81,
-	0x904a, 0xe0,
-	0x904b, 0x60,
-	0x904c, 0x08,
-	0x904d, 0x90,
-	0x904e, 0xc0,
-	0x904f, 0x43,
-	0x9050, 0x74,
-	0x9051, 0x01,
-	0x9052, 0xf0,
-	0x9053, 0x80,
-	0x9054, 0x05,
-	0x9055, 0xE4,
-	0x9056, 0x90,
-	0x9057, 0xc0,
-	0x9058, 0x43,
-	0x9059, 0xf0,
-	0x905a, 0x02,
-	0x905b, 0x07,
-	0x905c, 0xec,
-	0xc910, 0x5d,
-	0xc911, 0xca,
-	0xc912, 0x80,
-	0xc913, 0x5d,
-	0x905d, 0xa3,
-	0x905e, 0x04,
-	0x905f, 0xf0,
-	0x9060, 0xa3,
-	0x9061, 0x04,
-	0x9062, 0xf0,
-	0x9063, 0x22,
-	0xc914, 0x72,
-	0xc915, 0x92,
-	0xc916, 0x80,
-	0xc917, 0x64,
-	0x9064, 0x74,
-	0x9065, 0x01,
-	0x9066, 0x02,
-	0x9067, 0x72,
-	0x9068, 0x95,
-	0xc918, 0x47,
-	0xc919, 0xf2,
-	0xc91a, 0x81,
-	0xc91b, 0x69,
-	0x9169, 0x74,
-	0x916a, 0x02,
-	0x916b, 0xf0,
-	0x916c, 0xec,
-	0x916d, 0xb4,
-	0x916e, 0x10,
-	0x916f, 0x0a,
-	0x9170, 0x90,
-	0x9171, 0x80,
-	0x9172, 0x16,
-	0x9173, 0xe0,
-	0x9174, 0x70,
-	0x9175, 0x04,
-	0x9176, 0x90,
-	0x9177, 0xd3,
-	0x9178, 0xc4,
-	0x9179, 0xf0,
-	0x917a, 0x22,
-	0xc91c, 0x0a,
-	0xc91d, 0xbe,
-	0xc91e, 0x80,
-	0xc91f, 0x73,
-	0x9073, 0xfc,
-	0x9074, 0xa3,
-	0x9075, 0xe0,
-	0x9076, 0xf5,
-	0x9077, 0x82,
-	0x9078, 0x8c,
-	0x9079, 0x83,
-	0x907a, 0xa3,
-	0x907b, 0xa3,
-	0x907c, 0xe0,
-	0x907d, 0xfc,
-	0x907e, 0xa3,
-	0x907f, 0xe0,
-	0x9080, 0xc3,
-	0x9081, 0x9f,
-	0x9082, 0xff,
-	0x9083, 0xec,
-	0x9084, 0x9e,
-	0x9085, 0xfe,
-	0x9086, 0x02,
-	0x9087, 0x0a,
-	0x9088, 0xea,
-	0xc920, 0x47,
-	0xc921, 0x38,
-	0xc922, 0x80,
-	0xc923, 0x89,
-	0x9089, 0xec,
-	0x908a, 0xd3,
-	0x908b, 0x94,
-	0x908c, 0x20,
-	0x908d, 0x40,
-	0x908e, 0x01,
-	0x908f, 0x1c,
-	0x9090, 0x90,
-	0x9091, 0xd3,
-	0x9092, 0xd4,
-	0x9093, 0xec,
-	0x9094, 0xf0,
-	0x9095, 0x02,
-	0x9096, 0x47,
-	0x9097, 0x3d,
-	0xc924, 0x45,
-	0xc925, 0xca,
-	0xc926, 0x80,
-	0xc927, 0x98,
-	0x9098, 0x12,
-	0x9099, 0x77,
-	0x909a, 0xd6,
-	0x909b, 0x02,
-	0x909c, 0x45,
-	0x909d, 0xcd,
-	0xc928, 0x20,
-	0xc929, 0xd5,
-	0xc92a, 0x80,
-	0xc92b, 0x9e,
-	0x909e, 0x90,
-	0x909f, 0x82,
-	0x90a0, 0x18,
-	0x90a1, 0xe0,
-	0x90a2, 0xb4,
-	0x90a3, 0x03,
-	0x90a4, 0x0e,
-	0x90a5, 0x90,
-	0x90a6, 0x83,
-	0x90a7, 0xbf,
-	0x90a8, 0xe0,
-	0x90a9, 0x60,
-	0x90aa, 0x08,
-	0x90ab, 0x90,
-	0x90ac, 0x81,
-	0x90ad, 0xfc,
-	0x90ae, 0xe0,
-	0x90af, 0xff,
-	0x90b0, 0xc3,
-	0x90b1, 0x13,
-	0x90b2, 0xf0,
-	0x90b3, 0x90,
-	0x90b4, 0x81,
-	0x90b5, 0xfc,
-	0x90b6, 0xe0,
-	0x90b7, 0xff,
-	0x90b8, 0x02,
-	0x90b9, 0x20,
-	0x90ba, 0xda,
-	0xc92c, 0x70,
-	0xc92d, 0xbc,
-	0xc92e, 0x80,
-	0xc92f, 0xbb,
-	0x90bb, 0x90,
-	0x90bc, 0x82,
-	0x90bd, 0x18,
-	0x90be, 0xe0,
-	0x90bf, 0xb4,
-	0x90c0, 0x03,
-	0x90c1, 0x06,
-	0x90c2, 0x90,
-	0x90c3, 0xc1,
-	0x90c4, 0x06,
-	0x90c5, 0x74,
-	0x90c6, 0x05,
-	0x90c7, 0xf0,
-	0x90c8, 0x90,
-	0x90c9, 0xd3,
-	0x90ca, 0xa0,
-	0x90cb, 0x02,
-	0x90cc, 0x70,
-	0x90cd, 0xbf,
-	0xc930, 0x72,
-	0xc931, 0x21,
-	0xc932, 0x81,
-	0xc933, 0x3b,
-	0x913b, 0x7d,
-	0x913c, 0x02,
-	0x913d, 0x7f,
-	0x913e, 0x7b,
-	0x913f, 0x02,
-	0x9140, 0x72,
-	0x9141, 0x25,
-	0xc934, 0x28,
-	0xc935, 0xae,
-	0xc936, 0x80,
-	0xc937, 0xd2,
-	0x90d2, 0xf0,
-	0x90d3, 0x90,
-	0x90d4, 0xd2,
-	0x90d5, 0x0a,
-	0x90d6, 0x02,
-	0x90d7, 0x28,
-	0x90d8, 0xb4,
-	0xc938, 0x28,
-	0xc939, 0xb1,
-	0xc93a, 0x80,
-	0xc93b, 0xd9,
-	0x90d9, 0x90,
-	0x90da, 0x83,
-	0x90db, 0xba,
-	0x90dc, 0xe0,
-	0x90dd, 0xff,
-	0x90de, 0x90,
-	0x90df, 0xd2,
-	0x90e0, 0x08,
-	0x90e1, 0xe0,
-	0x90e2, 0xe4,
-	0x90e3, 0xef,
-	0x90e4, 0xf0,
-	0x90e5, 0xa3,
-	0x90e6, 0xe0,
-	0x90e7, 0x74,
-	0x90e8, 0xff,
-	0x90e9, 0xf0,
-	0x90ea, 0x90,
-	0x90eb, 0xd2,
-	0x90ec, 0x0a,
-	0x90ed, 0x02,
-	0x90ee, 0x28,
-	0x90ef, 0xb4,
-	0xc93c, 0x29,
-	0xc93d, 0x79,
-	0xc93e, 0x80,
-	0xc93f, 0xf0,
-	0x90f0, 0xf0,
-	0x90f1, 0x90,
-	0x90f2, 0xd2,
-	0x90f3, 0x0e,
-	0x90f4, 0x02,
-	0x90f5, 0x29,
-	0x90f6, 0x7f,
-	0xc940, 0x29,
-	0xc941, 0x7c,
-	0xc942, 0x80,
-	0xc943, 0xf7,
-	0x90f7, 0x90,
-	0x90f8, 0x83,
-	0x90f9, 0xba,
-	0x90fa, 0xe0,
-	0x90fb, 0xff,
-	0x90fc, 0x90,
-	0x90fd, 0xd2,
-	0x90fe, 0x0c,
-	0x90ff, 0xe0,
-	0x9100, 0xe4,
-	0x9101, 0xef,
-	0x9102, 0xf0,
-	0x9103, 0xa3,
-	0x9104, 0xe0,
-	0x9105, 0x74,
-	0x9106, 0xff,
-	0x9107, 0xf0,
-	0x9108, 0x90,
-	0x9109, 0xd2,
-	0x910a, 0x0e,
-	0x910b, 0x02,
-	0x910c, 0x29,
-	0x910d, 0x7f,
-	0xc944, 0x2a,
-	0xc945, 0x42,
-	0xc946, 0x81,
-	0xc947, 0x0e,
-	0x910e, 0xf0,
-	0x910f, 0x90,
-	0x9110, 0xd2,
-	0x9111, 0x12,
-	0x9112, 0x02,
-	0x9113, 0x2a,
-	0x9114, 0x48,
-	0xc948, 0x2a,
-	0xc949, 0x45,
-	0xc94a, 0x81,
-	0xc94b, 0x15,
-	0x9115, 0x90,
-	0x9116, 0x83,
-	0x9117, 0xba,
-	0x9118, 0xe0,
-	0x9119, 0xff,
-	0x911a, 0x90,
-	0x911b, 0xd2,
-	0x911c, 0x10,
-	0x911d, 0xe0,
-	0x911e, 0xe4,
-	0x911f, 0xef,
-	0x9120, 0xf0,
-	0x9121, 0xa3,
-	0x9122, 0xe0,
-	0x9123, 0x74,
-	0x9124, 0xff,
-	0x9125, 0xf0,
-	0x9126, 0x90,
-	0x9127, 0xd2,
-	0x9128, 0x12,
-	0x9129, 0x02,
-	0x912a, 0x2a,
-	0x912b, 0x48,
-	0xc900, 0x01,
-	0x0000, 0x00,
-};
-
-static const u16 vs6624_p2[] = {
-	0x806f, 0x01,
-	0x058c, 0x01,
-	0x0000, 0x00,
-};
-
-static const u16 vs6624_run_setup[] = {
-	0x1d18, 0x00,				/* Enableconstrainedwhitebalance */
-	VS6624_PEAK_MIN_OUT_G_MSB, 0x3c,	/* Damper PeakGain Output MSB */
-	VS6624_PEAK_MIN_OUT_G_LSB, 0x66,	/* Damper PeakGain Output LSB */
-	VS6624_CM_LOW_THR_MSB, 0x65,		/* Damper Low MSB */
-	VS6624_CM_LOW_THR_LSB, 0xd1,		/* Damper Low LSB */
-	VS6624_CM_HIGH_THR_MSB, 0x66,		/* Damper High MSB */
-	VS6624_CM_HIGH_THR_LSB, 0x62,		/* Damper High LSB */
-	VS6624_CM_MIN_OUT_MSB, 0x00,		/* Damper Min output MSB */
-	VS6624_CM_MIN_OUT_LSB, 0x00,		/* Damper Min output LSB */
-	VS6624_NORA_DISABLE, 0x00,		/* Nora fDisable */
-	VS6624_NORA_USAGE, 0x04,		/* Nora usage */
-	VS6624_NORA_LOW_THR_MSB, 0x63,		/* Damper Low MSB Changed 0x63 to 0x65 */
-	VS6624_NORA_LOW_THR_LSB, 0xd1,		/* Damper Low LSB */
-	VS6624_NORA_HIGH_THR_MSB, 0x68,		/* Damper High MSB */
-	VS6624_NORA_HIGH_THR_LSB, 0xdd,		/* Damper High LSB */
-	VS6624_NORA_MIN_OUT_MSB, 0x3a,		/* Damper Min output MSB */
-	VS6624_NORA_MIN_OUT_LSB, 0x00,		/* Damper Min output LSB */
-	VS6624_F2B_DISABLE, 0x00,		/* Disable */
-	0x1d8a, 0x30,				/* MAXWeightHigh */
-	0x1d91, 0x62,				/* fpDamperLowThresholdHigh MSB */
-	0x1d92, 0x4a,				/* fpDamperLowThresholdHigh LSB */
-	0x1d95, 0x65,				/* fpDamperHighThresholdHigh MSB */
-	0x1d96, 0x0e,				/* fpDamperHighThresholdHigh LSB */
-	0x1da1, 0x3a,				/* fpMinimumDamperOutputLow MSB */
-	0x1da2, 0xb8,				/* fpMinimumDamperOutputLow LSB */
-	0x1e08, 0x06,				/* MAXWeightLow */
-	0x1e0a, 0x0a,				/* MAXWeightHigh */
-	0x1601, 0x3a,				/* Red A MSB */
-	0x1602, 0x14,				/* Red A LSB */
-	0x1605, 0x3b,				/* Blue A MSB */
-	0x1606, 0x85,				/* BLue A LSB */
-	0x1609, 0x3b,				/* RED B MSB */
-	0x160a, 0x85,				/* RED B LSB */
-	0x160d, 0x3a,				/* Blue B MSB */
-	0x160e, 0x14,				/* Blue B LSB */
-	0x1611, 0x30,				/* Max Distance from Locus MSB */
-	0x1612, 0x8f,				/* Max Distance from Locus MSB */
-	0x1614, 0x01,				/* Enable constrainer */
-	0x0000, 0x00,
-};
-
-static const u16 vs6624_default[] = {
-	VS6624_CONTRAST0, 0x84,
-	VS6624_SATURATION0, 0x75,
-	VS6624_GAMMA0, 0x11,
-	VS6624_CONTRAST1, 0x84,
-	VS6624_SATURATION1, 0x75,
-	VS6624_GAMMA1, 0x11,
-	VS6624_MAN_RG, 0x80,
-	VS6624_MAN_GG, 0x80,
-	VS6624_MAN_BG, 0x80,
-	VS6624_WB_MODE, 0x1,
-	VS6624_EXPO_COMPENSATION, 0xfe,
-	VS6624_EXPO_METER, 0x0,
-	VS6624_LIGHT_FREQ, 0x64,
-	VS6624_PEAK_GAIN, 0xe,
-	VS6624_PEAK_LOW_THR, 0x28,
-	VS6624_HMIRROR0, 0x0,
-	VS6624_VFLIP0, 0x0,
-	VS6624_ZOOM_HSTEP0_MSB, 0x0,
-	VS6624_ZOOM_HSTEP0_LSB, 0x1,
-	VS6624_ZOOM_VSTEP0_MSB, 0x0,
-	VS6624_ZOOM_VSTEP0_LSB, 0x1,
-	VS6624_PAN_HSTEP0_MSB, 0x0,
-	VS6624_PAN_HSTEP0_LSB, 0xf,
-	VS6624_PAN_VSTEP0_MSB, 0x0,
-	VS6624_PAN_VSTEP0_LSB, 0xf,
-	VS6624_SENSOR_MODE, 0x1,
-	VS6624_SYNC_CODE_SETUP, 0x21,
-	VS6624_DISABLE_FR_DAMPER, 0x0,
-	VS6624_FR_DEN, 0x1,
-	VS6624_FR_NUM_LSB, 0xf,
-	VS6624_INIT_PIPE_SETUP, 0x0,
-	VS6624_IMG_FMT0, 0x0,
-	VS6624_YUV_SETUP, 0x1,
-	VS6624_IMAGE_SIZE0, 0x2,
-	0x0000, 0x00,
-};
-
-static inline struct vs6624 *to_vs6624(struct v4l2_subdev *sd)
-{
-	return container_of(sd, struct vs6624, sd);
-}
-static inline struct v4l2_subdev *to_sd(struct v4l2_ctrl *ctrl)
-{
-	return &container_of(ctrl->handler, struct vs6624, hdl)->sd;
-}
-
-#ifdef CONFIG_VIDEO_ADV_DEBUG
-static int vs6624_read(struct v4l2_subdev *sd, u16 index)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u8 buf[2];
-
-	buf[0] = index >> 8;
-	buf[1] = index;
-	i2c_master_send(client, buf, 2);
-	i2c_master_recv(client, buf, 1);
-
-	return buf[0];
-}
-#endif
-
-static int vs6624_write(struct v4l2_subdev *sd, u16 index,
-				u8 value)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u8 buf[3];
-
-	buf[0] = index >> 8;
-	buf[1] = index;
-	buf[2] = value;
-
-	return i2c_master_send(client, buf, 3);
-}
-
-static int vs6624_writeregs(struct v4l2_subdev *sd, const u16 *regs)
-{
-	u16 reg;
-	u8 data;
-
-	while (*regs != 0x00) {
-		reg = *regs++;
-		data = *regs++;
-
-		vs6624_write(sd, reg, data);
-	}
-	return 0;
-}
-
-static int vs6624_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct v4l2_subdev *sd = to_sd(ctrl);
-
-	switch (ctrl->id) {
-	case V4L2_CID_CONTRAST:
-		vs6624_write(sd, VS6624_CONTRAST0, ctrl->val);
-		break;
-	case V4L2_CID_SATURATION:
-		vs6624_write(sd, VS6624_SATURATION0, ctrl->val);
-		break;
-	case V4L2_CID_HFLIP:
-		vs6624_write(sd, VS6624_HMIRROR0, ctrl->val);
-		break;
-	case V4L2_CID_VFLIP:
-		vs6624_write(sd, VS6624_VFLIP0, ctrl->val);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int vs6624_enum_mbus_code(struct v4l2_subdev *sd,
-		struct v4l2_subdev_state *sd_state,
-		struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->pad || code->index >= ARRAY_SIZE(vs6624_formats))
-		return -EINVAL;
-
-	code->code = vs6624_formats[code->index].mbus_code;
-	return 0;
-}
-
-static int vs6624_set_fmt(struct v4l2_subdev *sd,
-		struct v4l2_subdev_state *sd_state,
-		struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct vs6624 *sensor = to_vs6624(sd);
-	int index;
-
-	if (format->pad)
-		return -EINVAL;
-
-	for (index = 0; index < ARRAY_SIZE(vs6624_formats); index++)
-		if (vs6624_formats[index].mbus_code == fmt->code)
-			break;
-	if (index >= ARRAY_SIZE(vs6624_formats)) {
-		/* default to first format */
-		index = 0;
-		fmt->code = vs6624_formats[0].mbus_code;
-	}
-
-	/* sensor mode is VGA */
-	if (fmt->width > VGA_WIDTH)
-		fmt->width = VGA_WIDTH;
-	if (fmt->height > VGA_HEIGHT)
-		fmt->height = VGA_HEIGHT;
-	fmt->width = fmt->width & (~3);
-	fmt->height = fmt->height & (~3);
-	fmt->field = V4L2_FIELD_NONE;
-	fmt->colorspace = vs6624_formats[index].colorspace;
-
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		sd_state->pads->try_fmt = *fmt;
-		return 0;
-	}
-
-	/* set image format */
-	switch (fmt->code) {
-	case MEDIA_BUS_FMT_UYVY8_2X8:
-		vs6624_write(sd, VS6624_IMG_FMT0, 0x0);
-		vs6624_write(sd, VS6624_YUV_SETUP, 0x1);
-		break;
-	case MEDIA_BUS_FMT_YUYV8_2X8:
-		vs6624_write(sd, VS6624_IMG_FMT0, 0x0);
-		vs6624_write(sd, VS6624_YUV_SETUP, 0x3);
-		break;
-	case MEDIA_BUS_FMT_RGB565_2X8_LE:
-		vs6624_write(sd, VS6624_IMG_FMT0, 0x4);
-		vs6624_write(sd, VS6624_RGB_SETUP, 0x0);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* set image size */
-	if ((fmt->width == VGA_WIDTH) && (fmt->height == VGA_HEIGHT))
-		vs6624_write(sd, VS6624_IMAGE_SIZE0, 0x2);
-	else if ((fmt->width == QVGA_WIDTH) && (fmt->height == QVGA_HEIGHT))
-		vs6624_write(sd, VS6624_IMAGE_SIZE0, 0x4);
-	else if ((fmt->width == QQVGA_WIDTH) && (fmt->height == QQVGA_HEIGHT))
-		vs6624_write(sd, VS6624_IMAGE_SIZE0, 0x6);
-	else if ((fmt->width == CIF_WIDTH) && (fmt->height == CIF_HEIGHT))
-		vs6624_write(sd, VS6624_IMAGE_SIZE0, 0x3);
-	else if ((fmt->width == QCIF_WIDTH) && (fmt->height == QCIF_HEIGHT))
-		vs6624_write(sd, VS6624_IMAGE_SIZE0, 0x5);
-	else if ((fmt->width == QQCIF_WIDTH) && (fmt->height == QQCIF_HEIGHT))
-		vs6624_write(sd, VS6624_IMAGE_SIZE0, 0x7);
-	else {
-		vs6624_write(sd, VS6624_IMAGE_SIZE0, 0x8);
-		vs6624_write(sd, VS6624_MAN_HSIZE0_MSB, fmt->width >> 8);
-		vs6624_write(sd, VS6624_MAN_HSIZE0_LSB, fmt->width & 0xFF);
-		vs6624_write(sd, VS6624_MAN_VSIZE0_MSB, fmt->height >> 8);
-		vs6624_write(sd, VS6624_MAN_VSIZE0_LSB, fmt->height & 0xFF);
-		vs6624_write(sd, VS6624_CROP_CTRL0, 0x1);
-	}
-
-	sensor->fmt = *fmt;
-
-	return 0;
-}
-
-static int vs6624_get_fmt(struct v4l2_subdev *sd,
-		struct v4l2_subdev_state *sd_state,
-		struct v4l2_subdev_format *format)
-{
-	struct vs6624 *sensor = to_vs6624(sd);
-
-	if (format->pad)
-		return -EINVAL;
-
-	format->format = sensor->fmt;
-	return 0;
-}
-
-static int vs6624_g_frame_interval(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_frame_interval *ival)
-{
-	struct vs6624 *sensor = to_vs6624(sd);
-
-	ival->interval.numerator = sensor->frame_rate.denominator;
-	ival->interval.denominator = sensor->frame_rate.numerator;
-	return 0;
-}
-
-static int vs6624_s_frame_interval(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_frame_interval *ival)
-{
-	struct vs6624 *sensor = to_vs6624(sd);
-	struct v4l2_fract *tpf = &ival->interval;
-
-
-	if (tpf->numerator == 0 || tpf->denominator == 0
-		|| (tpf->denominator > tpf->numerator * MAX_FRAME_RATE)) {
-		/* reset to max frame rate */
-		tpf->numerator = 1;
-		tpf->denominator = MAX_FRAME_RATE;
-	}
-	sensor->frame_rate.numerator = tpf->denominator;
-	sensor->frame_rate.denominator = tpf->numerator;
-	vs6624_write(sd, VS6624_DISABLE_FR_DAMPER, 0x0);
-	vs6624_write(sd, VS6624_FR_NUM_MSB,
-			sensor->frame_rate.numerator >> 8);
-	vs6624_write(sd, VS6624_FR_NUM_LSB,
-			sensor->frame_rate.numerator & 0xFF);
-	vs6624_write(sd, VS6624_FR_DEN,
-			sensor->frame_rate.denominator & 0xFF);
-	return 0;
-}
-
-static int vs6624_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	if (enable)
-		vs6624_write(sd, VS6624_USER_CMD, 0x2);
-	else
-		vs6624_write(sd, VS6624_USER_CMD, 0x4);
-	udelay(100);
-	return 0;
-}
-
-#ifdef CONFIG_VIDEO_ADV_DEBUG
-static int vs6624_g_register(struct v4l2_subdev *sd, struct v4l2_dbg_register *reg)
-{
-	reg->val = vs6624_read(sd, reg->reg & 0xffff);
-	reg->size = 1;
-	return 0;
-}
-
-static int vs6624_s_register(struct v4l2_subdev *sd, const struct v4l2_dbg_register *reg)
-{
-	vs6624_write(sd, reg->reg & 0xffff, reg->val & 0xff);
-	return 0;
-}
-#endif
-
-static const struct v4l2_ctrl_ops vs6624_ctrl_ops = {
-	.s_ctrl = vs6624_s_ctrl,
-};
-
-static const struct v4l2_subdev_core_ops vs6624_core_ops = {
-#ifdef CONFIG_VIDEO_ADV_DEBUG
-	.g_register = vs6624_g_register,
-	.s_register = vs6624_s_register,
-#endif
-};
-
-static const struct v4l2_subdev_video_ops vs6624_video_ops = {
-	.s_frame_interval = vs6624_s_frame_interval,
-	.g_frame_interval = vs6624_g_frame_interval,
-	.s_stream = vs6624_s_stream,
-};
-
-static const struct v4l2_subdev_pad_ops vs6624_pad_ops = {
-	.enum_mbus_code = vs6624_enum_mbus_code,
-	.get_fmt = vs6624_get_fmt,
-	.set_fmt = vs6624_set_fmt,
-};
-
-static const struct v4l2_subdev_ops vs6624_ops = {
-	.core = &vs6624_core_ops,
-	.video = &vs6624_video_ops,
-	.pad = &vs6624_pad_ops,
-};
-
-static int vs6624_probe(struct i2c_client *client)
-{
-	struct vs6624 *sensor;
-	struct v4l2_subdev *sd;
-	struct v4l2_ctrl_handler *hdl;
-	const unsigned *ce;
-	int ret;
-
-	/* Check if the adapter supports the needed features */
-	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
-		return -EIO;
-
-	ce = client->dev.platform_data;
-	if (ce == NULL)
-		return -EINVAL;
-
-	ret = devm_gpio_request_one(&client->dev, *ce, GPIOF_OUT_INIT_HIGH,
-				    "VS6624 Chip Enable");
-	if (ret) {
-		v4l_err(client, "failed to request GPIO %d\n", *ce);
-		return ret;
-	}
-	/* wait 100ms before any further i2c writes are performed */
-	msleep(100);
-
-	sensor = devm_kzalloc(&client->dev, sizeof(*sensor), GFP_KERNEL);
-	if (sensor == NULL)
-		return -ENOMEM;
-
-	sd = &sensor->sd;
-	v4l2_i2c_subdev_init(sd, client, &vs6624_ops);
-
-	vs6624_writeregs(sd, vs6624_p1);
-	vs6624_write(sd, VS6624_MICRO_EN, 0x2);
-	vs6624_write(sd, VS6624_DIO_EN, 0x1);
-	usleep_range(10000, 11000);
-	vs6624_writeregs(sd, vs6624_p2);
-
-	vs6624_writeregs(sd, vs6624_default);
-	vs6624_write(sd, VS6624_HSYNC_SETUP, 0xF);
-	vs6624_writeregs(sd, vs6624_run_setup);
-
-	/* set frame rate */
-	sensor->frame_rate.numerator = MAX_FRAME_RATE;
-	sensor->frame_rate.denominator = 1;
-	vs6624_write(sd, VS6624_DISABLE_FR_DAMPER, 0x0);
-	vs6624_write(sd, VS6624_FR_NUM_MSB,
-			sensor->frame_rate.numerator >> 8);
-	vs6624_write(sd, VS6624_FR_NUM_LSB,
-			sensor->frame_rate.numerator & 0xFF);
-	vs6624_write(sd, VS6624_FR_DEN,
-			sensor->frame_rate.denominator & 0xFF);
-
-	sensor->fmt = vs6624_default_fmt;
-	sensor->ce_pin = *ce;
-
-	v4l_info(client, "chip found @ 0x%02x (%s)\n",
-			client->addr << 1, client->adapter->name);
-
-	hdl = &sensor->hdl;
-	v4l2_ctrl_handler_init(hdl, 4);
-	v4l2_ctrl_new_std(hdl, &vs6624_ctrl_ops,
-			V4L2_CID_CONTRAST, 0, 0xFF, 1, 0x87);
-	v4l2_ctrl_new_std(hdl, &vs6624_ctrl_ops,
-			V4L2_CID_SATURATION, 0, 0xFF, 1, 0x78);
-	v4l2_ctrl_new_std(hdl, &vs6624_ctrl_ops,
-			V4L2_CID_HFLIP, 0, 1, 1, 0);
-	v4l2_ctrl_new_std(hdl, &vs6624_ctrl_ops,
-			V4L2_CID_VFLIP, 0, 1, 1, 0);
-	/* hook the control handler into the driver */
-	sd->ctrl_handler = hdl;
-	if (hdl->error) {
-		int err = hdl->error;
-
-		v4l2_ctrl_handler_free(hdl);
-		return err;
-	}
-
-	/* initialize the hardware to the default control values */
-	ret = v4l2_ctrl_handler_setup(hdl);
-	if (ret)
-		v4l2_ctrl_handler_free(hdl);
-	return ret;
-}
-
-static void vs6624_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-
-	v4l2_device_unregister_subdev(sd);
-	v4l2_ctrl_handler_free(sd->ctrl_handler);
-}
-
-static const struct i2c_device_id vs6624_id[] = {
-	{"vs6624", 0},
-	{},
-};
-
-MODULE_DEVICE_TABLE(i2c, vs6624_id);
-
-static struct i2c_driver vs6624_driver = {
-	.driver = {
-		.name   = "vs6624",
-	},
-	.probe_new      = vs6624_probe,
-	.remove         = vs6624_remove,
-	.id_table       = vs6624_id,
-};
-
-module_i2c_driver(vs6624_driver);
-
-MODULE_DESCRIPTION("VS6624 sensor driver");
-MODULE_AUTHOR("Scott Jiang <Scott.Jiang.Linux@gmail.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/i2c/vs6624_regs.h b/drivers/media/i2c/vs6624_regs.h
deleted file mode 100644
index 76c9ed0f2c89..000000000000
--- a/drivers/media/i2c/vs6624_regs.h
+++ /dev/null
@@ -1,325 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * vs6624 - ST VS6624 CMOS image sensor registers
- *
- * Copyright (c) 2011 Analog Devices Inc.
- */
-
-#ifndef _VS6624_REGS_H_
-#define _VS6624_REGS_H_
-
-/* low level control registers */
-#define VS6624_MICRO_EN               0xC003 /* power enable for all MCU clock */
-#define VS6624_DIO_EN                 0xC044 /* enable digital I/O */
-/* device parameters */
-#define VS6624_DEV_ID_MSB             0x0001 /* device id MSB */
-#define VS6624_DEV_ID_LSB             0x0002 /* device id LSB */
-#define VS6624_FW_VSN_MAJOR           0x0004 /* firmware version major */
-#define VS6624_FW_VSN_MINOR           0x0006 /* firmware version minor */
-#define VS6624_PATCH_VSN_MAJOR        0x0008 /* patch version major */
-#define VS6624_PATCH_VSN_MINOR        0x000A /* patch version minor */
-/* host interface manager control */
-#define VS6624_USER_CMD               0x0180 /* user level control of operating states */
-/* host interface manager status */
-#define VS6624_STATE                  0x0202 /* current state of the mode manager */
-/* run mode control */
-#define VS6624_METER_ON               0x0280 /* if false AE and AWB are disabled */
-/* mode setup */
-#define VS6624_ACTIVE_PIPE_SETUP      0x0302 /* select the active bank for non view live mode */
-#define VS6624_SENSOR_MODE            0x0308 /* select the different sensor mode */
-/* pipe setup bank0 */
-#define VS6624_IMAGE_SIZE0            0x0380 /* required output dimension */
-#define VS6624_MAN_HSIZE0_MSB         0x0383 /* input required manual H size MSB */
-#define VS6624_MAN_HSIZE0_LSB         0x0384 /* input required manual H size LSB */
-#define VS6624_MAN_VSIZE0_MSB         0x0387 /* input required manual V size MSB */
-#define VS6624_MAN_VSIZE0_LSB         0x0388 /* input required manual V size LSB */
-#define VS6624_ZOOM_HSTEP0_MSB        0x038B /* set the zoom H step MSB */
-#define VS6624_ZOOM_HSTEP0_LSB        0x038C /* set the zoom H step LSB */
-#define VS6624_ZOOM_VSTEP0_MSB        0x038F /* set the zoom V step MSB */
-#define VS6624_ZOOM_VSTEP0_LSB        0x0390 /* set the zoom V step LSB */
-#define VS6624_ZOOM_CTRL0             0x0392 /* control zoon in, out and stop */
-#define VS6624_PAN_HSTEP0_MSB         0x0395 /* set the pan H step MSB */
-#define VS6624_PAN_HSTEP0_LSB         0x0396 /* set the pan H step LSB */
-#define VS6624_PAN_VSTEP0_MSB         0x0399 /* set the pan V step MSB */
-#define VS6624_PAN_VSTEP0_LSB         0x039A /* set the pan V step LSB */
-#define VS6624_PAN_CTRL0              0x039C /* control pan operation */
-#define VS6624_CROP_CTRL0             0x039E /* select cropping mode */
-#define VS6624_CROP_HSTART0_MSB       0x03A1 /* set the cropping H start address MSB */
-#define VS6624_CROP_HSTART0_LSB       0x03A2 /* set the cropping H start address LSB */
-#define VS6624_CROP_HSIZE0_MSB        0x03A5 /* set the cropping H size MSB */
-#define VS6624_CROP_HSIZE0_LSB        0x03A6 /* set the cropping H size LSB */
-#define VS6624_CROP_VSTART0_MSB       0x03A9 /* set the cropping V start address MSB */
-#define VS6624_CROP_VSTART0_LSB       0x03AA /* set the cropping V start address LSB */
-#define VS6624_CROP_VSIZE0_MSB        0x03AD /* set the cropping V size MSB */
-#define VS6624_CROP_VSIZE0_LSB        0x03AE /* set the cropping V size LSB */
-#define VS6624_IMG_FMT0               0x03B0 /* select required output image format */
-#define VS6624_BAYER_OUT_ALIGN0       0x03B2 /* set bayer output alignment */
-#define VS6624_CONTRAST0              0x03B4 /* contrast control for output */
-#define VS6624_SATURATION0            0x03B6 /* saturation control for output */
-#define VS6624_GAMMA0                 0x03B8 /* gamma settings */
-#define VS6624_HMIRROR0               0x03BA /* horizontal image orientation flip */
-#define VS6624_VFLIP0                 0x03BC /* vertical image orientation flip */
-#define VS6624_CHANNEL_ID0            0x03BE /* logical DMA channel number */
-/* pipe setup bank1 */
-#define VS6624_IMAGE_SIZE1            0x0400 /* required output dimension */
-#define VS6624_MAN_HSIZE1_MSB         0x0403 /* input required manual H size MSB */
-#define VS6624_MAN_HSIZE1_LSB         0x0404 /* input required manual H size LSB */
-#define VS6624_MAN_VSIZE1_MSB         0x0407 /* input required manual V size MSB */
-#define VS6624_MAN_VSIZE1_LSB         0x0408 /* input required manual V size LSB */
-#define VS6624_ZOOM_HSTEP1_MSB        0x040B /* set the zoom H step MSB */
-#define VS6624_ZOOM_HSTEP1_LSB        0x040C /* set the zoom H step LSB */
-#define VS6624_ZOOM_VSTEP1_MSB        0x040F /* set the zoom V step MSB */
-#define VS6624_ZOOM_VSTEP1_LSB        0x0410 /* set the zoom V step LSB */
-#define VS6624_ZOOM_CTRL1             0x0412 /* control zoon in, out and stop */
-#define VS6624_PAN_HSTEP1_MSB         0x0415 /* set the pan H step MSB */
-#define VS6624_PAN_HSTEP1_LSB         0x0416 /* set the pan H step LSB */
-#define VS6624_PAN_VSTEP1_MSB         0x0419 /* set the pan V step MSB */
-#define VS6624_PAN_VSTEP1_LSB         0x041A /* set the pan V step LSB */
-#define VS6624_PAN_CTRL1              0x041C /* control pan operation */
-#define VS6624_CROP_CTRL1             0x041E /* select cropping mode */
-#define VS6624_CROP_HSTART1_MSB       0x0421 /* set the cropping H start address MSB */
-#define VS6624_CROP_HSTART1_LSB       0x0422 /* set the cropping H start address LSB */
-#define VS6624_CROP_HSIZE1_MSB        0x0425 /* set the cropping H size MSB */
-#define VS6624_CROP_HSIZE1_LSB        0x0426 /* set the cropping H size LSB */
-#define VS6624_CROP_VSTART1_MSB       0x0429 /* set the cropping V start address MSB */
-#define VS6624_CROP_VSTART1_LSB       0x042A /* set the cropping V start address LSB */
-#define VS6624_CROP_VSIZE1_MSB        0x042D /* set the cropping V size MSB */
-#define VS6624_CROP_VSIZE1_LSB        0x042E /* set the cropping V size LSB */
-#define VS6624_IMG_FMT1               0x0430 /* select required output image format */
-#define VS6624_BAYER_OUT_ALIGN1       0x0432 /* set bayer output alignment */
-#define VS6624_CONTRAST1              0x0434 /* contrast control for output */
-#define VS6624_SATURATION1            0x0436 /* saturation control for output */
-#define VS6624_GAMMA1                 0x0438 /* gamma settings */
-#define VS6624_HMIRROR1               0x043A /* horizontal image orientation flip */
-#define VS6624_VFLIP1                 0x043C /* vertical image orientation flip */
-#define VS6624_CHANNEL_ID1            0x043E /* logical DMA channel number */
-/* view live control */
-#define VS6624_VIEW_LIVE_EN           0x0480 /* enable view live mode */
-#define VS6624_INIT_PIPE_SETUP        0x0482 /* select initial pipe setup bank */
-/* view live status */
-#define VS6624_CUR_PIPE_SETUP         0x0500 /* indicates most recently applied setup bank */
-/* power management */
-#define VS6624_TIME_TO_POWER_DOWN     0x0580 /* automatically transition time to stop mode */
-/* video timing parameter host inputs */
-#define VS6624_EXT_CLK_FREQ_NUM_MSB   0x0605 /* external clock frequency numerator MSB */
-#define VS6624_EXT_CLK_FREQ_NUM_LSB   0x0606 /* external clock frequency numerator LSB */
-#define VS6624_EXT_CLK_FREQ_DEN       0x0608 /* external clock frequency denominator */
-/* video timing control */
-#define VS6624_SYS_CLK_MODE           0x0880 /* decides system clock frequency */
-/* frame dimension parameter host inputs */
-#define VS6624_LIGHT_FREQ             0x0C80 /* AC frequency used for flicker free time */
-#define VS6624_FLICKER_COMPAT         0x0C82 /* flicker compatible frame length */
-/* static frame rate control */
-#define VS6624_FR_NUM_MSB             0x0D81 /* desired frame rate numerator MSB */
-#define VS6624_FR_NUM_LSB             0x0D82 /* desired frame rate numerator LSB */
-#define VS6624_FR_DEN                 0x0D84 /* desired frame rate denominator */
-/* automatic frame rate control */
-#define VS6624_DISABLE_FR_DAMPER      0x0E80 /* defines frame rate mode */
-#define VS6624_MIN_DAMPER_OUT_MSB     0x0E8C /* minimum frame rate MSB */
-#define VS6624_MIN_DAMPER_OUT_LSB     0x0E8A /* minimum frame rate LSB */
-/* exposure controls */
-#define VS6624_EXPO_MODE              0x1180 /* exposure mode */
-#define VS6624_EXPO_METER             0x1182 /* weights to be associated with the zones */
-#define VS6624_EXPO_TIME_NUM          0x1184 /* exposure time numerator */
-#define VS6624_EXPO_TIME_DEN          0x1186 /* exposure time denominator */
-#define VS6624_EXPO_TIME_MSB          0x1189 /* exposure time for the Manual Mode MSB */
-#define VS6624_EXPO_TIME_LSB          0x118A /* exposure time for the Manual Mode LSB */
-#define VS6624_EXPO_COMPENSATION      0x1190 /* exposure compensation */
-#define VS6624_DIRECT_COARSE_MSB      0x1195 /* coarse integration lines for Direct Mode MSB */
-#define VS6624_DIRECT_COARSE_LSB      0x1196 /* coarse integration lines for Direct Mode LSB */
-#define VS6624_DIRECT_FINE_MSB        0x1199 /* fine integration pixels for Direct Mode MSB */
-#define VS6624_DIRECT_FINE_LSB        0x119A /* fine integration pixels for Direct Mode LSB */
-#define VS6624_DIRECT_ANAL_GAIN_MSB   0x119D /* analog gain for Direct Mode MSB */
-#define VS6624_DIRECT_ANAL_GAIN_LSB   0x119E /* analog gain for Direct Mode LSB */
-#define VS6624_DIRECT_DIGI_GAIN_MSB   0x11A1 /* digital gain for Direct Mode MSB */
-#define VS6624_DIRECT_DIGI_GAIN_LSB   0x11A2 /* digital gain for Direct Mode LSB */
-#define VS6624_FLASH_COARSE_MSB       0x11A5 /* coarse integration lines for Flash Gun Mode MSB */
-#define VS6624_FLASH_COARSE_LSB       0x11A6 /* coarse integration lines for Flash Gun Mode LSB */
-#define VS6624_FLASH_FINE_MSB         0x11A9 /* fine integration pixels for Flash Gun Mode MSB */
-#define VS6624_FLASH_FINE_LSB         0x11AA /* fine integration pixels for Flash Gun Mode LSB */
-#define VS6624_FLASH_ANAL_GAIN_MSB    0x11AD /* analog gain for Flash Gun Mode MSB */
-#define VS6624_FLASH_ANAL_GAIN_LSB    0x11AE /* analog gain for Flash Gun Mode LSB */
-#define VS6624_FLASH_DIGI_GAIN_MSB    0x11B1 /* digital gain for Flash Gun Mode MSB */
-#define VS6624_FLASH_DIGI_GAIN_LSB    0x11B2 /* digital gain for Flash Gun Mode LSB */
-#define VS6624_FREEZE_AE              0x11B4 /* freeze auto exposure */
-#define VS6624_MAX_INT_TIME_MSB       0x11B7 /* user maximum integration time MSB */
-#define VS6624_MAX_INT_TIME_LSB       0x11B8 /* user maximum integration time LSB */
-#define VS6624_FLASH_AG_THR_MSB       0x11BB /* recommend flash gun analog gain threshold MSB */
-#define VS6624_FLASH_AG_THR_LSB       0x11BC /* recommend flash gun analog gain threshold LSB */
-#define VS6624_ANTI_FLICKER_MODE      0x11C0 /* anti flicker mode */
-/* white balance control */
-#define VS6624_WB_MODE                0x1480 /* set white balance mode */
-#define VS6624_MAN_RG                 0x1482 /* user setting for red channel gain */
-#define VS6624_MAN_GG                 0x1484 /* user setting for green channel gain */
-#define VS6624_MAN_BG                 0x1486 /* user setting for blue channel gain */
-#define VS6624_FLASH_RG_MSB           0x148B /* red gain for Flash Gun MSB */
-#define VS6624_FLASH_RG_LSB           0x148C /* red gain for Flash Gun LSB */
-#define VS6624_FLASH_GG_MSB           0x148F /* green gain for Flash Gun MSB */
-#define VS6624_FLASH_GG_LSB           0x1490 /* green gain for Flash Gun LSB */
-#define VS6624_FLASH_BG_MSB           0x1493 /* blue gain for Flash Gun MSB */
-#define VS6624_FLASH_BG_LSB           0x1494 /* blue gain for Flash Gun LSB */
-/* sensor setup */
-#define VS6624_BC_OFFSET              0x1990 /* Black Correction Offset */
-/* image stability */
-#define VS6624_STABLE_WB              0x1900 /* white balance stable */
-#define VS6624_STABLE_EXPO            0x1902 /* exposure stable */
-#define VS6624_STABLE                 0x1906 /* system stable */
-/* flash control */
-#define VS6624_FLASH_MODE             0x1A80 /* flash mode */
-#define VS6624_FLASH_OFF_LINE_MSB     0x1A83 /* off line at flash pulse mode MSB */
-#define VS6624_FLASH_OFF_LINE_LSB     0x1A84 /* off line at flash pulse mode LSB */
-/* flash status */
-#define VS6624_FLASH_RECOM            0x1B00 /* flash gun is recommended */
-#define VS6624_FLASH_GRAB_COMPLETE    0x1B02 /* flash gun image has been grabbed */
-/* scythe filter controls */
-#define VS6624_SCYTHE_FILTER          0x1D80 /* disable scythe defect correction */
-/* jack filter controls */
-#define VS6624_JACK_FILTER            0x1E00 /* disable jack defect correction */
-/* demosaic control */
-#define VS6624_ANTI_ALIAS_FILTER      0x1E80 /* anti alias filter suppress */
-/* color matrix dampers */
-#define VS6624_CM_DISABLE             0x1F00 /* disable color matrix damper */
-#define VS6624_CM_LOW_THR_MSB         0x1F03 /* low threshold for exposure MSB */
-#define VS6624_CM_LOW_THR_LSB         0x1F04 /* low threshold for exposure LSB */
-#define VS6624_CM_HIGH_THR_MSB        0x1F07 /* high threshold for exposure MSB */
-#define VS6624_CM_HIGH_THR_LSB        0x1F08 /* high threshold for exposure LSB */
-#define VS6624_CM_MIN_OUT_MSB         0x1F0B /* minimum possible damper output MSB */
-#define VS6624_CM_MIN_OUT_LSB         0x1F0C /* minimum possible damper output LSB */
-/* peaking control */
-#define VS6624_PEAK_GAIN              0x2000 /* controls peaking gain */
-#define VS6624_PEAK_G_DISABLE         0x2002 /* disable peak gain damping */
-#define VS6624_PEAK_LOW_THR_G_MSB     0x2005 /* low threshold for exposure for gain MSB */
-#define VS6624_PEAK_LOW_THR_G_LSB     0x2006 /* low threshold for exposure for gain LSB */
-#define VS6624_PEAK_HIGH_THR_G_MSB    0x2009 /* high threshold for exposure for gain MSB */
-#define VS6624_PEAK_HIGH_THR_G_LSB    0x200A /* high threshold for exposure for gain LSB */
-#define VS6624_PEAK_MIN_OUT_G_MSB     0x200D /* minimum damper output for gain MSB */
-#define VS6624_PEAK_MIN_OUT_G_LSB     0x200E /* minimum damper output for gain LSB */
-#define VS6624_PEAK_LOW_THR           0x2010 /* adjust degree of coring */
-#define VS6624_PEAK_C_DISABLE         0x2012 /* disable coring damping */
-#define VS6624_PEAK_HIGH_THR          0x2014 /* adjust maximum gain */
-#define VS6624_PEAK_LOW_THR_C_MSB     0x2017 /* low threshold for exposure for coring MSB */
-#define VS6624_PEAK_LOW_THR_C_LSB     0x2018 /* low threshold for exposure for coring LSB */
-#define VS6624_PEAK_HIGH_THR_C_MSB    0x201B /* high threshold for exposure for coring MSB */
-#define VS6624_PEAK_HIGH_THR_C_LSB    0x201C /* high threshold for exposure for coring LSB */
-#define VS6624_PEAK_MIN_OUT_C_MSB     0x201F /* minimum damper output for coring MSB */
-#define VS6624_PEAK_MIN_OUT_C_LSB     0x2020 /* minimum damper output for coring LSB */
-/* pipe 0 RGB to YUV matrix manual control */
-#define VS6624_RYM0_MAN_CTRL          0x2180 /* enable manual RGB to YUV matrix */
-#define VS6624_RYM0_W00_MSB           0x2183 /* row 0 column 0 of YUV matrix MSB */
-#define VS6624_RYM0_W00_LSB           0x2184 /* row 0 column 0 of YUV matrix LSB */
-#define VS6624_RYM0_W01_MSB           0x2187 /* row 0 column 1 of YUV matrix MSB */
-#define VS6624_RYM0_W01_LSB           0x2188 /* row 0 column 1 of YUV matrix LSB */
-#define VS6624_RYM0_W02_MSB           0x218C /* row 0 column 2 of YUV matrix MSB */
-#define VS6624_RYM0_W02_LSB           0x218D /* row 0 column 2 of YUV matrix LSB */
-#define VS6624_RYM0_W10_MSB           0x2190 /* row 1 column 0 of YUV matrix MSB */
-#define VS6624_RYM0_W10_LSB           0x218F /* row 1 column 0 of YUV matrix LSB */
-#define VS6624_RYM0_W11_MSB           0x2193 /* row 1 column 1 of YUV matrix MSB */
-#define VS6624_RYM0_W11_LSB           0x2194 /* row 1 column 1 of YUV matrix LSB */
-#define VS6624_RYM0_W12_MSB           0x2197 /* row 1 column 2 of YUV matrix MSB */
-#define VS6624_RYM0_W12_LSB           0x2198 /* row 1 column 2 of YUV matrix LSB */
-#define VS6624_RYM0_W20_MSB           0x219B /* row 2 column 0 of YUV matrix MSB */
-#define VS6624_RYM0_W20_LSB           0x219C /* row 2 column 0 of YUV matrix LSB */
-#define VS6624_RYM0_W21_MSB           0x21A0 /* row 2 column 1 of YUV matrix MSB */
-#define VS6624_RYM0_W21_LSB           0x219F /* row 2 column 1 of YUV matrix LSB */
-#define VS6624_RYM0_W22_MSB           0x21A3 /* row 2 column 2 of YUV matrix MSB */
-#define VS6624_RYM0_W22_LSB           0x21A4 /* row 2 column 2 of YUV matrix LSB */
-#define VS6624_RYM0_YINY_MSB          0x21A7 /* Y in Y MSB */
-#define VS6624_RYM0_YINY_LSB          0x21A8 /* Y in Y LSB */
-#define VS6624_RYM0_YINCB_MSB         0x21AB /* Y in Cb MSB */
-#define VS6624_RYM0_YINCB_LSB         0x21AC /* Y in Cb LSB */
-#define VS6624_RYM0_YINCR_MSB         0x21B0 /* Y in Cr MSB */
-#define VS6624_RYM0_YINCR_LSB         0x21AF /* Y in Cr LSB */
-/* pipe 1 RGB to YUV matrix manual control */
-#define VS6624_RYM1_MAN_CTRL          0x2200 /* enable manual RGB to YUV matrix */
-#define VS6624_RYM1_W00_MSB           0x2203 /* row 0 column 0 of YUV matrix MSB */
-#define VS6624_RYM1_W00_LSB           0x2204 /* row 0 column 0 of YUV matrix LSB */
-#define VS6624_RYM1_W01_MSB           0x2207 /* row 0 column 1 of YUV matrix MSB */
-#define VS6624_RYM1_W01_LSB           0x2208 /* row 0 column 1 of YUV matrix LSB */
-#define VS6624_RYM1_W02_MSB           0x220C /* row 0 column 2 of YUV matrix MSB */
-#define VS6624_RYM1_W02_LSB           0x220D /* row 0 column 2 of YUV matrix LSB */
-#define VS6624_RYM1_W10_MSB           0x2210 /* row 1 column 0 of YUV matrix MSB */
-#define VS6624_RYM1_W10_LSB           0x220F /* row 1 column 0 of YUV matrix LSB */
-#define VS6624_RYM1_W11_MSB           0x2213 /* row 1 column 1 of YUV matrix MSB */
-#define VS6624_RYM1_W11_LSB           0x2214 /* row 1 column 1 of YUV matrix LSB */
-#define VS6624_RYM1_W12_MSB           0x2217 /* row 1 column 2 of YUV matrix MSB */
-#define VS6624_RYM1_W12_LSB           0x2218 /* row 1 column 2 of YUV matrix LSB */
-#define VS6624_RYM1_W20_MSB           0x221B /* row 2 column 0 of YUV matrix MSB */
-#define VS6624_RYM1_W20_LSB           0x221C /* row 2 column 0 of YUV matrix LSB */
-#define VS6624_RYM1_W21_MSB           0x2220 /* row 2 column 1 of YUV matrix MSB */
-#define VS6624_RYM1_W21_LSB           0x221F /* row 2 column 1 of YUV matrix LSB */
-#define VS6624_RYM1_W22_MSB           0x2223 /* row 2 column 2 of YUV matrix MSB */
-#define VS6624_RYM1_W22_LSB           0x2224 /* row 2 column 2 of YUV matrix LSB */
-#define VS6624_RYM1_YINY_MSB          0x2227 /* Y in Y MSB */
-#define VS6624_RYM1_YINY_LSB          0x2228 /* Y in Y LSB */
-#define VS6624_RYM1_YINCB_MSB         0x222B /* Y in Cb MSB */
-#define VS6624_RYM1_YINCB_LSB         0x222C /* Y in Cb LSB */
-#define VS6624_RYM1_YINCR_MSB         0x2220 /* Y in Cr MSB */
-#define VS6624_RYM1_YINCR_LSB         0x222F /* Y in Cr LSB */
-/* pipe 0 gamma manual control */
-#define VS6624_GAMMA_MAN_CTRL0        0x2280 /* enable manual gamma setup */
-#define VS6624_GAMMA_PEAK_R0          0x2282 /* peaked red channel gamma value */
-#define VS6624_GAMMA_PEAK_G0          0x2284 /* peaked green channel gamma value */
-#define VS6624_GAMMA_PEAK_B0          0x2286 /* peaked blue channel gamma value */
-#define VS6624_GAMMA_UNPEAK_R0        0x2288 /* unpeaked red channel gamma value */
-#define VS6624_GAMMA_UNPEAK_G0        0x228A /* unpeaked green channel gamma value */
-#define VS6624_GAMMA_UNPEAK_B0        0x228C /* unpeaked blue channel gamma value */
-/* pipe 1 gamma manual control */
-#define VS6624_GAMMA_MAN_CTRL1        0x2300 /* enable manual gamma setup */
-#define VS6624_GAMMA_PEAK_R1          0x2302 /* peaked red channel gamma value */
-#define VS6624_GAMMA_PEAK_G1          0x2304 /* peaked green channel gamma value */
-#define VS6624_GAMMA_PEAK_B1          0x2306 /* peaked blue channel gamma value */
-#define VS6624_GAMMA_UNPEAK_R1        0x2308 /* unpeaked red channel gamma value */
-#define VS6624_GAMMA_UNPEAK_G1        0x230A /* unpeaked green channel gamma value */
-#define VS6624_GAMMA_UNPEAK_B1        0x230C /* unpeaked blue channel gamma value */
-/* fade to black */
-#define VS6624_F2B_DISABLE            0x2480 /* disable fade to black */
-#define VS6624_F2B_BLACK_VAL_MSB      0x2483 /* black value MSB */
-#define VS6624_F2B_BLACK_VAL_LSB      0x2484 /* black value LSB */
-#define VS6624_F2B_LOW_THR_MSB        0x2487 /* low threshold for exposure MSB */
-#define VS6624_F2B_LOW_THR_LSB        0x2488 /* low threshold for exposure LSB */
-#define VS6624_F2B_HIGH_THR_MSB       0x248B /* high threshold for exposure MSB */
-#define VS6624_F2B_HIGH_THR_LSB       0x248C /* high threshold for exposure LSB */
-#define VS6624_F2B_MIN_OUT_MSB        0x248F /* minimum damper output MSB */
-#define VS6624_F2B_MIN_OUT_LSB        0x2490 /* minimum damper output LSB */
-/* output formatter control */
-#define VS6624_CODE_CK_EN             0x2580 /* code check enable */
-#define VS6624_BLANK_FMT              0x2582 /* blank format */
-#define VS6624_SYNC_CODE_SETUP        0x2584 /* sync code setup */
-#define VS6624_HSYNC_SETUP            0x2586 /* H sync setup */
-#define VS6624_VSYNC_SETUP            0x2588 /* V sync setup */
-#define VS6624_PCLK_SETUP             0x258A /* PCLK setup */
-#define VS6624_PCLK_EN                0x258C /* PCLK enable */
-#define VS6624_OPF_SP_SETUP           0x258E /* output formatter sp setup */
-#define VS6624_BLANK_DATA_MSB         0x2590 /* blank data MSB */
-#define VS6624_BLANK_DATA_LSB         0x2592 /* blank data LSB */
-#define VS6624_RGB_SETUP              0x2594 /* RGB setup */
-#define VS6624_YUV_SETUP              0x2596 /* YUV setup */
-#define VS6624_VSYNC_RIS_COARSE_H     0x2598 /* V sync rising coarse high */
-#define VS6624_VSYNC_RIS_COARSE_L     0x259A /* V sync rising coarse low */
-#define VS6624_VSYNC_RIS_FINE_H       0x259C /* V sync rising fine high */
-#define VS6624_VSYNC_RIS_FINE_L       0x259E /* V sync rising fine low */
-#define VS6624_VSYNC_FALL_COARSE_H    0x25A0 /* V sync falling coarse high */
-#define VS6624_VSYNC_FALL_COARSE_L    0x25A2 /* V sync falling coarse low */
-#define VS6624_VSYNC_FALL_FINE_H      0x25A4 /* V sync falling fine high */
-#define VS6624_VSYNC_FALL_FINE_L      0x25A6 /* V sync falling fine low */
-#define VS6624_HSYNC_RIS_H            0x25A8 /* H sync rising high */
-#define VS6624_HSYNC_RIS_L            0x25AA /* H sync rising low */
-#define VS6624_HSYNC_FALL_H           0x25AC /* H sync falling high */
-#define VS6624_HSYNC_FALL_L           0x25AE /* H sync falling low */
-#define VS6624_OUT_IF                 0x25B0 /* output interface */
-#define VS6624_CCP_EXT_DATA           0x25B2 /* CCP extra data */
-/* NoRA controls */
-#define VS6624_NORA_DISABLE           0x2600 /* NoRA control mode */
-#define VS6624_NORA_USAGE             0x2602 /* usage */
-#define VS6624_NORA_SPLIT_KN          0x2604 /* split kn */
-#define VS6624_NORA_SPLIT_NI          0x2606 /* split ni */
-#define VS6624_NORA_TIGHT_G           0x2608 /* tight green */
-#define VS6624_NORA_DISABLE_NP        0x260A /* disable noro promoting */
-#define VS6624_NORA_LOW_THR_MSB       0x260D /* low threshold for exposure MSB */
-#define VS6624_NORA_LOW_THR_LSB       0x260E /* low threshold for exposure LSB */
-#define VS6624_NORA_HIGH_THR_MSB      0x2611 /* high threshold for exposure MSB */
-#define VS6624_NORA_HIGH_THR_LSB      0x2612 /* high threshold for exposure LSB */
-#define VS6624_NORA_MIN_OUT_MSB       0x2615 /* minimum damper output MSB */
-#define VS6624_NORA_MIN_OUT_LSB       0x2616 /* minimum damper output LSB */
-
-#endif
diff --git a/drivers/media/mc/mc-device.c b/drivers/media/mc/mc-device.c
index 25020d58eb06..8cee956e38d4 100644
--- a/drivers/media/mc/mc-device.c
+++ b/drivers/media/mc/mc-device.c
@@ -756,13 +756,12 @@ int __must_check __media_device_register(struct media_device *mdev,
 }
 EXPORT_SYMBOL_GPL(__media_device_register);
 
-int __must_check media_device_register_entity_notify(struct media_device *mdev,
+void media_device_register_entity_notify(struct media_device *mdev,
 					struct media_entity_notify *nptr)
 {
 	mutex_lock(&mdev->graph_mutex);
 	list_add_tail(&nptr->list, &mdev->entity_notify);
 	mutex_unlock(&mdev->graph_mutex);
-	return 0;
 }
 EXPORT_SYMBOL_GPL(media_device_register_entity_notify);
 
diff --git a/drivers/media/pci/bt8xx/Kconfig b/drivers/media/pci/bt8xx/Kconfig
index 927190281bd5..2d674dc28cec 100644
--- a/drivers/media/pci/bt8xx/Kconfig
+++ b/drivers/media/pci/bt8xx/Kconfig
@@ -15,7 +15,7 @@ config VIDEO_BT848
 	select RADIO_ADAPTERS
 	select RADIO_TEA575X
 	help
-	  Support for BT848 based frame grabber/overlay boards. This includes
+	  Support for BT848 based frame grabber boards. This includes
 	  the Miro, Hauppauge and STB boards. Please read the material in
 	  <file:Documentation/admin-guide/media/bttv.rst> for more information.
 
diff --git a/drivers/media/pci/bt8xx/btcx-risc.c b/drivers/media/pci/bt8xx/btcx-risc.c
index b3179038b900..0adbf8233e1a 100644
--- a/drivers/media/pci/bt8xx/btcx-risc.c
+++ b/drivers/media/pci/bt8xx/btcx-risc.c
@@ -75,156 +75,3 @@ int btcx_riscmem_alloc(struct pci_dev *pci,
 	}
 	return 0;
 }
-
-/* ---------------------------------------------------------- */
-/* screen overlay helpers                                     */
-
-int
-btcx_screen_clips(int swidth, int sheight, struct v4l2_rect *win,
-		  struct v4l2_clip *clips, unsigned int n)
-{
-	if (win->left < 0) {
-		/* left */
-		clips[n].c.left = 0;
-		clips[n].c.top = 0;
-		clips[n].c.width  = -win->left;
-		clips[n].c.height = win->height;
-		n++;
-	}
-	if (win->left + win->width > swidth) {
-		/* right */
-		clips[n].c.left   = swidth - win->left;
-		clips[n].c.top    = 0;
-		clips[n].c.width  = win->width - clips[n].c.left;
-		clips[n].c.height = win->height;
-		n++;
-	}
-	if (win->top < 0) {
-		/* top */
-		clips[n].c.left = 0;
-		clips[n].c.top = 0;
-		clips[n].c.width  = win->width;
-		clips[n].c.height = -win->top;
-		n++;
-	}
-	if (win->top + win->height > sheight) {
-		/* bottom */
-		clips[n].c.left = 0;
-		clips[n].c.top = sheight - win->top;
-		clips[n].c.width  = win->width;
-		clips[n].c.height = win->height - clips[n].c.top;
-		n++;
-	}
-	return n;
-}
-
-int
-btcx_align(struct v4l2_rect *win, struct v4l2_clip *clips, unsigned int n, int mask)
-{
-	s32 nx,nw,dx;
-	unsigned int i;
-
-	/* fixup window */
-	nx = (win->left + mask) & ~mask;
-	nw = (win->width) & ~mask;
-	if (nx + nw > win->left + win->width)
-		nw -= mask+1;
-	dx = nx - win->left;
-	win->left  = nx;
-	win->width = nw;
-	dprintk("btcx: window align %dx%d+%d+%d [dx=%d]\n",
-	       win->width, win->height, win->left, win->top, dx);
-
-	/* fixup clips */
-	for (i = 0; i < n; i++) {
-		nx = (clips[i].c.left-dx) & ~mask;
-		nw = (clips[i].c.width) & ~mask;
-		if (nx + nw < clips[i].c.left-dx + clips[i].c.width)
-			nw += mask+1;
-		clips[i].c.left  = nx;
-		clips[i].c.width = nw;
-		dprintk("btcx:   clip align %dx%d+%d+%d\n",
-		       clips[i].c.width, clips[i].c.height,
-		       clips[i].c.left, clips[i].c.top);
-	}
-	return 0;
-}
-
-void
-btcx_sort_clips(struct v4l2_clip *clips, unsigned int nclips)
-{
-	int i,j,n;
-
-	if (nclips < 2)
-		return;
-	for (i = nclips-2; i >= 0; i--) {
-		for (n = 0, j = 0; j <= i; j++) {
-			if (clips[j].c.left > clips[j+1].c.left) {
-				swap(clips[j], clips[j + 1]);
-				n++;
-			}
-		}
-		if (0 == n)
-			break;
-	}
-}
-
-void
-btcx_calc_skips(int line, int width, int *maxy,
-		struct btcx_skiplist *skips, unsigned int *nskips,
-		const struct v4l2_clip *clips, unsigned int nclips)
-{
-	unsigned int clip,skip;
-	int end, maxline;
-
-	skip=0;
-	maxline = 9999;
-	for (clip = 0; clip < nclips; clip++) {
-
-		/* sanity checks */
-		if (clips[clip].c.left + clips[clip].c.width <= 0)
-			continue;
-		if (clips[clip].c.left > (signed)width)
-			break;
-
-		/* vertical range */
-		if (line > clips[clip].c.top+clips[clip].c.height-1)
-			continue;
-		if (line < clips[clip].c.top) {
-			if (maxline > clips[clip].c.top-1)
-				maxline = clips[clip].c.top-1;
-			continue;
-		}
-		if (maxline > clips[clip].c.top+clips[clip].c.height-1)
-			maxline = clips[clip].c.top+clips[clip].c.height-1;
-
-		/* horizontal range */
-		if (0 == skip || clips[clip].c.left > skips[skip-1].end) {
-			/* new one */
-			skips[skip].start = clips[clip].c.left;
-			if (skips[skip].start < 0)
-				skips[skip].start = 0;
-			skips[skip].end = clips[clip].c.left + clips[clip].c.width;
-			if (skips[skip].end > width)
-				skips[skip].end = width;
-			skip++;
-		} else {
-			/* overlaps -- expand last one */
-			end = clips[clip].c.left + clips[clip].c.width;
-			if (skips[skip-1].end < end)
-				skips[skip-1].end = end;
-			if (skips[skip-1].end > width)
-				skips[skip-1].end = width;
-		}
-	}
-	*nskips = skip;
-	*maxy = maxline;
-
-	if (btcx_debug) {
-		dprintk("btcx: skips line %d-%d:", line, maxline);
-		for (skip = 0; skip < *nskips; skip++) {
-			pr_cont(" %d-%d", skips[skip].start, skips[skip].end);
-		}
-		pr_cont("\n");
-	}
-}
diff --git a/drivers/media/pci/bt8xx/btcx-risc.h b/drivers/media/pci/bt8xx/btcx-risc.h
index dc774a64cd1f..6ac79a15776f 100644
--- a/drivers/media/pci/bt8xx/btcx-risc.h
+++ b/drivers/media/pci/bt8xx/btcx-risc.h
@@ -16,12 +16,3 @@ int  btcx_riscmem_alloc(struct pci_dev *pci,
 			unsigned int size);
 void btcx_riscmem_free(struct pci_dev *pci,
 		       struct btcx_riscmem *risc);
-
-int btcx_screen_clips(int swidth, int sheight, struct v4l2_rect *win,
-		      struct v4l2_clip *clips, unsigned int n);
-int btcx_align(struct v4l2_rect *win, struct v4l2_clip *clips,
-	       unsigned int n, int mask);
-void btcx_sort_clips(struct v4l2_clip *clips, unsigned int nclips);
-void btcx_calc_skips(int line, int width, int *maxy,
-		     struct btcx_skiplist *skips, unsigned int *nskips,
-		     const struct v4l2_clip *clips, unsigned int nclips);
diff --git a/drivers/media/pci/bt8xx/bttv-cards.c b/drivers/media/pci/bt8xx/bttv-cards.c
index c2b5ab287dd7..ec78f7fc5e1b 100644
--- a/drivers/media/pci/bt8xx/bttv-cards.c
+++ b/drivers/media/pci/bt8xx/bttv-cards.c
@@ -81,7 +81,6 @@ static int pvr_boot(struct bttv *btv);
 static unsigned int triton1;
 static unsigned int vsfx;
 static unsigned int latency = UNSET;
-int no_overlay=-1;
 
 static unsigned int card[BTTV_MAX]   = { [ 0 ... (BTTV_MAX-1) ] = UNSET };
 static unsigned int pll[BTTV_MAX]    = { [ 0 ... (BTTV_MAX-1) ] = UNSET };
@@ -99,7 +98,6 @@ static unsigned int audiomux[5] = { [ 0 ... 4 ] = UNSET };
 /* insmod options */
 module_param(triton1,    int, 0444);
 module_param(vsfx,       int, 0444);
-module_param(no_overlay, int, 0444);
 module_param(latency,    int, 0444);
 module_param(gpiomask,   int, 0444);
 module_param(audioall,   int, 0444);
@@ -127,7 +125,6 @@ MODULE_PARM_DESC(audiodev, "specify audio device:\n"
 		"\t\t 2 = tda7432\n"
 		"\t\t 3 = tvaudio");
 MODULE_PARM_DESC(saa6588, "if 1, then load the saa6588 RDS module, default (0) is to use the card definition.");
-MODULE_PARM_DESC(no_overlay, "allow override overlay default (0 disables, 1 enables) [some VIA/SIS chipsets are known to have problem with overlay]");
 
 
 /* I2C addresses list */
@@ -4869,11 +4866,8 @@ static void gv800s_init(struct bttv *btv)
 
 void __init bttv_check_chipset(void)
 {
-	int pcipci_fail = 0;
 	struct pci_dev *dev = NULL;
 
-	if (pci_pci_problems & (PCIPCI_FAIL|PCIAGP_FAIL))	/* should check if target is AGP */
-		pcipci_fail = 1;
 	if (pci_pci_problems & (PCIPCI_TRITON|PCIPCI_NATOMA|PCIPCI_VIAETBF))
 		triton1 = 1;
 	if (pci_pci_problems & PCIPCI_VSFX)
@@ -4889,15 +4883,6 @@ void __init bttv_check_chipset(void)
 		pr_info("Host bridge needs ETBF enabled\n");
 	if (vsfx)
 		pr_info("Host bridge needs VSFX enabled\n");
-	if (pcipci_fail) {
-		pr_info("bttv and your chipset may not work together\n");
-		if (!no_overlay) {
-			pr_info("overlay will be disabled\n");
-			no_overlay = 1;
-		} else {
-			pr_info("overlay forced. Use this option at your own risk.\n");
-		}
-	}
 	if (UNSET != latency)
 		pr_info("pci latency fixup [%d]\n", latency);
 	while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL,
diff --git a/drivers/media/pci/bt8xx/bttv-driver.c b/drivers/media/pci/bt8xx/bttv-driver.c
index d40b537f4e98..734f02b91aa3 100644
--- a/drivers/media/pci/bt8xx/bttv-driver.c
+++ b/drivers/media/pci/bt8xx/bttv-driver.c
@@ -624,20 +624,14 @@ static const unsigned int FORMATS = ARRAY_SIZE(formats);
 		 VIDIOC_QBUF 1)              bttv_release
 		 VIDIOCMCAPTURE 1)
 
-   OVERLAY	 VIDIOCCAPTURE on            VIDIOCCAPTURE off
-		 VIDIOC_OVERLAY on           VIDIOC_OVERLAY off
-		 3)                          bttv_release
-
    VBI		 VIDIOC_STREAMON             VIDIOC_STREAMOFF
 		 VIDIOC_QBUF 1)              bttv_release
-		 bttv_read, bttv_poll 1) 4)
+		 bttv_read, bttv_poll 1) 3)
 
    1) The resource must be allocated when we enter buffer prepare functions
       and remain allocated while buffers are in the DMA queue.
    2) This is a single frame read.
-   3) VIDIOC_S_FBUF and VIDIOC_S_FMT (OVERLAY) still work when
-      RESOURCE_OVERLAY is allocated.
-   4) This is a continuous read, implies VIDIOC_STREAMON.
+   3) This is a continuous read, implies VIDIOC_STREAMON.
 
    Note this driver permits video input and standard changes regardless if
    resources are allocated.
@@ -645,8 +639,7 @@ static const unsigned int FORMATS = ARRAY_SIZE(formats);
 
 #define VBI_RESOURCES (RESOURCE_VBI)
 #define VIDEO_RESOURCES (RESOURCE_VIDEO_READ | \
-			 RESOURCE_VIDEO_STREAM | \
-			 RESOURCE_OVERLAY)
+			 RESOURCE_VIDEO_STREAM)
 
 static
 int check_alloc_btres_lock(struct bttv *btv, struct bttv_fh *fh, int bit)
@@ -1492,37 +1485,6 @@ format_by_fourcc(int fourcc)
 }
 
 /* ----------------------------------------------------------------------- */
-/* misc helpers                                                            */
-
-static int
-bttv_switch_overlay(struct bttv *btv, struct bttv_fh *fh,
-		    struct bttv_buffer *new)
-{
-	struct bttv_buffer *old;
-	unsigned long flags;
-
-	dprintk("switch_overlay: enter [new=%p]\n", new);
-	if (new)
-		new->vb.state = VIDEOBUF_DONE;
-	spin_lock_irqsave(&btv->s_lock,flags);
-	old = btv->screen;
-	btv->screen = new;
-	btv->loop_irq |= 1;
-	bttv_set_dma(btv, 0x03);
-	spin_unlock_irqrestore(&btv->s_lock,flags);
-	if (NULL != old) {
-		dprintk("switch_overlay: old=%p state is %d\n",
-			old, old->vb.state);
-		bttv_dma_free(&fh->cap,btv, old);
-		kfree(old);
-	}
-	if (NULL == new)
-		free_btres_lock(btv,fh,RESOURCE_OVERLAY);
-	dprintk("switch_overlay: done\n");
-	return 0;
-}
-
-/* ----------------------------------------------------------------------- */
 /* video4linux (1) interface                                               */
 
 static int bttv_prepare_buffer(struct videobuf_queue *q,struct bttv *btv,
@@ -2045,150 +2007,6 @@ limit_scaled_size_lock       (struct bttv_fh *               fh,
 	return rc;
 }
 
-/* Returns an error if the given overlay window dimensions are not
-   possible with the current cropping parameters. If adjust_size is
-   TRUE the function may adjust the window width and/or height
-   instead, however it always rounds the horizontal position and
-   width as btcx_align() does. If adjust_crop is TRUE the function
-   may also adjust the current cropping parameters to get closer
-   to the desired window size. */
-static int
-verify_window_lock(struct bttv_fh *fh, struct v4l2_window *win,
-			 int adjust_size, int adjust_crop)
-{
-	enum v4l2_field field;
-	unsigned int width_mask;
-
-	if (win->w.width < 48)
-		win->w.width = 48;
-	if (win->w.height < 32)
-		win->w.height = 32;
-	if (win->clipcount > 2048)
-		win->clipcount = 2048;
-
-	win->chromakey = 0;
-	win->global_alpha = 0;
-	field = win->field;
-
-	switch (field) {
-	case V4L2_FIELD_TOP:
-	case V4L2_FIELD_BOTTOM:
-	case V4L2_FIELD_INTERLACED:
-		break;
-	default:
-		field = V4L2_FIELD_ANY;
-		break;
-	}
-	if (V4L2_FIELD_ANY == field) {
-		__s32 height2;
-
-		height2 = fh->btv->crop[!!fh->do_crop].rect.height >> 1;
-		field = (win->w.height > height2)
-			? V4L2_FIELD_INTERLACED
-			: V4L2_FIELD_TOP;
-	}
-	win->field = field;
-
-	if (NULL == fh->ovfmt)
-		return -EINVAL;
-	/* 4-byte alignment. */
-	width_mask = ~0;
-	switch (fh->ovfmt->depth) {
-	case 8:
-	case 24:
-		width_mask = ~3;
-		break;
-	case 16:
-		width_mask = ~1;
-		break;
-	case 32:
-		break;
-	default:
-		BUG();
-	}
-
-	win->w.width -= win->w.left & ~width_mask;
-	win->w.left = (win->w.left - width_mask - 1) & width_mask;
-
-	return limit_scaled_size_lock(fh, &win->w.width, &win->w.height,
-				      field, width_mask,
-				      /* width_bias: round down */ 0,
-				      adjust_size, adjust_crop);
-}
-
-static int setup_window_lock(struct bttv_fh *fh, struct bttv *btv,
-			struct v4l2_window *win, int fixup)
-{
-	struct v4l2_clip *clips = NULL;
-	int n,size,retval = 0;
-
-	if (NULL == fh->ovfmt)
-		return -EINVAL;
-	if (!(fh->ovfmt->flags & FORMAT_FLAGS_PACKED))
-		return -EINVAL;
-	retval = verify_window_lock(fh, win,
-			       /* adjust_size */ fixup,
-			       /* adjust_crop */ fixup);
-	if (0 != retval)
-		return retval;
-
-	/* copy clips  --  luckily v4l1 + v4l2 are binary
-	   compatible here ...*/
-	n = win->clipcount;
-	size = sizeof(*clips)*(n+4);
-	clips = kmalloc(size,GFP_KERNEL);
-	if (NULL == clips)
-		return -ENOMEM;
-	if (n > 0)
-		memcpy(clips, win->clips, sizeof(struct v4l2_clip) * n);
-
-	/* clip against screen */
-	if (NULL != btv->fbuf.base)
-		n = btcx_screen_clips(btv->fbuf.fmt.width, btv->fbuf.fmt.height,
-				      &win->w, clips, n);
-	btcx_sort_clips(clips,n);
-
-	/* 4-byte alignments */
-	switch (fh->ovfmt->depth) {
-	case 8:
-	case 24:
-		btcx_align(&win->w, clips, n, 3);
-		break;
-	case 16:
-		btcx_align(&win->w, clips, n, 1);
-		break;
-	case 32:
-		/* no alignment fixups needed */
-		break;
-	default:
-		BUG();
-	}
-
-	kfree(fh->ov.clips);
-	fh->ov.clips    = clips;
-	fh->ov.nclips   = n;
-
-	fh->ov.w        = win->w;
-	fh->ov.field    = win->field;
-	fh->ov.setup_ok = 1;
-
-	btv->init.ov.w.width   = win->w.width;
-	btv->init.ov.w.height  = win->w.height;
-	btv->init.ov.field     = win->field;
-
-	/* update overlay if needed */
-	retval = 0;
-	if (check_btres(fh, RESOURCE_OVERLAY)) {
-		struct bttv_buffer *new;
-
-		new = videobuf_sg_alloc(sizeof(*new));
-		new->crop = btv->crop[!!fh->do_crop].rect;
-		bttv_overlay_risc(btv, &fh->ov, fh->ovfmt, new);
-		retval = bttv_switch_overlay(btv,fh,new);
-	}
-	return retval;
-}
-
 /* ----------------------------------------------------------------------- */
 
 static struct videobuf_queue* bttv_queue(struct bttv_fh *fh)
@@ -2270,17 +2088,6 @@ static int bttv_g_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
-static int bttv_g_fmt_vid_overlay(struct file *file, void *priv,
-					struct v4l2_format *f)
-{
-	struct bttv_fh *fh  = priv;
-
-	f->fmt.win.w     = fh->ov.w;
-	f->fmt.win.field = fh->ov.field;
-
-	return 0;
-}
-
 static void bttv_get_width_mask_vid_cap(const struct bttv_format *fmt,
 					unsigned int *width_mask,
 					unsigned int *width_bias)
@@ -2352,17 +2159,6 @@ static int bttv_try_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
-static int bttv_try_fmt_vid_overlay(struct file *file, void *priv,
-						struct v4l2_format *f)
-{
-	struct bttv_fh *fh = priv;
-
-	verify_window_lock(fh, &f->fmt.win,
-			/* adjust_size */ 1,
-			/* adjust_crop */ 0);
-	return 0;
-}
-
 static int bttv_s_fmt_vid_cap(struct file *file, void *priv,
 				struct v4l2_format *f)
 {
@@ -2410,20 +2206,6 @@ static int bttv_s_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
-static int bttv_s_fmt_vid_overlay(struct file *file, void *priv,
-				struct v4l2_format *f)
-{
-	struct bttv_fh *fh = priv;
-	struct bttv *btv = fh->btv;
-
-	if (no_overlay > 0) {
-		pr_err("V4L2_BUF_TYPE_VIDEO_OVERLAY: no_overlay\n");
-		return -EINVAL;
-	}
-
-	return setup_window_lock(fh, btv, &f->fmt.win, 1);
-}
-
 static int bttv_querycap(struct file *file, void  *priv,
 				struct v4l2_capability *cap)
 {
@@ -2437,8 +2219,6 @@ static int bttv_querycap(struct file *file, void  *priv,
 	strscpy(cap->card, btv->video_dev.name, sizeof(cap->card));
 	cap->capabilities = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_READWRITE |
 			    V4L2_CAP_STREAMING | V4L2_CAP_DEVICE_CAPS;
-	if (no_overlay <= 0)
-		cap->capabilities |= V4L2_CAP_VIDEO_OVERLAY;
 	if (video_is_registered(&btv->vbi_dev))
 		cap->capabilities |= V4L2_CAP_VBI_CAPTURE;
 	if (video_is_registered(&btv->radio_dev)) {
@@ -2458,7 +2238,8 @@ static int bttv_querycap(struct file *file, void  *priv,
 	return 0;
 }
 
-static int bttv_enum_fmt_cap_ovr(struct v4l2_fmtdesc *f)
+static int bttv_enum_fmt_vid_cap(struct file *file, void  *priv,
+				 struct v4l2_fmtdesc *f)
 {
 	int index = -1, i;
 
@@ -2473,162 +2254,9 @@ static int bttv_enum_fmt_cap_ovr(struct v4l2_fmtdesc *f)
 
 	f->pixelformat = formats[i].fourcc;
 
-	return i;
-}
-
-static int bttv_enum_fmt_vid_cap(struct file *file, void  *priv,
-				struct v4l2_fmtdesc *f)
-{
-	int rc = bttv_enum_fmt_cap_ovr(f);
-
-	if (rc < 0)
-		return rc;
-
-	return 0;
-}
-
-static int bttv_enum_fmt_vid_overlay(struct file *file, void  *priv,
-					struct v4l2_fmtdesc *f)
-{
-	int rc;
-
-	if (no_overlay > 0) {
-		pr_err("V4L2_BUF_TYPE_VIDEO_OVERLAY: no_overlay\n");
-		return -EINVAL;
-	}
-
-	rc = bttv_enum_fmt_cap_ovr(f);
-
-	if (rc < 0)
-		return rc;
-
-	if (!(formats[rc].flags & FORMAT_FLAGS_PACKED))
-		return -EINVAL;
-
 	return 0;
 }
 
-static int bttv_g_fbuf(struct file *file, void *f,
-				struct v4l2_framebuffer *fb)
-{
-	struct bttv_fh *fh = f;
-	struct bttv *btv = fh->btv;
-
-	*fb = btv->fbuf;
-	fb->capability = V4L2_FBUF_CAP_LIST_CLIPPING;
-	fb->flags = V4L2_FBUF_FLAG_PRIMARY;
-	if (fh->ovfmt)
-		fb->fmt.pixelformat  = fh->ovfmt->fourcc;
-	return 0;
-}
-
-static int bttv_overlay(struct file *file, void *f, unsigned int on)
-{
-	struct bttv_fh *fh = f;
-	struct bttv *btv = fh->btv;
-	struct bttv_buffer *new;
-	int retval = 0;
-
-	if (on) {
-		/* verify args */
-		if (unlikely(!btv->fbuf.base)) {
-			return -EINVAL;
-		}
-		if (unlikely(!fh->ov.setup_ok)) {
-			dprintk("%d: overlay: !setup_ok\n", btv->c.nr);
-			retval = -EINVAL;
-		}
-		if (retval)
-			return retval;
-	}
-
-	if (!check_alloc_btres_lock(btv, fh, RESOURCE_OVERLAY))
-		return -EBUSY;
-
-	if (on) {
-		fh->ov.tvnorm = btv->tvnorm;
-		new = videobuf_sg_alloc(sizeof(*new));
-		new->crop = btv->crop[!!fh->do_crop].rect;
-		bttv_overlay_risc(btv, &fh->ov, fh->ovfmt, new);
-	} else {
-		new = NULL;
-	}
-
-	/* switch over */
-	retval = bttv_switch_overlay(btv, fh, new);
-	return retval;
-}
-
-static int bttv_s_fbuf(struct file *file, void *f,
-				const struct v4l2_framebuffer *fb)
-{
-	struct bttv_fh *fh = f;
-	struct bttv *btv = fh->btv;
-	const struct bttv_format *fmt;
-	int retval;
-
-	if (!capable(CAP_SYS_ADMIN) &&
-		!capable(CAP_SYS_RAWIO))
-		return -EPERM;
-
-	/* check args */
-	fmt = format_by_fourcc(fb->fmt.pixelformat);
-	if (NULL == fmt)
-		return -EINVAL;
-	if (0 == (fmt->flags & FORMAT_FLAGS_PACKED))
-		return -EINVAL;
-
-	retval = -EINVAL;
-	if (fb->flags & V4L2_FBUF_FLAG_OVERLAY) {
-		__s32 width = fb->fmt.width;
-		__s32 height = fb->fmt.height;
-
-		retval = limit_scaled_size_lock(fh, &width, &height,
-					   V4L2_FIELD_INTERLACED,
-					   /* width_mask */ ~3,
-					   /* width_bias */ 2,
-					   /* adjust_size */ 0,
-					   /* adjust_crop */ 0);
-		if (0 != retval)
-			return retval;
-	}
-
-	/* ok, accept it */
-	btv->fbuf.base       = fb->base;
-	btv->fbuf.fmt.width  = fb->fmt.width;
-	btv->fbuf.fmt.height = fb->fmt.height;
-	if (0 != fb->fmt.bytesperline)
-		btv->fbuf.fmt.bytesperline = fb->fmt.bytesperline;
-	else
-		btv->fbuf.fmt.bytesperline = btv->fbuf.fmt.width*fmt->depth/8;
-
-	retval = 0;
-	fh->ovfmt = fmt;
-	btv->init.ovfmt = fmt;
-	if (fb->flags & V4L2_FBUF_FLAG_OVERLAY) {
-		fh->ov.w.left   = 0;
-		fh->ov.w.top    = 0;
-		fh->ov.w.width  = fb->fmt.width;
-		fh->ov.w.height = fb->fmt.height;
-		btv->init.ov.w.width  = fb->fmt.width;
-		btv->init.ov.w.height = fb->fmt.height;
-
-		kfree(fh->ov.clips);
-		fh->ov.clips = NULL;
-		fh->ov.nclips = 0;
-
-		if (check_btres(fh, RESOURCE_OVERLAY)) {
-			struct bttv_buffer *new;
-
-			new = videobuf_sg_alloc(sizeof(*new));
-			new->crop = btv->crop[!!fh->do_crop].rect;
-			bttv_overlay_risc(btv, &fh->ov, fh->ovfmt, new);
-			retval = bttv_switch_overlay(btv, fh, new);
-		}
-	}
-	return retval;
-}
-
 static int bttv_reqbufs(struct file *file, void *priv,
 				struct v4l2_requestbuffers *p)
 {
@@ -2748,8 +2376,7 @@ static int bttv_g_selection(struct file *file, void *f, struct v4l2_selection *s
 	struct bttv_fh *fh = f;
 	struct bttv *btv = fh->btv;
 
-	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
-	    sel->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 
 	switch (sel->target) {
@@ -2786,8 +2413,7 @@ static int bttv_s_selection(struct file *file, void *f, struct v4l2_selection *s
 	__s32 b_right;
 	__s32 b_bottom;
 
-	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
-	    sel->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 
 	if (sel->target != V4L2_SEL_TGT_CROP)
@@ -2977,7 +2603,6 @@ static int bttv_open(struct file *file)
 	v4l2_fh_init(&fh->fh, vdev);
 
 	fh->type = type;
-	fh->ov.setup_ok = 0;
 
 	videobuf_queue_sg_init(&fh->cap, &bttv_video_qops,
 			    &btv->c.pci->dev, &btv->s_lock,
@@ -3021,10 +2646,6 @@ static int bttv_release(struct file *file)
 	struct bttv_fh *fh = file->private_data;
 	struct bttv *btv = fh->btv;
 
-	/* turn off overlay */
-	if (check_btres(fh, RESOURCE_OVERLAY))
-		bttv_switch_overlay(btv,fh,NULL);
-
 	/* stop video capture */
 	if (check_btres(fh, RESOURCE_VIDEO_STREAM)) {
 		videobuf_streamoff(&fh->cap);
@@ -3090,10 +2711,6 @@ static const struct v4l2_ioctl_ops bttv_ioctl_ops = {
 	.vidioc_g_fmt_vid_cap           = bttv_g_fmt_vid_cap,
 	.vidioc_try_fmt_vid_cap         = bttv_try_fmt_vid_cap,
 	.vidioc_s_fmt_vid_cap           = bttv_s_fmt_vid_cap,
-	.vidioc_enum_fmt_vid_overlay    = bttv_enum_fmt_vid_overlay,
-	.vidioc_g_fmt_vid_overlay       = bttv_g_fmt_vid_overlay,
-	.vidioc_try_fmt_vid_overlay     = bttv_try_fmt_vid_overlay,
-	.vidioc_s_fmt_vid_overlay       = bttv_s_fmt_vid_overlay,
 	.vidioc_g_fmt_vbi_cap           = bttv_g_fmt_vbi_cap,
 	.vidioc_try_fmt_vbi_cap         = bttv_try_fmt_vbi_cap,
 	.vidioc_s_fmt_vbi_cap           = bttv_s_fmt_vbi_cap,
@@ -3113,9 +2730,6 @@ static const struct v4l2_ioctl_ops bttv_ioctl_ops = {
 	.vidioc_s_tuner                 = bttv_s_tuner,
 	.vidioc_g_selection             = bttv_g_selection,
 	.vidioc_s_selection             = bttv_s_selection,
-	.vidioc_g_fbuf                  = bttv_g_fbuf,
-	.vidioc_s_fbuf                  = bttv_s_fbuf,
-	.vidioc_overlay                 = bttv_overlay,
 	.vidioc_g_parm                  = bttv_g_parm,
 	.vidioc_g_frequency             = bttv_g_frequency,
 	.vidioc_s_frequency             = bttv_s_frequency,
@@ -3385,9 +2999,6 @@ static void bttv_print_riscaddr(struct bttv *btv)
 		? (unsigned long long)btv->curr.top->top.dma : 0,
 		btv->curr.bottom
 		? (unsigned long long)btv->curr.bottom->bottom.dma : 0);
-	pr_info("  scr : o=%08llx e=%08llx\n",
-		btv->screen ? (unsigned long long)btv->screen->top.dma : 0,
-		btv->screen ? (unsigned long long)btv->screen->bottom.dma : 0);
 	bttv_risc_disasm(btv, &btv->main);
 }
 
@@ -3508,28 +3119,9 @@ bttv_irq_next_video(struct bttv *btv, struct bttv_buffer_set *set)
 		}
 	}
 
-	/* screen overlay ? */
-	if (NULL != btv->screen) {
-		if (V4L2_FIELD_HAS_BOTH(btv->screen->vb.field)) {
-			if (NULL == set->top && NULL == set->bottom) {
-				set->top    = btv->screen;
-				set->bottom = btv->screen;
-			}
-		} else {
-			if (V4L2_FIELD_TOP == btv->screen->vb.field &&
-			    NULL == set->top) {
-				set->top = btv->screen;
-			}
-			if (V4L2_FIELD_BOTTOM == btv->screen->vb.field &&
-			    NULL == set->bottom) {
-				set->bottom = btv->screen;
-			}
-		}
-	}
-
-	dprintk("%d: next set: top=%p bottom=%p [screen=%p,irq=%d,%d]\n",
+	dprintk("%d: next set: top=%p bottom=%p [irq=%d,%d]\n",
 		btv->c.nr, set->top, set->bottom,
-		btv->screen, set->frame_irq, set->top_irq);
+		set->frame_irq, set->top_irq);
 	return 0;
 }
 
@@ -3883,17 +3475,12 @@ static void bttv_unregister_video(struct bttv *btv)
 /* register video4linux devices */
 static int bttv_register_video(struct bttv *btv)
 {
-	if (no_overlay > 0)
-		pr_notice("Overlay support disabled\n");
-
 	/* video */
 	vdev_init(btv, &btv->video_dev, &bttv_video_template, "video");
 	btv->video_dev.device_caps = V4L2_CAP_VIDEO_CAPTURE |
 				     V4L2_CAP_READWRITE | V4L2_CAP_STREAMING;
 	if (btv->tuner_type != TUNER_ABSENT)
 		btv->video_dev.device_caps |= V4L2_CAP_TUNER;
-	if (no_overlay <= 0)
-		btv->video_dev.device_caps |= V4L2_CAP_VIDEO_OVERLAY;
 
 	if (video_register_device(&btv->video_dev, VFL_TYPE_VIDEO,
 				  video_nr[btv->c.nr]) < 0)
@@ -4084,14 +3671,9 @@ static int bttv_probe(struct pci_dev *dev, const struct pci_device_id *pci_id)
 
 	/* fill struct bttv with some useful defaults */
 	btv->init.btv         = btv;
-	btv->init.ov.w.width  = 320;
-	btv->init.ov.w.height = 240;
 	btv->init.fmt         = format_by_fourcc(V4L2_PIX_FMT_BGR24);
 	btv->init.width       = 320;
 	btv->init.height      = 240;
-	btv->init.ov.w.width  = 320;
-	btv->init.ov.w.height = 240;
-	btv->init.ov.field    = V4L2_FIELD_INTERLACED;
 	btv->input = 0;
 
 	v4l2_ctrl_new_std(hdl, &bttv_ctrl_ops,
diff --git a/drivers/media/pci/bt8xx/bttv-risc.c b/drivers/media/pci/bt8xx/bttv-risc.c
index 32fa4a7fe76f..4fa4b9da9634 100644
--- a/drivers/media/pci/bt8xx/bttv-risc.c
+++ b/drivers/media/pci/bt8xx/bttv-risc.c
@@ -231,95 +231,6 @@ bttv_risc_planar(struct bttv *btv, struct btcx_riscmem *risc,
 	return 0;
 }
 
-static int
-bttv_risc_overlay(struct bttv *btv, struct btcx_riscmem *risc,
-		  const struct bttv_format *fmt, struct bttv_overlay *ov,
-		  int skip_even, int skip_odd)
-{
-	int dwords, rc, line, maxy, start, end;
-	unsigned skip, nskips;
-	struct btcx_skiplist *skips;
-	__le32 *rp;
-	u32 ri,ra;
-	u32 addr;
-
-	/* skip list for window clipping */
-	skips = kmalloc_array(ov->nclips, sizeof(*skips),GFP_KERNEL);
-	if (NULL == skips)
-		return -ENOMEM;
-
-	/* estimate risc mem: worst case is (1.5*clip+1) * lines instructions
-	   + sync + jump (all 2 dwords) */
-	dwords  = (3 * ov->nclips + 2) *
-		((skip_even || skip_odd) ? (ov->w.height+1)>>1 :  ov->w.height);
-	dwords += 4;
-	if ((rc = btcx_riscmem_alloc(btv->c.pci,risc,dwords*4)) < 0) {
-		kfree(skips);
-		return rc;
-	}
-
-	/* sync instruction */
-	rp = risc->cpu;
-	*(rp++) = cpu_to_le32(BT848_RISC_SYNC|BT848_FIFO_STATUS_FM1);
-	*(rp++) = cpu_to_le32(0);
-
-	addr  = (unsigned long)btv->fbuf.base;
-	addr += btv->fbuf.fmt.bytesperline * ov->w.top;
-	addr += (fmt->depth >> 3)          * ov->w.left;
-
-	/* scan lines */
-	for (maxy = -1, line = 0; line < ov->w.height;
-	     line++, addr += btv->fbuf.fmt.bytesperline) {
-		if ((btv->opt_vcr_hack) &&
-		     (line >= (ov->w.height - VCR_HACK_LINES)))
-			continue;
-		if ((line%2) == 0  &&  skip_even)
-			continue;
-		if ((line%2) == 1  &&  skip_odd)
-			continue;
-
-		/* calculate clipping */
-		if (line > maxy)
-			btcx_calc_skips(line, ov->w.width, &maxy,
-					skips, &nskips, ov->clips, ov->nclips);
-
-		/* write out risc code */
-		for (start = 0, skip = 0; start < ov->w.width; start = end) {
-			if (skip >= nskips) {
-				ri  = BT848_RISC_WRITE;
-				end = ov->w.width;
-			} else if (start < skips[skip].start) {
-				ri  = BT848_RISC_WRITE;
-				end = skips[skip].start;
-			} else {
-				ri  = BT848_RISC_SKIP;
-				end = skips[skip].end;
-				skip++;
-			}
-			if (BT848_RISC_WRITE == ri)
-				ra = addr + (fmt->depth>>3)*start;
-			else
-				ra = 0;
-
-			if (0 == start)
-				ri |= BT848_RISC_SOL;
-			if (ov->w.width == end)
-				ri |= BT848_RISC_EOL;
-			ri |= (fmt->depth>>3) * (end-start);
-
-			*(rp++)=cpu_to_le32(ri);
-			if (0 != ra)
-				*(rp++)=cpu_to_le32(ra);
-		}
-	}
-
-	/* save pointer to jmp instruction address */
-	risc->jmp = rp;
-	BUG_ON((risc->jmp - risc->cpu + 2) * sizeof(*risc->cpu) > risc->size);
-	kfree(skips);
-	return 0;
-}
-
 /* ---------------------------------------------------------- */
 
 static void
@@ -848,45 +759,3 @@ bttv_buffer_risc(struct bttv *btv, struct bttv_buffer *buf)
 	buf->btswap   = buf->fmt->btswap;
 	return 0;
 }
-
-/* ---------------------------------------------------------- */
-
-/* calculate geometry, build risc code */
-int
-bttv_overlay_risc(struct bttv *btv,
-		  struct bttv_overlay *ov,
-		  const struct bttv_format *fmt,
-		  struct bttv_buffer *buf)
-{
-	/* check interleave, bottom+top fields */
-	dprintk("%d: overlay fields: %s format: 0x%08x  size: %dx%d\n",
-		btv->c.nr, v4l2_field_names[buf->vb.field],
-		fmt->fourcc, ov->w.width, ov->w.height);
-
-	/* calculate geometry */
-	bttv_calc_geo(btv,&buf->geo,ov->w.width,ov->w.height,
-		      V4L2_FIELD_HAS_BOTH(ov->field),
-		      &bttv_tvnorms[ov->tvnorm],&buf->crop);
-
-	/* build risc code */
-	switch (ov->field) {
-	case V4L2_FIELD_TOP:
-		bttv_risc_overlay(btv, &buf->top,    fmt, ov, 0, 0);
-		break;
-	case V4L2_FIELD_BOTTOM:
-		bttv_risc_overlay(btv, &buf->bottom, fmt, ov, 0, 0);
-		break;
-	case V4L2_FIELD_INTERLACED:
-		bttv_risc_overlay(btv, &buf->top,    fmt, ov, 0, 1);
-		bttv_risc_overlay(btv, &buf->bottom, fmt, ov, 1, 0);
-		break;
-	default:
-		BUG();
-	}
-
-	/* copy format info */
-	buf->btformat = fmt->btformat;
-	buf->btswap   = fmt->btswap;
-	buf->vb.field = ov->field;
-	return 0;
-}
diff --git a/drivers/media/pci/bt8xx/bttvp.h b/drivers/media/pci/bt8xx/bttvp.h
index 4abf43657846..717f002a41df 100644
--- a/drivers/media/pci/bt8xx/bttvp.h
+++ b/drivers/media/pci/bt8xx/bttvp.h
@@ -50,7 +50,6 @@
 #define RISC_SLOT_E_FIELD     12
 #define RISC_SLOT_LOOP        14
 
-#define RESOURCE_OVERLAY       1
 #define RESOURCE_VIDEO_STREAM  2
 #define RESOURCE_VBI           4
 #define RESOURCE_VIDEO_READ    8
@@ -165,15 +164,6 @@ struct bttv_buffer_set {
 	unsigned int           frame_irq;
 };
 
-struct bttv_overlay {
-	unsigned int           tvnorm;
-	struct v4l2_rect       w;
-	enum v4l2_field        field;
-	struct v4l2_clip       *clips;
-	int                    nclips;
-	int                    setup_ok;
-};
-
 struct bttv_vbi_fmt {
 	struct v4l2_vbi_format fmt;
 
@@ -216,10 +206,6 @@ struct bttv_fh {
 	int                      width;
 	int                      height;
 
-	/* video overlay */
-	const struct bttv_format *ovfmt;
-	struct bttv_overlay      ov;
-
 	/* Application called VIDIOC_S_SELECTION. */
 	int                      do_crop;
 
@@ -256,12 +242,6 @@ int bttv_buffer_activate_vbi(struct bttv *btv,
 void bttv_dma_free(struct videobuf_queue *q, struct bttv *btv,
 		   struct bttv_buffer *buf);
 
-/* overlay handling */
-int bttv_overlay_risc(struct bttv *btv, struct bttv_overlay *ov,
-		      const struct bttv_format *fmt,
-		      struct bttv_buffer *buf);
-
-
 /* ---------------------------------------------------------- */
 /* bttv-vbi.c                                                 */
 
@@ -279,11 +259,6 @@ int bttv_sub_add_device(struct bttv_core *core, char *name);
 int bttv_sub_del_devices(struct bttv_core *core);
 
 /* ---------------------------------------------------------- */
-/* bttv-cards.c                                               */
-
-extern int no_overlay;
-
-/* ---------------------------------------------------------- */
 /* bttv-input.c                                               */
 
 extern void init_bttv_i2c_ir(struct bttv *btv);
@@ -454,7 +429,6 @@ struct bttv {
 	   - must acquire s_lock before changing these
 	   - only the irq handler is supported to touch top + bottom + vcurr */
 	struct btcx_riscmem     main;
-	struct bttv_buffer      *screen;    /* overlay             */
 	struct list_head        capture;    /* video capture queue */
 	struct list_head        vcapture;   /* vbi capture queue   */
 	struct bttv_buffer_set  curr;       /* active buffers      */
@@ -479,7 +453,7 @@ struct bttv {
 	/* used to make dvb-bt8xx autoloadable */
 	struct work_struct request_module_wk;
 
-	/* Default (0) and current (1) video capturing and overlay
+	/* Default (0) and current (1) video capturing
 	   cropping parameters in bttv_tvnorm.cropcap units. Protected
 	   by bttv.lock. */
 	struct bttv_crop crop[2];
diff --git a/drivers/media/pci/cobalt/cobalt-v4l2.c b/drivers/media/pci/cobalt/cobalt-v4l2.c
index 0ff37496c9ab..4bfbcca14f60 100644
--- a/drivers/media/pci/cobalt/cobalt-v4l2.c
+++ b/drivers/media/pci/cobalt/cobalt-v4l2.c
@@ -708,7 +708,6 @@ static int cobalt_g_fmt_vid_cap(struct file *file, void *priv_fh,
 {
 	struct cobalt_stream *s = video_drvdata(file);
 	struct v4l2_pix_format *pix = &f->fmt.pix;
-	struct v4l2_subdev_format sd_fmt;
 
 	pix->width = s->width;
 	pix->height = s->height;
@@ -718,8 +717,11 @@ static int cobalt_g_fmt_vid_cap(struct file *file, void *priv_fh,
 	if (s->input == 1) {
 		pix->colorspace = V4L2_COLORSPACE_SRGB;
 	} else {
-		sd_fmt.pad = s->pad_source;
-		sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		struct v4l2_subdev_format sd_fmt = {
+			.pad = s->pad_source,
+			.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		};
+
 		v4l2_subdev_call(s->sd, pad, get_fmt, NULL, &sd_fmt);
 		v4l2_fill_pix_format(pix, &sd_fmt.format);
 	}
@@ -735,7 +737,6 @@ static int cobalt_try_fmt_vid_cap(struct file *file, void *priv_fh,
 {
 	struct cobalt_stream *s = video_drvdata(file);
 	struct v4l2_pix_format *pix = &f->fmt.pix;
-	struct v4l2_subdev_format sd_fmt;
 
 	/* Check for min (QCIF) and max (Full HD) size */
 	if ((pix->width < 176) || (pix->height < 144)) {
@@ -760,8 +761,11 @@ static int cobalt_try_fmt_vid_cap(struct file *file, void *priv_fh,
 		pix->height = 1080;
 		pix->colorspace = V4L2_COLORSPACE_SRGB;
 	} else {
-		sd_fmt.pad = s->pad_source;
-		sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		struct v4l2_subdev_format sd_fmt = {
+			.pad = s->pad_source,
+			.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		};
+
 		v4l2_subdev_call(s->sd, pad, get_fmt, NULL, &sd_fmt);
 		v4l2_fill_pix_format(pix, &sd_fmt.format);
 	}
@@ -906,7 +910,9 @@ static int cobalt_s_fmt_vid_out(struct file *file, void *priv_fh,
 {
 	struct cobalt_stream *s = video_drvdata(file);
 	struct v4l2_pix_format *pix = &f->fmt.pix;
-	struct v4l2_subdev_format sd_fmt = { 0 };
+	struct v4l2_subdev_format sd_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	u32 code;
 
 	if (cobalt_try_fmt_vid_out(file, priv_fh, f))
@@ -937,7 +943,6 @@ static int cobalt_s_fmt_vid_out(struct file *file, void *priv_fh,
 	s->xfer_func = pix->xfer_func;
 	s->ycbcr_enc = pix->ycbcr_enc;
 	s->quantization = pix->quantization;
-	sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	v4l2_fill_mbus_format(&sd_fmt.format, pix, code);
 	v4l2_subdev_call(s->sd, pad, set_fmt, NULL, &sd_fmt);
 	return 0;
diff --git a/drivers/media/pci/cx18/Kconfig b/drivers/media/pci/cx18/Kconfig
index a4e32fdcfd3d..5c959bb5e233 100644
--- a/drivers/media/pci/cx18/Kconfig
+++ b/drivers/media/pci/cx18/Kconfig
@@ -3,7 +3,7 @@ config VIDEO_CX18
 	tristate "Conexant cx23418 MPEG encoder support"
 	depends on VIDEO_DEV && DVB_CORE && PCI && I2C
 	select I2C_ALGOBIT
-	select VIDEOBUF_VMALLOC
+	select VIDEOBUF2_VMALLOC
 	depends on RC_CORE
 	select VIDEO_TUNER
 	select VIDEO_TVEEPROM
diff --git a/drivers/media/pci/cx18/cx18-driver.c b/drivers/media/pci/cx18/cx18-driver.c
index 84260972c343..743fcc961374 100644
--- a/drivers/media/pci/cx18/cx18-driver.c
+++ b/drivers/media/pci/cx18/cx18-driver.c
@@ -771,11 +771,11 @@ static void cx18_init_struct2(struct cx18 *cx)
 {
 	int i;
 
-	for (i = 0; i < CX18_CARD_MAX_VIDEO_INPUTS - 1; i++)
+	for (i = 0; i < CX18_CARD_MAX_VIDEO_INPUTS; i++)
 		if (cx->card->video_inputs[i].video_type == 0)
 			break;
 	cx->nof_inputs = i;
-	for (i = 0; i < CX18_CARD_MAX_AUDIO_INPUTS - 1; i++)
+	for (i = 0; i < CX18_CARD_MAX_AUDIO_INPUTS; i++)
 		if (cx->card->audio_inputs[i].audio_type == 0)
 			break;
 	cx->nof_audio_inputs = i;
diff --git a/drivers/media/pci/cx18/cx18-driver.h b/drivers/media/pci/cx18/cx18-driver.h
index ef545b96121d..887d2aa36447 100644
--- a/drivers/media/pci/cx18/cx18-driver.h
+++ b/drivers/media/pci/cx18/cx18-driver.h
@@ -48,9 +48,8 @@
 #include <media/dvb_net.h>
 #include <media/dvbdev.h>
 
-/* Videobuf / YUV support */
-#include <media/videobuf-core.h>
-#include <media/videobuf-vmalloc.h>
+/* vb2 YUV support */
+#include <media/videobuf2-vmalloc.h>
 
 #ifndef CONFIG_PCI
 #  error "This driver requires kernel PCI support."
@@ -284,6 +283,14 @@ struct cx18_options {
 #define list_entry_is_past_end(pos, head, member) \
 	(&pos->member == (head))
 
+struct cx18_vb2_buffer {
+	/* Common video buffer sub-system struct */
+	struct vb2_v4l2_buffer vb;
+	struct list_head list;
+	v4l2_std_id tvnorm; /* selected tv norm */
+	u32 bytes_used;
+};
+
 struct cx18_buffer {
 	struct list_head list;
 	dma_addr_t dma_handle;
@@ -399,19 +406,12 @@ struct cx18_stream {
 	struct list_head vb_capture;    /* video capture queue */
 	spinlock_t vb_lock;
 	struct timer_list vb_timeout;
+	u32 sequence;
 
-	struct videobuf_queue vbuf_q;
-	spinlock_t vbuf_q_lock; /* Protect vbuf_q */
+	struct vb2_queue vidq;
 	enum v4l2_buf_type vb_type;
 };
 
-struct cx18_videobuf_buffer {
-	/* Common video buffer sub-system struct */
-	struct videobuf_buffer vb;
-	v4l2_std_id tvnorm; /* selected tv norm */
-	u32 bytes_used;
-};
-
 struct cx18_open_id {
 	struct v4l2_fh fh;
 	u32 open_id;
diff --git a/drivers/media/pci/cx18/cx18-fileops.c b/drivers/media/pci/cx18/cx18-fileops.c
index 4cf5b9ca92e7..7e742733391b 100644
--- a/drivers/media/pci/cx18/cx18-fileops.c
+++ b/drivers/media/pci/cx18/cx18-fileops.c
@@ -584,12 +584,6 @@ ssize_t cx18_v4l2_read(struct file *filp, char __user *buf, size_t count,
 	if (rc)
 		return rc;
 
-	if ((s->vb_type == V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
-		(id->type == CX18_ENC_STREAM_TYPE_YUV)) {
-		return videobuf_read_stream(&s->vbuf_q, buf, count, pos, 0,
-			filp->f_flags & O_NONBLOCK);
-	}
-
 	return cx18_read_pos(s, buf, count, pos, filp->f_flags & O_NONBLOCK);
 }
 
@@ -618,17 +612,6 @@ __poll_t cx18_v4l2_enc_poll(struct file *filp, poll_table *wait)
 		CX18_DEBUG_FILE("Encoder poll started capture\n");
 	}
 
-	if ((s->vb_type == V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
-		(id->type == CX18_ENC_STREAM_TYPE_YUV)) {
-		__poll_t videobuf_poll = videobuf_poll_stream(filp, &s->vbuf_q, wait);
-
-		if (v4l2_event_pending(&id->fh))
-			res |= EPOLLPRI;
-		if (eof && videobuf_poll == EPOLLERR)
-			return res | EPOLLHUP;
-		return res | videobuf_poll;
-	}
-
 	/* add stream's waitq to the poll list */
 	CX18_DEBUG_HI_FILE("Encoder poll\n");
 	if (v4l2_event_pending(&id->fh))
@@ -643,62 +626,20 @@ __poll_t cx18_v4l2_enc_poll(struct file *filp, poll_table *wait)
 	return res;
 }
 
-int cx18_v4l2_mmap(struct file *file, struct vm_area_struct *vma)
-{
-	struct cx18_open_id *id = file->private_data;
-	struct cx18 *cx = id->cx;
-	struct cx18_stream *s = &cx->streams[id->type];
-	int eof = test_bit(CX18_F_S_STREAMOFF, &s->s_flags);
-
-	if ((s->vb_type == V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
-		(id->type == CX18_ENC_STREAM_TYPE_YUV)) {
-
-		/* Start a capture if there is none */
-		if (!eof && !test_bit(CX18_F_S_STREAMING, &s->s_flags)) {
-			int rc;
-
-			mutex_lock(&cx->serialize_lock);
-			rc = cx18_start_capture(id);
-			mutex_unlock(&cx->serialize_lock);
-			if (rc) {
-				CX18_DEBUG_INFO(
-					"Could not start capture for %s (%d)\n",
-					s->name, rc);
-				return -EINVAL;
-			}
-			CX18_DEBUG_FILE("Encoder mmap started capture\n");
-		}
-
-		return videobuf_mmap_mapper(&s->vbuf_q, vma);
-	}
-
-	return -EINVAL;
-}
-
 void cx18_vb_timeout(struct timer_list *t)
 {
 	struct cx18_stream *s = from_timer(s, t, vb_timeout);
-	struct cx18_videobuf_buffer *buf;
-	unsigned long flags;
 
-	/* Return all of the buffers in error state, so the vbi/vid inode
+	/*
+	 * Return all of the buffers in error state, so the vbi/vid inode
 	 * can return from blocking.
 	 */
-	spin_lock_irqsave(&s->vb_lock, flags);
-	while (!list_empty(&s->vb_capture)) {
-		buf = list_entry(s->vb_capture.next,
-			struct cx18_videobuf_buffer, vb.queue);
-		list_del(&buf->vb.queue);
-		buf->vb.state = VIDEOBUF_ERROR;
-		wake_up(&buf->vb.done);
-	}
-	spin_unlock_irqrestore(&s->vb_lock, flags);
+	cx18_clear_queue(s, VB2_BUF_STATE_ERROR);
 }
 
-void cx18_stop_capture(struct cx18_open_id *id, int gop_end)
+void cx18_stop_capture(struct cx18_stream *s, int gop_end)
 {
-	struct cx18 *cx = id->cx;
-	struct cx18_stream *s = &cx->streams[id->type];
+	struct cx18 *cx = s->cx;
 	struct cx18_stream *s_vbi = &cx->streams[CX18_ENC_STREAM_TYPE_VBI];
 	struct cx18_stream *s_idx = &cx->streams[CX18_ENC_STREAM_TYPE_IDX];
 
@@ -709,7 +650,7 @@ void cx18_stop_capture(struct cx18_open_id *id, int gop_end)
 	/* Stop capturing */
 	if (test_bit(CX18_F_S_STREAMING, &s->s_flags)) {
 		CX18_DEBUG_INFO("close stopping capture\n");
-		if (id->type == CX18_ENC_STREAM_TYPE_MPG) {
+		if (s->type == CX18_ENC_STREAM_TYPE_MPG) {
 			/* Stop internal use associated VBI and IDX streams */
 			if (test_bit(CX18_F_S_STREAMING, &s_vbi->s_flags) &&
 			    !test_bit(CX18_F_S_APPL_IO, &s_vbi->s_flags)) {
@@ -721,7 +662,7 @@ void cx18_stop_capture(struct cx18_open_id *id, int gop_end)
 				cx18_stop_v4l2_encode_stream(s_idx, 0);
 			}
 		}
-		if (id->type == CX18_ENC_STREAM_TYPE_VBI &&
+		if (s->type == CX18_ENC_STREAM_TYPE_VBI &&
 		    test_bit(CX18_F_S_INTERNAL_USE, &s->s_flags))
 			/* Also used internally, don't stop capturing */
 			s->id = -1;
@@ -741,13 +682,14 @@ int cx18_v4l2_close(struct file *filp)
 	struct cx18_open_id *id = fh2id(fh);
 	struct cx18 *cx = id->cx;
 	struct cx18_stream *s = &cx->streams[id->type];
+	struct video_device *vdev = &s->video_dev;
 
 	CX18_DEBUG_IOCTL("close() of %s\n", s->name);
 
 	mutex_lock(&cx->serialize_lock);
 	/* Stop radio */
 	if (id->type == CX18_ENC_STREAM_TYPE_RAD &&
-			v4l2_fh_is_singular_file(filp)) {
+	    v4l2_fh_is_singular_file(filp)) {
 		/* Closing radio device, return to TV mode */
 		cx18_mute(cx);
 		/* Mark that the radio is no longer in use */
@@ -766,12 +708,17 @@ int cx18_v4l2_close(struct file *filp)
 		cx18_unmute(cx);
 	}
 
+	if (id->type == CX18_ENC_STREAM_TYPE_YUV &&
+	    filp->private_data == vdev->queue->owner) {
+		vb2_queue_release(vdev->queue);
+		vdev->queue->owner = NULL;
+	}
 	v4l2_fh_del(fh);
 	v4l2_fh_exit(fh);
 
 	/* 'Unclaim' this stream */
-	if (s->id == id->open_id)
-		cx18_stop_capture(id, 0);
+	if (id->type != CX18_ENC_STREAM_TYPE_YUV && s->id == id->open_id)
+		cx18_stop_capture(s, 0);
 	kfree(id);
 	mutex_unlock(&cx->serialize_lock);
 	return 0;
diff --git a/drivers/media/pci/cx18/cx18-fileops.h b/drivers/media/pci/cx18/cx18-fileops.h
index 1985d6422347..943057b83d94 100644
--- a/drivers/media/pci/cx18/cx18-fileops.h
+++ b/drivers/media/pci/cx18/cx18-fileops.h
@@ -16,10 +16,11 @@ ssize_t cx18_v4l2_write(struct file *filp, const char __user *buf, size_t count,
 int cx18_v4l2_close(struct file *filp);
 __poll_t cx18_v4l2_enc_poll(struct file *filp, poll_table *wait);
 int cx18_start_capture(struct cx18_open_id *id);
-void cx18_stop_capture(struct cx18_open_id *id, int gop_end);
+void cx18_stop_capture(struct cx18_stream *s, int gop_end);
 void cx18_mute(struct cx18 *cx);
 void cx18_unmute(struct cx18 *cx);
 int cx18_v4l2_mmap(struct file *file, struct vm_area_struct *vma);
+void cx18_clear_queue(struct cx18_stream *s, enum vb2_buffer_state state);
 void cx18_vb_timeout(struct timer_list *t);
 
 /* Shared with cx18-alsa module */
diff --git a/drivers/media/pci/cx18/cx18-ioctl.c b/drivers/media/pci/cx18/cx18-ioctl.c
index c8ba7841c720..1817b9ed042c 100644
--- a/drivers/media/pci/cx18/cx18-ioctl.c
+++ b/drivers/media/pci/cx18/cx18-ioctl.c
@@ -27,6 +27,133 @@
 #include <media/tveeprom.h>
 #include <media/v4l2-event.h>
 
+static const struct v4l2_fmtdesc cx18_formats_yuv[] = {
+	{
+		.index = 0,
+		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
+		.pixelformat = V4L2_PIX_FMT_NV12_16L16,
+	},
+	{
+		.index = 1,
+		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
+		.pixelformat = V4L2_PIX_FMT_UYVY,
+	},
+};
+
+static const struct v4l2_fmtdesc cx18_formats_mpeg[] = {
+	{
+		.index = 0,
+		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
+		.flags = V4L2_FMT_FLAG_COMPRESSED,
+		.pixelformat = V4L2_PIX_FMT_MPEG,
+	},
+};
+
+static int cx18_g_fmt_vid_cap(struct file *file, void *fh,
+			      struct v4l2_format *fmt)
+{
+	struct cx18_open_id *id = fh2id(fh);
+	struct cx18 *cx = id->cx;
+	struct cx18_stream *s = &cx->streams[id->type];
+	struct v4l2_pix_format *pixfmt = &fmt->fmt.pix;
+
+	pixfmt->width = cx->cxhdl.width;
+	pixfmt->height = cx->cxhdl.height;
+	pixfmt->colorspace = V4L2_COLORSPACE_SMPTE170M;
+	pixfmt->field = V4L2_FIELD_INTERLACED;
+	if (id->type == CX18_ENC_STREAM_TYPE_YUV) {
+		pixfmt->pixelformat = s->pixelformat;
+		pixfmt->sizeimage = s->vb_bytes_per_frame;
+		pixfmt->bytesperline = s->vb_bytes_per_line;
+	} else {
+		pixfmt->pixelformat = V4L2_PIX_FMT_MPEG;
+		pixfmt->sizeimage = 128 * 1024;
+		pixfmt->bytesperline = 0;
+	}
+	return 0;
+}
+
+static int cx18_try_fmt_vid_cap(struct file *file, void *fh,
+				struct v4l2_format *fmt)
+{
+	struct cx18_open_id *id = fh2id(fh);
+	struct cx18 *cx = id->cx;
+	struct v4l2_pix_format *pixfmt = &fmt->fmt.pix;
+	int w = pixfmt->width;
+	int h = pixfmt->height;
+
+	w = min(w, 720);
+	w = max(w, 720 / 16);
+
+	h = min(h, cx->is_50hz ? 576 : 480);
+	h = max(h, (cx->is_50hz ? 576 : 480) / 8);
+
+	if (id->type == CX18_ENC_STREAM_TYPE_YUV) {
+		if (pixfmt->pixelformat != V4L2_PIX_FMT_NV12_16L16 &&
+		    pixfmt->pixelformat != V4L2_PIX_FMT_UYVY)
+			pixfmt->pixelformat = V4L2_PIX_FMT_UYVY;
+		/* YUV height must be a multiple of 32 */
+		h = round_up(h, 32);
+		/*
+		 * HM12 YUV size is (Y=(h*720) + UV=(h*(720/2)))
+		 * UYUV YUV size is (Y=(h*720) + UV=(h*(720)))
+		 */
+		if (pixfmt->pixelformat == V4L2_PIX_FMT_NV12_16L16) {
+			pixfmt->sizeimage = h * 720 * 3 / 2;
+			pixfmt->bytesperline = 720; /* First plane */
+		} else {
+			pixfmt->sizeimage = h * 720 * 2;
+			pixfmt->bytesperline = 1440; /* Packed */
+		}
+	} else {
+		pixfmt->pixelformat = V4L2_PIX_FMT_MPEG;
+		pixfmt->sizeimage = 128 * 1024;
+		pixfmt->bytesperline = 0;
+	}
+
+	pixfmt->width = w;
+	pixfmt->height = h;
+	pixfmt->colorspace = V4L2_COLORSPACE_SMPTE170M;
+	pixfmt->field = V4L2_FIELD_INTERLACED;
+	return 0;
+}
+
+static int cx18_s_fmt_vid_cap(struct file *file, void *fh,
+			      struct v4l2_format *fmt)
+{
+	struct cx18_open_id *id = fh2id(fh);
+	struct cx18 *cx = id->cx;
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	struct cx18_stream *s = &cx->streams[id->type];
+	int ret;
+	int w, h;
+
+	ret = cx18_try_fmt_vid_cap(file, fh, fmt);
+	if (ret)
+		return ret;
+	w = fmt->fmt.pix.width;
+	h = fmt->fmt.pix.height;
+
+	if (cx->cxhdl.width == w && cx->cxhdl.height == h &&
+	    s->pixelformat == fmt->fmt.pix.pixelformat)
+		return 0;
+
+	if (atomic_read(&cx->ana_capturing) > 0)
+		return -EBUSY;
+
+	s->pixelformat = fmt->fmt.pix.pixelformat;
+	s->vb_bytes_per_frame = fmt->fmt.pix.sizeimage;
+	s->vb_bytes_per_line = fmt->fmt.pix.bytesperline;
+
+	format.format.width = cx->cxhdl.width = w;
+	format.format.height = cx->cxhdl.height = h;
+	format.format.code = MEDIA_BUS_FMT_FIXED;
+	v4l2_subdev_call(cx->sd_av, pad, set_fmt, NULL, &format);
+	return cx18_g_fmt_vid_cap(file, fh, fmt);
+}
+
 u16 cx18_service2vbi(int type)
 {
 	switch (type) {
@@ -131,30 +258,6 @@ u16 cx18_get_service_set(struct v4l2_sliced_vbi_format *fmt)
 	return set;
 }
 
-static int cx18_g_fmt_vid_cap(struct file *file, void *fh,
-				struct v4l2_format *fmt)
-{
-	struct cx18_open_id *id = fh2id(fh);
-	struct cx18 *cx = id->cx;
-	struct cx18_stream *s = &cx->streams[id->type];
-	struct v4l2_pix_format *pixfmt = &fmt->fmt.pix;
-
-	pixfmt->width = cx->cxhdl.width;
-	pixfmt->height = cx->cxhdl.height;
-	pixfmt->colorspace = V4L2_COLORSPACE_SMPTE170M;
-	pixfmt->field = V4L2_FIELD_INTERLACED;
-	if (id->type == CX18_ENC_STREAM_TYPE_YUV) {
-		pixfmt->pixelformat = s->pixelformat;
-		pixfmt->sizeimage = s->vb_bytes_per_frame;
-		pixfmt->bytesperline = s->vb_bytes_per_line;
-	} else {
-		pixfmt->pixelformat = V4L2_PIX_FMT_MPEG;
-		pixfmt->sizeimage = 128 * 1024;
-		pixfmt->bytesperline = 0;
-	}
-	return 0;
-}
-
 static int cx18_g_fmt_vbi_cap(struct file *file, void *fh,
 				struct v4l2_format *fmt)
 {
@@ -199,30 +302,6 @@ static int cx18_g_fmt_sliced_vbi_cap(struct file *file, void *fh,
 	return 0;
 }
 
-static int cx18_try_fmt_vid_cap(struct file *file, void *fh,
-				struct v4l2_format *fmt)
-{
-	struct cx18_open_id *id = fh2id(fh);
-	struct cx18 *cx = id->cx;
-	int w = fmt->fmt.pix.width;
-	int h = fmt->fmt.pix.height;
-	int min_h = 2;
-
-	w = min(w, 720);
-	w = max(w, 2);
-	if (id->type == CX18_ENC_STREAM_TYPE_YUV) {
-		/* YUV height must be a multiple of 32 */
-		h &= ~0x1f;
-		min_h = 32;
-	}
-	h = min(h, cx->is_50hz ? 576 : 480);
-	h = max(h, min_h);
-
-	fmt->fmt.pix.width = w;
-	fmt->fmt.pix.height = h;
-	return 0;
-}
-
 static int cx18_try_fmt_vbi_cap(struct file *file, void *fh,
 				struct v4l2_format *fmt)
 {
@@ -248,49 +327,6 @@ static int cx18_try_fmt_sliced_vbi_cap(struct file *file, void *fh,
 	return 0;
 }
 
-static int cx18_s_fmt_vid_cap(struct file *file, void *fh,
-				struct v4l2_format *fmt)
-{
-	struct cx18_open_id *id = fh2id(fh);
-	struct cx18 *cx = id->cx;
-	struct v4l2_subdev_format format = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-	};
-	struct cx18_stream *s = &cx->streams[id->type];
-	int ret;
-	int w, h;
-
-	ret = cx18_try_fmt_vid_cap(file, fh, fmt);
-	if (ret)
-		return ret;
-	w = fmt->fmt.pix.width;
-	h = fmt->fmt.pix.height;
-
-	if (cx->cxhdl.width == w && cx->cxhdl.height == h &&
-	    s->pixelformat == fmt->fmt.pix.pixelformat)
-		return 0;
-
-	if (atomic_read(&cx->ana_capturing) > 0)
-		return -EBUSY;
-
-	s->pixelformat = fmt->fmt.pix.pixelformat;
-	/* HM12 YUV size is (Y=(h*720) + UV=(h*(720/2)))
-	   UYUV YUV size is (Y=(h*720) + UV=(h*(720))) */
-	if (s->pixelformat == V4L2_PIX_FMT_NV12_16L16) {
-		s->vb_bytes_per_frame = h * 720 * 3 / 2;
-		s->vb_bytes_per_line = 720; /* First plane */
-	} else {
-		s->vb_bytes_per_frame = h * 720 * 2;
-		s->vb_bytes_per_line = 1440; /* Packed */
-	}
-
-	format.format.width = cx->cxhdl.width = w;
-	format.format.height = cx->cxhdl.height = h;
-	format.format.code = MEDIA_BUS_FMT_FIXED;
-	v4l2_subdev_call(cx->sd_av, pad, set_fmt, NULL, &format);
-	return cx18_g_fmt_vid_cap(file, fh, fmt);
-}
-
 static int cx18_s_fmt_vbi_cap(struct file *file, void *fh,
 				struct v4l2_format *fmt)
 {
@@ -463,31 +499,17 @@ static int cx18_g_selection(struct file *file, void *fh,
 static int cx18_enum_fmt_vid_cap(struct file *file, void *fh,
 					struct v4l2_fmtdesc *fmt)
 {
-	static const struct v4l2_fmtdesc formats[] = {
-		{
-			.index = 0,
-			.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
-			.description = "HM12 (YUV 4:1:1)",
-			.pixelformat = V4L2_PIX_FMT_NV12_16L16,
-		},
-		{
-			.index = 1,
-			.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
-			.flags = V4L2_FMT_FLAG_COMPRESSED,
-			.description = "MPEG",
-			.pixelformat = V4L2_PIX_FMT_MPEG,
-		},
-		{
-			.index = 2,
-			.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
-			.description = "UYVY 4:2:2",
-			.pixelformat = V4L2_PIX_FMT_UYVY,
-		},
-	};
+	struct cx18_open_id *id = fh2id(fh);
 
-	if (fmt->index > ARRAY_SIZE(formats) - 1)
+	if (id->type == CX18_ENC_STREAM_TYPE_YUV) {
+		if (fmt->index >= ARRAY_SIZE(cx18_formats_yuv))
+			return -EINVAL;
+		*fmt = cx18_formats_yuv[fmt->index];
+		return 0;
+	}
+	if (fmt->index)
 		return -EINVAL;
-	*fmt = formats[fmt->index];
+	*fmt = cx18_formats_mpeg[0];
 	return 0;
 }
 
@@ -596,6 +618,19 @@ int cx18_s_std(struct file *file, void *fh, v4l2_std_id std)
 	cx2341x_handler_set_50hz(&cx->cxhdl, cx->is_50hz);
 	cx->cxhdl.width = 720;
 	cx->cxhdl.height = cx->is_50hz ? 576 : 480;
+	/*
+	 * HM12 YUV size is (Y=(h*720) + UV=(h*(720/2)))
+	 * UYUV YUV size is (Y=(h*720) + UV=(h*(720)))
+	 */
+	if (cx->streams[CX18_ENC_STREAM_TYPE_YUV].pixelformat == V4L2_PIX_FMT_NV12_16L16) {
+		cx->streams[CX18_ENC_STREAM_TYPE_YUV].vb_bytes_per_frame =
+			cx->cxhdl.height * 720 * 3 / 2;
+		cx->streams[CX18_ENC_STREAM_TYPE_YUV].vb_bytes_per_line = 720;
+	} else {
+		cx->streams[CX18_ENC_STREAM_TYPE_YUV].vb_bytes_per_frame =
+			cx->cxhdl.height * 720 * 2;
+		cx->streams[CX18_ENC_STREAM_TYPE_YUV].vb_bytes_per_line = 1440;
+	}
 	cx->vbi.count = cx->is_50hz ? 18 : 12;
 	cx->vbi.start[0] = cx->is_50hz ? 6 : 10;
 	cx->vbi.start[1] = cx->is_50hz ? 318 : 273;
@@ -803,117 +838,6 @@ static int cx18_g_enc_index(struct file *file, void *fh,
 	return 0;
 }
 
-static struct videobuf_queue *cx18_vb_queue(struct cx18_open_id *id)
-{
-	struct videobuf_queue *q = NULL;
-	struct cx18 *cx = id->cx;
-	struct cx18_stream *s = &cx->streams[id->type];
-
-	switch (s->vb_type) {
-	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		q = &s->vbuf_q;
-		break;
-	case V4L2_BUF_TYPE_VBI_CAPTURE:
-		break;
-	default:
-		break;
-	}
-	return q;
-}
-
-static int cx18_streamon(struct file *file, void *priv,
-	enum v4l2_buf_type type)
-{
-	struct cx18_open_id *id = file->private_data;
-	struct cx18 *cx = id->cx;
-	struct cx18_stream *s = &cx->streams[id->type];
-
-	/* Start the hardware only if we're the video device */
-	if ((s->vb_type != V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
-		(s->vb_type != V4L2_BUF_TYPE_VBI_CAPTURE))
-		return -EINVAL;
-
-	if (id->type != CX18_ENC_STREAM_TYPE_YUV)
-		return -EINVAL;
-
-	/* Establish a buffer timeout */
-	mod_timer(&s->vb_timeout, msecs_to_jiffies(2000) + jiffies);
-
-	return videobuf_streamon(cx18_vb_queue(id));
-}
-
-static int cx18_streamoff(struct file *file, void *priv,
-	enum v4l2_buf_type type)
-{
-	struct cx18_open_id *id = file->private_data;
-	struct cx18 *cx = id->cx;
-	struct cx18_stream *s = &cx->streams[id->type];
-
-	/* Start the hardware only if we're the video device */
-	if ((s->vb_type != V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
-		(s->vb_type != V4L2_BUF_TYPE_VBI_CAPTURE))
-		return -EINVAL;
-
-	if (id->type != CX18_ENC_STREAM_TYPE_YUV)
-		return -EINVAL;
-
-	return videobuf_streamoff(cx18_vb_queue(id));
-}
-
-static int cx18_reqbufs(struct file *file, void *priv,
-	struct v4l2_requestbuffers *rb)
-{
-	struct cx18_open_id *id = file->private_data;
-	struct cx18 *cx = id->cx;
-	struct cx18_stream *s = &cx->streams[id->type];
-
-	if ((s->vb_type != V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
-		(s->vb_type != V4L2_BUF_TYPE_VBI_CAPTURE))
-		return -EINVAL;
-
-	return videobuf_reqbufs(cx18_vb_queue(id), rb);
-}
-
-static int cx18_querybuf(struct file *file, void *priv,
-	struct v4l2_buffer *b)
-{
-	struct cx18_open_id *id = file->private_data;
-	struct cx18 *cx = id->cx;
-	struct cx18_stream *s = &cx->streams[id->type];
-
-	if ((s->vb_type != V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
-		(s->vb_type != V4L2_BUF_TYPE_VBI_CAPTURE))
-		return -EINVAL;
-
-	return videobuf_querybuf(cx18_vb_queue(id), b);
-}
-
-static int cx18_qbuf(struct file *file, void *priv, struct v4l2_buffer *b)
-{
-	struct cx18_open_id *id = file->private_data;
-	struct cx18 *cx = id->cx;
-	struct cx18_stream *s = &cx->streams[id->type];
-
-	if ((s->vb_type != V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
-		(s->vb_type != V4L2_BUF_TYPE_VBI_CAPTURE))
-		return -EINVAL;
-
-	return videobuf_qbuf(cx18_vb_queue(id), b);
-}
-
-static int cx18_dqbuf(struct file *file, void *priv, struct v4l2_buffer *b)
-{
-	struct cx18_open_id *id = file->private_data;
-	struct cx18 *cx = id->cx;
-	struct cx18_stream *s = &cx->streams[id->type];
-
-	if ((s->vb_type != V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
-		(s->vb_type != V4L2_BUF_TYPE_VBI_CAPTURE))
-		return -EINVAL;
-
-	return videobuf_dqbuf(cx18_vb_queue(id), b, file->f_flags & O_NONBLOCK);
-}
-
 static int cx18_encoder_cmd(struct file *file, void *fh,
 				struct v4l2_encoder_cmd *enc)
 {
@@ -930,7 +854,7 @@ static int cx18_encoder_cmd(struct file *file, void *fh,
 	case V4L2_ENC_CMD_STOP:
 		CX18_DEBUG_IOCTL("V4L2_ENC_CMD_STOP\n");
 		enc->flags &= V4L2_ENC_CMD_STOP_AT_GOP_END;
-		cx18_stop_capture(id,
+		cx18_stop_capture(&cx->streams[id->type],
 				  enc->flags & V4L2_ENC_CMD_STOP_AT_GOP_END);
 		break;
 
@@ -1106,12 +1030,15 @@ static const struct v4l2_ioctl_ops cx18_ioctl_ops = {
 	.vidioc_s_register              = cx18_s_register,
 #endif
 	.vidioc_default                 = cx18_default,
-	.vidioc_streamon                = cx18_streamon,
-	.vidioc_streamoff               = cx18_streamoff,
-	.vidioc_reqbufs                 = cx18_reqbufs,
-	.vidioc_querybuf                = cx18_querybuf,
-	.vidioc_qbuf                    = cx18_qbuf,
-	.vidioc_dqbuf                   = cx18_dqbuf,
+
+	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
+	.vidioc_querybuf		= vb2_ioctl_querybuf,
+	.vidioc_qbuf			= vb2_ioctl_qbuf,
+	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
+	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
+	.vidioc_streamon		= vb2_ioctl_streamon,
+	.vidioc_streamoff		= vb2_ioctl_streamoff,
+	.vidioc_prepare_buf		= vb2_ioctl_prepare_buf,
 	.vidioc_subscribe_event		= v4l2_ctrl_subscribe_event,
 	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
 };
diff --git a/drivers/media/pci/cx18/cx18-mailbox.c b/drivers/media/pci/cx18/cx18-mailbox.c
index 162480ec68ca..3b283f3c6726 100644
--- a/drivers/media/pci/cx18/cx18-mailbox.c
+++ b/drivers/media/pci/cx18/cx18-mailbox.c
@@ -145,36 +145,37 @@ static void cx18_mdl_send_to_dvb(struct cx18_stream *s, struct cx18_mdl *mdl)
 	}
 }
 
-static void cx18_mdl_send_to_videobuf(struct cx18_stream *s,
-	struct cx18_mdl *mdl)
+static void cx18_mdl_send_to_vb2(struct cx18_stream *s, struct cx18_mdl *mdl)
 {
-	struct cx18_videobuf_buffer *vb_buf;
+	struct cx18_vb2_buffer *vb_buf;
 	struct cx18_buffer *buf;
 	u8 *p;
 	u32 offset = 0;
 	int dispatch = 0;
+	unsigned long bsize;
 
 	if (mdl->bytesused == 0)
 		return;
 
-	/* Acquire a videobuf buffer, clone to and and release it */
+	/* Acquire a vb2 buffer, clone to and release it */
 	spin_lock(&s->vb_lock);
 	if (list_empty(&s->vb_capture))
 		goto out;
 
-	vb_buf = list_first_entry(&s->vb_capture, struct cx18_videobuf_buffer,
-		vb.queue);
+	vb_buf = list_first_entry(&s->vb_capture, struct cx18_vb2_buffer,
+				  list);
 
-	p = videobuf_to_vmalloc(&vb_buf->vb);
+	p = vb2_plane_vaddr(&vb_buf->vb.vb2_buf, 0);
 	if (!p)
 		goto out;
 
+	bsize = vb2_get_plane_payload(&vb_buf->vb.vb2_buf, 0);
 	offset = vb_buf->bytes_used;
 	list_for_each_entry(buf, &mdl->buf_list, list) {
 		if (buf->bytesused == 0)
 			break;
 
-		if ((offset + buf->bytesused) <= vb_buf->vb.bsize) {
+		if ((offset + buf->bytesused) <= bsize) {
 			memcpy(p + offset, buf->buf, buf->bytesused);
 			offset += buf->bytesused;
 			vb_buf->bytes_used += buf->bytesused;
@@ -188,10 +189,10 @@ static void cx18_mdl_send_to_videobuf(struct cx18_stream *s,
 	}
 
 	if (dispatch) {
-		vb_buf->vb.ts = ktime_get_ns();
-		list_del(&vb_buf->vb.queue);
-		vb_buf->vb.state = VIDEOBUF_DONE;
-		wake_up(&vb_buf->vb.done);
+		vb_buf->vb.vb2_buf.timestamp = ktime_get_ns();
+		vb_buf->vb.sequence = s->sequence++;
+		list_del(&vb_buf->list);
+		vb2_buffer_done(&vb_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
 	}
 
 	mod_timer(&s->vb_timeout, msecs_to_jiffies(2000) + jiffies);
@@ -304,7 +305,7 @@ static void epu_dma_done(struct cx18 *cx, struct cx18_in_work_order *order)
 				cx18_enqueue(s, mdl, &s->q_full);
 			}
 		} else if (s->type == CX18_ENC_STREAM_TYPE_YUV) {
-			cx18_mdl_send_to_videobuf(s, mdl);
+			cx18_mdl_send_to_vb2(s, mdl);
 			cx18_enqueue(s, mdl, &s->q_free);
 		} else {
 			cx18_enqueue(s, mdl, &s->q_full);
diff --git a/drivers/media/pci/cx18/cx18-streams.c b/drivers/media/pci/cx18/cx18-streams.c
index 87ff554bb2d2..597472754c4c 100644
--- a/drivers/media/pci/cx18/cx18-streams.c
+++ b/drivers/media/pci/cx18/cx18-streams.c
@@ -29,7 +29,16 @@ static const struct v4l2_file_operations cx18_v4l2_enc_fops = {
 	.unlocked_ioctl = video_ioctl2,
 	.release = cx18_v4l2_close,
 	.poll = cx18_v4l2_enc_poll,
-	.mmap = cx18_v4l2_mmap,
+};
+
+static const struct v4l2_file_operations cx18_v4l2_enc_yuv_fops = {
+	.owner = THIS_MODULE,
+	.open = cx18_v4l2_open,
+	.unlocked_ioctl = video_ioctl2,
+	.release = cx18_v4l2_close,
+	.poll = vb2_fop_poll,
+	.read = vb2_fop_read,
+	.mmap = vb2_fop_mmap,
 };
 
 /* offset from 0 to register ts v4l2 minors on */
@@ -70,7 +79,7 @@ static struct {
 		VFL_TYPE_VBI, 0,
 		DMA_FROM_DEVICE,
 		V4L2_CAP_VBI_CAPTURE | V4L2_CAP_SLICED_VBI_CAPTURE |
-		V4L2_CAP_READWRITE | V4L2_CAP_TUNER
+		V4L2_CAP_READWRITE | V4L2_CAP_AUDIO | V4L2_CAP_TUNER
 	},
 	{	/* CX18_ENC_STREAM_TYPE_PCM */
 		"encoder PCM audio",
@@ -91,156 +100,142 @@ static struct {
 	},
 };
 
-
-static void cx18_dma_free(struct videobuf_queue *q,
-	struct cx18_stream *s, struct cx18_videobuf_buffer *buf)
-{
-	videobuf_waiton(q, &buf->vb, 0, 0);
-	videobuf_vmalloc_free(&buf->vb);
-	buf->vb.state = VIDEOBUF_NEEDS_INIT;
-}
-
-static int cx18_prepare_buffer(struct videobuf_queue *q,
-	struct cx18_stream *s,
-	struct cx18_videobuf_buffer *buf,
-	u32 pixelformat,
-	unsigned int width, unsigned int height,
-	enum v4l2_field field)
+static int cx18_queue_setup(struct vb2_queue *vq,
+			    unsigned int *nbuffers, unsigned int *nplanes,
+			    unsigned int sizes[], struct device *alloc_devs[])
 {
+	struct cx18_stream *s = vb2_get_drv_priv(vq);
 	struct cx18 *cx = s->cx;
-	int rc = 0;
-
-	/* check settings */
-	buf->bytes_used = 0;
-
-	if ((width  < 48) || (height < 32))
-		return -EINVAL;
-
-	buf->vb.size = (width * height * 2);
-	if ((buf->vb.baddr != 0) && (buf->vb.bsize < buf->vb.size))
-		return -EINVAL;
+	unsigned int szimage;
 
-	/* alloc + fill struct (if changed) */
-	if (buf->vb.width != width || buf->vb.height != height ||
-	    buf->vb.field != field || s->pixelformat != pixelformat ||
-	    buf->tvnorm != cx->std) {
+	/*
+	 * HM12 YUV size is (Y=(h*720) + UV=(h*(720/2)))
+	 * UYUV YUV size is (Y=(h*720) + UV=(h*(720)))
+	 */
+	if (s->pixelformat == V4L2_PIX_FMT_NV12_16L16)
+		szimage = cx->cxhdl.height * 720 * 3 / 2;
+	else
+		szimage = cx->cxhdl.height * 720 * 2;
 
-		buf->vb.width  = width;
-		buf->vb.height = height;
-		buf->vb.field  = field;
-		buf->tvnorm    = cx->std;
-		s->pixelformat = pixelformat;
+	/*
+	 * Let's request at least three buffers: two for the
+	 * DMA engine and one for userspace.
+	 */
+	if (vq->num_buffers + *nbuffers < 3)
+		*nbuffers = 3 - vq->num_buffers;
 
-		/* HM12 YUV size is (Y=(h*720) + UV=(h*(720/2)))
-		   UYUV YUV size is (Y=(h*720) + UV=(h*(720))) */
-		if (s->pixelformat == V4L2_PIX_FMT_NV12_16L16)
-			s->vb_bytes_per_frame = height * 720 * 3 / 2;
-		else
-			s->vb_bytes_per_frame = height * 720 * 2;
-		cx18_dma_free(q, s, buf);
+	if (*nplanes) {
+		if (*nplanes != 1 || sizes[0] < szimage)
+			return -EINVAL;
+		return 0;
 	}
 
-	if ((buf->vb.baddr != 0) && (buf->vb.bsize < buf->vb.size))
-		return -EINVAL;
-
-	if (buf->vb.field == 0)
-		buf->vb.field = V4L2_FIELD_INTERLACED;
-
-	if (VIDEOBUF_NEEDS_INIT == buf->vb.state) {
-		buf->vb.width  = width;
-		buf->vb.height = height;
-		buf->vb.field  = field;
-		buf->tvnorm    = cx->std;
-		s->pixelformat = pixelformat;
-
-		/* HM12 YUV size is (Y=(h*720) + UV=(h*(720/2)))
-		   UYUV YUV size is (Y=(h*720) + UV=(h*(720))) */
-		if (s->pixelformat == V4L2_PIX_FMT_NV12_16L16)
-			s->vb_bytes_per_frame = height * 720 * 3 / 2;
-		else
-			s->vb_bytes_per_frame = height * 720 * 2;
-		rc = videobuf_iolock(q, &buf->vb, NULL);
-		if (rc != 0)
-			goto fail;
-	}
-	buf->vb.state = VIDEOBUF_PREPARED;
+	sizes[0] = szimage;
+	*nplanes = 1;
 	return 0;
-
-fail:
-	cx18_dma_free(q, s, buf);
-	return rc;
-
 }
 
-/* VB_MIN_BUFSIZE is lcm(1440 * 480, 1440 * 576)
-   1440 is a single line of 4:2:2 YUV at 720 luma samples wide
-*/
-#define VB_MIN_BUFFERS 32
-#define VB_MIN_BUFSIZE 4147200
+static void cx18_buf_queue(struct vb2_buffer *vb)
+{
+	struct cx18_stream *s = vb2_get_drv_priv(vb->vb2_queue);
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct cx18_vb2_buffer *buf =
+		container_of(vbuf, struct cx18_vb2_buffer, vb);
+	unsigned long flags;
+
+	spin_lock_irqsave(&s->vb_lock, flags);
+	list_add_tail(&buf->list, &s->vb_capture);
+	spin_unlock_irqrestore(&s->vb_lock, flags);
+}
 
-static int buffer_setup(struct videobuf_queue *q,
-	unsigned int *count, unsigned int *size)
+static int cx18_buf_prepare(struct vb2_buffer *vb)
 {
-	struct cx18_stream *s = q->priv_data;
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct cx18_stream *s = vb2_get_drv_priv(vb->vb2_queue);
 	struct cx18 *cx = s->cx;
+	unsigned int size;
 
-	*size = 2 * cx->cxhdl.width * cx->cxhdl.height;
-	if (*count == 0)
-		*count = VB_MIN_BUFFERS;
-
-	while (*size * *count > VB_MIN_BUFFERS * VB_MIN_BUFSIZE)
-		(*count)--;
-
-	q->field = V4L2_FIELD_INTERLACED;
-	q->last = V4L2_FIELD_INTERLACED;
+	/*
+	 * HM12 YUV size is (Y=(h*720) + UV=(h*(720/2)))
+	 * UYUV YUV size is (Y=(h*720) + UV=(h*(720)))
+	 */
+	if (s->pixelformat == V4L2_PIX_FMT_NV12_16L16)
+		size = cx->cxhdl.height * 720 * 3 / 2;
+	else
+		size = cx->cxhdl.height * 720 * 2;
 
+	if (vb2_plane_size(vb, 0) < size)
+		return -EINVAL;
+	vb2_set_plane_payload(vb, 0, size);
+	vbuf->field = V4L2_FIELD_INTERLACED;
 	return 0;
 }
 
-static int buffer_prepare(struct videobuf_queue *q,
-	struct videobuf_buffer *vb,
-	enum v4l2_field field)
+void cx18_clear_queue(struct cx18_stream *s, enum vb2_buffer_state state)
 {
-	struct cx18_videobuf_buffer *buf =
-		container_of(vb, struct cx18_videobuf_buffer, vb);
-	struct cx18_stream *s = q->priv_data;
-	struct cx18 *cx = s->cx;
+	while (!list_empty(&s->vb_capture)) {
+		struct cx18_vb2_buffer *buf;
 
-	return cx18_prepare_buffer(q, s, buf, s->pixelformat,
-		cx->cxhdl.width, cx->cxhdl.height, field);
+		buf = list_first_entry(&s->vb_capture,
+				       struct cx18_vb2_buffer, list);
+		list_del(&buf->list);
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+	}
 }
 
-static void buffer_release(struct videobuf_queue *q,
-	struct videobuf_buffer *vb)
+static int cx18_start_streaming(struct vb2_queue *vq, unsigned int count)
 {
-	struct cx18_videobuf_buffer *buf =
-		container_of(vb, struct cx18_videobuf_buffer, vb);
-	struct cx18_stream *s = q->priv_data;
+	struct v4l2_fh *owner = vq->owner;
+	struct cx18_stream *s = vb2_get_drv_priv(vq);
+	unsigned long flags;
+	int rc;
+
+	if (WARN_ON(!owner)) {
+		rc = -EIO;
+		goto clear_queue;
+	}
 
-	cx18_dma_free(q, s, buf);
+	s->sequence = 0;
+	rc = cx18_start_capture(fh2id(owner));
+	if (!rc) {
+		/* Establish a buffer timeout */
+		mod_timer(&s->vb_timeout, msecs_to_jiffies(2000) + jiffies);
+		return 0;
+	}
+
+clear_queue:
+	spin_lock_irqsave(&s->vb_lock, flags);
+	cx18_clear_queue(s, VB2_BUF_STATE_QUEUED);
+	spin_unlock_irqrestore(&s->vb_lock, flags);
+	return rc;
 }
 
-static void buffer_queue(struct videobuf_queue *q, struct videobuf_buffer *vb)
+static void cx18_stop_streaming(struct vb2_queue *vq)
 {
-	struct cx18_videobuf_buffer *buf =
-		container_of(vb, struct cx18_videobuf_buffer, vb);
-	struct cx18_stream *s = q->priv_data;
-
-	buf->vb.state = VIDEOBUF_QUEUED;
-
-	list_add_tail(&buf->vb.queue, &s->vb_capture);
+	struct cx18_stream *s = vb2_get_drv_priv(vq);
+	unsigned long flags;
+
+	cx18_stop_capture(s, 0);
+	timer_delete_sync(&s->vb_timeout);
+	spin_lock_irqsave(&s->vb_lock, flags);
+	cx18_clear_queue(s, VB2_BUF_STATE_ERROR);
+	spin_unlock_irqrestore(&s->vb_lock, flags);
 }
 
-static const struct videobuf_queue_ops cx18_videobuf_qops = {
-	.buf_setup    = buffer_setup,
-	.buf_prepare  = buffer_prepare,
-	.buf_queue    = buffer_queue,
-	.buf_release  = buffer_release,
+static const struct vb2_ops cx18_vb2_qops = {
+	.queue_setup		= cx18_queue_setup,
+	.buf_queue		= cx18_buf_queue,
+	.buf_prepare		= cx18_buf_prepare,
+	.start_streaming	= cx18_start_streaming,
+	.stop_streaming		= cx18_stop_streaming,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
 };
 
-static void cx18_stream_init(struct cx18 *cx, int type)
+static int cx18_stream_init(struct cx18 *cx, int type)
 {
 	struct cx18_stream *s = &cx->streams[type];
+	int err = 0;
 
 	memset(s, 0, sizeof(*s));
 
@@ -275,22 +270,33 @@ static void cx18_stream_init(struct cx18 *cx, int type)
 	INIT_LIST_HEAD(&s->vb_capture);
 	timer_setup(&s->vb_timeout, cx18_vb_timeout, 0);
 	spin_lock_init(&s->vb_lock);
-	if (type == CX18_ENC_STREAM_TYPE_YUV) {
-		spin_lock_init(&s->vbuf_q_lock);
 
+	if (type == CX18_ENC_STREAM_TYPE_YUV) {
 		s->vb_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-		videobuf_queue_vmalloc_init(&s->vbuf_q, &cx18_videobuf_qops,
-			&cx->pci_dev->dev, &s->vbuf_q_lock,
-			V4L2_BUF_TYPE_VIDEO_CAPTURE,
-			V4L2_FIELD_INTERLACED,
-			sizeof(struct cx18_videobuf_buffer),
-			s, &cx->serialize_lock);
 
 		/* Assume the previous pixel default */
 		s->pixelformat = V4L2_PIX_FMT_NV12_16L16;
 		s->vb_bytes_per_frame = cx->cxhdl.height * 720 * 3 / 2;
 		s->vb_bytes_per_line = 720;
+
+		s->vidq.io_modes = VB2_READ | VB2_MMAP | VB2_DMABUF;
+		s->vidq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+		s->vidq.drv_priv = s;
+		s->vidq.buf_struct_size = sizeof(struct cx18_vb2_buffer);
+		s->vidq.ops = &cx18_vb2_qops;
+		s->vidq.mem_ops = &vb2_vmalloc_memops;
+		s->vidq.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+		s->vidq.min_buffers_needed = 2;
+		s->vidq.gfp_flags = GFP_DMA32;
+		s->vidq.dev = &cx->pci_dev->dev;
+		s->vidq.lock = &cx->serialize_lock;
+
+		err = vb2_queue_init(&s->vidq);
+		if (err)
+			v4l2_err(&cx->v4l2_dev, "cannot init vb2 queue\n");
+		s->video_dev.queue = &s->vidq;
 	}
+	return err;
 }
 
 static int cx18_prep_dev(struct cx18 *cx, int type)
@@ -299,6 +305,7 @@ static int cx18_prep_dev(struct cx18 *cx, int type)
 	u32 cap = cx->v4l2_cap;
 	int num_offset = cx18_stream_info[type].num_offset;
 	int num = cx->instance + cx18_first_minor + num_offset;
+	int err;
 
 	/*
 	 * These five fields are always initialized.
@@ -330,7 +337,9 @@ static int cx18_prep_dev(struct cx18 *cx, int type)
 		return 0;
 	}
 
-	cx18_stream_init(cx, type);
+	err = cx18_stream_init(cx, type);
+	if (err)
+		return err;
 
 	/* Allocate the cx18_dvb struct only for the TS on cards with DTV */
 	if (type == CX18_ENC_STREAM_TYPE_TS) {
@@ -356,7 +365,10 @@ static int cx18_prep_dev(struct cx18 *cx, int type)
 
 	s->video_dev.num = num;
 	s->video_dev.v4l2_dev = &cx->v4l2_dev;
-	s->video_dev.fops = &cx18_v4l2_enc_fops;
+	if (type == CX18_ENC_STREAM_TYPE_YUV)
+		s->video_dev.fops = &cx18_v4l2_enc_yuv_fops;
+	else
+		s->video_dev.fops = &cx18_v4l2_enc_fops;
 	s->video_dev.release = video_device_release_empty;
 	if (cx->card->video_inputs->video_type == CX18_CARD_INPUT_VID_TUNER)
 		s->video_dev.tvnorms = cx->tuner_std;
@@ -525,12 +537,12 @@ void cx18_streams_cleanup(struct cx18 *cx, int unregister)
 		if (vdev->v4l2_dev == NULL)
 			continue;
 
-		if (type == CX18_ENC_STREAM_TYPE_YUV)
-			videobuf_mmap_free(&cx->streams[type].vbuf_q);
-
 		cx18_stream_free(&cx->streams[type]);
 
-		video_unregister_device(vdev);
+		if (type == CX18_ENC_STREAM_TYPE_YUV)
+			vb2_video_unregister_device(vdev);
+		else
+			video_unregister_device(vdev);
 	}
 }
 
diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c
index 9232a966bcab..2ce2914576cf 100644
--- a/drivers/media/pci/cx23885/cx23885-core.c
+++ b/drivers/media/pci/cx23885/cx23885-core.c
@@ -1325,7 +1325,9 @@ void cx23885_free_buffer(struct cx23885_dev *dev, struct cx23885_buffer *buf)
 {
 	struct cx23885_riscmem *risc = &buf->risc;
 
-	dma_free_coherent(&dev->pci->dev, risc->size, risc->cpu, risc->dma);
+	if (risc->cpu)
+		dma_free_coherent(&dev->pci->dev, risc->size, risc->cpu, risc->dma);
+	memset(risc, 0, sizeof(*risc));
 }
 
 static void cx23885_tsport_reg_dump(struct cx23885_tsport *port)
diff --git a/drivers/media/pci/cx23885/cx23885-video.c b/drivers/media/pci/cx23885/cx23885-video.c
index 3d03f5e95786..671fc0588e43 100644
--- a/drivers/media/pci/cx23885/cx23885-video.c
+++ b/drivers/media/pci/cx23885/cx23885-video.c
@@ -342,6 +342,7 @@ static int queue_setup(struct vb2_queue *q,
 
 static int buffer_prepare(struct vb2_buffer *vb)
 {
+	int ret;
 	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
 	struct cx23885_dev *dev = vb->vb2_queue->drv_priv;
 	struct cx23885_buffer *buf =
@@ -358,12 +359,12 @@ static int buffer_prepare(struct vb2_buffer *vb)
 
 	switch (dev->field) {
 	case V4L2_FIELD_TOP:
-		cx23885_risc_buffer(dev->pci, &buf->risc,
+		ret = cx23885_risc_buffer(dev->pci, &buf->risc,
 				sgt->sgl, 0, UNSET,
 				buf->bpl, 0, dev->height);
 		break;
 	case V4L2_FIELD_BOTTOM:
-		cx23885_risc_buffer(dev->pci, &buf->risc,
+		ret = cx23885_risc_buffer(dev->pci, &buf->risc,
 				sgt->sgl, UNSET, 0,
 				buf->bpl, 0, dev->height);
 		break;
@@ -391,21 +392,21 @@ static int buffer_prepare(struct vb2_buffer *vb)
 			line0_offset = 0;
 			line1_offset = buf->bpl;
 		}
-		cx23885_risc_buffer(dev->pci, &buf->risc,
+		ret = cx23885_risc_buffer(dev->pci, &buf->risc,
 				sgt->sgl, line0_offset,
 				line1_offset,
 				buf->bpl, buf->bpl,
 				dev->height >> 1);
 		break;
 	case V4L2_FIELD_SEQ_TB:
-		cx23885_risc_buffer(dev->pci, &buf->risc,
+		ret = cx23885_risc_buffer(dev->pci, &buf->risc,
 				sgt->sgl,
 				0, buf->bpl * (dev->height >> 1),
 				buf->bpl, 0,
 				dev->height >> 1);
 		break;
 	case V4L2_FIELD_SEQ_BT:
-		cx23885_risc_buffer(dev->pci, &buf->risc,
+		ret = cx23885_risc_buffer(dev->pci, &buf->risc,
 				sgt->sgl,
 				buf->bpl * (dev->height >> 1), 0,
 				buf->bpl, 0,
@@ -418,7 +419,7 @@ static int buffer_prepare(struct vb2_buffer *vb)
 		buf, buf->vb.vb2_buf.index,
 		dev->width, dev->height, dev->fmt->depth, dev->fmt->fourcc,
 		(unsigned long)buf->risc.dma);
-	return 0;
+	return ret;
 }
 
 static void buffer_finish(struct vb2_buffer *vb)
diff --git a/drivers/media/pci/ddbridge/ddbridge-core.c b/drivers/media/pci/ddbridge/ddbridge-core.c
index ee8087f29b2c..40e6c873c36d 100644
--- a/drivers/media/pci/ddbridge/ddbridge-core.c
+++ b/drivers/media/pci/ddbridge/ddbridge-core.c
@@ -3117,7 +3117,6 @@ static struct device_attribute ddb_attrs_fanspeed[] = {
 
 static struct class ddb_class = {
 	.name		= "ddbridge",
-	.owner          = THIS_MODULE,
 	.devnode        = ddb_devnode,
 };
 
diff --git a/drivers/media/pci/dm1105/dm1105.c b/drivers/media/pci/dm1105/dm1105.c
index 4ac645a56c14..9e9c7c071acc 100644
--- a/drivers/media/pci/dm1105/dm1105.c
+++ b/drivers/media/pci/dm1105/dm1105.c
@@ -1176,6 +1176,7 @@ static void dm1105_remove(struct pci_dev *pdev)
 	struct dvb_demux *dvbdemux = &dev->demux;
 	struct dmx_demux *dmx = &dvbdemux->dmx;
 
+	cancel_work_sync(&dev->ir.work);
 	dm1105_ir_exit(dev);
 	dmx->close(dmx);
 	dvb_net_release(&dev->dvbnet);
diff --git a/drivers/media/pci/intel/ipu3/cio2-bridge.c b/drivers/media/pci/intel/ipu3/cio2-bridge.c
index dfefe0d8aa95..3c2accfe5455 100644
--- a/drivers/media/pci/intel/ipu3/cio2-bridge.c
+++ b/drivers/media/pci/intel/ipu3/cio2-bridge.c
@@ -29,6 +29,14 @@ static const struct cio2_sensor_config cio2_supported_sensors[] = {
 	CIO2_SENSOR_CONFIG("INT347E", 1, 319200000),
 	/* Omnivision OV2680 */
 	CIO2_SENSOR_CONFIG("OVTI2680", 0),
+	/* Omnivision ov8856 */
+	CIO2_SENSOR_CONFIG("OVTI8856", 3, 180000000, 360000000, 720000000),
+	/* Omnivision ov2740 */
+	CIO2_SENSOR_CONFIG("INT3474", 1, 360000000),
+	/* Hynix hi556 */
+	CIO2_SENSOR_CONFIG("INT3537", 1, 437000000),
+	/* Omnivision ov13b10 */
+	CIO2_SENSOR_CONFIG("OVTIDB10", 1, 560000000),
 };
 
 static const struct cio2_property_names prop_names = {
@@ -212,6 +220,7 @@ static void cio2_bridge_create_connection_swnodes(struct cio2_bridge *bridge,
 						  struct cio2_sensor *sensor)
 {
 	struct software_node *nodes = sensor->swnodes;
+	char vcm_name[ACPI_ID_LEN + 4];
 
 	cio2_bridge_init_swnode_names(sensor);
 
@@ -229,9 +238,13 @@ static void cio2_bridge_create_connection_swnodes(struct cio2_bridge *bridge,
 						sensor->node_names.endpoint,
 						&nodes[SWNODE_CIO2_PORT],
 						sensor->cio2_properties);
-	if (sensor->ssdb.vcmtype)
-		nodes[SWNODE_VCM] =
-			NODE_VCM(cio2_vcm_types[sensor->ssdb.vcmtype - 1]);
+	if (sensor->ssdb.vcmtype) {
+		/* append ssdb.link to distinguish VCM nodes with same HID */
+		snprintf(vcm_name, sizeof(vcm_name), "%s-%u",
+			 cio2_vcm_types[sensor->ssdb.vcmtype - 1],
+			 sensor->ssdb.link);
+		nodes[SWNODE_VCM] = NODE_VCM(vcm_name);
+	}
 
 	cio2_bridge_init_swnode_group(sensor);
 }
@@ -295,7 +308,6 @@ static int cio2_bridge_connect_sensor(const struct cio2_sensor_config *cfg,
 		}
 
 		sensor = &bridge->sensors[bridge->n_sensors];
-		strscpy(sensor->name, cfg->hid, sizeof(sensor->name));
 
 		ret = cio2_bridge_read_acpi_buffer(adev, "SSDB",
 						   &sensor->ssdb,
@@ -303,6 +315,9 @@ static int cio2_bridge_connect_sensor(const struct cio2_sensor_config *cfg,
 		if (ret)
 			goto err_put_adev;
 
+		snprintf(sensor->name, sizeof(sensor->name), "%s-%u",
+			 cfg->hid, sensor->ssdb.link);
+
 		if (sensor->ssdb.vcmtype > ARRAY_SIZE(cio2_vcm_types)) {
 			dev_warn(&adev->dev, "Unknown VCM type %d\n",
 				 sensor->ssdb.vcmtype);
diff --git a/drivers/media/pci/intel/ipu3/cio2-bridge.h b/drivers/media/pci/intel/ipu3/cio2-bridge.h
index b93b749c65bd..b76ed8a641e2 100644
--- a/drivers/media/pci/intel/ipu3/cio2-bridge.h
+++ b/drivers/media/pci/intel/ipu3/cio2-bridge.h
@@ -113,7 +113,8 @@ struct cio2_sensor_config {
 };
 
 struct cio2_sensor {
-	char name[ACPI_ID_LEN];
+	/* append ssdb.link(u8) in "-%u" format as suffix of HID */
+	char name[ACPI_ID_LEN + 4];
 	struct acpi_device *adev;
 	struct i2c_client *vcm_i2c_client;
 
diff --git a/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c b/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c
index 3b76a9d0383a..3c84cb121632 100644
--- a/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c
+++ b/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c
@@ -1305,6 +1305,7 @@ static int cio2_subdev_link_validate_get_format(struct media_pad *pad,
 		struct v4l2_subdev *sd =
 			media_entity_to_v4l2_subdev(pad->entity);
 
+		memset(fmt, 0, sizeof(*fmt));
 		fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
 		fmt->pad = pad->index;
 		return v4l2_subdev_call(sd, pad, get_fmt, NULL, fmt);
diff --git a/drivers/media/pci/saa7134/saa7134-cards.c b/drivers/media/pci/saa7134/saa7134-cards.c
index 99be59af3560..1280696f65f2 100644
--- a/drivers/media/pci/saa7134/saa7134-cards.c
+++ b/drivers/media/pci/saa7134/saa7134-cards.c
@@ -2116,7 +2116,6 @@ struct saa7134_board saa7134_boards[] = {
 		- Remote control doesn't initialize properly.
 		- Audio volume starts muted,
 		then gradually increases after channel change.
-		- Overlay scaling problems (application error?)
 		- Composite S-Video untested.
 		From: Konrad Rzepecki <hannibal@megapolis.pl>
 		*/
diff --git a/drivers/media/pci/saa7134/saa7134-core.c b/drivers/media/pci/saa7134/saa7134-core.c
index cf2871306987..ea0585e43abb 100644
--- a/drivers/media/pci/saa7134/saa7134-core.c
+++ b/drivers/media/pci/saa7134/saa7134-core.c
@@ -51,10 +51,6 @@ static unsigned int latency = UNSET;
 module_param(latency, int, 0444);
 MODULE_PARM_DESC(latency,"pci latency timer");
 
-int saa7134_no_overlay=-1;
-module_param_named(no_overlay, saa7134_no_overlay, int, 0444);
-MODULE_PARM_DESC(no_overlay, "allow override overlay default (0 disables, 1 enables) [some VIA/SIS chipsets are known to have problem with overlay]");
-
 bool saa7134_userptr;
 module_param(saa7134_userptr, bool, 0644);
 MODULE_PARM_DESC(saa7134_userptr, "enable page-aligned userptr support");
@@ -400,13 +396,6 @@ int saa7134_set_dmabits(struct saa7134_dev *dev)
 			SAA7134_MAIN_CTRL_TE5;
 	}
 
-	/* screen overlay -- dma 0 + video task B */
-	if (dev->ovenable) {
-		task |= 0x10;
-		ctrl |= SAA7134_MAIN_CTRL_TE1;
-		ov = dev->ovfield;
-	}
-
 	/* vbi capture -- dma 0 + vbi task A+B */
 	if (dev->vbi_q.curr) {
 		task |= 0x22;
@@ -1066,18 +1055,6 @@ static int saa7134_initdev(struct pci_dev *pci_dev,
 			latency = 0x0A;
 		}
 #endif
-		if (pci_pci_problems & (PCIPCI_FAIL|PCIAGP_FAIL)) {
-			pr_info("%s: quirk: this driver and your chipset may not work together in overlay mode.\n",
-				dev->name);
-			if (!saa7134_no_overlay) {
-				pr_info("%s: quirk: overlay mode will be disabled.\n",
-						dev->name);
-				saa7134_no_overlay = 1;
-			} else {
-				pr_info("%s: quirk: overlay mode will be forced. Use this option at your own risk.\n",
-						dev->name);
-			}
-		}
 	}
 	if (UNSET != latency) {
 		pr_info("%s: setting pci latency timer to %d\n",
@@ -1198,9 +1175,6 @@ static int saa7134_initdev(struct pci_dev *pci_dev,
 		saa_call_all(dev, core, s_power, 0);
 
 	/* register v4l devices */
-	if (saa7134_no_overlay > 0)
-		pr_info("%s: Overlay support disabled.\n", dev->name);
-
 	dev->video_dev = vdev_init(dev,&saa7134_video_template,"video");
 	dev->video_dev->ctrl_handler = &dev->ctrl_handler;
 	dev->video_dev->lock = &dev->lock;
@@ -1210,9 +1184,6 @@ static int saa7134_initdev(struct pci_dev *pci_dev,
 	if (dev->tuner_type != TUNER_ABSENT && dev->tuner_type != UNSET)
 		dev->video_dev->device_caps |= V4L2_CAP_TUNER;
 
-	if (saa7134_no_overlay <= 0)
-		dev->video_dev->device_caps |= V4L2_CAP_VIDEO_OVERLAY;
-
 	err = video_register_device(dev->video_dev,VFL_TYPE_VIDEO,
 				    video_nr[dev->nr]);
 	if (err < 0) {
@@ -1403,9 +1374,6 @@ static int __maybe_unused saa7134_suspend(struct device *dev_d)
 	struct v4l2_device *v4l2_dev = pci_get_drvdata(pci_dev);
 	struct saa7134_dev *dev = container_of(v4l2_dev, struct saa7134_dev, v4l2_dev);
 
-	/* disable overlay - apps should enable it explicitly on resume*/
-	dev->ovenable = 0;
-
 	/* Disable interrupts, DMA, and rest of the chip*/
 	saa_writel(SAA7134_IRQ1, 0);
 	saa_writel(SAA7134_IRQ2, 0);
diff --git a/drivers/media/pci/saa7134/saa7134-empress.c b/drivers/media/pci/saa7134/saa7134-empress.c
index aafbb34765b0..434fa1ee1c33 100644
--- a/drivers/media/pci/saa7134/saa7134-empress.c
+++ b/drivers/media/pci/saa7134/saa7134-empress.c
@@ -139,8 +139,8 @@ static int empress_try_fmt_vid_cap(struct file *file, void *priv,
 	struct saa7134_dev *dev = video_drvdata(file);
 	struct v4l2_subdev_pad_config pad_cfg;
 	struct v4l2_subdev_state pad_state = {
-		.pads = &pad_cfg
-		};
+		.pads = &pad_cfg,
+	};
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_TRY,
 	};
diff --git a/drivers/media/pci/saa7134/saa7134-ts.c b/drivers/media/pci/saa7134/saa7134-ts.c
index 6a5053126237..437dbe5e75e2 100644
--- a/drivers/media/pci/saa7134/saa7134-ts.c
+++ b/drivers/media/pci/saa7134/saa7134-ts.c
@@ -300,6 +300,7 @@ int saa7134_ts_start(struct saa7134_dev *dev)
 
 int saa7134_ts_fini(struct saa7134_dev *dev)
 {
+	del_timer_sync(&dev->ts_q.timeout);
 	saa7134_pgtable_free(dev->pci, &dev->ts_q.pt);
 	return 0;
 }
diff --git a/drivers/media/pci/saa7134/saa7134-vbi.c b/drivers/media/pci/saa7134/saa7134-vbi.c
index 3f0b0933eed6..3e773690468b 100644
--- a/drivers/media/pci/saa7134/saa7134-vbi.c
+++ b/drivers/media/pci/saa7134/saa7134-vbi.c
@@ -185,6 +185,7 @@ int saa7134_vbi_init1(struct saa7134_dev *dev)
 int saa7134_vbi_fini(struct saa7134_dev *dev)
 {
 	/* nothing */
+	del_timer_sync(&dev->vbi_q.timeout);
 	return 0;
 }
 
diff --git a/drivers/media/pci/saa7134/saa7134-video.c b/drivers/media/pci/saa7134/saa7134-video.c
index 4d8974c9fcc9..56b4481a40e6 100644
--- a/drivers/media/pci/saa7134/saa7134-video.c
+++ b/drivers/media/pci/saa7134/saa7134-video.c
@@ -571,193 +571,6 @@ static void set_size(struct saa7134_dev *dev, int task,
 
 /* ------------------------------------------------------------------ */
 
-struct cliplist {
-	__u16 position;
-	__u8  enable;
-	__u8  disable;
-};
-
-static void set_cliplist(struct saa7134_dev *dev, int reg,
-			struct cliplist *cl, int entries, char *name)
-{
-	__u8 winbits = 0;
-	int i;
-
-	for (i = 0; i < entries; i++) {
-		winbits |= cl[i].enable;
-		winbits &= ~cl[i].disable;
-		if (i < 15 && cl[i].position == cl[i+1].position)
-			continue;
-		saa_writeb(reg + 0, winbits);
-		saa_writeb(reg + 2, cl[i].position & 0xff);
-		saa_writeb(reg + 3, cl[i].position >> 8);
-		video_dbg("clip: %s winbits=%02x pos=%d\n",
-			name,winbits,cl[i].position);
-		reg += 8;
-	}
-	for (; reg < 0x400; reg += 8) {
-		saa_writeb(reg+ 0, 0);
-		saa_writeb(reg + 1, 0);
-		saa_writeb(reg + 2, 0);
-		saa_writeb(reg + 3, 0);
-	}
-}
-
-static int clip_range(int val)
-{
-	if (val < 0)
-		val = 0;
-	return val;
-}
-
-/* Sort into smallest position first order */
-static int cliplist_cmp(const void *a, const void *b)
-{
-	const struct cliplist *cla = a;
-	const struct cliplist *clb = b;
-	if (cla->position < clb->position)
-		return -1;
-	if (cla->position > clb->position)
-		return 1;
-	return 0;
-}
-
-static int setup_clipping(struct saa7134_dev *dev, struct v4l2_clip *clips,
-			  int nclips, int interlace)
-{
-	struct cliplist col[16], row[16];
-	int cols = 0, rows = 0, i;
-	int div = interlace ? 2 : 1;
-
-	memset(col, 0, sizeof(col));
-	memset(row, 0, sizeof(row));
-	for (i = 0; i < nclips && i < 8; i++) {
-		col[cols].position = clip_range(clips[i].c.left);
-		col[cols].enable   = (1 << i);
-		cols++;
-		col[cols].position = clip_range(clips[i].c.left+clips[i].c.width);
-		col[cols].disable  = (1 << i);
-		cols++;
-		row[rows].position = clip_range(clips[i].c.top / div);
-		row[rows].enable   = (1 << i);
-		rows++;
-		row[rows].position = clip_range((clips[i].c.top + clips[i].c.height)
-						/ div);
-		row[rows].disable  = (1 << i);
-		rows++;
-	}
-	sort(col, cols, sizeof col[0], cliplist_cmp, NULL);
-	sort(row, rows, sizeof row[0], cliplist_cmp, NULL);
-	set_cliplist(dev,0x380,col,cols,"cols");
-	set_cliplist(dev,0x384,row,rows,"rows");
-	return 0;
-}
-
-static int verify_preview(struct saa7134_dev *dev, struct v4l2_window *win, bool try)
-{
-	enum v4l2_field field;
-	int maxw, maxh;
-
-	if (!try && (dev->ovbuf.base == NULL || dev->ovfmt == NULL))
-		return -EINVAL;
-	if (win->w.width < 48)
-		win->w.width = 48;
-	if (win->w.height < 32)
-		win->w.height = 32;
-	if (win->clipcount > 8)
-		win->clipcount = 8;
-
-	win->chromakey = 0;
-	win->global_alpha = 0;
-	field = win->field;
-	maxw  = dev->crop_current.width;
-	maxh  = dev->crop_current.height;
-
-	if (V4L2_FIELD_ANY == field) {
-		field = (win->w.height > maxh/2)
-			? V4L2_FIELD_INTERLACED
-			: V4L2_FIELD_TOP;
-	}
-	switch (field) {
-	case V4L2_FIELD_TOP:
-	case V4L2_FIELD_BOTTOM:
-		maxh = maxh / 2;
-		break;
-	default:
-		field = V4L2_FIELD_INTERLACED;
-		break;
-	}
-
-	win->field = field;
-	if (win->w.width > maxw)
-		win->w.width = maxw;
-	if (win->w.height > maxh)
-		win->w.height = maxh;
-	return 0;
-}
-
-static int start_preview(struct saa7134_dev *dev)
-{
-	unsigned long base,control,bpl;
-	int err;
-
-	err = verify_preview(dev, &dev->win, false);
-	if (0 != err)
-		return err;
-
-	dev->ovfield = dev->win.field;
-	video_dbg("%s %dx%d+%d+%d 0x%08x field=%s\n", __func__,
-		  dev->win.w.width, dev->win.w.height,
-		  dev->win.w.left, dev->win.w.top,
-		  dev->ovfmt->fourcc, v4l2_field_names[dev->ovfield]);
-
-	/* setup window + clipping */
-	set_size(dev, TASK_B, dev->win.w.width, dev->win.w.height,
-		 V4L2_FIELD_HAS_BOTH(dev->ovfield));
-	setup_clipping(dev, dev->clips, dev->nclips,
-		       V4L2_FIELD_HAS_BOTH(dev->ovfield));
-	if (dev->ovfmt->yuv)
-		saa_andorb(SAA7134_DATA_PATH(TASK_B), 0x3f, 0x03);
-	else
-		saa_andorb(SAA7134_DATA_PATH(TASK_B), 0x3f, 0x01);
-	saa_writeb(SAA7134_OFMT_VIDEO_B, dev->ovfmt->pm | 0x20);
-
-	/* dma: setup channel 1 (= Video Task B) */
-	base  = (unsigned long)dev->ovbuf.base;
-	base += dev->ovbuf.fmt.bytesperline * dev->win.w.top;
-	base += dev->ovfmt->depth/8         * dev->win.w.left;
-	bpl   = dev->ovbuf.fmt.bytesperline;
-	control = SAA7134_RS_CONTROL_BURST_16;
-	if (dev->ovfmt->bswap)
-		control |= SAA7134_RS_CONTROL_BSWAP;
-	if (dev->ovfmt->wswap)
-		control |= SAA7134_RS_CONTROL_WSWAP;
-	if (V4L2_FIELD_HAS_BOTH(dev->ovfield)) {
-		saa_writel(SAA7134_RS_BA1(1),base);
-		saa_writel(SAA7134_RS_BA2(1),base+bpl);
-		saa_writel(SAA7134_RS_PITCH(1),bpl*2);
-		saa_writel(SAA7134_RS_CONTROL(1),control);
-	} else {
-		saa_writel(SAA7134_RS_BA1(1),base);
-		saa_writel(SAA7134_RS_BA2(1),base);
-		saa_writel(SAA7134_RS_PITCH(1),bpl);
-		saa_writel(SAA7134_RS_CONTROL(1),control);
-	}
-
-	/* start dma */
-	dev->ovenable = 1;
-	saa7134_set_dmabits(dev);
-
-	return 0;
-}
-
-static int stop_preview(struct saa7134_dev *dev)
-{
-	dev->ovenable = 0;
-	saa7134_set_dmabits(dev);
-	return 0;
-}
-
 /*
  * Media Controller helper functions
  */
@@ -1042,8 +855,6 @@ static const struct vb2_ops vb2_qops = {
 static int saa7134_s_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct saa7134_dev *dev = container_of(ctrl->handler, struct saa7134_dev, ctrl_handler);
-	unsigned long flags;
-	int restart_overlay = 0;
 
 	switch (ctrl->id) {
 	case V4L2_CID_BRIGHTNESS:
@@ -1081,15 +892,12 @@ static int saa7134_s_ctrl(struct v4l2_ctrl *ctrl)
 		break;
 	case V4L2_CID_HFLIP:
 		dev->ctl_mirror = ctrl->val;
-		restart_overlay = 1;
 		break;
 	case V4L2_CID_PRIVATE_Y_EVEN:
 		dev->ctl_y_even = ctrl->val;
-		restart_overlay = 1;
 		break;
 	case V4L2_CID_PRIVATE_Y_ODD:
 		dev->ctl_y_odd = ctrl->val;
-		restart_overlay = 1;
 		break;
 	case V4L2_CID_PRIVATE_AUTOMUTE:
 	{
@@ -1112,12 +920,6 @@ static int saa7134_s_ctrl(struct v4l2_ctrl *ctrl)
 	default:
 		return -EINVAL;
 	}
-	if (restart_overlay && dev->overlay_owner) {
-		spin_lock_irqsave(&dev->slock, flags);
-		stop_preview(dev);
-		start_preview(dev);
-		spin_unlock_irqrestore(&dev->slock, flags);
-	}
 	return 0;
 }
 
@@ -1150,21 +952,11 @@ static int video_release(struct file *file)
 {
 	struct video_device *vdev = video_devdata(file);
 	struct saa7134_dev *dev = video_drvdata(file);
-	struct v4l2_fh *fh = file->private_data;
 	struct saa6588_command cmd;
-	unsigned long flags;
 
 	mutex_lock(&dev->lock);
 	saa7134_tvaudio_close(dev);
 
-	/* turn off overlay */
-	if (fh == dev->overlay_owner) {
-		spin_lock_irqsave(&dev->slock,flags);
-		stop_preview(dev);
-		spin_unlock_irqrestore(&dev->slock,flags);
-		dev->overlay_owner = NULL;
-	}
-
 	if (vdev->vfl_type == VFL_TYPE_RADIO)
 		v4l2_fh_release(file);
 	else
@@ -1261,34 +1053,6 @@ static int saa7134_g_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
-static int saa7134_g_fmt_vid_overlay(struct file *file, void *priv,
-				struct v4l2_format *f)
-{
-	struct saa7134_dev *dev = video_drvdata(file);
-	u32 clipcount = f->fmt.win.clipcount;
-	int i;
-
-	if (saa7134_no_overlay > 0) {
-		pr_err("V4L2_BUF_TYPE_VIDEO_OVERLAY: no_overlay\n");
-		return -EINVAL;
-	}
-	f->fmt.win = dev->win;
-	if (!f->fmt.win.clips) {
-		f->fmt.win.clipcount = 0;
-		return 0;
-	}
-	if (dev->nclips < clipcount)
-		clipcount = dev->nclips;
-	f->fmt.win.clipcount = clipcount;
-
-	for (i = 0; i < clipcount; i++) {
-		memcpy(&f->fmt.win.clips[i].c, &dev->clips[i].c,
-		       sizeof(struct v4l2_rect));
-	}
-
-	return 0;
-}
-
 static int saa7134_try_fmt_vid_cap(struct file *file, void *priv,
 						struct v4l2_format *f)
 {
@@ -1342,21 +1106,6 @@ static int saa7134_try_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
-static int saa7134_try_fmt_vid_overlay(struct file *file, void *priv,
-						struct v4l2_format *f)
-{
-	struct saa7134_dev *dev = video_drvdata(file);
-
-	if (saa7134_no_overlay > 0) {
-		pr_err("V4L2_BUF_TYPE_VIDEO_OVERLAY: no_overlay\n");
-		return -EINVAL;
-	}
-
-	if (f->fmt.win.clips == NULL)
-		f->fmt.win.clipcount = 0;
-	return verify_preview(dev, &f->fmt.win, true);
-}
-
 static int saa7134_s_fmt_vid_cap(struct file *file, void *priv,
 					struct v4l2_format *f)
 {
@@ -1374,39 +1123,6 @@ static int saa7134_s_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
-static int saa7134_s_fmt_vid_overlay(struct file *file, void *priv,
-					struct v4l2_format *f)
-{
-	struct saa7134_dev *dev = video_drvdata(file);
-	int err;
-	unsigned long flags;
-
-	if (saa7134_no_overlay > 0) {
-		pr_err("V4L2_BUF_TYPE_VIDEO_OVERLAY: no_overlay\n");
-		return -EINVAL;
-	}
-	if (f->fmt.win.clips == NULL)
-		f->fmt.win.clipcount = 0;
-	err = verify_preview(dev, &f->fmt.win, true);
-	if (0 != err)
-		return err;
-
-	dev->win    = f->fmt.win;
-	dev->nclips = f->fmt.win.clipcount;
-
-	memcpy(dev->clips, f->fmt.win.clips,
-	       sizeof(struct v4l2_clip) * dev->nclips);
-
-	if (priv == dev->overlay_owner) {
-		spin_lock_irqsave(&dev->slock, flags);
-		stop_preview(dev);
-		start_preview(dev);
-		spin_unlock_irqrestore(&dev->slock, flags);
-	}
-
-	return 0;
-}
-
 int saa7134_enum_input(struct file *file, void *priv, struct v4l2_input *i)
 {
 	struct saa7134_dev *dev = video_drvdata(file);
@@ -1482,8 +1198,6 @@ int saa7134_querycap(struct file *file, void *priv,
 		cap->capabilities |= V4L2_CAP_TUNER;
 	if (dev->has_rds)
 		cap->capabilities |= V4L2_CAP_RDS_CAPTURE;
-	if (saa7134_no_overlay <= 0)
-		cap->capabilities |= V4L2_CAP_VIDEO_OVERLAY;
 
 	return 0;
 }
@@ -1492,17 +1206,9 @@ EXPORT_SYMBOL_GPL(saa7134_querycap);
 int saa7134_s_std(struct file *file, void *priv, v4l2_std_id id)
 {
 	struct saa7134_dev *dev = video_drvdata(file);
-	struct v4l2_fh *fh = priv;
-	unsigned long flags;
 	unsigned int i;
 	v4l2_std_id fixup;
 
-	if (is_empress(file) && dev->overlay_owner) {
-		/* Don't change the std from the mpeg device
-		   if overlay is active. */
-		return -EBUSY;
-	}
-
 	for (i = 0; i < TVNORMS; i++)
 		if (id == tvnorms[i].id)
 			break;
@@ -1534,18 +1240,7 @@ int saa7134_s_std(struct file *file, void *priv, v4l2_std_id id)
 			return -EINVAL;
 	}
 
-	if (!is_empress(file) && fh == dev->overlay_owner) {
-		spin_lock_irqsave(&dev->slock, flags);
-		stop_preview(dev);
-		spin_unlock_irqrestore(&dev->slock, flags);
-
-		set_tvnorm(dev, &tvnorms[i]);
-
-		spin_lock_irqsave(&dev->slock, flags);
-		start_preview(dev);
-		spin_unlock_irqrestore(&dev->slock, flags);
-	} else
-		set_tvnorm(dev, &tvnorms[i]);
+	set_tvnorm(dev, &tvnorms[i]);
 
 	saa7134_tvaudio_do_scan(dev);
 	return 0;
@@ -1595,8 +1290,7 @@ static int saa7134_g_pixelaspect(struct file *file, void *priv,
 {
 	struct saa7134_dev *dev = video_drvdata(file);
 
-	if (type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
-	    type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+	if (type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 
 	if (dev->tvnorm->id & V4L2_STD_525_60) {
@@ -1614,8 +1308,7 @@ static int saa7134_g_selection(struct file *file, void *f, struct v4l2_selection
 {
 	struct saa7134_dev *dev = video_drvdata(file);
 
-	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
-	    sel->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 
 	switch (sel->target) {
@@ -1640,15 +1333,12 @@ static int saa7134_s_selection(struct file *file, void *f, struct v4l2_selection
 	struct v4l2_rect *b = &dev->crop_bounds;
 	struct v4l2_rect *c = &dev->crop_current;
 
-	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
-	    sel->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 
 	if (sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
 
-	if (dev->overlay_owner)
-		return -EBUSY;
 	if (vb2_is_streaming(&dev->video_vbq))
 		return -EBUSY;
 
@@ -1764,85 +1454,6 @@ static int saa7134_enum_fmt_vid_cap(struct file *file, void  *priv,
 	return 0;
 }
 
-static int saa7134_enum_fmt_vid_overlay(struct file *file, void  *priv,
-					struct v4l2_fmtdesc *f)
-{
-	if (saa7134_no_overlay > 0) {
-		pr_err("V4L2_BUF_TYPE_VIDEO_OVERLAY: no_overlay\n");
-		return -EINVAL;
-	}
-
-	if ((f->index >= FORMATS) || formats[f->index].planar)
-		return -EINVAL;
-
-	f->pixelformat = formats[f->index].fourcc;
-
-	return 0;
-}
-
-static int saa7134_g_fbuf(struct file *file, void *f,
-				struct v4l2_framebuffer *fb)
-{
-	struct saa7134_dev *dev = video_drvdata(file);
-
-	*fb = dev->ovbuf;
-	fb->capability = V4L2_FBUF_CAP_LIST_CLIPPING;
-
-	return 0;
-}
-
-static int saa7134_s_fbuf(struct file *file, void *f,
-					const struct v4l2_framebuffer *fb)
-{
-	struct saa7134_dev *dev = video_drvdata(file);
-	struct saa7134_format *fmt;
-
-	if (!capable(CAP_SYS_ADMIN) &&
-	   !capable(CAP_SYS_RAWIO))
-		return -EPERM;
-
-	/* check args */
-	fmt = format_by_fourcc(fb->fmt.pixelformat);
-	if (NULL == fmt)
-		return -EINVAL;
-
-	/* ok, accept it */
-	dev->ovbuf = *fb;
-	dev->ovfmt = fmt;
-	if (0 == dev->ovbuf.fmt.bytesperline)
-		dev->ovbuf.fmt.bytesperline =
-			dev->ovbuf.fmt.width*fmt->depth/8;
-	return 0;
-}
-
-static int saa7134_overlay(struct file *file, void *priv, unsigned int on)
-{
-	struct saa7134_dev *dev = video_drvdata(file);
-	unsigned long flags;
-
-	if (on) {
-		if (saa7134_no_overlay > 0) {
-			video_dbg("no_overlay\n");
-			return -EINVAL;
-		}
-
-		if (dev->overlay_owner && priv != dev->overlay_owner)
-			return -EBUSY;
-		dev->overlay_owner = priv;
-		spin_lock_irqsave(&dev->slock, flags);
-		start_preview(dev);
-		spin_unlock_irqrestore(&dev->slock, flags);
-	} else {
-		if (priv != dev->overlay_owner)
-			return -EINVAL;
-		spin_lock_irqsave(&dev->slock, flags);
-		stop_preview(dev);
-		spin_unlock_irqrestore(&dev->slock, flags);
-		dev->overlay_owner = NULL;
-	}
-	return 0;
-}
-
 #ifdef CONFIG_VIDEO_ADV_DEBUG
 static int vidioc_g_register (struct file *file, void *priv,
 			      struct v4l2_dbg_register *reg)
@@ -1912,10 +1523,6 @@ static const struct v4l2_ioctl_ops video_ioctl_ops = {
 	.vidioc_g_fmt_vid_cap		= saa7134_g_fmt_vid_cap,
 	.vidioc_try_fmt_vid_cap		= saa7134_try_fmt_vid_cap,
 	.vidioc_s_fmt_vid_cap		= saa7134_s_fmt_vid_cap,
-	.vidioc_enum_fmt_vid_overlay	= saa7134_enum_fmt_vid_overlay,
-	.vidioc_g_fmt_vid_overlay	= saa7134_g_fmt_vid_overlay,
-	.vidioc_try_fmt_vid_overlay	= saa7134_try_fmt_vid_overlay,
-	.vidioc_s_fmt_vid_overlay	= saa7134_s_fmt_vid_overlay,
 	.vidioc_g_fmt_vbi_cap		= saa7134_try_get_set_fmt_vbi_cap,
 	.vidioc_try_fmt_vbi_cap		= saa7134_try_get_set_fmt_vbi_cap,
 	.vidioc_s_fmt_vbi_cap		= saa7134_try_get_set_fmt_vbi_cap,
@@ -1937,9 +1544,6 @@ static const struct v4l2_ioctl_ops video_ioctl_ops = {
 	.vidioc_s_tuner			= saa7134_s_tuner,
 	.vidioc_g_selection		= saa7134_g_selection,
 	.vidioc_s_selection		= saa7134_s_selection,
-	.vidioc_g_fbuf			= saa7134_g_fbuf,
-	.vidioc_s_fbuf			= saa7134_s_fbuf,
-	.vidioc_overlay			= saa7134_overlay,
 	.vidioc_g_frequency		= saa7134_g_frequency,
 	.vidioc_s_frequency		= saa7134_s_frequency,
 #ifdef CONFIG_VIDEO_ADV_DEBUG
@@ -2086,13 +1690,6 @@ int saa7134_video_init1(struct saa7134_dev *dev)
 	dev->width    = 720;
 	dev->height   = 576;
 	dev->field = V4L2_FIELD_INTERLACED;
-	dev->win.w.width = dev->width;
-	dev->win.w.height = dev->height;
-	dev->win.field = V4L2_FIELD_INTERLACED;
-	dev->ovbuf.fmt.width = dev->width;
-	dev->ovbuf.fmt.height = dev->height;
-	dev->ovbuf.fmt.pixelformat = dev->fmt->fourcc;
-	dev->ovbuf.fmt.colorspace = V4L2_COLORSPACE_SMPTE170M;
 
 	if (saa7134_boards[dev->board].video_out)
 		saa7134_videoport_init(dev);
@@ -2146,6 +1743,7 @@ int saa7134_video_init1(struct saa7134_dev *dev)
 
 void saa7134_video_fini(struct saa7134_dev *dev)
 {
+	del_timer_sync(&dev->video_q.timeout);
 	/* free stuff */
 	saa7134_pgtable_free(dev->pci, &dev->video_q.pt);
 	saa7134_pgtable_free(dev->pci, &dev->vbi_q.pt);
diff --git a/drivers/media/pci/saa7134/saa7134.h b/drivers/media/pci/saa7134/saa7134.h
index 5c9b2912a9d1..9f27e3775c27 100644
--- a/drivers/media/pci/saa7134/saa7134.h
+++ b/drivers/media/pci/saa7134/saa7134.h
@@ -429,7 +429,6 @@ struct saa7134_board {
 /* ----------------------------------------------------------- */
 /* device / file handle status                                 */
 
-#define RESOURCE_OVERLAY       1
 #define RESOURCE_VIDEO         2
 #define RESOURCE_VBI           4
 #define RESOURCE_EMPRESS       8
@@ -589,17 +588,6 @@ struct saa7134_dev {
 	unsigned char              eedata[256];
 	int			   has_rds;
 
-	/* video overlay */
-	struct v4l2_framebuffer    ovbuf;
-	struct saa7134_format      *ovfmt;
-	unsigned int               ovenable;
-	enum v4l2_field            ovfield;
-	struct v4l2_window         win;
-	struct v4l2_clip           clips[8];
-	unsigned int               nclips;
-	struct v4l2_fh		   *overlay_owner;
-
-
 	/* video+ts+vbi capture */
 	struct saa7134_dmaqueue    video_q;
 	struct vb2_queue           video_vbq;
@@ -745,7 +733,6 @@ static inline bool is_empress(struct file *file)
 
 extern struct list_head  saa7134_devlist;
 extern struct mutex saa7134_devlist_lock;
-extern int saa7134_no_overlay;
 extern bool saa7134_userptr;
 
 void saa7134_track_gpio(struct saa7134_dev *dev, const char *msg);
diff --git a/drivers/media/pci/saa7146/hexium_gemini.c b/drivers/media/pci/saa7146/hexium_gemini.c
index 3947701cd6c7..40b35098f3ea 100644
--- a/drivers/media/pci/saa7146/hexium_gemini.c
+++ b/drivers/media/pci/saa7146/hexium_gemini.c
@@ -27,17 +27,18 @@ static int hexium_num;
 #define HEXIUM_GEMINI			4
 #define HEXIUM_GEMINI_DUAL		5
 
+#define HEXIUM_STD (V4L2_STD_PAL | V4L2_STD_SECAM | V4L2_STD_NTSC)
 #define HEXIUM_INPUTS	9
 static struct v4l2_input hexium_inputs[HEXIUM_INPUTS] = {
-	{ 0, "CVBS 1",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 1, "CVBS 2",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 2, "CVBS 3",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 3, "CVBS 4",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 4, "CVBS 5",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 5, "CVBS 6",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 6, "Y/C 1",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 7, "Y/C 2",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 8, "Y/C 3",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
+	{ 0, "CVBS 1",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 1, "CVBS 2",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 2, "CVBS 3",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 3, "CVBS 4",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 4, "CVBS 5",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 5, "CVBS 6",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 6, "Y/C 1",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 7, "Y/C 2",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 8, "Y/C 3",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
 };
 
 #define HEXIUM_AUDIOS	0
@@ -215,7 +216,7 @@ static int vidioc_enum_input(struct file *file, void *fh, struct v4l2_input *i)
 
 static int vidioc_g_input(struct file *file, void *fh, unsigned int *input)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct hexium *hexium = (struct hexium *) dev->ext_priv;
 
 	*input = hexium->cur_input;
@@ -226,7 +227,7 @@ static int vidioc_g_input(struct file *file, void *fh, unsigned int *input)
 
 static int vidioc_s_input(struct file *file, void *fh, unsigned int input)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct hexium *hexium = (struct hexium *) dev->ext_priv;
 
 	DEB_EE("VIDIOC_S_INPUT %d\n", input);
diff --git a/drivers/media/pci/saa7146/hexium_orion.c b/drivers/media/pci/saa7146/hexium_orion.c
index 2eb4bee16b71..a2076728c621 100644
--- a/drivers/media/pci/saa7146/hexium_orion.c
+++ b/drivers/media/pci/saa7146/hexium_orion.c
@@ -28,17 +28,18 @@ static int hexium_num;
 #define HEXIUM_ORION_1SVHS_3BNC		2
 #define HEXIUM_ORION_4BNC		3
 
+#define HEXIUM_STD (V4L2_STD_PAL | V4L2_STD_SECAM | V4L2_STD_NTSC)
 #define HEXIUM_INPUTS	9
 static struct v4l2_input hexium_inputs[HEXIUM_INPUTS] = {
-	{ 0, "CVBS 1",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 1, "CVBS 2",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 2, "CVBS 3",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 3, "CVBS 4",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 4, "CVBS 5",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 5, "CVBS 6",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 6, "Y/C 1",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 7, "Y/C 2",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
-	{ 8, "Y/C 3",	V4L2_INPUT_TYPE_CAMERA,	0, 0, V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
+	{ 0, "CVBS 1",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 1, "CVBS 2",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 2, "CVBS 3",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 3, "CVBS 4",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 4, "CVBS 5",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 5, "CVBS 6",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 6, "Y/C 1",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 7, "Y/C 2",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
+	{ 8, "Y/C 3",	V4L2_INPUT_TYPE_CAMERA,	0, 0, HEXIUM_STD, 0, V4L2_IN_CAP_STD },
 };
 
 #define HEXIUM_AUDIOS	0
@@ -326,7 +327,7 @@ static int vidioc_enum_input(struct file *file, void *fh, struct v4l2_input *i)
 
 static int vidioc_g_input(struct file *file, void *fh, unsigned int *input)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct hexium *hexium = (struct hexium *) dev->ext_priv;
 
 	*input = hexium->cur_input;
@@ -337,7 +338,7 @@ static int vidioc_g_input(struct file *file, void *fh, unsigned int *input)
 
 static int vidioc_s_input(struct file *file, void *fh, unsigned int input)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct hexium *hexium = (struct hexium *) dev->ext_priv;
 
 	if (input >= HEXIUM_INPUTS)
@@ -379,6 +380,7 @@ static int hexium_attach(struct saa7146_dev *dev, struct saa7146_pci_extension_d
 	/* the rest */
 	hexium->cur_input = 0;
 	hexium_init_done(dev);
+	hexium_set_input(hexium, 0);
 
 	return 0;
 }
diff --git a/drivers/media/pci/saa7146/mxb.c b/drivers/media/pci/saa7146/mxb.c
index 7ded8f5b05cb..a14b839098b8 100644
--- a/drivers/media/pci/saa7146/mxb.c
+++ b/drivers/media/pci/saa7146/mxb.c
@@ -48,6 +48,7 @@ static int debug;
 module_param(debug, int, 0644);
 MODULE_PARM_DESC(debug, "Turn on/off device debugging (default:off).");
 
+#define MXB_STD (V4L2_STD_PAL_BG | V4L2_STD_PAL_I | V4L2_STD_SECAM | V4L2_STD_NTSC)
 #define MXB_INPUTS 4
 enum { TUNER, AUX1, AUX3, AUX3_YC };
 
@@ -55,11 +56,11 @@ static struct v4l2_input mxb_inputs[MXB_INPUTS] = {
 	{ TUNER,   "Tuner",          V4L2_INPUT_TYPE_TUNER,  0x3f, 0,
 		V4L2_STD_PAL_BG | V4L2_STD_PAL_I, 0, V4L2_IN_CAP_STD },
 	{ AUX1,	   "AUX1",           V4L2_INPUT_TYPE_CAMERA, 0x3f, 0,
-		V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
+		MXB_STD, 0, V4L2_IN_CAP_STD },
 	{ AUX3,	   "AUX3 Composite", V4L2_INPUT_TYPE_CAMERA, 0x3f, 0,
-		V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
+		MXB_STD, 0, V4L2_IN_CAP_STD },
 	{ AUX3_YC, "AUX3 S-Video",   V4L2_INPUT_TYPE_CAMERA, 0x3f, 0,
-		V4L2_STD_ALL, 0, V4L2_IN_CAP_STD },
+		MXB_STD, 0, V4L2_IN_CAP_STD },
 };
 
 /* this array holds the information, which port of the saa7146 each
@@ -456,7 +457,7 @@ static int vidioc_enum_input(struct file *file, void *fh, struct v4l2_input *i)
 
 static int vidioc_g_input(struct file *file, void *fh, unsigned int *i)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct mxb *mxb = (struct mxb *)dev->ext_priv;
 	*i = mxb->cur_input;
 
@@ -466,7 +467,7 @@ static int vidioc_g_input(struct file *file, void *fh, unsigned int *i)
 
 static int vidioc_s_input(struct file *file, void *fh, unsigned int input)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct mxb *mxb = (struct mxb *)dev->ext_priv;
 	int err = 0;
 	int i = 0;
@@ -512,6 +513,9 @@ static int vidioc_s_input(struct file *file, void *fh, unsigned int input)
 	if (err)
 		return err;
 
+	mxb->video_dev.tvnorms = mxb_inputs[input].std;
+	mxb->vbi_dev.tvnorms = mxb_inputs[input].std;
+
 	/* switch video in saa7111a */
 	if (saa7111a_call(mxb, video, s_routing, i, SAA7111_FMT_CCIR, 0))
 		pr_err("VIDIOC_S_INPUT: could not address saa7111a\n");
@@ -528,7 +532,7 @@ static int vidioc_s_input(struct file *file, void *fh, unsigned int input)
 
 static int vidioc_g_tuner(struct file *file, void *fh, struct v4l2_tuner *t)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct mxb *mxb = (struct mxb *)dev->ext_priv;
 
 	if (t->index) {
@@ -550,7 +554,7 @@ static int vidioc_g_tuner(struct file *file, void *fh, struct v4l2_tuner *t)
 
 static int vidioc_s_tuner(struct file *file, void *fh, const struct v4l2_tuner *t)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct mxb *mxb = (struct mxb *)dev->ext_priv;
 
 	if (t->index) {
@@ -565,14 +569,14 @@ static int vidioc_s_tuner(struct file *file, void *fh, const struct v4l2_tuner *
 
 static int vidioc_querystd(struct file *file, void *fh, v4l2_std_id *norm)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 
 	return call_all(dev, video, querystd, norm);
 }
 
 static int vidioc_g_frequency(struct file *file, void *fh, struct v4l2_frequency *f)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct mxb *mxb = (struct mxb *)dev->ext_priv;
 
 	if (f->tuner)
@@ -585,9 +589,8 @@ static int vidioc_g_frequency(struct file *file, void *fh, struct v4l2_frequency
 
 static int vidioc_s_frequency(struct file *file, void *fh, const struct v4l2_frequency *f)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct mxb *mxb = (struct mxb *)dev->ext_priv;
-	struct saa7146_vv *vv = dev->vv_data;
 
 	if (f->tuner)
 		return -EINVAL;
@@ -604,15 +607,6 @@ static int vidioc_s_frequency(struct file *file, void *fh, const struct v4l2_fre
 	tuner_call(mxb, tuner, g_frequency, &mxb->cur_freq);
 	if (mxb->cur_audinput == 0)
 		mxb_update_audmode(mxb);
-
-	if (mxb->cur_input)
-		return 0;
-
-	/* hack: changing the frequency should invalidate the vbi-counter (=> alevt) */
-	spin_lock(&dev->slock);
-	vv->vbi_fieldcount = 0;
-	spin_unlock(&dev->slock);
-
 	return 0;
 }
 
@@ -626,7 +620,7 @@ static int vidioc_enumaudio(struct file *file, void *fh, struct v4l2_audio *a)
 
 static int vidioc_g_audio(struct file *file, void *fh, struct v4l2_audio *a)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct mxb *mxb = (struct mxb *)dev->ext_priv;
 
 	DEB_EE("VIDIOC_G_AUDIO\n");
@@ -636,7 +630,7 @@ static int vidioc_g_audio(struct file *file, void *fh, struct v4l2_audio *a)
 
 static int vidioc_s_audio(struct file *file, void *fh, const struct v4l2_audio *a)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct mxb *mxb = (struct mxb *)dev->ext_priv;
 
 	DEB_D("VIDIOC_S_AUDIO %d\n", a->index);
@@ -656,7 +650,7 @@ static int vidioc_s_audio(struct file *file, void *fh, const struct v4l2_audio *
 #ifdef CONFIG_VIDEO_ADV_DEBUG
 static int vidioc_g_register(struct file *file, void *fh, struct v4l2_dbg_register *reg)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 
 	if (reg->reg > pci_resource_len(dev->pci, 0) - 4)
 		return -EINVAL;
@@ -667,7 +661,7 @@ static int vidioc_g_register(struct file *file, void *fh, struct v4l2_dbg_regist
 
 static int vidioc_s_register(struct file *file, void *fh, const struct v4l2_dbg_register *reg)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 
 	if (reg->reg > pci_resource_len(dev->pci, 0) - 4)
 		return -EINVAL;
@@ -713,6 +707,17 @@ static int mxb_attach(struct saa7146_dev *dev, struct saa7146_pci_extension_data
 	vv_data.vid_ops.vidioc_g_register = vidioc_g_register;
 	vv_data.vid_ops.vidioc_s_register = vidioc_s_register;
 #endif
+	vv_data.vbi_ops.vidioc_enum_input = vidioc_enum_input;
+	vv_data.vbi_ops.vidioc_g_input = vidioc_g_input;
+	vv_data.vbi_ops.vidioc_s_input = vidioc_s_input;
+	vv_data.vbi_ops.vidioc_querystd = vidioc_querystd;
+	vv_data.vbi_ops.vidioc_g_tuner = vidioc_g_tuner;
+	vv_data.vbi_ops.vidioc_s_tuner = vidioc_s_tuner;
+	vv_data.vbi_ops.vidioc_g_frequency = vidioc_g_frequency;
+	vv_data.vbi_ops.vidioc_s_frequency = vidioc_s_frequency;
+	vv_data.vbi_ops.vidioc_enumaudio = vidioc_enumaudio;
+	vv_data.vbi_ops.vidioc_g_audio = vidioc_g_audio;
+	vv_data.vbi_ops.vidioc_s_audio = vidioc_s_audio;
 	if (saa7146_register_device(&mxb->video_dev, dev, "mxb", VFL_TYPE_VIDEO)) {
 		ERR("cannot register capture v4l2 device. skipping.\n");
 		saa7146_vv_release(dev);
diff --git a/drivers/media/pci/sta2x11/sta2x11_vip.c b/drivers/media/pci/sta2x11/sta2x11_vip.c
index 8535e49a4c4f..e4cf9d63e926 100644
--- a/drivers/media/pci/sta2x11/sta2x11_vip.c
+++ b/drivers/media/pci/sta2x11/sta2x11_vip.c
@@ -18,6 +18,7 @@
 #include <linux/pci.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/gpio/consumer.h>
 #include <linux/gpio.h>
 #include <linux/i2c.h>
 #include <linux/delay.h>
@@ -889,6 +890,7 @@ static int sta2x11_vip_init_controls(struct sta2x11_vip *vip)
 static int vip_gpio_reserve(struct device *dev, int pin, int dir,
 			    const char *name)
 {
+	struct gpio_desc *desc = gpio_to_desc(pin);
 	int ret = -ENODEV;
 
 	if (!gpio_is_valid(pin))
@@ -900,7 +902,7 @@ static int vip_gpio_reserve(struct device *dev, int pin, int dir,
 		return ret;
 	}
 
-	ret = gpio_direction_output(pin, dir);
+	ret = gpiod_direction_output(desc, dir);
 	if (ret) {
 		dev_err(dev, "Failed to set direction for pin %d (%s)\n",
 			pin, name);
@@ -908,7 +910,7 @@ static int vip_gpio_reserve(struct device *dev, int pin, int dir,
 		return ret;
 	}
 
-	ret = gpio_export(pin, false);
+	ret = gpiod_export(desc, false);
 	if (ret) {
 		dev_err(dev, "Failed to export pin %d (%s)\n", pin, name);
 		gpio_free(pin);
@@ -928,8 +930,10 @@ static int vip_gpio_reserve(struct device *dev, int pin, int dir,
 static void vip_gpio_release(struct device *dev, int pin, const char *name)
 {
 	if (gpio_is_valid(pin)) {
+		struct gpio_desc *desc = gpio_to_desc(pin);
+
 		dev_dbg(dev, "releasing pin %d (%s)\n",	pin, name);
-		gpio_unexport(pin);
+		gpiod_unexport(desc);
 		gpio_free(pin);
 	}
 }
diff --git a/drivers/media/pci/ttpci/budget-av.c b/drivers/media/pci/ttpci/budget-av.c
index 3cb83005cf09..824529f3c74b 100644
--- a/drivers/media/pci/ttpci/budget-av.c
+++ b/drivers/media/pci/ttpci/budget-av.c
@@ -31,6 +31,7 @@
 #include "dvb-pll.h"
 #include <media/drv-intf/saa7146_vv.h>
 #include <linux/module.h>
+#include <linux/etherdevice.h>
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/interrupt.h>
@@ -1410,7 +1411,7 @@ static int vidioc_enum_input(struct file *file, void *fh, struct v4l2_input *i)
 
 static int vidioc_g_input(struct file *file, void *fh, unsigned int *i)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct budget_av *budget_av = (struct budget_av *)dev->ext_priv;
 
 	*i = budget_av->cur_input;
@@ -1421,7 +1422,7 @@ static int vidioc_g_input(struct file *file, void *fh, unsigned int *i)
 
 static int vidioc_s_input(struct file *file, void *fh, unsigned int input)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct budget_av *budget_av = (struct budget_av *)dev->ext_priv;
 
 	dprintk(1, "VIDIOC_S_INPUT %d\n", input);
diff --git a/drivers/media/pci/tw68/tw68-video.c b/drivers/media/pci/tw68/tw68-video.c
index 0cbc5b038073..773a18702d36 100644
--- a/drivers/media/pci/tw68/tw68-video.c
+++ b/drivers/media/pci/tw68/tw68-video.c
@@ -437,6 +437,7 @@ static void tw68_buf_queue(struct vb2_buffer *vb)
  */
 static int tw68_buf_prepare(struct vb2_buffer *vb)
 {
+	int ret;
 	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
 	struct vb2_queue *vq = vb->vb2_queue;
 	struct tw68_dev *dev = vb2_get_drv_priv(vq);
@@ -452,30 +453,30 @@ static int tw68_buf_prepare(struct vb2_buffer *vb)
 	bpl = (dev->width * dev->fmt->depth) >> 3;
 	switch (dev->field) {
 	case V4L2_FIELD_TOP:
-		tw68_risc_buffer(dev->pci, buf, dma->sgl,
+		ret = tw68_risc_buffer(dev->pci, buf, dma->sgl,
 				 0, UNSET, bpl, 0, dev->height);
 		break;
 	case V4L2_FIELD_BOTTOM:
-		tw68_risc_buffer(dev->pci, buf, dma->sgl,
+		ret = tw68_risc_buffer(dev->pci, buf, dma->sgl,
 				 UNSET, 0, bpl, 0, dev->height);
 		break;
 	case V4L2_FIELD_SEQ_TB:
-		tw68_risc_buffer(dev->pci, buf, dma->sgl,
+		ret = tw68_risc_buffer(dev->pci, buf, dma->sgl,
 				 0, bpl * (dev->height >> 1),
 				 bpl, 0, dev->height >> 1);
 		break;
 	case V4L2_FIELD_SEQ_BT:
-		tw68_risc_buffer(dev->pci, buf, dma->sgl,
+		ret = tw68_risc_buffer(dev->pci, buf, dma->sgl,
 				 bpl * (dev->height >> 1), 0,
 				 bpl, 0, dev->height >> 1);
 		break;
 	case V4L2_FIELD_INTERLACED:
 	default:
-		tw68_risc_buffer(dev->pci, buf, dma->sgl,
+		ret = tw68_risc_buffer(dev->pci, buf, dma->sgl,
 				 0, bpl, bpl, bpl, dev->height >> 1);
 		break;
 	}
-	return 0;
+	return ret;
 }
 
 static void tw68_buf_finish(struct vb2_buffer *vb)
@@ -485,7 +486,8 @@ static void tw68_buf_finish(struct vb2_buffer *vb)
 	struct tw68_dev *dev = vb2_get_drv_priv(vq);
 	struct tw68_buf *buf = container_of(vbuf, struct tw68_buf, vb);
 
-	dma_free_coherent(&dev->pci->dev, buf->size, buf->cpu, buf->dma);
+	if (buf->cpu)
+		dma_free_coherent(&dev->pci->dev, buf->size, buf->cpu, buf->dma);
 }
 
 static int tw68_start_streaming(struct vb2_queue *q, unsigned int count)
diff --git a/drivers/media/pci/zoran/zoran_device.h b/drivers/media/pci/zoran/zoran_device.h
index 237e830ae726..6d2a526789a5 100644
--- a/drivers/media/pci/zoran/zoran_device.h
+++ b/drivers/media/pci/zoran/zoran_device.h
@@ -25,8 +25,6 @@ void jpeg_codec_sleep(struct zoran *zr, int sleep);
 int jpeg_codec_reset(struct zoran *zr);
 
 /* zr360x7 access to raw capture */
-void zr36057_overlay(struct zoran *zr, int on);
-void write_overlay_mask(struct zoran_fh *fh, struct v4l2_clip *vp, int count);
 void zr36057_set_memgrab(struct zoran *zr, int mode);
 int wait_grab_pending(struct zoran *zr);
 
diff --git a/drivers/media/platform/allegro-dvt/allegro-core.c b/drivers/media/platform/allegro-dvt/allegro-core.c
index 2423714afcb9..ec03e17727d7 100644
--- a/drivers/media/platform/allegro-dvt/allegro-core.c
+++ b/drivers/media/platform/allegro-dvt/allegro-core.c
@@ -3919,7 +3919,7 @@ static int allegro_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int allegro_remove(struct platform_device *pdev)
+static void allegro_remove(struct platform_device *pdev)
 {
 	struct allegro_dev *dev = platform_get_drvdata(pdev);
 
@@ -3935,8 +3935,6 @@ static int allegro_remove(struct platform_device *pdev)
 	pm_runtime_disable(&dev->plat_dev->dev);
 
 	v4l2_device_unregister(&dev->v4l2_dev);
-
-	return 0;
 }
 
 static int allegro_runtime_resume(struct device *device)
@@ -4006,7 +4004,7 @@ static const struct dev_pm_ops allegro_pm_ops = {
 
 static struct platform_driver allegro_driver = {
 	.probe = allegro_probe,
-	.remove = allegro_remove,
+	.remove_new = allegro_remove,
 	.driver = {
 		.name = "allegro",
 		.of_match_table = of_match_ptr(allegro_dt_ids),
diff --git a/drivers/media/platform/amlogic/meson-ge2d/ge2d.c b/drivers/media/platform/amlogic/meson-ge2d/ge2d.c
index 142d421a8d76..09409908ba5d 100644
--- a/drivers/media/platform/amlogic/meson-ge2d/ge2d.c
+++ b/drivers/media/platform/amlogic/meson-ge2d/ge2d.c
@@ -1024,7 +1024,7 @@ disable_clks:
 	return ret;
 }
 
-static int ge2d_remove(struct platform_device *pdev)
+static void ge2d_remove(struct platform_device *pdev)
 {
 	struct meson_ge2d *ge2d = platform_get_drvdata(pdev);
 
@@ -1032,8 +1032,6 @@ static int ge2d_remove(struct platform_device *pdev)
 	v4l2_m2m_release(ge2d->m2m_dev);
 	v4l2_device_unregister(&ge2d->v4l2_dev);
 	clk_disable_unprepare(ge2d->clk);
-
-	return 0;
 }
 
 static const struct of_device_id meson_ge2d_match[] = {
@@ -1047,7 +1045,7 @@ MODULE_DEVICE_TABLE(of, meson_ge2d_match);
 
 static struct platform_driver ge2d_drv = {
 	.probe = ge2d_probe,
-	.remove = ge2d_remove,
+	.remove_new = ge2d_remove,
 	.driver = {
 		.name = "meson-ge2d",
 		.of_match_table = meson_ge2d_match,
diff --git a/drivers/media/platform/amphion/vdec.c b/drivers/media/platform/amphion/vdec.c
index 87f9f8e90ab1..3fa1a74a2e20 100644
--- a/drivers/media/platform/amphion/vdec.c
+++ b/drivers/media/platform/amphion/vdec.c
@@ -165,10 +165,55 @@ static const struct vpu_format vdec_formats[] = {
 		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
 		.flags = V4L2_FMT_FLAG_DYN_RESOLUTION | V4L2_FMT_FLAG_COMPRESSED
 	},
+	{
+		.pixfmt = V4L2_PIX_FMT_SPK,
+		.mem_planes = 1,
+		.comp_planes = 1,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+		.flags = V4L2_FMT_FLAG_DYN_RESOLUTION | V4L2_FMT_FLAG_COMPRESSED
+	},
+	{
+		.pixfmt = V4L2_PIX_FMT_RV30,
+		.mem_planes = 1,
+		.comp_planes = 1,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+		.flags = V4L2_FMT_FLAG_DYN_RESOLUTION | V4L2_FMT_FLAG_COMPRESSED
+	},
+	{
+		.pixfmt = V4L2_PIX_FMT_RV40,
+		.mem_planes = 1,
+		.comp_planes = 1,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+		.flags = V4L2_FMT_FLAG_DYN_RESOLUTION | V4L2_FMT_FLAG_COMPRESSED
+	},
 	{0, 0, 0, 0},
 };
 
+static int vdec_op_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct vpu_inst *inst = ctrl_to_inst(ctrl);
+	struct vdec_t *vdec = inst->priv;
+	int ret = 0;
+
+	vpu_inst_lock(inst);
+	switch (ctrl->id) {
+	case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY_ENABLE:
+		vdec->params.display_delay_enable = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY:
+		vdec->params.display_delay = ctrl->val;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	vpu_inst_unlock(inst);
+
+	return ret;
+}
+
 static const struct v4l2_ctrl_ops vdec_ctrl_ops = {
+	.s_ctrl = vdec_op_s_ctrl,
 	.g_volatile_ctrl = vpu_helper_g_volatile_ctrl,
 };
 
@@ -181,6 +226,14 @@ static int vdec_ctrl_init(struct vpu_inst *inst)
 	if (ret)
 		return ret;
 
+	v4l2_ctrl_new_std(&inst->ctrl_handler, &vdec_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY,
+			  0, 0, 1, 0);
+
+	v4l2_ctrl_new_std(&inst->ctrl_handler, &vdec_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY_ENABLE,
+			  0, 1, 1, 0);
+
 	ctrl = v4l2_ctrl_new_std(&inst->ctrl_handler, &vdec_ctrl_ops,
 				 V4L2_CID_MIN_BUFFERS_FOR_CAPTURE, 1, 32, 1, 2);
 	if (ctrl)
diff --git a/drivers/media/platform/amphion/vpu_codec.h b/drivers/media/platform/amphion/vpu_codec.h
index 528a93f08ecd..bac6d0d94f8a 100644
--- a/drivers/media/platform/amphion/vpu_codec.h
+++ b/drivers/media/platform/amphion/vpu_codec.h
@@ -55,7 +55,8 @@ struct vpu_encode_params {
 struct vpu_decode_params {
 	u32 codec_format;
 	u32 output_format;
-	u32 b_dis_reorder;
+	u32 display_delay_enable;
+	u32 display_delay;
 	u32 b_non_frame;
 	u32 frame_count;
 	u32 end_flag;
diff --git a/drivers/media/platform/amphion/vpu_core.c b/drivers/media/platform/amphion/vpu_core.c
index f9ec1753f7c8..de23627a119a 100644
--- a/drivers/media/platform/amphion/vpu_core.c
+++ b/drivers/media/platform/amphion/vpu_core.c
@@ -709,7 +709,7 @@ err_runtime_disable:
 	return ret;
 }
 
-static int vpu_core_remove(struct platform_device *pdev)
+static void vpu_core_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct vpu_core *core = platform_get_drvdata(pdev);
@@ -728,8 +728,6 @@ static int vpu_core_remove(struct platform_device *pdev)
 	memunmap(core->rpc.virt);
 	mutex_destroy(&core->lock);
 	mutex_destroy(&core->cmd_lock);
-
-	return 0;
 }
 
 static int __maybe_unused vpu_core_runtime_resume(struct device *dev)
@@ -864,7 +862,7 @@ MODULE_DEVICE_TABLE(of, vpu_core_dt_match);
 
 static struct platform_driver amphion_vpu_core_driver = {
 	.probe = vpu_core_probe,
-	.remove = vpu_core_remove,
+	.remove_new = vpu_core_remove,
 	.driver = {
 		.name = "amphion-vpu-core",
 		.of_match_table = vpu_core_dt_match,
diff --git a/drivers/media/platform/amphion/vpu_drv.c b/drivers/media/platform/amphion/vpu_drv.c
index f01ce49d27e8..4187b2b5562f 100644
--- a/drivers/media/platform/amphion/vpu_drv.c
+++ b/drivers/media/platform/amphion/vpu_drv.c
@@ -157,7 +157,7 @@ err_vpu_deinit:
 	return ret;
 }
 
-static int vpu_remove(struct platform_device *pdev)
+static void vpu_remove(struct platform_device *pdev)
 {
 	struct vpu_dev *vpu = platform_get_drvdata(pdev);
 	struct device *dev = &pdev->dev;
@@ -173,8 +173,6 @@ static int vpu_remove(struct platform_device *pdev)
 	media_device_cleanup(&vpu->mdev);
 	v4l2_device_unregister(&vpu->v4l2_dev);
 	mutex_destroy(&vpu->lock);
-
-	return 0;
 }
 
 static int __maybe_unused vpu_runtime_resume(struct device *dev)
@@ -229,7 +227,7 @@ MODULE_DEVICE_TABLE(of, vpu_dt_match);
 
 static struct platform_driver amphion_vpu_driver = {
 	.probe = vpu_probe,
-	.remove = vpu_remove,
+	.remove_new = vpu_remove,
 	.driver = {
 		.name = "amphion-vpu",
 		.of_match_table = vpu_dt_match,
diff --git a/drivers/media/platform/amphion/vpu_malone.c b/drivers/media/platform/amphion/vpu_malone.c
index 2c9bfc6a5a72..ef44bff9fbaf 100644
--- a/drivers/media/platform/amphion/vpu_malone.c
+++ b/drivers/media/platform/amphion/vpu_malone.c
@@ -68,6 +68,8 @@
 #define STREAM_CONFIG_NUM_DBE_SET(x, y)		CONFIG_SET(x, y, 30, 0x40000000)
 #define STREAM_CONFIG_FS_CTRL_MODE_SET(x, y)	CONFIG_SET(x, y, 31, 0x80000000)
 
+#define MALONE_DEC_FMT_RV_MASK			BIT(21)
+
 enum vpu_malone_stream_input_mode {
 	INVALID_MODE = 0,
 	FRAME_LVL,
@@ -478,6 +480,9 @@ u32 vpu_malone_get_version(struct vpu_shared_addr *shared)
 {
 	struct malone_iface *iface = shared->iface;
 
+	vpu_malone_enable_format(V4L2_PIX_FMT_RV30, iface->fw_version & MALONE_DEC_FMT_RV_MASK);
+	vpu_malone_enable_format(V4L2_PIX_FMT_RV40, iface->fw_version & MALONE_DEC_FMT_RV_MASK);
+
 	return iface->fw_version;
 }
 
@@ -562,8 +567,23 @@ static struct malone_fmt_mapping fmt_mappings[] = {
 	{V4L2_PIX_FMT_H263,        MALONE_FMT_ASP},
 	{V4L2_PIX_FMT_JPEG,        MALONE_FMT_JPG},
 	{V4L2_PIX_FMT_VP8,         MALONE_FMT_VP8},
+	{V4L2_PIX_FMT_SPK,         MALONE_FMT_SPK},
+	{V4L2_PIX_FMT_RV30,        MALONE_FMT_RV},
+	{V4L2_PIX_FMT_RV40,        MALONE_FMT_RV},
 };
 
+void vpu_malone_enable_format(u32 pixelformat, int enable)
+{
+	u32 i;
+
+	for (i = 0; i < ARRAY_SIZE(fmt_mappings); i++) {
+		if (pixelformat == fmt_mappings[i].pixelformat) {
+			fmt_mappings[i].is_disabled = enable ? 0 : 1;
+			return;
+		}
+	}
+}
+
 static enum vpu_malone_format vpu_malone_format_remap(u32 pixelformat)
 {
 	u32 i;
@@ -641,7 +661,9 @@ static int vpu_malone_set_params(struct vpu_shared_addr *shared,
 		hc->jpg[instance].jpg_mjpeg_interlaced = 0;
 	}
 
-	hc->codec_param[instance].disp_imm = params->b_dis_reorder ? 1 : 0;
+	hc->codec_param[instance].disp_imm = params->display_delay_enable ? 1 : 0;
+	if (malone_format != MALONE_FMT_AVC)
+		hc->codec_param[instance].disp_imm = 0;
 	hc->codec_param[instance].dbglog_enable = 0;
 	iface->dbglog_desc.level = 0;
 
@@ -987,6 +1009,9 @@ static const struct malone_padding_scode padding_scodes[] = {
 	{SCODE_PADDING_EOS,      V4L2_PIX_FMT_XVID,        {0xb1010000, 0x0}},
 	{SCODE_PADDING_EOS,      V4L2_PIX_FMT_H263,        {0xb1010000, 0x0}},
 	{SCODE_PADDING_EOS,      V4L2_PIX_FMT_VP8,         {0x34010000, 0x0}},
+	{SCODE_PADDING_EOS,      V4L2_PIX_FMT_SPK,         {0x34010000, 0x0}},
+	{SCODE_PADDING_EOS,      V4L2_PIX_FMT_RV30,        {0x34010000, 0x0}},
+	{SCODE_PADDING_EOS,      V4L2_PIX_FMT_RV40,        {0x34010000, 0x0}},
 	{SCODE_PADDING_EOS,      V4L2_PIX_FMT_JPEG,        {0xefff0000, 0x0}},
 	{SCODE_PADDING_ABORT,    V4L2_PIX_FMT_H264,        {0x0B010000, 0}},
 	{SCODE_PADDING_ABORT,    V4L2_PIX_FMT_H264_MVC,    {0x0B010000, 0}},
@@ -998,6 +1023,9 @@ static const struct malone_padding_scode padding_scodes[] = {
 	{SCODE_PADDING_ABORT,    V4L2_PIX_FMT_XVID,        {0xb1010000, 0x0}},
 	{SCODE_PADDING_ABORT,    V4L2_PIX_FMT_H263,        {0xb1010000, 0x0}},
 	{SCODE_PADDING_ABORT,    V4L2_PIX_FMT_VP8,         {0x34010000, 0x0}},
+	{SCODE_PADDING_ABORT,    V4L2_PIX_FMT_SPK,         {0x34010000, 0x0}},
+	{SCODE_PADDING_ABORT,    V4L2_PIX_FMT_RV30,        {0x34010000, 0x0}},
+	{SCODE_PADDING_ABORT,    V4L2_PIX_FMT_RV40,        {0x34010000, 0x0}},
 	{SCODE_PADDING_EOS,      V4L2_PIX_FMT_JPEG,        {0x0, 0x0}},
 	{SCODE_PADDING_BUFFLUSH, V4L2_PIX_FMT_H264,        {0x15010000, 0x0}},
 	{SCODE_PADDING_BUFFLUSH, V4L2_PIX_FMT_H264_MVC,    {0x15010000, 0x0}},
@@ -1411,6 +1439,16 @@ static int vpu_malone_insert_scode_vp8_pic(struct malone_scode_t *scode)
 	return size;
 }
 
+static int vpu_malone_insert_scode_spk_seq(struct malone_scode_t *scode)
+{
+	return vpu_malone_insert_scode_seq(scode, MALONE_CODEC_ID_SPK, 0);
+}
+
+static int vpu_malone_insert_scode_spk_pic(struct malone_scode_t *scode)
+{
+	return vpu_malone_insert_scode_pic(scode, MALONE_CODEC_ID_SPK, 0);
+}
+
 static const struct malone_scode_handler scode_handlers[] = {
 	{
 		/* fix me, need to swap return operation after gstreamer swap */
@@ -1427,6 +1465,11 @@ static const struct malone_scode_handler scode_handlers[] = {
 		.insert_scode_seq = vpu_malone_insert_scode_vp8_seq,
 		.insert_scode_pic = vpu_malone_insert_scode_vp8_pic,
 	},
+	{
+		.pixelformat = V4L2_PIX_FMT_SPK,
+		.insert_scode_seq = vpu_malone_insert_scode_spk_seq,
+		.insert_scode_pic = vpu_malone_insert_scode_spk_pic,
+	},
 };
 
 static const struct malone_scode_handler *get_scode_handler(u32 pixelformat)
diff --git a/drivers/media/platform/amphion/vpu_malone.h b/drivers/media/platform/amphion/vpu_malone.h
index 02a9d9530970..c95b53629199 100644
--- a/drivers/media/platform/amphion/vpu_malone.h
+++ b/drivers/media/platform/amphion/vpu_malone.h
@@ -41,5 +41,6 @@ int vpu_malone_post_cmd(struct vpu_shared_addr *shared, u32 instance);
 int vpu_malone_init_instance(struct vpu_shared_addr *shared, u32 instance);
 u32 vpu_malone_get_max_instance_count(struct vpu_shared_addr *shared);
 bool vpu_malone_check_fmt(enum vpu_core_type type, u32 pixelfmt);
+void vpu_malone_enable_format(u32 pixelformat, int enable);
 
 #endif
diff --git a/drivers/media/platform/aspeed/aspeed-video.c b/drivers/media/platform/aspeed/aspeed-video.c
index 794d4dc3a654..374eb7781936 100644
--- a/drivers/media/platform/aspeed/aspeed-video.c
+++ b/drivers/media/platform/aspeed/aspeed-video.c
@@ -2206,7 +2206,7 @@ static int aspeed_video_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int aspeed_video_remove(struct platform_device *pdev)
+static void aspeed_video_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct v4l2_device *v4l2_dev = dev_get_drvdata(dev);
@@ -2228,8 +2228,6 @@ static int aspeed_video_remove(struct platform_device *pdev)
 	aspeed_video_free_buf(video, &video->jpeg);
 
 	of_reserved_mem_device_release(dev);
-
-	return 0;
 }
 
 static struct platform_driver aspeed_video_driver = {
@@ -2238,7 +2236,7 @@ static struct platform_driver aspeed_video_driver = {
 		.of_match_table = aspeed_video_of_match,
 	},
 	.probe = aspeed_video_probe,
-	.remove = aspeed_video_remove,
+	.remove_new = aspeed_video_remove,
 };
 
 module_platform_driver(aspeed_video_driver);
diff --git a/drivers/media/platform/atmel/atmel-isi.c b/drivers/media/platform/atmel/atmel-isi.c
index 4d15814e4481..c29e04864445 100644
--- a/drivers/media/platform/atmel/atmel-isi.c
+++ b/drivers/media/platform/atmel/atmel-isi.c
@@ -587,8 +587,8 @@ static int isi_try_fmt(struct atmel_isi *isi, struct v4l2_format *f,
 	struct v4l2_pix_format *pixfmt = &f->fmt.pix;
 	struct v4l2_subdev_pad_config pad_cfg = {};
 	struct v4l2_subdev_state pad_state = {
-		.pads = &pad_cfg
-		};
+		.pads = &pad_cfg,
+	};
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_TRY,
 	};
@@ -1317,7 +1317,7 @@ err_vdev_alloc:
 	return ret;
 }
 
-static int atmel_isi_remove(struct platform_device *pdev)
+static void atmel_isi_remove(struct platform_device *pdev)
 {
 	struct atmel_isi *isi = platform_get_drvdata(pdev);
 
@@ -1329,8 +1329,6 @@ static int atmel_isi_remove(struct platform_device *pdev)
 	v4l2_async_nf_unregister(&isi->notifier);
 	v4l2_async_nf_cleanup(&isi->notifier);
 	v4l2_device_unregister(&isi->v4l2_dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -1368,7 +1366,7 @@ static struct platform_driver atmel_isi_driver = {
 		.pm	= &atmel_isi_dev_pm_ops,
 	},
 	.probe		= atmel_isi_probe,
-	.remove		= atmel_isi_remove,
+	.remove_new	= atmel_isi_remove,
 };
 
 module_platform_driver(atmel_isi_driver);
diff --git a/drivers/media/platform/cadence/cdns-csi2rx.c b/drivers/media/platform/cadence/cdns-csi2rx.c
index cc3ebb0d96f6..9755d1c8ceb9 100644
--- a/drivers/media/platform/cadence/cdns-csi2rx.c
+++ b/drivers/media/platform/cadence/cdns-csi2rx.c
@@ -473,14 +473,12 @@ err_free_priv:
 	return ret;
 }
 
-static int csi2rx_remove(struct platform_device *pdev)
+static void csi2rx_remove(struct platform_device *pdev)
 {
 	struct csi2rx_priv *csi2rx = platform_get_drvdata(pdev);
 
 	v4l2_async_unregister_subdev(&csi2rx->subdev);
 	kfree(csi2rx);
-
-	return 0;
 }
 
 static const struct of_device_id csi2rx_of_table[] = {
@@ -491,7 +489,7 @@ MODULE_DEVICE_TABLE(of, csi2rx_of_table);
 
 static struct platform_driver csi2rx_driver = {
 	.probe	= csi2rx_probe,
-	.remove	= csi2rx_remove,
+	.remove_new = csi2rx_remove,
 
 	.driver	= {
 		.name		= "cdns-csi2rx",
diff --git a/drivers/media/platform/cadence/cdns-csi2tx.c b/drivers/media/platform/cadence/cdns-csi2tx.c
index 58e405b69f67..1e0400b7803e 100644
--- a/drivers/media/platform/cadence/cdns-csi2tx.c
+++ b/drivers/media/platform/cadence/cdns-csi2tx.c
@@ -635,19 +635,17 @@ err_free_priv:
 	return ret;
 }
 
-static int csi2tx_remove(struct platform_device *pdev)
+static void csi2tx_remove(struct platform_device *pdev)
 {
 	struct csi2tx_priv *csi2tx = platform_get_drvdata(pdev);
 
 	v4l2_async_unregister_subdev(&csi2tx->subdev);
 	kfree(csi2tx);
-
-	return 0;
 }
 
 static struct platform_driver csi2tx_driver = {
 	.probe	= csi2tx_probe,
-	.remove	= csi2tx_remove,
+	.remove_new = csi2tx_remove,
 
 	.driver	= {
 		.name		= "cdns-csi2tx",
diff --git a/drivers/media/platform/chips-media/coda-common.c b/drivers/media/platform/chips-media/coda-common.c
index af71eea04dbd..d013ea5d9d3d 100644
--- a/drivers/media/platform/chips-media/coda-common.c
+++ b/drivers/media/platform/chips-media/coda-common.c
@@ -3300,7 +3300,7 @@ err_v4l2_register:
 	return ret;
 }
 
-static int coda_remove(struct platform_device *pdev)
+static void coda_remove(struct platform_device *pdev)
 {
 	struct coda_dev *dev = platform_get_drvdata(pdev);
 	int i;
@@ -3322,7 +3322,6 @@ static int coda_remove(struct platform_device *pdev)
 	coda_free_aux_buf(dev, &dev->workbuf);
 	debugfs_remove_recursive(dev->debugfs_root);
 	ida_destroy(&dev->ida);
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -3347,7 +3346,7 @@ static const struct dev_pm_ops coda_pm_ops = {
 
 static struct platform_driver coda_driver = {
 	.probe	= coda_probe,
-	.remove	= coda_remove,
+	.remove_new = coda_remove,
 	.driver	= {
 		.name	= CODA_NAME,
 		.of_match_table = coda_dt_ids,
diff --git a/drivers/media/platform/intel/pxa_camera.c b/drivers/media/platform/intel/pxa_camera.c
index 54270d6b6f50..9ed3c2e063de 100644
--- a/drivers/media/platform/intel/pxa_camera.c
+++ b/drivers/media/platform/intel/pxa_camera.c
@@ -1794,8 +1794,8 @@ static int pxac_vidioc_try_fmt_vid_cap(struct file *filp, void *priv,
 	struct v4l2_pix_format *pix = &f->fmt.pix;
 	struct v4l2_subdev_pad_config pad_cfg;
 	struct v4l2_subdev_state pad_state = {
-		.pads = &pad_cfg
-		};
+		.pads = &pad_cfg,
+	};
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_TRY,
 	};
@@ -2421,7 +2421,7 @@ exit_free_dma_y:
 	return err;
 }
 
-static int pxa_camera_remove(struct platform_device *pdev)
+static void pxa_camera_remove(struct platform_device *pdev)
 {
 	struct pxa_camera_dev *pcdev = platform_get_drvdata(pdev);
 
@@ -2437,8 +2437,6 @@ static int pxa_camera_remove(struct platform_device *pdev)
 	v4l2_device_unregister(&pcdev->v4l2_dev);
 
 	dev_info(&pdev->dev, "PXA Camera driver unloaded\n");
-
-	return 0;
 }
 
 static const struct dev_pm_ops pxa_camera_pm = {
@@ -2459,7 +2457,7 @@ static struct platform_driver pxa_camera_driver = {
 		.of_match_table = of_match_ptr(pxa_camera_of_match),
 	},
 	.probe		= pxa_camera_probe,
-	.remove		= pxa_camera_remove,
+	.remove_new	= pxa_camera_remove,
 };
 
 module_platform_driver(pxa_camera_driver);
diff --git a/drivers/media/platform/m2m-deinterlace.c b/drivers/media/platform/m2m-deinterlace.c
index 1f89e71cdccf..96b35a5d6174 100644
--- a/drivers/media/platform/m2m-deinterlace.c
+++ b/drivers/media/platform/m2m-deinterlace.c
@@ -984,7 +984,7 @@ rel_dma:
 	return ret;
 }
 
-static int deinterlace_remove(struct platform_device *pdev)
+static void deinterlace_remove(struct platform_device *pdev)
 {
 	struct deinterlace_dev *pcdev = platform_get_drvdata(pdev);
 
@@ -993,13 +993,11 @@ static int deinterlace_remove(struct platform_device *pdev)
 	video_unregister_device(&pcdev->vfd);
 	v4l2_device_unregister(&pcdev->v4l2_dev);
 	dma_release_channel(pcdev->dma_chan);
-
-	return 0;
 }
 
 static struct platform_driver deinterlace_pdrv = {
 	.probe		= deinterlace_probe,
-	.remove		= deinterlace_remove,
+	.remove_new	= deinterlace_remove,
 	.driver		= {
 		.name	= MEM2MEM_NAME,
 	},
diff --git a/drivers/media/platform/marvell/mcam-core.c b/drivers/media/platform/marvell/mcam-core.c
index ad4a7922d0d7..154bdcb3f2cc 100644
--- a/drivers/media/platform/marvell/mcam-core.c
+++ b/drivers/media/platform/marvell/mcam-core.c
@@ -1351,8 +1351,8 @@ static int mcam_vidioc_try_fmt_vid_cap(struct file *filp, void *priv,
 	struct v4l2_pix_format *pix = &fmt->fmt.pix;
 	struct v4l2_subdev_pad_config pad_cfg;
 	struct v4l2_subdev_state pad_state = {
-		.pads = &pad_cfg
-		};
+		.pads = &pad_cfg,
+	};
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_TRY,
 	};
diff --git a/drivers/media/platform/marvell/mmp-driver.c b/drivers/media/platform/marvell/mmp-driver.c
index ef22bf8f276c..e93feefb447b 100644
--- a/drivers/media/platform/marvell/mmp-driver.c
+++ b/drivers/media/platform/marvell/mmp-driver.c
@@ -287,23 +287,13 @@ out:
 	return ret;
 }
 
-
-static int mmpcam_remove(struct mmp_camera *cam)
+static void mmpcam_remove(struct platform_device *pdev)
 {
+	struct mmp_camera *cam = platform_get_drvdata(pdev);
 	struct mcam_camera *mcam = &cam->mcam;
 
 	mccic_shutdown(mcam);
 	pm_runtime_force_suspend(mcam->dev);
-	return 0;
-}
-
-static int mmpcam_platform_remove(struct platform_device *pdev)
-{
-	struct mmp_camera *cam = platform_get_drvdata(pdev);
-
-	if (cam == NULL)
-		return -ENODEV;
-	return mmpcam_remove(cam);
 }
 
 /*
@@ -369,7 +359,7 @@ MODULE_DEVICE_TABLE(of, mmpcam_of_match);
 
 static struct platform_driver mmpcam_driver = {
 	.probe		= mmpcam_probe,
-	.remove		= mmpcam_platform_remove,
+	.remove_new	= mmpcam_remove,
 	.driver = {
 		.name	= "mmp-camera",
 		.of_match_table = of_match_ptr(mmpcam_of_match),
diff --git a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c
index 969516a940ba..6c7c5f3648df 100644
--- a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c
+++ b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c
@@ -1004,8 +1004,8 @@ static void mtk_jpegenc_worker(struct work_struct *work)
 retry_select:
 	hw_id = mtk_jpegenc_get_hw(ctx);
 	if (hw_id < 0) {
-		ret = wait_event_interruptible(jpeg->enc_hw_wq,
-					       atomic_read(&jpeg->enchw_rdy) > 0);
+		ret = wait_event_interruptible(jpeg->hw_wq,
+					       atomic_read(&jpeg->hw_rdy) > 0);
 		if (ret != 0 || (i++ > MTK_JPEG_MAX_RETRY_TIME)) {
 			dev_err(jpeg->dev, "%s : %d, all HW are busy\n",
 				__func__, __LINE__);
@@ -1016,7 +1016,7 @@ retry_select:
 		goto retry_select;
 	}
 
-	atomic_dec(&jpeg->enchw_rdy);
+	atomic_dec(&jpeg->hw_rdy);
 	src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
 	if (!src_buf)
 		goto getbuf_fail;
@@ -1025,9 +1025,6 @@ retry_select:
 	if (!dst_buf)
 		goto getbuf_fail;
 
-	v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
-	v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
-
 	v4l2_m2m_buf_copy_metadata(src_buf, dst_buf, true);
 
 	mtk_jpegenc_set_hw_param(ctx, hw_id, src_buf, dst_buf);
@@ -1045,6 +1042,9 @@ retry_select:
 		goto enc_end;
 	}
 
+	v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+	v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+
 	schedule_delayed_work(&comp_jpeg[hw_id]->job_timeout_work,
 			      msecs_to_jiffies(MTK_JPEG_HW_TIMEOUT_MSEC));
 
@@ -1073,7 +1073,7 @@ enc_end:
 	v4l2_m2m_buf_done(src_buf, buf_state);
 	v4l2_m2m_buf_done(dst_buf, buf_state);
 getbuf_fail:
-	atomic_inc(&jpeg->enchw_rdy);
+	atomic_inc(&jpeg->hw_rdy);
 	mtk_jpegenc_put_hw(jpeg, hw_id);
 	v4l2_m2m_job_finish(jpeg->m2m_dev, ctx->fh.m2m_ctx);
 }
@@ -1198,8 +1198,8 @@ static void mtk_jpegdec_worker(struct work_struct *work)
 retry_select:
 	hw_id = mtk_jpegdec_get_hw(ctx);
 	if (hw_id < 0) {
-		ret = wait_event_interruptible_timeout(jpeg->dec_hw_wq,
-						       atomic_read(&jpeg->dechw_rdy) > 0,
+		ret = wait_event_interruptible_timeout(jpeg->hw_wq,
+						       atomic_read(&jpeg->hw_rdy) > 0,
 						       MTK_JPEG_HW_TIMEOUT_MSEC);
 		if (ret != 0 || (i++ > MTK_JPEG_MAX_RETRY_TIME)) {
 			dev_err(jpeg->dev, "%s : %d, all HW are busy\n",
@@ -1211,7 +1211,7 @@ retry_select:
 		goto retry_select;
 	}
 
-	atomic_dec(&jpeg->dechw_rdy);
+	atomic_dec(&jpeg->hw_rdy);
 	src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
 	if (!src_buf)
 		goto getbuf_fail;
@@ -1220,9 +1220,6 @@ retry_select:
 	if (!dst_buf)
 		goto getbuf_fail;
 
-	v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
-	v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
-
 	v4l2_m2m_buf_copy_metadata(src_buf, dst_buf, true);
 	jpeg_src_buf = mtk_jpeg_vb2_to_srcbuf(&src_buf->vb2_buf);
 	jpeg_dst_buf = mtk_jpeg_vb2_to_srcbuf(&dst_buf->vb2_buf);
@@ -1231,7 +1228,7 @@ retry_select:
 					     &jpeg_src_buf->dec_param)) {
 		mtk_jpeg_queue_src_chg_event(ctx);
 		ctx->state = MTK_JPEG_SOURCE_CHANGE;
-		goto dec_end;
+		goto getbuf_fail;
 	}
 
 	jpeg_src_buf->curr_ctx = ctx;
@@ -1254,6 +1251,9 @@ retry_select:
 		goto clk_end;
 	}
 
+	v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+	v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+
 	schedule_delayed_work(&comp_jpeg[hw_id]->job_timeout_work,
 			      msecs_to_jiffies(MTK_JPEG_HW_TIMEOUT_MSEC));
 
@@ -1290,7 +1290,7 @@ dec_end:
 	v4l2_m2m_buf_done(src_buf, buf_state);
 	v4l2_m2m_buf_done(dst_buf, buf_state);
 getbuf_fail:
-	atomic_inc(&jpeg->dechw_rdy);
+	atomic_inc(&jpeg->hw_rdy);
 	mtk_jpegdec_put_hw(jpeg, hw_id);
 	v4l2_m2m_job_finish(jpeg->m2m_dev, ctx->fh.m2m_ctx);
 }
@@ -1575,12 +1575,7 @@ static int mtk_jpeg_open(struct file *file)
 		goto free;
 	}
 
-	if (jpeg->is_jpgenc_multihw)
-		INIT_WORK(&ctx->jpeg_work, mtk_jpegenc_worker);
-
-	if (jpeg->is_jpgdec_multihw)
-		INIT_WORK(&ctx->jpeg_work, mtk_jpegdec_worker);
-
+	INIT_WORK(&ctx->jpeg_work, jpeg->variant->jpeg_worker);
 	INIT_LIST_HEAD(&ctx->dst_done_queue);
 	spin_lock_init(&ctx->done_queue_lock);
 	v4l2_fh_init(&ctx->fh, vfd);
@@ -1671,10 +1666,52 @@ static void mtk_jpeg_job_timeout_work(struct work_struct *work)
 	v4l2_m2m_job_finish(jpeg->m2m_dev, ctx->fh.m2m_ctx);
 }
 
+static int mtk_jpeg_single_core_init(struct platform_device *pdev,
+				     struct mtk_jpeg_dev *jpeg_dev)
+{
+	struct mtk_jpeg_dev *jpeg = jpeg_dev;
+	int jpeg_irq, ret;
+
+	INIT_DELAYED_WORK(&jpeg->job_timeout_work,
+			  mtk_jpeg_job_timeout_work);
+
+	jpeg->reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(jpeg->reg_base)) {
+		ret = PTR_ERR(jpeg->reg_base);
+		return ret;
+	}
+
+	jpeg_irq = platform_get_irq(pdev, 0);
+	if (jpeg_irq < 0)
+		return jpeg_irq;
+
+	ret = devm_request_irq(&pdev->dev,
+			       jpeg_irq,
+			       jpeg->variant->irq_handler,
+			       0,
+			       pdev->name, jpeg);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to request jpeg_irq %d (%d)\n",
+			jpeg_irq, ret);
+		return ret;
+	}
+
+	ret = devm_clk_bulk_get(jpeg->dev,
+				jpeg->variant->num_clks,
+				jpeg->variant->clks);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to init clk\n");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int mtk_jpeg_probe(struct platform_device *pdev)
 {
 	struct mtk_jpeg_dev *jpeg;
-	int jpeg_irq;
+	struct device_node *child;
+	int num_child = 0;
 	int ret;
 
 	jpeg = devm_kzalloc(&pdev->dev, sizeof(*jpeg), GFP_KERNEL);
@@ -1692,38 +1729,26 @@ static int mtk_jpeg_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	if (list_empty(&pdev->dev.devres_head)) {
-		INIT_DELAYED_WORK(&jpeg->job_timeout_work,
-				  mtk_jpeg_job_timeout_work);
-
-		jpeg->reg_base = devm_platform_ioremap_resource(pdev, 0);
-		if (IS_ERR(jpeg->reg_base)) {
-			ret = PTR_ERR(jpeg->reg_base);
-			return ret;
+	if (!jpeg->variant->multi_core) {
+		ret = mtk_jpeg_single_core_init(pdev, jpeg);
+		if (ret) {
+			v4l2_err(&jpeg->v4l2_dev, "mtk_jpeg_single_core_init failed.");
+			return -EINVAL;
 		}
+	} else {
+		init_waitqueue_head(&jpeg->hw_wq);
 
-		jpeg_irq = platform_get_irq(pdev, 0);
-		if (jpeg_irq < 0)
-			return jpeg_irq;
+		for_each_child_of_node(pdev->dev.of_node, child)
+			num_child++;
 
-		ret = devm_request_irq(&pdev->dev,
-				       jpeg_irq,
-				       jpeg->variant->irq_handler,
-				       0,
-				       pdev->name, jpeg);
-		if (ret) {
-			dev_err(&pdev->dev, "Failed to request jpeg_irq %d (%d)\n",
-				jpeg_irq, ret);
-			return ret;
-		}
+		atomic_set(&jpeg->hw_rdy, num_child);
+		atomic_set(&jpeg->hw_index, 0);
 
-		ret = devm_clk_bulk_get(jpeg->dev,
-					jpeg->variant->num_clks,
-					jpeg->variant->clks);
-		if (ret) {
-			dev_err(&pdev->dev, "Failed to init clk\n");
-			return ret;
-		}
+		jpeg->workqueue = alloc_ordered_workqueue(MTK_JPEG_NAME,
+							  WQ_MEM_RECLAIM
+							  | WQ_FREEZABLE);
+		if (!jpeg->workqueue)
+			return -EINVAL;
 	}
 
 	ret = v4l2_device_register(&pdev->dev, &jpeg->v4l2_dev);
@@ -1757,7 +1782,7 @@ static int mtk_jpeg_probe(struct platform_device *pdev)
 	jpeg->vdev->device_caps = V4L2_CAP_STREAMING |
 				  V4L2_CAP_VIDEO_M2M_MPLANE;
 
-	if (of_get_property(pdev->dev.of_node, "dma-ranges", NULL))
+	if (of_property_present(pdev->dev.of_node, "dma-ranges"))
 		dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(34));
 
 	ret = video_register_device(jpeg->vdev, VFL_TYPE_VIDEO, -1);
@@ -1790,7 +1815,7 @@ err_m2m_init:
 	return ret;
 }
 
-static int mtk_jpeg_remove(struct platform_device *pdev)
+static void mtk_jpeg_remove(struct platform_device *pdev)
 {
 	struct mtk_jpeg_dev *jpeg = platform_get_drvdata(pdev);
 
@@ -1798,8 +1823,6 @@ static int mtk_jpeg_remove(struct platform_device *pdev)
 	video_unregister_device(jpeg->vdev);
 	v4l2_m2m_release(jpeg->m2m_dev);
 	v4l2_device_unregister(&jpeg->v4l2_dev);
-
-	return 0;
 }
 
 static __maybe_unused int mtk_jpeg_pm_suspend(struct device *dev)
@@ -1846,6 +1869,7 @@ static const struct dev_pm_ops mtk_jpeg_pm_ops = {
 	SET_RUNTIME_PM_OPS(mtk_jpeg_pm_suspend, mtk_jpeg_pm_resume, NULL)
 };
 
+#if defined(CONFIG_OF)
 static const struct mtk_jpeg_variant mt8173_jpeg_drvdata = {
 	.clks = mt8173_jpeg_dec_clocks,
 	.num_clks = ARRAY_SIZE(mt8173_jpeg_dec_clocks),
@@ -1874,6 +1898,7 @@ static const struct mtk_jpeg_variant mtk_jpeg_drvdata = {
 	.ioctl_ops = &mtk_jpeg_enc_ioctl_ops,
 	.out_q_default_fourcc = V4L2_PIX_FMT_YUYV,
 	.cap_q_default_fourcc = V4L2_PIX_FMT_JPEG,
+	.multi_core = false,
 };
 
 static struct mtk_jpeg_variant mtk8195_jpegenc_drvdata = {
@@ -1885,6 +1910,8 @@ static struct mtk_jpeg_variant mtk8195_jpegenc_drvdata = {
 	.ioctl_ops = &mtk_jpeg_enc_ioctl_ops,
 	.out_q_default_fourcc = V4L2_PIX_FMT_YUYV,
 	.cap_q_default_fourcc = V4L2_PIX_FMT_JPEG,
+	.multi_core = true,
+	.jpeg_worker = mtk_jpegenc_worker,
 };
 
 static const struct mtk_jpeg_variant mtk8195_jpegdec_drvdata = {
@@ -1896,9 +1923,10 @@ static const struct mtk_jpeg_variant mtk8195_jpegdec_drvdata = {
 	.ioctl_ops = &mtk_jpeg_dec_ioctl_ops,
 	.out_q_default_fourcc = V4L2_PIX_FMT_JPEG,
 	.cap_q_default_fourcc = V4L2_PIX_FMT_YUV420M,
+	.multi_core = true,
+	.jpeg_worker = mtk_jpegdec_worker,
 };
 
-#if defined(CONFIG_OF)
 static const struct of_device_id mtk_jpeg_match[] = {
 	{
 		.compatible = "mediatek,mt8173-jpgdec",
@@ -1928,7 +1956,7 @@ MODULE_DEVICE_TABLE(of, mtk_jpeg_match);
 
 static struct platform_driver mtk_jpeg_driver = {
 	.probe = mtk_jpeg_probe,
-	.remove = mtk_jpeg_remove,
+	.remove_new = mtk_jpeg_remove,
 	.driver = {
 		.name           = MTK_JPEG_NAME,
 		.of_match_table = of_match_ptr(mtk_jpeg_match),
diff --git a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.h b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.h
index b9126476be8f..8ba6e757e11a 100644
--- a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.h
+++ b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.h
@@ -60,6 +60,8 @@ enum mtk_jpeg_ctx_state {
  * @ioctl_ops:			the callback of jpeg v4l2_ioctl_ops
  * @out_q_default_fourcc:	output queue default fourcc
  * @cap_q_default_fourcc:	capture queue default fourcc
+ * @multi_core:		mark jpeg hw is multi_core or not
+ * @jpeg_worker:		jpeg dec or enc worker
  */
 struct mtk_jpeg_variant {
 	struct clk_bulk_data *clks;
@@ -74,6 +76,8 @@ struct mtk_jpeg_variant {
 	const struct v4l2_ioctl_ops *ioctl_ops;
 	u32 out_q_default_fourcc;
 	u32 cap_q_default_fourcc;
+	bool multi_core;
+	void (*jpeg_worker)(struct work_struct *work);
 };
 
 struct mtk_jpeg_src_buf {
@@ -199,16 +203,12 @@ struct mtk_jpegdec_comp_dev {
  * @job_timeout_work:	IRQ timeout structure
  * @variant:		driver variant to be used
  * @reg_encbase:	jpg encode register base addr
- * @enc_hw_dev:		jpg encode hardware device
- * @is_jpgenc_multihw:	the flag of multi-hw core
- * @enc_hw_wq:		jpg encode wait queue
- * @enchw_rdy:		jpg encode hw ready flag
+ * @enc_hw_dev:	jpg encode hardware device
+ * @hw_wq:		jpg wait queue
+ * @hw_rdy:		jpg hw ready flag
  * @reg_decbase:	jpg decode register base addr
- * @dec_hw_dev:		jpg decode hardware device
- * @is_jpgdec_multihw:	the flag of dec multi-hw core
- * @dec_hw_wq:		jpg decode wait queue
- * @dec_workqueue:	jpg decode work queue
- * @dechw_rdy:		jpg decode hw ready flag
+ * @dec_hw_dev:	jpg decode hardware device
+ * @hw_index:		jpg hw index
  */
 struct mtk_jpeg_dev {
 	struct mutex		lock;
@@ -225,16 +225,12 @@ struct mtk_jpeg_dev {
 
 	void __iomem *reg_encbase[MTK_JPEGENC_HW_MAX];
 	struct mtk_jpegenc_comp_dev *enc_hw_dev[MTK_JPEGENC_HW_MAX];
-	bool is_jpgenc_multihw;
-	wait_queue_head_t enc_hw_wq;
-	atomic_t enchw_rdy;
+	wait_queue_head_t hw_wq;
+	atomic_t hw_rdy;
 
 	void __iomem *reg_decbase[MTK_JPEGDEC_HW_MAX];
 	struct mtk_jpegdec_comp_dev *dec_hw_dev[MTK_JPEGDEC_HW_MAX];
-	bool is_jpgdec_multihw;
-	wait_queue_head_t dec_hw_wq;
-	struct workqueue_struct	*dec_workqueue;
-	atomic_t dechw_rdy;
+	atomic_t hw_index;
 };
 
 /**
diff --git a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_dec_hw.c b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_dec_hw.c
index 8c07fa02fd9a..869068fac5e2 100644
--- a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_dec_hw.c
+++ b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_dec_hw.c
@@ -503,8 +503,8 @@ static void mtk_jpegdec_timeout_work(struct work_struct *work)
 	clk_disable_unprepare(cjpeg->jdec_clk.clks->clk);
 	pm_runtime_put(cjpeg->dev);
 	cjpeg->hw_state = MTK_JPEG_HW_IDLE;
-	atomic_inc(&master_jpeg->dechw_rdy);
-	wake_up(&master_jpeg->dec_hw_wq);
+	atomic_inc(&master_jpeg->hw_rdy);
+	wake_up(&master_jpeg->hw_wq);
 	v4l2_m2m_buf_done(src_buf, buf_state);
 	mtk_jpegdec_put_buf(cjpeg);
 }
@@ -551,8 +551,8 @@ static irqreturn_t mtk_jpegdec_hw_irq_handler(int irq, void *priv)
 	clk_disable_unprepare(jpeg->jdec_clk.clks->clk);
 
 	jpeg->hw_state = MTK_JPEG_HW_IDLE;
-	wake_up(&master_jpeg->dec_hw_wq);
-	atomic_inc(&master_jpeg->dechw_rdy);
+	wake_up(&master_jpeg->hw_wq);
+	atomic_inc(&master_jpeg->hw_rdy);
 
 	return IRQ_HANDLED;
 }
@@ -608,25 +608,12 @@ static int mtk_jpegdec_hw_probe(struct platform_device *pdev)
 	dev->plat_dev = pdev;
 	dev->dev = &pdev->dev;
 
-	if (!master_dev->is_jpgdec_multihw) {
-		master_dev->is_jpgdec_multihw = true;
-		for (i = 0; i < MTK_JPEGDEC_HW_MAX; i++)
-			master_dev->dec_hw_dev[i] = NULL;
-
-		init_waitqueue_head(&master_dev->dec_hw_wq);
-		master_dev->workqueue = alloc_ordered_workqueue(MTK_JPEG_NAME,
-								WQ_MEM_RECLAIM
-								| WQ_FREEZABLE);
-		if (!master_dev->workqueue)
-			return -EINVAL;
-
-		ret = devm_add_action_or_reset(&pdev->dev, mtk_jpegdec_destroy_workqueue,
-					       master_dev->workqueue);
-		if (ret)
-			return ret;
-	}
+	ret = devm_add_action_or_reset(&pdev->dev,
+				       mtk_jpegdec_destroy_workqueue,
+				       master_dev->workqueue);
+	if (ret)
+		return ret;
 
-	atomic_set(&master_dev->dechw_rdy, MTK_JPEGDEC_HW_MAX);
 	spin_lock_init(&dev->hw_lock);
 	dev->hw_state = MTK_JPEG_HW_IDLE;
 
@@ -651,14 +638,10 @@ static int mtk_jpegdec_hw_probe(struct platform_device *pdev)
 		return dev_err_probe(&pdev->dev, ret,
 				     "Failed to register JPEGDEC irq handler.\n");
 
-	for (i = 0; i < MTK_JPEGDEC_HW_MAX; i++) {
-		if (master_dev->dec_hw_dev[i])
-			continue;
-
-		master_dev->dec_hw_dev[i] = dev;
-		master_dev->reg_decbase[i] = dev->reg_base;
-		dev->master_dev = master_dev;
-	}
+	i = atomic_add_return(1, &master_dev->hw_index) - 1;
+	master_dev->dec_hw_dev[i] = dev;
+	master_dev->reg_decbase[i] = dev->reg_base;
+	dev->master_dev = master_dev;
 
 	platform_set_drvdata(pdev, dev);
 	pm_runtime_enable(&pdev->dev);
diff --git a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_enc_hw.c b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_enc_hw.c
index 1bbb712d78d0..71e85b4bbf12 100644
--- a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_enc_hw.c
+++ b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_enc_hw.c
@@ -248,8 +248,8 @@ static void mtk_jpegenc_timeout_work(struct work_struct *work)
 	clk_disable_unprepare(cjpeg->venc_clk.clks->clk);
 	pm_runtime_put(cjpeg->dev);
 	cjpeg->hw_state = MTK_JPEG_HW_IDLE;
-	atomic_inc(&master_jpeg->enchw_rdy);
-	wake_up(&master_jpeg->enc_hw_wq);
+	atomic_inc(&master_jpeg->hw_rdy);
+	wake_up(&master_jpeg->hw_wq);
 	v4l2_m2m_buf_done(src_buf, buf_state);
 	mtk_jpegenc_put_buf(cjpeg);
 }
@@ -286,14 +286,10 @@ static irqreturn_t mtk_jpegenc_hw_irq_handler(int irq, void *priv)
 	mtk_jpegenc_put_buf(jpeg);
 	pm_runtime_put(ctx->jpeg->dev);
 	clk_disable_unprepare(jpeg->venc_clk.clks->clk);
-	if (!list_empty(&ctx->fh.m2m_ctx->out_q_ctx.rdy_queue) ||
-	    !list_empty(&ctx->fh.m2m_ctx->cap_q_ctx.rdy_queue)) {
-		queue_work(master_jpeg->workqueue, &ctx->jpeg_work);
-	}
 
 	jpeg->hw_state = MTK_JPEG_HW_IDLE;
-	wake_up(&master_jpeg->enc_hw_wq);
-	atomic_inc(&master_jpeg->enchw_rdy);
+	wake_up(&master_jpeg->hw_wq);
+	atomic_inc(&master_jpeg->hw_rdy);
 
 	return IRQ_HANDLED;
 }
@@ -344,20 +340,6 @@ static int mtk_jpegenc_hw_probe(struct platform_device *pdev)
 	dev->plat_dev = pdev;
 	dev->dev = &pdev->dev;
 
-	if (!master_dev->is_jpgenc_multihw) {
-		master_dev->is_jpgenc_multihw = true;
-		for (i = 0; i < MTK_JPEGENC_HW_MAX; i++)
-			master_dev->enc_hw_dev[i] = NULL;
-
-		init_waitqueue_head(&master_dev->enc_hw_wq);
-		master_dev->workqueue = alloc_ordered_workqueue(MTK_JPEG_NAME,
-								WQ_MEM_RECLAIM
-								| WQ_FREEZABLE);
-		if (!master_dev->workqueue)
-			return -EINVAL;
-	}
-
-	atomic_set(&master_dev->enchw_rdy, MTK_JPEGENC_HW_MAX);
 	spin_lock_init(&dev->hw_lock);
 	dev->hw_state = MTK_JPEG_HW_IDLE;
 
@@ -380,14 +362,10 @@ static int mtk_jpegenc_hw_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	for (i = 0; i < MTK_JPEGENC_HW_MAX; i++) {
-		if (master_dev->enc_hw_dev[i])
-			continue;
-
-		master_dev->enc_hw_dev[i] = dev;
-		master_dev->reg_encbase[i] = dev->reg_base;
-		dev->master_dev = master_dev;
-	}
+	i = atomic_add_return(1, &master_dev->hw_index) - 1;
+	master_dev->enc_hw_dev[i] = dev;
+	master_dev->reg_encbase[i] = dev->reg_base;
+	dev->master_dev = master_dev;
 
 	platform_set_drvdata(pdev, dev);
 	pm_runtime_enable(&pdev->dev);
diff --git a/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c b/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c
index d83c4964eaf9..77e310e588e5 100644
--- a/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c
+++ b/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c
@@ -235,7 +235,7 @@ err_comp:
 	return ret;
 }
 
-static int mtk_mdp_remove(struct platform_device *pdev)
+static void mtk_mdp_remove(struct platform_device *pdev)
 {
 	struct mtk_mdp_dev *mdp = platform_get_drvdata(pdev);
 	struct mtk_mdp_comp *comp, *comp_temp;
@@ -255,7 +255,6 @@ static int mtk_mdp_remove(struct platform_device *pdev)
 	}
 
 	dev_dbg(&pdev->dev, "%s driver unloaded\n", pdev->name);
-	return 0;
 }
 
 static int __maybe_unused mtk_mdp_pm_suspend(struct device *dev)
@@ -299,7 +298,7 @@ static const struct dev_pm_ops mtk_mdp_pm_ops = {
 
 static struct platform_driver mtk_mdp_driver = {
 	.probe		= mtk_mdp_probe,
-	.remove		= mtk_mdp_remove,
+	.remove_new	= mtk_mdp_remove,
 	.driver = {
 		.name	= MTK_MDP_MODULE_NAME,
 		.pm	= &mtk_mdp_pm_ops,
diff --git a/drivers/media/platform/mediatek/mdp3/Makefile b/drivers/media/platform/mediatek/mdp3/Makefile
index 63e6c87e480b..2ee24195a2dd 100644
--- a/drivers/media/platform/mediatek/mdp3/Makefile
+++ b/drivers/media/platform/mediatek/mdp3/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0-only
-mtk-mdp3-y += mtk-mdp3-core.o mtk-mdp3-vpu.o mtk-mdp3-regs.o
+mtk-mdp3-y += mdp_cfg_data.o mtk-mdp3-core.o mtk-mdp3-vpu.o mtk-mdp3-regs.o
 mtk-mdp3-y += mtk-mdp3-m2m.o
 mtk-mdp3-y += mtk-mdp3-comp.o mtk-mdp3-cmdq.o
 
diff --git a/drivers/media/platform/mediatek/mdp3/mdp_cfg_data.c b/drivers/media/platform/mediatek/mdp3/mdp_cfg_data.c
new file mode 100644
index 000000000000..502eeae0bfdc
--- /dev/null
+++ b/drivers/media/platform/mediatek/mdp3/mdp_cfg_data.c
@@ -0,0 +1,453 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2023 MediaTek Inc.
+ * Author: Ping-Hsun Wu <ping-hsun.wu@mediatek.com>
+ */
+
+#include "mtk-img-ipi.h"
+#include "mtk-mdp3-cfg.h"
+#include "mtk-mdp3-core.h"
+#include "mtk-mdp3-comp.h"
+#include "mtk-mdp3-regs.h"
+
+enum mt8183_mdp_comp_id {
+	/* ISP */
+	MT8183_MDP_COMP_WPEI = 0,
+	MT8183_MDP_COMP_WPEO,           /* 1 */
+	MT8183_MDP_COMP_WPEI2,          /* 2 */
+	MT8183_MDP_COMP_WPEO2,          /* 3 */
+	MT8183_MDP_COMP_ISP_IMGI,       /* 4 */
+	MT8183_MDP_COMP_ISP_IMGO,       /* 5 */
+	MT8183_MDP_COMP_ISP_IMG2O,      /* 6 */
+
+	/* IPU */
+	MT8183_MDP_COMP_IPUI,           /* 7 */
+	MT8183_MDP_COMP_IPUO,           /* 8 */
+
+	/* MDP */
+	MT8183_MDP_COMP_CAMIN,          /* 9 */
+	MT8183_MDP_COMP_CAMIN2,         /* 10 */
+	MT8183_MDP_COMP_RDMA0,          /* 11 */
+	MT8183_MDP_COMP_AAL0,           /* 12 */
+	MT8183_MDP_COMP_CCORR0,         /* 13 */
+	MT8183_MDP_COMP_RSZ0,           /* 14 */
+	MT8183_MDP_COMP_RSZ1,           /* 15 */
+	MT8183_MDP_COMP_TDSHP0,         /* 16 */
+	MT8183_MDP_COMP_COLOR0,         /* 17 */
+	MT8183_MDP_COMP_PATH0_SOUT,     /* 18 */
+	MT8183_MDP_COMP_PATH1_SOUT,     /* 19 */
+	MT8183_MDP_COMP_WROT0,          /* 20 */
+	MT8183_MDP_COMP_WDMA,           /* 21 */
+
+	/* Dummy Engine */
+	MT8183_MDP_COMP_RDMA1,          /* 22 */
+	MT8183_MDP_COMP_RSZ2,           /* 23 */
+	MT8183_MDP_COMP_TDSHP1,         /* 24 */
+	MT8183_MDP_COMP_WROT1,          /* 25 */
+};
+
+static const struct of_device_id mt8183_mdp_probe_infra[MDP_INFRA_MAX] = {
+	[MDP_INFRA_MMSYS] = { .compatible = "mediatek,mt8183-mmsys" },
+	[MDP_INFRA_MUTEX] = { .compatible = "mediatek,mt8183-disp-mutex" },
+	[MDP_INFRA_SCP] = { .compatible = "mediatek,mt8183-scp" }
+};
+
+static const struct mdp_platform_config mt8183_plat_cfg = {
+	.rdma_support_10bit		= true,
+	.rdma_rsz1_sram_sharing		= true,
+	.rdma_upsample_repeat_only	= true,
+	.rsz_disable_dcm_small_sample	= false,
+	.wrot_filter_constraint		= false,
+};
+
+static const u32 mt8183_mutex_idx[MDP_MAX_COMP_COUNT] = {
+	[MDP_COMP_RDMA0] = MUTEX_MOD_IDX_MDP_RDMA0,
+	[MDP_COMP_RSZ0] = MUTEX_MOD_IDX_MDP_RSZ0,
+	[MDP_COMP_RSZ1] = MUTEX_MOD_IDX_MDP_RSZ1,
+	[MDP_COMP_TDSHP0] = MUTEX_MOD_IDX_MDP_TDSHP0,
+	[MDP_COMP_WROT0] = MUTEX_MOD_IDX_MDP_WROT0,
+	[MDP_COMP_WDMA] = MUTEX_MOD_IDX_MDP_WDMA,
+	[MDP_COMP_AAL0] = MUTEX_MOD_IDX_MDP_AAL0,
+	[MDP_COMP_CCORR0] = MUTEX_MOD_IDX_MDP_CCORR0,
+};
+
+static const struct mdp_comp_data mt8183_mdp_comp_data[MDP_MAX_COMP_COUNT] = {
+	[MDP_COMP_WPEI] = {
+		{MDP_COMP_TYPE_WPEI, 0, MT8183_MDP_COMP_WPEI},
+		{0, 0, 0}
+	},
+	[MDP_COMP_WPEO] = {
+		{MDP_COMP_TYPE_EXTO, 2, MT8183_MDP_COMP_WPEO},
+		{0, 0, 0}
+	},
+	[MDP_COMP_WPEI2] = {
+		{MDP_COMP_TYPE_WPEI, 1, MT8183_MDP_COMP_WPEI2},
+		{0, 0, 0}
+	},
+	[MDP_COMP_WPEO2] = {
+		{MDP_COMP_TYPE_EXTO, 3, MT8183_MDP_COMP_WPEO2},
+		{0, 0, 0}
+	},
+	[MDP_COMP_ISP_IMGI] = {
+		{MDP_COMP_TYPE_IMGI, 0, MT8183_MDP_COMP_ISP_IMGI},
+		{0, 0, 4}
+	},
+	[MDP_COMP_ISP_IMGO] = {
+		{MDP_COMP_TYPE_EXTO, 0, MT8183_MDP_COMP_ISP_IMGO},
+		{0, 0, 4}
+	},
+	[MDP_COMP_ISP_IMG2O] = {
+		{MDP_COMP_TYPE_EXTO, 1, MT8183_MDP_COMP_ISP_IMG2O},
+		{0, 0, 0}
+	},
+	[MDP_COMP_CAMIN] = {
+		{MDP_COMP_TYPE_DL_PATH, 0, MT8183_MDP_COMP_CAMIN},
+		{2, 2, 1}
+	},
+	[MDP_COMP_CAMIN2] = {
+		{MDP_COMP_TYPE_DL_PATH, 1, MT8183_MDP_COMP_CAMIN2},
+		{2, 4, 1}
+	},
+	[MDP_COMP_RDMA0] = {
+		{MDP_COMP_TYPE_RDMA, 0, MT8183_MDP_COMP_RDMA0},
+		{2, 0, 0}
+	},
+	[MDP_COMP_CCORR0] = {
+		{MDP_COMP_TYPE_CCORR, 0, MT8183_MDP_COMP_CCORR0},
+		{1, 0, 0}
+	},
+	[MDP_COMP_RSZ0] = {
+		{MDP_COMP_TYPE_RSZ, 0, MT8183_MDP_COMP_RSZ0},
+		{1, 0, 0}
+	},
+	[MDP_COMP_RSZ1] = {
+		{MDP_COMP_TYPE_RSZ, 1, MT8183_MDP_COMP_RSZ1},
+		{1, 0, 0}
+	},
+	[MDP_COMP_TDSHP0] = {
+		{MDP_COMP_TYPE_TDSHP, 0, MT8183_MDP_COMP_TDSHP0},
+		{0, 0, 0}
+	},
+	[MDP_COMP_PATH0_SOUT] = {
+		{MDP_COMP_TYPE_PATH, 0, MT8183_MDP_COMP_PATH0_SOUT},
+		{0, 0, 0}
+	},
+	[MDP_COMP_PATH1_SOUT] = {
+		{MDP_COMP_TYPE_PATH, 1, MT8183_MDP_COMP_PATH1_SOUT},
+		{0, 0, 0}
+	},
+	[MDP_COMP_WROT0] = {
+		{MDP_COMP_TYPE_WROT, 0, MT8183_MDP_COMP_WROT0},
+		{1, 0, 0}
+	},
+	[MDP_COMP_WDMA] = {
+		{MDP_COMP_TYPE_WDMA, 0, MT8183_MDP_COMP_WDMA},
+		{1, 0, 0}
+	},
+};
+
+static const struct of_device_id mt8183_sub_comp_dt_ids[] = {
+	{
+		.compatible = "mediatek,mt8183-mdp3-wdma",
+		.data = (void *)MDP_COMP_TYPE_PATH,
+	}, {
+		.compatible = "mediatek,mt8183-mdp3-wrot",
+		.data = (void *)MDP_COMP_TYPE_PATH,
+	},
+	{}
+};
+
+/*
+ * All 10-bit related formats are not added in the basic format list,
+ * please add the corresponding format settings before use.
+ */
+static const struct mdp_format mt8183_formats[] = {
+	{
+		.pixelformat	= V4L2_PIX_FMT_GREY,
+		.mdp_color	= MDP_COLOR_GREY,
+		.depth		= { 8 },
+		.row_depth	= { 8 },
+		.num_planes	= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_RGB565X,
+		.mdp_color	= MDP_COLOR_BGR565,
+		.depth		= { 16 },
+		.row_depth	= { 16 },
+		.num_planes	= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_RGB565,
+		.mdp_color	= MDP_COLOR_RGB565,
+		.depth		= { 16 },
+		.row_depth	= { 16 },
+		.num_planes	= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_RGB24,
+		.mdp_color	= MDP_COLOR_RGB888,
+		.depth		= { 24 },
+		.row_depth	= { 24 },
+		.num_planes	= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_BGR24,
+		.mdp_color	= MDP_COLOR_BGR888,
+		.depth		= { 24 },
+		.row_depth	= { 24 },
+		.num_planes	= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_ABGR32,
+		.mdp_color	= MDP_COLOR_BGRA8888,
+		.depth		= { 32 },
+		.row_depth	= { 32 },
+		.num_planes	= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_ARGB32,
+		.mdp_color	= MDP_COLOR_ARGB8888,
+		.depth		= { 32 },
+		.row_depth	= { 32 },
+		.num_planes	= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_UYVY,
+		.mdp_color	= MDP_COLOR_UYVY,
+		.depth		= { 16 },
+		.row_depth	= { 16 },
+		.num_planes	= 1,
+		.walign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_VYUY,
+		.mdp_color	= MDP_COLOR_VYUY,
+		.depth		= { 16 },
+		.row_depth	= { 16 },
+		.num_planes	= 1,
+		.walign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_YUYV,
+		.mdp_color	= MDP_COLOR_YUYV,
+		.depth		= { 16 },
+		.row_depth	= { 16 },
+		.num_planes	= 1,
+		.walign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_YVYU,
+		.mdp_color	= MDP_COLOR_YVYU,
+		.depth		= { 16 },
+		.row_depth	= { 16 },
+		.num_planes	= 1,
+		.walign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_YUV420,
+		.mdp_color	= MDP_COLOR_I420,
+		.depth		= { 12 },
+		.row_depth	= { 8 },
+		.num_planes	= 1,
+		.walign		= 1,
+		.halign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_YVU420,
+		.mdp_color	= MDP_COLOR_YV12,
+		.depth		= { 12 },
+		.row_depth	= { 8 },
+		.num_planes	= 1,
+		.walign		= 1,
+		.halign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_NV12,
+		.mdp_color	= MDP_COLOR_NV12,
+		.depth		= { 12 },
+		.row_depth	= { 8 },
+		.num_planes	= 1,
+		.walign		= 1,
+		.halign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_NV21,
+		.mdp_color	= MDP_COLOR_NV21,
+		.depth		= { 12 },
+		.row_depth	= { 8 },
+		.num_planes	= 1,
+		.walign		= 1,
+		.halign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_NV16,
+		.mdp_color	= MDP_COLOR_NV16,
+		.depth		= { 16 },
+		.row_depth	= { 8 },
+		.num_planes	= 1,
+		.walign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_NV61,
+		.mdp_color	= MDP_COLOR_NV61,
+		.depth		= { 16 },
+		.row_depth	= { 8 },
+		.num_planes	= 1,
+		.walign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_NV24,
+		.mdp_color	= MDP_COLOR_NV24,
+		.depth		= { 24 },
+		.row_depth	= { 8 },
+		.num_planes	= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_NV42,
+		.mdp_color	= MDP_COLOR_NV42,
+		.depth		= { 24 },
+		.row_depth	= { 8 },
+		.num_planes	= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_MT21C,
+		.mdp_color	= MDP_COLOR_420_BLK_UFO,
+		.depth		= { 8, 4 },
+		.row_depth	= { 8, 8 },
+		.num_planes	= 2,
+		.walign		= 4,
+		.halign		= 5,
+		.flags		= MDP_FMT_FLAG_OUTPUT,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_MM21,
+		.mdp_color	= MDP_COLOR_420_BLK,
+		.depth		= { 8, 4 },
+		.row_depth	= { 8, 8 },
+		.num_planes	= 2,
+		.walign		= 4,
+		.halign		= 5,
+		.flags		= MDP_FMT_FLAG_OUTPUT,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_NV12M,
+		.mdp_color	= MDP_COLOR_NV12,
+		.depth		= { 8, 4 },
+		.row_depth	= { 8, 8 },
+		.num_planes	= 2,
+		.walign		= 1,
+		.halign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_NV21M,
+		.mdp_color	= MDP_COLOR_NV21,
+		.depth		= { 8, 4 },
+		.row_depth	= { 8, 8 },
+		.num_planes	= 2,
+		.walign		= 1,
+		.halign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_NV16M,
+		.mdp_color	= MDP_COLOR_NV16,
+		.depth		= { 8, 8 },
+		.row_depth	= { 8, 8 },
+		.num_planes	= 2,
+		.walign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_NV61M,
+		.mdp_color	= MDP_COLOR_NV61,
+		.depth		= { 8, 8 },
+		.row_depth	= { 8, 8 },
+		.num_planes	= 2,
+		.walign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_YUV420M,
+		.mdp_color	= MDP_COLOR_I420,
+		.depth		= { 8, 2, 2 },
+		.row_depth	= { 8, 4, 4 },
+		.num_planes	= 3,
+		.walign		= 1,
+		.halign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}, {
+		.pixelformat	= V4L2_PIX_FMT_YVU420M,
+		.mdp_color	= MDP_COLOR_YV12,
+		.depth		= { 8, 2, 2 },
+		.row_depth	= { 8, 4, 4 },
+		.num_planes	= 3,
+		.walign		= 1,
+		.halign		= 1,
+		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
+	}
+};
+
+static const struct mdp_limit mt8183_mdp_def_limit = {
+	.out_limit = {
+		.wmin	= 16,
+		.hmin	= 16,
+		.wmax	= 8176,
+		.hmax	= 8176,
+	},
+	.cap_limit = {
+		.wmin	= 2,
+		.hmin	= 2,
+		.wmax	= 8176,
+		.hmax	= 8176,
+	},
+	.h_scale_up_max = 32,
+	.v_scale_up_max = 32,
+	.h_scale_down_max = 20,
+	.v_scale_down_max = 128,
+};
+
+static const struct mdp_pipe_info mt8183_pipe_info[] = {
+	[MDP_PIPE_WPEI] = {MDP_PIPE_WPEI, 0},
+	[MDP_PIPE_WPEI2] = {MDP_PIPE_WPEI2, 1},
+	[MDP_PIPE_IMGI] = {MDP_PIPE_IMGI, 2},
+	[MDP_PIPE_RDMA0] = {MDP_PIPE_RDMA0, 3}
+};
+
+const struct mtk_mdp_driver_data mt8183_mdp_driver_data = {
+	.mdp_plat_id = MT8183,
+	.mdp_probe_infra = mt8183_mdp_probe_infra,
+	.mdp_cfg = &mt8183_plat_cfg,
+	.mdp_mutex_table_idx = mt8183_mutex_idx,
+	.comp_data = mt8183_mdp_comp_data,
+	.comp_data_len = ARRAY_SIZE(mt8183_mdp_comp_data),
+	.mdp_sub_comp_dt_ids = mt8183_sub_comp_dt_ids,
+	.format = mt8183_formats,
+	.format_len = ARRAY_SIZE(mt8183_formats),
+	.def_limit = &mt8183_mdp_def_limit,
+	.pipe_info = mt8183_pipe_info,
+	.pipe_info_len = ARRAY_SIZE(mt8183_pipe_info),
+};
+
+s32 mdp_cfg_get_id_inner(struct mdp_dev *mdp_dev, enum mtk_mdp_comp_id id)
+{
+	if (!mdp_dev)
+		return MDP_COMP_NONE;
+	if (id <= MDP_COMP_NONE || id >= MDP_MAX_COMP_COUNT)
+		return MDP_COMP_NONE;
+
+	return mdp_dev->mdp_data->comp_data[id].match.inner_id;
+}
+
+enum mtk_mdp_comp_id mdp_cfg_get_id_public(struct mdp_dev *mdp_dev, s32 inner_id)
+{
+	enum mtk_mdp_comp_id public_id = MDP_COMP_NONE;
+	u32 i;
+
+	if (IS_ERR(mdp_dev) || !inner_id)
+		goto err_public_id;
+
+	for (i = 0; i < MDP_MAX_COMP_COUNT; i++) {
+		if (mdp_dev->mdp_data->comp_data[i].match.inner_id == inner_id) {
+			public_id = i;
+			return public_id;
+		}
+	}
+
+err_public_id:
+	return public_id;
+}
diff --git a/drivers/media/platform/mediatek/mdp3/mdp_sm_mt8183.h b/drivers/media/platform/mediatek/mdp3/mdp_sm_mt8183.h
new file mode 100644
index 000000000000..85637084520f
--- /dev/null
+++ b/drivers/media/platform/mediatek/mdp3/mdp_sm_mt8183.h
@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2023 MediaTek Inc.
+ * Author: Ping-Hsun Wu <ping-hsun.wu@mediatek.com>
+ */
+
+#ifndef __MDP_SM_MT8183_H__
+#define __MDP_SM_MT8183_H__
+
+#include "mtk-mdp3-type.h"
+
+/*
+ * ISP-MDP generic output information
+ * MD5 of the target SCP prebuild:
+ *     2d995ddb5c3b0cf26e96d6a823481886
+ */
+
+#define IMG_MAX_SUBFRAMES_8183      14
+
+struct img_comp_frame_8183 {
+	u32 output_disable:1;
+	u32 bypass:1;
+	u16 in_width;
+	u16 in_height;
+	u16 out_width;
+	u16 out_height;
+	struct img_crop crop;
+	u16 in_total_width;
+	u16 out_total_width;
+} __packed;
+
+struct img_comp_subfrm_8183 {
+	u32 tile_disable:1;
+	struct img_region in;
+	struct img_region out;
+	struct img_offset luma;
+	struct img_offset chroma;
+	s16 out_vertical; /* Output vertical index */
+	s16 out_horizontal; /* Output horizontal index */
+} __packed;
+
+struct mdp_rdma_subfrm_8183 {
+	u32 offset[IMG_MAX_PLANES];
+	u32 offset_0_p;
+	u32 src;
+	u32 clip;
+	u32 clip_ofst;
+} __packed;
+
+struct mdp_rdma_data_8183 {
+	u32 src_ctrl;
+	u32 control;
+	u32 iova[IMG_MAX_PLANES];
+	u32 iova_end[IMG_MAX_PLANES];
+	u32 mf_bkgd;
+	u32 mf_bkgd_in_pxl;
+	u32 sf_bkgd;
+	u32 ufo_dec_y;
+	u32 ufo_dec_c;
+	u32 transform;
+	struct mdp_rdma_subfrm_8183 subfrms[IMG_MAX_SUBFRAMES_8183];
+} __packed;
+
+struct mdp_rsz_subfrm_8183 {
+	u32 control2;
+	u32 src;
+	u32 clip;
+} __packed;
+
+struct mdp_rsz_data_8183 {
+	u32 coeff_step_x;
+	u32 coeff_step_y;
+	u32 control1;
+	u32 control2;
+	struct mdp_rsz_subfrm_8183 subfrms[IMG_MAX_SUBFRAMES_8183];
+} __packed;
+
+struct mdp_wrot_subfrm_8183 {
+	u32 offset[IMG_MAX_PLANES];
+	u32 src;
+	u32 clip;
+	u32 clip_ofst;
+	u32 main_buf;
+} __packed;
+
+struct mdp_wrot_data_8183 {
+	u32 iova[IMG_MAX_PLANES];
+	u32 control;
+	u32 stride[IMG_MAX_PLANES];
+	u32 mat_ctrl;
+	u32 fifo_test;
+	u32 filter;
+	struct mdp_wrot_subfrm_8183 subfrms[IMG_MAX_SUBFRAMES_8183];
+} __packed;
+
+struct mdp_wdma_subfrm_8183 {
+	u32 offset[IMG_MAX_PLANES];
+	u32 src;
+	u32 clip;
+	u32 clip_ofst;
+} __packed;
+
+struct mdp_wdma_data_8183 {
+	u32 wdma_cfg;
+	u32 iova[IMG_MAX_PLANES];
+	u32 w_in_byte;
+	u32 uv_stride;
+	struct mdp_wdma_subfrm_8183 subfrms[IMG_MAX_SUBFRAMES_8183];
+} __packed;
+
+struct isp_data_8183 {
+	u64 dl_flags; /* 1 << (enum mdp_comp_type) */
+	u32 smxi_iova[4];
+	u32 cq_idx;
+	u32 cq_iova;
+	u32 tpipe_iova[IMG_MAX_SUBFRAMES_8183];
+} __packed;
+
+struct img_compparam_8183 {
+	u16 type; /* enum mdp_comp_id */
+	u16 id; /* engine alias_id */
+	u32 input;
+	u32 outputs[IMG_MAX_HW_OUTPUTS];
+	u32 num_outputs;
+	struct img_comp_frame_8183 frame;
+	struct img_comp_subfrm_8183 subfrms[IMG_MAX_SUBFRAMES_8183];
+	u32 num_subfrms;
+	union {
+		struct mdp_rdma_data_8183 rdma;
+		struct mdp_rsz_data_8183 rsz;
+		struct mdp_wrot_data_8183 wrot;
+		struct mdp_wdma_data_8183 wdma;
+		struct isp_data_8183 isp;
+	};
+} __packed;
+
+struct img_config_8183 {
+	struct img_compparam_8183 components[IMG_MAX_COMPONENTS];
+	u32 num_components;
+	struct img_mmsys_ctrl ctrls[IMG_MAX_SUBFRAMES_8183];
+	u32 num_subfrms;
+} __packed;
+
+#endif  /* __MDP_SM_MT8183_H__ */
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-img-ipi.h b/drivers/media/platform/mediatek/mdp3/mtk-img-ipi.h
index c7f231f8ea3e..22b8b9a10ef7 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-img-ipi.h
+++ b/drivers/media/platform/mediatek/mdp3/mtk-img-ipi.h
@@ -8,19 +8,11 @@
 #ifndef __MTK_IMG_IPI_H__
 #define __MTK_IMG_IPI_H__
 
-#include <linux/types.h>
+#include <linux/err.h>
+#include "mdp_sm_mt8183.h"
+#include "mtk-mdp3-type.h"
 
-/*
- * ISP-MDP generic input information
- * MD5 of the target SCP blob:
- *     6da52bdcf4bf76a0983b313e1d4745d6
- */
-
-#define IMG_MAX_HW_INPUTS	3
-
-#define IMG_MAX_HW_OUTPUTS	4
-
-#define IMG_MAX_PLANES		3
+/* ISP-MDP generic input information */
 
 #define IMG_IPI_INIT    1
 #define IMG_IPI_DEINIT  2
@@ -71,17 +63,6 @@ struct img_image_buffer {
 
 #define IMG_SUBPIXEL_SHIFT	20
 
-struct img_crop {
-	s32 left;
-	s32 top;
-	u32 width;
-	u32 height;
-	u32 left_subpix;
-	u32 top_subpix;
-	u32 width_subpix;
-	u32 height_subpix;
-} __packed;
-
 #define IMG_CTRL_FLAG_HFLIP	BIT(0)
 #define IMG_CTRL_FLAG_DITHER	BIT(1)
 #define IMG_CTRL_FLAG_SHARPNESS	BIT(4)
@@ -132,159 +113,37 @@ struct img_frameparam {
 	struct img_ipi_frameparam frameparam;
 } __packed;
 
-/* ISP-MDP generic output information */
+/* Platform config indicator */
+#define MT8183 8183
 
-struct img_comp_frame {
-	u32 output_disable;
-	u32 bypass;
-	u32 in_width;
-	u32 in_height;
-	u32 out_width;
-	u32 out_height;
-	struct img_crop crop;
-	u32 in_total_width;
-	u32 out_total_width;
-} __packed;
+#define CFG_CHECK(plat, p_id) ((plat) == (p_id))
 
-struct img_region {
-	s32 left;
-	s32 right;
-	s32 top;
-	s32 bottom;
-} __packed;
+#define _CFG_OFST(plat, cfg, ofst) ((void *)(&((cfg)->config_##plat) + (ofst)))
+#define CFG_OFST(plat, cfg, ofst) \
+	(IS_ERR_OR_NULL(cfg) ? NULL : _CFG_OFST(plat, cfg, ofst))
 
-struct img_offset {
-	s32 left;
-	s32 top;
-	u32 left_subpix;
-	u32 top_subpix;
-} __packed;
-
-struct img_comp_subfrm {
-	u32 tile_disable;
-	struct img_region in;
-	struct img_region out;
-	struct img_offset luma;
-	struct img_offset chroma;
-	s32 out_vertical; /* Output vertical index */
-	s32 out_horizontal; /* Output horizontal index */
-} __packed;
+#define _CFG_ADDR(plat, cfg, mem) (&((cfg)->config_##plat.mem))
+#define CFG_ADDR(plat, cfg, mem) \
+	(IS_ERR_OR_NULL(cfg) ? NULL : _CFG_ADDR(plat, cfg, mem))
 
-#define IMG_MAX_SUBFRAMES	14
+#define _CFG_GET(plat, cfg, mem) ((cfg)->config_##plat.mem)
+#define CFG_GET(plat, cfg, mem) \
+	(IS_ERR_OR_NULL(cfg) ? 0 : _CFG_GET(plat, cfg, mem))
 
-struct mdp_rdma_subfrm {
-	u32 offset[IMG_MAX_PLANES];
-	u32 offset_0_p;
-	u32 src;
-	u32 clip;
-	u32 clip_ofst;
-} __packed;
-
-struct mdp_rdma_data {
-	u32 src_ctrl;
-	u32 control;
-	u32 iova[IMG_MAX_PLANES];
-	u32 iova_end[IMG_MAX_PLANES];
-	u32 mf_bkgd;
-	u32 mf_bkgd_in_pxl;
-	u32 sf_bkgd;
-	u32 ufo_dec_y;
-	u32 ufo_dec_c;
-	u32 transform;
-	struct mdp_rdma_subfrm subfrms[IMG_MAX_SUBFRAMES];
-} __packed;
-
-struct mdp_rsz_subfrm {
-	u32 control2;
-	u32 src;
-	u32 clip;
-} __packed;
-
-struct mdp_rsz_data {
-	u32 coeff_step_x;
-	u32 coeff_step_y;
-	u32 control1;
-	u32 control2;
-	struct mdp_rsz_subfrm subfrms[IMG_MAX_SUBFRAMES];
-} __packed;
+#define _CFG_COMP(plat, comp, mem) ((comp)->comp_##plat.mem)
+#define CFG_COMP(plat, comp, mem) \
+	(IS_ERR_OR_NULL(comp) ? 0 : _CFG_COMP(plat, comp, mem))
 
-struct mdp_wrot_subfrm {
-	u32 offset[IMG_MAX_PLANES];
-	u32 src;
-	u32 clip;
-	u32 clip_ofst;
-	u32 main_buf;
-} __packed;
-
-struct mdp_wrot_data {
-	u32 iova[IMG_MAX_PLANES];
-	u32 control;
-	u32 stride[IMG_MAX_PLANES];
-	u32 mat_ctrl;
-	u32 fifo_test;
-	u32 filter;
-	struct mdp_wrot_subfrm subfrms[IMG_MAX_SUBFRAMES];
-} __packed;
-
-struct mdp_wdma_subfrm {
-	u32 offset[IMG_MAX_PLANES];
-	u32 src;
-	u32 clip;
-	u32 clip_ofst;
-} __packed;
-
-struct mdp_wdma_data {
-	u32 wdma_cfg;
-	u32 iova[IMG_MAX_PLANES];
-	u32 w_in_byte;
-	u32 uv_stride;
-	struct mdp_wdma_subfrm subfrms[IMG_MAX_SUBFRAMES];
-} __packed;
-
-struct isp_data {
-	u64 dl_flags; /* 1 << (enum mdp_comp_type) */
-	u32 smxi_iova[4];
-	u32 cq_idx;
-	u32 cq_iova;
-	u32 tpipe_iova[IMG_MAX_SUBFRAMES];
+struct img_config {
+	union {
+		struct img_config_8183 config_8183;
+	};
 } __packed;
 
 struct img_compparam {
-	u32 type; /* enum mdp_comp_id */
-	u32 id; /* engine alias_id */
-	u32 input;
-	u32 outputs[IMG_MAX_HW_OUTPUTS];
-	u32 num_outputs;
-	struct img_comp_frame frame;
-	struct img_comp_subfrm subfrms[IMG_MAX_SUBFRAMES];
-	u32 num_subfrms;
 	union {
-		struct mdp_rdma_data rdma;
-		struct mdp_rsz_data rsz;
-		struct mdp_wrot_data wrot;
-		struct mdp_wdma_data wdma;
-		struct isp_data isp;
+		struct img_compparam_8183 comp_8183;
 	};
 } __packed;
 
-#define IMG_MAX_COMPONENTS	20
-
-struct img_mux {
-	u32 reg;
-	u32 value;
-	u32 subsys_id;
-} __packed;
-
-struct img_mmsys_ctrl {
-	struct img_mux sets[IMG_MAX_COMPONENTS * 2];
-	u32 num_sets;
-} __packed;
-
-struct img_config {
-	struct img_compparam components[IMG_MAX_COMPONENTS];
-	u32 num_components;
-	struct img_mmsys_ctrl ctrls[IMG_MAX_SUBFRAMES];
-	u32 num_subfrms;
-} __packed;
-
 #endif  /* __MTK_IMG_IPI_H__ */
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cfg.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cfg.h
new file mode 100644
index 000000000000..dee57cc4a954
--- /dev/null
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cfg.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2023 MediaTek Inc.
+ * Author: Ping-Hsun Wu <ping-hsun.wu@mediatek.com>
+ */
+
+#ifndef __MTK_MDP3_CFG_H__
+#define __MTK_MDP3_CFG_H__
+
+#include <linux/types.h>
+
+extern const struct mtk_mdp_driver_data mt8183_mdp_driver_data;
+
+struct mdp_dev;
+enum mtk_mdp_comp_id;
+
+s32 mdp_cfg_get_id_inner(struct mdp_dev *mdp_dev, enum mtk_mdp_comp_id id);
+enum mtk_mdp_comp_id mdp_cfg_get_id_public(struct mdp_dev *mdp_dev, s32 id);
+
+#endif  /* __MTK_MDP3_CFG_H__ */
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c
index 124c1b96e96b..3177592490be 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c
@@ -10,6 +10,7 @@
 #include "mtk-mdp3-comp.h"
 #include "mtk-mdp3-core.h"
 #include "mtk-mdp3-m2m.h"
+#include "mtk-img-ipi.h"
 
 #define MDP_PATH_MAX_COMPS	IMG_MAX_COMPONENTS
 
@@ -28,50 +29,62 @@ struct mdp_path {
  #define call_op(ctx, op, ...) \
 	(has_op(ctx, op) ? (ctx)->comp->ops->op(ctx, ##__VA_ARGS__) : 0)
 
-static bool is_output_disabled(const struct img_compparam *param, u32 count)
+static bool is_output_disabled(int p_id, const struct img_compparam *param, u32 count)
 {
-	return (count < param->num_subfrms) ?
-		(param->frame.output_disable ||
-		param->subfrms[count].tile_disable) :
-		true;
+	u32 num = 0;
+	bool dis_output = false;
+	bool dis_tile = false;
+
+	if (CFG_CHECK(MT8183, p_id)) {
+		num = CFG_COMP(MT8183, param, num_subfrms);
+		dis_output = CFG_COMP(MT8183, param, frame.output_disable);
+		dis_tile = CFG_COMP(MT8183, param, frame.output_disable);
+	}
+
+	return (count < num) ? (dis_output || dis_tile) : true;
 }
 
 static int mdp_path_subfrm_require(const struct mdp_path *path,
 				   struct mdp_cmdq_cmd *cmd,
 				   s32 *mutex_id, u32 count)
 {
-	const struct img_config *config = path->config;
+	const int p_id = path->mdp_dev->mdp_data->mdp_plat_id;
 	const struct mdp_comp_ctx *ctx;
 	const struct mtk_mdp_driver_data *data = path->mdp_dev->mdp_data;
 	struct device *dev = &path->mdp_dev->pdev->dev;
 	struct mtk_mutex **mutex = path->mdp_dev->mdp_mutex;
 	int id, index;
+	u32 num_comp = 0;
+
+	if (CFG_CHECK(MT8183, p_id))
+		num_comp = CFG_GET(MT8183, path->config, num_components);
 
 	/* Decide which mutex to use based on the current pipeline */
-	switch (path->comps[0].comp->id) {
+	switch (path->comps[0].comp->public_id) {
 	case MDP_COMP_RDMA0:
-		*mutex_id = MDP_PIPE_RDMA0;
+		index = MDP_PIPE_RDMA0;
 		break;
 	case MDP_COMP_ISP_IMGI:
-		*mutex_id = MDP_PIPE_IMGI;
+		index = MDP_PIPE_IMGI;
 		break;
 	case MDP_COMP_WPEI:
-		*mutex_id = MDP_PIPE_WPEI;
+		index = MDP_PIPE_WPEI;
 		break;
 	case MDP_COMP_WPEI2:
-		*mutex_id = MDP_PIPE_WPEI2;
+		index = MDP_PIPE_WPEI2;
 		break;
 	default:
 		dev_err(dev, "Unknown pipeline and no mutex is assigned");
 		return -EINVAL;
 	}
+	*mutex_id = data->pipe_info[index].mutex_id;
 
 	/* Set mutex mod */
-	for (index = 0; index < config->num_components; index++) {
+	for (index = 0; index < num_comp; index++) {
 		ctx = &path->comps[index];
-		if (is_output_disabled(ctx->param, count))
+		if (is_output_disabled(p_id, ctx->param, count))
 			continue;
-		id = ctx->comp->id;
+		id = ctx->comp->public_id;
 		mtk_mutex_write_mod(mutex[*mutex_id],
 				    data->mdp_mutex_table_idx[id], false);
 	}
@@ -86,11 +99,12 @@ static int mdp_path_subfrm_run(const struct mdp_path *path,
 			       struct mdp_cmdq_cmd *cmd,
 			       s32 *mutex_id, u32 count)
 {
-	const struct img_config *config = path->config;
+	const int p_id = path->mdp_dev->mdp_data->mdp_plat_id;
 	const struct mdp_comp_ctx *ctx;
 	struct device *dev = &path->mdp_dev->pdev->dev;
 	struct mtk_mutex **mutex = path->mdp_dev->mdp_mutex;
 	int index;
+	u32 num_comp = 0;
 	s32 event;
 
 	if (-1 == *mutex_id) {
@@ -98,11 +112,14 @@ static int mdp_path_subfrm_run(const struct mdp_path *path,
 		return -EINVAL;
 	}
 
+	if (CFG_CHECK(MT8183, p_id))
+		num_comp = CFG_GET(MT8183, path->config, num_components);
+
 	/* Wait WROT SRAM shared to DISP RDMA */
 	/* Clear SOF event for each engine */
-	for (index = 0; index < config->num_components; index++) {
+	for (index = 0; index < num_comp; index++) {
 		ctx = &path->comps[index];
-		if (is_output_disabled(ctx->param, count))
+		if (is_output_disabled(p_id, ctx->param, count))
 			continue;
 		event = ctx->comp->gce_event[MDP_GCE_EVENT_SOF];
 		if (event != MDP_GCE_NO_EVENT)
@@ -113,9 +130,9 @@ static int mdp_path_subfrm_run(const struct mdp_path *path,
 	mtk_mutex_enable_by_cmdq(mutex[*mutex_id], (void *)&cmd->pkt);
 
 	/* Wait SOF events and clear mutex modules (optional) */
-	for (index = 0; index < config->num_components; index++) {
+	for (index = 0; index < num_comp; index++) {
 		ctx = &path->comps[index];
-		if (is_output_disabled(ctx->param, count))
+		if (is_output_disabled(p_id, ctx->param, count))
 			continue;
 		event = ctx->comp->gce_event[MDP_GCE_EVENT_SOF];
 		if (event != MDP_GCE_NO_EVENT)
@@ -127,16 +144,22 @@ static int mdp_path_subfrm_run(const struct mdp_path *path,
 
 static int mdp_path_ctx_init(struct mdp_dev *mdp, struct mdp_path *path)
 {
-	const struct img_config *config = path->config;
+	const int p_id = mdp->mdp_data->mdp_plat_id;
+	void *param = NULL;
 	int index, ret;
+	u32 num_comp = 0;
 
-	if (config->num_components < 1)
+	if (CFG_CHECK(MT8183, p_id))
+		num_comp = CFG_GET(MT8183, path->config, num_components);
+
+	if (num_comp < 1)
 		return -EINVAL;
 
-	for (index = 0; index < config->num_components; index++) {
+	for (index = 0; index < num_comp; index++) {
+		if (CFG_CHECK(MT8183, p_id))
+			param = (void *)CFG_ADDR(MT8183, path->config, components[index]);
 		ret = mdp_comp_ctx_config(mdp, &path->comps[index],
-					  &config->components[index],
-					  path->param);
+					  param, path->param);
 		if (ret)
 			return ret;
 	}
@@ -147,12 +170,19 @@ static int mdp_path_ctx_init(struct mdp_dev *mdp, struct mdp_path *path)
 static int mdp_path_config_subfrm(struct mdp_cmdq_cmd *cmd,
 				  struct mdp_path *path, u32 count)
 {
-	const struct img_config *config = path->config;
-	const struct img_mmsys_ctrl *ctrl = &config->ctrls[count];
+	const int p_id = path->mdp_dev->mdp_data->mdp_plat_id;
+	const struct img_mmsys_ctrl *ctrl = NULL;
 	const struct img_mux *set;
 	struct mdp_comp_ctx *ctx;
 	s32 mutex_id;
 	int index, ret;
+	u32 num_comp = 0;
+
+	if (CFG_CHECK(MT8183, p_id))
+		num_comp = CFG_GET(MT8183, path->config, num_components);
+
+	if (CFG_CHECK(MT8183, p_id))
+		ctrl = CFG_ADDR(MT8183, path->config, ctrls[count]);
 
 	/* Acquire components */
 	ret = mdp_path_subfrm_require(path, cmd, &mutex_id, count);
@@ -165,9 +195,9 @@ static int mdp_path_config_subfrm(struct mdp_cmdq_cmd *cmd,
 				    set->value, 0xFFFFFFFF);
 	}
 	/* Config sub-frame information */
-	for (index = (config->num_components - 1); index >= 0; index--) {
+	for (index = (num_comp - 1); index >= 0; index--) {
 		ctx = &path->comps[index];
-		if (is_output_disabled(ctx->param, count))
+		if (is_output_disabled(p_id, ctx->param, count))
 			continue;
 		ret = call_op(ctx, config_subfrm, cmd, count);
 		if (ret)
@@ -178,16 +208,16 @@ static int mdp_path_config_subfrm(struct mdp_cmdq_cmd *cmd,
 	if (ret)
 		return ret;
 	/* Wait components done */
-	for (index = 0; index < config->num_components; index++) {
+	for (index = 0; index < num_comp; index++) {
 		ctx = &path->comps[index];
-		if (is_output_disabled(ctx->param, count))
+		if (is_output_disabled(p_id, ctx->param, count))
 			continue;
 		ret = call_op(ctx, wait_comp_event, cmd);
 		if (ret)
 			return ret;
 	}
 	/* Advance to the next sub-frame */
-	for (index = 0; index < config->num_components; index++) {
+	for (index = 0; index < num_comp; index++) {
 		ctx = &path->comps[index];
 		ret = call_op(ctx, advance_subfrm, cmd, count);
 		if (ret)
@@ -206,23 +236,35 @@ static int mdp_path_config_subfrm(struct mdp_cmdq_cmd *cmd,
 static int mdp_path_config(struct mdp_dev *mdp, struct mdp_cmdq_cmd *cmd,
 			   struct mdp_path *path)
 {
-	const struct img_config *config = path->config;
+	const int p_id = mdp->mdp_data->mdp_plat_id;
 	struct mdp_comp_ctx *ctx;
 	int index, count, ret;
+	u32 num_comp = 0;
+	u32 num_sub = 0;
+
+	if (CFG_CHECK(MT8183, p_id))
+		num_comp = CFG_GET(MT8183, path->config, num_components);
+
+	if (CFG_CHECK(MT8183, p_id))
+		num_sub = CFG_GET(MT8183, path->config, num_subfrms);
 
 	/* Config path frame */
 	/* Reset components */
-	for (index = 0; index < config->num_components; index++) {
+	for (index = 0; index < num_comp; index++) {
 		ctx = &path->comps[index];
 		ret = call_op(ctx, init_comp, cmd);
 		if (ret)
 			return ret;
 	}
 	/* Config frame mode */
-	for (index = 0; index < config->num_components; index++) {
-		const struct v4l2_rect *compose =
-			path->composes[ctx->param->outputs[0]];
+	for (index = 0; index < num_comp; index++) {
+		const struct v4l2_rect *compose;
+		u32 out = 0;
+
+		if (CFG_CHECK(MT8183, p_id))
+			out = CFG_COMP(MT8183, ctx->param, outputs[0]);
 
+		compose = path->composes[out];
 		ctx = &path->comps[index];
 		ret = call_op(ctx, config_frame, cmd, compose);
 		if (ret)
@@ -230,13 +272,13 @@ static int mdp_path_config(struct mdp_dev *mdp, struct mdp_cmdq_cmd *cmd,
 	}
 
 	/* Config path sub-frames */
-	for (count = 0; count < config->num_subfrms; count++) {
+	for (count = 0; count < num_sub; count++) {
 		ret = mdp_path_config_subfrm(cmd, path, count);
 		if (ret)
 			return ret;
 	}
 	/* Post processing information */
-	for (index = 0; index < config->num_components; index++) {
+	for (index = 0; index < num_comp; index++) {
 		ctx = &path->comps[index];
 		ret = call_op(ctx, post_process, cmd);
 		if (ret)
@@ -286,11 +328,13 @@ static void mdp_auto_release_work(struct work_struct *work)
 {
 	struct mdp_cmdq_cmd *cmd;
 	struct mdp_dev *mdp;
+	int id;
 
 	cmd = container_of(work, struct mdp_cmdq_cmd, auto_release_work);
 	mdp = cmd->mdp;
 
-	mtk_mutex_unprepare(mdp->mdp_mutex[MDP_PIPE_RDMA0]);
+	id = mdp->mdp_data->pipe_info[MDP_PIPE_RDMA0].mutex_id;
+	mtk_mutex_unprepare(mdp->mdp_mutex[id]);
 	mdp_comp_clocks_off(&mdp->pdev->dev, cmd->comps,
 			    cmd->num_comps);
 
@@ -310,6 +354,7 @@ static void mdp_handle_cmdq_callback(struct mbox_client *cl, void *mssg)
 	struct cmdq_cb_data *data;
 	struct mdp_dev *mdp;
 	struct device *dev;
+	int id;
 
 	if (!mssg) {
 		pr_info("%s:no callback data\n", __func__);
@@ -335,7 +380,8 @@ static void mdp_handle_cmdq_callback(struct mbox_client *cl, void *mssg)
 	INIT_WORK(&cmd->auto_release_work, mdp_auto_release_work);
 	if (!queue_work(mdp->clock_wq, &cmd->auto_release_work)) {
 		dev_err(dev, "%s:queue_work fail!\n", __func__);
-		mtk_mutex_unprepare(mdp->mdp_mutex[MDP_PIPE_RDMA0]);
+		id = mdp->mdp_data->pipe_info[MDP_PIPE_RDMA0].mutex_id;
+		mtk_mutex_unprepare(mdp->mdp_mutex[id]);
 		mdp_comp_clocks_off(&mdp->pdev->dev, cmd->comps,
 				    cmd->num_comps);
 
@@ -356,7 +402,9 @@ int mdp_cmdq_send(struct mdp_dev *mdp, struct mdp_cmdq_param *param)
 	struct mdp_cmdq_cmd *cmd = NULL;
 	struct mdp_comp *comps = NULL;
 	struct device *dev = &mdp->pdev->dev;
+	const int p_id = mdp->mdp_data->mdp_plat_id;
 	int i, ret;
+	u32 num_comp = 0;
 
 	atomic_inc(&mdp->job_count);
 	if (atomic_read(&mdp->suspended)) {
@@ -374,8 +422,13 @@ int mdp_cmdq_send(struct mdp_dev *mdp, struct mdp_cmdq_param *param)
 	if (ret)
 		goto err_free_cmd;
 
-	comps = kcalloc(param->config->num_components, sizeof(*comps),
-			GFP_KERNEL);
+	if (CFG_CHECK(MT8183, p_id)) {
+		num_comp = CFG_GET(MT8183, param->config, num_components);
+	} else {
+		ret = -EINVAL;
+		goto err_destroy_pkt;
+	}
+	comps = kcalloc(num_comp, sizeof(*comps), GFP_KERNEL);
 	if (!comps) {
 		ret = -ENOMEM;
 		goto err_destroy_pkt;
@@ -387,7 +440,8 @@ int mdp_cmdq_send(struct mdp_dev *mdp, struct mdp_cmdq_param *param)
 		goto err_free_comps;
 	}
 
-	ret = mtk_mutex_prepare(mdp->mdp_mutex[MDP_PIPE_RDMA0]);
+	i = mdp->mdp_data->pipe_info[MDP_PIPE_RDMA0].mutex_id;
+	ret = mtk_mutex_prepare(mdp->mdp_mutex[i]);
 	if (ret) {
 		dev_err(dev, "Fail to enable mutex clk\n");
 		goto err_free_path;
@@ -406,7 +460,6 @@ int mdp_cmdq_send(struct mdp_dev *mdp, struct mdp_cmdq_param *param)
 		path->composes[i] = param->composes[i] ?
 			param->composes[i] : &path->bounds[i];
 	}
-
 	ret = mdp_path_ctx_init(mdp, path);
 	if (ret) {
 		dev_err(dev, "mdp_path_ctx_init error\n");
@@ -420,7 +473,7 @@ int mdp_cmdq_send(struct mdp_dev *mdp, struct mdp_cmdq_param *param)
 	}
 	cmdq_pkt_finalize(&cmd->pkt);
 
-	for (i = 0; i < param->config->num_components; i++)
+	for (i = 0; i < num_comp; i++)
 		memcpy(&comps[i], path->comps[i].comp,
 		       sizeof(struct mdp_comp));
 
@@ -429,7 +482,7 @@ int mdp_cmdq_send(struct mdp_dev *mdp, struct mdp_cmdq_param *param)
 	cmd->user_cmdq_cb = param->cmdq_cb;
 	cmd->user_cb_data = param->cb_data;
 	cmd->comps = comps;
-	cmd->num_comps = param->config->num_components;
+	cmd->num_comps = num_comp;
 	cmd->mdp_ctx = param->mdp_ctx;
 
 	ret = mdp_comp_clocks_on(&mdp->pdev->dev, cmd->comps, cmd->num_comps);
@@ -453,7 +506,8 @@ err_clock_off:
 	mdp_comp_clocks_off(&mdp->pdev->dev, cmd->comps,
 			    cmd->num_comps);
 err_free_path:
-	mtk_mutex_unprepare(mdp->mdp_mutex[MDP_PIPE_RDMA0]);
+	i = mdp->mdp_data->pipe_info[MDP_PIPE_RDMA0].mutex_id;
+	mtk_mutex_unprepare(mdp->mdp_mutex[i]);
 	kfree(path);
 err_free_comps:
 	kfree(comps);
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.c
index 091a68685590..75c92e282fa2 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.c
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.c
@@ -8,6 +8,7 @@
 #include <linux/of_platform.h>
 #include <linux/of_address.h>
 #include <linux/pm_runtime.h>
+#include "mtk-mdp3-cfg.h"
 #include "mtk-mdp3-comp.h"
 #include "mtk-mdp3-core.h"
 #include "mtk-mdp3-regs.h"
@@ -19,6 +20,7 @@
 #include "mdp_reg_wdma.h"
 
 static u32 mdp_comp_alias_id[MDP_COMP_TYPE_COUNT];
+static int p_id;
 
 static inline const struct mdp_platform_config *
 __get_plat_cfg(const struct mdp_comp_ctx *ctx)
@@ -32,12 +34,18 @@ __get_plat_cfg(const struct mdp_comp_ctx *ctx)
 static s64 get_comp_flag(const struct mdp_comp_ctx *ctx)
 {
 	const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx);
+	u32 rdma0, rsz1;
+
+	rdma0 = mdp_cfg_get_id_inner(ctx->comp->mdp_dev, MDP_COMP_RDMA0);
+	rsz1 = mdp_cfg_get_id_inner(ctx->comp->mdp_dev, MDP_COMP_RSZ1);
+	if (!rdma0 || !rsz1)
+		return MDP_COMP_NONE;
 
 	if (mdp_cfg && mdp_cfg->rdma_rsz1_sram_sharing)
-		if (ctx->comp->id == MDP_COMP_RDMA0)
-			return BIT(MDP_COMP_RDMA0) | BIT(MDP_COMP_RSZ1);
+		if (ctx->comp->inner_id == rdma0)
+			return BIT(rdma0) | BIT(rsz1);
 
-	return BIT(ctx->comp->id);
+	return BIT(ctx->comp->inner_id);
 }
 
 static int init_rdma(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd)
@@ -45,12 +53,17 @@ static int init_rdma(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd)
 	const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx);
 	phys_addr_t base = ctx->comp->reg_base;
 	u8 subsys_id = ctx->comp->subsys_id;
+	s32 rdma0;
+
+	rdma0 = mdp_cfg_get_id_inner(ctx->comp->mdp_dev, MDP_COMP_RDMA0);
+	if (!rdma0)
+		return -EINVAL;
 
 	if (mdp_cfg && mdp_cfg->rdma_support_10bit) {
 		struct mdp_comp *prz1 = ctx->comp->mdp_dev->comp[MDP_COMP_RSZ1];
 
 		/* Disable RSZ1 */
-		if (ctx->comp->id == MDP_COMP_RDMA0 && prz1)
+		if (ctx->comp->inner_id == rdma0 && prz1)
 			MM_REG_WRITE(cmd, subsys_id, prz1->reg_base, PRZ_ENABLE,
 				     0x0, BIT(0));
 	}
@@ -66,13 +79,13 @@ static int config_rdma_frame(struct mdp_comp_ctx *ctx,
 			     struct mdp_cmdq_cmd *cmd,
 			     const struct v4l2_rect *compose)
 {
-	const struct mdp_rdma_data *rdma = &ctx->param->rdma;
 	const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx);
 	u32 colorformat = ctx->input->buffer.format.colorformat;
 	bool block10bit = MDP_COLOR_IS_10BIT_PACKED(colorformat);
 	bool en_ufo = MDP_COLOR_IS_UFP(colorformat);
 	phys_addr_t base = ctx->comp->reg_base;
 	u8 subsys_id = ctx->comp->subsys_id;
+	u32 reg = 0;
 
 	if (mdp_cfg && mdp_cfg->rdma_support_10bit) {
 		if (block10bit)
@@ -90,49 +103,78 @@ static int config_rdma_frame(struct mdp_comp_ctx *ctx,
 		     0x00030071);
 
 	/* Setup source frame info */
-	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_CON, rdma->src_ctrl,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.src_ctrl);
+	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_CON, reg,
 		     0x03C8FE0F);
 
 	if (mdp_cfg)
 		if (mdp_cfg->rdma_support_10bit && en_ufo) {
 			/* Setup source buffer base */
+			if (CFG_CHECK(MT8183, p_id))
+				reg = CFG_COMP(MT8183, ctx->param, rdma.ufo_dec_y);
 			MM_REG_WRITE(cmd, subsys_id,
 				     base, MDP_RDMA_UFO_DEC_LENGTH_BASE_Y,
-				     rdma->ufo_dec_y, 0xFFFFFFFF);
+				     reg, 0xFFFFFFFF);
+			if (CFG_CHECK(MT8183, p_id))
+				reg = CFG_COMP(MT8183, ctx->param, rdma.ufo_dec_c);
 			MM_REG_WRITE(cmd, subsys_id,
 				     base, MDP_RDMA_UFO_DEC_LENGTH_BASE_C,
-				     rdma->ufo_dec_c, 0xFFFFFFFF);
+				     reg, 0xFFFFFFFF);
 			/* Set 10bit source frame pitch */
-			if (block10bit)
+			if (block10bit) {
+				if (CFG_CHECK(MT8183, p_id))
+					reg = CFG_COMP(MT8183, ctx->param, rdma.mf_bkgd_in_pxl);
 				MM_REG_WRITE(cmd, subsys_id,
 					     base, MDP_RDMA_MF_BKGD_SIZE_IN_PXL,
-					     rdma->mf_bkgd_in_pxl, 0x001FFFFF);
+					     reg, 0x001FFFFF);
+			}
 		}
 
-	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_CON, rdma->control,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.control);
+	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_CON, reg,
 		     0x1110);
 	/* Setup source buffer base */
-	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_BASE_0, rdma->iova[0],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.iova[0]);
+	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_BASE_0, reg,
 		     0xFFFFFFFF);
-	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_BASE_1, rdma->iova[1],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.iova[1]);
+	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_BASE_1, reg,
 		     0xFFFFFFFF);
-	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_BASE_2, rdma->iova[2],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.iova[2]);
+	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_BASE_2, reg,
 		     0xFFFFFFFF);
 	/* Setup source buffer end */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.iova_end[0]);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_END_0,
-		     rdma->iova_end[0], 0xFFFFFFFF);
+		     reg, 0xFFFFFFFF);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.iova_end[1]);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_END_1,
-		     rdma->iova_end[1], 0xFFFFFFFF);
+		     reg, 0xFFFFFFFF);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.iova_end[2]);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_END_2,
-		     rdma->iova_end[2], 0xFFFFFFFF);
+		     reg, 0xFFFFFFFF);
 	/* Setup source frame pitch */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.mf_bkgd);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_MF_BKGD_SIZE_IN_BYTE,
-		     rdma->mf_bkgd, 0x001FFFFF);
+		     reg, 0x001FFFFF);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.sf_bkgd);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SF_BKGD_SIZE_IN_BYTE,
-		     rdma->sf_bkgd, 0x001FFFFF);
+		     reg, 0x001FFFFF);
 	/* Setup color transform */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.transform);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_TRANSFORM_0,
-		     rdma->transform, 0x0F110000);
+		     reg, 0x0F110000);
 
 	return 0;
 }
@@ -140,47 +182,67 @@ static int config_rdma_frame(struct mdp_comp_ctx *ctx,
 static int config_rdma_subfrm(struct mdp_comp_ctx *ctx,
 			      struct mdp_cmdq_cmd *cmd, u32 index)
 {
-	const struct mdp_rdma_subfrm *subfrm = &ctx->param->rdma.subfrms[index];
-	const struct img_comp_subfrm *csf = &ctx->param->subfrms[index];
 	const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx);
 	u32 colorformat = ctx->input->buffer.format.colorformat;
 	bool block10bit = MDP_COLOR_IS_10BIT_PACKED(colorformat);
 	bool en_ufo = MDP_COLOR_IS_UFP(colorformat);
 	phys_addr_t base = ctx->comp->reg_base;
 	u8 subsys_id = ctx->comp->subsys_id;
+	u32 csf_l = 0, csf_r = 0;
+	u32 reg = 0;
 
 	/* Enable RDMA */
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_EN, BIT(0), BIT(0));
 
 	/* Set Y pixel offset */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.subfrms[index].offset[0]);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_OFFSET_0,
-		     subfrm->offset[0], 0xFFFFFFFF);
+		     reg, 0xFFFFFFFF);
 
 	/* Set 10bit UFO mode */
-	if (mdp_cfg)
-		if (mdp_cfg->rdma_support_10bit && block10bit && en_ufo)
+	if (mdp_cfg) {
+		if (mdp_cfg->rdma_support_10bit && block10bit && en_ufo) {
+			if (CFG_CHECK(MT8183, p_id))
+				reg = CFG_COMP(MT8183, ctx->param, rdma.subfrms[index].offset_0_p);
 			MM_REG_WRITE(cmd, subsys_id, base,
 				     MDP_RDMA_SRC_OFFSET_0_P,
-				     subfrm->offset_0_p, 0xFFFFFFFF);
+				     reg, 0xFFFFFFFF);
+		}
+	}
 
 	/* Set U pixel offset */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.subfrms[index].offset[1]);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_OFFSET_1,
-		     subfrm->offset[1], 0xFFFFFFFF);
+		     reg, 0xFFFFFFFF);
 	/* Set V pixel offset */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.subfrms[index].offset[2]);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_OFFSET_2,
-		     subfrm->offset[2], 0xFFFFFFFF);
+		     reg, 0xFFFFFFFF);
 	/* Set source size */
-	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_MF_SRC_SIZE, subfrm->src,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.subfrms[index].src);
+	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_MF_SRC_SIZE, reg,
 		     0x1FFF1FFF);
 	/* Set target size */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.subfrms[index].clip);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_MF_CLIP_SIZE,
-		     subfrm->clip, 0x1FFF1FFF);
+		     reg, 0x1FFF1FFF);
 	/* Set crop offset */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rdma.subfrms[index].clip_ofst);
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_MF_OFFSET_1,
-		     subfrm->clip_ofst, 0x003F001F);
+		     reg, 0x003F001F);
 
+	if (CFG_CHECK(MT8183, p_id)) {
+		csf_l = CFG_COMP(MT8183, ctx->param, subfrms[index].in.left);
+		csf_r = CFG_COMP(MT8183, ctx->param, subfrms[index].in.right);
+	}
 	if (mdp_cfg && mdp_cfg->rdma_upsample_repeat_only)
-		if ((csf->in.right - csf->in.left + 1) > 320)
+		if ((csf_r - csf_l + 1) > 320)
 			MM_REG_WRITE(cmd, subsys_id, base,
 				     MDP_RDMA_RESV_DUMMY_0, BIT(2), BIT(2));
 
@@ -228,63 +290,97 @@ static int config_rsz_frame(struct mdp_comp_ctx *ctx,
 			    struct mdp_cmdq_cmd *cmd,
 			    const struct v4l2_rect *compose)
 {
-	const struct mdp_rsz_data *rsz = &ctx->param->rsz;
 	phys_addr_t base = ctx->comp->reg_base;
 	u8 subsys_id = ctx->comp->subsys_id;
+	bool bypass = FALSE;
+	u32 reg = 0;
 
-	if (ctx->param->frame.bypass) {
+	if (CFG_CHECK(MT8183, p_id))
+		bypass = CFG_COMP(MT8183, ctx->param, frame.bypass);
+
+	if (bypass) {
 		/* Disable RSZ */
 		MM_REG_WRITE(cmd, subsys_id, base, PRZ_ENABLE, 0x0, BIT(0));
 		return 0;
 	}
 
-	MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_1, rsz->control1,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rsz.control1);
+	MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_1, reg,
 		     0x03FFFDF3);
-	MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_2, rsz->control2,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rsz.control2);
+	MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_2, reg,
 		     0x0FFFC290);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rsz.coeff_step_x);
 	MM_REG_WRITE(cmd, subsys_id, base, PRZ_HORIZONTAL_COEFF_STEP,
-		     rsz->coeff_step_x, 0x007FFFFF);
+		     reg, 0x007FFFFF);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rsz.coeff_step_y);
 	MM_REG_WRITE(cmd, subsys_id, base, PRZ_VERTICAL_COEFF_STEP,
-		     rsz->coeff_step_y, 0x007FFFFF);
+		     reg, 0x007FFFFF);
 	return 0;
 }
 
 static int config_rsz_subfrm(struct mdp_comp_ctx *ctx,
 			     struct mdp_cmdq_cmd *cmd, u32 index)
 {
-	const struct mdp_rsz_subfrm *subfrm = &ctx->param->rsz.subfrms[index];
-	const struct img_comp_subfrm *csf = &ctx->param->subfrms[index];
 	const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx);
 	phys_addr_t base = ctx->comp->reg_base;
 	u8 subsys_id = ctx->comp->subsys_id;
+	u32 csf_l = 0, csf_r = 0;
+	u32 reg = 0;
 
-	MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_2, subfrm->control2,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rsz.subfrms[index].control2);
+	MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_2, reg,
 		     0x00003800);
-	MM_REG_WRITE(cmd, subsys_id, base, PRZ_INPUT_IMAGE, subfrm->src,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rsz.subfrms[index].src);
+	MM_REG_WRITE(cmd, subsys_id, base, PRZ_INPUT_IMAGE, reg,
 		     0xFFFFFFFF);
 
+	if (CFG_CHECK(MT8183, p_id)) {
+		csf_l = CFG_COMP(MT8183, ctx->param, subfrms[index].in.left);
+		csf_r = CFG_COMP(MT8183, ctx->param, subfrms[index].in.right);
+	}
 	if (mdp_cfg && mdp_cfg->rsz_disable_dcm_small_sample)
-		if ((csf->in.right - csf->in.left + 1) <= 16)
+		if ((csf_r - csf_l + 1) <= 16)
 			MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_1,
 				     BIT(27), BIT(27));
 
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, subfrms[index].luma.left);
 	MM_REG_WRITE(cmd, subsys_id, base, PRZ_LUMA_HORIZONTAL_INTEGER_OFFSET,
-		     csf->luma.left, 0xFFFF);
+		     reg, 0xFFFF);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, subfrms[index].luma.left_subpix);
 	MM_REG_WRITE(cmd, subsys_id,
 		     base, PRZ_LUMA_HORIZONTAL_SUBPIXEL_OFFSET,
-		     csf->luma.left_subpix, 0x1FFFFF);
+		     reg, 0x1FFFFF);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, subfrms[index].luma.top);
 	MM_REG_WRITE(cmd, subsys_id, base, PRZ_LUMA_VERTICAL_INTEGER_OFFSET,
-		     csf->luma.top, 0xFFFF);
+		     reg, 0xFFFF);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, subfrms[index].luma.top_subpix);
 	MM_REG_WRITE(cmd, subsys_id, base, PRZ_LUMA_VERTICAL_SUBPIXEL_OFFSET,
-		     csf->luma.top_subpix, 0x1FFFFF);
+		     reg, 0x1FFFFF);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, subfrms[index].chroma.left);
 	MM_REG_WRITE(cmd, subsys_id,
 		     base, PRZ_CHROMA_HORIZONTAL_INTEGER_OFFSET,
-		     csf->chroma.left, 0xFFFF);
+		     reg, 0xFFFF);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, subfrms[index].chroma.left_subpix);
 	MM_REG_WRITE(cmd, subsys_id,
 		     base, PRZ_CHROMA_HORIZONTAL_SUBPIXEL_OFFSET,
-		     csf->chroma.left_subpix, 0x1FFFFF);
+		     reg, 0x1FFFFF);
 
-	MM_REG_WRITE(cmd, subsys_id, base, PRZ_OUTPUT_IMAGE, subfrm->clip,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, rsz.subfrms[index].clip);
+	MM_REG_WRITE(cmd, subsys_id, base, PRZ_OUTPUT_IMAGE, reg,
 		     0xFFFFFFFF);
 
 	return 0;
@@ -296,11 +392,16 @@ static int advance_rsz_subfrm(struct mdp_comp_ctx *ctx,
 	const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx);
 
 	if (mdp_cfg && mdp_cfg->rsz_disable_dcm_small_sample) {
-		const struct img_comp_subfrm *csf = &ctx->param->subfrms[index];
 		phys_addr_t base = ctx->comp->reg_base;
 		u8 subsys_id = ctx->comp->subsys_id;
+		u32 csf_l = 0, csf_r = 0;
+
+		if (CFG_CHECK(MT8183, p_id)) {
+			csf_l = CFG_COMP(MT8183, ctx->param, subfrms[index].in.left);
+			csf_r = CFG_COMP(MT8183, ctx->param, subfrms[index].in.right);
+		}
 
-		if ((csf->in.right - csf->in.left + 1) <= 16)
+		if ((csf_r - csf_l + 1) <= 16)
 			MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_1, 0x0,
 				     BIT(27));
 	}
@@ -333,31 +434,47 @@ static int config_wrot_frame(struct mdp_comp_ctx *ctx,
 			     struct mdp_cmdq_cmd *cmd,
 			     const struct v4l2_rect *compose)
 {
-	const struct mdp_wrot_data *wrot = &ctx->param->wrot;
 	const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx);
 	phys_addr_t base = ctx->comp->reg_base;
 	u8 subsys_id = ctx->comp->subsys_id;
+	u32 reg = 0;
 
 	/* Write frame base address */
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_BASE_ADDR, wrot->iova[0],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.iova[0]);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_BASE_ADDR, reg,
 		     0xFFFFFFFF);
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_BASE_ADDR_C, wrot->iova[1],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.iova[1]);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_BASE_ADDR_C, reg,
 		     0xFFFFFFFF);
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_BASE_ADDR_V, wrot->iova[2],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.iova[2]);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_BASE_ADDR_V, reg,
 		     0xFFFFFFFF);
 	/* Write frame related registers */
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_CTRL, wrot->control,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.control);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_CTRL, reg,
 		     0xF131510F);
 	/* Write frame Y pitch */
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_STRIDE, wrot->stride[0],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.stride[0]);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_STRIDE, reg,
 		     0x0000FFFF);
 	/* Write frame UV pitch */
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_STRIDE_C, wrot->stride[1],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.stride[1]);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_STRIDE_C, reg,
 		     0xFFFF);
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_STRIDE_V, wrot->stride[2],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.stride[2]);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_STRIDE_V, reg,
 		     0xFFFF);
 	/* Write matrix control */
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_MAT_CTRL, wrot->mat_ctrl, 0xF3);
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.mat_ctrl);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_MAT_CTRL, reg, 0xF3);
 
 	/* Set the fixed ALPHA as 0xFF */
 	MM_REG_WRITE(cmd, subsys_id, base, VIDO_DITHER, 0xFF000000,
@@ -365,13 +482,18 @@ static int config_wrot_frame(struct mdp_comp_ctx *ctx,
 	/* Set VIDO_EOL_SEL */
 	MM_REG_WRITE(cmd, subsys_id, base, VIDO_RSV_1, BIT(31), BIT(31));
 	/* Set VIDO_FIFO_TEST */
-	if (wrot->fifo_test != 0)
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.fifo_test);
+	if (reg != 0)
 		MM_REG_WRITE(cmd, subsys_id, base, VIDO_FIFO_TEST,
-			     wrot->fifo_test, 0xFFF);
+			     reg, 0xFFF);
 	/* Filter enable */
-	if (mdp_cfg && mdp_cfg->wrot_filter_constraint)
+	if (mdp_cfg && mdp_cfg->wrot_filter_constraint) {
+		if (CFG_CHECK(MT8183, p_id))
+			reg = CFG_COMP(MT8183, ctx->param, wrot.filter);
 		MM_REG_WRITE(cmd, subsys_id, base, VIDO_MAIN_BUF_SIZE,
-			     wrot->filter, 0x77);
+			     reg, 0x77);
+	}
 
 	return 0;
 }
@@ -379,30 +501,44 @@ static int config_wrot_frame(struct mdp_comp_ctx *ctx,
 static int config_wrot_subfrm(struct mdp_comp_ctx *ctx,
 			      struct mdp_cmdq_cmd *cmd, u32 index)
 {
-	const struct mdp_wrot_subfrm *subfrm = &ctx->param->wrot.subfrms[index];
 	phys_addr_t base = ctx->comp->reg_base;
 	u8 subsys_id = ctx->comp->subsys_id;
+	u32 reg = 0;
 
 	/* Write Y pixel offset */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.subfrms[index].offset[0]);
 	MM_REG_WRITE(cmd, subsys_id, base, VIDO_OFST_ADDR,
-		     subfrm->offset[0], 0x0FFFFFFF);
+		     reg, 0x0FFFFFFF);
 	/* Write U pixel offset */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.subfrms[index].offset[1]);
 	MM_REG_WRITE(cmd, subsys_id, base, VIDO_OFST_ADDR_C,
-		     subfrm->offset[1], 0x0FFFFFFF);
+		     reg, 0x0FFFFFFF);
 	/* Write V pixel offset */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.subfrms[index].offset[2]);
 	MM_REG_WRITE(cmd, subsys_id, base, VIDO_OFST_ADDR_V,
-		     subfrm->offset[2], 0x0FFFFFFF);
+		     reg, 0x0FFFFFFF);
 	/* Write source size */
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_IN_SIZE, subfrm->src,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.subfrms[index].src);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_IN_SIZE, reg,
 		     0x1FFF1FFF);
 	/* Write target size */
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_TAR_SIZE, subfrm->clip,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.subfrms[index].clip);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_TAR_SIZE, reg,
 		     0x1FFF1FFF);
-	MM_REG_WRITE(cmd, subsys_id, base, VIDO_CROP_OFST, subfrm->clip_ofst,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.subfrms[index].clip_ofst);
+	MM_REG_WRITE(cmd, subsys_id, base, VIDO_CROP_OFST, reg,
 		     0x1FFF1FFF);
 
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wrot.subfrms[index].main_buf);
 	MM_REG_WRITE(cmd, subsys_id, base, VIDO_MAIN_BUF_SIZE,
-		     subfrm->main_buf, 0x1FFF7F00);
+		     reg, 0x1FFF7F00);
 
 	/* Enable WROT */
 	MM_REG_WRITE(cmd, subsys_id, base, VIDO_ROT_EN, BIT(0), BIT(0));
@@ -456,29 +592,41 @@ static int config_wdma_frame(struct mdp_comp_ctx *ctx,
 			     struct mdp_cmdq_cmd *cmd,
 			     const struct v4l2_rect *compose)
 {
-	const struct mdp_wdma_data *wdma = &ctx->param->wdma;
 	phys_addr_t base = ctx->comp->reg_base;
 	u8 subsys_id = ctx->comp->subsys_id;
+	u32 reg = 0;
 
 	MM_REG_WRITE(cmd, subsys_id, base, WDMA_BUF_CON2, 0x10101050,
 		     0xFFFFFFFF);
 
 	/* Setup frame information */
-	MM_REG_WRITE(cmd, subsys_id, base, WDMA_CFG, wdma->wdma_cfg,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.wdma_cfg);
+	MM_REG_WRITE(cmd, subsys_id, base, WDMA_CFG, reg,
 		     0x0F01B8F0);
 	/* Setup frame base address */
-	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_ADDR,   wdma->iova[0],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.iova[0]);
+	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_ADDR, reg,
 		     0xFFFFFFFF);
-	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_U_ADDR, wdma->iova[1],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.iova[1]);
+	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_U_ADDR, reg,
 		     0xFFFFFFFF);
-	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_V_ADDR, wdma->iova[2],
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.iova[2]);
+	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_V_ADDR, reg,
 		     0xFFFFFFFF);
 	/* Setup Y pitch */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.w_in_byte);
 	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_W_IN_BYTE,
-		     wdma->w_in_byte, 0x0000FFFF);
+		     reg, 0x0000FFFF);
 	/* Setup UV pitch */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.uv_stride);
 	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_UV_PITCH,
-		     wdma->uv_stride, 0x0000FFFF);
+		     reg, 0x0000FFFF);
 	/* Set the fixed ALPHA as 0xFF */
 	MM_REG_WRITE(cmd, subsys_id, base, WDMA_ALPHA, 0x800000FF,
 		     0x800000FF);
@@ -489,27 +637,39 @@ static int config_wdma_frame(struct mdp_comp_ctx *ctx,
 static int config_wdma_subfrm(struct mdp_comp_ctx *ctx,
 			      struct mdp_cmdq_cmd *cmd, u32 index)
 {
-	const struct mdp_wdma_subfrm *subfrm = &ctx->param->wdma.subfrms[index];
 	phys_addr_t base = ctx->comp->reg_base;
 	u8 subsys_id = ctx->comp->subsys_id;
+	u32 reg = 0;
 
 	/* Write Y pixel offset */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.subfrms[index].offset[0]);
 	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_ADDR_OFFSET,
-		     subfrm->offset[0], 0x0FFFFFFF);
+		     reg, 0x0FFFFFFF);
 	/* Write U pixel offset */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.subfrms[index].offset[1]);
 	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_U_ADDR_OFFSET,
-		     subfrm->offset[1], 0x0FFFFFFF);
+		     reg, 0x0FFFFFFF);
 	/* Write V pixel offset */
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.subfrms[index].offset[2]);
 	MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_V_ADDR_OFFSET,
-		     subfrm->offset[2], 0x0FFFFFFF);
+		     reg, 0x0FFFFFFF);
 	/* Write source size */
-	MM_REG_WRITE(cmd, subsys_id, base, WDMA_SRC_SIZE, subfrm->src,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.subfrms[index].src);
+	MM_REG_WRITE(cmd, subsys_id, base, WDMA_SRC_SIZE, reg,
 		     0x3FFF3FFF);
 	/* Write target size */
-	MM_REG_WRITE(cmd, subsys_id, base, WDMA_CLIP_SIZE, subfrm->clip,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.subfrms[index].clip);
+	MM_REG_WRITE(cmd, subsys_id, base, WDMA_CLIP_SIZE, reg,
 		     0x3FFF3FFF);
 	/* Write clip offset */
-	MM_REG_WRITE(cmd, subsys_id, base, WDMA_CLIP_COORD, subfrm->clip_ofst,
+	if (CFG_CHECK(MT8183, p_id))
+		reg = CFG_COMP(MT8183, ctx->param, wdma.subfrms[index].clip_ofst);
+	MM_REG_WRITE(cmd, subsys_id, base, WDMA_CLIP_COORD, reg,
 		     0x3FFF3FFF);
 
 	/* Enable WDMA */
@@ -552,13 +712,21 @@ static int init_ccorr(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd)
 static int config_ccorr_subfrm(struct mdp_comp_ctx *ctx,
 			       struct mdp_cmdq_cmd *cmd, u32 index)
 {
-	const struct img_comp_subfrm *csf = &ctx->param->subfrms[index];
 	phys_addr_t base = ctx->comp->reg_base;
 	u8 subsys_id = ctx->comp->subsys_id;
+	u32 csf_l = 0, csf_r = 0;
+	u32 csf_t = 0, csf_b = 0;
 	u32 hsize, vsize;
 
-	hsize = csf->in.right - csf->in.left + 1;
-	vsize = csf->in.bottom - csf->in.top + 1;
+	if (CFG_CHECK(MT8183, p_id)) {
+		csf_l = CFG_COMP(MT8183, ctx->param, subfrms[index].in.left);
+		csf_r = CFG_COMP(MT8183, ctx->param, subfrms[index].in.right);
+		csf_t = CFG_COMP(MT8183, ctx->param, subfrms[index].in.top);
+		csf_b = CFG_COMP(MT8183, ctx->param, subfrms[index].in.bottom);
+	}
+
+	hsize = csf_r - csf_l + 1;
+	vsize = csf_b - csf_t + 1;
 	MM_REG_WRITE(cmd, subsys_id, base, MDP_CCORR_SIZE,
 		     (hsize << 16) + (vsize <<  0), 0x1FFF1FFF);
 	return 0;
@@ -578,32 +746,6 @@ static const struct mdp_comp_ops *mdp_comp_ops[MDP_COMP_TYPE_COUNT] = {
 	[MDP_COMP_TYPE_CCORR] =		&ccorr_ops,
 };
 
-struct mdp_comp_match {
-	enum mdp_comp_type	type;
-	u32			alias_id;
-};
-
-static const struct mdp_comp_match mdp_comp_matches[MDP_MAX_COMP_COUNT] = {
-	[MDP_COMP_WPEI] =	{ MDP_COMP_TYPE_WPEI, 0 },
-	[MDP_COMP_WPEO] =	{ MDP_COMP_TYPE_EXTO, 2 },
-	[MDP_COMP_WPEI2] =	{ MDP_COMP_TYPE_WPEI, 1 },
-	[MDP_COMP_WPEO2] =	{ MDP_COMP_TYPE_EXTO, 3 },
-	[MDP_COMP_ISP_IMGI] =	{ MDP_COMP_TYPE_IMGI, 0 },
-	[MDP_COMP_ISP_IMGO] =	{ MDP_COMP_TYPE_EXTO, 0 },
-	[MDP_COMP_ISP_IMG2O] =	{ MDP_COMP_TYPE_EXTO, 1 },
-
-	[MDP_COMP_CAMIN] =	{ MDP_COMP_TYPE_DL_PATH, 0 },
-	[MDP_COMP_CAMIN2] =	{ MDP_COMP_TYPE_DL_PATH, 1 },
-	[MDP_COMP_RDMA0] =	{ MDP_COMP_TYPE_RDMA, 0 },
-	[MDP_COMP_CCORR0] =	{ MDP_COMP_TYPE_CCORR, 0 },
-	[MDP_COMP_RSZ0] =	{ MDP_COMP_TYPE_RSZ, 0 },
-	[MDP_COMP_RSZ1] =	{ MDP_COMP_TYPE_RSZ, 1 },
-	[MDP_COMP_PATH0_SOUT] =	{ MDP_COMP_TYPE_PATH, 0 },
-	[MDP_COMP_PATH1_SOUT] =	{ MDP_COMP_TYPE_PATH, 1 },
-	[MDP_COMP_WROT0] =	{ MDP_COMP_TYPE_WROT, 0 },
-	[MDP_COMP_WDMA] =	{ MDP_COMP_TYPE_WDMA, 0 },
-};
-
 static const struct of_device_id mdp_comp_dt_ids[] = {
 	{
 		.compatible = "mediatek,mt8183-mdp3-rdma",
@@ -624,32 +766,6 @@ static const struct of_device_id mdp_comp_dt_ids[] = {
 	{}
 };
 
-static const struct of_device_id mdp_sub_comp_dt_ids[] = {
-	{
-		.compatible = "mediatek,mt8183-mdp3-wdma",
-		.data = (void *)MDP_COMP_TYPE_PATH,
-	}, {
-		.compatible = "mediatek,mt8183-mdp3-wrot",
-		.data = (void *)MDP_COMP_TYPE_PATH,
-	},
-	{}
-};
-
-/* Used to describe the item order in MDP property */
-struct mdp_comp_info {
-	u32	clk_num;
-	u32	clk_ofst;
-	u32	dts_reg_ofst;
-};
-
-static const struct mdp_comp_info mdp_comp_dt_info[MDP_MAX_COMP_COUNT] = {
-	[MDP_COMP_RDMA0]	= {2, 0, 0},
-	[MDP_COMP_RSZ0]		= {1, 0, 0},
-	[MDP_COMP_WROT0]	= {1, 0, 0},
-	[MDP_COMP_WDMA]		= {1, 0, 0},
-	[MDP_COMP_CCORR0]	= {1, 0, 0},
-};
-
 static inline bool is_dma_capable(const enum mdp_comp_type type)
 {
 	return (type == MDP_COMP_TYPE_RDMA ||
@@ -666,13 +782,13 @@ static inline bool is_bypass_gce_event(const enum mdp_comp_type type)
 	return (type == MDP_COMP_TYPE_PATH);
 }
 
-static int mdp_comp_get_id(enum mdp_comp_type type, int alias_id)
+static int mdp_comp_get_id(struct mdp_dev *mdp, enum mdp_comp_type type, u32 alias_id)
 {
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(mdp_comp_matches); i++)
-		if (mdp_comp_matches[i].type == type &&
-		    mdp_comp_matches[i].alias_id == alias_id)
+	for (i = 0; i < mdp->mdp_data->comp_data_len; i++)
+		if (mdp->mdp_data->comp_data[i].match.type == type &&
+		    mdp->mdp_data->comp_data[i].match.alias_id == alias_id)
 			return i;
 	return -ENODEV;
 }
@@ -681,24 +797,25 @@ int mdp_comp_clock_on(struct device *dev, struct mdp_comp *comp)
 {
 	int i, ret;
 
-	if (comp->comp_dev) {
+	/* Only DMA capable components need the pm control */
+	if (comp->comp_dev && is_dma_capable(comp->type)) {
 		ret = pm_runtime_resume_and_get(comp->comp_dev);
 		if (ret < 0) {
 			dev_err(dev,
 				"Failed to get power, err %d. type:%d id:%d\n",
-				ret, comp->type, comp->id);
+				ret, comp->type, comp->inner_id);
 			return ret;
 		}
 	}
 
-	for (i = 0; i < ARRAY_SIZE(comp->clks); i++) {
+	for (i = 0; i < comp->clk_num; i++) {
 		if (IS_ERR_OR_NULL(comp->clks[i]))
 			continue;
 		ret = clk_prepare_enable(comp->clks[i]);
 		if (ret) {
 			dev_err(dev,
 				"Failed to enable clk %d. type:%d id:%d\n",
-				i, comp->type, comp->id);
+				i, comp->type, comp->inner_id);
 			goto err_revert;
 		}
 	}
@@ -711,7 +828,7 @@ err_revert:
 			continue;
 		clk_disable_unprepare(comp->clks[i]);
 	}
-	if (comp->comp_dev)
+	if (comp->comp_dev && is_dma_capable(comp->type))
 		pm_runtime_put_sync(comp->comp_dev);
 
 	return ret;
@@ -721,13 +838,13 @@ void mdp_comp_clock_off(struct device *dev, struct mdp_comp *comp)
 {
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(comp->clks); i++) {
+	for (i = 0; i < comp->clk_num; i++) {
 		if (IS_ERR_OR_NULL(comp->clks[i]))
 			continue;
 		clk_disable_unprepare(comp->clks[i]);
 	}
 
-	if (comp->comp_dev)
+	if (comp->comp_dev && is_dma_capable(comp->type))
 		pm_runtime_put(comp->comp_dev);
 }
 
@@ -752,8 +869,8 @@ void mdp_comp_clocks_off(struct device *dev, struct mdp_comp *comps, int num)
 		mdp_comp_clock_off(dev, &comps[i]);
 }
 
-static int mdp_get_subsys_id(struct device *dev, struct device_node *node,
-			     struct mdp_comp *comp)
+static int mdp_get_subsys_id(struct mdp_dev *mdp, struct device *dev,
+			     struct device_node *node, struct mdp_comp *comp)
 {
 	struct platform_device *comp_pdev;
 	struct cmdq_client_reg  cmdq_reg;
@@ -766,12 +883,12 @@ static int mdp_get_subsys_id(struct device *dev, struct device_node *node,
 	comp_pdev = of_find_device_by_node(node);
 
 	if (!comp_pdev) {
-		dev_err(dev, "get comp_pdev fail! comp id=%d type=%d\n",
-			comp->id, comp->type);
+		dev_err(dev, "get comp_pdev fail! comp public id=%d, inner id=%d, type=%d\n",
+			comp->public_id, comp->inner_id, comp->type);
 		return -ENODEV;
 	}
 
-	index = mdp_comp_dt_info[comp->id].dts_reg_ofst;
+	index = mdp->mdp_data->comp_data[comp->public_id].info.dts_reg_ofst;
 	ret = cmdq_dev_get_client_reg(&comp_pdev->dev, &cmdq_reg, index);
 	if (ret != 0) {
 		dev_err(&comp_pdev->dev, "cmdq_dev_get_subsys fail!\n");
@@ -789,8 +906,9 @@ static void __mdp_comp_init(struct mdp_dev *mdp, struct device_node *node,
 {
 	struct resource res;
 	phys_addr_t base;
-	int index = mdp_comp_dt_info[comp->id].dts_reg_ofst;
+	int index;
 
+	index = mdp->mdp_data->comp_data[comp->public_id].info.dts_reg_ofst;
 	if (of_address_to_resource(node, index, &res) < 0)
 		base = 0L;
 	else
@@ -805,7 +923,7 @@ static int mdp_comp_init(struct mdp_dev *mdp, struct device_node *node,
 			 struct mdp_comp *comp, enum mtk_mdp_comp_id id)
 {
 	struct device *dev = &mdp->pdev->dev;
-	int clk_num;
+	struct platform_device *pdev_c;
 	int clk_ofst;
 	int i;
 	s32 event;
@@ -815,22 +933,36 @@ static int mdp_comp_init(struct mdp_dev *mdp, struct device_node *node,
 		return -EINVAL;
 	}
 
-	comp->id = id;
-	comp->type = mdp_comp_matches[id].type;
-	comp->alias_id = mdp_comp_matches[id].alias_id;
+	pdev_c = of_find_device_by_node(node);
+	if (!pdev_c) {
+		dev_warn(dev, "can't find platform device of node:%s\n",
+			 node->name);
+		return -ENODEV;
+	}
+
+	comp->comp_dev = &pdev_c->dev;
+	comp->public_id = id;
+	comp->type = mdp->mdp_data->comp_data[id].match.type;
+	comp->inner_id = mdp->mdp_data->comp_data[id].match.inner_id;
+	comp->alias_id = mdp->mdp_data->comp_data[id].match.alias_id;
 	comp->ops = mdp_comp_ops[comp->type];
 	__mdp_comp_init(mdp, node, comp);
 
-	clk_num = mdp_comp_dt_info[id].clk_num;
-	clk_ofst = mdp_comp_dt_info[id].clk_ofst;
+	comp->clk_num = mdp->mdp_data->comp_data[id].info.clk_num;
+	comp->clks = devm_kzalloc(dev, sizeof(struct clk *) * comp->clk_num,
+				  GFP_KERNEL);
+	if (!comp->clks)
+		return -ENOMEM;
 
-	for (i = 0; i < clk_num; i++) {
+	clk_ofst = mdp->mdp_data->comp_data[id].info.clk_ofst;
+
+	for (i = 0; i < comp->clk_num; i++) {
 		comp->clks[i] = of_clk_get(node, i + clk_ofst);
 		if (IS_ERR(comp->clks[i]))
 			break;
 	}
 
-	mdp_get_subsys_id(dev, node, comp);
+	mdp_get_subsys_id(mdp, dev, node, comp);
 
 	/* Set GCE SOF event */
 	if (is_bypass_gce_event(comp->type) ||
@@ -861,6 +993,11 @@ static void mdp_comp_deinit(struct mdp_comp *comp)
 	if (!comp)
 		return;
 
+	if (comp->comp_dev && comp->clks) {
+		devm_kfree(&comp->mdp_dev->pdev->dev, comp->clks);
+		comp->clks = NULL;
+	}
+
 	if (comp->regs)
 		iounmap(comp->regs);
 }
@@ -888,8 +1025,8 @@ static struct mdp_comp *mdp_comp_create(struct mdp_dev *mdp,
 	mdp->comp[id] = comp;
 	mdp->comp[id]->mdp_dev = mdp;
 
-	dev_dbg(dev, "%s type:%d alias:%d id:%d base:%#x regs:%p\n",
-		dev->of_node->name, comp->type, comp->alias_id, id,
+	dev_dbg(dev, "%s type:%d alias:%d public id:%d inner id:%d base:%#x regs:%p\n",
+		dev->of_node->name, comp->type, comp->alias_id, id, comp->inner_id,
 		(u32)comp->reg_base, comp->regs);
 	return comp;
 }
@@ -898,6 +1035,7 @@ static int mdp_comp_sub_create(struct mdp_dev *mdp)
 {
 	struct device *dev = &mdp->pdev->dev;
 	struct device_node *node, *parent;
+	const struct mtk_mdp_driver_data *data = mdp->mdp_data;
 
 	parent = dev->of_node->parent;
 
@@ -907,7 +1045,7 @@ static int mdp_comp_sub_create(struct mdp_dev *mdp)
 		int id, alias_id;
 		struct mdp_comp *comp;
 
-		of_id = of_match_node(mdp_sub_comp_dt_ids, node);
+		of_id = of_match_node(data->mdp_sub_comp_dt_ids, node);
 		if (!of_id)
 			continue;
 		if (!of_device_is_available(node)) {
@@ -918,7 +1056,7 @@ static int mdp_comp_sub_create(struct mdp_dev *mdp)
 
 		type = (enum mdp_comp_type)(uintptr_t)of_id->data;
 		alias_id = mdp_comp_alias_id[type];
-		id = mdp_comp_get_id(type, alias_id);
+		id = mdp_comp_get_id(mdp, type, alias_id);
 		if (id < 0) {
 			dev_err(dev,
 				"Fail to get sub comp. id: type %d alias %d\n",
@@ -941,7 +1079,8 @@ void mdp_comp_destroy(struct mdp_dev *mdp)
 
 	for (i = 0; i < ARRAY_SIZE(mdp->comp); i++) {
 		if (mdp->comp[i]) {
-			pm_runtime_disable(mdp->comp[i]->comp_dev);
+			if (is_dma_capable(mdp->comp[i]->type))
+				pm_runtime_disable(mdp->comp[i]->comp_dev);
 			mdp_comp_deinit(mdp->comp[i]);
 			devm_kfree(mdp->comp[i]->comp_dev, mdp->comp[i]);
 			mdp->comp[i] = NULL;
@@ -953,10 +1092,10 @@ int mdp_comp_config(struct mdp_dev *mdp)
 {
 	struct device *dev = &mdp->pdev->dev;
 	struct device_node *node, *parent;
-	struct platform_device *pdev;
 	int ret;
 
 	memset(mdp_comp_alias_id, 0, sizeof(mdp_comp_alias_id));
+	p_id = mdp->mdp_data->mdp_plat_id;
 
 	parent = dev->of_node->parent;
 	/* Iterate over sibling MDP function blocks */
@@ -978,7 +1117,7 @@ int mdp_comp_config(struct mdp_dev *mdp)
 
 		type = (enum mdp_comp_type)(uintptr_t)of_id->data;
 		alias_id = mdp_comp_alias_id[type];
-		id = mdp_comp_get_id(type, alias_id);
+		id = mdp_comp_get_id(mdp, type, alias_id);
 		if (id < 0) {
 			dev_err(dev,
 				"Fail to get component id: type %d alias %d\n",
@@ -994,19 +1133,8 @@ int mdp_comp_config(struct mdp_dev *mdp)
 		}
 
 		/* Only DMA capable components need the pm control */
-		comp->comp_dev = NULL;
 		if (!is_dma_capable(comp->type))
 			continue;
-
-		pdev = of_find_device_by_node(node);
-		if (!pdev) {
-			dev_warn(dev, "can't find platform device of node:%s\n",
-				 node->name);
-			ret = -ENODEV;
-			goto err_init_comps;
-		}
-
-		comp->comp_dev = &pdev->dev;
 		pm_runtime_enable(comp->comp_dev);
 	}
 
@@ -1026,22 +1154,47 @@ int mdp_comp_ctx_config(struct mdp_dev *mdp, struct mdp_comp_ctx *ctx,
 			const struct img_ipi_frameparam *frame)
 {
 	struct device *dev = &mdp->pdev->dev;
-	int i;
+	enum mtk_mdp_comp_id public_id = MDP_COMP_NONE;
+	u32 arg;
+	int i, idx;
 
-	if (param->type < 0 || param->type >= MDP_MAX_COMP_COUNT) {
-		dev_err(dev, "Invalid component id %d", param->type);
+	if (!param) {
+		dev_err(dev, "Invalid component param");
 		return -EINVAL;
 	}
 
-	ctx->comp = mdp->comp[param->type];
+	if (CFG_CHECK(MT8183, p_id))
+		arg = CFG_COMP(MT8183, param, type);
+	else
+		return -EINVAL;
+	public_id = mdp_cfg_get_id_public(mdp, arg);
+	if (public_id < 0) {
+		dev_err(dev, "Invalid component id %d", public_id);
+		return -EINVAL;
+	}
+
+	ctx->comp = mdp->comp[public_id];
 	if (!ctx->comp) {
-		dev_err(dev, "Uninit component id %d", param->type);
+		dev_err(dev, "Uninit component inner id %d", arg);
 		return -EINVAL;
 	}
 
 	ctx->param = param;
-	ctx->input = &frame->inputs[param->input];
-	for (i = 0; i < param->num_outputs; i++)
-		ctx->outputs[i] = &frame->outputs[param->outputs[i]];
+	if (CFG_CHECK(MT8183, p_id))
+		arg = CFG_COMP(MT8183, param, input);
+	else
+		return -EINVAL;
+	ctx->input = &frame->inputs[arg];
+	if (CFG_CHECK(MT8183, p_id))
+		idx = CFG_COMP(MT8183, param, num_outputs);
+	else
+		return -EINVAL;
+	for (i = 0; i < idx; i++) {
+		if (CFG_CHECK(MT8183, p_id))
+			arg = CFG_COMP(MT8183, param, outputs[i]);
+		else
+			return -EINVAL;
+		ctx->outputs[i] = &frame->outputs[arg];
+	}
 	return 0;
 }
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h
index dc48f55ac4f7..20d2bcb77ef9 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h
@@ -134,6 +134,24 @@ enum {
 	MDP_GCE_EVENT_MAX,
 };
 
+struct mdp_comp_match {
+	enum mdp_comp_type type;
+	u32 alias_id;
+	s32 inner_id;
+};
+
+/* Used to describe the item order in MDP property */
+struct mdp_comp_info {
+	u32 clk_num;
+	u32 clk_ofst;
+	u32 dts_reg_ofst;
+};
+
+struct mdp_comp_data {
+	struct mdp_comp_match match;
+	struct mdp_comp_info info;
+};
+
 struct mdp_comp_ops;
 
 struct mdp_comp {
@@ -141,10 +159,12 @@ struct mdp_comp {
 	void __iomem			*regs;
 	phys_addr_t			reg_base;
 	u8				subsys_id;
-	struct clk			*clks[6];
+	u8				clk_num;
+	struct clk			**clks;
 	struct device			*comp_dev;
 	enum mdp_comp_type		type;
-	enum mtk_mdp_comp_id		id;
+	enum mtk_mdp_comp_id		public_id;
+	s32				inner_id;
 	u32				alias_id;
 	s32				gce_event[MDP_GCE_EVENT_MAX];
 	const struct mdp_comp_ops	*ops;
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c
index 97edcd9d1c81..aa6c225302f0 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c
@@ -12,40 +12,11 @@
 #include <linux/remoteproc.h>
 #include <linux/remoteproc/mtk_scp.h>
 #include <media/videobuf2-dma-contig.h>
+
 #include "mtk-mdp3-core.h"
+#include "mtk-mdp3-cfg.h"
 #include "mtk-mdp3-m2m.h"
 
-static const struct mdp_platform_config mt8183_plat_cfg = {
-	.rdma_support_10bit		= true,
-	.rdma_rsz1_sram_sharing		= true,
-	.rdma_upsample_repeat_only	= true,
-	.rsz_disable_dcm_small_sample	= false,
-	.wrot_filter_constraint		= false,
-};
-
-static const struct of_device_id mt8183_mdp_probe_infra[MDP_INFRA_MAX] = {
-	[MDP_INFRA_MMSYS] = { .compatible = "mediatek,mt8183-mmsys" },
-	[MDP_INFRA_MUTEX] = { .compatible = "mediatek,mt8183-disp-mutex" },
-	[MDP_INFRA_SCP] = { .compatible = "mediatek,mt8183-scp" }
-};
-
-static const u32 mt8183_mutex_idx[MDP_MAX_COMP_COUNT] = {
-	[MDP_COMP_RDMA0] = MUTEX_MOD_IDX_MDP_RDMA0,
-	[MDP_COMP_RSZ0] = MUTEX_MOD_IDX_MDP_RSZ0,
-	[MDP_COMP_RSZ1] = MUTEX_MOD_IDX_MDP_RSZ1,
-	[MDP_COMP_TDSHP0] = MUTEX_MOD_IDX_MDP_TDSHP0,
-	[MDP_COMP_WROT0] = MUTEX_MOD_IDX_MDP_WROT0,
-	[MDP_COMP_WDMA] = MUTEX_MOD_IDX_MDP_WDMA,
-	[MDP_COMP_AAL0] = MUTEX_MOD_IDX_MDP_AAL0,
-	[MDP_COMP_CCORR0] = MUTEX_MOD_IDX_MDP_CCORR0,
-};
-
-static const struct mtk_mdp_driver_data mt8183_mdp_driver_data = {
-	.mdp_probe_infra = mt8183_mdp_probe_infra,
-	.mdp_cfg = &mt8183_plat_cfg,
-	.mdp_mutex_table_idx = mt8183_mutex_idx,
-};
-
 static const struct of_device_id mdp_of_ids[] = {
 	{ .compatible = "mediatek,mt8183-mdp3-rdma",
 	  .data = &mt8183_mdp_driver_data,
@@ -182,7 +153,7 @@ static int mdp_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct mdp_dev *mdp;
 	struct platform_device *mm_pdev;
-	int ret, i;
+	int ret, i, mutex_id;
 
 	mdp = kzalloc(sizeof(*mdp), GFP_KERNEL);
 	if (!mdp) {
@@ -205,10 +176,13 @@ static int mdp_probe(struct platform_device *pdev)
 		ret = -ENODEV;
 		goto err_destroy_device;
 	}
-	for (i = 0; i < MDP_PIPE_MAX; i++) {
-		mdp->mdp_mutex[i] = mtk_mutex_get(&mm_pdev->dev);
-		if (IS_ERR(mdp->mdp_mutex[i])) {
-			ret = PTR_ERR(mdp->mdp_mutex[i]);
+	for (i = 0; i < mdp->mdp_data->pipe_info_len; i++) {
+		mutex_id = mdp->mdp_data->pipe_info[i].mutex_id;
+		if (!IS_ERR_OR_NULL(mdp->mdp_mutex[mutex_id]))
+			continue;
+		mdp->mdp_mutex[mutex_id] = mtk_mutex_get(&mm_pdev->dev);
+		if (IS_ERR(mdp->mdp_mutex[mutex_id])) {
+			ret = PTR_ERR(mdp->mdp_mutex[mutex_id]);
 			goto err_free_mutex;
 		}
 	}
@@ -288,7 +262,7 @@ err_destroy_job_wq:
 err_deinit_comp:
 	mdp_comp_destroy(mdp);
 err_free_mutex:
-	for (i = 0; i < MDP_PIPE_MAX; i++)
+	for (i = 0; i < mdp->mdp_data->pipe_info_len; i++)
 		if (!IS_ERR_OR_NULL(mdp->mdp_mutex[i]))
 			mtk_mutex_put(mdp->mdp_mutex[i]);
 err_destroy_device:
@@ -298,14 +272,13 @@ err_return:
 	return ret;
 }
 
-static int mdp_remove(struct platform_device *pdev)
+static void mdp_remove(struct platform_device *pdev)
 {
 	struct mdp_dev *mdp = platform_get_drvdata(pdev);
 
 	v4l2_device_unregister(&mdp->v4l2_dev);
 
 	dev_dbg(&pdev->dev, "%s driver unloaded\n", pdev->name);
-	return 0;
 }
 
 static int __maybe_unused mdp_suspend(struct device *dev)
@@ -345,7 +318,7 @@ static const struct dev_pm_ops mdp_pm_ops = {
 
 static struct platform_driver mdp_driver = {
 	.probe		= mdp_probe,
-	.remove		= mdp_remove,
+	.remove_new	= mdp_remove,
 	.driver = {
 		.name	= MDP_MODULE_NAME,
 		.pm	= &mdp_pm_ops,
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.h
index 2ef5fbc4f25a..7e21d226ceb8 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.h
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.h
@@ -43,17 +43,26 @@ struct mdp_platform_config {
 
 /* indicate which mutex is used by each pipepline */
 enum mdp_pipe_id {
-	MDP_PIPE_RDMA0,
-	MDP_PIPE_IMGI,
 	MDP_PIPE_WPEI,
 	MDP_PIPE_WPEI2,
+	MDP_PIPE_IMGI,
+	MDP_PIPE_RDMA0,
 	MDP_PIPE_MAX
 };
 
 struct mtk_mdp_driver_data {
+	const int mdp_plat_id;
 	const struct of_device_id *mdp_probe_infra;
 	const struct mdp_platform_config *mdp_cfg;
 	const u32 *mdp_mutex_table_idx;
+	const struct mdp_comp_data *comp_data;
+	unsigned int comp_data_len;
+	const struct of_device_id *mdp_sub_comp_dt_ids;
+	const struct mdp_format *format;
+	unsigned int format_len;
+	const struct mdp_limit *def_limit;
+	const struct mdp_pipe_info *pipe_info;
+	unsigned int pipe_info_len;
 };
 
 struct mdp_dev {
@@ -85,6 +94,11 @@ struct mdp_dev {
 	atomic_t				job_count;
 };
 
+struct mdp_pipe_info {
+	enum mdp_pipe_id pipe_id;
+	u32 mutex_id;
+};
+
 int mdp_vpu_get_locked(struct mdp_dev *mdp);
 void mdp_vpu_put_locked(struct mdp_dev *mdp);
 int mdp_vpu_register(struct mdp_dev *mdp);
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.c
index 5f74ea3b7a52..a298c1b15b9e 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.c
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.c
@@ -87,14 +87,14 @@ static void mdp_m2m_device_run(void *priv)
 	dst_vb = v4l2_m2m_next_dst_buf(ctx->m2m_ctx);
 	mdp_set_dst_config(&param.outputs[0], frame, &dst_vb->vb2_buf);
 
-	ret = mdp_vpu_process(&ctx->vpu, &param);
+	ret = mdp_vpu_process(&ctx->mdp_dev->vpu, &param);
 	if (ret) {
 		dev_err(&ctx->mdp_dev->pdev->dev,
 			"VPU MDP process failed: %d\n", ret);
 		goto worker_end;
 	}
 
-	task.config = ctx->vpu.config;
+	task.config = ctx->mdp_dev->vpu.config;
 	task.param = &param;
 	task.composes[0] = &frame->compose;
 	task.cmdq_cb = NULL;
@@ -150,11 +150,6 @@ static int mdp_m2m_start_streaming(struct vb2_queue *q, unsigned int count)
 
 	if (!mdp_m2m_ctx_is_state_set(ctx, MDP_VPU_INIT)) {
 		ret = mdp_vpu_get_locked(ctx->mdp_dev);
-		if (ret)
-			return ret;
-
-		ret = mdp_vpu_ctx_init(&ctx->vpu, &ctx->mdp_dev->vpu,
-				       MDP_DEV_M2M);
 		if (ret) {
 			dev_err(&ctx->mdp_dev->pdev->dev,
 				"VPU init failed %d\n", ret);
@@ -277,7 +272,9 @@ static int mdp_m2m_querycap(struct file *file, void *fh,
 static int mdp_m2m_enum_fmt_mplane(struct file *file, void *fh,
 				   struct v4l2_fmtdesc *f)
 {
-	return mdp_enum_fmt_mplane(f);
+	struct mdp_m2m_ctx *ctx = fh_to_ctx(fh);
+
+	return mdp_enum_fmt_mplane(ctx->mdp_dev, f);
 }
 
 static int mdp_m2m_g_fmt_mplane(struct file *file, void *fh,
@@ -307,7 +304,7 @@ static int mdp_m2m_s_fmt_mplane(struct file *file, void *fh,
 	const struct mdp_format *fmt;
 	struct vb2_queue *vq;
 
-	fmt = mdp_try_fmt_mplane(f, &ctx->curr_param, ctx->id);
+	fmt = mdp_try_fmt_mplane(ctx->mdp_dev, f, &ctx->curr_param, ctx->id);
 	if (!fmt)
 		return -EINVAL;
 
@@ -346,7 +343,7 @@ static int mdp_m2m_try_fmt_mplane(struct file *file, void *fh,
 {
 	struct mdp_m2m_ctx *ctx = fh_to_ctx(fh);
 
-	if (!mdp_try_fmt_mplane(f, &ctx->curr_param, ctx->id))
+	if (!mdp_try_fmt_mplane(ctx->mdp_dev, f, &ctx->curr_param, ctx->id))
 		return -EINVAL;
 
 	return 0;
@@ -556,6 +553,7 @@ static int mdp_m2m_open(struct file *file)
 	struct device *dev = &mdp->pdev->dev;
 	int ret;
 	struct v4l2_format default_format = {};
+	const struct mdp_limit *limit = mdp->mdp_data->def_limit;
 
 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
@@ -566,7 +564,11 @@ static int mdp_m2m_open(struct file *file)
 		goto err_free_ctx;
 	}
 
-	ctx->id = ida_alloc(&mdp->mdp_ida, GFP_KERNEL);
+	ret = ida_alloc(&mdp->mdp_ida, GFP_KERNEL);
+	if (ret < 0)
+		goto err_unlock_mutex;
+	ctx->id = ret;
+
 	ctx->mdp_dev = mdp;
 
 	v4l2_fh_init(&ctx->fh, vdev);
@@ -589,7 +591,7 @@ static int mdp_m2m_open(struct file *file)
 	ctx->fh.m2m_ctx = ctx->m2m_ctx;
 
 	ctx->curr_param.ctx = ctx;
-	ret = mdp_frameparam_init(&ctx->curr_param);
+	ret = mdp_frameparam_init(mdp, &ctx->curr_param);
 	if (ret) {
 		dev_err(dev, "Failed to initialize mdp parameter\n");
 		goto err_release_m2m_ctx;
@@ -599,8 +601,8 @@ static int mdp_m2m_open(struct file *file)
 
 	/* Default format */
 	default_format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
-	default_format.fmt.pix_mp.width = 32;
-	default_format.fmt.pix_mp.height = 32;
+	default_format.fmt.pix_mp.width = limit->out_limit.wmin;
+	default_format.fmt.pix_mp.height = limit->out_limit.hmin;
 	default_format.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_YUV420M;
 	mdp_m2m_s_fmt_mplane(file, &ctx->fh, &default_format);
 	default_format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
@@ -617,6 +619,8 @@ err_release_handler:
 	v4l2_fh_del(&ctx->fh);
 err_exit_fh:
 	v4l2_fh_exit(&ctx->fh);
+	ida_free(&mdp->mdp_ida, ctx->id);
+err_unlock_mutex:
 	mutex_unlock(&mdp->m2m_lock);
 err_free_ctx:
 	kfree(ctx);
@@ -632,10 +636,8 @@ static int mdp_m2m_release(struct file *file)
 
 	mutex_lock(&mdp->m2m_lock);
 	v4l2_m2m_ctx_release(ctx->m2m_ctx);
-	if (mdp_m2m_ctx_is_state_set(ctx, MDP_VPU_INIT)) {
-		mdp_vpu_ctx_deinit(&ctx->vpu);
+	if (mdp_m2m_ctx_is_state_set(ctx, MDP_VPU_INIT))
 		mdp_vpu_put_locked(mdp);
-	}
 
 	v4l2_ctrl_handler_free(&ctx->ctrl_handler);
 	v4l2_fh_del(&ctx->fh);
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.h
index 61ddbaf1bf13..dfc59e5b3b87 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.h
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.h
@@ -33,7 +33,6 @@ struct mdp_m2m_ctx {
 	struct v4l2_ctrl_handler	ctrl_handler;
 	struct mdp_m2m_ctrls		ctrls;
 	struct v4l2_m2m_ctx		*m2m_ctx;
-	struct mdp_vpu_ctx		vpu;
 	u32				frame_count[MDP_M2M_MAX];
 
 	struct mdp_frameparam		curr_param;
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c
index 4e84a37ecdfc..9b436b911d92 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c
@@ -4,6 +4,7 @@
  * Author: Ping-Hsun Wu <ping-hsun.wu@mediatek.com>
  */
 
+#include <linux/math64.h>
 #include <media/v4l2-common.h>
 #include <media/videobuf2-v4l2.h>
 #include <media/videobuf2-dma-contig.h>
@@ -11,276 +12,34 @@
 #include "mtk-mdp3-regs.h"
 #include "mtk-mdp3-m2m.h"
 
-/*
- * All 10-bit related formats are not added in the basic format list,
- * please add the corresponding format settings before use.
- */
-static const struct mdp_format mdp_formats[] = {
-	{
-		.pixelformat	= V4L2_PIX_FMT_GREY,
-		.mdp_color	= MDP_COLOR_GREY,
-		.depth		= { 8 },
-		.row_depth	= { 8 },
-		.num_planes	= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_RGB565X,
-		.mdp_color	= MDP_COLOR_BGR565,
-		.depth		= { 16 },
-		.row_depth	= { 16 },
-		.num_planes	= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_RGB565,
-		.mdp_color	= MDP_COLOR_RGB565,
-		.depth		= { 16 },
-		.row_depth	= { 16 },
-		.num_planes	= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_RGB24,
-		.mdp_color	= MDP_COLOR_RGB888,
-		.depth		= { 24 },
-		.row_depth	= { 24 },
-		.num_planes	= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_BGR24,
-		.mdp_color	= MDP_COLOR_BGR888,
-		.depth		= { 24 },
-		.row_depth	= { 24 },
-		.num_planes	= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_ABGR32,
-		.mdp_color	= MDP_COLOR_BGRA8888,
-		.depth		= { 32 },
-		.row_depth	= { 32 },
-		.num_planes	= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_ARGB32,
-		.mdp_color	= MDP_COLOR_ARGB8888,
-		.depth		= { 32 },
-		.row_depth	= { 32 },
-		.num_planes	= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_UYVY,
-		.mdp_color	= MDP_COLOR_UYVY,
-		.depth		= { 16 },
-		.row_depth	= { 16 },
-		.num_planes	= 1,
-		.walign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_VYUY,
-		.mdp_color	= MDP_COLOR_VYUY,
-		.depth		= { 16 },
-		.row_depth	= { 16 },
-		.num_planes	= 1,
-		.walign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_YUYV,
-		.mdp_color	= MDP_COLOR_YUYV,
-		.depth		= { 16 },
-		.row_depth	= { 16 },
-		.num_planes	= 1,
-		.walign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_YVYU,
-		.mdp_color	= MDP_COLOR_YVYU,
-		.depth		= { 16 },
-		.row_depth	= { 16 },
-		.num_planes	= 1,
-		.walign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_YUV420,
-		.mdp_color	= MDP_COLOR_I420,
-		.depth		= { 12 },
-		.row_depth	= { 8 },
-		.num_planes	= 1,
-		.walign		= 1,
-		.halign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_YVU420,
-		.mdp_color	= MDP_COLOR_YV12,
-		.depth		= { 12 },
-		.row_depth	= { 8 },
-		.num_planes	= 1,
-		.walign		= 1,
-		.halign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_NV12,
-		.mdp_color	= MDP_COLOR_NV12,
-		.depth		= { 12 },
-		.row_depth	= { 8 },
-		.num_planes	= 1,
-		.walign		= 1,
-		.halign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_NV21,
-		.mdp_color	= MDP_COLOR_NV21,
-		.depth		= { 12 },
-		.row_depth	= { 8 },
-		.num_planes	= 1,
-		.walign		= 1,
-		.halign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_NV16,
-		.mdp_color	= MDP_COLOR_NV16,
-		.depth		= { 16 },
-		.row_depth	= { 8 },
-		.num_planes	= 1,
-		.walign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_NV61,
-		.mdp_color	= MDP_COLOR_NV61,
-		.depth		= { 16 },
-		.row_depth	= { 8 },
-		.num_planes	= 1,
-		.walign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_NV24,
-		.mdp_color	= MDP_COLOR_NV24,
-		.depth		= { 24 },
-		.row_depth	= { 8 },
-		.num_planes	= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_NV42,
-		.mdp_color	= MDP_COLOR_NV42,
-		.depth		= { 24 },
-		.row_depth	= { 8 },
-		.num_planes	= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_MT21C,
-		.mdp_color	= MDP_COLOR_420_BLK_UFO,
-		.depth		= { 8, 4 },
-		.row_depth	= { 8, 8 },
-		.num_planes	= 2,
-		.walign		= 4,
-		.halign		= 5,
-		.flags		= MDP_FMT_FLAG_OUTPUT,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_MM21,
-		.mdp_color	= MDP_COLOR_420_BLK,
-		.depth		= { 8, 4 },
-		.row_depth	= { 8, 8 },
-		.num_planes	= 2,
-		.walign		= 4,
-		.halign		= 5,
-		.flags		= MDP_FMT_FLAG_OUTPUT,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_NV12M,
-		.mdp_color	= MDP_COLOR_NV12,
-		.depth		= { 8, 4 },
-		.row_depth	= { 8, 8 },
-		.num_planes	= 2,
-		.walign		= 1,
-		.halign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_NV21M,
-		.mdp_color	= MDP_COLOR_NV21,
-		.depth		= { 8, 4 },
-		.row_depth	= { 8, 8 },
-		.num_planes	= 2,
-		.walign		= 1,
-		.halign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_NV16M,
-		.mdp_color	= MDP_COLOR_NV16,
-		.depth		= { 8, 8 },
-		.row_depth	= { 8, 8 },
-		.num_planes	= 2,
-		.walign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_NV61M,
-		.mdp_color	= MDP_COLOR_NV61,
-		.depth		= { 8, 8 },
-		.row_depth	= { 8, 8 },
-		.num_planes	= 2,
-		.walign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_YUV420M,
-		.mdp_color	= MDP_COLOR_I420,
-		.depth		= { 8, 2, 2 },
-		.row_depth	= { 8, 4, 4 },
-		.num_planes	= 3,
-		.walign		= 1,
-		.halign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}, {
-		.pixelformat	= V4L2_PIX_FMT_YVU420M,
-		.mdp_color	= MDP_COLOR_YV12,
-		.depth		= { 8, 2, 2 },
-		.row_depth	= { 8, 4, 4 },
-		.num_planes	= 3,
-		.walign		= 1,
-		.halign		= 1,
-		.flags		= MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE,
-	}
-};
-
-static const struct mdp_limit mdp_def_limit = {
-	.out_limit = {
-		.wmin	= 16,
-		.hmin	= 16,
-		.wmax	= 8176,
-		.hmax	= 8176,
-	},
-	.cap_limit = {
-		.wmin	= 2,
-		.hmin	= 2,
-		.wmax	= 8176,
-		.hmax	= 8176,
-	},
-	.h_scale_up_max = 32,
-	.v_scale_up_max = 32,
-	.h_scale_down_max = 20,
-	.v_scale_down_max = 128,
-};
-
-static const struct mdp_format *mdp_find_fmt(u32 pixelformat, u32 type)
+static const struct mdp_format *mdp_find_fmt(const struct mtk_mdp_driver_data *mdp_data,
+					     u32 pixelformat, u32 type)
 {
 	u32 i, flag;
 
 	flag = V4L2_TYPE_IS_OUTPUT(type) ? MDP_FMT_FLAG_OUTPUT :
 					MDP_FMT_FLAG_CAPTURE;
-	for (i = 0; i < ARRAY_SIZE(mdp_formats); ++i) {
-		if (!(mdp_formats[i].flags & flag))
+	for (i = 0; i < mdp_data->format_len; ++i) {
+		if (!(mdp_data->format[i].flags & flag))
 			continue;
-		if (mdp_formats[i].pixelformat == pixelformat)
-			return &mdp_formats[i];
+		if (mdp_data->format[i].pixelformat == pixelformat)
+			return &mdp_data->format[i];
 	}
 	return NULL;
 }
 
-static const struct mdp_format *mdp_find_fmt_by_index(u32 index, u32 type)
+static const struct mdp_format *mdp_find_fmt_by_index(const struct mtk_mdp_driver_data *mdp_data,
+						      u32 index, u32 type)
 {
 	u32 i, flag, num = 0;
 
 	flag = V4L2_TYPE_IS_OUTPUT(type) ? MDP_FMT_FLAG_OUTPUT :
 					MDP_FMT_FLAG_CAPTURE;
-	for (i = 0; i < ARRAY_SIZE(mdp_formats); ++i) {
-		if (!(mdp_formats[i].flags & flag))
+	for (i = 0; i < mdp_data->format_len; ++i) {
+		if (!(mdp_data->format[i].flags & flag))
 			continue;
 		if (index == num)
-			return &mdp_formats[i];
+			return &mdp_data->format[i];
 		num++;
 	}
 	return NULL;
@@ -354,11 +113,11 @@ static int mdp_clamp_align(s32 *x, int min, int max, unsigned int align)
 	return 0;
 }
 
-int mdp_enum_fmt_mplane(struct v4l2_fmtdesc *f)
+int mdp_enum_fmt_mplane(struct mdp_dev *mdp, struct v4l2_fmtdesc *f)
 {
 	const struct mdp_format *fmt;
 
-	fmt = mdp_find_fmt_by_index(f->index, f->type);
+	fmt = mdp_find_fmt_by_index(mdp->mdp_data, f->index, f->type);
 	if (!fmt)
 		return -EINVAL;
 
@@ -366,7 +125,8 @@ int mdp_enum_fmt_mplane(struct v4l2_fmtdesc *f)
 	return 0;
 }
 
-const struct mdp_format *mdp_try_fmt_mplane(struct v4l2_format *f,
+const struct mdp_format *mdp_try_fmt_mplane(struct mdp_dev *mdp,
+					    struct v4l2_format *f,
 					    struct mdp_frameparam *param,
 					    u32 ctx_id)
 {
@@ -378,9 +138,9 @@ const struct mdp_format *mdp_try_fmt_mplane(struct v4l2_format *f,
 	u32 org_w, org_h;
 	unsigned int i;
 
-	fmt = mdp_find_fmt(pix_mp->pixelformat, f->type);
+	fmt = mdp_find_fmt(mdp->mdp_data, pix_mp->pixelformat, f->type);
 	if (!fmt) {
-		fmt = mdp_find_fmt_by_index(0, f->type);
+		fmt = mdp_find_fmt_by_index(mdp->mdp_data, 0, f->type);
 		if (!fmt) {
 			dev_dbg(dev, "%d: pixelformat %c%c%c%c invalid", ctx_id,
 				(pix_mp->pixelformat & 0xff),
@@ -428,14 +188,15 @@ const struct mdp_format *mdp_try_fmt_mplane(struct v4l2_format *f,
 		u32 bpl = pix_mp->plane_fmt[i].bytesperline;
 		u32 min_si, max_si;
 		u32 si = pix_mp->plane_fmt[i].sizeimage;
+		u64 di;
 
 		bpl = clamp(bpl, min_bpl, max_bpl);
 		pix_mp->plane_fmt[i].bytesperline = bpl;
 
-		min_si = (bpl * pix_mp->height * fmt->depth[i]) /
-			 fmt->row_depth[i];
-		max_si = (bpl * s.max_height * fmt->depth[i]) /
-			 fmt->row_depth[i];
+		di = (u64)bpl * pix_mp->height * fmt->depth[i];
+		min_si = (u32)div_u64(di, fmt->row_depth[i]);
+		di = (u64)bpl * s.max_height * fmt->depth[i];
+		max_si = (u32)div_u64(di, fmt->row_depth[i]);
 
 		si = clamp(si, min_si, max_si);
 		pix_mp->plane_fmt[i].sizeimage = si;
@@ -699,7 +460,7 @@ void mdp_set_dst_config(struct img_output *out,
 	mdp_set_orientation(out, frame->rotation, frame->hflip, frame->vflip);
 }
 
-int mdp_frameparam_init(struct mdp_frameparam *param)
+int mdp_frameparam_init(struct mdp_dev *mdp, struct mdp_frameparam *param)
 {
 	struct mdp_frame *frame;
 
@@ -707,12 +468,12 @@ int mdp_frameparam_init(struct mdp_frameparam *param)
 		return -EINVAL;
 
 	INIT_LIST_HEAD(&param->list);
-	param->limit = &mdp_def_limit;
+	param->limit = mdp->mdp_data->def_limit;
 	param->type = MDP_STREAM_TYPE_BITBLT;
 
 	frame = &param->output;
 	frame->format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
-	frame->mdp_fmt = mdp_try_fmt_mplane(&frame->format, param, 0);
+	frame->mdp_fmt = mdp_try_fmt_mplane(mdp, &frame->format, param, 0);
 	frame->ycbcr_prof =
 		mdp_map_ycbcr_prof_mplane(&frame->format,
 					  frame->mdp_fmt->mdp_color);
@@ -721,7 +482,7 @@ int mdp_frameparam_init(struct mdp_frameparam *param)
 	param->num_captures = 1;
 	frame = &param->captures[0];
 	frame->format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
-	frame->mdp_fmt = mdp_try_fmt_mplane(&frame->format, param, 0);
+	frame->mdp_fmt = mdp_try_fmt_mplane(mdp, &frame->format, param, 0);
 	frame->ycbcr_prof =
 		mdp_map_ycbcr_prof_mplane(&frame->format,
 					  frame->mdp_fmt->mdp_color);
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.h
index f995e536d45f..e9ab8ac2c0e8 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.h
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.h
@@ -18,11 +18,12 @@
  * V-subsample: 0, 1
  * Color group: 0-RGB, 1-YUV, 2-raw
  */
-#define MDP_COLOR(PACKED, LOOSE, VIDEO, PLANE, HF, VF, BITS, GROUP, SWAP, ID)\
-	(((PACKED) << 27) | ((LOOSE) << 26) | ((VIDEO) << 23) |\
+#define MDP_COLOR(COMPRESS, PACKED, LOOSE, VIDEO, PLANE, HF, VF, BITS, GROUP, SWAP, ID)\
+	(((COMPRESS) << 29) | ((PACKED) << 27) | ((LOOSE) << 26) | ((VIDEO) << 23) |\
 	 ((PLANE) << 21) | ((HF) << 19) | ((VF) << 18) | ((BITS) << 8) |\
 	 ((GROUP) << 6) | ((SWAP) << 5) | ((ID) << 0))
 
+#define MDP_COLOR_IS_COMPRESS(c)        ((0x20000000 & (c)) >> 29)
 #define MDP_COLOR_IS_10BIT_PACKED(c)	((0x08000000 & (c)) >> 27)
 #define MDP_COLOR_IS_10BIT_LOOSE(c)	(((0x0c000000 & (c)) >> 26) == 1)
 #define MDP_COLOR_IS_10BIT_TILE(c)	(((0x0c000000 & (c)) >> 26) == 3)
@@ -44,144 +45,144 @@
 enum mdp_color {
 	MDP_COLOR_UNKNOWN	= 0,
 
-	//MDP_COLOR_FULLG8,
-	MDP_COLOR_FULLG8_RGGB	= MDP_COLOR(0, 0, 0, 1, 0, 0,  8, 2,  0, 21),
-	MDP_COLOR_FULLG8_GRBG	= MDP_COLOR(0, 0, 0, 1, 0, 1,  8, 2,  0, 21),
-	MDP_COLOR_FULLG8_GBRG	= MDP_COLOR(0, 0, 0, 1, 1, 0,  8, 2,  0, 21),
-	MDP_COLOR_FULLG8_BGGR	= MDP_COLOR(0, 0, 0, 1, 1, 1,  8, 2,  0, 21),
+	/* MDP_COLOR_FULLG8 */
+	MDP_COLOR_FULLG8_RGGB	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0,  8, 2,  0, 21),
+	MDP_COLOR_FULLG8_GRBG	= MDP_COLOR(0, 0, 0, 0, 1, 0, 1,  8, 2,  0, 21),
+	MDP_COLOR_FULLG8_GBRG	= MDP_COLOR(0, 0, 0, 0, 1, 1, 0,  8, 2,  0, 21),
+	MDP_COLOR_FULLG8_BGGR	= MDP_COLOR(0, 0, 0, 0, 1, 1, 1,  8, 2,  0, 21),
 	MDP_COLOR_FULLG8	= MDP_COLOR_FULLG8_BGGR,
 
-	//MDP_COLOR_FULLG10,
-	MDP_COLOR_FULLG10_RGGB	= MDP_COLOR(0, 0, 0, 1, 0, 0, 10, 2,  0, 21),
-	MDP_COLOR_FULLG10_GRBG	= MDP_COLOR(0, 0, 0, 1, 0, 1, 10, 2,  0, 21),
-	MDP_COLOR_FULLG10_GBRG	= MDP_COLOR(0, 0, 0, 1, 1, 0, 10, 2,  0, 21),
-	MDP_COLOR_FULLG10_BGGR	= MDP_COLOR(0, 0, 0, 1, 1, 1, 10, 2,  0, 21),
+	/* MDP_COLOR_FULLG10 */
+	MDP_COLOR_FULLG10_RGGB	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 10, 2,  0, 21),
+	MDP_COLOR_FULLG10_GRBG	= MDP_COLOR(0, 0, 0, 0, 1, 0, 1, 10, 2,  0, 21),
+	MDP_COLOR_FULLG10_GBRG	= MDP_COLOR(0, 0, 0, 0, 1, 1, 0, 10, 2,  0, 21),
+	MDP_COLOR_FULLG10_BGGR	= MDP_COLOR(0, 0, 0, 0, 1, 1, 1, 10, 2,  0, 21),
 	MDP_COLOR_FULLG10	= MDP_COLOR_FULLG10_BGGR,
 
-	//MDP_COLOR_FULLG12,
-	MDP_COLOR_FULLG12_RGGB	= MDP_COLOR(0, 0, 0, 1, 0, 0, 12, 2,  0, 21),
-	MDP_COLOR_FULLG12_GRBG	= MDP_COLOR(0, 0, 0, 1, 0, 1, 12, 2,  0, 21),
-	MDP_COLOR_FULLG12_GBRG	= MDP_COLOR(0, 0, 0, 1, 1, 0, 12, 2,  0, 21),
-	MDP_COLOR_FULLG12_BGGR	= MDP_COLOR(0, 0, 0, 1, 1, 1, 12, 2,  0, 21),
+	/* MDP_COLOR_FULLG12 */
+	MDP_COLOR_FULLG12_RGGB	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 12, 2,  0, 21),
+	MDP_COLOR_FULLG12_GRBG	= MDP_COLOR(0, 0, 0, 0, 1, 0, 1, 12, 2,  0, 21),
+	MDP_COLOR_FULLG12_GBRG	= MDP_COLOR(0, 0, 0, 0, 1, 1, 0, 12, 2,  0, 21),
+	MDP_COLOR_FULLG12_BGGR	= MDP_COLOR(0, 0, 0, 0, 1, 1, 1, 12, 2,  0, 21),
 	MDP_COLOR_FULLG12	= MDP_COLOR_FULLG12_BGGR,
 
-	//MDP_COLOR_FULLG14,
-	MDP_COLOR_FULLG14_RGGB	= MDP_COLOR(0, 0, 0, 1, 0, 0, 14, 2,  0, 21),
-	MDP_COLOR_FULLG14_GRBG	= MDP_COLOR(0, 0, 0, 1, 0, 1, 14, 2,  0, 21),
-	MDP_COLOR_FULLG14_GBRG	= MDP_COLOR(0, 0, 0, 1, 1, 0, 14, 2,  0, 21),
-	MDP_COLOR_FULLG14_BGGR	= MDP_COLOR(0, 0, 0, 1, 1, 1, 14, 2,  0, 21),
+	/* MDP_COLOR_FULLG14 */
+	MDP_COLOR_FULLG14_RGGB	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 14, 2,  0, 21),
+	MDP_COLOR_FULLG14_GRBG	= MDP_COLOR(0, 0, 0, 0, 1, 0, 1, 14, 2,  0, 21),
+	MDP_COLOR_FULLG14_GBRG	= MDP_COLOR(0, 0, 0, 0, 1, 1, 0, 14, 2,  0, 21),
+	MDP_COLOR_FULLG14_BGGR	= MDP_COLOR(0, 0, 0, 0, 1, 1, 1, 14, 2,  0, 21),
 	MDP_COLOR_FULLG14	= MDP_COLOR_FULLG14_BGGR,
 
-	MDP_COLOR_UFO10		= MDP_COLOR(0, 0, 0, 1, 0, 0, 10, 2,  0, 24),
+	MDP_COLOR_UFO10		= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 10, 2,  0, 24),
 
-	//MDP_COLOR_BAYER8,
-	MDP_COLOR_BAYER8_RGGB	= MDP_COLOR(0, 0, 0, 1, 0, 0,  8, 2,  0, 20),
-	MDP_COLOR_BAYER8_GRBG	= MDP_COLOR(0, 0, 0, 1, 0, 1,  8, 2,  0, 20),
-	MDP_COLOR_BAYER8_GBRG	= MDP_COLOR(0, 0, 0, 1, 1, 0,  8, 2,  0, 20),
-	MDP_COLOR_BAYER8_BGGR	= MDP_COLOR(0, 0, 0, 1, 1, 1,  8, 2,  0, 20),
+	/* MDP_COLOR_BAYER8 */
+	MDP_COLOR_BAYER8_RGGB	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0,  8, 2,  0, 20),
+	MDP_COLOR_BAYER8_GRBG	= MDP_COLOR(0, 0, 0, 0, 1, 0, 1,  8, 2,  0, 20),
+	MDP_COLOR_BAYER8_GBRG	= MDP_COLOR(0, 0, 0, 0, 1, 1, 0,  8, 2,  0, 20),
+	MDP_COLOR_BAYER8_BGGR	= MDP_COLOR(0, 0, 0, 0, 1, 1, 1,  8, 2,  0, 20),
 	MDP_COLOR_BAYER8	= MDP_COLOR_BAYER8_BGGR,
 
-	//MDP_COLOR_BAYER10,
-	MDP_COLOR_BAYER10_RGGB	= MDP_COLOR(0, 0, 0, 1, 0, 0, 10, 2,  0, 20),
-	MDP_COLOR_BAYER10_GRBG	= MDP_COLOR(0, 0, 0, 1, 0, 1, 10, 2,  0, 20),
-	MDP_COLOR_BAYER10_GBRG	= MDP_COLOR(0, 0, 0, 1, 1, 0, 10, 2,  0, 20),
-	MDP_COLOR_BAYER10_BGGR	= MDP_COLOR(0, 0, 0, 1, 1, 1, 10, 2,  0, 20),
+	/* MDP_COLOR_BAYER10 */
+	MDP_COLOR_BAYER10_RGGB	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 10, 2,  0, 20),
+	MDP_COLOR_BAYER10_GRBG	= MDP_COLOR(0, 0, 0, 0, 1, 0, 1, 10, 2,  0, 20),
+	MDP_COLOR_BAYER10_GBRG	= MDP_COLOR(0, 0, 0, 0, 1, 1, 0, 10, 2,  0, 20),
+	MDP_COLOR_BAYER10_BGGR	= MDP_COLOR(0, 0, 0, 0, 1, 1, 1, 10, 2,  0, 20),
 	MDP_COLOR_BAYER10	= MDP_COLOR_BAYER10_BGGR,
 
-	//MDP_COLOR_BAYER12,
-	MDP_COLOR_BAYER12_RGGB	= MDP_COLOR(0, 0, 0, 1, 0, 0, 12, 2,  0, 20),
-	MDP_COLOR_BAYER12_GRBG	= MDP_COLOR(0, 0, 0, 1, 0, 1, 12, 2,  0, 20),
-	MDP_COLOR_BAYER12_GBRG	= MDP_COLOR(0, 0, 0, 1, 1, 0, 12, 2,  0, 20),
-	MDP_COLOR_BAYER12_BGGR	= MDP_COLOR(0, 0, 0, 1, 1, 1, 12, 2,  0, 20),
+	/* MDP_COLOR_BAYER12 */
+	MDP_COLOR_BAYER12_RGGB	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 12, 2,  0, 20),
+	MDP_COLOR_BAYER12_GRBG	= MDP_COLOR(0, 0, 0, 0, 1, 0, 1, 12, 2,  0, 20),
+	MDP_COLOR_BAYER12_GBRG	= MDP_COLOR(0, 0, 0, 0, 1, 1, 0, 12, 2,  0, 20),
+	MDP_COLOR_BAYER12_BGGR	= MDP_COLOR(0, 0, 0, 0, 1, 1, 1, 12, 2,  0, 20),
 	MDP_COLOR_BAYER12	= MDP_COLOR_BAYER12_BGGR,
 
-	//MDP_COLOR_BAYER14,
-	MDP_COLOR_BAYER14_RGGB	= MDP_COLOR(0, 0, 0, 1, 0, 0, 14, 2,  0, 20),
-	MDP_COLOR_BAYER14_GRBG	= MDP_COLOR(0, 0, 0, 1, 0, 1, 14, 2,  0, 20),
-	MDP_COLOR_BAYER14_GBRG	= MDP_COLOR(0, 0, 0, 1, 1, 0, 14, 2,  0, 20),
-	MDP_COLOR_BAYER14_BGGR	= MDP_COLOR(0, 0, 0, 1, 1, 1, 14, 2,  0, 20),
+	/* MDP_COLOR_BAYER14 */
+	MDP_COLOR_BAYER14_RGGB	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 14, 2,  0, 20),
+	MDP_COLOR_BAYER14_GRBG	= MDP_COLOR(0, 0, 0, 0, 1, 0, 1, 14, 2,  0, 20),
+	MDP_COLOR_BAYER14_GBRG	= MDP_COLOR(0, 0, 0, 0, 1, 1, 0, 14, 2,  0, 20),
+	MDP_COLOR_BAYER14_BGGR	= MDP_COLOR(0, 0, 0, 0, 1, 1, 1, 14, 2,  0, 20),
 	MDP_COLOR_BAYER14	= MDP_COLOR_BAYER14_BGGR,
 
-	MDP_COLOR_RGB48		= MDP_COLOR(0, 0, 0, 1, 0, 0, 48, 0,  0, 23),
+	MDP_COLOR_RGB48		= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 48, 0,  0, 23),
 	/* For bayer+mono raw-16 */
-	MDP_COLOR_RGB565_RAW	= MDP_COLOR(0, 0, 0, 1, 0, 0, 16, 2,  0, 0),
+	MDP_COLOR_RGB565_RAW	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 16, 2,  0, 0),
 
-	MDP_COLOR_BAYER8_UNPAK	= MDP_COLOR(0, 0, 0, 1, 0, 0,  8, 2,  0, 22),
-	MDP_COLOR_BAYER10_UNPAK	= MDP_COLOR(0, 0, 0, 1, 0, 0, 10, 2,  0, 22),
-	MDP_COLOR_BAYER12_UNPAK	= MDP_COLOR(0, 0, 0, 1, 0, 0, 12, 2,  0, 22),
-	MDP_COLOR_BAYER14_UNPAK	= MDP_COLOR(0, 0, 0, 1, 0, 0, 14, 2,  0, 22),
+	MDP_COLOR_BAYER8_UNPAK	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0,  8, 2,  0, 22),
+	MDP_COLOR_BAYER10_UNPAK	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 10, 2,  0, 22),
+	MDP_COLOR_BAYER12_UNPAK	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 12, 2,  0, 22),
+	MDP_COLOR_BAYER14_UNPAK	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 14, 2,  0, 22),
 
 	/* Unified formats */
-	MDP_COLOR_GREY		= MDP_COLOR(0, 0, 0, 1, 0, 0,  8, 1,  0, 7),
-
-	MDP_COLOR_RGB565	= MDP_COLOR(0, 0, 0, 1, 0, 0, 16, 0,  0, 0),
-	MDP_COLOR_BGR565	= MDP_COLOR(0, 0, 0, 1, 0, 0, 16, 0,  1, 0),
-	MDP_COLOR_RGB888	= MDP_COLOR(0, 0, 0, 1, 0, 0, 24, 0,  1, 1),
-	MDP_COLOR_BGR888	= MDP_COLOR(0, 0, 0, 1, 0, 0, 24, 0,  0, 1),
-	MDP_COLOR_RGBA8888	= MDP_COLOR(0, 0, 0, 1, 0, 0, 32, 0,  1, 2),
-	MDP_COLOR_BGRA8888	= MDP_COLOR(0, 0, 0, 1, 0, 0, 32, 0,  0, 2),
-	MDP_COLOR_ARGB8888	= MDP_COLOR(0, 0, 0, 1, 0, 0, 32, 0,  1, 3),
-	MDP_COLOR_ABGR8888	= MDP_COLOR(0, 0, 0, 1, 0, 0, 32, 0,  0, 3),
-
-	MDP_COLOR_UYVY		= MDP_COLOR(0, 0, 0, 1, 1, 0, 16, 1,  0, 4),
-	MDP_COLOR_VYUY		= MDP_COLOR(0, 0, 0, 1, 1, 0, 16, 1,  1, 4),
-	MDP_COLOR_YUYV		= MDP_COLOR(0, 0, 0, 1, 1, 0, 16, 1,  0, 5),
-	MDP_COLOR_YVYU		= MDP_COLOR(0, 0, 0, 1, 1, 0, 16, 1,  1, 5),
-
-	MDP_COLOR_I420		= MDP_COLOR(0, 0, 0, 3, 1, 1,  8, 1,  0, 8),
-	MDP_COLOR_YV12		= MDP_COLOR(0, 0, 0, 3, 1, 1,  8, 1,  1, 8),
-	MDP_COLOR_I422		= MDP_COLOR(0, 0, 0, 3, 1, 0,  8, 1,  0, 9),
-	MDP_COLOR_YV16		= MDP_COLOR(0, 0, 0, 3, 1, 0,  8, 1,  1, 9),
-	MDP_COLOR_I444		= MDP_COLOR(0, 0, 0, 3, 0, 0,  8, 1,  0, 10),
-	MDP_COLOR_YV24		= MDP_COLOR(0, 0, 0, 3, 0, 0,  8, 1,  1, 10),
-
-	MDP_COLOR_NV12		= MDP_COLOR(0, 0, 0, 2, 1, 1,  8, 1,  0, 12),
-	MDP_COLOR_NV21		= MDP_COLOR(0, 0, 0, 2, 1, 1,  8, 1,  1, 12),
-	MDP_COLOR_NV16		= MDP_COLOR(0, 0, 0, 2, 1, 0,  8, 1,  0, 13),
-	MDP_COLOR_NV61		= MDP_COLOR(0, 0, 0, 2, 1, 0,  8, 1,  1, 13),
-	MDP_COLOR_NV24		= MDP_COLOR(0, 0, 0, 2, 0, 0,  8, 1,  0, 14),
-	MDP_COLOR_NV42		= MDP_COLOR(0, 0, 0, 2, 0, 0,  8, 1,  1, 14),
+	MDP_COLOR_GREY		= MDP_COLOR(0, 0, 0, 0, 1, 0, 0,  8, 1,  0, 7),
+
+	MDP_COLOR_RGB565	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 16, 0,  0, 0),
+	MDP_COLOR_BGR565	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 16, 0,  1, 0),
+	MDP_COLOR_RGB888	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 24, 0,  1, 1),
+	MDP_COLOR_BGR888	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 24, 0,  0, 1),
+	MDP_COLOR_RGBA8888	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 32, 0,  1, 2),
+	MDP_COLOR_BGRA8888	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 32, 0,  0, 2),
+	MDP_COLOR_ARGB8888	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 32, 0,  1, 3),
+	MDP_COLOR_ABGR8888	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 32, 0,  0, 3),
+
+	MDP_COLOR_UYVY		= MDP_COLOR(0, 0, 0, 0, 1, 1, 0, 16, 1,  0, 4),
+	MDP_COLOR_VYUY		= MDP_COLOR(0, 0, 0, 0, 1, 1, 0, 16, 1,  1, 4),
+	MDP_COLOR_YUYV		= MDP_COLOR(0, 0, 0, 0, 1, 1, 0, 16, 1,  0, 5),
+	MDP_COLOR_YVYU		= MDP_COLOR(0, 0, 0, 0, 1, 1, 0, 16, 1,  1, 5),
+
+	MDP_COLOR_I420		= MDP_COLOR(0, 0, 0, 0, 3, 1, 1,  8, 1,  0, 8),
+	MDP_COLOR_YV12		= MDP_COLOR(0, 0, 0, 0, 3, 1, 1,  8, 1,  1, 8),
+	MDP_COLOR_I422		= MDP_COLOR(0, 0, 0, 0, 3, 1, 0,  8, 1,  0, 9),
+	MDP_COLOR_YV16		= MDP_COLOR(0, 0, 0, 0, 3, 1, 0,  8, 1,  1, 9),
+	MDP_COLOR_I444		= MDP_COLOR(0, 0, 0, 0, 3, 0, 0,  8, 1,  0, 10),
+	MDP_COLOR_YV24		= MDP_COLOR(0, 0, 0, 0, 3, 0, 0,  8, 1,  1, 10),
+
+	MDP_COLOR_NV12		= MDP_COLOR(0, 0, 0, 0, 2, 1, 1,  8, 1,  0, 12),
+	MDP_COLOR_NV21		= MDP_COLOR(0, 0, 0, 0, 2, 1, 1,  8, 1,  1, 12),
+	MDP_COLOR_NV16		= MDP_COLOR(0, 0, 0, 0, 2, 1, 0,  8, 1,  0, 13),
+	MDP_COLOR_NV61		= MDP_COLOR(0, 0, 0, 0, 2, 1, 0,  8, 1,  1, 13),
+	MDP_COLOR_NV24		= MDP_COLOR(0, 0, 0, 0, 2, 0, 0,  8, 1,  0, 14),
+	MDP_COLOR_NV42		= MDP_COLOR(0, 0, 0, 0, 2, 0, 0,  8, 1,  1, 14),
 
 	/* MediaTek proprietary formats */
 	/* UFO encoded block mode */
-	MDP_COLOR_420_BLK_UFO	= MDP_COLOR(0, 0, 5, 2, 1, 1, 256, 1, 0, 12),
+	MDP_COLOR_420_BLK_UFO	= MDP_COLOR(0, 0, 0, 5, 2, 1, 1, 256, 1, 0, 12),
 	/* Block mode */
-	MDP_COLOR_420_BLK	= MDP_COLOR(0, 0, 1, 2, 1, 1, 256, 1, 0, 12),
+	MDP_COLOR_420_BLK	= MDP_COLOR(0, 0, 0, 1, 2, 1, 1, 256, 1, 0, 12),
 	/* Block mode + field mode */
-	MDP_COLOR_420_BLKI	= MDP_COLOR(0, 0, 3, 2, 1, 1, 256, 1, 0, 12),
+	MDP_COLOR_420_BLKI	= MDP_COLOR(0, 0, 0, 3, 2, 1, 1, 256, 1, 0, 12),
 	/* Block mode */
-	MDP_COLOR_422_BLK	= MDP_COLOR(0, 0, 1, 1, 1, 0, 512, 1, 0, 4),
+	MDP_COLOR_422_BLK	= MDP_COLOR(0, 0, 0, 1, 1, 1, 0, 512, 1, 0, 4),
 
-	MDP_COLOR_IYU2		= MDP_COLOR(0, 0, 0, 1, 0, 0, 24,  1, 0, 25),
-	MDP_COLOR_YUV444	= MDP_COLOR(0, 0, 0, 1, 0, 0, 24,  1, 0, 30),
+	MDP_COLOR_IYU2		= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 24,  1, 0, 25),
+	MDP_COLOR_YUV444	= MDP_COLOR(0, 0, 0, 0, 1, 0, 0, 24,  1, 0, 30),
 
 	/* Packed 10-bit formats */
-	MDP_COLOR_RGBA1010102	= MDP_COLOR(1, 0, 0, 1, 0, 0, 32,  0, 1, 2),
-	MDP_COLOR_BGRA1010102	= MDP_COLOR(1, 0, 0, 1, 0, 0, 32,  0, 0, 2),
+	MDP_COLOR_RGBA1010102	= MDP_COLOR(0, 1, 0, 0, 1, 0, 0, 32,  0, 1, 2),
+	MDP_COLOR_BGRA1010102	= MDP_COLOR(0, 1, 0, 0, 1, 0, 0, 32,  0, 0, 2),
 	/* Packed 10-bit UYVY */
-	MDP_COLOR_UYVY_10P	= MDP_COLOR(1, 0, 0, 1, 1, 0, 20,  1, 0, 4),
+	MDP_COLOR_UYVY_10P	= MDP_COLOR(0, 1, 0, 0, 1, 1, 0, 20,  1, 0, 4),
 	/* Packed 10-bit NV21 */
-	MDP_COLOR_NV21_10P	= MDP_COLOR(1, 0, 0, 2, 1, 1, 10,  1, 1, 12),
+	MDP_COLOR_NV21_10P	= MDP_COLOR(0, 1, 0, 0, 2, 1, 1, 10,  1, 1, 12),
 	/* 10-bit block mode */
-	MDP_COLOR_420_BLK_10_H	= MDP_COLOR(1, 0, 1, 2, 1, 1, 320, 1, 0, 12),
+	MDP_COLOR_420_BLK_10_H	= MDP_COLOR(0, 1, 0, 1, 2, 1, 1, 320, 1, 0, 12),
 	/* 10-bit HEVC tile mode */
-	MDP_COLOR_420_BLK_10_V	= MDP_COLOR(1, 1, 1, 2, 1, 1, 320, 1, 0, 12),
+	MDP_COLOR_420_BLK_10_V	= MDP_COLOR(0, 1, 1, 1, 2, 1, 1, 320, 1, 0, 12),
 	/* UFO encoded 10-bit block mode */
-	MDP_COLOR_420_BLK_U10_H	= MDP_COLOR(1, 0, 5, 2, 1, 1, 320, 1, 0, 12),
+	MDP_COLOR_420_BLK_U10_H	= MDP_COLOR(0, 1, 0, 5, 2, 1, 1, 320, 1, 0, 12),
 	/* UFO encoded 10-bit HEVC tile mode */
-	MDP_COLOR_420_BLK_U10_V	= MDP_COLOR(1, 1, 5, 2, 1, 1, 320, 1, 0, 12),
+	MDP_COLOR_420_BLK_U10_V	= MDP_COLOR(0, 1, 1, 5, 2, 1, 1, 320, 1, 0, 12),
 
 	/* Loose 10-bit formats */
-	MDP_COLOR_UYVY_10L	= MDP_COLOR(0, 1, 0, 1, 1, 0, 20,  1, 0, 4),
-	MDP_COLOR_VYUY_10L	= MDP_COLOR(0, 1, 0, 1, 1, 0, 20,  1, 1, 4),
-	MDP_COLOR_YUYV_10L	= MDP_COLOR(0, 1, 0, 1, 1, 0, 20,  1, 0, 5),
-	MDP_COLOR_YVYU_10L	= MDP_COLOR(0, 1, 0, 1, 1, 0, 20,  1, 1, 5),
-	MDP_COLOR_NV12_10L	= MDP_COLOR(0, 1, 0, 2, 1, 1, 10,  1, 0, 12),
-	MDP_COLOR_NV21_10L	= MDP_COLOR(0, 1, 0, 2, 1, 1, 10,  1, 1, 12),
-	MDP_COLOR_NV16_10L	= MDP_COLOR(0, 1, 0, 2, 1, 0, 10,  1, 0, 13),
-	MDP_COLOR_NV61_10L	= MDP_COLOR(0, 1, 0, 2, 1, 0, 10,  1, 1, 13),
-	MDP_COLOR_YV12_10L	= MDP_COLOR(0, 1, 0, 3, 1, 1, 10,  1, 1, 8),
-	MDP_COLOR_I420_10L	= MDP_COLOR(0, 1, 0, 3, 1, 1, 10,  1, 0, 8),
+	MDP_COLOR_UYVY_10L	= MDP_COLOR(0, 0, 1, 0, 1, 1, 0, 20,  1, 0, 4),
+	MDP_COLOR_VYUY_10L	= MDP_COLOR(0, 0, 1, 0, 1, 1, 0, 20,  1, 1, 4),
+	MDP_COLOR_YUYV_10L	= MDP_COLOR(0, 0, 1, 0, 1, 1, 0, 20,  1, 0, 5),
+	MDP_COLOR_YVYU_10L	= MDP_COLOR(0, 0, 1, 0, 1, 1, 0, 20,  1, 1, 5),
+	MDP_COLOR_NV12_10L	= MDP_COLOR(0, 0, 1, 0, 2, 1, 1, 10,  1, 0, 12),
+	MDP_COLOR_NV21_10L	= MDP_COLOR(0, 0, 1, 0, 2, 1, 1, 10,  1, 1, 12),
+	MDP_COLOR_NV16_10L	= MDP_COLOR(0, 0, 1, 0, 2, 1, 0, 10,  1, 0, 13),
+	MDP_COLOR_NV61_10L	= MDP_COLOR(0, 0, 1, 0, 2, 1, 0, 10,  1, 1, 13),
+	MDP_COLOR_YV12_10L	= MDP_COLOR(0, 0, 1, 0, 3, 1, 1, 10,  1, 1, 8),
+	MDP_COLOR_I420_10L	= MDP_COLOR(0, 0, 1, 0, 3, 1, 1, 10,  1, 0, 8),
 };
 
 static inline bool MDP_COLOR_IS_UV_COPLANE(enum mdp_color c)
@@ -353,8 +354,11 @@ struct mdp_frameparam {
 	enum v4l2_quantization		quant;
 };
 
-int mdp_enum_fmt_mplane(struct v4l2_fmtdesc *f);
-const struct mdp_format *mdp_try_fmt_mplane(struct v4l2_format *f,
+struct mdp_dev;
+
+int mdp_enum_fmt_mplane(struct mdp_dev *mdp, struct v4l2_fmtdesc *f);
+const struct mdp_format *mdp_try_fmt_mplane(struct mdp_dev *mdp,
+					    struct v4l2_format *f,
 					    struct mdp_frameparam *param,
 					    u32 ctx_id);
 enum mdp_ycbcr_profile mdp_map_ycbcr_prof_mplane(struct v4l2_format *f,
@@ -368,6 +372,6 @@ void mdp_set_src_config(struct img_input *in,
 			struct mdp_frame *frame, struct vb2_buffer *vb);
 void mdp_set_dst_config(struct img_output *out,
 			struct mdp_frame *frame, struct vb2_buffer *vb);
-int mdp_frameparam_init(struct mdp_frameparam *param);
+int mdp_frameparam_init(struct mdp_dev *mdp, struct mdp_frameparam *param);
 
 #endif  /* __MTK_MDP3_REGS_H__ */
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-type.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-type.h
new file mode 100644
index 000000000000..ae0396806152
--- /dev/null
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-type.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2023 MediaTek Inc.
+ * Author: Ping-Hsun Wu <ping-hsun.wu@mediatek.com>
+ */
+
+#ifndef __MTK_MDP3_TYPE_H__
+#define __MTK_MDP3_TYPE_H__
+
+#include <linux/types.h>
+
+#define IMG_MAX_HW_INPUTS	3
+#define IMG_MAX_HW_OUTPUTS	4
+#define IMG_MAX_PLANES		3
+#define IMG_MAX_COMPONENTS	20
+
+struct img_crop {
+	s32 left;
+	s32 top;
+	u32 width;
+	u32 height;
+	u32 left_subpix;
+	u32 top_subpix;
+	u32 width_subpix;
+	u32 height_subpix;
+} __packed;
+
+struct img_region {
+	s32 left;
+	s32 right;
+	s32 top;
+	s32 bottom;
+} __packed;
+
+struct img_offset {
+	s32 left;
+	s32 top;
+	u32 left_subpix;
+	u32 top_subpix;
+} __packed;
+
+struct img_mux {
+	u32 reg;
+	u32 value;
+	u32 subsys_id;
+} __packed;
+
+struct img_mmsys_ctrl {
+	struct img_mux sets[IMG_MAX_COMPONENTS * 2];
+	u32 num_sets;
+} __packed;
+
+#endif  /* __MTK_MDP3_TYPE_H__ */
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.c
index a72bed927bb6..49fc2e9d45dd 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.c
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.c
@@ -10,7 +10,6 @@
 #include "mtk-mdp3-core.h"
 
 #define MDP_VPU_MESSAGE_TIMEOUT 500U
-#define vpu_alloc_size		0x600000
 
 static inline struct mdp_dev *vpu_to_mdp(struct mdp_vpu_dev *vpu)
 {
@@ -19,23 +18,63 @@ static inline struct mdp_dev *vpu_to_mdp(struct mdp_vpu_dev *vpu)
 
 static int mdp_vpu_shared_mem_alloc(struct mdp_vpu_dev *vpu)
 {
-	if (vpu->work && vpu->work_addr)
-		return 0;
+	struct device *dev;
 
-	vpu->work = dma_alloc_coherent(scp_get_device(vpu->scp), vpu_alloc_size,
-				       &vpu->work_addr, GFP_KERNEL);
+	if (IS_ERR_OR_NULL(vpu))
+		goto err_return;
 
-	if (!vpu->work)
-		return -ENOMEM;
-	else
-		return 0;
+	dev = scp_get_device(vpu->scp);
+
+	if (!vpu->param) {
+		vpu->param = dma_alloc_wc(dev, vpu->param_size,
+					  &vpu->param_addr, GFP_KERNEL);
+		if (!vpu->param)
+			goto err_return;
+	}
+
+	if (!vpu->work) {
+		vpu->work = dma_alloc_wc(dev, vpu->work_size,
+					 &vpu->work_addr, GFP_KERNEL);
+		if (!vpu->work)
+			goto err_free_param;
+	}
+
+	if (!vpu->config) {
+		vpu->config = dma_alloc_wc(dev, vpu->config_size,
+					   &vpu->config_addr, GFP_KERNEL);
+		if (!vpu->config)
+			goto err_free_work;
+	}
+
+	return 0;
+
+err_free_work:
+	dma_free_wc(dev, vpu->work_size, vpu->work, vpu->work_addr);
+	vpu->work = NULL;
+err_free_param:
+	dma_free_wc(dev, vpu->param_size, vpu->param, vpu->param_addr);
+	vpu->param = NULL;
+err_return:
+	return -ENOMEM;
 }
 
 void mdp_vpu_shared_mem_free(struct mdp_vpu_dev *vpu)
 {
+	struct device *dev;
+
+	if (IS_ERR_OR_NULL(vpu))
+		return;
+
+	dev = scp_get_device(vpu->scp);
+
+	if (vpu->param && vpu->param_addr)
+		dma_free_wc(dev, vpu->param_size, vpu->param, vpu->param_addr);
+
 	if (vpu->work && vpu->work_addr)
-		dma_free_coherent(scp_get_device(vpu->scp), vpu_alloc_size,
-				  vpu->work, vpu->work_addr);
+		dma_free_wc(dev, vpu->work_size, vpu->work, vpu->work_addr);
+
+	if (vpu->config && vpu->config_addr)
+		dma_free_wc(dev, vpu->config_size, vpu->config, vpu->config_addr);
 }
 
 static void mdp_vpu_ipi_handle_init_ack(void *data, unsigned int len,
@@ -69,16 +108,16 @@ static void mdp_vpu_ipi_handle_frame_ack(void *data, unsigned int len,
 	struct img_sw_addr *addr = (struct img_sw_addr *)data;
 	struct img_ipi_frameparam *param =
 		(struct img_ipi_frameparam *)(unsigned long)addr->va;
-	struct mdp_vpu_ctx *ctx =
-		(struct mdp_vpu_ctx *)(unsigned long)param->drv_data;
+	struct mdp_vpu_dev *vpu =
+		(struct mdp_vpu_dev *)(unsigned long)param->drv_data;
 
 	if (param->state) {
-		struct mdp_dev *mdp = vpu_to_mdp(ctx->vpu_dev);
+		struct mdp_dev *mdp = vpu_to_mdp(vpu);
 
 		dev_err(&mdp->pdev->dev, "VPU MDP failure:%d\n", param->state);
 	}
-	ctx->vpu_dev->status = param->state;
-	complete(&ctx->vpu_dev->ipi_acked);
+	vpu->status = param->state;
+	complete(&vpu->ipi_acked);
 }
 
 int mdp_vpu_register(struct mdp_dev *mdp)
@@ -157,9 +196,6 @@ int mdp_vpu_dev_init(struct mdp_vpu_dev *vpu, struct mtk_scp *scp,
 	struct mdp_ipi_init_msg msg = {
 		.drv_data = (unsigned long)vpu,
 	};
-	size_t mem_size;
-	phys_addr_t pool;
-	const size_t pool_size = sizeof(struct mdp_config_pool);
 	struct mdp_dev *mdp = vpu_to_mdp(vpu);
 	int err;
 
@@ -172,34 +208,29 @@ int mdp_vpu_dev_init(struct mdp_vpu_dev *vpu, struct mtk_scp *scp,
 		goto err_work_size;
 	/* vpu work_size was set in mdp_vpu_ipi_handle_init_ack */
 
-	mem_size = vpu_alloc_size;
+	mutex_lock(vpu->lock);
+	vpu->work_size = ALIGN(vpu->work_size, 64);
+	vpu->param_size = ALIGN(sizeof(struct img_ipi_frameparam), 64);
+	vpu->config_size = ALIGN(sizeof(struct img_config), 64);
 	err = mdp_vpu_shared_mem_alloc(vpu);
+	mutex_unlock(vpu->lock);
 	if (err) {
 		dev_err(&mdp->pdev->dev, "VPU memory alloc fail!");
 		goto err_mem_alloc;
 	}
 
-	pool = ALIGN((uintptr_t)vpu->work + vpu->work_size, 8);
-	if (pool + pool_size - (uintptr_t)vpu->work > mem_size) {
-		dev_err(&mdp->pdev->dev,
-			"VPU memory insufficient: %zx + %zx > %zx",
-			vpu->work_size, pool_size, mem_size);
-		err = -ENOMEM;
-		goto err_mem_size;
-	}
-
 	dev_dbg(&mdp->pdev->dev,
-		"VPU work:%pK pa:%pad sz:%zx pool:%pa sz:%zx (mem sz:%zx)",
+		"VPU param:%pK pa:%pad sz:%zx, work:%pK pa:%pad sz:%zx, config:%pK pa:%pad sz:%zx",
+		vpu->param, &vpu->param_addr, vpu->param_size,
 		vpu->work, &vpu->work_addr, vpu->work_size,
-		&pool, pool_size, mem_size);
-	vpu->pool = (struct mdp_config_pool *)(uintptr_t)pool;
+		vpu->config, &vpu->config_addr, vpu->config_size);
+
 	msg.work_addr = vpu->work_addr;
 	msg.work_size = vpu->work_size;
 	err = mdp_vpu_sendmsg(vpu, SCP_IPI_MDP_INIT, &msg, sizeof(msg));
 	if (err)
 		goto err_work_size;
 
-	memset(vpu->pool, 0, sizeof(*vpu->pool));
 	return 0;
 
 err_work_size:
@@ -212,7 +243,6 @@ err_work_size:
 		break;
 	}
 	return err;
-err_mem_size:
 err_mem_alloc:
 	return err;
 }
@@ -227,88 +257,31 @@ int mdp_vpu_dev_deinit(struct mdp_vpu_dev *vpu)
 	return mdp_vpu_sendmsg(vpu, SCP_IPI_MDP_DEINIT, &msg, sizeof(msg));
 }
 
-static struct img_config *mdp_config_get(struct mdp_vpu_dev *vpu,
-					 enum mdp_config_id id, uint32_t *addr)
+int mdp_vpu_process(struct mdp_vpu_dev *vpu, struct img_ipi_frameparam *param)
 {
-	struct img_config *config;
-
-	if (id < 0 || id >= MDP_CONFIG_POOL_SIZE)
-		return ERR_PTR(-EINVAL);
+	struct mdp_dev *mdp = vpu_to_mdp(vpu);
+	struct img_sw_addr addr;
 
 	mutex_lock(vpu->lock);
-	vpu->pool->cfg_count[id]++;
-	config = &vpu->pool->configs[id];
-	*addr = vpu->work_addr + ((uintptr_t)config - (uintptr_t)vpu->work);
-	mutex_unlock(vpu->lock);
-
-	return config;
-}
-
-static int mdp_config_put(struct mdp_vpu_dev *vpu,
-			  enum mdp_config_id id,
-			  const struct img_config *config)
-{
-	int err = 0;
-
-	if (id < 0 || id >= MDP_CONFIG_POOL_SIZE)
-		return -EINVAL;
-	if (vpu->lock)
-		mutex_lock(vpu->lock);
-	if (!vpu->pool->cfg_count[id] || config != &vpu->pool->configs[id])
-		err = -EINVAL;
-	else
-		vpu->pool->cfg_count[id]--;
-	if (vpu->lock)
+	if (mdp_vpu_shared_mem_alloc(vpu)) {
+		dev_err(&mdp->pdev->dev, "VPU memory alloc fail!");
 		mutex_unlock(vpu->lock);
-	return err;
-}
-
-int mdp_vpu_ctx_init(struct mdp_vpu_ctx *ctx, struct mdp_vpu_dev *vpu,
-		     enum mdp_config_id id)
-{
-	ctx->config = mdp_config_get(vpu, id, &ctx->inst_addr);
-	if (IS_ERR(ctx->config)) {
-		int err = PTR_ERR(ctx->config);
-
-		ctx->config = NULL;
-		return err;
+		return -ENOMEM;
 	}
-	ctx->config_id = id;
-	ctx->vpu_dev = vpu;
-	return 0;
-}
 
-int mdp_vpu_ctx_deinit(struct mdp_vpu_ctx *ctx)
-{
-	int err = mdp_config_put(ctx->vpu_dev, ctx->config_id, ctx->config);
+	memset(vpu->param, 0, vpu->param_size);
+	memset(vpu->work, 0, vpu->work_size);
+	memset(vpu->config, 0, vpu->config_size);
 
-	ctx->config_id = 0;
-	ctx->config = NULL;
-	ctx->inst_addr = 0;
-	return err;
-}
+	param->self_data.va = (unsigned long)vpu->work;
+	param->self_data.pa = vpu->work_addr;
+	param->config_data.va = (unsigned long)vpu->config;
+	param->config_data.pa = vpu->config_addr;
+	param->drv_data = (unsigned long)vpu;
+	memcpy(vpu->param, param, sizeof(*param));
 
-int mdp_vpu_process(struct mdp_vpu_ctx *ctx, struct img_ipi_frameparam *param)
-{
-	struct mdp_vpu_dev *vpu = ctx->vpu_dev;
-	struct mdp_dev *mdp = vpu_to_mdp(vpu);
-	struct img_sw_addr addr;
-
-	if (!ctx->vpu_dev->work || !ctx->vpu_dev->work_addr) {
-		if (mdp_vpu_shared_mem_alloc(vpu)) {
-			dev_err(&mdp->pdev->dev, "VPU memory alloc fail!");
-			return -ENOMEM;
-		}
-	}
-	memset((void *)ctx->vpu_dev->work, 0, ctx->vpu_dev->work_size);
-	memset(ctx->config, 0, sizeof(*ctx->config));
-	param->config_data.va = (unsigned long)ctx->config;
-	param->config_data.pa = ctx->inst_addr;
-	param->drv_data = (unsigned long)ctx;
-
-	memcpy((void *)ctx->vpu_dev->work, param, sizeof(*param));
-	addr.pa = ctx->vpu_dev->work_addr;
-	addr.va = (uintptr_t)ctx->vpu_dev->work;
-	return mdp_vpu_sendmsg(ctx->vpu_dev, SCP_IPI_MDP_FRAME,
-		&addr, sizeof(addr));
+	addr.pa = vpu->param_addr;
+	addr.va = (unsigned long)vpu->param;
+	mutex_unlock(vpu->lock);
+	return mdp_vpu_sendmsg(vpu, SCP_IPI_MDP_FRAME, &addr, sizeof(addr));
 }
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.h
index 244b3a32d689..ad3551bc0730 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.h
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.h
@@ -37,42 +37,27 @@ struct mdp_ipi_deinit_msg {
 	u32	work_addr;
 } __packed;
 
-enum mdp_config_id {
-	MDP_DEV_M2M = 0,
-	MDP_CONFIG_POOL_SIZE	/* ALWAYS keep at the end */
-};
-
-struct mdp_config_pool {
-	u64			cfg_count[MDP_CONFIG_POOL_SIZE];
-	struct img_config	configs[MDP_CONFIG_POOL_SIZE];
-};
-
 struct mdp_vpu_dev {
 	/* synchronization protect for accessing vpu working buffer info */
 	struct mutex		*lock;
 	struct mtk_scp		*scp;
 	struct completion	ipi_acked;
+	void			*param;
+	dma_addr_t		param_addr;
+	size_t			param_size;
 	void			*work;
 	dma_addr_t		work_addr;
 	size_t			work_size;
-	struct mdp_config_pool	*pool;
+	void			*config;
+	dma_addr_t		config_addr;
+	size_t			config_size;
 	u32			status;
 };
 
-struct mdp_vpu_ctx {
-	struct mdp_vpu_dev	*vpu_dev;
-	u32			config_id;
-	struct img_config	*config;
-	u32			inst_addr;
-};
-
 void mdp_vpu_shared_mem_free(struct mdp_vpu_dev *vpu);
 int mdp_vpu_dev_init(struct mdp_vpu_dev *vpu, struct mtk_scp *scp,
 		     struct mutex *lock /* for sync */);
 int mdp_vpu_dev_deinit(struct mdp_vpu_dev *vpu);
-int mdp_vpu_ctx_init(struct mdp_vpu_ctx *ctx, struct mdp_vpu_dev *vpu,
-		     enum mdp_config_id id);
-int mdp_vpu_ctx_deinit(struct mdp_vpu_ctx *ctx);
-int mdp_vpu_process(struct mdp_vpu_ctx *vpu, struct img_ipi_frameparam *param);
+int mdp_vpu_process(struct mdp_vpu_dev *vpu, struct img_ipi_frameparam *param);
 
 #endif  /* __MTK_MDP3_VPU_H__ */
diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec.c
index 641f533c417f..93fcea821001 100644
--- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec.c
+++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec.c
@@ -39,10 +39,9 @@ static bool mtk_vdec_get_cap_fmt(struct mtk_vcodec_ctx *ctx, int format_index)
 {
 	const struct mtk_vcodec_dec_pdata *dec_pdata = ctx->dev->vdec_pdata;
 	const struct mtk_video_fmt *fmt;
-	struct mtk_q_data *q_data;
 	int num_frame_count = 0, i;
-	bool ret = true;
 
+	fmt = &dec_pdata->vdec_formats[format_index];
 	for (i = 0; i < *dec_pdata->num_formats; i++) {
 		if (dec_pdata->vdec_formats[i].type != MTK_FMT_FRAME)
 			continue;
@@ -50,27 +49,10 @@ static bool mtk_vdec_get_cap_fmt(struct mtk_vcodec_ctx *ctx, int format_index)
 		num_frame_count++;
 	}
 
-	if (num_frame_count == 1)
+	if (num_frame_count == 1 || fmt->fourcc == V4L2_PIX_FMT_MM21)
 		return true;
 
-	fmt = &dec_pdata->vdec_formats[format_index];
-	q_data = &ctx->q_data[MTK_Q_DATA_SRC];
-	switch (q_data->fmt->fourcc) {
-	case V4L2_PIX_FMT_VP8_FRAME:
-		if (fmt->fourcc == V4L2_PIX_FMT_MM21)
-			ret = true;
-		break;
-	case V4L2_PIX_FMT_H264_SLICE:
-	case V4L2_PIX_FMT_VP9_FRAME:
-		if (fmt->fourcc == V4L2_PIX_FMT_MM21)
-			ret = false;
-		break;
-	default:
-		ret = true;
-		break;
-	}
-
-	return ret;
+	return false;
 }
 
 static struct mtk_q_data *mtk_vdec_get_q_data(struct mtk_vcodec_ctx *ctx,
@@ -753,6 +735,13 @@ int vb2ops_vdec_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers,
 	}
 
 	if (*nplanes) {
+		if (vq->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
+			if (*nplanes != q_data->fmt->num_planes)
+				return -EINVAL;
+		} else {
+			if (*nplanes != 1)
+				return -EINVAL;
+		}
 		for (i = 0; i < *nplanes; i++) {
 			if (sizes[i] < q_data->sizeimage[i])
 				return -EINVAL;
diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_drv.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_drv.c
index 174a6eec2f54..9ba5dc5df648 100644
--- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_drv.c
+++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_drv.c
@@ -321,7 +321,7 @@ static int mtk_vcodec_probe(struct platform_device *pdev)
 		}
 	}
 
-	if (of_get_property(pdev->dev.of_node, "dma-ranges", NULL)) {
+	if (of_property_present(pdev->dev.of_node, "dma-ranges")) {
 		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(34));
 		if (ret) {
 			mtk_v4l2_err("Failed to set mask");
@@ -451,7 +451,8 @@ err_core_workq:
 	if (IS_VDEC_LAT_ARCH(dev->vdec_pdata->hw_arch))
 		destroy_workqueue(dev->core_workqueue);
 err_res:
-	pm_runtime_disable(dev->pm.dev);
+	if (!dev->vdec_pdata->is_subdev_supported)
+		pm_runtime_disable(dev->pm.dev);
 err_dec_pm:
 	mtk_vcodec_fw_release(dev->fw_handler);
 	return ret;
@@ -487,7 +488,7 @@ static const struct of_device_id mtk_vcodec_match[] = {
 
 MODULE_DEVICE_TABLE(of, mtk_vcodec_match);
 
-static int mtk_vcodec_dec_remove(struct platform_device *pdev)
+static void mtk_vcodec_dec_remove(struct platform_device *pdev)
 {
 	struct mtk_vcodec_dev *dev = platform_get_drvdata(pdev);
 
@@ -509,12 +510,11 @@ static int mtk_vcodec_dec_remove(struct platform_device *pdev)
 	if (!dev->vdec_pdata->is_subdev_supported)
 		pm_runtime_disable(dev->pm.dev);
 	mtk_vcodec_fw_release(dev->fw_handler);
-	return 0;
 }
 
 static struct platform_driver mtk_vcodec_dec_driver = {
 	.probe	= mtk_vcodec_probe,
-	.remove	= mtk_vcodec_dec_remove,
+	.remove_new = mtk_vcodec_dec_remove,
 	.driver	= {
 		.name	= MTK_VCODEC_DEC_NAME,
 		.of_match_table = mtk_vcodec_match,
diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_hw.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_hw.c
index 376db0e433d7..b753bf54ebd9 100644
--- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_hw.c
+++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_hw.c
@@ -193,8 +193,16 @@ err:
 	return ret;
 }
 
+static int mtk_vdec_hw_remove(struct platform_device *pdev)
+{
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
 static struct platform_driver mtk_vdec_driver = {
 	.probe	= mtk_vdec_hw_probe,
+	.remove = mtk_vdec_hw_remove,
 	.driver	= {
 		.name	= "mtk-vdec-comp",
 		.of_match_table = mtk_vdec_hw_match,
diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateful.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateful.c
index 035c86e7809f..29991551cf61 100644
--- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateful.c
+++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateful.c
@@ -11,7 +11,7 @@
 #include "mtk_vcodec_dec_pm.h"
 #include "vdec_drv_if.h"
 
-static const struct mtk_video_fmt mtk_video_formats[] = {
+static struct mtk_video_fmt mtk_video_formats[] = {
 	{
 		.fourcc = V4L2_PIX_FMT_H264,
 		.type = MTK_FMT_DEC,
@@ -580,6 +580,16 @@ static int mtk_vcodec_dec_ctrls_setup(struct mtk_vcodec_ctx *ctx)
 
 static void mtk_init_vdec_params(struct mtk_vcodec_ctx *ctx)
 {
+	unsigned int i;
+
+	if (!(ctx->dev->dec_capability & VCODEC_CAPABILITY_4K_DISABLED)) {
+		for (i = 0; i < num_supported_formats; i++) {
+			mtk_video_formats[i].frmsize.max_width =
+				VCODEC_DEC_4K_CODED_WIDTH;
+			mtk_video_formats[i].frmsize.max_height =
+				VCODEC_DEC_4K_CODED_HEIGHT;
+		}
+	}
 }
 
 static struct vb2_ops mtk_vdec_frame_vb2_ops = {
diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateless.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateless.c
index ffbcee04dc26..3000db975e5f 100644
--- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateless.c
+++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateless.c
@@ -258,8 +258,10 @@ static void mtk_vdec_worker(struct work_struct *work)
 		if (src_buf_req)
 			v4l2_ctrl_request_complete(src_buf_req, &ctx->ctrl_hdl);
 	} else {
-		v4l2_m2m_src_buf_remove(ctx->m2m_ctx);
-		v4l2_m2m_buf_done(vb2_v4l2_src, state);
+		if (ret != -EAGAIN) {
+			v4l2_m2m_src_buf_remove(ctx->m2m_ctx);
+			v4l2_m2m_buf_done(vb2_v4l2_src, state);
+		}
 		v4l2_m2m_job_finish(dev->m2m_dev_dec, ctx->m2m_ctx);
 	}
 }
@@ -390,14 +392,14 @@ static void mtk_vcodec_get_supported_formats(struct mtk_vcodec_ctx *ctx)
 	if (num_formats)
 		return;
 
-	if (ctx->dev->dec_capability & MTK_VDEC_FORMAT_MM21) {
-		mtk_vcodec_add_formats(V4L2_PIX_FMT_MM21, ctx);
-		cap_format_count++;
-	}
 	if (ctx->dev->dec_capability & MTK_VDEC_FORMAT_MT21C) {
 		mtk_vcodec_add_formats(V4L2_PIX_FMT_MT21C, ctx);
 		cap_format_count++;
 	}
+	if (ctx->dev->dec_capability & MTK_VDEC_FORMAT_MM21) {
+		mtk_vcodec_add_formats(V4L2_PIX_FMT_MM21, ctx);
+		cap_format_count++;
+	}
 	if (ctx->dev->dec_capability & MTK_VDEC_FORMAT_H264_SLICE) {
 		mtk_vcodec_add_formats(V4L2_PIX_FMT_H264_SLICE, ctx);
 		out_format_count++;
diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c
index d65800a3b89d..db65e77bd373 100644
--- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c
+++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c
@@ -943,7 +943,7 @@ err_start_stream:
 		 * FIXME: This check is not needed as only active buffers
 		 * can be marked as done.
 		 */
-		if (buf->state == VB2_BUF_STATE_ACTIVE) {
+		if (buf && buf->state == VB2_BUF_STATE_ACTIVE) {
 			mtk_v4l2_debug(0, "[%d] id=%d, type=%d, %d -> VB2_BUF_STATE_QUEUED",
 					ctx->id, i, q->type,
 					(int)buf->state);
diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c
index 9095186d5495..755f567b9e54 100644
--- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c
+++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c
@@ -89,16 +89,24 @@ static irqreturn_t mtk_vcodec_enc_irq_handler(int irq, void *priv)
 	struct mtk_vcodec_ctx *ctx;
 	unsigned long flags;
 	void __iomem *addr;
+	int core_id;
 
 	spin_lock_irqsave(&dev->irqlock, flags);
 	ctx = dev->curr_ctx;
 	spin_unlock_irqrestore(&dev->irqlock, flags);
 
-	mtk_v4l2_debug(1, "id=%d coreid:%d", ctx->id, dev->venc_pdata->core_id);
-	addr = dev->reg_base[dev->venc_pdata->core_id] +
-				MTK_VENC_IRQ_ACK_OFFSET;
+	core_id = dev->venc_pdata->core_id;
+	if (core_id < 0 || core_id >= NUM_MAX_VCODEC_REG_BASE) {
+		mtk_v4l2_err("Invalid core id: %d, ctx id: %d",
+			     core_id, ctx->id);
+		return IRQ_HANDLED;
+	}
+
+	mtk_v4l2_debug(1, "id: %d, core id: %d", ctx->id, core_id);
 
-	ctx->irq_status = readl(dev->reg_base[dev->venc_pdata->core_id] +
+	addr = dev->reg_base[core_id] + MTK_VENC_IRQ_ACK_OFFSET;
+
+	ctx->irq_status = readl(dev->reg_base[core_id] +
 				(MTK_VENC_IRQ_STATUS_OFFSET));
 
 	clean_irq_status(ctx->irq_status, addr);
@@ -344,7 +352,7 @@ static int mtk_vcodec_probe(struct platform_device *pdev)
 		goto err_event_workq;
 	}
 
-	if (of_get_property(pdev->dev.of_node, "dma-ranges", NULL))
+	if (of_property_present(pdev->dev.of_node, "dma-ranges"))
 		dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(34));
 
 	ret = video_register_device(vfd_enc, VFL_TYPE_VIDEO, -1);
@@ -451,7 +459,7 @@ static const struct of_device_id mtk_vcodec_enc_match[] = {
 };
 MODULE_DEVICE_TABLE(of, mtk_vcodec_enc_match);
 
-static int mtk_vcodec_enc_remove(struct platform_device *pdev)
+static void mtk_vcodec_enc_remove(struct platform_device *pdev)
 {
 	struct mtk_vcodec_dev *dev = platform_get_drvdata(pdev);
 
@@ -466,12 +474,11 @@ static int mtk_vcodec_enc_remove(struct platform_device *pdev)
 	v4l2_device_unregister(&dev->v4l2_dev);
 	pm_runtime_disable(dev->pm.dev);
 	mtk_vcodec_fw_release(dev->fw_handler);
-	return 0;
 }
 
 static struct platform_driver mtk_vcodec_enc_driver = {
 	.probe	= mtk_vcodec_probe,
-	.remove	= mtk_vcodec_enc_remove,
+	.remove_new = mtk_vcodec_enc_remove,
 	.driver	= {
 		.name	= MTK_VCODEC_ENC_NAME,
 		.of_match_table = mtk_vcodec_enc_match,
diff --git a/drivers/media/platform/mediatek/vcodec/vdec/vdec_h264_req_multi_if.c b/drivers/media/platform/mediatek/vcodec/vdec/vdec_h264_req_multi_if.c
index 955b2d0c8f53..999ce7ee5fdc 100644
--- a/drivers/media/platform/mediatek/vcodec/vdec/vdec_h264_req_multi_if.c
+++ b/drivers/media/platform/mediatek/vcodec/vdec/vdec_h264_req_multi_if.c
@@ -597,7 +597,7 @@ static int vdec_h264_slice_lat_decode(void *h_vdec, struct mtk_vcodec_mem *bs,
 	lat_buf = vdec_msg_queue_dqbuf(&inst->ctx->msg_queue.lat_ctx);
 	if (!lat_buf) {
 		mtk_vcodec_err(inst, "failed to get lat buffer");
-		return -EINVAL;
+		return -EAGAIN;
 	}
 	share_info = lat_buf->private_data;
 	src_buf_info = container_of(bs, struct mtk_video_dec_buf, bs_buffer);
diff --git a/drivers/media/platform/mediatek/vcodec/vdec/vdec_vp9_req_lat_if.c b/drivers/media/platform/mediatek/vcodec/vdec/vdec_vp9_req_lat_if.c
index cbb6728b8a40..cf16cf2807f0 100644
--- a/drivers/media/platform/mediatek/vcodec/vdec/vdec_vp9_req_lat_if.c
+++ b/drivers/media/platform/mediatek/vcodec/vdec/vdec_vp9_req_lat_if.c
@@ -2070,7 +2070,7 @@ static int vdec_vp9_slice_lat_decode(void *h_vdec, struct mtk_vcodec_mem *bs,
 	lat_buf = vdec_msg_queue_dqbuf(&instance->ctx->msg_queue.lat_ctx);
 	if (!lat_buf) {
 		mtk_vcodec_err(instance, "Failed to get VP9 lat buf\n");
-		return -EBUSY;
+		return -EAGAIN;
 	}
 	pfc = (struct vdec_vp9_slice_pfc *)lat_buf->private_data;
 	if (!pfc) {
diff --git a/drivers/media/platform/mediatek/vcodec/vdec_msg_queue.c b/drivers/media/platform/mediatek/vcodec/vdec_msg_queue.c
index dc2004790a47..f3073d1e7f42 100644
--- a/drivers/media/platform/mediatek/vcodec/vdec_msg_queue.c
+++ b/drivers/media/platform/mediatek/vcodec/vdec_msg_queue.c
@@ -52,9 +52,26 @@ static struct list_head *vdec_get_buf_list(int hardware_index, struct vdec_lat_b
 	}
 }
 
+static void vdec_msg_queue_inc(struct vdec_msg_queue *msg_queue, int hardware_index)
+{
+	if (hardware_index == MTK_VDEC_CORE)
+		atomic_inc(&msg_queue->core_list_cnt);
+	else
+		atomic_inc(&msg_queue->lat_list_cnt);
+}
+
+static void vdec_msg_queue_dec(struct vdec_msg_queue *msg_queue, int hardware_index)
+{
+	if (hardware_index == MTK_VDEC_CORE)
+		atomic_dec(&msg_queue->core_list_cnt);
+	else
+		atomic_dec(&msg_queue->lat_list_cnt);
+}
+
 int vdec_msg_queue_qbuf(struct vdec_msg_queue_ctx *msg_ctx, struct vdec_lat_buf *buf)
 {
 	struct list_head *head;
+	int status;
 
 	head = vdec_get_buf_list(msg_ctx->hardware_index, buf);
 	if (!head) {
@@ -66,11 +83,18 @@ int vdec_msg_queue_qbuf(struct vdec_msg_queue_ctx *msg_ctx, struct vdec_lat_buf
 	list_add_tail(head, &msg_ctx->ready_queue);
 	msg_ctx->ready_num++;
 
-	if (msg_ctx->hardware_index != MTK_VDEC_CORE)
+	vdec_msg_queue_inc(&buf->ctx->msg_queue, msg_ctx->hardware_index);
+	if (msg_ctx->hardware_index != MTK_VDEC_CORE) {
 		wake_up_all(&msg_ctx->ready_to_use);
-	else
-		queue_work(buf->ctx->dev->core_workqueue,
-			   &buf->ctx->msg_queue.core_work);
+	} else {
+		if (buf->ctx->msg_queue.core_work_cnt <
+			atomic_read(&buf->ctx->msg_queue.core_list_cnt)) {
+			status = queue_work(buf->ctx->dev->core_workqueue,
+					    &buf->ctx->msg_queue.core_work);
+			if (status)
+				buf->ctx->msg_queue.core_work_cnt++;
+		}
+	}
 
 	mtk_v4l2_debug(3, "enqueue buf type: %d addr: 0x%p num: %d",
 		       msg_ctx->hardware_index, buf, msg_ctx->ready_num);
@@ -127,6 +151,7 @@ struct vdec_lat_buf *vdec_msg_queue_dqbuf(struct vdec_msg_queue_ctx *msg_ctx)
 		return NULL;
 	}
 	list_del(head);
+	vdec_msg_queue_dec(&buf->ctx->msg_queue, msg_ctx->hardware_index);
 
 	msg_ctx->ready_num--;
 	mtk_v4l2_debug(3, "dqueue buf type:%d addr: 0x%p num: %d",
@@ -156,11 +181,29 @@ void vdec_msg_queue_update_ube_wptr(struct vdec_msg_queue *msg_queue, uint64_t u
 
 bool vdec_msg_queue_wait_lat_buf_full(struct vdec_msg_queue *msg_queue)
 {
+	struct vdec_lat_buf *buf, *tmp;
+	struct list_head *list_core[3];
+	struct vdec_msg_queue_ctx *core_ctx;
+	int ret, i, in_core_count = 0, count = 0;
 	long timeout_jiff;
-	int ret;
+
+	core_ctx = &msg_queue->ctx->dev->msg_queue_core_ctx;
+	spin_lock(&core_ctx->ready_lock);
+	list_for_each_entry_safe(buf, tmp, &core_ctx->ready_queue, core_list) {
+		if (buf && buf->ctx == msg_queue->ctx) {
+			list_core[in_core_count++] = &buf->core_list;
+			list_del(&buf->core_list);
+		}
+	}
+
+	for (i = 0; i < in_core_count; i++) {
+		list_add(list_core[in_core_count - (1 + i)], &core_ctx->ready_queue);
+		queue_work(msg_queue->ctx->dev->core_workqueue, &msg_queue->core_work);
+	}
+	spin_unlock(&core_ctx->ready_lock);
 
 	timeout_jiff = msecs_to_jiffies(1000 * (NUM_BUFFER_COUNT + 2));
-	ret = wait_event_timeout(msg_queue->lat_ctx.ready_to_use,
+	ret = wait_event_timeout(msg_queue->ctx->msg_queue.core_dec_done,
 				 msg_queue->lat_ctx.ready_num == NUM_BUFFER_COUNT,
 				 timeout_jiff);
 	if (ret) {
@@ -168,8 +211,20 @@ bool vdec_msg_queue_wait_lat_buf_full(struct vdec_msg_queue *msg_queue)
 			       msg_queue->lat_ctx.ready_num);
 		return true;
 	}
-	mtk_v4l2_err("failed with lat buf isn't full: %d",
-		     msg_queue->lat_ctx.ready_num);
+
+	spin_lock(&core_ctx->ready_lock);
+	list_for_each_entry_safe(buf, tmp, &core_ctx->ready_queue, core_list) {
+		if (buf && buf->ctx == msg_queue->ctx) {
+			count++;
+			list_del(&buf->core_list);
+		}
+	}
+	spin_unlock(&core_ctx->ready_lock);
+
+	mtk_v4l2_err("failed with lat buf isn't full: list(%d %d) count:%d",
+		     atomic_read(&msg_queue->lat_list_cnt),
+		     atomic_read(&msg_queue->core_list_cnt), count);
+
 	return false;
 }
 
@@ -206,6 +261,7 @@ static void vdec_msg_queue_core_work(struct work_struct *work)
 		container_of(msg_queue, struct mtk_vcodec_ctx, msg_queue);
 	struct mtk_vcodec_dev *dev = ctx->dev;
 	struct vdec_lat_buf *lat_buf;
+	int status;
 
 	lat_buf = vdec_msg_queue_dqbuf(&dev->msg_queue_core_ctx);
 	if (!lat_buf)
@@ -221,11 +277,18 @@ static void vdec_msg_queue_core_work(struct work_struct *work)
 	mtk_vcodec_dec_disable_hardware(ctx, MTK_VDEC_CORE);
 	vdec_msg_queue_qbuf(&ctx->msg_queue.lat_ctx, lat_buf);
 
-	if (!list_empty(&dev->msg_queue_core_ctx.ready_queue)) {
-		mtk_v4l2_debug(3, "re-schedule to decode for core: %d",
-			       dev->msg_queue_core_ctx.ready_num);
-		queue_work(dev->core_workqueue, &msg_queue->core_work);
+	wake_up_all(&ctx->msg_queue.core_dec_done);
+	spin_lock(&dev->msg_queue_core_ctx.ready_lock);
+	lat_buf->ctx->msg_queue.core_work_cnt--;
+
+	if (lat_buf->ctx->msg_queue.core_work_cnt <
+		atomic_read(&lat_buf->ctx->msg_queue.core_list_cnt)) {
+		status = queue_work(lat_buf->ctx->dev->core_workqueue,
+				    &lat_buf->ctx->msg_queue.core_work);
+		if (status)
+			lat_buf->ctx->msg_queue.core_work_cnt++;
 	}
+	spin_unlock(&dev->msg_queue_core_ctx.ready_lock);
 }
 
 int vdec_msg_queue_init(struct vdec_msg_queue *msg_queue,
@@ -239,12 +302,18 @@ int vdec_msg_queue_init(struct vdec_msg_queue *msg_queue,
 	if (msg_queue->wdma_addr.size)
 		return 0;
 
+	msg_queue->ctx = ctx;
+	msg_queue->core_work_cnt = 0;
 	vdec_msg_queue_init_ctx(&msg_queue->lat_ctx, MTK_VDEC_LAT0);
 	INIT_WORK(&msg_queue->core_work, vdec_msg_queue_core_work);
+
+	atomic_set(&msg_queue->lat_list_cnt, 0);
+	atomic_set(&msg_queue->core_list_cnt, 0);
+	init_waitqueue_head(&msg_queue->core_dec_done);
+
 	msg_queue->wdma_addr.size =
 		vde_msg_queue_get_trans_size(ctx->picinfo.buf_w,
 					     ctx->picinfo.buf_h);
-
 	err = mtk_vcodec_mem_alloc(ctx, &msg_queue->wdma_addr);
 	if (err) {
 		mtk_v4l2_err("failed to allocate wdma_addr buf");
diff --git a/drivers/media/platform/mediatek/vcodec/vdec_msg_queue.h b/drivers/media/platform/mediatek/vcodec/vdec_msg_queue.h
index c43d427f5f54..a5d44bc97c16 100644
--- a/drivers/media/platform/mediatek/vcodec/vdec_msg_queue.h
+++ b/drivers/media/platform/mediatek/vcodec/vdec_msg_queue.h
@@ -72,6 +72,12 @@ struct vdec_lat_buf {
  * @wdma_wptr_addr: ube write point
  * @core_work: core hardware work
  * @lat_ctx: used to store lat buffer list
+ * @ctx: point to mtk_vcodec_ctx
+ *
+ * @lat_list_cnt: used to record each instance lat list count
+ * @core_list_cnt: used to record each instance core list count
+ * @core_dec_done: core work queue decode done event
+ * @core_work_cnt: the number of core work in work queue
  */
 struct vdec_msg_queue {
 	struct vdec_lat_buf lat_buf[NUM_BUFFER_COUNT];
@@ -82,6 +88,12 @@ struct vdec_msg_queue {
 
 	struct work_struct core_work;
 	struct vdec_msg_queue_ctx lat_ctx;
+	struct mtk_vcodec_ctx *ctx;
+
+	atomic_t lat_list_cnt;
+	atomic_t core_list_cnt;
+	wait_queue_head_t core_dec_done;
+	int core_work_cnt;
 };
 
 /**
diff --git a/drivers/media/platform/mediatek/vpu/mtk_vpu.c b/drivers/media/platform/mediatek/vpu/mtk_vpu.c
index 47b684b92f81..5e2bc286f168 100644
--- a/drivers/media/platform/mediatek/vpu/mtk_vpu.c
+++ b/drivers/media/platform/mediatek/vpu/mtk_vpu.c
@@ -953,7 +953,7 @@ static const struct of_device_id mtk_vpu_match[] = {
 };
 MODULE_DEVICE_TABLE(of, mtk_vpu_match);
 
-static int mtk_vpu_remove(struct platform_device *pdev)
+static void mtk_vpu_remove(struct platform_device *pdev)
 {
 	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
 
@@ -966,8 +966,6 @@ static int mtk_vpu_remove(struct platform_device *pdev)
 	vpu_free_ext_mem(vpu, D_FW);
 	mutex_destroy(&vpu->vpu_mutex);
 	clk_unprepare(vpu->clk);
-
-	return 0;
 }
 
 static int mtk_vpu_suspend(struct device *dev)
@@ -1040,7 +1038,7 @@ static const struct dev_pm_ops mtk_vpu_pm = {
 
 static struct platform_driver mtk_vpu_driver = {
 	.probe	= mtk_vpu_probe,
-	.remove	= mtk_vpu_remove,
+	.remove_new = mtk_vpu_remove,
 	.driver	= {
 		.name	= "mtk_vpu",
 		.pm = &mtk_vpu_pm,
diff --git a/drivers/media/platform/microchip/microchip-csi2dc.c b/drivers/media/platform/microchip/microchip-csi2dc.c
index d5b359f607ae..bfb3edcf018a 100644
--- a/drivers/media/platform/microchip/microchip-csi2dc.c
+++ b/drivers/media/platform/microchip/microchip-csi2dc.c
@@ -741,7 +741,7 @@ csi2dc_probe_cleanup_entity:
 	return ret;
 }
 
-static int csi2dc_remove(struct platform_device *pdev)
+static void csi2dc_remove(struct platform_device *pdev)
 {
 	struct csi2dc_device *csi2dc = platform_get_drvdata(pdev);
 
@@ -751,8 +751,6 @@ static int csi2dc_remove(struct platform_device *pdev)
 	v4l2_async_nf_unregister(&csi2dc->notifier);
 	v4l2_async_nf_cleanup(&csi2dc->notifier);
 	media_entity_cleanup(&csi2dc->csi2dc_sd.entity);
-
-	return 0;
 }
 
 static int __maybe_unused csi2dc_runtime_suspend(struct device *dev)
@@ -782,7 +780,7 @@ MODULE_DEVICE_TABLE(of, csi2dc_of_match);
 
 static struct platform_driver csi2dc_driver = {
 	.probe	= csi2dc_probe,
-	.remove = csi2dc_remove,
+	.remove_new = csi2dc_remove,
 	.driver = {
 		.name =			"microchip-csi2dc",
 		.pm =			&csi2dc_dev_pm_ops,
diff --git a/drivers/media/platform/microchip/microchip-isc-base.c b/drivers/media/platform/microchip/microchip-isc-base.c
index 71758ee8474b..4e657fad33d0 100644
--- a/drivers/media/platform/microchip/microchip-isc-base.c
+++ b/drivers/media/platform/microchip/microchip-isc-base.c
@@ -858,8 +858,10 @@ static int isc_try_configure_pipeline(struct isc_device *isc)
 static void isc_try_fse(struct isc_device *isc,
 			struct v4l2_subdev_state *sd_state)
 {
+	struct v4l2_subdev_frame_size_enum fse = {
+		.which = V4L2_SUBDEV_FORMAT_TRY,
+	};
 	int ret;
-	struct v4l2_subdev_frame_size_enum fse = {};
 
 	/*
 	 * If we do not know yet which format the subdev is using, we cannot
@@ -869,7 +871,6 @@ static void isc_try_fse(struct isc_device *isc,
 		return;
 
 	fse.code = isc->try_config.sd_format->mbus_code;
-	fse.which = V4L2_SUBDEV_FORMAT_TRY;
 
 	ret = v4l2_subdev_call(isc->current_subdev->sd, pad, enum_frame_size,
 			       sd_state, &fse);
diff --git a/drivers/media/platform/microchip/microchip-sama5d2-isc.c b/drivers/media/platform/microchip/microchip-sama5d2-isc.c
index ac4715d91de6..746f4a2fa9f6 100644
--- a/drivers/media/platform/microchip/microchip-sama5d2-isc.c
+++ b/drivers/media/platform/microchip/microchip-sama5d2-isc.c
@@ -608,7 +608,7 @@ unprepare_hclk:
 	return ret;
 }
 
-static int microchip_isc_remove(struct platform_device *pdev)
+static void microchip_isc_remove(struct platform_device *pdev)
 {
 	struct isc_device *isc = platform_get_drvdata(pdev);
 
@@ -624,8 +624,6 @@ static int microchip_isc_remove(struct platform_device *pdev)
 	clk_disable_unprepare(isc->hclock);
 
 	microchip_isc_clk_cleanup(isc);
-
-	return 0;
 }
 
 static int __maybe_unused isc_runtime_suspend(struct device *dev)
@@ -668,7 +666,7 @@ MODULE_DEVICE_TABLE(of, microchip_isc_of_match);
 
 static struct platform_driver microchip_isc_driver = {
 	.probe	= microchip_isc_probe,
-	.remove	= microchip_isc_remove,
+	.remove_new = microchip_isc_remove,
 	.driver	= {
 		.name		= "microchip-sama5d2-isc",
 		.pm		= &microchip_isc_dev_pm_ops,
diff --git a/drivers/media/platform/microchip/microchip-sama7g5-isc.c b/drivers/media/platform/microchip/microchip-sama7g5-isc.c
index d583eafe5cc1..79ae696764d0 100644
--- a/drivers/media/platform/microchip/microchip-sama7g5-isc.c
+++ b/drivers/media/platform/microchip/microchip-sama7g5-isc.c
@@ -577,7 +577,7 @@ unprepare_hclk:
 	return ret;
 }
 
-static int microchip_xisc_remove(struct platform_device *pdev)
+static void microchip_xisc_remove(struct platform_device *pdev)
 {
 	struct isc_device *isc = platform_get_drvdata(pdev);
 
@@ -592,8 +592,6 @@ static int microchip_xisc_remove(struct platform_device *pdev)
 	clk_disable_unprepare(isc->hclock);
 
 	microchip_isc_clk_cleanup(isc);
-
-	return 0;
 }
 
 static int __maybe_unused xisc_runtime_suspend(struct device *dev)
@@ -631,7 +629,7 @@ MODULE_DEVICE_TABLE(of, microchip_xisc_of_match);
 
 static struct platform_driver microchip_xisc_driver = {
 	.probe	= microchip_xisc_probe,
-	.remove	= microchip_xisc_remove,
+	.remove_new = microchip_xisc_remove,
 	.driver	= {
 		.name		= "microchip-sama7g5-xisc",
 		.pm		= &microchip_xisc_dev_pm_ops,
diff --git a/drivers/media/platform/nvidia/tegra-vde/vde.c b/drivers/media/platform/nvidia/tegra-vde/vde.c
index f3e863a94c5a..7157734a1550 100644
--- a/drivers/media/platform/nvidia/tegra-vde/vde.c
+++ b/drivers/media/platform/nvidia/tegra-vde/vde.c
@@ -378,7 +378,7 @@ err_gen_free:
 	return err;
 }
 
-static int tegra_vde_remove(struct platform_device *pdev)
+static void tegra_vde_remove(struct platform_device *pdev)
 {
 	struct tegra_vde *vde = platform_get_drvdata(pdev);
 	struct device *dev = &pdev->dev;
@@ -407,8 +407,6 @@ static int tegra_vde_remove(struct platform_device *pdev)
 
 	gen_pool_free(vde->iram_pool, (unsigned long)vde->iram,
 		      gen_pool_size(vde->iram_pool));
-
-	return 0;
 }
 
 static void tegra_vde_shutdown(struct platform_device *pdev)
@@ -536,7 +534,7 @@ MODULE_DEVICE_TABLE(of, tegra_vde_of_match);
 
 static struct platform_driver tegra_vde_driver = {
 	.probe		= tegra_vde_probe,
-	.remove		= tegra_vde_remove,
+	.remove_new	= tegra_vde_remove,
 	.shutdown	= tegra_vde_shutdown,
 	.driver		= {
 		.name		= "tegra-vde",
diff --git a/drivers/media/platform/nxp/Kconfig b/drivers/media/platform/nxp/Kconfig
index 730f39971e1c..a0ca6b297fb8 100644
--- a/drivers/media/platform/nxp/Kconfig
+++ b/drivers/media/platform/nxp/Kconfig
@@ -28,6 +28,8 @@ config VIDEO_IMX_MIPI_CSIS
 	  Video4Linux2 sub-device driver for the MIPI CSI-2 CSIS receiver
 	  v3.3/v3.6.3 found on some i.MX7 and i.MX8 SoCs.
 
+source "drivers/media/platform/nxp/imx8-isi/Kconfig"
+
 # mem2mem drivers
 
 config VIDEO_IMX_PXP
diff --git a/drivers/media/platform/nxp/Makefile b/drivers/media/platform/nxp/Makefile
index 1a129b58d99e..b8e672b75fed 100644
--- a/drivers/media/platform/nxp/Makefile
+++ b/drivers/media/platform/nxp/Makefile
@@ -2,6 +2,7 @@
 
 obj-y += dw100/
 obj-y += imx-jpeg/
+obj-y += imx8-isi/
 
 obj-$(CONFIG_VIDEO_IMX7_CSI) += imx7-media-csi.o
 obj-$(CONFIG_VIDEO_IMX_MIPI_CSIS) += imx-mipi-csis.o
diff --git a/drivers/media/platform/nxp/dw100/dw100.c b/drivers/media/platform/nxp/dw100/dw100.c
index 189d60cd5ed1..0024d6175ad9 100644
--- a/drivers/media/platform/nxp/dw100/dw100.c
+++ b/drivers/media/platform/nxp/dw100/dw100.c
@@ -1532,7 +1532,6 @@ static int dw100_probe(struct platform_device *pdev)
 {
 	struct dw100_device *dw_dev;
 	struct video_device *vfd;
-	struct resource *res;
 	int ret, irq;
 
 	dw_dev = devm_kzalloc(&pdev->dev, sizeof(*dw_dev), GFP_KERNEL);
@@ -1547,8 +1546,7 @@ static int dw100_probe(struct platform_device *pdev)
 	}
 	dw_dev->num_clks = ret;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	dw_dev->mmio = devm_ioremap_resource(&pdev->dev, res);
+	dw_dev->mmio = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
 	if (IS_ERR(dw_dev->mmio))
 		return PTR_ERR(dw_dev->mmio);
 
@@ -1633,7 +1631,7 @@ err_pm:
 	return ret;
 }
 
-static int dw100_remove(struct platform_device *pdev)
+static void dw100_remove(struct platform_device *pdev)
 {
 	struct dw100_device *dw_dev = platform_get_drvdata(pdev);
 
@@ -1649,8 +1647,6 @@ static int dw100_remove(struct platform_device *pdev)
 	mutex_destroy(dw_dev->vfd.lock);
 	v4l2_m2m_release(dw_dev->m2m_dev);
 	v4l2_device_unregister(&dw_dev->v4l2_dev);
-
-	return 0;
 }
 
 static int __maybe_unused dw100_runtime_suspend(struct device *dev)
@@ -1692,7 +1688,7 @@ MODULE_DEVICE_TABLE(of, dw100_dt_ids);
 
 static struct platform_driver dw100_driver = {
 	.probe		= dw100_probe,
-	.remove		= dw100_remove,
+	.remove_new	= dw100_remove,
 	.driver		= {
 		.name	= DRV_NAME,
 		.pm = &dw100_pm,
diff --git a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg-hw.c b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg-hw.c
index ef28122a5ed4..9a6e8b332e12 100644
--- a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg-hw.c
+++ b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg-hw.c
@@ -97,25 +97,31 @@ void mxc_jpeg_sw_reset(void __iomem *reg)
 	writel(GLB_CTRL_SFT_RST, reg + GLB_CTRL);
 }
 
-void mxc_jpeg_enc_mode_conf(struct device *dev, void __iomem *reg)
+void mxc_jpeg_enc_mode_conf(struct device *dev, void __iomem *reg, u8 extseq)
 {
 	dev_dbg(dev, "CAST Encoder CONFIG...\n");
 	/*
 	 * "Config_Mode" enabled, "Config_Mode auto clear enabled",
 	 */
-	writel(0xa0, reg + CAST_MODE);
+	if (extseq)
+		writel(0xb0, reg + CAST_MODE);
+	else
+		writel(0xa0, reg + CAST_MODE);
 
 	/* all markers and segments */
 	writel(0x3ff, reg + CAST_CFG_MODE);
 }
 
-void mxc_jpeg_enc_mode_go(struct device *dev, void __iomem *reg)
+void mxc_jpeg_enc_mode_go(struct device *dev, void __iomem *reg, u8 extseq)
 {
 	dev_dbg(dev, "CAST Encoder GO...\n");
 	/*
 	 * "GO" enabled, "GO bit auto clear" enabled
 	 */
-	writel(0x140, reg + CAST_MODE);
+	if (extseq)
+		writel(0x150, reg + CAST_MODE);
+	else
+		writel(0x140, reg + CAST_MODE);
 }
 
 void mxc_jpeg_enc_set_quality(struct device *dev, void __iomem *reg, u8 quality)
@@ -178,3 +184,8 @@ void mxc_jpeg_set_desc(u32 desc, void __iomem *reg, int slot)
 	writel(desc | MXC_NXT_DESCPT_EN,
 	       reg + MXC_SLOT_OFFSET(slot, SLOT_NXT_DESCPT_PTR));
 }
+
+void mxc_jpeg_clr_desc(void __iomem *reg, int slot)
+{
+	writel(0, reg + MXC_SLOT_OFFSET(slot, SLOT_NXT_DESCPT_PTR));
+}
diff --git a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg-hw.h b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg-hw.h
index ecf3b6562ba2..ed15ea348f97 100644
--- a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg-hw.h
+++ b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg-hw.h
@@ -117,8 +117,8 @@ void print_wrapper_info(struct device *dev, void __iomem *reg);
 void mxc_jpeg_sw_reset(void __iomem *reg);
 int mxc_jpeg_enable(void __iomem *reg);
 void wait_frmdone(struct device *dev, void __iomem *reg);
-void mxc_jpeg_enc_mode_conf(struct device *dev, void __iomem *reg);
-void mxc_jpeg_enc_mode_go(struct device *dev, void __iomem *reg);
+void mxc_jpeg_enc_mode_conf(struct device *dev, void __iomem *reg, u8 extseq);
+void mxc_jpeg_enc_mode_go(struct device *dev, void __iomem *reg, u8 extseq);
 void mxc_jpeg_enc_set_quality(struct device *dev, void __iomem *reg, u8 quality);
 void mxc_jpeg_dec_mode_go(struct device *dev, void __iomem *reg);
 int mxc_jpeg_get_slot(void __iomem *reg);
@@ -137,6 +137,7 @@ void mxc_jpeg_set_bufsize(struct mxc_jpeg_desc *desc,  u32 bufsize);
 void mxc_jpeg_set_res(struct mxc_jpeg_desc *desc, u16 w, u16 h);
 void mxc_jpeg_set_line_pitch(struct mxc_jpeg_desc *desc, u32 line_pitch);
 void mxc_jpeg_set_desc(u32 desc, void __iomem *reg, int slot);
+void mxc_jpeg_clr_desc(void __iomem *reg, int slot);
 void mxc_jpeg_set_regs_from_desc(struct mxc_jpeg_desc *desc,
 				 void __iomem *reg);
 #endif
diff --git a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c
index f085f14d676a..c0e49be42450 100644
--- a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c
+++ b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c
@@ -88,6 +88,20 @@ static const struct mxc_jpeg_fmt mxc_formats[] = {
 		.is_rgb		= 1,
 	},
 	{
+		.name		= "BGR 12bit", /*12-bit BGR packed format*/
+		.fourcc		= V4L2_PIX_FMT_BGR48_12,
+		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_444,
+		.nc		= 3,
+		.depth		= 36,
+		.mem_planes	= 1,
+		.comp_planes	= 1,
+		.h_align	= 3,
+		.v_align	= 3,
+		.flags		= MXC_JPEG_FMT_TYPE_RAW,
+		.precision	= 12,
+		.is_rgb		= 1,
+	},
+	{
 		.name		= "ABGR", /* ABGR packed format */
 		.fourcc		= V4L2_PIX_FMT_ABGR32,
 		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_444,
@@ -102,6 +116,20 @@ static const struct mxc_jpeg_fmt mxc_formats[] = {
 		.is_rgb		= 1,
 	},
 	{
+		.name		= "ABGR 12bit", /* 12-bit ABGR packed format */
+		.fourcc		= V4L2_PIX_FMT_ABGR64_12,
+		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_444,
+		.nc		= 4,
+		.depth		= 48,
+		.mem_planes	= 1,
+		.comp_planes	= 1,
+		.h_align	= 3,
+		.v_align	= 3,
+		.flags		= MXC_JPEG_FMT_TYPE_RAW,
+		.precision	= 12,
+		.is_rgb		= 1,
+	},
+	{
 		.name		= "YUV420", /* 1st plane = Y, 2nd plane = UV */
 		.fourcc		= V4L2_PIX_FMT_NV12M,
 		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_420,
@@ -128,6 +156,32 @@ static const struct mxc_jpeg_fmt mxc_formats[] = {
 		.precision	= 8,
 	},
 	{
+		.name		= "YUV420 12bit", /* 1st plane = Y, 2nd plane = UV */
+		.fourcc		= V4L2_PIX_FMT_P012M,
+		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_420,
+		.nc		= 3,
+		.depth		= 18, /* 6 x 12 bits (4Y + UV) for 4 pixels */
+		.mem_planes	= 2,
+		.comp_planes	= 2, /* 1 plane Y, 1 plane UV interleaved */
+		.h_align	= 4,
+		.v_align	= 4,
+		.flags		= MXC_JPEG_FMT_TYPE_RAW,
+		.precision	= 12,
+	},
+	{
+		.name		= "YUV420 12bit", /* 1st plane = Y, 2nd plane = UV */
+		.fourcc		= V4L2_PIX_FMT_P012,
+		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_420,
+		.nc		= 3,
+		.depth		= 18, /* 6 x 12 bits (4Y + UV) for 4 pixels */
+		.mem_planes	= 1,
+		.comp_planes	= 2, /* 1 plane Y, 1 plane UV interleaved */
+		.h_align	= 4,
+		.v_align	= 4,
+		.flags		= MXC_JPEG_FMT_TYPE_RAW,
+		.precision	= 12,
+	},
+	{
 		.name		= "YUV422", /* YUYV */
 		.fourcc		= V4L2_PIX_FMT_YUYV,
 		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_422,
@@ -141,6 +195,19 @@ static const struct mxc_jpeg_fmt mxc_formats[] = {
 		.precision	= 8,
 	},
 	{
+		.name		= "YUV422 12bit", /* YUYV */
+		.fourcc		= V4L2_PIX_FMT_Y212,
+		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_422,
+		.nc		= 3,
+		.depth		= 24,
+		.mem_planes	= 1,
+		.comp_planes	= 1,
+		.h_align	= 4,
+		.v_align	= 3,
+		.flags		= MXC_JPEG_FMT_TYPE_RAW,
+		.precision	= 12,
+	},
+	{
 		.name		= "YUV444", /* YUVYUV */
 		.fourcc		= V4L2_PIX_FMT_YUV24,
 		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_444,
@@ -154,6 +221,19 @@ static const struct mxc_jpeg_fmt mxc_formats[] = {
 		.precision	= 8,
 	},
 	{
+		.name		= "YUV444 12bit", /* YUVYUV */
+		.fourcc		= V4L2_PIX_FMT_YUV48_12,
+		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_444,
+		.nc		= 3,
+		.depth		= 36,
+		.mem_planes	= 1,
+		.comp_planes	= 1,
+		.h_align	= 3,
+		.v_align	= 3,
+		.flags		= MXC_JPEG_FMT_TYPE_RAW,
+		.precision	= 12,
+	},
+	{
 		.name		= "Gray", /* Gray (Y8/Y12) or Single Comp */
 		.fourcc		= V4L2_PIX_FMT_GREY,
 		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_GRAY,
@@ -166,6 +246,19 @@ static const struct mxc_jpeg_fmt mxc_formats[] = {
 		.flags		= MXC_JPEG_FMT_TYPE_RAW,
 		.precision	= 8,
 	},
+	{
+		.name		= "Gray 12bit", /* Gray (Y8/Y12) or Single Comp */
+		.fourcc		= V4L2_PIX_FMT_Y012,
+		.subsampling	= V4L2_JPEG_CHROMA_SUBSAMPLING_GRAY,
+		.nc		= 1,
+		.depth		= 12,
+		.mem_planes	= 1,
+		.comp_planes	= 1,
+		.h_align	= 3,
+		.v_align	= 3,
+		.flags		= MXC_JPEG_FMT_TYPE_RAW,
+		.precision	= 12,
+	},
 };
 
 #define MXC_JPEG_NUM_FORMATS ARRAY_SIZE(mxc_formats)
@@ -229,6 +322,45 @@ static const unsigned char jpeg_dqt[] = {
 	0x63, 0x63, 0x63, 0x63, 0x63, 0x63
 };
 
+static const unsigned char jpeg_dqt_extseq[] = {
+	0xFF, 0xDB,
+	0x01, 0x04,
+	0x10,
+	0x00, 0x80, 0x00, 0x58, 0x00, 0x60, 0x00, 0x70,
+	0x00, 0x60, 0x00, 0x50, 0x00, 0x80, 0x00, 0x70,
+	0x00, 0x68, 0x00, 0x70, 0x00, 0x90, 0x00, 0x88,
+	0x00, 0x80, 0x00, 0x98, 0x00, 0xC0, 0x01, 0x40,
+	0x00, 0xD0, 0x00, 0xC0, 0x00, 0xB0, 0x00, 0xB0,
+	0x00, 0xC0, 0x01, 0x88, 0x01, 0x18, 0x01, 0x28,
+	0x00, 0xE8, 0x01, 0x40, 0x01, 0xD0, 0x01, 0x98,
+	0x01, 0xE8, 0x01, 0xE0, 0x01, 0xC8, 0x01, 0x98,
+	0x01, 0xC0, 0x01, 0xB8, 0x02, 0x00, 0x02, 0x40,
+	0x02, 0xE0, 0x02, 0x70, 0x02, 0x00, 0x02, 0x20,
+	0x02, 0xB8, 0x02, 0x28, 0x01, 0xB8, 0x01, 0xC0,
+	0x02, 0x80, 0x03, 0x68, 0x02, 0x88, 0x02, 0xB8,
+	0x02, 0xF8, 0x03, 0x10, 0x03, 0x38, 0x03, 0x40,
+	0x03, 0x38, 0x01, 0xF0, 0x02, 0x68, 0x03, 0x88,
+	0x03, 0xC8, 0x03, 0x80, 0x03, 0x20, 0x03, 0xC0,
+	0x02, 0xE0, 0x03, 0x28, 0x03, 0x38, 0x03, 0x18,
+	0x11,
+	0x00, 0x88, 0x00, 0x90, 0x00, 0x90, 0x00, 0xC0,
+	0x00, 0xA8, 0x00, 0xC0, 0x01, 0x78, 0x00, 0xD0,
+	0x00, 0xD0, 0x01, 0x78, 0x03, 0x18, 0x02, 0x10,
+	0x01, 0xC0, 0x02, 0x10, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+	0x03, 0x18, 0x03, 0x18, 0x03, 0x18, 0x03, 0x18,
+};
+
 static const unsigned char jpeg_sof_maximal[] = {
 	0xFF, 0xC0,
 	0x00, 0x14, 0x08, 0x00, 0x40, 0x00, 0x40,
@@ -236,6 +368,13 @@ static const unsigned char jpeg_sof_maximal[] = {
 	0x03, 0x11, 0x01, 0x04, 0x11, 0x01
 };
 
+static const unsigned char jpeg_sof_extseq[] = {
+	0xFF, 0xC1,
+	0x00, 0x14, 0x08, 0x00, 0x40, 0x00, 0x40,
+	0x04, 0x01, 0x11, 0x00, 0x02, 0x11, 0x01,
+	0x03, 0x11, 0x01, 0x04, 0x11, 0x01
+};
+
 static const unsigned char jpeg_dht[] = {
 	0xFF, 0xC4,
 	0x01, 0xA2, 0x00, 0x00, 0x01, 0x05, 0x01,
@@ -300,6 +439,90 @@ static const unsigned char jpeg_dht[] = {
 	0xF6, 0xF7, 0xF8, 0xF9, 0xFA
 };
 
+static const unsigned char jpeg_dht_extseq[] = {
+	0xFF, 0xC4,
+	0x02, 0x2a, 0x00, 0x00, 0x01, 0x05, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01,
+	0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+	0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+	0x10, 0x00, 0x02, 0x01, 0x03, 0x03, 0x02,
+	0x04, 0x03, 0x05, 0x05, 0x02, 0x03, 0x02,
+	0x00, 0x00, 0xbf, 0x01, 0x02, 0x03, 0x00,
+	0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41,
+	0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71,
+	0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23,
+	0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0,
+	0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a,
+	0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26,
+	0x27, 0x28, 0x29, 0x2a, 0x34, 0x35, 0x36,
+	0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45,
+	0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54,
+	0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63,
+	0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a,
+	0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79,
+	0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
+	0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96,
+	0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4,
+	0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2,
+	0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9,
+	0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+	0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5,
+	0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2,
+	0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9,
+	0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6,
+	0xf7, 0xf8, 0xf9, 0xfa, 0x0b, 0x0c, 0x0d,
+	0x0e, 0x1b, 0x1c, 0x1d, 0x1e, 0x2b, 0x2c,
+	0x2d, 0x2e, 0x3b, 0x3c, 0x3d, 0x3e, 0x4b,
+	0x4c, 0x4d, 0x4e, 0x5b, 0x5c, 0x5d, 0x5e,
+	0x6b, 0x6c, 0x6d, 0x6e, 0x7b, 0x7c, 0x7d,
+	0x7e, 0x8b, 0x8c, 0x8d, 0x8e, 0x9b, 0x9c,
+	0x9d, 0x9e, 0xab, 0xac, 0xad, 0xae, 0xbb,
+	0xbc, 0xbd, 0xbe, 0xcb, 0xcc, 0xcd, 0xce,
+	0xdb, 0xdc, 0xdd, 0xde, 0xeb, 0xec, 0xed,
+	0xee, 0xfb, 0xfc, 0xfd, 0xfe, 0x01, 0x00,
+	0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00,
+	0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
+	0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c,
+	0x0d, 0x0e, 0x0f, 0x11, 0x00, 0x02, 0x01,
+	0x03, 0x03, 0x02, 0x04, 0x03, 0x05, 0x05,
+	0x02, 0x03, 0x02, 0x00, 0x00, 0xbf, 0x01,
+	0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12,
+	0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61,
+	0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91,
+	0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15,
+	0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72,
+	0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19,
+	0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a,
+	0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a,
+	0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49,
+	0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
+	0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67,
+	0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76,
+	0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85,
+	0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93,
+	0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a,
+	0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8,
+	0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6,
+	0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4,
+	0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2,
+	0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9,
+	0xda, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6,
+	0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3,
+	0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa,
+	0x0b, 0x0c, 0x0d, 0x0e, 0x1b, 0x1c, 0x1d,
+	0x1e, 0x2b, 0x2c, 0x2d, 0x2e, 0x3b, 0x3c,
+	0x3d, 0x3e, 0x4b, 0x4c, 0x4d, 0x4e, 0x5b,
+	0x5c, 0x5d, 0x5e, 0x6b, 0x6c, 0x6d, 0x6e,
+	0x7b, 0x7c, 0x7d, 0x7e, 0x8b, 0x8c, 0x8d,
+	0x8e, 0x9b, 0x9c, 0x9d, 0x9e, 0xab, 0xac,
+	0xad, 0xae, 0xbb, 0xbc, 0xbd, 0xbe, 0xcb,
+	0xcc, 0xcd, 0xce, 0xdb, 0xdc, 0xdd, 0xde,
+	0xeb, 0xec, 0xed, 0xee, 0xfb, 0xfc, 0xfd,
+	0xfe,
+};
+
 static const unsigned char jpeg_dri[] = {
 	0xFF, 0xDD,
 	0x00, 0x04, 0x00, 0x20
@@ -420,8 +643,7 @@ static int enum_fmt(const struct mxc_jpeg_fmt *mxc_formats, int n,
 	return 0;
 }
 
-static const struct mxc_jpeg_fmt *mxc_jpeg_find_format(struct mxc_jpeg_ctx *ctx,
-						       u32 pixelformat)
+static const struct mxc_jpeg_fmt *mxc_jpeg_find_format(u32 pixelformat)
 {
 	unsigned int k;
 
@@ -438,17 +660,24 @@ static enum mxc_jpeg_image_format mxc_jpeg_fourcc_to_imgfmt(u32 fourcc)
 {
 	switch (fourcc) {
 	case V4L2_PIX_FMT_GREY:
+	case V4L2_PIX_FMT_Y012:
 		return MXC_JPEG_GRAY;
 	case V4L2_PIX_FMT_YUYV:
+	case V4L2_PIX_FMT_Y212:
 		return MXC_JPEG_YUV422;
 	case V4L2_PIX_FMT_NV12:
 	case V4L2_PIX_FMT_NV12M:
+	case V4L2_PIX_FMT_P012:
+	case V4L2_PIX_FMT_P012M:
 		return MXC_JPEG_YUV420;
 	case V4L2_PIX_FMT_YUV24:
+	case V4L2_PIX_FMT_YUV48_12:
 		return MXC_JPEG_YUV444;
 	case V4L2_PIX_FMT_BGR24:
+	case V4L2_PIX_FMT_BGR48_12:
 		return MXC_JPEG_BGR;
 	case V4L2_PIX_FMT_ABGR32:
+	case V4L2_PIX_FMT_ABGR64_12:
 		return MXC_JPEG_ABGR;
 	default:
 		return MXC_JPEG_INVALID;
@@ -484,6 +713,17 @@ static void mxc_jpeg_addrs(struct mxc_jpeg_desc *desc,
 		offset;
 }
 
+static bool mxc_jpeg_is_extended_sequential(const struct mxc_jpeg_fmt *fmt)
+{
+	if (!fmt || !(fmt->flags & MXC_JPEG_FMT_TYPE_RAW))
+		return false;
+
+	if (fmt->precision > 8)
+		return true;
+
+	return false;
+}
+
 static void notify_eos(struct mxc_jpeg_ctx *ctx)
 {
 	const struct v4l2_event ev = {
@@ -497,10 +737,9 @@ static void notify_eos(struct mxc_jpeg_ctx *ctx)
 static void notify_src_chg(struct mxc_jpeg_ctx *ctx)
 {
 	const struct v4l2_event ev = {
-			.type = V4L2_EVENT_SOURCE_CHANGE,
-			.u.src_change.changes =
-			V4L2_EVENT_SRC_CH_RESOLUTION,
-		};
+		.type = V4L2_EVENT_SOURCE_CHANGE,
+		.u.src_change.changes = V4L2_EVENT_SRC_CH_RESOLUTION,
+	};
 
 	dev_dbg(ctx->mxc_jpeg->dev, "Notify app event SRC_CH_RESOLUTION");
 	v4l2_event_queue_fh(&ctx->fh, &ev);
@@ -637,6 +876,11 @@ static u32 mxc_jpeg_get_plane_size(struct mxc_jpeg_q_data *q_data, u32 plane_no)
 		return q_data->sizeimage[plane_no];
 
 	size = q_data->sizeimage[fmt->mem_planes - 1];
+
+	/* Should be impossible given mxc_formats. */
+	if (WARN_ON_ONCE(fmt->comp_planes > ARRAY_SIZE(q_data->sizeimage)))
+		return size;
+
 	for (i = fmt->mem_planes; i < fmt->comp_planes; i++)
 		size += q_data->sizeimage[i];
 
@@ -692,7 +936,9 @@ static irqreturn_t mxc_jpeg_dec_irq(int irq, void *priv)
 	if (dec_ret & SLOT_STATUS_ENC_CONFIG_ERR) {
 		u32 ret = readl(reg + CAST_STATUS12);
 
-		dev_err(dev, "Encoder/decoder error, status=0x%08x", ret);
+		dev_err(dev, "Encoder/decoder error, dec_ret = 0x%08x, status=0x%08x",
+			dec_ret, ret);
+		mxc_jpeg_clr_desc(reg, slot);
 		mxc_jpeg_sw_reset(reg);
 		buf_state = VB2_BUF_STATE_ERROR;
 		goto buffers_done;
@@ -703,10 +949,11 @@ static irqreturn_t mxc_jpeg_dec_irq(int irq, void *priv)
 
 	if (jpeg->mode == MXC_JPEG_ENCODE &&
 	    ctx->enc_state == MXC_JPEG_ENC_CONF) {
+		q_data = mxc_jpeg_get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
 		ctx->enc_state = MXC_JPEG_ENCODING;
 		dev_dbg(dev, "Encoder config finished. Start encoding...\n");
 		mxc_jpeg_enc_set_quality(dev, reg, ctx->jpeg_quality);
-		mxc_jpeg_enc_mode_go(dev, reg);
+		mxc_jpeg_enc_mode_go(dev, reg, mxc_jpeg_is_extended_sequential(q_data->fmt));
 		goto job_unlock;
 	}
 	if (jpeg->mode == MXC_JPEG_DECODE && jpeg_src_buf->dht_needed) {
@@ -757,8 +1004,12 @@ static int mxc_jpeg_fixup_sof(struct mxc_jpeg_sof *sof,
 			      u16 w, u16 h)
 {
 	int sof_length;
+	const struct mxc_jpeg_fmt *fmt = mxc_jpeg_find_format(fourcc);
 
-	sof->precision = 8; /* TODO allow 8/12 bit precision*/
+	if (fmt)
+		sof->precision = fmt->precision;
+	else
+		sof->precision = 8; /* TODO allow 8/12 bit precision*/
 	sof->height = h;
 	_bswap16(&sof->height);
 	sof->width = w;
@@ -767,24 +1018,31 @@ static int mxc_jpeg_fixup_sof(struct mxc_jpeg_sof *sof,
 	switch (fourcc) {
 	case V4L2_PIX_FMT_NV12:
 	case V4L2_PIX_FMT_NV12M:
+	case V4L2_PIX_FMT_P012:
+	case V4L2_PIX_FMT_P012M:
 		sof->components_no = 3;
 		sof->comp[0].v = 0x2;
 		sof->comp[0].h = 0x2;
 		break;
 	case V4L2_PIX_FMT_YUYV:
+	case V4L2_PIX_FMT_Y212:
 		sof->components_no = 3;
 		sof->comp[0].v = 0x1;
 		sof->comp[0].h = 0x2;
 		break;
 	case V4L2_PIX_FMT_YUV24:
+	case V4L2_PIX_FMT_YUV48_12:
 	case V4L2_PIX_FMT_BGR24:
+	case V4L2_PIX_FMT_BGR48_12:
 	default:
 		sof->components_no = 3;
 		break;
 	case V4L2_PIX_FMT_ABGR32:
+	case V4L2_PIX_FMT_ABGR64_12:
 		sof->components_no = 4;
 		break;
 	case V4L2_PIX_FMT_GREY:
+	case V4L2_PIX_FMT_Y012:
 		sof->components_no = 1;
 		break;
 	}
@@ -804,20 +1062,27 @@ static int mxc_jpeg_fixup_sos(struct mxc_jpeg_sos *sos,
 	switch (fourcc) {
 	case V4L2_PIX_FMT_NV12:
 	case V4L2_PIX_FMT_NV12M:
+	case V4L2_PIX_FMT_P012:
+	case V4L2_PIX_FMT_P012M:
 		sos->components_no = 3;
 		break;
 	case V4L2_PIX_FMT_YUYV:
+	case V4L2_PIX_FMT_Y212:
 		sos->components_no = 3;
 		break;
 	case V4L2_PIX_FMT_YUV24:
+	case V4L2_PIX_FMT_YUV48_12:
 	case V4L2_PIX_FMT_BGR24:
+	case V4L2_PIX_FMT_BGR48_12:
 	default:
 		sos->components_no = 3;
 		break;
 	case V4L2_PIX_FMT_ABGR32:
+	case V4L2_PIX_FMT_ABGR64_12:
 		sos->components_no = 4;
 		break;
 	case V4L2_PIX_FMT_GREY:
+	case V4L2_PIX_FMT_Y012:
 		sos->components_no = 1;
 		break;
 	}
@@ -847,12 +1112,15 @@ static unsigned int mxc_jpeg_setup_cfg_stream(void *cfg_stream_vaddr,
 	u8 *cfg = (u8 *)cfg_stream_vaddr;
 	struct mxc_jpeg_sof *sof;
 	struct mxc_jpeg_sos *sos;
+	const struct mxc_jpeg_fmt *fmt = mxc_jpeg_find_format(fourcc);
+
+	if (!fmt)
+		return 0;
 
 	memcpy(cfg + offset, jpeg_soi, ARRAY_SIZE(jpeg_soi));
 	offset += ARRAY_SIZE(jpeg_soi);
 
-	if (fourcc == V4L2_PIX_FMT_BGR24 ||
-	    fourcc == V4L2_PIX_FMT_ABGR32) {
+	if (fmt->is_rgb) {
 		memcpy(cfg + offset, jpeg_app14, sizeof(jpeg_app14));
 		offset += sizeof(jpeg_app14);
 	} else {
@@ -860,16 +1128,28 @@ static unsigned int mxc_jpeg_setup_cfg_stream(void *cfg_stream_vaddr,
 		offset += sizeof(jpeg_app0);
 	}
 
-	memcpy(cfg + offset, jpeg_dqt, sizeof(jpeg_dqt));
-	offset += sizeof(jpeg_dqt);
+	if (mxc_jpeg_is_extended_sequential(fmt)) {
+		memcpy(cfg + offset, jpeg_dqt_extseq, sizeof(jpeg_dqt_extseq));
+		offset += sizeof(jpeg_dqt_extseq);
 
-	memcpy(cfg + offset, jpeg_sof_maximal, sizeof(jpeg_sof_maximal));
+		memcpy(cfg + offset, jpeg_sof_extseq, sizeof(jpeg_sof_extseq));
+	} else {
+		memcpy(cfg + offset, jpeg_dqt, sizeof(jpeg_dqt));
+		offset += sizeof(jpeg_dqt);
+
+		memcpy(cfg + offset, jpeg_sof_maximal, sizeof(jpeg_sof_maximal));
+	}
 	offset += 2; /* skip marker ID */
 	sof = (struct mxc_jpeg_sof *)(cfg + offset);
 	offset += mxc_jpeg_fixup_sof(sof, fourcc, w, h);
 
-	memcpy(cfg + offset, jpeg_dht, sizeof(jpeg_dht));
-	offset += sizeof(jpeg_dht);
+	if (mxc_jpeg_is_extended_sequential(fmt)) {
+		memcpy(cfg + offset, jpeg_dht_extseq, sizeof(jpeg_dht_extseq));
+		offset += sizeof(jpeg_dht_extseq);
+	} else {
+		memcpy(cfg + offset, jpeg_dht, sizeof(jpeg_dht));
+		offset += sizeof(jpeg_dht);
+	}
 
 	memcpy(cfg + offset, jpeg_dri, sizeof(jpeg_dri));
 	offset += sizeof(jpeg_dri);
@@ -918,6 +1198,10 @@ static void mxc_jpeg_config_dec_desc(struct vb2_buffer *out_buf,
 	desc->stm_ctrl &= ~STM_CTRL_IMAGE_FORMAT(0xF); /* clear image format */
 	desc->stm_ctrl |= STM_CTRL_IMAGE_FORMAT(img_fmt);
 	desc->stm_ctrl |= STM_CTRL_BITBUF_PTR_CLR(1);
+	if (mxc_jpeg_is_extended_sequential(jpeg_src_buf->fmt))
+		desc->stm_ctrl |= STM_CTRL_PIXEL_PRECISION;
+	else
+		desc->stm_ctrl &= ~STM_CTRL_PIXEL_PRECISION;
 	desc->line_pitch = q_data_cap->bytesperline[0];
 	mxc_jpeg_addrs(desc, dst_buf, src_buf, 0);
 	mxc_jpeg_set_bufsize(desc, ALIGN(vb2_plane_size(src_buf, 0), 1024));
@@ -1006,6 +1290,10 @@ static void mxc_jpeg_config_enc_desc(struct vb2_buffer *out_buf,
 	desc->stm_ctrl = STM_CTRL_CONFIG_MOD(0) |
 			 STM_CTRL_IMAGE_FORMAT(img_fmt);
 	desc->stm_ctrl |= STM_CTRL_BITBUF_PTR_CLR(1);
+	if (mxc_jpeg_is_extended_sequential(q_data->fmt))
+		desc->stm_ctrl |= STM_CTRL_PIXEL_PRECISION;
+	else
+		desc->stm_ctrl &= ~STM_CTRL_PIXEL_PRECISION;
 	mxc_jpeg_addrs(desc, src_buf, dst_buf, 0);
 	dev_dbg(jpeg->dev, "cfg_desc:\n");
 	print_descriptor_info(jpeg->dev, cfg_desc);
@@ -1206,7 +1494,9 @@ static void mxc_jpeg_device_run(void *priv)
 		ctx->enc_state = MXC_JPEG_ENC_CONF;
 		mxc_jpeg_config_enc_desc(&dst_buf->vb2_buf, ctx,
 					 &src_buf->vb2_buf, &dst_buf->vb2_buf);
-		mxc_jpeg_enc_mode_conf(dev, reg); /* start config phase */
+		/* start config phase */
+		mxc_jpeg_enc_mode_conf(dev, reg,
+				       mxc_jpeg_is_extended_sequential(q_data_out->fmt));
 	} else {
 		dev_dbg(dev, "Decoding on slot %d\n", ctx->slot);
 		print_mxc_buf(jpeg, &src_buf->vb2_buf, 0);
@@ -1562,7 +1852,7 @@ static int mxc_jpeg_parse(struct mxc_jpeg_ctx *ctx, struct vb2_buffer *vb)
 	if (fourcc == 0)
 		return -EINVAL;
 
-	jpeg_src_buf->fmt = mxc_jpeg_find_format(ctx, fourcc);
+	jpeg_src_buf->fmt = mxc_jpeg_find_format(fourcc);
 	jpeg_src_buf->w = header.frame.width;
 	jpeg_src_buf->h = header.frame.height;
 	ctx->header_parsed = true;
@@ -1702,11 +1992,11 @@ static void mxc_jpeg_set_default_params(struct mxc_jpeg_ctx *ctx)
 	int i;
 
 	if (ctx->mxc_jpeg->mode == MXC_JPEG_ENCODE) {
-		out_q->fmt = mxc_jpeg_find_format(ctx, MXC_JPEG_DEFAULT_PFMT);
-		cap_q->fmt = mxc_jpeg_find_format(ctx, V4L2_PIX_FMT_JPEG);
+		out_q->fmt = mxc_jpeg_find_format(MXC_JPEG_DEFAULT_PFMT);
+		cap_q->fmt = mxc_jpeg_find_format(V4L2_PIX_FMT_JPEG);
 	} else {
-		out_q->fmt = mxc_jpeg_find_format(ctx, V4L2_PIX_FMT_JPEG);
-		cap_q->fmt = mxc_jpeg_find_format(ctx, MXC_JPEG_DEFAULT_PFMT);
+		out_q->fmt = mxc_jpeg_find_format(V4L2_PIX_FMT_JPEG);
+		cap_q->fmt = mxc_jpeg_find_format(MXC_JPEG_DEFAULT_PFMT);
 	}
 
 	for (i = 0; i < 2; i++) {
@@ -1950,7 +2240,7 @@ static int mxc_jpeg_try_fmt(struct v4l2_format *f,
 		 pix_mp->height : MXC_JPEG_MAX_HEIGHT;
 	int i;
 
-	fmt = mxc_jpeg_find_format(ctx, fourcc);
+	fmt = mxc_jpeg_find_format(fourcc);
 	if (!fmt || fmt->flags != mxc_jpeg_get_fmt_type(ctx, f->type)) {
 		dev_warn(ctx->mxc_jpeg->dev, "Format not supported: %c%c%c%c, use the default.\n",
 			 (fourcc & 0xff),
@@ -1958,7 +2248,7 @@ static int mxc_jpeg_try_fmt(struct v4l2_format *f,
 			 (fourcc >> 16) & 0xff,
 			 (fourcc >> 24) & 0xff);
 		fourcc = mxc_jpeg_get_default_fourcc(ctx, f->type);
-		fmt = mxc_jpeg_find_format(ctx, fourcc);
+		fmt = mxc_jpeg_find_format(fourcc);
 		if (!fmt)
 			return -EINVAL;
 		f->fmt.pix_mp.pixelformat = fourcc;
@@ -2622,7 +2912,7 @@ static const struct dev_pm_ops	mxc_jpeg_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(mxc_jpeg_suspend, mxc_jpeg_resume)
 };
 
-static int mxc_jpeg_remove(struct platform_device *pdev)
+static void mxc_jpeg_remove(struct platform_device *pdev)
 {
 	unsigned int slot;
 	struct mxc_jpeg_dev *jpeg = platform_get_drvdata(pdev);
@@ -2635,15 +2925,13 @@ static int mxc_jpeg_remove(struct platform_device *pdev)
 	v4l2_m2m_release(jpeg->m2m_dev);
 	v4l2_device_unregister(&jpeg->v4l2_dev);
 	mxc_jpeg_detach_pm_domains(jpeg);
-
-	return 0;
 }
 
 MODULE_DEVICE_TABLE(of, mxc_jpeg_match);
 
 static struct platform_driver mxc_jpeg_driver = {
 	.probe = mxc_jpeg_probe,
-	.remove = mxc_jpeg_remove,
+	.remove_new = mxc_jpeg_remove,
 	.driver = {
 		.name = "mxc-jpeg",
 		.of_match_table = mxc_jpeg_match,
diff --git a/drivers/media/platform/nxp/imx-mipi-csis.c b/drivers/media/platform/nxp/imx-mipi-csis.c
index be2768a47995..05d52762e792 100644
--- a/drivers/media/platform/nxp/imx-mipi-csis.c
+++ b/drivers/media/platform/nxp/imx-mipi-csis.c
@@ -1503,7 +1503,7 @@ err_disable_clock:
 	return ret;
 }
 
-static int mipi_csis_remove(struct platform_device *pdev)
+static void mipi_csis_remove(struct platform_device *pdev)
 {
 	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
 	struct mipi_csis_device *csis = sd_to_mipi_csis_device(sd);
@@ -1520,8 +1520,6 @@ static int mipi_csis_remove(struct platform_device *pdev)
 	media_entity_cleanup(&csis->sd.entity);
 	fwnode_handle_put(csis->sd.fwnode);
 	pm_runtime_set_suspended(&pdev->dev);
-
-	return 0;
 }
 
 static const struct of_device_id mipi_csis_of_match[] = {
@@ -1544,7 +1542,7 @@ MODULE_DEVICE_TABLE(of, mipi_csis_of_match);
 
 static struct platform_driver mipi_csis_driver = {
 	.probe		= mipi_csis_probe,
-	.remove		= mipi_csis_remove,
+	.remove_new	= mipi_csis_remove,
 	.driver		= {
 		.of_match_table = mipi_csis_of_match,
 		.name		= CSIS_DRIVER_NAME,
diff --git a/drivers/media/platform/nxp/imx-pxp.c b/drivers/media/platform/nxp/imx-pxp.c
index fde3c36e5e1d..90f319857c23 100644
--- a/drivers/media/platform/nxp/imx-pxp.c
+++ b/drivers/media/platform/nxp/imx-pxp.c
@@ -1904,7 +1904,7 @@ err_clk:
 	return ret;
 }
 
-static int pxp_remove(struct platform_device *pdev)
+static void pxp_remove(struct platform_device *pdev)
 {
 	struct pxp_dev *dev = platform_get_drvdata(pdev);
 
@@ -1922,8 +1922,6 @@ static int pxp_remove(struct platform_device *pdev)
 	video_unregister_device(&dev->vfd);
 	v4l2_m2m_release(dev->m2m_dev);
 	v4l2_device_unregister(&dev->v4l2_dev);
-
-	return 0;
 }
 
 static const struct pxp_pdata pxp_imx6ull_pdata = {
@@ -1943,7 +1941,7 @@ MODULE_DEVICE_TABLE(of, pxp_dt_ids);
 
 static struct platform_driver pxp_driver = {
 	.probe		= pxp_probe,
-	.remove		= pxp_remove,
+	.remove_new	= pxp_remove,
 	.driver		= {
 		.name	= MEM2MEM_NAME,
 		.of_match_table = pxp_dt_ids,
diff --git a/drivers/media/platform/nxp/imx7-media-csi.c b/drivers/media/platform/nxp/imx7-media-csi.c
index c22bf5c827e7..b701e823436a 100644
--- a/drivers/media/platform/nxp/imx7-media-csi.c
+++ b/drivers/media/platform/nxp/imx7-media-csi.c
@@ -1214,6 +1214,9 @@ static int imx7_csi_video_g_selection(struct file *file, void *fh,
 {
 	struct imx7_csi *csi = video_drvdata(file);
 
+	if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
 	switch (s->target) {
 	case V4L2_SEL_TGT_COMPOSE:
 	case V4L2_SEL_TGT_COMPOSE_DEFAULT:
@@ -1610,6 +1613,7 @@ static int imx7_csi_video_init_format(struct imx7_csi *csi)
 	format.code = IMX7_CSI_DEF_MBUS_CODE;
 	format.width = IMX7_CSI_DEF_PIX_WIDTH;
 	format.height = IMX7_CSI_DEF_PIX_HEIGHT;
+	format.field = V4L2_FIELD_NONE;
 
 	imx7_csi_mbus_fmt_to_pix_fmt(&csi->vdev_fmt, &format, NULL);
 	csi->vdev_compose.width = format.width;
@@ -2107,18 +2111,21 @@ static int imx7_csi_async_register(struct imx7_csi *csi)
 
 	ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(csi->dev), 0, 0,
 					     FWNODE_GRAPH_ENDPOINT_NEXT);
-	if (ep) {
-		asd = v4l2_async_nf_add_fwnode_remote(&csi->notifier, ep,
-						      struct v4l2_async_subdev);
+	if (!ep) {
+		ret = dev_err_probe(csi->dev, -ENOTCONN,
+				    "Failed to get remote endpoint\n");
+		goto error;
+	}
 
-		fwnode_handle_put(ep);
+	asd = v4l2_async_nf_add_fwnode_remote(&csi->notifier, ep,
+					      struct v4l2_async_subdev);
 
-		if (IS_ERR(asd)) {
-			ret = PTR_ERR(asd);
-			/* OK if asd already exists */
-			if (ret != -EEXIST)
-				goto error;
-		}
+	fwnode_handle_put(ep);
+
+	if (IS_ERR(asd)) {
+		ret = dev_err_probe(csi->dev, PTR_ERR(asd),
+				    "Failed to add remote subdev to notifier\n");
+		goto error;
 	}
 
 	csi->notifier.ops = &imx7_csi_notify_ops;
@@ -2278,7 +2285,7 @@ err_media_cleanup:
 	return ret;
 }
 
-static int imx7_csi_remove(struct platform_device *pdev)
+static void imx7_csi_remove(struct platform_device *pdev)
 {
 	struct imx7_csi *csi = platform_get_drvdata(pdev);
 
@@ -2287,8 +2294,6 @@ static int imx7_csi_remove(struct platform_device *pdev)
 	v4l2_async_nf_unregister(&csi->notifier);
 	v4l2_async_nf_cleanup(&csi->notifier);
 	v4l2_async_unregister_subdev(&csi->sd);
-
-	return 0;
 }
 
 static const struct of_device_id imx7_csi_of_match[] = {
@@ -2301,7 +2306,7 @@ MODULE_DEVICE_TABLE(of, imx7_csi_of_match);
 
 static struct platform_driver imx7_csi_driver = {
 	.probe = imx7_csi_probe,
-	.remove = imx7_csi_remove,
+	.remove_new = imx7_csi_remove,
 	.driver = {
 		.of_match_table = imx7_csi_of_match,
 		.name = "imx7-csi",
diff --git a/drivers/media/platform/nxp/imx8-isi/Kconfig b/drivers/media/platform/nxp/imx8-isi/Kconfig
new file mode 100644
index 000000000000..fcff33fc2630
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/Kconfig
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+config VIDEO_IMX8_ISI
+	tristate "i.MX8 Image Sensor Interface (ISI) driver"
+	depends on ARCH_MXC || COMPILE_TEST
+	depends on HAS_DMA && PM
+	depends on VIDEO_DEV
+	select MEDIA_CONTROLLER
+	select V4L2_FWNODE
+	select V4L2_MEM2MEM_DEV if VIDEO_IMX8_ISI_M2M
+	select VIDEO_V4L2_SUBDEV_API
+	select VIDEOBUF2_DMA_CONTIG
+	help
+	  V4L2 driver for the Image Sensor Interface (ISI) found in various
+	  i.MX8 SoCs.
+
+config VIDEO_IMX8_ISI_M2M
+	bool "i.MX8 Image Sensor Interface (ISI) memory-to-memory support"
+	depends on VIDEO_IMX8_ISI
+	help
+	  Select 'yes' here to enable support for memory-to-memory processing
+	  in the ISI driver.
diff --git a/drivers/media/platform/nxp/imx8-isi/Makefile b/drivers/media/platform/nxp/imx8-isi/Makefile
new file mode 100644
index 000000000000..9bff9297686d
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+imx8-isi-y := imx8-isi-core.o imx8-isi-crossbar.o imx8-isi-hw.o \
+	imx8-isi-pipe.o imx8-isi-video.o
+imx8-isi-$(CONFIG_DEBUG_FS) += imx8-isi-debug.o
+imx8-isi-$(CONFIG_VIDEO_IMX8_ISI_M2M) += imx8-isi-m2m.o
+
+obj-$(CONFIG_VIDEO_IMX8_ISI) += imx8-isi.o
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c
new file mode 100644
index 000000000000..238521622b75
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c
@@ -0,0 +1,539 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019-2020 NXP
+ */
+
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/pm_runtime.h>
+#include <linux/property.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+#include <media/media-device.h>
+#include <media/v4l2-async.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-mc.h>
+
+#include "imx8-isi-core.h"
+
+/* -----------------------------------------------------------------------------
+ * V4L2 async subdevs
+ */
+
+struct mxc_isi_async_subdev {
+	struct v4l2_async_subdev asd;
+	unsigned int port;
+};
+
+static inline struct mxc_isi_async_subdev *
+asd_to_mxc_isi_async_subdev(struct v4l2_async_subdev *asd)
+{
+	return container_of(asd, struct mxc_isi_async_subdev, asd);
+};
+
+static inline struct mxc_isi_dev *
+notifier_to_mxc_isi_dev(struct v4l2_async_notifier *n)
+{
+	return container_of(n, struct mxc_isi_dev, notifier);
+};
+
+static int mxc_isi_async_notifier_bound(struct v4l2_async_notifier *notifier,
+					struct v4l2_subdev *sd,
+					struct v4l2_async_subdev *asd)
+{
+	const unsigned int link_flags = MEDIA_LNK_FL_IMMUTABLE
+				      | MEDIA_LNK_FL_ENABLED;
+	struct mxc_isi_dev *isi = notifier_to_mxc_isi_dev(notifier);
+	struct mxc_isi_async_subdev *masd = asd_to_mxc_isi_async_subdev(asd);
+	struct media_pad *pad = &isi->crossbar.pads[masd->port];
+	struct device_link *link;
+
+	dev_dbg(isi->dev, "Bound subdev %s to crossbar input %u\n", sd->name,
+		masd->port);
+
+	/*
+	 * Enforce suspend/resume ordering between the source (supplier) and
+	 * the ISI (consumer). The source will be suspended before and resume
+	 * after the ISI.
+	 */
+	link = device_link_add(isi->dev, sd->dev, DL_FLAG_STATELESS);
+	if (!link) {
+		dev_err(isi->dev,
+			"Failed to create device link to source %s\n", sd->name);
+		return -EINVAL;
+	}
+
+	return v4l2_create_fwnode_links_to_pad(sd, pad, link_flags);
+}
+
+static int mxc_isi_async_notifier_complete(struct v4l2_async_notifier *notifier)
+{
+	struct mxc_isi_dev *isi = notifier_to_mxc_isi_dev(notifier);
+	int ret;
+
+	dev_dbg(isi->dev, "All subdevs bound\n");
+
+	ret = v4l2_device_register_subdev_nodes(&isi->v4l2_dev);
+	if (ret < 0) {
+		dev_err(isi->dev,
+			"Failed to register subdev nodes: %d\n", ret);
+		return ret;
+	}
+
+	return media_device_register(&isi->media_dev);
+}
+
+static const struct v4l2_async_notifier_operations mxc_isi_async_notifier_ops = {
+	.bound = mxc_isi_async_notifier_bound,
+	.complete = mxc_isi_async_notifier_complete,
+};
+
+static int mxc_isi_pipe_register(struct mxc_isi_pipe *pipe)
+{
+	int ret;
+
+	ret = v4l2_device_register_subdev(&pipe->isi->v4l2_dev, &pipe->sd);
+	if (ret < 0)
+		return ret;
+
+	return mxc_isi_video_register(pipe, &pipe->isi->v4l2_dev);
+}
+
+static void mxc_isi_pipe_unregister(struct mxc_isi_pipe *pipe)
+{
+	mxc_isi_video_unregister(pipe);
+}
+
+static int mxc_isi_v4l2_init(struct mxc_isi_dev *isi)
+{
+	struct fwnode_handle *node = dev_fwnode(isi->dev);
+	struct media_device *media_dev = &isi->media_dev;
+	struct v4l2_device *v4l2_dev = &isi->v4l2_dev;
+	unsigned int i;
+	int ret;
+
+	/* Initialize the media device. */
+	strscpy(media_dev->model, "FSL Capture Media Device",
+		sizeof(media_dev->model));
+	media_dev->dev = isi->dev;
+
+	media_device_init(media_dev);
+
+	/* Initialize and register the V4L2 device. */
+	v4l2_dev->mdev = media_dev;
+	strscpy(v4l2_dev->name, "mx8-img-md", sizeof(v4l2_dev->name));
+
+	ret = v4l2_device_register(isi->dev, v4l2_dev);
+	if (ret < 0) {
+		dev_err(isi->dev,
+			"Failed to register V4L2 device: %d\n", ret);
+		goto err_media;
+	}
+
+	/* Register the crossbar switch subdev. */
+	ret = mxc_isi_crossbar_register(&isi->crossbar);
+	if (ret < 0) {
+		dev_err(isi->dev, "Failed to register crossbar: %d\n", ret);
+		goto err_v4l2;
+	}
+
+	/* Register the pipeline subdevs and link them to the crossbar switch. */
+	for (i = 0; i < isi->pdata->num_channels; ++i) {
+		struct mxc_isi_pipe *pipe = &isi->pipes[i];
+
+		ret = mxc_isi_pipe_register(pipe);
+		if (ret < 0) {
+			dev_err(isi->dev, "Failed to register pipe%u: %d\n", i,
+				ret);
+			goto err_v4l2;
+		}
+
+		ret = media_create_pad_link(&isi->crossbar.sd.entity,
+					    isi->crossbar.num_sinks + i,
+					    &pipe->sd.entity,
+					    MXC_ISI_PIPE_PAD_SINK,
+					    MEDIA_LNK_FL_IMMUTABLE |
+					    MEDIA_LNK_FL_ENABLED);
+		if (ret < 0)
+			goto err_v4l2;
+	}
+
+	/* Register the M2M device. */
+	ret = mxc_isi_m2m_register(isi, v4l2_dev);
+	if (ret < 0) {
+		dev_err(isi->dev, "Failed to register M2M device: %d\n", ret);
+		goto err_v4l2;
+	}
+
+	/* Initialize, fill and register the async notifier. */
+	v4l2_async_nf_init(&isi->notifier);
+	isi->notifier.ops = &mxc_isi_async_notifier_ops;
+
+	for (i = 0; i < isi->pdata->num_ports; ++i) {
+		struct mxc_isi_async_subdev *masd;
+		struct fwnode_handle *ep;
+
+		ep = fwnode_graph_get_endpoint_by_id(node, i, 0,
+						     FWNODE_GRAPH_ENDPOINT_NEXT);
+
+		if (!ep)
+			continue;
+
+		masd = v4l2_async_nf_add_fwnode_remote(&isi->notifier, ep,
+						       struct mxc_isi_async_subdev);
+		fwnode_handle_put(ep);
+
+		if (IS_ERR(masd)) {
+			ret = PTR_ERR(masd);
+			goto err_m2m;
+		}
+
+		masd->port = i;
+	}
+
+	ret = v4l2_async_nf_register(v4l2_dev, &isi->notifier);
+	if (ret < 0) {
+		dev_err(isi->dev,
+			"Failed to register async notifier: %d\n", ret);
+		goto err_m2m;
+	}
+
+	return 0;
+
+err_m2m:
+	mxc_isi_m2m_unregister(isi);
+	v4l2_async_nf_cleanup(&isi->notifier);
+err_v4l2:
+	v4l2_device_unregister(v4l2_dev);
+err_media:
+	media_device_cleanup(media_dev);
+	return ret;
+}
+
+static void mxc_isi_v4l2_cleanup(struct mxc_isi_dev *isi)
+{
+	unsigned int i;
+
+	v4l2_async_nf_unregister(&isi->notifier);
+	v4l2_async_nf_cleanup(&isi->notifier);
+
+	v4l2_device_unregister(&isi->v4l2_dev);
+	media_device_unregister(&isi->media_dev);
+
+	mxc_isi_m2m_unregister(isi);
+
+	for (i = 0; i < isi->pdata->num_channels; ++i)
+		mxc_isi_pipe_unregister(&isi->pipes[i]);
+
+	mxc_isi_crossbar_unregister(&isi->crossbar);
+
+	media_device_cleanup(&isi->media_dev);
+}
+
+/* -----------------------------------------------------------------------------
+ * Device information
+ */
+
+/* Panic will assert when the buffers are 50% full */
+
+/* For i.MX8QXP C0 and i.MX8MN ISI IER version */
+static const struct mxc_isi_ier_reg mxc_imx8_isi_ier_v1 = {
+	.oflw_y_buf_en = { .offset = 19, .mask = 0x80000  },
+	.oflw_u_buf_en = { .offset = 21, .mask = 0x200000 },
+	.oflw_v_buf_en = { .offset = 23, .mask = 0x800000 },
+
+	.panic_y_buf_en = {.offset = 20, .mask = 0x100000  },
+	.panic_u_buf_en = {.offset = 22, .mask = 0x400000  },
+	.panic_v_buf_en = {.offset = 24, .mask = 0x1000000 },
+};
+
+/* For i.MX8MP ISI IER version */
+static const struct mxc_isi_ier_reg mxc_imx8_isi_ier_v2 = {
+	.oflw_y_buf_en = { .offset = 18, .mask = 0x40000  },
+	.oflw_u_buf_en = { .offset = 20, .mask = 0x100000 },
+	.oflw_v_buf_en = { .offset = 22, .mask = 0x400000 },
+
+	.panic_y_buf_en = {.offset = 19, .mask = 0x80000  },
+	.panic_u_buf_en = {.offset = 21, .mask = 0x200000 },
+	.panic_v_buf_en = {.offset = 23, .mask = 0x800000 },
+};
+
+/* Panic will assert when the buffers are 50% full */
+static const struct mxc_isi_set_thd mxc_imx8_isi_thd_v1 = {
+	.panic_set_thd_y = { .mask = 0x0000f, .offset = 0,  .threshold = 0x7 },
+	.panic_set_thd_u = { .mask = 0x00f00, .offset = 8,  .threshold = 0x7 },
+	.panic_set_thd_v = { .mask = 0xf0000, .offset = 16, .threshold = 0x7 },
+};
+
+static const struct clk_bulk_data mxc_imx8mn_clks[] = {
+	{ .id = "axi" },
+	{ .id = "apb" },
+};
+
+static const struct mxc_isi_plat_data mxc_imx8mn_data = {
+	.model			= MXC_ISI_IMX8MN,
+	.num_ports		= 1,
+	.num_channels		= 1,
+	.reg_offset		= 0,
+	.ier_reg		= &mxc_imx8_isi_ier_v1,
+	.set_thd		= &mxc_imx8_isi_thd_v1,
+	.clks			= mxc_imx8mn_clks,
+	.num_clks		= ARRAY_SIZE(mxc_imx8mn_clks),
+	.buf_active_reverse	= false,
+	.has_gasket		= true,
+	.has_36bit_dma		= false,
+};
+
+static const struct mxc_isi_plat_data mxc_imx8mp_data = {
+	.model			= MXC_ISI_IMX8MP,
+	.num_ports		= 2,
+	.num_channels		= 2,
+	.reg_offset		= 0x2000,
+	.ier_reg		= &mxc_imx8_isi_ier_v2,
+	.set_thd		= &mxc_imx8_isi_thd_v1,
+	.clks			= mxc_imx8mn_clks,
+	.num_clks		= ARRAY_SIZE(mxc_imx8mn_clks),
+	.buf_active_reverse	= true,
+	.has_gasket		= true,
+	.has_36bit_dma		= true,
+};
+
+/* -----------------------------------------------------------------------------
+ * Power management
+ */
+
+static int mxc_isi_pm_suspend(struct device *dev)
+{
+	struct mxc_isi_dev *isi = dev_get_drvdata(dev);
+	unsigned int i;
+
+	for (i = 0; i < isi->pdata->num_channels; ++i) {
+		struct mxc_isi_pipe *pipe = &isi->pipes[i];
+
+		mxc_isi_video_suspend(pipe);
+	}
+
+	return pm_runtime_force_suspend(dev);
+}
+
+static int mxc_isi_pm_resume(struct device *dev)
+{
+	struct mxc_isi_dev *isi = dev_get_drvdata(dev);
+	unsigned int i;
+	int err = 0;
+	int ret;
+
+	ret = pm_runtime_force_resume(dev);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < isi->pdata->num_channels; ++i) {
+		struct mxc_isi_pipe *pipe = &isi->pipes[i];
+
+		ret = mxc_isi_video_resume(pipe);
+		if (ret) {
+			dev_err(dev, "Failed to resume pipeline %u (%d)\n", i,
+				ret);
+			/*
+			 * Record the last error as it's as meaningful as any,
+			 * and continue resuming the other pipelines.
+			 */
+			err = ret;
+		}
+	}
+
+	return err;
+}
+
+static int mxc_isi_runtime_suspend(struct device *dev)
+{
+	struct mxc_isi_dev *isi = dev_get_drvdata(dev);
+
+	clk_bulk_disable_unprepare(isi->pdata->num_clks, isi->clks);
+
+	return 0;
+}
+
+static int mxc_isi_runtime_resume(struct device *dev)
+{
+	struct mxc_isi_dev *isi = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_bulk_prepare_enable(isi->pdata->num_clks, isi->clks);
+	if (ret) {
+		dev_err(dev, "Failed to enable clocks (%d)\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct dev_pm_ops mxc_isi_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(mxc_isi_pm_suspend, mxc_isi_pm_resume)
+	SET_RUNTIME_PM_OPS(mxc_isi_runtime_suspend, mxc_isi_runtime_resume, NULL)
+};
+
+/* -----------------------------------------------------------------------------
+ * Probe, remove & driver
+ */
+
+static int mxc_isi_clk_get(struct mxc_isi_dev *isi)
+{
+	unsigned int size = isi->pdata->num_clks
+			  * sizeof(*isi->clks);
+	int ret;
+
+	isi->clks = devm_kmalloc(isi->dev, size, GFP_KERNEL);
+	if (!isi->clks)
+		return -ENOMEM;
+
+	memcpy(isi->clks, isi->pdata->clks, size);
+
+	ret = devm_clk_bulk_get(isi->dev, isi->pdata->num_clks,
+				isi->clks);
+	if (ret < 0) {
+		dev_err(isi->dev, "Failed to acquire clocks: %d\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int mxc_isi_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mxc_isi_dev *isi;
+	unsigned int dma_size;
+	unsigned int i;
+	int ret = 0;
+
+	isi = devm_kzalloc(dev, sizeof(*isi), GFP_KERNEL);
+	if (!isi)
+		return -ENOMEM;
+
+	isi->dev = dev;
+	platform_set_drvdata(pdev, isi);
+
+	isi->pdata = of_device_get_match_data(dev);
+
+	isi->pipes = kcalloc(isi->pdata->num_channels, sizeof(isi->pipes[0]),
+			     GFP_KERNEL);
+	if (!isi->pipes)
+		return -ENOMEM;
+
+	ret = mxc_isi_clk_get(isi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to get clocks\n");
+		return ret;
+	}
+
+	isi->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(isi->regs)) {
+		dev_err(dev, "Failed to get ISI register map\n");
+		return PTR_ERR(isi->regs);
+	}
+
+	if (isi->pdata->has_gasket) {
+		isi->gasket = syscon_regmap_lookup_by_phandle(dev->of_node,
+							      "fsl,blk-ctrl");
+		if (IS_ERR(isi->gasket)) {
+			ret = PTR_ERR(isi->gasket);
+			dev_err(dev, "failed to get gasket: %d\n", ret);
+			return ret;
+		}
+	}
+
+	dma_size = isi->pdata->has_36bit_dma ? 36 : 32;
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(dma_size));
+	if (ret) {
+		dev_err(dev, "failed to set DMA mask\n");
+		return ret;
+	}
+
+	pm_runtime_enable(dev);
+
+	ret = mxc_isi_crossbar_init(isi);
+	if (ret) {
+		dev_err(dev, "Failed to initialize crossbar: %d\n", ret);
+		goto err_pm;
+	}
+
+	for (i = 0; i < isi->pdata->num_channels; ++i) {
+		ret = mxc_isi_pipe_init(isi, i);
+		if (ret < 0) {
+			dev_err(dev, "Failed to initialize pipe%u: %d\n", i,
+				ret);
+			goto err_xbar;
+		}
+	}
+
+	ret = mxc_isi_v4l2_init(isi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to initialize V4L2: %d\n", ret);
+		goto err_xbar;
+	}
+
+	mxc_isi_debug_init(isi);
+
+	return 0;
+
+err_xbar:
+	mxc_isi_crossbar_cleanup(&isi->crossbar);
+err_pm:
+	pm_runtime_disable(isi->dev);
+	return ret;
+}
+
+static int mxc_isi_remove(struct platform_device *pdev)
+{
+	struct mxc_isi_dev *isi = platform_get_drvdata(pdev);
+	unsigned int i;
+
+	mxc_isi_debug_cleanup(isi);
+
+	for (i = 0; i < isi->pdata->num_channels; ++i) {
+		struct mxc_isi_pipe *pipe = &isi->pipes[i];
+
+		mxc_isi_pipe_cleanup(pipe);
+	}
+
+	mxc_isi_crossbar_cleanup(&isi->crossbar);
+	mxc_isi_v4l2_cleanup(isi);
+
+	pm_runtime_disable(isi->dev);
+
+	return 0;
+}
+
+static const struct of_device_id mxc_isi_of_match[] = {
+	{ .compatible = "fsl,imx8mn-isi", .data = &mxc_imx8mn_data },
+	{ .compatible = "fsl,imx8mp-isi", .data = &mxc_imx8mp_data },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, mxc_isi_of_match);
+
+static struct platform_driver mxc_isi_driver = {
+	.probe		= mxc_isi_probe,
+	.remove		= mxc_isi_remove,
+	.driver = {
+		.of_match_table = mxc_isi_of_match,
+		.name		= MXC_ISI_DRIVER_NAME,
+		.pm		= &mxc_isi_pm_ops,
+	}
+};
+module_platform_driver(mxc_isi_driver);
+
+MODULE_ALIAS("ISI");
+MODULE_AUTHOR("Freescale Semiconductor, Inc.");
+MODULE_DESCRIPTION("IMX8 Image Sensing Interface driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h
new file mode 100644
index 000000000000..e469788a9e6c
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h
@@ -0,0 +1,394 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * V4L2 Capture ISI subdev for i.MX8QXP/QM platform
+ *
+ * ISI is a Image Sensor Interface of i.MX8QXP/QM platform, which
+ * used to process image from camera sensor to memory or DC
+ * Copyright 2019-2020 NXP
+ */
+
+#ifndef __MXC_ISI_CORE_H__
+#define __MXC_ISI_CORE_H__
+
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+#include <linux/videodev2.h>
+
+#include <media/media-device.h>
+#include <media/media-entity.h>
+#include <media/v4l2-async.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-v4l2.h>
+
+struct clk_bulk_data;
+struct dentry;
+struct device;
+struct media_intf_devnode;
+struct regmap;
+struct v4l2_m2m_dev;
+
+/* Pipeline pads */
+#define MXC_ISI_PIPE_PAD_SINK		0
+#define MXC_ISI_PIPE_PAD_SOURCE		1
+#define MXC_ISI_PIPE_PADS_NUM		2
+
+#define MXC_ISI_MIN_WIDTH		1U
+#define MXC_ISI_MIN_HEIGHT		1U
+#define MXC_ISI_MAX_WIDTH_UNCHAINED	2048U
+#define MXC_ISI_MAX_WIDTH_CHAINED	4096U
+#define MXC_ISI_MAX_HEIGHT		8191U
+
+#define MXC_ISI_DEF_WIDTH		1920U
+#define MXC_ISI_DEF_HEIGHT		1080U
+#define MXC_ISI_DEF_MBUS_CODE_SINK	MEDIA_BUS_FMT_UYVY8_1X16
+#define MXC_ISI_DEF_MBUS_CODE_SOURCE	MEDIA_BUS_FMT_YUV8_1X24
+#define MXC_ISI_DEF_PIXEL_FORMAT	V4L2_PIX_FMT_YUYV
+#define MXC_ISI_DEF_COLOR_SPACE		V4L2_COLORSPACE_SRGB
+#define MXC_ISI_DEF_YCBCR_ENC		V4L2_YCBCR_ENC_601
+#define MXC_ISI_DEF_QUANTIZATION	V4L2_QUANTIZATION_LIM_RANGE
+#define MXC_ISI_DEF_XFER_FUNC		V4L2_XFER_FUNC_SRGB
+
+#define MXC_ISI_DRIVER_NAME		"mxc-isi"
+#define MXC_ISI_CAPTURE			"mxc-isi-cap"
+#define MXC_ISI_M2M			"mxc-isi-m2m"
+#define MXC_MAX_PLANES			3
+
+struct mxc_isi_dev;
+struct mxc_isi_m2m_ctx;
+
+enum mxc_isi_buf_id {
+	MXC_ISI_BUF1 = 0x0,
+	MXC_ISI_BUF2,
+};
+
+enum mxc_isi_encoding {
+	MXC_ISI_ENC_RAW,
+	MXC_ISI_ENC_RGB,
+	MXC_ISI_ENC_YUV,
+};
+
+enum mxc_isi_input_id {
+	/* Inputs from the crossbar switch range from 0 to 15 */
+	MXC_ISI_INPUT_MEM = 16,
+};
+
+enum mxc_isi_video_type {
+	MXC_ISI_VIDEO_CAP = BIT(0),
+	MXC_ISI_VIDEO_M2M_OUT = BIT(1),
+	MXC_ISI_VIDEO_M2M_CAP = BIT(2),
+};
+
+struct mxc_isi_format_info {
+	u32	mbus_code;
+	u32	fourcc;
+	enum mxc_isi_video_type type;
+	u32	isi_in_format;
+	u32	isi_out_format;
+	u8	mem_planes;
+	u8	color_planes;
+	u8	depth[MXC_MAX_PLANES];
+	u8	hsub;
+	u8	vsub;
+	enum mxc_isi_encoding encoding;
+};
+
+struct mxc_isi_bus_format_info {
+	u32	mbus_code;
+	u32	output;
+	u32	pads;
+	enum mxc_isi_encoding encoding;
+};
+
+struct mxc_isi_buffer {
+	struct vb2_v4l2_buffer  v4l2_buf;
+	struct list_head	list;
+	dma_addr_t		dma_addrs[3];
+	enum mxc_isi_buf_id	id;
+	bool discard;
+};
+
+struct mxc_isi_reg {
+	u32 offset;
+	u32 mask;
+};
+
+struct mxc_isi_ier_reg {
+	/* Overflow Y/U/V trigger enable*/
+	struct mxc_isi_reg oflw_y_buf_en;
+	struct mxc_isi_reg oflw_u_buf_en;
+	struct mxc_isi_reg oflw_v_buf_en;
+
+	/* Excess overflow Y/U/V trigger enable*/
+	struct mxc_isi_reg excs_oflw_y_buf_en;
+	struct mxc_isi_reg excs_oflw_u_buf_en;
+	struct mxc_isi_reg excs_oflw_v_buf_en;
+
+	/* Panic Y/U/V trigger enable*/
+	struct mxc_isi_reg panic_y_buf_en;
+	struct mxc_isi_reg panic_v_buf_en;
+	struct mxc_isi_reg panic_u_buf_en;
+};
+
+struct mxc_isi_panic_thd {
+	u32 mask;
+	u32 offset;
+	u32 threshold;
+};
+
+struct mxc_isi_set_thd {
+	struct mxc_isi_panic_thd panic_set_thd_y;
+	struct mxc_isi_panic_thd panic_set_thd_u;
+	struct mxc_isi_panic_thd panic_set_thd_v;
+};
+
+enum model {
+	MXC_ISI_IMX8MN,
+	MXC_ISI_IMX8MP,
+};
+
+struct mxc_isi_plat_data {
+	enum model model;
+	unsigned int num_ports;
+	unsigned int num_channels;
+	unsigned int reg_offset;
+	const struct mxc_isi_ier_reg  *ier_reg;
+	const struct mxc_isi_set_thd *set_thd;
+	const struct clk_bulk_data *clks;
+	unsigned int num_clks;
+	bool buf_active_reverse;
+	bool has_gasket;
+	bool has_36bit_dma;
+};
+
+struct mxc_isi_dma_buffer {
+	size_t				size;
+	void				*addr;
+	dma_addr_t			dma;
+};
+
+struct mxc_isi_input {
+	unsigned int			enable_count;
+};
+
+struct mxc_isi_crossbar {
+	struct mxc_isi_dev		*isi;
+
+	unsigned int			num_sinks;
+	unsigned int			num_sources;
+	struct mxc_isi_input		*inputs;
+
+	struct v4l2_subdev		sd;
+	struct media_pad		*pads;
+};
+
+struct mxc_isi_video {
+	struct mxc_isi_pipe		*pipe;
+
+	struct video_device		vdev;
+	struct media_pad		pad;
+
+	/* Protects is_streaming, and the vdev and vb2_q operations */
+	struct mutex			lock;
+	bool				is_streaming;
+
+	struct v4l2_pix_format_mplane	pix;
+	const struct mxc_isi_format_info *fmtinfo;
+
+	struct {
+		struct v4l2_ctrl_handler handler;
+		unsigned int		alpha;
+		bool			hflip;
+		bool			vflip;
+	} ctrls;
+
+	struct vb2_queue		vb2_q;
+	struct mxc_isi_buffer		buf_discard[3];
+	struct list_head		out_pending;
+	struct list_head		out_active;
+	struct list_head		out_discard;
+	u32				frame_count;
+	/* Protects out_pending, out_active, out_discard and frame_count */
+	spinlock_t			buf_lock;
+
+	struct mxc_isi_dma_buffer	discard_buffer[MXC_MAX_PLANES];
+};
+
+typedef void(*mxc_isi_pipe_irq_t)(struct mxc_isi_pipe *, u32);
+
+struct mxc_isi_pipe {
+	struct mxc_isi_dev		*isi;
+	u32				id;
+	void __iomem			*regs;
+
+	struct media_pipeline		pipe;
+
+	struct v4l2_subdev		sd;
+	struct media_pad		pads[MXC_ISI_PIPE_PADS_NUM];
+
+	struct mxc_isi_video		video;
+
+	/*
+	 * Protects use_count, irq_handler, res_available, res_acquired,
+	 * chained_res, and the CHNL_CTRL register.
+	 */
+	struct mutex			lock;
+	unsigned int			use_count;
+	mxc_isi_pipe_irq_t		irq_handler;
+
+#define MXC_ISI_CHANNEL_RES_LINE_BUF	BIT(0)
+#define MXC_ISI_CHANNEL_RES_OUTPUT_BUF	BIT(1)
+	u8				available_res;
+	u8				acquired_res;
+	u8				chained_res;
+	bool				chained;
+};
+
+struct mxc_isi_m2m {
+	struct mxc_isi_dev		*isi;
+	struct mxc_isi_pipe		*pipe;
+
+	struct media_pad		pad;
+	struct video_device		vdev;
+	struct media_intf_devnode	*intf;
+	struct v4l2_m2m_dev		*m2m_dev;
+
+	/* Protects last_ctx, usage_count and chained_count */
+	struct mutex			lock;
+
+	struct mxc_isi_m2m_ctx		*last_ctx;
+	int				usage_count;
+	int				chained_count;
+};
+
+struct mxc_isi_dev {
+	struct device			*dev;
+
+	const struct mxc_isi_plat_data	*pdata;
+
+	void __iomem			*regs;
+	struct clk_bulk_data		*clks;
+	struct regmap			*gasket;
+
+	struct mxc_isi_crossbar		crossbar;
+	struct mxc_isi_pipe		*pipes;
+	struct mxc_isi_m2m		m2m;
+
+	struct media_device		media_dev;
+	struct v4l2_device		v4l2_dev;
+	struct v4l2_async_notifier	notifier;
+
+	struct dentry			*debugfs_root;
+};
+
+int mxc_isi_crossbar_init(struct mxc_isi_dev *isi);
+void mxc_isi_crossbar_cleanup(struct mxc_isi_crossbar *xbar);
+int mxc_isi_crossbar_register(struct mxc_isi_crossbar *xbar);
+void mxc_isi_crossbar_unregister(struct mxc_isi_crossbar *xbar);
+
+const struct mxc_isi_bus_format_info *
+mxc_isi_bus_format_by_code(u32 code, unsigned int pad);
+const struct mxc_isi_bus_format_info *
+mxc_isi_bus_format_by_index(unsigned int index, unsigned int pad);
+const struct mxc_isi_format_info *
+mxc_isi_format_by_fourcc(u32 fourcc, enum mxc_isi_video_type type);
+const struct mxc_isi_format_info *
+mxc_isi_format_enum(unsigned int index, enum mxc_isi_video_type type);
+const struct mxc_isi_format_info *
+mxc_isi_format_try(struct mxc_isi_pipe *pipe, struct v4l2_pix_format_mplane *pix,
+		   enum mxc_isi_video_type type);
+
+int mxc_isi_pipe_init(struct mxc_isi_dev *isi, unsigned int id);
+void mxc_isi_pipe_cleanup(struct mxc_isi_pipe *pipe);
+int mxc_isi_pipe_acquire(struct mxc_isi_pipe *pipe,
+			 mxc_isi_pipe_irq_t irq_handler);
+void mxc_isi_pipe_release(struct mxc_isi_pipe *pipe);
+int mxc_isi_pipe_enable(struct mxc_isi_pipe *pipe);
+void mxc_isi_pipe_disable(struct mxc_isi_pipe *pipe);
+
+int mxc_isi_video_register(struct mxc_isi_pipe *pipe,
+			   struct v4l2_device *v4l2_dev);
+void mxc_isi_video_unregister(struct mxc_isi_pipe *pipe);
+void mxc_isi_video_suspend(struct mxc_isi_pipe *pipe);
+int mxc_isi_video_resume(struct mxc_isi_pipe *pipe);
+int mxc_isi_video_queue_setup(const struct v4l2_pix_format_mplane *format,
+			      const struct mxc_isi_format_info *info,
+			      unsigned int *num_buffers,
+			      unsigned int *num_planes, unsigned int sizes[]);
+void mxc_isi_video_buffer_init(struct vb2_buffer *vb2, dma_addr_t dma_addrs[3],
+			       const struct mxc_isi_format_info *info,
+			       const struct v4l2_pix_format_mplane *pix);
+int mxc_isi_video_buffer_prepare(struct mxc_isi_dev *isi, struct vb2_buffer *vb2,
+				 const struct mxc_isi_format_info *info,
+				 const struct v4l2_pix_format_mplane *pix);
+
+#ifdef CONFIG_VIDEO_IMX8_ISI_M2M
+int mxc_isi_m2m_register(struct mxc_isi_dev *isi, struct v4l2_device *v4l2_dev);
+int mxc_isi_m2m_unregister(struct mxc_isi_dev *isi);
+#else
+static inline int mxc_isi_m2m_register(struct mxc_isi_dev *isi,
+				       struct v4l2_device *v4l2_dev)
+{
+	return 0;
+}
+static inline int mxc_isi_m2m_unregister(struct mxc_isi_dev *isi)
+{
+	return 0;
+}
+#endif
+
+int mxc_isi_channel_acquire(struct mxc_isi_pipe *pipe,
+			    mxc_isi_pipe_irq_t irq_handler, bool bypass);
+void mxc_isi_channel_release(struct mxc_isi_pipe *pipe);
+void mxc_isi_channel_get(struct mxc_isi_pipe *pipe);
+void mxc_isi_channel_put(struct mxc_isi_pipe *pipe);
+void mxc_isi_channel_enable(struct mxc_isi_pipe *pipe);
+void mxc_isi_channel_disable(struct mxc_isi_pipe *pipe);
+int mxc_isi_channel_chain(struct mxc_isi_pipe *pipe, bool bypass);
+void mxc_isi_channel_unchain(struct mxc_isi_pipe *pipe);
+
+void mxc_isi_channel_config(struct mxc_isi_pipe *pipe,
+			    enum mxc_isi_input_id input,
+			    const struct v4l2_area *in_size,
+			    const struct v4l2_area *scale,
+			    const struct v4l2_rect *crop,
+			    enum mxc_isi_encoding in_encoding,
+			    enum mxc_isi_encoding out_encoding);
+
+void mxc_isi_channel_set_input_format(struct mxc_isi_pipe *pipe,
+				      const struct mxc_isi_format_info *info,
+				      const struct v4l2_pix_format_mplane *format);
+void mxc_isi_channel_set_output_format(struct mxc_isi_pipe *pipe,
+				       const struct mxc_isi_format_info *info,
+				       struct v4l2_pix_format_mplane *format);
+void mxc_isi_channel_m2m_start(struct mxc_isi_pipe *pipe);
+
+void mxc_isi_channel_set_alpha(struct mxc_isi_pipe *pipe, u8 alpha);
+void mxc_isi_channel_set_flip(struct mxc_isi_pipe *pipe, bool hflip, bool vflip);
+
+void mxc_isi_channel_set_inbuf(struct mxc_isi_pipe *pipe, dma_addr_t dma_addr);
+void mxc_isi_channel_set_outbuf(struct mxc_isi_pipe *pipe,
+				const dma_addr_t dma_addrs[3],
+				enum mxc_isi_buf_id buf_id);
+
+u32 mxc_isi_channel_irq_status(struct mxc_isi_pipe *pipe, bool clear);
+void mxc_isi_channel_irq_clear(struct mxc_isi_pipe *pipe);
+
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+void mxc_isi_debug_init(struct mxc_isi_dev *isi);
+void mxc_isi_debug_cleanup(struct mxc_isi_dev *isi);
+#else
+static inline void mxc_isi_debug_init(struct mxc_isi_dev *isi)
+{
+}
+static inline void mxc_isi_debug_cleanup(struct mxc_isi_dev *isi)
+{
+}
+#endif
+
+#endif /* __MXC_ISI_CORE_H__ */
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
new file mode 100644
index 000000000000..b5ffde46f31b
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
@@ -0,0 +1,529 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * i.MX8 ISI - Input crossbar switch
+ *
+ * Copyright (c) 2022 Laurent Pinchart <laurent.pinchart@ideasonboard.com>
+ */
+
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/minmax.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+#include <media/media-entity.h>
+#include <media/mipi-csi2.h>
+#include <media/v4l2-subdev.h>
+
+#include "imx8-isi-core.h"
+
+static inline struct mxc_isi_crossbar *to_isi_crossbar(struct v4l2_subdev *sd)
+{
+	return container_of(sd, struct mxc_isi_crossbar, sd);
+}
+
+/* -----------------------------------------------------------------------------
+ * Media block control (i.MX8MN and i.MX8MP only)
+ */
+#define GASKET_BASE(n)				(0x0060 + (n) * 0x30)
+
+#define GASKET_CTRL				0x0000
+#define GASKET_CTRL_DATA_TYPE(dt)		((dt) << 8)
+#define GASKET_CTRL_DATA_TYPE_MASK		(0x3f << 8)
+#define GASKET_CTRL_DUAL_COMP_ENABLE		BIT(1)
+#define GASKET_CTRL_ENABLE			BIT(0)
+
+#define GASKET_HSIZE				0x0004
+#define GASKET_VSIZE				0x0008
+
+static int mxc_isi_crossbar_gasket_enable(struct mxc_isi_crossbar *xbar,
+					  struct v4l2_subdev_state *state,
+					  struct v4l2_subdev *remote_sd,
+					  u32 remote_pad, unsigned int port)
+{
+	struct mxc_isi_dev *isi = xbar->isi;
+	const struct v4l2_mbus_framefmt *fmt;
+	struct v4l2_mbus_frame_desc fd;
+	u32 val;
+	int ret;
+
+	if (!isi->pdata->has_gasket)
+		return 0;
+
+	/*
+	 * Configure and enable the gasket with the frame size and CSI-2 data
+	 * type. For YUV422 8-bit, enable dual component mode unconditionally,
+	 * to match the configuration of the CSIS.
+	 */
+
+	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc, remote_pad, &fd);
+	if (ret) {
+		dev_err(isi->dev,
+			"failed to get frame descriptor from '%s':%u: %d\n",
+			remote_sd->name, remote_pad, ret);
+		return ret;
+	}
+
+	if (fd.num_entries != 1) {
+		dev_err(isi->dev, "invalid frame descriptor for '%s':%u\n",
+			remote_sd->name, remote_pad);
+		return -EINVAL;
+	}
+
+	fmt = v4l2_subdev_state_get_stream_format(state, port, 0);
+	if (!fmt)
+		return -EINVAL;
+
+	regmap_write(isi->gasket, GASKET_BASE(port) + GASKET_HSIZE, fmt->width);
+	regmap_write(isi->gasket, GASKET_BASE(port) + GASKET_VSIZE, fmt->height);
+
+	val = GASKET_CTRL_DATA_TYPE(fd.entry[0].bus.csi2.dt)
+	    | GASKET_CTRL_ENABLE;
+
+	if (fd.entry[0].bus.csi2.dt == MIPI_CSI2_DT_YUV422_8B)
+		val |= GASKET_CTRL_DUAL_COMP_ENABLE;
+
+	regmap_write(isi->gasket, GASKET_BASE(port) + GASKET_CTRL, val);
+
+	return 0;
+}
+
+static void mxc_isi_crossbar_gasket_disable(struct mxc_isi_crossbar *xbar,
+					    unsigned int port)
+{
+	struct mxc_isi_dev *isi = xbar->isi;
+
+	if (!isi->pdata->has_gasket)
+		return;
+
+	regmap_write(isi->gasket, GASKET_BASE(port) + GASKET_CTRL, 0);
+}
+
+/* -----------------------------------------------------------------------------
+ * V4L2 subdev operations
+ */
+
+static const struct v4l2_mbus_framefmt mxc_isi_crossbar_default_format = {
+	.code = MXC_ISI_DEF_MBUS_CODE_SINK,
+	.width = MXC_ISI_DEF_WIDTH,
+	.height = MXC_ISI_DEF_HEIGHT,
+	.field = V4L2_FIELD_NONE,
+	.colorspace = MXC_ISI_DEF_COLOR_SPACE,
+	.ycbcr_enc = MXC_ISI_DEF_YCBCR_ENC,
+	.quantization = MXC_ISI_DEF_QUANTIZATION,
+	.xfer_func = MXC_ISI_DEF_XFER_FUNC,
+};
+
+static int __mxc_isi_crossbar_set_routing(struct v4l2_subdev *sd,
+					  struct v4l2_subdev_state *state,
+					  struct v4l2_subdev_krouting *routing)
+{
+	struct mxc_isi_crossbar *xbar = to_isi_crossbar(sd);
+	struct v4l2_subdev_route *route;
+	int ret;
+
+	ret = v4l2_subdev_routing_validate(sd, routing,
+					   V4L2_SUBDEV_ROUTING_NO_N_TO_1);
+	if (ret)
+		return ret;
+
+	/* The memory input can be routed to the first pipeline only. */
+	for_each_active_route(&state->routing, route) {
+		if (route->sink_pad == xbar->num_sinks - 1 &&
+		    route->source_pad != xbar->num_sinks) {
+			dev_dbg(xbar->isi->dev,
+				"invalid route from memory input (%u) to pipe %u\n",
+				route->sink_pad,
+				route->source_pad - xbar->num_sinks);
+			return -EINVAL;
+		}
+	}
+
+	return v4l2_subdev_set_routing_with_fmt(sd, state, routing,
+						&mxc_isi_crossbar_default_format);
+}
+
+static struct v4l2_subdev *
+mxc_isi_crossbar_xlate_streams(struct mxc_isi_crossbar *xbar,
+			       struct v4l2_subdev_state *state,
+			       u32 source_pad, u64 source_streams,
+			       u32 *__sink_pad, u64 *__sink_streams,
+			       u32 *remote_pad)
+{
+	struct v4l2_subdev_route *route;
+	struct v4l2_subdev *sd;
+	struct media_pad *pad;
+	u64 sink_streams = 0;
+	int sink_pad = -1;
+
+	/*
+	 * Translate the source pad and streams to the sink side. The routing
+	 * validation forbids stream merging, so all matching entries in the
+	 * routing table are guaranteed to have the same sink pad.
+	 *
+	 * TODO: This is likely worth a helper function, it could perhaps be
+	 * supported by v4l2_subdev_state_xlate_streams() with pad1 set to -1.
+	 */
+	for_each_active_route(&state->routing, route) {
+		if (route->source_pad != source_pad ||
+		    !(source_streams & BIT(route->source_stream)))
+			continue;
+
+		sink_streams |= BIT(route->sink_stream);
+		sink_pad = route->sink_pad;
+	}
+
+	if (sink_pad < 0) {
+		dev_dbg(xbar->isi->dev,
+			"no stream connected to pipeline %u\n",
+			source_pad - xbar->num_sinks);
+		return ERR_PTR(-EPIPE);
+	}
+
+	pad = media_pad_remote_pad_first(&xbar->pads[sink_pad]);
+	sd = media_entity_to_v4l2_subdev(pad->entity);
+
+	if (!sd) {
+		dev_dbg(xbar->isi->dev,
+			"no entity connected to crossbar input %u\n",
+			sink_pad);
+		return ERR_PTR(-EPIPE);
+	}
+
+	*__sink_pad = sink_pad;
+	*__sink_streams = sink_streams;
+	*remote_pad = pad->index;
+
+	return sd;
+}
+
+static int mxc_isi_crossbar_init_cfg(struct v4l2_subdev *sd,
+				     struct v4l2_subdev_state *state)
+{
+	struct mxc_isi_crossbar *xbar = to_isi_crossbar(sd);
+	struct v4l2_subdev_krouting routing = { };
+	struct v4l2_subdev_route *routes;
+	unsigned int i;
+	int ret;
+
+	/*
+	 * Create a 1:1 mapping between pixel link inputs and outputs to
+	 * pipelines by default.
+	 */
+	routes = kcalloc(xbar->num_sources, sizeof(*routes), GFP_KERNEL);
+	if (!routes)
+		return -ENOMEM;
+
+	for (i = 0; i < xbar->num_sources; ++i) {
+		struct v4l2_subdev_route *route = &routes[i];
+
+		route->sink_pad = i;
+		route->source_pad = i + xbar->num_sinks;
+		route->flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE;
+	};
+
+	routing.num_routes = xbar->num_sources;
+	routing.routes = routes;
+
+	ret = __mxc_isi_crossbar_set_routing(sd, state, &routing);
+
+	kfree(routes);
+
+	return ret;
+}
+
+static int mxc_isi_crossbar_enum_mbus_code(struct v4l2_subdev *sd,
+					   struct v4l2_subdev_state *state,
+					   struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct mxc_isi_crossbar *xbar = to_isi_crossbar(sd);
+	const struct mxc_isi_bus_format_info *info;
+
+	if (code->pad >= xbar->num_sinks) {
+		const struct v4l2_mbus_framefmt *format;
+
+		/*
+		 * The media bus code on source pads is identical to the
+		 * connected sink pad.
+		 */
+		if (code->index > 0)
+			return -EINVAL;
+
+		format = v4l2_subdev_state_get_opposite_stream_format(state,
+								      code->pad,
+								      code->stream);
+		if (!format)
+			return -EINVAL;
+
+		code->code = format->code;
+
+		return 0;
+	}
+
+	info = mxc_isi_bus_format_by_index(code->index, MXC_ISI_PIPE_PAD_SINK);
+	if (!info)
+		return -EINVAL;
+
+	code->code = info->mbus_code;
+
+	return 0;
+}
+
+static int mxc_isi_crossbar_set_fmt(struct v4l2_subdev *sd,
+				    struct v4l2_subdev_state *state,
+				    struct v4l2_subdev_format *fmt)
+{
+	struct mxc_isi_crossbar *xbar = to_isi_crossbar(sd);
+	struct v4l2_mbus_framefmt *sink_fmt;
+	struct v4l2_subdev_route *route;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE &&
+	    media_pad_is_streaming(&xbar->pads[fmt->pad]))
+		return -EBUSY;
+
+	/*
+	 * The source pad format is always identical to the sink pad format and
+	 * can't be modified.
+	 */
+	if (fmt->pad >= xbar->num_sinks)
+		return v4l2_subdev_get_fmt(sd, state, fmt);
+
+	/* Validate the requested format. */
+	if (!mxc_isi_bus_format_by_code(fmt->format.code, MXC_ISI_PIPE_PAD_SINK))
+		fmt->format.code = MXC_ISI_DEF_MBUS_CODE_SINK;
+
+	fmt->format.width = clamp_t(unsigned int, fmt->format.width,
+				    MXC_ISI_MIN_WIDTH, MXC_ISI_MAX_WIDTH_CHAINED);
+	fmt->format.height = clamp_t(unsigned int, fmt->format.height,
+				     MXC_ISI_MIN_HEIGHT, MXC_ISI_MAX_HEIGHT);
+	fmt->format.field = V4L2_FIELD_NONE;
+
+	/*
+	 * Set the format on the sink stream and propagate it to the source
+	 * streams.
+	 */
+	sink_fmt = v4l2_subdev_state_get_stream_format(state, fmt->pad,
+						       fmt->stream);
+	if (!sink_fmt)
+		return -EINVAL;
+
+	*sink_fmt = fmt->format;
+
+	/* TODO: A format propagation helper would be useful. */
+	for_each_active_route(&state->routing, route) {
+		struct v4l2_mbus_framefmt *source_fmt;
+
+		if (route->sink_pad != fmt->pad ||
+		    route->sink_stream != fmt->stream)
+			continue;
+
+		source_fmt = v4l2_subdev_state_get_stream_format(state, route->source_pad,
+								 route->source_stream);
+		if (!source_fmt)
+			return -EINVAL;
+
+		*source_fmt = fmt->format;
+	}
+
+	return 0;
+}
+
+static int mxc_isi_crossbar_set_routing(struct v4l2_subdev *sd,
+					struct v4l2_subdev_state *state,
+					enum v4l2_subdev_format_whence which,
+					struct v4l2_subdev_krouting *routing)
+{
+	if (which == V4L2_SUBDEV_FORMAT_ACTIVE &&
+	    media_entity_is_streaming(&sd->entity))
+		return -EBUSY;
+
+	return __mxc_isi_crossbar_set_routing(sd, state, routing);
+}
+
+static int mxc_isi_crossbar_enable_streams(struct v4l2_subdev *sd,
+					   struct v4l2_subdev_state *state,
+					   u32 pad, u64 streams_mask)
+{
+	struct mxc_isi_crossbar *xbar = to_isi_crossbar(sd);
+	struct v4l2_subdev *remote_sd;
+	struct mxc_isi_input *input;
+	u64 sink_streams;
+	u32 sink_pad;
+	u32 remote_pad;
+	int ret;
+
+	remote_sd = mxc_isi_crossbar_xlate_streams(xbar, state, pad, streams_mask,
+						   &sink_pad, &sink_streams,
+						   &remote_pad);
+	if (IS_ERR(remote_sd))
+		return PTR_ERR(remote_sd);
+
+	input = &xbar->inputs[sink_pad];
+
+	/*
+	 * TODO: Track per-stream enable counts to support multiplexed
+	 * streams.
+	 */
+	if (!input->enable_count) {
+		ret = mxc_isi_crossbar_gasket_enable(xbar, state, remote_sd,
+						     remote_pad, sink_pad);
+		if (ret)
+			return ret;
+
+		ret = v4l2_subdev_enable_streams(remote_sd, remote_pad,
+						 sink_streams);
+		if (ret) {
+			dev_err(xbar->isi->dev,
+				"failed to %s streams 0x%llx on '%s':%u: %d\n",
+				"enable", sink_streams, remote_sd->name,
+				remote_pad, ret);
+			mxc_isi_crossbar_gasket_disable(xbar, sink_pad);
+			return ret;
+		}
+	}
+
+	input->enable_count++;
+
+	return 0;
+}
+
+static int mxc_isi_crossbar_disable_streams(struct v4l2_subdev *sd,
+					    struct v4l2_subdev_state *state,
+					    u32 pad, u64 streams_mask)
+{
+	struct mxc_isi_crossbar *xbar = to_isi_crossbar(sd);
+	struct v4l2_subdev *remote_sd;
+	struct mxc_isi_input *input;
+	u64 sink_streams;
+	u32 sink_pad;
+	u32 remote_pad;
+	int ret = 0;
+
+	remote_sd = mxc_isi_crossbar_xlate_streams(xbar, state, pad, streams_mask,
+						   &sink_pad, &sink_streams,
+						   &remote_pad);
+	if (IS_ERR(remote_sd))
+		return PTR_ERR(remote_sd);
+
+	input = &xbar->inputs[sink_pad];
+
+	input->enable_count--;
+
+	if (!input->enable_count) {
+		ret = v4l2_subdev_disable_streams(remote_sd, remote_pad,
+						  sink_streams);
+		if (ret)
+			dev_err(xbar->isi->dev,
+				"failed to %s streams 0x%llx on '%s':%u: %d\n",
+				"disable", sink_streams, remote_sd->name,
+				remote_pad, ret);
+
+		mxc_isi_crossbar_gasket_disable(xbar, sink_pad);
+	}
+
+	return ret;
+}
+
+static const struct v4l2_subdev_pad_ops mxc_isi_crossbar_subdev_pad_ops = {
+	.init_cfg = mxc_isi_crossbar_init_cfg,
+	.enum_mbus_code = mxc_isi_crossbar_enum_mbus_code,
+	.get_fmt = v4l2_subdev_get_fmt,
+	.set_fmt = mxc_isi_crossbar_set_fmt,
+	.set_routing = mxc_isi_crossbar_set_routing,
+	.enable_streams = mxc_isi_crossbar_enable_streams,
+	.disable_streams = mxc_isi_crossbar_disable_streams,
+};
+
+static const struct v4l2_subdev_ops mxc_isi_crossbar_subdev_ops = {
+	.pad = &mxc_isi_crossbar_subdev_pad_ops,
+};
+
+static const struct media_entity_operations mxc_isi_cross_entity_ops = {
+	.get_fwnode_pad = v4l2_subdev_get_fwnode_pad_1_to_1,
+	.link_validate	= v4l2_subdev_link_validate,
+	.has_pad_interdep = v4l2_subdev_has_pad_interdep,
+};
+
+/* -----------------------------------------------------------------------------
+ * Init & cleanup
+ */
+
+int mxc_isi_crossbar_init(struct mxc_isi_dev *isi)
+{
+	struct mxc_isi_crossbar *xbar = &isi->crossbar;
+	struct v4l2_subdev *sd = &xbar->sd;
+	unsigned int num_pads;
+	unsigned int i;
+	int ret;
+
+	xbar->isi = isi;
+
+	v4l2_subdev_init(sd, &mxc_isi_crossbar_subdev_ops);
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_STREAMS;
+	strscpy(sd->name, "crossbar", sizeof(sd->name));
+	sd->dev = isi->dev;
+
+	sd->entity.function = MEDIA_ENT_F_VID_MUX;
+	sd->entity.ops = &mxc_isi_cross_entity_ops;
+
+	/*
+	 * The subdev has one sink and one source per port, plus one sink for
+	 * the memory input.
+	 */
+	xbar->num_sinks = isi->pdata->num_ports + 1;
+	xbar->num_sources = isi->pdata->num_ports;
+	num_pads = xbar->num_sinks + xbar->num_sources;
+
+	xbar->pads = kcalloc(num_pads, sizeof(*xbar->pads), GFP_KERNEL);
+	if (!xbar->pads)
+		return -ENOMEM;
+
+	xbar->inputs = kcalloc(xbar->num_sinks, sizeof(*xbar->inputs),
+			       GFP_KERNEL);
+	if (!xbar->pads) {
+		ret = -ENOMEM;
+		goto err_free;
+	}
+
+	for (i = 0; i < xbar->num_sinks; ++i)
+		xbar->pads[i].flags = MEDIA_PAD_FL_SINK;
+	for (i = 0; i < xbar->num_sources; ++i)
+		xbar->pads[i + xbar->num_sinks].flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&sd->entity, num_pads, xbar->pads);
+	if (ret)
+		goto err_free;
+
+	ret = v4l2_subdev_init_finalize(sd);
+	if (ret < 0)
+		goto err_entity;
+
+	return 0;
+
+err_entity:
+	media_entity_cleanup(&sd->entity);
+err_free:
+	kfree(xbar->pads);
+	kfree(xbar->inputs);
+
+	return ret;
+}
+
+void mxc_isi_crossbar_cleanup(struct mxc_isi_crossbar *xbar)
+{
+	media_entity_cleanup(&xbar->sd.entity);
+	kfree(xbar->pads);
+	kfree(xbar->inputs);
+}
+
+int mxc_isi_crossbar_register(struct mxc_isi_crossbar *xbar)
+{
+	return v4l2_device_register_subdev(&xbar->isi->v4l2_dev, &xbar->sd);
+}
+
+void mxc_isi_crossbar_unregister(struct mxc_isi_crossbar *xbar)
+{
+}
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-debug.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-debug.c
new file mode 100644
index 000000000000..6709ab7ea1f3
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-debug.c
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019-2020 NXP
+ */
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/pm_runtime.h>
+#include <linux/seq_file.h>
+#include <linux/types.h>
+
+#include "imx8-isi-core.h"
+#include "imx8-isi-regs.h"
+
+static inline u32 mxc_isi_read(struct mxc_isi_pipe *pipe, u32 reg)
+{
+	return readl(pipe->regs + reg);
+}
+
+static int mxc_isi_debug_dump_regs_show(struct seq_file *m, void *p)
+{
+#define MXC_ISI_DEBUG_REG(name)		{ name, #name }
+	static const struct {
+		u32 offset;
+		const char * const name;
+	} registers[] = {
+		MXC_ISI_DEBUG_REG(CHNL_CTRL),
+		MXC_ISI_DEBUG_REG(CHNL_IMG_CTRL),
+		MXC_ISI_DEBUG_REG(CHNL_OUT_BUF_CTRL),
+		MXC_ISI_DEBUG_REG(CHNL_IMG_CFG),
+		MXC_ISI_DEBUG_REG(CHNL_IER),
+		MXC_ISI_DEBUG_REG(CHNL_STS),
+		MXC_ISI_DEBUG_REG(CHNL_SCALE_FACTOR),
+		MXC_ISI_DEBUG_REG(CHNL_SCALE_OFFSET),
+		MXC_ISI_DEBUG_REG(CHNL_CROP_ULC),
+		MXC_ISI_DEBUG_REG(CHNL_CROP_LRC),
+		MXC_ISI_DEBUG_REG(CHNL_CSC_COEFF0),
+		MXC_ISI_DEBUG_REG(CHNL_CSC_COEFF1),
+		MXC_ISI_DEBUG_REG(CHNL_CSC_COEFF2),
+		MXC_ISI_DEBUG_REG(CHNL_CSC_COEFF3),
+		MXC_ISI_DEBUG_REG(CHNL_CSC_COEFF4),
+		MXC_ISI_DEBUG_REG(CHNL_CSC_COEFF5),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_0_ALPHA),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_0_ULC),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_0_LRC),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_1_ALPHA),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_1_ULC),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_1_LRC),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_2_ALPHA),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_2_ULC),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_2_LRC),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_3_ALPHA),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_3_ULC),
+		MXC_ISI_DEBUG_REG(CHNL_ROI_3_LRC),
+		MXC_ISI_DEBUG_REG(CHNL_OUT_BUF1_ADDR_Y),
+		MXC_ISI_DEBUG_REG(CHNL_OUT_BUF1_ADDR_U),
+		MXC_ISI_DEBUG_REG(CHNL_OUT_BUF1_ADDR_V),
+		MXC_ISI_DEBUG_REG(CHNL_OUT_BUF_PITCH),
+		MXC_ISI_DEBUG_REG(CHNL_IN_BUF_ADDR),
+		MXC_ISI_DEBUG_REG(CHNL_IN_BUF_PITCH),
+		MXC_ISI_DEBUG_REG(CHNL_MEM_RD_CTRL),
+		MXC_ISI_DEBUG_REG(CHNL_OUT_BUF2_ADDR_Y),
+		MXC_ISI_DEBUG_REG(CHNL_OUT_BUF2_ADDR_U),
+		MXC_ISI_DEBUG_REG(CHNL_OUT_BUF2_ADDR_V),
+		MXC_ISI_DEBUG_REG(CHNL_SCL_IMG_CFG),
+		MXC_ISI_DEBUG_REG(CHNL_FLOW_CTRL),
+	};
+
+	struct mxc_isi_pipe *pipe = m->private;
+	unsigned int i;
+
+	if (!pm_runtime_get_if_in_use(pipe->isi->dev))
+		return 0;
+
+	seq_printf(m, "--- ISI pipe %u registers ---\n", pipe->id);
+
+	for (i = 0; i < ARRAY_SIZE(registers); ++i)
+		seq_printf(m, "%20s[0x%02x]: 0x%08x\n",
+			   registers[i].name, registers[i].offset,
+			   mxc_isi_read(pipe, registers[i].offset));
+
+	pm_runtime_put(pipe->isi->dev);
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(mxc_isi_debug_dump_regs);
+
+void mxc_isi_debug_init(struct mxc_isi_dev *isi)
+{
+	unsigned int i;
+
+	isi->debugfs_root = debugfs_create_dir(dev_name(isi->dev), NULL);
+
+	for (i = 0; i < isi->pdata->num_channels; ++i) {
+		struct mxc_isi_pipe *pipe = &isi->pipes[i];
+		char name[8];
+
+		sprintf(name, "pipe%u", pipe->id);
+		debugfs_create_file(name, 0444, isi->debugfs_root, pipe,
+				    &mxc_isi_debug_dump_regs_fops);
+	}
+}
+
+void mxc_isi_debug_cleanup(struct mxc_isi_dev *isi)
+{
+	debugfs_remove_recursive(isi->debugfs_root);
+}
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c
new file mode 100644
index 000000000000..db538f3d88ec
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c
@@ -0,0 +1,650 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019-2020 NXP
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/types.h>
+
+#include "imx8-isi-core.h"
+#include "imx8-isi-regs.h"
+
+#define	ISI_DOWNSCALE_THRESHOLD		0x4000
+
+static inline u32 mxc_isi_read(struct mxc_isi_pipe *pipe, u32 reg)
+{
+	return readl(pipe->regs + reg);
+}
+
+static inline void mxc_isi_write(struct mxc_isi_pipe *pipe, u32 reg, u32 val)
+{
+	writel(val, pipe->regs + reg);
+}
+
+/* -----------------------------------------------------------------------------
+ * Buffers & M2M operation
+ */
+
+void mxc_isi_channel_set_inbuf(struct mxc_isi_pipe *pipe, dma_addr_t dma_addr)
+{
+	mxc_isi_write(pipe, CHNL_IN_BUF_ADDR, dma_addr);
+#if CONFIG_ARCH_DMA_ADDR_T_64BIT
+	if (pipe->isi->pdata->has_36bit_dma)
+		mxc_isi_write(pipe, CHNL_IN_BUF_XTND_ADDR, dma_addr >> 32);
+#endif
+}
+
+void mxc_isi_channel_set_outbuf(struct mxc_isi_pipe *pipe,
+				const dma_addr_t dma_addrs[3],
+				enum mxc_isi_buf_id buf_id)
+{
+	int val;
+
+	val = mxc_isi_read(pipe, CHNL_OUT_BUF_CTRL);
+
+	if (buf_id == MXC_ISI_BUF1) {
+		mxc_isi_write(pipe, CHNL_OUT_BUF1_ADDR_Y, dma_addrs[0]);
+		mxc_isi_write(pipe, CHNL_OUT_BUF1_ADDR_U, dma_addrs[1]);
+		mxc_isi_write(pipe, CHNL_OUT_BUF1_ADDR_V, dma_addrs[2]);
+#if CONFIG_ARCH_DMA_ADDR_T_64BIT
+		if (pipe->isi->pdata->has_36bit_dma) {
+			mxc_isi_write(pipe, CHNL_Y_BUF1_XTND_ADDR,
+				      dma_addrs[0] >> 32);
+			mxc_isi_write(pipe, CHNL_U_BUF1_XTND_ADDR,
+				      dma_addrs[1] >> 32);
+			mxc_isi_write(pipe, CHNL_V_BUF1_XTND_ADDR,
+				      dma_addrs[2] >> 32);
+		}
+#endif
+		val ^= CHNL_OUT_BUF_CTRL_LOAD_BUF1_ADDR;
+	} else  {
+		mxc_isi_write(pipe, CHNL_OUT_BUF2_ADDR_Y, dma_addrs[0]);
+		mxc_isi_write(pipe, CHNL_OUT_BUF2_ADDR_U, dma_addrs[1]);
+		mxc_isi_write(pipe, CHNL_OUT_BUF2_ADDR_V, dma_addrs[2]);
+#if CONFIG_ARCH_DMA_ADDR_T_64BIT
+		if (pipe->isi->pdata->has_36bit_dma) {
+			mxc_isi_write(pipe, CHNL_Y_BUF2_XTND_ADDR,
+				      dma_addrs[0] >> 32);
+			mxc_isi_write(pipe, CHNL_U_BUF2_XTND_ADDR,
+				      dma_addrs[1] >> 32);
+			mxc_isi_write(pipe, CHNL_V_BUF2_XTND_ADDR,
+				      dma_addrs[2] >> 32);
+		}
+#endif
+		val ^= CHNL_OUT_BUF_CTRL_LOAD_BUF2_ADDR;
+	}
+
+	mxc_isi_write(pipe, CHNL_OUT_BUF_CTRL, val);
+}
+
+void mxc_isi_channel_m2m_start(struct mxc_isi_pipe *pipe)
+{
+	u32 val;
+
+	val = mxc_isi_read(pipe, CHNL_MEM_RD_CTRL);
+	val &= ~CHNL_MEM_RD_CTRL_READ_MEM;
+	mxc_isi_write(pipe, CHNL_MEM_RD_CTRL, val);
+
+	fsleep(300);
+
+	val |= CHNL_MEM_RD_CTRL_READ_MEM;
+	mxc_isi_write(pipe, CHNL_MEM_RD_CTRL, val);
+}
+
+/* -----------------------------------------------------------------------------
+ * Pipeline configuration
+ */
+
+static u32 mxc_isi_channel_scaling_ratio(unsigned int from, unsigned int to,
+					 u32 *dec)
+{
+	unsigned int ratio = from / to;
+
+	if (ratio < 2)
+		*dec = 1;
+	else if (ratio < 4)
+		*dec = 2;
+	else if (ratio < 8)
+		*dec = 4;
+	else
+		*dec = 8;
+
+	return min_t(u32, from * 0x1000 / (to * *dec), ISI_DOWNSCALE_THRESHOLD);
+}
+
+static void mxc_isi_channel_set_scaling(struct mxc_isi_pipe *pipe,
+					enum mxc_isi_encoding encoding,
+					const struct v4l2_area *in_size,
+					const struct v4l2_area *out_size,
+					bool *bypass)
+{
+	u32 xscale, yscale;
+	u32 decx, decy;
+	u32 val;
+
+	dev_dbg(pipe->isi->dev, "input %ux%u, output %ux%u\n",
+		in_size->width, in_size->height,
+		out_size->width, out_size->height);
+
+	xscale = mxc_isi_channel_scaling_ratio(in_size->width, out_size->width,
+					       &decx);
+	yscale = mxc_isi_channel_scaling_ratio(in_size->height, out_size->height,
+					       &decy);
+
+	val = mxc_isi_read(pipe, CHNL_IMG_CTRL);
+	val &= ~(CHNL_IMG_CTRL_DEC_X_MASK | CHNL_IMG_CTRL_DEC_Y_MASK |
+		 CHNL_IMG_CTRL_YCBCR_MODE);
+
+	val |= CHNL_IMG_CTRL_DEC_X(ilog2(decx))
+	    |  CHNL_IMG_CTRL_DEC_Y(ilog2(decy));
+
+	/*
+	 * Contrary to what the documentation states, YCBCR_MODE does not
+	 * control conversion between YCbCr and RGB, but whether the scaler
+	 * operates in YUV mode or in RGB mode. It must be set when the scaler
+	 * input is YUV.
+	 */
+	if (encoding == MXC_ISI_ENC_YUV)
+		val |= CHNL_IMG_CTRL_YCBCR_MODE;
+
+	mxc_isi_write(pipe, CHNL_IMG_CTRL, val);
+
+	mxc_isi_write(pipe, CHNL_SCALE_FACTOR,
+		      CHNL_SCALE_FACTOR_Y_SCALE(yscale) |
+		      CHNL_SCALE_FACTOR_X_SCALE(xscale));
+
+	mxc_isi_write(pipe, CHNL_SCALE_OFFSET, 0);
+
+	mxc_isi_write(pipe, CHNL_SCL_IMG_CFG,
+		      CHNL_SCL_IMG_CFG_HEIGHT(out_size->height) |
+		      CHNL_SCL_IMG_CFG_WIDTH(out_size->width));
+
+	*bypass = in_size->height == out_size->height &&
+		  in_size->width == out_size->width;
+}
+
+static void mxc_isi_channel_set_crop(struct mxc_isi_pipe *pipe,
+				     const struct v4l2_area *src,
+				     const struct v4l2_rect *dst)
+{
+	u32 val, val0, val1;
+
+	val = mxc_isi_read(pipe, CHNL_IMG_CTRL);
+	val &= ~CHNL_IMG_CTRL_CROP_EN;
+
+	if (src->height == dst->height && src->width == dst->width) {
+		mxc_isi_write(pipe, CHNL_IMG_CTRL, val);
+		return;
+	}
+
+	val |= CHNL_IMG_CTRL_CROP_EN;
+	val0 = CHNL_CROP_ULC_X(dst->left) | CHNL_CROP_ULC_Y(dst->top);
+	val1 = CHNL_CROP_LRC_X(dst->width) | CHNL_CROP_LRC_Y(dst->height);
+
+	mxc_isi_write(pipe, CHNL_CROP_ULC, val0);
+	mxc_isi_write(pipe, CHNL_CROP_LRC, val1 + val0);
+	mxc_isi_write(pipe, CHNL_IMG_CTRL, val);
+}
+
+/*
+ * A2,A1,      B1, A3,     B3, B2,
+ * C2, C1,     D1, C3,     D3, D2
+ */
+static const u32 mxc_isi_yuv2rgb_coeffs[6] = {
+	/* YUV -> RGB */
+	0x0000012a, 0x012a0198, 0x0730079c,
+	0x0204012a, 0x01f00000, 0x01800180
+};
+
+static const u32 mxc_isi_rgb2yuv_coeffs[6] = {
+	/* RGB->YUV */
+	0x00810041, 0x07db0019, 0x007007b6,
+	0x07a20070, 0x001007ee, 0x00800080
+};
+
+static void mxc_isi_channel_set_csc(struct mxc_isi_pipe *pipe,
+				    enum mxc_isi_encoding in_encoding,
+				    enum mxc_isi_encoding out_encoding,
+				    bool *bypass)
+{
+	static const char * const encodings[] = {
+		[MXC_ISI_ENC_RAW] = "RAW",
+		[MXC_ISI_ENC_RGB] = "RGB",
+		[MXC_ISI_ENC_YUV] = "YUV",
+	};
+	const u32 *coeffs;
+	bool cscen = true;
+	u32 val;
+
+	val = mxc_isi_read(pipe, CHNL_IMG_CTRL);
+	val &= ~(CHNL_IMG_CTRL_CSC_BYPASS | CHNL_IMG_CTRL_CSC_MODE_MASK);
+
+	if (in_encoding == MXC_ISI_ENC_YUV &&
+	    out_encoding == MXC_ISI_ENC_RGB) {
+		/* YUV2RGB */
+		coeffs = mxc_isi_yuv2rgb_coeffs;
+		/* YCbCr enable???  */
+		val |= CHNL_IMG_CTRL_CSC_MODE(CHNL_IMG_CTRL_CSC_MODE_YCBCR2RGB);
+	} else if (in_encoding == MXC_ISI_ENC_RGB &&
+		   out_encoding == MXC_ISI_ENC_YUV) {
+		/* RGB2YUV */
+		coeffs = mxc_isi_rgb2yuv_coeffs;
+		val |= CHNL_IMG_CTRL_CSC_MODE(CHNL_IMG_CTRL_CSC_MODE_RGB2YCBCR);
+	} else {
+		/* Bypass CSC */
+		cscen = false;
+		val |= CHNL_IMG_CTRL_CSC_BYPASS;
+	}
+
+	dev_dbg(pipe->isi->dev, "CSC: %s -> %s\n",
+		encodings[in_encoding], encodings[out_encoding]);
+
+	if (cscen) {
+		mxc_isi_write(pipe, CHNL_CSC_COEFF0, coeffs[0]);
+		mxc_isi_write(pipe, CHNL_CSC_COEFF1, coeffs[1]);
+		mxc_isi_write(pipe, CHNL_CSC_COEFF2, coeffs[2]);
+		mxc_isi_write(pipe, CHNL_CSC_COEFF3, coeffs[3]);
+		mxc_isi_write(pipe, CHNL_CSC_COEFF4, coeffs[4]);
+		mxc_isi_write(pipe, CHNL_CSC_COEFF5, coeffs[5]);
+	}
+
+	mxc_isi_write(pipe, CHNL_IMG_CTRL, val);
+
+	*bypass = !cscen;
+}
+
+void mxc_isi_channel_set_alpha(struct mxc_isi_pipe *pipe, u8 alpha)
+{
+	u32 val;
+
+	val = mxc_isi_read(pipe, CHNL_IMG_CTRL);
+	val &= ~CHNL_IMG_CTRL_GBL_ALPHA_VAL_MASK;
+	val |= CHNL_IMG_CTRL_GBL_ALPHA_VAL(alpha) |
+	       CHNL_IMG_CTRL_GBL_ALPHA_EN;
+	mxc_isi_write(pipe, CHNL_IMG_CTRL, val);
+}
+
+void mxc_isi_channel_set_flip(struct mxc_isi_pipe *pipe, bool hflip, bool vflip)
+{
+	u32 val;
+
+	val = mxc_isi_read(pipe, CHNL_IMG_CTRL);
+	val &= ~(CHNL_IMG_CTRL_VFLIP_EN | CHNL_IMG_CTRL_HFLIP_EN);
+
+	if (vflip)
+		val |= CHNL_IMG_CTRL_VFLIP_EN;
+	if (hflip)
+		val |= CHNL_IMG_CTRL_HFLIP_EN;
+
+	mxc_isi_write(pipe, CHNL_IMG_CTRL, val);
+}
+
+static void mxc_isi_channel_set_panic_threshold(struct mxc_isi_pipe *pipe)
+{
+	const struct mxc_isi_set_thd *set_thd = pipe->isi->pdata->set_thd;
+	u32 val;
+
+	val = mxc_isi_read(pipe, CHNL_OUT_BUF_CTRL);
+
+	val &= ~(set_thd->panic_set_thd_y.mask);
+	val |= set_thd->panic_set_thd_y.threshold << set_thd->panic_set_thd_y.offset;
+
+	val &= ~(set_thd->panic_set_thd_u.mask);
+	val |= set_thd->panic_set_thd_u.threshold << set_thd->panic_set_thd_u.offset;
+
+	val &= ~(set_thd->panic_set_thd_v.mask);
+	val |= set_thd->panic_set_thd_v.threshold << set_thd->panic_set_thd_v.offset;
+
+	mxc_isi_write(pipe, CHNL_OUT_BUF_CTRL, val);
+}
+
+static void mxc_isi_channel_set_control(struct mxc_isi_pipe *pipe,
+					enum mxc_isi_input_id input,
+					bool bypass)
+{
+	u32 val;
+
+	mutex_lock(&pipe->lock);
+
+	val = mxc_isi_read(pipe, CHNL_CTRL);
+	val &= ~(CHNL_CTRL_CHNL_BYPASS | CHNL_CTRL_CHAIN_BUF_MASK |
+		 CHNL_CTRL_BLANK_PXL_MASK | CHNL_CTRL_SRC_TYPE_MASK |
+		 CHNL_CTRL_MIPI_VC_ID_MASK | CHNL_CTRL_SRC_INPUT_MASK);
+
+	/*
+	 * If no scaling or color space conversion is needed, bypass the
+	 * channel.
+	 */
+	if (bypass)
+		val |= CHNL_CTRL_CHNL_BYPASS;
+
+	/* Chain line buffers if needed. */
+	if (pipe->chained)
+		val |= CHNL_CTRL_CHAIN_BUF(CHNL_CTRL_CHAIN_BUF_2_CHAIN);
+
+	val |= CHNL_CTRL_BLANK_PXL(0xff);
+
+	/* Input source (including VC configuration for CSI-2) */
+	if (input == MXC_ISI_INPUT_MEM) {
+		/*
+		 * The memory input is connected to the last port of the
+		 * crossbar switch, after all pixel link inputs. The SRC_INPUT
+		 * field controls the input selection and must be set
+		 * accordingly, despite being documented as ignored when using
+		 * the memory input in the i.MX8MP reference manual, and
+		 * reserved in the i.MX8MN reference manual.
+		 */
+		val |= CHNL_CTRL_SRC_TYPE(CHNL_CTRL_SRC_TYPE_MEMORY);
+		val |= CHNL_CTRL_SRC_INPUT(pipe->isi->pdata->num_ports);
+	} else {
+		val |= CHNL_CTRL_SRC_TYPE(CHNL_CTRL_SRC_TYPE_DEVICE);
+		val |= CHNL_CTRL_SRC_INPUT(input);
+		val |= CHNL_CTRL_MIPI_VC_ID(0); /* FIXME: For CSI-2 only */
+	}
+
+	mxc_isi_write(pipe, CHNL_CTRL, val);
+
+	mutex_unlock(&pipe->lock);
+}
+
+void mxc_isi_channel_config(struct mxc_isi_pipe *pipe,
+			    enum mxc_isi_input_id input,
+			    const struct v4l2_area *in_size,
+			    const struct v4l2_area *scale,
+			    const struct v4l2_rect *crop,
+			    enum mxc_isi_encoding in_encoding,
+			    enum mxc_isi_encoding out_encoding)
+{
+	bool csc_bypass;
+	bool scaler_bypass;
+
+	/* Input frame size */
+	mxc_isi_write(pipe, CHNL_IMG_CFG,
+		      CHNL_IMG_CFG_HEIGHT(in_size->height) |
+		      CHNL_IMG_CFG_WIDTH(in_size->width));
+
+	/* Scaling */
+	mxc_isi_channel_set_scaling(pipe, in_encoding, in_size, scale,
+				    &scaler_bypass);
+	mxc_isi_channel_set_crop(pipe, scale, crop);
+
+	/* CSC */
+	mxc_isi_channel_set_csc(pipe, in_encoding, out_encoding, &csc_bypass);
+
+	/* Output buffer management */
+	mxc_isi_channel_set_panic_threshold(pipe);
+
+	/* Channel control */
+	mxc_isi_channel_set_control(pipe, input, csc_bypass && scaler_bypass);
+}
+
+void mxc_isi_channel_set_input_format(struct mxc_isi_pipe *pipe,
+				      const struct mxc_isi_format_info *info,
+				      const struct v4l2_pix_format_mplane *format)
+{
+	unsigned int bpl = format->plane_fmt[0].bytesperline;
+
+	mxc_isi_write(pipe, CHNL_MEM_RD_CTRL,
+		      CHNL_MEM_RD_CTRL_IMG_TYPE(info->isi_in_format));
+	mxc_isi_write(pipe, CHNL_IN_BUF_PITCH,
+		      CHNL_IN_BUF_PITCH_LINE_PITCH(bpl));
+}
+
+void mxc_isi_channel_set_output_format(struct mxc_isi_pipe *pipe,
+				       const struct mxc_isi_format_info *info,
+				       struct v4l2_pix_format_mplane *format)
+{
+	u32 val;
+
+	/* set outbuf format */
+	dev_dbg(pipe->isi->dev, "output format %p4cc", &format->pixelformat);
+
+	val = mxc_isi_read(pipe, CHNL_IMG_CTRL);
+	val &= ~CHNL_IMG_CTRL_FORMAT_MASK;
+	val |= CHNL_IMG_CTRL_FORMAT(info->isi_out_format);
+	mxc_isi_write(pipe, CHNL_IMG_CTRL, val);
+
+	/* line pitch */
+	mxc_isi_write(pipe, CHNL_OUT_BUF_PITCH,
+		      format->plane_fmt[0].bytesperline);
+}
+
+/* -----------------------------------------------------------------------------
+ * IRQ
+ */
+
+u32 mxc_isi_channel_irq_status(struct mxc_isi_pipe *pipe, bool clear)
+{
+	u32 status;
+
+	status = mxc_isi_read(pipe, CHNL_STS);
+	if (clear)
+		mxc_isi_write(pipe, CHNL_STS, status);
+
+	return status;
+}
+
+void mxc_isi_channel_irq_clear(struct mxc_isi_pipe *pipe)
+{
+	mxc_isi_write(pipe, CHNL_STS, 0xffffffff);
+}
+
+static void mxc_isi_channel_irq_enable(struct mxc_isi_pipe *pipe)
+{
+	const struct mxc_isi_ier_reg *ier_reg = pipe->isi->pdata->ier_reg;
+	u32 val;
+
+	val = CHNL_IER_FRM_RCVD_EN |
+		CHNL_IER_AXI_WR_ERR_U_EN |
+		CHNL_IER_AXI_WR_ERR_V_EN |
+		CHNL_IER_AXI_WR_ERR_Y_EN;
+
+	/* Y/U/V overflow enable */
+	val |= ier_reg->oflw_y_buf_en.mask |
+	       ier_reg->oflw_u_buf_en.mask |
+	       ier_reg->oflw_v_buf_en.mask;
+
+	/* Y/U/V excess overflow enable */
+	val |= ier_reg->excs_oflw_y_buf_en.mask |
+	       ier_reg->excs_oflw_u_buf_en.mask |
+	       ier_reg->excs_oflw_v_buf_en.mask;
+
+	/* Y/U/V panic enable */
+	val |= ier_reg->panic_y_buf_en.mask |
+	       ier_reg->panic_u_buf_en.mask |
+	       ier_reg->panic_v_buf_en.mask;
+
+	mxc_isi_channel_irq_clear(pipe);
+	mxc_isi_write(pipe, CHNL_IER, val);
+}
+
+static void mxc_isi_channel_irq_disable(struct mxc_isi_pipe *pipe)
+{
+	mxc_isi_write(pipe, CHNL_IER, 0);
+}
+
+/* -----------------------------------------------------------------------------
+ * Init, deinit, enable, disable
+ */
+
+static void mxc_isi_channel_sw_reset(struct mxc_isi_pipe *pipe, bool enable_clk)
+{
+	mxc_isi_write(pipe, CHNL_CTRL, CHNL_CTRL_SW_RST);
+	mdelay(5);
+	mxc_isi_write(pipe, CHNL_CTRL, enable_clk ? CHNL_CTRL_CLK_EN : 0);
+}
+
+static void __mxc_isi_channel_get(struct mxc_isi_pipe *pipe)
+{
+	if (!pipe->use_count++)
+		mxc_isi_channel_sw_reset(pipe, true);
+}
+
+void mxc_isi_channel_get(struct mxc_isi_pipe *pipe)
+{
+	mutex_lock(&pipe->lock);
+	__mxc_isi_channel_get(pipe);
+	mutex_unlock(&pipe->lock);
+}
+
+static void __mxc_isi_channel_put(struct mxc_isi_pipe *pipe)
+{
+	if (!--pipe->use_count)
+		mxc_isi_channel_sw_reset(pipe, false);
+}
+
+void mxc_isi_channel_put(struct mxc_isi_pipe *pipe)
+{
+	mutex_lock(&pipe->lock);
+	__mxc_isi_channel_put(pipe);
+	mutex_unlock(&pipe->lock);
+}
+
+void mxc_isi_channel_enable(struct mxc_isi_pipe *pipe)
+{
+	u32 val;
+
+	mxc_isi_channel_irq_enable(pipe);
+
+	mutex_lock(&pipe->lock);
+
+	val = mxc_isi_read(pipe, CHNL_CTRL);
+	val |= CHNL_CTRL_CHNL_EN;
+	mxc_isi_write(pipe, CHNL_CTRL, val);
+
+	mutex_unlock(&pipe->lock);
+}
+
+void mxc_isi_channel_disable(struct mxc_isi_pipe *pipe)
+{
+	u32 val;
+
+	mxc_isi_channel_irq_disable(pipe);
+
+	mutex_lock(&pipe->lock);
+
+	val = mxc_isi_read(pipe, CHNL_CTRL);
+	val &= ~CHNL_CTRL_CHNL_EN;
+	mxc_isi_write(pipe, CHNL_CTRL, val);
+
+	mutex_unlock(&pipe->lock);
+}
+
+/* -----------------------------------------------------------------------------
+ * Resource management & chaining
+ */
+int mxc_isi_channel_acquire(struct mxc_isi_pipe *pipe,
+			    mxc_isi_pipe_irq_t irq_handler, bool bypass)
+{
+	u8 resources;
+	int ret = 0;
+
+	mutex_lock(&pipe->lock);
+
+	if (pipe->irq_handler) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	/*
+	 * Make sure the resources we need are available. The output buffer is
+	 * always needed to operate the channel, the line buffer is needed only
+	 * when the channel isn't in bypass mode.
+	 */
+	resources = MXC_ISI_CHANNEL_RES_OUTPUT_BUF
+		  | (!bypass ? MXC_ISI_CHANNEL_RES_LINE_BUF : 0);
+	if ((pipe->available_res & resources) != resources) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	/* Acquire the channel resources. */
+	pipe->acquired_res = resources;
+	pipe->available_res &= ~resources;
+	pipe->irq_handler = irq_handler;
+
+unlock:
+	mutex_unlock(&pipe->lock);
+
+	return ret;
+}
+
+void mxc_isi_channel_release(struct mxc_isi_pipe *pipe)
+{
+	mutex_lock(&pipe->lock);
+
+	pipe->irq_handler = NULL;
+	pipe->available_res |= pipe->acquired_res;
+	pipe->acquired_res = 0;
+
+	mutex_unlock(&pipe->lock);
+}
+
+/*
+ * We currently support line buffer chaining only, for handling images with a
+ * width larger than 2048 pixels.
+ *
+ * TODO: Support secondary line buffer for downscaling YUV420 images.
+ */
+int mxc_isi_channel_chain(struct mxc_isi_pipe *pipe, bool bypass)
+{
+	/* Channel chaining requires both line and output buffer. */
+	const u8 resources = MXC_ISI_CHANNEL_RES_OUTPUT_BUF
+			   | MXC_ISI_CHANNEL_RES_LINE_BUF;
+	struct mxc_isi_pipe *chained_pipe = pipe + 1;
+	int ret = 0;
+
+	/*
+	 * If buffer chaining is required, make sure this channel is not the
+	 * last one, otherwise there's no 'next' channel to chain with. This
+	 * should be prevented by checks in the set format handlers, but let's
+	 * be defensive.
+	 */
+	if (WARN_ON(pipe->id == pipe->isi->pdata->num_channels - 1))
+		return -EINVAL;
+
+	mutex_lock(&chained_pipe->lock);
+
+	/* Safety checks. */
+	if (WARN_ON(pipe->chained || chained_pipe->chained_res)) {
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	if ((chained_pipe->available_res & resources) != resources) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	pipe->chained = true;
+	chained_pipe->chained_res |= resources;
+	chained_pipe->available_res &= ~resources;
+
+	__mxc_isi_channel_get(chained_pipe);
+
+unlock:
+	mutex_unlock(&chained_pipe->lock);
+
+	return ret;
+}
+
+void mxc_isi_channel_unchain(struct mxc_isi_pipe *pipe)
+{
+	struct mxc_isi_pipe *chained_pipe = pipe + 1;
+
+	if (!pipe->chained)
+		return;
+
+	pipe->chained = false;
+
+	mutex_lock(&chained_pipe->lock);
+
+	chained_pipe->available_res |= chained_pipe->chained_res;
+	chained_pipe->chained_res = 0;
+
+	__mxc_isi_channel_put(chained_pipe);
+
+	mutex_unlock(&chained_pipe->lock);
+}
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c
new file mode 100644
index 000000000000..9745d6219a16
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c
@@ -0,0 +1,858 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ISI V4L2 memory to memory driver for i.MX8QXP/QM platform
+ *
+ * ISI is a Image Sensor Interface of i.MX8QXP/QM platform, which
+ * used to process image from camera sensor or memory to memory or DC
+ *
+ * Copyright (c) 2019 NXP Semiconductor
+ */
+
+#include <linux/container_of.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/limits.h>
+#include <linux/minmax.h>
+#include <linux/mutex.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/videodev2.h>
+
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mem2mem.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "imx8-isi-core.h"
+
+struct mxc_isi_m2m_buffer {
+	struct v4l2_m2m_buffer buf;
+	dma_addr_t dma_addrs[3];
+};
+
+struct mxc_isi_m2m_ctx_queue_data {
+	struct v4l2_pix_format_mplane format;
+	const struct mxc_isi_format_info *info;
+	u32 sequence;
+};
+
+struct mxc_isi_m2m_ctx {
+	struct v4l2_fh fh;
+	struct mxc_isi_m2m *m2m;
+
+	/* Protects the m2m vb2 queues */
+	struct mutex vb2_lock;
+
+	struct {
+		struct mxc_isi_m2m_ctx_queue_data out;
+		struct mxc_isi_m2m_ctx_queue_data cap;
+	} queues;
+
+	struct {
+		struct v4l2_ctrl_handler handler;
+		unsigned int alpha;
+		bool hflip;
+		bool vflip;
+	} ctrls;
+
+	bool chained;
+};
+
+static inline struct mxc_isi_m2m_buffer *
+to_isi_m2m_buffer(struct vb2_v4l2_buffer *buf)
+{
+	return container_of(buf, struct mxc_isi_m2m_buffer, buf.vb);
+}
+
+static inline struct mxc_isi_m2m_ctx *to_isi_m2m_ctx(struct v4l2_fh *fh)
+{
+	return container_of(fh, struct mxc_isi_m2m_ctx, fh);
+}
+
+static inline struct mxc_isi_m2m_ctx_queue_data *
+mxc_isi_m2m_ctx_qdata(struct mxc_isi_m2m_ctx *ctx, enum v4l2_buf_type type)
+{
+	if (V4L2_TYPE_IS_OUTPUT(type))
+		return &ctx->queues.out;
+	else
+		return &ctx->queues.cap;
+}
+
+/* -----------------------------------------------------------------------------
+ * V4L2 M2M device operations
+ */
+
+static void mxc_isi_m2m_frame_write_done(struct mxc_isi_pipe *pipe, u32 status)
+{
+	struct mxc_isi_m2m *m2m = &pipe->isi->m2m;
+	struct vb2_v4l2_buffer *src_vbuf, *dst_vbuf;
+	struct mxc_isi_m2m_ctx *ctx;
+
+	ctx = v4l2_m2m_get_curr_priv(m2m->m2m_dev);
+	if (!ctx) {
+		dev_err(m2m->isi->dev,
+			"Instance released before the end of transaction\n");
+		return;
+	}
+
+	src_vbuf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+	dst_vbuf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+
+	v4l2_m2m_buf_copy_metadata(src_vbuf, dst_vbuf, false);
+
+	src_vbuf->sequence = ctx->queues.out.sequence++;
+	dst_vbuf->sequence = ctx->queues.cap.sequence++;
+
+	v4l2_m2m_buf_done(src_vbuf, VB2_BUF_STATE_DONE);
+	v4l2_m2m_buf_done(dst_vbuf, VB2_BUF_STATE_DONE);
+
+	v4l2_m2m_job_finish(m2m->m2m_dev, ctx->fh.m2m_ctx);
+}
+
+static void mxc_isi_m2m_device_run(void *priv)
+{
+	struct mxc_isi_m2m_ctx *ctx = priv;
+	struct mxc_isi_m2m *m2m = ctx->m2m;
+	struct vb2_v4l2_buffer *src_vbuf, *dst_vbuf;
+	struct mxc_isi_m2m_buffer *src_buf, *dst_buf;
+
+	mxc_isi_channel_disable(m2m->pipe);
+
+	mutex_lock(&m2m->lock);
+
+	/* If the context has changed, reconfigure the channel. */
+	if (m2m->last_ctx != ctx) {
+		const struct v4l2_area in_size = {
+			.width = ctx->queues.out.format.width,
+			.height = ctx->queues.out.format.height,
+		};
+		const struct v4l2_area scale = {
+			.width = ctx->queues.cap.format.width,
+			.height = ctx->queues.cap.format.height,
+		};
+		const struct v4l2_rect crop = {
+			.width = ctx->queues.cap.format.width,
+			.height = ctx->queues.cap.format.height,
+		};
+
+		mxc_isi_channel_config(m2m->pipe, MXC_ISI_INPUT_MEM,
+				       &in_size, &scale, &crop,
+				       ctx->queues.out.info->encoding,
+				       ctx->queues.cap.info->encoding);
+		mxc_isi_channel_set_input_format(m2m->pipe,
+						 ctx->queues.out.info,
+						 &ctx->queues.out.format);
+		mxc_isi_channel_set_output_format(m2m->pipe,
+						  ctx->queues.cap.info,
+						  &ctx->queues.cap.format);
+
+		m2m->last_ctx = ctx;
+	}
+
+	mutex_unlock(&m2m->lock);
+
+	mutex_lock(ctx->ctrls.handler.lock);
+	mxc_isi_channel_set_alpha(m2m->pipe, ctx->ctrls.alpha);
+	mxc_isi_channel_set_flip(m2m->pipe, ctx->ctrls.hflip, ctx->ctrls.vflip);
+	mutex_unlock(ctx->ctrls.handler.lock);
+
+	src_vbuf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
+	dst_vbuf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
+
+	src_buf = to_isi_m2m_buffer(src_vbuf);
+	dst_buf = to_isi_m2m_buffer(dst_vbuf);
+
+	mxc_isi_channel_set_inbuf(m2m->pipe, src_buf->dma_addrs[0]);
+	mxc_isi_channel_set_outbuf(m2m->pipe, dst_buf->dma_addrs, MXC_ISI_BUF1);
+	mxc_isi_channel_set_outbuf(m2m->pipe, dst_buf->dma_addrs, MXC_ISI_BUF2);
+
+	mxc_isi_channel_enable(m2m->pipe);
+
+	mxc_isi_channel_m2m_start(m2m->pipe);
+}
+
+static const struct v4l2_m2m_ops mxc_isi_m2m_ops = {
+	.device_run = mxc_isi_m2m_device_run,
+};
+
+/* -----------------------------------------------------------------------------
+ * videobuf2 queue operations
+ */
+
+static int mxc_isi_m2m_vb2_queue_setup(struct vb2_queue *q,
+				       unsigned int *num_buffers,
+				       unsigned int *num_planes,
+				       unsigned int sizes[],
+				       struct device *alloc_devs[])
+{
+	struct mxc_isi_m2m_ctx *ctx = vb2_get_drv_priv(q);
+	const struct mxc_isi_m2m_ctx_queue_data *qdata =
+		mxc_isi_m2m_ctx_qdata(ctx, q->type);
+
+	return mxc_isi_video_queue_setup(&qdata->format, qdata->info,
+					 num_buffers, num_planes, sizes);
+}
+
+static int mxc_isi_m2m_vb2_buffer_init(struct vb2_buffer *vb2)
+{
+	struct vb2_queue *vq = vb2->vb2_queue;
+	struct mxc_isi_m2m_buffer *buf = to_isi_m2m_buffer(to_vb2_v4l2_buffer(vb2));
+	struct mxc_isi_m2m_ctx *ctx = vb2_get_drv_priv(vb2->vb2_queue);
+	const struct mxc_isi_m2m_ctx_queue_data *qdata =
+		mxc_isi_m2m_ctx_qdata(ctx, vq->type);
+
+	mxc_isi_video_buffer_init(vb2, buf->dma_addrs, qdata->info,
+				  &qdata->format);
+
+	return 0;
+}
+
+static int mxc_isi_m2m_vb2_buffer_prepare(struct vb2_buffer *vb2)
+{
+	struct vb2_queue *vq = vb2->vb2_queue;
+	struct mxc_isi_m2m_ctx *ctx = vb2_get_drv_priv(vq);
+	const struct mxc_isi_m2m_ctx_queue_data *qdata =
+		mxc_isi_m2m_ctx_qdata(ctx, vq->type);
+
+	return mxc_isi_video_buffer_prepare(ctx->m2m->isi, vb2, qdata->info,
+					    &qdata->format);
+}
+
+static void mxc_isi_m2m_vb2_buffer_queue(struct vb2_buffer *vb2)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb2);
+	struct mxc_isi_m2m_ctx *ctx = vb2_get_drv_priv(vb2->vb2_queue);
+
+	v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf);
+}
+
+static int mxc_isi_m2m_vb2_start_streaming(struct vb2_queue *q,
+					   unsigned int count)
+{
+	struct mxc_isi_m2m_ctx *ctx = vb2_get_drv_priv(q);
+	struct mxc_isi_m2m_ctx_queue_data *qdata =
+		mxc_isi_m2m_ctx_qdata(ctx, q->type);
+
+	qdata->sequence = 0;
+
+	return 0;
+}
+
+static void mxc_isi_m2m_vb2_stop_streaming(struct vb2_queue *q)
+{
+	struct mxc_isi_m2m_ctx *ctx = vb2_get_drv_priv(q);
+	struct vb2_v4l2_buffer *vbuf;
+
+	for (;;) {
+		if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)
+			vbuf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+		else
+			vbuf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+		if (!vbuf)
+			break;
+
+		v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_ERROR);
+	}
+}
+
+static const struct vb2_ops mxc_isi_m2m_vb2_qops = {
+	.queue_setup		= mxc_isi_m2m_vb2_queue_setup,
+	.buf_init		= mxc_isi_m2m_vb2_buffer_init,
+	.buf_prepare		= mxc_isi_m2m_vb2_buffer_prepare,
+	.buf_queue		= mxc_isi_m2m_vb2_buffer_queue,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
+	.start_streaming	= mxc_isi_m2m_vb2_start_streaming,
+	.stop_streaming		= mxc_isi_m2m_vb2_stop_streaming,
+};
+
+static int mxc_isi_m2m_queue_init(void *priv, struct vb2_queue *src_vq,
+				  struct vb2_queue *dst_vq)
+{
+	struct mxc_isi_m2m_ctx *ctx = priv;
+	struct mxc_isi_m2m *m2m = ctx->m2m;
+	int ret;
+
+	src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+	src_vq->io_modes = VB2_MMAP | VB2_DMABUF;
+	src_vq->drv_priv = ctx;
+	src_vq->buf_struct_size = sizeof(struct mxc_isi_m2m_buffer);
+	src_vq->ops = &mxc_isi_m2m_vb2_qops;
+	src_vq->mem_ops = &vb2_dma_contig_memops;
+	src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+	src_vq->lock = &ctx->vb2_lock;
+	src_vq->dev = m2m->isi->dev;
+
+	ret = vb2_queue_init(src_vq);
+	if (ret)
+		return ret;
+
+	dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	dst_vq->io_modes = VB2_MMAP | VB2_DMABUF;
+	dst_vq->drv_priv = ctx;
+	dst_vq->buf_struct_size = sizeof(struct mxc_isi_m2m_buffer);
+	dst_vq->ops = &mxc_isi_m2m_vb2_qops;
+	dst_vq->mem_ops = &vb2_dma_contig_memops;
+	dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+	dst_vq->lock = &ctx->vb2_lock;
+	dst_vq->dev = m2m->isi->dev;
+
+	return vb2_queue_init(dst_vq);
+}
+
+/* -----------------------------------------------------------------------------
+ * V4L2 controls
+ */
+
+static inline struct mxc_isi_m2m_ctx *
+ctrl_to_mxc_isi_m2m_ctx(struct v4l2_ctrl *ctrl)
+{
+	return container_of(ctrl->handler, struct mxc_isi_m2m_ctx, ctrls.handler);
+}
+
+static int mxc_isi_m2m_ctx_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct mxc_isi_m2m_ctx *ctx = ctrl_to_mxc_isi_m2m_ctx(ctrl);
+
+	switch (ctrl->id) {
+	case V4L2_CID_HFLIP:
+		ctx->ctrls.hflip = ctrl->val;
+		break;
+
+	case V4L2_CID_VFLIP:
+		ctx->ctrls.vflip = ctrl->val;
+		break;
+
+	case V4L2_CID_ALPHA_COMPONENT:
+		ctx->ctrls.alpha = ctrl->val;
+		break;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_ctrl_ops mxc_isi_m2m_ctx_ctrl_ops = {
+	.s_ctrl = mxc_isi_m2m_ctx_s_ctrl,
+};
+
+static int mxc_isi_m2m_ctx_ctrls_create(struct mxc_isi_m2m_ctx *ctx)
+{
+	struct v4l2_ctrl_handler *handler = &ctx->ctrls.handler;
+	int ret;
+
+	v4l2_ctrl_handler_init(handler, 3);
+
+	v4l2_ctrl_new_std(handler, &mxc_isi_m2m_ctx_ctrl_ops,
+			  V4L2_CID_ALPHA_COMPONENT, 0, 255, 1, 0);
+	v4l2_ctrl_new_std(handler, &mxc_isi_m2m_ctx_ctrl_ops,
+			  V4L2_CID_HFLIP, 0, 1, 1, 0);
+	v4l2_ctrl_new_std(handler, &mxc_isi_m2m_ctx_ctrl_ops,
+			  V4L2_CID_VFLIP, 0, 1, 1, 0);
+
+	if (handler->error) {
+		ret = handler->error;
+		v4l2_ctrl_handler_free(handler);
+		return ret;
+	}
+
+	ctx->fh.ctrl_handler = handler;
+
+	return 0;
+}
+
+static void mxc_isi_m2m_ctx_ctrls_delete(struct mxc_isi_m2m_ctx *ctx)
+{
+	v4l2_ctrl_handler_free(&ctx->ctrls.handler);
+}
+
+/* -----------------------------------------------------------------------------
+ * V4L2 ioctls
+ */
+
+static int mxc_isi_m2m_querycap(struct file *file, void *fh,
+				struct v4l2_capability *cap)
+{
+	strscpy(cap->driver, MXC_ISI_DRIVER_NAME, sizeof(cap->driver));
+	strscpy(cap->card, MXC_ISI_M2M, sizeof(cap->card));
+	cap->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M_MPLANE;
+	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
+
+	return 0;
+}
+
+static int mxc_isi_m2m_enum_fmt_vid(struct file *file, void *fh,
+				    struct v4l2_fmtdesc *f)
+{
+	const enum mxc_isi_video_type type =
+		f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE ?
+		MXC_ISI_VIDEO_M2M_OUT : MXC_ISI_VIDEO_M2M_CAP;
+	const struct mxc_isi_format_info *info;
+
+	info = mxc_isi_format_enum(f->index, type);
+	if (!info)
+		return -EINVAL;
+
+	f->pixelformat = info->fourcc;
+	f->flags |= V4L2_FMT_FLAG_CSC_COLORSPACE | V4L2_FMT_FLAG_CSC_YCBCR_ENC
+		 |  V4L2_FMT_FLAG_CSC_QUANTIZATION | V4L2_FMT_FLAG_CSC_XFER_FUNC;
+
+	return 0;
+}
+
+static const struct mxc_isi_format_info *
+__mxc_isi_m2m_try_fmt_vid(struct mxc_isi_m2m_ctx *ctx,
+			  struct v4l2_pix_format_mplane *pix,
+			  const enum mxc_isi_video_type type)
+{
+	if (type == MXC_ISI_VIDEO_M2M_CAP) {
+		/* Downscaling only  */
+		pix->width = min(pix->width, ctx->queues.out.format.width);
+		pix->height = min(pix->height, ctx->queues.out.format.height);
+	}
+
+	return mxc_isi_format_try(ctx->m2m->pipe, pix, type);
+}
+
+static int mxc_isi_m2m_try_fmt_vid(struct file *file, void *fh,
+				   struct v4l2_format *f)
+{
+	const enum mxc_isi_video_type type =
+		f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE ?
+		MXC_ISI_VIDEO_M2M_OUT : MXC_ISI_VIDEO_M2M_CAP;
+	struct mxc_isi_m2m_ctx *ctx = to_isi_m2m_ctx(fh);
+
+	__mxc_isi_m2m_try_fmt_vid(ctx, &f->fmt.pix_mp, type);
+
+	return 0;
+}
+
+static int mxc_isi_m2m_g_fmt_vid(struct file *file, void *fh,
+				 struct v4l2_format *f)
+{
+	struct mxc_isi_m2m_ctx *ctx = to_isi_m2m_ctx(fh);
+	const struct mxc_isi_m2m_ctx_queue_data *qdata =
+		mxc_isi_m2m_ctx_qdata(ctx, f->type);
+
+	f->fmt.pix_mp = qdata->format;
+
+	return 0;
+}
+
+static int mxc_isi_m2m_s_fmt_vid(struct file *file, void *fh,
+				 struct v4l2_format *f)
+{
+	const enum mxc_isi_video_type type =
+		f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE ?
+		MXC_ISI_VIDEO_M2M_OUT : MXC_ISI_VIDEO_M2M_CAP;
+	struct mxc_isi_m2m_ctx *ctx = to_isi_m2m_ctx(fh);
+	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
+	const struct mxc_isi_format_info *info;
+	struct vb2_queue *vq;
+
+	vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
+	if (!vq)
+		return -EINVAL;
+
+	if (vb2_is_busy(vq))
+		return -EBUSY;
+
+	info = __mxc_isi_m2m_try_fmt_vid(ctx, pix, type);
+
+	if (f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
+		ctx->queues.out.format = *pix;
+		ctx->queues.out.info = info;
+	}
+
+	/*
+	 * Always set the format on the capture side, due to either format
+	 * propagation or direct setting.
+	 */
+	ctx->queues.cap.format = *pix;
+	ctx->queues.cap.info = info;
+
+	return 0;
+}
+
+static int mxc_isi_m2m_streamon(struct file *file, void *fh,
+				enum v4l2_buf_type type)
+{
+	struct mxc_isi_m2m_ctx *ctx = to_isi_m2m_ctx(fh);
+	const struct v4l2_pix_format_mplane *out_pix = &ctx->queues.out.format;
+	const struct v4l2_pix_format_mplane *cap_pix = &ctx->queues.cap.format;
+	const struct mxc_isi_format_info *cap_info = ctx->queues.cap.info;
+	const struct mxc_isi_format_info *out_info = ctx->queues.out.info;
+	struct mxc_isi_m2m *m2m = ctx->m2m;
+	bool bypass;
+
+	int ret;
+
+	mutex_lock(&m2m->lock);
+
+	if (m2m->usage_count == INT_MAX) {
+		ret = -EOVERFLOW;
+		goto unlock;
+	}
+
+	bypass = cap_pix->width == out_pix->width &&
+		 cap_pix->height == out_pix->height &&
+		 cap_info->encoding == out_info->encoding;
+
+	/*
+	 * Acquire the pipe and initialize the channel with the first user of
+	 * the M2M device.
+	 */
+	if (m2m->usage_count == 0) {
+		ret = mxc_isi_channel_acquire(m2m->pipe,
+					      &mxc_isi_m2m_frame_write_done,
+					      bypass);
+		if (ret)
+			goto unlock;
+
+		mxc_isi_channel_get(m2m->pipe);
+	}
+
+	m2m->usage_count++;
+
+	/*
+	 * Allocate resources for the channel, counting how many users require
+	 * buffer chaining.
+	 */
+	if (!ctx->chained && out_pix->width > MXC_ISI_MAX_WIDTH_UNCHAINED) {
+		ret = mxc_isi_channel_chain(m2m->pipe, bypass);
+		if (ret)
+			goto deinit;
+
+		m2m->chained_count++;
+		ctx->chained = true;
+	}
+
+	/*
+	 * Drop the lock to start the stream, as the .device_run() operation
+	 * needs to acquire it.
+	 */
+	mutex_unlock(&m2m->lock);
+	ret = v4l2_m2m_ioctl_streamon(file, fh, type);
+	if (ret) {
+		/* Reacquire the lock for the cleanup path. */
+		mutex_lock(&m2m->lock);
+		goto unchain;
+	}
+
+	return 0;
+
+unchain:
+	if (ctx->chained && --m2m->chained_count == 0)
+		mxc_isi_channel_unchain(m2m->pipe);
+	ctx->chained = false;
+
+deinit:
+	if (--m2m->usage_count == 0) {
+		mxc_isi_channel_put(m2m->pipe);
+		mxc_isi_channel_release(m2m->pipe);
+	}
+
+unlock:
+	mutex_unlock(&m2m->lock);
+	return ret;
+}
+
+static int mxc_isi_m2m_streamoff(struct file *file, void *fh,
+				 enum v4l2_buf_type type)
+{
+	struct mxc_isi_m2m_ctx *ctx = to_isi_m2m_ctx(fh);
+	struct mxc_isi_m2m *m2m = ctx->m2m;
+
+	v4l2_m2m_ioctl_streamoff(file, fh, type);
+
+	mutex_lock(&m2m->lock);
+
+	/*
+	 * If the last context is this one, reset it to make sure the device
+	 * will be reconfigured when streaming is restarted.
+	 */
+	if (m2m->last_ctx == ctx)
+		m2m->last_ctx = NULL;
+
+	/* Free the channel resources if this is the last chained context. */
+	if (ctx->chained && --m2m->chained_count == 0)
+		mxc_isi_channel_unchain(m2m->pipe);
+	ctx->chained = false;
+
+	/* Turn off the light with the last user. */
+	if (--m2m->usage_count == 0) {
+		mxc_isi_channel_disable(m2m->pipe);
+		mxc_isi_channel_put(m2m->pipe);
+		mxc_isi_channel_release(m2m->pipe);
+	}
+
+	WARN_ON(m2m->usage_count < 0);
+
+	mutex_unlock(&m2m->lock);
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops mxc_isi_m2m_ioctl_ops = {
+	.vidioc_querycap		= mxc_isi_m2m_querycap,
+
+	.vidioc_enum_fmt_vid_cap	= mxc_isi_m2m_enum_fmt_vid,
+	.vidioc_enum_fmt_vid_out	= mxc_isi_m2m_enum_fmt_vid,
+	.vidioc_g_fmt_vid_cap_mplane	= mxc_isi_m2m_g_fmt_vid,
+	.vidioc_g_fmt_vid_out_mplane	= mxc_isi_m2m_g_fmt_vid,
+	.vidioc_s_fmt_vid_cap_mplane	= mxc_isi_m2m_s_fmt_vid,
+	.vidioc_s_fmt_vid_out_mplane	= mxc_isi_m2m_s_fmt_vid,
+	.vidioc_try_fmt_vid_cap_mplane	= mxc_isi_m2m_try_fmt_vid,
+	.vidioc_try_fmt_vid_out_mplane	= mxc_isi_m2m_try_fmt_vid,
+
+	.vidioc_reqbufs			= v4l2_m2m_ioctl_reqbufs,
+	.vidioc_querybuf		= v4l2_m2m_ioctl_querybuf,
+	.vidioc_qbuf			= v4l2_m2m_ioctl_qbuf,
+	.vidioc_dqbuf			= v4l2_m2m_ioctl_dqbuf,
+	.vidioc_expbuf			= v4l2_m2m_ioctl_expbuf,
+	.vidioc_prepare_buf		= v4l2_m2m_ioctl_prepare_buf,
+	.vidioc_create_bufs		= v4l2_m2m_ioctl_create_bufs,
+
+	.vidioc_streamon		= mxc_isi_m2m_streamon,
+	.vidioc_streamoff		= mxc_isi_m2m_streamoff,
+
+	.vidioc_subscribe_event		= v4l2_ctrl_subscribe_event,
+	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
+};
+
+/* -----------------------------------------------------------------------------
+ * Video device file operations
+ */
+
+static void mxc_isi_m2m_init_format(struct mxc_isi_m2m_ctx *ctx,
+				    struct mxc_isi_m2m_ctx_queue_data *qdata,
+				    enum mxc_isi_video_type type)
+{
+	qdata->format.width = MXC_ISI_DEF_WIDTH;
+	qdata->format.height = MXC_ISI_DEF_HEIGHT;
+	qdata->format.pixelformat = MXC_ISI_DEF_PIXEL_FORMAT;
+
+	qdata->info = mxc_isi_format_try(ctx->m2m->pipe, &qdata->format, type);
+}
+
+static int mxc_isi_m2m_open(struct file *file)
+{
+	struct video_device *vdev = video_devdata(file);
+	struct mxc_isi_m2m *m2m = video_drvdata(file);
+	struct mxc_isi_m2m_ctx *ctx;
+	int ret;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->m2m = m2m;
+	mutex_init(&ctx->vb2_lock);
+
+	v4l2_fh_init(&ctx->fh, vdev);
+	file->private_data = &ctx->fh;
+
+	ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(m2m->m2m_dev, ctx,
+					    &mxc_isi_m2m_queue_init);
+	if (IS_ERR(ctx->fh.m2m_ctx)) {
+		ret = PTR_ERR(ctx->fh.m2m_ctx);
+		ctx->fh.m2m_ctx = NULL;
+		goto err_fh;
+	}
+
+	mxc_isi_m2m_init_format(ctx, &ctx->queues.out, MXC_ISI_VIDEO_M2M_OUT);
+	mxc_isi_m2m_init_format(ctx, &ctx->queues.cap, MXC_ISI_VIDEO_M2M_CAP);
+
+	ret = mxc_isi_m2m_ctx_ctrls_create(ctx);
+	if (ret)
+		goto err_ctx;
+
+	ret = pm_runtime_resume_and_get(m2m->isi->dev);
+	if (ret)
+		goto err_ctrls;
+
+	v4l2_fh_add(&ctx->fh);
+
+	return 0;
+
+err_ctrls:
+	mxc_isi_m2m_ctx_ctrls_delete(ctx);
+err_ctx:
+	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+err_fh:
+	v4l2_fh_exit(&ctx->fh);
+	mutex_destroy(&ctx->vb2_lock);
+	kfree(ctx);
+	return ret;
+}
+
+static int mxc_isi_m2m_release(struct file *file)
+{
+	struct mxc_isi_m2m *m2m = video_drvdata(file);
+	struct mxc_isi_m2m_ctx *ctx = to_isi_m2m_ctx(file->private_data);
+
+	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+	mxc_isi_m2m_ctx_ctrls_delete(ctx);
+
+	v4l2_fh_del(&ctx->fh);
+	v4l2_fh_exit(&ctx->fh);
+
+	mutex_destroy(&ctx->vb2_lock);
+	kfree(ctx);
+
+	pm_runtime_put(m2m->isi->dev);
+
+	return 0;
+}
+
+static const struct v4l2_file_operations mxc_isi_m2m_fops = {
+	.owner		= THIS_MODULE,
+	.open		= mxc_isi_m2m_open,
+	.release	= mxc_isi_m2m_release,
+	.poll		= v4l2_m2m_fop_poll,
+	.unlocked_ioctl	= video_ioctl2,
+	.mmap		= v4l2_m2m_fop_mmap,
+};
+
+/* -----------------------------------------------------------------------------
+ * Registration
+ */
+
+int mxc_isi_m2m_register(struct mxc_isi_dev *isi, struct v4l2_device *v4l2_dev)
+{
+	struct mxc_isi_m2m *m2m = &isi->m2m;
+	struct video_device *vdev = &m2m->vdev;
+	struct media_link *link;
+	int ret;
+
+	m2m->isi = isi;
+	m2m->pipe = &isi->pipes[0];
+
+	mutex_init(&m2m->lock);
+
+	/* Initialize the video device and create controls. */
+	snprintf(vdev->name, sizeof(vdev->name), "mxc_isi.m2m");
+
+	vdev->fops	= &mxc_isi_m2m_fops;
+	vdev->ioctl_ops	= &mxc_isi_m2m_ioctl_ops;
+	vdev->v4l2_dev	= v4l2_dev;
+	vdev->minor	= -1;
+	vdev->release	= video_device_release_empty;
+	vdev->vfl_dir	= VFL_DIR_M2M;
+
+	vdev->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M_MPLANE;
+	video_set_drvdata(vdev, m2m);
+
+	/* Create the M2M device. */
+	m2m->m2m_dev = v4l2_m2m_init(&mxc_isi_m2m_ops);
+	if (IS_ERR(m2m->m2m_dev)) {
+		dev_err(isi->dev, "failed to initialize m2m device\n");
+		ret = PTR_ERR(m2m->m2m_dev);
+		goto err_mutex;
+	}
+
+	/* Register the video device. */
+	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
+	if (ret < 0) {
+		dev_err(isi->dev, "failed to register m2m device\n");
+		goto err_m2m;
+	}
+
+	/*
+	 * Populate the media graph. We can't use the mem2mem helper
+	 * v4l2_m2m_register_media_controller() as the M2M interface needs to
+	 * be connected to the existing entities in the graph, so we have to
+	 * wire things up manually:
+	 *
+	 * - The entity in the video_device, which isn't touched by the V4L2
+	 *   core for M2M devices, is used as the source I/O entity in the
+	 *   graph, connected to the crossbar switch.
+	 *
+	 * - The video device at the end of the pipeline provides the sink
+	 *   entity, and is already wired up in the graph.
+	 *
+	 * - A new interface is created, pointing at both entities. The sink
+	 *   entity will thus have two interfaces pointing to it.
+	 */
+	m2m->pad.flags = MEDIA_PAD_FL_SOURCE;
+	vdev->entity.name = "mxc_isi.output";
+	vdev->entity.function = MEDIA_ENT_F_IO_V4L;
+	ret = media_entity_pads_init(&vdev->entity, 1, &m2m->pad);
+	if (ret)
+		goto err_video;
+
+	ret = media_device_register_entity(v4l2_dev->mdev, &vdev->entity);
+	if (ret)
+		goto err_entity_cleanup;
+
+	ret = media_create_pad_link(&vdev->entity, 0,
+				    &m2m->isi->crossbar.sd.entity,
+				    m2m->isi->crossbar.num_sinks - 1,
+				    MEDIA_LNK_FL_IMMUTABLE |
+				    MEDIA_LNK_FL_ENABLED);
+	if (ret)
+		goto err_entity_unreg;
+
+	m2m->intf = media_devnode_create(v4l2_dev->mdev, MEDIA_INTF_T_V4L_VIDEO,
+					 0, VIDEO_MAJOR, vdev->minor);
+	if (!m2m->intf) {
+		ret = -ENOMEM;
+		goto err_entity_unreg;
+	}
+
+	link = media_create_intf_link(&vdev->entity, &m2m->intf->intf,
+				      MEDIA_LNK_FL_IMMUTABLE |
+				      MEDIA_LNK_FL_ENABLED);
+	if (!link) {
+		ret = -ENOMEM;
+		goto err_devnode;
+	}
+
+	link = media_create_intf_link(&m2m->pipe->video.vdev.entity,
+				      &m2m->intf->intf,
+				      MEDIA_LNK_FL_IMMUTABLE |
+				      MEDIA_LNK_FL_ENABLED);
+	if (!link) {
+		ret = -ENOMEM;
+		goto err_devnode;
+	}
+
+	return 0;
+
+err_devnode:
+	media_devnode_remove(m2m->intf);
+err_entity_unreg:
+	media_device_unregister_entity(&vdev->entity);
+err_entity_cleanup:
+	media_entity_cleanup(&vdev->entity);
+err_video:
+	video_unregister_device(vdev);
+err_m2m:
+	v4l2_m2m_release(m2m->m2m_dev);
+err_mutex:
+	mutex_destroy(&m2m->lock);
+	return ret;
+}
+
+int mxc_isi_m2m_unregister(struct mxc_isi_dev *isi)
+{
+	struct mxc_isi_m2m *m2m = &isi->m2m;
+	struct video_device *vdev = &m2m->vdev;
+
+	video_unregister_device(vdev);
+
+	v4l2_m2m_release(m2m->m2m_dev);
+	media_devnode_remove(m2m->intf);
+	media_entity_cleanup(&vdev->entity);
+	mutex_destroy(&m2m->lock);
+
+	return 0;
+}
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
new file mode 100644
index 000000000000..c4454aa1cb34
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
@@ -0,0 +1,867 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * V4L2 Capture ISI subdev driver for i.MX8QXP/QM platform
+ *
+ * ISI is a Image Sensor Interface of i.MX8QXP/QM platform, which
+ * used to process image from camera sensor to memory or DC
+ *
+ * Copyright (c) 2019 NXP Semiconductor
+ */
+
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/minmax.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+#include <linux/videodev2.h>
+
+#include <media/media-entity.h>
+#include <media/v4l2-subdev.h>
+#include <media/videobuf2-v4l2.h>
+
+#include "imx8-isi-core.h"
+#include "imx8-isi-regs.h"
+
+/*
+ * While the ISI receives data from the gasket on a 3x12-bit bus, the pipeline
+ * subdev conceptually includes the gasket in order to avoid exposing an extra
+ * subdev between the CSIS and the ISI. We thus need to expose media bus codes
+ * corresponding to the CSIS output, which is narrower.
+ */
+static const struct mxc_isi_bus_format_info mxc_isi_bus_formats[] = {
+	/* YUV formats */
+	{
+		.mbus_code	= MEDIA_BUS_FMT_UYVY8_1X16,
+		.output		= MEDIA_BUS_FMT_YUV8_1X24,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK),
+		.encoding	= MXC_ISI_ENC_YUV,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_YUV8_1X24,
+		.output		= MEDIA_BUS_FMT_YUV8_1X24,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_YUV,
+	},
+	/* RGB formats */
+	{
+		.mbus_code	= MEDIA_BUS_FMT_RGB565_1X16,
+		.output		= MEDIA_BUS_FMT_RGB888_1X24,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK),
+		.encoding	= MXC_ISI_ENC_RGB,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_RGB888_1X24,
+		.output		= MEDIA_BUS_FMT_RGB888_1X24,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RGB,
+	},
+	/* RAW formats */
+	{
+		.mbus_code	= MEDIA_BUS_FMT_Y8_1X8,
+		.output		= MEDIA_BUS_FMT_Y8_1X8,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_Y10_1X10,
+		.output		= MEDIA_BUS_FMT_Y10_1X10,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_Y12_1X12,
+		.output		= MEDIA_BUS_FMT_Y12_1X12,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_Y14_1X14,
+		.output		= MEDIA_BUS_FMT_Y14_1X14,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR8_1X8,
+		.output		= MEDIA_BUS_FMT_SBGGR8_1X8,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG8_1X8,
+		.output		= MEDIA_BUS_FMT_SGBRG8_1X8,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG8_1X8,
+		.output		= MEDIA_BUS_FMT_SGRBG8_1X8,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB8_1X8,
+		.output		= MEDIA_BUS_FMT_SRGGB8_1X8,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR10_1X10,
+		.output		= MEDIA_BUS_FMT_SBGGR10_1X10,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG10_1X10,
+		.output		= MEDIA_BUS_FMT_SGBRG10_1X10,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG10_1X10,
+		.output		= MEDIA_BUS_FMT_SGRBG10_1X10,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB10_1X10,
+		.output		= MEDIA_BUS_FMT_SRGGB10_1X10,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR12_1X12,
+		.output		= MEDIA_BUS_FMT_SBGGR12_1X12,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG12_1X12,
+		.output		= MEDIA_BUS_FMT_SGBRG12_1X12,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG12_1X12,
+		.output		= MEDIA_BUS_FMT_SGRBG12_1X12,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB12_1X12,
+		.output		= MEDIA_BUS_FMT_SRGGB12_1X12,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR14_1X14,
+		.output		= MEDIA_BUS_FMT_SBGGR14_1X14,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG14_1X14,
+		.output		= MEDIA_BUS_FMT_SGBRG14_1X14,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG14_1X14,
+		.output		= MEDIA_BUS_FMT_SGRBG14_1X14,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB14_1X14,
+		.output		= MEDIA_BUS_FMT_SRGGB14_1X14,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	},
+	/* JPEG */
+	{
+		.mbus_code	= MEDIA_BUS_FMT_JPEG_1X8,
+		.output		= MEDIA_BUS_FMT_JPEG_1X8,
+		.pads		= BIT(MXC_ISI_PIPE_PAD_SINK)
+				| BIT(MXC_ISI_PIPE_PAD_SOURCE),
+		.encoding	= MXC_ISI_ENC_RAW,
+	}
+};
+
+const struct mxc_isi_bus_format_info *
+mxc_isi_bus_format_by_code(u32 code, unsigned int pad)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(mxc_isi_bus_formats); i++) {
+		const struct mxc_isi_bus_format_info *info =
+			&mxc_isi_bus_formats[i];
+
+		if (info->mbus_code == code && info->pads & BIT(pad))
+			return info;
+	}
+
+	return NULL;
+}
+
+const struct mxc_isi_bus_format_info *
+mxc_isi_bus_format_by_index(unsigned int index, unsigned int pad)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(mxc_isi_bus_formats); i++) {
+		const struct mxc_isi_bus_format_info *info =
+			&mxc_isi_bus_formats[i];
+
+		if (!(info->pads & BIT(pad)))
+			continue;
+
+		if (!index)
+			return info;
+
+		index--;
+	}
+
+	return NULL;
+}
+
+static inline struct mxc_isi_pipe *to_isi_pipe(struct v4l2_subdev *sd)
+{
+	return container_of(sd, struct mxc_isi_pipe, sd);
+}
+
+int mxc_isi_pipe_enable(struct mxc_isi_pipe *pipe)
+{
+	struct mxc_isi_crossbar *xbar = &pipe->isi->crossbar;
+	const struct mxc_isi_bus_format_info *sink_info;
+	const struct mxc_isi_bus_format_info *src_info;
+	const struct v4l2_mbus_framefmt *sink_fmt;
+	const struct v4l2_mbus_framefmt *src_fmt;
+	const struct v4l2_rect *compose;
+	struct v4l2_subdev_state *state;
+	struct v4l2_subdev *sd = &pipe->sd;
+	struct v4l2_area in_size, scale;
+	struct v4l2_rect crop;
+	u32 input;
+	int ret;
+
+	/*
+	 * Find the connected input by inspecting the crossbar switch routing
+	 * table.
+	 */
+	state = v4l2_subdev_lock_and_get_active_state(&xbar->sd);
+	ret = v4l2_subdev_routing_find_opposite_end(&state->routing,
+						    xbar->num_sinks + pipe->id,
+						    0, &input, NULL);
+	v4l2_subdev_unlock_state(state);
+
+	if (ret)
+		return -EPIPE;
+
+	/* Configure the pipeline. */
+	state = v4l2_subdev_lock_and_get_active_state(sd);
+
+	sink_fmt = v4l2_subdev_get_try_format(sd, state, MXC_ISI_PIPE_PAD_SINK);
+	src_fmt = v4l2_subdev_get_try_format(sd, state, MXC_ISI_PIPE_PAD_SOURCE);
+	compose = v4l2_subdev_get_try_compose(sd, state, MXC_ISI_PIPE_PAD_SINK);
+	crop = *v4l2_subdev_get_try_crop(sd, state, MXC_ISI_PIPE_PAD_SOURCE);
+
+	sink_info = mxc_isi_bus_format_by_code(sink_fmt->code,
+					       MXC_ISI_PIPE_PAD_SINK);
+	src_info = mxc_isi_bus_format_by_code(src_fmt->code,
+					      MXC_ISI_PIPE_PAD_SOURCE);
+
+	in_size.width = sink_fmt->width;
+	in_size.height = sink_fmt->height;
+	scale.width = compose->width;
+	scale.height = compose->height;
+
+	v4l2_subdev_unlock_state(state);
+
+	/* Configure the ISI channel. */
+	mxc_isi_channel_config(pipe, input, &in_size, &scale, &crop,
+			       sink_info->encoding, src_info->encoding);
+
+	mxc_isi_channel_enable(pipe);
+
+	/* Enable streams on the crossbar switch. */
+	ret = v4l2_subdev_enable_streams(&xbar->sd, xbar->num_sinks + pipe->id,
+					 BIT(0));
+	if (ret) {
+		mxc_isi_channel_disable(pipe);
+		dev_err(pipe->isi->dev, "Failed to enable pipe %u\n",
+			pipe->id);
+		return ret;
+	}
+
+	return 0;
+}
+
+void mxc_isi_pipe_disable(struct mxc_isi_pipe *pipe)
+{
+	struct mxc_isi_crossbar *xbar = &pipe->isi->crossbar;
+	int ret;
+
+	ret = v4l2_subdev_disable_streams(&xbar->sd, xbar->num_sinks + pipe->id,
+					  BIT(0));
+	if (ret)
+		dev_err(pipe->isi->dev, "Failed to disable pipe %u\n",
+			pipe->id);
+
+	mxc_isi_channel_disable(pipe);
+}
+
+/* -----------------------------------------------------------------------------
+ * V4L2 subdev operations
+ */
+
+static struct v4l2_mbus_framefmt *
+mxc_isi_pipe_get_pad_format(struct mxc_isi_pipe *pipe,
+			    struct v4l2_subdev_state *state,
+			    unsigned int pad)
+{
+	return v4l2_subdev_get_try_format(&pipe->sd, state, pad);
+}
+
+static struct v4l2_rect *
+mxc_isi_pipe_get_pad_crop(struct mxc_isi_pipe *pipe,
+			  struct v4l2_subdev_state *state,
+			  unsigned int pad)
+{
+	return v4l2_subdev_get_try_crop(&pipe->sd, state, pad);
+}
+
+static struct v4l2_rect *
+mxc_isi_pipe_get_pad_compose(struct mxc_isi_pipe *pipe,
+			     struct v4l2_subdev_state *state,
+			     unsigned int pad)
+{
+	return v4l2_subdev_get_try_compose(&pipe->sd, state, pad);
+}
+
+static int mxc_isi_pipe_init_cfg(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *state)
+{
+	struct mxc_isi_pipe *pipe = to_isi_pipe(sd);
+	struct v4l2_mbus_framefmt *fmt_source;
+	struct v4l2_mbus_framefmt *fmt_sink;
+	struct v4l2_rect *compose;
+	struct v4l2_rect *crop;
+
+	fmt_sink = mxc_isi_pipe_get_pad_format(pipe, state,
+					       MXC_ISI_PIPE_PAD_SINK);
+	fmt_source = mxc_isi_pipe_get_pad_format(pipe, state,
+						 MXC_ISI_PIPE_PAD_SOURCE);
+
+	fmt_sink->width = MXC_ISI_DEF_WIDTH;
+	fmt_sink->height = MXC_ISI_DEF_HEIGHT;
+	fmt_sink->code = MXC_ISI_DEF_MBUS_CODE_SINK;
+	fmt_sink->field = V4L2_FIELD_NONE;
+	fmt_sink->colorspace = V4L2_COLORSPACE_JPEG;
+	fmt_sink->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(fmt_sink->colorspace);
+	fmt_sink->quantization =
+		V4L2_MAP_QUANTIZATION_DEFAULT(false, fmt_sink->colorspace,
+					      fmt_sink->ycbcr_enc);
+	fmt_sink->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(fmt_sink->colorspace);
+
+	*fmt_source = *fmt_sink;
+	fmt_source->code = MXC_ISI_DEF_MBUS_CODE_SOURCE;
+
+	compose = mxc_isi_pipe_get_pad_compose(pipe, state,
+					       MXC_ISI_PIPE_PAD_SINK);
+	crop = mxc_isi_pipe_get_pad_crop(pipe, state, MXC_ISI_PIPE_PAD_SOURCE);
+
+	compose->left = 0;
+	compose->top = 0;
+	compose->width = MXC_ISI_DEF_WIDTH;
+	compose->height = MXC_ISI_DEF_HEIGHT;
+
+	*crop = *compose;
+
+	return 0;
+}
+
+static int mxc_isi_pipe_enum_mbus_code(struct v4l2_subdev *sd,
+				       struct v4l2_subdev_state *state,
+				       struct v4l2_subdev_mbus_code_enum *code)
+{
+	static const u32 output_codes[] = {
+		MEDIA_BUS_FMT_YUV8_1X24,
+		MEDIA_BUS_FMT_RGB888_1X24,
+	};
+	struct mxc_isi_pipe *pipe = to_isi_pipe(sd);
+	const struct mxc_isi_bus_format_info *info;
+	unsigned int index;
+	unsigned int i;
+
+	if (code->pad == MXC_ISI_PIPE_PAD_SOURCE) {
+		const struct v4l2_mbus_framefmt *format;
+
+		format = mxc_isi_pipe_get_pad_format(pipe, state,
+						     MXC_ISI_PIPE_PAD_SINK);
+		info = mxc_isi_bus_format_by_code(format->code,
+						  MXC_ISI_PIPE_PAD_SINK);
+
+		if (info->encoding == MXC_ISI_ENC_RAW) {
+			/*
+			 * For RAW formats, the sink and source media bus codes
+			 * must match.
+			 */
+			if (code->index)
+				return -EINVAL;
+
+			code->code = info->output;
+		} else {
+			/*
+			 * For RGB or YUV formats, the ISI supports format
+			 * conversion. Either of the two output formats can be
+			 * used regardless of the input.
+			 */
+			if (code->index > 1)
+				return -EINVAL;
+
+			code->code = output_codes[code->index];
+		}
+
+		return 0;
+	}
+
+	index = code->index;
+
+	for (i = 0; i < ARRAY_SIZE(mxc_isi_bus_formats); ++i) {
+		info = &mxc_isi_bus_formats[i];
+
+		if (!(info->pads & BIT(MXC_ISI_PIPE_PAD_SINK)))
+			continue;
+
+		if (index == 0) {
+			code->code = info->mbus_code;
+			return 0;
+		}
+
+		index--;
+	}
+
+	return -EINVAL;
+}
+
+static int mxc_isi_pipe_set_fmt(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *state,
+				struct v4l2_subdev_format *fmt)
+{
+	struct mxc_isi_pipe *pipe = to_isi_pipe(sd);
+	struct v4l2_mbus_framefmt *mf = &fmt->format;
+	const struct mxc_isi_bus_format_info *info;
+	struct v4l2_mbus_framefmt *format;
+	struct v4l2_rect *rect;
+
+	if (vb2_is_busy(&pipe->video.vb2_q))
+		return -EBUSY;
+
+	if (fmt->pad == MXC_ISI_PIPE_PAD_SINK) {
+		unsigned int max_width;
+
+		info = mxc_isi_bus_format_by_code(mf->code,
+						  MXC_ISI_PIPE_PAD_SINK);
+		if (!info)
+			info = mxc_isi_bus_format_by_code(MXC_ISI_DEF_MBUS_CODE_SINK,
+							  MXC_ISI_PIPE_PAD_SINK);
+
+		/*
+		 * Limit the max line length if there's no adjacent pipe to
+		 * chain with.
+		 */
+		max_width = pipe->id == pipe->isi->pdata->num_channels - 1
+			  ? MXC_ISI_MAX_WIDTH_UNCHAINED
+			  : MXC_ISI_MAX_WIDTH_CHAINED;
+
+		mf->code = info->mbus_code;
+		mf->width = clamp(mf->width, MXC_ISI_MIN_WIDTH, max_width);
+		mf->height = clamp(mf->height, MXC_ISI_MIN_HEIGHT,
+				   MXC_ISI_MAX_HEIGHT);
+
+		/* Propagate the format to the source pad. */
+		rect = mxc_isi_pipe_get_pad_compose(pipe, state,
+						    MXC_ISI_PIPE_PAD_SINK);
+		rect->width = mf->width;
+		rect->height = mf->height;
+
+		rect = mxc_isi_pipe_get_pad_crop(pipe, state,
+						 MXC_ISI_PIPE_PAD_SOURCE);
+		rect->left = 0;
+		rect->top = 0;
+		rect->width = mf->width;
+		rect->height = mf->height;
+
+		format = mxc_isi_pipe_get_pad_format(pipe, state,
+						     MXC_ISI_PIPE_PAD_SOURCE);
+		format->code = info->output;
+		format->width = mf->width;
+		format->height = mf->height;
+	} else {
+		/*
+		 * For RGB or YUV formats, the ISI supports RGB <-> YUV format
+		 * conversion. For RAW formats, the sink and source media bus
+		 * codes must match.
+		 */
+		format = mxc_isi_pipe_get_pad_format(pipe, state,
+						     MXC_ISI_PIPE_PAD_SINK);
+		info = mxc_isi_bus_format_by_code(format->code,
+						  MXC_ISI_PIPE_PAD_SINK);
+
+		if (info->encoding != MXC_ISI_ENC_RAW) {
+			if (mf->code != MEDIA_BUS_FMT_YUV8_1X24 &&
+			    mf->code != MEDIA_BUS_FMT_RGB888_1X24)
+				mf->code = info->output;
+
+			info = mxc_isi_bus_format_by_code(mf->code,
+							  MXC_ISI_PIPE_PAD_SOURCE);
+		}
+
+		mf->code = info->output;
+
+		/*
+		 * The width and height on the source can't be changed, they
+		 * must match the crop rectangle size.
+		 */
+		rect = mxc_isi_pipe_get_pad_crop(pipe, state,
+						 MXC_ISI_PIPE_PAD_SOURCE);
+
+		mf->width = rect->width;
+		mf->height = rect->height;
+	}
+
+	format = mxc_isi_pipe_get_pad_format(pipe, state, fmt->pad);
+	*format = *mf;
+
+	dev_dbg(pipe->isi->dev, "pad%u: code: 0x%04x, %ux%u",
+		fmt->pad, mf->code, mf->width, mf->height);
+
+	return 0;
+}
+
+static int mxc_isi_pipe_get_selection(struct v4l2_subdev *sd,
+				      struct v4l2_subdev_state *state,
+				      struct v4l2_subdev_selection *sel)
+{
+	struct mxc_isi_pipe *pipe = to_isi_pipe(sd);
+	const struct v4l2_mbus_framefmt *format;
+	const struct v4l2_rect *rect;
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+		if (sel->pad != MXC_ISI_PIPE_PAD_SINK)
+			/* No compose rectangle on source pad. */
+			return -EINVAL;
+
+		/* The sink compose is bound by the sink format. */
+		format = mxc_isi_pipe_get_pad_format(pipe, state,
+						     MXC_ISI_PIPE_PAD_SINK);
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = format->width;
+		sel->r.height = format->height;
+		break;
+
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		if (sel->pad != MXC_ISI_PIPE_PAD_SOURCE)
+			/* No crop rectangle on sink pad. */
+			return -EINVAL;
+
+		/* The source crop is bound by the sink compose. */
+		rect = mxc_isi_pipe_get_pad_compose(pipe, state,
+						    MXC_ISI_PIPE_PAD_SINK);
+		sel->r = *rect;
+		break;
+
+	case V4L2_SEL_TGT_CROP:
+		if (sel->pad != MXC_ISI_PIPE_PAD_SOURCE)
+			/* No crop rectangle on sink pad. */
+			return -EINVAL;
+
+		rect = mxc_isi_pipe_get_pad_crop(pipe, state, sel->pad);
+		sel->r = *rect;
+		break;
+
+	case V4L2_SEL_TGT_COMPOSE:
+		if (sel->pad != MXC_ISI_PIPE_PAD_SINK)
+			/* No compose rectangle on source pad. */
+			return -EINVAL;
+
+		rect = mxc_isi_pipe_get_pad_compose(pipe, state, sel->pad);
+		sel->r = *rect;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int mxc_isi_pipe_set_selection(struct v4l2_subdev *sd,
+				      struct v4l2_subdev_state *state,
+				      struct v4l2_subdev_selection *sel)
+{
+	struct mxc_isi_pipe *pipe = to_isi_pipe(sd);
+	struct v4l2_mbus_framefmt *format;
+	struct v4l2_rect *rect;
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		if (sel->pad != MXC_ISI_PIPE_PAD_SOURCE)
+			/* The pipeline support cropping on the source only. */
+			return -EINVAL;
+
+		/* The source crop is bound by the sink compose. */
+		rect = mxc_isi_pipe_get_pad_compose(pipe, state,
+						    MXC_ISI_PIPE_PAD_SINK);
+		sel->r.left = clamp_t(s32, sel->r.left, 0, rect->width - 1);
+		sel->r.top = clamp_t(s32, sel->r.top, 0, rect->height - 1);
+		sel->r.width = clamp(sel->r.width, MXC_ISI_MIN_WIDTH,
+				     rect->width - sel->r.left);
+		sel->r.height = clamp(sel->r.height, MXC_ISI_MIN_HEIGHT,
+				      rect->height - sel->r.top);
+
+		rect = mxc_isi_pipe_get_pad_crop(pipe, state,
+						 MXC_ISI_PIPE_PAD_SOURCE);
+		*rect = sel->r;
+
+		/* Propagate the crop rectangle to the source pad. */
+		format = mxc_isi_pipe_get_pad_format(pipe, state,
+						     MXC_ISI_PIPE_PAD_SOURCE);
+		format->width = sel->r.width;
+		format->height = sel->r.height;
+		break;
+
+	case V4L2_SEL_TGT_COMPOSE:
+		if (sel->pad != MXC_ISI_PIPE_PAD_SINK)
+			/* Composing is supported on the sink only. */
+			return -EINVAL;
+
+		/* The sink crop is bound by the sink format downscaling only). */
+		format = mxc_isi_pipe_get_pad_format(pipe, state,
+						     MXC_ISI_PIPE_PAD_SINK);
+
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = clamp(sel->r.width, MXC_ISI_MIN_WIDTH,
+				     format->width);
+		sel->r.height = clamp(sel->r.height, MXC_ISI_MIN_HEIGHT,
+				      format->height);
+
+		rect = mxc_isi_pipe_get_pad_compose(pipe, state,
+						    MXC_ISI_PIPE_PAD_SINK);
+		*rect = sel->r;
+
+		/* Propagate the compose rectangle to the source pad. */
+		rect = mxc_isi_pipe_get_pad_crop(pipe, state,
+						 MXC_ISI_PIPE_PAD_SOURCE);
+		rect->left = 0;
+		rect->top = 0;
+		rect->width = sel->r.width;
+		rect->height = sel->r.height;
+
+		format = mxc_isi_pipe_get_pad_format(pipe, state,
+						     MXC_ISI_PIPE_PAD_SOURCE);
+		format->width = sel->r.width;
+		format->height = sel->r.height;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	dev_dbg(pipe->isi->dev, "%s, target %#x: (%d,%d)/%dx%d", __func__,
+		sel->target, sel->r.left, sel->r.top, sel->r.width,
+		sel->r.height);
+
+	return 0;
+}
+
+static const struct v4l2_subdev_pad_ops mxc_isi_pipe_subdev_pad_ops = {
+	.init_cfg = mxc_isi_pipe_init_cfg,
+	.enum_mbus_code = mxc_isi_pipe_enum_mbus_code,
+	.get_fmt = v4l2_subdev_get_fmt,
+	.set_fmt = mxc_isi_pipe_set_fmt,
+	.get_selection = mxc_isi_pipe_get_selection,
+	.set_selection = mxc_isi_pipe_set_selection,
+};
+
+static const struct v4l2_subdev_ops mxc_isi_pipe_subdev_ops = {
+	.pad = &mxc_isi_pipe_subdev_pad_ops,
+};
+
+/* -----------------------------------------------------------------------------
+ * IRQ handling
+ */
+
+static irqreturn_t mxc_isi_pipe_irq_handler(int irq, void *priv)
+{
+	struct mxc_isi_pipe *pipe = priv;
+	const struct mxc_isi_ier_reg *ier_reg = pipe->isi->pdata->ier_reg;
+	u32 status;
+
+	status = mxc_isi_channel_irq_status(pipe, true);
+
+	if (status & CHNL_STS_FRM_STRD) {
+		if (!WARN_ON(!pipe->irq_handler))
+			pipe->irq_handler(pipe, status);
+	}
+
+	if (status & (CHNL_STS_AXI_WR_ERR_Y |
+		      CHNL_STS_AXI_WR_ERR_U |
+		      CHNL_STS_AXI_WR_ERR_V))
+		dev_dbg(pipe->isi->dev, "%s: IRQ AXI Error stat=0x%X\n",
+			__func__, status);
+
+	if (status & (ier_reg->panic_y_buf_en.mask |
+		      ier_reg->panic_u_buf_en.mask |
+		      ier_reg->panic_v_buf_en.mask))
+		dev_dbg(pipe->isi->dev, "%s: IRQ Panic OFLW Error stat=0x%X\n",
+			__func__, status);
+
+	if (status & (ier_reg->oflw_y_buf_en.mask |
+		      ier_reg->oflw_u_buf_en.mask |
+		      ier_reg->oflw_v_buf_en.mask))
+		dev_dbg(pipe->isi->dev, "%s: IRQ OFLW Error stat=0x%X\n",
+			__func__, status);
+
+	if (status & (ier_reg->excs_oflw_y_buf_en.mask |
+		      ier_reg->excs_oflw_u_buf_en.mask |
+		      ier_reg->excs_oflw_v_buf_en.mask))
+		dev_dbg(pipe->isi->dev, "%s: IRQ EXCS OFLW Error stat=0x%X\n",
+			__func__, status);
+
+	return IRQ_HANDLED;
+}
+
+/* -----------------------------------------------------------------------------
+ * Init & cleanup
+ */
+
+static const struct media_entity_operations mxc_isi_pipe_entity_ops = {
+	.link_validate	= v4l2_subdev_link_validate,
+};
+
+int mxc_isi_pipe_init(struct mxc_isi_dev *isi, unsigned int id)
+{
+	struct mxc_isi_pipe *pipe = &isi->pipes[id];
+	struct v4l2_subdev *sd;
+	int irq;
+	int ret;
+
+	pipe->id = id;
+	pipe->isi = isi;
+	pipe->regs = isi->regs + id * isi->pdata->reg_offset;
+
+	mutex_init(&pipe->lock);
+
+	pipe->available_res = MXC_ISI_CHANNEL_RES_LINE_BUF
+			    | MXC_ISI_CHANNEL_RES_OUTPUT_BUF;
+	pipe->acquired_res = 0;
+	pipe->chained_res = 0;
+	pipe->chained = false;
+
+	sd = &pipe->sd;
+	v4l2_subdev_init(sd, &mxc_isi_pipe_subdev_ops);
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	snprintf(sd->name, sizeof(sd->name), "mxc_isi.%d", pipe->id);
+	sd->dev = isi->dev;
+
+	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
+	sd->entity.ops = &mxc_isi_pipe_entity_ops;
+
+	pipe->pads[MXC_ISI_PIPE_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
+	pipe->pads[MXC_ISI_PIPE_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&sd->entity, MXC_ISI_PIPE_PADS_NUM,
+				     pipe->pads);
+	if (ret)
+		goto error;
+
+	ret = v4l2_subdev_init_finalize(sd);
+	if (ret < 0)
+		goto error;
+
+	/* Register IRQ handler. */
+	mxc_isi_channel_irq_clear(pipe);
+
+	irq = platform_get_irq(to_platform_device(isi->dev), id);
+	if (irq < 0) {
+		dev_err(pipe->isi->dev, "Failed to get IRQ (%d)\n", irq);
+		ret = irq;
+		goto error;
+	}
+
+	ret = devm_request_irq(isi->dev, irq, mxc_isi_pipe_irq_handler,
+			       0, dev_name(isi->dev), pipe);
+	if (ret < 0) {
+		dev_err(isi->dev, "failed to request IRQ (%d)\n", ret);
+		goto error;
+	}
+
+	return 0;
+
+error:
+	media_entity_cleanup(&sd->entity);
+	mutex_destroy(&pipe->lock);
+
+	return ret;
+}
+
+void mxc_isi_pipe_cleanup(struct mxc_isi_pipe *pipe)
+{
+	struct v4l2_subdev *sd = &pipe->sd;
+
+	media_entity_cleanup(&sd->entity);
+	mutex_destroy(&pipe->lock);
+}
+
+int mxc_isi_pipe_acquire(struct mxc_isi_pipe *pipe,
+			 mxc_isi_pipe_irq_t irq_handler)
+{
+	const struct mxc_isi_bus_format_info *sink_info;
+	const struct mxc_isi_bus_format_info *src_info;
+	struct v4l2_mbus_framefmt *sink_fmt;
+	const struct v4l2_mbus_framefmt *src_fmt;
+	struct v4l2_subdev *sd = &pipe->sd;
+	struct v4l2_subdev_state *state;
+	bool bypass;
+	int ret;
+
+	state = v4l2_subdev_lock_and_get_active_state(sd);
+	sink_fmt = v4l2_subdev_get_try_format(sd, state, MXC_ISI_PIPE_PAD_SINK);
+	src_fmt = v4l2_subdev_get_try_format(sd, state, MXC_ISI_PIPE_PAD_SOURCE);
+	v4l2_subdev_unlock_state(state);
+
+	sink_info = mxc_isi_bus_format_by_code(sink_fmt->code,
+					       MXC_ISI_PIPE_PAD_SINK);
+	src_info = mxc_isi_bus_format_by_code(src_fmt->code,
+					      MXC_ISI_PIPE_PAD_SOURCE);
+
+	bypass = sink_fmt->width == src_fmt->width &&
+		 sink_fmt->height == src_fmt->height &&
+		 sink_info->encoding == src_info->encoding;
+
+	ret = mxc_isi_channel_acquire(pipe, irq_handler, bypass);
+	if (ret)
+		return ret;
+
+	/* Chain the channel if needed for wide resolutions. */
+	if (sink_fmt->width > MXC_ISI_MAX_WIDTH_UNCHAINED) {
+		ret = mxc_isi_channel_chain(pipe, bypass);
+		if (ret)
+			mxc_isi_channel_release(pipe);
+	}
+
+	return ret;
+}
+
+void mxc_isi_pipe_release(struct mxc_isi_pipe *pipe)
+{
+	mxc_isi_channel_release(pipe);
+	mxc_isi_channel_unchain(pipe);
+}
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-regs.h b/drivers/media/platform/nxp/imx8-isi/imx8-isi-regs.h
new file mode 100644
index 000000000000..1b65eccdf0da
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-regs.h
@@ -0,0 +1,418 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2019-2020 NXP
+ */
+
+#ifndef __IMX8_ISI_REGS_H__
+#define __IMX8_ISI_REGS_H__
+
+#include <linux/bits.h>
+
+/* ISI Registers Define  */
+/* Channel Control Register */
+#define CHNL_CTRL						0x0000
+#define CHNL_CTRL_CHNL_EN					BIT(31)
+#define CHNL_CTRL_CLK_EN					BIT(30)
+#define CHNL_CTRL_CHNL_BYPASS					BIT(29)
+#define CHNL_CTRL_CHAIN_BUF(n)					((n) << 25)
+#define CHNL_CTRL_CHAIN_BUF_MASK				GENMASK(26, 25)
+#define CHNL_CTRL_CHAIN_BUF_NO_CHAIN				0
+#define CHNL_CTRL_CHAIN_BUF_2_CHAIN				1
+#define CHNL_CTRL_SW_RST					BIT(24)
+#define CHNL_CTRL_BLANK_PXL(n)					((n) << 16)
+#define CHNL_CTRL_BLANK_PXL_MASK				GENMASK(23, 16)
+#define CHNL_CTRL_MIPI_VC_ID(n)					((n) << 6)
+#define CHNL_CTRL_MIPI_VC_ID_MASK				GENMASK(7, 6)
+#define CHNL_CTRL_SRC_TYPE(n)					((n) << 4)
+#define CHNL_CTRL_SRC_TYPE_MASK					BIT(4)
+#define CHNL_CTRL_SRC_TYPE_DEVICE				0
+#define CHNL_CTRL_SRC_TYPE_MEMORY				1
+#define CHNL_CTRL_SRC_INPUT(n)					((n) << 0)
+#define CHNL_CTRL_SRC_INPUT_MASK				GENMASK(2, 0)
+
+/* Channel Image Control Register */
+#define CHNL_IMG_CTRL						0x0004
+#define CHNL_IMG_CTRL_FORMAT(n)					((n) << 24)
+#define CHNL_IMG_CTRL_FORMAT_MASK				GENMASK(29, 24)
+#define CHNL_IMG_CTRL_FORMAT_RGBA8888				0x00
+#define CHNL_IMG_CTRL_FORMAT_ABGR8888				0x01
+#define CHNL_IMG_CTRL_FORMAT_ARGB8888				0x02
+#define CHNL_IMG_CTRL_FORMAT_RGBX888				0x03
+#define CHNL_IMG_CTRL_FORMAT_XBGR888				0x04
+#define CHNL_IMG_CTRL_FORMAT_XRGB888				0x05
+#define CHNL_IMG_CTRL_FORMAT_RGB888P				0x06
+#define CHNL_IMG_CTRL_FORMAT_BGR888P				0x07
+#define CHNL_IMG_CTRL_FORMAT_A2BGR10				0x08
+#define CHNL_IMG_CTRL_FORMAT_A2RGB10				0x09
+#define CHNL_IMG_CTRL_FORMAT_RGB565				0x0a
+#define CHNL_IMG_CTRL_FORMAT_RAW8				0x0b
+#define CHNL_IMG_CTRL_FORMAT_RAW10				0x0c
+#define CHNL_IMG_CTRL_FORMAT_RAW10P				0x0d
+#define CHNL_IMG_CTRL_FORMAT_RAW12				0x0e
+#define CHNL_IMG_CTRL_FORMAT_RAW16				0x0f
+#define CHNL_IMG_CTRL_FORMAT_YUV444_1P8P			0x10
+#define CHNL_IMG_CTRL_FORMAT_YUV444_2P8P			0x11
+#define CHNL_IMG_CTRL_FORMAT_YUV444_3P8P			0x12
+#define CHNL_IMG_CTRL_FORMAT_YUV444_1P8				0x13
+#define CHNL_IMG_CTRL_FORMAT_YUV444_1P10			0x14
+#define CHNL_IMG_CTRL_FORMAT_YUV444_2P10			0x15
+#define CHNL_IMG_CTRL_FORMAT_YUV444_3P10			0x16
+#define CHNL_IMG_CTRL_FORMAT_YUV444_1P10P			0x18
+#define CHNL_IMG_CTRL_FORMAT_YUV444_2P10P			0x19
+#define CHNL_IMG_CTRL_FORMAT_YUV444_3P10P			0x1a
+#define CHNL_IMG_CTRL_FORMAT_YUV444_1P12			0x1c
+#define CHNL_IMG_CTRL_FORMAT_YUV444_2P12			0x1d
+#define CHNL_IMG_CTRL_FORMAT_YUV444_3P12			0x1e
+#define CHNL_IMG_CTRL_FORMAT_YUV422_1P8P			0x20
+#define CHNL_IMG_CTRL_FORMAT_YUV422_2P8P			0x21
+#define CHNL_IMG_CTRL_FORMAT_YUV422_3P8P			0x22
+#define CHNL_IMG_CTRL_FORMAT_YUV422_1P10			0x24
+#define CHNL_IMG_CTRL_FORMAT_YUV422_2P10			0x25
+#define CHNL_IMG_CTRL_FORMAT_YUV422_3P10			0x26
+#define CHNL_IMG_CTRL_FORMAT_YUV422_1P10P			0x28
+#define CHNL_IMG_CTRL_FORMAT_YUV422_2P10P			0x29
+#define CHNL_IMG_CTRL_FORMAT_YUV422_3P10P			0x2a
+#define CHNL_IMG_CTRL_FORMAT_YUV422_1P12			0x2c
+#define CHNL_IMG_CTRL_FORMAT_YUV422_2P12			0x2d
+#define CHNL_IMG_CTRL_FORMAT_YUV422_3P12			0x2e
+#define CHNL_IMG_CTRL_FORMAT_YUV420_2P8P			0x31
+#define CHNL_IMG_CTRL_FORMAT_YUV420_3P8P			0x32
+#define CHNL_IMG_CTRL_FORMAT_YUV420_2P10			0x35
+#define CHNL_IMG_CTRL_FORMAT_YUV420_3P10			0x36
+#define CHNL_IMG_CTRL_FORMAT_YUV420_2P10P			0x39
+#define CHNL_IMG_CTRL_FORMAT_YUV420_3P10P			0x3a
+#define CHNL_IMG_CTRL_FORMAT_YUV420_2P12			0x3d
+#define CHNL_IMG_CTRL_FORMAT_YUV420_3P12			0x3e
+#define CHNL_IMG_CTRL_GBL_ALPHA_VAL(n)				((n) << 16)
+#define CHNL_IMG_CTRL_GBL_ALPHA_VAL_MASK			GENMASK(23, 16)
+#define CHNL_IMG_CTRL_GBL_ALPHA_EN				BIT(15)
+#define CHNL_IMG_CTRL_DEINT(n)					((n) << 12)
+#define CHNL_IMG_CTRL_DEINT_MASK				GENMASK(14, 12)
+#define CHNL_IMG_CTRL_DEINT_WEAVE_ODD_EVEN			2
+#define CHNL_IMG_CTRL_DEINT_WEAVE_EVEN_ODD			3
+#define CHNL_IMG_CTRL_DEINT_BLEND_ODD_EVEN			4
+#define CHNL_IMG_CTRL_DEINT_BLEND_EVEN_ODD			5
+#define CHNL_IMG_CTRL_DEINT_LDOUBLE_ODD_EVEN			6
+#define CHNL_IMG_CTRL_DEINT_LDOUBLE_EVEN_ODD			7
+#define CHNL_IMG_CTRL_DEC_X(n)					((n) << 10)
+#define CHNL_IMG_CTRL_DEC_X_MASK				GENMASK(11, 10)
+#define CHNL_IMG_CTRL_DEC_Y(n)					((n) << 8)
+#define CHNL_IMG_CTRL_DEC_Y_MASK				GENMASK(9, 8)
+#define CHNL_IMG_CTRL_CROP_EN					BIT(7)
+#define CHNL_IMG_CTRL_VFLIP_EN					BIT(6)
+#define CHNL_IMG_CTRL_HFLIP_EN					BIT(5)
+#define CHNL_IMG_CTRL_YCBCR_MODE				BIT(3)
+#define CHNL_IMG_CTRL_CSC_MODE(n)				((n) << 1)
+#define CHNL_IMG_CTRL_CSC_MODE_MASK				GENMASK(2, 1)
+#define CHNL_IMG_CTRL_CSC_MODE_YUV2RGB				0
+#define CHNL_IMG_CTRL_CSC_MODE_YCBCR2RGB			1
+#define CHNL_IMG_CTRL_CSC_MODE_RGB2YUV				2
+#define CHNL_IMG_CTRL_CSC_MODE_RGB2YCBCR			3
+#define CHNL_IMG_CTRL_CSC_BYPASS				BIT(0)
+
+/* Channel Output Buffer Control Register */
+#define CHNL_OUT_BUF_CTRL					0x0008
+#define CHNL_OUT_BUF_CTRL_LOAD_BUF2_ADDR			BIT(15)
+#define CHNL_OUT_BUF_CTRL_LOAD_BUF1_ADDR			BIT(14)
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_V(n)		((n) << 6)
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_V_MASK		GENMASK(7, 6)
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_V_NO_PANIC		0
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_V_PANIC_25		1
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_V_PANIC_50		2
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_V_PANIC_75		3
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_U(n)		((n) << 3)
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_U_MASK		GENMASK(4, 3)
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_U_NO_PANIC		0
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_U_PANIC_25		1
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_U_PANIC_50		2
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_U_PANIC_75		3
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_Y(n)		((n) << 0)
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_Y_MASK		GENMASK(1, 0)
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_Y_NO_PANIC		0
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_Y_PANIC_25		1
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_Y_PANIC_50		2
+#define CHNL_OUT_BUF_CTRL_OFLW_PANIC_SET_THD_Y_PANIC_75		3
+
+/* Channel Image Configuration */
+#define CHNL_IMG_CFG						0x000c
+#define CHNL_IMG_CFG_HEIGHT(n)					((n) << 16)
+#define CHNL_IMG_CFG_HEIGHT_MASK				GENMASK(28, 16)
+#define CHNL_IMG_CFG_WIDTH(n)					((n) << 0)
+#define CHNL_IMG_CFG_WIDTH_MASK					GENMASK(12, 0)
+
+/* Channel Interrupt Enable Register */
+#define CHNL_IER						0x0010
+#define CHNL_IER_MEM_RD_DONE_EN					BIT(31)
+#define CHNL_IER_LINE_RCVD_EN					BIT(30)
+#define CHNL_IER_FRM_RCVD_EN					BIT(29)
+#define CHNL_IER_AXI_WR_ERR_V_EN				BIT(28)
+#define CHNL_IER_AXI_WR_ERR_U_EN				BIT(27)
+#define CHNL_IER_AXI_WR_ERR_Y_EN				BIT(26)
+#define CHNL_IER_AXI_RD_ERR_EN					BIT(25)
+
+/* Channel Status Register */
+#define CHNL_STS						0x0014
+#define CHNL_STS_MEM_RD_DONE					BIT(31)
+#define CHNL_STS_LINE_STRD					BIT(30)
+#define CHNL_STS_FRM_STRD					BIT(29)
+#define CHNL_STS_AXI_WR_ERR_V					BIT(28)
+#define CHNL_STS_AXI_WR_ERR_U					BIT(27)
+#define CHNL_STS_AXI_WR_ERR_Y					BIT(26)
+#define CHNL_STS_AXI_RD_ERR					BIT(25)
+#define CHNL_STS_OFLW_PANIC_V_BUF				BIT(24)
+#define CHNL_STS_EXCS_OFLW_V_BUF				BIT(23)
+#define CHNL_STS_OFLW_V_BUF					BIT(22)
+#define CHNL_STS_OFLW_PANIC_U_BUF				BIT(21)
+#define CHNL_STS_EXCS_OFLW_U_BUF				BIT(20)
+#define CHNL_STS_OFLW_U_BUF					BIT(19)
+#define CHNL_STS_OFLW_PANIC_Y_BUF				BIT(18)
+#define CHNL_STS_EXCS_OFLW_Y_BUF				BIT(17)
+#define CHNL_STS_OFLW_Y_BUF					BIT(16)
+#define CHNL_STS_EARLY_VSYNC_ERR				BIT(15)
+#define CHNL_STS_LATE_VSYNC_ERR					BIT(14)
+#define CHNL_STS_MEM_RD_OFLOW					BIT(10)
+#define CHNL_STS_BUF2_ACTIVE					BIT(9)
+#define CHNL_STS_BUF1_ACTIVE					BIT(8)
+#define CHNL_STS_OFLW_BYTES(n)					((n) << 0)
+#define CHNL_STS_OFLW_BYTES_MASK				GENMASK(7, 0)
+
+/* Channel Scale Factor Register */
+#define CHNL_SCALE_FACTOR					0x0018
+#define CHNL_SCALE_FACTOR_Y_SCALE(n)				((n) << 16)
+#define CHNL_SCALE_FACTOR_Y_SCALE_MASK				GENMASK(29, 16)
+#define CHNL_SCALE_FACTOR_X_SCALE(n)				((n) << 0)
+#define CHNL_SCALE_FACTOR_X_SCALE_MASK				GENMASK(13, 0)
+
+/* Channel Scale Offset Register */
+#define CHNL_SCALE_OFFSET					0x001c
+#define CHNL_SCALE_OFFSET_Y_SCALE(n)				((n) << 16)
+#define CHNL_SCALE_OFFSET_Y_SCALE_MASK				GENMASK(27, 16)
+#define CHNL_SCALE_OFFSET_X_SCALE(n)				((n) << 0)
+#define CHNL_SCALE_OFFSET_X_SCALE_MASK				GENMASK(11, 0)
+
+/* Channel Crop Upper Left Corner Coordinate Register */
+#define CHNL_CROP_ULC						0x0020
+#define CHNL_CROP_ULC_X(n)					((n) << 16)
+#define CHNL_CROP_ULC_X_MASK					GENMASK(27, 16)
+#define CHNL_CROP_ULC_Y(n)					((n) << 0)
+#define CHNL_CROP_ULC_Y_MASK					GENMASK(11, 0)
+
+/* Channel Crop Lower Right Corner Coordinate Register */
+#define CHNL_CROP_LRC						0x0024
+#define CHNL_CROP_LRC_X(n)					((n) << 16)
+#define CHNL_CROP_LRC_X_MASK					GENMASK(27, 16)
+#define CHNL_CROP_LRC_Y(n)					((n) << 0)
+#define CHNL_CROP_LRC_Y_MASK					GENMASK(11, 0)
+
+/* Channel Color Space Conversion Coefficient Register 0 */
+#define CHNL_CSC_COEFF0						0x0028
+#define CHNL_CSC_COEFF0_A2(n)					((n) << 16)
+#define CHNL_CSC_COEFF0_A2_MASK					GENMASK(26, 16)
+#define CHNL_CSC_COEFF0_A1(n)					((n) << 0)
+#define CHNL_CSC_COEFF0_A1_MASK					GENMASK(10, 0)
+
+/* Channel Color Space Conversion Coefficient Register 1 */
+#define CHNL_CSC_COEFF1						0x002c
+#define CHNL_CSC_COEFF1_B1(n)					((n) << 16)
+#define CHNL_CSC_COEFF1_B1_MASK					GENMASK(26, 16)
+#define CHNL_CSC_COEFF1_A3(n)					((n) << 0)
+#define CHNL_CSC_COEFF1_A3_MASK					GENMASK(10, 0)
+
+/* Channel Color Space Conversion Coefficient Register 2 */
+#define CHNL_CSC_COEFF2						0x0030
+#define CHNL_CSC_COEFF2_B3(n)					((n) << 16)
+#define CHNL_CSC_COEFF2_B3_MASK					GENMASK(26, 16)
+#define CHNL_CSC_COEFF2_B2(n)					((n) << 0)
+#define CHNL_CSC_COEFF2_B2_MASK					GENMASK(10, 0)
+
+/* Channel Color Space Conversion Coefficient Register 3 */
+#define CHNL_CSC_COEFF3						0x0034
+#define CHNL_CSC_COEFF3_C2(n)					((n) << 16)
+#define CHNL_CSC_COEFF3_C2_MASK					GENMASK(26, 16)
+#define CHNL_CSC_COEFF3_C1(n)					((n) << 0)
+#define CHNL_CSC_COEFF3_C1_MASK					GENMASK(10, 0)
+
+/* Channel Color Space Conversion Coefficient Register 4 */
+#define CHNL_CSC_COEFF4						0x0038
+#define CHNL_CSC_COEFF4_D1(n)					((n) << 16)
+#define CHNL_CSC_COEFF4_D1_MASK					GENMASK(24, 16)
+#define CHNL_CSC_COEFF4_C3(n)					((n) << 0)
+#define CHNL_CSC_COEFF4_C3_MASK					GENMASK(10, 0)
+
+/* Channel Color Space Conversion Coefficient Register 5 */
+#define CHNL_CSC_COEFF5						0x003c
+#define CHNL_CSC_COEFF5_D3(n)					((n) << 16)
+#define CHNL_CSC_COEFF5_D3_MASK					GENMASK(24, 16)
+#define CHNL_CSC_COEFF5_D2(n)					((n) << 0)
+#define CHNL_CSC_COEFF5_D2_MASK					GENMASK(8, 0)
+
+/* Channel Alpha Value Register for ROI 0 */
+#define CHNL_ROI_0_ALPHA					0x0040
+#define CHNL_ROI_0_ALPHA_VAL(n)					((n) << 24)
+#define CHNL_ROI_0_ALPHA_MASK					GENMASK(31, 24)
+#define CHNL_ROI_0_ALPHA_EN					BIT(16)
+
+/* Channel Upper Left Coordinate Register for ROI 0 */
+#define CHNL_ROI_0_ULC						0x0044
+#define CHNL_ROI_0_ULC_X(n)					((n) << 16)
+#define CHNL_ROI_0_ULC_X_MASK					GENMASK(27, 16)
+#define CHNL_ROI_0_ULC_Y(n)					((n) << 0)
+#define CHNL_ROI_0_ULC_Y_MASK					GENMASK(11, 0)
+
+/* Channel Lower Right Coordinate Register for ROI 0 */
+#define CHNL_ROI_0_LRC						0x0048
+#define CHNL_ROI_0_LRC_X(n)					((n) << 16)
+#define CHNL_ROI_0_LRC_X_MASK					GENMASK(27, 16)
+#define CHNL_ROI_0_LRC_Y(n)					((n) << 0)
+#define CHNL_ROI_0_LRC_Y_MASK					GENMASK(11, 0)
+
+/* Channel Alpha Value Register for ROI 1 */
+#define CHNL_ROI_1_ALPHA					0x004c
+#define CHNL_ROI_1_ALPHA_VAL(n)					((n) << 24)
+#define CHNL_ROI_1_ALPHA_MASK					GENMASK(31, 24)
+#define CHNL_ROI_1_ALPHA_EN					BIT(16)
+
+/* Channel Upper Left Coordinate Register for ROI 1 */
+#define CHNL_ROI_1_ULC						0x0050
+#define CHNL_ROI_1_ULC_X(n)					((n) << 16)
+#define CHNL_ROI_1_ULC_X_MASK					GENMASK(27, 16)
+#define CHNL_ROI_1_ULC_Y(n)					((n) << 0)
+#define CHNL_ROI_1_ULC_Y_MASK					GENMASK(11, 0)
+
+/* Channel Lower Right Coordinate Register for ROI 1 */
+#define CHNL_ROI_1_LRC						0x0054
+#define CHNL_ROI_1_LRC_X(n)					((n) << 16)
+#define CHNL_ROI_1_LRC_X_MASK					GENMASK(27, 16)
+#define CHNL_ROI_1_LRC_Y(n)					((n) << 0)
+#define CHNL_ROI_1_LRC_Y_MASK					GENMASK(11, 0)
+
+/* Channel Alpha Value Register for ROI 2 */
+#define CHNL_ROI_2_ALPHA					0x0058
+#define CHNL_ROI_2_ALPHA_VAL(n)					((n) << 24)
+#define CHNL_ROI_2_ALPHA_MASK					GENMASK(31, 24)
+#define CHNL_ROI_2_ALPHA_EN					BIT(16)
+
+/* Channel Upper Left Coordinate Register for ROI 2 */
+#define CHNL_ROI_2_ULC						0x005c
+#define CHNL_ROI_2_ULC_X(n)					((n) << 16)
+#define CHNL_ROI_2_ULC_X_MASK					GENMASK(27, 16)
+#define CHNL_ROI_2_ULC_Y(n)					((n) << 0)
+#define CHNL_ROI_2_ULC_Y_MASK					GENMASK(11, 0)
+
+/* Channel Lower Right Coordinate Register for ROI 2 */
+#define CHNL_ROI_2_LRC						0x0060
+#define CHNL_ROI_2_LRC_X(n)					((n) << 16)
+#define CHNL_ROI_2_LRC_X_MASK					GENMASK(27, 16)
+#define CHNL_ROI_2_LRC_Y(n)					((n) << 0)
+#define CHNL_ROI_2_LRC_Y_MASK					GENMASK(11, 0)
+
+/* Channel Alpha Value Register for ROI 3 */
+#define CHNL_ROI_3_ALPHA					0x0064
+#define CHNL_ROI_3_ALPHA_VAL(n)					((n) << 24)
+#define CHNL_ROI_3_ALPHA_MASK					GENMASK(31, 24)
+#define CHNL_ROI_3_ALPHA_EN					BIT(16)
+
+/* Channel Upper Left Coordinate Register for ROI 3 */
+#define CHNL_ROI_3_ULC						0x0068
+#define CHNL_ROI_3_ULC_X(n)					((n) << 16)
+#define CHNL_ROI_3_ULC_X_MASK					GENMASK(27, 16)
+#define CHNL_ROI_3_ULC_Y(n)					((n) << 0)
+#define CHNL_ROI_3_ULC_Y_MASK					GENMASK(11, 0)
+
+/* Channel Lower Right Coordinate Register for ROI 3 */
+#define CHNL_ROI_3_LRC						0x006c
+#define CHNL_ROI_3_LRC_X(n)					((n) << 16)
+#define CHNL_ROI_3_LRC_X_MASK					GENMASK(27, 16)
+#define CHNL_ROI_3_LRC_Y(n)					((n) << 0)
+#define CHNL_ROI_3_LRC_Y_MASK					GENMASK(11, 0)
+/* Channel RGB or Luma (Y) Output Buffer 1 Address */
+#define CHNL_OUT_BUF1_ADDR_Y					0x0070
+
+/* Channel Chroma (U/Cb/UV/CbCr) Output Buffer 1 Address */
+#define CHNL_OUT_BUF1_ADDR_U					0x0074
+
+/* Channel Chroma (V/Cr) Output Buffer 1 Address */
+#define CHNL_OUT_BUF1_ADDR_V					0x0078
+
+/* Channel Output Buffer Pitch */
+#define CHNL_OUT_BUF_PITCH					0x007c
+#define CHNL_OUT_BUF_PITCH_LINE_PITCH(n)			((n) << 0)
+#define CHNL_OUT_BUF_PITCH_LINE_PITCH_MASK			GENMASK(15, 0)
+
+/* Channel Input Buffer Address */
+#define CHNL_IN_BUF_ADDR					0x0080
+
+/* Channel Input Buffer Pitch */
+#define CHNL_IN_BUF_PITCH					0x0084
+#define CHNL_IN_BUF_PITCH_FRM_PITCH(n)				((n) << 16)
+#define CHNL_IN_BUF_PITCH_FRM_PITCH_MASK			GENMASK(31, 16)
+#define CHNL_IN_BUF_PITCH_LINE_PITCH(n)				((n) << 0)
+#define CHNL_IN_BUF_PITCH_LINE_PITCH_MASK			GENMASK(15, 0)
+
+/* Channel Memory Read Control */
+#define CHNL_MEM_RD_CTRL					0x0088
+#define CHNL_MEM_RD_CTRL_IMG_TYPE(n)				((n) << 28)
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_MASK				GENMASK(31, 28)
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_BGR8P				0x00
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_RGB8P				0x01
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_XRGB8				0x02
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_RGBX8				0x03
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_XBGR8				0x04
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_RGB565			0x05
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_A2BGR10			0x06
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_A2RGB10			0x07
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_YUV444_1P8P			0x08
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_YUV444_1P10			0x09
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_YUV444_1P10P			0x0a
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_YUV444_1P12			0x0b
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_YUV444_1P8			0x0c
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_YUV422_1P8P			0x0d
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_YUV422_1P10			0x0e
+#define CHNL_MEM_RD_CTRL_IMG_TYPE_YUV422_1P12			0x0f
+#define CHNL_MEM_RD_CTRL_READ_MEM				BIT(0)
+
+/* Channel RGB or Luma (Y) Output Buffer 2 Address */
+#define CHNL_OUT_BUF2_ADDR_Y					0x008c
+
+/* Channel Chroma (U/Cb/UV/CbCr) Output Buffer 2 Address  */
+#define CHNL_OUT_BUF2_ADDR_U					0x0090
+
+/* Channel Chroma (V/Cr) Output Buffer 2 Address   */
+#define CHNL_OUT_BUF2_ADDR_V					0x0094
+
+/* Channel scale image config */
+#define CHNL_SCL_IMG_CFG					0x0098
+#define CHNL_SCL_IMG_CFG_HEIGHT(n)				((n) << 16)
+#define CHNL_SCL_IMG_CFG_HEIGHT_MASK				GENMASK(28, 16)
+#define CHNL_SCL_IMG_CFG_WIDTH(n)				((n) << 0)
+#define CHNL_SCL_IMG_CFG_WIDTH_MASK				GENMASK(12, 0)
+
+/* Channel Flow Control Register */
+#define CHNL_FLOW_CTRL						0x009c
+#define CHNL_FLOW_CTRL_FC_DENOM_MASK				GENMASK(7, 0)
+#define CHNL_FLOW_CTRL_FC_DENOM(n)				((n) << 0)
+#define CHNL_FLOW_CTRL_FC_NUMER_MASK				GENMASK(23, 16)
+#define CHNL_FLOW_CTRL_FC_NUMER(n)				((n) << 0)
+
+/* Channel Output Y-Buffer 1 Extended Address Bits */
+#define CHNL_Y_BUF1_XTND_ADDR					0x00a0
+
+/* Channel Output U-Buffer 1 Extended Address Bits */
+#define CHNL_U_BUF1_XTND_ADDR					0x00a4
+
+/* Channel Output V-Buffer 1 Extended Address Bits */
+#define CHNL_V_BUF1_XTND_ADDR					0x00a8
+
+/* Channel Output Y-Buffer 2 Extended Address Bits */
+#define CHNL_Y_BUF2_XTND_ADDR					0x00ac
+
+/* Channel Output U-Buffer 2 Extended Address Bits */
+#define CHNL_U_BUF2_XTND_ADDR					0x00b0
+
+/* Channel Output V-Buffer 2 Extended Address Bits */
+#define CHNL_V_BUF2_XTND_ADDR					0x00b4
+
+/* Channel Input Buffer Extended Address Bits */
+#define CHNL_IN_BUF_XTND_ADDR					0x00b8
+
+#endif /* __IMX8_ISI_REGS_H__ */
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c
new file mode 100644
index 000000000000..10840c9a0912
--- /dev/null
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c
@@ -0,0 +1,1512 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * V4L2 Capture ISI subdev driver for i.MX8QXP/QM platform
+ *
+ * ISI is a Image Sensor Interface of i.MX8QXP/QM platform, which
+ * used to process image from camera sensor to memory or DC
+ *
+ * Copyright (c) 2019 NXP Semiconductor
+ */
+
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/media-bus-format.h>
+#include <linux/minmax.h>
+#include <linux/pm_runtime.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/videodev2.h>
+
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-subdev.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/videobuf2-v4l2.h>
+
+#include "imx8-isi-core.h"
+#include "imx8-isi-regs.h"
+
+/* Keep the first entry matching MXC_ISI_DEF_PIXEL_FORMAT */
+static const struct mxc_isi_format_info mxc_isi_formats[] = {
+	/* YUV formats */
+	{
+		.mbus_code	= MEDIA_BUS_FMT_YUV8_1X24,
+		.fourcc		= V4L2_PIX_FMT_YUYV,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_OUT
+				| MXC_ISI_VIDEO_M2M_CAP,
+		.isi_in_format	= CHNL_MEM_RD_CTRL_IMG_TYPE_YUV422_1P8P,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_YUV422_1P8P,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_YUV,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_YUV8_1X24,
+		.fourcc		= V4L2_PIX_FMT_YUVA32,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_YUV444_1P8,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 32 },
+		.encoding	= MXC_ISI_ENC_YUV,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_YUV8_1X24,
+		.fourcc		= V4L2_PIX_FMT_NV12,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_YUV420_2P8P,
+		.color_planes	= 2,
+		.mem_planes	= 1,
+		.depth		= { 8, 16 },
+		.hsub		= 2,
+		.vsub		= 2,
+		.encoding	= MXC_ISI_ENC_YUV,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_YUV8_1X24,
+		.fourcc		= V4L2_PIX_FMT_NV12M,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_YUV420_2P8P,
+		.mem_planes	= 2,
+		.color_planes	= 2,
+		.depth		= { 8, 16 },
+		.hsub		= 2,
+		.vsub		= 2,
+		.encoding	= MXC_ISI_ENC_YUV,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_YUV8_1X24,
+		.fourcc		= V4L2_PIX_FMT_NV16,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_YUV422_2P8P,
+		.color_planes	= 2,
+		.mem_planes	= 1,
+		.depth		= { 8, 16 },
+		.hsub		= 2,
+		.vsub		= 1,
+		.encoding	= MXC_ISI_ENC_YUV,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_YUV8_1X24,
+		.fourcc		= V4L2_PIX_FMT_NV16M,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_YUV422_2P8P,
+		.mem_planes	= 2,
+		.color_planes	= 2,
+		.depth		= { 8, 16 },
+		.hsub		= 2,
+		.vsub		= 1,
+		.encoding	= MXC_ISI_ENC_YUV,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_YUV8_1X24,
+		.fourcc		= V4L2_PIX_FMT_YUV444M,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_YUV444_3P8P,
+		.mem_planes	= 3,
+		.color_planes	= 3,
+		.depth		= { 8, 8, 8 },
+		.hsub		= 1,
+		.vsub		= 1,
+		.encoding	= MXC_ISI_ENC_YUV,
+	},
+	/* RGB formats */
+	{
+		.mbus_code	= MEDIA_BUS_FMT_RGB888_1X24,
+		.fourcc		= V4L2_PIX_FMT_RGB565,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_OUT
+				| MXC_ISI_VIDEO_M2M_CAP,
+		.isi_in_format	= CHNL_MEM_RD_CTRL_IMG_TYPE_RGB565,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RGB565,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RGB,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_RGB888_1X24,
+		.fourcc		= V4L2_PIX_FMT_RGB24,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_OUT
+				| MXC_ISI_VIDEO_M2M_CAP,
+		.isi_in_format	= CHNL_MEM_RD_CTRL_IMG_TYPE_BGR8P,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_BGR888P,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 24 },
+		.encoding	= MXC_ISI_ENC_RGB,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_RGB888_1X24,
+		.fourcc		= V4L2_PIX_FMT_BGR24,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_OUT
+				| MXC_ISI_VIDEO_M2M_CAP,
+		.isi_in_format	= CHNL_MEM_RD_CTRL_IMG_TYPE_RGB8P,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RGB888P,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 24 },
+		.encoding	= MXC_ISI_ENC_RGB,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_RGB888_1X24,
+		.fourcc		= V4L2_PIX_FMT_XBGR32,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_OUT
+				| MXC_ISI_VIDEO_M2M_CAP,
+		.isi_in_format	= CHNL_MEM_RD_CTRL_IMG_TYPE_XBGR8,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_XRGB888,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 32 },
+		.encoding	= MXC_ISI_ENC_RGB,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_RGB888_1X24,
+		.fourcc		= V4L2_PIX_FMT_ABGR32,
+		.type		= MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_ARGB8888,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 32 },
+		.encoding	= MXC_ISI_ENC_RGB,
+	},
+	/*
+	 * RAW formats
+	 *
+	 * The ISI shifts the 10-bit and 12-bit formats left by 6 and 4 bits
+	 * when using CHNL_IMG_CTRL_FORMAT_RAW10 or MXC_ISI_OUT_FMT_RAW12
+	 * respectively, to align the bits to the left and pad with zeros in
+	 * the LSBs. The corresponding V4L2 formats are however right-aligned,
+	 * we have to use CHNL_IMG_CTRL_FORMAT_RAW16 to avoid the left shift.
+	 */
+	{
+		.mbus_code	= MEDIA_BUS_FMT_Y8_1X8,
+		.fourcc		= V4L2_PIX_FMT_GREY,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW8,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 8 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_Y10_1X10,
+		.fourcc		= V4L2_PIX_FMT_Y10,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_Y12_1X12,
+		.fourcc		= V4L2_PIX_FMT_Y12,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_Y14_1X14,
+		.fourcc		= V4L2_PIX_FMT_Y14,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR8_1X8,
+		.fourcc		= V4L2_PIX_FMT_SBGGR8,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW8,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 8 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG8_1X8,
+		.fourcc		= V4L2_PIX_FMT_SGBRG8,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW8,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 8 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG8_1X8,
+		.fourcc		= V4L2_PIX_FMT_SGRBG8,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW8,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 8 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB8_1X8,
+		.fourcc		= V4L2_PIX_FMT_SRGGB8,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW8,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 8 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR10_1X10,
+		.fourcc		= V4L2_PIX_FMT_SBGGR10,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG10_1X10,
+		.fourcc		= V4L2_PIX_FMT_SGBRG10,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG10_1X10,
+		.fourcc		= V4L2_PIX_FMT_SGRBG10,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB10_1X10,
+		.fourcc		= V4L2_PIX_FMT_SRGGB10,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR12_1X12,
+		.fourcc		= V4L2_PIX_FMT_SBGGR12,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG12_1X12,
+		.fourcc		= V4L2_PIX_FMT_SGBRG12,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG12_1X12,
+		.fourcc		= V4L2_PIX_FMT_SGRBG12,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB12_1X12,
+		.fourcc		= V4L2_PIX_FMT_SRGGB12,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR14_1X14,
+		.fourcc		= V4L2_PIX_FMT_SBGGR14,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG14_1X14,
+		.fourcc		= V4L2_PIX_FMT_SGBRG14,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG14_1X14,
+		.fourcc		= V4L2_PIX_FMT_SGRBG14,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB14_1X14,
+		.fourcc		= V4L2_PIX_FMT_SRGGB14,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW16,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 16 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	},
+	/* JPEG */
+	{
+		.mbus_code	= MEDIA_BUS_FMT_JPEG_1X8,
+		.fourcc		= V4L2_PIX_FMT_MJPEG,
+		.type		= MXC_ISI_VIDEO_CAP,
+		.isi_out_format	= CHNL_IMG_CTRL_FORMAT_RAW8,
+		.mem_planes	= 1,
+		.color_planes	= 1,
+		.depth		= { 8 },
+		.encoding	= MXC_ISI_ENC_RAW,
+	}
+};
+
+const struct mxc_isi_format_info *
+mxc_isi_format_by_fourcc(u32 fourcc, enum mxc_isi_video_type type)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(mxc_isi_formats); i++) {
+		const struct mxc_isi_format_info *fmt = &mxc_isi_formats[i];
+
+		if (fmt->fourcc == fourcc && fmt->type & type)
+			return fmt;
+	}
+
+	return NULL;
+}
+
+const struct mxc_isi_format_info *
+mxc_isi_format_enum(unsigned int index, enum mxc_isi_video_type type)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(mxc_isi_formats); i++) {
+		const struct mxc_isi_format_info *fmt = &mxc_isi_formats[i];
+
+		if (!(fmt->type & type))
+			continue;
+
+		if (!index)
+			return fmt;
+
+		index--;
+	}
+
+	return NULL;
+}
+
+const struct mxc_isi_format_info *
+mxc_isi_format_try(struct mxc_isi_pipe *pipe, struct v4l2_pix_format_mplane *pix,
+		   enum mxc_isi_video_type type)
+{
+	const struct mxc_isi_format_info *fmt;
+	unsigned int max_width;
+	unsigned int i;
+
+	max_width = pipe->id == pipe->isi->pdata->num_channels - 1
+		  ? MXC_ISI_MAX_WIDTH_UNCHAINED
+		  : MXC_ISI_MAX_WIDTH_CHAINED;
+
+	fmt = mxc_isi_format_by_fourcc(pix->pixelformat, type);
+	if (!fmt)
+		fmt = &mxc_isi_formats[0];
+
+	pix->width = clamp(pix->width, MXC_ISI_MIN_WIDTH, max_width);
+	pix->height = clamp(pix->height, MXC_ISI_MIN_HEIGHT, MXC_ISI_MAX_HEIGHT);
+	pix->pixelformat = fmt->fourcc;
+	pix->field = V4L2_FIELD_NONE;
+
+	if (pix->colorspace == V4L2_COLORSPACE_DEFAULT) {
+		pix->colorspace = MXC_ISI_DEF_COLOR_SPACE;
+		pix->ycbcr_enc = MXC_ISI_DEF_YCBCR_ENC;
+		pix->quantization = MXC_ISI_DEF_QUANTIZATION;
+		pix->xfer_func = MXC_ISI_DEF_XFER_FUNC;
+	}
+
+	if (pix->ycbcr_enc == V4L2_YCBCR_ENC_DEFAULT)
+		pix->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(pix->colorspace);
+	if (pix->quantization == V4L2_QUANTIZATION_DEFAULT) {
+		bool is_rgb = fmt->encoding == MXC_ISI_ENC_RGB;
+
+		pix->quantization =
+			V4L2_MAP_QUANTIZATION_DEFAULT(is_rgb, pix->colorspace,
+						      pix->ycbcr_enc);
+	}
+	if (pix->xfer_func == V4L2_XFER_FUNC_DEFAULT)
+		pix->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(pix->colorspace);
+
+	pix->num_planes = fmt->mem_planes;
+
+	for (i = 0; i < fmt->color_planes; ++i) {
+		struct v4l2_plane_pix_format *plane = &pix->plane_fmt[i];
+		unsigned int bpl;
+
+		/* The pitch must be identical for all planes. */
+		if (i == 0)
+			bpl = clamp(plane->bytesperline,
+				    pix->width * fmt->depth[0] / 8,
+				    65535U);
+		else
+			bpl = pix->plane_fmt[0].bytesperline;
+
+		plane->bytesperline = bpl;
+
+		plane->sizeimage = plane->bytesperline * pix->height;
+		if (i >= 1)
+			plane->sizeimage /= fmt->vsub;
+	}
+
+	/*
+	 * For single-planar pixel formats with multiple color planes,
+	 * concatenate the size of all planes and clear all planes but the
+	 * first one.
+	 */
+	if (fmt->color_planes != fmt->mem_planes) {
+		for (i = 1; i < fmt->color_planes; ++i) {
+			struct v4l2_plane_pix_format *plane = &pix->plane_fmt[i];
+
+			pix->plane_fmt[0].sizeimage += plane->sizeimage;
+			plane->bytesperline = 0;
+			plane->sizeimage = 0;
+		}
+	}
+
+	return fmt;
+}
+
+/* -----------------------------------------------------------------------------
+ * videobuf2 queue operations
+ */
+
+static void mxc_isi_video_frame_write_done(struct mxc_isi_pipe *pipe,
+					   u32 status)
+{
+	struct mxc_isi_video *video = &pipe->video;
+	struct device *dev = pipe->isi->dev;
+	struct mxc_isi_buffer *next_buf;
+	struct mxc_isi_buffer *buf;
+	enum mxc_isi_buf_id buf_id;
+
+	spin_lock(&video->buf_lock);
+
+	/*
+	 * The ISI hardware handles buffers using a ping-pong mechanism with
+	 * two sets of destination addresses (with shadow registers to allow
+	 * programming addresses for all planes atomically) named BUF1 and
+	 * BUF2. Addresses can be loaded and copied to shadow registers at any
+	 * at any time.
+	 *
+	 * The hardware keeps track of which buffer is being written to and
+	 * automatically switches to the other buffer at frame end, copying the
+	 * corresponding address to another set of shadow registers that track
+	 * the address being written to. The active buffer tracking bits are
+	 * accessible through the CHNL_STS register.
+	 *
+	 *  BUF1        BUF2  |   Event   | Action
+	 *                    |           |
+	 *                    |           | Program initial buffers
+	 *                    |           | B0 in BUF1, B1 in BUF2
+	 *                    | Start ISI |
+	 * +----+             |           |
+	 * | B0 |             |           |
+	 * +----+             |           |
+	 *             +----+ | FRM IRQ 0 | B0 complete, BUF2 now active
+	 *             | B1 | |           | Program B2 in BUF1
+	 *             +----+ |           |
+	 * +----+             | FRM IRQ 1 | B1 complete, BUF1 now active
+	 * | B2 |             |           | Program B3 in BUF2
+	 * +----+             |           |
+	 *             +----+ | FRM IRQ 2 | B2 complete, BUF2 now active
+	 *             | B3 | |           | Program B4 in BUF1
+	 *             +----+ |           |
+	 * +----+             | FRM IRQ 3 | B3 complete, BUF1 now active
+	 * | B4 |             |           | Program B5 in BUF2
+	 * +----+             |           |
+	 *        ...         |           |
+	 *
+	 * Races between address programming and buffer switching can be
+	 * detected by checking if a frame end interrupt occurred after
+	 * programming the addresses.
+	 *
+	 * As none of the shadow registers are accessible, races can occur
+	 * between address programming and buffer switching. It is possible to
+	 * detect the race condition by checking if a frame end interrupt
+	 * occurred after programming the addresses, but impossible to
+	 * determine if the race has been won or lost.
+	 *
+	 * In addition to this, we need to use discard buffers if no pending
+	 * buffers are available. To simplify handling of discard buffer, we
+	 * need to allocate three of them, as two can be active concurrently
+	 * and we need to still be able to get hold of a next buffer. The logic
+	 * could be improved to use two buffers only, but as all discard
+	 * buffers share the same memory, an additional buffer is cheap.
+	 */
+
+	/* Check which buffer has just completed. */
+	buf_id = pipe->isi->pdata->buf_active_reverse
+	       ? (status & CHNL_STS_BUF1_ACTIVE ? MXC_ISI_BUF2 : MXC_ISI_BUF1)
+	       : (status & CHNL_STS_BUF1_ACTIVE ? MXC_ISI_BUF1 : MXC_ISI_BUF2);
+
+	buf = list_first_entry_or_null(&video->out_active,
+				       struct mxc_isi_buffer, list);
+
+	/* Safety check, this should really never happen. */
+	if (!buf) {
+		dev_warn(dev, "trying to access empty active list\n");
+		goto done;
+	}
+
+	/*
+	 * If the buffer that has completed doesn't match the buffer on the
+	 * front of the active list, it means we have lost one frame end
+	 * interrupt (or possibly a large odd number of interrupts, although
+	 * quite unlikely).
+	 *
+	 * For instance, if IRQ1 is lost and we handle IRQ2, both B1 and B2
+	 * have been completed, but B3 hasn't been programmed, BUF2 still
+	 * addresses B1 and the ISI is now writing in B1 instead of B3. We
+	 * can't complete B2 as that would result in out-of-order completion.
+	 *
+	 * The only option is to ignore this interrupt and try again. When IRQ3
+	 * will be handled, we will complete B1 and be in sync again.
+	 */
+	if (buf->id != buf_id) {
+		dev_dbg(dev, "buffer ID mismatch (expected %u, got %u), skipping\n",
+			buf->id, buf_id);
+
+		/*
+		 * Increment the frame count by two to account for the missed
+		 * and the ignored interrupts.
+		 */
+		video->frame_count += 2;
+		goto done;
+	}
+
+	/* Pick the next buffer and queue it to the hardware. */
+	next_buf = list_first_entry_or_null(&video->out_pending,
+					    struct mxc_isi_buffer, list);
+	if (!next_buf) {
+		next_buf = list_first_entry_or_null(&video->out_discard,
+						    struct mxc_isi_buffer, list);
+
+		/* Safety check, this should never happen. */
+		if (!next_buf) {
+			dev_warn(dev, "trying to access empty discard list\n");
+			goto done;
+		}
+	}
+
+	mxc_isi_channel_set_outbuf(pipe, next_buf->dma_addrs, buf_id);
+	next_buf->id = buf_id;
+
+	/*
+	 * Check if we have raced with the end of frame interrupt. If so, we
+	 * can't tell if the ISI has recorded the new address, or is still
+	 * using the previous buffer. We must assume the latter as that is the
+	 * worst case.
+	 *
+	 * For instance, if we are handling IRQ1 and now detect the FRM
+	 * interrupt, assume B2 has completed and the ISI has switched to BUF2
+	 * using B1 just before we programmed B3. Unlike in the previous race
+	 * condition, B3 has been programmed and will be written to the next
+	 * time the ISI switches to BUF2. We can however handle this exactly as
+	 * the first race condition, as we'll program B3 (still at the head of
+	 * the pending list) when handling IRQ3.
+	 */
+	status = mxc_isi_channel_irq_status(pipe, false);
+	if (status & CHNL_STS_FRM_STRD) {
+		dev_dbg(dev, "raced with frame end interrupt\n");
+		video->frame_count += 2;
+		goto done;
+	}
+
+	/*
+	 * The next buffer has been queued successfully, move it to the active
+	 * list, and complete the current buffer.
+	 */
+	list_move_tail(&next_buf->list, &video->out_active);
+
+	if (!buf->discard) {
+		list_del_init(&buf->list);
+		buf->v4l2_buf.sequence = video->frame_count;
+		buf->v4l2_buf.vb2_buf.timestamp = ktime_get_ns();
+		vb2_buffer_done(&buf->v4l2_buf.vb2_buf, VB2_BUF_STATE_DONE);
+	} else {
+		list_move_tail(&buf->list, &video->out_discard);
+	}
+
+	video->frame_count++;
+
+done:
+	spin_unlock(&video->buf_lock);
+}
+
+static void mxc_isi_video_free_discard_buffers(struct mxc_isi_video *video)
+{
+	unsigned int i;
+
+	for (i = 0; i < video->pix.num_planes; i++) {
+		struct mxc_isi_dma_buffer *buf = &video->discard_buffer[i];
+
+		if (!buf->addr)
+			continue;
+
+		dma_free_coherent(video->pipe->isi->dev, buf->size, buf->addr,
+				  buf->dma);
+		buf->addr = NULL;
+	}
+}
+
+static int mxc_isi_video_alloc_discard_buffers(struct mxc_isi_video *video)
+{
+	unsigned int i, j;
+
+	/* Allocate memory for each plane. */
+	for (i = 0; i < video->pix.num_planes; i++) {
+		struct mxc_isi_dma_buffer *buf = &video->discard_buffer[i];
+
+		buf->size = PAGE_ALIGN(video->pix.plane_fmt[i].sizeimage);
+		buf->addr = dma_alloc_coherent(video->pipe->isi->dev, buf->size,
+					       &buf->dma, GFP_DMA | GFP_KERNEL);
+		if (!buf->addr) {
+			mxc_isi_video_free_discard_buffers(video);
+			return -ENOMEM;
+		}
+
+		dev_dbg(video->pipe->isi->dev,
+			"discard buffer plane %u: %zu bytes @%pad (CPU address %p)\n",
+			i, buf->size, &buf->dma, buf->addr);
+	}
+
+	/* Fill the DMA addresses in the discard buffers. */
+	for (i = 0; i < ARRAY_SIZE(video->buf_discard); ++i) {
+		struct mxc_isi_buffer *buf = &video->buf_discard[i];
+
+		buf->discard = true;
+
+		for (j = 0; j < video->pix.num_planes; ++j)
+			buf->dma_addrs[j] = video->discard_buffer[j].dma;
+	}
+
+	return 0;
+}
+
+static int mxc_isi_video_validate_format(struct mxc_isi_video *video)
+{
+	const struct v4l2_mbus_framefmt *format;
+	const struct mxc_isi_format_info *info;
+	struct v4l2_subdev_state *state;
+	struct v4l2_subdev *sd = &video->pipe->sd;
+	int ret = 0;
+
+	state = v4l2_subdev_lock_and_get_active_state(sd);
+
+	info = mxc_isi_format_by_fourcc(video->pix.pixelformat,
+					MXC_ISI_VIDEO_CAP);
+	format = v4l2_subdev_get_try_format(sd, state, MXC_ISI_PIPE_PAD_SOURCE);
+
+	if (format->code != info->mbus_code ||
+	    format->width != video->pix.width ||
+	    format->height != video->pix.height) {
+		dev_dbg(video->pipe->isi->dev,
+			"%s: configuration mismatch, 0x%04x/%ux%u != 0x%04x/%ux%u\n",
+			__func__, format->code, format->width, format->height,
+			info->mbus_code, video->pix.width, video->pix.height);
+		ret = -EINVAL;
+	}
+
+	v4l2_subdev_unlock_state(state);
+
+	return ret;
+}
+
+static void mxc_isi_video_return_buffers(struct mxc_isi_video *video,
+					 enum vb2_buffer_state state)
+{
+	struct mxc_isi_buffer *buf;
+
+	spin_lock_irq(&video->buf_lock);
+
+	while (!list_empty(&video->out_active)) {
+		buf = list_first_entry(&video->out_active,
+				       struct mxc_isi_buffer, list);
+		list_del_init(&buf->list);
+		if (buf->discard)
+			continue;
+
+		vb2_buffer_done(&buf->v4l2_buf.vb2_buf, state);
+	}
+
+	while (!list_empty(&video->out_pending)) {
+		buf = list_first_entry(&video->out_pending,
+				       struct mxc_isi_buffer, list);
+		list_del_init(&buf->list);
+		vb2_buffer_done(&buf->v4l2_buf.vb2_buf, state);
+	}
+
+	while (!list_empty(&video->out_discard)) {
+		buf = list_first_entry(&video->out_discard,
+				       struct mxc_isi_buffer, list);
+		list_del_init(&buf->list);
+	}
+
+	INIT_LIST_HEAD(&video->out_active);
+	INIT_LIST_HEAD(&video->out_pending);
+	INIT_LIST_HEAD(&video->out_discard);
+
+	spin_unlock_irq(&video->buf_lock);
+}
+
+static void mxc_isi_video_queue_first_buffers(struct mxc_isi_video *video)
+{
+	unsigned int discard;
+	unsigned int i;
+
+	lockdep_assert_held(&video->buf_lock);
+
+	/*
+	 * Queue two ISI channel output buffers. We are not guaranteed to have
+	 * any buffer in the pending list when this function is called from the
+	 * system resume handler. Use pending buffers as much as possible, and
+	 * use discard buffers to fill the remaining slots.
+	 */
+
+	/* How many discard buffers do we need to queue first ? */
+	discard = list_empty(&video->out_pending) ? 2
+		: list_is_singular(&video->out_pending) ? 1
+		: 0;
+
+	for (i = 0; i < 2; ++i) {
+		enum mxc_isi_buf_id buf_id = i == 0 ? MXC_ISI_BUF1
+					   : MXC_ISI_BUF2;
+		struct mxc_isi_buffer *buf;
+		struct list_head *list;
+
+		list = i < discard ? &video->out_discard : &video->out_pending;
+		buf = list_first_entry(list, struct mxc_isi_buffer, list);
+
+		mxc_isi_channel_set_outbuf(video->pipe, buf->dma_addrs, buf_id);
+		buf->id = buf_id;
+		list_move_tail(&buf->list, &video->out_active);
+	}
+}
+
+static inline struct mxc_isi_buffer *to_isi_buffer(struct vb2_v4l2_buffer *v4l2_buf)
+{
+	return container_of(v4l2_buf, struct mxc_isi_buffer, v4l2_buf);
+}
+
+int mxc_isi_video_queue_setup(const struct v4l2_pix_format_mplane *format,
+			      const struct mxc_isi_format_info *info,
+			      unsigned int *num_buffers,
+			      unsigned int *num_planes, unsigned int sizes[])
+{
+	unsigned int i;
+
+	if (*num_planes) {
+		if (*num_planes != info->mem_planes)
+			return -EINVAL;
+
+		for (i = 0; i < info->mem_planes; ++i) {
+			if (sizes[i] < format->plane_fmt[i].sizeimage)
+				return -EINVAL;
+		}
+
+		return 0;
+	}
+
+	*num_planes = info->mem_planes;
+
+	for (i = 0; i < info->mem_planes; ++i)
+		sizes[i] = format->plane_fmt[i].sizeimage;
+
+	return 0;
+}
+
+void mxc_isi_video_buffer_init(struct vb2_buffer *vb2, dma_addr_t dma_addrs[3],
+			       const struct mxc_isi_format_info *info,
+			       const struct v4l2_pix_format_mplane *pix)
+{
+	unsigned int i;
+
+	for (i = 0; i < info->mem_planes; ++i)
+		dma_addrs[i] = vb2_dma_contig_plane_dma_addr(vb2, i);
+
+	/*
+	 * For single-planar pixel formats with multiple color planes, split
+	 * the buffer into color planes.
+	 */
+	if (info->color_planes != info->mem_planes) {
+		unsigned int size = pix->plane_fmt[0].bytesperline * pix->height;
+
+		for (i = 1; i < info->color_planes; ++i) {
+			unsigned int vsub = i > 1 ? info->vsub : 1;
+
+			dma_addrs[i] = dma_addrs[i - 1] + size / vsub;
+		}
+	}
+}
+
+int mxc_isi_video_buffer_prepare(struct mxc_isi_dev *isi, struct vb2_buffer *vb2,
+				 const struct mxc_isi_format_info *info,
+				 const struct v4l2_pix_format_mplane *pix)
+{
+	unsigned int i;
+
+	for (i = 0; i < info->mem_planes; i++) {
+		unsigned long size = pix->plane_fmt[i].sizeimage;
+
+		if (vb2_plane_size(vb2, i) < size) {
+			dev_err(isi->dev, "User buffer too small (%ld < %ld)\n",
+				vb2_plane_size(vb2, i), size);
+			return -EINVAL;
+		}
+
+		vb2_set_plane_payload(vb2, i, size);
+	}
+
+	return 0;
+}
+
+static int mxc_isi_vb2_queue_setup(struct vb2_queue *q,
+				   unsigned int *num_buffers,
+				   unsigned int *num_planes,
+				   unsigned int sizes[],
+				   struct device *alloc_devs[])
+{
+	struct mxc_isi_video *video = vb2_get_drv_priv(q);
+
+	return mxc_isi_video_queue_setup(&video->pix, video->fmtinfo,
+					 num_buffers, num_planes, sizes);
+}
+
+static int mxc_isi_vb2_buffer_init(struct vb2_buffer *vb2)
+{
+	struct mxc_isi_buffer *buf = to_isi_buffer(to_vb2_v4l2_buffer(vb2));
+	struct mxc_isi_video *video = vb2_get_drv_priv(vb2->vb2_queue);
+
+	mxc_isi_video_buffer_init(vb2, buf->dma_addrs, video->fmtinfo,
+				  &video->pix);
+
+	return 0;
+}
+
+static int mxc_isi_vb2_buffer_prepare(struct vb2_buffer *vb2)
+{
+	struct mxc_isi_video *video = vb2_get_drv_priv(vb2->vb2_queue);
+
+	return mxc_isi_video_buffer_prepare(video->pipe->isi, vb2,
+					    video->fmtinfo, &video->pix);
+}
+
+static void mxc_isi_vb2_buffer_queue(struct vb2_buffer *vb2)
+{
+	struct vb2_v4l2_buffer *v4l2_buf = to_vb2_v4l2_buffer(vb2);
+	struct mxc_isi_buffer *buf = to_isi_buffer(v4l2_buf);
+	struct mxc_isi_video *video = vb2_get_drv_priv(vb2->vb2_queue);
+
+	spin_lock_irq(&video->buf_lock);
+	list_add_tail(&buf->list, &video->out_pending);
+	spin_unlock_irq(&video->buf_lock);
+}
+
+static void mxc_isi_video_init_channel(struct mxc_isi_video *video)
+{
+	struct mxc_isi_pipe *pipe = video->pipe;
+
+	mxc_isi_channel_get(pipe);
+
+	mutex_lock(video->ctrls.handler.lock);
+	mxc_isi_channel_set_alpha(pipe, video->ctrls.alpha);
+	mxc_isi_channel_set_flip(pipe, video->ctrls.hflip, video->ctrls.vflip);
+	mutex_unlock(video->ctrls.handler.lock);
+
+	mxc_isi_channel_set_output_format(pipe, video->fmtinfo, &video->pix);
+}
+
+static int mxc_isi_vb2_start_streaming(struct vb2_queue *q, unsigned int count)
+{
+	struct mxc_isi_video *video = vb2_get_drv_priv(q);
+	unsigned int i;
+	int ret;
+
+	/* Initialize the ISI channel. */
+	mxc_isi_video_init_channel(video);
+
+	spin_lock_irq(&video->buf_lock);
+
+	/* Add the discard buffers to the out_discard list. */
+	for (i = 0; i < ARRAY_SIZE(video->buf_discard); ++i) {
+		struct mxc_isi_buffer *buf = &video->buf_discard[i];
+
+		list_add_tail(&buf->list, &video->out_discard);
+	}
+
+	/* Queue the first buffers. */
+	mxc_isi_video_queue_first_buffers(video);
+
+	/* Clear frame count */
+	video->frame_count = 0;
+
+	spin_unlock_irq(&video->buf_lock);
+
+	ret = mxc_isi_pipe_enable(video->pipe);
+	if (ret)
+		goto error;
+
+	return 0;
+
+error:
+	mxc_isi_channel_put(video->pipe);
+	mxc_isi_video_return_buffers(video, VB2_BUF_STATE_QUEUED);
+	return ret;
+}
+
+static void mxc_isi_vb2_stop_streaming(struct vb2_queue *q)
+{
+	struct mxc_isi_video *video = vb2_get_drv_priv(q);
+
+	mxc_isi_pipe_disable(video->pipe);
+	mxc_isi_channel_put(video->pipe);
+
+	mxc_isi_video_return_buffers(video, VB2_BUF_STATE_ERROR);
+}
+
+static const struct vb2_ops mxc_isi_vb2_qops = {
+	.queue_setup		= mxc_isi_vb2_queue_setup,
+	.buf_init		= mxc_isi_vb2_buffer_init,
+	.buf_prepare		= mxc_isi_vb2_buffer_prepare,
+	.buf_queue		= mxc_isi_vb2_buffer_queue,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
+	.start_streaming	= mxc_isi_vb2_start_streaming,
+	.stop_streaming		= mxc_isi_vb2_stop_streaming,
+};
+
+/* -----------------------------------------------------------------------------
+ * V4L2 controls
+ */
+
+static inline struct mxc_isi_video *ctrl_to_isi_video(struct v4l2_ctrl *ctrl)
+{
+	return container_of(ctrl->handler, struct mxc_isi_video, ctrls.handler);
+}
+
+static int mxc_isi_video_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct mxc_isi_video *video = ctrl_to_isi_video(ctrl);
+
+	switch (ctrl->id) {
+	case V4L2_CID_ALPHA_COMPONENT:
+		video->ctrls.alpha = ctrl->val;
+		break;
+	case V4L2_CID_VFLIP:
+		video->ctrls.vflip = ctrl->val;
+		break;
+	case V4L2_CID_HFLIP:
+		video->ctrls.hflip = ctrl->val;
+		break;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_ctrl_ops mxc_isi_video_ctrl_ops = {
+	.s_ctrl = mxc_isi_video_s_ctrl,
+};
+
+static int mxc_isi_video_ctrls_create(struct mxc_isi_video *video)
+{
+	struct v4l2_ctrl_handler *handler = &video->ctrls.handler;
+	int ret;
+
+	v4l2_ctrl_handler_init(handler, 3);
+
+	v4l2_ctrl_new_std(handler, &mxc_isi_video_ctrl_ops,
+			  V4L2_CID_ALPHA_COMPONENT, 0, 255, 1, 0);
+
+	v4l2_ctrl_new_std(handler, &mxc_isi_video_ctrl_ops,
+			  V4L2_CID_VFLIP, 0, 1, 1, 0);
+
+	v4l2_ctrl_new_std(handler, &mxc_isi_video_ctrl_ops,
+			  V4L2_CID_HFLIP, 0, 1, 1, 0);
+
+	if (handler->error) {
+		ret = handler->error;
+		v4l2_ctrl_handler_free(handler);
+		return ret;
+	}
+
+	video->vdev.ctrl_handler = handler;
+
+	return 0;
+}
+
+static void mxc_isi_video_ctrls_delete(struct mxc_isi_video *video)
+{
+	v4l2_ctrl_handler_free(&video->ctrls.handler);
+}
+
+/* -----------------------------------------------------------------------------
+ * V4L2 ioctls
+ */
+
+static int mxc_isi_video_querycap(struct file *file, void *priv,
+				  struct v4l2_capability *cap)
+{
+	strscpy(cap->driver, MXC_ISI_DRIVER_NAME, sizeof(cap->driver));
+	strscpy(cap->card, MXC_ISI_CAPTURE, sizeof(cap->card));
+
+	return 0;
+}
+
+static int mxc_isi_video_enum_fmt(struct file *file, void *priv,
+				  struct v4l2_fmtdesc *f)
+{
+	const struct mxc_isi_format_info *fmt;
+	unsigned int index = f->index;
+	unsigned int i;
+
+	if (f->mbus_code) {
+		/*
+		 * If a media bus code is specified, only enumerate formats
+		 * compatible with it.
+		 */
+		for (i = 0; i < ARRAY_SIZE(mxc_isi_formats); i++) {
+			fmt = &mxc_isi_formats[i];
+			if (fmt->mbus_code != f->mbus_code)
+				continue;
+
+			if (index == 0)
+				break;
+
+			index--;
+		}
+
+		if (i == ARRAY_SIZE(mxc_isi_formats))
+			return -EINVAL;
+	} else {
+		/* Otherwise, enumerate all formatS. */
+		if (f->index >= ARRAY_SIZE(mxc_isi_formats))
+			return -EINVAL;
+
+		fmt = &mxc_isi_formats[f->index];
+	}
+
+	f->pixelformat = fmt->fourcc;
+	f->flags |= V4L2_FMT_FLAG_CSC_COLORSPACE | V4L2_FMT_FLAG_CSC_YCBCR_ENC
+		 |  V4L2_FMT_FLAG_CSC_QUANTIZATION | V4L2_FMT_FLAG_CSC_XFER_FUNC;
+
+	return 0;
+}
+
+static int mxc_isi_video_g_fmt(struct file *file, void *fh,
+			       struct v4l2_format *f)
+{
+	struct mxc_isi_video *video = video_drvdata(file);
+
+	f->fmt.pix_mp = video->pix;
+
+	return 0;
+}
+
+static int mxc_isi_video_try_fmt(struct file *file, void *fh,
+				 struct v4l2_format *f)
+{
+	struct mxc_isi_video *video = video_drvdata(file);
+
+	mxc_isi_format_try(video->pipe, &f->fmt.pix_mp, MXC_ISI_VIDEO_CAP);
+	return 0;
+}
+
+static int mxc_isi_video_s_fmt(struct file *file, void *priv,
+			       struct v4l2_format *f)
+{
+	struct mxc_isi_video *video = video_drvdata(file);
+	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
+
+	if (vb2_is_busy(&video->vb2_q))
+		return -EBUSY;
+
+	video->fmtinfo = mxc_isi_format_try(video->pipe, pix, MXC_ISI_VIDEO_CAP);
+	video->pix = *pix;
+
+	return 0;
+}
+
+static int mxc_isi_video_streamon(struct file *file, void *priv,
+				  enum v4l2_buf_type type)
+{
+	struct mxc_isi_video *video = video_drvdata(file);
+	struct media_device *mdev = &video->pipe->isi->media_dev;
+	struct media_pipeline *pipe;
+	int ret;
+
+	if (vb2_queue_is_busy(&video->vb2_q, file))
+		return -EBUSY;
+
+	/*
+	 * Get a pipeline for the video node and start it. This must be done
+	 * here and not in the queue .start_streaming() handler, so that
+	 * pipeline start errors can be reported from VIDIOC_STREAMON and not
+	 * delayed until subsequent VIDIOC_QBUF calls.
+	 */
+	mutex_lock(&mdev->graph_mutex);
+
+	ret = mxc_isi_pipe_acquire(video->pipe, &mxc_isi_video_frame_write_done);
+	if (ret) {
+		mutex_unlock(&mdev->graph_mutex);
+		return ret;
+	}
+
+	pipe = media_entity_pipeline(&video->vdev.entity) ? : &video->pipe->pipe;
+
+	ret = __video_device_pipeline_start(&video->vdev, pipe);
+	if (ret) {
+		mutex_unlock(&mdev->graph_mutex);
+		goto err_release;
+	}
+
+	mutex_unlock(&mdev->graph_mutex);
+
+	/* Verify that the video format matches the output of the subdev. */
+	ret = mxc_isi_video_validate_format(video);
+	if (ret)
+		goto err_stop;
+
+	/* Allocate buffers for discard operation. */
+	ret = mxc_isi_video_alloc_discard_buffers(video);
+	if (ret)
+		goto err_stop;
+
+	ret = vb2_streamon(&video->vb2_q, type);
+	if (ret)
+		goto err_free;
+
+	video->is_streaming = true;
+
+	return 0;
+
+err_free:
+	mxc_isi_video_free_discard_buffers(video);
+err_stop:
+	video_device_pipeline_stop(&video->vdev);
+err_release:
+	mxc_isi_pipe_release(video->pipe);
+	return ret;
+}
+
+static void mxc_isi_video_cleanup_streaming(struct mxc_isi_video *video)
+{
+	lockdep_assert_held(&video->lock);
+
+	if (!video->is_streaming)
+		return;
+
+	mxc_isi_video_free_discard_buffers(video);
+	video_device_pipeline_stop(&video->vdev);
+	mxc_isi_pipe_release(video->pipe);
+
+	video->is_streaming = false;
+}
+
+static int mxc_isi_video_streamoff(struct file *file, void *priv,
+				   enum v4l2_buf_type type)
+{
+	struct mxc_isi_video *video = video_drvdata(file);
+	int ret;
+
+	ret = vb2_ioctl_streamoff(file, priv, type);
+	if (ret)
+		return ret;
+
+	mxc_isi_video_cleanup_streaming(video);
+
+	return 0;
+}
+
+static int mxc_isi_video_enum_framesizes(struct file *file, void *priv,
+					 struct v4l2_frmsizeenum *fsize)
+{
+	struct mxc_isi_video *video = video_drvdata(file);
+	const struct mxc_isi_format_info *info;
+	unsigned int max_width;
+	unsigned int h_align;
+	unsigned int v_align;
+
+	if (fsize->index)
+		return -EINVAL;
+
+	info = mxc_isi_format_by_fourcc(fsize->pixel_format, MXC_ISI_VIDEO_CAP);
+	if (!info)
+		return -EINVAL;
+
+	h_align = max_t(unsigned int, info->hsub, 1);
+	v_align = max_t(unsigned int, info->vsub, 1);
+
+	max_width = video->pipe->id == video->pipe->isi->pdata->num_channels - 1
+		  ? MXC_ISI_MAX_WIDTH_UNCHAINED
+		  : MXC_ISI_MAX_WIDTH_CHAINED;
+
+	fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
+	fsize->stepwise.min_width = ALIGN(MXC_ISI_MIN_WIDTH, h_align);
+	fsize->stepwise.min_height = ALIGN(MXC_ISI_MIN_HEIGHT, v_align);
+	fsize->stepwise.max_width = ALIGN_DOWN(max_width, h_align);
+	fsize->stepwise.max_height = ALIGN_DOWN(MXC_ISI_MAX_HEIGHT, v_align);
+	fsize->stepwise.step_width = h_align;
+	fsize->stepwise.step_height = v_align;
+
+	/*
+	 * The width can be further restricted due to line buffer sharing
+	 * between pipelines when scaling, but we have no way to know here if
+	 * the scaler will be used.
+	 */
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops mxc_isi_video_ioctl_ops = {
+	.vidioc_querycap		= mxc_isi_video_querycap,
+
+	.vidioc_enum_fmt_vid_cap	= mxc_isi_video_enum_fmt,
+	.vidioc_try_fmt_vid_cap_mplane	= mxc_isi_video_try_fmt,
+	.vidioc_s_fmt_vid_cap_mplane	= mxc_isi_video_s_fmt,
+	.vidioc_g_fmt_vid_cap_mplane	= mxc_isi_video_g_fmt,
+
+	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
+	.vidioc_querybuf		= vb2_ioctl_querybuf,
+	.vidioc_qbuf			= vb2_ioctl_qbuf,
+	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
+	.vidioc_expbuf			= vb2_ioctl_expbuf,
+	.vidioc_prepare_buf		= vb2_ioctl_prepare_buf,
+	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
+
+	.vidioc_streamon		= mxc_isi_video_streamon,
+	.vidioc_streamoff		= mxc_isi_video_streamoff,
+
+	.vidioc_enum_framesizes		= mxc_isi_video_enum_framesizes,
+
+	.vidioc_subscribe_event		= v4l2_ctrl_subscribe_event,
+	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
+};
+
+/* -----------------------------------------------------------------------------
+ * Video device file operations
+ */
+
+static int mxc_isi_video_open(struct file *file)
+{
+	struct mxc_isi_video *video = video_drvdata(file);
+	int ret;
+
+	ret = v4l2_fh_open(file);
+	if (ret)
+		return ret;
+
+	ret = pm_runtime_resume_and_get(video->pipe->isi->dev);
+	if (ret) {
+		v4l2_fh_release(file);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int mxc_isi_video_release(struct file *file)
+{
+	struct mxc_isi_video *video = video_drvdata(file);
+	int ret;
+
+	ret = vb2_fop_release(file);
+	if (ret)
+		dev_err(video->pipe->isi->dev, "%s fail\n", __func__);
+
+	mutex_lock(&video->lock);
+	mxc_isi_video_cleanup_streaming(video);
+	mutex_unlock(&video->lock);
+
+	pm_runtime_put(video->pipe->isi->dev);
+	return ret;
+}
+
+static const struct v4l2_file_operations mxc_isi_video_fops = {
+	.owner		= THIS_MODULE,
+	.open		= mxc_isi_video_open,
+	.release	= mxc_isi_video_release,
+	.poll		= vb2_fop_poll,
+	.unlocked_ioctl	= video_ioctl2,
+	.mmap		= vb2_fop_mmap,
+};
+
+/* -----------------------------------------------------------------------------
+ * Suspend & resume
+ */
+
+void mxc_isi_video_suspend(struct mxc_isi_pipe *pipe)
+{
+	struct mxc_isi_video *video = &pipe->video;
+
+	if (!video->is_streaming)
+		return;
+
+	mxc_isi_pipe_disable(pipe);
+	mxc_isi_channel_put(pipe);
+
+	spin_lock_irq(&video->buf_lock);
+
+	/*
+	 * Move the active buffers back to the pending or discard list. We must
+	 * iterate the active list backward and move the buffers to the head of
+	 * the pending list to preserve the buffer queueing order.
+	 */
+	while (!list_empty(&video->out_active)) {
+		struct mxc_isi_buffer *buf =
+			list_last_entry(&video->out_active,
+					struct mxc_isi_buffer, list);
+
+		if (buf->discard)
+			list_move(&buf->list, &video->out_discard);
+		else
+			list_move(&buf->list, &video->out_pending);
+	}
+
+	spin_unlock_irq(&video->buf_lock);
+}
+
+int mxc_isi_video_resume(struct mxc_isi_pipe *pipe)
+{
+	struct mxc_isi_video *video = &pipe->video;
+
+	if (!video->is_streaming)
+		return 0;
+
+	mxc_isi_video_init_channel(video);
+
+	spin_lock_irq(&video->buf_lock);
+	mxc_isi_video_queue_first_buffers(video);
+	spin_unlock_irq(&video->buf_lock);
+
+	return mxc_isi_pipe_enable(pipe);
+}
+
+/* -----------------------------------------------------------------------------
+ * Registration
+ */
+
+int mxc_isi_video_register(struct mxc_isi_pipe *pipe,
+			   struct v4l2_device *v4l2_dev)
+{
+	struct mxc_isi_video *video = &pipe->video;
+	struct v4l2_pix_format_mplane *pix = &video->pix;
+	struct video_device *vdev = &video->vdev;
+	struct vb2_queue *q = &video->vb2_q;
+	int ret = -ENOMEM;
+
+	video->pipe = pipe;
+
+	mutex_init(&video->lock);
+	spin_lock_init(&video->buf_lock);
+
+	pix->width = MXC_ISI_DEF_WIDTH;
+	pix->height = MXC_ISI_DEF_HEIGHT;
+	pix->pixelformat = MXC_ISI_DEF_PIXEL_FORMAT;
+	pix->colorspace = MXC_ISI_DEF_COLOR_SPACE;
+	pix->ycbcr_enc = MXC_ISI_DEF_YCBCR_ENC;
+	pix->quantization = MXC_ISI_DEF_QUANTIZATION;
+	pix->xfer_func = MXC_ISI_DEF_XFER_FUNC;
+	video->fmtinfo = mxc_isi_format_try(video->pipe, pix, MXC_ISI_VIDEO_CAP);
+
+	memset(vdev, 0, sizeof(*vdev));
+	snprintf(vdev->name, sizeof(vdev->name), "mxc_isi.%d.capture", pipe->id);
+
+	vdev->fops	= &mxc_isi_video_fops;
+	vdev->ioctl_ops	= &mxc_isi_video_ioctl_ops;
+	vdev->v4l2_dev	= v4l2_dev;
+	vdev->minor	= -1;
+	vdev->release	= video_device_release_empty;
+	vdev->queue	= q;
+	vdev->lock	= &video->lock;
+
+	vdev->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE_MPLANE
+			  | V4L2_CAP_IO_MC;
+	video_set_drvdata(vdev, video);
+
+	INIT_LIST_HEAD(&video->out_pending);
+	INIT_LIST_HEAD(&video->out_active);
+	INIT_LIST_HEAD(&video->out_discard);
+
+	memset(q, 0, sizeof(*q));
+	q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	q->io_modes = VB2_MMAP | VB2_DMABUF;
+	q->drv_priv = video;
+	q->ops = &mxc_isi_vb2_qops;
+	q->mem_ops = &vb2_dma_contig_memops;
+	q->buf_struct_size = sizeof(struct mxc_isi_buffer);
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->min_buffers_needed = 2;
+	q->lock = &video->lock;
+	q->dev = pipe->isi->dev;
+
+	ret = vb2_queue_init(q);
+	if (ret)
+		goto err_free_ctx;
+
+	video->pad.flags = MEDIA_PAD_FL_SINK;
+	vdev->entity.function = MEDIA_ENT_F_PROC_VIDEO_SCALER;
+	ret = media_entity_pads_init(&vdev->entity, 1, &video->pad);
+	if (ret)
+		goto err_free_ctx;
+
+	ret = mxc_isi_video_ctrls_create(video);
+	if (ret)
+		goto err_me_cleanup;
+
+	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
+	if (ret)
+		goto err_ctrl_free;
+
+	ret = media_create_pad_link(&pipe->sd.entity,
+				    MXC_ISI_PIPE_PAD_SOURCE,
+				    &vdev->entity, 0,
+				    MEDIA_LNK_FL_IMMUTABLE |
+				    MEDIA_LNK_FL_ENABLED);
+	if (ret)
+		goto err_video_unreg;
+
+	return 0;
+
+err_video_unreg:
+	video_unregister_device(vdev);
+err_ctrl_free:
+	mxc_isi_video_ctrls_delete(video);
+err_me_cleanup:
+	media_entity_cleanup(&vdev->entity);
+err_free_ctx:
+	return ret;
+}
+
+void mxc_isi_video_unregister(struct mxc_isi_pipe *pipe)
+{
+	struct mxc_isi_video *video = &pipe->video;
+	struct video_device *vdev = &video->vdev;
+
+	mutex_lock(&video->lock);
+
+	if (video_is_registered(vdev)) {
+		video_unregister_device(vdev);
+		mxc_isi_video_ctrls_delete(video);
+		media_entity_cleanup(&vdev->entity);
+	}
+
+	mutex_unlock(&video->lock);
+}
diff --git a/drivers/media/platform/nxp/mx2_emmaprp.c b/drivers/media/platform/nxp/mx2_emmaprp.c
index 3ce84d0f078c..023ed40c6b08 100644
--- a/drivers/media/platform/nxp/mx2_emmaprp.c
+++ b/drivers/media/platform/nxp/mx2_emmaprp.c
@@ -888,7 +888,7 @@ unreg_dev:
 	return ret;
 }
 
-static int emmaprp_remove(struct platform_device *pdev)
+static void emmaprp_remove(struct platform_device *pdev)
 {
 	struct emmaprp_dev *pcdev = platform_get_drvdata(pdev);
 
@@ -898,13 +898,11 @@ static int emmaprp_remove(struct platform_device *pdev)
 	v4l2_m2m_release(pcdev->m2m_dev);
 	v4l2_device_unregister(&pcdev->v4l2_dev);
 	mutex_destroy(&pcdev->dev_mutex);
-
-	return 0;
 }
 
 static struct platform_driver emmaprp_pdrv = {
 	.probe		= emmaprp_probe,
-	.remove		= emmaprp_remove,
+	.remove_new	= emmaprp_remove,
 	.driver		= {
 		.name	= MEM2MEM_NAME,
 	},
diff --git a/drivers/media/platform/qcom/camss/camss-csid-gen2.c b/drivers/media/platform/qcom/camss/camss-csid-gen2.c
index 2031bde13a93..0f8ac29d038d 100644
--- a/drivers/media/platform/qcom/camss/camss-csid-gen2.c
+++ b/drivers/media/platform/qcom/camss/camss-csid-gen2.c
@@ -334,13 +334,14 @@ static const struct csid_format csid_formats[] = {
 	},
 };
 
-static void csid_configure_stream(struct csid_device *csid, u8 enable)
+static void __csid_configure_stream(struct csid_device *csid, u8 enable, u8 vc)
 {
 	struct csid_testgen_config *tg = &csid->testgen;
 	u32 val;
 	u32 phy_sel = 0;
 	u8 lane_cnt = csid->phy.lane_cnt;
-	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_SRC];
+	/* Source pads matching RDI channels on hardware. Pad 1 -> RDI0, Pad 2 -> RDI1, etc. */
+	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc];
 	const struct csid_format *format = csid_get_fmt_entry(csid->formats, csid->nformats,
 							      input_format->code);
 
@@ -351,8 +352,7 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable)
 		phy_sel = csid->phy.csiphy_id;
 
 	if (enable) {
-		u8 vc = 0; /* Virtual Channel 0 */
-		u8 dt_id = vc * 4;
+		u8 dt_id = vc;
 
 		if (tg->enabled) {
 			/* Config Test Generator */
@@ -395,42 +395,42 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable)
 		val |= format->data_type << RDI_CFG0_DATA_TYPE;
 		val |= vc << RDI_CFG0_VIRTUAL_CHANNEL;
 		val |= dt_id << RDI_CFG0_DT_ID;
-		writel_relaxed(val, csid->base + CSID_RDI_CFG0(0));
+		writel_relaxed(val, csid->base + CSID_RDI_CFG0(vc));
 
 		/* CSID_TIMESTAMP_STB_POST_IRQ */
 		val = 2 << RDI_CFG1_TIMESTAMP_STB_SEL;
-		writel_relaxed(val, csid->base + CSID_RDI_CFG1(0));
+		writel_relaxed(val, csid->base + CSID_RDI_CFG1(vc));
 
 		val = 1;
-		writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(0));
+		writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(vc));
 
 		val = 0;
-		writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PATTERN(0));
+		writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PATTERN(vc));
 
 		val = 1;
-		writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(0));
+		writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(vc));
 
 		val = 0;
-		writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(0));
+		writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(vc));
 
 		val = 1;
-		writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PERIOD(0));
+		writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PERIOD(vc));
 
 		val = 0;
-		writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PATTERN(0));
+		writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PATTERN(vc));
 
 		val = 1;
-		writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PERIOD(0));
+		writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PERIOD(vc));
 
 		val = 0;
-		writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PATTERN(0));
+		writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PATTERN(vc));
 
 		val = 0;
-		writel_relaxed(val, csid->base + CSID_RDI_CTRL(0));
+		writel_relaxed(val, csid->base + CSID_RDI_CTRL(vc));
 
-		val = readl_relaxed(csid->base + CSID_RDI_CFG0(0));
+		val = readl_relaxed(csid->base + CSID_RDI_CFG0(vc));
 		val |=  1 << RDI_CFG0_ENABLE;
-		writel_relaxed(val, csid->base + CSID_RDI_CFG0(0));
+		writel_relaxed(val, csid->base + CSID_RDI_CFG0(vc));
 	}
 
 	if (tg->enabled) {
@@ -456,7 +456,16 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable)
 		val = HALT_CMD_RESUME_AT_FRAME_BOUNDARY << RDI_CTRL_HALT_CMD;
 	else
 		val = HALT_CMD_HALT_AT_FRAME_BOUNDARY << RDI_CTRL_HALT_CMD;
-	writel_relaxed(val, csid->base + CSID_RDI_CTRL(0));
+	writel_relaxed(val, csid->base + CSID_RDI_CTRL(vc));
+}
+
+static void csid_configure_stream(struct csid_device *csid, u8 enable)
+{
+	u8 i;
+	/* Loop through all enabled VCs and configure stream for each */
+	for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++)
+		if (csid->phy.en_vc & BIT(i))
+			__csid_configure_stream(csid, enable, i);
 }
 
 static int csid_configure_testgen_pattern(struct csid_device *csid, s32 val)
@@ -502,6 +511,7 @@ static irqreturn_t csid_isr(int irq, void *dev)
 	struct csid_device *csid = dev;
 	u32 val;
 	u8 reset_done;
+	int i;
 
 	val = readl_relaxed(csid->base + CSID_TOP_IRQ_STATUS);
 	writel_relaxed(val, csid->base + CSID_TOP_IRQ_CLEAR);
@@ -510,8 +520,12 @@ static irqreturn_t csid_isr(int irq, void *dev)
 	val = readl_relaxed(csid->base + CSID_CSI2_RX_IRQ_STATUS);
 	writel_relaxed(val, csid->base + CSID_CSI2_RX_IRQ_CLEAR);
 
-	val = readl_relaxed(csid->base + CSID_CSI2_RDIN_IRQ_STATUS(0));
-	writel_relaxed(val, csid->base + CSID_CSI2_RDIN_IRQ_CLEAR(0));
+	/* Read and clear IRQ status for each enabled RDI channel */
+	for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++)
+		if (csid->phy.en_vc & BIT(i)) {
+			val = readl_relaxed(csid->base + CSID_CSI2_RDIN_IRQ_STATUS(i));
+			writel_relaxed(val, csid->base + CSID_CSI2_RDIN_IRQ_CLEAR(i));
+		}
 
 	val = 1 << IRQ_CMD_CLEAR;
 	writel_relaxed(val, csid->base + CSID_IRQ_CMD);
diff --git a/drivers/media/platform/qcom/camss/camss-csid.c b/drivers/media/platform/qcom/camss/camss-csid.c
index 88f188e0f750..6360314f04a6 100644
--- a/drivers/media/platform/qcom/camss/camss-csid.c
+++ b/drivers/media/platform/qcom/camss/camss-csid.c
@@ -196,6 +196,8 @@ static int csid_set_power(struct v4l2_subdev *sd, int on)
 			return ret;
 		}
 
+		csid->phy.need_vc_update = true;
+
 		enable_irq(csid->irq);
 
 		ret = csid->ops->reset(csid);
@@ -249,7 +251,10 @@ static int csid_set_stream(struct v4l2_subdev *sd, int enable)
 			return -ENOLINK;
 	}
 
-	csid->ops->configure_stream(csid, enable);
+	if (csid->phy.need_vc_update) {
+		csid->ops->configure_stream(csid, enable);
+		csid->phy.need_vc_update = false;
+	}
 
 	return 0;
 }
@@ -460,6 +465,7 @@ static int csid_set_format(struct v4l2_subdev *sd,
 {
 	struct csid_device *csid = v4l2_get_subdevdata(sd);
 	struct v4l2_mbus_framefmt *format;
+	int i;
 
 	format = __csid_get_format(csid, sd_state, fmt->pad, fmt->which);
 	if (format == NULL)
@@ -468,14 +474,14 @@ static int csid_set_format(struct v4l2_subdev *sd,
 	csid_try_format(csid, sd_state, fmt->pad, &fmt->format, fmt->which);
 	*format = fmt->format;
 
-	/* Propagate the format from sink to source */
+	/* Propagate the format from sink to source pads */
 	if (fmt->pad == MSM_CSID_PAD_SINK) {
-		format = __csid_get_format(csid, sd_state, MSM_CSID_PAD_SRC,
-					   fmt->which);
+		for (i = MSM_CSID_PAD_FIRST_SRC; i < MSM_CSID_PADS_NUM; ++i) {
+			format = __csid_get_format(csid, sd_state, i, fmt->which);
 
-		*format = fmt->format;
-		csid_try_format(csid, sd_state, MSM_CSID_PAD_SRC, format,
-				fmt->which);
+			*format = fmt->format;
+			csid_try_format(csid, sd_state, i, format, fmt->which);
+		}
 	}
 
 	return 0;
@@ -738,7 +744,6 @@ static int csid_link_setup(struct media_entity *entity,
 		struct csid_device *csid;
 		struct csiphy_device *csiphy;
 		struct csiphy_lanes_cfg *lane_cfg;
-		struct v4l2_subdev_format format = { 0 };
 
 		sd = media_entity_to_v4l2_subdev(entity);
 		csid = v4l2_get_subdevdata(sd);
@@ -761,11 +766,22 @@ static int csid_link_setup(struct media_entity *entity,
 		lane_cfg = &csiphy->cfg.csi2->lane_cfg;
 		csid->phy.lane_cnt = lane_cfg->num_data;
 		csid->phy.lane_assign = csid_get_lane_assign(lane_cfg);
+	}
+	/* Decide which virtual channels to enable based on which source pads are enabled */
+	if (local->flags & MEDIA_PAD_FL_SOURCE) {
+		struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(entity);
+		struct csid_device *csid = v4l2_get_subdevdata(sd);
+		struct device *dev = csid->camss->dev;
+
+		if (flags & MEDIA_LNK_FL_ENABLED)
+			csid->phy.en_vc |= BIT(local->index - 1);
+		else
+			csid->phy.en_vc &= ~BIT(local->index - 1);
 
-		/* Reset format on source pad to sink pad format */
-		format.pad = MSM_CSID_PAD_SRC;
-		format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-		csid_set_format(&csid->subdev, NULL, &format);
+		csid->phy.need_vc_update = true;
+
+		dev_dbg(dev, "%s: Enabled CSID virtual channels mask 0x%x\n",
+			__func__, csid->phy.en_vc);
 	}
 
 	return 0;
@@ -816,6 +832,7 @@ int msm_csid_register_entity(struct csid_device *csid,
 	struct v4l2_subdev *sd = &csid->subdev;
 	struct media_pad *pads = csid->pads;
 	struct device *dev = csid->camss->dev;
+	int i;
 	int ret;
 
 	v4l2_subdev_init(sd, &csid_v4l2_ops);
@@ -852,7 +869,8 @@ int msm_csid_register_entity(struct csid_device *csid,
 	}
 
 	pads[MSM_CSID_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
-	pads[MSM_CSID_PAD_SRC].flags = MEDIA_PAD_FL_SOURCE;
+	for (i = MSM_CSID_PAD_FIRST_SRC; i < MSM_CSID_PADS_NUM; ++i)
+		pads[i].flags = MEDIA_PAD_FL_SOURCE;
 
 	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
 	sd->entity.ops = &csid_media_ops;
diff --git a/drivers/media/platform/qcom/camss/camss-csid.h b/drivers/media/platform/qcom/camss/camss-csid.h
index f06040e44c51..d4b48432a097 100644
--- a/drivers/media/platform/qcom/camss/camss-csid.h
+++ b/drivers/media/platform/qcom/camss/camss-csid.h
@@ -19,8 +19,13 @@
 #include <media/v4l2-subdev.h>
 
 #define MSM_CSID_PAD_SINK 0
-#define MSM_CSID_PAD_SRC 1
-#define MSM_CSID_PADS_NUM 2
+#define MSM_CSID_PAD_FIRST_SRC 1
+#define MSM_CSID_PADS_NUM 5
+
+#define MSM_CSID_PAD_SRC (MSM_CSID_PAD_FIRST_SRC)
+
+/* CSID hardware can demultiplex up to 4 outputs */
+#define MSM_CSID_MAX_SRC_STREAMS	4
 
 #define DATA_TYPE_EMBEDDED_DATA_8BIT	0x12
 #define DATA_TYPE_YUV420_8BIT		0x18
@@ -81,6 +86,8 @@ struct csid_phy_config {
 	u8 csiphy_id;
 	u8 lane_cnt;
 	u32 lane_assign;
+	u32 en_vc;
+	u8 need_vc_update;
 };
 
 struct csid_device;
diff --git a/drivers/media/platform/qcom/camss/camss-vfe-170.c b/drivers/media/platform/qcom/camss/camss-vfe-170.c
index 8e506a805d11..02494c89da91 100644
--- a/drivers/media/platform/qcom/camss/camss-vfe-170.c
+++ b/drivers/media/platform/qcom/camss/camss-vfe-170.c
@@ -409,7 +409,7 @@ static int vfe_get_output(struct vfe_line *line)
 	spin_lock_irqsave(&vfe->output_lock, flags);
 
 	output = &line->output;
-	if (output->state != VFE_OUTPUT_OFF) {
+	if (output->state > VFE_OUTPUT_RESERVED) {
 		dev_err(vfe->camss->dev, "Output is running\n");
 		goto error;
 	}
@@ -462,7 +462,7 @@ static int vfe_enable_output(struct vfe_line *line)
 
 	ops->reg_update_clear(vfe, line->id);
 
-	if (output->state != VFE_OUTPUT_OFF) {
+	if (output->state > VFE_OUTPUT_RESERVED) {
 		dev_err(vfe->camss->dev, "Output is not in reserved state %d\n",
 			output->state);
 		spin_unlock_irqrestore(&vfe->output_lock, flags);
diff --git a/drivers/media/platform/qcom/camss/camss-vfe-480.c b/drivers/media/platform/qcom/camss/camss-vfe-480.c
index 3aa962b5663b..f70aad2e8c23 100644
--- a/drivers/media/platform/qcom/camss/camss-vfe-480.c
+++ b/drivers/media/platform/qcom/camss/camss-vfe-480.c
@@ -94,6 +94,8 @@ static inline int bus_irq_mask_0_comp_done(struct vfe_device *vfe, int n)
 #define RDI_WM(n)			((IS_LITE ? 0 : 23) + (n))
 #define RDI_COMP_GROUP(n)		((IS_LITE ? 0 : 11) + (n))
 
+#define MAX_VFE_OUTPUT_LINES	4
+
 static u32 vfe_hw_version(struct vfe_device *vfe)
 {
 	u32 hw_version = readl_relaxed(vfe->base + VFE_HW_VERSION);
@@ -171,12 +173,26 @@ static inline void vfe_reg_update_clear(struct vfe_device *vfe,
 
 static void vfe_enable_irq_common(struct vfe_device *vfe)
 {
-	/* enable only the IRQs used: rup and comp_done irqs for RDI0 */
+	/* enable reset ack IRQ and top BUS status IRQ */
 	writel_relaxed(IRQ_MASK_0_RESET_ACK | IRQ_MASK_0_BUS_TOP_IRQ,
 		       vfe->base + VFE_IRQ_MASK(0));
-	writel_relaxed(BUS_IRQ_MASK_0_RDI_RUP(vfe, 0) |
-		       BUS_IRQ_MASK_0_COMP_DONE(vfe, RDI_COMP_GROUP(0)),
-		       vfe->base + VFE_BUS_IRQ_MASK(0));
+}
+
+static void vfe_enable_lines_irq(struct vfe_device *vfe)
+{
+	int i;
+	u32 bus_irq_mask = 0;
+
+	for (i = 0; i < MAX_VFE_OUTPUT_LINES; i++) {
+		/* Enable IRQ for newly added lines, but also keep already running lines's IRQ */
+		if (vfe->line[i].output.state == VFE_OUTPUT_RESERVED ||
+		    vfe->line[i].output.state == VFE_OUTPUT_ON) {
+			bus_irq_mask |= BUS_IRQ_MASK_0_RDI_RUP(vfe, i)
+					| BUS_IRQ_MASK_0_COMP_DONE(vfe, RDI_COMP_GROUP(i));
+			}
+	}
+
+	writel_relaxed(bus_irq_mask, vfe->base + VFE_BUS_IRQ_MASK(0));
 }
 
 static void vfe_isr_reg_update(struct vfe_device *vfe, enum vfe_line_id line_id);
@@ -193,6 +209,7 @@ static irqreturn_t vfe_isr(int irq, void *dev)
 {
 	struct vfe_device *vfe = dev;
 	u32 status;
+	int i;
 
 	status = readl_relaxed(vfe->base + VFE_IRQ_STATUS(0));
 	writel_relaxed(status, vfe->base + VFE_IRQ_CLEAR(0));
@@ -207,11 +224,14 @@ static irqreturn_t vfe_isr(int irq, void *dev)
 		writel_relaxed(status, vfe->base + VFE_BUS_IRQ_CLEAR(0));
 		writel_relaxed(1, vfe->base + VFE_BUS_IRQ_CLEAR_GLOBAL);
 
-		if (status & BUS_IRQ_MASK_0_RDI_RUP(vfe, 0))
-			vfe_isr_reg_update(vfe, 0);
+		/* Loop through all WMs IRQs */
+		for (i = 0; i < MSM_VFE_IMAGE_MASTERS_NUM; i++) {
+			if (status & BUS_IRQ_MASK_0_RDI_RUP(vfe, i))
+				vfe_isr_reg_update(vfe, i);
 
-		if (status & BUS_IRQ_MASK_0_COMP_DONE(vfe, RDI_COMP_GROUP(0)))
-			vfe_isr_wm_done(vfe, 0);
+			if (status & BUS_IRQ_MASK_0_COMP_DONE(vfe, RDI_COMP_GROUP(i)))
+				vfe_isr_wm_done(vfe, i);
+		}
 	}
 
 	return IRQ_HANDLED;
@@ -234,24 +254,23 @@ static int vfe_get_output(struct vfe_line *line)
 	struct vfe_device *vfe = to_vfe(line);
 	struct vfe_output *output;
 	unsigned long flags;
-	int wm_idx;
 
 	spin_lock_irqsave(&vfe->output_lock, flags);
 
 	output = &line->output;
-	if (output->state != VFE_OUTPUT_OFF) {
+	if (output->state > VFE_OUTPUT_RESERVED) {
 		dev_err(vfe->camss->dev, "Output is running\n");
 		goto error;
 	}
 
 	output->wm_num = 1;
 
-	wm_idx = vfe_reserve_wm(vfe, line->id);
-	if (wm_idx < 0) {
-		dev_err(vfe->camss->dev, "Can not reserve wm\n");
-		goto error_get_wm;
-	}
-	output->wm_idx[0] = wm_idx;
+	/* Correspondence between VFE line number and WM number.
+	 * line 0 -> RDI 0, line 1 -> RDI1, line 2 -> RDI2, line 3 -> PIX/RDI3
+	 * Note this 1:1 mapping will not work for PIX streams.
+	 */
+	output->wm_idx[0] = line->id;
+	vfe->wm_output_map[line->id] = line->id;
 
 	output->drop_update_idx = 0;
 
@@ -259,11 +278,9 @@ static int vfe_get_output(struct vfe_line *line)
 
 	return 0;
 
-error_get_wm:
-	vfe_release_wm(vfe, output->wm_idx[0]);
-	output->state = VFE_OUTPUT_OFF;
 error:
 	spin_unlock_irqrestore(&vfe->output_lock, flags);
+	output->state = VFE_OUTPUT_OFF;
 
 	return -EINVAL;
 }
@@ -279,7 +296,7 @@ static int vfe_enable_output(struct vfe_line *line)
 
 	vfe_reg_update_clear(vfe, line->id);
 
-	if (output->state != VFE_OUTPUT_OFF) {
+	if (output->state > VFE_OUTPUT_RESERVED) {
 		dev_err(vfe->camss->dev, "Output is not in reserved state %d\n",
 			output->state);
 		spin_unlock_irqrestore(&vfe->output_lock, flags);
@@ -360,6 +377,8 @@ static int vfe_enable(struct vfe_line *line)
 
 	vfe->stream_count++;
 
+	vfe_enable_lines_irq(vfe);
+
 	mutex_unlock(&vfe->stream_lock);
 
 	ret = vfe_get_output(line);
@@ -566,7 +585,7 @@ static const struct camss_video_ops vfe_video_ops_480 = {
 static void vfe_subdev_init(struct device *dev, struct vfe_device *vfe)
 {
 	vfe->video_ops = vfe_video_ops_480;
-	vfe->line_num = 1;
+	vfe->line_num = MAX_VFE_OUTPUT_LINES;
 }
 
 const struct vfe_hw_ops vfe_ops_480 = {
diff --git a/drivers/media/platform/qcom/camss/camss-vfe-gen1.c b/drivers/media/platform/qcom/camss/camss-vfe-gen1.c
index 4fd265d01883..239d3d4ac666 100644
--- a/drivers/media/platform/qcom/camss/camss-vfe-gen1.c
+++ b/drivers/media/platform/qcom/camss/camss-vfe-gen1.c
@@ -194,7 +194,7 @@ static int vfe_enable_output(struct vfe_line *line)
 
 	ops->reg_update_clear(vfe, line->id);
 
-	if (output->state != VFE_OUTPUT_RESERVED) {
+	if (output->state > VFE_OUTPUT_RESERVED) {
 		dev_err(vfe->camss->dev, "Output is not in reserved state %d\n", output->state);
 		spin_unlock_irqrestore(&vfe->output_lock, flags);
 		return -EINVAL;
@@ -289,7 +289,7 @@ static int vfe_get_output(struct vfe_line *line)
 	spin_lock_irqsave(&vfe->output_lock, flags);
 
 	output = &line->output;
-	if (output->state != VFE_OUTPUT_OFF) {
+	if (output->state > VFE_OUTPUT_RESERVED) {
 		dev_err(vfe->camss->dev, "Output is running\n");
 		goto error;
 	}
diff --git a/drivers/media/platform/qcom/camss/camss-vfe.c b/drivers/media/platform/qcom/camss/camss-vfe.c
index a26e4a5d87b6..e0832f3f4f25 100644
--- a/drivers/media/platform/qcom/camss/camss-vfe.c
+++ b/drivers/media/platform/qcom/camss/camss-vfe.c
@@ -740,6 +740,7 @@ static int vfe_set_stream(struct v4l2_subdev *sd, int enable)
 	int ret;
 
 	if (enable) {
+		line->output.state = VFE_OUTPUT_RESERVED;
 		ret = vfe->ops->vfe_enable(line);
 		if (ret < 0)
 			dev_err(vfe->camss->dev,
diff --git a/drivers/media/platform/qcom/camss/camss-video.c b/drivers/media/platform/qcom/camss/camss-video.c
index 41deda232e4a..898f32177b12 100644
--- a/drivers/media/platform/qcom/camss/camss-video.c
+++ b/drivers/media/platform/qcom/camss/camss-video.c
@@ -342,7 +342,9 @@ static struct v4l2_subdev *video_remote_subdev(struct camss_video *video,
 static int video_get_subdev_format(struct camss_video *video,
 				   struct v4l2_format *format)
 {
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *subdev;
 	u32 pad;
 	int ret;
@@ -351,8 +353,8 @@ static int video_get_subdev_format(struct camss_video *video,
 	if (subdev == NULL)
 		return -EPIPE;
 
+	memset(&fmt, 0, sizeof(fmt));
 	fmt.pad = pad;
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 
 	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &fmt);
 	if (ret)
@@ -493,9 +495,11 @@ static int video_start_streaming(struct vb2_queue *q, unsigned int count)
 	struct v4l2_subdev *subdev;
 	int ret;
 
-	ret = video_device_pipeline_start(vdev, &video->pipe);
-	if (ret < 0)
+	ret = video_device_pipeline_alloc_start(vdev);
+	if (ret < 0) {
+		dev_err(video->camss->dev, "Failed to start media pipeline: %d\n", ret);
 		goto flush_buffers;
+	}
 
 	ret = video_check_format(video);
 	if (ret < 0)
@@ -537,6 +541,7 @@ static void video_stop_streaming(struct vb2_queue *q)
 	struct media_entity *entity;
 	struct media_pad *pad;
 	struct v4l2_subdev *subdev;
+	int ret;
 
 	entity = &vdev->entity;
 	while (1) {
@@ -551,7 +556,18 @@ static void video_stop_streaming(struct vb2_queue *q)
 		entity = pad->entity;
 		subdev = media_entity_to_v4l2_subdev(entity);
 
-		v4l2_subdev_call(subdev, video, s_stream, 0);
+		ret = v4l2_subdev_call(subdev, video, s_stream, 0);
+
+		if (entity->use_count > 1) {
+			/* Don't stop if other instances of the pipeline are still running */
+			dev_dbg(video->camss->dev, "Video pipeline still used, don't stop streaming.\n");
+			return;
+		}
+
+		if (ret) {
+			dev_err(video->camss->dev, "Video pipeline stop failed: %d\n", ret);
+			return;
+		}
 	}
 
 	video_device_pipeline_stop(vdev);
diff --git a/drivers/media/platform/qcom/camss/camss.c b/drivers/media/platform/qcom/camss/camss.c
index 9cda284f1e71..1ef26aea3eae 100644
--- a/drivers/media/platform/qcom/camss/camss.c
+++ b/drivers/media/platform/qcom/camss/camss.c
@@ -1320,7 +1320,7 @@ static int camss_register_entities(struct camss *camss)
 					struct v4l2_subdev *vfe = &camss->vfe[k].line[j].subdev;
 
 					ret = media_create_pad_link(&csid->entity,
-								    MSM_CSID_PAD_SRC,
+								    MSM_CSID_PAD_FIRST_SRC + j,
 								    &vfe->entity,
 								    MSM_VFE_PAD_SINK,
 								    0);
@@ -1725,7 +1725,7 @@ void camss_delete(struct camss *camss)
  *
  * Always returns 0.
  */
-static int camss_remove(struct platform_device *pdev)
+static void camss_remove(struct platform_device *pdev)
 {
 	struct camss *camss = platform_get_drvdata(pdev);
 
@@ -1735,8 +1735,6 @@ static int camss_remove(struct platform_device *pdev)
 
 	if (atomic_read(&camss->ref_count) == 0)
 		camss_delete(camss);
-
-	return 0;
 }
 
 static const struct of_device_id camss_dt_match[] = {
@@ -1798,7 +1796,7 @@ static const struct dev_pm_ops camss_pm_ops = {
 
 static struct platform_driver qcom_camss_driver = {
 	.probe = camss_probe,
-	.remove = camss_remove,
+	.remove_new = camss_remove,
 	.driver = {
 		.name = "qcom-camss",
 		.of_match_table = camss_dt_match,
diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c
index 990a1519f968..2ae867cb4c48 100644
--- a/drivers/media/platform/qcom/venus/core.c
+++ b/drivers/media/platform/qcom/venus/core.c
@@ -416,7 +416,7 @@ err_core_put:
 	return ret;
 }
 
-static int venus_remove(struct platform_device *pdev)
+static void venus_remove(struct platform_device *pdev)
 {
 	struct venus_core *core = platform_get_drvdata(pdev);
 	const struct venus_pm_ops *pm_ops = core->pm_ops;
@@ -447,8 +447,6 @@ static int venus_remove(struct platform_device *pdev)
 	mutex_destroy(&core->pm_lock);
 	mutex_destroy(&core->lock);
 	venus_dbgfs_deinit(core);
-
-	return ret;
 }
 
 static void venus_core_shutdown(struct platform_device *pdev)
@@ -891,7 +889,7 @@ MODULE_DEVICE_TABLE(of, venus_dt_match);
 
 static struct platform_driver qcom_venus_driver = {
 	.probe = venus_probe,
-	.remove = venus_remove,
+	.remove_new = venus_remove,
 	.driver = {
 		.name = "qcom-venus",
 		.of_match_table = venus_dt_match,
diff --git a/drivers/media/platform/qcom/venus/core.h b/drivers/media/platform/qcom/venus/core.h
index 32551c2602a9..4f81669986ba 100644
--- a/drivers/media/platform/qcom/venus/core.h
+++ b/drivers/media/platform/qcom/venus/core.h
@@ -107,7 +107,6 @@ struct venus_format {
  * @vcodec1_clks: an array of vcodec1 struct clk pointers
  * @video_path: an interconnect handle to video to/from memory path
  * @cpucfg_path: an interconnect handle to cpu configuration path
- * @opp_table: an device OPP table handle
  * @has_opp_table: does OPP table exist
  * @pmdomains:	an array of pmdomains struct device pointers
  * @opp_dl_venus: an device-link for device OPP
@@ -317,6 +316,14 @@ enum venus_dec_state {
 	VENUS_DEC_STATE_DRC		= 7,
 };
 
+enum venus_enc_state {
+	VENUS_ENC_STATE_DEINIT		= 0,
+	VENUS_ENC_STATE_INIT		= 1,
+	VENUS_ENC_STATE_ENCODING	= 2,
+	VENUS_ENC_STATE_STOPPED		= 3,
+	VENUS_ENC_STATE_DRAIN		= 4,
+};
+
 struct venus_ts_metadata {
 	bool used;
 	u64 ts_ns;
@@ -428,6 +435,7 @@ struct venus_inst {
 	u8 quantization;
 	u8 xfer_func;
 	enum venus_dec_state codec_state;
+	enum venus_enc_state enc_state;
 	wait_queue_head_t reconf_wait;
 	unsigned int subscriptions;
 	int buf_count;
diff --git a/drivers/media/platform/qcom/venus/helpers.c b/drivers/media/platform/qcom/venus/helpers.c
index ab6a29ffc81e..a2ceab7f9ddb 100644
--- a/drivers/media/platform/qcom/venus/helpers.c
+++ b/drivers/media/platform/qcom/venus/helpers.c
@@ -988,8 +988,8 @@ static u32 get_framesize_raw_p010(u32 width, u32 height)
 {
 	u32 y_plane, uv_plane, y_stride, uv_stride, y_sclines, uv_sclines;
 
-	y_stride = ALIGN(width * 2, 256);
-	uv_stride = ALIGN(width * 2, 256);
+	y_stride = ALIGN(width * 2, 128);
+	uv_stride = ALIGN(width * 2, 128);
 	y_sclines = ALIGN(height, 32);
 	uv_sclines = ALIGN((height + 1) >> 1, 16);
 	y_plane = y_stride * y_sclines;
diff --git a/drivers/media/platform/qcom/venus/hfi_cmds.c b/drivers/media/platform/qcom/venus/hfi_cmds.c
index 930b743f225e..bc3f8ff05840 100644
--- a/drivers/media/platform/qcom/venus/hfi_cmds.c
+++ b/drivers/media/platform/qcom/venus/hfi_cmds.c
@@ -1257,7 +1257,30 @@ pkt_session_set_property_4xx(struct hfi_session_set_property_pkt *pkt,
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*tm);
 		break;
 	}
+	case HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2: {
+		struct hfi_quantization_range_v2 *in = pdata, *range = prop_data;
+		u32 min_qp, max_qp;
+
+		min_qp = in->min_qp.qp_packed;
+		max_qp = in->max_qp.qp_packed;
 
+		/* We'll be packing in the qp, so make sure we
+		 * won't be losing data when masking
+		 */
+		if (min_qp > 0xff || max_qp > 0xff)
+			return -ERANGE;
+
+		range->min_qp.layer_id = 0xFF;
+		range->max_qp.layer_id = 0xFF;
+		range->min_qp.qp_packed = (min_qp & 0xFF) | ((min_qp & 0xFF) << 8) |
+			((min_qp & 0xFF) << 16);
+		range->max_qp.qp_packed = (max_qp & 0xFF) | ((max_qp & 0xFF) << 8) |
+			((max_qp & 0xFF) << 16);
+		range->min_qp.enable = 7;
+		range->max_qp.enable = 7;
+		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*range);
+		break;
+	}
 	case HFI_PROPERTY_CONFIG_VENC_MAX_BITRATE:
 	case HFI_PROPERTY_CONFIG_VDEC_POST_LOOP_DEBLOCKER:
 	case HFI_PROPERTY_PARAM_BUFFER_ALLOC_MODE:
diff --git a/drivers/media/platform/qcom/venus/hfi_helper.h b/drivers/media/platform/qcom/venus/hfi_helper.h
index d2d6719a2ba4..105792a68060 100644
--- a/drivers/media/platform/qcom/venus/hfi_helper.h
+++ b/drivers/media/platform/qcom/venus/hfi_helper.h
@@ -487,6 +487,11 @@
 #define HFI_PROPERTY_PARAM_VENC_SESSION_QP			0x2005006
 #define HFI_PROPERTY_PARAM_VENC_MPEG4_AC_PREDICTION		0x2005007
 #define HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE		0x2005008
+/*
+ * Note: HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2 is
+ * specific to HFI_VERSION_6XX and HFI_VERSION_4XX only
+ */
+#define HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2		0x2005009
 #define HFI_PROPERTY_PARAM_VENC_MPEG4_TIME_RESOLUTION		0x2005009
 #define HFI_PROPERTY_PARAM_VENC_MPEG4_SHORT_HEADER		0x200500a
 #define HFI_PROPERTY_PARAM_VENC_MPEG4_HEADER_EXTENSION		0x200500b
@@ -827,6 +832,19 @@ struct hfi_quantization_range {
 	u32 layer_id;
 };
 
+struct hfi_quantization_v2 {
+	u32 qp_packed;
+	u32 layer_id;
+	u32 enable;
+	u32 reserved[3];
+};
+
+struct hfi_quantization_range_v2 {
+	struct hfi_quantization_v2 min_qp;
+	struct hfi_quantization_v2 max_qp;
+	u32 reserved[4];
+};
+
 #define HFI_LTR_MODE_DISABLE	0x0
 #define HFI_LTR_MODE_MANUAL	0x1
 #define HFI_LTR_MODE_PERIODIC	0x2
diff --git a/drivers/media/platform/qcom/venus/hfi_plat_bufs_v6.c b/drivers/media/platform/qcom/venus/hfi_plat_bufs_v6.c
index ea25c451222b..a9be31ec6927 100644
--- a/drivers/media/platform/qcom/venus/hfi_plat_bufs_v6.c
+++ b/drivers/media/platform/qcom/venus/hfi_plat_bufs_v6.c
@@ -93,7 +93,7 @@
 #define LCU_MIN_SIZE_PELS		16
 #define SIZE_SEI_USERDATA		4096
 
-#define H265D_MAX_SLICE			600
+#define H265D_MAX_SLICE			3600
 #define SIZE_H265D_HW_PIC_T		SIZE_H264D_HW_PIC_T
 #define SIZE_H265D_BSE_CMD_PER_BUF	(16 * sizeof(u32))
 #define SIZE_H265D_VPP_CMD_PER_BUF	256
@@ -1021,7 +1021,7 @@ static u32 h264d_persist1_size(void)
 static u32 h265d_persist1_size(void)
 {
 	return ALIGN((SIZE_SLIST_BUF_H265 * NUM_SLIST_BUF_H265 + H265_NUM_TILE
-			* sizeof(u32)), HFI_DMA_ALIGNMENT);
+			* sizeof(u32) + NUM_HW_PIC_BUF * SIZE_SEI_USERDATA), HFI_DMA_ALIGNMENT);
 }
 
 static u32 vp8d_persist1_size(void)
diff --git a/drivers/media/platform/qcom/venus/vdec.c b/drivers/media/platform/qcom/venus/vdec.c
index 4ceaba37e2e5..51a53bf82bd3 100644
--- a/drivers/media/platform/qcom/venus/vdec.c
+++ b/drivers/media/platform/qcom/venus/vdec.c
@@ -31,15 +31,15 @@
  */
 static const struct venus_format vdec_formats[] = {
 	{
-		.pixfmt = V4L2_PIX_FMT_QC08C,
+		.pixfmt = V4L2_PIX_FMT_NV12,
 		.num_planes = 1,
 		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
 	}, {
-		.pixfmt = V4L2_PIX_FMT_QC10C,
+		.pixfmt = V4L2_PIX_FMT_QC08C,
 		.num_planes = 1,
 		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
-	},{
-		.pixfmt = V4L2_PIX_FMT_NV12,
+	}, {
+		.pixfmt = V4L2_PIX_FMT_QC10C,
 		.num_planes = 1,
 		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
 	}, {
@@ -204,8 +204,13 @@ vdec_try_fmt_common(struct venus_inst *inst, struct v4l2_format *f)
 					   pixmp->height);
 
 	if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
+		unsigned int stride = pixmp->width;
+
+		if (pixmp->pixelformat == V4L2_PIX_FMT_P010)
+			stride *= 2;
+
 		pfmt[0].sizeimage = szimage;
-		pfmt[0].bytesperline = ALIGN(pixmp->width, 128);
+		pfmt[0].bytesperline = ALIGN(stride, 128);
 	} else {
 		pfmt[0].sizeimage = clamp_t(u32, pfmt[0].sizeimage, 0, SZ_8M);
 		pfmt[0].sizeimage = max(pfmt[0].sizeimage, szimage);
@@ -526,6 +531,7 @@ static int
 vdec_decoder_cmd(struct file *file, void *fh, struct v4l2_decoder_cmd *cmd)
 {
 	struct venus_inst *inst = to_inst(file);
+	struct vb2_queue *dst_vq;
 	struct hfi_frame_data fdata = {0};
 	int ret;
 
@@ -556,6 +562,13 @@ vdec_decoder_cmd(struct file *file, void *fh, struct v4l2_decoder_cmd *cmd)
 			inst->codec_state = VENUS_DEC_STATE_DRAIN;
 			inst->drain_active = true;
 		}
+	} else if (cmd->cmd == V4L2_DEC_CMD_START &&
+		   inst->codec_state == VENUS_DEC_STATE_STOPPED) {
+		dst_vq = v4l2_m2m_get_vq(inst->fh.m2m_ctx,
+					 V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
+		vb2_clear_last_buffer_dequeued(dst_vq);
+
+		inst->codec_state = VENUS_DEC_STATE_DECODING;
 	}
 
 unlock:
@@ -1774,7 +1787,7 @@ err_vdev_release:
 	return ret;
 }
 
-static int vdec_remove(struct platform_device *pdev)
+static void vdec_remove(struct platform_device *pdev)
 {
 	struct venus_core *core = dev_get_drvdata(pdev->dev.parent);
 
@@ -1783,8 +1796,6 @@ static int vdec_remove(struct platform_device *pdev)
 
 	if (core->pm_ops->vdec_put)
 		core->pm_ops->vdec_put(core->dev_dec);
-
-	return 0;
 }
 
 static __maybe_unused int vdec_runtime_suspend(struct device *dev)
@@ -1825,7 +1836,7 @@ MODULE_DEVICE_TABLE(of, vdec_dt_match);
 
 static struct platform_driver qcom_venus_dec_driver = {
 	.probe = vdec_probe,
-	.remove = vdec_remove,
+	.remove_new = vdec_remove,
 	.driver = {
 		.name = "qcom-venus-decoder",
 		.of_match_table = vdec_dt_match,
diff --git a/drivers/media/platform/qcom/venus/venc.c b/drivers/media/platform/qcom/venus/venc.c
index cdb12546c4fa..4666f42feea3 100644
--- a/drivers/media/platform/qcom/venus/venc.c
+++ b/drivers/media/platform/qcom/venus/venc.c
@@ -520,6 +520,51 @@ static int venc_subscribe_event(struct v4l2_fh *fh,
 	}
 }
 
+static int
+venc_encoder_cmd(struct file *file, void *fh, struct v4l2_encoder_cmd *cmd)
+{
+	struct venus_inst *inst = to_inst(file);
+	struct hfi_frame_data fdata = {0};
+	int ret = 0;
+
+	ret = v4l2_m2m_ioctl_try_encoder_cmd(file, fh, cmd);
+	if (ret)
+		return ret;
+
+	mutex_lock(&inst->lock);
+
+	if (cmd->cmd == V4L2_ENC_CMD_STOP &&
+	    inst->enc_state == VENUS_ENC_STATE_ENCODING) {
+		/*
+		 * Implement V4L2_ENC_CMD_STOP by enqueue an empty buffer on
+		 * encoder input to signal EOS.
+		 */
+		if (!(inst->streamon_out && inst->streamon_cap))
+			goto unlock;
+
+		fdata.buffer_type = HFI_BUFFER_INPUT;
+		fdata.flags |= HFI_BUFFERFLAG_EOS;
+		fdata.device_addr = 0xdeadb000;
+
+		ret = hfi_session_process_buf(inst, &fdata);
+
+		inst->enc_state = VENUS_ENC_STATE_DRAIN;
+	} else if (cmd->cmd == V4L2_ENC_CMD_START) {
+		if (inst->enc_state == VENUS_ENC_STATE_DRAIN) {
+			ret = -EBUSY;
+			goto unlock;
+		}
+		if (inst->enc_state == VENUS_ENC_STATE_STOPPED) {
+			vb2_clear_last_buffer_dequeued(&inst->fh.m2m_ctx->cap_q_ctx.q);
+			inst->enc_state = VENUS_ENC_STATE_ENCODING;
+		}
+	}
+
+unlock:
+	mutex_unlock(&inst->lock);
+	return ret;
+}
+
 static const struct v4l2_ioctl_ops venc_ioctl_ops = {
 	.vidioc_querycap = venc_querycap,
 	.vidioc_enum_fmt_vid_cap = venc_enum_fmt,
@@ -548,6 +593,7 @@ static const struct v4l2_ioctl_ops venc_ioctl_ops = {
 	.vidioc_subscribe_event = venc_subscribe_event,
 	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
 	.vidioc_try_encoder_cmd = v4l2_m2m_ioctl_try_encoder_cmd,
+	.vidioc_encoder_cmd = venc_encoder_cmd,
 };
 
 static int venc_pm_get(struct venus_inst *inst)
@@ -617,6 +663,7 @@ static int venc_set_properties(struct venus_inst *inst)
 	struct hfi_idr_period idrp;
 	struct hfi_quantization quant;
 	struct hfi_quantization_range quant_range;
+	struct hfi_quantization_range_v2 quant_range_v2;
 	struct hfi_enable en;
 	struct hfi_ltr_mode ltr_mode;
 	struct hfi_intra_refresh intra_refresh = {};
@@ -825,16 +872,40 @@ static int venc_set_properties(struct venus_inst *inst)
 	if (ret)
 		return ret;
 
-	ptype = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE;
-	if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_HEVC) {
-		quant_range.min_qp = ctr->hevc_min_qp;
-		quant_range.max_qp = ctr->hevc_max_qp;
+	if (inst->core->res->hfi_version == HFI_VERSION_4XX ||
+	    inst->core->res->hfi_version == HFI_VERSION_6XX) {
+		ptype = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2;
+
+		if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_HEVC) {
+			quant_range_v2.min_qp.qp_packed = ctr->hevc_min_qp;
+			quant_range_v2.max_qp.qp_packed = ctr->hevc_max_qp;
+		} else if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_VP8) {
+			quant_range_v2.min_qp.qp_packed = ctr->vp8_min_qp;
+			quant_range_v2.max_qp.qp_packed = ctr->vp8_max_qp;
+		} else {
+			quant_range_v2.min_qp.qp_packed = ctr->h264_min_qp;
+			quant_range_v2.max_qp.qp_packed = ctr->h264_max_qp;
+		}
+
+		ret = hfi_session_set_property(inst, ptype, &quant_range_v2);
 	} else {
-		quant_range.min_qp = ctr->h264_min_qp;
-		quant_range.max_qp = ctr->h264_max_qp;
+		ptype = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE;
+
+		if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_HEVC) {
+			quant_range.min_qp = ctr->hevc_min_qp;
+			quant_range.max_qp = ctr->hevc_max_qp;
+		} else if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_VP8) {
+			quant_range.min_qp = ctr->vp8_min_qp;
+			quant_range.max_qp = ctr->vp8_max_qp;
+		} else {
+			quant_range.min_qp = ctr->h264_min_qp;
+			quant_range.max_qp = ctr->h264_max_qp;
+		}
+
+		quant_range.layer_id = 0;
+		ret = hfi_session_set_property(inst, ptype, &quant_range);
 	}
-	quant_range.layer_id = 0;
-	ret = hfi_session_set_property(inst, ptype, &quant_range);
+
 	if (ret)
 		return ret;
 
@@ -1196,6 +1267,8 @@ static int venc_start_streaming(struct vb2_queue *q, unsigned int count)
 	if (ret)
 		goto error;
 
+	inst->enc_state = VENUS_ENC_STATE_ENCODING;
+
 	mutex_unlock(&inst->lock);
 
 	return 0;
@@ -1215,10 +1288,21 @@ error:
 static void venc_vb2_buf_queue(struct vb2_buffer *vb)
 {
 	struct venus_inst *inst = vb2_get_drv_priv(vb->vb2_queue);
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
 
 	venc_pm_get_put(inst);
 
 	mutex_lock(&inst->lock);
+
+	if (inst->enc_state == VENUS_ENC_STATE_STOPPED) {
+		vbuf->sequence = inst->sequence_cap++;
+		vbuf->field = V4L2_FIELD_NONE;
+		vb2_set_plane_payload(vb, 0, 0);
+		v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_DONE);
+		mutex_unlock(&inst->lock);
+		return;
+	}
+
 	venus_helper_vb2_buf_queue(vb);
 	mutex_unlock(&inst->lock);
 }
@@ -1260,6 +1344,10 @@ static void venc_buf_done(struct venus_inst *inst, unsigned int buf_type,
 		vb->planes[0].data_offset = data_offset;
 		vb->timestamp = timestamp_us * NSEC_PER_USEC;
 		vbuf->sequence = inst->sequence_cap++;
+		if ((vbuf->flags & V4L2_BUF_FLAG_LAST) &&
+		    inst->enc_state == VENUS_ENC_STATE_DRAIN) {
+			inst->enc_state = VENUS_ENC_STATE_STOPPED;
+		}
 	} else {
 		vbuf->sequence = inst->sequence_out++;
 	}
@@ -1362,6 +1450,9 @@ static int venc_open(struct file *file)
 	inst->core_acquired = false;
 	inst->nonblock = file->f_flags & O_NONBLOCK;
 
+	if (inst->enc_state == VENUS_ENC_STATE_DEINIT)
+		inst->enc_state = VENUS_ENC_STATE_INIT;
+
 	venus_helper_init_instance(inst);
 
 	ret = venc_ctrl_init(inst);
@@ -1424,6 +1515,8 @@ static int venc_close(struct file *file)
 	v4l2_fh_del(&inst->fh);
 	v4l2_fh_exit(&inst->fh);
 
+	inst->enc_state = VENUS_ENC_STATE_DEINIT;
+
 	venc_pm_put(inst, false);
 
 	kfree(inst);
@@ -1492,7 +1585,7 @@ err_vdev_release:
 	return ret;
 }
 
-static int venc_remove(struct platform_device *pdev)
+static void venc_remove(struct platform_device *pdev)
 {
 	struct venus_core *core = dev_get_drvdata(pdev->dev.parent);
 
@@ -1501,8 +1594,6 @@ static int venc_remove(struct platform_device *pdev)
 
 	if (core->pm_ops->venc_put)
 		core->pm_ops->venc_put(core->dev_enc);
-
-	return 0;
 }
 
 static __maybe_unused int venc_runtime_suspend(struct device *dev)
@@ -1543,7 +1634,7 @@ MODULE_DEVICE_TABLE(of, venc_dt_match);
 
 static struct platform_driver qcom_venus_enc_driver = {
 	.probe = venc_probe,
-	.remove = venc_remove,
+	.remove_new = venc_remove,
 	.driver = {
 		.name = "qcom-venus-encoder",
 		.of_match_table = venc_dt_match,
diff --git a/drivers/media/platform/renesas/rcar-fcp.c b/drivers/media/platform/renesas/rcar-fcp.c
index eb59a3ba6d0f..bcef7b87da7c 100644
--- a/drivers/media/platform/renesas/rcar-fcp.c
+++ b/drivers/media/platform/renesas/rcar-fcp.c
@@ -144,7 +144,7 @@ static int rcar_fcp_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int rcar_fcp_remove(struct platform_device *pdev)
+static void rcar_fcp_remove(struct platform_device *pdev)
 {
 	struct rcar_fcp_device *fcp = platform_get_drvdata(pdev);
 
@@ -153,8 +153,6 @@ static int rcar_fcp_remove(struct platform_device *pdev)
 	mutex_unlock(&fcp_lock);
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static const struct of_device_id rcar_fcp_of_match[] = {
@@ -166,7 +164,7 @@ MODULE_DEVICE_TABLE(of, rcar_fcp_of_match);
 
 static struct platform_driver rcar_fcp_platform_driver = {
 	.probe		= rcar_fcp_probe,
-	.remove		= rcar_fcp_remove,
+	.remove_new	= rcar_fcp_remove,
 	.driver		= {
 		.name	= "rcar-fcp",
 		.of_match_table = rcar_fcp_of_match,
diff --git a/drivers/media/platform/renesas/rcar-isp.c b/drivers/media/platform/renesas/rcar-isp.c
index 10b3474f93a4..f666b621338d 100644
--- a/drivers/media/platform/renesas/rcar-isp.c
+++ b/drivers/media/platform/renesas/rcar-isp.c
@@ -419,10 +419,7 @@ static const struct media_entity_operations risp_entity_ops = {
 static int risp_probe_resources(struct rcar_isp *isp,
 				struct platform_device *pdev)
 {
-	struct resource *res;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	isp->base = devm_ioremap_resource(&pdev->dev, res);
+	isp->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
 	if (IS_ERR(isp->base))
 		return PTR_ERR(isp->base);
 
@@ -503,7 +500,7 @@ error_mutex:
 	return ret;
 }
 
-static int risp_remove(struct platform_device *pdev)
+static void risp_remove(struct platform_device *pdev)
 {
 	struct rcar_isp *isp = platform_get_drvdata(pdev);
 
@@ -515,8 +512,6 @@ static int risp_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 
 	mutex_destroy(&isp->lock);
-
-	return 0;
 }
 
 static struct platform_driver rcar_isp_driver = {
@@ -525,7 +520,7 @@ static struct platform_driver rcar_isp_driver = {
 		.of_match_table = risp_of_id_table,
 	},
 	.probe = risp_probe,
-	.remove = risp_remove,
+	.remove_new = risp_remove,
 };
 
 module_platform_driver(rcar_isp_driver);
diff --git a/drivers/media/platform/renesas/rcar-vin/rcar-core.c b/drivers/media/platform/renesas/rcar-vin/rcar-core.c
index 5e53d6b7036c..ff4bde9cc0e3 100644
--- a/drivers/media/platform/renesas/rcar-vin/rcar-core.c
+++ b/drivers/media/platform/renesas/rcar-vin/rcar-core.c
@@ -17,7 +17,6 @@
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
-#include <linux/sys_soc.h>
 
 #include <media/v4l2-async.h>
 #include <media/v4l2-fwnode.h>
@@ -1183,24 +1182,6 @@ static const struct rvin_info rcar_info_r8a7795 = {
 	.scaler = rvin_scaler_gen3,
 };
 
-static const struct rvin_group_route rcar_info_r8a7795es1_routes[] = {
-	{ .master = 0, .csi = RVIN_CSI20, .chsel = 0x04 },
-	{ .master = 0, .csi = RVIN_CSI21, .chsel = 0x05 },
-	{ .master = 0, .csi = RVIN_CSI40, .chsel = 0x03 },
-	{ .master = 4, .csi = RVIN_CSI20, .chsel = 0x04 },
-	{ .master = 4, .csi = RVIN_CSI21, .chsel = 0x05 },
-	{ .master = 4, .csi = RVIN_CSI41, .chsel = 0x03 },
-	{ /* Sentinel */ }
-};
-
-static const struct rvin_info rcar_info_r8a7795es1 = {
-	.model = RCAR_GEN3,
-	.use_mc = true,
-	.max_width = 4096,
-	.max_height = 4096,
-	.routes = rcar_info_r8a7795es1_routes,
-};
-
 static const struct rvin_group_route rcar_info_r8a7796_routes[] = {
 	{ .master = 0, .csi = RVIN_CSI20, .chsel = 0x04 },
 	{ .master = 0, .csi = RVIN_CSI40, .chsel = 0x03 },
@@ -1372,17 +1353,8 @@ static const struct of_device_id rvin_of_id_table[] = {
 };
 MODULE_DEVICE_TABLE(of, rvin_of_id_table);
 
-static const struct soc_device_attribute r8a7795es1[] = {
-	{
-		.soc_id = "r8a7795", .revision = "ES1.*",
-		.data = &rcar_info_r8a7795es1,
-	},
-	{ /* Sentinel */ }
-};
-
 static int rcar_vin_probe(struct platform_device *pdev)
 {
-	const struct soc_device_attribute *attr;
 	struct rvin_dev *vin;
 	int irq, ret;
 
@@ -1394,14 +1366,6 @@ static int rcar_vin_probe(struct platform_device *pdev)
 	vin->info = of_device_get_match_data(&pdev->dev);
 	vin->alpha = 0xff;
 
-	/*
-	 * Special care is needed on r8a7795 ES1.x since it
-	 * uses different routing than r8a7795 ES2.0.
-	 */
-	attr = soc_device_match(r8a7795es1);
-	if (attr)
-		vin->info = attr->data;
-
 	vin->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(vin->base))
 		return PTR_ERR(vin->base);
@@ -1442,7 +1406,7 @@ static int rcar_vin_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int rcar_vin_remove(struct platform_device *pdev)
+static void rcar_vin_remove(struct platform_device *pdev)
 {
 	struct rvin_dev *vin = platform_get_drvdata(pdev);
 
@@ -1458,8 +1422,6 @@ static int rcar_vin_remove(struct platform_device *pdev)
 		rvin_parallel_cleanup(vin);
 
 	rvin_dma_unregister(vin);
-
-	return 0;
 }
 
 static SIMPLE_DEV_PM_OPS(rvin_pm_ops, rvin_suspend, rvin_resume);
@@ -1472,7 +1434,7 @@ static struct platform_driver rcar_vin_driver = {
 		.of_match_table = rvin_of_id_table,
 	},
 	.probe = rcar_vin_probe,
-	.remove = rcar_vin_remove,
+	.remove_new = rcar_vin_remove,
 };
 
 module_platform_driver(rcar_vin_driver);
diff --git a/drivers/media/platform/renesas/rcar-vin/rcar-csi2.c b/drivers/media/platform/renesas/rcar-vin/rcar-csi2.c
index 174aa6176f54..e34060c2b039 100644
--- a/drivers/media/platform/renesas/rcar-vin/rcar-csi2.c
+++ b/drivers/media/platform/renesas/rcar-vin/rcar-csi2.c
@@ -345,7 +345,7 @@ static const struct rcsi2_mbps_reg hsfreqrange_h3_v3h_m3n[] = {
 	{ /* sentinel */ },
 };
 
-static const struct rcsi2_mbps_reg hsfreqrange_m3w_h3es1[] = {
+static const struct rcsi2_mbps_reg hsfreqrange_m3w[] = {
 	{ .mbps =   80,	.reg = 0x00 },
 	{ .mbps =   90,	.reg = 0x10 },
 	{ .mbps =  100,	.reg = 0x20 },
@@ -1369,11 +1369,6 @@ static const struct rcar_csi2_info rcar_csi2_info_r8a7795 = {
 	.clear_ulps = true,
 };
 
-static const struct rcar_csi2_info rcar_csi2_info_r8a7795es1 = {
-	.hsfreqrange = hsfreqrange_m3w_h3es1,
-	.num_channels = 4,
-};
-
 static const struct rcar_csi2_info rcar_csi2_info_r8a7795es2 = {
 	.init_phtw = rcsi2_init_phtw_h3es2,
 	.hsfreqrange = hsfreqrange_h3_v3h_m3n,
@@ -1383,12 +1378,12 @@ static const struct rcar_csi2_info rcar_csi2_info_r8a7795es2 = {
 };
 
 static const struct rcar_csi2_info rcar_csi2_info_r8a7796 = {
-	.hsfreqrange = hsfreqrange_m3w_h3es1,
+	.hsfreqrange = hsfreqrange_m3w,
 	.num_channels = 4,
 };
 
 static const struct rcar_csi2_info rcar_csi2_info_r8a77961 = {
-	.hsfreqrange = hsfreqrange_m3w_h3es1,
+	.hsfreqrange = hsfreqrange_m3w,
 	.num_channels = 4,
 };
 
@@ -1482,10 +1477,6 @@ MODULE_DEVICE_TABLE(of, rcar_csi2_of_table);
 
 static const struct soc_device_attribute r8a7795[] = {
 	{
-		.soc_id = "r8a7795", .revision = "ES1.*",
-		.data = &rcar_csi2_info_r8a7795es1,
-	},
-	{
 		.soc_id = "r8a7795", .revision = "ES2.*",
 		.data = &rcar_csi2_info_r8a7795es2,
 	},
@@ -1574,7 +1565,7 @@ error_mutex:
 	return ret;
 }
 
-static int rcsi2_remove(struct platform_device *pdev)
+static void rcsi2_remove(struct platform_device *pdev)
 {
 	struct rcar_csi2 *priv = platform_get_drvdata(pdev);
 
@@ -1585,12 +1576,10 @@ static int rcsi2_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 
 	mutex_destroy(&priv->lock);
-
-	return 0;
 }
 
 static struct platform_driver rcar_csi2_pdrv = {
-	.remove	= rcsi2_remove,
+	.remove_new = rcsi2_remove,
 	.probe	= rcsi2_probe,
 	.driver	= {
 		.name	= "rcar-csi2",
diff --git a/drivers/media/platform/renesas/rcar_drif.c b/drivers/media/platform/renesas/rcar_drif.c
index 3fec41f6e964..3a92f4535c18 100644
--- a/drivers/media/platform/renesas/rcar_drif.c
+++ b/drivers/media/platform/renesas/rcar_drif.c
@@ -1433,19 +1433,17 @@ static int rcar_drif_probe(struct platform_device *pdev)
 }
 
 /* DRIF channel remove */
-static int rcar_drif_remove(struct platform_device *pdev)
+static void rcar_drif_remove(struct platform_device *pdev)
 {
 	struct rcar_drif *ch = platform_get_drvdata(pdev);
 	struct rcar_drif_sdr *sdr = ch->sdr;
 
 	/* Channel 0 will be the SDR instance */
 	if (ch->num)
-		return 0;
+		return;
 
 	/* SDR instance */
 	rcar_drif_sdr_remove(sdr);
-
-	return 0;
 }
 
 /* FIXME: Implement suspend/resume support */
@@ -1476,7 +1474,7 @@ static struct platform_driver rcar_drif_driver = {
 		.pm = &rcar_drif_pm_ops,
 		},
 	.probe = rcar_drif_probe,
-	.remove = rcar_drif_remove,
+	.remove_new = rcar_drif_remove,
 };
 
 module_platform_driver(rcar_drif_driver);
diff --git a/drivers/media/platform/renesas/rcar_fdp1.c b/drivers/media/platform/renesas/rcar_fdp1.c
index 37ecf489d112..f43e458590b8 100644
--- a/drivers/media/platform/renesas/rcar_fdp1.c
+++ b/drivers/media/platform/renesas/rcar_fdp1.c
@@ -254,7 +254,6 @@ MODULE_PARM_DESC(debug, "activate debug info");
 
 /* Internal Data (HW Version) */
 #define FD1_IP_INTDATA			0x0800
-#define FD1_IP_H3_ES1			0x02010101
 #define FD1_IP_M3W			0x02010202
 #define FD1_IP_H3			0x02010203
 #define FD1_IP_M3N			0x02010204
@@ -2313,8 +2312,10 @@ static int fdp1_probe(struct platform_device *pdev)
 
 	/* Determine our clock rate */
 	clk = clk_get(&pdev->dev, NULL);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	if (IS_ERR(clk)) {
+		ret = PTR_ERR(clk);
+		goto put_dev;
+	}
 
 	fdp1->clk_rate = clk_get_rate(clk);
 	clk_put(clk);
@@ -2323,7 +2324,7 @@ static int fdp1_probe(struct platform_device *pdev)
 	ret = v4l2_device_register(&pdev->dev, &fdp1->v4l2_dev);
 	if (ret) {
 		v4l2_err(&fdp1->v4l2_dev, "Failed to register video device\n");
-		return ret;
+		goto put_dev;
 	}
 
 	/* M2M registration */
@@ -2359,9 +2360,6 @@ static int fdp1_probe(struct platform_device *pdev)
 
 	hw_version = fdp1_read(fdp1, FD1_IP_INTDATA);
 	switch (hw_version) {
-	case FD1_IP_H3_ES1:
-		dprintk(fdp1, "FDP1 Version R-Car H3 ES1\n");
-		break;
 	case FD1_IP_M3W:
 		dprintk(fdp1, "FDP1 Version R-Car M3-W\n");
 		break;
@@ -2393,10 +2391,12 @@ release_m2m:
 unreg_dev:
 	v4l2_device_unregister(&fdp1->v4l2_dev);
 
+put_dev:
+	rcar_fcp_put(fdp1->fcp);
 	return ret;
 }
 
-static int fdp1_remove(struct platform_device *pdev)
+static void fdp1_remove(struct platform_device *pdev)
 {
 	struct fdp1_dev *fdp1 = platform_get_drvdata(pdev);
 
@@ -2404,8 +2404,7 @@ static int fdp1_remove(struct platform_device *pdev)
 	video_unregister_device(&fdp1->vfd);
 	v4l2_device_unregister(&fdp1->v4l2_dev);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
+	rcar_fcp_put(fdp1->fcp);
 }
 
 static int __maybe_unused fdp1_pm_runtime_suspend(struct device *dev)
@@ -2441,7 +2440,7 @@ MODULE_DEVICE_TABLE(of, fdp1_dt_ids);
 
 static struct platform_driver fdp1_pdrv = {
 	.probe		= fdp1_probe,
-	.remove		= fdp1_remove,
+	.remove_new	= fdp1_remove,
 	.driver		= {
 		.name	= DRIVER_NAME,
 		.of_match_table = fdp1_dt_ids,
diff --git a/drivers/media/platform/renesas/rcar_jpu.c b/drivers/media/platform/renesas/rcar_jpu.c
index 2f4377cfbb42..e7f604807825 100644
--- a/drivers/media/platform/renesas/rcar_jpu.c
+++ b/drivers/media/platform/renesas/rcar_jpu.c
@@ -1702,7 +1702,7 @@ device_register_rollback:
 	return ret;
 }
 
-static int jpu_remove(struct platform_device *pdev)
+static void jpu_remove(struct platform_device *pdev)
 {
 	struct jpu *jpu = platform_get_drvdata(pdev);
 
@@ -1710,8 +1710,6 @@ static int jpu_remove(struct platform_device *pdev)
 	video_unregister_device(&jpu->vfd_encoder);
 	v4l2_m2m_release(jpu->m2m_dev);
 	v4l2_device_unregister(&jpu->v4l2_dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1746,7 +1744,7 @@ static const struct dev_pm_ops jpu_pm_ops = {
 
 static struct platform_driver jpu_driver = {
 	.probe = jpu_probe,
-	.remove = jpu_remove,
+	.remove_new = jpu_remove,
 	.driver = {
 		.of_match_table = jpu_dt_ids,
 		.name = DRV_NAME,
diff --git a/drivers/media/platform/renesas/renesas-ceu.c b/drivers/media/platform/renesas/renesas-ceu.c
index f70f91b006b7..56b9c59cfda8 100644
--- a/drivers/media/platform/renesas/renesas-ceu.c
+++ b/drivers/media/platform/renesas/renesas-ceu.c
@@ -795,8 +795,8 @@ static int __ceu_try_fmt(struct ceu_device *ceudev, struct v4l2_format *v4l2_fmt
 	struct v4l2_subdev *v4l2_sd = ceu_sd->v4l2_sd;
 	struct v4l2_subdev_pad_config pad_cfg;
 	struct v4l2_subdev_state pad_state = {
-		.pads = &pad_cfg
-		};
+		.pads = &pad_cfg,
+	};
 	const struct ceu_fmt *ceu_fmt;
 	u32 mbus_code_old;
 	u32 mbus_code;
@@ -1709,7 +1709,7 @@ error_free_ceudev:
 	return ret;
 }
 
-static int ceu_remove(struct platform_device *pdev)
+static void ceu_remove(struct platform_device *pdev)
 {
 	struct ceu_device *ceudev = platform_get_drvdata(pdev);
 
@@ -1722,8 +1722,6 @@ static int ceu_remove(struct platform_device *pdev)
 	v4l2_device_unregister(&ceudev->v4l2_dev);
 
 	video_unregister_device(&ceudev->vdev);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ceu_pm_ops = {
@@ -1739,7 +1737,7 @@ static struct platform_driver ceu_driver = {
 		.of_match_table = of_match_ptr(ceu_of_match),
 	},
 	.probe		= ceu_probe,
-	.remove		= ceu_remove,
+	.remove_new	= ceu_remove,
 };
 
 module_platform_driver(ceu_driver);
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-core.c b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-core.c
index 5939f5165a5e..7a71370fcc32 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-core.c
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-core.c
@@ -298,7 +298,7 @@ error_dma_unregister:
 	return ret;
 }
 
-static int rzg2l_cru_remove(struct platform_device *pdev)
+static void rzg2l_cru_remove(struct platform_device *pdev)
 {
 	struct rzg2l_cru_dev *cru = platform_get_drvdata(pdev);
 
@@ -312,8 +312,6 @@ static int rzg2l_cru_remove(struct platform_device *pdev)
 	mutex_destroy(&cru->mdev_lock);
 
 	rzg2l_cru_dma_unregister(cru);
-
-	return 0;
 }
 
 static const struct of_device_id rzg2l_cru_of_id_table[] = {
@@ -328,7 +326,7 @@ static struct platform_driver rzg2l_cru_driver = {
 		.of_match_table = rzg2l_cru_of_id_table,
 	},
 	.probe = rzg2l_cru_probe,
-	.remove = rzg2l_cru_remove,
+	.remove_new = rzg2l_cru_remove,
 };
 
 module_platform_driver(rzg2l_cru_driver);
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-csi2.c b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-csi2.c
index 384fb54e219a..30dad7383654 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-csi2.c
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-csi2.c
@@ -819,7 +819,7 @@ error_pm:
 	return ret;
 }
 
-static int rzg2l_csi2_remove(struct platform_device *pdev)
+static void rzg2l_csi2_remove(struct platform_device *pdev)
 {
 	struct rzg2l_csi2 *csi2 = platform_get_drvdata(pdev);
 
@@ -829,8 +829,6 @@ static int rzg2l_csi2_remove(struct platform_device *pdev)
 	v4l2_subdev_cleanup(&csi2->subdev);
 	media_entity_cleanup(&csi2->subdev.entity);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static int __maybe_unused rzg2l_csi2_pm_runtime_suspend(struct device *dev)
@@ -859,7 +857,7 @@ static const struct of_device_id rzg2l_csi2_of_table[] = {
 };
 
 static struct platform_driver rzg2l_csi2_pdrv = {
-	.remove	= rzg2l_csi2_remove,
+	.remove_new = rzg2l_csi2_remove,
 	.probe	= rzg2l_csi2_probe,
 	.driver	= {
 		.name = "rzg2l-csi2",
diff --git a/drivers/media/platform/renesas/sh_vou.c b/drivers/media/platform/renesas/sh_vou.c
index ca4310e26c49..8fe3272a541f 100644
--- a/drivers/media/platform/renesas/sh_vou.c
+++ b/drivers/media/platform/renesas/sh_vou.c
@@ -1343,7 +1343,7 @@ ei2cgadap:
 	return ret;
 }
 
-static int sh_vou_remove(struct platform_device *pdev)
+static void sh_vou_remove(struct platform_device *pdev)
 {
 	struct v4l2_device *v4l2_dev = platform_get_drvdata(pdev);
 	struct sh_vou_device *vou_dev = container_of(v4l2_dev,
@@ -1356,11 +1356,10 @@ static int sh_vou_remove(struct platform_device *pdev)
 	video_unregister_device(&vou_dev->vdev);
 	i2c_put_adapter(client->adapter);
 	v4l2_device_unregister(&vou_dev->v4l2_dev);
-	return 0;
 }
 
 static struct platform_driver sh_vou = {
-	.remove  = sh_vou_remove,
+	.remove_new = sh_vou_remove,
 	.driver  = {
 		.name	= "sh-vou",
 	},
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_drm.c b/drivers/media/platform/renesas/vsp1/vsp1_drm.c
index c6f25200982c..9b087bd8df7d 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_drm.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_drm.c
@@ -66,7 +66,9 @@ static int vsp1_du_insert_uif(struct vsp1_device *vsp1,
 			      struct vsp1_entity *prev, unsigned int prev_pad,
 			      struct vsp1_entity *next, unsigned int next_pad)
 {
-	struct v4l2_subdev_format format;
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	int ret;
 
 	if (!uif) {
@@ -82,8 +84,6 @@ static int vsp1_du_insert_uif(struct vsp1_device *vsp1,
 	prev->sink = uif;
 	prev->sink_pad = UIF_PAD_SINK;
 
-	memset(&format, 0, sizeof(format));
-	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	format.pad = prev_pad;
 
 	ret = v4l2_subdev_call(&prev->subdev, pad, get_fmt, NULL, &format);
@@ -118,8 +118,12 @@ static int vsp1_du_pipeline_setup_rpf(struct vsp1_device *vsp1,
 				      struct vsp1_entity *uif,
 				      unsigned int brx_input)
 {
-	struct v4l2_subdev_selection sel;
-	struct v4l2_subdev_format format;
+	struct v4l2_subdev_selection sel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	const struct v4l2_rect *crop;
 	int ret;
 
@@ -129,8 +133,6 @@ static int vsp1_du_pipeline_setup_rpf(struct vsp1_device *vsp1,
 	 */
 	crop = &vsp1->drm->inputs[rpf->entity.index].crop;
 
-	memset(&format, 0, sizeof(format));
-	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	format.pad = RWPF_PAD_SINK;
 	format.format.width = crop->width + crop->left;
 	format.format.height = crop->height + crop->top;
@@ -147,8 +149,6 @@ static int vsp1_du_pipeline_setup_rpf(struct vsp1_device *vsp1,
 		__func__, format.format.width, format.format.height,
 		format.format.code, rpf->entity.index);
 
-	memset(&sel, 0, sizeof(sel));
-	sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	sel.pad = RWPF_PAD_SINK;
 	sel.target = V4L2_SEL_TGT_CROP;
 	sel.r = *crop;
@@ -481,10 +481,11 @@ static int vsp1_du_pipeline_setup_output(struct vsp1_device *vsp1,
 					 struct vsp1_pipeline *pipe)
 {
 	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
-	struct v4l2_subdev_format format = { 0, };
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	int ret;
 
-	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	format.pad = RWPF_PAD_SINK;
 	format.format.width = drm_pipe->width;
 	format.format.height = drm_pipe->height;
@@ -710,6 +711,9 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 		return 0;
 	}
 
+	/* Reset the underrun counter */
+	pipe->underrun_count = 0;
+
 	drm_pipe->width = cfg->width;
 	drm_pipe->height = cfg->height;
 	pipe->interlaced = cfg->interlaced;
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_drv.c b/drivers/media/platform/renesas/vsp1/vsp1_drv.c
index 5710152d6511..a9db84be4822 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_drv.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_drv.c
@@ -45,7 +45,8 @@
 
 static irqreturn_t vsp1_irq_handler(int irq, void *data)
 {
-	u32 mask = VI6_WPF_IRQ_STA_DFE | VI6_WPF_IRQ_STA_FRE;
+	u32 mask = VI6_WPF_IRQ_STA_DFE | VI6_WPF_IRQ_STA_FRE |
+		   VI6_WPF_IRQ_STA_UND;
 	struct vsp1_device *vsp1 = data;
 	irqreturn_t ret = IRQ_NONE;
 	unsigned int i;
@@ -60,6 +61,14 @@ static irqreturn_t vsp1_irq_handler(int irq, void *data)
 		status = vsp1_read(vsp1, VI6_WPF_IRQ_STA(i));
 		vsp1_write(vsp1, VI6_WPF_IRQ_STA(i), ~status & mask);
 
+		if ((status & VI6_WPF_IRQ_STA_UND) && wpf->entity.pipe) {
+			wpf->entity.pipe->underrun_count++;
+
+			dev_warn_ratelimited(vsp1->dev,
+				"Underrun occurred at WPF%u (total underruns %u)\n",
+				i, wpf->entity.pipe->underrun_count);
+		}
+
 		if (status & VI6_WPF_IRQ_STA_DFE) {
 			vsp1_pipeline_frame_end(wpf->entity.pipe);
 			ret = IRQ_HANDLED;
@@ -977,7 +986,7 @@ done:
 	return ret;
 }
 
-static int vsp1_remove(struct platform_device *pdev)
+static void vsp1_remove(struct platform_device *pdev)
 {
 	struct vsp1_device *vsp1 = platform_get_drvdata(pdev);
 
@@ -985,8 +994,6 @@ static int vsp1_remove(struct platform_device *pdev)
 	rcar_fcp_put(vsp1->fcp);
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static const struct of_device_id vsp1_of_match[] = {
@@ -999,7 +1006,7 @@ MODULE_DEVICE_TABLE(of, vsp1_of_match);
 
 static struct platform_driver vsp1_platform_driver = {
 	.probe		= vsp1_probe,
-	.remove		= vsp1_remove,
+	.remove_new	= vsp1_remove,
 	.driver		= {
 		.name	= "vsp1",
 		.pm	= &vsp1_pm_ops,
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_entity.c b/drivers/media/platform/renesas/vsp1/vsp1_entity.c
index 4c3bd2b1ca28..c31f05a80bb5 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_entity.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_entity.c
@@ -184,15 +184,14 @@ vsp1_entity_get_pad_selection(struct vsp1_entity *entity,
 int vsp1_entity_init_cfg(struct v4l2_subdev *subdev,
 			 struct v4l2_subdev_state *sd_state)
 {
-	struct v4l2_subdev_format format;
 	unsigned int pad;
 
 	for (pad = 0; pad < subdev->entity.num_pads - 1; ++pad) {
-		memset(&format, 0, sizeof(format));
-
-		format.pad = pad;
-		format.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY
-			     : V4L2_SUBDEV_FORMAT_ACTIVE;
+		struct v4l2_subdev_format format = {
+			.pad = pad,
+			.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY
+			       : V4L2_SUBDEV_FORMAT_ACTIVE,
+		};
 
 		v4l2_subdev_call(subdev, pad, set_fmt, sd_state, &format);
 	}
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_entity.h b/drivers/media/platform/renesas/vsp1/vsp1_entity.h
index f22724439cdc..17f98a6a972e 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_entity.h
+++ b/drivers/media/platform/renesas/vsp1/vsp1_entity.h
@@ -130,8 +130,6 @@ int vsp1_entity_init(struct vsp1_device *vsp1, struct vsp1_entity *entity,
 		     const struct v4l2_subdev_ops *ops, u32 function);
 void vsp1_entity_destroy(struct vsp1_entity *entity);
 
-extern const struct v4l2_subdev_internal_ops vsp1_subdev_internal_ops;
-
 int vsp1_entity_link_setup(struct media_entity *entity,
 			   const struct media_pad *local,
 			   const struct media_pad *remote, u32 flags);
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_pipe.h b/drivers/media/platform/renesas/vsp1/vsp1_pipe.h
index ae646c9ef337..674b5748d929 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_pipe.h
+++ b/drivers/media/platform/renesas/vsp1/vsp1_pipe.h
@@ -148,6 +148,8 @@ struct vsp1_pipeline {
 	unsigned int partitions;
 	struct vsp1_partition *partition;
 	struct vsp1_partition *part_table;
+
+	u32 underrun_count;
 };
 
 void vsp1_pipeline_reset(struct vsp1_pipeline *pipe);
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_regs.h b/drivers/media/platform/renesas/vsp1/vsp1_regs.h
index d94343ae57a1..7eca82e0ba7e 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_regs.h
+++ b/drivers/media/platform/renesas/vsp1/vsp1_regs.h
@@ -32,10 +32,12 @@
 #define VI6_STATUS_SYS_ACT(n)		BIT((n) + 8)
 
 #define VI6_WPF_IRQ_ENB(n)		(0x0048 + (n) * 12)
+#define VI6_WPF_IRQ_ENB_UNDE		BIT(16)
 #define VI6_WPF_IRQ_ENB_DFEE		BIT(1)
 #define VI6_WPF_IRQ_ENB_FREE		BIT(0)
 
 #define VI6_WPF_IRQ_STA(n)		(0x004c + (n) * 12)
+#define VI6_WPF_IRQ_STA_UND		BIT(16)
 #define VI6_WPF_IRQ_STA_DFE		BIT(1)
 #define VI6_WPF_IRQ_STA_FRE		BIT(0)
 
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_video.c b/drivers/media/platform/renesas/vsp1/vsp1_video.c
index 544012fd1fe9..e9d5027761bb 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_video.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_video.c
@@ -62,7 +62,9 @@ vsp1_video_remote_subdev(struct media_pad *local, u32 *pad)
 
 static int vsp1_video_verify_format(struct vsp1_video *video)
 {
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *subdev;
 	int ret;
 
@@ -70,7 +72,6 @@ static int vsp1_video_verify_format(struct vsp1_video *video)
 	if (subdev == NULL)
 		return -EINVAL;
 
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &fmt);
 	if (ret < 0)
 		return ret == -ENOIOCTLCMD ? -EINVAL : ret;
@@ -776,7 +777,7 @@ static void vsp1_video_buffer_queue(struct vb2_buffer *vb)
 	video->rwpf->mem = buf->mem;
 	pipe->buffers_ready |= 1 << video->pipe_index;
 
-	if (vb2_is_streaming(&video->queue) &&
+	if (vb2_start_streaming_called(&video->queue) &&
 	    vsp1_pipeline_ready(pipe))
 		vsp1_video_pipeline_run(pipe);
 
diff --git a/drivers/media/platform/rockchip/rga/rga.c b/drivers/media/platform/rockchip/rga/rga.c
index 61b25fcf826e..67dcf22e5ba3 100644
--- a/drivers/media/platform/rockchip/rga/rga.c
+++ b/drivers/media/platform/rockchip/rga/rga.c
@@ -930,7 +930,7 @@ err_put_clk:
 	return ret;
 }
 
-static int rga_remove(struct platform_device *pdev)
+static void rga_remove(struct platform_device *pdev)
 {
 	struct rockchip_rga *rga = platform_get_drvdata(pdev);
 
@@ -947,8 +947,6 @@ static int rga_remove(struct platform_device *pdev)
 	v4l2_device_unregister(&rga->v4l2_dev);
 
 	pm_runtime_disable(rga->dev);
-
-	return 0;
 }
 
 static int __maybe_unused rga_runtime_suspend(struct device *dev)
@@ -986,7 +984,7 @@ MODULE_DEVICE_TABLE(of, rockchip_rga_match);
 
 static struct platform_driver rga_pdrv = {
 	.probe = rga_probe,
-	.remove = rga_remove,
+	.remove_new = rga_remove,
 	.driver = {
 		.name = RGA_NAME,
 		.pm = &rga_pm,
diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-capture.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-capture.c
index d1d1fdce03e3..8f3cba319762 100644
--- a/drivers/media/platform/rockchip/rkisp1/rkisp1-capture.c
+++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-capture.c
@@ -111,6 +111,16 @@ static const struct rkisp1_capture_fmt_cfg rkisp1_mp_fmts[] = {
 		.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA,
 		.mbus = MEDIA_BUS_FMT_YUYV8_2X8,
 	}, {
+		.fourcc = V4L2_PIX_FMT_NV16M,
+		.uv_swap = 0,
+		.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA,
+		.mbus = MEDIA_BUS_FMT_YUYV8_2X8,
+	}, {
+		.fourcc = V4L2_PIX_FMT_NV61M,
+		.uv_swap = 1,
+		.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA,
+		.mbus = MEDIA_BUS_FMT_YUYV8_2X8,
+	}, {
 		.fourcc = V4L2_PIX_FMT_YVU422M,
 		.uv_swap = 1,
 		.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
@@ -238,6 +248,18 @@ static const struct rkisp1_capture_fmt_cfg rkisp1_sp_fmts[] = {
 		.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422,
 		.mbus = MEDIA_BUS_FMT_YUYV8_2X8,
 	}, {
+		.fourcc = V4L2_PIX_FMT_NV16M,
+		.uv_swap = 0,
+		.write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA,
+		.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422,
+		.mbus = MEDIA_BUS_FMT_YUYV8_2X8,
+	}, {
+		.fourcc = V4L2_PIX_FMT_NV61M,
+		.uv_swap = 1,
+		.write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA,
+		.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422,
+		.mbus = MEDIA_BUS_FMT_YUYV8_2X8,
+	}, {
 		.fourcc = V4L2_PIX_FMT_YVU422M,
 		.uv_swap = 1,
 		.write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
@@ -1212,6 +1234,35 @@ static int rkisp1_enum_fmt_vid_cap_mplane(struct file *file, void *priv,
 	return -EINVAL;
 }
 
+static int rkisp1_enum_framesizes(struct file *file, void *fh,
+				  struct v4l2_frmsizeenum *fsize)
+{
+	static const unsigned int max_widths[] = {
+		RKISP1_RSZ_MP_SRC_MAX_WIDTH,
+		RKISP1_RSZ_SP_SRC_MAX_WIDTH,
+	};
+	static const unsigned int max_heights[] = {
+		RKISP1_RSZ_MP_SRC_MAX_HEIGHT,
+		RKISP1_RSZ_SP_SRC_MAX_HEIGHT,
+	};
+	struct rkisp1_capture *cap = video_drvdata(file);
+
+	if (fsize->index != 0)
+		return -EINVAL;
+
+	fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
+
+	fsize->stepwise.min_width = RKISP1_RSZ_SRC_MIN_WIDTH;
+	fsize->stepwise.max_width = max_widths[cap->id];
+	fsize->stepwise.step_width = 2;
+
+	fsize->stepwise.min_height = RKISP1_RSZ_SRC_MIN_HEIGHT;
+	fsize->stepwise.max_height = max_heights[cap->id];
+	fsize->stepwise.step_height = 2;
+
+	return 0;
+}
+
 static int rkisp1_s_fmt_vid_cap_mplane(struct file *file,
 				       void *priv, struct v4l2_format *f)
 {
@@ -1261,6 +1312,7 @@ static const struct v4l2_ioctl_ops rkisp1_v4l2_ioctl_ops = {
 	.vidioc_s_fmt_vid_cap_mplane = rkisp1_s_fmt_vid_cap_mplane,
 	.vidioc_g_fmt_vid_cap_mplane = rkisp1_g_fmt_vid_cap_mplane,
 	.vidioc_enum_fmt_vid_cap = rkisp1_enum_fmt_vid_cap_mplane,
+	.vidioc_enum_framesizes = rkisp1_enum_framesizes,
 	.vidioc_querycap = rkisp1_querycap,
 	.vidioc_subscribe_event = v4l2_ctrl_subscribe_event,
 	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c
index f2475c6235ea..4762cb32353d 100644
--- a/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c
+++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c
@@ -622,7 +622,7 @@ err_pm_runtime_disable:
 	return ret;
 }
 
-static int rkisp1_remove(struct platform_device *pdev)
+static void rkisp1_remove(struct platform_device *pdev)
 {
 	struct rkisp1_device *rkisp1 = platform_get_drvdata(pdev);
 
@@ -638,8 +638,6 @@ static int rkisp1_remove(struct platform_device *pdev)
 	v4l2_device_unregister(&rkisp1->v4l2_dev);
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static struct platform_driver rkisp1_drv = {
@@ -649,7 +647,7 @@ static struct platform_driver rkisp1_drv = {
 		.pm = &rkisp1_pm_ops,
 	},
 	.probe = rkisp1_probe,
-	.remove = rkisp1_remove,
+	.remove_new = rkisp1_remove,
 };
 
 module_platform_driver(rkisp1_drv);
diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c
index f76afd8112b2..c15ae0218118 100644
--- a/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c
+++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c
@@ -123,8 +123,8 @@ rkisp1_rsz_get_pad_fmt(struct rkisp1_resizer *rsz,
 		       unsigned int pad, u32 which)
 {
 	struct v4l2_subdev_state state = {
-		.pads = rsz->pad_cfg
-		};
+		.pads = rsz->pad_cfg,
+	};
 	if (which == V4L2_SUBDEV_FORMAT_TRY)
 		return v4l2_subdev_get_try_format(&rsz->sd, sd_state, pad);
 	else
@@ -137,8 +137,8 @@ rkisp1_rsz_get_pad_crop(struct rkisp1_resizer *rsz,
 			unsigned int pad, u32 which)
 {
 	struct v4l2_subdev_state state = {
-		.pads = rsz->pad_cfg
-		};
+		.pads = rsz->pad_cfg,
+	};
 	if (which == V4L2_SUBDEV_FORMAT_TRY)
 		return v4l2_subdev_get_try_crop(&rsz->sd, sd_state, pad);
 	else
@@ -366,7 +366,7 @@ static int rkisp1_rsz_enum_mbus_code(struct v4l2_subdev *sd,
 	struct v4l2_subdev_pad_config dummy_cfg;
 	struct v4l2_subdev_state pad_state = {
 		.pads = &dummy_cfg
-		};
+	};
 	u32 pad = code->pad;
 	int ret;
 
@@ -733,8 +733,8 @@ static void rkisp1_rsz_unregister(struct rkisp1_resizer *rsz)
 static int rkisp1_rsz_register(struct rkisp1_resizer *rsz)
 {
 	struct v4l2_subdev_state state = {
-		.pads = rsz->pad_cfg
-		};
+		.pads = rsz->pad_cfg,
+	};
 	static const char * const dev_names[] = {
 		RKISP1_RSZ_MP_DEV_NAME,
 		RKISP1_RSZ_SP_DEV_NAME
diff --git a/drivers/media/platform/samsung/exynos-gsc/gsc-core.c b/drivers/media/platform/samsung/exynos-gsc/gsc-core.c
index b147c645ae0b..1fb34de70649 100644
--- a/drivers/media/platform/samsung/exynos-gsc/gsc-core.c
+++ b/drivers/media/platform/samsung/exynos-gsc/gsc-core.c
@@ -1201,7 +1201,7 @@ err_clk:
 	return ret;
 }
 
-static int gsc_remove(struct platform_device *pdev)
+static void gsc_remove(struct platform_device *pdev)
 {
 	struct gsc_dev *gsc = platform_get_drvdata(pdev);
 	int i;
@@ -1220,7 +1220,6 @@ static int gsc_remove(struct platform_device *pdev)
 	pm_runtime_set_suspended(&pdev->dev);
 
 	dev_dbg(&pdev->dev, "%s driver unloaded\n", pdev->name);
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -1311,7 +1310,7 @@ static const struct dev_pm_ops gsc_pm_ops = {
 
 static struct platform_driver gsc_driver = {
 	.probe		= gsc_probe,
-	.remove		= gsc_remove,
+	.remove_new	= gsc_remove,
 	.driver = {
 		.name	= GSC_MODULE_NAME,
 		.pm	= &gsc_pm_ops,
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-capture.c b/drivers/media/platform/samsung/exynos4-is/fimc-capture.c
index e3b95a2b7e04..a0d43bf892e6 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-capture.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-capture.c
@@ -763,7 +763,10 @@ static int fimc_pipeline_try_format(struct fimc_ctx *ctx,
 	struct fimc_dev *fimc = ctx->fimc_dev;
 	struct fimc_pipeline *p = to_fimc_pipeline(fimc->vid_cap.ve.pipe);
 	struct v4l2_subdev *sd = p->subdevs[IDX_SENSOR];
-	struct v4l2_subdev_format sfmt;
+	struct v4l2_subdev_format sfmt = {
+		.which = set ? V4L2_SUBDEV_FORMAT_ACTIVE
+		       : V4L2_SUBDEV_FORMAT_TRY,
+	};
 	struct v4l2_mbus_framefmt *mf = &sfmt.format;
 	struct media_entity *me;
 	struct fimc_fmt *ffmt;
@@ -774,9 +777,7 @@ static int fimc_pipeline_try_format(struct fimc_ctx *ctx,
 	if (WARN_ON(!sd || !tfmt))
 		return -EINVAL;
 
-	memset(&sfmt, 0, sizeof(sfmt));
 	sfmt.format = *tfmt;
-	sfmt.which = set ? V4L2_SUBDEV_FORMAT_ACTIVE : V4L2_SUBDEV_FORMAT_TRY;
 
 	me = fimc_pipeline_get_head(&sd->entity);
 
@@ -854,7 +855,7 @@ static int fimc_get_sensor_frame_desc(struct v4l2_subdev *sensor,
 				      struct v4l2_plane_pix_format *plane_fmt,
 				      unsigned int num_planes, bool try)
 {
-	struct v4l2_mbus_frame_desc fd;
+	struct v4l2_mbus_frame_desc fd = { };
 	int i, ret;
 	int pad;
 
@@ -1095,7 +1096,12 @@ static int fimc_cap_g_input(struct file *file, void *priv, unsigned int *i)
  */
 static int fimc_pipeline_validate(struct fimc_dev *fimc)
 {
-	struct v4l2_subdev_format sink_fmt, src_fmt;
+	struct v4l2_subdev_format sink_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	struct v4l2_subdev_format src_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct fimc_vid_cap *vc = &fimc->vid_cap;
 	struct v4l2_subdev *sd = &vc->subdev;
 	struct fimc_pipeline *p = to_fimc_pipeline(vc->ve.pipe);
@@ -1132,7 +1138,6 @@ static int fimc_pipeline_validate(struct fimc_dev *fimc)
 			sink_fmt.format.code = ff->fmt ? ff->fmt->mbus_code : 0;
 		} else {
 			sink_fmt.pad = sink_pad->index;
-			sink_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 			ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &sink_fmt);
 			if (ret < 0 && ret != -ENOIOCTLCMD)
 				return -EPIPE;
@@ -1141,7 +1146,6 @@ static int fimc_pipeline_validate(struct fimc_dev *fimc)
 		/* Retrieve format at the source pad */
 		sd = media_entity_to_v4l2_subdev(src_pad->entity);
 		src_fmt.pad = src_pad->index;
-		src_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &src_fmt);
 		if (ret < 0 && ret != -ENOIOCTLCMD)
 			return -EPIPE;
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-core.c b/drivers/media/platform/samsung/exynos4-is/fimc-core.c
index 1791100b6935..a2034ade8b9e 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-core.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-core.c
@@ -1092,7 +1092,7 @@ static int fimc_suspend(struct device *dev)
 }
 #endif /* CONFIG_PM_SLEEP */
 
-static int fimc_remove(struct platform_device *pdev)
+static void fimc_remove(struct platform_device *pdev)
 {
 	struct fimc_dev *fimc = platform_get_drvdata(pdev);
 
@@ -1108,7 +1108,6 @@ static int fimc_remove(struct platform_device *pdev)
 	fimc_clk_put(fimc);
 
 	dev_info(&pdev->dev, "driver unloaded\n");
-	return 0;
 }
 
 /* S5PV210, S5PC110 */
@@ -1160,7 +1159,7 @@ static const struct dev_pm_ops fimc_pm_ops = {
 
 static struct platform_driver fimc_driver = {
 	.probe		= fimc_probe,
-	.remove		= fimc_remove,
+	.remove_new	= fimc_remove,
 	.driver = {
 		.of_match_table = fimc_of_match,
 		.name		= FIMC_DRIVER_NAME,
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-is-errno.c b/drivers/media/platform/samsung/exynos4-is/fimc-is-errno.c
index 5d9f4c1cdc5e..7a48fad1df16 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-is-errno.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-is-errno.c
@@ -54,7 +54,7 @@ const char *fimc_is_param_strerr(unsigned int error)
 	case ERROR_DMA_INPUT_ORDER:
 		return "ERROR_DMA_INPUT_ORDER: Invalid order(DRC: YYCbCr,YCbYCr,FD:NO,YYCbCr,YCbYCr,CbCr,CrCb)";
 	case ERROR_DMA_INPUT_PLANE:
-		return "ERROR_DMA_INPUT_PLANE: Invalid palne (DRC: 3, FD: 1, 2, 3)";
+		return "ERROR_DMA_INPUT_PLANE: Invalid plane (DRC: 3, FD: 1, 2, 3)";
 	case ERROR_OTF_OUTPUT_WIDTH:
 		return "ERROR_OTF_OUTPUT_WIDTH: Invalid width (DRC: 128~8192)";
 	case ERROR_OTF_OUTPUT_HEIGHT:
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-is-errno.h b/drivers/media/platform/samsung/exynos4-is/fimc-is-errno.h
index 9dcbb9853ac0..809e117331c0 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-is-errno.h
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-is-errno.h
@@ -156,7 +156,7 @@ enum fimc_is_error {
 	ERROR_DMA_INPUT_BIT_WIDTH	= 34,
 	/* invalid order(DRC: YYCbCrorYCbYCr, FD:NO,YYCbCr,YCbYCr,CbCr,CrCb) */
 	ERROR_DMA_INPUT_ORDER		= 35,
-	/* invalid palne (DRC: 3, FD: 1, 2, 3) */
+	/* invalid plane (DRC: 3, FD: 1, 2, 3) */
 	ERROR_DMA_INPUT_PLANE		= 36,
 
 	ERROR_OTF_OUTPUT_NONE		= ERROR_COMMON_NONE,
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-is-i2c.c b/drivers/media/platform/samsung/exynos4-is/fimc-is-i2c.c
index 83a28ef8e099..bef6e9b4a25e 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-is-i2c.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-is-i2c.c
@@ -82,14 +82,12 @@ err_pm_dis:
 	return ret;
 }
 
-static int fimc_is_i2c_remove(struct platform_device *pdev)
+static void fimc_is_i2c_remove(struct platform_device *pdev)
 {
 	struct fimc_is_i2c *isp_i2c = platform_get_drvdata(pdev);
 
 	pm_runtime_disable(&pdev->dev);
 	i2c_del_adapter(&isp_i2c->adapter);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -140,7 +138,7 @@ static const struct of_device_id fimc_is_i2c_of_match[] = {
 
 static struct platform_driver fimc_is_i2c_driver = {
 	.probe		= fimc_is_i2c_probe,
-	.remove		= fimc_is_i2c_remove,
+	.remove_new	= fimc_is_i2c_remove,
 	.driver = {
 		.of_match_table = fimc_is_i2c_of_match,
 		.name		= "fimc-isp-i2c",
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-is.c b/drivers/media/platform/samsung/exynos4-is/fimc-is.c
index a7704ff069d6..530a148fe4d3 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-is.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-is.c
@@ -915,7 +915,7 @@ static int fimc_is_suspend(struct device *dev)
 }
 #endif /* CONFIG_PM_SLEEP */
 
-static int fimc_is_remove(struct platform_device *pdev)
+static void fimc_is_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct fimc_is *is = dev_get_drvdata(dev);
@@ -932,8 +932,6 @@ static int fimc_is_remove(struct platform_device *pdev)
 	fimc_is_debugfs_remove(is);
 	release_firmware(is->fw.f_w);
 	fimc_is_free_cpu_memory(is);
-
-	return 0;
 }
 
 static const struct of_device_id fimc_is_of_match[] = {
@@ -950,7 +948,7 @@ static const struct dev_pm_ops fimc_is_pm_ops = {
 
 static struct platform_driver fimc_is_driver = {
 	.probe		= fimc_is_probe,
-	.remove		= fimc_is_remove,
+	.remove_new	= fimc_is_remove,
 	.driver = {
 		.of_match_table	= fimc_is_of_match,
 		.name		= FIMC_IS_DRV_NAME,
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-isp-video.c b/drivers/media/platform/samsung/exynos4-is/fimc-isp-video.c
index f6a302fa8d37..8fa26969c411 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-isp-video.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-isp-video.c
@@ -449,17 +449,22 @@ static int isp_video_s_fmt_mplane(struct file *file, void *priv,
 static int isp_video_pipeline_validate(struct fimc_isp *isp)
 {
 	struct v4l2_subdev *sd = &isp->subdev;
-	struct v4l2_subdev_format sink_fmt, src_fmt;
 	struct media_pad *pad;
 	int ret;
 
 	while (1) {
+		struct v4l2_subdev_format sink_fmt = {
+			.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		};
+		struct v4l2_subdev_format src_fmt = {
+			.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		};
+
 		/* Retrieve format at the sink pad */
 		pad = &sd->entity.pads[0];
 		if (!(pad->flags & MEDIA_PAD_FL_SINK))
 			break;
 		sink_fmt.pad = pad->index;
-		sink_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &sink_fmt);
 		if (ret < 0 && ret != -ENOIOCTLCMD)
 			return -EPIPE;
@@ -471,7 +476,6 @@ static int isp_video_pipeline_validate(struct fimc_isp *isp)
 
 		sd = media_entity_to_v4l2_subdev(pad->entity);
 		src_fmt.pad = pad->index;
-		src_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &src_fmt);
 		if (ret < 0 && ret != -ENOIOCTLCMD)
 			return -EPIPE;
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-lite.c b/drivers/media/platform/samsung/exynos4-is/fimc-lite.c
index e185a40305a8..24b3dda26714 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-lite.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-lite.c
@@ -765,7 +765,12 @@ static int fimc_lite_s_fmt_mplane(struct file *file, void *priv,
 static int fimc_pipeline_validate(struct fimc_lite *fimc)
 {
 	struct v4l2_subdev *sd = &fimc->subdev;
-	struct v4l2_subdev_format sink_fmt, src_fmt;
+	struct v4l2_subdev_format sink_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	struct v4l2_subdev_format src_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct media_pad *pad;
 	int ret;
 
@@ -782,7 +787,6 @@ static int fimc_pipeline_validate(struct fimc_lite *fimc)
 			sink_fmt.format.code = fimc->inp_frame.fmt->mbus_code;
 		} else {
 			sink_fmt.pad = pad->index;
-			sink_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 			ret = v4l2_subdev_call(sd, pad, get_fmt, NULL,
 					       &sink_fmt);
 			if (ret < 0 && ret != -ENOIOCTLCMD)
@@ -795,7 +799,6 @@ static int fimc_pipeline_validate(struct fimc_lite *fimc)
 
 		sd = media_entity_to_v4l2_subdev(pad->entity);
 		src_fmt.pad = pad->index;
-		src_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &src_fmt);
 		if (ret < 0 && ret != -ENOIOCTLCMD)
 			return -EPIPE;
@@ -1595,7 +1598,7 @@ static int fimc_lite_suspend(struct device *dev)
 }
 #endif /* CONFIG_PM_SLEEP */
 
-static int fimc_lite_remove(struct platform_device *pdev)
+static void fimc_lite_remove(struct platform_device *pdev)
 {
 	struct fimc_lite *fimc = platform_get_drvdata(pdev);
 	struct device *dev = &pdev->dev;
@@ -1610,7 +1613,6 @@ static int fimc_lite_remove(struct platform_device *pdev)
 	fimc_lite_clk_put(fimc);
 
 	dev_info(dev, "Driver unloaded\n");
-	return 0;
 }
 
 static const struct dev_pm_ops fimc_lite_pm_ops = {
@@ -1656,7 +1658,7 @@ MODULE_DEVICE_TABLE(of, flite_of_match);
 
 static struct platform_driver fimc_lite_driver = {
 	.probe		= fimc_lite_probe,
-	.remove		= fimc_lite_remove,
+	.remove_new	= fimc_lite_remove,
 	.driver = {
 		.of_match_table = flite_of_match,
 		.name		= FIMC_LITE_DRV_NAME,
diff --git a/drivers/media/platform/samsung/exynos4-is/media-dev.c b/drivers/media/platform/samsung/exynos4-is/media-dev.c
index 98a60f01129d..c9cb9a216fae 100644
--- a/drivers/media/platform/samsung/exynos4-is/media-dev.c
+++ b/drivers/media/platform/samsung/exynos4-is/media-dev.c
@@ -1440,6 +1440,10 @@ static int fimc_md_probe(struct platform_device *pdev)
 	if (!fmd)
 		return -ENOMEM;
 
+	ret = of_platform_populate(dev->of_node, NULL, NULL, dev);
+	if (ret < 0)
+		return -ENOMEM;
+
 	spin_lock_init(&fmd->slock);
 	INIT_LIST_HEAD(&fmd->pipelines);
 	fmd->pdev = pdev;
@@ -1470,10 +1474,8 @@ static int fimc_md_probe(struct platform_device *pdev)
 		goto err_v4l2dev;
 
 	pinctrl = devm_pinctrl_get(dev);
-	if (IS_ERR(pinctrl)) {
-		ret = dev_err_probe(dev, PTR_ERR(pinctrl), "Failed to get pinctrl\n");
-		goto err_clk;
-	}
+	if (IS_ERR(pinctrl))
+		dev_dbg(dev, "Failed to get pinctrl: %pe\n", pinctrl);
 
 	platform_set_drvdata(pdev, fmd);
 
@@ -1530,12 +1532,12 @@ err_md:
 	return ret;
 }
 
-static int fimc_md_remove(struct platform_device *pdev)
+static void fimc_md_remove(struct platform_device *pdev)
 {
 	struct fimc_md *fmd = platform_get_drvdata(pdev);
 
 	if (!fmd)
-		return 0;
+		return;
 
 	fimc_md_unregister_clk_provider(fmd);
 	v4l2_async_nf_unregister(&fmd->subdev_notifier);
@@ -1548,8 +1550,6 @@ static int fimc_md_remove(struct platform_device *pdev)
 	media_device_unregister(&fmd->media_dev);
 	media_device_cleanup(&fmd->media_dev);
 	fimc_md_put_clocks(fmd);
-
-	return 0;
 }
 
 static const struct platform_device_id fimc_driver_ids[] __always_unused = {
@@ -1566,7 +1566,7 @@ MODULE_DEVICE_TABLE(of, fimc_md_of_match);
 
 static struct platform_driver fimc_md_driver = {
 	.probe		= fimc_md_probe,
-	.remove		= fimc_md_remove,
+	.remove_new	= fimc_md_remove,
 	.driver = {
 		.of_match_table = of_match_ptr(fimc_md_of_match),
 		.name		= "s5p-fimc-md",
diff --git a/drivers/media/platform/samsung/exynos4-is/mipi-csis.c b/drivers/media/platform/samsung/exynos4-is/mipi-csis.c
index 6a0d35f33e8c..686ca8753ba2 100644
--- a/drivers/media/platform/samsung/exynos4-is/mipi-csis.c
+++ b/drivers/media/platform/samsung/exynos4-is/mipi-csis.c
@@ -975,7 +975,7 @@ static int s5pcsis_runtime_resume(struct device *dev)
 }
 #endif
 
-static int s5pcsis_remove(struct platform_device *pdev)
+static void s5pcsis_remove(struct platform_device *pdev)
 {
 	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
 	struct csis_state *state = sd_to_csis_state(sd);
@@ -987,8 +987,6 @@ static int s5pcsis_remove(struct platform_device *pdev)
 	s5pcsis_clk_put(state);
 
 	media_entity_cleanup(&state->sd.entity);
-
-	return 0;
 }
 
 static const struct dev_pm_ops s5pcsis_pm_ops = {
@@ -1022,7 +1020,7 @@ MODULE_DEVICE_TABLE(of, s5pcsis_of_match);
 
 static struct platform_driver s5pcsis_driver = {
 	.probe		= s5pcsis_probe,
-	.remove		= s5pcsis_remove,
+	.remove_new	= s5pcsis_remove,
 	.driver		= {
 		.of_match_table = s5pcsis_of_match,
 		.name		= CSIS_DRIVER_NAME,
diff --git a/drivers/media/platform/samsung/s3c-camif/camif-capture.c b/drivers/media/platform/samsung/s3c-camif/camif-capture.c
index db106ebdf870..76634d242b10 100644
--- a/drivers/media/platform/samsung/s3c-camif/camif-capture.c
+++ b/drivers/media/platform/samsung/s3c-camif/camif-capture.c
@@ -806,7 +806,9 @@ static int s3c_camif_vidioc_s_fmt(struct file *file, void *priv,
 /* Only check pixel formats at the sensor and the camif subdev pads */
 static int camif_pipeline_validate(struct camif_dev *camif)
 {
-	struct v4l2_subdev_format src_fmt;
+	struct v4l2_subdev_format src_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct media_pad *pad;
 	int ret;
 
@@ -816,7 +818,6 @@ static int camif_pipeline_validate(struct camif_dev *camif)
 		return -EPIPE;
 
 	src_fmt.pad = pad->index;
-	src_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(camif->sensor.sd, pad, get_fmt, NULL, &src_fmt);
 	if (ret < 0 && ret != -ENOIOCTLCMD)
 		return -EPIPE;
diff --git a/drivers/media/platform/samsung/s3c-camif/camif-core.c b/drivers/media/platform/samsung/s3c-camif/camif-core.c
index 6e8ef86566b7..afe1fcc37354 100644
--- a/drivers/media/platform/samsung/s3c-camif/camif-core.c
+++ b/drivers/media/platform/samsung/s3c-camif/camif-core.c
@@ -190,7 +190,9 @@ static int camif_register_sensor(struct camif_dev *camif)
 	struct s3c_camif_sensor_info *sensor = &camif->pdata.sensor;
 	struct v4l2_device *v4l2_dev = &camif->v4l2_dev;
 	struct i2c_adapter *adapter;
-	struct v4l2_subdev_format format;
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *sd;
 	int ret;
 
@@ -220,7 +222,6 @@ static int camif_register_sensor(struct camif_dev *camif)
 
 	/* Get initial pixel format and set it at the camif sink pad */
 	format.pad = 0;
-	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &format);
 
 	if (ret < 0)
@@ -507,7 +508,7 @@ err_sd:
 	return ret;
 }
 
-static int s3c_camif_remove(struct platform_device *pdev)
+static void s3c_camif_remove(struct platform_device *pdev)
 {
 	struct camif_dev *camif = platform_get_drvdata(pdev);
 	struct s3c_camif_plat_data *pdata = &camif->pdata;
@@ -521,8 +522,6 @@ static int s3c_camif_remove(struct platform_device *pdev)
 	camif_clk_put(camif);
 	s3c_camif_unregister_subdev(camif);
 	pdata->gpio_put();
-
-	return 0;
 }
 
 static int s3c_camif_runtime_resume(struct device *dev)
@@ -623,7 +622,7 @@ static const struct dev_pm_ops s3c_camif_pm_ops = {
 
 static struct platform_driver s3c_camif_driver = {
 	.probe		= s3c_camif_probe,
-	.remove		= s3c_camif_remove,
+	.remove_new	= s3c_camif_remove,
 	.id_table	= s3c_camif_driver_ids,
 	.driver = {
 		.name	= S3C_CAMIF_DRIVER_NAME,
diff --git a/drivers/media/platform/samsung/s5p-g2d/g2d.c b/drivers/media/platform/samsung/s5p-g2d/g2d.c
index dd8864779a7c..89aeba47ed07 100644
--- a/drivers/media/platform/samsung/s5p-g2d/g2d.c
+++ b/drivers/media/platform/samsung/s5p-g2d/g2d.c
@@ -740,7 +740,7 @@ put_clk:
 	return ret;
 }
 
-static int g2d_remove(struct platform_device *pdev)
+static void g2d_remove(struct platform_device *pdev)
 {
 	struct g2d_dev *dev = platform_get_drvdata(pdev);
 
@@ -753,7 +753,6 @@ static int g2d_remove(struct platform_device *pdev)
 	clk_put(dev->gate);
 	clk_unprepare(dev->clk);
 	clk_put(dev->clk);
-	return 0;
 }
 
 static struct g2d_variant g2d_drvdata_v3x = {
@@ -778,7 +777,7 @@ MODULE_DEVICE_TABLE(of, exynos_g2d_match);
 
 static struct platform_driver g2d_pdrv = {
 	.probe		= g2d_probe,
-	.remove		= g2d_remove,
+	.remove_new	= g2d_remove,
 	.driver		= {
 		.name = G2D_NAME,
 		.of_match_table = exynos_g2d_match,
diff --git a/drivers/media/platform/samsung/s5p-jpeg/jpeg-core.c b/drivers/media/platform/samsung/s5p-jpeg/jpeg-core.c
index 55814041b8d8..c3c7e48f1b6e 100644
--- a/drivers/media/platform/samsung/s5p-jpeg/jpeg-core.c
+++ b/drivers/media/platform/samsung/s5p-jpeg/jpeg-core.c
@@ -2991,7 +2991,7 @@ device_register_rollback:
 	return ret;
 }
 
-static int s5p_jpeg_remove(struct platform_device *pdev)
+static void s5p_jpeg_remove(struct platform_device *pdev)
 {
 	struct s5p_jpeg *jpeg = platform_get_drvdata(pdev);
 	int i;
@@ -3008,8 +3008,6 @@ static int s5p_jpeg_remove(struct platform_device *pdev)
 		for (i = jpeg->variant->num_clocks - 1; i >= 0; i--)
 			clk_disable_unprepare(jpeg->clocks[i]);
 	}
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -3164,7 +3162,7 @@ static void *jpeg_get_drv_data(struct device *dev)
 
 static struct platform_driver s5p_jpeg_driver = {
 	.probe = s5p_jpeg_probe,
-	.remove = s5p_jpeg_remove,
+	.remove_new = s5p_jpeg_remove,
 	.driver = {
 		.of_match_table	= of_match_ptr(samsung_jpeg_match),
 		.name		= S5P_JPEG_M2M_NAME,
diff --git a/drivers/media/platform/samsung/s5p-mfc/s5p_mfc.c b/drivers/media/platform/samsung/s5p-mfc/s5p_mfc.c
index 9d2cce124a34..e30e54935d79 100644
--- a/drivers/media/platform/samsung/s5p-mfc/s5p_mfc.c
+++ b/drivers/media/platform/samsung/s5p-mfc/s5p_mfc.c
@@ -1431,7 +1431,7 @@ err_dma:
 }
 
 /* Remove the driver */
-static int s5p_mfc_remove(struct platform_device *pdev)
+static void s5p_mfc_remove(struct platform_device *pdev)
 {
 	struct s5p_mfc_dev *dev = platform_get_drvdata(pdev);
 	struct s5p_mfc_ctx *ctx;
@@ -1463,7 +1463,6 @@ static int s5p_mfc_remove(struct platform_device *pdev)
 	s5p_mfc_unconfigure_dma_memory(dev);
 
 	s5p_mfc_final_pm(dev);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1690,7 +1689,7 @@ MODULE_DEVICE_TABLE(of, exynos_mfc_match);
 
 static struct platform_driver s5p_mfc_driver = {
 	.probe		= s5p_mfc_probe,
-	.remove		= s5p_mfc_remove,
+	.remove_new	= s5p_mfc_remove,
 	.driver	= {
 		.name	= S5P_MFC_NAME,
 		.pm	= &s5p_mfc_pm_ops,
diff --git a/drivers/media/platform/st/sti/bdisp/bdisp-v4l2.c b/drivers/media/platform/st/sti/bdisp/bdisp-v4l2.c
index dd74cc43920d..1328b4eb6b9f 100644
--- a/drivers/media/platform/st/sti/bdisp/bdisp-v4l2.c
+++ b/drivers/media/platform/st/sti/bdisp/bdisp-v4l2.c
@@ -1257,7 +1257,7 @@ static const struct dev_pm_ops bdisp_pm_ops = {
 	.runtime_resume         = bdisp_runtime_resume,
 };
 
-static int bdisp_remove(struct platform_device *pdev)
+static void bdisp_remove(struct platform_device *pdev)
 {
 	struct bdisp_dev *bdisp = platform_get_drvdata(pdev);
 
@@ -1277,8 +1277,6 @@ static int bdisp_remove(struct platform_device *pdev)
 	destroy_workqueue(bdisp->work_queue);
 
 	dev_dbg(&pdev->dev, "%s driver unloaded\n", pdev->name);
-
-	return 0;
 }
 
 static int bdisp_probe(struct platform_device *pdev)
@@ -1309,6 +1307,8 @@ static int bdisp_probe(struct platform_device *pdev)
 	init_waitqueue_head(&bdisp->irq_queue);
 	INIT_DELAYED_WORK(&bdisp->timeout_work, bdisp_irq_timeout);
 	bdisp->work_queue = create_workqueue(BDISP_NAME);
+	if (!bdisp->work_queue)
+		return -ENOMEM;
 
 	spin_lock_init(&bdisp->slock);
 	mutex_init(&bdisp->lock);
@@ -1411,7 +1411,7 @@ MODULE_DEVICE_TABLE(of, bdisp_match_types);
 
 static struct platform_driver bdisp_driver = {
 	.probe          = bdisp_probe,
-	.remove         = bdisp_remove,
+	.remove_new     = bdisp_remove,
 	.driver         = {
 		.name           = BDISP_NAME,
 		.of_match_table = bdisp_match_types,
diff --git a/drivers/media/platform/st/sti/c8sectpfe/c8sectpfe-core.c b/drivers/media/platform/st/sti/c8sectpfe/c8sectpfe-core.c
index c38b62d4f1ae..45ade7210d26 100644
--- a/drivers/media/platform/st/sti/c8sectpfe/c8sectpfe-core.c
+++ b/drivers/media/platform/st/sti/c8sectpfe/c8sectpfe-core.c
@@ -878,7 +878,7 @@ err_clk_disable:
 	return ret;
 }
 
-static int c8sectpfe_remove(struct platform_device *pdev)
+static void c8sectpfe_remove(struct platform_device *pdev)
 {
 	struct c8sectpfei *fei = platform_get_drvdata(pdev);
 	struct channel_info *channel;
@@ -910,8 +910,6 @@ static int c8sectpfe_remove(struct platform_device *pdev)
 		writel(0, fei->io + SYS_OTHER_CLKEN);
 
 	clk_disable_unprepare(fei->c8sectpfeclk);
-
-	return 0;
 }
 
 
@@ -1178,7 +1176,7 @@ static struct platform_driver c8sectpfe_driver = {
 		.of_match_table = of_match_ptr(c8sectpfe_match),
 	},
 	.probe	= c8sectpfe_probe,
-	.remove	= c8sectpfe_remove,
+	.remove_new = c8sectpfe_remove,
 };
 
 module_platform_driver(c8sectpfe_driver);
diff --git a/drivers/media/platform/st/sti/delta/delta-v4l2.c b/drivers/media/platform/st/sti/delta/delta-v4l2.c
index 03eaee6d15da..da402d1e9171 100644
--- a/drivers/media/platform/st/sti/delta/delta-v4l2.c
+++ b/drivers/media/platform/st/sti/delta/delta-v4l2.c
@@ -1900,7 +1900,7 @@ err:
 	return ret;
 }
 
-static int delta_remove(struct platform_device *pdev)
+static void delta_remove(struct platform_device *pdev)
 {
 	struct delta_dev *delta = platform_get_drvdata(pdev);
 
@@ -1914,8 +1914,6 @@ static int delta_remove(struct platform_device *pdev)
 	pm_runtime_disable(delta->dev);
 
 	v4l2_device_unregister(&delta->v4l2_dev);
-
-	return 0;
 }
 
 static int delta_runtime_suspend(struct device *dev)
@@ -1956,7 +1954,7 @@ MODULE_DEVICE_TABLE(of, delta_match_types);
 
 static struct platform_driver delta_driver = {
 	.probe = delta_probe,
-	.remove = delta_remove,
+	.remove_new = delta_remove,
 	.driver = {
 		   .name = DELTA_NAME,
 		   .of_match_table = delta_match_types,
diff --git a/drivers/media/platform/st/sti/hva/hva-v4l2.c b/drivers/media/platform/st/sti/hva/hva-v4l2.c
index bb34d6997d99..3a848ca32a0e 100644
--- a/drivers/media/platform/st/sti/hva/hva-v4l2.c
+++ b/drivers/media/platform/st/sti/hva/hva-v4l2.c
@@ -1422,7 +1422,7 @@ err:
 	return ret;
 }
 
-static int hva_remove(struct platform_device *pdev)
+static void hva_remove(struct platform_device *pdev)
 {
 	struct hva_dev *hva = platform_get_drvdata(pdev);
 	struct device *dev = hva_to_dev(hva);
@@ -1440,8 +1440,6 @@ static int hva_remove(struct platform_device *pdev)
 	v4l2_device_unregister(&hva->v4l2_dev);
 
 	dev_info(dev, "%s %s removed\n", HVA_PREFIX, pdev->name);
-
-	return 0;
 }
 
 /* PM ops */
@@ -1461,7 +1459,7 @@ MODULE_DEVICE_TABLE(of, hva_match_types);
 
 static struct platform_driver hva_driver = {
 	.probe  = hva_probe,
-	.remove = hva_remove,
+	.remove_new = hva_remove,
 	.driver = {
 		.name		= HVA_NAME,
 		.of_match_table	= hva_match_types,
diff --git a/drivers/media/platform/st/stm32/dma2d/dma2d.c b/drivers/media/platform/st/stm32/dma2d/dma2d.c
index 9706aa41b5d2..92f1edee58f8 100644
--- a/drivers/media/platform/st/stm32/dma2d/dma2d.c
+++ b/drivers/media/platform/st/stm32/dma2d/dma2d.c
@@ -603,7 +603,6 @@ static int dma2d_probe(struct platform_device *pdev)
 {
 	struct dma2d_dev *dev;
 	struct video_device *vfd;
-	struct resource *res;
 	int ret = 0;
 
 	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
@@ -614,9 +613,7 @@ static int dma2d_probe(struct platform_device *pdev)
 	mutex_init(&dev->mutex);
 	atomic_set(&dev->num_inst, 0);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-
-	dev->regs = devm_ioremap_resource(&pdev->dev, res);
+	dev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
 	if (IS_ERR(dev->regs))
 		return PTR_ERR(dev->regs);
 
@@ -696,7 +693,7 @@ put_clk_gate:
 	return ret;
 }
 
-static int dma2d_remove(struct platform_device *pdev)
+static void dma2d_remove(struct platform_device *pdev)
 {
 	struct dma2d_dev *dev = platform_get_drvdata(pdev);
 
@@ -707,8 +704,6 @@ static int dma2d_remove(struct platform_device *pdev)
 	vb2_dma_contig_clear_max_seg_size(&pdev->dev);
 	clk_unprepare(dev->gate);
 	clk_put(dev->gate);
-
-	return 0;
 }
 
 static const struct of_device_id stm32_dma2d_match[] = {
@@ -722,7 +717,7 @@ MODULE_DEVICE_TABLE(of, stm32_dma2d_match);
 
 static struct platform_driver dma2d_pdrv = {
 	.probe		= dma2d_probe,
-	.remove		= dma2d_remove,
+	.remove_new	= dma2d_remove,
 	.driver		= {
 		.name = DMA2D_NAME,
 		.of_match_table = stm32_dma2d_match,
diff --git a/drivers/media/platform/st/stm32/dma2d/dma2d.h b/drivers/media/platform/st/stm32/dma2d/dma2d.h
index 3f03a7ca9ee3..af12739fc774 100644
--- a/drivers/media/platform/st/stm32/dma2d/dma2d.h
+++ b/drivers/media/platform/st/stm32/dma2d/dma2d.h
@@ -90,8 +90,6 @@ struct dma2d_ctx {
 	struct dma2d_frame	cap;
 	struct dma2d_frame	out;
 	struct dma2d_frame	bg;
-	/* fb_buf always point to bg address */
-	struct v4l2_framebuffer	fb_buf;
 	/*
 	 * MODE[17:16] of DMA2D_CR
 	 */
diff --git a/drivers/media/platform/st/stm32/stm32-dcmi.c b/drivers/media/platform/st/stm32/stm32-dcmi.c
index ad8e9742e1ae..dad6e22e4ce4 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmi.c
+++ b/drivers/media/platform/st/stm32/stm32-dcmi.c
@@ -2084,6 +2084,7 @@ static int dcmi_probe(struct platform_device *pdev)
 	q->mem_ops = &vb2_dma_contig_memops;
 	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
 	q->min_buffers_needed = 2;
+	q->allow_cache_hints = 1;
 	q->dev = &pdev->dev;
 
 	ret = vb2_queue_init(q);
@@ -2134,7 +2135,7 @@ err_media_device_cleanup:
 	return ret;
 }
 
-static int dcmi_remove(struct platform_device *pdev)
+static void dcmi_remove(struct platform_device *pdev)
 {
 	struct stm32_dcmi *dcmi = platform_get_drvdata(pdev);
 
@@ -2147,8 +2148,6 @@ static int dcmi_remove(struct platform_device *pdev)
 	media_device_cleanup(&dcmi->mdev);
 
 	dma_release_channel(dcmi->dma_chan);
-
-	return 0;
 }
 
 static __maybe_unused int dcmi_runtime_suspend(struct device *dev)
@@ -2202,7 +2201,7 @@ static const struct dev_pm_ops dcmi_pm_ops = {
 
 static struct platform_driver stm32_dcmi_driver = {
 	.probe		= dcmi_probe,
-	.remove		= dcmi_remove,
+	.remove_new	= dcmi_remove,
 	.driver		= {
 		.name = DRV_NAME,
 		.of_match_table = of_match_ptr(stm32_dcmi_of_match),
diff --git a/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.c b/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.c
index 86c5235a0c7a..d6e7d1b36083 100644
--- a/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.c
+++ b/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.c
@@ -260,7 +260,7 @@ err_clean_pad:
 	return ret;
 }
 
-static int sun4i_csi_remove(struct platform_device *pdev)
+static void sun4i_csi_remove(struct platform_device *pdev)
 {
 	struct sun4i_csi *csi = platform_get_drvdata(pdev);
 
@@ -271,8 +271,6 @@ static int sun4i_csi_remove(struct platform_device *pdev)
 	media_device_unregister(&csi->mdev);
 	sun4i_csi_dma_unregister(csi);
 	media_device_cleanup(&csi->mdev);
-
-	return 0;
 }
 
 static const struct sun4i_csi_traits sun4i_a10_csi1_traits = {
@@ -330,7 +328,7 @@ static const struct dev_pm_ops sun4i_csi_pm_ops = {
 
 static struct platform_driver sun4i_csi_driver = {
 	.probe	= sun4i_csi_probe,
-	.remove	= sun4i_csi_remove,
+	.remove_new = sun4i_csi_remove,
 	.driver	= {
 		.name		= "sun4i-csi",
 		.of_match_table	= sun4i_csi_of_match,
diff --git a/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c b/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c
index e3e6650181c8..e2723cfa4515 100644
--- a/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c
+++ b/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c
@@ -375,7 +375,7 @@ error_resources:
 	return ret;
 }
 
-static int sun6i_csi_remove(struct platform_device *pdev)
+static void sun6i_csi_remove(struct platform_device *pdev)
 {
 	struct sun6i_csi_device *csi_dev = platform_get_drvdata(pdev);
 
@@ -386,8 +386,6 @@ static int sun6i_csi_remove(struct platform_device *pdev)
 		sun6i_csi_v4l2_cleanup(csi_dev);
 
 	sun6i_csi_resources_cleanup(csi_dev);
-
-	return 0;
 }
 
 static const struct sun6i_csi_variant sun6i_a31_csi_variant = {
@@ -426,7 +424,7 @@ MODULE_DEVICE_TABLE(of, sun6i_csi_of_match);
 
 static struct platform_driver sun6i_csi_platform_driver = {
 	.probe	= sun6i_csi_probe,
-	.remove	= sun6i_csi_remove,
+	.remove_new = sun6i_csi_remove,
 	.driver	= {
 		.name		= SUN6I_CSI_NAME,
 		.of_match_table	= of_match_ptr(sun6i_csi_of_match),
diff --git a/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c b/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c
index a220ce849b41..dce130b4c9f6 100644
--- a/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c
+++ b/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c
@@ -737,15 +737,13 @@ error_resources:
 	return ret;
 }
 
-static int sun6i_mipi_csi2_remove(struct platform_device *platform_dev)
+static void sun6i_mipi_csi2_remove(struct platform_device *platform_dev)
 {
 	struct sun6i_mipi_csi2_device *csi2_dev =
 		platform_get_drvdata(platform_dev);
 
 	sun6i_mipi_csi2_bridge_cleanup(csi2_dev);
 	sun6i_mipi_csi2_resources_cleanup(csi2_dev);
-
-	return 0;
 }
 
 static const struct of_device_id sun6i_mipi_csi2_of_match[] = {
@@ -756,7 +754,7 @@ MODULE_DEVICE_TABLE(of, sun6i_mipi_csi2_of_match);
 
 static struct platform_driver sun6i_mipi_csi2_platform_driver = {
 	.probe	= sun6i_mipi_csi2_probe,
-	.remove	= sun6i_mipi_csi2_remove,
+	.remove_new = sun6i_mipi_csi2_remove,
 	.driver	= {
 		.name		= SUN6I_MIPI_CSI2_NAME,
 		.of_match_table	= of_match_ptr(sun6i_mipi_csi2_of_match),
diff --git a/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c b/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c
index cd2e92ae2293..23d32e198aaa 100644
--- a/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c
+++ b/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c
@@ -804,15 +804,13 @@ error_resources:
 	return ret;
 }
 
-static int sun8i_a83t_mipi_csi2_remove(struct platform_device *platform_dev)
+static void sun8i_a83t_mipi_csi2_remove(struct platform_device *platform_dev)
 {
 	struct sun8i_a83t_mipi_csi2_device *csi2_dev =
 		platform_get_drvdata(platform_dev);
 
 	sun8i_a83t_mipi_csi2_bridge_cleanup(csi2_dev);
 	sun8i_a83t_mipi_csi2_resources_cleanup(csi2_dev);
-
-	return 0;
 }
 
 static const struct of_device_id sun8i_a83t_mipi_csi2_of_match[] = {
@@ -823,7 +821,7 @@ MODULE_DEVICE_TABLE(of, sun8i_a83t_mipi_csi2_of_match);
 
 static struct platform_driver sun8i_a83t_mipi_csi2_platform_driver = {
 	.probe	= sun8i_a83t_mipi_csi2_probe,
-	.remove	= sun8i_a83t_mipi_csi2_remove,
+	.remove_new = sun8i_a83t_mipi_csi2_remove,
 	.driver	= {
 		.name		= SUN8I_A83T_MIPI_CSI2_NAME,
 		.of_match_table	= of_match_ptr(sun8i_a83t_mipi_csi2_of_match),
diff --git a/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c b/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c
index aa65d70b6270..e4b0fd793f55 100644
--- a/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c
+++ b/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c
@@ -906,7 +906,7 @@ err_v4l2:
 	return ret;
 }
 
-static int deinterlace_remove(struct platform_device *pdev)
+static void deinterlace_remove(struct platform_device *pdev)
 {
 	struct deinterlace_dev *dev = platform_get_drvdata(pdev);
 
@@ -915,8 +915,6 @@ static int deinterlace_remove(struct platform_device *pdev)
 	v4l2_device_unregister(&dev->v4l2_dev);
 
 	pm_runtime_force_suspend(&pdev->dev);
-
-	return 0;
 }
 
 static int deinterlace_runtime_resume(struct device *device)
@@ -1002,7 +1000,7 @@ static const struct dev_pm_ops deinterlace_pm_ops = {
 
 static struct platform_driver deinterlace_driver = {
 	.probe		= deinterlace_probe,
-	.remove		= deinterlace_remove,
+	.remove_new	= deinterlace_remove,
 	.driver		= {
 		.name		= DEINTERLACE_NAME,
 		.of_match_table	= deinterlace_dt_match,
diff --git a/drivers/media/platform/sunxi/sun8i-rotate/sun8i_rotate.c b/drivers/media/platform/sunxi/sun8i-rotate/sun8i_rotate.c
index fbcca59a0517..bd0c4257bbff 100644
--- a/drivers/media/platform/sunxi/sun8i-rotate/sun8i_rotate.c
+++ b/drivers/media/platform/sunxi/sun8i-rotate/sun8i_rotate.c
@@ -833,7 +833,7 @@ err_v4l2:
 	return ret;
 }
 
-static int rotate_remove(struct platform_device *pdev)
+static void rotate_remove(struct platform_device *pdev)
 {
 	struct rotate_dev *dev = platform_get_drvdata(pdev);
 
@@ -842,8 +842,6 @@ static int rotate_remove(struct platform_device *pdev)
 	v4l2_device_unregister(&dev->v4l2_dev);
 
 	pm_runtime_force_suspend(&pdev->dev);
-
-	return 0;
 }
 
 static int rotate_runtime_resume(struct device *device)
@@ -907,7 +905,7 @@ static const struct dev_pm_ops rotate_pm_ops = {
 
 static struct platform_driver rotate_driver = {
 	.probe		= rotate_probe,
-	.remove		= rotate_remove,
+	.remove_new	= rotate_remove,
 	.driver		= {
 		.name		= ROTATE_NAME,
 		.of_match_table	= rotate_dt_match,
diff --git a/drivers/media/platform/ti/am437x/am437x-vpfe.c b/drivers/media/platform/ti/am437x/am437x-vpfe.c
index 2dfae9bc0bba..ffe1887cc429 100644
--- a/drivers/media/platform/ti/am437x/am437x-vpfe.c
+++ b/drivers/media/platform/ti/am437x/am437x-vpfe.c
@@ -1285,13 +1285,13 @@ static int __subdev_get_format(struct vpfe_device *vpfe,
 			       struct v4l2_mbus_framefmt *fmt)
 {
 	struct v4l2_subdev *sd = vpfe->current_subdev->sd;
-	struct v4l2_subdev_format sd_fmt;
+	struct v4l2_subdev_format sd_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.pad = 0,
+	};
 	struct v4l2_mbus_framefmt *mbus_fmt = &sd_fmt.format;
 	int ret;
 
-	sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	sd_fmt.pad = 0;
-
 	ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &sd_fmt);
 	if (ret)
 		return ret;
@@ -1309,12 +1309,13 @@ static int __subdev_set_format(struct vpfe_device *vpfe,
 			       struct v4l2_mbus_framefmt *fmt)
 {
 	struct v4l2_subdev *sd = vpfe->current_subdev->sd;
-	struct v4l2_subdev_format sd_fmt;
+	struct v4l2_subdev_format sd_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.pad = 0,
+	};
 	struct v4l2_mbus_framefmt *mbus_fmt = &sd_fmt.format;
 	int ret;
 
-	sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	sd_fmt.pad = 0;
 	*mbus_fmt = *fmt;
 
 	ret = v4l2_subdev_call(sd, pad, set_fmt, NULL, &sd_fmt);
@@ -1393,7 +1394,9 @@ static int vpfe_try_fmt(struct file *file, void *priv,
 	struct vpfe_device *vpfe = video_drvdata(file);
 	struct v4l2_subdev *sd = vpfe->current_subdev->sd;
 	const struct vpfe_fmt *fmt;
-	struct v4l2_subdev_frame_size_enum fse;
+	struct v4l2_subdev_frame_size_enum fse = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	int ret, found;
 
 	fmt = find_format_by_pix(vpfe, f->fmt.pix.pixelformat);
@@ -1412,7 +1415,6 @@ static int vpfe_try_fmt(struct file *file, void *priv,
 	found = false;
 	fse.pad = 0;
 	fse.code = fmt->code;
-	fse.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	for (fse.index = 0; ; fse.index++) {
 		ret = v4l2_subdev_call(sd, pad, enum_frame_size,
 				       NULL, &fse);
@@ -1499,7 +1501,9 @@ static int vpfe_enum_size(struct file *file, void  *priv,
 			  struct v4l2_frmsizeenum *fsize)
 {
 	struct vpfe_device *vpfe = video_drvdata(file);
-	struct v4l2_subdev_frame_size_enum fse;
+	struct v4l2_subdev_frame_size_enum fse = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *sd = vpfe->current_subdev->sd;
 	struct vpfe_fmt *fmt;
 	int ret;
@@ -1514,11 +1518,9 @@ static int vpfe_enum_size(struct file *file, void  *priv,
 
 	memset(fsize->reserved, 0x0, sizeof(fsize->reserved));
 
-	memset(&fse, 0x0, sizeof(fse));
 	fse.index = fsize->index;
 	fse.pad = 0;
 	fse.code = fmt->code;
-	fse.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(sd, pad, enum_frame_size, NULL, &fse);
 	if (ret)
 		return ret;
@@ -2146,7 +2148,6 @@ vpfe_async_bound(struct v4l2_async_notifier *notifier,
 {
 	struct vpfe_device *vpfe = container_of(notifier->v4l2_dev,
 					       struct vpfe_device, v4l2_dev);
-	struct v4l2_subdev_mbus_code_enum mbus_code;
 	struct vpfe_subdev_info *sdinfo;
 	struct vpfe_fmt *fmt;
 	int ret = 0;
@@ -2173,9 +2174,11 @@ vpfe_async_bound(struct v4l2_async_notifier *notifier,
 
 	vpfe->num_active_fmt = 0;
 	for (j = 0, i = 0; (ret != -EINVAL); ++j) {
-		memset(&mbus_code, 0, sizeof(mbus_code));
-		mbus_code.index = j;
-		mbus_code.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		struct v4l2_subdev_mbus_code_enum mbus_code = {
+			.index = j,
+			.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		};
+
 		ret = v4l2_subdev_call(subdev, pad, enum_mbus_code,
 				       NULL, &mbus_code);
 		if (ret)
@@ -2483,7 +2486,7 @@ probe_out_cleanup:
 /*
  * vpfe_remove : It un-register device from V4L2 driver
  */
-static int vpfe_remove(struct platform_device *pdev)
+static void vpfe_remove(struct platform_device *pdev)
 {
 	struct vpfe_device *vpfe = platform_get_drvdata(pdev);
 
@@ -2493,8 +2496,6 @@ static int vpfe_remove(struct platform_device *pdev)
 	v4l2_async_nf_cleanup(&vpfe->notifier);
 	v4l2_device_unregister(&vpfe->v4l2_dev);
 	video_unregister_device(&vpfe->video_dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -2625,7 +2626,7 @@ MODULE_DEVICE_TABLE(of, vpfe_of_match);
 
 static struct platform_driver vpfe_driver = {
 	.probe		= vpfe_probe,
-	.remove		= vpfe_remove,
+	.remove_new	= vpfe_remove,
 	.driver = {
 		.name	= VPFE_MODULE_NAME,
 		.pm	= &vpfe_pm_ops,
diff --git a/drivers/media/platform/ti/cal/cal-video.c b/drivers/media/platform/ti/cal/cal-video.c
index 4eade409d5d3..ca906a9e4222 100644
--- a/drivers/media/platform/ti/cal/cal-video.c
+++ b/drivers/media/platform/ti/cal/cal-video.c
@@ -117,13 +117,13 @@ static int cal_legacy_enum_fmt_vid_cap(struct file *file, void *priv,
 static int __subdev_get_format(struct cal_ctx *ctx,
 			       struct v4l2_mbus_framefmt *fmt)
 {
-	struct v4l2_subdev_format sd_fmt;
+	struct v4l2_subdev_format sd_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.pad = 0,
+	};
 	struct v4l2_mbus_framefmt *mbus_fmt = &sd_fmt.format;
 	int ret;
 
-	sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	sd_fmt.pad = 0;
-
 	ret = v4l2_subdev_call(ctx->phy->source, pad, get_fmt, NULL, &sd_fmt);
 	if (ret)
 		return ret;
@@ -139,12 +139,13 @@ static int __subdev_get_format(struct cal_ctx *ctx,
 static int __subdev_set_format(struct cal_ctx *ctx,
 			       struct v4l2_mbus_framefmt *fmt)
 {
-	struct v4l2_subdev_format sd_fmt;
+	struct v4l2_subdev_format sd_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.pad = 0,
+	};
 	struct v4l2_mbus_framefmt *mbus_fmt = &sd_fmt.format;
 	int ret;
 
-	sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	sd_fmt.pad = 0;
 	*mbus_fmt = *fmt;
 
 	ret = v4l2_subdev_call(ctx->phy->source, pad, set_fmt, NULL, &sd_fmt);
@@ -190,7 +191,9 @@ static int cal_legacy_try_fmt_vid_cap(struct file *file, void *priv,
 {
 	struct cal_ctx *ctx = video_drvdata(file);
 	const struct cal_format_info *fmtinfo;
-	struct v4l2_subdev_frame_size_enum fse;
+	struct v4l2_subdev_frame_size_enum fse = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	int found;
 
 	fmtinfo = find_format_by_pix(ctx, f->fmt.pix.pixelformat);
@@ -209,7 +212,6 @@ static int cal_legacy_try_fmt_vid_cap(struct file *file, void *priv,
 	found = false;
 	fse.pad = 0;
 	fse.code = fmtinfo->code;
-	fse.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	for (fse.index = 0; ; fse.index++) {
 		int ret;
 
@@ -302,7 +304,11 @@ static int cal_legacy_enum_framesizes(struct file *file, void *fh,
 {
 	struct cal_ctx *ctx = video_drvdata(file);
 	const struct cal_format_info *fmtinfo;
-	struct v4l2_subdev_frame_size_enum fse;
+	struct v4l2_subdev_frame_size_enum fse = {
+		.index = fsize->index,
+		.pad = 0,
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	int ret;
 
 	/* check for valid format */
@@ -313,10 +319,7 @@ static int cal_legacy_enum_framesizes(struct file *file, void *fh,
 		return -EINVAL;
 	}
 
-	fse.index = fsize->index;
-	fse.pad = 0;
 	fse.code = fmtinfo->code;
-	fse.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 
 	ret = v4l2_subdev_call(ctx->phy->source, pad, enum_frame_size, NULL,
 			       &fse);
@@ -811,7 +814,6 @@ static const struct v4l2_file_operations cal_fops = {
 
 static int cal_ctx_v4l2_init_formats(struct cal_ctx *ctx)
 {
-	struct v4l2_subdev_mbus_code_enum mbus_code;
 	struct v4l2_mbus_framefmt mbus_fmt;
 	const struct cal_format_info *fmtinfo;
 	unsigned int i, j, k;
@@ -826,10 +828,11 @@ static int cal_ctx_v4l2_init_formats(struct cal_ctx *ctx)
 	ctx->num_active_fmt = 0;
 
 	for (j = 0, i = 0; ; ++j) {
+		struct v4l2_subdev_mbus_code_enum mbus_code = {
+			.index = j,
+			.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		};
 
-		memset(&mbus_code, 0, sizeof(mbus_code));
-		mbus_code.index = j;
-		mbus_code.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 		ret = v4l2_subdev_call(ctx->phy->source, pad, enum_mbus_code,
 				       NULL, &mbus_code);
 		if (ret == -EINVAL)
diff --git a/drivers/media/platform/ti/cal/cal.c b/drivers/media/platform/ti/cal/cal.c
index 1236215ec70e..9c5105223d6b 100644
--- a/drivers/media/platform/ti/cal/cal.c
+++ b/drivers/media/platform/ti/cal/cal.c
@@ -1293,7 +1293,7 @@ error_pm_runtime:
 	return ret;
 }
 
-static int cal_remove(struct platform_device *pdev)
+static void cal_remove(struct platform_device *pdev)
 {
 	struct cal_dev *cal = platform_get_drvdata(pdev);
 	unsigned int i;
@@ -1319,8 +1319,6 @@ static int cal_remove(struct platform_device *pdev)
 	if (ret >= 0)
 		pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static int cal_runtime_resume(struct device *dev)
@@ -1364,7 +1362,7 @@ static const struct dev_pm_ops cal_pm_ops = {
 
 static struct platform_driver cal_pdrv = {
 	.probe		= cal_probe,
-	.remove		= cal_remove,
+	.remove_new	= cal_remove,
 	.driver		= {
 		.name	= CAL_MODULE_NAME,
 		.pm	= &cal_pm_ops,
diff --git a/drivers/media/platform/ti/davinci/vpif.c b/drivers/media/platform/ti/davinci/vpif.c
index 832489822706..63cdfed37bc9 100644
--- a/drivers/media/platform/ti/davinci/vpif.c
+++ b/drivers/media/platform/ti/davinci/vpif.c
@@ -538,7 +538,7 @@ err_put_rpm:
 	return ret;
 }
 
-static int vpif_remove(struct platform_device *pdev)
+static void vpif_remove(struct platform_device *pdev)
 {
 	struct vpif_data *data = platform_get_drvdata(pdev);
 
@@ -551,8 +551,6 @@ static int vpif_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 
 	kfree(data);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -592,7 +590,7 @@ static struct platform_driver vpif_driver = {
 		.name	= VPIF_DRIVER_NAME,
 		.pm	= vpif_pm_ops,
 	},
-	.remove = vpif_remove,
+	.remove_new = vpif_remove,
 	.probe = vpif_probe,
 };
 
diff --git a/drivers/media/platform/ti/davinci/vpif_capture.c b/drivers/media/platform/ti/davinci/vpif_capture.c
index 580723333fcc..44d269d6038c 100644
--- a/drivers/media/platform/ti/davinci/vpif_capture.c
+++ b/drivers/media/platform/ti/davinci/vpif_capture.c
@@ -1714,7 +1714,7 @@ cleanup:
  *
  * The vidoe device is unregistered
  */
-static int vpif_remove(struct platform_device *device)
+static void vpif_remove(struct platform_device *device)
 {
 	struct channel_obj *ch;
 	int i;
@@ -1732,7 +1732,6 @@ static int vpif_remove(struct platform_device *device)
 		video_unregister_device(&ch->video_dev);
 		kfree(vpif_obj.dev[i]);
 	}
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1815,7 +1814,7 @@ static __refdata struct platform_driver vpif_driver = {
 		.pm	= &vpif_pm_ops,
 	},
 	.probe = vpif_probe,
-	.remove = vpif_remove,
+	.remove_new = vpif_remove,
 };
 
 module_platform_driver(vpif_driver);
diff --git a/drivers/media/platform/ti/davinci/vpif_display.c b/drivers/media/platform/ti/davinci/vpif_display.c
index b2df81603f62..f8ec2991c667 100644
--- a/drivers/media/platform/ti/davinci/vpif_display.c
+++ b/drivers/media/platform/ti/davinci/vpif_display.c
@@ -1305,7 +1305,7 @@ vpif_free:
 /*
  * vpif_remove: It un-register channels from V4L2 driver
  */
-static int vpif_remove(struct platform_device *device)
+static void vpif_remove(struct platform_device *device)
 {
 	struct channel_obj *ch;
 	int i;
@@ -1321,8 +1321,6 @@ static int vpif_remove(struct platform_device *device)
 		video_unregister_device(&ch->video_dev);
 	}
 	free_vpif_objs();
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1399,7 +1397,7 @@ static __refdata struct platform_driver vpif_driver = {
 			.pm	= &vpif_pm_ops,
 	},
 	.probe	= vpif_probe,
-	.remove	= vpif_remove,
+	.remove_new = vpif_remove,
 };
 
 module_platform_driver(vpif_driver);
diff --git a/drivers/media/platform/ti/omap/omap_vout.c b/drivers/media/platform/ti/omap/omap_vout.c
index 3e0d9af7ffec..4143274089c3 100644
--- a/drivers/media/platform/ti/omap/omap_vout.c
+++ b/drivers/media/platform/ti/omap/omap_vout.c
@@ -1569,7 +1569,7 @@ static void omap_vout_cleanup_device(struct omap_vout_device *vout)
 	kfree(vout);
 }
 
-static int omap_vout_remove(struct platform_device *pdev)
+static void omap_vout_remove(struct platform_device *pdev)
 {
 	int k;
 	struct v4l2_device *v4l2_dev = platform_get_drvdata(pdev);
@@ -1587,7 +1587,6 @@ static int omap_vout_remove(struct platform_device *pdev)
 		omap_dss_put_device(vid_dev->displays[k]);
 	}
 	kfree(vid_dev);
-	return 0;
 }
 
 static int __init omap_vout_probe(struct platform_device *pdev)
@@ -1721,7 +1720,7 @@ static struct platform_driver omap_vout_driver = {
 	.driver = {
 		.name = VOUT_NAME,
 	},
-	.remove = omap_vout_remove,
+	.remove_new = omap_vout_remove,
 };
 
 static int __init omap_vout_init(void)
diff --git a/drivers/media/platform/ti/omap3isp/isp.c b/drivers/media/platform/ti/omap3isp/isp.c
index e7327e38482d..f3aaa9e76492 100644
--- a/drivers/media/platform/ti/omap3isp/isp.c
+++ b/drivers/media/platform/ti/omap3isp/isp.c
@@ -1997,7 +1997,7 @@ error:
  *
  * Always returns 0.
  */
-static int isp_remove(struct platform_device *pdev)
+static void isp_remove(struct platform_device *pdev)
 {
 	struct isp_device *isp = platform_get_drvdata(pdev);
 
@@ -2014,8 +2014,6 @@ static int isp_remove(struct platform_device *pdev)
 	v4l2_async_nf_cleanup(&isp->notifier);
 
 	kfree(isp);
-
-	return 0;
 }
 
 enum isp_of_phy {
@@ -2476,7 +2474,7 @@ MODULE_DEVICE_TABLE(of, omap3isp_of_table);
 
 static struct platform_driver omap3isp_driver = {
 	.probe = isp_probe,
-	.remove = isp_remove,
+	.remove_new = isp_remove,
 	.id_table = omap3isp_id_table,
 	.driver = {
 		.name = "omap3isp",
diff --git a/drivers/media/platform/ti/omap3isp/ispccdc.c b/drivers/media/platform/ti/omap3isp/ispccdc.c
index 11afb8aec292..fdcdffe5fecb 100644
--- a/drivers/media/platform/ti/omap3isp/ispccdc.c
+++ b/drivers/media/platform/ti/omap3isp/ispccdc.c
@@ -1118,7 +1118,9 @@ static void ccdc_configure(struct isp_ccdc_device *ccdc)
 	struct v4l2_mbus_framefmt *format;
 	const struct v4l2_rect *crop;
 	const struct isp_format_info *fmt_info;
-	struct v4l2_subdev_format fmt_src;
+	struct v4l2_subdev_format fmt_src = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	unsigned int depth_out;
 	unsigned int depth_in = 0;
 	struct media_pad *pad;
@@ -1150,7 +1152,6 @@ static void ccdc_configure(struct isp_ccdc_device *ccdc)
 	 * input format is a non-BT.656 YUV variant.
 	 */
 	fmt_src.pad = pad->index;
-	fmt_src.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	if (!v4l2_subdev_call(sensor, pad, get_fmt, NULL, &fmt_src)) {
 		fmt_info = omap3isp_video_format_info(fmt_src.format.code);
 		depth_in = fmt_info->width;
diff --git a/drivers/media/platform/ti/omap3isp/ispvideo.c b/drivers/media/platform/ti/omap3isp/ispvideo.c
index ddc7d08d4f96..daca689dc082 100644
--- a/drivers/media/platform/ti/omap3isp/ispvideo.c
+++ b/drivers/media/platform/ti/omap3isp/ispvideo.c
@@ -268,7 +268,9 @@ static int isp_video_get_graph_data(struct isp_video *video,
 static int
 __isp_video_get_format(struct isp_video *video, struct v4l2_format *format)
 {
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *subdev;
 	u32 pad;
 	int ret;
@@ -278,7 +280,6 @@ __isp_video_get_format(struct isp_video *video, struct v4l2_format *format)
 		return -EINVAL;
 
 	fmt.pad = pad;
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 
 	mutex_lock(&video->mutex);
 	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &fmt);
@@ -731,7 +732,9 @@ static int
 isp_video_try_format(struct file *file, void *fh, struct v4l2_format *format)
 {
 	struct isp_video *video = video_drvdata(file);
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *subdev;
 	u32 pad;
 	int ret;
@@ -746,7 +749,6 @@ isp_video_try_format(struct file *file, void *fh, struct v4l2_format *format)
 	isp_video_pix_to_mbus(&format->fmt.pix, &fmt.format);
 
 	fmt.pad = pad;
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &fmt);
 	if (ret)
 		return ret == -ENOIOCTLCMD ? -ENOTTY : ret;
@@ -759,7 +761,9 @@ static int
 isp_video_get_selection(struct file *file, void *fh, struct v4l2_selection *sel)
 {
 	struct isp_video *video = video_drvdata(file);
-	struct v4l2_subdev_format format;
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *subdev;
 	struct v4l2_subdev_selection sdsel = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
@@ -799,7 +803,6 @@ isp_video_get_selection(struct file *file, void *fh, struct v4l2_selection *sel)
 		return ret;
 
 	format.pad = pad;
-	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &format);
 	if (ret < 0)
 		return ret == -ENOIOCTLCMD ? -ENOTTY : ret;
@@ -957,7 +960,9 @@ static int isp_video_check_external_subdevs(struct isp_video *video,
 	struct media_pad *source_pad;
 	struct media_entity *source = NULL;
 	struct media_entity *sink;
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_ext_controls ctrls;
 	struct v4l2_ext_control ctrl;
 	unsigned int i;
@@ -993,7 +998,6 @@ static int isp_video_check_external_subdevs(struct isp_video *video,
 	pipe->external = media_entity_to_v4l2_subdev(source);
 
 	fmt.pad = source_pad->index;
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(media_entity_to_v4l2_subdev(sink),
 			       pad, get_fmt, NULL, &fmt);
 	if (unlikely(ret < 0)) {
diff --git a/drivers/media/platform/ti/vpe/vpe.c b/drivers/media/platform/ti/vpe/vpe.c
index 5b1c5d96a407..6848cbc82f52 100644
--- a/drivers/media/platform/ti/vpe/vpe.c
+++ b/drivers/media/platform/ti/vpe/vpe.c
@@ -2622,7 +2622,7 @@ v4l2_dev_unreg:
 	return ret;
 }
 
-static int vpe_remove(struct platform_device *pdev)
+static void vpe_remove(struct platform_device *pdev)
 {
 	struct vpe_dev *dev = platform_get_drvdata(pdev);
 
@@ -2635,8 +2635,6 @@ static int vpe_remove(struct platform_device *pdev)
 	vpe_set_clock_enable(dev, 0);
 	vpe_runtime_put(pdev);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 #if defined(CONFIG_OF)
@@ -2651,7 +2649,7 @@ MODULE_DEVICE_TABLE(of, vpe_of_match);
 
 static struct platform_driver vpe_pdrv = {
 	.probe		= vpe_probe,
-	.remove		= vpe_remove,
+	.remove_new	= vpe_remove,
 	.driver		= {
 		.name	= VPE_MODULE_NAME,
 		.of_match_table = of_match_ptr(vpe_of_match),
diff --git a/drivers/media/platform/verisilicon/hantro_drv.c b/drivers/media/platform/verisilicon/hantro_drv.c
index b0aeedae7b65..09c74a573ddb 100644
--- a/drivers/media/platform/verisilicon/hantro_drv.c
+++ b/drivers/media/platform/verisilicon/hantro_drv.c
@@ -251,11 +251,6 @@ queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq)
 
 static int hantro_try_ctrl(struct v4l2_ctrl *ctrl)
 {
-	struct hantro_ctx *ctx;
-
-	ctx = container_of(ctrl->handler,
-			   struct hantro_ctx, ctrl_handler);
-
 	if (ctrl->id == V4L2_CID_STATELESS_H264_SPS) {
 		const struct v4l2_ctrl_h264_sps *sps = ctrl->p_new.p_h264_sps;
 
@@ -274,8 +269,6 @@ static int hantro_try_ctrl(struct v4l2_ctrl *ctrl)
 		if (sps->bit_depth_luma_minus8 != 0 && sps->bit_depth_luma_minus8 != 2)
 			/* Only 8-bit and 10-bit are supported */
 			return -EINVAL;
-
-		ctx->bit_depth = sps->bit_depth_luma_minus8 + 8;
 	} else if (ctrl->id == V4L2_CID_STATELESS_VP9_FRAME) {
 		const struct v4l2_ctrl_vp9_frame *dec_params = ctrl->p_new.p_vp9_frame;
 
@@ -314,9 +307,38 @@ static int hantro_vp9_s_ctrl(struct v4l2_ctrl *ctrl)
 			   struct hantro_ctx, ctrl_handler);
 
 	switch (ctrl->id) {
-	case V4L2_CID_STATELESS_VP9_FRAME:
-		ctx->bit_depth = ctrl->p_new.p_vp9_frame->bit_depth;
-		break;
+	case V4L2_CID_STATELESS_VP9_FRAME: {
+		int bit_depth = ctrl->p_new.p_vp9_frame->bit_depth;
+
+		if (ctx->bit_depth == bit_depth)
+			return 0;
+
+		return hantro_reset_raw_fmt(ctx, bit_depth);
+	}
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int hantro_hevc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct hantro_ctx *ctx;
+
+	ctx = container_of(ctrl->handler,
+			   struct hantro_ctx, ctrl_handler);
+
+	switch (ctrl->id) {
+	case V4L2_CID_STATELESS_HEVC_SPS: {
+		const struct v4l2_ctrl_hevc_sps *sps = ctrl->p_new.p_hevc_sps;
+		int bit_depth = sps->bit_depth_luma_minus8 + 8;
+
+		if (ctx->bit_depth == bit_depth)
+			return 0;
+
+		return hantro_reset_raw_fmt(ctx, bit_depth);
+	}
 	default:
 		return -EINVAL;
 	}
@@ -336,6 +358,11 @@ static const struct v4l2_ctrl_ops hantro_vp9_ctrl_ops = {
 	.s_ctrl = hantro_vp9_s_ctrl,
 };
 
+static const struct v4l2_ctrl_ops hantro_hevc_ctrl_ops = {
+	.try_ctrl = hantro_try_ctrl,
+	.s_ctrl = hantro_hevc_s_ctrl,
+};
+
 #define HANTRO_JPEG_ACTIVE_MARKERS	(V4L2_JPEG_ACTIVE_MARKER_APP0 | \
 					 V4L2_JPEG_ACTIVE_MARKER_COM | \
 					 V4L2_JPEG_ACTIVE_MARKER_DQT | \
@@ -470,7 +497,7 @@ static const struct hantro_ctrl controls[] = {
 		.codec = HANTRO_HEVC_DECODER,
 		.cfg = {
 			.id = V4L2_CID_STATELESS_HEVC_SPS,
-			.ops = &hantro_ctrl_ops,
+			.ops = &hantro_hevc_ctrl_ops,
 		},
 	}, {
 		.codec = HANTRO_HEVC_DECODER,
@@ -945,7 +972,7 @@ static int hantro_probe(struct platform_device *pdev)
 			return PTR_ERR(vpu->clocks[0].clk);
 	}
 
-	vpu->resets = devm_reset_control_array_get(&pdev->dev, false, true);
+	vpu->resets = devm_reset_control_array_get_optional_exclusive(&pdev->dev);
 	if (IS_ERR(vpu->resets))
 		return PTR_ERR(vpu->resets);
 
@@ -1093,7 +1120,7 @@ err_pm_disable:
 	return ret;
 }
 
-static int hantro_remove(struct platform_device *pdev)
+static void hantro_remove(struct platform_device *pdev)
 {
 	struct hantro_dev *vpu = platform_get_drvdata(pdev);
 
@@ -1109,7 +1136,6 @@ static int hantro_remove(struct platform_device *pdev)
 	reset_control_assert(vpu->resets);
 	pm_runtime_dont_use_autosuspend(vpu->dev);
 	pm_runtime_disable(vpu->dev);
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -1132,7 +1158,7 @@ static const struct dev_pm_ops hantro_pm_ops = {
 
 static struct platform_driver hantro_driver = {
 	.probe = hantro_probe,
-	.remove = hantro_remove,
+	.remove_new = hantro_remove,
 	.driver = {
 		   .name = DRIVER_NAME,
 		   .of_match_table = of_match_ptr(of_hantro_match),
diff --git a/drivers/media/platform/verisilicon/hantro_postproc.c b/drivers/media/platform/verisilicon/hantro_postproc.c
index 09d8cf942689..6437423ccf3a 100644
--- a/drivers/media/platform/verisilicon/hantro_postproc.c
+++ b/drivers/media/platform/verisilicon/hantro_postproc.c
@@ -197,7 +197,7 @@ int hantro_postproc_alloc(struct hantro_ctx *ctx)
 	unsigned int i, buf_size;
 
 	/* this should always pick native format */
-	fmt = hantro_get_default_fmt(ctx, false);
+	fmt = hantro_get_default_fmt(ctx, false, ctx->bit_depth);
 	if (!fmt)
 		return -EINVAL;
 	v4l2_fill_pixfmt_mp(&pix_mp, fmt->fourcc, ctx->src_fmt.width,
diff --git a/drivers/media/platform/verisilicon/hantro_v4l2.c b/drivers/media/platform/verisilicon/hantro_v4l2.c
index c0d427956210..835518534e3b 100644
--- a/drivers/media/platform/verisilicon/hantro_v4l2.c
+++ b/drivers/media/platform/verisilicon/hantro_v4l2.c
@@ -28,6 +28,8 @@
 #include "hantro_hw.h"
 #include "hantro_v4l2.h"
 
+#define  HANTRO_DEFAULT_BIT_DEPTH 8
+
 static int hantro_set_fmt_out(struct hantro_ctx *ctx,
 			      struct v4l2_pix_format_mplane *pix_mp);
 static int hantro_set_fmt_cap(struct hantro_ctx *ctx,
@@ -76,18 +78,13 @@ int hantro_get_format_depth(u32 fourcc)
 }
 
 static bool
-hantro_check_depth_match(const struct hantro_ctx *ctx,
-			 const struct hantro_fmt *fmt)
+hantro_check_depth_match(const struct hantro_fmt *fmt, int bit_depth)
 {
-	int fmt_depth, ctx_depth = 8;
+	int fmt_depth;
 
 	if (!fmt->match_depth && !fmt->postprocessed)
 		return true;
 
-	/* 0 means default depth, which is 8 */
-	if (ctx->bit_depth)
-		ctx_depth = ctx->bit_depth;
-
 	fmt_depth = hantro_get_format_depth(fmt->fourcc);
 
 	/*
@@ -95,9 +92,9 @@ hantro_check_depth_match(const struct hantro_ctx *ctx,
 	 * It may be possible to relax that on some HW.
 	 */
 	if (!fmt->match_depth)
-		return fmt_depth <= ctx_depth;
+		return fmt_depth <= bit_depth;
 
-	return fmt_depth == ctx_depth;
+	return fmt_depth == bit_depth;
 }
 
 static const struct hantro_fmt *
@@ -119,7 +116,7 @@ hantro_find_format(const struct hantro_ctx *ctx, u32 fourcc)
 }
 
 const struct hantro_fmt *
-hantro_get_default_fmt(const struct hantro_ctx *ctx, bool bitstream)
+hantro_get_default_fmt(const struct hantro_ctx *ctx, bool bitstream, int bit_depth)
 {
 	const struct hantro_fmt *formats;
 	unsigned int i, num_fmts;
@@ -128,7 +125,7 @@ hantro_get_default_fmt(const struct hantro_ctx *ctx, bool bitstream)
 	for (i = 0; i < num_fmts; i++) {
 		if (bitstream == (formats[i].codec_mode !=
 				  HANTRO_MODE_NONE) &&
-		    hantro_check_depth_match(ctx, &formats[i]))
+		    hantro_check_depth_match(&formats[i], bit_depth))
 			return &formats[i];
 	}
 	return NULL;
@@ -204,7 +201,7 @@ static int vidioc_enum_fmt(struct file *file, void *priv,
 
 		if (skip_mode_none == mode_none)
 			continue;
-		if (!hantro_check_depth_match(ctx, fmt))
+		if (!hantro_check_depth_match(fmt, ctx->bit_depth))
 			continue;
 		if (j == f->index) {
 			f->pixelformat = fmt->fourcc;
@@ -224,7 +221,7 @@ static int vidioc_enum_fmt(struct file *file, void *priv,
 	for (i = 0; i < num_fmts; i++) {
 		fmt = &formats[i];
 
-		if (!hantro_check_depth_match(ctx, fmt))
+		if (!hantro_check_depth_match(fmt, ctx->bit_depth))
 			continue;
 		if (j == f->index) {
 			f->pixelformat = fmt->fourcc;
@@ -278,7 +275,7 @@ static int hantro_try_fmt(const struct hantro_ctx *ctx,
 			  struct v4l2_pix_format_mplane *pix_mp,
 			  enum v4l2_buf_type type)
 {
-	const struct hantro_fmt *fmt, *vpu_fmt;
+	const struct hantro_fmt *fmt;
 	bool capture = V4L2_TYPE_IS_CAPTURE(type);
 	bool coded;
 
@@ -292,17 +289,13 @@ static int hantro_try_fmt(const struct hantro_ctx *ctx,
 
 	fmt = hantro_find_format(ctx, pix_mp->pixelformat);
 	if (!fmt) {
-		fmt = hantro_get_default_fmt(ctx, coded);
+		fmt = hantro_get_default_fmt(ctx, coded, HANTRO_DEFAULT_BIT_DEPTH);
 		pix_mp->pixelformat = fmt->fourcc;
 	}
 
 	if (coded) {
 		pix_mp->num_planes = 1;
-		vpu_fmt = fmt;
-	} else if (ctx->is_encoder) {
-		vpu_fmt = ctx->vpu_dst_fmt;
-	} else {
-		vpu_fmt = fmt;
+	} else if (!ctx->is_encoder) {
 		/*
 		 * Width/height on the CAPTURE end of a decoder are ignored and
 		 * replaced by the OUTPUT ones.
@@ -314,7 +307,7 @@ static int hantro_try_fmt(const struct hantro_ctx *ctx,
 	pix_mp->field = V4L2_FIELD_NONE;
 
 	v4l2_apply_frmsize_constraints(&pix_mp->width, &pix_mp->height,
-				       &vpu_fmt->frmsize);
+				       &fmt->frmsize);
 
 	if (!coded) {
 		/* Fill remaining fields */
@@ -378,58 +371,55 @@ static void
 hantro_reset_encoded_fmt(struct hantro_ctx *ctx)
 {
 	const struct hantro_fmt *vpu_fmt;
-	struct v4l2_pix_format_mplane *fmt;
-
-	vpu_fmt = hantro_get_default_fmt(ctx, true);
+	struct v4l2_pix_format_mplane fmt;
 
-	if (ctx->is_encoder) {
-		ctx->vpu_dst_fmt = vpu_fmt;
-		fmt = &ctx->dst_fmt;
-	} else {
-		ctx->vpu_src_fmt = vpu_fmt;
-		fmt = &ctx->src_fmt;
-	}
+	vpu_fmt = hantro_get_default_fmt(ctx, true, HANTRO_DEFAULT_BIT_DEPTH);
+	if (!vpu_fmt)
+		return;
 
-	hantro_reset_fmt(fmt, vpu_fmt);
-	fmt->width = vpu_fmt->frmsize.min_width;
-	fmt->height = vpu_fmt->frmsize.min_height;
+	hantro_reset_fmt(&fmt, vpu_fmt);
+	fmt.width = vpu_fmt->frmsize.min_width;
+	fmt.height = vpu_fmt->frmsize.min_height;
 	if (ctx->is_encoder)
-		hantro_set_fmt_cap(ctx, fmt);
+		hantro_set_fmt_cap(ctx, &fmt);
 	else
-		hantro_set_fmt_out(ctx, fmt);
+		hantro_set_fmt_out(ctx, &fmt);
 }
 
-static void
-hantro_reset_raw_fmt(struct hantro_ctx *ctx)
+int
+hantro_reset_raw_fmt(struct hantro_ctx *ctx, int bit_depth)
 {
 	const struct hantro_fmt *raw_vpu_fmt;
-	struct v4l2_pix_format_mplane *raw_fmt, *encoded_fmt;
+	struct v4l2_pix_format_mplane raw_fmt, *encoded_fmt;
+	int ret;
 
-	raw_vpu_fmt = hantro_get_default_fmt(ctx, false);
+	raw_vpu_fmt = hantro_get_default_fmt(ctx, false, bit_depth);
+	if (!raw_vpu_fmt)
+		return -EINVAL;
 
-	if (ctx->is_encoder) {
-		ctx->vpu_src_fmt = raw_vpu_fmt;
-		raw_fmt = &ctx->src_fmt;
+	if (ctx->is_encoder)
 		encoded_fmt = &ctx->dst_fmt;
-	} else {
-		ctx->vpu_dst_fmt = raw_vpu_fmt;
-		raw_fmt = &ctx->dst_fmt;
+	else
 		encoded_fmt = &ctx->src_fmt;
-	}
 
-	hantro_reset_fmt(raw_fmt, raw_vpu_fmt);
-	raw_fmt->width = encoded_fmt->width;
-	raw_fmt->height = encoded_fmt->height;
+	hantro_reset_fmt(&raw_fmt, raw_vpu_fmt);
+	raw_fmt.width = encoded_fmt->width;
+	raw_fmt.height = encoded_fmt->height;
 	if (ctx->is_encoder)
-		hantro_set_fmt_out(ctx, raw_fmt);
+		ret = hantro_set_fmt_out(ctx, &raw_fmt);
 	else
-		hantro_set_fmt_cap(ctx, raw_fmt);
+		ret = hantro_set_fmt_cap(ctx, &raw_fmt);
+
+	if (!ret)
+		ctx->bit_depth = bit_depth;
+
+	return ret;
 }
 
 void hantro_reset_fmts(struct hantro_ctx *ctx)
 {
 	hantro_reset_encoded_fmt(ctx);
-	hantro_reset_raw_fmt(ctx);
+	hantro_reset_raw_fmt(ctx, HANTRO_DEFAULT_BIT_DEPTH);
 }
 
 static void
@@ -529,7 +519,7 @@ static int hantro_set_fmt_out(struct hantro_ctx *ctx,
 	 * changes to the raw format.
 	 */
 	if (!ctx->is_encoder)
-		hantro_reset_raw_fmt(ctx);
+		hantro_reset_raw_fmt(ctx, hantro_get_format_depth(pix_mp->pixelformat));
 
 	/* Colorimetry information are always propagated. */
 	ctx->dst_fmt.colorspace = pix_mp->colorspace;
@@ -592,7 +582,7 @@ static int hantro_set_fmt_cap(struct hantro_ctx *ctx,
 	 * changes to the raw format.
 	 */
 	if (ctx->is_encoder)
-		hantro_reset_raw_fmt(ctx);
+		hantro_reset_raw_fmt(ctx, HANTRO_DEFAULT_BIT_DEPTH);
 
 	/* Colorimetry information are always propagated. */
 	ctx->src_fmt.colorspace = pix_mp->colorspace;
diff --git a/drivers/media/platform/verisilicon/hantro_v4l2.h b/drivers/media/platform/verisilicon/hantro_v4l2.h
index 64f6f57e9d7a..9ea2fef57dcd 100644
--- a/drivers/media/platform/verisilicon/hantro_v4l2.h
+++ b/drivers/media/platform/verisilicon/hantro_v4l2.h
@@ -21,9 +21,10 @@
 extern const struct v4l2_ioctl_ops hantro_ioctl_ops;
 extern const struct vb2_ops hantro_queue_ops;
 
+int hantro_reset_raw_fmt(struct hantro_ctx *ctx, int bit_depth);
 void hantro_reset_fmts(struct hantro_ctx *ctx);
 int hantro_get_format_depth(u32 fourcc);
 const struct hantro_fmt *
-hantro_get_default_fmt(const struct hantro_ctx *ctx, bool bitstream);
+hantro_get_default_fmt(const struct hantro_ctx *ctx, bool bitstream, int bit_depth);
 
 #endif /* HANTRO_V4L2_H_ */
diff --git a/drivers/media/platform/verisilicon/imx8m_vpu_hw.c b/drivers/media/platform/verisilicon/imx8m_vpu_hw.c
index b390228fd3b4..f850d8bddef6 100644
--- a/drivers/media/platform/verisilicon/imx8m_vpu_hw.c
+++ b/drivers/media/platform/verisilicon/imx8m_vpu_hw.c
@@ -152,6 +152,7 @@ static const struct hantro_fmt imx8m_vpu_g2_postproc_fmts[] = {
 	{
 		.fourcc = V4L2_PIX_FMT_NV12,
 		.codec_mode = HANTRO_MODE_NONE,
+		.match_depth = true,
 		.postprocessed = true,
 		.frmsize = {
 			.min_width = FMT_MIN_WIDTH,
@@ -165,6 +166,7 @@ static const struct hantro_fmt imx8m_vpu_g2_postproc_fmts[] = {
 	{
 		.fourcc = V4L2_PIX_FMT_P010,
 		.codec_mode = HANTRO_MODE_NONE,
+		.match_depth = true,
 		.postprocessed = true,
 		.frmsize = {
 			.min_width = FMT_MIN_WIDTH,
diff --git a/drivers/media/platform/via/via-camera.c b/drivers/media/platform/via/via-camera.c
index 95483c84c3f2..450254403fa8 100644
--- a/drivers/media/platform/via/via-camera.c
+++ b/drivers/media/platform/via/via-camera.c
@@ -845,8 +845,8 @@ static int viacam_do_try_fmt(struct via_camera *cam,
 	int ret;
 	struct v4l2_subdev_pad_config pad_cfg;
 	struct v4l2_subdev_state pad_state = {
-		.pads = &pad_cfg
-		};
+		.pads = &pad_cfg,
+	};
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_TRY,
 	};
@@ -1208,7 +1208,9 @@ static int viacam_probe(struct platform_device *pdev)
 	 * Convince the system that we can do DMA.
 	 */
 	pdev->dev.dma_mask = &viadev->pdev->dma_mask;
-	dma_set_mask(&pdev->dev, 0xffffffff);
+	ret = dma_set_mask(&pdev->dev, 0xffffffff);
+	if (ret)
+		goto out_ctrl_hdl_free;
 	/*
 	 * Fire up the capture port.  The write to 0x78 looks purely
 	 * OLPCish; any system will need to tweak 0x1e.
@@ -1294,7 +1296,7 @@ out_free:
 	return ret;
 }
 
-static int viacam_remove(struct platform_device *pdev)
+static void viacam_remove(struct platform_device *pdev)
 {
 	struct via_camera *cam = via_cam_info;
 	struct viafb_dev *viadev = pdev->dev.platform_data;
@@ -1309,7 +1311,6 @@ static int viacam_remove(struct platform_device *pdev)
 	v4l2_ctrl_handler_free(&cam->ctrl_handler);
 	kfree(cam);
 	via_cam_info = NULL;
-	return 0;
 }
 
 static struct platform_driver viacam_driver = {
@@ -1317,7 +1318,7 @@ static struct platform_driver viacam_driver = {
 		.name = "viafb-camera",
 	},
 	.probe = viacam_probe,
-	.remove = viacam_remove,
+	.remove_new = viacam_remove,
 };
 
 module_platform_driver(viacam_driver);
diff --git a/drivers/media/platform/video-mux.c b/drivers/media/platform/video-mux.c
index 71d97042a470..1d9f32e5a917 100644
--- a/drivers/media/platform/video-mux.c
+++ b/drivers/media/platform/video-mux.c
@@ -481,7 +481,7 @@ static int video_mux_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int video_mux_remove(struct platform_device *pdev)
+static void video_mux_remove(struct platform_device *pdev)
 {
 	struct video_mux *vmux = platform_get_drvdata(pdev);
 	struct v4l2_subdev *sd = &vmux->subdev;
@@ -490,8 +490,6 @@ static int video_mux_remove(struct platform_device *pdev)
 	v4l2_async_nf_cleanup(&vmux->notifier);
 	v4l2_async_unregister_subdev(sd);
 	media_entity_cleanup(&sd->entity);
-
-	return 0;
 }
 
 static const struct of_device_id video_mux_dt_ids[] = {
@@ -502,7 +500,7 @@ MODULE_DEVICE_TABLE(of, video_mux_dt_ids);
 
 static struct platform_driver video_mux_driver = {
 	.probe		= video_mux_probe,
-	.remove		= video_mux_remove,
+	.remove_new	= video_mux_remove,
 	.driver		= {
 		.of_match_table = video_mux_dt_ids,
 		.name = "video-mux",
diff --git a/drivers/media/platform/xilinx/xilinx-csi2rxss.c b/drivers/media/platform/xilinx/xilinx-csi2rxss.c
index d8a23f18cfbc..5b53745fe44e 100644
--- a/drivers/media/platform/xilinx/xilinx-csi2rxss.c
+++ b/drivers/media/platform/xilinx/xilinx-csi2rxss.c
@@ -1059,7 +1059,7 @@ err_clk_put:
 	return ret;
 }
 
-static int xcsi2rxss_remove(struct platform_device *pdev)
+static void xcsi2rxss_remove(struct platform_device *pdev)
 {
 	struct xcsi2rxss_state *xcsi2rxss = platform_get_drvdata(pdev);
 	struct v4l2_subdev *subdev = &xcsi2rxss->subdev;
@@ -1070,8 +1070,6 @@ static int xcsi2rxss_remove(struct platform_device *pdev)
 	mutex_destroy(&xcsi2rxss->lock);
 	clk_bulk_disable_unprepare(num_clks, xcsi2rxss->clks);
 	clk_bulk_put(num_clks, xcsi2rxss->clks);
-
-	return 0;
 }
 
 static const struct of_device_id xcsi2rxss_of_id_table[] = {
@@ -1086,7 +1084,7 @@ static struct platform_driver xcsi2rxss_driver = {
 		.of_match_table	= xcsi2rxss_of_id_table,
 	},
 	.probe			= xcsi2rxss_probe,
-	.remove			= xcsi2rxss_remove,
+	.remove_new		= xcsi2rxss_remove,
 };
 
 module_platform_driver(xcsi2rxss_driver);
diff --git a/drivers/media/platform/xilinx/xilinx-dma.c b/drivers/media/platform/xilinx/xilinx-dma.c
index fee02c8c85fd..80d6f5b072ea 100644
--- a/drivers/media/platform/xilinx/xilinx-dma.c
+++ b/drivers/media/platform/xilinx/xilinx-dma.c
@@ -56,7 +56,9 @@ xvip_dma_remote_subdev(struct media_pad *local, u32 *pad)
 
 static int xvip_dma_verify_format(struct xvip_dma *dma)
 {
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *subdev;
 	int ret;
 
@@ -64,7 +66,6 @@ static int xvip_dma_verify_format(struct xvip_dma *dma)
 	if (subdev == NULL)
 		return -EPIPE;
 
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &fmt);
 	if (ret < 0)
 		return ret == -ENOIOCTLCMD ? -EINVAL : ret;
diff --git a/drivers/media/platform/xilinx/xilinx-tpg.c b/drivers/media/platform/xilinx/xilinx-tpg.c
index 0f2d5a0edf0c..80353ca44402 100644
--- a/drivers/media/platform/xilinx/xilinx-tpg.c
+++ b/drivers/media/platform/xilinx/xilinx-tpg.c
@@ -894,7 +894,7 @@ error_resource:
 	return ret;
 }
 
-static int xtpg_remove(struct platform_device *pdev)
+static void xtpg_remove(struct platform_device *pdev)
 {
 	struct xtpg_device *xtpg = platform_get_drvdata(pdev);
 	struct v4l2_subdev *subdev = &xtpg->xvip.subdev;
@@ -904,8 +904,6 @@ static int xtpg_remove(struct platform_device *pdev)
 	media_entity_cleanup(&subdev->entity);
 
 	xvip_cleanup_resources(&xtpg->xvip);
-
-	return 0;
 }
 
 static SIMPLE_DEV_PM_OPS(xtpg_pm_ops, xtpg_pm_suspend, xtpg_pm_resume);
@@ -923,7 +921,7 @@ static struct platform_driver xtpg_driver = {
 		.of_match_table	= xtpg_of_id_table,
 	},
 	.probe			= xtpg_probe,
-	.remove			= xtpg_remove,
+	.remove_new		= xtpg_remove,
 };
 
 module_platform_driver(xtpg_driver);
diff --git a/drivers/media/platform/xilinx/xilinx-vipp.c b/drivers/media/platform/xilinx/xilinx-vipp.c
index 0a16c218a50a..3123216b3f70 100644
--- a/drivers/media/platform/xilinx/xilinx-vipp.c
+++ b/drivers/media/platform/xilinx/xilinx-vipp.c
@@ -617,14 +617,12 @@ error:
 	return ret;
 }
 
-static int xvip_composite_remove(struct platform_device *pdev)
+static void xvip_composite_remove(struct platform_device *pdev)
 {
 	struct xvip_composite_device *xdev = platform_get_drvdata(pdev);
 
 	xvip_graph_cleanup(xdev);
 	xvip_composite_v4l2_cleanup(xdev);
-
-	return 0;
 }
 
 static const struct of_device_id xvip_composite_of_id_table[] = {
@@ -639,7 +637,7 @@ static struct platform_driver xvip_composite_driver = {
 		.of_match_table = xvip_composite_of_id_table,
 	},
 	.probe = xvip_composite_probe,
-	.remove = xvip_composite_remove,
+	.remove_new = xvip_composite_remove,
 };
 
 module_platform_driver(xvip_composite_driver);
diff --git a/drivers/media/platform/xilinx/xilinx-vtc.c b/drivers/media/platform/xilinx/xilinx-vtc.c
index 0ae0208d7529..dda70719f004 100644
--- a/drivers/media/platform/xilinx/xilinx-vtc.c
+++ b/drivers/media/platform/xilinx/xilinx-vtc.c
@@ -254,7 +254,7 @@ struct xvtc_device *xvtc_of_get(struct device_node *np)
 	struct xvtc_device *found = NULL;
 	struct xvtc_device *xvtc;
 
-	if (!of_find_property(np, "xlnx,vtc", NULL))
+	if (!of_property_present(np, "xlnx,vtc"))
 		return NULL;
 
 	xvtc_node = of_parse_phandle(np, "xlnx,vtc", 0);
@@ -344,15 +344,13 @@ static int xvtc_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int xvtc_remove(struct platform_device *pdev)
+static void xvtc_remove(struct platform_device *pdev)
 {
 	struct xvtc_device *xvtc = platform_get_drvdata(pdev);
 
 	xvtc_unregister_device(xvtc);
 
 	xvip_cleanup_resources(&xvtc->xvip);
-
-	return 0;
 }
 
 static const struct of_device_id xvtc_of_id_table[] = {
@@ -367,7 +365,7 @@ static struct platform_driver xvtc_driver = {
 		.of_match_table = xvtc_of_id_table,
 	},
 	.probe = xvtc_probe,
-	.remove = xvtc_remove,
+	.remove_new = xvtc_remove,
 };
 
 module_platform_driver(xvtc_driver);
diff --git a/drivers/media/radio/radio-shark.c b/drivers/media/radio/radio-shark.c
index 8230da828d0e..127a3be0e0f0 100644
--- a/drivers/media/radio/radio-shark.c
+++ b/drivers/media/radio/radio-shark.c
@@ -316,6 +316,16 @@ static int usb_shark_probe(struct usb_interface *intf,
 {
 	struct shark_device *shark;
 	int retval = -ENOMEM;
+	static const u8 ep_addresses[] = {
+		SHARK_IN_EP | USB_DIR_IN,
+		SHARK_OUT_EP | USB_DIR_OUT,
+		0};
+
+	/* Are the expected endpoints present? */
+	if (!usb_check_int_endpoints(intf, ep_addresses)) {
+		dev_err(&intf->dev, "Invalid radioSHARK device\n");
+		return -EINVAL;
+	}
 
 	shark = kzalloc(sizeof(struct shark_device), GFP_KERNEL);
 	if (!shark)
diff --git a/drivers/media/radio/radio-shark2.c b/drivers/media/radio/radio-shark2.c
index d150f12382c6..f1c5c0a6a335 100644
--- a/drivers/media/radio/radio-shark2.c
+++ b/drivers/media/radio/radio-shark2.c
@@ -282,6 +282,16 @@ static int usb_shark_probe(struct usb_interface *intf,
 {
 	struct shark_device *shark;
 	int retval = -ENOMEM;
+	static const u8 ep_addresses[] = {
+		SHARK_IN_EP | USB_DIR_IN,
+		SHARK_OUT_EP | USB_DIR_OUT,
+		0};
+
+	/* Are the expected endpoints present? */
+	if (!usb_check_int_endpoints(intf, ep_addresses)) {
+		dev_err(&intf->dev, "Invalid radioSHARK2 device\n");
+		return -EINVAL;
+	}
 
 	shark = kzalloc(sizeof(struct shark_device), GFP_KERNEL);
 	if (!shark)
diff --git a/drivers/media/radio/radio-si476x.c b/drivers/media/radio/radio-si476x.c
index 171f9cc9ee5e..6061506159f1 100644
--- a/drivers/media/radio/radio-si476x.c
+++ b/drivers/media/radio/radio-si476x.c
@@ -1498,7 +1498,7 @@ exit:
 	return rval;
 }
 
-static int si476x_radio_remove(struct platform_device *pdev)
+static void si476x_radio_remove(struct platform_device *pdev)
 {
 	struct si476x_radio *radio = platform_get_drvdata(pdev);
 
@@ -1506,8 +1506,6 @@ static int si476x_radio_remove(struct platform_device *pdev)
 	video_unregister_device(&radio->videodev);
 	v4l2_device_unregister(&radio->v4l2dev);
 	debugfs_remove_recursive(radio->debugfs);
-
-	return 0;
 }
 
 MODULE_ALIAS("platform:si476x-radio");
@@ -1517,7 +1515,7 @@ static struct platform_driver si476x_radio_driver = {
 		.name	= DRIVER_NAME,
 	},
 	.probe		= si476x_radio_probe,
-	.remove		= si476x_radio_remove,
+	.remove_new	= si476x_radio_remove,
 };
 module_platform_driver(si476x_radio_driver);
 
diff --git a/drivers/media/radio/radio-timb.c b/drivers/media/radio/radio-timb.c
index 948ee3eec914..04daa9c358c2 100644
--- a/drivers/media/radio/radio-timb.c
+++ b/drivers/media/radio/radio-timb.c
@@ -151,13 +151,12 @@ err:
 	return err;
 }
 
-static int timbradio_remove(struct platform_device *pdev)
+static void timbradio_remove(struct platform_device *pdev)
 {
 	struct timbradio *tr = platform_get_drvdata(pdev);
 
 	video_unregister_device(&tr->video_dev);
 	v4l2_device_unregister(&tr->v4l2_dev);
-	return 0;
 }
 
 static struct platform_driver timbradio_platform_driver = {
@@ -165,7 +164,7 @@ static struct platform_driver timbradio_platform_driver = {
 		.name	= DRIVER_NAME,
 	},
 	.probe		= timbradio_probe,
-	.remove		= timbradio_remove,
+	.remove_new	= timbradio_remove,
 };
 
 module_platform_driver(timbradio_platform_driver);
diff --git a/drivers/media/radio/radio-wl1273.c b/drivers/media/radio/radio-wl1273.c
index 484046471c03..e8166eac9efe 100644
--- a/drivers/media/radio/radio-wl1273.c
+++ b/drivers/media/radio/radio-wl1273.c
@@ -1977,7 +1977,7 @@ static const struct video_device wl1273_viddev_template = {
 				  V4L2_CAP_RDS_OUTPUT,
 };
 
-static int wl1273_fm_radio_remove(struct platform_device *pdev)
+static void wl1273_fm_radio_remove(struct platform_device *pdev)
 {
 	struct wl1273_device *radio = platform_get_drvdata(pdev);
 	struct wl1273_core *core = radio->core;
@@ -1990,8 +1990,6 @@ static int wl1273_fm_radio_remove(struct platform_device *pdev)
 	v4l2_ctrl_handler_free(&radio->ctrl_handler);
 	video_unregister_device(&radio->videodev);
 	v4l2_device_unregister(&radio->v4l2dev);
-
-	return 0;
 }
 
 static int wl1273_fm_radio_probe(struct platform_device *pdev)
@@ -2147,7 +2145,7 @@ pdata_err:
 
 static struct platform_driver wl1273_fm_radio_driver = {
 	.probe		= wl1273_fm_radio_probe,
-	.remove		= wl1273_fm_radio_remove,
+	.remove_new	= wl1273_fm_radio_remove,
 	.driver		= {
 		.name	= "wl1273_fm_radio",
 	},
diff --git a/drivers/media/radio/si4713/radio-platform-si4713.c b/drivers/media/radio/si4713/radio-platform-si4713.c
index 433f9642786d..9fdaed68a962 100644
--- a/drivers/media/radio/si4713/radio-platform-si4713.c
+++ b/drivers/media/radio/si4713/radio-platform-si4713.c
@@ -190,7 +190,7 @@ exit:
 }
 
 /* radio_si4713_pdriver_remove - remove the device */
-static int radio_si4713_pdriver_remove(struct platform_device *pdev)
+static void radio_si4713_pdriver_remove(struct platform_device *pdev)
 {
 	struct v4l2_device *v4l2_dev = platform_get_drvdata(pdev);
 	struct radio_si4713_device *rsdev;
@@ -198,8 +198,6 @@ static int radio_si4713_pdriver_remove(struct platform_device *pdev)
 	rsdev = container_of(v4l2_dev, struct radio_si4713_device, v4l2_dev);
 	video_unregister_device(&rsdev->radio_dev);
 	v4l2_device_unregister(&rsdev->v4l2_dev);
-
-	return 0;
 }
 
 static struct platform_driver radio_si4713_pdriver = {
@@ -207,7 +205,7 @@ static struct platform_driver radio_si4713_pdriver = {
 		.name	= "radio-si4713",
 	},
 	.probe		= radio_si4713_pdriver_probe,
-	.remove         = radio_si4713_pdriver_remove,
+	.remove_new     = radio_si4713_pdriver_remove,
 };
 
 module_platform_driver(radio_si4713_pdriver);
diff --git a/drivers/media/rc/gpio-ir-recv.c b/drivers/media/rc/gpio-ir-recv.c
index 8dbe780dae4e..27f55ea966c6 100644
--- a/drivers/media/rc/gpio-ir-recv.c
+++ b/drivers/media/rc/gpio-ir-recv.c
@@ -103,6 +103,8 @@ static int gpio_ir_recv_probe(struct platform_device *pdev)
 		rcdev->map_name = RC_MAP_EMPTY;
 
 	gpio_dev->rcdev = rcdev;
+	if (of_property_read_bool(np, "wakeup-source"))
+		device_init_wakeup(dev, true);
 
 	rc = devm_rc_register_device(dev, rcdev);
 	if (rc < 0) {
@@ -126,7 +128,7 @@ static int gpio_ir_recv_probe(struct platform_device *pdev)
 				"gpio-ir-recv-irq", gpio_dev);
 }
 
-static int gpio_ir_recv_remove(struct platform_device *pdev)
+static void gpio_ir_recv_remove(struct platform_device *pdev)
 {
 	struct gpio_rc_dev *gpio_dev = platform_get_drvdata(pdev);
 	struct device *pmdev = gpio_dev->pmdev;
@@ -139,8 +141,6 @@ static int gpio_ir_recv_remove(struct platform_device *pdev)
 		pm_runtime_put_noidle(pmdev);
 		pm_runtime_set_suspended(pmdev);
 	}
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -202,7 +202,7 @@ MODULE_DEVICE_TABLE(of, gpio_ir_recv_of_match);
 
 static struct platform_driver gpio_ir_recv_driver = {
 	.probe  = gpio_ir_recv_probe,
-	.remove = gpio_ir_recv_remove,
+	.remove_new = gpio_ir_recv_remove,
 	.driver = {
 		.name   = KBUILD_MODNAME,
 		.of_match_table = of_match_ptr(gpio_ir_recv_of_match),
diff --git a/drivers/media/rc/img-ir/img-ir-core.c b/drivers/media/rc/img-ir/img-ir-core.c
index 6f8464872033..d87d8e14c556 100644
--- a/drivers/media/rc/img-ir/img-ir-core.c
+++ b/drivers/media/rc/img-ir/img-ir-core.c
@@ -152,7 +152,7 @@ err_probe:
 	return error;
 }
 
-static int img_ir_remove(struct platform_device *pdev)
+static void img_ir_remove(struct platform_device *pdev)
 {
 	struct img_ir_priv *priv = platform_get_drvdata(pdev);
 
@@ -164,7 +164,6 @@ static int img_ir_remove(struct platform_device *pdev)
 		clk_disable_unprepare(priv->clk);
 	if (!IS_ERR(priv->sys_clk))
 		clk_disable_unprepare(priv->sys_clk);
-	return 0;
 }
 
 static SIMPLE_DEV_PM_OPS(img_ir_pmops, img_ir_suspend, img_ir_resume);
@@ -182,7 +181,7 @@ static struct platform_driver img_ir_driver = {
 		.pm = &img_ir_pmops,
 	},
 	.probe = img_ir_probe,
-	.remove = img_ir_remove,
+	.remove_new = img_ir_remove,
 };
 
 module_platform_driver(img_ir_driver);
diff --git a/drivers/media/rc/ir-hix5hd2.c b/drivers/media/rc/ir-hix5hd2.c
index 4ff954b11dc7..0034f615b466 100644
--- a/drivers/media/rc/ir-hix5hd2.c
+++ b/drivers/media/rc/ir-hix5hd2.c
@@ -340,13 +340,12 @@ err:
 	return ret;
 }
 
-static int hix5hd2_ir_remove(struct platform_device *pdev)
+static void hix5hd2_ir_remove(struct platform_device *pdev)
 {
 	struct hix5hd2_ir_priv *priv = platform_get_drvdata(pdev);
 
 	clk_disable_unprepare(priv->clock);
 	rc_unregister_device(priv->rdev);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -395,7 +394,7 @@ static struct platform_driver hix5hd2_ir_driver = {
 		.pm     = &hix5hd2_ir_pm_ops,
 	},
 	.probe = hix5hd2_ir_probe,
-	.remove = hix5hd2_ir_remove,
+	.remove_new = hix5hd2_ir_remove,
 };
 
 module_platform_driver(hix5hd2_ir_driver);
diff --git a/drivers/media/rc/keymaps/Makefile b/drivers/media/rc/keymaps/Makefile
index f513ff5caf4e..f19558fdab0c 100644
--- a/drivers/media/rc/keymaps/Makefile
+++ b/drivers/media/rc/keymaps/Makefile
@@ -22,6 +22,7 @@ obj-$(CONFIG_RC_MAP) += \
 			rc-avertv-303.o \
 			rc-azurewave-ad-tu700.o \
 			rc-beelink-gs1.o \
+			rc-beelink-mxiii.o \
 			rc-behold-columbus.o \
 			rc-behold.o \
 			rc-budget-ci-old.o \
@@ -37,6 +38,7 @@ obj-$(CONFIG_RC_MAP) += \
 			rc-dm1105-nec.o \
 			rc-dntv-live-dvb-t.o \
 			rc-dntv-live-dvbt-pro.o \
+			rc-dreambox.o \
 			rc-dtt200u.o \
 			rc-dvbsky.o \
 			rc-dvico-mce.o \
diff --git a/drivers/media/rc/keymaps/rc-beelink-mxiii.c b/drivers/media/rc/keymaps/rc-beelink-mxiii.c
new file mode 100644
index 000000000000..01180cd92205
--- /dev/null
+++ b/drivers/media/rc/keymaps/rc-beelink-mxiii.c
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019 Christian Hewitt <christianshewitt@gmail.com>
+ *
+ */
+
+#include <media/rc-map.h>
+#include <linux/module.h>
+
+/*
+ * Keytable for the Beelink Mini MXIII remote control
+ *
+ */
+
+static struct rc_map_table beelink_mxiii[] = {
+	{ 0xb2dc, KEY_POWER },
+
+	{ 0xb288, KEY_MUTE },
+	{ 0xb282, KEY_HOME },
+
+	{ 0xb2ca, KEY_UP },
+	{ 0xb299, KEY_LEFT },
+	{ 0xb2ce, KEY_OK },
+	{ 0xb2c1, KEY_RIGHT },
+	{ 0xb2d2, KEY_DOWN },
+
+	{ 0xb2c5, KEY_MENU },
+	{ 0xb29a, KEY_BACK },
+
+	{ 0xb281, KEY_VOLUMEDOWN },
+	{ 0xb280, KEY_VOLUMEUP },
+};
+
+static struct rc_map_list beelink_mxiii_map = {
+	.map = {
+		.scan     = beelink_mxiii,
+		.size     = ARRAY_SIZE(beelink_mxiii),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_BEELINK_MXIII,
+	}
+};
+
+static int __init init_rc_map_beelink_mxiii(void)
+{
+	return rc_map_register(&beelink_mxiii_map);
+}
+
+static void __exit exit_rc_map_beelink_mxiii(void)
+{
+	rc_map_unregister(&beelink_mxiii_map);
+}
+
+module_init(init_rc_map_beelink_mxiii)
+module_exit(exit_rc_map_beelink_mxiii)
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Christian Hewitt <christianshewitt@gmail.com");
diff --git a/drivers/media/rc/keymaps/rc-dreambox.c b/drivers/media/rc/keymaps/rc-dreambox.c
new file mode 100644
index 000000000000..dea024fa3a22
--- /dev/null
+++ b/drivers/media/rc/keymaps/rc-dreambox.c
@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2021 Emanuel Strobel <emanuel.strobel@yahoo.com>
+ */
+
+#include <media/rc-map.h>
+#include <linux/module.h>
+
+/*
+ * Keytable for Dreambox RC10/RC0 and RC20/RC-BT remote controls
+ *
+ * Keys that are not IR addressable:
+ *
+ * // DREAM switches to STB control mode
+ * // TV switches to TV control mode
+ * // MODE toggles STB/TV/BT control modes
+ *
+ */
+
+static struct rc_map_table dreambox[] = {
+	/* Dreambox RC10/RC0/RCU-BT remote */
+	{ 0x3200, KEY_POWER },
+
+	// DREAM
+	{ 0x3290, KEY_HELP },
+	// TV
+
+	{ 0x3201, KEY_1 },
+	{ 0x3202, KEY_2 },
+	{ 0x3203, KEY_3 },
+	{ 0x3204, KEY_4 },
+	{ 0x3205, KEY_5 },
+	{ 0x3206, KEY_6 },
+	{ 0x3207, KEY_7 },
+	{ 0x3208, KEY_8 },
+	{ 0x3209, KEY_9 },
+	{ 0x320a, KEY_PREVIOUS },
+	{ 0x320b, KEY_0 },
+	{ 0x320c, KEY_NEXT },
+
+	{ 0x321f, KEY_RED },
+	{ 0x3220, KEY_GREEN },
+	{ 0x3221, KEY_YELLOW },
+	{ 0x3222, KEY_BLUE },
+
+	{ 0x3210, KEY_INFO },
+	{ 0x3212, KEY_MENU },
+	{ 0x320e, KEY_AUDIO },
+	{ 0x3218, KEY_PVR },
+
+	{ 0x3213, KEY_LEFT },
+	{ 0x3211, KEY_UP },
+	{ 0x3215, KEY_RIGHT },
+	{ 0x3217, KEY_DOWN },
+	{ 0x3214, KEY_OK },
+
+	{ 0x3219, KEY_VOLUMEUP },
+	{ 0x321c, KEY_VOLUMEDOWN },
+
+	{ 0x321d, KEY_ESC }, // EXIT
+	{ 0x321a, KEY_MUTE },
+
+	{ 0x321b, KEY_PAGEUP },
+	{ 0x321e, KEY_PAGEDOWN },
+
+	{ 0x3223, KEY_PREVIOUSSONG },
+	{ 0x3224, KEY_PLAYPAUSE },
+	{ 0x3225, KEY_STOP },
+	{ 0x3226, KEY_NEXTSONG },
+
+	{ 0x3227, KEY_TV },
+	{ 0x3228, KEY_RADIO },
+	{ 0x3229, KEY_TEXT },
+	{ 0x322a, KEY_RECORD },
+
+	/* Dreambox RC20/RC-BT */
+	{ 0x3407, KEY_MUTE },
+	// MODE
+	{ 0x3401, KEY_POWER },
+
+	{ 0x3432, KEY_PREVIOUSSONG },
+	{ 0x3433, KEY_PLAYPAUSE },
+	{ 0x3435, KEY_NEXTSONG },
+
+	{ 0x3436, KEY_RECORD },
+	{ 0x3434, KEY_STOP },
+	{ 0x3425, KEY_TEXT },
+
+	{ 0x341f, KEY_RED },
+	{ 0x3420, KEY_GREEN },
+	{ 0x3421, KEY_YELLOW },
+	{ 0x3422, KEY_BLUE },
+
+	{ 0x341b, KEY_INFO },
+	{ 0x341c, KEY_MENU },
+	{ 0x3430, KEY_AUDIO },
+	{ 0x3431, KEY_PVR },
+
+	{ 0x3414, KEY_LEFT },
+	{ 0x3411, KEY_UP },
+	{ 0x3416, KEY_RIGHT },
+	{ 0x3419, KEY_DOWN },
+	{ 0x3415, KEY_OK },
+
+	{ 0x3413, KEY_VOLUMEUP },
+	{ 0x3418, KEY_VOLUMEDOWN },
+
+	{ 0x3412, KEY_ESC }, // EXIT
+	{ 0x3426, KEY_HELP }, // MIC
+
+	{ 0x3417, KEY_PAGEUP },
+	{ 0x341a, KEY_PAGEDOWN },
+
+	{ 0x3404, KEY_1 },
+	{ 0x3405, KEY_2 },
+	{ 0x3406, KEY_3 },
+	{ 0x3408, KEY_4 },
+	{ 0x3409, KEY_5 },
+	{ 0x340a, KEY_6 },
+	{ 0x340c, KEY_7 },
+	{ 0x340d, KEY_8 },
+	{ 0x340e, KEY_9 },
+	{ 0x340b, KEY_PREVIOUS },
+	{ 0x3410, KEY_0 },
+	{ 0x340f, KEY_NEXT },
+};
+
+static struct rc_map_list dreambox_map = {
+	.map = {
+		.scan     = dreambox,
+		.size     = ARRAY_SIZE(dreambox),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_DREAMBOX,
+	}
+};
+
+static int __init init_rc_map_dreambox(void)
+{
+	return rc_map_register(&dreambox_map);
+}
+
+static void __exit exit_rc_map_dreambox(void)
+{
+	rc_map_unregister(&dreambox_map);
+}
+
+module_init(init_rc_map_dreambox)
+module_exit(exit_rc_map_dreambox)
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Emanuel Strobel <emanuel.strobel@yahoo.com>");
diff --git a/drivers/media/rc/lirc_dev.c b/drivers/media/rc/lirc_dev.c
index 25ab61dae126..043d23aaa3cb 100644
--- a/drivers/media/rc/lirc_dev.c
+++ b/drivers/media/rc/lirc_dev.c
@@ -785,7 +785,7 @@ int __init lirc_dev_init(void)
 {
 	int retval;
 
-	lirc_class = class_create(THIS_MODULE, "lirc");
+	lirc_class = class_create("lirc");
 	if (IS_ERR(lirc_class)) {
 		pr_err("class_create failed\n");
 		return PTR_ERR(lirc_class);
diff --git a/drivers/media/rc/meson-ir-tx.c b/drivers/media/rc/meson-ir-tx.c
index abdb62b16e98..6355b79893fb 100644
--- a/drivers/media/rc/meson-ir-tx.c
+++ b/drivers/media/rc/meson-ir-tx.c
@@ -372,13 +372,11 @@ static int __init meson_irtx_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int meson_irtx_remove(struct platform_device *pdev)
+static void meson_irtx_remove(struct platform_device *pdev)
 {
 	struct rc_dev *rc = platform_get_drvdata(pdev);
 
 	rc_unregister_device(rc);
-
-	return 0;
 }
 
 static const struct of_device_id meson_irtx_dt_match[] = {
@@ -390,7 +388,7 @@ static const struct of_device_id meson_irtx_dt_match[] = {
 MODULE_DEVICE_TABLE(of, meson_irtx_dt_match);
 
 static struct platform_driver meson_irtx_pd = {
-	.remove = meson_irtx_remove,
+	.remove_new = meson_irtx_remove,
 	.driver = {
 		.name = DRIVER_NAME,
 		.of_match_table = meson_irtx_dt_match,
diff --git a/drivers/media/rc/meson-ir.c b/drivers/media/rc/meson-ir.c
index 4b769111f78e..49aa309d1a8c 100644
--- a/drivers/media/rc/meson-ir.c
+++ b/drivers/media/rc/meson-ir.c
@@ -177,7 +177,7 @@ static int meson_ir_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int meson_ir_remove(struct platform_device *pdev)
+static void meson_ir_remove(struct platform_device *pdev)
 {
 	struct meson_ir *ir = platform_get_drvdata(pdev);
 	unsigned long flags;
@@ -186,8 +186,6 @@ static int meson_ir_remove(struct platform_device *pdev)
 	spin_lock_irqsave(&ir->lock, flags);
 	meson_ir_set_mask(ir, IR_DEC_REG1, REG1_ENABLE, 0);
 	spin_unlock_irqrestore(&ir->lock, flags);
-
-	return 0;
 }
 
 static void meson_ir_shutdown(struct platform_device *pdev)
@@ -226,7 +224,7 @@ MODULE_DEVICE_TABLE(of, meson_ir_match);
 
 static struct platform_driver meson_ir_driver = {
 	.probe		= meson_ir_probe,
-	.remove		= meson_ir_remove,
+	.remove_new	= meson_ir_remove,
 	.shutdown	= meson_ir_shutdown,
 	.driver = {
 		.name		= DRIVER_NAME,
diff --git a/drivers/media/rc/mtk-cir.c b/drivers/media/rc/mtk-cir.c
index 27b7412d02a5..df9349330a93 100644
--- a/drivers/media/rc/mtk-cir.c
+++ b/drivers/media/rc/mtk-cir.c
@@ -420,7 +420,7 @@ exit_clkdisable_clk:
 	return ret;
 }
 
-static int mtk_ir_remove(struct platform_device *pdev)
+static void mtk_ir_remove(struct platform_device *pdev)
 {
 	struct mtk_ir *ir = platform_get_drvdata(pdev);
 
@@ -434,13 +434,11 @@ static int mtk_ir_remove(struct platform_device *pdev)
 
 	clk_disable_unprepare(ir->bus);
 	clk_disable_unprepare(ir->clk);
-
-	return 0;
 }
 
 static struct platform_driver mtk_ir_driver = {
 	.probe          = mtk_ir_probe,
-	.remove         = mtk_ir_remove,
+	.remove_new     = mtk_ir_remove,
 	.driver = {
 		.name = MTK_IR_DEV,
 		.of_match_table = mtk_ir_match,
diff --git a/drivers/media/rc/st_rc.c b/drivers/media/rc/st_rc.c
index 19e987a048cc..28477aa95563 100644
--- a/drivers/media/rc/st_rc.c
+++ b/drivers/media/rc/st_rc.c
@@ -194,7 +194,7 @@ static int st_rc_hardware_init(struct st_rc_device *dev)
 	return 0;
 }
 
-static int st_rc_remove(struct platform_device *pdev)
+static void st_rc_remove(struct platform_device *pdev)
 {
 	struct st_rc_device *rc_dev = platform_get_drvdata(pdev);
 
@@ -202,7 +202,6 @@ static int st_rc_remove(struct platform_device *pdev)
 	device_init_wakeup(&pdev->dev, false);
 	clk_disable_unprepare(rc_dev->sys_clock);
 	rc_unregister_device(rc_dev->rdev);
-	return 0;
 }
 
 static int st_rc_open(struct rc_dev *rdev)
@@ -408,7 +407,7 @@ static struct platform_driver st_rc_driver = {
 		.pm     = &st_rc_pm_ops,
 	},
 	.probe = st_rc_probe,
-	.remove = st_rc_remove,
+	.remove_new = st_rc_remove,
 };
 
 module_platform_driver(st_rc_driver);
diff --git a/drivers/media/rc/sunxi-cir.c b/drivers/media/rc/sunxi-cir.c
index b631a81e58bb..75b7aed1579c 100644
--- a/drivers/media/rc/sunxi-cir.c
+++ b/drivers/media/rc/sunxi-cir.c
@@ -364,14 +364,12 @@ exit_free_dev:
 	return ret;
 }
 
-static int sunxi_ir_remove(struct platform_device *pdev)
+static void sunxi_ir_remove(struct platform_device *pdev)
 {
 	struct sunxi_ir *ir = platform_get_drvdata(pdev);
 
 	rc_unregister_device(ir->rc);
 	sunxi_ir_hw_exit(&pdev->dev);
-
-	return 0;
 }
 
 static void sunxi_ir_shutdown(struct platform_device *pdev)
@@ -413,7 +411,7 @@ MODULE_DEVICE_TABLE(of, sunxi_ir_match);
 
 static struct platform_driver sunxi_ir_driver = {
 	.probe          = sunxi_ir_probe,
-	.remove         = sunxi_ir_remove,
+	.remove_new     = sunxi_ir_remove,
 	.shutdown       = sunxi_ir_shutdown,
 	.driver = {
 		.name = SUNXI_IR_DEV,
diff --git a/drivers/media/test-drivers/vicodec/vicodec-core.c b/drivers/media/test-drivers/vicodec/vicodec-core.c
index 1d1bee111732..6f0e20df74e9 100644
--- a/drivers/media/test-drivers/vicodec/vicodec-core.c
+++ b/drivers/media/test-drivers/vicodec/vicodec-core.c
@@ -2179,7 +2179,7 @@ free_dev:
 	return ret;
 }
 
-static int vicodec_remove(struct platform_device *pdev)
+static void vicodec_remove(struct platform_device *pdev)
 {
 	struct vicodec_dev *dev = platform_get_drvdata(pdev);
 
@@ -2196,13 +2196,11 @@ static int vicodec_remove(struct platform_device *pdev)
 	video_unregister_device(&dev->stateful_dec.vfd);
 	video_unregister_device(&dev->stateless_dec.vfd);
 	v4l2_device_put(&dev->v4l2_dev);
-
-	return 0;
 }
 
 static struct platform_driver vicodec_pdrv = {
 	.probe		= vicodec_probe,
-	.remove		= vicodec_remove,
+	.remove_new	= vicodec_remove,
 	.driver		= {
 		.name	= VICODEC_NAME,
 	},
diff --git a/drivers/media/test-drivers/vidtv/vidtv_bridge.c b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
index dff7265a42ca..8b04e12af286 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_bridge.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
@@ -528,7 +528,7 @@ err_dvb:
 	return ret;
 }
 
-static int vidtv_bridge_remove(struct platform_device *pdev)
+static void vidtv_bridge_remove(struct platform_device *pdev)
 {
 	struct vidtv_dvb *dvb;
 	u32 i;
@@ -552,8 +552,6 @@ static int vidtv_bridge_remove(struct platform_device *pdev)
 	dvb_dmx_release(&dvb->demux);
 	dvb_unregister_adapter(&dvb->adapter);
 	dev_info(&pdev->dev, "Successfully removed vidtv\n");
-
-	return 0;
 }
 
 static void vidtv_bridge_dev_release(struct device *dev)
@@ -574,7 +572,7 @@ static struct platform_driver vidtv_bridge_driver = {
 		.name = VIDTV_PDEV_NAME,
 	},
 	.probe    = vidtv_bridge_probe,
-	.remove   = vidtv_bridge_remove,
+	.remove_new = vidtv_bridge_remove,
 };
 
 static void __exit vidtv_bridge_exit(void)
diff --git a/drivers/media/test-drivers/vim2m.c b/drivers/media/test-drivers/vim2m.c
index 7964426bf2f7..3e3b424b4860 100644
--- a/drivers/media/test-drivers/vim2m.c
+++ b/drivers/media/test-drivers/vim2m.c
@@ -1379,7 +1379,7 @@ error_free:
 	return ret;
 }
 
-static int vim2m_remove(struct platform_device *pdev)
+static void vim2m_remove(struct platform_device *pdev)
 {
 	struct vim2m_dev *dev = platform_get_drvdata(pdev);
 
@@ -1390,13 +1390,11 @@ static int vim2m_remove(struct platform_device *pdev)
 	v4l2_m2m_unregister_media_controller(dev->m2m_dev);
 #endif
 	video_unregister_device(&dev->vfd);
-
-	return 0;
 }
 
 static struct platform_driver vim2m_pdrv = {
 	.probe		= vim2m_probe,
-	.remove		= vim2m_remove,
+	.remove_new	= vim2m_remove,
 	.driver		= {
 		.name	= MEM2MEM_NAME,
 	},
diff --git a/drivers/media/test-drivers/vimc/vimc-common.c b/drivers/media/test-drivers/vimc/vimc-common.c
index 7b27153c0728..2e72974e35b4 100644
--- a/drivers/media/test-drivers/vimc/vimc-common.c
+++ b/drivers/media/test-drivers/vimc/vimc-common.c
@@ -241,13 +241,13 @@ static int vimc_get_pix_format(struct media_pad *pad,
 	if (is_media_entity_v4l2_subdev(pad->entity)) {
 		struct v4l2_subdev *sd =
 			media_entity_to_v4l2_subdev(pad->entity);
-		struct v4l2_subdev_format sd_fmt;
+		struct v4l2_subdev_format sd_fmt = {
+			.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+			.pad = pad->index,
+		};
 		const struct vimc_pix_map *pix_map;
 		int ret;
 
-		sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-		sd_fmt.pad = pad->index;
-
 		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &sd_fmt);
 		if (ret)
 			return ret;
diff --git a/drivers/media/test-drivers/vimc/vimc-core.c b/drivers/media/test-drivers/vimc/vimc-core.c
index e82cfa5ffbf4..af127476e920 100644
--- a/drivers/media/test-drivers/vimc/vimc-core.c
+++ b/drivers/media/test-drivers/vimc/vimc-core.c
@@ -387,7 +387,7 @@ static int vimc_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int vimc_remove(struct platform_device *pdev)
+static void vimc_remove(struct platform_device *pdev)
 {
 	struct vimc_device *vimc = platform_get_drvdata(pdev);
 
@@ -397,8 +397,6 @@ static int vimc_remove(struct platform_device *pdev)
 	media_device_unregister(&vimc->mdev);
 	v4l2_device_unregister(&vimc->v4l2_dev);
 	v4l2_device_put(&vimc->v4l2_dev);
-
-	return 0;
 }
 
 static void vimc_dev_release(struct device *dev)
@@ -412,7 +410,7 @@ static struct platform_device vimc_pdev = {
 
 static struct platform_driver vimc_pdrv = {
 	.probe		= vimc_probe,
-	.remove		= vimc_remove,
+	.remove_new	= vimc_remove,
 	.driver		= {
 		.name	= VIMC_PDEV_NAME,
 	},
diff --git a/drivers/media/test-drivers/visl/visl-core.c b/drivers/media/test-drivers/visl/visl-core.c
index 9cb60ab653bf..9970dc739ca5 100644
--- a/drivers/media/test-drivers/visl/visl-core.c
+++ b/drivers/media/test-drivers/visl/visl-core.c
@@ -480,7 +480,7 @@ error_visl_dev:
 	return ret;
 }
 
-static int visl_remove(struct platform_device *pdev)
+static void visl_remove(struct platform_device *pdev)
 {
 	struct visl_dev *dev = platform_get_drvdata(pdev);
 
@@ -493,13 +493,11 @@ static int visl_remove(struct platform_device *pdev)
 	}
 #endif
 	video_unregister_device(&dev->vfd);
-
-	return 0;
 }
 
 static struct platform_driver visl_pdrv = {
 	.probe		= visl_probe,
-	.remove		= visl_remove,
+	.remove_new	= visl_remove,
 	.driver		= {
 		.name	= VISL_NAME,
 	},
diff --git a/drivers/media/test-drivers/vivid/vivid-core.c b/drivers/media/test-drivers/vivid/vivid-core.c
index f28440e6c9f8..c2167ccfd222 100644
--- a/drivers/media/test-drivers/vivid/vivid-core.c
+++ b/drivers/media/test-drivers/vivid/vivid-core.c
@@ -126,7 +126,7 @@ MODULE_PARM_DESC(node_types, " node types, default is 0xe1d3d. Bitmask with the
 			     "\t\t    bit 8: Video Output node\n"
 			     "\t\t    bit 10-11: VBI Output node: 0 = none, 1 = raw vbi, 2 = sliced vbi, 3 = both\n"
 			     "\t\t    bit 12: Radio Transmitter node\n"
-			     "\t\t    bit 16: Framebuffer for testing overlays\n"
+			     "\t\t    bit 16: Framebuffer for testing output overlays\n"
 			     "\t\t    bit 17: Metadata Capture node\n"
 			     "\t\t    bit 18: Metadata Output node\n"
 			     "\t\t    bit 19: Touch Capture node\n");
@@ -326,7 +326,7 @@ static int vidioc_overlay(struct file *file, void *fh, unsigned i)
 	struct video_device *vdev = video_devdata(file);
 
 	if (vdev->vfl_dir == VFL_DIR_RX)
-		return vivid_vid_cap_overlay(file, fh, i);
+		return -ENOTTY;
 	return vivid_vid_out_overlay(file, fh, i);
 }
 
@@ -335,38 +335,16 @@ static int vidioc_g_fbuf(struct file *file, void *fh, struct v4l2_framebuffer *a
 	struct video_device *vdev = video_devdata(file);
 
 	if (vdev->vfl_dir == VFL_DIR_RX)
-		return vivid_vid_cap_g_fbuf(file, fh, a);
+		return -ENOTTY;
 	return vivid_vid_out_g_fbuf(file, fh, a);
 }
 
-/*
- * Only support the framebuffer of one of the vivid instances.
- * Anything else is rejected.
- */
-bool vivid_validate_fb(const struct v4l2_framebuffer *a)
-{
-	struct vivid_dev *dev;
-	int i;
-
-	for (i = 0; i < n_devs; i++) {
-		dev = vivid_devs[i];
-		if (!dev || !dev->video_pbase)
-			continue;
-		if ((unsigned long)a->base == dev->video_pbase &&
-		    a->fmt.width <= dev->display_width &&
-		    a->fmt.height <= dev->display_height &&
-		    a->fmt.bytesperline <= dev->display_byte_stride)
-			return true;
-	}
-	return false;
-}
-
 static int vidioc_s_fbuf(struct file *file, void *fh, const struct v4l2_framebuffer *a)
 {
 	struct video_device *vdev = video_devdata(file);
 
 	if (vdev->vfl_dir == VFL_DIR_RX)
-		return vivid_vid_cap_s_fbuf(file, fh, a);
+		return -ENOTTY;
 	return vivid_vid_out_s_fbuf(file, fh, a);
 }
 
@@ -651,8 +629,6 @@ static int vivid_fop_release(struct file *file)
 		vivid_reconnect(dev);
 	}
 	mutex_unlock(&dev->mutex);
-	if (file->private_data == dev->overlay_cap_owner)
-		dev->overlay_cap_owner = NULL;
 	if (file->private_data == dev->radio_rx_rds_owner) {
 		dev->radio_rx_rds_last_block = 0;
 		dev->radio_rx_rds_owner = NULL;
@@ -778,10 +754,6 @@ static const struct v4l2_ioctl_ops vivid_ioctl_ops = {
 	.vidioc_g_parm			= vidioc_g_parm,
 	.vidioc_s_parm			= vidioc_s_parm,
 
-	.vidioc_enum_fmt_vid_overlay	= vidioc_enum_fmt_vid_overlay,
-	.vidioc_g_fmt_vid_overlay	= vidioc_g_fmt_vid_overlay,
-	.vidioc_try_fmt_vid_overlay	= vidioc_try_fmt_vid_overlay,
-	.vidioc_s_fmt_vid_overlay	= vidioc_s_fmt_vid_overlay,
 	.vidioc_g_fmt_vid_out_overlay	= vidioc_g_fmt_vid_out_overlay,
 	.vidioc_try_fmt_vid_out_overlay	= vidioc_try_fmt_vid_out_overlay,
 	.vidioc_s_fmt_vid_out_overlay	= vidioc_s_fmt_vid_out_overlay,
@@ -862,8 +834,6 @@ static void vivid_dev_release(struct v4l2_device *v4l2_dev)
 	vfree(dev->scaled_line);
 	vfree(dev->blended_line);
 	vfree(dev->edid);
-	vfree(dev->bitmap_cap);
-	vfree(dev->bitmap_out);
 	tpg_free(&dev->tpg);
 	kfree(dev->query_dv_timings_qmenu);
 	kfree(dev->query_dv_timings_qmenu_strings);
@@ -1107,7 +1077,7 @@ static void vivid_set_capabilities(struct vivid_dev *dev)
 		/* set up the capabilities of the video capture device */
 		dev->vid_cap_caps = dev->multiplanar ?
 			V4L2_CAP_VIDEO_CAPTURE_MPLANE :
-			V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_OVERLAY;
+			V4L2_CAP_VIDEO_CAPTURE;
 		dev->vid_cap_caps |= V4L2_CAP_STREAMING | V4L2_CAP_READWRITE;
 		if (dev->has_audio_inputs)
 			dev->vid_cap_caps |= V4L2_CAP_AUDIO;
@@ -1396,7 +1366,7 @@ static int vivid_create_queues(struct vivid_dev *dev)
 	}
 
 	if (dev->has_fb) {
-		/* Create framebuffer for testing capture/output overlay */
+		/* Create framebuffer for testing output overlay */
 		ret = vivid_fb_init(dev);
 		if (ret)
 			return ret;
@@ -1892,13 +1862,6 @@ static int vivid_create_instance(struct platform_device *pdev, int inst)
 	vivid_update_format_cap(dev, false);
 	vivid_update_format_out(dev);
 
-	/* initialize overlay */
-	dev->fb_cap.fmt.width = dev->src_rect.width;
-	dev->fb_cap.fmt.height = dev->src_rect.height;
-	dev->fb_cap.fmt.pixelformat = dev->fmt_cap->fourcc;
-	dev->fb_cap.fmt.bytesperline = dev->src_rect.width * tpg_g_twopixelsize(&dev->tpg, 0) / 2;
-	dev->fb_cap.fmt.sizeimage = dev->src_rect.height * dev->fb_cap.fmt.bytesperline;
-
 	/* update touch configuration */
 	dev->timeperframe_tch_cap.numerator = 1;
 	dev->timeperframe_tch_cap.denominator = 10;
@@ -2058,7 +2021,7 @@ static int vivid_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int vivid_remove(struct platform_device *pdev)
+static void vivid_remove(struct platform_device *pdev)
 {
 	struct vivid_dev *dev;
 	unsigned int i, j;
@@ -2138,7 +2101,6 @@ static int vivid_remove(struct platform_device *pdev)
 		v4l2_device_put(&dev->v4l2_dev);
 		vivid_devs[i] = NULL;
 	}
-	return 0;
 }
 
 static void vivid_pdev_release(struct device *dev)
@@ -2152,7 +2114,7 @@ static struct platform_device vivid_pdev = {
 
 static struct platform_driver vivid_pdrv = {
 	.probe		= vivid_probe,
-	.remove		= vivid_remove,
+	.remove_new	= vivid_remove,
 	.driver		= {
 		.name	= "vivid",
 	},
diff --git a/drivers/media/test-drivers/vivid/vivid-core.h b/drivers/media/test-drivers/vivid/vivid-core.h
index 473f3598db5a..cfb8e66083f6 100644
--- a/drivers/media/test-drivers/vivid/vivid-core.h
+++ b/drivers/media/test-drivers/vivid/vivid-core.h
@@ -22,8 +22,6 @@
 #define dprintk(dev, level, fmt, arg...) \
 	v4l2_dbg(level, vivid_debug, &dev->v4l2_dev, fmt, ## arg)
 
-/* The maximum number of clip rectangles */
-#define MAX_CLIPS  16
 /* The maximum number of inputs */
 #define MAX_INPUTS 16
 /* The maximum number of outputs */
@@ -345,17 +343,6 @@ struct vivid_dev {
 
 	u32				power_present;
 
-	/* Capture Overlay */
-	struct v4l2_framebuffer		fb_cap;
-	struct v4l2_fh			*overlay_cap_owner;
-	void				*fb_vbase_cap;
-	int				overlay_cap_top, overlay_cap_left;
-	enum v4l2_field			overlay_cap_field;
-	void				*bitmap_cap;
-	struct v4l2_clip		clips_cap[MAX_CLIPS];
-	struct v4l2_clip		try_clips_cap[MAX_CLIPS];
-	unsigned			clipcount_cap;
-
 	/* Output */
 	unsigned			output;
 	v4l2_std_id			std_out;
@@ -383,10 +370,6 @@ struct vivid_dev {
 	void				*fb_vbase_out;
 	bool				overlay_out_enabled;
 	int				overlay_out_top, overlay_out_left;
-	void				*bitmap_out;
-	struct v4l2_clip		clips_out[MAX_CLIPS];
-	struct v4l2_clip		try_clips_out[MAX_CLIPS];
-	unsigned			clipcount_out;
 	unsigned			fbuf_out_flags;
 	u32				chromakey_out;
 	u8				global_alpha_out;
@@ -613,6 +596,4 @@ static inline bool vivid_is_hdmi_out(const struct vivid_dev *dev)
 	return dev->output_type[dev->output] == HDMI;
 }
 
-bool vivid_validate_fb(const struct v4l2_framebuffer *a);
-
 #endif
diff --git a/drivers/media/test-drivers/vivid/vivid-kthread-cap.c b/drivers/media/test-drivers/vivid/vivid-kthread-cap.c
index ee65d20314d3..42048727d7ff 100644
--- a/drivers/media/test-drivers/vivid/vivid-kthread-cap.c
+++ b/drivers/media/test-drivers/vivid/vivid-kthread-cap.c
@@ -53,31 +53,10 @@ static void copy_pix(struct vivid_dev *dev, int win_y, int win_x,
 			u16 *cap, const u16 *osd)
 {
 	u16 out;
-	int left = dev->overlay_out_left;
-	int top = dev->overlay_out_top;
-	int fb_x = win_x + left;
-	int fb_y = win_y + top;
-	int i;
 
 	out = *cap;
 	*cap = *osd;
-	if (dev->bitmap_out) {
-		const u8 *p = dev->bitmap_out;
-		unsigned stride = (dev->compose_out.width + 7) / 8;
 
-		win_x -= dev->compose_out.left;
-		win_y -= dev->compose_out.top;
-		if (!(p[stride * win_y + win_x / 8] & (1 << (win_x & 7))))
-			return;
-	}
-
-	for (i = 0; i < dev->clipcount_out; i++) {
-		struct v4l2_rect *r = &dev->clips_out[i].c;
-
-		if (fb_y >= r->top && fb_y < r->top + r->height &&
-		    fb_x >= r->left && fb_x < r->left + r->width)
-			return;
-	}
 	if ((dev->fbuf_out_flags & V4L2_FBUF_FLAG_CHROMAKEY) &&
 	    *osd != dev->chromakey_out)
 		return;
@@ -251,7 +230,7 @@ static noinline_for_stack int vivid_copy_buffer(struct vivid_dev *dev, unsigned
 	u8 *voutbuf;
 	u8 *vosdbuf = NULL;
 	unsigned y;
-	bool blend = dev->bitmap_out || dev->clipcount_out || dev->fbuf_out_flags;
+	bool blend = dev->fbuf_out_flags;
 	/* Coarse scaling with Bresenham */
 	unsigned vid_out_int_part;
 	unsigned vid_out_fract_part;
@@ -554,109 +533,6 @@ static void vivid_fillbuff(struct vivid_dev *dev, struct vivid_buffer *buf)
 	}
 }
 
-/*
- * Return true if this pixel coordinate is a valid video pixel.
- */
-static bool valid_pix(struct vivid_dev *dev, int win_y, int win_x, int fb_y, int fb_x)
-{
-	int i;
-
-	if (dev->bitmap_cap) {
-		/*
-		 * Only if the corresponding bit in the bitmap is set can
-		 * the video pixel be shown. Coordinates are relative to
-		 * the overlay window set by VIDIOC_S_FMT.
-		 */
-		const u8 *p = dev->bitmap_cap;
-		unsigned stride = (dev->compose_cap.width + 7) / 8;
-
-		if (!(p[stride * win_y + win_x / 8] & (1 << (win_x & 7))))
-			return false;
-	}
-
-	for (i = 0; i < dev->clipcount_cap; i++) {
-		/*
-		 * Only if the framebuffer coordinate is not in any of the
-		 * clip rectangles will be video pixel be shown.
-		 */
-		struct v4l2_rect *r = &dev->clips_cap[i].c;
-
-		if (fb_y >= r->top && fb_y < r->top + r->height &&
-		    fb_x >= r->left && fb_x < r->left + r->width)
-			return false;
-	}
-	return true;
-}
-
-/*
- * Draw the image into the overlay buffer.
- * Note that the combination of overlay and multiplanar is not supported.
- */
-static void vivid_overlay(struct vivid_dev *dev, struct vivid_buffer *buf)
-{
-	struct tpg_data *tpg = &dev->tpg;
-	unsigned pixsize = tpg_g_twopixelsize(tpg, 0) / 2;
-	void *vbase = dev->fb_vbase_cap;
-	void *vbuf = vb2_plane_vaddr(&buf->vb.vb2_buf, 0);
-	unsigned img_width = dev->compose_cap.width;
-	unsigned img_height = dev->compose_cap.height;
-	unsigned stride = tpg->bytesperline[0];
-	/* if quick is true, then valid_pix() doesn't have to be called */
-	bool quick = dev->bitmap_cap == NULL && dev->clipcount_cap == 0;
-	int x, y, w, out_x = 0;
-
-	/*
-	 * Overlay support is only supported for formats that have a twopixelsize
-	 * that's >= 2. Warn and bail out if that's not the case.
-	 */
-	if (WARN_ON(pixsize == 0))
-		return;
-	if ((dev->overlay_cap_field == V4L2_FIELD_TOP ||
-	     dev->overlay_cap_field == V4L2_FIELD_BOTTOM) &&
-	    dev->overlay_cap_field != buf->vb.field)
-		return;
-
-	vbuf += dev->compose_cap.left * pixsize + dev->compose_cap.top * stride;
-	x = dev->overlay_cap_left;
-	w = img_width;
-	if (x < 0) {
-		out_x = -x;
-		w = w - out_x;
-		x = 0;
-	} else {
-		w = dev->fb_cap.fmt.width - x;
-		if (w > img_width)
-			w = img_width;
-	}
-	if (w <= 0)
-		return;
-	if (dev->overlay_cap_top >= 0)
-		vbase += dev->overlay_cap_top * dev->fb_cap.fmt.bytesperline;
-	for (y = dev->overlay_cap_top;
-	     y < dev->overlay_cap_top + (int)img_height;
-	     y++, vbuf += stride) {
-		int px;
-
-		if (y < 0 || y > dev->fb_cap.fmt.height)
-			continue;
-		if (quick) {
-			memcpy(vbase + x * pixsize,
-			       vbuf + out_x * pixsize, w * pixsize);
-			vbase += dev->fb_cap.fmt.bytesperline;
-			continue;
-		}
-		for (px = 0; px < w; px++) {
-			if (!valid_pix(dev, y - dev->overlay_cap_top,
-				       px + out_x, y, px + x))
-				continue;
-			memcpy(vbase + (px + x) * pixsize,
-			       vbuf + (px + out_x) * pixsize,
-			       pixsize);
-		}
-		vbase += dev->fb_cap.fmt.bytesperline;
-	}
-}
-
 static void vivid_cap_update_frame_period(struct vivid_dev *dev)
 {
 	u64 f_period;
@@ -730,11 +606,6 @@ static noinline_for_stack void vivid_thread_vid_cap_tick(struct vivid_dev *dev,
 		dprintk(dev, 1, "filled buffer %d\n",
 			vid_cap_buf->vb.vb2_buf.index);
 
-		/* Handle overlay */
-		if (dev->overlay_cap_owner && dev->fb_cap.base &&
-			dev->fb_cap.fmt.pixelformat == dev->fmt_cap->fourcc)
-			vivid_overlay(dev, vid_cap_buf);
-
 		v4l2_ctrl_request_complete(vid_cap_buf->vb.vb2_buf.req_obj.req,
 					   &dev->ctrl_hdl_vid_cap);
 		vb2_buffer_done(&vid_cap_buf->vb.vb2_buf, dev->dqbuf_error ?
diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c
index c0999581c599..801286dc1448 100644
--- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c
+++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c
@@ -21,30 +21,6 @@
 #include "vivid-kthread-cap.h"
 #include "vivid-vid-cap.h"
 
-static const struct vivid_fmt formats_ovl[] = {
-	{
-		.fourcc   = V4L2_PIX_FMT_RGB565, /* gggbbbbb rrrrrggg */
-		.vdownsampling = { 1 },
-		.bit_depth = { 16 },
-		.planes   = 1,
-		.buffers = 1,
-	},
-	{
-		.fourcc   = V4L2_PIX_FMT_XRGB555, /* gggbbbbb arrrrrgg */
-		.vdownsampling = { 1 },
-		.bit_depth = { 16 },
-		.planes   = 1,
-		.buffers = 1,
-	},
-	{
-		.fourcc   = V4L2_PIX_FMT_ARGB555, /* gggbbbbb arrrrrgg */
-		.vdownsampling = { 1 },
-		.bit_depth = { 16 },
-		.planes   = 1,
-		.buffers = 1,
-	},
-};
-
 /* The number of discrete webcam framesizes */
 #define VIVID_WEBCAM_SIZES 6
 /* The number of discrete webcam frameintervals */
@@ -447,18 +423,10 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls)
 		tpg_s_rgb_range(&dev->tpg, v4l2_ctrl_g_ctrl(dev->rgb_range_cap));
 		break;
 	}
-	vfree(dev->bitmap_cap);
-	dev->bitmap_cap = NULL;
 	vivid_update_quality(dev);
 	tpg_reset_source(&dev->tpg, dev->src_rect.width, dev->src_rect.height, dev->field_cap);
 	dev->crop_cap = dev->src_rect;
 	dev->crop_bounds_cap = dev->src_rect;
-	if (dev->bitmap_cap &&
-	    (dev->compose_cap.width != dev->crop_cap.width ||
-	     dev->compose_cap.height != dev->crop_cap.height)) {
-		vfree(dev->bitmap_cap);
-		dev->bitmap_cap = NULL;
-	}
 	dev->compose_cap = dev->crop_cap;
 	if (V4L2_FIELD_HAS_T_OR_B(dev->field_cap))
 		dev->compose_cap.height /= 2;
@@ -701,11 +669,6 @@ int vivid_s_fmt_vid_cap(struct file *file, void *priv,
 		return -EBUSY;
 	}
 
-	if (dev->overlay_cap_owner && dev->fb_cap.fmt.pixelformat != mp->pixelformat) {
-		dprintk(dev, 1, "overlay is active, can't change pixelformat\n");
-		return -EBUSY;
-	}
-
 	dev->fmt_cap = vivid_get_format(dev, mp->pixelformat);
 	if (V4L2_FIELD_HAS_T_OR_B(mp->field))
 		factor = 2;
@@ -927,8 +890,6 @@ int vivid_vid_cap_s_selection(struct file *file, void *fh, struct v4l2_selection
 	struct vivid_dev *dev = video_drvdata(file);
 	struct v4l2_rect *crop = &dev->crop_cap;
 	struct v4l2_rect *compose = &dev->compose_cap;
-	unsigned orig_compose_w = compose->width;
-	unsigned orig_compose_h = compose->height;
 	unsigned factor = V4L2_FIELD_HAS_T_OR_B(dev->field_cap) ? 2 : 1;
 	int ret;
 
@@ -1052,11 +1013,6 @@ int vivid_vid_cap_s_selection(struct file *file, void *fh, struct v4l2_selection
 		return -EINVAL;
 	}
 
-	if (dev->bitmap_cap && (compose->width != orig_compose_w ||
-				compose->height != orig_compose_h)) {
-		vfree(dev->bitmap_cap);
-		dev->bitmap_cap = NULL;
-	}
 	tpg_s_crop_compose(&dev->tpg, crop, compose);
 	return 0;
 }
@@ -1084,234 +1040,6 @@ int vivid_vid_cap_g_pixelaspect(struct file *file, void *priv,
 	return 0;
 }
 
-int vidioc_enum_fmt_vid_overlay(struct file *file, void  *priv,
-					struct v4l2_fmtdesc *f)
-{
-	struct vivid_dev *dev = video_drvdata(file);
-	const struct vivid_fmt *fmt;
-
-	if (dev->multiplanar)
-		return -ENOTTY;
-
-	if (f->index >= ARRAY_SIZE(formats_ovl))
-		return -EINVAL;
-
-	fmt = &formats_ovl[f->index];
-
-	f->pixelformat = fmt->fourcc;
-	return 0;
-}
-
-int vidioc_g_fmt_vid_overlay(struct file *file, void *priv,
-					struct v4l2_format *f)
-{
-	struct vivid_dev *dev = video_drvdata(file);
-	const struct v4l2_rect *compose = &dev->compose_cap;
-	struct v4l2_window *win = &f->fmt.win;
-	unsigned clipcount = win->clipcount;
-
-	if (dev->multiplanar)
-		return -ENOTTY;
-
-	win->w.top = dev->overlay_cap_top;
-	win->w.left = dev->overlay_cap_left;
-	win->w.width = compose->width;
-	win->w.height = compose->height;
-	win->field = dev->overlay_cap_field;
-	win->clipcount = dev->clipcount_cap;
-	if (clipcount > dev->clipcount_cap)
-		clipcount = dev->clipcount_cap;
-	if (dev->bitmap_cap == NULL)
-		win->bitmap = NULL;
-	else if (win->bitmap) {
-		if (copy_to_user(win->bitmap, dev->bitmap_cap,
-		    ((compose->width + 7) / 8) * compose->height))
-			return -EFAULT;
-	}
-	if (clipcount && win->clips)
-		memcpy(win->clips, dev->clips_cap,
-		       clipcount * sizeof(dev->clips_cap[0]));
-	return 0;
-}
-
-int vidioc_try_fmt_vid_overlay(struct file *file, void *priv,
-					struct v4l2_format *f)
-{
-	struct vivid_dev *dev = video_drvdata(file);
-	const struct v4l2_rect *compose = &dev->compose_cap;
-	struct v4l2_window *win = &f->fmt.win;
-	int i, j;
-
-	if (dev->multiplanar)
-		return -ENOTTY;
-
-	win->w.left = clamp_t(int, win->w.left,
-			      -dev->fb_cap.fmt.width, dev->fb_cap.fmt.width);
-	win->w.top = clamp_t(int, win->w.top,
-			     -dev->fb_cap.fmt.height, dev->fb_cap.fmt.height);
-	win->w.width = compose->width;
-	win->w.height = compose->height;
-	if (win->field != V4L2_FIELD_BOTTOM && win->field != V4L2_FIELD_TOP)
-		win->field = V4L2_FIELD_ANY;
-	win->chromakey = 0;
-	win->global_alpha = 0;
-	if (win->clipcount && !win->clips)
-		win->clipcount = 0;
-	if (win->clipcount > MAX_CLIPS)
-		win->clipcount = MAX_CLIPS;
-	if (win->clipcount) {
-		memcpy(dev->try_clips_cap, win->clips,
-		       win->clipcount * sizeof(dev->clips_cap[0]));
-		for (i = 0; i < win->clipcount; i++) {
-			struct v4l2_rect *r = &dev->try_clips_cap[i].c;
-
-			r->top = clamp_t(s32, r->top, 0, dev->fb_cap.fmt.height - 1);
-			r->height = clamp_t(s32, r->height, 1, dev->fb_cap.fmt.height - r->top);
-			r->left = clamp_t(u32, r->left, 0, dev->fb_cap.fmt.width - 1);
-			r->width = clamp_t(u32, r->width, 1, dev->fb_cap.fmt.width - r->left);
-		}
-		/*
-		 * Yeah, so sue me, it's an O(n^2) algorithm. But n is a small
-		 * number and it's typically a one-time deal.
-		 */
-		for (i = 0; i < win->clipcount - 1; i++) {
-			struct v4l2_rect *r1 = &dev->try_clips_cap[i].c;
-
-			for (j = i + 1; j < win->clipcount; j++) {
-				struct v4l2_rect *r2 = &dev->try_clips_cap[j].c;
-
-				if (v4l2_rect_overlap(r1, r2))
-					return -EINVAL;
-			}
-		}
-		memcpy(win->clips, dev->try_clips_cap,
-		       win->clipcount * sizeof(dev->clips_cap[0]));
-	}
-	return 0;
-}
-
-int vidioc_s_fmt_vid_overlay(struct file *file, void *priv,
-					struct v4l2_format *f)
-{
-	struct vivid_dev *dev = video_drvdata(file);
-	const struct v4l2_rect *compose = &dev->compose_cap;
-	struct v4l2_window *win = &f->fmt.win;
-	int ret = vidioc_try_fmt_vid_overlay(file, priv, f);
-	unsigned bitmap_size = ((compose->width + 7) / 8) * compose->height;
-	unsigned clips_size = win->clipcount * sizeof(dev->clips_cap[0]);
-	void *new_bitmap = NULL;
-
-	if (ret)
-		return ret;
-
-	if (win->bitmap) {
-		new_bitmap = vzalloc(bitmap_size);
-
-		if (new_bitmap == NULL)
-			return -ENOMEM;
-		if (copy_from_user(new_bitmap, win->bitmap, bitmap_size)) {
-			vfree(new_bitmap);
-			return -EFAULT;
-		}
-	}
-
-	dev->overlay_cap_top = win->w.top;
-	dev->overlay_cap_left = win->w.left;
-	dev->overlay_cap_field = win->field;
-	vfree(dev->bitmap_cap);
-	dev->bitmap_cap = new_bitmap;
-	dev->clipcount_cap = win->clipcount;
-	if (dev->clipcount_cap)
-		memcpy(dev->clips_cap, dev->try_clips_cap, clips_size);
-	return 0;
-}
-
-int vivid_vid_cap_overlay(struct file *file, void *fh, unsigned i)
-{
-	struct vivid_dev *dev = video_drvdata(file);
-
-	if (dev->multiplanar)
-		return -ENOTTY;
-
-	if (i && dev->fb_vbase_cap == NULL)
-		return -EINVAL;
-
-	if (i && dev->fb_cap.fmt.pixelformat != dev->fmt_cap->fourcc) {
-		dprintk(dev, 1, "mismatch between overlay and video capture pixelformats\n");
-		return -EINVAL;
-	}
-
-	if (dev->overlay_cap_owner && dev->overlay_cap_owner != fh)
-		return -EBUSY;
-	dev->overlay_cap_owner = i ? fh : NULL;
-	return 0;
-}
-
-int vivid_vid_cap_g_fbuf(struct file *file, void *fh,
-				struct v4l2_framebuffer *a)
-{
-	struct vivid_dev *dev = video_drvdata(file);
-
-	if (dev->multiplanar)
-		return -ENOTTY;
-
-	*a = dev->fb_cap;
-	a->capability = V4L2_FBUF_CAP_BITMAP_CLIPPING |
-			V4L2_FBUF_CAP_LIST_CLIPPING;
-	a->flags = V4L2_FBUF_FLAG_PRIMARY;
-	a->fmt.field = V4L2_FIELD_NONE;
-	a->fmt.colorspace = V4L2_COLORSPACE_SRGB;
-	a->fmt.priv = 0;
-	return 0;
-}
-
-int vivid_vid_cap_s_fbuf(struct file *file, void *fh,
-				const struct v4l2_framebuffer *a)
-{
-	struct vivid_dev *dev = video_drvdata(file);
-	const struct vivid_fmt *fmt;
-
-	if (dev->multiplanar)
-		return -ENOTTY;
-
-	if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RAWIO))
-		return -EPERM;
-
-	if (dev->overlay_cap_owner)
-		return -EBUSY;
-
-	if (a->base == NULL) {
-		dev->fb_cap.base = NULL;
-		dev->fb_vbase_cap = NULL;
-		return 0;
-	}
-
-	if (a->fmt.width < 48 || a->fmt.height < 32)
-		return -EINVAL;
-	fmt = vivid_get_format(dev, a->fmt.pixelformat);
-	if (!fmt || !fmt->can_do_overlay)
-		return -EINVAL;
-	if (a->fmt.bytesperline < (a->fmt.width * fmt->bit_depth[0]) / 8)
-		return -EINVAL;
-	if (a->fmt.bytesperline > a->fmt.sizeimage / a->fmt.height)
-		return -EINVAL;
-
-	/*
-	 * Only support the framebuffer of one of the vivid instances.
-	 * Anything else is rejected.
-	 */
-	if (!vivid_validate_fb(a))
-		return -EINVAL;
-
-	dev->fb_vbase_cap = phys_to_virt((unsigned long)a->base);
-	dev->fb_cap = *a;
-	dev->overlay_cap_left = clamp_t(int, dev->overlay_cap_left,
-				    -dev->fb_cap.fmt.width, dev->fb_cap.fmt.width);
-	dev->overlay_cap_top = clamp_t(int, dev->overlay_cap_top,
-				   -dev->fb_cap.fmt.height, dev->fb_cap.fmt.height);
-	return 0;
-}
-
 static const struct v4l2_audio vivid_audio_inputs[] = {
 	{ 0, "TV", V4L2_AUDCAP_STEREO },
 	{ 1, "Line-In", V4L2_AUDCAP_STEREO },
diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.h b/drivers/media/test-drivers/vivid/vivid-vid-cap.h
index 1e422a59eeab..949768652d38 100644
--- a/drivers/media/test-drivers/vivid/vivid-vid-cap.h
+++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.h
@@ -33,9 +33,6 @@ int vidioc_enum_fmt_vid_overlay(struct file *file, void  *priv, struct v4l2_fmtd
 int vidioc_g_fmt_vid_overlay(struct file *file, void *priv, struct v4l2_format *f);
 int vidioc_try_fmt_vid_overlay(struct file *file, void *priv, struct v4l2_format *f);
 int vidioc_s_fmt_vid_overlay(struct file *file, void *priv, struct v4l2_format *f);
-int vivid_vid_cap_overlay(struct file *file, void *fh, unsigned i);
-int vivid_vid_cap_g_fbuf(struct file *file, void *fh, struct v4l2_framebuffer *a);
-int vivid_vid_cap_s_fbuf(struct file *file, void *fh, const struct v4l2_framebuffer *a);
 int vidioc_enum_input(struct file *file, void *priv, struct v4l2_input *inp);
 int vidioc_g_input(struct file *file, void *priv, unsigned *i);
 int vidioc_s_input(struct file *file, void *priv, unsigned i);
diff --git a/drivers/media/test-drivers/vivid/vivid-vid-out.c b/drivers/media/test-drivers/vivid/vivid-vid-out.c
index 9f731f085179..184a6df2c29f 100644
--- a/drivers/media/test-drivers/vivid/vivid-vid-out.c
+++ b/drivers/media/test-drivers/vivid/vivid-vid-out.c
@@ -793,11 +793,6 @@ int vivid_vid_out_s_selection(struct file *file, void *fh, struct v4l2_selection
 		}
 		s->r.top *= factor;
 		s->r.height *= factor;
-		if (dev->bitmap_out && (compose->width != s->r.width ||
-					compose->height != s->r.height)) {
-			vfree(dev->bitmap_out);
-			dev->bitmap_out = NULL;
-		}
 		*compose = s->r;
 		break;
 	default:
@@ -836,7 +831,6 @@ int vidioc_g_fmt_vid_out_overlay(struct file *file, void *priv,
 	struct vivid_dev *dev = video_drvdata(file);
 	const struct v4l2_rect *compose = &dev->compose_out;
 	struct v4l2_window *win = &f->fmt.win;
-	unsigned clipcount = win->clipcount;
 
 	if (!dev->has_fb)
 		return -EINVAL;
@@ -844,22 +838,9 @@ int vidioc_g_fmt_vid_out_overlay(struct file *file, void *priv,
 	win->w.left = dev->overlay_out_left;
 	win->w.width = compose->width;
 	win->w.height = compose->height;
-	win->clipcount = dev->clipcount_out;
 	win->field = V4L2_FIELD_ANY;
 	win->chromakey = dev->chromakey_out;
 	win->global_alpha = dev->global_alpha_out;
-	if (clipcount > dev->clipcount_out)
-		clipcount = dev->clipcount_out;
-	if (dev->bitmap_out == NULL)
-		win->bitmap = NULL;
-	else if (win->bitmap) {
-		if (copy_to_user(win->bitmap, dev->bitmap_out,
-		    ((dev->compose_out.width + 7) / 8) * dev->compose_out.height))
-			return -EFAULT;
-	}
-	if (clipcount && win->clips)
-		memcpy(win->clips, dev->clips_out,
-		       clipcount * sizeof(dev->clips_out[0]));
 	return 0;
 }
 
@@ -869,7 +850,6 @@ int vidioc_try_fmt_vid_out_overlay(struct file *file, void *priv,
 	struct vivid_dev *dev = video_drvdata(file);
 	const struct v4l2_rect *compose = &dev->compose_out;
 	struct v4l2_window *win = &f->fmt.win;
-	int i, j;
 
 	if (!dev->has_fb)
 		return -EINVAL;
@@ -884,38 +864,6 @@ int vidioc_try_fmt_vid_out_overlay(struct file *file, void *priv,
 	 * so always set this to ANY.
 	 */
 	win->field = V4L2_FIELD_ANY;
-	if (win->clipcount && !win->clips)
-		win->clipcount = 0;
-	if (win->clipcount > MAX_CLIPS)
-		win->clipcount = MAX_CLIPS;
-	if (win->clipcount) {
-		memcpy(dev->try_clips_out, win->clips,
-		       win->clipcount * sizeof(dev->clips_out[0]));
-		for (i = 0; i < win->clipcount; i++) {
-			struct v4l2_rect *r = &dev->try_clips_out[i].c;
-
-			r->top = clamp_t(s32, r->top, 0, dev->display_height - 1);
-			r->height = clamp_t(s32, r->height, 1, dev->display_height - r->top);
-			r->left = clamp_t(u32, r->left, 0, dev->display_width - 1);
-			r->width = clamp_t(u32, r->width, 1, dev->display_width - r->left);
-		}
-		/*
-		 * Yeah, so sue me, it's an O(n^2) algorithm. But n is a small
-		 * number and it's typically a one-time deal.
-		 */
-		for (i = 0; i < win->clipcount - 1; i++) {
-			struct v4l2_rect *r1 = &dev->try_clips_out[i].c;
-
-			for (j = i + 1; j < win->clipcount; j++) {
-				struct v4l2_rect *r2 = &dev->try_clips_out[j].c;
-
-				if (v4l2_rect_overlap(r1, r2))
-					return -EINVAL;
-			}
-		}
-		memcpy(win->clips, dev->try_clips_out,
-		       win->clipcount * sizeof(dev->clips_out[0]));
-	}
 	return 0;
 }
 
@@ -923,34 +871,14 @@ int vidioc_s_fmt_vid_out_overlay(struct file *file, void *priv,
 					struct v4l2_format *f)
 {
 	struct vivid_dev *dev = video_drvdata(file);
-	const struct v4l2_rect *compose = &dev->compose_out;
 	struct v4l2_window *win = &f->fmt.win;
 	int ret = vidioc_try_fmt_vid_out_overlay(file, priv, f);
-	unsigned bitmap_size = ((compose->width + 7) / 8) * compose->height;
-	unsigned clips_size = win->clipcount * sizeof(dev->clips_out[0]);
-	void *new_bitmap = NULL;
 
 	if (ret)
 		return ret;
 
-	if (win->bitmap) {
-		new_bitmap = vzalloc(bitmap_size);
-
-		if (!new_bitmap)
-			return -ENOMEM;
-		if (copy_from_user(new_bitmap, win->bitmap, bitmap_size)) {
-			vfree(new_bitmap);
-			return -EFAULT;
-		}
-	}
-
 	dev->overlay_out_top = win->w.top;
 	dev->overlay_out_left = win->w.left;
-	vfree(dev->bitmap_out);
-	dev->bitmap_out = new_bitmap;
-	dev->clipcount_out = win->clipcount;
-	if (dev->clipcount_out)
-		memcpy(dev->clips_out, dev->try_clips_out, clips_size);
 	dev->chromakey_out = win->chromakey;
 	dev->global_alpha_out = win->global_alpha;
 	return ret;
@@ -975,8 +903,6 @@ int vivid_vid_out_g_fbuf(struct file *file, void *fh,
 	struct vivid_dev *dev = video_drvdata(file);
 
 	a->capability = V4L2_FBUF_CAP_EXTERNOVERLAY |
-			V4L2_FBUF_CAP_BITMAP_CLIPPING |
-			V4L2_FBUF_CAP_LIST_CLIPPING |
 			V4L2_FBUF_CAP_CHROMAKEY |
 			V4L2_FBUF_CAP_SRC_CHROMAKEY |
 			V4L2_FBUF_CAP_GLOBAL_ALPHA |
diff --git a/drivers/media/tuners/it913x.c b/drivers/media/tuners/it913x.c
index 7696a28fe407..4d5b1c878028 100644
--- a/drivers/media/tuners/it913x.c
+++ b/drivers/media/tuners/it913x.c
@@ -419,7 +419,7 @@ err:
 	return ret;
 }
 
-static int it913x_remove(struct platform_device *pdev)
+static void it913x_remove(struct platform_device *pdev)
 {
 	struct it913x_dev *dev = platform_get_drvdata(pdev);
 	struct dvb_frontend *fe = dev->fe;
@@ -429,8 +429,6 @@ static int it913x_remove(struct platform_device *pdev)
 	memset(&fe->ops.tuner_ops, 0, sizeof(struct dvb_tuner_ops));
 	fe->tuner_priv = NULL;
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct platform_device_id it913x_id_table[] = {
@@ -446,7 +444,7 @@ static struct platform_driver it913x_driver = {
 		.suppress_bind_attrs	= true,
 	},
 	.probe		= it913x_probe,
-	.remove		= it913x_remove,
+	.remove_new	= it913x_remove,
 	.id_table	= it913x_id_table,
 };
 
diff --git a/drivers/media/tuners/mxl5005s.c b/drivers/media/tuners/mxl5005s.c
index 3a509038c8df..06dfab9fb8cb 100644
--- a/drivers/media/tuners/mxl5005s.c
+++ b/drivers/media/tuners/mxl5005s.c
@@ -3423,9 +3423,11 @@ static u16 MXL_ControlWrite_Group(struct dvb_frontend *fe, u16 controlNum,
 
 			if (controlNum == state->Init_Ctrl[i].Ctrl_Num) {
 
-				highLimit = 1 << state->Init_Ctrl[i].size;
+				u16 size = min_t(u16, state->Init_Ctrl[i].size,
+					       ARRAY_SIZE(state->Init_Ctrl[i].val));
+				highLimit = 1 << size;
 				if (value < highLimit) {
-					for (j = 0; j < state->Init_Ctrl[i].size; j++) {
+					for (j = 0; j < size; j++) {
 						state->Init_Ctrl[i].val[j] = (u8)((value >> j) & 0x01);
 						MXL_RegWriteBit(fe, (u8)(state->Init_Ctrl[i].addr[j]),
 							(u8)(state->Init_Ctrl[i].bit[j]),
@@ -3442,9 +3444,11 @@ static u16 MXL_ControlWrite_Group(struct dvb_frontend *fe, u16 controlNum,
 
 			if (controlNum == state->CH_Ctrl[i].Ctrl_Num) {
 
-				highLimit = 1 << state->CH_Ctrl[i].size;
+				u16 size = min_t(u16, state->CH_Ctrl[i].size,
+					       ARRAY_SIZE(state->CH_Ctrl[i].val));
+				highLimit = 1 << size;
 				if (value < highLimit) {
-					for (j = 0; j < state->CH_Ctrl[i].size; j++) {
+					for (j = 0; j < size; j++) {
 						state->CH_Ctrl[i].val[j] = (u8)((value >> j) & 0x01);
 						MXL_RegWriteBit(fe, (u8)(state->CH_Ctrl[i].addr[j]),
 							(u8)(state->CH_Ctrl[i].bit[j]),
diff --git a/drivers/media/usb/au0828/au0828-core.c b/drivers/media/usb/au0828/au0828-core.c
index 877e85a451cb..b3a09d3ac7d2 100644
--- a/drivers/media/usb/au0828/au0828-core.c
+++ b/drivers/media/usb/au0828/au0828-core.c
@@ -211,7 +211,7 @@ static int au0828_media_device_init(struct au0828_dev *dev,
 static void au0828_media_graph_notify(struct media_entity *new,
 				      void *notify_data)
 {
-	struct au0828_dev *dev = (struct au0828_dev *) notify_data;
+	struct au0828_dev *dev = notify_data;
 	int ret;
 	struct media_entity *entity, *mixer = NULL, *decoder = NULL;
 
@@ -627,14 +627,9 @@ static int au0828_media_device_register(struct au0828_dev *dev,
 	/* register entity_notify callback */
 	dev->entity_notify.notify_data = (void *) dev;
 	dev->entity_notify.notify = (void *) au0828_media_graph_notify;
-	ret = media_device_register_entity_notify(dev->media_dev,
+	media_device_register_entity_notify(dev->media_dev,
 						  &dev->entity_notify);
-	if (ret) {
-		dev_err(&udev->dev,
-			"Media Device register entity_notify Error: %d\n",
-			ret);
-		return ret;
-	}
+
 	/* set enable_source */
 	mutex_lock(&dev->media_dev->graph_mutex);
 	dev->media_dev->source_priv = (void *) dev;
diff --git a/drivers/media/usb/au0828/au0828-dvb.c b/drivers/media/usb/au0828/au0828-dvb.c
index 2a8691a0d7fa..09f9948c6f8e 100644
--- a/drivers/media/usb/au0828/au0828-dvb.c
+++ b/drivers/media/usb/au0828/au0828-dvb.c
@@ -273,7 +273,7 @@ static void au0828_stop_transport(struct au0828_dev *dev, int full_stop)
 static int au0828_dvb_start_feed(struct dvb_demux_feed *feed)
 {
 	struct dvb_demux *demux = feed->demux;
-	struct au0828_dev *dev = (struct au0828_dev *) demux->priv;
+	struct au0828_dev *dev = demux->priv;
 	struct au0828_dvb *dvb = &dev->dvb;
 	int ret = 0;
 
@@ -305,7 +305,7 @@ static int au0828_dvb_start_feed(struct dvb_demux_feed *feed)
 static int au0828_dvb_stop_feed(struct dvb_demux_feed *feed)
 {
 	struct dvb_demux *demux = feed->demux;
-	struct au0828_dev *dev = (struct au0828_dev *) demux->priv;
+	struct au0828_dev *dev = demux->priv;
 	struct au0828_dvb *dvb = &dev->dvb;
 	int ret = 0;
 
diff --git a/drivers/media/usb/dvb-usb/cxusb-analog.c b/drivers/media/usb/dvb-usb/cxusb-analog.c
index e93183ddd797..deba5224cb8d 100644
--- a/drivers/media/usb/dvb-usb/cxusb-analog.c
+++ b/drivers/media/usb/dvb-usb/cxusb-analog.c
@@ -1014,7 +1014,10 @@ static int cxusb_medion_try_s_fmt_vid_cap(struct file *file,
 {
 	struct dvb_usb_device *dvbdev = video_drvdata(file);
 	struct cxusb_medion_dev *cxdev = dvbdev->priv;
-	struct v4l2_subdev_format subfmt;
+	struct v4l2_subdev_format subfmt = {
+		.which = isset ? V4L2_SUBDEV_FORMAT_ACTIVE :
+			 V4L2_SUBDEV_FORMAT_TRY,
+	};
 	u32 field;
 	int ret;
 
@@ -1024,9 +1027,6 @@ static int cxusb_medion_try_s_fmt_vid_cap(struct file *file,
 	field = vb2_start_streaming_called(&cxdev->videoqueue) ?
 		cxdev->field_order : cxusb_medion_field_order(cxdev);
 
-	memset(&subfmt, 0, sizeof(subfmt));
-	subfmt.which = isset ? V4L2_SUBDEV_FORMAT_ACTIVE :
-		V4L2_SUBDEV_FORMAT_TRY;
 	subfmt.format.width = f->fmt.pix.width & ~1;
 	subfmt.format.height = f->fmt.pix.height & ~1;
 	subfmt.format.code = MEDIA_BUS_FMT_FIXED;
@@ -1464,7 +1464,9 @@ int cxusb_medion_analog_init(struct dvb_usb_device *dvbdev)
 					    .buf = tuner_analog_msg_data,
 					    .len =
 					    sizeof(tuner_analog_msg_data) };
-	struct v4l2_subdev_format subfmt;
+	struct v4l2_subdev_format subfmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	int ret;
 
 	/* switch tuner to analog mode so IF demod will become accessible */
@@ -1507,8 +1509,6 @@ int cxusb_medion_analog_init(struct dvb_usb_device *dvbdev)
 	v4l2_subdev_call(cxdev->tuner, video, s_std, cxdev->norm);
 	v4l2_subdev_call(cxdev->cx25840, video, s_std, cxdev->norm);
 
-	memset(&subfmt, 0, sizeof(subfmt));
-	subfmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	subfmt.format.width = cxdev->width;
 	subfmt.format.height = cxdev->height;
 	subfmt.format.code = MEDIA_BUS_FMT_FIXED;
diff --git a/drivers/media/usb/dvb-usb/pctv452e.c b/drivers/media/usb/dvb-usb/pctv452e.c
index f0794c68c622..da42c989e071 100644
--- a/drivers/media/usb/dvb-usb/pctv452e.c
+++ b/drivers/media/usb/dvb-usb/pctv452e.c
@@ -26,6 +26,8 @@
 #include <media/dvb_ca_en50221.h>
 #include "ttpci-eeprom.h"
 
+#include <linux/etherdevice.h>
+
 static int debug;
 module_param(debug, int, 0644);
 MODULE_PARM_DESC(debug, "Turn on/off debugging (default:off).");
diff --git a/drivers/media/usb/pvrusb2/Kconfig b/drivers/media/usb/pvrusb2/Kconfig
index f2b64e49c5a2..9501b10b31aa 100644
--- a/drivers/media/usb/pvrusb2/Kconfig
+++ b/drivers/media/usb/pvrusb2/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config VIDEO_PVRUSB2
 	tristate "Hauppauge WinTV-PVR USB2 support"
-	depends on VIDEO_DEV && I2C
+	depends on VIDEO_DEV && I2C && DVB_CORE
 	select VIDEO_TUNER
 	select VIDEO_TVEEPROM
 	select VIDEO_CX2341X
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-main.c b/drivers/media/usb/pvrusb2/pvrusb2-main.c
index ce4d566e4e5a..721dafd2c14b 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-main.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-main.c
@@ -16,9 +16,7 @@
 #include "pvrusb2-context.h"
 #include "pvrusb2-debug.h"
 #include "pvrusb2-v4l2.h"
-#ifdef CONFIG_VIDEO_PVRUSB2_SYSFS
 #include "pvrusb2-sysfs.h"
-#endif /* CONFIG_VIDEO_PVRUSB2_SYSFS */
 
 #define DRIVER_AUTHOR "Mike Isely <isely@pobox.com>"
 #define DRIVER_DESC "Hauppauge WinTV-PVR-USB2 MPEG2 Encoder/Tuner"
@@ -36,10 +34,6 @@ int pvrusb2_debug = DEFAULT_DEBUG_MASK;
 module_param_named(debug,pvrusb2_debug,int,S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(debug, "Debug trace mask");
 
-#ifdef CONFIG_VIDEO_PVRUSB2_SYSFS
-static struct pvr2_sysfs_class *class_ptr = NULL;
-#endif /* CONFIG_VIDEO_PVRUSB2_SYSFS */
-
 static void pvr_setup_attach(struct pvr2_context *pvr)
 {
 	/* Create association with v4l layer */
@@ -48,9 +42,7 @@ static void pvr_setup_attach(struct pvr2_context *pvr)
 	/* Create association with dvb layer */
 	pvr2_dvb_create(pvr);
 #endif
-#ifdef CONFIG_VIDEO_PVRUSB2_SYSFS
-	pvr2_sysfs_create(pvr,class_ptr);
-#endif /* CONFIG_VIDEO_PVRUSB2_SYSFS */
+	pvr2_sysfs_create(pvr);
 }
 
 static int pvr_probe(struct usb_interface *intf,
@@ -115,9 +107,7 @@ static int __init pvr_init(void)
 		return ret;
 	}
 
-#ifdef CONFIG_VIDEO_PVRUSB2_SYSFS
-	class_ptr = pvr2_sysfs_class_create();
-#endif /* CONFIG_VIDEO_PVRUSB2_SYSFS */
+	pvr2_sysfs_class_create();
 
 	ret = usb_register(&pvr_driver);
 
@@ -141,9 +131,7 @@ static void __exit pvr_exit(void)
 
 	pvr2_context_global_done();
 
-#ifdef CONFIG_VIDEO_PVRUSB2_SYSFS
-	pvr2_sysfs_class_destroy(class_ptr);
-#endif /* CONFIG_VIDEO_PVRUSB2_SYSFS */
+	pvr2_sysfs_class_destroy();
 
 	pvr2_trace(PVR2_TRACE_INIT,"pvr_exit complete");
 }
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-sysfs.c b/drivers/media/usb/pvrusb2/pvrusb2-sysfs.c
index 3e42e209be37..a8c0b513e58e 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-sysfs.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-sysfs.c
@@ -66,10 +66,6 @@ struct pvr2_sysfs_ctl_item {
 	char name[80];
 };
 
-struct pvr2_sysfs_class {
-	struct class class;
-};
-
 static ssize_t show_name(struct device *class_dev,
 			 struct device_attribute *attr,
 			 char *buf)
@@ -487,15 +483,6 @@ static void pvr2_sysfs_tear_down_controls(struct pvr2_sysfs *sfp)
 }
 
 
-static void pvr2_sysfs_class_release(struct class *class)
-{
-	struct pvr2_sysfs_class *clp;
-	clp = container_of(class,struct pvr2_sysfs_class,class);
-	pvr2_sysfs_trace("Destroying pvr2_sysfs_class id=%p",clp);
-	kfree(clp);
-}
-
-
 static void pvr2_sysfs_release(struct device *class_dev)
 {
 	pvr2_sysfs_trace("Releasing class_dev id=%p",class_dev);
@@ -503,6 +490,12 @@ static void pvr2_sysfs_release(struct device *class_dev)
 }
 
 
+static struct class pvr2_class = {
+	.name		= "pvrusb2",
+	.dev_release	= pvr2_sysfs_release,
+};
+
+
 static void class_dev_destroy(struct pvr2_sysfs *sfp)
 {
 	struct device *dev;
@@ -614,8 +607,7 @@ static ssize_t unit_number_show(struct device *class_dev,
 }
 
 
-static void class_dev_create(struct pvr2_sysfs *sfp,
-			     struct pvr2_sysfs_class *class_ptr)
+static void class_dev_create(struct pvr2_sysfs *sfp)
 {
 	struct usb_device *usb_dev;
 	struct device *class_dev;
@@ -628,7 +620,7 @@ static void class_dev_create(struct pvr2_sysfs *sfp,
 
 	pvr2_sysfs_trace("Creating class_dev id=%p",class_dev);
 
-	class_dev->class = &class_ptr->class;
+	class_dev->class = &pvr2_class;
 
 	dev_set_name(class_dev, "%s",
 		     pvr2_hdw_get_device_identifier(sfp->channel.hdw));
@@ -753,47 +745,30 @@ static void pvr2_sysfs_internal_check(struct pvr2_channel *chp)
 }
 
 
-struct pvr2_sysfs *pvr2_sysfs_create(struct pvr2_context *mp,
-				     struct pvr2_sysfs_class *class_ptr)
+void pvr2_sysfs_create(struct pvr2_context *mp)
 {
 	struct pvr2_sysfs *sfp;
 	sfp = kzalloc(sizeof(*sfp),GFP_KERNEL);
-	if (!sfp) return sfp;
+	if (!sfp)
+		return;
 	pvr2_trace(PVR2_TRACE_STRUCT,"Creating pvr2_sysfs id=%p",sfp);
 	pvr2_channel_init(&sfp->channel,mp);
 	sfp->channel.check_func = pvr2_sysfs_internal_check;
 
-	class_dev_create(sfp,class_ptr);
-	return sfp;
+	class_dev_create(sfp);
 }
 
 
-
-struct pvr2_sysfs_class *pvr2_sysfs_class_create(void)
+void pvr2_sysfs_class_create(void)
 {
-	struct pvr2_sysfs_class *clp;
-	clp = kzalloc(sizeof(*clp),GFP_KERNEL);
-	if (!clp) return clp;
-	pvr2_sysfs_trace("Creating and registering pvr2_sysfs_class id=%p",
-			 clp);
-	clp->class.name = "pvrusb2";
-	clp->class.class_release = pvr2_sysfs_class_release;
-	clp->class.dev_release = pvr2_sysfs_release;
-	if (class_register(&clp->class)) {
-		pvr2_sysfs_trace(
-			"Registration failed for pvr2_sysfs_class id=%p",clp);
-		kfree(clp);
-		clp = NULL;
-	}
-	return clp;
+	if (class_register(&pvr2_class))
+		pvr2_sysfs_trace("Registration failed for pvr2_sysfs_class");
 }
 
 
-void pvr2_sysfs_class_destroy(struct pvr2_sysfs_class *clp)
+void pvr2_sysfs_class_destroy(void)
 {
-	pvr2_sysfs_trace("Unregistering pvr2_sysfs_class id=%p", clp);
-	if (clp)
-		class_unregister(&clp->class);
+	class_unregister(&pvr2_class);
 }
 
 
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-sysfs.h b/drivers/media/usb/pvrusb2/pvrusb2-sysfs.h
index ac580ff39b5f..375a5372e95c 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-sysfs.h
+++ b/drivers/media/usb/pvrusb2/pvrusb2-sysfs.h
@@ -10,13 +10,15 @@
 #include <linux/sysfs.h>
 #include "pvrusb2-context.h"
 
-struct pvr2_sysfs;
-struct pvr2_sysfs_class;
+#ifdef CONFIG_VIDEO_PVRUSB2_SYSFS
+void pvr2_sysfs_class_create(void);
+void pvr2_sysfs_class_destroy(void);
+void pvr2_sysfs_create(struct pvr2_context *mp);
+#else
+static inline void pvr2_sysfs_class_create(void) { }
+static inline void pvr2_sysfs_class_destroy(void) { }
+static inline void pvr2_sysfs_create(struct pvr2_context *mp) { }
+#endif
 
-struct pvr2_sysfs_class *pvr2_sysfs_class_create(void);
-void pvr2_sysfs_class_destroy(struct pvr2_sysfs_class *);
-
-struct pvr2_sysfs *pvr2_sysfs_create(struct pvr2_context *,
-				     struct pvr2_sysfs_class *);
 
 #endif /* __PVRUSB2_SYSFS_H */
diff --git a/drivers/media/usb/usbtv/usbtv-core.c b/drivers/media/usb/usbtv/usbtv-core.c
index 2308c0b4f5e7..1f7620cd2996 100644
--- a/drivers/media/usb/usbtv/usbtv-core.c
+++ b/drivers/media/usb/usbtv/usbtv-core.c
@@ -47,7 +47,7 @@
 int usbtv_set_regs(struct usbtv *usbtv, const u16 regs[][2], int size)
 {
 	int ret;
-	int pipe = usb_rcvctrlpipe(usbtv->udev, 0);
+	int pipe = usb_sndctrlpipe(usbtv->udev, 0);
 	int i;
 
 	for (i = 0; i < size; i++) {
diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c
index d7e9ffc7aa23..b16b5f4cb91e 100644
--- a/drivers/media/v4l2-core/v4l2-async.c
+++ b/drivers/media/v4l2-core/v4l2-async.c
@@ -416,7 +416,8 @@ static void v4l2_async_cleanup(struct v4l2_subdev *sd)
 
 /* Unbind all sub-devices in the notifier tree. */
 static void
-v4l2_async_nf_unbind_all_subdevs(struct v4l2_async_notifier *notifier)
+v4l2_async_nf_unbind_all_subdevs(struct v4l2_async_notifier *notifier,
+				 bool readd)
 {
 	struct v4l2_subdev *sd, *tmp;
 
@@ -425,9 +426,11 @@ v4l2_async_nf_unbind_all_subdevs(struct v4l2_async_notifier *notifier)
 			v4l2_async_find_subdev_notifier(sd);
 
 		if (subdev_notifier)
-			v4l2_async_nf_unbind_all_subdevs(subdev_notifier);
+			v4l2_async_nf_unbind_all_subdevs(subdev_notifier, true);
 
 		v4l2_async_nf_call_unbind(notifier, sd, sd->asd);
+		if (readd)
+			list_add_tail(&sd->asd->list, &notifier->waiting);
 		v4l2_async_cleanup(sd);
 
 		list_move(&sd->async_list, &subdev_list);
@@ -559,7 +562,7 @@ err_unbind:
 	/*
 	 * On failure, unbind all sub-devices registered through this notifier.
 	 */
-	v4l2_async_nf_unbind_all_subdevs(notifier);
+	v4l2_async_nf_unbind_all_subdevs(notifier, false);
 
 err_unlock:
 	mutex_unlock(&list_lock);
@@ -609,7 +612,7 @@ __v4l2_async_nf_unregister(struct v4l2_async_notifier *notifier)
 	if (!notifier || (!notifier->v4l2_dev && !notifier->sd))
 		return;
 
-	v4l2_async_nf_unbind_all_subdevs(notifier);
+	v4l2_async_nf_unbind_all_subdevs(notifier, false);
 
 	notifier->sd = NULL;
 	notifier->v4l2_dev = NULL;
@@ -807,7 +810,7 @@ err_unbind:
 	 */
 	subdev_notifier = v4l2_async_find_subdev_notifier(sd);
 	if (subdev_notifier)
-		v4l2_async_nf_unbind_all_subdevs(subdev_notifier);
+		v4l2_async_nf_unbind_all_subdevs(subdev_notifier, false);
 
 	if (sd->asd)
 		v4l2_async_nf_call_unbind(notifier, sd, sd->asd);
diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c
index 40f56e044640..3c5ab5ecd678 100644
--- a/drivers/media/v4l2-core/v4l2-common.c
+++ b/drivers/media/v4l2-core/v4l2-common.c
@@ -252,12 +252,16 @@ const struct v4l2_format_info *v4l2_format_info(u32 format)
 		{ .format = V4L2_PIX_FMT_RGB565,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_RGB555,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_BGR666,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_BGR48_12, .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 6, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_ABGR64_12, .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 8, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
 
 		/* YUV packed formats */
 		{ .format = V4L2_PIX_FMT_YUYV,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 2, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_YVYU,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 2, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_UYVY,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 2, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_VYUY,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_Y212,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_YUV48_12, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 6, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
 
 		/* YUV planar formats */
 		{ .format = V4L2_PIX_FMT_NV12,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 2 },
@@ -267,6 +271,7 @@ const struct v4l2_format_info *v4l2_format_info(u32 format)
 		{ .format = V4L2_PIX_FMT_NV24,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 1, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_NV42,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 1, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_P010,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 2, 2, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_P012,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 2, 4, 0, 0 }, .hdiv = 2, .vdiv = 2 },
 
 		{ .format = V4L2_PIX_FMT_YUV410,  .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 4, .vdiv = 4 },
 		{ .format = V4L2_PIX_FMT_YVU410,  .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 4, .vdiv = 4 },
@@ -292,6 +297,7 @@ const struct v4l2_format_info *v4l2_format_info(u32 format)
 		{ .format = V4L2_PIX_FMT_NV21M,   .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 2 },
 		{ .format = V4L2_PIX_FMT_NV16M,   .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_NV61M,   .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_P012M,   .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 2, 4, 0, 0 }, .hdiv = 2, .vdiv = 2 },
 
 		/* Bayer RGB formats */
 		{ .format = V4L2_PIX_FMT_SBGGR8,	.pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
diff --git a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
index 55c26e7d370e..f3bed37859a2 100644
--- a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
+++ b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
@@ -38,18 +38,13 @@
  * data to the routine.
  */
 
-struct v4l2_clip32 {
-	struct v4l2_rect        c;
-	compat_caddr_t		next;
-};
-
 struct v4l2_window32 {
 	struct v4l2_rect        w;
 	__u32			field;	/* enum v4l2_field */
 	__u32			chromakey;
-	compat_caddr_t		clips; /* actually struct v4l2_clip32 * */
-	__u32			clipcount;
-	compat_caddr_t		bitmap;
+	compat_caddr_t		clips; /* always NULL */
+	__u32			clipcount; /* always 0 */
+	compat_caddr_t		bitmap; /* always NULL */
 	__u8                    global_alpha;
 };
 
@@ -65,17 +60,12 @@ static int get_v4l2_window32(struct v4l2_window *p64,
 		.w		= w32.w,
 		.field		= w32.field,
 		.chromakey	= w32.chromakey,
-		.clips		= (void __force *)compat_ptr(w32.clips),
-		.clipcount	= w32.clipcount,
-		.bitmap		= compat_ptr(w32.bitmap),
+		.clips		= NULL,
+		.clipcount	= 0,
+		.bitmap		= NULL,
 		.global_alpha	= w32.global_alpha,
 	};
 
-	if (p64->clipcount > 2048)
-		return -EINVAL;
-	if (!p64->clipcount)
-		p64->clips = NULL;
-
 	return 0;
 }
 
@@ -89,16 +79,13 @@ static int put_v4l2_window32(struct v4l2_window *p64,
 		.w		= p64->w,
 		.field		= p64->field,
 		.chromakey	= p64->chromakey,
-		.clips		= (uintptr_t)p64->clips,
-		.clipcount	= p64->clipcount,
-		.bitmap		= ptr_to_compat(p64->bitmap),
+		.clips		= 0,
+		.clipcount	= 0,
+		.bitmap		= 0,
 		.global_alpha	= p64->global_alpha,
 	};
 
-	/* copy everything except the clips pointer */
-	if (copy_to_user(p32, &w32, offsetof(struct v4l2_window32, clips)) ||
-	    copy_to_user(&p32->clipcount, &w32.clipcount,
-			 sizeof(w32) - offsetof(struct v4l2_window32, clipcount)))
+	if (copy_to_user(p32, &w32, sizeof(w32)))
 		return -EFAULT;
 
 	return 0;
@@ -600,14 +587,11 @@ struct v4l2_framebuffer32 {
 static int get_v4l2_framebuffer32(struct v4l2_framebuffer *p64,
 				  struct v4l2_framebuffer32 __user *p32)
 {
-	compat_caddr_t tmp;
-
-	if (get_user(tmp, &p32->base) ||
-	    get_user(p64->capability, &p32->capability) ||
+	if (get_user(p64->capability, &p32->capability) ||
 	    get_user(p64->flags, &p32->flags) ||
 	    copy_from_user(&p64->fmt, &p32->fmt, sizeof(p64->fmt)))
 		return -EFAULT;
-	p64->base = (void __force *)compat_ptr(tmp);
+	p64->base = NULL;
 
 	return 0;
 }
@@ -1043,29 +1027,6 @@ int v4l2_compat_get_array_args(struct file *file, void *mbuf,
 	memset(mbuf, 0, array_size);
 
 	switch (cmd) {
-	case VIDIOC_G_FMT32:
-	case VIDIOC_S_FMT32:
-	case VIDIOC_TRY_FMT32: {
-		struct v4l2_format *f64 = arg;
-		struct v4l2_clip *c64 = mbuf;
-		struct v4l2_clip32 __user *c32 = user_ptr;
-		u32 clipcount = f64->fmt.win.clipcount;
-
-		if ((f64->type != V4L2_BUF_TYPE_VIDEO_OVERLAY &&
-		     f64->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY) ||
-		    clipcount == 0)
-			return 0;
-		if (clipcount > 2048)
-			return -EINVAL;
-		while (clipcount--) {
-			if (copy_from_user(c64, c32, sizeof(c64->c)))
-				return -EFAULT;
-			c64->next = NULL;
-			c64++;
-			c32++;
-		}
-		break;
-	}
 #ifdef CONFIG_COMPAT_32BIT_TIME
 	case VIDIOC_QUERYBUF32_TIME32:
 	case VIDIOC_QBUF32_TIME32:
@@ -1136,28 +1097,6 @@ int v4l2_compat_put_array_args(struct file *file, void __user *user_ptr,
 	int err = 0;
 
 	switch (cmd) {
-	case VIDIOC_G_FMT32:
-	case VIDIOC_S_FMT32:
-	case VIDIOC_TRY_FMT32: {
-		struct v4l2_format *f64 = arg;
-		struct v4l2_clip *c64 = mbuf;
-		struct v4l2_clip32 __user *c32 = user_ptr;
-		u32 clipcount = f64->fmt.win.clipcount;
-
-		if ((f64->type != V4L2_BUF_TYPE_VIDEO_OVERLAY &&
-		     f64->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY) ||
-		    clipcount == 0)
-			return 0;
-		if (clipcount > 2048)
-			return -EINVAL;
-		while (clipcount--) {
-			if (copy_to_user(c32, c64, sizeof(c64->c)))
-				return -EFAULT;
-			c64++;
-			c32++;
-		}
-		break;
-	}
 #ifdef CONFIG_COMPAT_32BIT_TIME
 	case VIDIOC_QUERYBUF32_TIME32:
 	case VIDIOC_QBUF32_TIME32:
diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c
index 397d553177fa..f81279492682 100644
--- a/drivers/media/v4l2-core/v4l2-dev.c
+++ b/drivers/media/v4l2-core/v4l2-dev.c
@@ -556,6 +556,7 @@ static void determine_valid_ioctls(struct video_device *vdev)
 	bool is_rx = vdev->vfl_dir != VFL_DIR_TX;
 	bool is_tx = vdev->vfl_dir != VFL_DIR_RX;
 	bool is_io_mc = vdev->device_caps & V4L2_CAP_IO_MC;
+	bool has_streaming = vdev->device_caps & V4L2_CAP_STREAMING;
 
 	bitmap_zero(valid_ioctls, BASE_VIDIOC_PRIVATE);
 
@@ -708,8 +709,8 @@ static void determine_valid_ioctls(struct video_device *vdev)
 		SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_sdr_out);
 	}
 
-	if (is_vid || is_vbi || is_sdr || is_tch || is_meta) {
-		/* ioctls valid for video, vbi, sdr, touch and metadata */
+	if (has_streaming) {
+		/* ioctls valid for streaming I/O */
 		SET_VALID_IOCTL(ops, VIDIOC_REQBUFS, vidioc_reqbufs);
 		SET_VALID_IOCTL(ops, VIDIOC_QUERYBUF, vidioc_querybuf);
 		SET_VALID_IOCTL(ops, VIDIOC_QBUF, vidioc_qbuf);
diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c
index 87f163a89c80..a858acea6547 100644
--- a/drivers/media/v4l2-core/v4l2-ioctl.c
+++ b/drivers/media/v4l2-core/v4l2-ioctl.c
@@ -310,14 +310,10 @@ static void v4l_print_format(const void *arg, bool write_only)
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY:
 		win = &p->fmt.win;
-		/* Note: we can't print the clip list here since the clips
-		 * pointer is a userspace pointer, not a kernelspace
-		 * pointer. */
-		pr_cont(", wxh=%dx%d, x,y=%d,%d, field=%s, chromakey=0x%08x, clipcount=%u, clips=%p, bitmap=%p, global_alpha=0x%02x\n",
+		pr_cont(", wxh=%dx%d, x,y=%d,%d, field=%s, chromakey=0x%08x, global_alpha=0x%02x\n",
 			win->w.width, win->w.height, win->w.left, win->w.top,
 			prt_names(win->field, v4l2_field_names),
-			win->chromakey, win->clipcount, win->clips,
-			win->bitmap, win->global_alpha);
+			win->chromakey, win->global_alpha);
 		break;
 	case V4L2_BUF_TYPE_VBI_CAPTURE:
 	case V4L2_BUF_TYPE_VBI_OUTPUT:
@@ -1302,11 +1298,14 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_PIX_FMT_RGBX1010102:	descr = "32-bit RGBX 10-10-10-2"; break;
 	case V4L2_PIX_FMT_RGBA1010102:	descr = "32-bit RGBA 10-10-10-2"; break;
 	case V4L2_PIX_FMT_ARGB2101010:	descr = "32-bit ARGB 2-10-10-10"; break;
+	case V4L2_PIX_FMT_BGR48_12:	descr = "12-bit Depth BGR"; break;
+	case V4L2_PIX_FMT_ABGR64_12:	descr = "12-bit Depth BGRA"; break;
 	case V4L2_PIX_FMT_GREY:		descr = "8-bit Greyscale"; break;
 	case V4L2_PIX_FMT_Y4:		descr = "4-bit Greyscale"; break;
 	case V4L2_PIX_FMT_Y6:		descr = "6-bit Greyscale"; break;
 	case V4L2_PIX_FMT_Y10:		descr = "10-bit Greyscale"; break;
 	case V4L2_PIX_FMT_Y12:		descr = "12-bit Greyscale"; break;
+	case V4L2_PIX_FMT_Y012:		descr = "12-bit Greyscale (bits 15-4)"; break;
 	case V4L2_PIX_FMT_Y14:		descr = "14-bit Greyscale"; break;
 	case V4L2_PIX_FMT_Y16:		descr = "16-bit Greyscale"; break;
 	case V4L2_PIX_FMT_Y16_BE:	descr = "16-bit Greyscale BE"; break;
@@ -1345,6 +1344,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_PIX_FMT_YUV420:	descr = "Planar YUV 4:2:0"; break;
 	case V4L2_PIX_FMT_HI240:	descr = "8-bit Dithered RGB (BTTV)"; break;
 	case V4L2_PIX_FMT_M420:		descr = "YUV 4:2:0 (M420)"; break;
+	case V4L2_PIX_FMT_YUV48_12:	descr = "12-bit YUV 4:4:4 Packed"; break;
 	case V4L2_PIX_FMT_NV12:		descr = "Y/UV 4:2:0"; break;
 	case V4L2_PIX_FMT_NV21:		descr = "Y/VU 4:2:0"; break;
 	case V4L2_PIX_FMT_NV16:		descr = "Y/UV 4:2:2"; break;
@@ -1352,6 +1352,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_PIX_FMT_NV24:		descr = "Y/UV 4:4:4"; break;
 	case V4L2_PIX_FMT_NV42:		descr = "Y/VU 4:4:4"; break;
 	case V4L2_PIX_FMT_P010:		descr = "10-bit Y/UV 4:2:0"; break;
+	case V4L2_PIX_FMT_P012:		descr = "12-bit Y/UV 4:2:0"; break;
 	case V4L2_PIX_FMT_NV12_4L4:	descr = "Y/UV 4:2:0 (4x4 Linear)"; break;
 	case V4L2_PIX_FMT_NV12_16L16:	descr = "Y/UV 4:2:0 (16x16 Linear)"; break;
 	case V4L2_PIX_FMT_NV12_32L32:   descr = "Y/UV 4:2:0 (32x32 Linear)"; break;
@@ -1362,6 +1363,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_PIX_FMT_NV61M:	descr = "Y/VU 4:2:2 (N-C)"; break;
 	case V4L2_PIX_FMT_NV12MT:	descr = "Y/UV 4:2:0 (64x32 MB, N-C)"; break;
 	case V4L2_PIX_FMT_NV12MT_16X16:	descr = "Y/UV 4:2:0 (16x16 MB, N-C)"; break;
+	case V4L2_PIX_FMT_P012M:	descr = "12-bit Y/UV 4:2:0 (N-C)"; break;
 	case V4L2_PIX_FMT_YUV420M:	descr = "Planar YUV 4:2:0 (N-C)"; break;
 	case V4L2_PIX_FMT_YVU420M:	descr = "Planar YVU 4:2:0 (N-C)"; break;
 	case V4L2_PIX_FMT_YUV422M:	descr = "Planar YUV 4:2:2 (N-C)"; break;
@@ -1479,6 +1481,9 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 		case V4L2_PIX_FMT_HEVC_SLICE:	descr = "HEVC Parsed Slice Data"; break;
 		case V4L2_PIX_FMT_FWHT:		descr = "FWHT"; break; /* used in vicodec */
 		case V4L2_PIX_FMT_FWHT_STATELESS:	descr = "FWHT Stateless"; break; /* used in vicodec */
+		case V4L2_PIX_FMT_SPK:		descr = "Sorenson Spark"; break;
+		case V4L2_PIX_FMT_RV30:		descr = "RealVideo 8"; break;
+		case V4L2_PIX_FMT_RV40:		descr = "RealVideo 9 & 10"; break;
 		case V4L2_PIX_FMT_CPIA1:	descr = "GSPCA CPiA YUV"; break;
 		case V4L2_PIX_FMT_WNVA:		descr = "WNVA"; break;
 		case V4L2_PIX_FMT_SN9C10X:	descr = "GSPCA SN9C10X"; break;
@@ -1618,29 +1623,7 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops,
 	if (ret)
 		return ret;
 
-	/*
-	 * fmt can't be cleared for these overlay types due to the 'clips'
-	 * 'clipcount' and 'bitmap' pointers in struct v4l2_window.
-	 * Those are provided by the user. So handle these two overlay types
-	 * first, and then just do a simple memset for the other types.
-	 */
-	switch (p->type) {
-	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
-	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY: {
-		struct v4l2_clip *clips = p->fmt.win.clips;
-		u32 clipcount = p->fmt.win.clipcount;
-		void __user *bitmap = p->fmt.win.bitmap;
-
-		memset(&p->fmt, 0, sizeof(p->fmt));
-		p->fmt.win.clips = clips;
-		p->fmt.win.clipcount = clipcount;
-		p->fmt.win.bitmap = bitmap;
-		break;
-	}
-	default:
-		memset(&p->fmt, 0, sizeof(p->fmt));
-		break;
-	}
+	memset(&p->fmt, 0, sizeof(p->fmt));
 
 	switch (p->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
@@ -1728,6 +1711,9 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops,
 		if (unlikely(!ops->vidioc_s_fmt_vid_overlay))
 			break;
 		memset_after(p, 0, fmt.win);
+		p->fmt.win.clips = NULL;
+		p->fmt.win.clipcount = 0;
+		p->fmt.win.bitmap = NULL;
 		return ops->vidioc_s_fmt_vid_overlay(file, fh, arg);
 	case V4L2_BUF_TYPE_VBI_CAPTURE:
 		if (unlikely(!ops->vidioc_s_fmt_vbi_cap))
@@ -1759,6 +1745,9 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops,
 		if (unlikely(!ops->vidioc_s_fmt_vid_out_overlay))
 			break;
 		memset_after(p, 0, fmt.win);
+		p->fmt.win.clips = NULL;
+		p->fmt.win.clipcount = 0;
+		p->fmt.win.bitmap = NULL;
 		return ops->vidioc_s_fmt_vid_out_overlay(file, fh, arg);
 	case V4L2_BUF_TYPE_VBI_OUTPUT:
 		if (unlikely(!ops->vidioc_s_fmt_vbi_out))
@@ -1830,6 +1819,9 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops,
 		if (unlikely(!ops->vidioc_try_fmt_vid_overlay))
 			break;
 		memset_after(p, 0, fmt.win);
+		p->fmt.win.clips = NULL;
+		p->fmt.win.clipcount = 0;
+		p->fmt.win.bitmap = NULL;
 		return ops->vidioc_try_fmt_vid_overlay(file, fh, arg);
 	case V4L2_BUF_TYPE_VBI_CAPTURE:
 		if (unlikely(!ops->vidioc_try_fmt_vbi_cap))
@@ -1861,6 +1853,9 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops,
 		if (unlikely(!ops->vidioc_try_fmt_vid_out_overlay))
 			break;
 		memset_after(p, 0, fmt.win);
+		p->fmt.win.clips = NULL;
+		p->fmt.win.clipcount = 0;
+		p->fmt.win.bitmap = NULL;
 		return ops->vidioc_try_fmt_vid_out_overlay(file, fh, arg);
 	case V4L2_BUF_TYPE_VBI_OUTPUT:
 		if (unlikely(!ops->vidioc_try_fmt_vbi_out))
@@ -2074,6 +2069,15 @@ static int v4l_s_hw_freq_seek(const struct v4l2_ioctl_ops *ops,
 	return ops->vidioc_s_hw_freq_seek(file, fh, p);
 }
 
+static int v4l_s_fbuf(const struct v4l2_ioctl_ops *ops,
+		      struct file *file, void *fh, void *arg)
+{
+	struct v4l2_framebuffer *p = arg;
+
+	p->base = NULL;
+	return ops->vidioc_s_fbuf(file, fh, p);
+}
+
 static int v4l_overlay(const struct v4l2_ioctl_ops *ops,
 				struct file *file, void *fh, void *arg)
 {
@@ -2793,7 +2797,6 @@ struct v4l2_ioctl_info {
 	}
 
 DEFINE_V4L_STUB_FUNC(g_fbuf)
-DEFINE_V4L_STUB_FUNC(s_fbuf)
 DEFINE_V4L_STUB_FUNC(expbuf)
 DEFINE_V4L_STUB_FUNC(g_std)
 DEFINE_V4L_STUB_FUNC(g_audio)
@@ -2827,7 +2830,7 @@ static const struct v4l2_ioctl_info v4l2_ioctls[] = {
 	IOCTL_INFO(VIDIOC_REQBUFS, v4l_reqbufs, v4l_print_requestbuffers, INFO_FL_PRIO | INFO_FL_QUEUE),
 	IOCTL_INFO(VIDIOC_QUERYBUF, v4l_querybuf, v4l_print_buffer, INFO_FL_QUEUE | INFO_FL_CLEAR(v4l2_buffer, length)),
 	IOCTL_INFO(VIDIOC_G_FBUF, v4l_stub_g_fbuf, v4l_print_framebuffer, 0),
-	IOCTL_INFO(VIDIOC_S_FBUF, v4l_stub_s_fbuf, v4l_print_framebuffer, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_S_FBUF, v4l_s_fbuf, v4l_print_framebuffer, INFO_FL_PRIO),
 	IOCTL_INFO(VIDIOC_OVERLAY, v4l_overlay, v4l_print_u32, INFO_FL_PRIO),
 	IOCTL_INFO(VIDIOC_QBUF, v4l_qbuf, v4l_print_buffer, INFO_FL_QUEUE),
 	IOCTL_INFO(VIDIOC_EXPBUF, v4l_stub_expbuf, v4l_print_exportbuffer, INFO_FL_QUEUE | INFO_FL_CLEAR(v4l2_exportbuffer, flags)),
@@ -3134,27 +3137,6 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size,
 		}
 		break;
 	}
-	case VIDIOC_G_FMT:
-	case VIDIOC_S_FMT:
-	case VIDIOC_TRY_FMT: {
-		struct v4l2_format *fmt = parg;
-
-		if (fmt->type != V4L2_BUF_TYPE_VIDEO_OVERLAY &&
-		    fmt->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY)
-			break;
-		if (fmt->fmt.win.clipcount > 2048)
-			return -EINVAL;
-		if (!fmt->fmt.win.clipcount)
-			break;
-
-		*user_ptr = (void __user *)fmt->fmt.win.clips;
-		*kernel_ptr = (void **)&fmt->fmt.win.clips;
-		*array_size = sizeof(struct v4l2_clip)
-				* fmt->fmt.win.clipcount;
-
-		ret = 1;
-		break;
-	}
 
 	case VIDIOC_SUBDEV_G_ROUTING:
 	case VIDIOC_SUBDEV_S_ROUTING: {
diff --git a/drivers/media/v4l2-core/v4l2-mc.c b/drivers/media/v4l2-core/v4l2-mc.c
index b01474717dca..bf0c18100664 100644
--- a/drivers/media/v4l2-core/v4l2-mc.c
+++ b/drivers/media/v4l2-core/v4l2-mc.c
@@ -313,14 +313,11 @@ int v4l2_create_fwnode_links_to_pad(struct v4l2_subdev *src_sd,
 				    struct media_pad *sink, u32 flags)
 {
 	struct fwnode_handle *endpoint;
-	struct v4l2_subdev *sink_sd;
 
 	if (!(sink->flags & MEDIA_PAD_FL_SINK) ||
 	    !is_media_entity_v4l2_subdev(sink->entity))
 		return -EINVAL;
 
-	sink_sd = media_entity_to_v4l2_subdev(sink->entity);
-
 	fwnode_graph_for_each_endpoint(dev_fwnode(src_sd->dev), endpoint) {
 		struct fwnode_handle *remote_ep;
 		int src_idx, sink_idx, ret;
@@ -340,7 +337,7 @@ int v4l2_create_fwnode_links_to_pad(struct v4l2_subdev *src_sd,
 		 * ask the sink to verify it owns the remote endpoint,
 		 * and translate to a sink pad.
 		 */
-		sink_idx = media_entity_get_fwnode_pad(&sink_sd->entity,
+		sink_idx = media_entity_get_fwnode_pad(sink->entity,
 						       remote_ep,
 						       MEDIA_PAD_FL_SINK);
 		fwnode_handle_put(remote_ep);
@@ -362,17 +359,17 @@ int v4l2_create_fwnode_links_to_pad(struct v4l2_subdev *src_sd,
 		if (media_entity_find_link(src, sink))
 			continue;
 
-		dev_dbg(sink_sd->dev, "creating link %s:%d -> %s:%d\n",
+		dev_dbg(src_sd->dev, "creating link %s:%d -> %s:%d\n",
 			src_sd->entity.name, src_idx,
-			sink_sd->entity.name, sink_idx);
+			sink->entity->name, sink_idx);
 
 		ret = media_create_pad_link(&src_sd->entity, src_idx,
-					    &sink_sd->entity, sink_idx, flags);
+					    sink->entity, sink_idx, flags);
 		if (ret) {
-			dev_err(sink_sd->dev,
+			dev_err(src_sd->dev,
 				"link %s:%d -> %s:%d failed with %d\n",
 				src_sd->entity.name, src_idx,
-				sink_sd->entity.name, sink_idx, ret);
+				sink->entity->name, sink_idx, ret);
 
 			fwnode_handle_put(endpoint);
 			return ret;
diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c
index b10045c02f43..2ec179cd1264 100644
--- a/drivers/media/v4l2-core/v4l2-subdev.c
+++ b/drivers/media/v4l2-core/v4l2-subdev.c
@@ -510,8 +510,11 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 	struct video_device *vdev = video_devdata(file);
 	struct v4l2_subdev *sd = vdev_to_v4l2_subdev(vdev);
 	struct v4l2_fh *vfh = file->private_data;
+	struct v4l2_subdev_fh *subdev_fh = to_v4l2_subdev_fh(vfh);
 	bool ro_subdev = test_bit(V4L2_FL_SUBDEV_RO_DEVNODE, &vdev->flags);
 	bool streams_subdev = sd->flags & V4L2_SUBDEV_FL_STREAMS;
+	bool client_supports_streams = subdev_fh->client_caps &
+				       V4L2_SUBDEV_CLIENT_CAP_STREAMS;
 	int rval;
 
 	switch (cmd) {
@@ -636,6 +639,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 	case VIDIOC_SUBDEV_G_FMT: {
 		struct v4l2_subdev_format *format = arg;
 
+		if (!client_supports_streams)
+			format->stream = 0;
+
 		memset(format->reserved, 0, sizeof(format->reserved));
 		memset(format->format.reserved, 0, sizeof(format->format.reserved));
 		return v4l2_subdev_call(sd, pad, get_fmt, state, format);
@@ -647,6 +653,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		if (format->which != V4L2_SUBDEV_FORMAT_TRY && ro_subdev)
 			return -EPERM;
 
+		if (!client_supports_streams)
+			format->stream = 0;
+
 		memset(format->reserved, 0, sizeof(format->reserved));
 		memset(format->format.reserved, 0, sizeof(format->format.reserved));
 		return v4l2_subdev_call(sd, pad, set_fmt, state, format);
@@ -656,6 +665,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		struct v4l2_subdev_crop *crop = arg;
 		struct v4l2_subdev_selection sel;
 
+		if (!client_supports_streams)
+			crop->stream = 0;
+
 		memset(crop->reserved, 0, sizeof(crop->reserved));
 		memset(&sel, 0, sizeof(sel));
 		sel.which = crop->which;
@@ -677,6 +689,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		if (crop->which != V4L2_SUBDEV_FORMAT_TRY && ro_subdev)
 			return -EPERM;
 
+		if (!client_supports_streams)
+			crop->stream = 0;
+
 		memset(crop->reserved, 0, sizeof(crop->reserved));
 		memset(&sel, 0, sizeof(sel));
 		sel.which = crop->which;
@@ -695,6 +710,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 	case VIDIOC_SUBDEV_ENUM_MBUS_CODE: {
 		struct v4l2_subdev_mbus_code_enum *code = arg;
 
+		if (!client_supports_streams)
+			code->stream = 0;
+
 		memset(code->reserved, 0, sizeof(code->reserved));
 		return v4l2_subdev_call(sd, pad, enum_mbus_code, state,
 					code);
@@ -703,6 +721,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 	case VIDIOC_SUBDEV_ENUM_FRAME_SIZE: {
 		struct v4l2_subdev_frame_size_enum *fse = arg;
 
+		if (!client_supports_streams)
+			fse->stream = 0;
+
 		memset(fse->reserved, 0, sizeof(fse->reserved));
 		return v4l2_subdev_call(sd, pad, enum_frame_size, state,
 					fse);
@@ -711,6 +732,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 	case VIDIOC_SUBDEV_G_FRAME_INTERVAL: {
 		struct v4l2_subdev_frame_interval *fi = arg;
 
+		if (!client_supports_streams)
+			fi->stream = 0;
+
 		memset(fi->reserved, 0, sizeof(fi->reserved));
 		return v4l2_subdev_call(sd, video, g_frame_interval, arg);
 	}
@@ -721,6 +745,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		if (ro_subdev)
 			return -EPERM;
 
+		if (!client_supports_streams)
+			fi->stream = 0;
+
 		memset(fi->reserved, 0, sizeof(fi->reserved));
 		return v4l2_subdev_call(sd, video, s_frame_interval, arg);
 	}
@@ -728,6 +755,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 	case VIDIOC_SUBDEV_ENUM_FRAME_INTERVAL: {
 		struct v4l2_subdev_frame_interval_enum *fie = arg;
 
+		if (!client_supports_streams)
+			fie->stream = 0;
+
 		memset(fie->reserved, 0, sizeof(fie->reserved));
 		return v4l2_subdev_call(sd, pad, enum_frame_interval, state,
 					fie);
@@ -736,6 +766,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 	case VIDIOC_SUBDEV_G_SELECTION: {
 		struct v4l2_subdev_selection *sel = arg;
 
+		if (!client_supports_streams)
+			sel->stream = 0;
+
 		memset(sel->reserved, 0, sizeof(sel->reserved));
 		return v4l2_subdev_call(
 			sd, pad, get_selection, state, sel);
@@ -747,6 +780,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		if (sel->which != V4L2_SUBDEV_FORMAT_TRY && ro_subdev)
 			return -EPERM;
 
+		if (!client_supports_streams)
+			sel->stream = 0;
+
 		memset(sel->reserved, 0, sizeof(sel->reserved));
 		return v4l2_subdev_call(
 			sd, pad, set_selection, state, sel);
@@ -888,6 +924,33 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 					routing->which, &krouting);
 	}
 
+	case VIDIOC_SUBDEV_G_CLIENT_CAP: {
+		struct v4l2_subdev_client_capability *client_cap = arg;
+
+		client_cap->capabilities = subdev_fh->client_caps;
+
+		return 0;
+	}
+
+	case VIDIOC_SUBDEV_S_CLIENT_CAP: {
+		struct v4l2_subdev_client_capability *client_cap = arg;
+
+		/*
+		 * Clear V4L2_SUBDEV_CLIENT_CAP_STREAMS if streams API is not
+		 * enabled. Remove this when streams API is no longer
+		 * experimental.
+		 */
+		if (!v4l2_subdev_enable_streams_api)
+			client_cap->capabilities &= ~V4L2_SUBDEV_CLIENT_CAP_STREAMS;
+
+		/* Filter out unsupported capabilities */
+		client_cap->capabilities &= V4L2_SUBDEV_CLIENT_CAP_STREAMS;
+
+		subdev_fh->client_caps = client_cap->capabilities;
+
+		return 0;
+	}
+
 	default:
 		return v4l2_subdev_call(sd, core, ioctl, cmd, arg);
 	}
@@ -1069,32 +1132,45 @@ EXPORT_SYMBOL_GPL(v4l2_subdev_link_validate_default);
 
 static int
 v4l2_subdev_link_validate_get_format(struct media_pad *pad, u32 stream,
-				     struct v4l2_subdev_format *fmt)
+				     struct v4l2_subdev_format *fmt,
+				     bool states_locked)
 {
-	if (is_media_entity_v4l2_subdev(pad->entity)) {
-		struct v4l2_subdev *sd =
-			media_entity_to_v4l2_subdev(pad->entity);
+	struct v4l2_subdev_state *state;
+	struct v4l2_subdev *sd;
+	int ret;
 
-		fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
-		fmt->pad = pad->index;
-		fmt->stream = stream;
+	if (!is_media_entity_v4l2_subdev(pad->entity)) {
+		WARN(pad->entity->function != MEDIA_ENT_F_IO_V4L,
+		     "Driver bug! Wrong media entity type 0x%08x, entity %s\n",
+		     pad->entity->function, pad->entity->name);
 
-		return v4l2_subdev_call(sd, pad, get_fmt,
-					v4l2_subdev_get_locked_active_state(sd),
-					fmt);
+		return -EINVAL;
 	}
 
-	WARN(pad->entity->function != MEDIA_ENT_F_IO_V4L,
-	     "Driver bug! Wrong media entity type 0x%08x, entity %s\n",
-	     pad->entity->function, pad->entity->name);
+	sd = media_entity_to_v4l2_subdev(pad->entity);
 
-	return -EINVAL;
+	fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	fmt->pad = pad->index;
+	fmt->stream = stream;
+
+	if (states_locked)
+		state = v4l2_subdev_get_locked_active_state(sd);
+	else
+		state = v4l2_subdev_lock_and_get_active_state(sd);
+
+	ret = v4l2_subdev_call(sd, pad, get_fmt, state, fmt);
+
+	if (!states_locked && state)
+		v4l2_subdev_unlock_state(state);
+
+	return ret;
 }
 
 #if defined(CONFIG_VIDEO_V4L2_SUBDEV_API)
 
 static void __v4l2_link_validate_get_streams(struct media_pad *pad,
-					     u64 *streams_mask)
+					     u64 *streams_mask,
+					     bool states_locked)
 {
 	struct v4l2_subdev_route *route;
 	struct v4l2_subdev_state *state;
@@ -1104,7 +1180,11 @@ static void __v4l2_link_validate_get_streams(struct media_pad *pad,
 
 	*streams_mask = 0;
 
-	state = v4l2_subdev_get_locked_active_state(subdev);
+	if (states_locked)
+		state = v4l2_subdev_get_locked_active_state(subdev);
+	else
+		state = v4l2_subdev_lock_and_get_active_state(subdev);
+
 	if (WARN_ON(!state))
 		return;
 
@@ -1125,12 +1205,16 @@ static void __v4l2_link_validate_get_streams(struct media_pad *pad,
 
 		*streams_mask |= BIT_ULL(route_stream);
 	}
+
+	if (!states_locked)
+		v4l2_subdev_unlock_state(state);
 }
 
 #endif /* CONFIG_VIDEO_V4L2_SUBDEV_API */
 
 static void v4l2_link_validate_get_streams(struct media_pad *pad,
-					   u64 *streams_mask)
+					   u64 *streams_mask,
+					   bool states_locked)
 {
 	struct v4l2_subdev *subdev = media_entity_to_v4l2_subdev(pad->entity);
 
@@ -1141,14 +1225,14 @@ static void v4l2_link_validate_get_streams(struct media_pad *pad,
 	}
 
 #if defined(CONFIG_VIDEO_V4L2_SUBDEV_API)
-	__v4l2_link_validate_get_streams(pad, streams_mask);
+	__v4l2_link_validate_get_streams(pad, streams_mask, states_locked);
 #else
 	/* This shouldn't happen */
 	*streams_mask = 0;
 #endif
 }
 
-static int v4l2_subdev_link_validate_locked(struct media_link *link)
+static int v4l2_subdev_link_validate_locked(struct media_link *link, bool states_locked)
 {
 	struct v4l2_subdev *sink_subdev =
 		media_entity_to_v4l2_subdev(link->sink->entity);
@@ -1163,8 +1247,8 @@ static int v4l2_subdev_link_validate_locked(struct media_link *link)
 		link->source->entity->name, link->source->index,
 		link->sink->entity->name, link->sink->index);
 
-	v4l2_link_validate_get_streams(link->source, &source_streams_mask);
-	v4l2_link_validate_get_streams(link->sink, &sink_streams_mask);
+	v4l2_link_validate_get_streams(link->source, &source_streams_mask, states_locked);
+	v4l2_link_validate_get_streams(link->sink, &sink_streams_mask, states_locked);
 
 	/*
 	 * It is ok to have more source streams than sink streams as extra
@@ -1192,7 +1276,7 @@ static int v4l2_subdev_link_validate_locked(struct media_link *link)
 			link->sink->entity->name, link->sink->index, stream);
 
 		ret = v4l2_subdev_link_validate_get_format(link->source, stream,
-							   &source_fmt);
+							   &source_fmt, states_locked);
 		if (ret < 0) {
 			dev_dbg(dev,
 				"Failed to get format for \"%s\":%u:%u (but that's ok)\n",
@@ -1202,7 +1286,7 @@ static int v4l2_subdev_link_validate_locked(struct media_link *link)
 		}
 
 		ret = v4l2_subdev_link_validate_get_format(link->sink, stream,
-							   &sink_fmt);
+							   &sink_fmt, states_locked);
 		if (ret < 0) {
 			dev_dbg(dev,
 				"Failed to get format for \"%s\":%u:%u (but that's ok)\n",
@@ -1234,27 +1318,38 @@ int v4l2_subdev_link_validate(struct media_link *link)
 {
 	struct v4l2_subdev *source_sd, *sink_sd;
 	struct v4l2_subdev_state *source_state, *sink_state;
+	bool states_locked;
 	int ret;
 
+	if (!is_media_entity_v4l2_subdev(link->sink->entity) ||
+	    !is_media_entity_v4l2_subdev(link->source->entity)) {
+		pr_warn_once("%s of link '%s':%u->'%s':%u is not a V4L2 sub-device, driver bug!\n",
+			     !is_media_entity_v4l2_subdev(link->sink->entity) ?
+			     "sink" : "source",
+			     link->source->entity->name, link->source->index,
+			     link->sink->entity->name, link->sink->index);
+		return 0;
+	}
+
 	sink_sd = media_entity_to_v4l2_subdev(link->sink->entity);
 	source_sd = media_entity_to_v4l2_subdev(link->source->entity);
 
 	sink_state = v4l2_subdev_get_unlocked_active_state(sink_sd);
 	source_state = v4l2_subdev_get_unlocked_active_state(source_sd);
 
-	if (sink_state)
-		v4l2_subdev_lock_state(sink_state);
+	states_locked = sink_state && source_state;
 
-	if (source_state)
+	if (states_locked) {
+		v4l2_subdev_lock_state(sink_state);
 		v4l2_subdev_lock_state(source_state);
+	}
 
-	ret = v4l2_subdev_link_validate_locked(link);
+	ret = v4l2_subdev_link_validate_locked(link, states_locked);
 
-	if (sink_state)
+	if (states_locked) {
 		v4l2_subdev_unlock_state(sink_state);
-
-	if (source_state)
 		v4l2_subdev_unlock_state(source_state);
+	}
 
 	return ret;
 }
@@ -1676,7 +1771,8 @@ int v4l2_subdev_routing_validate(struct v4l2_subdev *sd,
 	unsigned int i, j;
 	int ret = -EINVAL;
 
-	if (disallow & V4L2_SUBDEV_ROUTING_NO_STREAM_MIX) {
+	if (disallow & (V4L2_SUBDEV_ROUTING_NO_STREAM_MIX |
+			V4L2_SUBDEV_ROUTING_NO_MULTIPLEXING)) {
 		remote_pads = kcalloc(sd->entity.num_pads, sizeof(*remote_pads),
 				      GFP_KERNEL);
 		if (!remote_pads)
@@ -1705,10 +1801,10 @@ int v4l2_subdev_routing_validate(struct v4l2_subdev *sd,
 		}
 
 		/*
-		 * V4L2_SUBDEV_ROUTING_NO_STREAM_MIX: Streams on the same pad
-		 * may not be routed to streams on different pads.
+		 * V4L2_SUBDEV_ROUTING_NO_SINK_STREAM_MIX: all streams from a
+		 * sink pad must be routed to a single source pad.
 		 */
-		if (disallow & V4L2_SUBDEV_ROUTING_NO_STREAM_MIX) {
+		if (disallow & V4L2_SUBDEV_ROUTING_NO_SINK_STREAM_MIX) {
 			if (remote_pads[route->sink_pad] != U32_MAX &&
 			    remote_pads[route->sink_pad] != route->source_pad) {
 				dev_dbg(sd->dev,
@@ -1716,7 +1812,13 @@ int v4l2_subdev_routing_validate(struct v4l2_subdev *sd,
 					i, "sink");
 				goto out;
 			}
+		}
 
+		/*
+		 * V4L2_SUBDEV_ROUTING_NO_SOURCE_STREAM_MIX: all streams on a
+		 * source pad must originate from a single sink pad.
+		 */
+		if (disallow & V4L2_SUBDEV_ROUTING_NO_SOURCE_STREAM_MIX) {
 			if (remote_pads[route->source_pad] != U32_MAX &&
 			    remote_pads[route->source_pad] != route->sink_pad) {
 				dev_dbg(sd->dev,
@@ -1724,7 +1826,37 @@ int v4l2_subdev_routing_validate(struct v4l2_subdev *sd,
 					i, "source");
 				goto out;
 			}
+		}
+
+		/*
+		 * V4L2_SUBDEV_ROUTING_NO_SINK_MULTIPLEXING: Pads on the sink
+		 * side can not do stream multiplexing, i.e. there can be only
+		 * a single stream in a sink pad.
+		 */
+		if (disallow & V4L2_SUBDEV_ROUTING_NO_SINK_MULTIPLEXING) {
+			if (remote_pads[route->sink_pad] != U32_MAX) {
+				dev_dbg(sd->dev,
+					"route %u attempts to multiplex on %s pad %u\n",
+					i, "sink", route->sink_pad);
+				goto out;
+			}
+		}
+
+		/*
+		 * V4L2_SUBDEV_ROUTING_NO_SOURCE_MULTIPLEXING: Pads on the
+		 * source side can not do stream multiplexing, i.e. there can
+		 * be only a single stream in a source pad.
+		 */
+		if (disallow & V4L2_SUBDEV_ROUTING_NO_SOURCE_MULTIPLEXING) {
+			if (remote_pads[route->source_pad] != U32_MAX) {
+				dev_dbg(sd->dev,
+					"route %u attempts to multiplex on %s pad %u\n",
+					i, "source", route->source_pad);
+				goto out;
+			}
+		}
 
+		if (remote_pads) {
 			remote_pads[route->sink_pad] = route->source_pad;
 			remote_pads[route->source_pad] = route->sink_pad;
 		}
diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig
index fac290e48e0b..91774e6ee624 100644
--- a/drivers/memory/Kconfig
+++ b/drivers/memory/Kconfig
@@ -228,7 +228,7 @@ config RENESAS_RPCIF
 
 config STM32_FMC2_EBI
 	tristate "Support for FMC2 External Bus Interface on STM32MP SoCs"
-	depends on MACH_STM32MP157 || COMPILE_TEST
+	depends on ARCH_STM32 || COMPILE_TEST
 	select MFD_SYSCON
 	help
 	  Select this option to enable the STM32 FMC2 External Bus Interface
diff --git a/drivers/memory/atmel-ebi.c b/drivers/memory/atmel-ebi.c
index e749dcb3ddea..635966d705cb 100644
--- a/drivers/memory/atmel-ebi.c
+++ b/drivers/memory/atmel-ebi.c
@@ -598,7 +598,7 @@ static int atmel_ebi_probe(struct platform_device *pdev)
 	reg_cells += val;
 
 	for_each_available_child_of_node(np, child) {
-		if (!of_find_property(child, "reg", NULL))
+		if (!of_property_present(child, "reg"))
 			continue;
 
 		ret = atmel_ebi_dev_setup(ebi, child, reg_cells);
diff --git a/drivers/memory/bt1-l2-ctl.c b/drivers/memory/bt1-l2-ctl.c
index 85965fa26e0b..78bd71b203f2 100644
--- a/drivers/memory/bt1-l2-ctl.c
+++ b/drivers/memory/bt1-l2-ctl.c
@@ -321,4 +321,3 @@ module_platform_driver(l2_ctl_driver);
 
 MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
 MODULE_DESCRIPTION("Baikal-T1 L2-cache driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/memory/da8xx-ddrctl.c b/drivers/memory/da8xx-ddrctl.c
index b32005bf269c..0ef8cc878b95 100644
--- a/drivers/memory/da8xx-ddrctl.c
+++ b/drivers/memory/da8xx-ddrctl.c
@@ -164,4 +164,3 @@ module_platform_driver(da8xx_ddrctl_driver);
 
 MODULE_AUTHOR("Bartosz Golaszewski <bgolaszewski@baylibre.com>");
 MODULE_DESCRIPTION("TI da8xx DDR2/mDDR controller driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/memory/fsl_ifc.c b/drivers/memory/fsl_ifc.c
index e83b61c925a4..9e8d8e9c5ad8 100644
--- a/drivers/memory/fsl_ifc.c
+++ b/drivers/memory/fsl_ifc.c
@@ -327,6 +327,5 @@ static int __init fsl_ifc_init(void)
 }
 subsys_initcall(fsl_ifc_init);
 
-MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Freescale Semiconductor");
 MODULE_DESCRIPTION("Freescale Integrated Flash Controller driver");
diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c
index 5a9754442bc7..6523cb510518 100644
--- a/drivers/memory/mtk-smi.c
+++ b/drivers/memory/mtk-smi.c
@@ -713,6 +713,11 @@ static const struct mtk_smi_common_plat mtk_smi_sub_common_mt8195 = {
 	.has_gals = true,
 };
 
+static const struct mtk_smi_common_plat mtk_smi_common_mt8365 = {
+	.type     = MTK_SMI_GEN2,
+	.bus_sel  = F_MMU1_LARB(2) | F_MMU1_LARB(4),
+};
+
 static const struct of_device_id mtk_smi_common_of_ids[] = {
 	{.compatible = "mediatek,mt2701-smi-common", .data = &mtk_smi_common_gen1},
 	{.compatible = "mediatek,mt2712-smi-common", .data = &mtk_smi_common_gen2},
@@ -728,6 +733,7 @@ static const struct of_device_id mtk_smi_common_of_ids[] = {
 	{.compatible = "mediatek,mt8195-smi-common-vdo", .data = &mtk_smi_common_mt8195_vdo},
 	{.compatible = "mediatek,mt8195-smi-common-vpp", .data = &mtk_smi_common_mt8195_vpp},
 	{.compatible = "mediatek,mt8195-smi-sub-common", .data = &mtk_smi_sub_common_mt8195},
+	{.compatible = "mediatek,mt8365-smi-common", .data = &mtk_smi_common_mt8365},
 	{}
 };
 
diff --git a/drivers/memory/mvebu-devbus.c b/drivers/memory/mvebu-devbus.c
index efc6c08db2b7..406fddcdba02 100644
--- a/drivers/memory/mvebu-devbus.c
+++ b/drivers/memory/mvebu-devbus.c
@@ -341,6 +341,5 @@ static int __init mvebu_devbus_init(void)
 }
 module_init(mvebu_devbus_init);
 
-MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Ezequiel Garcia <ezequiel.garcia@free-electrons.com>");
 MODULE_DESCRIPTION("Marvell EBU SoC Device Bus controller");
diff --git a/drivers/memory/tegra/mc.c b/drivers/memory/tegra/mc.c
index 5cd28619ea9f..9082b6c3763d 100644
--- a/drivers/memory/tegra/mc.c
+++ b/drivers/memory/tegra/mc.c
@@ -983,4 +983,3 @@ arch_initcall(tegra_mc_init);
 
 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/memory/tegra/tegra186-emc.c b/drivers/memory/tegra/tegra186-emc.c
index 26e763bde92a..e935ad4e95b6 100644
--- a/drivers/memory/tegra/tegra186-emc.c
+++ b/drivers/memory/tegra/tegra186-emc.c
@@ -280,4 +280,3 @@ module_platform_driver(tegra186_emc_driver);
 
 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
 MODULE_DESCRIPTION("NVIDIA Tegra186 External Memory Controller driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/memory/tegra/tegra210-emc-cc-r21021.c b/drivers/memory/tegra/tegra210-emc-cc-r21021.c
index cc76adb8d7e8..4cb608c71ead 100644
--- a/drivers/memory/tegra/tegra210-emc-cc-r21021.c
+++ b/drivers/memory/tegra/tegra210-emc-cc-r21021.c
@@ -277,7 +277,7 @@ static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type)
 		/*
 		 * Dev1 LSB.
 		 */
-		value = tegra210_emc_mrr_read(emc, 2, 18);
+		value = tegra210_emc_mrr_read(emc, 1, 18);
 
 		for (i = 0; i < emc->num_channels; i++) {
 			temp[i][0] |= (value & 0x00ff) >> 0;
diff --git a/drivers/memory/tegra/tegra210-emc-table.c b/drivers/memory/tegra/tegra210-emc-table.c
index 3e0598363b87..34a8785d2861 100644
--- a/drivers/memory/tegra/tegra210-emc-table.c
+++ b/drivers/memory/tegra/tegra210-emc-table.c
@@ -22,8 +22,6 @@ static int tegra210_emc_table_device_init(struct reserved_mem *rmem,
 		return -ENOMEM;
 	}
 
-	count = 0;
-
 	for (i = 0; i < TEGRA_EMC_MAX_FREQS; i++) {
 		if (timings[i].revision == 0)
 			break;
diff --git a/drivers/memstick/core/memstick.c b/drivers/memstick/core/memstick.c
index bf7667845459..bbfaf6536903 100644
--- a/drivers/memstick/core/memstick.c
+++ b/drivers/memstick/core/memstick.c
@@ -410,6 +410,7 @@ static struct memstick_dev *memstick_alloc_card(struct memstick_host *host)
 	return card;
 err_out:
 	host->card = old_card;
+	kfree_const(card->dev.kobj.name);
 	kfree(card);
 	return NULL;
 }
@@ -468,8 +469,10 @@ static void memstick_check(struct work_struct *work)
 				put_device(&card->dev);
 				host->card = NULL;
 			}
-		} else
+		} else {
+			kfree_const(card->dev.kobj.name);
 			kfree(card);
+		}
 	}
 
 out_power_off:
diff --git a/drivers/memstick/host/r592.c b/drivers/memstick/host/r592.c
index 1d35d147552d..42bfc46842b8 100644
--- a/drivers/memstick/host/r592.c
+++ b/drivers/memstick/host/r592.c
@@ -829,7 +829,7 @@ static void r592_remove(struct pci_dev *pdev)
 	/* Stop the processing thread.
 	That ensures that we won't take any more requests */
 	kthread_stop(dev->io_thread);
-
+	del_timer_sync(&dev->detect_timer);
 	r592_enable_device(dev, false);
 
 	while (!error && dev->req) {
diff --git a/drivers/message/fusion/mptbase.h b/drivers/message/fusion/mptbase.h
index 4bd0682c65d3..e0ba783de368 100644
--- a/drivers/message/fusion/mptbase.h
+++ b/drivers/message/fusion/mptbase.h
@@ -757,7 +757,6 @@ typedef struct _MPT_ADAPTER
 	u8			 wait_on_reset_completion;
 	MPT_SCHEDULE_TARGET_RESET schedule_target_reset;
 	MPT_FLUSH_RUNNING_CMDS schedule_dead_ioc_flush_running_cmds;
-	struct work_struct	 sas_persist_task;
 
 	struct work_struct	 fc_setup_reset_work;
 	struct list_head	 fc_rports;
diff --git a/drivers/message/fusion/mptfc.c b/drivers/message/fusion/mptfc.c
index fac747109209..22e7779a332b 100644
--- a/drivers/message/fusion/mptfc.c
+++ b/drivers/message/fusion/mptfc.c
@@ -105,7 +105,7 @@ static int mptfc_abort(struct scsi_cmnd *SCpnt);
 static int mptfc_dev_reset(struct scsi_cmnd *SCpnt);
 static int mptfc_bus_reset(struct scsi_cmnd *SCpnt);
 
-static struct scsi_host_template mptfc_driver_template = {
+static const struct scsi_host_template mptfc_driver_template = {
 	.module				= THIS_MODULE,
 	.proc_name			= "mptfc",
 	.show_info			= mptscsih_show_info,
diff --git a/drivers/message/fusion/mptlan.c b/drivers/message/fusion/mptlan.c
index 142eb5d5d9df..de2e7bcf4784 100644
--- a/drivers/message/fusion/mptlan.c
+++ b/drivers/message/fusion/mptlan.c
@@ -1433,7 +1433,9 @@ mptlan_remove(struct pci_dev *pdev)
 {
 	MPT_ADAPTER 		*ioc = pci_get_drvdata(pdev);
 	struct net_device	*dev = ioc->netdev;
+	struct mpt_lan_priv *priv = netdev_priv(dev);
 
+	cancel_delayed_work_sync(&priv->post_buckets_task);
 	if(dev != NULL) {
 		unregister_netdev(dev);
 		free_netdev(dev);
diff --git a/drivers/message/fusion/mptsas.c b/drivers/message/fusion/mptsas.c
index 88fe4a860ae5..86f16f3ea478 100644
--- a/drivers/message/fusion/mptsas.c
+++ b/drivers/message/fusion/mptsas.c
@@ -1997,7 +1997,7 @@ done:
 }
 
 
-static struct scsi_host_template mptsas_driver_template = {
+static const struct scsi_host_template mptsas_driver_template = {
 	.module				= THIS_MODULE,
 	.proc_name			= "mptsas",
 	.show_info			= mptscsih_show_info,
diff --git a/drivers/message/fusion/mptscsih.c b/drivers/message/fusion/mptscsih.c
index 276084ed04a6..2bc17087d17d 100644
--- a/drivers/message/fusion/mptscsih.c
+++ b/drivers/message/fusion/mptscsih.c
@@ -1231,7 +1231,6 @@ mptscsih_suspend(struct pci_dev *pdev, pm_message_t state)
 	MPT_ADAPTER 		*ioc = pci_get_drvdata(pdev);
 
 	scsi_block_requests(ioc->sh);
-	flush_scheduled_work();
 	mptscsih_shutdown(pdev);
 	return mpt_suspend(pdev,state);
 }
diff --git a/drivers/message/fusion/mptspi.c b/drivers/message/fusion/mptspi.c
index 62089a8caa2f..6c5920db1e9d 100644
--- a/drivers/message/fusion/mptspi.c
+++ b/drivers/message/fusion/mptspi.c
@@ -820,7 +820,7 @@ static void mptspi_slave_destroy(struct scsi_device *sdev)
 	mptscsih_slave_destroy(sdev);
 }
 
-static struct scsi_host_template mptspi_driver_template = {
+static const struct scsi_host_template mptspi_driver_template = {
 	.module				= THIS_MODULE,
 	.proc_name			= "mptspi",
 	.show_info			= mptscsih_show_info,
diff --git a/drivers/mfd/altera-sysmgr.c b/drivers/mfd/altera-sysmgr.c
index 5d3715a28b28..af205813b281 100644
--- a/drivers/mfd/altera-sysmgr.c
+++ b/drivers/mfd/altera-sysmgr.c
@@ -198,4 +198,3 @@ module_exit(altr_sysmgr_exit);
 
 MODULE_AUTHOR("Thor Thayer <>");
 MODULE_DESCRIPTION("SOCFPGA System Manager driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/cros_ec_dev.c b/drivers/mfd/cros_ec_dev.c
index 02d4271dfe06..92f4dfccc3cc 100644
--- a/drivers/mfd/cros_ec_dev.c
+++ b/drivers/mfd/cros_ec_dev.c
@@ -20,7 +20,6 @@
 #define DRV_NAME "cros-ec-dev"
 
 static struct class cros_class = {
-	.owner          = THIS_MODULE,
 	.name           = "chromeos",
 };
 
diff --git a/drivers/mfd/ocelot-core.c b/drivers/mfd/ocelot-core.c
index e1772ff00cad..9cccf54fc9c8 100644
--- a/drivers/mfd/ocelot-core.c
+++ b/drivers/mfd/ocelot-core.c
@@ -45,6 +45,9 @@
 #define VSC7512_SIO_CTRL_RES_START	0x710700f8
 #define VSC7512_SIO_CTRL_RES_SIZE	0x00000100
 
+#define VSC7512_HSIO_RES_START		0x710d0000
+#define VSC7512_HSIO_RES_SIZE		0x00000128
+
 #define VSC7512_ANA_RES_START		0x71880000
 #define VSC7512_ANA_RES_SIZE		0x00010000
 
@@ -129,8 +132,13 @@ static const struct resource vsc7512_sgpio_resources[] = {
 	DEFINE_RES_REG_NAMED(VSC7512_SIO_CTRL_RES_START, VSC7512_SIO_CTRL_RES_SIZE, "gcb_sio"),
 };
 
+static const struct resource vsc7512_serdes_resources[] = {
+	DEFINE_RES_REG_NAMED(VSC7512_HSIO_RES_START, VSC7512_HSIO_RES_SIZE, "hsio"),
+};
+
 static const struct resource vsc7512_switch_resources[] = {
 	DEFINE_RES_REG_NAMED(VSC7512_ANA_RES_START, VSC7512_ANA_RES_SIZE, "ana"),
+	DEFINE_RES_REG_NAMED(VSC7512_HSIO_RES_START, VSC7512_HSIO_RES_SIZE, "hsio"),
 	DEFINE_RES_REG_NAMED(VSC7512_QS_RES_START, VSC7512_QS_RES_SIZE, "qs"),
 	DEFINE_RES_REG_NAMED(VSC7512_QSYS_RES_START, VSC7512_QSYS_RES_SIZE, "qsys"),
 	DEFINE_RES_REG_NAMED(VSC7512_REW_RES_START, VSC7512_REW_RES_SIZE, "rew"),
@@ -177,6 +185,11 @@ static const struct mfd_cell vsc7512_devs[] = {
 		.num_resources = ARRAY_SIZE(vsc7512_miim1_resources),
 		.resources = vsc7512_miim1_resources,
 	}, {
+		.name = "ocelot-serdes",
+		.of_compatible = "mscc,vsc7514-serdes",
+		.num_resources = ARRAY_SIZE(vsc7512_serdes_resources),
+		.resources = vsc7512_serdes_resources,
+	}, {
 		.name = "ocelot-ext-switch",
 		.of_compatible = "mscc,vsc7512-switch",
 		.num_resources = ARRAY_SIZE(vsc7512_switch_resources),
diff --git a/drivers/mfd/ocelot-spi.c b/drivers/mfd/ocelot-spi.c
index 2ecd271de2fb..2d1349a10ca9 100644
--- a/drivers/mfd/ocelot-spi.c
+++ b/drivers/mfd/ocelot-spi.c
@@ -125,7 +125,7 @@ static int ocelot_spi_initialize(struct device *dev)
 static const struct regmap_config ocelot_spi_regmap_config = {
 	.reg_bits = 24,
 	.reg_stride = 4,
-	.reg_downshift = 2,
+	.reg_shift = REGMAP_DOWNSHIFT(2),
 	.val_bits = 32,
 
 	.write_flag_mask = 0x80,
diff --git a/drivers/misc/c2port/core.c b/drivers/misc/c2port/core.c
index fb9a1b49ff6d..f574c83b82cf 100644
--- a/drivers/misc/c2port/core.c
+++ b/drivers/misc/c2port/core.c
@@ -977,7 +977,7 @@ static int __init c2port_init(void)
 	printk(KERN_INFO "Silicon Labs C2 port support v. " DRIVER_VERSION
 		" - (C) 2007 Rodolfo Giometti\n");
 
-	c2port_class = class_create(THIS_MODULE, "c2port");
+	c2port_class = class_create("c2port");
 	if (IS_ERR(c2port_class)) {
 		printk(KERN_ERR "c2port: failed to allocate class\n");
 		return PTR_ERR(c2port_class);
diff --git a/drivers/misc/cardreader/alcor_pci.c b/drivers/misc/cardreader/alcor_pci.c
index 9080f9f150a2..0142c4bf4f42 100644
--- a/drivers/misc/cardreader/alcor_pci.c
+++ b/drivers/misc/cardreader/alcor_pci.c
@@ -95,160 +95,6 @@ u32 alcor_read32be(struct alcor_pci_priv *priv, unsigned int addr)
 }
 EXPORT_SYMBOL_GPL(alcor_read32be);
 
-static int alcor_pci_find_cap_offset(struct alcor_pci_priv *priv,
-				     struct pci_dev *pci)
-{
-	int where;
-	u8 val8;
-	u32 val32;
-
-	where = ALCOR_CAP_START_OFFSET;
-	pci_read_config_byte(pci, where, &val8);
-	if (!val8)
-		return 0;
-
-	where = (int)val8;
-	while (1) {
-		pci_read_config_dword(pci, where, &val32);
-		if (val32 == 0xffffffff) {
-			dev_dbg(priv->dev, "find_cap_offset invalid value %x.\n",
-				val32);
-			return 0;
-		}
-
-		if ((val32 & 0xff) == 0x10) {
-			dev_dbg(priv->dev, "pcie cap offset: %x\n", where);
-			return where;
-		}
-
-		if ((val32 & 0xff00) == 0x00) {
-			dev_dbg(priv->dev, "pci_find_cap_offset invalid value %x.\n",
-				val32);
-			break;
-		}
-		where = (int)((val32 >> 8) & 0xff);
-	}
-
-	return 0;
-}
-
-static void alcor_pci_init_check_aspm(struct alcor_pci_priv *priv)
-{
-	struct pci_dev *pci;
-	int where;
-	u32 val32;
-
-	priv->pdev_cap_off    = alcor_pci_find_cap_offset(priv, priv->pdev);
-	/*
-	 * A device might be attached to root complex directly and
-	 * priv->parent_pdev will be NULL. In this case we don't check its
-	 * capability and disable ASPM completely.
-	 */
-	if (priv->parent_pdev)
-		priv->parent_cap_off = alcor_pci_find_cap_offset(priv,
-							 priv->parent_pdev);
-
-	if ((priv->pdev_cap_off == 0) || (priv->parent_cap_off == 0)) {
-		dev_dbg(priv->dev, "pci_cap_off: %x, parent_cap_off: %x\n",
-			priv->pdev_cap_off, priv->parent_cap_off);
-		return;
-	}
-
-	/* link capability */
-	pci   = priv->pdev;
-	where = priv->pdev_cap_off + ALCOR_PCIE_LINK_CAP_OFFSET;
-	pci_read_config_dword(pci, where, &val32);
-	priv->pdev_aspm_cap = (u8)(val32 >> 10) & 0x03;
-
-	pci   = priv->parent_pdev;
-	where = priv->parent_cap_off + ALCOR_PCIE_LINK_CAP_OFFSET;
-	pci_read_config_dword(pci, where, &val32);
-	priv->parent_aspm_cap = (u8)(val32 >> 10) & 0x03;
-
-	if (priv->pdev_aspm_cap != priv->parent_aspm_cap) {
-		u8 aspm_cap;
-
-		dev_dbg(priv->dev, "pdev_aspm_cap: %x, parent_aspm_cap: %x\n",
-			priv->pdev_aspm_cap, priv->parent_aspm_cap);
-		aspm_cap = priv->pdev_aspm_cap & priv->parent_aspm_cap;
-		priv->pdev_aspm_cap    = aspm_cap;
-		priv->parent_aspm_cap = aspm_cap;
-	}
-
-	dev_dbg(priv->dev, "ext_config_dev_aspm: %x, pdev_aspm_cap: %x\n",
-		priv->ext_config_dev_aspm, priv->pdev_aspm_cap);
-	priv->ext_config_dev_aspm &= priv->pdev_aspm_cap;
-}
-
-static void alcor_pci_aspm_ctrl(struct alcor_pci_priv *priv, u8 aspm_enable)
-{
-	struct pci_dev *pci;
-	u8 aspm_ctrl, i;
-	int where;
-	u32 val32;
-
-	if ((!priv->pdev_cap_off) || (!priv->parent_cap_off)) {
-		dev_dbg(priv->dev, "pci_cap_off: %x, parent_cap_off: %x\n",
-			priv->pdev_cap_off, priv->parent_cap_off);
-		return;
-	}
-
-	if (!priv->pdev_aspm_cap)
-		return;
-
-	aspm_ctrl = 0;
-	if (aspm_enable) {
-		aspm_ctrl = priv->ext_config_dev_aspm;
-
-		if (!aspm_ctrl) {
-			dev_dbg(priv->dev, "aspm_ctrl == 0\n");
-			return;
-		}
-	}
-
-	for (i = 0; i < 2; i++) {
-
-		if (i) {
-			pci   = priv->parent_pdev;
-			where = priv->parent_cap_off
-				+ ALCOR_PCIE_LINK_CTRL_OFFSET;
-		} else {
-			pci   = priv->pdev;
-			where = priv->pdev_cap_off
-				+ ALCOR_PCIE_LINK_CTRL_OFFSET;
-		}
-
-		pci_read_config_dword(pci, where, &val32);
-		val32 &= (~0x03);
-		val32 |= (aspm_ctrl & priv->pdev_aspm_cap);
-		pci_write_config_byte(pci, where, (u8)val32);
-	}
-
-}
-
-static inline void alcor_mask_sd_irqs(struct alcor_pci_priv *priv)
-{
-	alcor_write32(priv, 0, AU6601_REG_INT_ENABLE);
-}
-
-static inline void alcor_unmask_sd_irqs(struct alcor_pci_priv *priv)
-{
-	alcor_write32(priv, AU6601_INT_CMD_MASK | AU6601_INT_DATA_MASK |
-		  AU6601_INT_CARD_INSERT | AU6601_INT_CARD_REMOVE |
-		  AU6601_INT_OVER_CURRENT_ERR,
-		  AU6601_REG_INT_ENABLE);
-}
-
-static inline void alcor_mask_ms_irqs(struct alcor_pci_priv *priv)
-{
-	alcor_write32(priv, 0, AU6601_MS_INT_ENABLE);
-}
-
-static inline void alcor_unmask_ms_irqs(struct alcor_pci_priv *priv)
-{
-	alcor_write32(priv, 0x3d00fa, AU6601_MS_INT_ENABLE);
-}
-
 static int alcor_pci_probe(struct pci_dev *pdev,
 			   const struct pci_device_id *ent)
 {
@@ -308,7 +154,6 @@ static int alcor_pci_probe(struct pci_dev *pdev,
 
 	pci_set_master(pdev);
 	pci_set_drvdata(pdev, priv);
-	alcor_pci_init_check_aspm(priv);
 
 	for (i = 0; i < ARRAY_SIZE(alcor_pci_cells); i++) {
 		alcor_pci_cells[i].platform_data = priv;
@@ -319,7 +164,7 @@ static int alcor_pci_probe(struct pci_dev *pdev,
 	if (ret < 0)
 		goto error_clear_drvdata;
 
-	alcor_pci_aspm_ctrl(priv, 0);
+	pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
 
 	return 0;
 
@@ -339,8 +184,6 @@ static void alcor_pci_remove(struct pci_dev *pdev)
 
 	priv = pci_get_drvdata(pdev);
 
-	alcor_pci_aspm_ctrl(priv, 1);
-
 	mfd_remove_devices(&pdev->dev);
 
 	ida_free(&alcor_pci_idr, priv->id);
@@ -353,18 +196,16 @@ static void alcor_pci_remove(struct pci_dev *pdev)
 #ifdef CONFIG_PM_SLEEP
 static int alcor_suspend(struct device *dev)
 {
-	struct alcor_pci_priv *priv = dev_get_drvdata(dev);
-
-	alcor_pci_aspm_ctrl(priv, 1);
 	return 0;
 }
 
 static int alcor_resume(struct device *dev)
 {
-
 	struct alcor_pci_priv *priv = dev_get_drvdata(dev);
 
-	alcor_pci_aspm_ctrl(priv, 0);
+	pci_disable_link_state(priv->pdev,
+			       PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
+
 	return 0;
 }
 #endif /* CONFIG_PM_SLEEP */
diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c
index 5878329b011a..144d1f2d78ce 100644
--- a/drivers/misc/cxl/file.c
+++ b/drivers/misc/cxl/file.c
@@ -678,7 +678,7 @@ int __init cxl_file_init(void)
 
 	pr_devel("CXL device allocated, MAJOR %i\n", MAJOR(cxl_dev));
 
-	cxl_class = class_create(THIS_MODULE, "cxl");
+	cxl_class = class_create("cxl");
 	if (IS_ERR(cxl_class)) {
 		pr_err("Unable to create CXL class\n");
 		rc = PTR_ERR(cxl_class);
diff --git a/drivers/misc/enclosure.c b/drivers/misc/enclosure.c
index 4ba966529458..76511d279aff 100644
--- a/drivers/misc/enclosure.c
+++ b/drivers/misc/enclosure.c
@@ -451,7 +451,6 @@ ATTRIBUTE_GROUPS(enclosure_class);
 
 static struct class enclosure_class = {
 	.name			= "enclosure",
-	.owner			= THIS_MODULE,
 	.dev_release		= enclosure_release,
 	.dev_groups		= enclosure_class_groups,
 };
diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c
index a701132638cf..f48466960f1b 100644
--- a/drivers/misc/fastrpc.c
+++ b/drivers/misc/fastrpc.c
@@ -262,7 +262,7 @@ struct fastrpc_channel_ctx {
 	int domain_id;
 	int sesscount;
 	int vmcount;
-	u32 perms;
+	u64 perms;
 	struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];
 	struct rpmsg_device *rpdev;
 	struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
diff --git a/drivers/misc/genwqe/card_base.c b/drivers/misc/genwqe/card_base.c
index 5b63d179b24e..b03010810b89 100644
--- a/drivers/misc/genwqe/card_base.c
+++ b/drivers/misc/genwqe/card_base.c
@@ -19,7 +19,6 @@
 #include <linux/types.h>
 #include <linux/pci.h>
 #include <linux/err.h>
-#include <linux/aer.h>
 #include <linux/string.h>
 #include <linux/sched.h>
 #include <linux/wait.h>
@@ -1099,7 +1098,6 @@ static int genwqe_pci_setup(struct genwqe_dev *cd)
 	}
 
 	pci_set_master(pci_dev);
-	pci_enable_pcie_error_reporting(pci_dev);
 
 	/* EEH recovery requires PCIe fundamental reset */
 	pci_dev->needs_freset = 1;
@@ -1363,7 +1361,7 @@ static int __init genwqe_init_module(void)
 {
 	int rc;
 
-	class_genwqe = class_create(THIS_MODULE, GENWQE_DEVNAME);
+	class_genwqe = class_create(GENWQE_DEVNAME);
 	if (IS_ERR(class_genwqe)) {
 		pr_err("[%s] create class failed\n", __func__);
 		return -ENOMEM;
diff --git a/drivers/misc/hpilo.c b/drivers/misc/hpilo.c
index 8d00df9243c4..2fde8d63c5fe 100644
--- a/drivers/misc/hpilo.c
+++ b/drivers/misc/hpilo.c
@@ -392,12 +392,6 @@ static inline int is_db_reset(int db_out)
 	return db_out & (1 << DB_RESET);
 }
 
-static inline int is_device_reset(struct ilo_hwinfo *hw)
-{
-	/* check for global reset condition */
-	return is_db_reset(get_device_outbound(hw));
-}
-
 static inline void clear_pending_db(struct ilo_hwinfo *hw, int clr)
 {
 	iowrite32(clr, &hw->mmio_vaddr[DB_OUT]);
@@ -888,7 +882,7 @@ static int __init ilo_init(void)
 	int error;
 	dev_t dev;
 
-	ilo_class = class_create(THIS_MODULE, "iLO");
+	ilo_class = class_create("iLO");
 	if (IS_ERR(ilo_class)) {
 		error = PTR_ERR(ilo_class);
 		goto out;
diff --git a/drivers/misc/lis3lv02d/lis3lv02d.c b/drivers/misc/lis3lv02d/lis3lv02d.c
index 3a7808b796b1..299d316f1bda 100644
--- a/drivers/misc/lis3lv02d/lis3lv02d.c
+++ b/drivers/misc/lis3lv02d/lis3lv02d.c
@@ -965,19 +965,19 @@ int lis3lv02d_init_dt(struct lis3lv02d *lis3)
 	if (!pdata)
 		return -ENOMEM;
 
-	if (of_get_property(np, "st,click-single-x", NULL))
+	if (of_property_read_bool(np, "st,click-single-x"))
 		pdata->click_flags |= LIS3_CLICK_SINGLE_X;
-	if (of_get_property(np, "st,click-double-x", NULL))
+	if (of_property_read_bool(np, "st,click-double-x"))
 		pdata->click_flags |= LIS3_CLICK_DOUBLE_X;
 
-	if (of_get_property(np, "st,click-single-y", NULL))
+	if (of_property_read_bool(np, "st,click-single-y"))
 		pdata->click_flags |= LIS3_CLICK_SINGLE_Y;
-	if (of_get_property(np, "st,click-double-y", NULL))
+	if (of_property_read_bool(np, "st,click-double-y"))
 		pdata->click_flags |= LIS3_CLICK_DOUBLE_Y;
 
-	if (of_get_property(np, "st,click-single-z", NULL))
+	if (of_property_read_bool(np, "st,click-single-z"))
 		pdata->click_flags |= LIS3_CLICK_SINGLE_Z;
-	if (of_get_property(np, "st,click-double-z", NULL))
+	if (of_property_read_bool(np, "st,click-double-z"))
 		pdata->click_flags |= LIS3_CLICK_DOUBLE_Z;
 
 	if (!of_property_read_u32(np, "st,click-threshold-x", &val))
@@ -994,31 +994,31 @@ int lis3lv02d_init_dt(struct lis3lv02d *lis3)
 	if (!of_property_read_u32(np, "st,click-window", &val))
 		pdata->click_window = val;
 
-	if (of_get_property(np, "st,irq1-disable", NULL))
+	if (of_property_read_bool(np, "st,irq1-disable"))
 		pdata->irq_cfg |= LIS3_IRQ1_DISABLE;
-	if (of_get_property(np, "st,irq1-ff-wu-1", NULL))
+	if (of_property_read_bool(np, "st,irq1-ff-wu-1"))
 		pdata->irq_cfg |= LIS3_IRQ1_FF_WU_1;
-	if (of_get_property(np, "st,irq1-ff-wu-2", NULL))
+	if (of_property_read_bool(np, "st,irq1-ff-wu-2"))
 		pdata->irq_cfg |= LIS3_IRQ1_FF_WU_2;
-	if (of_get_property(np, "st,irq1-data-ready", NULL))
+	if (of_property_read_bool(np, "st,irq1-data-ready"))
 		pdata->irq_cfg |= LIS3_IRQ1_DATA_READY;
-	if (of_get_property(np, "st,irq1-click", NULL))
+	if (of_property_read_bool(np, "st,irq1-click"))
 		pdata->irq_cfg |= LIS3_IRQ1_CLICK;
 
-	if (of_get_property(np, "st,irq2-disable", NULL))
+	if (of_property_read_bool(np, "st,irq2-disable"))
 		pdata->irq_cfg |= LIS3_IRQ2_DISABLE;
-	if (of_get_property(np, "st,irq2-ff-wu-1", NULL))
+	if (of_property_read_bool(np, "st,irq2-ff-wu-1"))
 		pdata->irq_cfg |= LIS3_IRQ2_FF_WU_1;
-	if (of_get_property(np, "st,irq2-ff-wu-2", NULL))
+	if (of_property_read_bool(np, "st,irq2-ff-wu-2"))
 		pdata->irq_cfg |= LIS3_IRQ2_FF_WU_2;
-	if (of_get_property(np, "st,irq2-data-ready", NULL))
+	if (of_property_read_bool(np, "st,irq2-data-ready"))
 		pdata->irq_cfg |= LIS3_IRQ2_DATA_READY;
-	if (of_get_property(np, "st,irq2-click", NULL))
+	if (of_property_read_bool(np, "st,irq2-click"))
 		pdata->irq_cfg |= LIS3_IRQ2_CLICK;
 
-	if (of_get_property(np, "st,irq-open-drain", NULL))
+	if (of_property_read_bool(np, "st,irq-open-drain"))
 		pdata->irq_cfg |= LIS3_IRQ_OPEN_DRAIN;
-	if (of_get_property(np, "st,irq-active-low", NULL))
+	if (of_property_read_bool(np, "st,irq-active-low"))
 		pdata->irq_cfg |= LIS3_IRQ_ACTIVE_LOW;
 
 	if (!of_property_read_u32(np, "st,wu-duration-1", &val))
@@ -1026,32 +1026,32 @@ int lis3lv02d_init_dt(struct lis3lv02d *lis3)
 	if (!of_property_read_u32(np, "st,wu-duration-2", &val))
 		pdata->duration2 = val;
 
-	if (of_get_property(np, "st,wakeup-x-lo", NULL))
+	if (of_property_read_bool(np, "st,wakeup-x-lo"))
 		pdata->wakeup_flags |= LIS3_WAKEUP_X_LO;
-	if (of_get_property(np, "st,wakeup-x-hi", NULL))
+	if (of_property_read_bool(np, "st,wakeup-x-hi"))
 		pdata->wakeup_flags |= LIS3_WAKEUP_X_HI;
-	if (of_get_property(np, "st,wakeup-y-lo", NULL))
+	if (of_property_read_bool(np, "st,wakeup-y-lo"))
 		pdata->wakeup_flags |= LIS3_WAKEUP_Y_LO;
-	if (of_get_property(np, "st,wakeup-y-hi", NULL))
+	if (of_property_read_bool(np, "st,wakeup-y-hi"))
 		pdata->wakeup_flags |= LIS3_WAKEUP_Y_HI;
-	if (of_get_property(np, "st,wakeup-z-lo", NULL))
+	if (of_property_read_bool(np, "st,wakeup-z-lo"))
 		pdata->wakeup_flags |= LIS3_WAKEUP_Z_LO;
-	if (of_get_property(np, "st,wakeup-z-hi", NULL))
+	if (of_property_read_bool(np, "st,wakeup-z-hi"))
 		pdata->wakeup_flags |= LIS3_WAKEUP_Z_HI;
 	if (of_get_property(np, "st,wakeup-threshold", &val))
 		pdata->wakeup_thresh = val;
 
-	if (of_get_property(np, "st,wakeup2-x-lo", NULL))
+	if (of_property_read_bool(np, "st,wakeup2-x-lo"))
 		pdata->wakeup_flags2 |= LIS3_WAKEUP_X_LO;
-	if (of_get_property(np, "st,wakeup2-x-hi", NULL))
+	if (of_property_read_bool(np, "st,wakeup2-x-hi"))
 		pdata->wakeup_flags2 |= LIS3_WAKEUP_X_HI;
-	if (of_get_property(np, "st,wakeup2-y-lo", NULL))
+	if (of_property_read_bool(np, "st,wakeup2-y-lo"))
 		pdata->wakeup_flags2 |= LIS3_WAKEUP_Y_LO;
-	if (of_get_property(np, "st,wakeup2-y-hi", NULL))
+	if (of_property_read_bool(np, "st,wakeup2-y-hi"))
 		pdata->wakeup_flags2 |= LIS3_WAKEUP_Y_HI;
-	if (of_get_property(np, "st,wakeup2-z-lo", NULL))
+	if (of_property_read_bool(np, "st,wakeup2-z-lo"))
 		pdata->wakeup_flags2 |= LIS3_WAKEUP_Z_LO;
-	if (of_get_property(np, "st,wakeup2-z-hi", NULL))
+	if (of_property_read_bool(np, "st,wakeup2-z-hi"))
 		pdata->wakeup_flags2 |= LIS3_WAKEUP_Z_HI;
 	if (of_get_property(np, "st,wakeup2-threshold", &val))
 		pdata->wakeup_thresh2 = val;
@@ -1073,9 +1073,9 @@ int lis3lv02d_init_dt(struct lis3lv02d *lis3)
 		}
 	}
 
-	if (of_get_property(np, "st,hipass1-disable", NULL))
+	if (of_property_read_bool(np, "st,hipass1-disable"))
 		pdata->hipass_ctrl |= LIS3_HIPASS1_DISABLE;
-	if (of_get_property(np, "st,hipass2-disable", NULL))
+	if (of_property_read_bool(np, "st,hipass2-disable"))
 		pdata->hipass_ctrl |= LIS3_HIPASS2_DISABLE;
 
 	if (of_property_read_s32(np, "st,axis-x", &sval) == 0)
@@ -1085,7 +1085,7 @@ int lis3lv02d_init_dt(struct lis3lv02d *lis3)
 	if (of_property_read_s32(np, "st,axis-z", &sval) == 0)
 		pdata->axis_z = sval;
 
-	if (of_get_property(np, "st,default-rate", NULL))
+	if (of_property_read_u32(np, "st,default-rate", &val) == 0)
 		pdata->default_rate = val;
 
 	if (of_property_read_s32(np, "st,min-limit-x", &sval) == 0)
diff --git a/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gpio.c b/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gpio.c
index 3389803cb281..e616e3ec2b42 100644
--- a/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gpio.c
+++ b/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gpio.c
@@ -175,9 +175,13 @@ static void pci1xxxx_gpio_irq_set_mask(struct irq_data *data, bool set)
 	unsigned int gpio = irqd_to_hwirq(data);
 	unsigned long flags;
 
+	if (!set)
+		gpiochip_enable_irq(chip, gpio);
 	spin_lock_irqsave(&priv->lock, flags);
 	pci1xxx_assign_bit(priv->reg_base, INTR_MASK_OFFSET(gpio), (gpio % 32), set);
 	spin_unlock_irqrestore(&priv->lock, flags);
+	if (set)
+		gpiochip_disable_irq(chip, gpio);
 }
 
 static void pci1xxxx_gpio_irq_mask(struct irq_data *data)
@@ -283,12 +287,14 @@ static irqreturn_t pci1xxxx_gpio_irq_handler(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static struct irq_chip pci1xxxx_gpio_irqchip = {
+static const struct irq_chip pci1xxxx_gpio_irqchip = {
 	.name = "pci1xxxx_gpio",
 	.irq_ack = pci1xxxx_gpio_irq_ack,
 	.irq_mask = pci1xxxx_gpio_irq_mask,
 	.irq_unmask = pci1xxxx_gpio_irq_unmask,
 	.irq_set_type = pci1xxxx_gpio_set_type,
+	.flags = IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static int pci1xxxx_gpio_suspend(struct device *dev)
@@ -351,7 +357,7 @@ static int pci1xxxx_gpio_setup(struct pci1xxxx_gpio *priv, int irq)
 		return retval;
 
 	girq = &priv->gpio.irq;
-	girq->chip = &pci1xxxx_gpio_irqchip;
+	gpio_irq_chip_set_chip(girq, &pci1xxxx_gpio_irqchip);
 	girq->parent_handler = NULL;
 	girq->num_parents = 0;
 	girq->parents = NULL;
diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c
index 211536109308..31e3c74ca1f1 100644
--- a/drivers/misc/mei/bus-fixup.c
+++ b/drivers/misc/mei/bus-fixup.c
@@ -9,8 +9,8 @@
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/slab.h>
-#include <linux/uuid.h>
 
+#include <linux/mei.h>
 #include <linux/mei_cl_bus.h>
 
 #include "mei_dev.h"
diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c
index 9ddb854b8155..5c19097266fe 100644
--- a/drivers/misc/mei/client.c
+++ b/drivers/misc/mei/client.c
@@ -1343,7 +1343,9 @@ static void mei_cl_reset_read_by_vtag(const struct mei_cl *cl, u8 vtag)
 	struct mei_cl_vtag *vtag_l;
 
 	list_for_each_entry(vtag_l, &cl->vtag_map, list) {
-		if (vtag_l->vtag == vtag) {
+		/* The client on bus has one fixed vtag map */
+		if ((cl->cldev && mei_cldev_enabled(cl->cldev)) ||
+		    vtag_l->vtag == vtag) {
 			vtag_l->pending_read = false;
 			break;
 		}
diff --git a/drivers/misc/mei/hdcp/mei_hdcp.c b/drivers/misc/mei/hdcp/mei_hdcp.c
index e0dcd5c114db..85cbfc3413ee 100644
--- a/drivers/misc/mei/hdcp/mei_hdcp.c
+++ b/drivers/misc/mei/hdcp/mei_hdcp.c
@@ -18,12 +18,12 @@
 
 #include <linux/module.h>
 #include <linux/slab.h>
-#include <linux/uuid.h>
+#include <linux/mei.h>
 #include <linux/mei_cl_bus.h>
 #include <linux/component.h>
 #include <drm/drm_connector.h>
 #include <drm/i915_component.h>
-#include <drm/i915_mei_hdcp_interface.h>
+#include <drm/i915_hdcp_interface.h>
 
 #include "mei_hdcp.h"
 
@@ -52,13 +52,13 @@ mei_hdcp_initiate_session(struct device *dev, struct hdcp_port_data *data,
 
 	session_init_in.header.api_version = HDCP_API_VERSION;
 	session_init_in.header.command_id = WIRED_INITIATE_HDCP2_SESSION;
-	session_init_in.header.status = ME_HDCP_STATUS_SUCCESS;
+	session_init_in.header.status = FW_HDCP_STATUS_SUCCESS;
 	session_init_in.header.buffer_len =
 				WIRED_CMD_BUF_LEN_INITIATE_HDCP2_SESSION_IN;
 
 	session_init_in.port.integrated_port_type = data->port_type;
-	session_init_in.port.physical_port = (u8)data->fw_ddi;
-	session_init_in.port.attached_transcoder = (u8)data->fw_tc;
+	session_init_in.port.physical_port = (u8)data->hdcp_ddi;
+	session_init_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 	session_init_in.protocol = data->protocol;
 
 	byte = mei_cldev_send(cldev, (u8 *)&session_init_in,
@@ -75,7 +75,7 @@ mei_hdcp_initiate_session(struct device *dev, struct hdcp_port_data *data,
 		return byte;
 	}
 
-	if (session_init_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (session_init_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "ME cmd 0x%08X Failed. Status: 0x%X\n",
 			WIRED_INITIATE_HDCP2_SESSION,
 			session_init_out.header.status);
@@ -122,13 +122,13 @@ mei_hdcp_verify_receiver_cert_prepare_km(struct device *dev,
 
 	verify_rxcert_in.header.api_version = HDCP_API_VERSION;
 	verify_rxcert_in.header.command_id = WIRED_VERIFY_RECEIVER_CERT;
-	verify_rxcert_in.header.status = ME_HDCP_STATUS_SUCCESS;
+	verify_rxcert_in.header.status = FW_HDCP_STATUS_SUCCESS;
 	verify_rxcert_in.header.buffer_len =
 				WIRED_CMD_BUF_LEN_VERIFY_RECEIVER_CERT_IN;
 
 	verify_rxcert_in.port.integrated_port_type = data->port_type;
-	verify_rxcert_in.port.physical_port = (u8)data->fw_ddi;
-	verify_rxcert_in.port.attached_transcoder = (u8)data->fw_tc;
+	verify_rxcert_in.port.physical_port = (u8)data->hdcp_ddi;
+	verify_rxcert_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
 	verify_rxcert_in.cert_rx = rx_cert->cert_rx;
 	memcpy(verify_rxcert_in.r_rx, &rx_cert->r_rx, HDCP_2_2_RRX_LEN);
@@ -148,7 +148,7 @@ mei_hdcp_verify_receiver_cert_prepare_km(struct device *dev,
 		return byte;
 	}
 
-	if (verify_rxcert_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (verify_rxcert_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "ME cmd 0x%08X Failed. Status: 0x%X\n",
 			WIRED_VERIFY_RECEIVER_CERT,
 			verify_rxcert_out.header.status);
@@ -194,12 +194,12 @@ mei_hdcp_verify_hprime(struct device *dev, struct hdcp_port_data *data,
 
 	send_hprime_in.header.api_version = HDCP_API_VERSION;
 	send_hprime_in.header.command_id = WIRED_AKE_SEND_HPRIME;
-	send_hprime_in.header.status = ME_HDCP_STATUS_SUCCESS;
+	send_hprime_in.header.status = FW_HDCP_STATUS_SUCCESS;
 	send_hprime_in.header.buffer_len = WIRED_CMD_BUF_LEN_AKE_SEND_HPRIME_IN;
 
 	send_hprime_in.port.integrated_port_type = data->port_type;
-	send_hprime_in.port.physical_port = (u8)data->fw_ddi;
-	send_hprime_in.port.attached_transcoder = (u8)data->fw_tc;
+	send_hprime_in.port.physical_port = (u8)data->hdcp_ddi;
+	send_hprime_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
 	memcpy(send_hprime_in.h_prime, rx_hprime->h_prime,
 	       HDCP_2_2_H_PRIME_LEN);
@@ -218,7 +218,7 @@ mei_hdcp_verify_hprime(struct device *dev, struct hdcp_port_data *data,
 		return byte;
 	}
 
-	if (send_hprime_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (send_hprime_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "ME cmd 0x%08X Failed. Status: 0x%X\n",
 			WIRED_AKE_SEND_HPRIME, send_hprime_out.header.status);
 		return -EIO;
@@ -251,13 +251,13 @@ mei_hdcp_store_pairing_info(struct device *dev, struct hdcp_port_data *data,
 
 	pairing_info_in.header.api_version = HDCP_API_VERSION;
 	pairing_info_in.header.command_id = WIRED_AKE_SEND_PAIRING_INFO;
-	pairing_info_in.header.status = ME_HDCP_STATUS_SUCCESS;
+	pairing_info_in.header.status = FW_HDCP_STATUS_SUCCESS;
 	pairing_info_in.header.buffer_len =
 					WIRED_CMD_BUF_LEN_SEND_PAIRING_INFO_IN;
 
 	pairing_info_in.port.integrated_port_type = data->port_type;
-	pairing_info_in.port.physical_port = (u8)data->fw_ddi;
-	pairing_info_in.port.attached_transcoder = (u8)data->fw_tc;
+	pairing_info_in.port.physical_port = (u8)data->hdcp_ddi;
+	pairing_info_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
 	memcpy(pairing_info_in.e_kh_km, pairing_info->e_kh_km,
 	       HDCP_2_2_E_KH_KM_LEN);
@@ -276,7 +276,7 @@ mei_hdcp_store_pairing_info(struct device *dev, struct hdcp_port_data *data,
 		return byte;
 	}
 
-	if (pairing_info_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (pairing_info_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "ME cmd 0x%08X failed. Status: 0x%X\n",
 			WIRED_AKE_SEND_PAIRING_INFO,
 			pairing_info_out.header.status);
@@ -311,12 +311,12 @@ mei_hdcp_initiate_locality_check(struct device *dev,
 
 	lc_init_in.header.api_version = HDCP_API_VERSION;
 	lc_init_in.header.command_id = WIRED_INIT_LOCALITY_CHECK;
-	lc_init_in.header.status = ME_HDCP_STATUS_SUCCESS;
+	lc_init_in.header.status = FW_HDCP_STATUS_SUCCESS;
 	lc_init_in.header.buffer_len = WIRED_CMD_BUF_LEN_INIT_LOCALITY_CHECK_IN;
 
 	lc_init_in.port.integrated_port_type = data->port_type;
-	lc_init_in.port.physical_port = (u8)data->fw_ddi;
-	lc_init_in.port.attached_transcoder = (u8)data->fw_tc;
+	lc_init_in.port.physical_port = (u8)data->hdcp_ddi;
+	lc_init_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
 	byte = mei_cldev_send(cldev, (u8 *)&lc_init_in, sizeof(lc_init_in));
 	if (byte < 0) {
@@ -330,7 +330,7 @@ mei_hdcp_initiate_locality_check(struct device *dev,
 		return byte;
 	}
 
-	if (lc_init_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (lc_init_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "ME cmd 0x%08X Failed. status: 0x%X\n",
 			WIRED_INIT_LOCALITY_CHECK, lc_init_out.header.status);
 		return -EIO;
@@ -366,13 +366,13 @@ mei_hdcp_verify_lprime(struct device *dev, struct hdcp_port_data *data,
 
 	verify_lprime_in.header.api_version = HDCP_API_VERSION;
 	verify_lprime_in.header.command_id = WIRED_VALIDATE_LOCALITY;
-	verify_lprime_in.header.status = ME_HDCP_STATUS_SUCCESS;
+	verify_lprime_in.header.status = FW_HDCP_STATUS_SUCCESS;
 	verify_lprime_in.header.buffer_len =
 					WIRED_CMD_BUF_LEN_VALIDATE_LOCALITY_IN;
 
 	verify_lprime_in.port.integrated_port_type = data->port_type;
-	verify_lprime_in.port.physical_port = (u8)data->fw_ddi;
-	verify_lprime_in.port.attached_transcoder = (u8)data->fw_tc;
+	verify_lprime_in.port.physical_port = (u8)data->hdcp_ddi;
+	verify_lprime_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
 	memcpy(verify_lprime_in.l_prime, rx_lprime->l_prime,
 	       HDCP_2_2_L_PRIME_LEN);
@@ -391,7 +391,7 @@ mei_hdcp_verify_lprime(struct device *dev, struct hdcp_port_data *data,
 		return byte;
 	}
 
-	if (verify_lprime_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (verify_lprime_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "ME cmd 0x%08X failed. status: 0x%X\n",
 			WIRED_VALIDATE_LOCALITY,
 			verify_lprime_out.header.status);
@@ -425,12 +425,12 @@ static int mei_hdcp_get_session_key(struct device *dev,
 
 	get_skey_in.header.api_version = HDCP_API_VERSION;
 	get_skey_in.header.command_id = WIRED_GET_SESSION_KEY;
-	get_skey_in.header.status = ME_HDCP_STATUS_SUCCESS;
+	get_skey_in.header.status = FW_HDCP_STATUS_SUCCESS;
 	get_skey_in.header.buffer_len = WIRED_CMD_BUF_LEN_GET_SESSION_KEY_IN;
 
 	get_skey_in.port.integrated_port_type = data->port_type;
-	get_skey_in.port.physical_port = (u8)data->fw_ddi;
-	get_skey_in.port.attached_transcoder = (u8)data->fw_tc;
+	get_skey_in.port.physical_port = (u8)data->hdcp_ddi;
+	get_skey_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
 	byte = mei_cldev_send(cldev, (u8 *)&get_skey_in, sizeof(get_skey_in));
 	if (byte < 0) {
@@ -445,7 +445,7 @@ static int mei_hdcp_get_session_key(struct device *dev,
 		return byte;
 	}
 
-	if (get_skey_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (get_skey_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "ME cmd 0x%08X failed. status: 0x%X\n",
 			WIRED_GET_SESSION_KEY, get_skey_out.header.status);
 		return -EIO;
@@ -489,13 +489,13 @@ mei_hdcp_repeater_check_flow_prepare_ack(struct device *dev,
 
 	verify_repeater_in.header.api_version = HDCP_API_VERSION;
 	verify_repeater_in.header.command_id = WIRED_VERIFY_REPEATER;
-	verify_repeater_in.header.status = ME_HDCP_STATUS_SUCCESS;
+	verify_repeater_in.header.status = FW_HDCP_STATUS_SUCCESS;
 	verify_repeater_in.header.buffer_len =
 					WIRED_CMD_BUF_LEN_VERIFY_REPEATER_IN;
 
 	verify_repeater_in.port.integrated_port_type = data->port_type;
-	verify_repeater_in.port.physical_port = (u8)data->fw_ddi;
-	verify_repeater_in.port.attached_transcoder = (u8)data->fw_tc;
+	verify_repeater_in.port.physical_port = (u8)data->hdcp_ddi;
+	verify_repeater_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
 	memcpy(verify_repeater_in.rx_info, rep_topology->rx_info,
 	       HDCP_2_2_RXINFO_LEN);
@@ -520,7 +520,7 @@ mei_hdcp_repeater_check_flow_prepare_ack(struct device *dev,
 		return byte;
 	}
 
-	if (verify_repeater_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (verify_repeater_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "ME cmd 0x%08X failed. status: 0x%X\n",
 			WIRED_VERIFY_REPEATER,
 			verify_repeater_out.header.status);
@@ -568,12 +568,12 @@ static int mei_hdcp_verify_mprime(struct device *dev,
 
 	verify_mprime_in->header.api_version = HDCP_API_VERSION;
 	verify_mprime_in->header.command_id = WIRED_REPEATER_AUTH_STREAM_REQ;
-	verify_mprime_in->header.status = ME_HDCP_STATUS_SUCCESS;
+	verify_mprime_in->header.status = FW_HDCP_STATUS_SUCCESS;
 	verify_mprime_in->header.buffer_len = cmd_size  - sizeof(verify_mprime_in->header);
 
 	verify_mprime_in->port.integrated_port_type = data->port_type;
-	verify_mprime_in->port.physical_port = (u8)data->fw_ddi;
-	verify_mprime_in->port.attached_transcoder = (u8)data->fw_tc;
+	verify_mprime_in->port.physical_port = (u8)data->hdcp_ddi;
+	verify_mprime_in->port.attached_transcoder = (u8)data->hdcp_transcoder;
 
 	memcpy(verify_mprime_in->m_prime, stream_ready->m_prime, HDCP_2_2_MPRIME_LEN);
 	drm_hdcp_cpu_to_be24(verify_mprime_in->seq_num_m, data->seq_num_m);
@@ -597,7 +597,7 @@ static int mei_hdcp_verify_mprime(struct device *dev,
 		return byte;
 	}
 
-	if (verify_mprime_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (verify_mprime_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "ME cmd 0x%08X failed. status: 0x%X\n",
 			WIRED_REPEATER_AUTH_STREAM_REQ,
 			verify_mprime_out.header.status);
@@ -630,12 +630,12 @@ static int mei_hdcp_enable_authentication(struct device *dev,
 
 	enable_auth_in.header.api_version = HDCP_API_VERSION;
 	enable_auth_in.header.command_id = WIRED_ENABLE_AUTH;
-	enable_auth_in.header.status = ME_HDCP_STATUS_SUCCESS;
+	enable_auth_in.header.status = FW_HDCP_STATUS_SUCCESS;
 	enable_auth_in.header.buffer_len = WIRED_CMD_BUF_LEN_ENABLE_AUTH_IN;
 
 	enable_auth_in.port.integrated_port_type = data->port_type;
-	enable_auth_in.port.physical_port = (u8)data->fw_ddi;
-	enable_auth_in.port.attached_transcoder = (u8)data->fw_tc;
+	enable_auth_in.port.physical_port = (u8)data->hdcp_ddi;
+	enable_auth_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 	enable_auth_in.stream_type = data->streams[0].stream_type;
 
 	byte = mei_cldev_send(cldev, (u8 *)&enable_auth_in,
@@ -652,7 +652,7 @@ static int mei_hdcp_enable_authentication(struct device *dev,
 		return byte;
 	}
 
-	if (enable_auth_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (enable_auth_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "ME cmd 0x%08X failed. status: 0x%X\n",
 			WIRED_ENABLE_AUTH, enable_auth_out.header.status);
 		return -EIO;
@@ -684,13 +684,13 @@ mei_hdcp_close_session(struct device *dev, struct hdcp_port_data *data)
 
 	session_close_in.header.api_version = HDCP_API_VERSION;
 	session_close_in.header.command_id = WIRED_CLOSE_SESSION;
-	session_close_in.header.status = ME_HDCP_STATUS_SUCCESS;
+	session_close_in.header.status = FW_HDCP_STATUS_SUCCESS;
 	session_close_in.header.buffer_len =
 				WIRED_CMD_BUF_LEN_CLOSE_SESSION_IN;
 
 	session_close_in.port.integrated_port_type = data->port_type;
-	session_close_in.port.physical_port = (u8)data->fw_ddi;
-	session_close_in.port.attached_transcoder = (u8)data->fw_tc;
+	session_close_in.port.physical_port = (u8)data->hdcp_ddi;
+	session_close_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
 	byte = mei_cldev_send(cldev, (u8 *)&session_close_in,
 			      sizeof(session_close_in));
@@ -706,7 +706,7 @@ mei_hdcp_close_session(struct device *dev, struct hdcp_port_data *data)
 		return byte;
 	}
 
-	if (session_close_out.header.status != ME_HDCP_STATUS_SUCCESS) {
+	if (session_close_out.header.status != FW_HDCP_STATUS_SUCCESS) {
 		dev_dbg(dev, "Session Close Failed. status: 0x%X\n",
 			session_close_out.header.status);
 		return -EIO;
@@ -715,7 +715,7 @@ mei_hdcp_close_session(struct device *dev, struct hdcp_port_data *data)
 	return 0;
 }
 
-static const struct i915_hdcp_component_ops mei_hdcp_ops = {
+static const struct i915_hdcp_ops mei_hdcp_ops = {
 	.owner = THIS_MODULE,
 	.initiate_hdcp2_session = mei_hdcp_initiate_session,
 	.verify_receiver_cert_prepare_km =
@@ -735,13 +735,12 @@ static const struct i915_hdcp_component_ops mei_hdcp_ops = {
 static int mei_component_master_bind(struct device *dev)
 {
 	struct mei_cl_device *cldev = to_mei_cl_device(dev);
-	struct i915_hdcp_comp_master *comp_master =
-						mei_cldev_get_drvdata(cldev);
+	struct i915_hdcp_master *comp_master = mei_cldev_get_drvdata(cldev);
 	int ret;
 
 	dev_dbg(dev, "%s\n", __func__);
 	comp_master->ops = &mei_hdcp_ops;
-	comp_master->mei_dev = dev;
+	comp_master->hdcp_dev = dev;
 	ret = component_bind_all(dev, comp_master);
 	if (ret < 0)
 		return ret;
@@ -752,8 +751,7 @@ static int mei_component_master_bind(struct device *dev)
 static void mei_component_master_unbind(struct device *dev)
 {
 	struct mei_cl_device *cldev = to_mei_cl_device(dev);
-	struct i915_hdcp_comp_master *comp_master =
-						mei_cldev_get_drvdata(cldev);
+	struct i915_hdcp_master *comp_master = mei_cldev_get_drvdata(cldev);
 
 	dev_dbg(dev, "%s\n", __func__);
 	component_unbind_all(dev, comp_master);
@@ -801,7 +799,7 @@ static int mei_hdcp_component_match(struct device *dev, int subcomponent,
 static int mei_hdcp_probe(struct mei_cl_device *cldev,
 			  const struct mei_cl_device_id *id)
 {
-	struct i915_hdcp_comp_master *comp_master;
+	struct i915_hdcp_master *comp_master;
 	struct component_match *master_match;
 	int ret;
 
@@ -846,8 +844,7 @@ enable_err_exit:
 
 static void mei_hdcp_remove(struct mei_cl_device *cldev)
 {
-	struct i915_hdcp_comp_master *comp_master =
-						mei_cldev_get_drvdata(cldev);
+	struct i915_hdcp_master *comp_master = mei_cldev_get_drvdata(cldev);
 	int ret;
 
 	component_master_del(&cldev->dev, &mei_component_master_ops);
diff --git a/drivers/misc/mei/hdcp/mei_hdcp.h b/drivers/misc/mei/hdcp/mei_hdcp.h
index ca09c8f83d6b..0683d83ec17a 100644
--- a/drivers/misc/mei/hdcp/mei_hdcp.h
+++ b/drivers/misc/mei/hdcp/mei_hdcp.h
@@ -11,358 +11,4 @@
 
 #include <drm/display/drm_hdcp.h>
 
-/* me_hdcp_status: Enumeration of all HDCP Status Codes */
-enum me_hdcp_status {
-	ME_HDCP_STATUS_SUCCESS			= 0x0000,
-
-	/* WiDi Generic Status Codes */
-	ME_HDCP_STATUS_INTERNAL_ERROR		= 0x1000,
-	ME_HDCP_STATUS_UNKNOWN_ERROR		= 0x1001,
-	ME_HDCP_STATUS_INCORRECT_API_VERSION	= 0x1002,
-	ME_HDCP_STATUS_INVALID_FUNCTION		= 0x1003,
-	ME_HDCP_STATUS_INVALID_BUFFER_LENGTH	= 0x1004,
-	ME_HDCP_STATUS_INVALID_PARAMS		= 0x1005,
-	ME_HDCP_STATUS_AUTHENTICATION_FAILED	= 0x1006,
-
-	/* WiDi Status Codes */
-	ME_HDCP_INVALID_SESSION_STATE		= 0x6000,
-	ME_HDCP_SRM_FRAGMENT_UNEXPECTED		= 0x6001,
-	ME_HDCP_SRM_INVALID_LENGTH		= 0x6002,
-	ME_HDCP_SRM_FRAGMENT_OFFSET_INVALID	= 0x6003,
-	ME_HDCP_SRM_VERIFICATION_FAILED		= 0x6004,
-	ME_HDCP_SRM_VERSION_TOO_OLD		= 0x6005,
-	ME_HDCP_RX_CERT_VERIFICATION_FAILED	= 0x6006,
-	ME_HDCP_RX_REVOKED			= 0x6007,
-	ME_HDCP_H_VERIFICATION_FAILED		= 0x6008,
-	ME_HDCP_REPEATER_CHECK_UNEXPECTED	= 0x6009,
-	ME_HDCP_TOPOLOGY_MAX_EXCEEDED		= 0x600A,
-	ME_HDCP_V_VERIFICATION_FAILED		= 0x600B,
-	ME_HDCP_L_VERIFICATION_FAILED		= 0x600C,
-	ME_HDCP_STREAM_KEY_ALLOC_FAILED		= 0x600D,
-	ME_HDCP_BASE_KEY_RESET_FAILED		= 0x600E,
-	ME_HDCP_NONCE_GENERATION_FAILED		= 0x600F,
-	ME_HDCP_STATUS_INVALID_E_KEY_STATE	= 0x6010,
-	ME_HDCP_STATUS_INVALID_CS_ICV		= 0x6011,
-	ME_HDCP_STATUS_INVALID_KB_KEY_STATE	= 0x6012,
-	ME_HDCP_STATUS_INVALID_PAVP_MODE_ICV	= 0x6013,
-	ME_HDCP_STATUS_INVALID_PAVP_MODE	= 0x6014,
-	ME_HDCP_STATUS_LC_MAX_ATTEMPTS		= 0x6015,
-
-	/* New status for HDCP 2.1 */
-	ME_HDCP_STATUS_MISMATCH_IN_M		= 0x6016,
-
-	/* New status code for HDCP 2.2 Rx */
-	ME_HDCP_STATUS_RX_PROV_NOT_ALLOWED	= 0x6017,
-	ME_HDCP_STATUS_RX_PROV_WRONG_SUBJECT	= 0x6018,
-	ME_HDCP_RX_NEEDS_PROVISIONING		= 0x6019,
-	ME_HDCP_BKSV_ICV_AUTH_FAILED		= 0x6020,
-	ME_HDCP_STATUS_INVALID_STREAM_ID	= 0x6021,
-	ME_HDCP_STATUS_CHAIN_NOT_INITIALIZED	= 0x6022,
-	ME_HDCP_FAIL_NOT_EXPECTED		= 0x6023,
-	ME_HDCP_FAIL_HDCP_OFF			= 0x6024,
-	ME_HDCP_FAIL_INVALID_PAVP_MEMORY_MODE	= 0x6025,
-	ME_HDCP_FAIL_AES_ECB_FAILURE		= 0x6026,
-	ME_HDCP_FEATURE_NOT_SUPPORTED		= 0x6027,
-	ME_HDCP_DMA_READ_ERROR			= 0x6028,
-	ME_HDCP_DMA_WRITE_ERROR			= 0x6029,
-	ME_HDCP_FAIL_INVALID_PACKET_SIZE	= 0x6030,
-	ME_HDCP_H264_PARSING_ERROR		= 0x6031,
-	ME_HDCP_HDCP2_ERRATA_VIDEO_VIOLATION	= 0x6032,
-	ME_HDCP_HDCP2_ERRATA_AUDIO_VIOLATION	= 0x6033,
-	ME_HDCP_TX_ACTIVE_ERROR			= 0x6034,
-	ME_HDCP_MODE_CHANGE_ERROR		= 0x6035,
-	ME_HDCP_STREAM_TYPE_ERROR		= 0x6036,
-	ME_HDCP_STREAM_MANAGE_NOT_POSSIBLE	= 0x6037,
-
-	ME_HDCP_STATUS_PORT_INVALID_COMMAND	= 0x6038,
-	ME_HDCP_STATUS_UNSUPPORTED_PROTOCOL	= 0x6039,
-	ME_HDCP_STATUS_INVALID_PORT_INDEX	= 0x603a,
-	ME_HDCP_STATUS_TX_AUTH_NEEDED		= 0x603b,
-	ME_HDCP_STATUS_NOT_INTEGRATED_PORT	= 0x603c,
-	ME_HDCP_STATUS_SESSION_MAX_REACHED	= 0x603d,
-
-	/* hdcp capable bit is not set in rx_caps(error is unique to DP) */
-	ME_HDCP_STATUS_NOT_HDCP_CAPABLE		= 0x6041,
-
-	ME_HDCP_STATUS_INVALID_STREAM_COUNT	= 0x6042,
-};
-
-#define HDCP_API_VERSION				0x00010000
-
-#define HDCP_M_LEN					16
-#define HDCP_KH_LEN					16
-
-/* Payload Buffer size(Excluding Header) for CMDs and corresponding response */
-/* Wired_Tx_AKE  */
-#define	WIRED_CMD_BUF_LEN_INITIATE_HDCP2_SESSION_IN	(4 + 1)
-#define	WIRED_CMD_BUF_LEN_INITIATE_HDCP2_SESSION_OUT	(4 + 8 + 3)
-
-#define	WIRED_CMD_BUF_LEN_VERIFY_RECEIVER_CERT_IN	(4 + 522 + 8 + 3)
-#define	WIRED_CMD_BUF_LEN_VERIFY_RECEIVER_CERT_MIN_OUT	(4 + 1 + 3 + 16 + 16)
-#define	WIRED_CMD_BUF_LEN_VERIFY_RECEIVER_CERT_MAX_OUT	(4 + 1 + 3 + 128)
-
-#define	WIRED_CMD_BUF_LEN_AKE_SEND_HPRIME_IN		(4 + 32)
-#define	WIRED_CMD_BUF_LEN_AKE_SEND_HPRIME_OUT		(4)
-
-#define	WIRED_CMD_BUF_LEN_SEND_PAIRING_INFO_IN		(4 + 16)
-#define	WIRED_CMD_BUF_LEN_SEND_PAIRING_INFO_OUT		(4)
-
-#define	WIRED_CMD_BUF_LEN_CLOSE_SESSION_IN		(4)
-#define	WIRED_CMD_BUF_LEN_CLOSE_SESSION_OUT		(4)
-
-/* Wired_Tx_LC */
-#define	WIRED_CMD_BUF_LEN_INIT_LOCALITY_CHECK_IN	(4)
-#define	WIRED_CMD_BUF_LEN_INIT_LOCALITY_CHECK_OUT	(4 + 8)
-
-#define	WIRED_CMD_BUF_LEN_VALIDATE_LOCALITY_IN		(4 + 32)
-#define	WIRED_CMD_BUF_LEN_VALIDATE_LOCALITY_OUT		(4)
-
-/* Wired_Tx_SKE */
-#define	WIRED_CMD_BUF_LEN_GET_SESSION_KEY_IN		(4)
-#define	WIRED_CMD_BUF_LEN_GET_SESSION_KEY_OUT		(4 + 16 + 8)
-
-/* Wired_Tx_SKE */
-#define	WIRED_CMD_BUF_LEN_ENABLE_AUTH_IN		(4 + 1)
-#define	WIRED_CMD_BUF_LEN_ENABLE_AUTH_OUT		(4)
-
-/* Wired_Tx_Repeater */
-#define	WIRED_CMD_BUF_LEN_VERIFY_REPEATER_IN		(4 + 2 + 3 + 16 + 155)
-#define	WIRED_CMD_BUF_LEN_VERIFY_REPEATER_OUT		(4 + 1 + 16)
-
-#define	WIRED_CMD_BUF_LEN_REPEATER_AUTH_STREAM_REQ_MIN_IN	(4 + 3 + \
-								32 + 2 + 2)
-
-#define	WIRED_CMD_BUF_LEN_REPEATER_AUTH_STREAM_REQ_OUT		(4)
-
-/* hdcp_command_id: Enumeration of all WIRED HDCP Command IDs */
-enum hdcp_command_id {
-	_WIDI_COMMAND_BASE		= 0x00030000,
-	WIDI_INITIATE_HDCP2_SESSION	= _WIDI_COMMAND_BASE,
-	HDCP_GET_SRM_STATUS,
-	HDCP_SEND_SRM_FRAGMENT,
-
-	/* The wired HDCP Tx commands */
-	_WIRED_COMMAND_BASE		= 0x00031000,
-	WIRED_INITIATE_HDCP2_SESSION	= _WIRED_COMMAND_BASE,
-	WIRED_VERIFY_RECEIVER_CERT,
-	WIRED_AKE_SEND_HPRIME,
-	WIRED_AKE_SEND_PAIRING_INFO,
-	WIRED_INIT_LOCALITY_CHECK,
-	WIRED_VALIDATE_LOCALITY,
-	WIRED_GET_SESSION_KEY,
-	WIRED_ENABLE_AUTH,
-	WIRED_VERIFY_REPEATER,
-	WIRED_REPEATER_AUTH_STREAM_REQ,
-	WIRED_CLOSE_SESSION,
-
-	_WIRED_COMMANDS_COUNT,
-};
-
-union encrypted_buff {
-	u8		e_kpub_km[HDCP_2_2_E_KPUB_KM_LEN];
-	u8		e_kh_km_m[HDCP_2_2_E_KH_KM_M_LEN];
-	struct {
-		u8	e_kh_km[HDCP_KH_LEN];
-		u8	m[HDCP_M_LEN];
-	} __packed;
-};
-
-/* HDCP HECI message header. All header values are little endian. */
-struct hdcp_cmd_header {
-	u32			api_version;
-	u32			command_id;
-	enum me_hdcp_status	status;
-	/* Length of the HECI message (excluding the header) */
-	u32			buffer_len;
-} __packed;
-
-/* Empty command request or response. No data follows the header. */
-struct hdcp_cmd_no_data {
-	struct hdcp_cmd_header header;
-} __packed;
-
-/* Uniquely identifies the hdcp port being addressed for a given command. */
-struct hdcp_port_id {
-	u8	integrated_port_type;
-	/* physical_port is used until Gen11.5. Must be zero for Gen11.5+ */
-	u8	physical_port;
-	/* attached_transcoder is for Gen11.5+. Set to zero for <Gen11.5 */
-	u8	attached_transcoder;
-	u8	reserved;
-} __packed;
-
-/*
- * Data structures for integrated wired HDCP2 Tx in
- * support of the AKE protocol
- */
-/* HECI struct for integrated wired HDCP Tx session initiation. */
-struct wired_cmd_initiate_hdcp2_session_in {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			protocol; /* for HDMI vs DP */
-} __packed;
-
-struct wired_cmd_initiate_hdcp2_session_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			r_tx[HDCP_2_2_RTX_LEN];
-	struct hdcp2_tx_caps	tx_caps;
-} __packed;
-
-/* HECI struct for ending an integrated wired HDCP Tx session. */
-struct wired_cmd_close_session_in {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-} __packed;
-
-struct wired_cmd_close_session_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-} __packed;
-
-/* HECI struct for integrated wired HDCP Tx Rx Cert verification. */
-struct wired_cmd_verify_receiver_cert_in {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	struct hdcp2_cert_rx	cert_rx;
-	u8			r_rx[HDCP_2_2_RRX_LEN];
-	u8			rx_caps[HDCP_2_2_RXCAPS_LEN];
-} __packed;
-
-struct wired_cmd_verify_receiver_cert_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			km_stored;
-	u8			reserved[3];
-	union encrypted_buff	ekm_buff;
-} __packed;
-
-/* HECI struct for verification of Rx's Hprime in a HDCP Tx session */
-struct wired_cmd_ake_send_hprime_in {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			h_prime[HDCP_2_2_H_PRIME_LEN];
-} __packed;
-
-struct wired_cmd_ake_send_hprime_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-} __packed;
-
-/*
- * HECI struct for sending in AKE pairing data generated by the Rx in an
- * integrated wired HDCP Tx session.
- */
-struct wired_cmd_ake_send_pairing_info_in {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			e_kh_km[HDCP_2_2_E_KH_KM_LEN];
-} __packed;
-
-struct wired_cmd_ake_send_pairing_info_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-} __packed;
-
-/* Data structures for integrated wired HDCP2 Tx in support of the LC protocol*/
-/*
- * HECI struct for initiating locality check with an
- * integrated wired HDCP Tx session.
- */
-struct wired_cmd_init_locality_check_in {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-} __packed;
-
-struct wired_cmd_init_locality_check_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			r_n[HDCP_2_2_RN_LEN];
-} __packed;
-
-/*
- * HECI struct for validating an Rx's LPrime value in an
- * integrated wired HDCP Tx session.
- */
-struct wired_cmd_validate_locality_in {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			l_prime[HDCP_2_2_L_PRIME_LEN];
-} __packed;
-
-struct wired_cmd_validate_locality_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-} __packed;
-
-/*
- * Data structures for integrated wired HDCP2 Tx in support of the
- * SKE protocol
- */
-/* HECI struct for creating session key */
-struct wired_cmd_get_session_key_in {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-} __packed;
-
-struct wired_cmd_get_session_key_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			e_dkey_ks[HDCP_2_2_E_DKEY_KS_LEN];
-	u8			r_iv[HDCP_2_2_RIV_LEN];
-} __packed;
-
-/* HECI struct for the Tx enable authentication command */
-struct wired_cmd_enable_auth_in {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			stream_type;
-} __packed;
-
-struct wired_cmd_enable_auth_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-} __packed;
-
-/*
- * Data structures for integrated wired HDCP2 Tx in support of
- * the repeater protocols
- */
-/*
- * HECI struct for verifying the downstream repeater's HDCP topology in an
- * integrated wired HDCP Tx session.
- */
-struct wired_cmd_verify_repeater_in {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			rx_info[HDCP_2_2_RXINFO_LEN];
-	u8			seq_num_v[HDCP_2_2_SEQ_NUM_LEN];
-	u8			v_prime[HDCP_2_2_V_PRIME_HALF_LEN];
-	u8			receiver_ids[HDCP_2_2_RECEIVER_IDS_MAX_LEN];
-} __packed;
-
-struct wired_cmd_verify_repeater_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-	u8			content_type_supported;
-	u8			v[HDCP_2_2_V_PRIME_HALF_LEN];
-} __packed;
-
-/*
- * HECI struct in support of stream management in an
- * integrated wired HDCP Tx session.
- */
-struct wired_cmd_repeater_auth_stream_req_in {
-	struct hdcp_cmd_header		header;
-	struct hdcp_port_id		port;
-	u8				seq_num_m[HDCP_2_2_SEQ_NUM_LEN];
-	u8				m_prime[HDCP_2_2_MPRIME_LEN];
-	__be16				k;
-	struct hdcp2_streamid_type	streams[];
-} __packed;
-
-struct wired_cmd_repeater_auth_stream_req_out {
-	struct hdcp_cmd_header	header;
-	struct hdcp_port_id	port;
-} __packed;
 #endif /* __MEI_HDCP_H__ */
diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h
index 319418ddf4fb..e910302fcd1f 100644
--- a/drivers/misc/mei/hw.h
+++ b/drivers/misc/mei/hw.h
@@ -7,7 +7,7 @@
 #ifndef _MEI_HW_TYPES_H_
 #define _MEI_HW_TYPES_H_
 
-#include <linux/uuid.h>
+#include <linux/mei.h>
 
 /*
  * Timeouts in Seconds
diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c
index 632d4ae21e46..51876da3fd65 100644
--- a/drivers/misc/mei/main.c
+++ b/drivers/misc/mei/main.c
@@ -18,7 +18,6 @@
 #include <linux/ioctl.h>
 #include <linux/cdev.h>
 #include <linux/sched/signal.h>
-#include <linux/uuid.h>
 #include <linux/compat.h>
 #include <linux/jiffies.h>
 #include <linux/interrupt.h>
@@ -1275,7 +1274,7 @@ static int __init mei_init(void)
 {
 	int ret;
 
-	mei_class = class_create(THIS_MODULE, "mei");
+	mei_class = class_create("mei");
 	if (IS_ERR(mei_class)) {
 		pr_err("couldn't create class\n");
 		ret = PTR_ERR(mei_class);
diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c
index 5bf0d50d55a0..676d566f38dd 100644
--- a/drivers/misc/mei/pci-me.c
+++ b/drivers/misc/mei/pci-me.c
@@ -342,6 +342,12 @@ static void mei_me_remove(struct pci_dev *pdev)
 }
 
 #ifdef CONFIG_PM_SLEEP
+static int mei_me_pci_prepare(struct device *device)
+{
+	pm_runtime_resume(device);
+	return 0;
+}
+
 static int mei_me_pci_suspend(struct device *device)
 {
 	struct pci_dev *pdev = to_pci_dev(device);
@@ -398,7 +404,17 @@ static int mei_me_pci_resume(struct device *device)
 
 	return 0;
 }
-#endif /* CONFIG_PM_SLEEP */
+
+static void mei_me_pci_complete(struct device *device)
+{
+	pm_runtime_suspend(device);
+}
+#else /* CONFIG_PM_SLEEP */
+
+#define mei_me_pci_prepare NULL
+#define mei_me_pci_complete NULL
+
+#endif /* !CONFIG_PM_SLEEP */
 
 #ifdef CONFIG_PM
 static int mei_me_pm_runtime_idle(struct device *device)
@@ -501,6 +517,8 @@ static inline void mei_me_unset_pm_domain(struct mei_device *dev)
 }
 
 static const struct dev_pm_ops mei_me_pm_ops = {
+	.prepare = mei_me_pci_prepare,
+	.complete = mei_me_pci_complete,
 	SET_SYSTEM_SLEEP_PM_OPS(mei_me_pci_suspend,
 				mei_me_pci_resume)
 	SET_RUNTIME_PM_OPS(
diff --git a/drivers/misc/mei/pxp/mei_pxp.c b/drivers/misc/mei/pxp/mei_pxp.c
index 7ee1fa7b1cb3..3bf560bbdee0 100644
--- a/drivers/misc/mei/pxp/mei_pxp.c
+++ b/drivers/misc/mei/pxp/mei_pxp.c
@@ -13,7 +13,7 @@
 
 #include <linux/module.h>
 #include <linux/slab.h>
-#include <linux/uuid.h>
+#include <linux/mei.h>
 #include <linux/mei_cl_bus.h>
 #include <linux/component.h>
 #include <drm/drm_connector.h>
diff --git a/drivers/misc/ocxl/file.c b/drivers/misc/ocxl/file.c
index 3b058654b45b..6e63f060e4cc 100644
--- a/drivers/misc/ocxl/file.c
+++ b/drivers/misc/ocxl/file.c
@@ -601,7 +601,7 @@ int ocxl_file_init(void)
 		return rc;
 	}
 
-	ocxl_class = class_create(THIS_MODULE, "ocxl");
+	ocxl_class = class_create("ocxl");
 	if (IS_ERR(ocxl_class)) {
 		pr_err("Unable to create ocxl class\n");
 		unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS);
diff --git a/drivers/misc/phantom.c b/drivers/misc/phantom.c
index ce72e46a2e73..7966a6b8b5b3 100644
--- a/drivers/misc/phantom.c
+++ b/drivers/misc/phantom.c
@@ -503,7 +503,7 @@ static int __init phantom_init(void)
 	int retval;
 	dev_t dev;
 
-	phantom_class = class_create(THIS_MODULE, "phantom");
+	phantom_class = class_create("phantom");
 	if (IS_ERR(phantom_class)) {
 		retval = PTR_ERR(phantom_class);
 		printk(KERN_ERR "phantom: can't register phantom class\n");
diff --git a/drivers/misc/sgi-xp/xpc_main.c b/drivers/misc/sgi-xp/xpc_main.c
index b2c3c22fc13c..6da509d692bb 100644
--- a/drivers/misc/sgi-xp/xpc_main.c
+++ b/drivers/misc/sgi-xp/xpc_main.c
@@ -93,7 +93,7 @@ int xpc_disengage_timelimit = XPC_DISENGAGE_DEFAULT_TIMELIMIT;
 static int xpc_disengage_min_timelimit;	/* = 0 */
 static int xpc_disengage_max_timelimit = 120;
 
-static struct ctl_table xpc_sys_xpc_hb_dir[] = {
+static struct ctl_table xpc_sys_xpc_hb[] = {
 	{
 	 .procname = "hb_interval",
 	 .data = &xpc_hb_interval,
@@ -112,11 +112,7 @@ static struct ctl_table xpc_sys_xpc_hb_dir[] = {
 	 .extra2 = &xpc_hb_check_max_interval},
 	{}
 };
-static struct ctl_table xpc_sys_xpc_dir[] = {
-	{
-	 .procname = "hb",
-	 .mode = 0555,
-	 .child = xpc_sys_xpc_hb_dir},
+static struct ctl_table xpc_sys_xpc[] = {
 	{
 	 .procname = "disengage_timelimit",
 	 .data = &xpc_disengage_timelimit,
@@ -127,14 +123,9 @@ static struct ctl_table xpc_sys_xpc_dir[] = {
 	 .extra2 = &xpc_disengage_max_timelimit},
 	{}
 };
-static struct ctl_table xpc_sys_dir[] = {
-	{
-	 .procname = "xpc",
-	 .mode = 0555,
-	 .child = xpc_sys_xpc_dir},
-	{}
-};
+
 static struct ctl_table_header *xpc_sysctl;
+static struct ctl_table_header *xpc_sysctl_hb;
 
 /* non-zero if any remote partition disengage was timed out */
 int xpc_disengage_timedout;
@@ -1041,6 +1032,8 @@ xpc_do_exit(enum xp_retval reason)
 
 	if (xpc_sysctl)
 		unregister_sysctl_table(xpc_sysctl);
+	if (xpc_sysctl_hb)
+		unregister_sysctl_table(xpc_sysctl_hb);
 
 	xpc_teardown_partitions();
 
@@ -1243,7 +1236,8 @@ xpc_init(void)
 		goto out_1;
 	}
 
-	xpc_sysctl = register_sysctl_table(xpc_sys_dir);
+	xpc_sysctl = register_sysctl("xpc", xpc_sys_xpc);
+	xpc_sysctl_hb = register_sysctl("xpc/hb", xpc_sys_xpc_hb);
 
 	/*
 	 * Fill the partition reserved page with the information needed by
@@ -1308,6 +1302,8 @@ out_3:
 	(void)unregister_die_notifier(&xpc_die_notifier);
 	(void)unregister_reboot_notifier(&xpc_reboot_notifier);
 out_2:
+	if (xpc_sysctl_hb)
+		unregister_sysctl_table(xpc_sysctl_hb);
 	if (xpc_sysctl)
 		unregister_sysctl_table(xpc_sysctl);
 
diff --git a/drivers/misc/smpro-errmon.c b/drivers/misc/smpro-errmon.c
index d1431d419aa4..a1f0b2c77fac 100644
--- a/drivers/misc/smpro-errmon.c
+++ b/drivers/misc/smpro-errmon.c
@@ -47,6 +47,12 @@
 #define WARN_PMPRO_INFO_LO	0xAC
 #define WARN_PMPRO_INFO_HI	0xAD
 
+/* Boot Stage Register */
+#define BOOTSTAGE		0xB0
+#define DIMM_SYNDROME_SEL	0xB4
+#define DIMM_SYNDROME_ERR	0xB5
+#define DIMM_SYNDROME_STAGE	4
+
 /* PCIE Error Registers */
 #define PCIE_CE_ERR_CNT		0xC0
 #define PCIE_CE_ERR_LEN		0xC1
@@ -67,6 +73,7 @@
 #define VRD_WARN_FAULT_EVENT_DATA	0x78
 #define VRD_HOT_EVENT_DATA		0x79
 #define DIMM_HOT_EVENT_DATA		0x7A
+#define DIMM_2X_REFRESH_EVENT_DATA	0x96
 
 #define MAX_READ_BLOCK_LENGTH	48
 
@@ -190,6 +197,7 @@ enum EVENT_TYPES {
 	VRD_WARN_FAULT_EVENT,
 	VRD_HOT_EVENT,
 	DIMM_HOT_EVENT,
+	DIMM_2X_REFRESH_EVENT,
 	NUM_EVENTS_TYPE,
 };
 
@@ -198,6 +206,7 @@ static u8 smpro_event_table[NUM_EVENTS_TYPE] = {
 	VRD_WARN_FAULT_EVENT_DATA,
 	VRD_HOT_EVENT_DATA,
 	DIMM_HOT_EVENT_DATA,
+	DIMM_2X_REFRESH_EVENT_DATA,
 };
 
 static ssize_t smpro_event_data_read(struct device *dev,
@@ -463,6 +472,62 @@ static DEVICE_ATTR_RO(warn_pmpro);
 EVENT_RO(vrd_warn_fault, VRD_WARN_FAULT_EVENT);
 EVENT_RO(vrd_hot, VRD_HOT_EVENT);
 EVENT_RO(dimm_hot, DIMM_HOT_EVENT);
+EVENT_RO(dimm_2x_refresh, DIMM_2X_REFRESH_EVENT);
+
+static ssize_t smpro_dimm_syndrome_read(struct device *dev, struct device_attribute *da,
+					char *buf, unsigned int slot)
+{
+	struct smpro_errmon *errmon = dev_get_drvdata(dev);
+	unsigned int data;
+	int ret;
+
+	ret = regmap_read(errmon->regmap, BOOTSTAGE, &data);
+	if (ret)
+		return ret;
+
+	/* check for valid stage */
+	data = (data >> 8) & 0xff;
+	if (data != DIMM_SYNDROME_STAGE)
+		return ret;
+
+	/* Write the slot ID to retrieve Error Syndrome */
+	ret = regmap_write(errmon->regmap, DIMM_SYNDROME_SEL, slot);
+	if (ret)
+		return ret;
+
+	/* Read the Syndrome error */
+	ret = regmap_read(errmon->regmap, DIMM_SYNDROME_ERR, &data);
+	if (ret || !data)
+		return ret;
+
+	return sysfs_emit(buf, "%04x\n", data);
+}
+
+#define EVENT_DIMM_SYNDROME(_slot) \
+	static ssize_t event_dimm##_slot##_syndrome_show(struct device *dev,          \
+							 struct device_attribute *da, \
+							 char *buf)                   \
+	{                                                                             \
+		return smpro_dimm_syndrome_read(dev, da, buf, _slot);                 \
+	}                                                                             \
+	static DEVICE_ATTR_RO(event_dimm##_slot##_syndrome)
+
+EVENT_DIMM_SYNDROME(0);
+EVENT_DIMM_SYNDROME(1);
+EVENT_DIMM_SYNDROME(2);
+EVENT_DIMM_SYNDROME(3);
+EVENT_DIMM_SYNDROME(4);
+EVENT_DIMM_SYNDROME(5);
+EVENT_DIMM_SYNDROME(6);
+EVENT_DIMM_SYNDROME(7);
+EVENT_DIMM_SYNDROME(8);
+EVENT_DIMM_SYNDROME(9);
+EVENT_DIMM_SYNDROME(10);
+EVENT_DIMM_SYNDROME(11);
+EVENT_DIMM_SYNDROME(12);
+EVENT_DIMM_SYNDROME(13);
+EVENT_DIMM_SYNDROME(14);
+EVENT_DIMM_SYNDROME(15);
 
 static struct attribute *smpro_errmon_attrs[] = {
 	&dev_attr_overflow_core_ce.attr,
@@ -488,6 +553,23 @@ static struct attribute *smpro_errmon_attrs[] = {
 	&dev_attr_event_vrd_warn_fault.attr,
 	&dev_attr_event_vrd_hot.attr,
 	&dev_attr_event_dimm_hot.attr,
+	&dev_attr_event_dimm_2x_refresh.attr,
+	&dev_attr_event_dimm0_syndrome.attr,
+	&dev_attr_event_dimm1_syndrome.attr,
+	&dev_attr_event_dimm2_syndrome.attr,
+	&dev_attr_event_dimm3_syndrome.attr,
+	&dev_attr_event_dimm4_syndrome.attr,
+	&dev_attr_event_dimm5_syndrome.attr,
+	&dev_attr_event_dimm6_syndrome.attr,
+	&dev_attr_event_dimm7_syndrome.attr,
+	&dev_attr_event_dimm8_syndrome.attr,
+	&dev_attr_event_dimm9_syndrome.attr,
+	&dev_attr_event_dimm10_syndrome.attr,
+	&dev_attr_event_dimm11_syndrome.attr,
+	&dev_attr_event_dimm12_syndrome.attr,
+	&dev_attr_event_dimm13_syndrome.attr,
+	&dev_attr_event_dimm14_syndrome.attr,
+	&dev_attr_event_dimm15_syndrome.attr,
 	NULL
 };
 
diff --git a/drivers/misc/sram.c b/drivers/misc/sram.c
index f0e7f02605eb..99413310956b 100644
--- a/drivers/misc/sram.c
+++ b/drivers/misc/sram.c
@@ -218,14 +218,9 @@ static int sram_reserve_regions(struct sram_dev *sram, struct resource *res)
 		block->res = child_res;
 		list_add_tail(&block->list, &reserve_list);
 
-		if (of_find_property(child, "export", NULL))
-			block->export = true;
-
-		if (of_find_property(child, "pool", NULL))
-			block->pool = true;
-
-		if (of_find_property(child, "protect-exec", NULL))
-			block->protect_exec = true;
+		block->export = of_property_read_bool(child, "export");
+		block->pool = of_property_read_bool(child, "pool");
+		block->protect_exec = of_property_read_bool(child, "protect-exec");
 
 		if ((block->export || block->pool || block->protect_exec) &&
 		    block->size) {
@@ -381,6 +376,7 @@ static int sram_probe(struct platform_device *pdev)
 	struct sram_dev *sram;
 	int ret;
 	struct resource *res;
+	struct clk *clk;
 
 	config = of_device_get_match_data(&pdev->dev);
 
@@ -409,16 +405,14 @@ static int sram_probe(struct platform_device *pdev)
 			return PTR_ERR(sram->pool);
 	}
 
-	sram->clk = devm_clk_get(sram->dev, NULL);
-	if (IS_ERR(sram->clk))
-		sram->clk = NULL;
-	else
-		clk_prepare_enable(sram->clk);
+	clk = devm_clk_get_optional_enabled(sram->dev, NULL);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
 
 	ret = sram_reserve_regions(sram,
 			platform_get_resource(pdev, IORESOURCE_MEM, 0));
 	if (ret)
-		goto err_disable_clk;
+		return ret;
 
 	platform_set_drvdata(pdev, sram);
 
@@ -436,9 +430,6 @@ static int sram_probe(struct platform_device *pdev)
 
 err_free_partitions:
 	sram_free_partitions(sram);
-err_disable_clk:
-	if (sram->clk)
-		clk_disable_unprepare(sram->clk);
 
 	return ret;
 }
@@ -452,9 +443,6 @@ static int sram_remove(struct platform_device *pdev)
 	if (sram->pool && gen_pool_avail(sram->pool) < gen_pool_size(sram->pool))
 		dev_err(sram->dev, "removed while SRAM allocated\n");
 
-	if (sram->clk)
-		clk_disable_unprepare(sram->clk);
-
 	return 0;
 }
 
diff --git a/drivers/misc/sram.h b/drivers/misc/sram.h
index d2058d8c8f1d..397205b8bf6f 100644
--- a/drivers/misc/sram.h
+++ b/drivers/misc/sram.h
@@ -27,7 +27,6 @@ struct sram_dev {
 	bool no_memory_wc;
 
 	struct gen_pool *pool;
-	struct clk *clk;
 
 	struct sram_partition *partition;
 	u32 partitions;
diff --git a/drivers/misc/uacce/uacce.c b/drivers/misc/uacce/uacce.c
index 07023397afc7..346bd7cf2e94 100644
--- a/drivers/misc/uacce/uacce.c
+++ b/drivers/misc/uacce/uacce.c
@@ -620,7 +620,7 @@ static int __init uacce_init(void)
 {
 	int ret;
 
-	uacce_class = class_create(THIS_MODULE, UACCE_NAME);
+	uacce_class = class_create(UACCE_NAME);
 	if (IS_ERR(uacce_class))
 		return PTR_ERR(uacce_class);
 
diff --git a/drivers/misc/vmw_vmci/vmci_context.c b/drivers/misc/vmw_vmci/vmci_context.c
index 172696abce31..f22b44827e92 100644
--- a/drivers/misc/vmw_vmci/vmci_context.c
+++ b/drivers/misc/vmw_vmci/vmci_context.c
@@ -687,7 +687,7 @@ int vmci_ctx_remove_notification(u32 context_id, u32 remote_cid)
 	spin_unlock(&context->lock);
 
 	if (notifier)
-		kvfree_rcu(notifier);
+		kvfree_rcu_mightsleep(notifier);
 
 	vmci_ctx_put(context);
 
diff --git a/drivers/misc/vmw_vmci/vmci_event.c b/drivers/misc/vmw_vmci/vmci_event.c
index 2100297c94ad..5d7ac07623c2 100644
--- a/drivers/misc/vmw_vmci/vmci_event.c
+++ b/drivers/misc/vmw_vmci/vmci_event.c
@@ -209,7 +209,7 @@ int vmci_event_unsubscribe(u32 sub_id)
 	if (!s)
 		return VMCI_ERROR_NOT_FOUND;
 
-	kvfree_rcu(s);
+	kvfree_rcu_mightsleep(s);
 
 	return VMCI_SUCCESS;
 }
diff --git a/drivers/misc/vmw_vmci/vmci_host.c b/drivers/misc/vmw_vmci/vmci_host.c
index 857b9851402a..abe79f6fd2a7 100644
--- a/drivers/misc/vmw_vmci/vmci_host.c
+++ b/drivers/misc/vmw_vmci/vmci_host.c
@@ -165,10 +165,16 @@ static int vmci_host_close(struct inode *inode, struct file *filp)
 static __poll_t vmci_host_poll(struct file *filp, poll_table *wait)
 {
 	struct vmci_host_dev *vmci_host_dev = filp->private_data;
-	struct vmci_ctx *context = vmci_host_dev->context;
+	struct vmci_ctx *context;
 	__poll_t mask = 0;
 
 	if (vmci_host_dev->ct_type == VMCIOBJ_CONTEXT) {
+		/*
+		 * Read context only if ct_type == VMCIOBJ_CONTEXT to make
+		 * sure that context is initialized
+		 */
+		context = vmci_host_dev->context;
+
 		/* Check for VMCI calls to this VM context. */
 		if (wait)
 			poll_wait(filp, &context->host_context.wait_queue,
diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c
index 672ab90c4b2d..00c33edb9fb9 100644
--- a/drivers/mmc/core/block.c
+++ b/drivers/mmc/core/block.c
@@ -76,8 +76,6 @@ MODULE_ALIAS("mmc:block");
 #define MMC_EXTRACT_INDEX_FROM_ARG(x) ((x & 0x00FF0000) >> 16)
 #define MMC_EXTRACT_VALUE_FROM_ARG(x) ((x & 0x0000FF00) >> 8)
 
-#define mmc_req_rel_wr(req)	((req->cmd_flags & REQ_FUA) && \
-				  (rq_data_dir(req) == WRITE))
 static DEFINE_MUTEX(block_mutex);
 
 /*
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
index 426c7f66b349..3d3e0ca52614 100644
--- a/drivers/mmc/core/core.c
+++ b/drivers/mmc/core/core.c
@@ -2257,6 +2257,11 @@ void mmc_rescan(struct work_struct *work)
 			break;
 	}
 
+	/* A non-removable card should have been detected by now. */
+	if (!mmc_card_is_removable(host) && !host->bus_ops)
+		pr_info("%s: Failed to initialize a non-removable card",
+			mmc_hostname(host));
+
 	/*
 	 * Ignore the command timeout errors observed during
 	 * the card init as those are excepted.
diff --git a/drivers/mmc/core/debugfs.c b/drivers/mmc/core/debugfs.c
index fe6808771bc7..2c97b94aab23 100644
--- a/drivers/mmc/core/debugfs.c
+++ b/drivers/mmc/core/debugfs.c
@@ -246,7 +246,7 @@ DEFINE_DEBUGFS_ATTRIBUTE(mmc_err_state, mmc_err_state_get, NULL, "%llu\n");
 
 static int mmc_err_stats_show(struct seq_file *file, void *data)
 {
-	struct mmc_host *host = (struct mmc_host *)file->private;
+	struct mmc_host *host = file->private;
 	const char *desc[MMC_ERR_MAX] = {
 		[MMC_ERR_CMD_TIMEOUT] = "Command Timeout Occurred",
 		[MMC_ERR_CMD_CRC] = "Command CRC Errors Occurred",
diff --git a/drivers/mmc/core/mmc_test.c b/drivers/mmc/core/mmc_test.c
index 156d34b2ed4d..0f6a563103fd 100644
--- a/drivers/mmc/core/mmc_test.c
+++ b/drivers/mmc/core/mmc_test.c
@@ -3045,7 +3045,7 @@ static LIST_HEAD(mmc_test_file_test);
 
 static int mtf_test_show(struct seq_file *sf, void *data)
 {
-	struct mmc_card *card = (struct mmc_card *)sf->private;
+	struct mmc_card *card = sf->private;
 	struct mmc_test_general_result *gr;
 
 	mutex_lock(&mmc_test_lock);
@@ -3079,8 +3079,8 @@ static int mtf_test_open(struct inode *inode, struct file *file)
 static ssize_t mtf_test_write(struct file *file, const char __user *buf,
 	size_t count, loff_t *pos)
 {
-	struct seq_file *sf = (struct seq_file *)file->private_data;
-	struct mmc_card *card = (struct mmc_card *)sf->private;
+	struct seq_file *sf = file->private_data;
+	struct mmc_card *card = sf->private;
 	struct mmc_test_card *test;
 	long testcase;
 	int ret;
diff --git a/drivers/mmc/core/regulator.c b/drivers/mmc/core/regulator.c
index 609201a467ef..005247a49e51 100644
--- a/drivers/mmc/core/regulator.c
+++ b/drivers/mmc/core/regulator.c
@@ -110,6 +110,9 @@ int mmc_regulator_set_ocr(struct mmc_host *mmc,
 	int			result = 0;
 	int			min_uV, max_uV;
 
+	if (IS_ERR(supply))
+		return 0;
+
 	if (vdd_bit) {
 		mmc_ocrbitnum_to_vdd(vdd_bit, &min_uV, &max_uV);
 
@@ -271,3 +274,44 @@ int mmc_regulator_get_supply(struct mmc_host *mmc)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mmc_regulator_get_supply);
+
+/**
+ * mmc_regulator_enable_vqmmc - enable VQMMC regulator for a host
+ * @mmc: the host to regulate
+ *
+ * Returns 0 or errno. Enables the regulator for vqmmc.
+ * Keeps track of the enable status for ensuring that calls to
+ * regulator_enable/disable are balanced.
+ */
+int mmc_regulator_enable_vqmmc(struct mmc_host *mmc)
+{
+	int ret = 0;
+
+	if (!IS_ERR(mmc->supply.vqmmc) && !mmc->vqmmc_enabled) {
+		ret = regulator_enable(mmc->supply.vqmmc);
+		if (ret < 0)
+			dev_err(mmc_dev(mmc), "enabling vqmmc regulator failed\n");
+		else
+			mmc->vqmmc_enabled = true;
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(mmc_regulator_enable_vqmmc);
+
+/**
+ * mmc_regulator_disable_vqmmc - disable VQMMC regulator for a host
+ * @mmc: the host to regulate
+ *
+ * Returns 0 or errno. Disables the regulator for vqmmc.
+ * Keeps track of the enable status for ensuring that calls to
+ * regulator_enable/disable are balanced.
+ */
+void mmc_regulator_disable_vqmmc(struct mmc_host *mmc)
+{
+	if (!IS_ERR(mmc->supply.vqmmc) && mmc->vqmmc_enabled) {
+		regulator_disable(mmc->supply.vqmmc);
+		mmc->vqmmc_enabled = false;
+	}
+}
+EXPORT_SYMBOL_GPL(mmc_regulator_disable_vqmmc);
diff --git a/drivers/mmc/core/sdio_uart.c b/drivers/mmc/core/sdio_uart.c
index 50536fe59f1a..aa659758563f 100644
--- a/drivers/mmc/core/sdio_uart.c
+++ b/drivers/mmc/core/sdio_uart.c
@@ -478,13 +478,13 @@ static void sdio_uart_check_modem_status(struct sdio_uart_port *port)
 			int cts = (status & UART_MSR_CTS);
 			if (tty->hw_stopped) {
 				if (cts) {
-					tty->hw_stopped = 0;
+					tty->hw_stopped = false;
 					sdio_uart_start_tx(port);
 					tty_wakeup(tty);
 				}
 			} else {
 				if (!cts) {
-					tty->hw_stopped = 1;
+					tty->hw_stopped = true;
 					sdio_uart_stop_tx(port);
 				}
 			}
@@ -633,7 +633,7 @@ static int sdio_uart_activate(struct tty_port *tport, struct tty_struct *tty)
 
 	if (C_CRTSCTS(tty))
 		if (!(sdio_uart_get_mctrl(port) & TIOCM_CTS))
-			tty->hw_stopped = 1;
+			tty->hw_stopped = true;
 
 	clear_bit(TTY_IO_ERROR, &tty->flags);
 
@@ -882,14 +882,14 @@ static void sdio_uart_set_termios(struct tty_struct *tty,
 
 	/* Handle turning off CRTSCTS */
 	if ((old_termios->c_cflag & CRTSCTS) && !(cflag & CRTSCTS)) {
-		tty->hw_stopped = 0;
+		tty->hw_stopped = false;
 		sdio_uart_start_tx(port);
 	}
 
 	/* Handle turning on CRTSCTS */
 	if (!(old_termios->c_cflag & CRTSCTS) && (cflag & CRTSCTS)) {
 		if (!(sdio_uart_get_mctrl(port) & TIOCM_CTS)) {
-			tty->hw_stopped = 1;
+			tty->hw_stopped = true;
 			sdio_uart_stop_tx(port);
 		}
 	}
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 4745fe217ade..9f793892123c 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -255,6 +255,7 @@ config MMC_SDHCI_CADENCE
 	tristate "SDHCI support for the Cadence SD/SDIO/eMMC controller"
 	depends on MMC_SDHCI_PLTFM
 	depends on OF
+	select MMC_SDHCI_IO_ACCESSORS
 	help
 	  This selects the Cadence SD/SDIO/eMMC driver.
 
diff --git a/drivers/mmc/host/dw_mmc-pltfm.c b/drivers/mmc/host/dw_mmc-pltfm.c
index 13e55cff8237..48b7da2b86b3 100644
--- a/drivers/mmc/host/dw_mmc-pltfm.c
+++ b/drivers/mmc/host/dw_mmc-pltfm.c
@@ -46,8 +46,7 @@ int dw_mci_pltfm_register(struct platform_device *pdev,
 	host->irq_flags = 0;
 	host->pdata = pdev->dev.platform_data;
 
-	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	host->regs = devm_ioremap_resource(&pdev->dev, regs);
+	host->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &regs);
 	if (IS_ERR(host->regs))
 		return PTR_ERR(host->regs);
 
diff --git a/drivers/mmc/host/jz4740_mmc.c b/drivers/mmc/host/jz4740_mmc.c
index 698450afa7bb..1846a05210e3 100644
--- a/drivers/mmc/host/jz4740_mmc.c
+++ b/drivers/mmc/host/jz4740_mmc.c
@@ -1079,8 +1079,7 @@ static int jz4740_mmc_probe(struct platform_device* pdev)
 		goto err_free_host;
 	}
 
-	host->mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	host->base = devm_ioremap_resource(&pdev->dev, host->mem_res);
+	host->base = devm_platform_get_and_ioremap_resource(pdev, 0, &host->mem_res);
 	if (IS_ERR(host->base)) {
 		ret = PTR_ERR(host->base);
 		goto err_free_host;
diff --git a/drivers/mmc/host/meson-gx-mmc.c b/drivers/mmc/host/meson-gx-mmc.c
index 2b963a81c2ad..b8514d9d5e73 100644
--- a/drivers/mmc/host/meson-gx-mmc.c
+++ b/drivers/mmc/host/meson-gx-mmc.c
@@ -174,7 +174,6 @@ struct meson_host {
 
 	int irq;
 
-	bool vqmmc_enabled;
 	bool needs_pre_post_req;
 
 	spinlock_t lock;
@@ -604,32 +603,18 @@ static void meson_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	 */
 	switch (ios->power_mode) {
 	case MMC_POWER_OFF:
-		if (!IS_ERR(mmc->supply.vmmc))
-			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
-
-		if (!IS_ERR(mmc->supply.vqmmc) && host->vqmmc_enabled) {
-			regulator_disable(mmc->supply.vqmmc);
-			host->vqmmc_enabled = false;
-		}
+		mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
+		mmc_regulator_disable_vqmmc(mmc);
 
 		break;
 
 	case MMC_POWER_UP:
-		if (!IS_ERR(mmc->supply.vmmc))
-			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, ios->vdd);
+		mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, ios->vdd);
 
 		break;
 
 	case MMC_POWER_ON:
-		if (!IS_ERR(mmc->supply.vqmmc) && !host->vqmmc_enabled) {
-			int ret = regulator_enable(mmc->supply.vqmmc);
-
-			if (ret < 0)
-				dev_err(host->dev,
-					"failed to enable vqmmc regulator\n");
-			else
-				host->vqmmc_enabled = true;
-		}
+		mmc_regulator_enable_vqmmc(mmc);
 
 		break;
 	}
@@ -1181,7 +1166,6 @@ static int meson_mmc_probe(struct platform_device *pdev)
 					"amlogic,dram-access-quirk");
 
 	/* Get regulators and the supported OCR mask */
-	host->vqmmc_enabled = false;
 	ret = mmc_regulator_get_supply(mmc);
 	if (ret)
 		return ret;
diff --git a/drivers/mmc/host/mmci.c b/drivers/mmc/host/mmci.c
index b9e5dfe74e5c..f2b2e8b0574e 100644
--- a/drivers/mmc/host/mmci.c
+++ b/drivers/mmc/host/mmci.c
@@ -1962,28 +1962,28 @@ static int mmci_of_parse(struct device_node *np, struct mmc_host *mmc)
 	if (ret)
 		return ret;
 
-	if (of_get_property(np, "st,sig-dir-dat0", NULL))
+	if (of_property_read_bool(np, "st,sig-dir-dat0"))
 		host->pwr_reg_add |= MCI_ST_DATA0DIREN;
-	if (of_get_property(np, "st,sig-dir-dat2", NULL))
+	if (of_property_read_bool(np, "st,sig-dir-dat2"))
 		host->pwr_reg_add |= MCI_ST_DATA2DIREN;
-	if (of_get_property(np, "st,sig-dir-dat31", NULL))
+	if (of_property_read_bool(np, "st,sig-dir-dat31"))
 		host->pwr_reg_add |= MCI_ST_DATA31DIREN;
-	if (of_get_property(np, "st,sig-dir-dat74", NULL))
+	if (of_property_read_bool(np, "st,sig-dir-dat74"))
 		host->pwr_reg_add |= MCI_ST_DATA74DIREN;
-	if (of_get_property(np, "st,sig-dir-cmd", NULL))
+	if (of_property_read_bool(np, "st,sig-dir-cmd"))
 		host->pwr_reg_add |= MCI_ST_CMDDIREN;
-	if (of_get_property(np, "st,sig-pin-fbclk", NULL))
+	if (of_property_read_bool(np, "st,sig-pin-fbclk"))
 		host->pwr_reg_add |= MCI_ST_FBCLKEN;
-	if (of_get_property(np, "st,sig-dir", NULL))
+	if (of_property_read_bool(np, "st,sig-dir"))
 		host->pwr_reg_add |= MCI_STM32_DIRPOL;
-	if (of_get_property(np, "st,neg-edge", NULL))
+	if (of_property_read_bool(np, "st,neg-edge"))
 		host->clk_reg_add |= MCI_STM32_CLK_NEGEDGE;
-	if (of_get_property(np, "st,use-ckin", NULL))
+	if (of_property_read_bool(np, "st,use-ckin"))
 		mmci_probe_level_translator(mmc);
 
-	if (of_get_property(np, "mmc-cap-mmc-highspeed", NULL))
+	if (of_property_read_bool(np, "mmc-cap-mmc-highspeed"))
 		mmc->caps |= MMC_CAP_MMC_HIGHSPEED;
-	if (of_get_property(np, "mmc-cap-sd-highspeed", NULL))
+	if (of_property_read_bool(np, "mmc-cap-sd-highspeed"))
 		mmc->caps |= MMC_CAP_SD_HIGHSPEED;
 
 	return 0;
diff --git a/drivers/mmc/host/omap.c b/drivers/mmc/host/omap.c
index 57d39283924d..ce78edfb402b 100644
--- a/drivers/mmc/host/omap.c
+++ b/drivers/mmc/host/omap.c
@@ -1345,8 +1345,7 @@ static int mmc_omap_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return -ENXIO;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	host->virt_base = devm_ioremap_resource(&pdev->dev, res);
+	host->virt_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(host->virt_base))
 		return PTR_ERR(host->virt_base);
 
diff --git a/drivers/mmc/host/omap_hsmmc.c b/drivers/mmc/host/omap_hsmmc.c
index 4bd744755205..517dde777413 100644
--- a/drivers/mmc/host/omap_hsmmc.c
+++ b/drivers/mmc/host/omap_hsmmc.c
@@ -1736,18 +1736,18 @@ static struct omap_hsmmc_platform_data *of_get_hsmmc_pdata(struct device *dev)
 	if (legacy && legacy->name)
 		pdata->name = legacy->name;
 
-	if (of_find_property(np, "ti,dual-volt", NULL))
+	if (of_property_read_bool(np, "ti,dual-volt"))
 		pdata->controller_flags |= OMAP_HSMMC_SUPPORTS_DUAL_VOLT;
 
-	if (of_find_property(np, "ti,non-removable", NULL)) {
+	if (of_property_read_bool(np, "ti,non-removable")) {
 		pdata->nonremovable = true;
 		pdata->no_regulator_off_init = true;
 	}
 
-	if (of_find_property(np, "ti,needs-special-reset", NULL))
+	if (of_property_read_bool(np, "ti,needs-special-reset"))
 		pdata->features |= HSMMC_HAS_UPDATED_RESET;
 
-	if (of_find_property(np, "ti,needs-special-hs-handling", NULL))
+	if (of_property_read_bool(np, "ti,needs-special-hs-handling"))
 		pdata->features |= HSMMC_HAS_HSPE_SUPPORT;
 
 	return pdata;
diff --git a/drivers/mmc/host/owl-mmc.c b/drivers/mmc/host/owl-mmc.c
index 3dc143b03939..6f9d31a886ba 100644
--- a/drivers/mmc/host/owl-mmc.c
+++ b/drivers/mmc/host/owl-mmc.c
@@ -578,8 +578,7 @@ static int owl_mmc_probe(struct platform_device *pdev)
 	owl_host->mmc = mmc;
 	spin_lock_init(&owl_host->lock);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	owl_host->base = devm_ioremap_resource(&pdev->dev, res);
+	owl_host->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(owl_host->base)) {
 		ret = PTR_ERR(owl_host->base);
 		goto err_free_host;
diff --git a/drivers/mmc/host/renesas_sdhi_internal_dmac.c b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
index f38003f6b1ca..9ab813903b2c 100644
--- a/drivers/mmc/host/renesas_sdhi_internal_dmac.c
+++ b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
@@ -72,11 +72,10 @@ enum renesas_sdhi_dma_cookie {
 
 static unsigned long global_flags;
 /*
- * Workaround for avoiding to use RX DMAC by multiple channels.
- * On R-Car H3 ES1.* and M3-W ES1.0, when multiple SDHI channels use
- * RX DMAC simultaneously, sometimes hundreds of bytes data are not
- * stored into the system memory even if the DMAC interrupt happened.
- * So, this driver then uses one RX DMAC channel only.
+ * Workaround for avoiding to use RX DMAC by multiple channels. On R-Car M3-W
+ * ES1.0, when multiple SDHI channels use RX DMAC simultaneously, sometimes
+ * hundreds of data bytes are not stored into the system memory even if the
+ * DMAC interrupt happened. So, this driver then uses one RX DMAC channel only.
  */
 #define SDHI_INTERNAL_DMAC_RX_IN_USE	0
 
@@ -222,7 +221,6 @@ static const struct renesas_sdhi_quirks sdhi_quirks_r9a09g011 = {
  */
 static const struct soc_device_attribute sdhi_quirks_match[]  = {
 	{ .soc_id = "r8a774a1", .revision = "ES1.[012]", .data = &sdhi_quirks_4tap_nohs400 },
-	{ .soc_id = "r8a7795", .revision = "ES1.*", .data = &sdhi_quirks_4tap_nohs400_one_rx },
 	{ .soc_id = "r8a7795", .revision = "ES2.0", .data = &sdhi_quirks_4tap },
 	{ .soc_id = "r8a7796", .revision = "ES1.0", .data = &sdhi_quirks_4tap_nohs400_one_rx },
 	{ .soc_id = "r8a7796", .revision = "ES1.[12]", .data = &sdhi_quirks_4tap_nohs400 },
diff --git a/drivers/mmc/host/sdhci-cadence.c b/drivers/mmc/host/sdhci-cadence.c
index 6f2de54a5987..b24aa27da50c 100644
--- a/drivers/mmc/host/sdhci-cadence.c
+++ b/drivers/mmc/host/sdhci-cadence.c
@@ -12,6 +12,7 @@
 #include <linux/mmc/mmc.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/reset.h>
 
 #include "sdhci-pltfm.h"
 
@@ -66,7 +67,11 @@ struct sdhci_cdns_phy_param {
 
 struct sdhci_cdns_priv {
 	void __iomem *hrs_addr;
+	void __iomem *ctl_addr;	/* write control */
+	spinlock_t wrlock;	/* write lock */
 	bool enhanced_strobe;
+	void (*priv_writel)(struct sdhci_cdns_priv *priv, u32 val, void __iomem *reg);
+	struct reset_control *rst_hw;
 	unsigned int nr_phy_params;
 	struct sdhci_cdns_phy_param phy_params[];
 };
@@ -76,6 +81,11 @@ struct sdhci_cdns_phy_cfg {
 	u8 addr;
 };
 
+struct sdhci_cdns_drv_data {
+	int (*init)(struct platform_device *pdev);
+	const struct sdhci_pltfm_data pltfm_data;
+};
+
 static const struct sdhci_cdns_phy_cfg sdhci_cdns_phy_cfgs[] = {
 	{ "cdns,phy-input-delay-sd-highspeed", SDHCI_CDNS_PHY_DLY_SD_HS, },
 	{ "cdns,phy-input-delay-legacy", SDHCI_CDNS_PHY_DLY_SD_DEFAULT, },
@@ -90,6 +100,12 @@ static const struct sdhci_cdns_phy_cfg sdhci_cdns_phy_cfgs[] = {
 	{ "cdns,phy-dll-delay-strobe", SDHCI_CDNS_PHY_DLY_STROBE, },
 };
 
+static inline void cdns_writel(struct sdhci_cdns_priv *priv, u32 val,
+			       void __iomem *reg)
+{
+	writel(val, reg);
+}
+
 static int sdhci_cdns_write_phy_reg(struct sdhci_cdns_priv *priv,
 				    u8 addr, u8 data)
 {
@@ -104,17 +120,17 @@ static int sdhci_cdns_write_phy_reg(struct sdhci_cdns_priv *priv,
 
 	tmp = FIELD_PREP(SDHCI_CDNS_HRS04_WDATA, data) |
 	      FIELD_PREP(SDHCI_CDNS_HRS04_ADDR, addr);
-	writel(tmp, reg);
+	priv->priv_writel(priv, tmp, reg);
 
 	tmp |= SDHCI_CDNS_HRS04_WR;
-	writel(tmp, reg);
+	priv->priv_writel(priv, tmp, reg);
 
 	ret = readl_poll_timeout(reg, tmp, tmp & SDHCI_CDNS_HRS04_ACK, 0, 10);
 	if (ret)
 		return ret;
 
 	tmp &= ~SDHCI_CDNS_HRS04_WR;
-	writel(tmp, reg);
+	priv->priv_writel(priv, tmp, reg);
 
 	ret = readl_poll_timeout(reg, tmp, !(tmp & SDHCI_CDNS_HRS04_ACK),
 				 0, 10);
@@ -191,7 +207,7 @@ static void sdhci_cdns_set_emmc_mode(struct sdhci_cdns_priv *priv, u32 mode)
 	tmp = readl(priv->hrs_addr + SDHCI_CDNS_HRS06);
 	tmp &= ~SDHCI_CDNS_HRS06_MODE;
 	tmp |= FIELD_PREP(SDHCI_CDNS_HRS06_MODE, mode);
-	writel(tmp, priv->hrs_addr + SDHCI_CDNS_HRS06);
+	priv->priv_writel(priv, tmp, priv->hrs_addr + SDHCI_CDNS_HRS06);
 }
 
 static u32 sdhci_cdns_get_emmc_mode(struct sdhci_cdns_priv *priv)
@@ -223,7 +239,7 @@ static int sdhci_cdns_set_tune_val(struct sdhci_host *host, unsigned int val)
 	 */
 	for (i = 0; i < 2; i++) {
 		tmp |= SDHCI_CDNS_HRS06_TUNE_UP;
-		writel(tmp, reg);
+		priv->priv_writel(priv, tmp, reg);
 
 		ret = readl_poll_timeout(reg, tmp,
 					 !(tmp & SDHCI_CDNS_HRS06_TUNE_UP),
@@ -309,6 +325,91 @@ static void sdhci_cdns_set_uhs_signaling(struct sdhci_host *host,
 		sdhci_set_uhs_signaling(host, timing);
 }
 
+/* Elba control register bits [6:3] are byte-lane enables */
+#define ELBA_BYTE_ENABLE_MASK(x)	((x) << 3)
+
+/*
+ * The Pensando Elba SoC explicitly controls byte-lane enabling on writes
+ * which includes writes to the HRS registers.  The write lock (wrlock)
+ * is used to ensure byte-lane enable, using write control (ctl_addr),
+ * occurs before the data write.
+ */
+static void elba_priv_writel(struct sdhci_cdns_priv *priv, u32 val,
+			     void __iomem *reg)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->wrlock, flags);
+	writel(GENMASK(7, 3), priv->ctl_addr);
+	writel(val, reg);
+	spin_unlock_irqrestore(&priv->wrlock, flags);
+}
+
+static void elba_write_l(struct sdhci_host *host, u32 val, int reg)
+{
+	elba_priv_writel(sdhci_cdns_priv(host), val, host->ioaddr + reg);
+}
+
+static void elba_write_w(struct sdhci_host *host, u16 val, int reg)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	u32 shift = reg & GENMASK(1, 0);
+	unsigned long flags;
+	u32 byte_enables;
+
+	byte_enables = GENMASK(1, 0) << shift;
+	spin_lock_irqsave(&priv->wrlock, flags);
+	writel(ELBA_BYTE_ENABLE_MASK(byte_enables), priv->ctl_addr);
+	writew(val, host->ioaddr + reg);
+	spin_unlock_irqrestore(&priv->wrlock, flags);
+}
+
+static void elba_write_b(struct sdhci_host *host, u8 val, int reg)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	u32 shift = reg & GENMASK(1, 0);
+	unsigned long flags;
+	u32 byte_enables;
+
+	byte_enables = BIT(0) << shift;
+	spin_lock_irqsave(&priv->wrlock, flags);
+	writel(ELBA_BYTE_ENABLE_MASK(byte_enables), priv->ctl_addr);
+	writeb(val, host->ioaddr + reg);
+	spin_unlock_irqrestore(&priv->wrlock, flags);
+}
+
+static const struct sdhci_ops sdhci_elba_ops = {
+	.write_l = elba_write_l,
+	.write_w = elba_write_w,
+	.write_b = elba_write_b,
+	.set_clock = sdhci_set_clock,
+	.get_timeout_clock = sdhci_cdns_get_timeout_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = sdhci_reset,
+	.set_uhs_signaling = sdhci_cdns_set_uhs_signaling,
+};
+
+static int elba_drv_init(struct platform_device *pdev)
+{
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	void __iomem *ioaddr;
+
+	host->mmc->caps |= MMC_CAP_1_8V_DDR | MMC_CAP_8_BIT_DATA;
+	spin_lock_init(&priv->wrlock);
+
+	/* Byte-lane control register */
+	ioaddr = devm_platform_ioremap_resource(pdev, 1);
+	if (IS_ERR(ioaddr))
+		return PTR_ERR(ioaddr);
+
+	priv->ctl_addr = ioaddr;
+	priv->priv_writel = elba_priv_writel;
+	writel(ELBA_BYTE_ENABLE_MASK(0xf), priv->ctl_addr);
+
+	return 0;
+}
+
 static const struct sdhci_ops sdhci_cdns_ops = {
 	.set_clock = sdhci_set_clock,
 	.get_timeout_clock = sdhci_cdns_get_timeout_clock,
@@ -318,13 +419,24 @@ static const struct sdhci_ops sdhci_cdns_ops = {
 	.set_uhs_signaling = sdhci_cdns_set_uhs_signaling,
 };
 
-static const struct sdhci_pltfm_data sdhci_cdns_uniphier_pltfm_data = {
-	.ops = &sdhci_cdns_ops,
-	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+static const struct sdhci_cdns_drv_data sdhci_cdns_uniphier_drv_data = {
+	.pltfm_data = {
+		.ops = &sdhci_cdns_ops,
+		.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+	},
 };
 
-static const struct sdhci_pltfm_data sdhci_cdns_pltfm_data = {
-	.ops = &sdhci_cdns_ops,
+static const struct sdhci_cdns_drv_data sdhci_elba_drv_data = {
+	.init = elba_drv_init,
+	.pltfm_data = {
+		.ops = &sdhci_elba_ops,
+	},
+};
+
+static const struct sdhci_cdns_drv_data sdhci_cdns_drv_data = {
+	.pltfm_data = {
+		.ops = &sdhci_cdns_ops,
+	},
 };
 
 static void sdhci_cdns_hs400_enhanced_strobe(struct mmc_host *mmc,
@@ -347,10 +459,26 @@ static void sdhci_cdns_hs400_enhanced_strobe(struct mmc_host *mmc,
 					 SDHCI_CDNS_HRS06_MODE_MMC_HS400);
 }
 
+static void sdhci_cdns_mmc_hw_reset(struct mmc_host *mmc)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+
+	dev_dbg(mmc_dev(host->mmc), "emmc hardware reset\n");
+
+	reset_control_assert(priv->rst_hw);
+	/* For eMMC, minimum is 1us but give it 3us for good measure */
+	udelay(3);
+
+	reset_control_deassert(priv->rst_hw);
+	/* For eMMC, minimum is 200us but give it 300us for good measure */
+	usleep_range(300, 1000);
+}
+
 static int sdhci_cdns_probe(struct platform_device *pdev)
 {
 	struct sdhci_host *host;
-	const struct sdhci_pltfm_data *data;
+	const struct sdhci_cdns_drv_data *data;
 	struct sdhci_pltfm_host *pltfm_host;
 	struct sdhci_cdns_priv *priv;
 	struct clk *clk;
@@ -369,10 +497,10 @@ static int sdhci_cdns_probe(struct platform_device *pdev)
 
 	data = of_device_get_match_data(dev);
 	if (!data)
-		data = &sdhci_cdns_pltfm_data;
+		data = &sdhci_cdns_drv_data;
 
 	nr_phy_params = sdhci_cdns_phy_param_count(dev->of_node);
-	host = sdhci_pltfm_init(pdev, data,
+	host = sdhci_pltfm_init(pdev, &data->pltfm_data,
 				struct_size(priv, phy_params, nr_phy_params));
 	if (IS_ERR(host)) {
 		ret = PTR_ERR(host);
@@ -386,9 +514,15 @@ static int sdhci_cdns_probe(struct platform_device *pdev)
 	priv->nr_phy_params = nr_phy_params;
 	priv->hrs_addr = host->ioaddr;
 	priv->enhanced_strobe = false;
+	priv->priv_writel = cdns_writel;
 	host->ioaddr += SDHCI_CDNS_SRS_BASE;
 	host->mmc_host_ops.hs400_enhanced_strobe =
 				sdhci_cdns_hs400_enhanced_strobe;
+	if (data->init) {
+		ret = data->init(pdev);
+		if (ret)
+			goto free;
+	}
 	sdhci_enable_v4_mode(host);
 	__sdhci_read_caps(host, &version, NULL, NULL);
 
@@ -404,6 +538,15 @@ static int sdhci_cdns_probe(struct platform_device *pdev)
 	if (ret)
 		goto free;
 
+	if (host->mmc->caps & MMC_CAP_HW_RESET) {
+		priv->rst_hw = devm_reset_control_get_optional_exclusive(dev, NULL);
+		if (IS_ERR(priv->rst_hw))
+			return dev_err_probe(mmc_dev(host->mmc), PTR_ERR(priv->rst_hw),
+					     "reset controller error\n");
+		if (priv->rst_hw)
+			host->mmc_host_ops.card_hw_reset = sdhci_cdns_mmc_hw_reset;
+	}
+
 	ret = sdhci_add_host(host);
 	if (ret)
 		goto free;
@@ -453,7 +596,11 @@ static const struct dev_pm_ops sdhci_cdns_pm_ops = {
 static const struct of_device_id sdhci_cdns_match[] = {
 	{
 		.compatible = "socionext,uniphier-sd4hc",
-		.data = &sdhci_cdns_uniphier_pltfm_data,
+		.data = &sdhci_cdns_uniphier_drv_data,
+	},
+	{
+		.compatible = "amd,pensando-elba-sd4hc",
+		.data = &sdhci_elba_drv_data,
 	},
 	{ .compatible = "cdns,sd4hc" },
 	{ /* sentinel */ }
diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c
index 58f042fdd4f4..d7c0c0b9e26c 100644
--- a/drivers/mmc/host/sdhci-esdhc-imx.c
+++ b/drivers/mmc/host/sdhci-esdhc-imx.c
@@ -1597,7 +1597,7 @@ sdhci_esdhc_imx_probe_dt(struct platform_device *pdev,
 	struct esdhc_platform_data *boarddata = &imx_data->boarddata;
 	int ret;
 
-	if (of_get_property(np, "fsl,wp-controller", NULL))
+	if (of_property_read_bool(np, "fsl,wp-controller"))
 		boarddata->wp_type = ESDHC_WP_CONTROLLER;
 
 	/*
@@ -1614,7 +1614,7 @@ sdhci_esdhc_imx_probe_dt(struct platform_device *pdev,
 
 	of_property_read_u32(np, "fsl,strobe-dll-delay-target",
 				&boarddata->strobe_dll_delay_target);
-	if (of_find_property(np, "no-1-8-v", NULL))
+	if (of_property_read_bool(np, "no-1-8-v"))
 		host->quirks2 |= SDHCI_QUIRK2_NO_1_8_V;
 
 	if (of_property_read_u32(np, "fsl,delay-line", &boarddata->delay_line))
diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c
index 89c431a34c43..294dd605fd2b 100644
--- a/drivers/mmc/host/sdhci-of-arasan.c
+++ b/drivers/mmc/host/sdhci-of-arasan.c
@@ -41,11 +41,41 @@
 #define VENDOR_ENHANCED_STROBE		BIT(0)
 
 #define PHY_CLK_TOO_SLOW_HZ		400000
+#define MIN_PHY_CLK_HZ			50000000
 
 #define SDHCI_ITAPDLY_CHGWIN		0x200
 #define SDHCI_ITAPDLY_ENABLE		0x100
 #define SDHCI_OTAPDLY_ENABLE		0x40
 
+#define PHY_CTRL_REG1			0x270
+#define PHY_CTRL_ITAPDLY_ENA_MASK	BIT(0)
+#define PHY_CTRL_ITAPDLY_SEL_MASK	GENMASK(5, 1)
+#define PHY_CTRL_ITAPDLY_SEL_SHIFT	1
+#define PHY_CTRL_ITAP_CHG_WIN_MASK	BIT(6)
+#define PHY_CTRL_OTAPDLY_ENA_MASK	BIT(8)
+#define PHY_CTRL_OTAPDLY_SEL_MASK	GENMASK(15, 12)
+#define PHY_CTRL_OTAPDLY_SEL_SHIFT	12
+#define PHY_CTRL_STRB_SEL_MASK		GENMASK(23, 16)
+#define PHY_CTRL_STRB_SEL_SHIFT		16
+#define PHY_CTRL_TEST_CTRL_MASK		GENMASK(31, 24)
+
+#define PHY_CTRL_REG2			0x274
+#define PHY_CTRL_EN_DLL_MASK		BIT(0)
+#define PHY_CTRL_DLL_RDY_MASK		BIT(1)
+#define PHY_CTRL_FREQ_SEL_MASK		GENMASK(6, 4)
+#define PHY_CTRL_FREQ_SEL_SHIFT		4
+#define PHY_CTRL_SEL_DLY_TX_MASK	BIT(16)
+#define PHY_CTRL_SEL_DLY_RX_MASK	BIT(17)
+#define FREQSEL_200M_170M		0x0
+#define FREQSEL_170M_140M	        0x1
+#define FREQSEL_140M_110M	        0x2
+#define FREQSEL_110M_80M	        0x3
+#define FREQSEL_80M_50M			0x4
+#define FREQSEL_275M_250M		0x5
+#define FREQSEL_250M_225M		0x6
+#define FREQSEL_225M_200M		0x7
+#define PHY_DLL_TIMEOUT_MS		100
+
 /* Default settings for ZynqMP Clock Phases */
 #define ZYNQMP_ICLK_PHASE {0, 63, 63, 0, 63,  0,   0, 183, 54,  0, 0}
 #define ZYNQMP_OCLK_PHASE {0, 72, 60, 0, 60, 72, 135, 48, 72, 135, 0}
@@ -53,6 +83,11 @@
 #define VERSAL_ICLK_PHASE {0, 132, 132, 0, 132, 0, 0, 162, 90, 0, 0}
 #define VERSAL_OCLK_PHASE {0,  60, 48, 0, 48, 72, 90, 36, 60, 90, 0}
 
+#define VERSAL_NET_EMMC_ICLK_PHASE {0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0}
+#define VERSAL_NET_EMMC_OCLK_PHASE {0, 113, 0, 0, 0, 0, 0, 0, 113, 79, 45}
+
+#define VERSAL_NET_PHY_CTRL_STRB90_STRB180_VAL		0X77
+
 /*
  * On some SoCs the syscon area has a feature where the upper 16-bits of
  * each 32-bit register act as a write mask for the lower 16-bits.  This allows
@@ -135,6 +170,7 @@ struct sdhci_arasan_clk_data {
  * @clk_ahb:		Pointer to the AHB clock
  * @phy:		Pointer to the generic phy
  * @is_phy_on:		True if the PHY is on; false if not.
+ * @internal_phy_reg:	True if the PHY is within the Host controller.
  * @has_cqe:		True if controller has command queuing engine.
  * @clk_data:		Struct for the Arasan Controller Clock Data.
  * @clk_ops:		Struct for the Arasan Controller Clock Operations.
@@ -147,6 +183,7 @@ struct sdhci_arasan_data {
 	struct clk	*clk_ahb;
 	struct phy	*phy;
 	bool		is_phy_on;
+	bool		internal_phy_reg;
 
 	bool		has_cqe;
 	struct sdhci_arasan_clk_data clk_data;
@@ -193,13 +230,6 @@ static const struct sdhci_arasan_soc_ctl_map intel_lgm_sdxc_soc_ctl_map = {
 	.hiword_update = false,
 };
 
-static const struct sdhci_arasan_soc_ctl_map thunderbay_soc_ctl_map = {
-	.baseclkfreq = { .reg = 0x0, .width = 8, .shift = 14 },
-	.clockmultiplier = { .reg = 0x4, .width = 8, .shift = 14 },
-	.support64b = { .reg = 0x4, .width = 1, .shift = 24 },
-	.hiword_update = false,
-};
-
 static const struct sdhci_arasan_soc_ctl_map intel_keembay_soc_ctl_map = {
 	.baseclkfreq = { .reg = 0x0, .width = 8, .shift = 14 },
 	.clockmultiplier = { .reg = 0x4, .width = 8, .shift = 14 },
@@ -207,6 +237,61 @@ static const struct sdhci_arasan_soc_ctl_map intel_keembay_soc_ctl_map = {
 	.hiword_update = false,
 };
 
+static void sdhci_arasan_phy_set_delaychain(struct sdhci_host *host, bool enable)
+{
+	u32 reg;
+
+	reg = readl(host->ioaddr + PHY_CTRL_REG2);
+	if (enable)
+		reg |= (PHY_CTRL_SEL_DLY_TX_MASK | PHY_CTRL_SEL_DLY_RX_MASK);
+	else
+		reg &= ~(PHY_CTRL_SEL_DLY_TX_MASK | PHY_CTRL_SEL_DLY_RX_MASK);
+
+	writel(reg, host->ioaddr + PHY_CTRL_REG2);
+}
+
+static int sdhci_arasan_phy_set_dll(struct sdhci_host *host, bool enable)
+{
+	u32 reg;
+
+	reg = readl(host->ioaddr + PHY_CTRL_REG2);
+	if (enable)
+		reg |= PHY_CTRL_EN_DLL_MASK;
+	else
+		reg &= ~PHY_CTRL_EN_DLL_MASK;
+
+	writel(reg, host->ioaddr + PHY_CTRL_REG2);
+
+	if (!enable)
+		return 0;
+
+	return readl_relaxed_poll_timeout(host->ioaddr + PHY_CTRL_REG2, reg,
+					  (reg & PHY_CTRL_DLL_RDY_MASK), 10,
+					  1000 * PHY_DLL_TIMEOUT_MS);
+}
+
+static void sdhci_arasan_phy_dll_set_freq(struct sdhci_host *host, int clock)
+{
+	u32 reg, freq_sel, freq;
+
+	freq = DIV_ROUND_CLOSEST(clock, 1000000);
+	if (freq <= 200 && freq > 170)
+		freq_sel = FREQSEL_200M_170M;
+	else if (freq <= 170 && freq > 140)
+		freq_sel = FREQSEL_170M_140M;
+	else if (freq <= 140 && freq > 110)
+		freq_sel = FREQSEL_140M_110M;
+	else if (freq <= 110 && freq > 80)
+		freq_sel = FREQSEL_110M_80M;
+	else
+		freq_sel = FREQSEL_80M_50M;
+
+	reg = readl(host->ioaddr + PHY_CTRL_REG2);
+	reg &= ~PHY_CTRL_FREQ_SEL_MASK;
+	reg |= (freq_sel << PHY_CTRL_FREQ_SEL_SHIFT);
+	writel(reg, host->ioaddr + PHY_CTRL_REG2);
+}
+
 /**
  * sdhci_arasan_syscon_write - Write to a field in soc_ctl registers
  *
@@ -321,11 +406,24 @@ static void sdhci_arasan_set_clock(struct sdhci_host *host, unsigned int clock)
 	}
 
 	/* Set the Input and Output Clock Phase Delays */
-	if (clk_data->set_clk_delays)
+	if (clk_data->set_clk_delays && clock > PHY_CLK_TOO_SLOW_HZ)
 		clk_data->set_clk_delays(host);
 
+	if (sdhci_arasan->internal_phy_reg && clock >= MIN_PHY_CLK_HZ) {
+		sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
+		sdhci_arasan_phy_set_dll(host, 0);
+		sdhci_arasan_phy_set_delaychain(host, 0);
+		sdhci_arasan_phy_dll_set_freq(host, clock);
+	} else if (sdhci_arasan->internal_phy_reg) {
+		sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
+		sdhci_arasan_phy_set_delaychain(host, 1);
+	}
+
 	sdhci_set_clock(host, clock);
 
+	if (sdhci_arasan->internal_phy_reg && clock >= MIN_PHY_CLK_HZ)
+		sdhci_arasan_phy_set_dll(host, 1);
+
 	if (sdhci_arasan->quirks & SDHCI_ARASAN_QUIRK_CLOCK_UNSTABLE)
 		/*
 		 * Some controllers immediately report SDHCI_CLOCK_INT_STABLE
@@ -465,15 +563,6 @@ static const struct sdhci_pltfm_data sdhci_arasan_cqe_pdata = {
 			SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN,
 };
 
-static const struct sdhci_pltfm_data sdhci_arasan_thunderbay_pdata = {
-	.ops = &sdhci_arasan_cqe_ops,
-	.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN | SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC,
-	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
-		SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN |
-		SDHCI_QUIRK2_STOP_WITH_TC |
-		SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400,
-};
-
 #ifdef CONFIG_PM_SLEEP
 /**
  * sdhci_arasan_suspend - Suspend method for the driver
@@ -897,6 +986,101 @@ static const struct clk_ops versal_sampleclk_ops = {
 	.set_phase = sdhci_versal_sampleclk_set_phase,
 };
 
+static int sdhci_versal_net_emmc_sdcardclk_set_phase(struct clk_hw *hw, int degrees)
+{
+	struct sdhci_arasan_clk_data *clk_data =
+		container_of(hw, struct sdhci_arasan_clk_data, sdcardclk_hw);
+	struct sdhci_arasan_data *sdhci_arasan =
+		container_of(clk_data, struct sdhci_arasan_data, clk_data);
+	struct sdhci_host *host = sdhci_arasan->host;
+	u8 tap_delay, tap_max = 0;
+
+	switch (host->timing) {
+	case MMC_TIMING_MMC_HS:
+	case MMC_TIMING_MMC_DDR52:
+		tap_max = 16;
+		break;
+	case MMC_TIMING_MMC_HS200:
+	case MMC_TIMING_MMC_HS400:
+		 /* For 200MHz clock, 32 Taps are available */
+		tap_max = 32;
+		break;
+	default:
+		break;
+	}
+
+	tap_delay = (degrees * tap_max) / 360;
+
+	/* Set the Clock Phase */
+	if (tap_delay) {
+		u32 regval;
+
+		regval = sdhci_readl(host, PHY_CTRL_REG1);
+		regval |= PHY_CTRL_OTAPDLY_ENA_MASK;
+		sdhci_writel(host, regval, PHY_CTRL_REG1);
+		regval &= ~PHY_CTRL_OTAPDLY_SEL_MASK;
+		regval |= tap_delay << PHY_CTRL_OTAPDLY_SEL_SHIFT;
+		sdhci_writel(host, regval, PHY_CTRL_REG1);
+	}
+
+	return 0;
+}
+
+static const struct clk_ops versal_net_sdcardclk_ops = {
+	.recalc_rate = sdhci_arasan_sdcardclk_recalc_rate,
+	.set_phase = sdhci_versal_net_emmc_sdcardclk_set_phase,
+};
+
+static int sdhci_versal_net_emmc_sampleclk_set_phase(struct clk_hw *hw, int degrees)
+{
+	struct sdhci_arasan_clk_data *clk_data =
+		container_of(hw, struct sdhci_arasan_clk_data, sampleclk_hw);
+	struct sdhci_arasan_data *sdhci_arasan =
+		container_of(clk_data, struct sdhci_arasan_data, clk_data);
+	struct sdhci_host *host = sdhci_arasan->host;
+	u8 tap_delay, tap_max = 0;
+	u32 regval;
+
+	switch (host->timing) {
+	case MMC_TIMING_MMC_HS:
+	case MMC_TIMING_MMC_DDR52:
+		tap_max = 32;
+		break;
+	case MMC_TIMING_MMC_HS400:
+		/* Strobe select tap point for strb90 and strb180 */
+		regval = sdhci_readl(host, PHY_CTRL_REG1);
+		regval &= ~PHY_CTRL_STRB_SEL_MASK;
+		regval |= VERSAL_NET_PHY_CTRL_STRB90_STRB180_VAL << PHY_CTRL_STRB_SEL_SHIFT;
+		sdhci_writel(host, regval, PHY_CTRL_REG1);
+		break;
+	default:
+		break;
+	}
+
+	tap_delay = (degrees * tap_max) / 360;
+
+	/* Set the Clock Phase */
+	if (tap_delay) {
+		regval = sdhci_readl(host, PHY_CTRL_REG1);
+		regval |= PHY_CTRL_ITAP_CHG_WIN_MASK;
+		sdhci_writel(host, regval, PHY_CTRL_REG1);
+		regval |= PHY_CTRL_ITAPDLY_ENA_MASK;
+		sdhci_writel(host, regval, PHY_CTRL_REG1);
+		regval &= ~PHY_CTRL_ITAPDLY_SEL_MASK;
+		regval |= tap_delay << PHY_CTRL_ITAPDLY_SEL_SHIFT;
+		sdhci_writel(host, regval, PHY_CTRL_REG1);
+		regval &= ~PHY_CTRL_ITAP_CHG_WIN_MASK;
+		sdhci_writel(host, regval, PHY_CTRL_REG1);
+	}
+
+	return 0;
+}
+
+static const struct clk_ops versal_net_sampleclk_ops = {
+	.recalc_rate = sdhci_arasan_sampleclk_recalc_rate,
+	.set_phase = sdhci_versal_net_emmc_sampleclk_set_phase,
+};
+
 static void arasan_zynqmp_dll_reset(struct sdhci_host *host, u32 deviceid)
 {
 	u16 clk;
@@ -1107,7 +1291,17 @@ static void arasan_dt_parse_clk_phases(struct device *dev,
 			clk_data->clk_phase_out[i] = versal_oclk_phase[i];
 		}
 	}
+	if (of_device_is_compatible(dev->of_node, "xlnx,versal-net-emmc")) {
+		u32 versal_net_iclk_phase[MMC_TIMING_MMC_HS400 + 1] =
+			VERSAL_NET_EMMC_ICLK_PHASE;
+		u32 versal_net_oclk_phase[MMC_TIMING_MMC_HS400 + 1] =
+			VERSAL_NET_EMMC_OCLK_PHASE;
 
+		for (i = 0; i <= MMC_TIMING_MMC_HS400; i++) {
+			clk_data->clk_phase_in[i] = versal_net_iclk_phase[i];
+			clk_data->clk_phase_out[i] = versal_net_oclk_phase[i];
+		}
+	}
 	arasan_dt_read_clk_phase(dev, clk_data, MMC_TIMING_LEGACY,
 				 "clk-phase-legacy");
 	arasan_dt_read_clk_phase(dev, clk_data, MMC_TIMING_MMC_HS,
@@ -1150,12 +1344,6 @@ static struct sdhci_arasan_of_data sdhci_arasan_generic_data = {
 	.clk_ops = &arasan_clk_ops,
 };
 
-static const struct sdhci_arasan_of_data sdhci_arasan_thunderbay_data = {
-	.soc_ctl_map = &thunderbay_soc_ctl_map,
-	.pdata = &sdhci_arasan_thunderbay_pdata,
-	.clk_ops = &arasan_clk_ops,
-};
-
 static const struct sdhci_pltfm_data sdhci_keembay_emmc_pdata = {
 	.ops = &sdhci_arasan_cqe_ops,
 	.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN |
@@ -1225,6 +1413,14 @@ static const struct sdhci_pltfm_data sdhci_arasan_zynqmp_pdata = {
 			SDHCI_QUIRK2_STOP_WITH_TC,
 };
 
+static const struct sdhci_pltfm_data sdhci_arasan_versal_net_pdata = {
+	.ops = &sdhci_arasan_ops,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
+			SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN |
+			SDHCI_QUIRK2_STOP_WITH_TC |
+			SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400,
+};
+
 static const struct sdhci_arasan_clk_ops zynqmp_clk_ops = {
 	.sdcardclk_ops = &zynqmp_sdcardclk_ops,
 	.sampleclk_ops = &zynqmp_sampleclk_ops,
@@ -1245,6 +1441,16 @@ static struct sdhci_arasan_of_data sdhci_arasan_versal_data = {
 	.clk_ops = &versal_clk_ops,
 };
 
+static const struct sdhci_arasan_clk_ops versal_net_clk_ops = {
+	.sdcardclk_ops = &versal_net_sdcardclk_ops,
+	.sampleclk_ops = &versal_net_sampleclk_ops,
+};
+
+static struct sdhci_arasan_of_data sdhci_arasan_versal_net_data = {
+	.pdata = &sdhci_arasan_versal_net_pdata,
+	.clk_ops = &versal_net_clk_ops,
+};
+
 static struct sdhci_arasan_of_data intel_keembay_emmc_data = {
 	.soc_ctl_map = &intel_keembay_soc_ctl_map,
 	.pdata = &sdhci_keembay_emmc_pdata,
@@ -1289,10 +1495,6 @@ static const struct of_device_id sdhci_arasan_of_match[] = {
 		.compatible = "intel,keembay-sdhci-5.1-sdio",
 		.data = &intel_keembay_sdio_data,
 	},
-	{
-		.compatible = "intel,thunderbay-sdhci-5.1",
-		.data = &sdhci_arasan_thunderbay_data,
-	},
 	/* Generic compatible below here */
 	{
 		.compatible = "arasan,sdhci-8.9a",
@@ -1314,6 +1516,10 @@ static const struct of_device_id sdhci_arasan_of_match[] = {
 		.compatible = "xlnx,versal-8.9a",
 		.data = &sdhci_arasan_versal_data,
 	},
+	{
+		.compatible = "xlnx,versal-net-emmc",
+		.data = &sdhci_arasan_versal_net_data,
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, sdhci_arasan_of_match);
@@ -1434,7 +1640,7 @@ static void sdhci_arasan_unregister_sdclk(struct device *dev)
 {
 	struct device_node *np = dev->of_node;
 
-	if (!of_find_property(np, "#clock-cells", NULL))
+	if (!of_property_present(np, "#clock-cells"))
 		return;
 
 	of_clk_del_provider(dev->of_node);
@@ -1630,6 +1836,7 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 	int ret;
 	struct device_node *node;
 	struct clk *clk_xin;
+	struct clk *clk_dll;
 	struct sdhci_host *host;
 	struct sdhci_pltfm_host *pltfm_host;
 	struct device *dev = &pdev->dev;
@@ -1703,6 +1910,12 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 		goto clk_dis_ahb;
 	}
 
+	clk_dll = devm_clk_get_optional_enabled(dev, "gate");
+	if (IS_ERR(clk_dll)) {
+		ret = dev_err_probe(dev, PTR_ERR(clk_dll), "failed to get dll clk\n");
+		goto clk_disable_all;
+	}
+
 	if (of_property_read_bool(np, "xlnx,fails-without-test-cd"))
 		sdhci_arasan->quirks |= SDHCI_ARASAN_QUIRK_FORCE_CDTEST;
 
@@ -1716,8 +1929,7 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 
 	if (of_device_is_compatible(np, "intel,keembay-sdhci-5.1-emmc") ||
 	    of_device_is_compatible(np, "intel,keembay-sdhci-5.1-sd") ||
-	    of_device_is_compatible(np, "intel,keembay-sdhci-5.1-sdio") ||
-	    of_device_is_compatible(np, "intel,thunderbay-sdhci-5.1")) {
+	    of_device_is_compatible(np, "intel,keembay-sdhci-5.1-sdio")) {
 		sdhci_arasan_update_clockmultiplier(host, 0x0);
 		sdhci_arasan_update_support64b(host, 0x0);
 
@@ -1781,6 +1993,9 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 			host->mmc->caps2 |= MMC_CAP2_CQE_DCMD;
 	}
 
+	if (of_device_is_compatible(np, "xlnx,versal-net-emmc"))
+		sdhci_arasan->internal_phy_reg = true;
+
 	ret = sdhci_arasan_add_host(sdhci_arasan);
 	if (ret)
 		goto err_add_host;
diff --git a/drivers/mmc/host/sdhci-of-aspeed.c b/drivers/mmc/host/sdhci-of-aspeed.c
index ba6677bf7372..25b4073f698b 100644
--- a/drivers/mmc/host/sdhci-of-aspeed.c
+++ b/drivers/mmc/host/sdhci-of-aspeed.c
@@ -547,8 +547,7 @@ static int aspeed_sdc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	sdc->res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	sdc->regs = devm_ioremap_resource(&pdev->dev, sdc->res);
+	sdc->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &sdc->res);
 	if (IS_ERR(sdc->regs)) {
 		ret = PTR_ERR(sdc->regs);
 		goto err_clk;
diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c
index d1490469184b..e68cd87998c8 100644
--- a/drivers/mmc/host/sdhci-of-dwcmshc.c
+++ b/drivers/mmc/host/sdhci-of-dwcmshc.c
@@ -126,6 +126,13 @@ static unsigned int dwcmshc_get_max_clock(struct sdhci_host *host)
 		return pltfm_host->clock;
 }
 
+static unsigned int rk35xx_get_max_clock(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+
+	return clk_round_rate(pltfm_host->clk, ULONG_MAX);
+}
+
 static void dwcmshc_check_auto_cmd23(struct mmc_host *mmc,
 				     struct mmc_request *mrq)
 {
@@ -343,7 +350,7 @@ static const struct sdhci_ops sdhci_dwcmshc_rk35xx_ops = {
 	.set_clock		= dwcmshc_rk3568_set_clock,
 	.set_bus_width		= sdhci_set_bus_width,
 	.set_uhs_signaling	= dwcmshc_set_uhs_signaling,
-	.get_max_clock		= sdhci_pltfm_clk_get_max_clock,
+	.get_max_clock		= rk35xx_get_max_clock,
 	.reset			= rk35xx_sdhci_reset,
 	.adma_write_desc	= dwcmshc_adma_write_desc,
 };
diff --git a/drivers/mmc/host/sdhci-of-esdhc.c b/drivers/mmc/host/sdhci-of-esdhc.c
index 4712adac7f7c..48ca1cf15b19 100644
--- a/drivers/mmc/host/sdhci-of-esdhc.c
+++ b/drivers/mmc/host/sdhci-of-esdhc.c
@@ -133,6 +133,7 @@ static u32 esdhc_readl_fixup(struct sdhci_host *host,
 			return ret;
 		}
 	}
+
 	/*
 	 * The DAT[3:0] line signal levels and the CMD line signal level are
 	 * not compatible with standard SDHC register. The line signal levels
@@ -144,6 +145,16 @@ static u32 esdhc_readl_fixup(struct sdhci_host *host,
 		ret = value & 0x000fffff;
 		ret |= (value >> 4) & SDHCI_DATA_LVL_MASK;
 		ret |= (value << 1) & SDHCI_CMD_LVL;
+
+		/*
+		 * Some controllers have unreliable Data Line Active
+		 * bit for commands with busy signal. This affects
+		 * Command Inhibit (data) bit. Just ignore it since
+		 * MMC core driver has already polled card status
+		 * with CMD13 after any command with busy siganl.
+		 */
+		if (esdhc->quirk_ignore_data_inhibit)
+			ret &= ~SDHCI_DATA_INHIBIT;
 		return ret;
 	}
 
@@ -158,19 +169,6 @@ static u32 esdhc_readl_fixup(struct sdhci_host *host,
 		return ret;
 	}
 
-	/*
-	 * Some controllers have unreliable Data Line Active
-	 * bit for commands with busy signal. This affects
-	 * Command Inhibit (data) bit. Just ignore it since
-	 * MMC core driver has already polled card status
-	 * with CMD13 after any command with busy siganl.
-	 */
-	if ((spec_reg == SDHCI_PRESENT_STATE) &&
-	(esdhc->quirk_ignore_data_inhibit == true)) {
-		ret = value & ~SDHCI_DATA_INHIBIT;
-		return ret;
-	}
-
 	ret = value;
 	return ret;
 }
diff --git a/drivers/mmc/host/sdhci-pci-o2micro.c b/drivers/mmc/host/sdhci-pci-o2micro.c
index 98cadff47b2b..620f52ad9667 100644
--- a/drivers/mmc/host/sdhci-pci-o2micro.c
+++ b/drivers/mmc/host/sdhci-pci-o2micro.c
@@ -339,22 +339,24 @@ static int sdhci_o2_execute_tuning(struct mmc_host *mmc, u32 opcode)
 	reg_val &= ~SDHCI_CLOCK_CARD_EN;
 	sdhci_writew(host, reg_val, SDHCI_CLOCK_CONTROL);
 
-	/* UnLock WP */
-	pci_read_config_byte(chip->pdev, O2_SD_LOCK_WP, &scratch_8);
-	scratch_8 &= 0x7f;
-	pci_write_config_byte(chip->pdev, O2_SD_LOCK_WP, scratch_8);
-
-	/* Set pcr 0x354[16] to choose dll clock, and set the default phase */
-	pci_read_config_dword(chip->pdev, O2_SD_OUTPUT_CLK_SOURCE_SWITCH, &reg_val);
-	reg_val &= ~(O2_SD_SEL_DLL | O2_SD_PHASE_MASK);
-	reg_val |= (O2_SD_SEL_DLL | O2_SD_FIX_PHASE);
-	pci_write_config_dword(chip->pdev, O2_SD_OUTPUT_CLK_SOURCE_SWITCH, reg_val);
+	if ((host->timing == MMC_TIMING_MMC_HS200) ||
+		(host->timing == MMC_TIMING_UHS_SDR104)) {
+		/* UnLock WP */
+		pci_read_config_byte(chip->pdev, O2_SD_LOCK_WP, &scratch_8);
+		scratch_8 &= 0x7f;
+		pci_write_config_byte(chip->pdev, O2_SD_LOCK_WP, scratch_8);
 
-	/* Lock WP */
-	pci_read_config_byte(chip->pdev, O2_SD_LOCK_WP, &scratch_8);
-	scratch_8 |= 0x80;
-	pci_write_config_byte(chip->pdev, O2_SD_LOCK_WP, scratch_8);
+		/* Set pcr 0x354[16] to choose dll clock, and set the default phase */
+		pci_read_config_dword(chip->pdev, O2_SD_OUTPUT_CLK_SOURCE_SWITCH, &reg_val);
+		reg_val &= ~(O2_SD_SEL_DLL | O2_SD_PHASE_MASK);
+		reg_val |= (O2_SD_SEL_DLL | O2_SD_FIX_PHASE);
+		pci_write_config_dword(chip->pdev, O2_SD_OUTPUT_CLK_SOURCE_SWITCH, reg_val);
 
+		/* Lock WP */
+		pci_read_config_byte(chip->pdev, O2_SD_LOCK_WP, &scratch_8);
+		scratch_8 |= 0x80;
+		pci_write_config_byte(chip->pdev, O2_SD_LOCK_WP, scratch_8);
+	}
 	/* Start clk */
 	reg_val = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
 	reg_val |= SDHCI_CLOCK_CARD_EN;
diff --git a/drivers/mmc/host/sdhci-pltfm.c b/drivers/mmc/host/sdhci-pltfm.c
index 328b132bbe57..673e750a8490 100644
--- a/drivers/mmc/host/sdhci-pltfm.c
+++ b/drivers/mmc/host/sdhci-pltfm.c
@@ -54,7 +54,6 @@ static bool sdhci_wp_inverted(struct device *dev)
 #endif /* CONFIG_PPC */
 }
 
-#ifdef CONFIG_OF
 static void sdhci_get_compatibility(struct platform_device *pdev)
 {
 	struct sdhci_host *host = platform_get_drvdata(pdev);
@@ -72,9 +71,6 @@ static void sdhci_get_compatibility(struct platform_device *pdev)
 	    of_device_is_compatible(np, "fsl,mpc8536-esdhc"))
 		host->quirks |= SDHCI_QUIRK_BROKEN_TIMEOUT_VAL;
 }
-#else
-void sdhci_get_compatibility(struct platform_device *pdev) {}
-#endif /* CONFIG_OF */
 
 void sdhci_get_property(struct platform_device *pdev)
 {
diff --git a/drivers/mmc/host/sdhci-pxav2.c b/drivers/mmc/host/sdhci-pxav2.c
index fc306eb1f845..91aca8f8d6ef 100644
--- a/drivers/mmc/host/sdhci-pxav2.c
+++ b/drivers/mmc/host/sdhci-pxav2.c
@@ -228,7 +228,7 @@ static struct sdhci_pxa_platdata *pxav2_get_mmc_pdata(struct device *dev)
 	if (!pdata)
 		return NULL;
 
-	if (of_find_property(np, "non-removable", NULL))
+	if (of_property_read_bool(np, "non-removable"))
 		pdata->flags |= PXA_FLAG_CARD_PERMANENT;
 
 	of_property_read_u32(np, "bus-width", &bus_width);
diff --git a/drivers/mmc/host/sdhci-s3c.c b/drivers/mmc/host/sdhci-s3c.c
index 9085f3932443..504015e84308 100644
--- a/drivers/mmc/host/sdhci-s3c.c
+++ b/drivers/mmc/host/sdhci-s3c.c
@@ -437,12 +437,12 @@ static int sdhci_s3c_parse_dt(struct device *dev,
 	pdata->max_width = max_width;
 
 	/* get the card detection method */
-	if (of_get_property(node, "broken-cd", NULL)) {
+	if (of_property_read_bool(node, "broken-cd")) {
 		pdata->cd_type = S3C_SDHCI_CD_NONE;
 		return 0;
 	}
 
-	if (of_get_property(node, "non-removable", NULL)) {
+	if (of_property_read_bool(node, "non-removable")) {
 		pdata->cd_type = S3C_SDHCI_CD_PERMANENT;
 		return 0;
 	}
diff --git a/drivers/mmc/host/sdhci_am654.c b/drivers/mmc/host/sdhci_am654.c
index 89953093e20c..7cdf0f54e3a5 100644
--- a/drivers/mmc/host/sdhci_am654.c
+++ b/drivers/mmc/host/sdhci_am654.c
@@ -85,6 +85,7 @@
 #define DRIVER_STRENGTH_40_OHM	0x4
 
 #define CLOCK_TOO_SLOW_HZ	50000000
+#define SDHCI_AM654_AUTOSUSPEND_DELAY	-1
 
 /* Command Queue Host Controller Interface Base address */
 #define SDHCI_AM654_CQE_BASE_ADDR 0x200
@@ -351,8 +352,6 @@ static void sdhci_am654_write_b(struct sdhci_host *host, u8 val, int reg)
 		 */
 		case MMC_TIMING_SD_HS:
 		case MMC_TIMING_MMC_HS:
-		case MMC_TIMING_UHS_SDR12:
-		case MMC_TIMING_UHS_SDR25:
 			val &= ~SDHCI_CTRL_HISPD;
 		}
 	}
@@ -810,16 +809,10 @@ static int sdhci_am654_probe(struct platform_device *pdev)
 
 	pltfm_host->clk = clk_xin;
 
-	/* Clocks are enabled using pm_runtime */
-	pm_runtime_enable(dev);
-	ret = pm_runtime_resume_and_get(dev);
-	if (ret)
-		goto pm_runtime_disable;
-
 	base = devm_platform_ioremap_resource(pdev, 1);
 	if (IS_ERR(base)) {
 		ret = PTR_ERR(base);
-		goto pm_runtime_put;
+		goto err_pltfm_free;
 	}
 
 	sdhci_am654->base = devm_regmap_init_mmio(dev, base,
@@ -827,31 +820,47 @@ static int sdhci_am654_probe(struct platform_device *pdev)
 	if (IS_ERR(sdhci_am654->base)) {
 		dev_err(dev, "Failed to initialize regmap\n");
 		ret = PTR_ERR(sdhci_am654->base);
-		goto pm_runtime_put;
+		goto err_pltfm_free;
 	}
 
 	ret = sdhci_am654_get_of_property(pdev, sdhci_am654);
 	if (ret)
-		goto pm_runtime_put;
+		goto err_pltfm_free;
 
 	ret = mmc_of_parse(host->mmc);
 	if (ret) {
 		dev_err_probe(dev, ret, "parsing dt failed\n");
-		goto pm_runtime_put;
+		goto err_pltfm_free;
 	}
 
 	host->mmc_host_ops.execute_tuning = sdhci_am654_execute_tuning;
 
+	pm_runtime_get_noresume(dev);
+	ret = pm_runtime_set_active(dev);
+	if (ret)
+		goto pm_put;
+	pm_runtime_enable(dev);
+	ret = clk_prepare_enable(pltfm_host->clk);
+	if (ret)
+		goto pm_disable;
+
 	ret = sdhci_am654_init(host);
 	if (ret)
-		goto pm_runtime_put;
+		goto clk_disable;
 
+	/* Setting up autosuspend */
+	pm_runtime_set_autosuspend_delay(dev, SDHCI_AM654_AUTOSUSPEND_DELAY);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
 	return 0;
 
-pm_runtime_put:
-	pm_runtime_put_sync(dev);
-pm_runtime_disable:
+clk_disable:
+	clk_disable_unprepare(pltfm_host->clk);
+pm_disable:
 	pm_runtime_disable(dev);
+pm_put:
+	pm_runtime_put_noidle(dev);
 err_pltfm_free:
 	sdhci_pltfm_free(pdev);
 	return ret;
@@ -860,23 +869,127 @@ err_pltfm_free:
 static int sdhci_am654_remove(struct platform_device *pdev)
 {
 	struct sdhci_host *host = platform_get_drvdata(pdev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 	int ret;
 
-	sdhci_remove_host(host, true);
-	ret = pm_runtime_put_sync(&pdev->dev);
+	ret = pm_runtime_resume_and_get(&pdev->dev);
 	if (ret < 0)
 		return ret;
 
+	sdhci_remove_host(host, true);
+	clk_disable_unprepare(pltfm_host->clk);
 	pm_runtime_disable(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
 	sdhci_pltfm_free(pdev);
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int sdhci_am654_restore(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_am654_data *sdhci_am654 = sdhci_pltfm_priv(pltfm_host);
+	u32 ctl_cfg_2 = 0;
+	u32 val;
+	int ret;
+
+	if (sdhci_am654->flags & DLL_CALIB) {
+		regmap_read(sdhci_am654->base, PHY_STAT1, &val);
+		if (~val & CALDONE_MASK) {
+			/* Calibrate IO lines */
+			regmap_update_bits(sdhci_am654->base, PHY_CTRL1,
+					   PDB_MASK, PDB_MASK);
+			ret = regmap_read_poll_timeout(sdhci_am654->base,
+						       PHY_STAT1, val,
+						       val & CALDONE_MASK,
+						       1, 20);
+			if (ret)
+				return ret;
+		}
+	}
+
+	/* Enable pins by setting IO mux to 0 */
+	if (sdhci_am654->flags & IOMUX_PRESENT)
+		regmap_update_bits(sdhci_am654->base, PHY_CTRL1,
+				   IOMUX_ENABLE_MASK, 0);
 
+	/* Set slot type based on SD or eMMC */
+	if (host->mmc->caps & MMC_CAP_NONREMOVABLE)
+		ctl_cfg_2 = SLOTTYPE_EMBEDDED;
+
+	regmap_update_bits(sdhci_am654->base, CTL_CFG_2, SLOTTYPE_MASK,
+			   ctl_cfg_2);
+
+	regmap_read(sdhci_am654->base, CTL_CFG_3, &val);
+	if (~val & TUNINGFORSDR50_MASK)
+		/* Enable tuning for SDR50 */
+		regmap_update_bits(sdhci_am654->base, CTL_CFG_3, TUNINGFORSDR50_MASK,
+				   TUNINGFORSDR50_MASK);
+
+	return 0;
+}
+
+static int sdhci_am654_runtime_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	int ret;
+
+	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
+		mmc_retune_needed(host->mmc);
+
+	ret = cqhci_suspend(host->mmc);
+	if (ret)
+		return ret;
+
+	ret = sdhci_runtime_suspend_host(host);
+	if (ret)
+		return ret;
+
+	/* disable the clock */
+	clk_disable_unprepare(pltfm_host->clk);
 	return 0;
 }
 
+static int sdhci_am654_runtime_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	int ret;
+
+	/* Enable the clock */
+	ret = clk_prepare_enable(pltfm_host->clk);
+	if (ret)
+		return ret;
+
+	ret = sdhci_am654_restore(host);
+	if (ret)
+		return ret;
+
+	ret = sdhci_runtime_resume_host(host, 0);
+	if (ret)
+		return ret;
+
+	ret = cqhci_resume(host->mmc);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops sdhci_am654_dev_pm_ops = {
+	SET_RUNTIME_PM_OPS(sdhci_am654_runtime_suspend,
+			   sdhci_am654_runtime_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+};
+
 static struct platform_driver sdhci_am654_driver = {
 	.driver = {
 		.name = "sdhci-am654",
 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+		.pm = &sdhci_am654_dev_pm_ops,
 		.of_match_table = sdhci_am654_of_match,
 	},
 	.probe = sdhci_am654_probe,
diff --git a/drivers/mmc/host/sdricoh_cs.c b/drivers/mmc/host/sdricoh_cs.c
index 76a8cd3a186f..57b8c1a96756 100644
--- a/drivers/mmc/host/sdricoh_cs.c
+++ b/drivers/mmc/host/sdricoh_cs.c
@@ -101,14 +101,6 @@ static inline void sdricoh_writel(struct sdricoh_host *host, unsigned int reg,
 
 }
 
-static inline unsigned int sdricoh_readw(struct sdricoh_host *host,
-					 unsigned int reg)
-{
-	unsigned int value = readw(host->iobase + reg);
-	dev_vdbg(host->dev, "rb %x 0x%x\n", reg, value);
-	return value;
-}
-
 static inline void sdricoh_writew(struct sdricoh_host *host, unsigned int reg,
 					 unsigned short value)
 {
diff --git a/drivers/mmc/host/tmio_mmc_core.c b/drivers/mmc/host/tmio_mmc_core.c
index e24c3d284515..be7f18fd4836 100644
--- a/drivers/mmc/host/tmio_mmc_core.c
+++ b/drivers/mmc/host/tmio_mmc_core.c
@@ -1084,7 +1084,7 @@ static void tmio_mmc_of_parse(struct platform_device *pdev,
 	 * For new platforms, please use "disable-wp" instead of
 	 * "toshiba,mmc-wrprotect-disable"
 	 */
-	if (of_get_property(np, "toshiba,mmc-wrprotect-disable", NULL))
+	if (of_property_read_bool(np, "toshiba,mmc-wrprotect-disable"))
 		mmc->caps2 |= MMC_CAP2_NO_WRITE_PROTECT;
 }
 
diff --git a/drivers/mmc/host/usdhi6rol0.c b/drivers/mmc/host/usdhi6rol0.c
index 99515be6e5e5..2f59917b105e 100644
--- a/drivers/mmc/host/usdhi6rol0.c
+++ b/drivers/mmc/host/usdhi6rol0.c
@@ -1790,8 +1790,7 @@ static int usdhi6_probe(struct platform_device *pdev)
 
 	host->pins_uhs = pinctrl_lookup_state(host->pinctrl, "state_uhs");
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	host->base = devm_ioremap_resource(dev, res);
+	host->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(host->base)) {
 		ret = PTR_ERR(host->base);
 		goto e_free_mmc;
diff --git a/drivers/mmc/host/vub300.c b/drivers/mmc/host/vub300.c
index 72f65f32abbc..e4c4bfac3763 100644
--- a/drivers/mmc/host/vub300.c
+++ b/drivers/mmc/host/vub300.c
@@ -1341,8 +1341,6 @@ static void __download_offload_pseudocode(struct vub300_mmc_host *vub300,
 		return;
 	}
 
-	return;
-
 copy_error_message:
 	strncpy(vub300->vub_name, "SDIO pseudocode download failed",
 		sizeof(vub300->vub_name));
diff --git a/drivers/mmc/host/wmt-sdmmc.c b/drivers/mmc/host/wmt-sdmmc.c
index 9aa3027ca25e..68525d900046 100644
--- a/drivers/mmc/host/wmt-sdmmc.c
+++ b/drivers/mmc/host/wmt-sdmmc.c
@@ -802,10 +802,8 @@ static int wmt_mci_probe(struct platform_device *pdev)
 	priv->power_inverted = 0;
 	priv->cd_inverted = 0;
 
-	if (of_get_property(np, "sdon-inverted", NULL))
-		priv->power_inverted = 1;
-	if (of_get_property(np, "cd-inverted", NULL))
-		priv->cd_inverted = 1;
+	priv->power_inverted = of_property_read_bool(np, "sdon-inverted");
+	priv->cd_inverted = of_property_read_bool(np, "cd-inverted");
 
 	priv->sdmmc_base = of_iomap(np, 0);
 	if (!priv->sdmmc_base) {
diff --git a/drivers/most/most_cdev.c b/drivers/most/most_cdev.c
index 4ee536980f71..3ed8f461e01e 100644
--- a/drivers/most/most_cdev.c
+++ b/drivers/most/most_cdev.c
@@ -491,7 +491,7 @@ static int __init most_cdev_init(void)
 {
 	int err;
 
-	comp.class = class_create(THIS_MODULE, "most_cdev");
+	comp.class = class_create("most_cdev");
 	if (IS_ERR(comp.class))
 		return PTR_ERR(comp.class);
 
diff --git a/drivers/mtd/devices/mtd_dataflash.c b/drivers/mtd/devices/mtd_dataflash.c
index 3bbaa590c768..1d3b2a94581f 100644
--- a/drivers/mtd/devices/mtd_dataflash.c
+++ b/drivers/mtd/devices/mtd_dataflash.c
@@ -639,7 +639,7 @@ static int add_dataflash_otp(struct spi_device *spi, char *name, int nr_pages,
 
 	/* name must be usable with cmdlinepart */
 	sprintf(priv->name, "spi%d.%d-%s",
-			spi->master->bus_num, spi->chip_select,
+			spi->master->bus_num, spi_get_chipselect(spi, 0),
 			name);
 
 	device = &priv->mtd;
diff --git a/drivers/mtd/devices/spear_smi.c b/drivers/mtd/devices/spear_smi.c
index f58742486d3d..cc17133be297 100644
--- a/drivers/mtd/devices/spear_smi.c
+++ b/drivers/mtd/devices/spear_smi.c
@@ -820,8 +820,8 @@ static int spear_smi_probe_config_dt(struct platform_device *pdev,
 		pdata->board_flash_info->mem_base = be32_to_cpup(&addr[0]);
 		pdata->board_flash_info->size = be32_to_cpup(&addr[1]);
 
-		if (of_get_property(pp, "st,smi-fast-mode", NULL))
-			pdata->board_flash_info->fast_mode = 1;
+		pdata->board_flash_info->fast_mode =
+			of_property_read_bool(pp, "st,smi-fast-mode");
 
 		i++;
 	}
diff --git a/drivers/mtd/lpddr/lpddr_cmds.c b/drivers/mtd/lpddr/lpddr_cmds.c
index ee063baed136..3c3939bc2dad 100644
--- a/drivers/mtd/lpddr/lpddr_cmds.c
+++ b/drivers/mtd/lpddr/lpddr_cmds.c
@@ -406,7 +406,7 @@ static int do_write_buffer(struct map_info *map, struct flchip *chip,
 {
 	struct lpddr_private *lpddr = map->fldrv_priv;
 	map_word datum;
-	int ret, wbufsize, word_gap, words;
+	int ret, wbufsize, word_gap;
 	const struct kvec *vec;
 	unsigned long vec_seek;
 	unsigned long prog_buf_ofs;
@@ -421,10 +421,7 @@ static int do_write_buffer(struct map_info *map, struct flchip *chip,
 	}
 	/* Figure out the number of words to write */
 	word_gap = (-adr & (map_bankwidth(map)-1));
-	words = (len - word_gap + map_bankwidth(map) - 1) / map_bankwidth(map);
-	if (!word_gap) {
-		words--;
-	} else {
+	if (word_gap) {
 		word_gap = map_bankwidth(map) - word_gap;
 		adr -= word_gap;
 		datum = map_word_ff(map);
diff --git a/drivers/mtd/maps/sun_uflash.c b/drivers/mtd/maps/sun_uflash.c
index 6c0c91bfec05..860b19f77090 100644
--- a/drivers/mtd/maps/sun_uflash.c
+++ b/drivers/mtd/maps/sun_uflash.c
@@ -112,7 +112,7 @@ static int uflash_probe(struct platform_device *op)
 	/* Flashprom must have the "user" property in order to
 	 * be used by this driver.
 	 */
-	if (!of_find_property(dp, "user", NULL))
+	if (!of_property_read_bool(dp, "user"))
 		return -ENODEV;
 
 	return uflash_devinit(op, dp);
diff --git a/drivers/mtd/mtdblock_ro.c b/drivers/mtd/mtdblock_ro.c
index 7c51952ce55d..66ffc9f1ead2 100644
--- a/drivers/mtd/mtdblock_ro.c
+++ b/drivers/mtd/mtdblock_ro.c
@@ -16,8 +16,10 @@ static int mtdblock_readsect(struct mtd_blktrans_dev *dev,
 			      unsigned long block, char *buf)
 {
 	size_t retlen;
+	int err;
 
-	if (mtd_read(dev->mtd, (block * 512), 512, &retlen, buf))
+	err = mtd_read(dev->mtd, (block * 512), 512, &retlen, buf);
+	if (err && !mtd_is_bitflip(err))
 		return 1;
 	return 0;
 }
diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
index 0feacb9fbdac..60670b2f70b9 100644
--- a/drivers/mtd/mtdcore.c
+++ b/drivers/mtd/mtdcore.c
@@ -63,7 +63,6 @@ static SIMPLE_DEV_PM_OPS(mtd_cls_pm_ops, mtd_cls_suspend, mtd_cls_resume);
 
 static struct class mtd_class = {
 	.name = "mtd",
-	.owner = THIS_MODULE,
 	.pm = MTD_CLS_PM_OPS,
 };
 
@@ -519,7 +518,7 @@ static int mtd_nvmem_add(struct mtd_info *mtd)
 	struct device_node *node = mtd_get_of_node(mtd);
 	struct nvmem_config config = {};
 
-	config.id = -1;
+	config.id = NVMEM_DEVID_NONE;
 	config.dev = &mtd->dev;
 	config.name = dev_name(&mtd->dev);
 	config.owner = THIS_MODULE;
@@ -536,12 +535,11 @@ static int mtd_nvmem_add(struct mtd_info *mtd)
 	mtd->nvmem = nvmem_register(&config);
 	if (IS_ERR(mtd->nvmem)) {
 		/* Just ignore if there is no NVMEM support in the kernel */
-		if (PTR_ERR(mtd->nvmem) == -EOPNOTSUPP) {
+		if (PTR_ERR(mtd->nvmem) == -EOPNOTSUPP)
 			mtd->nvmem = NULL;
-		} else {
-			dev_err(&mtd->dev, "Failed to register NVMEM device\n");
-			return PTR_ERR(mtd->nvmem);
-		}
+		else
+			return dev_err_probe(&mtd->dev, PTR_ERR(mtd->nvmem),
+					     "Failed to register NVMEM device\n");
 	}
 
 	return 0;
@@ -739,7 +737,7 @@ int add_mtd_device(struct mtd_info *mtd)
 
 	mutex_unlock(&mtd_table_mutex);
 
-	if (of_find_property(mtd_get_of_node(mtd), "linux,rootfs", NULL)) {
+	if (of_property_read_bool(mtd_get_of_node(mtd), "linux,rootfs")) {
 		if (IS_BUILTIN(CONFIG_MTD)) {
 			pr_info("mtd: setting mtd%d (%s) as root device\n", mtd->index, mtd->name);
 			ROOT_DEV = MKDEV(MTD_BLOCK_MAJOR, mtd->index);
@@ -888,8 +886,8 @@ static struct nvmem_device *mtd_otp_nvmem_register(struct mtd_info *mtd,
 
 	/* OTP nvmem will be registered on the physical device */
 	config.dev = mtd->dev.parent;
-	config.name = kasprintf(GFP_KERNEL, "%s-%s", dev_name(&mtd->dev), compatible);
-	config.id = NVMEM_DEVID_NONE;
+	config.name = compatible;
+	config.id = NVMEM_DEVID_AUTO;
 	config.owner = THIS_MODULE;
 	config.type = NVMEM_TYPE_OTP;
 	config.root_only = true;
@@ -905,7 +903,6 @@ static struct nvmem_device *mtd_otp_nvmem_register(struct mtd_info *mtd,
 		nvmem = NULL;
 
 	of_node_put(np);
-	kfree(config.name);
 
 	return nvmem;
 }
@@ -940,6 +937,7 @@ static int mtd_nvmem_fact_otp_reg_read(void *priv, unsigned int offset,
 
 static int mtd_otp_nvmem_add(struct mtd_info *mtd)
 {
+	struct device *dev = mtd->dev.parent;
 	struct nvmem_device *nvmem;
 	ssize_t size;
 	int err;
@@ -953,8 +951,8 @@ static int mtd_otp_nvmem_add(struct mtd_info *mtd)
 			nvmem = mtd_otp_nvmem_register(mtd, "user-otp", size,
 						       mtd_nvmem_user_otp_reg_read);
 			if (IS_ERR(nvmem)) {
-				dev_err(&mtd->dev, "Failed to register OTP NVMEM device\n");
-				return PTR_ERR(nvmem);
+				err = PTR_ERR(nvmem);
+				goto err;
 			}
 			mtd->otp_user_nvmem = nvmem;
 		}
@@ -971,7 +969,6 @@ static int mtd_otp_nvmem_add(struct mtd_info *mtd)
 			nvmem = mtd_otp_nvmem_register(mtd, "factory-otp", size,
 						       mtd_nvmem_fact_otp_reg_read);
 			if (IS_ERR(nvmem)) {
-				dev_err(&mtd->dev, "Failed to register OTP NVMEM device\n");
 				err = PTR_ERR(nvmem);
 				goto err;
 			}
@@ -983,7 +980,7 @@ static int mtd_otp_nvmem_add(struct mtd_info *mtd)
 
 err:
 	nvmem_unregister(mtd->otp_user_nvmem);
-	return err;
+	return dev_err_probe(dev, err, "Failed to register OTP NVMEM device\n");
 }
 
 /**
@@ -1023,10 +1020,14 @@ int mtd_device_parse_register(struct mtd_info *mtd, const char * const *types,
 
 	mtd_set_dev_defaults(mtd);
 
+	ret = mtd_otp_nvmem_add(mtd);
+	if (ret)
+		goto out;
+
 	if (IS_ENABLED(CONFIG_MTD_PARTITIONED_MASTER)) {
 		ret = add_mtd_device(mtd);
 		if (ret)
-			return ret;
+			goto out;
 	}
 
 	/* Prefer parsed partitions over driver-provided fallback */
@@ -1061,9 +1062,12 @@ int mtd_device_parse_register(struct mtd_info *mtd, const char * const *types,
 		register_reboot_notifier(&mtd->reboot_notifier);
 	}
 
-	ret = mtd_otp_nvmem_add(mtd);
-
 out:
+	if (ret) {
+		nvmem_unregister(mtd->otp_user_nvmem);
+		nvmem_unregister(mtd->otp_factory_nvmem);
+	}
+
 	if (ret && device_is_registered(&mtd->dev))
 		del_mtd_device(mtd);
 
diff --git a/drivers/mtd/nand/ecc-mxic.c b/drivers/mtd/nand/ecc-mxic.c
index 6b487ffe2f2d..22a760e6024e 100644
--- a/drivers/mtd/nand/ecc-mxic.c
+++ b/drivers/mtd/nand/ecc-mxic.c
@@ -848,13 +848,11 @@ static int mxic_ecc_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int mxic_ecc_remove(struct platform_device *pdev)
+static void mxic_ecc_remove(struct platform_device *pdev)
 {
 	struct mxic_ecc_engine *mxic = platform_get_drvdata(pdev);
 
 	nand_ecc_unregister_on_host_hw_engine(&mxic->external_engine);
-
-	return 0;
 }
 
 static const struct of_device_id mxic_ecc_of_ids[] = {
@@ -871,7 +869,7 @@ static struct platform_driver mxic_ecc_driver = {
 		.of_match_table = mxic_ecc_of_ids,
 	},
 	.probe = mxic_ecc_probe,
-	.remove	= mxic_ecc_remove,
+	.remove_new = mxic_ecc_remove,
 };
 module_platform_driver(mxic_ecc_driver);
 
diff --git a/drivers/mtd/nand/onenand/Kconfig b/drivers/mtd/nand/onenand/Kconfig
index c94bf483541e..7d57836bf32f 100644
--- a/drivers/mtd/nand/onenand/Kconfig
+++ b/drivers/mtd/nand/onenand/Kconfig
@@ -25,7 +25,7 @@ config MTD_ONENAND_GENERIC
 config MTD_ONENAND_OMAP2
 	tristate "OneNAND on OMAP2/OMAP3 support"
 	depends on ARCH_OMAP2 || ARCH_OMAP3 || (COMPILE_TEST && ARM)
-	depends on OF || COMPILE_TEST
+	depends on OF
 	depends on OMAP_GPMC
 	help
 	  Support for a OneNAND flash device connected to an OMAP2/OMAP3 SoC
diff --git a/drivers/mtd/nand/onenand/generic.c b/drivers/mtd/nand/onenand/generic.c
index a4b8b65fe15f..4e7de48f07a6 100644
--- a/drivers/mtd/nand/onenand/generic.c
+++ b/drivers/mtd/nand/onenand/generic.c
@@ -85,7 +85,7 @@ out_free_info:
 	return err;
 }
 
-static int generic_onenand_remove(struct platform_device *pdev)
+static void generic_onenand_remove(struct platform_device *pdev)
 {
 	struct onenand_info *info = platform_get_drvdata(pdev);
 	struct resource *res = pdev->resource;
@@ -97,8 +97,6 @@ static int generic_onenand_remove(struct platform_device *pdev)
 		iounmap(info->onenand.base);
 		kfree(info);
 	}
-
-	return 0;
 }
 
 static struct platform_driver generic_onenand_driver = {
@@ -106,7 +104,7 @@ static struct platform_driver generic_onenand_driver = {
 		.name		= DRIVER_NAME,
 	},
 	.probe		= generic_onenand_probe,
-	.remove		= generic_onenand_remove,
+	.remove_new	= generic_onenand_remove,
 };
 
 module_platform_driver(generic_onenand_driver);
diff --git a/drivers/mtd/nand/onenand/onenand_omap2.c b/drivers/mtd/nand/onenand/onenand_omap2.c
index 12825eb97938..ff7af98604df 100644
--- a/drivers/mtd/nand/onenand/onenand_omap2.c
+++ b/drivers/mtd/nand/onenand/onenand_omap2.c
@@ -581,7 +581,7 @@ err_release_dma:
 	return r;
 }
 
-static int omap2_onenand_remove(struct platform_device *pdev)
+static void omap2_onenand_remove(struct platform_device *pdev)
 {
 	struct omap2_onenand *c = dev_get_drvdata(&pdev->dev);
 
@@ -589,8 +589,6 @@ static int omap2_onenand_remove(struct platform_device *pdev)
 	if (c->dma_chan)
 		dma_release_channel(c->dma_chan);
 	omap2_onenand_shutdown(pdev);
-
-	return 0;
 }
 
 static const struct of_device_id omap2_onenand_id_table[] = {
@@ -601,7 +599,7 @@ MODULE_DEVICE_TABLE(of, omap2_onenand_id_table);
 
 static struct platform_driver omap2_onenand_driver = {
 	.probe		= omap2_onenand_probe,
-	.remove		= omap2_onenand_remove,
+	.remove_new	= omap2_onenand_remove,
 	.shutdown	= omap2_onenand_shutdown,
 	.driver		= {
 		.name	= DRIVER_NAME,
diff --git a/drivers/mtd/nand/onenand/onenand_samsung.c b/drivers/mtd/nand/onenand/onenand_samsung.c
index b64895573515..92151aa52964 100644
--- a/drivers/mtd/nand/onenand/onenand_samsung.c
+++ b/drivers/mtd/nand/onenand/onenand_samsung.c
@@ -943,13 +943,11 @@ static int s3c_onenand_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int s3c_onenand_remove(struct platform_device *pdev)
+static void s3c_onenand_remove(struct platform_device *pdev)
 {
 	struct mtd_info *mtd = platform_get_drvdata(pdev);
 
 	onenand_release(mtd);
-
-	return 0;
 }
 
 static int s3c_pm_ops_suspend(struct device *dev)
@@ -996,7 +994,7 @@ static struct platform_driver s3c_onenand_driver = {
 	},
 	.id_table	= s3c_onenand_driver_ids,
 	.probe          = s3c_onenand_probe,
-	.remove         = s3c_onenand_remove,
+	.remove_new     = s3c_onenand_remove,
 };
 
 module_platform_driver(s3c_onenand_driver);
diff --git a/drivers/mtd/nand/raw/Kconfig b/drivers/mtd/nand/raw/Kconfig
index 170f1185ddc4..b523354dfb00 100644
--- a/drivers/mtd/nand/raw/Kconfig
+++ b/drivers/mtd/nand/raw/Kconfig
@@ -373,7 +373,7 @@ config MTD_NAND_TEGRA
 
 config MTD_NAND_STM32_FMC2
 	tristate "Support for NAND controller on STM32MP SoCs"
-	depends on MACH_STM32MP157 || COMPILE_TEST
+	depends on ARCH_STM32 || COMPILE_TEST
 	select MFD_SYSCON
 	help
 	  Enables support for NAND Flash chips on SoCs containing the FMC2
diff --git a/drivers/mtd/nand/raw/ams-delta.c b/drivers/mtd/nand/raw/ams-delta.c
index 13de39aa3288..fa621ffa6490 100644
--- a/drivers/mtd/nand/raw/ams-delta.c
+++ b/drivers/mtd/nand/raw/ams-delta.c
@@ -397,7 +397,7 @@ err_nand_cleanup:
 /*
  * Clean up routine
  */
-static int gpio_nand_remove(struct platform_device *pdev)
+static void gpio_nand_remove(struct platform_device *pdev)
 {
 	struct gpio_nand *priv = platform_get_drvdata(pdev);
 	struct mtd_info *mtd = nand_to_mtd(&priv->nand_chip);
@@ -410,8 +410,6 @@ static int gpio_nand_remove(struct platform_device *pdev)
 	ret = mtd_device_unregister(mtd);
 	WARN_ON(ret);
 	nand_cleanup(mtd_to_nand(mtd));
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
@@ -434,7 +432,7 @@ MODULE_DEVICE_TABLE(platform, gpio_nand_plat_id_table);
 
 static struct platform_driver gpio_nand_driver = {
 	.probe		= gpio_nand_probe,
-	.remove		= gpio_nand_remove,
+	.remove_new	= gpio_nand_remove,
 	.id_table	= gpio_nand_plat_id_table,
 	.driver		= {
 		.name	= "ams-delta-nand",
diff --git a/drivers/mtd/nand/raw/arasan-nand-controller.c b/drivers/mtd/nand/raw/arasan-nand-controller.c
index ec7e6eeac55f..d513d2db3549 100644
--- a/drivers/mtd/nand/raw/arasan-nand-controller.c
+++ b/drivers/mtd/nand/raw/arasan-nand-controller.c
@@ -1496,7 +1496,7 @@ disable_controller_clk:
 	return ret;
 }
 
-static int anfc_remove(struct platform_device *pdev)
+static void anfc_remove(struct platform_device *pdev)
 {
 	struct arasan_nfc *nfc = platform_get_drvdata(pdev);
 
@@ -1504,8 +1504,6 @@ static int anfc_remove(struct platform_device *pdev)
 
 	clk_disable_unprepare(nfc->bus_clk);
 	clk_disable_unprepare(nfc->controller_clk);
-
-	return 0;
 }
 
 static const struct of_device_id anfc_ids[] = {
@@ -1525,7 +1523,7 @@ static struct platform_driver anfc_driver = {
 		.of_match_table = anfc_ids,
 	},
 	.probe = anfc_probe,
-	.remove = anfc_remove,
+	.remove_new = anfc_remove,
 };
 module_platform_driver(anfc_driver);
 
diff --git a/drivers/mtd/nand/raw/atmel/nand-controller.c b/drivers/mtd/nand/raw/atmel/nand-controller.c
index 41c6bd6e2d72..81e3d682a8cd 100644
--- a/drivers/mtd/nand/raw/atmel/nand-controller.c
+++ b/drivers/mtd/nand/raw/atmel/nand-controller.c
@@ -2626,13 +2626,11 @@ static int atmel_nand_controller_probe(struct platform_device *pdev)
 	return caps->ops->probe(pdev, caps);
 }
 
-static int atmel_nand_controller_remove(struct platform_device *pdev)
+static void atmel_nand_controller_remove(struct platform_device *pdev)
 {
 	struct atmel_nand_controller *nc = platform_get_drvdata(pdev);
 
 	WARN_ON(nc->caps->ops->remove(nc));
-
-	return 0;
 }
 
 static __maybe_unused int atmel_nand_controller_resume(struct device *dev)
@@ -2663,7 +2661,7 @@ static struct platform_driver atmel_nand_controller_driver = {
 		.pm = &atmel_nand_controller_pm_ops,
 	},
 	.probe = atmel_nand_controller_probe,
-	.remove = atmel_nand_controller_remove,
+	.remove_new = atmel_nand_controller_remove,
 };
 module_platform_driver(atmel_nand_controller_driver);
 
diff --git a/drivers/mtd/nand/raw/au1550nd.c b/drivers/mtd/nand/raw/au1550nd.c
index 5aa3a06d740c..063a5e0b8d4b 100644
--- a/drivers/mtd/nand/raw/au1550nd.c
+++ b/drivers/mtd/nand/raw/au1550nd.c
@@ -337,7 +337,7 @@ out1:
 	return ret;
 }
 
-static int au1550nd_remove(struct platform_device *pdev)
+static void au1550nd_remove(struct platform_device *pdev)
 {
 	struct au1550nd_ctx *ctx = platform_get_drvdata(pdev);
 	struct resource *r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -350,7 +350,6 @@ static int au1550nd_remove(struct platform_device *pdev)
 	iounmap(ctx->base);
 	release_mem_region(r->start, 0x1000);
 	kfree(ctx);
-	return 0;
 }
 
 static struct platform_driver au1550nd_driver = {
@@ -358,7 +357,7 @@ static struct platform_driver au1550nd_driver = {
 		.name	= "au1550-nand",
 	},
 	.probe		= au1550nd_probe,
-	.remove		= au1550nd_remove,
+	.remove_new	= au1550nd_remove,
 };
 
 module_platform_driver(au1550nd_driver);
diff --git a/drivers/mtd/nand/raw/bcm47xxnflash/main.c b/drivers/mtd/nand/raw/bcm47xxnflash/main.c
index dcc70d9dc6e5..ebcf508e0606 100644
--- a/drivers/mtd/nand/raw/bcm47xxnflash/main.c
+++ b/drivers/mtd/nand/raw/bcm47xxnflash/main.c
@@ -57,7 +57,7 @@ static int bcm47xxnflash_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int bcm47xxnflash_remove(struct platform_device *pdev)
+static void bcm47xxnflash_remove(struct platform_device *pdev)
 {
 	struct bcm47xxnflash *nflash = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &nflash->nand_chip;
@@ -66,13 +66,11 @@ static int bcm47xxnflash_remove(struct platform_device *pdev)
 	ret = mtd_device_unregister(nand_to_mtd(chip));
 	WARN_ON(ret);
 	nand_cleanup(chip);
-
-	return 0;
 }
 
 static struct platform_driver bcm47xxnflash_driver = {
 	.probe	= bcm47xxnflash_probe,
-	.remove = bcm47xxnflash_remove,
+	.remove_new = bcm47xxnflash_remove,
 	.driver = {
 		.name = "bcma_nflash",
 	},
diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c
index 7661a5cf1883..034ec564c2ed 100644
--- a/drivers/mtd/nand/raw/cadence-nand-controller.c
+++ b/drivers/mtd/nand/raw/cadence-nand-controller.c
@@ -3055,18 +3055,16 @@ static int cadence_nand_dt_probe(struct platform_device *ofdev)
 	return 0;
 }
 
-static int cadence_nand_dt_remove(struct platform_device *ofdev)
+static void cadence_nand_dt_remove(struct platform_device *ofdev)
 {
 	struct cadence_nand_dt *dt = platform_get_drvdata(ofdev);
 
 	cadence_nand_remove(&dt->cdns_ctrl);
-
-	return 0;
 }
 
 static struct platform_driver cadence_nand_dt_driver = {
 	.probe		= cadence_nand_dt_probe,
-	.remove		= cadence_nand_dt_remove,
+	.remove_new	= cadence_nand_dt_remove,
 	.driver		= {
 		.name	= "cadence-nand-controller",
 		.of_match_table = cadence_nand_dt_ids,
diff --git a/drivers/mtd/nand/raw/davinci_nand.c b/drivers/mtd/nand/raw/davinci_nand.c
index 3e98e3c255bf..415d6aaa8255 100644
--- a/drivers/mtd/nand/raw/davinci_nand.c
+++ b/drivers/mtd/nand/raw/davinci_nand.c
@@ -821,7 +821,7 @@ err_cleanup_nand:
 	return ret;
 }
 
-static int nand_davinci_remove(struct platform_device *pdev)
+static void nand_davinci_remove(struct platform_device *pdev)
 {
 	struct davinci_nand_info *info = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &info->chip;
@@ -835,13 +835,11 @@ static int nand_davinci_remove(struct platform_device *pdev)
 	ret = mtd_device_unregister(nand_to_mtd(chip));
 	WARN_ON(ret);
 	nand_cleanup(chip);
-
-	return 0;
 }
 
 static struct platform_driver nand_davinci_driver = {
 	.probe		= nand_davinci_probe,
-	.remove		= nand_davinci_remove,
+	.remove_new	= nand_davinci_remove,
 	.driver		= {
 		.name	= "davinci_nand",
 		.of_match_table = of_match_ptr(davinci_nand_of_match),
diff --git a/drivers/mtd/nand/raw/denali_dt.c b/drivers/mtd/nand/raw/denali_dt.c
index 8513bb9fcfcc..915047e3fbc2 100644
--- a/drivers/mtd/nand/raw/denali_dt.c
+++ b/drivers/mtd/nand/raw/denali_dt.c
@@ -233,7 +233,7 @@ out_disable_clk:
 	return ret;
 }
 
-static int denali_dt_remove(struct platform_device *pdev)
+static void denali_dt_remove(struct platform_device *pdev)
 {
 	struct denali_dt *dt = platform_get_drvdata(pdev);
 
@@ -243,13 +243,11 @@ static int denali_dt_remove(struct platform_device *pdev)
 	clk_disable_unprepare(dt->clk_ecc);
 	clk_disable_unprepare(dt->clk_x);
 	clk_disable_unprepare(dt->clk);
-
-	return 0;
 }
 
 static struct platform_driver denali_dt_driver = {
 	.probe		= denali_dt_probe,
-	.remove		= denali_dt_remove,
+	.remove_new	= denali_dt_remove,
 	.driver		= {
 		.name	= "denali-nand-dt",
 		.of_match_table	= denali_nand_dt_ids,
diff --git a/drivers/mtd/nand/raw/fsl_elbc_nand.c b/drivers/mtd/nand/raw/fsl_elbc_nand.c
index e25119e58b69..1e3a80f06f33 100644
--- a/drivers/mtd/nand/raw/fsl_elbc_nand.c
+++ b/drivers/mtd/nand/raw/fsl_elbc_nand.c
@@ -963,7 +963,7 @@ err:
 	return ret;
 }
 
-static int fsl_elbc_nand_remove(struct platform_device *pdev)
+static void fsl_elbc_nand_remove(struct platform_device *pdev)
 {
 	struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = fsl_lbc_ctrl_dev->nand;
 	struct fsl_elbc_mtd *priv = dev_get_drvdata(&pdev->dev);
@@ -984,8 +984,6 @@ static int fsl_elbc_nand_remove(struct platform_device *pdev)
 	}
 	mutex_unlock(&fsl_elbc_nand_mutex);
 
-	return 0;
-
 }
 
 static const struct of_device_id fsl_elbc_nand_match[] = {
@@ -1000,7 +998,7 @@ static struct platform_driver fsl_elbc_nand_driver = {
 		.of_match_table = fsl_elbc_nand_match,
 	},
 	.probe = fsl_elbc_nand_probe,
-	.remove = fsl_elbc_nand_remove,
+	.remove_new = fsl_elbc_nand_remove,
 };
 
 module_platform_driver(fsl_elbc_nand_driver);
diff --git a/drivers/mtd/nand/raw/fsl_ifc_nand.c b/drivers/mtd/nand/raw/fsl_ifc_nand.c
index 02d500176838..fa537fee6701 100644
--- a/drivers/mtd/nand/raw/fsl_ifc_nand.c
+++ b/drivers/mtd/nand/raw/fsl_ifc_nand.c
@@ -1094,7 +1094,7 @@ err:
 	return ret;
 }
 
-static int fsl_ifc_nand_remove(struct platform_device *dev)
+static void fsl_ifc_nand_remove(struct platform_device *dev)
 {
 	struct fsl_ifc_mtd *priv = dev_get_drvdata(&dev->dev);
 	struct nand_chip *chip = &priv->chip;
@@ -1113,8 +1113,6 @@ static int fsl_ifc_nand_remove(struct platform_device *dev)
 		kfree(ifc_nand_ctrl);
 	}
 	mutex_unlock(&fsl_ifc_nand_mutex);
-
-	return 0;
 }
 
 static const struct of_device_id fsl_ifc_nand_match[] = {
@@ -1131,7 +1129,7 @@ static struct platform_driver fsl_ifc_nand_driver = {
 		.of_match_table = fsl_ifc_nand_match,
 	},
 	.probe       = fsl_ifc_nand_probe,
-	.remove      = fsl_ifc_nand_remove,
+	.remove_new  = fsl_ifc_nand_remove,
 };
 
 module_platform_driver(fsl_ifc_nand_driver);
diff --git a/drivers/mtd/nand/raw/fsl_upm.c b/drivers/mtd/nand/raw/fsl_upm.c
index b3cc427100a2..086426139173 100644
--- a/drivers/mtd/nand/raw/fsl_upm.c
+++ b/drivers/mtd/nand/raw/fsl_upm.c
@@ -235,7 +235,7 @@ static int fun_probe(struct platform_device *ofdev)
 	return 0;
 }
 
-static int fun_remove(struct platform_device *ofdev)
+static void fun_remove(struct platform_device *ofdev)
 {
 	struct fsl_upm_nand *fun = dev_get_drvdata(&ofdev->dev);
 	struct nand_chip *chip = &fun->chip;
@@ -245,8 +245,6 @@ static int fun_remove(struct platform_device *ofdev)
 	ret = mtd_device_unregister(mtd);
 	WARN_ON(ret);
 	nand_cleanup(chip);
-
-	return 0;
 }
 
 static const struct of_device_id of_fun_match[] = {
@@ -261,7 +259,7 @@ static struct platform_driver of_fun_driver = {
 		.of_match_table = of_fun_match,
 	},
 	.probe		= fun_probe,
-	.remove		= fun_remove,
+	.remove_new	= fun_remove,
 };
 
 module_platform_driver(of_fun_driver);
diff --git a/drivers/mtd/nand/raw/fsmc_nand.c b/drivers/mtd/nand/raw/fsmc_nand.c
index 6b2bda815b88..7b4742420dfc 100644
--- a/drivers/mtd/nand/raw/fsmc_nand.c
+++ b/drivers/mtd/nand/raw/fsmc_nand.c
@@ -880,7 +880,7 @@ static int fsmc_nand_probe_config_dt(struct platform_device *pdev,
 		}
 	}
 
-	if (of_get_property(np, "nand-skip-bbtscan", NULL))
+	if (of_property_read_bool(np, "nand-skip-bbtscan"))
 		nand->options |= NAND_SKIP_BBTSCAN;
 
 	host->dev_timings = devm_kzalloc(&pdev->dev,
@@ -1165,7 +1165,7 @@ disable_clk:
 /*
  * Clean up routine
  */
-static int fsmc_nand_remove(struct platform_device *pdev)
+static void fsmc_nand_remove(struct platform_device *pdev)
 {
 	struct fsmc_nand_data *host = platform_get_drvdata(pdev);
 
@@ -1184,8 +1184,6 @@ static int fsmc_nand_remove(struct platform_device *pdev)
 		}
 		clk_disable_unprepare(host->clk);
 	}
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1224,7 +1222,7 @@ static const struct of_device_id fsmc_nand_id_table[] = {
 MODULE_DEVICE_TABLE(of, fsmc_nand_id_table);
 
 static struct platform_driver fsmc_nand_driver = {
-	.remove = fsmc_nand_remove,
+	.remove_new = fsmc_nand_remove,
 	.driver = {
 		.name = "fsmc-nand",
 		.of_match_table = fsmc_nand_id_table,
diff --git a/drivers/mtd/nand/raw/gpio.c b/drivers/mtd/nand/raw/gpio.c
index dcf28cff760d..d6cc2cb65214 100644
--- a/drivers/mtd/nand/raw/gpio.c
+++ b/drivers/mtd/nand/raw/gpio.c
@@ -265,7 +265,7 @@ gpio_nand_get_io_sync(struct platform_device *pdev)
 	return platform_get_resource(pdev, IORESOURCE_MEM, 1);
 }
 
-static int gpio_nand_remove(struct platform_device *pdev)
+static void gpio_nand_remove(struct platform_device *pdev)
 {
 	struct gpiomtd *gpiomtd = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &gpiomtd->nand_chip;
@@ -280,8 +280,6 @@ static int gpio_nand_remove(struct platform_device *pdev)
 		gpiod_set_value(gpiomtd->nwp, 0);
 	if (gpiomtd->nce && !IS_ERR(gpiomtd->nce))
 		gpiod_set_value(gpiomtd->nce, 0);
-
-	return 0;
 }
 
 static int gpio_nand_probe(struct platform_device *pdev)
@@ -394,7 +392,7 @@ out_ce:
 
 static struct platform_driver gpio_nand_driver = {
 	.probe		= gpio_nand_probe,
-	.remove		= gpio_nand_remove,
+	.remove_new	= gpio_nand_remove,
 	.driver		= {
 		.name	= "gpio-nand",
 		.of_match_table = of_match_ptr(gpio_nand_id_table),
diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
index ada83344b0f9..500e7a28d2e4 100644
--- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
+++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
@@ -2777,7 +2777,7 @@ exit_acquire_resources:
 	return ret;
 }
 
-static int gpmi_nand_remove(struct platform_device *pdev)
+static void gpmi_nand_remove(struct platform_device *pdev)
 {
 	struct gpmi_nand_data *this = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &this->nand;
@@ -2791,7 +2791,6 @@ static int gpmi_nand_remove(struct platform_device *pdev)
 	nand_cleanup(chip);
 	gpmi_free_dma_buffer(this);
 	release_resources(this);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -2860,7 +2859,7 @@ static struct platform_driver gpmi_nand_driver = {
 		.of_match_table = gpmi_nand_id_table,
 	},
 	.probe   = gpmi_nand_probe,
-	.remove  = gpmi_nand_remove,
+	.remove_new = gpmi_nand_remove,
 };
 module_platform_driver(gpmi_nand_driver);
 
diff --git a/drivers/mtd/nand/raw/hisi504_nand.c b/drivers/mtd/nand/raw/hisi504_nand.c
index c74f6b2192fc..fe291a2e5c77 100644
--- a/drivers/mtd/nand/raw/hisi504_nand.c
+++ b/drivers/mtd/nand/raw/hisi504_nand.c
@@ -798,7 +798,7 @@ static int hisi_nfc_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int hisi_nfc_remove(struct platform_device *pdev)
+static void hisi_nfc_remove(struct platform_device *pdev)
 {
 	struct hinfc_host *host = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &host->chip;
@@ -807,8 +807,6 @@ static int hisi_nfc_remove(struct platform_device *pdev)
 	ret = mtd_device_unregister(nand_to_mtd(chip));
 	WARN_ON(ret);
 	nand_cleanup(chip);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -860,7 +858,7 @@ static struct platform_driver hisi_nfc_driver = {
 		.pm = &hisi_nfc_pm_ops,
 	},
 	.probe		= hisi_nfc_probe,
-	.remove		= hisi_nfc_remove,
+	.remove_new	= hisi_nfc_remove,
 };
 
 module_platform_driver(hisi_nfc_driver);
diff --git a/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c b/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c
index ff26c10f295d..b9f135297aa0 100644
--- a/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c
+++ b/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c
@@ -522,7 +522,7 @@ static int ingenic_nand_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int ingenic_nand_remove(struct platform_device *pdev)
+static void ingenic_nand_remove(struct platform_device *pdev)
 {
 	struct ingenic_nfc *nfc = platform_get_drvdata(pdev);
 
@@ -530,8 +530,6 @@ static int ingenic_nand_remove(struct platform_device *pdev)
 		ingenic_ecc_release(nfc->ecc);
 
 	ingenic_nand_cleanup_chips(nfc);
-
-	return 0;
 }
 
 static const struct jz_soc_info jz4740_soc_info = {
@@ -564,7 +562,7 @@ MODULE_DEVICE_TABLE(of, ingenic_nand_dt_match);
 
 static struct platform_driver ingenic_nand_driver = {
 	.probe		= ingenic_nand_probe,
-	.remove		= ingenic_nand_remove,
+	.remove_new	= ingenic_nand_remove,
 	.driver	= {
 		.name	= DRV_NAME,
 		.of_match_table = ingenic_nand_dt_match,
diff --git a/drivers/mtd/nand/raw/intel-nand-controller.c b/drivers/mtd/nand/raw/intel-nand-controller.c
index 6f4cea81f97c..a9909eb08124 100644
--- a/drivers/mtd/nand/raw/intel-nand-controller.c
+++ b/drivers/mtd/nand/raw/intel-nand-controller.c
@@ -706,7 +706,7 @@ err_of_node_put:
 	return ret;
 }
 
-static int ebu_nand_remove(struct platform_device *pdev)
+static void ebu_nand_remove(struct platform_device *pdev)
 {
 	struct ebu_nand_controller *ebu_host = platform_get_drvdata(pdev);
 	int ret;
@@ -717,8 +717,6 @@ static int ebu_nand_remove(struct platform_device *pdev)
 	ebu_nand_disable(&ebu_host->chip);
 	ebu_dma_cleanup(ebu_host);
 	clk_disable_unprepare(ebu_host->clk);
-
-	return 0;
 }
 
 static const struct of_device_id ebu_nand_match[] = {
@@ -729,7 +727,7 @@ MODULE_DEVICE_TABLE(of, ebu_nand_match);
 
 static struct platform_driver ebu_nand_driver = {
 	.probe = ebu_nand_probe,
-	.remove = ebu_nand_remove,
+	.remove_new = ebu_nand_remove,
 	.driver = {
 		.name = "intel-nand-controller",
 		.of_match_table = ebu_nand_match,
diff --git a/drivers/mtd/nand/raw/lpc32xx_mlc.c b/drivers/mtd/nand/raw/lpc32xx_mlc.c
index ae7f6429a5f6..b3136ae6f4e9 100644
--- a/drivers/mtd/nand/raw/lpc32xx_mlc.c
+++ b/drivers/mtd/nand/raw/lpc32xx_mlc.c
@@ -827,7 +827,7 @@ free_gpio:
 /*
  * Remove NAND device
  */
-static int lpc32xx_nand_remove(struct platform_device *pdev)
+static void lpc32xx_nand_remove(struct platform_device *pdev)
 {
 	struct lpc32xx_nand_host *host = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &host->nand_chip;
@@ -846,8 +846,6 @@ static int lpc32xx_nand_remove(struct platform_device *pdev)
 
 	lpc32xx_wp_enable(host);
 	gpiod_put(host->wp_gpio);
-
-	return 0;
 }
 
 static int lpc32xx_nand_resume(struct platform_device *pdev)
@@ -889,7 +887,7 @@ MODULE_DEVICE_TABLE(of, lpc32xx_nand_match);
 
 static struct platform_driver lpc32xx_nand_driver = {
 	.probe		= lpc32xx_nand_probe,
-	.remove		= lpc32xx_nand_remove,
+	.remove_new	= lpc32xx_nand_remove,
 	.resume		= pm_ptr(lpc32xx_nand_resume),
 	.suspend	= pm_ptr(lpc32xx_nand_suspend),
 	.driver		= {
diff --git a/drivers/mtd/nand/raw/lpc32xx_slc.c b/drivers/mtd/nand/raw/lpc32xx_slc.c
index 6918737346c9..3139b6107660 100644
--- a/drivers/mtd/nand/raw/lpc32xx_slc.c
+++ b/drivers/mtd/nand/raw/lpc32xx_slc.c
@@ -946,7 +946,7 @@ enable_wp:
 /*
  * Remove NAND device.
  */
-static int lpc32xx_nand_remove(struct platform_device *pdev)
+static void lpc32xx_nand_remove(struct platform_device *pdev)
 {
 	uint32_t tmp;
 	struct lpc32xx_nand_host *host = platform_get_drvdata(pdev);
@@ -965,8 +965,6 @@ static int lpc32xx_nand_remove(struct platform_device *pdev)
 
 	clk_disable_unprepare(host->clk);
 	lpc32xx_wp_enable(host);
-
-	return 0;
 }
 
 static int lpc32xx_nand_resume(struct platform_device *pdev)
@@ -1015,7 +1013,7 @@ MODULE_DEVICE_TABLE(of, lpc32xx_nand_match);
 
 static struct platform_driver lpc32xx_nand_driver = {
 	.probe		= lpc32xx_nand_probe,
-	.remove		= lpc32xx_nand_remove,
+	.remove_new	= lpc32xx_nand_remove,
 	.resume		= pm_ptr(lpc32xx_nand_resume),
 	.suspend	= pm_ptr(lpc32xx_nand_suspend),
 	.driver		= {
diff --git a/drivers/mtd/nand/raw/marvell_nand.c b/drivers/mtd/nand/raw/marvell_nand.c
index 3034916d2e25..afb424579f0b 100644
--- a/drivers/mtd/nand/raw/marvell_nand.c
+++ b/drivers/mtd/nand/raw/marvell_nand.c
@@ -3004,7 +3004,7 @@ unprepare_core_clk:
 	return ret;
 }
 
-static int marvell_nfc_remove(struct platform_device *pdev)
+static void marvell_nfc_remove(struct platform_device *pdev)
 {
 	struct marvell_nfc *nfc = platform_get_drvdata(pdev);
 
@@ -3017,8 +3017,6 @@ static int marvell_nfc_remove(struct platform_device *pdev)
 
 	clk_disable_unprepare(nfc->reg_clk);
 	clk_disable_unprepare(nfc->core_clk);
-
-	return 0;
 }
 
 static int __maybe_unused marvell_nfc_suspend(struct device *dev)
@@ -3154,7 +3152,7 @@ static struct platform_driver marvell_nfc_driver = {
 	},
 	.id_table = marvell_nfc_platform_ids,
 	.probe = marvell_nfc_probe,
-	.remove	= marvell_nfc_remove,
+	.remove_new = marvell_nfc_remove,
 };
 module_platform_driver(marvell_nfc_driver);
 
diff --git a/drivers/mtd/nand/raw/meson_nand.c b/drivers/mtd/nand/raw/meson_nand.c
index 074e14225c06..1feea7d82252 100644
--- a/drivers/mtd/nand/raw/meson_nand.c
+++ b/drivers/mtd/nand/raw/meson_nand.c
@@ -1440,20 +1440,18 @@ err_clk:
 	return ret;
 }
 
-static int meson_nfc_remove(struct platform_device *pdev)
+static void meson_nfc_remove(struct platform_device *pdev)
 {
 	struct meson_nfc *nfc = platform_get_drvdata(pdev);
 
 	meson_nfc_nand_chip_cleanup(nfc);
 
 	meson_nfc_disable_clk(nfc);
-
-	return 0;
 }
 
 static struct platform_driver meson_nfc_driver = {
 	.probe  = meson_nfc_probe,
-	.remove = meson_nfc_remove,
+	.remove_new = meson_nfc_remove,
 	.driver = {
 		.name  = "meson-nand",
 		.of_match_table = meson_nfc_id_table,
diff --git a/drivers/mtd/nand/raw/mpc5121_nfc.c b/drivers/mtd/nand/raw/mpc5121_nfc.c
index f68349cb7824..ab05ee65702c 100644
--- a/drivers/mtd/nand/raw/mpc5121_nfc.c
+++ b/drivers/mtd/nand/raw/mpc5121_nfc.c
@@ -822,7 +822,7 @@ error:
 	return retval;
 }
 
-static int mpc5121_nfc_remove(struct platform_device *op)
+static void mpc5121_nfc_remove(struct platform_device *op)
 {
 	struct device *dev = &op->dev;
 	struct mtd_info *mtd = dev_get_drvdata(dev);
@@ -832,8 +832,6 @@ static int mpc5121_nfc_remove(struct platform_device *op)
 	WARN_ON(ret);
 	nand_cleanup(mtd_to_nand(mtd));
 	mpc5121_nfc_free(dev, mtd);
-
-	return 0;
 }
 
 static const struct of_device_id mpc5121_nfc_match[] = {
@@ -844,7 +842,7 @@ MODULE_DEVICE_TABLE(of, mpc5121_nfc_match);
 
 static struct platform_driver mpc5121_nfc_driver = {
 	.probe		= mpc5121_nfc_probe,
-	.remove		= mpc5121_nfc_remove,
+	.remove_new	= mpc5121_nfc_remove,
 	.driver		= {
 		.name = DRV_NAME,
 		.of_match_table = mpc5121_nfc_match,
diff --git a/drivers/mtd/nand/raw/mtk_nand.c b/drivers/mtd/nand/raw/mtk_nand.c
index d540454cbbdf..b2fa6b2074ab 100644
--- a/drivers/mtd/nand/raw/mtk_nand.c
+++ b/drivers/mtd/nand/raw/mtk_nand.c
@@ -1601,7 +1601,7 @@ release_ecc:
 	return ret;
 }
 
-static int mtk_nfc_remove(struct platform_device *pdev)
+static void mtk_nfc_remove(struct platform_device *pdev)
 {
 	struct mtk_nfc *nfc = platform_get_drvdata(pdev);
 	struct mtk_nfc_nand_chip *mtk_chip;
@@ -1620,8 +1620,6 @@ static int mtk_nfc_remove(struct platform_device *pdev)
 
 	mtk_ecc_release(nfc->ecc);
 	mtk_nfc_disable_clk(&nfc->clk);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1663,7 +1661,7 @@ static SIMPLE_DEV_PM_OPS(mtk_nfc_pm_ops, mtk_nfc_suspend, mtk_nfc_resume);
 
 static struct platform_driver mtk_nfc_driver = {
 	.probe  = mtk_nfc_probe,
-	.remove = mtk_nfc_remove,
+	.remove_new = mtk_nfc_remove,
 	.driver = {
 		.name  = MTK_NAME,
 		.of_match_table = mtk_nfc_id_table,
diff --git a/drivers/mtd/nand/raw/mxc_nand.c b/drivers/mtd/nand/raw/mxc_nand.c
index f6c96341b896..3d4b2e8294ea 100644
--- a/drivers/mtd/nand/raw/mxc_nand.c
+++ b/drivers/mtd/nand/raw/mxc_nand.c
@@ -1599,16 +1599,6 @@ static inline int is_imx25_nfc(struct mxc_nand_host *host)
 	return host->devtype_data == &imx25_nand_devtype_data;
 }
 
-static inline int is_imx51_nfc(struct mxc_nand_host *host)
-{
-	return host->devtype_data == &imx51_nand_devtype_data;
-}
-
-static inline int is_imx53_nfc(struct mxc_nand_host *host)
-{
-	return host->devtype_data == &imx53_nand_devtype_data;
-}
-
 static const struct of_device_id mxcnd_dt_ids[] = {
 	{ .compatible = "fsl,imx21-nand", .data = &imx21_nand_devtype_data, },
 	{ .compatible = "fsl,imx27-nand", .data = &imx27_nand_devtype_data, },
@@ -1831,7 +1821,7 @@ escan:
 	return err;
 }
 
-static int mxcnd_remove(struct platform_device *pdev)
+static void mxcnd_remove(struct platform_device *pdev)
 {
 	struct mxc_nand_host *host = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &host->nand;
@@ -1842,8 +1832,6 @@ static int mxcnd_remove(struct platform_device *pdev)
 	nand_cleanup(chip);
 	if (host->clk_act)
 		clk_disable_unprepare(host->clk);
-
-	return 0;
 }
 
 static struct platform_driver mxcnd_driver = {
@@ -1852,7 +1840,7 @@ static struct platform_driver mxcnd_driver = {
 		   .of_match_table = mxcnd_dt_ids,
 	},
 	.probe = mxcnd_probe,
-	.remove = mxcnd_remove,
+	.remove_new = mxcnd_remove,
 };
 module_platform_driver(mxcnd_driver);
 
diff --git a/drivers/mtd/nand/raw/mxic_nand.c b/drivers/mtd/nand/raw/mxic_nand.c
index da1070993994..be8050e84b4f 100644
--- a/drivers/mtd/nand/raw/mxic_nand.c
+++ b/drivers/mtd/nand/raw/mxic_nand.c
@@ -553,7 +553,7 @@ fail:
 	return err;
 }
 
-static int mxic_nfc_remove(struct platform_device *pdev)
+static void mxic_nfc_remove(struct platform_device *pdev)
 {
 	struct mxic_nand_ctlr *nfc = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &nfc->chip;
@@ -564,7 +564,6 @@ static int mxic_nfc_remove(struct platform_device *pdev)
 	nand_cleanup(chip);
 
 	mxic_nfc_clk_disable(nfc);
-	return 0;
 }
 
 static const struct of_device_id mxic_nfc_of_ids[] = {
@@ -575,7 +574,7 @@ MODULE_DEVICE_TABLE(of, mxic_nfc_of_ids);
 
 static struct platform_driver mxic_nfc_driver = {
 	.probe = mxic_nfc_probe,
-	.remove = mxic_nfc_remove,
+	.remove_new = mxic_nfc_remove,
 	.driver = {
 		.name = "mxic-nfc",
 		.of_match_table = mxic_nfc_of_ids,
diff --git a/drivers/mtd/nand/raw/nand_hynix.c b/drivers/mtd/nand/raw/nand_hynix.c
index 0d4d4bbfdece..39076735a3fb 100644
--- a/drivers/mtd/nand/raw/nand_hynix.c
+++ b/drivers/mtd/nand/raw/nand_hynix.c
@@ -728,8 +728,21 @@ static int hynix_nand_init(struct nand_chip *chip)
 	return ret;
 }
 
+static void hynix_fixup_onfi_param_page(struct nand_chip *chip,
+					struct nand_onfi_params *p)
+{
+	/*
+	 * Certain chips might report a 0 on sdr_timing_mode field
+	 * (bytes 129-130). This has been seen on H27U4G8F2GDA-BI.
+	 * According to ONFI specification, bit 0 of this field "shall be 1".
+	 * Forcibly set this bit.
+	 */
+	p->sdr_timing_modes |= cpu_to_le16(BIT(0));
+}
+
 const struct nand_manufacturer_ops hynix_nand_manuf_ops = {
 	.detect = hynix_nand_decode_id,
 	.init = hynix_nand_init,
 	.cleanup = hynix_nand_cleanup,
+	.fixup_onfi_param_page = hynix_fixup_onfi_param_page,
 };
diff --git a/drivers/mtd/nand/raw/nand_macronix.c b/drivers/mtd/nand/raw/nand_macronix.c
index 1472f925f386..385957eb6762 100644
--- a/drivers/mtd/nand/raw/nand_macronix.c
+++ b/drivers/mtd/nand/raw/nand_macronix.c
@@ -93,14 +93,13 @@ static void macronix_nand_onfi_init(struct nand_chip *chip)
 	struct nand_parameters *p = &chip->parameters;
 	struct nand_onfi_vendor_macronix *mxic;
 	struct device_node *dn = nand_get_flash_node(chip);
-	int rand_otp = 0;
+	int rand_otp;
 	int ret;
 
 	if (!p->onfi)
 		return;
 
-	if (of_find_property(dn, "mxic,enable-randomizer-otp", NULL))
-		rand_otp = 1;
+	rand_otp = of_property_read_bool(dn, "mxic,enable-randomizer-otp");
 
 	mxic = (struct nand_onfi_vendor_macronix *)p->onfi->vendor;
 	/* Subpage write is prohibited in randomizer operatoin */
diff --git a/drivers/mtd/nand/raw/ndfc.c b/drivers/mtd/nand/raw/ndfc.c
index 338d6b1a189e..57f3db32122d 100644
--- a/drivers/mtd/nand/raw/ndfc.c
+++ b/drivers/mtd/nand/raw/ndfc.c
@@ -240,7 +240,7 @@ static int ndfc_probe(struct platform_device *ofdev)
 	return 0;
 }
 
-static int ndfc_remove(struct platform_device *ofdev)
+static void ndfc_remove(struct platform_device *ofdev)
 {
 	struct ndfc_controller *ndfc = dev_get_drvdata(&ofdev->dev);
 	struct nand_chip *chip = &ndfc->chip;
@@ -251,8 +251,6 @@ static int ndfc_remove(struct platform_device *ofdev)
 	WARN_ON(ret);
 	nand_cleanup(chip);
 	kfree(mtd->name);
-
-	return 0;
 }
 
 static const struct of_device_id ndfc_match[] = {
@@ -267,7 +265,7 @@ static struct platform_driver ndfc_driver = {
 		.of_match_table = ndfc_match,
 	},
 	.probe = ndfc_probe,
-	.remove = ndfc_remove,
+	.remove_new = ndfc_remove,
 };
 
 module_platform_driver(ndfc_driver);
diff --git a/drivers/mtd/nand/raw/omap2.c b/drivers/mtd/nand/raw/omap2.c
index 4a9f2b6c772d..db22b3af16d8 100644
--- a/drivers/mtd/nand/raw/omap2.c
+++ b/drivers/mtd/nand/raw/omap2.c
@@ -2273,7 +2273,7 @@ return_error:
 	return err;
 }
 
-static int omap_nand_remove(struct platform_device *pdev)
+static void omap_nand_remove(struct platform_device *pdev)
 {
 	struct mtd_info *mtd = platform_get_drvdata(pdev);
 	struct nand_chip *nand_chip = mtd_to_nand(mtd);
@@ -2285,7 +2285,6 @@ static int omap_nand_remove(struct platform_device *pdev)
 		dma_release_channel(info->dma);
 	WARN_ON(mtd_device_unregister(mtd));
 	nand_cleanup(nand_chip);
-	return 0;
 }
 
 /* omap_nand_ids defined in linux/platform_data/mtd-nand-omap2.h */
@@ -2293,7 +2292,7 @@ MODULE_DEVICE_TABLE(of, omap_nand_ids);
 
 static struct platform_driver omap_nand_driver = {
 	.probe		= omap_nand_probe,
-	.remove		= omap_nand_remove,
+	.remove_new	= omap_nand_remove,
 	.driver		= {
 		.name	= DRIVER_NAME,
 		.of_match_table = omap_nand_ids,
diff --git a/drivers/mtd/nand/raw/omap_elm.c b/drivers/mtd/nand/raw/omap_elm.c
index 4796a48e1012..6e1eac6644a6 100644
--- a/drivers/mtd/nand/raw/omap_elm.c
+++ b/drivers/mtd/nand/raw/omap_elm.c
@@ -422,11 +422,10 @@ static int elm_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int elm_remove(struct platform_device *pdev)
+static void elm_remove(struct platform_device *pdev)
 {
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -561,7 +560,7 @@ static struct platform_driver elm_driver = {
 		.pm	= &elm_pm_ops,
 	},
 	.probe	= elm_probe,
-	.remove	= elm_remove,
+	.remove_new = elm_remove,
 };
 
 module_platform_driver(elm_driver);
diff --git a/drivers/mtd/nand/raw/orion_nand.c b/drivers/mtd/nand/raw/orion_nand.c
index 1bfecf502216..7e0313889b50 100644
--- a/drivers/mtd/nand/raw/orion_nand.c
+++ b/drivers/mtd/nand/raw/orion_nand.c
@@ -102,7 +102,6 @@ static int __init orion_nand_probe(struct platform_device *pdev)
 	struct mtd_info *mtd;
 	struct nand_chip *nc;
 	struct orion_nand_data *board;
-	struct resource *res;
 	void __iomem *io_base;
 	int ret = 0;
 	u32 val = 0;
@@ -119,8 +118,7 @@ static int __init orion_nand_probe(struct platform_device *pdev)
 	info->controller.ops = &orion_nand_ops;
 	nc->controller = &info->controller;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	io_base = devm_ioremap_resource(&pdev->dev, res);
+	io_base = devm_platform_ioremap_resource(pdev, 0);
 
 	if (IS_ERR(io_base))
 		return PTR_ERR(io_base);
@@ -207,7 +205,7 @@ no_dev:
 	return ret;
 }
 
-static int orion_nand_remove(struct platform_device *pdev)
+static void orion_nand_remove(struct platform_device *pdev)
 {
 	struct orion_nand_info *info = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &info->chip;
@@ -219,8 +217,6 @@ static int orion_nand_remove(struct platform_device *pdev)
 	nand_cleanup(chip);
 
 	clk_disable_unprepare(info->clk);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
@@ -232,7 +228,7 @@ MODULE_DEVICE_TABLE(of, orion_nand_of_match_table);
 #endif
 
 static struct platform_driver orion_nand_driver = {
-	.remove		= orion_nand_remove,
+	.remove_new	= orion_nand_remove,
 	.driver		= {
 		.name	= "orion_nand",
 		.of_match_table = of_match_ptr(orion_nand_of_match_table),
diff --git a/drivers/mtd/nand/raw/oxnas_nand.c b/drivers/mtd/nand/raw/oxnas_nand.c
index cd112d45e0b5..e3c9807df1cd 100644
--- a/drivers/mtd/nand/raw/oxnas_nand.c
+++ b/drivers/mtd/nand/raw/oxnas_nand.c
@@ -171,7 +171,7 @@ err_clk_unprepare:
 	return err;
 }
 
-static int oxnas_nand_remove(struct platform_device *pdev)
+static void oxnas_nand_remove(struct platform_device *pdev)
 {
 	struct oxnas_nand_ctrl *oxnas = platform_get_drvdata(pdev);
 	struct nand_chip *chip;
@@ -184,8 +184,6 @@ static int oxnas_nand_remove(struct platform_device *pdev)
 	}
 
 	clk_disable_unprepare(oxnas->clk);
-
-	return 0;
 }
 
 static const struct of_device_id oxnas_nand_match[] = {
@@ -196,7 +194,7 @@ MODULE_DEVICE_TABLE(of, oxnas_nand_match);
 
 static struct platform_driver oxnas_nand_driver = {
 	.probe	= oxnas_nand_probe,
-	.remove	= oxnas_nand_remove,
+	.remove_new = oxnas_nand_remove,
 	.driver	= {
 		.name		= "oxnas_nand",
 		.of_match_table = oxnas_nand_match,
diff --git a/drivers/mtd/nand/raw/pasemi_nand.c b/drivers/mtd/nand/raw/pasemi_nand.c
index f7ef6ca06ca9..19b2c9d25863 100644
--- a/drivers/mtd/nand/raw/pasemi_nand.c
+++ b/drivers/mtd/nand/raw/pasemi_nand.c
@@ -197,7 +197,7 @@ static int pasemi_nand_probe(struct platform_device *ofdev)
 	return err;
 }
 
-static int pasemi_nand_remove(struct platform_device *ofdev)
+static void pasemi_nand_remove(struct platform_device *ofdev)
 {
 	struct pasemi_ddata *ddata = platform_get_drvdata(ofdev);
 	struct mtd_info *pasemi_nand_mtd;
@@ -218,8 +218,6 @@ static int pasemi_nand_remove(struct platform_device *ofdev)
 
 	/* Free the MTD device structure */
 	kfree(ddata);
-
-	return 0;
 }
 
 static const struct of_device_id pasemi_nand_match[] =
@@ -239,7 +237,7 @@ static struct platform_driver pasemi_nand_driver =
 		.of_match_table = pasemi_nand_match,
 	},
 	.probe		= pasemi_nand_probe,
-	.remove		= pasemi_nand_remove,
+	.remove_new	= pasemi_nand_remove,
 };
 
 module_platform_driver(pasemi_nand_driver);
diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c
index 3c6f6aff649f..28b7bd7e22eb 100644
--- a/drivers/mtd/nand/raw/pl35x-nand-controller.c
+++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c
@@ -1163,13 +1163,11 @@ static int pl35x_nand_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int pl35x_nand_remove(struct platform_device *pdev)
+static void pl35x_nand_remove(struct platform_device *pdev)
 {
 	struct pl35x_nandc *nfc = platform_get_drvdata(pdev);
 
 	pl35x_nand_chips_cleanup(nfc);
-
-	return 0;
 }
 
 static const struct of_device_id pl35x_nand_of_match[] = {
@@ -1180,7 +1178,7 @@ MODULE_DEVICE_TABLE(of, pl35x_nand_of_match);
 
 static struct platform_driver pl35x_nandc_driver = {
 	.probe = pl35x_nand_probe,
-	.remove	= pl35x_nand_remove,
+	.remove_new = pl35x_nand_remove,
 	.driver = {
 		.name = PL35X_NANDC_DRIVER_NAME,
 		.of_match_table = pl35x_nand_of_match,
diff --git a/drivers/mtd/nand/raw/plat_nand.c b/drivers/mtd/nand/raw/plat_nand.c
index 7e0d0a8dfd1e..b5c374b51ecd 100644
--- a/drivers/mtd/nand/raw/plat_nand.c
+++ b/drivers/mtd/nand/raw/plat_nand.c
@@ -122,7 +122,7 @@ out:
 /*
  * Remove a NAND device.
  */
-static int plat_nand_remove(struct platform_device *pdev)
+static void plat_nand_remove(struct platform_device *pdev)
 {
 	struct plat_nand_data *data = platform_get_drvdata(pdev);
 	struct platform_nand_data *pdata = dev_get_platdata(&pdev->dev);
@@ -134,8 +134,6 @@ static int plat_nand_remove(struct platform_device *pdev)
 	nand_cleanup(chip);
 	if (pdata->ctrl.remove)
 		pdata->ctrl.remove(pdev);
-
-	return 0;
 }
 
 static const struct of_device_id plat_nand_match[] = {
@@ -146,7 +144,7 @@ MODULE_DEVICE_TABLE(of, plat_nand_match);
 
 static struct platform_driver plat_nand_driver = {
 	.probe	= plat_nand_probe,
-	.remove	= plat_nand_remove,
+	.remove_new = plat_nand_remove,
 	.driver	= {
 		.name		= "gen_nand",
 		.of_match_table = plat_nand_match,
diff --git a/drivers/mtd/nand/raw/qcom_nandc.c b/drivers/mtd/nand/raw/qcom_nandc.c
index 198a44794d2d..72d6168d8a1b 100644
--- a/drivers/mtd/nand/raw/qcom_nandc.c
+++ b/drivers/mtd/nand/raw/qcom_nandc.c
@@ -3054,7 +3054,7 @@ static int qcom_nand_host_parse_boot_partitions(struct qcom_nand_controller *nan
 	struct device *dev = nandc->dev;
 	int partitions_count, i, j, ret;
 
-	if (!of_find_property(dn, "qcom,boot-partitions", NULL))
+	if (!of_property_present(dn, "qcom,boot-partitions"))
 		return 0;
 
 	partitions_count = of_property_count_u32_elems(dn, "qcom,boot-partitions");
@@ -3269,8 +3269,7 @@ static int qcom_nandc_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	nandc->base = devm_ioremap_resource(dev, res);
+	nandc->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(nandc->base))
 		return PTR_ERR(nandc->base);
 
@@ -3315,7 +3314,7 @@ err_core_clk:
 	return ret;
 }
 
-static int qcom_nandc_remove(struct platform_device *pdev)
+static void qcom_nandc_remove(struct platform_device *pdev)
 {
 	struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
 	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -3337,8 +3336,6 @@ static int qcom_nandc_remove(struct platform_device *pdev)
 
 	dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res),
 			   DMA_BIDIRECTIONAL, 0);
-
-	return 0;
 }
 
 static const struct qcom_nandc_props ipq806x_nandc_props = {
@@ -3405,7 +3402,7 @@ static struct platform_driver qcom_nandc_driver = {
 		.of_match_table = qcom_nandc_of_match,
 	},
 	.probe   = qcom_nandc_probe,
-	.remove  = qcom_nandc_remove,
+	.remove_new = qcom_nandc_remove,
 };
 module_platform_driver(qcom_nandc_driver);
 
diff --git a/drivers/mtd/nand/raw/renesas-nand-controller.c b/drivers/mtd/nand/raw/renesas-nand-controller.c
index 1620e25a1147..589021ea9eb2 100644
--- a/drivers/mtd/nand/raw/renesas-nand-controller.c
+++ b/drivers/mtd/nand/raw/renesas-nand-controller.c
@@ -1386,15 +1386,13 @@ dis_runtime_pm:
 	return ret;
 }
 
-static int rnandc_remove(struct platform_device *pdev)
+static void rnandc_remove(struct platform_device *pdev)
 {
 	struct rnandc *rnandc = platform_get_drvdata(pdev);
 
 	rnandc_chips_cleanup(rnandc);
 
 	pm_runtime_put(&pdev->dev);
-
-	return 0;
 }
 
 static const struct of_device_id rnandc_id_table[] = {
@@ -1410,7 +1408,7 @@ static struct platform_driver rnandc_driver = {
 		.of_match_table = rnandc_id_table,
 	},
 	.probe = rnandc_probe,
-	.remove = rnandc_remove,
+	.remove_new = rnandc_remove,
 };
 module_platform_driver(rnandc_driver);
 
diff --git a/drivers/mtd/nand/raw/rockchip-nand-controller.c b/drivers/mtd/nand/raw/rockchip-nand-controller.c
index f133985cc053..2312e27362cb 100644
--- a/drivers/mtd/nand/raw/rockchip-nand-controller.c
+++ b/drivers/mtd/nand/raw/rockchip-nand-controller.c
@@ -1427,7 +1427,7 @@ release_nfc:
 	return ret;
 }
 
-static int rk_nfc_remove(struct platform_device *pdev)
+static void rk_nfc_remove(struct platform_device *pdev)
 {
 	struct rk_nfc *nfc = platform_get_drvdata(pdev);
 
@@ -1435,8 +1435,6 @@ static int rk_nfc_remove(struct platform_device *pdev)
 	kfree(nfc->oob_buf);
 	rk_nfc_chips_cleanup(nfc);
 	rk_nfc_disable_clks(nfc);
-
-	return 0;
 }
 
 static int __maybe_unused rk_nfc_suspend(struct device *dev)
@@ -1476,7 +1474,7 @@ static const struct dev_pm_ops rk_nfc_pm_ops = {
 
 static struct platform_driver rk_nfc_driver = {
 	.probe = rk_nfc_probe,
-	.remove = rk_nfc_remove,
+	.remove_new = rk_nfc_remove,
 	.driver = {
 		.name = "rockchip-nfc",
 		.of_match_table = rk_nfc_id_table,
diff --git a/drivers/mtd/nand/raw/s3c2410.c b/drivers/mtd/nand/raw/s3c2410.c
index 80d96f94d6cb..ac80aaf5b4e3 100644
--- a/drivers/mtd/nand/raw/s3c2410.c
+++ b/drivers/mtd/nand/raw/s3c2410.c
@@ -709,12 +709,12 @@ static void s3c2440_nand_write_buf(struct nand_chip *this, const u_char *buf,
 
 /* device management functions */
 
-static int s3c24xx_nand_remove(struct platform_device *pdev)
+static void s3c24xx_nand_remove(struct platform_device *pdev)
 {
 	struct s3c2410_nand_info *info = to_nand_info(pdev);
 
 	if (info == NULL)
-		return 0;
+		return;
 
 	/* Release all our mtds  and their partitions, then go through
 	 * freeing the resources used
@@ -735,8 +735,6 @@ static int s3c24xx_nand_remove(struct platform_device *pdev)
 
 	if (!IS_ERR(info->clk))
 		s3c2410_nand_clk_set_state(info, CLOCK_DISABLE);
-
-	return 0;
 }
 
 static int s3c2410_nand_add_partition(struct s3c2410_nand_info *info,
@@ -1218,7 +1216,7 @@ MODULE_DEVICE_TABLE(platform, s3c24xx_driver_ids);
 
 static struct platform_driver s3c24xx_nand_driver = {
 	.probe		= s3c24xx_nand_probe,
-	.remove		= s3c24xx_nand_remove,
+	.remove_new	= s3c24xx_nand_remove,
 	.suspend	= s3c24xx_nand_suspend,
 	.resume		= s3c24xx_nand_resume,
 	.id_table	= s3c24xx_driver_ids,
diff --git a/drivers/mtd/nand/raw/sh_flctl.c b/drivers/mtd/nand/raw/sh_flctl.c
index a278829469d6..63bf20c41719 100644
--- a/drivers/mtd/nand/raw/sh_flctl.c
+++ b/drivers/mtd/nand/raw/sh_flctl.c
@@ -1203,7 +1203,7 @@ err_chip:
 	return ret;
 }
 
-static int flctl_remove(struct platform_device *pdev)
+static void flctl_remove(struct platform_device *pdev)
 {
 	struct sh_flctl *flctl = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &flctl->chip;
@@ -1214,12 +1214,10 @@ static int flctl_remove(struct platform_device *pdev)
 	WARN_ON(ret);
 	nand_cleanup(chip);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static struct platform_driver flctl_driver = {
-	.remove		= flctl_remove,
+	.remove_new	= flctl_remove,
 	.driver = {
 		.name	= "sh_flctl",
 		.of_match_table = of_flctl_match,
diff --git a/drivers/mtd/nand/raw/sharpsl.c b/drivers/mtd/nand/raw/sharpsl.c
index 52ce5162538a..2402dc5465d5 100644
--- a/drivers/mtd/nand/raw/sharpsl.c
+++ b/drivers/mtd/nand/raw/sharpsl.c
@@ -210,7 +210,7 @@ err_get_res:
 /*
  * Clean up routine
  */
-static int sharpsl_nand_remove(struct platform_device *pdev)
+static void sharpsl_nand_remove(struct platform_device *pdev)
 {
 	struct sharpsl_nand *sharpsl = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &sharpsl->chip;
@@ -227,8 +227,6 @@ static int sharpsl_nand_remove(struct platform_device *pdev)
 
 	/* Free the driver's structure */
 	kfree(sharpsl);
-
-	return 0;
 }
 
 static struct platform_driver sharpsl_nand_driver = {
@@ -236,7 +234,7 @@ static struct platform_driver sharpsl_nand_driver = {
 		.name	= "sharpsl-nand",
 	},
 	.probe		= sharpsl_nand_probe,
-	.remove		= sharpsl_nand_remove,
+	.remove_new	= sharpsl_nand_remove,
 };
 
 module_platform_driver(sharpsl_nand_driver);
diff --git a/drivers/mtd/nand/raw/socrates_nand.c b/drivers/mtd/nand/raw/socrates_nand.c
index fb39cc7ebce0..a8b720ffe9e8 100644
--- a/drivers/mtd/nand/raw/socrates_nand.c
+++ b/drivers/mtd/nand/raw/socrates_nand.c
@@ -201,7 +201,7 @@ out:
 /*
  * Remove a NAND device.
  */
-static int socrates_nand_remove(struct platform_device *ofdev)
+static void socrates_nand_remove(struct platform_device *ofdev)
 {
 	struct socrates_nand_host *host = dev_get_drvdata(&ofdev->dev);
 	struct nand_chip *chip = &host->nand_chip;
@@ -212,8 +212,6 @@ static int socrates_nand_remove(struct platform_device *ofdev)
 	nand_cleanup(chip);
 
 	iounmap(host->io_base);
-
-	return 0;
 }
 
 static const struct of_device_id socrates_nand_match[] =
@@ -232,7 +230,7 @@ static struct platform_driver socrates_nand_driver = {
 		.of_match_table = socrates_nand_match,
 	},
 	.probe		= socrates_nand_probe,
-	.remove		= socrates_nand_remove,
+	.remove_new	= socrates_nand_remove,
 };
 
 module_platform_driver(socrates_nand_driver);
diff --git a/drivers/mtd/nand/raw/stm32_fmc2_nand.c b/drivers/mtd/nand/raw/stm32_fmc2_nand.c
index 9e74bcd90aaa..10c11cecac08 100644
--- a/drivers/mtd/nand/raw/stm32_fmc2_nand.c
+++ b/drivers/mtd/nand/raw/stm32_fmc2_nand.c
@@ -2024,7 +2024,7 @@ err_clk_disable:
 	return ret;
 }
 
-static int stm32_fmc2_nfc_remove(struct platform_device *pdev)
+static void stm32_fmc2_nfc_remove(struct platform_device *pdev)
 {
 	struct stm32_fmc2_nfc *nfc = platform_get_drvdata(pdev);
 	struct stm32_fmc2_nand *nand = &nfc->nand;
@@ -2048,8 +2048,6 @@ static int stm32_fmc2_nfc_remove(struct platform_device *pdev)
 	clk_disable_unprepare(nfc->clk);
 
 	stm32_fmc2_nfc_wp_enable(nand);
-
-	return 0;
 }
 
 static int __maybe_unused stm32_fmc2_nfc_suspend(struct device *dev)
@@ -2106,7 +2104,7 @@ MODULE_DEVICE_TABLE(of, stm32_fmc2_nfc_match);
 
 static struct platform_driver stm32_fmc2_nfc_driver = {
 	.probe	= stm32_fmc2_nfc_probe,
-	.remove	= stm32_fmc2_nfc_remove,
+	.remove_new = stm32_fmc2_nfc_remove,
 	.driver	= {
 		.name = "stm32_fmc2_nfc",
 		.of_match_table = stm32_fmc2_nfc_match,
diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c
index 13e3e0198d15..9884304634f6 100644
--- a/drivers/mtd/nand/raw/sunxi_nand.c
+++ b/drivers/mtd/nand/raw/sunxi_nand.c
@@ -2173,7 +2173,7 @@ out_ahb_clk_unprepare:
 	return ret;
 }
 
-static int sunxi_nfc_remove(struct platform_device *pdev)
+static void sunxi_nfc_remove(struct platform_device *pdev)
 {
 	struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
 
@@ -2185,8 +2185,6 @@ static int sunxi_nfc_remove(struct platform_device *pdev)
 		dma_release_channel(nfc->dmac);
 	clk_disable_unprepare(nfc->mod_clk);
 	clk_disable_unprepare(nfc->ahb_clk);
-
-	return 0;
 }
 
 static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = {
@@ -2219,7 +2217,7 @@ static struct platform_driver sunxi_nfc_driver = {
 		.of_match_table = sunxi_nfc_ids,
 	},
 	.probe = sunxi_nfc_probe,
-	.remove = sunxi_nfc_remove,
+	.remove_new = sunxi_nfc_remove,
 };
 module_platform_driver(sunxi_nfc_driver);
 
diff --git a/drivers/mtd/nand/raw/tegra_nand.c b/drivers/mtd/nand/raw/tegra_nand.c
index a9b9031ce616..eb0b9d16e8da 100644
--- a/drivers/mtd/nand/raw/tegra_nand.c
+++ b/drivers/mtd/nand/raw/tegra_nand.c
@@ -1220,7 +1220,7 @@ err_dis_pm:
 	return err;
 }
 
-static int tegra_nand_remove(struct platform_device *pdev)
+static void tegra_nand_remove(struct platform_device *pdev)
 {
 	struct tegra_nand_controller *ctrl = platform_get_drvdata(pdev);
 	struct nand_chip *chip = ctrl->chip;
@@ -1232,8 +1232,6 @@ static int tegra_nand_remove(struct platform_device *pdev)
 
 	pm_runtime_put_sync_suspend(ctrl->dev);
 	pm_runtime_force_suspend(ctrl->dev);
-
-	return 0;
 }
 
 static int __maybe_unused tegra_nand_runtime_resume(struct device *dev)
@@ -1277,7 +1275,7 @@ static struct platform_driver tegra_nand_driver = {
 		.pm = &tegra_nand_pm,
 	},
 	.probe = tegra_nand_probe,
-	.remove = tegra_nand_remove,
+	.remove_new = tegra_nand_remove,
 };
 module_platform_driver(tegra_nand_driver);
 
diff --git a/drivers/mtd/nand/raw/vf610_nfc.c b/drivers/mtd/nand/raw/vf610_nfc.c
index b643332ea1ff..86522048e271 100644
--- a/drivers/mtd/nand/raw/vf610_nfc.c
+++ b/drivers/mtd/nand/raw/vf610_nfc.c
@@ -909,7 +909,7 @@ err_disable_clk:
 	return err;
 }
 
-static int vf610_nfc_remove(struct platform_device *pdev)
+static void vf610_nfc_remove(struct platform_device *pdev)
 {
 	struct vf610_nfc *nfc = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &nfc->chip;
@@ -919,7 +919,6 @@ static int vf610_nfc_remove(struct platform_device *pdev)
 	WARN_ON(ret);
 	nand_cleanup(chip);
 	clk_disable_unprepare(nfc->clk);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -955,7 +954,7 @@ static struct platform_driver vf610_nfc_driver = {
 		.pm	= &vf610_nfc_pm_ops,
 	},
 	.probe		= vf610_nfc_probe,
-	.remove		= vf610_nfc_remove,
+	.remove_new	= vf610_nfc_remove,
 };
 
 module_platform_driver(vf610_nfc_driver);
diff --git a/drivers/mtd/nand/raw/xway_nand.c b/drivers/mtd/nand/raw/xway_nand.c
index 035b82aa2f4a..6b1e2a2bba15 100644
--- a/drivers/mtd/nand/raw/xway_nand.c
+++ b/drivers/mtd/nand/raw/xway_nand.c
@@ -238,7 +238,7 @@ static int xway_nand_probe(struct platform_device *pdev)
 /*
  * Remove a NAND device.
  */
-static int xway_nand_remove(struct platform_device *pdev)
+static void xway_nand_remove(struct platform_device *pdev)
 {
 	struct xway_nand_data *data = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &data->chip;
@@ -247,8 +247,6 @@ static int xway_nand_remove(struct platform_device *pdev)
 	ret = mtd_device_unregister(nand_to_mtd(chip));
 	WARN_ON(ret);
 	nand_cleanup(chip);
-
-	return 0;
 }
 
 static const struct of_device_id xway_nand_match[] = {
@@ -258,7 +256,7 @@ static const struct of_device_id xway_nand_match[] = {
 
 static struct platform_driver xway_nand_driver = {
 	.probe	= xway_nand_probe,
-	.remove	= xway_nand_remove,
+	.remove_new = xway_nand_remove,
 	.driver	= {
 		.name		= "lantiq,nand-xway",
 		.of_match_table = xway_nand_match,
diff --git a/drivers/mtd/nand/spi/Makefile b/drivers/mtd/nand/spi/Makefile
index 4ec973b8b6bf..cd8b66bf7740 100644
--- a/drivers/mtd/nand/spi/Makefile
+++ b/drivers/mtd/nand/spi/Makefile
@@ -1,3 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-spinand-objs := core.o alliancememory.o ato.o gigadevice.o macronix.o micron.o paragon.o toshiba.o winbond.o xtx.o
+spinand-objs := core.o alliancememory.o ato.o esmt.o gigadevice.o macronix.o
+spinand-objs += micron.o paragon.o toshiba.o winbond.o xtx.o
 obj-$(CONFIG_MTD_SPI_NAND) += spinand.o
diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c
index 638391f77d8c..393ff37f0d23 100644
--- a/drivers/mtd/nand/spi/core.c
+++ b/drivers/mtd/nand/spi/core.c
@@ -939,6 +939,7 @@ static const struct nand_ops spinand_ops = {
 static const struct spinand_manufacturer *spinand_manufacturers[] = {
 	&alliancememory_spinand_manufacturer,
 	&ato_spinand_manufacturer,
+	&esmt_c8_spinand_manufacturer,
 	&gigadevice_spinand_manufacturer,
 	&macronix_spinand_manufacturer,
 	&micron_spinand_manufacturer,
diff --git a/drivers/mtd/nand/spi/esmt.c b/drivers/mtd/nand/spi/esmt.c
new file mode 100644
index 000000000000..1a3ffb982335
--- /dev/null
+++ b/drivers/mtd/nand/spi/esmt.c
@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Author:
+ *	Chuanhong Guo <gch981213@gmail.com> - the main driver logic
+ *	Martin Kurbanov <mmkurbanov@sberdevices.ru> - OOB layout
+ */
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/mtd/spinand.h>
+
+/* ESMT uses GigaDevice 0xc8 JECDEC ID on some SPI NANDs */
+#define SPINAND_MFR_ESMT_C8			0xc8
+
+static SPINAND_OP_VARIANTS(read_cache_variants,
+			   SPINAND_PAGE_READ_FROM_CACHE_X4_OP(0, 1, NULL, 0),
+			   SPINAND_PAGE_READ_FROM_CACHE_X2_OP(0, 1, NULL, 0),
+			   SPINAND_PAGE_READ_FROM_CACHE_OP(true, 0, 1, NULL, 0),
+			   SPINAND_PAGE_READ_FROM_CACHE_OP(false, 0, 1, NULL, 0));
+
+static SPINAND_OP_VARIANTS(write_cache_variants,
+			   SPINAND_PROG_LOAD_X4(true, 0, NULL, 0),
+			   SPINAND_PROG_LOAD(true, 0, NULL, 0));
+
+static SPINAND_OP_VARIANTS(update_cache_variants,
+			   SPINAND_PROG_LOAD_X4(false, 0, NULL, 0),
+			   SPINAND_PROG_LOAD(false, 0, NULL, 0));
+
+/*
+ * OOB spare area map (64 bytes)
+ *
+ * Bad Block Markers
+ * filled by HW and kernel                 Reserved
+ *   |                 +-----------------------+-----------------------+
+ *   |                 |                       |                       |
+ *   |                 |    OOB free data Area |non ECC protected      |
+ *   |   +-------------|-----+-----------------|-----+-----------------|-----+
+ *   |   |             |     |                 |     |                 |     |
+ * +-|---|----------+--|-----|--------------+--|-----|--------------+--|-----|--------------+
+ * | |   | section0 |  |     |    section1  |  |     |    section2  |  |     |    section3  |
+ * +-v-+-v-+---+----+--v--+--v--+-----+-----+--v--+--v--+-----+-----+--v--+--v--+-----+-----+
+ * |   |   |   |    |     |     |     |     |     |     |     |     |     |     |     |     |
+ * |0:1|2:3|4:7|8:15|16:17|18:19|20:23|24:31|32:33|34:35|36:39|40:47|48:49|50:51|52:55|56:63|
+ * |   |   |   |    |     |     |     |     |     |     |     |     |     |     |     |     |
+ * +---+---+-^-+--^-+-----+-----+--^--+--^--+-----+-----+--^--+--^--+-----+-----+--^--+--^--+
+ *           |    |                |     |                 |     |                 |     |
+ *           |    +----------------|-----+-----------------|-----+-----------------|-----+
+ *           |             ECC Area|(Main + Spare) - filled|by ESMT NAND HW        |
+ *           |                     |                       |                       |
+ *           +---------------------+-----------------------+-----------------------+
+ *                         OOB ECC protected Area - not used due to
+ *                         partial programming from some filesystems
+ *                             (like JFFS2 with cleanmarkers)
+ */
+
+#define ESMT_OOB_SECTION_COUNT			4
+#define ESMT_OOB_SECTION_SIZE(nand) \
+	(nanddev_per_page_oobsize(nand) / ESMT_OOB_SECTION_COUNT)
+#define ESMT_OOB_FREE_SIZE(nand) \
+	(ESMT_OOB_SECTION_SIZE(nand) / 2)
+#define ESMT_OOB_ECC_SIZE(nand) \
+	(ESMT_OOB_SECTION_SIZE(nand) - ESMT_OOB_FREE_SIZE(nand))
+#define ESMT_OOB_BBM_SIZE			2
+
+static int f50l1g41lb_ooblayout_ecc(struct mtd_info *mtd, int section,
+				    struct mtd_oob_region *region)
+{
+	struct nand_device *nand = mtd_to_nanddev(mtd);
+
+	if (section >= ESMT_OOB_SECTION_COUNT)
+		return -ERANGE;
+
+	region->offset = section * ESMT_OOB_SECTION_SIZE(nand) +
+			 ESMT_OOB_FREE_SIZE(nand);
+	region->length = ESMT_OOB_ECC_SIZE(nand);
+
+	return 0;
+}
+
+static int f50l1g41lb_ooblayout_free(struct mtd_info *mtd, int section,
+				     struct mtd_oob_region *region)
+{
+	struct nand_device *nand = mtd_to_nanddev(mtd);
+
+	if (section >= ESMT_OOB_SECTION_COUNT)
+		return -ERANGE;
+
+	/*
+	 * Reserve space for bad blocks markers (section0) and
+	 * reserved bytes (sections 1-3)
+	 */
+	region->offset = section * ESMT_OOB_SECTION_SIZE(nand) + 2;
+
+	/* Use only 2 non-protected ECC bytes per each OOB section */
+	region->length = 2;
+
+	return 0;
+}
+
+static const struct mtd_ooblayout_ops f50l1g41lb_ooblayout = {
+	.ecc = f50l1g41lb_ooblayout_ecc,
+	.free = f50l1g41lb_ooblayout_free,
+};
+
+static const struct spinand_info esmt_c8_spinand_table[] = {
+	SPINAND_INFO("F50L1G41LB",
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_ADDR, 0x01),
+		     NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1),
+		     NAND_ECCREQ(1, 512),
+		     SPINAND_INFO_OP_VARIANTS(&read_cache_variants,
+					      &write_cache_variants,
+					      &update_cache_variants),
+		     0,
+		     SPINAND_ECCINFO(&f50l1g41lb_ooblayout, NULL)),
+	SPINAND_INFO("F50D1G41LB",
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_ADDR, 0x11),
+		     NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1),
+		     NAND_ECCREQ(1, 512),
+		     SPINAND_INFO_OP_VARIANTS(&read_cache_variants,
+					      &write_cache_variants,
+					      &update_cache_variants),
+		     0,
+		     SPINAND_ECCINFO(&f50l1g41lb_ooblayout, NULL)),
+};
+
+static const struct spinand_manufacturer_ops esmt_spinand_manuf_ops = {
+};
+
+const struct spinand_manufacturer esmt_c8_spinand_manufacturer = {
+	.id = SPINAND_MFR_ESMT_C8,
+	.name = "ESMT",
+	.chips = esmt_c8_spinand_table,
+	.nchips = ARRAY_SIZE(esmt_c8_spinand_table),
+	.ops = &esmt_spinand_manuf_ops,
+};
diff --git a/drivers/mtd/parsers/Kconfig b/drivers/mtd/parsers/Kconfig
index b20e0c38b517..60738edcd5d5 100644
--- a/drivers/mtd/parsers/Kconfig
+++ b/drivers/mtd/parsers/Kconfig
@@ -149,7 +149,7 @@ config MTD_PARSER_TRX
 
 config MTD_SHARPSL_PARTS
 	tristate "Sharp SL Series NAND flash partition parser"
-	depends on MTD_NAND_SHARPSL || MTD_NAND_TMIO || COMPILE_TEST
+	depends on MTD_NAND_SHARPSL || COMPILE_TEST
 	help
 	  This provides the read-only FTL logic necessary to read the partition
 	  table from the NAND flash of Sharp SL Series (Zaurus) and the MTD
diff --git a/drivers/mtd/parsers/bcm63xxpart.c b/drivers/mtd/parsers/bcm63xxpart.c
index b15bdadaedb5..6e7b1ae8f58f 100644
--- a/drivers/mtd/parsers/bcm63xxpart.c
+++ b/drivers/mtd/parsers/bcm63xxpart.c
@@ -164,7 +164,6 @@ static struct mtd_part_parser bcm63xx_cfe_parser = {
 };
 module_mtd_part_parser(bcm63xx_cfe_parser);
 
-MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Daniel Dickinson <openwrt@cshore.neomailbox.net>");
 MODULE_AUTHOR("Florian Fainelli <florian@openwrt.org>");
 MODULE_AUTHOR("Mike Albon <malbon@openwrt.org>");
diff --git a/drivers/mtd/spi-nor/controllers/nxp-spifi.c b/drivers/mtd/spi-nor/controllers/nxp-spifi.c
index ab3990e6ac25..794c7b7d5c92 100644
--- a/drivers/mtd/spi-nor/controllers/nxp-spifi.c
+++ b/drivers/mtd/spi-nor/controllers/nxp-spifi.c
@@ -305,10 +305,10 @@ static int nxp_spifi_setup_flash(struct nxp_spifi *spifi,
 		}
 	}
 
-	if (of_find_property(np, "spi-cpha", NULL))
+	if (of_property_read_bool(np, "spi-cpha"))
 		mode |= SPI_CPHA;
 
-	if (of_find_property(np, "spi-cpol", NULL))
+	if (of_property_read_bool(np, "spi-cpol"))
 		mode |= SPI_CPOL;
 
 	/* Setup control register defaults */
diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c
index 522d375aeccf..0bb0ad14a2fc 100644
--- a/drivers/mtd/spi-nor/core.c
+++ b/drivers/mtd/spi-nor/core.c
@@ -508,14 +508,16 @@ int spi_nor_read_cr(struct spi_nor *nor, u8 *cr)
 }
 
 /**
- * spi_nor_set_4byte_addr_mode() - Enter/Exit 4-byte address mode.
+ * spi_nor_set_4byte_addr_mode_en4b_ex4b() - Enter/Exit 4-byte address mode
+ *			using SPINOR_OP_EN4B/SPINOR_OP_EX4B. Typically used by
+ *			Winbond and Macronix.
  * @nor:	pointer to 'struct spi_nor'.
  * @enable:	true to enter the 4-byte address mode, false to exit the 4-byte
  *		address mode.
  *
  * Return: 0 on success, -errno otherwise.
  */
-int spi_nor_set_4byte_addr_mode(struct spi_nor *nor, bool enable)
+int spi_nor_set_4byte_addr_mode_en4b_ex4b(struct spi_nor *nor, bool enable)
 {
 	int ret;
 
@@ -539,15 +541,45 @@ int spi_nor_set_4byte_addr_mode(struct spi_nor *nor, bool enable)
 }
 
 /**
- * spansion_set_4byte_addr_mode() - Set 4-byte address mode for Spansion
- * flashes.
+ * spi_nor_set_4byte_addr_mode_wren_en4b_ex4b() - Set 4-byte address mode using
+ * SPINOR_OP_WREN followed by SPINOR_OP_EN4B or SPINOR_OP_EX4B. Typically used
+ * by ST and Micron flashes.
+ * @nor:	pointer to 'struct spi_nor'.
+ * @enable:	true to enter the 4-byte address mode, false to exit the 4-byte
+ *		address mode.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+int spi_nor_set_4byte_addr_mode_wren_en4b_ex4b(struct spi_nor *nor, bool enable)
+{
+	int ret;
+
+	ret = spi_nor_write_enable(nor);
+	if (ret)
+		return ret;
+
+	ret = spi_nor_set_4byte_addr_mode_en4b_ex4b(nor, enable);
+	if (ret)
+		return ret;
+
+	return spi_nor_write_disable(nor);
+}
+
+/**
+ * spi_nor_set_4byte_addr_mode_brwr() - Set 4-byte address mode using
+ *			SPINOR_OP_BRWR. Typically used by Spansion flashes.
  * @nor:	pointer to 'struct spi_nor'.
  * @enable:	true to enter the 4-byte address mode, false to exit the 4-byte
  *		address mode.
  *
+ * 8-bit volatile bank register used to define A[30:A24] bits. MSB (bit[7]) is
+ * used to enable/disable 4-byte address mode. When MSB is set to ‘1’, 4-byte
+ * address mode is active and A[30:24] bits are don’t care. Write instruction is
+ * SPINOR_OP_BRWR(17h) with 1 byte of data.
+ *
  * Return: 0 on success, -errno otherwise.
  */
-static int spansion_set_4byte_addr_mode(struct spi_nor *nor, bool enable)
+int spi_nor_set_4byte_addr_mode_brwr(struct spi_nor *nor, bool enable)
 {
 	int ret;
 
@@ -589,6 +621,65 @@ int spi_nor_sr_ready(struct spi_nor *nor)
 }
 
 /**
+ * spi_nor_use_parallel_locking() - Checks if RWW locking scheme shall be used
+ * @nor:	pointer to 'struct spi_nor'.
+ *
+ * Return: true if parallel locking is enabled, false otherwise.
+ */
+static bool spi_nor_use_parallel_locking(struct spi_nor *nor)
+{
+	return nor->flags & SNOR_F_RWW;
+}
+
+/* Locking helpers for status read operations */
+static int spi_nor_rww_start_rdst(struct spi_nor *nor)
+{
+	struct spi_nor_rww *rww = &nor->rww;
+	int ret = -EAGAIN;
+
+	mutex_lock(&nor->lock);
+
+	if (rww->ongoing_io || rww->ongoing_rd)
+		goto busy;
+
+	rww->ongoing_io = true;
+	rww->ongoing_rd = true;
+	ret = 0;
+
+busy:
+	mutex_unlock(&nor->lock);
+	return ret;
+}
+
+static void spi_nor_rww_end_rdst(struct spi_nor *nor)
+{
+	struct spi_nor_rww *rww = &nor->rww;
+
+	mutex_lock(&nor->lock);
+
+	rww->ongoing_io = false;
+	rww->ongoing_rd = false;
+
+	mutex_unlock(&nor->lock);
+}
+
+static int spi_nor_lock_rdst(struct spi_nor *nor)
+{
+	if (spi_nor_use_parallel_locking(nor))
+		return spi_nor_rww_start_rdst(nor);
+
+	return 0;
+}
+
+static void spi_nor_unlock_rdst(struct spi_nor *nor)
+{
+	if (spi_nor_use_parallel_locking(nor)) {
+		spi_nor_rww_end_rdst(nor);
+		wake_up(&nor->rww.wait);
+	}
+}
+
+/**
  * spi_nor_ready() - Query the flash to see if it is ready for new commands.
  * @nor:	pointer to 'struct spi_nor'.
  *
@@ -596,11 +687,21 @@ int spi_nor_sr_ready(struct spi_nor *nor)
  */
 static int spi_nor_ready(struct spi_nor *nor)
 {
+	int ret;
+
+	ret = spi_nor_lock_rdst(nor);
+	if (ret)
+		return 0;
+
 	/* Flashes might override the standard routine. */
 	if (nor->params->ready)
-		return nor->params->ready(nor);
+		ret = nor->params->ready(nor);
+	else
+		ret = spi_nor_sr_ready(nor);
+
+	spi_nor_unlock_rdst(nor);
 
-	return spi_nor_sr_ready(nor);
+	return ret;
 }
 
 /**
@@ -1070,29 +1171,289 @@ static void spi_nor_set_4byte_opcodes(struct spi_nor *nor)
 	}
 }
 
-int spi_nor_lock_and_prep(struct spi_nor *nor)
+static int spi_nor_prep(struct spi_nor *nor)
 {
 	int ret = 0;
 
+	if (nor->controller_ops && nor->controller_ops->prepare)
+		ret = nor->controller_ops->prepare(nor);
+
+	return ret;
+}
+
+static void spi_nor_unprep(struct spi_nor *nor)
+{
+	if (nor->controller_ops && nor->controller_ops->unprepare)
+		nor->controller_ops->unprepare(nor);
+}
+
+static void spi_nor_offset_to_banks(u64 bank_size, loff_t start, size_t len,
+				    u8 *first, u8 *last)
+{
+	/* This is currently safe, the number of banks being very small */
+	*first = DIV_ROUND_DOWN_ULL(start, bank_size);
+	*last = DIV_ROUND_DOWN_ULL(start + len - 1, bank_size);
+}
+
+/* Generic helpers for internal locking and serialization */
+static bool spi_nor_rww_start_io(struct spi_nor *nor)
+{
+	struct spi_nor_rww *rww = &nor->rww;
+	bool start = false;
+
 	mutex_lock(&nor->lock);
 
-	if (nor->controller_ops &&  nor->controller_ops->prepare) {
-		ret = nor->controller_ops->prepare(nor);
-		if (ret) {
-			mutex_unlock(&nor->lock);
-			return ret;
-		}
+	if (rww->ongoing_io)
+		goto busy;
+
+	rww->ongoing_io = true;
+	start = true;
+
+busy:
+	mutex_unlock(&nor->lock);
+	return start;
+}
+
+static void spi_nor_rww_end_io(struct spi_nor *nor)
+{
+	mutex_lock(&nor->lock);
+	nor->rww.ongoing_io = false;
+	mutex_unlock(&nor->lock);
+}
+
+static int spi_nor_lock_device(struct spi_nor *nor)
+{
+	if (!spi_nor_use_parallel_locking(nor))
+		return 0;
+
+	return wait_event_killable(nor->rww.wait, spi_nor_rww_start_io(nor));
+}
+
+static void spi_nor_unlock_device(struct spi_nor *nor)
+{
+	if (spi_nor_use_parallel_locking(nor)) {
+		spi_nor_rww_end_io(nor);
+		wake_up(&nor->rww.wait);
 	}
+}
+
+/* Generic helpers for internal locking and serialization */
+static bool spi_nor_rww_start_exclusive(struct spi_nor *nor)
+{
+	struct spi_nor_rww *rww = &nor->rww;
+	bool start = false;
+
+	mutex_lock(&nor->lock);
+
+	if (rww->ongoing_io || rww->ongoing_rd || rww->ongoing_pe)
+		goto busy;
+
+	rww->ongoing_io = true;
+	rww->ongoing_rd = true;
+	rww->ongoing_pe = true;
+	start = true;
+
+busy:
+	mutex_unlock(&nor->lock);
+	return start;
+}
+
+static void spi_nor_rww_end_exclusive(struct spi_nor *nor)
+{
+	struct spi_nor_rww *rww = &nor->rww;
+
+	mutex_lock(&nor->lock);
+	rww->ongoing_io = false;
+	rww->ongoing_rd = false;
+	rww->ongoing_pe = false;
+	mutex_unlock(&nor->lock);
+}
+
+int spi_nor_prep_and_lock(struct spi_nor *nor)
+{
+	int ret;
+
+	ret = spi_nor_prep(nor);
+	if (ret)
+		return ret;
+
+	if (!spi_nor_use_parallel_locking(nor))
+		mutex_lock(&nor->lock);
+	else
+		ret = wait_event_killable(nor->rww.wait,
+					  spi_nor_rww_start_exclusive(nor));
+
 	return ret;
 }
 
 void spi_nor_unlock_and_unprep(struct spi_nor *nor)
 {
-	if (nor->controller_ops && nor->controller_ops->unprepare)
-		nor->controller_ops->unprepare(nor);
+	if (!spi_nor_use_parallel_locking(nor)) {
+		mutex_unlock(&nor->lock);
+	} else {
+		spi_nor_rww_end_exclusive(nor);
+		wake_up(&nor->rww.wait);
+	}
+
+	spi_nor_unprep(nor);
+}
+
+/* Internal locking helpers for program and erase operations */
+static bool spi_nor_rww_start_pe(struct spi_nor *nor, loff_t start, size_t len)
+{
+	struct spi_nor_rww *rww = &nor->rww;
+	unsigned int used_banks = 0;
+	bool started = false;
+	u8 first, last;
+	int bank;
+
+	mutex_lock(&nor->lock);
+
+	if (rww->ongoing_io || rww->ongoing_rd || rww->ongoing_pe)
+		goto busy;
+
+	spi_nor_offset_to_banks(nor->params->bank_size, start, len, &first, &last);
+	for (bank = first; bank <= last; bank++) {
+		if (rww->used_banks & BIT(bank))
+			goto busy;
+
+		used_banks |= BIT(bank);
+	}
+
+	rww->used_banks |= used_banks;
+	rww->ongoing_pe = true;
+	started = true;
+
+busy:
+	mutex_unlock(&nor->lock);
+	return started;
+}
+
+static void spi_nor_rww_end_pe(struct spi_nor *nor, loff_t start, size_t len)
+{
+	struct spi_nor_rww *rww = &nor->rww;
+	u8 first, last;
+	int bank;
+
+	mutex_lock(&nor->lock);
+
+	spi_nor_offset_to_banks(nor->params->bank_size, start, len, &first, &last);
+	for (bank = first; bank <= last; bank++)
+		rww->used_banks &= ~BIT(bank);
+
+	rww->ongoing_pe = false;
+
 	mutex_unlock(&nor->lock);
 }
 
+static int spi_nor_prep_and_lock_pe(struct spi_nor *nor, loff_t start, size_t len)
+{
+	int ret;
+
+	ret = spi_nor_prep(nor);
+	if (ret)
+		return ret;
+
+	if (!spi_nor_use_parallel_locking(nor))
+		mutex_lock(&nor->lock);
+	else
+		ret = wait_event_killable(nor->rww.wait,
+					  spi_nor_rww_start_pe(nor, start, len));
+
+	return ret;
+}
+
+static void spi_nor_unlock_and_unprep_pe(struct spi_nor *nor, loff_t start, size_t len)
+{
+	if (!spi_nor_use_parallel_locking(nor)) {
+		mutex_unlock(&nor->lock);
+	} else {
+		spi_nor_rww_end_pe(nor, start, len);
+		wake_up(&nor->rww.wait);
+	}
+
+	spi_nor_unprep(nor);
+}
+
+/* Internal locking helpers for read operations */
+static bool spi_nor_rww_start_rd(struct spi_nor *nor, loff_t start, size_t len)
+{
+	struct spi_nor_rww *rww = &nor->rww;
+	unsigned int used_banks = 0;
+	bool started = false;
+	u8 first, last;
+	int bank;
+
+	mutex_lock(&nor->lock);
+
+	if (rww->ongoing_io || rww->ongoing_rd)
+		goto busy;
+
+	spi_nor_offset_to_banks(nor->params->bank_size, start, len, &first, &last);
+	for (bank = first; bank <= last; bank++) {
+		if (rww->used_banks & BIT(bank))
+			goto busy;
+
+		used_banks |= BIT(bank);
+	}
+
+	rww->used_banks |= used_banks;
+	rww->ongoing_io = true;
+	rww->ongoing_rd = true;
+	started = true;
+
+busy:
+	mutex_unlock(&nor->lock);
+	return started;
+}
+
+static void spi_nor_rww_end_rd(struct spi_nor *nor, loff_t start, size_t len)
+{
+	struct spi_nor_rww *rww = &nor->rww;
+	u8 first, last;
+	int bank;
+
+	mutex_lock(&nor->lock);
+
+	spi_nor_offset_to_banks(nor->params->bank_size, start, len, &first, &last);
+	for (bank = first; bank <= last; bank++)
+		nor->rww.used_banks &= ~BIT(bank);
+
+	rww->ongoing_io = false;
+	rww->ongoing_rd = false;
+
+	mutex_unlock(&nor->lock);
+}
+
+static int spi_nor_prep_and_lock_rd(struct spi_nor *nor, loff_t start, size_t len)
+{
+	int ret;
+
+	ret = spi_nor_prep(nor);
+	if (ret)
+		return ret;
+
+	if (!spi_nor_use_parallel_locking(nor))
+		mutex_lock(&nor->lock);
+	else
+		ret = wait_event_killable(nor->rww.wait,
+					  spi_nor_rww_start_rd(nor, start, len));
+
+	return ret;
+}
+
+static void spi_nor_unlock_and_unprep_rd(struct spi_nor *nor, loff_t start, size_t len)
+{
+	if (!spi_nor_use_parallel_locking(nor)) {
+		mutex_unlock(&nor->lock);
+	} else {
+		spi_nor_rww_end_rd(nor, start, len);
+		wake_up(&nor->rww.wait);
+	}
+
+	spi_nor_unprep(nor);
+}
+
 static u32 spi_nor_convert_addr(struct spi_nor *nor, loff_t addr)
 {
 	if (!nor->params->convert_addr)
@@ -1397,11 +1758,18 @@ static int spi_nor_erase_multi_sectors(struct spi_nor *nor, u64 addr, u32 len)
 			dev_vdbg(nor->dev, "erase_cmd->size = 0x%08x, erase_cmd->opcode = 0x%02x, erase_cmd->count = %u\n",
 				 cmd->size, cmd->opcode, cmd->count);
 
-			ret = spi_nor_write_enable(nor);
+			ret = spi_nor_lock_device(nor);
 			if (ret)
 				goto destroy_erase_cmd_list;
 
+			ret = spi_nor_write_enable(nor);
+			if (ret) {
+				spi_nor_unlock_device(nor);
+				goto destroy_erase_cmd_list;
+			}
+
 			ret = spi_nor_erase_sector(nor, addr);
+			spi_nor_unlock_device(nor);
 			if (ret)
 				goto destroy_erase_cmd_list;
 
@@ -1446,7 +1814,7 @@ static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr)
 	addr = instr->addr;
 	len = instr->len;
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock_pe(nor, instr->addr, instr->len);
 	if (ret)
 		return ret;
 
@@ -1454,11 +1822,18 @@ static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr)
 	if (len == mtd->size && !(nor->flags & SNOR_F_NO_OP_CHIP_ERASE)) {
 		unsigned long timeout;
 
-		ret = spi_nor_write_enable(nor);
+		ret = spi_nor_lock_device(nor);
 		if (ret)
 			goto erase_err;
 
+		ret = spi_nor_write_enable(nor);
+		if (ret) {
+			spi_nor_unlock_device(nor);
+			goto erase_err;
+		}
+
 		ret = spi_nor_erase_chip(nor);
+		spi_nor_unlock_device(nor);
 		if (ret)
 			goto erase_err;
 
@@ -1483,11 +1858,18 @@ static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr)
 	/* "sector"-at-a-time erase */
 	} else if (spi_nor_has_uniform_erase(nor)) {
 		while (len) {
-			ret = spi_nor_write_enable(nor);
+			ret = spi_nor_lock_device(nor);
 			if (ret)
 				goto erase_err;
 
+			ret = spi_nor_write_enable(nor);
+			if (ret) {
+				spi_nor_unlock_device(nor);
+				goto erase_err;
+			}
+
 			ret = spi_nor_erase_sector(nor, addr);
+			spi_nor_unlock_device(nor);
 			if (ret)
 				goto erase_err;
 
@@ -1509,7 +1891,7 @@ static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr)
 	ret = spi_nor_write_disable(nor);
 
 erase_err:
-	spi_nor_unlock_and_unprep(nor);
+	spi_nor_unlock_and_unprep_pe(nor, instr->addr, instr->len);
 
 	return ret;
 }
@@ -1702,11 +2084,13 @@ static int spi_nor_read(struct mtd_info *mtd, loff_t from, size_t len,
 			size_t *retlen, u_char *buf)
 {
 	struct spi_nor *nor = mtd_to_spi_nor(mtd);
+	loff_t from_lock = from;
+	size_t len_lock = len;
 	ssize_t ret;
 
 	dev_dbg(nor->dev, "from 0x%08x, len %zd\n", (u32)from, len);
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock_rd(nor, from_lock, len_lock);
 	if (ret)
 		return ret;
 
@@ -1733,7 +2117,8 @@ static int spi_nor_read(struct mtd_info *mtd, loff_t from, size_t len,
 	ret = 0;
 
 read_err:
-	spi_nor_unlock_and_unprep(nor);
+	spi_nor_unlock_and_unprep_rd(nor, from_lock, len_lock);
+
 	return ret;
 }
 
@@ -1752,7 +2137,7 @@ static int spi_nor_write(struct mtd_info *mtd, loff_t to, size_t len,
 
 	dev_dbg(nor->dev, "to 0x%08x, len %zd\n", (u32)to, len);
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock_pe(nor, to, len);
 	if (ret)
 		return ret;
 
@@ -1777,11 +2162,18 @@ static int spi_nor_write(struct mtd_info *mtd, loff_t to, size_t len,
 
 		addr = spi_nor_convert_addr(nor, addr);
 
-		ret = spi_nor_write_enable(nor);
+		ret = spi_nor_lock_device(nor);
 		if (ret)
 			goto write_err;
 
+		ret = spi_nor_write_enable(nor);
+		if (ret) {
+			spi_nor_unlock_device(nor);
+			goto write_err;
+		}
+
 		ret = spi_nor_write_data(nor, addr, page_remain, buf + i);
+		spi_nor_unlock_device(nor);
 		if (ret < 0)
 			goto write_err;
 		written = ret;
@@ -1794,7 +2186,8 @@ static int spi_nor_write(struct mtd_info *mtd, loff_t to, size_t len,
 	}
 
 write_err:
-	spi_nor_unlock_and_unprep(nor);
+	spi_nor_unlock_and_unprep_pe(nor, to, len);
+
 	return ret;
 }
 
@@ -2470,6 +2863,10 @@ static void spi_nor_init_flags(struct spi_nor *nor)
 
 	if (flags & NO_CHIP_ERASE)
 		nor->flags |= SNOR_F_NO_OP_CHIP_ERASE;
+
+	if (flags & SPI_NOR_RWW && nor->info->n_banks > 1 &&
+	    !nor->controller_ops)
+		nor->flags |= SNOR_F_RWW;
 }
 
 /**
@@ -2501,6 +2898,8 @@ static void spi_nor_init_fixup_flags(struct spi_nor *nor)
  */
 static void spi_nor_late_init_params(struct spi_nor *nor)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
+
 	if (nor->manufacturer && nor->manufacturer->fixups &&
 	    nor->manufacturer->fixups->late_init)
 		nor->manufacturer->fixups->late_init(nor);
@@ -2508,6 +2907,10 @@ static void spi_nor_late_init_params(struct spi_nor *nor)
 	if (nor->info->fixups && nor->info->fixups->late_init)
 		nor->info->fixups->late_init(nor);
 
+	/* Default method kept for backward compatibility. */
+	if (!params->set_4byte_addr_mode)
+		params->set_4byte_addr_mode = spi_nor_set_4byte_addr_mode_brwr;
+
 	spi_nor_init_flags(nor);
 	spi_nor_init_fixup_flags(nor);
 
@@ -2517,6 +2920,8 @@ static void spi_nor_late_init_params(struct spi_nor *nor)
 	 */
 	if (nor->flags & SNOR_F_HAS_LOCK && !nor->params->locking_ops)
 		spi_nor_init_default_locking_ops(nor);
+
+	nor->params->bank_size = div64_u64(nor->params->size, nor->info->n_banks);
 }
 
 /**
@@ -2574,7 +2979,6 @@ static void spi_nor_init_default_params(struct spi_nor *nor)
 	struct device_node *np = spi_nor_get_flash_node(nor);
 
 	params->quad_enable = spi_nor_sr2_bit1_quad_enable;
-	params->set_4byte_addr_mode = spansion_set_4byte_addr_mode;
 	params->otp.org = &info->otp_org;
 
 	/* Default to 16-bit Write Status (01h) Command */
@@ -2730,6 +3134,33 @@ static int spi_nor_quad_enable(struct spi_nor *nor)
 	return nor->params->quad_enable(nor);
 }
 
+/**
+ * spi_nor_set_4byte_addr_mode() - Set address mode.
+ * @nor:                pointer to a 'struct spi_nor'.
+ * @enable:             enable/disable 4 byte address mode.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+int spi_nor_set_4byte_addr_mode(struct spi_nor *nor, bool enable)
+{
+	struct spi_nor_flash_parameter *params = nor->params;
+	int ret;
+
+	ret = params->set_4byte_addr_mode(nor, enable);
+	if (ret && ret != -ENOTSUPP)
+		return ret;
+
+	if (enable) {
+		params->addr_nbytes = 4;
+		params->addr_mode_nbytes = 4;
+	} else {
+		params->addr_nbytes = 3;
+		params->addr_mode_nbytes = 3;
+	}
+
+	return 0;
+}
+
 static int spi_nor_init(struct spi_nor *nor)
 {
 	int err;
@@ -2773,8 +3204,8 @@ static int spi_nor_init(struct spi_nor *nor)
 		 */
 		WARN_ONCE(nor->flags & SNOR_F_BROKEN_RESET,
 			  "enabling reset hack; may not recover from unexpected reboots\n");
-		err = nor->params->set_4byte_addr_mode(nor, true);
-		if (err && err != -ENOTSUPP)
+		err = spi_nor_set_4byte_addr_mode(nor, true);
+		if (err)
 			return err;
 	}
 
@@ -2887,14 +3318,14 @@ static void spi_nor_put_device(struct mtd_info *mtd)
 	module_put(dev->driver->owner);
 }
 
-void spi_nor_restore(struct spi_nor *nor)
+static void spi_nor_restore(struct spi_nor *nor)
 {
 	int ret;
 
 	/* restore the addressing mode */
 	if (nor->addr_nbytes == 4 && !(nor->flags & SNOR_F_4B_OPCODES) &&
 	    nor->flags & SNOR_F_BROKEN_RESET) {
-		ret = nor->params->set_4byte_addr_mode(nor, false);
+		ret = spi_nor_set_4byte_addr_mode(nor, false);
 		if (ret)
 			/*
 			 * Do not stop the execution in the hope that the flash
@@ -2907,7 +3338,6 @@ void spi_nor_restore(struct spi_nor *nor)
 	if (nor->flags & SNOR_F_SOFT_RESET)
 		spi_nor_soft_reset(nor);
 }
-EXPORT_SYMBOL_GPL(spi_nor_restore);
 
 static const struct flash_info *spi_nor_match_name(struct spi_nor *nor,
 						   const char *name)
@@ -2952,7 +3382,7 @@ static const struct flash_info *spi_nor_get_flash_info(struct spi_nor *nor,
 			 * JEDEC knows better, so overwrite platform ID. We
 			 * can't trust partitions any longer, but we'll let
 			 * mtd apply them anyway, since some partitions may be
-			 * marked read-only, and we don't want to lose that
+			 * marked read-only, and we don't want to loose that
 			 * information, even if it's not 100% accurate.
 			 */
 			dev_warn(nor->dev, "found %s, expected %s\n",
@@ -2977,6 +3407,9 @@ static void spi_nor_set_mtd_info(struct spi_nor *nor)
 		mtd->name = dev_name(dev);
 	mtd->type = MTD_NORFLASH;
 	mtd->flags = MTD_CAP_NORFLASH;
+	/* Unset BIT_WRITEABLE to enable JFFS2 write buffer for ECC'd NOR */
+	if (nor->flags & SNOR_F_ECC)
+		mtd->flags &= ~MTD_BIT_WRITEABLE;
 	if (nor->info->flags & SPI_NOR_NO_ERASE)
 		mtd->flags |= MTD_NO_ERASE;
 	else
@@ -3064,6 +3497,9 @@ int spi_nor_scan(struct spi_nor *nor, const char *name,
 	if (ret)
 		return ret;
 
+	if (spi_nor_use_parallel_locking(nor))
+		init_waitqueue_head(&nor->rww.wait);
+
 	/*
 	 * Configure the SPI memory:
 	 * - select op codes for (Fast) Read, Page Program and Sector Erase.
diff --git a/drivers/mtd/spi-nor/core.h b/drivers/mtd/spi-nor/core.h
index e0cc42a4a0c8..4fb5ff09c63a 100644
--- a/drivers/mtd/spi-nor/core.h
+++ b/drivers/mtd/spi-nor/core.h
@@ -130,6 +130,8 @@ enum spi_nor_option_flags {
 	SNOR_F_IO_MODE_EN_VOLATILE = BIT(11),
 	SNOR_F_SOFT_RESET	= BIT(12),
 	SNOR_F_SWP_IS_VOLATILE	= BIT(13),
+	SNOR_F_RWW		= BIT(14),
+	SNOR_F_ECC		= BIT(15),
 };
 
 struct spi_nor_read_command {
@@ -336,7 +338,8 @@ struct spi_nor_otp {
  * by the spi_nor_fixups hooks, or dynamically when parsing the JESD216
  * Serial Flash Discoverable Parameters (SFDP) tables.
  *
- * @size:		the flash memory density in bytes.
+ * @bank_size:		the flash memory bank density in bytes.
+ * @size:		the total flash memory density in bytes.
  * @writesize		Minimal writable flash unit size. Defaults to 1. Set to
  *			ECC unit size for ECC-ed flashes.
  * @page_size:		the page size of the SPI NOR flash memory.
@@ -349,6 +352,8 @@ struct spi_nor_otp {
  *			in octal DTR mode.
  * @rdsr_addr_nbytes:	dummy address bytes needed for Read Status Register
  *			command in octal DTR mode.
+ * @n_dice:		number of dice in the flash memory.
+ * @vreg_offset:	volatile register offset for each die.
  * @hwcaps:		describes the read and page program hardware
  *			capabilities.
  * @reads:		read capabilities ordered by priority: the higher index
@@ -374,6 +379,7 @@ struct spi_nor_otp {
  * @locking_ops:	SPI NOR locking methods.
  */
 struct spi_nor_flash_parameter {
+	u64				bank_size;
 	u64				size;
 	u32				writesize;
 	u32				page_size;
@@ -381,6 +387,8 @@ struct spi_nor_flash_parameter {
 	u8				addr_mode_nbytes;
 	u8				rdsr_dummy;
 	u8				rdsr_addr_nbytes;
+	u8				n_dice;
+	u32				*vreg_offset;
 
 	struct spi_nor_hwcaps		hwcaps;
 	struct spi_nor_read_command	reads[SNOR_CMD_READ_MAX];
@@ -422,7 +430,7 @@ struct spi_nor_fixups {
 	int (*post_bfpt)(struct spi_nor *nor,
 			 const struct sfdp_parameter_header *bfpt_header,
 			 const struct sfdp_bfpt *bfpt);
-	void (*post_sfdp)(struct spi_nor *nor);
+	int (*post_sfdp)(struct spi_nor *nor);
 	void (*late_init)(struct spi_nor *nor);
 };
 
@@ -435,6 +443,7 @@ struct spi_nor_fixups {
  * @sector_size:    the size listed here is what works with SPINOR_OP_SE, which
  *                  isn't necessarily called a "sector" by the vendor.
  * @n_sectors:      the number of sectors.
+ * @n_banks:        the number of banks.
  * @page_size:      the flash's page size.
  * @addr_nbytes:    number of address bytes to send.
  *
@@ -459,6 +468,7 @@ struct spi_nor_fixups {
  *   NO_CHIP_ERASE:           chip does not support chip erase.
  *   SPI_NOR_NO_FR:           can't do fastread.
  *   SPI_NOR_QUAD_PP:         flash supports Quad Input Page Program.
+ *   SPI_NOR_RWW:             flash supports reads while write.
  *
  * @no_sfdp_flags:  flags that indicate support that can be discovered via SFDP.
  *                  Used when SFDP tables are not defined in the flash. These
@@ -495,6 +505,7 @@ struct flash_info {
 	unsigned sector_size;
 	u16 n_sectors;
 	u16 page_size;
+	u8 n_banks;
 	u8 addr_nbytes;
 
 	bool parse_sfdp;
@@ -509,6 +520,7 @@ struct flash_info {
 #define NO_CHIP_ERASE			BIT(7)
 #define SPI_NOR_NO_FR			BIT(8)
 #define SPI_NOR_QUAD_PP			BIT(9)
+#define SPI_NOR_RWW			BIT(10)
 
 	u8 no_sfdp_flags;
 #define SPI_NOR_SKIP_SFDP		BIT(0)
@@ -540,24 +552,30 @@ struct flash_info {
 	.id = { SPI_NOR_ID_3ITEMS(_jedec_id), SPI_NOR_ID_3ITEMS(_ext_id) }, \
 	.id_len = 6
 
-#define SPI_NOR_GEOMETRY(_sector_size, _n_sectors)			\
+#define SPI_NOR_GEOMETRY(_sector_size, _n_sectors, _n_banks)		\
 	.sector_size = (_sector_size),					\
 	.n_sectors = (_n_sectors),					\
-	.page_size = 256
+	.page_size = 256,						\
+	.n_banks = (_n_banks)
 
 /* Used when the "_ext_id" is two bytes at most */
 #define INFO(_jedec_id, _ext_id, _sector_size, _n_sectors)		\
 	SPI_NOR_ID((_jedec_id), (_ext_id)),				\
-	SPI_NOR_GEOMETRY((_sector_size), (_n_sectors)),
+	SPI_NOR_GEOMETRY((_sector_size), (_n_sectors), 1),
+
+#define INFOB(_jedec_id, _ext_id, _sector_size, _n_sectors, _n_banks)	\
+	SPI_NOR_ID((_jedec_id), (_ext_id)),				\
+	SPI_NOR_GEOMETRY((_sector_size), (_n_sectors), (_n_banks)),
 
 #define INFO6(_jedec_id, _ext_id, _sector_size, _n_sectors)		\
 	SPI_NOR_ID6((_jedec_id), (_ext_id)),				\
-	SPI_NOR_GEOMETRY((_sector_size), (_n_sectors)),
+	SPI_NOR_GEOMETRY((_sector_size), (_n_sectors), 1),
 
 #define CAT25_INFO(_sector_size, _n_sectors, _page_size, _addr_nbytes)	\
 		.sector_size = (_sector_size),				\
 		.n_sectors = (_n_sectors),				\
 		.page_size = (_page_size),				\
+		.n_banks = 1,						\
 		.addr_nbytes = (_addr_nbytes),				\
 		.flags = SPI_NOR_NO_ERASE | SPI_NOR_NO_FR,		\
 
@@ -634,10 +652,14 @@ void spi_nor_spimem_setup_op(const struct spi_nor *nor,
 			     const enum spi_nor_protocol proto);
 int spi_nor_write_enable(struct spi_nor *nor);
 int spi_nor_write_disable(struct spi_nor *nor);
+int spi_nor_set_4byte_addr_mode_en4b_ex4b(struct spi_nor *nor, bool enable);
+int spi_nor_set_4byte_addr_mode_wren_en4b_ex4b(struct spi_nor *nor,
+					       bool enable);
+int spi_nor_set_4byte_addr_mode_brwr(struct spi_nor *nor, bool enable);
 int spi_nor_set_4byte_addr_mode(struct spi_nor *nor, bool enable);
 int spi_nor_wait_till_ready(struct spi_nor *nor);
 int spi_nor_global_block_unlock(struct spi_nor *nor);
-int spi_nor_lock_and_prep(struct spi_nor *nor);
+int spi_nor_prep_and_lock(struct spi_nor *nor);
 void spi_nor_unlock_and_unprep(struct spi_nor *nor);
 int spi_nor_sr1_bit6_quad_enable(struct spi_nor *nor);
 int spi_nor_sr2_bit1_quad_enable(struct spi_nor *nor);
diff --git a/drivers/mtd/spi-nor/debugfs.c b/drivers/mtd/spi-nor/debugfs.c
index fc7ad203df12..e11536fffe0f 100644
--- a/drivers/mtd/spi-nor/debugfs.c
+++ b/drivers/mtd/spi-nor/debugfs.c
@@ -25,6 +25,8 @@ static const char *const snor_f_names[] = {
 	SNOR_F_NAME(IO_MODE_EN_VOLATILE),
 	SNOR_F_NAME(SOFT_RESET),
 	SNOR_F_NAME(SWP_IS_VOLATILE),
+	SNOR_F_NAME(RWW),
+	SNOR_F_NAME(ECC),
 };
 #undef SNOR_F_NAME
 
diff --git a/drivers/mtd/spi-nor/macronix.c b/drivers/mtd/spi-nor/macronix.c
index 6853ec9ae65d..04888258e891 100644
--- a/drivers/mtd/spi-nor/macronix.c
+++ b/drivers/mtd/spi-nor/macronix.c
@@ -82,6 +82,9 @@ static const struct flash_info macronix_nor_parts[] = {
 	{ "mx25u51245g", INFO(0xc2253a, 0, 64 * 1024, 1024)
 		NO_SFDP_FLAGS(SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ)
 		FIXUP_FLAGS(SPI_NOR_4B_OPCODES) },
+	{ "mx25uw51245g", INFOB(0xc2813a, 0, 0, 0, 4)
+		PARSE_SFDP
+		FLAGS(SPI_NOR_RWW) },
 	{ "mx25v8035f",  INFO(0xc22314, 0, 64 * 1024,  16)
 		NO_SFDP_FLAGS(SECT_4K | SPI_NOR_DUAL_READ |
 			      SPI_NOR_QUAD_READ) },
@@ -105,11 +108,17 @@ static const struct flash_info macronix_nor_parts[] = {
 static void macronix_nor_default_init(struct spi_nor *nor)
 {
 	nor->params->quad_enable = spi_nor_sr1_bit6_quad_enable;
-	nor->params->set_4byte_addr_mode = spi_nor_set_4byte_addr_mode;
+}
+
+static void macronix_nor_late_init(struct spi_nor *nor)
+{
+	if (!nor->params->set_4byte_addr_mode)
+		nor->params->set_4byte_addr_mode = spi_nor_set_4byte_addr_mode_en4b_ex4b;
 }
 
 static const struct spi_nor_fixups macronix_nor_fixups = {
 	.default_init = macronix_nor_default_init,
+	.late_init = macronix_nor_late_init,
 };
 
 const struct spi_nor_manufacturer spi_nor_macronix = {
diff --git a/drivers/mtd/spi-nor/micron-st.c b/drivers/mtd/spi-nor/micron-st.c
index 7bb86df52f0b..4b919756a205 100644
--- a/drivers/mtd/spi-nor/micron-st.c
+++ b/drivers/mtd/spi-nor/micron-st.c
@@ -131,7 +131,7 @@ static void mt35xu512aba_default_init(struct spi_nor *nor)
 	nor->params->octal_dtr_enable = micron_st_nor_octal_dtr_enable;
 }
 
-static void mt35xu512aba_post_sfdp_fixup(struct spi_nor *nor)
+static int mt35xu512aba_post_sfdp_fixup(struct spi_nor *nor)
 {
 	/* Set the Fast Read settings. */
 	nor->params->hwcaps.mask |= SNOR_HWCAPS_READ_8_8_8_DTR;
@@ -149,6 +149,8 @@ static void mt35xu512aba_post_sfdp_fixup(struct spi_nor *nor)
 	 * disable it.
 	 */
 	nor->params->quad_enable = NULL;
+
+	return 0;
 }
 
 static const struct spi_nor_fixups mt35xu512aba_fixups = {
@@ -302,30 +304,6 @@ static const struct flash_info st_nor_parts[] = {
 };
 
 /**
- * micron_st_nor_set_4byte_addr_mode() - Set 4-byte address mode for ST and
- * Micron flashes.
- * @nor:	pointer to 'struct spi_nor'.
- * @enable:	true to enter the 4-byte address mode, false to exit the 4-byte
- *		address mode.
- *
- * Return: 0 on success, -errno otherwise.
- */
-static int micron_st_nor_set_4byte_addr_mode(struct spi_nor *nor, bool enable)
-{
-	int ret;
-
-	ret = spi_nor_write_enable(nor);
-	if (ret)
-		return ret;
-
-	ret = spi_nor_set_4byte_addr_mode(nor, enable);
-	if (ret)
-		return ret;
-
-	return spi_nor_write_disable(nor);
-}
-
-/**
  * micron_st_nor_read_fsr() - Read the Flag Status Register.
  * @nor:	pointer to 'struct spi_nor'
  * @fsr:	pointer to a DMA-able buffer where the value of the
@@ -449,13 +427,17 @@ static void micron_st_nor_default_init(struct spi_nor *nor)
 	nor->flags |= SNOR_F_HAS_LOCK;
 	nor->flags &= ~SNOR_F_HAS_16BIT_SR;
 	nor->params->quad_enable = NULL;
-	nor->params->set_4byte_addr_mode = micron_st_nor_set_4byte_addr_mode;
 }
 
 static void micron_st_nor_late_init(struct spi_nor *nor)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
+
 	if (nor->info->mfr_flags & USE_FSR)
-		nor->params->ready = micron_st_nor_ready;
+		params->ready = micron_st_nor_ready;
+
+	if (!params->set_4byte_addr_mode)
+		params->set_4byte_addr_mode = spi_nor_set_4byte_addr_mode_wren_en4b_ex4b;
 }
 
 static const struct spi_nor_fixups micron_st_nor_fixups = {
diff --git a/drivers/mtd/spi-nor/otp.c b/drivers/mtd/spi-nor/otp.c
index 00ab0d2d6d2f..9a729aa3452d 100644
--- a/drivers/mtd/spi-nor/otp.c
+++ b/drivers/mtd/spi-nor/otp.c
@@ -255,7 +255,7 @@ static int spi_nor_mtd_otp_info(struct mtd_info *mtd, size_t len,
 	if (len < n_regions * sizeof(*buf))
 		return -ENOSPC;
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock(nor);
 	if (ret)
 		return ret;
 
@@ -325,7 +325,7 @@ static int spi_nor_mtd_otp_read_write(struct mtd_info *mtd, loff_t ofs,
 	if (!total_len)
 		return 0;
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock(nor);
 	if (ret)
 		return ret;
 
@@ -415,7 +415,7 @@ static int spi_nor_mtd_otp_erase(struct mtd_info *mtd, loff_t from, size_t len)
 	if (!IS_ALIGNED(len, rlen) || !IS_ALIGNED(from, rlen))
 		return -EINVAL;
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock(nor);
 	if (ret)
 		return ret;
 
@@ -460,7 +460,7 @@ static int spi_nor_mtd_otp_lock(struct mtd_info *mtd, loff_t from, size_t len)
 	if (!IS_ALIGNED(len, rlen) || !IS_ALIGNED(from, rlen))
 		return -EINVAL;
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock(nor);
 	if (ret)
 		return ret;
 
diff --git a/drivers/mtd/spi-nor/sfdp.c b/drivers/mtd/spi-nor/sfdp.c
index 298ab5e53a8c..b3b11dfed789 100644
--- a/drivers/mtd/spi-nor/sfdp.c
+++ b/drivers/mtd/spi-nor/sfdp.c
@@ -26,6 +26,11 @@
 					 * Status, Control and Configuration
 					 * Register Map.
 					 */
+#define SFDP_SCCR_MAP_MC_ID	0xff88	/*
+					 * Status, Control and Configuration
+					 * Register Map Offsets for Multi-Chip
+					 * SPI Memory Devices.
+					 */
 
 #define SFDP_SIGNATURE		0x50444653U
 
@@ -438,6 +443,7 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 	size_t len;
 	int i, cmd, err;
 	u32 addr, val;
+	u32 dword;
 	u16 half;
 	u8 erase_mask;
 
@@ -607,6 +613,16 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 		break;
 	}
 
+	dword = bfpt.dwords[SFDP_DWORD(16)] & BFPT_DWORD16_4B_ADDR_MODE_MASK;
+	if (SFDP_MASK_CHECK(dword, BFPT_DWORD16_4B_ADDR_MODE_BRWR))
+		params->set_4byte_addr_mode = spi_nor_set_4byte_addr_mode_brwr;
+	else if (SFDP_MASK_CHECK(dword, BFPT_DWORD16_4B_ADDR_MODE_WREN_EN4B_EX4B))
+		params->set_4byte_addr_mode = spi_nor_set_4byte_addr_mode_wren_en4b_ex4b;
+	else if (SFDP_MASK_CHECK(dword, BFPT_DWORD16_4B_ADDR_MODE_EN4B_EX4B))
+		params->set_4byte_addr_mode = spi_nor_set_4byte_addr_mode_en4b_ex4b;
+	else
+		dev_dbg(nor->dev, "BFPT: 4-Byte Address Mode method is not recognized or not implemented\n");
+
 	/* Soft Reset support. */
 	if (bfpt.dwords[SFDP_DWORD(16)] & BFPT_DWORD16_SWRST_EN_RST)
 		nor->flags |= SNOR_F_SOFT_RESET;
@@ -1215,6 +1231,7 @@ out:
 static int spi_nor_parse_sccr(struct spi_nor *nor,
 			      const struct sfdp_parameter_header *sccr_header)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	u32 *dwords, addr;
 	size_t len;
 	int ret;
@@ -1231,6 +1248,18 @@ static int spi_nor_parse_sccr(struct spi_nor *nor,
 
 	le32_to_cpu_array(dwords, sccr_header->length);
 
+	/* Address offset for volatile registers (die 0) */
+	if (!params->vreg_offset) {
+		params->vreg_offset = devm_kmalloc(nor->dev, sizeof(*dwords),
+						   GFP_KERNEL);
+		if (!params->vreg_offset) {
+			ret = -ENOMEM;
+			goto out;
+		}
+	}
+	params->vreg_offset[0] = dwords[SFDP_DWORD(1)];
+	params->n_dice = 1;
+
 	if (FIELD_GET(SCCR_DWORD22_OCTAL_DTR_EN_VOLATILE,
 		      dwords[SFDP_DWORD(22)]))
 		nor->flags |= SNOR_F_IO_MODE_EN_VOLATILE;
@@ -1241,6 +1270,63 @@ out:
 }
 
 /**
+ * spi_nor_parse_sccr_mc() - Parse the Status, Control and Configuration
+ *                           Register Map Offsets for Multi-Chip SPI Memory
+ *                           Devices.
+ * @nor:		pointer to a 'struct spi_nor'
+ * @sccr_mc_header:	pointer to the 'struct sfdp_parameter_header' describing
+ *			the SCCR Map offsets table length and version.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int spi_nor_parse_sccr_mc(struct spi_nor *nor,
+				 const struct sfdp_parameter_header *sccr_mc_header)
+{
+	struct spi_nor_flash_parameter *params = nor->params;
+	u32 *dwords, addr;
+	u8 i, n_dice;
+	size_t len;
+	int ret;
+
+	len = sccr_mc_header->length * sizeof(*dwords);
+	dwords = kmalloc(len, GFP_KERNEL);
+	if (!dwords)
+		return -ENOMEM;
+
+	addr = SFDP_PARAM_HEADER_PTP(sccr_mc_header);
+	ret = spi_nor_read_sfdp(nor, addr, len, dwords);
+	if (ret)
+		goto out;
+
+	le32_to_cpu_array(dwords, sccr_mc_header->length);
+
+	/*
+	 * Pair of DOWRDs (volatile and non-volatile register offsets) per
+	 * additional die. Hence, length = 2 * (number of additional dice).
+	 */
+	n_dice = 1 + sccr_mc_header->length / 2;
+
+	/* Address offset for volatile registers of additional dice */
+	params->vreg_offset =
+			devm_krealloc(nor->dev, params->vreg_offset,
+				      n_dice * sizeof(*dwords),
+				      GFP_KERNEL);
+	if (!params->vreg_offset) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	for (i = 1; i < n_dice; i++)
+		params->vreg_offset[i] = dwords[SFDP_DWORD(i) * 2];
+
+	params->n_dice = n_dice;
+
+out:
+	kfree(dwords);
+	return ret;
+}
+
+/**
  * spi_nor_post_sfdp_fixups() - Updates the flash's parameters and settings
  * after SFDP has been parsed. Called only for flashes that define JESD216 SFDP
  * tables.
@@ -1249,14 +1335,21 @@ out:
  * Used to tweak various flash parameters when information provided by the SFDP
  * tables are wrong.
  */
-static void spi_nor_post_sfdp_fixups(struct spi_nor *nor)
+static int spi_nor_post_sfdp_fixups(struct spi_nor *nor)
 {
+	int ret;
+
 	if (nor->manufacturer && nor->manufacturer->fixups &&
-	    nor->manufacturer->fixups->post_sfdp)
-		nor->manufacturer->fixups->post_sfdp(nor);
+	    nor->manufacturer->fixups->post_sfdp) {
+		ret = nor->manufacturer->fixups->post_sfdp(nor);
+		if (ret)
+			return ret;
+	}
 
 	if (nor->info->fixups && nor->info->fixups->post_sfdp)
-		nor->info->fixups->post_sfdp(nor);
+		return nor->info->fixups->post_sfdp(nor);
+
+	return 0;
 }
 
 /**
@@ -1449,6 +1542,10 @@ int spi_nor_parse_sfdp(struct spi_nor *nor)
 			err = spi_nor_parse_sccr(nor, param_header);
 			break;
 
+		case SFDP_SCCR_MAP_MC_ID:
+			err = spi_nor_parse_sccr_mc(nor, param_header);
+			break;
+
 		default:
 			break;
 		}
@@ -1466,7 +1563,7 @@ int spi_nor_parse_sfdp(struct spi_nor *nor)
 		}
 	}
 
-	spi_nor_post_sfdp_fixups(nor);
+	err = spi_nor_post_sfdp_fixups(nor);
 exit:
 	kfree(param_headers);
 	return err;
diff --git a/drivers/mtd/spi-nor/sfdp.h b/drivers/mtd/spi-nor/sfdp.h
index 500659b35655..6eb99e1cdd61 100644
--- a/drivers/mtd/spi-nor/sfdp.h
+++ b/drivers/mtd/spi-nor/sfdp.h
@@ -15,6 +15,7 @@
 
 /* SFDP DWORDS are indexed from 1 but C arrays are indexed from 0. */
 #define SFDP_DWORD(i)		((i) - 1)
+#define SFDP_MASK_CHECK(dword, mask)		(((dword) & (mask)) == (mask))
 
 /* Basic Flash Parameter Table */
 
@@ -89,6 +90,32 @@ struct sfdp_bfpt {
 #define BFPT_DWORD15_QER_SR2_BIT1_NO_RD		(0x4UL << 20)
 #define BFPT_DWORD15_QER_SR2_BIT1		(0x5UL << 20) /* Spansion */
 
+#define BFPT_DWORD16_EN4B_MASK			GENMASK(31, 24)
+#define BFPT_DWORD16_EN4B_ALWAYS_4B		BIT(30)
+#define BFPT_DWORD16_EN4B_4B_OPCODES		BIT(29)
+#define BFPT_DWORD16_EN4B_16BIT_NV_CR		BIT(28)
+#define BFPT_DWORD16_EN4B_BRWR			BIT(27)
+#define BFPT_DWORD16_EN4B_WREAR			BIT(26)
+#define BFPT_DWORD16_EN4B_WREN_EN4B		BIT(25)
+#define BFPT_DWORD16_EN4B_EN4B			BIT(24)
+#define BFPT_DWORD16_EX4B_MASK			GENMASK(18, 14)
+#define BFPT_DWORD16_EX4B_16BIT_NV_CR		BIT(18)
+#define BFPT_DWORD16_EX4B_BRWR			BIT(17)
+#define BFPT_DWORD16_EX4B_WREAR			BIT(16)
+#define BFPT_DWORD16_EX4B_WREN_EX4B		BIT(15)
+#define BFPT_DWORD16_EX4B_EX4B			BIT(14)
+#define BFPT_DWORD16_4B_ADDR_MODE_MASK			\
+	(BFPT_DWORD16_EN4B_MASK | BFPT_DWORD16_EX4B_MASK)
+#define BFPT_DWORD16_4B_ADDR_MODE_16BIT_NV_CR		\
+	(BFPT_DWORD16_EN4B_16BIT_NV_CR | BFPT_DWORD16_EX4B_16BIT_NV_CR)
+#define BFPT_DWORD16_4B_ADDR_MODE_BRWR			\
+	(BFPT_DWORD16_EN4B_BRWR | BFPT_DWORD16_EX4B_BRWR)
+#define BFPT_DWORD16_4B_ADDR_MODE_WREAR			\
+	(BFPT_DWORD16_EN4B_WREAR | BFPT_DWORD16_EX4B_WREAR)
+#define BFPT_DWORD16_4B_ADDR_MODE_WREN_EN4B_EX4B	\
+	(BFPT_DWORD16_EN4B_WREN_EN4B | BFPT_DWORD16_EX4B_WREN_EX4B)
+#define BFPT_DWORD16_4B_ADDR_MODE_EN4B_EX4B		\
+	(BFPT_DWORD16_EN4B_EN4B | BFPT_DWORD16_EX4B_EX4B)
 #define BFPT_DWORD16_SWRST_EN_RST		BIT(12)
 
 #define BFPT_DWORD18_CMD_EXT_MASK		GENMASK(30, 29)
diff --git a/drivers/mtd/spi-nor/spansion.c b/drivers/mtd/spi-nor/spansion.c
index 12a256c0ef4c..15f9a80c10b9 100644
--- a/drivers/mtd/spi-nor/spansion.c
+++ b/drivers/mtd/spi-nor/spansion.c
@@ -14,13 +14,26 @@
 #define SPINOR_OP_CLSR		0x30	/* Clear status register 1 */
 #define SPINOR_OP_RD_ANY_REG			0x65	/* Read any register */
 #define SPINOR_OP_WR_ANY_REG			0x71	/* Write any register */
-#define SPINOR_REG_CYPRESS_CFR1V		0x00800002
+#define SPINOR_REG_CYPRESS_VREG			0x00800000
+#define SPINOR_REG_CYPRESS_STR1			0x0
+#define SPINOR_REG_CYPRESS_STR1V					\
+	(SPINOR_REG_CYPRESS_VREG + SPINOR_REG_CYPRESS_STR1)
+#define SPINOR_REG_CYPRESS_CFR1			0x2
+#define SPINOR_REG_CYPRESS_CFR1V					\
+	(SPINOR_REG_CYPRESS_VREG + SPINOR_REG_CYPRESS_CFR1)
 #define SPINOR_REG_CYPRESS_CFR1_QUAD_EN		BIT(1)	/* Quad Enable */
-#define SPINOR_REG_CYPRESS_CFR2V		0x00800003
+#define SPINOR_REG_CYPRESS_CFR2			0x3
+#define SPINOR_REG_CYPRESS_CFR2V					\
+	(SPINOR_REG_CYPRESS_VREG + SPINOR_REG_CYPRESS_CFR2)
 #define SPINOR_REG_CYPRESS_CFR2_MEMLAT_11_24	0xb
-#define SPINOR_REG_CYPRESS_CFR3V		0x00800004
+#define SPINOR_REG_CYPRESS_CFR2_ADRBYT		BIT(7)
+#define SPINOR_REG_CYPRESS_CFR3			0x4
+#define SPINOR_REG_CYPRESS_CFR3V					\
+	(SPINOR_REG_CYPRESS_VREG + SPINOR_REG_CYPRESS_CFR3)
 #define SPINOR_REG_CYPRESS_CFR3_PGSZ		BIT(4) /* Page size. */
-#define SPINOR_REG_CYPRESS_CFR5V		0x00800006
+#define SPINOR_REG_CYPRESS_CFR5			0x6
+#define SPINOR_REG_CYPRESS_CFR5V					\
+	(SPINOR_REG_CYPRESS_VREG + SPINOR_REG_CYPRESS_CFR5)
 #define SPINOR_REG_CYPRESS_CFR5_BIT6		BIT(6)
 #define SPINOR_REG_CYPRESS_CFR5_DDR		BIT(1)
 #define SPINOR_REG_CYPRESS_CFR5_OPI		BIT(0)
@@ -29,6 +42,7 @@
 	 SPINOR_REG_CYPRESS_CFR5_OPI)
 #define SPINOR_REG_CYPRESS_CFR5_OCT_DTR_DS	SPINOR_REG_CYPRESS_CFR5_BIT6
 #define SPINOR_OP_CYPRESS_RD_FAST		0xee
+#define SPINOR_REG_CYPRESS_ARCFN		0x00000006
 
 /* Cypress SPI NOR flash operations. */
 #define CYPRESS_NOR_WR_ANY_REG_OP(naddr, addr, ndata, buf)		\
@@ -37,10 +51,10 @@
 		   SPI_MEM_OP_NO_DUMMY,					\
 		   SPI_MEM_OP_DATA_OUT(ndata, buf, 0))
 
-#define CYPRESS_NOR_RD_ANY_REG_OP(naddr, addr, buf)			\
+#define CYPRESS_NOR_RD_ANY_REG_OP(naddr, addr, ndummy, buf)		\
 	SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RD_ANY_REG, 0),		\
 		   SPI_MEM_OP_ADDR(naddr, addr, 0),			\
-		   SPI_MEM_OP_NO_DUMMY,					\
+		   SPI_MEM_OP_DUMMY(ndummy, 0),				\
 		   SPI_MEM_OP_DATA_IN(1, buf, 0))
 
 #define SPANSION_CLSR_OP						\
@@ -49,6 +63,84 @@
 		   SPI_MEM_OP_NO_DUMMY,					\
 		   SPI_MEM_OP_NO_DATA)
 
+/**
+ * spansion_nor_clear_sr() - Clear the Status Register.
+ * @nor:	pointer to 'struct spi_nor'.
+ */
+static void spansion_nor_clear_sr(struct spi_nor *nor)
+{
+	int ret;
+
+	if (nor->spimem) {
+		struct spi_mem_op op = SPANSION_CLSR_OP;
+
+		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
+
+		ret = spi_mem_exec_op(nor->spimem, &op);
+	} else {
+		ret = spi_nor_controller_ops_write_reg(nor, SPINOR_OP_CLSR,
+						       NULL, 0);
+	}
+
+	if (ret)
+		dev_dbg(nor->dev, "error %d clearing SR\n", ret);
+}
+
+static int cypress_nor_sr_ready_and_clear_reg(struct spi_nor *nor, u64 addr)
+{
+	struct spi_mem_op op =
+		CYPRESS_NOR_RD_ANY_REG_OP(nor->params->addr_mode_nbytes, addr,
+					  0, nor->bouncebuf);
+	int ret;
+
+	ret = spi_nor_read_any_reg(nor, &op, nor->reg_proto);
+	if (ret)
+		return ret;
+
+	if (nor->bouncebuf[0] & (SR_E_ERR | SR_P_ERR)) {
+		if (nor->bouncebuf[0] & SR_E_ERR)
+			dev_err(nor->dev, "Erase Error occurred\n");
+		else
+			dev_err(nor->dev, "Programming Error occurred\n");
+
+		spansion_nor_clear_sr(nor);
+
+		ret = spi_nor_write_disable(nor);
+		if (ret)
+			return ret;
+
+		return -EIO;
+	}
+
+	return !(nor->bouncebuf[0] & SR_WIP);
+}
+/**
+ * cypress_nor_sr_ready_and_clear() - Query the Status Register of each die by
+ * using Read Any Register command to see if the whole flash is ready for new
+ * commands and clear it if there are any errors.
+ * @nor:	pointer to 'struct spi_nor'.
+ *
+ * Return: 1 if ready, 0 if not ready, -errno on errors.
+ */
+static int cypress_nor_sr_ready_and_clear(struct spi_nor *nor)
+{
+	struct spi_nor_flash_parameter *params = nor->params;
+	u64 addr;
+	int ret;
+	u8 i;
+
+	for (i = 0; i < params->n_dice; i++) {
+		addr = params->vreg_offset[i] + SPINOR_REG_CYPRESS_STR1;
+		ret = cypress_nor_sr_ready_and_clear_reg(nor, addr);
+		if (ret < 0)
+			return ret;
+		else if (ret == 0)
+			return 0;
+	}
+
+	return 1;
+}
+
 static int cypress_nor_octal_dtr_en(struct spi_nor *nor)
 {
 	struct spi_mem_op op;
@@ -125,21 +217,7 @@ static int cypress_nor_octal_dtr_dis(struct spi_nor *nor)
 	return 0;
 }
 
-/**
- * cypress_nor_quad_enable_volatile() - enable Quad I/O mode in volatile
- *                                      register.
- * @nor:	pointer to a 'struct spi_nor'
- *
- * It is recommended to update volatile registers in the field application due
- * to a risk of the non-volatile registers corruption by power interrupt. This
- * function sets Quad Enable bit in CFR1 volatile. If users set the Quad Enable
- * bit in the CFR1 non-volatile in advance (typically by a Flash programmer
- * before mounting Flash on PCB), the Quad Enable bit in the CFR1 volatile is
- * also set during Flash power-up.
- *
- * Return: 0 on success, -errno otherwise.
- */
-static int cypress_nor_quad_enable_volatile(struct spi_nor *nor)
+static int cypress_nor_quad_enable_volatile_reg(struct spi_nor *nor, u64 addr)
 {
 	struct spi_mem_op op;
 	u8 addr_mode_nbytes = nor->params->addr_mode_nbytes;
@@ -147,8 +225,7 @@ static int cypress_nor_quad_enable_volatile(struct spi_nor *nor)
 	int ret;
 
 	op = (struct spi_mem_op)
-		CYPRESS_NOR_RD_ANY_REG_OP(addr_mode_nbytes,
-					  SPINOR_REG_CYPRESS_CFR1V,
+		CYPRESS_NOR_RD_ANY_REG_OP(addr_mode_nbytes, addr, 0,
 					  nor->bouncebuf);
 
 	ret = spi_nor_read_any_reg(nor, &op, nor->reg_proto);
@@ -161,8 +238,7 @@ static int cypress_nor_quad_enable_volatile(struct spi_nor *nor)
 	/* Update the Quad Enable bit. */
 	nor->bouncebuf[0] |= SPINOR_REG_CYPRESS_CFR1_QUAD_EN;
 	op = (struct spi_mem_op)
-		CYPRESS_NOR_WR_ANY_REG_OP(addr_mode_nbytes,
-					  SPINOR_REG_CYPRESS_CFR1V, 1,
+		CYPRESS_NOR_WR_ANY_REG_OP(addr_mode_nbytes, addr, 1,
 					  nor->bouncebuf);
 	ret = spi_nor_write_any_volatile_reg(nor, &op, nor->reg_proto);
 	if (ret)
@@ -172,8 +248,7 @@ static int cypress_nor_quad_enable_volatile(struct spi_nor *nor)
 
 	/* Read back and check it. */
 	op = (struct spi_mem_op)
-		CYPRESS_NOR_RD_ANY_REG_OP(addr_mode_nbytes,
-					  SPINOR_REG_CYPRESS_CFR1V,
+		CYPRESS_NOR_RD_ANY_REG_OP(addr_mode_nbytes, addr, 0,
 					  nor->bouncebuf);
 	ret = spi_nor_read_any_reg(nor, &op, nor->reg_proto);
 	if (ret)
@@ -188,21 +263,156 @@ static int cypress_nor_quad_enable_volatile(struct spi_nor *nor)
 }
 
 /**
- * cypress_nor_set_page_size() - Set page size which corresponds to the flash
- *                               configuration.
+ * cypress_nor_quad_enable_volatile() - enable Quad I/O mode in volatile
+ *                                      register.
  * @nor:	pointer to a 'struct spi_nor'
  *
- * The BFPT table advertises a 512B or 256B page size depending on part but the
- * page size is actually configurable (with the default being 256B). Read from
- * CFR3V[4] and set the correct size.
+ * It is recommended to update volatile registers in the field application due
+ * to a risk of the non-volatile registers corruption by power interrupt. This
+ * function sets Quad Enable bit in CFR1 volatile. If users set the Quad Enable
+ * bit in the CFR1 non-volatile in advance (typically by a Flash programmer
+ * before mounting Flash on PCB), the Quad Enable bit in the CFR1 volatile is
+ * also set during Flash power-up.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int cypress_nor_quad_enable_volatile(struct spi_nor *nor)
+{
+	struct spi_nor_flash_parameter *params = nor->params;
+	u64 addr;
+	u8 i;
+	int ret;
+
+	if (!params->n_dice)
+		return cypress_nor_quad_enable_volatile_reg(nor,
+						SPINOR_REG_CYPRESS_CFR1V);
+
+	for (i = 0; i < params->n_dice; i++) {
+		addr = params->vreg_offset[i] + SPINOR_REG_CYPRESS_CFR1;
+		ret = cypress_nor_quad_enable_volatile_reg(nor, addr);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * cypress_nor_determine_addr_mode_by_sr1() - Determine current address mode
+ *                                            (3 or 4-byte) by querying status
+ *                                            register 1 (SR1).
+ * @nor:		pointer to a 'struct spi_nor'
+ * @addr_mode:		ponter to a buffer where we return the determined
+ *			address mode.
+ *
+ * This function tries to determine current address mode by comparing SR1 value
+ * from RDSR1(no address), RDAR(3-byte address), and RDAR(4-byte address).
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int cypress_nor_determine_addr_mode_by_sr1(struct spi_nor *nor,
+						  u8 *addr_mode)
+{
+	struct spi_mem_op op =
+		CYPRESS_NOR_RD_ANY_REG_OP(3, SPINOR_REG_CYPRESS_STR1V, 0,
+					  nor->bouncebuf);
+	bool is3byte, is4byte;
+	int ret;
+
+	ret = spi_nor_read_sr(nor, &nor->bouncebuf[1]);
+	if (ret)
+		return ret;
+
+	ret = spi_nor_read_any_reg(nor, &op, nor->reg_proto);
+	if (ret)
+		return ret;
+
+	is3byte = (nor->bouncebuf[0] == nor->bouncebuf[1]);
+
+	op = (struct spi_mem_op)
+		CYPRESS_NOR_RD_ANY_REG_OP(4, SPINOR_REG_CYPRESS_STR1V, 0,
+					  nor->bouncebuf);
+	ret = spi_nor_read_any_reg(nor, &op, nor->reg_proto);
+	if (ret)
+		return ret;
+
+	is4byte = (nor->bouncebuf[0] == nor->bouncebuf[1]);
+
+	if (is3byte == is4byte)
+		return -EIO;
+	if (is3byte)
+		*addr_mode = 3;
+	else
+		*addr_mode = 4;
+
+	return 0;
+}
+
+/**
+ * cypress_nor_set_addr_mode_nbytes() - Set the number of address bytes mode of
+ *                                      current address mode.
+ * @nor:		pointer to a 'struct spi_nor'
+ *
+ * Determine current address mode by reading SR1 with different methods, then
+ * query CFR2V[7] to confirm. If determination is failed, force enter to 4-byte
+ * address mode.
  *
  * Return: 0 on success, -errno otherwise.
  */
-static int cypress_nor_set_page_size(struct spi_nor *nor)
+static int cypress_nor_set_addr_mode_nbytes(struct spi_nor *nor)
+{
+	struct spi_mem_op op;
+	u8 addr_mode;
+	int ret;
+
+	/*
+	 * Read SR1 by RDSR1 and RDAR(3- AND 4-byte addr). Use write enable
+	 * that sets bit-1 in SR1.
+	 */
+	ret = spi_nor_write_enable(nor);
+	if (ret)
+		return ret;
+	ret = cypress_nor_determine_addr_mode_by_sr1(nor, &addr_mode);
+	if (ret) {
+		ret = spi_nor_set_4byte_addr_mode(nor, true);
+		if (ret)
+			return ret;
+		return spi_nor_write_disable(nor);
+	}
+	ret = spi_nor_write_disable(nor);
+	if (ret)
+		return ret;
+
+	/*
+	 * Query CFR2V and make sure no contradiction between determined address
+	 * mode and CFR2V[7].
+	 */
+	op = (struct spi_mem_op)
+		CYPRESS_NOR_RD_ANY_REG_OP(addr_mode, SPINOR_REG_CYPRESS_CFR2V,
+					  0, nor->bouncebuf);
+	ret = spi_nor_read_any_reg(nor, &op, nor->reg_proto);
+	if (ret)
+		return ret;
+
+	if (nor->bouncebuf[0] & SPINOR_REG_CYPRESS_CFR2_ADRBYT) {
+		if (addr_mode != 4)
+			return spi_nor_set_4byte_addr_mode(nor, true);
+	} else {
+		if (addr_mode != 3)
+			return spi_nor_set_4byte_addr_mode(nor, true);
+	}
+
+	nor->params->addr_nbytes = addr_mode;
+	nor->params->addr_mode_nbytes = addr_mode;
+
+	return 0;
+}
+
+static int cypress_nor_get_page_size_single_chip(struct spi_nor *nor)
 {
 	struct spi_mem_op op =
 		CYPRESS_NOR_RD_ANY_REG_OP(nor->params->addr_mode_nbytes,
-					  SPINOR_REG_CYPRESS_CFR3V,
+					  SPINOR_REG_CYPRESS_CFR3V, 0,
 					  nor->bouncebuf);
 	int ret;
 
@@ -218,18 +428,135 @@ static int cypress_nor_set_page_size(struct spi_nor *nor)
 	return 0;
 }
 
+
+static int cypress_nor_get_page_size_mcp(struct spi_nor *nor)
+{
+	struct spi_mem_op op =
+		CYPRESS_NOR_RD_ANY_REG_OP(nor->params->addr_mode_nbytes,
+					  0, 0, nor->bouncebuf);
+	struct spi_nor_flash_parameter *params = nor->params;
+	int ret;
+	u8 i;
+
+	/*
+	 * Use the minimum common page size configuration. Programming 256-byte
+	 * under 512-byte page size configuration is safe.
+	 */
+	params->page_size = 256;
+	for (i = 0; i < params->n_dice; i++) {
+		op.addr.val = params->vreg_offset[i] + SPINOR_REG_CYPRESS_CFR3;
+
+		ret = spi_nor_read_any_reg(nor, &op, nor->reg_proto);
+		if (ret)
+			return ret;
+
+		if (!(nor->bouncebuf[0] & SPINOR_REG_CYPRESS_CFR3_PGSZ))
+			return 0;
+	}
+
+	params->page_size = 512;
+
+	return 0;
+}
+
+/**
+ * cypress_nor_get_page_size() - Get flash page size configuration.
+ * @nor:	pointer to a 'struct spi_nor'
+ *
+ * The BFPT table advertises a 512B or 256B page size depending on part but the
+ * page size is actually configurable (with the default being 256B). Read from
+ * CFR3V[4] and set the correct size.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int cypress_nor_get_page_size(struct spi_nor *nor)
+{
+	if (nor->params->n_dice)
+		return cypress_nor_get_page_size_mcp(nor);
+	return cypress_nor_get_page_size_single_chip(nor);
+}
+
+static void cypress_nor_ecc_init(struct spi_nor *nor)
+{
+	/*
+	 * Programming is supported only in 16-byte ECC data unit granularity.
+	 * Byte-programming, bit-walking, or multiple program operations to the
+	 * same ECC data unit without an erase are not allowed.
+	 */
+	nor->params->writesize = 16;
+	nor->flags |= SNOR_F_ECC;
+}
+
+static int
+s25fs256t_post_bfpt_fixup(struct spi_nor *nor,
+			  const struct sfdp_parameter_header *bfpt_header,
+			  const struct sfdp_bfpt *bfpt)
+{
+	struct spi_mem_op op;
+	int ret;
+
+	ret = cypress_nor_set_addr_mode_nbytes(nor);
+	if (ret)
+		return ret;
+
+	/* Read Architecture Configuration Register (ARCFN) */
+	op = (struct spi_mem_op)
+		CYPRESS_NOR_RD_ANY_REG_OP(nor->params->addr_mode_nbytes,
+					  SPINOR_REG_CYPRESS_ARCFN, 1,
+					  nor->bouncebuf);
+	ret = spi_nor_read_any_reg(nor, &op, nor->reg_proto);
+	if (ret)
+		return ret;
+
+	/* ARCFN value must be 0 if uniform sector is selected  */
+	if (nor->bouncebuf[0])
+		return -ENODEV;
+
+	return cypress_nor_get_page_size(nor);
+}
+
+static int s25fs256t_post_sfdp_fixup(struct spi_nor *nor)
+{
+	struct spi_nor_flash_parameter *params = nor->params;
+
+	/* PP_1_1_4_4B is supported but missing in 4BAIT. */
+	params->hwcaps.mask |= SNOR_HWCAPS_PP_1_1_4;
+	spi_nor_set_pp_settings(&params->page_programs[SNOR_CMD_PP_1_1_4],
+				SPINOR_OP_PP_1_1_4_4B,
+				SNOR_PROTO_1_1_4);
+
+	return 0;
+}
+
+static void s25fs256t_late_init(struct spi_nor *nor)
+{
+	cypress_nor_ecc_init(nor);
+}
+
+static struct spi_nor_fixups s25fs256t_fixups = {
+	.post_bfpt = s25fs256t_post_bfpt_fixup,
+	.post_sfdp = s25fs256t_post_sfdp_fixup,
+	.late_init = s25fs256t_late_init,
+};
+
 static int
 s25hx_t_post_bfpt_fixup(struct spi_nor *nor,
 			const struct sfdp_parameter_header *bfpt_header,
 			const struct sfdp_bfpt *bfpt)
 {
+	int ret;
+
+	ret = cypress_nor_set_addr_mode_nbytes(nor);
+	if (ret)
+		return ret;
+
 	/* Replace Quad Enable with volatile version */
 	nor->params->quad_enable = cypress_nor_quad_enable_volatile;
 
-	return cypress_nor_set_page_size(nor);
+	return 0;
 }
 
-static void s25hx_t_post_sfdp_fixup(struct spi_nor *nor)
+static int s25hx_t_post_sfdp_fixup(struct spi_nor *nor)
 {
 	struct spi_nor_erase_type *erase_type =
 					nor->params->erase_map.erase_type;
@@ -251,6 +578,12 @@ static void s25hx_t_post_sfdp_fixup(struct spi_nor *nor)
 			break;
 		}
 	}
+
+	/* The 2 Gb parts duplicate info and advertise 4 dice instead of 2. */
+	if (nor->params->size == SZ_256M)
+		nor->params->n_dice = 2;
+
+	return cypress_nor_get_page_size(nor);
 }
 
 static void s25hx_t_late_init(struct spi_nor *nor)
@@ -260,8 +593,11 @@ static void s25hx_t_late_init(struct spi_nor *nor)
 	/* Fast Read 4B requires mode cycles */
 	params->reads[SNOR_CMD_READ_FAST].num_mode_clocks = 8;
 
-	/* The writesize should be ECC data unit size */
-	params->writesize = 16;
+	cypress_nor_ecc_init(nor);
+
+	/* Replace ready() with multi die version */
+	if (params->n_dice)
+		params->ready = cypress_nor_sr_ready_and_clear;
 }
 
 static struct spi_nor_fixups s25hx_t_fixups = {
@@ -286,7 +622,7 @@ static int cypress_nor_octal_dtr_enable(struct spi_nor *nor, bool enable)
 			cypress_nor_octal_dtr_dis(nor);
 }
 
-static void s28hx_t_post_sfdp_fixup(struct spi_nor *nor)
+static int s28hx_t_post_sfdp_fixup(struct spi_nor *nor)
 {
 	/*
 	 * On older versions of the flash the xSPI Profile 1.0 table has the
@@ -312,19 +648,27 @@ static void s28hx_t_post_sfdp_fixup(struct spi_nor *nor)
 	 * actual value for that is 4.
 	 */
 	nor->params->rdsr_addr_nbytes = 4;
+
+	return cypress_nor_get_page_size(nor);
 }
 
 static int s28hx_t_post_bfpt_fixup(struct spi_nor *nor,
 				   const struct sfdp_parameter_header *bfpt_header,
 				   const struct sfdp_bfpt *bfpt)
 {
-	return cypress_nor_set_page_size(nor);
+	int ret;
+
+	ret = cypress_nor_set_addr_mode_nbytes(nor);
+	if (ret)
+		return ret;
+
+	return 0;
 }
 
 static void s28hx_t_late_init(struct spi_nor *nor)
 {
 	nor->params->octal_dtr_enable = cypress_nor_octal_dtr_enable;
-	nor->params->writesize = 16;
+	cypress_nor_ecc_init(nor);
 }
 
 static const struct spi_nor_fixups s28hx_t_fixups = {
@@ -446,6 +790,9 @@ static const struct flash_info spansion_nor_parts[] = {
 	{ "s25fl256l",  INFO(0x016019,      0,  64 * 1024, 512)
 		NO_SFDP_FLAGS(SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ)
 		FIXUP_FLAGS(SPI_NOR_4B_OPCODES) },
+	{ "s25fs256t",  INFO6(0x342b19, 0x0f0890, 0, 0)
+		PARSE_SFDP
+		.fixups = &s25fs256t_fixups },
 	{ "s25hl512t",  INFO6(0x342a1a, 0x0f0390, 256 * 1024, 256)
 		PARSE_SFDP
 		MFR_FLAGS(USE_CLSR)
@@ -454,6 +801,10 @@ static const struct flash_info spansion_nor_parts[] = {
 		PARSE_SFDP
 		MFR_FLAGS(USE_CLSR)
 		.fixups = &s25hx_t_fixups },
+	{ "s25hl02gt",  INFO6(0x342a1c, 0x0f0090, 0, 0)
+		PARSE_SFDP
+		FLAGS(NO_CHIP_ERASE)
+		.fixups = &s25hx_t_fixups },
 	{ "s25hs512t",  INFO6(0x342b1a, 0x0f0390, 256 * 1024, 256)
 		PARSE_SFDP
 		MFR_FLAGS(USE_CLSR)
@@ -462,6 +813,10 @@ static const struct flash_info spansion_nor_parts[] = {
 		PARSE_SFDP
 		MFR_FLAGS(USE_CLSR)
 		.fixups = &s25hx_t_fixups },
+	{ "s25hs02gt",  INFO6(0x342b1c, 0x0f0090, 0, 0)
+		PARSE_SFDP
+		FLAGS(NO_CHIP_ERASE)
+		.fixups = &s25hx_t_fixups },
 	{ "cy15x104q",  INFO6(0x042cc2, 0x7f7f7f, 512 * 1024, 1)
 		FLAGS(SPI_NOR_NO_ERASE) },
 	{ "s28hl512t",   INFO(0x345a1a,      0, 256 * 1024, 256)
@@ -483,29 +838,6 @@ static const struct flash_info spansion_nor_parts[] = {
 };
 
 /**
- * spansion_nor_clear_sr() - Clear the Status Register.
- * @nor:	pointer to 'struct spi_nor'.
- */
-static void spansion_nor_clear_sr(struct spi_nor *nor)
-{
-	int ret;
-
-	if (nor->spimem) {
-		struct spi_mem_op op = SPANSION_CLSR_OP;
-
-		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
-
-		ret = spi_mem_exec_op(nor->spimem, &op);
-	} else {
-		ret = spi_nor_controller_ops_write_reg(nor, SPINOR_OP_CLSR,
-						       NULL, 0);
-	}
-
-	if (ret)
-		dev_dbg(nor->dev, "error %d clearing SR\n", ret);
-}
-
-/**
  * spansion_nor_sr_ready_and_clear() - Query the Status Register to see if the
  * flash is ready for new commands and clear it if there are any errors.
  * @nor:	pointer to 'struct spi_nor'.
diff --git a/drivers/mtd/spi-nor/sst.c b/drivers/mtd/spi-nor/sst.c
index 63bcc97bf978..688eb20c763e 100644
--- a/drivers/mtd/spi-nor/sst.c
+++ b/drivers/mtd/spi-nor/sst.c
@@ -126,7 +126,7 @@ static int sst_nor_write(struct mtd_info *mtd, loff_t to, size_t len,
 
 	dev_dbg(nor->dev, "to 0x%08x, len %zd\n", (u32)to, len);
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock(nor);
 	if (ret)
 		return ret;
 
diff --git a/drivers/mtd/spi-nor/swp.c b/drivers/mtd/spi-nor/swp.c
index 1f178313ba8f..0ba716e84377 100644
--- a/drivers/mtd/spi-nor/swp.c
+++ b/drivers/mtd/spi-nor/swp.c
@@ -348,7 +348,7 @@ static int spi_nor_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 	struct spi_nor *nor = mtd_to_spi_nor(mtd);
 	int ret;
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock(nor);
 	if (ret)
 		return ret;
 
@@ -363,7 +363,7 @@ static int spi_nor_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 	struct spi_nor *nor = mtd_to_spi_nor(mtd);
 	int ret;
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock(nor);
 	if (ret)
 		return ret;
 
@@ -378,7 +378,7 @@ static int spi_nor_is_locked(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 	struct spi_nor *nor = mtd_to_spi_nor(mtd);
 	int ret;
 
-	ret = spi_nor_lock_and_prep(nor);
+	ret = spi_nor_prep_and_lock(nor);
 	if (ret)
 		return ret;
 
diff --git a/drivers/mtd/spi-nor/winbond.c b/drivers/mtd/spi-nor/winbond.c
index ca39acf4112c..834d6ba5ce70 100644
--- a/drivers/mtd/spi-nor/winbond.c
+++ b/drivers/mtd/spi-nor/winbond.c
@@ -188,7 +188,7 @@ static int winbond_nor_set_4byte_addr_mode(struct spi_nor *nor, bool enable)
 {
 	int ret;
 
-	ret = spi_nor_set_4byte_addr_mode(nor, enable);
+	ret = spi_nor_set_4byte_addr_mode_en4b_ex4b(nor, enable);
 	if (ret || enable)
 		return ret;
 
@@ -216,19 +216,25 @@ static const struct spi_nor_otp_ops winbond_nor_otp_ops = {
 	.is_locked = spi_nor_otp_is_locked_sr2,
 };
 
-static void winbond_nor_default_init(struct spi_nor *nor)
-{
-	nor->params->set_4byte_addr_mode = winbond_nor_set_4byte_addr_mode;
-}
-
 static void winbond_nor_late_init(struct spi_nor *nor)
 {
-	if (nor->params->otp.org->n_regions)
-		nor->params->otp.ops = &winbond_nor_otp_ops;
+	struct spi_nor_flash_parameter *params = nor->params;
+
+	if (params->otp.org->n_regions)
+		params->otp.ops = &winbond_nor_otp_ops;
+
+	/*
+	 * Winbond seems to require that the Extended Address Register to be set
+	 * to zero when exiting the 4-Byte Address Mode, at least for W25Q256FV.
+	 * This requirement is not described in the JESD216 SFDP standard, thus
+	 * it is Winbond specific. Since we do not know if other Winbond flashes
+	 * have the same requirement, play safe and overwrite the method parsed
+	 * from BFPT, if any.
+	 */
+	params->set_4byte_addr_mode = winbond_nor_set_4byte_addr_mode;
 }
 
 static const struct spi_nor_fixups winbond_nor_fixups = {
-	.default_init = winbond_nor_default_init,
 	.late_init = winbond_nor_late_init,
 };
 
diff --git a/drivers/mtd/spi-nor/xilinx.c b/drivers/mtd/spi-nor/xilinx.c
index 5723157739fc..7175de8aa336 100644
--- a/drivers/mtd/spi-nor/xilinx.c
+++ b/drivers/mtd/spi-nor/xilinx.c
@@ -31,6 +31,7 @@
 		.sector_size = (8 * (_page_size)),			\
 		.n_sectors = (_n_sectors),				\
 		.page_size = (_page_size),				\
+		.n_banks = 1,						\
 		.addr_nbytes = 3,					\
 		.flags = SPI_NOR_NO_FR
 
diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c
index ad025b2ee417..9cd565daad36 100644
--- a/drivers/mtd/ubi/build.c
+++ b/drivers/mtd/ubi/build.c
@@ -95,7 +95,7 @@ static DEFINE_SPINLOCK(ubi_devices_lock);
 
 /* "Show" method for files in '/<sysfs>/class/ubi/' */
 /* UBI version attribute ('/<sysfs>/class/ubi/version') */
-static ssize_t version_show(struct class *class, struct class_attribute *attr,
+static ssize_t version_show(const struct class *class, const struct class_attribute *attr,
 			    char *buf)
 {
 	return sprintf(buf, "%d\n", UBI_VERSION);
@@ -111,7 +111,6 @@ ATTRIBUTE_GROUPS(ubi_class);
 /* Root UBI "class" object (corresponds to '/<sysfs>/class/ubi/') */
 struct class ubi_class = {
 	.name		= UBI_NAME_STR,
-	.owner		= THIS_MODULE,
 	.class_groups	= ubi_class_groups,
 };
 
diff --git a/drivers/mux/core.c b/drivers/mux/core.c
index 49bedbe6316c..990e7bc17c85 100644
--- a/drivers/mux/core.c
+++ b/drivers/mux/core.c
@@ -45,7 +45,6 @@ struct mux_state {
 
 static struct class mux_class = {
 	.name = "mux",
-	.owner = THIS_MODULE,
 };
 
 static DEFINE_IDA(mux_ida);
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index c34bd432da27..d0a1ed216d15 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -402,6 +402,8 @@ config TUN_VNET_CROSS_LE
 
 config VETH
 	tristate "Virtual ethernet pair device"
+	select PAGE_POOL
+	select PAGE_POOL_STATS
 	help
 	  This device is a local ethernet tunnel. Devices are created in pairs.
 	  When one end receives the packet it appears on its pair and vice
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 8cc9a74789b7..710548dbd0c1 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -1777,14 +1777,15 @@ void bond_lower_state_changed(struct slave *slave)
 
 /* The bonding driver uses ether_setup() to convert a master bond device
  * to ARPHRD_ETHER, that resets the target netdevice's flags so we always
- * have to restore the IFF_MASTER flag, and only restore IFF_SLAVE if it was set
+ * have to restore the IFF_MASTER flag, and only restore IFF_SLAVE and IFF_UP
+ * if they were set
  */
 static void bond_ether_setup(struct net_device *bond_dev)
 {
-	unsigned int slave_flag = bond_dev->flags & IFF_SLAVE;
+	unsigned int flags = bond_dev->flags & (IFF_SLAVE | IFF_UP);
 
 	ether_setup(bond_dev);
-	bond_dev->flags |= IFF_MASTER | slave_flag;
+	bond_dev->flags |= IFF_MASTER | flags;
 	bond_dev->priv_flags &= ~IFF_TX_SKB_SHARING;
 }
 
@@ -5696,9 +5697,13 @@ static int bond_ethtool_get_ts_info(struct net_device *bond_dev,
 				    struct ethtool_ts_info *info)
 {
 	struct bonding *bond = netdev_priv(bond_dev);
+	struct ethtool_ts_info ts_info;
 	const struct ethtool_ops *ops;
 	struct net_device *real_dev;
+	bool sw_tx_support = false;
 	struct phy_device *phydev;
+	struct list_head *iter;
+	struct slave *slave;
 	int ret = 0;
 
 	rcu_read_lock();
@@ -5717,10 +5722,36 @@ static int bond_ethtool_get_ts_info(struct net_device *bond_dev,
 			ret = ops->get_ts_info(real_dev, info);
 			goto out;
 		}
+	} else {
+		/* Check if all slaves support software tx timestamping */
+		rcu_read_lock();
+		bond_for_each_slave_rcu(bond, slave, iter) {
+			ret = -1;
+			ops = slave->dev->ethtool_ops;
+			phydev = slave->dev->phydev;
+
+			if (phy_has_tsinfo(phydev))
+				ret = phy_ts_info(phydev, &ts_info);
+			else if (ops->get_ts_info)
+				ret = ops->get_ts_info(slave->dev, &ts_info);
+
+			if (!ret && (ts_info.so_timestamping & SOF_TIMESTAMPING_TX_SOFTWARE)) {
+				sw_tx_support = true;
+				continue;
+			}
+
+			sw_tx_support = false;
+			break;
+		}
+		rcu_read_unlock();
 	}
 
+	ret = 0;
 	info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE |
 				SOF_TIMESTAMPING_SOFTWARE;
+	if (sw_tx_support)
+		info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE;
+
 	info->phc_index = -1;
 
 out:
diff --git a/drivers/net/bonding/bond_sysfs.c b/drivers/net/bonding/bond_sysfs.c
index 8996bd0a194a..0bb59da24922 100644
--- a/drivers/net/bonding/bond_sysfs.c
+++ b/drivers/net/bonding/bond_sysfs.c
@@ -31,12 +31,12 @@
 /* "show" function for the bond_masters attribute.
  * The class parameter is ignored.
  */
-static ssize_t bonding_show_bonds(struct class *cls,
-				  struct class_attribute *attr,
+static ssize_t bonding_show_bonds(const struct class *cls,
+				  const struct class_attribute *attr,
 				  char *buf)
 {
-	struct bond_net *bn =
-		container_of(attr, struct bond_net, class_attr_bonding_masters);
+	const struct bond_net *bn =
+		container_of_const(attr, struct bond_net, class_attr_bonding_masters);
 	int res = 0;
 	struct bonding *bond;
 
@@ -59,7 +59,7 @@ static ssize_t bonding_show_bonds(struct class *cls,
 	return res;
 }
 
-static struct net_device *bond_get_by_name(struct bond_net *bn, const char *ifname)
+static struct net_device *bond_get_by_name(const struct bond_net *bn, const char *ifname)
 {
 	struct bonding *bond;
 
@@ -75,12 +75,12 @@ static struct net_device *bond_get_by_name(struct bond_net *bn, const char *ifna
  *
  * The class parameter is ignored.
  */
-static ssize_t bonding_store_bonds(struct class *cls,
-				   struct class_attribute *attr,
+static ssize_t bonding_store_bonds(const struct class *cls,
+				   const struct class_attribute *attr,
 				   const char *buffer, size_t count)
 {
-	struct bond_net *bn =
-		container_of(attr, struct bond_net, class_attr_bonding_masters);
+	const struct bond_net *bn =
+		container_of_const(attr, struct bond_net, class_attr_bonding_masters);
 	char command[IFNAMSIZ + 1] = {0, };
 	char *ifname;
 	int rv, res = count;
diff --git a/drivers/net/can/Kconfig b/drivers/net/can/Kconfig
index cd34e8dc9394..3ceccafd701b 100644
--- a/drivers/net/can/Kconfig
+++ b/drivers/net/can/Kconfig
@@ -93,6 +93,18 @@ config CAN_AT91
 	  This is a driver for the SoC CAN controller in Atmel's AT91SAM9263
 	  and AT91SAM9X5 processors.
 
+config CAN_BXCAN
+	tristate "STM32 Basic Extended CAN (bxCAN) devices"
+	depends on OF || ARCH_STM32 || COMPILE_TEST
+	depends on HAS_IOMEM
+	select CAN_RX_OFFLOAD
+	help
+	  Say yes here to build support for the STMicroelectronics STM32 basic
+	  extended CAN Controller (bxCAN).
+
+	  This driver can also be built as a module. If so, the module
+	  will be called bxcan.
+
 config CAN_CAN327
 	tristate "Serial / USB serial ELM327 based OBD-II Interfaces (can327)"
 	depends on TTY
diff --git a/drivers/net/can/Makefile b/drivers/net/can/Makefile
index 52b0f6e10668..ff8f76295d13 100644
--- a/drivers/net/can/Makefile
+++ b/drivers/net/can/Makefile
@@ -14,6 +14,7 @@ obj-y				+= usb/
 obj-y				+= softing/
 
 obj-$(CONFIG_CAN_AT91)		+= at91_can.o
+obj-$(CONFIG_CAN_BXCAN)		+= bxcan.o
 obj-$(CONFIG_CAN_CAN327)	+= can327.o
 obj-$(CONFIG_CAN_CC770)		+= cc770/
 obj-$(CONFIG_CAN_C_CAN)		+= c_can/
diff --git a/drivers/net/can/bxcan.c b/drivers/net/can/bxcan.c
new file mode 100644
index 000000000000..e26ccd41e3cb
--- /dev/null
+++ b/drivers/net/can/bxcan.c
@@ -0,0 +1,1098 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// bxcan.c - STM32 Basic Extended CAN controller driver
+//
+// Copyright (c) 2022 Dario Binacchi <dario.binacchi@amarulasolutions.com>
+//
+// NOTE: The ST documentation uses the terms master/slave instead of
+// primary/secondary.
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/bitfield.h>
+#include <linux/can.h>
+#include <linux/can/dev.h>
+#include <linux/can/error.h>
+#include <linux/can/rx-offload.h>
+#include <linux/clk.h>
+#include <linux/ethtool.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#define BXCAN_NAPI_WEIGHT 3
+#define BXCAN_TIMEOUT_US 10000
+
+#define BXCAN_RX_MB_NUM 2
+#define BXCAN_TX_MB_NUM 3
+
+/* Primary control register (MCR) bits */
+#define BXCAN_MCR_RESET BIT(15)
+#define BXCAN_MCR_TTCM BIT(7)
+#define BXCAN_MCR_ABOM BIT(6)
+#define BXCAN_MCR_AWUM BIT(5)
+#define BXCAN_MCR_NART BIT(4)
+#define BXCAN_MCR_RFLM BIT(3)
+#define BXCAN_MCR_TXFP BIT(2)
+#define BXCAN_MCR_SLEEP BIT(1)
+#define BXCAN_MCR_INRQ BIT(0)
+
+/* Primary status register (MSR) bits */
+#define BXCAN_MSR_ERRI BIT(2)
+#define BXCAN_MSR_SLAK BIT(1)
+#define BXCAN_MSR_INAK BIT(0)
+
+/* Transmit status register (TSR) bits */
+#define BXCAN_TSR_RQCP2 BIT(16)
+#define BXCAN_TSR_RQCP1 BIT(8)
+#define BXCAN_TSR_RQCP0 BIT(0)
+
+/* Receive FIFO 0 register (RF0R) bits */
+#define BXCAN_RF0R_RFOM0 BIT(5)
+#define BXCAN_RF0R_FMP0_MASK GENMASK(1, 0)
+
+/* Interrupt enable register (IER) bits */
+#define BXCAN_IER_SLKIE BIT(17)
+#define BXCAN_IER_WKUIE BIT(16)
+#define BXCAN_IER_ERRIE BIT(15)
+#define BXCAN_IER_LECIE BIT(11)
+#define BXCAN_IER_BOFIE BIT(10)
+#define BXCAN_IER_EPVIE BIT(9)
+#define BXCAN_IER_EWGIE BIT(8)
+#define BXCAN_IER_FOVIE1 BIT(6)
+#define BXCAN_IER_FFIE1 BIT(5)
+#define BXCAN_IER_FMPIE1 BIT(4)
+#define BXCAN_IER_FOVIE0 BIT(3)
+#define BXCAN_IER_FFIE0 BIT(2)
+#define BXCAN_IER_FMPIE0 BIT(1)
+#define BXCAN_IER_TMEIE BIT(0)
+
+/* Error status register (ESR) bits */
+#define BXCAN_ESR_REC_MASK GENMASK(31, 24)
+#define BXCAN_ESR_TEC_MASK GENMASK(23, 16)
+#define BXCAN_ESR_LEC_MASK GENMASK(6, 4)
+#define BXCAN_ESR_BOFF BIT(2)
+#define BXCAN_ESR_EPVF BIT(1)
+#define BXCAN_ESR_EWGF BIT(0)
+
+/* Bit timing register (BTR) bits */
+#define BXCAN_BTR_SILM BIT(31)
+#define BXCAN_BTR_LBKM BIT(30)
+#define BXCAN_BTR_SJW_MASK GENMASK(25, 24)
+#define BXCAN_BTR_TS2_MASK GENMASK(22, 20)
+#define BXCAN_BTR_TS1_MASK GENMASK(19, 16)
+#define BXCAN_BTR_BRP_MASK GENMASK(9, 0)
+
+/* TX mailbox identifier register (TIxR, x = 0..2) bits */
+#define BXCAN_TIxR_STID_MASK GENMASK(31, 21)
+#define BXCAN_TIxR_EXID_MASK GENMASK(31, 3)
+#define BXCAN_TIxR_IDE BIT(2)
+#define BXCAN_TIxR_RTR BIT(1)
+#define BXCAN_TIxR_TXRQ BIT(0)
+
+/* TX mailbox data length and time stamp register (TDTxR, x = 0..2 bits */
+#define BXCAN_TDTxR_DLC_MASK GENMASK(3, 0)
+
+/* RX FIFO mailbox identifier register (RIxR, x = 0..1 */
+#define BXCAN_RIxR_STID_MASK GENMASK(31, 21)
+#define BXCAN_RIxR_EXID_MASK GENMASK(31, 3)
+#define BXCAN_RIxR_IDE BIT(2)
+#define BXCAN_RIxR_RTR BIT(1)
+
+/* RX FIFO mailbox data length and timestamp register (RDTxR, x = 0..1) bits */
+#define BXCAN_RDTxR_TIME_MASK GENMASK(31, 16)
+#define BXCAN_RDTxR_DLC_MASK GENMASK(3, 0)
+
+#define BXCAN_FMR_REG 0x00
+#define BXCAN_FM1R_REG 0x04
+#define BXCAN_FS1R_REG 0x0c
+#define BXCAN_FFA1R_REG 0x14
+#define BXCAN_FA1R_REG 0x1c
+#define BXCAN_FiR1_REG(b) (0x40 + (b) * 8)
+#define BXCAN_FiR2_REG(b) (0x44 + (b) * 8)
+
+#define BXCAN_FILTER_ID(primary) (primary ? 0 : 14)
+
+/* Filter primary register (FMR) bits */
+#define BXCAN_FMR_CANSB_MASK GENMASK(13, 8)
+#define BXCAN_FMR_FINIT BIT(0)
+
+enum bxcan_lec_code {
+	BXCAN_LEC_NO_ERROR = 0,
+	BXCAN_LEC_STUFF_ERROR,
+	BXCAN_LEC_FORM_ERROR,
+	BXCAN_LEC_ACK_ERROR,
+	BXCAN_LEC_BIT1_ERROR,
+	BXCAN_LEC_BIT0_ERROR,
+	BXCAN_LEC_CRC_ERROR,
+	BXCAN_LEC_UNUSED
+};
+
+/* Structure of the message buffer */
+struct bxcan_mb {
+	u32 id;			/* can identifier */
+	u32 dlc;		/* data length control and timestamp */
+	u32 data[2];		/* data */
+};
+
+/* Structure of the hardware registers */
+struct bxcan_regs {
+	u32 mcr;			/* 0x00 - primary control */
+	u32 msr;			/* 0x04 - primary status */
+	u32 tsr;			/* 0x08 - transmit status */
+	u32 rf0r;			/* 0x0c - FIFO 0 */
+	u32 rf1r;			/* 0x10 - FIFO 1 */
+	u32 ier;			/* 0x14 - interrupt enable */
+	u32 esr;			/* 0x18 - error status */
+	u32 btr;			/* 0x1c - bit timing*/
+	u32 reserved0[88];		/* 0x20 */
+	struct bxcan_mb tx_mb[BXCAN_TX_MB_NUM];	/* 0x180 - tx mailbox */
+	struct bxcan_mb rx_mb[BXCAN_RX_MB_NUM];	/* 0x1b0 - rx mailbox */
+};
+
+struct bxcan_priv {
+	struct can_priv can;
+	struct can_rx_offload offload;
+	struct device *dev;
+	struct net_device *ndev;
+
+	struct bxcan_regs __iomem *regs;
+	struct regmap *gcan;
+	int tx_irq;
+	int sce_irq;
+	bool primary;
+	struct clk *clk;
+	spinlock_t rmw_lock;	/* lock for read-modify-write operations */
+	unsigned int tx_head;
+	unsigned int tx_tail;
+	u32 timestamp;
+};
+
+static const struct can_bittiming_const bxcan_bittiming_const = {
+	.name = KBUILD_MODNAME,
+	.tseg1_min = 1,
+	.tseg1_max = 16,
+	.tseg2_min = 1,
+	.tseg2_max = 8,
+	.sjw_max = 4,
+	.brp_min = 1,
+	.brp_max = 1024,
+	.brp_inc = 1,
+};
+
+static inline void bxcan_rmw(struct bxcan_priv *priv, void __iomem *addr,
+			     u32 clear, u32 set)
+{
+	unsigned long flags;
+	u32 old, val;
+
+	spin_lock_irqsave(&priv->rmw_lock, flags);
+	old = readl(addr);
+	val = (old & ~clear) | set;
+	if (val != old)
+		writel(val, addr);
+
+	spin_unlock_irqrestore(&priv->rmw_lock, flags);
+}
+
+static void bxcan_disable_filters(struct bxcan_priv *priv, bool primary)
+{
+	unsigned int fid = BXCAN_FILTER_ID(primary);
+	u32 fmask = BIT(fid);
+
+	regmap_update_bits(priv->gcan, BXCAN_FA1R_REG, fmask, 0);
+}
+
+static void bxcan_enable_filters(struct bxcan_priv *priv, bool primary)
+{
+	unsigned int fid = BXCAN_FILTER_ID(primary);
+	u32 fmask = BIT(fid);
+
+	/* Filter settings:
+	 *
+	 * Accept all messages.
+	 * Assign filter 0 to CAN1 and filter 14 to CAN2 in identifier
+	 * mask mode with 32 bits width.
+	 */
+
+	/* Enter filter initialization mode and assing filters to CAN
+	 * controllers.
+	 */
+	regmap_update_bits(priv->gcan, BXCAN_FMR_REG,
+			   BXCAN_FMR_CANSB_MASK | BXCAN_FMR_FINIT,
+			   FIELD_PREP(BXCAN_FMR_CANSB_MASK, 14) |
+			   BXCAN_FMR_FINIT);
+
+	/* Deactivate filter */
+	regmap_update_bits(priv->gcan, BXCAN_FA1R_REG, fmask, 0);
+
+	/* Two 32-bit registers in identifier mask mode */
+	regmap_update_bits(priv->gcan, BXCAN_FM1R_REG, fmask, 0);
+
+	/* Single 32-bit scale configuration */
+	regmap_update_bits(priv->gcan, BXCAN_FS1R_REG, fmask, fmask);
+
+	/* Assign filter to FIFO 0 */
+	regmap_update_bits(priv->gcan, BXCAN_FFA1R_REG, fmask, 0);
+
+	/* Accept all messages */
+	regmap_write(priv->gcan, BXCAN_FiR1_REG(fid), 0);
+	regmap_write(priv->gcan, BXCAN_FiR2_REG(fid), 0);
+
+	/* Activate filter */
+	regmap_update_bits(priv->gcan, BXCAN_FA1R_REG, fmask, fmask);
+
+	/* Exit filter initialization mode */
+	regmap_update_bits(priv->gcan, BXCAN_FMR_REG, BXCAN_FMR_FINIT, 0);
+}
+
+static inline u8 bxcan_get_tx_head(const struct bxcan_priv *priv)
+{
+	return priv->tx_head % BXCAN_TX_MB_NUM;
+}
+
+static inline u8 bxcan_get_tx_tail(const struct bxcan_priv *priv)
+{
+	return priv->tx_tail % BXCAN_TX_MB_NUM;
+}
+
+static inline u8 bxcan_get_tx_free(const struct bxcan_priv *priv)
+{
+	return BXCAN_TX_MB_NUM - (priv->tx_head - priv->tx_tail);
+}
+
+static bool bxcan_tx_busy(const struct bxcan_priv *priv)
+{
+	if (bxcan_get_tx_free(priv) > 0)
+		return false;
+
+	netif_stop_queue(priv->ndev);
+
+	/* Memory barrier before checking tx_free (head and tail) */
+	smp_mb();
+
+	if (bxcan_get_tx_free(priv) == 0) {
+		netdev_dbg(priv->ndev,
+			   "Stopping tx-queue (tx_head=0x%08x, tx_tail=0x%08x, len=%d).\n",
+			   priv->tx_head, priv->tx_tail,
+			   priv->tx_head - priv->tx_tail);
+
+		return true;
+	}
+
+	netif_start_queue(priv->ndev);
+
+	return false;
+}
+
+static int bxcan_chip_softreset(struct bxcan_priv *priv)
+{
+	struct bxcan_regs __iomem *regs = priv->regs;
+	u32 value;
+
+	bxcan_rmw(priv, &regs->mcr, 0, BXCAN_MCR_RESET);
+	return readx_poll_timeout(readl, &regs->msr, value,
+				  value & BXCAN_MSR_SLAK, BXCAN_TIMEOUT_US,
+				  USEC_PER_SEC);
+}
+
+static int bxcan_enter_init_mode(struct bxcan_priv *priv)
+{
+	struct bxcan_regs __iomem *regs = priv->regs;
+	u32 value;
+
+	bxcan_rmw(priv, &regs->mcr, 0, BXCAN_MCR_INRQ);
+	return readx_poll_timeout(readl, &regs->msr, value,
+				  value & BXCAN_MSR_INAK, BXCAN_TIMEOUT_US,
+				  USEC_PER_SEC);
+}
+
+static int bxcan_leave_init_mode(struct bxcan_priv *priv)
+{
+	struct bxcan_regs __iomem *regs = priv->regs;
+	u32 value;
+
+	bxcan_rmw(priv, &regs->mcr, BXCAN_MCR_INRQ, 0);
+	return readx_poll_timeout(readl, &regs->msr, value,
+				  !(value & BXCAN_MSR_INAK), BXCAN_TIMEOUT_US,
+				  USEC_PER_SEC);
+}
+
+static int bxcan_enter_sleep_mode(struct bxcan_priv *priv)
+{
+	struct bxcan_regs __iomem *regs = priv->regs;
+	u32 value;
+
+	bxcan_rmw(priv, &regs->mcr, 0, BXCAN_MCR_SLEEP);
+	return readx_poll_timeout(readl, &regs->msr, value,
+				  value & BXCAN_MSR_SLAK, BXCAN_TIMEOUT_US,
+				  USEC_PER_SEC);
+}
+
+static int bxcan_leave_sleep_mode(struct bxcan_priv *priv)
+{
+	struct bxcan_regs __iomem *regs = priv->regs;
+	u32 value;
+
+	bxcan_rmw(priv, &regs->mcr, BXCAN_MCR_SLEEP, 0);
+	return readx_poll_timeout(readl, &regs->msr, value,
+				  !(value & BXCAN_MSR_SLAK), BXCAN_TIMEOUT_US,
+				  USEC_PER_SEC);
+}
+
+static inline
+struct bxcan_priv *rx_offload_to_priv(struct can_rx_offload *offload)
+{
+	return container_of(offload, struct bxcan_priv, offload);
+}
+
+static struct sk_buff *bxcan_mailbox_read(struct can_rx_offload *offload,
+					  unsigned int mbxno, u32 *timestamp,
+					  bool drop)
+{
+	struct bxcan_priv *priv = rx_offload_to_priv(offload);
+	struct bxcan_regs __iomem *regs = priv->regs;
+	struct bxcan_mb __iomem *mb_regs = &regs->rx_mb[0];
+	struct sk_buff *skb = NULL;
+	struct can_frame *cf;
+	u32 rf0r, id, dlc;
+
+	rf0r = readl(&regs->rf0r);
+	if (unlikely(drop)) {
+		skb = ERR_PTR(-ENOBUFS);
+		goto mark_as_read;
+	}
+
+	if (!(rf0r & BXCAN_RF0R_FMP0_MASK))
+		goto mark_as_read;
+
+	skb = alloc_can_skb(offload->dev, &cf);
+	if (unlikely(!skb)) {
+		skb = ERR_PTR(-ENOMEM);
+		goto mark_as_read;
+	}
+
+	id = readl(&mb_regs->id);
+	if (id & BXCAN_RIxR_IDE)
+		cf->can_id = FIELD_GET(BXCAN_RIxR_EXID_MASK, id) | CAN_EFF_FLAG;
+	else
+		cf->can_id = FIELD_GET(BXCAN_RIxR_STID_MASK, id) & CAN_SFF_MASK;
+
+	dlc = readl(&mb_regs->dlc);
+	priv->timestamp = FIELD_GET(BXCAN_RDTxR_TIME_MASK, dlc);
+	cf->len = can_cc_dlc2len(FIELD_GET(BXCAN_RDTxR_DLC_MASK, dlc));
+
+	if (id & BXCAN_RIxR_RTR) {
+		cf->can_id |= CAN_RTR_FLAG;
+	} else {
+		int i, j;
+
+		for (i = 0, j = 0; i < cf->len; i += 4, j++)
+			*(u32 *)(cf->data + i) = readl(&mb_regs->data[j]);
+	}
+
+ mark_as_read:
+	rf0r |= BXCAN_RF0R_RFOM0;
+	writel(rf0r, &regs->rf0r);
+	return skb;
+}
+
+static irqreturn_t bxcan_rx_isr(int irq, void *dev_id)
+{
+	struct net_device *ndev = dev_id;
+	struct bxcan_priv *priv = netdev_priv(ndev);
+	struct bxcan_regs __iomem *regs = priv->regs;
+	u32 rf0r;
+
+	rf0r = readl(&regs->rf0r);
+	if (!(rf0r & BXCAN_RF0R_FMP0_MASK))
+		return IRQ_NONE;
+
+	can_rx_offload_irq_offload_fifo(&priv->offload);
+	can_rx_offload_irq_finish(&priv->offload);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t bxcan_tx_isr(int irq, void *dev_id)
+{
+	struct net_device *ndev = dev_id;
+	struct bxcan_priv *priv = netdev_priv(ndev);
+	struct bxcan_regs __iomem *regs = priv->regs;
+	struct net_device_stats *stats = &ndev->stats;
+	u32 tsr, rqcp_bit;
+	int idx;
+
+	tsr = readl(&regs->tsr);
+	if (!(tsr & (BXCAN_TSR_RQCP0 | BXCAN_TSR_RQCP1 | BXCAN_TSR_RQCP2)))
+		return IRQ_NONE;
+
+	while (priv->tx_head - priv->tx_tail > 0) {
+		idx = bxcan_get_tx_tail(priv);
+		rqcp_bit = BXCAN_TSR_RQCP0 << (idx << 3);
+		if (!(tsr & rqcp_bit))
+			break;
+
+		stats->tx_packets++;
+		stats->tx_bytes += can_get_echo_skb(ndev, idx, NULL);
+		priv->tx_tail++;
+	}
+
+	writel(tsr, &regs->tsr);
+
+	if (bxcan_get_tx_free(priv)) {
+		/* Make sure that anybody stopping the queue after
+		 * this sees the new tx_ring->tail.
+		 */
+		smp_mb();
+		netif_wake_queue(ndev);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void bxcan_handle_state_change(struct net_device *ndev, u32 esr)
+{
+	struct bxcan_priv *priv = netdev_priv(ndev);
+	enum can_state new_state = priv->can.state;
+	struct can_berr_counter bec;
+	enum can_state rx_state, tx_state;
+	struct sk_buff *skb;
+	struct can_frame *cf;
+
+	/* Early exit if no error flag is set */
+	if (!(esr & (BXCAN_ESR_EWGF | BXCAN_ESR_EPVF | BXCAN_ESR_BOFF)))
+		return;
+
+	bec.txerr = FIELD_GET(BXCAN_ESR_TEC_MASK, esr);
+	bec.rxerr = FIELD_GET(BXCAN_ESR_REC_MASK, esr);
+
+	if (esr & BXCAN_ESR_BOFF)
+		new_state = CAN_STATE_BUS_OFF;
+	else if (esr & BXCAN_ESR_EPVF)
+		new_state = CAN_STATE_ERROR_PASSIVE;
+	else if (esr & BXCAN_ESR_EWGF)
+		new_state = CAN_STATE_ERROR_WARNING;
+
+	/* state hasn't changed */
+	if (unlikely(new_state == priv->can.state))
+		return;
+
+	skb = alloc_can_err_skb(ndev, &cf);
+
+	tx_state = bec.txerr >= bec.rxerr ? new_state : 0;
+	rx_state = bec.txerr <= bec.rxerr ? new_state : 0;
+	can_change_state(ndev, cf, tx_state, rx_state);
+
+	if (new_state == CAN_STATE_BUS_OFF) {
+		can_bus_off(ndev);
+	} else if (skb) {
+		cf->can_id |= CAN_ERR_CNT;
+		cf->data[6] = bec.txerr;
+		cf->data[7] = bec.rxerr;
+	}
+
+	if (skb) {
+		int err;
+
+		err = can_rx_offload_queue_timestamp(&priv->offload, skb,
+						     priv->timestamp);
+		if (err)
+			ndev->stats.rx_fifo_errors++;
+	}
+}
+
+static void bxcan_handle_bus_err(struct net_device *ndev, u32 esr)
+{
+	struct bxcan_priv *priv = netdev_priv(ndev);
+	enum bxcan_lec_code lec_code;
+	struct can_frame *cf;
+	struct sk_buff *skb;
+
+	lec_code = FIELD_GET(BXCAN_ESR_LEC_MASK, esr);
+
+	/* Early exit if no lec update or no error.
+	 * No lec update means that no CAN bus event has been detected
+	 * since CPU wrote BXCAN_LEC_UNUSED value to status reg.
+	 */
+	if (lec_code == BXCAN_LEC_UNUSED || lec_code == BXCAN_LEC_NO_ERROR)
+		return;
+
+	/* Common for all type of bus errors */
+	priv->can.can_stats.bus_error++;
+
+	/* Propagate the error condition to the CAN stack */
+	skb = alloc_can_err_skb(ndev, &cf);
+	if (skb)
+		cf->can_id |= CAN_ERR_PROT | CAN_ERR_BUSERROR;
+
+	switch (lec_code) {
+	case BXCAN_LEC_STUFF_ERROR:
+		netdev_dbg(ndev, "Stuff error\n");
+		ndev->stats.rx_errors++;
+		if (skb)
+			cf->data[2] |= CAN_ERR_PROT_STUFF;
+		break;
+
+	case BXCAN_LEC_FORM_ERROR:
+		netdev_dbg(ndev, "Form error\n");
+		ndev->stats.rx_errors++;
+		if (skb)
+			cf->data[2] |= CAN_ERR_PROT_FORM;
+		break;
+
+	case BXCAN_LEC_ACK_ERROR:
+		netdev_dbg(ndev, "Ack error\n");
+		ndev->stats.tx_errors++;
+		if (skb) {
+			cf->can_id |= CAN_ERR_ACK;
+			cf->data[3] = CAN_ERR_PROT_LOC_ACK;
+		}
+		break;
+
+	case BXCAN_LEC_BIT1_ERROR:
+		netdev_dbg(ndev, "Bit error (recessive)\n");
+		ndev->stats.tx_errors++;
+		if (skb)
+			cf->data[2] |= CAN_ERR_PROT_BIT1;
+		break;
+
+	case BXCAN_LEC_BIT0_ERROR:
+		netdev_dbg(ndev, "Bit error (dominant)\n");
+		ndev->stats.tx_errors++;
+		if (skb)
+			cf->data[2] |= CAN_ERR_PROT_BIT0;
+		break;
+
+	case BXCAN_LEC_CRC_ERROR:
+		netdev_dbg(ndev, "CRC error\n");
+		ndev->stats.rx_errors++;
+		if (skb) {
+			cf->data[2] |= CAN_ERR_PROT_BIT;
+			cf->data[3] = CAN_ERR_PROT_LOC_CRC_SEQ;
+		}
+		break;
+
+	default:
+		break;
+	}
+
+	if (skb) {
+		int err;
+
+		err = can_rx_offload_queue_timestamp(&priv->offload, skb,
+						     priv->timestamp);
+		if (err)
+			ndev->stats.rx_fifo_errors++;
+	}
+}
+
+static irqreturn_t bxcan_state_change_isr(int irq, void *dev_id)
+{
+	struct net_device *ndev = dev_id;
+	struct bxcan_priv *priv = netdev_priv(ndev);
+	struct bxcan_regs __iomem *regs = priv->regs;
+	u32 msr, esr;
+
+	msr = readl(&regs->msr);
+	if (!(msr & BXCAN_MSR_ERRI))
+		return IRQ_NONE;
+
+	esr = readl(&regs->esr);
+	bxcan_handle_state_change(ndev, esr);
+
+	if (priv->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING)
+		bxcan_handle_bus_err(ndev, esr);
+
+	msr |= BXCAN_MSR_ERRI;
+	writel(msr, &regs->msr);
+	can_rx_offload_irq_finish(&priv->offload);
+
+	return IRQ_HANDLED;
+}
+
+static int bxcan_chip_start(struct net_device *ndev)
+{
+	struct bxcan_priv *priv = netdev_priv(ndev);
+	struct bxcan_regs __iomem *regs = priv->regs;
+	struct can_bittiming *bt = &priv->can.bittiming;
+	u32 clr, set;
+	int err;
+
+	err = bxcan_chip_softreset(priv);
+	if (err) {
+		netdev_err(ndev, "failed to reset chip, error %pe\n",
+			   ERR_PTR(err));
+		return err;
+	}
+
+	err = bxcan_leave_sleep_mode(priv);
+	if (err) {
+		netdev_err(ndev, "failed to leave sleep mode, error %pe\n",
+			   ERR_PTR(err));
+		goto failed_leave_sleep;
+	}
+
+	err = bxcan_enter_init_mode(priv);
+	if (err) {
+		netdev_err(ndev, "failed to enter init mode, error %pe\n",
+			   ERR_PTR(err));
+		goto failed_enter_init;
+	}
+
+	/* MCR
+	 *
+	 * select request order priority
+	 * enable time triggered mode
+	 * bus-off state left on sw request
+	 * sleep mode left on sw request
+	 * retransmit automatically on error
+	 * do not lock RX FIFO on overrun
+	 */
+	bxcan_rmw(priv, &regs->mcr,
+		  BXCAN_MCR_ABOM | BXCAN_MCR_AWUM | BXCAN_MCR_NART |
+		  BXCAN_MCR_RFLM, BXCAN_MCR_TTCM | BXCAN_MCR_TXFP);
+
+	/* Bit timing register settings */
+	set = FIELD_PREP(BXCAN_BTR_BRP_MASK, bt->brp - 1) |
+		FIELD_PREP(BXCAN_BTR_TS1_MASK, bt->phase_seg1 +
+			   bt->prop_seg - 1) |
+		FIELD_PREP(BXCAN_BTR_TS2_MASK, bt->phase_seg2 - 1) |
+		FIELD_PREP(BXCAN_BTR_SJW_MASK, bt->sjw - 1);
+
+	/* loopback + silent mode put the controller in test mode,
+	 * useful for hot self-test
+	 */
+	if (priv->can.ctrlmode & CAN_CTRLMODE_LOOPBACK)
+		set |= BXCAN_BTR_LBKM;
+
+	if (priv->can.ctrlmode & CAN_CTRLMODE_LISTENONLY)
+		set |= BXCAN_BTR_SILM;
+
+	bxcan_rmw(priv, &regs->btr, BXCAN_BTR_SILM | BXCAN_BTR_LBKM |
+		  BXCAN_BTR_BRP_MASK | BXCAN_BTR_TS1_MASK | BXCAN_BTR_TS2_MASK |
+		  BXCAN_BTR_SJW_MASK, set);
+
+	bxcan_enable_filters(priv, priv->primary);
+
+	/* Clear all internal status */
+	priv->tx_head = 0;
+	priv->tx_tail = 0;
+
+	err = bxcan_leave_init_mode(priv);
+	if (err) {
+		netdev_err(ndev, "failed to leave init mode, error %pe\n",
+			   ERR_PTR(err));
+		goto failed_leave_init;
+	}
+
+	/* Set a `lec` value so that we can check for updates later */
+	bxcan_rmw(priv, &regs->esr, BXCAN_ESR_LEC_MASK,
+		  FIELD_PREP(BXCAN_ESR_LEC_MASK, BXCAN_LEC_UNUSED));
+
+	/* IER
+	 *
+	 * Enable interrupt for:
+	 * bus-off
+	 * passive error
+	 * warning error
+	 * last error code
+	 * RX FIFO pending message
+	 * TX mailbox empty
+	 */
+	clr = BXCAN_IER_WKUIE | BXCAN_IER_SLKIE |  BXCAN_IER_FOVIE1 |
+		BXCAN_IER_FFIE1 | BXCAN_IER_FMPIE1 | BXCAN_IER_FOVIE0 |
+		BXCAN_IER_FFIE0;
+	set = BXCAN_IER_ERRIE | BXCAN_IER_BOFIE | BXCAN_IER_EPVIE |
+		BXCAN_IER_EWGIE | BXCAN_IER_FMPIE0 | BXCAN_IER_TMEIE;
+
+	if (priv->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING)
+		set |= BXCAN_IER_LECIE;
+	else
+		clr |= BXCAN_IER_LECIE;
+
+	bxcan_rmw(priv, &regs->ier, clr, set);
+
+	priv->can.state = CAN_STATE_ERROR_ACTIVE;
+	return 0;
+
+failed_leave_init:
+failed_enter_init:
+failed_leave_sleep:
+	bxcan_chip_softreset(priv);
+	return err;
+}
+
+static int bxcan_open(struct net_device *ndev)
+{
+	struct bxcan_priv *priv = netdev_priv(ndev);
+	int err;
+
+	err = clk_prepare_enable(priv->clk);
+	if (err) {
+		netdev_err(ndev, "failed to enable clock, error %pe\n",
+			   ERR_PTR(err));
+		return err;
+	}
+
+	err = open_candev(ndev);
+	if (err) {
+		netdev_err(ndev, "open_candev() failed, error %pe\n",
+			   ERR_PTR(err));
+		goto out_disable_clock;
+	}
+
+	can_rx_offload_enable(&priv->offload);
+	err = request_irq(ndev->irq, bxcan_rx_isr, IRQF_SHARED, ndev->name,
+			  ndev);
+	if (err) {
+		netdev_err(ndev, "failed to register rx irq(%d), error %pe\n",
+			   ndev->irq, ERR_PTR(err));
+		goto out_close_candev;
+	}
+
+	err = request_irq(priv->tx_irq, bxcan_tx_isr, IRQF_SHARED, ndev->name,
+			  ndev);
+	if (err) {
+		netdev_err(ndev, "failed to register tx irq(%d), error %pe\n",
+			   priv->tx_irq, ERR_PTR(err));
+		goto out_free_rx_irq;
+	}
+
+	err = request_irq(priv->sce_irq, bxcan_state_change_isr, IRQF_SHARED,
+			  ndev->name, ndev);
+	if (err) {
+		netdev_err(ndev, "failed to register sce irq(%d), error %pe\n",
+			   priv->sce_irq, ERR_PTR(err));
+		goto out_free_tx_irq;
+	}
+
+	err = bxcan_chip_start(ndev);
+	if (err)
+		goto out_free_sce_irq;
+
+	netif_start_queue(ndev);
+	return 0;
+
+out_free_sce_irq:
+	free_irq(priv->sce_irq, ndev);
+out_free_tx_irq:
+	free_irq(priv->tx_irq, ndev);
+out_free_rx_irq:
+	free_irq(ndev->irq, ndev);
+out_close_candev:
+	can_rx_offload_disable(&priv->offload);
+	close_candev(ndev);
+out_disable_clock:
+	clk_disable_unprepare(priv->clk);
+	return err;
+}
+
+static void bxcan_chip_stop(struct net_device *ndev)
+{
+	struct bxcan_priv *priv = netdev_priv(ndev);
+	struct bxcan_regs __iomem *regs = priv->regs;
+
+	/* disable all interrupts */
+	bxcan_rmw(priv, &regs->ier, BXCAN_IER_SLKIE | BXCAN_IER_WKUIE |
+		  BXCAN_IER_ERRIE | BXCAN_IER_LECIE | BXCAN_IER_BOFIE |
+		  BXCAN_IER_EPVIE | BXCAN_IER_EWGIE | BXCAN_IER_FOVIE1 |
+		  BXCAN_IER_FFIE1 | BXCAN_IER_FMPIE1 | BXCAN_IER_FOVIE0 |
+		  BXCAN_IER_FFIE0 | BXCAN_IER_FMPIE0 | BXCAN_IER_TMEIE, 0);
+	bxcan_disable_filters(priv, priv->primary);
+	bxcan_enter_sleep_mode(priv);
+	priv->can.state = CAN_STATE_STOPPED;
+}
+
+static int bxcan_stop(struct net_device *ndev)
+{
+	struct bxcan_priv *priv = netdev_priv(ndev);
+
+	netif_stop_queue(ndev);
+	bxcan_chip_stop(ndev);
+	free_irq(ndev->irq, ndev);
+	free_irq(priv->tx_irq, ndev);
+	free_irq(priv->sce_irq, ndev);
+	can_rx_offload_disable(&priv->offload);
+	close_candev(ndev);
+	clk_disable_unprepare(priv->clk);
+	return 0;
+}
+
+static netdev_tx_t bxcan_start_xmit(struct sk_buff *skb,
+				    struct net_device *ndev)
+{
+	struct bxcan_priv *priv = netdev_priv(ndev);
+	struct can_frame *cf = (struct can_frame *)skb->data;
+	struct bxcan_regs __iomem *regs = priv->regs;
+	struct bxcan_mb __iomem *mb_regs;
+	unsigned int idx;
+	u32 id;
+	int i, j;
+
+	if (can_dropped_invalid_skb(ndev, skb))
+		return NETDEV_TX_OK;
+
+	if (bxcan_tx_busy(priv))
+		return NETDEV_TX_BUSY;
+
+	idx = bxcan_get_tx_head(priv);
+	priv->tx_head++;
+	if (bxcan_get_tx_free(priv) == 0)
+		netif_stop_queue(ndev);
+
+	mb_regs = &regs->tx_mb[idx];
+	if (cf->can_id & CAN_EFF_FLAG)
+		id = FIELD_PREP(BXCAN_TIxR_EXID_MASK, cf->can_id) |
+			BXCAN_TIxR_IDE;
+	else
+		id = FIELD_PREP(BXCAN_TIxR_STID_MASK, cf->can_id);
+
+	if (cf->can_id & CAN_RTR_FLAG) { /* Remote transmission request */
+		id |= BXCAN_TIxR_RTR;
+	} else {
+		for (i = 0, j = 0; i < cf->len; i += 4, j++)
+			writel(*(u32 *)(cf->data + i), &mb_regs->data[j]);
+	}
+
+	writel(FIELD_PREP(BXCAN_TDTxR_DLC_MASK, cf->len), &mb_regs->dlc);
+
+	can_put_echo_skb(skb, ndev, idx, 0);
+
+	/* Start transmission */
+	writel(id | BXCAN_TIxR_TXRQ, &mb_regs->id);
+
+	return NETDEV_TX_OK;
+}
+
+static const struct net_device_ops bxcan_netdev_ops = {
+	.ndo_open = bxcan_open,
+	.ndo_stop = bxcan_stop,
+	.ndo_start_xmit = bxcan_start_xmit,
+	.ndo_change_mtu = can_change_mtu,
+};
+
+static const struct ethtool_ops bxcan_ethtool_ops = {
+	.get_ts_info = ethtool_op_get_ts_info,
+};
+
+static int bxcan_do_set_mode(struct net_device *ndev, enum can_mode mode)
+{
+	int err;
+
+	switch (mode) {
+	case CAN_MODE_START:
+		err = bxcan_chip_start(ndev);
+		if (err)
+			return err;
+
+		netif_wake_queue(ndev);
+		break;
+
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int bxcan_get_berr_counter(const struct net_device *ndev,
+				  struct can_berr_counter *bec)
+{
+	struct bxcan_priv *priv = netdev_priv(ndev);
+	struct bxcan_regs __iomem *regs = priv->regs;
+	u32 esr;
+	int err;
+
+	err = clk_prepare_enable(priv->clk);
+	if (err)
+		return err;
+
+	esr = readl(&regs->esr);
+	bec->txerr = FIELD_GET(BXCAN_ESR_TEC_MASK, esr);
+	bec->rxerr = FIELD_GET(BXCAN_ESR_REC_MASK, esr);
+	clk_disable_unprepare(priv->clk);
+	return 0;
+}
+
+static int bxcan_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+	struct net_device *ndev;
+	struct bxcan_priv *priv;
+	struct clk *clk = NULL;
+	void __iomem *regs;
+	struct regmap *gcan;
+	bool primary;
+	int err, rx_irq, tx_irq, sce_irq;
+
+	regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(regs)) {
+		dev_err(dev, "failed to get base address\n");
+		return PTR_ERR(regs);
+	}
+
+	gcan = syscon_regmap_lookup_by_phandle(np, "st,gcan");
+	if (IS_ERR(gcan)) {
+		dev_err(dev, "failed to get shared memory base address\n");
+		return PTR_ERR(gcan);
+	}
+
+	primary = of_property_read_bool(np, "st,can-primary");
+	clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(clk)) {
+		dev_err(dev, "failed to get clock\n");
+		return PTR_ERR(clk);
+	}
+
+	rx_irq = platform_get_irq_byname(pdev, "rx0");
+	if (rx_irq < 0) {
+		dev_err(dev, "failed to get rx0 irq\n");
+		return rx_irq;
+	}
+
+	tx_irq = platform_get_irq_byname(pdev, "tx");
+	if (tx_irq < 0) {
+		dev_err(dev, "failed to get tx irq\n");
+		return tx_irq;
+	}
+
+	sce_irq = platform_get_irq_byname(pdev, "sce");
+	if (sce_irq < 0) {
+		dev_err(dev, "failed to get sce irq\n");
+		return sce_irq;
+	}
+
+	ndev = alloc_candev(sizeof(struct bxcan_priv), BXCAN_TX_MB_NUM);
+	if (!ndev) {
+		dev_err(dev, "alloc_candev() failed\n");
+		return -ENOMEM;
+	}
+
+	priv = netdev_priv(ndev);
+	platform_set_drvdata(pdev, ndev);
+	SET_NETDEV_DEV(ndev, dev);
+	ndev->netdev_ops = &bxcan_netdev_ops;
+	ndev->ethtool_ops = &bxcan_ethtool_ops;
+	ndev->irq = rx_irq;
+	ndev->flags |= IFF_ECHO;
+
+	priv->dev = dev;
+	priv->ndev = ndev;
+	priv->regs = regs;
+	priv->gcan = gcan;
+	priv->clk = clk;
+	priv->tx_irq = tx_irq;
+	priv->sce_irq = sce_irq;
+	priv->primary = primary;
+	priv->can.clock.freq = clk_get_rate(clk);
+	spin_lock_init(&priv->rmw_lock);
+	priv->tx_head = 0;
+	priv->tx_tail = 0;
+	priv->can.bittiming_const = &bxcan_bittiming_const;
+	priv->can.do_set_mode = bxcan_do_set_mode;
+	priv->can.do_get_berr_counter = bxcan_get_berr_counter;
+	priv->can.ctrlmode_supported = CAN_CTRLMODE_LOOPBACK |
+		CAN_CTRLMODE_LISTENONLY	| CAN_CTRLMODE_BERR_REPORTING;
+
+	priv->offload.mailbox_read = bxcan_mailbox_read;
+	err = can_rx_offload_add_fifo(ndev, &priv->offload, BXCAN_NAPI_WEIGHT);
+	if (err) {
+		dev_err(dev, "failed to add FIFO rx_offload\n");
+		goto out_free_candev;
+	}
+
+	err = register_candev(ndev);
+	if (err) {
+		dev_err(dev, "failed to register netdev\n");
+		goto out_can_rx_offload_del;
+	}
+
+	dev_info(dev, "clk: %d Hz, IRQs: %d, %d, %d\n", priv->can.clock.freq,
+		 tx_irq, rx_irq, sce_irq);
+	return 0;
+
+out_can_rx_offload_del:
+	can_rx_offload_del(&priv->offload);
+out_free_candev:
+	free_candev(ndev);
+	return err;
+}
+
+static int bxcan_remove(struct platform_device *pdev)
+{
+	struct net_device *ndev = platform_get_drvdata(pdev);
+	struct bxcan_priv *priv = netdev_priv(ndev);
+
+	unregister_candev(ndev);
+	clk_disable_unprepare(priv->clk);
+	can_rx_offload_del(&priv->offload);
+	free_candev(ndev);
+	return 0;
+}
+
+static int __maybe_unused bxcan_suspend(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct bxcan_priv *priv = netdev_priv(ndev);
+
+	if (!netif_running(ndev))
+		return 0;
+
+	netif_stop_queue(ndev);
+	netif_device_detach(ndev);
+
+	bxcan_enter_sleep_mode(priv);
+	priv->can.state = CAN_STATE_SLEEPING;
+	clk_disable_unprepare(priv->clk);
+	return 0;
+}
+
+static int __maybe_unused bxcan_resume(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct bxcan_priv *priv = netdev_priv(ndev);
+
+	if (!netif_running(ndev))
+		return 0;
+
+	clk_prepare_enable(priv->clk);
+	bxcan_leave_sleep_mode(priv);
+	priv->can.state = CAN_STATE_ERROR_ACTIVE;
+
+	netif_device_attach(ndev);
+	netif_start_queue(ndev);
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(bxcan_pm_ops, bxcan_suspend, bxcan_resume);
+
+static const struct of_device_id bxcan_of_match[] = {
+	{.compatible = "st,stm32f4-bxcan"},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, bxcan_of_match);
+
+static struct platform_driver bxcan_driver = {
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.pm = &bxcan_pm_ops,
+		.of_match_table = bxcan_of_match,
+	},
+	.probe = bxcan_probe,
+	.remove = bxcan_remove,
+};
+
+module_platform_driver(bxcan_driver);
+
+MODULE_AUTHOR("Dario Binacchi <dario.binacchi@amarulasolutions.com>");
+MODULE_DESCRIPTION("STMicroelectronics Basic Extended CAN controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/can/c_can/c_can_pci.c b/drivers/net/can/c_can/c_can_pci.c
index bf2f8c3da1c1..093bea597f4e 100644
--- a/drivers/net/can/c_can/c_can_pci.c
+++ b/drivers/net/can/c_can/c_can_pci.c
@@ -227,7 +227,6 @@ out_iounmap:
 	pci_iounmap(pdev, addr);
 out_release_regions:
 	pci_disable_msi(pdev);
-	pci_clear_master(pdev);
 	pci_release_regions(pdev);
 out_disable_device:
 	pci_disable_device(pdev);
@@ -247,7 +246,6 @@ static void c_can_pci_remove(struct pci_dev *pdev)
 
 	pci_iounmap(pdev, addr);
 	pci_disable_msi(pdev);
-	pci_clear_master(pdev);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
 }
diff --git a/drivers/net/can/ctucanfd/ctucanfd_pci.c b/drivers/net/can/ctucanfd/ctucanfd_pci.c
index 8f2956a8ae43..9da09e7dd63a 100644
--- a/drivers/net/can/ctucanfd/ctucanfd_pci.c
+++ b/drivers/net/can/ctucanfd/ctucanfd_pci.c
@@ -206,10 +206,8 @@ err_pci_iounmap_bar0:
 err_pci_iounmap_bar1:
 	pci_iounmap(pdev, addr);
 err_release_regions:
-	if (msi_ok) {
+	if (msi_ok)
 		pci_disable_msi(pdev);
-		pci_clear_master(pdev);
-	}
 	pci_release_regions(pdev);
 err_disable_device:
 	pci_disable_device(pdev);
@@ -257,10 +255,8 @@ static void ctucan_pci_remove(struct pci_dev *pdev)
 
 	pci_iounmap(pdev, bdata->bar1_base);
 
-	if (bdata->use_msi) {
+	if (bdata->use_msi)
 		pci_disable_msi(pdev);
-		pci_clear_master(pdev);
-	}
 
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
diff --git a/drivers/net/can/kvaser_pciefd.c b/drivers/net/can/kvaser_pciefd.c
index bcad11709bc9..53e8a914c88b 100644
--- a/drivers/net/can/kvaser_pciefd.c
+++ b/drivers/net/can/kvaser_pciefd.c
@@ -1907,7 +1907,6 @@ static void kvaser_pciefd_remove(struct pci_dev *pdev)
 
 	free_irq(pcie->pci->irq, pcie);
 
-	pci_clear_master(pdev);
 	pci_iounmap(pdev, pcie->reg_base);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c
index 8e83d6963d85..a5003435802b 100644
--- a/drivers/net/can/m_can/m_can.c
+++ b/drivers/net/can/m_can/m_can.c
@@ -972,8 +972,8 @@ static int m_can_rx_peripheral(struct net_device *dev, u32 irqstatus)
 	/* Don't re-enable interrupts if the driver had a fatal error
 	 * (e.g., FIFO read failure).
 	 */
-	if (work_done >= 0)
-		m_can_enable_all_interrupts(cdev);
+	if (work_done < 0)
+		m_can_disable_all_interrupts(cdev);
 
 	return work_done;
 }
@@ -1083,8 +1083,7 @@ static irqreturn_t m_can_isr(int irq, void *dev_id)
 		return IRQ_NONE;
 
 	/* ACK all irqs */
-	if (ir & IR_ALL_INT)
-		m_can_write(cdev, M_CAN_IR, ir);
+	m_can_write(cdev, M_CAN_IR, ir);
 
 	if (cdev->ops->clear_interrupts)
 		cdev->ops->clear_interrupts(cdev);
@@ -1096,11 +1095,12 @@ static irqreturn_t m_can_isr(int irq, void *dev_id)
 	 */
 	if ((ir & IR_RF0N) || (ir & IR_ERR_ALL_30X)) {
 		cdev->irqstatus = ir;
-		m_can_disable_all_interrupts(cdev);
-		if (!cdev->is_peripheral)
+		if (!cdev->is_peripheral) {
+			m_can_disable_all_interrupts(cdev);
 			napi_schedule(&cdev->napi);
-		else if (m_can_rx_peripheral(dev, ir) < 0)
+		} else if (m_can_rx_peripheral(dev, ir) < 0) {
 			goto out_fail;
+		}
 	}
 
 	if (cdev->version == 30) {
@@ -1262,6 +1262,7 @@ static int m_can_set_bittiming(struct net_device *dev)
 static int m_can_chip_config(struct net_device *dev)
 {
 	struct m_can_classdev *cdev = netdev_priv(dev);
+	u32 interrupts = IR_ALL_INT;
 	u32 cccr, test;
 	int err;
 
@@ -1271,6 +1272,11 @@ static int m_can_chip_config(struct net_device *dev)
 		return err;
 	}
 
+	/* Disable unused interrupts */
+	interrupts &= ~(IR_ARA | IR_ELO | IR_DRX | IR_TEFF | IR_TEFW | IR_TFE |
+			IR_TCF | IR_HPM | IR_RF1F | IR_RF1W | IR_RF1N |
+			IR_RF0F | IR_RF0W);
+
 	m_can_config_endisable(cdev, true);
 
 	/* RX Buffer/FIFO Element Size 64 bytes data field */
@@ -1365,16 +1371,13 @@ static int m_can_chip_config(struct net_device *dev)
 	m_can_write(cdev, M_CAN_TEST, test);
 
 	/* Enable interrupts */
-	m_can_write(cdev, M_CAN_IR, IR_ALL_INT);
-	if (!(cdev->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING))
+	if (!(cdev->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING)) {
 		if (cdev->version == 30)
-			m_can_write(cdev, M_CAN_IE, IR_ALL_INT &
-				    ~(IR_ERR_LEC_30X));
+			interrupts &= ~(IR_ERR_LEC_30X);
 		else
-			m_can_write(cdev, M_CAN_IE, IR_ALL_INT &
-				    ~(IR_ERR_LEC_31X));
-	else
-		m_can_write(cdev, M_CAN_IE, IR_ALL_INT);
+			interrupts &= ~(IR_ERR_LEC_31X);
+	}
+	m_can_write(cdev, M_CAN_IE, interrupts);
 
 	/* route all interrupts to INT0 */
 	m_can_write(cdev, M_CAN_ILS, ILS_ALL_INT0);
@@ -1592,10 +1595,8 @@ static int m_can_close(struct net_device *dev)
 		cdev->tx_skb = NULL;
 		destroy_workqueue(cdev->tx_wq);
 		cdev->tx_wq = NULL;
-	}
-
-	if (cdev->is_peripheral)
 		can_rx_offload_disable(&cdev->offload);
+	}
 
 	close_candev(dev);
 
diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c
index ef4e1b9a9e1e..963c42f43755 100644
--- a/drivers/net/can/rcar/rcar_canfd.c
+++ b/drivers/net/can/rcar/rcar_canfd.c
@@ -35,6 +35,7 @@
 #include <linux/netdevice.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/reset.h>
 #include <linux/types.h>
@@ -530,6 +531,7 @@ struct rcar_canfd_channel {
 	struct net_device *ndev;
 	struct rcar_canfd_global *gpriv;	/* Controller reference */
 	void __iomem *base;			/* Register base address */
+	struct phy *transceiver;		/* Optional transceiver */
 	struct napi_struct napi;
 	u32 tx_head;				/* Incremented on xmit */
 	u32 tx_tail;				/* Incremented on xmit done */
@@ -1413,16 +1415,22 @@ static int rcar_canfd_open(struct net_device *ndev)
 	struct rcar_canfd_global *gpriv = priv->gpriv;
 	int err;
 
+	err = phy_power_on(priv->transceiver);
+	if (err) {
+		netdev_err(ndev, "failed to power on PHY: %pe\n", ERR_PTR(err));
+		return err;
+	}
+
 	/* Peripheral clock is already enabled in probe */
 	err = clk_prepare_enable(gpriv->can_clk);
 	if (err) {
-		netdev_err(ndev, "failed to enable CAN clock, error %d\n", err);
-		goto out_clock;
+		netdev_err(ndev, "failed to enable CAN clock: %pe\n", ERR_PTR(err));
+		goto out_phy;
 	}
 
 	err = open_candev(ndev);
 	if (err) {
-		netdev_err(ndev, "open_candev() failed, error %d\n", err);
+		netdev_err(ndev, "open_candev() failed: %pe\n", ERR_PTR(err));
 		goto out_can_clock;
 	}
 
@@ -1437,7 +1445,8 @@ out_close:
 	close_candev(ndev);
 out_can_clock:
 	clk_disable_unprepare(gpriv->can_clk);
-out_clock:
+out_phy:
+	phy_power_off(priv->transceiver);
 	return err;
 }
 
@@ -1480,6 +1489,7 @@ static int rcar_canfd_close(struct net_device *ndev)
 	napi_disable(&priv->napi);
 	clk_disable_unprepare(gpriv->can_clk);
 	close_candev(ndev);
+	phy_power_off(priv->transceiver);
 	return 0;
 }
 
@@ -1711,7 +1721,7 @@ static const struct ethtool_ops rcar_canfd_ethtool_ops = {
 };
 
 static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch,
-				    u32 fcan_freq)
+				    u32 fcan_freq, struct phy *transceiver)
 {
 	const struct rcar_canfd_hw_info *info = gpriv->info;
 	struct platform_device *pdev = gpriv->pdev;
@@ -1721,10 +1731,9 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch,
 	int err = -ENODEV;
 
 	ndev = alloc_candev(sizeof(*priv), RCANFD_FIFO_DEPTH);
-	if (!ndev) {
-		dev_err(dev, "alloc_candev() failed\n");
+	if (!ndev)
 		return -ENOMEM;
-	}
+
 	priv = netdev_priv(ndev);
 
 	ndev->netdev_ops = &rcar_canfd_netdev_ops;
@@ -1732,8 +1741,11 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch,
 	ndev->flags |= IFF_ECHO;
 	priv->ndev = ndev;
 	priv->base = gpriv->base;
+	priv->transceiver = transceiver;
 	priv->channel = ch;
 	priv->gpriv = gpriv;
+	if (transceiver)
+		priv->can.bitrate_max = transceiver->attrs.max_link_rate;
 	priv->can.clock.freq = fcan_freq;
 	dev_info(dev, "can_clk rate is %u\n", priv->can.clock.freq);
 
@@ -1764,8 +1776,8 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch,
 				       rcar_canfd_channel_err_interrupt, 0,
 				       irq_name, priv);
 		if (err) {
-			dev_err(dev, "devm_request_irq CH Err(%d) failed, error %d\n",
-				err_irq, err);
+			dev_err(dev, "devm_request_irq CH Err %d failed: %pe\n",
+				err_irq, ERR_PTR(err));
 			goto fail;
 		}
 		irq_name = devm_kasprintf(dev, GFP_KERNEL, "canfd.ch%d_trx",
@@ -1778,8 +1790,8 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch,
 				       rcar_canfd_channel_tx_interrupt, 0,
 				       irq_name, priv);
 		if (err) {
-			dev_err(dev, "devm_request_irq Tx (%d) failed, error %d\n",
-				tx_irq, err);
+			dev_err(dev, "devm_request_irq Tx %d failed: %pe\n",
+				tx_irq, ERR_PTR(err));
 			goto fail;
 		}
 	}
@@ -1810,7 +1822,7 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch,
 	gpriv->ch[priv->channel] = priv;
 	err = register_candev(ndev);
 	if (err) {
-		dev_err(dev, "register_candev() failed, error %d\n", err);
+		dev_err(dev, "register_candev() failed: %pe\n", ERR_PTR(err));
 		goto fail_candev;
 	}
 	dev_info(dev, "device registered (channel %u)\n", priv->channel);
@@ -1836,6 +1848,7 @@ static void rcar_canfd_channel_remove(struct rcar_canfd_global *gpriv, u32 ch)
 
 static int rcar_canfd_probe(struct platform_device *pdev)
 {
+	struct phy *transceivers[RCANFD_NUM_CHANNELS] = { NULL, };
 	const struct rcar_canfd_hw_info *info;
 	struct device *dev = &pdev->dev;
 	void __iomem *addr;
@@ -1857,9 +1870,14 @@ static int rcar_canfd_probe(struct platform_device *pdev)
 	for (i = 0; i < info->max_channels; ++i) {
 		name[7] = '0' + i;
 		of_child = of_get_child_by_name(dev->of_node, name);
-		if (of_child && of_device_is_available(of_child))
+		if (of_child && of_device_is_available(of_child)) {
 			channels_mask |= BIT(i);
+			transceivers[i] = devm_of_phy_optional_get(dev,
+							of_child, NULL);
+		}
 		of_node_put(of_child);
+		if (IS_ERR(transceivers[i]))
+			return PTR_ERR(transceivers[i]);
 	}
 
 	if (info->shared_global_irqs) {
@@ -1948,16 +1966,16 @@ static int rcar_canfd_probe(struct platform_device *pdev)
 				       rcar_canfd_channel_interrupt, 0,
 				       "canfd.ch_int", gpriv);
 		if (err) {
-			dev_err(dev, "devm_request_irq(%d) failed, error %d\n",
-				ch_irq, err);
+			dev_err(dev, "devm_request_irq %d failed: %pe\n",
+				ch_irq, ERR_PTR(err));
 			goto fail_dev;
 		}
 
 		err = devm_request_irq(dev, g_irq, rcar_canfd_global_interrupt,
 				       0, "canfd.g_int", gpriv);
 		if (err) {
-			dev_err(dev, "devm_request_irq(%d) failed, error %d\n",
-				g_irq, err);
+			dev_err(dev, "devm_request_irq %d failed: %pe\n",
+				g_irq, ERR_PTR(err));
 			goto fail_dev;
 		}
 	} else {
@@ -1966,8 +1984,8 @@ static int rcar_canfd_probe(struct platform_device *pdev)
 				       "canfd.g_recc", gpriv);
 
 		if (err) {
-			dev_err(dev, "devm_request_irq(%d) failed, error %d\n",
-				g_recc_irq, err);
+			dev_err(dev, "devm_request_irq %d failed: %pe\n",
+				g_recc_irq, ERR_PTR(err));
 			goto fail_dev;
 		}
 
@@ -1975,8 +1993,8 @@ static int rcar_canfd_probe(struct platform_device *pdev)
 				       rcar_canfd_global_err_interrupt, 0,
 				       "canfd.g_err", gpriv);
 		if (err) {
-			dev_err(dev, "devm_request_irq(%d) failed, error %d\n",
-				g_err_irq, err);
+			dev_err(dev, "devm_request_irq %d failed: %pe\n",
+				g_err_irq, ERR_PTR(err));
 			goto fail_dev;
 		}
 	}
@@ -1993,14 +2011,14 @@ static int rcar_canfd_probe(struct platform_device *pdev)
 	/* Enable peripheral clock for register access */
 	err = clk_prepare_enable(gpriv->clkp);
 	if (err) {
-		dev_err(dev, "failed to enable peripheral clock, error %d\n",
-			err);
+		dev_err(dev, "failed to enable peripheral clock: %pe\n",
+			ERR_PTR(err));
 		goto fail_reset;
 	}
 
 	err = rcar_canfd_reset_controller(gpriv);
 	if (err) {
-		dev_err(dev, "reset controller failed\n");
+		dev_err(dev, "reset controller failed: %pe\n", ERR_PTR(err));
 		goto fail_clk;
 	}
 
@@ -2035,7 +2053,8 @@ static int rcar_canfd_probe(struct platform_device *pdev)
 	}
 
 	for_each_set_bit(ch, &gpriv->channels_mask, info->max_channels) {
-		err = rcar_canfd_channel_probe(gpriv, ch, fcan_freq);
+		err = rcar_canfd_channel_probe(gpriv, ch, fcan_freq,
+					       transceivers[ch]);
 		if (err)
 			goto fail_channel;
 	}
diff --git a/drivers/net/can/usb/esd_usb.c b/drivers/net/can/usb/esd_usb.c
index 55b36973952d..d33bac3a6c10 100644
--- a/drivers/net/can/usb/esd_usb.c
+++ b/drivers/net/can/usb/esd_usb.c
@@ -174,17 +174,15 @@ struct set_baudrate_msg {
 };
 
 /* Main message type used between library and application */
-struct __packed esd_usb_msg {
-	union {
-		struct header_msg hdr;
-		struct version_msg version;
-		struct version_reply_msg version_reply;
-		struct rx_msg rx;
-		struct tx_msg tx;
-		struct tx_done_msg txdone;
-		struct set_baudrate_msg setbaud;
-		struct id_filter_msg filter;
-	} msg;
+union __packed esd_usb_msg {
+	struct header_msg hdr;
+	struct version_msg version;
+	struct version_reply_msg version_reply;
+	struct rx_msg rx;
+	struct tx_msg tx;
+	struct tx_done_msg txdone;
+	struct set_baudrate_msg setbaud;
+	struct id_filter_msg filter;
 };
 
 static struct usb_device_id esd_usb_table[] = {
@@ -229,24 +227,33 @@ struct esd_usb_net_priv {
 };
 
 static void esd_usb_rx_event(struct esd_usb_net_priv *priv,
-			     struct esd_usb_msg *msg)
+			     union esd_usb_msg *msg)
 {
 	struct net_device_stats *stats = &priv->netdev->stats;
 	struct can_frame *cf;
 	struct sk_buff *skb;
-	u32 id = le32_to_cpu(msg->msg.rx.id) & ESD_IDMASK;
+	u32 id = le32_to_cpu(msg->rx.id) & ESD_IDMASK;
 
 	if (id == ESD_EV_CAN_ERROR_EXT) {
-		u8 state = msg->msg.rx.ev_can_err_ext.status;
-		u8 ecc = msg->msg.rx.ev_can_err_ext.ecc;
-		u8 rxerr = msg->msg.rx.ev_can_err_ext.rec;
-		u8 txerr = msg->msg.rx.ev_can_err_ext.tec;
+		u8 state = msg->rx.ev_can_err_ext.status;
+		u8 ecc = msg->rx.ev_can_err_ext.ecc;
+
+		priv->bec.rxerr = msg->rx.ev_can_err_ext.rec;
+		priv->bec.txerr = msg->rx.ev_can_err_ext.tec;
 
 		netdev_dbg(priv->netdev,
 			   "CAN_ERR_EV_EXT: dlc=%#02x state=%02x ecc=%02x rec=%02x tec=%02x\n",
-			   msg->msg.rx.dlc, state, ecc, rxerr, txerr);
+			   msg->rx.dlc, state, ecc,
+			   priv->bec.rxerr, priv->bec.txerr);
+
+		/* if berr-reporting is off, only pass through on state change ... */
+		if (!(priv->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING) &&
+		    state == priv->old_state)
+			return;
 
 		skb = alloc_can_err_skb(priv->netdev, &cf);
+		if (!skb)
+			stats->rx_dropped++;
 
 		if (state != priv->old_state) {
 			enum can_state tx_state, rx_state;
@@ -267,14 +274,14 @@ static void esd_usb_rx_event(struct esd_usb_net_priv *priv,
 				break;
 			default:
 				new_state = CAN_STATE_ERROR_ACTIVE;
-				txerr = 0;
-				rxerr = 0;
+				priv->bec.txerr = 0;
+				priv->bec.rxerr = 0;
 				break;
 			}
 
 			if (new_state != priv->can.state) {
-				tx_state = (txerr >= rxerr) ? new_state : 0;
-				rx_state = (txerr <= rxerr) ? new_state : 0;
+				tx_state = (priv->bec.txerr >= priv->bec.rxerr) ? new_state : 0;
+				rx_state = (priv->bec.txerr <= priv->bec.rxerr) ? new_state : 0;
 				can_change_state(priv->netdev, cf,
 						 tx_state, rx_state);
 			}
@@ -306,23 +313,18 @@ static void esd_usb_rx_event(struct esd_usb_net_priv *priv,
 			cf->data[3] = ecc & SJA1000_ECC_SEG;
 		}
 
-		priv->bec.txerr = txerr;
-		priv->bec.rxerr = rxerr;
-
 		if (skb) {
 			cf->can_id |= CAN_ERR_CNT;
-			cf->data[6] = txerr;
-			cf->data[7] = rxerr;
+			cf->data[6] = priv->bec.txerr;
+			cf->data[7] = priv->bec.rxerr;
 
 			netif_rx(skb);
-		} else {
-			stats->rx_dropped++;
 		}
 	}
 }
 
 static void esd_usb_rx_can_msg(struct esd_usb_net_priv *priv,
-			       struct esd_usb_msg *msg)
+			       union esd_usb_msg *msg)
 {
 	struct net_device_stats *stats = &priv->netdev->stats;
 	struct can_frame *cf;
@@ -333,7 +335,7 @@ static void esd_usb_rx_can_msg(struct esd_usb_net_priv *priv,
 	if (!netif_device_present(priv->netdev))
 		return;
 
-	id = le32_to_cpu(msg->msg.rx.id);
+	id = le32_to_cpu(msg->rx.id);
 
 	if (id & ESD_EVENT) {
 		esd_usb_rx_event(priv, msg);
@@ -345,17 +347,17 @@ static void esd_usb_rx_can_msg(struct esd_usb_net_priv *priv,
 		}
 
 		cf->can_id = id & ESD_IDMASK;
-		can_frame_set_cc_len(cf, msg->msg.rx.dlc & ~ESD_RTR,
+		can_frame_set_cc_len(cf, msg->rx.dlc & ~ESD_RTR,
 				     priv->can.ctrlmode);
 
 		if (id & ESD_EXTID)
 			cf->can_id |= CAN_EFF_FLAG;
 
-		if (msg->msg.rx.dlc & ESD_RTR) {
+		if (msg->rx.dlc & ESD_RTR) {
 			cf->can_id |= CAN_RTR_FLAG;
 		} else {
 			for (i = 0; i < cf->len; i++)
-				cf->data[i] = msg->msg.rx.data[i];
+				cf->data[i] = msg->rx.data[i];
 
 			stats->rx_bytes += cf->len;
 		}
@@ -366,7 +368,7 @@ static void esd_usb_rx_can_msg(struct esd_usb_net_priv *priv,
 }
 
 static void esd_usb_tx_done_msg(struct esd_usb_net_priv *priv,
-				struct esd_usb_msg *msg)
+				union esd_usb_msg *msg)
 {
 	struct net_device_stats *stats = &priv->netdev->stats;
 	struct net_device *netdev = priv->netdev;
@@ -375,9 +377,9 @@ static void esd_usb_tx_done_msg(struct esd_usb_net_priv *priv,
 	if (!netif_device_present(netdev))
 		return;
 
-	context = &priv->tx_contexts[msg->msg.txdone.hnd & (MAX_TX_URBS - 1)];
+	context = &priv->tx_contexts[msg->txdone.hnd & (MAX_TX_URBS - 1)];
 
-	if (!msg->msg.txdone.status) {
+	if (!msg->txdone.status) {
 		stats->tx_packets++;
 		stats->tx_bytes += can_get_echo_skb(netdev, context->echo_index,
 						    NULL);
@@ -417,32 +419,32 @@ static void esd_usb_read_bulk_callback(struct urb *urb)
 	}
 
 	while (pos < urb->actual_length) {
-		struct esd_usb_msg *msg;
+		union esd_usb_msg *msg;
 
-		msg = (struct esd_usb_msg *)(urb->transfer_buffer + pos);
+		msg = (union esd_usb_msg *)(urb->transfer_buffer + pos);
 
-		switch (msg->msg.hdr.cmd) {
+		switch (msg->hdr.cmd) {
 		case CMD_CAN_RX:
-			if (msg->msg.rx.net >= dev->net_count) {
+			if (msg->rx.net >= dev->net_count) {
 				dev_err(dev->udev->dev.parent, "format error\n");
 				break;
 			}
 
-			esd_usb_rx_can_msg(dev->nets[msg->msg.rx.net], msg);
+			esd_usb_rx_can_msg(dev->nets[msg->rx.net], msg);
 			break;
 
 		case CMD_CAN_TX:
-			if (msg->msg.txdone.net >= dev->net_count) {
+			if (msg->txdone.net >= dev->net_count) {
 				dev_err(dev->udev->dev.parent, "format error\n");
 				break;
 			}
 
-			esd_usb_tx_done_msg(dev->nets[msg->msg.txdone.net],
+			esd_usb_tx_done_msg(dev->nets[msg->txdone.net],
 					    msg);
 			break;
 		}
 
-		pos += msg->msg.hdr.len << 2;
+		pos += msg->hdr.len << 2;
 
 		if (pos > urb->actual_length) {
 			dev_err(dev->udev->dev.parent, "format error\n");
@@ -473,7 +475,7 @@ static void esd_usb_write_bulk_callback(struct urb *urb)
 	struct esd_tx_urb_context *context = urb->context;
 	struct esd_usb_net_priv *priv;
 	struct net_device *netdev;
-	size_t size = sizeof(struct esd_usb_msg);
+	size_t size = sizeof(union esd_usb_msg);
 
 	WARN_ON(!context);
 
@@ -529,20 +531,20 @@ static ssize_t nets_show(struct device *d,
 }
 static DEVICE_ATTR_RO(nets);
 
-static int esd_usb_send_msg(struct esd_usb *dev, struct esd_usb_msg *msg)
+static int esd_usb_send_msg(struct esd_usb *dev, union esd_usb_msg *msg)
 {
 	int actual_length;
 
 	return usb_bulk_msg(dev->udev,
 			    usb_sndbulkpipe(dev->udev, 2),
 			    msg,
-			    msg->msg.hdr.len << 2,
+			    msg->hdr.len << 2,
 			    &actual_length,
 			    1000);
 }
 
 static int esd_usb_wait_msg(struct esd_usb *dev,
-			    struct esd_usb_msg *msg)
+			    union esd_usb_msg *msg)
 {
 	int actual_length;
 
@@ -630,7 +632,7 @@ static int esd_usb_start(struct esd_usb_net_priv *priv)
 {
 	struct esd_usb *dev = priv->usb;
 	struct net_device *netdev = priv->netdev;
-	struct esd_usb_msg *msg;
+	union esd_usb_msg *msg;
 	int err, i;
 
 	msg = kmalloc(sizeof(*msg), GFP_KERNEL);
@@ -651,14 +653,14 @@ static int esd_usb_start(struct esd_usb_net_priv *priv)
 	 * the number of the starting bitmask (0..64) to the filter.option
 	 * field followed by only some bitmasks.
 	 */
-	msg->msg.hdr.cmd = CMD_IDADD;
-	msg->msg.hdr.len = 2 + ESD_MAX_ID_SEGMENT;
-	msg->msg.filter.net = priv->index;
-	msg->msg.filter.option = ESD_ID_ENABLE; /* start with segment 0 */
+	msg->hdr.cmd = CMD_IDADD;
+	msg->hdr.len = 2 + ESD_MAX_ID_SEGMENT;
+	msg->filter.net = priv->index;
+	msg->filter.option = ESD_ID_ENABLE; /* start with segment 0 */
 	for (i = 0; i < ESD_MAX_ID_SEGMENT; i++)
-		msg->msg.filter.mask[i] = cpu_to_le32(0xffffffff);
+		msg->filter.mask[i] = cpu_to_le32(0xffffffff);
 	/* enable 29bit extended IDs */
-	msg->msg.filter.mask[ESD_MAX_ID_SEGMENT] = cpu_to_le32(0x00000001);
+	msg->filter.mask[ESD_MAX_ID_SEGMENT] = cpu_to_le32(0x00000001);
 
 	err = esd_usb_send_msg(dev, msg);
 	if (err)
@@ -734,12 +736,12 @@ static netdev_tx_t esd_usb_start_xmit(struct sk_buff *skb,
 	struct esd_tx_urb_context *context = NULL;
 	struct net_device_stats *stats = &netdev->stats;
 	struct can_frame *cf = (struct can_frame *)skb->data;
-	struct esd_usb_msg *msg;
+	union esd_usb_msg *msg;
 	struct urb *urb;
 	u8 *buf;
 	int i, err;
 	int ret = NETDEV_TX_OK;
-	size_t size = sizeof(struct esd_usb_msg);
+	size_t size = sizeof(union esd_usb_msg);
 
 	if (can_dev_dropped_skb(netdev, skb))
 		return NETDEV_TX_OK;
@@ -761,24 +763,24 @@ static netdev_tx_t esd_usb_start_xmit(struct sk_buff *skb,
 		goto nobufmem;
 	}
 
-	msg = (struct esd_usb_msg *)buf;
+	msg = (union esd_usb_msg *)buf;
 
-	msg->msg.hdr.len = 3; /* minimal length */
-	msg->msg.hdr.cmd = CMD_CAN_TX;
-	msg->msg.tx.net = priv->index;
-	msg->msg.tx.dlc = can_get_cc_dlc(cf, priv->can.ctrlmode);
-	msg->msg.tx.id = cpu_to_le32(cf->can_id & CAN_ERR_MASK);
+	msg->hdr.len = 3; /* minimal length */
+	msg->hdr.cmd = CMD_CAN_TX;
+	msg->tx.net = priv->index;
+	msg->tx.dlc = can_get_cc_dlc(cf, priv->can.ctrlmode);
+	msg->tx.id = cpu_to_le32(cf->can_id & CAN_ERR_MASK);
 
 	if (cf->can_id & CAN_RTR_FLAG)
-		msg->msg.tx.dlc |= ESD_RTR;
+		msg->tx.dlc |= ESD_RTR;
 
 	if (cf->can_id & CAN_EFF_FLAG)
-		msg->msg.tx.id |= cpu_to_le32(ESD_EXTID);
+		msg->tx.id |= cpu_to_le32(ESD_EXTID);
 
 	for (i = 0; i < cf->len; i++)
-		msg->msg.tx.data[i] = cf->data[i];
+		msg->tx.data[i] = cf->data[i];
 
-	msg->msg.hdr.len += (cf->len + 3) >> 2;
+	msg->hdr.len += (cf->len + 3) >> 2;
 
 	for (i = 0; i < MAX_TX_URBS; i++) {
 		if (priv->tx_contexts[i].echo_index == MAX_TX_URBS) {
@@ -798,10 +800,10 @@ static netdev_tx_t esd_usb_start_xmit(struct sk_buff *skb,
 	context->echo_index = i;
 
 	/* hnd must not be 0 - MSB is stripped in txdone handling */
-	msg->msg.tx.hnd = 0x80000000 | i; /* returned in TX done message */
+	msg->tx.hnd = 0x80000000 | i; /* returned in TX done message */
 
 	usb_fill_bulk_urb(urb, dev->udev, usb_sndbulkpipe(dev->udev, 2), buf,
-			  msg->msg.hdr.len << 2,
+			  msg->hdr.len << 2,
 			  esd_usb_write_bulk_callback, context);
 
 	urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
@@ -855,7 +857,7 @@ nourbmem:
 static int esd_usb_close(struct net_device *netdev)
 {
 	struct esd_usb_net_priv *priv = netdev_priv(netdev);
-	struct esd_usb_msg *msg;
+	union esd_usb_msg *msg;
 	int i;
 
 	msg = kmalloc(sizeof(*msg), GFP_KERNEL);
@@ -863,21 +865,21 @@ static int esd_usb_close(struct net_device *netdev)
 		return -ENOMEM;
 
 	/* Disable all IDs (see esd_usb_start()) */
-	msg->msg.hdr.cmd = CMD_IDADD;
-	msg->msg.hdr.len = 2 + ESD_MAX_ID_SEGMENT;
-	msg->msg.filter.net = priv->index;
-	msg->msg.filter.option = ESD_ID_ENABLE; /* start with segment 0 */
+	msg->hdr.cmd = CMD_IDADD;
+	msg->hdr.len = 2 + ESD_MAX_ID_SEGMENT;
+	msg->filter.net = priv->index;
+	msg->filter.option = ESD_ID_ENABLE; /* start with segment 0 */
 	for (i = 0; i <= ESD_MAX_ID_SEGMENT; i++)
-		msg->msg.filter.mask[i] = 0;
+		msg->filter.mask[i] = 0;
 	if (esd_usb_send_msg(priv->usb, msg) < 0)
 		netdev_err(netdev, "sending idadd message failed\n");
 
 	/* set CAN controller to reset mode */
-	msg->msg.hdr.len = 2;
-	msg->msg.hdr.cmd = CMD_SETBAUD;
-	msg->msg.setbaud.net = priv->index;
-	msg->msg.setbaud.rsvd = 0;
-	msg->msg.setbaud.baud = cpu_to_le32(ESD_USB_NO_BAUDRATE);
+	msg->hdr.len = 2;
+	msg->hdr.cmd = CMD_SETBAUD;
+	msg->setbaud.net = priv->index;
+	msg->setbaud.rsvd = 0;
+	msg->setbaud.baud = cpu_to_le32(ESD_USB_NO_BAUDRATE);
 	if (esd_usb_send_msg(priv->usb, msg) < 0)
 		netdev_err(netdev, "sending setbaud message failed\n");
 
@@ -919,7 +921,7 @@ static int esd_usb2_set_bittiming(struct net_device *netdev)
 {
 	struct esd_usb_net_priv *priv = netdev_priv(netdev);
 	struct can_bittiming *bt = &priv->can.bittiming;
-	struct esd_usb_msg *msg;
+	union esd_usb_msg *msg;
 	int err;
 	u32 canbtr;
 	int sjw_shift;
@@ -950,11 +952,11 @@ static int esd_usb2_set_bittiming(struct net_device *netdev)
 	if (!msg)
 		return -ENOMEM;
 
-	msg->msg.hdr.len = 2;
-	msg->msg.hdr.cmd = CMD_SETBAUD;
-	msg->msg.setbaud.net = priv->index;
-	msg->msg.setbaud.rsvd = 0;
-	msg->msg.setbaud.baud = cpu_to_le32(canbtr);
+	msg->hdr.len = 2;
+	msg->hdr.cmd = CMD_SETBAUD;
+	msg->setbaud.net = priv->index;
+	msg->setbaud.rsvd = 0;
+	msg->setbaud.baud = cpu_to_le32(canbtr);
 
 	netdev_info(netdev, "setting BTR=%#x\n", canbtr);
 
@@ -1018,7 +1020,8 @@ static int esd_usb_probe_one_net(struct usb_interface *intf, int index)
 
 	priv->can.state = CAN_STATE_STOPPED;
 	priv->can.ctrlmode_supported = CAN_CTRLMODE_LISTENONLY |
-		CAN_CTRLMODE_CC_LEN8_DLC;
+		CAN_CTRLMODE_CC_LEN8_DLC |
+		CAN_CTRLMODE_BERR_REPORTING;
 
 	if (le16_to_cpu(dev->udev->descriptor.idProduct) ==
 	    USB_CANUSBM_PRODUCT_ID)
@@ -1065,7 +1068,7 @@ static int esd_usb_probe(struct usb_interface *intf,
 			 const struct usb_device_id *id)
 {
 	struct esd_usb *dev;
-	struct esd_usb_msg *msg;
+	union esd_usb_msg *msg;
 	int i, err;
 
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
@@ -1087,11 +1090,11 @@ static int esd_usb_probe(struct usb_interface *intf,
 	}
 
 	/* query number of CAN interfaces (nets) */
-	msg->msg.hdr.cmd = CMD_VERSION;
-	msg->msg.hdr.len = 2;
-	msg->msg.version.rsvd = 0;
-	msg->msg.version.flags = 0;
-	msg->msg.version.drv_version = 0;
+	msg->hdr.cmd = CMD_VERSION;
+	msg->hdr.len = 2;
+	msg->version.rsvd = 0;
+	msg->version.flags = 0;
+	msg->version.drv_version = 0;
 
 	err = esd_usb_send_msg(dev, msg);
 	if (err < 0) {
@@ -1105,8 +1108,8 @@ static int esd_usb_probe(struct usb_interface *intf,
 		goto free_msg;
 	}
 
-	dev->net_count = (int)msg->msg.version_reply.nets;
-	dev->version = le32_to_cpu(msg->msg.version_reply.version);
+	dev->net_count = (int)msg->version_reply.nets;
+	dev->version = le32_to_cpu(msg->version_reply.version);
 
 	if (device_create_file(&intf->dev, &dev_attr_firmware))
 		dev_err(&intf->dev,
diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c
index d4c5356d5884..7135ec851341 100644
--- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c
+++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c
@@ -31,63 +31,63 @@
 #include "kvaser_usb.h"
 
 /* Kvaser USB vendor id. */
-#define KVASER_VENDOR_ID			0x0bfd
+#define KVASER_VENDOR_ID 0x0bfd
 
 /* Kvaser Leaf USB devices product ids */
-#define USB_LEAF_DEVEL_PRODUCT_ID		10
-#define USB_LEAF_LITE_PRODUCT_ID		11
-#define USB_LEAF_PRO_PRODUCT_ID			12
-#define USB_LEAF_SPRO_PRODUCT_ID		14
-#define USB_LEAF_PRO_LS_PRODUCT_ID		15
-#define USB_LEAF_PRO_SWC_PRODUCT_ID		16
-#define USB_LEAF_PRO_LIN_PRODUCT_ID		17
-#define USB_LEAF_SPRO_LS_PRODUCT_ID		18
-#define USB_LEAF_SPRO_SWC_PRODUCT_ID		19
-#define USB_MEMO2_DEVEL_PRODUCT_ID		22
-#define USB_MEMO2_HSHS_PRODUCT_ID		23
-#define USB_UPRO_HSHS_PRODUCT_ID		24
-#define USB_LEAF_LITE_GI_PRODUCT_ID		25
-#define USB_LEAF_PRO_OBDII_PRODUCT_ID		26
-#define USB_MEMO2_HSLS_PRODUCT_ID		27
-#define USB_LEAF_LITE_CH_PRODUCT_ID		28
-#define USB_BLACKBIRD_SPRO_PRODUCT_ID		29
-#define USB_OEM_MERCURY_PRODUCT_ID		34
-#define USB_OEM_LEAF_PRODUCT_ID			35
-#define USB_CAN_R_PRODUCT_ID			39
-#define USB_LEAF_LITE_V2_PRODUCT_ID		288
-#define USB_MINI_PCIE_HS_PRODUCT_ID		289
-#define USB_LEAF_LIGHT_HS_V2_OEM_PRODUCT_ID	290
-#define USB_USBCAN_LIGHT_2HS_PRODUCT_ID		291
-#define USB_MINI_PCIE_2HS_PRODUCT_ID		292
-#define USB_USBCAN_R_V2_PRODUCT_ID		294
-#define USB_LEAF_LIGHT_R_V2_PRODUCT_ID		295
-#define USB_LEAF_LIGHT_HS_V2_OEM2_PRODUCT_ID	296
+#define USB_LEAF_DEVEL_PRODUCT_ID 0x000a
+#define USB_LEAF_LITE_PRODUCT_ID 0x000b
+#define USB_LEAF_PRO_PRODUCT_ID 0x000c
+#define USB_LEAF_SPRO_PRODUCT_ID 0x000e
+#define USB_LEAF_PRO_LS_PRODUCT_ID 0x000f
+#define USB_LEAF_PRO_SWC_PRODUCT_ID 0x0010
+#define USB_LEAF_PRO_LIN_PRODUCT_ID 0x0011
+#define USB_LEAF_SPRO_LS_PRODUCT_ID 0x0012
+#define USB_LEAF_SPRO_SWC_PRODUCT_ID 0x0013
+#define USB_MEMO2_DEVEL_PRODUCT_ID 0x0016
+#define USB_MEMO2_HSHS_PRODUCT_ID 0x0017
+#define USB_UPRO_HSHS_PRODUCT_ID 0x0018
+#define USB_LEAF_LITE_GI_PRODUCT_ID 0x0019
+#define USB_LEAF_PRO_OBDII_PRODUCT_ID 0x001a
+#define USB_MEMO2_HSLS_PRODUCT_ID 0x001b
+#define USB_LEAF_LITE_CH_PRODUCT_ID 0x001c
+#define USB_BLACKBIRD_SPRO_PRODUCT_ID 0x001d
+#define USB_OEM_MERCURY_PRODUCT_ID 0x0022
+#define USB_OEM_LEAF_PRODUCT_ID 0x0023
+#define USB_CAN_R_PRODUCT_ID 0x0027
+#define USB_LEAF_LITE_V2_PRODUCT_ID 0x0120
+#define USB_MINI_PCIE_HS_PRODUCT_ID 0x0121
+#define USB_LEAF_LIGHT_HS_V2_OEM_PRODUCT_ID 0x0122
+#define USB_USBCAN_LIGHT_2HS_PRODUCT_ID 0x0123
+#define USB_MINI_PCIE_2HS_PRODUCT_ID 0x0124
+#define USB_USBCAN_R_V2_PRODUCT_ID 0x0126
+#define USB_LEAF_LIGHT_R_V2_PRODUCT_ID 0x0127
+#define USB_LEAF_LIGHT_HS_V2_OEM2_PRODUCT_ID 0x0128
 
 /* Kvaser USBCan-II devices product ids */
-#define USB_USBCAN_REVB_PRODUCT_ID		2
-#define USB_VCI2_PRODUCT_ID			3
-#define USB_USBCAN2_PRODUCT_ID			4
-#define USB_MEMORATOR_PRODUCT_ID		5
+#define USB_USBCAN_REVB_PRODUCT_ID 0x0002
+#define USB_VCI2_PRODUCT_ID 0x0003
+#define USB_USBCAN2_PRODUCT_ID 0x0004
+#define USB_MEMORATOR_PRODUCT_ID 0x0005
 
 /* Kvaser Minihydra USB devices product ids */
-#define USB_BLACKBIRD_V2_PRODUCT_ID		258
-#define USB_MEMO_PRO_5HS_PRODUCT_ID		260
-#define USB_USBCAN_PRO_5HS_PRODUCT_ID		261
-#define USB_USBCAN_LIGHT_4HS_PRODUCT_ID		262
-#define USB_LEAF_PRO_HS_V2_PRODUCT_ID		263
-#define USB_USBCAN_PRO_2HS_V2_PRODUCT_ID	264
-#define USB_MEMO_2HS_PRODUCT_ID			265
-#define USB_MEMO_PRO_2HS_V2_PRODUCT_ID		266
-#define USB_HYBRID_2CANLIN_PRODUCT_ID		267
-#define USB_ATI_USBCAN_PRO_2HS_V2_PRODUCT_ID	268
-#define USB_ATI_MEMO_PRO_2HS_V2_PRODUCT_ID	269
-#define USB_HYBRID_PRO_2CANLIN_PRODUCT_ID	270
-#define USB_U100_PRODUCT_ID			273
-#define USB_U100P_PRODUCT_ID			274
-#define USB_U100S_PRODUCT_ID			275
-#define USB_USBCAN_PRO_4HS_PRODUCT_ID		276
-#define USB_HYBRID_CANLIN_PRODUCT_ID		277
-#define USB_HYBRID_PRO_CANLIN_PRODUCT_ID	278
+#define USB_BLACKBIRD_V2_PRODUCT_ID 0x0102
+#define USB_MEMO_PRO_5HS_PRODUCT_ID 0x0104
+#define USB_USBCAN_PRO_5HS_PRODUCT_ID 0x0105
+#define USB_USBCAN_LIGHT_4HS_PRODUCT_ID 0x0106
+#define USB_LEAF_PRO_HS_V2_PRODUCT_ID 0x0107
+#define USB_USBCAN_PRO_2HS_V2_PRODUCT_ID 0x0108
+#define USB_MEMO_2HS_PRODUCT_ID 0x0109
+#define USB_MEMO_PRO_2HS_V2_PRODUCT_ID 0x010a
+#define USB_HYBRID_2CANLIN_PRODUCT_ID 0x010b
+#define USB_ATI_USBCAN_PRO_2HS_V2_PRODUCT_ID 0x010c
+#define USB_ATI_MEMO_PRO_2HS_V2_PRODUCT_ID 0x010d
+#define USB_HYBRID_PRO_2CANLIN_PRODUCT_ID 0x010e
+#define USB_U100_PRODUCT_ID 0x0111
+#define USB_U100P_PRODUCT_ID 0x0112
+#define USB_U100S_PRODUCT_ID 0x0113
+#define USB_USBCAN_PRO_4HS_PRODUCT_ID 0x0114
+#define USB_HYBRID_CANLIN_PRODUCT_ID 0x0115
+#define USB_HYBRID_PRO_CANLIN_PRODUCT_ID 0x0116
 
 static const struct kvaser_usb_driver_info kvaser_usb_driver_info_hydra = {
 	.quirks = KVASER_USB_QUIRK_HAS_HARDWARE_TIMESTAMP,
diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig
index f6f3b43dfb06..3ed5391bb18d 100644
--- a/drivers/net/dsa/Kconfig
+++ b/drivers/net/dsa/Kconfig
@@ -38,10 +38,34 @@ config NET_DSA_MT7530
 	tristate "MediaTek MT7530 and MT7531 Ethernet switch support"
 	select NET_DSA_TAG_MTK
 	select MEDIATEK_GE_PHY
+	imply NET_DSA_MT7530_MDIO
+	imply NET_DSA_MT7530_MMIO
 	help
 	  This enables support for the MediaTek MT7530 and MT7531 Ethernet
 	  switch chips. Multi-chip module MT7530 in MT7621AT, MT7621DAT,
-	  MT7621ST and MT7623AI SoCs is supported.
+	  MT7621ST and MT7623AI SoCs, and built-in switch in MT7988 SoC are
+	  supported as well.
+
+config NET_DSA_MT7530_MDIO
+	tristate "MediaTek MT7530 MDIO interface driver"
+	depends on NET_DSA_MT7530
+	select PCS_MTK_LYNXI
+	help
+	  This enables support for the MediaTek MT7530 and MT7531 switch
+	  chips which are connected via MDIO, as well as multi-chip
+	  module MT7530 which can be found in the MT7621AT, MT7621DAT,
+	  MT7621ST and MT7623AI SoCs.
+
+config NET_DSA_MT7530_MMIO
+	tristate "MediaTek MT7530 MMIO interface driver"
+	depends on NET_DSA_MT7530
+	depends on HAS_IOMEM
+	help
+	  This enables support for the built-in Ethernet switch found
+	  in the MediaTek MT7988 SoC.
+	  The switch is a similar design as MT7531, but the switch registers
+	  are directly mapped into the SoCs register space rather than being
+	  accessible via MDIO.
 
 config NET_DSA_MV88E6060
 	tristate "Marvell 88E6060 ethernet switch chip support"
diff --git a/drivers/net/dsa/Makefile b/drivers/net/dsa/Makefile
index 16eb879e0cb4..cb9a97340e58 100644
--- a/drivers/net/dsa/Makefile
+++ b/drivers/net/dsa/Makefile
@@ -7,6 +7,8 @@ obj-$(CONFIG_FIXED_PHY)		+= dsa_loop_bdinfo.o
 endif
 obj-$(CONFIG_NET_DSA_LANTIQ_GSWIP) += lantiq_gswip.o
 obj-$(CONFIG_NET_DSA_MT7530)	+= mt7530.o
+obj-$(CONFIG_NET_DSA_MT7530_MDIO) += mt7530-mdio.o
+obj-$(CONFIG_NET_DSA_MT7530_MMIO) += mt7530-mmio.o
 obj-$(CONFIG_NET_DSA_MV88E6060) += mv88e6060.o
 obj-$(CONFIG_NET_DSA_RZN1_A5PSW) += rzn1_a5psw.o
 obj-$(CONFIG_NET_DSA_SMSC_LAN9303) += lan9303-core.o
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 59cdfc51ce06..3464ce5e7470 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -1209,6 +1209,50 @@ static void b53_force_port_config(struct b53_device *dev, int port,
 	b53_write8(dev, B53_CTRL_PAGE, off, reg);
 }
 
+static void b53_adjust_63xx_rgmii(struct dsa_switch *ds, int port,
+				  phy_interface_t interface)
+{
+	struct b53_device *dev = ds->priv;
+	u8 rgmii_ctrl = 0, off;
+
+	if (port == dev->imp_port)
+		off = B53_RGMII_CTRL_IMP;
+	else
+		off = B53_RGMII_CTRL_P(port);
+
+	b53_read8(dev, B53_CTRL_PAGE, off, &rgmii_ctrl);
+
+	switch (interface) {
+	case PHY_INTERFACE_MODE_RGMII_ID:
+		rgmii_ctrl |= (RGMII_CTRL_DLL_RXC | RGMII_CTRL_DLL_TXC);
+		break;
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+		rgmii_ctrl &= ~(RGMII_CTRL_DLL_TXC);
+		rgmii_ctrl |= RGMII_CTRL_DLL_RXC;
+		break;
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		rgmii_ctrl &= ~(RGMII_CTRL_DLL_RXC);
+		rgmii_ctrl |= RGMII_CTRL_DLL_TXC;
+		break;
+	case PHY_INTERFACE_MODE_RGMII:
+	default:
+		rgmii_ctrl &= ~(RGMII_CTRL_DLL_RXC | RGMII_CTRL_DLL_TXC);
+		break;
+	}
+
+	if (port != dev->imp_port) {
+		if (is63268(dev))
+			rgmii_ctrl |= RGMII_CTRL_MII_OVERRIDE;
+
+		rgmii_ctrl |= RGMII_CTRL_ENABLE_GMII;
+	}
+
+	b53_write8(dev, B53_CTRL_PAGE, off, rgmii_ctrl);
+
+	dev_dbg(ds->dev, "Configured port %d for %s\n", port,
+		phy_modes(interface));
+}
+
 static void b53_adjust_link(struct dsa_switch *ds, int port,
 			    struct phy_device *phydev)
 {
@@ -1235,6 +1279,9 @@ static void b53_adjust_link(struct dsa_switch *ds, int port,
 			      tx_pause, rx_pause);
 	b53_force_link(dev, port, phydev->link);
 
+	if (is63xx(dev) && port >= B53_63XX_RGMII0)
+		b53_adjust_63xx_rgmii(ds, port, phydev->interface);
+
 	if (is531x5(dev) && phy_interface_is_rgmii(phydev)) {
 		if (port == dev->imp_port)
 			off = B53_RGMII_CTRL_IMP;
@@ -1402,6 +1449,9 @@ void b53_phylink_mac_link_up(struct dsa_switch *ds, int port,
 {
 	struct b53_device *dev = ds->priv;
 
+	if (is63xx(dev) && port >= B53_63XX_RGMII0)
+		b53_adjust_63xx_rgmii(ds, port, interface);
+
 	if (mode == MLO_AN_PHY)
 		return;
 
@@ -2420,6 +2470,19 @@ static const struct b53_chip_data b53_switch_chips[] = {
 		.jumbo_size_reg = B53_JUMBO_MAX_SIZE_63XX,
 	},
 	{
+		.chip_id = BCM63268_DEVICE_ID,
+		.dev_name = "BCM63268",
+		.vlans = 4096,
+		.enabled_ports = 0, /* pdata must provide them */
+		.arl_bins = 4,
+		.arl_buckets = 1024,
+		.imp_port = 8,
+		.vta_regs = B53_VTA_REGS_63XX,
+		.duplex_reg = B53_DUPLEX_STAT_63XX,
+		.jumbo_pm_reg = B53_JUMBO_PORT_MASK_63XX,
+		.jumbo_size_reg = B53_JUMBO_MAX_SIZE_63XX,
+	},
+	{
 		.chip_id = BCM53010_DEVICE_ID,
 		.dev_name = "BCM53010",
 		.vlans = 4096,
@@ -2550,6 +2613,20 @@ static const struct b53_chip_data b53_switch_chips[] = {
 		.jumbo_pm_reg = B53_JUMBO_PORT_MASK,
 		.jumbo_size_reg = B53_JUMBO_MAX_SIZE,
 	},
+	{
+		.chip_id = BCM53134_DEVICE_ID,
+		.dev_name = "BCM53134",
+		.vlans = 4096,
+		.enabled_ports = 0x12f,
+		.imp_port = 8,
+		.cpu_port = B53_CPU_PORT,
+		.vta_regs = B53_VTA_REGS,
+		.arl_bins = 4,
+		.arl_buckets = 1024,
+		.duplex_reg = B53_DUPLEX_STAT_GE,
+		.jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+		.jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+	},
 };
 
 static int b53_switch_init(struct b53_device *dev)
@@ -2727,6 +2804,7 @@ int b53_switch_detect(struct b53_device *dev)
 		case BCM53012_DEVICE_ID:
 		case BCM53018_DEVICE_ID:
 		case BCM53019_DEVICE_ID:
+		case BCM53134_DEVICE_ID:
 			dev->chip_id = id32;
 			break;
 		default:
diff --git a/drivers/net/dsa/b53/b53_mdio.c b/drivers/net/dsa/b53/b53_mdio.c
index 6ddc03b58b28..8b422b298cd5 100644
--- a/drivers/net/dsa/b53/b53_mdio.c
+++ b/drivers/net/dsa/b53/b53_mdio.c
@@ -286,6 +286,7 @@ static const struct b53_io_ops b53_mdio_ops = {
 #define B53_BRCM_OUI_2	0x03625c00
 #define B53_BRCM_OUI_3	0x00406000
 #define B53_BRCM_OUI_4	0x01410c00
+#define B53_BRCM_OUI_5	0xae025000
 
 static int b53_mdio_probe(struct mdio_device *mdiodev)
 {
@@ -313,7 +314,8 @@ static int b53_mdio_probe(struct mdio_device *mdiodev)
 	if ((phy_id & 0xfffffc00) != B53_BRCM_OUI_1 &&
 	    (phy_id & 0xfffffc00) != B53_BRCM_OUI_2 &&
 	    (phy_id & 0xfffffc00) != B53_BRCM_OUI_3 &&
-	    (phy_id & 0xfffffc00) != B53_BRCM_OUI_4) {
+	    (phy_id & 0xfffffc00) != B53_BRCM_OUI_4 &&
+	    (phy_id & 0xfffffc00) != B53_BRCM_OUI_5) {
 		dev_err(&mdiodev->dev, "Unsupported device: 0x%08x\n", phy_id);
 		return -ENODEV;
 	}
@@ -375,6 +377,7 @@ static const struct of_device_id b53_of_match[] = {
 	{ .compatible = "brcm,bcm53115" },
 	{ .compatible = "brcm,bcm53125" },
 	{ .compatible = "brcm,bcm53128" },
+	{ .compatible = "brcm,bcm53134" },
 	{ .compatible = "brcm,bcm5365" },
 	{ .compatible = "brcm,bcm5389" },
 	{ .compatible = "brcm,bcm5395" },
diff --git a/drivers/net/dsa/b53/b53_mmap.c b/drivers/net/dsa/b53/b53_mmap.c
index d9434ed9450d..5db1ed26f03a 100644
--- a/drivers/net/dsa/b53/b53_mmap.c
+++ b/drivers/net/dsa/b53/b53_mmap.c
@@ -262,7 +262,7 @@ static int b53_mmap_probe_of(struct platform_device *pdev,
 		return -ENOMEM;
 
 	pdata->regs = mem;
-	pdata->chip_id = BCM63XX_DEVICE_ID;
+	pdata->chip_id = (u32)(unsigned long)device_get_match_data(dev);
 	pdata->big_endian = of_property_read_bool(np, "big-endian");
 
 	of_ports = of_get_child_by_name(np, "ports");
@@ -344,11 +344,28 @@ static void b53_mmap_shutdown(struct platform_device *pdev)
 }
 
 static const struct of_device_id b53_mmap_of_table[] = {
-	{ .compatible = "brcm,bcm3384-switch" },
-	{ .compatible = "brcm,bcm6328-switch" },
-	{ .compatible = "brcm,bcm6368-switch" },
-	{ .compatible = "brcm,bcm63xx-switch" },
-	{ /* sentinel */ },
+	{
+		.compatible = "brcm,bcm3384-switch",
+		.data = (void *)BCM63XX_DEVICE_ID,
+	}, {
+		.compatible = "brcm,bcm6318-switch",
+		.data = (void *)BCM63268_DEVICE_ID,
+	}, {
+		.compatible = "brcm,bcm6328-switch",
+		.data = (void *)BCM63XX_DEVICE_ID,
+	}, {
+		.compatible = "brcm,bcm6362-switch",
+		.data = (void *)BCM63XX_DEVICE_ID,
+	}, {
+		.compatible = "brcm,bcm6368-switch",
+		.data = (void *)BCM63XX_DEVICE_ID,
+	}, {
+		.compatible = "brcm,bcm63268-switch",
+		.data = (void *)BCM63268_DEVICE_ID,
+	}, {
+		.compatible = "brcm,bcm63xx-switch",
+		.data = (void *)BCM63XX_DEVICE_ID,
+	}, { /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, b53_mmap_of_table);
 
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
index 795cbffd5c2b..fdcfd5081c28 100644
--- a/drivers/net/dsa/b53/b53_priv.h
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -70,6 +70,7 @@ enum {
 	BCM53125_DEVICE_ID = 0x53125,
 	BCM53128_DEVICE_ID = 0x53128,
 	BCM63XX_DEVICE_ID = 0x6300,
+	BCM63268_DEVICE_ID = 0x63268,
 	BCM53010_DEVICE_ID = 0x53010,
 	BCM53011_DEVICE_ID = 0x53011,
 	BCM53012_DEVICE_ID = 0x53012,
@@ -79,6 +80,7 @@ enum {
 	BCM583XX_DEVICE_ID = 0x58300,
 	BCM7445_DEVICE_ID = 0x7445,
 	BCM7278_DEVICE_ID = 0x7278,
+	BCM53134_DEVICE_ID = 0x5075,
 };
 
 struct b53_pcs {
@@ -186,12 +188,19 @@ static inline int is531x5(struct b53_device *dev)
 {
 	return dev->chip_id == BCM53115_DEVICE_ID ||
 		dev->chip_id == BCM53125_DEVICE_ID ||
-		dev->chip_id == BCM53128_DEVICE_ID;
+		dev->chip_id == BCM53128_DEVICE_ID ||
+		dev->chip_id == BCM53134_DEVICE_ID;
 }
 
 static inline int is63xx(struct b53_device *dev)
 {
-	return dev->chip_id == BCM63XX_DEVICE_ID;
+	return dev->chip_id == BCM63XX_DEVICE_ID ||
+		dev->chip_id == BCM63268_DEVICE_ID;
+}
+
+static inline int is63268(struct b53_device *dev)
+{
+	return dev->chip_id == BCM63268_DEVICE_ID;
 }
 
 static inline int is5301x(struct b53_device *dev)
@@ -208,9 +217,11 @@ static inline int is58xx(struct b53_device *dev)
 	return dev->chip_id == BCM58XX_DEVICE_ID ||
 		dev->chip_id == BCM583XX_DEVICE_ID ||
 		dev->chip_id == BCM7445_DEVICE_ID ||
-		dev->chip_id == BCM7278_DEVICE_ID;
+		dev->chip_id == BCM7278_DEVICE_ID ||
+		dev->chip_id == BCM53134_DEVICE_ID;
 }
 
+#define B53_63XX_RGMII0	4
 #define B53_CPU_PORT_25	5
 #define B53_CPU_PORT	8
 
diff --git a/drivers/net/dsa/b53/b53_regs.h b/drivers/net/dsa/b53/b53_regs.h
index b2c539a42154..bfbcb66bef66 100644
--- a/drivers/net/dsa/b53/b53_regs.h
+++ b/drivers/net/dsa/b53/b53_regs.h
@@ -138,6 +138,7 @@
 
 #define B53_RGMII_CTRL_IMP		0x60
 #define   RGMII_CTRL_ENABLE_GMII	BIT(7)
+#define   RGMII_CTRL_MII_OVERRIDE	BIT(6)
 #define   RGMII_CTRL_TIMING_SEL		BIT(2)
 #define   RGMII_CTRL_DLL_RXC		BIT(1)
 #define   RGMII_CTRL_DLL_TXC		BIT(0)
diff --git a/drivers/net/dsa/hirschmann/hellcreek_ptp.c b/drivers/net/dsa/hirschmann/hellcreek_ptp.c
index b28baab6d56a..3e44ccb7db84 100644
--- a/drivers/net/dsa/hirschmann/hellcreek_ptp.c
+++ b/drivers/net/dsa/hirschmann/hellcreek_ptp.c
@@ -297,7 +297,8 @@ static enum led_brightness hellcreek_led_is_gm_get(struct led_classdev *ldev)
 static int hellcreek_led_setup(struct hellcreek *hellcreek)
 {
 	struct device_node *leds, *led = NULL;
-	const char *label, *state;
+	enum led_default_state state;
+	const char *label;
 	int ret = -EINVAL;
 
 	of_node_get(hellcreek->dev->of_node);
@@ -318,16 +319,17 @@ static int hellcreek_led_setup(struct hellcreek *hellcreek)
 	ret = of_property_read_string(led, "label", &label);
 	hellcreek->led_sync_good.name = ret ? "sync_good" : label;
 
-	ret = of_property_read_string(led, "default-state", &state);
-	if (!ret) {
-		if (!strcmp(state, "on"))
-			hellcreek->led_sync_good.brightness = 1;
-		else if (!strcmp(state, "off"))
-			hellcreek->led_sync_good.brightness = 0;
-		else if (!strcmp(state, "keep"))
-			hellcreek->led_sync_good.brightness =
-				hellcreek_get_brightness(hellcreek,
-							 STATUS_OUT_SYNC_GOOD);
+	state = led_init_default_state_get(of_fwnode_handle(led));
+	switch (state) {
+	case LEDS_DEFSTATE_ON:
+		hellcreek->led_sync_good.brightness = 1;
+		break;
+	case LEDS_DEFSTATE_KEEP:
+		hellcreek->led_sync_good.brightness =
+			hellcreek_get_brightness(hellcreek, STATUS_OUT_SYNC_GOOD);
+		break;
+	default:
+		hellcreek->led_sync_good.brightness = 0;
 	}
 
 	hellcreek->led_sync_good.max_brightness = 1;
@@ -344,16 +346,17 @@ static int hellcreek_led_setup(struct hellcreek *hellcreek)
 	ret = of_property_read_string(led, "label", &label);
 	hellcreek->led_is_gm.name = ret ? "is_gm" : label;
 
-	ret = of_property_read_string(led, "default-state", &state);
-	if (!ret) {
-		if (!strcmp(state, "on"))
-			hellcreek->led_is_gm.brightness = 1;
-		else if (!strcmp(state, "off"))
-			hellcreek->led_is_gm.brightness = 0;
-		else if (!strcmp(state, "keep"))
-			hellcreek->led_is_gm.brightness =
-				hellcreek_get_brightness(hellcreek,
-							 STATUS_OUT_IS_GM);
+	state = led_init_default_state_get(of_fwnode_handle(led));
+	switch (state) {
+	case LEDS_DEFSTATE_ON:
+		hellcreek->led_is_gm.brightness = 1;
+		break;
+	case LEDS_DEFSTATE_KEEP:
+		hellcreek->led_is_gm.brightness =
+			hellcreek_get_brightness(hellcreek, STATUS_OUT_IS_GM);
+		break;
+	default:
+		hellcreek->led_is_gm.brightness = 0;
 	}
 
 	hellcreek->led_is_gm.max_brightness = 1;
diff --git a/drivers/net/dsa/lan9303_i2c.c b/drivers/net/dsa/lan9303_i2c.c
index 1cb41c36bd47..e8844820c3a9 100644
--- a/drivers/net/dsa/lan9303_i2c.c
+++ b/drivers/net/dsa/lan9303_i2c.c
@@ -103,7 +103,7 @@ MODULE_DEVICE_TABLE(of, lan9303_i2c_of_match);
 static struct i2c_driver lan9303_i2c_driver = {
 	.driver = {
 		.name = "LAN9303_I2C",
-		.of_match_table = of_match_ptr(lan9303_i2c_of_match),
+		.of_match_table = lan9303_i2c_of_match,
 	},
 	.probe_new = lan9303_i2c_probe,
 	.remove = lan9303_i2c_remove,
diff --git a/drivers/net/dsa/lan9303_mdio.c b/drivers/net/dsa/lan9303_mdio.c
index 4f33369a2de5..d8ab2b77d201 100644
--- a/drivers/net/dsa/lan9303_mdio.c
+++ b/drivers/net/dsa/lan9303_mdio.c
@@ -164,7 +164,7 @@ MODULE_DEVICE_TABLE(of, lan9303_mdio_of_match);
 static struct mdio_driver lan9303_mdio_driver = {
 	.mdiodrv.driver = {
 		.name = "LAN9303_MDIO",
-		.of_match_table = of_match_ptr(lan9303_mdio_of_match),
+		.of_match_table = lan9303_mdio_of_match,
 	},
 	.probe  = lan9303_mdio_probe,
 	.remove = lan9303_mdio_remove,
diff --git a/drivers/net/dsa/lantiq_gswip.c b/drivers/net/dsa/lantiq_gswip.c
index 05ecaa007ab1..3c76a1a14aee 100644
--- a/drivers/net/dsa/lantiq_gswip.c
+++ b/drivers/net/dsa/lantiq_gswip.c
@@ -1885,7 +1885,7 @@ static const struct xway_gphy_match_data xrx300_gphy_data = {
 	.ge_firmware_name = "lantiq/xrx300_phy11g_a21.bin",
 };
 
-static const struct of_device_id xway_gphy_match[] = {
+static const struct of_device_id xway_gphy_match[] __maybe_unused = {
 	{ .compatible = "lantiq,xrx200-gphy-fw", .data = NULL },
 	{ .compatible = "lantiq,xrx200a1x-gphy-fw", .data = &xrx200a1x_gphy_data },
 	{ .compatible = "lantiq,xrx200a2x-gphy-fw", .data = &xrx200a2x_gphy_data },
diff --git a/drivers/net/dsa/microchip/ksz8.h b/drivers/net/dsa/microchip/ksz8.h
index ea05abfbd51d..e68465fdf6b9 100644
--- a/drivers/net/dsa/microchip/ksz8.h
+++ b/drivers/net/dsa/microchip/ksz8.h
@@ -21,10 +21,6 @@ int ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val);
 int ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val);
 int ksz8_r_dyn_mac_table(struct ksz_device *dev, u16 addr, u8 *mac_addr,
 			 u8 *fid, u8 *src_port, u8 *timestamp, u16 *entries);
-int ksz8_r_sta_mac_table(struct ksz_device *dev, u16 addr,
-			 struct alu_struct *alu);
-void ksz8_w_sta_mac_table(struct ksz_device *dev, u16 addr,
-			  struct alu_struct *alu);
 void ksz8_r_mib_cnt(struct ksz_device *dev, int port, u16 addr, u64 *cnt);
 void ksz8_r_mib_pkt(struct ksz_device *dev, int port, u16 addr,
 		    u64 *dropped, u64 *cnt);
@@ -32,6 +28,10 @@ void ksz8_freeze_mib(struct ksz_device *dev, int port, bool freeze);
 void ksz8_port_init_cnt(struct ksz_device *dev, int port);
 int ksz8_fdb_dump(struct ksz_device *dev, int port,
 		  dsa_fdb_dump_cb_t *cb, void *data);
+int ksz8_fdb_add(struct ksz_device *dev, int port, const unsigned char *addr,
+		 u16 vid, struct dsa_db db);
+int ksz8_fdb_del(struct ksz_device *dev, int port, const unsigned char *addr,
+		 u16 vid, struct dsa_db db);
 int ksz8_mdb_add(struct ksz_device *dev, int port,
 		 const struct switchdev_obj_port_mdb *mdb, struct dsa_db db);
 int ksz8_mdb_del(struct ksz_device *dev, int port,
diff --git a/drivers/net/dsa/microchip/ksz8795.c b/drivers/net/dsa/microchip/ksz8795.c
index 3fffd5da8d3b..f56fca1b1a22 100644
--- a/drivers/net/dsa/microchip/ksz8795.c
+++ b/drivers/net/dsa/microchip/ksz8795.c
@@ -96,7 +96,7 @@ static int ksz8795_change_mtu(struct ksz_device *dev, int frame_size)
 
 	if (frame_size > KSZ8_LEGAL_PACKET_SIZE)
 		ctrl2 |= SW_LEGAL_PACKET_DISABLE;
-	else if (frame_size > KSZ8863_NORMAL_PACKET_SIZE)
+	if (frame_size > KSZ8863_NORMAL_PACKET_SIZE)
 		ctrl1 |= SW_HUGE_PACKET;
 
 	ret = ksz_rmw8(dev, REG_SW_CTRL_1, SW_HUGE_PACKET, ctrl1);
@@ -336,34 +336,48 @@ void ksz8_port_init_cnt(struct ksz_device *dev, int port)
 	}
 }
 
-static void ksz8_r_table(struct ksz_device *dev, int table, u16 addr, u64 *data)
+static int ksz8_r_table(struct ksz_device *dev, int table, u16 addr, u64 *data)
 {
 	const u16 *regs;
 	u16 ctrl_addr;
+	int ret;
 
 	regs = dev->info->regs;
 
 	ctrl_addr = IND_ACC_TABLE(table | TABLE_READ) | addr;
 
 	mutex_lock(&dev->alu_mutex);
-	ksz_write16(dev, regs[REG_IND_CTRL_0], ctrl_addr);
-	ksz_read64(dev, regs[REG_IND_DATA_HI], data);
+	ret = ksz_write16(dev, regs[REG_IND_CTRL_0], ctrl_addr);
+	if (ret)
+		goto unlock_alu;
+
+	ret = ksz_read64(dev, regs[REG_IND_DATA_HI], data);
+unlock_alu:
 	mutex_unlock(&dev->alu_mutex);
+
+	return ret;
 }
 
-static void ksz8_w_table(struct ksz_device *dev, int table, u16 addr, u64 data)
+static int ksz8_w_table(struct ksz_device *dev, int table, u16 addr, u64 data)
 {
 	const u16 *regs;
 	u16 ctrl_addr;
+	int ret;
 
 	regs = dev->info->regs;
 
 	ctrl_addr = IND_ACC_TABLE(table) | addr;
 
 	mutex_lock(&dev->alu_mutex);
-	ksz_write64(dev, regs[REG_IND_DATA_HI], data);
-	ksz_write16(dev, regs[REG_IND_CTRL_0], ctrl_addr);
+	ret = ksz_write64(dev, regs[REG_IND_DATA_HI], data);
+	if (ret)
+		goto unlock_alu;
+
+	ret = ksz_write16(dev, regs[REG_IND_CTRL_0], ctrl_addr);
+unlock_alu:
 	mutex_unlock(&dev->alu_mutex);
+
+	return ret;
 }
 
 static int ksz8_valid_dyn_entry(struct ksz_device *dev, u8 *data)
@@ -457,46 +471,54 @@ int ksz8_r_dyn_mac_table(struct ksz_device *dev, u16 addr, u8 *mac_addr,
 	return rc;
 }
 
-int ksz8_r_sta_mac_table(struct ksz_device *dev, u16 addr,
-			 struct alu_struct *alu)
+static int ksz8_r_sta_mac_table(struct ksz_device *dev, u16 addr,
+				struct alu_struct *alu, bool *valid)
 {
 	u32 data_hi, data_lo;
 	const u8 *shifts;
 	const u32 *masks;
 	u64 data;
+	int ret;
 
 	shifts = dev->info->shifts;
 	masks = dev->info->masks;
 
-	ksz8_r_table(dev, TABLE_STATIC_MAC, addr, &data);
+	ret = ksz8_r_table(dev, TABLE_STATIC_MAC, addr, &data);
+	if (ret)
+		return ret;
+
 	data_hi = data >> 32;
 	data_lo = (u32)data;
-	if (data_hi & (masks[STATIC_MAC_TABLE_VALID] |
-			masks[STATIC_MAC_TABLE_OVERRIDE])) {
-		alu->mac[5] = (u8)data_lo;
-		alu->mac[4] = (u8)(data_lo >> 8);
-		alu->mac[3] = (u8)(data_lo >> 16);
-		alu->mac[2] = (u8)(data_lo >> 24);
-		alu->mac[1] = (u8)data_hi;
-		alu->mac[0] = (u8)(data_hi >> 8);
-		alu->port_forward =
-			(data_hi & masks[STATIC_MAC_TABLE_FWD_PORTS]) >>
-				shifts[STATIC_MAC_FWD_PORTS];
-		alu->is_override =
-			(data_hi & masks[STATIC_MAC_TABLE_OVERRIDE]) ? 1 : 0;
-		data_hi >>= 1;
-		alu->is_static = true;
-		alu->is_use_fid =
-			(data_hi & masks[STATIC_MAC_TABLE_USE_FID]) ? 1 : 0;
-		alu->fid = (data_hi & masks[STATIC_MAC_TABLE_FID]) >>
-				shifts[STATIC_MAC_FID];
+
+	if (!(data_hi & (masks[STATIC_MAC_TABLE_VALID] |
+			 masks[STATIC_MAC_TABLE_OVERRIDE]))) {
+		*valid = false;
 		return 0;
 	}
-	return -ENXIO;
+
+	alu->mac[5] = (u8)data_lo;
+	alu->mac[4] = (u8)(data_lo >> 8);
+	alu->mac[3] = (u8)(data_lo >> 16);
+	alu->mac[2] = (u8)(data_lo >> 24);
+	alu->mac[1] = (u8)data_hi;
+	alu->mac[0] = (u8)(data_hi >> 8);
+	alu->port_forward =
+		(data_hi & masks[STATIC_MAC_TABLE_FWD_PORTS]) >>
+			shifts[STATIC_MAC_FWD_PORTS];
+	alu->is_override = (data_hi & masks[STATIC_MAC_TABLE_OVERRIDE]) ? 1 : 0;
+	data_hi >>= 1;
+	alu->is_static = true;
+	alu->is_use_fid = (data_hi & masks[STATIC_MAC_TABLE_USE_FID]) ? 1 : 0;
+	alu->fid = (data_hi & masks[STATIC_MAC_TABLE_FID]) >>
+		shifts[STATIC_MAC_FID];
+
+	*valid = true;
+
+	return 0;
 }
 
-void ksz8_w_sta_mac_table(struct ksz_device *dev, u16 addr,
-			  struct alu_struct *alu)
+static int ksz8_w_sta_mac_table(struct ksz_device *dev, u16 addr,
+				struct alu_struct *alu)
 {
 	u32 data_hi, data_lo;
 	const u8 *shifts;
@@ -524,7 +546,8 @@ void ksz8_w_sta_mac_table(struct ksz_device *dev, u16 addr,
 		data_hi &= ~masks[STATIC_MAC_TABLE_OVERRIDE];
 
 	data = (u64)data_hi << 32 | data_lo;
-	ksz8_w_table(dev, TABLE_STATIC_MAC, addr, data);
+
+	return ksz8_w_table(dev, TABLE_STATIC_MAC, addr, data);
 }
 
 static void ksz8_from_vlan(struct ksz_device *dev, u32 vlan, u8 *fid,
@@ -977,24 +1000,29 @@ int ksz8_fdb_dump(struct ksz_device *dev, int port,
 	return ret;
 }
 
-int ksz8_mdb_add(struct ksz_device *dev, int port,
-		 const struct switchdev_obj_port_mdb *mdb, struct dsa_db db)
+static int ksz8_add_sta_mac(struct ksz_device *dev, int port,
+			    const unsigned char *addr, u16 vid)
 {
 	struct alu_struct alu;
-	int index;
+	int index, ret;
 	int empty = 0;
 
 	alu.port_forward = 0;
 	for (index = 0; index < dev->info->num_statics; index++) {
-		if (!ksz8_r_sta_mac_table(dev, index, &alu)) {
-			/* Found one already in static MAC table. */
-			if (!memcmp(alu.mac, mdb->addr, ETH_ALEN) &&
-			    alu.fid == mdb->vid)
-				break;
-		/* Remember the first empty entry. */
-		} else if (!empty) {
-			empty = index + 1;
+		bool valid;
+
+		ret = ksz8_r_sta_mac_table(dev, index, &alu, &valid);
+		if (ret)
+			return ret;
+		if (!valid) {
+			/* Remember the first empty entry. */
+			if (!empty)
+				empty = index + 1;
+			continue;
 		}
+
+		if (!memcmp(alu.mac, addr, ETH_ALEN) && alu.fid == vid)
+			break;
 	}
 
 	/* no available entry */
@@ -1005,48 +1033,73 @@ int ksz8_mdb_add(struct ksz_device *dev, int port,
 	if (index == dev->info->num_statics) {
 		index = empty - 1;
 		memset(&alu, 0, sizeof(alu));
-		memcpy(alu.mac, mdb->addr, ETH_ALEN);
+		memcpy(alu.mac, addr, ETH_ALEN);
 		alu.is_static = true;
 	}
 	alu.port_forward |= BIT(port);
-	if (mdb->vid) {
+	if (vid) {
 		alu.is_use_fid = true;
 
 		/* Need a way to map VID to FID. */
-		alu.fid = mdb->vid;
+		alu.fid = vid;
 	}
-	ksz8_w_sta_mac_table(dev, index, &alu);
 
-	return 0;
+	return ksz8_w_sta_mac_table(dev, index, &alu);
 }
 
-int ksz8_mdb_del(struct ksz_device *dev, int port,
-		 const struct switchdev_obj_port_mdb *mdb, struct dsa_db db)
+static int ksz8_del_sta_mac(struct ksz_device *dev, int port,
+			    const unsigned char *addr, u16 vid)
 {
 	struct alu_struct alu;
-	int index;
+	int index, ret;
 
 	for (index = 0; index < dev->info->num_statics; index++) {
-		if (!ksz8_r_sta_mac_table(dev, index, &alu)) {
-			/* Found one already in static MAC table. */
-			if (!memcmp(alu.mac, mdb->addr, ETH_ALEN) &&
-			    alu.fid == mdb->vid)
-				break;
-		}
+		bool valid;
+
+		ret = ksz8_r_sta_mac_table(dev, index, &alu, &valid);
+		if (ret)
+			return ret;
+		if (!valid)
+			continue;
+
+		if (!memcmp(alu.mac, addr, ETH_ALEN) && alu.fid == vid)
+			break;
 	}
 
 	/* no available entry */
 	if (index == dev->info->num_statics)
-		goto exit;
+		return 0;
 
 	/* clear port */
 	alu.port_forward &= ~BIT(port);
 	if (!alu.port_forward)
 		alu.is_static = false;
-	ksz8_w_sta_mac_table(dev, index, &alu);
 
-exit:
-	return 0;
+	return ksz8_w_sta_mac_table(dev, index, &alu);
+}
+
+int ksz8_mdb_add(struct ksz_device *dev, int port,
+		 const struct switchdev_obj_port_mdb *mdb, struct dsa_db db)
+{
+	return ksz8_add_sta_mac(dev, port, mdb->addr, mdb->vid);
+}
+
+int ksz8_mdb_del(struct ksz_device *dev, int port,
+		 const struct switchdev_obj_port_mdb *mdb, struct dsa_db db)
+{
+	return ksz8_del_sta_mac(dev, port, mdb->addr, mdb->vid);
+}
+
+int ksz8_fdb_add(struct ksz_device *dev, int port, const unsigned char *addr,
+		 u16 vid, struct dsa_db db)
+{
+	return ksz8_add_sta_mac(dev, port, addr, vid);
+}
+
+int ksz8_fdb_del(struct ksz_device *dev, int port, const unsigned char *addr,
+		 u16 vid, struct dsa_db db)
+{
+	return ksz8_del_sta_mac(dev, port, addr, vid);
 }
 
 int ksz8_port_vlan_filtering(struct ksz_device *dev, int port, bool flag,
@@ -1346,9 +1399,7 @@ int ksz8_enable_stp_addr(struct ksz_device *dev)
 	alu.is_override = true;
 	alu.port_forward = dev->info->cpu_ports;
 
-	ksz8_w_sta_mac_table(dev, 0, &alu);
-
-	return 0;
+	return ksz8_w_sta_mac_table(dev, 0, &alu);
 }
 
 int ksz8_setup(struct dsa_switch *ds)
diff --git a/drivers/net/dsa/microchip/ksz9477_i2c.c b/drivers/net/dsa/microchip/ksz9477_i2c.c
index e315f669ec06..97a317263a2f 100644
--- a/drivers/net/dsa/microchip/ksz9477_i2c.c
+++ b/drivers/net/dsa/microchip/ksz9477_i2c.c
@@ -117,7 +117,7 @@ MODULE_DEVICE_TABLE(of, ksz9477_dt_ids);
 static struct i2c_driver ksz9477_i2c_driver = {
 	.driver = {
 		.name	= "ksz9477-switch",
-		.of_match_table = of_match_ptr(ksz9477_dt_ids),
+		.of_match_table = ksz9477_dt_ids,
 	},
 	.probe_new = ksz9477_i2c_probe,
 	.remove	= ksz9477_i2c_remove,
diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c
index 74c56d05ab0b..a4428be5f483 100644
--- a/drivers/net/dsa/microchip/ksz_common.c
+++ b/drivers/net/dsa/microchip/ksz_common.c
@@ -32,10 +32,6 @@
 #include "ksz9477.h"
 #include "lan937x.h"
 
-#define KSZ_CBS_ENABLE ((MTI_SCHEDULE_STRICT_PRIO << MTI_SCHEDULE_MODE_S) | \
-			(MTI_SHAPING_SRP << MTI_SHAPING_S))
-#define KSZ_CBS_DISABLE ((MTI_SCHEDULE_WRR << MTI_SCHEDULE_MODE_S) |\
-			 (MTI_SHAPING_OFF << MTI_SHAPING_S))
 #define MIB_COUNTER_NUM 0x20
 
 struct ksz_stats_raw {
@@ -204,6 +200,8 @@ static const struct ksz_dev_ops ksz8_dev_ops = {
 	.freeze_mib = ksz8_freeze_mib,
 	.port_init_cnt = ksz8_port_init_cnt,
 	.fdb_dump = ksz8_fdb_dump,
+	.fdb_add = ksz8_fdb_add,
+	.fdb_del = ksz8_fdb_del,
 	.mdb_add = ksz8_mdb_add,
 	.mdb_del = ksz8_mdb_del,
 	.vlan_filtering = ksz8_port_vlan_filtering,
@@ -1089,6 +1087,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.port_nirqs = 3,
 		.num_tx_queues = 4,
 		.tc_cbs_supported = true,
+		.tc_ets_supported = true,
 		.ops = &ksz9477_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -1228,6 +1227,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.port_nirqs = 4,
 		.num_tx_queues = 4,
 		.tc_cbs_supported = true,
+		.tc_ets_supported = true,
 		.ops = &ksz9477_dev_ops,
 		.phy_errata_9477 = true,
 		.mib_names = ksz9477_mib_names,
@@ -1352,6 +1352,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.port_nirqs = 3,
 		.num_tx_queues = 4,
 		.tc_cbs_supported = true,
+		.tc_ets_supported = true,
 		.ops = &ksz9477_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -1379,6 +1380,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.port_nirqs = 3,
 		.num_tx_queues = 4,
 		.tc_cbs_supported = true,
+		.tc_ets_supported = true,
 		.ops = &ksz9477_dev_ops,
 		.phy_errata_9477 = true,
 		.mib_names = ksz9477_mib_names,
@@ -1411,6 +1413,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.port_nirqs = 6,
 		.num_tx_queues = 8,
 		.tc_cbs_supported = true,
+		.tc_ets_supported = true,
 		.ops = &lan937x_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -1437,6 +1440,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.port_nirqs = 6,
 		.num_tx_queues = 8,
 		.tc_cbs_supported = true,
+		.tc_ets_supported = true,
 		.ops = &lan937x_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -1463,6 +1467,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.port_nirqs = 6,
 		.num_tx_queues = 8,
 		.tc_cbs_supported = true,
+		.tc_ets_supported = true,
 		.ops = &lan937x_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -1493,6 +1498,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.port_nirqs = 6,
 		.num_tx_queues = 8,
 		.tc_cbs_supported = true,
+		.tc_ets_supported = true,
 		.ops = &lan937x_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -1523,6 +1529,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.port_nirqs = 6,
 		.num_tx_queues = 8,
 		.tc_cbs_supported = true,
+		.tc_ets_supported = true,
 		.ops = &lan937x_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -3091,6 +3098,14 @@ static int cinc_cal(s32 idle_slope, s32 send_slope, u32 *bw)
 	return 0;
 }
 
+static int ksz_setup_tc_mode(struct ksz_device *dev, int port, u8 scheduler,
+			     u8 shaper)
+{
+	return ksz_pwrite8(dev, port, REG_PORT_MTI_QUEUE_CTRL_0,
+			   FIELD_PREP(MTI_SCHEDULE_MODE_M, scheduler) |
+			   FIELD_PREP(MTI_SHAPING_M, shaper));
+}
+
 static int ksz_setup_tc_cbs(struct dsa_switch *ds, int port,
 			    struct tc_cbs_qopt_offload *qopt)
 {
@@ -3110,8 +3125,8 @@ static int ksz_setup_tc_cbs(struct dsa_switch *ds, int port,
 		return ret;
 
 	if (!qopt->enable)
-		return ksz_pwrite8(dev, port, REG_PORT_MTI_QUEUE_CTRL_0,
-				   KSZ_CBS_DISABLE);
+		return ksz_setup_tc_mode(dev, port, MTI_SCHEDULE_WRR,
+					 MTI_SHAPING_OFF);
 
 	/* High Credit */
 	ret = ksz_pwrite16(dev, port, REG_PORT_MTI_HI_WATER_MARK,
@@ -3136,8 +3151,215 @@ static int ksz_setup_tc_cbs(struct dsa_switch *ds, int port,
 			return ret;
 	}
 
-	return ksz_pwrite8(dev, port, REG_PORT_MTI_QUEUE_CTRL_0,
-			   KSZ_CBS_ENABLE);
+	return ksz_setup_tc_mode(dev, port, MTI_SCHEDULE_STRICT_PRIO,
+				 MTI_SHAPING_SRP);
+}
+
+static int ksz_disable_egress_rate_limit(struct ksz_device *dev, int port)
+{
+	int queue, ret;
+
+	/* Configuration will not take effect until the last Port Queue X
+	 * Egress Limit Control Register is written.
+	 */
+	for (queue = 0; queue < dev->info->num_tx_queues; queue++) {
+		ret = ksz_pwrite8(dev, port, KSZ9477_REG_PORT_OUT_RATE_0 + queue,
+				  KSZ9477_OUT_RATE_NO_LIMIT);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int ksz_ets_band_to_queue(struct tc_ets_qopt_offload_replace_params *p,
+				 int band)
+{
+	/* Compared to queues, bands prioritize packets differently. In strict
+	 * priority mode, the lowest priority is assigned to Queue 0 while the
+	 * highest priority is given to Band 0.
+	 */
+	return p->bands - 1 - band;
+}
+
+static int ksz_queue_set_strict(struct ksz_device *dev, int port, int queue)
+{
+	int ret;
+
+	ret = ksz_pwrite32(dev, port, REG_PORT_MTI_QUEUE_INDEX__4, queue);
+	if (ret)
+		return ret;
+
+	return ksz_setup_tc_mode(dev, port, MTI_SCHEDULE_STRICT_PRIO,
+				 MTI_SHAPING_OFF);
+}
+
+static int ksz_queue_set_wrr(struct ksz_device *dev, int port, int queue,
+			     int weight)
+{
+	int ret;
+
+	ret = ksz_pwrite32(dev, port, REG_PORT_MTI_QUEUE_INDEX__4, queue);
+	if (ret)
+		return ret;
+
+	ret = ksz_setup_tc_mode(dev, port, MTI_SCHEDULE_WRR,
+				MTI_SHAPING_OFF);
+	if (ret)
+		return ret;
+
+	return ksz_pwrite8(dev, port, KSZ9477_PORT_MTI_QUEUE_CTRL_1, weight);
+}
+
+static int ksz_tc_ets_add(struct ksz_device *dev, int port,
+			  struct tc_ets_qopt_offload_replace_params *p)
+{
+	int ret, band, tc_prio;
+	u32 queue_map = 0;
+
+	/* In order to ensure proper prioritization, it is necessary to set the
+	 * rate limit for the related queue to zero. Otherwise strict priority
+	 * or WRR mode will not work. This is a hardware limitation.
+	 */
+	ret = ksz_disable_egress_rate_limit(dev, port);
+	if (ret)
+		return ret;
+
+	/* Configure queue scheduling mode for all bands. Currently only strict
+	 * prio mode is supported.
+	 */
+	for (band = 0; band < p->bands; band++) {
+		int queue = ksz_ets_band_to_queue(p, band);
+
+		ret = ksz_queue_set_strict(dev, port, queue);
+		if (ret)
+			return ret;
+	}
+
+	/* Configure the mapping between traffic classes and queues. Note:
+	 * priomap variable support 16 traffic classes, but the chip can handle
+	 * only 8 classes.
+	 */
+	for (tc_prio = 0; tc_prio < ARRAY_SIZE(p->priomap); tc_prio++) {
+		int queue;
+
+		if (tc_prio > KSZ9477_MAX_TC_PRIO)
+			break;
+
+		queue = ksz_ets_band_to_queue(p, p->priomap[tc_prio]);
+		queue_map |= queue << (tc_prio * KSZ9477_PORT_TC_MAP_S);
+	}
+
+	return ksz_pwrite32(dev, port, KSZ9477_PORT_MRI_TC_MAP__4, queue_map);
+}
+
+static int ksz_tc_ets_del(struct ksz_device *dev, int port)
+{
+	int ret, queue, tc_prio, s;
+	u32 queue_map = 0;
+
+	/* To restore the default chip configuration, set all queues to use the
+	 * WRR scheduler with a weight of 1.
+	 */
+	for (queue = 0; queue < dev->info->num_tx_queues; queue++) {
+		ret = ksz_queue_set_wrr(dev, port, queue,
+					KSZ9477_DEFAULT_WRR_WEIGHT);
+		if (ret)
+			return ret;
+	}
+
+	switch (dev->info->num_tx_queues) {
+	case 2:
+		s = 2;
+		break;
+	case 4:
+		s = 1;
+		break;
+	case 8:
+		s = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Revert the queue mapping for TC-priority to its default setting on
+	 * the chip.
+	 */
+	for (tc_prio = 0; tc_prio <= KSZ9477_MAX_TC_PRIO; tc_prio++) {
+		int queue;
+
+		queue = tc_prio >> s;
+		queue_map |= queue << (tc_prio * KSZ9477_PORT_TC_MAP_S);
+	}
+
+	return ksz_pwrite32(dev, port, KSZ9477_PORT_MRI_TC_MAP__4, queue_map);
+}
+
+static int ksz_tc_ets_validate(struct ksz_device *dev, int port,
+			       struct tc_ets_qopt_offload_replace_params *p)
+{
+	int band;
+
+	/* Since it is not feasible to share one port among multiple qdisc,
+	 * the user must configure all available queues appropriately.
+	 */
+	if (p->bands != dev->info->num_tx_queues) {
+		dev_err(dev->dev, "Not supported amount of bands. It should be %d\n",
+			dev->info->num_tx_queues);
+		return -EOPNOTSUPP;
+	}
+
+	for (band = 0; band < p->bands; ++band) {
+		/* The KSZ switches utilize a weighted round robin configuration
+		 * where a certain number of packets can be transmitted from a
+		 * queue before the next queue is serviced. For more information
+		 * on this, refer to section 5.2.8.4 of the KSZ8565R
+		 * documentation on the Port Transmit Queue Control 1 Register.
+		 * However, the current ETS Qdisc implementation (as of February
+		 * 2023) assigns a weight to each queue based on the number of
+		 * bytes or extrapolated bandwidth in percentages. Since this
+		 * differs from the KSZ switches' method and we don't want to
+		 * fake support by converting bytes to packets, it is better to
+		 * return an error instead.
+		 */
+		if (p->quanta[band]) {
+			dev_err(dev->dev, "Quanta/weights configuration is not supported.\n");
+			return -EOPNOTSUPP;
+		}
+	}
+
+	return 0;
+}
+
+static int ksz_tc_setup_qdisc_ets(struct dsa_switch *ds, int port,
+				  struct tc_ets_qopt_offload *qopt)
+{
+	struct ksz_device *dev = ds->priv;
+	int ret;
+
+	if (!dev->info->tc_ets_supported)
+		return -EOPNOTSUPP;
+
+	if (qopt->parent != TC_H_ROOT) {
+		dev_err(dev->dev, "Parent should be \"root\"\n");
+		return -EOPNOTSUPP;
+	}
+
+	switch (qopt->command) {
+	case TC_ETS_REPLACE:
+		ret = ksz_tc_ets_validate(dev, port, &qopt->replace_params);
+		if (ret)
+			return ret;
+
+		return ksz_tc_ets_add(dev, port, &qopt->replace_params);
+	case TC_ETS_DESTROY:
+		return ksz_tc_ets_del(dev, port);
+	case TC_ETS_STATS:
+	case TC_ETS_GRAFT:
+		return -EOPNOTSUPP;
+	}
+
+	return -EOPNOTSUPP;
 }
 
 static int ksz_setup_tc(struct dsa_switch *ds, int port,
@@ -3146,6 +3368,8 @@ static int ksz_setup_tc(struct dsa_switch *ds, int port,
 	switch (type) {
 	case TC_SETUP_QDISC_CBS:
 		return ksz_setup_tc_cbs(ds, port, type_data);
+	case TC_SETUP_QDISC_ETS:
+		return ksz_tc_setup_qdisc_ets(ds, port, type_data);
 	default:
 		return -EOPNOTSUPP;
 	}
diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h
index d2d5761d58e9..8abecaf6089e 100644
--- a/drivers/net/dsa/microchip/ksz_common.h
+++ b/drivers/net/dsa/microchip/ksz_common.h
@@ -51,6 +51,7 @@ struct ksz_chip_data {
 	u8 port_nirqs;
 	u8 num_tx_queues;
 	bool tc_cbs_supported;
+	bool tc_ets_supported;
 	const struct ksz_dev_ops *ops;
 	bool phy_errata_9477;
 	bool ksz87xx_eee_link_erratum;
@@ -649,21 +650,30 @@ static inline int is_lan937x(struct ksz_device *dev)
 #define KSZ8_LEGAL_PACKET_SIZE		1518
 #define KSZ9477_MAX_FRAME_SIZE		9000
 
+#define KSZ9477_REG_PORT_OUT_RATE_0	0x0420
+#define KSZ9477_OUT_RATE_NO_LIMIT	0
+
+#define KSZ9477_PORT_MRI_TC_MAP__4	0x0808
+
+#define KSZ9477_PORT_TC_MAP_S		4
+#define KSZ9477_MAX_TC_PRIO		7
+
 /* CBS related registers */
 #define REG_PORT_MTI_QUEUE_INDEX__4	0x0900
 
 #define REG_PORT_MTI_QUEUE_CTRL_0	0x0914
 
-#define MTI_SCHEDULE_MODE_M		0x3
-#define MTI_SCHEDULE_MODE_S		6
+#define MTI_SCHEDULE_MODE_M		GENMASK(7, 6)
 #define MTI_SCHEDULE_STRICT_PRIO	0
 #define MTI_SCHEDULE_WRR		2
-#define MTI_SHAPING_M			0x3
-#define MTI_SHAPING_S			4
+#define MTI_SHAPING_M			GENMASK(5, 4)
 #define MTI_SHAPING_OFF			0
 #define MTI_SHAPING_SRP			1
 #define MTI_SHAPING_TIME_AWARE		2
 
+#define KSZ9477_PORT_MTI_QUEUE_CTRL_1	0x0915
+#define KSZ9477_DEFAULT_WRR_WEIGHT	1
+
 #define REG_PORT_MTI_HI_WATER_MARK	0x0916
 #define REG_PORT_MTI_LO_WATER_MARK	0x0918
 
diff --git a/drivers/net/dsa/mt7530-mdio.c b/drivers/net/dsa/mt7530-mdio.c
new file mode 100644
index 000000000000..088533663b83
--- /dev/null
+++ b/drivers/net/dsa/mt7530-mdio.c
@@ -0,0 +1,271 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/gpio/consumer.h>
+#include <linux/mdio.h>
+#include <linux/module.h>
+#include <linux/pcs/pcs-mtk-lynxi.h>
+#include <linux/of_irq.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
+#include <linux/of_platform.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/regulator/consumer.h>
+#include <net/dsa.h>
+
+#include "mt7530.h"
+
+static int
+mt7530_regmap_write(void *context, unsigned int reg, unsigned int val)
+{
+	struct mii_bus *bus = context;
+	u16 page, r, lo, hi;
+	int ret;
+
+	page = (reg >> 6) & 0x3ff;
+	r  = (reg >> 2) & 0xf;
+	lo = val & 0xffff;
+	hi = val >> 16;
+
+	/* MT7530 uses 31 as the pseudo port */
+	ret = bus->write(bus, 0x1f, 0x1f, page);
+	if (ret < 0)
+		return ret;
+
+	ret = bus->write(bus, 0x1f, r,  lo);
+	if (ret < 0)
+		return ret;
+
+	ret = bus->write(bus, 0x1f, 0x10, hi);
+	return ret;
+}
+
+static int
+mt7530_regmap_read(void *context, unsigned int reg, unsigned int *val)
+{
+	struct mii_bus *bus = context;
+	u16 page, r, lo, hi;
+	int ret;
+
+	page = (reg >> 6) & 0x3ff;
+	r = (reg >> 2) & 0xf;
+
+	/* MT7530 uses 31 as the pseudo port */
+	ret = bus->write(bus, 0x1f, 0x1f, page);
+	if (ret < 0)
+		return ret;
+
+	lo = bus->read(bus, 0x1f, r);
+	hi = bus->read(bus, 0x1f, 0x10);
+
+	*val = (hi << 16) | (lo & 0xffff);
+
+	return 0;
+}
+
+static void
+mt7530_mdio_regmap_lock(void *mdio_lock)
+{
+	mutex_lock_nested(mdio_lock, MDIO_MUTEX_NESTED);
+}
+
+static void
+mt7530_mdio_regmap_unlock(void *mdio_lock)
+{
+	mutex_unlock(mdio_lock);
+}
+
+static const struct regmap_bus mt7530_regmap_bus = {
+	.reg_write = mt7530_regmap_write,
+	.reg_read = mt7530_regmap_read,
+};
+
+static int
+mt7531_create_sgmii(struct mt7530_priv *priv, bool dual_sgmii)
+{
+	struct regmap_config *mt7531_pcs_config[2] = {};
+	struct phylink_pcs *pcs;
+	struct regmap *regmap;
+	int i, ret = 0;
+
+	/* MT7531AE has two SGMII units for port 5 and port 6
+	 * MT7531BE has only one SGMII unit for port 6
+	 */
+	for (i = dual_sgmii ? 0 : 1; i < 2; i++) {
+		mt7531_pcs_config[i] = devm_kzalloc(priv->dev,
+						    sizeof(struct regmap_config),
+						    GFP_KERNEL);
+		if (!mt7531_pcs_config[i]) {
+			ret = -ENOMEM;
+			break;
+		}
+
+		mt7531_pcs_config[i]->name = i ? "port6" : "port5";
+		mt7531_pcs_config[i]->reg_bits = 16;
+		mt7531_pcs_config[i]->val_bits = 32;
+		mt7531_pcs_config[i]->reg_stride = 4;
+		mt7531_pcs_config[i]->reg_base = MT7531_SGMII_REG_BASE(5 + i);
+		mt7531_pcs_config[i]->max_register = 0x17c;
+		mt7531_pcs_config[i]->lock = mt7530_mdio_regmap_lock;
+		mt7531_pcs_config[i]->unlock = mt7530_mdio_regmap_unlock;
+		mt7531_pcs_config[i]->lock_arg = &priv->bus->mdio_lock;
+
+		regmap = devm_regmap_init(priv->dev,
+					  &mt7530_regmap_bus, priv->bus,
+					  mt7531_pcs_config[i]);
+		if (IS_ERR(regmap)) {
+			ret = PTR_ERR(regmap);
+			break;
+		}
+		pcs = mtk_pcs_lynxi_create(priv->dev, regmap,
+					   MT7531_PHYA_CTRL_SIGNAL3, 0);
+		if (!pcs) {
+			ret = -ENXIO;
+			break;
+		}
+		priv->ports[5 + i].sgmii_pcs = pcs;
+	}
+
+	if (ret && i)
+		mtk_pcs_lynxi_destroy(priv->ports[5].sgmii_pcs);
+
+	return ret;
+}
+
+static const struct of_device_id mt7530_of_match[] = {
+	{ .compatible = "mediatek,mt7621", .data = &mt753x_table[ID_MT7621], },
+	{ .compatible = "mediatek,mt7530", .data = &mt753x_table[ID_MT7530], },
+	{ .compatible = "mediatek,mt7531", .data = &mt753x_table[ID_MT7531], },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, mt7530_of_match);
+
+static int
+mt7530_probe(struct mdio_device *mdiodev)
+{
+	static struct regmap_config *regmap_config;
+	struct mt7530_priv *priv;
+	struct device_node *dn;
+	int ret;
+
+	dn = mdiodev->dev.of_node;
+
+	priv = devm_kzalloc(&mdiodev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->bus = mdiodev->bus;
+	priv->dev = &mdiodev->dev;
+
+	ret = mt7530_probe_common(priv);
+	if (ret)
+		return ret;
+
+	/* Use medatek,mcm property to distinguish hardware type that would
+	 * cause a little bit differences on power-on sequence.
+	 * Not MCM that indicates switch works as the remote standalone
+	 * integrated circuit so the GPIO pin would be used to complete
+	 * the reset, otherwise memory-mapped register accessing used
+	 * through syscon provides in the case of MCM.
+	 */
+	priv->mcm = of_property_read_bool(dn, "mediatek,mcm");
+	if (priv->mcm) {
+		dev_info(&mdiodev->dev, "MT7530 adapts as multi-chip module\n");
+
+		priv->rstc = devm_reset_control_get(&mdiodev->dev, "mcm");
+		if (IS_ERR(priv->rstc)) {
+			dev_err(&mdiodev->dev, "Couldn't get our reset line\n");
+			return PTR_ERR(priv->rstc);
+		}
+	} else {
+		priv->reset = devm_gpiod_get_optional(&mdiodev->dev, "reset",
+						      GPIOD_OUT_LOW);
+		if (IS_ERR(priv->reset)) {
+			dev_err(&mdiodev->dev, "Couldn't get our reset line\n");
+			return PTR_ERR(priv->reset);
+		}
+	}
+
+	if (priv->id == ID_MT7530) {
+		priv->core_pwr = devm_regulator_get(&mdiodev->dev, "core");
+		if (IS_ERR(priv->core_pwr))
+			return PTR_ERR(priv->core_pwr);
+
+		priv->io_pwr = devm_regulator_get(&mdiodev->dev, "io");
+		if (IS_ERR(priv->io_pwr))
+			return PTR_ERR(priv->io_pwr);
+	}
+
+	regmap_config = devm_kzalloc(&mdiodev->dev, sizeof(*regmap_config),
+				     GFP_KERNEL);
+	if (!regmap_config)
+		return -ENOMEM;
+
+	regmap_config->reg_bits = 16;
+	regmap_config->val_bits = 32;
+	regmap_config->reg_stride = 4;
+	regmap_config->max_register = MT7530_CREV;
+	regmap_config->disable_locking = true;
+	priv->regmap = devm_regmap_init(priv->dev, &mt7530_regmap_bus,
+					priv->bus, regmap_config);
+	if (IS_ERR(priv->regmap))
+		return PTR_ERR(priv->regmap);
+
+	if (priv->id == ID_MT7531)
+		priv->create_sgmii = mt7531_create_sgmii;
+
+	return dsa_register_switch(priv->ds);
+}
+
+static void
+mt7530_remove(struct mdio_device *mdiodev)
+{
+	struct mt7530_priv *priv = dev_get_drvdata(&mdiodev->dev);
+	int ret = 0, i;
+
+	if (!priv)
+		return;
+
+	ret = regulator_disable(priv->core_pwr);
+	if (ret < 0)
+		dev_err(priv->dev,
+			"Failed to disable core power: %d\n", ret);
+
+	ret = regulator_disable(priv->io_pwr);
+	if (ret < 0)
+		dev_err(priv->dev, "Failed to disable io pwr: %d\n",
+			ret);
+
+	mt7530_remove_common(priv);
+
+	for (i = 0; i < 2; ++i)
+		mtk_pcs_lynxi_destroy(priv->ports[5 + i].sgmii_pcs);
+}
+
+static void mt7530_shutdown(struct mdio_device *mdiodev)
+{
+	struct mt7530_priv *priv = dev_get_drvdata(&mdiodev->dev);
+
+	if (!priv)
+		return;
+
+	dsa_switch_shutdown(priv->ds);
+
+	dev_set_drvdata(&mdiodev->dev, NULL);
+}
+
+static struct mdio_driver mt7530_mdio_driver = {
+	.probe  = mt7530_probe,
+	.remove = mt7530_remove,
+	.shutdown = mt7530_shutdown,
+	.mdiodrv.driver = {
+		.name = "mt7530-mdio",
+		.of_match_table = mt7530_of_match,
+	},
+};
+
+mdio_module_driver(mt7530_mdio_driver);
+
+MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
+MODULE_DESCRIPTION("Driver for Mediatek MT7530 Switch (MDIO)");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/dsa/mt7530-mmio.c b/drivers/net/dsa/mt7530-mmio.c
new file mode 100644
index 000000000000..1a3d4b692f34
--- /dev/null
+++ b/drivers/net/dsa/mt7530-mmio.c
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <net/dsa.h>
+
+#include "mt7530.h"
+
+static const struct of_device_id mt7988_of_match[] = {
+	{ .compatible = "mediatek,mt7988-switch", .data = &mt753x_table[ID_MT7988], },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, mt7988_of_match);
+
+static int
+mt7988_probe(struct platform_device *pdev)
+{
+	static struct regmap_config *sw_regmap_config;
+	struct mt7530_priv *priv;
+	void __iomem *base_addr;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->bus = NULL;
+	priv->dev = &pdev->dev;
+
+	ret = mt7530_probe_common(priv);
+	if (ret)
+		return ret;
+
+	priv->rstc = devm_reset_control_get(&pdev->dev, NULL);
+	if (IS_ERR(priv->rstc)) {
+		dev_err(&pdev->dev, "Couldn't get our reset line\n");
+		return PTR_ERR(priv->rstc);
+	}
+
+	base_addr = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base_addr)) {
+		dev_err(&pdev->dev, "cannot request I/O memory space\n");
+		return -ENXIO;
+	}
+
+	sw_regmap_config = devm_kzalloc(&pdev->dev, sizeof(*sw_regmap_config), GFP_KERNEL);
+	if (!sw_regmap_config)
+		return -ENOMEM;
+
+	sw_regmap_config->name = "switch";
+	sw_regmap_config->reg_bits = 16;
+	sw_regmap_config->val_bits = 32;
+	sw_regmap_config->reg_stride = 4;
+	sw_regmap_config->max_register = MT7530_CREV;
+	priv->regmap = devm_regmap_init_mmio(&pdev->dev, base_addr, sw_regmap_config);
+	if (IS_ERR(priv->regmap))
+		return PTR_ERR(priv->regmap);
+
+	return dsa_register_switch(priv->ds);
+}
+
+static int
+mt7988_remove(struct platform_device *pdev)
+{
+	struct mt7530_priv *priv = platform_get_drvdata(pdev);
+
+	if (priv)
+		mt7530_remove_common(priv);
+
+	return 0;
+}
+
+static void mt7988_shutdown(struct platform_device *pdev)
+{
+	struct mt7530_priv *priv = platform_get_drvdata(pdev);
+
+	if (!priv)
+		return;
+
+	dsa_switch_shutdown(priv->ds);
+
+	dev_set_drvdata(&pdev->dev, NULL);
+}
+
+static struct platform_driver mt7988_platform_driver = {
+	.probe  = mt7988_probe,
+	.remove = mt7988_remove,
+	.shutdown = mt7988_shutdown,
+	.driver = {
+		.name = "mt7530-mmio",
+		.of_match_table = mt7988_of_match,
+	},
+};
+module_platform_driver(mt7988_platform_driver);
+
+MODULE_AUTHOR("Daniel Golle <daniel@makrotopia.org>");
+MODULE_DESCRIPTION("Driver for Mediatek MT7530 Switch (MMIO)");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 02410ac439b7..c680873819b0 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -142,31 +142,42 @@ err:
 }
 
 static void
-core_write(struct mt7530_priv *priv, u32 reg, u32 val)
+mt7530_mutex_lock(struct mt7530_priv *priv)
 {
-	struct mii_bus *bus = priv->bus;
+	if (priv->bus)
+		mutex_lock_nested(&priv->bus->mdio_lock, MDIO_MUTEX_NESTED);
+}
+
+static void
+mt7530_mutex_unlock(struct mt7530_priv *priv)
+{
+	if (priv->bus)
+		mutex_unlock(&priv->bus->mdio_lock);
+}
 
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+static void
+core_write(struct mt7530_priv *priv, u32 reg, u32 val)
+{
+	mt7530_mutex_lock(priv);
 
 	core_write_mmd_indirect(priv, reg, MDIO_MMD_VEND2, val);
 
-	mutex_unlock(&bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 }
 
 static void
 core_rmw(struct mt7530_priv *priv, u32 reg, u32 mask, u32 set)
 {
-	struct mii_bus *bus = priv->bus;
 	u32 val;
 
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(priv);
 
 	val = core_read_mmd_indirect(priv, reg, MDIO_MMD_VEND2);
 	val &= ~mask;
 	val |= set;
 	core_write_mmd_indirect(priv, reg, MDIO_MMD_VEND2, val);
 
-	mutex_unlock(&bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 }
 
 static void
@@ -184,66 +195,42 @@ core_clear(struct mt7530_priv *priv, u32 reg, u32 val)
 static int
 mt7530_mii_write(struct mt7530_priv *priv, u32 reg, u32 val)
 {
-	struct mii_bus *bus = priv->bus;
-	u16 page, r, lo, hi;
 	int ret;
 
-	page = (reg >> 6) & 0x3ff;
-	r  = (reg >> 2) & 0xf;
-	lo = val & 0xffff;
-	hi = val >> 16;
+	ret = regmap_write(priv->regmap, reg, val);
 
-	/* MT7530 uses 31 as the pseudo port */
-	ret = bus->write(bus, 0x1f, 0x1f, page);
 	if (ret < 0)
-		goto err;
-
-	ret = bus->write(bus, 0x1f, r,  lo);
-	if (ret < 0)
-		goto err;
-
-	ret = bus->write(bus, 0x1f, 0x10, hi);
-err:
-	if (ret < 0)
-		dev_err(&bus->dev,
+		dev_err(priv->dev,
 			"failed to write mt7530 register\n");
+
 	return ret;
 }
 
 static u32
 mt7530_mii_read(struct mt7530_priv *priv, u32 reg)
 {
-	struct mii_bus *bus = priv->bus;
-	u16 page, r, lo, hi;
 	int ret;
+	u32 val;
 
-	page = (reg >> 6) & 0x3ff;
-	r = (reg >> 2) & 0xf;
-
-	/* MT7530 uses 31 as the pseudo port */
-	ret = bus->write(bus, 0x1f, 0x1f, page);
-	if (ret < 0) {
-		dev_err(&bus->dev,
+	ret = regmap_read(priv->regmap, reg, &val);
+	if (ret) {
+		WARN_ON_ONCE(1);
+		dev_err(priv->dev,
 			"failed to read mt7530 register\n");
-		return ret;
+		return 0;
 	}
 
-	lo = bus->read(bus, 0x1f, r);
-	hi = bus->read(bus, 0x1f, 0x10);
-
-	return (hi << 16) | (lo & 0xffff);
+	return val;
 }
 
 static void
 mt7530_write(struct mt7530_priv *priv, u32 reg, u32 val)
 {
-	struct mii_bus *bus = priv->bus;
-
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(priv);
 
 	mt7530_mii_write(priv, reg, val);
 
-	mutex_unlock(&bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 }
 
 static u32
@@ -255,14 +242,13 @@ _mt7530_unlocked_read(struct mt7530_dummy_poll *p)
 static u32
 _mt7530_read(struct mt7530_dummy_poll *p)
 {
-	struct mii_bus		*bus = p->priv->bus;
 	u32 val;
 
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(p->priv);
 
 	val = mt7530_mii_read(p->priv, p->reg);
 
-	mutex_unlock(&bus->mdio_lock);
+	mt7530_mutex_unlock(p->priv);
 
 	return val;
 }
@@ -280,23 +266,17 @@ static void
 mt7530_rmw(struct mt7530_priv *priv, u32 reg,
 	   u32 mask, u32 set)
 {
-	struct mii_bus *bus = priv->bus;
-	u32 val;
-
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(priv);
 
-	val = mt7530_mii_read(priv, reg);
-	val &= ~mask;
-	val |= set;
-	mt7530_mii_write(priv, reg, val);
+	regmap_update_bits(priv->regmap, reg, mask, set);
 
-	mutex_unlock(&bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 }
 
 static void
 mt7530_set(struct mt7530_priv *priv, u32 reg, u32 val)
 {
-	mt7530_rmw(priv, reg, 0, val);
+	mt7530_rmw(priv, reg, val, val);
 }
 
 static void
@@ -634,14 +614,13 @@ static int
 mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
 			int regnum)
 {
-	struct mii_bus *bus = priv->bus;
 	struct mt7530_dummy_poll p;
 	u32 reg, val;
 	int ret;
 
 	INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC);
 
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(priv);
 
 	ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
 				 !(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -674,7 +653,7 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
 
 	ret = val & MT7531_MDIO_RW_DATA_MASK;
 out:
-	mutex_unlock(&bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 
 	return ret;
 }
@@ -683,14 +662,13 @@ static int
 mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
 			 int regnum, u16 data)
 {
-	struct mii_bus *bus = priv->bus;
 	struct mt7530_dummy_poll p;
 	u32 val, reg;
 	int ret;
 
 	INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC);
 
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(priv);
 
 	ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
 				 !(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -722,7 +700,7 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
 	}
 
 out:
-	mutex_unlock(&bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 
 	return ret;
 }
@@ -730,14 +708,13 @@ out:
 static int
 mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum)
 {
-	struct mii_bus *bus = priv->bus;
 	struct mt7530_dummy_poll p;
 	int ret;
 	u32 val;
 
 	INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC);
 
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(priv);
 
 	ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
 				 !(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -760,7 +737,7 @@ mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum)
 
 	ret = val & MT7531_MDIO_RW_DATA_MASK;
 out:
-	mutex_unlock(&bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 
 	return ret;
 }
@@ -769,14 +746,13 @@ static int
 mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum,
 			 u16 data)
 {
-	struct mii_bus *bus = priv->bus;
 	struct mt7530_dummy_poll p;
 	int ret;
 	u32 reg;
 
 	INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC);
 
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(priv);
 
 	ret = readx_poll_timeout(_mt7530_unlocked_read, &p, reg,
 				 !(reg & MT7531_PHY_ACS_ST), 20, 100000);
@@ -798,7 +774,7 @@ mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum,
 	}
 
 out:
-	mutex_unlock(&bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 
 	return ret;
 }
@@ -920,6 +896,24 @@ mt7530_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
 	return 0;
 }
 
+static const char *p5_intf_modes(unsigned int p5_interface)
+{
+	switch (p5_interface) {
+	case P5_DISABLED:
+		return "DISABLED";
+	case P5_INTF_SEL_PHY_P0:
+		return "PHY P0";
+	case P5_INTF_SEL_PHY_P4:
+		return "PHY P4";
+	case P5_INTF_SEL_GMAC5:
+		return "GMAC5";
+	case P5_INTF_SEL_GMAC5_SGMII:
+		return "GMAC5_SGMII";
+	default:
+		return "unknown";
+	}
+}
+
 static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
 {
 	struct mt7530_priv *priv = ds->priv;
@@ -1079,7 +1073,6 @@ static int
 mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
 {
 	struct mt7530_priv *priv = ds->priv;
-	struct mii_bus *bus = priv->bus;
 	int length;
 	u32 val;
 
@@ -1090,7 +1083,7 @@ mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
 	if (!dsa_is_cpu_port(ds, port))
 		return 0;
 
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(priv);
 
 	val = mt7530_mii_read(priv, MT7530_GMACCR);
 	val &= ~MAX_RX_PKT_LEN_MASK;
@@ -1111,7 +1104,7 @@ mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
 
 	mt7530_mii_write(priv, MT7530_GMACCR, val);
 
-	mutex_unlock(&bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 
 	return 0;
 }
@@ -1912,10 +1905,10 @@ mt7530_irq_thread_fn(int irq, void *dev_id)
 	u32 val;
 	int p;
 
-	mutex_lock_nested(&priv->bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(priv);
 	val = mt7530_mii_read(priv, MT7530_SYS_INT_STS);
 	mt7530_mii_write(priv, MT7530_SYS_INT_STS, val);
-	mutex_unlock(&priv->bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 
 	for (p = 0; p < MT7530_NUM_PHYS; p++) {
 		if (BIT(p) & val) {
@@ -1951,7 +1944,7 @@ mt7530_irq_bus_lock(struct irq_data *d)
 {
 	struct mt7530_priv *priv = irq_data_get_irq_chip_data(d);
 
-	mutex_lock_nested(&priv->bus->mdio_lock, MDIO_MUTEX_NESTED);
+	mt7530_mutex_lock(priv);
 }
 
 static void
@@ -1960,7 +1953,7 @@ mt7530_irq_bus_sync_unlock(struct irq_data *d)
 	struct mt7530_priv *priv = irq_data_get_irq_chip_data(d);
 
 	mt7530_mii_write(priv, MT7530_SYS_INT_EN, priv->irq_enable);
-	mutex_unlock(&priv->bus->mdio_lock);
+	mt7530_mutex_unlock(priv);
 }
 
 static struct irq_chip mt7530_irq_chip = {
@@ -1989,6 +1982,47 @@ static const struct irq_domain_ops mt7530_irq_domain_ops = {
 };
 
 static void
+mt7988_irq_mask(struct irq_data *d)
+{
+	struct mt7530_priv *priv = irq_data_get_irq_chip_data(d);
+
+	priv->irq_enable &= ~BIT(d->hwirq);
+	mt7530_mii_write(priv, MT7530_SYS_INT_EN, priv->irq_enable);
+}
+
+static void
+mt7988_irq_unmask(struct irq_data *d)
+{
+	struct mt7530_priv *priv = irq_data_get_irq_chip_data(d);
+
+	priv->irq_enable |= BIT(d->hwirq);
+	mt7530_mii_write(priv, MT7530_SYS_INT_EN, priv->irq_enable);
+}
+
+static struct irq_chip mt7988_irq_chip = {
+	.name = KBUILD_MODNAME,
+	.irq_mask = mt7988_irq_mask,
+	.irq_unmask = mt7988_irq_unmask,
+};
+
+static int
+mt7988_irq_map(struct irq_domain *domain, unsigned int irq,
+	       irq_hw_number_t hwirq)
+{
+	irq_set_chip_data(irq, domain->host_data);
+	irq_set_chip_and_handler(irq, &mt7988_irq_chip, handle_simple_irq);
+	irq_set_nested_thread(irq, true);
+	irq_set_noprobe(irq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops mt7988_irq_domain_ops = {
+	.map = mt7988_irq_map,
+	.xlate = irq_domain_xlate_onecell,
+};
+
+static void
 mt7530_setup_mdio_irq(struct mt7530_priv *priv)
 {
 	struct dsa_switch *ds = priv->ds;
@@ -2022,8 +2056,15 @@ mt7530_setup_irq(struct mt7530_priv *priv)
 		return priv->irq ? : -EINVAL;
 	}
 
-	priv->irq_domain = irq_domain_add_linear(np, MT7530_NUM_PHYS,
-						 &mt7530_irq_domain_ops, priv);
+	if (priv->id == ID_MT7988)
+		priv->irq_domain = irq_domain_add_linear(np, MT7530_NUM_PHYS,
+							 &mt7988_irq_domain_ops,
+							 priv);
+	else
+		priv->irq_domain = irq_domain_add_linear(np, MT7530_NUM_PHYS,
+							 &mt7530_irq_domain_ops,
+							 priv);
+
 	if (!priv->irq_domain) {
 		dev_err(dev, "failed to create IRQ domain\n");
 		return -ENOMEM;
@@ -2308,11 +2349,64 @@ mt7530_setup(struct dsa_switch *ds)
 }
 
 static int
+mt7531_setup_common(struct dsa_switch *ds)
+{
+	struct mt7530_priv *priv = ds->priv;
+	struct dsa_port *cpu_dp;
+	int ret, i;
+
+	/* BPDU to CPU port */
+	dsa_switch_for_each_cpu_port(cpu_dp, ds) {
+		mt7530_rmw(priv, MT7531_CFC, MT7531_CPU_PMAP_MASK,
+			   BIT(cpu_dp->index));
+		break;
+	}
+	mt7530_rmw(priv, MT753X_BPC, MT753X_BPDU_PORT_FW_MASK,
+		   MT753X_BPDU_CPU_ONLY);
+
+	/* Enable and reset MIB counters */
+	mt7530_mib_reset(ds);
+
+	for (i = 0; i < MT7530_NUM_PORTS; i++) {
+		/* Disable forwarding by default on all ports */
+		mt7530_rmw(priv, MT7530_PCR_P(i), PCR_MATRIX_MASK,
+			   PCR_MATRIX_CLR);
+
+		/* Disable learning by default on all ports */
+		mt7530_set(priv, MT7530_PSC_P(i), SA_DIS);
+
+		mt7530_set(priv, MT7531_DBG_CNT(i), MT7531_DIS_CLR);
+
+		if (dsa_is_cpu_port(ds, i)) {
+			ret = mt753x_cpu_port_enable(ds, i);
+			if (ret)
+				return ret;
+		} else {
+			mt7530_port_disable(ds, i);
+
+			/* Set default PVID to 0 on all user ports */
+			mt7530_rmw(priv, MT7530_PPBV1_P(i), G0_PORT_VID_MASK,
+				   G0_PORT_VID_DEF);
+		}
+
+		/* Enable consistent egress tag */
+		mt7530_rmw(priv, MT7530_PVC_P(i), PVC_EG_TAG_MASK,
+			   PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
+	}
+
+	/* Flush the FDB table */
+	ret = mt7530_fdb_cmd(priv, MT7530_FDB_FLUSH, NULL);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
 mt7531_setup(struct dsa_switch *ds)
 {
 	struct mt7530_priv *priv = ds->priv;
 	struct mt7530_dummy_poll p;
-	struct dsa_port *cpu_dp;
 	u32 val, id;
 	int ret, i;
 
@@ -2390,44 +2484,7 @@ mt7531_setup(struct dsa_switch *ds)
 	mt7531_ind_c45_phy_write(priv, MT753X_CTRL_PHY_ADDR, MDIO_MMD_VEND2,
 				 CORE_PLL_GROUP4, val);
 
-	/* BPDU to CPU port */
-	dsa_switch_for_each_cpu_port(cpu_dp, ds) {
-		mt7530_rmw(priv, MT7531_CFC, MT7531_CPU_PMAP_MASK,
-			   BIT(cpu_dp->index));
-		break;
-	}
-	mt7530_rmw(priv, MT753X_BPC, MT753X_BPDU_PORT_FW_MASK,
-		   MT753X_BPDU_CPU_ONLY);
-
-	/* Enable and reset MIB counters */
-	mt7530_mib_reset(ds);
-
-	for (i = 0; i < MT7530_NUM_PORTS; i++) {
-		/* Disable forwarding by default on all ports */
-		mt7530_rmw(priv, MT7530_PCR_P(i), PCR_MATRIX_MASK,
-			   PCR_MATRIX_CLR);
-
-		/* Disable learning by default on all ports */
-		mt7530_set(priv, MT7530_PSC_P(i), SA_DIS);
-
-		mt7530_set(priv, MT7531_DBG_CNT(i), MT7531_DIS_CLR);
-
-		if (dsa_is_cpu_port(ds, i)) {
-			ret = mt753x_cpu_port_enable(ds, i);
-			if (ret)
-				return ret;
-		} else {
-			mt7530_port_disable(ds, i);
-
-			/* Set default PVID to 0 on all user ports */
-			mt7530_rmw(priv, MT7530_PPBV1_P(i), G0_PORT_VID_MASK,
-				   G0_PORT_VID_DEF);
-		}
-
-		/* Enable consistent egress tag */
-		mt7530_rmw(priv, MT7530_PVC_P(i), PVC_EG_TAG_MASK,
-			   PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
-	}
+	mt7531_setup_common(ds);
 
 	/* Setup VLAN ID 0 for VLAN-unaware bridges */
 	ret = mt7530_setup_vlan0(priv);
@@ -2437,11 +2494,6 @@ mt7531_setup(struct dsa_switch *ds)
 	ds->assisted_learning_on_cpu_port = true;
 	ds->mtu_enforcement_ingress = true;
 
-	/* Flush the FDB table */
-	ret = mt7530_fdb_cmd(priv, MT7530_FDB_FLUSH, NULL);
-	if (ret < 0)
-		return ret;
-
 	return 0;
 }
 
@@ -2507,6 +2559,25 @@ static void mt7531_mac_port_get_caps(struct dsa_switch *ds, int port,
 	}
 }
 
+static void mt7988_mac_port_get_caps(struct dsa_switch *ds, int port,
+				     struct phylink_config *config)
+{
+	phy_interface_zero(config->supported_interfaces);
+
+	switch (port) {
+	case 0 ... 4: /* Internal phy */
+		__set_bit(PHY_INTERFACE_MODE_INTERNAL,
+			  config->supported_interfaces);
+		break;
+
+	case 6:
+		__set_bit(PHY_INTERFACE_MODE_INTERNAL,
+			  config->supported_interfaces);
+		config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
+					   MAC_10000FD;
+	}
+}
+
 static int
 mt753x_pad_setup(struct dsa_switch *ds, const struct phylink_link_state *state)
 {
@@ -2577,126 +2648,20 @@ static int mt7531_rgmii_setup(struct mt7530_priv *priv, u32 port,
 	return 0;
 }
 
-static void mt7531_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode,
-			       phy_interface_t interface, int speed, int duplex)
-{
-	struct mt7530_priv *priv = pcs_to_mt753x_pcs(pcs)->priv;
-	int port = pcs_to_mt753x_pcs(pcs)->port;
-	unsigned int val;
-
-	/* For adjusting speed and duplex of SGMII force mode. */
-	if (interface != PHY_INTERFACE_MODE_SGMII ||
-	    phylink_autoneg_inband(mode))
-		return;
-
-	/* SGMII force mode setting */
-	val = mt7530_read(priv, MT7531_SGMII_MODE(port));
-	val &= ~MT7531_SGMII_IF_MODE_MASK;
-
-	switch (speed) {
-	case SPEED_10:
-		val |= MT7531_SGMII_FORCE_SPEED_10;
-		break;
-	case SPEED_100:
-		val |= MT7531_SGMII_FORCE_SPEED_100;
-		break;
-	case SPEED_1000:
-		val |= MT7531_SGMII_FORCE_SPEED_1000;
-		break;
-	}
-
-	/* MT7531 SGMII 1G force mode can only work in full duplex mode,
-	 * no matter MT7531_SGMII_FORCE_HALF_DUPLEX is set or not.
-	 *
-	 * The speed check is unnecessary as the MAC capabilities apply
-	 * this restriction. --rmk
-	 */
-	if ((speed == SPEED_10 || speed == SPEED_100) &&
-	    duplex != DUPLEX_FULL)
-		val |= MT7531_SGMII_FORCE_HALF_DUPLEX;
-
-	mt7530_write(priv, MT7531_SGMII_MODE(port), val);
-}
-
 static bool mt753x_is_mac_port(u32 port)
 {
 	return (port == 5 || port == 6);
 }
 
-static int mt7531_sgmii_setup_mode_force(struct mt7530_priv *priv, u32 port,
-					 phy_interface_t interface)
-{
-	u32 val;
-
-	if (!mt753x_is_mac_port(port))
-		return -EINVAL;
-
-	mt7530_set(priv, MT7531_QPHY_PWR_STATE_CTRL(port),
-		   MT7531_SGMII_PHYA_PWD);
-
-	val = mt7530_read(priv, MT7531_PHYA_CTRL_SIGNAL3(port));
-	val &= ~MT7531_RG_TPHY_SPEED_MASK;
-	/* Setup 2.5 times faster clock for 2.5Gbps data speeds with 10B/8B
-	 * encoding.
-	 */
-	val |= (interface == PHY_INTERFACE_MODE_2500BASEX) ?
-		MT7531_RG_TPHY_SPEED_3_125G : MT7531_RG_TPHY_SPEED_1_25G;
-	mt7530_write(priv, MT7531_PHYA_CTRL_SIGNAL3(port), val);
-
-	mt7530_clear(priv, MT7531_PCS_CONTROL_1(port), MT7531_SGMII_AN_ENABLE);
-
-	/* MT7531 SGMII 1G and 2.5G force mode can only work in full duplex
-	 * mode, no matter MT7531_SGMII_FORCE_HALF_DUPLEX is set or not.
-	 */
-	mt7530_rmw(priv, MT7531_SGMII_MODE(port),
-		   MT7531_SGMII_IF_MODE_MASK | MT7531_SGMII_REMOTE_FAULT_DIS,
-		   MT7531_SGMII_FORCE_SPEED_1000);
-
-	mt7530_write(priv, MT7531_QPHY_PWR_STATE_CTRL(port), 0);
-
-	return 0;
-}
-
-static int mt7531_sgmii_setup_mode_an(struct mt7530_priv *priv, int port,
-				      phy_interface_t interface)
-{
-	if (!mt753x_is_mac_port(port))
-		return -EINVAL;
-
-	mt7530_set(priv, MT7531_QPHY_PWR_STATE_CTRL(port),
-		   MT7531_SGMII_PHYA_PWD);
-
-	mt7530_rmw(priv, MT7531_PHYA_CTRL_SIGNAL3(port),
-		   MT7531_RG_TPHY_SPEED_MASK, MT7531_RG_TPHY_SPEED_1_25G);
-
-	mt7530_set(priv, MT7531_SGMII_MODE(port),
-		   MT7531_SGMII_REMOTE_FAULT_DIS |
-		   MT7531_SGMII_SPEED_DUPLEX_AN);
-
-	mt7530_rmw(priv, MT7531_PCS_SPEED_ABILITY(port),
-		   MT7531_SGMII_TX_CONFIG_MASK, 1);
-
-	mt7530_set(priv, MT7531_PCS_CONTROL_1(port), MT7531_SGMII_AN_ENABLE);
-
-	mt7530_set(priv, MT7531_PCS_CONTROL_1(port), MT7531_SGMII_AN_RESTART);
-
-	mt7530_write(priv, MT7531_QPHY_PWR_STATE_CTRL(port), 0);
-
-	return 0;
-}
-
-static void mt7531_pcs_an_restart(struct phylink_pcs *pcs)
+static int
+mt7988_mac_config(struct dsa_switch *ds, int port, unsigned int mode,
+		  phy_interface_t interface)
 {
-	struct mt7530_priv *priv = pcs_to_mt753x_pcs(pcs)->priv;
-	int port = pcs_to_mt753x_pcs(pcs)->port;
-	u32 val;
+	if (dsa_is_cpu_port(ds, port) &&
+	    interface == PHY_INTERFACE_MODE_INTERNAL)
+		return 0;
 
-	/* Only restart AN when AN is enabled */
-	val = mt7530_read(priv, MT7531_PCS_CONTROL_1(port));
-	if (val & MT7531_SGMII_AN_ENABLE) {
-		val |= MT7531_SGMII_AN_RESTART;
-		mt7530_write(priv, MT7531_PCS_CONTROL_1(port), val);
-	}
+	return -EINVAL;
 }
 
 static int
@@ -2721,11 +2686,11 @@ mt7531_mac_config(struct dsa_switch *ds, int port, unsigned int mode,
 		phydev = dp->slave->phydev;
 		return mt7531_rgmii_setup(priv, port, interface, phydev);
 	case PHY_INTERFACE_MODE_SGMII:
-		return mt7531_sgmii_setup_mode_an(priv, port, interface);
 	case PHY_INTERFACE_MODE_NA:
 	case PHY_INTERFACE_MODE_1000BASEX:
 	case PHY_INTERFACE_MODE_2500BASEX:
-		return mt7531_sgmii_setup_mode_force(priv, port, interface);
+		/* handled in SGMII PCS driver */
+		return 0;
 	default:
 		return -EINVAL;
 	}
@@ -2750,11 +2715,11 @@ mt753x_phylink_mac_select_pcs(struct dsa_switch *ds, int port,
 
 	switch (interface) {
 	case PHY_INTERFACE_MODE_TRGMII:
+		return &priv->pcs[port].pcs;
 	case PHY_INTERFACE_MODE_SGMII:
 	case PHY_INTERFACE_MODE_1000BASEX:
 	case PHY_INTERFACE_MODE_2500BASEX:
-		return &priv->pcs[port].pcs;
-
+		return priv->ports[port].sgmii_pcs;
 	default:
 		return NULL;
 	}
@@ -2769,7 +2734,8 @@ mt753x_phylink_mac_config(struct dsa_switch *ds, int port, unsigned int mode,
 
 	switch (port) {
 	case 0 ... 4: /* Internal phy */
-		if (state->interface != PHY_INTERFACE_MODE_GMII)
+		if (state->interface != PHY_INTERFACE_MODE_GMII &&
+		    state->interface != PHY_INTERFACE_MODE_INTERNAL)
 			goto unsupported;
 		break;
 	case 5: /* 2nd cpu port with phy of port 0 or 4 / external phy */
@@ -2847,7 +2813,8 @@ static void mt753x_phylink_mac_link_up(struct dsa_switch *ds, int port,
 	/* MT753x MAC works in 1G full duplex mode for all up-clocked
 	 * variants.
 	 */
-	if (interface == PHY_INTERFACE_MODE_TRGMII ||
+	if (interface == PHY_INTERFACE_MODE_INTERNAL ||
+	    interface == PHY_INTERFACE_MODE_TRGMII ||
 	    (phy_interface_mode_is_8023z(interface))) {
 		speed = SPEED_1000;
 		duplex = DUPLEX_FULL;
@@ -2927,6 +2894,21 @@ mt7531_cpu_port_config(struct dsa_switch *ds, int port)
 	return 0;
 }
 
+static int
+mt7988_cpu_port_config(struct dsa_switch *ds, int port)
+{
+	struct mt7530_priv *priv = ds->priv;
+
+	mt7530_write(priv, MT7530_PMCR_P(port),
+		     PMCR_CPU_PORT_SETTING(priv->id));
+
+	mt753x_phylink_mac_link_up(ds, port, MLO_AN_FIXED,
+				   PHY_INTERFACE_MODE_INTERNAL, NULL,
+				   SPEED_10000, DUPLEX_FULL, true, true);
+
+	return 0;
+}
+
 static void mt753x_phylink_get_caps(struct dsa_switch *ds, int port,
 				    struct phylink_config *config)
 {
@@ -2992,86 +2974,6 @@ static void mt7530_pcs_get_state(struct phylink_pcs *pcs,
 		state->pause |= MLO_PAUSE_TX;
 }
 
-static int
-mt7531_sgmii_pcs_get_state_an(struct mt7530_priv *priv, int port,
-			      struct phylink_link_state *state)
-{
-	u32 status, val;
-	u16 config_reg;
-
-	status = mt7530_read(priv, MT7531_PCS_CONTROL_1(port));
-	state->link = !!(status & MT7531_SGMII_LINK_STATUS);
-	state->an_complete = !!(status & MT7531_SGMII_AN_COMPLETE);
-	if (state->interface == PHY_INTERFACE_MODE_SGMII &&
-	    (status & MT7531_SGMII_AN_ENABLE)) {
-		val = mt7530_read(priv, MT7531_PCS_SPEED_ABILITY(port));
-		config_reg = val >> 16;
-
-		switch (config_reg & LPA_SGMII_SPD_MASK) {
-		case LPA_SGMII_1000:
-			state->speed = SPEED_1000;
-			break;
-		case LPA_SGMII_100:
-			state->speed = SPEED_100;
-			break;
-		case LPA_SGMII_10:
-			state->speed = SPEED_10;
-			break;
-		default:
-			dev_err(priv->dev, "invalid sgmii PHY speed\n");
-			state->link = false;
-			return -EINVAL;
-		}
-
-		if (config_reg & LPA_SGMII_FULL_DUPLEX)
-			state->duplex = DUPLEX_FULL;
-		else
-			state->duplex = DUPLEX_HALF;
-	}
-
-	return 0;
-}
-
-static void
-mt7531_sgmii_pcs_get_state_inband(struct mt7530_priv *priv, int port,
-				  struct phylink_link_state *state)
-{
-	unsigned int val;
-
-	val = mt7530_read(priv, MT7531_PCS_CONTROL_1(port));
-	state->link = !!(val & MT7531_SGMII_LINK_STATUS);
-	if (!state->link)
-		return;
-
-	state->an_complete = state->link;
-
-	if (state->interface == PHY_INTERFACE_MODE_2500BASEX)
-		state->speed = SPEED_2500;
-	else
-		state->speed = SPEED_1000;
-
-	state->duplex = DUPLEX_FULL;
-	state->pause = MLO_PAUSE_NONE;
-}
-
-static void mt7531_pcs_get_state(struct phylink_pcs *pcs,
-				 struct phylink_link_state *state)
-{
-	struct mt7530_priv *priv = pcs_to_mt753x_pcs(pcs)->priv;
-	int port = pcs_to_mt753x_pcs(pcs)->port;
-
-	if (state->interface == PHY_INTERFACE_MODE_SGMII) {
-		mt7531_sgmii_pcs_get_state_an(priv, port, state);
-		return;
-	} else if ((state->interface == PHY_INTERFACE_MODE_1000BASEX) ||
-		   (state->interface == PHY_INTERFACE_MODE_2500BASEX)) {
-		mt7531_sgmii_pcs_get_state_inband(priv, port, state);
-		return;
-	}
-
-	state->link = false;
-}
-
 static int mt753x_pcs_config(struct phylink_pcs *pcs, unsigned int mode,
 			     phy_interface_t interface,
 			     const unsigned long *advertising,
@@ -3091,14 +2993,6 @@ static const struct phylink_pcs_ops mt7530_pcs_ops = {
 	.pcs_an_restart = mt7530_pcs_an_restart,
 };
 
-static const struct phylink_pcs_ops mt7531_pcs_ops = {
-	.pcs_validate = mt753x_pcs_validate,
-	.pcs_get_state = mt7531_pcs_get_state,
-	.pcs_config = mt753x_pcs_config,
-	.pcs_an_restart = mt7531_pcs_an_restart,
-	.pcs_link_up = mt7531_pcs_link_up,
-};
-
 static int
 mt753x_setup(struct dsa_switch *ds)
 {
@@ -3110,8 +3004,6 @@ mt753x_setup(struct dsa_switch *ds)
 		priv->pcs[i].pcs.ops = priv->info->pcs_ops;
 		priv->pcs[i].priv = priv;
 		priv->pcs[i].port = i;
-		if (mt753x_is_mac_port(i))
-			priv->pcs[i].pcs.poll = 1;
 	}
 
 	ret = priv->info->sw_setup(ds);
@@ -3126,6 +3018,12 @@ mt753x_setup(struct dsa_switch *ds)
 	if (ret && priv->irq)
 		mt7530_free_irq_common(priv);
 
+	if (priv->create_sgmii) {
+		ret = priv->create_sgmii(priv, mt7531_dual_sgmii_supported(priv));
+		if (ret && priv->irq)
+			mt7530_free_irq(priv);
+	}
+
 	return ret;
 }
 
@@ -3159,7 +3057,28 @@ static int mt753x_set_mac_eee(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static const struct dsa_switch_ops mt7530_switch_ops = {
+static int mt7988_pad_setup(struct dsa_switch *ds, phy_interface_t interface)
+{
+	return 0;
+}
+
+static int mt7988_setup(struct dsa_switch *ds)
+{
+	struct mt7530_priv *priv = ds->priv;
+
+	/* Reset the switch */
+	reset_control_assert(priv->rstc);
+	usleep_range(20, 50);
+	reset_control_deassert(priv->rstc);
+	usleep_range(20, 50);
+
+	/* Reset the switch PHYs */
+	mt7530_write(priv, MT7530_SYS_CTRL, SYS_CTRL_PHY_RST);
+
+	return mt7531_setup_common(ds);
+}
+
+const struct dsa_switch_ops mt7530_switch_ops = {
 	.get_tag_protocol	= mtk_get_tag_protocol,
 	.setup			= mt753x_setup,
 	.get_strings		= mt7530_get_strings,
@@ -3193,8 +3112,9 @@ static const struct dsa_switch_ops mt7530_switch_ops = {
 	.get_mac_eee		= mt753x_get_mac_eee,
 	.set_mac_eee		= mt753x_set_mac_eee,
 };
+EXPORT_SYMBOL_GPL(mt7530_switch_ops);
 
-static const struct mt753x_info mt753x_table[] = {
+const struct mt753x_info mt753x_table[] = {
 	[ID_MT7621] = {
 		.id = ID_MT7621,
 		.pcs_ops = &mt7530_pcs_ops,
@@ -3221,7 +3141,7 @@ static const struct mt753x_info mt753x_table[] = {
 	},
 	[ID_MT7531] = {
 		.id = ID_MT7531,
-		.pcs_ops = &mt7531_pcs_ops,
+		.pcs_ops = &mt7530_pcs_ops,
 		.sw_setup = mt7531_setup,
 		.phy_read_c22 = mt7531_ind_c22_phy_read,
 		.phy_write_c22 = mt7531_ind_c22_phy_write,
@@ -3232,53 +3152,38 @@ static const struct mt753x_info mt753x_table[] = {
 		.mac_port_get_caps = mt7531_mac_port_get_caps,
 		.mac_port_config = mt7531_mac_config,
 	},
+	[ID_MT7988] = {
+		.id = ID_MT7988,
+		.pcs_ops = &mt7530_pcs_ops,
+		.sw_setup = mt7988_setup,
+		.phy_read_c22 = mt7531_ind_c22_phy_read,
+		.phy_write_c22 = mt7531_ind_c22_phy_write,
+		.phy_read_c45 = mt7531_ind_c45_phy_read,
+		.phy_write_c45 = mt7531_ind_c45_phy_write,
+		.pad_setup = mt7988_pad_setup,
+		.cpu_port_config = mt7988_cpu_port_config,
+		.mac_port_get_caps = mt7988_mac_port_get_caps,
+		.mac_port_config = mt7988_mac_config,
+	},
 };
+EXPORT_SYMBOL_GPL(mt753x_table);
 
-static const struct of_device_id mt7530_of_match[] = {
-	{ .compatible = "mediatek,mt7621", .data = &mt753x_table[ID_MT7621], },
-	{ .compatible = "mediatek,mt7530", .data = &mt753x_table[ID_MT7530], },
-	{ .compatible = "mediatek,mt7531", .data = &mt753x_table[ID_MT7531], },
-	{ /* sentinel */ },
-};
-MODULE_DEVICE_TABLE(of, mt7530_of_match);
-
-static int
-mt7530_probe(struct mdio_device *mdiodev)
+int
+mt7530_probe_common(struct mt7530_priv *priv)
 {
-	struct mt7530_priv *priv;
-	struct device_node *dn;
-
-	dn = mdiodev->dev.of_node;
-
-	priv = devm_kzalloc(&mdiodev->dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
+	struct device *dev = priv->dev;
 
-	priv->ds = devm_kzalloc(&mdiodev->dev, sizeof(*priv->ds), GFP_KERNEL);
+	priv->ds = devm_kzalloc(dev, sizeof(*priv->ds), GFP_KERNEL);
 	if (!priv->ds)
 		return -ENOMEM;
 
-	priv->ds->dev = &mdiodev->dev;
+	priv->ds->dev = dev;
 	priv->ds->num_ports = MT7530_NUM_PORTS;
 
-	/* Use medatek,mcm property to distinguish hardware type that would
-	 * casues a little bit differences on power-on sequence.
-	 */
-	priv->mcm = of_property_read_bool(dn, "mediatek,mcm");
-	if (priv->mcm) {
-		dev_info(&mdiodev->dev, "MT7530 adapts as multi-chip module\n");
-
-		priv->rstc = devm_reset_control_get(&mdiodev->dev, "mcm");
-		if (IS_ERR(priv->rstc)) {
-			dev_err(&mdiodev->dev, "Couldn't get our reset line\n");
-			return PTR_ERR(priv->rstc);
-		}
-	}
-
 	/* Get the hardware identifier from the devicetree node.
 	 * We will need it for some of the clock and regulator setup.
 	 */
-	priv->info = of_device_get_match_data(&mdiodev->dev);
+	priv->info = of_device_get_match_data(dev);
 	if (!priv->info)
 		return -EINVAL;
 
@@ -3292,90 +3197,27 @@ mt7530_probe(struct mdio_device *mdiodev)
 		return -EINVAL;
 
 	priv->id = priv->info->id;
-
-	if (priv->id == ID_MT7530) {
-		priv->core_pwr = devm_regulator_get(&mdiodev->dev, "core");
-		if (IS_ERR(priv->core_pwr))
-			return PTR_ERR(priv->core_pwr);
-
-		priv->io_pwr = devm_regulator_get(&mdiodev->dev, "io");
-		if (IS_ERR(priv->io_pwr))
-			return PTR_ERR(priv->io_pwr);
-	}
-
-	/* Not MCM that indicates switch works as the remote standalone
-	 * integrated circuit so the GPIO pin would be used to complete
-	 * the reset, otherwise memory-mapped register accessing used
-	 * through syscon provides in the case of MCM.
-	 */
-	if (!priv->mcm) {
-		priv->reset = devm_gpiod_get_optional(&mdiodev->dev, "reset",
-						      GPIOD_OUT_LOW);
-		if (IS_ERR(priv->reset)) {
-			dev_err(&mdiodev->dev, "Couldn't get our reset line\n");
-			return PTR_ERR(priv->reset);
-		}
-	}
-
-	priv->bus = mdiodev->bus;
-	priv->dev = &mdiodev->dev;
+	priv->dev = dev;
 	priv->ds->priv = priv;
 	priv->ds->ops = &mt7530_switch_ops;
 	mutex_init(&priv->reg_mutex);
-	dev_set_drvdata(&mdiodev->dev, priv);
+	dev_set_drvdata(dev, priv);
 
-	return dsa_register_switch(priv->ds);
+	return 0;
 }
+EXPORT_SYMBOL_GPL(mt7530_probe_common);
 
-static void
-mt7530_remove(struct mdio_device *mdiodev)
+void
+mt7530_remove_common(struct mt7530_priv *priv)
 {
-	struct mt7530_priv *priv = dev_get_drvdata(&mdiodev->dev);
-	int ret = 0;
-
-	if (!priv)
-		return;
-
-	ret = regulator_disable(priv->core_pwr);
-	if (ret < 0)
-		dev_err(priv->dev,
-			"Failed to disable core power: %d\n", ret);
-
-	ret = regulator_disable(priv->io_pwr);
-	if (ret < 0)
-		dev_err(priv->dev, "Failed to disable io pwr: %d\n",
-			ret);
-
 	if (priv->irq)
 		mt7530_free_irq(priv);
 
 	dsa_unregister_switch(priv->ds);
-	mutex_destroy(&priv->reg_mutex);
-}
-
-static void mt7530_shutdown(struct mdio_device *mdiodev)
-{
-	struct mt7530_priv *priv = dev_get_drvdata(&mdiodev->dev);
-
-	if (!priv)
-		return;
-
-	dsa_switch_shutdown(priv->ds);
 
-	dev_set_drvdata(&mdiodev->dev, NULL);
+	mutex_destroy(&priv->reg_mutex);
 }
-
-static struct mdio_driver mt7530_mdio_driver = {
-	.probe  = mt7530_probe,
-	.remove = mt7530_remove,
-	.shutdown = mt7530_shutdown,
-	.mdiodrv.driver = {
-		.name = "mt7530",
-		.of_match_table = mt7530_of_match,
-	},
-};
-
-mdio_module_driver(mt7530_mdio_driver);
+EXPORT_SYMBOL_GPL(mt7530_remove_common);
 
 MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
 MODULE_DESCRIPTION("Driver for Mediatek MT7530 Switch");
diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h
index 6b2fc6290ea8..5084f48a8869 100644
--- a/drivers/net/dsa/mt7530.h
+++ b/drivers/net/dsa/mt7530.h
@@ -18,6 +18,7 @@ enum mt753x_id {
 	ID_MT7530 = 0,
 	ID_MT7621 = 1,
 	ID_MT7531 = 2,
+	ID_MT7988 = 3,
 };
 
 #define	NUM_TRGMII_CTRL			5
@@ -54,11 +55,11 @@ enum mt753x_id {
 #define  MT7531_MIRROR_PORT_SET(x)	(((x) & MIRROR_MASK) << 16)
 #define  MT7531_CPU_PMAP_MASK		GENMASK(7, 0)
 
-#define MT753X_MIRROR_REG(id)		(((id) == ID_MT7531) ? \
+#define MT753X_MIRROR_REG(id)		((((id) == ID_MT7531) || ((id) == ID_MT7988)) ?	\
 					 MT7531_CFC : MT7530_MFC)
-#define MT753X_MIRROR_EN(id)		(((id) == ID_MT7531) ? \
+#define MT753X_MIRROR_EN(id)		((((id) == ID_MT7531) || ((id) == ID_MT7988)) ?	\
 					 MT7531_MIRROR_EN : MIRROR_EN)
-#define MT753X_MIRROR_MASK(id)		(((id) == ID_MT7531) ? \
+#define MT753X_MIRROR_MASK(id)		((((id) == ID_MT7531) || ((id) == ID_MT7988)) ?	\
 					 MT7531_MIRROR_MASK : MIRROR_MASK)
 
 /* Registers for BPDU and PAE frame control*/
@@ -295,9 +296,8 @@ enum mt7530_vlan_port_acc_frm {
 					 MT7531_FORCE_DPX | \
 					 MT7531_FORCE_RX_FC | \
 					 MT7531_FORCE_TX_FC)
-#define  PMCR_FORCE_MODE_ID(id)		(((id) == ID_MT7531) ? \
-					 MT7531_FORCE_MODE : \
-					 PMCR_FORCE_MODE)
+#define  PMCR_FORCE_MODE_ID(id)		((((id) == ID_MT7531) || ((id) == ID_MT7988)) ?	\
+					 MT7531_FORCE_MODE : PMCR_FORCE_MODE)
 #define  PMCR_LINK_SETTINGS_MASK	(PMCR_TX_EN | PMCR_FORCE_SPEED_1000 | \
 					 PMCR_RX_EN | PMCR_FORCE_SPEED_100 | \
 					 PMCR_TX_FC_EN | PMCR_RX_FC_EN | \
@@ -364,47 +364,8 @@ enum mt7530_vlan_port_acc_frm {
 					 CCR_TX_OCT_CNT_BAD)
 
 /* MT7531 SGMII register group */
-#define MT7531_SGMII_REG_BASE		0x5000
-#define MT7531_SGMII_REG(p, r)		(MT7531_SGMII_REG_BASE + \
-					((p) - 5) * 0x1000 + (r))
-
-/* Register forSGMII PCS_CONTROL_1 */
-#define MT7531_PCS_CONTROL_1(p)		MT7531_SGMII_REG(p, 0x00)
-#define  MT7531_SGMII_LINK_STATUS	BIT(18)
-#define  MT7531_SGMII_AN_ENABLE		BIT(12)
-#define  MT7531_SGMII_AN_RESTART	BIT(9)
-#define  MT7531_SGMII_AN_COMPLETE	BIT(21)
-
-/* Register for SGMII PCS_SPPED_ABILITY */
-#define MT7531_PCS_SPEED_ABILITY(p)	MT7531_SGMII_REG(p, 0x08)
-#define  MT7531_SGMII_TX_CONFIG_MASK	GENMASK(15, 0)
-#define  MT7531_SGMII_TX_CONFIG		BIT(0)
-
-/* Register for SGMII_MODE */
-#define MT7531_SGMII_MODE(p)		MT7531_SGMII_REG(p, 0x20)
-#define  MT7531_SGMII_REMOTE_FAULT_DIS	BIT(8)
-#define  MT7531_SGMII_IF_MODE_MASK	GENMASK(5, 1)
-#define  MT7531_SGMII_FORCE_DUPLEX	BIT(4)
-#define  MT7531_SGMII_FORCE_SPEED_MASK	GENMASK(3, 2)
-#define  MT7531_SGMII_FORCE_SPEED_1000	BIT(3)
-#define  MT7531_SGMII_FORCE_SPEED_100	BIT(2)
-#define  MT7531_SGMII_FORCE_SPEED_10	0
-#define  MT7531_SGMII_SPEED_DUPLEX_AN	BIT(1)
-
-enum mt7531_sgmii_force_duplex {
-	MT7531_SGMII_FORCE_FULL_DUPLEX = 0,
-	MT7531_SGMII_FORCE_HALF_DUPLEX = 0x10,
-};
-
-/* Fields of QPHY_PWR_STATE_CTRL */
-#define MT7531_QPHY_PWR_STATE_CTRL(p)	MT7531_SGMII_REG(p, 0xe8)
-#define  MT7531_SGMII_PHYA_PWD		BIT(4)
-
-/* Values of SGMII SPEED */
-#define MT7531_PHYA_CTRL_SIGNAL3(p)	MT7531_SGMII_REG(p, 0x128)
-#define  MT7531_RG_TPHY_SPEED_MASK	(BIT(2) | BIT(3))
-#define  MT7531_RG_TPHY_SPEED_1_25G	0x0
-#define  MT7531_RG_TPHY_SPEED_3_125G	BIT(2)
+#define MT7531_SGMII_REG_BASE(p)	(0x5000 + ((p) - 5) * 0x1000)
+#define MT7531_PHYA_CTRL_SIGNAL3	0x128
 
 /* Register for system reset */
 #define MT7530_SYS_CTRL			0x7000
@@ -703,13 +664,13 @@ struct mt7530_fdb {
  * @pm:		The matrix used to show all connections with the port.
  * @pvid:	The VLAN specified is to be considered a PVID at ingress.  Any
  *		untagged frames will be assigned to the related VLAN.
- * @vlan_filtering: The flags indicating whether the port that can recognize
- *		    VLAN-tagged frames.
+ * @sgmii_pcs:	Pointer to PCS instance for SerDes ports
  */
 struct mt7530_port {
 	bool enable;
 	u32 pm;
 	u16 pvid;
+	struct phylink_pcs *sgmii_pcs;
 };
 
 /* Port 5 interface select definitions */
@@ -721,24 +682,6 @@ enum p5_interface_select {
 	P5_INTF_SEL_GMAC5_SGMII,
 };
 
-static const char *p5_intf_modes(unsigned int p5_interface)
-{
-	switch (p5_interface) {
-	case P5_DISABLED:
-		return "DISABLED";
-	case P5_INTF_SEL_PHY_P0:
-		return "PHY P0";
-	case P5_INTF_SEL_PHY_P4:
-		return "PHY P4";
-	case P5_INTF_SEL_GMAC5:
-		return "GMAC5";
-	case P5_INTF_SEL_GMAC5_SGMII:
-		return "GMAC5_SGMII";
-	default:
-		return "unknown";
-	}
-}
-
 struct mt7530_priv;
 
 struct mt753x_pcs {
@@ -793,6 +736,7 @@ struct mt753x_info {
  * @dev:		The device pointer
  * @ds:			The pointer to the dsa core structure
  * @bus:		The bus used for the device and built-in PHY
+ * @regmap:		The regmap instance representing all switch registers
  * @rstc:		The pointer to reset control used by MCM
  * @core_pwr:		The power supplied into the core
  * @io_pwr:		The power supplied into the I/O
@@ -804,15 +748,16 @@ struct mt753x_info {
  *			registers
  * @p6_interface	Holding the current port 6 interface
  * @p5_intf_sel:	Holding the current port 5 interface select
- *
  * @irq:		IRQ number of the switch
  * @irq_domain:		IRQ domain of the switch irq_chip
  * @irq_enable:		IRQ enable bits, synced to SYS_INT_EN
+ * @create_sgmii:	Pointer to function creating SGMII PCS instance(s)
  */
 struct mt7530_priv {
 	struct device		*dev;
 	struct dsa_switch	*ds;
 	struct mii_bus		*bus;
+	struct regmap		*regmap;
 	struct reset_control	*rstc;
 	struct regulator	*core_pwr;
 	struct regulator	*io_pwr;
@@ -825,7 +770,6 @@ struct mt7530_priv {
 	unsigned int		p5_intf_sel;
 	u8			mirror_rx;
 	u8			mirror_tx;
-
 	struct mt7530_port	ports[MT7530_NUM_PORTS];
 	struct mt753x_pcs	pcs[MT7530_NUM_PORTS];
 	/* protect among processes for registers access*/
@@ -833,6 +777,7 @@ struct mt7530_priv {
 	int irq;
 	struct irq_domain *irq_domain;
 	u32 irq_enable;
+	int (*create_sgmii)(struct mt7530_priv *priv, bool dual_sgmii);
 };
 
 struct mt7530_hw_vlan_entry {
@@ -869,4 +814,10 @@ static inline void INIT_MT7530_DUMMY_POLL(struct mt7530_dummy_poll *p,
 	p->reg = reg;
 }
 
+int mt7530_probe_common(struct mt7530_priv *priv);
+void mt7530_remove_common(struct mt7530_priv *priv);
+
+extern const struct dsa_switch_ops mt7530_switch_ops;
+extern const struct mt753x_info mt753x_table[];
+
 #endif /* __MT7530_H */
diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
index 7108f745fbf0..dc263cea205f 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -611,10 +611,10 @@ static void mv88e6185_phylink_get_caps(struct mv88e6xxx_chip *chip, int port,
 }
 
 static const u8 mv88e6xxx_phy_interface_modes[] = {
-	[MV88E6XXX_PORT_STS_CMODE_MII_PHY]	= PHY_INTERFACE_MODE_MII,
+	[MV88E6XXX_PORT_STS_CMODE_MII_PHY]	= PHY_INTERFACE_MODE_REVMII,
 	[MV88E6XXX_PORT_STS_CMODE_MII]		= PHY_INTERFACE_MODE_MII,
 	[MV88E6XXX_PORT_STS_CMODE_GMII]		= PHY_INTERFACE_MODE_GMII,
-	[MV88E6XXX_PORT_STS_CMODE_RMII_PHY]	= PHY_INTERFACE_MODE_RMII,
+	[MV88E6XXX_PORT_STS_CMODE_RMII_PHY]	= PHY_INTERFACE_MODE_REVRMII,
 	[MV88E6XXX_PORT_STS_CMODE_RMII]		= PHY_INTERFACE_MODE_RMII,
 	[MV88E6XXX_PORT_STS_CMODE_100BASEX]	= PHY_INTERFACE_MODE_100BASEX,
 	[MV88E6XXX_PORT_STS_CMODE_1000BASEX]	= PHY_INTERFACE_MODE_1000BASEX,
@@ -3685,185 +3685,6 @@ static int mv88e6390_setup_errata(struct mv88e6xxx_chip *chip)
 	return mv88e6xxx_software_reset(chip);
 }
 
-static void mv88e6xxx_teardown(struct dsa_switch *ds)
-{
-	mv88e6xxx_teardown_devlink_params(ds);
-	dsa_devlink_resources_unregister(ds);
-	mv88e6xxx_teardown_devlink_regions_global(ds);
-}
-
-static int mv88e6xxx_setup(struct dsa_switch *ds)
-{
-	struct mv88e6xxx_chip *chip = ds->priv;
-	u8 cmode;
-	int err;
-	int i;
-
-	chip->ds = ds;
-	ds->slave_mii_bus = mv88e6xxx_default_mdio_bus(chip);
-
-	/* Since virtual bridges are mapped in the PVT, the number we support
-	 * depends on the physical switch topology. We need to let DSA figure
-	 * that out and therefore we cannot set this at dsa_register_switch()
-	 * time.
-	 */
-	if (mv88e6xxx_has_pvt(chip))
-		ds->max_num_bridges = MV88E6XXX_MAX_PVT_SWITCHES -
-				      ds->dst->last_switch - 1;
-
-	mv88e6xxx_reg_lock(chip);
-
-	if (chip->info->ops->setup_errata) {
-		err = chip->info->ops->setup_errata(chip);
-		if (err)
-			goto unlock;
-	}
-
-	/* Cache the cmode of each port. */
-	for (i = 0; i < mv88e6xxx_num_ports(chip); i++) {
-		if (chip->info->ops->port_get_cmode) {
-			err = chip->info->ops->port_get_cmode(chip, i, &cmode);
-			if (err)
-				goto unlock;
-
-			chip->ports[i].cmode = cmode;
-		}
-	}
-
-	err = mv88e6xxx_vtu_setup(chip);
-	if (err)
-		goto unlock;
-
-	/* Must be called after mv88e6xxx_vtu_setup (which flushes the
-	 * VTU, thereby also flushing the STU).
-	 */
-	err = mv88e6xxx_stu_setup(chip);
-	if (err)
-		goto unlock;
-
-	/* Setup Switch Port Registers */
-	for (i = 0; i < mv88e6xxx_num_ports(chip); i++) {
-		if (dsa_is_unused_port(ds, i))
-			continue;
-
-		/* Prevent the use of an invalid port. */
-		if (mv88e6xxx_is_invalid_port(chip, i)) {
-			dev_err(chip->dev, "port %d is invalid\n", i);
-			err = -EINVAL;
-			goto unlock;
-		}
-
-		err = mv88e6xxx_setup_port(chip, i);
-		if (err)
-			goto unlock;
-	}
-
-	err = mv88e6xxx_irl_setup(chip);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_mac_setup(chip);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_phy_setup(chip);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_pvt_setup(chip);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_atu_setup(chip);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_broadcast_setup(chip, 0);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_pot_setup(chip);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_rmu_setup(chip);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_rsvd2cpu_setup(chip);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_trunk_setup(chip);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_devmap_setup(chip);
-	if (err)
-		goto unlock;
-
-	err = mv88e6xxx_pri_setup(chip);
-	if (err)
-		goto unlock;
-
-	/* Setup PTP Hardware Clock and timestamping */
-	if (chip->info->ptp_support) {
-		err = mv88e6xxx_ptp_setup(chip);
-		if (err)
-			goto unlock;
-
-		err = mv88e6xxx_hwtstamp_setup(chip);
-		if (err)
-			goto unlock;
-	}
-
-	err = mv88e6xxx_stats_setup(chip);
-	if (err)
-		goto unlock;
-
-unlock:
-	mv88e6xxx_reg_unlock(chip);
-
-	if (err)
-		return err;
-
-	/* Have to be called without holding the register lock, since
-	 * they take the devlink lock, and we later take the locks in
-	 * the reverse order when getting/setting parameters or
-	 * resource occupancy.
-	 */
-	err = mv88e6xxx_setup_devlink_resources(ds);
-	if (err)
-		return err;
-
-	err = mv88e6xxx_setup_devlink_params(ds);
-	if (err)
-		goto out_resources;
-
-	err = mv88e6xxx_setup_devlink_regions_global(ds);
-	if (err)
-		goto out_params;
-
-	return 0;
-
-out_params:
-	mv88e6xxx_teardown_devlink_params(ds);
-out_resources:
-	dsa_devlink_resources_unregister(ds);
-
-	return err;
-}
-
-static int mv88e6xxx_port_setup(struct dsa_switch *ds, int port)
-{
-	return mv88e6xxx_setup_devlink_regions_port(ds, port);
-}
-
-static void mv88e6xxx_port_teardown(struct dsa_switch *ds, int port)
-{
-	mv88e6xxx_teardown_devlink_regions_port(ds, port);
-}
-
 /* prod_id for switch families which do not have a PHY model number */
 static const u16 family_prod_id_table[] = {
 	[MV88E6XXX_FAMILY_6341] = MV88E6XXX_PORT_SWITCH_ID_PROD_6341,
@@ -3989,6 +3810,9 @@ static int mv88e6xxx_mdio_register(struct mv88e6xxx_chip *chip,
 	bus->read_c45 = mv88e6xxx_mdio_read_c45;
 	bus->write_c45 = mv88e6xxx_mdio_write_c45;
 	bus->parent = chip->dev;
+	bus->phy_mask = ~GENMASK(chip->info->phy_base_addr +
+				 mv88e6xxx_num_ports(chip) - 1,
+				 chip->info->phy_base_addr);
 
 	if (!external) {
 		err = mv88e6xxx_g2_irq_mdio_setup(chip, bus);
@@ -4032,9 +3856,9 @@ static void mv88e6xxx_mdios_unregister(struct mv88e6xxx_chip *chip)
 	}
 }
 
-static int mv88e6xxx_mdios_register(struct mv88e6xxx_chip *chip,
-				    struct device_node *np)
+static int mv88e6xxx_mdios_register(struct mv88e6xxx_chip *chip)
 {
+	struct device_node *np = chip->dev->of_node;
 	struct device_node *child;
 	int err;
 
@@ -4067,6 +3891,194 @@ static int mv88e6xxx_mdios_register(struct mv88e6xxx_chip *chip,
 	return 0;
 }
 
+static void mv88e6xxx_teardown(struct dsa_switch *ds)
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+
+	mv88e6xxx_teardown_devlink_params(ds);
+	dsa_devlink_resources_unregister(ds);
+	mv88e6xxx_teardown_devlink_regions_global(ds);
+	mv88e6xxx_mdios_unregister(chip);
+}
+
+static int mv88e6xxx_setup(struct dsa_switch *ds)
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+	u8 cmode;
+	int err;
+	int i;
+
+	err = mv88e6xxx_mdios_register(chip);
+	if (err)
+		return err;
+
+	chip->ds = ds;
+	ds->slave_mii_bus = mv88e6xxx_default_mdio_bus(chip);
+
+	/* Since virtual bridges are mapped in the PVT, the number we support
+	 * depends on the physical switch topology. We need to let DSA figure
+	 * that out and therefore we cannot set this at dsa_register_switch()
+	 * time.
+	 */
+	if (mv88e6xxx_has_pvt(chip))
+		ds->max_num_bridges = MV88E6XXX_MAX_PVT_SWITCHES -
+				      ds->dst->last_switch - 1;
+
+	mv88e6xxx_reg_lock(chip);
+
+	if (chip->info->ops->setup_errata) {
+		err = chip->info->ops->setup_errata(chip);
+		if (err)
+			goto unlock;
+	}
+
+	/* Cache the cmode of each port. */
+	for (i = 0; i < mv88e6xxx_num_ports(chip); i++) {
+		if (chip->info->ops->port_get_cmode) {
+			err = chip->info->ops->port_get_cmode(chip, i, &cmode);
+			if (err)
+				goto unlock;
+
+			chip->ports[i].cmode = cmode;
+		}
+	}
+
+	err = mv88e6xxx_vtu_setup(chip);
+	if (err)
+		goto unlock;
+
+	/* Must be called after mv88e6xxx_vtu_setup (which flushes the
+	 * VTU, thereby also flushing the STU).
+	 */
+	err = mv88e6xxx_stu_setup(chip);
+	if (err)
+		goto unlock;
+
+	/* Setup Switch Port Registers */
+	for (i = 0; i < mv88e6xxx_num_ports(chip); i++) {
+		if (dsa_is_unused_port(ds, i))
+			continue;
+
+		/* Prevent the use of an invalid port. */
+		if (mv88e6xxx_is_invalid_port(chip, i)) {
+			dev_err(chip->dev, "port %d is invalid\n", i);
+			err = -EINVAL;
+			goto unlock;
+		}
+
+		err = mv88e6xxx_setup_port(chip, i);
+		if (err)
+			goto unlock;
+	}
+
+	err = mv88e6xxx_irl_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_mac_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_phy_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_pvt_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_atu_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_broadcast_setup(chip, 0);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_pot_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_rmu_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_rsvd2cpu_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_trunk_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_devmap_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_pri_setup(chip);
+	if (err)
+		goto unlock;
+
+	/* Setup PTP Hardware Clock and timestamping */
+	if (chip->info->ptp_support) {
+		err = mv88e6xxx_ptp_setup(chip);
+		if (err)
+			goto unlock;
+
+		err = mv88e6xxx_hwtstamp_setup(chip);
+		if (err)
+			goto unlock;
+	}
+
+	err = mv88e6xxx_stats_setup(chip);
+	if (err)
+		goto unlock;
+
+unlock:
+	mv88e6xxx_reg_unlock(chip);
+
+	if (err)
+		goto out_mdios;
+
+	/* Have to be called without holding the register lock, since
+	 * they take the devlink lock, and we later take the locks in
+	 * the reverse order when getting/setting parameters or
+	 * resource occupancy.
+	 */
+	err = mv88e6xxx_setup_devlink_resources(ds);
+	if (err)
+		goto out_mdios;
+
+	err = mv88e6xxx_setup_devlink_params(ds);
+	if (err)
+		goto out_resources;
+
+	err = mv88e6xxx_setup_devlink_regions_global(ds);
+	if (err)
+		goto out_params;
+
+	return 0;
+
+out_params:
+	mv88e6xxx_teardown_devlink_params(ds);
+out_resources:
+	dsa_devlink_resources_unregister(ds);
+out_mdios:
+	mv88e6xxx_mdios_unregister(chip);
+
+	return err;
+}
+
+static int mv88e6xxx_port_setup(struct dsa_switch *ds, int port)
+{
+	return mv88e6xxx_setup_devlink_regions_port(ds, port);
+}
+
+static void mv88e6xxx_port_teardown(struct dsa_switch *ds, int port)
+{
+	mv88e6xxx_teardown_devlink_regions_port(ds, port);
+}
+
 static int mv88e6xxx_get_eeprom_len(struct dsa_switch *ds)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
@@ -7233,18 +7245,12 @@ static int mv88e6xxx_probe(struct mdio_device *mdiodev)
 	if (err)
 		goto out_g1_atu_prob_irq;
 
-	err = mv88e6xxx_mdios_register(chip, np);
-	if (err)
-		goto out_g1_vtu_prob_irq;
-
 	err = mv88e6xxx_register_switch(chip);
 	if (err)
-		goto out_mdio;
+		goto out_g1_vtu_prob_irq;
 
 	return 0;
 
-out_mdio:
-	mv88e6xxx_mdios_unregister(chip);
 out_g1_vtu_prob_irq:
 	mv88e6xxx_g1_vtu_prob_irq_free(chip);
 out_g1_atu_prob_irq:
@@ -7281,7 +7287,6 @@ static void mv88e6xxx_remove(struct mdio_device *mdiodev)
 
 	mv88e6xxx_phy_destroy(chip);
 	mv88e6xxx_unregister_switch(chip);
-	mv88e6xxx_mdios_unregister(chip);
 
 	mv88e6xxx_g1_vtu_prob_irq_free(chip);
 	mv88e6xxx_g1_atu_prob_irq_free(chip);
diff --git a/drivers/net/dsa/mv88e6xxx/global2.c b/drivers/net/dsa/mv88e6xxx/global2.c
index a7af3cebae97..615896893076 100644
--- a/drivers/net/dsa/mv88e6xxx/global2.c
+++ b/drivers/net/dsa/mv88e6xxx/global2.c
@@ -1196,31 +1196,19 @@ out:
 int mv88e6xxx_g2_irq_mdio_setup(struct mv88e6xxx_chip *chip,
 				struct mii_bus *bus)
 {
-	int phy, irq, err, err_phy;
+	int phy, irq;
 
 	for (phy = 0; phy < chip->info->num_internal_phys; phy++) {
 		irq = irq_find_mapping(chip->g2_irq.domain, phy);
-		if (irq < 0) {
-			err = irq;
-			goto out;
-		}
+		if (irq < 0)
+			return irq;
+
 		bus->irq[chip->info->phy_base_addr + phy] = irq;
 	}
 	return 0;
-out:
-	err_phy = phy;
-
-	for (phy = 0; phy < err_phy; phy++)
-		irq_dispose_mapping(bus->irq[phy]);
-
-	return err;
 }
 
 void mv88e6xxx_g2_irq_mdio_free(struct mv88e6xxx_chip *chip,
 				struct mii_bus *bus)
 {
-	int phy;
-
-	for (phy = 0; phy < chip->info->num_internal_phys; phy++)
-		irq_dispose_mapping(bus->irq[phy]);
 }
diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c
index d4cc9e60f369..80861ac090ae 100644
--- a/drivers/net/dsa/ocelot/felix.c
+++ b/drivers/net/dsa/ocelot/felix.c
@@ -1056,6 +1056,17 @@ static void felix_phylink_get_caps(struct dsa_switch *ds, int port,
 		  config->supported_interfaces);
 }
 
+static void felix_phylink_mac_config(struct dsa_switch *ds, int port,
+				     unsigned int mode,
+				     const struct phylink_link_state *state)
+{
+	struct ocelot *ocelot = ds->priv;
+	struct felix *felix = ocelot_to_felix(ocelot);
+
+	if (felix->info->phylink_mac_config)
+		felix->info->phylink_mac_config(ocelot, port, mode, state);
+}
+
 static struct phylink_pcs *felix_phylink_mac_select_pcs(struct dsa_switch *ds,
 							int port,
 							phy_interface_t iface)
@@ -1539,11 +1550,6 @@ static int felix_connect_tag_protocol(struct dsa_switch *ds,
 	}
 }
 
-/* Hardware initialization done here so that we can allocate structures with
- * devm without fear of dsa_register_switch returning -EPROBE_DEFER and causing
- * us to allocate structures twice (leak memory) and map PCI memory twice
- * (which will not work).
- */
 static int felix_setup(struct dsa_switch *ds)
 {
 	struct ocelot *ocelot = ds->priv;
@@ -1555,6 +1561,9 @@ static int felix_setup(struct dsa_switch *ds)
 	if (err)
 		return err;
 
+	if (ocelot->targets[HSIO])
+		ocelot_pll5_init(ocelot);
+
 	err = ocelot_init(ocelot);
 	if (err)
 		goto out_mdiobus_free;
@@ -1571,6 +1580,10 @@ static int felix_setup(struct dsa_switch *ds)
 	dsa_switch_for_each_available_port(dp, ds) {
 		ocelot_init_port(ocelot, dp->index);
 
+		if (felix->info->configure_serdes)
+			felix->info->configure_serdes(ocelot, dp->index,
+						      dp->dn);
+
 		/* Set the default QoS Classification based on PCP and DEI
 		 * bits of vlan tag.
 		 */
@@ -2085,6 +2098,7 @@ const struct dsa_switch_ops felix_switch_ops = {
 	.get_sset_count			= felix_get_sset_count,
 	.get_ts_info			= felix_get_ts_info,
 	.phylink_get_caps		= felix_phylink_get_caps,
+	.phylink_mac_config		= felix_phylink_mac_config,
 	.phylink_mac_select_pcs		= felix_phylink_mac_select_pcs,
 	.phylink_mac_link_down		= felix_phylink_mac_link_down,
 	.phylink_mac_link_up		= felix_phylink_mac_link_up,
diff --git a/drivers/net/dsa/ocelot/felix.h b/drivers/net/dsa/ocelot/felix.h
index d5d0b30c0b75..96008c046da5 100644
--- a/drivers/net/dsa/ocelot/felix.h
+++ b/drivers/net/dsa/ocelot/felix.h
@@ -15,6 +15,8 @@
 #define OCELOT_PORT_MODE_USXGMII	BIT(4)
 #define OCELOT_PORT_MODE_1000BASEX	BIT(5)
 
+struct device_node;
+
 /* Platform-specific information */
 struct felix_info {
 	/* Hardcoded resources provided by the hardware instantiation. */
@@ -58,6 +60,11 @@ struct felix_info {
 	void	(*tas_guard_bands_update)(struct ocelot *ocelot, int port);
 	void	(*port_sched_speed_set)(struct ocelot *ocelot, int port,
 					u32 speed);
+	void	(*phylink_mac_config)(struct ocelot *ocelot, int port,
+				      unsigned int mode,
+				      const struct phylink_link_state *state);
+	int	(*configure_serdes)(struct ocelot *ocelot, int port,
+				    struct device_node *portnp);
 };
 
 /* Methods for initializing the hardware resources specific to a tagging
diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c
index dddb28984bdf..cfb3faeaa5bf 100644
--- a/drivers/net/dsa/ocelot/felix_vsc9959.c
+++ b/drivers/net/dsa/ocelot/felix_vsc9959.c
@@ -1424,6 +1424,7 @@ static int vsc9959_qos_port_tas_set(struct ocelot *ocelot, int port,
 	mutex_lock(&ocelot->tas_lock);
 
 	if (!taprio->enable) {
+		ocelot_port_mqprio(ocelot, port, &taprio->mqprio);
 		ocelot_rmw_rix(ocelot, 0, QSYS_TAG_CONFIG_ENABLE,
 			       QSYS_TAG_CONFIG, port);
 
@@ -1436,15 +1437,19 @@ static int vsc9959_qos_port_tas_set(struct ocelot *ocelot, int port,
 		return 0;
 	}
 
+	ret = ocelot_port_mqprio(ocelot, port, &taprio->mqprio);
+	if (ret)
+		goto err_unlock;
+
 	if (taprio->cycle_time > NSEC_PER_SEC ||
 	    taprio->cycle_time_extension >= NSEC_PER_SEC) {
 		ret = -EINVAL;
-		goto err;
+		goto err_reset_tc;
 	}
 
 	if (taprio->num_entries > VSC9959_TAS_GCL_ENTRY_MAX) {
 		ret = -ERANGE;
-		goto err;
+		goto err_reset_tc;
 	}
 
 	/* Enable guard band. The switch will schedule frames without taking
@@ -1468,7 +1473,7 @@ static int vsc9959_qos_port_tas_set(struct ocelot *ocelot, int port,
 	val = ocelot_read(ocelot, QSYS_PARAM_STATUS_REG_8);
 	if (val & QSYS_PARAM_STATUS_REG_8_CONFIG_PENDING) {
 		ret = -EBUSY;
-		goto err;
+		goto err_reset_tc;
 	}
 
 	ocelot_rmw_rix(ocelot,
@@ -1503,12 +1508,19 @@ static int vsc9959_qos_port_tas_set(struct ocelot *ocelot, int port,
 				 !(val & QSYS_TAS_PARAM_CFG_CTRL_CONFIG_CHANGE),
 				 10, 100000);
 	if (ret)
-		goto err;
+		goto err_reset_tc;
 
 	ocelot_port->taprio = taprio_offload_get(taprio);
 	vsc9959_tas_guard_bands_update(ocelot, port);
 
-err:
+	mutex_unlock(&ocelot->tas_lock);
+
+	return 0;
+
+err_reset_tc:
+	taprio->mqprio.qopt.num_tc = 0;
+	ocelot_port_mqprio(ocelot, port, &taprio->mqprio);
+err_unlock:
 	mutex_unlock(&ocelot->tas_lock);
 
 	return ret;
@@ -1612,6 +1624,13 @@ static int vsc9959_qos_port_cbs_set(struct dsa_switch *ds, int port,
 static int vsc9959_qos_query_caps(struct tc_query_caps_base *base)
 {
 	switch (base->type) {
+	case TC_SETUP_QDISC_MQPRIO: {
+		struct tc_mqprio_caps *caps = base->caps;
+
+		caps->validate_queue_counts = true;
+
+		return 0;
+	}
 	case TC_SETUP_QDISC_TAPRIO: {
 		struct tc_taprio_caps *caps = base->caps;
 
@@ -1635,6 +1654,8 @@ static int vsc9959_port_setup_tc(struct dsa_switch *ds, int port,
 		return vsc9959_qos_query_caps(type_data);
 	case TC_SETUP_QDISC_TAPRIO:
 		return vsc9959_qos_port_tas_set(ocelot, port, type_data);
+	case TC_SETUP_QDISC_MQPRIO:
+		return ocelot_port_mqprio(ocelot, port, type_data);
 	case TC_SETUP_QDISC_CBS:
 		return vsc9959_qos_port_cbs_set(ds, port, type_data);
 	default:
@@ -2498,6 +2519,7 @@ static void vsc9959_cut_through_fwd(struct ocelot *ocelot)
 
 	for (port = 0; port < ocelot->num_phys_ports; port++) {
 		struct ocelot_port *ocelot_port = ocelot->ports[port];
+		struct ocelot_mm_state *mm = &ocelot->mm[port];
 		int min_speed = ocelot_port->speed;
 		unsigned long mask = 0;
 		u32 tmp, val = 0;
@@ -2538,10 +2560,12 @@ static void vsc9959_cut_through_fwd(struct ocelot *ocelot)
 
 		/* Enable cut-through forwarding for all traffic classes that
 		 * don't have oversized dropping enabled, since this check is
-		 * bypassed in cut-through mode.
+		 * bypassed in cut-through mode. Also exclude preemptible
+		 * traffic classes, since these would hang the port for some
+		 * reason, if sent as cut-through.
 		 */
 		if (ocelot_port->speed == min_speed) {
-			val = GENMASK(7, 0);
+			val = GENMASK(7, 0) & ~mm->active_preemptible_tcs;
 
 			for (tc = 0; tc < OCELOT_NUM_TC; tc++)
 				if (vsc9959_port_qmaxsdu_get(ocelot, port, tc))
@@ -2610,12 +2634,9 @@ static const struct felix_info felix_info_vsc9959 = {
 static irqreturn_t felix_irq_handler(int irq, void *data)
 {
 	struct ocelot *ocelot = (struct ocelot *)data;
-	int port;
 
 	ocelot_get_txtstamp(ocelot);
-
-	for (port = 0; port < ocelot->num_phys_ports; port++)
-		ocelot_port_mm_irq(ocelot, port);
+	ocelot_mm_irq(ocelot);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/net/dsa/ocelot/ocelot_ext.c b/drivers/net/dsa/ocelot/ocelot_ext.c
index 063150659816..c29bee5a5c48 100644
--- a/drivers/net/dsa/ocelot/ocelot_ext.c
+++ b/drivers/net/dsa/ocelot/ocelot_ext.c
@@ -20,13 +20,13 @@ static const u32 vsc7512_port_modes[VSC7514_NUM_PORTS] = {
 	OCELOT_PORT_MODE_INTERNAL,
 	OCELOT_PORT_MODE_INTERNAL,
 	OCELOT_PORT_MODE_INTERNAL,
-	OCELOT_PORT_MODE_NONE,
-	OCELOT_PORT_MODE_NONE,
-	OCELOT_PORT_MODE_NONE,
-	OCELOT_PORT_MODE_NONE,
-	OCELOT_PORT_MODE_NONE,
-	OCELOT_PORT_MODE_NONE,
-	OCELOT_PORT_MODE_NONE,
+	OCELOT_PORT_MODE_SERDES,
+	OCELOT_PORT_MODE_SERDES,
+	OCELOT_PORT_MODE_SERDES,
+	OCELOT_PORT_MODE_SERDES,
+	OCELOT_PORT_MODE_SERDES,
+	OCELOT_PORT_MODE_SGMII,
+	OCELOT_PORT_MODE_SERDES,
 };
 
 static const struct ocelot_ops ocelot_ext_ops = {
@@ -59,6 +59,8 @@ static const struct felix_info vsc7512_info = {
 	.num_ports			= VSC7514_NUM_PORTS,
 	.num_tx_queues			= OCELOT_NUM_TC,
 	.port_modes			= vsc7512_port_modes,
+	.phylink_mac_config		= ocelot_phylink_mac_config,
+	.configure_serdes		= ocelot_port_configure_serdes,
 };
 
 static int ocelot_ext_probe(struct platform_device *pdev)
@@ -149,7 +151,7 @@ MODULE_DEVICE_TABLE(of, ocelot_ext_switch_of_match);
 static struct platform_driver ocelot_ext_switch_driver = {
 	.driver = {
 		.name = "ocelot-ext-switch",
-		.of_match_table = of_match_ptr(ocelot_ext_switch_of_match),
+		.of_match_table = ocelot_ext_switch_of_match,
 	},
 	.probe = ocelot_ext_probe,
 	.remove = ocelot_ext_remove,
diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c
index 563ad338da25..96d4972a62f0 100644
--- a/drivers/net/dsa/ocelot/seville_vsc9953.c
+++ b/drivers/net/dsa/ocelot/seville_vsc9953.c
@@ -1079,7 +1079,7 @@ static struct platform_driver seville_vsc9953_driver = {
 	.shutdown	= seville_shutdown,
 	.driver = {
 		.name		= "mscc_seville",
-		.of_match_table	= of_match_ptr(seville_of_match),
+		.of_match_table	= seville_of_match,
 	},
 };
 module_platform_driver(seville_vsc9953_driver);
diff --git a/drivers/net/dsa/qca/Kconfig b/drivers/net/dsa/qca/Kconfig
index ba339747362c..4347b42c50fd 100644
--- a/drivers/net/dsa/qca/Kconfig
+++ b/drivers/net/dsa/qca/Kconfig
@@ -15,3 +15,11 @@ config NET_DSA_QCA8K
 	help
 	  This enables support for the Qualcomm Atheros QCA8K Ethernet
 	  switch chips.
+
+config NET_DSA_QCA8K_LEDS_SUPPORT
+	bool "Qualcomm Atheros QCA8K Ethernet switch family LEDs support"
+	depends on NET_DSA_QCA8K
+	depends on LEDS_CLASS=y || LEDS_CLASS=NET_DSA_QCA8K
+	help
+	  This enabled support for LEDs present on the Qualcomm Atheros
+	  QCA8K Ethernet switch chips.
diff --git a/drivers/net/dsa/qca/Makefile b/drivers/net/dsa/qca/Makefile
index 701f1d199e93..ce66b1984e5f 100644
--- a/drivers/net/dsa/qca/Makefile
+++ b/drivers/net/dsa/qca/Makefile
@@ -2,3 +2,6 @@
 obj-$(CONFIG_NET_DSA_AR9331)	+= ar9331.o
 obj-$(CONFIG_NET_DSA_QCA8K)	+= qca8k.o
 qca8k-y 			+= qca8k-common.o qca8k-8xxx.o
+ifdef CONFIG_NET_DSA_QCA8K_LEDS_SUPPORT
+qca8k-y				+= qca8k-leds.o
+endif
diff --git a/drivers/net/dsa/qca/qca8k-8xxx.c b/drivers/net/dsa/qca/qca8k-8xxx.c
index 55df4479ea30..6d5ac7588a69 100644
--- a/drivers/net/dsa/qca/qca8k-8xxx.c
+++ b/drivers/net/dsa/qca/qca8k-8xxx.c
@@ -22,6 +22,7 @@
 #include <linux/dsa/tag_qca.h>
 
 #include "qca8k.h"
+#include "qca8k_leds.h"
 
 static void
 qca8k_split_addr(u32 regaddr, u16 *r1, u16 *r2, u16 *page)
@@ -772,21 +773,6 @@ err_clear_skb:
 	return ret;
 }
 
-static u32
-qca8k_port_to_phy(int port)
-{
-	/* From Andrew Lunn:
-	 * Port 0 has no internal phy.
-	 * Port 1 has an internal PHY at MDIO address 0.
-	 * Port 2 has an internal PHY at MDIO address 1.
-	 * ...
-	 * Port 5 has an internal PHY at MDIO address 4.
-	 * Port 6 has no internal PHY.
-	 */
-
-	return port - 1;
-}
-
 static int
 qca8k_mdio_busy_wait(struct mii_bus *bus, u32 reg, u32 mask)
 {
@@ -1483,7 +1469,6 @@ static void qca8k_pcs_get_state(struct phylink_pcs *pcs,
 
 	state->link = !!(reg & QCA8K_PORT_STATUS_LINK_UP);
 	state->an_complete = state->link;
-	state->an_enabled = !!(reg & QCA8K_PORT_STATUS_LINK_AUTO);
 	state->duplex = (reg & QCA8K_PORT_STATUS_DUPLEX) ? DUPLEX_FULL :
 							   DUPLEX_HALF;
 
@@ -1798,6 +1783,10 @@ qca8k_setup(struct dsa_switch *ds)
 	if (ret)
 		return ret;
 
+	ret = qca8k_setup_led_ctrl(priv);
+	if (ret)
+		return ret;
+
 	qca8k_setup_pcs(priv, &priv->pcs_port_0, 0);
 	qca8k_setup_pcs(priv, &priv->pcs_port_6, 6);
 
diff --git a/drivers/net/dsa/qca/qca8k-leds.c b/drivers/net/dsa/qca/qca8k-leds.c
new file mode 100644
index 000000000000..b883692b7d86
--- /dev/null
+++ b/drivers/net/dsa/qca/qca8k-leds.c
@@ -0,0 +1,277 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/regmap.h>
+#include <net/dsa.h>
+
+#include "qca8k.h"
+#include "qca8k_leds.h"
+
+static int
+qca8k_get_enable_led_reg(int port_num, int led_num, struct qca8k_led_pattern_en *reg_info)
+{
+	switch (port_num) {
+	case 0:
+		reg_info->reg = QCA8K_LED_CTRL_REG(led_num);
+		reg_info->shift = QCA8K_LED_PHY0123_CONTROL_RULE_SHIFT;
+		break;
+	case 1:
+	case 2:
+	case 3:
+		/* Port 123 are controlled on a different reg */
+		reg_info->reg = QCA8K_LED_CTRL3_REG;
+		reg_info->shift = QCA8K_LED_PHY123_PATTERN_EN_SHIFT(port_num, led_num);
+		break;
+	case 4:
+		reg_info->reg = QCA8K_LED_CTRL_REG(led_num);
+		reg_info->shift = QCA8K_LED_PHY4_CONTROL_RULE_SHIFT;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+qca8k_led_brightness_set(struct qca8k_led *led,
+			 enum led_brightness brightness)
+{
+	struct qca8k_led_pattern_en reg_info;
+	struct qca8k_priv *priv = led->priv;
+	u32 mask, val;
+
+	qca8k_get_enable_led_reg(led->port_num, led->led_num, &reg_info);
+
+	val = QCA8K_LED_ALWAYS_OFF;
+	if (brightness)
+		val = QCA8K_LED_ALWAYS_ON;
+
+	/* HW regs to control brightness is special and port 1-2-3
+	 * are placed in a different reg.
+	 *
+	 * To control port 0 brightness:
+	 * - the 2 bit (15, 14) of:
+	 *   - QCA8K_LED_CTRL0_REG for led1
+	 *   - QCA8K_LED_CTRL1_REG for led2
+	 *   - QCA8K_LED_CTRL2_REG for led3
+	 *
+	 * To control port 4:
+	 * - the 2 bit (31, 30) of:
+	 *   - QCA8K_LED_CTRL0_REG for led1
+	 *   - QCA8K_LED_CTRL1_REG for led2
+	 *   - QCA8K_LED_CTRL2_REG for led3
+	 *
+	 * To control port 1:
+	 *   - the 2 bit at (9, 8) of QCA8K_LED_CTRL3_REG are used for led1
+	 *   - the 2 bit at (11, 10) of QCA8K_LED_CTRL3_REG are used for led2
+	 *   - the 2 bit at (13, 12) of QCA8K_LED_CTRL3_REG are used for led3
+	 *
+	 * To control port 2:
+	 *   - the 2 bit at (15, 14) of QCA8K_LED_CTRL3_REG are used for led1
+	 *   - the 2 bit at (17, 16) of QCA8K_LED_CTRL3_REG are used for led2
+	 *   - the 2 bit at (19, 18) of QCA8K_LED_CTRL3_REG are used for led3
+	 *
+	 * To control port 3:
+	 *   - the 2 bit at (21, 20) of QCA8K_LED_CTRL3_REG are used for led1
+	 *   - the 2 bit at (23, 22) of QCA8K_LED_CTRL3_REG are used for led2
+	 *   - the 2 bit at (25, 24) of QCA8K_LED_CTRL3_REG are used for led3
+	 *
+	 * To abstract this and have less code, we use the port and led numm
+	 * to calculate the shift and the correct reg due to this problem of
+	 * not having a 1:1 map of LED with the regs.
+	 */
+	if (led->port_num == 0 || led->port_num == 4) {
+		mask = QCA8K_LED_PATTERN_EN_MASK;
+		val <<= QCA8K_LED_PATTERN_EN_SHIFT;
+	} else {
+		mask = QCA8K_LED_PHY123_PATTERN_EN_MASK;
+	}
+
+	return regmap_update_bits(priv->regmap, reg_info.reg,
+				  mask << reg_info.shift,
+				  val << reg_info.shift);
+}
+
+static int
+qca8k_cled_brightness_set_blocking(struct led_classdev *ldev,
+				   enum led_brightness brightness)
+{
+	struct qca8k_led *led = container_of(ldev, struct qca8k_led, cdev);
+
+	return qca8k_led_brightness_set(led, brightness);
+}
+
+static enum led_brightness
+qca8k_led_brightness_get(struct qca8k_led *led)
+{
+	struct qca8k_led_pattern_en reg_info;
+	struct qca8k_priv *priv = led->priv;
+	u32 val;
+	int ret;
+
+	qca8k_get_enable_led_reg(led->port_num, led->led_num, &reg_info);
+
+	ret = regmap_read(priv->regmap, reg_info.reg, &val);
+	if (ret)
+		return 0;
+
+	val >>= reg_info.shift;
+
+	if (led->port_num == 0 || led->port_num == 4) {
+		val &= QCA8K_LED_PATTERN_EN_MASK;
+		val >>= QCA8K_LED_PATTERN_EN_SHIFT;
+	} else {
+		val &= QCA8K_LED_PHY123_PATTERN_EN_MASK;
+	}
+
+	/* Assume brightness ON only when the LED is set to always ON */
+	return val == QCA8K_LED_ALWAYS_ON;
+}
+
+static int
+qca8k_cled_blink_set(struct led_classdev *ldev,
+		     unsigned long *delay_on,
+		     unsigned long *delay_off)
+{
+	struct qca8k_led *led = container_of(ldev, struct qca8k_led, cdev);
+	u32 mask, val = QCA8K_LED_ALWAYS_BLINK_4HZ;
+	struct qca8k_led_pattern_en reg_info;
+	struct qca8k_priv *priv = led->priv;
+
+	if (*delay_on == 0 && *delay_off == 0) {
+		*delay_on = 125;
+		*delay_off = 125;
+	}
+
+	if (*delay_on != 125 || *delay_off != 125) {
+		/* The hardware only supports blinking at 4Hz. Fall back
+		 * to software implementation in other cases.
+		 */
+		return -EINVAL;
+	}
+
+	qca8k_get_enable_led_reg(led->port_num, led->led_num, &reg_info);
+
+	if (led->port_num == 0 || led->port_num == 4) {
+		mask = QCA8K_LED_PATTERN_EN_MASK;
+		val <<= QCA8K_LED_PATTERN_EN_SHIFT;
+	} else {
+		mask = QCA8K_LED_PHY123_PATTERN_EN_MASK;
+	}
+
+	regmap_update_bits(priv->regmap, reg_info.reg, mask << reg_info.shift,
+			   val << reg_info.shift);
+
+	return 0;
+}
+
+static int
+qca8k_parse_port_leds(struct qca8k_priv *priv, struct fwnode_handle *port, int port_num)
+{
+	struct fwnode_handle *led = NULL, *leds = NULL;
+	struct led_init_data init_data = { };
+	struct dsa_switch *ds = priv->ds;
+	enum led_default_state state;
+	struct qca8k_led *port_led;
+	int led_num, led_index;
+	int ret;
+
+	leds = fwnode_get_named_child_node(port, "leds");
+	if (!leds) {
+		dev_dbg(priv->dev, "No Leds node specified in device tree for port %d!\n",
+			port_num);
+		return 0;
+	}
+
+	fwnode_for_each_child_node(leds, led) {
+		/* Reg represent the led number of the port.
+		 * Each port can have at most 3 leds attached
+		 * Commonly:
+		 * 1. is gigabit led
+		 * 2. is mbit led
+		 * 3. additional status led
+		 */
+		if (fwnode_property_read_u32(led, "reg", &led_num))
+			continue;
+
+		if (led_num >= QCA8K_LED_PORT_COUNT) {
+			dev_warn(priv->dev, "Invalid LED reg %d defined for port %d",
+				 led_num, port_num);
+			continue;
+		}
+
+		led_index = QCA8K_LED_PORT_INDEX(port_num, led_num);
+
+		port_led = &priv->ports_led[led_index];
+		port_led->port_num = port_num;
+		port_led->led_num = led_num;
+		port_led->priv = priv;
+
+		state = led_init_default_state_get(led);
+		switch (state) {
+		case LEDS_DEFSTATE_ON:
+			port_led->cdev.brightness = 1;
+			qca8k_led_brightness_set(port_led, 1);
+			break;
+		case LEDS_DEFSTATE_KEEP:
+			port_led->cdev.brightness =
+					qca8k_led_brightness_get(port_led);
+			break;
+		default:
+			port_led->cdev.brightness = 0;
+			qca8k_led_brightness_set(port_led, 0);
+		}
+
+		port_led->cdev.max_brightness = 1;
+		port_led->cdev.brightness_set_blocking = qca8k_cled_brightness_set_blocking;
+		port_led->cdev.blink_set = qca8k_cled_blink_set;
+		init_data.default_label = ":port";
+		init_data.fwnode = led;
+		init_data.devname_mandatory = true;
+		init_data.devicename = kasprintf(GFP_KERNEL, "%s:0%d", ds->slave_mii_bus->id,
+						 port_num);
+		if (!init_data.devicename)
+			return -ENOMEM;
+
+		ret = devm_led_classdev_register_ext(priv->dev, &port_led->cdev, &init_data);
+		if (ret)
+			dev_warn(priv->dev, "Failed to init LED %d for port %d", led_num, port_num);
+
+		kfree(init_data.devicename);
+	}
+
+	return 0;
+}
+
+int
+qca8k_setup_led_ctrl(struct qca8k_priv *priv)
+{
+	struct fwnode_handle *ports, *port;
+	int port_num;
+	int ret;
+
+	ports = device_get_named_child_node(priv->dev, "ports");
+	if (!ports) {
+		dev_info(priv->dev, "No ports node specified in device tree!");
+		return 0;
+	}
+
+	fwnode_for_each_child_node(ports, port) {
+		if (fwnode_property_read_u32(port, "reg", &port_num))
+			continue;
+
+		/* Skip checking for CPU port 0 and CPU port 6 as not supported */
+		if (port_num == 0 || port_num == 6)
+			continue;
+
+		/* Each port can have at most 3 different leds attached.
+		 * Switch port starts from 0 to 6, but port 0 and 6 are CPU
+		 * port. The port index needs to be decreased by one to identify
+		 * the correct port for LED setup.
+		 */
+		ret = qca8k_parse_port_leds(priv, port, qca8k_port_to_phy(port_num));
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/dsa/qca/qca8k.h b/drivers/net/dsa/qca/qca8k.h
index 7996975d29d3..c5cc8a172d65 100644
--- a/drivers/net/dsa/qca/qca8k.h
+++ b/drivers/net/dsa/qca/qca8k.h
@@ -11,6 +11,7 @@
 #include <linux/delay.h>
 #include <linux/regmap.h>
 #include <linux/gpio.h>
+#include <linux/leds.h>
 #include <linux/dsa/tag_qca.h>
 
 #define QCA8K_ETHERNET_MDIO_PRIORITY			7
@@ -85,6 +86,51 @@
 #define   QCA8K_MDIO_MASTER_DATA(x)			FIELD_PREP(QCA8K_MDIO_MASTER_DATA_MASK, x)
 #define   QCA8K_MDIO_MASTER_MAX_PORTS			5
 #define   QCA8K_MDIO_MASTER_MAX_REG			32
+
+/* LED control register */
+#define QCA8K_LED_PORT_COUNT				3
+#define QCA8K_LED_COUNT					((QCA8K_NUM_PORTS - QCA8K_NUM_CPU_PORTS) * QCA8K_LED_PORT_COUNT)
+#define QCA8K_LED_RULE_COUNT				6
+#define QCA8K_LED_RULE_MAX				11
+#define QCA8K_LED_PORT_INDEX(_phy, _led)		(((_phy) * QCA8K_LED_PORT_COUNT) + (_led))
+
+#define QCA8K_LED_PHY123_PATTERN_EN_SHIFT(_phy, _led)	((((_phy) - 1) * 6) + 8 + (2 * (_led)))
+#define QCA8K_LED_PHY123_PATTERN_EN_MASK		GENMASK(1, 0)
+
+#define QCA8K_LED_PHY0123_CONTROL_RULE_SHIFT		0
+#define QCA8K_LED_PHY4_CONTROL_RULE_SHIFT		16
+
+#define QCA8K_LED_CTRL_REG(_i)				(0x050 + (_i) * 4)
+#define QCA8K_LED_CTRL0_REG				0x50
+#define QCA8K_LED_CTRL1_REG				0x54
+#define QCA8K_LED_CTRL2_REG				0x58
+#define QCA8K_LED_CTRL3_REG				0x5C
+#define   QCA8K_LED_CTRL_SHIFT(_i)			(((_i) % 2) * 16)
+#define   QCA8K_LED_CTRL_MASK				GENMASK(15, 0)
+#define QCA8K_LED_RULE_MASK				GENMASK(13, 0)
+#define QCA8K_LED_BLINK_FREQ_MASK			GENMASK(1, 0)
+#define QCA8K_LED_BLINK_FREQ_SHITF			0
+#define   QCA8K_LED_BLINK_2HZ				0
+#define   QCA8K_LED_BLINK_4HZ				1
+#define   QCA8K_LED_BLINK_8HZ				2
+#define   QCA8K_LED_BLINK_AUTO				3
+#define QCA8K_LED_LINKUP_OVER_MASK			BIT(2)
+#define QCA8K_LED_TX_BLINK_MASK				BIT(4)
+#define QCA8K_LED_RX_BLINK_MASK				BIT(5)
+#define QCA8K_LED_COL_BLINK_MASK			BIT(7)
+#define QCA8K_LED_LINK_10M_EN_MASK			BIT(8)
+#define QCA8K_LED_LINK_100M_EN_MASK			BIT(9)
+#define QCA8K_LED_LINK_1000M_EN_MASK			BIT(10)
+#define QCA8K_LED_POWER_ON_LIGHT_MASK			BIT(11)
+#define QCA8K_LED_HALF_DUPLEX_MASK			BIT(12)
+#define QCA8K_LED_FULL_DUPLEX_MASK			BIT(13)
+#define QCA8K_LED_PATTERN_EN_MASK			GENMASK(15, 14)
+#define QCA8K_LED_PATTERN_EN_SHIFT			14
+#define   QCA8K_LED_ALWAYS_OFF				0
+#define   QCA8K_LED_ALWAYS_BLINK_4HZ			1
+#define   QCA8K_LED_ALWAYS_ON				2
+#define   QCA8K_LED_RULE_CONTROLLED			3
+
 #define QCA8K_GOL_MAC_ADDR0				0x60
 #define QCA8K_GOL_MAC_ADDR1				0x64
 #define QCA8K_MAX_FRAME_SIZE				0x78
@@ -382,6 +428,19 @@ struct qca8k_pcs {
 	int port;
 };
 
+struct qca8k_led_pattern_en {
+	u32 reg;
+	u8 shift;
+};
+
+struct qca8k_led {
+	u8 port_num;
+	u8 led_num;
+	u16 old_rule;
+	struct qca8k_priv *priv;
+	struct led_classdev cdev;
+};
+
 struct qca8k_priv {
 	u8 switch_id;
 	u8 switch_revision;
@@ -406,6 +465,7 @@ struct qca8k_priv {
 	struct qca8k_pcs pcs_port_0;
 	struct qca8k_pcs pcs_port_6;
 	const struct qca8k_match_data *info;
+	struct qca8k_led ports_led[QCA8K_LED_COUNT];
 };
 
 struct qca8k_mib_desc {
@@ -421,6 +481,20 @@ struct qca8k_fdb {
 	u8 mac[6];
 };
 
+static inline u32 qca8k_port_to_phy(int port)
+{
+	/* From Andrew Lunn:
+	 * Port 0 has no internal phy.
+	 * Port 1 has an internal PHY at MDIO address 0.
+	 * Port 2 has an internal PHY at MDIO address 1.
+	 * ...
+	 * Port 5 has an internal PHY at MDIO address 4.
+	 * Port 6 has no internal PHY.
+	 */
+
+	return port - 1;
+}
+
 /* Common setup function */
 extern const struct qca8k_mib_desc ar8327_mib[];
 extern const struct regmap_access_table qca8k_readable_table;
diff --git a/drivers/net/dsa/qca/qca8k_leds.h b/drivers/net/dsa/qca/qca8k_leds.h
new file mode 100644
index 000000000000..ab367f05b173
--- /dev/null
+++ b/drivers/net/dsa/qca/qca8k_leds.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef __QCA8K_LEDS_H
+#define __QCA8K_LEDS_H
+
+/* Leds Support function */
+#ifdef CONFIG_NET_DSA_QCA8K_LEDS_SUPPORT
+int qca8k_setup_led_ctrl(struct qca8k_priv *priv);
+#else
+static inline int qca8k_setup_led_ctrl(struct qca8k_priv *priv)
+{
+	return 0;
+}
+#endif
+
+#endif /* __QCA8K_LEDS_H */
diff --git a/drivers/net/dsa/realtek/rtl8365mb.c b/drivers/net/dsa/realtek/rtl8365mb.c
index da31d8b839ac..41ea3b5a42b1 100644
--- a/drivers/net/dsa/realtek/rtl8365mb.c
+++ b/drivers/net/dsa/realtek/rtl8365mb.c
@@ -98,6 +98,7 @@
 #include <linux/of_irq.h>
 #include <linux/regmap.h>
 #include <linux/if_bridge.h>
+#include <linux/if_vlan.h>
 
 #include "realtek.h"
 
@@ -267,6 +268,7 @@
 /* Maximum packet length register */
 #define RTL8365MB_CFG0_MAX_LEN_REG	0x088C
 #define   RTL8365MB_CFG0_MAX_LEN_MASK	0x3FFF
+#define RTL8365MB_CFG0_MAX_LEN_MAX	0x3FFF
 
 /* Port learning limit registers */
 #define RTL8365MB_LUT_PORT_LEARN_LIMIT_BASE		0x0A20
@@ -1135,6 +1137,35 @@ static void rtl8365mb_phylink_mac_link_up(struct dsa_switch *ds, int port,
 	}
 }
 
+static int rtl8365mb_port_change_mtu(struct dsa_switch *ds, int port,
+				     int new_mtu)
+{
+	struct realtek_priv *priv = ds->priv;
+	int frame_size;
+
+	/* When a new MTU is set, DSA always sets the CPU port's MTU to the
+	 * largest MTU of the slave ports. Because the switch only has a global
+	 * RX length register, only allowing CPU port here is enough.
+	 */
+	if (!dsa_is_cpu_port(ds, port))
+		return 0;
+
+	frame_size = new_mtu + VLAN_ETH_HLEN + ETH_FCS_LEN;
+
+	dev_dbg(priv->dev, "changing mtu to %d (frame size: %d)\n",
+		new_mtu, frame_size);
+
+	return regmap_update_bits(priv->map, RTL8365MB_CFG0_MAX_LEN_REG,
+				  RTL8365MB_CFG0_MAX_LEN_MASK,
+				  FIELD_PREP(RTL8365MB_CFG0_MAX_LEN_MASK,
+					     frame_size));
+}
+
+static int rtl8365mb_port_max_mtu(struct dsa_switch *ds, int port)
+{
+	return RTL8365MB_CFG0_MAX_LEN_MAX - VLAN_ETH_HLEN - ETH_FCS_LEN;
+}
+
 static void rtl8365mb_port_stp_state_set(struct dsa_switch *ds, int port,
 					 u8 state)
 {
@@ -1980,10 +2011,7 @@ static int rtl8365mb_setup(struct dsa_switch *ds)
 		p->index = i;
 	}
 
-	/* Set maximum packet length to 1536 bytes */
-	ret = regmap_update_bits(priv->map, RTL8365MB_CFG0_MAX_LEN_REG,
-				 RTL8365MB_CFG0_MAX_LEN_MASK,
-				 FIELD_PREP(RTL8365MB_CFG0_MAX_LEN_MASK, 1536));
+	ret = rtl8365mb_port_change_mtu(ds, cpu->trap_port, ETH_DATA_LEN);
 	if (ret)
 		goto out_teardown_irq;
 
@@ -2103,6 +2131,8 @@ static const struct dsa_switch_ops rtl8365mb_switch_ops_smi = {
 	.get_eth_mac_stats = rtl8365mb_get_mac_stats,
 	.get_eth_ctrl_stats = rtl8365mb_get_ctrl_stats,
 	.get_stats64 = rtl8365mb_get_stats64,
+	.port_change_mtu = rtl8365mb_port_change_mtu,
+	.port_max_mtu = rtl8365mb_port_max_mtu,
 };
 
 static const struct dsa_switch_ops rtl8365mb_switch_ops_mdio = {
@@ -2124,6 +2154,8 @@ static const struct dsa_switch_ops rtl8365mb_switch_ops_mdio = {
 	.get_eth_mac_stats = rtl8365mb_get_mac_stats,
 	.get_eth_ctrl_stats = rtl8365mb_get_ctrl_stats,
 	.get_stats64 = rtl8365mb_get_stats64,
+	.port_change_mtu = rtl8365mb_port_change_mtu,
+	.port_max_mtu = rtl8365mb_port_max_mtu,
 };
 
 static const struct realtek_ops rtl8365mb_ops = {
diff --git a/drivers/net/ethernet/8390/axnet_cs.c b/drivers/net/ethernet/8390/axnet_cs.c
index 3aef959fc25b..78f985885547 100644
--- a/drivers/net/ethernet/8390/axnet_cs.c
+++ b/drivers/net/ethernet/8390/axnet_cs.c
@@ -650,7 +650,6 @@ static void block_input(struct net_device *dev, int count,
 {
     unsigned int nic_base = dev->base_addr;
     struct ei_device *ei_local = netdev_priv(dev);
-    int xfer_count = count;
     char *buf = skb->data;
 
     if ((netif_msg_rx_status(ei_local)) && (count != 4))
@@ -662,9 +661,7 @@ static void block_input(struct net_device *dev, int count,
     insw(nic_base + AXNET_DATAPORT,buf,count>>1);
     if (count & 0x01) {
 	buf[count-1] = inb(nic_base + AXNET_DATAPORT);
-	xfer_count++;
     }
-
 }
 
 /*====================================================================*/
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
index 1917da784191..5a274b99f299 100644
--- a/drivers/net/ethernet/Kconfig
+++ b/drivers/net/ethernet/Kconfig
@@ -84,7 +84,6 @@ source "drivers/net/ethernet/huawei/Kconfig"
 source "drivers/net/ethernet/i825xx/Kconfig"
 source "drivers/net/ethernet/ibm/Kconfig"
 source "drivers/net/ethernet/intel/Kconfig"
-source "drivers/net/ethernet/wangxun/Kconfig"
 source "drivers/net/ethernet/xscale/Kconfig"
 
 config JME
@@ -189,6 +188,7 @@ source "drivers/net/ethernet/toshiba/Kconfig"
 source "drivers/net/ethernet/tundra/Kconfig"
 source "drivers/net/ethernet/vertexcom/Kconfig"
 source "drivers/net/ethernet/via/Kconfig"
+source "drivers/net/ethernet/wangxun/Kconfig"
 source "drivers/net/ethernet/wiznet/Kconfig"
 source "drivers/net/ethernet/xilinx/Kconfig"
 source "drivers/net/ethernet/xircom/Kconfig"
diff --git a/drivers/net/ethernet/adi/adin1110.c b/drivers/net/ethernet/adi/adin1110.c
index 3f316a0f4158..f5c2d7a9abc1 100644
--- a/drivers/net/ethernet/adi/adin1110.c
+++ b/drivers/net/ethernet/adi/adin1110.c
@@ -515,7 +515,7 @@ static int adin1110_register_mdiobus(struct adin1110_priv *priv,
 		return -ENOMEM;
 
 	snprintf(priv->mii_bus_name, MII_BUS_ID_SIZE, "%s-%u",
-		 priv->cfg->name, priv->spidev->chip_select);
+		 priv->cfg->name, spi_get_chipselect(priv->spidev, 0));
 
 	mii_bus->name = priv->mii_bus_name;
 	mii_bus->read = adin1110_mdio_read;
diff --git a/drivers/net/ethernet/alteon/acenic.c b/drivers/net/ethernet/alteon/acenic.c
index d7762da8b2c0..eafef84fe3be 100644
--- a/drivers/net/ethernet/alteon/acenic.c
+++ b/drivers/net/ethernet/alteon/acenic.c
@@ -2435,7 +2435,7 @@ restart:
 	} else {
 		dma_addr_t mapping;
 		u32 vlan_tag = 0;
-		int i, len = 0;
+		int i;
 
 		mapping = ace_map_tx_skb(ap, skb, NULL, idx);
 		flagsize = (skb_headlen(skb) << 16);
@@ -2454,7 +2454,6 @@ restart:
 			const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 			struct tx_ring_info *info;
 
-			len += skb_frag_size(frag);
 			info = ap->skb->tx_skbuff + idx;
 			desc = ap->tx_ring + idx;
 
diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.h b/drivers/net/ethernet/amazon/ena/ena_eth_com.h
index 689313ee25a8..372b259279ec 100644
--- a/drivers/net/ethernet/amazon/ena/ena_eth_com.h
+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.h
@@ -10,6 +10,10 @@
 
 /* head update threshold in units of (queue size / ENA_COMP_HEAD_THRESH) */
 #define ENA_COMP_HEAD_THRESH 4
+/* we allow 2 DMA descriptors per LLQ entry */
+#define ENA_LLQ_ENTRY_DESC_CHUNK_SIZE	(2 * sizeof(struct ena_eth_io_tx_desc))
+#define ENA_LLQ_HEADER		(128UL - ENA_LLQ_ENTRY_DESC_CHUNK_SIZE)
+#define ENA_LLQ_LARGE_HEADER	(256UL - ENA_LLQ_ENTRY_DESC_CHUNK_SIZE)
 
 struct ena_com_tx_ctx {
 	struct ena_com_tx_meta ena_meta;
diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c
index 1d4f2f4d10f2..d671df4b76bc 100644
--- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c
+++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c
@@ -476,6 +476,21 @@ static void ena_get_ringparam(struct net_device *netdev,
 
 	ring->tx_max_pending = adapter->max_tx_ring_size;
 	ring->rx_max_pending = adapter->max_rx_ring_size;
+	if (adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
+		bool large_llq_supported = adapter->large_llq_header_supported;
+
+		kernel_ring->tx_push = true;
+		kernel_ring->tx_push_buf_len = adapter->ena_dev->tx_max_header_size;
+		if (large_llq_supported)
+			kernel_ring->tx_push_buf_max_len = ENA_LLQ_LARGE_HEADER;
+		else
+			kernel_ring->tx_push_buf_max_len = ENA_LLQ_HEADER;
+	} else {
+		kernel_ring->tx_push = false;
+		kernel_ring->tx_push_buf_max_len = 0;
+		kernel_ring->tx_push_buf_len = 0;
+	}
+
 	ring->tx_pending = adapter->tx_ring[0].ring_size;
 	ring->rx_pending = adapter->rx_ring[0].ring_size;
 }
@@ -486,7 +501,8 @@ static int ena_set_ringparam(struct net_device *netdev,
 			     struct netlink_ext_ack *extack)
 {
 	struct ena_adapter *adapter = netdev_priv(netdev);
-	u32 new_tx_size, new_rx_size;
+	u32 new_tx_size, new_rx_size, new_tx_push_buf_len;
+	bool changed = false;
 
 	new_tx_size = ring->tx_pending < ENA_MIN_RING_SIZE ?
 			ENA_MIN_RING_SIZE : ring->tx_pending;
@@ -496,11 +512,51 @@ static int ena_set_ringparam(struct net_device *netdev,
 			ENA_MIN_RING_SIZE : ring->rx_pending;
 	new_rx_size = rounddown_pow_of_two(new_rx_size);
 
-	if (new_tx_size == adapter->requested_tx_ring_size &&
-	    new_rx_size == adapter->requested_rx_ring_size)
+	changed |= new_tx_size != adapter->requested_tx_ring_size ||
+		   new_rx_size != adapter->requested_rx_ring_size;
+
+	/* This value is ignored if LLQ is not supported */
+	new_tx_push_buf_len = adapter->ena_dev->tx_max_header_size;
+
+	if ((adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) !=
+	    kernel_ring->tx_push) {
+		NL_SET_ERR_MSG_MOD(extack, "Push mode state cannot be modified");
+		return -EINVAL;
+	}
+
+	/* Validate that the push buffer is supported on the underlying device */
+	if (kernel_ring->tx_push_buf_len) {
+		enum ena_admin_placement_policy_type placement;
+
+		new_tx_push_buf_len = kernel_ring->tx_push_buf_len;
+
+		placement = adapter->ena_dev->tx_mem_queue_type;
+		if (placement == ENA_ADMIN_PLACEMENT_POLICY_HOST)
+			return -EOPNOTSUPP;
+
+		if (new_tx_push_buf_len != ENA_LLQ_HEADER &&
+		    new_tx_push_buf_len != ENA_LLQ_LARGE_HEADER) {
+			bool large_llq_sup = adapter->large_llq_header_supported;
+			char large_llq_size_str[40];
+
+			snprintf(large_llq_size_str, 40, ", %lu", ENA_LLQ_LARGE_HEADER);
+
+			NL_SET_ERR_MSG_FMT_MOD(extack,
+					       "Supported tx push buff values: [%lu%s]",
+					       ENA_LLQ_HEADER,
+					       large_llq_sup ? large_llq_size_str : "");
+
+			return -EINVAL;
+		}
+
+		changed |= new_tx_push_buf_len != adapter->ena_dev->tx_max_header_size;
+	}
+
+	if (!changed)
 		return 0;
 
-	return ena_update_queue_sizes(adapter, new_tx_size, new_rx_size);
+	return ena_update_queue_params(adapter, new_tx_size, new_rx_size,
+				       new_tx_push_buf_len);
 }
 
 static u32 ena_flow_hash_to_flow_type(u16 hash_fields)
@@ -909,6 +965,8 @@ static int ena_set_tunable(struct net_device *netdev,
 static const struct ethtool_ops ena_ethtool_ops = {
 	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
 				     ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
+	.supported_ring_params	= ETHTOOL_RING_USE_TX_PUSH_BUF_LEN |
+				  ETHTOOL_RING_USE_TX_PUSH,
 	.get_link_ksettings	= ena_get_link_ksettings,
 	.get_drvinfo		= ena_get_drvinfo,
 	.get_msglevel		= ena_get_msglevel,
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
index cbfe7f977270..e6a6efaeb87c 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
@@ -1898,7 +1898,6 @@ static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget)
 {
 	u32 total_done = 0;
 	u16 next_to_clean;
-	u32 tx_bytes = 0;
 	int tx_pkts = 0;
 	u16 req_id;
 	int rc;
@@ -1936,7 +1935,6 @@ static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget)
 			  "tx_poll: q %d skb %p completed\n", xdp_ring->qid,
 			  xdpf);
 
-		tx_bytes += xdpf->len;
 		tx_pkts++;
 		total_done += tx_info->tx_descs;
 
@@ -2809,11 +2807,13 @@ static int ena_close(struct net_device *netdev)
 	return 0;
 }
 
-int ena_update_queue_sizes(struct ena_adapter *adapter,
-			   u32 new_tx_size,
-			   u32 new_rx_size)
+int ena_update_queue_params(struct ena_adapter *adapter,
+			    u32 new_tx_size,
+			    u32 new_rx_size,
+			    u32 new_llq_header_len)
 {
-	bool dev_was_up;
+	bool dev_was_up, large_llq_changed = false;
+	int rc = 0;
 
 	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
 	ena_close(adapter->netdev);
@@ -2823,7 +2823,21 @@ int ena_update_queue_sizes(struct ena_adapter *adapter,
 			  0,
 			  adapter->xdp_num_queues +
 			  adapter->num_io_queues);
-	return dev_was_up ? ena_up(adapter) : 0;
+
+	large_llq_changed = adapter->ena_dev->tx_mem_queue_type ==
+			    ENA_ADMIN_PLACEMENT_POLICY_DEV;
+	large_llq_changed &=
+		new_llq_header_len != adapter->ena_dev->tx_max_header_size;
+
+	/* a check that the configuration is valid is done by caller */
+	if (large_llq_changed) {
+		adapter->large_llq_header_enabled = !adapter->large_llq_header_enabled;
+
+		ena_destroy_device(adapter, false);
+		rc = ena_restore_device(adapter);
+	}
+
+	return dev_was_up && !rc ? ena_up(adapter) : rc;
 }
 
 int ena_set_rx_copybreak(struct ena_adapter *adapter, u32 rx_copybreak)
@@ -3364,6 +3378,98 @@ static const struct net_device_ops ena_netdev_ops = {
 	.ndo_xdp_xmit		= ena_xdp_xmit,
 };
 
+static void ena_calc_io_queue_size(struct ena_adapter *adapter,
+				   struct ena_com_dev_get_features_ctx *get_feat_ctx)
+{
+	struct ena_admin_feature_llq_desc *llq = &get_feat_ctx->llq;
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	u32 tx_queue_size = ENA_DEFAULT_RING_SIZE;
+	u32 rx_queue_size = ENA_DEFAULT_RING_SIZE;
+	u32 max_tx_queue_size;
+	u32 max_rx_queue_size;
+
+	/* If this function is called after driver load, the ring sizes have already
+	 * been configured. Take it into account when recalculating ring size.
+	 */
+	if (adapter->tx_ring->ring_size)
+		tx_queue_size = adapter->tx_ring->ring_size;
+
+	if (adapter->rx_ring->ring_size)
+		rx_queue_size = adapter->rx_ring->ring_size;
+
+	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
+		struct ena_admin_queue_ext_feature_fields *max_queue_ext =
+			&get_feat_ctx->max_queue_ext.max_queue_ext;
+		max_rx_queue_size = min_t(u32, max_queue_ext->max_rx_cq_depth,
+					  max_queue_ext->max_rx_sq_depth);
+		max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
+
+		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
+			max_tx_queue_size = min_t(u32, max_tx_queue_size,
+						  llq->max_llq_depth);
+		else
+			max_tx_queue_size = min_t(u32, max_tx_queue_size,
+						  max_queue_ext->max_tx_sq_depth);
+
+		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
+						 max_queue_ext->max_per_packet_tx_descs);
+		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
+						 max_queue_ext->max_per_packet_rx_descs);
+	} else {
+		struct ena_admin_queue_feature_desc *max_queues =
+			&get_feat_ctx->max_queues;
+		max_rx_queue_size = min_t(u32, max_queues->max_cq_depth,
+					  max_queues->max_sq_depth);
+		max_tx_queue_size = max_queues->max_cq_depth;
+
+		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
+			max_tx_queue_size = min_t(u32, max_tx_queue_size,
+						  llq->max_llq_depth);
+		else
+			max_tx_queue_size = min_t(u32, max_tx_queue_size,
+						  max_queues->max_sq_depth);
+
+		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
+						 max_queues->max_packet_tx_descs);
+		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
+						 max_queues->max_packet_rx_descs);
+	}
+
+	max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size);
+	max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size);
+
+	/* When forcing large headers, we multiply the entry size by 2, and therefore divide
+	 * the queue size by 2, leaving the amount of memory used by the queues unchanged.
+	 */
+	if (adapter->large_llq_header_enabled) {
+		if ((llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B) &&
+		    ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
+			max_tx_queue_size /= 2;
+			dev_info(&adapter->pdev->dev,
+				 "Forcing large headers and decreasing maximum TX queue size to %d\n",
+				 max_tx_queue_size);
+		} else {
+			dev_err(&adapter->pdev->dev,
+				"Forcing large headers failed: LLQ is disabled or device does not support large headers\n");
+
+			adapter->large_llq_header_enabled = false;
+		}
+	}
+
+	tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
+				  max_tx_queue_size);
+	rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
+				  max_rx_queue_size);
+
+	tx_queue_size = rounddown_pow_of_two(tx_queue_size);
+	rx_queue_size = rounddown_pow_of_two(rx_queue_size);
+
+	adapter->max_tx_ring_size  = max_tx_queue_size;
+	adapter->max_rx_ring_size = max_rx_queue_size;
+	adapter->requested_tx_ring_size = tx_queue_size;
+	adapter->requested_rx_ring_size = rx_queue_size;
+}
+
 static int ena_device_validate_params(struct ena_adapter *adapter,
 				      struct ena_com_dev_get_features_ctx *get_feat_ctx)
 {
@@ -3387,13 +3493,30 @@ static int ena_device_validate_params(struct ena_adapter *adapter,
 	return 0;
 }
 
-static void set_default_llq_configurations(struct ena_llq_configurations *llq_config)
+static void set_default_llq_configurations(struct ena_adapter *adapter,
+					   struct ena_llq_configurations *llq_config,
+					   struct ena_admin_feature_llq_desc *llq)
 {
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+
 	llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
 	llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
 	llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
-	llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
-	llq_config->llq_ring_entry_size_value = 128;
+
+	adapter->large_llq_header_supported =
+		!!(ena_dev->supported_features & BIT(ENA_ADMIN_LLQ));
+	adapter->large_llq_header_supported &=
+		!!(llq->entry_size_ctrl_supported &
+			ENA_ADMIN_LIST_ENTRY_SIZE_256B);
+
+	if ((llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B) &&
+	    adapter->large_llq_header_enabled) {
+		llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
+		llq_config->llq_ring_entry_size_value = 256;
+	} else {
+		llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
+		llq_config->llq_ring_entry_size_value = 128;
+	}
 }
 
 static int ena_set_queues_placement_policy(struct pci_dev *pdev,
@@ -3412,6 +3535,13 @@ static int ena_set_queues_placement_policy(struct pci_dev *pdev,
 		return 0;
 	}
 
+	if (!ena_dev->mem_bar) {
+		netdev_err(ena_dev->net_device,
+			   "LLQ is advertised as supported but device doesn't expose mem bar\n");
+		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
+		return 0;
+	}
+
 	rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
 	if (unlikely(rc)) {
 		dev_err(&pdev->dev,
@@ -3427,15 +3557,8 @@ static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev
 {
 	bool has_mem_bar = !!(bars & BIT(ENA_MEM_BAR));
 
-	if (!has_mem_bar) {
-		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
-			dev_err(&pdev->dev,
-				"ENA device does not expose LLQ bar. Fallback to host mode policy.\n");
-			ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
-		}
-
+	if (!has_mem_bar)
 		return 0;
-	}
 
 	ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
 					   pci_resource_start(pdev, ENA_MEM_BAR),
@@ -3447,10 +3570,11 @@ static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev
 	return 0;
 }
 
-static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev,
+static int ena_device_init(struct ena_adapter *adapter, struct pci_dev *pdev,
 			   struct ena_com_dev_get_features_ctx *get_feat_ctx,
 			   bool *wd_state)
 {
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
 	struct ena_llq_configurations llq_config;
 	struct device *dev = &pdev->dev;
 	bool readless_supported;
@@ -3535,7 +3659,7 @@ static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev,
 
 	*wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
 
-	set_default_llq_configurations(&llq_config);
+	set_default_llq_configurations(adapter, &llq_config, &get_feat_ctx->llq);
 
 	rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq,
 					     &llq_config);
@@ -3544,6 +3668,8 @@ static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev,
 		goto err_admin_init;
 	}
 
+	ena_calc_io_queue_size(adapter, get_feat_ctx);
+
 	return 0;
 
 err_admin_init:
@@ -3638,17 +3764,25 @@ static int ena_restore_device(struct ena_adapter *adapter)
 	struct ena_com_dev_get_features_ctx get_feat_ctx;
 	struct ena_com_dev *ena_dev = adapter->ena_dev;
 	struct pci_dev *pdev = adapter->pdev;
+	struct ena_ring *txr;
+	int rc, count, i;
 	bool wd_state;
-	int rc;
 
 	set_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
-	rc = ena_device_init(ena_dev, adapter->pdev, &get_feat_ctx, &wd_state);
+	rc = ena_device_init(adapter, adapter->pdev, &get_feat_ctx, &wd_state);
 	if (rc) {
 		dev_err(&pdev->dev, "Can not initialize device\n");
 		goto err;
 	}
 	adapter->wd_state = wd_state;
 
+	count =  adapter->xdp_num_queues + adapter->num_io_queues;
+	for (i = 0 ; i < count; i++) {
+		txr = &adapter->tx_ring[i];
+		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
+		txr->tx_max_header_size = ena_dev->tx_max_header_size;
+	}
+
 	rc = ena_device_validate_params(adapter, &get_feat_ctx);
 	if (rc) {
 		dev_err(&pdev->dev, "Validation of device parameters failed\n");
@@ -4162,72 +4296,6 @@ static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
 	pci_release_selected_regions(pdev, release_bars);
 }
 
-
-static void ena_calc_io_queue_size(struct ena_adapter *adapter,
-				   struct ena_com_dev_get_features_ctx *get_feat_ctx)
-{
-	struct ena_admin_feature_llq_desc *llq = &get_feat_ctx->llq;
-	struct ena_com_dev *ena_dev = adapter->ena_dev;
-	u32 tx_queue_size = ENA_DEFAULT_RING_SIZE;
-	u32 rx_queue_size = ENA_DEFAULT_RING_SIZE;
-	u32 max_tx_queue_size;
-	u32 max_rx_queue_size;
-
-	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
-		struct ena_admin_queue_ext_feature_fields *max_queue_ext =
-			&get_feat_ctx->max_queue_ext.max_queue_ext;
-		max_rx_queue_size = min_t(u32, max_queue_ext->max_rx_cq_depth,
-					  max_queue_ext->max_rx_sq_depth);
-		max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
-
-		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
-			max_tx_queue_size = min_t(u32, max_tx_queue_size,
-						  llq->max_llq_depth);
-		else
-			max_tx_queue_size = min_t(u32, max_tx_queue_size,
-						  max_queue_ext->max_tx_sq_depth);
-
-		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
-						 max_queue_ext->max_per_packet_tx_descs);
-		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
-						 max_queue_ext->max_per_packet_rx_descs);
-	} else {
-		struct ena_admin_queue_feature_desc *max_queues =
-			&get_feat_ctx->max_queues;
-		max_rx_queue_size = min_t(u32, max_queues->max_cq_depth,
-					  max_queues->max_sq_depth);
-		max_tx_queue_size = max_queues->max_cq_depth;
-
-		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
-			max_tx_queue_size = min_t(u32, max_tx_queue_size,
-						  llq->max_llq_depth);
-		else
-			max_tx_queue_size = min_t(u32, max_tx_queue_size,
-						  max_queues->max_sq_depth);
-
-		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
-						 max_queues->max_packet_tx_descs);
-		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
-						 max_queues->max_packet_rx_descs);
-	}
-
-	max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size);
-	max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size);
-
-	tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
-				  max_tx_queue_size);
-	rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
-				  max_rx_queue_size);
-
-	tx_queue_size = rounddown_pow_of_two(tx_queue_size);
-	rx_queue_size = rounddown_pow_of_two(rx_queue_size);
-
-	adapter->max_tx_ring_size  = max_tx_queue_size;
-	adapter->max_rx_ring_size = max_rx_queue_size;
-	adapter->requested_tx_ring_size = tx_queue_size;
-	adapter->requested_rx_ring_size = rx_queue_size;
-}
-
 /* ena_probe - Device Initialization Routine
  * @pdev: PCI device information struct
  * @ent: entry in ena_pci_tbl
@@ -4310,7 +4378,13 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pci_set_drvdata(pdev, adapter);
 
-	rc = ena_device_init(ena_dev, pdev, &get_feat_ctx, &wd_state);
+	rc = ena_map_llq_mem_bar(pdev, ena_dev, bars);
+	if (rc) {
+		dev_err(&pdev->dev, "ENA LLQ bar mapping failed\n");
+		goto err_netdev_destroy;
+	}
+
+	rc = ena_device_init(adapter, pdev, &get_feat_ctx, &wd_state);
 	if (rc) {
 		dev_err(&pdev->dev, "ENA device init failed\n");
 		if (rc == -ETIME)
@@ -4318,12 +4392,6 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto err_netdev_destroy;
 	}
 
-	rc = ena_map_llq_mem_bar(pdev, ena_dev, bars);
-	if (rc) {
-		dev_err(&pdev->dev, "ENA llq bar mapping failed\n");
-		goto err_device_destroy;
-	}
-
 	/* Initial TX and RX interrupt delay. Assumes 1 usec granularity.
 	 * Updated during device initialization with the real granularity
 	 */
@@ -4331,7 +4399,6 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS;
 	ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
 	max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, &get_feat_ctx);
-	ena_calc_io_queue_size(adapter, &get_feat_ctx);
 	if (unlikely(!max_num_io_queues)) {
 		rc = -EFAULT;
 		goto err_device_destroy;
@@ -4364,6 +4431,7 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 			"Failed to query interrupt moderation feature\n");
 		goto err_device_destroy;
 	}
+
 	ena_init_io_rings(adapter,
 			  0,
 			  adapter->xdp_num_queues +
@@ -4488,6 +4556,7 @@ static void __ena_shutoff(struct pci_dev *pdev, bool shutdown)
 	rtnl_lock(); /* lock released inside the below if-else block */
 	adapter->reset_reason = ENA_REGS_RESET_SHUTDOWN;
 	ena_destroy_device(adapter, true);
+
 	if (shutdown) {
 		netif_device_detach(netdev);
 		dev_close(netdev);
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.h b/drivers/net/ethernet/amazon/ena/ena_netdev.h
index 2cb141079474..5a0d4ee76172 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.h
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.h
@@ -334,6 +334,14 @@ struct ena_adapter {
 
 	u32 msg_enable;
 
+	/* large_llq_header_enabled is used for two purposes:
+	 * 1. Indicates that large LLQ has been requested.
+	 * 2. Indicates whether large LLQ is set or not after device
+	 *    initialization / configuration.
+	 */
+	bool large_llq_header_enabled;
+	bool large_llq_header_supported;
+
 	u16 max_tx_sgl_size;
 	u16 max_rx_sgl_size;
 
@@ -388,9 +396,10 @@ void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf);
 
 int ena_update_hw_stats(struct ena_adapter *adapter);
 
-int ena_update_queue_sizes(struct ena_adapter *adapter,
-			   u32 new_tx_size,
-			   u32 new_rx_size);
+int ena_update_queue_params(struct ena_adapter *adapter,
+			    u32 new_tx_size,
+			    u32 new_rx_size,
+			    u32 new_llq_header_len);
 
 int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count);
 
diff --git a/drivers/net/ethernet/amd/Kconfig b/drivers/net/ethernet/amd/Kconfig
index ab42f75b9413..235fcacef5c5 100644
--- a/drivers/net/ethernet/amd/Kconfig
+++ b/drivers/net/ethernet/amd/Kconfig
@@ -186,4 +186,16 @@ config AMD_XGBE_HAVE_ECC
 	bool
 	default n
 
+config PDS_CORE
+	tristate "AMD/Pensando Data Systems Core Device Support"
+	depends on 64BIT && PCI
+	help
+	  This enables the support for the AMD/Pensando Core device family of
+	  adapters.  More specific information on this driver can be
+	  found in
+	  <file:Documentation/networking/device_drivers/ethernet/amd/pds_core.rst>.
+
+	  To compile this driver as a module, choose M here. The module
+	  will be called pds_core.
+
 endif # NET_VENDOR_AMD
diff --git a/drivers/net/ethernet/amd/Makefile b/drivers/net/ethernet/amd/Makefile
index 42742afe9115..2dcfb84731e1 100644
--- a/drivers/net/ethernet/amd/Makefile
+++ b/drivers/net/ethernet/amd/Makefile
@@ -17,3 +17,4 @@ obj-$(CONFIG_PCNET32) += pcnet32.o
 obj-$(CONFIG_SUN3LANCE) += sun3lance.o
 obj-$(CONFIG_SUNLANCE) += sunlance.o
 obj-$(CONFIG_AMD_XGBE) += xgbe/
+obj-$(CONFIG_PDS_CORE) += pds_core/
diff --git a/drivers/net/ethernet/amd/nmclan_cs.c b/drivers/net/ethernet/amd/nmclan_cs.c
index 823a329a921f..0dd391c84c13 100644
--- a/drivers/net/ethernet/amd/nmclan_cs.c
+++ b/drivers/net/ethernet/amd/nmclan_cs.c
@@ -651,7 +651,7 @@ static int nmclan_config(struct pcmcia_device *link)
     } else {
       pr_notice("mace id not found: %x %x should be 0x40 0x?9\n",
 		sig[0], sig[1]);
-      return -ENODEV;
+      goto failed;
     }
   }
 
diff --git a/drivers/net/ethernet/amd/pds_core/Makefile b/drivers/net/ethernet/amd/pds_core/Makefile
new file mode 100644
index 000000000000..0abc33ce826c
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/Makefile
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2023 Advanced Micro Devices, Inc.
+
+obj-$(CONFIG_PDS_CORE) := pds_core.o
+
+pds_core-y := main.o \
+	      devlink.o \
+	      auxbus.o \
+	      dev.o \
+	      adminq.o \
+	      core.o \
+	      fw.o
+
+pds_core-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/net/ethernet/amd/pds_core/adminq.c b/drivers/net/ethernet/amd/pds_core/adminq.c
new file mode 100644
index 000000000000..045fe133f6ee
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/adminq.c
@@ -0,0 +1,290 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/dynamic_debug.h>
+
+#include "core.h"
+
+struct pdsc_wait_context {
+	struct pdsc_qcq *qcq;
+	struct completion wait_completion;
+};
+
+static int pdsc_process_notifyq(struct pdsc_qcq *qcq)
+{
+	union pds_core_notifyq_comp *comp;
+	struct pdsc *pdsc = qcq->pdsc;
+	struct pdsc_cq *cq = &qcq->cq;
+	struct pdsc_cq_info *cq_info;
+	int nq_work = 0;
+	u64 eid;
+
+	cq_info = &cq->info[cq->tail_idx];
+	comp = cq_info->comp;
+	eid = le64_to_cpu(comp->event.eid);
+	while (eid > pdsc->last_eid) {
+		u16 ecode = le16_to_cpu(comp->event.ecode);
+
+		switch (ecode) {
+		case PDS_EVENT_LINK_CHANGE:
+			dev_info(pdsc->dev, "NotifyQ LINK_CHANGE ecode %d eid %lld\n",
+				 ecode, eid);
+			pdsc_notify(PDS_EVENT_LINK_CHANGE, comp);
+			break;
+
+		case PDS_EVENT_RESET:
+			dev_info(pdsc->dev, "NotifyQ RESET ecode %d eid %lld\n",
+				 ecode, eid);
+			pdsc_notify(PDS_EVENT_RESET, comp);
+			break;
+
+		case PDS_EVENT_XCVR:
+			dev_info(pdsc->dev, "NotifyQ XCVR ecode %d eid %lld\n",
+				 ecode, eid);
+			break;
+
+		default:
+			dev_info(pdsc->dev, "NotifyQ ecode %d eid %lld\n",
+				 ecode, eid);
+			break;
+		}
+
+		pdsc->last_eid = eid;
+		cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
+		cq_info = &cq->info[cq->tail_idx];
+		comp = cq_info->comp;
+		eid = le64_to_cpu(comp->event.eid);
+
+		nq_work++;
+	}
+
+	qcq->accum_work += nq_work;
+
+	return nq_work;
+}
+
+void pdsc_process_adminq(struct pdsc_qcq *qcq)
+{
+	union pds_core_adminq_comp *comp;
+	struct pdsc_queue *q = &qcq->q;
+	struct pdsc *pdsc = qcq->pdsc;
+	struct pdsc_cq *cq = &qcq->cq;
+	struct pdsc_q_info *q_info;
+	unsigned long irqflags;
+	int nq_work = 0;
+	int aq_work = 0;
+	int credits;
+
+	/* Don't process AdminQ when shutting down */
+	if (pdsc->state & BIT_ULL(PDSC_S_STOPPING_DRIVER)) {
+		dev_err(pdsc->dev, "%s: called while PDSC_S_STOPPING_DRIVER\n",
+			__func__);
+		return;
+	}
+
+	/* Check for NotifyQ event */
+	nq_work = pdsc_process_notifyq(&pdsc->notifyqcq);
+
+	/* Check for empty queue, which can happen if the interrupt was
+	 * for a NotifyQ event and there are no new AdminQ completions.
+	 */
+	if (q->tail_idx == q->head_idx)
+		goto credits;
+
+	/* Find the first completion to clean,
+	 * run the callback in the related q_info,
+	 * and continue while we still match done color
+	 */
+	spin_lock_irqsave(&pdsc->adminq_lock, irqflags);
+	comp = cq->info[cq->tail_idx].comp;
+	while (pdsc_color_match(comp->color, cq->done_color)) {
+		q_info = &q->info[q->tail_idx];
+		q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
+
+		/* Copy out the completion data */
+		memcpy(q_info->dest, comp, sizeof(*comp));
+
+		complete_all(&q_info->wc->wait_completion);
+
+		if (cq->tail_idx == cq->num_descs - 1)
+			cq->done_color = !cq->done_color;
+		cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
+		comp = cq->info[cq->tail_idx].comp;
+
+		aq_work++;
+	}
+	spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags);
+
+	qcq->accum_work += aq_work;
+
+credits:
+	/* Return the interrupt credits, one for each completion */
+	credits = nq_work + aq_work;
+	if (credits)
+		pds_core_intr_credits(&pdsc->intr_ctrl[qcq->intx],
+				      credits,
+				      PDS_CORE_INTR_CRED_REARM);
+}
+
+void pdsc_work_thread(struct work_struct *work)
+{
+	struct pdsc_qcq *qcq = container_of(work, struct pdsc_qcq, work);
+
+	pdsc_process_adminq(qcq);
+}
+
+irqreturn_t pdsc_adminq_isr(int irq, void *data)
+{
+	struct pdsc_qcq *qcq = data;
+	struct pdsc *pdsc = qcq->pdsc;
+
+	/* Don't process AdminQ when shutting down */
+	if (pdsc->state & BIT_ULL(PDSC_S_STOPPING_DRIVER)) {
+		dev_err(pdsc->dev, "%s: called while PDSC_S_STOPPING_DRIVER\n",
+			__func__);
+		return IRQ_HANDLED;
+	}
+
+	queue_work(pdsc->wq, &qcq->work);
+	pds_core_intr_mask(&pdsc->intr_ctrl[irq], PDS_CORE_INTR_MASK_CLEAR);
+
+	return IRQ_HANDLED;
+}
+
+static int __pdsc_adminq_post(struct pdsc *pdsc,
+			      struct pdsc_qcq *qcq,
+			      union pds_core_adminq_cmd *cmd,
+			      union pds_core_adminq_comp *comp,
+			      struct pdsc_wait_context *wc)
+{
+	struct pdsc_queue *q = &qcq->q;
+	struct pdsc_q_info *q_info;
+	unsigned long irqflags;
+	unsigned int avail;
+	int index;
+	int ret;
+
+	spin_lock_irqsave(&pdsc->adminq_lock, irqflags);
+
+	/* Check for space in the queue */
+	avail = q->tail_idx;
+	if (q->head_idx >= avail)
+		avail += q->num_descs - q->head_idx - 1;
+	else
+		avail -= q->head_idx + 1;
+	if (!avail) {
+		ret = -ENOSPC;
+		goto err_out_unlock;
+	}
+
+	/* Check that the FW is running */
+	if (!pdsc_is_fw_running(pdsc)) {
+		u8 fw_status = ioread8(&pdsc->info_regs->fw_status);
+
+		dev_info(pdsc->dev, "%s: post failed - fw not running %#02x:\n",
+			 __func__, fw_status);
+		ret = -ENXIO;
+
+		goto err_out_unlock;
+	}
+
+	/* Post the request */
+	index = q->head_idx;
+	q_info = &q->info[index];
+	q_info->wc = wc;
+	q_info->dest = comp;
+	memcpy(q_info->desc, cmd, sizeof(*cmd));
+
+	dev_dbg(pdsc->dev, "head_idx %d tail_idx %d\n",
+		q->head_idx, q->tail_idx);
+	dev_dbg(pdsc->dev, "post admin queue command:\n");
+	dynamic_hex_dump("cmd ", DUMP_PREFIX_OFFSET, 16, 1,
+			 cmd, sizeof(*cmd), true);
+
+	q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);
+
+	pds_core_dbell_ring(pdsc->kern_dbpage,
+			    q->hw_type, q->dbval | q->head_idx);
+	ret = index;
+
+err_out_unlock:
+	spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags);
+	return ret;
+}
+
+int pdsc_adminq_post(struct pdsc *pdsc,
+		     union pds_core_adminq_cmd *cmd,
+		     union pds_core_adminq_comp *comp,
+		     bool fast_poll)
+{
+	struct pdsc_wait_context wc = {
+		.wait_completion =
+			COMPLETION_INITIALIZER_ONSTACK(wc.wait_completion),
+	};
+	unsigned long poll_interval = 1;
+	unsigned long poll_jiffies;
+	unsigned long time_limit;
+	unsigned long time_start;
+	unsigned long time_done;
+	unsigned long remaining;
+	int err = 0;
+	int index;
+
+	wc.qcq = &pdsc->adminqcq;
+	index = __pdsc_adminq_post(pdsc, &pdsc->adminqcq, cmd, comp, &wc);
+	if (index < 0) {
+		err = index;
+		goto err_out;
+	}
+
+	time_start = jiffies;
+	time_limit = time_start + HZ * pdsc->devcmd_timeout;
+	do {
+		/* Timeslice the actual wait to catch IO errors etc early */
+		poll_jiffies = msecs_to_jiffies(poll_interval);
+		remaining = wait_for_completion_timeout(&wc.wait_completion,
+							poll_jiffies);
+		if (remaining)
+			break;
+
+		if (!pdsc_is_fw_running(pdsc)) {
+			u8 fw_status = ioread8(&pdsc->info_regs->fw_status);
+
+			dev_dbg(pdsc->dev, "%s: post wait failed - fw not running %#02x:\n",
+				__func__, fw_status);
+			err = -ENXIO;
+			break;
+		}
+
+		/* When fast_poll is not requested, prevent aggressive polling
+		 * on failures due to timeouts by doing exponential back off.
+		 */
+		if (!fast_poll && poll_interval < PDSC_ADMINQ_MAX_POLL_INTERVAL)
+			poll_interval <<= 1;
+	} while (time_before(jiffies, time_limit));
+	time_done = jiffies;
+	dev_dbg(pdsc->dev, "%s: elapsed %d msecs\n",
+		__func__, jiffies_to_msecs(time_done - time_start));
+
+	/* Check the results */
+	if (time_after_eq(time_done, time_limit))
+		err = -ETIMEDOUT;
+
+	dev_dbg(pdsc->dev, "read admin queue completion idx %d:\n", index);
+	dynamic_hex_dump("comp ", DUMP_PREFIX_OFFSET, 16, 1,
+			 comp, sizeof(*comp), true);
+
+	if (remaining && comp->status)
+		err = pdsc_err_to_errno(comp->status);
+
+err_out:
+	if (err) {
+		dev_dbg(pdsc->dev, "%s: opcode %d status %d err %pe\n",
+			__func__, cmd->opcode, comp->status, ERR_PTR(err));
+		if (err == -ENXIO || err == -ETIMEDOUT)
+			queue_work(pdsc->wq, &pdsc->health_work);
+	}
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(pdsc_adminq_post);
diff --git a/drivers/net/ethernet/amd/pds_core/auxbus.c b/drivers/net/ethernet/amd/pds_core/auxbus.c
new file mode 100644
index 000000000000..561af8e5b3ea
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/auxbus.c
@@ -0,0 +1,264 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/pci.h>
+
+#include "core.h"
+#include <linux/pds/pds_auxbus.h>
+
+/**
+ * pds_client_register - Link the client to the firmware
+ * @pf_pdev:	ptr to the PF driver struct
+ * @devname:	name that includes service into, e.g. pds_core.vDPA
+ *
+ * Return: 0 on success, or
+ *         negative for error
+ */
+int pds_client_register(struct pci_dev *pf_pdev, char *devname)
+{
+	union pds_core_adminq_comp comp = {};
+	union pds_core_adminq_cmd cmd = {};
+	struct pdsc *pf;
+	int err;
+	u16 ci;
+
+	pf = pci_get_drvdata(pf_pdev);
+	if (pf->state)
+		return -ENXIO;
+
+	cmd.client_reg.opcode = PDS_AQ_CMD_CLIENT_REG;
+	strscpy(cmd.client_reg.devname, devname,
+		sizeof(cmd.client_reg.devname));
+
+	err = pdsc_adminq_post(pf, &cmd, &comp, false);
+	if (err) {
+		dev_info(pf->dev, "register dev_name %s with DSC failed, status %d: %pe\n",
+			 devname, comp.status, ERR_PTR(err));
+		return err;
+	}
+
+	ci = le16_to_cpu(comp.client_reg.client_id);
+	if (!ci) {
+		dev_err(pf->dev, "%s: device returned null client_id\n",
+			__func__);
+		return -EIO;
+	}
+
+	dev_dbg(pf->dev, "%s: device returned client_id %d for %s\n",
+		__func__, ci, devname);
+
+	return ci;
+}
+EXPORT_SYMBOL_GPL(pds_client_register);
+
+/**
+ * pds_client_unregister - Unlink the client from the firmware
+ * @pf_pdev:	ptr to the PF driver struct
+ * @client_id:	id returned from pds_client_register()
+ *
+ * Return: 0 on success, or
+ *         negative for error
+ */
+int pds_client_unregister(struct pci_dev *pf_pdev, u16 client_id)
+{
+	union pds_core_adminq_comp comp = {};
+	union pds_core_adminq_cmd cmd = {};
+	struct pdsc *pf;
+	int err;
+
+	pf = pci_get_drvdata(pf_pdev);
+	if (pf->state)
+		return -ENXIO;
+
+	cmd.client_unreg.opcode = PDS_AQ_CMD_CLIENT_UNREG;
+	cmd.client_unreg.client_id = cpu_to_le16(client_id);
+
+	err = pdsc_adminq_post(pf, &cmd, &comp, false);
+	if (err)
+		dev_info(pf->dev, "unregister client_id %d failed, status %d: %pe\n",
+			 client_id, comp.status, ERR_PTR(err));
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(pds_client_unregister);
+
+/**
+ * pds_client_adminq_cmd - Process an adminq request for the client
+ * @padev:   ptr to the client device
+ * @req:     ptr to buffer with request
+ * @req_len: length of actual struct used for request
+ * @resp:    ptr to buffer where answer is to be copied
+ * @flags:   optional flags from pds_core_adminq_flags
+ *
+ * Return: 0 on success, or
+ *         negative for error
+ *
+ * Client sends pointers to request and response buffers
+ * Core copies request data into pds_core_client_request_cmd
+ * Core sets other fields as needed
+ * Core posts to AdminQ
+ * Core copies completion data into response buffer
+ */
+int pds_client_adminq_cmd(struct pds_auxiliary_dev *padev,
+			  union pds_core_adminq_cmd *req,
+			  size_t req_len,
+			  union pds_core_adminq_comp *resp,
+			  u64 flags)
+{
+	union pds_core_adminq_cmd cmd = {};
+	struct pci_dev *pf_pdev;
+	struct pdsc *pf;
+	size_t cp_len;
+	int err;
+
+	pf_pdev = pci_physfn(padev->vf_pdev);
+	pf = pci_get_drvdata(pf_pdev);
+
+	dev_dbg(pf->dev, "%s: %s opcode %d\n",
+		__func__, dev_name(&padev->aux_dev.dev), req->opcode);
+
+	if (pf->state)
+		return -ENXIO;
+
+	/* Wrap the client's request */
+	cmd.client_request.opcode = PDS_AQ_CMD_CLIENT_CMD;
+	cmd.client_request.client_id = cpu_to_le16(padev->client_id);
+	cp_len = min_t(size_t, req_len, sizeof(cmd.client_request.client_cmd));
+	memcpy(cmd.client_request.client_cmd, req, cp_len);
+
+	err = pdsc_adminq_post(pf, &cmd, resp,
+			       !!(flags & PDS_AQ_FLAG_FASTPOLL));
+	if (err && err != -EAGAIN)
+		dev_info(pf->dev, "client admin cmd failed: %pe\n",
+			 ERR_PTR(err));
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(pds_client_adminq_cmd);
+
+static void pdsc_auxbus_dev_release(struct device *dev)
+{
+	struct pds_auxiliary_dev *padev =
+		container_of(dev, struct pds_auxiliary_dev, aux_dev.dev);
+
+	kfree(padev);
+}
+
+static struct pds_auxiliary_dev *pdsc_auxbus_dev_register(struct pdsc *cf,
+							  struct pdsc *pf,
+							  u16 client_id,
+							  char *name)
+{
+	struct auxiliary_device *aux_dev;
+	struct pds_auxiliary_dev *padev;
+	int err;
+
+	padev = kzalloc(sizeof(*padev), GFP_KERNEL);
+	if (!padev)
+		return ERR_PTR(-ENOMEM);
+
+	padev->vf_pdev = cf->pdev;
+	padev->client_id = client_id;
+
+	aux_dev = &padev->aux_dev;
+	aux_dev->name = name;
+	aux_dev->id = cf->uid;
+	aux_dev->dev.parent = cf->dev;
+	aux_dev->dev.release = pdsc_auxbus_dev_release;
+
+	err = auxiliary_device_init(aux_dev);
+	if (err < 0) {
+		dev_warn(cf->dev, "auxiliary_device_init of %s failed: %pe\n",
+			 name, ERR_PTR(err));
+		goto err_out;
+	}
+
+	err = auxiliary_device_add(aux_dev);
+	if (err) {
+		dev_warn(cf->dev, "auxiliary_device_add of %s failed: %pe\n",
+			 name, ERR_PTR(err));
+		goto err_out_uninit;
+	}
+
+	return padev;
+
+err_out_uninit:
+	auxiliary_device_uninit(aux_dev);
+err_out:
+	kfree(padev);
+	return ERR_PTR(err);
+}
+
+int pdsc_auxbus_dev_del(struct pdsc *cf, struct pdsc *pf)
+{
+	struct pds_auxiliary_dev *padev;
+	int err = 0;
+
+	mutex_lock(&pf->config_lock);
+
+	padev = pf->vfs[cf->vf_id].padev;
+	if (padev) {
+		pds_client_unregister(pf->pdev, padev->client_id);
+		auxiliary_device_delete(&padev->aux_dev);
+		auxiliary_device_uninit(&padev->aux_dev);
+		padev->client_id = 0;
+	}
+	pf->vfs[cf->vf_id].padev = NULL;
+
+	mutex_unlock(&pf->config_lock);
+	return err;
+}
+
+int pdsc_auxbus_dev_add(struct pdsc *cf, struct pdsc *pf)
+{
+	struct pds_auxiliary_dev *padev;
+	enum pds_core_vif_types vt;
+	char devname[PDS_DEVNAME_LEN];
+	u16 vt_support;
+	int client_id;
+	int err = 0;
+
+	mutex_lock(&pf->config_lock);
+
+	/* We only support vDPA so far, so it is the only one to
+	 * be verified that it is available in the Core device and
+	 * enabled in the devlink param.  In the future this might
+	 * become a loop for several VIF types.
+	 */
+
+	/* Verify that the type is supported and enabled.  It is not
+	 * an error if there is no auxbus device support for this
+	 * VF, it just means something else needs to happen with it.
+	 */
+	vt = PDS_DEV_TYPE_VDPA;
+	vt_support = !!le16_to_cpu(pf->dev_ident.vif_types[vt]);
+	if (!(vt_support &&
+	      pf->viftype_status[vt].supported &&
+	      pf->viftype_status[vt].enabled))
+		goto out_unlock;
+
+	/* Need to register with FW and get the client_id before
+	 * creating the aux device so that the aux client can run
+	 * adminq commands as part its probe
+	 */
+	snprintf(devname, sizeof(devname), "%s.%s.%d",
+		 PDS_CORE_DRV_NAME, pf->viftype_status[vt].name, cf->uid);
+	client_id = pds_client_register(pf->pdev, devname);
+	if (client_id < 0) {
+		err = client_id;
+		goto out_unlock;
+	}
+
+	padev = pdsc_auxbus_dev_register(cf, pf, client_id,
+					 pf->viftype_status[vt].name);
+	if (IS_ERR(padev)) {
+		pds_client_unregister(pf->pdev, client_id);
+		err = PTR_ERR(padev);
+		goto out_unlock;
+	}
+	pf->vfs[cf->vf_id].padev = padev;
+
+out_unlock:
+	mutex_unlock(&pf->config_lock);
+	return err;
+}
diff --git a/drivers/net/ethernet/amd/pds_core/core.c b/drivers/net/ethernet/amd/pds_core/core.c
new file mode 100644
index 000000000000..483a070d96fa
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/core.c
@@ -0,0 +1,597 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/pci.h>
+#include <linux/vmalloc.h>
+
+#include "core.h"
+
+static BLOCKING_NOTIFIER_HEAD(pds_notify_chain);
+
+int pdsc_register_notify(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_register(&pds_notify_chain, nb);
+}
+EXPORT_SYMBOL_GPL(pdsc_register_notify);
+
+void pdsc_unregister_notify(struct notifier_block *nb)
+{
+	blocking_notifier_chain_unregister(&pds_notify_chain, nb);
+}
+EXPORT_SYMBOL_GPL(pdsc_unregister_notify);
+
+void pdsc_notify(unsigned long event, void *data)
+{
+	blocking_notifier_call_chain(&pds_notify_chain, event, data);
+}
+
+void pdsc_intr_free(struct pdsc *pdsc, int index)
+{
+	struct pdsc_intr_info *intr_info;
+
+	if (index >= pdsc->nintrs || index < 0) {
+		WARN(true, "bad intr index %d\n", index);
+		return;
+	}
+
+	intr_info = &pdsc->intr_info[index];
+	if (!intr_info->vector)
+		return;
+	dev_dbg(pdsc->dev, "%s: idx %d vec %d name %s\n",
+		__func__, index, intr_info->vector, intr_info->name);
+
+	pds_core_intr_mask(&pdsc->intr_ctrl[index], PDS_CORE_INTR_MASK_SET);
+	pds_core_intr_clean(&pdsc->intr_ctrl[index]);
+
+	free_irq(intr_info->vector, intr_info->data);
+
+	memset(intr_info, 0, sizeof(*intr_info));
+}
+
+int pdsc_intr_alloc(struct pdsc *pdsc, char *name,
+		    irq_handler_t handler, void *data)
+{
+	struct pdsc_intr_info *intr_info;
+	unsigned int index;
+	int err;
+
+	/* Find the first available interrupt */
+	for (index = 0; index < pdsc->nintrs; index++)
+		if (!pdsc->intr_info[index].vector)
+			break;
+	if (index >= pdsc->nintrs) {
+		dev_warn(pdsc->dev, "%s: no intr, index=%d nintrs=%d\n",
+			 __func__, index, pdsc->nintrs);
+		return -ENOSPC;
+	}
+
+	pds_core_intr_clean_flags(&pdsc->intr_ctrl[index],
+				  PDS_CORE_INTR_CRED_RESET_COALESCE);
+
+	intr_info = &pdsc->intr_info[index];
+
+	intr_info->index = index;
+	intr_info->data = data;
+	strscpy(intr_info->name, name, sizeof(intr_info->name));
+
+	/* Get the OS vector number for the interrupt */
+	err = pci_irq_vector(pdsc->pdev, index);
+	if (err < 0) {
+		dev_err(pdsc->dev, "failed to get intr vector index %d: %pe\n",
+			index, ERR_PTR(err));
+		goto err_out_free_intr;
+	}
+	intr_info->vector = err;
+
+	/* Init the device's intr mask */
+	pds_core_intr_clean(&pdsc->intr_ctrl[index]);
+	pds_core_intr_mask_assert(&pdsc->intr_ctrl[index], 1);
+	pds_core_intr_mask(&pdsc->intr_ctrl[index], PDS_CORE_INTR_MASK_SET);
+
+	/* Register the isr with a name */
+	err = request_irq(intr_info->vector, handler, 0, intr_info->name, data);
+	if (err) {
+		dev_err(pdsc->dev, "failed to get intr irq vector %d: %pe\n",
+			intr_info->vector, ERR_PTR(err));
+		goto err_out_free_intr;
+	}
+
+	return index;
+
+err_out_free_intr:
+	pdsc_intr_free(pdsc, index);
+	return err;
+}
+
+static void pdsc_qcq_intr_free(struct pdsc *pdsc, struct pdsc_qcq *qcq)
+{
+	if (!(qcq->flags & PDS_CORE_QCQ_F_INTR) ||
+	    qcq->intx == PDS_CORE_INTR_INDEX_NOT_ASSIGNED)
+		return;
+
+	pdsc_intr_free(pdsc, qcq->intx);
+	qcq->intx = PDS_CORE_INTR_INDEX_NOT_ASSIGNED;
+}
+
+static int pdsc_qcq_intr_alloc(struct pdsc *pdsc, struct pdsc_qcq *qcq)
+{
+	char name[PDSC_INTR_NAME_MAX_SZ];
+	int index;
+
+	if (!(qcq->flags & PDS_CORE_QCQ_F_INTR)) {
+		qcq->intx = PDS_CORE_INTR_INDEX_NOT_ASSIGNED;
+		return 0;
+	}
+
+	snprintf(name, sizeof(name), "%s-%d-%s",
+		 PDS_CORE_DRV_NAME, pdsc->pdev->bus->number, qcq->q.name);
+	index = pdsc_intr_alloc(pdsc, name, pdsc_adminq_isr, qcq);
+	if (index < 0)
+		return index;
+	qcq->intx = index;
+
+	return 0;
+}
+
+void pdsc_qcq_free(struct pdsc *pdsc, struct pdsc_qcq *qcq)
+{
+	struct device *dev = pdsc->dev;
+
+	if (!(qcq && qcq->pdsc))
+		return;
+
+	pdsc_debugfs_del_qcq(qcq);
+
+	pdsc_qcq_intr_free(pdsc, qcq);
+
+	if (qcq->q_base)
+		dma_free_coherent(dev, qcq->q_size,
+				  qcq->q_base, qcq->q_base_pa);
+
+	if (qcq->cq_base)
+		dma_free_coherent(dev, qcq->cq_size,
+				  qcq->cq_base, qcq->cq_base_pa);
+
+	if (qcq->cq.info)
+		vfree(qcq->cq.info);
+
+	if (qcq->q.info)
+		vfree(qcq->q.info);
+
+	memset(qcq, 0, sizeof(*qcq));
+}
+
+static void pdsc_q_map(struct pdsc_queue *q, void *base, dma_addr_t base_pa)
+{
+	struct pdsc_q_info *cur;
+	unsigned int i;
+
+	q->base = base;
+	q->base_pa = base_pa;
+
+	for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
+		cur->desc = base + (i * q->desc_size);
+}
+
+static void pdsc_cq_map(struct pdsc_cq *cq, void *base, dma_addr_t base_pa)
+{
+	struct pdsc_cq_info *cur;
+	unsigned int i;
+
+	cq->base = base;
+	cq->base_pa = base_pa;
+
+	for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++)
+		cur->comp = base + (i * cq->desc_size);
+}
+
+int pdsc_qcq_alloc(struct pdsc *pdsc, unsigned int type, unsigned int index,
+		   const char *name, unsigned int flags, unsigned int num_descs,
+		   unsigned int desc_size, unsigned int cq_desc_size,
+		   unsigned int pid, struct pdsc_qcq *qcq)
+{
+	struct device *dev = pdsc->dev;
+	void *q_base, *cq_base;
+	dma_addr_t cq_base_pa;
+	dma_addr_t q_base_pa;
+	int err;
+
+	qcq->q.info = vzalloc(num_descs * sizeof(*qcq->q.info));
+	if (!qcq->q.info) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	qcq->pdsc = pdsc;
+	qcq->flags = flags;
+	INIT_WORK(&qcq->work, pdsc_work_thread);
+
+	qcq->q.type = type;
+	qcq->q.index = index;
+	qcq->q.num_descs = num_descs;
+	qcq->q.desc_size = desc_size;
+	qcq->q.tail_idx = 0;
+	qcq->q.head_idx = 0;
+	qcq->q.pid = pid;
+	snprintf(qcq->q.name, sizeof(qcq->q.name), "%s%u", name, index);
+
+	err = pdsc_qcq_intr_alloc(pdsc, qcq);
+	if (err)
+		goto err_out_free_q_info;
+
+	qcq->cq.info = vzalloc(num_descs * sizeof(*qcq->cq.info));
+	if (!qcq->cq.info) {
+		err = -ENOMEM;
+		goto err_out_free_irq;
+	}
+
+	qcq->cq.bound_intr = &pdsc->intr_info[qcq->intx];
+	qcq->cq.num_descs = num_descs;
+	qcq->cq.desc_size = cq_desc_size;
+	qcq->cq.tail_idx = 0;
+	qcq->cq.done_color = 1;
+
+	if (flags & PDS_CORE_QCQ_F_NOTIFYQ) {
+		/* q & cq need to be contiguous in case of notifyq */
+		qcq->q_size = PDS_PAGE_SIZE +
+			      ALIGN(num_descs * desc_size, PDS_PAGE_SIZE) +
+			      ALIGN(num_descs * cq_desc_size, PDS_PAGE_SIZE);
+		qcq->q_base = dma_alloc_coherent(dev,
+						 qcq->q_size + qcq->cq_size,
+						 &qcq->q_base_pa,
+						 GFP_KERNEL);
+		if (!qcq->q_base) {
+			err = -ENOMEM;
+			goto err_out_free_cq_info;
+		}
+		q_base = PTR_ALIGN(qcq->q_base, PDS_PAGE_SIZE);
+		q_base_pa = ALIGN(qcq->q_base_pa, PDS_PAGE_SIZE);
+		pdsc_q_map(&qcq->q, q_base, q_base_pa);
+
+		cq_base = PTR_ALIGN(q_base +
+				    ALIGN(num_descs * desc_size, PDS_PAGE_SIZE),
+				    PDS_PAGE_SIZE);
+		cq_base_pa = ALIGN(qcq->q_base_pa +
+				   ALIGN(num_descs * desc_size, PDS_PAGE_SIZE),
+				   PDS_PAGE_SIZE);
+
+	} else {
+		/* q DMA descriptors */
+		qcq->q_size = PDS_PAGE_SIZE + (num_descs * desc_size);
+		qcq->q_base = dma_alloc_coherent(dev, qcq->q_size,
+						 &qcq->q_base_pa,
+						 GFP_KERNEL);
+		if (!qcq->q_base) {
+			err = -ENOMEM;
+			goto err_out_free_cq_info;
+		}
+		q_base = PTR_ALIGN(qcq->q_base, PDS_PAGE_SIZE);
+		q_base_pa = ALIGN(qcq->q_base_pa, PDS_PAGE_SIZE);
+		pdsc_q_map(&qcq->q, q_base, q_base_pa);
+
+		/* cq DMA descriptors */
+		qcq->cq_size = PDS_PAGE_SIZE + (num_descs * cq_desc_size);
+		qcq->cq_base = dma_alloc_coherent(dev, qcq->cq_size,
+						  &qcq->cq_base_pa,
+						  GFP_KERNEL);
+		if (!qcq->cq_base) {
+			err = -ENOMEM;
+			goto err_out_free_q;
+		}
+		cq_base = PTR_ALIGN(qcq->cq_base, PDS_PAGE_SIZE);
+		cq_base_pa = ALIGN(qcq->cq_base_pa, PDS_PAGE_SIZE);
+	}
+
+	pdsc_cq_map(&qcq->cq, cq_base, cq_base_pa);
+	qcq->cq.bound_q = &qcq->q;
+
+	pdsc_debugfs_add_qcq(pdsc, qcq);
+
+	return 0;
+
+err_out_free_q:
+	dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
+err_out_free_cq_info:
+	vfree(qcq->cq.info);
+err_out_free_irq:
+	pdsc_qcq_intr_free(pdsc, qcq);
+err_out_free_q_info:
+	vfree(qcq->q.info);
+	memset(qcq, 0, sizeof(*qcq));
+err_out:
+	dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
+	return err;
+}
+
+static int pdsc_core_init(struct pdsc *pdsc)
+{
+	union pds_core_dev_comp comp = {};
+	union pds_core_dev_cmd cmd = {
+		.init.opcode = PDS_CORE_CMD_INIT,
+	};
+	struct pds_core_dev_init_data_out cido;
+	struct pds_core_dev_init_data_in cidi;
+	u32 dbid_count;
+	u32 dbpage_num;
+	size_t sz;
+	int err;
+
+	cidi.adminq_q_base = cpu_to_le64(pdsc->adminqcq.q_base_pa);
+	cidi.adminq_cq_base = cpu_to_le64(pdsc->adminqcq.cq_base_pa);
+	cidi.notifyq_cq_base = cpu_to_le64(pdsc->notifyqcq.cq.base_pa);
+	cidi.flags = cpu_to_le32(PDS_CORE_QINIT_F_IRQ | PDS_CORE_QINIT_F_ENA);
+	cidi.intr_index = cpu_to_le16(pdsc->adminqcq.intx);
+	cidi.adminq_ring_size = ilog2(pdsc->adminqcq.q.num_descs);
+	cidi.notifyq_ring_size = ilog2(pdsc->notifyqcq.q.num_descs);
+
+	mutex_lock(&pdsc->devcmd_lock);
+
+	sz = min_t(size_t, sizeof(cidi), sizeof(pdsc->cmd_regs->data));
+	memcpy_toio(&pdsc->cmd_regs->data, &cidi, sz);
+
+	err = pdsc_devcmd_locked(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
+	if (!err) {
+		sz = min_t(size_t, sizeof(cido), sizeof(pdsc->cmd_regs->data));
+		memcpy_fromio(&cido, &pdsc->cmd_regs->data, sz);
+	}
+
+	mutex_unlock(&pdsc->devcmd_lock);
+	if (err) {
+		dev_err(pdsc->dev, "Device init command failed: %pe\n",
+			ERR_PTR(err));
+		return err;
+	}
+
+	pdsc->hw_index = le32_to_cpu(cido.core_hw_index);
+
+	dbid_count = le32_to_cpu(pdsc->dev_ident.ndbpgs_per_lif);
+	dbpage_num = pdsc->hw_index * dbid_count;
+	pdsc->kern_dbpage = pdsc_map_dbpage(pdsc, dbpage_num);
+	if (!pdsc->kern_dbpage) {
+		dev_err(pdsc->dev, "Cannot map dbpage, aborting\n");
+		return -ENOMEM;
+	}
+
+	pdsc->adminqcq.q.hw_type = cido.adminq_hw_type;
+	pdsc->adminqcq.q.hw_index = le32_to_cpu(cido.adminq_hw_index);
+	pdsc->adminqcq.q.dbval = PDS_CORE_DBELL_QID(pdsc->adminqcq.q.hw_index);
+
+	pdsc->notifyqcq.q.hw_type = cido.notifyq_hw_type;
+	pdsc->notifyqcq.q.hw_index = le32_to_cpu(cido.notifyq_hw_index);
+	pdsc->notifyqcq.q.dbval = PDS_CORE_DBELL_QID(pdsc->notifyqcq.q.hw_index);
+
+	pdsc->last_eid = 0;
+
+	return err;
+}
+
+static struct pdsc_viftype pdsc_viftype_defaults[] = {
+	[PDS_DEV_TYPE_VDPA] = { .name = PDS_DEV_TYPE_VDPA_STR,
+				.vif_id = PDS_DEV_TYPE_VDPA,
+				.dl_id = DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET },
+	[PDS_DEV_TYPE_MAX] = {}
+};
+
+static int pdsc_viftypes_init(struct pdsc *pdsc)
+{
+	enum pds_core_vif_types vt;
+
+	pdsc->viftype_status = kzalloc(sizeof(pdsc_viftype_defaults),
+				       GFP_KERNEL);
+	if (!pdsc->viftype_status)
+		return -ENOMEM;
+
+	for (vt = 0; vt < PDS_DEV_TYPE_MAX; vt++) {
+		bool vt_support;
+
+		if (!pdsc_viftype_defaults[vt].name)
+			continue;
+
+		/* Grab the defaults */
+		pdsc->viftype_status[vt] = pdsc_viftype_defaults[vt];
+
+		/* See what the Core device has for support */
+		vt_support = !!le16_to_cpu(pdsc->dev_ident.vif_types[vt]);
+		dev_dbg(pdsc->dev, "VIF %s is %ssupported\n",
+			pdsc->viftype_status[vt].name,
+			vt_support ? "" : "not ");
+
+		pdsc->viftype_status[vt].supported = vt_support;
+	}
+
+	return 0;
+}
+
+int pdsc_setup(struct pdsc *pdsc, bool init)
+{
+	int numdescs;
+	int err;
+
+	if (init)
+		err = pdsc_dev_init(pdsc);
+	else
+		err = pdsc_dev_reinit(pdsc);
+	if (err)
+		return err;
+
+	/* Scale the descriptor ring length based on number of CPUs and VFs */
+	numdescs = max_t(int, PDSC_ADMINQ_MIN_LENGTH, num_online_cpus());
+	numdescs += 2 * pci_sriov_get_totalvfs(pdsc->pdev);
+	numdescs = roundup_pow_of_two(numdescs);
+	err = pdsc_qcq_alloc(pdsc, PDS_CORE_QTYPE_ADMINQ, 0, "adminq",
+			     PDS_CORE_QCQ_F_CORE | PDS_CORE_QCQ_F_INTR,
+			     numdescs,
+			     sizeof(union pds_core_adminq_cmd),
+			     sizeof(union pds_core_adminq_comp),
+			     0, &pdsc->adminqcq);
+	if (err)
+		goto err_out_teardown;
+
+	err = pdsc_qcq_alloc(pdsc, PDS_CORE_QTYPE_NOTIFYQ, 0, "notifyq",
+			     PDS_CORE_QCQ_F_NOTIFYQ,
+			     PDSC_NOTIFYQ_LENGTH,
+			     sizeof(struct pds_core_notifyq_cmd),
+			     sizeof(union pds_core_notifyq_comp),
+			     0, &pdsc->notifyqcq);
+	if (err)
+		goto err_out_teardown;
+
+	/* NotifyQ rides on the AdminQ interrupt */
+	pdsc->notifyqcq.intx = pdsc->adminqcq.intx;
+
+	/* Set up the Core with the AdminQ and NotifyQ info */
+	err = pdsc_core_init(pdsc);
+	if (err)
+		goto err_out_teardown;
+
+	/* Set up the VIFs */
+	err = pdsc_viftypes_init(pdsc);
+	if (err)
+		goto err_out_teardown;
+
+	if (init)
+		pdsc_debugfs_add_viftype(pdsc);
+
+	clear_bit(PDSC_S_FW_DEAD, &pdsc->state);
+	return 0;
+
+err_out_teardown:
+	pdsc_teardown(pdsc, init);
+	return err;
+}
+
+void pdsc_teardown(struct pdsc *pdsc, bool removing)
+{
+	int i;
+
+	pdsc_devcmd_reset(pdsc);
+	pdsc_qcq_free(pdsc, &pdsc->notifyqcq);
+	pdsc_qcq_free(pdsc, &pdsc->adminqcq);
+
+	kfree(pdsc->viftype_status);
+	pdsc->viftype_status = NULL;
+
+	if (pdsc->intr_info) {
+		for (i = 0; i < pdsc->nintrs; i++)
+			pdsc_intr_free(pdsc, i);
+
+		if (removing) {
+			kfree(pdsc->intr_info);
+			pdsc->intr_info = NULL;
+		}
+	}
+
+	if (pdsc->kern_dbpage) {
+		iounmap(pdsc->kern_dbpage);
+		pdsc->kern_dbpage = NULL;
+	}
+
+	set_bit(PDSC_S_FW_DEAD, &pdsc->state);
+}
+
+int pdsc_start(struct pdsc *pdsc)
+{
+	pds_core_intr_mask(&pdsc->intr_ctrl[pdsc->adminqcq.intx],
+			   PDS_CORE_INTR_MASK_CLEAR);
+
+	return 0;
+}
+
+void pdsc_stop(struct pdsc *pdsc)
+{
+	int i;
+
+	if (!pdsc->intr_info)
+		return;
+
+	/* Mask interrupts that are in use */
+	for (i = 0; i < pdsc->nintrs; i++)
+		if (pdsc->intr_info[i].vector)
+			pds_core_intr_mask(&pdsc->intr_ctrl[i],
+					   PDS_CORE_INTR_MASK_SET);
+}
+
+static void pdsc_fw_down(struct pdsc *pdsc)
+{
+	union pds_core_notifyq_comp reset_event = {
+		.reset.ecode = cpu_to_le16(PDS_EVENT_RESET),
+		.reset.state = 0,
+	};
+
+	if (test_and_set_bit(PDSC_S_FW_DEAD, &pdsc->state)) {
+		dev_err(pdsc->dev, "%s: already happening\n", __func__);
+		return;
+	}
+
+	/* Notify clients of fw_down */
+	devlink_health_report(pdsc->fw_reporter, "FW down reported", pdsc);
+	pdsc_notify(PDS_EVENT_RESET, &reset_event);
+
+	pdsc_stop(pdsc);
+	pdsc_teardown(pdsc, PDSC_TEARDOWN_RECOVERY);
+}
+
+static void pdsc_fw_up(struct pdsc *pdsc)
+{
+	union pds_core_notifyq_comp reset_event = {
+		.reset.ecode = cpu_to_le16(PDS_EVENT_RESET),
+		.reset.state = 1,
+	};
+	int err;
+
+	if (!test_bit(PDSC_S_FW_DEAD, &pdsc->state)) {
+		dev_err(pdsc->dev, "%s: fw not dead\n", __func__);
+		return;
+	}
+
+	err = pdsc_setup(pdsc, PDSC_SETUP_RECOVERY);
+	if (err)
+		goto err_out;
+
+	err = pdsc_start(pdsc);
+	if (err)
+		goto err_out;
+
+	/* Notify clients of fw_up */
+	pdsc->fw_recoveries++;
+	devlink_health_reporter_state_update(pdsc->fw_reporter,
+					     DEVLINK_HEALTH_REPORTER_STATE_HEALTHY);
+	pdsc_notify(PDS_EVENT_RESET, &reset_event);
+
+	return;
+
+err_out:
+	pdsc_teardown(pdsc, PDSC_TEARDOWN_RECOVERY);
+}
+
+void pdsc_health_thread(struct work_struct *work)
+{
+	struct pdsc *pdsc = container_of(work, struct pdsc, health_work);
+	unsigned long mask;
+	bool healthy;
+
+	mutex_lock(&pdsc->config_lock);
+
+	/* Don't do a check when in a transition state */
+	mask = BIT_ULL(PDSC_S_INITING_DRIVER) |
+	       BIT_ULL(PDSC_S_STOPPING_DRIVER);
+	if (pdsc->state & mask)
+		goto out_unlock;
+
+	healthy = pdsc_is_fw_good(pdsc);
+	dev_dbg(pdsc->dev, "%s: health %d fw_status %#02x fw_heartbeat %d\n",
+		__func__, healthy, pdsc->fw_status, pdsc->last_hb);
+
+	if (test_bit(PDSC_S_FW_DEAD, &pdsc->state)) {
+		if (healthy)
+			pdsc_fw_up(pdsc);
+	} else {
+		if (!healthy)
+			pdsc_fw_down(pdsc);
+	}
+
+	pdsc->fw_generation = pdsc->fw_status & PDS_CORE_FW_STS_F_GENERATION;
+
+out_unlock:
+	mutex_unlock(&pdsc->config_lock);
+}
diff --git a/drivers/net/ethernet/amd/pds_core/core.h b/drivers/net/ethernet/amd/pds_core/core.h
new file mode 100644
index 000000000000..e545fafc4819
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/core.h
@@ -0,0 +1,312 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _PDSC_H_
+#define _PDSC_H_
+
+#include <linux/debugfs.h>
+#include <net/devlink.h>
+
+#include <linux/pds/pds_common.h>
+#include <linux/pds/pds_core_if.h>
+#include <linux/pds/pds_adminq.h>
+#include <linux/pds/pds_intr.h>
+
+#define PDSC_DRV_DESCRIPTION	"AMD/Pensando Core Driver"
+
+#define PDSC_WATCHDOG_SECS	5
+#define PDSC_QUEUE_NAME_MAX_SZ  32
+#define PDSC_ADMINQ_MIN_LENGTH	16	/* must be a power of two */
+#define PDSC_NOTIFYQ_LENGTH	64	/* must be a power of two */
+#define PDSC_TEARDOWN_RECOVERY	false
+#define PDSC_TEARDOWN_REMOVING	true
+#define PDSC_SETUP_RECOVERY	false
+#define PDSC_SETUP_INIT		true
+
+struct pdsc_dev_bar {
+	void __iomem *vaddr;
+	phys_addr_t bus_addr;
+	unsigned long len;
+	int res_index;
+};
+
+struct pdsc;
+
+struct pdsc_vf {
+	struct pds_auxiliary_dev *padev;
+	struct pdsc *vf;
+	u16     index;
+	__le16  vif_types[PDS_DEV_TYPE_MAX];
+};
+
+struct pdsc_devinfo {
+	u8 asic_type;
+	u8 asic_rev;
+	char fw_version[PDS_CORE_DEVINFO_FWVERS_BUFLEN + 1];
+	char serial_num[PDS_CORE_DEVINFO_SERIAL_BUFLEN + 1];
+};
+
+struct pdsc_queue {
+	struct pdsc_q_info *info;
+	u64 dbval;
+	u16 head_idx;
+	u16 tail_idx;
+	u8 hw_type;
+	unsigned int index;
+	unsigned int num_descs;
+	u64 dbell_count;
+	u64 features;
+	unsigned int type;
+	unsigned int hw_index;
+	union {
+		void *base;
+		struct pds_core_admin_cmd *adminq;
+	};
+	dma_addr_t base_pa;	/* must be page aligned */
+	unsigned int desc_size;
+	unsigned int pid;
+	char name[PDSC_QUEUE_NAME_MAX_SZ];
+};
+
+#define PDSC_INTR_NAME_MAX_SZ		32
+
+struct pdsc_intr_info {
+	char name[PDSC_INTR_NAME_MAX_SZ];
+	unsigned int index;
+	unsigned int vector;
+	void *data;
+};
+
+struct pdsc_cq_info {
+	void *comp;
+};
+
+struct pdsc_buf_info {
+	struct page *page;
+	dma_addr_t dma_addr;
+	u32 page_offset;
+	u32 len;
+};
+
+struct pdsc_q_info {
+	union {
+		void *desc;
+		struct pdsc_admin_cmd *adminq_desc;
+	};
+	unsigned int bytes;
+	unsigned int nbufs;
+	struct pdsc_buf_info bufs[PDS_CORE_MAX_FRAGS];
+	struct pdsc_wait_context *wc;
+	void *dest;
+};
+
+struct pdsc_cq {
+	struct pdsc_cq_info *info;
+	struct pdsc_queue *bound_q;
+	struct pdsc_intr_info *bound_intr;
+	u16 tail_idx;
+	bool done_color;
+	unsigned int num_descs;
+	unsigned int desc_size;
+	void *base;
+	dma_addr_t base_pa;	/* must be page aligned */
+} ____cacheline_aligned_in_smp;
+
+struct pdsc_qcq {
+	struct pdsc *pdsc;
+	void *q_base;
+	dma_addr_t q_base_pa;	/* might not be page aligned */
+	void *cq_base;
+	dma_addr_t cq_base_pa;	/* might not be page aligned */
+	u32 q_size;
+	u32 cq_size;
+	bool armed;
+	unsigned int flags;
+
+	struct work_struct work;
+	struct pdsc_queue q;
+	struct pdsc_cq cq;
+	int intx;
+
+	u32 accum_work;
+	struct dentry *dentry;
+};
+
+struct pdsc_viftype {
+	char *name;
+	bool supported;
+	bool enabled;
+	int dl_id;
+	int vif_id;
+	struct pds_auxiliary_dev *padev;
+};
+
+/* No state flags set means we are in a steady running state */
+enum pdsc_state_flags {
+	PDSC_S_FW_DEAD,		    /* stopped, wait on startup or recovery */
+	PDSC_S_INITING_DRIVER,	    /* initial startup from probe */
+	PDSC_S_STOPPING_DRIVER,	    /* driver remove */
+
+	/* leave this as last */
+	PDSC_S_STATE_SIZE
+};
+
+struct pdsc {
+	struct pci_dev *pdev;
+	struct dentry *dentry;
+	struct device *dev;
+	struct pdsc_dev_bar bars[PDS_CORE_BARS_MAX];
+	struct pdsc_vf *vfs;
+	int num_vfs;
+	int vf_id;
+	int hw_index;
+	int uid;
+
+	unsigned long state;
+	u8 fw_status;
+	u8 fw_generation;
+	unsigned long last_fw_time;
+	u32 last_hb;
+	struct timer_list wdtimer;
+	unsigned int wdtimer_period;
+	struct work_struct health_work;
+	struct devlink_health_reporter *fw_reporter;
+	u32 fw_recoveries;
+
+	struct pdsc_devinfo dev_info;
+	struct pds_core_dev_identity dev_ident;
+	unsigned int nintrs;
+	struct pdsc_intr_info *intr_info;	/* array of nintrs elements */
+
+	struct workqueue_struct *wq;
+
+	unsigned int devcmd_timeout;
+	struct mutex devcmd_lock;	/* lock for dev_cmd operations */
+	struct mutex config_lock;	/* lock for configuration operations */
+	spinlock_t adminq_lock;		/* lock for adminq operations */
+	struct pds_core_dev_info_regs __iomem *info_regs;
+	struct pds_core_dev_cmd_regs __iomem *cmd_regs;
+	struct pds_core_intr __iomem *intr_ctrl;
+	u64 __iomem *intr_status;
+	u64 __iomem *db_pages;
+	dma_addr_t phy_db_pages;
+	u64 __iomem *kern_dbpage;
+
+	struct pdsc_qcq adminqcq;
+	struct pdsc_qcq notifyqcq;
+	u64 last_eid;
+	struct pdsc_viftype *viftype_status;
+};
+
+/** enum pds_core_dbell_bits - bitwise composition of dbell values.
+ *
+ * @PDS_CORE_DBELL_QID_MASK:	unshifted mask of valid queue id bits.
+ * @PDS_CORE_DBELL_QID_SHIFT:	queue id shift amount in dbell value.
+ * @PDS_CORE_DBELL_QID:		macro to build QID component of dbell value.
+ *
+ * @PDS_CORE_DBELL_RING_MASK:	unshifted mask of valid ring bits.
+ * @PDS_CORE_DBELL_RING_SHIFT:	ring shift amount in dbell value.
+ * @PDS_CORE_DBELL_RING:	macro to build ring component of dbell value.
+ *
+ * @PDS_CORE_DBELL_RING_0:	ring zero dbell component value.
+ * @PDS_CORE_DBELL_RING_1:	ring one dbell component value.
+ * @PDS_CORE_DBELL_RING_2:	ring two dbell component value.
+ * @PDS_CORE_DBELL_RING_3:	ring three dbell component value.
+ *
+ * @PDS_CORE_DBELL_INDEX_MASK:	bit mask of valid index bits, no shift needed.
+ */
+enum pds_core_dbell_bits {
+	PDS_CORE_DBELL_QID_MASK		= 0xffffff,
+	PDS_CORE_DBELL_QID_SHIFT		= 24,
+
+#define PDS_CORE_DBELL_QID(n) \
+	(((u64)(n) & PDS_CORE_DBELL_QID_MASK) << PDS_CORE_DBELL_QID_SHIFT)
+
+	PDS_CORE_DBELL_RING_MASK		= 0x7,
+	PDS_CORE_DBELL_RING_SHIFT		= 16,
+
+#define PDS_CORE_DBELL_RING(n) \
+	(((u64)(n) & PDS_CORE_DBELL_RING_MASK) << PDS_CORE_DBELL_RING_SHIFT)
+
+	PDS_CORE_DBELL_RING_0		= 0,
+	PDS_CORE_DBELL_RING_1		= PDS_CORE_DBELL_RING(1),
+	PDS_CORE_DBELL_RING_2		= PDS_CORE_DBELL_RING(2),
+	PDS_CORE_DBELL_RING_3		= PDS_CORE_DBELL_RING(3),
+
+	PDS_CORE_DBELL_INDEX_MASK		= 0xffff,
+};
+
+static inline void pds_core_dbell_ring(u64 __iomem *db_page,
+				       enum pds_core_logical_qtype qtype,
+				       u64 val)
+{
+	writeq(val, &db_page[qtype]);
+}
+
+int pdsc_fw_reporter_diagnose(struct devlink_health_reporter *reporter,
+			      struct devlink_fmsg *fmsg,
+			      struct netlink_ext_ack *extack);
+int pdsc_dl_info_get(struct devlink *dl, struct devlink_info_req *req,
+		     struct netlink_ext_ack *extack);
+int pdsc_dl_flash_update(struct devlink *dl,
+			 struct devlink_flash_update_params *params,
+			 struct netlink_ext_ack *extack);
+int pdsc_dl_enable_get(struct devlink *dl, u32 id,
+		       struct devlink_param_gset_ctx *ctx);
+int pdsc_dl_enable_set(struct devlink *dl, u32 id,
+		       struct devlink_param_gset_ctx *ctx);
+int pdsc_dl_enable_validate(struct devlink *dl, u32 id,
+			    union devlink_param_value val,
+			    struct netlink_ext_ack *extack);
+
+void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num);
+
+void pdsc_debugfs_create(void);
+void pdsc_debugfs_destroy(void);
+void pdsc_debugfs_add_dev(struct pdsc *pdsc);
+void pdsc_debugfs_del_dev(struct pdsc *pdsc);
+void pdsc_debugfs_add_ident(struct pdsc *pdsc);
+void pdsc_debugfs_add_viftype(struct pdsc *pdsc);
+void pdsc_debugfs_add_irqs(struct pdsc *pdsc);
+void pdsc_debugfs_add_qcq(struct pdsc *pdsc, struct pdsc_qcq *qcq);
+void pdsc_debugfs_del_qcq(struct pdsc_qcq *qcq);
+
+int pdsc_err_to_errno(enum pds_core_status_code code);
+bool pdsc_is_fw_running(struct pdsc *pdsc);
+bool pdsc_is_fw_good(struct pdsc *pdsc);
+int pdsc_devcmd(struct pdsc *pdsc, union pds_core_dev_cmd *cmd,
+		union pds_core_dev_comp *comp, int max_seconds);
+int pdsc_devcmd_locked(struct pdsc *pdsc, union pds_core_dev_cmd *cmd,
+		       union pds_core_dev_comp *comp, int max_seconds);
+int pdsc_devcmd_init(struct pdsc *pdsc);
+int pdsc_devcmd_reset(struct pdsc *pdsc);
+int pdsc_dev_reinit(struct pdsc *pdsc);
+int pdsc_dev_init(struct pdsc *pdsc);
+
+int pdsc_intr_alloc(struct pdsc *pdsc, char *name,
+		    irq_handler_t handler, void *data);
+void pdsc_intr_free(struct pdsc *pdsc, int index);
+void pdsc_qcq_free(struct pdsc *pdsc, struct pdsc_qcq *qcq);
+int pdsc_qcq_alloc(struct pdsc *pdsc, unsigned int type, unsigned int index,
+		   const char *name, unsigned int flags, unsigned int num_descs,
+		   unsigned int desc_size, unsigned int cq_desc_size,
+		   unsigned int pid, struct pdsc_qcq *qcq);
+int pdsc_setup(struct pdsc *pdsc, bool init);
+void pdsc_teardown(struct pdsc *pdsc, bool removing);
+int pdsc_start(struct pdsc *pdsc);
+void pdsc_stop(struct pdsc *pdsc);
+void pdsc_health_thread(struct work_struct *work);
+
+int pdsc_register_notify(struct notifier_block *nb);
+void pdsc_unregister_notify(struct notifier_block *nb);
+void pdsc_notify(unsigned long event, void *data);
+int pdsc_auxbus_dev_add(struct pdsc *cf, struct pdsc *pf);
+int pdsc_auxbus_dev_del(struct pdsc *cf, struct pdsc *pf);
+
+void pdsc_process_adminq(struct pdsc_qcq *qcq);
+void pdsc_work_thread(struct work_struct *work);
+irqreturn_t pdsc_adminq_isr(int irq, void *data);
+
+int pdsc_firmware_update(struct pdsc *pdsc, const struct firmware *fw,
+			 struct netlink_ext_ack *extack);
+#endif /* _PDSC_H_ */
diff --git a/drivers/net/ethernet/amd/pds_core/debugfs.c b/drivers/net/ethernet/amd/pds_core/debugfs.c
new file mode 100644
index 000000000000..8ec392299b7d
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/debugfs.c
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/pci.h>
+
+#include "core.h"
+
+static struct dentry *pdsc_dir;
+
+void pdsc_debugfs_create(void)
+{
+	pdsc_dir = debugfs_create_dir(PDS_CORE_DRV_NAME, NULL);
+}
+
+void pdsc_debugfs_destroy(void)
+{
+	debugfs_remove_recursive(pdsc_dir);
+}
+
+void pdsc_debugfs_add_dev(struct pdsc *pdsc)
+{
+	pdsc->dentry = debugfs_create_dir(pci_name(pdsc->pdev), pdsc_dir);
+
+	debugfs_create_ulong("state", 0400, pdsc->dentry, &pdsc->state);
+}
+
+void pdsc_debugfs_del_dev(struct pdsc *pdsc)
+{
+	debugfs_remove_recursive(pdsc->dentry);
+	pdsc->dentry = NULL;
+}
+
+static int identity_show(struct seq_file *seq, void *v)
+{
+	struct pdsc *pdsc = seq->private;
+	struct pds_core_dev_identity *ident;
+	int vt;
+
+	ident = &pdsc->dev_ident;
+
+	seq_printf(seq, "fw_heartbeat:     0x%x\n",
+		   ioread32(&pdsc->info_regs->fw_heartbeat));
+
+	seq_printf(seq, "nlifs:            %d\n",
+		   le32_to_cpu(ident->nlifs));
+	seq_printf(seq, "nintrs:           %d\n",
+		   le32_to_cpu(ident->nintrs));
+	seq_printf(seq, "ndbpgs_per_lif:   %d\n",
+		   le32_to_cpu(ident->ndbpgs_per_lif));
+	seq_printf(seq, "intr_coal_mult:   %d\n",
+		   le32_to_cpu(ident->intr_coal_mult));
+	seq_printf(seq, "intr_coal_div:    %d\n",
+		   le32_to_cpu(ident->intr_coal_div));
+
+	seq_puts(seq, "vif_types:        ");
+	for (vt = 0; vt < PDS_DEV_TYPE_MAX; vt++)
+		seq_printf(seq, "%d ",
+			   le16_to_cpu(pdsc->dev_ident.vif_types[vt]));
+	seq_puts(seq, "\n");
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(identity);
+
+void pdsc_debugfs_add_ident(struct pdsc *pdsc)
+{
+	debugfs_create_file("identity", 0400, pdsc->dentry,
+			    pdsc, &identity_fops);
+}
+
+static int viftype_show(struct seq_file *seq, void *v)
+{
+	struct pdsc *pdsc = seq->private;
+	int vt;
+
+	for (vt = 0; vt < PDS_DEV_TYPE_MAX; vt++) {
+		if (!pdsc->viftype_status[vt].name)
+			continue;
+
+		seq_printf(seq, "%s\t%d supported %d enabled\n",
+			   pdsc->viftype_status[vt].name,
+			   pdsc->viftype_status[vt].supported,
+			   pdsc->viftype_status[vt].enabled);
+	}
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(viftype);
+
+void pdsc_debugfs_add_viftype(struct pdsc *pdsc)
+{
+	debugfs_create_file("viftypes", 0400, pdsc->dentry,
+			    pdsc, &viftype_fops);
+}
+
+static const struct debugfs_reg32 intr_ctrl_regs[] = {
+	{ .name = "coal_init", .offset = 0, },
+	{ .name = "mask", .offset = 4, },
+	{ .name = "credits", .offset = 8, },
+	{ .name = "mask_on_assert", .offset = 12, },
+	{ .name = "coal_timer", .offset = 16, },
+};
+
+void pdsc_debugfs_add_qcq(struct pdsc *pdsc, struct pdsc_qcq *qcq)
+{
+	struct dentry *qcq_dentry, *q_dentry, *cq_dentry;
+	struct dentry *intr_dentry;
+	struct debugfs_regset32 *intr_ctrl_regset;
+	struct pdsc_intr_info *intr = &pdsc->intr_info[qcq->intx];
+	struct pdsc_queue *q = &qcq->q;
+	struct pdsc_cq *cq = &qcq->cq;
+
+	qcq_dentry = debugfs_create_dir(q->name, pdsc->dentry);
+	if (IS_ERR_OR_NULL(qcq_dentry))
+		return;
+	qcq->dentry = qcq_dentry;
+
+	debugfs_create_x64("q_base_pa", 0400, qcq_dentry, &qcq->q_base_pa);
+	debugfs_create_x32("q_size", 0400, qcq_dentry, &qcq->q_size);
+	debugfs_create_x64("cq_base_pa", 0400, qcq_dentry, &qcq->cq_base_pa);
+	debugfs_create_x32("cq_size", 0400, qcq_dentry, &qcq->cq_size);
+	debugfs_create_x32("accum_work", 0400, qcq_dentry, &qcq->accum_work);
+
+	q_dentry = debugfs_create_dir("q", qcq->dentry);
+	if (IS_ERR_OR_NULL(q_dentry))
+		return;
+
+	debugfs_create_u32("index", 0400, q_dentry, &q->index);
+	debugfs_create_u32("num_descs", 0400, q_dentry, &q->num_descs);
+	debugfs_create_u32("desc_size", 0400, q_dentry, &q->desc_size);
+	debugfs_create_u32("pid", 0400, q_dentry, &q->pid);
+
+	debugfs_create_u16("tail", 0400, q_dentry, &q->tail_idx);
+	debugfs_create_u16("head", 0400, q_dentry, &q->head_idx);
+
+	cq_dentry = debugfs_create_dir("cq", qcq->dentry);
+	if (IS_ERR_OR_NULL(cq_dentry))
+		return;
+
+	debugfs_create_x64("base_pa", 0400, cq_dentry, &cq->base_pa);
+	debugfs_create_u32("num_descs", 0400, cq_dentry, &cq->num_descs);
+	debugfs_create_u32("desc_size", 0400, cq_dentry, &cq->desc_size);
+	debugfs_create_bool("done_color", 0400, cq_dentry, &cq->done_color);
+	debugfs_create_u16("tail", 0400, cq_dentry, &cq->tail_idx);
+
+	if (qcq->flags & PDS_CORE_QCQ_F_INTR) {
+		intr_dentry = debugfs_create_dir("intr", qcq->dentry);
+		if (IS_ERR_OR_NULL(intr_dentry))
+			return;
+
+		debugfs_create_u32("index", 0400, intr_dentry, &intr->index);
+		debugfs_create_u32("vector", 0400, intr_dentry, &intr->vector);
+
+		intr_ctrl_regset = kzalloc(sizeof(*intr_ctrl_regset),
+					   GFP_KERNEL);
+		if (!intr_ctrl_regset)
+			return;
+		intr_ctrl_regset->regs = intr_ctrl_regs;
+		intr_ctrl_regset->nregs = ARRAY_SIZE(intr_ctrl_regs);
+		intr_ctrl_regset->base = &pdsc->intr_ctrl[intr->index];
+
+		debugfs_create_regset32("intr_ctrl", 0400, intr_dentry,
+					intr_ctrl_regset);
+	}
+};
+
+void pdsc_debugfs_del_qcq(struct pdsc_qcq *qcq)
+{
+	debugfs_remove_recursive(qcq->dentry);
+	qcq->dentry = NULL;
+}
diff --git a/drivers/net/ethernet/amd/pds_core/dev.c b/drivers/net/ethernet/amd/pds_core/dev.c
new file mode 100644
index 000000000000..f7c597ea5daf
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/dev.c
@@ -0,0 +1,351 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/utsname.h>
+
+#include "core.h"
+
+int pdsc_err_to_errno(enum pds_core_status_code code)
+{
+	switch (code) {
+	case PDS_RC_SUCCESS:
+		return 0;
+	case PDS_RC_EVERSION:
+	case PDS_RC_EQTYPE:
+	case PDS_RC_EQID:
+	case PDS_RC_EINVAL:
+	case PDS_RC_ENOSUPP:
+		return -EINVAL;
+	case PDS_RC_EPERM:
+		return -EPERM;
+	case PDS_RC_ENOENT:
+		return -ENOENT;
+	case PDS_RC_EAGAIN:
+		return -EAGAIN;
+	case PDS_RC_ENOMEM:
+		return -ENOMEM;
+	case PDS_RC_EFAULT:
+		return -EFAULT;
+	case PDS_RC_EBUSY:
+		return -EBUSY;
+	case PDS_RC_EEXIST:
+		return -EEXIST;
+	case PDS_RC_EVFID:
+		return -ENODEV;
+	case PDS_RC_ECLIENT:
+		return -ECHILD;
+	case PDS_RC_ENOSPC:
+		return -ENOSPC;
+	case PDS_RC_ERANGE:
+		return -ERANGE;
+	case PDS_RC_BAD_ADDR:
+		return -EFAULT;
+	case PDS_RC_EOPCODE:
+	case PDS_RC_EINTR:
+	case PDS_RC_DEV_CMD:
+	case PDS_RC_ERROR:
+	case PDS_RC_ERDMA:
+	case PDS_RC_EIO:
+	default:
+		return -EIO;
+	}
+}
+
+bool pdsc_is_fw_running(struct pdsc *pdsc)
+{
+	pdsc->fw_status = ioread8(&pdsc->info_regs->fw_status);
+	pdsc->last_fw_time = jiffies;
+	pdsc->last_hb = ioread32(&pdsc->info_regs->fw_heartbeat);
+
+	/* Firmware is useful only if the running bit is set and
+	 * fw_status != 0xff (bad PCI read)
+	 */
+	return (pdsc->fw_status != 0xff) &&
+		(pdsc->fw_status & PDS_CORE_FW_STS_F_RUNNING);
+}
+
+bool pdsc_is_fw_good(struct pdsc *pdsc)
+{
+	u8 gen = pdsc->fw_status & PDS_CORE_FW_STS_F_GENERATION;
+
+	return pdsc_is_fw_running(pdsc) && gen == pdsc->fw_generation;
+}
+
+static u8 pdsc_devcmd_status(struct pdsc *pdsc)
+{
+	return ioread8(&pdsc->cmd_regs->comp.status);
+}
+
+static bool pdsc_devcmd_done(struct pdsc *pdsc)
+{
+	return ioread32(&pdsc->cmd_regs->done) & PDS_CORE_DEV_CMD_DONE;
+}
+
+static void pdsc_devcmd_dbell(struct pdsc *pdsc)
+{
+	iowrite32(0, &pdsc->cmd_regs->done);
+	iowrite32(1, &pdsc->cmd_regs->doorbell);
+}
+
+static void pdsc_devcmd_clean(struct pdsc *pdsc)
+{
+	iowrite32(0, &pdsc->cmd_regs->doorbell);
+	memset_io(&pdsc->cmd_regs->cmd, 0, sizeof(pdsc->cmd_regs->cmd));
+}
+
+static const char *pdsc_devcmd_str(int opcode)
+{
+	switch (opcode) {
+	case PDS_CORE_CMD_NOP:
+		return "PDS_CORE_CMD_NOP";
+	case PDS_CORE_CMD_IDENTIFY:
+		return "PDS_CORE_CMD_IDENTIFY";
+	case PDS_CORE_CMD_RESET:
+		return "PDS_CORE_CMD_RESET";
+	case PDS_CORE_CMD_INIT:
+		return "PDS_CORE_CMD_INIT";
+	case PDS_CORE_CMD_FW_DOWNLOAD:
+		return "PDS_CORE_CMD_FW_DOWNLOAD";
+	case PDS_CORE_CMD_FW_CONTROL:
+		return "PDS_CORE_CMD_FW_CONTROL";
+	default:
+		return "PDS_CORE_CMD_UNKNOWN";
+	}
+}
+
+static int pdsc_devcmd_wait(struct pdsc *pdsc, int max_seconds)
+{
+	struct device *dev = pdsc->dev;
+	unsigned long start_time;
+	unsigned long max_wait;
+	unsigned long duration;
+	int timeout = 0;
+	int done = 0;
+	int err = 0;
+	int status;
+	int opcode;
+
+	opcode = ioread8(&pdsc->cmd_regs->cmd.opcode);
+
+	start_time = jiffies;
+	max_wait = start_time + (max_seconds * HZ);
+
+	while (!done && !timeout) {
+		done = pdsc_devcmd_done(pdsc);
+		if (done)
+			break;
+
+		timeout = time_after(jiffies, max_wait);
+		if (timeout)
+			break;
+
+		usleep_range(100, 200);
+	}
+	duration = jiffies - start_time;
+
+	if (done && duration > HZ)
+		dev_dbg(dev, "DEVCMD %d %s after %ld secs\n",
+			opcode, pdsc_devcmd_str(opcode), duration / HZ);
+
+	if (!done || timeout) {
+		dev_err(dev, "DEVCMD %d %s timeout, done %d timeout %d max_seconds=%d\n",
+			opcode, pdsc_devcmd_str(opcode), done, timeout,
+			max_seconds);
+		err = -ETIMEDOUT;
+		pdsc_devcmd_clean(pdsc);
+	}
+
+	status = pdsc_devcmd_status(pdsc);
+	err = pdsc_err_to_errno(status);
+	if (err && err != -EAGAIN)
+		dev_err(dev, "DEVCMD %d %s failed, status=%d err %d %pe\n",
+			opcode, pdsc_devcmd_str(opcode), status, err,
+			ERR_PTR(err));
+
+	return err;
+}
+
+int pdsc_devcmd_locked(struct pdsc *pdsc, union pds_core_dev_cmd *cmd,
+		       union pds_core_dev_comp *comp, int max_seconds)
+{
+	int err;
+
+	memcpy_toio(&pdsc->cmd_regs->cmd, cmd, sizeof(*cmd));
+	pdsc_devcmd_dbell(pdsc);
+	err = pdsc_devcmd_wait(pdsc, max_seconds);
+	memcpy_fromio(comp, &pdsc->cmd_regs->comp, sizeof(*comp));
+
+	if (err == -ENXIO || err == -ETIMEDOUT)
+		queue_work(pdsc->wq, &pdsc->health_work);
+
+	return err;
+}
+
+int pdsc_devcmd(struct pdsc *pdsc, union pds_core_dev_cmd *cmd,
+		union pds_core_dev_comp *comp, int max_seconds)
+{
+	int err;
+
+	mutex_lock(&pdsc->devcmd_lock);
+	err = pdsc_devcmd_locked(pdsc, cmd, comp, max_seconds);
+	mutex_unlock(&pdsc->devcmd_lock);
+
+	return err;
+}
+
+int pdsc_devcmd_init(struct pdsc *pdsc)
+{
+	union pds_core_dev_comp comp = {};
+	union pds_core_dev_cmd cmd = {
+		.opcode = PDS_CORE_CMD_INIT,
+	};
+
+	return pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
+}
+
+int pdsc_devcmd_reset(struct pdsc *pdsc)
+{
+	union pds_core_dev_comp comp = {};
+	union pds_core_dev_cmd cmd = {
+		.reset.opcode = PDS_CORE_CMD_RESET,
+	};
+
+	return pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
+}
+
+static int pdsc_devcmd_identify_locked(struct pdsc *pdsc)
+{
+	union pds_core_dev_comp comp = {};
+	union pds_core_dev_cmd cmd = {
+		.identify.opcode = PDS_CORE_CMD_IDENTIFY,
+		.identify.ver = PDS_CORE_IDENTITY_VERSION_1,
+	};
+
+	return pdsc_devcmd_locked(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
+}
+
+static void pdsc_init_devinfo(struct pdsc *pdsc)
+{
+	pdsc->dev_info.asic_type = ioread8(&pdsc->info_regs->asic_type);
+	pdsc->dev_info.asic_rev = ioread8(&pdsc->info_regs->asic_rev);
+	pdsc->fw_generation = PDS_CORE_FW_STS_F_GENERATION &
+			      ioread8(&pdsc->info_regs->fw_status);
+
+	memcpy_fromio(pdsc->dev_info.fw_version,
+		      pdsc->info_regs->fw_version,
+		      PDS_CORE_DEVINFO_FWVERS_BUFLEN);
+	pdsc->dev_info.fw_version[PDS_CORE_DEVINFO_FWVERS_BUFLEN] = 0;
+
+	memcpy_fromio(pdsc->dev_info.serial_num,
+		      pdsc->info_regs->serial_num,
+		      PDS_CORE_DEVINFO_SERIAL_BUFLEN);
+	pdsc->dev_info.serial_num[PDS_CORE_DEVINFO_SERIAL_BUFLEN] = 0;
+
+	dev_dbg(pdsc->dev, "fw_version %s\n", pdsc->dev_info.fw_version);
+}
+
+static int pdsc_identify(struct pdsc *pdsc)
+{
+	struct pds_core_drv_identity drv = {};
+	size_t sz;
+	int err;
+
+	drv.drv_type = cpu_to_le32(PDS_DRIVER_LINUX);
+	snprintf(drv.driver_ver_str, sizeof(drv.driver_ver_str),
+		 "%s %s", PDS_CORE_DRV_NAME, utsname()->release);
+
+	/* Next let's get some info about the device
+	 * We use the devcmd_lock at this level in order to
+	 * get safe access to the cmd_regs->data before anyone
+	 * else can mess it up
+	 */
+	mutex_lock(&pdsc->devcmd_lock);
+
+	sz = min_t(size_t, sizeof(drv), sizeof(pdsc->cmd_regs->data));
+	memcpy_toio(&pdsc->cmd_regs->data, &drv, sz);
+
+	err = pdsc_devcmd_identify_locked(pdsc);
+	if (!err) {
+		sz = min_t(size_t, sizeof(pdsc->dev_ident),
+			   sizeof(pdsc->cmd_regs->data));
+		memcpy_fromio(&pdsc->dev_ident, &pdsc->cmd_regs->data, sz);
+	}
+	mutex_unlock(&pdsc->devcmd_lock);
+
+	if (err) {
+		dev_err(pdsc->dev, "Cannot identify device: %pe\n",
+			ERR_PTR(err));
+		return err;
+	}
+
+	if (isprint(pdsc->dev_info.fw_version[0]) &&
+	    isascii(pdsc->dev_info.fw_version[0]))
+		dev_info(pdsc->dev, "FW: %.*s\n",
+			 (int)(sizeof(pdsc->dev_info.fw_version) - 1),
+			 pdsc->dev_info.fw_version);
+	else
+		dev_info(pdsc->dev, "FW: (invalid string) 0x%02x 0x%02x 0x%02x 0x%02x ...\n",
+			 (u8)pdsc->dev_info.fw_version[0],
+			 (u8)pdsc->dev_info.fw_version[1],
+			 (u8)pdsc->dev_info.fw_version[2],
+			 (u8)pdsc->dev_info.fw_version[3]);
+
+	return 0;
+}
+
+int pdsc_dev_reinit(struct pdsc *pdsc)
+{
+	pdsc_init_devinfo(pdsc);
+
+	return pdsc_identify(pdsc);
+}
+
+int pdsc_dev_init(struct pdsc *pdsc)
+{
+	unsigned int nintrs;
+	int err;
+
+	/* Initial init and reset of device */
+	pdsc_init_devinfo(pdsc);
+	pdsc->devcmd_timeout = PDS_CORE_DEVCMD_TIMEOUT;
+
+	err = pdsc_devcmd_reset(pdsc);
+	if (err)
+		return err;
+
+	err = pdsc_identify(pdsc);
+	if (err)
+		return err;
+
+	pdsc_debugfs_add_ident(pdsc);
+
+	/* Now we can reserve interrupts */
+	nintrs = le32_to_cpu(pdsc->dev_ident.nintrs);
+	nintrs = min_t(unsigned int, num_online_cpus(), nintrs);
+
+	/* Get intr_info struct array for tracking */
+	pdsc->intr_info = kcalloc(nintrs, sizeof(*pdsc->intr_info), GFP_KERNEL);
+	if (!pdsc->intr_info) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	err = pci_alloc_irq_vectors(pdsc->pdev, nintrs, nintrs, PCI_IRQ_MSIX);
+	if (err != nintrs) {
+		dev_err(pdsc->dev, "Can't get %d intrs from OS: %pe\n",
+			nintrs, ERR_PTR(err));
+		err = -ENOSPC;
+		goto err_out;
+	}
+	pdsc->nintrs = nintrs;
+
+	return 0;
+
+err_out:
+	kfree(pdsc->intr_info);
+	pdsc->intr_info = NULL;
+
+	return err;
+}
diff --git a/drivers/net/ethernet/amd/pds_core/devlink.c b/drivers/net/ethernet/amd/pds_core/devlink.c
new file mode 100644
index 000000000000..9c6b3653c1c7
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/devlink.c
@@ -0,0 +1,183 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include "core.h"
+#include <linux/pds/pds_auxbus.h>
+
+static struct
+pdsc_viftype *pdsc_dl_find_viftype_by_id(struct pdsc *pdsc,
+					 enum devlink_param_type dl_id)
+{
+	int vt;
+
+	for (vt = 0; vt < PDS_DEV_TYPE_MAX; vt++) {
+		if (pdsc->viftype_status[vt].dl_id == dl_id)
+			return &pdsc->viftype_status[vt];
+	}
+
+	return NULL;
+}
+
+int pdsc_dl_enable_get(struct devlink *dl, u32 id,
+		       struct devlink_param_gset_ctx *ctx)
+{
+	struct pdsc *pdsc = devlink_priv(dl);
+	struct pdsc_viftype *vt_entry;
+
+	vt_entry = pdsc_dl_find_viftype_by_id(pdsc, id);
+	if (!vt_entry)
+		return -ENOENT;
+
+	ctx->val.vbool = vt_entry->enabled;
+
+	return 0;
+}
+
+int pdsc_dl_enable_set(struct devlink *dl, u32 id,
+		       struct devlink_param_gset_ctx *ctx)
+{
+	struct pdsc *pdsc = devlink_priv(dl);
+	struct pdsc_viftype *vt_entry;
+	int err = 0;
+	int vf_id;
+
+	vt_entry = pdsc_dl_find_viftype_by_id(pdsc, id);
+	if (!vt_entry || !vt_entry->supported)
+		return -EOPNOTSUPP;
+
+	if (vt_entry->enabled == ctx->val.vbool)
+		return 0;
+
+	vt_entry->enabled = ctx->val.vbool;
+	for (vf_id = 0; vf_id < pdsc->num_vfs; vf_id++) {
+		struct pdsc *vf = pdsc->vfs[vf_id].vf;
+
+		err = ctx->val.vbool ? pdsc_auxbus_dev_add(vf, pdsc) :
+				       pdsc_auxbus_dev_del(vf, pdsc);
+	}
+
+	return err;
+}
+
+int pdsc_dl_enable_validate(struct devlink *dl, u32 id,
+			    union devlink_param_value val,
+			    struct netlink_ext_ack *extack)
+{
+	struct pdsc *pdsc = devlink_priv(dl);
+	struct pdsc_viftype *vt_entry;
+
+	vt_entry = pdsc_dl_find_viftype_by_id(pdsc, id);
+	if (!vt_entry || !vt_entry->supported)
+		return -EOPNOTSUPP;
+
+	if (!pdsc->viftype_status[vt_entry->vif_id].supported)
+		return -ENODEV;
+
+	return 0;
+}
+
+int pdsc_dl_flash_update(struct devlink *dl,
+			 struct devlink_flash_update_params *params,
+			 struct netlink_ext_ack *extack)
+{
+	struct pdsc *pdsc = devlink_priv(dl);
+
+	return pdsc_firmware_update(pdsc, params->fw, extack);
+}
+
+static char *fw_slotnames[] = {
+	"fw.goldfw",
+	"fw.mainfwa",
+	"fw.mainfwb",
+};
+
+int pdsc_dl_info_get(struct devlink *dl, struct devlink_info_req *req,
+		     struct netlink_ext_ack *extack)
+{
+	union pds_core_dev_cmd cmd = {
+		.fw_control.opcode = PDS_CORE_CMD_FW_CONTROL,
+		.fw_control.oper = PDS_CORE_FW_GET_LIST,
+	};
+	struct pds_core_fw_list_info fw_list;
+	struct pdsc *pdsc = devlink_priv(dl);
+	union pds_core_dev_comp comp;
+	char buf[16];
+	int listlen;
+	int err;
+	int i;
+
+	mutex_lock(&pdsc->devcmd_lock);
+	err = pdsc_devcmd_locked(pdsc, &cmd, &comp, pdsc->devcmd_timeout * 2);
+	memcpy_fromio(&fw_list, pdsc->cmd_regs->data, sizeof(fw_list));
+	mutex_unlock(&pdsc->devcmd_lock);
+	if (err && err != -EIO)
+		return err;
+
+	listlen = fw_list.num_fw_slots;
+	for (i = 0; i < listlen; i++) {
+		if (i < ARRAY_SIZE(fw_slotnames))
+			strscpy(buf, fw_slotnames[i], sizeof(buf));
+		else
+			snprintf(buf, sizeof(buf), "fw.slot_%d", i);
+		err = devlink_info_version_stored_put(req, buf,
+						      fw_list.fw_names[i].fw_version);
+	}
+
+	err = devlink_info_version_running_put(req,
+					       DEVLINK_INFO_VERSION_GENERIC_FW,
+					       pdsc->dev_info.fw_version);
+	if (err)
+		return err;
+
+	snprintf(buf, sizeof(buf), "0x%x", pdsc->dev_info.asic_type);
+	err = devlink_info_version_fixed_put(req,
+					     DEVLINK_INFO_VERSION_GENERIC_ASIC_ID,
+					     buf);
+	if (err)
+		return err;
+
+	snprintf(buf, sizeof(buf), "0x%x", pdsc->dev_info.asic_rev);
+	err = devlink_info_version_fixed_put(req,
+					     DEVLINK_INFO_VERSION_GENERIC_ASIC_REV,
+					     buf);
+	if (err)
+		return err;
+
+	return devlink_info_serial_number_put(req, pdsc->dev_info.serial_num);
+}
+
+int pdsc_fw_reporter_diagnose(struct devlink_health_reporter *reporter,
+			      struct devlink_fmsg *fmsg,
+			      struct netlink_ext_ack *extack)
+{
+	struct pdsc *pdsc = devlink_health_reporter_priv(reporter);
+	int err;
+
+	mutex_lock(&pdsc->config_lock);
+
+	if (test_bit(PDSC_S_FW_DEAD, &pdsc->state))
+		err = devlink_fmsg_string_pair_put(fmsg, "Status", "dead");
+	else if (!pdsc_is_fw_good(pdsc))
+		err = devlink_fmsg_string_pair_put(fmsg, "Status", "unhealthy");
+	else
+		err = devlink_fmsg_string_pair_put(fmsg, "Status", "healthy");
+
+	mutex_unlock(&pdsc->config_lock);
+
+	if (err)
+		return err;
+
+	err = devlink_fmsg_u32_pair_put(fmsg, "State",
+					pdsc->fw_status &
+						~PDS_CORE_FW_STS_F_GENERATION);
+	if (err)
+		return err;
+
+	err = devlink_fmsg_u32_pair_put(fmsg, "Generation",
+					pdsc->fw_generation >> 4);
+	if (err)
+		return err;
+
+	return devlink_fmsg_u32_pair_put(fmsg, "Recoveries",
+					 pdsc->fw_recoveries);
+}
diff --git a/drivers/net/ethernet/amd/pds_core/fw.c b/drivers/net/ethernet/amd/pds_core/fw.c
new file mode 100644
index 000000000000..90a811f3878a
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/fw.c
@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include "core.h"
+
+/* The worst case wait for the install activity is about 25 minutes when
+ * installing a new CPLD, which is very seldom.  Normal is about 30-35
+ * seconds.  Since the driver can't tell if a CPLD update will happen we
+ * set the timeout for the ugly case.
+ */
+#define PDSC_FW_INSTALL_TIMEOUT	(25 * 60)
+#define PDSC_FW_SELECT_TIMEOUT	30
+
+/* Number of periodic log updates during fw file download */
+#define PDSC_FW_INTERVAL_FRACTION	32
+
+static int pdsc_devcmd_fw_download_locked(struct pdsc *pdsc, u64 addr,
+					  u32 offset, u32 length)
+{
+	union pds_core_dev_cmd cmd = {
+		.fw_download.opcode = PDS_CORE_CMD_FW_DOWNLOAD,
+		.fw_download.offset = cpu_to_le32(offset),
+		.fw_download.addr = cpu_to_le64(addr),
+		.fw_download.length = cpu_to_le32(length),
+	};
+	union pds_core_dev_comp comp;
+
+	return pdsc_devcmd_locked(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
+}
+
+static int pdsc_devcmd_fw_install(struct pdsc *pdsc)
+{
+	union pds_core_dev_cmd cmd = {
+		.fw_control.opcode = PDS_CORE_CMD_FW_CONTROL,
+		.fw_control.oper = PDS_CORE_FW_INSTALL_ASYNC
+	};
+	union pds_core_dev_comp comp;
+	int err;
+
+	err = pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
+	if (err < 0)
+		return err;
+
+	return comp.fw_control.slot;
+}
+
+static int pdsc_devcmd_fw_activate(struct pdsc *pdsc,
+				   enum pds_core_fw_slot slot)
+{
+	union pds_core_dev_cmd cmd = {
+		.fw_control.opcode = PDS_CORE_CMD_FW_CONTROL,
+		.fw_control.oper = PDS_CORE_FW_ACTIVATE_ASYNC,
+		.fw_control.slot = slot
+	};
+	union pds_core_dev_comp comp;
+
+	return pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
+}
+
+static int pdsc_fw_status_long_wait(struct pdsc *pdsc,
+				    const char *label,
+				    unsigned long timeout,
+				    u8 fw_cmd,
+				    struct netlink_ext_ack *extack)
+{
+	union pds_core_dev_cmd cmd = {
+		.fw_control.opcode = PDS_CORE_CMD_FW_CONTROL,
+		.fw_control.oper = fw_cmd,
+	};
+	union pds_core_dev_comp comp;
+	unsigned long start_time;
+	unsigned long end_time;
+	int err;
+
+	/* Ping on the status of the long running async install
+	 * command.  We get EAGAIN while the command is still
+	 * running, else we get the final command status.
+	 */
+	start_time = jiffies;
+	end_time = start_time + (timeout * HZ);
+	do {
+		err = pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
+		msleep(20);
+	} while (time_before(jiffies, end_time) &&
+		 (err == -EAGAIN || err == -ETIMEDOUT));
+
+	if (err == -EAGAIN || err == -ETIMEDOUT) {
+		NL_SET_ERR_MSG_MOD(extack, "Firmware wait timed out");
+		dev_err(pdsc->dev, "DEV_CMD firmware wait %s timed out\n",
+			label);
+	} else if (err) {
+		NL_SET_ERR_MSG_MOD(extack, "Firmware wait failed");
+	}
+
+	return err;
+}
+
+int pdsc_firmware_update(struct pdsc *pdsc, const struct firmware *fw,
+			 struct netlink_ext_ack *extack)
+{
+	u32 buf_sz, copy_sz, offset;
+	struct devlink *dl;
+	int next_interval;
+	u64 data_addr;
+	int err = 0;
+	int fw_slot;
+
+	dev_info(pdsc->dev, "Installing firmware\n");
+
+	dl = priv_to_devlink(pdsc);
+	devlink_flash_update_status_notify(dl, "Preparing to flash",
+					   NULL, 0, 0);
+
+	buf_sz = sizeof(pdsc->cmd_regs->data);
+
+	dev_dbg(pdsc->dev,
+		"downloading firmware - size %d part_sz %d nparts %lu\n",
+		(int)fw->size, buf_sz, DIV_ROUND_UP(fw->size, buf_sz));
+
+	offset = 0;
+	next_interval = 0;
+	data_addr = offsetof(struct pds_core_dev_cmd_regs, data);
+	while (offset < fw->size) {
+		if (offset >= next_interval) {
+			devlink_flash_update_status_notify(dl, "Downloading",
+							   NULL, offset,
+							   fw->size);
+			next_interval = offset +
+					(fw->size / PDSC_FW_INTERVAL_FRACTION);
+		}
+
+		copy_sz = min_t(unsigned int, buf_sz, fw->size - offset);
+		mutex_lock(&pdsc->devcmd_lock);
+		memcpy_toio(&pdsc->cmd_regs->data, fw->data + offset, copy_sz);
+		err = pdsc_devcmd_fw_download_locked(pdsc, data_addr,
+						     offset, copy_sz);
+		mutex_unlock(&pdsc->devcmd_lock);
+		if (err) {
+			dev_err(pdsc->dev,
+				"download failed offset 0x%x addr 0x%llx len 0x%x: %pe\n",
+				offset, data_addr, copy_sz, ERR_PTR(err));
+			NL_SET_ERR_MSG_MOD(extack, "Segment download failed");
+			goto err_out;
+		}
+		offset += copy_sz;
+	}
+	devlink_flash_update_status_notify(dl, "Downloading", NULL,
+					   fw->size, fw->size);
+
+	devlink_flash_update_timeout_notify(dl, "Installing", NULL,
+					    PDSC_FW_INSTALL_TIMEOUT);
+
+	fw_slot = pdsc_devcmd_fw_install(pdsc);
+	if (fw_slot < 0) {
+		err = fw_slot;
+		dev_err(pdsc->dev, "install failed: %pe\n", ERR_PTR(err));
+		NL_SET_ERR_MSG_MOD(extack, "Failed to start firmware install");
+		goto err_out;
+	}
+
+	err = pdsc_fw_status_long_wait(pdsc, "Installing",
+				       PDSC_FW_INSTALL_TIMEOUT,
+				       PDS_CORE_FW_INSTALL_STATUS,
+				       extack);
+	if (err)
+		goto err_out;
+
+	devlink_flash_update_timeout_notify(dl, "Selecting", NULL,
+					    PDSC_FW_SELECT_TIMEOUT);
+
+	err = pdsc_devcmd_fw_activate(pdsc, fw_slot);
+	if (err) {
+		NL_SET_ERR_MSG_MOD(extack, "Failed to start firmware select");
+		goto err_out;
+	}
+
+	err = pdsc_fw_status_long_wait(pdsc, "Selecting",
+				       PDSC_FW_SELECT_TIMEOUT,
+				       PDS_CORE_FW_ACTIVATE_STATUS,
+				       extack);
+	if (err)
+		goto err_out;
+
+	dev_info(pdsc->dev, "Firmware update completed, slot %d\n", fw_slot);
+
+err_out:
+	if (err)
+		devlink_flash_update_status_notify(dl, "Flash failed",
+						   NULL, 0, 0);
+	else
+		devlink_flash_update_status_notify(dl, "Flash done",
+						   NULL, 0, 0);
+	return err;
+}
diff --git a/drivers/net/ethernet/amd/pds_core/main.c b/drivers/net/ethernet/amd/pds_core/main.c
new file mode 100644
index 000000000000..e2d14b1ca471
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/main.c
@@ -0,0 +1,475 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/pci.h>
+
+#include <linux/pds/pds_common.h>
+
+#include "core.h"
+
+MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION);
+MODULE_AUTHOR("Advanced Micro Devices, Inc");
+MODULE_LICENSE("GPL");
+
+/* Supported devices */
+static const struct pci_device_id pdsc_id_table[] = {
+	{ PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) },
+	{ PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },
+	{ 0, }	/* end of table */
+};
+MODULE_DEVICE_TABLE(pci, pdsc_id_table);
+
+static void pdsc_wdtimer_cb(struct timer_list *t)
+{
+	struct pdsc *pdsc = from_timer(pdsc, t, wdtimer);
+
+	dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies);
+	mod_timer(&pdsc->wdtimer,
+		  round_jiffies(jiffies + pdsc->wdtimer_period));
+
+	queue_work(pdsc->wq, &pdsc->health_work);
+}
+
+static void pdsc_unmap_bars(struct pdsc *pdsc)
+{
+	struct pdsc_dev_bar *bars = pdsc->bars;
+	unsigned int i;
+
+	for (i = 0; i < PDS_CORE_BARS_MAX; i++) {
+		if (bars[i].vaddr)
+			pci_iounmap(pdsc->pdev, bars[i].vaddr);
+	}
+}
+
+static int pdsc_map_bars(struct pdsc *pdsc)
+{
+	struct pdsc_dev_bar *bar = pdsc->bars;
+	struct pci_dev *pdev = pdsc->pdev;
+	struct device *dev = pdsc->dev;
+	struct pdsc_dev_bar *bars;
+	unsigned int i, j;
+	int num_bars = 0;
+	int err;
+	u32 sig;
+
+	bars = pdsc->bars;
+
+	/* Since the PCI interface in the hardware is configurable,
+	 * we need to poke into all the bars to find the set we're
+	 * expecting.
+	 */
+	for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) {
+		if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
+			continue;
+
+		bars[j].len = pci_resource_len(pdev, i);
+		bars[j].bus_addr = pci_resource_start(pdev, i);
+		bars[j].res_index = i;
+
+		/* only map the whole bar 0 */
+		if (j > 0) {
+			bars[j].vaddr = NULL;
+		} else {
+			bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);
+			if (!bars[j].vaddr) {
+				dev_err(dev, "Cannot map BAR %d, aborting\n", i);
+				return -ENODEV;
+			}
+		}
+
+		j++;
+	}
+	num_bars = j;
+
+	/* BAR0: dev_cmd and interrupts */
+	if (num_bars < 1) {
+		dev_err(dev, "No bars found\n");
+		err = -EFAULT;
+		goto err_out;
+	}
+
+	if (bar->len < PDS_CORE_BAR0_SIZE) {
+		dev_err(dev, "Resource bar size %lu too small\n", bar->len);
+		err = -EFAULT;
+		goto err_out;
+	}
+
+	pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET;
+	pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET;
+	pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET;
+	pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET;
+
+	sig = ioread32(&pdsc->info_regs->signature);
+	if (sig != PDS_CORE_DEV_INFO_SIGNATURE) {
+		dev_err(dev, "Incompatible firmware signature %x", sig);
+		err = -EFAULT;
+		goto err_out;
+	}
+
+	/* BAR1: doorbells */
+	bar++;
+	if (num_bars < 2) {
+		dev_err(dev, "Doorbell bar missing\n");
+		err = -EFAULT;
+		goto err_out;
+	}
+
+	pdsc->db_pages = bar->vaddr;
+	pdsc->phy_db_pages = bar->bus_addr;
+
+	return 0;
+
+err_out:
+	pdsc_unmap_bars(pdsc);
+	return err;
+}
+
+void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num)
+{
+	return pci_iomap_range(pdsc->pdev,
+			       pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index,
+			       (u64)page_num << PAGE_SHIFT, PAGE_SIZE);
+}
+
+static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+	struct pdsc *pdsc = pci_get_drvdata(pdev);
+	struct device *dev = pdsc->dev;
+	int ret = 0;
+
+	if (num_vfs > 0) {
+		pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf),
+				    GFP_KERNEL);
+		if (!pdsc->vfs)
+			return -ENOMEM;
+		pdsc->num_vfs = num_vfs;
+
+		ret = pci_enable_sriov(pdev, num_vfs);
+		if (ret) {
+			dev_err(dev, "Cannot enable SRIOV: %pe\n",
+				ERR_PTR(ret));
+			goto no_vfs;
+		}
+
+		return num_vfs;
+	}
+
+no_vfs:
+	pci_disable_sriov(pdev);
+
+	kfree(pdsc->vfs);
+	pdsc->vfs = NULL;
+	pdsc->num_vfs = 0;
+
+	return ret;
+}
+
+static int pdsc_init_vf(struct pdsc *vf)
+{
+	struct devlink *dl;
+	struct pdsc *pf;
+	int err;
+
+	pf = pdsc_get_pf_struct(vf->pdev);
+	if (IS_ERR_OR_NULL(pf))
+		return PTR_ERR(pf) ?: -1;
+
+	vf->vf_id = pci_iov_vf_id(vf->pdev);
+
+	dl = priv_to_devlink(vf);
+	devl_lock(dl);
+	devl_register(dl);
+	devl_unlock(dl);
+
+	pf->vfs[vf->vf_id].vf = vf;
+	err = pdsc_auxbus_dev_add(vf, pf);
+	if (err) {
+		devl_lock(dl);
+		devl_unregister(dl);
+		devl_unlock(dl);
+	}
+
+	return err;
+}
+
+static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = {
+	.name = "fw",
+	.diagnose = pdsc_fw_reporter_diagnose,
+};
+
+static const struct devlink_param pdsc_dl_params[] = {
+	DEVLINK_PARAM_GENERIC(ENABLE_VNET,
+			      BIT(DEVLINK_PARAM_CMODE_RUNTIME),
+			      pdsc_dl_enable_get,
+			      pdsc_dl_enable_set,
+			      pdsc_dl_enable_validate),
+};
+
+#define PDSC_WQ_NAME_LEN 24
+
+static int pdsc_init_pf(struct pdsc *pdsc)
+{
+	struct devlink_health_reporter *hr;
+	char wq_name[PDSC_WQ_NAME_LEN];
+	struct devlink *dl;
+	int err;
+
+	pcie_print_link_status(pdsc->pdev);
+
+	err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
+	if (err) {
+		dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
+			ERR_PTR(err));
+		return err;
+	}
+
+	err = pdsc_map_bars(pdsc);
+	if (err)
+		goto err_out_release_regions;
+
+	/* General workqueue and timer, but don't start timer yet */
+	snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);
+	pdsc->wq = create_singlethread_workqueue(wq_name);
+	INIT_WORK(&pdsc->health_work, pdsc_health_thread);
+	timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
+	pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ;
+
+	mutex_init(&pdsc->devcmd_lock);
+	mutex_init(&pdsc->config_lock);
+	spin_lock_init(&pdsc->adminq_lock);
+
+	mutex_lock(&pdsc->config_lock);
+	set_bit(PDSC_S_FW_DEAD, &pdsc->state);
+
+	err = pdsc_setup(pdsc, PDSC_SETUP_INIT);
+	if (err)
+		goto err_out_unmap_bars;
+	err = pdsc_start(pdsc);
+	if (err)
+		goto err_out_teardown;
+
+	mutex_unlock(&pdsc->config_lock);
+
+	dl = priv_to_devlink(pdsc);
+	devl_lock(dl);
+	err = devl_params_register(dl, pdsc_dl_params,
+				   ARRAY_SIZE(pdsc_dl_params));
+	if (err) {
+		dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n",
+			 ERR_PTR(err));
+		goto err_out_unlock_dl;
+	}
+
+	hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc);
+	if (IS_ERR(hr)) {
+		dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr);
+		err = PTR_ERR(hr);
+		goto err_out_unreg_params;
+	}
+	pdsc->fw_reporter = hr;
+
+	devl_register(dl);
+	devl_unlock(dl);
+
+	/* Lastly, start the health check timer */
+	mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period));
+
+	return 0;
+
+err_out_unreg_params:
+	devl_params_unregister(dl, pdsc_dl_params,
+			       ARRAY_SIZE(pdsc_dl_params));
+err_out_unlock_dl:
+	devl_unlock(dl);
+	pdsc_stop(pdsc);
+err_out_teardown:
+	pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
+err_out_unmap_bars:
+	mutex_unlock(&pdsc->config_lock);
+	del_timer_sync(&pdsc->wdtimer);
+	if (pdsc->wq)
+		destroy_workqueue(pdsc->wq);
+	mutex_destroy(&pdsc->config_lock);
+	mutex_destroy(&pdsc->devcmd_lock);
+	pci_free_irq_vectors(pdsc->pdev);
+	pdsc_unmap_bars(pdsc);
+err_out_release_regions:
+	pci_release_regions(pdsc->pdev);
+
+	return err;
+}
+
+static const struct devlink_ops pdsc_dl_ops = {
+	.info_get	= pdsc_dl_info_get,
+	.flash_update	= pdsc_dl_flash_update,
+};
+
+static const struct devlink_ops pdsc_dl_vf_ops = {
+};
+
+static DEFINE_IDA(pdsc_ida);
+
+static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	struct device *dev = &pdev->dev;
+	const struct devlink_ops *ops;
+	struct devlink *dl;
+	struct pdsc *pdsc;
+	bool is_pf;
+	int err;
+
+	is_pf = !pdev->is_virtfn;
+	ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops;
+	dl = devlink_alloc(ops, sizeof(struct pdsc), dev);
+	if (!dl)
+		return -ENOMEM;
+	pdsc = devlink_priv(dl);
+
+	pdsc->pdev = pdev;
+	pdsc->dev = &pdev->dev;
+	set_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
+	pci_set_drvdata(pdev, pdsc);
+	pdsc_debugfs_add_dev(pdsc);
+
+	err = ida_alloc(&pdsc_ida, GFP_KERNEL);
+	if (err < 0) {
+		dev_err(pdsc->dev, "%s: id alloc failed: %pe\n",
+			__func__, ERR_PTR(err));
+		goto err_out_free_devlink;
+	}
+	pdsc->uid = err;
+
+	/* Query system for DMA addressing limitation for the device. */
+	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));
+	if (err) {
+		dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n",
+			ERR_PTR(err));
+		goto err_out_free_ida;
+	}
+
+	err = pci_enable_device(pdev);
+	if (err) {
+		dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
+		goto err_out_free_ida;
+	}
+	pci_set_master(pdev);
+
+	if (is_pf)
+		err = pdsc_init_pf(pdsc);
+	else
+		err = pdsc_init_vf(pdsc);
+	if (err) {
+		dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err));
+		goto err_out_clear_master;
+	}
+
+	clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
+	return 0;
+
+err_out_clear_master:
+	pci_clear_master(pdev);
+	pci_disable_device(pdev);
+err_out_free_ida:
+	ida_free(&pdsc_ida, pdsc->uid);
+err_out_free_devlink:
+	pdsc_debugfs_del_dev(pdsc);
+	devlink_free(dl);
+
+	return err;
+}
+
+static void pdsc_remove(struct pci_dev *pdev)
+{
+	struct pdsc *pdsc = pci_get_drvdata(pdev);
+	struct devlink *dl;
+
+	/* Unhook the registrations first to be sure there
+	 * are no requests while we're stopping.
+	 */
+	dl = priv_to_devlink(pdsc);
+	devl_lock(dl);
+	devl_unregister(dl);
+	if (!pdev->is_virtfn) {
+		if (pdsc->fw_reporter) {
+			devl_health_reporter_destroy(pdsc->fw_reporter);
+			pdsc->fw_reporter = NULL;
+		}
+		devl_params_unregister(dl, pdsc_dl_params,
+				       ARRAY_SIZE(pdsc_dl_params));
+	}
+	devl_unlock(dl);
+
+	if (pdev->is_virtfn) {
+		struct pdsc *pf;
+
+		pf = pdsc_get_pf_struct(pdsc->pdev);
+		if (!IS_ERR(pf)) {
+			pdsc_auxbus_dev_del(pdsc, pf);
+			pf->vfs[pdsc->vf_id].vf = NULL;
+		}
+	} else {
+		/* Remove the VFs and their aux_bus connections before other
+		 * cleanup so that the clients can use the AdminQ to cleanly
+		 * shut themselves down.
+		 */
+		pdsc_sriov_configure(pdev, 0);
+
+		del_timer_sync(&pdsc->wdtimer);
+		if (pdsc->wq)
+			destroy_workqueue(pdsc->wq);
+
+		mutex_lock(&pdsc->config_lock);
+		set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);
+
+		pdsc_stop(pdsc);
+		pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
+		mutex_unlock(&pdsc->config_lock);
+		mutex_destroy(&pdsc->config_lock);
+		mutex_destroy(&pdsc->devcmd_lock);
+
+		pci_free_irq_vectors(pdev);
+		pdsc_unmap_bars(pdsc);
+		pci_release_regions(pdev);
+	}
+
+	pci_clear_master(pdev);
+	pci_disable_device(pdev);
+
+	ida_free(&pdsc_ida, pdsc->uid);
+	pdsc_debugfs_del_dev(pdsc);
+	devlink_free(dl);
+}
+
+static struct pci_driver pdsc_driver = {
+	.name = PDS_CORE_DRV_NAME,
+	.id_table = pdsc_id_table,
+	.probe = pdsc_probe,
+	.remove = pdsc_remove,
+	.sriov_configure = pdsc_sriov_configure,
+};
+
+void *pdsc_get_pf_struct(struct pci_dev *vf_pdev)
+{
+	return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver);
+}
+EXPORT_SYMBOL_GPL(pdsc_get_pf_struct);
+
+static int __init pdsc_init_module(void)
+{
+	if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME))
+		return -EINVAL;
+
+	pdsc_debugfs_create();
+	return pci_register_driver(&pdsc_driver);
+}
+
+static void __exit pdsc_cleanup_module(void)
+{
+	pci_unregister_driver(&pdsc_driver);
+	pdsc_debugfs_destroy();
+}
+
+module_init(pdsc_init_module);
+module_exit(pdsc_cleanup_module);
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_drvinfo.c b/drivers/net/ethernet/aquantia/atlantic/aq_drvinfo.c
index d3526cd38f3d..414b2e448d59 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_drvinfo.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_drvinfo.c
@@ -124,7 +124,7 @@ static const struct hwmon_channel_info aq_hwmon_temp = {
 	.config = aq_hwmon_temp_config,
 };
 
-static const struct hwmon_channel_info *aq_hwmon_info[] = {
+static const struct hwmon_channel_info * const aq_hwmon_info[] = {
 	&aq_hwmon_temp,
 	NULL,
 };
diff --git a/drivers/net/ethernet/asix/ax88796c_main.c b/drivers/net/ethernet/asix/ax88796c_main.c
index 21376c79f671..e551ffaed20d 100644
--- a/drivers/net/ethernet/asix/ax88796c_main.c
+++ b/drivers/net/ethernet/asix/ax88796c_main.c
@@ -1006,7 +1006,7 @@ static int ax88796c_probe(struct spi_device *spi)
 	ax_local->mdiobus->parent = &spi->dev;
 
 	snprintf(ax_local->mdiobus->id, MII_BUS_ID_SIZE,
-		 "ax88796c-%s.%u", dev_name(&spi->dev), spi->chip_select);
+		 "ax88796c-%s.%u", dev_name(&spi->dev), spi_get_chipselect(spi, 0));
 
 	ret = devm_mdiobus_register(&spi->dev, ax_local->mdiobus);
 	if (ret < 0) {
diff --git a/drivers/net/ethernet/atheros/alx/main.c b/drivers/net/ethernet/atheros/alx/main.c
index 306393f8eeca..49bb9a8f00e6 100644
--- a/drivers/net/ethernet/atheros/alx/main.c
+++ b/drivers/net/ethernet/atheros/alx/main.c
@@ -39,7 +39,6 @@
 #include <linux/ipv6.h>
 #include <linux/if_vlan.h>
 #include <linux/mdio.h>
-#include <linux/aer.h>
 #include <linux/bitops.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
@@ -1745,7 +1744,6 @@ static int alx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto out_pci_disable;
 	}
 
-	pci_enable_pcie_error_reporting(pdev);
 	pci_set_master(pdev);
 
 	if (!pdev->pm_cap) {
@@ -1879,7 +1877,6 @@ out_free_netdev:
 	free_netdev(netdev);
 out_pci_release:
 	pci_release_mem_regions(pdev);
-	pci_disable_pcie_error_reporting(pdev);
 out_pci_disable:
 	pci_disable_device(pdev);
 	return err;
@@ -1897,7 +1894,6 @@ static void alx_remove(struct pci_dev *pdev)
 	iounmap(hw->hw_addr);
 	pci_release_mem_regions(pdev);
 
-	pci_disable_pcie_error_reporting(pdev);
 	pci_disable_device(pdev);
 
 	mutex_destroy(&alx->mtx);
diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
index 40c781695d58..4a288799633f 100644
--- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
+++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
@@ -207,16 +207,6 @@ static inline void atl1c_irq_disable(struct atl1c_adapter *adapter)
 	synchronize_irq(adapter->pdev->irq);
 }
 
-/**
- * atl1c_irq_reset - reset interrupt confiure on the NIC
- * @adapter: board private structure
- */
-static inline void atl1c_irq_reset(struct atl1c_adapter *adapter)
-{
-	atomic_set(&adapter->irq_sem, 1);
-	atl1c_irq_enable(adapter);
-}
-
 /*
  * atl1c_wait_until_idle - wait up to AT_HW_MAX_IDLE_DELAY reads
  * of the idle status register until the device is actually idle
diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c
index 9f473854b0f4..466e1d62bcf6 100644
--- a/drivers/net/ethernet/broadcom/bnx2.c
+++ b/drivers/net/ethernet/broadcom/bnx2.c
@@ -48,7 +48,6 @@
 #include <linux/cache.h>
 #include <linux/firmware.h>
 #include <linux/log2.h>
-#include <linux/aer.h>
 #include <linux/crash_dump.h>
 
 #if IS_ENABLED(CONFIG_CNIC)
@@ -3829,7 +3828,7 @@ load_rv2p_fw(struct bnx2 *bp, u32 rv2p_proc,
 	return 0;
 }
 
-static int
+static void
 load_cpu_fw(struct bnx2 *bp, const struct cpu_reg *cpu_reg,
 	    const struct bnx2_mips_fw_file_entry *fw_entry)
 {
@@ -3897,48 +3896,34 @@ load_cpu_fw(struct bnx2 *bp, const struct cpu_reg *cpu_reg,
 	val &= ~cpu_reg->mode_value_halt;
 	bnx2_reg_wr_ind(bp, cpu_reg->state, cpu_reg->state_value_clear);
 	bnx2_reg_wr_ind(bp, cpu_reg->mode, val);
-
-	return 0;
 }
 
-static int
+static void
 bnx2_init_cpus(struct bnx2 *bp)
 {
 	const struct bnx2_mips_fw_file *mips_fw =
 		(const struct bnx2_mips_fw_file *) bp->mips_firmware->data;
 	const struct bnx2_rv2p_fw_file *rv2p_fw =
 		(const struct bnx2_rv2p_fw_file *) bp->rv2p_firmware->data;
-	int rc;
 
 	/* Initialize the RV2P processor. */
 	load_rv2p_fw(bp, RV2P_PROC1, &rv2p_fw->proc1);
 	load_rv2p_fw(bp, RV2P_PROC2, &rv2p_fw->proc2);
 
 	/* Initialize the RX Processor. */
-	rc = load_cpu_fw(bp, &cpu_reg_rxp, &mips_fw->rxp);
-	if (rc)
-		goto init_cpu_err;
+	load_cpu_fw(bp, &cpu_reg_rxp, &mips_fw->rxp);
 
 	/* Initialize the TX Processor. */
-	rc = load_cpu_fw(bp, &cpu_reg_txp, &mips_fw->txp);
-	if (rc)
-		goto init_cpu_err;
+	load_cpu_fw(bp, &cpu_reg_txp, &mips_fw->txp);
 
 	/* Initialize the TX Patch-up Processor. */
-	rc = load_cpu_fw(bp, &cpu_reg_tpat, &mips_fw->tpat);
-	if (rc)
-		goto init_cpu_err;
+	load_cpu_fw(bp, &cpu_reg_tpat, &mips_fw->tpat);
 
 	/* Initialize the Completion Processor. */
-	rc = load_cpu_fw(bp, &cpu_reg_com, &mips_fw->com);
-	if (rc)
-		goto init_cpu_err;
+	load_cpu_fw(bp, &cpu_reg_com, &mips_fw->com);
 
 	/* Initialize the Command Processor. */
-	rc = load_cpu_fw(bp, &cpu_reg_cp, &mips_fw->cp);
-
-init_cpu_err:
-	return rc;
+	load_cpu_fw(bp, &cpu_reg_cp, &mips_fw->cp);
 }
 
 static void
@@ -4951,8 +4936,7 @@ bnx2_init_chip(struct bnx2 *bp)
 	} else
 		bnx2_init_context(bp);
 
-	if ((rc = bnx2_init_cpus(bp)) != 0)
-		return rc;
+	bnx2_init_cpus(bp);
 
 	bnx2_init_nvram(bp);
 
@@ -8093,7 +8077,6 @@ bnx2_init_board(struct pci_dev *pdev, struct net_device *dev)
 	int rc, i, j;
 	u32 reg;
 	u64 dma_mask, persist_dma_mask;
-	int err;
 
 	SET_NETDEV_DEV(dev, &pdev->dev);
 	bp = netdev_priv(dev);
@@ -8176,12 +8159,6 @@ bnx2_init_board(struct pci_dev *pdev, struct net_device *dev)
 		bp->flags |= BNX2_FLAG_PCIE;
 		if (BNX2_CHIP_REV(bp) == BNX2_CHIP_REV_Ax)
 			bp->flags |= BNX2_FLAG_JUMBO_BROKEN;
-
-		/* AER (Advanced Error Reporting) hooks */
-		err = pci_enable_pcie_error_reporting(pdev);
-		if (!err)
-			bp->flags |= BNX2_FLAG_AER_ENABLED;
-
 	} else {
 		bp->pcix_cap = pci_find_capability(pdev, PCI_CAP_ID_PCIX);
 		if (bp->pcix_cap == 0) {
@@ -8460,11 +8437,6 @@ bnx2_init_board(struct pci_dev *pdev, struct net_device *dev)
 	return 0;
 
 err_out_unmap:
-	if (bp->flags & BNX2_FLAG_AER_ENABLED) {
-		pci_disable_pcie_error_reporting(pdev);
-		bp->flags &= ~BNX2_FLAG_AER_ENABLED;
-	}
-
 	pci_iounmap(pdev, bp->regview);
 	bp->regview = NULL;
 
@@ -8638,11 +8610,6 @@ bnx2_remove_one(struct pci_dev *pdev)
 	bnx2_free_stats_blk(dev);
 	kfree(bp->temp_stats_blk);
 
-	if (bp->flags & BNX2_FLAG_AER_ENABLED) {
-		pci_disable_pcie_error_reporting(pdev);
-		bp->flags &= ~BNX2_FLAG_AER_ENABLED;
-	}
-
 	bnx2_release_firmware(bp);
 
 	free_netdev(dev);
@@ -8766,9 +8733,6 @@ static pci_ers_result_t bnx2_io_slot_reset(struct pci_dev *pdev)
 	}
 	rtnl_unlock();
 
-	if (!(bp->flags & BNX2_FLAG_AER_ENABLED))
-		return result;
-
 	return result;
 }
 
diff --git a/drivers/net/ethernet/broadcom/bnx2.h b/drivers/net/ethernet/broadcom/bnx2.h
index a09ec47461c9..315b08c64edd 100644
--- a/drivers/net/ethernet/broadcom/bnx2.h
+++ b/drivers/net/ethernet/broadcom/bnx2.h
@@ -6808,7 +6808,6 @@ struct bnx2 {
 #define BNX2_FLAG_JUMBO_BROKEN		0x00000800
 #define BNX2_FLAG_CAN_KEEP_VLAN		0x00001000
 #define BNX2_FLAG_BROKEN_STATS		0x00002000
-#define BNX2_FLAG_AER_ENABLED		0x00004000
 
 	struct bnx2_napi	bnx2_napi[BNX2_MAX_MSIX_VEC];
 
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h
index dd5945c4bfec..8bcde0a6e011 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h
@@ -1486,7 +1486,6 @@ struct bnx2x {
 #define IS_VF_FLAG			(1 << 22)
 #define BC_SUPPORTS_RMMOD_CMD		(1 << 23)
 #define HAS_PHYS_PORT_ID		(1 << 24)
-#define AER_ENABLED			(1 << 25)
 #define PTP_SUPPORTED			(1 << 26)
 #define TX_TIMESTAMPING_EN		(1 << 27)
 
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
index 12083b9679b5..6ea5521074d3 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
@@ -1935,8 +1935,7 @@ u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
 
 		/* Skip VLAN tag if present */
 		if (ether_type == ETH_P_8021Q) {
-			struct vlan_ethhdr *vhdr =
-				(struct vlan_ethhdr *)skb->data;
+			struct vlan_ethhdr *vhdr = skb_vlan_eth_hdr(skb);
 
 			ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
 		}
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
index 5d1e4fe335aa..637d162bbcfa 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
@@ -29,7 +29,6 @@
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/init.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
@@ -298,7 +297,7 @@ const u32 dmae_reg_go_c[] = {
 
 /* Global resources for unloading a previously loaded device */
 #define BNX2X_PREV_WAIT_NEEDED 1
-static DEFINE_SEMAPHORE(bnx2x_prev_sem);
+static DEFINE_SEMAPHORE(bnx2x_prev_sem, 1);
 static LIST_HEAD(bnx2x_prev_list);
 
 /* Forward declaration */
@@ -13037,14 +13036,6 @@ static const struct net_device_ops bnx2x_netdev_ops = {
 	.ndo_features_check	= bnx2x_features_check,
 };
 
-static void bnx2x_disable_pcie_error_reporting(struct bnx2x *bp)
-{
-	if (bp->flags & AER_ENABLED) {
-		pci_disable_pcie_error_reporting(bp->pdev);
-		bp->flags &= ~AER_ENABLED;
-	}
-}
-
 static int bnx2x_init_dev(struct bnx2x *bp, struct pci_dev *pdev,
 			  struct net_device *dev, unsigned long board_type)
 {
@@ -13157,13 +13148,6 @@ static int bnx2x_init_dev(struct bnx2x *bp, struct pci_dev *pdev,
 	/* Set PCIe reset type to fundamental for EEH recovery */
 	pdev->needs_freset = 1;
 
-	/* AER (Advanced Error reporting) configuration */
-	rc = pci_enable_pcie_error_reporting(pdev);
-	if (!rc)
-		bp->flags |= AER_ENABLED;
-	else
-		BNX2X_DEV_INFO("Failed To configure PCIe AER [%d]\n", rc);
-
 	/*
 	 * Clean the following indirect addresses for all functions since it
 	 * is not used by the driver.
@@ -14020,8 +14004,6 @@ init_one_freemem:
 	bnx2x_free_mem_bp(bp);
 
 init_one_exit:
-	bnx2x_disable_pcie_error_reporting(bp);
-
 	if (bp->regview)
 		iounmap(bp->regview);
 
@@ -14102,7 +14084,6 @@ static void __bnx2x_remove(struct pci_dev *pdev,
 		pci_set_power_state(pdev, PCI_D3hot);
 	}
 
-	bnx2x_disable_pcie_error_reporting(bp);
 	if (remove_netdev) {
 		if (bp->regview)
 			iounmap(bp->regview);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index c23e3b397bcf..dcd9367f05af 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -48,7 +48,6 @@
 #include <linux/prefetch.h>
 #include <linux/cache.h>
 #include <linux/log2.h>
-#include <linux/aer.h>
 #include <linux/bitmap.h>
 #include <linux/cpu_rmap.h>
 #include <linux/cpumask.h>
@@ -57,6 +56,7 @@
 #include <linux/hwmon-sysfs.h>
 #include <net/page_pool.h>
 #include <linux/align.h>
+#include <net/netdev_queues.h>
 
 #include "bnxt_hsi.h"
 #include "bnxt.h"
@@ -332,26 +332,6 @@ static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 	txr->kick_pending = 0;
 }
 
-static bool bnxt_txr_netif_try_stop_queue(struct bnxt *bp,
-					  struct bnxt_tx_ring_info *txr,
-					  struct netdev_queue *txq)
-{
-	netif_tx_stop_queue(txq);
-
-	/* netif_tx_stop_queue() must be done before checking
-	 * tx index in bnxt_tx_avail() below, because in
-	 * bnxt_tx_int(), we update tx index before checking for
-	 * netif_tx_queue_stopped().
-	 */
-	smp_mb();
-	if (bnxt_tx_avail(bp, txr) >= bp->tx_wake_thresh) {
-		netif_tx_wake_queue(txq);
-		return false;
-	}
-
-	return true;
-}
-
 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct bnxt *bp = netdev_priv(dev);
@@ -385,7 +365,8 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		if (net_ratelimit() && txr->kick_pending)
 			netif_warn(bp, tx_err, dev,
 				   "bnxt: ring busy w/ flush pending!\n");
-		if (bnxt_txr_netif_try_stop_queue(bp, txr, txq))
+		if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
+					bp->tx_wake_thresh))
 			return NETDEV_TX_BUSY;
 	}
 
@@ -491,7 +472,7 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		prod = NEXT_TX(prod);
 		tx_push->doorbell =
 			cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
-		txr->tx_prod = prod;
+		WRITE_ONCE(txr->tx_prod, prod);
 
 		tx_buf->is_push = 1;
 		netdev_tx_sent_queue(txq, skb->len);
@@ -602,7 +583,7 @@ normal_tx:
 	wmb();
 
 	prod = NEXT_TX(prod);
-	txr->tx_prod = prod;
+	WRITE_ONCE(txr->tx_prod, prod);
 
 	if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 		bnxt_txr_db_kick(bp, txr, prod);
@@ -615,7 +596,8 @@ tx_done:
 		if (netdev_xmit_more() && !tx_buf->is_push)
 			bnxt_txr_db_kick(bp, txr, prod);
 
-		bnxt_txr_netif_try_stop_queue(bp, txr, txq);
+		netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
+				   bp->tx_wake_thresh);
 	}
 	return NETDEV_TX_OK;
 
@@ -706,20 +688,11 @@ next_tx_int:
 		dev_kfree_skb_any(skb);
 	}
 
-	netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
-	txr->tx_cons = cons;
+	WRITE_ONCE(txr->tx_cons, cons);
 
-	/* Need to make the tx_cons update visible to bnxt_start_xmit()
-	 * before checking for netif_tx_queue_stopped().  Without the
-	 * memory barrier, there is a small possibility that bnxt_start_xmit()
-	 * will miss it and cause the queue to be stopped forever.
-	 */
-	smp_mb();
-
-	if (unlikely(netif_tx_queue_stopped(txq)) &&
-	    bnxt_tx_avail(bp, txr) >= bp->tx_wake_thresh &&
-	    READ_ONCE(txr->dev_state) != BNXT_DEV_STATE_CLOSING)
-		netif_tx_wake_queue(txq);
+	__netif_txq_completed_wake(txq, nr_pkts, tx_bytes,
+				   bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
+				   READ_ONCE(txr->dev_state) != BNXT_DEV_STATE_CLOSING);
 }
 
 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
@@ -2388,7 +2361,7 @@ static int bnxt_async_event_process(struct bnxt *bp,
 	case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
 		switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
 		case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
-			if (bp->fw_cap & BNXT_FW_CAP_PTP_RTC) {
+			if (BNXT_PTP_USE_RTC(bp)) {
 				struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
 				u64 ns;
 
@@ -3238,6 +3211,7 @@ static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
 
 	pp.pool_size = bp->rx_ring_size;
 	pp.nid = dev_to_node(&bp->pdev->dev);
+	pp.napi = &rxr->bnapi->napi;
 	pp.dev = &bp->pdev->dev;
 	pp.dma_dir = DMA_BIDIRECTIONAL;
 
@@ -7627,7 +7601,7 @@ static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
 	u8 flags;
 	int rc;
 
-	if (bp->hwrm_spec_code < 0x10801) {
+	if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5_THOR(bp)) {
 		rc = -ENODEV;
 		goto no_ptp;
 	}
@@ -7770,7 +7744,7 @@ static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
 		if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
 			bp->flags |= BNXT_FLAG_WOL_CAP;
 		if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
-			__bnxt_hwrm_ptp_qcfg(bp);
+			bp->fw_cap |= BNXT_FW_CAP_PTP;
 		} else {
 			bnxt_ptp_clear(bp);
 			kfree(bp->ptp_cfg);
@@ -12299,6 +12273,8 @@ static int bnxt_fw_init_one_p2(struct bnxt *bp)
 	bnxt_hwrm_vnic_qcaps(bp);
 	bnxt_hwrm_port_led_qcaps(bp);
 	bnxt_ethtool_init(bp);
+	if (bp->fw_cap & BNXT_FW_CAP_PTP)
+		__bnxt_hwrm_ptp_qcfg(bp);
 	bnxt_dcb_init(bp);
 	return 0;
 }
@@ -12705,8 +12681,6 @@ static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
 		goto init_err_release;
 	}
 
-	pci_enable_pcie_error_reporting(pdev);
-
 	INIT_WORK(&bp->sp_task, bnxt_sp_task);
 	INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
 
@@ -13186,7 +13160,6 @@ static void bnxt_remove_one(struct pci_dev *pdev)
 	bnxt_rdma_aux_device_uninit(bp);
 
 	bnxt_ptp_clear(bp);
-	pci_disable_pcie_error_reporting(pdev);
 	unregister_netdev(dev);
 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
 	/* Flush any pending tasks */
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index 5928430f6f51..080e73496066 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -1969,34 +1969,35 @@ struct bnxt {
 
 	u32			msg_enable;
 
-	u32			fw_cap;
-	#define BNXT_FW_CAP_SHORT_CMD			0x00000001
-	#define BNXT_FW_CAP_LLDP_AGENT			0x00000002
-	#define BNXT_FW_CAP_DCBX_AGENT			0x00000004
-	#define BNXT_FW_CAP_NEW_RM			0x00000008
-	#define BNXT_FW_CAP_IF_CHANGE			0x00000010
-	#define BNXT_FW_CAP_KONG_MB_CHNL		0x00000080
-	#define BNXT_FW_CAP_OVS_64BIT_HANDLE		0x00000400
-	#define BNXT_FW_CAP_TRUSTED_VF			0x00000800
-	#define BNXT_FW_CAP_ERROR_RECOVERY		0x00002000
-	#define BNXT_FW_CAP_PKG_VER			0x00004000
-	#define BNXT_FW_CAP_CFA_ADV_FLOW		0x00008000
-	#define BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2	0x00010000
-	#define BNXT_FW_CAP_PCIE_STATS_SUPPORTED	0x00020000
-	#define BNXT_FW_CAP_EXT_STATS_SUPPORTED		0x00040000
-	#define BNXT_FW_CAP_RSS_HASH_TYPE_DELTA		0x00080000
-	#define BNXT_FW_CAP_ERR_RECOVER_RELOAD		0x00100000
-	#define BNXT_FW_CAP_HOT_RESET			0x00200000
-	#define BNXT_FW_CAP_PTP_RTC			0x00400000
-	#define BNXT_FW_CAP_RX_ALL_PKT_TS		0x00800000
-	#define BNXT_FW_CAP_VLAN_RX_STRIP		0x01000000
-	#define BNXT_FW_CAP_VLAN_TX_INSERT		0x02000000
-	#define BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED	0x04000000
-	#define BNXT_FW_CAP_LIVEPATCH			0x08000000
-	#define BNXT_FW_CAP_PTP_PPS			0x10000000
-	#define BNXT_FW_CAP_HOT_RESET_IF		0x20000000
-	#define BNXT_FW_CAP_RING_MONITOR		0x40000000
-	#define BNXT_FW_CAP_DBG_QCAPS			0x80000000
+	u64			fw_cap;
+	#define BNXT_FW_CAP_SHORT_CMD			BIT_ULL(0)
+	#define BNXT_FW_CAP_LLDP_AGENT			BIT_ULL(1)
+	#define BNXT_FW_CAP_DCBX_AGENT			BIT_ULL(2)
+	#define BNXT_FW_CAP_NEW_RM			BIT_ULL(3)
+	#define BNXT_FW_CAP_IF_CHANGE			BIT_ULL(4)
+	#define BNXT_FW_CAP_KONG_MB_CHNL		BIT_ULL(7)
+	#define BNXT_FW_CAP_OVS_64BIT_HANDLE		BIT_ULL(10)
+	#define BNXT_FW_CAP_TRUSTED_VF			BIT_ULL(11)
+	#define BNXT_FW_CAP_ERROR_RECOVERY		BIT_ULL(13)
+	#define BNXT_FW_CAP_PKG_VER			BIT_ULL(14)
+	#define BNXT_FW_CAP_CFA_ADV_FLOW		BIT_ULL(15)
+	#define BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2	BIT_ULL(16)
+	#define BNXT_FW_CAP_PCIE_STATS_SUPPORTED	BIT_ULL(17)
+	#define BNXT_FW_CAP_EXT_STATS_SUPPORTED		BIT_ULL(18)
+	#define BNXT_FW_CAP_RSS_HASH_TYPE_DELTA		BIT_ULL(19)
+	#define BNXT_FW_CAP_ERR_RECOVER_RELOAD		BIT_ULL(20)
+	#define BNXT_FW_CAP_HOT_RESET			BIT_ULL(21)
+	#define BNXT_FW_CAP_PTP_RTC			BIT_ULL(22)
+	#define BNXT_FW_CAP_RX_ALL_PKT_TS		BIT_ULL(23)
+	#define BNXT_FW_CAP_VLAN_RX_STRIP		BIT_ULL(24)
+	#define BNXT_FW_CAP_VLAN_TX_INSERT		BIT_ULL(25)
+	#define BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED	BIT_ULL(26)
+	#define BNXT_FW_CAP_LIVEPATCH			BIT_ULL(27)
+	#define BNXT_FW_CAP_PTP_PPS			BIT_ULL(28)
+	#define BNXT_FW_CAP_HOT_RESET_IF		BIT_ULL(29)
+	#define BNXT_FW_CAP_RING_MONITOR		BIT_ULL(30)
+	#define BNXT_FW_CAP_DBG_QCAPS			BIT_ULL(31)
+	#define BNXT_FW_CAP_PTP				BIT_ULL(32)
 
 	u32			fw_dbg_cap;
 
@@ -2230,13 +2231,12 @@ struct bnxt {
 #define SFF_MODULE_ID_QSFP28			0x11
 #define BNXT_MAX_PHY_I2C_RESP_SIZE		64
 
-static inline u32 bnxt_tx_avail(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
+static inline u32 bnxt_tx_avail(struct bnxt *bp,
+				const struct bnxt_tx_ring_info *txr)
 {
-	/* Tell compiler to fetch tx indices from memory. */
-	barrier();
+	u32 used = READ_ONCE(txr->tx_prod) - READ_ONCE(txr->tx_cons);
 
-	return bp->tx_ring_size -
-		((txr->tx_prod - txr->tx_cons) & bp->tx_ring_mask);
+	return bp->tx_ring_size - (used & bp->tx_ring_mask);
 }
 
 static inline void bnxt_writeq(struct bnxt *bp, u64 val,
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index 6bd18eb5137f..2dd8ee4a6f75 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -2864,7 +2864,7 @@ static int bnxt_get_nvram_directory(struct net_device *dev, u32 len, u8 *data)
 	if (rc)
 		return rc;
 
-	buflen = dir_entries * entry_length;
+	buflen = mul_u32_u32(dir_entries, entry_length);
 	buf = hwrm_req_dma_slice(bp, req, buflen, &dma_handle);
 	if (!buf) {
 		hwrm_req_drop(bp, req);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
index a3a3978a4d1c..e46689128e32 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
@@ -230,7 +230,7 @@ static int bnxt_ptp_adjfine(struct ptp_clock_info *ptp_info, long scaled_ppm)
 						ptp_info);
 	struct bnxt *bp = ptp->bp;
 
-	if (BNXT_PTP_USE_RTC(bp))
+	if (!BNXT_MH(bp))
 		return bnxt_ptp_adjfine_rtc(bp, scaled_ppm);
 
 	spin_lock_bh(&ptp->ptp_lock);
@@ -861,9 +861,15 @@ static void bnxt_ptp_timecounter_init(struct bnxt *bp, bool init_tc)
 		memset(&ptp->cc, 0, sizeof(ptp->cc));
 		ptp->cc.read = bnxt_cc_read;
 		ptp->cc.mask = CYCLECOUNTER_MASK(48);
-		ptp->cc.shift = BNXT_CYCLES_SHIFT;
-		ptp->cc.mult = clocksource_khz2mult(BNXT_DEVCLK_FREQ, ptp->cc.shift);
-		ptp->cmult = ptp->cc.mult;
+		if (BNXT_MH(bp)) {
+			/* Use timecounter based non-real time mode */
+			ptp->cc.shift = BNXT_CYCLES_SHIFT;
+			ptp->cc.mult = clocksource_khz2mult(BNXT_DEVCLK_FREQ, ptp->cc.shift);
+			ptp->cmult = ptp->cc.mult;
+		} else {
+			ptp->cc.shift = 0;
+			ptp->cc.mult = 1;
+		}
 		ptp->next_overflow_check = jiffies + BNXT_PHC_OVERFLOW_PERIOD;
 	}
 	if (init_tc)
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
index 3ed3a2b3b3a9..dde327f2c57e 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
@@ -825,8 +825,24 @@ static int bnxt_sriov_enable(struct bnxt *bp, int *num_vfs)
 	if (rc)
 		goto err_out2;
 
+	if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
+		return 0;
+
+	/* Create representors for VFs in switchdev mode */
+	devl_lock(bp->dl);
+	rc = bnxt_vf_reps_create(bp);
+	devl_unlock(bp->dl);
+	if (rc) {
+		netdev_info(bp->dev, "Cannot enable VFS as representors cannot be created\n");
+		goto err_out3;
+	}
+
 	return 0;
 
+err_out3:
+	/* Disable SR-IOV */
+	pci_disable_sriov(bp->pdev);
+
 err_out2:
 	/* Free the resources reserved for various VF's */
 	bnxt_hwrm_func_vf_resource_free(bp, *num_vfs);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
index e7b5e28ee29f..852eb449ccae 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
@@ -304,7 +304,7 @@ void bnxt_rdma_aux_device_uninit(struct bnxt *bp)
 	struct auxiliary_device *adev;
 
 	/* Skip if no auxiliary device init was done. */
-	if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
+	if (!bp->aux_priv)
 		return;
 
 	aux_priv = bp->aux_priv;
@@ -324,6 +324,7 @@ static void bnxt_aux_dev_release(struct device *dev)
 	bp->edev = NULL;
 	kfree(aux_priv->edev);
 	kfree(aux_priv);
+	bp->aux_priv = NULL;
 }
 
 static void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt *bp)
@@ -359,19 +360,18 @@ void bnxt_rdma_aux_device_init(struct bnxt *bp)
 	if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
 		return;
 
-	bp->aux_priv = kzalloc(sizeof(*bp->aux_priv), GFP_KERNEL);
-	if (!bp->aux_priv)
+	aux_priv = kzalloc(sizeof(*bp->aux_priv), GFP_KERNEL);
+	if (!aux_priv)
 		goto exit;
 
-	bp->aux_priv->id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL);
-	if (bp->aux_priv->id < 0) {
+	aux_priv->id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL);
+	if (aux_priv->id < 0) {
 		netdev_warn(bp->dev,
 			    "ida alloc failed for ROCE auxiliary device\n");
-		kfree(bp->aux_priv);
+		kfree(aux_priv);
 		goto exit;
 	}
 
-	aux_priv = bp->aux_priv;
 	aux_dev = &aux_priv->aux_dev;
 	aux_dev->id = aux_priv->id;
 	aux_dev->name = "rdma";
@@ -380,10 +380,11 @@ void bnxt_rdma_aux_device_init(struct bnxt *bp)
 
 	rc = auxiliary_device_init(aux_dev);
 	if (rc) {
-		ida_free(&bnxt_aux_dev_ids, bp->aux_priv->id);
-		kfree(bp->aux_priv);
+		ida_free(&bnxt_aux_dev_ids, aux_priv->id);
+		kfree(aux_priv);
 		goto exit;
 	}
+	bp->aux_priv = aux_priv;
 
 	/* From this point, all cleanup will happen via the .release callback &
 	 * any error unwinding will need to include a call to
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
index fcc65890820a..2f1a1f2d2157 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
@@ -356,10 +356,15 @@ void bnxt_vf_reps_destroy(struct bnxt *bp)
 	/* un-publish cfa_code_map so that RX path can't see it anymore */
 	kfree(bp->cfa_code_map);
 	bp->cfa_code_map = NULL;
-	bp->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
 
-	if (closed)
+	if (closed) {
+		/* Temporarily set legacy mode to avoid re-opening
+		 * representors and restore switchdev mode after that.
+		 */
+		bp->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
 		bnxt_open_nic(bp, false, false);
+		bp->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
+	}
 	rtnl_unlock();
 
 	/* Need to call vf_reps_destroy() outside of rntl_lock
@@ -482,7 +487,7 @@ static void bnxt_vf_rep_netdev_init(struct bnxt *bp, struct bnxt_vf_rep *vf_rep,
 	dev->min_mtu = ETH_ZLEN;
 }
 
-static int bnxt_vf_reps_create(struct bnxt *bp)
+int bnxt_vf_reps_create(struct bnxt *bp)
 {
 	u16 *cfa_code_map = NULL, num_vfs = pci_num_vf(bp->pdev);
 	struct bnxt_vf_rep *vf_rep;
@@ -535,7 +540,6 @@ static int bnxt_vf_reps_create(struct bnxt *bp)
 
 	/* publish cfa_code_map only after all VF-reps have been initialized */
 	bp->cfa_code_map = cfa_code_map;
-	bp->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
 	netif_keep_dst(bp->dev);
 	return 0;
 
@@ -559,6 +563,7 @@ int bnxt_dl_eswitch_mode_set(struct devlink *devlink, u16 mode,
 			     struct netlink_ext_ack *extack)
 {
 	struct bnxt *bp = bnxt_get_bp_from_dl(devlink);
+	int ret = 0;
 
 	if (bp->eswitch_mode == mode) {
 		netdev_info(bp->dev, "already in %s eswitch mode\n",
@@ -570,7 +575,7 @@ int bnxt_dl_eswitch_mode_set(struct devlink *devlink, u16 mode,
 	switch (mode) {
 	case DEVLINK_ESWITCH_MODE_LEGACY:
 		bnxt_vf_reps_destroy(bp);
-		return 0;
+		break;
 
 	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
 		if (bp->hwrm_spec_code < 0x10803) {
@@ -578,15 +583,19 @@ int bnxt_dl_eswitch_mode_set(struct devlink *devlink, u16 mode,
 			return -ENOTSUPP;
 		}
 
-		if (pci_num_vf(bp->pdev) == 0) {
-			netdev_info(bp->dev, "Enable VFs before setting switchdev mode\n");
-			return -EPERM;
-		}
-		return bnxt_vf_reps_create(bp);
+		/* Create representors for existing VFs */
+		if (pci_num_vf(bp->pdev) > 0)
+			ret = bnxt_vf_reps_create(bp);
+		break;
 
 	default:
 		return -EINVAL;
 	}
+
+	if (!ret)
+		bp->eswitch_mode = mode;
+
+	return ret;
 }
 
 #endif
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h
index 5637a84884d7..33a965631d0b 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h
@@ -14,6 +14,7 @@
 
 #define	MAX_CFA_CODE			65536
 
+int bnxt_vf_reps_create(struct bnxt *bp);
 void bnxt_vf_reps_destroy(struct bnxt *bp);
 void bnxt_vf_reps_close(struct bnxt *bp);
 void bnxt_vf_reps_open(struct bnxt *bp);
@@ -37,6 +38,11 @@ int bnxt_dl_eswitch_mode_set(struct devlink *devlink, u16 mode,
 
 #else
 
+static inline int bnxt_vf_reps_create(struct bnxt *bp)
+{
+	return 0;
+}
+
 static inline void bnxt_vf_reps_close(struct bnxt *bp)
 {
 }
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
index 5843c93b1711..4efa5fe6972b 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
@@ -64,7 +64,7 @@ struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,
 		int frag_len;
 
 		prod = NEXT_TX(prod);
-		txr->tx_prod = prod;
+		WRITE_ONCE(txr->tx_prod, prod);
 
 		/* first fill up the first buffer */
 		frag_tx_buf = &txr->tx_buf_ring[prod];
@@ -94,7 +94,7 @@ struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,
 	/* Sync TX BD */
 	wmb();
 	prod = NEXT_TX(prod);
-	txr->tx_prod = prod;
+	WRITE_ONCE(txr->tx_prod, prod);
 
 	return tx_buf;
 }
@@ -161,7 +161,7 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 		}
 		tx_cons = NEXT_TX(tx_cons);
 	}
-	txr->tx_cons = tx_cons;
+	WRITE_ONCE(txr->tx_cons, tx_cons);
 	if (rx_doorbell_needed) {
 		tx_buf = &txr->tx_buf_ring[last_tx_cons];
 		bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod);
diff --git a/drivers/net/ethernet/broadcom/sb1250-mac.c b/drivers/net/ethernet/broadcom/sb1250-mac.c
index f02facb60fd1..3a6763c5e8b3 100644
--- a/drivers/net/ethernet/broadcom/sb1250-mac.c
+++ b/drivers/net/ethernet/broadcom/sb1250-mac.c
@@ -73,7 +73,7 @@ MODULE_PARM_DESC(int_timeout_rx, "RX timeout value");
 
 #include <asm/sibyte/board.h>
 #include <asm/sibyte/sb1250.h>
-#if defined(CONFIG_SIBYTE_BCM1x55) || defined(CONFIG_SIBYTE_BCM1x80)
+#if defined(CONFIG_SIBYTE_BCM1x80)
 #include <asm/sibyte/bcm1480_regs.h>
 #include <asm/sibyte/bcm1480_int.h>
 #define R_MAC_DMA_OODPKTLOST_RX	R_MAC_DMA_OODPKTLOST
@@ -87,7 +87,7 @@ MODULE_PARM_DESC(int_timeout_rx, "RX timeout value");
 #include <asm/sibyte/sb1250_mac.h>
 #include <asm/sibyte/sb1250_dma.h>
 
-#if defined(CONFIG_SIBYTE_BCM1x55) || defined(CONFIG_SIBYTE_BCM1x80)
+#if defined(CONFIG_SIBYTE_BCM1x80)
 #define UNIT_INT(n)		(K_BCM1480_INT_MAC_0 + ((n) * 2))
 #elif defined(CONFIG_SIBYTE_SB1250) || defined(CONFIG_SIBYTE_BCM112X)
 #define UNIT_INT(n)		(K_INT_MAC_0 + (n))
@@ -1527,7 +1527,7 @@ static void sbmac_channel_start(struct sbmac_softc *s)
 	 * Turn on the rest of the bits in the enable register
 	 */
 
-#if defined(CONFIG_SIBYTE_BCM1x55) || defined(CONFIG_SIBYTE_BCM1x80)
+#if defined(CONFIG_SIBYTE_BCM1x80)
 	__raw_writeq(M_MAC_RXDMA_EN0 |
 		       M_MAC_TXDMA_EN0, s->sbm_macenable);
 #elif defined(CONFIG_SIBYTE_SB1250) || defined(CONFIG_SIBYTE_BCM112X)
diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h
index 14dfec4db8f9..cfbdd0022764 100644
--- a/drivers/net/ethernet/cadence/macb.h
+++ b/drivers/net/ethernet/cadence/macb.h
@@ -95,6 +95,8 @@
 #define GEM_SA4B		0x00A0 /* Specific4 Bottom */
 #define GEM_SA4T		0x00A4 /* Specific4 Top */
 #define GEM_WOL			0x00b8 /* Wake on LAN */
+#define GEM_RXPTPUNI		0x00D4 /* PTP RX Unicast address */
+#define GEM_TXPTPUNI		0x00D8 /* PTP TX Unicast address */
 #define GEM_EFTSH		0x00e8 /* PTP Event Frame Transmitted Seconds Register 47:32 */
 #define GEM_EFRSH		0x00ec /* PTP Event Frame Received Seconds Register 47:32 */
 #define GEM_PEFTSH		0x00f0 /* PTP Peer Event Frame Transmitted Seconds Register 47:32 */
@@ -245,6 +247,8 @@
 #define MACB_TZQ_OFFSET		12 /* Transmit zero quantum pause frame */
 #define MACB_TZQ_SIZE		1
 #define MACB_SRTSM_OFFSET	15 /* Store Receive Timestamp to Memory */
+#define MACB_PTPUNI_OFFSET	20 /* PTP Unicast packet enable */
+#define MACB_PTPUNI_SIZE	1
 #define MACB_OSSMODE_OFFSET	24 /* Enable One Step Synchro Mode */
 #define MACB_OSSMODE_SIZE	1
 #define MACB_MIIONRGMII_OFFSET	28 /* MII Usage on RGMII Interface */
@@ -692,6 +696,8 @@
 #define GEM_CLK_DIV48				3
 #define GEM_CLK_DIV64				4
 #define GEM_CLK_DIV96				5
+#define GEM_CLK_DIV128				6
+#define GEM_CLK_DIV224				7
 
 /* Constants for MAN register */
 #define MACB_MAN_C22_SOF			1
@@ -1361,7 +1367,7 @@ static inline bool macb_is_gem(struct macb *bp)
 
 static inline bool gem_has_ptp(struct macb *bp)
 {
-	return !!(bp->caps & MACB_CAPS_GEM_HAS_PTP);
+	return IS_ENABLED(CONFIG_MACB_USE_HWSTAMP) && (bp->caps & MACB_CAPS_GEM_HAS_PTP);
 }
 
 /**
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
index e43d99ec50ba..29a1199dad14 100644
--- a/drivers/net/ethernet/cadence/macb_main.c
+++ b/drivers/net/ethernet/cadence/macb_main.c
@@ -94,8 +94,7 @@ struct sifive_fu540_macb_mgmt {
 /* Graceful stop timeouts in us. We should allow up to
  * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions)
  */
-#define MACB_HALT_TIMEOUT	1230
-
+#define MACB_HALT_TIMEOUT	14000
 #define MACB_PM_TIMEOUT  100 /* ms */
 
 #define MACB_MDIO_TIMEOUT	1000000 /* in usecs */
@@ -288,6 +287,11 @@ static void macb_set_hwaddr(struct macb *bp)
 	top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4)));
 	macb_or_gem_writel(bp, SA1T, top);
 
+	if (gem_has_ptp(bp)) {
+		gem_writel(bp, RXPTPUNI, bottom);
+		gem_writel(bp, TXPTPUNI, bottom);
+	}
+
 	/* Clear unused address register sets */
 	macb_or_gem_writel(bp, SA2B, 0);
 	macb_or_gem_writel(bp, SA2T, 0);
@@ -774,8 +778,12 @@ static void macb_mac_link_up(struct phylink_config *config,
 
 	spin_unlock_irqrestore(&bp->lock, flags);
 
-	/* Enable Rx and Tx */
-	macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE));
+	/* Enable Rx and Tx; Enable PTP unicast */
+	ctrl = macb_readl(bp, NCR);
+	if (gem_has_ptp(bp))
+		ctrl |= MACB_BIT(PTPUNI);
+
+	macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE));
 
 	netif_tx_wake_all_queues(ndev);
 }
@@ -1075,6 +1083,7 @@ static void macb_tx_error_task(struct work_struct *work)
 {
 	struct macb_queue	*queue = container_of(work, struct macb_queue,
 						      tx_error_task);
+	bool			halt_timeout = false;
 	struct macb		*bp = queue->bp;
 	struct macb_tx_skb	*tx_skb;
 	struct macb_dma_desc	*desc;
@@ -1102,9 +1111,11 @@ static void macb_tx_error_task(struct work_struct *work)
 	 * (in case we have just queued new packets)
 	 * macb/gem must be halted to write TBQP register
 	 */
-	if (macb_halt_tx(bp))
-		/* Just complain for now, reinitializing TX path can be good */
+	if (macb_halt_tx(bp)) {
 		netdev_err(bp->dev, "BUG: halt tx timed out\n");
+		macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE)));
+		halt_timeout = true;
+	}
 
 	/* Treat frames in TX queue including the ones that caused the error.
 	 * Free transmit buffers in upper layer.
@@ -1175,6 +1186,9 @@ static void macb_tx_error_task(struct work_struct *work)
 	macb_writel(bp, TSR, macb_readl(bp, TSR));
 	queue_writel(queue, IER, MACB_TX_INT_FLAGS);
 
+	if (halt_timeout)
+		macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE));
+
 	/* Now we are ready to start transmission again */
 	netif_tx_start_all_queues(bp->dev);
 	macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
@@ -2645,8 +2659,12 @@ static u32 gem_mdc_clk_div(struct macb *bp)
 		config = GEM_BF(CLK, GEM_CLK_DIV48);
 	else if (pclk_hz <= 160000000)
 		config = GEM_BF(CLK, GEM_CLK_DIV64);
-	else
+	else if (pclk_hz <= 240000000)
 		config = GEM_BF(CLK, GEM_CLK_DIV96);
+	else if (pclk_hz <= 320000000)
+		config = GEM_BF(CLK, GEM_CLK_DIV128);
+	else
+		config = GEM_BF(CLK, GEM_CLK_DIV224);
 
 	return config;
 }
@@ -3884,17 +3902,17 @@ static void macb_configure_caps(struct macb *bp,
 		dcfg = gem_readl(bp, DCFG2);
 		if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0)
 			bp->caps |= MACB_CAPS_FIFO_MODE;
-#ifdef CONFIG_MACB_USE_HWSTAMP
 		if (gem_has_ptp(bp)) {
 			if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5)))
 				dev_err(&bp->pdev->dev,
 					"GEM doesn't support hardware ptp.\n");
 			else {
+#ifdef CONFIG_MACB_USE_HWSTAMP
 				bp->hw_dma_cap |= HW_DMA_CAP_PTP;
 				bp->ptp_info = &gem_ptp_info;
+#endif
 			}
 		}
-#endif
 	}
 
 	dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps);
@@ -4848,7 +4866,7 @@ static const struct macb_config mpfs_config = {
 
 static const struct macb_config sama7g5_gem_config = {
 	.caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG |
-		MACB_CAPS_MIIONRGMII,
+		MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP,
 	.dma_burst_length = 16,
 	.clk_init = macb_clk_init,
 	.init = macb_init,
@@ -4857,7 +4875,8 @@ static const struct macb_config sama7g5_gem_config = {
 
 static const struct macb_config sama7g5_emac_config = {
 	.caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII |
-		MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII,
+		MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII |
+		MACB_CAPS_GEM_HAS_PTP,
 	.dma_burst_length = 16,
 	.clk_init = macb_clk_init,
 	.init = macb_init,
diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c
index f962a95068a0..51d26fa190d7 100644
--- a/drivers/net/ethernet/cadence/macb_ptp.c
+++ b/drivers/net/ethernet/cadence/macb_ptp.c
@@ -258,6 +258,8 @@ static int gem_hw_timestamp(struct macb *bp, u32 dma_desc_ts_1,
 	 */
 	gem_tsu_get_time(&bp->ptp_clock_info, &tsu, NULL);
 
+	ts->tv_sec |= ((~GEM_DMA_SEC_MASK) & tsu.tv_sec);
+
 	/* If the top bit is set in the timestamp,
 	 * but not in 1588 timer, it has rolled over,
 	 * so subtract max size
@@ -266,8 +268,6 @@ static int gem_hw_timestamp(struct macb *bp, u32 dma_desc_ts_1,
 	    !(tsu.tv_sec & (GEM_DMA_SEC_TOP >> 1)))
 		ts->tv_sec -= GEM_DMA_SEC_TOP;
 
-	ts->tv_sec += ((~GEM_DMA_SEC_MASK) & tsu.tv_sec);
-
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c
index fd7c80edb6e8..9bd1d2d7027d 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c
@@ -1129,7 +1129,6 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 
 		fallthrough;
 	case OCT_DEV_PCI_ENABLE_DONE:
-		pci_clear_master(oct->pci_dev);
 		/* Disable the device, releasing the PCI INT */
 		pci_disable_device(oct->pci_dev);
 
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
index ac196883f07e..e2921aec3da0 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
@@ -577,7 +577,6 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 
 		fallthrough;
 	case OCT_DEV_PCI_ENABLE_DONE:
-		pci_clear_master(oct->pci_dev);
 		/* Disable the device, releasing the PCI INT */
 		pci_disable_device(oct->pci_dev);
 
diff --git a/drivers/net/ethernet/cavium/liquidio/request_manager.c b/drivers/net/ethernet/cavium/liquidio/request_manager.c
index 8e59c2825533..32f854c0cd79 100644
--- a/drivers/net/ethernet/cavium/liquidio/request_manager.c
+++ b/drivers/net/ethernet/cavium/liquidio/request_manager.c
@@ -40,15 +40,6 @@ static void  __check_db_timeout(struct octeon_device *oct, u64 iq_no);
 
 static void (*reqtype_free_fn[MAX_OCTEON_DEVICES][REQTYPE_LAST + 1]) (void *);
 
-static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no)
-{
-	struct octeon_instr_queue *iq =
-	    (struct octeon_instr_queue *)oct->instr_queue[iq_no];
-	return iq->iqcmd_64B;
-}
-
-#define IQ_INSTR_MODE_32B(oct, iq_no)  (!IQ_INSTR_MODE_64B(oct, iq_no))
-
 /* Define this to return the request status comaptible to old code */
 /*#define OCTEON_USE_OLD_REQ_STATUS*/
 
diff --git a/drivers/net/ethernet/chelsio/cxgb3/sge.c b/drivers/net/ethernet/chelsio/cxgb3/sge.c
index 62dfbdd33365..efa7f401529e 100644
--- a/drivers/net/ethernet/chelsio/cxgb3/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb3/sge.c
@@ -166,11 +166,6 @@ static u8 flit_desc_map[] = {
 #endif
 };
 
-static inline struct sge_qset *fl_to_qset(const struct sge_fl *q, int qidx)
-{
-	return container_of(q, struct sge_qset, fl[qidx]);
-}
-
 static inline struct sge_qset *rspq_to_qset(const struct sge_rspq *q)
 {
 	return container_of(q, struct sge_qset, rspq);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 7db2403c4c9c..f0bc7396ce2b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -51,7 +51,6 @@
 #include <linux/mutex.h>
 #include <linux/netdevice.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/rtnetlink.h>
 #include <linux/sched.h>
 #include <linux/seq_file.h>
@@ -6687,7 +6686,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto out_free_adapter;
 	}
 
-	pci_enable_pcie_error_reporting(pdev);
 	pci_set_master(pdev);
 	pci_save_state(pdev);
 	adap_idx++;
@@ -7092,7 +7090,6 @@ fw_attach_fail:
  out_unmap_bar0:
 	iounmap(regs);
  out_disable_device:
-	pci_disable_pcie_error_reporting(pdev);
 	pci_disable_device(pdev);
  out_release_regions:
 	pci_release_regions(pdev);
@@ -7171,7 +7168,6 @@ static void remove_one(struct pci_dev *pdev)
 	}
 #endif
 	iounmap(adapter->regs);
-	pci_disable_pcie_error_reporting(pdev);
 	if ((adapter->flags & CXGB4_DEV_ENABLED)) {
 		pci_disable_device(pdev);
 		adapter->flags &= ~CXGB4_DEV_ENABLED;
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c
index dd9be229819a..d3541159487d 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c
@@ -1135,7 +1135,7 @@ void cxgb4_cleanup_tc_flower(struct adapter *adap)
 		return;
 
 	if (adap->flower_stats_timer.function)
-		del_timer_sync(&adap->flower_stats_timer);
+		timer_shutdown_sync(&adap->flower_stats_timer);
 	cancel_work_sync(&adap->flower_stats_work);
 	rhashtable_destroy(&adap->flower_tbl);
 	adap->tc_flower_initialized = false;
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c
index 95e1b415ba13..dea9d2907666 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c
@@ -12,7 +12,7 @@
 static int cxgb4_thermal_get_temp(struct thermal_zone_device *tzdev,
 				  int *temp)
 {
-	struct adapter *adap = tzdev->devdata;
+	struct adapter *adap = thermal_zone_device_priv(tzdev);
 	u32 param, val;
 	int ret;
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
index 63b2bd084130..9ba0864592e8 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
@@ -3258,7 +3258,6 @@ err_free_adapter:
 
 err_release_regions:
 	pci_release_regions(pdev);
-	pci_clear_master(pdev);
 
 err_disable_device:
 	pci_disable_device(pdev);
@@ -3338,7 +3337,6 @@ static void cxgb4vf_pci_remove(struct pci_dev *pdev)
 	 * Disable the device and release its PCI resources.
 	 */
 	pci_disable_device(pdev);
-	pci_clear_master(pdev);
 	pci_release_regions(pdev);
 }
 
diff --git a/drivers/net/ethernet/davicom/dm9051.c b/drivers/net/ethernet/davicom/dm9051.c
index de7105a84747..70728b2e5f18 100644
--- a/drivers/net/ethernet/davicom/dm9051.c
+++ b/drivers/net/ethernet/davicom/dm9051.c
@@ -1123,7 +1123,7 @@ static int dm9051_mdio_register(struct board_info *db)
 	db->mdiobus->phy_mask = (u32)~BIT(1);
 	db->mdiobus->parent = &spi->dev;
 	snprintf(db->mdiobus->id, MII_BUS_ID_SIZE,
-		 "dm9051-%s.%u", dev_name(&spi->dev), spi->chip_select);
+		 "dm9051-%s.%u", dev_name(&spi->dev), spi_get_chipselect(spi, 0));
 
 	ret = devm_mdiobus_register(&spi->dev, db->mdiobus);
 	if (ret)
diff --git a/drivers/net/ethernet/ec_bhf.c b/drivers/net/ethernet/ec_bhf.c
index 46e3a05e9582..c2c5c589a5e3 100644
--- a/drivers/net/ethernet/ec_bhf.c
+++ b/drivers/net/ethernet/ec_bhf.c
@@ -558,7 +558,6 @@ err_unmap:
 err_release_regions:
 	pci_release_regions(dev);
 err_disable_dev:
-	pci_clear_master(dev);
 	pci_disable_device(dev);
 
 	return err;
@@ -577,7 +576,6 @@ static void ec_bhf_remove(struct pci_dev *dev)
 	free_netdev(net_dev);
 
 	pci_release_regions(dev);
-	pci_clear_master(dev);
 	pci_disable_device(dev);
 }
 
diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c
index 08ec84cd21c0..61adcebeef01 100644
--- a/drivers/net/ethernet/emulex/benet/be_cmds.c
+++ b/drivers/net/ethernet/emulex/benet/be_cmds.c
@@ -135,7 +135,8 @@ static int be_mcc_notify(struct be_adapter *adapter)
 
 /* To check if valid bit is set, check the entire word as we don't know
  * the endianness of the data (old entry is host endian while a new entry is
- * little endian) */
+ * little endian)
+ */
 static inline bool be_mcc_compl_is_new(struct be_mcc_compl *compl)
 {
 	u32 flags;
@@ -248,7 +249,8 @@ static int be_mcc_compl_process(struct be_adapter *adapter,
 	u8 opcode = 0, subsystem = 0;
 
 	/* Just swap the status to host endian; mcc tag is opaquely copied
-	 * from mcc_wrb */
+	 * from mcc_wrb
+	 */
 	be_dws_le_to_cpu(compl, 4);
 
 	base_status = base_status(compl->status);
@@ -657,8 +659,7 @@ static int be_mbox_db_ready_wait(struct be_adapter *adapter, void __iomem *db)
 	return 0;
 }
 
-/*
- * Insert the mailbox address into the doorbell in two steps
+/* Insert the mailbox address into the doorbell in two steps
  * Polls on the mbox doorbell till a command completion (or a timeout) occurs
  */
 static int be_mbox_notify_wait(struct be_adapter *adapter)
@@ -802,7 +803,7 @@ static void be_wrb_cmd_hdr_prepare(struct be_cmd_req_hdr *req_hdr,
 	req_hdr->subsystem = subsystem;
 	req_hdr->request_length = cpu_to_le32(cmd_len - sizeof(*req_hdr));
 	req_hdr->version = 0;
-	fill_wrb_tags(wrb, (ulong) req_hdr);
+	fill_wrb_tags(wrb, (ulong)req_hdr);
 	wrb->payload_length = cmd_len;
 	if (mem) {
 		wrb->embedded |= (1 & MCC_WRB_SGE_CNT_MASK) <<
@@ -832,8 +833,8 @@ static void be_cmd_page_addrs_prepare(struct phys_addr *pages, u32 max_pages,
 static inline struct be_mcc_wrb *wrb_from_mbox(struct be_adapter *adapter)
 {
 	struct be_dma_mem *mbox_mem = &adapter->mbox_mem;
-	struct be_mcc_wrb *wrb
-		= &((struct be_mcc_mailbox *)(mbox_mem->va))->wrb;
+	struct be_mcc_wrb *wrb = &((struct be_mcc_mailbox *)(mbox_mem->va))->wrb;
+
 	memset(wrb, 0, sizeof(*wrb));
 	return wrb;
 }
@@ -896,7 +897,7 @@ static struct be_mcc_wrb *be_cmd_copy(struct be_adapter *adapter,
 
 	memcpy(dest_wrb, wrb, sizeof(*wrb));
 	if (wrb->embedded & cpu_to_le32(MCC_WRB_EMBEDDED_MASK))
-		fill_wrb_tags(dest_wrb, (ulong) embedded_payload(wrb));
+		fill_wrb_tags(dest_wrb, (ulong)embedded_payload(wrb));
 
 	return dest_wrb;
 }
@@ -1114,7 +1115,7 @@ int be_cmd_pmac_add(struct be_adapter *adapter, const u8 *mac_addr,
 err:
 	mutex_unlock(&adapter->mcc_lock);
 
-	 if (base_status(status) == MCC_STATUS_UNAUTHORIZED_REQUEST)
+	if (base_status(status) == MCC_STATUS_UNAUTHORIZED_REQUEST)
 		status = -EPERM;
 
 	return status;
@@ -1803,7 +1804,7 @@ int be_cmd_get_fat_dump(struct be_adapter *adapter, u32 buf_len, void *buf)
 
 	total_size = buf_len;
 
-	get_fat_cmd.size = sizeof(struct be_cmd_req_get_fat) + 60*1024;
+	get_fat_cmd.size = sizeof(struct be_cmd_req_get_fat) + 60 * 1024;
 	get_fat_cmd.va = dma_alloc_coherent(&adapter->pdev->dev,
 					    get_fat_cmd.size,
 					    &get_fat_cmd.dma, GFP_ATOMIC);
@@ -1813,7 +1814,7 @@ int be_cmd_get_fat_dump(struct be_adapter *adapter, u32 buf_len, void *buf)
 	mutex_lock(&adapter->mcc_lock);
 
 	while (total_size) {
-		buf_size = min(total_size, (u32)60*1024);
+		buf_size = min(total_size, (u32)60 * 1024);
 		total_size -= buf_size;
 
 		wrb = wrb_from_mccq(adapter);
@@ -3362,7 +3363,7 @@ int be_cmd_ddr_dma_test(struct be_adapter *adapter, u64 pattern,
 	req->pattern = cpu_to_le64(pattern);
 	req->byte_count = cpu_to_le32(byte_cnt);
 	for (i = 0; i < byte_cnt; i++) {
-		req->snd_buff[i] = (u8)(pattern >> (j*8));
+		req->snd_buff[i] = (u8)(pattern >> (j * 8));
 		j++;
 		if (j > 7)
 			j = 0;
@@ -3846,7 +3847,7 @@ int be_cmd_set_mac_list(struct be_adapter *adapter, u8 *mac_array,
 	req->hdr.domain = domain;
 	req->mac_count = mac_count;
 	if (mac_count)
-		memcpy(req->mac, mac_array, ETH_ALEN*mac_count);
+		memcpy(req->mac, mac_array, ETH_ALEN * mac_count);
 
 	status = be_mcc_notify_wait(adapter);
 
diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c
index 46fe3d74e2e9..7e408bcc88de 100644
--- a/drivers/net/ethernet/emulex/benet/be_main.c
+++ b/drivers/net/ethernet/emulex/benet/be_main.c
@@ -16,7 +16,6 @@
 #include "be.h"
 #include "be_cmds.h"
 #include <asm/div64.h>
-#include <linux/aer.h>
 #include <linux/if_bridge.h>
 #include <net/busy_poll.h>
 #include <net/vxlan.h>
@@ -1125,7 +1124,7 @@ static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 						  struct be_wrb_params
 						  *wrb_params)
 {
-	struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
+	struct vlan_ethhdr *veh = skb_vlan_eth_hdr(skb);
 	unsigned int eth_hdr_len;
 	struct iphdr *ip;
 
@@ -5726,8 +5725,6 @@ static void be_remove(struct pci_dev *pdev)
 	be_unmap_pci_bars(adapter);
 	be_drv_cleanup(adapter);
 
-	pci_disable_pcie_error_reporting(pdev);
-
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
 
@@ -5845,10 +5842,6 @@ static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
 		goto free_netdev;
 	}
 
-	status = pci_enable_pcie_error_reporting(pdev);
-	if (!status)
-		dev_info(&pdev->dev, "PCIe error reporting enabled\n");
-
 	status = be_map_pci_bars(adapter);
 	if (status)
 		goto free_netdev;
@@ -5893,7 +5886,6 @@ drv_cleanup:
 unmap_bars:
 	be_unmap_pci_bars(adapter);
 free_netdev:
-	pci_disable_pcie_error_reporting(pdev);
 	free_netdev(netdev);
 rel_reg:
 	pci_release_regions(pdev);
diff --git a/drivers/net/ethernet/engleder/tsnep.h b/drivers/net/ethernet/engleder/tsnep.h
index 058c2bcf31a7..11b29f56aaf9 100644
--- a/drivers/net/ethernet/engleder/tsnep.h
+++ b/drivers/net/ethernet/engleder/tsnep.h
@@ -18,6 +18,7 @@
 #define TSNEP "tsnep"
 
 #define TSNEP_RING_SIZE 256
+#define TSNEP_RING_MASK (TSNEP_RING_SIZE - 1)
 #define TSNEP_RING_RX_REFILL 16
 #define TSNEP_RING_RX_REUSE (TSNEP_RING_SIZE - TSNEP_RING_SIZE / 4)
 #define TSNEP_RING_ENTRIES_PER_PAGE (PAGE_SIZE / TSNEP_DESC_SIZE)
@@ -69,6 +70,7 @@ struct tsnep_tx_entry {
 	union {
 		struct sk_buff *skb;
 		struct xdp_frame *xdpf;
+		bool zc;
 	};
 	size_t len;
 	DEFINE_DMA_UNMAP_ADDR(dma);
@@ -87,6 +89,7 @@ struct tsnep_tx {
 	int read;
 	u32 owner_counter;
 	int increment_owner_counter;
+	struct xsk_buff_pool *xsk_pool;
 
 	u32 packets;
 	u32 bytes;
@@ -100,7 +103,10 @@ struct tsnep_rx_entry {
 
 	u32 properties;
 
-	struct page *page;
+	union {
+		struct page *page;
+		struct xdp_buff *xdp;
+	};
 	size_t len;
 	dma_addr_t dma;
 };
@@ -120,6 +126,9 @@ struct tsnep_rx {
 	u32 owner_counter;
 	int increment_owner_counter;
 	struct page_pool *page_pool;
+	struct page **page_buffer;
+	struct xsk_buff_pool *xsk_pool;
+	struct xdp_buff **xdp_batch;
 
 	u32 packets;
 	u32 bytes;
@@ -128,6 +137,7 @@ struct tsnep_rx {
 	u32 alloc_failed;
 
 	struct xdp_rxq_info xdp_rxq;
+	struct xdp_rxq_info xdp_rxq_zc;
 };
 
 struct tsnep_queue {
@@ -213,6 +223,8 @@ int tsnep_rxnfc_del_rule(struct tsnep_adapter *adapter,
 
 int tsnep_xdp_setup_prog(struct tsnep_adapter *adapter, struct bpf_prog *prog,
 			 struct netlink_ext_ack *extack);
+int tsnep_xdp_setup_pool(struct tsnep_adapter *adapter,
+			 struct xsk_buff_pool *pool, u16 queue_id);
 
 #if IS_ENABLED(CONFIG_TSNEP_SELFTESTS)
 int tsnep_ethtool_get_test_count(void);
@@ -241,5 +253,7 @@ static inline void tsnep_ethtool_self_test(struct net_device *dev,
 void tsnep_get_system_time(struct tsnep_adapter *adapter, u64 *time);
 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs);
 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue);
+int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool);
+void tsnep_disable_xsk(struct tsnep_queue *queue);
 
 #endif /* _TSNEP_H */
diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c
index 6982aaa928b5..84751bb303a6 100644
--- a/drivers/net/ethernet/engleder/tsnep_main.c
+++ b/drivers/net/ethernet/engleder/tsnep_main.c
@@ -28,11 +28,16 @@
 #include <linux/iopoll.h>
 #include <linux/bpf.h>
 #include <linux/bpf_trace.h>
+#include <net/xdp_sock_drv.h>
 
 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)
 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4)
 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \
 			       SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
+/* XSK buffer shall store at least Q-in-Q frame */
+#define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \
+				     ETH_FRAME_LEN + ETH_FCS_LEN + \
+				     VLAN_HLEN * 2, 4))
 
 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF))
@@ -49,6 +54,8 @@
 #define TSNEP_TX_TYPE_SKB_FRAG	BIT(1)
 #define TSNEP_TX_TYPE_XDP_TX	BIT(2)
 #define TSNEP_TX_TYPE_XDP_NDO	BIT(3)
+#define TSNEP_TX_TYPE_XDP	(TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO)
+#define TSNEP_TX_TYPE_XSK	BIT(4)
 
 #define TSNEP_XDP_TX		BIT(0)
 #define TSNEP_XDP_REDIRECT	BIT(1)
@@ -246,7 +253,6 @@ static void tsnep_phy_close(struct tsnep_adapter *adapter)
 {
 	phy_stop(adapter->netdev->phydev);
 	phy_disconnect(adapter->netdev->phydev);
-	adapter->netdev->phydev = NULL;
 }
 
 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx)
@@ -266,7 +272,7 @@ static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx)
 	}
 }
 
-static int tsnep_tx_ring_init(struct tsnep_tx *tx)
+static int tsnep_tx_ring_create(struct tsnep_tx *tx)
 {
 	struct device *dmadev = tx->adapter->dmadev;
 	struct tsnep_tx_entry *entry;
@@ -289,11 +295,12 @@ static int tsnep_tx_ring_init(struct tsnep_tx *tx)
 			entry->desc = (struct tsnep_tx_desc *)
 				(((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
 			entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j;
+			entry->owner_user_flag = false;
 		}
 	}
 	for (i = 0; i < TSNEP_RING_SIZE; i++) {
 		entry = &tx->entry[i];
-		next_entry = &tx->entry[(i + 1) % TSNEP_RING_SIZE];
+		next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK];
 		entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
 	}
 
@@ -304,13 +311,60 @@ alloc_failed:
 	return retval;
 }
 
+static void tsnep_tx_init(struct tsnep_tx *tx)
+{
+	dma_addr_t dma;
+
+	dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
+	iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW);
+	iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH);
+	tx->write = 0;
+	tx->read = 0;
+	tx->owner_counter = 1;
+	tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
+}
+
+static void tsnep_tx_enable(struct tsnep_tx *tx)
+{
+	struct netdev_queue *nq;
+
+	nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
+
+	__netif_tx_lock_bh(nq);
+	netif_tx_wake_queue(nq);
+	__netif_tx_unlock_bh(nq);
+}
+
+static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi)
+{
+	struct netdev_queue *nq;
+	u32 val;
+
+	nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
+
+	__netif_tx_lock_bh(nq);
+	netif_tx_stop_queue(nq);
+	__netif_tx_unlock_bh(nq);
+
+	/* wait until TX is done in hardware */
+	readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,
+			   ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,
+			   1000000);
+
+	/* wait until TX is also done in software */
+	while (READ_ONCE(tx->read) != tx->write) {
+		napi_schedule(napi);
+		napi_synchronize(napi);
+	}
+}
+
 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length,
 			      bool last)
 {
 	struct tsnep_tx_entry *entry = &tx->entry[index];
 
 	entry->properties = 0;
-	/* xdpf is union with skb */
+	/* xdpf and zc are union with skb */
 	if (entry->skb) {
 		entry->properties = length & TSNEP_DESC_LENGTH_MASK;
 		entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
@@ -382,7 +436,7 @@ static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count)
 	int i;
 
 	for (i = 0; i < count; i++) {
-		entry = &tx->entry[(tx->write + i) % TSNEP_RING_SIZE];
+		entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
 
 		if (!i) {
 			len = skb_headlen(skb);
@@ -420,7 +474,7 @@ static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)
 	int i;
 
 	for (i = 0; i < count; i++) {
-		entry = &tx->entry[(index + i) % TSNEP_RING_SIZE];
+		entry = &tx->entry[(index + i) & TSNEP_RING_MASK];
 
 		if (entry->len) {
 			if (entry->type & TSNEP_TX_TYPE_SKB)
@@ -482,9 +536,9 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 
 	for (i = 0; i < count; i++)
-		tsnep_tx_activate(tx, (tx->write + i) % TSNEP_RING_SIZE, length,
+		tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
 				  i == count - 1);
-	tx->write = (tx->write + count) % TSNEP_RING_SIZE;
+	tx->write = (tx->write + count) & TSNEP_RING_MASK;
 
 	skb_tx_timestamp(skb);
 
@@ -517,7 +571,7 @@ static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx,
 	frag = NULL;
 	len = xdpf->len;
 	for (i = 0; i < count; i++) {
-		entry = &tx->entry[(tx->write + i) % TSNEP_RING_SIZE];
+		entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
 		if (type & TSNEP_TX_TYPE_XDP_NDO) {
 			data = unlikely(frag) ? skb_frag_address(frag) :
 						xdpf->data;
@@ -590,9 +644,9 @@ static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf,
 	length = retval;
 
 	for (i = 0; i < count; i++)
-		tsnep_tx_activate(tx, (tx->write + i) % TSNEP_RING_SIZE, length,
+		tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
 				  i == count - 1);
-	tx->write = (tx->write + count) % TSNEP_RING_SIZE;
+	tx->write = (tx->write + count) & TSNEP_RING_MASK;
 
 	/* descriptor properties shall be valid before hardware is notified */
 	dma_wmb();
@@ -628,10 +682,69 @@ static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter,
 	return xmit;
 }
 
+static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx)
+{
+	struct tsnep_tx_entry *entry;
+	dma_addr_t dma;
+
+	entry = &tx->entry[tx->write];
+	entry->zc = true;
+
+	dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr);
+	xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len);
+
+	entry->type = TSNEP_TX_TYPE_XSK;
+	entry->len = xdpd->len;
+
+	entry->desc->tx = __cpu_to_le64(dma);
+
+	return xdpd->len;
+}
+
+static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd,
+					 struct tsnep_tx *tx)
+{
+	int length;
+
+	length = tsnep_xdp_tx_map_zc(xdpd, tx);
+
+	tsnep_tx_activate(tx, tx->write, length, true);
+	tx->write = (tx->write + 1) & TSNEP_RING_MASK;
+}
+
+static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx)
+{
+	int desc_available = tsnep_tx_desc_available(tx);
+	struct xdp_desc *descs = tx->xsk_pool->tx_descs;
+	int batch, i;
+
+	/* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
+	 * will be available for normal TX path and queue is stopped there if
+	 * necessary
+	 */
+	if (desc_available <= (MAX_SKB_FRAGS + 1))
+		return;
+	desc_available -= MAX_SKB_FRAGS + 1;
+
+	batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available);
+	for (i = 0; i < batch; i++)
+		tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx);
+
+	if (batch) {
+		/* descriptor properties shall be valid before hardware is
+		 * notified
+		 */
+		dma_wmb();
+
+		tsnep_xdp_xmit_flush(tx);
+	}
+}
+
 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
 {
 	struct tsnep_tx_entry *entry;
 	struct netdev_queue *nq;
+	int xsk_frames = 0;
 	int budget = 128;
 	int length;
 	int count;
@@ -658,7 +771,7 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
 		if ((entry->type & TSNEP_TX_TYPE_SKB) &&
 		    skb_shinfo(entry->skb)->nr_frags > 0)
 			count += skb_shinfo(entry->skb)->nr_frags;
-		else if (!(entry->type & TSNEP_TX_TYPE_SKB) &&
+		else if ((entry->type & TSNEP_TX_TYPE_XDP) &&
 			 xdp_frame_has_frags(entry->xdpf))
 			count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags;
 
@@ -687,12 +800,14 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
 
 		if (entry->type & TSNEP_TX_TYPE_SKB)
 			napi_consume_skb(entry->skb, napi_budget);
-		else
+		else if (entry->type & TSNEP_TX_TYPE_XDP)
 			xdp_return_frame_rx_napi(entry->xdpf);
-		/* xdpf is union with skb */
+		else
+			xsk_frames++;
+		/* xdpf and zc are union with skb */
 		entry->skb = NULL;
 
-		tx->read = (tx->read + count) % TSNEP_RING_SIZE;
+		tx->read = (tx->read + count) & TSNEP_RING_MASK;
 
 		tx->packets++;
 		tx->bytes += length + ETH_FCS_LEN;
@@ -700,6 +815,14 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
 		budget--;
 	} while (likely(budget));
 
+	if (tx->xsk_pool) {
+		if (xsk_frames)
+			xsk_tx_completed(tx->xsk_pool, xsk_frames);
+		if (xsk_uses_need_wakeup(tx->xsk_pool))
+			xsk_set_tx_need_wakeup(tx->xsk_pool);
+		tsnep_xdp_xmit_zc(tx);
+	}
+
 	if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) &&
 	    netif_tx_queue_stopped(nq)) {
 		netif_tx_wake_queue(nq);
@@ -732,38 +855,21 @@ static bool tsnep_tx_pending(struct tsnep_tx *tx)
 	return pending;
 }
 
-static int tsnep_tx_open(struct tsnep_adapter *adapter, void __iomem *addr,
-			 int queue_index, struct tsnep_tx *tx)
+static int tsnep_tx_open(struct tsnep_tx *tx)
 {
-	dma_addr_t dma;
 	int retval;
 
-	memset(tx, 0, sizeof(*tx));
-	tx->adapter = adapter;
-	tx->addr = addr;
-	tx->queue_index = queue_index;
-
-	retval = tsnep_tx_ring_init(tx);
+	retval = tsnep_tx_ring_create(tx);
 	if (retval)
 		return retval;
 
-	dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
-	iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW);
-	iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH);
-	tx->owner_counter = 1;
-	tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
+	tsnep_tx_init(tx);
 
 	return 0;
 }
 
 static void tsnep_tx_close(struct tsnep_tx *tx)
 {
-	u32 val;
-
-	readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,
-			   ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,
-			   1000000);
-
 	tsnep_tx_ring_cleanup(tx);
 }
 
@@ -775,9 +881,12 @@ static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
 
 	for (i = 0; i < TSNEP_RING_SIZE; i++) {
 		entry = &rx->entry[i];
-		if (entry->page)
+		if (!rx->xsk_pool && entry->page)
 			page_pool_put_full_page(rx->page_pool, entry->page,
 						false);
+		if (rx->xsk_pool && entry->xdp)
+			xsk_buff_free(entry->xdp);
+		/* xdp is union with page */
 		entry->page = NULL;
 	}
 
@@ -796,7 +905,7 @@ static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
 	}
 }
 
-static int tsnep_rx_ring_init(struct tsnep_rx *rx)
+static int tsnep_rx_ring_create(struct tsnep_rx *rx)
 {
 	struct device *dmadev = rx->adapter->dmadev;
 	struct tsnep_rx_entry *entry;
@@ -840,7 +949,7 @@ static int tsnep_rx_ring_init(struct tsnep_rx *rx)
 
 	for (i = 0; i < TSNEP_RING_SIZE; i++) {
 		entry = &rx->entry[i];
-		next_entry = &rx->entry[(i + 1) % TSNEP_RING_SIZE];
+		next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK];
 		entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
 	}
 
@@ -851,6 +960,37 @@ failed:
 	return retval;
 }
 
+static void tsnep_rx_init(struct tsnep_rx *rx)
+{
+	dma_addr_t dma;
+
+	dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
+	iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);
+	iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);
+	rx->write = 0;
+	rx->read = 0;
+	rx->owner_counter = 1;
+	rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
+}
+
+static void tsnep_rx_enable(struct tsnep_rx *rx)
+{
+	/* descriptor properties shall be valid before hardware is notified */
+	dma_wmb();
+
+	iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
+}
+
+static void tsnep_rx_disable(struct tsnep_rx *rx)
+{
+	u32 val;
+
+	iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);
+	readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,
+			   ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000,
+			   1000000);
+}
+
 static int tsnep_rx_desc_available(struct tsnep_rx *rx)
 {
 	if (rx->read <= rx->write)
@@ -859,6 +999,40 @@ static int tsnep_rx_desc_available(struct tsnep_rx *rx)
 		return rx->read - rx->write - 1;
 }
 
+static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx)
+{
+	struct page **page;
+
+	/* last entry of page_buffer is always zero, because ring cannot be
+	 * filled completely
+	 */
+	page = rx->page_buffer;
+	while (*page) {
+		page_pool_put_full_page(rx->page_pool, *page, false);
+		*page = NULL;
+		page++;
+	}
+}
+
+static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx)
+{
+	int i;
+
+	/* alloc for all ring entries except the last one, because ring cannot
+	 * be filled completely
+	 */
+	for (i = 0; i < TSNEP_RING_SIZE - 1; i++) {
+		rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool);
+		if (!rx->page_buffer[i]) {
+			tsnep_rx_free_page_buffer(rx);
+
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
 			      struct page *page)
 {
@@ -894,7 +1068,7 @@ static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
 {
 	struct tsnep_rx_entry *entry = &rx->entry[index];
 
-	/* TSNEP_MAX_RX_BUF_SIZE is a multiple of 4 */
+	/* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */
 	entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK;
 	entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
 	if (index == rx->increment_owner_counter) {
@@ -917,19 +1091,15 @@ static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
 	entry->desc->properties = __cpu_to_le32(entry->properties);
 }
 
-static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)
+static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse)
 {
-	int index;
 	bool alloc_failed = false;
-	bool enable = false;
-	int i;
-	int retval;
+	int i, index;
 
 	for (i = 0; i < count && !alloc_failed; i++) {
-		index = (rx->write + i) % TSNEP_RING_SIZE;
+		index = (rx->write + i) & TSNEP_RING_MASK;
 
-		retval = tsnep_rx_alloc_buffer(rx, index);
-		if (unlikely(retval)) {
+		if (unlikely(tsnep_rx_alloc_buffer(rx, index))) {
 			rx->alloc_failed++;
 			alloc_failed = true;
 
@@ -941,24 +1111,95 @@ static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)
 		}
 
 		tsnep_rx_activate(rx, index);
-
-		enable = true;
 	}
 
-	if (enable) {
-		rx->write = (rx->write + i) % TSNEP_RING_SIZE;
+	if (i)
+		rx->write = (rx->write + i) & TSNEP_RING_MASK;
 
-		/* descriptor properties shall be valid before hardware is
-		 * notified
-		 */
-		dma_wmb();
+	return i;
+}
+
+static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)
+{
+	int desc_refilled;
+
+	desc_refilled = tsnep_rx_alloc(rx, count, reuse);
+	if (desc_refilled)
+		tsnep_rx_enable(rx);
+
+	return desc_refilled;
+}
+
+static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
+			     struct xdp_buff *xdp)
+{
+	entry->xdp = xdp;
+	entry->len = TSNEP_XSK_RX_BUF_SIZE;
+	entry->dma = xsk_buff_xdp_get_dma(entry->xdp);
+	entry->desc->rx = __cpu_to_le64(entry->dma);
+}
 
-		iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
+static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index)
+{
+	struct tsnep_rx_entry *entry = &rx->entry[index];
+	struct tsnep_rx_entry *read = &rx->entry[rx->read];
+
+	tsnep_rx_set_xdp(rx, entry, read->xdp);
+	read->xdp = NULL;
+}
+
+static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse)
+{
+	u32 allocated;
+	int i;
+
+	allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count);
+	for (i = 0; i < allocated; i++) {
+		int index = (rx->write + i) & TSNEP_RING_MASK;
+		struct tsnep_rx_entry *entry = &rx->entry[index];
+
+		tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]);
+		tsnep_rx_activate(rx, index);
+	}
+	if (i == 0) {
+		rx->alloc_failed++;
+
+		if (reuse) {
+			tsnep_rx_reuse_buffer_zc(rx, rx->write);
+			tsnep_rx_activate(rx, rx->write);
+		}
 	}
 
+	if (i)
+		rx->write = (rx->write + i) & TSNEP_RING_MASK;
+
 	return i;
 }
 
+static void tsnep_rx_free_zc(struct tsnep_rx *rx)
+{
+	int i;
+
+	for (i = 0; i < TSNEP_RING_SIZE; i++) {
+		struct tsnep_rx_entry *entry = &rx->entry[i];
+
+		if (entry->xdp)
+			xsk_buff_free(entry->xdp);
+		entry->xdp = NULL;
+	}
+}
+
+static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse)
+{
+	int desc_refilled;
+
+	desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse);
+	if (desc_refilled)
+		tsnep_rx_enable(rx);
+
+	return desc_refilled;
+}
+
 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog,
 			       struct xdp_buff *xdp, int *status,
 			       struct netdev_queue *tx_nq, struct tsnep_tx *tx)
@@ -970,11 +1211,6 @@ static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog,
 	length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;
 
 	act = bpf_prog_run_xdp(prog, xdp);
-
-	/* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */
-	sync = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;
-	sync = max(sync, length);
-
 	switch (act) {
 	case XDP_PASS:
 		return false;
@@ -996,12 +1232,56 @@ out_failure:
 		trace_xdp_exception(rx->adapter->netdev, prog, act);
 		fallthrough;
 	case XDP_DROP:
+		/* Due xdp_adjust_tail: DMA sync for_device cover max len CPU
+		 * touch
+		 */
+		sync = xdp->data_end - xdp->data_hard_start -
+		       XDP_PACKET_HEADROOM;
+		sync = max(sync, length);
 		page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data),
 				   sync, true);
 		return true;
 	}
 }
 
+static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog,
+				  struct xdp_buff *xdp, int *status,
+				  struct netdev_queue *tx_nq,
+				  struct tsnep_tx *tx)
+{
+	u32 act;
+
+	act = bpf_prog_run_xdp(prog, xdp);
+
+	/* XDP_REDIRECT is the main action for zero-copy */
+	if (likely(act == XDP_REDIRECT)) {
+		if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
+			goto out_failure;
+		*status |= TSNEP_XDP_REDIRECT;
+		return true;
+	}
+
+	switch (act) {
+	case XDP_PASS:
+		return false;
+	case XDP_TX:
+		if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx))
+			goto out_failure;
+		*status |= TSNEP_XDP_TX;
+		return true;
+	default:
+		bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
+		fallthrough;
+	case XDP_ABORTED:
+out_failure:
+		trace_xdp_exception(rx->adapter->netdev, prog, act);
+		fallthrough;
+	case XDP_DROP:
+		xsk_buff_free(xdp);
+		return true;
+	}
+}
+
 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status,
 			       struct netdev_queue *tx_nq, struct tsnep_tx *tx)
 {
@@ -1046,6 +1326,28 @@ static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,
 	return skb;
 }
 
+static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi,
+			  struct page *page, int length)
+{
+	struct sk_buff *skb;
+
+	skb = tsnep_build_skb(rx, page, length);
+	if (skb) {
+		page_pool_release_page(rx->page_pool, page);
+
+		rx->packets++;
+		rx->bytes += length;
+		if (skb->pkt_type == PACKET_MULTICAST)
+			rx->multicast++;
+
+		napi_gro_receive(napi, skb);
+	} else {
+		page_pool_recycle_direct(rx->page_pool, page);
+
+		rx->dropped++;
+	}
+}
+
 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
 			 int budget)
 {
@@ -1055,7 +1357,6 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
 	struct netdev_queue *tx_nq;
 	struct bpf_prog *prog;
 	struct xdp_buff xdp;
-	struct sk_buff *skb;
 	struct tsnep_tx *tx;
 	int desc_available;
 	int xdp_status = 0;
@@ -1091,7 +1392,7 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
 				 * empty RX ring, thus buffer cannot be used for
 				 * RX processing
 				 */
-				rx->read = (rx->read + 1) % TSNEP_RING_SIZE;
+				rx->read = (rx->read + 1) & TSNEP_RING_MASK;
 				desc_available++;
 
 				rx->dropped++;
@@ -1118,7 +1419,7 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
 		 */
 		length -= TSNEP_RX_INLINE_METADATA_SIZE;
 
-		rx->read = (rx->read + 1) % TSNEP_RING_SIZE;
+		rx->read = (rx->read + 1) & TSNEP_RING_MASK;
 		desc_available++;
 
 		if (prog) {
@@ -1140,31 +1441,135 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
 			}
 		}
 
-		skb = tsnep_build_skb(rx, entry->page, length);
-		if (skb) {
-			page_pool_release_page(rx->page_pool, entry->page);
+		tsnep_rx_page(rx, napi, entry->page, length);
+		entry->page = NULL;
+	}
+
+	if (xdp_status)
+		tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
+
+	if (desc_available)
+		tsnep_rx_refill(rx, desc_available, false);
 
-			rx->packets++;
-			rx->bytes += length;
-			if (skb->pkt_type == PACKET_MULTICAST)
-				rx->multicast++;
+	return done;
+}
 
-			napi_gro_receive(napi, skb);
-		} else {
-			page_pool_recycle_direct(rx->page_pool, entry->page);
+static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi,
+			    int budget)
+{
+	struct tsnep_rx_entry *entry;
+	struct netdev_queue *tx_nq;
+	struct bpf_prog *prog;
+	struct tsnep_tx *tx;
+	int desc_available;
+	int xdp_status = 0;
+	struct page *page;
+	int done = 0;
+	int length;
+
+	desc_available = tsnep_rx_desc_available(rx);
+	prog = READ_ONCE(rx->adapter->xdp_prog);
+	if (prog) {
+		tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
+					    rx->tx_queue_index);
+		tx = &rx->adapter->tx[rx->tx_queue_index];
+	}
+
+	while (likely(done < budget) && (rx->read != rx->write)) {
+		entry = &rx->entry[rx->read];
+		if ((__le32_to_cpu(entry->desc_wb->properties) &
+		     TSNEP_DESC_OWNER_COUNTER_MASK) !=
+		    (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
+			break;
+		done++;
+
+		if (desc_available >= TSNEP_RING_RX_REFILL) {
+			bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
+
+			desc_available -= tsnep_rx_refill_zc(rx, desc_available,
+							     reuse);
+			if (!entry->xdp) {
+				/* buffer has been reused for refill to prevent
+				 * empty RX ring, thus buffer cannot be used for
+				 * RX processing
+				 */
+				rx->read = (rx->read + 1) & TSNEP_RING_MASK;
+				desc_available++;
+
+				rx->dropped++;
+
+				continue;
+			}
+		}
+
+		/* descriptor properties shall be read first, because valid data
+		 * is signaled there
+		 */
+		dma_rmb();
+
+		prefetch(entry->xdp->data);
+		length = __le32_to_cpu(entry->desc_wb->properties) &
+			 TSNEP_DESC_LENGTH_MASK;
+		xsk_buff_set_size(entry->xdp, length);
+		xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool);
+
+		/* RX metadata with timestamps is in front of actual data,
+		 * subtract metadata size to get length of actual data and
+		 * consider metadata size as offset of actual data during RX
+		 * processing
+		 */
+		length -= TSNEP_RX_INLINE_METADATA_SIZE;
+
+		rx->read = (rx->read + 1) & TSNEP_RING_MASK;
+		desc_available++;
+
+		if (prog) {
+			bool consume;
+
+			entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE;
+			entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE;
+
+			consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp,
+							&xdp_status, tx_nq, tx);
+			if (consume) {
+				rx->packets++;
+				rx->bytes += length;
+
+				entry->xdp = NULL;
 
+				continue;
+			}
+		}
+
+		page = page_pool_dev_alloc_pages(rx->page_pool);
+		if (page) {
+			memcpy(page_address(page) + TSNEP_RX_OFFSET,
+			       entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE,
+			       length + TSNEP_RX_INLINE_METADATA_SIZE);
+			tsnep_rx_page(rx, napi, page, length);
+		} else {
 			rx->dropped++;
 		}
-		entry->page = NULL;
+		xsk_buff_free(entry->xdp);
+		entry->xdp = NULL;
 	}
 
 	if (xdp_status)
 		tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
 
 	if (desc_available)
-		tsnep_rx_refill(rx, desc_available, false);
+		desc_available -= tsnep_rx_refill_zc(rx, desc_available, false);
 
-	return done;
+	if (xsk_uses_need_wakeup(rx->xsk_pool)) {
+		if (desc_available)
+			xsk_set_rx_need_wakeup(rx->xsk_pool);
+		else
+			xsk_clear_rx_need_wakeup(rx->xsk_pool);
+
+		return done;
+	}
+
+	return desc_available ? budget : done;
 }
 
 static bool tsnep_rx_pending(struct tsnep_rx *rx)
@@ -1182,44 +1587,125 @@ static bool tsnep_rx_pending(struct tsnep_rx *rx)
 	return false;
 }
 
-static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr,
-			 int queue_index, struct tsnep_rx *rx)
+static int tsnep_rx_open(struct tsnep_rx *rx)
 {
-	dma_addr_t dma;
+	int desc_available;
 	int retval;
 
-	memset(rx, 0, sizeof(*rx));
-	rx->adapter = adapter;
-	rx->addr = addr;
-	rx->queue_index = queue_index;
-
-	retval = tsnep_rx_ring_init(rx);
+	retval = tsnep_rx_ring_create(rx);
 	if (retval)
 		return retval;
 
-	dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
-	iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);
-	iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);
-	rx->owner_counter = 1;
-	rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
+	tsnep_rx_init(rx);
 
-	tsnep_rx_refill(rx, tsnep_rx_desc_available(rx), false);
+	desc_available = tsnep_rx_desc_available(rx);
+	if (rx->xsk_pool)
+		retval = tsnep_rx_alloc_zc(rx, desc_available, false);
+	else
+		retval = tsnep_rx_alloc(rx, desc_available, false);
+	if (retval != desc_available) {
+		retval = -ENOMEM;
+
+		goto alloc_failed;
+	}
+
+	/* prealloc pages to prevent allocation failures when XSK pool is
+	 * disabled at runtime
+	 */
+	if (rx->xsk_pool) {
+		retval = tsnep_rx_alloc_page_buffer(rx);
+		if (retval)
+			goto alloc_failed;
+	}
 
 	return 0;
+
+alloc_failed:
+	tsnep_rx_ring_cleanup(rx);
+	return retval;
 }
 
 static void tsnep_rx_close(struct tsnep_rx *rx)
 {
-	u32 val;
-
-	iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);
-	readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,
-			   ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000,
-			   1000000);
+	if (rx->xsk_pool)
+		tsnep_rx_free_page_buffer(rx);
 
 	tsnep_rx_ring_cleanup(rx);
 }
 
+static void tsnep_rx_reopen(struct tsnep_rx *rx)
+{
+	struct page **page = rx->page_buffer;
+	int i;
+
+	tsnep_rx_init(rx);
+
+	for (i = 0; i < TSNEP_RING_SIZE; i++) {
+		struct tsnep_rx_entry *entry = &rx->entry[i];
+
+		/* defined initial values for properties are required for
+		 * correct owner counter checking
+		 */
+		entry->desc->properties = 0;
+		entry->desc_wb->properties = 0;
+
+		/* prevent allocation failures by reusing kept pages */
+		if (*page) {
+			tsnep_rx_set_page(rx, entry, *page);
+			tsnep_rx_activate(rx, rx->write);
+			rx->write++;
+
+			*page = NULL;
+			page++;
+		}
+	}
+}
+
+static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx)
+{
+	struct page **page = rx->page_buffer;
+	u32 allocated;
+	int i;
+
+	tsnep_rx_init(rx);
+
+	/* alloc all ring entries except the last one, because ring cannot be
+	 * filled completely, as many buffers as possible is enough as wakeup is
+	 * done if new buffers are available
+	 */
+	allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch,
+					 TSNEP_RING_SIZE - 1);
+
+	for (i = 0; i < TSNEP_RING_SIZE; i++) {
+		struct tsnep_rx_entry *entry = &rx->entry[i];
+
+		/* keep pages to prevent allocation failures when xsk is
+		 * disabled
+		 */
+		if (entry->page) {
+			*page = entry->page;
+			entry->page = NULL;
+
+			page++;
+		}
+
+		/* defined initial values for properties are required for
+		 * correct owner counter checking
+		 */
+		entry->desc->properties = 0;
+		entry->desc_wb->properties = 0;
+
+		if (allocated) {
+			tsnep_rx_set_xdp(rx, entry,
+					 rx->xdp_batch[allocated - 1]);
+			tsnep_rx_activate(rx, rx->write);
+			rx->write++;
+
+			allocated--;
+		}
+	}
+}
+
 static bool tsnep_pending(struct tsnep_queue *queue)
 {
 	if (queue->tx && tsnep_tx_pending(queue->tx))
@@ -1242,7 +1728,9 @@ static int tsnep_poll(struct napi_struct *napi, int budget)
 		complete = tsnep_tx_poll(queue->tx, budget);
 
 	if (queue->rx) {
-		done = tsnep_rx_poll(queue->rx, napi, budget);
+		done = queue->rx->xsk_pool ?
+		       tsnep_rx_poll_zc(queue->rx, napi, budget) :
+		       tsnep_rx_poll(queue->rx, napi, budget);
 		if (done >= budget)
 			complete = false;
 	}
@@ -1323,8 +1811,12 @@ static void tsnep_queue_close(struct tsnep_queue *queue, bool first)
 
 	tsnep_free_irq(queue, first);
 
-	if (rx && xdp_rxq_info_is_reg(&rx->xdp_rxq))
-		xdp_rxq_info_unreg(&rx->xdp_rxq);
+	if (rx) {
+		if (xdp_rxq_info_is_reg(&rx->xdp_rxq))
+			xdp_rxq_info_unreg(&rx->xdp_rxq);
+		if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc))
+			xdp_rxq_info_unreg(&rx->xdp_rxq_zc);
+	}
 
 	netif_napi_del(&queue->napi);
 }
@@ -1336,8 +1828,6 @@ static int tsnep_queue_open(struct tsnep_adapter *adapter,
 	struct tsnep_tx *tx = queue->tx;
 	int retval;
 
-	queue->adapter = adapter;
-
 	netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll);
 
 	if (rx) {
@@ -1349,6 +1839,10 @@ static int tsnep_queue_open(struct tsnep_adapter *adapter,
 		else
 			rx->tx_queue_index = 0;
 
+		/* prepare both memory models to eliminate possible registration
+		 * errors when memory model is switched between page pool and
+		 * XSK pool during runtime
+		 */
 		retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev,
 					  rx->queue_index, queue->napi.napi_id);
 		if (retval)
@@ -1358,6 +1852,17 @@ static int tsnep_queue_open(struct tsnep_adapter *adapter,
 						    rx->page_pool);
 		if (retval)
 			goto failed;
+		retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev,
+					  rx->queue_index, queue->napi.napi_id);
+		if (retval)
+			goto failed;
+		retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc,
+						    MEM_TYPE_XSK_BUFF_POOL,
+						    NULL);
+		if (retval)
+			goto failed;
+		if (rx->xsk_pool)
+			xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc);
 	}
 
 	retval = tsnep_request_irq(queue, first);
@@ -1375,30 +1880,48 @@ failed:
 	return retval;
 }
 
+static void tsnep_queue_enable(struct tsnep_queue *queue)
+{
+	napi_enable(&queue->napi);
+	tsnep_enable_irq(queue->adapter, queue->irq_mask);
+
+	if (queue->tx)
+		tsnep_tx_enable(queue->tx);
+
+	if (queue->rx)
+		tsnep_rx_enable(queue->rx);
+}
+
+static void tsnep_queue_disable(struct tsnep_queue *queue)
+{
+	if (queue->tx)
+		tsnep_tx_disable(queue->tx, &queue->napi);
+
+	napi_disable(&queue->napi);
+	tsnep_disable_irq(queue->adapter, queue->irq_mask);
+
+	/* disable RX after NAPI polling has been disabled, because RX can be
+	 * enabled during NAPI polling
+	 */
+	if (queue->rx)
+		tsnep_rx_disable(queue->rx);
+}
+
 static int tsnep_netdev_open(struct net_device *netdev)
 {
 	struct tsnep_adapter *adapter = netdev_priv(netdev);
-	int tx_queue_index = 0;
-	int rx_queue_index = 0;
-	void __iomem *addr;
 	int i, retval;
 
 	for (i = 0; i < adapter->num_queues; i++) {
 		if (adapter->queue[i].tx) {
-			addr = adapter->addr + TSNEP_QUEUE(tx_queue_index);
-			retval = tsnep_tx_open(adapter, addr, tx_queue_index,
-					       adapter->queue[i].tx);
+			retval = tsnep_tx_open(adapter->queue[i].tx);
 			if (retval)
 				goto failed;
-			tx_queue_index++;
 		}
 		if (adapter->queue[i].rx) {
-			addr = adapter->addr + TSNEP_QUEUE(rx_queue_index);
-			retval = tsnep_rx_open(adapter, addr, rx_queue_index,
-					       adapter->queue[i].rx);
+			retval = tsnep_rx_open(adapter->queue[i].rx);
 			if (retval)
 				goto failed;
-			rx_queue_index++;
 		}
 
 		retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0);
@@ -1420,11 +1943,8 @@ static int tsnep_netdev_open(struct net_device *netdev)
 	if (retval)
 		goto phy_failed;
 
-	for (i = 0; i < adapter->num_queues; i++) {
-		napi_enable(&adapter->queue[i].napi);
-
-		tsnep_enable_irq(adapter, adapter->queue[i].irq_mask);
-	}
+	for (i = 0; i < adapter->num_queues; i++)
+		tsnep_queue_enable(&adapter->queue[i]);
 
 	return 0;
 
@@ -1451,9 +1971,7 @@ static int tsnep_netdev_close(struct net_device *netdev)
 	tsnep_phy_close(adapter);
 
 	for (i = 0; i < adapter->num_queues; i++) {
-		tsnep_disable_irq(adapter, adapter->queue[i].irq_mask);
-
-		napi_disable(&adapter->queue[i].napi);
+		tsnep_queue_disable(&adapter->queue[i]);
 
 		tsnep_queue_close(&adapter->queue[i], i == 0);
 
@@ -1466,6 +1984,69 @@ static int tsnep_netdev_close(struct net_device *netdev)
 	return 0;
 }
 
+int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool)
+{
+	bool running = netif_running(queue->adapter->netdev);
+	u32 frame_size;
+
+	frame_size = xsk_pool_get_rx_frame_size(pool);
+	if (frame_size < TSNEP_XSK_RX_BUF_SIZE)
+		return -EOPNOTSUPP;
+
+	queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE,
+					 sizeof(*queue->rx->page_buffer),
+					 GFP_KERNEL);
+	if (!queue->rx->page_buffer)
+		return -ENOMEM;
+	queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE,
+				       sizeof(*queue->rx->xdp_batch),
+				       GFP_KERNEL);
+	if (!queue->rx->xdp_batch) {
+		kfree(queue->rx->page_buffer);
+		queue->rx->page_buffer = NULL;
+
+		return -ENOMEM;
+	}
+
+	xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc);
+
+	if (running)
+		tsnep_queue_disable(queue);
+
+	queue->tx->xsk_pool = pool;
+	queue->rx->xsk_pool = pool;
+
+	if (running) {
+		tsnep_rx_reopen_xsk(queue->rx);
+		tsnep_queue_enable(queue);
+	}
+
+	return 0;
+}
+
+void tsnep_disable_xsk(struct tsnep_queue *queue)
+{
+	bool running = netif_running(queue->adapter->netdev);
+
+	if (running)
+		tsnep_queue_disable(queue);
+
+	tsnep_rx_free_zc(queue->rx);
+
+	queue->rx->xsk_pool = NULL;
+	queue->tx->xsk_pool = NULL;
+
+	if (running) {
+		tsnep_rx_reopen(queue->rx);
+		tsnep_queue_enable(queue);
+	}
+
+	kfree(queue->rx->xdp_batch);
+	queue->rx->xdp_batch = NULL;
+	kfree(queue->rx->page_buffer);
+	queue->rx->page_buffer = NULL;
+}
+
 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb,
 					   struct net_device *netdev)
 {
@@ -1615,6 +2196,9 @@ static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf)
 	switch (bpf->command) {
 	case XDP_SETUP_PROG:
 		return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack);
+	case XDP_SETUP_XSK_POOL:
+		return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool,
+					    bpf->xsk.queue_id);
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -1669,6 +2253,24 @@ static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n,
 	return nxmit;
 }
 
+static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id,
+				   u32 flags)
+{
+	struct tsnep_adapter *adapter = netdev_priv(dev);
+	struct tsnep_queue *queue;
+
+	if (queue_id >= adapter->num_rx_queues ||
+	    queue_id >= adapter->num_tx_queues)
+		return -EINVAL;
+
+	queue = &adapter->queue[queue_id];
+
+	if (!napi_if_scheduled_mark_missed(&queue->napi))
+		napi_schedule(&queue->napi);
+
+	return 0;
+}
+
 static const struct net_device_ops tsnep_netdev_ops = {
 	.ndo_open = tsnep_netdev_open,
 	.ndo_stop = tsnep_netdev_close,
@@ -1682,6 +2284,7 @@ static const struct net_device_ops tsnep_netdev_ops = {
 	.ndo_setup_tc = tsnep_tc_setup,
 	.ndo_bpf = tsnep_netdev_bpf,
 	.ndo_xdp_xmit = tsnep_netdev_xdp_xmit,
+	.ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup,
 };
 
 static int tsnep_mac_init(struct tsnep_adapter *adapter)
@@ -1797,9 +2400,16 @@ static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count)
 	adapter->num_tx_queues = 1;
 	adapter->num_rx_queues = 1;
 	adapter->num_queues = 1;
+	adapter->queue[0].adapter = adapter;
 	adapter->queue[0].irq = retval;
 	adapter->queue[0].tx = &adapter->tx[0];
+	adapter->queue[0].tx->adapter = adapter;
+	adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0);
+	adapter->queue[0].tx->queue_index = 0;
 	adapter->queue[0].rx = &adapter->rx[0];
+	adapter->queue[0].rx->adapter = adapter;
+	adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0);
+	adapter->queue[0].rx->queue_index = 0;
 	adapter->queue[0].irq_mask = irq_mask;
 	adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY;
 	retval = tsnep_set_irq_coalesce(&adapter->queue[0],
@@ -1821,9 +2431,16 @@ static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count)
 		adapter->num_tx_queues++;
 		adapter->num_rx_queues++;
 		adapter->num_queues++;
+		adapter->queue[i].adapter = adapter;
 		adapter->queue[i].irq = retval;
 		adapter->queue[i].tx = &adapter->tx[i];
+		adapter->queue[i].tx->adapter = adapter;
+		adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i);
+		adapter->queue[i].tx->queue_index = i;
 		adapter->queue[i].rx = &adapter->rx[i];
+		adapter->queue[i].rx->adapter = adapter;
+		adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i);
+		adapter->queue[i].rx->queue_index = i;
 		adapter->queue[i].irq_mask =
 			irq_mask << (ECM_INT_TXRX_SHIFT * i);
 		adapter->queue[i].irq_delay_addr =
@@ -1928,7 +2545,8 @@ static int tsnep_probe(struct platform_device *pdev)
 
 	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
 			       NETDEV_XDP_ACT_NDO_XMIT |
-			       NETDEV_XDP_ACT_NDO_XMIT_SG;
+			       NETDEV_XDP_ACT_NDO_XMIT_SG |
+			       NETDEV_XDP_ACT_XSK_ZEROCOPY;
 
 	/* carrier off reporting is important to ethtool even BEFORE open */
 	netif_carrier_off(netdev);
diff --git a/drivers/net/ethernet/engleder/tsnep_xdp.c b/drivers/net/ethernet/engleder/tsnep_xdp.c
index 4d14cb1fd772..c0513848c547 100644
--- a/drivers/net/ethernet/engleder/tsnep_xdp.c
+++ b/drivers/net/ethernet/engleder/tsnep_xdp.c
@@ -17,3 +17,69 @@ int tsnep_xdp_setup_prog(struct tsnep_adapter *adapter, struct bpf_prog *prog,
 
 	return 0;
 }
+
+static int tsnep_xdp_enable_pool(struct tsnep_adapter *adapter,
+				 struct xsk_buff_pool *pool, u16 queue_id)
+{
+	struct tsnep_queue *queue;
+	int retval;
+
+	if (queue_id >= adapter->num_rx_queues ||
+	    queue_id >= adapter->num_tx_queues)
+		return -EINVAL;
+
+	queue = &adapter->queue[queue_id];
+	if (queue->rx->queue_index != queue_id ||
+	    queue->tx->queue_index != queue_id) {
+		netdev_err(adapter->netdev,
+			   "XSK support only for TX/RX queue pairs\n");
+
+		return -EOPNOTSUPP;
+	}
+
+	retval = xsk_pool_dma_map(pool, adapter->dmadev,
+				  DMA_ATTR_SKIP_CPU_SYNC);
+	if (retval) {
+		netdev_err(adapter->netdev, "failed to map XSK pool\n");
+
+		return retval;
+	}
+
+	retval = tsnep_enable_xsk(queue, pool);
+	if (retval) {
+		xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC);
+
+		return retval;
+	}
+
+	return 0;
+}
+
+static int tsnep_xdp_disable_pool(struct tsnep_adapter *adapter, u16 queue_id)
+{
+	struct xsk_buff_pool *pool;
+	struct tsnep_queue *queue;
+
+	if (queue_id >= adapter->num_rx_queues ||
+	    queue_id >= adapter->num_tx_queues)
+		return -EINVAL;
+
+	pool = xsk_get_pool_from_qid(adapter->netdev, queue_id);
+	if (!pool)
+		return -EINVAL;
+
+	queue = &adapter->queue[queue_id];
+
+	tsnep_disable_xsk(queue);
+
+	xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC);
+
+	return 0;
+}
+
+int tsnep_xdp_setup_pool(struct tsnep_adapter *adapter,
+			 struct xsk_buff_pool *pool, u16 queue_id)
+{
+	return pool ? tsnep_xdp_enable_pool(adapter, pool, queue_id) :
+		      tsnep_xdp_disable_pool(adapter, queue_id);
+}
diff --git a/drivers/net/ethernet/freescale/Kconfig b/drivers/net/ethernet/freescale/Kconfig
index f1e80d6996ef..1c78f66a89da 100644
--- a/drivers/net/ethernet/freescale/Kconfig
+++ b/drivers/net/ethernet/freescale/Kconfig
@@ -71,6 +71,7 @@ config FSL_XGMAC_MDIO
 	tristate "Freescale XGMAC MDIO"
 	select PHYLIB
 	depends on OF
+	select MDIO_DEVRES
 	select OF_MDIO
 	help
 	  This driver supports the MDIO bus on the Fman 10G Ethernet MACs, and
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
index 9318a2554056..431f8917dc39 100644
--- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
@@ -299,7 +299,8 @@ static int dpaa_stop(struct net_device *net_dev)
 {
 	struct mac_device *mac_dev;
 	struct dpaa_priv *priv;
-	int i, err, error;
+	int i, error;
+	int err = 0;
 
 	priv = netdev_priv(net_dev);
 	mac_dev = priv->mac_dev;
@@ -1482,13 +1483,8 @@ static int dpaa_enable_tx_csum(struct dpaa_priv *priv,
 	parse_result = (struct fman_prs_result *)parse_results;
 
 	/* If we're dealing with VLAN, get the real Ethernet type */
-	if (ethertype == ETH_P_8021Q) {
-		/* We can't always assume the MAC header is set correctly
-		 * by the stack, so reset to beginning of skb->data
-		 */
-		skb_reset_mac_header(skb);
-		ethertype = ntohs(vlan_eth_hdr(skb)->h_vlan_encapsulated_proto);
-	}
+	if (ethertype == ETH_P_8021Q)
+		ethertype = ntohs(skb_vlan_eth_hdr(skb)->h_vlan_encapsulated_proto);
 
 	/* Fill in the relevant L3 parse result fields
 	 * and read the L4 protocol type
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c
index c886f33f8c6f..b1871e6c4006 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c
+++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c
@@ -159,7 +159,8 @@ static void dpaa2_mac_config(struct phylink_config *config, unsigned int mode,
 	struct dpmac_link_state *dpmac_state = &mac->state;
 	int err;
 
-	if (state->an_enabled)
+	if (linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
+			      state->advertising))
 		dpmac_state->options |= DPMAC_LINK_OPT_AUTONEG;
 	else
 		dpmac_state->options &= ~DPMAC_LINK_OPT_AUTONEG;
diff --git a/drivers/net/ethernet/freescale/enetc/Kconfig b/drivers/net/ethernet/freescale/enetc/Kconfig
index 9bc099cf3cb1..4d75e6807e92 100644
--- a/drivers/net/ethernet/freescale/enetc/Kconfig
+++ b/drivers/net/ethernet/freescale/enetc/Kconfig
@@ -10,6 +10,7 @@ config FSL_ENETC_CORE
 config FSL_ENETC
 	tristate "ENETC PF driver"
 	depends on PCI_MSI
+	select MDIO_DEVRES
 	select FSL_ENETC_CORE
 	select FSL_ENETC_IERB
 	select FSL_ENETC_MDIO
diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c
index 2fc712b24d12..3c4fa26f0f9b 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc.c
@@ -25,6 +25,13 @@ void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
 }
 EXPORT_SYMBOL_GPL(enetc_port_mac_wr);
 
+static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
+					 u8 preemptible_tcs)
+{
+	priv->preemptible_tcs = preemptible_tcs;
+	enetc_mm_commit_preemptible_tcs(priv);
+}
+
 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
 {
 	int num_tx_rings = priv->num_tx_rings;
@@ -2640,16 +2647,19 @@ static void enetc_reset_tc_mqprio(struct net_device *ndev)
 	}
 
 	enetc_debug_tx_ring_prios(priv);
+
+	enetc_change_preemptible_tcs(priv, 0);
 }
 
 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
 {
+	struct tc_mqprio_qopt_offload *mqprio = type_data;
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
-	struct tc_mqprio_qopt *mqprio = type_data;
+	struct tc_mqprio_qopt *qopt = &mqprio->qopt;
 	struct enetc_hw *hw = &priv->si->hw;
 	int num_stack_tx_queues = 0;
-	u8 num_tc = mqprio->num_tc;
 	struct enetc_bdr *tx_ring;
+	u8 num_tc = qopt->num_tc;
 	int offset, count;
 	int err, tc, q;
 
@@ -2663,8 +2673,8 @@ int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
 		return err;
 
 	for (tc = 0; tc < num_tc; tc++) {
-		offset = mqprio->offset[tc];
-		count = mqprio->count[tc];
+		offset = qopt->offset[tc];
+		count = qopt->count[tc];
 		num_stack_tx_queues += count;
 
 		err = netdev_set_tc_queue(ndev, tc, count, offset);
@@ -2693,6 +2703,8 @@ int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
 
 	enetc_debug_tx_ring_prios(priv);
 
+	enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);
+
 	return 0;
 
 err_reset_tc:
diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h
index 8010f31cd10d..c97a8e3d7a7f 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc.h
@@ -355,6 +355,9 @@ struct enetc_ndev_priv {
 	u16 rx_bd_count, tx_bd_count;
 
 	u16 msg_enable;
+
+	u8 preemptible_tcs;
+
 	enum enetc_active_offloads active_offloads;
 
 	u32 speed; /* store speed for compare update pspeed */
@@ -433,6 +436,7 @@ int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
 /* ethtool */
 void enetc_set_ethtool_ops(struct net_device *ndev);
 void enetc_mm_link_state_update(struct enetc_ndev_priv *priv, bool link);
+void enetc_mm_commit_preemptible_tcs(struct enetc_ndev_priv *priv);
 
 /* control buffer descriptor ring (CBDR) */
 int enetc_setup_cbdr(struct device *dev, struct enetc_hw *hw, int bd_count,
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
index 838750a03cf6..e993ed04ab57 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
@@ -32,6 +32,12 @@ static const u32 enetc_port_regs[] = {
 	ENETC_PM0_CMD_CFG, ENETC_PM0_MAXFRM, ENETC_PM0_IF_MODE
 };
 
+static const u32 enetc_port_mm_regs[] = {
+	ENETC_MMCSR, ENETC_PFPMR, ENETC_PTCFPR(0), ENETC_PTCFPR(1),
+	ENETC_PTCFPR(2), ENETC_PTCFPR(3), ENETC_PTCFPR(4), ENETC_PTCFPR(5),
+	ENETC_PTCFPR(6), ENETC_PTCFPR(7),
+};
+
 static int enetc_get_reglen(struct net_device *ndev)
 {
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
@@ -45,6 +51,9 @@ static int enetc_get_reglen(struct net_device *ndev)
 	if (hw->port)
 		len += ARRAY_SIZE(enetc_port_regs);
 
+	if (hw->port && !!(priv->si->hw_features & ENETC_SI_F_QBU))
+		len += ARRAY_SIZE(enetc_port_mm_regs);
+
 	len *= sizeof(u32) * 2; /* store 2 entries per reg: addr and value */
 
 	return len;
@@ -90,6 +99,14 @@ static void enetc_get_regs(struct net_device *ndev, struct ethtool_regs *regs,
 		*buf++ = addr;
 		*buf++ = enetc_rd(hw, addr);
 	}
+
+	if (priv->si->hw_features & ENETC_SI_F_QBU) {
+		for (i = 0; i < ARRAY_SIZE(enetc_port_mm_regs); i++) {
+			addr = ENETC_PORT_BASE + enetc_port_mm_regs[i];
+			*buf++ = addr;
+			*buf++ = enetc_rd(hw, addr);
+		}
+	}
 }
 
 static const struct {
@@ -976,7 +993,9 @@ static int enetc_get_mm(struct net_device *ndev, struct ethtool_mm_state *state)
 	lafs = ENETC_MMCSR_GET_LAFS(val);
 	state->rx_min_frag_size = ethtool_mm_frag_size_add_to_min(lafs);
 	state->tx_enabled = !!(val & ENETC_MMCSR_LPE); /* mirror of MMCSR_ME */
-	state->tx_active = !!(val & ENETC_MMCSR_LPA);
+	state->tx_active = state->tx_enabled &&
+			   (state->verify_status == ETHTOOL_MM_VERIFY_STATUS_SUCCEEDED ||
+			    state->verify_status == ETHTOOL_MM_VERIFY_STATUS_DISABLED);
 	state->verify_enabled = !(val & ENETC_MMCSR_VDIS);
 	state->verify_time = ENETC_MMCSR_GET_VT(val);
 	/* A verifyTime of 128 ms would exceed the 7 bit width
@@ -989,6 +1008,64 @@ static int enetc_get_mm(struct net_device *ndev, struct ethtool_mm_state *state)
 	return 0;
 }
 
+static int enetc_mm_wait_tx_active(struct enetc_hw *hw, int verify_time)
+{
+	int timeout = verify_time * USEC_PER_MSEC * ENETC_MM_VERIFY_RETRIES;
+	u32 val;
+
+	/* This will time out after the standard value of 3 verification
+	 * attempts. To not sleep forever, it relies on a non-zero verify_time,
+	 * guarantee which is provided by the ethtool nlattr policy.
+	 */
+	return read_poll_timeout(enetc_port_rd, val,
+				 ENETC_MMCSR_GET_VSTS(val) == 3,
+				 ENETC_MM_VERIFY_SLEEP_US, timeout,
+				 true, hw, ENETC_MMCSR);
+}
+
+static void enetc_set_ptcfpr(struct enetc_hw *hw, u8 preemptible_tcs)
+{
+	u32 val;
+	int tc;
+
+	for (tc = 0; tc < 8; tc++) {
+		val = enetc_port_rd(hw, ENETC_PTCFPR(tc));
+
+		if (preemptible_tcs & BIT(tc))
+			val |= ENETC_PTCFPR_FPE;
+		else
+			val &= ~ENETC_PTCFPR_FPE;
+
+		enetc_port_wr(hw, ENETC_PTCFPR(tc), val);
+	}
+}
+
+/* ENETC does not have an IRQ to notify changes to the MAC Merge TX status
+ * (active/inactive), but the preemptible traffic classes should only be
+ * committed to hardware once TX is active. Resort to polling.
+ */
+void enetc_mm_commit_preemptible_tcs(struct enetc_ndev_priv *priv)
+{
+	struct enetc_hw *hw = &priv->si->hw;
+	u8 preemptible_tcs = 0;
+	u32 val;
+	int err;
+
+	val = enetc_port_rd(hw, ENETC_MMCSR);
+	if (!(val & ENETC_MMCSR_ME))
+		goto out;
+
+	if (!(val & ENETC_MMCSR_VDIS)) {
+		err = enetc_mm_wait_tx_active(hw, ENETC_MMCSR_GET_VT(val));
+		if (err)
+			goto out;
+	}
+
+	preemptible_tcs = priv->preemptible_tcs;
+out:
+	enetc_set_ptcfpr(hw, preemptible_tcs);
+}
+
 /* FIXME: Workaround for the link partner's verification failing if ENETC
  * priorly received too much express traffic. The documentation doesn't
  * suggest this is needed.
@@ -1041,10 +1118,13 @@ static int enetc_set_mm(struct net_device *ndev, struct ethtool_mm_cfg *cfg,
 	else
 		priv->active_offloads &= ~ENETC_F_QBU;
 
-	/* If link is up, enable MAC Merge right away */
-	if (!!(priv->active_offloads & ENETC_F_QBU) &&
-	    !(val & ENETC_MMCSR_LINK_FAIL))
-		val |= ENETC_MMCSR_ME;
+	/* If link is up, enable/disable MAC Merge right away */
+	if (!(val & ENETC_MMCSR_LINK_FAIL)) {
+		if (!!(priv->active_offloads & ENETC_F_QBU))
+			val |= ENETC_MMCSR_ME;
+		else
+			val &= ~ENETC_MMCSR_ME;
+	}
 
 	val &= ~ENETC_MMCSR_VT_MASK;
 	val |= ENETC_MMCSR_VT(cfg->verify_time);
@@ -1056,6 +1136,8 @@ static int enetc_set_mm(struct net_device *ndev, struct ethtool_mm_cfg *cfg,
 
 	enetc_restart_emac_rx(priv->si);
 
+	enetc_mm_commit_preemptible_tcs(priv);
+
 	mutex_unlock(&priv->mm_lock);
 
 	return 0;
@@ -1089,6 +1171,8 @@ void enetc_mm_link_state_update(struct enetc_ndev_priv *priv, bool link)
 
 	enetc_port_wr(hw, ENETC_MMCSR, val);
 
+	enetc_mm_commit_preemptible_tcs(priv);
+
 	mutex_unlock(&priv->mm_lock);
 }
 EXPORT_SYMBOL_GPL(enetc_mm_link_state_update);
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_hw.h b/drivers/net/ethernet/freescale/enetc/enetc_hw.h
index de2e0ee8cdcb..1619943fb263 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h
@@ -3,6 +3,9 @@
 
 #include <linux/bitops.h>
 
+#define ENETC_MM_VERIFY_SLEEP_US	USEC_PER_MSEC
+#define ENETC_MM_VERIFY_RETRIES		3
+
 /* ENETC device IDs */
 #define ENETC_DEV_ID_PF		0xe100
 #define ENETC_DEV_ID_VF		0xef00
@@ -965,6 +968,10 @@ static inline u32 enetc_usecs_to_cycles(u32 usecs)
 	return (u32)div_u64(usecs * ENETC_CLK, 1000000ULL);
 }
 
+/* Port traffic class frame preemption register */
+#define ENETC_PTCFPR(n)			(0x1910 + (n) * 4) /* n = [0 ..7] */
+#define ENETC_PTCFPR_FPE		BIT(31)
+
 /* port time gating control register */
 #define ENETC_PTGCR			0x11a00
 #define ENETC_PTGCR_TGE			BIT(31)
diff --git a/drivers/net/ethernet/fungible/funcore/fun_dev.c b/drivers/net/ethernet/fungible/funcore/fun_dev.c
index fb5120d90f26..a7fbd4cd560a 100644
--- a/drivers/net/ethernet/fungible/funcore/fun_dev.c
+++ b/drivers/net/ethernet/fungible/funcore/fun_dev.c
@@ -1,6 +1,5 @@
 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
 
-#include <linux/aer.h>
 #include <linux/bitmap.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
@@ -747,8 +746,6 @@ void fun_dev_disable(struct fun_dev *fdev)
 	bitmap_free(fdev->irq_map);
 	pci_free_irq_vectors(pdev);
 
-	pci_clear_master(pdev);
-	pci_disable_pcie_error_reporting(pdev);
 	pci_disable_device(pdev);
 
 	fun_unmap_bars(fdev);
@@ -781,8 +778,6 @@ int fun_dev_enable(struct fun_dev *fdev, struct pci_dev *pdev,
 		goto unmap;
 	}
 
-	pci_enable_pcie_error_reporting(pdev);
-
 	rc = sanitize_dev(fdev);
 	if (rc)
 		goto disable_dev;
@@ -825,12 +820,10 @@ int fun_dev_enable(struct fun_dev *fdev, struct pci_dev *pdev,
 disable_admin:
 	fun_disable_admin_queue(fdev);
 free_irq_mgr:
-	pci_clear_master(pdev);
 	bitmap_free(fdev->irq_map);
 free_irqs:
 	pci_free_irq_vectors(pdev);
 disable_dev:
-	pci_disable_pcie_error_reporting(pdev);
 	pci_disable_device(pdev);
 unmap:
 	fun_unmap_bars(fdev);
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 005cb9dfe078..98eb78d98e9f 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -47,6 +47,8 @@
 
 #define GVE_RX_BUFFER_SIZE_DQO 2048
 
+#define GVE_XDP_ACTIONS 5
+
 #define GVE_GQ_TX_MIN_PKT_DESC_BYTES 182
 
 /* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */
@@ -232,7 +234,10 @@ struct gve_rx_ring {
 	u64 rx_frag_flip_cnt; /* free-running count of rx segments where page_flip was used */
 	u64 rx_frag_copy_cnt; /* free-running count of rx segments copied */
 	u64 rx_frag_alloc_cnt; /* free-running count of rx page allocations */
-
+	u64 xdp_tx_errors;
+	u64 xdp_redirect_errors;
+	u64 xdp_alloc_fails;
+	u64 xdp_actions[GVE_XDP_ACTIONS];
 	u32 q_num; /* queue index */
 	u32 ntfy_id; /* notification block index */
 	struct gve_queue_resources *q_resources; /* head and tail pointer idx */
@@ -240,6 +245,12 @@ struct gve_rx_ring {
 	struct u64_stats_sync statss; /* sync stats for 32bit archs */
 
 	struct gve_rx_ctx ctx; /* Info for packet currently being processed in this ring. */
+
+	/* XDP stuff */
+	struct xdp_rxq_info xdp_rxq;
+	struct xdp_rxq_info xsk_rxq;
+	struct xsk_buff_pool *xsk_pool;
+	struct page_frag_cache page_cache; /* Page cache to allocate XDP frames */
 };
 
 /* A TX desc ring entry */
@@ -260,7 +271,14 @@ struct gve_tx_iovec {
  * ring entry but only used for a pkt_desc not a seg_desc
  */
 struct gve_tx_buffer_state {
-	struct sk_buff *skb; /* skb for this pkt */
+	union {
+		struct sk_buff *skb; /* skb for this pkt */
+		struct xdp_frame *xdp_frame; /* xdp_frame */
+	};
+	struct {
+		u16 size; /* size of xmitted xdp pkt */
+		u8 is_xsk; /* xsk buff */
+	} xdp;
 	union {
 		struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */
 		struct {
@@ -375,6 +393,8 @@ struct gve_tx_ring {
 		struct {
 			/* Spinlock for when cleanup in progress */
 			spinlock_t clean_lock;
+			/* Spinlock for XDP tx traffic */
+			spinlock_t xdp_lock;
 		};
 
 		/* DQO fields. */
@@ -452,6 +472,12 @@ struct gve_tx_ring {
 	dma_addr_t q_resources_bus; /* dma address of the queue resources */
 	dma_addr_t complq_bus_dqo; /* dma address of the dqo.compl_ring */
 	struct u64_stats_sync statss; /* sync stats for 32bit archs */
+	struct xsk_buff_pool *xsk_pool;
+	u32 xdp_xsk_wakeup;
+	u32 xdp_xsk_done;
+	u64 xdp_xsk_sent;
+	u64 xdp_xmit;
+	u64 xdp_xmit_errors;
 } ____cacheline_aligned;
 
 /* Wraps the info for one irq including the napi struct and the queues
@@ -528,9 +554,11 @@ struct gve_priv {
 	u16 rx_data_slot_cnt; /* rx buffer length */
 	u64 max_registered_pages;
 	u64 num_registered_pages; /* num pages registered with NIC */
+	struct bpf_prog *xdp_prog; /* XDP BPF program */
 	u32 rx_copybreak; /* copy packets smaller than this */
 	u16 default_num_queues; /* default num queues to set up */
 
+	u16 num_xdp_queues;
 	struct gve_queue_config tx_cfg;
 	struct gve_queue_config rx_cfg;
 	struct gve_qpl_config qpl_cfg; /* map used QPL ids */
@@ -787,7 +815,17 @@ static inline u32 gve_num_tx_qpls(struct gve_priv *priv)
 	if (priv->queue_format != GVE_GQI_QPL_FORMAT)
 		return 0;
 
-	return priv->tx_cfg.num_queues;
+	return priv->tx_cfg.num_queues + priv->num_xdp_queues;
+}
+
+/* Returns the number of XDP tx queue page lists
+ */
+static inline u32 gve_num_xdp_qpls(struct gve_priv *priv)
+{
+	if (priv->queue_format != GVE_GQI_QPL_FORMAT)
+		return 0;
+
+	return priv->num_xdp_queues;
 }
 
 /* Returns the number of rx queue page lists
@@ -800,16 +838,35 @@ static inline u32 gve_num_rx_qpls(struct gve_priv *priv)
 	return priv->rx_cfg.num_queues;
 }
 
+static inline u32 gve_tx_qpl_id(struct gve_priv *priv, int tx_qid)
+{
+	return tx_qid;
+}
+
+static inline u32 gve_rx_qpl_id(struct gve_priv *priv, int rx_qid)
+{
+	return priv->tx_cfg.max_queues + rx_qid;
+}
+
+static inline u32 gve_tx_start_qpl_id(struct gve_priv *priv)
+{
+	return gve_tx_qpl_id(priv, 0);
+}
+
+static inline u32 gve_rx_start_qpl_id(struct gve_priv *priv)
+{
+	return gve_rx_qpl_id(priv, 0);
+}
+
 /* Returns a pointer to the next available tx qpl in the list of qpls
  */
 static inline
-struct gve_queue_page_list *gve_assign_tx_qpl(struct gve_priv *priv)
+struct gve_queue_page_list *gve_assign_tx_qpl(struct gve_priv *priv, int tx_qid)
 {
-	int id = find_first_zero_bit(priv->qpl_cfg.qpl_id_map,
-				     priv->qpl_cfg.qpl_map_size);
+	int id = gve_tx_qpl_id(priv, tx_qid);
 
-	/* we are out of tx qpls */
-	if (id >= gve_num_tx_qpls(priv))
+	/* QPL already in use */
+	if (test_bit(id, priv->qpl_cfg.qpl_id_map))
 		return NULL;
 
 	set_bit(id, priv->qpl_cfg.qpl_id_map);
@@ -819,14 +876,12 @@ struct gve_queue_page_list *gve_assign_tx_qpl(struct gve_priv *priv)
 /* Returns a pointer to the next available rx qpl in the list of qpls
  */
 static inline
-struct gve_queue_page_list *gve_assign_rx_qpl(struct gve_priv *priv)
+struct gve_queue_page_list *gve_assign_rx_qpl(struct gve_priv *priv, int rx_qid)
 {
-	int id = find_next_zero_bit(priv->qpl_cfg.qpl_id_map,
-				    priv->qpl_cfg.qpl_map_size,
-				    gve_num_tx_qpls(priv));
+	int id = gve_rx_qpl_id(priv, rx_qid);
 
-	/* we are out of rx qpls */
-	if (id == gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv))
+	/* QPL already in use */
+	if (test_bit(id, priv->qpl_cfg.qpl_id_map))
 		return NULL;
 
 	set_bit(id, priv->qpl_cfg.qpl_id_map);
@@ -845,7 +900,7 @@ static inline void gve_unassign_qpl(struct gve_priv *priv, int id)
 static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv,
 						      int id)
 {
-	if (id < gve_num_tx_qpls(priv))
+	if (id < gve_rx_start_qpl_id(priv))
 		return DMA_TO_DEVICE;
 	else
 		return DMA_FROM_DEVICE;
@@ -857,6 +912,21 @@ static inline bool gve_is_gqi(struct gve_priv *priv)
 		priv->queue_format == GVE_GQI_QPL_FORMAT;
 }
 
+static inline u32 gve_num_tx_queues(struct gve_priv *priv)
+{
+	return priv->tx_cfg.num_queues + priv->num_xdp_queues;
+}
+
+static inline u32 gve_xdp_tx_queue_id(struct gve_priv *priv, u32 queue_id)
+{
+	return priv->tx_cfg.num_queues + queue_id;
+}
+
+static inline u32 gve_xdp_tx_start_queue_id(struct gve_priv *priv)
+{
+	return gve_xdp_tx_queue_id(priv, 0);
+}
+
 /* buffers */
 int gve_alloc_page(struct gve_priv *priv, struct device *dev,
 		   struct page **page, dma_addr_t *dma,
@@ -865,9 +935,15 @@ void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma,
 		   enum dma_data_direction);
 /* tx handling */
 netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev);
+int gve_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
+		 u32 flags);
+int gve_xdp_xmit_one(struct gve_priv *priv, struct gve_tx_ring *tx,
+		     void *data, int len, void *frame_p);
+void gve_xdp_tx_flush(struct gve_priv *priv, u32 xdp_qid);
 bool gve_tx_poll(struct gve_notify_block *block, int budget);
-int gve_tx_alloc_rings(struct gve_priv *priv);
-void gve_tx_free_rings_gqi(struct gve_priv *priv);
+bool gve_xdp_poll(struct gve_notify_block *block, int budget);
+int gve_tx_alloc_rings(struct gve_priv *priv, int start_id, int num_rings);
+void gve_tx_free_rings_gqi(struct gve_priv *priv, int start_id, int num_rings);
 u32 gve_tx_load_event_counter(struct gve_priv *priv,
 			      struct gve_tx_ring *tx);
 bool gve_tx_clean_pending(struct gve_priv *priv, struct gve_tx_ring *tx);
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 60061288ad9d..252974202a3f 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -516,12 +516,12 @@ static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
 	return gve_adminq_issue_cmd(priv, &cmd);
 }
 
-int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues)
+int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues)
 {
 	int err;
 	int i;
 
-	for (i = 0; i < num_queues; i++) {
+	for (i = start_id; i < start_id + num_queues; i++) {
 		err = gve_adminq_create_tx_queue(priv, i);
 		if (err)
 			return err;
@@ -604,12 +604,12 @@ static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
 	return 0;
 }
 
-int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 num_queues)
+int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues)
 {
 	int err;
 	int i;
 
-	for (i = 0; i < num_queues; i++) {
+	for (i = start_id; i < start_id + num_queues; i++) {
 		err = gve_adminq_destroy_tx_queue(priv, i);
 		if (err)
 			return err;
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index cf29662e6ad1..f894beb3deaf 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -410,8 +410,8 @@ int gve_adminq_configure_device_resources(struct gve_priv *priv,
 					  dma_addr_t db_array_bus_addr,
 					  u32 num_ntfy_blks);
 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv);
-int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues);
-int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 queue_id);
+int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues);
+int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues);
 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues);
 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 queue_id);
 int gve_adminq_register_page_list(struct gve_priv *priv,
diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c
index 5f81470843b4..cfd4b8d284d1 100644
--- a/drivers/net/ethernet/google/gve/gve_ethtool.c
+++ b/drivers/net/ethernet/google/gve/gve_ethtool.c
@@ -34,6 +34,11 @@ static u32 gve_get_msglevel(struct net_device *netdev)
 	return priv->msg_enable;
 }
 
+/* For the following stats column string names, make sure the order
+ * matches how it is filled in the code. For xdp_aborted, xdp_drop,
+ * xdp_pass, xdp_tx, xdp_redirect, make sure it also matches the order
+ * as declared in enum xdp_action inside file uapi/linux/bpf.h .
+ */
 static const char gve_gstrings_main_stats[][ETH_GSTRING_LEN] = {
 	"rx_packets", "tx_packets", "rx_bytes", "tx_bytes",
 	"rx_dropped", "tx_dropped", "tx_timeouts",
@@ -49,12 +54,16 @@ static const char gve_gstrings_rx_stats[][ETH_GSTRING_LEN] = {
 	"rx_dropped_pkt[%u]", "rx_copybreak_pkt[%u]", "rx_copied_pkt[%u]",
 	"rx_queue_drop_cnt[%u]", "rx_no_buffers_posted[%u]",
 	"rx_drops_packet_over_mru[%u]", "rx_drops_invalid_checksum[%u]",
+	"rx_xdp_aborted[%u]", "rx_xdp_drop[%u]", "rx_xdp_pass[%u]",
+	"rx_xdp_tx[%u]", "rx_xdp_redirect[%u]",
+	"rx_xdp_tx_errors[%u]", "rx_xdp_redirect_errors[%u]", "rx_xdp_alloc_fails[%u]",
 };
 
 static const char gve_gstrings_tx_stats[][ETH_GSTRING_LEN] = {
 	"tx_posted_desc[%u]", "tx_completed_desc[%u]", "tx_consumed_desc[%u]", "tx_bytes[%u]",
 	"tx_wake[%u]", "tx_stop[%u]", "tx_event_counter[%u]",
-	"tx_dma_mapping_error[%u]",
+	"tx_dma_mapping_error[%u]", "tx_xsk_wakeup[%u]",
+	"tx_xsk_done[%u]", "tx_xsk_sent[%u]", "tx_xdp_xmit[%u]", "tx_xdp_xmit_errors[%u]"
 };
 
 static const char gve_gstrings_adminq_stats[][ETH_GSTRING_LEN] = {
@@ -81,8 +90,10 @@ static void gve_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
 {
 	struct gve_priv *priv = netdev_priv(netdev);
 	char *s = (char *)data;
+	int num_tx_queues;
 	int i, j;
 
+	num_tx_queues = gve_num_tx_queues(priv);
 	switch (stringset) {
 	case ETH_SS_STATS:
 		memcpy(s, *gve_gstrings_main_stats,
@@ -97,7 +108,7 @@ static void gve_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
 			}
 		}
 
-		for (i = 0; i < priv->tx_cfg.num_queues; i++) {
+		for (i = 0; i < num_tx_queues; i++) {
 			for (j = 0; j < NUM_GVE_TX_CNTS; j++) {
 				snprintf(s, ETH_GSTRING_LEN,
 					 gve_gstrings_tx_stats[j], i);
@@ -124,12 +135,14 @@ static void gve_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
 static int gve_get_sset_count(struct net_device *netdev, int sset)
 {
 	struct gve_priv *priv = netdev_priv(netdev);
+	int num_tx_queues;
 
+	num_tx_queues = gve_num_tx_queues(priv);
 	switch (sset) {
 	case ETH_SS_STATS:
 		return GVE_MAIN_STATS_LEN + GVE_ADMINQ_STATS_LEN +
 		       (priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS) +
-		       (priv->tx_cfg.num_queues * NUM_GVE_TX_CNTS);
+		       (num_tx_queues * NUM_GVE_TX_CNTS);
 	case ETH_SS_PRIV_FLAGS:
 		return GVE_PRIV_FLAGS_STR_LEN;
 	default:
@@ -153,18 +166,20 @@ gve_get_ethtool_stats(struct net_device *netdev,
 	struct gve_priv *priv;
 	bool skip_nic_stats;
 	unsigned int start;
+	int num_tx_queues;
 	int ring;
 	int i, j;
 
 	ASSERT_RTNL();
 
 	priv = netdev_priv(netdev);
+	num_tx_queues = gve_num_tx_queues(priv);
 	report_stats = priv->stats_report->stats;
 	rx_qid_to_stats_idx = kmalloc_array(priv->rx_cfg.num_queues,
 					    sizeof(int), GFP_KERNEL);
 	if (!rx_qid_to_stats_idx)
 		return;
-	tx_qid_to_stats_idx = kmalloc_array(priv->tx_cfg.num_queues,
+	tx_qid_to_stats_idx = kmalloc_array(num_tx_queues,
 					    sizeof(int), GFP_KERNEL);
 	if (!tx_qid_to_stats_idx) {
 		kfree(rx_qid_to_stats_idx);
@@ -195,7 +210,7 @@ gve_get_ethtool_stats(struct net_device *netdev,
 		}
 	}
 	for (tx_pkts = 0, tx_bytes = 0, tx_dropped = 0, ring = 0;
-	     ring < priv->tx_cfg.num_queues; ring++) {
+	     ring < num_tx_queues; ring++) {
 		if (priv->tx) {
 			do {
 				start =
@@ -232,7 +247,7 @@ gve_get_ethtool_stats(struct net_device *netdev,
 	i = GVE_MAIN_STATS_LEN;
 
 	/* For rx cross-reporting stats, start from nic rx stats in report */
-	base_stats_idx = GVE_TX_STATS_REPORT_NUM * priv->tx_cfg.num_queues +
+	base_stats_idx = GVE_TX_STATS_REPORT_NUM * num_tx_queues +
 		GVE_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues;
 	max_stats_idx = NIC_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues +
 		base_stats_idx;
@@ -283,14 +298,26 @@ gve_get_ethtool_stats(struct net_device *netdev,
 			if (skip_nic_stats) {
 				/* skip NIC rx stats */
 				i += NIC_RX_STATS_REPORT_NUM;
-				continue;
-			}
-			for (j = 0; j < NIC_RX_STATS_REPORT_NUM; j++) {
-				u64 value =
-				be64_to_cpu(report_stats[rx_qid_to_stats_idx[ring] + j].value);
+			} else {
+				stats_idx = rx_qid_to_stats_idx[ring];
+				for (j = 0; j < NIC_RX_STATS_REPORT_NUM; j++) {
+					u64 value =
+						be64_to_cpu(report_stats[stats_idx + j].value);
 
-				data[i++] = value;
+					data[i++] = value;
+				}
 			}
+			/* XDP rx counters */
+			do {
+				start =	u64_stats_fetch_begin(&priv->rx[ring].statss);
+				for (j = 0; j < GVE_XDP_ACTIONS; j++)
+					data[i + j] = rx->xdp_actions[j];
+				data[i + j++] = rx->xdp_tx_errors;
+				data[i + j++] = rx->xdp_redirect_errors;
+				data[i + j++] = rx->xdp_alloc_fails;
+			} while (u64_stats_fetch_retry(&priv->rx[ring].statss,
+						       start));
+			i += GVE_XDP_ACTIONS + 3; /* XDP rx counters */
 		}
 	} else {
 		i += priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS;
@@ -298,7 +325,7 @@ gve_get_ethtool_stats(struct net_device *netdev,
 
 	/* For tx cross-reporting stats, start from nic tx stats in report */
 	base_stats_idx = max_stats_idx;
-	max_stats_idx = NIC_TX_STATS_REPORT_NUM * priv->tx_cfg.num_queues +
+	max_stats_idx = NIC_TX_STATS_REPORT_NUM * num_tx_queues +
 		max_stats_idx;
 	/* Preprocess the stats report for tx, map queue id to start index */
 	skip_nic_stats = false;
@@ -316,7 +343,7 @@ gve_get_ethtool_stats(struct net_device *netdev,
 	}
 	/* walk TX rings */
 	if (priv->tx) {
-		for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) {
+		for (ring = 0; ring < num_tx_queues; ring++) {
 			struct gve_tx_ring *tx = &priv->tx[ring];
 
 			if (gve_is_gqi(priv)) {
@@ -346,16 +373,28 @@ gve_get_ethtool_stats(struct net_device *netdev,
 			if (skip_nic_stats) {
 				/* skip NIC tx stats */
 				i += NIC_TX_STATS_REPORT_NUM;
-				continue;
-			}
-			for (j = 0; j < NIC_TX_STATS_REPORT_NUM; j++) {
-				u64 value =
-				be64_to_cpu(report_stats[tx_qid_to_stats_idx[ring] + j].value);
-				data[i++] = value;
+			} else {
+				stats_idx = tx_qid_to_stats_idx[ring];
+				for (j = 0; j < NIC_TX_STATS_REPORT_NUM; j++) {
+					u64 value =
+						be64_to_cpu(report_stats[stats_idx + j].value);
+					data[i++] = value;
+				}
 			}
+			/* XDP xsk counters */
+			data[i++] = tx->xdp_xsk_wakeup;
+			data[i++] = tx->xdp_xsk_done;
+			do {
+				start = u64_stats_fetch_begin(&priv->tx[ring].statss);
+				data[i] = tx->xdp_xsk_sent;
+				data[i + 1] = tx->xdp_xmit;
+				data[i + 2] = tx->xdp_xmit_errors;
+			} while (u64_stats_fetch_retry(&priv->tx[ring].statss,
+						       start));
+			i += 3; /* XDP tx counters */
 		}
 	} else {
-		i += priv->tx_cfg.num_queues * NUM_GVE_TX_CNTS;
+		i += num_tx_queues * NUM_GVE_TX_CNTS;
 	}
 
 	kfree(rx_qid_to_stats_idx);
@@ -412,6 +451,12 @@ static int gve_set_channels(struct net_device *netdev,
 	if (!new_rx || !new_tx)
 		return -EINVAL;
 
+	if (priv->num_xdp_queues &&
+	    (new_tx != new_rx || (2 * new_tx > priv->tx_cfg.max_queues))) {
+		dev_err(&priv->pdev->dev, "XDP load failed: The number of configured RX queues should be equal to the number of configured TX queues and the number of configured RX/TX queues should be less than or equal to half the maximum number of RX/TX queues");
+		return -EINVAL;
+	}
+
 	if (!netif_carrier_ok(netdev)) {
 		priv->tx_cfg.num_queues = new_tx;
 		priv->rx_cfg.num_queues = new_rx;
@@ -502,7 +547,9 @@ static int gve_set_priv_flags(struct net_device *netdev, u32 flags)
 {
 	struct gve_priv *priv = netdev_priv(netdev);
 	u64 ori_flags, new_flags;
+	int num_tx_queues;
 
+	num_tx_queues = gve_num_tx_queues(priv);
 	ori_flags = READ_ONCE(priv->ethtool_flags);
 	new_flags = ori_flags;
 
@@ -522,7 +569,7 @@ static int gve_set_priv_flags(struct net_device *netdev, u32 flags)
 	/* delete report stats timer. */
 	if (!(flags & BIT(0)) && (ori_flags & BIT(0))) {
 		int tx_stats_num = GVE_TX_STATS_REPORT_NUM *
-			priv->tx_cfg.num_queues;
+			num_tx_queues;
 		int rx_stats_num = GVE_RX_STATS_REPORT_NUM *
 			priv->rx_cfg.num_queues;
 
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 07111c241e0e..57ce74315eba 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -4,8 +4,10 @@
  * Copyright (C) 2015-2021 Google, Inc.
  */
 
+#include <linux/bpf.h>
 #include <linux/cpumask.h>
 #include <linux/etherdevice.h>
+#include <linux/filter.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/pci.h>
@@ -15,6 +17,7 @@
 #include <linux/utsname.h>
 #include <linux/version.h>
 #include <net/sch_generic.h>
+#include <net/xdp_sock_drv.h>
 #include "gve.h"
 #include "gve_dqo.h"
 #include "gve_adminq.h"
@@ -90,8 +93,10 @@ static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s)
 	struct gve_priv *priv = netdev_priv(dev);
 	unsigned int start;
 	u64 packets, bytes;
+	int num_tx_queues;
 	int ring;
 
+	num_tx_queues = gve_num_tx_queues(priv);
 	if (priv->rx) {
 		for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) {
 			do {
@@ -106,7 +111,7 @@ static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s)
 		}
 	}
 	if (priv->tx) {
-		for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) {
+		for (ring = 0; ring < num_tx_queues; ring++) {
 			do {
 				start =
 				  u64_stats_fetch_begin(&priv->tx[ring].statss);
@@ -180,7 +185,7 @@ static int gve_alloc_stats_report(struct gve_priv *priv)
 	int tx_stats_num, rx_stats_num;
 
 	tx_stats_num = (GVE_TX_STATS_REPORT_NUM + NIC_TX_STATS_REPORT_NUM) *
-		       priv->tx_cfg.num_queues;
+		       gve_num_tx_queues(priv);
 	rx_stats_num = (GVE_RX_STATS_REPORT_NUM + NIC_RX_STATS_REPORT_NUM) *
 		       priv->rx_cfg.num_queues;
 	priv->stats_report_len = struct_size(priv->stats_report, stats,
@@ -245,8 +250,13 @@ static int gve_napi_poll(struct napi_struct *napi, int budget)
 	block = container_of(napi, struct gve_notify_block, napi);
 	priv = block->priv;
 
-	if (block->tx)
-		reschedule |= gve_tx_poll(block, budget);
+	if (block->tx) {
+		if (block->tx->q_num < priv->tx_cfg.num_queues)
+			reschedule |= gve_tx_poll(block, budget);
+		else
+			reschedule |= gve_xdp_poll(block, budget);
+	}
+
 	if (block->rx) {
 		work_done = gve_rx_poll(block, budget);
 		reschedule |= work_done == budget;
@@ -580,13 +590,36 @@ static void gve_remove_napi(struct gve_priv *priv, int ntfy_idx)
 	netif_napi_del(&block->napi);
 }
 
+static int gve_register_xdp_qpls(struct gve_priv *priv)
+{
+	int start_id;
+	int err;
+	int i;
+
+	start_id = gve_tx_qpl_id(priv, gve_xdp_tx_start_queue_id(priv));
+	for (i = start_id; i < start_id + gve_num_xdp_qpls(priv); i++) {
+		err = gve_adminq_register_page_list(priv, &priv->qpls[i]);
+		if (err) {
+			netif_err(priv, drv, priv->dev,
+				  "failed to register queue page list %d\n",
+				  priv->qpls[i].id);
+			/* This failure will trigger a reset - no need to clean
+			 * up
+			 */
+			return err;
+		}
+	}
+	return 0;
+}
+
 static int gve_register_qpls(struct gve_priv *priv)
 {
-	int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
+	int start_id;
 	int err;
 	int i;
 
-	for (i = 0; i < num_qpls; i++) {
+	start_id = gve_tx_start_qpl_id(priv);
+	for (i = start_id; i < start_id + gve_num_tx_qpls(priv); i++) {
 		err = gve_adminq_register_page_list(priv, &priv->qpls[i]);
 		if (err) {
 			netif_err(priv, drv, priv->dev,
@@ -598,16 +631,63 @@ static int gve_register_qpls(struct gve_priv *priv)
 			return err;
 		}
 	}
+
+	start_id = gve_rx_start_qpl_id(priv);
+	for (i = start_id; i < start_id + gve_num_rx_qpls(priv); i++) {
+		err = gve_adminq_register_page_list(priv, &priv->qpls[i]);
+		if (err) {
+			netif_err(priv, drv, priv->dev,
+				  "failed to register queue page list %d\n",
+				  priv->qpls[i].id);
+			/* This failure will trigger a reset - no need to clean
+			 * up
+			 */
+			return err;
+		}
+	}
+	return 0;
+}
+
+static int gve_unregister_xdp_qpls(struct gve_priv *priv)
+{
+	int start_id;
+	int err;
+	int i;
+
+	start_id = gve_tx_qpl_id(priv, gve_xdp_tx_start_queue_id(priv));
+	for (i = start_id; i < start_id + gve_num_xdp_qpls(priv); i++) {
+		err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id);
+		/* This failure will trigger a reset - no need to clean up */
+		if (err) {
+			netif_err(priv, drv, priv->dev,
+				  "Failed to unregister queue page list %d\n",
+				  priv->qpls[i].id);
+			return err;
+		}
+	}
 	return 0;
 }
 
 static int gve_unregister_qpls(struct gve_priv *priv)
 {
-	int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
+	int start_id;
 	int err;
 	int i;
 
-	for (i = 0; i < num_qpls; i++) {
+	start_id = gve_tx_start_qpl_id(priv);
+	for (i = start_id; i < start_id + gve_num_tx_qpls(priv); i++) {
+		err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id);
+		/* This failure will trigger a reset - no need to clean up */
+		if (err) {
+			netif_err(priv, drv, priv->dev,
+				  "Failed to unregister queue page list %d\n",
+				  priv->qpls[i].id);
+			return err;
+		}
+	}
+
+	start_id = gve_rx_start_qpl_id(priv);
+	for (i = start_id; i < start_id + gve_num_rx_qpls(priv); i++) {
 		err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id);
 		/* This failure will trigger a reset - no need to clean up */
 		if (err) {
@@ -620,22 +700,44 @@ static int gve_unregister_qpls(struct gve_priv *priv)
 	return 0;
 }
 
+static int gve_create_xdp_rings(struct gve_priv *priv)
+{
+	int err;
+
+	err = gve_adminq_create_tx_queues(priv,
+					  gve_xdp_tx_start_queue_id(priv),
+					  priv->num_xdp_queues);
+	if (err) {
+		netif_err(priv, drv, priv->dev, "failed to create %d XDP tx queues\n",
+			  priv->num_xdp_queues);
+		/* This failure will trigger a reset - no need to clean
+		 * up
+		 */
+		return err;
+	}
+	netif_dbg(priv, drv, priv->dev, "created %d XDP tx queues\n",
+		  priv->num_xdp_queues);
+
+	return 0;
+}
+
 static int gve_create_rings(struct gve_priv *priv)
 {
+	int num_tx_queues = gve_num_tx_queues(priv);
 	int err;
 	int i;
 
-	err = gve_adminq_create_tx_queues(priv, priv->tx_cfg.num_queues);
+	err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues);
 	if (err) {
 		netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n",
-			  priv->tx_cfg.num_queues);
+			  num_tx_queues);
 		/* This failure will trigger a reset - no need to clean
 		 * up
 		 */
 		return err;
 	}
 	netif_dbg(priv, drv, priv->dev, "created %d tx queues\n",
-		  priv->tx_cfg.num_queues);
+		  num_tx_queues);
 
 	err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);
 	if (err) {
@@ -668,6 +770,23 @@ static int gve_create_rings(struct gve_priv *priv)
 	return 0;
 }
 
+static void add_napi_init_xdp_sync_stats(struct gve_priv *priv,
+					 int (*napi_poll)(struct napi_struct *napi,
+							  int budget))
+{
+	int start_id = gve_xdp_tx_start_queue_id(priv);
+	int i;
+
+	/* Add xdp tx napi & init sync stats*/
+	for (i = start_id; i < start_id + priv->num_xdp_queues; i++) {
+		int ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
+
+		u64_stats_init(&priv->tx[i].statss);
+		priv->tx[i].ntfy_id = ntfy_idx;
+		gve_add_napi(priv, ntfy_idx, napi_poll);
+	}
+}
+
 static void add_napi_init_sync_stats(struct gve_priv *priv,
 				     int (*napi_poll)(struct napi_struct *napi,
 						      int budget))
@@ -675,7 +794,7 @@ static void add_napi_init_sync_stats(struct gve_priv *priv,
 	int i;
 
 	/* Add tx napi & init sync stats*/
-	for (i = 0; i < priv->tx_cfg.num_queues; i++) {
+	for (i = 0; i < gve_num_tx_queues(priv); i++) {
 		int ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
 
 		u64_stats_init(&priv->tx[i].statss);
@@ -692,34 +811,51 @@ static void add_napi_init_sync_stats(struct gve_priv *priv,
 	}
 }
 
-static void gve_tx_free_rings(struct gve_priv *priv)
+static void gve_tx_free_rings(struct gve_priv *priv, int start_id, int num_rings)
 {
 	if (gve_is_gqi(priv)) {
-		gve_tx_free_rings_gqi(priv);
+		gve_tx_free_rings_gqi(priv, start_id, num_rings);
 	} else {
 		gve_tx_free_rings_dqo(priv);
 	}
 }
 
+static int gve_alloc_xdp_rings(struct gve_priv *priv)
+{
+	int start_id;
+	int err = 0;
+
+	if (!priv->num_xdp_queues)
+		return 0;
+
+	start_id = gve_xdp_tx_start_queue_id(priv);
+	err = gve_tx_alloc_rings(priv, start_id, priv->num_xdp_queues);
+	if (err)
+		return err;
+	add_napi_init_xdp_sync_stats(priv, gve_napi_poll);
+
+	return 0;
+}
+
 static int gve_alloc_rings(struct gve_priv *priv)
 {
 	int err;
 
 	/* Setup tx rings */
-	priv->tx = kvcalloc(priv->tx_cfg.num_queues, sizeof(*priv->tx),
+	priv->tx = kvcalloc(priv->tx_cfg.max_queues, sizeof(*priv->tx),
 			    GFP_KERNEL);
 	if (!priv->tx)
 		return -ENOMEM;
 
 	if (gve_is_gqi(priv))
-		err = gve_tx_alloc_rings(priv);
+		err = gve_tx_alloc_rings(priv, 0, gve_num_tx_queues(priv));
 	else
 		err = gve_tx_alloc_rings_dqo(priv);
 	if (err)
 		goto free_tx;
 
 	/* Setup rx rings */
-	priv->rx = kvcalloc(priv->rx_cfg.num_queues, sizeof(*priv->rx),
+	priv->rx = kvcalloc(priv->rx_cfg.max_queues, sizeof(*priv->rx),
 			    GFP_KERNEL);
 	if (!priv->rx) {
 		err = -ENOMEM;
@@ -744,18 +880,39 @@ free_rx:
 	kvfree(priv->rx);
 	priv->rx = NULL;
 free_tx_queue:
-	gve_tx_free_rings(priv);
+	gve_tx_free_rings(priv, 0, gve_num_tx_queues(priv));
 free_tx:
 	kvfree(priv->tx);
 	priv->tx = NULL;
 	return err;
 }
 
+static int gve_destroy_xdp_rings(struct gve_priv *priv)
+{
+	int start_id;
+	int err;
+
+	start_id = gve_xdp_tx_start_queue_id(priv);
+	err = gve_adminq_destroy_tx_queues(priv,
+					   start_id,
+					   priv->num_xdp_queues);
+	if (err) {
+		netif_err(priv, drv, priv->dev,
+			  "failed to destroy XDP queues\n");
+		/* This failure will trigger a reset - no need to clean up */
+		return err;
+	}
+	netif_dbg(priv, drv, priv->dev, "destroyed XDP queues\n");
+
+	return 0;
+}
+
 static int gve_destroy_rings(struct gve_priv *priv)
 {
+	int num_tx_queues = gve_num_tx_queues(priv);
 	int err;
 
-	err = gve_adminq_destroy_tx_queues(priv, priv->tx_cfg.num_queues);
+	err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues);
 	if (err) {
 		netif_err(priv, drv, priv->dev,
 			  "failed to destroy tx queues\n");
@@ -782,17 +939,33 @@ static void gve_rx_free_rings(struct gve_priv *priv)
 		gve_rx_free_rings_dqo(priv);
 }
 
+static void gve_free_xdp_rings(struct gve_priv *priv)
+{
+	int ntfy_idx, start_id;
+	int i;
+
+	start_id = gve_xdp_tx_start_queue_id(priv);
+	if (priv->tx) {
+		for (i = start_id; i <  start_id + priv->num_xdp_queues; i++) {
+			ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
+			gve_remove_napi(priv, ntfy_idx);
+		}
+		gve_tx_free_rings(priv, start_id, priv->num_xdp_queues);
+	}
+}
+
 static void gve_free_rings(struct gve_priv *priv)
 {
+	int num_tx_queues = gve_num_tx_queues(priv);
 	int ntfy_idx;
 	int i;
 
 	if (priv->tx) {
-		for (i = 0; i < priv->tx_cfg.num_queues; i++) {
+		for (i = 0; i < num_tx_queues; i++) {
 			ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
 			gve_remove_napi(priv, ntfy_idx);
 		}
-		gve_tx_free_rings(priv);
+		gve_tx_free_rings(priv, 0, num_tx_queues);
 		kvfree(priv->tx);
 		priv->tx = NULL;
 	}
@@ -889,40 +1062,68 @@ static void gve_free_queue_page_list(struct gve_priv *priv, u32 id)
 			      qpl->page_buses[i], gve_qpl_dma_dir(priv, id));
 
 	kvfree(qpl->page_buses);
+	qpl->page_buses = NULL;
 free_pages:
 	kvfree(qpl->pages);
+	qpl->pages = NULL;
 	priv->num_registered_pages -= qpl->num_entries;
 }
 
+static int gve_alloc_xdp_qpls(struct gve_priv *priv)
+{
+	int start_id;
+	int i, j;
+	int err;
+
+	start_id = gve_tx_qpl_id(priv, gve_xdp_tx_start_queue_id(priv));
+	for (i = start_id; i < start_id + gve_num_xdp_qpls(priv); i++) {
+		err = gve_alloc_queue_page_list(priv, i,
+						priv->tx_pages_per_qpl);
+		if (err)
+			goto free_qpls;
+	}
+
+	return 0;
+
+free_qpls:
+	for (j = start_id; j <= i; j++)
+		gve_free_queue_page_list(priv, j);
+	return err;
+}
+
 static int gve_alloc_qpls(struct gve_priv *priv)
 {
-	int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
+	int max_queues = priv->tx_cfg.max_queues + priv->rx_cfg.max_queues;
+	int start_id;
 	int i, j;
 	int err;
 
-	if (num_qpls == 0)
+	if (priv->queue_format != GVE_GQI_QPL_FORMAT)
 		return 0;
 
-	priv->qpls = kvcalloc(num_qpls, sizeof(*priv->qpls), GFP_KERNEL);
+	priv->qpls = kvcalloc(max_queues, sizeof(*priv->qpls), GFP_KERNEL);
 	if (!priv->qpls)
 		return -ENOMEM;
 
-	for (i = 0; i < gve_num_tx_qpls(priv); i++) {
+	start_id = gve_tx_start_qpl_id(priv);
+	for (i = start_id; i < start_id + gve_num_tx_qpls(priv); i++) {
 		err = gve_alloc_queue_page_list(priv, i,
 						priv->tx_pages_per_qpl);
 		if (err)
 			goto free_qpls;
 	}
-	for (; i < num_qpls; i++) {
+
+	start_id = gve_rx_start_qpl_id(priv);
+	for (i = start_id; i < start_id + gve_num_rx_qpls(priv); i++) {
 		err = gve_alloc_queue_page_list(priv, i,
 						priv->rx_data_slot_cnt);
 		if (err)
 			goto free_qpls;
 	}
 
-	priv->qpl_cfg.qpl_map_size = BITS_TO_LONGS(num_qpls) *
+	priv->qpl_cfg.qpl_map_size = BITS_TO_LONGS(max_queues) *
 				     sizeof(unsigned long) * BITS_PER_BYTE;
-	priv->qpl_cfg.qpl_id_map = kvcalloc(BITS_TO_LONGS(num_qpls),
+	priv->qpl_cfg.qpl_id_map = kvcalloc(BITS_TO_LONGS(max_queues),
 					    sizeof(unsigned long), GFP_KERNEL);
 	if (!priv->qpl_cfg.qpl_id_map) {
 		err = -ENOMEM;
@@ -935,23 +1136,36 @@ free_qpls:
 	for (j = 0; j <= i; j++)
 		gve_free_queue_page_list(priv, j);
 	kvfree(priv->qpls);
+	priv->qpls = NULL;
 	return err;
 }
 
+static void gve_free_xdp_qpls(struct gve_priv *priv)
+{
+	int start_id;
+	int i;
+
+	start_id = gve_tx_qpl_id(priv, gve_xdp_tx_start_queue_id(priv));
+	for (i = start_id; i < start_id + gve_num_xdp_qpls(priv); i++)
+		gve_free_queue_page_list(priv, i);
+}
+
 static void gve_free_qpls(struct gve_priv *priv)
 {
-	int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
+	int max_queues = priv->tx_cfg.max_queues + priv->rx_cfg.max_queues;
 	int i;
 
-	if (num_qpls == 0)
+	if (!priv->qpls)
 		return;
 
 	kvfree(priv->qpl_cfg.qpl_id_map);
+	priv->qpl_cfg.qpl_id_map = NULL;
 
-	for (i = 0; i < num_qpls; i++)
+	for (i = 0; i < max_queues; i++)
 		gve_free_queue_page_list(priv, i);
 
 	kvfree(priv->qpls);
+	priv->qpls = NULL;
 }
 
 /* Use this to schedule a reset when the device is capable of continuing
@@ -969,11 +1183,109 @@ static int gve_reset_recovery(struct gve_priv *priv, bool was_up);
 static void gve_turndown(struct gve_priv *priv);
 static void gve_turnup(struct gve_priv *priv);
 
+static int gve_reg_xdp_info(struct gve_priv *priv, struct net_device *dev)
+{
+	struct napi_struct *napi;
+	struct gve_rx_ring *rx;
+	int err = 0;
+	int i, j;
+	u32 tx_qid;
+
+	if (!priv->num_xdp_queues)
+		return 0;
+
+	for (i = 0; i < priv->rx_cfg.num_queues; i++) {
+		rx = &priv->rx[i];
+		napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
+
+		err = xdp_rxq_info_reg(&rx->xdp_rxq, dev, i,
+				       napi->napi_id);
+		if (err)
+			goto err;
+		err = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq,
+						 MEM_TYPE_PAGE_SHARED, NULL);
+		if (err)
+			goto err;
+		rx->xsk_pool = xsk_get_pool_from_qid(dev, i);
+		if (rx->xsk_pool) {
+			err = xdp_rxq_info_reg(&rx->xsk_rxq, dev, i,
+					       napi->napi_id);
+			if (err)
+				goto err;
+			err = xdp_rxq_info_reg_mem_model(&rx->xsk_rxq,
+							 MEM_TYPE_XSK_BUFF_POOL, NULL);
+			if (err)
+				goto err;
+			xsk_pool_set_rxq_info(rx->xsk_pool,
+					      &rx->xsk_rxq);
+		}
+	}
+
+	for (i = 0; i < priv->num_xdp_queues; i++) {
+		tx_qid = gve_xdp_tx_queue_id(priv, i);
+		priv->tx[tx_qid].xsk_pool = xsk_get_pool_from_qid(dev, i);
+	}
+	return 0;
+
+err:
+	for (j = i; j >= 0; j--) {
+		rx = &priv->rx[j];
+		if (xdp_rxq_info_is_reg(&rx->xdp_rxq))
+			xdp_rxq_info_unreg(&rx->xdp_rxq);
+		if (xdp_rxq_info_is_reg(&rx->xsk_rxq))
+			xdp_rxq_info_unreg(&rx->xsk_rxq);
+	}
+	return err;
+}
+
+static void gve_unreg_xdp_info(struct gve_priv *priv)
+{
+	int i, tx_qid;
+
+	if (!priv->num_xdp_queues)
+		return;
+
+	for (i = 0; i < priv->rx_cfg.num_queues; i++) {
+		struct gve_rx_ring *rx = &priv->rx[i];
+
+		xdp_rxq_info_unreg(&rx->xdp_rxq);
+		if (rx->xsk_pool) {
+			xdp_rxq_info_unreg(&rx->xsk_rxq);
+			rx->xsk_pool = NULL;
+		}
+	}
+
+	for (i = 0; i < priv->num_xdp_queues; i++) {
+		tx_qid = gve_xdp_tx_queue_id(priv, i);
+		priv->tx[tx_qid].xsk_pool = NULL;
+	}
+}
+
+static void gve_drain_page_cache(struct gve_priv *priv)
+{
+	struct page_frag_cache *nc;
+	int i;
+
+	for (i = 0; i < priv->rx_cfg.num_queues; i++) {
+		nc = &priv->rx[i].page_cache;
+		if (nc->va) {
+			__page_frag_cache_drain(virt_to_page(nc->va),
+						nc->pagecnt_bias);
+			nc->va = NULL;
+		}
+	}
+}
+
 static int gve_open(struct net_device *dev)
 {
 	struct gve_priv *priv = netdev_priv(dev);
 	int err;
 
+	if (priv->xdp_prog)
+		priv->num_xdp_queues = priv->rx_cfg.num_queues;
+	else
+		priv->num_xdp_queues = 0;
+
 	err = gve_alloc_qpls(priv);
 	if (err)
 		return err;
@@ -989,6 +1301,10 @@ static int gve_open(struct net_device *dev)
 	if (err)
 		goto free_rings;
 
+	err = gve_reg_xdp_info(priv, dev);
+	if (err)
+		goto free_rings;
+
 	err = gve_register_qpls(priv);
 	if (err)
 		goto reset;
@@ -1043,6 +1359,7 @@ static int gve_close(struct net_device *dev)
 	netif_carrier_off(dev);
 	if (gve_get_device_rings_ok(priv)) {
 		gve_turndown(priv);
+		gve_drain_page_cache(priv);
 		err = gve_destroy_rings(priv);
 		if (err)
 			goto err;
@@ -1053,6 +1370,7 @@ static int gve_close(struct net_device *dev)
 	}
 	del_timer_sync(&priv->stats_report_timer);
 
+	gve_unreg_xdp_info(priv);
 	gve_free_rings(priv);
 	gve_free_qpls(priv);
 	priv->interface_down_cnt++;
@@ -1069,6 +1387,306 @@ err:
 	return gve_reset_recovery(priv, false);
 }
 
+static int gve_remove_xdp_queues(struct gve_priv *priv)
+{
+	int err;
+
+	err = gve_destroy_xdp_rings(priv);
+	if (err)
+		return err;
+
+	err = gve_unregister_xdp_qpls(priv);
+	if (err)
+		return err;
+
+	gve_unreg_xdp_info(priv);
+	gve_free_xdp_rings(priv);
+	gve_free_xdp_qpls(priv);
+	priv->num_xdp_queues = 0;
+	return 0;
+}
+
+static int gve_add_xdp_queues(struct gve_priv *priv)
+{
+	int err;
+
+	priv->num_xdp_queues = priv->tx_cfg.num_queues;
+
+	err = gve_alloc_xdp_qpls(priv);
+	if (err)
+		goto err;
+
+	err = gve_alloc_xdp_rings(priv);
+	if (err)
+		goto free_xdp_qpls;
+
+	err = gve_reg_xdp_info(priv, priv->dev);
+	if (err)
+		goto free_xdp_rings;
+
+	err = gve_register_xdp_qpls(priv);
+	if (err)
+		goto free_xdp_rings;
+
+	err = gve_create_xdp_rings(priv);
+	if (err)
+		goto free_xdp_rings;
+
+	return 0;
+
+free_xdp_rings:
+	gve_free_xdp_rings(priv);
+free_xdp_qpls:
+	gve_free_xdp_qpls(priv);
+err:
+	priv->num_xdp_queues = 0;
+	return err;
+}
+
+static void gve_handle_link_status(struct gve_priv *priv, bool link_status)
+{
+	if (!gve_get_napi_enabled(priv))
+		return;
+
+	if (link_status == netif_carrier_ok(priv->dev))
+		return;
+
+	if (link_status) {
+		netdev_info(priv->dev, "Device link is up.\n");
+		netif_carrier_on(priv->dev);
+	} else {
+		netdev_info(priv->dev, "Device link is down.\n");
+		netif_carrier_off(priv->dev);
+	}
+}
+
+static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog,
+		       struct netlink_ext_ack *extack)
+{
+	struct bpf_prog *old_prog;
+	int err = 0;
+	u32 status;
+
+	old_prog = READ_ONCE(priv->xdp_prog);
+	if (!netif_carrier_ok(priv->dev)) {
+		WRITE_ONCE(priv->xdp_prog, prog);
+		if (old_prog)
+			bpf_prog_put(old_prog);
+		return 0;
+	}
+
+	gve_turndown(priv);
+	if (!old_prog && prog) {
+		// Allocate XDP TX queues if an XDP program is
+		// being installed
+		err = gve_add_xdp_queues(priv);
+		if (err)
+			goto out;
+	} else if (old_prog && !prog) {
+		// Remove XDP TX queues if an XDP program is
+		// being uninstalled
+		err = gve_remove_xdp_queues(priv);
+		if (err)
+			goto out;
+	}
+	WRITE_ONCE(priv->xdp_prog, prog);
+	if (old_prog)
+		bpf_prog_put(old_prog);
+
+out:
+	gve_turnup(priv);
+	status = ioread32be(&priv->reg_bar0->device_status);
+	gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
+	return err;
+}
+
+static int gve_xsk_pool_enable(struct net_device *dev,
+			       struct xsk_buff_pool *pool,
+			       u16 qid)
+{
+	struct gve_priv *priv = netdev_priv(dev);
+	struct napi_struct *napi;
+	struct gve_rx_ring *rx;
+	int tx_qid;
+	int err;
+
+	if (qid >= priv->rx_cfg.num_queues) {
+		dev_err(&priv->pdev->dev, "xsk pool invalid qid %d", qid);
+		return -EINVAL;
+	}
+	if (xsk_pool_get_rx_frame_size(pool) <
+	     priv->dev->max_mtu + sizeof(struct ethhdr)) {
+		dev_err(&priv->pdev->dev, "xsk pool frame_len too small");
+		return -EINVAL;
+	}
+
+	err = xsk_pool_dma_map(pool, &priv->pdev->dev,
+			       DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
+	if (err)
+		return err;
+
+	/* If XDP prog is not installed, return */
+	if (!priv->xdp_prog)
+		return 0;
+
+	rx = &priv->rx[qid];
+	napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
+	err = xdp_rxq_info_reg(&rx->xsk_rxq, dev, qid, napi->napi_id);
+	if (err)
+		goto err;
+
+	err = xdp_rxq_info_reg_mem_model(&rx->xsk_rxq,
+					 MEM_TYPE_XSK_BUFF_POOL, NULL);
+	if (err)
+		goto err;
+
+	xsk_pool_set_rxq_info(pool, &rx->xsk_rxq);
+	rx->xsk_pool = pool;
+
+	tx_qid = gve_xdp_tx_queue_id(priv, qid);
+	priv->tx[tx_qid].xsk_pool = pool;
+
+	return 0;
+err:
+	if (xdp_rxq_info_is_reg(&rx->xsk_rxq))
+		xdp_rxq_info_unreg(&rx->xsk_rxq);
+
+	xsk_pool_dma_unmap(pool,
+			   DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
+	return err;
+}
+
+static int gve_xsk_pool_disable(struct net_device *dev,
+				u16 qid)
+{
+	struct gve_priv *priv = netdev_priv(dev);
+	struct napi_struct *napi_rx;
+	struct napi_struct *napi_tx;
+	struct xsk_buff_pool *pool;
+	int tx_qid;
+
+	pool = xsk_get_pool_from_qid(dev, qid);
+	if (!pool)
+		return -EINVAL;
+	if (qid >= priv->rx_cfg.num_queues)
+		return -EINVAL;
+
+	/* If XDP prog is not installed, unmap DMA and return */
+	if (!priv->xdp_prog)
+		goto done;
+
+	tx_qid = gve_xdp_tx_queue_id(priv, qid);
+	if (!netif_running(dev)) {
+		priv->rx[qid].xsk_pool = NULL;
+		xdp_rxq_info_unreg(&priv->rx[qid].xsk_rxq);
+		priv->tx[tx_qid].xsk_pool = NULL;
+		goto done;
+	}
+
+	napi_rx = &priv->ntfy_blocks[priv->rx[qid].ntfy_id].napi;
+	napi_disable(napi_rx); /* make sure current rx poll is done */
+
+	napi_tx = &priv->ntfy_blocks[priv->tx[tx_qid].ntfy_id].napi;
+	napi_disable(napi_tx); /* make sure current tx poll is done */
+
+	priv->rx[qid].xsk_pool = NULL;
+	xdp_rxq_info_unreg(&priv->rx[qid].xsk_rxq);
+	priv->tx[tx_qid].xsk_pool = NULL;
+	smp_mb(); /* Make sure it is visible to the workers on datapath */
+
+	napi_enable(napi_rx);
+	if (gve_rx_work_pending(&priv->rx[qid]))
+		napi_schedule(napi_rx);
+
+	napi_enable(napi_tx);
+	if (gve_tx_clean_pending(priv, &priv->tx[tx_qid]))
+		napi_schedule(napi_tx);
+
+done:
+	xsk_pool_dma_unmap(pool,
+			   DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
+	return 0;
+}
+
+static int gve_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
+{
+	struct gve_priv *priv = netdev_priv(dev);
+	int tx_queue_id = gve_xdp_tx_queue_id(priv, queue_id);
+
+	if (queue_id >= priv->rx_cfg.num_queues || !priv->xdp_prog)
+		return -EINVAL;
+
+	if (flags & XDP_WAKEUP_TX) {
+		struct gve_tx_ring *tx = &priv->tx[tx_queue_id];
+		struct napi_struct *napi =
+			&priv->ntfy_blocks[tx->ntfy_id].napi;
+
+		if (!napi_if_scheduled_mark_missed(napi)) {
+			/* Call local_bh_enable to trigger SoftIRQ processing */
+			local_bh_disable();
+			napi_schedule(napi);
+			local_bh_enable();
+		}
+
+		tx->xdp_xsk_wakeup++;
+	}
+
+	return 0;
+}
+
+static int verify_xdp_configuration(struct net_device *dev)
+{
+	struct gve_priv *priv = netdev_priv(dev);
+
+	if (dev->features & NETIF_F_LRO) {
+		netdev_warn(dev, "XDP is not supported when LRO is on.\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (priv->queue_format != GVE_GQI_QPL_FORMAT) {
+		netdev_warn(dev, "XDP is not supported in mode %d.\n",
+			    priv->queue_format);
+		return -EOPNOTSUPP;
+	}
+
+	if (dev->mtu > (PAGE_SIZE / 2) - sizeof(struct ethhdr) - GVE_RX_PAD) {
+		netdev_warn(dev, "XDP is not supported for mtu %d.\n",
+			    dev->mtu);
+		return -EOPNOTSUPP;
+	}
+
+	if (priv->rx_cfg.num_queues != priv->tx_cfg.num_queues ||
+	    (2 * priv->tx_cfg.num_queues > priv->tx_cfg.max_queues)) {
+		netdev_warn(dev, "XDP load failed: The number of configured RX queues %d should be equal to the number of configured TX queues %d and the number of configured RX/TX queues should be less than or equal to half the maximum number of RX/TX queues %d",
+			    priv->rx_cfg.num_queues,
+			    priv->tx_cfg.num_queues,
+			    priv->tx_cfg.max_queues);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int gve_xdp(struct net_device *dev, struct netdev_bpf *xdp)
+{
+	struct gve_priv *priv = netdev_priv(dev);
+	int err;
+
+	err = verify_xdp_configuration(dev);
+	if (err)
+		return err;
+	switch (xdp->command) {
+	case XDP_SETUP_PROG:
+		return gve_set_xdp(priv, xdp->prog, xdp->extack);
+	case XDP_SETUP_XSK_POOL:
+		if (xdp->xsk.pool)
+			return gve_xsk_pool_enable(dev, xdp->xsk.pool, xdp->xsk.queue_id);
+		else
+			return gve_xsk_pool_disable(dev, xdp->xsk.queue_id);
+	default:
+		return -EINVAL;
+	}
+}
+
 int gve_adjust_queues(struct gve_priv *priv,
 		      struct gve_queue_config new_rx_config,
 		      struct gve_queue_config new_tx_config)
@@ -1118,7 +1736,7 @@ static void gve_turndown(struct gve_priv *priv)
 		return;
 
 	/* Disable napi to prevent more work from coming in */
-	for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
+	for (idx = 0; idx < gve_num_tx_queues(priv); idx++) {
 		int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
 		struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 
@@ -1146,7 +1764,7 @@ static void gve_turnup(struct gve_priv *priv)
 	netif_tx_start_all_queues(priv->dev);
 
 	/* Enable napi and unmask interrupts for all queues */
-	for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
+	for (idx = 0; idx < gve_num_tx_queues(priv); idx++) {
 		int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
 		struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 
@@ -1263,6 +1881,9 @@ static const struct net_device_ops gve_netdev_ops = {
 	.ndo_get_stats64	=	gve_get_stats,
 	.ndo_tx_timeout         =       gve_tx_timeout,
 	.ndo_set_features	=	gve_set_features,
+	.ndo_bpf		=	gve_xdp,
+	.ndo_xdp_xmit		=	gve_xdp_xmit,
+	.ndo_xsk_wakeup		=	gve_xsk_wakeup,
 };
 
 static void gve_handle_status(struct gve_priv *priv, u32 status)
@@ -1306,7 +1927,7 @@ void gve_handle_report_stats(struct gve_priv *priv)
 	be64_add_cpu(&priv->stats_report->written_count, 1);
 	/* tx stats */
 	if (priv->tx) {
-		for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
+		for (idx = 0; idx < gve_num_tx_queues(priv); idx++) {
 			u32 last_completion = 0;
 			u32 tx_frames = 0;
 
@@ -1369,23 +1990,6 @@ void gve_handle_report_stats(struct gve_priv *priv)
 	}
 }
 
-static void gve_handle_link_status(struct gve_priv *priv, bool link_status)
-{
-	if (!gve_get_napi_enabled(priv))
-		return;
-
-	if (link_status == netif_carrier_ok(priv->dev))
-		return;
-
-	if (link_status) {
-		netdev_info(priv->dev, "Device link is up.\n");
-		netif_carrier_on(priv->dev);
-	} else {
-		netdev_info(priv->dev, "Device link is down.\n");
-		netif_carrier_off(priv->dev);
-	}
-}
-
 /* Handle NIC status register changes, reset requests and report stats */
 static void gve_service_task(struct work_struct *work)
 {
@@ -1399,6 +2003,18 @@ static void gve_service_task(struct work_struct *work)
 	gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
 }
 
+static void gve_set_netdev_xdp_features(struct gve_priv *priv)
+{
+	if (priv->queue_format == GVE_GQI_QPL_FORMAT) {
+		priv->dev->xdp_features = NETDEV_XDP_ACT_BASIC;
+		priv->dev->xdp_features |= NETDEV_XDP_ACT_REDIRECT;
+		priv->dev->xdp_features |= NETDEV_XDP_ACT_NDO_XMIT;
+		priv->dev->xdp_features |= NETDEV_XDP_ACT_XSK_ZEROCOPY;
+	} else {
+		priv->dev->xdp_features = 0;
+	}
+}
+
 static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
 {
 	int num_ntfy;
@@ -1477,6 +2093,7 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
 	}
 
 setup_device:
+	gve_set_netdev_xdp_features(priv);
 	err = gve_setup_device_resources(priv);
 	if (!err)
 		return 0;
diff --git a/drivers/net/ethernet/google/gve/gve_rx.c b/drivers/net/ethernet/google/gve/gve_rx.c
index 1f55137722b0..d1da7413dc4d 100644
--- a/drivers/net/ethernet/google/gve/gve_rx.c
+++ b/drivers/net/ethernet/google/gve/gve_rx.c
@@ -8,6 +8,9 @@
 #include "gve_adminq.h"
 #include "gve_utils.h"
 #include <linux/etherdevice.h>
+#include <linux/filter.h>
+#include <net/xdp.h>
+#include <net/xdp_sock_drv.h>
 
 static void gve_rx_free_buffer(struct device *dev,
 			       struct gve_rx_slot_page_info *page_info,
@@ -124,7 +127,7 @@ static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
 		return -ENOMEM;
 
 	if (!rx->data.raw_addressing) {
-		rx->data.qpl = gve_assign_rx_qpl(priv);
+		rx->data.qpl = gve_assign_rx_qpl(priv, rx->q_num);
 		if (!rx->data.qpl) {
 			kvfree(rx->data.page_info);
 			rx->data.page_info = NULL;
@@ -556,7 +559,7 @@ static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
 
 	if (len <= priv->rx_copybreak && is_only_frag)  {
 		/* Just copy small packets */
-		skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD);
+		skb = gve_rx_copy(netdev, napi, page_info, len);
 		if (skb) {
 			u64_stats_update_begin(&rx->statss);
 			rx->rx_copied_pkt++;
@@ -591,6 +594,107 @@ static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
 	return skb;
 }
 
+static int gve_xsk_pool_redirect(struct net_device *dev,
+				 struct gve_rx_ring *rx,
+				 void *data, int len,
+				 struct bpf_prog *xdp_prog)
+{
+	struct xdp_buff *xdp;
+	int err;
+
+	if (rx->xsk_pool->frame_len < len)
+		return -E2BIG;
+	xdp = xsk_buff_alloc(rx->xsk_pool);
+	if (!xdp) {
+		u64_stats_update_begin(&rx->statss);
+		rx->xdp_alloc_fails++;
+		u64_stats_update_end(&rx->statss);
+		return -ENOMEM;
+	}
+	xdp->data_end = xdp->data + len;
+	memcpy(xdp->data, data, len);
+	err = xdp_do_redirect(dev, xdp, xdp_prog);
+	if (err)
+		xsk_buff_free(xdp);
+	return err;
+}
+
+static int gve_xdp_redirect(struct net_device *dev, struct gve_rx_ring *rx,
+			    struct xdp_buff *orig, struct bpf_prog *xdp_prog)
+{
+	int total_len, len = orig->data_end - orig->data;
+	int headroom = XDP_PACKET_HEADROOM;
+	struct xdp_buff new;
+	void *frame;
+	int err;
+
+	if (rx->xsk_pool)
+		return gve_xsk_pool_redirect(dev, rx, orig->data,
+					     len, xdp_prog);
+
+	total_len = headroom + SKB_DATA_ALIGN(len) +
+		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+	frame = page_frag_alloc(&rx->page_cache, total_len, GFP_ATOMIC);
+	if (!frame) {
+		u64_stats_update_begin(&rx->statss);
+		rx->xdp_alloc_fails++;
+		u64_stats_update_end(&rx->statss);
+		return -ENOMEM;
+	}
+	xdp_init_buff(&new, total_len, &rx->xdp_rxq);
+	xdp_prepare_buff(&new, frame, headroom, len, false);
+	memcpy(new.data, orig->data, len);
+
+	err = xdp_do_redirect(dev, &new, xdp_prog);
+	if (err)
+		page_frag_free(frame);
+
+	return err;
+}
+
+static void gve_xdp_done(struct gve_priv *priv, struct gve_rx_ring *rx,
+			 struct xdp_buff *xdp, struct bpf_prog *xprog,
+			 int xdp_act)
+{
+	struct gve_tx_ring *tx;
+	int tx_qid;
+	int err;
+
+	switch (xdp_act) {
+	case XDP_ABORTED:
+	case XDP_DROP:
+	default:
+		break;
+	case XDP_TX:
+		tx_qid = gve_xdp_tx_queue_id(priv, rx->q_num);
+		tx = &priv->tx[tx_qid];
+		spin_lock(&tx->xdp_lock);
+		err = gve_xdp_xmit_one(priv, tx, xdp->data,
+				       xdp->data_end - xdp->data, NULL);
+		spin_unlock(&tx->xdp_lock);
+
+		if (unlikely(err)) {
+			u64_stats_update_begin(&rx->statss);
+			rx->xdp_tx_errors++;
+			u64_stats_update_end(&rx->statss);
+		}
+		break;
+	case XDP_REDIRECT:
+		err = gve_xdp_redirect(priv->dev, rx, xdp, xprog);
+
+		if (unlikely(err)) {
+			u64_stats_update_begin(&rx->statss);
+			rx->xdp_redirect_errors++;
+			u64_stats_update_end(&rx->statss);
+		}
+		break;
+	}
+	u64_stats_update_begin(&rx->statss);
+	if ((u32)xdp_act < GVE_XDP_ACTIONS)
+		rx->xdp_actions[xdp_act]++;
+	u64_stats_update_end(&rx->statss);
+}
+
 #define GVE_PKTCONT_BIT_IS_SET(x) (GVE_RXF_PKT_CONT & (x))
 static void gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
 		   struct gve_rx_desc *desc, u32 idx,
@@ -603,9 +707,12 @@ static void gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
 	union gve_rx_data_slot *data_slot;
 	struct gve_priv *priv = rx->gve;
 	struct sk_buff *skb = NULL;
+	struct bpf_prog *xprog;
+	struct xdp_buff xdp;
 	dma_addr_t page_bus;
 	void *va;
 
+	u16 len = frag_size;
 	struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
 	bool is_first_frag = ctx->frag_cnt == 0;
 
@@ -645,9 +752,35 @@ static void gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
 	dma_sync_single_for_cpu(&priv->pdev->dev, page_bus,
 				PAGE_SIZE, DMA_FROM_DEVICE);
 	page_info->pad = is_first_frag ? GVE_RX_PAD : 0;
+	len -= page_info->pad;
 	frag_size -= page_info->pad;
 
-	skb = gve_rx_skb(priv, rx, page_info, napi, frag_size,
+	xprog = READ_ONCE(priv->xdp_prog);
+	if (xprog && is_only_frag) {
+		void *old_data;
+		int xdp_act;
+
+		xdp_init_buff(&xdp, rx->packet_buffer_size, &rx->xdp_rxq);
+		xdp_prepare_buff(&xdp, page_info->page_address +
+				 page_info->page_offset, GVE_RX_PAD,
+				 len, false);
+		old_data = xdp.data;
+		xdp_act = bpf_prog_run_xdp(xprog, &xdp);
+		if (xdp_act != XDP_PASS) {
+			gve_xdp_done(priv, rx, &xdp, xprog, xdp_act);
+			ctx->total_size += frag_size;
+			goto finish_ok_pkt;
+		}
+
+		page_info->pad += xdp.data - old_data;
+		len = xdp.data_end - xdp.data;
+
+		u64_stats_update_begin(&rx->statss);
+		rx->xdp_actions[XDP_PASS]++;
+		u64_stats_update_end(&rx->statss);
+	}
+
+	skb = gve_rx_skb(priv, rx, page_info, napi, len,
 			 data_slot, is_only_frag);
 	if (!skb) {
 		u64_stats_update_begin(&rx->statss);
@@ -773,6 +906,8 @@ static bool gve_rx_refill_buffers(struct gve_priv *priv, struct gve_rx_ring *rx)
 static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
 			     netdev_features_t feat)
 {
+	u64 xdp_redirects = rx->xdp_actions[XDP_REDIRECT];
+	u64 xdp_txs = rx->xdp_actions[XDP_TX];
 	struct gve_rx_ctx *ctx = &rx->ctx;
 	struct gve_priv *priv = rx->gve;
 	struct gve_rx_cnts cnts = {0};
@@ -820,6 +955,12 @@ static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
 		u64_stats_update_end(&rx->statss);
 	}
 
+	if (xdp_txs != rx->xdp_actions[XDP_TX])
+		gve_xdp_tx_flush(priv, rx->q_num);
+
+	if (xdp_redirects != rx->xdp_actions[XDP_REDIRECT])
+		xdp_do_flush();
+
 	/* restock ring slots */
 	if (!rx->data.raw_addressing) {
 		/* In QPL mode buffs are refilled as the desc are processed */
diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
index 630f42a3037b..e57b73eb70f6 100644
--- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
@@ -568,7 +568,7 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
 
 	if (eop && buf_len <= priv->rx_copybreak) {
 		rx->ctx.skb_head = gve_rx_copy(priv->dev, napi,
-					       &buf_state->page_info, buf_len, 0);
+					       &buf_state->page_info, buf_len);
 		if (unlikely(!rx->ctx.skb_head))
 			goto error;
 		rx->ctx.skb_tail = rx->ctx.skb_head;
diff --git a/drivers/net/ethernet/google/gve/gve_tx.c b/drivers/net/ethernet/google/gve/gve_tx.c
index 5e11b8236754..813da572abca 100644
--- a/drivers/net/ethernet/google/gve/gve_tx.c
+++ b/drivers/net/ethernet/google/gve/gve_tx.c
@@ -11,6 +11,7 @@
 #include <linux/tcp.h>
 #include <linux/vmalloc.h>
 #include <linux/skbuff.h>
+#include <net/xdp_sock_drv.h>
 
 static inline void gve_tx_put_doorbell(struct gve_priv *priv,
 				       struct gve_queue_resources *q_resources,
@@ -19,6 +20,14 @@ static inline void gve_tx_put_doorbell(struct gve_priv *priv,
 	iowrite32be(val, &priv->db_bar2[be32_to_cpu(q_resources->db_index)]);
 }
 
+void gve_xdp_tx_flush(struct gve_priv *priv, u32 xdp_qid)
+{
+	u32 tx_qid = gve_xdp_tx_queue_id(priv, xdp_qid);
+	struct gve_tx_ring *tx = &priv->tx[tx_qid];
+
+	gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
+}
+
 /* gvnic can only transmit from a Registered Segment.
  * We copy skb payloads into the registered segment before writing Tx
  * descriptors and ringing the Tx doorbell.
@@ -132,6 +141,58 @@ static void gve_tx_free_fifo(struct gve_tx_fifo *fifo, size_t bytes)
 	atomic_add(bytes, &fifo->available);
 }
 
+static size_t gve_tx_clear_buffer_state(struct gve_tx_buffer_state *info)
+{
+	size_t space_freed = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(info->iov); i++) {
+		space_freed += info->iov[i].iov_len + info->iov[i].iov_padding;
+		info->iov[i].iov_len = 0;
+		info->iov[i].iov_padding = 0;
+	}
+	return space_freed;
+}
+
+static int gve_clean_xdp_done(struct gve_priv *priv, struct gve_tx_ring *tx,
+			      u32 to_do)
+{
+	struct gve_tx_buffer_state *info;
+	u32 clean_end = tx->done + to_do;
+	u64 pkts = 0, bytes = 0;
+	size_t space_freed = 0;
+	u32 xsk_complete = 0;
+	u32 idx;
+
+	for (; tx->done < clean_end; tx->done++) {
+		idx = tx->done & tx->mask;
+		info = &tx->info[idx];
+
+		if (unlikely(!info->xdp.size))
+			continue;
+
+		bytes += info->xdp.size;
+		pkts++;
+		xsk_complete += info->xdp.is_xsk;
+
+		info->xdp.size = 0;
+		if (info->xdp_frame) {
+			xdp_return_frame(info->xdp_frame);
+			info->xdp_frame = NULL;
+		}
+		space_freed += gve_tx_clear_buffer_state(info);
+	}
+
+	gve_tx_free_fifo(&tx->tx_fifo, space_freed);
+	if (xsk_complete > 0 && tx->xsk_pool)
+		xsk_tx_completed(tx->xsk_pool, xsk_complete);
+	u64_stats_update_begin(&tx->statss);
+	tx->bytes_done += bytes;
+	tx->pkt_done += pkts;
+	u64_stats_update_end(&tx->statss);
+	return pkts;
+}
+
 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
 			     u32 to_do, bool try_to_wake);
 
@@ -144,8 +205,12 @@ static void gve_tx_free_ring(struct gve_priv *priv, int idx)
 
 	gve_tx_remove_from_block(priv, idx);
 	slots = tx->mask + 1;
-	gve_clean_tx_done(priv, tx, priv->tx_desc_cnt, false);
-	netdev_tx_reset_queue(tx->netdev_txq);
+	if (tx->q_num < priv->tx_cfg.num_queues) {
+		gve_clean_tx_done(priv, tx, priv->tx_desc_cnt, false);
+		netdev_tx_reset_queue(tx->netdev_txq);
+	} else {
+		gve_clean_xdp_done(priv, tx, priv->tx_desc_cnt);
+	}
 
 	dma_free_coherent(hdev, sizeof(*tx->q_resources),
 			  tx->q_resources, tx->q_resources_bus);
@@ -177,6 +242,7 @@ static int gve_tx_alloc_ring(struct gve_priv *priv, int idx)
 	/* Make sure everything is zeroed to start */
 	memset(tx, 0, sizeof(*tx));
 	spin_lock_init(&tx->clean_lock);
+	spin_lock_init(&tx->xdp_lock);
 	tx->q_num = idx;
 
 	tx->mask = slots - 1;
@@ -195,7 +261,7 @@ static int gve_tx_alloc_ring(struct gve_priv *priv, int idx)
 	tx->raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT;
 	tx->dev = &priv->pdev->dev;
 	if (!tx->raw_addressing) {
-		tx->tx_fifo.qpl = gve_assign_tx_qpl(priv);
+		tx->tx_fifo.qpl = gve_assign_tx_qpl(priv, idx);
 		if (!tx->tx_fifo.qpl)
 			goto abort_with_desc;
 		/* map Tx FIFO */
@@ -213,7 +279,8 @@ static int gve_tx_alloc_ring(struct gve_priv *priv, int idx)
 
 	netif_dbg(priv, drv, priv->dev, "tx[%d]->bus=%lx\n", idx,
 		  (unsigned long)tx->bus);
-	tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx);
+	if (idx < priv->tx_cfg.num_queues)
+		tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx);
 	gve_tx_add_to_block(priv, idx);
 
 	return 0;
@@ -233,12 +300,12 @@ abort_with_info:
 	return -ENOMEM;
 }
 
-int gve_tx_alloc_rings(struct gve_priv *priv)
+int gve_tx_alloc_rings(struct gve_priv *priv, int start_id, int num_rings)
 {
 	int err = 0;
 	int i;
 
-	for (i = 0; i < priv->tx_cfg.num_queues; i++) {
+	for (i = start_id; i < start_id + num_rings; i++) {
 		err = gve_tx_alloc_ring(priv, i);
 		if (err) {
 			netif_err(priv, drv, priv->dev,
@@ -251,17 +318,17 @@ int gve_tx_alloc_rings(struct gve_priv *priv)
 	if (err) {
 		int j;
 
-		for (j = 0; j < i; j++)
+		for (j = start_id; j < i; j++)
 			gve_tx_free_ring(priv, j);
 	}
 	return err;
 }
 
-void gve_tx_free_rings_gqi(struct gve_priv *priv)
+void gve_tx_free_rings_gqi(struct gve_priv *priv, int start_id, int num_rings)
 {
 	int i;
 
-	for (i = 0; i < priv->tx_cfg.num_queues; i++)
+	for (i = start_id; i < start_id + num_rings; i++)
 		gve_tx_free_ring(priv, i);
 }
 
@@ -374,18 +441,18 @@ static int gve_maybe_stop_tx(struct gve_priv *priv, struct gve_tx_ring *tx,
 }
 
 static void gve_tx_fill_pkt_desc(union gve_tx_desc *pkt_desc,
-				 struct sk_buff *skb, bool is_gso,
+				 u16 csum_offset, u8 ip_summed, bool is_gso,
 				 int l4_hdr_offset, u32 desc_cnt,
-				 u16 hlen, u64 addr)
+				 u16 hlen, u64 addr, u16 pkt_len)
 {
 	/* l4_hdr_offset and csum_offset are in units of 16-bit words */
 	if (is_gso) {
 		pkt_desc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM;
-		pkt_desc->pkt.l4_csum_offset = skb->csum_offset >> 1;
+		pkt_desc->pkt.l4_csum_offset = csum_offset >> 1;
 		pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
-	} else if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
+	} else if (likely(ip_summed == CHECKSUM_PARTIAL)) {
 		pkt_desc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM;
-		pkt_desc->pkt.l4_csum_offset = skb->csum_offset >> 1;
+		pkt_desc->pkt.l4_csum_offset = csum_offset >> 1;
 		pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
 	} else {
 		pkt_desc->pkt.type_flags = GVE_TXD_STD;
@@ -393,7 +460,7 @@ static void gve_tx_fill_pkt_desc(union gve_tx_desc *pkt_desc,
 		pkt_desc->pkt.l4_hdr_offset = 0;
 	}
 	pkt_desc->pkt.desc_cnt = desc_cnt;
-	pkt_desc->pkt.len = cpu_to_be16(skb->len);
+	pkt_desc->pkt.len = cpu_to_be16(pkt_len);
 	pkt_desc->pkt.seg_len = cpu_to_be16(hlen);
 	pkt_desc->pkt.seg_addr = cpu_to_be64(addr);
 }
@@ -412,15 +479,16 @@ static void gve_tx_fill_mtd_desc(union gve_tx_desc *mtd_desc,
 }
 
 static void gve_tx_fill_seg_desc(union gve_tx_desc *seg_desc,
-				 struct sk_buff *skb, bool is_gso,
+				 u16 l3_offset, u16 gso_size,
+				 bool is_gso_v6, bool is_gso,
 				 u16 len, u64 addr)
 {
 	seg_desc->seg.type_flags = GVE_TXD_SEG;
 	if (is_gso) {
-		if (skb_is_gso_v6(skb))
+		if (is_gso_v6)
 			seg_desc->seg.type_flags |= GVE_TXSF_IPV6;
-		seg_desc->seg.l3_offset = skb_network_offset(skb) >> 1;
-		seg_desc->seg.mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
+		seg_desc->seg.l3_offset = l3_offset >> 1;
+		seg_desc->seg.mss = cpu_to_be16(gso_size);
 	}
 	seg_desc->seg.seg_len = cpu_to_be16(len);
 	seg_desc->seg.seg_addr = cpu_to_be64(addr);
@@ -471,9 +539,10 @@ static int gve_tx_add_skb_copy(struct gve_priv *priv, struct gve_tx_ring *tx, st
 	payload_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, skb->len - hlen,
 					   &info->iov[payload_iov]);
 
-	gve_tx_fill_pkt_desc(pkt_desc, skb, is_gso, l4_hdr_offset,
+	gve_tx_fill_pkt_desc(pkt_desc, skb->csum_offset, skb->ip_summed,
+			     is_gso, l4_hdr_offset,
 			     1 + mtd_desc_nr + payload_nfrags, hlen,
-			     info->iov[hdr_nfrags - 1].iov_offset);
+			     info->iov[hdr_nfrags - 1].iov_offset, skb->len);
 
 	skb_copy_bits(skb, 0,
 		      tx->tx_fifo.base + info->iov[hdr_nfrags - 1].iov_offset,
@@ -492,7 +561,9 @@ static int gve_tx_add_skb_copy(struct gve_priv *priv, struct gve_tx_ring *tx, st
 		next_idx = (tx->req + 1 + mtd_desc_nr + i - payload_iov) & tx->mask;
 		seg_desc = &tx->desc[next_idx];
 
-		gve_tx_fill_seg_desc(seg_desc, skb, is_gso,
+		gve_tx_fill_seg_desc(seg_desc, skb_network_offset(skb),
+				     skb_shinfo(skb)->gso_size,
+				     skb_is_gso_v6(skb), is_gso,
 				     info->iov[i].iov_len,
 				     info->iov[i].iov_offset);
 
@@ -550,8 +621,9 @@ static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx,
 	if (mtd_desc_nr)
 		num_descriptors++;
 
-	gve_tx_fill_pkt_desc(pkt_desc, skb, is_gso, l4_hdr_offset,
-			     num_descriptors, hlen, addr);
+	gve_tx_fill_pkt_desc(pkt_desc, skb->csum_offset, skb->ip_summed,
+			     is_gso, l4_hdr_offset,
+			     num_descriptors, hlen, addr, skb->len);
 
 	if (mtd_desc_nr) {
 		idx = (idx + 1) & tx->mask;
@@ -567,7 +639,9 @@ static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx,
 		addr += hlen;
 		idx = (idx + 1) & tx->mask;
 		seg_desc = &tx->desc[idx];
-		gve_tx_fill_seg_desc(seg_desc, skb, is_gso, len, addr);
+		gve_tx_fill_seg_desc(seg_desc, skb_network_offset(skb),
+				     skb_shinfo(skb)->gso_size,
+				     skb_is_gso_v6(skb), is_gso, len, addr);
 	}
 
 	for (i = 0; i < shinfo->nr_frags; i++) {
@@ -585,7 +659,9 @@ static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx,
 		dma_unmap_len_set(&tx->info[idx], len, len);
 		dma_unmap_addr_set(&tx->info[idx], dma, addr);
 
-		gve_tx_fill_seg_desc(seg_desc, skb, is_gso, len, addr);
+		gve_tx_fill_seg_desc(seg_desc, skb_network_offset(skb),
+				     skb_shinfo(skb)->gso_size,
+				     skb_is_gso_v6(skb), is_gso, len, addr);
 	}
 
 	return num_descriptors;
@@ -646,6 +722,103 @@ netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev)
 	return NETDEV_TX_OK;
 }
 
+static int gve_tx_fill_xdp(struct gve_priv *priv, struct gve_tx_ring *tx,
+			   void *data, int len, void *frame_p, bool is_xsk)
+{
+	int pad, nfrags, ndescs, iovi, offset;
+	struct gve_tx_buffer_state *info;
+	u32 reqi = tx->req;
+
+	pad = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, len);
+	if (pad >= GVE_GQ_TX_MIN_PKT_DESC_BYTES)
+		pad = 0;
+	info = &tx->info[reqi & tx->mask];
+	info->xdp_frame = frame_p;
+	info->xdp.size = len;
+	info->xdp.is_xsk = is_xsk;
+
+	nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, pad + len,
+				   &info->iov[0]);
+	iovi = pad > 0;
+	ndescs = nfrags - iovi;
+	offset = 0;
+
+	while (iovi < nfrags) {
+		if (!offset)
+			gve_tx_fill_pkt_desc(&tx->desc[reqi & tx->mask], 0,
+					     CHECKSUM_NONE, false, 0, ndescs,
+					     info->iov[iovi].iov_len,
+					     info->iov[iovi].iov_offset, len);
+		else
+			gve_tx_fill_seg_desc(&tx->desc[reqi & tx->mask],
+					     0, 0, false, false,
+					     info->iov[iovi].iov_len,
+					     info->iov[iovi].iov_offset);
+
+		memcpy(tx->tx_fifo.base + info->iov[iovi].iov_offset,
+		       data + offset, info->iov[iovi].iov_len);
+		gve_dma_sync_for_device(&priv->pdev->dev,
+					tx->tx_fifo.qpl->page_buses,
+					info->iov[iovi].iov_offset,
+					info->iov[iovi].iov_len);
+		offset += info->iov[iovi].iov_len;
+		iovi++;
+		reqi++;
+	}
+
+	return ndescs;
+}
+
+int gve_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
+		 u32 flags)
+{
+	struct gve_priv *priv = netdev_priv(dev);
+	struct gve_tx_ring *tx;
+	int i, err = 0, qid;
+
+	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
+		return -EINVAL;
+
+	qid = gve_xdp_tx_queue_id(priv,
+				  smp_processor_id() % priv->num_xdp_queues);
+
+	tx = &priv->tx[qid];
+
+	spin_lock(&tx->xdp_lock);
+	for (i = 0; i < n; i++) {
+		err = gve_xdp_xmit_one(priv, tx, frames[i]->data,
+				       frames[i]->len, frames[i]);
+		if (err)
+			break;
+	}
+
+	if (flags & XDP_XMIT_FLUSH)
+		gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
+
+	spin_unlock(&tx->xdp_lock);
+
+	u64_stats_update_begin(&tx->statss);
+	tx->xdp_xmit += n;
+	tx->xdp_xmit_errors += n - i;
+	u64_stats_update_end(&tx->statss);
+
+	return i ? i : err;
+}
+
+int gve_xdp_xmit_one(struct gve_priv *priv, struct gve_tx_ring *tx,
+		     void *data, int len, void *frame_p)
+{
+	int nsegs;
+
+	if (!gve_can_tx(tx, len + GVE_GQ_TX_MIN_PKT_DESC_BYTES - 1))
+		return -EBUSY;
+
+	nsegs = gve_tx_fill_xdp(priv, tx, data, len, frame_p, false);
+	tx->req += nsegs;
+
+	return 0;
+}
+
 #define GVE_TX_START_THRESH	PAGE_SIZE
 
 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
@@ -655,8 +828,8 @@ static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
 	u64 pkts = 0, bytes = 0;
 	size_t space_freed = 0;
 	struct sk_buff *skb;
-	int i, j;
 	u32 idx;
+	int j;
 
 	for (j = 0; j < to_do; j++) {
 		idx = tx->done & tx->mask;
@@ -678,12 +851,7 @@ static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
 			dev_consume_skb_any(skb);
 			if (tx->raw_addressing)
 				continue;
-			/* FIFO free */
-			for (i = 0; i < ARRAY_SIZE(info->iov); i++) {
-				space_freed += info->iov[i].iov_len + info->iov[i].iov_padding;
-				info->iov[i].iov_len = 0;
-				info->iov[i].iov_padding = 0;
-			}
+			space_freed += gve_tx_clear_buffer_state(info);
 		}
 	}
 
@@ -718,6 +886,70 @@ u32 gve_tx_load_event_counter(struct gve_priv *priv,
 	return be32_to_cpu(counter);
 }
 
+static int gve_xsk_tx(struct gve_priv *priv, struct gve_tx_ring *tx,
+		      int budget)
+{
+	struct xdp_desc desc;
+	int sent = 0, nsegs;
+	void *data;
+
+	spin_lock(&tx->xdp_lock);
+	while (sent < budget) {
+		if (!gve_can_tx(tx, GVE_TX_START_THRESH))
+			goto out;
+
+		if (!xsk_tx_peek_desc(tx->xsk_pool, &desc)) {
+			tx->xdp_xsk_done = tx->xdp_xsk_wakeup;
+			goto out;
+		}
+
+		data = xsk_buff_raw_get_data(tx->xsk_pool, desc.addr);
+		nsegs = gve_tx_fill_xdp(priv, tx, data, desc.len, NULL, true);
+		tx->req += nsegs;
+		sent++;
+	}
+out:
+	if (sent > 0) {
+		gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
+		xsk_tx_release(tx->xsk_pool);
+	}
+	spin_unlock(&tx->xdp_lock);
+	return sent;
+}
+
+bool gve_xdp_poll(struct gve_notify_block *block, int budget)
+{
+	struct gve_priv *priv = block->priv;
+	struct gve_tx_ring *tx = block->tx;
+	u32 nic_done;
+	bool repoll;
+	u32 to_do;
+
+	/* If budget is 0, do all the work */
+	if (budget == 0)
+		budget = INT_MAX;
+
+	/* Find out how much work there is to be done */
+	nic_done = gve_tx_load_event_counter(priv, tx);
+	to_do = min_t(u32, (nic_done - tx->done), budget);
+	gve_clean_xdp_done(priv, tx, to_do);
+	repoll = nic_done != tx->done;
+
+	if (tx->xsk_pool) {
+		int sent = gve_xsk_tx(priv, tx, budget);
+
+		u64_stats_update_begin(&tx->statss);
+		tx->xdp_xsk_sent += sent;
+		u64_stats_update_end(&tx->statss);
+		repoll |= (sent == budget);
+		if (xsk_uses_need_wakeup(tx->xsk_pool))
+			xsk_set_tx_need_wakeup(tx->xsk_pool);
+	}
+
+	/* If we still have work we want to repoll */
+	return repoll;
+}
+
 bool gve_tx_poll(struct gve_notify_block *block, int budget)
 {
 	struct gve_priv *priv = block->priv;
diff --git a/drivers/net/ethernet/google/gve/gve_utils.c b/drivers/net/ethernet/google/gve/gve_utils.c
index 6ba46adaaee3..26e08d753270 100644
--- a/drivers/net/ethernet/google/gve/gve_utils.c
+++ b/drivers/net/ethernet/google/gve/gve_utils.c
@@ -49,10 +49,10 @@ void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx)
 }
 
 struct sk_buff *gve_rx_copy(struct net_device *dev, struct napi_struct *napi,
-			    struct gve_rx_slot_page_info *page_info, u16 len,
-			    u16 padding)
+			    struct gve_rx_slot_page_info *page_info, u16 len)
 {
-	void *va = page_info->page_address + padding + page_info->page_offset;
+	void *va = page_info->page_address + page_info->page_offset +
+		page_info->pad;
 	struct sk_buff *skb;
 
 	skb = napi_alloc_skb(napi, len);
diff --git a/drivers/net/ethernet/google/gve/gve_utils.h b/drivers/net/ethernet/google/gve/gve_utils.h
index 79595940b351..324fd98a6112 100644
--- a/drivers/net/ethernet/google/gve/gve_utils.h
+++ b/drivers/net/ethernet/google/gve/gve_utils.h
@@ -18,8 +18,7 @@ void gve_rx_remove_from_block(struct gve_priv *priv, int queue_idx);
 void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx);
 
 struct sk_buff *gve_rx_copy(struct net_device *dev, struct napi_struct *napi,
-			    struct gve_rx_slot_page_info *page_info, u16 len,
-			    u16 pad);
+			    struct gve_rx_slot_page_info *page_info, u16 len);
 
 /* Decrement pagecnt_bias. Set it back to INT_MAX if it reached zero. */
 void gve_dec_pagecnt_bias(struct gve_rx_slot_page_info *page_info);
diff --git a/drivers/net/ethernet/hisilicon/hns/hnae.c b/drivers/net/ethernet/hisilicon/hns/hnae.c
index 9b26f0f2c748..8a1027ad340d 100644
--- a/drivers/net/ethernet/hisilicon/hns/hnae.c
+++ b/drivers/net/ethernet/hisilicon/hns/hnae.c
@@ -448,7 +448,7 @@ EXPORT_SYMBOL(hnae_ae_unregister);
 
 static int __init hnae_init(void)
 {
-	hnae_class = class_create(THIS_MODULE, "hnae");
+	hnae_class = class_create("hnae");
 	return PTR_ERR_OR_ZERO(hnae_class);
 }
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
index 40f4306449eb..9c9c72dc57e0 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
@@ -100,6 +100,7 @@ enum HNAE3_DEV_CAP_BITS {
 	HNAE3_DEV_SUPPORT_CQ_B,
 	HNAE3_DEV_SUPPORT_FEC_STATS_B,
 	HNAE3_DEV_SUPPORT_LANE_NUM_B,
+	HNAE3_DEV_SUPPORT_WOL_B,
 };
 
 #define hnae3_ae_dev_fd_supported(ae_dev) \
@@ -168,6 +169,9 @@ enum HNAE3_DEV_CAP_BITS {
 #define hnae3_ae_dev_lane_num_supported(ae_dev) \
 	test_bit(HNAE3_DEV_SUPPORT_LANE_NUM_B, (ae_dev)->caps)
 
+#define hnae3_ae_dev_wol_supported(ae_dev) \
+	test_bit(HNAE3_DEV_SUPPORT_WOL_B, (ae_dev)->caps)
+
 enum HNAE3_PF_CAP_BITS {
 	HNAE3_PF_SUPPORT_VLAN_FLTR_MDF_B = 0,
 };
@@ -561,6 +565,10 @@ struct hnae3_ae_dev {
  *   Get phc info
  * clean_vf_config
  *   Clean residual vf info after disable sriov
+ * get_wol
+ *   Get wake on lan info
+ * set_wol
+ *   Config wake on lan
  */
 struct hnae3_ae_ops {
 	int (*init_ae_dev)(struct hnae3_ae_dev *ae_dev);
@@ -760,6 +768,10 @@ struct hnae3_ae_ops {
 	void (*clean_vf_config)(struct hnae3_ae_dev *ae_dev, int num_vfs);
 	int (*get_dscp_prio)(struct hnae3_handle *handle, u8 dscp,
 			     u8 *tc_map_mode, u8 *priority);
+	void (*get_wol)(struct hnae3_handle *handle,
+			struct ethtool_wolinfo *wol);
+	int (*set_wol)(struct hnae3_handle *handle,
+		       struct ethtool_wolinfo *wol);
 };
 
 struct hnae3_dcb_ops {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c
index f671a63cecde..cbbab5b2b402 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c
@@ -155,6 +155,7 @@ static const struct hclge_comm_caps_bit_map hclge_pf_cmd_caps[] = {
 	{HCLGE_COMM_CAP_FD_B, HNAE3_DEV_SUPPORT_FD_B},
 	{HCLGE_COMM_CAP_FEC_STATS_B, HNAE3_DEV_SUPPORT_FEC_STATS_B},
 	{HCLGE_COMM_CAP_LANE_NUM_B, HNAE3_DEV_SUPPORT_LANE_NUM_B},
+	{HCLGE_COMM_CAP_WOL_B, HNAE3_DEV_SUPPORT_WOL_B},
 };
 
 static const struct hclge_comm_caps_bit_map hclge_vf_cmd_caps[] = {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h
index b1f9383b418f..de72ecbfd5ad 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h
@@ -294,6 +294,8 @@ enum hclge_opcode_type {
 	HCLGE_PPP_CMD0_INT_CMD		= 0x2100,
 	HCLGE_PPP_CMD1_INT_CMD		= 0x2101,
 	HCLGE_MAC_ETHERTYPE_IDX_RD      = 0x2105,
+	HCLGE_OPC_WOL_GET_SUPPORTED_MODE	= 0x2201,
+	HCLGE_OPC_WOL_CFG		= 0x2202,
 	HCLGE_NCSI_INT_EN		= 0x2401,
 
 	/* ROH MAC commands */
@@ -345,6 +347,7 @@ enum HCLGE_COMM_CAP_BITS {
 	HCLGE_COMM_CAP_FD_B = 21,
 	HCLGE_COMM_CAP_FEC_STATS_B = 25,
 	HCLGE_COMM_CAP_LANE_NUM_B = 27,
+	HCLGE_COMM_CAP_WOL_B = 28,
 };
 
 enum HCLGE_COMM_API_CAP_BITS {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c
index 66feb23f7b7b..4c3e90a1c4d0 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c
@@ -408,6 +408,9 @@ static struct hns3_dbg_cap_info hns3_dbg_cap[] = {
 	}, {
 		.name = "support lane num",
 		.cap_bit = HNAE3_DEV_SUPPORT_LANE_NUM_B,
+	}, {
+		.name = "support wake on lan",
+		.cap_bit = HNAE3_DEV_SUPPORT_WOL_B,
 	}
 };
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
index 25be7f8ac7cd..7356ad965487 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
@@ -13,7 +13,6 @@
 #include <linux/ipv6.h>
 #include <linux/module.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/skbuff.h>
 #include <linux/sctp.h>
 #include <net/gre.h>
@@ -1533,7 +1532,7 @@ static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring,
 	if (unlikely(rc < 0))
 		return rc;
 
-	vhdr = (struct vlan_ethhdr *)skb->data;
+	vhdr = skb_vlan_eth_hdr(skb);
 	vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority << VLAN_PRIO_SHIFT)
 					 & VLAN_PRIO_MASK);
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
index 294a14b4fdef..88af34bbee34 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
@@ -695,6 +695,12 @@ static inline unsigned int hns3_page_order(struct hns3_enet_ring *ring)
 #define hns3_get_handle(ndev) \
 	(((struct hns3_nic_priv *)netdev_priv(ndev))->ae_handle)
 
+#define hns3_get_ae_dev(handle) \
+	(pci_get_drvdata((handle)->pdev))
+
+#define hns3_get_ops(handle) \
+	((handle)->ae_algo->ops)
+
 #define hns3_gl_usec_to_reg(int_gl) ((int_gl) >> 1)
 #define hns3_gl_round_down(int_gl) round_down(int_gl, 2)
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
index 55306fe8a540..51d1278b18f6 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
@@ -2063,6 +2063,31 @@ static int hns3_get_link_ext_state(struct net_device *netdev,
 	return -ENODATA;
 }
 
+static void hns3_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
+{
+	struct hnae3_handle *handle = hns3_get_handle(netdev);
+	const struct hnae3_ae_ops *ops = hns3_get_ops(handle);
+	struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(handle);
+
+	if (!hnae3_ae_dev_wol_supported(ae_dev))
+		return;
+
+	ops->get_wol(handle, wol);
+}
+
+static int hns3_set_wol(struct net_device *netdev,
+			struct ethtool_wolinfo *wol)
+{
+	struct hnae3_handle *handle = hns3_get_handle(netdev);
+	const struct hnae3_ae_ops *ops = hns3_get_ops(handle);
+	struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(handle);
+
+	if (!hnae3_ae_dev_wol_supported(ae_dev))
+		return -EOPNOTSUPP;
+
+	return ops->set_wol(handle, wol);
+}
+
 static const struct ethtool_ops hns3vf_ethtool_ops = {
 	.supported_coalesce_params = HNS3_ETHTOOL_COALESCE,
 	.supported_ring_params = HNS3_ETHTOOL_RING,
@@ -2139,6 +2164,8 @@ static const struct ethtool_ops hns3_ethtool_ops = {
 	.set_tunable = hns3_set_tunable,
 	.reset = hns3_set_reset,
 	.get_link_ext_state = hns3_get_link_ext_state,
+	.get_wol = hns3_get_wol,
+	.set_wol = hns3_set_wol,
 };
 
 void hns3_ethtool_set_ops(struct net_device *netdev)
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
index 43cada51d8cb..91c173f40701 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
@@ -872,6 +872,18 @@ struct hclge_phy_reg_cmd {
 	u8 rsv1[18];
 };
 
+struct hclge_wol_cfg_cmd {
+	__le32 wake_on_lan_mode;
+	u8 sopass[SOPASS_MAX];
+	u8 sopass_size;
+	u8 rsv[13];
+};
+
+struct hclge_query_wol_supported_cmd {
+	__le32 supported_wake_mode;
+	u8 rsv[20];
+};
+
 struct hclge_hw;
 int hclge_cmd_send(struct hclge_hw *hw, struct hclge_desc *desc, int num);
 enum hclge_comm_cmd_status hclge_cmd_mdio_write(struct hclge_hw *hw,
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
index 07ad5f35219e..4fb5406c1951 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
@@ -11365,7 +11365,7 @@ static int hclge_pci_init(struct hclge_dev *hdev)
 	if (!hw->hw.io_base) {
 		dev_err(&pdev->dev, "Can't map configuration register space\n");
 		ret = -ENOMEM;
-		goto err_clr_master;
+		goto err_release_regions;
 	}
 
 	ret = hclge_dev_mem_map(hdev);
@@ -11378,8 +11378,7 @@ static int hclge_pci_init(struct hclge_dev *hdev)
 
 err_unmap_io_base:
 	pcim_iounmap(pdev, hdev->hw.hw.io_base);
-err_clr_master:
-	pci_clear_master(pdev);
+err_release_regions:
 	pci_release_regions(pdev);
 err_disable_device:
 	pci_disable_device(pdev);
@@ -11396,7 +11395,6 @@ static void hclge_pci_uninit(struct hclge_dev *hdev)
 
 	pcim_iounmap(pdev, hdev->hw.hw.io_base);
 	pci_free_irq_vectors(pdev);
-	pci_clear_master(pdev);
 	pci_release_mem_regions(pdev);
 	pci_disable_device(pdev);
 }
@@ -11524,6 +11522,124 @@ static void hclge_uninit_rxd_adv_layout(struct hclge_dev *hdev)
 		hclge_write_dev(&hdev->hw, HCLGE_RXD_ADV_LAYOUT_EN_REG, 0);
 }
 
+static struct hclge_wol_info *hclge_get_wol_info(struct hnae3_handle *handle)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+
+	return &vport->back->hw.mac.wol;
+}
+
+static int hclge_get_wol_supported_mode(struct hclge_dev *hdev,
+					u32 *wol_supported)
+{
+	struct hclge_query_wol_supported_cmd *wol_supported_cmd;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_WOL_GET_SUPPORTED_MODE,
+				   true);
+	wol_supported_cmd = (struct hclge_query_wol_supported_cmd *)desc.data;
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed to query wol supported, ret = %d\n", ret);
+		return ret;
+	}
+
+	*wol_supported = le32_to_cpu(wol_supported_cmd->supported_wake_mode);
+
+	return 0;
+}
+
+static int hclge_set_wol_cfg(struct hclge_dev *hdev,
+			     struct hclge_wol_info *wol_info)
+{
+	struct hclge_wol_cfg_cmd *wol_cfg_cmd;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_WOL_CFG, false);
+	wol_cfg_cmd = (struct hclge_wol_cfg_cmd *)desc.data;
+	wol_cfg_cmd->wake_on_lan_mode = cpu_to_le32(wol_info->wol_current_mode);
+	wol_cfg_cmd->sopass_size = wol_info->wol_sopass_size;
+	memcpy(wol_cfg_cmd->sopass, wol_info->wol_sopass, SOPASS_MAX);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret)
+		dev_err(&hdev->pdev->dev,
+			"failed to set wol config, ret = %d\n", ret);
+
+	return ret;
+}
+
+static int hclge_update_wol(struct hclge_dev *hdev)
+{
+	struct hclge_wol_info *wol_info = &hdev->hw.mac.wol;
+
+	if (!hnae3_ae_dev_wol_supported(hdev->ae_dev))
+		return 0;
+
+	return hclge_set_wol_cfg(hdev, wol_info);
+}
+
+static int hclge_init_wol(struct hclge_dev *hdev)
+{
+	struct hclge_wol_info *wol_info = &hdev->hw.mac.wol;
+	int ret;
+
+	if (!hnae3_ae_dev_wol_supported(hdev->ae_dev))
+		return 0;
+
+	memset(wol_info, 0, sizeof(struct hclge_wol_info));
+	ret = hclge_get_wol_supported_mode(hdev,
+					   &wol_info->wol_support_mode);
+	if (ret) {
+		wol_info->wol_support_mode = 0;
+		return ret;
+	}
+
+	return hclge_update_wol(hdev);
+}
+
+static void hclge_get_wol(struct hnae3_handle *handle,
+			  struct ethtool_wolinfo *wol)
+{
+	struct hclge_wol_info *wol_info = hclge_get_wol_info(handle);
+
+	wol->supported = wol_info->wol_support_mode;
+	wol->wolopts = wol_info->wol_current_mode;
+	if (wol_info->wol_current_mode & WAKE_MAGICSECURE)
+		memcpy(wol->sopass, wol_info->wol_sopass, SOPASS_MAX);
+}
+
+static int hclge_set_wol(struct hnae3_handle *handle,
+			 struct ethtool_wolinfo *wol)
+{
+	struct hclge_wol_info *wol_info = hclge_get_wol_info(handle);
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	u32 wol_mode;
+	int ret;
+
+	wol_mode = wol->wolopts;
+	if (wol_mode & ~wol_info->wol_support_mode)
+		return -EINVAL;
+
+	wol_info->wol_current_mode = wol_mode;
+	if (wol_mode & WAKE_MAGICSECURE) {
+		memcpy(wol_info->wol_sopass, wol->sopass, SOPASS_MAX);
+		wol_info->wol_sopass_size = SOPASS_MAX;
+	} else {
+		wol_info->wol_sopass_size = 0;
+	}
+
+	ret = hclge_set_wol_cfg(vport->back, wol_info);
+	if (ret)
+		wol_info->wol_current_mode = 0;
+
+	return ret;
+}
+
 static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
 {
 	struct pci_dev *pdev = ae_dev->pdev;
@@ -11720,6 +11836,11 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
 	/* Enable MISC vector(vector0) */
 	hclge_enable_vector(&hdev->misc_vector, true);
 
+	ret = hclge_init_wol(hdev);
+	if (ret)
+		dev_warn(&pdev->dev,
+			 "failed to wake on lan init, ret = %d\n", ret);
+
 	hclge_state_init(hdev);
 	hdev->last_reset_time = jiffies;
 
@@ -11743,7 +11864,6 @@ err_devlink_uninit:
 	hclge_devlink_uninit(hdev);
 err_pci_uninit:
 	pcim_iounmap(pdev, hdev->hw.hw.io_base);
-	pci_clear_master(pdev);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
 out:
@@ -12099,6 +12219,11 @@ static int hclge_reset_ae_dev(struct hnae3_ae_dev *ae_dev)
 
 	hclge_init_rxd_adv_layout(hdev);
 
+	ret = hclge_update_wol(hdev);
+	if (ret)
+		dev_warn(&pdev->dev,
+			 "failed to update wol config, ret = %d\n", ret);
+
 	dev_info(&pdev->dev, "Reset done, %s driver initialization finished.\n",
 		 HCLGE_DRIVER_NAME);
 
@@ -13145,6 +13270,8 @@ static const struct hnae3_ae_ops hclge_ops = {
 	.get_link_diagnosis_info = hclge_get_link_diagnosis_info,
 	.clean_vf_config = hclge_clean_vport_config,
 	.get_dscp_prio = hclge_get_dscp_prio,
+	.get_wol = hclge_get_wol,
+	.set_wol = hclge_set_wol,
 };
 
 static struct hnae3_ae_algo ae_algo = {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
index 13f23d606e77..81aa6b0facf5 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
@@ -249,6 +249,13 @@ enum HCLGE_MAC_DUPLEX {
 #define QUERY_SFP_SPEED		0
 #define QUERY_ACTIVE_SPEED	1
 
+struct hclge_wol_info {
+	u32 wol_support_mode; /* store the wake on lan info */
+	u32 wol_current_mode;
+	u8 wol_sopass[SOPASS_MAX];
+	u8 wol_sopass_size;
+};
+
 struct hclge_mac {
 	u8 mac_id;
 	u8 phy_addr;
@@ -268,6 +275,7 @@ struct hclge_mac {
 	u32 user_fec_mode;
 	u32 fec_ability;
 	int link;	/* store the link status of mac & phy (if phy exists) */
+	struct hclge_wol_info wol;
 	struct phy_device *phydev;
 	struct mii_bus *mdio_bus;
 	phy_interface_t phy_if;
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
index e84e5be8e59e..f24046250341 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
@@ -2598,7 +2598,7 @@ static int hclgevf_pci_init(struct hclgevf_dev *hdev)
 	if (!hw->hw.io_base) {
 		dev_err(&pdev->dev, "can't map configuration register space\n");
 		ret = -ENOMEM;
-		goto err_clr_master;
+		goto err_release_regions;
 	}
 
 	ret = hclgevf_dev_mem_map(hdev);
@@ -2609,8 +2609,7 @@ static int hclgevf_pci_init(struct hclgevf_dev *hdev)
 
 err_unmap_io_base:
 	pci_iounmap(pdev, hdev->hw.hw.io_base);
-err_clr_master:
-	pci_clear_master(pdev);
+err_release_regions:
 	pci_release_regions(pdev);
 err_disable_device:
 	pci_disable_device(pdev);
@@ -2626,7 +2625,6 @@ static void hclgevf_pci_uninit(struct hclgevf_dev *hdev)
 		devm_iounmap(&pdev->dev, hdev->hw.hw.mem_base);
 
 	pci_iounmap(pdev, hdev->hw.hw.io_base);
-	pci_clear_master(pdev);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
 }
diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig
index c18c3b373846..9bc0a9519899 100644
--- a/drivers/net/ethernet/intel/Kconfig
+++ b/drivers/net/ethernet/intel/Kconfig
@@ -139,23 +139,6 @@ config IGBVF
 	  To compile this driver as a module, choose M here. The module
 	  will be called igbvf.
 
-config IXGB
-	tristate "Intel(R) PRO/10GbE support"
-	depends on PCI
-	help
-	  This driver supports Intel(R) PRO/10GbE family of adapters for
-	  PCI-X type cards. For PCI-E type cards, use the "ixgbe" driver
-	  instead. For more information on how to identify your adapter, go
-	  to the Adapter & Driver ID Guide that can be located at:
-
-	  <http://support.intel.com>
-
-	  More specific information on configuring the driver is in
-	  <file:Documentation/networking/device_drivers/ethernet/intel/ixgb.rst>.
-
-	  To compile this driver as a module, choose M here. The module
-	  will be called ixgb.
-
 config IXGBE
 	tristate "Intel(R) 10GbE PCI Express adapters support"
 	depends on PCI
diff --git a/drivers/net/ethernet/intel/Makefile b/drivers/net/ethernet/intel/Makefile
index 3075290063f6..d80d04132073 100644
--- a/drivers/net/ethernet/intel/Makefile
+++ b/drivers/net/ethernet/intel/Makefile
@@ -12,7 +12,6 @@ obj-$(CONFIG_IGBVF) += igbvf/
 obj-$(CONFIG_IXGBE) += ixgbe/
 obj-$(CONFIG_IXGBEVF) += ixgbevf/
 obj-$(CONFIG_I40E) += i40e/
-obj-$(CONFIG_IXGB) += ixgb/
 obj-$(CONFIG_IAVF) += iavf/
 obj-$(CONFIG_FM10K) += fm10k/
 obj-$(CONFIG_ICE) += ice/
diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c
index e1eb1de88bf9..bd7ef59b1f2e 100644
--- a/drivers/net/ethernet/intel/e1000e/netdev.c
+++ b/drivers/net/ethernet/intel/e1000e/netdev.c
@@ -23,7 +23,6 @@
 #include <linux/smp.h>
 #include <linux/pm_qos.h>
 #include <linux/pm_runtime.h>
-#include <linux/aer.h>
 #include <linux/prefetch.h>
 #include <linux/suspend.h>
 
@@ -5288,31 +5287,6 @@ static void e1000_watchdog_task(struct work_struct *work)
 				ew32(TARC(0), tarc0);
 			}
 
-			/* disable TSO for pcie and 10/100 speeds, to avoid
-			 * some hardware issues
-			 */
-			if (!(adapter->flags & FLAG_TSO_FORCE)) {
-				switch (adapter->link_speed) {
-				case SPEED_10:
-				case SPEED_100:
-					e_info("10/100 speed: disabling TSO\n");
-					netdev->features &= ~NETIF_F_TSO;
-					netdev->features &= ~NETIF_F_TSO6;
-					break;
-				case SPEED_1000:
-					netdev->features |= NETIF_F_TSO;
-					netdev->features |= NETIF_F_TSO6;
-					break;
-				default:
-					/* oops */
-					break;
-				}
-				if (hw->mac.type == e1000_pch_spt) {
-					netdev->features &= ~NETIF_F_TSO;
-					netdev->features &= ~NETIF_F_TSO6;
-				}
-			}
-
 			/* enable transmits in the hardware, need to do this
 			 * after setting TARC(0)
 			 */
@@ -7526,6 +7500,32 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 			    NETIF_F_RXCSUM |
 			    NETIF_F_HW_CSUM);
 
+	/* disable TSO for pcie and 10/100 speeds to avoid
+	 * some hardware issues and for i219 to fix transfer
+	 * speed being capped at 60%
+	 */
+	if (!(adapter->flags & FLAG_TSO_FORCE)) {
+		switch (adapter->link_speed) {
+		case SPEED_10:
+		case SPEED_100:
+			e_info("10/100 speed: disabling TSO\n");
+			netdev->features &= ~NETIF_F_TSO;
+			netdev->features &= ~NETIF_F_TSO6;
+			break;
+		case SPEED_1000:
+			netdev->features |= NETIF_F_TSO;
+			netdev->features |= NETIF_F_TSO6;
+			break;
+		default:
+			/* oops */
+			break;
+		}
+		if (hw->mac.type == e1000_pch_spt) {
+			netdev->features &= ~NETIF_F_TSO;
+			netdev->features &= ~NETIF_F_TSO6;
+		}
+	}
+
 	/* Set user-changeable features (subset of all device features) */
 	netdev->hw_features = netdev->features;
 	netdev->hw_features |= NETIF_F_RXFCS;
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
index 027d721feb18..d748b98274e7 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
@@ -3,7 +3,6 @@
 
 #include <linux/module.h>
 #include <linux/interrupt.h>
-#include <linux/aer.h>
 
 #include "fm10k.h"
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index 60ce4d15d82a..6e310a539467 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -10,7 +10,6 @@
 #include <linux/errno.h>
 #include <linux/module.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/netdevice.h>
 #include <linux/ioport.h>
 #include <linux/iommu.h>
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index 4934ff58332c..afc4fa8c66af 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -5402,6 +5402,13 @@ flags_complete:
 		return -EOPNOTSUPP;
 	}
 
+	if ((changed_flags & I40E_FLAG_LEGACY_RX) &&
+	    I40E_2K_TOO_SMALL_WITH_PADDING) {
+		dev_warn(&pf->pdev->dev,
+			 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");
+		return -EOPNOTSUPP;
+	}
+
 	if ((changed_flags & new_flags &
 	     I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED) &&
 	    (new_flags & I40E_FLAG_MFP_ENABLED))
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 228cd502bb48..b847bd105b16 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -2896,15 +2896,35 @@ static void i40e_sync_filters_subtask(struct i40e_pf *pf)
 }
 
 /**
- * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
+ * i40e_calculate_vsi_rx_buf_len - Calculates buffer length
+ *
+ * @vsi: VSI to calculate rx_buf_len from
+ */
+static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)
+{
+	if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
+		return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048);
+
+	return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;
+}
+
+/**
+ * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI
  * @vsi: the vsi
+ * @xdp_prog: XDP program
  **/
-static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
+static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi,
+				   struct bpf_prog *xdp_prog)
 {
-	if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
-		return I40E_RXBUFFER_2048;
+	u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
+	u16 chain_len;
+
+	if (xdp_prog && !xdp_prog->aux->xdp_has_frags)
+		chain_len = 1;
 	else
-		return I40E_RXBUFFER_3072;
+		chain_len = I40E_MAX_CHAINED_RX_BUFFERS;
+
+	return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER);
 }
 
 /**
@@ -2919,12 +2939,13 @@ static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
 	struct i40e_netdev_priv *np = netdev_priv(netdev);
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
+	int frame_size;
 
-	if (i40e_enabled_xdp_vsi(vsi)) {
-		int frame_size = new_mtu + I40E_PACKET_HDR_PAD;
-
-		if (frame_size > i40e_max_xdp_frame_size(vsi))
-			return -EINVAL;
+	frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
+	if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) {
+		netdev_err(netdev, "Error changing mtu to %d, Max is %d\n",
+			   new_mtu, frame_size - I40E_PACKET_HDR_PAD);
+		return -EINVAL;
 	}
 
 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
@@ -3595,6 +3616,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
 		}
 	}
 
+	xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq);
+
 	rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
 				    BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
 
@@ -3640,10 +3663,16 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
 	}
 
 	/* configure Rx buffer alignment */
-	if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
+	if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
+		if (I40E_2K_TOO_SMALL_WITH_PADDING) {
+			dev_info(&vsi->back->pdev->dev,
+				 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");
+			return -EOPNOTSUPP;
+		}
 		clear_ring_build_skb_enabled(ring);
-	else
+	} else {
 		set_ring_build_skb_enabled(ring);
+	}
 
 	ring->rx_offset = i40e_rx_offset(ring);
 
@@ -3694,24 +3723,6 @@ static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
 }
 
 /**
- * i40e_calculate_vsi_rx_buf_len - Calculates buffer length
- *
- * @vsi: VSI to calculate rx_buf_len from
- */
-static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)
-{
-	if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
-		return I40E_RXBUFFER_2048;
-
-#if (PAGE_SIZE < 8192)
-	if (!I40E_2K_TOO_SMALL_WITH_PADDING && vsi->netdev->mtu <= ETH_DATA_LEN)
-		return I40E_RXBUFFER_1536 - NET_IP_ALIGN;
-#endif
-
-	return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;
-}
-
-/**
  * i40e_vsi_configure_rx - Configure the VSI for Rx
  * @vsi: the VSI being configured
  *
@@ -3722,13 +3733,15 @@ static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
 	int err = 0;
 	u16 i;
 
-	vsi->max_frame = I40E_MAX_RXBUFFER;
+	vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
 	vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
 
 #if (PAGE_SIZE < 8192)
 	if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING &&
-	    vsi->netdev->mtu <= ETH_DATA_LEN)
-		vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
+	    vsi->netdev->mtu <= ETH_DATA_LEN) {
+		vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
+		vsi->max_frame = vsi->rx_buf_len;
+	}
 #endif
 
 	/* set up individual rings */
@@ -11059,8 +11072,11 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
 					     pf->hw.aq.asq_last_status));
 	}
 	/* reinit the misc interrupt */
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
 		ret = i40e_setup_misc_vector(pf);
+		if (ret)
+			goto end_unlock;
+	}
 
 	/* Add a filter to drop all Flow control frames from any VSI from being
 	 * transmitted. By doing so we stop a malicious VF from sending out
@@ -13316,15 +13332,15 @@ out_err:
 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
 			  struct netlink_ext_ack *extack)
 {
-	int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
+	int frame_size = i40e_max_vsi_frame_size(vsi, prog);
 	struct i40e_pf *pf = vsi->back;
 	struct bpf_prog *old_prog;
 	bool need_reset;
 	int i;
 
 	/* Don't allow frames that span over multiple buffers */
-	if (frame_size > i40e_calculate_vsi_rx_buf_len(vsi)) {
-		NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
+	if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) {
+		NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags");
 		return -EINVAL;
 	}
 
@@ -13810,7 +13826,8 @@ static int i40e_config_netdev(struct i40e_vsi *vsi)
 
 		netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
 				       NETDEV_XDP_ACT_REDIRECT |
-				       NETDEV_XDP_ACT_XSK_ZEROCOPY;
+				       NETDEV_XDP_ACT_XSK_ZEROCOPY |
+				       NETDEV_XDP_ACT_RX_SG;
 	} else {
 		/* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
 		 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
@@ -14133,15 +14150,15 @@ static int i40e_add_vsi(struct i40e_vsi *vsi)
 		vsi->id = ctxt.vsi_number;
 	}
 
-	vsi->active_filters = 0;
-	clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
 	spin_lock_bh(&vsi->mac_filter_hash_lock);
+	vsi->active_filters = 0;
 	/* If macvlan filters already exist, force them to get loaded */
 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
 		f->state = I40E_FILTER_NEW;
 		f_count++;
 	}
 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
+	clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
 
 	if (f_count) {
 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_trace.h b/drivers/net/ethernet/intel/i40e/i40e_trace.h
index 79d587ad5409..33b4e30f5e00 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_trace.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_trace.h
@@ -162,45 +162,45 @@ DECLARE_EVENT_CLASS(
 
 	TP_PROTO(struct i40e_ring *ring,
 		 union i40e_16byte_rx_desc *desc,
-		 struct sk_buff *skb),
+		 struct xdp_buff *xdp),
 
-	TP_ARGS(ring, desc, skb),
+	TP_ARGS(ring, desc, xdp),
 
 	TP_STRUCT__entry(
 		__field(void*, ring)
 		__field(void*, desc)
-		__field(void*, skb)
+		__field(void*, xdp)
 		__string(devname, ring->netdev->name)
 	),
 
 	TP_fast_assign(
 		__entry->ring = ring;
 		__entry->desc = desc;
-		__entry->skb = skb;
+		__entry->xdp = xdp;
 		__assign_str(devname, ring->netdev->name);
 	),
 
 	TP_printk(
-		"netdev: %s ring: %p desc: %p skb %p",
+		"netdev: %s ring: %p desc: %p xdp %p",
 		__get_str(devname), __entry->ring,
-		__entry->desc, __entry->skb)
+		__entry->desc, __entry->xdp)
 );
 
 DEFINE_EVENT(
 	i40e_rx_template, i40e_clean_rx_irq,
 	TP_PROTO(struct i40e_ring *ring,
 		 union i40e_16byte_rx_desc *desc,
-		 struct sk_buff *skb),
+		 struct xdp_buff *xdp),
 
-	TP_ARGS(ring, desc, skb));
+	TP_ARGS(ring, desc, xdp));
 
 DEFINE_EVENT(
 	i40e_rx_template, i40e_clean_rx_irq_rx,
 	TP_PROTO(struct i40e_ring *ring,
 		 union i40e_16byte_rx_desc *desc,
-		 struct sk_buff *skb),
+		 struct xdp_buff *xdp),
 
-	TP_ARGS(ring, desc, skb));
+	TP_ARGS(ring, desc, xdp));
 
 DECLARE_EVENT_CLASS(
 	i40e_xmit_template,
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index 72b091f2509d..8b8bf4880faa 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -1477,9 +1477,6 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
 	if (!rx_ring->rx_bi)
 		return;
 
-	dev_kfree_skb(rx_ring->skb);
-	rx_ring->skb = NULL;
-
 	if (rx_ring->xsk_pool) {
 		i40e_xsk_clean_rx_ring(rx_ring);
 		goto skip_free;
@@ -1524,6 +1521,7 @@ skip_free:
 
 	rx_ring->next_to_alloc = 0;
 	rx_ring->next_to_clean = 0;
+	rx_ring->next_to_process = 0;
 	rx_ring->next_to_use = 0;
 }
 
@@ -1576,6 +1574,7 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
 
 	rx_ring->next_to_alloc = 0;
 	rx_ring->next_to_clean = 0;
+	rx_ring->next_to_process = 0;
 	rx_ring->next_to_use = 0;
 
 	/* XDP RX-queue info only needed for RX rings exposed to XDP */
@@ -1617,21 +1616,19 @@ void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
 	writel(val, rx_ring->tail);
 }
 
+#if (PAGE_SIZE >= 8192)
 static unsigned int i40e_rx_frame_truesize(struct i40e_ring *rx_ring,
 					   unsigned int size)
 {
 	unsigned int truesize;
 
-#if (PAGE_SIZE < 8192)
-	truesize = i40e_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */
-#else
 	truesize = rx_ring->rx_offset ?
 		SKB_DATA_ALIGN(size + rx_ring->rx_offset) +
 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
 		SKB_DATA_ALIGN(size);
-#endif
 	return truesize;
 }
+#endif
 
 /**
  * i40e_alloc_mapped_page - recycle or make a new page
@@ -1970,7 +1967,6 @@ static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb,
  * i40e_can_reuse_rx_page - Determine if page can be reused for another Rx
  * @rx_buffer: buffer containing the page
  * @rx_stats: rx stats structure for the rx ring
- * @rx_buffer_pgcnt: buffer page refcount pre xdp_do_redirect() call
  *
  * If page is reusable, we have a green light for calling i40e_reuse_rx_page,
  * which will assign the current buffer to the buffer that next_to_alloc is
@@ -1981,8 +1977,7 @@ static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb,
  * or busy if it could not be reused.
  */
 static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer,
-				   struct i40e_rx_queue_stats *rx_stats,
-				   int rx_buffer_pgcnt)
+				   struct i40e_rx_queue_stats *rx_stats)
 {
 	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
 	struct page *page = rx_buffer->page;
@@ -1995,7 +1990,7 @@ static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer,
 
 #if (PAGE_SIZE < 8192)
 	/* if we are only owner of page we can reuse it */
-	if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1)) {
+	if (unlikely((rx_buffer->page_count - pagecnt_bias) > 1)) {
 		rx_stats->page_busy_count++;
 		return false;
 	}
@@ -2021,33 +2016,14 @@ static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer,
 }
 
 /**
- * i40e_add_rx_frag - Add contents of Rx buffer to sk_buff
- * @rx_ring: rx descriptor ring to transact packets on
- * @rx_buffer: buffer containing page to add
- * @skb: sk_buff to place the data into
- * @size: packet length from rx_desc
- *
- * This function will add the data contained in rx_buffer->page to the skb.
- * It will just attach the page as a frag to the skb.
- *
- * The function will then update the page offset.
+ * i40e_rx_buffer_flip - adjusted rx_buffer to point to an unused region
+ * @rx_buffer: Rx buffer to adjust
+ * @truesize: Size of adjustment
  **/
-static void i40e_add_rx_frag(struct i40e_ring *rx_ring,
-			     struct i40e_rx_buffer *rx_buffer,
-			     struct sk_buff *skb,
-			     unsigned int size)
+static void i40e_rx_buffer_flip(struct i40e_rx_buffer *rx_buffer,
+				unsigned int truesize)
 {
 #if (PAGE_SIZE < 8192)
-	unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
-#else
-	unsigned int truesize = SKB_DATA_ALIGN(size + rx_ring->rx_offset);
-#endif
-
-	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
-			rx_buffer->page_offset, size, truesize);
-
-	/* page is being used so we must update the page offset */
-#if (PAGE_SIZE < 8192)
 	rx_buffer->page_offset ^= truesize;
 #else
 	rx_buffer->page_offset += truesize;
@@ -2058,19 +2034,17 @@ static void i40e_add_rx_frag(struct i40e_ring *rx_ring,
  * i40e_get_rx_buffer - Fetch Rx buffer and synchronize data for use
  * @rx_ring: rx descriptor ring to transact packets on
  * @size: size of buffer to add to skb
- * @rx_buffer_pgcnt: buffer page refcount
  *
  * This function will pull an Rx buffer from the ring and synchronize it
  * for use by the CPU.
  */
 static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
-						 const unsigned int size,
-						 int *rx_buffer_pgcnt)
+						 const unsigned int size)
 {
 	struct i40e_rx_buffer *rx_buffer;
 
-	rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
-	*rx_buffer_pgcnt =
+	rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_process);
+	rx_buffer->page_count =
 #if (PAGE_SIZE < 8192)
 		page_count(rx_buffer->page);
 #else
@@ -2092,25 +2066,82 @@ static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
 }
 
 /**
- * i40e_construct_skb - Allocate skb and populate it
+ * i40e_put_rx_buffer - Clean up used buffer and either recycle or free
  * @rx_ring: rx descriptor ring to transact packets on
  * @rx_buffer: rx buffer to pull data from
+ *
+ * This function will clean up the contents of the rx_buffer.  It will
+ * either recycle the buffer or unmap it and free the associated resources.
+ */
+static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
+			       struct i40e_rx_buffer *rx_buffer)
+{
+	if (i40e_can_reuse_rx_page(rx_buffer, &rx_ring->rx_stats)) {
+		/* hand second half of page back to the ring */
+		i40e_reuse_rx_page(rx_ring, rx_buffer);
+	} else {
+		/* we are not reusing the buffer so unmap it */
+		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
+				     i40e_rx_pg_size(rx_ring),
+				     DMA_FROM_DEVICE, I40E_RX_DMA_ATTR);
+		__page_frag_cache_drain(rx_buffer->page,
+					rx_buffer->pagecnt_bias);
+		/* clear contents of buffer_info */
+		rx_buffer->page = NULL;
+	}
+}
+
+/**
+ * i40e_process_rx_buffs- Processing of buffers post XDP prog or on error
+ * @rx_ring: Rx descriptor ring to transact packets on
+ * @xdp_res: Result of the XDP program
+ * @xdp: xdp_buff pointing to the data
+ **/
+static void i40e_process_rx_buffs(struct i40e_ring *rx_ring, int xdp_res,
+				  struct xdp_buff *xdp)
+{
+	u32 next = rx_ring->next_to_clean;
+	struct i40e_rx_buffer *rx_buffer;
+
+	xdp->flags = 0;
+
+	while (1) {
+		rx_buffer = i40e_rx_bi(rx_ring, next);
+		if (++next == rx_ring->count)
+			next = 0;
+
+		if (!rx_buffer->page)
+			continue;
+
+		if (xdp_res == I40E_XDP_CONSUMED)
+			rx_buffer->pagecnt_bias++;
+		else
+			i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);
+
+		/* EOP buffer will be put in i40e_clean_rx_irq() */
+		if (next == rx_ring->next_to_process)
+			return;
+
+		i40e_put_rx_buffer(rx_ring, rx_buffer);
+	}
+}
+
+/**
+ * i40e_construct_skb - Allocate skb and populate it
+ * @rx_ring: rx descriptor ring to transact packets on
  * @xdp: xdp_buff pointing to the data
+ * @nr_frags: number of buffers for the packet
  *
  * This function allocates an skb.  It then populates it with the page
  * data from the current receive descriptor, taking care to set up the
  * skb correctly.
  */
 static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
-					  struct i40e_rx_buffer *rx_buffer,
-					  struct xdp_buff *xdp)
+					  struct xdp_buff *xdp,
+					  u32 nr_frags)
 {
 	unsigned int size = xdp->data_end - xdp->data;
-#if (PAGE_SIZE < 8192)
-	unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
-#else
-	unsigned int truesize = SKB_DATA_ALIGN(size);
-#endif
+	struct i40e_rx_buffer *rx_buffer;
 	unsigned int headlen;
 	struct sk_buff *skb;
 
@@ -2150,48 +2181,60 @@ static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
 	memcpy(__skb_put(skb, headlen), xdp->data,
 	       ALIGN(headlen, sizeof(long)));
 
+	rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
 	/* update all of the pointers */
 	size -= headlen;
 	if (size) {
+		if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
+			dev_kfree_skb(skb);
+			return NULL;
+		}
 		skb_add_rx_frag(skb, 0, rx_buffer->page,
 				rx_buffer->page_offset + headlen,
-				size, truesize);
-
+				size, xdp->frame_sz);
 		/* buffer is used by skb, update page_offset */
-#if (PAGE_SIZE < 8192)
-		rx_buffer->page_offset ^= truesize;
-#else
-		rx_buffer->page_offset += truesize;
-#endif
+		i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);
 	} else {
 		/* buffer is unused, reset bias back to rx_buffer */
 		rx_buffer->pagecnt_bias++;
 	}
 
+	if (unlikely(xdp_buff_has_frags(xdp))) {
+		struct skb_shared_info *sinfo, *skinfo = skb_shinfo(skb);
+
+		sinfo = xdp_get_shared_info_from_buff(xdp);
+		memcpy(&skinfo->frags[skinfo->nr_frags], &sinfo->frags[0],
+		       sizeof(skb_frag_t) * nr_frags);
+
+		xdp_update_skb_shared_info(skb, skinfo->nr_frags + nr_frags,
+					   sinfo->xdp_frags_size,
+					   nr_frags * xdp->frame_sz,
+					   xdp_buff_is_frag_pfmemalloc(xdp));
+
+		/* First buffer has already been processed, so bump ntc */
+		if (++rx_ring->next_to_clean == rx_ring->count)
+			rx_ring->next_to_clean = 0;
+
+		i40e_process_rx_buffs(rx_ring, I40E_XDP_PASS, xdp);
+	}
+
 	return skb;
 }
 
 /**
  * i40e_build_skb - Build skb around an existing buffer
  * @rx_ring: Rx descriptor ring to transact packets on
- * @rx_buffer: Rx buffer to pull data from
  * @xdp: xdp_buff pointing to the data
+ * @nr_frags: number of buffers for the packet
  *
  * This function builds an skb around an existing Rx buffer, taking care
  * to set up the skb correctly and avoid any memcpy overhead.
  */
 static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
-				      struct i40e_rx_buffer *rx_buffer,
-				      struct xdp_buff *xdp)
+				      struct xdp_buff *xdp,
+				      u32 nr_frags)
 {
 	unsigned int metasize = xdp->data - xdp->data_meta;
-#if (PAGE_SIZE < 8192)
-	unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
-#else
-	unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
-				SKB_DATA_ALIGN(xdp->data_end -
-					       xdp->data_hard_start);
-#endif
 	struct sk_buff *skb;
 
 	/* Prefetch first cache line of first page. If xdp->data_meta
@@ -2202,7 +2245,7 @@ static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
 	net_prefetch(xdp->data_meta);
 
 	/* build an skb around the page buffer */
-	skb = napi_build_skb(xdp->data_hard_start, truesize);
+	skb = napi_build_skb(xdp->data_hard_start, xdp->frame_sz);
 	if (unlikely(!skb))
 		return NULL;
 
@@ -2212,42 +2255,25 @@ static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
 	if (metasize)
 		skb_metadata_set(skb, metasize);
 
-	/* buffer is used by skb, update page_offset */
-#if (PAGE_SIZE < 8192)
-	rx_buffer->page_offset ^= truesize;
-#else
-	rx_buffer->page_offset += truesize;
-#endif
+	if (unlikely(xdp_buff_has_frags(xdp))) {
+		struct skb_shared_info *sinfo;
 
-	return skb;
-}
+		sinfo = xdp_get_shared_info_from_buff(xdp);
+		xdp_update_skb_shared_info(skb, nr_frags,
+					   sinfo->xdp_frags_size,
+					   nr_frags * xdp->frame_sz,
+					   xdp_buff_is_frag_pfmemalloc(xdp));
 
-/**
- * i40e_put_rx_buffer - Clean up used buffer and either recycle or free
- * @rx_ring: rx descriptor ring to transact packets on
- * @rx_buffer: rx buffer to pull data from
- * @rx_buffer_pgcnt: rx buffer page refcount pre xdp_do_redirect() call
- *
- * This function will clean up the contents of the rx_buffer.  It will
- * either recycle the buffer or unmap it and free the associated resources.
- */
-static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
-			       struct i40e_rx_buffer *rx_buffer,
-			       int rx_buffer_pgcnt)
-{
-	if (i40e_can_reuse_rx_page(rx_buffer, &rx_ring->rx_stats, rx_buffer_pgcnt)) {
-		/* hand second half of page back to the ring */
-		i40e_reuse_rx_page(rx_ring, rx_buffer);
+		i40e_process_rx_buffs(rx_ring, I40E_XDP_PASS, xdp);
 	} else {
-		/* we are not reusing the buffer so unmap it */
-		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
-				     i40e_rx_pg_size(rx_ring),
-				     DMA_FROM_DEVICE, I40E_RX_DMA_ATTR);
-		__page_frag_cache_drain(rx_buffer->page,
-					rx_buffer->pagecnt_bias);
-		/* clear contents of buffer_info */
-		rx_buffer->page = NULL;
+		struct i40e_rx_buffer *rx_buffer;
+
+		rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
+		/* buffer is used by skb, update page_offset */
+		i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);
 	}
+
+	return skb;
 }
 
 /**
@@ -2333,25 +2359,6 @@ xdp_out:
 }
 
 /**
- * i40e_rx_buffer_flip - adjusted rx_buffer to point to an unused region
- * @rx_ring: Rx ring
- * @rx_buffer: Rx buffer to adjust
- * @size: Size of adjustment
- **/
-static void i40e_rx_buffer_flip(struct i40e_ring *rx_ring,
-				struct i40e_rx_buffer *rx_buffer,
-				unsigned int size)
-{
-	unsigned int truesize = i40e_rx_frame_truesize(rx_ring, size);
-
-#if (PAGE_SIZE < 8192)
-	rx_buffer->page_offset ^= truesize;
-#else
-	rx_buffer->page_offset += truesize;
-#endif
-}
-
-/**
  * i40e_xdp_ring_update_tail - Updates the XDP Tx ring tail register
  * @xdp_ring: XDP Tx ring
  *
@@ -2409,16 +2416,65 @@ void i40e_finalize_xdp_rx(struct i40e_ring *rx_ring, unsigned int xdp_res)
 }
 
 /**
- * i40e_inc_ntc: Advance the next_to_clean index
+ * i40e_inc_ntp: Advance the next_to_process index
  * @rx_ring: Rx ring
  **/
-static void i40e_inc_ntc(struct i40e_ring *rx_ring)
+static void i40e_inc_ntp(struct i40e_ring *rx_ring)
+{
+	u32 ntp = rx_ring->next_to_process + 1;
+
+	ntp = (ntp < rx_ring->count) ? ntp : 0;
+	rx_ring->next_to_process = ntp;
+	prefetch(I40E_RX_DESC(rx_ring, ntp));
+}
+
+/**
+ * i40e_add_xdp_frag: Add a frag to xdp_buff
+ * @xdp: xdp_buff pointing to the data
+ * @nr_frags: return number of buffers for the packet
+ * @rx_buffer: rx_buffer holding data of the current frag
+ * @size: size of data of current frag
+ */
+static int i40e_add_xdp_frag(struct xdp_buff *xdp, u32 *nr_frags,
+			     struct i40e_rx_buffer *rx_buffer, u32 size)
 {
-	u32 ntc = rx_ring->next_to_clean + 1;
+	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
+
+	if (!xdp_buff_has_frags(xdp)) {
+		sinfo->nr_frags = 0;
+		sinfo->xdp_frags_size = 0;
+		xdp_buff_set_frags_flag(xdp);
+	} else if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) {
+		/* Overflowing packet: All frags need to be dropped */
+		return -ENOMEM;
+	}
+
+	__skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, rx_buffer->page,
+				   rx_buffer->page_offset, size);
+
+	sinfo->xdp_frags_size += size;
 
-	ntc = (ntc < rx_ring->count) ? ntc : 0;
-	rx_ring->next_to_clean = ntc;
-	prefetch(I40E_RX_DESC(rx_ring, ntc));
+	if (page_is_pfmemalloc(rx_buffer->page))
+		xdp_buff_set_frag_pfmemalloc(xdp);
+	*nr_frags = sinfo->nr_frags;
+
+	return 0;
+}
+
+/**
+ * i40e_consume_xdp_buff - Consume all the buffers of the packet and update ntc
+ * @rx_ring: rx descriptor ring to transact packets on
+ * @xdp: xdp_buff pointing to the data
+ * @rx_buffer: rx_buffer of eop desc
+ */
+static void i40e_consume_xdp_buff(struct i40e_ring *rx_ring,
+				  struct xdp_buff *xdp,
+				  struct i40e_rx_buffer *rx_buffer)
+{
+	i40e_process_rx_buffs(rx_ring, I40E_XDP_CONSUMED, xdp);
+	i40e_put_rx_buffer(rx_ring, rx_buffer);
+	rx_ring->next_to_clean = rx_ring->next_to_process;
+	xdp->data = NULL;
 }
 
 /**
@@ -2437,38 +2493,36 @@ static void i40e_inc_ntc(struct i40e_ring *rx_ring)
 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget,
 			     unsigned int *rx_cleaned)
 {
-	unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0;
+	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
+	u16 clean_threshold = rx_ring->count / 2;
 	unsigned int offset = rx_ring->rx_offset;
-	struct sk_buff *skb = rx_ring->skb;
+	struct xdp_buff *xdp = &rx_ring->xdp;
 	unsigned int xdp_xmit = 0;
 	struct bpf_prog *xdp_prog;
 	bool failure = false;
-	struct xdp_buff xdp;
 	int xdp_res = 0;
 
-#if (PAGE_SIZE < 8192)
-	frame_sz = i40e_rx_frame_truesize(rx_ring, 0);
-#endif
-	xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);
-
 	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
 
 	while (likely(total_rx_packets < (unsigned int)budget)) {
+		u16 ntp = rx_ring->next_to_process;
 		struct i40e_rx_buffer *rx_buffer;
 		union i40e_rx_desc *rx_desc;
-		int rx_buffer_pgcnt;
+		struct sk_buff *skb;
 		unsigned int size;
+		u32 nfrags = 0;
+		bool neop;
 		u64 qword;
 
 		/* return some buffers to hardware, one at a time is too slow */
-		if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
+		if (cleaned_count >= clean_threshold) {
 			failure = failure ||
 				  i40e_alloc_rx_buffers(rx_ring, cleaned_count);
 			cleaned_count = 0;
 		}
 
-		rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean);
+		rx_desc = I40E_RX_DESC(rx_ring, ntp);
 
 		/* status_error_len will always be zero for unused descriptors
 		 * because it's cleared in cleanup, and overlaps with hdr_addr
@@ -2487,8 +2541,8 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget,
 			i40e_clean_programming_status(rx_ring,
 						      rx_desc->raw.qword[0],
 						      qword);
-			rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
-			i40e_inc_ntc(rx_ring);
+			rx_buffer = i40e_rx_bi(rx_ring, ntp);
+			i40e_inc_ntp(rx_ring);
 			i40e_reuse_rx_page(rx_ring, rx_buffer);
 			cleaned_count++;
 			continue;
@@ -2499,76 +2553,84 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget,
 		if (!size)
 			break;
 
-		i40e_trace(clean_rx_irq, rx_ring, rx_desc, skb);
-		rx_buffer = i40e_get_rx_buffer(rx_ring, size, &rx_buffer_pgcnt);
-
+		i40e_trace(clean_rx_irq, rx_ring, rx_desc, xdp);
 		/* retrieve a buffer from the ring */
-		if (!skb) {
+		rx_buffer = i40e_get_rx_buffer(rx_ring, size);
+
+		neop = i40e_is_non_eop(rx_ring, rx_desc);
+		i40e_inc_ntp(rx_ring);
+
+		if (!xdp->data) {
 			unsigned char *hard_start;
 
 			hard_start = page_address(rx_buffer->page) +
 				     rx_buffer->page_offset - offset;
-			xdp_prepare_buff(&xdp, hard_start, offset, size, true);
-			xdp_buff_clear_frags_flag(&xdp);
+			xdp_prepare_buff(xdp, hard_start, offset, size, true);
 #if (PAGE_SIZE > 4096)
 			/* At larger PAGE_SIZE, frame_sz depend on len size */
-			xdp.frame_sz = i40e_rx_frame_truesize(rx_ring, size);
+			xdp->frame_sz = i40e_rx_frame_truesize(rx_ring, size);
 #endif
-			xdp_res = i40e_run_xdp(rx_ring, &xdp, xdp_prog);
+		} else if (i40e_add_xdp_frag(xdp, &nfrags, rx_buffer, size) &&
+			   !neop) {
+			/* Overflowing packet: Drop all frags on EOP */
+			i40e_consume_xdp_buff(rx_ring, xdp, rx_buffer);
+			break;
 		}
 
+		if (neop)
+			continue;
+
+		xdp_res = i40e_run_xdp(rx_ring, xdp, xdp_prog);
+
 		if (xdp_res) {
-			if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR)) {
-				xdp_xmit |= xdp_res;
-				i40e_rx_buffer_flip(rx_ring, rx_buffer, size);
+			xdp_xmit |= xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR);
+
+			if (unlikely(xdp_buff_has_frags(xdp))) {
+				i40e_process_rx_buffs(rx_ring, xdp_res, xdp);
+				size = xdp_get_buff_len(xdp);
+			} else if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR)) {
+				i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);
 			} else {
 				rx_buffer->pagecnt_bias++;
 			}
 			total_rx_bytes += size;
-			total_rx_packets++;
-		} else if (skb) {
-			i40e_add_rx_frag(rx_ring, rx_buffer, skb, size);
-		} else if (ring_uses_build_skb(rx_ring)) {
-			skb = i40e_build_skb(rx_ring, rx_buffer, &xdp);
 		} else {
-			skb = i40e_construct_skb(rx_ring, rx_buffer, &xdp);
-		}
+			if (ring_uses_build_skb(rx_ring))
+				skb = i40e_build_skb(rx_ring, xdp, nfrags);
+			else
+				skb = i40e_construct_skb(rx_ring, xdp, nfrags);
+
+			/* drop if we failed to retrieve a buffer */
+			if (!skb) {
+				rx_ring->rx_stats.alloc_buff_failed++;
+				i40e_consume_xdp_buff(rx_ring, xdp, rx_buffer);
+				break;
+			}
 
-		/* exit if we failed to retrieve a buffer */
-		if (!xdp_res && !skb) {
-			rx_ring->rx_stats.alloc_buff_failed++;
-			rx_buffer->pagecnt_bias++;
-			break;
-		}
+			if (i40e_cleanup_headers(rx_ring, skb, rx_desc))
+				goto process_next;
 
-		i40e_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt);
-		cleaned_count++;
+			/* probably a little skewed due to removing CRC */
+			total_rx_bytes += skb->len;
 
-		i40e_inc_ntc(rx_ring);
-		if (i40e_is_non_eop(rx_ring, rx_desc))
-			continue;
+			/* populate checksum, VLAN, and protocol */
+			i40e_process_skb_fields(rx_ring, rx_desc, skb);
 
-		if (xdp_res || i40e_cleanup_headers(rx_ring, skb, rx_desc)) {
-			skb = NULL;
-			continue;
+			i40e_trace(clean_rx_irq_rx, rx_ring, rx_desc, xdp);
+			napi_gro_receive(&rx_ring->q_vector->napi, skb);
 		}
 
-		/* probably a little skewed due to removing CRC */
-		total_rx_bytes += skb->len;
-
-		/* populate checksum, VLAN, and protocol */
-		i40e_process_skb_fields(rx_ring, rx_desc, skb);
-
-		i40e_trace(clean_rx_irq_rx, rx_ring, rx_desc, skb);
-		napi_gro_receive(&rx_ring->q_vector->napi, skb);
-		skb = NULL;
-
 		/* update budget accounting */
 		total_rx_packets++;
+process_next:
+		cleaned_count += nfrags + 1;
+		i40e_put_rx_buffer(rx_ring, rx_buffer);
+		rx_ring->next_to_clean = rx_ring->next_to_process;
+
+		xdp->data = NULL;
 	}
 
 	i40e_finalize_xdp_rx(rx_ring, xdp_xmit);
-	rx_ring->skb = skb;
 
 	i40e_update_rx_stats(rx_ring, total_rx_bytes, total_rx_packets);
 
@@ -3001,7 +3063,7 @@ static inline int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
 			rc = skb_cow_head(skb, 0);
 			if (rc < 0)
 				return rc;
-			vhdr = (struct vlan_ethhdr *)skb->data;
+			vhdr = skb_vlan_eth_hdr(skb);
 			vhdr->h_vlan_TCI = htons(tx_flags >>
 						 I40E_TX_FLAGS_VLAN_SHIFT);
 		} else {
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
index 768290dc6f48..8c3d24012c54 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
@@ -277,6 +277,7 @@ struct i40e_rx_buffer {
 	struct page *page;
 	__u32 page_offset;
 	__u16 pagecnt_bias;
+	__u32 page_count;
 };
 
 struct i40e_queue_stats {
@@ -336,6 +337,17 @@ struct i40e_ring {
 	u8 dcb_tc;			/* Traffic class of ring */
 	u8 __iomem *tail;
 
+	/* Storing xdp_buff on ring helps in saving the state of partially built
+	 * packet when i40e_clean_rx_ring_irq() must return before it sees EOP
+	 * and to resume packet building for this ring in the next call to
+	 * i40e_clean_rx_ring_irq().
+	 */
+	struct xdp_buff xdp;
+
+	/* Next descriptor to be processed; next_to_clean is updated only on
+	 * processing EOP descriptor
+	 */
+	u16 next_to_process;
 	/* high bit set means dynamic, use accessor routines to read/write.
 	 * hardware only supports 2us resolution for the ITR registers.
 	 * these values always store the USER setting, and must be converted
@@ -380,14 +392,6 @@ struct i40e_ring {
 
 	struct rcu_head rcu;		/* to avoid race on free */
 	u16 next_to_alloc;
-	struct sk_buff *skb;		/* When i40e_clean_rx_ring_irq() must
-					 * return before it sees the EOP for
-					 * the current packet, we save that skb
-					 * here and resume receiving this
-					 * packet the next time
-					 * i40e_clean_rx_ring_irq() is called
-					 * for this ring.
-					 */
 
 	struct i40e_channel *ch;
 	u16 rx_offset;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
index 8a4587585acd..be59ba3774e1 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
@@ -2915,6 +2915,72 @@ static inline int i40e_check_vf_permission(struct i40e_vf *vf,
 }
 
 /**
+ * i40e_vc_ether_addr_type - get type of virtchnl_ether_addr
+ * @vc_ether_addr: used to extract the type
+ **/
+static u8
+i40e_vc_ether_addr_type(struct virtchnl_ether_addr *vc_ether_addr)
+{
+	return vc_ether_addr->type & VIRTCHNL_ETHER_ADDR_TYPE_MASK;
+}
+
+/**
+ * i40e_is_vc_addr_legacy
+ * @vc_ether_addr: VIRTCHNL structure that contains MAC and type
+ *
+ * check if the MAC address is from an older VF
+ **/
+static bool
+i40e_is_vc_addr_legacy(struct virtchnl_ether_addr *vc_ether_addr)
+{
+	return i40e_vc_ether_addr_type(vc_ether_addr) ==
+		VIRTCHNL_ETHER_ADDR_LEGACY;
+}
+
+/**
+ * i40e_is_vc_addr_primary
+ * @vc_ether_addr: VIRTCHNL structure that contains MAC and type
+ *
+ * check if the MAC address is the VF's primary MAC
+ * This function should only be called when the MAC address in
+ * virtchnl_ether_addr is a valid unicast MAC
+ **/
+static bool
+i40e_is_vc_addr_primary(struct virtchnl_ether_addr *vc_ether_addr)
+{
+	return i40e_vc_ether_addr_type(vc_ether_addr) ==
+		VIRTCHNL_ETHER_ADDR_PRIMARY;
+}
+
+/**
+ * i40e_update_vf_mac_addr
+ * @vf: VF to update
+ * @vc_ether_addr: structure from VIRTCHNL with MAC to add
+ *
+ * update the VF's cached hardware MAC if allowed
+ **/
+static void
+i40e_update_vf_mac_addr(struct i40e_vf *vf,
+			struct virtchnl_ether_addr *vc_ether_addr)
+{
+	u8 *mac_addr = vc_ether_addr->addr;
+
+	if (!is_valid_ether_addr(mac_addr))
+		return;
+
+	/* If request to add MAC filter is a primary request update its default
+	 * MAC address with the requested one. If it is a legacy request then
+	 * check if current default is empty if so update the default MAC
+	 */
+	if (i40e_is_vc_addr_primary(vc_ether_addr)) {
+		ether_addr_copy(vf->default_lan_addr.addr, mac_addr);
+	} else if (i40e_is_vc_addr_legacy(vc_ether_addr)) {
+		if (is_zero_ether_addr(vf->default_lan_addr.addr))
+			ether_addr_copy(vf->default_lan_addr.addr, mac_addr);
+	}
+}
+
+/**
  * i40e_vc_add_mac_addr_msg
  * @vf: pointer to the VF info
  * @msg: pointer to the msg buffer
@@ -2965,11 +3031,8 @@ static int i40e_vc_add_mac_addr_msg(struct i40e_vf *vf, u8 *msg)
 				spin_unlock_bh(&vsi->mac_filter_hash_lock);
 				goto error_param;
 			}
-			if (is_valid_ether_addr(al->list[i].addr) &&
-			    is_zero_ether_addr(vf->default_lan_addr.addr))
-				ether_addr_copy(vf->default_lan_addr.addr,
-						al->list[i].addr);
 		}
+		i40e_update_vf_mac_addr(vf, &al->list[i]);
 	}
 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
 
@@ -3032,6 +3095,9 @@ static int i40e_vc_del_mac_addr_msg(struct i40e_vf *vf, u8 *msg)
 
 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
 
+	if (was_unimac_deleted)
+		eth_zero_addr(vf->default_lan_addr.addr);
+
 	/* program the updated filter list */
 	ret = i40e_sync_vsi_filters(vsi);
 	if (ret)
diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h
index 746ff76f2fb1..9abaff1f2aff 100644
--- a/drivers/net/ethernet/intel/iavf/iavf.h
+++ b/drivers/net/ethernet/intel/iavf/iavf.h
@@ -6,7 +6,6 @@
 
 #include <linux/module.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/netdevice.h>
 #include <linux/vmalloc.h>
 #include <linux/interrupt.h>
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index e809249500e1..aa32111afd6e 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -20,7 +20,6 @@
 #include <linux/pci.h>
 #include <linux/workqueue.h>
 #include <linux/wait.h>
-#include <linux/aer.h>
 #include <linux/interrupt.h>
 #include <linux/ethtool.h>
 #include <linux/timer.h>
diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c
index c2fda4fa4188..0157f6e98d3e 100644
--- a/drivers/net/ethernet/intel/ice/ice_common.c
+++ b/drivers/net/ethernet/intel/ice/ice_common.c
@@ -1619,7 +1619,6 @@ ice_sq_send_cmd_retry(struct ice_hw *hw, struct ice_ctl_q_info *cq,
 {
 	struct ice_aq_desc desc_cpy;
 	bool is_cmd_for_retry;
-	u8 *buf_cpy = NULL;
 	u8 idx = 0;
 	u16 opcode;
 	int status;
@@ -1629,11 +1628,8 @@ ice_sq_send_cmd_retry(struct ice_hw *hw, struct ice_ctl_q_info *cq,
 	memset(&desc_cpy, 0, sizeof(desc_cpy));
 
 	if (is_cmd_for_retry) {
-		if (buf) {
-			buf_cpy = kzalloc(buf_size, GFP_KERNEL);
-			if (!buf_cpy)
-				return -ENOMEM;
-		}
+		/* All retryable cmds are direct, without buf. */
+		WARN_ON(buf);
 
 		memcpy(&desc_cpy, desc, sizeof(desc_cpy));
 	}
@@ -1645,17 +1641,12 @@ ice_sq_send_cmd_retry(struct ice_hw *hw, struct ice_ctl_q_info *cq,
 		    hw->adminq.sq_last_status != ICE_AQ_RC_EBUSY)
 			break;
 
-		if (buf_cpy)
-			memcpy(buf, buf_cpy, buf_size);
-
 		memcpy(desc, &desc_cpy, sizeof(desc_cpy));
 
-		mdelay(ICE_SQ_SEND_DELAY_TIME_MS);
+		msleep(ICE_SQ_SEND_DELAY_TIME_MS);
 
 	} while (++idx < ICE_SQ_SEND_MAX_EXECUTE);
 
-	kfree(buf_cpy);
-
 	return status;
 }
 
@@ -1992,19 +1983,19 @@ ice_acquire_res_exit:
  */
 void ice_release_res(struct ice_hw *hw, enum ice_aq_res_ids res)
 {
-	u32 total_delay = 0;
+	unsigned long timeout;
 	int status;
 
-	status = ice_aq_release_res(hw, res, 0, NULL);
-
 	/* there are some rare cases when trying to release the resource
 	 * results in an admin queue timeout, so handle them correctly
 	 */
-	while ((status == -EIO) && (total_delay < hw->adminq.sq_cmd_timeout)) {
-		mdelay(1);
+	timeout = jiffies + 10 * ICE_CTL_Q_SQ_CMD_TIMEOUT;
+	do {
 		status = ice_aq_release_res(hw, res, 0, NULL);
-		total_delay++;
-	}
+		if (status != -EIO)
+			break;
+		usleep_range(1000, 2000);
+	} while (time_before(jiffies, timeout));
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/ice/ice_controlq.c b/drivers/net/ethernet/intel/ice/ice_controlq.c
index 6bcfee295991..d2faf1baad2f 100644
--- a/drivers/net/ethernet/intel/ice/ice_controlq.c
+++ b/drivers/net/ethernet/intel/ice/ice_controlq.c
@@ -637,9 +637,6 @@ static int ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
 		return -EIO;
 	}
 
-	/* setup SQ command write back timeout */
-	cq->sq_cmd_timeout = ICE_CTL_Q_SQ_CMD_TIMEOUT;
-
 	/* allocate the ATQ */
 	ret_code = ice_init_sq(hw, cq);
 	if (ret_code)
@@ -967,7 +964,7 @@ ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq,
 	struct ice_aq_desc *desc_on_ring;
 	bool cmd_completed = false;
 	struct ice_sq_cd *details;
-	u32 total_delay = 0;
+	unsigned long timeout;
 	int status = 0;
 	u16 retval = 0;
 	u32 val = 0;
@@ -1060,13 +1057,14 @@ ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq,
 		cq->sq.next_to_use = 0;
 	wr32(hw, cq->sq.tail, cq->sq.next_to_use);
 
+	timeout = jiffies + ICE_CTL_Q_SQ_CMD_TIMEOUT;
 	do {
 		if (ice_sq_done(hw, cq))
 			break;
 
-		udelay(ICE_CTL_Q_SQ_CMD_USEC);
-		total_delay++;
-	} while (total_delay < cq->sq_cmd_timeout);
+		usleep_range(ICE_CTL_Q_SQ_CMD_USEC,
+			     ICE_CTL_Q_SQ_CMD_USEC * 3 / 2);
+	} while (time_before(jiffies, timeout));
 
 	/* if ready, copy the desc back to temp */
 	if (ice_sq_done(hw, cq)) {
diff --git a/drivers/net/ethernet/intel/ice/ice_controlq.h b/drivers/net/ethernet/intel/ice/ice_controlq.h
index c07e9cc9fc6e..950b7f4a7a05 100644
--- a/drivers/net/ethernet/intel/ice/ice_controlq.h
+++ b/drivers/net/ethernet/intel/ice/ice_controlq.h
@@ -34,7 +34,7 @@ enum ice_ctl_q {
 };
 
 /* Control Queue timeout settings - max delay 1s */
-#define ICE_CTL_Q_SQ_CMD_TIMEOUT	10000 /* Count 10000 times */
+#define ICE_CTL_Q_SQ_CMD_TIMEOUT	HZ    /* Wait max 1s */
 #define ICE_CTL_Q_SQ_CMD_USEC		100   /* Check every 100usec */
 #define ICE_CTL_Q_ADMIN_INIT_TIMEOUT	10    /* Count 10 times */
 #define ICE_CTL_Q_ADMIN_INIT_MSEC	100   /* Check every 100msec */
@@ -87,7 +87,6 @@ struct ice_ctl_q_info {
 	enum ice_ctl_q qtype;
 	struct ice_ctl_q_ring rq;	/* receive queue */
 	struct ice_ctl_q_ring sq;	/* send queue */
-	u32 sq_cmd_timeout;		/* send queue cmd write back timeout */
 	u16 num_rq_entries;		/* receive queue depth */
 	u16 num_sq_entries;		/* send queue depth */
 	u16 rq_buf_size;		/* receive queue buffer size */
diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
index 05f216af8c81..bc44cc220818 100644
--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
@@ -1254,7 +1254,6 @@ static const struct devlink_ops ice_devlink_ops = {
 	.supported_flash_update_params = DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK,
 	.reload_actions = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
 			  BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE),
-	/* The ice driver currently does not support driver reinit */
 	.reload_down = ice_devlink_reload_down,
 	.reload_up = ice_devlink_reload_up,
 	.port_split = ice_devlink_port_split,
diff --git a/drivers/net/ethernet/intel/ice/ice_gnss.c b/drivers/net/ethernet/intel/ice/ice_gnss.c
index 8dec748bb53a..2ea8a2b11bcd 100644
--- a/drivers/net/ethernet/intel/ice/ice_gnss.c
+++ b/drivers/net/ethernet/intel/ice/ice_gnss.c
@@ -117,6 +117,7 @@ static void ice_gnss_read(struct kthread_work *work)
 {
 	struct gnss_serial *gnss = container_of(work, struct gnss_serial,
 						read_work.work);
+	unsigned long delay = ICE_GNSS_POLL_DATA_DELAY_TIME;
 	unsigned int i, bytes_read, data_len, count;
 	struct ice_aqc_link_topo_addr link_topo;
 	struct ice_pf *pf;
@@ -136,11 +137,6 @@ static void ice_gnss_read(struct kthread_work *work)
 		return;
 
 	hw = &pf->hw;
-	buf = (char *)get_zeroed_page(GFP_KERNEL);
-	if (!buf) {
-		err = -ENOMEM;
-		goto exit;
-	}
 
 	memset(&link_topo, 0, sizeof(struct ice_aqc_link_topo_addr));
 	link_topo.topo_params.index = ICE_E810T_GNSS_I2C_BUS;
@@ -151,25 +147,24 @@ static void ice_gnss_read(struct kthread_work *work)
 	i2c_params = ICE_GNSS_UBX_DATA_LEN_WIDTH |
 		     ICE_AQC_I2C_USE_REPEATED_START;
 
-	/* Read data length in a loop, when it's not 0 the data is ready */
-	for (i = 0; i < ICE_MAX_UBX_READ_TRIES; i++) {
-		err = ice_aq_read_i2c(hw, link_topo, ICE_GNSS_UBX_I2C_BUS_ADDR,
-				      cpu_to_le16(ICE_GNSS_UBX_DATA_LEN_H),
-				      i2c_params, (u8 *)&data_len_b, NULL);
-		if (err)
-			goto exit_buf;
+	err = ice_aq_read_i2c(hw, link_topo, ICE_GNSS_UBX_I2C_BUS_ADDR,
+			      cpu_to_le16(ICE_GNSS_UBX_DATA_LEN_H),
+			      i2c_params, (u8 *)&data_len_b, NULL);
+	if (err)
+		goto requeue;
 
-		data_len = be16_to_cpu(data_len_b);
-		if (data_len != 0 && data_len != U16_MAX)
-			break;
+	data_len = be16_to_cpu(data_len_b);
+	if (data_len == 0 || data_len == U16_MAX)
+		goto requeue;
 
-		mdelay(10);
-	}
+	/* The u-blox has data_len bytes for us to read */
 
 	data_len = min_t(typeof(data_len), data_len, PAGE_SIZE);
-	if (!data_len) {
+
+	buf = (char *)get_zeroed_page(GFP_KERNEL);
+	if (!buf) {
 		err = -ENOMEM;
-		goto exit_buf;
+		goto requeue;
 	}
 
 	/* Read received data */
@@ -183,7 +178,7 @@ static void ice_gnss_read(struct kthread_work *work)
 				      cpu_to_le16(ICE_GNSS_UBX_EMPTY_DATA),
 				      bytes_read, &buf[i], NULL);
 		if (err)
-			goto exit_buf;
+			goto free_buf;
 	}
 
 	count = gnss_insert_raw(pf->gnss_dev, buf, i);
@@ -191,10 +186,11 @@ static void ice_gnss_read(struct kthread_work *work)
 		dev_warn(ice_pf_to_dev(pf),
 			 "gnss_insert_raw ret=%d size=%d\n",
 			 count, i);
-exit_buf:
+	delay = ICE_GNSS_TIMER_DELAY_TIME;
+free_buf:
 	free_page((unsigned long)buf);
-	kthread_queue_delayed_work(gnss->kworker, &gnss->read_work,
-				   ICE_GNSS_TIMER_DELAY_TIME);
+requeue:
+	kthread_queue_delayed_work(gnss->kworker, &gnss->read_work, delay);
 exit:
 	if (err)
 		dev_dbg(ice_pf_to_dev(pf), "GNSS failed to read err=%d\n", err);
diff --git a/drivers/net/ethernet/intel/ice/ice_gnss.h b/drivers/net/ethernet/intel/ice/ice_gnss.h
index 4d49e5b0b4b8..b8bb8b63d081 100644
--- a/drivers/net/ethernet/intel/ice/ice_gnss.h
+++ b/drivers/net/ethernet/intel/ice/ice_gnss.h
@@ -5,6 +5,7 @@
 #define _ICE_GNSS_H_
 
 #define ICE_E810T_GNSS_I2C_BUS		0x2
+#define ICE_GNSS_POLL_DATA_DELAY_TIME	(HZ / 50) /* poll every 20 ms */
 #define ICE_GNSS_TIMER_DELAY_TIME	(HZ / 10) /* 0.1 second per message */
 #define ICE_GNSS_TTY_WRITE_BUF		250
 #define ICE_MAX_I2C_DATA_SIZE		FIELD_MAX(ICE_AQC_I2C_DATA_SIZE_M)
@@ -20,8 +21,6 @@
  * passed as I2C addr parameter.
  */
 #define ICE_GNSS_UBX_WRITE_BYTES	(ICE_MAX_I2C_WRITE_BYTES + 1)
-#define ICE_MAX_UBX_READ_TRIES		255
-#define ICE_MAX_UBX_ACK_READ_TRIES	4095
 
 struct gnss_write_buf {
 	struct list_head queue;
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index 0d8b8c6f9bd3..a1f7c8edc22f 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -1393,6 +1393,8 @@ static void ice_aq_cancel_waiting_tasks(struct ice_pf *pf)
 	wake_up(&pf->aq_wait_queue);
 }
 
+#define ICE_MBX_OVERFLOW_WATERMARK 64
+
 /**
  * __ice_clean_ctrlq - helper function to clean controlq rings
  * @pf: ptr to struct ice_pf
@@ -1483,6 +1485,7 @@ static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type)
 		return 0;
 
 	do {
+		struct ice_mbx_data data = {};
 		u16 opcode;
 		int ret;
 
@@ -1509,8 +1512,12 @@ static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type)
 			ice_vf_lan_overflow_event(pf, &event);
 			break;
 		case ice_mbx_opc_send_msg_to_pf:
-			if (!ice_is_malicious_vf(pf, &event, i, pending))
-				ice_vc_process_vf_msg(pf, &event);
+			data.num_msg_proc = i;
+			data.num_pending_arq = pending;
+			data.max_num_msgs_mbx = hw->mailboxq.num_rq_entries;
+			data.async_watermark_val = ICE_MBX_OVERFLOW_WATERMARK;
+
+			ice_vc_process_vf_msg(pf, &event, &data);
 			break;
 		case ice_aqc_opc_fw_logging:
 			ice_output_fw_log(hw, &event.desc, event.msg_buf);
@@ -3888,6 +3895,7 @@ static int ice_init_pf(struct ice_pf *pf)
 
 	mutex_init(&pf->vfs.table_lock);
 	hash_init(pf->vfs.table);
+	ice_mbx_init_snapshot(&pf->hw);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.c b/drivers/net/ethernet/intel/ice/ice_sriov.c
index 0cc05e54a781..f1dca59bd844 100644
--- a/drivers/net/ethernet/intel/ice/ice_sriov.c
+++ b/drivers/net/ethernet/intel/ice/ice_sriov.c
@@ -204,10 +204,7 @@ void ice_free_vfs(struct ice_pf *pf)
 		}
 
 		/* clear malicious info since the VF is getting released */
-		if (ice_mbx_clear_malvf(&hw->mbx_snapshot, pf->vfs.malvfs,
-					ICE_MAX_SRIOV_VFS, vf->vf_id))
-			dev_dbg(dev, "failed to clear malicious VF state for VF %u\n",
-				vf->vf_id);
+		list_del(&vf->mbx_info.list_entry);
 
 		mutex_unlock(&vf->cfg_lock);
 	}
@@ -1017,7 +1014,6 @@ int ice_sriov_configure(struct pci_dev *pdev, int num_vfs)
 	if (!num_vfs) {
 		if (!pci_vfs_assigned(pdev)) {
 			ice_free_vfs(pf);
-			ice_mbx_deinit_snapshot(&pf->hw);
 			if (pf->lag)
 				ice_enable_lag(pf->lag);
 			return 0;
@@ -1027,15 +1023,9 @@ int ice_sriov_configure(struct pci_dev *pdev, int num_vfs)
 		return -EBUSY;
 	}
 
-	err = ice_mbx_init_snapshot(&pf->hw, num_vfs);
-	if (err)
-		return err;
-
 	err = ice_pci_sriov_ena(pf, num_vfs);
-	if (err) {
-		ice_mbx_deinit_snapshot(&pf->hw);
+	if (err)
 		return err;
-	}
 
 	if (pf->lag)
 		ice_disable_lag(pf->lag);
@@ -1787,66 +1777,3 @@ void ice_restore_all_vfs_msi_state(struct pci_dev *pdev)
 		}
 	}
 }
-
-/**
- * ice_is_malicious_vf - helper function to detect a malicious VF
- * @pf: ptr to struct ice_pf
- * @event: pointer to the AQ event
- * @num_msg_proc: the number of messages processed so far
- * @num_msg_pending: the number of messages peinding in admin queue
- */
-bool
-ice_is_malicious_vf(struct ice_pf *pf, struct ice_rq_event_info *event,
-		    u16 num_msg_proc, u16 num_msg_pending)
-{
-	s16 vf_id = le16_to_cpu(event->desc.retval);
-	struct device *dev = ice_pf_to_dev(pf);
-	struct ice_mbx_data mbxdata;
-	bool malvf = false;
-	struct ice_vf *vf;
-	int status;
-
-	vf = ice_get_vf_by_id(pf, vf_id);
-	if (!vf)
-		return false;
-
-	if (test_bit(ICE_VF_STATE_DIS, vf->vf_states))
-		goto out_put_vf;
-
-	mbxdata.num_msg_proc = num_msg_proc;
-	mbxdata.num_pending_arq = num_msg_pending;
-	mbxdata.max_num_msgs_mbx = pf->hw.mailboxq.num_rq_entries;
-#define ICE_MBX_OVERFLOW_WATERMARK 64
-	mbxdata.async_watermark_val = ICE_MBX_OVERFLOW_WATERMARK;
-
-	/* check to see if we have a malicious VF */
-	status = ice_mbx_vf_state_handler(&pf->hw, &mbxdata, vf_id, &malvf);
-	if (status)
-		goto out_put_vf;
-
-	if (malvf) {
-		bool report_vf = false;
-
-		/* if the VF is malicious and we haven't let the user
-		 * know about it, then let them know now
-		 */
-		status = ice_mbx_report_malvf(&pf->hw, pf->vfs.malvfs,
-					      ICE_MAX_SRIOV_VFS, vf_id,
-					      &report_vf);
-		if (status)
-			dev_dbg(dev, "Error reporting malicious VF\n");
-
-		if (report_vf) {
-			struct ice_vsi *pf_vsi = ice_get_main_vsi(pf);
-
-			if (pf_vsi)
-				dev_warn(dev, "VF MAC %pM on PF MAC %pM is generating asynchronous messages and may be overflowing the PF message queue. Please see the Adapter User Guide for more information\n",
-					 &vf->dev_lan_addr[0],
-					 pf_vsi->netdev->dev_addr);
-		}
-	}
-
-out_put_vf:
-	ice_put_vf(vf);
-	return malvf;
-}
diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.h b/drivers/net/ethernet/intel/ice/ice_sriov.h
index 955ab810a198..346cb2666f3a 100644
--- a/drivers/net/ethernet/intel/ice/ice_sriov.h
+++ b/drivers/net/ethernet/intel/ice/ice_sriov.h
@@ -33,11 +33,7 @@ int
 ice_get_vf_cfg(struct net_device *netdev, int vf_id, struct ifla_vf_info *ivi);
 
 void ice_free_vfs(struct ice_pf *pf);
-void ice_vc_process_vf_msg(struct ice_pf *pf, struct ice_rq_event_info *event);
 void ice_restore_all_vfs_msi_state(struct pci_dev *pdev);
-bool
-ice_is_malicious_vf(struct ice_pf *pf, struct ice_rq_event_info *event,
-		    u16 num_msg_proc, u16 num_msg_pending);
 
 int
 ice_set_vf_port_vlan(struct net_device *netdev, int vf_id, u16 vlan_id, u8 qos,
@@ -68,22 +64,11 @@ ice_vc_validate_pattern(struct ice_vf *vf, struct virtchnl_proto_hdrs *proto);
 static inline void ice_process_vflr_event(struct ice_pf *pf) { }
 static inline void ice_free_vfs(struct ice_pf *pf) { }
 static inline
-void ice_vc_process_vf_msg(struct ice_pf *pf, struct ice_rq_event_info *event) { }
-static inline
 void ice_vf_lan_overflow_event(struct ice_pf *pf, struct ice_rq_event_info *event) { }
 static inline void ice_print_vfs_mdd_events(struct ice_pf *pf) { }
 static inline void ice_print_vf_rx_mdd_event(struct ice_vf *vf) { }
 static inline void ice_restore_all_vfs_msi_state(struct pci_dev *pdev) { }
 
-static inline bool
-ice_is_malicious_vf(struct ice_pf __always_unused *pf,
-		    struct ice_rq_event_info __always_unused *event,
-		    u16 __always_unused num_msg_proc,
-		    u16 __always_unused num_msg_pending)
-{
-	return false;
-}
-
 static inline int
 ice_sriov_configure(struct pci_dev __always_unused *pdev,
 		    int __always_unused num_vfs)
diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h
index e3f622cad425..a09556e57803 100644
--- a/drivers/net/ethernet/intel/ice/ice_type.h
+++ b/drivers/net/ethernet/intel/ice/ice_type.h
@@ -784,14 +784,15 @@ struct ice_mbx_snap_buffer_data {
 	u16 max_num_msgs_mbx;
 };
 
-/* Structure to track messages sent by VFs on mailbox:
- * 1. vf_cntr: a counter array of VFs to track the number of
- * asynchronous messages sent by each VF
- * 2. vfcntr_len: number of entries in VF counter array
+/* Structure used to track a single VF's messages on the mailbox:
+ * 1. list_entry: linked list entry node
+ * 2. msg_count: the number of asynchronous messages sent by this VF
+ * 3. malicious: whether this VF has been detected as malicious before
  */
-struct ice_mbx_vf_counter {
-	u32 *vf_cntr;
-	u32 vfcntr_len;
+struct ice_mbx_vf_info {
+	struct list_head list_entry;
+	u32 msg_count;
+	u8 malicious : 1;
 };
 
 /* Structure to hold data relevant to the captured static snapshot
@@ -799,7 +800,7 @@ struct ice_mbx_vf_counter {
  */
 struct ice_mbx_snapshot {
 	struct ice_mbx_snap_buffer_data mbx_buf;
-	struct ice_mbx_vf_counter mbx_vf;
+	struct list_head mbx_vf;
 };
 
 /* Structure to hold data to be used for capturing or updating a
diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c
index 0e57bd1b85fd..89fd6982df09 100644
--- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c
@@ -496,10 +496,7 @@ void ice_reset_all_vfs(struct ice_pf *pf)
 
 	/* clear all malicious info if the VFs are getting reset */
 	ice_for_each_vf(pf, bkt, vf)
-		if (ice_mbx_clear_malvf(&hw->mbx_snapshot, pf->vfs.malvfs,
-					ICE_MAX_SRIOV_VFS, vf->vf_id))
-			dev_dbg(dev, "failed to clear malicious VF state for VF %u\n",
-				vf->vf_id);
+		ice_mbx_clear_malvf(&vf->mbx_info);
 
 	/* If VFs have been disabled, there is no need to reset */
 	if (test_and_set_bit(ICE_VF_DIS, pf->state)) {
@@ -601,12 +598,10 @@ int ice_reset_vf(struct ice_vf *vf, u32 flags)
 	struct ice_pf *pf = vf->pf;
 	struct ice_vsi *vsi;
 	struct device *dev;
-	struct ice_hw *hw;
 	int err = 0;
 	bool rsd;
 
 	dev = ice_pf_to_dev(pf);
-	hw = &pf->hw;
 
 	if (flags & ICE_VF_RESET_NOTIFY)
 		ice_notify_vf_reset(vf);
@@ -705,10 +700,7 @@ int ice_reset_vf(struct ice_vf *vf, u32 flags)
 	ice_eswitch_replay_vf_mac_rule(vf);
 
 	/* if the VF has been reset allow it to come up again */
-	if (ice_mbx_clear_malvf(&hw->mbx_snapshot, pf->vfs.malvfs,
-				ICE_MAX_SRIOV_VFS, vf->vf_id))
-		dev_dbg(dev, "failed to clear malicious VF state for VF %u\n",
-			vf->vf_id);
+	ice_mbx_clear_malvf(&vf->mbx_info);
 
 out_unlock:
 	if (flags & ICE_VF_RESET_LOCK)
@@ -764,6 +756,9 @@ void ice_initialize_vf_entry(struct ice_vf *vf)
 	ice_vf_ctrl_invalidate_vsi(vf);
 	ice_vf_fdir_init(vf);
 
+	/* Initialize mailbox info for this VF */
+	ice_mbx_init_vf_info(&pf->hw, &vf->mbx_info);
+
 	mutex_init(&vf->cfg_lock);
 }
 
diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.h b/drivers/net/ethernet/intel/ice/ice_vf_lib.h
index ef30f05b5d02..e3cda6fb71ab 100644
--- a/drivers/net/ethernet/intel/ice/ice_vf_lib.h
+++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.h
@@ -74,7 +74,6 @@ struct ice_vfs {
 	u16 num_qps_per;		/* number of queue pairs per VF */
 	u16 num_msix_per;		/* number of MSI-X vectors per VF */
 	unsigned long last_printed_mdd_jiffies;	/* MDD message rate limit */
-	DECLARE_BITMAP(malvfs, ICE_MAX_SRIOV_VFS); /* malicious VF indicator */
 };
 
 /* VF information structure */
@@ -105,6 +104,7 @@ struct ice_vf {
 	DECLARE_BITMAP(rxq_ena, ICE_MAX_RSS_QS_PER_VF);
 	struct ice_vlan port_vlan_info;	/* Port VLAN ID, QoS, and TPID */
 	struct virtchnl_vlan_caps vlan_v2_caps;
+	struct ice_mbx_vf_info mbx_info;
 	u8 pf_set_mac:1;		/* VF MAC address set by VMM admin */
 	u8 trusted:1;
 	u8 spoofchk:1;
diff --git a/drivers/net/ethernet/intel/ice/ice_vf_mbx.c b/drivers/net/ethernet/intel/ice/ice_vf_mbx.c
index f56fa94ff3d0..40cb4ba0789c 100644
--- a/drivers/net/ethernet/intel/ice/ice_vf_mbx.c
+++ b/drivers/net/ethernet/intel/ice/ice_vf_mbx.c
@@ -93,36 +93,31 @@ u32 ice_conv_link_speed_to_virtchnl(bool adv_link_support, u16 link_speed)
  *
  * 2. When the caller starts processing its mailbox queue in response to an
  * interrupt, the structure ice_mbx_snapshot is expected to be cleared before
- * the algorithm can be run for the first time for that interrupt. This can be
- * done via ice_mbx_reset_snapshot().
+ * the algorithm can be run for the first time for that interrupt. This
+ * requires calling ice_mbx_reset_snapshot() as well as calling
+ * ice_mbx_reset_vf_info() for each VF tracking structure.
  *
  * 3. For every message read by the caller from the MBX Queue, the caller must
  * call the detection algorithm's entry function ice_mbx_vf_state_handler().
  * Before every call to ice_mbx_vf_state_handler() the struct ice_mbx_data is
  * filled as it is required to be passed to the algorithm.
  *
- * 4. Every time a message is read from the MBX queue, a VFId is received which
- * is passed to the state handler. The boolean output is_malvf of the state
- * handler ice_mbx_vf_state_handler() serves as an indicator to the caller
- * whether this VF is malicious or not.
+ * 4. Every time a message is read from the MBX queue, a tracking structure
+ * for the VF must be passed to the state handler. The boolean output
+ * report_malvf from ice_mbx_vf_state_handler() serves as an indicator to the
+ * caller whether it must report this VF as malicious or not.
  *
  * 5. When a VF is identified to be malicious, the caller can send a message
- * to the system administrator. The caller can invoke ice_mbx_report_malvf()
- * to help determine if a malicious VF is to be reported or not. This function
- * requires the caller to maintain a global bitmap to track all malicious VFs
- * and pass that to ice_mbx_report_malvf() along with the VFID which was identified
- * to be malicious by ice_mbx_vf_state_handler().
+ * to the system administrator.
  *
- * 6. The global bitmap maintained by PF can be cleared completely if PF is in
- * reset or the bit corresponding to a VF can be cleared if that VF is in reset.
- * When a VF is shut down and brought back up, we assume that the new VF
- * brought up is not malicious and hence report it if found malicious.
+ * 6. The PF is responsible for maintaining the struct ice_mbx_vf_info
+ * structure for each VF. The PF should clear the VF tracking structure if the
+ * VF is reset. When a VF is shut down and brought back up, we will then
+ * assume that the new VF is not malicious and may report it again if we
+ * detect it again.
  *
  * 7. The function ice_mbx_reset_snapshot() is called to reset the information
  * in ice_mbx_snapshot for every new mailbox interrupt handled.
- *
- * 8. The memory allocated for variables in ice_mbx_snapshot is de-allocated
- * when driver is unloaded.
  */
 #define ICE_RQ_DATA_MASK(rq_data) ((rq_data) & PF_MBX_ARQH_ARQH_M)
 /* Using the highest value for an unsigned 16-bit value 0xFFFF to indicate that
@@ -131,6 +126,25 @@ u32 ice_conv_link_speed_to_virtchnl(bool adv_link_support, u16 link_speed)
 #define ICE_IGNORE_MAX_MSG_CNT	0xFFFF
 
 /**
+ * ice_mbx_reset_snapshot - Reset mailbox snapshot structure
+ * @snap: pointer to the mailbox snapshot
+ */
+static void ice_mbx_reset_snapshot(struct ice_mbx_snapshot *snap)
+{
+	struct ice_mbx_vf_info *vf_info;
+
+	/* Clear mbx_buf in the mailbox snaphot structure and setting the
+	 * mailbox snapshot state to a new capture.
+	 */
+	memset(&snap->mbx_buf, 0, sizeof(snap->mbx_buf));
+	snap->mbx_buf.state = ICE_MAL_VF_DETECT_STATE_NEW_SNAPSHOT;
+
+	/* Reset message counts for all VFs to zero */
+	list_for_each_entry(vf_info, &snap->mbx_vf, list_entry)
+		vf_info->msg_count = 0;
+}
+
+/**
  * ice_mbx_traverse - Pass through mailbox snapshot
  * @hw: pointer to the HW struct
  * @new_state: new algorithm state
@@ -171,7 +185,7 @@ ice_mbx_traverse(struct ice_hw *hw,
 /**
  * ice_mbx_detect_malvf - Detect malicious VF in snapshot
  * @hw: pointer to the HW struct
- * @vf_id: relative virtual function ID
+ * @vf_info: mailbox tracking structure for a VF
  * @new_state: new algorithm state
  * @is_malvf: boolean output to indicate if VF is malicious
  *
@@ -180,19 +194,14 @@ ice_mbx_traverse(struct ice_hw *hw,
  * the permissible number of messages to send.
  */
 static int
-ice_mbx_detect_malvf(struct ice_hw *hw, u16 vf_id,
+ice_mbx_detect_malvf(struct ice_hw *hw, struct ice_mbx_vf_info *vf_info,
 		     enum ice_mbx_snapshot_state *new_state,
 		     bool *is_malvf)
 {
-	struct ice_mbx_snapshot *snap = &hw->mbx_snapshot;
-
-	if (vf_id >= snap->mbx_vf.vfcntr_len)
-		return -EIO;
-
-	/* increment the message count in the VF array */
-	snap->mbx_vf.vf_cntr[vf_id]++;
+	/* increment the message count for this VF */
+	vf_info->msg_count++;
 
-	if (snap->mbx_vf.vf_cntr[vf_id] >= ICE_ASYNC_VF_MSG_THRESHOLD)
+	if (vf_info->msg_count >= ICE_ASYNC_VF_MSG_THRESHOLD)
 		*is_malvf = true;
 
 	/* continue to iterate through the mailbox snapshot */
@@ -202,35 +211,11 @@ ice_mbx_detect_malvf(struct ice_hw *hw, u16 vf_id,
 }
 
 /**
- * ice_mbx_reset_snapshot - Reset mailbox snapshot structure
- * @snap: pointer to mailbox snapshot structure in the ice_hw struct
- *
- * Reset the mailbox snapshot structure and clear VF counter array.
- */
-static void ice_mbx_reset_snapshot(struct ice_mbx_snapshot *snap)
-{
-	u32 vfcntr_len;
-
-	if (!snap || !snap->mbx_vf.vf_cntr)
-		return;
-
-	/* Clear VF counters. */
-	vfcntr_len = snap->mbx_vf.vfcntr_len;
-	if (vfcntr_len)
-		memset(snap->mbx_vf.vf_cntr, 0,
-		       (vfcntr_len * sizeof(*snap->mbx_vf.vf_cntr)));
-
-	/* Reset mailbox snapshot for a new capture. */
-	memset(&snap->mbx_buf, 0, sizeof(snap->mbx_buf));
-	snap->mbx_buf.state = ICE_MAL_VF_DETECT_STATE_NEW_SNAPSHOT;
-}
-
-/**
  * ice_mbx_vf_state_handler - Handle states of the overflow algorithm
  * @hw: pointer to the HW struct
  * @mbx_data: pointer to structure containing mailbox data
- * @vf_id: relative virtual function (VF) ID
- * @is_malvf: boolean output to indicate if VF is malicious
+ * @vf_info: mailbox tracking structure for the VF in question
+ * @report_malvf: boolean output to indicate whether VF should be reported
  *
  * The function serves as an entry point for the malicious VF
  * detection algorithm by handling the different states and state
@@ -249,24 +234,24 @@ static void ice_mbx_reset_snapshot(struct ice_mbx_snapshot *snap)
  * the static snapshot and look for a malicious VF.
  */
 int
-ice_mbx_vf_state_handler(struct ice_hw *hw,
-			 struct ice_mbx_data *mbx_data, u16 vf_id,
-			 bool *is_malvf)
+ice_mbx_vf_state_handler(struct ice_hw *hw, struct ice_mbx_data *mbx_data,
+			 struct ice_mbx_vf_info *vf_info, bool *report_malvf)
 {
 	struct ice_mbx_snapshot *snap = &hw->mbx_snapshot;
 	struct ice_mbx_snap_buffer_data *snap_buf;
 	struct ice_ctl_q_info *cq = &hw->mailboxq;
 	enum ice_mbx_snapshot_state new_state;
+	bool is_malvf = false;
 	int status = 0;
 
-	if (!is_malvf || !mbx_data)
+	if (!report_malvf || !mbx_data || !vf_info)
 		return -EINVAL;
 
+	*report_malvf = false;
+
 	/* When entering the mailbox state machine assume that the VF
 	 * is not malicious until detected.
 	 */
-	*is_malvf = false;
-
 	 /* Checking if max messages allowed to be processed while servicing current
 	  * interrupt is not less than the defined AVF message threshold.
 	  */
@@ -315,7 +300,7 @@ ice_mbx_vf_state_handler(struct ice_hw *hw,
 		if (snap_buf->num_pending_arq >=
 		    mbx_data->async_watermark_val) {
 			new_state = ICE_MAL_VF_DETECT_STATE_DETECT;
-			status = ice_mbx_detect_malvf(hw, vf_id, &new_state, is_malvf);
+			status = ice_mbx_detect_malvf(hw, vf_info, &new_state, &is_malvf);
 		} else {
 			new_state = ICE_MAL_VF_DETECT_STATE_TRAVERSE;
 			ice_mbx_traverse(hw, &new_state);
@@ -329,7 +314,7 @@ ice_mbx_vf_state_handler(struct ice_hw *hw,
 
 	case ICE_MAL_VF_DETECT_STATE_DETECT:
 		new_state = ICE_MAL_VF_DETECT_STATE_DETECT;
-		status = ice_mbx_detect_malvf(hw, vf_id, &new_state, is_malvf);
+		status = ice_mbx_detect_malvf(hw, vf_info, &new_state, &is_malvf);
 		break;
 
 	default:
@@ -339,145 +324,57 @@ ice_mbx_vf_state_handler(struct ice_hw *hw,
 
 	snap_buf->state = new_state;
 
-	return status;
-}
-
-/**
- * ice_mbx_report_malvf - Track and note malicious VF
- * @hw: pointer to the HW struct
- * @all_malvfs: all malicious VFs tracked by PF
- * @bitmap_len: length of bitmap in bits
- * @vf_id: relative virtual function ID of the malicious VF
- * @report_malvf: boolean to indicate if malicious VF must be reported
- *
- * This function will update a bitmap that keeps track of the malicious
- * VFs attached to the PF. A malicious VF must be reported only once if
- * discovered between VF resets or loading so the function checks
- * the input vf_id against the bitmap to verify if the VF has been
- * detected in any previous mailbox iterations.
- */
-int
-ice_mbx_report_malvf(struct ice_hw *hw, unsigned long *all_malvfs,
-		     u16 bitmap_len, u16 vf_id, bool *report_malvf)
-{
-	if (!all_malvfs || !report_malvf)
-		return -EINVAL;
-
-	*report_malvf = false;
-
-	if (bitmap_len < hw->mbx_snapshot.mbx_vf.vfcntr_len)
-		return -EINVAL;
-
-	if (vf_id >= bitmap_len)
-		return -EIO;
-
-	/* If the vf_id is found in the bitmap set bit and boolean to true */
-	if (!test_and_set_bit(vf_id, all_malvfs))
+	/* Only report VFs as malicious the first time we detect it */
+	if (is_malvf && !vf_info->malicious) {
+		vf_info->malicious = 1;
 		*report_malvf = true;
+	}
 
-	return 0;
+	return status;
 }
 
 /**
- * ice_mbx_clear_malvf - Clear VF bitmap and counter for VF ID
- * @snap: pointer to the mailbox snapshot structure
- * @all_malvfs: all malicious VFs tracked by PF
- * @bitmap_len: length of bitmap in bits
- * @vf_id: relative virtual function ID of the malicious VF
+ * ice_mbx_clear_malvf - Clear VF mailbox info
+ * @vf_info: the mailbox tracking structure for a VF
  *
- * In case of a VF reset, this function can be called to clear
- * the bit corresponding to the VF ID in the bitmap tracking all
- * malicious VFs attached to the PF. The function also clears the
- * VF counter array at the index of the VF ID. This is to ensure
- * that the new VF loaded is not considered malicious before going
- * through the overflow detection algorithm.
+ * In case of a VF reset, this function shall be called to clear the VF's
+ * current mailbox tracking state.
  */
-int
-ice_mbx_clear_malvf(struct ice_mbx_snapshot *snap, unsigned long *all_malvfs,
-		    u16 bitmap_len, u16 vf_id)
+void ice_mbx_clear_malvf(struct ice_mbx_vf_info *vf_info)
 {
-	if (!snap || !all_malvfs)
-		return -EINVAL;
-
-	if (bitmap_len < snap->mbx_vf.vfcntr_len)
-		return -EINVAL;
-
-	/* Ensure VF ID value is not larger than bitmap or VF counter length */
-	if (vf_id >= bitmap_len || vf_id >= snap->mbx_vf.vfcntr_len)
-		return -EIO;
-
-	/* Clear VF ID bit in the bitmap tracking malicious VFs attached to PF */
-	clear_bit(vf_id, all_malvfs);
-
-	/* Clear the VF counter in the mailbox snapshot structure for that VF ID.
-	 * This is to ensure that if a VF is unloaded and a new one brought back
-	 * up with the same VF ID for a snapshot currently in traversal or detect
-	 * state the counter for that VF ID does not increment on top of existing
-	 * values in the mailbox overflow detection algorithm.
-	 */
-	snap->mbx_vf.vf_cntr[vf_id] = 0;
-
-	return 0;
+	vf_info->malicious = 0;
+	vf_info->msg_count = 0;
 }
 
 /**
- * ice_mbx_init_snapshot - Initialize mailbox snapshot structure
+ * ice_mbx_init_vf_info - Initialize a new VF mailbox tracking info
  * @hw: pointer to the hardware structure
- * @vf_count: number of VFs allocated on a PF
+ * @vf_info: the mailbox tracking info structure for a VF
  *
- * Clear the mailbox snapshot structure and allocate memory
- * for the VF counter array based on the number of VFs allocated
- * on that PF.
+ * Initialize a VF mailbox tracking info structure and insert it into the
+ * snapshot list.
  *
- * Assumption: This function will assume ice_get_caps() has already been
- * called to ensure that the vf_count can be compared against the number
- * of VFs supported as defined in the functional capabilities of the device.
+ * If you remove the VF, you must also delete the associated VF info structure
+ * from the linked list.
  */
-int ice_mbx_init_snapshot(struct ice_hw *hw, u16 vf_count)
+void ice_mbx_init_vf_info(struct ice_hw *hw, struct ice_mbx_vf_info *vf_info)
 {
 	struct ice_mbx_snapshot *snap = &hw->mbx_snapshot;
 
-	/* Ensure that the number of VFs allocated is non-zero and
-	 * is not greater than the number of supported VFs defined in
-	 * the functional capabilities of the PF.
-	 */
-	if (!vf_count || vf_count > hw->func_caps.num_allocd_vfs)
-		return -EINVAL;
-
-	snap->mbx_vf.vf_cntr = devm_kcalloc(ice_hw_to_dev(hw), vf_count,
-					    sizeof(*snap->mbx_vf.vf_cntr),
-					    GFP_KERNEL);
-	if (!snap->mbx_vf.vf_cntr)
-		return -ENOMEM;
-
-	/* Setting the VF counter length to the number of allocated
-	 * VFs for given PF's functional capabilities.
-	 */
-	snap->mbx_vf.vfcntr_len = vf_count;
-
-	/* Clear mbx_buf in the mailbox snaphot structure and setting the
-	 * mailbox snapshot state to a new capture.
-	 */
-	memset(&snap->mbx_buf, 0, sizeof(snap->mbx_buf));
-	snap->mbx_buf.state = ICE_MAL_VF_DETECT_STATE_NEW_SNAPSHOT;
-
-	return 0;
+	ice_mbx_clear_malvf(vf_info);
+	list_add(&vf_info->list_entry, &snap->mbx_vf);
 }
 
 /**
- * ice_mbx_deinit_snapshot - Free mailbox snapshot structure
+ * ice_mbx_init_snapshot - Initialize mailbox snapshot data
  * @hw: pointer to the hardware structure
  *
- * Clear the mailbox snapshot structure and free the VF counter array.
+ * Clear the mailbox snapshot structure and initialize the VF mailbox list.
  */
-void ice_mbx_deinit_snapshot(struct ice_hw *hw)
+void ice_mbx_init_snapshot(struct ice_hw *hw)
 {
 	struct ice_mbx_snapshot *snap = &hw->mbx_snapshot;
 
-	/* Free VF counter array and reset VF counter length */
-	devm_kfree(ice_hw_to_dev(hw), snap->mbx_vf.vf_cntr);
-	snap->mbx_vf.vfcntr_len = 0;
-
-	/* Clear mbx_buf in the mailbox snaphot structure */
-	memset(&snap->mbx_buf, 0, sizeof(snap->mbx_buf));
+	INIT_LIST_HEAD(&snap->mbx_vf);
+	ice_mbx_reset_snapshot(snap);
 }
diff --git a/drivers/net/ethernet/intel/ice/ice_vf_mbx.h b/drivers/net/ethernet/intel/ice/ice_vf_mbx.h
index 582716e6d5f9..44bc030d17e0 100644
--- a/drivers/net/ethernet/intel/ice/ice_vf_mbx.h
+++ b/drivers/net/ethernet/intel/ice/ice_vf_mbx.h
@@ -21,15 +21,10 @@ ice_aq_send_msg_to_vf(struct ice_hw *hw, u16 vfid, u32 v_opcode, u32 v_retval,
 u32 ice_conv_link_speed_to_virtchnl(bool adv_link_support, u16 link_speed);
 int
 ice_mbx_vf_state_handler(struct ice_hw *hw, struct ice_mbx_data *mbx_data,
-			 u16 vf_id, bool *is_mal_vf);
-int
-ice_mbx_clear_malvf(struct ice_mbx_snapshot *snap, unsigned long *all_malvfs,
-		    u16 bitmap_len, u16 vf_id);
-int ice_mbx_init_snapshot(struct ice_hw *hw, u16 vf_count);
-void ice_mbx_deinit_snapshot(struct ice_hw *hw);
-int
-ice_mbx_report_malvf(struct ice_hw *hw, unsigned long *all_malvfs,
-		     u16 bitmap_len, u16 vf_id, bool *report_malvf);
+			 struct ice_mbx_vf_info *vf_info, bool *report_malvf);
+void ice_mbx_clear_malvf(struct ice_mbx_vf_info *vf_info);
+void ice_mbx_init_vf_info(struct ice_hw *hw, struct ice_mbx_vf_info *vf_info);
+void ice_mbx_init_snapshot(struct ice_hw *hw);
 #else /* CONFIG_PCI_IOV */
 static inline int
 ice_aq_send_msg_to_vf(struct ice_hw __always_unused *hw,
@@ -48,5 +43,9 @@ ice_conv_link_speed_to_virtchnl(bool __always_unused adv_link_support,
 	return 0;
 }
 
+static inline void ice_mbx_init_snapshot(struct ice_hw *hw)
+{
+}
+
 #endif /* CONFIG_PCI_IOV */
 #endif /* _ICE_VF_MBX_H_ */
diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/ice_virtchnl.c
index e24e3f5017ca..97243c616d5d 100644
--- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c
+++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.c
@@ -3834,14 +3834,57 @@ void ice_virtchnl_set_repr_ops(struct ice_vf *vf)
 }
 
 /**
+ * ice_is_malicious_vf - check if this vf might be overflowing mailbox
+ * @vf: the VF to check
+ * @mbxdata: data about the state of the mailbox
+ *
+ * Detect if a given VF might be malicious and attempting to overflow the PF
+ * mailbox. If so, log a warning message and ignore this event.
+ */
+static bool
+ice_is_malicious_vf(struct ice_vf *vf, struct ice_mbx_data *mbxdata)
+{
+	bool report_malvf = false;
+	struct device *dev;
+	struct ice_pf *pf;
+	int status;
+
+	pf = vf->pf;
+	dev = ice_pf_to_dev(pf);
+
+	if (test_bit(ICE_VF_STATE_DIS, vf->vf_states))
+		return vf->mbx_info.malicious;
+
+	/* check to see if we have a newly malicious VF */
+	status = ice_mbx_vf_state_handler(&pf->hw, mbxdata, &vf->mbx_info,
+					  &report_malvf);
+	if (status)
+		dev_warn_ratelimited(dev, "Unable to check status of mailbox overflow for VF %u MAC %pM, status %d\n",
+				     vf->vf_id, vf->dev_lan_addr, status);
+
+	if (report_malvf) {
+		struct ice_vsi *pf_vsi = ice_get_main_vsi(pf);
+		u8 zero_addr[ETH_ALEN] = {};
+
+		dev_warn(dev, "VF MAC %pM on PF MAC %pM is generating asynchronous messages and may be overflowing the PF message queue. Please see the Adapter User Guide for more information\n",
+			 vf->dev_lan_addr,
+			 pf_vsi ? pf_vsi->netdev->dev_addr : zero_addr);
+	}
+
+	return vf->mbx_info.malicious;
+}
+
+/**
  * ice_vc_process_vf_msg - Process request from VF
  * @pf: pointer to the PF structure
  * @event: pointer to the AQ event
+ * @mbxdata: information used to detect VF attempting mailbox overflow
  *
  * called from the common asq/arq handler to
  * process request from VF
  */
-void ice_vc_process_vf_msg(struct ice_pf *pf, struct ice_rq_event_info *event)
+void ice_vc_process_vf_msg(struct ice_pf *pf, struct ice_rq_event_info *event,
+			   struct ice_mbx_data *mbxdata)
 {
 	u32 v_opcode = le32_to_cpu(event->desc.cookie_high);
 	s16 vf_id = le16_to_cpu(event->desc.retval);
@@ -3863,6 +3906,10 @@ void ice_vc_process_vf_msg(struct ice_pf *pf, struct ice_rq_event_info *event)
 
 	mutex_lock(&vf->cfg_lock);
 
+	/* Check if the VF is trying to overflow the mailbox */
+	if (ice_is_malicious_vf(vf, mbxdata))
+		goto finish;
+
 	/* Check if VF is disabled. */
 	if (test_bit(ICE_VF_STATE_DIS, vf->vf_states)) {
 		err = -EPERM;
diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.h b/drivers/net/ethernet/intel/ice/ice_virtchnl.h
index b454654d7b0c..cd747718de73 100644
--- a/drivers/net/ethernet/intel/ice/ice_virtchnl.h
+++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.h
@@ -63,6 +63,8 @@ int
 ice_vc_send_msg_to_vf(struct ice_vf *vf, u32 v_opcode,
 		      enum virtchnl_status_code v_retval, u8 *msg, u16 msglen);
 bool ice_vc_isvalid_vsi_id(struct ice_vf *vf, u16 vsi_id);
+void ice_vc_process_vf_msg(struct ice_pf *pf, struct ice_rq_event_info *event,
+			   struct ice_mbx_data *mbxdata);
 #else /* CONFIG_PCI_IOV */
 static inline void ice_virtchnl_set_dflt_ops(struct ice_vf *vf) { }
 static inline void ice_virtchnl_set_repr_ops(struct ice_vf *vf) { }
@@ -81,6 +83,12 @@ static inline bool ice_vc_isvalid_vsi_id(struct ice_vf *vf, u16 vsi_id)
 {
 	return false;
 }
+
+static inline void
+ice_vc_process_vf_msg(struct ice_pf *pf, struct ice_rq_event_info *event,
+		      struct ice_mbx_data *mbxdata)
+{
+}
 #endif /* !CONFIG_PCI_IOV */
 
 #endif /* _ICE_VIRTCHNL_H_ */
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 274c781b5547..58872a4c2540 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -28,7 +28,6 @@
 #include <linux/tcp.h>
 #include <linux/sctp.h>
 #include <linux/if_ether.h>
-#include <linux/aer.h>
 #include <linux/prefetch.h>
 #include <linux/bpf.h>
 #include <linux/bpf_trace.h>
diff --git a/drivers/net/ethernet/intel/igb/igb_ptp.c b/drivers/net/ethernet/intel/igb/igb_ptp.c
index 6f471b91f562..405886ee5261 100644
--- a/drivers/net/ethernet/intel/igb/igb_ptp.c
+++ b/drivers/net/ethernet/intel/igb/igb_ptp.c
@@ -67,6 +67,7 @@
 #define INCVALUE_82576_MASK		GENMASK(E1000_TIMINCA_16NS_SHIFT - 1, 0)
 #define INCVALUE_82576			(16u << IGB_82576_TSYNC_SHIFT)
 #define IGB_NBITS_82580			40
+#define IGB_82580_BASE_PERIOD		0x800000000
 
 static void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter);
 static void igb_ptp_sdp_init(struct igb_adapter *adapter);
@@ -209,17 +210,11 @@ static int igb_ptp_adjfine_82580(struct ptp_clock_info *ptp, long scaled_ppm)
 	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
 					       ptp_caps);
 	struct e1000_hw *hw = &igb->hw;
-	int neg_adj = 0;
+	bool neg_adj;
 	u64 rate;
 	u32 inca;
 
-	if (scaled_ppm < 0) {
-		neg_adj = 1;
-		scaled_ppm = -scaled_ppm;
-	}
-	rate = scaled_ppm;
-	rate <<= 13;
-	rate = div_u64(rate, 15625);
+	neg_adj = diff_by_scaled_ppm(IGB_82580_BASE_PERIOD, scaled_ppm, &rate);
 
 	inca = rate & INCVALUE_MASK;
 	if (neg_adj)
diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c
index 72cb1b56e9f2..7ff2752dd763 100644
--- a/drivers/net/ethernet/intel/igbvf/netdev.c
+++ b/drivers/net/ethernet/intel/igbvf/netdev.c
@@ -2593,6 +2593,33 @@ static void igbvf_io_resume(struct pci_dev *pdev)
 	netif_device_attach(netdev);
 }
 
+/**
+ * igbvf_io_prepare - prepare device driver for PCI reset
+ * @pdev: PCI device information struct
+ */
+static void igbvf_io_prepare(struct pci_dev *pdev)
+{
+	struct net_device *netdev = pci_get_drvdata(pdev);
+	struct igbvf_adapter *adapter = netdev_priv(netdev);
+
+	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
+		usleep_range(1000, 2000);
+	igbvf_down(adapter);
+}
+
+/**
+ * igbvf_io_reset_done - PCI reset done, device driver reset can begin
+ * @pdev: PCI device information struct
+ */
+static void igbvf_io_reset_done(struct pci_dev *pdev)
+{
+	struct net_device *netdev = pci_get_drvdata(pdev);
+	struct igbvf_adapter *adapter = netdev_priv(netdev);
+
+	igbvf_up(adapter);
+	clear_bit(__IGBVF_RESETTING, &adapter->state);
+}
+
 static void igbvf_print_device_info(struct igbvf_adapter *adapter)
 {
 	struct e1000_hw *hw = &adapter->hw;
@@ -2920,6 +2947,8 @@ static const struct pci_error_handlers igbvf_err_handler = {
 	.error_detected = igbvf_io_error_detected,
 	.slot_reset = igbvf_io_slot_reset,
 	.resume = igbvf_io_resume,
+	.reset_prepare = igbvf_io_prepare,
+	.reset_done = igbvf_io_reset_done,
 };
 
 static const struct pci_device_id igbvf_pci_tbl[] = {
diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h
index df3e26c0cf01..34aebf00a512 100644
--- a/drivers/net/ethernet/intel/igc/igc.h
+++ b/drivers/net/ethernet/intel/igc/igc.h
@@ -99,6 +99,7 @@ struct igc_ring {
 
 	u32 start_time;
 	u32 end_time;
+	u32 max_sdu;
 
 	/* CBS parameters */
 	bool cbs_enable;                /* indicates if CBS is enabled */
@@ -185,6 +186,7 @@ struct igc_adapter {
 	ktime_t base_time;
 	ktime_t cycle_time;
 	bool qbv_enable;
+	u32 qbv_config_change_errors;
 
 	/* OS defined structs */
 	struct pci_dev *pdev;
@@ -292,8 +294,6 @@ extern char igc_driver_name[];
 #define IGC_FLAG_PTP			BIT(8)
 #define IGC_FLAG_WOL_SUPPORTED		BIT(8)
 #define IGC_FLAG_NEED_LINK_UPDATE	BIT(9)
-#define IGC_FLAG_MEDIA_RESET		BIT(10)
-#define IGC_FLAG_MAS_ENABLE		BIT(12)
 #define IGC_FLAG_HAS_MSIX		BIT(13)
 #define IGC_FLAG_EEE			BIT(14)
 #define IGC_FLAG_VLAN_PROMISC		BIT(15)
diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h
index 9dec3563ce3a..44a507029946 100644
--- a/drivers/net/ethernet/intel/igc/igc_defines.h
+++ b/drivers/net/ethernet/intel/igc/igc_defines.h
@@ -662,9 +662,6 @@
  */
 #define IGC_TW_SYSTEM_100_MASK		0x0000FF00
 #define IGC_TW_SYSTEM_100_SHIFT		8
-#define IGC_DMACR_DMAC_EN		0x80000000 /* Enable DMA Coalescing */
-#define IGC_DMACR_DMACTHR_MASK		0x00FF0000
-#define IGC_DMACR_DMACTHR_SHIFT		16
 /* Reg val to set scale to 1024 nsec */
 #define IGC_LTRMINV_SCALE_1024		2
 /* Reg val to set scale to 32768 nsec */
diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c
index 5a26a7805ef8..0e2cb00622d1 100644
--- a/drivers/net/ethernet/intel/igc/igc_ethtool.c
+++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c
@@ -67,6 +67,7 @@ static const struct igc_stats igc_gstrings_stats[] = {
 	IGC_STAT("rx_hwtstamp_cleared", rx_hwtstamp_cleared),
 	IGC_STAT("tx_lpi_counter", stats.tlpic),
 	IGC_STAT("rx_lpi_counter", stats.rlpic),
+	IGC_STAT("qbv_config_change_errors", qbv_config_change_errors),
 };
 
 #define IGC_NETDEV_STAT(_net_stat) { \
diff --git a/drivers/net/ethernet/intel/igc/igc_hw.h b/drivers/net/ethernet/intel/igc/igc_hw.h
index 88680e3d613d..e1c572e0d4ef 100644
--- a/drivers/net/ethernet/intel/igc/igc_hw.h
+++ b/drivers/net/ethernet/intel/igc/igc_hw.h
@@ -273,6 +273,7 @@ struct igc_hw_stats {
 	u64 o2bspc;
 	u64 b2ospc;
 	u64 b2ogprc;
+	u64 txdrop;
 };
 
 struct net_device *igc_get_hw_dev(struct igc_hw *hw);
diff --git a/drivers/net/ethernet/intel/igc/igc_i225.c b/drivers/net/ethernet/intel/igc/igc_i225.c
index 59d5c467ea6e..17546a035ab1 100644
--- a/drivers/net/ethernet/intel/igc/igc_i225.c
+++ b/drivers/net/ethernet/intel/igc/igc_i225.c
@@ -593,20 +593,11 @@ s32 igc_set_ltr_i225(struct igc_hw *hw, bool link)
 		size = rd32(IGC_RXPBS) &
 		       IGC_RXPBS_SIZE_I225_MASK;
 
-		/* Calculations vary based on DMAC settings. */
-		if (rd32(IGC_DMACR) & IGC_DMACR_DMAC_EN) {
-			size -= (rd32(IGC_DMACR) &
-				 IGC_DMACR_DMACTHR_MASK) >>
-				 IGC_DMACR_DMACTHR_SHIFT;
-			/* Convert size to bits. */
-			size *= 1024 * 8;
-		} else {
-			/* Convert size to bytes, subtract the MTU, and then
-			 * convert the size to bits.
-			 */
-			size *= 1024;
-			size *= 8;
-		}
+		/* Convert size to bytes, subtract the MTU, and then
+		 * convert the size to bits.
+		 */
+		size *= 1024;
+		size *= 8;
 
 		if (size < 0) {
 			hw_dbg("Invalid effective Rx buffer size %d\n",
diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c
index 25fc6c65209b..ba49728be919 100644
--- a/drivers/net/ethernet/intel/igc/igc_main.c
+++ b/drivers/net/ethernet/intel/igc/igc_main.c
@@ -4,7 +4,6 @@
 #include <linux/module.h>
 #include <linux/types.h>
 #include <linux/if_vlan.h>
-#include <linux/aer.h>
 #include <linux/tcp.h>
 #include <linux/udp.h>
 #include <linux/ip.h>
@@ -1501,6 +1500,7 @@ static int igc_tso(struct igc_ring *tx_ring,
 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
 				       struct igc_ring *tx_ring)
 {
+	struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
 	bool first_flag = false, insert_empty = false;
 	u16 count = TXD_USE_COUNT(skb_headlen(skb));
 	__be16 protocol = vlan_get_protocol(skb);
@@ -1563,9 +1563,19 @@ done:
 	first->bytecount = skb->len;
 	first->gso_segs = 1;
 
-	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
-		struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
+	if (tx_ring->max_sdu > 0) {
+		u32 max_sdu = 0;
+
+		max_sdu = tx_ring->max_sdu +
+			  (skb_vlan_tagged(first->skb) ? VLAN_HLEN : 0);
+
+		if (first->bytecount > max_sdu) {
+			adapter->stats.txdrop++;
+			goto out_drop;
+		}
+	}
 
+	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
 		/* FIXME: add support for retrieving timestamps from
 		 * the other timer registers before skipping the
 		 * timestamping request.
@@ -4920,7 +4930,8 @@ void igc_update_stats(struct igc_adapter *adapter)
 	net_stats->tx_window_errors = adapter->stats.latecol;
 	net_stats->tx_carrier_errors = adapter->stats.tncrs;
 
-	/* Tx Dropped needs to be maintained elsewhere */
+	/* Tx Dropped */
+	net_stats->tx_dropped = adapter->stats.txdrop;
 
 	/* Management Stats */
 	adapter->stats.mgptc += rd32(IGC_MGTPTC);
@@ -5566,25 +5577,8 @@ no_wait:
 				mod_timer(&adapter->phy_info_timer,
 					  round_jiffies(jiffies + 2 * HZ));
 
-			/* link is down, time to check for alternate media */
-			if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
-				if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
-					schedule_work(&adapter->reset_task);
-					/* return immediately */
-					return;
-				}
-			}
 			pm_schedule_suspend(netdev->dev.parent,
 					    MSEC_PER_SEC * 5);
-
-		/* also check for alternate media here */
-		} else if (!netif_carrier_ok(netdev) &&
-			   (adapter->flags & IGC_FLAG_MAS_ENABLE)) {
-			if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
-				schedule_work(&adapter->reset_task);
-				/* return immediately */
-				return;
-			}
 		}
 	}
 
@@ -6049,12 +6043,14 @@ static int igc_tsn_clear_schedule(struct igc_adapter *adapter)
 
 	adapter->base_time = 0;
 	adapter->cycle_time = NSEC_PER_SEC;
+	adapter->qbv_config_change_errors = 0;
 
 	for (i = 0; i < adapter->num_tx_queues; i++) {
 		struct igc_ring *ring = adapter->tx_ring[i];
 
 		ring->start_time = 0;
 		ring->end_time = NSEC_PER_SEC;
+		ring->max_sdu = 0;
 	}
 
 	return 0;
@@ -6138,6 +6134,16 @@ static int igc_save_qbv_schedule(struct igc_adapter *adapter,
 		}
 	}
 
+	for (i = 0; i < adapter->num_tx_queues; i++) {
+		struct igc_ring *ring = adapter->tx_ring[i];
+		struct net_device *dev = adapter->netdev;
+
+		if (qopt->max_sdu[i])
+			ring->max_sdu = qopt->max_sdu[i] + dev->hard_header_len;
+		else
+			ring->max_sdu = 0;
+	}
+
 	return 0;
 }
 
@@ -6236,8 +6242,10 @@ static int igc_tc_query_caps(struct igc_adapter *adapter,
 
 		caps->broken_mqprio = true;
 
-		if (hw->mac.type == igc_i225)
+		if (hw->mac.type == igc_i225) {
+			caps->supports_queue_max_sdu = true;
 			caps->gate_mask_per_txq = true;
+		}
 
 		return 0;
 	}
diff --git a/drivers/net/ethernet/intel/igc/igc_regs.h b/drivers/net/ethernet/intel/igc/igc_regs.h
index 01c86d36856d..dba5a5759b1c 100644
--- a/drivers/net/ethernet/intel/igc/igc_regs.h
+++ b/drivers/net/ethernet/intel/igc/igc_regs.h
@@ -292,7 +292,6 @@
 
 /* LTR registers */
 #define IGC_LTRC	0x01A0 /* Latency Tolerance Reporting Control */
-#define IGC_DMACR	0x02508 /* DMA Coalescing Control Register */
 #define IGC_LTRMINV	0x5BB0 /* LTR Minimum Value */
 #define IGC_LTRMAXV	0x5BB4 /* LTR Maximum Value */
 
diff --git a/drivers/net/ethernet/intel/igc/igc_tsn.c b/drivers/net/ethernet/intel/igc/igc_tsn.c
index a386c8d61dbf..94a2b0dfb54d 100644
--- a/drivers/net/ethernet/intel/igc/igc_tsn.c
+++ b/drivers/net/ethernet/intel/igc/igc_tsn.c
@@ -114,6 +114,7 @@ static int igc_tsn_disable_offload(struct igc_adapter *adapter)
 static int igc_tsn_enable_offload(struct igc_adapter *adapter)
 {
 	struct igc_hw *hw = &adapter->hw;
+	bool tsn_mode_reconfig = false;
 	u32 tqavctrl, baset_l, baset_h;
 	u32 sec, nsec, cycle;
 	ktime_t base_time, systim;
@@ -226,6 +227,10 @@ skip_cbs:
 	}
 
 	tqavctrl = rd32(IGC_TQAVCTRL) & ~IGC_TQAVCTRL_FUTSCDDIS;
+
+	if (tqavctrl & IGC_TQAVCTRL_TRANSMIT_MODE_TSN)
+		tsn_mode_reconfig = true;
+
 	tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN | IGC_TQAVCTRL_ENHANCED_QAV;
 
 	cycle = adapter->cycle_time;
@@ -239,6 +244,13 @@ skip_cbs:
 		s64 n = div64_s64(ktime_sub_ns(systim, base_time), cycle);
 
 		base_time = ktime_add_ns(base_time, (n + 1) * cycle);
+
+		/* Increase the counter if scheduling into the past while
+		 * Gate Control List (GCL) is running.
+		 */
+		if ((rd32(IGC_BASET_H) || rd32(IGC_BASET_L)) &&
+		    tsn_mode_reconfig)
+			adapter->qbv_config_change_errors++;
 	} else {
 		/* According to datasheet section 7.5.2.9.3.3, FutScdDis bit
 		 * has to be configured before the cycle time and base time.
diff --git a/drivers/net/ethernet/intel/ixgb/Makefile b/drivers/net/ethernet/intel/ixgb/Makefile
deleted file mode 100644
index 2433e9300a33..000000000000
--- a/drivers/net/ethernet/intel/ixgb/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-# Copyright(c) 1999 - 2008 Intel Corporation.
-#
-# Makefile for the Intel(R) PRO/10GbE ethernet driver
-#
-
-obj-$(CONFIG_IXGB) += ixgb.o
-
-ixgb-objs := ixgb_main.o ixgb_hw.o ixgb_ee.o ixgb_ethtool.o ixgb_param.o
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb.h b/drivers/net/ethernet/intel/ixgb/ixgb.h
deleted file mode 100644
index 81ac39576803..000000000000
--- a/drivers/net/ethernet/intel/ixgb/ixgb.h
+++ /dev/null
@@ -1,179 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* Copyright(c) 1999 - 2008 Intel Corporation. */
-
-#ifndef _IXGB_H_
-#define _IXGB_H_
-
-#include <linux/stddef.h>
-#include <linux/module.h>
-#include <linux/types.h>
-#include <asm/byteorder.h>
-#include <linux/mm.h>
-#include <linux/errno.h>
-#include <linux/ioport.h>
-#include <linux/pci.h>
-#include <linux/kernel.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/skbuff.h>
-#include <linux/delay.h>
-#include <linux/timer.h>
-#include <linux/slab.h>
-#include <linux/vmalloc.h>
-#include <linux/interrupt.h>
-#include <linux/string.h>
-#include <linux/pagemap.h>
-#include <linux/dma-mapping.h>
-#include <linux/bitops.h>
-#include <asm/io.h>
-#include <asm/irq.h>
-#include <linux/capability.h>
-#include <linux/in.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/udp.h>
-#include <net/pkt_sched.h>
-#include <linux/list.h>
-#include <linux/reboot.h>
-#include <net/checksum.h>
-
-#include <linux/ethtool.h>
-#include <linux/if_vlan.h>
-
-#define BAR_0		0
-#define BAR_1		1
-
-struct ixgb_adapter;
-#include "ixgb_hw.h"
-#include "ixgb_ee.h"
-#include "ixgb_ids.h"
-
-/* TX/RX descriptor defines */
-#define DEFAULT_TXD      256
-#define MAX_TXD         4096
-#define MIN_TXD           64
-
-/* hardware cannot reliably support more than 512 descriptors owned by
- * hardware descriptor cache otherwise an unreliable ring under heavy
- * receive load may result */
-#define DEFAULT_RXD      512
-#define MAX_RXD          512
-#define MIN_RXD           64
-
-/* Supported Rx Buffer Sizes */
-#define IXGB_RXBUFFER_2048  2048
-#define IXGB_RXBUFFER_4096  4096
-#define IXGB_RXBUFFER_8192  8192
-#define IXGB_RXBUFFER_16384 16384
-
-/* How many Rx Buffers do we bundle into one write to the hardware ? */
-#define IXGB_RX_BUFFER_WRITE	8	/* Must be power of 2 */
-
-/* wrapper around a pointer to a socket buffer,
- * so a DMA handle can be stored along with the buffer */
-struct ixgb_buffer {
-	struct sk_buff *skb;
-	dma_addr_t dma;
-	unsigned long time_stamp;
-	u16 length;
-	u16 next_to_watch;
-	u16 mapped_as_page;
-};
-
-struct ixgb_desc_ring {
-	/* pointer to the descriptor ring memory */
-	void *desc;
-	/* physical address of the descriptor ring */
-	dma_addr_t dma;
-	/* length of descriptor ring in bytes */
-	unsigned int size;
-	/* number of descriptors in the ring */
-	unsigned int count;
-	/* next descriptor to associate a buffer with */
-	unsigned int next_to_use;
-	/* next descriptor to check for DD status bit */
-	unsigned int next_to_clean;
-	/* array of buffer information structs */
-	struct ixgb_buffer *buffer_info;
-};
-
-#define IXGB_DESC_UNUSED(R) \
-	((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->count) + \
-	(R)->next_to_clean - (R)->next_to_use - 1)
-
-#define IXGB_GET_DESC(R, i, type)	(&(((struct type *)((R).desc))[i]))
-#define IXGB_RX_DESC(R, i)		IXGB_GET_DESC(R, i, ixgb_rx_desc)
-#define IXGB_TX_DESC(R, i)		IXGB_GET_DESC(R, i, ixgb_tx_desc)
-#define IXGB_CONTEXT_DESC(R, i)	IXGB_GET_DESC(R, i, ixgb_context_desc)
-
-/* board specific private data structure */
-
-struct ixgb_adapter {
-	struct timer_list watchdog_timer;
-	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
-	u32 bd_number;
-	u32 rx_buffer_len;
-	u32 part_num;
-	u16 link_speed;
-	u16 link_duplex;
-	struct work_struct tx_timeout_task;
-
-	/* TX */
-	struct ixgb_desc_ring tx_ring ____cacheline_aligned_in_smp;
-	unsigned int restart_queue;
-	unsigned long timeo_start;
-	u32 tx_cmd_type;
-	u64 hw_csum_tx_good;
-	u64 hw_csum_tx_error;
-	u32 tx_int_delay;
-	u32 tx_timeout_count;
-	bool tx_int_delay_enable;
-	bool detect_tx_hung;
-
-	/* RX */
-	struct ixgb_desc_ring rx_ring;
-	u64 hw_csum_rx_error;
-	u64 hw_csum_rx_good;
-	u32 rx_int_delay;
-	bool rx_csum;
-
-	/* OS defined structs */
-	struct napi_struct napi;
-	struct net_device *netdev;
-	struct pci_dev *pdev;
-
-	/* structs defined in ixgb_hw.h */
-	struct ixgb_hw hw;
-	u16 msg_enable;
-	struct ixgb_hw_stats stats;
-	u32 alloc_rx_buff_failed;
-	bool have_msi;
-	unsigned long flags;
-};
-
-enum ixgb_state_t {
-	/* TBD
-	__IXGB_TESTING,
-	__IXGB_RESETTING,
-	*/
-	__IXGB_DOWN
-};
-
-/* Exported from other modules */
-void ixgb_check_options(struct ixgb_adapter *adapter);
-void ixgb_set_ethtool_ops(struct net_device *netdev);
-extern char ixgb_driver_name[];
-
-void ixgb_set_speed_duplex(struct net_device *netdev);
-
-int ixgb_up(struct ixgb_adapter *adapter);
-void ixgb_down(struct ixgb_adapter *adapter, bool kill_watchdog);
-void ixgb_reset(struct ixgb_adapter *adapter);
-int ixgb_setup_rx_resources(struct ixgb_adapter *adapter);
-int ixgb_setup_tx_resources(struct ixgb_adapter *adapter);
-void ixgb_free_rx_resources(struct ixgb_adapter *adapter);
-void ixgb_free_tx_resources(struct ixgb_adapter *adapter);
-void ixgb_update_stats(struct ixgb_adapter *adapter);
-
-
-#endif /* _IXGB_H_ */
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ee.c b/drivers/net/ethernet/intel/ixgb/ixgb_ee.c
deleted file mode 100644
index 129286fc1634..000000000000
--- a/drivers/net/ethernet/intel/ixgb/ixgb_ee.c
+++ /dev/null
@@ -1,580 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright(c) 1999 - 2008 Intel Corporation. */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include "ixgb_hw.h"
-#include "ixgb_ee.h"
-/* Local prototypes */
-static u16 ixgb_shift_in_bits(struct ixgb_hw *hw);
-
-static void ixgb_shift_out_bits(struct ixgb_hw *hw,
-				u16 data,
-				u16 count);
-static void ixgb_standby_eeprom(struct ixgb_hw *hw);
-
-static bool ixgb_wait_eeprom_command(struct ixgb_hw *hw);
-
-static void ixgb_cleanup_eeprom(struct ixgb_hw *hw);
-
-/******************************************************************************
- * Raises the EEPROM's clock input.
- *
- * hw - Struct containing variables accessed by shared code
- * eecd_reg - EECD's current value
- *****************************************************************************/
-static void
-ixgb_raise_clock(struct ixgb_hw *hw,
-		  u32 *eecd_reg)
-{
-	/* Raise the clock input to the EEPROM (by setting the SK bit), and then
-	 *  wait 50 microseconds.
-	 */
-	*eecd_reg = *eecd_reg | IXGB_EECD_SK;
-	IXGB_WRITE_REG(hw, EECD, *eecd_reg);
-	IXGB_WRITE_FLUSH(hw);
-	udelay(50);
-}
-
-/******************************************************************************
- * Lowers the EEPROM's clock input.
- *
- * hw - Struct containing variables accessed by shared code
- * eecd_reg - EECD's current value
- *****************************************************************************/
-static void
-ixgb_lower_clock(struct ixgb_hw *hw,
-		  u32 *eecd_reg)
-{
-	/* Lower the clock input to the EEPROM (by clearing the SK bit), and then
-	 * wait 50 microseconds.
-	 */
-	*eecd_reg = *eecd_reg & ~IXGB_EECD_SK;
-	IXGB_WRITE_REG(hw, EECD, *eecd_reg);
-	IXGB_WRITE_FLUSH(hw);
-	udelay(50);
-}
-
-/******************************************************************************
- * Shift data bits out to the EEPROM.
- *
- * hw - Struct containing variables accessed by shared code
- * data - data to send to the EEPROM
- * count - number of bits to shift out
- *****************************************************************************/
-static void
-ixgb_shift_out_bits(struct ixgb_hw *hw,
-					 u16 data,
-					 u16 count)
-{
-	u32 eecd_reg;
-	u32 mask;
-
-	/* We need to shift "count" bits out to the EEPROM. So, value in the
-	 * "data" parameter will be shifted out to the EEPROM one bit at a time.
-	 * In order to do this, "data" must be broken down into bits.
-	 */
-	mask = 0x01 << (count - 1);
-	eecd_reg = IXGB_READ_REG(hw, EECD);
-	eecd_reg &= ~(IXGB_EECD_DO | IXGB_EECD_DI);
-	do {
-		/* A "1" is shifted out to the EEPROM by setting bit "DI" to a "1",
-		 * and then raising and then lowering the clock (the SK bit controls
-		 * the clock input to the EEPROM).  A "0" is shifted out to the EEPROM
-		 * by setting "DI" to "0" and then raising and then lowering the clock.
-		 */
-		eecd_reg &= ~IXGB_EECD_DI;
-
-		if (data & mask)
-			eecd_reg |= IXGB_EECD_DI;
-
-		IXGB_WRITE_REG(hw, EECD, eecd_reg);
-		IXGB_WRITE_FLUSH(hw);
-
-		udelay(50);
-
-		ixgb_raise_clock(hw, &eecd_reg);
-		ixgb_lower_clock(hw, &eecd_reg);
-
-		mask = mask >> 1;
-
-	} while (mask);
-
-	/* We leave the "DI" bit set to "0" when we leave this routine. */
-	eecd_reg &= ~IXGB_EECD_DI;
-	IXGB_WRITE_REG(hw, EECD, eecd_reg);
-}
-
-/******************************************************************************
- * Shift data bits in from the EEPROM
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-static u16
-ixgb_shift_in_bits(struct ixgb_hw *hw)
-{
-	u32 eecd_reg;
-	u32 i;
-	u16 data;
-
-	/* In order to read a register from the EEPROM, we need to shift 16 bits
-	 * in from the EEPROM. Bits are "shifted in" by raising the clock input to
-	 * the EEPROM (setting the SK bit), and then reading the value of the "DO"
-	 * bit.  During this "shifting in" process the "DI" bit should always be
-	 * clear..
-	 */
-
-	eecd_reg = IXGB_READ_REG(hw, EECD);
-
-	eecd_reg &= ~(IXGB_EECD_DO | IXGB_EECD_DI);
-	data = 0;
-
-	for (i = 0; i < 16; i++) {
-		data = data << 1;
-		ixgb_raise_clock(hw, &eecd_reg);
-
-		eecd_reg = IXGB_READ_REG(hw, EECD);
-
-		eecd_reg &= ~(IXGB_EECD_DI);
-		if (eecd_reg & IXGB_EECD_DO)
-			data |= 1;
-
-		ixgb_lower_clock(hw, &eecd_reg);
-	}
-
-	return data;
-}
-
-/******************************************************************************
- * Prepares EEPROM for access
- *
- * hw - Struct containing variables accessed by shared code
- *
- * Lowers EEPROM clock. Clears input pin. Sets the chip select pin. This
- * function should be called before issuing a command to the EEPROM.
- *****************************************************************************/
-static void
-ixgb_setup_eeprom(struct ixgb_hw *hw)
-{
-	u32 eecd_reg;
-
-	eecd_reg = IXGB_READ_REG(hw, EECD);
-
-	/*  Clear SK and DI  */
-	eecd_reg &= ~(IXGB_EECD_SK | IXGB_EECD_DI);
-	IXGB_WRITE_REG(hw, EECD, eecd_reg);
-
-	/*  Set CS  */
-	eecd_reg |= IXGB_EECD_CS;
-	IXGB_WRITE_REG(hw, EECD, eecd_reg);
-}
-
-/******************************************************************************
- * Returns EEPROM to a "standby" state
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-static void
-ixgb_standby_eeprom(struct ixgb_hw *hw)
-{
-	u32 eecd_reg;
-
-	eecd_reg = IXGB_READ_REG(hw, EECD);
-
-	/*  Deselect EEPROM  */
-	eecd_reg &= ~(IXGB_EECD_CS | IXGB_EECD_SK);
-	IXGB_WRITE_REG(hw, EECD, eecd_reg);
-	IXGB_WRITE_FLUSH(hw);
-	udelay(50);
-
-	/*  Clock high  */
-	eecd_reg |= IXGB_EECD_SK;
-	IXGB_WRITE_REG(hw, EECD, eecd_reg);
-	IXGB_WRITE_FLUSH(hw);
-	udelay(50);
-
-	/*  Select EEPROM  */
-	eecd_reg |= IXGB_EECD_CS;
-	IXGB_WRITE_REG(hw, EECD, eecd_reg);
-	IXGB_WRITE_FLUSH(hw);
-	udelay(50);
-
-	/*  Clock low  */
-	eecd_reg &= ~IXGB_EECD_SK;
-	IXGB_WRITE_REG(hw, EECD, eecd_reg);
-	IXGB_WRITE_FLUSH(hw);
-	udelay(50);
-}
-
-/******************************************************************************
- * Raises then lowers the EEPROM's clock pin
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-static void
-ixgb_clock_eeprom(struct ixgb_hw *hw)
-{
-	u32 eecd_reg;
-
-	eecd_reg = IXGB_READ_REG(hw, EECD);
-
-	/*  Rising edge of clock  */
-	eecd_reg |= IXGB_EECD_SK;
-	IXGB_WRITE_REG(hw, EECD, eecd_reg);
-	IXGB_WRITE_FLUSH(hw);
-	udelay(50);
-
-	/*  Falling edge of clock  */
-	eecd_reg &= ~IXGB_EECD_SK;
-	IXGB_WRITE_REG(hw, EECD, eecd_reg);
-	IXGB_WRITE_FLUSH(hw);
-	udelay(50);
-}
-
-/******************************************************************************
- * Terminates a command by lowering the EEPROM's chip select pin
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-static void
-ixgb_cleanup_eeprom(struct ixgb_hw *hw)
-{
-	u32 eecd_reg;
-
-	eecd_reg = IXGB_READ_REG(hw, EECD);
-
-	eecd_reg &= ~(IXGB_EECD_CS | IXGB_EECD_DI);
-
-	IXGB_WRITE_REG(hw, EECD, eecd_reg);
-
-	ixgb_clock_eeprom(hw);
-}
-
-/******************************************************************************
- * Waits for the EEPROM to finish the current command.
- *
- * hw - Struct containing variables accessed by shared code
- *
- * The command is done when the EEPROM's data out pin goes high.
- *
- * Returns:
- *      true: EEPROM data pin is high before timeout.
- *      false:  Time expired.
- *****************************************************************************/
-static bool
-ixgb_wait_eeprom_command(struct ixgb_hw *hw)
-{
-	u32 eecd_reg;
-	u32 i;
-
-	/* Toggle the CS line.  This in effect tells to EEPROM to actually execute
-	 * the command in question.
-	 */
-	ixgb_standby_eeprom(hw);
-
-	/* Now read DO repeatedly until is high (equal to '1').  The EEPROM will
-	 * signal that the command has been completed by raising the DO signal.
-	 * If DO does not go high in 10 milliseconds, then error out.
-	 */
-	for (i = 0; i < 200; i++) {
-		eecd_reg = IXGB_READ_REG(hw, EECD);
-
-		if (eecd_reg & IXGB_EECD_DO)
-			return true;
-
-		udelay(50);
-	}
-	ASSERT(0);
-	return false;
-}
-
-/******************************************************************************
- * Verifies that the EEPROM has a valid checksum
- *
- * hw - Struct containing variables accessed by shared code
- *
- * Reads the first 64 16 bit words of the EEPROM and sums the values read.
- * If the sum of the 64 16 bit words is 0xBABA, the EEPROM's checksum is
- * valid.
- *
- * Returns:
- *  true: Checksum is valid
- *  false: Checksum is not valid.
- *****************************************************************************/
-bool
-ixgb_validate_eeprom_checksum(struct ixgb_hw *hw)
-{
-	u16 checksum = 0;
-	u16 i;
-
-	for (i = 0; i < (EEPROM_CHECKSUM_REG + 1); i++)
-		checksum += ixgb_read_eeprom(hw, i);
-
-	if (checksum == (u16) EEPROM_SUM)
-		return true;
-	else
-		return false;
-}
-
-/******************************************************************************
- * Calculates the EEPROM checksum and writes it to the EEPROM
- *
- * hw - Struct containing variables accessed by shared code
- *
- * Sums the first 63 16 bit words of the EEPROM. Subtracts the sum from 0xBABA.
- * Writes the difference to word offset 63 of the EEPROM.
- *****************************************************************************/
-void
-ixgb_update_eeprom_checksum(struct ixgb_hw *hw)
-{
-	u16 checksum = 0;
-	u16 i;
-
-	for (i = 0; i < EEPROM_CHECKSUM_REG; i++)
-		checksum += ixgb_read_eeprom(hw, i);
-
-	checksum = (u16) EEPROM_SUM - checksum;
-
-	ixgb_write_eeprom(hw, EEPROM_CHECKSUM_REG, checksum);
-}
-
-/******************************************************************************
- * Writes a 16 bit word to a given offset in the EEPROM.
- *
- * hw - Struct containing variables accessed by shared code
- * reg - offset within the EEPROM to be written to
- * data - 16 bit word to be written to the EEPROM
- *
- * If ixgb_update_eeprom_checksum is not called after this function, the
- * EEPROM will most likely contain an invalid checksum.
- *
- *****************************************************************************/
-void
-ixgb_write_eeprom(struct ixgb_hw *hw, u16 offset, u16 data)
-{
-	struct ixgb_ee_map_type *ee_map = (struct ixgb_ee_map_type *)hw->eeprom;
-
-	/* Prepare the EEPROM for writing */
-	ixgb_setup_eeprom(hw);
-
-	/*  Send the 9-bit EWEN (write enable) command to the EEPROM (5-bit opcode
-	 *  plus 4-bit dummy).  This puts the EEPROM into write/erase mode.
-	 */
-	ixgb_shift_out_bits(hw, EEPROM_EWEN_OPCODE, 5);
-	ixgb_shift_out_bits(hw, 0, 4);
-
-	/*  Prepare the EEPROM  */
-	ixgb_standby_eeprom(hw);
-
-	/*  Send the Write command (3-bit opcode + 6-bit addr)  */
-	ixgb_shift_out_bits(hw, EEPROM_WRITE_OPCODE, 3);
-	ixgb_shift_out_bits(hw, offset, 6);
-
-	/*  Send the data  */
-	ixgb_shift_out_bits(hw, data, 16);
-
-	ixgb_wait_eeprom_command(hw);
-
-	/*  Recover from write  */
-	ixgb_standby_eeprom(hw);
-
-	/* Send the 9-bit EWDS (write disable) command to the EEPROM (5-bit
-	 * opcode plus 4-bit dummy).  This takes the EEPROM out of write/erase
-	 * mode.
-	 */
-	ixgb_shift_out_bits(hw, EEPROM_EWDS_OPCODE, 5);
-	ixgb_shift_out_bits(hw, 0, 4);
-
-	/*  Done with writing  */
-	ixgb_cleanup_eeprom(hw);
-
-	/* clear the init_ctrl_reg_1 to signify that the cache is invalidated */
-	ee_map->init_ctrl_reg_1 = cpu_to_le16(EEPROM_ICW1_SIGNATURE_CLEAR);
-}
-
-/******************************************************************************
- * Reads a 16 bit word from the EEPROM.
- *
- * hw - Struct containing variables accessed by shared code
- * offset - offset of 16 bit word in the EEPROM to read
- *
- * Returns:
- *  The 16-bit value read from the eeprom
- *****************************************************************************/
-u16
-ixgb_read_eeprom(struct ixgb_hw *hw,
-		  u16 offset)
-{
-	u16 data;
-
-	/*  Prepare the EEPROM for reading  */
-	ixgb_setup_eeprom(hw);
-
-	/*  Send the READ command (opcode + addr)  */
-	ixgb_shift_out_bits(hw, EEPROM_READ_OPCODE, 3);
-	/*
-	 * We have a 64 word EEPROM, there are 6 address bits
-	 */
-	ixgb_shift_out_bits(hw, offset, 6);
-
-	/*  Read the data  */
-	data = ixgb_shift_in_bits(hw);
-
-	/*  End this read operation  */
-	ixgb_standby_eeprom(hw);
-
-	return data;
-}
-
-/******************************************************************************
- * Reads eeprom and stores data in shared structure.
- * Validates eeprom checksum and eeprom signature.
- *
- * hw - Struct containing variables accessed by shared code
- *
- * Returns:
- *      true: if eeprom read is successful
- *      false: otherwise.
- *****************************************************************************/
-bool
-ixgb_get_eeprom_data(struct ixgb_hw *hw)
-{
-	u16 i;
-	u16 checksum = 0;
-	struct ixgb_ee_map_type *ee_map;
-
-	ENTER();
-
-	ee_map = (struct ixgb_ee_map_type *)hw->eeprom;
-
-	pr_debug("Reading eeprom data\n");
-	for (i = 0; i < IXGB_EEPROM_SIZE ; i++) {
-		u16 ee_data;
-		ee_data = ixgb_read_eeprom(hw, i);
-		checksum += ee_data;
-		hw->eeprom[i] = cpu_to_le16(ee_data);
-	}
-
-	if (checksum != (u16) EEPROM_SUM) {
-		pr_debug("Checksum invalid\n");
-		/* clear the init_ctrl_reg_1 to signify that the cache is
-		 * invalidated */
-		ee_map->init_ctrl_reg_1 = cpu_to_le16(EEPROM_ICW1_SIGNATURE_CLEAR);
-		return false;
-	}
-
-	if ((ee_map->init_ctrl_reg_1 & cpu_to_le16(EEPROM_ICW1_SIGNATURE_MASK))
-		 != cpu_to_le16(EEPROM_ICW1_SIGNATURE_VALID)) {
-		pr_debug("Signature invalid\n");
-		return false;
-	}
-
-	return true;
-}
-
-/******************************************************************************
- * Local function to check if the eeprom signature is good
- * If the eeprom signature is good, calls ixgb)get_eeprom_data.
- *
- * hw - Struct containing variables accessed by shared code
- *
- * Returns:
- *      true: eeprom signature was good and the eeprom read was successful
- *      false: otherwise.
- ******************************************************************************/
-static bool
-ixgb_check_and_get_eeprom_data (struct ixgb_hw* hw)
-{
-	struct ixgb_ee_map_type *ee_map = (struct ixgb_ee_map_type *)hw->eeprom;
-
-	if ((ee_map->init_ctrl_reg_1 & cpu_to_le16(EEPROM_ICW1_SIGNATURE_MASK))
-	    == cpu_to_le16(EEPROM_ICW1_SIGNATURE_VALID)) {
-		return true;
-	} else {
-		return ixgb_get_eeprom_data(hw);
-	}
-}
-
-/******************************************************************************
- * return a word from the eeprom
- *
- * hw - Struct containing variables accessed by shared code
- * index - Offset of eeprom word
- *
- * Returns:
- *          Word at indexed offset in eeprom, if valid, 0 otherwise.
- ******************************************************************************/
-__le16
-ixgb_get_eeprom_word(struct ixgb_hw *hw, u16 index)
-{
-
-	if (index < IXGB_EEPROM_SIZE && ixgb_check_and_get_eeprom_data(hw))
-		return hw->eeprom[index];
-
-	return 0;
-}
-
-/******************************************************************************
- * return the mac address from EEPROM
- *
- * hw       - Struct containing variables accessed by shared code
- * mac_addr - Ethernet Address if EEPROM contents are valid, 0 otherwise
- *
- * Returns: None.
- ******************************************************************************/
-void
-ixgb_get_ee_mac_addr(struct ixgb_hw *hw,
-			u8 *mac_addr)
-{
-	int i;
-	struct ixgb_ee_map_type *ee_map = (struct ixgb_ee_map_type *)hw->eeprom;
-
-	ENTER();
-
-	if (ixgb_check_and_get_eeprom_data(hw)) {
-		for (i = 0; i < ETH_ALEN; i++) {
-			mac_addr[i] = ee_map->mac_addr[i];
-		}
-		pr_debug("eeprom mac address = %pM\n", mac_addr);
-	}
-}
-
-
-/******************************************************************************
- * return the Printed Board Assembly number from EEPROM
- *
- * hw - Struct containing variables accessed by shared code
- *
- * Returns:
- *          PBA number if EEPROM contents are valid, 0 otherwise
- ******************************************************************************/
-u32
-ixgb_get_ee_pba_number(struct ixgb_hw *hw)
-{
-	if (ixgb_check_and_get_eeprom_data(hw))
-		return le16_to_cpu(hw->eeprom[EEPROM_PBA_1_2_REG])
-			| (le16_to_cpu(hw->eeprom[EEPROM_PBA_3_4_REG])<<16);
-
-	return 0;
-}
-
-
-/******************************************************************************
- * return the Device Id from EEPROM
- *
- * hw - Struct containing variables accessed by shared code
- *
- * Returns:
- *          Device Id if EEPROM contents are valid, 0 otherwise
- ******************************************************************************/
-u16
-ixgb_get_ee_device_id(struct ixgb_hw *hw)
-{
-	struct ixgb_ee_map_type *ee_map = (struct ixgb_ee_map_type *)hw->eeprom;
-
-	if (ixgb_check_and_get_eeprom_data(hw))
-		return le16_to_cpu(ee_map->device_id);
-
-	return 0;
-}
-
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ee.h b/drivers/net/ethernet/intel/ixgb/ixgb_ee.h
deleted file mode 100644
index 3ee0a09e5d0a..000000000000
--- a/drivers/net/ethernet/intel/ixgb/ixgb_ee.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* Copyright(c) 1999 - 2008 Intel Corporation. */
-
-#ifndef _IXGB_EE_H_
-#define _IXGB_EE_H_
-
-#define IXGB_EEPROM_SIZE    64	/* Size in words */
-
-/* EEPROM Commands */
-#define EEPROM_READ_OPCODE  0x6	/* EEPROM read opcode */
-#define EEPROM_WRITE_OPCODE 0x5	/* EEPROM write opcode */
-#define EEPROM_ERASE_OPCODE 0x7	/* EEPROM erase opcode */
-#define EEPROM_EWEN_OPCODE  0x13	/* EEPROM erase/write enable */
-#define EEPROM_EWDS_OPCODE  0x10	/* EEPROM erase/write disable */
-
-/* EEPROM MAP (Word Offsets) */
-#define EEPROM_IA_1_2_REG        0x0000
-#define EEPROM_IA_3_4_REG        0x0001
-#define EEPROM_IA_5_6_REG        0x0002
-#define EEPROM_COMPATIBILITY_REG 0x0003
-#define EEPROM_PBA_1_2_REG       0x0008
-#define EEPROM_PBA_3_4_REG       0x0009
-#define EEPROM_INIT_CONTROL1_REG 0x000A
-#define EEPROM_SUBSYS_ID_REG     0x000B
-#define EEPROM_SUBVEND_ID_REG    0x000C
-#define EEPROM_DEVICE_ID_REG     0x000D
-#define EEPROM_VENDOR_ID_REG     0x000E
-#define EEPROM_INIT_CONTROL2_REG 0x000F
-#define EEPROM_SWDPINS_REG       0x0020
-#define EEPROM_CIRCUIT_CTRL_REG  0x0021
-#define EEPROM_D0_D3_POWER_REG   0x0022
-#define EEPROM_FLASH_VERSION     0x0032
-#define EEPROM_CHECKSUM_REG      0x003F
-
-/* Mask bits for fields in Word 0x0a of the EEPROM */
-
-#define EEPROM_ICW1_SIGNATURE_MASK  0xC000
-#define EEPROM_ICW1_SIGNATURE_VALID 0x4000
-#define EEPROM_ICW1_SIGNATURE_CLEAR 0x0000
-
-/* For checksumming, the sum of all words in the EEPROM should equal 0xBABA. */
-#define EEPROM_SUM 0xBABA
-
-/* EEPROM Map Sizes (Byte Counts) */
-#define PBA_SIZE 4
-
-/* EEPROM Map defines (WORD OFFSETS)*/
-
-/* EEPROM structure */
-struct ixgb_ee_map_type {
-	u8 mac_addr[ETH_ALEN];
-	__le16 compatibility;
-	__le16 reserved1[4];
-	__le32 pba_number;
-	__le16 init_ctrl_reg_1;
-	__le16 subsystem_id;
-	__le16 subvendor_id;
-	__le16 device_id;
-	__le16 vendor_id;
-	__le16 init_ctrl_reg_2;
-	__le16 oem_reserved[16];
-	__le16 swdpins_reg;
-	__le16 circuit_ctrl_reg;
-	u8 d3_power;
-	u8 d0_power;
-	__le16 reserved2[28];
-	__le16 checksum;
-};
-
-/* EEPROM Functions */
-u16 ixgb_read_eeprom(struct ixgb_hw *hw, u16 reg);
-
-bool ixgb_validate_eeprom_checksum(struct ixgb_hw *hw);
-
-void ixgb_update_eeprom_checksum(struct ixgb_hw *hw);
-
-void ixgb_write_eeprom(struct ixgb_hw *hw, u16 reg, u16 data);
-
-#endif				/* IXGB_EE_H */
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c
deleted file mode 100644
index efa980514944..000000000000
--- a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c
+++ /dev/null
@@ -1,642 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright(c) 1999 - 2008 Intel Corporation. */
-
-/* ethtool support for ixgb */
-
-#include "ixgb.h"
-
-#include <linux/uaccess.h>
-
-#define IXGB_ALL_RAR_ENTRIES 16
-
-enum {NETDEV_STATS, IXGB_STATS};
-
-struct ixgb_stats {
-	char stat_string[ETH_GSTRING_LEN];
-	int type;
-	int sizeof_stat;
-	int stat_offset;
-};
-
-#define IXGB_STAT(m)		IXGB_STATS, \
-				sizeof_field(struct ixgb_adapter, m), \
-				offsetof(struct ixgb_adapter, m)
-#define IXGB_NETDEV_STAT(m)	NETDEV_STATS, \
-				sizeof_field(struct net_device, m), \
-				offsetof(struct net_device, m)
-
-static struct ixgb_stats ixgb_gstrings_stats[] = {
-	{"rx_packets", IXGB_NETDEV_STAT(stats.rx_packets)},
-	{"tx_packets", IXGB_NETDEV_STAT(stats.tx_packets)},
-	{"rx_bytes", IXGB_NETDEV_STAT(stats.rx_bytes)},
-	{"tx_bytes", IXGB_NETDEV_STAT(stats.tx_bytes)},
-	{"rx_errors", IXGB_NETDEV_STAT(stats.rx_errors)},
-	{"tx_errors", IXGB_NETDEV_STAT(stats.tx_errors)},
-	{"rx_dropped", IXGB_NETDEV_STAT(stats.rx_dropped)},
-	{"tx_dropped", IXGB_NETDEV_STAT(stats.tx_dropped)},
-	{"multicast", IXGB_NETDEV_STAT(stats.multicast)},
-	{"collisions", IXGB_NETDEV_STAT(stats.collisions)},
-
-/*	{ "rx_length_errors", IXGB_NETDEV_STAT(stats.rx_length_errors) },	*/
-	{"rx_over_errors", IXGB_NETDEV_STAT(stats.rx_over_errors)},
-	{"rx_crc_errors", IXGB_NETDEV_STAT(stats.rx_crc_errors)},
-	{"rx_frame_errors", IXGB_NETDEV_STAT(stats.rx_frame_errors)},
-	{"rx_no_buffer_count", IXGB_STAT(stats.rnbc)},
-	{"rx_fifo_errors", IXGB_NETDEV_STAT(stats.rx_fifo_errors)},
-	{"rx_missed_errors", IXGB_NETDEV_STAT(stats.rx_missed_errors)},
-	{"tx_aborted_errors", IXGB_NETDEV_STAT(stats.tx_aborted_errors)},
-	{"tx_carrier_errors", IXGB_NETDEV_STAT(stats.tx_carrier_errors)},
-	{"tx_fifo_errors", IXGB_NETDEV_STAT(stats.tx_fifo_errors)},
-	{"tx_heartbeat_errors", IXGB_NETDEV_STAT(stats.tx_heartbeat_errors)},
-	{"tx_window_errors", IXGB_NETDEV_STAT(stats.tx_window_errors)},
-	{"tx_deferred_ok", IXGB_STAT(stats.dc)},
-	{"tx_timeout_count", IXGB_STAT(tx_timeout_count) },
-	{"tx_restart_queue", IXGB_STAT(restart_queue) },
-	{"rx_long_length_errors", IXGB_STAT(stats.roc)},
-	{"rx_short_length_errors", IXGB_STAT(stats.ruc)},
-	{"tx_tcp_seg_good", IXGB_STAT(stats.tsctc)},
-	{"tx_tcp_seg_failed", IXGB_STAT(stats.tsctfc)},
-	{"rx_flow_control_xon", IXGB_STAT(stats.xonrxc)},
-	{"rx_flow_control_xoff", IXGB_STAT(stats.xoffrxc)},
-	{"tx_flow_control_xon", IXGB_STAT(stats.xontxc)},
-	{"tx_flow_control_xoff", IXGB_STAT(stats.xofftxc)},
-	{"rx_csum_offload_good", IXGB_STAT(hw_csum_rx_good)},
-	{"rx_csum_offload_errors", IXGB_STAT(hw_csum_rx_error)},
-	{"tx_csum_offload_good", IXGB_STAT(hw_csum_tx_good)},
-	{"tx_csum_offload_errors", IXGB_STAT(hw_csum_tx_error)}
-};
-
-#define IXGB_STATS_LEN	ARRAY_SIZE(ixgb_gstrings_stats)
-
-static int
-ixgb_get_link_ksettings(struct net_device *netdev,
-			struct ethtool_link_ksettings *cmd)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-
-	ethtool_link_ksettings_zero_link_mode(cmd, supported);
-	ethtool_link_ksettings_add_link_mode(cmd, supported, 10000baseT_Full);
-	ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE);
-
-	ethtool_link_ksettings_zero_link_mode(cmd, advertising);
-	ethtool_link_ksettings_add_link_mode(cmd, advertising, 10000baseT_Full);
-	ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE);
-
-	cmd->base.port = PORT_FIBRE;
-
-	if (netif_carrier_ok(adapter->netdev)) {
-		cmd->base.speed = SPEED_10000;
-		cmd->base.duplex = DUPLEX_FULL;
-	} else {
-		cmd->base.speed = SPEED_UNKNOWN;
-		cmd->base.duplex = DUPLEX_UNKNOWN;
-	}
-
-	cmd->base.autoneg = AUTONEG_DISABLE;
-	return 0;
-}
-
-void ixgb_set_speed_duplex(struct net_device *netdev)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	/* be optimistic about our link, since we were up before */
-	adapter->link_speed = 10000;
-	adapter->link_duplex = FULL_DUPLEX;
-	netif_carrier_on(netdev);
-	netif_wake_queue(netdev);
-}
-
-static int
-ixgb_set_link_ksettings(struct net_device *netdev,
-			const struct ethtool_link_ksettings *cmd)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	u32 speed = cmd->base.speed;
-
-	if (cmd->base.autoneg == AUTONEG_ENABLE ||
-	    (speed + cmd->base.duplex != SPEED_10000 + DUPLEX_FULL))
-		return -EINVAL;
-
-	if (netif_running(adapter->netdev)) {
-		ixgb_down(adapter, true);
-		ixgb_reset(adapter);
-		ixgb_up(adapter);
-		ixgb_set_speed_duplex(netdev);
-	} else
-		ixgb_reset(adapter);
-
-	return 0;
-}
-
-static void
-ixgb_get_pauseparam(struct net_device *netdev,
-			 struct ethtool_pauseparam *pause)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct ixgb_hw *hw = &adapter->hw;
-
-	pause->autoneg = AUTONEG_DISABLE;
-
-	if (hw->fc.type == ixgb_fc_rx_pause)
-		pause->rx_pause = 1;
-	else if (hw->fc.type == ixgb_fc_tx_pause)
-		pause->tx_pause = 1;
-	else if (hw->fc.type == ixgb_fc_full) {
-		pause->rx_pause = 1;
-		pause->tx_pause = 1;
-	}
-}
-
-static int
-ixgb_set_pauseparam(struct net_device *netdev,
-			 struct ethtool_pauseparam *pause)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct ixgb_hw *hw = &adapter->hw;
-
-	if (pause->autoneg == AUTONEG_ENABLE)
-		return -EINVAL;
-
-	if (pause->rx_pause && pause->tx_pause)
-		hw->fc.type = ixgb_fc_full;
-	else if (pause->rx_pause && !pause->tx_pause)
-		hw->fc.type = ixgb_fc_rx_pause;
-	else if (!pause->rx_pause && pause->tx_pause)
-		hw->fc.type = ixgb_fc_tx_pause;
-	else if (!pause->rx_pause && !pause->tx_pause)
-		hw->fc.type = ixgb_fc_none;
-
-	if (netif_running(adapter->netdev)) {
-		ixgb_down(adapter, true);
-		ixgb_up(adapter);
-		ixgb_set_speed_duplex(netdev);
-	} else
-		ixgb_reset(adapter);
-
-	return 0;
-}
-
-static u32
-ixgb_get_msglevel(struct net_device *netdev)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	return adapter->msg_enable;
-}
-
-static void
-ixgb_set_msglevel(struct net_device *netdev, u32 data)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	adapter->msg_enable = data;
-}
-#define IXGB_GET_STAT(_A_, _R_) _A_->stats._R_
-
-static int
-ixgb_get_regs_len(struct net_device *netdev)
-{
-#define IXGB_REG_DUMP_LEN  136*sizeof(u32)
-	return IXGB_REG_DUMP_LEN;
-}
-
-static void
-ixgb_get_regs(struct net_device *netdev,
-		   struct ethtool_regs *regs, void *p)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct ixgb_hw *hw = &adapter->hw;
-	u32 *reg = p;
-	u32 *reg_start = reg;
-	u8 i;
-
-	/* the 1 (one) below indicates an attempt at versioning, if the
-	 * interface in ethtool or the driver changes, this 1 should be
-	 * incremented */
-	regs->version = (1<<24) | hw->revision_id << 16 | hw->device_id;
-
-	/* General Registers */
-	*reg++ = IXGB_READ_REG(hw, CTRL0);	/*   0 */
-	*reg++ = IXGB_READ_REG(hw, CTRL1);	/*   1 */
-	*reg++ = IXGB_READ_REG(hw, STATUS);	/*   2 */
-	*reg++ = IXGB_READ_REG(hw, EECD);	/*   3 */
-	*reg++ = IXGB_READ_REG(hw, MFS);	/*   4 */
-
-	/* Interrupt */
-	*reg++ = IXGB_READ_REG(hw, ICR);	/*   5 */
-	*reg++ = IXGB_READ_REG(hw, ICS);	/*   6 */
-	*reg++ = IXGB_READ_REG(hw, IMS);	/*   7 */
-	*reg++ = IXGB_READ_REG(hw, IMC);	/*   8 */
-
-	/* Receive */
-	*reg++ = IXGB_READ_REG(hw, RCTL);	/*   9 */
-	*reg++ = IXGB_READ_REG(hw, FCRTL);	/*  10 */
-	*reg++ = IXGB_READ_REG(hw, FCRTH);	/*  11 */
-	*reg++ = IXGB_READ_REG(hw, RDBAL);	/*  12 */
-	*reg++ = IXGB_READ_REG(hw, RDBAH);	/*  13 */
-	*reg++ = IXGB_READ_REG(hw, RDLEN);	/*  14 */
-	*reg++ = IXGB_READ_REG(hw, RDH);	/*  15 */
-	*reg++ = IXGB_READ_REG(hw, RDT);	/*  16 */
-	*reg++ = IXGB_READ_REG(hw, RDTR);	/*  17 */
-	*reg++ = IXGB_READ_REG(hw, RXDCTL);	/*  18 */
-	*reg++ = IXGB_READ_REG(hw, RAIDC);	/*  19 */
-	*reg++ = IXGB_READ_REG(hw, RXCSUM);	/*  20 */
-
-	/* there are 16 RAR entries in hardware, we only use 3 */
-	for (i = 0; i < IXGB_ALL_RAR_ENTRIES; i++) {
-		*reg++ = IXGB_READ_REG_ARRAY(hw, RAL, (i << 1)); /*21,...,51 */
-		*reg++ = IXGB_READ_REG_ARRAY(hw, RAH, (i << 1)); /*22,...,52 */
-	}
-
-	/* Transmit */
-	*reg++ = IXGB_READ_REG(hw, TCTL);	/*  53 */
-	*reg++ = IXGB_READ_REG(hw, TDBAL);	/*  54 */
-	*reg++ = IXGB_READ_REG(hw, TDBAH);	/*  55 */
-	*reg++ = IXGB_READ_REG(hw, TDLEN);	/*  56 */
-	*reg++ = IXGB_READ_REG(hw, TDH);	/*  57 */
-	*reg++ = IXGB_READ_REG(hw, TDT);	/*  58 */
-	*reg++ = IXGB_READ_REG(hw, TIDV);	/*  59 */
-	*reg++ = IXGB_READ_REG(hw, TXDCTL);	/*  60 */
-	*reg++ = IXGB_READ_REG(hw, TSPMT);	/*  61 */
-	*reg++ = IXGB_READ_REG(hw, PAP);	/*  62 */
-
-	/* Physical */
-	*reg++ = IXGB_READ_REG(hw, PCSC1);	/*  63 */
-	*reg++ = IXGB_READ_REG(hw, PCSC2);	/*  64 */
-	*reg++ = IXGB_READ_REG(hw, PCSS1);	/*  65 */
-	*reg++ = IXGB_READ_REG(hw, PCSS2);	/*  66 */
-	*reg++ = IXGB_READ_REG(hw, XPCSS);	/*  67 */
-	*reg++ = IXGB_READ_REG(hw, UCCR);	/*  68 */
-	*reg++ = IXGB_READ_REG(hw, XPCSTC);	/*  69 */
-	*reg++ = IXGB_READ_REG(hw, MACA);	/*  70 */
-	*reg++ = IXGB_READ_REG(hw, APAE);	/*  71 */
-	*reg++ = IXGB_READ_REG(hw, ARD);	/*  72 */
-	*reg++ = IXGB_READ_REG(hw, AIS);	/*  73 */
-	*reg++ = IXGB_READ_REG(hw, MSCA);	/*  74 */
-	*reg++ = IXGB_READ_REG(hw, MSRWD);	/*  75 */
-
-	/* Statistics */
-	*reg++ = IXGB_GET_STAT(adapter, tprl);	/*  76 */
-	*reg++ = IXGB_GET_STAT(adapter, tprh);	/*  77 */
-	*reg++ = IXGB_GET_STAT(adapter, gprcl);	/*  78 */
-	*reg++ = IXGB_GET_STAT(adapter, gprch);	/*  79 */
-	*reg++ = IXGB_GET_STAT(adapter, bprcl);	/*  80 */
-	*reg++ = IXGB_GET_STAT(adapter, bprch);	/*  81 */
-	*reg++ = IXGB_GET_STAT(adapter, mprcl);	/*  82 */
-	*reg++ = IXGB_GET_STAT(adapter, mprch);	/*  83 */
-	*reg++ = IXGB_GET_STAT(adapter, uprcl);	/*  84 */
-	*reg++ = IXGB_GET_STAT(adapter, uprch);	/*  85 */
-	*reg++ = IXGB_GET_STAT(adapter, vprcl);	/*  86 */
-	*reg++ = IXGB_GET_STAT(adapter, vprch);	/*  87 */
-	*reg++ = IXGB_GET_STAT(adapter, jprcl);	/*  88 */
-	*reg++ = IXGB_GET_STAT(adapter, jprch);	/*  89 */
-	*reg++ = IXGB_GET_STAT(adapter, gorcl);	/*  90 */
-	*reg++ = IXGB_GET_STAT(adapter, gorch);	/*  91 */
-	*reg++ = IXGB_GET_STAT(adapter, torl);	/*  92 */
-	*reg++ = IXGB_GET_STAT(adapter, torh);	/*  93 */
-	*reg++ = IXGB_GET_STAT(adapter, rnbc);	/*  94 */
-	*reg++ = IXGB_GET_STAT(adapter, ruc);	/*  95 */
-	*reg++ = IXGB_GET_STAT(adapter, roc);	/*  96 */
-	*reg++ = IXGB_GET_STAT(adapter, rlec);	/*  97 */
-	*reg++ = IXGB_GET_STAT(adapter, crcerrs);	/*  98 */
-	*reg++ = IXGB_GET_STAT(adapter, icbc);	/*  99 */
-	*reg++ = IXGB_GET_STAT(adapter, ecbc);	/* 100 */
-	*reg++ = IXGB_GET_STAT(adapter, mpc);	/* 101 */
-	*reg++ = IXGB_GET_STAT(adapter, tptl);	/* 102 */
-	*reg++ = IXGB_GET_STAT(adapter, tpth);	/* 103 */
-	*reg++ = IXGB_GET_STAT(adapter, gptcl);	/* 104 */
-	*reg++ = IXGB_GET_STAT(adapter, gptch);	/* 105 */
-	*reg++ = IXGB_GET_STAT(adapter, bptcl);	/* 106 */
-	*reg++ = IXGB_GET_STAT(adapter, bptch);	/* 107 */
-	*reg++ = IXGB_GET_STAT(adapter, mptcl);	/* 108 */
-	*reg++ = IXGB_GET_STAT(adapter, mptch);	/* 109 */
-	*reg++ = IXGB_GET_STAT(adapter, uptcl);	/* 110 */
-	*reg++ = IXGB_GET_STAT(adapter, uptch);	/* 111 */
-	*reg++ = IXGB_GET_STAT(adapter, vptcl);	/* 112 */
-	*reg++ = IXGB_GET_STAT(adapter, vptch);	/* 113 */
-	*reg++ = IXGB_GET_STAT(adapter, jptcl);	/* 114 */
-	*reg++ = IXGB_GET_STAT(adapter, jptch);	/* 115 */
-	*reg++ = IXGB_GET_STAT(adapter, gotcl);	/* 116 */
-	*reg++ = IXGB_GET_STAT(adapter, gotch);	/* 117 */
-	*reg++ = IXGB_GET_STAT(adapter, totl);	/* 118 */
-	*reg++ = IXGB_GET_STAT(adapter, toth);	/* 119 */
-	*reg++ = IXGB_GET_STAT(adapter, dc);	/* 120 */
-	*reg++ = IXGB_GET_STAT(adapter, plt64c);	/* 121 */
-	*reg++ = IXGB_GET_STAT(adapter, tsctc);	/* 122 */
-	*reg++ = IXGB_GET_STAT(adapter, tsctfc);	/* 123 */
-	*reg++ = IXGB_GET_STAT(adapter, ibic);	/* 124 */
-	*reg++ = IXGB_GET_STAT(adapter, rfc);	/* 125 */
-	*reg++ = IXGB_GET_STAT(adapter, lfc);	/* 126 */
-	*reg++ = IXGB_GET_STAT(adapter, pfrc);	/* 127 */
-	*reg++ = IXGB_GET_STAT(adapter, pftc);	/* 128 */
-	*reg++ = IXGB_GET_STAT(adapter, mcfrc);	/* 129 */
-	*reg++ = IXGB_GET_STAT(adapter, mcftc);	/* 130 */
-	*reg++ = IXGB_GET_STAT(adapter, xonrxc);	/* 131 */
-	*reg++ = IXGB_GET_STAT(adapter, xontxc);	/* 132 */
-	*reg++ = IXGB_GET_STAT(adapter, xoffrxc);	/* 133 */
-	*reg++ = IXGB_GET_STAT(adapter, xofftxc);	/* 134 */
-	*reg++ = IXGB_GET_STAT(adapter, rjc);	/* 135 */
-
-	regs->len = (reg - reg_start) * sizeof(u32);
-}
-
-static int
-ixgb_get_eeprom_len(struct net_device *netdev)
-{
-	/* return size in bytes */
-	return IXGB_EEPROM_SIZE << 1;
-}
-
-static int
-ixgb_get_eeprom(struct net_device *netdev,
-		  struct ethtool_eeprom *eeprom, u8 *bytes)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct ixgb_hw *hw = &adapter->hw;
-	__le16 *eeprom_buff;
-	int i, max_len, first_word, last_word;
-	int ret_val = 0;
-
-	if (eeprom->len == 0) {
-		ret_val = -EINVAL;
-		goto geeprom_error;
-	}
-
-	eeprom->magic = hw->vendor_id | (hw->device_id << 16);
-
-	max_len = ixgb_get_eeprom_len(netdev);
-
-	if (eeprom->offset > eeprom->offset + eeprom->len) {
-		ret_val = -EINVAL;
-		goto geeprom_error;
-	}
-
-	if ((eeprom->offset + eeprom->len) > max_len)
-		eeprom->len = (max_len - eeprom->offset);
-
-	first_word = eeprom->offset >> 1;
-	last_word = (eeprom->offset + eeprom->len - 1) >> 1;
-
-	eeprom_buff = kmalloc_array(last_word - first_word + 1,
-				    sizeof(__le16),
-				    GFP_KERNEL);
-	if (!eeprom_buff)
-		return -ENOMEM;
-
-	/* note the eeprom was good because the driver loaded */
-	for (i = 0; i <= (last_word - first_word); i++)
-		eeprom_buff[i] = ixgb_get_eeprom_word(hw, (first_word + i));
-
-	memcpy(bytes, (u8 *)eeprom_buff + (eeprom->offset & 1), eeprom->len);
-	kfree(eeprom_buff);
-
-geeprom_error:
-	return ret_val;
-}
-
-static int
-ixgb_set_eeprom(struct net_device *netdev,
-		  struct ethtool_eeprom *eeprom, u8 *bytes)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct ixgb_hw *hw = &adapter->hw;
-	u16 *eeprom_buff;
-	void *ptr;
-	int max_len, first_word, last_word;
-	u16 i;
-
-	if (eeprom->len == 0)
-		return -EINVAL;
-
-	if (eeprom->magic != (hw->vendor_id | (hw->device_id << 16)))
-		return -EFAULT;
-
-	max_len = ixgb_get_eeprom_len(netdev);
-
-	if (eeprom->offset > eeprom->offset + eeprom->len)
-		return -EINVAL;
-
-	if ((eeprom->offset + eeprom->len) > max_len)
-		eeprom->len = (max_len - eeprom->offset);
-
-	first_word = eeprom->offset >> 1;
-	last_word = (eeprom->offset + eeprom->len - 1) >> 1;
-	eeprom_buff = kmalloc(max_len, GFP_KERNEL);
-	if (!eeprom_buff)
-		return -ENOMEM;
-
-	ptr = (void *)eeprom_buff;
-
-	if (eeprom->offset & 1) {
-		/* need read/modify/write of first changed EEPROM word */
-		/* only the second byte of the word is being modified */
-		eeprom_buff[0] = ixgb_read_eeprom(hw, first_word);
-		ptr++;
-	}
-	if ((eeprom->offset + eeprom->len) & 1) {
-		/* need read/modify/write of last changed EEPROM word */
-		/* only the first byte of the word is being modified */
-		eeprom_buff[last_word - first_word]
-			= ixgb_read_eeprom(hw, last_word);
-	}
-
-	memcpy(ptr, bytes, eeprom->len);
-	for (i = 0; i <= (last_word - first_word); i++)
-		ixgb_write_eeprom(hw, first_word + i, eeprom_buff[i]);
-
-	/* Update the checksum over the first part of the EEPROM if needed */
-	if (first_word <= EEPROM_CHECKSUM_REG)
-		ixgb_update_eeprom_checksum(hw);
-
-	kfree(eeprom_buff);
-	return 0;
-}
-
-static void
-ixgb_get_drvinfo(struct net_device *netdev,
-		   struct ethtool_drvinfo *drvinfo)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-
-	strscpy(drvinfo->driver,  ixgb_driver_name,
-		sizeof(drvinfo->driver));
-	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
-		sizeof(drvinfo->bus_info));
-}
-
-static void
-ixgb_get_ringparam(struct net_device *netdev,
-		   struct ethtool_ringparam *ring,
-		   struct kernel_ethtool_ringparam *kernel_ring,
-		   struct netlink_ext_ack *extack)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct ixgb_desc_ring *txdr = &adapter->tx_ring;
-	struct ixgb_desc_ring *rxdr = &adapter->rx_ring;
-
-	ring->rx_max_pending = MAX_RXD;
-	ring->tx_max_pending = MAX_TXD;
-	ring->rx_pending = rxdr->count;
-	ring->tx_pending = txdr->count;
-}
-
-static int
-ixgb_set_ringparam(struct net_device *netdev,
-		   struct ethtool_ringparam *ring,
-		   struct kernel_ethtool_ringparam *kernel_ring,
-		   struct netlink_ext_ack *extack)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct ixgb_desc_ring *txdr = &adapter->tx_ring;
-	struct ixgb_desc_ring *rxdr = &adapter->rx_ring;
-	struct ixgb_desc_ring tx_old, tx_new, rx_old, rx_new;
-	int err;
-
-	tx_old = adapter->tx_ring;
-	rx_old = adapter->rx_ring;
-
-	if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))
-		return -EINVAL;
-
-	if (netif_running(adapter->netdev))
-		ixgb_down(adapter, true);
-
-	rxdr->count = max(ring->rx_pending,(u32)MIN_RXD);
-	rxdr->count = min(rxdr->count,(u32)MAX_RXD);
-	rxdr->count = ALIGN(rxdr->count, IXGB_REQ_RX_DESCRIPTOR_MULTIPLE);
-
-	txdr->count = max(ring->tx_pending,(u32)MIN_TXD);
-	txdr->count = min(txdr->count,(u32)MAX_TXD);
-	txdr->count = ALIGN(txdr->count, IXGB_REQ_TX_DESCRIPTOR_MULTIPLE);
-
-	if (netif_running(adapter->netdev)) {
-		/* Try to get new resources before deleting old */
-		if ((err = ixgb_setup_rx_resources(adapter)))
-			goto err_setup_rx;
-		if ((err = ixgb_setup_tx_resources(adapter)))
-			goto err_setup_tx;
-
-		/* save the new, restore the old in order to free it,
-		 * then restore the new back again */
-
-		rx_new = adapter->rx_ring;
-		tx_new = adapter->tx_ring;
-		adapter->rx_ring = rx_old;
-		adapter->tx_ring = tx_old;
-		ixgb_free_rx_resources(adapter);
-		ixgb_free_tx_resources(adapter);
-		adapter->rx_ring = rx_new;
-		adapter->tx_ring = tx_new;
-		if ((err = ixgb_up(adapter)))
-			return err;
-		ixgb_set_speed_duplex(netdev);
-	}
-
-	return 0;
-err_setup_tx:
-	ixgb_free_rx_resources(adapter);
-err_setup_rx:
-	adapter->rx_ring = rx_old;
-	adapter->tx_ring = tx_old;
-	ixgb_up(adapter);
-	return err;
-}
-
-static int
-ixgb_set_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-
-	switch (state) {
-	case ETHTOOL_ID_ACTIVE:
-		return 2;
-
-	case ETHTOOL_ID_ON:
-		ixgb_led_on(&adapter->hw);
-		break;
-
-	case ETHTOOL_ID_OFF:
-	case ETHTOOL_ID_INACTIVE:
-		ixgb_led_off(&adapter->hw);
-	}
-
-	return 0;
-}
-
-static int
-ixgb_get_sset_count(struct net_device *netdev, int sset)
-{
-	switch (sset) {
-	case ETH_SS_STATS:
-		return IXGB_STATS_LEN;
-	default:
-		return -EOPNOTSUPP;
-	}
-}
-
-static void
-ixgb_get_ethtool_stats(struct net_device *netdev,
-		struct ethtool_stats *stats, u64 *data)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	int i;
-	char *p = NULL;
-
-	ixgb_update_stats(adapter);
-	for (i = 0; i < IXGB_STATS_LEN; i++) {
-		switch (ixgb_gstrings_stats[i].type) {
-		case NETDEV_STATS:
-			p = (char *) netdev +
-					ixgb_gstrings_stats[i].stat_offset;
-			break;
-		case IXGB_STATS:
-			p = (char *) adapter +
-					ixgb_gstrings_stats[i].stat_offset;
-			break;
-		}
-
-		data[i] = (ixgb_gstrings_stats[i].sizeof_stat ==
-			sizeof(u64)) ? *(u64 *)p : *(u32 *)p;
-	}
-}
-
-static void
-ixgb_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
-{
-	int i;
-
-	switch(stringset) {
-	case ETH_SS_STATS:
-		for (i = 0; i < IXGB_STATS_LEN; i++) {
-			memcpy(data + i * ETH_GSTRING_LEN,
-			ixgb_gstrings_stats[i].stat_string,
-			ETH_GSTRING_LEN);
-		}
-		break;
-	}
-}
-
-static const struct ethtool_ops ixgb_ethtool_ops = {
-	.get_drvinfo = ixgb_get_drvinfo,
-	.get_regs_len = ixgb_get_regs_len,
-	.get_regs = ixgb_get_regs,
-	.get_link = ethtool_op_get_link,
-	.get_eeprom_len = ixgb_get_eeprom_len,
-	.get_eeprom = ixgb_get_eeprom,
-	.set_eeprom = ixgb_set_eeprom,
-	.get_ringparam = ixgb_get_ringparam,
-	.set_ringparam = ixgb_set_ringparam,
-	.get_pauseparam	= ixgb_get_pauseparam,
-	.set_pauseparam	= ixgb_set_pauseparam,
-	.get_msglevel = ixgb_get_msglevel,
-	.set_msglevel = ixgb_set_msglevel,
-	.get_strings = ixgb_get_strings,
-	.set_phys_id = ixgb_set_phys_id,
-	.get_sset_count = ixgb_get_sset_count,
-	.get_ethtool_stats = ixgb_get_ethtool_stats,
-	.get_link_ksettings = ixgb_get_link_ksettings,
-	.set_link_ksettings = ixgb_set_link_ksettings,
-};
-
-void ixgb_set_ethtool_ops(struct net_device *netdev)
-{
-	netdev->ethtool_ops = &ixgb_ethtool_ops;
-}
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_hw.c b/drivers/net/ethernet/intel/ixgb/ixgb_hw.c
deleted file mode 100644
index 98bd3267b99b..000000000000
--- a/drivers/net/ethernet/intel/ixgb/ixgb_hw.c
+++ /dev/null
@@ -1,1229 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright(c) 1999 - 2008 Intel Corporation. */
-
-/* ixgb_hw.c
- * Shared functions for accessing and configuring the adapter
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/pci_ids.h>
-#include "ixgb_hw.h"
-#include "ixgb_ids.h"
-
-#include <linux/etherdevice.h>
-
-/*  Local function prototypes */
-
-static u32 ixgb_hash_mc_addr(struct ixgb_hw *hw, u8 * mc_addr);
-
-static void ixgb_mta_set(struct ixgb_hw *hw, u32 hash_value);
-
-static void ixgb_get_bus_info(struct ixgb_hw *hw);
-
-static bool ixgb_link_reset(struct ixgb_hw *hw);
-
-static void ixgb_optics_reset(struct ixgb_hw *hw);
-
-static void ixgb_optics_reset_bcm(struct ixgb_hw *hw);
-
-static ixgb_phy_type ixgb_identify_phy(struct ixgb_hw *hw);
-
-static void ixgb_clear_hw_cntrs(struct ixgb_hw *hw);
-
-static void ixgb_clear_vfta(struct ixgb_hw *hw);
-
-static void ixgb_init_rx_addrs(struct ixgb_hw *hw);
-
-static u16 ixgb_read_phy_reg(struct ixgb_hw *hw,
-				  u32 reg_address,
-				  u32 phy_address,
-				  u32 device_type);
-
-static bool ixgb_setup_fc(struct ixgb_hw *hw);
-
-static bool mac_addr_valid(u8 *mac_addr);
-
-static u32 ixgb_mac_reset(struct ixgb_hw *hw)
-{
-	u32 ctrl_reg;
-
-	ctrl_reg =  IXGB_CTRL0_RST |
-				IXGB_CTRL0_SDP3_DIR |   /* All pins are Output=1 */
-				IXGB_CTRL0_SDP2_DIR |
-				IXGB_CTRL0_SDP1_DIR |
-				IXGB_CTRL0_SDP0_DIR |
-				IXGB_CTRL0_SDP3	 |   /* Initial value 1101   */
-				IXGB_CTRL0_SDP2	 |
-				IXGB_CTRL0_SDP0;
-
-#ifdef HP_ZX1
-	/* Workaround for 82597EX reset errata */
-	IXGB_WRITE_REG_IO(hw, CTRL0, ctrl_reg);
-#else
-	IXGB_WRITE_REG(hw, CTRL0, ctrl_reg);
-#endif
-
-	/* Delay a few ms just to allow the reset to complete */
-	msleep(IXGB_DELAY_AFTER_RESET);
-	ctrl_reg = IXGB_READ_REG(hw, CTRL0);
-#ifdef DBG
-	/* Make sure the self-clearing global reset bit did self clear */
-	ASSERT(!(ctrl_reg & IXGB_CTRL0_RST));
-#endif
-
-	if (hw->subsystem_vendor_id == PCI_VENDOR_ID_SUN) {
-		ctrl_reg =  /* Enable interrupt from XFP and SerDes */
-			   IXGB_CTRL1_GPI0_EN |
-			   IXGB_CTRL1_SDP6_DIR |
-			   IXGB_CTRL1_SDP7_DIR |
-			   IXGB_CTRL1_SDP6 |
-			   IXGB_CTRL1_SDP7;
-		IXGB_WRITE_REG(hw, CTRL1, ctrl_reg);
-		ixgb_optics_reset_bcm(hw);
-	}
-
-	if (hw->phy_type == ixgb_phy_type_txn17401)
-		ixgb_optics_reset(hw);
-
-	return ctrl_reg;
-}
-
-/******************************************************************************
- * Reset the transmit and receive units; mask and clear all interrupts.
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-bool
-ixgb_adapter_stop(struct ixgb_hw *hw)
-{
-	u32 ctrl_reg;
-
-	ENTER();
-
-	/* If we are stopped or resetting exit gracefully and wait to be
-	 * started again before accessing the hardware.
-	 */
-	if (hw->adapter_stopped) {
-		pr_debug("Exiting because the adapter is already stopped!!!\n");
-		return false;
-	}
-
-	/* Set the Adapter Stopped flag so other driver functions stop
-	 * touching the Hardware.
-	 */
-	hw->adapter_stopped = true;
-
-	/* Clear interrupt mask to stop board from generating interrupts */
-	pr_debug("Masking off all interrupts\n");
-	IXGB_WRITE_REG(hw, IMC, 0xFFFFFFFF);
-
-	/* Disable the Transmit and Receive units.  Then delay to allow
-	 * any pending transactions to complete before we hit the MAC with
-	 * the global reset.
-	 */
-	IXGB_WRITE_REG(hw, RCTL, IXGB_READ_REG(hw, RCTL) & ~IXGB_RCTL_RXEN);
-	IXGB_WRITE_REG(hw, TCTL, IXGB_READ_REG(hw, TCTL) & ~IXGB_TCTL_TXEN);
-	IXGB_WRITE_FLUSH(hw);
-	msleep(IXGB_DELAY_BEFORE_RESET);
-
-	/* Issue a global reset to the MAC.  This will reset the chip's
-	 * transmit, receive, DMA, and link units.  It will not effect
-	 * the current PCI configuration.  The global reset bit is self-
-	 * clearing, and should clear within a microsecond.
-	 */
-	pr_debug("Issuing a global reset to MAC\n");
-
-	ctrl_reg = ixgb_mac_reset(hw);
-
-	/* Clear interrupt mask to stop board from generating interrupts */
-	pr_debug("Masking off all interrupts\n");
-	IXGB_WRITE_REG(hw, IMC, 0xffffffff);
-
-	/* Clear any pending interrupt events. */
-	IXGB_READ_REG(hw, ICR);
-
-	return ctrl_reg & IXGB_CTRL0_RST;
-}
-
-
-/******************************************************************************
- * Identifies the vendor of the optics module on the adapter.  The SR adapters
- * support two different types of XPAK optics, so it is necessary to determine
- * which optics are present before applying any optics-specific workarounds.
- *
- * hw - Struct containing variables accessed by shared code.
- *
- * Returns: the vendor of the XPAK optics module.
- *****************************************************************************/
-static ixgb_xpak_vendor
-ixgb_identify_xpak_vendor(struct ixgb_hw *hw)
-{
-	u32 i;
-	u16 vendor_name[5];
-	ixgb_xpak_vendor xpak_vendor;
-
-	ENTER();
-
-	/* Read the first few bytes of the vendor string from the XPAK NVR
-	 * registers.  These are standard XENPAK/XPAK registers, so all XPAK
-	 * devices should implement them. */
-	for (i = 0; i < 5; i++) {
-		vendor_name[i] = ixgb_read_phy_reg(hw,
-						   MDIO_PMA_PMD_XPAK_VENDOR_NAME
-						   + i, IXGB_PHY_ADDRESS,
-						   MDIO_MMD_PMAPMD);
-	}
-
-	/* Determine the actual vendor */
-	if (vendor_name[0] == 'I' &&
-	    vendor_name[1] == 'N' &&
-	    vendor_name[2] == 'T' &&
-	    vendor_name[3] == 'E' && vendor_name[4] == 'L') {
-		xpak_vendor = ixgb_xpak_vendor_intel;
-	} else {
-		xpak_vendor = ixgb_xpak_vendor_infineon;
-	}
-
-	return xpak_vendor;
-}
-
-/******************************************************************************
- * Determine the physical layer module on the adapter.
- *
- * hw - Struct containing variables accessed by shared code.  The device_id
- *      field must be (correctly) populated before calling this routine.
- *
- * Returns: the phy type of the adapter.
- *****************************************************************************/
-static ixgb_phy_type
-ixgb_identify_phy(struct ixgb_hw *hw)
-{
-	ixgb_phy_type phy_type;
-	ixgb_xpak_vendor xpak_vendor;
-
-	ENTER();
-
-	/* Infer the transceiver/phy type from the device id */
-	switch (hw->device_id) {
-	case IXGB_DEVICE_ID_82597EX:
-		pr_debug("Identified TXN17401 optics\n");
-		phy_type = ixgb_phy_type_txn17401;
-		break;
-
-	case IXGB_DEVICE_ID_82597EX_SR:
-		/* The SR adapters carry two different types of XPAK optics
-		 * modules; read the vendor identifier to determine the exact
-		 * type of optics. */
-		xpak_vendor = ixgb_identify_xpak_vendor(hw);
-		if (xpak_vendor == ixgb_xpak_vendor_intel) {
-			pr_debug("Identified TXN17201 optics\n");
-			phy_type = ixgb_phy_type_txn17201;
-		} else {
-			pr_debug("Identified G6005 optics\n");
-			phy_type = ixgb_phy_type_g6005;
-		}
-		break;
-	case IXGB_DEVICE_ID_82597EX_LR:
-		pr_debug("Identified G6104 optics\n");
-		phy_type = ixgb_phy_type_g6104;
-		break;
-	case IXGB_DEVICE_ID_82597EX_CX4:
-		pr_debug("Identified CX4\n");
-		xpak_vendor = ixgb_identify_xpak_vendor(hw);
-		if (xpak_vendor == ixgb_xpak_vendor_intel) {
-			pr_debug("Identified TXN17201 optics\n");
-			phy_type = ixgb_phy_type_txn17201;
-		} else {
-			pr_debug("Identified G6005 optics\n");
-			phy_type = ixgb_phy_type_g6005;
-		}
-		break;
-	default:
-		pr_debug("Unknown physical layer module\n");
-		phy_type = ixgb_phy_type_unknown;
-		break;
-	}
-
-	/* update phy type for sun specific board */
-	if (hw->subsystem_vendor_id == PCI_VENDOR_ID_SUN)
-		phy_type = ixgb_phy_type_bcm;
-
-	return phy_type;
-}
-
-/******************************************************************************
- * Performs basic configuration of the adapter.
- *
- * hw - Struct containing variables accessed by shared code
- *
- * Resets the controller.
- * Reads and validates the EEPROM.
- * Initializes the receive address registers.
- * Initializes the multicast table.
- * Clears all on-chip counters.
- * Calls routine to setup flow control settings.
- * Leaves the transmit and receive units disabled and uninitialized.
- *
- * Returns:
- *      true if successful,
- *      false if unrecoverable problems were encountered.
- *****************************************************************************/
-bool
-ixgb_init_hw(struct ixgb_hw *hw)
-{
-	u32 i;
-	bool status;
-
-	ENTER();
-
-	/* Issue a global reset to the MAC.  This will reset the chip's
-	 * transmit, receive, DMA, and link units.  It will not effect
-	 * the current PCI configuration.  The global reset bit is self-
-	 * clearing, and should clear within a microsecond.
-	 */
-	pr_debug("Issuing a global reset to MAC\n");
-
-	ixgb_mac_reset(hw);
-
-	pr_debug("Issuing an EE reset to MAC\n");
-#ifdef HP_ZX1
-	/* Workaround for 82597EX reset errata */
-	IXGB_WRITE_REG_IO(hw, CTRL1, IXGB_CTRL1_EE_RST);
-#else
-	IXGB_WRITE_REG(hw, CTRL1, IXGB_CTRL1_EE_RST);
-#endif
-
-	/* Delay a few ms just to allow the reset to complete */
-	msleep(IXGB_DELAY_AFTER_EE_RESET);
-
-	if (!ixgb_get_eeprom_data(hw))
-		return false;
-
-	/* Use the device id to determine the type of phy/transceiver. */
-	hw->device_id = ixgb_get_ee_device_id(hw);
-	hw->phy_type = ixgb_identify_phy(hw);
-
-	/* Setup the receive addresses.
-	 * Receive Address Registers (RARs 0 - 15).
-	 */
-	ixgb_init_rx_addrs(hw);
-
-	/*
-	 * Check that a valid MAC address has been set.
-	 * If it is not valid, we fail hardware init.
-	 */
-	if (!mac_addr_valid(hw->curr_mac_addr)) {
-		pr_debug("MAC address invalid after ixgb_init_rx_addrs\n");
-		return(false);
-	}
-
-	/* tell the routines in this file they can access hardware again */
-	hw->adapter_stopped = false;
-
-	/* Fill in the bus_info structure */
-	ixgb_get_bus_info(hw);
-
-	/* Zero out the Multicast HASH table */
-	pr_debug("Zeroing the MTA\n");
-	for (i = 0; i < IXGB_MC_TBL_SIZE; i++)
-		IXGB_WRITE_REG_ARRAY(hw, MTA, i, 0);
-
-	/* Zero out the VLAN Filter Table Array */
-	ixgb_clear_vfta(hw);
-
-	/* Zero all of the hardware counters */
-	ixgb_clear_hw_cntrs(hw);
-
-	/* Call a subroutine to setup flow control. */
-	status = ixgb_setup_fc(hw);
-
-	/* 82597EX errata: Call check-for-link in case lane deskew is locked */
-	ixgb_check_for_link(hw);
-
-	return status;
-}
-
-/******************************************************************************
- * Initializes receive address filters.
- *
- * hw - Struct containing variables accessed by shared code
- *
- * Places the MAC address in receive address register 0 and clears the rest
- * of the receive address registers. Clears the multicast table. Assumes
- * the receiver is in reset when the routine is called.
- *****************************************************************************/
-static void
-ixgb_init_rx_addrs(struct ixgb_hw *hw)
-{
-	u32 i;
-
-	ENTER();
-
-	/*
-	 * If the current mac address is valid, assume it is a software override
-	 * to the permanent address.
-	 * Otherwise, use the permanent address from the eeprom.
-	 */
-	if (!mac_addr_valid(hw->curr_mac_addr)) {
-
-		/* Get the MAC address from the eeprom for later reference */
-		ixgb_get_ee_mac_addr(hw, hw->curr_mac_addr);
-
-		pr_debug("Keeping Permanent MAC Addr = %pM\n",
-			 hw->curr_mac_addr);
-	} else {
-
-		/* Setup the receive address. */
-		pr_debug("Overriding MAC Address in RAR[0]\n");
-		pr_debug("New MAC Addr = %pM\n", hw->curr_mac_addr);
-
-		ixgb_rar_set(hw, hw->curr_mac_addr, 0);
-	}
-
-	/* Zero out the other 15 receive addresses. */
-	pr_debug("Clearing RAR[1-15]\n");
-	for (i = 1; i < IXGB_RAR_ENTRIES; i++) {
-		/* Write high reg first to disable the AV bit first */
-		IXGB_WRITE_REG_ARRAY(hw, RA, ((i << 1) + 1), 0);
-		IXGB_WRITE_REG_ARRAY(hw, RA, (i << 1), 0);
-	}
-}
-
-/******************************************************************************
- * Updates the MAC's list of multicast addresses.
- *
- * hw - Struct containing variables accessed by shared code
- * mc_addr_list - the list of new multicast addresses
- * mc_addr_count - number of addresses
- * pad - number of bytes between addresses in the list
- *
- * The given list replaces any existing list. Clears the last 15 receive
- * address registers and the multicast table. Uses receive address registers
- * for the first 15 multicast addresses, and hashes the rest into the
- * multicast table.
- *****************************************************************************/
-void
-ixgb_mc_addr_list_update(struct ixgb_hw *hw,
-			  u8 *mc_addr_list,
-			  u32 mc_addr_count,
-			  u32 pad)
-{
-	u32 hash_value;
-	u32 i;
-	u32 rar_used_count = 1;		/* RAR[0] is used for our MAC address */
-	u8 *mca;
-
-	ENTER();
-
-	/* Set the new number of MC addresses that we are being requested to use. */
-	hw->num_mc_addrs = mc_addr_count;
-
-	/* Clear RAR[1-15] */
-	pr_debug("Clearing RAR[1-15]\n");
-	for (i = rar_used_count; i < IXGB_RAR_ENTRIES; i++) {
-		IXGB_WRITE_REG_ARRAY(hw, RA, (i << 1), 0);
-		IXGB_WRITE_REG_ARRAY(hw, RA, ((i << 1) + 1), 0);
-	}
-
-	/* Clear the MTA */
-	pr_debug("Clearing MTA\n");
-	for (i = 0; i < IXGB_MC_TBL_SIZE; i++)
-		IXGB_WRITE_REG_ARRAY(hw, MTA, i, 0);
-
-	/* Add the new addresses */
-	mca = mc_addr_list;
-	for (i = 0; i < mc_addr_count; i++) {
-		pr_debug("Adding the multicast addresses:\n");
-		pr_debug("MC Addr #%d = %pM\n", i, mca);
-
-		/* Place this multicast address in the RAR if there is room, *
-		 * else put it in the MTA
-		 */
-		if (rar_used_count < IXGB_RAR_ENTRIES) {
-			ixgb_rar_set(hw, mca, rar_used_count);
-			pr_debug("Added a multicast address to RAR[%d]\n", i);
-			rar_used_count++;
-		} else {
-			hash_value = ixgb_hash_mc_addr(hw, mca);
-
-			pr_debug("Hash value = 0x%03X\n", hash_value);
-
-			ixgb_mta_set(hw, hash_value);
-		}
-
-		mca += ETH_ALEN + pad;
-	}
-
-	pr_debug("MC Update Complete\n");
-}
-
-/******************************************************************************
- * Hashes an address to determine its location in the multicast table
- *
- * hw - Struct containing variables accessed by shared code
- * mc_addr - the multicast address to hash
- *
- * Returns:
- *      The hash value
- *****************************************************************************/
-static u32
-ixgb_hash_mc_addr(struct ixgb_hw *hw,
-		   u8 *mc_addr)
-{
-	u32 hash_value = 0;
-
-	ENTER();
-
-	/* The portion of the address that is used for the hash table is
-	 * determined by the mc_filter_type setting.
-	 */
-	switch (hw->mc_filter_type) {
-		/* [0] [1] [2] [3] [4] [5]
-		 * 01  AA  00  12  34  56
-		 * LSB                 MSB - According to H/W docs */
-	case 0:
-		/* [47:36] i.e. 0x563 for above example address */
-		hash_value =
-		    ((mc_addr[4] >> 4) | (((u16) mc_addr[5]) << 4));
-		break;
-	case 1:		/* [46:35] i.e. 0xAC6 for above example address */
-		hash_value =
-		    ((mc_addr[4] >> 3) | (((u16) mc_addr[5]) << 5));
-		break;
-	case 2:		/* [45:34] i.e. 0x5D8 for above example address */
-		hash_value =
-		    ((mc_addr[4] >> 2) | (((u16) mc_addr[5]) << 6));
-		break;
-	case 3:		/* [43:32] i.e. 0x634 for above example address */
-		hash_value = ((mc_addr[4]) | (((u16) mc_addr[5]) << 8));
-		break;
-	default:
-		/* Invalid mc_filter_type, what should we do? */
-		pr_debug("MC filter type param set incorrectly\n");
-		ASSERT(0);
-		break;
-	}
-
-	hash_value &= 0xFFF;
-	return hash_value;
-}
-
-/******************************************************************************
- * Sets the bit in the multicast table corresponding to the hash value.
- *
- * hw - Struct containing variables accessed by shared code
- * hash_value - Multicast address hash value
- *****************************************************************************/
-static void
-ixgb_mta_set(struct ixgb_hw *hw,
-		  u32 hash_value)
-{
-	u32 hash_bit, hash_reg;
-	u32 mta_reg;
-
-	/* The MTA is a register array of 128 32-bit registers.
-	 * It is treated like an array of 4096 bits.  We want to set
-	 * bit BitArray[hash_value]. So we figure out what register
-	 * the bit is in, read it, OR in the new bit, then write
-	 * back the new value.  The register is determined by the
-	 * upper 7 bits of the hash value and the bit within that
-	 * register are determined by the lower 5 bits of the value.
-	 */
-	hash_reg = (hash_value >> 5) & 0x7F;
-	hash_bit = hash_value & 0x1F;
-
-	mta_reg = IXGB_READ_REG_ARRAY(hw, MTA, hash_reg);
-
-	mta_reg |= (1 << hash_bit);
-
-	IXGB_WRITE_REG_ARRAY(hw, MTA, hash_reg, mta_reg);
-}
-
-/******************************************************************************
- * Puts an ethernet address into a receive address register.
- *
- * hw - Struct containing variables accessed by shared code
- * addr - Address to put into receive address register
- * index - Receive address register to write
- *****************************************************************************/
-void
-ixgb_rar_set(struct ixgb_hw *hw,
-		  const u8 *addr,
-		  u32 index)
-{
-	u32 rar_low, rar_high;
-
-	ENTER();
-
-	/* HW expects these in little endian so we reverse the byte order
-	 * from network order (big endian) to little endian
-	 */
-	rar_low = ((u32) addr[0] |
-		   ((u32)addr[1] << 8) |
-		   ((u32)addr[2] << 16) |
-		   ((u32)addr[3] << 24));
-
-	rar_high = ((u32) addr[4] |
-			((u32)addr[5] << 8) |
-			IXGB_RAH_AV);
-
-	IXGB_WRITE_REG_ARRAY(hw, RA, (index << 1), rar_low);
-	IXGB_WRITE_REG_ARRAY(hw, RA, ((index << 1) + 1), rar_high);
-}
-
-/******************************************************************************
- * Writes a value to the specified offset in the VLAN filter table.
- *
- * hw - Struct containing variables accessed by shared code
- * offset - Offset in VLAN filter table to write
- * value - Value to write into VLAN filter table
- *****************************************************************************/
-void
-ixgb_write_vfta(struct ixgb_hw *hw,
-		 u32 offset,
-		 u32 value)
-{
-	IXGB_WRITE_REG_ARRAY(hw, VFTA, offset, value);
-}
-
-/******************************************************************************
- * Clears the VLAN filter table
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-static void
-ixgb_clear_vfta(struct ixgb_hw *hw)
-{
-	u32 offset;
-
-	for (offset = 0; offset < IXGB_VLAN_FILTER_TBL_SIZE; offset++)
-		IXGB_WRITE_REG_ARRAY(hw, VFTA, offset, 0);
-}
-
-/******************************************************************************
- * Configures the flow control settings based on SW configuration.
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-
-static bool
-ixgb_setup_fc(struct ixgb_hw *hw)
-{
-	u32 ctrl_reg;
-	u32 pap_reg = 0;   /* by default, assume no pause time */
-	bool status = true;
-
-	ENTER();
-
-	/* Get the current control reg 0 settings */
-	ctrl_reg = IXGB_READ_REG(hw, CTRL0);
-
-	/* Clear the Receive Pause Enable and Transmit Pause Enable bits */
-	ctrl_reg &= ~(IXGB_CTRL0_RPE | IXGB_CTRL0_TPE);
-
-	/* The possible values of the "flow_control" parameter are:
-	 *      0:  Flow control is completely disabled
-	 *      1:  Rx flow control is enabled (we can receive pause frames
-	 *          but not send pause frames).
-	 *      2:  Tx flow control is enabled (we can send pause frames
-	 *          but we do not support receiving pause frames).
-	 *      3:  Both Rx and TX flow control (symmetric) are enabled.
-	 *  other:  Invalid.
-	 */
-	switch (hw->fc.type) {
-	case ixgb_fc_none:	/* 0 */
-		/* Set CMDC bit to disable Rx Flow control */
-		ctrl_reg |= (IXGB_CTRL0_CMDC);
-		break;
-	case ixgb_fc_rx_pause:	/* 1 */
-		/* RX Flow control is enabled, and TX Flow control is
-		 * disabled.
-		 */
-		ctrl_reg |= (IXGB_CTRL0_RPE);
-		break;
-	case ixgb_fc_tx_pause:	/* 2 */
-		/* TX Flow control is enabled, and RX Flow control is
-		 * disabled, by a software over-ride.
-		 */
-		ctrl_reg |= (IXGB_CTRL0_TPE);
-		pap_reg = hw->fc.pause_time;
-		break;
-	case ixgb_fc_full:	/* 3 */
-		/* Flow control (both RX and TX) is enabled by a software
-		 * over-ride.
-		 */
-		ctrl_reg |= (IXGB_CTRL0_RPE | IXGB_CTRL0_TPE);
-		pap_reg = hw->fc.pause_time;
-		break;
-	default:
-		/* We should never get here.  The value should be 0-3. */
-		pr_debug("Flow control param set incorrectly\n");
-		ASSERT(0);
-		break;
-	}
-
-	/* Write the new settings */
-	IXGB_WRITE_REG(hw, CTRL0, ctrl_reg);
-
-	if (pap_reg != 0)
-		IXGB_WRITE_REG(hw, PAP, pap_reg);
-
-	/* Set the flow control receive threshold registers.  Normally,
-	 * these registers will be set to a default threshold that may be
-	 * adjusted later by the driver's runtime code.  However, if the
-	 * ability to transmit pause frames in not enabled, then these
-	 * registers will be set to 0.
-	 */
-	if (!(hw->fc.type & ixgb_fc_tx_pause)) {
-		IXGB_WRITE_REG(hw, FCRTL, 0);
-		IXGB_WRITE_REG(hw, FCRTH, 0);
-	} else {
-	   /* We need to set up the Receive Threshold high and low water
-	    * marks as well as (optionally) enabling the transmission of XON
-	    * frames. */
-		if (hw->fc.send_xon) {
-			IXGB_WRITE_REG(hw, FCRTL,
-				(hw->fc.low_water | IXGB_FCRTL_XONE));
-		} else {
-			IXGB_WRITE_REG(hw, FCRTL, hw->fc.low_water);
-		}
-		IXGB_WRITE_REG(hw, FCRTH, hw->fc.high_water);
-	}
-	return status;
-}
-
-/******************************************************************************
- * Reads a word from a device over the Management Data Interface (MDI) bus.
- * This interface is used to manage Physical layer devices.
- *
- * hw          - Struct containing variables accessed by hw code
- * reg_address - Offset of device register being read.
- * phy_address - Address of device on MDI.
- *
- * Returns:  Data word (16 bits) from MDI device.
- *
- * The 82597EX has support for several MDI access methods.  This routine
- * uses the new protocol MDI Single Command and Address Operation.
- * This requires that first an address cycle command is sent, followed by a
- * read command.
- *****************************************************************************/
-static u16
-ixgb_read_phy_reg(struct ixgb_hw *hw,
-		u32 reg_address,
-		u32 phy_address,
-		u32 device_type)
-{
-	u32 i;
-	u32 data;
-	u32 command = 0;
-
-	ASSERT(reg_address <= IXGB_MAX_PHY_REG_ADDRESS);
-	ASSERT(phy_address <= IXGB_MAX_PHY_ADDRESS);
-	ASSERT(device_type <= IXGB_MAX_PHY_DEV_TYPE);
-
-	/* Setup and write the address cycle command */
-	command = ((reg_address << IXGB_MSCA_NP_ADDR_SHIFT) |
-		   (device_type << IXGB_MSCA_DEV_TYPE_SHIFT) |
-		   (phy_address << IXGB_MSCA_PHY_ADDR_SHIFT) |
-		   (IXGB_MSCA_ADDR_CYCLE | IXGB_MSCA_MDI_COMMAND));
-
-	IXGB_WRITE_REG(hw, MSCA, command);
-
-    /**************************************************************
-    ** Check every 10 usec to see if the address cycle completed
-    ** The COMMAND bit will clear when the operation is complete.
-    ** This may take as long as 64 usecs (we'll wait 100 usecs max)
-    ** from the CPU Write to the Ready bit assertion.
-    **************************************************************/
-
-	for (i = 0; i < 10; i++)
-	{
-		udelay(10);
-
-		command = IXGB_READ_REG(hw, MSCA);
-
-		if ((command & IXGB_MSCA_MDI_COMMAND) == 0)
-			break;
-	}
-
-	ASSERT((command & IXGB_MSCA_MDI_COMMAND) == 0);
-
-	/* Address cycle complete, setup and write the read command */
-	command = ((reg_address << IXGB_MSCA_NP_ADDR_SHIFT) |
-		   (device_type << IXGB_MSCA_DEV_TYPE_SHIFT) |
-		   (phy_address << IXGB_MSCA_PHY_ADDR_SHIFT) |
-		   (IXGB_MSCA_READ | IXGB_MSCA_MDI_COMMAND));
-
-	IXGB_WRITE_REG(hw, MSCA, command);
-
-    /**************************************************************
-    ** Check every 10 usec to see if the read command completed
-    ** The COMMAND bit will clear when the operation is complete.
-    ** The read may take as long as 64 usecs (we'll wait 100 usecs max)
-    ** from the CPU Write to the Ready bit assertion.
-    **************************************************************/
-
-	for (i = 0; i < 10; i++)
-	{
-		udelay(10);
-
-		command = IXGB_READ_REG(hw, MSCA);
-
-		if ((command & IXGB_MSCA_MDI_COMMAND) == 0)
-			break;
-	}
-
-	ASSERT((command & IXGB_MSCA_MDI_COMMAND) == 0);
-
-	/* Operation is complete, get the data from the MDIO Read/Write Data
-	 * register and return.
-	 */
-	data = IXGB_READ_REG(hw, MSRWD);
-	data >>= IXGB_MSRWD_READ_DATA_SHIFT;
-	return((u16) data);
-}
-
-/******************************************************************************
- * Writes a word to a device over the Management Data Interface (MDI) bus.
- * This interface is used to manage Physical layer devices.
- *
- * hw          - Struct containing variables accessed by hw code
- * reg_address - Offset of device register being read.
- * phy_address - Address of device on MDI.
- * device_type - Also known as the Device ID or DID.
- * data        - 16-bit value to be written
- *
- * Returns:  void.
- *
- * The 82597EX has support for several MDI access methods.  This routine
- * uses the new protocol MDI Single Command and Address Operation.
- * This requires that first an address cycle command is sent, followed by a
- * write command.
- *****************************************************************************/
-static void
-ixgb_write_phy_reg(struct ixgb_hw *hw,
-			u32 reg_address,
-			u32 phy_address,
-			u32 device_type,
-			u16 data)
-{
-	u32 i;
-	u32 command = 0;
-
-	ASSERT(reg_address <= IXGB_MAX_PHY_REG_ADDRESS);
-	ASSERT(phy_address <= IXGB_MAX_PHY_ADDRESS);
-	ASSERT(device_type <= IXGB_MAX_PHY_DEV_TYPE);
-
-	/* Put the data in the MDIO Read/Write Data register */
-	IXGB_WRITE_REG(hw, MSRWD, (u32)data);
-
-	/* Setup and write the address cycle command */
-	command = ((reg_address << IXGB_MSCA_NP_ADDR_SHIFT)  |
-			   (device_type << IXGB_MSCA_DEV_TYPE_SHIFT) |
-			   (phy_address << IXGB_MSCA_PHY_ADDR_SHIFT) |
-			   (IXGB_MSCA_ADDR_CYCLE | IXGB_MSCA_MDI_COMMAND));
-
-	IXGB_WRITE_REG(hw, MSCA, command);
-
-	/**************************************************************
-	** Check every 10 usec to see if the address cycle completed
-	** The COMMAND bit will clear when the operation is complete.
-	** This may take as long as 64 usecs (we'll wait 100 usecs max)
-	** from the CPU Write to the Ready bit assertion.
-	**************************************************************/
-
-	for (i = 0; i < 10; i++)
-	{
-		udelay(10);
-
-		command = IXGB_READ_REG(hw, MSCA);
-
-		if ((command & IXGB_MSCA_MDI_COMMAND) == 0)
-			break;
-	}
-
-	ASSERT((command & IXGB_MSCA_MDI_COMMAND) == 0);
-
-	/* Address cycle complete, setup and write the write command */
-	command = ((reg_address << IXGB_MSCA_NP_ADDR_SHIFT)  |
-			   (device_type << IXGB_MSCA_DEV_TYPE_SHIFT) |
-			   (phy_address << IXGB_MSCA_PHY_ADDR_SHIFT) |
-			   (IXGB_MSCA_WRITE | IXGB_MSCA_MDI_COMMAND));
-
-	IXGB_WRITE_REG(hw, MSCA, command);
-
-	/**************************************************************
-	** Check every 10 usec to see if the read command completed
-	** The COMMAND bit will clear when the operation is complete.
-	** The write may take as long as 64 usecs (we'll wait 100 usecs max)
-	** from the CPU Write to the Ready bit assertion.
-	**************************************************************/
-
-	for (i = 0; i < 10; i++)
-	{
-		udelay(10);
-
-		command = IXGB_READ_REG(hw, MSCA);
-
-		if ((command & IXGB_MSCA_MDI_COMMAND) == 0)
-			break;
-	}
-
-	ASSERT((command & IXGB_MSCA_MDI_COMMAND) == 0);
-
-	/* Operation is complete, return. */
-}
-
-/******************************************************************************
- * Checks to see if the link status of the hardware has changed.
- *
- * hw - Struct containing variables accessed by hw code
- *
- * Called by any function that needs to check the link status of the adapter.
- *****************************************************************************/
-void
-ixgb_check_for_link(struct ixgb_hw *hw)
-{
-	u32 status_reg;
-	u32 xpcss_reg;
-
-	ENTER();
-
-	xpcss_reg = IXGB_READ_REG(hw, XPCSS);
-	status_reg = IXGB_READ_REG(hw, STATUS);
-
-	if ((xpcss_reg & IXGB_XPCSS_ALIGN_STATUS) &&
-	    (status_reg & IXGB_STATUS_LU)) {
-		hw->link_up = true;
-	} else if (!(xpcss_reg & IXGB_XPCSS_ALIGN_STATUS) &&
-		   (status_reg & IXGB_STATUS_LU)) {
-		pr_debug("XPCSS Not Aligned while Status:LU is set\n");
-		hw->link_up = ixgb_link_reset(hw);
-	} else {
-		/*
-		 * 82597EX errata.  Since the lane deskew problem may prevent
-		 * link, reset the link before reporting link down.
-		 */
-		hw->link_up = ixgb_link_reset(hw);
-	}
-	/*  Anything else for 10 Gig?? */
-}
-
-/******************************************************************************
- * Check for a bad link condition that may have occurred.
- * The indication is that the RFC / LFC registers may be incrementing
- * continually.  A full adapter reset is required to recover.
- *
- * hw - Struct containing variables accessed by hw code
- *
- * Called by any function that needs to check the link status of the adapter.
- *****************************************************************************/
-bool ixgb_check_for_bad_link(struct ixgb_hw *hw)
-{
-	u32 newLFC, newRFC;
-	bool bad_link_returncode = false;
-
-	if (hw->phy_type == ixgb_phy_type_txn17401) {
-		newLFC = IXGB_READ_REG(hw, LFC);
-		newRFC = IXGB_READ_REG(hw, RFC);
-		if ((hw->lastLFC + 250 < newLFC)
-		    || (hw->lastRFC + 250 < newRFC)) {
-			pr_debug("BAD LINK! too many LFC/RFC since last check\n");
-			bad_link_returncode = true;
-		}
-		hw->lastLFC = newLFC;
-		hw->lastRFC = newRFC;
-	}
-
-	return bad_link_returncode;
-}
-
-/******************************************************************************
- * Clears all hardware statistics counters.
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-static void
-ixgb_clear_hw_cntrs(struct ixgb_hw *hw)
-{
-	ENTER();
-
-	/* if we are stopped or resetting exit gracefully */
-	if (hw->adapter_stopped) {
-		pr_debug("Exiting because the adapter is stopped!!!\n");
-		return;
-	}
-
-	IXGB_READ_REG(hw, TPRL);
-	IXGB_READ_REG(hw, TPRH);
-	IXGB_READ_REG(hw, GPRCL);
-	IXGB_READ_REG(hw, GPRCH);
-	IXGB_READ_REG(hw, BPRCL);
-	IXGB_READ_REG(hw, BPRCH);
-	IXGB_READ_REG(hw, MPRCL);
-	IXGB_READ_REG(hw, MPRCH);
-	IXGB_READ_REG(hw, UPRCL);
-	IXGB_READ_REG(hw, UPRCH);
-	IXGB_READ_REG(hw, VPRCL);
-	IXGB_READ_REG(hw, VPRCH);
-	IXGB_READ_REG(hw, JPRCL);
-	IXGB_READ_REG(hw, JPRCH);
-	IXGB_READ_REG(hw, GORCL);
-	IXGB_READ_REG(hw, GORCH);
-	IXGB_READ_REG(hw, TORL);
-	IXGB_READ_REG(hw, TORH);
-	IXGB_READ_REG(hw, RNBC);
-	IXGB_READ_REG(hw, RUC);
-	IXGB_READ_REG(hw, ROC);
-	IXGB_READ_REG(hw, RLEC);
-	IXGB_READ_REG(hw, CRCERRS);
-	IXGB_READ_REG(hw, ICBC);
-	IXGB_READ_REG(hw, ECBC);
-	IXGB_READ_REG(hw, MPC);
-	IXGB_READ_REG(hw, TPTL);
-	IXGB_READ_REG(hw, TPTH);
-	IXGB_READ_REG(hw, GPTCL);
-	IXGB_READ_REG(hw, GPTCH);
-	IXGB_READ_REG(hw, BPTCL);
-	IXGB_READ_REG(hw, BPTCH);
-	IXGB_READ_REG(hw, MPTCL);
-	IXGB_READ_REG(hw, MPTCH);
-	IXGB_READ_REG(hw, UPTCL);
-	IXGB_READ_REG(hw, UPTCH);
-	IXGB_READ_REG(hw, VPTCL);
-	IXGB_READ_REG(hw, VPTCH);
-	IXGB_READ_REG(hw, JPTCL);
-	IXGB_READ_REG(hw, JPTCH);
-	IXGB_READ_REG(hw, GOTCL);
-	IXGB_READ_REG(hw, GOTCH);
-	IXGB_READ_REG(hw, TOTL);
-	IXGB_READ_REG(hw, TOTH);
-	IXGB_READ_REG(hw, DC);
-	IXGB_READ_REG(hw, PLT64C);
-	IXGB_READ_REG(hw, TSCTC);
-	IXGB_READ_REG(hw, TSCTFC);
-	IXGB_READ_REG(hw, IBIC);
-	IXGB_READ_REG(hw, RFC);
-	IXGB_READ_REG(hw, LFC);
-	IXGB_READ_REG(hw, PFRC);
-	IXGB_READ_REG(hw, PFTC);
-	IXGB_READ_REG(hw, MCFRC);
-	IXGB_READ_REG(hw, MCFTC);
-	IXGB_READ_REG(hw, XONRXC);
-	IXGB_READ_REG(hw, XONTXC);
-	IXGB_READ_REG(hw, XOFFRXC);
-	IXGB_READ_REG(hw, XOFFTXC);
-	IXGB_READ_REG(hw, RJC);
-}
-
-/******************************************************************************
- * Turns on the software controllable LED
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-void
-ixgb_led_on(struct ixgb_hw *hw)
-{
-	u32 ctrl0_reg = IXGB_READ_REG(hw, CTRL0);
-
-	/* To turn on the LED, clear software-definable pin 0 (SDP0). */
-	ctrl0_reg &= ~IXGB_CTRL0_SDP0;
-	IXGB_WRITE_REG(hw, CTRL0, ctrl0_reg);
-}
-
-/******************************************************************************
- * Turns off the software controllable LED
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-void
-ixgb_led_off(struct ixgb_hw *hw)
-{
-	u32 ctrl0_reg = IXGB_READ_REG(hw, CTRL0);
-
-	/* To turn off the LED, set software-definable pin 0 (SDP0). */
-	ctrl0_reg |= IXGB_CTRL0_SDP0;
-	IXGB_WRITE_REG(hw, CTRL0, ctrl0_reg);
-}
-
-/******************************************************************************
- * Gets the current PCI bus type, speed, and width of the hardware
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-static void
-ixgb_get_bus_info(struct ixgb_hw *hw)
-{
-	u32 status_reg;
-
-	status_reg = IXGB_READ_REG(hw, STATUS);
-
-	hw->bus.type = (status_reg & IXGB_STATUS_PCIX_MODE) ?
-		ixgb_bus_type_pcix : ixgb_bus_type_pci;
-
-	if (hw->bus.type == ixgb_bus_type_pci) {
-		hw->bus.speed = (status_reg & IXGB_STATUS_PCI_SPD) ?
-			ixgb_bus_speed_66 : ixgb_bus_speed_33;
-	} else {
-		switch (status_reg & IXGB_STATUS_PCIX_SPD_MASK) {
-		case IXGB_STATUS_PCIX_SPD_66:
-			hw->bus.speed = ixgb_bus_speed_66;
-			break;
-		case IXGB_STATUS_PCIX_SPD_100:
-			hw->bus.speed = ixgb_bus_speed_100;
-			break;
-		case IXGB_STATUS_PCIX_SPD_133:
-			hw->bus.speed = ixgb_bus_speed_133;
-			break;
-		default:
-			hw->bus.speed = ixgb_bus_speed_reserved;
-			break;
-		}
-	}
-
-	hw->bus.width = (status_reg & IXGB_STATUS_BUS64) ?
-		ixgb_bus_width_64 : ixgb_bus_width_32;
-}
-
-/******************************************************************************
- * Tests a MAC address to ensure it is a valid Individual Address
- *
- * mac_addr - pointer to MAC address.
- *
- *****************************************************************************/
-static bool
-mac_addr_valid(u8 *mac_addr)
-{
-	bool is_valid = true;
-	ENTER();
-
-	/* Make sure it is not a multicast address */
-	if (is_multicast_ether_addr(mac_addr)) {
-		pr_debug("MAC address is multicast\n");
-		is_valid = false;
-	}
-	/* Not a broadcast address */
-	else if (is_broadcast_ether_addr(mac_addr)) {
-		pr_debug("MAC address is broadcast\n");
-		is_valid = false;
-	}
-	/* Reject the zero address */
-	else if (is_zero_ether_addr(mac_addr)) {
-		pr_debug("MAC address is all zeros\n");
-		is_valid = false;
-	}
-	return is_valid;
-}
-
-/******************************************************************************
- * Resets the 10GbE link.  Waits the settle time and returns the state of
- * the link.
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-static bool
-ixgb_link_reset(struct ixgb_hw *hw)
-{
-	bool link_status = false;
-	u8 wait_retries = MAX_RESET_ITERATIONS;
-	u8 lrst_retries = MAX_RESET_ITERATIONS;
-
-	do {
-		/* Reset the link */
-		IXGB_WRITE_REG(hw, CTRL0,
-			       IXGB_READ_REG(hw, CTRL0) | IXGB_CTRL0_LRST);
-
-		/* Wait for link-up and lane re-alignment */
-		do {
-			udelay(IXGB_DELAY_USECS_AFTER_LINK_RESET);
-			link_status =
-			    ((IXGB_READ_REG(hw, STATUS) & IXGB_STATUS_LU)
-			     && (IXGB_READ_REG(hw, XPCSS) &
-				 IXGB_XPCSS_ALIGN_STATUS)) ? true : false;
-		} while (!link_status && --wait_retries);
-
-	} while (!link_status && --lrst_retries);
-
-	return link_status;
-}
-
-/******************************************************************************
- * Resets the 10GbE optics module.
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-static void
-ixgb_optics_reset(struct ixgb_hw *hw)
-{
-	if (hw->phy_type == ixgb_phy_type_txn17401) {
-		ixgb_write_phy_reg(hw,
-				   MDIO_CTRL1,
-				   IXGB_PHY_ADDRESS,
-				   MDIO_MMD_PMAPMD,
-				   MDIO_CTRL1_RESET);
-
-		ixgb_read_phy_reg(hw, MDIO_CTRL1, IXGB_PHY_ADDRESS, MDIO_MMD_PMAPMD);
-	}
-}
-
-/******************************************************************************
- * Resets the 10GbE optics module for Sun variant NIC.
- *
- * hw - Struct containing variables accessed by shared code
- *****************************************************************************/
-
-#define   IXGB_BCM8704_USER_PMD_TX_CTRL_REG         0xC803
-#define   IXGB_BCM8704_USER_PMD_TX_CTRL_REG_VAL     0x0164
-#define   IXGB_BCM8704_USER_CTRL_REG                0xC800
-#define   IXGB_BCM8704_USER_CTRL_REG_VAL            0x7FBF
-#define   IXGB_BCM8704_USER_DEV3_ADDR               0x0003
-#define   IXGB_SUN_PHY_ADDRESS                      0x0000
-#define   IXGB_SUN_PHY_RESET_DELAY                     305
-
-static void
-ixgb_optics_reset_bcm(struct ixgb_hw *hw)
-{
-	u32 ctrl = IXGB_READ_REG(hw, CTRL0);
-	ctrl &= ~IXGB_CTRL0_SDP2;
-	ctrl |= IXGB_CTRL0_SDP3;
-	IXGB_WRITE_REG(hw, CTRL0, ctrl);
-	IXGB_WRITE_FLUSH(hw);
-
-	/* SerDes needs extra delay */
-	msleep(IXGB_SUN_PHY_RESET_DELAY);
-
-	/* Broadcom 7408L configuration */
-	/* Reference clock config */
-	ixgb_write_phy_reg(hw,
-			   IXGB_BCM8704_USER_PMD_TX_CTRL_REG,
-			   IXGB_SUN_PHY_ADDRESS,
-			   IXGB_BCM8704_USER_DEV3_ADDR,
-			   IXGB_BCM8704_USER_PMD_TX_CTRL_REG_VAL);
-	/*  we must read the registers twice */
-	ixgb_read_phy_reg(hw,
-			  IXGB_BCM8704_USER_PMD_TX_CTRL_REG,
-			  IXGB_SUN_PHY_ADDRESS,
-			  IXGB_BCM8704_USER_DEV3_ADDR);
-	ixgb_read_phy_reg(hw,
-			  IXGB_BCM8704_USER_PMD_TX_CTRL_REG,
-			  IXGB_SUN_PHY_ADDRESS,
-			  IXGB_BCM8704_USER_DEV3_ADDR);
-
-	ixgb_write_phy_reg(hw,
-			   IXGB_BCM8704_USER_CTRL_REG,
-			   IXGB_SUN_PHY_ADDRESS,
-			   IXGB_BCM8704_USER_DEV3_ADDR,
-			   IXGB_BCM8704_USER_CTRL_REG_VAL);
-	ixgb_read_phy_reg(hw,
-			  IXGB_BCM8704_USER_CTRL_REG,
-			  IXGB_SUN_PHY_ADDRESS,
-			  IXGB_BCM8704_USER_DEV3_ADDR);
-	ixgb_read_phy_reg(hw,
-			  IXGB_BCM8704_USER_CTRL_REG,
-			  IXGB_SUN_PHY_ADDRESS,
-			  IXGB_BCM8704_USER_DEV3_ADDR);
-
-	/* SerDes needs extra delay */
-	msleep(IXGB_SUN_PHY_RESET_DELAY);
-}
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_hw.h b/drivers/net/ethernet/intel/ixgb/ixgb_hw.h
deleted file mode 100644
index 70bcff5fb3db..000000000000
--- a/drivers/net/ethernet/intel/ixgb/ixgb_hw.h
+++ /dev/null
@@ -1,767 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* Copyright(c) 1999 - 2008 Intel Corporation. */
-
-#ifndef _IXGB_HW_H_
-#define _IXGB_HW_H_
-
-#include <linux/mdio.h>
-
-#include "ixgb_osdep.h"
-
-/* Enums */
-typedef enum {
-	ixgb_mac_unknown = 0,
-	ixgb_82597,
-	ixgb_num_macs
-} ixgb_mac_type;
-
-/* Types of physical layer modules */
-typedef enum {
-	ixgb_phy_type_unknown = 0,
-	ixgb_phy_type_g6005,	/* 850nm, MM fiber, XPAK transceiver */
-	ixgb_phy_type_g6104,	/* 1310nm, SM fiber, XPAK transceiver */
-	ixgb_phy_type_txn17201,	/* 850nm, MM fiber, XPAK transceiver */
-	ixgb_phy_type_txn17401,	/* 1310nm, SM fiber, XENPAK transceiver */
-	ixgb_phy_type_bcm	/* SUN specific board */
-} ixgb_phy_type;
-
-/* XPAK transceiver vendors, for the SR adapters */
-typedef enum {
-	ixgb_xpak_vendor_intel,
-	ixgb_xpak_vendor_infineon
-} ixgb_xpak_vendor;
-
-/* Media Types */
-typedef enum {
-	ixgb_media_type_unknown = 0,
-	ixgb_media_type_fiber = 1,
-	ixgb_media_type_copper = 2,
-	ixgb_num_media_types
-} ixgb_media_type;
-
-/* Flow Control Settings */
-typedef enum {
-	ixgb_fc_none = 0,
-	ixgb_fc_rx_pause = 1,
-	ixgb_fc_tx_pause = 2,
-	ixgb_fc_full = 3,
-	ixgb_fc_default = 0xFF
-} ixgb_fc_type;
-
-/* PCI bus types */
-typedef enum {
-	ixgb_bus_type_unknown = 0,
-	ixgb_bus_type_pci,
-	ixgb_bus_type_pcix
-} ixgb_bus_type;
-
-/* PCI bus speeds */
-typedef enum {
-	ixgb_bus_speed_unknown = 0,
-	ixgb_bus_speed_33,
-	ixgb_bus_speed_66,
-	ixgb_bus_speed_100,
-	ixgb_bus_speed_133,
-	ixgb_bus_speed_reserved
-} ixgb_bus_speed;
-
-/* PCI bus widths */
-typedef enum {
-	ixgb_bus_width_unknown = 0,
-	ixgb_bus_width_32,
-	ixgb_bus_width_64
-} ixgb_bus_width;
-
-#define IXGB_EEPROM_SIZE    64	/* Size in words */
-
-#define SPEED_10000  10000
-#define FULL_DUPLEX  2
-
-#define MIN_NUMBER_OF_DESCRIPTORS       8
-#define MAX_NUMBER_OF_DESCRIPTORS  0xFFF8	/* 13 bits in RDLEN/TDLEN, 128B aligned     */
-
-#define IXGB_DELAY_BEFORE_RESET        10	/* allow 10ms after idling rx/tx units      */
-#define IXGB_DELAY_AFTER_RESET          1	/* allow 1ms after the reset                */
-#define IXGB_DELAY_AFTER_EE_RESET      10	/* allow 10ms after the EEPROM reset        */
-
-#define IXGB_DELAY_USECS_AFTER_LINK_RESET    13	/* allow 13 microseconds after the reset    */
-					   /* NOTE: this is MICROSECONDS               */
-#define MAX_RESET_ITERATIONS            8	/* number of iterations to get things right */
-
-/* General Registers */
-#define IXGB_CTRL0   0x00000	/* Device Control Register 0 - RW */
-#define IXGB_CTRL1   0x00008	/* Device Control Register 1 - RW */
-#define IXGB_STATUS  0x00010	/* Device Status Register - RO */
-#define IXGB_EECD    0x00018	/* EEPROM/Flash Control/Data Register - RW */
-#define IXGB_MFS     0x00020	/* Maximum Frame Size - RW */
-
-/* Interrupt */
-#define IXGB_ICR     0x00080	/* Interrupt Cause Read - R/clr */
-#define IXGB_ICS     0x00088	/* Interrupt Cause Set - RW */
-#define IXGB_IMS     0x00090	/* Interrupt Mask Set/Read - RW */
-#define IXGB_IMC     0x00098	/* Interrupt Mask Clear - WO */
-
-/* Receive */
-#define IXGB_RCTL    0x00100	/* RX Control - RW */
-#define IXGB_FCRTL   0x00108	/* Flow Control Receive Threshold Low - RW */
-#define IXGB_FCRTH   0x00110	/* Flow Control Receive Threshold High - RW */
-#define IXGB_RDBAL   0x00118	/* RX Descriptor Base Low - RW */
-#define IXGB_RDBAH   0x0011C	/* RX Descriptor Base High - RW */
-#define IXGB_RDLEN   0x00120	/* RX Descriptor Length - RW */
-#define IXGB_RDH     0x00128	/* RX Descriptor Head - RW */
-#define IXGB_RDT     0x00130	/* RX Descriptor Tail - RW */
-#define IXGB_RDTR    0x00138	/* RX Delay Timer Ring - RW */
-#define IXGB_RXDCTL  0x00140	/* Receive Descriptor Control - RW */
-#define IXGB_RAIDC   0x00148	/* Receive Adaptive Interrupt Delay Control - RW */
-#define IXGB_RXCSUM  0x00158	/* Receive Checksum Control - RW */
-#define IXGB_RA      0x00180	/* Receive Address Array Base - RW */
-#define IXGB_RAL     0x00180	/* Receive Address Low [0:15] - RW */
-#define IXGB_RAH     0x00184	/* Receive Address High [0:15] - RW */
-#define IXGB_MTA     0x00200	/* Multicast Table Array [0:127] - RW */
-#define IXGB_VFTA    0x00400	/* VLAN Filter Table Array [0:127] - RW */
-#define IXGB_REQ_RX_DESCRIPTOR_MULTIPLE 8
-
-/* Transmit */
-#define IXGB_TCTL    0x00600	/* TX Control - RW */
-#define IXGB_TDBAL   0x00608	/* TX Descriptor Base Low - RW */
-#define IXGB_TDBAH   0x0060C	/* TX Descriptor Base High - RW */
-#define IXGB_TDLEN   0x00610	/* TX Descriptor Length - RW */
-#define IXGB_TDH     0x00618	/* TX Descriptor Head - RW */
-#define IXGB_TDT     0x00620	/* TX Descriptor Tail - RW */
-#define IXGB_TIDV    0x00628	/* TX Interrupt Delay Value - RW */
-#define IXGB_TXDCTL  0x00630	/* Transmit Descriptor Control - RW */
-#define IXGB_TSPMT   0x00638	/* TCP Segmentation PAD & Min Threshold - RW */
-#define IXGB_PAP     0x00640	/* Pause and Pace - RW */
-#define IXGB_REQ_TX_DESCRIPTOR_MULTIPLE 8
-
-/* Physical */
-#define IXGB_PCSC1   0x00700	/* PCS Control 1 - RW */
-#define IXGB_PCSC2   0x00708	/* PCS Control 2 - RW */
-#define IXGB_PCSS1   0x00710	/* PCS Status 1 - RO */
-#define IXGB_PCSS2   0x00718	/* PCS Status 2 - RO */
-#define IXGB_XPCSS   0x00720	/* 10GBASE-X PCS Status (or XGXS Lane Status) - RO */
-#define IXGB_UCCR    0x00728	/* Unilink Circuit Control Register */
-#define IXGB_XPCSTC  0x00730	/* 10GBASE-X PCS Test Control */
-#define IXGB_MACA    0x00738	/* MDI Autoscan Command and Address - RW */
-#define IXGB_APAE    0x00740	/* Autoscan PHY Address Enable - RW */
-#define IXGB_ARD     0x00748	/* Autoscan Read Data - RO */
-#define IXGB_AIS     0x00750	/* Autoscan Interrupt Status - RO */
-#define IXGB_MSCA    0x00758	/* MDI Single Command and Address - RW */
-#define IXGB_MSRWD   0x00760	/* MDI Single Read and Write Data - RW, RO */
-
-/* Wake-up */
-#define IXGB_WUFC    0x00808	/* Wake Up Filter Control - RW */
-#define IXGB_WUS     0x00810	/* Wake Up Status - RO */
-#define IXGB_FFLT    0x01000	/* Flexible Filter Length Table - RW */
-#define IXGB_FFMT    0x01020	/* Flexible Filter Mask Table - RW */
-#define IXGB_FTVT    0x01420	/* Flexible Filter Value Table - RW */
-
-/* Statistics */
-#define IXGB_TPRL    0x02000	/* Total Packets Received (Low) */
-#define IXGB_TPRH    0x02004	/* Total Packets Received (High) */
-#define IXGB_GPRCL   0x02008	/* Good Packets Received Count (Low) */
-#define IXGB_GPRCH   0x0200C	/* Good Packets Received Count (High) */
-#define IXGB_BPRCL   0x02010	/* Broadcast Packets Received Count (Low) */
-#define IXGB_BPRCH   0x02014	/* Broadcast Packets Received Count (High) */
-#define IXGB_MPRCL   0x02018	/* Multicast Packets Received Count (Low) */
-#define IXGB_MPRCH   0x0201C	/* Multicast Packets Received Count (High) */
-#define IXGB_UPRCL   0x02020	/* Unicast Packets Received Count (Low) */
-#define IXGB_UPRCH   0x02024	/* Unicast Packets Received Count (High) */
-#define IXGB_VPRCL   0x02028	/* VLAN Packets Received Count (Low) */
-#define IXGB_VPRCH   0x0202C	/* VLAN Packets Received Count (High) */
-#define IXGB_JPRCL   0x02030	/* Jumbo Packets Received Count (Low) */
-#define IXGB_JPRCH   0x02034	/* Jumbo Packets Received Count (High) */
-#define IXGB_GORCL   0x02038	/* Good Octets Received Count (Low) */
-#define IXGB_GORCH   0x0203C	/* Good Octets Received Count (High) */
-#define IXGB_TORL    0x02040	/* Total Octets Received (Low) */
-#define IXGB_TORH    0x02044	/* Total Octets Received (High) */
-#define IXGB_RNBC    0x02048	/* Receive No Buffers Count */
-#define IXGB_RUC     0x02050	/* Receive Undersize Count */
-#define IXGB_ROC     0x02058	/* Receive Oversize Count */
-#define IXGB_RLEC    0x02060	/* Receive Length Error Count */
-#define IXGB_CRCERRS 0x02068	/* CRC Error Count */
-#define IXGB_ICBC    0x02070	/* Illegal control byte in mid-packet Count */
-#define IXGB_ECBC    0x02078	/* Error Control byte in mid-packet Count */
-#define IXGB_MPC     0x02080	/* Missed Packets Count */
-#define IXGB_TPTL    0x02100	/* Total Packets Transmitted (Low) */
-#define IXGB_TPTH    0x02104	/* Total Packets Transmitted (High) */
-#define IXGB_GPTCL   0x02108	/* Good Packets Transmitted Count (Low) */
-#define IXGB_GPTCH   0x0210C	/* Good Packets Transmitted Count (High) */
-#define IXGB_BPTCL   0x02110	/* Broadcast Packets Transmitted Count (Low) */
-#define IXGB_BPTCH   0x02114	/* Broadcast Packets Transmitted Count (High) */
-#define IXGB_MPTCL   0x02118	/* Multicast Packets Transmitted Count (Low) */
-#define IXGB_MPTCH   0x0211C	/* Multicast Packets Transmitted Count (High) */
-#define IXGB_UPTCL   0x02120	/* Unicast Packets Transmitted Count (Low) */
-#define IXGB_UPTCH   0x02124	/* Unicast Packets Transmitted Count (High) */
-#define IXGB_VPTCL   0x02128	/* VLAN Packets Transmitted Count (Low) */
-#define IXGB_VPTCH   0x0212C	/* VLAN Packets Transmitted Count (High) */
-#define IXGB_JPTCL   0x02130	/* Jumbo Packets Transmitted Count (Low) */
-#define IXGB_JPTCH   0x02134	/* Jumbo Packets Transmitted Count (High) */
-#define IXGB_GOTCL   0x02138	/* Good Octets Transmitted Count (Low) */
-#define IXGB_GOTCH   0x0213C	/* Good Octets Transmitted Count (High) */
-#define IXGB_TOTL    0x02140	/* Total Octets Transmitted Count (Low) */
-#define IXGB_TOTH    0x02144	/* Total Octets Transmitted Count (High) */
-#define IXGB_DC      0x02148	/* Defer Count */
-#define IXGB_PLT64C  0x02150	/* Packet Transmitted was less than 64 bytes Count */
-#define IXGB_TSCTC   0x02170	/* TCP Segmentation Context Transmitted Count */
-#define IXGB_TSCTFC  0x02178	/* TCP Segmentation Context Tx Fail Count */
-#define IXGB_IBIC    0x02180	/* Illegal byte during Idle stream count */
-#define IXGB_RFC     0x02188	/* Remote Fault Count */
-#define IXGB_LFC     0x02190	/* Local Fault Count */
-#define IXGB_PFRC    0x02198	/* Pause Frame Receive Count */
-#define IXGB_PFTC    0x021A0	/* Pause Frame Transmit Count */
-#define IXGB_MCFRC   0x021A8	/* MAC Control Frames (non-Pause) Received Count */
-#define IXGB_MCFTC   0x021B0	/* MAC Control Frames (non-Pause) Transmitted Count */
-#define IXGB_XONRXC  0x021B8	/* XON Received Count */
-#define IXGB_XONTXC  0x021C0	/* XON Transmitted Count */
-#define IXGB_XOFFRXC 0x021C8	/* XOFF Received Count */
-#define IXGB_XOFFTXC 0x021D0	/* XOFF Transmitted Count */
-#define IXGB_RJC     0x021D8	/* Receive Jabber Count */
-
-/* CTRL0 Bit Masks */
-#define IXGB_CTRL0_LRST     0x00000008
-#define IXGB_CTRL0_JFE      0x00000010
-#define IXGB_CTRL0_XLE      0x00000020
-#define IXGB_CTRL0_MDCS     0x00000040
-#define IXGB_CTRL0_CMDC     0x00000080
-#define IXGB_CTRL0_SDP0     0x00040000
-#define IXGB_CTRL0_SDP1     0x00080000
-#define IXGB_CTRL0_SDP2     0x00100000
-#define IXGB_CTRL0_SDP3     0x00200000
-#define IXGB_CTRL0_SDP0_DIR 0x00400000
-#define IXGB_CTRL0_SDP1_DIR 0x00800000
-#define IXGB_CTRL0_SDP2_DIR 0x01000000
-#define IXGB_CTRL0_SDP3_DIR 0x02000000
-#define IXGB_CTRL0_RST      0x04000000
-#define IXGB_CTRL0_RPE      0x08000000
-#define IXGB_CTRL0_TPE      0x10000000
-#define IXGB_CTRL0_VME      0x40000000
-
-/* CTRL1 Bit Masks */
-#define IXGB_CTRL1_GPI0_EN     0x00000001
-#define IXGB_CTRL1_GPI1_EN     0x00000002
-#define IXGB_CTRL1_GPI2_EN     0x00000004
-#define IXGB_CTRL1_GPI3_EN     0x00000008
-#define IXGB_CTRL1_SDP4        0x00000010
-#define IXGB_CTRL1_SDP5        0x00000020
-#define IXGB_CTRL1_SDP6        0x00000040
-#define IXGB_CTRL1_SDP7        0x00000080
-#define IXGB_CTRL1_SDP4_DIR    0x00000100
-#define IXGB_CTRL1_SDP5_DIR    0x00000200
-#define IXGB_CTRL1_SDP6_DIR    0x00000400
-#define IXGB_CTRL1_SDP7_DIR    0x00000800
-#define IXGB_CTRL1_EE_RST      0x00002000
-#define IXGB_CTRL1_RO_DIS      0x00020000
-#define IXGB_CTRL1_PCIXHM_MASK 0x00C00000
-#define IXGB_CTRL1_PCIXHM_1_2  0x00000000
-#define IXGB_CTRL1_PCIXHM_5_8  0x00400000
-#define IXGB_CTRL1_PCIXHM_3_4  0x00800000
-#define IXGB_CTRL1_PCIXHM_7_8  0x00C00000
-
-/* STATUS Bit Masks */
-#define IXGB_STATUS_LU            0x00000002
-#define IXGB_STATUS_AIP           0x00000004
-#define IXGB_STATUS_TXOFF         0x00000010
-#define IXGB_STATUS_XAUIME        0x00000020
-#define IXGB_STATUS_RES           0x00000040
-#define IXGB_STATUS_RIS           0x00000080
-#define IXGB_STATUS_RIE           0x00000100
-#define IXGB_STATUS_RLF           0x00000200
-#define IXGB_STATUS_RRF           0x00000400
-#define IXGB_STATUS_PCI_SPD       0x00000800
-#define IXGB_STATUS_BUS64         0x00001000
-#define IXGB_STATUS_PCIX_MODE     0x00002000
-#define IXGB_STATUS_PCIX_SPD_MASK 0x0000C000
-#define IXGB_STATUS_PCIX_SPD_66   0x00000000
-#define IXGB_STATUS_PCIX_SPD_100  0x00004000
-#define IXGB_STATUS_PCIX_SPD_133  0x00008000
-#define IXGB_STATUS_REV_ID_MASK   0x000F0000
-#define IXGB_STATUS_REV_ID_SHIFT  16
-
-/* EECD Bit Masks */
-#define IXGB_EECD_SK       0x00000001
-#define IXGB_EECD_CS       0x00000002
-#define IXGB_EECD_DI       0x00000004
-#define IXGB_EECD_DO       0x00000008
-#define IXGB_EECD_FWE_MASK 0x00000030
-#define IXGB_EECD_FWE_DIS  0x00000010
-#define IXGB_EECD_FWE_EN   0x00000020
-
-/* MFS */
-#define IXGB_MFS_SHIFT 16
-
-/* Interrupt Register Bit Masks (used for ICR, ICS, IMS, and IMC) */
-#define IXGB_INT_TXDW     0x00000001
-#define IXGB_INT_TXQE     0x00000002
-#define IXGB_INT_LSC      0x00000004
-#define IXGB_INT_RXSEQ    0x00000008
-#define IXGB_INT_RXDMT0   0x00000010
-#define IXGB_INT_RXO      0x00000040
-#define IXGB_INT_RXT0     0x00000080
-#define IXGB_INT_AUTOSCAN 0x00000200
-#define IXGB_INT_GPI0     0x00000800
-#define IXGB_INT_GPI1     0x00001000
-#define IXGB_INT_GPI2     0x00002000
-#define IXGB_INT_GPI3     0x00004000
-
-/* RCTL Bit Masks */
-#define IXGB_RCTL_RXEN        0x00000002
-#define IXGB_RCTL_SBP         0x00000004
-#define IXGB_RCTL_UPE         0x00000008
-#define IXGB_RCTL_MPE         0x00000010
-#define IXGB_RCTL_RDMTS_MASK  0x00000300
-#define IXGB_RCTL_RDMTS_1_2   0x00000000
-#define IXGB_RCTL_RDMTS_1_4   0x00000100
-#define IXGB_RCTL_RDMTS_1_8   0x00000200
-#define IXGB_RCTL_MO_MASK     0x00003000
-#define IXGB_RCTL_MO_47_36    0x00000000
-#define IXGB_RCTL_MO_46_35    0x00001000
-#define IXGB_RCTL_MO_45_34    0x00002000
-#define IXGB_RCTL_MO_43_32    0x00003000
-#define IXGB_RCTL_MO_SHIFT    12
-#define IXGB_RCTL_BAM         0x00008000
-#define IXGB_RCTL_BSIZE_MASK  0x00030000
-#define IXGB_RCTL_BSIZE_2048  0x00000000
-#define IXGB_RCTL_BSIZE_4096  0x00010000
-#define IXGB_RCTL_BSIZE_8192  0x00020000
-#define IXGB_RCTL_BSIZE_16384 0x00030000
-#define IXGB_RCTL_VFE         0x00040000
-#define IXGB_RCTL_CFIEN       0x00080000
-#define IXGB_RCTL_CFI         0x00100000
-#define IXGB_RCTL_RPDA_MASK   0x00600000
-#define IXGB_RCTL_RPDA_MC_MAC 0x00000000
-#define IXGB_RCTL_MC_ONLY     0x00400000
-#define IXGB_RCTL_CFF         0x00800000
-#define IXGB_RCTL_SECRC       0x04000000
-#define IXGB_RDT_FPDB         0x80000000
-
-#define IXGB_RCTL_IDLE_RX_UNIT 0
-
-/* FCRTL Bit Masks */
-#define IXGB_FCRTL_XONE       0x80000000
-
-/* RXDCTL Bit Masks */
-#define IXGB_RXDCTL_PTHRESH_MASK  0x000001FF
-#define IXGB_RXDCTL_PTHRESH_SHIFT 0
-#define IXGB_RXDCTL_HTHRESH_MASK  0x0003FE00
-#define IXGB_RXDCTL_HTHRESH_SHIFT 9
-#define IXGB_RXDCTL_WTHRESH_MASK  0x07FC0000
-#define IXGB_RXDCTL_WTHRESH_SHIFT 18
-
-/* RAIDC Bit Masks */
-#define IXGB_RAIDC_HIGHTHRS_MASK 0x0000003F
-#define IXGB_RAIDC_DELAY_MASK    0x000FF800
-#define IXGB_RAIDC_DELAY_SHIFT   11
-#define IXGB_RAIDC_POLL_MASK     0x1FF00000
-#define IXGB_RAIDC_POLL_SHIFT    20
-#define IXGB_RAIDC_RXT_GATE      0x40000000
-#define IXGB_RAIDC_EN            0x80000000
-
-#define IXGB_RAIDC_POLL_1000_INTERRUPTS_PER_SECOND      1220
-#define IXGB_RAIDC_POLL_5000_INTERRUPTS_PER_SECOND      244
-#define IXGB_RAIDC_POLL_10000_INTERRUPTS_PER_SECOND     122
-#define IXGB_RAIDC_POLL_20000_INTERRUPTS_PER_SECOND     61
-
-/* RXCSUM Bit Masks */
-#define IXGB_RXCSUM_IPOFL 0x00000100
-#define IXGB_RXCSUM_TUOFL 0x00000200
-
-/* RAH Bit Masks */
-#define IXGB_RAH_ASEL_MASK 0x00030000
-#define IXGB_RAH_ASEL_DEST 0x00000000
-#define IXGB_RAH_ASEL_SRC  0x00010000
-#define IXGB_RAH_AV        0x80000000
-
-/* TCTL Bit Masks */
-#define IXGB_TCTL_TCE  0x00000001
-#define IXGB_TCTL_TXEN 0x00000002
-#define IXGB_TCTL_TPDE 0x00000004
-
-#define IXGB_TCTL_IDLE_TX_UNIT  0
-
-/* TXDCTL Bit Masks */
-#define IXGB_TXDCTL_PTHRESH_MASK  0x0000007F
-#define IXGB_TXDCTL_HTHRESH_MASK  0x00007F00
-#define IXGB_TXDCTL_HTHRESH_SHIFT 8
-#define IXGB_TXDCTL_WTHRESH_MASK  0x007F0000
-#define IXGB_TXDCTL_WTHRESH_SHIFT 16
-
-/* TSPMT Bit Masks */
-#define IXGB_TSPMT_TSMT_MASK   0x0000FFFF
-#define IXGB_TSPMT_TSPBP_MASK  0xFFFF0000
-#define IXGB_TSPMT_TSPBP_SHIFT 16
-
-/* PAP Bit Masks */
-#define IXGB_PAP_TXPC_MASK 0x0000FFFF
-#define IXGB_PAP_TXPV_MASK 0x000F0000
-#define IXGB_PAP_TXPV_10G  0x00000000
-#define IXGB_PAP_TXPV_1G   0x00010000
-#define IXGB_PAP_TXPV_2G   0x00020000
-#define IXGB_PAP_TXPV_3G   0x00030000
-#define IXGB_PAP_TXPV_4G   0x00040000
-#define IXGB_PAP_TXPV_5G   0x00050000
-#define IXGB_PAP_TXPV_6G   0x00060000
-#define IXGB_PAP_TXPV_7G   0x00070000
-#define IXGB_PAP_TXPV_8G   0x00080000
-#define IXGB_PAP_TXPV_9G   0x00090000
-#define IXGB_PAP_TXPV_WAN  0x000F0000
-
-/* PCSC1 Bit Masks */
-#define IXGB_PCSC1_LOOPBACK 0x00004000
-
-/* PCSC2 Bit Masks */
-#define IXGB_PCSC2_PCS_TYPE_MASK  0x00000003
-#define IXGB_PCSC2_PCS_TYPE_10GBX 0x00000001
-
-/* PCSS1 Bit Masks */
-#define IXGB_PCSS1_LOCAL_FAULT    0x00000080
-#define IXGB_PCSS1_RX_LINK_STATUS 0x00000004
-
-/* PCSS2 Bit Masks */
-#define IXGB_PCSS2_DEV_PRES_MASK 0x0000C000
-#define IXGB_PCSS2_DEV_PRES      0x00004000
-#define IXGB_PCSS2_TX_LF         0x00000800
-#define IXGB_PCSS2_RX_LF         0x00000400
-#define IXGB_PCSS2_10GBW         0x00000004
-#define IXGB_PCSS2_10GBX         0x00000002
-#define IXGB_PCSS2_10GBR         0x00000001
-
-/* XPCSS Bit Masks */
-#define IXGB_XPCSS_ALIGN_STATUS 0x00001000
-#define IXGB_XPCSS_PATTERN_TEST 0x00000800
-#define IXGB_XPCSS_LANE_3_SYNC  0x00000008
-#define IXGB_XPCSS_LANE_2_SYNC  0x00000004
-#define IXGB_XPCSS_LANE_1_SYNC  0x00000002
-#define IXGB_XPCSS_LANE_0_SYNC  0x00000001
-
-/* XPCSTC Bit Masks */
-#define IXGB_XPCSTC_BERT_TRIG       0x00200000
-#define IXGB_XPCSTC_BERT_SST        0x00100000
-#define IXGB_XPCSTC_BERT_PSZ_MASK   0x000C0000
-#define IXGB_XPCSTC_BERT_PSZ_SHIFT  17
-#define IXGB_XPCSTC_BERT_PSZ_INF    0x00000003
-#define IXGB_XPCSTC_BERT_PSZ_68     0x00000001
-#define IXGB_XPCSTC_BERT_PSZ_1028   0x00000000
-
-/* MSCA bit Masks */
-/* New Protocol Address */
-#define IXGB_MSCA_NP_ADDR_MASK      0x0000FFFF
-#define IXGB_MSCA_NP_ADDR_SHIFT     0
-/* Either Device Type or Register Address,depending on ST_CODE */
-#define IXGB_MSCA_DEV_TYPE_MASK     0x001F0000
-#define IXGB_MSCA_DEV_TYPE_SHIFT    16
-#define IXGB_MSCA_PHY_ADDR_MASK     0x03E00000
-#define IXGB_MSCA_PHY_ADDR_SHIFT    21
-#define IXGB_MSCA_OP_CODE_MASK      0x0C000000
-/* OP_CODE == 00, Address cycle, New Protocol           */
-/* OP_CODE == 01, Write operation                       */
-/* OP_CODE == 10, Read operation                        */
-/* OP_CODE == 11, Read, auto increment, New Protocol    */
-#define IXGB_MSCA_ADDR_CYCLE        0x00000000
-#define IXGB_MSCA_WRITE             0x04000000
-#define IXGB_MSCA_READ              0x08000000
-#define IXGB_MSCA_READ_AUTOINC      0x0C000000
-#define IXGB_MSCA_OP_CODE_SHIFT     26
-#define IXGB_MSCA_ST_CODE_MASK      0x30000000
-/* ST_CODE == 00, New Protocol  */
-/* ST_CODE == 01, Old Protocol  */
-#define IXGB_MSCA_NEW_PROTOCOL      0x00000000
-#define IXGB_MSCA_OLD_PROTOCOL      0x10000000
-#define IXGB_MSCA_ST_CODE_SHIFT     28
-/* Initiate command, self-clearing when command completes */
-#define IXGB_MSCA_MDI_COMMAND       0x40000000
-/*MDI In Progress Enable. */
-#define IXGB_MSCA_MDI_IN_PROG_EN    0x80000000
-
-/* MSRWD bit masks */
-#define IXGB_MSRWD_WRITE_DATA_MASK  0x0000FFFF
-#define IXGB_MSRWD_WRITE_DATA_SHIFT 0
-#define IXGB_MSRWD_READ_DATA_MASK   0xFFFF0000
-#define IXGB_MSRWD_READ_DATA_SHIFT  16
-
-/* Definitions for the optics devices on the MDIO bus. */
-#define IXGB_PHY_ADDRESS             0x0	/* Single PHY, multiple "Devices" */
-
-#define MDIO_PMA_PMD_XPAK_VENDOR_NAME       0x803A	/* XPAK/XENPAK devices only */
-
-/* Vendor-specific MDIO registers */
-#define G6XXX_PMA_PMD_VS1                   0xC001	/* Vendor-specific register */
-#define G6XXX_XGXS_XAUI_VS2                 0x18	/* Vendor-specific register */
-
-#define G6XXX_PMA_PMD_VS1_PLL_RESET         0x80
-#define G6XXX_PMA_PMD_VS1_REMOVE_PLL_RESET  0x00
-#define G6XXX_XGXS_XAUI_VS2_INPUT_MASK      0x0F	/* XAUI lanes synchronized */
-
-/* Layout of a single receive descriptor.  The controller assumes that this
- * structure is packed into 16 bytes, which is a safe assumption with most
- * compilers.  However, some compilers may insert padding between the fields,
- * in which case the structure must be packed in some compiler-specific
- * manner. */
-struct ixgb_rx_desc {
-	__le64 buff_addr;
-	__le16 length;
-	__le16 reserved;
-	u8 status;
-	u8 errors;
-	__le16 special;
-};
-
-#define IXGB_RX_DESC_STATUS_DD    0x01
-#define IXGB_RX_DESC_STATUS_EOP   0x02
-#define IXGB_RX_DESC_STATUS_IXSM  0x04
-#define IXGB_RX_DESC_STATUS_VP    0x08
-#define IXGB_RX_DESC_STATUS_TCPCS 0x20
-#define IXGB_RX_DESC_STATUS_IPCS  0x40
-#define IXGB_RX_DESC_STATUS_PIF   0x80
-
-#define IXGB_RX_DESC_ERRORS_CE   0x01
-#define IXGB_RX_DESC_ERRORS_SE   0x02
-#define IXGB_RX_DESC_ERRORS_P    0x08
-#define IXGB_RX_DESC_ERRORS_TCPE 0x20
-#define IXGB_RX_DESC_ERRORS_IPE  0x40
-#define IXGB_RX_DESC_ERRORS_RXE  0x80
-
-#define IXGB_RX_DESC_SPECIAL_VLAN_MASK  0x0FFF	/* VLAN ID is in lower 12 bits */
-#define IXGB_RX_DESC_SPECIAL_PRI_MASK   0xE000	/* Priority is in upper 3 bits */
-#define IXGB_RX_DESC_SPECIAL_PRI_SHIFT  0x000D	/* Priority is in upper 3 of 16 */
-
-/* Layout of a single transmit descriptor.  The controller assumes that this
- * structure is packed into 16 bytes, which is a safe assumption with most
- * compilers.  However, some compilers may insert padding between the fields,
- * in which case the structure must be packed in some compiler-specific
- * manner. */
-struct ixgb_tx_desc {
-	__le64 buff_addr;
-	__le32 cmd_type_len;
-	u8 status;
-	u8 popts;
-	__le16 vlan;
-};
-
-#define IXGB_TX_DESC_LENGTH_MASK    0x000FFFFF
-#define IXGB_TX_DESC_TYPE_MASK      0x00F00000
-#define IXGB_TX_DESC_TYPE_SHIFT     20
-#define IXGB_TX_DESC_CMD_MASK       0xFF000000
-#define IXGB_TX_DESC_CMD_SHIFT      24
-#define IXGB_TX_DESC_CMD_EOP        0x01000000
-#define IXGB_TX_DESC_CMD_TSE        0x04000000
-#define IXGB_TX_DESC_CMD_RS         0x08000000
-#define IXGB_TX_DESC_CMD_VLE        0x40000000
-#define IXGB_TX_DESC_CMD_IDE        0x80000000
-
-#define IXGB_TX_DESC_TYPE           0x00100000
-
-#define IXGB_TX_DESC_STATUS_DD  0x01
-
-#define IXGB_TX_DESC_POPTS_IXSM 0x01
-#define IXGB_TX_DESC_POPTS_TXSM 0x02
-#define IXGB_TX_DESC_SPECIAL_PRI_SHIFT  IXGB_RX_DESC_SPECIAL_PRI_SHIFT	/* Priority is in upper 3 of 16 */
-
-struct ixgb_context_desc {
-	u8 ipcss;
-	u8 ipcso;
-	__le16 ipcse;
-	u8 tucss;
-	u8 tucso;
-	__le16 tucse;
-	__le32 cmd_type_len;
-	u8 status;
-	u8 hdr_len;
-	__le16 mss;
-};
-
-#define IXGB_CONTEXT_DESC_CMD_TCP 0x01000000
-#define IXGB_CONTEXT_DESC_CMD_IP  0x02000000
-#define IXGB_CONTEXT_DESC_CMD_TSE 0x04000000
-#define IXGB_CONTEXT_DESC_CMD_RS  0x08000000
-#define IXGB_CONTEXT_DESC_CMD_IDE 0x80000000
-
-#define IXGB_CONTEXT_DESC_TYPE 0x00000000
-
-#define IXGB_CONTEXT_DESC_STATUS_DD 0x01
-
-/* Filters */
-#define IXGB_MC_TBL_SIZE          128	/* Multicast Filter Table (4096 bits) */
-#define IXGB_VLAN_FILTER_TBL_SIZE 128	/* VLAN Filter Table (4096 bits) */
-#define IXGB_RAR_ENTRIES		  3	/* Number of entries in Rx Address array */
-
-#define IXGB_MEMORY_REGISTER_BASE_ADDRESS   0
-#define ENET_HEADER_SIZE			14
-#define ENET_FCS_LENGTH			 4
-#define IXGB_MAX_NUM_MULTICAST_ADDRESSES	128
-#define IXGB_MIN_ENET_FRAME_SIZE_WITHOUT_FCS	60
-#define IXGB_MAX_ENET_FRAME_SIZE_WITHOUT_FCS	1514
-#define IXGB_MAX_JUMBO_FRAME_SIZE		0x3F00
-
-/* Phy Addresses */
-#define IXGB_OPTICAL_PHY_ADDR 0x0	/* Optical Module phy address */
-#define IXGB_XAUII_PHY_ADDR   0x1	/* Xauii transceiver phy address */
-#define IXGB_DIAG_PHY_ADDR    0x1F	/* Diagnostic Device phy address */
-
-/* This structure takes a 64k flash and maps it for identification commands */
-struct ixgb_flash_buffer {
-	u8 manufacturer_id;
-	u8 device_id;
-	u8 filler1[0x2AA8];
-	u8 cmd2;
-	u8 filler2[0x2AAA];
-	u8 cmd1;
-	u8 filler3[0xAAAA];
-};
-
-/* Flow control parameters */
-struct ixgb_fc {
-	u32 high_water;	/* Flow Control High-water          */
-	u32 low_water;	/* Flow Control Low-water           */
-	u16 pause_time;	/* Flow Control Pause timer         */
-	bool send_xon;		/* Flow control send XON            */
-	ixgb_fc_type type;	/* Type of flow control             */
-};
-
-/* The historical defaults for the flow control values are given below. */
-#define FC_DEFAULT_HI_THRESH        (0x8000)	/* 32KB */
-#define FC_DEFAULT_LO_THRESH        (0x4000)	/* 16KB */
-#define FC_DEFAULT_TX_TIMER         (0x100)	/* ~130 us */
-
-/* Phy definitions */
-#define IXGB_MAX_PHY_REG_ADDRESS    0xFFFF
-#define IXGB_MAX_PHY_ADDRESS        31
-#define IXGB_MAX_PHY_DEV_TYPE       31
-
-/* Bus parameters */
-struct ixgb_bus {
-	ixgb_bus_speed speed;
-	ixgb_bus_width width;
-	ixgb_bus_type type;
-};
-
-struct ixgb_hw {
-	u8 __iomem *hw_addr;/* Base Address of the hardware     */
-	void *back;		/* Pointer to OS-dependent struct   */
-	struct ixgb_fc fc;	/* Flow control parameters          */
-	struct ixgb_bus bus;	/* Bus parameters                   */
-	u32 phy_id;	/* Phy Identifier                   */
-	u32 phy_addr;	/* XGMII address of Phy             */
-	ixgb_mac_type mac_type;	/* Identifier for MAC controller    */
-	ixgb_phy_type phy_type;	/* Transceiver/phy identifier       */
-	u32 max_frame_size;	/* Maximum frame size supported     */
-	u32 mc_filter_type;	/* Multicast filter hash type       */
-	u32 num_mc_addrs;	/* Number of current Multicast addrs */
-	u8 curr_mac_addr[ETH_ALEN];	/* Individual address currently programmed in MAC */
-	u32 num_tx_desc;	/* Number of Transmit descriptors   */
-	u32 num_rx_desc;	/* Number of Receive descriptors    */
-	u32 rx_buffer_size;	/* Size of Receive buffer           */
-	bool link_up;		/* true if link is valid            */
-	bool adapter_stopped;	/* State of adapter                 */
-	u16 device_id;	/* device id from PCI configuration space */
-	u16 vendor_id;	/* vendor id from PCI configuration space */
-	u8 revision_id;	/* revision id from PCI configuration space */
-	u16 subsystem_vendor_id;	/* subsystem vendor id from PCI configuration space */
-	u16 subsystem_id;	/* subsystem id from PCI configuration space */
-	u32 bar0;		/* Base Address registers           */
-	u32 bar1;
-	u32 bar2;
-	u32 bar3;
-	u16 pci_cmd_word;	/* PCI command register id from PCI configuration space */
-	__le16 eeprom[IXGB_EEPROM_SIZE];	/* EEPROM contents read at init time  */
-	unsigned long io_base;	/* Our I/O mapped location */
-	u32 lastLFC;
-	u32 lastRFC;
-};
-
-/* Statistics reported by the hardware */
-struct ixgb_hw_stats {
-	u64 tprl;
-	u64 tprh;
-	u64 gprcl;
-	u64 gprch;
-	u64 bprcl;
-	u64 bprch;
-	u64 mprcl;
-	u64 mprch;
-	u64 uprcl;
-	u64 uprch;
-	u64 vprcl;
-	u64 vprch;
-	u64 jprcl;
-	u64 jprch;
-	u64 gorcl;
-	u64 gorch;
-	u64 torl;
-	u64 torh;
-	u64 rnbc;
-	u64 ruc;
-	u64 roc;
-	u64 rlec;
-	u64 crcerrs;
-	u64 icbc;
-	u64 ecbc;
-	u64 mpc;
-	u64 tptl;
-	u64 tpth;
-	u64 gptcl;
-	u64 gptch;
-	u64 bptcl;
-	u64 bptch;
-	u64 mptcl;
-	u64 mptch;
-	u64 uptcl;
-	u64 uptch;
-	u64 vptcl;
-	u64 vptch;
-	u64 jptcl;
-	u64 jptch;
-	u64 gotcl;
-	u64 gotch;
-	u64 totl;
-	u64 toth;
-	u64 dc;
-	u64 plt64c;
-	u64 tsctc;
-	u64 tsctfc;
-	u64 ibic;
-	u64 rfc;
-	u64 lfc;
-	u64 pfrc;
-	u64 pftc;
-	u64 mcfrc;
-	u64 mcftc;
-	u64 xonrxc;
-	u64 xontxc;
-	u64 xoffrxc;
-	u64 xofftxc;
-	u64 rjc;
-};
-
-/* Function Prototypes */
-bool ixgb_adapter_stop(struct ixgb_hw *hw);
-bool ixgb_init_hw(struct ixgb_hw *hw);
-bool ixgb_adapter_start(struct ixgb_hw *hw);
-void ixgb_check_for_link(struct ixgb_hw *hw);
-bool ixgb_check_for_bad_link(struct ixgb_hw *hw);
-
-void ixgb_rar_set(struct ixgb_hw *hw, const u8 *addr, u32 index);
-
-/* Filters (multicast, vlan, receive) */
-void ixgb_mc_addr_list_update(struct ixgb_hw *hw, u8 *mc_addr_list,
-			      u32 mc_addr_count, u32 pad);
-
-/* Vfta functions */
-void ixgb_write_vfta(struct ixgb_hw *hw, u32 offset, u32 value);
-
-/* Access functions to eeprom data */
-void ixgb_get_ee_mac_addr(struct ixgb_hw *hw, u8 *mac_addr);
-u32 ixgb_get_ee_pba_number(struct ixgb_hw *hw);
-u16 ixgb_get_ee_device_id(struct ixgb_hw *hw);
-bool ixgb_get_eeprom_data(struct ixgb_hw *hw);
-__le16 ixgb_get_eeprom_word(struct ixgb_hw *hw, u16 index);
-
-/* Everything else */
-void ixgb_led_on(struct ixgb_hw *hw);
-void ixgb_led_off(struct ixgb_hw *hw);
-void ixgb_write_pci_cfg(struct ixgb_hw *hw,
-			 u32 reg,
-			 u16 * value);
-
-
-#endif /* _IXGB_HW_H_ */
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ids.h b/drivers/net/ethernet/intel/ixgb/ixgb_ids.h
deleted file mode 100644
index 9695b8215f01..000000000000
--- a/drivers/net/ethernet/intel/ixgb/ixgb_ids.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* Copyright(c) 1999 - 2008 Intel Corporation. */
-
-#ifndef _IXGB_IDS_H_
-#define _IXGB_IDS_H_
-
-/**********************************************************************
-** The Device and Vendor IDs for 10 Gigabit MACs
-**********************************************************************/
-
-#define IXGB_DEVICE_ID_82597EX      0x1048
-#define IXGB_DEVICE_ID_82597EX_SR   0x1A48
-#define IXGB_DEVICE_ID_82597EX_LR   0x1B48
-#define IXGB_SUBDEVICE_ID_A11F      0xA11F
-#define IXGB_SUBDEVICE_ID_A01F      0xA01F
-
-#define IXGB_DEVICE_ID_82597EX_CX4   0x109E
-#define IXGB_SUBDEVICE_ID_A00C  0xA00C
-#define IXGB_SUBDEVICE_ID_A01C  0xA01C
-#define IXGB_SUBDEVICE_ID_7036  0x7036
-
-#endif /* #ifndef _IXGB_IDS_H_ */
-/* End of File */
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_main.c b/drivers/net/ethernet/intel/ixgb/ixgb_main.c
deleted file mode 100644
index b4d47e7a76c8..000000000000
--- a/drivers/net/ethernet/intel/ixgb/ixgb_main.c
+++ /dev/null
@@ -1,2285 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright(c) 1999 - 2008 Intel Corporation. */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/prefetch.h>
-#include "ixgb.h"
-
-char ixgb_driver_name[] = "ixgb";
-static char ixgb_driver_string[] = "Intel(R) PRO/10GbE Network Driver";
-
-static const char ixgb_copyright[] = "Copyright (c) 1999-2008 Intel Corporation.";
-
-#define IXGB_CB_LENGTH 256
-static unsigned int copybreak __read_mostly = IXGB_CB_LENGTH;
-module_param(copybreak, uint, 0644);
-MODULE_PARM_DESC(copybreak,
-	"Maximum size of packet that is copied to a new buffer on receive");
-
-/* ixgb_pci_tbl - PCI Device ID Table
- *
- * Wildcard entries (PCI_ANY_ID) should come last
- * Last entry must be all 0s
- *
- * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
- *   Class, Class Mask, private data (not used) }
- */
-static const struct pci_device_id ixgb_pci_tbl[] = {
-	{PCI_VENDOR_ID_INTEL, IXGB_DEVICE_ID_82597EX,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, IXGB_DEVICE_ID_82597EX_CX4,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, IXGB_DEVICE_ID_82597EX_SR,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, IXGB_DEVICE_ID_82597EX_LR,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-
-	/* required last entry */
-	{0,}
-};
-
-MODULE_DEVICE_TABLE(pci, ixgb_pci_tbl);
-
-/* Local Function Prototypes */
-static int ixgb_init_module(void);
-static void ixgb_exit_module(void);
-static int ixgb_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
-static void ixgb_remove(struct pci_dev *pdev);
-static int ixgb_sw_init(struct ixgb_adapter *adapter);
-static int ixgb_open(struct net_device *netdev);
-static int ixgb_close(struct net_device *netdev);
-static void ixgb_configure_tx(struct ixgb_adapter *adapter);
-static void ixgb_configure_rx(struct ixgb_adapter *adapter);
-static void ixgb_setup_rctl(struct ixgb_adapter *adapter);
-static void ixgb_clean_tx_ring(struct ixgb_adapter *adapter);
-static void ixgb_clean_rx_ring(struct ixgb_adapter *adapter);
-static void ixgb_set_multi(struct net_device *netdev);
-static void ixgb_watchdog(struct timer_list *t);
-static netdev_tx_t ixgb_xmit_frame(struct sk_buff *skb,
-				   struct net_device *netdev);
-static int ixgb_change_mtu(struct net_device *netdev, int new_mtu);
-static int ixgb_set_mac(struct net_device *netdev, void *p);
-static irqreturn_t ixgb_intr(int irq, void *data);
-static bool ixgb_clean_tx_irq(struct ixgb_adapter *adapter);
-
-static int ixgb_clean(struct napi_struct *, int);
-static bool ixgb_clean_rx_irq(struct ixgb_adapter *, int *, int);
-static void ixgb_alloc_rx_buffers(struct ixgb_adapter *, int);
-
-static void ixgb_tx_timeout(struct net_device *dev, unsigned int txqueue);
-static void ixgb_tx_timeout_task(struct work_struct *work);
-
-static void ixgb_vlan_strip_enable(struct ixgb_adapter *adapter);
-static void ixgb_vlan_strip_disable(struct ixgb_adapter *adapter);
-static int ixgb_vlan_rx_add_vid(struct net_device *netdev,
-				__be16 proto, u16 vid);
-static int ixgb_vlan_rx_kill_vid(struct net_device *netdev,
-				 __be16 proto, u16 vid);
-static void ixgb_restore_vlan(struct ixgb_adapter *adapter);
-
-static pci_ers_result_t ixgb_io_error_detected (struct pci_dev *pdev,
-                             pci_channel_state_t state);
-static pci_ers_result_t ixgb_io_slot_reset (struct pci_dev *pdev);
-static void ixgb_io_resume (struct pci_dev *pdev);
-
-static const struct pci_error_handlers ixgb_err_handler = {
-	.error_detected = ixgb_io_error_detected,
-	.slot_reset = ixgb_io_slot_reset,
-	.resume = ixgb_io_resume,
-};
-
-static struct pci_driver ixgb_driver = {
-	.name     = ixgb_driver_name,
-	.id_table = ixgb_pci_tbl,
-	.probe    = ixgb_probe,
-	.remove   = ixgb_remove,
-	.err_handler = &ixgb_err_handler
-};
-
-MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
-MODULE_DESCRIPTION("Intel(R) PRO/10GbE Network Driver");
-MODULE_LICENSE("GPL v2");
-
-#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
-static int debug = -1;
-module_param(debug, int, 0);
-MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
-
-/**
- * ixgb_init_module - Driver Registration Routine
- *
- * ixgb_init_module is the first routine called when the driver is
- * loaded. All it does is register with the PCI subsystem.
- **/
-
-static int __init
-ixgb_init_module(void)
-{
-	pr_info("%s\n", ixgb_driver_string);
-	pr_info("%s\n", ixgb_copyright);
-
-	return pci_register_driver(&ixgb_driver);
-}
-
-module_init(ixgb_init_module);
-
-/**
- * ixgb_exit_module - Driver Exit Cleanup Routine
- *
- * ixgb_exit_module is called just before the driver is removed
- * from memory.
- **/
-
-static void __exit
-ixgb_exit_module(void)
-{
-	pci_unregister_driver(&ixgb_driver);
-}
-
-module_exit(ixgb_exit_module);
-
-/**
- * ixgb_irq_disable - Mask off interrupt generation on the NIC
- * @adapter: board private structure
- **/
-
-static void
-ixgb_irq_disable(struct ixgb_adapter *adapter)
-{
-	IXGB_WRITE_REG(&adapter->hw, IMC, ~0);
-	IXGB_WRITE_FLUSH(&adapter->hw);
-	synchronize_irq(adapter->pdev->irq);
-}
-
-/**
- * ixgb_irq_enable - Enable default interrupt generation settings
- * @adapter: board private structure
- **/
-
-static void
-ixgb_irq_enable(struct ixgb_adapter *adapter)
-{
-	u32 val = IXGB_INT_RXT0 | IXGB_INT_RXDMT0 |
-		  IXGB_INT_TXDW | IXGB_INT_LSC;
-	if (adapter->hw.subsystem_vendor_id == PCI_VENDOR_ID_SUN)
-		val |= IXGB_INT_GPI0;
-	IXGB_WRITE_REG(&adapter->hw, IMS, val);
-	IXGB_WRITE_FLUSH(&adapter->hw);
-}
-
-int
-ixgb_up(struct ixgb_adapter *adapter)
-{
-	struct net_device *netdev = adapter->netdev;
-	int err, irq_flags = IRQF_SHARED;
-	int max_frame = netdev->mtu + ENET_HEADER_SIZE + ENET_FCS_LENGTH;
-	struct ixgb_hw *hw = &adapter->hw;
-
-	/* hardware has been reset, we need to reload some things */
-
-	ixgb_rar_set(hw, netdev->dev_addr, 0);
-	ixgb_set_multi(netdev);
-
-	ixgb_restore_vlan(adapter);
-
-	ixgb_configure_tx(adapter);
-	ixgb_setup_rctl(adapter);
-	ixgb_configure_rx(adapter);
-	ixgb_alloc_rx_buffers(adapter, IXGB_DESC_UNUSED(&adapter->rx_ring));
-
-	/* disable interrupts and get the hardware into a known state */
-	IXGB_WRITE_REG(&adapter->hw, IMC, 0xffffffff);
-
-	/* only enable MSI if bus is in PCI-X mode */
-	if (IXGB_READ_REG(&adapter->hw, STATUS) & IXGB_STATUS_PCIX_MODE) {
-		err = pci_enable_msi(adapter->pdev);
-		if (!err) {
-			adapter->have_msi = true;
-			irq_flags = 0;
-		}
-		/* proceed to try to request regular interrupt */
-	}
-
-	err = request_irq(adapter->pdev->irq, ixgb_intr, irq_flags,
-	                  netdev->name, netdev);
-	if (err) {
-		if (adapter->have_msi)
-			pci_disable_msi(adapter->pdev);
-		netif_err(adapter, probe, adapter->netdev,
-			  "Unable to allocate interrupt Error: %d\n", err);
-		return err;
-	}
-
-	if ((hw->max_frame_size != max_frame) ||
-		(hw->max_frame_size !=
-		(IXGB_READ_REG(hw, MFS) >> IXGB_MFS_SHIFT))) {
-
-		hw->max_frame_size = max_frame;
-
-		IXGB_WRITE_REG(hw, MFS, hw->max_frame_size << IXGB_MFS_SHIFT);
-
-		if (hw->max_frame_size >
-		   IXGB_MAX_ENET_FRAME_SIZE_WITHOUT_FCS + ENET_FCS_LENGTH) {
-			u32 ctrl0 = IXGB_READ_REG(hw, CTRL0);
-
-			if (!(ctrl0 & IXGB_CTRL0_JFE)) {
-				ctrl0 |= IXGB_CTRL0_JFE;
-				IXGB_WRITE_REG(hw, CTRL0, ctrl0);
-			}
-		}
-	}
-
-	clear_bit(__IXGB_DOWN, &adapter->flags);
-
-	napi_enable(&adapter->napi);
-	ixgb_irq_enable(adapter);
-
-	netif_wake_queue(netdev);
-
-	mod_timer(&adapter->watchdog_timer, jiffies);
-
-	return 0;
-}
-
-void
-ixgb_down(struct ixgb_adapter *adapter, bool kill_watchdog)
-{
-	struct net_device *netdev = adapter->netdev;
-
-	/* prevent the interrupt handler from restarting watchdog */
-	set_bit(__IXGB_DOWN, &adapter->flags);
-
-	netif_carrier_off(netdev);
-
-	napi_disable(&adapter->napi);
-	/* waiting for NAPI to complete can re-enable interrupts */
-	ixgb_irq_disable(adapter);
-	free_irq(adapter->pdev->irq, netdev);
-
-	if (adapter->have_msi)
-		pci_disable_msi(adapter->pdev);
-
-	if (kill_watchdog)
-		del_timer_sync(&adapter->watchdog_timer);
-
-	adapter->link_speed = 0;
-	adapter->link_duplex = 0;
-	netif_stop_queue(netdev);
-
-	ixgb_reset(adapter);
-	ixgb_clean_tx_ring(adapter);
-	ixgb_clean_rx_ring(adapter);
-}
-
-void
-ixgb_reset(struct ixgb_adapter *adapter)
-{
-	struct ixgb_hw *hw = &adapter->hw;
-
-	ixgb_adapter_stop(hw);
-	if (!ixgb_init_hw(hw))
-		netif_err(adapter, probe, adapter->netdev, "ixgb_init_hw failed\n");
-
-	/* restore frame size information */
-	IXGB_WRITE_REG(hw, MFS, hw->max_frame_size << IXGB_MFS_SHIFT);
-	if (hw->max_frame_size >
-	    IXGB_MAX_ENET_FRAME_SIZE_WITHOUT_FCS + ENET_FCS_LENGTH) {
-		u32 ctrl0 = IXGB_READ_REG(hw, CTRL0);
-		if (!(ctrl0 & IXGB_CTRL0_JFE)) {
-			ctrl0 |= IXGB_CTRL0_JFE;
-			IXGB_WRITE_REG(hw, CTRL0, ctrl0);
-		}
-	}
-}
-
-static netdev_features_t
-ixgb_fix_features(struct net_device *netdev, netdev_features_t features)
-{
-	/*
-	 * Tx VLAN insertion does not work per HW design when Rx stripping is
-	 * disabled.
-	 */
-	if (!(features & NETIF_F_HW_VLAN_CTAG_RX))
-		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
-
-	return features;
-}
-
-static int
-ixgb_set_features(struct net_device *netdev, netdev_features_t features)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	netdev_features_t changed = features ^ netdev->features;
-
-	if (!(changed & (NETIF_F_RXCSUM|NETIF_F_HW_VLAN_CTAG_RX)))
-		return 0;
-
-	adapter->rx_csum = !!(features & NETIF_F_RXCSUM);
-
-	if (netif_running(netdev)) {
-		ixgb_down(adapter, true);
-		ixgb_up(adapter);
-		ixgb_set_speed_duplex(netdev);
-	} else
-		ixgb_reset(adapter);
-
-	return 0;
-}
-
-
-static const struct net_device_ops ixgb_netdev_ops = {
-	.ndo_open 		= ixgb_open,
-	.ndo_stop		= ixgb_close,
-	.ndo_start_xmit		= ixgb_xmit_frame,
-	.ndo_set_rx_mode	= ixgb_set_multi,
-	.ndo_validate_addr	= eth_validate_addr,
-	.ndo_set_mac_address	= ixgb_set_mac,
-	.ndo_change_mtu		= ixgb_change_mtu,
-	.ndo_tx_timeout		= ixgb_tx_timeout,
-	.ndo_vlan_rx_add_vid	= ixgb_vlan_rx_add_vid,
-	.ndo_vlan_rx_kill_vid	= ixgb_vlan_rx_kill_vid,
-	.ndo_fix_features       = ixgb_fix_features,
-	.ndo_set_features       = ixgb_set_features,
-};
-
-/**
- * ixgb_probe - Device Initialization Routine
- * @pdev: PCI device information struct
- * @ent: entry in ixgb_pci_tbl
- *
- * Returns 0 on success, negative on failure
- *
- * ixgb_probe initializes an adapter identified by a pci_dev structure.
- * The OS initialization, configuring of the adapter private structure,
- * and a hardware reset occur.
- **/
-
-static int
-ixgb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-	struct net_device *netdev = NULL;
-	struct ixgb_adapter *adapter;
-	static int cards_found = 0;
-	u8 addr[ETH_ALEN];
-	int i;
-	int err;
-
-	err = pci_enable_device(pdev);
-	if (err)
-		return err;
-
-	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
-	if (err) {
-		pr_err("No usable DMA configuration, aborting\n");
-		goto err_dma_mask;
-	}
-
-	err = pci_request_regions(pdev, ixgb_driver_name);
-	if (err)
-		goto err_request_regions;
-
-	pci_set_master(pdev);
-
-	netdev = alloc_etherdev(sizeof(struct ixgb_adapter));
-	if (!netdev) {
-		err = -ENOMEM;
-		goto err_alloc_etherdev;
-	}
-
-	SET_NETDEV_DEV(netdev, &pdev->dev);
-
-	pci_set_drvdata(pdev, netdev);
-	adapter = netdev_priv(netdev);
-	adapter->netdev = netdev;
-	adapter->pdev = pdev;
-	adapter->hw.back = adapter;
-	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
-
-	adapter->hw.hw_addr = pci_ioremap_bar(pdev, BAR_0);
-	if (!adapter->hw.hw_addr) {
-		err = -EIO;
-		goto err_ioremap;
-	}
-
-	for (i = BAR_1; i < PCI_STD_NUM_BARS; i++) {
-		if (pci_resource_len(pdev, i) == 0)
-			continue;
-		if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
-			adapter->hw.io_base = pci_resource_start(pdev, i);
-			break;
-		}
-	}
-
-	netdev->netdev_ops = &ixgb_netdev_ops;
-	ixgb_set_ethtool_ops(netdev);
-	netdev->watchdog_timeo = 5 * HZ;
-	netif_napi_add(netdev, &adapter->napi, ixgb_clean);
-
-	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
-
-	adapter->bd_number = cards_found;
-	adapter->link_speed = 0;
-	adapter->link_duplex = 0;
-
-	/* setup the private structure */
-
-	err = ixgb_sw_init(adapter);
-	if (err)
-		goto err_sw_init;
-
-	netdev->hw_features = NETIF_F_SG |
-			   NETIF_F_TSO |
-			   NETIF_F_HW_CSUM |
-			   NETIF_F_HW_VLAN_CTAG_TX |
-			   NETIF_F_HW_VLAN_CTAG_RX;
-	netdev->features = netdev->hw_features |
-			   NETIF_F_HW_VLAN_CTAG_FILTER;
-	netdev->hw_features |= NETIF_F_RXCSUM;
-
-	netdev->features |= NETIF_F_HIGHDMA;
-	netdev->vlan_features |= NETIF_F_HIGHDMA;
-
-	/* MTU range: 68 - 16114 */
-	netdev->min_mtu = ETH_MIN_MTU;
-	netdev->max_mtu = IXGB_MAX_JUMBO_FRAME_SIZE - ETH_HLEN;
-
-	/* make sure the EEPROM is good */
-
-	if (!ixgb_validate_eeprom_checksum(&adapter->hw)) {
-		netif_err(adapter, probe, adapter->netdev,
-			  "The EEPROM Checksum Is Not Valid\n");
-		err = -EIO;
-		goto err_eeprom;
-	}
-
-	ixgb_get_ee_mac_addr(&adapter->hw, addr);
-	eth_hw_addr_set(netdev, addr);
-
-	if (!is_valid_ether_addr(netdev->dev_addr)) {
-		netif_err(adapter, probe, adapter->netdev, "Invalid MAC Address\n");
-		err = -EIO;
-		goto err_eeprom;
-	}
-
-	adapter->part_num = ixgb_get_ee_pba_number(&adapter->hw);
-
-	timer_setup(&adapter->watchdog_timer, ixgb_watchdog, 0);
-
-	INIT_WORK(&adapter->tx_timeout_task, ixgb_tx_timeout_task);
-
-	strcpy(netdev->name, "eth%d");
-	err = register_netdev(netdev);
-	if (err)
-		goto err_register;
-
-	/* carrier off reporting is important to ethtool even BEFORE open */
-	netif_carrier_off(netdev);
-
-	netif_info(adapter, probe, adapter->netdev,
-		   "Intel(R) PRO/10GbE Network Connection\n");
-	ixgb_check_options(adapter);
-	/* reset the hardware with the new settings */
-
-	ixgb_reset(adapter);
-
-	cards_found++;
-	return 0;
-
-err_register:
-err_sw_init:
-err_eeprom:
-	iounmap(adapter->hw.hw_addr);
-err_ioremap:
-	free_netdev(netdev);
-err_alloc_etherdev:
-	pci_release_regions(pdev);
-err_request_regions:
-err_dma_mask:
-	pci_disable_device(pdev);
-	return err;
-}
-
-/**
- * ixgb_remove - Device Removal Routine
- * @pdev: PCI device information struct
- *
- * ixgb_remove is called by the PCI subsystem to alert the driver
- * that it should release a PCI device.  The could be caused by a
- * Hot-Plug event, or because the driver is going to be removed from
- * memory.
- **/
-
-static void
-ixgb_remove(struct pci_dev *pdev)
-{
-	struct net_device *netdev = pci_get_drvdata(pdev);
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-
-	cancel_work_sync(&adapter->tx_timeout_task);
-
-	unregister_netdev(netdev);
-
-	iounmap(adapter->hw.hw_addr);
-	pci_release_regions(pdev);
-
-	free_netdev(netdev);
-	pci_disable_device(pdev);
-}
-
-/**
- * ixgb_sw_init - Initialize general software structures (struct ixgb_adapter)
- * @adapter: board private structure to initialize
- *
- * ixgb_sw_init initializes the Adapter private data structure.
- * Fields are initialized based on PCI device information and
- * OS network device settings (MTU size).
- **/
-
-static int
-ixgb_sw_init(struct ixgb_adapter *adapter)
-{
-	struct ixgb_hw *hw = &adapter->hw;
-	struct net_device *netdev = adapter->netdev;
-	struct pci_dev *pdev = adapter->pdev;
-
-	/* PCI config space info */
-
-	hw->vendor_id = pdev->vendor;
-	hw->device_id = pdev->device;
-	hw->subsystem_vendor_id = pdev->subsystem_vendor;
-	hw->subsystem_id = pdev->subsystem_device;
-
-	hw->max_frame_size = netdev->mtu + ENET_HEADER_SIZE + ENET_FCS_LENGTH;
-	adapter->rx_buffer_len = hw->max_frame_size + 8; /* + 8 for errata */
-
-	if ((hw->device_id == IXGB_DEVICE_ID_82597EX) ||
-	    (hw->device_id == IXGB_DEVICE_ID_82597EX_CX4) ||
-	    (hw->device_id == IXGB_DEVICE_ID_82597EX_LR) ||
-	    (hw->device_id == IXGB_DEVICE_ID_82597EX_SR))
-		hw->mac_type = ixgb_82597;
-	else {
-		/* should never have loaded on this device */
-		netif_err(adapter, probe, adapter->netdev, "unsupported device id\n");
-	}
-
-	/* enable flow control to be programmed */
-	hw->fc.send_xon = 1;
-
-	set_bit(__IXGB_DOWN, &adapter->flags);
-	return 0;
-}
-
-/**
- * ixgb_open - Called when a network interface is made active
- * @netdev: network interface device structure
- *
- * Returns 0 on success, negative value on failure
- *
- * The open entry point is called when a network interface is made
- * active by the system (IFF_UP).  At this point all resources needed
- * for transmit and receive operations are allocated, the interrupt
- * handler is registered with the OS, the watchdog timer is started,
- * and the stack is notified that the interface is ready.
- **/
-
-static int
-ixgb_open(struct net_device *netdev)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	int err;
-
-	/* allocate transmit descriptors */
-	err = ixgb_setup_tx_resources(adapter);
-	if (err)
-		goto err_setup_tx;
-
-	netif_carrier_off(netdev);
-
-	/* allocate receive descriptors */
-
-	err = ixgb_setup_rx_resources(adapter);
-	if (err)
-		goto err_setup_rx;
-
-	err = ixgb_up(adapter);
-	if (err)
-		goto err_up;
-
-	netif_start_queue(netdev);
-
-	return 0;
-
-err_up:
-	ixgb_free_rx_resources(adapter);
-err_setup_rx:
-	ixgb_free_tx_resources(adapter);
-err_setup_tx:
-	ixgb_reset(adapter);
-
-	return err;
-}
-
-/**
- * ixgb_close - Disables a network interface
- * @netdev: network interface device structure
- *
- * Returns 0, this is not allowed to fail
- *
- * The close entry point is called when an interface is de-activated
- * by the OS.  The hardware is still under the drivers control, but
- * needs to be disabled.  A global MAC reset is issued to stop the
- * hardware, and all transmit and receive resources are freed.
- **/
-
-static int
-ixgb_close(struct net_device *netdev)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-
-	ixgb_down(adapter, true);
-
-	ixgb_free_tx_resources(adapter);
-	ixgb_free_rx_resources(adapter);
-
-	return 0;
-}
-
-/**
- * ixgb_setup_tx_resources - allocate Tx resources (Descriptors)
- * @adapter: board private structure
- *
- * Return 0 on success, negative on failure
- **/
-
-int
-ixgb_setup_tx_resources(struct ixgb_adapter *adapter)
-{
-	struct ixgb_desc_ring *txdr = &adapter->tx_ring;
-	struct pci_dev *pdev = adapter->pdev;
-	int size;
-
-	size = sizeof(struct ixgb_buffer) * txdr->count;
-	txdr->buffer_info = vzalloc(size);
-	if (!txdr->buffer_info)
-		return -ENOMEM;
-
-	/* round up to nearest 4K */
-
-	txdr->size = txdr->count * sizeof(struct ixgb_tx_desc);
-	txdr->size = ALIGN(txdr->size, 4096);
-
-	txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size, &txdr->dma,
-					GFP_KERNEL);
-	if (!txdr->desc) {
-		vfree(txdr->buffer_info);
-		return -ENOMEM;
-	}
-
-	txdr->next_to_use = 0;
-	txdr->next_to_clean = 0;
-
-	return 0;
-}
-
-/**
- * ixgb_configure_tx - Configure 82597 Transmit Unit after Reset.
- * @adapter: board private structure
- *
- * Configure the Tx unit of the MAC after a reset.
- **/
-
-static void
-ixgb_configure_tx(struct ixgb_adapter *adapter)
-{
-	u64 tdba = adapter->tx_ring.dma;
-	u32 tdlen = adapter->tx_ring.count * sizeof(struct ixgb_tx_desc);
-	u32 tctl;
-	struct ixgb_hw *hw = &adapter->hw;
-
-	/* Setup the Base and Length of the Tx Descriptor Ring
-	 * tx_ring.dma can be either a 32 or 64 bit value
-	 */
-
-	IXGB_WRITE_REG(hw, TDBAL, (tdba & 0x00000000ffffffffULL));
-	IXGB_WRITE_REG(hw, TDBAH, (tdba >> 32));
-
-	IXGB_WRITE_REG(hw, TDLEN, tdlen);
-
-	/* Setup the HW Tx Head and Tail descriptor pointers */
-
-	IXGB_WRITE_REG(hw, TDH, 0);
-	IXGB_WRITE_REG(hw, TDT, 0);
-
-	/* don't set up txdctl, it induces performance problems if configured
-	 * incorrectly */
-	/* Set the Tx Interrupt Delay register */
-
-	IXGB_WRITE_REG(hw, TIDV, adapter->tx_int_delay);
-
-	/* Program the Transmit Control Register */
-
-	tctl = IXGB_TCTL_TCE | IXGB_TCTL_TXEN | IXGB_TCTL_TPDE;
-	IXGB_WRITE_REG(hw, TCTL, tctl);
-
-	/* Setup Transmit Descriptor Settings for this adapter */
-	adapter->tx_cmd_type =
-		IXGB_TX_DESC_TYPE |
-		(adapter->tx_int_delay_enable ? IXGB_TX_DESC_CMD_IDE : 0);
-}
-
-/**
- * ixgb_setup_rx_resources - allocate Rx resources (Descriptors)
- * @adapter: board private structure
- *
- * Returns 0 on success, negative on failure
- **/
-
-int
-ixgb_setup_rx_resources(struct ixgb_adapter *adapter)
-{
-	struct ixgb_desc_ring *rxdr = &adapter->rx_ring;
-	struct pci_dev *pdev = adapter->pdev;
-	int size;
-
-	size = sizeof(struct ixgb_buffer) * rxdr->count;
-	rxdr->buffer_info = vzalloc(size);
-	if (!rxdr->buffer_info)
-		return -ENOMEM;
-
-	/* Round up to nearest 4K */
-
-	rxdr->size = rxdr->count * sizeof(struct ixgb_rx_desc);
-	rxdr->size = ALIGN(rxdr->size, 4096);
-
-	rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma,
-					GFP_KERNEL);
-
-	if (!rxdr->desc) {
-		vfree(rxdr->buffer_info);
-		return -ENOMEM;
-	}
-
-	rxdr->next_to_clean = 0;
-	rxdr->next_to_use = 0;
-
-	return 0;
-}
-
-/**
- * ixgb_setup_rctl - configure the receive control register
- * @adapter: Board private structure
- **/
-
-static void
-ixgb_setup_rctl(struct ixgb_adapter *adapter)
-{
-	u32 rctl;
-
-	rctl = IXGB_READ_REG(&adapter->hw, RCTL);
-
-	rctl &= ~(3 << IXGB_RCTL_MO_SHIFT);
-
-	rctl |=
-		IXGB_RCTL_BAM | IXGB_RCTL_RDMTS_1_2 |
-		IXGB_RCTL_RXEN | IXGB_RCTL_CFF |
-		(adapter->hw.mc_filter_type << IXGB_RCTL_MO_SHIFT);
-
-	rctl |= IXGB_RCTL_SECRC;
-
-	if (adapter->rx_buffer_len <= IXGB_RXBUFFER_2048)
-		rctl |= IXGB_RCTL_BSIZE_2048;
-	else if (adapter->rx_buffer_len <= IXGB_RXBUFFER_4096)
-		rctl |= IXGB_RCTL_BSIZE_4096;
-	else if (adapter->rx_buffer_len <= IXGB_RXBUFFER_8192)
-		rctl |= IXGB_RCTL_BSIZE_8192;
-	else if (adapter->rx_buffer_len <= IXGB_RXBUFFER_16384)
-		rctl |= IXGB_RCTL_BSIZE_16384;
-
-	IXGB_WRITE_REG(&adapter->hw, RCTL, rctl);
-}
-
-/**
- * ixgb_configure_rx - Configure 82597 Receive Unit after Reset.
- * @adapter: board private structure
- *
- * Configure the Rx unit of the MAC after a reset.
- **/
-
-static void
-ixgb_configure_rx(struct ixgb_adapter *adapter)
-{
-	u64 rdba = adapter->rx_ring.dma;
-	u32 rdlen = adapter->rx_ring.count * sizeof(struct ixgb_rx_desc);
-	struct ixgb_hw *hw = &adapter->hw;
-	u32 rctl;
-	u32 rxcsum;
-
-	/* make sure receives are disabled while setting up the descriptors */
-
-	rctl = IXGB_READ_REG(hw, RCTL);
-	IXGB_WRITE_REG(hw, RCTL, rctl & ~IXGB_RCTL_RXEN);
-
-	/* set the Receive Delay Timer Register */
-
-	IXGB_WRITE_REG(hw, RDTR, adapter->rx_int_delay);
-
-	/* Setup the Base and Length of the Rx Descriptor Ring */
-
-	IXGB_WRITE_REG(hw, RDBAL, (rdba & 0x00000000ffffffffULL));
-	IXGB_WRITE_REG(hw, RDBAH, (rdba >> 32));
-
-	IXGB_WRITE_REG(hw, RDLEN, rdlen);
-
-	/* Setup the HW Rx Head and Tail Descriptor Pointers */
-	IXGB_WRITE_REG(hw, RDH, 0);
-	IXGB_WRITE_REG(hw, RDT, 0);
-
-	/* due to the hardware errata with RXDCTL, we are unable to use any of
-	 * the performance enhancing features of it without causing other
-	 * subtle bugs, some of the bugs could include receive length
-	 * corruption at high data rates (WTHRESH > 0) and/or receive
-	 * descriptor ring irregularites (particularly in hardware cache) */
-	IXGB_WRITE_REG(hw, RXDCTL, 0);
-
-	/* Enable Receive Checksum Offload for TCP and UDP */
-	if (adapter->rx_csum) {
-		rxcsum = IXGB_READ_REG(hw, RXCSUM);
-		rxcsum |= IXGB_RXCSUM_TUOFL;
-		IXGB_WRITE_REG(hw, RXCSUM, rxcsum);
-	}
-
-	/* Enable Receives */
-
-	IXGB_WRITE_REG(hw, RCTL, rctl);
-}
-
-/**
- * ixgb_free_tx_resources - Free Tx Resources
- * @adapter: board private structure
- *
- * Free all transmit software resources
- **/
-
-void
-ixgb_free_tx_resources(struct ixgb_adapter *adapter)
-{
-	struct pci_dev *pdev = adapter->pdev;
-
-	ixgb_clean_tx_ring(adapter);
-
-	vfree(adapter->tx_ring.buffer_info);
-	adapter->tx_ring.buffer_info = NULL;
-
-	dma_free_coherent(&pdev->dev, adapter->tx_ring.size,
-			  adapter->tx_ring.desc, adapter->tx_ring.dma);
-
-	adapter->tx_ring.desc = NULL;
-}
-
-static void
-ixgb_unmap_and_free_tx_resource(struct ixgb_adapter *adapter,
-                                struct ixgb_buffer *buffer_info)
-{
-	if (buffer_info->dma) {
-		if (buffer_info->mapped_as_page)
-			dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
-				       buffer_info->length, DMA_TO_DEVICE);
-		else
-			dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
-					 buffer_info->length, DMA_TO_DEVICE);
-		buffer_info->dma = 0;
-	}
-
-	if (buffer_info->skb) {
-		dev_kfree_skb_any(buffer_info->skb);
-		buffer_info->skb = NULL;
-	}
-	buffer_info->time_stamp = 0;
-	/* these fields must always be initialized in tx
-	 * buffer_info->length = 0;
-	 * buffer_info->next_to_watch = 0; */
-}
-
-/**
- * ixgb_clean_tx_ring - Free Tx Buffers
- * @adapter: board private structure
- **/
-
-static void
-ixgb_clean_tx_ring(struct ixgb_adapter *adapter)
-{
-	struct ixgb_desc_ring *tx_ring = &adapter->tx_ring;
-	struct ixgb_buffer *buffer_info;
-	unsigned long size;
-	unsigned int i;
-
-	/* Free all the Tx ring sk_buffs */
-
-	for (i = 0; i < tx_ring->count; i++) {
-		buffer_info = &tx_ring->buffer_info[i];
-		ixgb_unmap_and_free_tx_resource(adapter, buffer_info);
-	}
-
-	size = sizeof(struct ixgb_buffer) * tx_ring->count;
-	memset(tx_ring->buffer_info, 0, size);
-
-	/* Zero out the descriptor ring */
-
-	memset(tx_ring->desc, 0, tx_ring->size);
-
-	tx_ring->next_to_use = 0;
-	tx_ring->next_to_clean = 0;
-
-	IXGB_WRITE_REG(&adapter->hw, TDH, 0);
-	IXGB_WRITE_REG(&adapter->hw, TDT, 0);
-}
-
-/**
- * ixgb_free_rx_resources - Free Rx Resources
- * @adapter: board private structure
- *
- * Free all receive software resources
- **/
-
-void
-ixgb_free_rx_resources(struct ixgb_adapter *adapter)
-{
-	struct ixgb_desc_ring *rx_ring = &adapter->rx_ring;
-	struct pci_dev *pdev = adapter->pdev;
-
-	ixgb_clean_rx_ring(adapter);
-
-	vfree(rx_ring->buffer_info);
-	rx_ring->buffer_info = NULL;
-
-	dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
-			  rx_ring->dma);
-
-	rx_ring->desc = NULL;
-}
-
-/**
- * ixgb_clean_rx_ring - Free Rx Buffers
- * @adapter: board private structure
- **/
-
-static void
-ixgb_clean_rx_ring(struct ixgb_adapter *adapter)
-{
-	struct ixgb_desc_ring *rx_ring = &adapter->rx_ring;
-	struct ixgb_buffer *buffer_info;
-	struct pci_dev *pdev = adapter->pdev;
-	unsigned long size;
-	unsigned int i;
-
-	/* Free all the Rx ring sk_buffs */
-
-	for (i = 0; i < rx_ring->count; i++) {
-		buffer_info = &rx_ring->buffer_info[i];
-		if (buffer_info->dma) {
-			dma_unmap_single(&pdev->dev,
-					 buffer_info->dma,
-					 buffer_info->length,
-					 DMA_FROM_DEVICE);
-			buffer_info->dma = 0;
-			buffer_info->length = 0;
-		}
-
-		if (buffer_info->skb) {
-			dev_kfree_skb(buffer_info->skb);
-			buffer_info->skb = NULL;
-		}
-	}
-
-	size = sizeof(struct ixgb_buffer) * rx_ring->count;
-	memset(rx_ring->buffer_info, 0, size);
-
-	/* Zero out the descriptor ring */
-
-	memset(rx_ring->desc, 0, rx_ring->size);
-
-	rx_ring->next_to_clean = 0;
-	rx_ring->next_to_use = 0;
-
-	IXGB_WRITE_REG(&adapter->hw, RDH, 0);
-	IXGB_WRITE_REG(&adapter->hw, RDT, 0);
-}
-
-/**
- * ixgb_set_mac - Change the Ethernet Address of the NIC
- * @netdev: network interface device structure
- * @p: pointer to an address structure
- *
- * Returns 0 on success, negative on failure
- **/
-
-static int
-ixgb_set_mac(struct net_device *netdev, void *p)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct sockaddr *addr = p;
-
-	if (!is_valid_ether_addr(addr->sa_data))
-		return -EADDRNOTAVAIL;
-
-	eth_hw_addr_set(netdev, addr->sa_data);
-
-	ixgb_rar_set(&adapter->hw, addr->sa_data, 0);
-
-	return 0;
-}
-
-/**
- * ixgb_set_multi - Multicast and Promiscuous mode set
- * @netdev: network interface device structure
- *
- * The set_multi entry point is called whenever the multicast address
- * list or the network interface flags are updated.  This routine is
- * responsible for configuring the hardware for proper multicast,
- * promiscuous mode, and all-multi behavior.
- **/
-
-static void
-ixgb_set_multi(struct net_device *netdev)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct ixgb_hw *hw = &adapter->hw;
-	struct netdev_hw_addr *ha;
-	u32 rctl;
-
-	/* Check for Promiscuous and All Multicast modes */
-
-	rctl = IXGB_READ_REG(hw, RCTL);
-
-	if (netdev->flags & IFF_PROMISC) {
-		rctl |= (IXGB_RCTL_UPE | IXGB_RCTL_MPE);
-		/* disable VLAN filtering */
-		rctl &= ~IXGB_RCTL_CFIEN;
-		rctl &= ~IXGB_RCTL_VFE;
-	} else {
-		if (netdev->flags & IFF_ALLMULTI) {
-			rctl |= IXGB_RCTL_MPE;
-			rctl &= ~IXGB_RCTL_UPE;
-		} else {
-			rctl &= ~(IXGB_RCTL_UPE | IXGB_RCTL_MPE);
-		}
-		/* enable VLAN filtering */
-		rctl |= IXGB_RCTL_VFE;
-		rctl &= ~IXGB_RCTL_CFIEN;
-	}
-
-	if (netdev_mc_count(netdev) > IXGB_MAX_NUM_MULTICAST_ADDRESSES) {
-		rctl |= IXGB_RCTL_MPE;
-		IXGB_WRITE_REG(hw, RCTL, rctl);
-	} else {
-		u8 *mta = kmalloc_array(ETH_ALEN,
-				        IXGB_MAX_NUM_MULTICAST_ADDRESSES,
-				        GFP_ATOMIC);
-		u8 *addr;
-		if (!mta)
-			goto alloc_failed;
-
-		IXGB_WRITE_REG(hw, RCTL, rctl);
-
-		addr = mta;
-		netdev_for_each_mc_addr(ha, netdev) {
-			memcpy(addr, ha->addr, ETH_ALEN);
-			addr += ETH_ALEN;
-		}
-
-		ixgb_mc_addr_list_update(hw, mta, netdev_mc_count(netdev), 0);
-		kfree(mta);
-	}
-
-alloc_failed:
-	if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
-		ixgb_vlan_strip_enable(adapter);
-	else
-		ixgb_vlan_strip_disable(adapter);
-
-}
-
-/**
- * ixgb_watchdog - Timer Call-back
- * @t: pointer to timer_list containing our private info pointer
- **/
-
-static void
-ixgb_watchdog(struct timer_list *t)
-{
-	struct ixgb_adapter *adapter = from_timer(adapter, t, watchdog_timer);
-	struct net_device *netdev = adapter->netdev;
-	struct ixgb_desc_ring *txdr = &adapter->tx_ring;
-
-	ixgb_check_for_link(&adapter->hw);
-
-	if (ixgb_check_for_bad_link(&adapter->hw)) {
-		/* force the reset path */
-		netif_stop_queue(netdev);
-	}
-
-	if (adapter->hw.link_up) {
-		if (!netif_carrier_ok(netdev)) {
-			netdev_info(netdev,
-				    "NIC Link is Up 10 Gbps Full Duplex, Flow Control: %s\n",
-				    (adapter->hw.fc.type == ixgb_fc_full) ?
-				    "RX/TX" :
-				    (adapter->hw.fc.type == ixgb_fc_rx_pause) ?
-				     "RX" :
-				    (adapter->hw.fc.type == ixgb_fc_tx_pause) ?
-				    "TX" : "None");
-			adapter->link_speed = 10000;
-			adapter->link_duplex = FULL_DUPLEX;
-			netif_carrier_on(netdev);
-		}
-	} else {
-		if (netif_carrier_ok(netdev)) {
-			adapter->link_speed = 0;
-			adapter->link_duplex = 0;
-			netdev_info(netdev, "NIC Link is Down\n");
-			netif_carrier_off(netdev);
-		}
-	}
-
-	ixgb_update_stats(adapter);
-
-	if (!netif_carrier_ok(netdev)) {
-		if (IXGB_DESC_UNUSED(txdr) + 1 < txdr->count) {
-			/* We've lost link, so the controller stops DMA,
-			 * but we've got queued Tx work that's never going
-			 * to get done, so reset controller to flush Tx.
-			 * (Do the reset outside of interrupt context). */
-			schedule_work(&adapter->tx_timeout_task);
-			/* return immediately since reset is imminent */
-			return;
-		}
-	}
-
-	/* Force detection of hung controller every watchdog period */
-	adapter->detect_tx_hung = true;
-
-	/* generate an interrupt to force clean up of any stragglers */
-	IXGB_WRITE_REG(&adapter->hw, ICS, IXGB_INT_TXDW);
-
-	/* Reset the timer */
-	mod_timer(&adapter->watchdog_timer, jiffies + 2 * HZ);
-}
-
-#define IXGB_TX_FLAGS_CSUM		0x00000001
-#define IXGB_TX_FLAGS_VLAN		0x00000002
-#define IXGB_TX_FLAGS_TSO		0x00000004
-
-static int
-ixgb_tso(struct ixgb_adapter *adapter, struct sk_buff *skb)
-{
-	struct ixgb_context_desc *context_desc;
-	unsigned int i;
-	u8 ipcss, ipcso, tucss, tucso, hdr_len;
-	u16 ipcse, tucse, mss;
-
-	if (likely(skb_is_gso(skb))) {
-		struct ixgb_buffer *buffer_info;
-		struct iphdr *iph;
-		int err;
-
-		err = skb_cow_head(skb, 0);
-		if (err < 0)
-			return err;
-
-		hdr_len = skb_tcp_all_headers(skb);
-		mss = skb_shinfo(skb)->gso_size;
-		iph = ip_hdr(skb);
-		iph->tot_len = 0;
-		iph->check = 0;
-		tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
-							 iph->daddr, 0,
-							 IPPROTO_TCP, 0);
-		ipcss = skb_network_offset(skb);
-		ipcso = (void *)&(iph->check) - (void *)skb->data;
-		ipcse = skb_transport_offset(skb) - 1;
-		tucss = skb_transport_offset(skb);
-		tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
-		tucse = 0;
-
-		i = adapter->tx_ring.next_to_use;
-		context_desc = IXGB_CONTEXT_DESC(adapter->tx_ring, i);
-		buffer_info = &adapter->tx_ring.buffer_info[i];
-		WARN_ON(buffer_info->dma != 0);
-
-		context_desc->ipcss = ipcss;
-		context_desc->ipcso = ipcso;
-		context_desc->ipcse = cpu_to_le16(ipcse);
-		context_desc->tucss = tucss;
-		context_desc->tucso = tucso;
-		context_desc->tucse = cpu_to_le16(tucse);
-		context_desc->mss = cpu_to_le16(mss);
-		context_desc->hdr_len = hdr_len;
-		context_desc->status = 0;
-		context_desc->cmd_type_len = cpu_to_le32(
-						  IXGB_CONTEXT_DESC_TYPE
-						| IXGB_CONTEXT_DESC_CMD_TSE
-						| IXGB_CONTEXT_DESC_CMD_IP
-						| IXGB_CONTEXT_DESC_CMD_TCP
-						| IXGB_CONTEXT_DESC_CMD_IDE
-						| (skb->len - (hdr_len)));
-
-
-		if (++i == adapter->tx_ring.count) i = 0;
-		adapter->tx_ring.next_to_use = i;
-
-		return 1;
-	}
-
-	return 0;
-}
-
-static bool
-ixgb_tx_csum(struct ixgb_adapter *adapter, struct sk_buff *skb)
-{
-	struct ixgb_context_desc *context_desc;
-	unsigned int i;
-	u8 css, cso;
-
-	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
-		struct ixgb_buffer *buffer_info;
-		css = skb_checksum_start_offset(skb);
-		cso = css + skb->csum_offset;
-
-		i = adapter->tx_ring.next_to_use;
-		context_desc = IXGB_CONTEXT_DESC(adapter->tx_ring, i);
-		buffer_info = &adapter->tx_ring.buffer_info[i];
-		WARN_ON(buffer_info->dma != 0);
-
-		context_desc->tucss = css;
-		context_desc->tucso = cso;
-		context_desc->tucse = 0;
-		/* zero out any previously existing data in one instruction */
-		*(u32 *)&(context_desc->ipcss) = 0;
-		context_desc->status = 0;
-		context_desc->hdr_len = 0;
-		context_desc->mss = 0;
-		context_desc->cmd_type_len =
-			cpu_to_le32(IXGB_CONTEXT_DESC_TYPE
-				    | IXGB_TX_DESC_CMD_IDE);
-
-		if (++i == adapter->tx_ring.count) i = 0;
-		adapter->tx_ring.next_to_use = i;
-
-		return true;
-	}
-
-	return false;
-}
-
-#define IXGB_MAX_TXD_PWR	14
-#define IXGB_MAX_DATA_PER_TXD	(1<<IXGB_MAX_TXD_PWR)
-
-static int
-ixgb_tx_map(struct ixgb_adapter *adapter, struct sk_buff *skb,
-	    unsigned int first)
-{
-	struct ixgb_desc_ring *tx_ring = &adapter->tx_ring;
-	struct pci_dev *pdev = adapter->pdev;
-	struct ixgb_buffer *buffer_info;
-	int len = skb_headlen(skb);
-	unsigned int offset = 0, size, count = 0, i;
-	unsigned int mss = skb_shinfo(skb)->gso_size;
-	unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
-	unsigned int f;
-
-	i = tx_ring->next_to_use;
-
-	while (len) {
-		buffer_info = &tx_ring->buffer_info[i];
-		size = min(len, IXGB_MAX_DATA_PER_TXD);
-		/* Workaround for premature desc write-backs
-		 * in TSO mode.  Append 4-byte sentinel desc */
-		if (unlikely(mss && !nr_frags && size == len && size > 8))
-			size -= 4;
-
-		buffer_info->length = size;
-		WARN_ON(buffer_info->dma != 0);
-		buffer_info->time_stamp = jiffies;
-		buffer_info->mapped_as_page = false;
-		buffer_info->dma = dma_map_single(&pdev->dev,
-						  skb->data + offset,
-						  size, DMA_TO_DEVICE);
-		if (dma_mapping_error(&pdev->dev, buffer_info->dma))
-			goto dma_error;
-		buffer_info->next_to_watch = 0;
-
-		len -= size;
-		offset += size;
-		count++;
-		if (len) {
-			i++;
-			if (i == tx_ring->count)
-				i = 0;
-		}
-	}
-
-	for (f = 0; f < nr_frags; f++) {
-		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
-		len = skb_frag_size(frag);
-		offset = 0;
-
-		while (len) {
-			i++;
-			if (i == tx_ring->count)
-				i = 0;
-
-			buffer_info = &tx_ring->buffer_info[i];
-			size = min(len, IXGB_MAX_DATA_PER_TXD);
-
-			/* Workaround for premature desc write-backs
-			 * in TSO mode.  Append 4-byte sentinel desc */
-			if (unlikely(mss && (f == (nr_frags - 1))
-				     && size == len && size > 8))
-				size -= 4;
-
-			buffer_info->length = size;
-			buffer_info->time_stamp = jiffies;
-			buffer_info->mapped_as_page = true;
-			buffer_info->dma =
-				skb_frag_dma_map(&pdev->dev, frag, offset, size,
-						 DMA_TO_DEVICE);
-			if (dma_mapping_error(&pdev->dev, buffer_info->dma))
-				goto dma_error;
-			buffer_info->next_to_watch = 0;
-
-			len -= size;
-			offset += size;
-			count++;
-		}
-	}
-	tx_ring->buffer_info[i].skb = skb;
-	tx_ring->buffer_info[first].next_to_watch = i;
-
-	return count;
-
-dma_error:
-	dev_err(&pdev->dev, "TX DMA map failed\n");
-	buffer_info->dma = 0;
-	if (count)
-		count--;
-
-	while (count--) {
-		if (i==0)
-			i += tx_ring->count;
-		i--;
-		buffer_info = &tx_ring->buffer_info[i];
-		ixgb_unmap_and_free_tx_resource(adapter, buffer_info);
-	}
-
-	return 0;
-}
-
-static void
-ixgb_tx_queue(struct ixgb_adapter *adapter, int count, int vlan_id,int tx_flags)
-{
-	struct ixgb_desc_ring *tx_ring = &adapter->tx_ring;
-	struct ixgb_tx_desc *tx_desc = NULL;
-	struct ixgb_buffer *buffer_info;
-	u32 cmd_type_len = adapter->tx_cmd_type;
-	u8 status = 0;
-	u8 popts = 0;
-	unsigned int i;
-
-	if (tx_flags & IXGB_TX_FLAGS_TSO) {
-		cmd_type_len |= IXGB_TX_DESC_CMD_TSE;
-		popts |= (IXGB_TX_DESC_POPTS_IXSM | IXGB_TX_DESC_POPTS_TXSM);
-	}
-
-	if (tx_flags & IXGB_TX_FLAGS_CSUM)
-		popts |= IXGB_TX_DESC_POPTS_TXSM;
-
-	if (tx_flags & IXGB_TX_FLAGS_VLAN)
-		cmd_type_len |= IXGB_TX_DESC_CMD_VLE;
-
-	i = tx_ring->next_to_use;
-
-	while (count--) {
-		buffer_info = &tx_ring->buffer_info[i];
-		tx_desc = IXGB_TX_DESC(*tx_ring, i);
-		tx_desc->buff_addr = cpu_to_le64(buffer_info->dma);
-		tx_desc->cmd_type_len =
-			cpu_to_le32(cmd_type_len | buffer_info->length);
-		tx_desc->status = status;
-		tx_desc->popts = popts;
-		tx_desc->vlan = cpu_to_le16(vlan_id);
-
-		if (++i == tx_ring->count) i = 0;
-	}
-
-	tx_desc->cmd_type_len |=
-		cpu_to_le32(IXGB_TX_DESC_CMD_EOP | IXGB_TX_DESC_CMD_RS);
-
-	/* Force memory writes to complete before letting h/w
-	 * know there are new descriptors to fetch.  (Only
-	 * applicable for weak-ordered memory model archs,
-	 * such as IA-64). */
-	wmb();
-
-	tx_ring->next_to_use = i;
-	IXGB_WRITE_REG(&adapter->hw, TDT, i);
-}
-
-static int __ixgb_maybe_stop_tx(struct net_device *netdev, int size)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct ixgb_desc_ring *tx_ring = &adapter->tx_ring;
-
-	netif_stop_queue(netdev);
-	/* Herbert's original patch had:
-	 *  smp_mb__after_netif_stop_queue();
-	 * but since that doesn't exist yet, just open code it. */
-	smp_mb();
-
-	/* We need to check again in a case another CPU has just
-	 * made room available. */
-	if (likely(IXGB_DESC_UNUSED(tx_ring) < size))
-		return -EBUSY;
-
-	/* A reprieve! */
-	netif_start_queue(netdev);
-	++adapter->restart_queue;
-	return 0;
-}
-
-static int ixgb_maybe_stop_tx(struct net_device *netdev,
-                              struct ixgb_desc_ring *tx_ring, int size)
-{
-	if (likely(IXGB_DESC_UNUSED(tx_ring) >= size))
-		return 0;
-	return __ixgb_maybe_stop_tx(netdev, size);
-}
-
-
-/* Tx Descriptors needed, worst case */
-#define TXD_USE_COUNT(S) (((S) >> IXGB_MAX_TXD_PWR) + \
-			 (((S) & (IXGB_MAX_DATA_PER_TXD - 1)) ? 1 : 0))
-#define DESC_NEEDED TXD_USE_COUNT(IXGB_MAX_DATA_PER_TXD) /* skb->date */ + \
-	MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1 /* for context */ \
-	+ 1 /* one more needed for sentinel TSO workaround */
-
-static netdev_tx_t
-ixgb_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	unsigned int first;
-	unsigned int tx_flags = 0;
-	int vlan_id = 0;
-	int count = 0;
-	int tso;
-
-	if (test_bit(__IXGB_DOWN, &adapter->flags)) {
-		dev_kfree_skb_any(skb);
-		return NETDEV_TX_OK;
-	}
-
-	if (skb->len <= 0) {
-		dev_kfree_skb_any(skb);
-		return NETDEV_TX_OK;
-	}
-
-	if (unlikely(ixgb_maybe_stop_tx(netdev, &adapter->tx_ring,
-                     DESC_NEEDED)))
-		return NETDEV_TX_BUSY;
-
-	if (skb_vlan_tag_present(skb)) {
-		tx_flags |= IXGB_TX_FLAGS_VLAN;
-		vlan_id = skb_vlan_tag_get(skb);
-	}
-
-	first = adapter->tx_ring.next_to_use;
-
-	tso = ixgb_tso(adapter, skb);
-	if (tso < 0) {
-		dev_kfree_skb_any(skb);
-		return NETDEV_TX_OK;
-	}
-
-	if (likely(tso))
-		tx_flags |= IXGB_TX_FLAGS_TSO;
-	else if (ixgb_tx_csum(adapter, skb))
-		tx_flags |= IXGB_TX_FLAGS_CSUM;
-
-	count = ixgb_tx_map(adapter, skb, first);
-
-	if (count) {
-		ixgb_tx_queue(adapter, count, vlan_id, tx_flags);
-		/* Make sure there is space in the ring for the next send. */
-		ixgb_maybe_stop_tx(netdev, &adapter->tx_ring, DESC_NEEDED);
-
-	} else {
-		dev_kfree_skb_any(skb);
-		adapter->tx_ring.buffer_info[first].time_stamp = 0;
-		adapter->tx_ring.next_to_use = first;
-	}
-
-	return NETDEV_TX_OK;
-}
-
-/**
- * ixgb_tx_timeout - Respond to a Tx Hang
- * @netdev: network interface device structure
- * @txqueue: queue hanging (unused)
- **/
-
-static void
-ixgb_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-
-	/* Do the reset outside of interrupt context */
-	schedule_work(&adapter->tx_timeout_task);
-}
-
-static void
-ixgb_tx_timeout_task(struct work_struct *work)
-{
-	struct ixgb_adapter *adapter =
-		container_of(work, struct ixgb_adapter, tx_timeout_task);
-
-	adapter->tx_timeout_count++;
-	ixgb_down(adapter, true);
-	ixgb_up(adapter);
-}
-
-/**
- * ixgb_change_mtu - Change the Maximum Transfer Unit
- * @netdev: network interface device structure
- * @new_mtu: new value for maximum frame size
- *
- * Returns 0 on success, negative on failure
- **/
-
-static int
-ixgb_change_mtu(struct net_device *netdev, int new_mtu)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	int max_frame = new_mtu + ENET_HEADER_SIZE + ENET_FCS_LENGTH;
-
-	if (netif_running(netdev))
-		ixgb_down(adapter, true);
-
-	adapter->rx_buffer_len = max_frame + 8; /* + 8 for errata */
-
-	netdev->mtu = new_mtu;
-
-	if (netif_running(netdev))
-		ixgb_up(adapter);
-
-	return 0;
-}
-
-/**
- * ixgb_update_stats - Update the board statistics counters.
- * @adapter: board private structure
- **/
-
-void
-ixgb_update_stats(struct ixgb_adapter *adapter)
-{
-	struct net_device *netdev = adapter->netdev;
-	struct pci_dev *pdev = adapter->pdev;
-
-	/* Prevent stats update while adapter is being reset */
-	if (pci_channel_offline(pdev))
-		return;
-
-	if ((netdev->flags & IFF_PROMISC) || (netdev->flags & IFF_ALLMULTI) ||
-	   (netdev_mc_count(netdev) > IXGB_MAX_NUM_MULTICAST_ADDRESSES)) {
-		u64 multi = IXGB_READ_REG(&adapter->hw, MPRCL);
-		u32 bcast_l = IXGB_READ_REG(&adapter->hw, BPRCL);
-		u32 bcast_h = IXGB_READ_REG(&adapter->hw, BPRCH);
-		u64 bcast = ((u64)bcast_h << 32) | bcast_l;
-
-		multi |= ((u64)IXGB_READ_REG(&adapter->hw, MPRCH) << 32);
-		/* fix up multicast stats by removing broadcasts */
-		if (multi >= bcast)
-			multi -= bcast;
-
-		adapter->stats.mprcl += (multi & 0xFFFFFFFF);
-		adapter->stats.mprch += (multi >> 32);
-		adapter->stats.bprcl += bcast_l;
-		adapter->stats.bprch += bcast_h;
-	} else {
-		adapter->stats.mprcl += IXGB_READ_REG(&adapter->hw, MPRCL);
-		adapter->stats.mprch += IXGB_READ_REG(&adapter->hw, MPRCH);
-		adapter->stats.bprcl += IXGB_READ_REG(&adapter->hw, BPRCL);
-		adapter->stats.bprch += IXGB_READ_REG(&adapter->hw, BPRCH);
-	}
-	adapter->stats.tprl += IXGB_READ_REG(&adapter->hw, TPRL);
-	adapter->stats.tprh += IXGB_READ_REG(&adapter->hw, TPRH);
-	adapter->stats.gprcl += IXGB_READ_REG(&adapter->hw, GPRCL);
-	adapter->stats.gprch += IXGB_READ_REG(&adapter->hw, GPRCH);
-	adapter->stats.uprcl += IXGB_READ_REG(&adapter->hw, UPRCL);
-	adapter->stats.uprch += IXGB_READ_REG(&adapter->hw, UPRCH);
-	adapter->stats.vprcl += IXGB_READ_REG(&adapter->hw, VPRCL);
-	adapter->stats.vprch += IXGB_READ_REG(&adapter->hw, VPRCH);
-	adapter->stats.jprcl += IXGB_READ_REG(&adapter->hw, JPRCL);
-	adapter->stats.jprch += IXGB_READ_REG(&adapter->hw, JPRCH);
-	adapter->stats.gorcl += IXGB_READ_REG(&adapter->hw, GORCL);
-	adapter->stats.gorch += IXGB_READ_REG(&adapter->hw, GORCH);
-	adapter->stats.torl += IXGB_READ_REG(&adapter->hw, TORL);
-	adapter->stats.torh += IXGB_READ_REG(&adapter->hw, TORH);
-	adapter->stats.rnbc += IXGB_READ_REG(&adapter->hw, RNBC);
-	adapter->stats.ruc += IXGB_READ_REG(&adapter->hw, RUC);
-	adapter->stats.roc += IXGB_READ_REG(&adapter->hw, ROC);
-	adapter->stats.rlec += IXGB_READ_REG(&adapter->hw, RLEC);
-	adapter->stats.crcerrs += IXGB_READ_REG(&adapter->hw, CRCERRS);
-	adapter->stats.icbc += IXGB_READ_REG(&adapter->hw, ICBC);
-	adapter->stats.ecbc += IXGB_READ_REG(&adapter->hw, ECBC);
-	adapter->stats.mpc += IXGB_READ_REG(&adapter->hw, MPC);
-	adapter->stats.tptl += IXGB_READ_REG(&adapter->hw, TPTL);
-	adapter->stats.tpth += IXGB_READ_REG(&adapter->hw, TPTH);
-	adapter->stats.gptcl += IXGB_READ_REG(&adapter->hw, GPTCL);
-	adapter->stats.gptch += IXGB_READ_REG(&adapter->hw, GPTCH);
-	adapter->stats.bptcl += IXGB_READ_REG(&adapter->hw, BPTCL);
-	adapter->stats.bptch += IXGB_READ_REG(&adapter->hw, BPTCH);
-	adapter->stats.mptcl += IXGB_READ_REG(&adapter->hw, MPTCL);
-	adapter->stats.mptch += IXGB_READ_REG(&adapter->hw, MPTCH);
-	adapter->stats.uptcl += IXGB_READ_REG(&adapter->hw, UPTCL);
-	adapter->stats.uptch += IXGB_READ_REG(&adapter->hw, UPTCH);
-	adapter->stats.vptcl += IXGB_READ_REG(&adapter->hw, VPTCL);
-	adapter->stats.vptch += IXGB_READ_REG(&adapter->hw, VPTCH);
-	adapter->stats.jptcl += IXGB_READ_REG(&adapter->hw, JPTCL);
-	adapter->stats.jptch += IXGB_READ_REG(&adapter->hw, JPTCH);
-	adapter->stats.gotcl += IXGB_READ_REG(&adapter->hw, GOTCL);
-	adapter->stats.gotch += IXGB_READ_REG(&adapter->hw, GOTCH);
-	adapter->stats.totl += IXGB_READ_REG(&adapter->hw, TOTL);
-	adapter->stats.toth += IXGB_READ_REG(&adapter->hw, TOTH);
-	adapter->stats.dc += IXGB_READ_REG(&adapter->hw, DC);
-	adapter->stats.plt64c += IXGB_READ_REG(&adapter->hw, PLT64C);
-	adapter->stats.tsctc += IXGB_READ_REG(&adapter->hw, TSCTC);
-	adapter->stats.tsctfc += IXGB_READ_REG(&adapter->hw, TSCTFC);
-	adapter->stats.ibic += IXGB_READ_REG(&adapter->hw, IBIC);
-	adapter->stats.rfc += IXGB_READ_REG(&adapter->hw, RFC);
-	adapter->stats.lfc += IXGB_READ_REG(&adapter->hw, LFC);
-	adapter->stats.pfrc += IXGB_READ_REG(&adapter->hw, PFRC);
-	adapter->stats.pftc += IXGB_READ_REG(&adapter->hw, PFTC);
-	adapter->stats.mcfrc += IXGB_READ_REG(&adapter->hw, MCFRC);
-	adapter->stats.mcftc += IXGB_READ_REG(&adapter->hw, MCFTC);
-	adapter->stats.xonrxc += IXGB_READ_REG(&adapter->hw, XONRXC);
-	adapter->stats.xontxc += IXGB_READ_REG(&adapter->hw, XONTXC);
-	adapter->stats.xoffrxc += IXGB_READ_REG(&adapter->hw, XOFFRXC);
-	adapter->stats.xofftxc += IXGB_READ_REG(&adapter->hw, XOFFTXC);
-	adapter->stats.rjc += IXGB_READ_REG(&adapter->hw, RJC);
-
-	/* Fill out the OS statistics structure */
-
-	netdev->stats.rx_packets = adapter->stats.gprcl;
-	netdev->stats.tx_packets = adapter->stats.gptcl;
-	netdev->stats.rx_bytes = adapter->stats.gorcl;
-	netdev->stats.tx_bytes = adapter->stats.gotcl;
-	netdev->stats.multicast = adapter->stats.mprcl;
-	netdev->stats.collisions = 0;
-
-	/* ignore RLEC as it reports errors for padded (<64bytes) frames
-	 * with a length in the type/len field */
-	netdev->stats.rx_errors =
-	    /* adapter->stats.rnbc + */ adapter->stats.crcerrs +
-	    adapter->stats.ruc +
-	    adapter->stats.roc /*+ adapter->stats.rlec */  +
-	    adapter->stats.icbc +
-	    adapter->stats.ecbc + adapter->stats.mpc;
-
-	/* see above
-	 * netdev->stats.rx_length_errors = adapter->stats.rlec;
-	 */
-
-	netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
-	netdev->stats.rx_fifo_errors = adapter->stats.mpc;
-	netdev->stats.rx_missed_errors = adapter->stats.mpc;
-	netdev->stats.rx_over_errors = adapter->stats.mpc;
-
-	netdev->stats.tx_errors = 0;
-	netdev->stats.rx_frame_errors = 0;
-	netdev->stats.tx_aborted_errors = 0;
-	netdev->stats.tx_carrier_errors = 0;
-	netdev->stats.tx_fifo_errors = 0;
-	netdev->stats.tx_heartbeat_errors = 0;
-	netdev->stats.tx_window_errors = 0;
-}
-
-/**
- * ixgb_intr - Interrupt Handler
- * @irq: interrupt number
- * @data: pointer to a network interface device structure
- **/
-
-static irqreturn_t
-ixgb_intr(int irq, void *data)
-{
-	struct net_device *netdev = data;
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	struct ixgb_hw *hw = &adapter->hw;
-	u32 icr = IXGB_READ_REG(hw, ICR);
-
-	if (unlikely(!icr))
-		return IRQ_NONE;  /* Not our interrupt */
-
-	if (unlikely(icr & (IXGB_INT_RXSEQ | IXGB_INT_LSC)))
-		if (!test_bit(__IXGB_DOWN, &adapter->flags))
-			mod_timer(&adapter->watchdog_timer, jiffies);
-
-	if (napi_schedule_prep(&adapter->napi)) {
-
-		/* Disable interrupts and register for poll. The flush
-		  of the posted write is intentionally left out.
-		*/
-
-		IXGB_WRITE_REG(&adapter->hw, IMC, ~0);
-		__napi_schedule(&adapter->napi);
-	}
-	return IRQ_HANDLED;
-}
-
-/**
- * ixgb_clean - NAPI Rx polling callback
- * @napi: napi struct pointer
- * @budget: max number of receives to clean
- **/
-
-static int
-ixgb_clean(struct napi_struct *napi, int budget)
-{
-	struct ixgb_adapter *adapter = container_of(napi, struct ixgb_adapter, napi);
-	int work_done = 0;
-
-	ixgb_clean_tx_irq(adapter);
-	ixgb_clean_rx_irq(adapter, &work_done, budget);
-
-	/* If budget not fully consumed, exit the polling mode */
-	if (work_done < budget) {
-		napi_complete_done(napi, work_done);
-		if (!test_bit(__IXGB_DOWN, &adapter->flags))
-			ixgb_irq_enable(adapter);
-	}
-
-	return work_done;
-}
-
-/**
- * ixgb_clean_tx_irq - Reclaim resources after transmit completes
- * @adapter: board private structure
- **/
-
-static bool
-ixgb_clean_tx_irq(struct ixgb_adapter *adapter)
-{
-	struct ixgb_desc_ring *tx_ring = &adapter->tx_ring;
-	struct net_device *netdev = adapter->netdev;
-	struct ixgb_tx_desc *tx_desc, *eop_desc;
-	struct ixgb_buffer *buffer_info;
-	unsigned int i, eop;
-	bool cleaned = false;
-
-	i = tx_ring->next_to_clean;
-	eop = tx_ring->buffer_info[i].next_to_watch;
-	eop_desc = IXGB_TX_DESC(*tx_ring, eop);
-
-	while (eop_desc->status & IXGB_TX_DESC_STATUS_DD) {
-
-		rmb(); /* read buffer_info after eop_desc */
-		for (cleaned = false; !cleaned; ) {
-			tx_desc = IXGB_TX_DESC(*tx_ring, i);
-			buffer_info = &tx_ring->buffer_info[i];
-
-			if (tx_desc->popts &
-			   (IXGB_TX_DESC_POPTS_TXSM |
-			    IXGB_TX_DESC_POPTS_IXSM))
-				adapter->hw_csum_tx_good++;
-
-			ixgb_unmap_and_free_tx_resource(adapter, buffer_info);
-
-			*(u32 *)&(tx_desc->status) = 0;
-
-			cleaned = (i == eop);
-			if (++i == tx_ring->count) i = 0;
-		}
-
-		eop = tx_ring->buffer_info[i].next_to_watch;
-		eop_desc = IXGB_TX_DESC(*tx_ring, eop);
-	}
-
-	tx_ring->next_to_clean = i;
-
-	if (unlikely(cleaned && netif_carrier_ok(netdev) &&
-		     IXGB_DESC_UNUSED(tx_ring) >= DESC_NEEDED)) {
-		/* Make sure that anybody stopping the queue after this
-		 * sees the new next_to_clean. */
-		smp_mb();
-
-		if (netif_queue_stopped(netdev) &&
-		    !(test_bit(__IXGB_DOWN, &adapter->flags))) {
-			netif_wake_queue(netdev);
-			++adapter->restart_queue;
-		}
-	}
-
-	if (adapter->detect_tx_hung) {
-		/* detect a transmit hang in hardware, this serializes the
-		 * check with the clearing of time_stamp and movement of i */
-		adapter->detect_tx_hung = false;
-		if (tx_ring->buffer_info[eop].time_stamp &&
-		   time_after(jiffies, tx_ring->buffer_info[eop].time_stamp + HZ)
-		   && !(IXGB_READ_REG(&adapter->hw, STATUS) &
-		        IXGB_STATUS_TXOFF)) {
-			/* detected Tx unit hang */
-			netif_err(adapter, drv, adapter->netdev,
-				  "Detected Tx Unit Hang\n"
-				  "  TDH                  <%x>\n"
-				  "  TDT                  <%x>\n"
-				  "  next_to_use          <%x>\n"
-				  "  next_to_clean        <%x>\n"
-				  "buffer_info[next_to_clean]\n"
-				  "  time_stamp           <%lx>\n"
-				  "  next_to_watch        <%x>\n"
-				  "  jiffies              <%lx>\n"
-				  "  next_to_watch.status <%x>\n",
-				  IXGB_READ_REG(&adapter->hw, TDH),
-				  IXGB_READ_REG(&adapter->hw, TDT),
-				  tx_ring->next_to_use,
-				  tx_ring->next_to_clean,
-				  tx_ring->buffer_info[eop].time_stamp,
-				  eop,
-				  jiffies,
-				  eop_desc->status);
-			netif_stop_queue(netdev);
-		}
-	}
-
-	return cleaned;
-}
-
-/**
- * ixgb_rx_checksum - Receive Checksum Offload for 82597.
- * @adapter: board private structure
- * @rx_desc: receive descriptor
- * @skb: socket buffer with received data
- **/
-
-static void
-ixgb_rx_checksum(struct ixgb_adapter *adapter,
-                 struct ixgb_rx_desc *rx_desc,
-                 struct sk_buff *skb)
-{
-	/* Ignore Checksum bit is set OR
-	 * TCP Checksum has not been calculated
-	 */
-	if ((rx_desc->status & IXGB_RX_DESC_STATUS_IXSM) ||
-	   (!(rx_desc->status & IXGB_RX_DESC_STATUS_TCPCS))) {
-		skb_checksum_none_assert(skb);
-		return;
-	}
-
-	/* At this point we know the hardware did the TCP checksum */
-	/* now look at the TCP checksum error bit */
-	if (rx_desc->errors & IXGB_RX_DESC_ERRORS_TCPE) {
-		/* let the stack verify checksum errors */
-		skb_checksum_none_assert(skb);
-		adapter->hw_csum_rx_error++;
-	} else {
-		/* TCP checksum is good */
-		skb->ip_summed = CHECKSUM_UNNECESSARY;
-		adapter->hw_csum_rx_good++;
-	}
-}
-
-/*
- * this should improve performance for small packets with large amounts
- * of reassembly being done in the stack
- */
-static void ixgb_check_copybreak(struct napi_struct *napi,
-				 struct ixgb_buffer *buffer_info,
-				 u32 length, struct sk_buff **skb)
-{
-	struct sk_buff *new_skb;
-
-	if (length > copybreak)
-		return;
-
-	new_skb = napi_alloc_skb(napi, length);
-	if (!new_skb)
-		return;
-
-	skb_copy_to_linear_data_offset(new_skb, -NET_IP_ALIGN,
-				       (*skb)->data - NET_IP_ALIGN,
-				       length + NET_IP_ALIGN);
-	/* save the skb in buffer_info as good */
-	buffer_info->skb = *skb;
-	*skb = new_skb;
-}
-
-/**
- * ixgb_clean_rx_irq - Send received data up the network stack,
- * @adapter: board private structure
- * @work_done: output pointer to amount of packets cleaned
- * @work_to_do: how much work we can complete
- **/
-
-static bool
-ixgb_clean_rx_irq(struct ixgb_adapter *adapter, int *work_done, int work_to_do)
-{
-	struct ixgb_desc_ring *rx_ring = &adapter->rx_ring;
-	struct net_device *netdev = adapter->netdev;
-	struct pci_dev *pdev = adapter->pdev;
-	struct ixgb_rx_desc *rx_desc, *next_rxd;
-	struct ixgb_buffer *buffer_info, *next_buffer, *next2_buffer;
-	u32 length;
-	unsigned int i, j;
-	int cleaned_count = 0;
-	bool cleaned = false;
-
-	i = rx_ring->next_to_clean;
-	rx_desc = IXGB_RX_DESC(*rx_ring, i);
-	buffer_info = &rx_ring->buffer_info[i];
-
-	while (rx_desc->status & IXGB_RX_DESC_STATUS_DD) {
-		struct sk_buff *skb;
-		u8 status;
-
-		if (*work_done >= work_to_do)
-			break;
-
-		(*work_done)++;
-		rmb();	/* read descriptor and rx_buffer_info after status DD */
-		status = rx_desc->status;
-		skb = buffer_info->skb;
-		buffer_info->skb = NULL;
-
-		prefetch(skb->data - NET_IP_ALIGN);
-
-		if (++i == rx_ring->count)
-			i = 0;
-		next_rxd = IXGB_RX_DESC(*rx_ring, i);
-		prefetch(next_rxd);
-
-		j = i + 1;
-		if (j == rx_ring->count)
-			j = 0;
-		next2_buffer = &rx_ring->buffer_info[j];
-		prefetch(next2_buffer);
-
-		next_buffer = &rx_ring->buffer_info[i];
-
-		cleaned = true;
-		cleaned_count++;
-
-		dma_unmap_single(&pdev->dev,
-				 buffer_info->dma,
-				 buffer_info->length,
-				 DMA_FROM_DEVICE);
-		buffer_info->dma = 0;
-
-		length = le16_to_cpu(rx_desc->length);
-		rx_desc->length = 0;
-
-		if (unlikely(!(status & IXGB_RX_DESC_STATUS_EOP))) {
-
-			/* All receives must fit into a single buffer */
-
-			pr_debug("Receive packet consumed multiple buffers length<%x>\n",
-				 length);
-
-			dev_kfree_skb_irq(skb);
-			goto rxdesc_done;
-		}
-
-		if (unlikely(rx_desc->errors &
-		    (IXGB_RX_DESC_ERRORS_CE | IXGB_RX_DESC_ERRORS_SE |
-		     IXGB_RX_DESC_ERRORS_P | IXGB_RX_DESC_ERRORS_RXE))) {
-			dev_kfree_skb_irq(skb);
-			goto rxdesc_done;
-		}
-
-		ixgb_check_copybreak(&adapter->napi, buffer_info, length, &skb);
-
-		/* Good Receive */
-		skb_put(skb, length);
-
-		/* Receive Checksum Offload */
-		ixgb_rx_checksum(adapter, rx_desc, skb);
-
-		skb->protocol = eth_type_trans(skb, netdev);
-		if (status & IXGB_RX_DESC_STATUS_VP)
-			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
-				       le16_to_cpu(rx_desc->special));
-
-		netif_receive_skb(skb);
-
-rxdesc_done:
-		/* clean up descriptor, might be written over by hw */
-		rx_desc->status = 0;
-
-		/* return some buffers to hardware, one at a time is too slow */
-		if (unlikely(cleaned_count >= IXGB_RX_BUFFER_WRITE)) {
-			ixgb_alloc_rx_buffers(adapter, cleaned_count);
-			cleaned_count = 0;
-		}
-
-		/* use prefetched values */
-		rx_desc = next_rxd;
-		buffer_info = next_buffer;
-	}
-
-	rx_ring->next_to_clean = i;
-
-	cleaned_count = IXGB_DESC_UNUSED(rx_ring);
-	if (cleaned_count)
-		ixgb_alloc_rx_buffers(adapter, cleaned_count);
-
-	return cleaned;
-}
-
-/**
- * ixgb_alloc_rx_buffers - Replace used receive buffers
- * @adapter: address of board private structure
- * @cleaned_count: how many buffers to allocate
- **/
-
-static void
-ixgb_alloc_rx_buffers(struct ixgb_adapter *adapter, int cleaned_count)
-{
-	struct ixgb_desc_ring *rx_ring = &adapter->rx_ring;
-	struct net_device *netdev = adapter->netdev;
-	struct pci_dev *pdev = adapter->pdev;
-	struct ixgb_rx_desc *rx_desc;
-	struct ixgb_buffer *buffer_info;
-	struct sk_buff *skb;
-	unsigned int i;
-	long cleancount;
-
-	i = rx_ring->next_to_use;
-	buffer_info = &rx_ring->buffer_info[i];
-	cleancount = IXGB_DESC_UNUSED(rx_ring);
-
-
-	/* leave three descriptors unused */
-	while (--cleancount > 2 && cleaned_count--) {
-		/* recycle! its good for you */
-		skb = buffer_info->skb;
-		if (skb) {
-			skb_trim(skb, 0);
-			goto map_skb;
-		}
-
-		skb = netdev_alloc_skb_ip_align(netdev, adapter->rx_buffer_len);
-		if (unlikely(!skb)) {
-			/* Better luck next round */
-			adapter->alloc_rx_buff_failed++;
-			break;
-		}
-
-		buffer_info->skb = skb;
-		buffer_info->length = adapter->rx_buffer_len;
-map_skb:
-		buffer_info->dma = dma_map_single(&pdev->dev,
-		                                  skb->data,
-		                                  adapter->rx_buffer_len,
-						  DMA_FROM_DEVICE);
-		if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
-			adapter->alloc_rx_buff_failed++;
-			break;
-		}
-
-		rx_desc = IXGB_RX_DESC(*rx_ring, i);
-		rx_desc->buff_addr = cpu_to_le64(buffer_info->dma);
-		/* guarantee DD bit not set now before h/w gets descriptor
-		 * this is the rest of the workaround for h/w double
-		 * writeback. */
-		rx_desc->status = 0;
-
-
-		if (++i == rx_ring->count)
-			i = 0;
-		buffer_info = &rx_ring->buffer_info[i];
-	}
-
-	if (likely(rx_ring->next_to_use != i)) {
-		rx_ring->next_to_use = i;
-		if (unlikely(i-- == 0))
-			i = (rx_ring->count - 1);
-
-		/* Force memory writes to complete before letting h/w
-		 * know there are new descriptors to fetch.  (Only
-		 * applicable for weak-ordered memory model archs, such
-		 * as IA-64). */
-		wmb();
-		IXGB_WRITE_REG(&adapter->hw, RDT, i);
-	}
-}
-
-static void
-ixgb_vlan_strip_enable(struct ixgb_adapter *adapter)
-{
-	u32 ctrl;
-
-	/* enable VLAN tag insert/strip */
-	ctrl = IXGB_READ_REG(&adapter->hw, CTRL0);
-	ctrl |= IXGB_CTRL0_VME;
-	IXGB_WRITE_REG(&adapter->hw, CTRL0, ctrl);
-}
-
-static void
-ixgb_vlan_strip_disable(struct ixgb_adapter *adapter)
-{
-	u32 ctrl;
-
-	/* disable VLAN tag insert/strip */
-	ctrl = IXGB_READ_REG(&adapter->hw, CTRL0);
-	ctrl &= ~IXGB_CTRL0_VME;
-	IXGB_WRITE_REG(&adapter->hw, CTRL0, ctrl);
-}
-
-static int
-ixgb_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	u32 vfta, index;
-
-	/* add VID to filter table */
-
-	index = (vid >> 5) & 0x7F;
-	vfta = IXGB_READ_REG_ARRAY(&adapter->hw, VFTA, index);
-	vfta |= (1 << (vid & 0x1F));
-	ixgb_write_vfta(&adapter->hw, index, vfta);
-	set_bit(vid, adapter->active_vlans);
-
-	return 0;
-}
-
-static int
-ixgb_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
-{
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	u32 vfta, index;
-
-	/* remove VID from filter table */
-
-	index = (vid >> 5) & 0x7F;
-	vfta = IXGB_READ_REG_ARRAY(&adapter->hw, VFTA, index);
-	vfta &= ~(1 << (vid & 0x1F));
-	ixgb_write_vfta(&adapter->hw, index, vfta);
-	clear_bit(vid, adapter->active_vlans);
-
-	return 0;
-}
-
-static void
-ixgb_restore_vlan(struct ixgb_adapter *adapter)
-{
-	u16 vid;
-
-	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
-		ixgb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
-}
-
-/**
- * ixgb_io_error_detected - called when PCI error is detected
- * @pdev:    pointer to pci device with error
- * @state:   pci channel state after error
- *
- * This callback is called by the PCI subsystem whenever
- * a PCI bus error is detected.
- */
-static pci_ers_result_t ixgb_io_error_detected(struct pci_dev *pdev,
-                                               pci_channel_state_t state)
-{
-	struct net_device *netdev = pci_get_drvdata(pdev);
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-
-	netif_device_detach(netdev);
-
-	if (state == pci_channel_io_perm_failure)
-		return PCI_ERS_RESULT_DISCONNECT;
-
-	if (netif_running(netdev))
-		ixgb_down(adapter, true);
-
-	pci_disable_device(pdev);
-
-	/* Request a slot reset. */
-	return PCI_ERS_RESULT_NEED_RESET;
-}
-
-/**
- * ixgb_io_slot_reset - called after the pci bus has been reset.
- * @pdev: pointer to pci device with error
- *
- * This callback is called after the PCI bus has been reset.
- * Basically, this tries to restart the card from scratch.
- * This is a shortened version of the device probe/discovery code,
- * it resembles the first-half of the ixgb_probe() routine.
- */
-static pci_ers_result_t ixgb_io_slot_reset(struct pci_dev *pdev)
-{
-	struct net_device *netdev = pci_get_drvdata(pdev);
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-	u8 addr[ETH_ALEN];
-
-	if (pci_enable_device(pdev)) {
-		netif_err(adapter, probe, adapter->netdev,
-			  "Cannot re-enable PCI device after reset\n");
-		return PCI_ERS_RESULT_DISCONNECT;
-	}
-
-	/* Perform card reset only on one instance of the card */
-	if (0 != PCI_FUNC (pdev->devfn))
-		return PCI_ERS_RESULT_RECOVERED;
-
-	pci_set_master(pdev);
-
-	netif_carrier_off(netdev);
-	netif_stop_queue(netdev);
-	ixgb_reset(adapter);
-
-	/* Make sure the EEPROM is good */
-	if (!ixgb_validate_eeprom_checksum(&adapter->hw)) {
-		netif_err(adapter, probe, adapter->netdev,
-			  "After reset, the EEPROM checksum is not valid\n");
-		return PCI_ERS_RESULT_DISCONNECT;
-	}
-	ixgb_get_ee_mac_addr(&adapter->hw, addr);
-	eth_hw_addr_set(netdev, addr);
-	memcpy(netdev->perm_addr, netdev->dev_addr, netdev->addr_len);
-
-	if (!is_valid_ether_addr(netdev->perm_addr)) {
-		netif_err(adapter, probe, adapter->netdev,
-			  "After reset, invalid MAC address\n");
-		return PCI_ERS_RESULT_DISCONNECT;
-	}
-
-	return PCI_ERS_RESULT_RECOVERED;
-}
-
-/**
- * ixgb_io_resume - called when its OK to resume normal operations
- * @pdev: pointer to pci device with error
- *
- * The error recovery driver tells us that its OK to resume
- * normal operation. Implementation resembles the second-half
- * of the ixgb_probe() routine.
- */
-static void ixgb_io_resume(struct pci_dev *pdev)
-{
-	struct net_device *netdev = pci_get_drvdata(pdev);
-	struct ixgb_adapter *adapter = netdev_priv(netdev);
-
-	pci_set_master(pdev);
-
-	if (netif_running(netdev)) {
-		if (ixgb_up(adapter)) {
-			pr_err("can't bring device back up after reset\n");
-			return;
-		}
-	}
-
-	netif_device_attach(netdev);
-	mod_timer(&adapter->watchdog_timer, jiffies);
-}
-
-/* ixgb_main.c */
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_osdep.h b/drivers/net/ethernet/intel/ixgb/ixgb_osdep.h
deleted file mode 100644
index 7bd54efa698d..000000000000
--- a/drivers/net/ethernet/intel/ixgb/ixgb_osdep.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* Copyright(c) 1999 - 2008 Intel Corporation. */
-
-/* glue for the OS independent part of ixgb
- * includes register access macros
- */
-
-#ifndef _IXGB_OSDEP_H_
-#define _IXGB_OSDEP_H_
-
-#include <linux/types.h>
-#include <linux/delay.h>
-#include <asm/io.h>
-#include <linux/interrupt.h>
-#include <linux/sched.h>
-#include <linux/if_ether.h>
-
-#undef ASSERT
-#define ASSERT(x)	BUG_ON(!(x))
-
-#define ENTER() pr_debug("%s\n", __func__);
-
-#define IXGB_WRITE_REG(a, reg, value) ( \
-	writel((value), ((a)->hw_addr + IXGB_##reg)))
-
-#define IXGB_READ_REG(a, reg) ( \
-	readl((a)->hw_addr + IXGB_##reg))
-
-#define IXGB_WRITE_REG_ARRAY(a, reg, offset, value) ( \
-	writel((value), ((a)->hw_addr + IXGB_##reg + ((offset) << 2))))
-
-#define IXGB_READ_REG_ARRAY(a, reg, offset) ( \
-	readl((a)->hw_addr + IXGB_##reg + ((offset) << 2)))
-
-#define IXGB_WRITE_FLUSH(a) IXGB_READ_REG(a, STATUS)
-
-#define IXGB_MEMCPY memcpy
-
-#endif /* _IXGB_OSDEP_H_ */
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_param.c b/drivers/net/ethernet/intel/ixgb/ixgb_param.c
deleted file mode 100644
index d40f96250691..000000000000
--- a/drivers/net/ethernet/intel/ixgb/ixgb_param.c
+++ /dev/null
@@ -1,442 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright(c) 1999 - 2008 Intel Corporation. */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include "ixgb.h"
-
-/* This is the only thing that needs to be changed to adjust the
- * maximum number of ports that the driver can manage.
- */
-
-#define IXGB_MAX_NIC 8
-
-#define OPTION_UNSET	-1
-#define OPTION_DISABLED 0
-#define OPTION_ENABLED  1
-
-/* All parameters are treated the same, as an integer array of values.
- * This macro just reduces the need to repeat the same declaration code
- * over and over (plus this helps to avoid typo bugs).
- */
-
-#define IXGB_PARAM_INIT { [0 ... IXGB_MAX_NIC] = OPTION_UNSET }
-#define IXGB_PARAM(X, desc)					\
-	static int X[IXGB_MAX_NIC+1]		\
-		= IXGB_PARAM_INIT;				\
-	static unsigned int num_##X = 0;			\
-	module_param_array_named(X, X, int, &num_##X, 0);	\
-	MODULE_PARM_DESC(X, desc);
-
-/* Transmit Descriptor Count
- *
- * Valid Range: 64-4096
- *
- * Default Value: 256
- */
-
-IXGB_PARAM(TxDescriptors, "Number of transmit descriptors");
-
-/* Receive Descriptor Count
- *
- * Valid Range: 64-4096
- *
- * Default Value: 1024
- */
-
-IXGB_PARAM(RxDescriptors, "Number of receive descriptors");
-
-/* User Specified Flow Control Override
- *
- * Valid Range: 0-3
- *  - 0 - No Flow Control
- *  - 1 - Rx only, respond to PAUSE frames but do not generate them
- *  - 2 - Tx only, generate PAUSE frames but ignore them on receive
- *  - 3 - Full Flow Control Support
- *
- * Default Value: 2 - Tx only (silicon bug avoidance)
- */
-
-IXGB_PARAM(FlowControl, "Flow Control setting");
-
-/* XsumRX - Receive Checksum Offload Enable/Disable
- *
- * Valid Range: 0, 1
- *  - 0 - disables all checksum offload
- *  - 1 - enables receive IP/TCP/UDP checksum offload
- *        on 82597 based NICs
- *
- * Default Value: 1
- */
-
-IXGB_PARAM(XsumRX, "Disable or enable Receive Checksum offload");
-
-/* Transmit Interrupt Delay in units of 0.8192 microseconds
- *
- * Valid Range: 0-65535
- *
- * Default Value: 32
- */
-
-IXGB_PARAM(TxIntDelay, "Transmit Interrupt Delay");
-
-/* Receive Interrupt Delay in units of 0.8192 microseconds
- *
- * Valid Range: 0-65535
- *
- * Default Value: 72
- */
-
-IXGB_PARAM(RxIntDelay, "Receive Interrupt Delay");
-
-/* Receive Flow control high threshold (when we send a pause frame)
- * (FCRTH)
- *
- * Valid Range: 1,536 - 262,136 (0x600 - 0x3FFF8, 8 byte granularity)
- *
- * Default Value: 196,608 (0x30000)
- */
-
-IXGB_PARAM(RxFCHighThresh, "Receive Flow Control High Threshold");
-
-/* Receive Flow control low threshold (when we send a resume frame)
- * (FCRTL)
- *
- * Valid Range: 64 - 262,136 (0x40 - 0x3FFF8, 8 byte granularity)
- *              must be less than high threshold by at least 8 bytes
- *
- * Default Value:  163,840 (0x28000)
- */
-
-IXGB_PARAM(RxFCLowThresh, "Receive Flow Control Low Threshold");
-
-/* Flow control request timeout (how long to pause the link partner's tx)
- * (PAP 15:0)
- *
- * Valid Range: 1 - 65535
- *
- * Default Value:  65535 (0xffff) (we'll send an xon if we recover)
- */
-
-IXGB_PARAM(FCReqTimeout, "Flow Control Request Timeout");
-
-/* Interrupt Delay Enable
- *
- * Valid Range: 0, 1
- *
- *  - 0 - disables transmit interrupt delay
- *  - 1 - enables transmmit interrupt delay
- *
- * Default Value: 1
- */
-
-IXGB_PARAM(IntDelayEnable, "Transmit Interrupt Delay Enable");
-
-
-#define DEFAULT_TIDV	   		     32
-#define MAX_TIDV			 0xFFFF
-#define MIN_TIDV			      0
-
-#define DEFAULT_RDTR		   	     72
-#define MAX_RDTR			 0xFFFF
-#define MIN_RDTR			      0
-
-#define DEFAULT_FCRTL	  		0x28000
-#define DEFAULT_FCRTH			0x30000
-#define MIN_FCRTL			      0
-#define MAX_FCRTL			0x3FFE8
-#define MIN_FCRTH			      8
-#define MAX_FCRTH			0x3FFF0
-
-#define MIN_FCPAUSE			      1
-#define MAX_FCPAUSE			 0xffff
-#define DEFAULT_FCPAUSE		  	 0xFFFF /* this may be too long */
-
-struct ixgb_option {
-	enum { enable_option, range_option, list_option } type;
-	const char *name;
-	const char *err;
-	int def;
-	union {
-		struct {	/* range_option info */
-			int min;
-			int max;
-		} r;
-		struct {	/* list_option info */
-			int nr;
-			const struct ixgb_opt_list {
-				int i;
-				const char *str;
-			} *p;
-		} l;
-	} arg;
-};
-
-static int
-ixgb_validate_option(unsigned int *value, const struct ixgb_option *opt)
-{
-	if (*value == OPTION_UNSET) {
-		*value = opt->def;
-		return 0;
-	}
-
-	switch (opt->type) {
-	case enable_option:
-		switch (*value) {
-		case OPTION_ENABLED:
-			pr_info("%s Enabled\n", opt->name);
-			return 0;
-		case OPTION_DISABLED:
-			pr_info("%s Disabled\n", opt->name);
-			return 0;
-		}
-		break;
-	case range_option:
-		if (*value >= opt->arg.r.min && *value <= opt->arg.r.max) {
-			pr_info("%s set to %i\n", opt->name, *value);
-			return 0;
-		}
-		break;
-	case list_option: {
-		int i;
-		const struct ixgb_opt_list *ent;
-
-		for (i = 0; i < opt->arg.l.nr; i++) {
-			ent = &opt->arg.l.p[i];
-			if (*value == ent->i) {
-				if (ent->str[0] != '\0')
-					pr_info("%s\n", ent->str);
-				return 0;
-			}
-		}
-	}
-		break;
-	default:
-		BUG();
-	}
-
-	pr_info("Invalid %s specified (%i) %s\n", opt->name, *value, opt->err);
-	*value = opt->def;
-	return -1;
-}
-
-/**
- * ixgb_check_options - Range Checking for Command Line Parameters
- * @adapter: board private structure
- *
- * This routine checks all command line parameters for valid user
- * input.  If an invalid value is given, or if no user specified
- * value exists, a default value is used.  The final value is stored
- * in a variable in the adapter structure.
- **/
-
-void
-ixgb_check_options(struct ixgb_adapter *adapter)
-{
-	int bd = adapter->bd_number;
-	if (bd >= IXGB_MAX_NIC) {
-		pr_notice("Warning: no configuration for board #%i\n", bd);
-		pr_notice("Using defaults for all values\n");
-	}
-
-	{ /* Transmit Descriptor Count */
-		static const struct ixgb_option opt = {
-			.type = range_option,
-			.name = "Transmit Descriptors",
-			.err  = "using default of " __MODULE_STRING(DEFAULT_TXD),
-			.def  = DEFAULT_TXD,
-			.arg  = { .r = { .min = MIN_TXD,
-					 .max = MAX_TXD}}
-		};
-		struct ixgb_desc_ring *tx_ring = &adapter->tx_ring;
-
-		if (num_TxDescriptors > bd) {
-			tx_ring->count = TxDescriptors[bd];
-			ixgb_validate_option(&tx_ring->count, &opt);
-		} else {
-			tx_ring->count = opt.def;
-		}
-		tx_ring->count = ALIGN(tx_ring->count, IXGB_REQ_TX_DESCRIPTOR_MULTIPLE);
-	}
-	{ /* Receive Descriptor Count */
-		static const struct ixgb_option opt = {
-			.type = range_option,
-			.name = "Receive Descriptors",
-			.err  = "using default of " __MODULE_STRING(DEFAULT_RXD),
-			.def  = DEFAULT_RXD,
-			.arg  = { .r = { .min = MIN_RXD,
-					 .max = MAX_RXD}}
-		};
-		struct ixgb_desc_ring *rx_ring = &adapter->rx_ring;
-
-		if (num_RxDescriptors > bd) {
-			rx_ring->count = RxDescriptors[bd];
-			ixgb_validate_option(&rx_ring->count, &opt);
-		} else {
-			rx_ring->count = opt.def;
-		}
-		rx_ring->count = ALIGN(rx_ring->count, IXGB_REQ_RX_DESCRIPTOR_MULTIPLE);
-	}
-	{ /* Receive Checksum Offload Enable */
-		static const struct ixgb_option opt = {
-			.type = enable_option,
-			.name = "Receive Checksum Offload",
-			.err  = "defaulting to Enabled",
-			.def  = OPTION_ENABLED
-		};
-
-		if (num_XsumRX > bd) {
-			unsigned int rx_csum = XsumRX[bd];
-			ixgb_validate_option(&rx_csum, &opt);
-			adapter->rx_csum = rx_csum;
-		} else {
-			adapter->rx_csum = opt.def;
-		}
-	}
-	{ /* Flow Control */
-
-		static const struct ixgb_opt_list fc_list[] = {
-		       { ixgb_fc_none, "Flow Control Disabled" },
-		       { ixgb_fc_rx_pause, "Flow Control Receive Only" },
-		       { ixgb_fc_tx_pause, "Flow Control Transmit Only" },
-		       { ixgb_fc_full, "Flow Control Enabled" },
-		       { ixgb_fc_default, "Flow Control Hardware Default" }
-		};
-
-		static const struct ixgb_option opt = {
-			.type = list_option,
-			.name = "Flow Control",
-			.err  = "reading default settings from EEPROM",
-			.def  = ixgb_fc_tx_pause,
-			.arg  = { .l = { .nr = ARRAY_SIZE(fc_list),
-					 .p = fc_list }}
-		};
-
-		if (num_FlowControl > bd) {
-			unsigned int fc = FlowControl[bd];
-			ixgb_validate_option(&fc, &opt);
-			adapter->hw.fc.type = fc;
-		} else {
-			adapter->hw.fc.type = opt.def;
-		}
-	}
-	{ /* Receive Flow Control High Threshold */
-		static const struct ixgb_option opt = {
-			.type = range_option,
-			.name = "Rx Flow Control High Threshold",
-			.err  = "using default of " __MODULE_STRING(DEFAULT_FCRTH),
-			.def  = DEFAULT_FCRTH,
-			.arg  = { .r = { .min = MIN_FCRTH,
-					 .max = MAX_FCRTH}}
-		};
-
-		if (num_RxFCHighThresh > bd) {
-			adapter->hw.fc.high_water = RxFCHighThresh[bd];
-			ixgb_validate_option(&adapter->hw.fc.high_water, &opt);
-		} else {
-			adapter->hw.fc.high_water = opt.def;
-		}
-		if (!(adapter->hw.fc.type & ixgb_fc_tx_pause) )
-			pr_info("Ignoring RxFCHighThresh when no RxFC\n");
-	}
-	{ /* Receive Flow Control Low Threshold */
-		static const struct ixgb_option opt = {
-			.type = range_option,
-			.name = "Rx Flow Control Low Threshold",
-			.err  = "using default of " __MODULE_STRING(DEFAULT_FCRTL),
-			.def  = DEFAULT_FCRTL,
-			.arg  = { .r = { .min = MIN_FCRTL,
-					 .max = MAX_FCRTL}}
-		};
-
-		if (num_RxFCLowThresh > bd) {
-			adapter->hw.fc.low_water = RxFCLowThresh[bd];
-			ixgb_validate_option(&adapter->hw.fc.low_water, &opt);
-		} else {
-			adapter->hw.fc.low_water = opt.def;
-		}
-		if (!(adapter->hw.fc.type & ixgb_fc_tx_pause) )
-			pr_info("Ignoring RxFCLowThresh when no RxFC\n");
-	}
-	{ /* Flow Control Pause Time Request*/
-		static const struct ixgb_option opt = {
-			.type = range_option,
-			.name = "Flow Control Pause Time Request",
-			.err  = "using default of "__MODULE_STRING(DEFAULT_FCPAUSE),
-			.def  = DEFAULT_FCPAUSE,
-			.arg = { .r = { .min = MIN_FCPAUSE,
-					.max = MAX_FCPAUSE}}
-		};
-
-		if (num_FCReqTimeout > bd) {
-			unsigned int pause_time = FCReqTimeout[bd];
-			ixgb_validate_option(&pause_time, &opt);
-			adapter->hw.fc.pause_time = pause_time;
-		} else {
-			adapter->hw.fc.pause_time = opt.def;
-		}
-		if (!(adapter->hw.fc.type & ixgb_fc_tx_pause) )
-			pr_info("Ignoring FCReqTimeout when no RxFC\n");
-	}
-	/* high low and spacing check for rx flow control thresholds */
-	if (adapter->hw.fc.type & ixgb_fc_tx_pause) {
-		/* high must be greater than low */
-		if (adapter->hw.fc.high_water < (adapter->hw.fc.low_water + 8)) {
-			/* set defaults */
-			pr_info("RxFCHighThresh must be >= (RxFCLowThresh + 8), Using Defaults\n");
-			adapter->hw.fc.high_water = DEFAULT_FCRTH;
-			adapter->hw.fc.low_water  = DEFAULT_FCRTL;
-		}
-	}
-	{ /* Receive Interrupt Delay */
-		static const struct ixgb_option opt = {
-			.type = range_option,
-			.name = "Receive Interrupt Delay",
-			.err  = "using default of " __MODULE_STRING(DEFAULT_RDTR),
-			.def  = DEFAULT_RDTR,
-			.arg  = { .r = { .min = MIN_RDTR,
-					 .max = MAX_RDTR}}
-		};
-
-		if (num_RxIntDelay > bd) {
-			adapter->rx_int_delay = RxIntDelay[bd];
-			ixgb_validate_option(&adapter->rx_int_delay, &opt);
-		} else {
-			adapter->rx_int_delay = opt.def;
-		}
-	}
-	{ /* Transmit Interrupt Delay */
-		static const struct ixgb_option opt = {
-			.type = range_option,
-			.name = "Transmit Interrupt Delay",
-			.err  = "using default of " __MODULE_STRING(DEFAULT_TIDV),
-			.def  = DEFAULT_TIDV,
-			.arg  = { .r = { .min = MIN_TIDV,
-					 .max = MAX_TIDV}}
-		};
-
-		if (num_TxIntDelay > bd) {
-			adapter->tx_int_delay = TxIntDelay[bd];
-			ixgb_validate_option(&adapter->tx_int_delay, &opt);
-		} else {
-			adapter->tx_int_delay = opt.def;
-		}
-	}
-
-	{ /* Transmit Interrupt Delay Enable */
-		static const struct ixgb_option opt = {
-			.type = enable_option,
-			.name = "Tx Interrupt Delay Enable",
-			.err  = "defaulting to Enabled",
-			.def  = OPTION_ENABLED
-		};
-
-		if (num_IntDelayEnable > bd) {
-			unsigned int ide = IntDelayEnable[bd];
-			ixgb_validate_option(&ide, &opt);
-			adapter->tx_int_delay_enable = ide;
-		} else {
-			adapter->tx_int_delay_enable = opt.def;
-		}
-	}
-}
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
index 8736ca4b2628..63d4e32df029 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
@@ -9,7 +9,6 @@
 #include <linux/pci.h>
 #include <linux/netdevice.h>
 #include <linux/cpumask.h>
-#include <linux/aer.h>
 #include <linux/if_vlan.h>
 #include <linux/jiffies.h>
 #include <linux/phy.h>
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
index 6cfc9dc16537..0bbad4a5cc2f 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
@@ -2665,6 +2665,14 @@ static int ixgbe_get_rss_hash_opts(struct ixgbe_adapter *adapter,
 	return 0;
 }
 
+static int ixgbe_rss_indir_tbl_max(struct ixgbe_adapter *adapter)
+{
+	if (adapter->hw.mac.type < ixgbe_mac_X550)
+		return 16;
+	else
+		return 64;
+}
+
 static int ixgbe_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
 			   u32 *rule_locs)
 {
@@ -2673,7 +2681,8 @@ static int ixgbe_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
 
 	switch (cmd->cmd) {
 	case ETHTOOL_GRXRINGS:
-		cmd->data = adapter->num_rx_queues;
+		cmd->data = min_t(int, adapter->num_rx_queues,
+				  ixgbe_rss_indir_tbl_max(adapter));
 		ret = 0;
 		break;
 	case ETHTOOL_GRXCLSRLCNT:
@@ -3075,14 +3084,6 @@ static int ixgbe_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
 	return ret;
 }
 
-static int ixgbe_rss_indir_tbl_max(struct ixgbe_adapter *adapter)
-{
-	if (adapter->hw.mac.type < ixgbe_mac_X550)
-		return 16;
-	else
-		return 64;
-}
-
 static u32 ixgbe_get_rxfh_key_size(struct net_device *netdev)
 {
 	return IXGBE_RSS_KEY_SIZE;
@@ -3131,8 +3132,8 @@ static int ixgbe_set_rxfh(struct net_device *netdev, const u32 *indir,
 	int i;
 	u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
 
-	if (hfunc)
-		return -EINVAL;
+	if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
+		return -EOPNOTSUPP;
 
 	/* Fill out the redirection table */
 	if (indir) {
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 773c35fecace..e961ef4bbf4d 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -36,6 +36,7 @@
 #include <net/tc_act/tc_mirred.h>
 #include <net/vxlan.h>
 #include <net/mpls.h>
+#include <net/netdev_queues.h>
 #include <net/xdp_sock_drv.h>
 #include <net/xfrm.h>
 
@@ -1119,6 +1120,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
 	unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
 	unsigned int budget = q_vector->tx.work_limit;
 	unsigned int i = tx_ring->next_to_clean;
+	struct netdev_queue *txq;
 
 	if (test_bit(__IXGBE_DOWN, &adapter->state))
 		return true;
@@ -1249,24 +1251,14 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
 	if (ring_is_xdp(tx_ring))
 		return !!budget;
 
-	netdev_tx_completed_queue(txring_txq(tx_ring),
-				  total_packets, total_bytes);
-
 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
-	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
-		     (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
-		/* Make sure that anybody stopping the queue after this
-		 * sees the new next_to_clean.
-		 */
-		smp_mb();
-		if (__netif_subqueue_stopped(tx_ring->netdev,
-					     tx_ring->queue_index)
-		    && !test_bit(__IXGBE_DOWN, &adapter->state)) {
-			netif_wake_subqueue(tx_ring->netdev,
-					    tx_ring->queue_index);
-			++tx_ring->tx_stats.restart_queue;
-		}
-	}
+	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->queue_index);
+	if (!__netif_txq_completed_wake(txq, total_packets, total_bytes,
+					ixgbe_desc_unused(tx_ring),
+					TX_WAKE_THRESHOLD,
+					netif_carrier_ok(tx_ring->netdev) &&
+					test_bit(__IXGBE_DOWN, &adapter->state)))
+		++tx_ring->tx_stats.restart_queue;
 
 	return !!budget;
 }
@@ -8270,22 +8262,10 @@ static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
 
 static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
 {
-	netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
-
-	/* Herbert's original patch had:
-	 *  smp_mb__after_netif_stop_queue();
-	 * but since that doesn't exist yet, just open code it.
-	 */
-	smp_mb();
-
-	/* We need to check again in a case another CPU has just
-	 * made room available.
-	 */
-	if (likely(ixgbe_desc_unused(tx_ring) < size))
+	if (!netif_subqueue_try_stop(tx_ring->netdev, tx_ring->queue_index,
+				     ixgbe_desc_unused(tx_ring), size))
 		return -EBUSY;
 
-	/* A reprieve! - use start_queue because it doesn't call schedule */
-	netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
 	++tx_ring->tx_stats.restart_queue;
 	return 0;
 }
@@ -8818,7 +8798,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
 
 			if (skb_cow_head(skb, 0))
 				goto out_drop;
-			vhdr = (struct vlan_ethhdr *)skb->data;
+			vhdr = skb_vlan_eth_hdr(skb);
 			vhdr->h_vlan_TCI = htons(tx_flags >>
 						 IXGBE_TX_FLAGS_VLAN_SHIFT);
 		} else {
diff --git a/drivers/net/ethernet/marvell/Kconfig b/drivers/net/ethernet/marvell/Kconfig
index f58a1c0144ba..884d64114bff 100644
--- a/drivers/net/ethernet/marvell/Kconfig
+++ b/drivers/net/ethernet/marvell/Kconfig
@@ -34,6 +34,7 @@ config MV643XX_ETH
 config MVMDIO
 	tristate "Marvell MDIO interface support"
 	depends on HAS_IOMEM
+	select MDIO_DEVRES
 	select PHYLIB
 	help
 	  This driver supports the MDIO interface found in the network
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
index 3ea00bc9b91c..adc953611913 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
@@ -6089,18 +6089,19 @@ static bool mvpp2_port_has_irqs(struct mvpp2 *priv,
 	return true;
 }
 
-static void mvpp2_port_copy_mac_addr(struct net_device *dev, struct mvpp2 *priv,
-				     struct fwnode_handle *fwnode,
-				     char **mac_from)
+static int mvpp2_port_copy_mac_addr(struct net_device *dev, struct mvpp2 *priv,
+				    struct fwnode_handle *fwnode,
+				    char **mac_from)
 {
 	struct mvpp2_port *port = netdev_priv(dev);
 	char hw_mac_addr[ETH_ALEN] = {0};
 	char fw_mac_addr[ETH_ALEN];
+	int ret;
 
 	if (!fwnode_get_mac_address(fwnode, fw_mac_addr)) {
 		*mac_from = "firmware node";
 		eth_hw_addr_set(dev, fw_mac_addr);
-		return;
+		return 0;
 	}
 
 	if (priv->hw_version == MVPP21) {
@@ -6108,19 +6109,24 @@ static void mvpp2_port_copy_mac_addr(struct net_device *dev, struct mvpp2 *priv,
 		if (is_valid_ether_addr(hw_mac_addr)) {
 			*mac_from = "hardware";
 			eth_hw_addr_set(dev, hw_mac_addr);
-			return;
+			return 0;
 		}
 	}
 
 	/* Only valid on OF enabled platforms */
-	if (!of_get_mac_address_nvmem(to_of_node(fwnode), fw_mac_addr)) {
+	ret = of_get_mac_address_nvmem(to_of_node(fwnode), fw_mac_addr);
+	if (ret == -EPROBE_DEFER)
+		return ret;
+	if (!ret) {
 		*mac_from = "nvmem cell";
 		eth_hw_addr_set(dev, fw_mac_addr);
-		return;
+		return 0;
 	}
 
 	*mac_from = "random";
 	eth_hw_addr_random(dev);
+
+	return 0;
 }
 
 static struct mvpp2_port *mvpp2_phylink_to_port(struct phylink_config *config)
@@ -6823,7 +6829,9 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	mutex_init(&port->gather_stats_lock);
 	INIT_DELAYED_WORK(&port->stats_work, mvpp2_gather_hw_statistics);
 
-	mvpp2_port_copy_mac_addr(dev, priv, port_fwnode, &mac_from);
+	err = mvpp2_port_copy_mac_addr(dev, priv, port_fwnode, &mac_from);
+	if (err < 0)
+		goto err_free_stats;
 
 	port->tx_ring_size = MVPP2_MAX_TXD_DFLT;
 	port->rx_ring_size = MVPP2_MAX_RXD_DFLT;
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c
index 6ad88d0fe43f..90c3a419932d 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c
@@ -13,6 +13,12 @@
 #include "octep_main.h"
 #include "octep_regs_cn9k_pf.h"
 
+#define CTRL_MBOX_MAX_PF	128
+#define CTRL_MBOX_SZ		((size_t)(0x400000 / CTRL_MBOX_MAX_PF))
+
+#define FW_HB_INTERVAL_IN_SECS		1
+#define FW_HB_MISS_COUNT		10
+
 /* Names of Hardware non-queue generic interrupts */
 static char *cn93_non_ioq_msix_names[] = {
 	"epf_ire_rint",
@@ -198,7 +204,9 @@ static void octep_init_config_cn93_pf(struct octep_device *oct)
 {
 	struct octep_config *conf = oct->conf;
 	struct pci_dev *pdev = oct->pdev;
+	u8 link = 0;
 	u64 val;
+	int pos;
 
 	/* Read ring configuration:
 	 * PF ring count, number of VFs and rings per VF supported
@@ -234,7 +242,20 @@ static void octep_init_config_cn93_pf(struct octep_device *oct)
 	conf->msix_cfg.ioq_msix = conf->pf_ring_cfg.active_io_rings;
 	conf->msix_cfg.non_ioq_msix_names = cn93_non_ioq_msix_names;
 
-	conf->ctrl_mbox_cfg.barmem_addr = (void __iomem *)oct->mmio[2].hw_addr + (0x400000ull * 7);
+	pos = pci_find_ext_capability(oct->pdev, PCI_EXT_CAP_ID_SRIOV);
+	if (pos) {
+		pci_read_config_byte(oct->pdev,
+				     pos + PCI_SRIOV_FUNC_LINK,
+				     &link);
+		link = PCI_DEVFN(PCI_SLOT(oct->pdev->devfn), link);
+	}
+	conf->ctrl_mbox_cfg.barmem_addr = (void __iomem *)oct->mmio[2].hw_addr +
+					   (0x400000ull * 7) +
+					   (link * CTRL_MBOX_SZ);
+
+	conf->hb_interval = FW_HB_INTERVAL_IN_SECS;
+	conf->max_hb_miss_cnt = FW_HB_MISS_COUNT;
+
 }
 
 /* Setup registers for a hardware Tx Queue  */
@@ -352,19 +373,30 @@ static void octep_setup_mbox_regs_cn93_pf(struct octep_device *oct, int q_no)
 	mbox->mbox_read_reg = oct->mmio[0].hw_addr + CN93_SDP_R_MBOX_VF_PF_DATA(q_no);
 }
 
-/* Mailbox Interrupt handler */
-static void cn93_handle_pf_mbox_intr(struct octep_device *oct)
+/* Process non-ioq interrupts required to keep pf interface running.
+ * OEI_RINT is needed for control mailbox
+ */
+static bool octep_poll_non_ioq_interrupts_cn93_pf(struct octep_device *oct)
 {
-	u64 mbox_int_val = 0ULL, val = 0ULL, qno = 0ULL;
+	bool handled = false;
+	u64 reg0;
 
-	mbox_int_val = readq(oct->mbox[0]->mbox_int_reg);
-	for (qno = 0; qno < OCTEP_MAX_VF; qno++) {
-		val = readq(oct->mbox[qno]->mbox_read_reg);
-		dev_dbg(&oct->pdev->dev,
-			"PF MBOX READ: val:%llx from VF:%llx\n", val, qno);
+	/* Check for OEI INTR */
+	reg0 = octep_read_csr64(oct, CN93_SDP_EPF_OEI_RINT);
+	if (reg0) {
+		dev_info(&oct->pdev->dev,
+			 "Received OEI_RINT intr: 0x%llx\n",
+			 reg0);
+		octep_write_csr64(oct, CN93_SDP_EPF_OEI_RINT, reg0);
+		if (reg0 & CN93_SDP_EPF_OEI_RINT_DATA_BIT_MBOX)
+			queue_work(octep_wq, &oct->ctrl_mbox_task);
+		else if (reg0 & CN93_SDP_EPF_OEI_RINT_DATA_BIT_HBEAT)
+			atomic_set(&oct->hb_miss_cnt, 0);
+
+		handled = true;
 	}
 
-	writeq(mbox_int_val, oct->mbox[0]->mbox_int_reg);
+	return handled;
 }
 
 /* Interrupts handler for all non-queue generic interrupts. */
@@ -434,24 +466,9 @@ static irqreturn_t octep_non_ioq_intr_handler_cn93_pf(void *dev)
 		goto irq_handled;
 	}
 
-	/* Check for MBOX INTR */
-	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_MBOX_RINT(0));
-	if (reg_val) {
-		dev_info(&pdev->dev,
-			 "Received MBOX_RINT intr: 0x%llx\n", reg_val);
-		cn93_handle_pf_mbox_intr(oct);
-		goto irq_handled;
-	}
-
-	/* Check for OEI INTR */
-	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_OEI_RINT);
-	if (reg_val) {
-		dev_info(&pdev->dev,
-			 "Received OEI_EINT intr: 0x%llx\n", reg_val);
-		octep_write_csr64(oct, CN93_SDP_EPF_OEI_RINT, reg_val);
-		queue_work(octep_wq, &oct->ctrl_mbox_task);
+	/* Check for MBOX INTR and OEI INTR */
+	if (octep_poll_non_ioq_interrupts_cn93_pf(oct))
 		goto irq_handled;
-	}
 
 	/* Check for DMA INTR */
 	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_DMA_RINT);
@@ -712,6 +729,7 @@ void octep_device_setup_cn93_pf(struct octep_device *oct)
 
 	oct->hw_ops.enable_interrupts = octep_enable_interrupts_cn93_pf;
 	oct->hw_ops.disable_interrupts = octep_disable_interrupts_cn93_pf;
+	oct->hw_ops.poll_non_ioq_interrupts = octep_poll_non_ioq_interrupts_cn93_pf;
 
 	oct->hw_ops.update_iq_read_idx = octep_update_iq_read_index_cn93_pf;
 
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_config.h b/drivers/net/ethernet/marvell/octeon_ep/octep_config.h
index f208f3f9a447..df7cd39d9fce 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_config.h
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_config.h
@@ -200,5 +200,11 @@ struct octep_config {
 
 	/* ctrl mbox config */
 	struct octep_ctrl_mbox_config ctrl_mbox_cfg;
+
+	/* Configured maximum heartbeat miss count */
+	u32 max_hb_miss_cnt;
+
+	/* Configured firmware heartbeat interval in secs */
+	u32 hb_interval;
 };
 #endif /* _OCTEP_CONFIG_H_ */
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.c b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.c
index 39322e4dd100..035ead7935c7 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.c
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.c
@@ -24,41 +24,49 @@
 /* Time in msecs to wait for message response */
 #define OCTEP_CTRL_MBOX_MSG_WAIT_MS			10
 
-#define OCTEP_CTRL_MBOX_INFO_MAGIC_NUM_OFFSET(m)	(m)
-#define OCTEP_CTRL_MBOX_INFO_BARMEM_SZ_OFFSET(m)	((m) + 8)
-#define OCTEP_CTRL_MBOX_INFO_HOST_STATUS_OFFSET(m)	((m) + 24)
-#define OCTEP_CTRL_MBOX_INFO_FW_STATUS_OFFSET(m)	((m) + 144)
-
-#define OCTEP_CTRL_MBOX_H2FQ_INFO_OFFSET(m)		((m) + OCTEP_CTRL_MBOX_INFO_SZ)
-#define OCTEP_CTRL_MBOX_H2FQ_PROD_OFFSET(m)		(OCTEP_CTRL_MBOX_H2FQ_INFO_OFFSET(m))
-#define OCTEP_CTRL_MBOX_H2FQ_CONS_OFFSET(m)		((OCTEP_CTRL_MBOX_H2FQ_INFO_OFFSET(m)) + 4)
-#define OCTEP_CTRL_MBOX_H2FQ_ELEM_SZ_OFFSET(m)		((OCTEP_CTRL_MBOX_H2FQ_INFO_OFFSET(m)) + 8)
-#define OCTEP_CTRL_MBOX_H2FQ_ELEM_CNT_OFFSET(m)		((OCTEP_CTRL_MBOX_H2FQ_INFO_OFFSET(m)) + 12)
-
-#define OCTEP_CTRL_MBOX_F2HQ_INFO_OFFSET(m)		((m) + \
-							 OCTEP_CTRL_MBOX_INFO_SZ + \
-							 OCTEP_CTRL_MBOX_H2FQ_INFO_SZ)
-#define OCTEP_CTRL_MBOX_F2HQ_PROD_OFFSET(m)		(OCTEP_CTRL_MBOX_F2HQ_INFO_OFFSET(m))
-#define OCTEP_CTRL_MBOX_F2HQ_CONS_OFFSET(m)		((OCTEP_CTRL_MBOX_F2HQ_INFO_OFFSET(m)) + 4)
-#define OCTEP_CTRL_MBOX_F2HQ_ELEM_SZ_OFFSET(m)		((OCTEP_CTRL_MBOX_F2HQ_INFO_OFFSET(m)) + 8)
-#define OCTEP_CTRL_MBOX_F2HQ_ELEM_CNT_OFFSET(m)		((OCTEP_CTRL_MBOX_F2HQ_INFO_OFFSET(m)) + 12)
-
-#define OCTEP_CTRL_MBOX_Q_OFFSET(m, i)			((m) + \
-							 (sizeof(struct octep_ctrl_mbox_msg) * (i)))
-
-static u32 octep_ctrl_mbox_circq_inc(u32 index, u32 mask)
+/* Size of mbox info in bytes */
+#define OCTEP_CTRL_MBOX_INFO_SZ				256
+/* Size of mbox host to fw queue info in bytes */
+#define OCTEP_CTRL_MBOX_H2FQ_INFO_SZ			16
+/* Size of mbox fw to host queue info in bytes */
+#define OCTEP_CTRL_MBOX_F2HQ_INFO_SZ			16
+
+#define OCTEP_CTRL_MBOX_TOTAL_INFO_SZ	(OCTEP_CTRL_MBOX_INFO_SZ + \
+					 OCTEP_CTRL_MBOX_H2FQ_INFO_SZ + \
+					 OCTEP_CTRL_MBOX_F2HQ_INFO_SZ)
+
+#define OCTEP_CTRL_MBOX_INFO_MAGIC_NUM(m)	(m)
+#define OCTEP_CTRL_MBOX_INFO_BARMEM_SZ(m)	((m) + 8)
+#define OCTEP_CTRL_MBOX_INFO_HOST_STATUS(m)	((m) + 24)
+#define OCTEP_CTRL_MBOX_INFO_FW_STATUS(m)	((m) + 144)
+
+#define OCTEP_CTRL_MBOX_H2FQ_INFO(m)	((m) + OCTEP_CTRL_MBOX_INFO_SZ)
+#define OCTEP_CTRL_MBOX_H2FQ_PROD(m)	(OCTEP_CTRL_MBOX_H2FQ_INFO(m))
+#define OCTEP_CTRL_MBOX_H2FQ_CONS(m)	((OCTEP_CTRL_MBOX_H2FQ_INFO(m)) + 4)
+#define OCTEP_CTRL_MBOX_H2FQ_SZ(m)	((OCTEP_CTRL_MBOX_H2FQ_INFO(m)) + 8)
+
+#define OCTEP_CTRL_MBOX_F2HQ_INFO(m)	((m) + \
+					 OCTEP_CTRL_MBOX_INFO_SZ + \
+					 OCTEP_CTRL_MBOX_H2FQ_INFO_SZ)
+#define OCTEP_CTRL_MBOX_F2HQ_PROD(m)	(OCTEP_CTRL_MBOX_F2HQ_INFO(m))
+#define OCTEP_CTRL_MBOX_F2HQ_CONS(m)	((OCTEP_CTRL_MBOX_F2HQ_INFO(m)) + 4)
+#define OCTEP_CTRL_MBOX_F2HQ_SZ(m)	((OCTEP_CTRL_MBOX_F2HQ_INFO(m)) + 8)
+
+static const u32 mbox_hdr_sz = sizeof(union octep_ctrl_mbox_msg_hdr);
+
+static u32 octep_ctrl_mbox_circq_inc(u32 index, u32 inc, u32 sz)
 {
-	return (index + 1) & mask;
+	return (index + inc) % sz;
 }
 
-static u32 octep_ctrl_mbox_circq_space(u32 pi, u32 ci, u32 mask)
+static u32 octep_ctrl_mbox_circq_space(u32 pi, u32 ci, u32 sz)
 {
-	return mask - ((pi - ci) & mask);
+	return sz - (abs(pi - ci) % sz);
 }
 
-static u32 octep_ctrl_mbox_circq_depth(u32 pi, u32 ci, u32 mask)
+static u32 octep_ctrl_mbox_circq_depth(u32 pi, u32 ci, u32 sz)
 {
-	return ((pi - ci) & mask);
+	return (abs(pi - ci) % sz);
 }
 
 int octep_ctrl_mbox_init(struct octep_ctrl_mbox *mbox)
@@ -73,153 +81,170 @@ int octep_ctrl_mbox_init(struct octep_ctrl_mbox *mbox)
 		return -EINVAL;
 	}
 
-	magic_num = readq(OCTEP_CTRL_MBOX_INFO_MAGIC_NUM_OFFSET(mbox->barmem));
+	magic_num = readq(OCTEP_CTRL_MBOX_INFO_MAGIC_NUM(mbox->barmem));
 	if (magic_num != OCTEP_CTRL_MBOX_MAGIC_NUMBER) {
 		pr_info("octep_ctrl_mbox : Invalid magic number %llx\n", magic_num);
 		return -EINVAL;
 	}
 
-	status = readq(OCTEP_CTRL_MBOX_INFO_FW_STATUS_OFFSET(mbox->barmem));
+	status = readq(OCTEP_CTRL_MBOX_INFO_FW_STATUS(mbox->barmem));
 	if (status != OCTEP_CTRL_MBOX_STATUS_READY) {
 		pr_info("octep_ctrl_mbox : Firmware is not ready.\n");
 		return -EINVAL;
 	}
 
-	mbox->barmem_sz = readl(OCTEP_CTRL_MBOX_INFO_BARMEM_SZ_OFFSET(mbox->barmem));
+	mbox->barmem_sz = readl(OCTEP_CTRL_MBOX_INFO_BARMEM_SZ(mbox->barmem));
 
-	writeq(OCTEP_CTRL_MBOX_STATUS_INIT, OCTEP_CTRL_MBOX_INFO_HOST_STATUS_OFFSET(mbox->barmem));
+	writeq(OCTEP_CTRL_MBOX_STATUS_INIT,
+	       OCTEP_CTRL_MBOX_INFO_HOST_STATUS(mbox->barmem));
 
-	mbox->h2fq.elem_cnt = readl(OCTEP_CTRL_MBOX_H2FQ_ELEM_CNT_OFFSET(mbox->barmem));
-	mbox->h2fq.elem_sz = readl(OCTEP_CTRL_MBOX_H2FQ_ELEM_SZ_OFFSET(mbox->barmem));
-	mbox->h2fq.mask = (mbox->h2fq.elem_cnt - 1);
-	mutex_init(&mbox->h2fq_lock);
+	mbox->h2fq.sz = readl(OCTEP_CTRL_MBOX_H2FQ_SZ(mbox->barmem));
+	mbox->h2fq.hw_prod = OCTEP_CTRL_MBOX_H2FQ_PROD(mbox->barmem);
+	mbox->h2fq.hw_cons = OCTEP_CTRL_MBOX_H2FQ_CONS(mbox->barmem);
+	mbox->h2fq.hw_q = mbox->barmem + OCTEP_CTRL_MBOX_TOTAL_INFO_SZ;
 
-	mbox->f2hq.elem_cnt = readl(OCTEP_CTRL_MBOX_F2HQ_ELEM_CNT_OFFSET(mbox->barmem));
-	mbox->f2hq.elem_sz = readl(OCTEP_CTRL_MBOX_F2HQ_ELEM_SZ_OFFSET(mbox->barmem));
-	mbox->f2hq.mask = (mbox->f2hq.elem_cnt - 1);
-	mutex_init(&mbox->f2hq_lock);
-
-	mbox->h2fq.hw_prod = OCTEP_CTRL_MBOX_H2FQ_PROD_OFFSET(mbox->barmem);
-	mbox->h2fq.hw_cons = OCTEP_CTRL_MBOX_H2FQ_CONS_OFFSET(mbox->barmem);
-	mbox->h2fq.hw_q = mbox->barmem +
-			  OCTEP_CTRL_MBOX_INFO_SZ +
-			  OCTEP_CTRL_MBOX_H2FQ_INFO_SZ +
-			  OCTEP_CTRL_MBOX_F2HQ_INFO_SZ;
-
-	mbox->f2hq.hw_prod = OCTEP_CTRL_MBOX_F2HQ_PROD_OFFSET(mbox->barmem);
-	mbox->f2hq.hw_cons = OCTEP_CTRL_MBOX_F2HQ_CONS_OFFSET(mbox->barmem);
-	mbox->f2hq.hw_q = mbox->h2fq.hw_q +
-			  ((mbox->h2fq.elem_sz + sizeof(union octep_ctrl_mbox_msg_hdr)) *
-			   mbox->h2fq.elem_cnt);
+	mbox->f2hq.sz = readl(OCTEP_CTRL_MBOX_F2HQ_SZ(mbox->barmem));
+	mbox->f2hq.hw_prod = OCTEP_CTRL_MBOX_F2HQ_PROD(mbox->barmem);
+	mbox->f2hq.hw_cons = OCTEP_CTRL_MBOX_F2HQ_CONS(mbox->barmem);
+	mbox->f2hq.hw_q = mbox->barmem +
+			  OCTEP_CTRL_MBOX_TOTAL_INFO_SZ +
+			  mbox->h2fq.sz;
 
 	/* ensure ready state is seen after everything is initialized */
 	wmb();
-	writeq(OCTEP_CTRL_MBOX_STATUS_READY, OCTEP_CTRL_MBOX_INFO_HOST_STATUS_OFFSET(mbox->barmem));
+	writeq(OCTEP_CTRL_MBOX_STATUS_READY,
+	       OCTEP_CTRL_MBOX_INFO_HOST_STATUS(mbox->barmem));
 
 	pr_info("Octep ctrl mbox : Init successful.\n");
 
 	return 0;
 }
 
+static void
+octep_write_mbox_data(struct octep_ctrl_mbox_q *q, u32 *pi, u32 ci, void *buf, u32 w_sz)
+{
+	u8 __iomem *qbuf;
+	u32 cp_sz;
+
+	/* Assumption: Caller has ensured enough write space */
+	qbuf = (q->hw_q + *pi);
+	if (*pi < ci) {
+		/* copy entire w_sz */
+		memcpy_toio(qbuf, buf, w_sz);
+		*pi = octep_ctrl_mbox_circq_inc(*pi, w_sz, q->sz);
+	} else {
+		/* copy up to end of queue */
+		cp_sz = min((q->sz - *pi), w_sz);
+		memcpy_toio(qbuf, buf, cp_sz);
+		w_sz -= cp_sz;
+		*pi = octep_ctrl_mbox_circq_inc(*pi, cp_sz, q->sz);
+		if (w_sz) {
+			/* roll over and copy remaining w_sz */
+			buf += cp_sz;
+			qbuf = (q->hw_q + *pi);
+			memcpy_toio(qbuf, buf, w_sz);
+			*pi = octep_ctrl_mbox_circq_inc(*pi, w_sz, q->sz);
+		}
+	}
+}
+
 int octep_ctrl_mbox_send(struct octep_ctrl_mbox *mbox, struct octep_ctrl_mbox_msg *msg)
 {
-	unsigned long timeout = msecs_to_jiffies(OCTEP_CTRL_MBOX_MSG_TIMEOUT_MS);
-	unsigned long period = msecs_to_jiffies(OCTEP_CTRL_MBOX_MSG_WAIT_MS);
+	struct octep_ctrl_mbox_msg_buf *sg;
 	struct octep_ctrl_mbox_q *q;
-	unsigned long expire;
-	u64 *mbuf, *word0;
-	u8 __iomem *qidx;
-	u16 pi, ci;
-	int i;
+	u32 pi, ci, buf_sz, w_sz;
+	int s;
 
 	if (!mbox || !msg)
 		return -EINVAL;
 
+	if (readq(OCTEP_CTRL_MBOX_INFO_FW_STATUS(mbox->barmem)) != OCTEP_CTRL_MBOX_STATUS_READY)
+		return -EIO;
+
+	mutex_lock(&mbox->h2fq_lock);
 	q = &mbox->h2fq;
 	pi = readl(q->hw_prod);
 	ci = readl(q->hw_cons);
 
-	if (!octep_ctrl_mbox_circq_space(pi, ci, q->mask))
-		return -ENOMEM;
-
-	qidx = OCTEP_CTRL_MBOX_Q_OFFSET(q->hw_q, pi);
-	mbuf = (u64 *)msg->msg;
-	word0 = &msg->hdr.word0;
-
-	mutex_lock(&mbox->h2fq_lock);
-	for (i = 1; i <= msg->hdr.sizew; i++)
-		writeq(*mbuf++, (qidx + (i * 8)));
-
-	writeq(*word0, qidx);
+	if (octep_ctrl_mbox_circq_space(pi, ci, q->sz) < (msg->hdr.s.sz + mbox_hdr_sz)) {
+		mutex_unlock(&mbox->h2fq_lock);
+		return -EAGAIN;
+	}
 
-	pi = octep_ctrl_mbox_circq_inc(pi, q->mask);
+	octep_write_mbox_data(q, &pi, ci, (void *)&msg->hdr, mbox_hdr_sz);
+	buf_sz = msg->hdr.s.sz;
+	for (s = 0; ((s < msg->sg_num) && (buf_sz > 0)); s++) {
+		sg = &msg->sg_list[s];
+		w_sz = (sg->sz <= buf_sz) ? sg->sz : buf_sz;
+		octep_write_mbox_data(q, &pi, ci, sg->msg, w_sz);
+		buf_sz -= w_sz;
+	}
 	writel(pi, q->hw_prod);
 	mutex_unlock(&mbox->h2fq_lock);
 
-	/* don't check for notification response */
-	if (msg->hdr.flags & OCTEP_CTRL_MBOX_MSG_HDR_FLAG_NOTIFY)
-		return 0;
-
-	expire = jiffies + timeout;
-	while (true) {
-		*word0 = readq(qidx);
-		if (msg->hdr.flags == OCTEP_CTRL_MBOX_MSG_HDR_FLAG_RESP)
-			break;
-		schedule_timeout_interruptible(period);
-		if (signal_pending(current) || time_after(jiffies, expire)) {
-			pr_info("octep_ctrl_mbox: Timed out\n");
-			return -EBUSY;
+	return 0;
+}
+
+static void
+octep_read_mbox_data(struct octep_ctrl_mbox_q *q, u32 pi, u32 *ci, void *buf, u32 r_sz)
+{
+	u8 __iomem *qbuf;
+	u32 cp_sz;
+
+	/* Assumption: Caller has ensured enough read space */
+	qbuf = (q->hw_q + *ci);
+	if (*ci < pi) {
+		/* copy entire r_sz */
+		memcpy_fromio(buf, qbuf, r_sz);
+		*ci = octep_ctrl_mbox_circq_inc(*ci, r_sz, q->sz);
+	} else {
+		/* copy up to end of queue */
+		cp_sz = min((q->sz - *ci), r_sz);
+		memcpy_fromio(buf, qbuf, cp_sz);
+		r_sz -= cp_sz;
+		*ci = octep_ctrl_mbox_circq_inc(*ci, cp_sz, q->sz);
+		if (r_sz) {
+			/* roll over and copy remaining r_sz */
+			buf += cp_sz;
+			qbuf = (q->hw_q + *ci);
+			memcpy_fromio(buf, qbuf, r_sz);
+			*ci = octep_ctrl_mbox_circq_inc(*ci, r_sz, q->sz);
 		}
 	}
-	mbuf = (u64 *)msg->msg;
-	for (i = 1; i <= msg->hdr.sizew; i++)
-		*mbuf++ = readq(qidx + (i * 8));
-
-	return 0;
 }
 
 int octep_ctrl_mbox_recv(struct octep_ctrl_mbox *mbox, struct octep_ctrl_mbox_msg *msg)
 {
+	struct octep_ctrl_mbox_msg_buf *sg;
+	u32 pi, ci, r_sz, buf_sz, q_depth;
 	struct octep_ctrl_mbox_q *q;
-	u32 count, pi, ci;
-	u8 __iomem *qidx;
-	u64 *mbuf;
-	int i;
+	int s;
 
-	if (!mbox || !msg)
-		return -EINVAL;
+	if (readq(OCTEP_CTRL_MBOX_INFO_FW_STATUS(mbox->barmem)) != OCTEP_CTRL_MBOX_STATUS_READY)
+		return -EIO;
 
+	mutex_lock(&mbox->f2hq_lock);
 	q = &mbox->f2hq;
 	pi = readl(q->hw_prod);
 	ci = readl(q->hw_cons);
-	count = octep_ctrl_mbox_circq_depth(pi, ci, q->mask);
-	if (!count)
-		return -EAGAIN;
-
-	qidx = OCTEP_CTRL_MBOX_Q_OFFSET(q->hw_q, ci);
-	mbuf = (u64 *)msg->msg;
 
-	mutex_lock(&mbox->f2hq_lock);
-
-	msg->hdr.word0 = readq(qidx);
-	for (i = 1; i <= msg->hdr.sizew; i++)
-		*mbuf++ = readq(qidx + (i * 8));
+	q_depth = octep_ctrl_mbox_circq_depth(pi, ci, q->sz);
+	if (q_depth < mbox_hdr_sz) {
+		mutex_unlock(&mbox->f2hq_lock);
+		return -EAGAIN;
+	}
 
-	ci = octep_ctrl_mbox_circq_inc(ci, q->mask);
+	octep_read_mbox_data(q, pi, &ci, (void *)&msg->hdr, mbox_hdr_sz);
+	buf_sz = msg->hdr.s.sz;
+	for (s = 0; ((s < msg->sg_num) && (buf_sz > 0)); s++) {
+		sg = &msg->sg_list[s];
+		r_sz = (sg->sz <= buf_sz) ? sg->sz : buf_sz;
+		octep_read_mbox_data(q, pi, &ci, sg->msg, r_sz);
+		buf_sz -= r_sz;
+	}
 	writel(ci, q->hw_cons);
-
 	mutex_unlock(&mbox->f2hq_lock);
 
-	if (msg->hdr.flags != OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ || !mbox->process_req)
-		return 0;
-
-	mbox->process_req(mbox->user_ctx, msg);
-	mbuf = (u64 *)msg->msg;
-	for (i = 1; i <= msg->hdr.sizew; i++)
-		writeq(*mbuf++, (qidx + (i * 8)));
-
-	writeq(msg->hdr.word0, qidx);
-
 	return 0;
 }
 
@@ -227,18 +252,17 @@ int octep_ctrl_mbox_uninit(struct octep_ctrl_mbox *mbox)
 {
 	if (!mbox)
 		return -EINVAL;
+	if (!mbox->barmem)
+		return -EINVAL;
 
-	writeq(OCTEP_CTRL_MBOX_STATUS_UNINIT,
-	       OCTEP_CTRL_MBOX_INFO_HOST_STATUS_OFFSET(mbox->barmem));
+	writeq(OCTEP_CTRL_MBOX_STATUS_INVALID,
+	       OCTEP_CTRL_MBOX_INFO_HOST_STATUS(mbox->barmem));
 	/* ensure uninit state is written before uninitialization */
 	wmb();
 
 	mutex_destroy(&mbox->h2fq_lock);
 	mutex_destroy(&mbox->f2hq_lock);
 
-	writeq(OCTEP_CTRL_MBOX_STATUS_INVALID,
-	       OCTEP_CTRL_MBOX_INFO_HOST_STATUS_OFFSET(mbox->barmem));
-
 	pr_info("Octep ctrl mbox : Uninit successful.\n");
 
 	return 0;
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.h b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.h
index 2dc5753cfec6..9c4ff0fba6a0 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.h
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.h
@@ -27,50 +27,39 @@
  * |-------------------------------------------|
  * |producer index (4 bytes)                   |
  * |consumer index (4 bytes)                   |
- * |element size (4 bytes)                     |
- * |element count (4 bytes)                    |
+ * |max element size (4 bytes)                 |
+ * |reserved (4 bytes)                         |
  * |===========================================|
  * |Fw to Host Queue info (16 bytes)           |
  * |-------------------------------------------|
  * |producer index (4 bytes)                   |
  * |consumer index (4 bytes)                   |
- * |element size (4 bytes)                     |
- * |element count (4 bytes)                    |
+ * |max element size (4 bytes)                 |
+ * |reserved (4 bytes)                         |
  * |===========================================|
- * |Host to Fw Queue                           |
+ * |Host to Fw Queue ((total size-288/2) bytes)|
  * |-------------------------------------------|
- * |((elem_sz + hdr(8 bytes)) * elem_cnt) bytes|
+ * |                                           |
  * |===========================================|
  * |===========================================|
- * |Fw to Host Queue                           |
+ * |Fw to Host Queue ((total size-288/2) bytes)|
  * |-------------------------------------------|
- * |((elem_sz + hdr(8 bytes)) * elem_cnt) bytes|
+ * |                                           |
  * |===========================================|
  */
 
 #define OCTEP_CTRL_MBOX_MAGIC_NUMBER			0xdeaddeadbeefbeefull
 
-/* Size of mbox info in bytes */
-#define OCTEP_CTRL_MBOX_INFO_SZ				256
-/* Size of mbox host to target queue info in bytes */
-#define OCTEP_CTRL_MBOX_H2FQ_INFO_SZ			16
-/* Size of mbox target to host queue info in bytes */
-#define OCTEP_CTRL_MBOX_F2HQ_INFO_SZ			16
-/* Size of mbox queue in bytes */
-#define OCTEP_CTRL_MBOX_Q_SZ(sz, cnt)			(((sz) + 8) * (cnt))
-/* Size of mbox in bytes */
-#define OCTEP_CTRL_MBOX_SZ(hsz, hcnt, fsz, fcnt)	(OCTEP_CTRL_MBOX_INFO_SZ + \
-							 OCTEP_CTRL_MBOX_H2FQ_INFO_SZ + \
-							 OCTEP_CTRL_MBOX_F2HQ_INFO_SZ + \
-							 OCTEP_CTRL_MBOX_Q_SZ(hsz, hcnt) + \
-							 OCTEP_CTRL_MBOX_Q_SZ(fsz, fcnt))
-
 /* Valid request message */
 #define OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ		BIT(0)
 /* Valid response message */
 #define OCTEP_CTRL_MBOX_MSG_HDR_FLAG_RESP		BIT(1)
 /* Valid notification, no response required */
 #define OCTEP_CTRL_MBOX_MSG_HDR_FLAG_NOTIFY		BIT(2)
+/* Valid custom message */
+#define OCTEP_CTRL_MBOX_MSG_HDR_FLAG_CUSTOM		BIT(3)
+
+#define OCTEP_CTRL_MBOX_MSG_DESC_MAX			4
 
 enum octep_ctrl_mbox_status {
 	OCTEP_CTRL_MBOX_STATUS_INVALID = 0,
@@ -81,31 +70,48 @@ enum octep_ctrl_mbox_status {
 
 /* mbox message */
 union octep_ctrl_mbox_msg_hdr {
-	u64 word0;
+	u64 words[2];
 	struct {
+		/* must be 0 */
+		u16 reserved1:15;
+		/* vf_idx is valid if 1 */
+		u16 is_vf:1;
+		/* sender vf index 0-(n-1), 0 if (is_vf==0) */
+		u16 vf_idx;
+		/* total size of message excluding header */
+		u32 sz;
 		/* OCTEP_CTRL_MBOX_MSG_HDR_FLAG_* */
 		u32 flags;
-		/* size of message in words excluding header */
-		u32 sizew;
-	};
+		/* identifier to match responses */
+		u16 msg_id;
+		u16 reserved2;
+	} s;
+};
+
+/* mbox message buffer */
+struct octep_ctrl_mbox_msg_buf {
+	u32 reserved1;
+	u16 reserved2;
+	/* size of buffer */
+	u16 sz;
+	/* pointer to message buffer */
+	void *msg;
 };
 
 /* mbox message */
 struct octep_ctrl_mbox_msg {
 	/* mbox transaction header */
 	union octep_ctrl_mbox_msg_hdr hdr;
-	/* pointer to message buffer */
-	void *msg;
+	/* number of sg buffer's */
+	int sg_num;
+	/* message buffer's */
+	struct octep_ctrl_mbox_msg_buf sg_list[OCTEP_CTRL_MBOX_MSG_DESC_MAX];
 };
 
 /* Mbox queue */
 struct octep_ctrl_mbox_q {
-	/* q element size, should be aligned to unsigned long */
-	u16 elem_sz;
-	/* q element count, should be power of 2 */
-	u16 elem_cnt;
-	/* q mask */
-	u16 mask;
+	/* size of queue buffer */
+	u32 sz;
 	/* producer address in bar mem */
 	u8 __iomem *hw_prod;
 	/* consumer address in bar mem */
@@ -115,16 +121,10 @@ struct octep_ctrl_mbox_q {
 };
 
 struct octep_ctrl_mbox {
-	/* host driver version */
-	u64 version;
 	/* size of bar memory */
 	u32 barmem_sz;
 	/* pointer to BAR memory */
 	u8 __iomem *barmem;
-	/* user context for callback, can be null */
-	void *user_ctx;
-	/* callback handler for processing request, called from octep_ctrl_mbox_recv */
-	int (*process_req)(void *user_ctx, struct octep_ctrl_mbox_msg *msg);
 	/* host-to-fw queue */
 	struct octep_ctrl_mbox_q h2fq;
 	/* fw-to-host queue */
@@ -146,6 +146,8 @@ int octep_ctrl_mbox_init(struct octep_ctrl_mbox *mbox);
 /* Send mbox message.
  *
  * @param mbox: non-null pointer to struct octep_ctrl_mbox.
+ * @param msg:  non-null pointer to struct octep_ctrl_mbox_msg.
+ *              Caller should fill msg.sz and msg.desc.sz for each message.
  *
  * return value: 0 on success, -errno on failure.
  */
@@ -154,6 +156,8 @@ int octep_ctrl_mbox_send(struct octep_ctrl_mbox *mbox, struct octep_ctrl_mbox_ms
 /* Retrieve mbox message.
  *
  * @param mbox: non-null pointer to struct octep_ctrl_mbox.
+ * @param msg:  non-null pointer to struct octep_ctrl_mbox_msg.
+ *              Caller should fill msg.sz and msg.desc.sz for each message.
  *
  * return value: 0 on success, -errno on failure.
  */
@@ -161,7 +165,7 @@ int octep_ctrl_mbox_recv(struct octep_ctrl_mbox *mbox, struct octep_ctrl_mbox_ms
 
 /* Uninitialize control mbox.
  *
- * @param ep: non-null pointer to struct octep_ctrl_mbox.
+ * @param mbox: non-null pointer to struct octep_ctrl_mbox.
  *
  * return value: 0 on success, -errno on failure.
  */
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.c b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.c
index 7c00c896ab98..1cc6af2feb38 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.c
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.c
@@ -8,187 +8,328 @@
 #include <linux/types.h>
 #include <linux/etherdevice.h>
 #include <linux/pci.h>
+#include <linux/wait.h>
 
 #include "octep_config.h"
 #include "octep_main.h"
 #include "octep_ctrl_net.h"
 
-int octep_get_link_status(struct octep_device *oct)
+static const u32 req_hdr_sz = sizeof(union octep_ctrl_net_req_hdr);
+static const u32 mtu_sz = sizeof(struct octep_ctrl_net_h2f_req_cmd_mtu);
+static const u32 mac_sz = sizeof(struct octep_ctrl_net_h2f_req_cmd_mac);
+static const u32 state_sz = sizeof(struct octep_ctrl_net_h2f_req_cmd_state);
+static const u32 link_info_sz = sizeof(struct octep_ctrl_net_link_info);
+static atomic_t ctrl_net_msg_id;
+
+static void init_send_req(struct octep_ctrl_mbox_msg *msg, void *buf,
+			  u16 sz, int vfid)
 {
-	struct octep_ctrl_net_h2f_req req = {};
-	struct octep_ctrl_net_h2f_resp *resp;
-	struct octep_ctrl_mbox_msg msg = {};
-	int err;
+	msg->hdr.s.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ;
+	msg->hdr.s.msg_id = atomic_inc_return(&ctrl_net_msg_id) &
+			    GENMASK(sizeof(msg->hdr.s.msg_id) * BITS_PER_BYTE, 0);
+	msg->hdr.s.sz = req_hdr_sz + sz;
+	msg->sg_num = 1;
+	msg->sg_list[0].msg = buf;
+	msg->sg_list[0].sz = msg->hdr.s.sz;
+	if (vfid != OCTEP_CTRL_NET_INVALID_VFID) {
+		msg->hdr.s.is_vf = 1;
+		msg->hdr.s.vf_idx = vfid;
+	}
+}
+
+static int octep_send_mbox_req(struct octep_device *oct,
+			       struct octep_ctrl_net_wait_data *d,
+			       bool wait_for_response)
+{
+	int err, ret;
+
+	err = octep_ctrl_mbox_send(&oct->ctrl_mbox, &d->msg);
+	if (err < 0)
+		return err;
+
+	if (!wait_for_response)
+		return 0;
+
+	d->done = 0;
+	INIT_LIST_HEAD(&d->list);
+	list_add_tail(&d->list, &oct->ctrl_req_wait_list);
+	ret = wait_event_interruptible_timeout(oct->ctrl_req_wait_q,
+					       (d->done != 0),
+					       jiffies + msecs_to_jiffies(500));
+	list_del(&d->list);
+	if (ret == 0 || ret == 1)
+		return -EAGAIN;
+
+	/**
+	 * (ret == 0)  cond = false && timeout, return 0
+	 * (ret < 0) interrupted by signal, return 0
+	 * (ret == 1) cond = true && timeout, return 1
+	 * (ret >= 1) cond = true && !timeout, return 1
+	 */
+
+	if (d->data.resp.hdr.s.reply != OCTEP_CTRL_NET_REPLY_OK)
+		return -EAGAIN;
+
+	return 0;
+}
 
-	req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_STATUS;
-	req.link.cmd = OCTEP_CTRL_NET_CMD_GET;
+int octep_ctrl_net_init(struct octep_device *oct)
+{
+	struct octep_ctrl_mbox *ctrl_mbox;
+	struct pci_dev *pdev = oct->pdev;
+	int ret;
+
+	init_waitqueue_head(&oct->ctrl_req_wait_q);
+	INIT_LIST_HEAD(&oct->ctrl_req_wait_list);
+
+	/* Initialize control mbox */
+	ctrl_mbox = &oct->ctrl_mbox;
+	ctrl_mbox->barmem = CFG_GET_CTRL_MBOX_MEM_ADDR(oct->conf);
+	ret = octep_ctrl_mbox_init(ctrl_mbox);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to initialize control mbox\n");
+		return ret;
+	}
+	oct->ctrl_mbox_ifstats_offset = ctrl_mbox->barmem_sz;
+
+	return 0;
+}
 
-	msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ;
-	msg.hdr.sizew = OCTEP_CTRL_NET_H2F_STATE_REQ_SZW;
-	msg.msg = &req;
-	err = octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg);
-	if (err)
+int octep_ctrl_net_get_link_status(struct octep_device *oct, int vfid)
+{
+	struct octep_ctrl_net_wait_data d = {0};
+	struct octep_ctrl_net_h2f_req *req = &d.data.req;
+	int err;
+
+	init_send_req(&d.msg, (void *)req, state_sz, vfid);
+	req->hdr.s.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_STATUS;
+	req->link.cmd = OCTEP_CTRL_NET_CMD_GET;
+	err = octep_send_mbox_req(oct, &d, true);
+	if (err < 0)
 		return err;
 
-	resp = (struct octep_ctrl_net_h2f_resp *)&req;
-	return resp->link.state;
+	return d.data.resp.link.state;
 }
 
-void octep_set_link_status(struct octep_device *oct, bool up)
+int octep_ctrl_net_set_link_status(struct octep_device *oct, int vfid, bool up,
+				   bool wait_for_response)
 {
-	struct octep_ctrl_net_h2f_req req = {};
-	struct octep_ctrl_mbox_msg msg = {};
+	struct octep_ctrl_net_wait_data d = {0};
+	struct octep_ctrl_net_h2f_req *req = &d.data.req;
 
-	req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_STATUS;
-	req.link.cmd = OCTEP_CTRL_NET_CMD_SET;
-	req.link.state = (up) ? OCTEP_CTRL_NET_STATE_UP : OCTEP_CTRL_NET_STATE_DOWN;
+	init_send_req(&d.msg, req, state_sz, vfid);
+	req->hdr.s.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_STATUS;
+	req->link.cmd = OCTEP_CTRL_NET_CMD_SET;
+	req->link.state = (up) ? OCTEP_CTRL_NET_STATE_UP :
+				OCTEP_CTRL_NET_STATE_DOWN;
 
-	msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ;
-	msg.hdr.sizew = OCTEP_CTRL_NET_H2F_STATE_REQ_SZW;
-	msg.msg = &req;
-	octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg);
+	return octep_send_mbox_req(oct, &d, wait_for_response);
 }
 
-void octep_set_rx_state(struct octep_device *oct, bool up)
+int octep_ctrl_net_set_rx_state(struct octep_device *oct, int vfid, bool up,
+				bool wait_for_response)
 {
-	struct octep_ctrl_net_h2f_req req = {};
-	struct octep_ctrl_mbox_msg msg = {};
+	struct octep_ctrl_net_wait_data d = {0};
+	struct octep_ctrl_net_h2f_req *req = &d.data.req;
 
-	req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_RX_STATE;
-	req.link.cmd = OCTEP_CTRL_NET_CMD_SET;
-	req.link.state = (up) ? OCTEP_CTRL_NET_STATE_UP : OCTEP_CTRL_NET_STATE_DOWN;
+	init_send_req(&d.msg, req, state_sz, vfid);
+	req->hdr.s.cmd = OCTEP_CTRL_NET_H2F_CMD_RX_STATE;
+	req->link.cmd = OCTEP_CTRL_NET_CMD_SET;
+	req->link.state = (up) ? OCTEP_CTRL_NET_STATE_UP :
+				OCTEP_CTRL_NET_STATE_DOWN;
 
-	msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ;
-	msg.hdr.sizew = OCTEP_CTRL_NET_H2F_STATE_REQ_SZW;
-	msg.msg = &req;
-	octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg);
+	return octep_send_mbox_req(oct, &d, wait_for_response);
 }
 
-int octep_get_mac_addr(struct octep_device *oct, u8 *addr)
+int octep_ctrl_net_get_mac_addr(struct octep_device *oct, int vfid, u8 *addr)
 {
-	struct octep_ctrl_net_h2f_req req = {};
-	struct octep_ctrl_net_h2f_resp *resp;
-	struct octep_ctrl_mbox_msg msg = {};
+	struct octep_ctrl_net_wait_data d = {0};
+	struct octep_ctrl_net_h2f_req *req = &d.data.req;
 	int err;
 
-	req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_MAC;
-	req.link.cmd = OCTEP_CTRL_NET_CMD_GET;
-
-	msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ;
-	msg.hdr.sizew = OCTEP_CTRL_NET_H2F_MAC_REQ_SZW;
-	msg.msg = &req;
-	err = octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg);
-	if (err)
+	init_send_req(&d.msg, req, mac_sz, vfid);
+	req->hdr.s.cmd = OCTEP_CTRL_NET_H2F_CMD_MAC;
+	req->link.cmd = OCTEP_CTRL_NET_CMD_GET;
+	err = octep_send_mbox_req(oct, &d, true);
+	if (err < 0)
 		return err;
 
-	resp = (struct octep_ctrl_net_h2f_resp *)&req;
-	memcpy(addr, resp->mac.addr, ETH_ALEN);
+	memcpy(addr, d.data.resp.mac.addr, ETH_ALEN);
 
-	return err;
+	return 0;
 }
 
-int octep_set_mac_addr(struct octep_device *oct, u8 *addr)
+int octep_ctrl_net_set_mac_addr(struct octep_device *oct, int vfid, u8 *addr,
+				bool wait_for_response)
 {
-	struct octep_ctrl_net_h2f_req req = {};
-	struct octep_ctrl_mbox_msg msg = {};
-
-	req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_MAC;
-	req.mac.cmd = OCTEP_CTRL_NET_CMD_SET;
-	memcpy(&req.mac.addr, addr, ETH_ALEN);
+	struct octep_ctrl_net_wait_data d = {0};
+	struct octep_ctrl_net_h2f_req *req = &d.data.req;
 
-	msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ;
-	msg.hdr.sizew = OCTEP_CTRL_NET_H2F_MAC_REQ_SZW;
-	msg.msg = &req;
+	init_send_req(&d.msg, req, mac_sz, vfid);
+	req->hdr.s.cmd = OCTEP_CTRL_NET_H2F_CMD_MAC;
+	req->mac.cmd = OCTEP_CTRL_NET_CMD_SET;
+	memcpy(&req->mac.addr, addr, ETH_ALEN);
 
-	return octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg);
+	return octep_send_mbox_req(oct, &d, wait_for_response);
 }
 
-int octep_set_mtu(struct octep_device *oct, int mtu)
+int octep_ctrl_net_set_mtu(struct octep_device *oct, int vfid, int mtu,
+			   bool wait_for_response)
 {
-	struct octep_ctrl_net_h2f_req req = {};
-	struct octep_ctrl_mbox_msg msg = {};
+	struct octep_ctrl_net_wait_data d = {0};
+	struct octep_ctrl_net_h2f_req *req = &d.data.req;
 
-	req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_MTU;
-	req.mtu.cmd = OCTEP_CTRL_NET_CMD_SET;
-	req.mtu.val = mtu;
+	init_send_req(&d.msg, req, mtu_sz, vfid);
+	req->hdr.s.cmd = OCTEP_CTRL_NET_H2F_CMD_MTU;
+	req->mtu.cmd = OCTEP_CTRL_NET_CMD_SET;
+	req->mtu.val = mtu;
 
-	msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ;
-	msg.hdr.sizew = OCTEP_CTRL_NET_H2F_MTU_REQ_SZW;
-	msg.msg = &req;
-
-	return octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg);
+	return octep_send_mbox_req(oct, &d, wait_for_response);
 }
 
-int octep_get_if_stats(struct octep_device *oct)
+int octep_ctrl_net_get_if_stats(struct octep_device *oct, int vfid,
+				struct octep_iface_rx_stats *rx_stats,
+				struct octep_iface_tx_stats *tx_stats)
 {
-	void __iomem *iface_rx_stats;
-	void __iomem *iface_tx_stats;
-	struct octep_ctrl_net_h2f_req req = {};
-	struct octep_ctrl_mbox_msg msg = {};
+	struct octep_ctrl_net_wait_data d = {0};
+	struct octep_ctrl_net_h2f_req *req = &d.data.req;
+	struct octep_ctrl_net_h2f_resp *resp;
 	int err;
 
-	req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_GET_IF_STATS;
-	req.mac.cmd = OCTEP_CTRL_NET_CMD_GET;
-	req.get_stats.offset = oct->ctrl_mbox_ifstats_offset;
-
-	msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ;
-	msg.hdr.sizew = OCTEP_CTRL_NET_H2F_GET_STATS_REQ_SZW;
-	msg.msg = &req;
-	err = octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg);
-	if (err)
+	init_send_req(&d.msg, req, 0, vfid);
+	req->hdr.s.cmd = OCTEP_CTRL_NET_H2F_CMD_GET_IF_STATS;
+	err = octep_send_mbox_req(oct, &d, true);
+	if (err < 0)
 		return err;
 
-	iface_rx_stats = oct->ctrl_mbox.barmem + oct->ctrl_mbox_ifstats_offset;
-	iface_tx_stats = oct->ctrl_mbox.barmem + oct->ctrl_mbox_ifstats_offset +
-			 sizeof(struct octep_iface_rx_stats);
-	memcpy_fromio(&oct->iface_rx_stats, iface_rx_stats, sizeof(struct octep_iface_rx_stats));
-	memcpy_fromio(&oct->iface_tx_stats, iface_tx_stats, sizeof(struct octep_iface_tx_stats));
-
-	return err;
+	resp = &d.data.resp;
+	memcpy(rx_stats, &resp->if_stats.rx_stats, sizeof(struct octep_iface_rx_stats));
+	memcpy(tx_stats, &resp->if_stats.tx_stats, sizeof(struct octep_iface_tx_stats));
+	return 0;
 }
 
-int octep_get_link_info(struct octep_device *oct)
+int octep_ctrl_net_get_link_info(struct octep_device *oct, int vfid,
+				 struct octep_iface_link_info *link_info)
 {
-	struct octep_ctrl_net_h2f_req req = {};
+	struct octep_ctrl_net_wait_data d = {0};
+	struct octep_ctrl_net_h2f_req *req = &d.data.req;
 	struct octep_ctrl_net_h2f_resp *resp;
-	struct octep_ctrl_mbox_msg msg = {};
 	int err;
 
-	req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_INFO;
-	req.mac.cmd = OCTEP_CTRL_NET_CMD_GET;
-
-	msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ;
-	msg.hdr.sizew = OCTEP_CTRL_NET_H2F_LINK_INFO_REQ_SZW;
-	msg.msg = &req;
-	err = octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg);
-	if (err)
+	init_send_req(&d.msg, req, link_info_sz, vfid);
+	req->hdr.s.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_INFO;
+	req->link_info.cmd = OCTEP_CTRL_NET_CMD_GET;
+	err = octep_send_mbox_req(oct, &d, true);
+	if (err < 0)
 		return err;
 
-	resp = (struct octep_ctrl_net_h2f_resp *)&req;
-	oct->link_info.supported_modes = resp->link_info.supported_modes;
-	oct->link_info.advertised_modes = resp->link_info.advertised_modes;
-	oct->link_info.autoneg = resp->link_info.autoneg;
-	oct->link_info.pause = resp->link_info.pause;
-	oct->link_info.speed = resp->link_info.speed;
+	resp = &d.data.resp;
+	link_info->supported_modes = resp->link_info.supported_modes;
+	link_info->advertised_modes = resp->link_info.advertised_modes;
+	link_info->autoneg = resp->link_info.autoneg;
+	link_info->pause = resp->link_info.pause;
+	link_info->speed = resp->link_info.speed;
+
+	return 0;
+}
+
+int octep_ctrl_net_set_link_info(struct octep_device *oct, int vfid,
+				 struct octep_iface_link_info *link_info,
+				 bool wait_for_response)
+{
+	struct octep_ctrl_net_wait_data d = {0};
+	struct octep_ctrl_net_h2f_req *req = &d.data.req;
+
+	init_send_req(&d.msg, req, link_info_sz, vfid);
+	req->hdr.s.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_INFO;
+	req->link_info.cmd = OCTEP_CTRL_NET_CMD_SET;
+	req->link_info.info.advertised_modes = link_info->advertised_modes;
+	req->link_info.info.autoneg = link_info->autoneg;
+	req->link_info.info.pause = link_info->pause;
+	req->link_info.info.speed = link_info->speed;
+
+	return octep_send_mbox_req(oct, &d, wait_for_response);
+}
+
+static void process_mbox_resp(struct octep_device *oct,
+			      struct octep_ctrl_mbox_msg *msg)
+{
+	struct octep_ctrl_net_wait_data *pos, *n;
+
+	list_for_each_entry_safe(pos, n, &oct->ctrl_req_wait_list, list) {
+		if (pos->msg.hdr.s.msg_id == msg->hdr.s.msg_id) {
+			memcpy(&pos->data.resp,
+			       msg->sg_list[0].msg,
+			       msg->hdr.s.sz);
+			pos->done = 1;
+			wake_up_interruptible_all(&oct->ctrl_req_wait_q);
+			break;
+		}
+	}
+}
 
-	return err;
+static int process_mbox_notify(struct octep_device *oct,
+			       struct octep_ctrl_mbox_msg *msg)
+{
+	struct net_device *netdev = oct->netdev;
+	struct octep_ctrl_net_f2h_req *req;
+
+	req = (struct octep_ctrl_net_f2h_req *)msg->sg_list[0].msg;
+	switch (req->hdr.s.cmd) {
+	case OCTEP_CTRL_NET_F2H_CMD_LINK_STATUS:
+		if (netif_running(netdev)) {
+			if (req->link.state) {
+				dev_info(&oct->pdev->dev, "netif_carrier_on\n");
+				netif_carrier_on(netdev);
+			} else {
+				dev_info(&oct->pdev->dev, "netif_carrier_off\n");
+				netif_carrier_off(netdev);
+			}
+		}
+		break;
+	default:
+		pr_info("Unknown mbox req : %u\n", req->hdr.s.cmd);
+		break;
+	}
+
+	return 0;
 }
 
-int octep_set_link_info(struct octep_device *oct, struct octep_iface_link_info *link_info)
+void octep_ctrl_net_recv_fw_messages(struct octep_device *oct)
 {
-	struct octep_ctrl_net_h2f_req req = {};
-	struct octep_ctrl_mbox_msg msg = {};
+	static u16 msg_sz = sizeof(union octep_ctrl_net_max_data);
+	union octep_ctrl_net_max_data data = {0};
+	struct octep_ctrl_mbox_msg msg = {0};
+	int ret;
+
+	msg.hdr.s.sz = msg_sz;
+	msg.sg_num = 1;
+	msg.sg_list[0].sz = msg_sz;
+	msg.sg_list[0].msg = &data;
+	while (true) {
+		/* mbox will overwrite msg.hdr.s.sz so initialize it */
+		msg.hdr.s.sz = msg_sz;
+		ret = octep_ctrl_mbox_recv(&oct->ctrl_mbox, (struct octep_ctrl_mbox_msg *)&msg);
+		if (ret < 0)
+			break;
+
+		if (msg.hdr.s.flags & OCTEP_CTRL_MBOX_MSG_HDR_FLAG_RESP)
+			process_mbox_resp(oct, &msg);
+		else if (msg.hdr.s.flags & OCTEP_CTRL_MBOX_MSG_HDR_FLAG_NOTIFY)
+			process_mbox_notify(oct, &msg);
+	}
+}
+
+int octep_ctrl_net_uninit(struct octep_device *oct)
+{
+	struct octep_ctrl_net_wait_data *pos, *n;
+
+	list_for_each_entry_safe(pos, n, &oct->ctrl_req_wait_list, list)
+		pos->done = 1;
 
-	req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_INFO;
-	req.link_info.cmd = OCTEP_CTRL_NET_CMD_SET;
-	req.link_info.info.advertised_modes = link_info->advertised_modes;
-	req.link_info.info.autoneg = link_info->autoneg;
-	req.link_info.info.pause = link_info->pause;
-	req.link_info.info.speed = link_info->speed;
+	wake_up_interruptible_all(&oct->ctrl_req_wait_q);
 
-	msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ;
-	msg.hdr.sizew = OCTEP_CTRL_NET_H2F_LINK_INFO_REQ_SZW;
-	msg.msg = &req;
+	octep_ctrl_mbox_uninit(&oct->ctrl_mbox);
 
-	return octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg);
+	return 0;
 }
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.h b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.h
index f23b58381322..37880dd79116 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.h
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.h
@@ -7,6 +7,8 @@
 #ifndef __OCTEP_CTRL_NET_H__
 #define __OCTEP_CTRL_NET_H__
 
+#define OCTEP_CTRL_NET_INVALID_VFID	(-1)
+
 /* Supported commands */
 enum octep_ctrl_net_cmd {
 	OCTEP_CTRL_NET_CMD_GET = 0,
@@ -45,15 +47,18 @@ enum octep_ctrl_net_f2h_cmd {
 	OCTEP_CTRL_NET_F2H_CMD_LINK_STATUS,
 };
 
-struct octep_ctrl_net_req_hdr {
-	/* sender id */
-	u16 sender;
-	/* receiver id */
-	u16 receiver;
-	/* octep_ctrl_net_h2t_cmd */
-	u16 cmd;
-	/* reserved */
-	u16 rsvd0;
+union octep_ctrl_net_req_hdr {
+	u64 words[1];
+	struct {
+		/* sender id */
+		u16 sender;
+		/* receiver id */
+		u16 receiver;
+		/* octep_ctrl_net_h2t_cmd */
+		u16 cmd;
+		/* reserved */
+		u16 rsvd0;
+	} s;
 };
 
 /* get/set mtu request */
@@ -72,12 +77,6 @@ struct octep_ctrl_net_h2f_req_cmd_mac {
 	u8 addr[ETH_ALEN];
 };
 
-/* get if_stats, xstats, q_stats request */
-struct octep_ctrl_net_h2f_req_cmd_get_stats {
-	/* offset into barmem where fw should copy over stats */
-	u32 offset;
-};
-
 /* get/set link state, rx state */
 struct octep_ctrl_net_h2f_req_cmd_state {
 	/* enum octep_ctrl_net_cmd */
@@ -110,26 +109,28 @@ struct octep_ctrl_net_h2f_req_cmd_link_info {
 
 /* Host to fw request data */
 struct octep_ctrl_net_h2f_req {
-	struct octep_ctrl_net_req_hdr hdr;
+	union octep_ctrl_net_req_hdr hdr;
 	union {
 		struct octep_ctrl_net_h2f_req_cmd_mtu mtu;
 		struct octep_ctrl_net_h2f_req_cmd_mac mac;
-		struct octep_ctrl_net_h2f_req_cmd_get_stats get_stats;
 		struct octep_ctrl_net_h2f_req_cmd_state link;
 		struct octep_ctrl_net_h2f_req_cmd_state rx;
 		struct octep_ctrl_net_h2f_req_cmd_link_info link_info;
 	};
 } __packed;
 
-struct octep_ctrl_net_resp_hdr {
-	/* sender id */
-	u16 sender;
-	/* receiver id */
-	u16 receiver;
-	/* octep_ctrl_net_h2t_cmd */
-	u16 cmd;
-	/* octep_ctrl_net_reply */
-	u16 reply;
+union octep_ctrl_net_resp_hdr {
+	u64 words[1];
+	struct {
+		/* sender id */
+		u16 sender;
+		/* receiver id */
+		u16 receiver;
+		/* octep_ctrl_net_h2t_cmd */
+		u16 cmd;
+		/* octep_ctrl_net_reply */
+		u16 reply;
+	} s;
 };
 
 /* get mtu response */
@@ -144,6 +145,12 @@ struct octep_ctrl_net_h2f_resp_cmd_mac {
 	u8 addr[ETH_ALEN];
 };
 
+/* get if_stats, xstats, q_stats request */
+struct octep_ctrl_net_h2f_resp_cmd_get_stats {
+	struct octep_iface_rx_stats rx_stats;
+	struct octep_iface_tx_stats tx_stats;
+};
+
 /* get link state, rx state response */
 struct octep_ctrl_net_h2f_resp_cmd_state {
 	/* enum octep_ctrl_net_state */
@@ -152,10 +159,11 @@ struct octep_ctrl_net_h2f_resp_cmd_state {
 
 /* Host to fw response data */
 struct octep_ctrl_net_h2f_resp {
-	struct octep_ctrl_net_resp_hdr hdr;
+	union octep_ctrl_net_resp_hdr hdr;
 	union {
 		struct octep_ctrl_net_h2f_resp_cmd_mtu mtu;
 		struct octep_ctrl_net_h2f_resp_cmd_mac mac;
+		struct octep_ctrl_net_h2f_resp_cmd_get_stats if_stats;
 		struct octep_ctrl_net_h2f_resp_cmd_state link;
 		struct octep_ctrl_net_h2f_resp_cmd_state rx;
 		struct octep_ctrl_net_link_info link_info;
@@ -170,7 +178,7 @@ struct octep_ctrl_net_f2h_req_cmd_state {
 
 /* Fw to host request data */
 struct octep_ctrl_net_f2h_req {
-	struct octep_ctrl_net_req_hdr hdr;
+	union octep_ctrl_net_req_hdr hdr;
 	union {
 		struct octep_ctrl_net_f2h_req_cmd_state link;
 	};
@@ -178,122 +186,152 @@ struct octep_ctrl_net_f2h_req {
 
 /* Fw to host response data */
 struct octep_ctrl_net_f2h_resp {
-	struct octep_ctrl_net_resp_hdr hdr;
+	union octep_ctrl_net_resp_hdr hdr;
 };
 
-/* Size of host to fw octep_ctrl_mbox queue element */
-union octep_ctrl_net_h2f_data_sz {
+/* Max data size to be transferred over mbox */
+union octep_ctrl_net_max_data {
 	struct octep_ctrl_net_h2f_req h2f_req;
 	struct octep_ctrl_net_h2f_resp h2f_resp;
-};
-
-/* Size of fw to host octep_ctrl_mbox queue element */
-union octep_ctrl_net_f2h_data_sz {
 	struct octep_ctrl_net_f2h_req f2h_req;
 	struct octep_ctrl_net_f2h_resp f2h_resp;
 };
 
-/* size of host to fw data in words */
-#define OCTEP_CTRL_NET_H2F_DATA_SZW		((sizeof(union octep_ctrl_net_h2f_data_sz)) / \
-						 (sizeof(unsigned long)))
-
-/* size of fw to host data in words */
-#define OCTEP_CTRL_NET_F2H_DATA_SZW		((sizeof(union octep_ctrl_net_f2h_data_sz)) / \
-						 (sizeof(unsigned long)))
-
-/* size in words of get/set mtu request */
-#define OCTEP_CTRL_NET_H2F_MTU_REQ_SZW			2
-/* size in words of get/set mac request */
-#define OCTEP_CTRL_NET_H2F_MAC_REQ_SZW			2
-/* size in words of get stats request */
-#define OCTEP_CTRL_NET_H2F_GET_STATS_REQ_SZW		2
-/* size in words of get/set state request */
-#define OCTEP_CTRL_NET_H2F_STATE_REQ_SZW		2
-/* size in words of get/set link info request */
-#define OCTEP_CTRL_NET_H2F_LINK_INFO_REQ_SZW		4
-
-/* size in words of get mtu response */
-#define OCTEP_CTRL_NET_H2F_GET_MTU_RESP_SZW		2
-/* size in words of set mtu response */
-#define OCTEP_CTRL_NET_H2F_SET_MTU_RESP_SZW		1
-/* size in words of get mac response */
-#define OCTEP_CTRL_NET_H2F_GET_MAC_RESP_SZW		2
-/* size in words of set mac response */
-#define OCTEP_CTRL_NET_H2F_SET_MAC_RESP_SZW		1
-/* size in words of get state request */
-#define OCTEP_CTRL_NET_H2F_GET_STATE_RESP_SZW		2
-/* size in words of set state request */
-#define OCTEP_CTRL_NET_H2F_SET_STATE_RESP_SZW		1
-/* size in words of get link info request */
-#define OCTEP_CTRL_NET_H2F_GET_LINK_INFO_RESP_SZW	4
-/* size in words of set link info request */
-#define OCTEP_CTRL_NET_H2F_SET_LINK_INFO_RESP_SZW	1
+struct octep_ctrl_net_wait_data {
+	struct list_head list;
+	int done;
+	struct octep_ctrl_mbox_msg msg;
+	union {
+		struct octep_ctrl_net_h2f_req req;
+		struct octep_ctrl_net_h2f_resp resp;
+	} data;
+};
+
+/** Initialize data for ctrl net.
+ *
+ * @param oct: non-null pointer to struct octep_device.
+ *
+ * return value: 0 on success, -errno on error.
+ */
+int octep_ctrl_net_init(struct octep_device *oct);
 
 /** Get link status from firmware.
  *
  * @param oct: non-null pointer to struct octep_device.
+ * @param vfid: Index of virtual function.
  *
  * return value: link status 0=down, 1=up.
  */
-int octep_get_link_status(struct octep_device *oct);
+int octep_ctrl_net_get_link_status(struct octep_device *oct, int vfid);
 
 /** Set link status in firmware.
  *
  * @param oct: non-null pointer to struct octep_device.
+ * @param vfid: Index of virtual function.
  * @param up: boolean status.
+ * @param wait_for_response: poll for response.
+ *
+ * return value: 0 on success, -errno on failure
  */
-void octep_set_link_status(struct octep_device *oct, bool up);
+int octep_ctrl_net_set_link_status(struct octep_device *oct, int vfid, bool up,
+				   bool wait_for_response);
 
 /** Set rx state in firmware.
  *
  * @param oct: non-null pointer to struct octep_device.
+ * @param vfid: Index of virtual function.
  * @param up: boolean status.
+ * @param wait_for_response: poll for response.
+ *
+ * return value: 0 on success, -errno on failure.
  */
-void octep_set_rx_state(struct octep_device *oct, bool up);
+int octep_ctrl_net_set_rx_state(struct octep_device *oct, int vfid, bool up,
+				bool wait_for_response);
 
 /** Get mac address from firmware.
  *
  * @param oct: non-null pointer to struct octep_device.
+ * @param vfid: Index of virtual function.
  * @param addr: non-null pointer to mac address.
  *
  * return value: 0 on success, -errno on failure.
  */
-int octep_get_mac_addr(struct octep_device *oct, u8 *addr);
+int octep_ctrl_net_get_mac_addr(struct octep_device *oct, int vfid, u8 *addr);
 
 /** Set mac address in firmware.
  *
  * @param oct: non-null pointer to struct octep_device.
+ * @param vfid: Index of virtual function.
  * @param addr: non-null pointer to mac address.
+ * @param wait_for_response: poll for response.
+ *
+ * return value: 0 on success, -errno on failure.
  */
-int octep_set_mac_addr(struct octep_device *oct, u8 *addr);
+int octep_ctrl_net_set_mac_addr(struct octep_device *oct, int vfid, u8 *addr,
+				bool wait_for_response);
 
 /** Set mtu in firmware.
  *
  * @param oct: non-null pointer to struct octep_device.
+ * @param vfid: Index of virtual function.
  * @param mtu: mtu.
+ * @param wait_for_response: poll for response.
+ *
+ * return value: 0 on success, -errno on failure.
  */
-int octep_set_mtu(struct octep_device *oct, int mtu);
+int octep_ctrl_net_set_mtu(struct octep_device *oct, int vfid, int mtu,
+			   bool wait_for_response);
 
 /** Get interface statistics from firmware.
  *
  * @param oct: non-null pointer to struct octep_device.
+ * @param vfid: Index of virtual function.
+ * @param rx_stats: non-null pointer struct octep_iface_rx_stats.
+ * @param tx_stats: non-null pointer struct octep_iface_tx_stats.
  *
  * return value: 0 on success, -errno on failure.
  */
-int octep_get_if_stats(struct octep_device *oct);
+int octep_ctrl_net_get_if_stats(struct octep_device *oct, int vfid,
+				struct octep_iface_rx_stats *rx_stats,
+				struct octep_iface_tx_stats *tx_stats);
 
 /** Get link info from firmware.
  *
  * @param oct: non-null pointer to struct octep_device.
+ * @param vfid: Index of virtual function.
+ * @param link_info: non-null pointer to struct octep_iface_link_info.
  *
  * return value: 0 on success, -errno on failure.
  */
-int octep_get_link_info(struct octep_device *oct);
+int octep_ctrl_net_get_link_info(struct octep_device *oct, int vfid,
+				 struct octep_iface_link_info *link_info);
 
 /** Set link info in firmware.
  *
  * @param oct: non-null pointer to struct octep_device.
+ * @param vfid: Index of virtual function.
+ * @param link_info: non-null pointer to struct octep_iface_link_info.
+ * @param wait_for_response: poll for response.
+ *
+ * return value: 0 on success, -errno on failure.
+ */
+int octep_ctrl_net_set_link_info(struct octep_device *oct,
+				 int vfid,
+				 struct octep_iface_link_info *link_info,
+				 bool wait_for_response);
+
+/** Poll for firmware messages and process them.
+ *
+ * @param oct: non-null pointer to struct octep_device.
+ */
+void octep_ctrl_net_recv_fw_messages(struct octep_device *oct);
+
+/** Uninitialize data for ctrl net.
+ *
+ * @param oct: non-null pointer to struct octep_device.
+ *
+ * return value: 0 on success, -errno on error.
  */
-int octep_set_link_info(struct octep_device *oct, struct octep_iface_link_info *link_info);
+int octep_ctrl_net_uninit(struct octep_device *oct);
 
 #endif /* __OCTEP_CTRL_NET_H__ */
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_ethtool.c b/drivers/net/ethernet/marvell/octeon_ep/octep_ethtool.c
index 87ef129b269a..7d0124b283da 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_ethtool.c
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_ethtool.c
@@ -150,9 +150,12 @@ octep_get_ethtool_stats(struct net_device *netdev,
 	rx_packets = 0;
 	rx_bytes = 0;
 
-	octep_get_if_stats(oct);
 	iface_tx_stats = &oct->iface_tx_stats;
 	iface_rx_stats = &oct->iface_rx_stats;
+	octep_ctrl_net_get_if_stats(oct,
+				    OCTEP_CTRL_NET_INVALID_VFID,
+				    iface_rx_stats,
+				    iface_tx_stats);
 
 	for (q = 0; q < oct->num_oqs; q++) {
 		struct octep_iq *iq = oct->iq[q];
@@ -283,11 +286,11 @@ static int octep_get_link_ksettings(struct net_device *netdev,
 	ethtool_link_ksettings_zero_link_mode(cmd, supported);
 	ethtool_link_ksettings_zero_link_mode(cmd, advertising);
 
-	octep_get_link_info(oct);
+	link_info = &oct->link_info;
+	octep_ctrl_net_get_link_info(oct, OCTEP_CTRL_NET_INVALID_VFID, link_info);
 
 	advertised_modes = oct->link_info.advertised_modes;
 	supported_modes = oct->link_info.supported_modes;
-	link_info = &oct->link_info;
 
 	OCTEP_SET_ETHTOOL_LINK_MODES_BITMAP(supported_modes, cmd, supported);
 	OCTEP_SET_ETHTOOL_LINK_MODES_BITMAP(advertised_modes, cmd, advertising);
@@ -439,7 +442,8 @@ static int octep_set_link_ksettings(struct net_device *netdev,
 	link_info_new.speed = cmd->base.speed;
 	link_info_new.autoneg = autoneg;
 
-	err = octep_set_link_info(oct, &link_info_new);
+	err = octep_ctrl_net_set_link_info(oct, OCTEP_CTRL_NET_INVALID_VFID,
+					   &link_info_new, true);
 	if (err)
 		return err;
 
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c
index 5a898fb88e37..e1853da280f9 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c
@@ -8,7 +8,6 @@
 #include <linux/types.h>
 #include <linux/module.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
 #include <linux/rtnetlink.h>
@@ -18,6 +17,7 @@
 #include "octep_main.h"
 #include "octep_ctrl_net.h"
 
+#define OCTEP_INTR_POLL_TIME_MSECS    100
 struct workqueue_struct *octep_wq;
 
 /* Supported Devices */
@@ -507,11 +507,11 @@ static int octep_open(struct net_device *netdev)
 	octep_napi_enable(oct);
 
 	oct->link_info.admin_up = 1;
-	octep_set_rx_state(oct, true);
-
-	ret = octep_get_link_status(oct);
-	if (!ret)
-		octep_set_link_status(oct, true);
+	octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, true,
+				    false);
+	octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, true,
+				       false);
+	oct->poll_non_ioq_intr = false;
 
 	/* Enable the input and output queues for this Octeon device */
 	oct->hw_ops.enable_io_queues(oct);
@@ -521,7 +521,7 @@ static int octep_open(struct net_device *netdev)
 
 	octep_oq_dbell_init(oct);
 
-	ret = octep_get_link_status(oct);
+	ret = octep_ctrl_net_get_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID);
 	if (ret > 0)
 		octep_link_up(netdev);
 
@@ -551,14 +551,16 @@ static int octep_stop(struct net_device *netdev)
 
 	netdev_info(netdev, "Stopping the device ...\n");
 
+	octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, false,
+				       false);
+	octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, false,
+				    false);
+
 	/* Stop Tx from stack */
 	netif_tx_stop_all_queues(netdev);
 	netif_carrier_off(netdev);
 	netif_tx_disable(netdev);
 
-	octep_set_link_status(oct, false);
-	octep_set_rx_state(oct, false);
-
 	oct->link_info.admin_up = 0;
 	oct->link_info.oper_up = 0;
 
@@ -573,6 +575,11 @@ static int octep_stop(struct net_device *netdev)
 	oct->hw_ops.reset_io_queues(oct);
 	octep_free_oqs(oct);
 	octep_free_iqs(oct);
+
+	oct->poll_non_ioq_intr = true;
+	queue_delayed_work(octep_wq, &oct->intr_poll_task,
+			   msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
+
 	netdev_info(netdev, "Device stopped !!\n");
 	return 0;
 }
@@ -755,7 +762,12 @@ static void octep_get_stats64(struct net_device *netdev,
 	struct octep_device *oct = netdev_priv(netdev);
 	int q;
 
-	octep_get_if_stats(oct);
+	if (netif_running(netdev))
+		octep_ctrl_net_get_if_stats(oct,
+					    OCTEP_CTRL_NET_INVALID_VFID,
+					    &oct->iface_rx_stats,
+					    &oct->iface_tx_stats);
+
 	tx_packets = 0;
 	tx_bytes = 0;
 	rx_packets = 0;
@@ -826,7 +838,8 @@ static int octep_set_mac(struct net_device *netdev, void *p)
 	if (!is_valid_ether_addr(addr->sa_data))
 		return -EADDRNOTAVAIL;
 
-	err = octep_set_mac_addr(oct, addr->sa_data);
+	err = octep_ctrl_net_set_mac_addr(oct, OCTEP_CTRL_NET_INVALID_VFID,
+					  addr->sa_data, true);
 	if (err)
 		return err;
 
@@ -846,7 +859,8 @@ static int octep_change_mtu(struct net_device *netdev, int new_mtu)
 	if (link_info->mtu == new_mtu)
 		return 0;
 
-	err = octep_set_mtu(oct, new_mtu);
+	err = octep_ctrl_net_set_mtu(oct, OCTEP_CTRL_NET_INVALID_VFID, new_mtu,
+				     true);
 	if (!err) {
 		oct->link_info.mtu = new_mtu;
 		netdev->mtu = new_mtu;
@@ -866,6 +880,59 @@ static const struct net_device_ops octep_netdev_ops = {
 };
 
 /**
+ * octep_intr_poll_task - work queue task to process non-ioq interrupts.
+ *
+ * @work: pointer to mbox work_struct
+ *
+ * Process non-ioq interrupts to handle control mailbox, pfvf mailbox.
+ **/
+static void octep_intr_poll_task(struct work_struct *work)
+{
+	struct octep_device *oct = container_of(work, struct octep_device,
+						intr_poll_task.work);
+
+	if (!oct->poll_non_ioq_intr) {
+		dev_info(&oct->pdev->dev, "Interrupt poll task stopped.\n");
+		return;
+	}
+
+	oct->hw_ops.poll_non_ioq_interrupts(oct);
+	queue_delayed_work(octep_wq, &oct->intr_poll_task,
+			   msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
+}
+
+/**
+ * octep_hb_timeout_task - work queue task to check firmware heartbeat.
+ *
+ * @work: pointer to hb work_struct
+ *
+ * Check for heartbeat miss count. Uninitialize oct device if miss count
+ * exceeds configured max heartbeat miss count.
+ *
+ **/
+static void octep_hb_timeout_task(struct work_struct *work)
+{
+	struct octep_device *oct = container_of(work, struct octep_device,
+						hb_task.work);
+
+	int miss_cnt;
+
+	miss_cnt = atomic_inc_return(&oct->hb_miss_cnt);
+	if (miss_cnt < oct->conf->max_hb_miss_cnt) {
+		queue_delayed_work(octep_wq, &oct->hb_task,
+				   msecs_to_jiffies(oct->conf->hb_interval * 1000));
+		return;
+	}
+
+	dev_err(&oct->pdev->dev, "Missed %u heartbeats. Uninitializing\n",
+		miss_cnt);
+	rtnl_lock();
+	if (netif_running(oct->netdev))
+		octep_stop(oct->netdev);
+	rtnl_unlock();
+}
+
+/**
  * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages.
  *
  * @work: pointer to ctrl mbox work_struct
@@ -876,34 +943,8 @@ static void octep_ctrl_mbox_task(struct work_struct *work)
 {
 	struct octep_device *oct = container_of(work, struct octep_device,
 						ctrl_mbox_task);
-	struct net_device *netdev = oct->netdev;
-	struct octep_ctrl_net_f2h_req req = {};
-	struct octep_ctrl_mbox_msg msg;
-	int ret = 0;
-
-	msg.msg = &req;
-	while (true) {
-		ret = octep_ctrl_mbox_recv(&oct->ctrl_mbox, &msg);
-		if (ret)
-			break;
-
-		switch (req.hdr.cmd) {
-		case OCTEP_CTRL_NET_F2H_CMD_LINK_STATUS:
-			if (netif_running(netdev)) {
-				if (req.link.state) {
-					dev_info(&oct->pdev->dev, "netif_carrier_on\n");
-					netif_carrier_on(netdev);
-				} else {
-					dev_info(&oct->pdev->dev, "netif_carrier_off\n");
-					netif_carrier_off(netdev);
-				}
-			}
-			break;
-		default:
-			pr_info("Unknown mbox req : %u\n", req.hdr.cmd);
-			break;
-		}
-	}
+
+	octep_ctrl_net_recv_fw_messages(oct);
 }
 
 static const char *octep_devid_to_str(struct octep_device *oct)
@@ -927,7 +968,6 @@ static const char *octep_devid_to_str(struct octep_device *oct)
  */
 int octep_device_setup(struct octep_device *oct)
 {
-	struct octep_ctrl_mbox *ctrl_mbox;
 	struct pci_dev *pdev = oct->pdev;
 	int i, ret;
 
@@ -964,19 +1004,14 @@ int octep_device_setup(struct octep_device *oct)
 
 	oct->pkind = CFG_GET_IQ_PKIND(oct->conf);
 
-	/* Initialize control mbox */
-	ctrl_mbox = &oct->ctrl_mbox;
-	ctrl_mbox->barmem = CFG_GET_CTRL_MBOX_MEM_ADDR(oct->conf);
-	ret = octep_ctrl_mbox_init(ctrl_mbox);
-	if (ret) {
-		dev_err(&pdev->dev, "Failed to initialize control mbox\n");
-		goto unsupported_dev;
-	}
-	oct->ctrl_mbox_ifstats_offset = OCTEP_CTRL_MBOX_SZ(ctrl_mbox->h2fq.elem_sz,
-							   ctrl_mbox->h2fq.elem_cnt,
-							   ctrl_mbox->f2hq.elem_sz,
-							   ctrl_mbox->f2hq.elem_cnt);
+	ret = octep_ctrl_net_init(oct);
+	if (ret)
+		return ret;
 
+	atomic_set(&oct->hb_miss_cnt, 0);
+	INIT_DELAYED_WORK(&oct->hb_task, octep_hb_timeout_task);
+	queue_delayed_work(octep_wq, &oct->hb_task,
+			   msecs_to_jiffies(oct->conf->hb_interval * 1000));
 	return 0;
 
 unsupported_dev:
@@ -1005,7 +1040,8 @@ static void octep_device_cleanup(struct octep_device *oct)
 		oct->mbox[i] = NULL;
 	}
 
-	octep_ctrl_mbox_uninit(&oct->ctrl_mbox);
+	octep_ctrl_net_uninit(oct);
+	cancel_delayed_work_sync(&oct->hb_task);
 
 	oct->hw_ops.soft_reset(oct);
 	for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
@@ -1017,6 +1053,26 @@ static void octep_device_cleanup(struct octep_device *oct)
 	oct->conf = NULL;
 }
 
+static bool get_fw_ready_status(struct pci_dev *pdev)
+{
+	u32 pos = 0;
+	u16 vsec_id;
+	u8 status;
+
+	while ((pos = pci_find_next_ext_capability(pdev, pos,
+						   PCI_EXT_CAP_ID_VNDR))) {
+		pci_read_config_word(pdev, pos + 4, &vsec_id);
+#define FW_STATUS_VSEC_ID  0xA3
+		if (vsec_id != FW_STATUS_VSEC_ID)
+			continue;
+
+		pci_read_config_byte(pdev, (pos + 8), &status);
+		dev_info(&pdev->dev, "Firmware ready status = %u\n", status);
+		return status;
+	}
+	return false;
+}
+
 /**
  * octep_probe() - Octeon PCI device probe handler.
  *
@@ -1050,9 +1106,14 @@ static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto err_pci_regions;
 	}
 
-	pci_enable_pcie_error_reporting(pdev);
 	pci_set_master(pdev);
 
+	if (!get_fw_ready_status(pdev)) {
+		dev_notice(&pdev->dev, "Firmware not ready; defer probe.\n");
+		err = -EPROBE_DEFER;
+		goto err_alloc_netdev;
+	}
+
 	netdev = alloc_etherdev_mq(sizeof(struct octep_device),
 				   OCTEP_MAX_QUEUES);
 	if (!netdev) {
@@ -1075,6 +1136,10 @@ static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 	INIT_WORK(&octep_dev->tx_timeout_task, octep_tx_timeout_task);
 	INIT_WORK(&octep_dev->ctrl_mbox_task, octep_ctrl_mbox_task);
+	INIT_DELAYED_WORK(&octep_dev->intr_poll_task, octep_intr_poll_task);
+	octep_dev->poll_non_ioq_intr = true;
+	queue_delayed_work(octep_wq, &octep_dev->intr_poll_task,
+			   msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
 
 	netdev->netdev_ops = &octep_netdev_ops;
 	octep_set_ethtool_ops(netdev);
@@ -1086,7 +1151,8 @@ static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	netdev->max_mtu = OCTEP_MAX_MTU;
 	netdev->mtu = OCTEP_DEFAULT_MTU;
 
-	err = octep_get_mac_addr(octep_dev, octep_dev->mac_addr);
+	err = octep_ctrl_net_get_mac_addr(octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
+					  octep_dev->mac_addr);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to get mac address\n");
 		goto register_dev_err;
@@ -1106,7 +1172,6 @@ register_dev_err:
 err_octep_config:
 	free_netdev(netdev);
 err_alloc_netdev:
-	pci_disable_pcie_error_reporting(pdev);
 	pci_release_mem_regions(pdev);
 err_pci_regions:
 err_dma_mask:
@@ -1136,10 +1201,11 @@ static void octep_remove(struct pci_dev *pdev)
 	if (netdev->reg_state == NETREG_REGISTERED)
 		unregister_netdev(netdev);
 
+	oct->poll_non_ioq_intr = false;
+	cancel_delayed_work_sync(&oct->intr_poll_task);
 	octep_device_cleanup(oct);
 	pci_release_mem_regions(pdev);
 	free_netdev(netdev);
-	pci_disable_pcie_error_reporting(pdev);
 	pci_disable_device(pdev);
 }
 
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h
index 123ffc13754d..e0907a719133 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h
@@ -73,6 +73,7 @@ struct octep_hw_ops {
 
 	void (*enable_interrupts)(struct octep_device *oct);
 	void (*disable_interrupts)(struct octep_device *oct);
+	bool (*poll_non_ioq_interrupts)(struct octep_device *oct);
 
 	void (*enable_io_queues)(struct octep_device *oct);
 	void (*disable_io_queues)(struct octep_device *oct);
@@ -270,7 +271,22 @@ struct octep_device {
 
 	/* Work entry to handle ctrl mbox interrupt */
 	struct work_struct ctrl_mbox_task;
-
+	/* Wait queue for host to firmware requests */
+	wait_queue_head_t ctrl_req_wait_q;
+	/* List of objects waiting for h2f response */
+	struct list_head ctrl_req_wait_list;
+
+	/* Enable non-ioq interrupt polling */
+	bool poll_non_ioq_intr;
+	/* Work entry to poll non-ioq interrupts */
+	struct delayed_work intr_poll_task;
+
+	/* Firmware heartbeat timer */
+	struct timer_list hb_timer;
+	/* Firmware heartbeat miss count tracked by timer */
+	atomic_t hb_miss_cnt;
+	/* Task to reset device on heartbeat miss */
+	struct delayed_work hb_task;
 };
 
 static inline u16 OCTEP_MAJOR_REV(struct octep_device *oct)
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h
index 3d5d39a52fe6..b25c3093dc7b 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h
@@ -364,4 +364,10 @@
 
 /* Number of non-queue interrupts in CN93xx */
 #define    CN93_NUM_NON_IOQ_INTR    16
+
+/* bit 0 for control mbox interrupt */
+#define CN93_SDP_EPF_OEI_RINT_DATA_BIT_MBOX	BIT_ULL(0)
+/* bit 1 for firmware heartbeat interrupt */
+#define CN93_SDP_EPF_OEI_RINT_DATA_BIT_HBEAT	BIT_ULL(1)
+
 #endif /* _OCTEP_REGS_CN9K_PF_H_ */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
index 5727d67e0259..8fb5cae7285b 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
@@ -936,7 +936,7 @@ struct nix_aq_enq_req {
 		struct nix_cq_ctx_s cq;
 		struct nix_rsse_s   rss;
 		struct nix_rx_mce_s mce;
-		u64 prof;
+		struct nix_bandprof_s prof;
 	};
 	union {
 		struct nix_rq_ctx_s rq_mask;
@@ -944,7 +944,7 @@ struct nix_aq_enq_req {
 		struct nix_cq_ctx_s cq_mask;
 		struct nix_rsse_s   rss_mask;
 		struct nix_rx_mce_s mce_mask;
-		u64 prof_mask;
+		struct nix_bandprof_s prof_mask;
 	};
 };
 
diff --git a/drivers/net/ethernet/marvell/pxa168_eth.c b/drivers/net/ethernet/marvell/pxa168_eth.c
index 87fff539d39d..d5691b6a2bc5 100644
--- a/drivers/net/ethernet/marvell/pxa168_eth.c
+++ b/drivers/net/ethernet/marvell/pxa168_eth.c
@@ -1586,7 +1586,7 @@ static struct platform_driver pxa168_eth_driver = {
 	.suspend = pxa168_eth_suspend,
 	.driver = {
 		.name		= DRIVER_NAME,
-		.of_match_table	= of_match_ptr(pxa168_eth_of_match),
+		.of_match_table	= pxa168_eth_of_match,
 	},
 };
 
diff --git a/drivers/net/ethernet/mediatek/Kconfig b/drivers/net/ethernet/mediatek/Kconfig
index 97374fb3ee79..da0db417ab69 100644
--- a/drivers/net/ethernet/mediatek/Kconfig
+++ b/drivers/net/ethernet/mediatek/Kconfig
@@ -19,6 +19,8 @@ config NET_MEDIATEK_SOC
 	select DIMLIB
 	select PAGE_POOL
 	select PAGE_POOL_STATS
+	select PCS_MTK_LYNXI
+	select REGMAP_MMIO
 	help
 	  This driver supports the gigabit ethernet MACs in the
 	  MediaTek SoC family.
diff --git a/drivers/net/ethernet/mediatek/Makefile b/drivers/net/ethernet/mediatek/Makefile
index 8e0c61c33ff8..03e008fbc859 100644
--- a/drivers/net/ethernet/mediatek/Makefile
+++ b/drivers/net/ethernet/mediatek/Makefile
@@ -4,7 +4,7 @@
 #
 
 obj-$(CONFIG_NET_MEDIATEK_SOC) += mtk_eth.o
-mtk_eth-y := mtk_eth_soc.o mtk_sgmii.o mtk_eth_path.o mtk_ppe.o mtk_ppe_debugfs.o mtk_ppe_offload.o
+mtk_eth-y := mtk_eth_soc.o mtk_eth_path.o mtk_ppe.o mtk_ppe_debugfs.o mtk_ppe_offload.o
 mtk_eth-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed.o mtk_wed_mcu.o mtk_wed_wo.o
 ifdef CONFIG_DEBUG_FS
 mtk_eth-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed_debugfs.o
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_path.c b/drivers/net/ethernet/mediatek/mtk_eth_path.c
index 72648535a14d..317e447f4991 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_path.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_path.c
@@ -96,12 +96,20 @@ static int set_mux_gmac2_gmac0_to_gephy(struct mtk_eth *eth, int path)
 
 static int set_mux_u3_gmac2_to_qphy(struct mtk_eth *eth, int path)
 {
-	unsigned int val = 0;
+	unsigned int val = 0, mask = 0, reg = 0;
 	bool updated = true;
 
 	switch (path) {
 	case MTK_ETH_PATH_GMAC2_SGMII:
-		val = CO_QPHY_SEL;
+		if (MTK_HAS_CAPS(eth->soc->caps, MTK_U3_COPHY_V2)) {
+			reg = USB_PHY_SWITCH_REG;
+			val = SGMII_QPHY_SEL;
+			mask = QPHY_SEL_MASK;
+		} else {
+			reg = INFRA_MISC2;
+			val = CO_QPHY_SEL;
+			mask = val;
+		}
 		break;
 	default:
 		updated = false;
@@ -109,7 +117,7 @@ static int set_mux_u3_gmac2_to_qphy(struct mtk_eth *eth, int path)
 	}
 
 	if (updated)
-		regmap_update_bits(eth->infra, INFRA_MISC2, CO_QPHY_SEL, val);
+		regmap_update_bits(eth->infra, reg, mask, val);
 
 	dev_dbg(eth->dev, "path %s in %s updated = %d\n",
 		mtk_eth_path_name(path), __func__, updated);
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index e14050e17862..9e948d091a69 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -20,6 +20,7 @@
 #include <linux/interrupt.h>
 #include <linux/pinctrl/devinfo.h>
 #include <linux/phylink.h>
+#include <linux/pcs/pcs-mtk-lynxi.h>
 #include <linux/jhash.h>
 #include <linux/bitfield.h>
 #include <net/dsa.h>
@@ -374,17 +375,6 @@ static int mt7621_gmac0_rgmii_adjust(struct mtk_eth *eth,
 {
 	u32 val;
 
-	/* Check DDR memory type.
-	 * Currently TRGMII mode with DDR2 memory is not supported.
-	 */
-	regmap_read(eth->ethsys, ETHSYS_SYSCFG, &val);
-	if (interface == PHY_INTERFACE_MODE_TRGMII &&
-	    val & SYSCFG_DRAM_TYPE_DDR2) {
-		dev_err(eth->dev,
-			"TRGMII mode with DDR2 memory is not supported!\n");
-		return -EOPNOTSUPP;
-	}
-
 	val = (interface == PHY_INTERFACE_MODE_TRGMII) ?
 		ETHSYS_TRGMII_MT7621_DDR_PLL : 0;
 
@@ -397,38 +387,42 @@ static int mt7621_gmac0_rgmii_adjust(struct mtk_eth *eth,
 static void mtk_gmac0_rgmii_adjust(struct mtk_eth *eth,
 				   phy_interface_t interface, int speed)
 {
-	u32 val;
+	unsigned long rate;
+	u32 tck, rck, intf;
 	int ret;
 
 	if (interface == PHY_INTERFACE_MODE_TRGMII) {
 		mtk_w32(eth, TRGMII_MODE, INTF_MODE);
-		val = 500000000;
-		ret = clk_set_rate(eth->clks[MTK_CLK_TRGPLL], val);
+		ret = clk_set_rate(eth->clks[MTK_CLK_TRGPLL], 500000000);
 		if (ret)
 			dev_err(eth->dev, "Failed to set trgmii pll: %d\n", ret);
 		return;
 	}
 
-	val = (speed == SPEED_1000) ?
-		INTF_MODE_RGMII_1000 : INTF_MODE_RGMII_10_100;
-	mtk_w32(eth, val, INTF_MODE);
+	if (speed == SPEED_1000) {
+		intf = INTF_MODE_RGMII_1000;
+		rate = 250000000;
+		rck = RCK_CTRL_RGMII_1000;
+		tck = TCK_CTRL_RGMII_1000;
+	} else {
+		intf = INTF_MODE_RGMII_10_100;
+		rate = 500000000;
+		rck = RCK_CTRL_RGMII_10_100;
+		tck = TCK_CTRL_RGMII_10_100;
+	}
+
+	mtk_w32(eth, intf, INTF_MODE);
 
 	regmap_update_bits(eth->ethsys, ETHSYS_CLKCFG0,
 			   ETHSYS_TRGMII_CLK_SEL362_5,
 			   ETHSYS_TRGMII_CLK_SEL362_5);
 
-	val = (speed == SPEED_1000) ? 250000000 : 500000000;
-	ret = clk_set_rate(eth->clks[MTK_CLK_TRGPLL], val);
+	ret = clk_set_rate(eth->clks[MTK_CLK_TRGPLL], rate);
 	if (ret)
 		dev_err(eth->dev, "Failed to set trgmii pll: %d\n", ret);
 
-	val = (speed == SPEED_1000) ?
-		RCK_CTRL_RGMII_1000 : RCK_CTRL_RGMII_10_100;
-	mtk_w32(eth, val, TRGMII_RCK_CTRL);
-
-	val = (speed == SPEED_1000) ?
-		TCK_CTRL_RGMII_1000 : TCK_CTRL_RGMII_10_100;
-	mtk_w32(eth, val, TRGMII_TCK_CTRL);
+	mtk_w32(eth, rck, TRGMII_RCK_CTRL);
+	mtk_w32(eth, tck, TRGMII_TCK_CTRL);
 }
 
 static struct phylink_pcs *mtk_mac_select_pcs(struct phylink_config *config,
@@ -444,7 +438,7 @@ static struct phylink_pcs *mtk_mac_select_pcs(struct phylink_config *config,
 		sid = (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_SGMII)) ?
 		       0 : mac->id;
 
-		return mtk_sgmii_select_pcs(eth->sgmii, sid);
+		return eth->sgmii_pcs[sid];
 	}
 
 	return NULL;
@@ -465,19 +459,11 @@ static void mtk_mac_config(struct phylink_config *config, unsigned int mode,
 		/* Setup soc pin functions */
 		switch (state->interface) {
 		case PHY_INTERFACE_MODE_TRGMII:
-			if (mac->id)
-				goto err_phy;
-			if (!MTK_HAS_CAPS(mac->hw->soc->caps,
-					  MTK_GMAC1_TRGMII))
-				goto err_phy;
-			fallthrough;
 		case PHY_INTERFACE_MODE_RGMII_TXID:
 		case PHY_INTERFACE_MODE_RGMII_RXID:
 		case PHY_INTERFACE_MODE_RGMII_ID:
 		case PHY_INTERFACE_MODE_RGMII:
 		case PHY_INTERFACE_MODE_MII:
-		case PHY_INTERFACE_MODE_REVMII:
-		case PHY_INTERFACE_MODE_RMII:
 			if (MTK_HAS_CAPS(eth->soc->caps, MTK_RGMII)) {
 				err = mtk_gmac_rgmii_path_setup(eth, mac->id);
 				if (err)
@@ -487,11 +473,9 @@ static void mtk_mac_config(struct phylink_config *config, unsigned int mode,
 		case PHY_INTERFACE_MODE_1000BASEX:
 		case PHY_INTERFACE_MODE_2500BASEX:
 		case PHY_INTERFACE_MODE_SGMII:
-			if (MTK_HAS_CAPS(eth->soc->caps, MTK_SGMII)) {
-				err = mtk_gmac_sgmii_path_setup(eth, mac->id);
-				if (err)
-					goto init_err;
-			}
+			err = mtk_gmac_sgmii_path_setup(eth, mac->id);
+			if (err)
+				goto init_err;
 			break;
 		case PHY_INTERFACE_MODE_GMII:
 			if (MTK_HAS_CAPS(eth->soc->caps, MTK_GEPHY)) {
@@ -539,21 +523,13 @@ static void mtk_mac_config(struct phylink_config *config, unsigned int mode,
 			}
 		}
 
-		ge_mode = 0;
 		switch (state->interface) {
 		case PHY_INTERFACE_MODE_MII:
 		case PHY_INTERFACE_MODE_GMII:
 			ge_mode = 1;
 			break;
-		case PHY_INTERFACE_MODE_REVMII:
-			ge_mode = 2;
-			break;
-		case PHY_INTERFACE_MODE_RMII:
-			if (mac->id)
-				goto err_phy;
-			ge_mode = 3;
-			break;
 		default:
+			ge_mode = 0;
 			break;
 		}
 
@@ -789,8 +765,10 @@ static const struct phylink_mac_ops mtk_phylink_ops = {
 
 static int mtk_mdio_init(struct mtk_eth *eth)
 {
+	unsigned int max_clk = 2500000, divider;
 	struct device_node *mii_np;
 	int ret;
+	u32 val;
 
 	mii_np = of_get_child_by_name(eth->dev->of_node, "mdio-bus");
 	if (!mii_np) {
@@ -818,6 +796,25 @@ static int mtk_mdio_init(struct mtk_eth *eth)
 	eth->mii_bus->parent = eth->dev;
 
 	snprintf(eth->mii_bus->id, MII_BUS_ID_SIZE, "%pOFn", mii_np);
+
+	if (!of_property_read_u32(mii_np, "clock-frequency", &val)) {
+		if (val > MDC_MAX_FREQ || val < MDC_MAX_FREQ / MDC_MAX_DIVIDER) {
+			dev_err(eth->dev, "MDIO clock frequency out of range");
+			ret = -EINVAL;
+			goto err_put_node;
+		}
+		max_clk = val;
+	}
+	divider = min_t(unsigned int, DIV_ROUND_UP(MDC_MAX_FREQ, max_clk), 63);
+
+	/* Configure MDC Divider */
+	val = mtk_r32(eth, MTK_PPSC);
+	val &= ~PPSC_MDC_CFG;
+	val |= FIELD_PREP(PPSC_MDC_CFG, divider) | PPSC_MDC_TURBO;
+	mtk_w32(eth, val, MTK_PPSC);
+
+	dev_dbg(eth->dev, "MDC is running on %d Hz\n", MDC_MAX_FREQ / divider);
+
 	ret = of_mdiobus_register(eth->mii_bus, mii_np);
 
 err_put_node:
@@ -4059,8 +4056,17 @@ static int mtk_unreg_dev(struct mtk_eth *eth)
 	return 0;
 }
 
+static void mtk_sgmii_destroy(struct mtk_eth *eth)
+{
+	int i;
+
+	for (i = 0; i < MTK_MAX_DEVS; i++)
+		mtk_pcs_lynxi_destroy(eth->sgmii_pcs[i]);
+}
+
 static int mtk_cleanup(struct mtk_eth *eth)
 {
+	mtk_sgmii_destroy(eth);
 	mtk_unreg_dev(eth);
 	mtk_free_dev(eth);
 	cancel_work_sync(&eth->pending_work);
@@ -4332,6 +4338,7 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
 	struct mtk_mac *mac;
 	int id, err;
 	int txqs = 1;
+	u32 val;
 
 	if (!_id) {
 		dev_err(eth->dev, "missing mac id\n");
@@ -4408,6 +4415,15 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
 		__set_bit(PHY_INTERFACE_MODE_TRGMII,
 			  mac->phylink_config.supported_interfaces);
 
+	/* TRGMII is not permitted on MT7621 if using DDR2 */
+	if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_GMAC1_TRGMII) &&
+	    MTK_HAS_CAPS(mac->hw->soc->caps, MTK_TRGMII_MT7621_CLK)) {
+		regmap_read(eth->ethsys, ETHSYS_SYSCFG, &val);
+		if (val & SYSCFG_DRAM_TYPE_DDR2)
+			__clear_bit(PHY_INTERFACE_MODE_TRGMII,
+				    mac->phylink_config.supported_interfaces);
+	}
+
 	if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_SGMII)) {
 		__set_bit(PHY_INTERFACE_MODE_SGMII,
 			  mac->phylink_config.supported_interfaces);
@@ -4496,6 +4512,36 @@ void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev)
 	rtnl_unlock();
 }
 
+static int mtk_sgmii_init(struct mtk_eth *eth)
+{
+	struct device_node *np;
+	struct regmap *regmap;
+	u32 flags;
+	int i;
+
+	for (i = 0; i < MTK_MAX_DEVS; i++) {
+		np = of_parse_phandle(eth->dev->of_node, "mediatek,sgmiisys", i);
+		if (!np)
+			break;
+
+		regmap = syscon_node_to_regmap(np);
+		flags = 0;
+		if (of_property_read_bool(np, "mediatek,pnswap"))
+			flags |= MTK_SGMII_FLAG_PN_SWAP;
+
+		of_node_put(np);
+
+		if (IS_ERR(regmap))
+			return PTR_ERR(regmap);
+
+		eth->sgmii_pcs[i] = mtk_pcs_lynxi_create(eth->dev, regmap,
+							 eth->soc->ana_rgc3,
+							 flags);
+	}
+
+	return 0;
+}
+
 static int mtk_probe(struct platform_device *pdev)
 {
 	struct resource *res = NULL;
@@ -4559,13 +4605,7 @@ static int mtk_probe(struct platform_device *pdev)
 	}
 
 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SGMII)) {
-		eth->sgmii = devm_kzalloc(eth->dev, sizeof(*eth->sgmii),
-					  GFP_KERNEL);
-		if (!eth->sgmii)
-			return -ENOMEM;
-
-		err = mtk_sgmii_init(eth->sgmii, pdev->dev.of_node,
-				     eth->soc->ana_rgc3);
+		err = mtk_sgmii_init(eth);
 
 		if (err)
 			return err;
@@ -4576,14 +4616,17 @@ static int mtk_probe(struct platform_device *pdev)
 							    "mediatek,pctl");
 		if (IS_ERR(eth->pctl)) {
 			dev_err(&pdev->dev, "no pctl regmap found\n");
-			return PTR_ERR(eth->pctl);
+			err = PTR_ERR(eth->pctl);
+			goto err_destroy_sgmii;
 		}
 	}
 
 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
 		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-		if (!res)
-			return -EINVAL;
+		if (!res) {
+			err = -EINVAL;
+			goto err_destroy_sgmii;
+		}
 	}
 
 	if (eth->soc->offload_version) {
@@ -4693,8 +4736,8 @@ static int mtk_probe(struct platform_device *pdev)
 		for (i = 0; i < num_ppe; i++) {
 			u32 ppe_addr = eth->soc->reg_map->ppe_base + i * 0x400;
 
-			eth->ppe[i] = mtk_ppe_init(eth, eth->base + ppe_addr,
-						   eth->soc->offload_version, i);
+			eth->ppe[i] = mtk_ppe_init(eth, eth->base + ppe_addr, i);
+
 			if (!eth->ppe[i]) {
 				err = -ENOMEM;
 				goto err_deinit_ppe;
@@ -4742,6 +4785,8 @@ err_deinit_hw:
 	mtk_hw_deinit(eth);
 err_wed_exit:
 	mtk_wed_exit();
+err_destroy_sgmii:
+	mtk_sgmii_destroy(eth);
 
 	return err;
 }
@@ -4816,6 +4861,7 @@ static const struct mtk_soc_data mt7622_data = {
 	.required_pctl = false,
 	.offload_version = 2,
 	.hash_offset = 2,
+	.has_accounting = true,
 	.foe_entry_size = sizeof(struct mtk_foe_entry) - 16,
 	.txrx = {
 		.txd_size = sizeof(struct mtk_tx_dma),
@@ -4853,6 +4899,7 @@ static const struct mtk_soc_data mt7629_data = {
 	.hw_features = MTK_HW_FEATURES,
 	.required_clks = MT7629_CLKS_BITMAP,
 	.required_pctl = false,
+	.has_accounting = true,
 	.txrx = {
 		.txd_size = sizeof(struct mtk_tx_dma),
 		.rxd_size = sizeof(struct mtk_rx_dma),
@@ -4863,6 +4910,27 @@ static const struct mtk_soc_data mt7629_data = {
 	},
 };
 
+static const struct mtk_soc_data mt7981_data = {
+	.reg_map = &mt7986_reg_map,
+	.ana_rgc3 = 0x128,
+	.caps = MT7981_CAPS,
+	.hw_features = MTK_HW_FEATURES,
+	.required_clks = MT7981_CLKS_BITMAP,
+	.required_pctl = false,
+	.offload_version = 2,
+	.hash_offset = 4,
+	.foe_entry_size = sizeof(struct mtk_foe_entry),
+	.has_accounting = true,
+	.txrx = {
+		.txd_size = sizeof(struct mtk_tx_dma_v2),
+		.rxd_size = sizeof(struct mtk_rx_dma_v2),
+		.rx_irq_done_mask = MTK_RX_DONE_INT_V2,
+		.rx_dma_l4_valid = RX_DMA_L4_VALID_V2,
+		.dma_max_len = MTK_TX_DMA_BUF_LEN_V2,
+		.dma_len_offset = 8,
+	},
+};
+
 static const struct mtk_soc_data mt7986_data = {
 	.reg_map = &mt7986_reg_map,
 	.ana_rgc3 = 0x128,
@@ -4873,6 +4941,7 @@ static const struct mtk_soc_data mt7986_data = {
 	.offload_version = 2,
 	.hash_offset = 4,
 	.foe_entry_size = sizeof(struct mtk_foe_entry),
+	.has_accounting = true,
 	.txrx = {
 		.txd_size = sizeof(struct mtk_tx_dma_v2),
 		.rxd_size = sizeof(struct mtk_rx_dma_v2),
@@ -4905,6 +4974,7 @@ const struct of_device_id of_mtk_match[] = {
 	{ .compatible = "mediatek,mt7622-eth", .data = &mt7622_data},
 	{ .compatible = "mediatek,mt7623-eth", .data = &mt7623_data},
 	{ .compatible = "mediatek,mt7629-eth", .data = &mt7629_data},
+	{ .compatible = "mediatek,mt7981-eth", .data = &mt7981_data},
 	{ .compatible = "mediatek,mt7986-eth", .data = &mt7986_data},
 	{ .compatible = "ralink,rt5350-eth", .data = &rt5350_data},
 	{},
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
index 084a6badef6d..cdcf8534283e 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
@@ -363,6 +363,13 @@
 #define RX_DMA_VTAG_V2		BIT(0)
 #define RX_DMA_L4_VALID_V2	BIT(2)
 
+/* PHY Polling and SMI Master Control registers */
+#define MTK_PPSC		0x10000
+#define PPSC_MDC_CFG		GENMASK(29, 24)
+#define PPSC_MDC_TURBO		BIT(20)
+#define MDC_MAX_FREQ		25000000
+#define MDC_MAX_DIVIDER		63
+
 /* PHY Indirect Access Control registers */
 #define MTK_PHY_IAC		0x10004
 #define PHY_IAC_ACCESS		BIT(31)
@@ -503,64 +510,16 @@
 #define ETHSYS_DMA_AG_MAP_QDMA	BIT(1)
 #define ETHSYS_DMA_AG_MAP_PPE	BIT(2)
 
-/* SGMII subsystem config registers */
-/* BMCR (low 16) BMSR (high 16) */
-#define SGMSYS_PCS_CONTROL_1	0x0
-#define SGMII_BMCR		GENMASK(15, 0)
-#define SGMII_BMSR		GENMASK(31, 16)
-#define SGMII_AN_RESTART	BIT(9)
-#define SGMII_ISOLATE		BIT(10)
-#define SGMII_AN_ENABLE		BIT(12)
-#define SGMII_LINK_STATYS	BIT(18)
-#define SGMII_AN_ABILITY	BIT(19)
-#define SGMII_AN_COMPLETE	BIT(21)
-#define SGMII_PCS_FAULT		BIT(23)
-#define SGMII_AN_EXPANSION_CLR	BIT(30)
-
-#define SGMSYS_PCS_ADVERTISE	0x8
-#define SGMII_ADVERTISE		GENMASK(15, 0)
-#define SGMII_LPA		GENMASK(31, 16)
-
-/* Register to programmable link timer, the unit in 2 * 8ns */
-#define SGMSYS_PCS_LINK_TIMER	0x18
-#define SGMII_LINK_TIMER_MASK	GENMASK(19, 0)
-#define SGMII_LINK_TIMER_DEFAULT	(0x186a0 & SGMII_LINK_TIMER_MASK)
-
-/* Register to control remote fault */
-#define SGMSYS_SGMII_MODE		0x20
-#define SGMII_IF_MODE_SGMII		BIT(0)
-#define SGMII_SPEED_DUPLEX_AN		BIT(1)
-#define SGMII_SPEED_MASK		GENMASK(3, 2)
-#define SGMII_SPEED_10			FIELD_PREP(SGMII_SPEED_MASK, 0)
-#define SGMII_SPEED_100			FIELD_PREP(SGMII_SPEED_MASK, 1)
-#define SGMII_SPEED_1000		FIELD_PREP(SGMII_SPEED_MASK, 2)
-#define SGMII_DUPLEX_HALF		BIT(4)
-#define SGMII_IF_MODE_BIT5		BIT(5)
-#define SGMII_REMOTE_FAULT_DIS		BIT(8)
-#define SGMII_CODE_SYNC_SET_VAL		BIT(9)
-#define SGMII_CODE_SYNC_SET_EN		BIT(10)
-#define SGMII_SEND_AN_ERROR_EN		BIT(11)
-#define SGMII_IF_MODE_MASK		GENMASK(5, 1)
-
-/* Register to reset SGMII design */
-#define SGMII_RESERVED_0	0x34
-#define SGMII_SW_RESET		BIT(0)
-
-/* Register to set SGMII speed, ANA RG_ Control Signals III*/
-#define SGMSYS_ANA_RG_CS3	0x2028
-#define RG_PHY_SPEED_MASK	(BIT(2) | BIT(3))
-#define RG_PHY_SPEED_1_25G	0x0
-#define RG_PHY_SPEED_3_125G	BIT(2)
-
-/* Register to power up QPHY */
-#define SGMSYS_QPHY_PWR_STATE_CTRL 0xe8
-#define	SGMII_PHYA_PWD		BIT(4)
-
 /* Infrasys subsystem config registers */
 #define INFRA_MISC2            0x70c
 #define CO_QPHY_SEL            BIT(0)
 #define GEPHY_MAC_SEL          BIT(1)
 
+/* Top misc registers */
+#define USB_PHY_SWITCH_REG	0x218
+#define QPHY_SEL_MASK		GENMASK(1, 0)
+#define SGMII_QPHY_SEL		0x2
+
 /* MT7628/88 specific stuff */
 #define MT7628_PDMA_OFFSET	0x0800
 #define MT7628_SDM_OFFSET	0x0c00
@@ -741,6 +700,17 @@ enum mtk_clks_map {
 				 BIT(MTK_CLK_SGMII2_CDR_FB) | \
 				 BIT(MTK_CLK_SGMII_CK) | \
 				 BIT(MTK_CLK_ETH2PLL) | BIT(MTK_CLK_SGMIITOP))
+#define MT7981_CLKS_BITMAP	(BIT(MTK_CLK_FE) | BIT(MTK_CLK_GP2) | BIT(MTK_CLK_GP1) | \
+				 BIT(MTK_CLK_WOCPU0) | \
+				 BIT(MTK_CLK_SGMII_TX_250M) | \
+				 BIT(MTK_CLK_SGMII_RX_250M) | \
+				 BIT(MTK_CLK_SGMII_CDR_REF) | \
+				 BIT(MTK_CLK_SGMII_CDR_FB) | \
+				 BIT(MTK_CLK_SGMII2_TX_250M) | \
+				 BIT(MTK_CLK_SGMII2_RX_250M) | \
+				 BIT(MTK_CLK_SGMII2_CDR_REF) | \
+				 BIT(MTK_CLK_SGMII2_CDR_FB) | \
+				 BIT(MTK_CLK_SGMII_CK))
 #define MT7986_CLKS_BITMAP	(BIT(MTK_CLK_FE) | BIT(MTK_CLK_GP2) | BIT(MTK_CLK_GP1) | \
 				 BIT(MTK_CLK_WOCPU1) | BIT(MTK_CLK_WOCPU0) | \
 				 BIT(MTK_CLK_SGMII_TX_250M) | \
@@ -854,6 +824,7 @@ enum mkt_eth_capabilities {
 	MTK_NETSYS_V2_BIT,
 	MTK_SOC_MT7628_BIT,
 	MTK_RSTCTRL_PPE1_BIT,
+	MTK_U3_COPHY_V2_BIT,
 
 	/* MUX BITS*/
 	MTK_ETH_MUX_GDM1_TO_GMAC1_ESW_BIT,
@@ -888,6 +859,7 @@ enum mkt_eth_capabilities {
 #define MTK_NETSYS_V2		BIT(MTK_NETSYS_V2_BIT)
 #define MTK_SOC_MT7628		BIT(MTK_SOC_MT7628_BIT)
 #define MTK_RSTCTRL_PPE1	BIT(MTK_RSTCTRL_PPE1_BIT)
+#define MTK_U3_COPHY_V2		BIT(MTK_U3_COPHY_V2_BIT)
 
 #define MTK_ETH_MUX_GDM1_TO_GMAC1_ESW		\
 	BIT(MTK_ETH_MUX_GDM1_TO_GMAC1_ESW_BIT)
@@ -960,6 +932,11 @@ enum mkt_eth_capabilities {
 		      MTK_MUX_U3_GMAC2_TO_QPHY | \
 		      MTK_MUX_GMAC12_TO_GEPHY_SGMII | MTK_QDMA)
 
+#define MT7981_CAPS  (MTK_GMAC1_SGMII | MTK_GMAC2_SGMII | MTK_GMAC2_GEPHY | \
+		      MTK_MUX_GMAC12_TO_GEPHY_SGMII | MTK_QDMA | \
+		      MTK_MUX_U3_GMAC2_TO_QPHY | MTK_U3_COPHY_V2 | \
+		      MTK_NETSYS_V2 | MTK_RSTCTRL_PPE1)
+
 #define MT7986_CAPS  (MTK_GMAC1_SGMII | MTK_GMAC2_SGMII | \
 		      MTK_MUX_GMAC12_TO_GEPHY_SGMII | MTK_QDMA | \
 		      MTK_NETSYS_V2 | MTK_RSTCTRL_PPE1)
@@ -1034,6 +1011,8 @@ struct mtk_reg_map {
  *				the extra setup for those pins used by GMAC.
  * @hash_offset			Flow table hash offset.
  * @foe_entry_size		Foe table entry size.
+ * @has_accounting		Bool indicating support for accounting of
+ *				offloaded flows.
  * @txd_size			Tx DMA descriptor size.
  * @rxd_size			Rx DMA descriptor size.
  * @rx_irq_done_mask		Rx irq done register mask.
@@ -1051,6 +1030,7 @@ struct mtk_soc_data {
 	u8		hash_offset;
 	u16		foe_entry_size;
 	netdev_features_t hw_features;
+	bool		has_accounting;
 	struct {
 		u32	txd_size;
 		u32	rxd_size;
@@ -1066,29 +1046,6 @@ struct mtk_soc_data {
 /* currently no SoC has more than 2 macs */
 #define MTK_MAX_DEVS			2
 
-/* struct mtk_pcs -    This structure holds each sgmii regmap and associated
- *                     data
- * @regmap:            The register map pointing at the range used to setup
- *                     SGMII modes
- * @ana_rgc3:          The offset refers to register ANA_RGC3 related to regmap
- * @interface:         Currently configured interface mode
- * @pcs:               Phylink PCS structure
- */
-struct mtk_pcs {
-	struct regmap	*regmap;
-	u32             ana_rgc3;
-	phy_interface_t	interface;
-	struct phylink_pcs pcs;
-};
-
-/* struct mtk_sgmii -  This is the structure holding sgmii regmap and its
- *                     characteristics
- * @pcs                Array of individual PCS structures
- */
-struct mtk_sgmii {
-	struct mtk_pcs	pcs[MTK_MAX_DEVS];
-};
-
 /* struct mtk_eth -	This is the main datasructure for holding the state
  *			of the driver
  * @dev:		The device pointer
@@ -1108,6 +1065,7 @@ struct mtk_sgmii {
  *			MII modes
  * @infra:              The register map pointing at the range used to setup
  *                      SGMII and GePHY path
+ * @sgmii_pcs:		Pointers to mtk-pcs-lynxi phylink_pcs instances
  * @pctl:		The register map pointing at the range used to setup
  *			GMAC port drive/slew values
  * @dma_refcnt:		track how many netdevs are using the DMA engine
@@ -1148,8 +1106,8 @@ struct mtk_eth {
 	u32				msg_enable;
 	unsigned long			sysclk;
 	struct regmap			*ethsys;
-	struct regmap                   *infra;
-	struct mtk_sgmii                *sgmii;
+	struct regmap			*infra;
+	struct phylink_pcs		*sgmii_pcs[MTK_MAX_DEVS];
 	struct regmap			*pctl;
 	bool				hwlro;
 	refcount_t			dma_refcnt;
@@ -1311,10 +1269,6 @@ void mtk_stats_update_mac(struct mtk_mac *mac);
 void mtk_w32(struct mtk_eth *eth, u32 val, unsigned reg);
 u32 mtk_r32(struct mtk_eth *eth, unsigned reg);
 
-struct phylink_pcs *mtk_sgmii_select_pcs(struct mtk_sgmii *ss, int id);
-int mtk_sgmii_init(struct mtk_sgmii *ss, struct device_node *np,
-		   u32 ana_rgc3);
-
 int mtk_gmac_sgmii_path_setup(struct mtk_eth *eth, int mac_id);
 int mtk_gmac_gephy_path_setup(struct mtk_eth *eth, int mac_id);
 int mtk_gmac_rgmii_path_setup(struct mtk_eth *eth, int mac_id);
@@ -1322,6 +1276,9 @@ int mtk_gmac_rgmii_path_setup(struct mtk_eth *eth, int mac_id);
 int mtk_eth_offload_init(struct mtk_eth *eth);
 int mtk_eth_setup_tc(struct net_device *dev, enum tc_setup_type type,
 		     void *type_data);
+int mtk_flow_offload_cmd(struct mtk_eth *eth, struct flow_cls_offload *cls,
+			 int ppe_index);
+void mtk_flow_offload_cleanup(struct mtk_eth *eth, struct list_head *list);
 void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev);
 
 
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c
index fd07d6e14273..9129821f3ab8 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe.c
+++ b/drivers/net/ethernet/mediatek/mtk_ppe.c
@@ -75,6 +75,48 @@ static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)
 	return ret;
 }
 
+static int mtk_ppe_mib_wait_busy(struct mtk_ppe *ppe)
+{
+	int ret;
+	u32 val;
+
+	ret = readl_poll_timeout(ppe->base + MTK_PPE_MIB_SER_CR, val,
+				 !(val & MTK_PPE_MIB_SER_CR_ST),
+				 20, MTK_PPE_WAIT_TIMEOUT_US);
+
+	if (ret)
+		dev_err(ppe->dev, "MIB table busy");
+
+	return ret;
+}
+
+static int mtk_mib_entry_read(struct mtk_ppe *ppe, u16 index, u64 *bytes, u64 *packets)
+{
+	u32 byte_cnt_low, byte_cnt_high, pkt_cnt_low, pkt_cnt_high;
+	u32 val, cnt_r0, cnt_r1, cnt_r2;
+	int ret;
+
+	val = FIELD_PREP(MTK_PPE_MIB_SER_CR_ADDR, index) | MTK_PPE_MIB_SER_CR_ST;
+	ppe_w32(ppe, MTK_PPE_MIB_SER_CR, val);
+
+	ret = mtk_ppe_mib_wait_busy(ppe);
+	if (ret)
+		return ret;
+
+	cnt_r0 = readl(ppe->base + MTK_PPE_MIB_SER_R0);
+	cnt_r1 = readl(ppe->base + MTK_PPE_MIB_SER_R1);
+	cnt_r2 = readl(ppe->base + MTK_PPE_MIB_SER_R2);
+
+	byte_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW, cnt_r0);
+	byte_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH, cnt_r1);
+	pkt_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R1_PKT_CNT_LOW, cnt_r1);
+	pkt_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH, cnt_r2);
+	*bytes = ((u64)byte_cnt_high << 32) | byte_cnt_low;
+	*packets = (pkt_cnt_high << 16) | pkt_cnt_low;
+
+	return 0;
+}
+
 static void mtk_ppe_cache_clear(struct mtk_ppe *ppe)
 {
 	ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
@@ -460,6 +502,14 @@ __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 		hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID);
 		dma_wmb();
 		mtk_ppe_cache_clear(ppe);
+
+		if (ppe->accounting) {
+			struct mtk_foe_accounting *acct;
+
+			acct = ppe->acct_table + entry->hash * sizeof(*acct);
+			acct->packets = 0;
+			acct->bytes = 0;
+		}
 	}
 	entry->hash = 0xffff;
 
@@ -551,6 +601,7 @@ __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
 	struct mtk_eth *eth = ppe->eth;
 	u16 timestamp = mtk_eth_timestamp(eth);
 	struct mtk_foe_entry *hwe;
+	u32 val;
 
 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
 		entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2;
@@ -567,6 +618,14 @@ __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
 	wmb();
 	hwe->ib1 = entry->ib1;
 
+	if (ppe->accounting) {
+		if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
+			val = MTK_FOE_IB2_MIB_CNT_V2;
+		else
+			val = MTK_FOE_IB2_MIB_CNT;
+		*mtk_foe_entry_ib2(eth, hwe) |= val;
+	}
+
 	dma_wmb();
 
 	mtk_ppe_cache_clear(ppe);
@@ -582,10 +641,20 @@ void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 static int
 mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 {
+	struct mtk_flow_entry *prev;
+
 	entry->type = MTK_FLOW_TYPE_L2;
 
-	return rhashtable_insert_fast(&ppe->l2_flows, &entry->l2_node,
-				      mtk_flow_l2_ht_params);
+	prev = rhashtable_lookup_get_insert_fast(&ppe->l2_flows, &entry->l2_node,
+						 mtk_flow_l2_ht_params);
+	if (likely(!prev))
+		return 0;
+
+	if (IS_ERR(prev))
+		return PTR_ERR(prev);
+
+	return rhashtable_replace_fast(&ppe->l2_flows, &prev->l2_node,
+				       &entry->l2_node, mtk_flow_l2_ht_params);
 }
 
 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
@@ -760,11 +829,39 @@ int mtk_ppe_prepare_reset(struct mtk_ppe *ppe)
 	return mtk_ppe_wait_busy(ppe);
 }
 
-struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
-			     int version, int index)
+struct mtk_foe_accounting *mtk_foe_entry_get_mib(struct mtk_ppe *ppe, u32 index,
+						 struct mtk_foe_accounting *diff)
+{
+	struct mtk_foe_accounting *acct;
+	int size = sizeof(struct mtk_foe_accounting);
+	u64 bytes, packets;
+
+	if (!ppe->accounting)
+		return NULL;
+
+	if (mtk_mib_entry_read(ppe, index, &bytes, &packets))
+		return NULL;
+
+	acct = ppe->acct_table + index * size;
+
+	acct->bytes += bytes;
+	acct->packets += packets;
+
+	if (diff) {
+		diff->bytes = bytes;
+		diff->packets = packets;
+	}
+
+	return acct;
+}
+
+struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index)
 {
+	bool accounting = eth->soc->has_accounting;
 	const struct mtk_soc_data *soc = eth->soc;
+	struct mtk_foe_accounting *acct;
 	struct device *dev = eth->dev;
+	struct mtk_mib_entry *mib;
 	struct mtk_ppe *ppe;
 	u32 foe_flow_size;
 	void *foe;
@@ -781,7 +878,8 @@ struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
 	ppe->base = base;
 	ppe->eth = eth;
 	ppe->dev = dev;
-	ppe->version = version;
+	ppe->version = eth->soc->offload_version;
+	ppe->accounting = accounting;
 
 	foe = dmam_alloc_coherent(ppe->dev,
 				  MTK_PPE_ENTRIES * soc->foe_entry_size,
@@ -797,6 +895,23 @@ struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
 	if (!ppe->foe_flow)
 		goto err_free_l2_flows;
 
+	if (accounting) {
+		mib = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*mib),
+					  &ppe->mib_phys, GFP_KERNEL);
+		if (!mib)
+			return NULL;
+
+		ppe->mib_table = mib;
+
+		acct = devm_kzalloc(dev, MTK_PPE_ENTRIES * sizeof(*acct),
+				    GFP_KERNEL);
+
+		if (!acct)
+			return NULL;
+
+		ppe->acct_table = acct;
+	}
+
 	mtk_ppe_debugfs_init(ppe, index);
 
 	return ppe;
@@ -926,6 +1041,16 @@ void mtk_ppe_start(struct mtk_ppe *ppe)
 		ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777);
 		ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f);
 	}
+
+	if (ppe->accounting && ppe->mib_phys) {
+		ppe_w32(ppe, MTK_PPE_MIB_TB_BASE, ppe->mib_phys);
+		ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_EN,
+			MTK_PPE_MIB_CFG_EN);
+		ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_RD_CLR,
+			MTK_PPE_MIB_CFG_RD_CLR);
+		ppe_m32(ppe, MTK_PPE_MIB_CACHE_CTL, MTK_PPE_MIB_CACHE_CTL_EN,
+			MTK_PPE_MIB_CFG_RD_CLR);
+	}
 }
 
 int mtk_ppe_stop(struct mtk_ppe *ppe)
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.h b/drivers/net/ethernet/mediatek/mtk_ppe.h
index 5e8bc48252b1..e51de31a52ec 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe.h
+++ b/drivers/net/ethernet/mediatek/mtk_ppe.h
@@ -55,8 +55,10 @@ enum {
 #define MTK_FOE_IB2_PSE_QOS		BIT(4)
 #define MTK_FOE_IB2_DEST_PORT		GENMASK(7, 5)
 #define MTK_FOE_IB2_MULTICAST		BIT(8)
+#define MTK_FOE_IB2_MIB_CNT		BIT(10)
 
 #define MTK_FOE_IB2_WDMA_QID2		GENMASK(13, 12)
+#define MTK_FOE_IB2_MIB_CNT_V2		BIT(15)
 #define MTK_FOE_IB2_WDMA_DEVIDX		BIT(16)
 #define MTK_FOE_IB2_WDMA_WINFO		BIT(17)
 
@@ -285,16 +287,34 @@ struct mtk_flow_entry {
 	unsigned long cookie;
 };
 
+struct mtk_mib_entry {
+	u32	byt_cnt_l;
+	u16	byt_cnt_h;
+	u32	pkt_cnt_l;
+	u8	pkt_cnt_h;
+	u8	_rsv0;
+	u32	_rsv1;
+} __packed;
+
+struct mtk_foe_accounting {
+	u64	bytes;
+	u64	packets;
+};
+
 struct mtk_ppe {
 	struct mtk_eth *eth;
 	struct device *dev;
 	void __iomem *base;
 	int version;
 	char dirname[5];
+	bool accounting;
 
 	void *foe_table;
 	dma_addr_t foe_phys;
 
+	struct mtk_mib_entry *mib_table;
+	dma_addr_t mib_phys;
+
 	u16 foe_check_time[MTK_PPE_ENTRIES];
 	struct hlist_head *foe_flow;
 
@@ -303,8 +323,8 @@ struct mtk_ppe {
 	void *acct_table;
 };
 
-struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
-			     int version, int index);
+struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index);
+
 void mtk_ppe_deinit(struct mtk_eth *eth);
 void mtk_ppe_start(struct mtk_ppe *ppe);
 int mtk_ppe_stop(struct mtk_ppe *ppe);
@@ -359,5 +379,7 @@ int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
 void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
 int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
 int mtk_ppe_debugfs_init(struct mtk_ppe *ppe, int index);
+struct mtk_foe_accounting *mtk_foe_entry_get_mib(struct mtk_ppe *ppe, u32 index,
+						 struct mtk_foe_accounting *diff);
 
 #endif
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
index 391b071bcff3..316fe2e70fea 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
+++ b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
@@ -47,7 +47,7 @@ static const char *mtk_foe_pkt_type_str(int type)
 static void
 mtk_print_addr(struct seq_file *m, u32 *addr, bool ipv6)
 {
-	u32 n_addr[4];
+	__be32 n_addr[4];
 	int i;
 
 	if (!ipv6) {
@@ -82,6 +82,7 @@ mtk_ppe_debugfs_foe_show(struct seq_file *m, void *private, bool bind)
 		struct mtk_foe_entry *entry = mtk_foe_get_entry(ppe, i);
 		struct mtk_foe_mac_info *l2;
 		struct mtk_flow_addr_info ai = {};
+		struct mtk_foe_accounting *acct;
 		unsigned char h_source[ETH_ALEN];
 		unsigned char h_dest[ETH_ALEN];
 		int type, state;
@@ -95,6 +96,8 @@ mtk_ppe_debugfs_foe_show(struct seq_file *m, void *private, bool bind)
 		if (bind && state != MTK_FOE_STATE_BIND)
 			continue;
 
+		acct = mtk_foe_entry_get_mib(ppe, i, NULL);
+
 		type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
 		seq_printf(m, "%05x %s %7s", i,
 			   mtk_foe_entry_state_str(state),
@@ -153,9 +156,11 @@ mtk_ppe_debugfs_foe_show(struct seq_file *m, void *private, bool bind)
 		*((__be16 *)&h_dest[4]) = htons(l2->dest_mac_lo);
 
 		seq_printf(m, " eth=%pM->%pM etype=%04x"
-			      " vlan=%d,%d ib1=%08x ib2=%08x\n",
+			      " vlan=%d,%d ib1=%08x ib2=%08x"
+			      " packets=%llu bytes=%llu\n",
 			   h_source, h_dest, ntohs(l2->etype),
-			   l2->vlan1, l2->vlan2, entry->ib1, ib2);
+			   l2->vlan1, l2->vlan2, entry->ib1, ib2,
+			   acct ? acct->packets : 0, acct ? acct->bytes : 0);
 	}
 
 	return 0;
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
index 161751bb36c9..02eebff02d45 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
+++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
@@ -235,7 +235,8 @@ out:
 }
 
 static int
-mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
+mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f,
+			 int ppe_index)
 {
 	struct flow_rule *rule = flow_cls_offload_flow_rule(f);
 	struct flow_action_entry *act;
@@ -452,6 +453,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
 	entry->cookie = f->cookie;
 	memcpy(&entry->data, &foe, sizeof(entry->data));
 	entry->wed_index = wed_index;
+	entry->ppe_index = ppe_index;
 
 	err = mtk_foe_entry_commit(eth->ppe[entry->ppe_index], entry);
 	if (err < 0)
@@ -497,6 +499,7 @@ static int
 mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f)
 {
 	struct mtk_flow_entry *entry;
+	struct mtk_foe_accounting diff;
 	u32 idle;
 
 	entry = rhashtable_lookup(&eth->flow_table, &f->cookie,
@@ -507,30 +510,27 @@ mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f)
 	idle = mtk_foe_entry_idle_time(eth->ppe[entry->ppe_index], entry);
 	f->stats.lastused = jiffies - idle * HZ;
 
+	if (entry->hash != 0xFFFF &&
+	    mtk_foe_entry_get_mib(eth->ppe[entry->ppe_index], entry->hash,
+				  &diff)) {
+		f->stats.pkts += diff.packets;
+		f->stats.bytes += diff.bytes;
+	}
+
 	return 0;
 }
 
 static DEFINE_MUTEX(mtk_flow_offload_mutex);
 
-static int
-mtk_eth_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
+int mtk_flow_offload_cmd(struct mtk_eth *eth, struct flow_cls_offload *cls,
+			 int ppe_index)
 {
-	struct flow_cls_offload *cls = type_data;
-	struct net_device *dev = cb_priv;
-	struct mtk_mac *mac = netdev_priv(dev);
-	struct mtk_eth *eth = mac->hw;
 	int err;
 
-	if (!tc_can_offload(dev))
-		return -EOPNOTSUPP;
-
-	if (type != TC_SETUP_CLSFLOWER)
-		return -EOPNOTSUPP;
-
 	mutex_lock(&mtk_flow_offload_mutex);
 	switch (cls->command) {
 	case FLOW_CLS_REPLACE:
-		err = mtk_flow_offload_replace(eth, cls);
+		err = mtk_flow_offload_replace(eth, cls, ppe_index);
 		break;
 	case FLOW_CLS_DESTROY:
 		err = mtk_flow_offload_destroy(eth, cls);
@@ -548,6 +548,26 @@ mtk_eth_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_pri
 }
 
 static int
+mtk_eth_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
+{
+	struct flow_cls_offload *cls = type_data;
+	struct net_device *dev = cb_priv;
+	struct mtk_mac *mac;
+	struct mtk_eth *eth;
+
+	mac = netdev_priv(dev);
+	eth = mac->hw;
+
+	if (!tc_can_offload(dev))
+		return -EOPNOTSUPP;
+
+	if (type != TC_SETUP_CLSFLOWER)
+		return -EOPNOTSUPP;
+
+	return mtk_flow_offload_cmd(eth, cls, 0);
+}
+
+static int
 mtk_eth_setup_tc_block(struct net_device *dev, struct flow_block_offload *f)
 {
 	struct mtk_mac *mac = netdev_priv(dev);
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
index 0fdb983b0a88..a2e61b3eb006 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
+++ b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
@@ -149,6 +149,20 @@ enum {
 
 #define MTK_PPE_MIB_TB_BASE			0x338
 
+#define MTK_PPE_MIB_SER_CR			0x33C
+#define MTK_PPE_MIB_SER_CR_ST			BIT(16)
+#define MTK_PPE_MIB_SER_CR_ADDR			GENMASK(13, 0)
+
+#define MTK_PPE_MIB_SER_R0			0x340
+#define MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW		GENMASK(31, 0)
+
+#define MTK_PPE_MIB_SER_R1			0x344
+#define MTK_PPE_MIB_SER_R1_PKT_CNT_LOW		GENMASK(31, 16)
+#define MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH	GENMASK(15, 0)
+
+#define MTK_PPE_MIB_SER_R2			0x348
+#define MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH		GENMASK(23, 0)
+
 #define MTK_PPE_MIB_CACHE_CTL			0x350
 #define MTK_PPE_MIB_CACHE_CTL_EN		BIT(0)
 #define MTK_PPE_MIB_CACHE_CTL_FLUSH		BIT(2)
diff --git a/drivers/net/ethernet/mediatek/mtk_sgmii.c b/drivers/net/ethernet/mediatek/mtk_sgmii.c
deleted file mode 100644
index 83976dc86887..000000000000
--- a/drivers/net/ethernet/mediatek/mtk_sgmii.c
+++ /dev/null
@@ -1,207 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-// Copyright (c) 2018-2019 MediaTek Inc.
-
-/* A library for MediaTek SGMII circuit
- *
- * Author: Sean Wang <sean.wang@mediatek.com>
- *
- */
-
-#include <linux/mfd/syscon.h>
-#include <linux/of.h>
-#include <linux/phylink.h>
-#include <linux/regmap.h>
-
-#include "mtk_eth_soc.h"
-
-static struct mtk_pcs *pcs_to_mtk_pcs(struct phylink_pcs *pcs)
-{
-	return container_of(pcs, struct mtk_pcs, pcs);
-}
-
-static void mtk_pcs_get_state(struct phylink_pcs *pcs,
-			      struct phylink_link_state *state)
-{
-	struct mtk_pcs *mpcs = pcs_to_mtk_pcs(pcs);
-	unsigned int bm, adv;
-
-	/* Read the BMSR and LPA */
-	regmap_read(mpcs->regmap, SGMSYS_PCS_CONTROL_1, &bm);
-	regmap_read(mpcs->regmap, SGMSYS_PCS_ADVERTISE, &adv);
-
-	phylink_mii_c22_pcs_decode_state(state, FIELD_GET(SGMII_BMSR, bm),
-					 FIELD_GET(SGMII_LPA, adv));
-}
-
-static int mtk_pcs_config(struct phylink_pcs *pcs, unsigned int mode,
-			  phy_interface_t interface,
-			  const unsigned long *advertising,
-			  bool permit_pause_to_mac)
-{
-	bool mode_changed = false, changed, use_an;
-	struct mtk_pcs *mpcs = pcs_to_mtk_pcs(pcs);
-	unsigned int rgc3, sgm_mode, bmcr;
-	int advertise, link_timer;
-
-	advertise = phylink_mii_c22_pcs_encode_advertisement(interface,
-							     advertising);
-	if (advertise < 0)
-		return advertise;
-
-	/* Clearing IF_MODE_BIT0 switches the PCS to BASE-X mode, and
-	 * we assume that fixes it's speed at bitrate = line rate (in
-	 * other words, 1000Mbps or 2500Mbps).
-	 */
-	if (interface == PHY_INTERFACE_MODE_SGMII) {
-		sgm_mode = SGMII_IF_MODE_SGMII;
-		if (phylink_autoneg_inband(mode)) {
-			sgm_mode |= SGMII_REMOTE_FAULT_DIS |
-				    SGMII_SPEED_DUPLEX_AN;
-			use_an = true;
-		} else {
-			use_an = false;
-		}
-	} else if (phylink_autoneg_inband(mode)) {
-		/* 1000base-X or 2500base-X autoneg */
-		sgm_mode = SGMII_REMOTE_FAULT_DIS;
-		use_an = linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
-					   advertising);
-	} else {
-		/* 1000base-X or 2500base-X without autoneg */
-		sgm_mode = 0;
-		use_an = false;
-	}
-
-	if (use_an) {
-		bmcr = SGMII_AN_ENABLE;
-	} else {
-		bmcr = 0;
-	}
-
-	if (mpcs->interface != interface) {
-		link_timer = phylink_get_link_timer_ns(interface);
-		if (link_timer < 0)
-			return link_timer;
-
-		/* PHYA power down */
-		regmap_update_bits(mpcs->regmap, SGMSYS_QPHY_PWR_STATE_CTRL,
-				   SGMII_PHYA_PWD, SGMII_PHYA_PWD);
-
-		/* Reset SGMII PCS state */
-		regmap_update_bits(mpcs->regmap, SGMII_RESERVED_0,
-				   SGMII_SW_RESET, SGMII_SW_RESET);
-
-		if (interface == PHY_INTERFACE_MODE_2500BASEX)
-			rgc3 = RG_PHY_SPEED_3_125G;
-		else
-			rgc3 = 0;
-
-		/* Configure the underlying interface speed */
-		regmap_update_bits(mpcs->regmap, mpcs->ana_rgc3,
-				   RG_PHY_SPEED_3_125G, rgc3);
-
-		/* Setup the link timer */
-		regmap_write(mpcs->regmap, SGMSYS_PCS_LINK_TIMER, link_timer / 2 / 8);
-
-		mpcs->interface = interface;
-		mode_changed = true;
-	}
-
-	/* Update the advertisement, noting whether it has changed */
-	regmap_update_bits_check(mpcs->regmap, SGMSYS_PCS_ADVERTISE,
-				 SGMII_ADVERTISE, advertise, &changed);
-
-	/* Update the sgmsys mode register */
-	regmap_update_bits(mpcs->regmap, SGMSYS_SGMII_MODE,
-			   SGMII_REMOTE_FAULT_DIS | SGMII_SPEED_DUPLEX_AN |
-			   SGMII_IF_MODE_SGMII, sgm_mode);
-
-	/* Update the BMCR */
-	regmap_update_bits(mpcs->regmap, SGMSYS_PCS_CONTROL_1,
-			   SGMII_AN_ENABLE, bmcr);
-
-	/* Release PHYA power down state
-	 * Only removing bit SGMII_PHYA_PWD isn't enough.
-	 * There are cases when the SGMII_PHYA_PWD register contains 0x9 which
-	 * prevents SGMII from working. The SGMII still shows link but no traffic
-	 * can flow. Writing 0x0 to the PHYA_PWD register fix the issue. 0x0 was
-	 * taken from a good working state of the SGMII interface.
-	 * Unknown how much the QPHY needs but it is racy without a sleep.
-	 * Tested on mt7622 & mt7986.
-	 */
-	usleep_range(50, 100);
-	regmap_write(mpcs->regmap, SGMSYS_QPHY_PWR_STATE_CTRL, 0);
-
-	return changed || mode_changed;
-}
-
-static void mtk_pcs_restart_an(struct phylink_pcs *pcs)
-{
-	struct mtk_pcs *mpcs = pcs_to_mtk_pcs(pcs);
-
-	regmap_update_bits(mpcs->regmap, SGMSYS_PCS_CONTROL_1,
-			   SGMII_AN_RESTART, SGMII_AN_RESTART);
-}
-
-static void mtk_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode,
-			    phy_interface_t interface, int speed, int duplex)
-{
-	struct mtk_pcs *mpcs = pcs_to_mtk_pcs(pcs);
-	unsigned int sgm_mode;
-
-	if (!phylink_autoneg_inband(mode)) {
-		/* Force the speed and duplex setting */
-		if (speed == SPEED_10)
-			sgm_mode = SGMII_SPEED_10;
-		else if (speed == SPEED_100)
-			sgm_mode = SGMII_SPEED_100;
-		else
-			sgm_mode = SGMII_SPEED_1000;
-
-		if (duplex != DUPLEX_FULL)
-			sgm_mode |= SGMII_DUPLEX_HALF;
-
-		regmap_update_bits(mpcs->regmap, SGMSYS_SGMII_MODE,
-				   SGMII_DUPLEX_HALF | SGMII_SPEED_MASK,
-				   sgm_mode);
-	}
-}
-
-static const struct phylink_pcs_ops mtk_pcs_ops = {
-	.pcs_get_state = mtk_pcs_get_state,
-	.pcs_config = mtk_pcs_config,
-	.pcs_an_restart = mtk_pcs_restart_an,
-	.pcs_link_up = mtk_pcs_link_up,
-};
-
-int mtk_sgmii_init(struct mtk_sgmii *ss, struct device_node *r, u32 ana_rgc3)
-{
-	struct device_node *np;
-	int i;
-
-	for (i = 0; i < MTK_MAX_DEVS; i++) {
-		np = of_parse_phandle(r, "mediatek,sgmiisys", i);
-		if (!np)
-			break;
-
-		ss->pcs[i].ana_rgc3 = ana_rgc3;
-		ss->pcs[i].regmap = syscon_node_to_regmap(np);
-		of_node_put(np);
-		if (IS_ERR(ss->pcs[i].regmap))
-			return PTR_ERR(ss->pcs[i].regmap);
-
-		ss->pcs[i].pcs.ops = &mtk_pcs_ops;
-		ss->pcs[i].pcs.poll = true;
-		ss->pcs[i].interface = PHY_INTERFACE_MODE_NA;
-	}
-
-	return 0;
-}
-
-struct phylink_pcs *mtk_sgmii_select_pcs(struct mtk_sgmii *ss, int id)
-{
-	if (!ss->pcs[id].regmap)
-		return NULL;
-
-	return &ss->pcs[id].pcs;
-}
diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c
index 95d890870984..4c205afbd230 100644
--- a/drivers/net/ethernet/mediatek/mtk_wed.c
+++ b/drivers/net/ethernet/mediatek/mtk_wed.c
@@ -13,6 +13,8 @@
 #include <linux/mfd/syscon.h>
 #include <linux/debugfs.h>
 #include <linux/soc/mediatek/mtk_wed.h>
+#include <net/flow_offload.h>
+#include <net/pkt_cls.h>
 #include "mtk_eth_soc.h"
 #include "mtk_wed_regs.h"
 #include "mtk_wed.h"
@@ -41,6 +43,11 @@
 static struct mtk_wed_hw *hw_list[2];
 static DEFINE_MUTEX(hw_lock);
 
+struct mtk_wed_flow_block_priv {
+	struct mtk_wed_hw *hw;
+	struct net_device *dev;
+};
+
 static void
 wed_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val)
 {
@@ -1745,6 +1752,99 @@ out:
 	mutex_unlock(&hw_lock);
 }
 
+static int
+mtk_wed_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
+{
+	struct mtk_wed_flow_block_priv *priv = cb_priv;
+	struct flow_cls_offload *cls = type_data;
+	struct mtk_wed_hw *hw = priv->hw;
+
+	if (!tc_can_offload(priv->dev))
+		return -EOPNOTSUPP;
+
+	if (type != TC_SETUP_CLSFLOWER)
+		return -EOPNOTSUPP;
+
+	return mtk_flow_offload_cmd(hw->eth, cls, hw->index);
+}
+
+static int
+mtk_wed_setup_tc_block(struct mtk_wed_hw *hw, struct net_device *dev,
+		       struct flow_block_offload *f)
+{
+	struct mtk_wed_flow_block_priv *priv;
+	static LIST_HEAD(block_cb_list);
+	struct flow_block_cb *block_cb;
+	struct mtk_eth *eth = hw->eth;
+	flow_setup_cb_t *cb;
+
+	if (!eth->soc->offload_version)
+		return -EOPNOTSUPP;
+
+	if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
+		return -EOPNOTSUPP;
+
+	cb = mtk_wed_setup_tc_block_cb;
+	f->driver_block_list = &block_cb_list;
+
+	switch (f->command) {
+	case FLOW_BLOCK_BIND:
+		block_cb = flow_block_cb_lookup(f->block, cb, dev);
+		if (block_cb) {
+			flow_block_cb_incref(block_cb);
+			return 0;
+		}
+
+		priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+		if (!priv)
+			return -ENOMEM;
+
+		priv->hw = hw;
+		priv->dev = dev;
+		block_cb = flow_block_cb_alloc(cb, dev, priv, NULL);
+		if (IS_ERR(block_cb)) {
+			kfree(priv);
+			return PTR_ERR(block_cb);
+		}
+
+		flow_block_cb_incref(block_cb);
+		flow_block_cb_add(block_cb, f);
+		list_add_tail(&block_cb->driver_list, &block_cb_list);
+		return 0;
+	case FLOW_BLOCK_UNBIND:
+		block_cb = flow_block_cb_lookup(f->block, cb, dev);
+		if (!block_cb)
+			return -ENOENT;
+
+		if (!flow_block_cb_decref(block_cb)) {
+			flow_block_cb_remove(block_cb, f);
+			list_del(&block_cb->driver_list);
+			kfree(block_cb->cb_priv);
+		}
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int
+mtk_wed_setup_tc(struct mtk_wed_device *wed, struct net_device *dev,
+		 enum tc_setup_type type, void *type_data)
+{
+	struct mtk_wed_hw *hw = wed->hw;
+
+	if (hw->version < 2)
+		return -EOPNOTSUPP;
+
+	switch (type) {
+	case TC_SETUP_BLOCK:
+	case TC_SETUP_FT:
+		return mtk_wed_setup_tc_block(hw, dev, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,
 		    void __iomem *wdma, phys_addr_t wdma_phy,
 		    int index)
@@ -1764,6 +1864,7 @@ void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,
 		.irq_set_mask = mtk_wed_irq_set_mask,
 		.detach = mtk_wed_detach,
 		.ppe_check = mtk_wed_ppe_check,
+		.setup_tc = mtk_wed_setup_tc,
 	};
 	struct device_node *eth_np = eth->dev->of_node;
 	struct platform_device *pdev;
diff --git a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c
index 56f663439721..b244c02c5b51 100644
--- a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c
+++ b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c
@@ -252,8 +252,6 @@ void mtk_wed_hw_add_debugfs(struct mtk_wed_hw *hw)
 
 	snprintf(hw->dirname, sizeof(hw->dirname), "wed%d", hw->index);
 	dir = debugfs_create_dir(hw->dirname, NULL);
-	if (!dir)
-		return;
 
 	hw->debugfs_dir = dir;
 	debugfs_create_u32("regidx", 0600, dir, &hw->debugfs_reg);
diff --git a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c
index 6bad0d262f28..071ed3dea860 100644
--- a/drivers/net/ethernet/mediatek/mtk_wed_mcu.c
+++ b/drivers/net/ethernet/mediatek/mtk_wed_mcu.c
@@ -326,7 +326,11 @@ mtk_wed_mcu_load_firmware(struct mtk_wed_wo *wo)
 		wo->hw->index + 1);
 
 	/* load firmware */
-	fw_name = wo->hw->index ? MT7986_FIRMWARE_WO1 : MT7986_FIRMWARE_WO0;
+	if (of_device_is_compatible(wo->hw->node, "mediatek,mt7981-wed"))
+		fw_name = MT7981_FIRMWARE_WO;
+	else
+		fw_name = wo->hw->index ? MT7986_FIRMWARE_WO1 : MT7986_FIRMWARE_WO0;
+
 	ret = request_firmware(&fw, fw_name, wo->hw->dev);
 	if (ret)
 		return ret;
@@ -386,5 +390,6 @@ int mtk_wed_mcu_init(struct mtk_wed_wo *wo)
 				  100, MTK_FW_DL_TIMEOUT);
 }
 
+MODULE_FIRMWARE(MT7981_FIRMWARE_WO);
 MODULE_FIRMWARE(MT7986_FIRMWARE_WO0);
 MODULE_FIRMWARE(MT7986_FIRMWARE_WO1);
diff --git a/drivers/net/ethernet/mediatek/mtk_wed_wo.h b/drivers/net/ethernet/mediatek/mtk_wed_wo.h
index dbcf42ce9173..7a1a2a28f1ac 100644
--- a/drivers/net/ethernet/mediatek/mtk_wed_wo.h
+++ b/drivers/net/ethernet/mediatek/mtk_wed_wo.h
@@ -88,6 +88,7 @@ enum mtk_wed_dummy_cr_idx {
 	MTK_WED_DUMMY_CR_WO_STATUS,
 };
 
+#define MT7981_FIRMWARE_WO	"mediatek/mt7981_wo.bin"
 #define MT7986_FIRMWARE_WO0	"mediatek/mt7986_wo_0.bin"
 #define MT7986_FIRMWARE_WO1	"mediatek/mt7986_wo_1.bin"
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
index 2f79378fbf6e..65cb63f6c465 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
@@ -228,7 +228,9 @@ void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv,
 
 static inline bool mlx4_en_is_tx_ring_full(struct mlx4_en_tx_ring *ring)
 {
-	return ring->prod - ring->cons > ring->full_size;
+	u32 used = READ_ONCE(ring->prod) - READ_ONCE(ring->cons);
+
+	return used > ring->full_size;
 }
 
 static void mlx4_en_stamp_wqe(struct mlx4_en_priv *priv,
@@ -1083,7 +1085,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
 			op_own |= cpu_to_be32(MLX4_WQE_CTRL_IIP);
 	}
 
-	ring->prod += nr_txbb;
+	WRITE_ONCE(ring->prod, ring->prod + nr_txbb);
 
 	/* If we used a bounce buffer then copy descriptor back into place */
 	if (unlikely(bounce))
@@ -1214,7 +1216,7 @@ netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_ring *rx_ring,
 
 	rx_ring->xdp_tx++;
 
-	ring->prod += MLX4_EN_XDP_TX_NRTXBB;
+	WRITE_ONCE(ring->prod, ring->prod + MLX4_EN_XDP_TX_NRTXBB);
 
 	/* Ensure new descriptor hits memory
 	 * before setting ownership of this descriptor to HW
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
index 4ac4d883047b..321f801c1d7c 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
@@ -323,7 +323,7 @@ struct mlx4_en_tx_ring {
 
 struct mlx4_en_rx_desc {
 	/* actual number of entries depends on rx ring stride */
-	struct mlx4_wqe_data_seg data[0];
+	DECLARE_FLEX_ARRAY(struct mlx4_wqe_data_seg, data);
 };
 
 struct mlx4_en_rx_ring {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 8d4e25cc54ea..ddf1e352f51d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -16,7 +16,7 @@ mlx5_core-y :=	main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
 		transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \
 		fs_counters.o fs_ft_pool.o rl.o lag/debugfs.o lag/lag.o dev.o events.o wq.o lib/gid.o \
 		lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \
-		diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o \
+		diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o diag/reporter_vnic.o \
 		fw_reset.o qos.o lib/tout.o lib/aso.o
 
 #
@@ -69,14 +69,15 @@ mlx5_core-$(CONFIG_MLX5_TC_SAMPLE)   += en/tc/sample.o
 #
 mlx5_core-$(CONFIG_MLX5_ESWITCH)   += eswitch.o eswitch_offloads.o eswitch_offloads_termtbl.o \
 				      ecpf.o rdma.o esw/legacy.o \
-				      esw/debugfs.o esw/devlink_port.o esw/vporttbl.o esw/qos.o
+				      esw/devlink_port.o esw/vporttbl.o esw/qos.o
 
 mlx5_core-$(CONFIG_MLX5_ESWITCH)   += esw/acl/helper.o \
 				      esw/acl/egress_lgcy.o esw/acl/egress_ofld.o \
 				      esw/acl/ingress_lgcy.o esw/acl/ingress_ofld.o
 
-mlx5_core-$(CONFIG_MLX5_BRIDGE)    += esw/bridge.o en/rep/bridge.o
+mlx5_core-$(CONFIG_MLX5_BRIDGE)    += esw/bridge.o esw/bridge_mcast.o en/rep/bridge.o
 
+mlx5_core-$(CONFIG_THERMAL)        += thermal.o
 mlx5_core-$(CONFIG_MLX5_MPFS)      += lib/mpfs.o
 mlx5_core-$(CONFIG_VXLAN)          += lib/vxlan.o
 mlx5_core-$(CONFIG_PTP_1588_CLOCK) += lib/clock.o
@@ -111,8 +112,8 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o
 					steering/dr_ste_v2.o \
 					steering/dr_cmd.o steering/dr_fw.o \
 					steering/dr_action.o steering/fs_dr.o \
-					steering/dr_definer.o \
-					steering/dr_dbg.o lib/smfs.o
+					steering/dr_definer.o steering/dr_ptrn.o \
+					steering/dr_arg.o steering/dr_dbg.o lib/smfs.o
 #
 # SF device
 #
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
index b00e33ed05e9..d53de39539a8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
@@ -1802,7 +1802,7 @@ static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,
 	if (in_size <= 16)
 		goto cache_miss;
 
-	for (i = 0; i < MLX5_NUM_COMMAND_CACHES; i++) {
+	for (i = 0; i < dev->profile.num_cmd_caches; i++) {
 		ch = &cmd->cache[i];
 		if (in_size > ch->max_inbox_size)
 			continue;
@@ -2097,7 +2097,7 @@ static void destroy_msg_cache(struct mlx5_core_dev *dev)
 	struct mlx5_cmd_msg *n;
 	int i;
 
-	for (i = 0; i < MLX5_NUM_COMMAND_CACHES; i++) {
+	for (i = 0; i < dev->profile.num_cmd_caches; i++) {
 		ch = &dev->cmd.cache[i];
 		list_for_each_entry_safe(msg, n, &ch->head, list) {
 			list_del(&msg->list);
@@ -2127,7 +2127,7 @@ static void create_msg_cache(struct mlx5_core_dev *dev)
 	int k;
 
 	/* Initialize and fill the caches with initial entries */
-	for (k = 0; k < MLX5_NUM_COMMAND_CACHES; k++) {
+	for (k = 0; k < dev->profile.num_cmd_caches; k++) {
 		ch = &cmd->cache[k];
 		spin_lock_init(&ch->lock);
 		INIT_LIST_HEAD(&ch->head);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
index 445fe30c3d0b..1b33533b15de 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
@@ -35,6 +35,7 @@
 #include <linux/mlx5/mlx5_ifc_vdpa.h>
 #include <linux/mlx5/vport.h>
 #include "mlx5_core.h"
+#include "devlink.h"
 
 /* intf dev list mutex */
 static DEFINE_MUTEX(mlx5_intf_mutex);
@@ -59,9 +60,6 @@ bool mlx5_eth_supported(struct mlx5_core_dev *dev)
 	if (!IS_ENABLED(CONFIG_MLX5_CORE_EN))
 		return false;
 
-	if (mlx5_core_is_management_pf(dev))
-		return false;
-
 	if (MLX5_CAP_GEN(dev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
 		return false;
 
@@ -109,17 +107,6 @@ bool mlx5_eth_supported(struct mlx5_core_dev *dev)
 	return true;
 }
 
-static bool is_eth_enabled(struct mlx5_core_dev *dev)
-{
-	union devlink_param_value val;
-	int err;
-
-	err = devl_param_driverinit_value_get(priv_to_devlink(dev),
-					      DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
-					      &val);
-	return err ? false : val.vbool;
-}
-
 bool mlx5_vnet_supported(struct mlx5_core_dev *dev)
 {
 	if (!IS_ENABLED(CONFIG_MLX5_VDPA_NET))
@@ -201,9 +188,6 @@ bool mlx5_rdma_supported(struct mlx5_core_dev *dev)
 	if (!IS_ENABLED(CONFIG_MLX5_INFINIBAND))
 		return false;
 
-	if (mlx5_core_is_management_pf(dev))
-		return false;
-
 	if (dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_IB_ADEV)
 		return false;
 
@@ -251,7 +235,7 @@ static const struct mlx5_adev_device {
 					 .is_enabled = &is_ib_enabled },
 	[MLX5_INTERFACE_PROTOCOL_ETH] = { .suffix = "eth",
 					  .is_supported = &mlx5_eth_supported,
-					  .is_enabled = &is_eth_enabled },
+					  .is_enabled = &mlx5_core_is_eth_enabled },
 	[MLX5_INTERFACE_PROTOCOL_ETH_REP] = { .suffix = "eth-rep",
 					   .is_supported = &is_eth_rep_supported },
 	[MLX5_INTERFACE_PROTOCOL_IB_REP] = { .suffix = "rdma-rep",
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index c5d2fdcabd56..4b607785d694 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -202,7 +202,7 @@ static int mlx5_devlink_reload_up(struct devlink *devlink, enum devlink_reload_a
 			break;
 		/* On fw_activate action, also driver is reloaded and reinit performed */
 		*actions_performed |= BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT);
-		ret = mlx5_load_one_devl_locked(dev, false);
+		ret = mlx5_load_one_devl_locked(dev, true);
 		break;
 	default:
 		/* Unsupported action should not get to this function */
@@ -494,6 +494,61 @@ static int mlx5_devlink_eq_depth_validate(struct devlink *devlink, u32 id,
 	return (val.vu32 >= 64 && val.vu32 <= 4096) ? 0 : -EINVAL;
 }
 
+static int
+mlx5_devlink_hairpin_num_queues_validate(struct devlink *devlink, u32 id,
+					 union devlink_param_value val,
+					 struct netlink_ext_ack *extack)
+{
+	return val.vu32 ? 0 : -EINVAL;
+}
+
+static int
+mlx5_devlink_hairpin_queue_size_validate(struct devlink *devlink, u32 id,
+					 union devlink_param_value val,
+					 struct netlink_ext_ack *extack)
+{
+	struct mlx5_core_dev *dev = devlink_priv(devlink);
+	u32 val32 = val.vu32;
+
+	if (!is_power_of_2(val32)) {
+		NL_SET_ERR_MSG_MOD(extack, "Value is not power of two");
+		return -EINVAL;
+	}
+
+	if (val32 > BIT(MLX5_CAP_GEN(dev, log_max_hairpin_num_packets))) {
+		NL_SET_ERR_MSG_FMT_MOD(
+			extack, "Maximum hairpin queue size is %lu",
+			BIT(MLX5_CAP_GEN(dev, log_max_hairpin_num_packets)));
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void mlx5_devlink_hairpin_params_init_values(struct devlink *devlink)
+{
+	struct mlx5_core_dev *dev = devlink_priv(devlink);
+	union devlink_param_value value;
+	u32 link_speed = 0;
+	u64 link_speed64;
+
+	/* set hairpin pair per each 50Gbs share of the link */
+	mlx5_port_max_linkspeed(dev, &link_speed);
+	link_speed = max_t(u32, link_speed, 50000);
+	link_speed64 = link_speed;
+	do_div(link_speed64, 50000);
+
+	value.vu32 = link_speed64;
+	devl_param_driverinit_value_set(
+		devlink, MLX5_DEVLINK_PARAM_ID_HAIRPIN_NUM_QUEUES, value);
+
+	value.vu32 =
+		BIT(min_t(u32, 16 - MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(dev),
+			  MLX5_CAP_GEN(dev, log_max_hairpin_num_packets)));
+	devl_param_driverinit_value_set(
+		devlink, MLX5_DEVLINK_PARAM_ID_HAIRPIN_QUEUE_SIZE, value);
+}
+
 static const struct devlink_param mlx5_devlink_params[] = {
 	DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
 			      NULL, NULL, mlx5_devlink_enable_roce_validate),
@@ -547,6 +602,14 @@ static void mlx5_devlink_set_params_init_values(struct devlink *devlink)
 static const struct devlink_param mlx5_devlink_eth_params[] = {
 	DEVLINK_PARAM_GENERIC(ENABLE_ETH, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
 			      NULL, NULL, NULL),
+	DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_HAIRPIN_NUM_QUEUES,
+			     "hairpin_num_queues", DEVLINK_PARAM_TYPE_U32,
+			     BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL,
+			     mlx5_devlink_hairpin_num_queues_validate),
+	DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_HAIRPIN_QUEUE_SIZE,
+			     "hairpin_queue_size", DEVLINK_PARAM_TYPE_U32,
+			     BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL,
+			     mlx5_devlink_hairpin_queue_size_validate),
 };
 
 static int mlx5_devlink_eth_params_register(struct devlink *devlink)
@@ -567,6 +630,9 @@ static int mlx5_devlink_eth_params_register(struct devlink *devlink)
 	devl_param_driverinit_value_set(devlink,
 					DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
 					value);
+
+	mlx5_devlink_hairpin_params_init_values(devlink);
+
 	return 0;
 }
 
@@ -805,6 +871,11 @@ int mlx5_devlink_params_register(struct devlink *devlink)
 {
 	int err;
 
+	/* Here only the driver init params should be registered.
+	 * Runtime params should be registered by the code which
+	 * behaviour they configure.
+	 */
+
 	err = devl_params_register(devlink, mlx5_devlink_params,
 				   ARRAY_SIZE(mlx5_devlink_params));
 	if (err)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h
index 212b12424146..defba5bd91d9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h
@@ -12,6 +12,8 @@ enum mlx5_devlink_param_id {
 	MLX5_DEVLINK_PARAM_ID_ESW_LARGE_GROUP_NUM,
 	MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA,
 	MLX5_DEVLINK_PARAM_ID_ESW_MULTIPORT,
+	MLX5_DEVLINK_PARAM_ID_HAIRPIN_NUM_QUEUES,
+	MLX5_DEVLINK_PARAM_ID_HAIRPIN_QUEUE_SIZE,
 };
 
 struct mlx5_trap_ctx {
@@ -44,4 +46,15 @@ void mlx5_devlink_free(struct devlink *devlink);
 int mlx5_devlink_params_register(struct devlink *devlink);
 void mlx5_devlink_params_unregister(struct devlink *devlink);
 
+static inline bool mlx5_core_is_eth_enabled(struct mlx5_core_dev *dev)
+{
+	union devlink_param_value val;
+	int err;
+
+	err = devl_param_driverinit_value_get(priv_to_devlink(dev),
+					      DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
+					      &val);
+	return err ? false : val.vbool;
+}
+
 #endif /* __MLX5_DEVLINK_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/reporter_vnic.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/reporter_vnic.c
new file mode 100644
index 000000000000..9114661cd967
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/reporter_vnic.c
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. */
+
+#include "reporter_vnic.h"
+#include "devlink.h"
+
+#define VNIC_ENV_GET64(vnic_env_stats, c) \
+	MLX5_GET64(query_vnic_env_out, (vnic_env_stats)->query_vnic_env_out, \
+		 vport_env.c)
+
+struct mlx5_vnic_diag_stats {
+	__be64 query_vnic_env_out[MLX5_ST_SZ_QW(query_vnic_env_out)];
+};
+
+int mlx5_reporter_vnic_diagnose_counters(struct mlx5_core_dev *dev,
+					 struct devlink_fmsg *fmsg,
+					 u16 vport_num, bool other_vport)
+{
+	u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {};
+	struct mlx5_vnic_diag_stats vnic;
+	int err;
+
+	MLX5_SET(query_vnic_env_in, in, opcode, MLX5_CMD_OP_QUERY_VNIC_ENV);
+	MLX5_SET(query_vnic_env_in, in, vport_number, vport_num);
+	MLX5_SET(query_vnic_env_in, in, other_vport, !!other_vport);
+
+	err = mlx5_cmd_exec_inout(dev, query_vnic_env, in, &vnic.query_vnic_env_out);
+	if (err)
+		return err;
+
+	err = devlink_fmsg_pair_nest_start(fmsg, "vNIC env counters");
+	if (err)
+		return err;
+
+	err = devlink_fmsg_obj_nest_start(fmsg);
+	if (err)
+		return err;
+
+	err = devlink_fmsg_u64_pair_put(fmsg, "total_error_queues",
+					VNIC_ENV_GET64(&vnic, total_error_queues));
+	if (err)
+		return err;
+
+	err = devlink_fmsg_u64_pair_put(fmsg, "send_queue_priority_update_flow",
+					VNIC_ENV_GET64(&vnic, send_queue_priority_update_flow));
+	if (err)
+		return err;
+
+	err = devlink_fmsg_u64_pair_put(fmsg, "comp_eq_overrun",
+					VNIC_ENV_GET64(&vnic, comp_eq_overrun));
+	if (err)
+		return err;
+
+	err = devlink_fmsg_u64_pair_put(fmsg, "async_eq_overrun",
+					VNIC_ENV_GET64(&vnic, async_eq_overrun));
+	if (err)
+		return err;
+
+	err = devlink_fmsg_u64_pair_put(fmsg, "cq_overrun",
+					VNIC_ENV_GET64(&vnic, cq_overrun));
+	if (err)
+		return err;
+
+	err = devlink_fmsg_u64_pair_put(fmsg, "invalid_command",
+					VNIC_ENV_GET64(&vnic, invalid_command));
+	if (err)
+		return err;
+
+	err = devlink_fmsg_u64_pair_put(fmsg, "quota_exceeded_command",
+					VNIC_ENV_GET64(&vnic, quota_exceeded_command));
+	if (err)
+		return err;
+
+	err = devlink_fmsg_u64_pair_put(fmsg, "nic_receive_steering_discard",
+					VNIC_ENV_GET64(&vnic, nic_receive_steering_discard));
+	if (err)
+		return err;
+
+	err = devlink_fmsg_obj_nest_end(fmsg);
+	if (err)
+		return err;
+
+	err = devlink_fmsg_pair_nest_end(fmsg);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int mlx5_reporter_vnic_diagnose(struct devlink_health_reporter *reporter,
+				       struct devlink_fmsg *fmsg,
+				       struct netlink_ext_ack *extack)
+{
+	struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter);
+
+	return mlx5_reporter_vnic_diagnose_counters(dev, fmsg, 0, false);
+}
+
+static const struct devlink_health_reporter_ops mlx5_reporter_vnic_ops = {
+	.name = "vnic",
+	.diagnose = mlx5_reporter_vnic_diagnose,
+};
+
+void mlx5_reporter_vnic_create(struct mlx5_core_dev *dev)
+{
+	struct mlx5_core_health *health = &dev->priv.health;
+	struct devlink *devlink = priv_to_devlink(dev);
+
+	health->vnic_reporter =
+		devlink_health_reporter_create(devlink,
+					       &mlx5_reporter_vnic_ops,
+					       0, dev);
+	if (IS_ERR(health->vnic_reporter))
+		mlx5_core_warn(dev,
+			       "Failed to create vnic reporter, err = %ld\n",
+			       PTR_ERR(health->vnic_reporter));
+}
+
+void mlx5_reporter_vnic_destroy(struct mlx5_core_dev *dev)
+{
+	struct mlx5_core_health *health = &dev->priv.health;
+
+	if (!IS_ERR_OR_NULL(health->vnic_reporter))
+		devlink_health_reporter_destroy(health->vnic_reporter);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/reporter_vnic.h b/drivers/net/ethernet/mellanox/mlx5/core/diag/reporter_vnic.h
new file mode 100644
index 000000000000..eba87a39e9b1
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/reporter_vnic.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+ * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.
+ */
+#ifndef __MLX5_REPORTER_VNIC_H
+#define __MLX5_REPORTER_VNIC_H
+
+#include "mlx5_core.h"
+
+void mlx5_reporter_vnic_create(struct mlx5_core_dev *dev);
+void mlx5_reporter_vnic_destroy(struct mlx5_core_dev *dev);
+
+int mlx5_reporter_vnic_diagnose_counters(struct mlx5_core_dev *dev,
+					 struct devlink_fmsg *fmsg,
+					 u16 vport_num, bool other_vport);
+
+#endif /* __MLX5_REPORTER_VNIC_H */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c
index 7c9c4e40c019..d000236ddbac 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c
@@ -75,10 +75,6 @@ int mlx5_ec_init(struct mlx5_core_dev *dev)
 	if (!mlx5_core_is_ecpf(dev))
 		return 0;
 
-	/* Management PF don't have a peer PF */
-	if (mlx5_core_is_management_pf(dev))
-		return 0;
-
 	return mlx5_host_pf_init(dev);
 }
 
@@ -89,10 +85,6 @@ void mlx5_ec_cleanup(struct mlx5_core_dev *dev)
 	if (!mlx5_core_is_ecpf(dev))
 		return;
 
-	/* Management PF don't have a peer PF */
-	if (mlx5_core_is_management_pf(dev))
-		return;
-
 	mlx5_host_pf_cleanup(dev);
 
 	err = mlx5_wait_for_pages(dev, &dev->priv.page_counters[MLX5_HOST_PF]);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 4a19ef4a9811..b8987a404d75 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -335,15 +335,20 @@ static inline u8 mlx5e_get_dcb_num_tc(struct mlx5e_params *params)
 		params->mqprio.num_tc : 1;
 }
 
+/* Keep this enum consistent with the corresponding strings array
+ * declared in en/reporter_rx.c
+ */
 enum {
-	MLX5E_RQ_STATE_ENABLED,
+	MLX5E_RQ_STATE_ENABLED = 0,
 	MLX5E_RQ_STATE_RECOVERING,
-	MLX5E_RQ_STATE_AM,
+	MLX5E_RQ_STATE_DIM,
 	MLX5E_RQ_STATE_NO_CSUM_COMPLETE,
 	MLX5E_RQ_STATE_CSUM_FULL, /* cqe_csum_full hw bit is set */
 	MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, /* set when mini_cqe_resp_stride_index cap is used */
 	MLX5E_RQ_STATE_SHAMPO, /* set when SHAMPO cap is used */
 	MLX5E_RQ_STATE_MINI_CQE_ENHANCED,  /* set when enhanced mini_cqe_cap is used */
+	MLX5E_RQ_STATE_XSK, /* set to indicate an xsk rq */
+	MLX5E_NUM_RQ_STATES, /* Must be kept last */
 };
 
 struct mlx5e_cq {
@@ -384,16 +389,20 @@ struct mlx5e_sq_dma {
 	enum mlx5e_dma_map_type type;
 };
 
+/* Keep this enum consistent with with the corresponding strings array
+ * declared in en/reporter_tx.c
+ */
 enum {
-	MLX5E_SQ_STATE_ENABLED,
+	MLX5E_SQ_STATE_ENABLED = 0,
 	MLX5E_SQ_STATE_MPWQE,
 	MLX5E_SQ_STATE_RECOVERING,
 	MLX5E_SQ_STATE_IPSEC,
-	MLX5E_SQ_STATE_AM,
+	MLX5E_SQ_STATE_DIM,
 	MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE,
 	MLX5E_SQ_STATE_PENDING_XSK_TX,
 	MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC,
 	MLX5E_SQ_STATE_XDP_MULTIBUF,
+	MLX5E_NUM_SQ_STATES, /* Must be kept last */
 };
 
 struct mlx5e_tx_mpwqe {
@@ -466,64 +475,18 @@ struct mlx5e_txqsq {
 	cqe_ts_to_ns               ptp_cyc2time;
 } ____cacheline_aligned_in_smp;
 
-union mlx5e_alloc_unit {
-	struct page *page;
-	struct xdp_buff *xsk;
-};
-
-/* XDP packets can be transmitted in different ways. On completion, we need to
- * distinguish between them to clean up things in a proper way.
- */
-enum mlx5e_xdp_xmit_mode {
-	/* An xdp_frame was transmitted due to either XDP_REDIRECT from another
-	 * device or XDP_TX from an XSK RQ. The frame has to be unmapped and
-	 * returned.
-	 */
-	MLX5E_XDP_XMIT_MODE_FRAME,
-
-	/* The xdp_frame was created in place as a result of XDP_TX from a
-	 * regular RQ. No DMA remapping happened, and the page belongs to us.
-	 */
-	MLX5E_XDP_XMIT_MODE_PAGE,
-
-	/* No xdp_frame was created at all, the transmit happened from a UMEM
-	 * page. The UMEM Completion Ring producer pointer has to be increased.
-	 */
-	MLX5E_XDP_XMIT_MODE_XSK,
-};
-
-struct mlx5e_xdp_info {
-	enum mlx5e_xdp_xmit_mode mode;
-	union {
-		struct {
-			struct xdp_frame *xdpf;
-			dma_addr_t dma_addr;
-		} frame;
-		struct {
-			struct mlx5e_rq *rq;
-			struct page *page;
-		} page;
-	};
-};
-
-struct mlx5e_xmit_data {
-	dma_addr_t  dma_addr;
-	void       *data;
-	u32         len;
-};
-
 struct mlx5e_xdp_info_fifo {
-	struct mlx5e_xdp_info *xi;
+	union mlx5e_xdp_info *xi;
 	u32 *cc;
 	u32 *pc;
 	u32 mask;
 };
 
 struct mlx5e_xdpsq;
+struct mlx5e_xmit_data;
 typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *);
 typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *,
 					struct mlx5e_xmit_data *,
-					struct skb_shared_info *,
 					int);
 
 struct mlx5e_xdpsq {
@@ -596,16 +559,36 @@ struct mlx5e_icosq {
 	struct work_struct         recover_work;
 } ____cacheline_aligned_in_smp;
 
+struct mlx5e_frag_page {
+	struct page *page;
+	u16 frags;
+};
+
+enum mlx5e_wqe_frag_flag {
+	MLX5E_WQE_FRAG_LAST_IN_PAGE,
+	MLX5E_WQE_FRAG_SKIP_RELEASE,
+};
+
 struct mlx5e_wqe_frag_info {
-	union mlx5e_alloc_unit *au;
+	union {
+		struct mlx5e_frag_page *frag_page;
+		struct xdp_buff **xskp;
+	};
 	u32 offset;
-	bool last_in_page;
+	u8 flags;
+};
+
+union mlx5e_alloc_units {
+	DECLARE_FLEX_ARRAY(struct mlx5e_frag_page, frag_pages);
+	DECLARE_FLEX_ARRAY(struct page *, pages);
+	DECLARE_FLEX_ARRAY(struct xdp_buff *, xsk_buffs);
 };
 
 struct mlx5e_mpw_info {
 	u16 consumed_strides;
-	DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE);
-	union mlx5e_alloc_unit alloc_units[];
+	DECLARE_BITMAP(skip_release_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE);
+	struct mlx5e_frag_page linear_page;
+	union mlx5e_alloc_units alloc_units;
 };
 
 #define MLX5E_MAX_RX_FRAGS 4
@@ -616,11 +599,6 @@ struct mlx5e_mpw_info {
 #define MLX5E_CACHE_UNIT (MLX5_MPWRQ_MAX_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \
 			  MLX5_MPWRQ_MAX_PAGES_PER_WQE : NAPI_POLL_WEIGHT)
 #define MLX5E_CACHE_SIZE	(4 * roundup_pow_of_two(MLX5E_CACHE_UNIT))
-struct mlx5e_page_cache {
-	u32 head;
-	u32 tail;
-	struct page *page_cache[MLX5E_CACHE_SIZE];
-};
 
 struct mlx5e_rq;
 typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*);
@@ -652,19 +630,24 @@ struct mlx5e_rq_frags_info {
 	struct mlx5e_rq_frag_info arr[MLX5E_MAX_RX_FRAGS];
 	u8 num_frags;
 	u8 log_num_frags;
-	u8 wqe_bulk;
+	u16 wqe_bulk;
+	u16 refill_unit;
 	u8 wqe_index_mask;
 };
 
 struct mlx5e_dma_info {
 	dma_addr_t addr;
-	struct page *page;
+	union {
+		struct mlx5e_frag_page *frag_page;
+		struct page *page;
+	};
 };
 
 struct mlx5e_shampo_hd {
 	u32 mkey;
 	struct mlx5e_dma_info *info;
-	struct page *last_page;
+	struct mlx5e_frag_page *pages;
+	u16 curr_page_index;
 	u16 hd_per_wq;
 	u16 hd_per_wqe;
 	unsigned long *bitmap;
@@ -693,7 +676,7 @@ struct mlx5e_rq {
 		struct {
 			struct mlx5_wq_cyc          wq;
 			struct mlx5e_wqe_frag_info *frags;
-			union mlx5e_alloc_unit     *alloc_units;
+			union mlx5e_alloc_units    *alloc_units;
 			struct mlx5e_rq_frags_info  info;
 			mlx5e_fp_skb_from_cqe       skb_from_cqe;
 		} wqe;
@@ -729,7 +712,6 @@ struct mlx5e_rq {
 	struct mlx5e_rq_stats *stats;
 	struct mlx5e_cq        cq;
 	struct mlx5e_cq_decomp cqd;
-	struct mlx5e_page_cache page_cache;
 	struct hwtstamp_config *tstamp;
 	struct mlx5_clock      *clock;
 	struct mlx5e_icosq    *icosq;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
index a21bd1179477..ef546ed8b4d9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
@@ -253,17 +253,20 @@ static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev,
 					 struct mlx5e_xsk_param *xsk,
 					 bool mpwqe)
 {
+	u32 sz;
+
 	/* XSK frames are mapped as individual pages, because frames may come in
 	 * an arbitrary order from random locations in the UMEM.
 	 */
 	if (xsk)
 		return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE;
 
-	/* XDP in mlx5e doesn't support multiple packets per page. */
-	if (params->xdp_prog)
-		return PAGE_SIZE;
+	sz = roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, false));
 
-	return roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, false));
+	/* XDP in mlx5e doesn't support multiple packets per page.
+	 * Do not assume sz <= PAGE_SIZE if params->xdp_prog is set.
+	 */
+	return params->xdp_prog && sz < PAGE_SIZE ? PAGE_SIZE : sz;
 }
 
 static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev,
@@ -320,6 +323,20 @@ static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
 	return log_num_strides >= MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
 }
 
+bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
+					  struct mlx5e_params *params,
+					  struct mlx5e_xsk_param *xsk)
+{
+	u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
+	u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+
+	return mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
+					     log_wqe_num_of_strides,
+					     page_shift, umr_mode);
+}
+
 bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
 				  struct mlx5e_params *params,
 				  struct mlx5e_xsk_param *xsk)
@@ -402,6 +419,10 @@ u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
 	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
 		return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
 
+	/* XDP in mlx5e doesn't support multiple packets per page. */
+	if (params->xdp_prog)
+		return PAGE_SHIFT;
+
 	return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
 }
 
@@ -553,7 +574,7 @@ bool slow_pci_heuristic(struct mlx5_core_dev *mdev)
 	u32 link_speed = 0;
 	u32 pci_bw = 0;
 
-	mlx5e_port_max_linkspeed(mdev, &link_speed);
+	mlx5_port_max_linkspeed(mdev, &link_speed);
 	pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL);
 	mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n",
 			   link_speed, pci_bw);
@@ -572,9 +593,6 @@ int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params
 	if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode))
 		return -EOPNOTSUPP;
 
-	if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
-		return -EINVAL;
-
 	return 0;
 }
 
@@ -667,6 +685,48 @@ static int mlx5e_max_nonlinear_mtu(int first_frag_size, int frag_size, bool xdp)
 	return first_frag_size + (MLX5E_MAX_RX_FRAGS - 2) * frag_size + PAGE_SIZE;
 }
 
+static void mlx5e_rx_compute_wqe_bulk_params(struct mlx5e_params *params,
+					     struct mlx5e_rq_frags_info *info)
+{
+	u16 bulk_bound_rq_size = (1 << params->log_rq_mtu_frames) / 4;
+	u32 bulk_bound_rq_size_in_bytes;
+	u32 sum_frag_strides = 0;
+	u32 wqe_bulk_in_bytes;
+	u16 split_factor;
+	u32 wqe_bulk;
+	int i;
+
+	for (i = 0; i < info->num_frags; i++)
+		sum_frag_strides += info->arr[i].frag_stride;
+
+	/* For MTUs larger than PAGE_SIZE, align to PAGE_SIZE to reflect
+	 * amount of consumed pages per wqe in bytes.
+	 */
+	if (sum_frag_strides > PAGE_SIZE)
+		sum_frag_strides = ALIGN(sum_frag_strides, PAGE_SIZE);
+
+	bulk_bound_rq_size_in_bytes = bulk_bound_rq_size * sum_frag_strides;
+
+#define MAX_WQE_BULK_BYTES(xdp) ((xdp ? 256 : 512) * 1024)
+
+	/* A WQE bulk should not exceed min(512KB, 1/4 of rq size). For XDP
+	 * keep bulk size smaller to avoid filling the page_pool cache on
+	 * every bulk refill.
+	 */
+	wqe_bulk_in_bytes = min_t(u32, MAX_WQE_BULK_BYTES(params->xdp_prog),
+				  bulk_bound_rq_size_in_bytes);
+	wqe_bulk = DIV_ROUND_UP(wqe_bulk_in_bytes, sum_frag_strides);
+
+	/* Make sure that allocations don't start when the page is still used
+	 * by older WQEs.
+	 */
+	info->wqe_bulk = max_t(u16, info->wqe_index_mask + 1, wqe_bulk);
+
+	split_factor = DIV_ROUND_UP(MAX_WQE_BULK_BYTES(params->xdp_prog),
+				    PP_ALLOC_CACHE_REFILL * PAGE_SIZE);
+	info->refill_unit = DIV_ROUND_UP(info->wqe_bulk, split_factor);
+}
+
 #define DEFAULT_FRAG_SIZE (2048)
 
 static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
@@ -774,11 +834,14 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
 	}
 
 out:
-	/* Bulking optimization to skip allocation until at least 8 WQEs can be
-	 * allocated in a row. At the same time, never start allocation when
-	 * the page is still used by older WQEs.
+	/* Bulking optimization to skip allocation until a large enough number
+	 * of WQEs can be allocated in a row. Bulking also influences how well
+	 * deferred page release works.
 	 */
-	info->wqe_bulk = max_t(u8, info->wqe_index_mask + 1, 8);
+	mlx5e_rx_compute_wqe_bulk_params(params, info);
+
+	mlx5_core_dbg(mdev, "%s: wqe_bulk = %u, wqe_bulk_refill_unit = %u\n",
+		      __func__, info->wqe_bulk, info->refill_unit);
 
 	info->log_num_frags = order_base_2(info->num_frags);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
index c9be6eb88012..a5d20f6d6d9c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
@@ -153,6 +153,9 @@ int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
 
 u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
+bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
+					  struct mlx5e_params *params,
+					  struct mlx5e_xsk_param *xsk);
 
 static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev,
 					   struct mlx5e_params *params)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port.c b/drivers/net/ethernet/mellanox/mlx5/core/en/port.c
index 505ba41195b9..dbe2b19a9570 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/port.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port.c
@@ -32,101 +32,6 @@
 
 #include "port.h"
 
-/* speed in units of 1Mb */
-static const u32 mlx5e_link_speed[MLX5E_LINK_MODES_NUMBER] = {
-	[MLX5E_1000BASE_CX_SGMII] = 1000,
-	[MLX5E_1000BASE_KX]       = 1000,
-	[MLX5E_10GBASE_CX4]       = 10000,
-	[MLX5E_10GBASE_KX4]       = 10000,
-	[MLX5E_10GBASE_KR]        = 10000,
-	[MLX5E_20GBASE_KR2]       = 20000,
-	[MLX5E_40GBASE_CR4]       = 40000,
-	[MLX5E_40GBASE_KR4]       = 40000,
-	[MLX5E_56GBASE_R4]        = 56000,
-	[MLX5E_10GBASE_CR]        = 10000,
-	[MLX5E_10GBASE_SR]        = 10000,
-	[MLX5E_10GBASE_ER]        = 10000,
-	[MLX5E_40GBASE_SR4]       = 40000,
-	[MLX5E_40GBASE_LR4]       = 40000,
-	[MLX5E_50GBASE_SR2]       = 50000,
-	[MLX5E_100GBASE_CR4]      = 100000,
-	[MLX5E_100GBASE_SR4]      = 100000,
-	[MLX5E_100GBASE_KR4]      = 100000,
-	[MLX5E_100GBASE_LR4]      = 100000,
-	[MLX5E_100BASE_TX]        = 100,
-	[MLX5E_1000BASE_T]        = 1000,
-	[MLX5E_10GBASE_T]         = 10000,
-	[MLX5E_25GBASE_CR]        = 25000,
-	[MLX5E_25GBASE_KR]        = 25000,
-	[MLX5E_25GBASE_SR]        = 25000,
-	[MLX5E_50GBASE_CR2]       = 50000,
-	[MLX5E_50GBASE_KR2]       = 50000,
-};
-
-static const u32 mlx5e_ext_link_speed[MLX5E_EXT_LINK_MODES_NUMBER] = {
-	[MLX5E_SGMII_100M]			= 100,
-	[MLX5E_1000BASE_X_SGMII]		= 1000,
-	[MLX5E_5GBASE_R]			= 5000,
-	[MLX5E_10GBASE_XFI_XAUI_1]		= 10000,
-	[MLX5E_40GBASE_XLAUI_4_XLPPI_4]		= 40000,
-	[MLX5E_25GAUI_1_25GBASE_CR_KR]		= 25000,
-	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2]	= 50000,
-	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR]	= 50000,
-	[MLX5E_CAUI_4_100GBASE_CR4_KR4]		= 100000,
-	[MLX5E_100GAUI_2_100GBASE_CR2_KR2]	= 100000,
-	[MLX5E_200GAUI_4_200GBASE_CR4_KR4]	= 200000,
-	[MLX5E_400GAUI_8]			= 400000,
-	[MLX5E_100GAUI_1_100GBASE_CR_KR]	= 100000,
-	[MLX5E_200GAUI_2_200GBASE_CR2_KR2]	= 200000,
-	[MLX5E_400GAUI_4_400GBASE_CR4_KR4]	= 400000,
-};
-
-bool mlx5e_ptys_ext_supported(struct mlx5_core_dev *mdev)
-{
-	struct mlx5e_port_eth_proto eproto;
-	int err;
-
-	if (MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet))
-		return true;
-
-	err = mlx5_port_query_eth_proto(mdev, 1, true, &eproto);
-	if (err)
-		return false;
-
-	return !!eproto.cap;
-}
-
-static void mlx5e_port_get_speed_arr(struct mlx5_core_dev *mdev,
-				     const u32 **arr, u32 *size,
-				     bool force_legacy)
-{
-	bool ext = force_legacy ? false : mlx5e_ptys_ext_supported(mdev);
-
-	*size = ext ? ARRAY_SIZE(mlx5e_ext_link_speed) :
-		      ARRAY_SIZE(mlx5e_link_speed);
-	*arr  = ext ? mlx5e_ext_link_speed : mlx5e_link_speed;
-}
-
-int mlx5_port_query_eth_proto(struct mlx5_core_dev *dev, u8 port, bool ext,
-			      struct mlx5e_port_eth_proto *eproto)
-{
-	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
-	int err;
-
-	if (!eproto)
-		return -EINVAL;
-
-	err = mlx5_query_port_ptys(dev, out, sizeof(out), MLX5_PTYS_EN, port);
-	if (err)
-		return err;
-
-	eproto->cap   = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
-					   eth_proto_capability);
-	eproto->admin = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_admin);
-	eproto->oper  = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);
-	return 0;
-}
-
 void mlx5_port_query_eth_autoneg(struct mlx5_core_dev *dev, u8 *an_status,
 				 u8 *an_disable_cap, u8 *an_disable_admin)
 {
@@ -172,30 +77,14 @@ int mlx5_port_set_eth_ptys(struct mlx5_core_dev *dev, bool an_disable,
 			    sizeof(out), MLX5_REG_PTYS, 0, 1);
 }
 
-u32 mlx5e_port_ptys2speed(struct mlx5_core_dev *mdev, u32 eth_proto_oper,
-			  bool force_legacy)
-{
-	unsigned long temp = eth_proto_oper;
-	const u32 *table;
-	u32 speed = 0;
-	u32 max_size;
-	int i;
-
-	mlx5e_port_get_speed_arr(mdev, &table, &max_size, force_legacy);
-	i = find_first_bit(&temp, max_size);
-	if (i < max_size)
-		speed = table[i];
-	return speed;
-}
-
 int mlx5e_port_linkspeed(struct mlx5_core_dev *mdev, u32 *speed)
 {
-	struct mlx5e_port_eth_proto eproto;
+	struct mlx5_port_eth_proto eproto;
 	bool force_legacy = false;
 	bool ext;
 	int err;
 
-	ext = mlx5e_ptys_ext_supported(mdev);
+	ext = mlx5_ptys_ext_supported(mdev);
 	err = mlx5_port_query_eth_proto(mdev, 1, ext, &eproto);
 	if (err)
 		goto out;
@@ -205,7 +94,7 @@ int mlx5e_port_linkspeed(struct mlx5_core_dev *mdev, u32 *speed)
 		if (err)
 			goto out;
 	}
-	*speed = mlx5e_port_ptys2speed(mdev, eproto.oper, force_legacy);
+	*speed = mlx5_port_ptys2speed(mdev, eproto.oper, force_legacy);
 	if (!(*speed))
 		err = -EINVAL;
 
@@ -213,46 +102,6 @@ out:
 	return err;
 }
 
-int mlx5e_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed)
-{
-	struct mlx5e_port_eth_proto eproto;
-	u32 max_speed = 0;
-	const u32 *table;
-	u32 max_size;
-	bool ext;
-	int err;
-	int i;
-
-	ext = mlx5e_ptys_ext_supported(mdev);
-	err = mlx5_port_query_eth_proto(mdev, 1, ext, &eproto);
-	if (err)
-		return err;
-
-	mlx5e_port_get_speed_arr(mdev, &table, &max_size, false);
-	for (i = 0; i < max_size; ++i)
-		if (eproto.cap & MLX5E_PROT_MASK(i))
-			max_speed = max(max_speed, table[i]);
-
-	*speed = max_speed;
-	return 0;
-}
-
-u32 mlx5e_port_speed2linkmodes(struct mlx5_core_dev *mdev, u32 speed,
-			       bool force_legacy)
-{
-	u32 link_modes = 0;
-	const u32 *table;
-	u32 max_size;
-	int i;
-
-	mlx5e_port_get_speed_arr(mdev, &table, &max_size, force_legacy);
-	for (i = 0; i < max_size; ++i) {
-		if (table[i] == speed)
-			link_modes |= MLX5E_PROT_MASK(i);
-	}
-	return link_modes;
-}
-
 int mlx5e_port_query_pbmc(struct mlx5_core_dev *mdev, void *out)
 {
 	int sz = MLX5_ST_SZ_BYTES(pbmc_reg);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port.h b/drivers/net/ethernet/mellanox/mlx5/core/en/port.h
index 3f474e370828..d1da225f35da 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/port.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port.h
@@ -36,25 +36,11 @@
 #include <linux/mlx5/driver.h>
 #include "en.h"
 
-struct mlx5e_port_eth_proto {
-	u32 cap;
-	u32 admin;
-	u32 oper;
-};
-
-int mlx5_port_query_eth_proto(struct mlx5_core_dev *dev, u8 port, bool ext,
-			      struct mlx5e_port_eth_proto *eproto);
 void mlx5_port_query_eth_autoneg(struct mlx5_core_dev *dev, u8 *an_status,
 				 u8 *an_disable_cap, u8 *an_disable_admin);
 int mlx5_port_set_eth_ptys(struct mlx5_core_dev *dev, bool an_disable,
 			   u32 proto_admin, bool ext);
-u32 mlx5e_port_ptys2speed(struct mlx5_core_dev *mdev, u32 eth_proto_oper,
-			  bool force_legacy);
 int mlx5e_port_linkspeed(struct mlx5_core_dev *mdev, u32 *speed);
-int mlx5e_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed);
-u32 mlx5e_port_speed2linkmodes(struct mlx5_core_dev *mdev, u32 speed,
-			       bool force_legacy);
-bool mlx5e_ptys_ext_supported(struct mlx5_core_dev *mdev);
 int mlx5e_port_query_pbmc(struct mlx5_core_dev *mdev, void *out);
 int mlx5e_port_set_pbmc(struct mlx5_core_dev *mdev, void *in);
 int mlx5e_port_query_sbpr(struct mlx5_core_dev *mdev, u32 desc, u8 dir,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
index eb5aeba3addf..eb5abd0e55d9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
@@ -81,23 +81,23 @@ void mlx5e_skb_cb_hwtstamp_handler(struct sk_buff *skb, int hwtstamp_type,
 
 #define PTP_WQE_CTR2IDX(val) ((val) & ptpsq->ts_cqe_ctr_mask)
 
-static bool mlx5e_ptp_ts_cqe_drop(struct mlx5e_ptpsq *ptpsq, u16 skb_cc, u16 skb_id)
+static bool mlx5e_ptp_ts_cqe_drop(struct mlx5e_ptpsq *ptpsq, u16 skb_ci, u16 skb_id)
 {
-	return (ptpsq->ts_cqe_ctr_mask && (skb_cc != skb_id));
+	return (ptpsq->ts_cqe_ctr_mask && (skb_ci != skb_id));
 }
 
 static bool mlx5e_ptp_ts_cqe_ooo(struct mlx5e_ptpsq *ptpsq, u16 skb_id)
 {
-	u16 skb_cc = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc);
-	u16 skb_pc = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_pc);
+	u16 skb_ci = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc);
+	u16 skb_pi = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_pc);
 
-	if (PTP_WQE_CTR2IDX(skb_id - skb_cc) >= PTP_WQE_CTR2IDX(skb_pc - skb_cc))
+	if (PTP_WQE_CTR2IDX(skb_id - skb_ci) >= PTP_WQE_CTR2IDX(skb_pi - skb_ci))
 		return true;
 
 	return false;
 }
 
-static void mlx5e_ptp_skb_fifo_ts_cqe_resync(struct mlx5e_ptpsq *ptpsq, u16 skb_cc,
+static void mlx5e_ptp_skb_fifo_ts_cqe_resync(struct mlx5e_ptpsq *ptpsq, u16 skb_ci,
 					     u16 skb_id, int budget)
 {
 	struct skb_shared_hwtstamps hwts = {};
@@ -105,13 +105,13 @@ static void mlx5e_ptp_skb_fifo_ts_cqe_resync(struct mlx5e_ptpsq *ptpsq, u16 skb_
 
 	ptpsq->cq_stats->resync_event++;
 
-	while (skb_cc != skb_id) {
+	while (skb_ci != skb_id) {
 		skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo);
 		hwts.hwtstamp = mlx5e_skb_cb_get_hwts(skb)->cqe_hwtstamp;
 		skb_tstamp_tx(skb, &hwts);
 		ptpsq->cq_stats->resync_cqe++;
 		napi_consume_skb(skb, budget);
-		skb_cc = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc);
+		skb_ci = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc);
 	}
 }
 
@@ -120,7 +120,7 @@ static void mlx5e_ptp_handle_ts_cqe(struct mlx5e_ptpsq *ptpsq,
 				    int budget)
 {
 	u16 skb_id = PTP_WQE_CTR2IDX(be16_to_cpu(cqe->wqe_counter));
-	u16 skb_cc = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc);
+	u16 skb_ci = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc);
 	struct mlx5e_txqsq *sq = &ptpsq->txqsq;
 	struct sk_buff *skb;
 	ktime_t hwtstamp;
@@ -131,13 +131,13 @@ static void mlx5e_ptp_handle_ts_cqe(struct mlx5e_ptpsq *ptpsq,
 		goto out;
 	}
 
-	if (mlx5e_ptp_ts_cqe_drop(ptpsq, skb_cc, skb_id)) {
+	if (mlx5e_ptp_ts_cqe_drop(ptpsq, skb_ci, skb_id)) {
 		if (mlx5e_ptp_ts_cqe_ooo(ptpsq, skb_id)) {
 			/* already handled by a previous resync */
 			ptpsq->cq_stats->ooo_cqe_drop++;
 			return;
 		}
-		mlx5e_ptp_skb_fifo_ts_cqe_resync(ptpsq, skb_cc, skb_id, budget);
+		mlx5e_ptp_skb_fifo_ts_cqe_resync(ptpsq, skb_ci, skb_id, budget);
 	}
 
 	skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c
index ce85b48d327d..fd191925ab4b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c
@@ -220,6 +220,7 @@ mlx5_esw_bridge_port_obj_add(struct net_device *dev,
 	struct netlink_ext_ack *extack = switchdev_notifier_info_to_extack(&port_obj_info->info);
 	const struct switchdev_obj *obj = port_obj_info->obj;
 	const struct switchdev_obj_port_vlan *vlan;
+	const struct switchdev_obj_port_mdb *mdb;
 	u16 vport_num, esw_owner_vhca_id;
 	int err;
 
@@ -235,6 +236,11 @@ mlx5_esw_bridge_port_obj_add(struct net_device *dev,
 		err = mlx5_esw_bridge_port_vlan_add(vport_num, esw_owner_vhca_id, vlan->vid,
 						    vlan->flags, br_offloads, extack);
 		break;
+	case SWITCHDEV_OBJ_ID_PORT_MDB:
+		mdb = SWITCHDEV_OBJ_PORT_MDB(obj);
+		err = mlx5_esw_bridge_port_mdb_add(dev, vport_num, esw_owner_vhca_id, mdb->addr,
+						   mdb->vid, br_offloads, extack);
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -248,6 +254,7 @@ mlx5_esw_bridge_port_obj_del(struct net_device *dev,
 {
 	const struct switchdev_obj *obj = port_obj_info->obj;
 	const struct switchdev_obj_port_vlan *vlan;
+	const struct switchdev_obj_port_mdb *mdb;
 	u16 vport_num, esw_owner_vhca_id;
 
 	if (!mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, br_offloads->esw, &vport_num,
@@ -261,6 +268,11 @@ mlx5_esw_bridge_port_obj_del(struct net_device *dev,
 		vlan = SWITCHDEV_OBJ_PORT_VLAN(obj);
 		mlx5_esw_bridge_port_vlan_del(vport_num, esw_owner_vhca_id, vlan->vid, br_offloads);
 		break;
+	case SWITCHDEV_OBJ_ID_PORT_MDB:
+		mdb = SWITCHDEV_OBJ_PORT_MDB(obj);
+		mlx5_esw_bridge_port_mdb_del(dev, vport_num, esw_owner_vhca_id, mdb->addr, mdb->vid,
+					     br_offloads);
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -306,6 +318,10 @@ mlx5_esw_bridge_port_obj_attr_set(struct net_device *dev,
 						     attr->u.vlan_protocol,
 						     br_offloads);
 		break;
+	case SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED:
+		err = mlx5_esw_bridge_mcast_set(vport_num, esw_owner_vhca_id,
+						!attr->u.mc_disabled, br_offloads);
+		break;
 	default:
 		err = -EOPNOTSUPP;
 	}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
index 8f7452dc00ee..b5c773ffc763 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
@@ -426,39 +426,58 @@ static bool mlx5e_rep_macvlan_mode_supported(const struct net_device *dev)
 	return macvlan->mode == MACVLAN_MODE_PASSTHRU;
 }
 
-static int
-mlx5e_rep_indr_setup_block(struct net_device *netdev, struct Qdisc *sch,
-			   struct mlx5e_rep_priv *rpriv,
-			   struct flow_block_offload *f,
-			   flow_setup_cb_t *setup_cb,
-			   void *data,
-			   void (*cleanup)(struct flow_block_cb *block_cb))
+static bool
+mlx5e_rep_check_indr_block_supported(struct mlx5e_rep_priv *rpriv,
+				     struct net_device *netdev,
+				     struct flow_block_offload *f)
 {
 	struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
-	bool is_ovs_int_port = netif_is_ovs_master(netdev);
-	struct mlx5e_rep_indr_block_priv *indr_priv;
-	struct flow_block_cb *block_cb;
+	struct net_device *macvlan_real_dev;
 
-	if (!mlx5e_tc_tun_device_to_offload(priv, netdev) &&
-	    !(is_vlan_dev(netdev) && vlan_dev_real_dev(netdev) == rpriv->netdev) &&
-	    !is_ovs_int_port) {
-		if (!(netif_is_macvlan(netdev) && macvlan_dev_real_dev(netdev) == rpriv->netdev))
-			return -EOPNOTSUPP;
+	if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
+	    f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
+		return false;
+
+	if (mlx5e_tc_tun_device_to_offload(priv, netdev))
+		return true;
+
+	if (is_vlan_dev(netdev) && vlan_dev_real_dev(netdev) == rpriv->netdev)
+		return true;
+
+	if (netif_is_macvlan(netdev)) {
 		if (!mlx5e_rep_macvlan_mode_supported(netdev)) {
 			netdev_warn(netdev, "Offloading ingress filter is supported only with macvlan passthru mode");
-			return -EOPNOTSUPP;
+			return false;
 		}
+
+		macvlan_real_dev = macvlan_dev_real_dev(netdev);
+
+		if (macvlan_real_dev == rpriv->netdev)
+			return true;
+		if (netif_is_bond_master(macvlan_real_dev))
+			return true;
 	}
 
-	if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
-	    f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
-		return -EOPNOTSUPP;
+	if (netif_is_ovs_master(netdev) && f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS &&
+	    mlx5e_tc_int_port_supported(esw))
+		return true;
 
-	if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS && !is_ovs_int_port)
-		return -EOPNOTSUPP;
+	return false;
+}
 
-	if (is_ovs_int_port && !mlx5e_tc_int_port_supported(esw))
+static int
+mlx5e_rep_indr_setup_block(struct net_device *netdev, struct Qdisc *sch,
+			   struct mlx5e_rep_priv *rpriv,
+			   struct flow_block_offload *f,
+			   flow_setup_cb_t *setup_cb,
+			   void *data,
+			   void (*cleanup)(struct flow_block_cb *block_cb))
+{
+	struct mlx5e_rep_indr_block_priv *indr_priv;
+	struct flow_block_cb *block_cb;
+
+	if (!mlx5e_rep_check_indr_block_supported(rpriv, netdev, f))
 		return -EOPNOTSUPP;
 
 	f->unlocked_driver_cb = true;
@@ -715,5 +734,6 @@ forward:
 	return;
 
 free_skb:
+	dev_put(tc_priv.fwd_dev);
 	dev_kfree_skb_any(skb);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
index c462fe76495b..e8eea9ffd5eb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
@@ -8,6 +8,19 @@
 #include "ptp.h"
 #include "lib/tout.h"
 
+/* Keep this string array consistent with the MLX5E_RQ_STATE_* enums in en.h */
+static const char * const rq_sw_state_type_name[] = {
+	[MLX5E_RQ_STATE_ENABLED] = "enabled",
+	[MLX5E_RQ_STATE_RECOVERING] = "recovering",
+	[MLX5E_RQ_STATE_DIM] = "dim",
+	[MLX5E_RQ_STATE_NO_CSUM_COMPLETE] = "no_csum_complete",
+	[MLX5E_RQ_STATE_CSUM_FULL] = "csum_full",
+	[MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX] = "mini_cqe_hw_stridx",
+	[MLX5E_RQ_STATE_SHAMPO] = "shampo",
+	[MLX5E_RQ_STATE_MINI_CQE_ENHANCED] = "mini_cqe_enhanced",
+	[MLX5E_RQ_STATE_XSK] = "xsk",
+};
+
 static int mlx5e_query_rq_state(struct mlx5_core_dev *dev, u32 rqn, u8 *state)
 {
 	int outlen = MLX5_ST_SZ_BYTES(query_rq_out);
@@ -108,9 +121,9 @@ static int mlx5e_rx_reporter_err_icosq_cqe_recover(void *ctx)
 
 	mlx5e_reset_icosq_cc_pc(icosq);
 
-	mlx5e_free_rx_in_progress_descs(rq);
+	mlx5e_free_rx_missing_descs(rq);
 	if (xskrq)
-		mlx5e_free_rx_in_progress_descs(xskrq);
+		mlx5e_free_rx_missing_descs(xskrq);
 
 	clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state);
 	mlx5e_activate_icosq(icosq);
@@ -239,6 +252,27 @@ static int mlx5e_reporter_icosq_diagnose(struct mlx5e_icosq *icosq, u8 hw_state,
 	return mlx5e_health_fmsg_named_obj_nest_end(fmsg);
 }
 
+static int mlx5e_health_rq_put_sw_state(struct devlink_fmsg *fmsg, struct mlx5e_rq *rq)
+{
+	int err;
+	int i;
+
+	BUILD_BUG_ON_MSG(ARRAY_SIZE(rq_sw_state_type_name) != MLX5E_NUM_RQ_STATES,
+			 "rq_sw_state_type_name string array must be consistent with MLX5E_RQ_STATE_* enum in en.h");
+	err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SW State");
+	if (err)
+		return err;
+
+	for (i = 0; i < ARRAY_SIZE(rq_sw_state_type_name); ++i) {
+		err = devlink_fmsg_u32_pair_put(fmsg, rq_sw_state_type_name[i],
+						test_bit(i, &rq->state));
+		if (err)
+			return err;
+	}
+
+	return mlx5e_health_fmsg_named_obj_nest_end(fmsg);
+}
+
 static int
 mlx5e_rx_reporter_build_diagnose_output_rq_common(struct mlx5e_rq *rq,
 						  struct devlink_fmsg *fmsg)
@@ -265,10 +299,6 @@ mlx5e_rx_reporter_build_diagnose_output_rq_common(struct mlx5e_rq *rq,
 	if (err)
 		return err;
 
-	err = devlink_fmsg_u8_pair_put(fmsg, "SW state", rq->state);
-	if (err)
-		return err;
-
 	err = devlink_fmsg_u32_pair_put(fmsg, "WQE counter", wqe_counter);
 	if (err)
 		return err;
@@ -281,6 +311,10 @@ mlx5e_rx_reporter_build_diagnose_output_rq_common(struct mlx5e_rq *rq,
 	if (err)
 		return err;
 
+	err = mlx5e_health_rq_put_sw_state(fmsg, rq);
+	if (err)
+		return err;
+
 	err = mlx5e_health_cq_diag_fmsg(&rq->cq, fmsg);
 	if (err)
 		return err;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
index 34666e2b3871..b35ff289af49 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
@@ -6,6 +6,19 @@
 #include "en/devlink.h"
 #include "lib/tout.h"
 
+/* Keep this string array consistent with the MLX5E_SQ_STATE_* enums in en.h */
+static const char * const sq_sw_state_type_name[] = {
+	[MLX5E_SQ_STATE_ENABLED] = "enabled",
+	[MLX5E_SQ_STATE_MPWQE] = "mpwqe",
+	[MLX5E_SQ_STATE_RECOVERING] = "recovering",
+	[MLX5E_SQ_STATE_IPSEC] = "ipsec",
+	[MLX5E_SQ_STATE_DIM] = "dim",
+	[MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE] = "vlan_need_l2_inline",
+	[MLX5E_SQ_STATE_PENDING_XSK_TX] = "pending_xsk_tx",
+	[MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC] = "pending_tls_rx_resync",
+	[MLX5E_SQ_STATE_XDP_MULTIBUF] = "xdp_multibuf",
+};
+
 static int mlx5e_wait_for_sq_flush(struct mlx5e_txqsq *sq)
 {
 	struct mlx5_core_dev *dev = sq->mdev;
@@ -37,6 +50,27 @@ static void mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq *sq)
 	sq->pc = 0;
 }
 
+static int mlx5e_health_sq_put_sw_state(struct devlink_fmsg *fmsg, struct mlx5e_txqsq *sq)
+{
+	int err;
+	int i;
+
+	BUILD_BUG_ON_MSG(ARRAY_SIZE(sq_sw_state_type_name) != MLX5E_NUM_SQ_STATES,
+			 "sq_sw_state_type_name string array must be consistent with MLX5E_SQ_STATE_* enum in en.h");
+	err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SW State");
+	if (err)
+		return err;
+
+	for (i = 0; i < ARRAY_SIZE(sq_sw_state_type_name); ++i) {
+		err = devlink_fmsg_u32_pair_put(fmsg, sq_sw_state_type_name[i],
+						test_bit(i, &sq->state));
+		if (err)
+			return err;
+	}
+
+	return mlx5e_health_fmsg_named_obj_nest_end(fmsg);
+}
+
 static int mlx5e_tx_reporter_err_cqe_recover(void *ctx)
 {
 	struct mlx5_core_dev *mdev;
@@ -190,6 +224,10 @@ mlx5e_tx_reporter_build_diagnose_output_sq_common(struct devlink_fmsg *fmsg,
 	if (err)
 		return err;
 
+	err = mlx5e_health_sq_put_sw_state(fmsg, sq);
+	if (err)
+		return err;
+
 	err = mlx5e_health_cq_diag_fmsg(&sq->cq, fmsg);
 	if (err)
 		return err;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/accept.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/accept.c
index a278f52d52b0..9db1b5307a8d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/accept.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/accept.c
@@ -4,15 +4,6 @@
 #include "act.h"
 #include "en/tc_priv.h"
 
-static bool
-tc_act_can_offload_accept(struct mlx5e_tc_act_parse_state *parse_state,
-			  const struct flow_action_entry *act,
-			  int act_index,
-			  struct mlx5_flow_attr *attr)
-{
-	return true;
-}
-
 static int
 tc_act_parse_accept(struct mlx5e_tc_act_parse_state *parse_state,
 		    const struct flow_action_entry *act,
@@ -26,7 +17,6 @@ tc_act_parse_accept(struct mlx5e_tc_act_parse_state *parse_state,
 }
 
 struct mlx5e_tc_act mlx5e_tc_act_accept = {
-	.can_offload = tc_act_can_offload_accept,
 	.parse_action = tc_act_parse_accept,
 	.is_terminating_action = true,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c
index eba0c8698926..fc923a99b6a4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c
@@ -82,26 +82,6 @@ mlx5e_tc_act_init_parse_state(struct mlx5e_tc_act_parse_state *parse_state,
 	parse_state->flow_action = flow_action;
 }
 
-void
-mlx5e_tc_act_reorder_flow_actions(struct flow_action *flow_action,
-				  struct mlx5e_tc_flow_action *flow_action_reorder)
-{
-	struct flow_action_entry *act;
-	int i, j = 0;
-
-	flow_action_for_each(i, act, flow_action) {
-		/* Add CT action to be first. */
-		if (act->id == FLOW_ACTION_CT)
-			flow_action_reorder->entries[j++] = act;
-	}
-
-	flow_action_for_each(i, act, flow_action) {
-		if (act->id == FLOW_ACTION_CT)
-			continue;
-		flow_action_reorder->entries[j++] = act;
-	}
-}
-
 int
 mlx5e_tc_act_post_parse(struct mlx5e_tc_act_parse_state *parse_state,
 			struct flow_action *flow_action,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h
index 8346557eeaf6..0e6e1872ac62 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h
@@ -17,8 +17,6 @@ struct mlx5e_tc_act_parse_state {
 	struct mlx5e_tc_flow *flow;
 	struct netlink_ext_ack *extack;
 	u32 actions;
-	bool ct;
-	bool ct_clear;
 	bool encap;
 	bool decap;
 	bool mpls_push;
@@ -56,6 +54,8 @@ struct mlx5e_tc_act {
 				   const struct flow_action_entry *act,
 				   struct mlx5_flow_attr *attr);
 
+	bool (*is_missable)(const struct flow_action_entry *act);
+
 	int (*offload_action)(struct mlx5e_priv *priv,
 			      struct flow_offload_action *fl_act,
 			      struct flow_action_entry *act);
@@ -110,10 +110,6 @@ mlx5e_tc_act_init_parse_state(struct mlx5e_tc_act_parse_state *parse_state,
 			      struct flow_action *flow_action,
 			      struct netlink_ext_ack *extack);
 
-void
-mlx5e_tc_act_reorder_flow_actions(struct flow_action *flow_action,
-				  struct mlx5e_tc_flow_action *flow_action_reorder);
-
 int
 mlx5e_tc_act_post_parse(struct mlx5e_tc_act_parse_state *parse_state,
 			struct flow_action *flow_action,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/ct.c
index a829c94289c1..92d3952dfa8b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/ct.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/ct.c
@@ -5,53 +5,22 @@
 #include "en/tc_priv.h"
 #include "en/tc_ct.h"
 
-static bool
-tc_act_can_offload_ct(struct mlx5e_tc_act_parse_state *parse_state,
-		      const struct flow_action_entry *act,
-		      int act_index,
-		      struct mlx5_flow_attr *attr)
-{
-	bool clear_action = act->ct.action & TCA_CT_ACT_CLEAR;
-	struct netlink_ext_ack *extack = parse_state->extack;
-
-	if (parse_state->ct && !clear_action) {
-		NL_SET_ERR_MSG_MOD(extack, "Multiple CT actions are not supported");
-		return false;
-	}
-
-	return true;
-}
-
 static int
 tc_act_parse_ct(struct mlx5e_tc_act_parse_state *parse_state,
 		const struct flow_action_entry *act,
 		struct mlx5e_priv *priv,
 		struct mlx5_flow_attr *attr)
 {
-	bool clear_action = act->ct.action & TCA_CT_ACT_CLEAR;
 	int err;
 
-	/* It's redundant to do ct clear more than once. */
-	if (clear_action && parse_state->ct_clear)
-		return 0;
-
-	err = mlx5_tc_ct_parse_action(parse_state->ct_priv, attr,
-				      &attr->parse_attr->mod_hdr_acts,
-				      act, parse_state->extack);
+	err = mlx5_tc_ct_parse_action(parse_state->ct_priv, attr, act, parse_state->extack);
 	if (err)
 		return err;
 
-
 	if (mlx5e_is_eswitch_flow(parse_state->flow))
 		attr->esw_attr->split_count = attr->esw_attr->out_count;
 
-	if (clear_action) {
-		parse_state->ct_clear = true;
-	} else {
-		attr->flags |= MLX5_ATTR_FLAG_CT;
-		flow_flag_set(parse_state->flow, CT);
-		parse_state->ct = true;
-	}
+	attr->flags |= MLX5_ATTR_FLAG_CT;
 
 	return 0;
 }
@@ -61,27 +30,10 @@ tc_act_post_parse_ct(struct mlx5e_tc_act_parse_state *parse_state,
 		     struct mlx5e_priv *priv,
 		     struct mlx5_flow_attr *attr)
 {
-	struct mlx5e_tc_mod_hdr_acts *mod_acts = &attr->parse_attr->mod_hdr_acts;
-	int err;
-
-	/* If ct action exist, we can ignore previous ct_clear actions */
-	if (parse_state->ct)
+	if (!(attr->flags & MLX5_ATTR_FLAG_CT))
 		return 0;
 
-	if (parse_state->ct_clear) {
-		err = mlx5_tc_ct_set_ct_clear_regs(parse_state->ct_priv, mod_acts);
-		if (err) {
-			NL_SET_ERR_MSG_MOD(parse_state->extack,
-					   "Failed to set registers for ct clear");
-			return err;
-		}
-		attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
-
-		/* Prevent handling of additional, redundant clear actions */
-		parse_state->ct_clear = false;
-	}
-
-	return 0;
+	return mlx5_tc_ct_flow_offload(parse_state->ct_priv, attr);
 }
 
 static bool
@@ -95,10 +47,16 @@ tc_act_is_multi_table_act_ct(struct mlx5e_priv *priv,
 	return true;
 }
 
+static bool
+tc_act_is_missable_ct(const struct flow_action_entry *act)
+{
+	return !(act->ct.action & TCA_CT_ACT_CLEAR);
+}
+
 struct mlx5e_tc_act mlx5e_tc_act_ct = {
-	.can_offload = tc_act_can_offload_ct,
 	.parse_action = tc_act_parse_ct,
-	.is_multi_table_act = tc_act_is_multi_table_act_ct,
 	.post_parse = tc_act_post_parse_ct,
+	.is_multi_table_act = tc_act_is_multi_table_act_ct,
+	.is_missable = tc_act_is_missable_ct,
 };
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/drop.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/drop.c
index 7d16aeabb119..5dc81715d625 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/drop.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/drop.c
@@ -4,15 +4,6 @@
 #include "act.h"
 #include "en/tc_priv.h"
 
-static bool
-tc_act_can_offload_drop(struct mlx5e_tc_act_parse_state *parse_state,
-			const struct flow_action_entry *act,
-			int act_index,
-			struct mlx5_flow_attr *attr)
-{
-	return true;
-}
-
 static int
 tc_act_parse_drop(struct mlx5e_tc_act_parse_state *parse_state,
 		  const struct flow_action_entry *act,
@@ -25,7 +16,6 @@ tc_act_parse_drop(struct mlx5e_tc_act_parse_state *parse_state,
 }
 
 struct mlx5e_tc_act mlx5e_tc_act_drop = {
-	.can_offload = tc_act_can_offload_drop,
 	.parse_action = tc_act_parse_drop,
 	.is_terminating_action = true,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c
index 07cc65596f89..291193f7120d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c
@@ -234,6 +234,9 @@ parse_mirred(struct mlx5e_tc_act_parse_state *parse_state,
 	if (mlx5_lag_mpesw_do_mirred(priv->mdev, out_dev, extack))
 		return -EOPNOTSUPP;
 
+	if (netif_is_macvlan(out_dev))
+		out_dev = macvlan_dev_real_dev(out_dev);
+
 	out_dev = get_fdb_out_dev(uplink_dev, out_dev);
 	if (!out_dev)
 		return -ENODEV;
@@ -250,9 +253,6 @@ parse_mirred(struct mlx5e_tc_act_parse_state *parse_state,
 			return err;
 	}
 
-	if (netif_is_macvlan(out_dev))
-		out_dev = macvlan_dev_real_dev(out_dev);
-
 	err = verify_uplink_forwarding(priv, attr, out_dev, extack);
 	if (err)
 		return err;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/pedit.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/pedit.c
index 47597c524e59..3b272bbf4c53 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/pedit.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/pedit.c
@@ -78,15 +78,6 @@ out_err:
 	return err;
 }
 
-static bool
-tc_act_can_offload_pedit(struct mlx5e_tc_act_parse_state *parse_state,
-			 const struct flow_action_entry *act,
-			 int act_index,
-			 struct mlx5_flow_attr *attr)
-{
-	return true;
-}
-
 static int
 tc_act_parse_pedit(struct mlx5e_tc_act_parse_state *parse_state,
 		   const struct flow_action_entry *act,
@@ -114,6 +105,5 @@ tc_act_parse_pedit(struct mlx5e_tc_act_parse_state *parse_state,
 }
 
 struct mlx5e_tc_act mlx5e_tc_act_pedit = {
-	.can_offload = tc_act_can_offload_pedit,
 	.parse_action = tc_act_parse_pedit,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/ptype.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/ptype.c
index 6454b031ff7a..80b4bc64380a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/ptype.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/ptype.c
@@ -4,15 +4,6 @@
 #include "act.h"
 #include "en/tc_priv.h"
 
-static bool
-tc_act_can_offload_ptype(struct mlx5e_tc_act_parse_state *parse_state,
-			 const struct flow_action_entry *act,
-			 int act_index,
-			 struct mlx5_flow_attr *attr)
-{
-	return true;
-}
-
 static int
 tc_act_parse_ptype(struct mlx5e_tc_act_parse_state *parse_state,
 		   const struct flow_action_entry *act,
@@ -31,6 +22,5 @@ tc_act_parse_ptype(struct mlx5e_tc_act_parse_state *parse_state,
 }
 
 struct mlx5e_tc_act mlx5e_tc_act_ptype = {
-	.can_offload = tc_act_can_offload_ptype,
 	.parse_action = tc_act_parse_ptype,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/sample.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/sample.c
index 2c0196431302..2df02f99cecf 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/sample.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/sample.c
@@ -6,25 +6,6 @@
 #include "en/tc_priv.h"
 #include "en/tc/act/sample.h"
 
-static bool
-tc_act_can_offload_sample(struct mlx5e_tc_act_parse_state *parse_state,
-			  const struct flow_action_entry *act,
-			  int act_index,
-			  struct mlx5_flow_attr *attr)
-{
-	struct netlink_ext_ack *extack = parse_state->extack;
-	bool ct_nat;
-
-	ct_nat = attr->ct_attr.ct_action & TCA_CT_ACT_NAT;
-
-	if (flow_flag_test(parse_state->flow, CT) && ct_nat) {
-		NL_SET_ERR_MSG_MOD(extack, "Sample action with CT NAT is not supported");
-		return false;
-	}
-
-	return true;
-}
-
 static int
 tc_act_parse_sample(struct mlx5e_tc_act_parse_state *parse_state,
 		    const struct flow_action_entry *act,
@@ -65,7 +46,6 @@ tc_act_is_multi_table_act_sample(struct mlx5e_priv *priv,
 }
 
 struct mlx5e_tc_act mlx5e_tc_act_sample = {
-	.can_offload = tc_act_can_offload_sample,
 	.parse_action = tc_act_parse_sample,
 	.is_multi_table_act = tc_act_is_multi_table_act_sample,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c
index 915ce201aeb2..1b78bd9c106a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c
@@ -5,15 +5,6 @@
 #include "en/tc_priv.h"
 #include "eswitch.h"
 
-static bool
-tc_act_can_offload_trap(struct mlx5e_tc_act_parse_state *parse_state,
-			const struct flow_action_entry *act,
-			int act_index,
-			struct mlx5_flow_attr *attr)
-{
-	return true;
-}
-
 static int
 tc_act_parse_trap(struct mlx5e_tc_act_parse_state *parse_state,
 		  const struct flow_action_entry *act,
@@ -27,6 +18,5 @@ tc_act_parse_trap(struct mlx5e_tc_act_parse_state *parse_state,
 }
 
 struct mlx5e_tc_act mlx5e_tc_act_trap = {
-	.can_offload = tc_act_can_offload_trap,
 	.parse_action = tc_act_parse_trap,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/tun.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/tun.c
index b4fa2de9711d..f1cae21c2c37 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/tun.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/tun.c
@@ -32,15 +32,6 @@ tc_act_parse_tun_encap(struct mlx5e_tc_act_parse_state *parse_state,
 	return 0;
 }
 
-static bool
-tc_act_can_offload_tun_decap(struct mlx5e_tc_act_parse_state *parse_state,
-			     const struct flow_action_entry *act,
-			     int act_index,
-			     struct mlx5_flow_attr *attr)
-{
-	return true;
-}
-
 static int
 tc_act_parse_tun_decap(struct mlx5e_tc_act_parse_state *parse_state,
 		       const struct flow_action_entry *act,
@@ -58,6 +49,5 @@ struct mlx5e_tc_act mlx5e_tc_act_tun_encap = {
 };
 
 struct mlx5e_tc_act mlx5e_tc_act_tun_decap = {
-	.can_offload = tc_act_can_offload_tun_decap,
 	.parse_action = tc_act_parse_tun_decap,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan.c
index 2e0d88b513aa..c8a3eaf189f6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan.c
@@ -141,15 +141,6 @@ mlx5e_tc_act_vlan_add_pop_action(struct mlx5e_priv *priv,
 	return err;
 }
 
-static bool
-tc_act_can_offload_vlan(struct mlx5e_tc_act_parse_state *parse_state,
-			const struct flow_action_entry *act,
-			int act_index,
-			struct mlx5_flow_attr *attr)
-{
-	return true;
-}
-
 static int
 tc_act_parse_vlan(struct mlx5e_tc_act_parse_state *parse_state,
 		  const struct flow_action_entry *act,
@@ -205,7 +196,6 @@ tc_act_post_parse_vlan(struct mlx5e_tc_act_parse_state *parse_state,
 }
 
 struct mlx5e_tc_act mlx5e_tc_act_vlan = {
-	.can_offload = tc_act_can_offload_vlan,
 	.parse_action = tc_act_parse_vlan,
 	.post_parse = tc_act_post_parse_vlan,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan_mangle.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan_mangle.c
index 9a8a1a6bd99e..310b99230760 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan_mangle.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan_mangle.c
@@ -50,15 +50,6 @@ mlx5e_tc_act_vlan_add_rewrite_action(struct mlx5e_priv *priv, int namespace,
 	return err;
 }
 
-static bool
-tc_act_can_offload_vlan_mangle(struct mlx5e_tc_act_parse_state *parse_state,
-			       const struct flow_action_entry *act,
-			       int act_index,
-			       struct mlx5_flow_attr *attr)
-{
-	return true;
-}
-
 static int
 tc_act_parse_vlan_mangle(struct mlx5e_tc_act_parse_state *parse_state,
 			 const struct flow_action_entry *act,
@@ -81,6 +72,5 @@ tc_act_parse_vlan_mangle(struct mlx5e_tc_act_parse_state *parse_state,
 }
 
 struct mlx5e_tc_act mlx5e_tc_act_vlan_mangle = {
-	.can_offload = tc_act_can_offload_vlan_mangle,
 	.parse_action = tc_act_parse_vlan_mangle,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/int_port.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/int_port.c
index ca834bbcb44f..8afcec0c5d3c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/int_port.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/int_port.c
@@ -242,7 +242,7 @@ mlx5e_int_port_remove(struct mlx5e_tc_int_port_priv *priv,
 		mlx5_del_flow_rules(int_port->rx_rule);
 	mapping_remove(ctx, int_port->mapping);
 	mlx5e_int_port_metadata_free(priv, int_port->match_metadata);
-	kfree_rcu(int_port);
+	kfree_rcu_mightsleep(int_port);
 	priv->num_ports--;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c
index 4e48946c4c2a..0290e0dea539 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c
@@ -106,22 +106,17 @@ err_rule:
 }
 
 struct mlx5e_post_act_handle *
-mlx5e_tc_post_act_add(struct mlx5e_post_act *post_act, struct mlx5_flow_attr *attr)
+mlx5e_tc_post_act_add(struct mlx5e_post_act *post_act, struct mlx5_flow_attr *post_attr)
 {
-	u32 attr_sz = ns_to_attr_sz(post_act->ns_type);
 	struct mlx5e_post_act_handle *handle;
-	struct mlx5_flow_attr *post_attr;
 	int err;
 
 	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
-	post_attr = mlx5_alloc_flow_attr(post_act->ns_type);
-	if (!handle || !post_attr) {
-		kfree(post_attr);
+	if (!handle) {
 		kfree(handle);
 		return ERR_PTR(-ENOMEM);
 	}
 
-	memcpy(post_attr, attr, attr_sz);
 	post_attr->chain = 0;
 	post_attr->prio = 0;
 	post_attr->ft = post_act->ft;
@@ -145,7 +140,6 @@ mlx5e_tc_post_act_add(struct mlx5e_post_act *post_act, struct mlx5_flow_attr *at
 	return handle;
 
 err_xarray:
-	kfree(post_attr);
 	kfree(handle);
 	return ERR_PTR(err);
 }
@@ -164,7 +158,6 @@ mlx5e_tc_post_act_del(struct mlx5e_post_act *post_act, struct mlx5e_post_act_han
 	if (!IS_ERR_OR_NULL(handle->rule))
 		mlx5e_tc_post_act_unoffload(post_act, handle);
 	xa_erase(&post_act->ids, handle->id);
-	kfree(handle->attr);
 	kfree(handle);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.h
index f476774c0b75..40b8df184af5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.h
@@ -19,7 +19,7 @@ void
 mlx5e_tc_post_act_destroy(struct mlx5e_post_act *post_act);
 
 struct mlx5e_post_act_handle *
-mlx5e_tc_post_act_add(struct mlx5e_post_act *post_act, struct mlx5_flow_attr *attr);
+mlx5e_tc_post_act_add(struct mlx5e_post_act *post_act, struct mlx5_flow_attr *post_attr);
 
 void
 mlx5e_tc_post_act_del(struct mlx5e_post_act *post_act, struct mlx5e_post_act_handle *handle);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c
index 558a776359af..5db239cae814 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c
@@ -14,10 +14,10 @@
 
 #define MLX5_ESW_VPORT_TBL_SIZE_SAMPLE (64 * 1024)
 
-static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_sample_ns = {
+static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_sample_ns = {
 	.max_fte = MLX5_ESW_VPORT_TBL_SIZE_SAMPLE,
 	.max_num_groups = 0,    /* default num of groups */
-	.flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | MLX5_FLOW_TABLE_TUNNEL_EN_DECAP,
+	.flags = 0,
 };
 
 struct mlx5e_tc_psample {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
index 314983bc6f08..ead38ef69483 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
@@ -83,12 +83,6 @@ struct mlx5_tc_ct_priv {
 	struct mlx5_tc_ct_debugfs debugfs;
 };
 
-struct mlx5_ct_flow {
-	struct mlx5_flow_attr *pre_ct_attr;
-	struct mlx5_flow_handle *pre_ct_rule;
-	struct mlx5_ct_ft *ft;
-};
-
 struct mlx5_ct_zone_rule {
 	struct mlx5_ct_fs_rule *rule;
 	struct mlx5e_mod_hdr_handle *mh;
@@ -598,12 +592,6 @@ mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv *ct_priv,
 	return 0;
 }
 
-int mlx5_tc_ct_set_ct_clear_regs(struct mlx5_tc_ct_priv *priv,
-				 struct mlx5e_tc_mod_hdr_acts *mod_acts)
-{
-		return mlx5_tc_ct_entry_set_registers(priv, mod_acts, 0, 0, 0, 0);
-}
-
 static int
 mlx5_tc_ct_parse_mangle_to_mod_act(struct flow_action_entry *act,
 				   char *modact)
@@ -920,6 +908,7 @@ mlx5_tc_ct_entry_replace_rule(struct mlx5_tc_ct_priv *ct_priv,
 	zone_rule->rule = rule;
 	mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, old_attr, zone_rule->mh);
 	zone_rule->mh = mh;
+	mlx5_put_label_mapping(ct_priv, old_attr->ct_attr.ct_labels_id);
 
 	kfree(old_attr);
 	kvfree(spec);
@@ -1545,7 +1534,6 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv,
 int
 mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
 			struct mlx5_flow_attr *attr,
-			struct mlx5e_tc_mod_hdr_acts *mod_acts,
 			const struct flow_action_entry *act,
 			struct netlink_ext_ack *extack)
 {
@@ -1555,8 +1543,8 @@ mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
 		return -EOPNOTSUPP;
 	}
 
+	attr->ct_attr.ct_action |= act->ct.action; /* So we can have clear + ct */
 	attr->ct_attr.zone = act->ct.zone;
-	attr->ct_attr.ct_action = act->ct.action;
 	attr->ct_attr.nf_ft = act->ct.flow_table;
 	attr->ct_attr.act_miss_cookie = act->miss_cookie;
 
@@ -1892,14 +1880,14 @@ mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft)
 
 /* We translate the tc filter with CT action to the following HW model:
  *
- *	+---------------------+
- *	+ ft prio (tc chain)  +
- *	+ original match      +
- *	+---------------------+
+ *	+-----------------------+
+ *	+ rule (either original +
+ *	+ or post_act rule)     +
+ *	+-----------------------+
  *		 | set act_miss_cookie mapping
  *		 | set fte_id
  *		 | set tunnel_id
- *		 | do decap
+ *		 | rest of actions before the CT action (for this orig/post_act rule)
  *		 |
  * +-------------+
  * | Chain 0	 |
@@ -1924,32 +1912,21 @@ mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft)
  *		 | do nat (if needed)
  *		 v
  *	+--------------+
- *	+ post_act     + original filter actions
+ *	+ post_act     + rest of parsed filter's actions
  *	+ fte_id match +------------------------>
  *	+--------------+
  *
  */
-static struct mlx5_flow_handle *
+static int
 __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv,
-			  struct mlx5_flow_spec *orig_spec,
 			  struct mlx5_flow_attr *attr)
 {
 	bool nat = attr->ct_attr.ct_action & TCA_CT_ACT_NAT;
 	struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
-	struct mlx5e_tc_mod_hdr_acts *pre_mod_acts;
-	u32 attr_sz = ns_to_attr_sz(ct_priv->ns_type);
-	struct mlx5_flow_attr *pre_ct_attr;
-	struct mlx5_modify_hdr *mod_hdr;
-	struct mlx5_ct_flow *ct_flow;
 	int act_miss_mapping = 0, err;
 	struct mlx5_ct_ft *ft;
 	u16 zone;
 
-	ct_flow = kzalloc(sizeof(*ct_flow), GFP_KERNEL);
-	if (!ct_flow) {
-		return ERR_PTR(-ENOMEM);
-	}
-
 	/* Register for CT established events */
 	ft = mlx5_tc_ct_add_ft_cb(ct_priv, attr->ct_attr.zone,
 				  attr->ct_attr.nf_ft);
@@ -1958,23 +1935,7 @@ __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv,
 		ct_dbg("Failed to register to ft callback");
 		goto err_ft;
 	}
-	ct_flow->ft = ft;
-
-	/* Base flow attributes of both rules on original rule attribute */
-	ct_flow->pre_ct_attr = mlx5_alloc_flow_attr(ct_priv->ns_type);
-	if (!ct_flow->pre_ct_attr) {
-		err = -ENOMEM;
-		goto err_alloc_pre;
-	}
-
-	pre_ct_attr = ct_flow->pre_ct_attr;
-	memcpy(pre_ct_attr, attr, attr_sz);
-	pre_mod_acts = &pre_ct_attr->parse_attr->mod_hdr_acts;
-
-	/* Modify the original rule's action to fwd and modify, leave decap */
-	pre_ct_attr->action = attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP;
-	pre_ct_attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
-			       MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+	attr->ct_attr.ft = ft;
 
 	err = mlx5e_tc_action_miss_mapping_get(ct_priv->priv, attr, attr->ct_attr.act_miss_cookie,
 					       &act_miss_mapping);
@@ -1982,136 +1943,89 @@ __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv,
 		ct_dbg("Failed to get register mapping for act miss");
 		goto err_get_act_miss;
 	}
-	attr->ct_attr.act_miss_mapping = act_miss_mapping;
 
-	err = mlx5e_tc_match_to_reg_set(priv->mdev, pre_mod_acts, ct_priv->ns_type,
-					MAPPED_OBJ_TO_REG, act_miss_mapping);
+	err = mlx5e_tc_match_to_reg_set(priv->mdev, &attr->parse_attr->mod_hdr_acts,
+					ct_priv->ns_type, MAPPED_OBJ_TO_REG, act_miss_mapping);
 	if (err) {
 		ct_dbg("Failed to set act miss register mapping");
 		goto err_mapping;
 	}
 
-	/* If original flow is decap, we do it before going into ct table
-	 * so add a rewrite for the tunnel match_id.
-	 */
-	if ((pre_ct_attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) &&
-	    attr->chain == 0) {
-		err = mlx5e_tc_match_to_reg_set(priv->mdev, pre_mod_acts,
-						ct_priv->ns_type,
-						TUNNEL_TO_REG,
-						attr->tunnel_id);
-		if (err) {
-			ct_dbg("Failed to set tunnel register mapping");
-			goto err_mapping;
-		}
-	}
-
-	/* Change original rule point to ct table
-	 * Chain 0 sets the zone and jumps to ct table
+	/* Chain 0 sets the zone and jumps to ct table
 	 * Other chains jump to pre_ct table to align with act_ct cached logic
 	 */
-	pre_ct_attr->dest_chain = 0;
 	if (!attr->chain) {
 		zone = ft->zone & MLX5_CT_ZONE_MASK;
-		err = mlx5e_tc_match_to_reg_set(priv->mdev, pre_mod_acts, ct_priv->ns_type,
-						ZONE_TO_REG, zone);
+		err = mlx5e_tc_match_to_reg_set(priv->mdev, &attr->parse_attr->mod_hdr_acts,
+						ct_priv->ns_type, ZONE_TO_REG, zone);
 		if (err) {
 			ct_dbg("Failed to set zone register mapping");
 			goto err_mapping;
 		}
 
-		pre_ct_attr->dest_ft = nat ? ct_priv->ct_nat : ct_priv->ct;
+		attr->dest_ft = nat ? ct_priv->ct_nat : ct_priv->ct;
 	} else {
-		pre_ct_attr->dest_ft = nat ? ft->pre_ct_nat.ft : ft->pre_ct.ft;
-	}
-
-	mod_hdr = mlx5_modify_header_alloc(priv->mdev, ct_priv->ns_type,
-					   pre_mod_acts->num_actions,
-					   pre_mod_acts->actions);
-	if (IS_ERR(mod_hdr)) {
-		err = PTR_ERR(mod_hdr);
-		ct_dbg("Failed to create pre ct mod hdr");
-		goto err_mapping;
-	}
-	pre_ct_attr->modify_hdr = mod_hdr;
-	ct_flow->pre_ct_rule = mlx5_tc_rule_insert(priv, orig_spec,
-						   pre_ct_attr);
-	if (IS_ERR(ct_flow->pre_ct_rule)) {
-		err = PTR_ERR(ct_flow->pre_ct_rule);
-		ct_dbg("Failed to add pre ct rule");
-		goto err_insert_orig;
+		attr->dest_ft = nat ? ft->pre_ct_nat.ft : ft->pre_ct.ft;
 	}
 
-	attr->ct_attr.ct_flow = ct_flow;
-	mlx5e_mod_hdr_dealloc(pre_mod_acts);
+	attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+	attr->ct_attr.act_miss_mapping = act_miss_mapping;
 
-	return ct_flow->pre_ct_rule;
+	return 0;
 
-err_insert_orig:
-	mlx5_modify_header_dealloc(priv->mdev, pre_ct_attr->modify_hdr);
 err_mapping:
-	mlx5e_mod_hdr_dealloc(pre_mod_acts);
 	mlx5e_tc_action_miss_mapping_put(ct_priv->priv, attr, act_miss_mapping);
 err_get_act_miss:
-	kfree(ct_flow->pre_ct_attr);
-err_alloc_pre:
 	mlx5_tc_ct_del_ft_cb(ct_priv, ft);
 err_ft:
-	kfree(ct_flow);
 	netdev_warn(priv->netdev, "Failed to offload ct flow, err %d\n", err);
-	return ERR_PTR(err);
+	return err;
 }
 
-struct mlx5_flow_handle *
-mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *priv,
-			struct mlx5_flow_spec *spec,
-			struct mlx5_flow_attr *attr,
-			struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts)
+int
+mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *priv, struct mlx5_flow_attr *attr)
 {
-	struct mlx5_flow_handle *rule;
+	int err;
 
 	if (!priv)
-		return ERR_PTR(-EOPNOTSUPP);
+		return -EOPNOTSUPP;
+
+	if (attr->ct_attr.ct_action & TCA_CT_ACT_CLEAR) {
+		err = mlx5_tc_ct_entry_set_registers(priv, &attr->parse_attr->mod_hdr_acts,
+						     0, 0, 0, 0);
+		if (err)
+			return err;
+
+		attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+	}
+
+	if (!attr->ct_attr.nf_ft) /* means only ct clear action, and not ct_clear,ct() */
+		return 0;
 
 	mutex_lock(&priv->control_lock);
-	rule = __mlx5_tc_ct_flow_offload(priv, spec, attr);
+	err = __mlx5_tc_ct_flow_offload(priv, attr);
 	mutex_unlock(&priv->control_lock);
 
-	return rule;
+	return err;
 }
 
 static void
 __mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *ct_priv,
-			 struct mlx5_ct_flow *ct_flow,
 			 struct mlx5_flow_attr *attr)
 {
-	struct mlx5_flow_attr *pre_ct_attr = ct_flow->pre_ct_attr;
-	struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
-
-	mlx5_tc_rule_delete(priv, ct_flow->pre_ct_rule, pre_ct_attr);
-	mlx5_modify_header_dealloc(priv->mdev, pre_ct_attr->modify_hdr);
-
 	mlx5e_tc_action_miss_mapping_put(ct_priv->priv, attr, attr->ct_attr.act_miss_mapping);
-	mlx5_tc_ct_del_ft_cb(ct_priv, ct_flow->ft);
-
-	kfree(ct_flow->pre_ct_attr);
-	kfree(ct_flow);
+	mlx5_tc_ct_del_ft_cb(ct_priv, attr->ct_attr.ft);
 }
 
 void
 mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *priv,
 		       struct mlx5_flow_attr *attr)
 {
-	struct mlx5_ct_flow *ct_flow = attr->ct_attr.ct_flow;
-
-	/* We are called on error to clean up stuff from parsing
-	 * but we don't have anything for now
-	 */
-	if (!ct_flow)
+	if (!attr->ct_attr.nf_ft) /* means only ct clear action, and not ct_clear,ct() */
 		return;
 
 	mutex_lock(&priv->control_lock);
-	__mlx5_tc_ct_delete_flow(priv, ct_flow, attr);
+	__mlx5_tc_ct_delete_flow(priv, attr);
 	mutex_unlock(&priv->control_lock);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h
index 5c5ddaa83055..8e9316fa46d4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h
@@ -25,11 +25,11 @@ struct nf_flowtable;
 struct mlx5_ct_attr {
 	u16 zone;
 	u16 ct_action;
-	struct mlx5_ct_flow *ct_flow;
 	struct nf_flowtable *nf_ft;
 	u32 ct_labels_id;
 	u32 act_miss_mapping;
 	u64 act_miss_cookie;
+	struct mlx5_ct_ft *ft;
 };
 
 #define zone_to_reg_ct {\
@@ -113,15 +113,12 @@ int mlx5_tc_ct_add_no_trk_match(struct mlx5_flow_spec *spec);
 int
 mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
 			struct mlx5_flow_attr *attr,
-			struct mlx5e_tc_mod_hdr_acts *mod_acts,
 			const struct flow_action_entry *act,
 			struct netlink_ext_ack *extack);
 
-struct mlx5_flow_handle *
-mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *priv,
-			struct mlx5_flow_spec *spec,
-			struct mlx5_flow_attr *attr,
-			struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts);
+int
+mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *priv, struct mlx5_flow_attr *attr);
+
 void
 mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *priv,
 		       struct mlx5_flow_attr *attr);
@@ -130,10 +127,6 @@ bool
 mlx5e_tc_ct_restore_flow(struct mlx5_tc_ct_priv *ct_priv,
 			 struct sk_buff *skb, u8 zone_restore_id);
 
-int
-mlx5_tc_ct_set_ct_clear_regs(struct mlx5_tc_ct_priv *priv,
-			     struct mlx5e_tc_mod_hdr_acts *mod_acts);
-
 #else /* CONFIG_MLX5_TC_CT */
 
 static inline struct mlx5_tc_ct_priv *
@@ -176,16 +169,8 @@ mlx5_tc_ct_add_no_trk_match(struct mlx5_flow_spec *spec)
 }
 
 static inline int
-mlx5_tc_ct_set_ct_clear_regs(struct mlx5_tc_ct_priv *priv,
-			     struct mlx5e_tc_mod_hdr_acts *mod_acts)
-{
-	return -EOPNOTSUPP;
-}
-
-static inline int
 mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
 			struct mlx5_flow_attr *attr,
-			struct mlx5e_tc_mod_hdr_acts *mod_acts,
 			const struct flow_action_entry *act,
 			struct netlink_ext_ack *extack)
 {
@@ -193,13 +178,11 @@ mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
 	return -EOPNOTSUPP;
 }
 
-static inline struct mlx5_flow_handle *
+static inline int
 mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *priv,
-			struct mlx5_flow_spec *spec,
-			struct mlx5_flow_attr *attr,
-			struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts)
+			struct mlx5_flow_attr *attr)
 {
-	return ERR_PTR(-EOPNOTSUPP);
+	return -EOPNOTSUPP;
 }
 
 static inline void
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
index 451fd4342a5a..ba2b1f24ff14 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
@@ -25,12 +25,11 @@ enum {
 	MLX5E_TC_FLOW_FLAG_DUP                   = MLX5E_TC_FLOW_BASE + 4,
 	MLX5E_TC_FLOW_FLAG_NOT_READY             = MLX5E_TC_FLOW_BASE + 5,
 	MLX5E_TC_FLOW_FLAG_DELETED               = MLX5E_TC_FLOW_BASE + 6,
-	MLX5E_TC_FLOW_FLAG_CT                    = MLX5E_TC_FLOW_BASE + 7,
-	MLX5E_TC_FLOW_FLAG_L3_TO_L2_DECAP        = MLX5E_TC_FLOW_BASE + 8,
-	MLX5E_TC_FLOW_FLAG_TUN_RX                = MLX5E_TC_FLOW_BASE + 9,
-	MLX5E_TC_FLOW_FLAG_FAILED                = MLX5E_TC_FLOW_BASE + 10,
-	MLX5E_TC_FLOW_FLAG_SAMPLE                = MLX5E_TC_FLOW_BASE + 11,
-	MLX5E_TC_FLOW_FLAG_USE_ACT_STATS	 = MLX5E_TC_FLOW_BASE + 12,
+	MLX5E_TC_FLOW_FLAG_L3_TO_L2_DECAP        = MLX5E_TC_FLOW_BASE + 7,
+	MLX5E_TC_FLOW_FLAG_TUN_RX                = MLX5E_TC_FLOW_BASE + 8,
+	MLX5E_TC_FLOW_FLAG_FAILED                = MLX5E_TC_FLOW_BASE + 9,
+	MLX5E_TC_FLOW_FLAG_SAMPLE                = MLX5E_TC_FLOW_BASE + 10,
+	MLX5E_TC_FLOW_FLAG_USE_ACT_STATS         = MLX5E_TC_FLOW_BASE + 11,
 };
 
 struct mlx5e_tc_flow_parse_attr {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.h
index b38f693bbb52..92065568bb19 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.h
@@ -115,6 +115,9 @@ int mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv *priv,
 bool mlx5e_tc_tun_encap_info_equal_generic(struct mlx5e_encap_key *a,
 					   struct mlx5e_encap_key *b);
 
+bool mlx5e_tc_tun_encap_info_equal_options(struct mlx5e_encap_key *a,
+					   struct mlx5e_encap_key *b,
+					   __be16 tun_flags);
 #endif /* CONFIG_MLX5_ESWITCH */
 
 #endif //__MLX5_EN_TC_TUNNEL_H__
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c
index 780224fd67a1..20c2d2ecaf93 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c
@@ -3,6 +3,7 @@
 
 #include <net/fib_notifier.h>
 #include <net/nexthop.h>
+#include <net/ip_tunnels.h>
 #include "tc_tun_encap.h"
 #include "en_tc.h"
 #include "tc_tun.h"
@@ -97,7 +98,6 @@ int mlx5e_tc_set_attr_rx_tun(struct mlx5e_tc_flow *flow,
 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
 	else if (ip_version == 6) {
 		int ipv6_size = MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6);
-		struct in6_addr zerov6 = {};
 
 		daddr = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 				     outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
@@ -105,8 +105,8 @@ int mlx5e_tc_set_attr_rx_tun(struct mlx5e_tc_flow *flow,
 				     outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6);
 		memcpy(&tun_attr->dst_ip.v6, daddr, ipv6_size);
 		memcpy(&tun_attr->src_ip.v6, saddr, ipv6_size);
-		if (!memcmp(&tun_attr->dst_ip.v6, &zerov6, sizeof(zerov6)) ||
-		    !memcmp(&tun_attr->src_ip.v6, &zerov6, sizeof(zerov6)))
+		if (ipv6_addr_any(&tun_attr->dst_ip.v6) ||
+		    ipv6_addr_any(&tun_attr->src_ip.v6))
 			return 0;
 	}
 #endif
@@ -571,6 +571,37 @@ bool mlx5e_tc_tun_encap_info_equal_generic(struct mlx5e_encap_key *a,
 		a->tc_tunnel->tunnel_type == b->tc_tunnel->tunnel_type;
 }
 
+bool mlx5e_tc_tun_encap_info_equal_options(struct mlx5e_encap_key *a,
+					   struct mlx5e_encap_key *b,
+					   __be16 tun_flags)
+{
+	struct ip_tunnel_info *a_info;
+	struct ip_tunnel_info *b_info;
+	bool a_has_opts, b_has_opts;
+
+	if (!mlx5e_tc_tun_encap_info_equal_generic(a, b))
+		return false;
+
+	a_has_opts = !!(a->ip_tun_key->tun_flags & tun_flags);
+	b_has_opts = !!(b->ip_tun_key->tun_flags & tun_flags);
+
+	/* keys are equal when both don't have any options attached */
+	if (!a_has_opts && !b_has_opts)
+		return true;
+
+	if (a_has_opts != b_has_opts)
+		return false;
+
+	/* options stored in memory next to ip_tunnel_info struct */
+	a_info = container_of(a->ip_tun_key, struct ip_tunnel_info, key);
+	b_info = container_of(b->ip_tun_key, struct ip_tunnel_info, key);
+
+	return a_info->options_len == b_info->options_len &&
+	       !memcmp(ip_tunnel_info_opts(a_info),
+		       ip_tunnel_info_opts(b_info),
+		       a_info->options_len);
+}
+
 static int cmp_decap_info(struct mlx5e_decap_key *a,
 			  struct mlx5e_decap_key *b)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_geneve.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_geneve.c
index 054d80c4e65c..2bcd10b6d653 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_geneve.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_geneve.c
@@ -337,29 +337,7 @@ static int mlx5e_tc_tun_parse_geneve(struct mlx5e_priv *priv,
 static bool mlx5e_tc_tun_encap_info_equal_geneve(struct mlx5e_encap_key *a,
 						 struct mlx5e_encap_key *b)
 {
-	struct ip_tunnel_info *a_info;
-	struct ip_tunnel_info *b_info;
-	bool a_has_opts, b_has_opts;
-
-	if (!mlx5e_tc_tun_encap_info_equal_generic(a, b))
-		return false;
-
-	a_has_opts = !!(a->ip_tun_key->tun_flags & TUNNEL_GENEVE_OPT);
-	b_has_opts = !!(b->ip_tun_key->tun_flags & TUNNEL_GENEVE_OPT);
-
-	/* keys are equal when both don't have any options attached */
-	if (!a_has_opts && !b_has_opts)
-		return true;
-
-	if (a_has_opts != b_has_opts)
-		return false;
-
-	/* geneve options stored in memory next to ip_tunnel_info struct */
-	a_info = container_of(a->ip_tun_key, struct ip_tunnel_info, key);
-	b_info = container_of(b->ip_tun_key, struct ip_tunnel_info, key);
-
-	return a_info->options_len == b_info->options_len &&
-		memcmp(a_info + 1, b_info + 1, a_info->options_len) == 0;
+	return mlx5e_tc_tun_encap_info_equal_options(a, b, TUNNEL_GENEVE_OPT);
 }
 
 struct mlx5e_tc_tunnel geneve_tunnel = {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c
index 1f62c702b625..a184d739d5f8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
 /* Copyright (c) 2018 Mellanox Technologies. */
 
+#include <net/ip_tunnels.h>
 #include <net/vxlan.h>
 #include "lib/vxlan.h"
 #include "en/tc_tun.h"
@@ -86,9 +87,11 @@ static int mlx5e_gen_ip_tunnel_header_vxlan(char buf[],
 	const struct ip_tunnel_key *tun_key = &e->tun_info->key;
 	__be32 tun_id = tunnel_id_to_key32(tun_key->tun_id);
 	struct udphdr *udp = (struct udphdr *)(buf);
+	const struct vxlan_metadata *md;
 	struct vxlanhdr *vxh;
 
-	if (tun_key->tun_flags & TUNNEL_VXLAN_OPT)
+	if ((tun_key->tun_flags & TUNNEL_VXLAN_OPT) &&
+	    e->tun_info->options_len != sizeof(*md))
 		return -EOPNOTSUPP;
 	vxh = (struct vxlanhdr *)((char *)udp + sizeof(struct udphdr));
 	*ip_proto = IPPROTO_UDP;
@@ -96,6 +99,57 @@ static int mlx5e_gen_ip_tunnel_header_vxlan(char buf[],
 	udp->dest = tun_key->tp_dst;
 	vxh->vx_flags = VXLAN_HF_VNI;
 	vxh->vx_vni = vxlan_vni_field(tun_id);
+	if (tun_key->tun_flags & TUNNEL_VXLAN_OPT) {
+		md = ip_tunnel_info_opts(e->tun_info);
+		vxlan_build_gbp_hdr(vxh, md);
+	}
+
+	return 0;
+}
+
+static int mlx5e_tc_tun_parse_vxlan_gbp_option(struct mlx5e_priv *priv,
+					       struct mlx5_flow_spec *spec,
+					       struct flow_cls_offload *f)
+{
+	struct flow_rule *rule = flow_cls_offload_flow_rule(f);
+	struct netlink_ext_ack *extack = f->common.extack;
+	struct flow_match_enc_opts enc_opts;
+	void *misc5_c, *misc5_v;
+	u32 *gbp, *gbp_mask;
+
+	flow_rule_match_enc_opts(rule, &enc_opts);
+
+	if (memchr_inv(&enc_opts.mask->data, 0, sizeof(enc_opts.mask->data)) &&
+	    !MLX5_CAP_ESW_FT_FIELD_SUPPORT_2(priv->mdev, tunnel_header_0_1)) {
+		NL_SET_ERR_MSG_MOD(extack, "Matching on VxLAN GBP is not supported");
+		return -EOPNOTSUPP;
+	}
+
+	if (enc_opts.key->dst_opt_type != TUNNEL_VXLAN_OPT) {
+		NL_SET_ERR_MSG_MOD(extack, "Wrong VxLAN option type: not GBP");
+		return -EOPNOTSUPP;
+	}
+
+	if (enc_opts.key->len != sizeof(*gbp) ||
+	    enc_opts.mask->len != sizeof(*gbp_mask)) {
+		NL_SET_ERR_MSG_MOD(extack, "VxLAN GBP option/mask len is not 32 bits");
+		return -EINVAL;
+	}
+
+	gbp = (u32 *)&enc_opts.key->data[0];
+	gbp_mask = (u32 *)&enc_opts.mask->data[0];
+
+	if (*gbp_mask & ~VXLAN_GBP_MASK) {
+		NL_SET_ERR_MSG_FMT_MOD(extack, "Wrong VxLAN GBP mask(0x%08X)\n", *gbp_mask);
+		return -EINVAL;
+	}
+
+	misc5_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_5);
+	misc5_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_5);
+	MLX5_SET(fte_match_set_misc5, misc5_c, tunnel_header_0, *gbp_mask);
+	MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_0, *gbp);
+
+	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5;
 
 	return 0;
 }
@@ -122,6 +176,14 @@ static int mlx5e_tc_tun_parse_vxlan(struct mlx5e_priv *priv,
 	if (!enc_keyid.mask->keyid)
 		return 0;
 
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_OPTS)) {
+		int err;
+
+		err = mlx5e_tc_tun_parse_vxlan_gbp_option(priv, spec, f);
+		if (err)
+			return err;
+	}
+
 	/* match on VNI is required */
 
 	if (!MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev,
@@ -143,6 +205,12 @@ static int mlx5e_tc_tun_parse_vxlan(struct mlx5e_priv *priv,
 	return 0;
 }
 
+static bool mlx5e_tc_tun_encap_info_equal_vxlan(struct mlx5e_encap_key *a,
+						struct mlx5e_encap_key *b)
+{
+	return mlx5e_tc_tun_encap_info_equal_options(a, b, TUNNEL_VXLAN_OPT);
+}
+
 static int mlx5e_tc_tun_get_remote_ifindex(struct net_device *mirred_dev)
 {
 	const struct vxlan_dev *vxlan = netdev_priv(mirred_dev);
@@ -160,6 +228,6 @@ struct mlx5e_tc_tunnel vxlan_tunnel = {
 	.generate_ip_tun_hdr  = mlx5e_gen_ip_tunnel_header_vxlan,
 	.parse_udp_ports      = mlx5e_tc_tun_parse_udp_ports_vxlan,
 	.parse_tunnel         = mlx5e_tc_tun_parse_vxlan,
-	.encap_info_equal     = mlx5e_tc_tun_encap_info_equal_generic,
+	.encap_info_equal     = mlx5e_tc_tun_encap_info_equal_vxlan,
 	.get_remote_ifindex   = mlx5e_tc_tun_get_remote_ifindex,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
index b9c2f67d3794..47381e949f1f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
@@ -65,13 +65,11 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget);
 int mlx5e_poll_ico_cq(struct mlx5e_cq *cq);
 
 /* RX */
-void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct page *page);
-void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, struct page *page, bool recycle);
 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq));
 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq));
 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
 void mlx5e_free_rx_descs(struct mlx5e_rq *rq);
-void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq);
+void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq);
 
 static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
 {
@@ -79,6 +77,19 @@ static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
 }
 
 /* TX */
+struct mlx5e_xmit_data {
+	dma_addr_t  dma_addr;
+	void       *data;
+	u32         len : 31;
+	u32         has_frags : 1;
+};
+
+struct mlx5e_xmit_data_frags {
+	struct mlx5e_xmit_data xd;
+	struct skb_shared_info *sinfo;
+	dma_addr_t *dma_arr;
+};
+
 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev);
 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
 void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
@@ -86,7 +97,7 @@ void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
 static inline bool
 mlx5e_skb_fifo_has_room(struct mlx5e_skb_fifo *fifo)
 {
-	return (u16)(*fifo->pc - *fifo->cc) < fifo->mask;
+	return (u16)(*fifo->pc - *fifo->cc) <= fifo->mask;
 }
 
 static inline bool
@@ -489,7 +500,7 @@ static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size
 
 static inline struct mlx5e_mpw_info *mlx5e_get_mpw_info(struct mlx5e_rq *rq, int i)
 {
-	size_t isz = struct_size(rq->mpwqe.info, alloc_units, rq->mpwqe.pages_per_wqe);
+	size_t isz = struct_size(rq->mpwqe.info, alloc_units.frag_pages, rq->mpwqe.pages_per_wqe);
 
 	return (struct mlx5e_mpw_info *)((char *)rq->mpwqe.info + array_size(i, isz));
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index d9d3b9e1f15a..f0e6095809fa 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -61,9 +61,8 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
 		    struct xdp_buff *xdp)
 {
 	struct page *page = virt_to_page(xdp->data);
-	struct skb_shared_info *sinfo = NULL;
-	struct mlx5e_xmit_data xdptxd;
-	struct mlx5e_xdp_info xdpi;
+	struct mlx5e_xmit_data_frags xdptxdf = {};
+	struct mlx5e_xmit_data *xdptxd;
 	struct xdp_frame *xdpf;
 	dma_addr_t dma_addr;
 	int i;
@@ -72,8 +71,10 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
 	if (unlikely(!xdpf))
 		return false;
 
-	xdptxd.data = xdpf->data;
-	xdptxd.len  = xdpf->len;
+	xdptxd = &xdptxdf.xd;
+	xdptxd->data = xdpf->data;
+	xdptxd->len  = xdpf->len;
+	xdptxd->has_frags = xdp_frame_has_frags(xdpf);
 
 	if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) {
 		/* The xdp_buff was in the UMEM and was copied into a newly
@@ -88,24 +89,29 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
 		 */
 		__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
 
-		xdpi.mode = MLX5E_XDP_XMIT_MODE_FRAME;
+		if (unlikely(xdptxd->has_frags))
+			return false;
 
-		dma_addr = dma_map_single(sq->pdev, xdptxd.data, xdptxd.len,
+		dma_addr = dma_map_single(sq->pdev, xdptxd->data, xdptxd->len,
 					  DMA_TO_DEVICE);
 		if (dma_mapping_error(sq->pdev, dma_addr)) {
 			xdp_return_frame(xdpf);
 			return false;
 		}
 
-		xdptxd.dma_addr     = dma_addr;
-		xdpi.frame.xdpf     = xdpf;
-		xdpi.frame.dma_addr = dma_addr;
+		xdptxd->dma_addr = dma_addr;
 
 		if (unlikely(!INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
-					      mlx5e_xmit_xdp_frame, sq, &xdptxd, NULL, 0)))
+					      mlx5e_xmit_xdp_frame, sq, xdptxd, 0)))
 			return false;
 
-		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, &xdpi);
+		/* xmit_mode == MLX5E_XDP_XMIT_MODE_FRAME */
+		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+				     (union mlx5e_xdp_info) { .mode = MLX5E_XDP_XMIT_MODE_FRAME });
+		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+				     (union mlx5e_xdp_info) { .frame.xdpf = xdpf });
+		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+				     (union mlx5e_xdp_info) { .frame.dma_addr = dma_addr });
 		return true;
 	}
 
@@ -115,17 +121,15 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
 	 * mode.
 	 */
 
-	xdpi.mode = MLX5E_XDP_XMIT_MODE_PAGE;
-	xdpi.page.rq = rq;
-
 	dma_addr = page_pool_get_dma_addr(page) + (xdpf->data - (void *)xdpf);
-	dma_sync_single_for_device(sq->pdev, dma_addr, xdptxd.len, DMA_BIDIRECTIONAL);
+	dma_sync_single_for_device(sq->pdev, dma_addr, xdptxd->len, DMA_BIDIRECTIONAL);
 
-	if (unlikely(xdp_frame_has_frags(xdpf))) {
-		sinfo = xdp_get_shared_info_from_frame(xdpf);
+	if (xdptxd->has_frags) {
+		xdptxdf.sinfo = xdp_get_shared_info_from_frame(xdpf);
+		xdptxdf.dma_arr = NULL;
 
-		for (i = 0; i < sinfo->nr_frags; i++) {
-			skb_frag_t *frag = &sinfo->frags[i];
+		for (i = 0; i < xdptxdf.sinfo->nr_frags; i++) {
+			skb_frag_t *frag = &xdptxdf.sinfo->frags[i];
 			dma_addr_t addr;
 			u32 len;
 
@@ -137,22 +141,34 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
 		}
 	}
 
-	xdptxd.dma_addr = dma_addr;
+	xdptxd->dma_addr = dma_addr;
 
 	if (unlikely(!INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
-				      mlx5e_xmit_xdp_frame, sq, &xdptxd, sinfo, 0)))
+				      mlx5e_xmit_xdp_frame, sq, xdptxd, 0)))
 		return false;
 
-	xdpi.page.page = page;
-	mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, &xdpi);
-
-	if (unlikely(xdp_frame_has_frags(xdpf))) {
-		for (i = 0; i < sinfo->nr_frags; i++) {
-			skb_frag_t *frag = &sinfo->frags[i];
-
-			xdpi.page.page = skb_frag_page(frag);
-			mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, &xdpi);
+	/* xmit_mode == MLX5E_XDP_XMIT_MODE_PAGE */
+	mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+			     (union mlx5e_xdp_info) { .mode = MLX5E_XDP_XMIT_MODE_PAGE });
+
+	if (xdptxd->has_frags) {
+		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+				     (union mlx5e_xdp_info)
+				     { .page.num = 1 + xdptxdf.sinfo->nr_frags });
+		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+				     (union mlx5e_xdp_info) { .page.page = page });
+		for (i = 0; i < xdptxdf.sinfo->nr_frags; i++) {
+			skb_frag_t *frag = &xdptxdf.sinfo->frags[i];
+
+			mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+					     (union mlx5e_xdp_info)
+					     { .page.page = skb_frag_page(frag) });
 		}
+	} else {
+		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+				     (union mlx5e_xdp_info) { .page.num = 1 });
+		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+				     (union mlx5e_xdp_info) { .page.page = page });
 	}
 
 	return true;
@@ -268,8 +284,6 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
 			goto xdp_abort;
 		__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
 		__set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags);
-		if (xdp->rxq->mem.type != MEM_TYPE_XSK_BUFF_POOL)
-			mlx5e_page_dma_unmap(rq, virt_to_page(xdp->data));
 		rq->stats->xdp_redirect++;
 		return true;
 	default:
@@ -383,26 +397,43 @@ INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq
 
 INDIRECT_CALLABLE_SCOPE bool
 mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
-		     struct skb_shared_info *sinfo, int check_result);
+		     int check_result);
 
 INDIRECT_CALLABLE_SCOPE bool
 mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
-			   struct skb_shared_info *sinfo, int check_result)
+			   int check_result)
 {
 	struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
 	struct mlx5e_xdpsq_stats *stats = sq->stats;
+	struct mlx5e_xmit_data *p = xdptxd;
+	struct mlx5e_xmit_data tmp;
 
-	if (unlikely(sinfo)) {
-		/* MPWQE is enabled, but a multi-buffer packet is queued for
-		 * transmission. MPWQE can't send fragmented packets, so close
-		 * the current session and fall back to a regular WQE.
-		 */
-		if (unlikely(sq->mpwqe.wqe))
-			mlx5e_xdp_mpwqe_complete(sq);
-		return mlx5e_xmit_xdp_frame(sq, xdptxd, sinfo, 0);
+	if (xdptxd->has_frags) {
+		struct mlx5e_xmit_data_frags *xdptxdf =
+			container_of(xdptxd, struct mlx5e_xmit_data_frags, xd);
+
+		if (!!xdptxd->len + xdptxdf->sinfo->nr_frags > 1) {
+			/* MPWQE is enabled, but a multi-buffer packet is queued for
+			 * transmission. MPWQE can't send fragmented packets, so close
+			 * the current session and fall back to a regular WQE.
+			 */
+			if (unlikely(sq->mpwqe.wqe))
+				mlx5e_xdp_mpwqe_complete(sq);
+			return mlx5e_xmit_xdp_frame(sq, xdptxd, 0);
+		}
+		if (!xdptxd->len) {
+			skb_frag_t *frag = &xdptxdf->sinfo->frags[0];
+
+			tmp.data = skb_frag_address(frag);
+			tmp.len = skb_frag_size(frag);
+			tmp.dma_addr = xdptxdf->dma_arr ? xdptxdf->dma_arr[0] :
+				page_pool_get_dma_addr(skb_frag_page(frag)) +
+				skb_frag_off(frag);
+			p = &tmp;
+		}
 	}
 
-	if (unlikely(xdptxd->len > sq->hw_mtu)) {
+	if (unlikely(p->len > sq->hw_mtu)) {
 		stats->err++;
 		return false;
 	}
@@ -420,7 +451,7 @@ mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptx
 		mlx5e_xdp_mpwqe_session_start(sq);
 	}
 
-	mlx5e_xdp_mpwqe_add_dseg(sq, xdptxd, stats);
+	mlx5e_xdp_mpwqe_add_dseg(sq, p, stats);
 
 	if (unlikely(mlx5e_xdp_mpwqe_is_full(session, sq->max_sq_mpw_wqebbs)))
 		mlx5e_xdp_mpwqe_complete(sq);
@@ -448,8 +479,10 @@ INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)
 
 INDIRECT_CALLABLE_SCOPE bool
 mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
-		     struct skb_shared_info *sinfo, int check_result)
+		     int check_result)
 {
+	struct mlx5e_xmit_data_frags *xdptxdf =
+		container_of(xdptxd, struct mlx5e_xmit_data_frags, xd);
 	struct mlx5_wq_cyc       *wq   = &sq->wq;
 	struct mlx5_wqe_ctrl_seg *cseg;
 	struct mlx5_wqe_data_seg *dseg;
@@ -461,26 +494,34 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
 	u16 ds_cnt, inline_hdr_sz;
 	u8 num_wqebbs = 1;
 	int num_frags = 0;
+	bool inline_ok;
+	bool linear;
 	u16 pi;
 
 	struct mlx5e_xdpsq_stats *stats = sq->stats;
 
-	if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || sq->hw_mtu < dma_len)) {
+	inline_ok = sq->min_inline_mode == MLX5_INLINE_MODE_NONE ||
+		dma_len >= MLX5E_XDP_MIN_INLINE;
+
+	if (unlikely(!inline_ok || sq->hw_mtu < dma_len)) {
 		stats->err++;
 		return false;
 	}
 
-	ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT + 1;
+	inline_hdr_sz = 0;
 	if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE)
-		ds_cnt++;
+		inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
+
+	linear = !!(dma_len - inline_hdr_sz);
+	ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT + linear + !!inline_hdr_sz;
 
 	/* check_result must be 0 if sinfo is passed. */
 	if (!check_result) {
 		int stop_room = 1;
 
-		if (unlikely(sinfo)) {
-			ds_cnt += sinfo->nr_frags;
-			num_frags = sinfo->nr_frags;
+		if (xdptxd->has_frags) {
+			ds_cnt += xdptxdf->sinfo->nr_frags;
+			num_frags = xdptxdf->sinfo->nr_frags;
 			num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
 			/* Assuming MLX5_CAP_GEN(mdev, max_wqe_sz_sq) is big
 			 * enough to hold all fragments.
@@ -501,53 +542,53 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
 	eseg = &wqe->eth;
 	dseg = wqe->data;
 
-	inline_hdr_sz = 0;
-
 	/* copy the inline part if required */
-	if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
+	if (inline_hdr_sz) {
 		memcpy(eseg->inline_hdr.start, xdptxd->data, sizeof(eseg->inline_hdr.start));
 		memcpy(dseg, xdptxd->data + sizeof(eseg->inline_hdr.start),
-		       MLX5E_XDP_MIN_INLINE - sizeof(eseg->inline_hdr.start));
-		dma_len  -= MLX5E_XDP_MIN_INLINE;
-		dma_addr += MLX5E_XDP_MIN_INLINE;
-		inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
+		       inline_hdr_sz - sizeof(eseg->inline_hdr.start));
+		dma_len  -= inline_hdr_sz;
+		dma_addr += inline_hdr_sz;
 		dseg++;
 	}
 
 	/* write the dma part */
-	dseg->addr       = cpu_to_be64(dma_addr);
-	dseg->byte_count = cpu_to_be32(dma_len);
+	if (linear) {
+		dseg->addr       = cpu_to_be64(dma_addr);
+		dseg->byte_count = cpu_to_be32(dma_len);
+		dseg->lkey       = sq->mkey_be;
+		dseg++;
+	}
 
 	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_SEND);
 
-	if (unlikely(test_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state))) {
-		u8 num_pkts = 1 + num_frags;
+	if (test_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state)) {
 		int i;
 
 		memset(&cseg->trailer, 0, sizeof(cseg->trailer));
 		memset(eseg, 0, sizeof(*eseg) - sizeof(eseg->trailer));
 
 		eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
-		dseg->lkey = sq->mkey_be;
 
 		for (i = 0; i < num_frags; i++) {
-			skb_frag_t *frag = &sinfo->frags[i];
+			skb_frag_t *frag = &xdptxdf->sinfo->frags[i];
 			dma_addr_t addr;
 
-			addr = page_pool_get_dma_addr(skb_frag_page(frag)) +
+			addr = xdptxdf->dma_arr ? xdptxdf->dma_arr[i] :
+				page_pool_get_dma_addr(skb_frag_page(frag)) +
 				skb_frag_off(frag);
 
-			dseg++;
 			dseg->addr = cpu_to_be64(addr);
 			dseg->byte_count = cpu_to_be32(skb_frag_size(frag));
 			dseg->lkey = sq->mkey_be;
+			dseg++;
 		}
 
 		cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
 
 		sq->db.wqe_info[pi] = (struct mlx5e_xdp_wqe_info) {
 			.num_wqebbs = num_wqebbs,
-			.num_pkts = num_pkts,
+			.num_pkts = 1,
 		};
 
 		sq->pc += num_wqebbs;
@@ -566,26 +607,67 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
 static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
 				  struct mlx5e_xdp_wqe_info *wi,
 				  u32 *xsk_frames,
-				  bool recycle,
 				  struct xdp_frame_bulk *bq)
 {
 	struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
 	u16 i;
 
 	for (i = 0; i < wi->num_pkts; i++) {
-		struct mlx5e_xdp_info xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
+		union mlx5e_xdp_info xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
 
 		switch (xdpi.mode) {
-		case MLX5E_XDP_XMIT_MODE_FRAME:
+		case MLX5E_XDP_XMIT_MODE_FRAME: {
 			/* XDP_TX from the XSK RQ and XDP_REDIRECT */
-			dma_unmap_single(sq->pdev, xdpi.frame.dma_addr,
-					 xdpi.frame.xdpf->len, DMA_TO_DEVICE);
-			xdp_return_frame_bulk(xdpi.frame.xdpf, bq);
+			struct xdp_frame *xdpf;
+			dma_addr_t dma_addr;
+
+			xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
+			xdpf = xdpi.frame.xdpf;
+			xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
+			dma_addr = xdpi.frame.dma_addr;
+
+			dma_unmap_single(sq->pdev, dma_addr,
+					 xdpf->len, DMA_TO_DEVICE);
+			if (xdp_frame_has_frags(xdpf)) {
+				struct skb_shared_info *sinfo;
+				int j;
+
+				sinfo = xdp_get_shared_info_from_frame(xdpf);
+				for (j = 0; j < sinfo->nr_frags; j++) {
+					skb_frag_t *frag = &sinfo->frags[j];
+
+					xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
+					dma_addr = xdpi.frame.dma_addr;
+
+					dma_unmap_single(sq->pdev, dma_addr,
+							 skb_frag_size(frag), DMA_TO_DEVICE);
+				}
+			}
+			xdp_return_frame_bulk(xdpf, bq);
 			break;
-		case MLX5E_XDP_XMIT_MODE_PAGE:
+		}
+		case MLX5E_XDP_XMIT_MODE_PAGE: {
 			/* XDP_TX from the regular RQ */
-			mlx5e_page_release_dynamic(xdpi.page.rq, xdpi.page.page, recycle);
+			u8 num, n = 0;
+
+			xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
+			num = xdpi.page.num;
+
+			do {
+				struct page *page;
+
+				xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
+				page = xdpi.page.page;
+
+				/* No need to check ((page->pp_magic & ~0x3UL) == PP_SIGNATURE)
+				 * as we know this is a page_pool page.
+				 */
+				page_pool_put_defragged_page(page->pp,
+							     page, -1, true);
+			} while (++n < num);
+
 			break;
+		}
 		case MLX5E_XDP_XMIT_MODE_XSK:
 			/* AF_XDP send */
 			(*xsk_frames)++;
@@ -638,7 +720,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
 
 			sqcc += wi->num_wqebbs;
 
-			mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, true, &bq);
+			mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, &bq);
 		} while (!last_wqe);
 
 		if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
@@ -685,7 +767,7 @@ void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
 
 		sq->cc += wi->num_wqebbs;
 
-		mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, false, &bq);
+		mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, &bq);
 	}
 
 	xdp_flush_frame_bulk(&bq);
@@ -719,34 +801,79 @@ int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
 	sq = &priv->channels.c[sq_num]->xdpsq;
 
 	for (i = 0; i < n; i++) {
+		struct mlx5e_xmit_data_frags xdptxdf = {};
 		struct xdp_frame *xdpf = frames[i];
-		struct mlx5e_xmit_data xdptxd;
-		struct mlx5e_xdp_info xdpi;
+		dma_addr_t dma_arr[MAX_SKB_FRAGS];
+		struct mlx5e_xmit_data *xdptxd;
 		bool ret;
 
-		xdptxd.data = xdpf->data;
-		xdptxd.len = xdpf->len;
-		xdptxd.dma_addr = dma_map_single(sq->pdev, xdptxd.data,
-						 xdptxd.len, DMA_TO_DEVICE);
+		xdptxd = &xdptxdf.xd;
+		xdptxd->data = xdpf->data;
+		xdptxd->len = xdpf->len;
+		xdptxd->has_frags = xdp_frame_has_frags(xdpf);
+		xdptxd->dma_addr = dma_map_single(sq->pdev, xdptxd->data,
+						  xdptxd->len, DMA_TO_DEVICE);
 
-		if (unlikely(dma_mapping_error(sq->pdev, xdptxd.dma_addr)))
+		if (unlikely(dma_mapping_error(sq->pdev, xdptxd->dma_addr)))
 			break;
 
-		xdpi.mode           = MLX5E_XDP_XMIT_MODE_FRAME;
-		xdpi.frame.xdpf     = xdpf;
-		xdpi.frame.dma_addr = xdptxd.dma_addr;
+		if (xdptxd->has_frags) {
+			int j;
+
+			xdptxdf.sinfo = xdp_get_shared_info_from_frame(xdpf);
+			xdptxdf.dma_arr = dma_arr;
+			for (j = 0; j < xdptxdf.sinfo->nr_frags; j++) {
+				skb_frag_t *frag = &xdptxdf.sinfo->frags[j];
+
+				dma_arr[j] = dma_map_single(sq->pdev, skb_frag_address(frag),
+							    skb_frag_size(frag), DMA_TO_DEVICE);
+
+				if (!dma_mapping_error(sq->pdev, dma_arr[j]))
+					continue;
+				/* mapping error */
+				while (--j >= 0)
+					dma_unmap_single(sq->pdev, dma_arr[j],
+							 skb_frag_size(&xdptxdf.sinfo->frags[j]),
+							 DMA_TO_DEVICE);
+				goto out;
+			}
+		}
 
 		ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
-				      mlx5e_xmit_xdp_frame, sq, &xdptxd, NULL, 0);
+				      mlx5e_xmit_xdp_frame, sq, xdptxd, 0);
 		if (unlikely(!ret)) {
-			dma_unmap_single(sq->pdev, xdptxd.dma_addr,
-					 xdptxd.len, DMA_TO_DEVICE);
+			int j;
+
+			dma_unmap_single(sq->pdev, xdptxd->dma_addr,
+					 xdptxd->len, DMA_TO_DEVICE);
+			if (!xdptxd->has_frags)
+				break;
+			for (j = 0; j < xdptxdf.sinfo->nr_frags; j++)
+				dma_unmap_single(sq->pdev, dma_arr[j],
+						 skb_frag_size(&xdptxdf.sinfo->frags[j]),
+						 DMA_TO_DEVICE);
 			break;
 		}
-		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, &xdpi);
+
+		/* xmit_mode == MLX5E_XDP_XMIT_MODE_FRAME */
+		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+				     (union mlx5e_xdp_info) { .mode = MLX5E_XDP_XMIT_MODE_FRAME });
+		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+				     (union mlx5e_xdp_info) { .frame.xdpf = xdpf });
+		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+				     (union mlx5e_xdp_info) { .frame.dma_addr = xdptxd->dma_addr });
+		if (xdptxd->has_frags) {
+			int j;
+
+			for (j = 0; j < xdptxdf.sinfo->nr_frags; j++)
+				mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+						     (union mlx5e_xdp_info)
+						     { .frame.dma_addr = dma_arr[j] });
+		}
 		nxmit++;
 	}
 
+out:
 	if (flags & XDP_XMIT_FLUSH) {
 		if (sq->mpwqe.wqe)
 			mlx5e_xdp_mpwqe_complete(sq);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
index 10bcfa6f88c1..9e8e6184f9e4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
@@ -50,6 +50,53 @@ struct mlx5e_xdp_buff {
 	struct mlx5e_rq *rq;
 };
 
+/* XDP packets can be transmitted in different ways. On completion, we need to
+ * distinguish between them to clean up things in a proper way.
+ */
+enum mlx5e_xdp_xmit_mode {
+	/* An xdp_frame was transmitted due to either XDP_REDIRECT from another
+	 * device or XDP_TX from an XSK RQ. The frame has to be unmapped and
+	 * returned.
+	 */
+	MLX5E_XDP_XMIT_MODE_FRAME,
+
+	/* The xdp_frame was created in place as a result of XDP_TX from a
+	 * regular RQ. No DMA remapping happened, and the page belongs to us.
+	 */
+	MLX5E_XDP_XMIT_MODE_PAGE,
+
+	/* No xdp_frame was created at all, the transmit happened from a UMEM
+	 * page. The UMEM Completion Ring producer pointer has to be increased.
+	 */
+	MLX5E_XDP_XMIT_MODE_XSK,
+};
+
+/* xmit_mode entry is pushed to the fifo per packet, followed by multiple
+ * entries, as follows:
+ *
+ * MLX5E_XDP_XMIT_MODE_FRAME:
+ *    xdpf, dma_addr_1, dma_addr_2, ... , dma_addr_num.
+ *    'num' is derived from xdpf.
+ *
+ * MLX5E_XDP_XMIT_MODE_PAGE:
+ *    num, page_1, page_2, ... , page_num.
+ *
+ * MLX5E_XDP_XMIT_MODE_XSK:
+ *    none.
+ */
+union mlx5e_xdp_info {
+	enum mlx5e_xdp_xmit_mode mode;
+	union {
+		struct xdp_frame *xdpf;
+		dma_addr_t dma_addr;
+	} frame;
+	union {
+		struct mlx5e_rq *rq;
+		u8 num;
+		struct page *page;
+	} page;
+};
+
 struct mlx5e_xsk_param;
 int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);
 bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
@@ -66,11 +113,9 @@ extern const struct xdp_metadata_ops mlx5e_xdp_metadata_ops;
 
 INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,
 							  struct mlx5e_xmit_data *xdptxd,
-							  struct skb_shared_info *sinfo,
 							  int check_result));
 INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
 						    struct mlx5e_xmit_data *xdptxd,
-						    struct skb_shared_info *sinfo,
 						    int check_result));
 INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq));
 INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq));
@@ -179,14 +224,14 @@ mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq,
 
 static inline void
 mlx5e_xdpi_fifo_push(struct mlx5e_xdp_info_fifo *fifo,
-		     struct mlx5e_xdp_info *xi)
+		     union mlx5e_xdp_info xi)
 {
 	u32 i = (*fifo->pc)++ & fifo->mask;
 
-	fifo->xi[i] = *xi;
+	fifo->xi[i] = xi;
 }
 
-static inline struct mlx5e_xdp_info
+static inline union mlx5e_xdp_info
 mlx5e_xdpi_fifo_pop(struct mlx5e_xdp_info_fifo *fifo)
 {
 	return fifo->xi[(*fifo->cc)++ & fifo->mask];
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
index fab787600459..d97e6df66f45 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
@@ -22,6 +22,7 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 	struct mlx5e_icosq *icosq = rq->icosq;
 	struct mlx5_wq_cyc *wq = &icosq->wq;
 	struct mlx5e_umr_wqe *umr_wqe;
+	struct xdp_buff **xsk_buffs;
 	int batch, i;
 	u32 offset; /* 17-bit value with MTT. */
 	u16 pi;
@@ -29,9 +30,9 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 	if (unlikely(!xsk_buff_can_alloc(rq->xsk_pool, rq->mpwqe.pages_per_wqe)))
 		goto err;
 
-	BUILD_BUG_ON(sizeof(wi->alloc_units[0]) != sizeof(wi->alloc_units[0].xsk));
 	XSK_CHECK_PRIV_TYPE(struct mlx5e_xdp_buff);
-	batch = xsk_buff_alloc_batch(rq->xsk_pool, (struct xdp_buff **)wi->alloc_units,
+	xsk_buffs = (struct xdp_buff **)wi->alloc_units.xsk_buffs;
+	batch = xsk_buff_alloc_batch(rq->xsk_pool, xsk_buffs,
 				     rq->mpwqe.pages_per_wqe);
 
 	/* If batch < pages_per_wqe, either:
@@ -41,8 +42,8 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 	 * the first error, which will mean there are no more valid descriptors.
 	 */
 	for (; batch < rq->mpwqe.pages_per_wqe; batch++) {
-		wi->alloc_units[batch].xsk = xsk_buff_alloc(rq->xsk_pool);
-		if (unlikely(!wi->alloc_units[batch].xsk))
+		xsk_buffs[batch] = xsk_buff_alloc(rq->xsk_pool);
+		if (unlikely(!xsk_buffs[batch]))
 			goto err_reuse_batch;
 	}
 
@@ -52,8 +53,8 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 
 	if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED)) {
 		for (i = 0; i < batch; i++) {
-			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);
-			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);
+			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(xsk_buffs[i]);
+			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(xsk_buffs[i]);
 
 			umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {
 				.ptag = cpu_to_be64(addr | MLX5_EN_WR),
@@ -62,8 +63,8 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 		}
 	} else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED)) {
 		for (i = 0; i < batch; i++) {
-			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);
-			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);
+			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(xsk_buffs[i]);
+			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(xsk_buffs[i]);
 
 			umr_wqe->inline_ksms[i] = (struct mlx5_ksm) {
 				.key = rq->mkey_be,
@@ -75,8 +76,8 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 		u32 mapping_size = 1 << (rq->mpwqe.page_shift - 2);
 
 		for (i = 0; i < batch; i++) {
-			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);
-			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);
+			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(xsk_buffs[i]);
+			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(xsk_buffs[i]);
 
 			umr_wqe->inline_ksms[i << 2] = (struct mlx5_ksm) {
 				.key = rq->mkey_be,
@@ -102,8 +103,8 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 		__be32 frame_size = cpu_to_be32(rq->xsk_pool->chunk_size);
 
 		for (i = 0; i < batch; i++) {
-			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);
-			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);
+			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(xsk_buffs[i]);
+			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(xsk_buffs[i]);
 
 			umr_wqe->inline_klms[i << 1] = (struct mlx5_klm) {
 				.key = rq->mkey_be,
@@ -119,7 +120,7 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 		}
 	}
 
-	bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe);
+	bitmap_zero(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
 	wi->consumed_strides = 0;
 
 	umr_wqe->ctrl.opmod_idx_opcode =
@@ -149,7 +150,7 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 
 err_reuse_batch:
 	while (--batch >= 0)
-		xsk_buff_free(wi->alloc_units[batch].xsk);
+		xsk_buff_free(xsk_buffs[batch]);
 
 err:
 	rq->stats->buff_alloc_err++;
@@ -163,13 +164,10 @@ int mlx5e_xsk_alloc_rx_wqes_batched(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
 	u32 contig, alloc;
 	int i;
 
-	/* mlx5e_init_frags_partition creates a 1:1 mapping between
-	 * rq->wqe.frags and rq->wqe.alloc_units, which allows us to
-	 * allocate XDP buffers straight into alloc_units.
+	/* Each rq->wqe.frags->xskp is 1:1 mapped to an element inside the
+	 * rq->wqe.alloc_units->xsk_buffs array allocated here.
 	 */
-	BUILD_BUG_ON(sizeof(rq->wqe.alloc_units[0]) !=
-		     sizeof(rq->wqe.alloc_units[0].xsk));
-	buffs = (struct xdp_buff **)rq->wqe.alloc_units;
+	buffs = rq->wqe.alloc_units->xsk_buffs;
 	contig = mlx5_wq_cyc_get_size(wq) - ix;
 	if (wqe_bulk <= contig) {
 		alloc = xsk_buff_alloc_batch(rq->xsk_pool, buffs + ix, wqe_bulk);
@@ -189,8 +187,9 @@ int mlx5e_xsk_alloc_rx_wqes_batched(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
 		/* Assumes log_num_frags == 0. */
 		frag = &rq->wqe.frags[j];
 
-		addr = xsk_buff_xdp_get_frame_dma(frag->au->xsk);
+		addr = xsk_buff_xdp_get_frame_dma(*frag->xskp);
 		wqe->data[0].addr = cpu_to_be64(addr + rq->buff.headroom);
+		frag->flags &= ~BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
 	}
 
 	return alloc;
@@ -211,12 +210,13 @@ int mlx5e_xsk_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
 		/* Assumes log_num_frags == 0. */
 		frag = &rq->wqe.frags[j];
 
-		frag->au->xsk = xsk_buff_alloc(rq->xsk_pool);
-		if (unlikely(!frag->au->xsk))
+		*frag->xskp = xsk_buff_alloc(rq->xsk_pool);
+		if (unlikely(!*frag->xskp))
 			return i;
 
-		addr = xsk_buff_xdp_get_frame_dma(frag->au->xsk);
+		addr = xsk_buff_xdp_get_frame_dma(*frag->xskp);
 		wqe->data[0].addr = cpu_to_be64(addr + rq->buff.headroom);
+		frag->flags &= ~BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
 	}
 
 	return wqe_bulk;
@@ -251,7 +251,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
 						    u32 head_offset,
 						    u32 page_idx)
 {
-	struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[page_idx].xsk);
+	struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units.xsk_buffs[page_idx]);
 	struct bpf_prog *prog;
 
 	/* Check packet size. Note LRO doesn't use linear SKB */
@@ -291,7 +291,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
 	prog = rcu_dereference(rq->xdp_prog);
 	if (likely(prog && mlx5e_xdp_handle(rq, prog, mxbuf))) {
 		if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)))
-			__set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
+			__set_bit(page_idx, wi->skip_release_bitmap); /* non-atomic */
 		return NULL; /* page/packet was consumed by XDP */
 	}
 
@@ -306,7 +306,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
 					      struct mlx5_cqe64 *cqe,
 					      u32 cqe_bcnt)
 {
-	struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->au->xsk);
+	struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(*wi->xskp);
 	struct bpf_prog *prog;
 
 	/* wi->offset is not used in this function, because xdp->data and the
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
index 81a567e17264..ed279f450976 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
@@ -93,13 +93,19 @@ static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *param
 			     struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool,
 			     struct mlx5e_xsk_param *xsk)
 {
+	struct mlx5e_rq *xskrq = &c->xskrq;
 	int err;
 
-	err = mlx5e_init_xsk_rq(c, params, pool, xsk, &c->xskrq);
+	err = mlx5e_init_xsk_rq(c, params, pool, xsk, xskrq);
 	if (err)
 		return err;
 
-	return mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), &c->xskrq);
+	err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), xskrq);
+	if (err)
+		return err;
+
+	__set_bit(MLX5E_RQ_STATE_XSK, &xskrq->state);
+	return 0;
 }
 
 int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
index 367a9505ca4f..597f319d4770 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
@@ -44,7 +44,7 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
  * same.
  */
 static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq,
-				  struct mlx5e_xdp_info *xdpi)
+				  union mlx5e_xdp_info *xdpi)
 {
 	u16 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
 	struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[pi];
@@ -54,15 +54,14 @@ static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq,
 	wi->num_pkts = 1;
 
 	nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
-	mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi);
+	mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, *xdpi);
 	sq->doorbell_cseg = &nopwqe->ctrl;
 }
 
 bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
 {
 	struct xsk_buff_pool *pool = sq->xsk_pool;
-	struct mlx5e_xmit_data xdptxd;
-	struct mlx5e_xdp_info xdpi;
+	union mlx5e_xdp_info xdpi;
 	bool work_done = true;
 	bool flush = false;
 
@@ -73,6 +72,7 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
 						   mlx5e_xmit_xdp_frame_check_mpwqe,
 						   mlx5e_xmit_xdp_frame_check,
 						   sq);
+		struct mlx5e_xmit_data xdptxd = {};
 		struct xdp_desc desc;
 		bool ret;
 
@@ -97,7 +97,7 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
 		xsk_buff_raw_dma_sync_for_device(pool, xdptxd.dma_addr, xdptxd.len);
 
 		ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
-				      mlx5e_xmit_xdp_frame, sq, &xdptxd, NULL,
+				      mlx5e_xmit_xdp_frame, sq, &xdptxd,
 				      check_result);
 		if (unlikely(!ret)) {
 			if (sq->mpwqe.wqe)
@@ -105,7 +105,7 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
 
 			mlx5e_xsk_tx_post_err(sq, &xdpi);
 		} else {
-			mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, &xdpi);
+			mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi);
 		}
 
 		flush = true;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
index 7b0d3de0ec6c..55b38544422f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
@@ -35,11 +35,15 @@
 #include <crypto/aead.h>
 #include <linux/inetdevice.h>
 #include <linux/netdevice.h>
+#include <net/netevent.h>
 
 #include "en.h"
 #include "ipsec.h"
 #include "ipsec_rxtx.h"
 
+#define MLX5_IPSEC_RESCHED msecs_to_jiffies(1000)
+#define MLX5E_IPSEC_TUNNEL_SA XA_MARK_1
+
 static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x)
 {
 	return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;
@@ -50,27 +54,71 @@ static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(struct xfrm_policy *x)
 	return (struct mlx5e_ipsec_pol_entry *)x->xdo.offload_handle;
 }
 
+static void mlx5e_ipsec_handle_tx_limit(struct work_struct *_work)
+{
+	struct mlx5e_ipsec_dwork *dwork =
+		container_of(_work, struct mlx5e_ipsec_dwork, dwork.work);
+	struct mlx5e_ipsec_sa_entry *sa_entry = dwork->sa_entry;
+	struct xfrm_state *x = sa_entry->x;
+
+	spin_lock(&x->lock);
+	xfrm_state_check_expire(x);
+	if (x->km.state == XFRM_STATE_EXPIRED) {
+		sa_entry->attrs.drop = true;
+		mlx5e_accel_ipsec_fs_modify(sa_entry);
+	}
+	spin_unlock(&x->lock);
+
+	if (sa_entry->attrs.drop)
+		return;
+
+	queue_delayed_work(sa_entry->ipsec->wq, &dwork->dwork,
+			   MLX5_IPSEC_RESCHED);
+}
+
 static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry)
 {
-	struct xfrm_replay_state_esn *replay_esn;
+	struct xfrm_state *x = sa_entry->x;
 	u32 seq_bottom = 0;
+	u32 esn, esn_msb;
 	u8 overlap;
 
-	if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) {
-		sa_entry->esn_state.trigger = 0;
+	switch (x->xso.type) {
+	case XFRM_DEV_OFFLOAD_PACKET:
+		switch (x->xso.dir) {
+		case XFRM_DEV_OFFLOAD_IN:
+			esn = x->replay_esn->seq;
+			esn_msb = x->replay_esn->seq_hi;
+			break;
+		case XFRM_DEV_OFFLOAD_OUT:
+			esn = x->replay_esn->oseq;
+			esn_msb = x->replay_esn->oseq_hi;
+			break;
+		default:
+			WARN_ON(true);
+			return false;
+		}
+		break;
+	case XFRM_DEV_OFFLOAD_CRYPTO:
+		/* Already parsed by XFRM core */
+		esn = x->replay_esn->seq;
+		break;
+	default:
+		WARN_ON(true);
 		return false;
 	}
 
-	replay_esn = sa_entry->x->replay_esn;
-	if (replay_esn->seq >= replay_esn->replay_window)
-		seq_bottom = replay_esn->seq - replay_esn->replay_window + 1;
-
 	overlap = sa_entry->esn_state.overlap;
 
-	sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x,
-						    htonl(seq_bottom));
+	if (esn >= x->replay_esn->replay_window)
+		seq_bottom = esn - x->replay_esn->replay_window + 1;
+
+	if (x->xso.type == XFRM_DEV_OFFLOAD_CRYPTO)
+		esn_msb = xfrm_replay_seqhi(x, htonl(seq_bottom));
+
+	sa_entry->esn_state.esn = esn;
+	sa_entry->esn_state.esn_msb = esn_msb;
 
-	sa_entry->esn_state.trigger = 1;
 	if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) {
 		sa_entry->esn_state.overlap = 0;
 		return true;
@@ -87,25 +135,161 @@ static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry,
 				    struct mlx5_accel_esp_xfrm_attrs *attrs)
 {
 	struct xfrm_state *x = sa_entry->x;
+	s64 start_value, n;
 
-	attrs->hard_packet_limit = x->lft.hard_packet_limit;
+	attrs->lft.hard_packet_limit = x->lft.hard_packet_limit;
+	attrs->lft.soft_packet_limit = x->lft.soft_packet_limit;
 	if (x->lft.soft_packet_limit == XFRM_INF)
 		return;
 
-	/* Hardware decrements hard_packet_limit counter through
-	 * the operation. While fires an event when soft_packet_limit
-	 * is reached. It emans that we need substitute the numbers
-	 * in order to properly count soft limit.
+	/* Compute hard limit initial value and number of rounds.
+	 *
+	 * The counting pattern of hardware counter goes:
+	 *                value  -> 2^31-1
+	 *      2^31  | (2^31-1) -> 2^31-1
+	 *      2^31  | (2^31-1) -> 2^31-1
+	 *      [..]
+	 *      2^31  | (2^31-1) -> 0
 	 *
-	 * As an example:
-	 * XFRM user sets soft limit is 2 and hard limit is 9 and
-	 * expects to see soft event after 2 packets and hard event
-	 * after 9 packets. In our case, the hard limit will be set
-	 * to 9 and soft limit is comparator to 7 so user gets the
-	 * soft event after 2 packeta
+	 * The pattern is created by using an ASO operation to atomically set
+	 * bit 31 after the down counter clears bit 31. This is effectively an
+	 * atomic addition of 2**31 to the counter.
+	 *
+	 * We wish to configure the counter, within the above pattern, so that
+	 * when it reaches 0, it has hit the hard limit. This is defined by this
+	 * system of equations:
+	 *
+	 *      hard_limit == start_value + n * 2^31
+	 *      n >= 0
+	 *      start_value < 2^32, start_value >= 0
+	 *
+	 * These equations are not single-solution, there are often two choices:
+	 *      hard_limit == start_value + n * 2^31
+	 *      hard_limit == (start_value+2^31) + (n-1) * 2^31
+	 *
+	 * The algorithm selects the solution that keeps the counter value
+	 * above 2^31 until the final iteration.
+	 */
+
+	/* Start by estimating n and compute start_value */
+	n = attrs->lft.hard_packet_limit / BIT_ULL(31);
+	start_value = attrs->lft.hard_packet_limit - n * BIT_ULL(31);
+
+	/* Choose the best of the two solutions: */
+	if (n >= 1)
+		n -= 1;
+
+	/* Computed values solve the system of equations: */
+	start_value = attrs->lft.hard_packet_limit - n * BIT_ULL(31);
+
+	/* The best solution means: when there are multiple iterations we must
+	 * start above 2^31 and count down to 2**31 to get the interrupt.
+	 */
+	attrs->lft.hard_packet_limit = lower_32_bits(start_value);
+	attrs->lft.numb_rounds_hard = (u64)n;
+
+	/* Compute soft limit initial value and number of rounds.
+	 *
+	 * The soft_limit is achieved by adjusting the counter's
+	 * interrupt_value. This is embedded in the counting pattern created by
+	 * hard packet calculations above.
+	 *
+	 * We wish to compute the interrupt_value for the soft_limit. This is
+	 * defined by this system of equations:
+	 *
+	 *      soft_limit == start_value - soft_value + n * 2^31
+	 *      n >= 0
+	 *      soft_value < 2^32, soft_value >= 0
+	 *      for n == 0 start_value > soft_value
+	 *
+	 * As with compute_hard_n_value() the equations are not single-solution.
+	 * The algorithm selects the solution that has:
+	 *      2^30 <= soft_limit < 2^31 + 2^30
+	 * for the interior iterations, which guarantees a large guard band
+	 * around the counter hard limit and next interrupt.
+	 */
+
+	/* Start by estimating n and compute soft_value */
+	n = (x->lft.soft_packet_limit - attrs->lft.hard_packet_limit) / BIT_ULL(31);
+	start_value = attrs->lft.hard_packet_limit + n * BIT_ULL(31) -
+		      x->lft.soft_packet_limit;
+
+	/* Compare against constraints and adjust n */
+	if (n < 0)
+		n = 0;
+	else if (start_value >= BIT_ULL(32))
+		n -= 1;
+	else if (start_value < 0)
+		n += 1;
+
+	/* Choose the best of the two solutions: */
+	start_value = attrs->lft.hard_packet_limit + n * BIT_ULL(31) - start_value;
+	if (n != attrs->lft.numb_rounds_hard && start_value < BIT_ULL(30))
+		n += 1;
+
+	/* Note that the upper limit of soft_value happens naturally because we
+	 * always select the lowest soft_value.
 	 */
-	attrs->soft_packet_limit =
-		x->lft.hard_packet_limit - x->lft.soft_packet_limit;
+
+	/* Computed values solve the system of equations: */
+	start_value = attrs->lft.hard_packet_limit + n * BIT_ULL(31) - start_value;
+
+	/* The best solution means: when there are multiple iterations we must
+	 * not fall below 2^30 as that would get too close to the false
+	 * hard_limit and when we reach an interior iteration for soft_limit it
+	 * has to be far away from 2**32-1 which is the counter reset point
+	 * after the +2^31 to accommodate latency.
+	 */
+	attrs->lft.soft_packet_limit = lower_32_bits(start_value);
+	attrs->lft.numb_rounds_soft = (u64)n;
+}
+
+static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry,
+				  struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+	struct xfrm_state *x = sa_entry->x;
+	struct net_device *netdev;
+	struct neighbour *n;
+	u8 addr[ETH_ALEN];
+	const void *pkey;
+	u8 *dst, *src;
+
+	if (attrs->mode != XFRM_MODE_TUNNEL ||
+	    attrs->type != XFRM_DEV_OFFLOAD_PACKET)
+		return;
+
+	netdev = x->xso.real_dev;
+
+	mlx5_query_mac_address(mdev, addr);
+	switch (attrs->dir) {
+	case XFRM_DEV_OFFLOAD_IN:
+		src = attrs->dmac;
+		dst = attrs->smac;
+		pkey = &attrs->saddr.a4;
+		break;
+	case XFRM_DEV_OFFLOAD_OUT:
+		src = attrs->smac;
+		dst = attrs->dmac;
+		pkey = &attrs->daddr.a4;
+		break;
+	default:
+		return;
+	}
+
+	ether_addr_copy(src, addr);
+	n = neigh_lookup(&arp_tbl, pkey, netdev);
+	if (!n) {
+		n = neigh_create(&arp_tbl, pkey, netdev);
+		if (IS_ERR(n))
+			return;
+		neigh_event_send(n, NULL);
+		attrs->drop = true;
+	} else {
+		neigh_ha_snapshot(addr, n, netdev);
+		ether_addr_copy(dst, addr);
+	}
+	neigh_release(n);
 }
 
 void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
@@ -141,11 +325,11 @@ void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
 	aes_gcm->icv_len = x->aead->alg_icv_len;
 
 	/* esn */
-	if (sa_entry->esn_state.trigger) {
-		attrs->esn_trigger = true;
-		attrs->esn = sa_entry->esn_state.esn;
-		attrs->esn_overlap = sa_entry->esn_state.overlap;
-		attrs->replay_window = x->replay_esn->replay_window;
+	if (x->props.flags & XFRM_STATE_ESN) {
+		attrs->replay_esn.trigger = true;
+		attrs->replay_esn.esn = sa_entry->esn_state.esn;
+		attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb;
+		attrs->replay_esn.overlap = sa_entry->esn_state.overlap;
 	}
 
 	attrs->dir = x->xso.dir;
@@ -163,8 +347,10 @@ void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
 	attrs->upspec.sport = ntohs(x->sel.sport);
 	attrs->upspec.sport_mask = ntohs(x->sel.sport_mask);
 	attrs->upspec.proto = x->sel.proto;
+	attrs->mode = x->props.mode;
 
 	mlx5e_ipsec_init_limits(sa_entry, attrs);
+	mlx5e_ipsec_init_macs(sa_entry, attrs);
 }
 
 static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev,
@@ -233,6 +419,11 @@ static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev,
 		return -EINVAL;
 	}
 
+	if (x->props.mode != XFRM_MODE_TRANSPORT && x->props.mode != XFRM_MODE_TUNNEL) {
+		NL_SET_ERR_MSG_MOD(extack, "Only transport and tunnel xfrm states may be offloaded");
+		return -EINVAL;
+	}
+
 	switch (x->xso.type) {
 	case XFRM_DEV_OFFLOAD_CRYPTO:
 		if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_CRYPTO)) {
@@ -240,11 +431,6 @@ static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev,
 			return -EINVAL;
 		}
 
-		if (x->props.mode != XFRM_MODE_TRANSPORT &&
-		    x->props.mode != XFRM_MODE_TUNNEL) {
-			NL_SET_ERR_MSG_MOD(extack, "Only transport and tunnel xfrm states may be offloaded");
-			return -EINVAL;
-		}
 		break;
 	case XFRM_DEV_OFFLOAD_PACKET:
 		if (!(mlx5_ipsec_device_caps(mdev) &
@@ -253,8 +439,9 @@ static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev,
 			return -EINVAL;
 		}
 
-		if (x->props.mode != XFRM_MODE_TRANSPORT) {
-			NL_SET_ERR_MSG_MOD(extack, "Only transport xfrm states may be offloaded in packet mode");
+		if (x->props.mode == XFRM_MODE_TUNNEL &&
+		    !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL)) {
+			NL_SET_ERR_MSG_MOD(extack, "Packet offload is not supported for tunnel mode");
 			return -EINVAL;
 		}
 
@@ -283,6 +470,11 @@ static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev,
 			NL_SET_ERR_MSG_MOD(extack, "Hard packet limit must be greater than soft one");
 			return -EINVAL;
 		}
+
+		if (!x->lft.soft_packet_limit || !x->lft.hard_packet_limit) {
+			NL_SET_ERR_MSG_MOD(extack, "Soft/hard packet limits can't be 0");
+			return -EINVAL;
+		}
 		break;
 	default:
 		NL_SET_ERR_MSG_MOD(extack, "Unsupported xfrm offload type");
@@ -291,14 +483,134 @@ static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev,
 	return 0;
 }
 
-static void _update_xfrm_state(struct work_struct *work)
+static void mlx5e_ipsec_modify_state(struct work_struct *_work)
+{
+	struct mlx5e_ipsec_work *work =
+		container_of(_work, struct mlx5e_ipsec_work, work);
+	struct mlx5e_ipsec_sa_entry *sa_entry = work->sa_entry;
+	struct mlx5_accel_esp_xfrm_attrs *attrs;
+
+	attrs = &((struct mlx5e_ipsec_sa_entry *)work->data)->attrs;
+
+	mlx5_accel_esp_modify_xfrm(sa_entry, attrs);
+}
+
+static void mlx5e_ipsec_set_esn_ops(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct xfrm_state *x = sa_entry->x;
+
+	if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO ||
+	    x->xso.dir != XFRM_DEV_OFFLOAD_OUT)
+		return;
+
+	if (x->props.flags & XFRM_STATE_ESN) {
+		sa_entry->set_iv_op = mlx5e_ipsec_set_iv_esn;
+		return;
+	}
+
+	sa_entry->set_iv_op = mlx5e_ipsec_set_iv;
+}
+
+static void mlx5e_ipsec_handle_netdev_event(struct work_struct *_work)
+{
+	struct mlx5e_ipsec_work *work =
+		container_of(_work, struct mlx5e_ipsec_work, work);
+	struct mlx5e_ipsec_sa_entry *sa_entry = work->sa_entry;
+	struct mlx5e_ipsec_netevent_data *data = work->data;
+	struct mlx5_accel_esp_xfrm_attrs *attrs;
+
+	attrs = &sa_entry->attrs;
+
+	switch (attrs->dir) {
+	case XFRM_DEV_OFFLOAD_IN:
+		ether_addr_copy(attrs->smac, data->addr);
+		break;
+	case XFRM_DEV_OFFLOAD_OUT:
+		ether_addr_copy(attrs->dmac, data->addr);
+		break;
+	default:
+		WARN_ON_ONCE(true);
+	}
+	attrs->drop = false;
+	mlx5e_accel_ipsec_fs_modify(sa_entry);
+}
+
+static int mlx5_ipsec_create_work(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct xfrm_state *x = sa_entry->x;
+	struct mlx5e_ipsec_work *work;
+	void *data = NULL;
+
+	switch (x->xso.type) {
+	case XFRM_DEV_OFFLOAD_CRYPTO:
+		if (!(x->props.flags & XFRM_STATE_ESN))
+			return 0;
+		break;
+	case XFRM_DEV_OFFLOAD_PACKET:
+		if (x->props.mode != XFRM_MODE_TUNNEL)
+			return 0;
+		break;
+	default:
+		break;
+	}
+
+	work = kzalloc(sizeof(*work), GFP_KERNEL);
+	if (!work)
+		return -ENOMEM;
+
+	switch (x->xso.type) {
+	case XFRM_DEV_OFFLOAD_CRYPTO:
+		data = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
+		if (!data)
+			goto free_work;
+
+		INIT_WORK(&work->work, mlx5e_ipsec_modify_state);
+		break;
+	case XFRM_DEV_OFFLOAD_PACKET:
+		data = kzalloc(sizeof(struct mlx5e_ipsec_netevent_data),
+			       GFP_KERNEL);
+		if (!data)
+			goto free_work;
+
+		INIT_WORK(&work->work, mlx5e_ipsec_handle_netdev_event);
+		break;
+	default:
+		break;
+	}
+
+	work->data = data;
+	work->sa_entry = sa_entry;
+	sa_entry->work = work;
+	return 0;
+
+free_work:
+	kfree(work);
+	return -ENOMEM;
+}
+
+static int mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry *sa_entry)
 {
-	struct mlx5e_ipsec_modify_state_work *modify_work =
-		container_of(work, struct mlx5e_ipsec_modify_state_work, work);
-	struct mlx5e_ipsec_sa_entry *sa_entry = container_of(
-		modify_work, struct mlx5e_ipsec_sa_entry, modify_work);
+	struct xfrm_state *x = sa_entry->x;
+	struct mlx5e_ipsec_dwork *dwork;
+
+	if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
+		return 0;
+
+	if (x->xso.dir != XFRM_DEV_OFFLOAD_OUT)
+		return 0;
+
+	if (x->lft.soft_packet_limit == XFRM_INF &&
+	    x->lft.hard_packet_limit == XFRM_INF)
+		return 0;
 
-	mlx5_accel_esp_modify_xfrm(sa_entry, &modify_work->attrs);
+	dwork = kzalloc(sizeof(*dwork), GFP_KERNEL);
+	if (!dwork)
+		return -ENOMEM;
+
+	dwork->sa_entry = sa_entry;
+	INIT_DELAYED_WORK(&dwork->dwork, mlx5e_ipsec_handle_tx_limit);
+	sa_entry->dwork = dwork;
+	return 0;
 }
 
 static int mlx5e_xfrm_add_state(struct xfrm_state *x,
@@ -308,6 +620,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x,
 	struct net_device *netdev = x->xso.real_dev;
 	struct mlx5e_ipsec *ipsec;
 	struct mlx5e_priv *priv;
+	gfp_t gfp;
 	int err;
 
 	priv = netdev_priv(netdev);
@@ -315,30 +628,52 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x,
 		return -EOPNOTSUPP;
 
 	ipsec = priv->ipsec;
-	err = mlx5e_xfrm_validate_state(priv->mdev, x, extack);
-	if (err)
-		return err;
-
-	sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
+	gfp = (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) ? GFP_ATOMIC : GFP_KERNEL;
+	sa_entry = kzalloc(sizeof(*sa_entry), gfp);
 	if (!sa_entry)
 		return -ENOMEM;
 
 	sa_entry->x = x;
 	sa_entry->ipsec = ipsec;
+	/* Check if this SA is originated from acquire flow temporary SA */
+	if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ)
+		goto out;
+
+	err = mlx5e_xfrm_validate_state(priv->mdev, x, extack);
+	if (err)
+		goto err_xfrm;
 
 	/* check esn */
-	mlx5e_ipsec_update_esn_state(sa_entry);
+	if (x->props.flags & XFRM_STATE_ESN)
+		mlx5e_ipsec_update_esn_state(sa_entry);
 
 	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs);
+
+	err = mlx5_ipsec_create_work(sa_entry);
+	if (err)
+		goto err_xfrm;
+
+	err = mlx5e_ipsec_create_dwork(sa_entry);
+	if (err)
+		goto release_work;
+
 	/* create hw context */
 	err = mlx5_ipsec_create_sa_ctx(sa_entry);
 	if (err)
-		goto err_xfrm;
+		goto release_dwork;
 
 	err = mlx5e_accel_ipsec_fs_add_rule(sa_entry);
 	if (err)
 		goto err_hw_ctx;
 
+	if (x->props.mode == XFRM_MODE_TUNNEL &&
+	    x->xso.type == XFRM_DEV_OFFLOAD_PACKET &&
+	    !mlx5e_ipsec_fs_tunnel_enabled(sa_entry)) {
+		NL_SET_ERR_MSG_MOD(extack, "Packet offload tunnel mode is disabled due to encap settings");
+		err = -EINVAL;
+		goto err_add_rule;
+	}
+
 	/* We use *_bh() variant because xfrm_timer_handler(), which runs
 	 * in softirq context, can reach our state delete logic and we need
 	 * xa_erase_bh() there.
@@ -348,11 +683,18 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x,
 	if (err)
 		goto err_add_rule;
 
-	if (x->xso.dir == XFRM_DEV_OFFLOAD_OUT)
-		sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ?
-				mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv;
+	mlx5e_ipsec_set_esn_ops(sa_entry);
+
+	if (sa_entry->dwork)
+		queue_delayed_work(ipsec->wq, &sa_entry->dwork->dwork,
+				   MLX5_IPSEC_RESCHED);
 
-	INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state);
+	if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET &&
+	    x->props.mode == XFRM_MODE_TUNNEL)
+		xa_set_mark(&ipsec->sadb, sa_entry->ipsec_obj_id,
+			    MLX5E_IPSEC_TUNNEL_SA);
+
+out:
 	x->xso.offload_handle = (unsigned long)sa_entry;
 	return 0;
 
@@ -360,32 +702,101 @@ err_add_rule:
 	mlx5e_accel_ipsec_fs_del_rule(sa_entry);
 err_hw_ctx:
 	mlx5_ipsec_free_sa_ctx(sa_entry);
+release_dwork:
+	kfree(sa_entry->dwork);
+release_work:
+	if (sa_entry->work)
+		kfree(sa_entry->work->data);
+	kfree(sa_entry->work);
 err_xfrm:
 	kfree(sa_entry);
-	NL_SET_ERR_MSG_MOD(extack, "Device failed to offload this policy");
+	NL_SET_ERR_MSG_WEAK_MOD(extack, "Device failed to offload this state");
 	return err;
 }
 
 static void mlx5e_xfrm_del_state(struct xfrm_state *x)
 {
 	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
+	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
 	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
 	struct mlx5e_ipsec_sa_entry *old;
 
+	if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ)
+		return;
+
 	old = xa_erase_bh(&ipsec->sadb, sa_entry->ipsec_obj_id);
 	WARN_ON(old != sa_entry);
+
+	if (attrs->mode == XFRM_MODE_TUNNEL &&
+	    attrs->type == XFRM_DEV_OFFLOAD_PACKET)
+		/* Make sure that no ARP requests are running in parallel */
+		flush_workqueue(ipsec->wq);
+
 }
 
 static void mlx5e_xfrm_free_state(struct xfrm_state *x)
 {
 	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
 
-	cancel_work_sync(&sa_entry->modify_work.work);
+	if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ)
+		goto sa_entry_free;
+
+	if (sa_entry->work)
+		cancel_work_sync(&sa_entry->work->work);
+
+	if (sa_entry->dwork)
+		cancel_delayed_work_sync(&sa_entry->dwork->dwork);
+
 	mlx5e_accel_ipsec_fs_del_rule(sa_entry);
 	mlx5_ipsec_free_sa_ctx(sa_entry);
+	kfree(sa_entry->dwork);
+	if (sa_entry->work)
+		kfree(sa_entry->work->data);
+	kfree(sa_entry->work);
+sa_entry_free:
 	kfree(sa_entry);
 }
 
+static int mlx5e_ipsec_netevent_event(struct notifier_block *nb,
+				      unsigned long event, void *ptr)
+{
+	struct mlx5_accel_esp_xfrm_attrs *attrs;
+	struct mlx5e_ipsec_netevent_data *data;
+	struct mlx5e_ipsec_sa_entry *sa_entry;
+	struct mlx5e_ipsec *ipsec;
+	struct neighbour *n = ptr;
+	struct net_device *netdev;
+	struct xfrm_state *x;
+	unsigned long idx;
+
+	if (event != NETEVENT_NEIGH_UPDATE || !(n->nud_state & NUD_VALID))
+		return NOTIFY_DONE;
+
+	ipsec = container_of(nb, struct mlx5e_ipsec, netevent_nb);
+	xa_for_each_marked(&ipsec->sadb, idx, sa_entry, MLX5E_IPSEC_TUNNEL_SA) {
+		attrs = &sa_entry->attrs;
+
+		if (attrs->family == AF_INET) {
+			if (!neigh_key_eq32(n, &attrs->saddr.a4) &&
+			    !neigh_key_eq32(n, &attrs->daddr.a4))
+				continue;
+		} else {
+			if (!neigh_key_eq128(n, &attrs->saddr.a4) &&
+			    !neigh_key_eq128(n, &attrs->daddr.a4))
+				continue;
+		}
+
+		x = sa_entry->x;
+		netdev = x->xso.real_dev;
+		data = sa_entry->work->data;
+
+		neigh_ha_snapshot(data->addr, n, netdev);
+		queue_work(ipsec->wq, &sa_entry->work->work);
+	}
+
+	return NOTIFY_DONE;
+}
+
 void mlx5e_ipsec_init(struct mlx5e_priv *priv)
 {
 	struct mlx5e_ipsec *ipsec;
@@ -402,8 +813,8 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv)
 
 	xa_init_flags(&ipsec->sadb, XA_FLAGS_ALLOC);
 	ipsec->mdev = priv->mdev;
-	ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0,
-					    priv->netdev->name);
+	ipsec->wq = alloc_workqueue("mlx5e_ipsec: %s", WQ_UNBOUND, 0,
+				    priv->netdev->name);
 	if (!ipsec->wq)
 		goto err_wq;
 
@@ -414,6 +825,13 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv)
 			goto err_aso;
 	}
 
+	if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_TUNNEL) {
+		ipsec->netevent_nb.notifier_call = mlx5e_ipsec_netevent_event;
+		ret = register_netevent_notifier(&ipsec->netevent_nb);
+		if (ret)
+			goto clear_aso;
+	}
+
 	ret = mlx5e_accel_ipsec_fs_init(ipsec);
 	if (ret)
 		goto err_fs_init;
@@ -424,6 +842,9 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv)
 	return;
 
 err_fs_init:
+	if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_TUNNEL)
+		unregister_netevent_notifier(&ipsec->netevent_nb);
+clear_aso:
 	if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
 		mlx5e_ipsec_aso_cleanup(ipsec);
 err_aso:
@@ -442,6 +863,8 @@ void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv)
 		return;
 
 	mlx5e_accel_ipsec_fs_cleanup(ipsec);
+	if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_TUNNEL)
+		unregister_netevent_notifier(&ipsec->netevent_nb);
 	if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
 		mlx5e_ipsec_aso_cleanup(ipsec);
 	destroy_workqueue(ipsec->wq);
@@ -467,41 +890,43 @@ static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
 static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x)
 {
 	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
-	struct mlx5e_ipsec_modify_state_work *modify_work =
-		&sa_entry->modify_work;
+	struct mlx5e_ipsec_work *work = sa_entry->work;
+	struct mlx5e_ipsec_sa_entry *sa_entry_shadow;
 	bool need_update;
 
 	need_update = mlx5e_ipsec_update_esn_state(sa_entry);
 	if (!need_update)
 		return;
 
-	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs);
-	queue_work(sa_entry->ipsec->wq, &modify_work->work);
+	sa_entry_shadow = work->data;
+	memset(sa_entry_shadow, 0x00, sizeof(*sa_entry_shadow));
+	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry_shadow->attrs);
+	queue_work(sa_entry->ipsec->wq, &work->work);
 }
 
 static void mlx5e_xfrm_update_curlft(struct xfrm_state *x)
 {
 	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
-	int err;
+	struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+	u64 packets, bytes, lastuse;
 
-	lockdep_assert_held(&x->lock);
+	lockdep_assert(lockdep_is_held(&x->lock) ||
+		       lockdep_is_held(&dev_net(x->xso.real_dev)->xfrm.xfrm_cfg_mutex));
 
-	if (sa_entry->attrs.soft_packet_limit == XFRM_INF)
-		/* Limits are not configured, as soft limit
-		 * must be lowever than hard limit.
-		 */
+	if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ)
 		return;
 
-	err = mlx5e_ipsec_aso_query(sa_entry, NULL);
-	if (err)
-		return;
-
-	mlx5e_ipsec_aso_update_curlft(sa_entry, &x->curlft.packets);
+	mlx5_fc_query_cached(ipsec_rule->fc, &bytes, &packets, &lastuse);
+	x->curlft.packets += packets;
+	x->curlft.bytes += bytes;
 }
 
-static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x,
+static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev,
+				      struct xfrm_policy *x,
 				      struct netlink_ext_ack *extack)
 {
+	struct xfrm_selector *sel = &x->selector;
+
 	if (x->type != XFRM_POLICY_TYPE_MAIN) {
 		NL_SET_ERR_MSG_MOD(extack, "Cannot offload non-main policy types");
 		return -EINVAL;
@@ -519,8 +944,9 @@ static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x,
 		return -EINVAL;
 	}
 
-	if (!x->xfrm_vec[0].reqid) {
-		NL_SET_ERR_MSG_MOD(extack, "Cannot offload policy without reqid");
+	if (!x->xfrm_vec[0].reqid && sel->proto == IPPROTO_IP &&
+	    addr6_all_zero(sel->saddr.a6) && addr6_all_zero(sel->daddr.a6)) {
+		NL_SET_ERR_MSG_MOD(extack, "Unsupported policy with reqid 0 without at least one of upper protocol or ip addr(s) different than 0");
 		return -EINVAL;
 	}
 
@@ -529,12 +955,24 @@ static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x,
 		return -EINVAL;
 	}
 
-	if (x->selector.proto != IPPROTO_IP &&
-	    (x->selector.proto != IPPROTO_UDP || x->xdo.dir != XFRM_DEV_OFFLOAD_OUT)) {
+	if (sel->proto != IPPROTO_IP &&
+	    (sel->proto != IPPROTO_UDP || x->xdo.dir != XFRM_DEV_OFFLOAD_OUT)) {
 		NL_SET_ERR_MSG_MOD(extack, "Device does not support upper protocol other than UDP, and only Tx direction");
 		return -EINVAL;
 	}
 
+	if (x->priority) {
+		if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO)) {
+			NL_SET_ERR_MSG_MOD(extack, "Device does not support policy priority");
+			return -EINVAL;
+		}
+
+		if (x->priority == U32_MAX) {
+			NL_SET_ERR_MSG_MOD(extack, "Device does not support requested policy priority");
+			return -EINVAL;
+		}
+	}
+
 	return 0;
 }
 
@@ -560,6 +998,7 @@ mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry,
 	attrs->upspec.sport = ntohs(sel->sport);
 	attrs->upspec.sport_mask = ntohs(sel->sport_mask);
 	attrs->upspec.proto = sel->proto;
+	attrs->prio = x->priority;
 }
 
 static int mlx5e_xfrm_add_policy(struct xfrm_policy *x,
@@ -576,7 +1015,7 @@ static int mlx5e_xfrm_add_policy(struct xfrm_policy *x,
 		return -EOPNOTSUPP;
 	}
 
-	err = mlx5e_xfrm_validate_policy(x, extack);
+	err = mlx5e_xfrm_validate_policy(priv->mdev, x, extack);
 	if (err)
 		return err;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
index 12f044330639..4e9887171508 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
@@ -60,10 +60,24 @@ struct upspec {
 	u8 proto;
 };
 
+struct mlx5_ipsec_lft {
+	u64 hard_packet_limit;
+	u64 soft_packet_limit;
+	u64 numb_rounds_hard;
+	u64 numb_rounds_soft;
+};
+
+struct mlx5_replay_esn {
+	u32 replay_window;
+	u32 esn;
+	u32 esn_msb;
+	u8 overlap : 1;
+	u8 trigger : 1;
+};
+
 struct mlx5_accel_esp_xfrm_attrs {
-	u32   esn;
 	u32   spi;
-	u32   flags;
+	u32   mode;
 	struct aes_gcm_keymat aes_gcm;
 
 	union {
@@ -78,15 +92,15 @@ struct mlx5_accel_esp_xfrm_attrs {
 
 	struct upspec upspec;
 	u8 dir : 2;
-	u8 esn_overlap : 1;
-	u8 esn_trigger : 1;
 	u8 type : 2;
+	u8 drop : 1;
 	u8 family;
-	u32 replay_window;
+	struct mlx5_replay_esn replay_esn;
 	u32 authsize;
 	u32 reqid;
-	u64 hard_packet_limit;
-	u64 soft_packet_limit;
+	struct mlx5_ipsec_lft lft;
+	u8 smac[ETH_ALEN];
+	u8 dmac[ETH_ALEN];
 };
 
 enum mlx5_ipsec_cap {
@@ -94,6 +108,8 @@ enum mlx5_ipsec_cap {
 	MLX5_IPSEC_CAP_ESN		= 1 << 1,
 	MLX5_IPSEC_CAP_PACKET_OFFLOAD	= 1 << 2,
 	MLX5_IPSEC_CAP_ROCE             = 1 << 3,
+	MLX5_IPSEC_CAP_PRIO             = 1 << 4,
+	MLX5_IPSEC_CAP_TUNNEL           = 1 << 5,
 };
 
 struct mlx5e_priv;
@@ -124,8 +140,17 @@ struct mlx5e_ipsec_tx;
 
 struct mlx5e_ipsec_work {
 	struct work_struct work;
-	struct mlx5e_ipsec *ipsec;
-	u32 id;
+	struct mlx5e_ipsec_sa_entry *sa_entry;
+	void *data;
+};
+
+struct mlx5e_ipsec_netevent_data {
+	u8 addr[ETH_ALEN];
+};
+
+struct mlx5e_ipsec_dwork {
+	struct delayed_work dwork;
+	struct mlx5e_ipsec_sa_entry *sa_entry;
 };
 
 struct mlx5e_ipsec_aso {
@@ -148,12 +173,13 @@ struct mlx5e_ipsec {
 	struct mlx5e_ipsec_tx *tx;
 	struct mlx5e_ipsec_aso *aso;
 	struct notifier_block nb;
+	struct notifier_block netevent_nb;
 	struct mlx5_ipsec_fs *roce;
 };
 
 struct mlx5e_ipsec_esn_state {
 	u32 esn;
-	u8 trigger: 1;
+	u32 esn_msb;
 	u8 overlap: 1;
 };
 
@@ -161,11 +187,13 @@ struct mlx5e_ipsec_rule {
 	struct mlx5_flow_handle *rule;
 	struct mlx5_modify_hdr *modify_hdr;
 	struct mlx5_pkt_reformat *pkt_reformat;
+	struct mlx5_fc *fc;
 };
 
-struct mlx5e_ipsec_modify_state_work {
-	struct work_struct		work;
-	struct mlx5_accel_esp_xfrm_attrs attrs;
+struct mlx5e_ipsec_limits {
+	u64 round;
+	u8 soft_limit_hit : 1;
+	u8 fix_limit : 1;
 };
 
 struct mlx5e_ipsec_sa_entry {
@@ -178,7 +206,9 @@ struct mlx5e_ipsec_sa_entry {
 	u32 ipsec_obj_id;
 	u32 enc_key_id;
 	struct mlx5e_ipsec_rule ipsec_rule;
-	struct mlx5e_ipsec_modify_state_work modify_work;
+	struct mlx5e_ipsec_work *work;
+	struct mlx5e_ipsec_dwork *dwork;
+	struct mlx5e_ipsec_limits limits;
 };
 
 struct mlx5_accel_pol_xfrm_attrs {
@@ -198,6 +228,7 @@ struct mlx5_accel_pol_xfrm_attrs {
 	u8 type : 2;
 	u8 dir : 2;
 	u32 reqid;
+	u32 prio;
 };
 
 struct mlx5e_ipsec_pol_entry {
@@ -219,6 +250,8 @@ int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry);
 void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry);
 int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry);
 void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry);
+void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry);
+bool mlx5e_ipsec_fs_tunnel_enabled(struct mlx5e_ipsec_sa_entry *sa_entry);
 
 int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);
 void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);
@@ -233,9 +266,6 @@ void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec);
 
 int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
 			  struct mlx5_wqe_aso_ctrl_seg *data);
-void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry,
-				   u64 *packets);
-
 void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv,
 				     void *ipsec_stats);
 
@@ -252,6 +282,13 @@ mlx5e_ipsec_pol2dev(struct mlx5e_ipsec_pol_entry *pol_entry)
 {
 	return pol_entry->ipsec->mdev;
 }
+
+static inline bool addr6_all_zero(__be32 *addr6)
+{
+	static const __be32 zaddr6[4] = {};
+
+	return !memcmp(addr6, zaddr6, sizeof(zaddr6));
+}
 #else
 static inline void mlx5e_ipsec_init(struct mlx5e_priv *priv)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
index 9871ba1b25ff..dbe87bf89c0d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
@@ -4,11 +4,15 @@
 #include <linux/netdevice.h>
 #include "en.h"
 #include "en/fs.h"
+#include "eswitch.h"
 #include "ipsec.h"
 #include "fs_core.h"
 #include "lib/ipsec_fs_roce.h"
+#include "lib/fs_chains.h"
 
 #define NUM_IPSEC_FTE BIT(15)
+#define MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE 16
+#define IPSEC_TUNNEL_DEFAULT_TTL 0x40
 
 struct mlx5e_ipsec_fc {
 	struct mlx5_fc *cnt;
@@ -34,13 +38,18 @@ struct mlx5e_ipsec_rx {
 	struct mlx5e_ipsec_miss sa;
 	struct mlx5e_ipsec_rule status;
 	struct mlx5e_ipsec_fc *fc;
+	struct mlx5_fs_chains *chains;
+	u8 allow_tunnel_mode : 1;
 };
 
 struct mlx5e_ipsec_tx {
 	struct mlx5e_ipsec_ft ft;
 	struct mlx5e_ipsec_miss pol;
+	struct mlx5e_ipsec_rule status;
 	struct mlx5_flow_namespace *ns;
 	struct mlx5e_ipsec_fc *fc;
+	struct mlx5_fs_chains *chains;
+	u8 allow_tunnel_mode : 1;
 };
 
 /* IPsec RX flow steering */
@@ -51,9 +60,70 @@ static enum mlx5_traffic_types family2tt(u32 family)
 	return MLX5_TT_IPV6_IPSEC_ESP;
 }
 
+static struct mlx5e_ipsec_rx *ipsec_rx(struct mlx5e_ipsec *ipsec, u32 family)
+{
+	if (family == AF_INET)
+		return ipsec->rx_ipv4;
+
+	return ipsec->rx_ipv6;
+}
+
+static struct mlx5_fs_chains *
+ipsec_chains_create(struct mlx5_core_dev *mdev, struct mlx5_flow_table *miss_ft,
+		    enum mlx5_flow_namespace_type ns, int base_prio,
+		    int base_level, struct mlx5_flow_table **root_ft)
+{
+	struct mlx5_chains_attr attr = {};
+	struct mlx5_fs_chains *chains;
+	struct mlx5_flow_table *ft;
+	int err;
+
+	attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED |
+		     MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
+	attr.max_grp_num = 2;
+	attr.default_ft = miss_ft;
+	attr.ns = ns;
+	attr.fs_base_prio = base_prio;
+	attr.fs_base_level = base_level;
+	chains = mlx5_chains_create(mdev, &attr);
+	if (IS_ERR(chains))
+		return chains;
+
+	/* Create chain 0, prio 1, level 0 to connect chains to prev in fs_core */
+	ft = mlx5_chains_get_table(chains, 0, 1, 0);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto err_chains_get;
+	}
+
+	*root_ft = ft;
+	return chains;
+
+err_chains_get:
+	mlx5_chains_destroy(chains);
+	return ERR_PTR(err);
+}
+
+static void ipsec_chains_destroy(struct mlx5_fs_chains *chains)
+{
+	mlx5_chains_put_table(chains, 0, 1, 0);
+	mlx5_chains_destroy(chains);
+}
+
+static struct mlx5_flow_table *
+ipsec_chains_get_table(struct mlx5_fs_chains *chains, u32 prio)
+{
+	return mlx5_chains_get_table(chains, 0, prio + 1, 0);
+}
+
+static void ipsec_chains_put_table(struct mlx5_fs_chains *chains, u32 prio)
+{
+	mlx5_chains_put_table(chains, 0, prio + 1, 0);
+}
+
 static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns,
 					       int level, int prio,
-					       int max_num_groups)
+					       int max_num_groups, u32 flags)
 {
 	struct mlx5_flow_table_attr ft_attr = {};
 
@@ -62,6 +132,7 @@ static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns,
 	ft_attr.max_fte = NUM_IPSEC_FTE;
 	ft_attr.level = level;
 	ft_attr.prio = prio;
+	ft_attr.flags = flags;
 
 	return mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
 }
@@ -170,14 +241,24 @@ out:
 static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
 		       struct mlx5e_ipsec_rx *rx, u32 family)
 {
-	mlx5_del_flow_rules(rx->pol.rule);
-	mlx5_destroy_flow_group(rx->pol.group);
-	mlx5_destroy_flow_table(rx->ft.pol);
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
+
+	/* disconnect */
+	mlx5_ttc_fwd_default_dest(ttc, family2tt(family));
+
+	if (rx->chains) {
+		ipsec_chains_destroy(rx->chains);
+	} else {
+		mlx5_del_flow_rules(rx->pol.rule);
+		mlx5_destroy_flow_group(rx->pol.group);
+		mlx5_destroy_flow_table(rx->ft.pol);
+	}
 
 	mlx5_del_flow_rules(rx->sa.rule);
 	mlx5_destroy_flow_group(rx->sa.group);
 	mlx5_destroy_flow_table(rx->ft.sa);
-
+	if (rx->allow_tunnel_mode)
+		mlx5_eswitch_unblock_encap(mdev);
 	mlx5_del_flow_rules(rx->status.rule);
 	mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr);
 	mlx5_destroy_flow_table(rx->ft.status);
@@ -193,6 +274,7 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
 	struct mlx5_flow_destination default_dest;
 	struct mlx5_flow_destination dest[2];
 	struct mlx5_flow_table *ft;
+	u32 flags = 0;
 	int err;
 
 	default_dest = mlx5_ttc_get_default_dest(ttc, family2tt(family));
@@ -203,7 +285,7 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
 		return err;
 
 	ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL,
-			     MLX5E_NIC_PRIO, 1);
+			     MLX5E_NIC_PRIO, 1, 0);
 	if (IS_ERR(ft)) {
 		err = PTR_ERR(ft);
 		goto err_fs_ft_status;
@@ -226,8 +308,12 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
 		goto err_add;
 
 	/* Create FT */
-	ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_NIC_PRIO,
-			     2);
+	if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL)
+		rx->allow_tunnel_mode = mlx5_eswitch_block_encap(mdev);
+	if (rx->allow_tunnel_mode)
+		flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
+	ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_NIC_PRIO, 2,
+			     flags);
 	if (IS_ERR(ft)) {
 		err = PTR_ERR(ft);
 		goto err_fs_ft;
@@ -238,8 +324,22 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
 	if (err)
 		goto err_fs;
 
+	if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) {
+		rx->chains = ipsec_chains_create(mdev, rx->ft.sa,
+						 MLX5_FLOW_NAMESPACE_KERNEL,
+						 MLX5E_NIC_PRIO,
+						 MLX5E_ACCEL_FS_POL_FT_LEVEL,
+						 &rx->ft.pol);
+		if (IS_ERR(rx->chains)) {
+			err = PTR_ERR(rx->chains);
+			goto err_pol_ft;
+		}
+
+		goto connect;
+	}
+
 	ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_POL_FT_LEVEL, MLX5E_NIC_PRIO,
-			     2);
+			     2, 0);
 	if (IS_ERR(ft)) {
 		err = PTR_ERR(ft);
 		goto err_pol_ft;
@@ -252,6 +352,12 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
 	if (err)
 		goto err_pol_miss;
 
+connect:
+	/* connect */
+	memset(dest, 0x00, sizeof(*dest));
+	dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest[0].ft = rx->ft.pol;
+	mlx5_ttc_fwd_dest(ttc, family2tt(family), &dest[0]);
 	return 0;
 
 err_pol_miss:
@@ -262,6 +368,8 @@ err_pol_ft:
 err_fs:
 	mlx5_destroy_flow_table(rx->ft.sa);
 err_fs_ft:
+	if (rx->allow_tunnel_mode)
+		mlx5_eswitch_unblock_encap(mdev);
 	mlx5_del_flow_rules(rx->status.rule);
 	mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr);
 err_add:
@@ -271,83 +379,191 @@ err_fs_ft_status:
 	return err;
 }
 
-static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev,
-					struct mlx5e_ipsec *ipsec, u32 family)
+static int rx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
+		  struct mlx5e_ipsec_rx *rx, u32 family)
 {
-	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
-	struct mlx5_flow_destination dest = {};
-	struct mlx5e_ipsec_rx *rx;
-	int err = 0;
-
-	if (family == AF_INET)
-		rx = ipsec->rx_ipv4;
-	else
-		rx = ipsec->rx_ipv6;
+	int err;
 
-	mutex_lock(&rx->ft.mutex);
 	if (rx->ft.refcnt)
 		goto skip;
 
-	/* create FT */
 	err = rx_create(mdev, ipsec, rx, family);
 	if (err)
-		goto out;
-
-	/* connect */
-	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
-	dest.ft = rx->ft.pol;
-	mlx5_ttc_fwd_dest(ttc, family2tt(family), &dest);
+		return err;
 
 skip:
 	rx->ft.refcnt++;
-out:
+	return 0;
+}
+
+static void rx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_rx *rx,
+		   u32 family)
+{
+	if (--rx->ft.refcnt)
+		return;
+
+	rx_destroy(ipsec->mdev, ipsec, rx, family);
+}
+
+static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev,
+					struct mlx5e_ipsec *ipsec, u32 family)
+{
+	struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family);
+	int err;
+
+	mutex_lock(&rx->ft.mutex);
+	err = rx_get(mdev, ipsec, rx, family);
 	mutex_unlock(&rx->ft.mutex);
 	if (err)
 		return ERR_PTR(err);
+
 	return rx;
 }
 
-static void rx_ft_put(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
-		      u32 family)
+static struct mlx5_flow_table *rx_ft_get_policy(struct mlx5_core_dev *mdev,
+						struct mlx5e_ipsec *ipsec,
+						u32 family, u32 prio)
 {
-	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
-	struct mlx5e_ipsec_rx *rx;
+	struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family);
+	struct mlx5_flow_table *ft;
+	int err;
 
-	if (family == AF_INET)
-		rx = ipsec->rx_ipv4;
-	else
-		rx = ipsec->rx_ipv6;
+	mutex_lock(&rx->ft.mutex);
+	err = rx_get(mdev, ipsec, rx, family);
+	if (err)
+		goto err_get;
+
+	ft = rx->chains ? ipsec_chains_get_table(rx->chains, prio) : rx->ft.pol;
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto err_get_ft;
+	}
+
+	mutex_unlock(&rx->ft.mutex);
+	return ft;
+
+err_get_ft:
+	rx_put(ipsec, rx, family);
+err_get:
+	mutex_unlock(&rx->ft.mutex);
+	return ERR_PTR(err);
+}
+
+static void rx_ft_put(struct mlx5e_ipsec *ipsec, u32 family)
+{
+	struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family);
 
 	mutex_lock(&rx->ft.mutex);
-	rx->ft.refcnt--;
-	if (rx->ft.refcnt)
-		goto out;
+	rx_put(ipsec, rx, family);
+	mutex_unlock(&rx->ft.mutex);
+}
 
-	/* disconnect */
-	mlx5_ttc_fwd_default_dest(ttc, family2tt(family));
+static void rx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 family, u32 prio)
+{
+	struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family);
 
-	/* remove FT */
-	rx_destroy(mdev, ipsec, rx, family);
+	mutex_lock(&rx->ft.mutex);
+	if (rx->chains)
+		ipsec_chains_put_table(rx->chains, prio);
 
-out:
+	rx_put(ipsec, rx, family);
 	mutex_unlock(&rx->ft.mutex);
 }
 
+static int ipsec_counter_rule_tx(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx)
+{
+	struct mlx5_flow_destination dest = {};
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_handle *fte;
+	struct mlx5_flow_spec *spec;
+	int err;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return -ENOMEM;
+
+	/* create fte */
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW |
+			  MLX5_FLOW_CONTEXT_ACTION_COUNT;
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+	dest.counter_id = mlx5_fc_id(tx->fc->cnt);
+	fte = mlx5_add_flow_rules(tx->ft.status, spec, &flow_act, &dest, 1);
+	if (IS_ERR(fte)) {
+		err = PTR_ERR(fte);
+		mlx5_core_err(mdev, "Fail to add ipsec tx counter rule err=%d\n", err);
+		goto err_rule;
+	}
+
+	kvfree(spec);
+	tx->status.rule = fte;
+	return 0;
+
+err_rule:
+	kvfree(spec);
+	return err;
+}
+
 /* IPsec TX flow steering */
+static void tx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx,
+		       struct mlx5_ipsec_fs *roce)
+{
+	mlx5_ipsec_fs_roce_tx_destroy(roce);
+	if (tx->chains) {
+		ipsec_chains_destroy(tx->chains);
+	} else {
+		mlx5_del_flow_rules(tx->pol.rule);
+		mlx5_destroy_flow_group(tx->pol.group);
+		mlx5_destroy_flow_table(tx->ft.pol);
+	}
+
+	mlx5_destroy_flow_table(tx->ft.sa);
+	if (tx->allow_tunnel_mode)
+		mlx5_eswitch_unblock_encap(mdev);
+	mlx5_del_flow_rules(tx->status.rule);
+	mlx5_destroy_flow_table(tx->ft.status);
+}
+
 static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx,
 		     struct mlx5_ipsec_fs *roce)
 {
 	struct mlx5_flow_destination dest = {};
 	struct mlx5_flow_table *ft;
+	u32 flags = 0;
 	int err;
 
-	ft = ipsec_ft_create(tx->ns, 1, 0, 4);
+	ft = ipsec_ft_create(tx->ns, 2, 0, 1, 0);
 	if (IS_ERR(ft))
 		return PTR_ERR(ft);
+	tx->ft.status = ft;
+
+	err = ipsec_counter_rule_tx(mdev, tx);
+	if (err)
+		goto err_status_rule;
 
+	if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL)
+		tx->allow_tunnel_mode = mlx5_eswitch_block_encap(mdev);
+	if (tx->allow_tunnel_mode)
+		flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
+	ft = ipsec_ft_create(tx->ns, 1, 0, 4, flags);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto err_sa_ft;
+	}
 	tx->ft.sa = ft;
 
-	ft = ipsec_ft_create(tx->ns, 0, 0, 2);
+	if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) {
+		tx->chains = ipsec_chains_create(
+			mdev, tx->ft.sa, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, 0, 0,
+			&tx->ft.pol);
+		if (IS_ERR(tx->chains)) {
+			err = PTR_ERR(tx->chains);
+			goto err_pol_ft;
+		}
+
+		goto connect_roce;
+	}
+
+	ft = ipsec_ft_create(tx->ns, 0, 0, 2, 0);
 	if (IS_ERR(ft)) {
 		err = PTR_ERR(ft);
 		goto err_pol_ft;
@@ -356,44 +572,102 @@ static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx,
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	dest.ft = tx->ft.sa;
 	err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest);
-	if (err)
-		goto err_pol_miss;
+	if (err) {
+		mlx5_destroy_flow_table(tx->ft.pol);
+		goto err_pol_ft;
+	}
 
+connect_roce:
 	err = mlx5_ipsec_fs_roce_tx_create(mdev, roce, tx->ft.pol);
 	if (err)
 		goto err_roce;
 	return 0;
 
 err_roce:
-	mlx5_del_flow_rules(tx->pol.rule);
-	mlx5_destroy_flow_group(tx->pol.group);
-err_pol_miss:
-	mlx5_destroy_flow_table(tx->ft.pol);
+	if (tx->chains) {
+		ipsec_chains_destroy(tx->chains);
+	} else {
+		mlx5_del_flow_rules(tx->pol.rule);
+		mlx5_destroy_flow_group(tx->pol.group);
+		mlx5_destroy_flow_table(tx->ft.pol);
+	}
 err_pol_ft:
 	mlx5_destroy_flow_table(tx->ft.sa);
+err_sa_ft:
+	if (tx->allow_tunnel_mode)
+		mlx5_eswitch_unblock_encap(mdev);
+	mlx5_del_flow_rules(tx->status.rule);
+err_status_rule:
+	mlx5_destroy_flow_table(tx->ft.status);
 	return err;
 }
 
-static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev,
-					struct mlx5e_ipsec *ipsec)
+static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
+		  struct mlx5e_ipsec_tx *tx)
 {
-	struct mlx5e_ipsec_tx *tx = ipsec->tx;
-	int err = 0;
+	int err;
 
-	mutex_lock(&tx->ft.mutex);
 	if (tx->ft.refcnt)
 		goto skip;
 
 	err = tx_create(mdev, tx, ipsec->roce);
 	if (err)
-		goto out;
+		return err;
 
 skip:
 	tx->ft.refcnt++;
-out:
+	return 0;
+}
+
+static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx)
+{
+	if (--tx->ft.refcnt)
+		return;
+
+	tx_destroy(ipsec->mdev, tx, ipsec->roce);
+}
+
+static struct mlx5_flow_table *tx_ft_get_policy(struct mlx5_core_dev *mdev,
+						struct mlx5e_ipsec *ipsec,
+						u32 prio)
+{
+	struct mlx5e_ipsec_tx *tx = ipsec->tx;
+	struct mlx5_flow_table *ft;
+	int err;
+
+	mutex_lock(&tx->ft.mutex);
+	err = tx_get(mdev, ipsec, tx);
+	if (err)
+		goto err_get;
+
+	ft = tx->chains ? ipsec_chains_get_table(tx->chains, prio) : tx->ft.pol;
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto err_get_ft;
+	}
+
+	mutex_unlock(&tx->ft.mutex);
+	return ft;
+
+err_get_ft:
+	tx_put(ipsec, tx);
+err_get:
+	mutex_unlock(&tx->ft.mutex);
+	return ERR_PTR(err);
+}
+
+static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev,
+					struct mlx5e_ipsec *ipsec)
+{
+	struct mlx5e_ipsec_tx *tx = ipsec->tx;
+	int err;
+
+	mutex_lock(&tx->ft.mutex);
+	err = tx_get(mdev, ipsec, tx);
 	mutex_unlock(&tx->ft.mutex);
 	if (err)
 		return ERR_PTR(err);
+
 	return tx;
 }
 
@@ -402,53 +676,72 @@ static void tx_ft_put(struct mlx5e_ipsec *ipsec)
 	struct mlx5e_ipsec_tx *tx = ipsec->tx;
 
 	mutex_lock(&tx->ft.mutex);
-	tx->ft.refcnt--;
-	if (tx->ft.refcnt)
-		goto out;
+	tx_put(ipsec, tx);
+	mutex_unlock(&tx->ft.mutex);
+}
 
-	mlx5_ipsec_fs_roce_tx_destroy(ipsec->roce);
-	mlx5_del_flow_rules(tx->pol.rule);
-	mlx5_destroy_flow_group(tx->pol.group);
-	mlx5_destroy_flow_table(tx->ft.pol);
-	mlx5_destroy_flow_table(tx->ft.sa);
-out:
+static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio)
+{
+	struct mlx5e_ipsec_tx *tx = ipsec->tx;
+
+	mutex_lock(&tx->ft.mutex);
+	if (tx->chains)
+		ipsec_chains_put_table(tx->chains, prio);
+
+	tx_put(ipsec, tx);
 	mutex_unlock(&tx->ft.mutex);
 }
 
 static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr,
 			    __be32 *daddr)
 {
+	if (!*saddr && !*daddr)
+		return;
+
 	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
 
 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
 	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
 
-	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
-			    outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4);
-	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
-			    outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4);
-	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
-			 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
-	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
-			 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
+	if (*saddr) {
+		memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+				    outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4);
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+				 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
+	}
+
+	if (*daddr) {
+		memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+				    outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4);
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+				 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
+	}
 }
 
 static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr,
 			    __be32 *daddr)
 {
+	if (addr6_all_zero(saddr) && addr6_all_zero(daddr))
+		return;
+
 	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
 
 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
 	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
 
-	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
-			    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16);
-	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
-			    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16);
-	memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
-			    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16);
-	memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
-			    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16);
+	if (!addr6_all_zero(saddr)) {
+		memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+				    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16);
+		memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
+				    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16);
+	}
+
+	if (!addr6_all_zero(daddr)) {
+		memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+				    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16);
+		memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
+				    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16);
+	}
 }
 
 static void setup_fte_esp(struct mlx5_flow_spec *spec)
@@ -560,40 +853,181 @@ static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir,
 	return 0;
 }
 
+static int
+setup_pkt_tunnel_reformat(struct mlx5_core_dev *mdev,
+			  struct mlx5_accel_esp_xfrm_attrs *attrs,
+			  struct mlx5_pkt_reformat_params *reformat_params)
+{
+	struct ip_esp_hdr *esp_hdr;
+	struct ipv6hdr *ipv6hdr;
+	struct ethhdr *eth_hdr;
+	struct iphdr *iphdr;
+	char *reformatbf;
+	size_t bfflen;
+	void *hdr;
+
+	bfflen = sizeof(*eth_hdr);
+
+	if (attrs->dir == XFRM_DEV_OFFLOAD_OUT) {
+		bfflen += sizeof(*esp_hdr) + 8;
+
+		switch (attrs->family) {
+		case AF_INET:
+			bfflen += sizeof(*iphdr);
+			break;
+		case AF_INET6:
+			bfflen += sizeof(*ipv6hdr);
+			break;
+		default:
+			return -EINVAL;
+		}
+	}
+
+	reformatbf = kzalloc(bfflen, GFP_KERNEL);
+	if (!reformatbf)
+		return -ENOMEM;
+
+	eth_hdr = (struct ethhdr *)reformatbf;
+	switch (attrs->family) {
+	case AF_INET:
+		eth_hdr->h_proto = htons(ETH_P_IP);
+		break;
+	case AF_INET6:
+		eth_hdr->h_proto = htons(ETH_P_IPV6);
+		break;
+	default:
+		goto free_reformatbf;
+	}
+
+	ether_addr_copy(eth_hdr->h_dest, attrs->dmac);
+	ether_addr_copy(eth_hdr->h_source, attrs->smac);
+
+	switch (attrs->dir) {
+	case XFRM_DEV_OFFLOAD_IN:
+		reformat_params->type = MLX5_REFORMAT_TYPE_L3_ESP_TUNNEL_TO_L2;
+		break;
+	case XFRM_DEV_OFFLOAD_OUT:
+		reformat_params->type = MLX5_REFORMAT_TYPE_L2_TO_L3_ESP_TUNNEL;
+		reformat_params->param_0 = attrs->authsize;
+
+		hdr = reformatbf + sizeof(*eth_hdr);
+		switch (attrs->family) {
+		case AF_INET:
+			iphdr = (struct iphdr *)hdr;
+			memcpy(&iphdr->saddr, &attrs->saddr.a4, 4);
+			memcpy(&iphdr->daddr, &attrs->daddr.a4, 4);
+			iphdr->version = 4;
+			iphdr->ihl = 5;
+			iphdr->ttl = IPSEC_TUNNEL_DEFAULT_TTL;
+			iphdr->protocol = IPPROTO_ESP;
+			hdr += sizeof(*iphdr);
+			break;
+		case AF_INET6:
+			ipv6hdr = (struct ipv6hdr *)hdr;
+			memcpy(&ipv6hdr->saddr, &attrs->saddr.a6, 16);
+			memcpy(&ipv6hdr->daddr, &attrs->daddr.a6, 16);
+			ipv6hdr->nexthdr = IPPROTO_ESP;
+			ipv6hdr->version = 6;
+			ipv6hdr->hop_limit = IPSEC_TUNNEL_DEFAULT_TTL;
+			hdr += sizeof(*ipv6hdr);
+			break;
+		default:
+			goto free_reformatbf;
+		}
+
+		esp_hdr = (struct ip_esp_hdr *)hdr;
+		esp_hdr->spi = htonl(attrs->spi);
+		break;
+	default:
+		goto free_reformatbf;
+	}
+
+	reformat_params->size = bfflen;
+	reformat_params->data = reformatbf;
+	return 0;
+
+free_reformatbf:
+	kfree(reformatbf);
+	return -EINVAL;
+}
+
+static int
+setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs,
+			     struct mlx5_pkt_reformat_params *reformat_params)
+{
+	u8 *reformatbf;
+	__be32 spi;
+
+	switch (attrs->dir) {
+	case XFRM_DEV_OFFLOAD_IN:
+		reformat_params->type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT;
+		break;
+	case XFRM_DEV_OFFLOAD_OUT:
+		if (attrs->family == AF_INET)
+			reformat_params->type =
+				MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4;
+		else
+			reformat_params->type =
+				MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6;
+
+		reformatbf = kzalloc(MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE,
+				     GFP_KERNEL);
+		if (!reformatbf)
+			return -ENOMEM;
+
+		/* convert to network format */
+		spi = htonl(attrs->spi);
+		memcpy(reformatbf, &spi, sizeof(spi));
+
+		reformat_params->param_0 = attrs->authsize;
+		reformat_params->size =
+			MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE;
+		reformat_params->data = reformatbf;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int setup_pkt_reformat(struct mlx5_core_dev *mdev,
 			      struct mlx5_accel_esp_xfrm_attrs *attrs,
 			      struct mlx5_flow_act *flow_act)
 {
-	enum mlx5_flow_namespace_type ns_type = MLX5_FLOW_NAMESPACE_EGRESS;
 	struct mlx5_pkt_reformat_params reformat_params = {};
 	struct mlx5_pkt_reformat *pkt_reformat;
-	u8 reformatbf[16] = {};
-	__be32 spi;
+	enum mlx5_flow_namespace_type ns_type;
+	int ret;
 
-	if (attrs->dir == XFRM_DEV_OFFLOAD_IN) {
-		reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT;
+	switch (attrs->dir) {
+	case XFRM_DEV_OFFLOAD_IN:
 		ns_type = MLX5_FLOW_NAMESPACE_KERNEL;
-		goto cmd;
+		break;
+	case XFRM_DEV_OFFLOAD_OUT:
+		ns_type = MLX5_FLOW_NAMESPACE_EGRESS;
+		break;
+	default:
+		return -EINVAL;
 	}
 
-	if (attrs->family == AF_INET)
-		reformat_params.type =
-			MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4;
-	else
-		reformat_params.type =
-			MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6;
-
-	/* convert to network format */
-	spi = htonl(attrs->spi);
-	memcpy(reformatbf, &spi, 4);
+	switch (attrs->mode) {
+	case XFRM_MODE_TRANSPORT:
+		ret = setup_pkt_transport_reformat(attrs, &reformat_params);
+		break;
+	case XFRM_MODE_TUNNEL:
+		ret = setup_pkt_tunnel_reformat(mdev, attrs, &reformat_params);
+		break;
+	default:
+		ret = -EINVAL;
+	}
 
-	reformat_params.param_0 = attrs->authsize;
-	reformat_params.size = sizeof(reformatbf);
-	reformat_params.data = &reformatbf;
+	if (ret)
+		return ret;
 
-cmd:
 	pkt_reformat =
 		mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type);
+	kfree(reformat_params.data);
 	if (IS_ERR(pkt_reformat))
 		return PTR_ERR(pkt_reformat);
 
@@ -607,11 +1041,12 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
 	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
 	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
 	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
-	struct mlx5_flow_destination dest = {};
+	struct mlx5_flow_destination dest[2];
 	struct mlx5_flow_act flow_act = {};
 	struct mlx5_flow_handle *rule;
 	struct mlx5_flow_spec *spec;
 	struct mlx5e_ipsec_rx *rx;
+	struct mlx5_fc *counter;
 	int err;
 
 	rx = rx_ft_get(mdev, ipsec, attrs->family);
@@ -648,14 +1083,25 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
 		break;
 	}
 
+	counter = mlx5_fc_create(mdev, true);
+	if (IS_ERR(counter)) {
+		err = PTR_ERR(counter);
+		goto err_add_cnt;
+	}
 	flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
 	flow_act.crypto.obj_id = sa_entry->ipsec_obj_id;
 	flow_act.flags |= FLOW_ACT_NO_APPEND;
-	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
-			   MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT;
-	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
-	dest.ft = rx->ft.status;
-	rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, &dest, 1);
+	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
+			   MLX5_FLOW_CONTEXT_ACTION_COUNT;
+	if (attrs->drop)
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+	else
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+	dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest[0].ft = rx->ft.status;
+	dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+	dest[1].counter_id = mlx5_fc_id(counter);
+	rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
 		mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err);
@@ -665,10 +1111,13 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
 
 	sa_entry->ipsec_rule.rule = rule;
 	sa_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr;
+	sa_entry->ipsec_rule.fc = counter;
 	sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat;
 	return 0;
 
 err_add_flow:
+	mlx5_fc_destroy(mdev, counter);
+err_add_cnt:
 	if (flow_act.pkt_reformat)
 		mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat);
 err_pkt_reformat:
@@ -676,7 +1125,7 @@ err_pkt_reformat:
 err_mod_header:
 	kvfree(spec);
 err_alloc:
-	rx_ft_put(mdev, ipsec, attrs->family);
+	rx_ft_put(ipsec, attrs->family);
 	return err;
 }
 
@@ -685,12 +1134,13 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
 	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
 	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
 	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
-	struct mlx5_flow_destination dest = {};
+	struct mlx5_flow_destination dest[2];
 	struct mlx5_flow_act flow_act = {};
 	struct mlx5_flow_handle *rule;
 	struct mlx5_flow_spec *spec;
 	struct mlx5e_ipsec_tx *tx;
-	int err = 0;
+	struct mlx5_fc *counter;
+	int err;
 
 	tx = tx_ft_get(mdev, ipsec);
 	if (IS_ERR(tx))
@@ -717,7 +1167,8 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
 		setup_fte_reg_a(spec);
 		break;
 	case XFRM_DEV_OFFLOAD_PACKET:
-		setup_fte_reg_c0(spec, attrs->reqid);
+		if (attrs->reqid)
+			setup_fte_reg_c0(spec, attrs->reqid);
 		err = setup_pkt_reformat(mdev, attrs, &flow_act);
 		if (err)
 			goto err_pkt_reformat;
@@ -726,15 +1177,27 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
 		break;
 	}
 
+	counter = mlx5_fc_create(mdev, true);
+	if (IS_ERR(counter)) {
+		err = PTR_ERR(counter);
+		goto err_add_cnt;
+	}
+
 	flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
 	flow_act.crypto.obj_id = sa_entry->ipsec_obj_id;
 	flow_act.flags |= FLOW_ACT_NO_APPEND;
-	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW |
-			   MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT |
+	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT |
 			   MLX5_FLOW_CONTEXT_ACTION_COUNT;
-	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
-	dest.counter_id = mlx5_fc_id(tx->fc->cnt);
-	rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, &dest, 1);
+	if (attrs->drop)
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+	else
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+
+	dest[0].ft = tx->ft.status;
+	dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+	dest[1].counter_id = mlx5_fc_id(counter);
+	rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, dest, 2);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
 		mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err);
@@ -743,10 +1206,13 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
 
 	kvfree(spec);
 	sa_entry->ipsec_rule.rule = rule;
+	sa_entry->ipsec_rule.fc = counter;
 	sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat;
 	return 0;
 
 err_add_flow:
+	mlx5_fc_destroy(mdev, counter);
+err_add_cnt:
 	if (flow_act.pkt_reformat)
 		mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat);
 err_pkt_reformat:
@@ -760,16 +1226,17 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
 {
 	struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs;
 	struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
+	struct mlx5e_ipsec_tx *tx = pol_entry->ipsec->tx;
 	struct mlx5_flow_destination dest[2] = {};
 	struct mlx5_flow_act flow_act = {};
 	struct mlx5_flow_handle *rule;
 	struct mlx5_flow_spec *spec;
-	struct mlx5e_ipsec_tx *tx;
+	struct mlx5_flow_table *ft;
 	int err, dstn = 0;
 
-	tx = tx_ft_get(mdev, pol_entry->ipsec);
-	if (IS_ERR(tx))
-		return PTR_ERR(tx);
+	ft = tx_ft_get_policy(mdev, pol_entry->ipsec, attrs->prio);
+	if (IS_ERR(ft))
+		return PTR_ERR(ft);
 
 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 	if (!spec) {
@@ -785,14 +1252,16 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
 	setup_fte_no_frags(spec);
 	setup_fte_upper_proto_match(spec, &attrs->upspec);
 
-	err = setup_modify_header(mdev, attrs->reqid, XFRM_DEV_OFFLOAD_OUT,
-				  &flow_act);
-	if (err)
-		goto err_mod_header;
-
 	switch (attrs->action) {
 	case XFRM_POLICY_ALLOW:
 		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+		if (!attrs->reqid)
+			break;
+
+		err = setup_modify_header(mdev, attrs->reqid,
+					  XFRM_DEV_OFFLOAD_OUT, &flow_act);
+		if (err)
+			goto err_mod_header;
 		break;
 	case XFRM_POLICY_BLOCK:
 		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP |
@@ -804,14 +1273,14 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
 	default:
 		WARN_ON(true);
 		err = -EINVAL;
-		goto err_action;
+		goto err_mod_header;
 	}
 
 	flow_act.flags |= FLOW_ACT_NO_APPEND;
 	dest[dstn].ft = tx->ft.sa;
 	dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	dstn++;
-	rule = mlx5_add_flow_rules(tx->ft.pol, spec, &flow_act, dest, dstn);
+	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
 		mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err);
@@ -824,11 +1293,12 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
 	return 0;
 
 err_action:
-	mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr);
+	if (flow_act.modify_hdr)
+		mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr);
 err_mod_header:
 	kvfree(spec);
 err_alloc:
-	tx_ft_put(pol_entry->ipsec);
+	tx_ft_put_policy(pol_entry->ipsec, attrs->prio);
 	return err;
 }
 
@@ -840,12 +1310,15 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
 	struct mlx5_flow_act flow_act = {};
 	struct mlx5_flow_handle *rule;
 	struct mlx5_flow_spec *spec;
+	struct mlx5_flow_table *ft;
 	struct mlx5e_ipsec_rx *rx;
 	int err, dstn = 0;
 
-	rx = rx_ft_get(mdev, pol_entry->ipsec, attrs->family);
-	if (IS_ERR(rx))
-		return PTR_ERR(rx);
+	ft = rx_ft_get_policy(mdev, pol_entry->ipsec, attrs->family, attrs->prio);
+	if (IS_ERR(ft))
+		return PTR_ERR(ft);
+
+	rx = ipsec_rx(pol_entry->ipsec, attrs->family);
 
 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 	if (!spec) {
@@ -880,7 +1353,7 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
 	dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	dest[dstn].ft = rx->ft.sa;
 	dstn++;
-	rule = mlx5_add_flow_rules(rx->ft.pol, spec, &flow_act, dest, dstn);
+	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
 		mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err);
@@ -894,7 +1367,7 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
 err_action:
 	kvfree(spec);
 err_alloc:
-	rx_ft_put(mdev, pol_entry->ipsec, attrs->family);
+	rx_ft_put_policy(pol_entry->ipsec, attrs->family, attrs->prio);
 	return err;
 }
 
@@ -1022,7 +1495,7 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
 	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
 
 	mlx5_del_flow_rules(ipsec_rule->rule);
-
+	mlx5_fc_destroy(mdev, ipsec_rule->fc);
 	if (ipsec_rule->pkt_reformat)
 		mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat);
 
@@ -1032,7 +1505,7 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
 	}
 
 	mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
-	rx_ft_put(mdev, sa_entry->ipsec, sa_entry->attrs.family);
+	rx_ft_put(sa_entry->ipsec, sa_entry->attrs.family);
 }
 
 int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry)
@@ -1051,12 +1524,15 @@ void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry)
 	mlx5_del_flow_rules(ipsec_rule->rule);
 
 	if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) {
-		rx_ft_put(mdev, pol_entry->ipsec, pol_entry->attrs.family);
+		rx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.family,
+				 pol_entry->attrs.prio);
 		return;
 	}
 
-	mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
-	tx_ft_put(pol_entry->ipsec);
+	if (ipsec_rule->modify_hdr)
+		mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
+
+	tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio);
 }
 
 void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec)
@@ -1126,3 +1602,31 @@ err_rx_ipv4:
 	kfree(ipsec->tx);
 	return err;
 }
+
+void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct mlx5e_ipsec_sa_entry sa_entry_shadow = {};
+	int err;
+
+	memcpy(&sa_entry_shadow, sa_entry, sizeof(*sa_entry));
+	memset(&sa_entry_shadow.ipsec_rule, 0x00, sizeof(sa_entry->ipsec_rule));
+
+	err = mlx5e_accel_ipsec_fs_add_rule(&sa_entry_shadow);
+	if (err)
+		return;
+
+	mlx5e_accel_ipsec_fs_del_rule(sa_entry);
+	memcpy(sa_entry, &sa_entry_shadow, sizeof(*sa_entry));
+}
+
+bool mlx5e_ipsec_fs_tunnel_enabled(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct mlx5e_ipsec_rx *rx =
+		ipsec_rx(sa_entry->ipsec, sa_entry->attrs.family);
+	struct mlx5e_ipsec_tx *tx = sa_entry->ipsec->tx;
+
+	if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT)
+		return tx->allow_tunnel_mode;
+
+	return rx->allow_tunnel_mode;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c
index 5fa7a4c40429..df90e19066bc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c
@@ -8,6 +8,7 @@
 
 enum {
 	MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET,
+	MLX5_IPSEC_ASO_REMOVE_FLOW_SOFT_LFT_OFFSET,
 };
 
 u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
@@ -36,11 +37,24 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
 	    MLX5_CAP_ETH(mdev, insert_trailer) && MLX5_CAP_ETH(mdev, swp))
 		caps |= MLX5_IPSEC_CAP_CRYPTO;
 
-	if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload) &&
-	    MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) &&
-	    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) &&
-	    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap))
-		caps |= MLX5_IPSEC_CAP_PACKET_OFFLOAD;
+	if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload)) {
+		if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev,
+					      reformat_add_esp_trasport) &&
+		    MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+					      reformat_del_esp_trasport) &&
+		    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap))
+			caps |= MLX5_IPSEC_CAP_PACKET_OFFLOAD;
+
+		if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) &&
+		    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level))
+			caps |= MLX5_IPSEC_CAP_PRIO;
+
+		if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev,
+					      reformat_l2_to_l3_esp_tunnel) &&
+		    MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+					      reformat_l3_esp_tunnel_to_l2))
+			caps |= MLX5_IPSEC_CAP_TUNNEL;
+	}
 
 	if (mlx5_get_roce_state(mdev) &&
 	    MLX5_CAP_GEN_2(mdev, flow_table_type_2_type) & MLX5_FT_NIC_RX_2_NIC_RX_RDMA &&
@@ -68,15 +82,17 @@ static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn,
 	void *aso_ctx;
 
 	aso_ctx = MLX5_ADDR_OF(ipsec_obj, obj, ipsec_aso);
-	if (attrs->esn_trigger) {
+	if (attrs->replay_esn.trigger) {
 		MLX5_SET(ipsec_aso, aso_ctx, esn_event_arm, 1);
 
 		if (attrs->dir == XFRM_DEV_OFFLOAD_IN) {
 			MLX5_SET(ipsec_aso, aso_ctx, window_sz,
-				 attrs->replay_window / 64);
+				 attrs->replay_esn.replay_window / 64);
 			MLX5_SET(ipsec_aso, aso_ctx, mode,
 				 MLX5_IPSEC_ASO_REPLAY_PROTECTION);
-			}
+		}
+		MLX5_SET(ipsec_aso, aso_ctx, mode_parameter,
+			 attrs->replay_esn.esn);
 	}
 
 	/* ASO context */
@@ -93,15 +109,15 @@ static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn,
 	if (attrs->dir == XFRM_DEV_OFFLOAD_OUT)
 		MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN);
 
-	if (attrs->hard_packet_limit != XFRM_INF) {
+	if (attrs->lft.hard_packet_limit != XFRM_INF) {
 		MLX5_SET(ipsec_aso, aso_ctx, remove_flow_pkt_cnt,
-			 lower_32_bits(attrs->hard_packet_limit));
+			 attrs->lft.hard_packet_limit);
 		MLX5_SET(ipsec_aso, aso_ctx, hard_lft_arm, 1);
 	}
 
-	if (attrs->soft_packet_limit != XFRM_INF) {
+	if (attrs->lft.soft_packet_limit != XFRM_INF) {
 		MLX5_SET(ipsec_aso, aso_ctx, remove_flow_soft_lft,
-			 lower_32_bits(attrs->soft_packet_limit));
+			 attrs->lft.soft_packet_limit);
 
 		MLX5_SET(ipsec_aso, aso_ctx, soft_lft_arm, 1);
 	}
@@ -128,10 +144,10 @@ static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
 	salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
 	memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
 	/* esn */
-	if (attrs->esn_trigger) {
+	if (attrs->replay_esn.trigger) {
 		MLX5_SET(ipsec_obj, obj, esn_en, 1);
-		MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn);
-		MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->esn_overlap);
+		MLX5_SET(ipsec_obj, obj, esn_msb, attrs->replay_esn.esn_msb);
+		MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->replay_esn.overlap);
 	}
 
 	MLX5_SET(ipsec_obj, obj, dekn, sa_entry->enc_key_id);
@@ -217,9 +233,6 @@ static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry,
 	void *obj;
 	int err;
 
-	if (!attrs->esn_trigger)
-		return 0;
-
 	general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
 	if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
 		return -EINVAL;
@@ -247,8 +260,8 @@ static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry,
 	MLX5_SET64(ipsec_obj, obj, modify_field_select,
 		   MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP |
 			   MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB);
-	MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn);
-	MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->esn_overlap);
+	MLX5_SET(ipsec_obj, obj, esn_msb, attrs->replay_esn.esn_msb);
+	MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->replay_esn.overlap);
 
 	/* general object fields set */
 	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
@@ -268,29 +281,24 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry,
 	memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs));
 }
 
-static void
-mlx5e_ipsec_aso_update_esn(struct mlx5e_ipsec_sa_entry *sa_entry,
-			   const struct mlx5_accel_esp_xfrm_attrs *attrs)
+static void mlx5e_ipsec_aso_update(struct mlx5e_ipsec_sa_entry *sa_entry,
+				   struct mlx5_wqe_aso_ctrl_seg *data)
 {
-	struct mlx5_wqe_aso_ctrl_seg data = {};
+	data->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT << 6;
+	data->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE |
+				      MLX5_ASO_ALWAYS_TRUE << 4;
 
-	data.data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT << 6;
-	data.condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | MLX5_ASO_ALWAYS_TRUE
-								    << 4;
-	data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
-	data.bitwise_data = cpu_to_be64(BIT_ULL(54));
-	data.data_mask = data.bitwise_data;
-
-	mlx5e_ipsec_aso_query(sa_entry, &data);
+	mlx5e_ipsec_aso_query(sa_entry, data);
 }
 
 static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
 					 u32 mode_param)
 {
 	struct mlx5_accel_esp_xfrm_attrs attrs = {};
+	struct mlx5_wqe_aso_ctrl_seg data = {};
 
 	if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) {
-		sa_entry->esn_state.esn++;
+		sa_entry->esn_state.esn_msb++;
 		sa_entry->esn_state.overlap = 0;
 	} else {
 		sa_entry->esn_state.overlap = 1;
@@ -298,25 +306,129 @@ static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
 
 	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs);
 	mlx5_accel_esp_modify_xfrm(sa_entry, &attrs);
-	mlx5e_ipsec_aso_update_esn(sa_entry, &attrs);
+
+	data.data_offset_condition_operand =
+		MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
+	data.bitwise_data = cpu_to_be64(BIT_ULL(54));
+	data.data_mask = data.bitwise_data;
+
+	mlx5e_ipsec_aso_update(sa_entry, &data);
+}
+
+static void mlx5e_ipsec_aso_update_hard(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct mlx5_wqe_aso_ctrl_seg data = {};
+
+	data.data_offset_condition_operand =
+		MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
+	data.bitwise_data = cpu_to_be64(BIT_ULL(57) + BIT_ULL(31));
+	data.data_mask = data.bitwise_data;
+	mlx5e_ipsec_aso_update(sa_entry, &data);
+}
+
+static void mlx5e_ipsec_aso_update_soft(struct mlx5e_ipsec_sa_entry *sa_entry,
+					u32 val)
+{
+	struct mlx5_wqe_aso_ctrl_seg data = {};
+
+	data.data_offset_condition_operand =
+		MLX5_IPSEC_ASO_REMOVE_FLOW_SOFT_LFT_OFFSET;
+	data.bitwise_data = cpu_to_be64(val);
+	data.data_mask = cpu_to_be64(U32_MAX);
+	mlx5e_ipsec_aso_update(sa_entry, &data);
+}
+
+static void mlx5e_ipsec_handle_limits(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
+	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
+	struct mlx5e_ipsec_aso *aso = ipsec->aso;
+	bool soft_arm, hard_arm;
+	u64 hard_cnt;
+
+	lockdep_assert_held(&sa_entry->x->lock);
+
+	soft_arm = !MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm);
+	hard_arm = !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm);
+	if (!soft_arm && !hard_arm)
+		/* It is not lifetime event */
+		return;
+
+	hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt);
+	if (!hard_cnt || hard_arm) {
+		/* It is possible to see packet counter equal to zero without
+		 * hard limit event armed. Such situation can be if packet
+		 * decreased, while we handled soft limit event.
+		 *
+		 * However it will be HW/FW bug if hard limit event is raised
+		 * and packet counter is not zero.
+		 */
+		WARN_ON_ONCE(hard_arm && hard_cnt);
+
+		/* Notify about hard limit */
+		xfrm_state_check_expire(sa_entry->x);
+		return;
+	}
+
+	/* We are in soft limit event. */
+	if (!sa_entry->limits.soft_limit_hit &&
+	    sa_entry->limits.round == attrs->lft.numb_rounds_soft) {
+		sa_entry->limits.soft_limit_hit = true;
+		/* Notify about soft limit */
+		xfrm_state_check_expire(sa_entry->x);
+
+		if (sa_entry->limits.round == attrs->lft.numb_rounds_hard)
+			goto hard;
+
+		if (attrs->lft.soft_packet_limit > BIT_ULL(31)) {
+			/* We cannot avoid a soft_value that might have the high
+			 * bit set. For instance soft_value=2^31+1 cannot be
+			 * adjusted to the low bit clear version of soft_value=1
+			 * because it is too close to 0.
+			 *
+			 * Thus we have this corner case where we can hit the
+			 * soft_limit with the high bit set, but cannot adjust
+			 * the counter. Thus we set a temporary interrupt_value
+			 * at least 2^30 away from here and do the adjustment
+			 * then.
+			 */
+			mlx5e_ipsec_aso_update_soft(sa_entry,
+						    BIT_ULL(31) - BIT_ULL(30));
+			sa_entry->limits.fix_limit = true;
+			return;
+		}
+
+		sa_entry->limits.fix_limit = true;
+	}
+
+hard:
+	if (sa_entry->limits.round == attrs->lft.numb_rounds_hard) {
+		mlx5e_ipsec_aso_update_soft(sa_entry, 0);
+		attrs->lft.soft_packet_limit = XFRM_INF;
+		return;
+	}
+
+	mlx5e_ipsec_aso_update_hard(sa_entry);
+	sa_entry->limits.round++;
+	if (sa_entry->limits.round == attrs->lft.numb_rounds_soft)
+		mlx5e_ipsec_aso_update_soft(sa_entry,
+					    attrs->lft.soft_packet_limit);
+	if (sa_entry->limits.fix_limit) {
+		sa_entry->limits.fix_limit = false;
+		mlx5e_ipsec_aso_update_soft(sa_entry, BIT_ULL(31) - 1);
+	}
 }
 
 static void mlx5e_ipsec_handle_event(struct work_struct *_work)
 {
 	struct mlx5e_ipsec_work *work =
 		container_of(_work, struct mlx5e_ipsec_work, work);
+	struct mlx5e_ipsec_sa_entry *sa_entry = work->data;
 	struct mlx5_accel_esp_xfrm_attrs *attrs;
-	struct mlx5e_ipsec_sa_entry *sa_entry;
 	struct mlx5e_ipsec_aso *aso;
-	struct mlx5e_ipsec *ipsec;
 	int ret;
 
-	sa_entry = xa_load(&work->ipsec->sadb, work->id);
-	if (!sa_entry)
-		goto out;
-
-	ipsec = sa_entry->ipsec;
-	aso = ipsec->aso;
+	aso = sa_entry->ipsec->aso;
 	attrs = &sa_entry->attrs;
 
 	spin_lock(&sa_entry->x->lock);
@@ -324,21 +436,18 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work)
 	if (ret)
 		goto unlock;
 
-	if (attrs->esn_trigger &&
+	if (attrs->replay_esn.trigger &&
 	    !MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) {
 		u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter);
 
 		mlx5e_ipsec_update_esn_state(sa_entry, mode_param);
 	}
 
-	if (attrs->soft_packet_limit != XFRM_INF)
-		if (!MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm) ||
-		    !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm))
-			xfrm_state_check_expire(sa_entry->x);
+	if (attrs->lft.soft_packet_limit != XFRM_INF)
+		mlx5e_ipsec_handle_limits(sa_entry);
 
 unlock:
 	spin_unlock(&sa_entry->x->lock);
-out:
 	kfree(work);
 }
 
@@ -346,6 +455,7 @@ static int mlx5e_ipsec_event(struct notifier_block *nb, unsigned long event,
 			     void *data)
 {
 	struct mlx5e_ipsec *ipsec = container_of(nb, struct mlx5e_ipsec, nb);
+	struct mlx5e_ipsec_sa_entry *sa_entry;
 	struct mlx5_eqe_obj_change *object;
 	struct mlx5e_ipsec_work *work;
 	struct mlx5_eqe *eqe = data;
@@ -360,13 +470,16 @@ static int mlx5e_ipsec_event(struct notifier_block *nb, unsigned long event,
 	if (type != MLX5_GENERAL_OBJECT_TYPES_IPSEC)
 		return NOTIFY_DONE;
 
+	sa_entry = xa_load(&ipsec->sadb, be32_to_cpu(object->obj_id));
+	if (!sa_entry)
+		return NOTIFY_DONE;
+
 	work = kmalloc(sizeof(*work), GFP_ATOMIC);
 	if (!work)
 		return NOTIFY_DONE;
 
 	INIT_WORK(&work->work, mlx5e_ipsec_handle_event);
-	work->ipsec = ipsec;
-	work->id = be32_to_cpu(object->obj_id);
+	work->data = sa_entry;
 
 	queue_work(ipsec->wq, &work->work);
 	return NOTIFY_OK;
@@ -457,6 +570,7 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
 	struct mlx5_wqe_aso_ctrl_seg *ctrl;
 	struct mlx5e_hw_objs *res;
 	struct mlx5_aso_wqe *wqe;
+	unsigned long expires;
 	u8 ds_cnt;
 	int ret;
 
@@ -478,22 +592,12 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
 	mlx5e_ipsec_aso_copy(ctrl, data);
 
 	mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl);
-	ret = mlx5_aso_poll_cq(aso->aso, false);
+	expires = jiffies + msecs_to_jiffies(10);
+	do {
+		ret = mlx5_aso_poll_cq(aso->aso, false);
+		if (ret)
+			usleep_range(2, 10);
+	} while (ret && time_is_after_jiffies(expires));
 	spin_unlock_bh(&aso->lock);
 	return ret;
 }
-
-void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry,
-				   u64 *packets)
-{
-	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
-	struct mlx5e_ipsec_aso *aso = ipsec->aso;
-	u64 hard_cnt;
-
-	hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt);
-	/* HW decresases the limit till it reaches zero to fire an avent.
-	 * We need to fix the calculations, so the returned count is a total
-	 * number of passed packets and not how much left.
-	 */
-	*packets = sa_entry->attrs.hard_packet_limit - hard_cnt;
-}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c
index 33b3620ea45c..6b7b563f844a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c
@@ -4,6 +4,7 @@
 #include <linux/mlx5/device.h>
 #include <linux/mlx5/mlx5_ifc.h>
 #include <linux/xarray.h>
+#include <linux/if_vlan.h>
 
 #include "en.h"
 #include "lib/aso.h"
@@ -348,12 +349,21 @@ static void mlx5e_macsec_cleanup_sa(struct mlx5e_macsec *macsec,
 	sa->macsec_rule = NULL;
 }
 
+static struct mlx5e_priv *macsec_netdev_priv(const struct net_device *dev)
+{
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
+	if (is_vlan_dev(dev))
+		return netdev_priv(vlan_dev_priv(dev)->real_dev);
+#endif
+	return netdev_priv(dev);
+}
+
 static int mlx5e_macsec_init_sa(struct macsec_context *ctx,
 				struct mlx5e_macsec_sa *sa,
 				bool encrypt,
 				bool is_tx)
 {
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	struct mlx5e_macsec *macsec = priv->macsec;
 	struct mlx5_macsec_rule_attrs rule_attrs;
 	struct mlx5_core_dev *mdev = priv->mdev;
@@ -427,7 +437,7 @@ static int macsec_rx_sa_active_update(struct macsec_context *ctx,
 				      struct mlx5e_macsec_sa *rx_sa,
 				      bool active)
 {
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	struct mlx5e_macsec *macsec = priv->macsec;
 	int err = 0;
 
@@ -508,9 +518,9 @@ static void update_macsec_epn(struct mlx5e_macsec_sa *sa, const struct macsec_ke
 
 static int mlx5e_macsec_add_txsa(struct macsec_context *ctx)
 {
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc;
 	const struct macsec_tx_sa *ctx_tx_sa = ctx->sa.tx_sa;
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
 	const struct macsec_secy *secy = ctx->secy;
 	struct mlx5e_macsec_device *macsec_device;
 	struct mlx5_core_dev *mdev = priv->mdev;
@@ -583,9 +593,9 @@ out:
 
 static int mlx5e_macsec_upd_txsa(struct macsec_context *ctx)
 {
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc;
 	const struct macsec_tx_sa *ctx_tx_sa = ctx->sa.tx_sa;
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
 	struct mlx5e_macsec_device *macsec_device;
 	u8 assoc_num = ctx->sa.assoc_num;
 	struct mlx5e_macsec_sa *tx_sa;
@@ -645,7 +655,7 @@ out:
 
 static int mlx5e_macsec_del_txsa(struct macsec_context *ctx)
 {
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	struct mlx5e_macsec_device *macsec_device;
 	u8 assoc_num = ctx->sa.assoc_num;
 	struct mlx5e_macsec_sa *tx_sa;
@@ -670,7 +680,7 @@ static int mlx5e_macsec_del_txsa(struct macsec_context *ctx)
 
 	mlx5e_macsec_cleanup_sa(macsec, tx_sa, true);
 	mlx5_destroy_encryption_key(macsec->mdev, tx_sa->enc_key_id);
-	kfree_rcu(tx_sa);
+	kfree_rcu_mightsleep(tx_sa);
 	macsec_device->tx_sa[assoc_num] = NULL;
 
 out:
@@ -696,7 +706,7 @@ static u32 mlx5e_macsec_get_sa_from_hashtable(struct rhashtable *sci_hash, sci_t
 static int mlx5e_macsec_add_rxsc(struct macsec_context *ctx)
 {
 	struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element;
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	const struct macsec_rx_sc *ctx_rx_sc = ctx->rx_sc;
 	struct mlx5e_macsec_device *macsec_device;
 	struct mlx5e_macsec_rx_sc *rx_sc;
@@ -776,7 +786,7 @@ out:
 
 static int mlx5e_macsec_upd_rxsc(struct macsec_context *ctx)
 {
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	const struct macsec_rx_sc *ctx_rx_sc = ctx->rx_sc;
 	struct mlx5e_macsec_device *macsec_device;
 	struct mlx5e_macsec_rx_sc *rx_sc;
@@ -849,12 +859,12 @@ static void macsec_del_rxsc_ctx(struct mlx5e_macsec *macsec, struct mlx5e_macsec
 	xa_erase(&macsec->sc_xarray, rx_sc->sc_xarray_element->fs_id);
 	metadata_dst_free(rx_sc->md_dst);
 	kfree(rx_sc->sc_xarray_element);
-	kfree_rcu(rx_sc);
+	kfree_rcu_mightsleep(rx_sc);
 }
 
 static int mlx5e_macsec_del_rxsc(struct macsec_context *ctx)
 {
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	struct mlx5e_macsec_device *macsec_device;
 	struct mlx5e_macsec_rx_sc *rx_sc;
 	struct mlx5e_macsec *macsec;
@@ -890,8 +900,8 @@ out:
 
 static int mlx5e_macsec_add_rxsa(struct macsec_context *ctx)
 {
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	const struct macsec_rx_sa *ctx_rx_sa = ctx->sa.rx_sa;
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
 	struct mlx5e_macsec_device *macsec_device;
 	struct mlx5_core_dev *mdev = priv->mdev;
 	u8 assoc_num = ctx->sa.assoc_num;
@@ -976,8 +986,8 @@ out:
 
 static int mlx5e_macsec_upd_rxsa(struct macsec_context *ctx)
 {
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	const struct macsec_rx_sa *ctx_rx_sa = ctx->sa.rx_sa;
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
 	struct mlx5e_macsec_device *macsec_device;
 	u8 assoc_num = ctx->sa.assoc_num;
 	struct mlx5e_macsec_rx_sc *rx_sc;
@@ -1033,7 +1043,7 @@ out:
 
 static int mlx5e_macsec_del_rxsa(struct macsec_context *ctx)
 {
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	struct mlx5e_macsec_device *macsec_device;
 	sci_t sci = ctx->sa.rx_sa->sc->sci;
 	struct mlx5e_macsec_rx_sc *rx_sc;
@@ -1085,7 +1095,7 @@ out:
 
 static int mlx5e_macsec_add_secy(struct macsec_context *ctx)
 {
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	const struct net_device *dev = ctx->secy->netdev;
 	const struct net_device *netdev = ctx->netdev;
 	struct mlx5e_macsec_device *macsec_device;
@@ -1137,7 +1147,7 @@ out:
 static int macsec_upd_secy_hw_address(struct macsec_context *ctx,
 				      struct mlx5e_macsec_device *macsec_device)
 {
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	const struct net_device *dev = ctx->secy->netdev;
 	struct mlx5e_macsec *macsec = priv->macsec;
 	struct mlx5e_macsec_rx_sc *rx_sc, *tmp;
@@ -1184,8 +1194,8 @@ out:
  */
 static int mlx5e_macsec_upd_secy(struct macsec_context *ctx)
 {
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc;
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
 	const struct net_device *dev = ctx->secy->netdev;
 	struct mlx5e_macsec_device *macsec_device;
 	struct mlx5e_macsec_sa *tx_sa;
@@ -1240,7 +1250,7 @@ out:
 
 static int mlx5e_macsec_del_secy(struct macsec_context *ctx)
 {
-	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
 	struct mlx5e_macsec_device *macsec_device;
 	struct mlx5e_macsec_rx_sc *rx_sc, *tmp;
 	struct mlx5e_macsec_sa *tx_sa;
@@ -1741,7 +1751,7 @@ void mlx5e_macsec_offload_handle_rx_skb(struct net_device *netdev,
 {
 	struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element;
 	u32 macsec_meta_data = be32_to_cpu(cqe->ft_metadata);
-	struct mlx5e_priv *priv = netdev_priv(netdev);
+	struct mlx5e_priv *priv = macsec_netdev_priv(netdev);
 	struct mlx5e_macsec_rx_sc *rx_sc;
 	struct mlx5e_macsec *macsec;
 	u32  fs_id;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c
index 5b658a5588c6..7fc901a6ec5f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c
@@ -4,6 +4,7 @@
 #include <net/macsec.h>
 #include <linux/netdevice.h>
 #include <linux/mlx5/qp.h>
+#include <linux/if_vlan.h>
 #include "fs_core.h"
 #include "en/fs.h"
 #include "en_accel/macsec_fs.h"
@@ -292,8 +293,6 @@ static int macsec_fs_tx_create(struct mlx5e_macsec_fs *macsec_fs)
 	}
 
 	/* Tx crypto table MKE rule - MKE packets shouldn't be offloaded */
-	memset(&flow_act, 0, sizeof(flow_act));
-	memset(spec, 0, sizeof(*spec));
 	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 
 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
@@ -510,6 +509,8 @@ static void macsec_fs_tx_del_rule(struct mlx5e_macsec_fs *macsec_fs,
 	macsec_fs_tx_ft_put(macsec_fs);
 }
 
+#define MLX5_REFORMAT_PARAM_ADD_MACSEC_OFFSET_4_BYTES 1
+
 static union mlx5e_macsec_rule *
 macsec_fs_tx_add_rule(struct mlx5e_macsec_fs *macsec_fs,
 		      const struct macsec_context *macsec_ctx,
@@ -555,6 +556,10 @@ macsec_fs_tx_add_rule(struct mlx5e_macsec_fs *macsec_fs,
 	reformat_params.type = MLX5_REFORMAT_TYPE_ADD_MACSEC;
 	reformat_params.size = reformat_size;
 	reformat_params.data = reformatbf;
+
+	if (is_vlan_dev(macsec_ctx->netdev))
+		reformat_params.param_0 = MLX5_REFORMAT_PARAM_ADD_MACSEC_OFFSET_4_BYTES;
+
 	flow_act.pkt_reformat = mlx5_packet_reformat_alloc(macsec_fs->mdev,
 							   &reformat_params,
 							   MLX5_FLOW_NAMESPACE_EGRESS_MACSEC);
@@ -1109,7 +1114,6 @@ static void macsec_fs_rx_setup_fte(struct mlx5_flow_spec *spec,
 
 static union mlx5e_macsec_rule *
 macsec_fs_rx_add_rule(struct mlx5e_macsec_fs *macsec_fs,
-		      const struct macsec_context *macsec_ctx,
 		      struct mlx5_macsec_rule_attrs *attrs,
 		      u32 fs_id)
 {
@@ -1334,7 +1338,7 @@ mlx5e_macsec_fs_add_rule(struct mlx5e_macsec_fs *macsec_fs,
 {
 	return (attrs->action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ?
 		macsec_fs_tx_add_rule(macsec_fs, macsec_ctx, attrs, sa_fs_id) :
-		macsec_fs_rx_add_rule(macsec_fs, macsec_ctx, attrs, *sa_fs_id);
+		macsec_fs_rx_add_rule(macsec_fs, attrs, *sa_fs_id);
 }
 
 void mlx5e_macsec_fs_del_rule(struct mlx5e_macsec_fs *macsec_fs,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index 79fd21ecb9cb..1f5a2110d31f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -220,7 +220,7 @@ static void mlx5e_ethtool_get_speed_arr(struct mlx5_core_dev *mdev,
 					struct ptys2ethtool_config **arr,
 					u32 *size)
 {
-	bool ext = mlx5e_ptys_ext_supported(mdev);
+	bool ext = mlx5_ptys_ext_supported(mdev);
 
 	*arr = ext ? ptys2ext_ethtool_table : ptys2legacy_ethtool_table;
 	*size = ext ? ARRAY_SIZE(ptys2ext_ethtool_table) :
@@ -895,7 +895,7 @@ static void get_speed_duplex(struct net_device *netdev,
 	if (!netif_carrier_ok(netdev))
 		goto out;
 
-	speed = mlx5e_port_ptys2speed(priv->mdev, eth_proto_oper, force_legacy);
+	speed = mlx5_port_ptys2speed(priv->mdev, eth_proto_oper, force_legacy);
 	if (!speed) {
 		if (data_rate_oper)
 			speed = 100 * data_rate_oper;
@@ -980,7 +980,7 @@ static void get_lp_advertising(struct mlx5_core_dev *mdev, u32 eth_proto_lp,
 			       struct ethtool_link_ksettings *link_ksettings)
 {
 	unsigned long *lp_advertising = link_ksettings->link_modes.lp_advertising;
-	bool ext = mlx5e_ptys_ext_supported(mdev);
+	bool ext = mlx5_ptys_ext_supported(mdev);
 
 	ptys2ethtool_adver_link(lp_advertising, eth_proto_lp, ext);
 }
@@ -1160,7 +1160,7 @@ int mlx5e_ethtool_set_link_ksettings(struct mlx5e_priv *priv,
 				     const struct ethtool_link_ksettings *link_ksettings)
 {
 	struct mlx5_core_dev *mdev = priv->mdev;
-	struct mlx5e_port_eth_proto eproto;
+	struct mlx5_port_eth_proto eproto;
 	const unsigned long *adver;
 	bool an_changes = false;
 	u8 an_disable_admin;
@@ -1180,7 +1180,7 @@ int mlx5e_ethtool_set_link_ksettings(struct mlx5e_priv *priv,
 	autoneg = link_ksettings->base.autoneg;
 	speed = link_ksettings->base.speed;
 
-	ext_supported = mlx5e_ptys_ext_supported(mdev);
+	ext_supported = mlx5_ptys_ext_supported(mdev);
 	ext = ext_requested(autoneg, adver, ext_supported);
 	if (!ext_supported && ext)
 		return -EOPNOTSUPP;
@@ -1194,7 +1194,7 @@ int mlx5e_ethtool_set_link_ksettings(struct mlx5e_priv *priv,
 		goto out;
 	}
 	link_modes = autoneg == AUTONEG_ENABLE ? ethtool2ptys_adver_func(adver) :
-		mlx5e_port_speed2linkmodes(mdev, speed, !ext);
+		mlx5_port_speed2linkmodes(mdev, speed, !ext);
 
 	err = mlx5e_speed_validate(priv->netdev, ext, link_modes, autoneg);
 	if (err)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
index 05796f8b1d7c..33bfe4d7338b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
@@ -783,6 +783,7 @@ static int mlx5e_create_promisc_table(struct mlx5e_flow_steering *fs)
 	ft->t = mlx5_create_auto_grouped_flow_table(fs->ns, &ft_attr);
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
+		ft->t = NULL;
 		fs_err(fs, "fail to create promisc table err=%d\n", err);
 		return err;
 	}
@@ -810,7 +811,7 @@ static void mlx5e_del_promisc_rule(struct mlx5e_flow_steering *fs)
 
 static void mlx5e_destroy_promisc_table(struct mlx5e_flow_steering *fs)
 {
-	if (WARN(!fs->promisc.ft.t, "Trying to remove non-existing promiscuous table"))
+	if (!fs->promisc.ft.t)
 		return;
 	mlx5e_del_promisc_rule(fs);
 	mlx5_destroy_flow_table(fs->promisc.ft.t);
@@ -1490,6 +1491,8 @@ err:
 
 void mlx5e_fs_cleanup(struct mlx5e_flow_steering *fs)
 {
+	if (!fs)
+		return;
 	debugfs_remove_recursive(fs->dfs_root);
 	mlx5e_fs_ethtool_free(fs);
 	mlx5e_fs_tc_free(fs);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 7ca7e9b57607..2944691f06ad 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -262,23 +262,30 @@ static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, int node)
 
 	shampo->bitmap = bitmap_zalloc_node(shampo->hd_per_wq, GFP_KERNEL,
 					    node);
-	if (!shampo->bitmap)
-		return -ENOMEM;
-
 	shampo->info = kvzalloc_node(array_size(shampo->hd_per_wq,
 						sizeof(*shampo->info)),
 				     GFP_KERNEL, node);
-	if (!shampo->info) {
-		kvfree(shampo->bitmap);
-		return -ENOMEM;
-	}
+	shampo->pages = kvzalloc_node(array_size(shampo->hd_per_wq,
+						 sizeof(*shampo->pages)),
+				     GFP_KERNEL, node);
+	if (!shampo->bitmap || !shampo->info || !shampo->pages)
+		goto err_nomem;
+
 	return 0;
+
+err_nomem:
+	kvfree(shampo->info);
+	kvfree(shampo->bitmap);
+	kvfree(shampo->pages);
+
+	return -ENOMEM;
 }
 
 static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq)
 {
 	kvfree(rq->mpwqe.shampo->bitmap);
 	kvfree(rq->mpwqe.shampo->info);
+	kvfree(rq->mpwqe.shampo->pages);
 }
 
 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
@@ -286,13 +293,23 @@ static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
 	int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 	size_t alloc_size;
 
-	alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, alloc_units,
+	alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info,
+						   alloc_units.frag_pages,
 						   rq->mpwqe.pages_per_wqe));
 
 	rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node);
 	if (!rq->mpwqe.info)
 		return -ENOMEM;
 
+	/* For deferred page release (release right before alloc), make sure
+	 * that on first round release is not called.
+	 */
+	for (int i = 0; i < wq_sz; i++) {
+		struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, i);
+
+		bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
+	}
+
 	mlx5e_build_umr_wqe(rq, rq->icosq, &rq->mpwqe.umr_wqe);
 
 	return 0;
@@ -499,14 +516,12 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
 	struct mlx5e_wqe_frag_info *prev = NULL;
 	int i;
 
-	if (rq->xsk_pool) {
-		/* Assumptions used by XSK batched allocator. */
-		WARN_ON(rq->wqe.info.num_frags != 1);
-		WARN_ON(rq->wqe.info.log_num_frags != 0);
-		WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE);
-	}
+	WARN_ON(rq->xsk_pool);
+
+	next_frag.frag_page = &rq->wqe.alloc_units->frag_pages[0];
 
-	next_frag.au = &rq->wqe.alloc_units[0];
+	/* Skip first release due to deferred release. */
+	next_frag.flags = BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
 
 	for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
 		struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
@@ -516,10 +531,11 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
 
 		for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
 			if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
-				next_frag.au++;
+				/* Pages are assigned at runtime. */
+				next_frag.frag_page++;
 				next_frag.offset = 0;
 				if (prev)
-					prev->last_in_page = true;
+					prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE);
 			}
 			*frag = next_frag;
 
@@ -530,25 +546,68 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
 	}
 
 	if (prev)
-		prev->last_in_page = true;
+		prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE);
+}
+
+static void mlx5e_init_xsk_buffs(struct mlx5e_rq *rq)
+{
+	int i;
+
+	/* Assumptions used by XSK batched allocator. */
+	WARN_ON(rq->wqe.info.num_frags != 1);
+	WARN_ON(rq->wqe.info.log_num_frags != 0);
+	WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE);
+
+	/* Considering the above assumptions a fragment maps to a single
+	 * xsk_buff.
+	 */
+	for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
+		rq->wqe.frags[i].xskp = &rq->wqe.alloc_units->xsk_buffs[i];
+
+		/* Skip first release due to deferred release as WQES are
+		 * not allocated yet.
+		 */
+		rq->wqe.frags[i].flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
+	}
 }
 
-static int mlx5e_init_au_list(struct mlx5e_rq *rq, int wq_sz, int node)
+static int mlx5e_init_wqe_alloc_info(struct mlx5e_rq *rq, int node)
 {
+	int wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
 	int len = wq_sz << rq->wqe.info.log_num_frags;
+	struct mlx5e_wqe_frag_info *frags;
+	union mlx5e_alloc_units *aus;
+	int aus_sz;
 
-	rq->wqe.alloc_units = kvzalloc_node(array_size(len, sizeof(*rq->wqe.alloc_units)),
-					    GFP_KERNEL, node);
-	if (!rq->wqe.alloc_units)
+	if (rq->xsk_pool)
+		aus_sz = sizeof(*aus->xsk_buffs);
+	else
+		aus_sz = sizeof(*aus->frag_pages);
+
+	aus = kvzalloc_node(array_size(len, aus_sz), GFP_KERNEL, node);
+	if (!aus)
 		return -ENOMEM;
 
-	mlx5e_init_frags_partition(rq);
+	frags = kvzalloc_node(array_size(len, sizeof(*frags)), GFP_KERNEL, node);
+	if (!frags) {
+		kvfree(aus);
+		return -ENOMEM;
+	}
+
+	rq->wqe.alloc_units = aus;
+	rq->wqe.frags = frags;
+
+	if (rq->xsk_pool)
+		mlx5e_init_xsk_buffs(rq);
+	else
+		mlx5e_init_frags_partition(rq);
 
 	return 0;
 }
 
-static void mlx5e_free_au_list(struct mlx5e_rq *rq)
+static void mlx5e_free_wqe_alloc_info(struct mlx5e_rq *rq)
 {
+	kvfree(rq->wqe.frags);
 	kvfree(rq->wqe.alloc_units);
 }
 
@@ -693,7 +752,6 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 			  struct mlx5e_rq_param *rqp,
 			  int node, struct mlx5e_rq *rq)
 {
-	struct page_pool_params pp_params = { 0 };
 	struct mlx5_core_dev *mdev = rq->mdev;
 	void *rqc = rqp->rqc;
 	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
@@ -745,6 +803,9 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 		pool_size = rq->mpwqe.pages_per_wqe <<
 			mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk);
 
+		if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk) && params->xdp_prog)
+			pool_size *= 2; /* additional page per packet for the linear part */
+
 		rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
 		rq->mpwqe.num_strides =
 			BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
@@ -778,18 +839,9 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 		rq->wqe.info = rqp->frags_info;
 		rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride;
 
-		rq->wqe.frags =
-			kvzalloc_node(array_size(sizeof(*rq->wqe.frags),
-					(wq_sz << rq->wqe.info.log_num_frags)),
-				      GFP_KERNEL, node);
-		if (!rq->wqe.frags) {
-			err = -ENOMEM;
-			goto err_rq_wq_destroy;
-		}
-
-		err = mlx5e_init_au_list(rq, wq_sz, node);
+		err = mlx5e_init_wqe_alloc_info(rq, node);
 		if (err)
-			goto err_rq_frags;
+			goto err_rq_wq_destroy;
 	}
 
 	if (xsk) {
@@ -798,12 +850,16 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 		xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq);
 	} else {
 		/* Create a page_pool and register it with rxq */
+		struct page_pool_params pp_params = { 0 };
+
 		pp_params.order     = 0;
-		pp_params.flags     = 0; /* No-internal DMA mapping in page_pool */
+		pp_params.flags     = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | PP_FLAG_PAGE_FRAG;
 		pp_params.pool_size = pool_size;
 		pp_params.nid       = node;
 		pp_params.dev       = rq->pdev;
+		pp_params.napi      = rq->cq.napi;
 		pp_params.dma_dir   = rq->buff.map_dir;
+		pp_params.max_len   = PAGE_SIZE;
 
 		/* page_pool can be used even when there is no rq->xdp_prog,
 		 * given page_pool does not handle DMA mapping there is no
@@ -869,9 +925,6 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 		rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 	}
 
-	rq->page_cache.head = 0;
-	rq->page_cache.tail = 0;
-
 	return 0;
 
 err_destroy_page_pool:
@@ -888,9 +941,7 @@ err_rq_drop_page:
 		mlx5e_free_mpwqe_rq_drop_page(rq);
 		break;
 	default: /* MLX5_WQ_TYPE_CYCLIC */
-		mlx5e_free_au_list(rq);
-err_rq_frags:
-		kvfree(rq->wqe.frags);
+		mlx5e_free_wqe_alloc_info(rq);
 	}
 err_rq_wq_destroy:
 	mlx5_wq_destroy(&rq->wq_ctrl);
@@ -904,7 +955,6 @@ err_rq_xdp_prog:
 static void mlx5e_free_rq(struct mlx5e_rq *rq)
 {
 	struct bpf_prog *old_prog;
-	int i;
 
 	if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) {
 		old_prog = rcu_dereference_protected(rq->xdp_prog,
@@ -921,17 +971,7 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
 		mlx5e_rq_free_shampo(rq);
 		break;
 	default: /* MLX5_WQ_TYPE_CYCLIC */
-		kvfree(rq->wqe.frags);
-		mlx5e_free_au_list(rq);
-	}
-
-	for (i = rq->page_cache.head; i != rq->page_cache.tail;
-	     i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
-		/* With AF_XDP, page_cache is not used, so this loop is not
-		 * entered, and it's safe to call mlx5e_page_release_dynamic
-		 * directly.
-		 */
-		mlx5e_page_release_dynamic(rq, rq->page_cache.page_cache[i], false);
+		mlx5e_free_wqe_alloc_info(rq);
 	}
 
 	xdp_rxq_info_unreg(&rq->xdp_rxq);
@@ -1094,7 +1134,7 @@ int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
 	return -ETIMEDOUT;
 }
 
-void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq)
+void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq)
 {
 	struct mlx5_wq_ll *wq;
 	u16 head;
@@ -1106,8 +1146,12 @@ void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq)
 	wq = &rq->mpwqe.wq;
 	head = wq->head;
 
-	/* Outstanding UMR WQEs (in progress) start at wq->head */
-	for (i = 0; i < rq->mpwqe.umr_in_progress; i++) {
+	/* Release WQEs that are in missing state: they have been
+	 * popped from the list after completion but were not freed
+	 * due to deferred release.
+	 * Also free the linked-list reserved entry, hence the "+ 1".
+	 */
+	for (i = 0; i < mlx5_wq_ll_missing(wq) + 1; i++) {
 		rq->dealloc_wqe(rq, head);
 		head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
 	}
@@ -1134,7 +1178,7 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 	if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
 		struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
 
-		mlx5e_free_rx_in_progress_descs(rq);
+		mlx5e_free_rx_missing_descs(rq);
 
 		while (!mlx5_wq_ll_is_empty(wq)) {
 			struct mlx5e_rx_wqe_ll *wqe;
@@ -1152,12 +1196,21 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 						0, true);
 	} else {
 		struct mlx5_wq_cyc *wq = &rq->wqe.wq;
+		u16 missing = mlx5_wq_cyc_missing(wq);
+		u16 head = mlx5_wq_cyc_get_head(wq);
 
 		while (!mlx5_wq_cyc_is_empty(wq)) {
 			wqe_ix = mlx5_wq_cyc_get_tail(wq);
 			rq->dealloc_wqe(rq, wqe_ix);
 			mlx5_wq_cyc_pop(wq);
 		}
+		/* Missing slots might also contain unreleased pages due to
+		 * deferred release.
+		 */
+		while (missing--) {
+			wqe_ix = mlx5_wq_cyc_ctr2ix(wq, head++);
+			rq->dealloc_wqe(rq, wqe_ix);
+		}
 	}
 
 }
@@ -1188,7 +1241,7 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
 		__set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state);
 
 	if (params->rx_dim_enabled)
-		__set_bit(MLX5E_RQ_STATE_AM, &rq->state);
+		__set_bit(MLX5E_RQ_STATE_DIM, &rq->state);
 
 	/* We disable csum_complete when XDP is enabled since
 	 * XDP programs might manipulate packets which will render
@@ -1251,17 +1304,19 @@ static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa)
 {
 	struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
 	int wq_sz        = mlx5_wq_cyc_get_size(&sq->wq);
-	int dsegs_per_wq = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
+	int entries = wq_sz * MLX5_SEND_WQEBB_NUM_DS * 2; /* upper bound for maximum num of
+							   * entries of all xmit_modes.
+							   */
 	size_t size;
 
-	size = array_size(sizeof(*xdpi_fifo->xi), dsegs_per_wq);
+	size = array_size(sizeof(*xdpi_fifo->xi), entries);
 	xdpi_fifo->xi = kvzalloc_node(size, GFP_KERNEL, numa);
 	if (!xdpi_fifo->xi)
 		return -ENOMEM;
 
 	xdpi_fifo->pc   = &sq->xdpi_fifo_pc;
 	xdpi_fifo->cc   = &sq->xdpi_fifo_cc;
-	xdpi_fifo->mask = dsegs_per_wq - 1;
+	xdpi_fifo->mask = entries - 1;
 
 	return 0;
 }
@@ -1664,7 +1719,7 @@ int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
 		mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
 
 	if (params->tx_dim_enabled)
-		sq->state |= BIT(MLX5E_SQ_STATE_AM);
+		sq->state |= BIT(MLX5E_SQ_STATE_DIM);
 
 	return 0;
 
@@ -1811,11 +1866,7 @@ int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
 	csp.min_inline_mode = sq->min_inline_mode;
 	set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
 
-	/* Don't enable multi buffer on XDP_REDIRECT SQ, as it's not yet
-	 * supported by upstream, and there is no defined trigger to allow
-	 * transmitting redirected multi-buffer frames.
-	 */
-	if (param->is_xdp_mb && !is_redirect)
+	if (param->is_xdp_mb)
 		set_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state);
 
 	err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
@@ -1839,7 +1890,6 @@ int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
 			struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(&sq->wq, i);
 			struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
 			struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
-			struct mlx5_wqe_data_seg *dseg;
 
 			sq->db.wqe_info[i] = (struct mlx5e_xdp_wqe_info) {
 				.num_wqebbs = 1,
@@ -1848,9 +1898,6 @@ int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
 
 			cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
 			eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
-
-			dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1);
-			dseg->lkey = sq->mkey_be;
 		}
 	}
 
@@ -4017,9 +4064,9 @@ void mlx5e_set_xdp_feature(struct net_device *netdev)
 
 	val = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
 	      NETDEV_XDP_ACT_XSK_ZEROCOPY |
-	      NETDEV_XDP_ACT_NDO_XMIT;
-	if (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC)
-		val |= NETDEV_XDP_ACT_RX_SG;
+	      NETDEV_XDP_ACT_RX_SG |
+	      NETDEV_XDP_ACT_NDO_XMIT |
+	      NETDEV_XDP_ACT_NDO_XMIT_SG;
 	xdp_set_features_flag(netdev, val);
 }
 
@@ -4213,19 +4260,24 @@ static bool mlx5e_params_validate_xdp(struct net_device *netdev,
 	/* No XSK params: AF_XDP can't be enabled yet at the point of setting
 	 * the XDP program.
 	 */
-	is_linear = mlx5e_rx_is_linear_skb(mdev, params, NULL);
-
-	if (!is_linear && params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) {
-		netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n",
-			    params->sw_mtu,
-			    mlx5e_xdp_max_mtu(params, NULL));
-		return false;
-	}
-	if (!is_linear && !params->xdp_prog->aux->xdp_has_frags) {
-		netdev_warn(netdev, "MTU(%d) > %d, too big for an XDP program not aware of multi buffer\n",
-			    params->sw_mtu,
-			    mlx5e_xdp_max_mtu(params, NULL));
-		return false;
+	is_linear = params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC ?
+		mlx5e_rx_is_linear_skb(mdev, params, NULL) :
+		mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL);
+
+	if (!is_linear) {
+		if (!params->xdp_prog->aux->xdp_has_frags) {
+			netdev_warn(netdev, "MTU(%d) > %d, too big for an XDP program not aware of multi buffer\n",
+				    params->sw_mtu,
+				    mlx5e_xdp_max_mtu(params, NULL));
+			return false;
+		}
+		if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
+		    !mlx5e_verify_params_rx_mpwqe_strides(mdev, params, NULL)) {
+			netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n",
+				    params->sw_mtu,
+				    mlx5e_xdp_max_mtu(params, NULL));
+			return false;
+		}
 	}
 
 	return true;
@@ -4717,20 +4769,15 @@ static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue)
 	queue_work(priv->wq, &priv->tx_timeout_work);
 }
 
-static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog)
+static int mlx5e_xdp_allowed(struct net_device *netdev, struct mlx5_core_dev *mdev,
+			     struct mlx5e_params *params)
 {
-	struct net_device *netdev = priv->netdev;
-	struct mlx5e_params new_params;
-
-	if (priv->channels.params.packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
+	if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
 		netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n");
 		return -EINVAL;
 	}
 
-	new_params = priv->channels.params;
-	new_params.xdp_prog = prog;
-
-	if (!mlx5e_params_validate_xdp(netdev, priv->mdev, &new_params))
+	if (!mlx5e_params_validate_xdp(netdev, mdev, params))
 		return -EINVAL;
 
 	return 0;
@@ -4757,8 +4804,11 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
 
 	mutex_lock(&priv->state_lock);
 
+	new_params = priv->channels.params;
+	new_params.xdp_prog = prog;
+
 	if (prog) {
-		err = mlx5e_xdp_allowed(priv, prog);
+		err = mlx5e_xdp_allowed(netdev, priv->mdev, &new_params);
 		if (err)
 			goto unlock;
 	}
@@ -4766,22 +4816,6 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
 	/* no need for full reset when exchanging programs */
 	reset = (!priv->channels.params.xdp_prog || !prog);
 
-	new_params = priv->channels.params;
-	new_params.xdp_prog = prog;
-
-	/* XDP affects striding RQ parameters. Block XDP if striding RQ won't be
-	 * supported with the new parameters: if PAGE_SIZE is bigger than
-	 * MLX5_MPWQE_LOG_STRIDE_SZ_MAX, striding RQ can't be used, even though
-	 * the MTU is small enough for the linear mode, because XDP uses strides
-	 * of PAGE_SIZE on regular RQs.
-	 */
-	if (reset && MLX5E_GET_PFLAG(&new_params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
-		/* Checking for regular RQs here; XSK RQs were checked on XSK bind. */
-		err = mlx5e_mpwrq_validate_regular(priv->mdev, &new_params);
-		if (err)
-			goto unlock;
-	}
-
 	old_prog = priv->channels.params.xdp_prog;
 
 	err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset);
@@ -5076,6 +5110,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 
 	netdev->vlan_features    |= NETIF_F_SG;
 	netdev->vlan_features    |= NETIF_F_HW_CSUM;
+	netdev->vlan_features    |= NETIF_F_HW_MACSEC;
 	netdev->vlan_features    |= NETIF_F_GRO;
 	netdev->vlan_features    |= NETIF_F_TSO;
 	netdev->vlan_features    |= NETIF_F_TSO6;
@@ -5270,6 +5305,7 @@ static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
 	mlx5e_health_destroy_reporters(priv);
 	mlx5e_ktls_cleanup(priv);
 	mlx5e_fs_cleanup(priv->fs);
+	priv->fs = NULL;
 }
 
 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
@@ -5725,8 +5761,8 @@ int mlx5e_attach_netdev(struct mlx5e_priv *priv)
 
 	/* Validate the max_wqe_size_sq capability. */
 	if (WARN_ON_ONCE(mlx5e_get_max_sq_wqebbs(priv->mdev) < MLX5E_MAX_TX_WQEBBS)) {
-		mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %lu\n",
-			       mlx5e_get_max_sq_wqebbs(priv->mdev), MLX5E_MAX_TX_WQEBBS);
+		mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %u\n",
+			       mlx5e_get_max_sq_wqebbs(priv->mdev), (unsigned int)MLX5E_MAX_TX_WQEBBS);
 		return -EIO;
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 8ff654b4e9e1..1fc386eccaf8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -53,6 +53,7 @@
 #include "lib/vxlan.h"
 #define CREATE_TRACE_POINTS
 #include "diag/en_rep_tracepoint.h"
+#include "diag/reporter_vnic.h"
 #include "en_accel/ipsec.h"
 #include "en/tc/int_port.h"
 #include "en/ptp.h"
@@ -828,6 +829,7 @@ static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev,
 static void mlx5e_cleanup_rep(struct mlx5e_priv *priv)
 {
 	mlx5e_fs_cleanup(priv->fs);
+	priv->fs = NULL;
 }
 
 static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv)
@@ -994,6 +996,7 @@ err_close_drop_rq:
 	priv->rx_res = NULL;
 err_free_fs:
 	mlx5e_fs_cleanup(priv->fs);
+	priv->fs = NULL;
 	return err;
 }
 
@@ -1294,6 +1297,50 @@ static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv *priv)
 	return ARRAY_SIZE(mlx5e_ul_rep_stats_grps);
 }
 
+static int
+mlx5e_rep_vnic_reporter_diagnose(struct devlink_health_reporter *reporter,
+				 struct devlink_fmsg *fmsg,
+				 struct netlink_ext_ack *extack)
+{
+	struct mlx5e_rep_priv *rpriv = devlink_health_reporter_priv(reporter);
+	struct mlx5_eswitch_rep *rep = rpriv->rep;
+
+	return mlx5_reporter_vnic_diagnose_counters(rep->esw->dev, fmsg,
+						    rep->vport, true);
+}
+
+static const struct devlink_health_reporter_ops mlx5_rep_vnic_reporter_ops = {
+	.name = "vnic",
+	.diagnose = mlx5e_rep_vnic_reporter_diagnose,
+};
+
+static void mlx5e_rep_vnic_reporter_create(struct mlx5e_priv *priv,
+					   struct devlink_port *dl_port)
+{
+	struct mlx5e_rep_priv *rpriv = priv->ppriv;
+	struct devlink_health_reporter *reporter;
+
+	reporter = devl_port_health_reporter_create(dl_port,
+						    &mlx5_rep_vnic_reporter_ops,
+						    0, rpriv);
+	if (IS_ERR(reporter)) {
+		mlx5_core_err(priv->mdev,
+			      "Failed to create representor vnic reporter, err = %ld\n",
+			      PTR_ERR(reporter));
+		return;
+	}
+
+	rpriv->rep_vnic_reporter = reporter;
+}
+
+static void mlx5e_rep_vnic_reporter_destroy(struct mlx5e_priv *priv)
+{
+	struct mlx5e_rep_priv *rpriv = priv->ppriv;
+
+	if (!IS_ERR_OR_NULL(rpriv->rep_vnic_reporter))
+		devl_health_reporter_destroy(rpriv->rep_vnic_reporter);
+}
+
 static const struct mlx5e_profile mlx5e_rep_profile = {
 	.init			= mlx5e_init_rep,
 	.cleanup		= mlx5e_cleanup_rep,
@@ -1394,8 +1441,10 @@ mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
 
 	dl_port = mlx5_esw_offloads_devlink_port(dev->priv.eswitch,
 						 rpriv->rep->vport);
-	if (dl_port)
+	if (dl_port) {
 		SET_NETDEV_DEVLINK_PORT(netdev, dl_port);
+		mlx5e_rep_vnic_reporter_create(priv, dl_port);
+	}
 
 	err = register_netdev(netdev);
 	if (err) {
@@ -1408,8 +1457,8 @@ mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
 	return 0;
 
 err_detach_netdev:
+	mlx5e_rep_vnic_reporter_destroy(priv);
 	mlx5e_detach_netdev(netdev_priv(netdev));
-
 err_cleanup_profile:
 	priv->profile->cleanup(priv);
 
@@ -1458,6 +1507,7 @@ mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep)
 	}
 
 	unregister_netdev(netdev);
+	mlx5e_rep_vnic_reporter_destroy(priv);
 	mlx5e_detach_netdev(priv);
 	priv->profile->cleanup(priv);
 	mlx5e_destroy_netdev(priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
index dcfad0bf0f45..80b7f5079a5a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
@@ -118,6 +118,7 @@ struct mlx5e_rep_priv {
 	struct rtnl_link_stats64 prev_vf_vport_stats;
 	struct mlx5_flow_handle *send_to_vport_meta_rule;
 	struct rhashtable tc_ht;
+	struct devlink_health_reporter *rep_vnic_reporter;
 };
 
 static inline
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 3f7b63d6616b..69634829558e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -271,98 +271,35 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
 	return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem);
 }
 
-static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, struct page *page)
-{
-	struct mlx5e_page_cache *cache = &rq->page_cache;
-	u32 tail_next = (cache->tail + 1) & (MLX5E_CACHE_SIZE - 1);
-	struct mlx5e_rq_stats *stats = rq->stats;
-
-	if (tail_next == cache->head) {
-		stats->cache_full++;
-		return false;
-	}
-
-	if (!dev_page_is_reusable(page)) {
-		stats->cache_waive++;
-		return false;
-	}
-
-	cache->page_cache[cache->tail] = page;
-	cache->tail = tail_next;
-	return true;
-}
-
-static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, union mlx5e_alloc_unit *au)
-{
-	struct mlx5e_page_cache *cache = &rq->page_cache;
-	struct mlx5e_rq_stats *stats = rq->stats;
-	dma_addr_t addr;
-
-	if (unlikely(cache->head == cache->tail)) {
-		stats->cache_empty++;
-		return false;
-	}
-
-	if (page_ref_count(cache->page_cache[cache->head]) != 1) {
-		stats->cache_busy++;
-		return false;
-	}
-
-	au->page = cache->page_cache[cache->head];
-	cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);
-	stats->cache_reuse++;
+#define MLX5E_PAGECNT_BIAS_MAX (PAGE_SIZE / 64)
 
-	addr = page_pool_get_dma_addr(au->page);
-	/* Non-XSK always uses PAGE_SIZE. */
-	dma_sync_single_for_device(rq->pdev, addr, PAGE_SIZE, rq->buff.map_dir);
-	return true;
-}
-
-static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, union mlx5e_alloc_unit *au)
+static int mlx5e_page_alloc_fragmented(struct mlx5e_rq *rq,
+				       struct mlx5e_frag_page *frag_page)
 {
-	dma_addr_t addr;
+	struct page *page;
 
-	if (mlx5e_rx_cache_get(rq, au))
-		return 0;
-
-	au->page = page_pool_dev_alloc_pages(rq->page_pool);
-	if (unlikely(!au->page))
+	page = page_pool_dev_alloc_pages(rq->page_pool);
+	if (unlikely(!page))
 		return -ENOMEM;
 
-	/* Non-XSK always uses PAGE_SIZE. */
-	addr = dma_map_page(rq->pdev, au->page, 0, PAGE_SIZE, rq->buff.map_dir);
-	if (unlikely(dma_mapping_error(rq->pdev, addr))) {
-		page_pool_recycle_direct(rq->page_pool, au->page);
-		au->page = NULL;
-		return -ENOMEM;
-	}
-	page_pool_set_dma_addr(au->page, addr);
+	page_pool_fragment_page(page, MLX5E_PAGECNT_BIAS_MAX);
+
+	*frag_page = (struct mlx5e_frag_page) {
+		.page	= page,
+		.frags	= 0,
+	};
 
 	return 0;
 }
 
-void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct page *page)
+static void mlx5e_page_release_fragmented(struct mlx5e_rq *rq,
+					  struct mlx5e_frag_page *frag_page)
 {
-	dma_addr_t dma_addr = page_pool_get_dma_addr(page);
+	u16 drain_count = MLX5E_PAGECNT_BIAS_MAX - frag_page->frags;
+	struct page *page = frag_page->page;
 
-	dma_unmap_page_attrs(rq->pdev, dma_addr, PAGE_SIZE, rq->buff.map_dir,
-			     DMA_ATTR_SKIP_CPU_SYNC);
-	page_pool_set_dma_addr(page, 0);
-}
-
-void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, struct page *page, bool recycle)
-{
-	if (likely(recycle)) {
-		if (mlx5e_rx_cache_put(rq, page))
-			return;
-
-		mlx5e_page_dma_unmap(rq, page);
-		page_pool_recycle_direct(rq->page_pool, page);
-	} else {
-		mlx5e_page_dma_unmap(rq, page);
-		page_pool_release_page(rq->page_pool, page);
-		put_page(page);
-	}
+	if (page_pool_defrag_page(page, drain_count) == 0)
+		page_pool_put_defragged_page(rq->page_pool, page, -1, true);
 }
 
 static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
@@ -371,22 +308,31 @@ static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
 	int err = 0;
 
 	if (!frag->offset)
-		/* On first frag (offset == 0), replenish page (alloc_unit actually).
-		 * Other frags that point to the same alloc_unit (with a different
+		/* On first frag (offset == 0), replenish page.
+		 * Other frags that point to the same page (with a different
 		 * offset) should just use the new one without replenishing again
 		 * by themselves.
 		 */
-		err = mlx5e_page_alloc_pool(rq, frag->au);
+		err = mlx5e_page_alloc_fragmented(rq, frag->frag_page);
 
 	return err;
 }
 
+static bool mlx5e_frag_can_release(struct mlx5e_wqe_frag_info *frag)
+{
+#define CAN_RELEASE_MASK \
+	(BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE) | BIT(MLX5E_WQE_FRAG_SKIP_RELEASE))
+
+#define CAN_RELEASE_VALUE BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE)
+
+	return (frag->flags & CAN_RELEASE_MASK) == CAN_RELEASE_VALUE;
+}
+
 static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,
-				     struct mlx5e_wqe_frag_info *frag,
-				     bool recycle)
+				     struct mlx5e_wqe_frag_info *frag)
 {
-	if (frag->last_in_page)
-		mlx5e_page_release_dynamic(rq, frag->au->page, recycle);
+	if (mlx5e_frag_can_release(frag))
+		mlx5e_page_release_fragmented(rq, frag->frag_page);
 }
 
 static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)
@@ -409,8 +355,10 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe,
 		if (unlikely(err))
 			goto free_frags;
 
+		frag->flags &= ~BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
+
 		headroom = i == 0 ? rq->buff.headroom : 0;
-		addr = page_pool_get_dma_addr(frag->au->page);
+		addr = page_pool_get_dma_addr(frag->frag_page->page);
 		wqe->data[i].addr = cpu_to_be64(addr + frag->offset + headroom);
 	}
 
@@ -418,35 +366,66 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe,
 
 free_frags:
 	while (--i >= 0)
-		mlx5e_put_rx_frag(rq, --frag, true);
+		mlx5e_put_rx_frag(rq, --frag);
 
 	return err;
 }
 
 static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
-				     struct mlx5e_wqe_frag_info *wi,
-				     bool recycle)
+				     struct mlx5e_wqe_frag_info *wi)
 {
 	int i;
 
-	if (rq->xsk_pool) {
-		/* The `recycle` parameter is ignored, and the page is always
-		 * put into the Reuse Ring, because there is no way to return
-		 * the page to the userspace when the interface goes down.
-		 */
-		xsk_buff_free(wi->au->xsk);
-		return;
-	}
-
 	for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)
-		mlx5e_put_rx_frag(rq, wi, recycle);
+		mlx5e_put_rx_frag(rq, wi);
+}
+
+static void mlx5e_xsk_free_rx_wqe(struct mlx5e_wqe_frag_info *wi)
+{
+	if (!(wi->flags & BIT(MLX5E_WQE_FRAG_SKIP_RELEASE)))
+		xsk_buff_free(*wi->xskp);
 }
 
 static void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
 {
 	struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);
 
-	mlx5e_free_rx_wqe(rq, wi, false);
+	if (rq->xsk_pool)
+		mlx5e_xsk_free_rx_wqe(wi);
+	else
+		mlx5e_free_rx_wqe(rq, wi);
+}
+
+static void mlx5e_xsk_free_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
+{
+	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
+	int i;
+
+	for (i = 0; i < wqe_bulk; i++) {
+		int j = mlx5_wq_cyc_ctr2ix(wq, ix + i);
+		struct mlx5e_wqe_frag_info *wi;
+
+		wi = get_frag(rq, j);
+		/* The page is always put into the Reuse Ring, because there
+		 * is no way to return the page to the userspace when the
+		 * interface goes down.
+		 */
+		mlx5e_xsk_free_rx_wqe(wi);
+	}
+}
+
+static void mlx5e_free_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
+{
+	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
+	int i;
+
+	for (i = 0; i < wqe_bulk; i++) {
+		int j = mlx5_wq_cyc_ctr2ix(wq, ix + i);
+		struct mlx5e_wqe_frag_info *wi;
+
+		wi = get_frag(rq, j);
+		mlx5e_free_rx_wqe(rq, wi);
+	}
 }
 
 static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
@@ -467,18 +446,71 @@ static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
 	return i;
 }
 
+static int mlx5e_refill_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
+{
+	int remaining = wqe_bulk;
+	int i = 0;
+
+	/* The WQE bulk is split into smaller bulks that are sized
+	 * according to the page pool cache refill size to avoid overflowing
+	 * the page pool cache due to too many page releases at once.
+	 */
+	do {
+		int refill = min_t(u16, rq->wqe.info.refill_unit, remaining);
+		int alloc_count;
+
+		mlx5e_free_rx_wqes(rq, ix + i, refill);
+		alloc_count = mlx5e_alloc_rx_wqes(rq, ix + i, refill);
+		i += alloc_count;
+		if (unlikely(alloc_count != refill))
+			break;
+
+		remaining -= refill;
+	} while (remaining);
+
+	return i;
+}
+
+static void
+mlx5e_add_skb_shared_info_frag(struct mlx5e_rq *rq, struct skb_shared_info *sinfo,
+			       struct xdp_buff *xdp, struct mlx5e_frag_page *frag_page,
+			       u32 frag_offset, u32 len)
+{
+	skb_frag_t *frag;
+
+	dma_addr_t addr = page_pool_get_dma_addr(frag_page->page);
+
+	dma_sync_single_for_cpu(rq->pdev, addr + frag_offset, len, rq->buff.map_dir);
+	if (!xdp_buff_has_frags(xdp)) {
+		/* Init on the first fragment to avoid cold cache access
+		 * when possible.
+		 */
+		sinfo->nr_frags = 0;
+		sinfo->xdp_frags_size = 0;
+		xdp_buff_set_frags_flag(xdp);
+	}
+
+	frag = &sinfo->frags[sinfo->nr_frags++];
+	__skb_frag_set_page(frag, frag_page->page);
+	skb_frag_off_set(frag, frag_offset);
+	skb_frag_size_set(frag, len);
+
+	if (page_is_pfmemalloc(frag_page->page))
+		xdp_buff_set_frag_pfmemalloc(xdp);
+	sinfo->xdp_frags_size += len;
+}
+
 static inline void
 mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb,
-		   union mlx5e_alloc_unit *au, u32 frag_offset, u32 len,
+		   struct page *page, u32 frag_offset, u32 len,
 		   unsigned int truesize)
 {
-	dma_addr_t addr = page_pool_get_dma_addr(au->page);
+	dma_addr_t addr = page_pool_get_dma_addr(page);
 
 	dma_sync_single_for_cpu(rq->pdev, addr + frag_offset, len,
 				rq->buff.map_dir);
-	page_ref_inc(au->page);
 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
-			au->page, frag_offset, len, truesize);
+			page, frag_offset, len, truesize);
 }
 
 static inline void
@@ -496,30 +528,36 @@ mlx5e_copy_skb_header(struct mlx5e_rq *rq, struct sk_buff *skb,
 }
 
 static void
-mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle)
+mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
 {
-	union mlx5e_alloc_unit *alloc_units = wi->alloc_units;
 	bool no_xdp_xmit;
 	int i;
 
 	/* A common case for AF_XDP. */
-	if (bitmap_full(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe))
+	if (bitmap_full(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe))
 		return;
 
-	no_xdp_xmit = bitmap_empty(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe);
+	no_xdp_xmit = bitmap_empty(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
 
 	if (rq->xsk_pool) {
-		/* The `recycle` parameter is ignored, and the page is always
-		 * put into the Reuse Ring, because there is no way to return
-		 * the page to the userspace when the interface goes down.
+		struct xdp_buff **xsk_buffs = wi->alloc_units.xsk_buffs;
+
+		/* The page is always put into the Reuse Ring, because there
+		 * is no way to return the page to userspace when the interface
+		 * goes down.
 		 */
 		for (i = 0; i < rq->mpwqe.pages_per_wqe; i++)
-			if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
-				xsk_buff_free(alloc_units[i].xsk);
+			if (no_xdp_xmit || !test_bit(i, wi->skip_release_bitmap))
+				xsk_buff_free(xsk_buffs[i]);
 	} else {
-		for (i = 0; i < rq->mpwqe.pages_per_wqe; i++)
-			if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
-				mlx5e_page_release_dynamic(rq, alloc_units[i].page, recycle);
+		for (i = 0; i < rq->mpwqe.pages_per_wqe; i++) {
+			if (no_xdp_xmit || !test_bit(i, wi->skip_release_bitmap)) {
+				struct mlx5e_frag_page *frag_page;
+
+				frag_page = &wi->alloc_units.frag_pages[i];
+				mlx5e_page_release_fragmented(rq, frag_page);
+			}
+		}
 	}
 }
 
@@ -583,7 +621,8 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
 	struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
 	u16 entries, pi, header_offset, err, wqe_bbs, new_entries;
 	u32 lkey = rq->mdev->mlx5e_res.hw_objs.mkey;
-	struct page *page = shampo->last_page;
+	u16 page_index = shampo->curr_page_index;
+	struct mlx5e_frag_page *frag_page;
 	u64 addr = shampo->last_addr;
 	struct mlx5e_dma_info *dma_info;
 	struct mlx5e_umr_wqe *umr_wqe;
@@ -597,6 +636,8 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
 	umr_wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
 	build_klm_umr(sq, umr_wqe, shampo->key, index, entries, wqe_bbs);
 
+	frag_page = &shampo->pages[page_index];
+
 	for (i = 0; i < entries; i++, index++) {
 		dma_info = &shampo->info[index];
 		if (i >= klm_entries || (index < shampo->pi && shampo->pi - index <
@@ -605,16 +646,20 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
 		header_offset = (index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) <<
 			MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE;
 		if (!(header_offset & (PAGE_SIZE - 1))) {
-			union mlx5e_alloc_unit au;
+			page_index = (page_index + 1) & (shampo->hd_per_wq - 1);
+			frag_page = &shampo->pages[page_index];
 
-			err = mlx5e_page_alloc_pool(rq, &au);
+			err = mlx5e_page_alloc_fragmented(rq, frag_page);
 			if (unlikely(err))
 				goto err_unmap;
-			page = dma_info->page = au.page;
-			addr = dma_info->addr = page_pool_get_dma_addr(au.page);
+
+			addr = page_pool_get_dma_addr(frag_page->page);
+
+			dma_info->addr = addr;
+			dma_info->frag_page = frag_page;
 		} else {
 			dma_info->addr = addr + header_offset;
-			dma_info->page = page;
+			dma_info->frag_page = frag_page;
 		}
 
 update_klm:
@@ -632,7 +677,7 @@ update_klm:
 	};
 
 	shampo->pi = (shampo->pi + new_entries) & (shampo->hd_per_wq - 1);
-	shampo->last_page = page;
+	shampo->curr_page_index = page_index;
 	shampo->last_addr = addr;
 	sq->pc += wqe_bbs;
 	sq->doorbell_cseg = &umr_wqe->ctrl;
@@ -644,7 +689,7 @@ err_unmap:
 		dma_info = &shampo->info[--index];
 		if (!(i & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1))) {
 			dma_info->addr = ALIGN_DOWN(dma_info->addr, PAGE_SIZE);
-			mlx5e_page_release_dynamic(rq, dma_info->page, true);
+			mlx5e_page_release_fragmented(rq, dma_info->frag_page);
 		}
 	}
 	rq->stats->buff_alloc_err++;
@@ -693,8 +738,8 @@ static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq)
 static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 {
 	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix);
-	union mlx5e_alloc_unit *au = &wi->alloc_units[0];
 	struct mlx5e_icosq *sq = rq->icosq;
+	struct mlx5e_frag_page *frag_page;
 	struct mlx5_wq_cyc *wq = &sq->wq;
 	struct mlx5e_umr_wqe *umr_wqe;
 	u32 offset; /* 17-bit value with MTT. */
@@ -712,13 +757,15 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 	umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
 	memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe));
 
-	for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, au++) {
+	frag_page = &wi->alloc_units.frag_pages[0];
+
+	for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, frag_page++) {
 		dma_addr_t addr;
 
-		err = mlx5e_page_alloc_pool(rq, au);
+		err = mlx5e_page_alloc_fragmented(rq, frag_page);
 		if (unlikely(err))
 			goto err_unmap;
-		addr = page_pool_get_dma_addr(au->page);
+		addr = page_pool_get_dma_addr(frag_page->page);
 		umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {
 			.ptag = cpu_to_be64(addr | MLX5_EN_WR),
 		};
@@ -735,7 +782,7 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 		       sizeof(*umr_wqe->inline_mtts) * pad);
 	}
 
-	bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe);
+	bitmap_zero(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
 	wi->consumed_strides = 0;
 
 	umr_wqe->ctrl.opmod_idx_opcode =
@@ -759,8 +806,8 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 
 err_unmap:
 	while (--i >= 0) {
-		au--;
-		mlx5e_page_release_dynamic(rq, au->page, true);
+		frag_page--;
+		mlx5e_page_release_fragmented(rq, frag_page);
 	}
 
 err:
@@ -778,8 +825,8 @@ err:
 void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close)
 {
 	struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
+	struct mlx5e_frag_page *deleted_page = NULL;
 	int hd_per_wq = shampo->hd_per_wq;
-	struct page *deleted_page = NULL;
 	struct mlx5e_dma_info *hd_info;
 	int i, index = start;
 
@@ -792,10 +839,12 @@ void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close
 
 		hd_info = &shampo->info[index];
 		hd_info->addr = ALIGN_DOWN(hd_info->addr, PAGE_SIZE);
-		if (hd_info->page != deleted_page) {
-			deleted_page = hd_info->page;
-			mlx5e_page_release_dynamic(rq, hd_info->page, false);
+		if (hd_info->frag_page && hd_info->frag_page != deleted_page) {
+			deleted_page = hd_info->frag_page;
+			mlx5e_page_release_fragmented(rq, hd_info->frag_page);
 		}
+
+		hd_info->frag_page = NULL;
 	}
 
 	if (start + len > hd_per_wq) {
@@ -810,8 +859,13 @@ void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close
 static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 {
 	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix);
-	/* Don't recycle, this function is called on rq/netdev close */
-	mlx5e_free_rx_mpwqe(rq, wi, false);
+	/* This function is called on rq/netdev close. */
+	mlx5e_free_rx_mpwqe(rq, wi);
+
+	/* Avoid a second release of the wqe pages: dealloc is called also
+	 * for missing wqes on an already flushed RQ.
+	 */
+	bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
 }
 
 INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
@@ -838,17 +892,20 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
 	 */
 	wqe_bulk -= (head + wqe_bulk) & rq->wqe.info.wqe_index_mask;
 
-	if (!rq->xsk_pool)
-		count = mlx5e_alloc_rx_wqes(rq, head, wqe_bulk);
-	else if (likely(!rq->xsk_pool->dma_need_sync))
+	if (!rq->xsk_pool) {
+		count = mlx5e_refill_rx_wqes(rq, head, wqe_bulk);
+	} else if (likely(!rq->xsk_pool->dma_need_sync)) {
+		mlx5e_xsk_free_rx_wqes(rq, head, wqe_bulk);
 		count = mlx5e_xsk_alloc_rx_wqes_batched(rq, head, wqe_bulk);
-	else
+	} else {
+		mlx5e_xsk_free_rx_wqes(rq, head, wqe_bulk);
 		/* If dma_need_sync is true, it's more efficient to call
 		 * xsk_buff_alloc in a loop, rather than xsk_buff_alloc_batch,
 		 * because the latter does the same check and returns only one
 		 * frame.
 		 */
 		count = mlx5e_xsk_alloc_rx_wqes(rq, head, wqe_bulk);
+	}
 
 	mlx5_wq_cyc_push_n(wq, count);
 	if (unlikely(count != wqe_bulk)) {
@@ -1029,6 +1086,11 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
 	head = rq->mpwqe.actual_wq_head;
 	i = missing;
 	do {
+		struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, head);
+
+		/* Deferred free for better page pool cache usage. */
+		mlx5e_free_rx_mpwqe(rq, wi);
+
 		alloc_err = rq->xsk_pool ? mlx5e_xsk_alloc_rx_mpwqe(rq, head) :
 					   mlx5e_alloc_rx_mpwqe(rq, head);
 
@@ -1133,7 +1195,7 @@ static void *mlx5e_shampo_get_packet_hd(struct mlx5e_rq *rq, u16 header_index)
 	struct mlx5e_dma_info *last_head = &rq->mpwqe.shampo->info[header_index];
 	u16 head_offset = (last_head->addr & (PAGE_SIZE - 1)) + rq->buff.headroom;
 
-	return page_address(last_head->page) + head_offset;
+	return page_address(last_head->frag_page->page) + head_offset;
 }
 
 static void mlx5e_shampo_update_ipv4_udp_hdr(struct mlx5e_rq *rq, struct iphdr *ipv4)
@@ -1573,10 +1635,10 @@ struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
 }
 
 static void mlx5e_fill_mxbuf(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
-			     void *va, u16 headroom, u32 len,
+			     void *va, u16 headroom, u32 frame_sz, u32 len,
 			     struct mlx5e_xdp_buff *mxbuf)
 {
-	xdp_init_buff(&mxbuf->xdp, rq->buff.frame0_sz, &rq->xdp_rxq);
+	xdp_init_buff(&mxbuf->xdp, frame_sz, &rq->xdp_rxq);
 	xdp_prepare_buff(&mxbuf->xdp, va, headroom, len, true);
 	mxbuf->cqe = cqe;
 	mxbuf->rq = rq;
@@ -1586,7 +1648,7 @@ static struct sk_buff *
 mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
 			  struct mlx5_cqe64 *cqe, u32 cqe_bcnt)
 {
-	union mlx5e_alloc_unit *au = wi->au;
+	struct mlx5e_frag_page *frag_page = wi->frag_page;
 	u16 rx_headroom = rq->buff.headroom;
 	struct bpf_prog *prog;
 	struct sk_buff *skb;
@@ -1595,11 +1657,11 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
 	dma_addr_t addr;
 	u32 frag_size;
 
-	va             = page_address(au->page) + wi->offset;
+	va             = page_address(frag_page->page) + wi->offset;
 	data           = va + rx_headroom;
 	frag_size      = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
 
-	addr = page_pool_get_dma_addr(au->page);
+	addr = page_pool_get_dma_addr(frag_page->page);
 	dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset,
 				      frag_size, rq->buff.map_dir);
 	net_prefetch(data);
@@ -1609,7 +1671,8 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
 		struct mlx5e_xdp_buff mxbuf;
 
 		net_prefetchw(va); /* xdp_frame data area */
-		mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, cqe_bcnt, &mxbuf);
+		mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, rq->buff.frame0_sz,
+				 cqe_bcnt, &mxbuf);
 		if (mlx5e_xdp_handle(rq, prog, &mxbuf))
 			return NULL; /* page/packet was consumed by XDP */
 
@@ -1623,7 +1686,8 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
 		return NULL;
 
 	/* queue up for recycling/reuse */
-	page_ref_inc(au->page);
+	skb_mark_for_recycle(skb);
+	frag_page->frags++;
 
 	return skb;
 }
@@ -1634,8 +1698,8 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
 {
 	struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
 	struct mlx5e_wqe_frag_info *head_wi = wi;
-	union mlx5e_alloc_unit *au = wi->au;
 	u16 rx_headroom = rq->buff.headroom;
+	struct mlx5e_frag_page *frag_page;
 	struct skb_shared_info *sinfo;
 	struct mlx5e_xdp_buff mxbuf;
 	u32 frag_consumed_bytes;
@@ -1645,16 +1709,19 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
 	u32 truesize;
 	void *va;
 
-	va = page_address(au->page) + wi->offset;
+	frag_page = wi->frag_page;
+
+	va = page_address(frag_page->page) + wi->offset;
 	frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt);
 
-	addr = page_pool_get_dma_addr(au->page);
+	addr = page_pool_get_dma_addr(frag_page->page);
 	dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset,
 				      rq->buff.frame0_sz, rq->buff.map_dir);
 	net_prefetchw(va); /* xdp_frame data area */
 	net_prefetch(va + rx_headroom);
 
-	mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, frag_consumed_bytes, &mxbuf);
+	mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, rq->buff.frame0_sz,
+			 frag_consumed_bytes, &mxbuf);
 	sinfo = xdp_get_shared_info_from_buff(&mxbuf.xdp);
 	truesize = 0;
 
@@ -1663,34 +1730,12 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
 	wi++;
 
 	while (cqe_bcnt) {
-		skb_frag_t *frag;
-
-		au = wi->au;
+		frag_page = wi->frag_page;
 
 		frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt);
 
-		addr = page_pool_get_dma_addr(au->page);
-		dma_sync_single_for_cpu(rq->pdev, addr + wi->offset,
-					frag_consumed_bytes, rq->buff.map_dir);
-
-		if (!xdp_buff_has_frags(&mxbuf.xdp)) {
-			/* Init on the first fragment to avoid cold cache access
-			 * when possible.
-			 */
-			sinfo->nr_frags = 0;
-			sinfo->xdp_frags_size = 0;
-			xdp_buff_set_frags_flag(&mxbuf.xdp);
-		}
-
-		frag = &sinfo->frags[sinfo->nr_frags++];
-		__skb_frag_set_page(frag, au->page);
-		skb_frag_off_set(frag, wi->offset);
-		skb_frag_size_set(frag, frag_consumed_bytes);
-
-		if (page_is_pfmemalloc(au->page))
-			xdp_buff_set_frag_pfmemalloc(&mxbuf.xdp);
-
-		sinfo->xdp_frags_size += frag_consumed_bytes;
+		mlx5e_add_skb_shared_info_frag(rq, sinfo, &mxbuf.xdp, frag_page,
+					       wi->offset, frag_consumed_bytes);
 		truesize += frag_info->frag_stride;
 
 		cqe_bcnt -= frag_consumed_bytes;
@@ -1701,10 +1746,10 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
 	prog = rcu_dereference(rq->xdp_prog);
 	if (prog && mlx5e_xdp_handle(rq, prog, &mxbuf)) {
 		if (test_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
-			int i;
+			struct mlx5e_wqe_frag_info *pwi;
 
-			for (i = wi - head_wi; i < rq->wqe.info.num_frags; i++)
-				mlx5e_put_rx_frag(rq, &head_wi[i], true);
+			for (pwi = head_wi; pwi < wi; pwi++)
+				pwi->flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
 		}
 		return NULL; /* page/packet was consumed by XDP */
 	}
@@ -1716,21 +1761,17 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
 	if (unlikely(!skb))
 		return NULL;
 
-	page_ref_inc(head_wi->au->page);
+	skb_mark_for_recycle(skb);
+	head_wi->frag_page->frags++;
 
 	if (xdp_buff_has_frags(&mxbuf.xdp)) {
-		int i;
-
 		/* sinfo->nr_frags is reset by build_skb, calculate again. */
 		xdp_update_skb_shared_info(skb, wi - head_wi - 1,
 					   sinfo->xdp_frags_size, truesize,
 					   xdp_buff_is_frag_pfmemalloc(&mxbuf.xdp));
 
-		for (i = 0; i < sinfo->nr_frags; i++) {
-			skb_frag_t *frag = &sinfo->frags[i];
-
-			page_ref_inc(skb_frag_page(frag));
-		}
+		for (struct mlx5e_wqe_frag_info *pwi = head_wi + 1; pwi < wi; pwi++)
+			pwi->frag_page->frags++;
 	}
 
 	return skb;
@@ -1768,7 +1809,7 @@ static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 
 	if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
 		mlx5e_handle_rx_err_cqe(rq, cqe);
-		goto free_wqe;
+		goto wq_cyc_pop;
 	}
 
 	skb = INDIRECT_CALL_3(rq->wqe.skb_from_cqe,
@@ -1782,9 +1823,9 @@ static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 			/* do not return page to cache,
 			 * it will be returned on XDP_TX completion.
 			 */
-			goto wq_cyc_pop;
+			wi->flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
 		}
-		goto free_wqe;
+		goto wq_cyc_pop;
 	}
 
 	mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
@@ -1792,13 +1833,11 @@ static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 	if (mlx5e_cqe_regb_chain(cqe))
 		if (!mlx5e_tc_update_skb_nic(cqe, skb)) {
 			dev_kfree_skb_any(skb);
-			goto free_wqe;
+			goto wq_cyc_pop;
 		}
 
 	napi_gro_receive(rq->cq.napi, skb);
 
-free_wqe:
-	mlx5e_free_rx_wqe(rq, wi, true);
 wq_cyc_pop:
 	mlx5_wq_cyc_pop(wq);
 }
@@ -1822,7 +1861,7 @@ static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 
 	if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
 		mlx5e_handle_rx_err_cqe(rq, cqe);
-		goto free_wqe;
+		goto wq_cyc_pop;
 	}
 
 	skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
@@ -1835,9 +1874,9 @@ static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 			/* do not return page to cache,
 			 * it will be returned on XDP_TX completion.
 			 */
-			goto wq_cyc_pop;
+			wi->flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
 		}
-		goto free_wqe;
+		goto wq_cyc_pop;
 	}
 
 	mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
@@ -1847,8 +1886,6 @@ static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 
 	mlx5e_rep_tc_receive(cqe, rq, skb);
 
-free_wqe:
-	mlx5e_free_rx_wqe(rq, wi, true);
 wq_cyc_pop:
 	mlx5_wq_cyc_pop(wq);
 }
@@ -1901,7 +1938,6 @@ mpwrq_cqe_out:
 
 	wq  = &rq->mpwqe.wq;
 	wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
-	mlx5e_free_rx_mpwqe(rq, wi, true);
 	mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
 }
 
@@ -1913,7 +1949,8 @@ const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep = {
 
 static void
 mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq,
-		    union mlx5e_alloc_unit *au, u32 data_bcnt, u32 data_offset)
+		    struct mlx5e_frag_page *frag_page,
+		    u32 data_bcnt, u32 data_offset)
 {
 	net_prefetchw(skb->data);
 
@@ -1927,12 +1964,13 @@ mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq,
 		else
 			truesize = ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz));
 
-		mlx5e_add_skb_frag(rq, skb, au, data_offset,
+		frag_page->frags++;
+		mlx5e_add_skb_frag(rq, skb, frag_page->page, data_offset,
 				   pg_consumed_bytes, truesize);
 
 		data_bcnt -= pg_consumed_bytes;
 		data_offset = 0;
-		au++;
+		frag_page++;
 	}
 }
 
@@ -1941,37 +1979,142 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
 				   struct mlx5_cqe64 *cqe, u16 cqe_bcnt, u32 head_offset,
 				   u32 page_idx)
 {
-	union mlx5e_alloc_unit *au = &wi->alloc_units[page_idx];
+	struct mlx5e_frag_page *frag_page = &wi->alloc_units.frag_pages[page_idx];
 	u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt);
-	u32 frag_offset    = head_offset + headlen;
-	u32 byte_cnt       = cqe_bcnt - headlen;
-	union mlx5e_alloc_unit *head_au = au;
+	struct mlx5e_frag_page *head_page = frag_page;
+	u32 frag_offset    = head_offset;
+	u32 byte_cnt       = cqe_bcnt;
+	struct skb_shared_info *sinfo;
+	struct mlx5e_xdp_buff mxbuf;
+	unsigned int truesize = 0;
+	struct bpf_prog *prog;
 	struct sk_buff *skb;
-	dma_addr_t addr;
+	u32 linear_frame_sz;
+	u16 linear_data_len;
+	u16 linear_hr;
+	void *va;
 
-	skb = napi_alloc_skb(rq->cq.napi,
-			     ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long)));
-	if (unlikely(!skb)) {
-		rq->stats->buff_alloc_err++;
-		return NULL;
+	prog = rcu_dereference(rq->xdp_prog);
+
+	if (prog) {
+		/* area for bpf_xdp_[store|load]_bytes */
+		net_prefetchw(page_address(frag_page->page) + frag_offset);
+		if (unlikely(mlx5e_page_alloc_fragmented(rq, &wi->linear_page))) {
+			rq->stats->buff_alloc_err++;
+			return NULL;
+		}
+		va = page_address(wi->linear_page.page);
+		net_prefetchw(va); /* xdp_frame data area */
+		linear_hr = XDP_PACKET_HEADROOM;
+		linear_data_len = 0;
+		linear_frame_sz = MLX5_SKB_FRAG_SZ(linear_hr + MLX5E_RX_MAX_HEAD);
+	} else {
+		skb = napi_alloc_skb(rq->cq.napi,
+				     ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long)));
+		if (unlikely(!skb)) {
+			rq->stats->buff_alloc_err++;
+			return NULL;
+		}
+		skb_mark_for_recycle(skb);
+		va = skb->head;
+		net_prefetchw(va); /* xdp_frame data area */
+		net_prefetchw(skb->data);
+
+		frag_offset += headlen;
+		byte_cnt -= headlen;
+		linear_hr = skb_headroom(skb);
+		linear_data_len = headlen;
+		linear_frame_sz = MLX5_SKB_FRAG_SZ(skb_end_offset(skb));
+		if (unlikely(frag_offset >= PAGE_SIZE)) {
+			frag_page++;
+			frag_offset -= PAGE_SIZE;
+		}
 	}
 
-	net_prefetchw(skb->data);
+	mlx5e_fill_mxbuf(rq, cqe, va, linear_hr, linear_frame_sz, linear_data_len, &mxbuf);
+
+	sinfo = xdp_get_shared_info_from_buff(&mxbuf.xdp);
+
+	while (byte_cnt) {
+		/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
+		u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - frag_offset, byte_cnt);
 
-	/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
-	if (unlikely(frag_offset >= PAGE_SIZE)) {
-		au++;
-		frag_offset -= PAGE_SIZE;
+		if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
+			truesize += pg_consumed_bytes;
+		else
+			truesize += ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz));
+
+		mlx5e_add_skb_shared_info_frag(rq, sinfo, &mxbuf.xdp, frag_page, frag_offset,
+					       pg_consumed_bytes);
+		byte_cnt -= pg_consumed_bytes;
+		frag_offset = 0;
+		frag_page++;
 	}
 
-	mlx5e_fill_skb_data(skb, rq, au, byte_cnt, frag_offset);
-	/* copy header */
-	addr = page_pool_get_dma_addr(head_au->page);
-	mlx5e_copy_skb_header(rq, skb, head_au->page, addr,
-			      head_offset, head_offset, headlen);
-	/* skb linear part was allocated with headlen and aligned to long */
-	skb->tail += headlen;
-	skb->len  += headlen;
+	if (prog) {
+		if (mlx5e_xdp_handle(rq, prog, &mxbuf)) {
+			if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
+				int i;
+
+				for (i = 0; i < sinfo->nr_frags; i++)
+					/* non-atomic */
+					__set_bit(page_idx + i, wi->skip_release_bitmap);
+				return NULL;
+			}
+			mlx5e_page_release_fragmented(rq, &wi->linear_page);
+			return NULL; /* page/packet was consumed by XDP */
+		}
+
+		skb = mlx5e_build_linear_skb(rq, mxbuf.xdp.data_hard_start,
+					     linear_frame_sz,
+					     mxbuf.xdp.data - mxbuf.xdp.data_hard_start, 0,
+					     mxbuf.xdp.data - mxbuf.xdp.data_meta);
+		if (unlikely(!skb)) {
+			mlx5e_page_release_fragmented(rq, &wi->linear_page);
+			return NULL;
+		}
+
+		skb_mark_for_recycle(skb);
+		wi->linear_page.frags++;
+		mlx5e_page_release_fragmented(rq, &wi->linear_page);
+
+		if (xdp_buff_has_frags(&mxbuf.xdp)) {
+			struct mlx5e_frag_page *pagep;
+
+			/* sinfo->nr_frags is reset by build_skb, calculate again. */
+			xdp_update_skb_shared_info(skb, frag_page - head_page,
+						   sinfo->xdp_frags_size, truesize,
+						   xdp_buff_is_frag_pfmemalloc(&mxbuf.xdp));
+
+			pagep = head_page;
+			do
+				pagep->frags++;
+			while (++pagep < frag_page);
+		}
+		__pskb_pull_tail(skb, headlen);
+	} else {
+		dma_addr_t addr;
+
+		if (xdp_buff_has_frags(&mxbuf.xdp)) {
+			struct mlx5e_frag_page *pagep;
+
+			xdp_update_skb_shared_info(skb, sinfo->nr_frags,
+						   sinfo->xdp_frags_size, truesize,
+						   xdp_buff_is_frag_pfmemalloc(&mxbuf.xdp));
+
+			pagep = frag_page - sinfo->nr_frags;
+			do
+				pagep->frags++;
+			while (++pagep < frag_page);
+		}
+		/* copy header */
+		addr = page_pool_get_dma_addr(head_page->page);
+		mlx5e_copy_skb_header(rq, skb, head_page->page, addr,
+				      head_offset, head_offset, headlen);
+		/* skb linear part was allocated with headlen and aligned to long */
+		skb->tail += headlen;
+		skb->len  += headlen;
+	}
 
 	return skb;
 }
@@ -1981,7 +2124,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 				struct mlx5_cqe64 *cqe, u16 cqe_bcnt, u32 head_offset,
 				u32 page_idx)
 {
-	union mlx5e_alloc_unit *au = &wi->alloc_units[page_idx];
+	struct mlx5e_frag_page *frag_page = &wi->alloc_units.frag_pages[page_idx];
 	u16 rx_headroom = rq->buff.headroom;
 	struct bpf_prog *prog;
 	struct sk_buff *skb;
@@ -1996,11 +2139,11 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 		return NULL;
 	}
 
-	va             = page_address(au->page) + head_offset;
+	va             = page_address(frag_page->page) + head_offset;
 	data           = va + rx_headroom;
 	frag_size      = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
 
-	addr = page_pool_get_dma_addr(au->page);
+	addr = page_pool_get_dma_addr(frag_page->page);
 	dma_sync_single_range_for_cpu(rq->pdev, addr, head_offset,
 				      frag_size, rq->buff.map_dir);
 	net_prefetch(data);
@@ -2010,10 +2153,11 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 		struct mlx5e_xdp_buff mxbuf;
 
 		net_prefetchw(va); /* xdp_frame data area */
-		mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, cqe_bcnt, &mxbuf);
+		mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, rq->buff.frame0_sz,
+				 cqe_bcnt, &mxbuf);
 		if (mlx5e_xdp_handle(rq, prog, &mxbuf)) {
 			if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
-				__set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
+				__set_bit(page_idx, wi->skip_release_bitmap); /* non-atomic */
 			return NULL; /* page/packet was consumed by XDP */
 		}
 
@@ -2027,7 +2171,8 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 		return NULL;
 
 	/* queue up for recycling/reuse */
-	page_ref_inc(au->page);
+	skb_mark_for_recycle(skb);
+	frag_page->frags++;
 
 	return skb;
 }
@@ -2044,7 +2189,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 	void *hdr, *data;
 	u32 frag_size;
 
-	hdr		= page_address(head->page) + head_offset;
+	hdr		= page_address(head->frag_page->page) + head_offset;
 	data		= hdr + rx_headroom;
 	frag_size	= MLX5_SKB_FRAG_SZ(rx_headroom + head_size);
 
@@ -2058,9 +2203,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 		if (unlikely(!skb))
 			return NULL;
 
-		/* queue up for recycling/reuse */
-		page_ref_inc(head->page);
-
+		head->frag_page->frags++;
 	} else {
 		/* allocate SKB and copy header for large header */
 		rq->stats->gro_large_hds++;
@@ -2072,13 +2215,17 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 		}
 
 		prefetchw(skb->data);
-		mlx5e_copy_skb_header(rq, skb, head->page, head->addr,
+		mlx5e_copy_skb_header(rq, skb, head->frag_page->page, head->addr,
 				      head_offset + rx_headroom,
 				      rx_headroom, head_size);
 		/* skb linear part was allocated with headlen and aligned to long */
 		skb->tail += head_size;
 		skb->len  += head_size;
 	}
+
+	/* queue up for recycling/reuse */
+	skb_mark_for_recycle(skb);
+
 	return skb;
 }
 
@@ -2123,8 +2270,10 @@ mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index)
 	u64 addr = shampo->info[header_index].addr;
 
 	if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) {
-		shampo->info[header_index].addr = ALIGN_DOWN(addr, PAGE_SIZE);
-		mlx5e_page_release_dynamic(rq, shampo->info[header_index].page, true);
+		struct mlx5e_dma_info *dma_info = &shampo->info[header_index];
+
+		dma_info->addr = ALIGN_DOWN(addr, PAGE_SIZE);
+		mlx5e_page_release_fragmented(rq, dma_info->frag_page);
 	}
 	bitmap_clear(shampo->bitmap, header_index, 1);
 }
@@ -2145,7 +2294,6 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq
 	bool match		= cqe->shampo.match;
 	struct mlx5e_rq_stats *stats = rq->stats;
 	struct mlx5e_rx_wqe_ll *wqe;
-	union mlx5e_alloc_unit *au;
 	struct mlx5e_mpw_info *wi;
 	struct mlx5_wq_ll *wq;
 
@@ -2195,8 +2343,10 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq
 	}
 
 	if (likely(head_size)) {
-		au = &wi->alloc_units[page_idx];
-		mlx5e_fill_skb_data(*skb, rq, au, data_bcnt, data_offset);
+		struct mlx5e_frag_page *frag_page;
+
+		frag_page = &wi->alloc_units.frag_pages[page_idx];
+		mlx5e_fill_skb_data(*skb, rq, frag_page, data_bcnt, data_offset);
 	}
 
 	mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb);
@@ -2210,7 +2360,6 @@ mpwrq_cqe_out:
 
 	wq  = &rq->mpwqe.wq;
 	wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
-	mlx5e_free_rx_mpwqe(rq, wi, true);
 	mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
 }
 
@@ -2270,7 +2419,6 @@ mpwrq_cqe_out:
 
 	wq  = &rq->mpwqe.wq;
 	wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
-	mlx5e_free_rx_mpwqe(rq, wi, true);
 	mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
 }
 
@@ -2489,7 +2637,7 @@ static void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 
 	if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
 		rq->stats->wqe_err++;
-		goto wq_free_wqe;
+		goto wq_cyc_pop;
 	}
 
 	skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
@@ -2497,17 +2645,16 @@ static void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 			      mlx5e_skb_from_cqe_nonlinear,
 			      rq, wi, cqe, cqe_bcnt);
 	if (!skb)
-		goto wq_free_wqe;
+		goto wq_cyc_pop;
 
 	mlx5i_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
 	if (unlikely(!skb->dev)) {
 		dev_kfree_skb_any(skb);
-		goto wq_free_wqe;
+		goto wq_cyc_pop;
 	}
 	napi_gro_receive(rq->cq.napi, skb);
 
-wq_free_wqe:
-	mlx5e_free_rx_wqe(rq, wi, true);
+wq_cyc_pop:
 	mlx5_wq_cyc_pop(wq);
 }
 
@@ -2582,12 +2729,12 @@ static void mlx5e_trap_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe
 
 	if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
 		rq->stats->wqe_err++;
-		goto free_wqe;
+		goto wq_cyc_pop;
 	}
 
 	skb = mlx5e_skb_from_cqe_nonlinear(rq, wi, cqe, cqe_bcnt);
 	if (!skb)
-		goto free_wqe;
+		goto wq_cyc_pop;
 
 	mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
 	skb_push(skb, ETH_HLEN);
@@ -2596,8 +2743,7 @@ static void mlx5e_trap_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe
 				 rq->netdev->devlink_port);
 	dev_kfree_skb_any(skb);
 
-free_wqe:
-	mlx5e_free_rx_wqe(rq, wi, false);
+wq_cyc_pop:
 	mlx5_wq_cyc_pop(wq);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
index 4478223c1720..f1d9596905c6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
@@ -179,11 +179,6 @@ static const struct counter_desc sw_stats_desc[] = {
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_buff_alloc_err) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_blks) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_pkts) },
-	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_reuse) },
-	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) },
-	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },
-	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) },
-	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_waive) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_congst_umr) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_arfs_err) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_recover) },
@@ -358,11 +353,6 @@ static void mlx5e_stats_grp_sw_update_stats_rq_stats(struct mlx5e_sw_stats *s,
 	s->rx_buff_alloc_err          += rq_stats->buff_alloc_err;
 	s->rx_cqe_compress_blks       += rq_stats->cqe_compress_blks;
 	s->rx_cqe_compress_pkts       += rq_stats->cqe_compress_pkts;
-	s->rx_cache_reuse             += rq_stats->cache_reuse;
-	s->rx_cache_full              += rq_stats->cache_full;
-	s->rx_cache_empty             += rq_stats->cache_empty;
-	s->rx_cache_busy              += rq_stats->cache_busy;
-	s->rx_cache_waive             += rq_stats->cache_waive;
 	s->rx_congst_umr              += rq_stats->congst_umr;
 	s->rx_arfs_err                += rq_stats->arfs_err;
 	s->rx_recover                 += rq_stats->recover;
@@ -1978,11 +1968,6 @@ static const struct counter_desc rq_stats_desc[] = {
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, buff_alloc_err) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_blks) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) },
-	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_reuse) },
-	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) },
-	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) },
-	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_busy) },
-	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_waive) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, congst_umr) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, arfs_err) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, recover) },
@@ -2163,11 +2148,6 @@ static const struct counter_desc ptp_rq_stats_desc[] = {
 	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, buff_alloc_err) },
 	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, cqe_compress_blks) },
 	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) },
-	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, cache_reuse) },
-	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, cache_full) },
-	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, cache_empty) },
-	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, cache_busy) },
-	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, cache_waive) },
 	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, congst_umr) },
 	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, arfs_err) },
 	{ MLX5E_DECLARE_PTP_RQ_STAT(struct mlx5e_rq_stats, recover) },
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
index b77100b60b50..1ff8a06027dc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
@@ -193,11 +193,6 @@ struct mlx5e_sw_stats {
 	u64 rx_buff_alloc_err;
 	u64 rx_cqe_compress_blks;
 	u64 rx_cqe_compress_pkts;
-	u64 rx_cache_reuse;
-	u64 rx_cache_full;
-	u64 rx_cache_empty;
-	u64 rx_cache_busy;
-	u64 rx_cache_waive;
 	u64 rx_congst_umr;
 	u64 rx_arfs_err;
 	u64 rx_recover;
@@ -362,11 +357,6 @@ struct mlx5e_rq_stats {
 	u64 buff_alloc_err;
 	u64 cqe_compress_blks;
 	u64 cqe_compress_pkts;
-	u64 cache_reuse;
-	u64 cache_full;
-	u64 cache_empty;
-	u64 cache_busy;
-	u64 cache_waive;
 	u64 congst_umr;
 	u64 arfs_err;
 	u64 recover;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index 87a2850b32d0..728b82ce4031 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -44,6 +44,7 @@
 #include <net/bareudp.h>
 #include <net/bonding.h>
 #include <net/dst_metadata.h>
+#include "devlink.h"
 #include "en.h"
 #include "en/tc/post_act.h"
 #include "en/tc/act_stats.h"
@@ -73,12 +74,6 @@
 #define MLX5E_TC_TABLE_NUM_GROUPS 4
 #define MLX5E_TC_TABLE_MAX_GROUP_SIZE BIT(18)
 
-struct mlx5e_hairpin_params {
-	struct mlx5_core_dev *mdev;
-	u32 num_queues;
-	u32 queue_size;
-};
-
 struct mlx5e_tc_table {
 	/* Protects the dynamic assignment of the t parameter
 	 * which is the nic tc root table.
@@ -101,7 +96,6 @@ struct mlx5e_tc_table {
 
 	struct mlx5_tc_ct_priv         *ct;
 	struct mapping_ctx             *mapping;
-	struct mlx5e_hairpin_params    hairpin_params;
 	struct dentry                  *dfs_root;
 
 	/* tc action stats */
@@ -183,7 +177,8 @@ static struct lock_class_key tc_ht_wq_key;
 
 static void mlx5e_put_flow_tunnel_id(struct mlx5e_tc_flow *flow);
 static void free_flow_post_acts(struct mlx5e_tc_flow *flow);
-static void mlx5_free_flow_attr(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr);
+static void mlx5_free_flow_attr_actions(struct mlx5e_tc_flow *flow,
+					struct mlx5_flow_attr *attr);
 
 void
 mlx5e_tc_match_to_reg_match(struct mlx5_flow_spec *spec,
@@ -493,15 +488,6 @@ mlx5e_tc_rule_offload(struct mlx5e_priv *priv,
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	int err;
 
-	if (attr->flags & MLX5_ATTR_FLAG_CT) {
-		struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts =
-			&attr->parse_attr->mod_hdr_acts;
-
-		return mlx5_tc_ct_flow_offload(get_ct_priv(priv),
-					       spec, attr,
-					       mod_hdr_acts);
-	}
-
 	if (!is_mdev_switchdev_mode(priv->mdev))
 		return mlx5e_add_offloaded_nic_rule(priv, spec, attr);
 
@@ -524,11 +510,6 @@ mlx5e_tc_rule_unoffload(struct mlx5e_priv *priv,
 {
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 
-	if (attr->flags & MLX5_ATTR_FLAG_CT) {
-		mlx5_tc_ct_delete_flow(get_ct_priv(priv), attr);
-		return;
-	}
-
 	if (!is_mdev_switchdev_mode(priv->mdev)) {
 		mlx5e_del_offloaded_nic_rule(priv, rule, attr);
 		return;
@@ -589,6 +570,7 @@ struct mlx5e_hairpin {
 	struct mlx5e_tir direct_tir;
 
 	int num_channels;
+	u8 log_num_packets;
 	struct mlx5e_rqt indir_rqt;
 	struct mlx5e_tir indir_tir[MLX5E_NUM_INDIR_TIRS];
 	struct mlx5_ttc_table *ttc;
@@ -935,6 +917,7 @@ mlx5e_hairpin_create(struct mlx5e_priv *priv, struct mlx5_hairpin_params *params
 	hp->func_mdev = func_mdev;
 	hp->func_priv = priv;
 	hp->num_channels = params->num_channels;
+	hp->log_num_packets = params->log_num_packets;
 
 	err = mlx5e_hairpin_create_transport(hp);
 	if (err)
@@ -1076,9 +1059,11 @@ static int debugfs_hairpin_table_dump_show(struct seq_file *file, void *priv)
 
 	mutex_lock(&tc->hairpin_tbl_lock);
 	hash_for_each(tc->hairpin_tbl, bkt, hpe, hairpin_hlist)
-		seq_printf(file, "Hairpin peer_vhca_id %u prio %u refcnt %u\n",
+		seq_printf(file,
+			   "Hairpin peer_vhca_id %u prio %u refcnt %u num_channels %u num_packets %lu\n",
 			   hpe->peer_vhca_id, hpe->prio,
-			   refcount_read(&hpe->refcnt));
+			   refcount_read(&hpe->refcnt), hpe->hp->num_channels,
+			   BIT(hpe->hp->log_num_packets));
 	mutex_unlock(&tc->hairpin_tbl_lock);
 
 	return 0;
@@ -1099,33 +1084,15 @@ static void mlx5e_tc_debugfs_init(struct mlx5e_tc_table *tc,
 			    &debugfs_hairpin_table_dump_fops);
 }
 
-static void
-mlx5e_hairpin_params_init(struct mlx5e_hairpin_params *hairpin_params,
-			  struct mlx5_core_dev *mdev)
-{
-	u32 link_speed = 0;
-	u64 link_speed64;
-
-	hairpin_params->mdev = mdev;
-	/* set hairpin pair per each 50Gbs share of the link */
-	mlx5e_port_max_linkspeed(mdev, &link_speed);
-	link_speed = max_t(u32, link_speed, 50000);
-	link_speed64 = link_speed;
-	do_div(link_speed64, 50000);
-	hairpin_params->num_queues = link_speed64;
-
-	hairpin_params->queue_size =
-		BIT(min_t(u32, 16 - MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev),
-			  MLX5_CAP_GEN(mdev, log_max_hairpin_num_packets)));
-}
-
 static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
 				  struct mlx5e_tc_flow *flow,
 				  struct mlx5e_tc_flow_parse_attr *parse_attr,
 				  struct netlink_ext_ack *extack)
 {
 	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
+	struct devlink *devlink = priv_to_devlink(priv->mdev);
 	int peer_ifindex = parse_attr->mirred_ifindex[0];
+	union devlink_param_value val = {};
 	struct mlx5_hairpin_params params;
 	struct mlx5_core_dev *peer_mdev;
 	struct mlx5e_hairpin_entry *hpe;
@@ -1182,7 +1149,14 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
 		 hash_hairpin_info(peer_id, match_prio));
 	mutex_unlock(&tc->hairpin_tbl_lock);
 
-	params.log_num_packets = ilog2(tc->hairpin_params.queue_size);
+	err = devl_param_driverinit_value_get(
+		devlink, MLX5_DEVLINK_PARAM_ID_HAIRPIN_QUEUE_SIZE, &val);
+	if (err) {
+		err = -ENOMEM;
+		goto out_err;
+	}
+
+	params.log_num_packets = ilog2(val.vu32);
 	params.log_data_size =
 		clamp_t(u32,
 			params.log_num_packets +
@@ -1191,7 +1165,14 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
 			MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz));
 
 	params.q_counter = priv->q_counter;
-	params.num_channels = tc->hairpin_params.num_queues;
+	err = devl_param_driverinit_value_get(
+		devlink, MLX5_DEVLINK_PARAM_ID_HAIRPIN_NUM_QUEUES, &val);
+	if (err) {
+		err = -ENOMEM;
+		goto out_err;
+	}
+
+	params.num_channels = val.vu32;
 
 	hp = mlx5e_hairpin_create(priv, &params, peer_ifindex);
 	hpe->hp = hp;
@@ -1401,13 +1382,7 @@ mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
 			return err;
 	}
 
-	if (attr->flags & MLX5_ATTR_FLAG_CT)
-		flow->rule[0] = mlx5_tc_ct_flow_offload(get_ct_priv(priv), &parse_attr->spec,
-							attr, &parse_attr->mod_hdr_acts);
-	else
-		flow->rule[0] = mlx5e_add_offloaded_nic_rule(priv, &parse_attr->spec,
-							     attr);
-
+	flow->rule[0] = mlx5e_add_offloaded_nic_rule(priv, &parse_attr->spec, attr);
 	return PTR_ERR_OR_ZERO(flow->rule[0]);
 }
 
@@ -1438,9 +1413,7 @@ static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv,
 
 	flow_flag_clear(flow, OFFLOADED);
 
-	if (attr->flags & MLX5_ATTR_FLAG_CT)
-		mlx5_tc_ct_delete_flow(get_ct_priv(flow->priv), attr);
-	else if (!IS_ERR_OR_NULL(flow->rule[0]))
+	if (!IS_ERR_OR_NULL(flow->rule[0]))
 		mlx5e_del_offloaded_nic_rule(priv, flow->rule[0], attr);
 
 	/* Remove root table if no rules are left to avoid
@@ -1791,8 +1764,7 @@ out:
 static void
 clean_encap_dests(struct mlx5e_priv *priv,
 		  struct mlx5e_tc_flow *flow,
-		  struct mlx5_flow_attr *attr,
-		  bool *vf_tun)
+		  struct mlx5_flow_attr *attr)
 {
 	struct mlx5_esw_flow_attr *esw_attr;
 	int out_index;
@@ -1801,17 +1773,11 @@ clean_encap_dests(struct mlx5e_priv *priv,
 		return;
 
 	esw_attr = attr->esw_attr;
-	*vf_tun = false;
 
 	for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) {
 		if (!(esw_attr->dests[out_index].flags & MLX5_ESW_DEST_ENCAP))
 			continue;
 
-		if (esw_attr->dests[out_index].flags &
-		    MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE &&
-		    !esw_attr->dest_int_port)
-			*vf_tun = true;
-
 		mlx5e_detach_encap(priv, flow, attr, out_index);
 		kfree(attr->parse_attr->tun_info[out_index]);
 	}
@@ -2034,7 +2000,7 @@ static void free_branch_attr(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *
 	if (!attr)
 		return;
 
-	mlx5_free_flow_attr(flow, attr);
+	mlx5_free_flow_attr_actions(flow, attr);
 	kvfree(attr->parse_attr);
 	kfree(attr);
 }
@@ -2045,7 +2011,6 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv,
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	struct mlx5_flow_attr *attr = flow->attr;
 	struct mlx5_esw_flow_attr *esw_attr;
-	bool vf_tun;
 
 	esw_attr = attr->esw_attr;
 	mlx5e_put_flow_tunnel_id(flow);
@@ -2067,18 +2032,8 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv,
 	if (flow->decap_route)
 		mlx5e_detach_decap_route(priv, flow);
 
-	clean_encap_dests(priv, flow, attr, &vf_tun);
-
 	mlx5_tc_ct_match_del(get_ct_priv(priv), &flow->attr->ct_attr);
 
-	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
-		mlx5e_mod_hdr_dealloc(&attr->parse_attr->mod_hdr_acts);
-		mlx5e_tc_detach_mod_hdr(priv, flow, attr);
-	}
-
-	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
-		mlx5_fc_destroy(esw_attr->counter_dev, attr->counter);
-
 	if (esw_attr->int_port)
 		mlx5e_tc_int_port_put(mlx5e_get_int_port_priv(priv), esw_attr->int_port);
 
@@ -2091,8 +2046,7 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv,
 	mlx5e_tc_act_stats_del_flow(get_act_stats_handle(priv), flow);
 
 	free_flow_post_acts(flow);
-	free_branch_attr(flow, attr->branch_true);
-	free_branch_attr(flow, attr->branch_false);
+	mlx5_free_flow_attr_actions(flow, attr);
 
 	kvfree(attr->esw_attr->rx_tun_attr);
 	kvfree(attr->parse_attr);
@@ -3469,114 +3423,59 @@ struct ipv6_hoplimit_word {
 };
 
 static bool
-is_action_keys_supported(const struct flow_action_entry *act, bool ct_flow,
-			 bool *modify_ip_header, bool *modify_tuple,
-			 struct netlink_ext_ack *extack)
+is_flow_action_modify_ip_header(struct flow_action *flow_action)
 {
+	const struct flow_action_entry *act;
 	u32 mask, offset;
 	u8 htype;
+	int i;
 
-	htype = act->mangle.htype;
-	offset = act->mangle.offset;
-	mask = ~act->mangle.mask;
 	/* For IPv4 & IPv6 header check 4 byte word,
 	 * to determine that modified fields
 	 * are NOT ttl & hop_limit only.
 	 */
-	if (htype == FLOW_ACT_MANGLE_HDR_TYPE_IP4) {
-		struct ip_ttl_word *ttl_word =
-			(struct ip_ttl_word *)&mask;
-
-		if (offset != offsetof(struct iphdr, ttl) ||
-		    ttl_word->protocol ||
-		    ttl_word->check) {
-			*modify_ip_header = true;
-		}
-
-		if (offset >= offsetof(struct iphdr, saddr))
-			*modify_tuple = true;
-
-		if (ct_flow && *modify_tuple) {
-			NL_SET_ERR_MSG_MOD(extack,
-					   "can't offload re-write of ipv4 address with action ct");
-			return false;
-		}
-	} else if (htype == FLOW_ACT_MANGLE_HDR_TYPE_IP6) {
-		struct ipv6_hoplimit_word *hoplimit_word =
-			(struct ipv6_hoplimit_word *)&mask;
-
-		if (offset != offsetof(struct ipv6hdr, payload_len) ||
-		    hoplimit_word->payload_len ||
-		    hoplimit_word->nexthdr) {
-			*modify_ip_header = true;
-		}
-
-		if (ct_flow && offset >= offsetof(struct ipv6hdr, saddr))
-			*modify_tuple = true;
+	flow_action_for_each(i, act, flow_action) {
+		if (act->id != FLOW_ACTION_MANGLE &&
+		    act->id != FLOW_ACTION_ADD)
+			continue;
 
-		if (ct_flow && *modify_tuple) {
-			NL_SET_ERR_MSG_MOD(extack,
-					   "can't offload re-write of ipv6 address with action ct");
-			return false;
-		}
-	} else if (htype == FLOW_ACT_MANGLE_HDR_TYPE_TCP ||
-		   htype == FLOW_ACT_MANGLE_HDR_TYPE_UDP) {
-		*modify_tuple = true;
-		if (ct_flow) {
-			NL_SET_ERR_MSG_MOD(extack,
-					   "can't offload re-write of transport header ports with action ct");
-			return false;
+		htype = act->mangle.htype;
+		offset = act->mangle.offset;
+		mask = ~act->mangle.mask;
+
+		if (htype == FLOW_ACT_MANGLE_HDR_TYPE_IP4) {
+			struct ip_ttl_word *ttl_word =
+				(struct ip_ttl_word *)&mask;
+
+			if (offset != offsetof(struct iphdr, ttl) ||
+			    ttl_word->protocol ||
+			    ttl_word->check)
+				return true;
+		} else if (htype == FLOW_ACT_MANGLE_HDR_TYPE_IP6) {
+			struct ipv6_hoplimit_word *hoplimit_word =
+				(struct ipv6_hoplimit_word *)&mask;
+
+			if (offset != offsetof(struct ipv6hdr, payload_len) ||
+			    hoplimit_word->payload_len ||
+			    hoplimit_word->nexthdr)
+				return true;
 		}
 	}
 
-	return true;
-}
-
-static bool modify_tuple_supported(bool modify_tuple, bool ct_clear,
-				   bool ct_flow, struct netlink_ext_ack *extack,
-				   struct mlx5e_priv *priv,
-				   struct mlx5_flow_spec *spec)
-{
-	if (!modify_tuple || ct_clear)
-		return true;
-
-	if (ct_flow) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "can't offload tuple modification with non-clear ct()");
-		netdev_info(priv->netdev,
-			    "can't offload tuple modification with non-clear ct()");
-		return false;
-	}
-
-	/* Add ct_state=-trk match so it will be offloaded for non ct flows
-	 * (or after clear action), as otherwise, since the tuple is changed,
-	 * we can't restore ct state
-	 */
-	if (mlx5_tc_ct_add_no_trk_match(spec)) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "can't offload tuple modification with ct matches and no ct(clear) action");
-		netdev_info(priv->netdev,
-			    "can't offload tuple modification with ct matches and no ct(clear) action");
-		return false;
-	}
-
-	return true;
+	return false;
 }
 
 static bool modify_header_match_supported(struct mlx5e_priv *priv,
 					  struct mlx5_flow_spec *spec,
 					  struct flow_action *flow_action,
-					  u32 actions, bool ct_flow,
-					  bool ct_clear,
+					  u32 actions,
 					  struct netlink_ext_ack *extack)
 {
-	const struct flow_action_entry *act;
-	bool modify_ip_header, modify_tuple;
+	bool modify_ip_header;
 	void *headers_c;
 	void *headers_v;
 	u16 ethertype;
 	u8 ip_proto;
-	int i;
 
 	headers_c = mlx5e_get_match_headers_criteria(actions, spec);
 	headers_v = mlx5e_get_match_headers_value(actions, spec);
@@ -3587,23 +3486,7 @@ static bool modify_header_match_supported(struct mlx5e_priv *priv,
 	    ethertype != ETH_P_IP && ethertype != ETH_P_IPV6)
 		goto out_ok;
 
-	modify_ip_header = false;
-	modify_tuple = false;
-	flow_action_for_each(i, act, flow_action) {
-		if (act->id != FLOW_ACTION_MANGLE &&
-		    act->id != FLOW_ACTION_ADD)
-			continue;
-
-		if (!is_action_keys_supported(act, ct_flow,
-					      &modify_ip_header,
-					      &modify_tuple, extack))
-			return false;
-	}
-
-	if (!modify_tuple_supported(modify_tuple, ct_clear, ct_flow, extack,
-				    priv, spec))
-		return false;
-
+	modify_ip_header = is_flow_action_modify_ip_header(flow_action);
 	ip_proto = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ip_protocol);
 	if (modify_ip_header && ip_proto != IPPROTO_TCP &&
 	    ip_proto != IPPROTO_UDP && ip_proto != IPPROTO_ICMP) {
@@ -3624,19 +3507,6 @@ actions_match_supported_fdb(struct mlx5e_priv *priv,
 			    struct netlink_ext_ack *extack)
 {
 	struct mlx5_esw_flow_attr *esw_attr = flow->attr->esw_attr;
-	bool ct_flow, ct_clear;
-
-	ct_clear = flow->attr->ct_attr.ct_action & TCA_CT_ACT_CLEAR;
-	ct_flow = flow_flag_test(flow, CT) && !ct_clear;
-
-	if (esw_attr->split_count && ct_flow &&
-	    !MLX5_CAP_GEN(esw_attr->in_mdev, reg_c_preserve)) {
-		/* All registers used by ct are cleared when using
-		 * split rules.
-		 */
-		NL_SET_ERR_MSG_MOD(extack, "Can't offload mirroring with action ct");
-		return false;
-	}
 
 	if (esw_attr->split_count > 0 && !mlx5_esw_has_fwd_fdb(priv->mdev)) {
 		NL_SET_ERR_MSG_MOD(extack,
@@ -3657,14 +3527,9 @@ actions_match_supported(struct mlx5e_priv *priv,
 			struct mlx5e_tc_flow *flow,
 			struct netlink_ext_ack *extack)
 {
-	bool ct_flow, ct_clear;
-
-	ct_clear = flow->attr->ct_attr.ct_action & TCA_CT_ACT_CLEAR;
-	ct_flow = flow_flag_test(flow, CT) && !ct_clear;
-
 	if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR &&
-	    !modify_header_match_supported(priv, &parse_attr->spec, flow_action,
-					   actions, ct_flow, ct_clear, extack))
+	    !modify_header_match_supported(priv, &parse_attr->spec, flow_action, actions,
+					   extack))
 		return false;
 
 	if (mlx5e_is_eswitch_flow(flow) &&
@@ -3758,6 +3623,7 @@ mlx5e_clone_flow_attr_for_post_act(struct mlx5_flow_attr *attr,
 	attr2->dest_chain = 0;
 	attr2->dest_ft = NULL;
 	attr2->act_id_restore_rule = NULL;
+	memset(&attr2->ct_attr, 0, sizeof(attr2->ct_attr));
 
 	if (ns_type == MLX5_FLOW_NAMESPACE_FDB) {
 		attr2->esw_attr->out_count = 0;
@@ -3811,9 +3677,7 @@ free_flow_post_acts(struct mlx5e_tc_flow *flow)
 		if (list_is_last(&attr->list, &flow->attrs))
 			break;
 
-		mlx5_free_flow_attr(flow, attr);
-		free_branch_attr(flow, attr->branch_true);
-		free_branch_attr(flow, attr->branch_false);
+		mlx5_free_flow_attr_actions(flow, attr);
 
 		list_del(&attr->list);
 		kvfree(attr->parse_attr);
@@ -4071,76 +3935,79 @@ parse_tc_actions(struct mlx5e_tc_act_parse_state *parse_state,
 		 struct flow_action *flow_action)
 {
 	struct netlink_ext_ack *extack = parse_state->extack;
-	struct mlx5e_tc_flow_action flow_action_reorder;
 	struct mlx5e_tc_flow *flow = parse_state->flow;
 	struct mlx5e_tc_jump_state jump_state = {};
 	struct mlx5_flow_attr *attr = flow->attr;
 	enum mlx5_flow_namespace_type ns_type;
 	struct mlx5e_priv *priv = flow->priv;
-	struct flow_action_entry *act, **_act;
+	struct mlx5_flow_attr *prev_attr;
+	struct flow_action_entry *act;
 	struct mlx5e_tc_act *tc_act;
+	bool is_missable;
 	int err, i;
 
-	flow_action_reorder.num_entries = flow_action->num_entries;
-	flow_action_reorder.entries = kcalloc(flow_action->num_entries,
-					      sizeof(flow_action), GFP_KERNEL);
-	if (!flow_action_reorder.entries)
-		return -ENOMEM;
-
-	mlx5e_tc_act_reorder_flow_actions(flow_action, &flow_action_reorder);
-
 	ns_type = mlx5e_get_flow_namespace(flow);
 	list_add(&attr->list, &flow->attrs);
 
-	flow_action_for_each(i, _act, &flow_action_reorder) {
+	flow_action_for_each(i, act, flow_action) {
 		jump_state.jump_target = false;
-		act = *_act;
+		is_missable = false;
+		prev_attr = attr;
+
 		tc_act = mlx5e_tc_act_get(act->id, ns_type);
 		if (!tc_act) {
 			NL_SET_ERR_MSG_MOD(extack, "Not implemented offload action");
 			err = -EOPNOTSUPP;
-			goto out_free;
+			goto out_free_post_acts;
 		}
 
-		if (!tc_act->can_offload(parse_state, act, i, attr)) {
+		if (tc_act->can_offload && !tc_act->can_offload(parse_state, act, i, attr)) {
 			err = -EOPNOTSUPP;
-			goto out_free;
+			goto out_free_post_acts;
 		}
 
 		err = tc_act->parse_action(parse_state, act, priv, attr);
 		if (err)
-			goto out_free;
+			goto out_free_post_acts;
 
 		dec_jump_count(act, tc_act, attr, priv, &jump_state);
 
 		err = parse_branch_ctrl(act, tc_act, flow, attr, &jump_state, extack);
 		if (err)
-			goto out_free;
+			goto out_free_post_acts;
 
 		parse_state->actions |= attr->action;
-		if (!tc_act->stats_action)
-			attr->tc_act_cookies[attr->tc_act_cookies_count++] = act->cookie;
 
 		/* Split attr for multi table act if not the last act. */
 		if (jump_state.jump_target ||
 		    (tc_act->is_multi_table_act &&
 		    tc_act->is_multi_table_act(priv, act, attr) &&
-		    i < flow_action_reorder.num_entries - 1)) {
+		    i < flow_action->num_entries - 1)) {
+			is_missable = tc_act->is_missable ? tc_act->is_missable(act) : false;
+
 			err = mlx5e_tc_act_post_parse(parse_state, flow_action, attr, ns_type);
 			if (err)
-				goto out_free;
+				goto out_free_post_acts;
 
 			attr = mlx5e_clone_flow_attr_for_post_act(flow->attr, ns_type);
 			if (!attr) {
 				err = -ENOMEM;
-				goto out_free;
+				goto out_free_post_acts;
 			}
 
 			list_add(&attr->list, &flow->attrs);
 		}
-	}
 
-	kfree(flow_action_reorder.entries);
+		if (is_missable) {
+			/* Add counter to prev, and assign act to new (next) attr */
+			prev_attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
+			flow_flag_set(flow, USE_ACT_STATS);
+
+			attr->tc_act_cookies[attr->tc_act_cookies_count++] = act->cookie;
+		} else if (!tc_act->stats_action) {
+			prev_attr->tc_act_cookies[prev_attr->tc_act_cookies_count++] = act->cookie;
+		}
+	}
 
 	err = mlx5e_tc_act_post_parse(parse_state, flow_action, attr, ns_type);
 	if (err)
@@ -4152,8 +4019,6 @@ parse_tc_actions(struct mlx5e_tc_act_parse_state *parse_state,
 
 	return 0;
 
-out_free:
-	kfree(flow_action_reorder.entries);
 out_free_post_acts:
 	free_flow_post_acts(flow);
 
@@ -4448,10 +4313,9 @@ mlx5_alloc_flow_attr(enum mlx5_flow_namespace_type type)
 }
 
 static void
-mlx5_free_flow_attr(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr)
+mlx5_free_flow_attr_actions(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr)
 {
 	struct mlx5_core_dev *counter_dev = get_flow_counter_dev(flow);
-	bool vf_tun;
 
 	if (!attr)
 		return;
@@ -4459,7 +4323,7 @@ mlx5_free_flow_attr(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr)
 	if (attr->post_act_handle)
 		mlx5e_tc_post_act_del(get_post_action(flow->priv), attr->post_act_handle);
 
-	clean_encap_dests(flow->priv, flow, attr, &vf_tun);
+	clean_encap_dests(flow->priv, flow, attr);
 
 	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
 		mlx5_fc_destroy(counter_dev, attr->counter);
@@ -4468,6 +4332,11 @@ mlx5_free_flow_attr(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr)
 		mlx5e_mod_hdr_dealloc(&attr->parse_attr->mod_hdr_acts);
 		mlx5e_tc_detach_mod_hdr(flow->priv, flow, attr);
 	}
+
+	mlx5_tc_ct_delete_flow(get_ct_priv(flow->priv), attr);
+
+	free_branch_attr(flow, attr->branch_true);
+	free_branch_attr(flow, attr->branch_false);
 }
 
 static int
@@ -4929,7 +4798,7 @@ int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv,
 		goto errout;
 	}
 
-	if (mlx5e_is_offloaded_flow(flow) || flow_flag_test(flow, CT)) {
+	if (mlx5e_is_offloaded_flow(flow)) {
 		if (flow_flag_test(flow, USE_ACT_STATS)) {
 			f->use_act_stats = true;
 		} else {
@@ -5187,22 +5056,6 @@ static int mlx5e_tc_netdev_event(struct notifier_block *this,
 	return NOTIFY_DONE;
 }
 
-static int mlx5e_tc_nic_get_ft_size(struct mlx5_core_dev *dev)
-{
-	int tc_grp_size, tc_tbl_size;
-	u32 max_flow_counter;
-
-	max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
-			    MLX5_CAP_GEN(dev, max_flow_counter_15_0);
-
-	tc_grp_size = min_t(int, max_flow_counter, MLX5E_TC_TABLE_MAX_GROUP_SIZE);
-
-	tc_tbl_size = min_t(int, tc_grp_size * MLX5E_TC_TABLE_NUM_GROUPS,
-			    BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev, log_max_ft_size)));
-
-	return tc_tbl_size;
-}
-
 static int mlx5e_tc_nic_create_miss_table(struct mlx5e_priv *priv)
 {
 	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
@@ -5275,10 +5128,10 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv)
 		attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED |
 			MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
 	attr.ns = MLX5_FLOW_NAMESPACE_KERNEL;
-	attr.max_ft_sz = mlx5e_tc_nic_get_ft_size(dev);
 	attr.max_grp_num = MLX5E_TC_TABLE_NUM_GROUPS;
 	attr.default_ft = tc->miss_t;
 	attr.mapping = chains_mapping;
+	attr.fs_base_prio = MLX5E_TC_PRIO;
 
 	tc->chains = mlx5_chains_create(dev, &attr);
 	if (IS_ERR(tc->chains)) {
@@ -5286,12 +5139,12 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv)
 		goto err_miss;
 	}
 
+	mlx5_chains_print_info(tc->chains);
+
 	tc->post_act = mlx5e_tc_post_act_init(priv, tc->chains, MLX5_FLOW_NAMESPACE_KERNEL);
 	tc->ct = mlx5_tc_ct_init(priv, tc->chains, &tc->mod_hdr,
 				 MLX5_FLOW_NAMESPACE_KERNEL, tc->post_act);
 
-	mlx5e_hairpin_params_init(&tc->hairpin_params, dev);
-
 	tc->netdevice_nb.notifier_call = mlx5e_tc_netdev_event;
 	err = register_netdevice_notifier_dev_net(priv->netdev,
 						  &tc->netdevice_nb,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index 9a458a5d9853..a50bfda18e96 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -51,7 +51,7 @@ static void mlx5e_handle_tx_dim(struct mlx5e_txqsq *sq)
 	struct mlx5e_sq_stats *stats = sq->stats;
 	struct dim_sample dim_sample = {};
 
-	if (unlikely(!test_bit(MLX5E_SQ_STATE_AM, &sq->state)))
+	if (unlikely(!test_bit(MLX5E_SQ_STATE_DIM, &sq->state)))
 		return;
 
 	dim_update_sample(sq->cq.event_ctr, stats->packets, stats->bytes, &dim_sample);
@@ -63,7 +63,7 @@ static void mlx5e_handle_rx_dim(struct mlx5e_rq *rq)
 	struct mlx5e_rq_stats *stats = rq->stats;
 	struct dim_sample dim_sample = {};
 
-	if (unlikely(!test_bit(MLX5E_RQ_STATE_AM, &rq->state)))
+	if (unlikely(!test_bit(MLX5E_RQ_STATE_DIM, &rq->state)))
 		return;
 
 	dim_update_sample(rq->cq.event_ctr, stats->packets, stats->bytes, &dim_sample);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 38b32e98f3bd..1c35d721a31d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -18,6 +18,7 @@
 #include "lib/clock.h"
 #include "diag/fw_tracer.h"
 #include "mlx5_irq.h"
+#include "pci_irq.h"
 #include "devlink.h"
 #include "en_accel/ipsec.h"
 
@@ -61,9 +62,7 @@ struct mlx5_eq_table {
 	struct mlx5_irq_table	*irq_table;
 	struct mlx5_irq         **comp_irqs;
 	struct mlx5_irq         *ctrl_irq;
-#ifdef CONFIG_RFS_ACCEL
 	struct cpu_rmap		*rmap;
-#endif
 };
 
 #define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG)	    | \
@@ -637,6 +636,7 @@ static u16 async_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
 	mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err);
 	return MLX5_NUM_ASYNC_EQE;
 }
+
 static int create_async_eqs(struct mlx5_core_dev *dev)
 {
 	struct mlx5_eq_table *table = dev->priv.eq_table;
@@ -803,44 +803,28 @@ void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm)
 }
 EXPORT_SYMBOL(mlx5_eq_update_ci);
 
-static void comp_irqs_release(struct mlx5_core_dev *dev)
+static void comp_irqs_release_pci(struct mlx5_core_dev *dev)
 {
 	struct mlx5_eq_table *table = dev->priv.eq_table;
 
-	if (mlx5_core_is_sf(dev))
-		mlx5_irq_affinity_irqs_release(dev, table->comp_irqs, table->num_comp_eqs);
-	else
-		mlx5_irqs_release_vectors(table->comp_irqs, table->num_comp_eqs);
-	kfree(table->comp_irqs);
+	mlx5_irqs_release_vectors(table->comp_irqs, table->num_comp_eqs);
 }
 
-static int comp_irqs_request(struct mlx5_core_dev *dev)
+static int comp_irqs_request_pci(struct mlx5_core_dev *dev)
 {
 	struct mlx5_eq_table *table = dev->priv.eq_table;
 	const struct cpumask *prev = cpu_none_mask;
 	const struct cpumask *mask;
-	int ncomp_eqs = table->num_comp_eqs;
+	int ncomp_eqs;
 	u16 *cpus;
 	int ret;
 	int cpu;
 	int i;
 
 	ncomp_eqs = table->num_comp_eqs;
-	table->comp_irqs = kcalloc(ncomp_eqs, sizeof(*table->comp_irqs), GFP_KERNEL);
-	if (!table->comp_irqs)
-		return -ENOMEM;
-	if (mlx5_core_is_sf(dev)) {
-		ret = mlx5_irq_affinity_irqs_request_auto(dev, ncomp_eqs, table->comp_irqs);
-		if (ret < 0)
-			goto free_irqs;
-		return ret;
-	}
-
 	cpus = kcalloc(ncomp_eqs, sizeof(*cpus), GFP_KERNEL);
-	if (!cpus) {
+	if (!cpus)
 		ret = -ENOMEM;
-		goto free_irqs;
-	}
 
 	i = 0;
 	rcu_read_lock();
@@ -854,17 +838,89 @@ static int comp_irqs_request(struct mlx5_core_dev *dev)
 	}
 spread_done:
 	rcu_read_unlock();
-	ret = mlx5_irqs_request_vectors(dev, cpus, ncomp_eqs, table->comp_irqs);
+	ret = mlx5_irqs_request_vectors(dev, cpus, ncomp_eqs, table->comp_irqs, &table->rmap);
 	kfree(cpus);
-	if (ret < 0)
-		goto free_irqs;
 	return ret;
+}
+
+static void comp_irqs_release_sf(struct mlx5_core_dev *dev)
+{
+	struct mlx5_eq_table *table = dev->priv.eq_table;
+
+	mlx5_irq_affinity_irqs_release(dev, table->comp_irqs, table->num_comp_eqs);
+}
+
+static int comp_irqs_request_sf(struct mlx5_core_dev *dev)
+{
+	struct mlx5_eq_table *table = dev->priv.eq_table;
+	int ncomp_eqs = table->num_comp_eqs;
+
+	return mlx5_irq_affinity_irqs_request_auto(dev, ncomp_eqs, table->comp_irqs);
+}
+
+static void comp_irqs_release(struct mlx5_core_dev *dev)
+{
+	struct mlx5_eq_table *table = dev->priv.eq_table;
+
+	mlx5_core_is_sf(dev) ? comp_irqs_release_sf(dev) :
+			       comp_irqs_release_pci(dev);
 
-free_irqs:
 	kfree(table->comp_irqs);
+}
+
+static int comp_irqs_request(struct mlx5_core_dev *dev)
+{
+	struct mlx5_eq_table *table = dev->priv.eq_table;
+	int ncomp_eqs;
+	int ret;
+
+	ncomp_eqs = table->num_comp_eqs;
+	table->comp_irqs = kcalloc(ncomp_eqs, sizeof(*table->comp_irqs), GFP_KERNEL);
+	if (!table->comp_irqs)
+		return -ENOMEM;
+
+	ret = mlx5_core_is_sf(dev) ? comp_irqs_request_sf(dev) :
+				     comp_irqs_request_pci(dev);
+	if (ret < 0)
+		kfree(table->comp_irqs);
+
 	return ret;
 }
 
+#ifdef CONFIG_RFS_ACCEL
+static int alloc_rmap(struct mlx5_core_dev *mdev)
+{
+	struct mlx5_eq_table *eq_table = mdev->priv.eq_table;
+
+	/* rmap is a mapping between irq number and queue number.
+	 * Each irq can be assigned only to a single rmap.
+	 * Since SFs share IRQs, rmap mapping cannot function correctly
+	 * for irqs that are shared between different core/netdev RX rings.
+	 * Hence we don't allow netdev rmap for SFs.
+	 */
+	if (mlx5_core_is_sf(mdev))
+		return 0;
+
+	eq_table->rmap = alloc_irq_cpu_rmap(eq_table->num_comp_eqs);
+	if (!eq_table->rmap)
+		return -ENOMEM;
+	return 0;
+}
+
+static void free_rmap(struct mlx5_core_dev *mdev)
+{
+	struct mlx5_eq_table *eq_table = mdev->priv.eq_table;
+
+	if (eq_table->rmap) {
+		free_irq_cpu_rmap(eq_table->rmap);
+		eq_table->rmap = NULL;
+	}
+}
+#else
+static int alloc_rmap(struct mlx5_core_dev *mdev) { return 0; }
+static void free_rmap(struct mlx5_core_dev *mdev) {}
+#endif
+
 static void destroy_comp_eqs(struct mlx5_core_dev *dev)
 {
 	struct mlx5_eq_table *table = dev->priv.eq_table;
@@ -880,6 +936,7 @@ static void destroy_comp_eqs(struct mlx5_core_dev *dev)
 		kfree(eq);
 	}
 	comp_irqs_release(dev);
+	free_rmap(dev);
 }
 
 static u16 comp_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
@@ -906,9 +963,16 @@ static int create_comp_eqs(struct mlx5_core_dev *dev)
 	int err;
 	int i;
 
+	err = alloc_rmap(dev);
+	if (err)
+		return err;
+
 	ncomp_eqs = comp_irqs_request(dev);
-	if (ncomp_eqs < 0)
-		return ncomp_eqs;
+	if (ncomp_eqs < 0) {
+		err = ncomp_eqs;
+		goto err_irqs_req;
+	}
+
 	INIT_LIST_HEAD(&table->comp_eqs_list);
 	nent = comp_eq_depth_devlink_param_get(dev);
 
@@ -953,6 +1017,8 @@ clean_eq:
 	kfree(eq);
 clean:
 	destroy_comp_eqs(dev);
+err_irqs_req:
+	free_rmap(dev);
 	return err;
 }
 
@@ -1004,10 +1070,11 @@ mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector)
 
 	list_for_each_entry(eq, &table->comp_eqs_list, list) {
 		if (i++ == vector)
-			break;
+			return mlx5_irq_get_affinity_mask(eq->core.irq);
 	}
 
-	return mlx5_irq_get_affinity_mask(eq->core.irq);
+	WARN_ON_ONCE(1);
+	return NULL;
 }
 EXPORT_SYMBOL(mlx5_comp_irq_get_affinity_mask);
 
@@ -1031,55 +1098,12 @@ struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn)
 	return ERR_PTR(-ENOENT);
 }
 
-static void clear_rmap(struct mlx5_core_dev *dev)
-{
-#ifdef CONFIG_RFS_ACCEL
-	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
-
-	free_irq_cpu_rmap(eq_table->rmap);
-#endif
-}
-
-static int set_rmap(struct mlx5_core_dev *mdev)
-{
-	int err = 0;
-#ifdef CONFIG_RFS_ACCEL
-	struct mlx5_eq_table *eq_table = mdev->priv.eq_table;
-	int vecidx;
-
-	eq_table->rmap = alloc_irq_cpu_rmap(eq_table->num_comp_eqs);
-	if (!eq_table->rmap) {
-		err = -ENOMEM;
-		mlx5_core_err(mdev, "Failed to allocate cpu_rmap. err %d", err);
-		goto err_out;
-	}
-
-	for (vecidx = 0; vecidx < eq_table->num_comp_eqs; vecidx++) {
-		err = irq_cpu_rmap_add(eq_table->rmap,
-				       pci_irq_vector(mdev->pdev, vecidx));
-		if (err) {
-			mlx5_core_err(mdev, "irq_cpu_rmap_add failed. err %d",
-				      err);
-			goto err_irq_cpu_rmap_add;
-		}
-	}
-	return 0;
-
-err_irq_cpu_rmap_add:
-	clear_rmap(mdev);
-err_out:
-#endif
-	return err;
-}
-
 /* This function should only be called after mlx5_cmd_force_teardown_hca */
 void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev)
 {
 	struct mlx5_eq_table *table = dev->priv.eq_table;
 
 	mutex_lock(&table->lock); /* sync with create/destroy_async_eq */
-	if (!mlx5_core_is_sf(dev))
-		clear_rmap(dev);
 	mlx5_irq_table_destroy(dev);
 	mutex_unlock(&table->lock);
 }
@@ -1090,44 +1114,47 @@ void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev)
 #define MLX5_MAX_ASYNC_EQS 3
 #endif
 
-int mlx5_eq_table_create(struct mlx5_core_dev *dev)
+static int get_num_eqs(struct mlx5_core_dev *dev)
 {
 	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
-	int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
+	int max_dev_eqs;
+	int max_eqs_sf;
+	int num_eqs;
+
+	/* If ethernet is disabled we use just a single completion vector to
+	 * have the other vectors available for other drivers using mlx5_core. For
+	 * example, mlx5_vdpa
+	 */
+	if (!mlx5_core_is_eth_enabled(dev) && mlx5_eth_supported(dev))
+		return 1;
+
+	max_dev_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
 		      MLX5_CAP_GEN(dev, max_num_eqs) :
 		      1 << MLX5_CAP_GEN(dev, log_max_eq);
-	int max_eqs_sf;
-	int err;
 
-	eq_table->num_comp_eqs =
-		min_t(int,
-		      mlx5_irq_table_get_num_comp(eq_table->irq_table),
-		      num_eqs - MLX5_MAX_ASYNC_EQS);
+	num_eqs = min_t(int, mlx5_irq_table_get_num_comp(eq_table->irq_table),
+			max_dev_eqs - MLX5_MAX_ASYNC_EQS);
 	if (mlx5_core_is_sf(dev)) {
 		max_eqs_sf = min_t(int, MLX5_COMP_EQS_PER_SF,
 				   mlx5_irq_table_get_sfs_vec(eq_table->irq_table));
-		eq_table->num_comp_eqs = min_t(int, eq_table->num_comp_eqs,
-					       max_eqs_sf);
+		num_eqs = min_t(int, num_eqs, max_eqs_sf);
 	}
 
+	return num_eqs;
+}
+
+int mlx5_eq_table_create(struct mlx5_core_dev *dev)
+{
+	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
+	int err;
+
+	eq_table->num_comp_eqs = get_num_eqs(dev);
 	err = create_async_eqs(dev);
 	if (err) {
 		mlx5_core_err(dev, "Failed to create async EQs\n");
 		goto err_async_eqs;
 	}
 
-	if (!mlx5_core_is_sf(dev)) {
-		/* rmap is a mapping between irq number and queue number.
-		 * each irq can be assign only to a single rmap.
-		 * since SFs share IRQs, rmap mapping cannot function correctly
-		 * for irqs that are shared for different core/netdev RX rings.
-		 * Hence we don't allow netdev rmap for SFs
-		 */
-		err = set_rmap(dev);
-		if (err)
-			goto err_rmap;
-	}
-
 	err = create_comp_eqs(dev);
 	if (err) {
 		mlx5_core_err(dev, "Failed to create completion EQs\n");
@@ -1135,10 +1162,8 @@ int mlx5_eq_table_create(struct mlx5_core_dev *dev)
 	}
 
 	return 0;
+
 err_comp_eqs:
-	if (!mlx5_core_is_sf(dev))
-		clear_rmap(dev);
-err_rmap:
 	destroy_async_eqs(dev);
 err_async_eqs:
 	return err;
@@ -1146,8 +1171,6 @@ err_async_eqs:
 
 void mlx5_eq_table_destroy(struct mlx5_core_dev *dev)
 {
-	if (!mlx5_core_is_sf(dev))
-		clear_rmap(dev);
 	destroy_comp_eqs(dev);
 	destroy_async_eqs(dev);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
index 3cdcb0e0b20f..1ba03e219111 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
@@ -13,66 +13,6 @@
 #define CREATE_TRACE_POINTS
 #include "diag/bridge_tracepoint.h"
 
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE 12000
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_UNTAGGED_GRP_SIZE 16000
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM 0
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO		\
-	(MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1)
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM	\
-	(MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO + 1)
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO		\
-	(MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM +	\
-	 MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1)
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM			\
-	(MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO + 1)
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO			\
-	(MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM +		\
-	 MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1)
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM	\
-	(MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO + 1)
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO		\
-	(MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM +	\
-	 MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1)
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM			\
-	(MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO + 1)
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO			\
-	(MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM +		\
-	 MLX5_ESW_BRIDGE_INGRESS_TABLE_UNTAGGED_GRP_SIZE - 1)
-#define MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE			\
-	(MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO + 1)
-static_assert(MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE == 64000);
-
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE 16000
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_SIZE (32000 - 1)
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM 0
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO		\
-	(MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE - 1)
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM		\
-	(MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO + 1)
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO			\
-	(MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM +		\
-	 MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE - 1)
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM \
-	(MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO + 1)
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO			\
-	(MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM +		\
-	 MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_SIZE - 1)
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_FROM \
-	(MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO + 1)
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO	\
-	MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_FROM
-#define MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE			\
-	(MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO + 1)
-static_assert(MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE == 64000);
-
-#define MLX5_ESW_BRIDGE_SKIP_TABLE_SIZE 0
-
-enum {
-	MLX5_ESW_BRIDGE_LEVEL_INGRESS_TABLE,
-	MLX5_ESW_BRIDGE_LEVEL_EGRESS_TABLE,
-	MLX5_ESW_BRIDGE_LEVEL_SKIP_TABLE,
-};
-
 static const struct rhashtable_params fdb_ht_params = {
 	.key_offset = offsetof(struct mlx5_esw_bridge_fdb_entry, key),
 	.key_len = sizeof(struct mlx5_esw_bridge_fdb_key),
@@ -80,31 +20,6 @@ static const struct rhashtable_params fdb_ht_params = {
 	.automatic_shrinking = true,
 };
 
-enum {
-	MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG = BIT(0),
-};
-
-struct mlx5_esw_bridge {
-	int ifindex;
-	int refcnt;
-	struct list_head list;
-	struct mlx5_esw_bridge_offloads *br_offloads;
-
-	struct list_head fdb_list;
-	struct rhashtable fdb_ht;
-
-	struct mlx5_flow_table *egress_ft;
-	struct mlx5_flow_group *egress_vlan_fg;
-	struct mlx5_flow_group *egress_qinq_fg;
-	struct mlx5_flow_group *egress_mac_fg;
-	struct mlx5_flow_group *egress_miss_fg;
-	struct mlx5_pkt_reformat *egress_miss_pkt_reformat;
-	struct mlx5_flow_handle *egress_miss_handle;
-	unsigned long ageing_time;
-	u32 flags;
-	u16 vlan_proto;
-};
-
 static void
 mlx5_esw_bridge_fdb_offload_notify(struct net_device *dev, const unsigned char *addr, u16 vid,
 				   unsigned long val)
@@ -146,7 +61,7 @@ mlx5_esw_bridge_pkt_reformat_vlan_pop_create(struct mlx5_eswitch *esw)
 	return mlx5_packet_reformat_alloc(esw->dev, &reformat_params, MLX5_FLOW_NAMESPACE_FDB);
 }
 
-static struct mlx5_flow_table *
+struct mlx5_flow_table *
 mlx5_esw_bridge_table_create(int max_fte, u32 level, struct mlx5_eswitch *esw)
 {
 	struct mlx5_flow_table_attr ft_attr = {};
@@ -925,6 +840,10 @@ static struct mlx5_esw_bridge *mlx5_esw_bridge_create(int ifindex,
 	if (err)
 		goto err_fdb_ht;
 
+	err = mlx5_esw_bridge_mdb_init(bridge);
+	if (err)
+		goto err_mdb_ht;
+
 	INIT_LIST_HEAD(&bridge->fdb_list);
 	bridge->ifindex = ifindex;
 	bridge->refcnt = 1;
@@ -934,6 +853,8 @@ static struct mlx5_esw_bridge *mlx5_esw_bridge_create(int ifindex,
 
 	return bridge;
 
+err_mdb_ht:
+	rhashtable_destroy(&bridge->fdb_ht);
 err_fdb_ht:
 	mlx5_esw_bridge_egress_table_cleanup(bridge);
 err_egress_tbl:
@@ -953,7 +874,9 @@ static void mlx5_esw_bridge_put(struct mlx5_esw_bridge_offloads *br_offloads,
 		return;
 
 	mlx5_esw_bridge_egress_table_cleanup(bridge);
+	mlx5_esw_bridge_mcast_disable(bridge);
 	list_del(&bridge->list);
+	mlx5_esw_bridge_mdb_cleanup(bridge);
 	rhashtable_destroy(&bridge->fdb_ht);
 	kvfree(bridge);
 
@@ -993,7 +916,7 @@ static unsigned long mlx5_esw_bridge_port_key_from_data(u16 vport_num, u16 esw_o
 	return vport_num | (unsigned long)esw_owner_vhca_id << sizeof(vport_num) * BITS_PER_BYTE;
 }
 
-static unsigned long mlx5_esw_bridge_port_key(struct mlx5_esw_bridge_port *port)
+unsigned long mlx5_esw_bridge_port_key(struct mlx5_esw_bridge_port *port)
 {
 	return mlx5_esw_bridge_port_key_from_data(port->vport_num, port->esw_owner_vhca_id);
 }
@@ -1018,6 +941,19 @@ static void mlx5_esw_bridge_port_erase(struct mlx5_esw_bridge_port *port,
 	xa_erase(&br_offloads->ports, mlx5_esw_bridge_port_key(port));
 }
 
+static struct mlx5_esw_bridge *
+mlx5_esw_bridge_from_port_lookup(u16 vport_num, u16 esw_owner_vhca_id,
+				 struct mlx5_esw_bridge_offloads *br_offloads)
+{
+	struct mlx5_esw_bridge_port *port;
+
+	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
+	if (!port)
+		return NULL;
+
+	return port->bridge;
+}
+
 static void mlx5_esw_bridge_fdb_entry_refresh(struct mlx5_esw_bridge_fdb_entry *entry)
 {
 	trace_mlx5_esw_bridge_fdb_entry_refresh(entry);
@@ -1166,8 +1102,21 @@ mlx5_esw_bridge_vlan_push_mark_cleanup(struct mlx5_esw_bridge_vlan *vlan, struct
 }
 
 static int
-mlx5_esw_bridge_vlan_push_pop_create(u16 vlan_proto, u16 flags, struct mlx5_esw_bridge_vlan *vlan,
-				     struct mlx5_eswitch *esw)
+mlx5_esw_bridge_vlan_push_pop_fhs_create(u16 vlan_proto, struct mlx5_esw_bridge_port *port,
+					 struct mlx5_esw_bridge_vlan *vlan)
+{
+	return mlx5_esw_bridge_vlan_mcast_init(vlan_proto, port, vlan);
+}
+
+static void
+mlx5_esw_bridge_vlan_push_pop_fhs_cleanup(struct mlx5_esw_bridge_vlan *vlan)
+{
+	mlx5_esw_bridge_vlan_mcast_cleanup(vlan);
+}
+
+static int
+mlx5_esw_bridge_vlan_push_pop_create(u16 vlan_proto, u16 flags, struct mlx5_esw_bridge_port *port,
+				     struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw)
 {
 	int err;
 
@@ -1185,10 +1134,16 @@ mlx5_esw_bridge_vlan_push_pop_create(u16 vlan_proto, u16 flags, struct mlx5_esw_
 		err = mlx5_esw_bridge_vlan_pop_create(vlan, esw);
 		if (err)
 			goto err_vlan_pop;
+
+		err = mlx5_esw_bridge_vlan_push_pop_fhs_create(vlan_proto, port, vlan);
+		if (err)
+			goto err_vlan_pop_fhs;
 	}
 
 	return 0;
 
+err_vlan_pop_fhs:
+	mlx5_esw_bridge_vlan_pop_cleanup(vlan, esw);
 err_vlan_pop:
 	if (vlan->pkt_mod_hdr_push_mark)
 		mlx5_esw_bridge_vlan_push_mark_cleanup(vlan, esw);
@@ -1213,7 +1168,7 @@ mlx5_esw_bridge_vlan_create(u16 vlan_proto, u16 vid, u16 flags, struct mlx5_esw_
 	vlan->flags = flags;
 	INIT_LIST_HEAD(&vlan->fdb_list);
 
-	err = mlx5_esw_bridge_vlan_push_pop_create(vlan_proto, flags, vlan, esw);
+	err = mlx5_esw_bridge_vlan_push_pop_create(vlan_proto, flags, port, vlan, esw);
 	if (err)
 		goto err_vlan_push_pop;
 
@@ -1225,6 +1180,8 @@ mlx5_esw_bridge_vlan_create(u16 vlan_proto, u16 vid, u16 flags, struct mlx5_esw_
 	return vlan;
 
 err_xa_insert:
+	if (vlan->mcast_handle)
+		mlx5_esw_bridge_vlan_push_pop_fhs_cleanup(vlan);
 	if (vlan->pkt_reformat_pop)
 		mlx5_esw_bridge_vlan_pop_cleanup(vlan, esw);
 	if (vlan->pkt_mod_hdr_push_mark)
@@ -1242,7 +1199,8 @@ static void mlx5_esw_bridge_vlan_erase(struct mlx5_esw_bridge_port *port,
 	xa_erase(&port->vlans, vlan->vid);
 }
 
-static void mlx5_esw_bridge_vlan_flush(struct mlx5_esw_bridge_vlan *vlan,
+static void mlx5_esw_bridge_vlan_flush(struct mlx5_esw_bridge_port *port,
+				       struct mlx5_esw_bridge_vlan *vlan,
 				       struct mlx5_esw_bridge *bridge)
 {
 	struct mlx5_eswitch *esw = bridge->br_offloads->esw;
@@ -1250,7 +1208,10 @@ static void mlx5_esw_bridge_vlan_flush(struct mlx5_esw_bridge_vlan *vlan,
 
 	list_for_each_entry_safe(entry, tmp, &vlan->fdb_list, vlan_list)
 		mlx5_esw_bridge_fdb_entry_notify_and_cleanup(entry, bridge);
+	mlx5_esw_bridge_port_mdb_vlan_flush(port, vlan);
 
+	if (vlan->mcast_handle)
+		mlx5_esw_bridge_vlan_push_pop_fhs_cleanup(vlan);
 	if (vlan->pkt_reformat_pop)
 		mlx5_esw_bridge_vlan_pop_cleanup(vlan, esw);
 	if (vlan->pkt_mod_hdr_push_mark)
@@ -1264,7 +1225,7 @@ static void mlx5_esw_bridge_vlan_cleanup(struct mlx5_esw_bridge_port *port,
 					 struct mlx5_esw_bridge *bridge)
 {
 	trace_mlx5_esw_bridge_vlan_cleanup(vlan);
-	mlx5_esw_bridge_vlan_flush(vlan, bridge);
+	mlx5_esw_bridge_vlan_flush(port, vlan, bridge);
 	mlx5_esw_bridge_vlan_erase(port, vlan);
 	kvfree(vlan);
 }
@@ -1288,9 +1249,9 @@ static int mlx5_esw_bridge_port_vlans_recreate(struct mlx5_esw_bridge_port *port
 	int err;
 
 	xa_for_each(&port->vlans, i, vlan) {
-		mlx5_esw_bridge_vlan_flush(vlan, bridge);
-		err = mlx5_esw_bridge_vlan_push_pop_create(bridge->vlan_proto, vlan->flags, vlan,
-							   br_offloads->esw);
+		mlx5_esw_bridge_vlan_flush(port, vlan, bridge);
+		err = mlx5_esw_bridge_vlan_push_pop_create(bridge->vlan_proto, vlan->flags, port,
+							   vlan, br_offloads->esw);
 		if (err) {
 			esw_warn(br_offloads->esw->dev,
 				 "Failed to create VLAN=%u(proto=%x) push/pop actions (vport=%u,err=%d)\n",
@@ -1473,33 +1434,32 @@ err_ingress_fc_create:
 int mlx5_esw_bridge_ageing_time_set(u16 vport_num, u16 esw_owner_vhca_id, unsigned long ageing_time,
 				    struct mlx5_esw_bridge_offloads *br_offloads)
 {
-	struct mlx5_esw_bridge_port *port;
+	struct mlx5_esw_bridge *bridge;
 
-	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
-	if (!port)
+	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
+	if (!bridge)
 		return -EINVAL;
 
-	port->bridge->ageing_time = clock_t_to_jiffies(ageing_time);
+	bridge->ageing_time = clock_t_to_jiffies(ageing_time);
 	return 0;
 }
 
 int mlx5_esw_bridge_vlan_filtering_set(u16 vport_num, u16 esw_owner_vhca_id, bool enable,
 				       struct mlx5_esw_bridge_offloads *br_offloads)
 {
-	struct mlx5_esw_bridge_port *port;
 	struct mlx5_esw_bridge *bridge;
 	bool filtering;
 
-	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
-	if (!port)
+	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
+	if (!bridge)
 		return -EINVAL;
 
-	bridge = port->bridge;
 	filtering = bridge->flags & MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG;
 	if (filtering == enable)
 		return 0;
 
 	mlx5_esw_bridge_fdb_flush(bridge);
+	mlx5_esw_bridge_mdb_flush(bridge);
 	if (enable)
 		bridge->flags |= MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG;
 	else
@@ -1511,15 +1471,13 @@ int mlx5_esw_bridge_vlan_filtering_set(u16 vport_num, u16 esw_owner_vhca_id, boo
 int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 proto,
 				   struct mlx5_esw_bridge_offloads *br_offloads)
 {
-	struct mlx5_esw_bridge_port *port;
 	struct mlx5_esw_bridge *bridge;
 
-	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id,
-					   br_offloads);
-	if (!port)
+	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id,
+						  br_offloads);
+	if (!bridge)
 		return -EINVAL;
 
-	bridge = port->bridge;
 	if (bridge->vlan_proto == proto)
 		return 0;
 	if (proto != ETH_P_8021Q && proto != ETH_P_8021AD) {
@@ -1528,12 +1486,43 @@ int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 pro
 	}
 
 	mlx5_esw_bridge_fdb_flush(bridge);
+	mlx5_esw_bridge_mdb_flush(bridge);
 	bridge->vlan_proto = proto;
 	mlx5_esw_bridge_vlans_recreate(bridge);
 
 	return 0;
 }
 
+int mlx5_esw_bridge_mcast_set(u16 vport_num, u16 esw_owner_vhca_id, bool enable,
+			      struct mlx5_esw_bridge_offloads *br_offloads)
+{
+	struct mlx5_eswitch *esw = br_offloads->esw;
+	struct mlx5_esw_bridge *bridge;
+	int err = 0;
+	bool mcast;
+
+	if (!(MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_multi_path_any_table) ||
+	      MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_multi_path_any_table_limit_regc)) ||
+	    !MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_uplink_hairpin) ||
+	    !MLX5_CAP_ESW_FLOWTABLE_FDB((esw)->dev, ignore_flow_level))
+		return -EOPNOTSUPP;
+
+	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
+	if (!bridge)
+		return -EINVAL;
+
+	mcast = bridge->flags & MLX5_ESW_BRIDGE_MCAST_FLAG;
+	if (mcast == enable)
+		return 0;
+
+	if (enable)
+		err = mlx5_esw_bridge_mcast_enable(bridge);
+	else
+		mlx5_esw_bridge_mcast_disable(bridge);
+
+	return err;
+}
+
 static int mlx5_esw_bridge_vport_init(u16 vport_num, u16 esw_owner_vhca_id, u16 flags,
 				      struct mlx5_esw_bridge_offloads *br_offloads,
 				      struct mlx5_esw_bridge *bridge)
@@ -1551,6 +1540,15 @@ static int mlx5_esw_bridge_vport_init(u16 vport_num, u16 esw_owner_vhca_id, u16
 	port->bridge = bridge;
 	port->flags |= flags;
 	xa_init(&port->vlans);
+
+	err = mlx5_esw_bridge_port_mcast_init(port);
+	if (err) {
+		esw_warn(esw->dev,
+			 "Failed to initialize port multicast (vport=%u,esw_owner_vhca_id=%u,err=%d)\n",
+			 port->vport_num, port->esw_owner_vhca_id, err);
+		goto err_port_mcast;
+	}
+
 	err = mlx5_esw_bridge_port_insert(port, br_offloads);
 	if (err) {
 		esw_warn(esw->dev,
@@ -1563,6 +1561,8 @@ static int mlx5_esw_bridge_vport_init(u16 vport_num, u16 esw_owner_vhca_id, u16
 	return 0;
 
 err_port_insert:
+	mlx5_esw_bridge_port_mcast_cleanup(port);
+err_port_mcast:
 	kvfree(port);
 	return err;
 }
@@ -1580,6 +1580,7 @@ static int mlx5_esw_bridge_vport_cleanup(struct mlx5_esw_bridge_offloads *br_off
 
 	trace_mlx5_esw_bridge_vport_cleanup(port);
 	mlx5_esw_bridge_port_vlans_flush(port, bridge);
+	mlx5_esw_bridge_port_mcast_cleanup(port);
 	mlx5_esw_bridge_port_erase(port, br_offloads);
 	kvfree(port);
 	mlx5_esw_bridge_put(br_offloads, bridge);
@@ -1711,14 +1712,12 @@ void mlx5_esw_bridge_fdb_update_used(struct net_device *dev, u16 vport_num, u16
 				     struct switchdev_notifier_fdb_info *fdb_info)
 {
 	struct mlx5_esw_bridge_fdb_entry *entry;
-	struct mlx5_esw_bridge_port *port;
 	struct mlx5_esw_bridge *bridge;
 
-	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
-	if (!port)
+	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
+	if (!bridge)
 		return;
 
-	bridge = port->bridge;
 	entry = mlx5_esw_bridge_fdb_lookup(bridge, fdb_info->addr, fdb_info->vid);
 	if (!entry) {
 		esw_debug(br_offloads->esw->dev,
@@ -1765,14 +1764,12 @@ void mlx5_esw_bridge_fdb_remove(struct net_device *dev, u16 vport_num, u16 esw_o
 {
 	struct mlx5_eswitch *esw = br_offloads->esw;
 	struct mlx5_esw_bridge_fdb_entry *entry;
-	struct mlx5_esw_bridge_port *port;
 	struct mlx5_esw_bridge *bridge;
 
-	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
-	if (!port)
+	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
+	if (!bridge)
 		return;
 
-	bridge = port->bridge;
 	entry = mlx5_esw_bridge_fdb_lookup(bridge, fdb_info->addr, fdb_info->vid);
 	if (!entry) {
 		esw_debug(esw->dev,
@@ -1806,6 +1803,64 @@ void mlx5_esw_bridge_update(struct mlx5_esw_bridge_offloads *br_offloads)
 	}
 }
 
+int mlx5_esw_bridge_port_mdb_add(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
+				 const unsigned char *addr, u16 vid,
+				 struct mlx5_esw_bridge_offloads *br_offloads,
+				 struct netlink_ext_ack *extack)
+{
+	struct mlx5_esw_bridge_vlan *vlan;
+	struct mlx5_esw_bridge_port *port;
+	struct mlx5_esw_bridge *bridge;
+	int err;
+
+	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
+	if (!port) {
+		esw_warn(br_offloads->esw->dev,
+			 "Failed to lookup bridge port to add MDB (MAC=%pM,vport=%u)\n",
+			 addr, vport_num);
+		NL_SET_ERR_MSG_FMT_MOD(extack,
+				       "Failed to lookup bridge port to add MDB (MAC=%pM,vport=%u)\n",
+				       addr, vport_num);
+		return -EINVAL;
+	}
+
+	bridge = port->bridge;
+	if (bridge->flags & MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG && vid) {
+		vlan = mlx5_esw_bridge_vlan_lookup(vid, port);
+		if (!vlan) {
+			esw_warn(br_offloads->esw->dev,
+				 "Failed to lookup bridge port vlan metadata to create MDB (MAC=%pM,vid=%u,vport=%u)\n",
+				 addr, vid, vport_num);
+			NL_SET_ERR_MSG_FMT_MOD(extack,
+					       "Failed to lookup bridge port vlan metadata to create MDB (MAC=%pM,vid=%u,vport=%u)\n",
+					       addr, vid, vport_num);
+			return -EINVAL;
+		}
+	}
+
+	err = mlx5_esw_bridge_port_mdb_attach(dev, port, addr, vid);
+	if (err) {
+		NL_SET_ERR_MSG_FMT_MOD(extack, "Failed to add MDB (MAC=%pM,vid=%u,vport=%u)\n",
+				       addr, vid, vport_num);
+		return err;
+	}
+
+	return 0;
+}
+
+void mlx5_esw_bridge_port_mdb_del(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
+				  const unsigned char *addr, u16 vid,
+				  struct mlx5_esw_bridge_offloads *br_offloads)
+{
+	struct mlx5_esw_bridge_port *port;
+
+	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
+	if (!port)
+		return;
+
+	mlx5_esw_bridge_port_mdb_detach(dev, port, addr, vid);
+}
+
 static void mlx5_esw_bridge_flush(struct mlx5_esw_bridge_offloads *br_offloads)
 {
 	struct mlx5_esw_bridge_port *port;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h
index 10851a515bca..a9dd18c73d6a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h
@@ -25,12 +25,19 @@ struct mlx5_esw_bridge_offloads {
 	struct delayed_work update_work;
 
 	struct mlx5_flow_table *ingress_ft;
+	struct mlx5_flow_group *ingress_igmp_fg;
+	struct mlx5_flow_group *ingress_mld_fg;
 	struct mlx5_flow_group *ingress_vlan_fg;
 	struct mlx5_flow_group *ingress_vlan_filter_fg;
 	struct mlx5_flow_group *ingress_qinq_fg;
 	struct mlx5_flow_group *ingress_qinq_filter_fg;
 	struct mlx5_flow_group *ingress_mac_fg;
 
+	struct mlx5_flow_handle *igmp_handle;
+	struct mlx5_flow_handle *mld_query_handle;
+	struct mlx5_flow_handle *mld_report_handle;
+	struct mlx5_flow_handle *mld_done_handle;
+
 	struct mlx5_flow_table *skip_ft;
 };
 
@@ -64,10 +71,20 @@ int mlx5_esw_bridge_vlan_filtering_set(u16 vport_num, u16 esw_owner_vhca_id, boo
 				       struct mlx5_esw_bridge_offloads *br_offloads);
 int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 proto,
 				   struct mlx5_esw_bridge_offloads *br_offloads);
+int mlx5_esw_bridge_mcast_set(u16 vport_num, u16 esw_owner_vhca_id, bool enable,
+			      struct mlx5_esw_bridge_offloads *br_offloads);
 int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, u16 flags,
 				  struct mlx5_esw_bridge_offloads *br_offloads,
 				  struct netlink_ext_ack *extack);
 void mlx5_esw_bridge_port_vlan_del(u16 vport_num, u16 esw_owner_vhca_id, u16 vid,
 				   struct mlx5_esw_bridge_offloads *br_offloads);
 
+int mlx5_esw_bridge_port_mdb_add(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
+				 const unsigned char *addr, u16 vid,
+				 struct mlx5_esw_bridge_offloads *br_offloads,
+				 struct netlink_ext_ack *extack);
+void mlx5_esw_bridge_port_mdb_del(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
+				  const unsigned char *addr, u16 vid,
+				  struct mlx5_esw_bridge_offloads *br_offloads);
+
 #endif /* __MLX5_ESW_BRIDGE_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_mcast.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_mcast.c
new file mode 100644
index 000000000000..2eae594a5e80
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_mcast.c
@@ -0,0 +1,1126 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#include "lib/devcom.h"
+#include "bridge.h"
+#include "eswitch.h"
+#include "bridge_priv.h"
+#include "diag/bridge_tracepoint.h"
+
+static const struct rhashtable_params mdb_ht_params = {
+	.key_offset = offsetof(struct mlx5_esw_bridge_mdb_entry, key),
+	.key_len = sizeof(struct mlx5_esw_bridge_mdb_key),
+	.head_offset = offsetof(struct mlx5_esw_bridge_mdb_entry, ht_node),
+	.automatic_shrinking = true,
+};
+
+int mlx5_esw_bridge_mdb_init(struct mlx5_esw_bridge *bridge)
+{
+	INIT_LIST_HEAD(&bridge->mdb_list);
+	return rhashtable_init(&bridge->mdb_ht, &mdb_ht_params);
+}
+
+void mlx5_esw_bridge_mdb_cleanup(struct mlx5_esw_bridge *bridge)
+{
+	rhashtable_destroy(&bridge->mdb_ht);
+}
+
+static struct mlx5_esw_bridge_port *
+mlx5_esw_bridge_mdb_port_lookup(struct mlx5_esw_bridge_port *port,
+				struct mlx5_esw_bridge_mdb_entry *entry)
+{
+	return xa_load(&entry->ports, mlx5_esw_bridge_port_key(port));
+}
+
+static int mlx5_esw_bridge_mdb_port_insert(struct mlx5_esw_bridge_port *port,
+					   struct mlx5_esw_bridge_mdb_entry *entry)
+{
+	int err = xa_insert(&entry->ports, mlx5_esw_bridge_port_key(port), port, GFP_KERNEL);
+
+	if (!err)
+		entry->num_ports++;
+	return err;
+}
+
+static void mlx5_esw_bridge_mdb_port_remove(struct mlx5_esw_bridge_port *port,
+					    struct mlx5_esw_bridge_mdb_entry *entry)
+{
+	xa_erase(&entry->ports, mlx5_esw_bridge_port_key(port));
+	entry->num_ports--;
+}
+
+static struct mlx5_flow_handle *
+mlx5_esw_bridge_mdb_flow_create(u16 esw_owner_vhca_id, struct mlx5_esw_bridge_mdb_entry *entry,
+				struct mlx5_esw_bridge *bridge)
+{
+	struct mlx5_flow_act flow_act = {
+		.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
+		.flags = FLOW_ACT_NO_APPEND | FLOW_ACT_IGNORE_FLOW_LEVEL,
+	};
+	int num_dests = entry->num_ports, i = 0;
+	struct mlx5_flow_destination *dests;
+	struct mlx5_esw_bridge_port *port;
+	struct mlx5_flow_spec *rule_spec;
+	struct mlx5_flow_handle *handle;
+	u8 *dmac_v, *dmac_c;
+	unsigned long idx;
+
+	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
+	if (!rule_spec)
+		return ERR_PTR(-ENOMEM);
+
+	dests = kvcalloc(num_dests, sizeof(*dests), GFP_KERNEL);
+	if (!dests) {
+		kvfree(rule_spec);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	xa_for_each(&entry->ports, idx, port) {
+		dests[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+		dests[i].ft = port->mcast.ft;
+		i++;
+	}
+
+	rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	dmac_v = MLX5_ADDR_OF(fte_match_param, rule_spec->match_value, outer_headers.dmac_47_16);
+	ether_addr_copy(dmac_v, entry->key.addr);
+	dmac_c = MLX5_ADDR_OF(fte_match_param, rule_spec->match_criteria, outer_headers.dmac_47_16);
+	eth_broadcast_addr(dmac_c);
+
+	if (entry->key.vid) {
+		if (bridge->vlan_proto == ETH_P_8021Q) {
+			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
+					 outer_headers.cvlan_tag);
+			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value,
+					 outer_headers.cvlan_tag);
+		} else if (bridge->vlan_proto == ETH_P_8021AD) {
+			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
+					 outer_headers.svlan_tag);
+			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value,
+					 outer_headers.svlan_tag);
+		}
+		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
+				 outer_headers.first_vid);
+		MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.first_vid,
+			 entry->key.vid);
+	}
+
+	handle = mlx5_add_flow_rules(bridge->egress_ft, rule_spec, &flow_act, dests, num_dests);
+
+	kvfree(dests);
+	kvfree(rule_spec);
+	return handle;
+}
+
+static int
+mlx5_esw_bridge_port_mdb_offload(struct mlx5_esw_bridge_port *port,
+				 struct mlx5_esw_bridge_mdb_entry *entry)
+{
+	struct mlx5_flow_handle *handle;
+
+	handle = mlx5_esw_bridge_mdb_flow_create(port->esw_owner_vhca_id, entry, port->bridge);
+	if (entry->egress_handle) {
+		mlx5_del_flow_rules(entry->egress_handle);
+		entry->egress_handle = NULL;
+	}
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+
+	entry->egress_handle = handle;
+	return 0;
+}
+
+static struct mlx5_esw_bridge_mdb_entry *
+mlx5_esw_bridge_mdb_lookup(struct mlx5_esw_bridge *bridge,
+			   const unsigned char *addr, u16 vid)
+{
+	struct mlx5_esw_bridge_mdb_key key = {};
+
+	ether_addr_copy(key.addr, addr);
+	key.vid = vid;
+	return rhashtable_lookup_fast(&bridge->mdb_ht, &key, mdb_ht_params);
+}
+
+static struct mlx5_esw_bridge_mdb_entry *
+mlx5_esw_bridge_port_mdb_entry_init(struct mlx5_esw_bridge_port *port,
+				    const unsigned char *addr, u16 vid)
+{
+	struct mlx5_esw_bridge *bridge = port->bridge;
+	struct mlx5_esw_bridge_mdb_entry *entry;
+	int err;
+
+	entry = kvzalloc(sizeof(*entry), GFP_KERNEL);
+	if (!entry)
+		return ERR_PTR(-ENOMEM);
+
+	ether_addr_copy(entry->key.addr, addr);
+	entry->key.vid = vid;
+	xa_init(&entry->ports);
+	err = rhashtable_insert_fast(&bridge->mdb_ht, &entry->ht_node, mdb_ht_params);
+	if (err)
+		goto err_ht_insert;
+
+	list_add(&entry->list, &bridge->mdb_list);
+
+	return entry;
+
+err_ht_insert:
+	xa_destroy(&entry->ports);
+	kvfree(entry);
+	return ERR_PTR(err);
+}
+
+static void mlx5_esw_bridge_port_mdb_entry_cleanup(struct mlx5_esw_bridge *bridge,
+						   struct mlx5_esw_bridge_mdb_entry *entry)
+{
+	if (entry->egress_handle)
+		mlx5_del_flow_rules(entry->egress_handle);
+	list_del(&entry->list);
+	rhashtable_remove_fast(&bridge->mdb_ht, &entry->ht_node, mdb_ht_params);
+	xa_destroy(&entry->ports);
+	kvfree(entry);
+}
+
+int mlx5_esw_bridge_port_mdb_attach(struct net_device *dev, struct mlx5_esw_bridge_port *port,
+				    const unsigned char *addr, u16 vid)
+{
+	struct mlx5_esw_bridge *bridge = port->bridge;
+	struct mlx5_esw_bridge_mdb_entry *entry;
+	int err;
+
+	if (!(bridge->flags & MLX5_ESW_BRIDGE_MCAST_FLAG))
+		return -EOPNOTSUPP;
+
+	entry = mlx5_esw_bridge_mdb_lookup(bridge, addr, vid);
+	if (entry) {
+		if (mlx5_esw_bridge_mdb_port_lookup(port, entry)) {
+			esw_warn(bridge->br_offloads->esw->dev, "MDB attach entry is already attached to port (MAC=%pM,vid=%u,vport=%u)\n",
+				 addr, vid, port->vport_num);
+			return 0;
+		}
+	} else {
+		entry = mlx5_esw_bridge_port_mdb_entry_init(port, addr, vid);
+		if (IS_ERR(entry)) {
+			err = PTR_ERR(entry);
+			esw_warn(bridge->br_offloads->esw->dev, "MDB attach failed to init entry (MAC=%pM,vid=%u,vport=%u,err=%d)\n",
+				 addr, vid, port->vport_num, err);
+			return err;
+		}
+	}
+
+	err = mlx5_esw_bridge_mdb_port_insert(port, entry);
+	if (err) {
+		if (!entry->num_ports)
+			mlx5_esw_bridge_port_mdb_entry_cleanup(bridge, entry); /* new mdb entry */
+		esw_warn(bridge->br_offloads->esw->dev,
+			 "MDB attach failed to insert port (MAC=%pM,vid=%u,vport=%u,err=%d)\n",
+			 addr, vid, port->vport_num, err);
+		return err;
+	}
+
+	err = mlx5_esw_bridge_port_mdb_offload(port, entry);
+	if (err)
+		/* Single mdb can be used by multiple ports, so just log the
+		 * error and continue.
+		 */
+		esw_warn(bridge->br_offloads->esw->dev, "MDB attach failed to offload (MAC=%pM,vid=%u,vport=%u,err=%d)\n",
+			 addr, vid, port->vport_num, err);
+
+	trace_mlx5_esw_bridge_port_mdb_attach(dev, entry);
+	return 0;
+}
+
+static void mlx5_esw_bridge_port_mdb_entry_detach(struct mlx5_esw_bridge_port *port,
+						  struct mlx5_esw_bridge_mdb_entry *entry)
+{
+	struct mlx5_esw_bridge *bridge = port->bridge;
+	int err;
+
+	mlx5_esw_bridge_mdb_port_remove(port, entry);
+	if (!entry->num_ports) {
+		mlx5_esw_bridge_port_mdb_entry_cleanup(bridge, entry);
+		return;
+	}
+
+	err = mlx5_esw_bridge_port_mdb_offload(port, entry);
+	if (err)
+		/* Single mdb can be used by multiple ports, so just log the
+		 * error and continue.
+		 */
+		esw_warn(bridge->br_offloads->esw->dev, "MDB detach failed to offload (MAC=%pM,vid=%u,vport=%u)\n",
+			 entry->key.addr, entry->key.vid, port->vport_num);
+}
+
+void mlx5_esw_bridge_port_mdb_detach(struct net_device *dev, struct mlx5_esw_bridge_port *port,
+				     const unsigned char *addr, u16 vid)
+{
+	struct mlx5_esw_bridge *bridge = port->bridge;
+	struct mlx5_esw_bridge_mdb_entry *entry;
+
+	entry = mlx5_esw_bridge_mdb_lookup(bridge, addr, vid);
+	if (!entry) {
+		esw_debug(bridge->br_offloads->esw->dev,
+			  "MDB detach entry not found (MAC=%pM,vid=%u,vport=%u)\n",
+			  addr, vid, port->vport_num);
+		return;
+	}
+
+	if (!mlx5_esw_bridge_mdb_port_lookup(port, entry)) {
+		esw_debug(bridge->br_offloads->esw->dev,
+			  "MDB detach entry not attached to the port (MAC=%pM,vid=%u,vport=%u)\n",
+			  addr, vid, port->vport_num);
+		return;
+	}
+
+	trace_mlx5_esw_bridge_port_mdb_detach(dev, entry);
+	mlx5_esw_bridge_port_mdb_entry_detach(port, entry);
+}
+
+void mlx5_esw_bridge_port_mdb_vlan_flush(struct mlx5_esw_bridge_port *port,
+					 struct mlx5_esw_bridge_vlan *vlan)
+{
+	struct mlx5_esw_bridge *bridge = port->bridge;
+	struct mlx5_esw_bridge_mdb_entry *entry, *tmp;
+
+	list_for_each_entry_safe(entry, tmp, &bridge->mdb_list, list)
+		if (entry->key.vid == vlan->vid && mlx5_esw_bridge_mdb_port_lookup(port, entry))
+			mlx5_esw_bridge_port_mdb_entry_detach(port, entry);
+}
+
+static void mlx5_esw_bridge_port_mdb_flush(struct mlx5_esw_bridge_port *port)
+{
+	struct mlx5_esw_bridge *bridge = port->bridge;
+	struct mlx5_esw_bridge_mdb_entry *entry, *tmp;
+
+	list_for_each_entry_safe(entry, tmp, &bridge->mdb_list, list)
+		if (mlx5_esw_bridge_mdb_port_lookup(port, entry))
+			mlx5_esw_bridge_port_mdb_entry_detach(port, entry);
+}
+
+void mlx5_esw_bridge_mdb_flush(struct mlx5_esw_bridge *bridge)
+{
+	struct mlx5_esw_bridge_mdb_entry *entry, *tmp;
+
+	list_for_each_entry_safe(entry, tmp, &bridge->mdb_list, list)
+		mlx5_esw_bridge_port_mdb_entry_cleanup(bridge, entry);
+}
+static int mlx5_esw_bridge_port_mcast_fts_init(struct mlx5_esw_bridge_port *port,
+					       struct mlx5_esw_bridge *bridge)
+{
+	struct mlx5_eswitch *esw = bridge->br_offloads->esw;
+	struct mlx5_flow_table *mcast_ft;
+
+	mcast_ft = mlx5_esw_bridge_table_create(MLX5_ESW_BRIDGE_MCAST_TABLE_SIZE,
+						MLX5_ESW_BRIDGE_LEVEL_MCAST_TABLE,
+						esw);
+	if (IS_ERR(mcast_ft))
+		return PTR_ERR(mcast_ft);
+
+	port->mcast.ft = mcast_ft;
+	return 0;
+}
+
+static void mlx5_esw_bridge_port_mcast_fts_cleanup(struct mlx5_esw_bridge_port *port)
+{
+	if (port->mcast.ft)
+		mlx5_destroy_flow_table(port->mcast.ft);
+	port->mcast.ft = NULL;
+}
+
+static struct mlx5_flow_group *
+mlx5_esw_bridge_mcast_filter_fg_create(struct mlx5_eswitch *esw,
+				       struct mlx5_flow_table *mcast_ft)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *fg;
+	u32 *in, *match;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return ERR_PTR(-ENOMEM);
+
+	MLX5_SET(create_flow_group_in, in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS_2);
+	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
+
+	MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0,
+		 mlx5_eswitch_get_vport_metadata_mask());
+
+	MLX5_SET(create_flow_group_in, in, start_flow_index,
+		 MLX5_ESW_BRIDGE_MCAST_TABLE_FILTER_GRP_IDX_FROM);
+	MLX5_SET(create_flow_group_in, in, end_flow_index,
+		 MLX5_ESW_BRIDGE_MCAST_TABLE_FILTER_GRP_IDX_TO);
+
+	fg = mlx5_create_flow_group(mcast_ft, in);
+	kvfree(in);
+	if (IS_ERR(fg))
+		esw_warn(esw->dev,
+			 "Failed to create filter flow group for bridge mcast table (err=%pe)\n",
+			 fg);
+
+	return fg;
+}
+
+static struct mlx5_flow_group *
+mlx5_esw_bridge_mcast_vlan_proto_fg_create(unsigned int from, unsigned int to, u16 vlan_proto,
+					   struct mlx5_eswitch *esw,
+					   struct mlx5_flow_table *mcast_ft)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *fg;
+	u32 *in, *match;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return ERR_PTR(-ENOMEM);
+
+	MLX5_SET(create_flow_group_in, in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
+	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
+
+	if (vlan_proto == ETH_P_8021Q)
+		MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag);
+	else if (vlan_proto == ETH_P_8021AD)
+		MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.svlan_tag);
+	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.first_vid);
+
+	MLX5_SET(create_flow_group_in, in, start_flow_index, from);
+	MLX5_SET(create_flow_group_in, in, end_flow_index, to);
+
+	fg = mlx5_create_flow_group(mcast_ft, in);
+	kvfree(in);
+	if (IS_ERR(fg))
+		esw_warn(esw->dev,
+			 "Failed to create VLAN(proto=%x) flow group for bridge mcast table (err=%pe)\n",
+			 vlan_proto, fg);
+
+	return fg;
+}
+
+static struct mlx5_flow_group *
+mlx5_esw_bridge_mcast_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *mcast_ft)
+{
+	unsigned int from = MLX5_ESW_BRIDGE_MCAST_TABLE_VLAN_GRP_IDX_FROM;
+	unsigned int to = MLX5_ESW_BRIDGE_MCAST_TABLE_VLAN_GRP_IDX_TO;
+
+	return mlx5_esw_bridge_mcast_vlan_proto_fg_create(from, to, ETH_P_8021Q, esw, mcast_ft);
+}
+
+static struct mlx5_flow_group *
+mlx5_esw_bridge_mcast_qinq_fg_create(struct mlx5_eswitch *esw,
+				     struct mlx5_flow_table *mcast_ft)
+{
+	unsigned int from = MLX5_ESW_BRIDGE_MCAST_TABLE_QINQ_GRP_IDX_FROM;
+	unsigned int to = MLX5_ESW_BRIDGE_MCAST_TABLE_QINQ_GRP_IDX_TO;
+
+	return mlx5_esw_bridge_mcast_vlan_proto_fg_create(from, to, ETH_P_8021AD, esw, mcast_ft);
+}
+
+static struct mlx5_flow_group *
+mlx5_esw_bridge_mcast_fwd_fg_create(struct mlx5_eswitch *esw,
+				    struct mlx5_flow_table *mcast_ft)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *fg;
+	u32 *in;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return ERR_PTR(-ENOMEM);
+
+	MLX5_SET(create_flow_group_in, in, start_flow_index,
+		 MLX5_ESW_BRIDGE_MCAST_TABLE_FWD_GRP_IDX_FROM);
+	MLX5_SET(create_flow_group_in, in, end_flow_index,
+		 MLX5_ESW_BRIDGE_MCAST_TABLE_FWD_GRP_IDX_TO);
+
+	fg = mlx5_create_flow_group(mcast_ft, in);
+	kvfree(in);
+	if (IS_ERR(fg))
+		esw_warn(esw->dev,
+			 "Failed to create forward flow group for bridge mcast table (err=%pe)\n",
+			 fg);
+
+	return fg;
+}
+
+static int mlx5_esw_bridge_port_mcast_fgs_init(struct mlx5_esw_bridge_port *port)
+{
+	struct mlx5_flow_group *fwd_fg, *qinq_fg, *vlan_fg, *filter_fg;
+	struct mlx5_eswitch *esw = port->bridge->br_offloads->esw;
+	struct mlx5_flow_table *mcast_ft = port->mcast.ft;
+	int err;
+
+	filter_fg = mlx5_esw_bridge_mcast_filter_fg_create(esw, mcast_ft);
+	if (IS_ERR(filter_fg))
+		return PTR_ERR(filter_fg);
+
+	vlan_fg = mlx5_esw_bridge_mcast_vlan_fg_create(esw, mcast_ft);
+	if (IS_ERR(vlan_fg)) {
+		err = PTR_ERR(vlan_fg);
+		goto err_vlan_fg;
+	}
+
+	qinq_fg = mlx5_esw_bridge_mcast_qinq_fg_create(esw, mcast_ft);
+	if (IS_ERR(qinq_fg)) {
+		err = PTR_ERR(qinq_fg);
+		goto err_qinq_fg;
+	}
+
+	fwd_fg = mlx5_esw_bridge_mcast_fwd_fg_create(esw, mcast_ft);
+	if (IS_ERR(fwd_fg)) {
+		err = PTR_ERR(fwd_fg);
+		goto err_fwd_fg;
+	}
+
+	port->mcast.filter_fg = filter_fg;
+	port->mcast.vlan_fg = vlan_fg;
+	port->mcast.qinq_fg = qinq_fg;
+	port->mcast.fwd_fg = fwd_fg;
+
+	return 0;
+
+err_fwd_fg:
+	mlx5_destroy_flow_group(qinq_fg);
+err_qinq_fg:
+	mlx5_destroy_flow_group(vlan_fg);
+err_vlan_fg:
+	mlx5_destroy_flow_group(filter_fg);
+	return err;
+}
+
+static void mlx5_esw_bridge_port_mcast_fgs_cleanup(struct mlx5_esw_bridge_port *port)
+{
+	if (port->mcast.fwd_fg)
+		mlx5_destroy_flow_group(port->mcast.fwd_fg);
+	port->mcast.fwd_fg = NULL;
+	if (port->mcast.qinq_fg)
+		mlx5_destroy_flow_group(port->mcast.qinq_fg);
+	port->mcast.qinq_fg = NULL;
+	if (port->mcast.vlan_fg)
+		mlx5_destroy_flow_group(port->mcast.vlan_fg);
+	port->mcast.vlan_fg = NULL;
+	if (port->mcast.filter_fg)
+		mlx5_destroy_flow_group(port->mcast.filter_fg);
+	port->mcast.filter_fg = NULL;
+}
+
+static struct mlx5_flow_handle *
+mlx5_esw_bridge_mcast_flow_with_esw_create(struct mlx5_esw_bridge_port *port,
+					   struct mlx5_eswitch *esw)
+{
+	struct mlx5_flow_act flow_act = {
+		.action = MLX5_FLOW_CONTEXT_ACTION_DROP,
+		.flags = FLOW_ACT_NO_APPEND,
+	};
+	struct mlx5_flow_spec *rule_spec;
+	struct mlx5_flow_handle *handle;
+
+	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
+	if (!rule_spec)
+		return ERR_PTR(-ENOMEM);
+
+	rule_spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
+
+	MLX5_SET(fte_match_param, rule_spec->match_criteria,
+		 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
+	MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_2.metadata_reg_c_0,
+		 mlx5_eswitch_get_vport_metadata_for_match(esw, port->vport_num));
+
+	handle = mlx5_add_flow_rules(port->mcast.ft, rule_spec, &flow_act, NULL, 0);
+
+	kvfree(rule_spec);
+	return handle;
+}
+
+static struct mlx5_flow_handle *
+mlx5_esw_bridge_mcast_filter_flow_create(struct mlx5_esw_bridge_port *port)
+{
+	return mlx5_esw_bridge_mcast_flow_with_esw_create(port, port->bridge->br_offloads->esw);
+}
+
+static struct mlx5_flow_handle *
+mlx5_esw_bridge_mcast_filter_flow_peer_create(struct mlx5_esw_bridge_port *port)
+{
+	struct mlx5_devcom *devcom = port->bridge->br_offloads->esw->dev->priv.devcom;
+	static struct mlx5_flow_handle *handle;
+	struct mlx5_eswitch *peer_esw;
+
+	peer_esw = mlx5_devcom_get_peer_data(devcom, MLX5_DEVCOM_ESW_OFFLOADS);
+	if (!peer_esw)
+		return ERR_PTR(-ENODEV);
+
+	handle = mlx5_esw_bridge_mcast_flow_with_esw_create(port, peer_esw);
+
+	mlx5_devcom_release_peer_data(devcom, MLX5_DEVCOM_ESW_OFFLOADS);
+	return handle;
+}
+
+static struct mlx5_flow_handle *
+mlx5_esw_bridge_mcast_vlan_flow_create(u16 vlan_proto, struct mlx5_esw_bridge_port *port,
+				       struct mlx5_esw_bridge_vlan *vlan)
+{
+	struct mlx5_flow_act flow_act = {
+		.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
+		.flags = FLOW_ACT_NO_APPEND,
+	};
+	struct mlx5_flow_destination dest = {
+		.type = MLX5_FLOW_DESTINATION_TYPE_VPORT,
+		.vport.num = port->vport_num,
+	};
+	struct mlx5_esw_bridge *bridge = port->bridge;
+	struct mlx5_flow_spec *rule_spec;
+	struct mlx5_flow_handle *handle;
+
+	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
+	if (!rule_spec)
+		return ERR_PTR(-ENOMEM);
+
+	if (MLX5_CAP_ESW_FLOWTABLE(bridge->br_offloads->esw->dev, flow_source) &&
+	    port->vport_num == MLX5_VPORT_UPLINK)
+		rule_spec->flow_context.flow_source =
+			MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
+	rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+
+	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
+	flow_act.pkt_reformat = vlan->pkt_reformat_pop;
+
+	if (vlan_proto == ETH_P_8021Q) {
+		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
+				 outer_headers.cvlan_tag);
+		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value,
+				 outer_headers.cvlan_tag);
+	} else if (vlan_proto == ETH_P_8021AD) {
+		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
+				 outer_headers.svlan_tag);
+		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value,
+				 outer_headers.svlan_tag);
+	}
+	MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, outer_headers.first_vid);
+	MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.first_vid, vlan->vid);
+
+	if (MLX5_CAP_ESW(bridge->br_offloads->esw->dev, merged_eswitch)) {
+		dest.vport.flags = MLX5_FLOW_DEST_VPORT_VHCA_ID;
+		dest.vport.vhca_id = port->esw_owner_vhca_id;
+	}
+	handle = mlx5_add_flow_rules(port->mcast.ft, rule_spec, &flow_act, &dest, 1);
+
+	kvfree(rule_spec);
+	return handle;
+}
+
+int mlx5_esw_bridge_vlan_mcast_init(u16 vlan_proto, struct mlx5_esw_bridge_port *port,
+				    struct mlx5_esw_bridge_vlan *vlan)
+{
+	struct mlx5_flow_handle *handle;
+
+	if (!(port->bridge->flags & MLX5_ESW_BRIDGE_MCAST_FLAG))
+		return 0;
+
+	handle = mlx5_esw_bridge_mcast_vlan_flow_create(vlan_proto, port, vlan);
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+
+	vlan->mcast_handle = handle;
+	return 0;
+}
+
+void mlx5_esw_bridge_vlan_mcast_cleanup(struct mlx5_esw_bridge_vlan *vlan)
+{
+	if (vlan->mcast_handle)
+		mlx5_del_flow_rules(vlan->mcast_handle);
+	vlan->mcast_handle = NULL;
+}
+
+static struct mlx5_flow_handle *
+mlx5_esw_bridge_mcast_fwd_flow_create(struct mlx5_esw_bridge_port *port)
+{
+	struct mlx5_flow_act flow_act = {
+		.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
+		.flags = FLOW_ACT_NO_APPEND,
+	};
+	struct mlx5_flow_destination dest = {
+		.type = MLX5_FLOW_DESTINATION_TYPE_VPORT,
+		.vport.num = port->vport_num,
+	};
+	struct mlx5_esw_bridge *bridge = port->bridge;
+	struct mlx5_flow_spec *rule_spec;
+	struct mlx5_flow_handle *handle;
+
+	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
+	if (!rule_spec)
+		return ERR_PTR(-ENOMEM);
+
+	if (MLX5_CAP_ESW_FLOWTABLE(bridge->br_offloads->esw->dev, flow_source) &&
+	    port->vport_num == MLX5_VPORT_UPLINK)
+		rule_spec->flow_context.flow_source =
+			MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
+
+	if (MLX5_CAP_ESW(bridge->br_offloads->esw->dev, merged_eswitch)) {
+		dest.vport.flags = MLX5_FLOW_DEST_VPORT_VHCA_ID;
+		dest.vport.vhca_id = port->esw_owner_vhca_id;
+	}
+	handle = mlx5_add_flow_rules(port->mcast.ft, rule_spec, &flow_act, &dest, 1);
+
+	kvfree(rule_spec);
+	return handle;
+}
+
+static int mlx5_esw_bridge_port_mcast_fhs_init(struct mlx5_esw_bridge_port *port)
+{
+	struct mlx5_flow_handle *filter_handle, *fwd_handle;
+	struct mlx5_esw_bridge_vlan *vlan, *failed;
+	unsigned long index;
+	int err;
+
+
+	filter_handle = (port->flags & MLX5_ESW_BRIDGE_PORT_FLAG_PEER) ?
+		mlx5_esw_bridge_mcast_filter_flow_peer_create(port) :
+		mlx5_esw_bridge_mcast_filter_flow_create(port);
+	if (IS_ERR(filter_handle))
+		return PTR_ERR(filter_handle);
+
+	fwd_handle = mlx5_esw_bridge_mcast_fwd_flow_create(port);
+	if (IS_ERR(fwd_handle)) {
+		err = PTR_ERR(fwd_handle);
+		goto err_fwd;
+	}
+
+	xa_for_each(&port->vlans, index, vlan) {
+		err = mlx5_esw_bridge_vlan_mcast_init(port->bridge->vlan_proto, port, vlan);
+		if (err) {
+			failed = vlan;
+			goto err_vlan;
+		}
+	}
+
+	port->mcast.filter_handle = filter_handle;
+	port->mcast.fwd_handle = fwd_handle;
+
+	return 0;
+
+err_vlan:
+	xa_for_each(&port->vlans, index, vlan) {
+		if (vlan == failed)
+			break;
+
+		mlx5_esw_bridge_vlan_mcast_cleanup(vlan);
+	}
+	mlx5_del_flow_rules(fwd_handle);
+err_fwd:
+	mlx5_del_flow_rules(filter_handle);
+	return err;
+}
+
+static void mlx5_esw_bridge_port_mcast_fhs_cleanup(struct mlx5_esw_bridge_port *port)
+{
+	struct mlx5_esw_bridge_vlan *vlan;
+	unsigned long index;
+
+	xa_for_each(&port->vlans, index, vlan)
+		mlx5_esw_bridge_vlan_mcast_cleanup(vlan);
+
+	if (port->mcast.fwd_handle)
+		mlx5_del_flow_rules(port->mcast.fwd_handle);
+	port->mcast.fwd_handle = NULL;
+	if (port->mcast.filter_handle)
+		mlx5_del_flow_rules(port->mcast.filter_handle);
+	port->mcast.filter_handle = NULL;
+}
+
+int mlx5_esw_bridge_port_mcast_init(struct mlx5_esw_bridge_port *port)
+{
+	struct mlx5_esw_bridge *bridge = port->bridge;
+	int err;
+
+	if (!(bridge->flags & MLX5_ESW_BRIDGE_MCAST_FLAG))
+		return 0;
+
+	err = mlx5_esw_bridge_port_mcast_fts_init(port, bridge);
+	if (err)
+		return err;
+
+	err = mlx5_esw_bridge_port_mcast_fgs_init(port);
+	if (err)
+		goto err_fgs;
+
+	err = mlx5_esw_bridge_port_mcast_fhs_init(port);
+	if (err)
+		goto err_fhs;
+	return err;
+
+err_fhs:
+	mlx5_esw_bridge_port_mcast_fgs_cleanup(port);
+err_fgs:
+	mlx5_esw_bridge_port_mcast_fts_cleanup(port);
+	return err;
+}
+
+void mlx5_esw_bridge_port_mcast_cleanup(struct mlx5_esw_bridge_port *port)
+{
+	mlx5_esw_bridge_port_mdb_flush(port);
+	mlx5_esw_bridge_port_mcast_fhs_cleanup(port);
+	mlx5_esw_bridge_port_mcast_fgs_cleanup(port);
+	mlx5_esw_bridge_port_mcast_fts_cleanup(port);
+}
+
+static struct mlx5_flow_group *
+mlx5_esw_bridge_ingress_igmp_fg_create(struct mlx5_eswitch *esw,
+				       struct mlx5_flow_table *ingress_ft)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *fg;
+	u32 *in, *match;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return ERR_PTR(-ENOMEM);
+
+	MLX5_SET(create_flow_group_in, in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
+	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
+
+	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.ip_version);
+	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.ip_protocol);
+
+	MLX5_SET(create_flow_group_in, in, start_flow_index,
+		 MLX5_ESW_BRIDGE_INGRESS_TABLE_IGMP_GRP_IDX_FROM);
+	MLX5_SET(create_flow_group_in, in, end_flow_index,
+		 MLX5_ESW_BRIDGE_INGRESS_TABLE_IGMP_GRP_IDX_TO);
+
+	fg = mlx5_create_flow_group(ingress_ft, in);
+	kvfree(in);
+	if (IS_ERR(fg))
+		esw_warn(esw->dev,
+			 "Failed to create IGMP flow group for bridge ingress table (err=%pe)\n",
+			 fg);
+
+	return fg;
+}
+
+static struct mlx5_flow_group *
+mlx5_esw_bridge_ingress_mld_fg_create(struct mlx5_eswitch *esw,
+				      struct mlx5_flow_table *ingress_ft)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *fg;
+	u32 *in, *match;
+
+	if (!(MLX5_CAP_GEN(esw->dev, flex_parser_protocols) & MLX5_FLEX_PROTO_ICMPV6)) {
+		esw_warn(esw->dev,
+			 "Can't create MLD flow group due to missing hardware ICMPv6 parsing support\n");
+		return NULL;
+	}
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return ERR_PTR(-ENOMEM);
+
+	MLX5_SET(create_flow_group_in, in, match_criteria_enable,
+		 MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_3);
+	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
+
+	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.ip_version);
+	MLX5_SET_TO_ONES(fte_match_param, match, misc_parameters_3.icmpv6_type);
+
+	MLX5_SET(create_flow_group_in, in, start_flow_index,
+		 MLX5_ESW_BRIDGE_INGRESS_TABLE_MLD_GRP_IDX_FROM);
+	MLX5_SET(create_flow_group_in, in, end_flow_index,
+		 MLX5_ESW_BRIDGE_INGRESS_TABLE_MLD_GRP_IDX_TO);
+
+	fg = mlx5_create_flow_group(ingress_ft, in);
+	kvfree(in);
+	if (IS_ERR(fg))
+		esw_warn(esw->dev,
+			 "Failed to create MLD flow group for bridge ingress table (err=%pe)\n",
+			 fg);
+
+	return fg;
+}
+
+static int
+mlx5_esw_bridge_ingress_mcast_fgs_init(struct mlx5_esw_bridge_offloads *br_offloads)
+{
+	struct mlx5_flow_table *ingress_ft = br_offloads->ingress_ft;
+	struct mlx5_eswitch *esw = br_offloads->esw;
+	struct mlx5_flow_group *igmp_fg, *mld_fg;
+
+	igmp_fg = mlx5_esw_bridge_ingress_igmp_fg_create(esw, ingress_ft);
+	if (IS_ERR(igmp_fg))
+		return PTR_ERR(igmp_fg);
+
+	mld_fg = mlx5_esw_bridge_ingress_mld_fg_create(esw, ingress_ft);
+	if (IS_ERR(mld_fg)) {
+		mlx5_destroy_flow_group(igmp_fg);
+		return PTR_ERR(mld_fg);
+	}
+
+	br_offloads->ingress_igmp_fg = igmp_fg;
+	br_offloads->ingress_mld_fg = mld_fg;
+	return 0;
+}
+
+static void
+mlx5_esw_bridge_ingress_mcast_fgs_cleanup(struct mlx5_esw_bridge_offloads *br_offloads)
+{
+	if (br_offloads->ingress_mld_fg)
+		mlx5_destroy_flow_group(br_offloads->ingress_mld_fg);
+	br_offloads->ingress_mld_fg = NULL;
+	if (br_offloads->ingress_igmp_fg)
+		mlx5_destroy_flow_group(br_offloads->ingress_igmp_fg);
+	br_offloads->ingress_igmp_fg = NULL;
+}
+
+static struct mlx5_flow_handle *
+mlx5_esw_bridge_ingress_igmp_fh_create(struct mlx5_flow_table *ingress_ft,
+				       struct mlx5_flow_table *skip_ft)
+{
+	struct mlx5_flow_destination dest = {
+		.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE,
+		.ft = skip_ft,
+	};
+	struct mlx5_flow_act flow_act = {
+		.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
+		.flags = FLOW_ACT_NO_APPEND,
+	};
+	struct mlx5_flow_spec *rule_spec;
+	struct mlx5_flow_handle *handle;
+
+	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
+	if (!rule_spec)
+		return ERR_PTR(-ENOMEM);
+
+	rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+
+	MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, outer_headers.ip_version);
+	MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.ip_version, 4);
+	MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, outer_headers.ip_protocol);
+	MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.ip_protocol, IPPROTO_IGMP);
+
+	handle = mlx5_add_flow_rules(ingress_ft, rule_spec, &flow_act, &dest, 1);
+
+	kvfree(rule_spec);
+	return handle;
+}
+
+static struct mlx5_flow_handle *
+mlx5_esw_bridge_ingress_mld_fh_create(u8 type, struct mlx5_flow_table *ingress_ft,
+				      struct mlx5_flow_table *skip_ft)
+{
+	struct mlx5_flow_destination dest = {
+		.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE,
+		.ft = skip_ft,
+	};
+	struct mlx5_flow_act flow_act = {
+		.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
+		.flags = FLOW_ACT_NO_APPEND,
+	};
+	struct mlx5_flow_spec *rule_spec;
+	struct mlx5_flow_handle *handle;
+
+	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
+	if (!rule_spec)
+		return ERR_PTR(-ENOMEM);
+
+	rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_3;
+
+	MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, outer_headers.ip_version);
+	MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.ip_version, 6);
+	MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, misc_parameters_3.icmpv6_type);
+	MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_3.icmpv6_type, type);
+
+	handle = mlx5_add_flow_rules(ingress_ft, rule_spec, &flow_act, &dest, 1);
+
+	kvfree(rule_spec);
+	return handle;
+}
+
+static int
+mlx5_esw_bridge_ingress_mcast_fhs_create(struct mlx5_esw_bridge_offloads *br_offloads)
+{
+	struct mlx5_flow_handle *igmp_handle, *mld_query_handle, *mld_report_handle,
+		*mld_done_handle;
+	struct mlx5_flow_table *ingress_ft = br_offloads->ingress_ft,
+		*skip_ft = br_offloads->skip_ft;
+	int err;
+
+	igmp_handle = mlx5_esw_bridge_ingress_igmp_fh_create(ingress_ft, skip_ft);
+	if (IS_ERR(igmp_handle))
+		return PTR_ERR(igmp_handle);
+
+	if (br_offloads->ingress_mld_fg) {
+		mld_query_handle = mlx5_esw_bridge_ingress_mld_fh_create(ICMPV6_MGM_QUERY,
+									 ingress_ft,
+									 skip_ft);
+		if (IS_ERR(mld_query_handle)) {
+			err = PTR_ERR(mld_query_handle);
+			goto err_mld_query;
+		}
+
+		mld_report_handle = mlx5_esw_bridge_ingress_mld_fh_create(ICMPV6_MGM_REPORT,
+									  ingress_ft,
+									  skip_ft);
+		if (IS_ERR(mld_report_handle)) {
+			err = PTR_ERR(mld_report_handle);
+			goto err_mld_report;
+		}
+
+		mld_done_handle = mlx5_esw_bridge_ingress_mld_fh_create(ICMPV6_MGM_REDUCTION,
+									ingress_ft,
+									skip_ft);
+		if (IS_ERR(mld_done_handle)) {
+			err = PTR_ERR(mld_done_handle);
+			goto err_mld_done;
+		}
+	} else {
+		mld_query_handle = NULL;
+		mld_report_handle = NULL;
+		mld_done_handle = NULL;
+	}
+
+	br_offloads->igmp_handle = igmp_handle;
+	br_offloads->mld_query_handle = mld_query_handle;
+	br_offloads->mld_report_handle = mld_report_handle;
+	br_offloads->mld_done_handle = mld_done_handle;
+
+	return 0;
+
+err_mld_done:
+	mlx5_del_flow_rules(mld_report_handle);
+err_mld_report:
+	mlx5_del_flow_rules(mld_query_handle);
+err_mld_query:
+	mlx5_del_flow_rules(igmp_handle);
+	return err;
+}
+
+static void
+mlx5_esw_bridge_ingress_mcast_fhs_cleanup(struct mlx5_esw_bridge_offloads *br_offloads)
+{
+	if (br_offloads->mld_done_handle)
+		mlx5_del_flow_rules(br_offloads->mld_done_handle);
+	br_offloads->mld_done_handle = NULL;
+	if (br_offloads->mld_report_handle)
+		mlx5_del_flow_rules(br_offloads->mld_report_handle);
+	br_offloads->mld_report_handle = NULL;
+	if (br_offloads->mld_query_handle)
+		mlx5_del_flow_rules(br_offloads->mld_query_handle);
+	br_offloads->mld_query_handle = NULL;
+	if (br_offloads->igmp_handle)
+		mlx5_del_flow_rules(br_offloads->igmp_handle);
+	br_offloads->igmp_handle = NULL;
+}
+
+static int mlx5_esw_brige_mcast_init(struct mlx5_esw_bridge *bridge)
+{
+	struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads;
+	struct mlx5_esw_bridge_port *port, *failed;
+	unsigned long i;
+	int err;
+
+	xa_for_each(&br_offloads->ports, i, port) {
+		if (port->bridge != bridge)
+			continue;
+
+		err = mlx5_esw_bridge_port_mcast_init(port);
+		if (err) {
+			failed = port;
+			goto err_port;
+		}
+	}
+	return 0;
+
+err_port:
+	xa_for_each(&br_offloads->ports, i, port) {
+		if (port == failed)
+			break;
+		if (port->bridge != bridge)
+			continue;
+
+		mlx5_esw_bridge_port_mcast_cleanup(port);
+	}
+	return err;
+}
+
+static void mlx5_esw_brige_mcast_cleanup(struct mlx5_esw_bridge *bridge)
+{
+	struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads;
+	struct mlx5_esw_bridge_port *port;
+	unsigned long i;
+
+	xa_for_each(&br_offloads->ports, i, port) {
+		if (port->bridge != bridge)
+			continue;
+
+		mlx5_esw_bridge_port_mcast_cleanup(port);
+	}
+}
+
+static int mlx5_esw_brige_mcast_global_enable(struct mlx5_esw_bridge_offloads *br_offloads)
+{
+	int err;
+
+	if (br_offloads->ingress_igmp_fg)
+		return 0; /* already enabled by another bridge */
+
+	err = mlx5_esw_bridge_ingress_mcast_fgs_init(br_offloads);
+	if (err) {
+		esw_warn(br_offloads->esw->dev,
+			 "Failed to create global multicast flow groups (err=%d)\n",
+			 err);
+		return err;
+	}
+
+	err = mlx5_esw_bridge_ingress_mcast_fhs_create(br_offloads);
+	if (err) {
+		esw_warn(br_offloads->esw->dev,
+			 "Failed to create global multicast flows (err=%d)\n",
+			 err);
+		goto err_fhs;
+	}
+
+	return 0;
+
+err_fhs:
+	mlx5_esw_bridge_ingress_mcast_fgs_cleanup(br_offloads);
+	return err;
+}
+
+static void mlx5_esw_brige_mcast_global_disable(struct mlx5_esw_bridge_offloads *br_offloads)
+{
+	struct mlx5_esw_bridge *br;
+
+	list_for_each_entry(br, &br_offloads->bridges, list) {
+		/* Ingress table is global, so only disable snooping when all
+		 * bridges on esw have multicast disabled.
+		 */
+		if (br->flags & MLX5_ESW_BRIDGE_MCAST_FLAG)
+			return;
+	}
+
+	mlx5_esw_bridge_ingress_mcast_fhs_cleanup(br_offloads);
+	mlx5_esw_bridge_ingress_mcast_fgs_cleanup(br_offloads);
+}
+
+int mlx5_esw_bridge_mcast_enable(struct mlx5_esw_bridge *bridge)
+{
+	int err;
+
+	err = mlx5_esw_brige_mcast_global_enable(bridge->br_offloads);
+	if (err)
+		return err;
+
+	bridge->flags |= MLX5_ESW_BRIDGE_MCAST_FLAG;
+
+	err = mlx5_esw_brige_mcast_init(bridge);
+	if (err) {
+		esw_warn(bridge->br_offloads->esw->dev, "Failed to enable multicast (err=%d)\n",
+			 err);
+		bridge->flags &= ~MLX5_ESW_BRIDGE_MCAST_FLAG;
+		mlx5_esw_brige_mcast_global_disable(bridge->br_offloads);
+	}
+	return err;
+}
+
+void mlx5_esw_bridge_mcast_disable(struct mlx5_esw_bridge *bridge)
+{
+	mlx5_esw_brige_mcast_cleanup(bridge);
+	bridge->flags &= ~MLX5_ESW_BRIDGE_MCAST_FLAG;
+	mlx5_esw_brige_mcast_global_disable(bridge->br_offloads);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_priv.h
index 878311fe950a..c9595801bdb4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_priv.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_priv.h
@@ -12,11 +12,124 @@
 #include <linux/xarray.h>
 #include "fs_core.h"
 
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_IGMP_GRP_SIZE 1
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MLD_GRP_SIZE 3
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE 131072
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_UNTAGGED_GRP_SIZE			\
+	(524288 - MLX5_ESW_BRIDGE_INGRESS_TABLE_IGMP_GRP_SIZE -		\
+	 MLX5_ESW_BRIDGE_INGRESS_TABLE_MLD_GRP_SIZE)
+
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_IGMP_GRP_IDX_FROM 0
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_IGMP_GRP_IDX_TO		\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_IGMP_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MLD_GRP_IDX_FROM	\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_IGMP_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MLD_GRP_IDX_TO		\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_MLD_GRP_IDX_FROM +	\
+	 MLX5_ESW_BRIDGE_INGRESS_TABLE_MLD_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM			\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_MLD_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO			\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM +		\
+	 MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM	\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO		\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM +	\
+	 MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM			\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO			\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM +		\
+	 MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM	\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO		\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM +	\
+	 MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM			\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO			\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM +		\
+	 MLX5_ESW_BRIDGE_INGRESS_TABLE_UNTAGGED_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE			\
+	(MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO + 1)
+static_assert(MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE == 1048576);
+
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE 131072
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_SIZE (262144 - 1)
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM 0
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO		\
+	(MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM		\
+	(MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO			\
+	(MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM +		\
+	 MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM \
+	(MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO			\
+	(MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM +		\
+	 MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_FROM \
+	(MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO	\
+	MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_FROM
+#define MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE			\
+	(MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO + 1)
+static_assert(MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE == 524288);
+
+#define MLX5_ESW_BRIDGE_SKIP_TABLE_SIZE 0
+
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_FILTER_GRP_SIZE 1
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_FWD_GRP_SIZE 1
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_VLAN_GRP_SIZE 4095
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_QINQ_GRP_SIZE MLX5_ESW_BRIDGE_MCAST_TABLE_VLAN_GRP_SIZE
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_FILTER_GRP_IDX_FROM 0
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_FILTER_GRP_IDX_TO		\
+	(MLX5_ESW_BRIDGE_MCAST_TABLE_FILTER_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_VLAN_GRP_IDX_FROM		\
+	(MLX5_ESW_BRIDGE_MCAST_TABLE_FILTER_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_VLAN_GRP_IDX_TO			\
+	(MLX5_ESW_BRIDGE_MCAST_TABLE_VLAN_GRP_IDX_FROM +		\
+	 MLX5_ESW_BRIDGE_MCAST_TABLE_VLAN_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_QINQ_GRP_IDX_FROM		\
+	(MLX5_ESW_BRIDGE_MCAST_TABLE_VLAN_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_QINQ_GRP_IDX_TO			\
+	(MLX5_ESW_BRIDGE_MCAST_TABLE_QINQ_GRP_IDX_FROM +		\
+	 MLX5_ESW_BRIDGE_MCAST_TABLE_QINQ_GRP_SIZE - 1)
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_FWD_GRP_IDX_FROM		\
+	(MLX5_ESW_BRIDGE_MCAST_TABLE_QINQ_GRP_IDX_TO + 1)
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_FWD_GRP_IDX_TO			\
+	(MLX5_ESW_BRIDGE_MCAST_TABLE_FWD_GRP_IDX_FROM +			\
+	 MLX5_ESW_BRIDGE_MCAST_TABLE_FWD_GRP_SIZE - 1)
+
+#define MLX5_ESW_BRIDGE_MCAST_TABLE_SIZE			\
+	(MLX5_ESW_BRIDGE_MCAST_TABLE_FWD_GRP_IDX_TO + 1)
+static_assert(MLX5_ESW_BRIDGE_MCAST_TABLE_SIZE == 8192);
+
+enum {
+	MLX5_ESW_BRIDGE_LEVEL_INGRESS_TABLE,
+	MLX5_ESW_BRIDGE_LEVEL_EGRESS_TABLE,
+	MLX5_ESW_BRIDGE_LEVEL_MCAST_TABLE,
+	MLX5_ESW_BRIDGE_LEVEL_SKIP_TABLE,
+};
+
+enum {
+	MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG = BIT(0),
+	MLX5_ESW_BRIDGE_MCAST_FLAG = BIT(1),
+};
+
 struct mlx5_esw_bridge_fdb_key {
 	unsigned char addr[ETH_ALEN];
 	u16 vid;
 };
 
+struct mlx5_esw_bridge_mdb_key {
+	unsigned char addr[ETH_ALEN];
+	u16 vid;
+};
+
 enum {
 	MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER = BIT(0),
 	MLX5_ESW_BRIDGE_FLAG_PEER = BIT(1),
@@ -43,6 +156,16 @@ struct mlx5_esw_bridge_fdb_entry {
 	struct mlx5_flow_handle *filter_handle;
 };
 
+struct mlx5_esw_bridge_mdb_entry {
+	struct mlx5_esw_bridge_mdb_key key;
+	struct rhash_head ht_node;
+	struct list_head list;
+	struct xarray ports;
+	int num_ports;
+
+	struct mlx5_flow_handle *egress_handle;
+};
+
 struct mlx5_esw_bridge_vlan {
 	u16 vid;
 	u16 flags;
@@ -50,6 +173,7 @@ struct mlx5_esw_bridge_vlan {
 	struct mlx5_pkt_reformat *pkt_reformat_push;
 	struct mlx5_pkt_reformat *pkt_reformat_pop;
 	struct mlx5_modify_hdr *pkt_mod_hdr_push_mark;
+	struct mlx5_flow_handle *mcast_handle;
 };
 
 struct mlx5_esw_bridge_port {
@@ -58,6 +182,63 @@ struct mlx5_esw_bridge_port {
 	u16 flags;
 	struct mlx5_esw_bridge *bridge;
 	struct xarray vlans;
+	struct {
+		struct mlx5_flow_table *ft;
+		struct mlx5_flow_group *filter_fg;
+		struct mlx5_flow_group *vlan_fg;
+		struct mlx5_flow_group *qinq_fg;
+		struct mlx5_flow_group *fwd_fg;
+
+		struct mlx5_flow_handle *filter_handle;
+		struct mlx5_flow_handle *fwd_handle;
+	} mcast;
 };
 
+struct mlx5_esw_bridge {
+	int ifindex;
+	int refcnt;
+	struct list_head list;
+	struct mlx5_esw_bridge_offloads *br_offloads;
+
+	struct list_head fdb_list;
+	struct rhashtable fdb_ht;
+
+	struct list_head mdb_list;
+	struct rhashtable mdb_ht;
+
+	struct mlx5_flow_table *egress_ft;
+	struct mlx5_flow_group *egress_vlan_fg;
+	struct mlx5_flow_group *egress_qinq_fg;
+	struct mlx5_flow_group *egress_mac_fg;
+	struct mlx5_flow_group *egress_miss_fg;
+	struct mlx5_pkt_reformat *egress_miss_pkt_reformat;
+	struct mlx5_flow_handle *egress_miss_handle;
+	unsigned long ageing_time;
+	u32 flags;
+	u16 vlan_proto;
+};
+
+struct mlx5_flow_table *mlx5_esw_bridge_table_create(int max_fte, u32 level,
+						     struct mlx5_eswitch *esw);
+unsigned long mlx5_esw_bridge_port_key(struct mlx5_esw_bridge_port *port);
+
+int mlx5_esw_bridge_port_mcast_init(struct mlx5_esw_bridge_port *port);
+void mlx5_esw_bridge_port_mcast_cleanup(struct mlx5_esw_bridge_port *port);
+int mlx5_esw_bridge_vlan_mcast_init(u16 vlan_proto, struct mlx5_esw_bridge_port *port,
+				    struct mlx5_esw_bridge_vlan *vlan);
+void mlx5_esw_bridge_vlan_mcast_cleanup(struct mlx5_esw_bridge_vlan *vlan);
+
+int mlx5_esw_bridge_mcast_enable(struct mlx5_esw_bridge *bridge);
+void mlx5_esw_bridge_mcast_disable(struct mlx5_esw_bridge *bridge);
+
+int mlx5_esw_bridge_mdb_init(struct mlx5_esw_bridge *bridge);
+void mlx5_esw_bridge_mdb_cleanup(struct mlx5_esw_bridge *bridge);
+int mlx5_esw_bridge_port_mdb_attach(struct net_device *dev, struct mlx5_esw_bridge_port *port,
+				    const unsigned char *addr, u16 vid);
+void mlx5_esw_bridge_port_mdb_detach(struct net_device *dev, struct mlx5_esw_bridge_port *port,
+				     const unsigned char *addr, u16 vid);
+void mlx5_esw_bridge_port_mdb_vlan_flush(struct mlx5_esw_bridge_port *port,
+					 struct mlx5_esw_bridge_vlan *vlan);
+void mlx5_esw_bridge_mdb_flush(struct mlx5_esw_bridge *bridge);
+
 #endif /* _MLX5_ESW_BRIDGE_PRIVATE_ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c
deleted file mode 100644
index 3d0bbcca1cb9..000000000000
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c
+++ /dev/null
@@ -1,198 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
-/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
-
-#include <linux/debugfs.h>
-#include "eswitch.h"
-
-enum vnic_diag_counter {
-	MLX5_VNIC_DIAG_TOTAL_Q_UNDER_PROCESSOR_HANDLE,
-	MLX5_VNIC_DIAG_SEND_QUEUE_PRIORITY_UPDATE_FLOW,
-	MLX5_VNIC_DIAG_COMP_EQ_OVERRUN,
-	MLX5_VNIC_DIAG_ASYNC_EQ_OVERRUN,
-	MLX5_VNIC_DIAG_CQ_OVERRUN,
-	MLX5_VNIC_DIAG_INVALID_COMMAND,
-	MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND,
-	MLX5_VNIC_DIAG_RX_STEERING_DISCARD,
-};
-
-static int mlx5_esw_query_vnic_diag(struct mlx5_vport *vport, enum vnic_diag_counter counter,
-				    u64 *val)
-{
-	u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {};
-	struct mlx5_core_dev *dev = vport->dev;
-	u16 vport_num = vport->vport;
-	void *vnic_diag_out;
-	int err;
-
-	MLX5_SET(query_vnic_env_in, in, opcode, MLX5_CMD_OP_QUERY_VNIC_ENV);
-	MLX5_SET(query_vnic_env_in, in, vport_number, vport_num);
-	if (!mlx5_esw_is_manager_vport(dev->priv.eswitch, vport_num))
-		MLX5_SET(query_vnic_env_in, in, other_vport, 1);
-
-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-	if (err)
-		return err;
-
-	vnic_diag_out = MLX5_ADDR_OF(query_vnic_env_out, out, vport_env);
-	switch (counter) {
-	case MLX5_VNIC_DIAG_TOTAL_Q_UNDER_PROCESSOR_HANDLE:
-		*val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, total_error_queues);
-		break;
-	case MLX5_VNIC_DIAG_SEND_QUEUE_PRIORITY_UPDATE_FLOW:
-		*val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out,
-				send_queue_priority_update_flow);
-		break;
-	case MLX5_VNIC_DIAG_COMP_EQ_OVERRUN:
-		*val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, comp_eq_overrun);
-		break;
-	case MLX5_VNIC_DIAG_ASYNC_EQ_OVERRUN:
-		*val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, async_eq_overrun);
-		break;
-	case MLX5_VNIC_DIAG_CQ_OVERRUN:
-		*val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, cq_overrun);
-		break;
-	case MLX5_VNIC_DIAG_INVALID_COMMAND:
-		*val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, invalid_command);
-		break;
-	case MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND:
-		*val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, quota_exceeded_command);
-		break;
-	case MLX5_VNIC_DIAG_RX_STEERING_DISCARD:
-		*val = MLX5_GET64(vnic_diagnostic_statistics, vnic_diag_out,
-				  nic_receive_steering_discard);
-		break;
-	}
-
-	return 0;
-}
-
-static int __show_vnic_diag(struct seq_file *file, struct mlx5_vport *vport,
-			    enum vnic_diag_counter type)
-{
-	u64 val = 0;
-	int ret;
-
-	ret = mlx5_esw_query_vnic_diag(vport, type, &val);
-	if (ret)
-		return ret;
-
-	seq_printf(file, "%llu\n", val);
-	return 0;
-}
-
-static int total_q_under_processor_handle_show(struct seq_file *file, void *priv)
-{
-	return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_TOTAL_Q_UNDER_PROCESSOR_HANDLE);
-}
-
-static int send_queue_priority_update_flow_show(struct seq_file *file, void *priv)
-{
-	return __show_vnic_diag(file, file->private,
-				MLX5_VNIC_DIAG_SEND_QUEUE_PRIORITY_UPDATE_FLOW);
-}
-
-static int comp_eq_overrun_show(struct seq_file *file, void *priv)
-{
-	return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_COMP_EQ_OVERRUN);
-}
-
-static int async_eq_overrun_show(struct seq_file *file, void *priv)
-{
-	return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_ASYNC_EQ_OVERRUN);
-}
-
-static int cq_overrun_show(struct seq_file *file, void *priv)
-{
-	return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_CQ_OVERRUN);
-}
-
-static int invalid_command_show(struct seq_file *file, void *priv)
-{
-	return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_INVALID_COMMAND);
-}
-
-static int quota_exceeded_command_show(struct seq_file *file, void *priv)
-{
-	return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND);
-}
-
-static int rx_steering_discard_show(struct seq_file *file, void *priv)
-{
-	return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_RX_STEERING_DISCARD);
-}
-
-DEFINE_SHOW_ATTRIBUTE(total_q_under_processor_handle);
-DEFINE_SHOW_ATTRIBUTE(send_queue_priority_update_flow);
-DEFINE_SHOW_ATTRIBUTE(comp_eq_overrun);
-DEFINE_SHOW_ATTRIBUTE(async_eq_overrun);
-DEFINE_SHOW_ATTRIBUTE(cq_overrun);
-DEFINE_SHOW_ATTRIBUTE(invalid_command);
-DEFINE_SHOW_ATTRIBUTE(quota_exceeded_command);
-DEFINE_SHOW_ATTRIBUTE(rx_steering_discard);
-
-void mlx5_esw_vport_debugfs_destroy(struct mlx5_eswitch *esw, u16 vport_num)
-{
-	struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
-
-	debugfs_remove_recursive(vport->dbgfs);
-	vport->dbgfs = NULL;
-}
-
-/* vnic diag dir name is "pf", "ecpf" or "{vf/sf}_xxxx" */
-#define VNIC_DIAG_DIR_NAME_MAX_LEN 8
-
-void mlx5_esw_vport_debugfs_create(struct mlx5_eswitch *esw, u16 vport_num, bool is_sf, u16 sf_num)
-{
-	struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
-	struct dentry *vnic_diag;
-	char dir_name[VNIC_DIAG_DIR_NAME_MAX_LEN];
-	int err;
-
-	if (!MLX5_CAP_GEN(esw->dev, vport_group_manager))
-		return;
-
-	if (vport_num == MLX5_VPORT_PF) {
-		strcpy(dir_name, "pf");
-	} else if (vport_num == MLX5_VPORT_ECPF) {
-		strcpy(dir_name, "ecpf");
-	} else {
-		err = snprintf(dir_name, VNIC_DIAG_DIR_NAME_MAX_LEN, "%s_%d", is_sf ? "sf" : "vf",
-			       is_sf ? sf_num : vport_num - MLX5_VPORT_FIRST_VF);
-		if (WARN_ON(err < 0))
-			return;
-	}
-
-	vport->dbgfs = debugfs_create_dir(dir_name, esw->dbgfs);
-	vnic_diag = debugfs_create_dir("vnic_diag", vport->dbgfs);
-
-	if (MLX5_CAP_GEN(esw->dev, vnic_env_queue_counters)) {
-		debugfs_create_file("total_q_under_processor_handle", 0444, vnic_diag, vport,
-				    &total_q_under_processor_handle_fops);
-		debugfs_create_file("send_queue_priority_update_flow", 0444, vnic_diag, vport,
-				    &send_queue_priority_update_flow_fops);
-	}
-
-	if (MLX5_CAP_GEN(esw->dev, eq_overrun_count)) {
-		debugfs_create_file("comp_eq_overrun", 0444, vnic_diag, vport,
-				    &comp_eq_overrun_fops);
-		debugfs_create_file("async_eq_overrun", 0444, vnic_diag, vport,
-				    &async_eq_overrun_fops);
-	}
-
-	if (MLX5_CAP_GEN(esw->dev, vnic_env_cq_overrun))
-		debugfs_create_file("cq_overrun", 0444, vnic_diag, vport, &cq_overrun_fops);
-
-	if (MLX5_CAP_GEN(esw->dev, invalid_command_count))
-		debugfs_create_file("invalid_command", 0444, vnic_diag, vport,
-				    &invalid_command_fops);
-
-	if (MLX5_CAP_GEN(esw->dev, quota_exceeded_count))
-		debugfs_create_file("quota_exceeded_command", 0444, vnic_diag, vport,
-				    &quota_exceeded_command_fops);
-
-	if (MLX5_CAP_GEN(esw->dev, nic_receive_steering_discard))
-		debugfs_create_file("rx_steering_discard", 0444, vnic_diag, vport,
-				    &rx_steering_discard_fops);
-
-}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h
index 51ac24e6ec3c..1808da214094 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h
@@ -110,6 +110,41 @@ DEFINE_EVENT(mlx5_esw_bridge_port_template,
 	     TP_ARGS(port)
 	);
 
+DECLARE_EVENT_CLASS(mlx5_esw_bridge_mdb_port_change_template,
+		    TP_PROTO(const struct net_device *dev,
+			     const struct mlx5_esw_bridge_mdb_entry *mdb),
+		    TP_ARGS(dev, mdb),
+		    TP_STRUCT__entry(
+			    __array(char, dev_name, IFNAMSIZ)
+			    __array(unsigned char, addr, ETH_ALEN)
+			    __field(u16, vid)
+			    __field(int, num_ports)
+			    __field(bool, offloaded)),
+		    TP_fast_assign(
+			    strscpy(__entry->dev_name, netdev_name(dev), IFNAMSIZ);
+			    memcpy(__entry->addr, mdb->key.addr, ETH_ALEN);
+			    __entry->vid = mdb->key.vid;
+			    __entry->num_ports = mdb->num_ports;
+			    __entry->offloaded = mdb->egress_handle;),
+		    TP_printk("net_device=%s addr=%pM vid=%u num_ports=%d offloaded=%d",
+			      __entry->dev_name,
+			      __entry->addr,
+			      __entry->vid,
+			      __entry->num_ports,
+			      __entry->offloaded));
+
+DEFINE_EVENT(mlx5_esw_bridge_mdb_port_change_template,
+	     mlx5_esw_bridge_port_mdb_attach,
+	     TP_PROTO(const struct net_device *dev,
+		      const struct mlx5_esw_bridge_mdb_entry *mdb),
+	     TP_ARGS(dev, mdb));
+
+DEFINE_EVENT(mlx5_esw_bridge_mdb_port_change_template,
+	     mlx5_esw_bridge_port_mdb_detach,
+	     TP_PROTO(const struct net_device *dev,
+		      const struct mlx5_esw_bridge_mdb_entry *mdb),
+	     TP_ARGS(dev, mdb));
+
 #endif
 
 /* This part must be outside protection */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
index 75015d370922..7c79476cc5f9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
@@ -744,7 +744,7 @@ static int esw_qos_devlink_rate_to_mbps(struct mlx5_core_dev *mdev, const char *
 	u64 value;
 	int err;
 
-	err = mlx5e_port_max_linkspeed(mdev, &link_speed_max);
+	err = mlx5_port_max_linkspeed(mdev, &link_speed_max);
 	if (err) {
 		NL_SET_ERR_MSG_MOD(extack, "Failed to get link maximum speed");
 		return err;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c
index 9e72118f2e4c..749c3957a128 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/vporttbl.c
@@ -11,7 +11,7 @@ struct mlx5_vport_key {
 	u16 prio;
 	u16 vport;
 	u16 vhca_id;
-	const struct esw_vport_tbl_namespace *vport_ns;
+	struct esw_vport_tbl_namespace *vport_ns;
 } __packed;
 
 struct mlx5_vport_table {
@@ -21,6 +21,14 @@ struct mlx5_vport_table {
 	struct mlx5_vport_key key;
 };
 
+static void
+esw_vport_tbl_init(struct mlx5_eswitch *esw, struct esw_vport_tbl_namespace *ns)
+{
+	if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
+		ns->flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
+			      MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
+}
+
 static struct mlx5_flow_table *
 esw_vport_tbl_create(struct mlx5_eswitch *esw, struct mlx5_flow_namespace *ns,
 		     const struct esw_vport_tbl_namespace *vport_ns)
@@ -80,6 +88,7 @@ mlx5_esw_vporttbl_get(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr
 	u32 hkey;
 
 	mutex_lock(&esw->fdb_table.offloads.vports.lock);
+	esw_vport_tbl_init(esw, attr->vport_ns);
 	hkey = flow_attr_to_vport_key(esw, attr, &skey);
 	e = esw_vport_tbl_lookup(esw, &skey, hkey);
 	if (e) {
@@ -127,6 +136,7 @@ mlx5_esw_vporttbl_put(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr
 	u32 hkey;
 
 	mutex_lock(&esw->fdb_table.offloads.vports.lock);
+	esw_vport_tbl_init(esw, attr->vport_ns);
 	hkey = flow_attr_to_vport_key(esw, attr, &key);
 	e = esw_vport_tbl_lookup(esw, &key, hkey);
 	if (!e || --e->num_rules)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index 8bdf28762f41..901c53751b0a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -36,7 +36,6 @@
 #include <linux/mlx5/vport.h>
 #include <linux/mlx5/fs.h>
 #include <linux/mlx5/mpfs.h>
-#include <linux/debugfs.h>
 #include "esw/acl/lgcy.h"
 #include "esw/legacy.h"
 #include "esw/qos.h"
@@ -1056,7 +1055,6 @@ int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, u16 vport_num,
 	if (err)
 		return err;
 
-	mlx5_esw_vport_debugfs_create(esw, vport_num, false, 0);
 	err = esw_offloads_load_rep(esw, vport_num);
 	if (err)
 		goto err_rep;
@@ -1064,7 +1062,6 @@ int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, u16 vport_num,
 	return err;
 
 err_rep:
-	mlx5_esw_vport_debugfs_destroy(esw, vport_num);
 	mlx5_esw_vport_disable(esw, vport_num);
 	return err;
 }
@@ -1072,7 +1069,6 @@ err_rep:
 void mlx5_eswitch_unload_vport(struct mlx5_eswitch *esw, u16 vport_num)
 {
 	esw_offloads_unload_rep(esw, vport_num);
-	mlx5_esw_vport_debugfs_destroy(esw, vport_num);
 	mlx5_esw_vport_disable(esw, vport_num);
 }
 
@@ -1488,7 +1484,7 @@ int mlx5_esw_sf_max_hpf_functions(struct mlx5_core_dev *dev, u16 *max_sfs, u16 *
 	void *hca_caps;
 	int err;
 
-	if (!mlx5_core_is_ecpf(dev) || mlx5_core_is_management_pf(dev)) {
+	if (!mlx5_core_is_ecpf(dev)) {
 		*max_sfs = 0;
 		return 0;
 	}
@@ -1510,7 +1506,7 @@ out_free:
 	return err;
 }
 
-static int mlx5_esw_vport_alloc(struct mlx5_eswitch *esw, struct mlx5_core_dev *dev,
+static int mlx5_esw_vport_alloc(struct mlx5_eswitch *esw,
 				int index, u16 vport_num)
 {
 	struct mlx5_vport *vport;
@@ -1564,7 +1560,7 @@ static int mlx5_esw_vports_init(struct mlx5_eswitch *esw)
 
 	xa_init(&esw->vports);
 
-	err = mlx5_esw_vport_alloc(esw, dev, idx, MLX5_VPORT_PF);
+	err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_PF);
 	if (err)
 		goto err;
 	if (esw->first_host_vport == MLX5_VPORT_PF)
@@ -1572,7 +1568,7 @@ static int mlx5_esw_vports_init(struct mlx5_eswitch *esw)
 	idx++;
 
 	for (i = 0; i < mlx5_core_max_vfs(dev); i++) {
-		err = mlx5_esw_vport_alloc(esw, dev, idx, idx);
+		err = mlx5_esw_vport_alloc(esw, idx, idx);
 		if (err)
 			goto err;
 		xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_VF);
@@ -1581,7 +1577,7 @@ static int mlx5_esw_vports_init(struct mlx5_eswitch *esw)
 	}
 	base_sf_num = mlx5_sf_start_function_id(dev);
 	for (i = 0; i < mlx5_sf_max_functions(dev); i++) {
-		err = mlx5_esw_vport_alloc(esw, dev, idx, base_sf_num + i);
+		err = mlx5_esw_vport_alloc(esw, idx, base_sf_num + i);
 		if (err)
 			goto err;
 		xa_set_mark(&esw->vports, base_sf_num + i, MLX5_ESW_VPT_SF);
@@ -1592,7 +1588,7 @@ static int mlx5_esw_vports_init(struct mlx5_eswitch *esw)
 	if (err)
 		goto err;
 	for (i = 0; i < max_host_pf_sfs; i++) {
-		err = mlx5_esw_vport_alloc(esw, dev, idx, base_sf_num + i);
+		err = mlx5_esw_vport_alloc(esw, idx, base_sf_num + i);
 		if (err)
 			goto err;
 		xa_set_mark(&esw->vports, base_sf_num + i, MLX5_ESW_VPT_SF);
@@ -1600,12 +1596,12 @@ static int mlx5_esw_vports_init(struct mlx5_eswitch *esw)
 	}
 
 	if (mlx5_ecpf_vport_exists(dev)) {
-		err = mlx5_esw_vport_alloc(esw, dev, idx, MLX5_VPORT_ECPF);
+		err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_ECPF);
 		if (err)
 			goto err;
 		idx++;
 	}
-	err = mlx5_esw_vport_alloc(esw, dev, idx, MLX5_VPORT_UPLINK);
+	err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_UPLINK);
 	if (err)
 		goto err;
 	return 0;
@@ -1672,7 +1668,6 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 	dev->priv.eswitch = esw;
 	BLOCKING_INIT_NOTIFIER_HEAD(&esw->n_head);
 
-	esw->dbgfs = debugfs_create_dir("esw", mlx5_debugfs_get_dev_root(esw->dev));
 	esw_info(dev,
 		 "Total vports %d, per vport: max uc(%d) max mc(%d)\n",
 		 esw->total_vports,
@@ -1696,7 +1691,6 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw)
 
 	esw_info(esw->dev, "cleanup\n");
 
-	debugfs_remove_recursive(esw->dbgfs);
 	esw->dev->priv.eswitch = NULL;
 	destroy_workqueue(esw->work_queue);
 	WARN_ON(refcount_read(&esw->qos.refcnt));
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 19e9a77c4633..1a042c981713 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -195,7 +195,6 @@ struct mlx5_vport {
 	enum mlx5_eswitch_vport_event enabled_events;
 	int index;
 	struct devlink_port *dl_port;
-	struct dentry *dbgfs;
 };
 
 struct mlx5_esw_indir_table;
@@ -263,6 +262,7 @@ struct mlx5_esw_offload {
 	const struct mlx5_eswitch_rep_ops *rep_ops[NUM_REP_TYPES];
 	u8 inline_mode;
 	atomic64_t num_flows;
+	u64 num_block_encap;
 	enum devlink_eswitch_encap_mode encap;
 	struct ida vport_metadata_ida;
 	unsigned int host_number; /* ECPF supports one external host */
@@ -342,7 +342,6 @@ struct mlx5_eswitch {
 		u32             large_group_num;
 	}  params;
 	struct blocking_notifier_head n_head;
-	struct dentry *dbgfs;
 };
 
 void esw_offloads_disable(struct mlx5_eswitch *esw);
@@ -355,7 +354,6 @@ mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num
 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule);
 
 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw);
-int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable);
 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw);
 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata);
 
@@ -674,7 +672,7 @@ struct mlx5_vport_tbl_attr {
 	u32 chain;
 	u16 prio;
 	u16 vport;
-	const struct esw_vport_tbl_namespace *vport_ns;
+	struct esw_vport_tbl_namespace *vport_ns;
 };
 
 struct mlx5_flow_table *
@@ -703,9 +701,6 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, u16 vport_
 void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_eswitch *esw, u16 vport_num);
 struct devlink_port *mlx5_esw_offloads_devlink_port(struct mlx5_eswitch *esw, u16 vport_num);
 
-void mlx5_esw_vport_debugfs_create(struct mlx5_eswitch *esw, u16 vport_num, bool is_sf, u16 sf_num);
-void mlx5_esw_vport_debugfs_destroy(struct mlx5_eswitch *esw, u16 vport_num);
-
 int mlx5_esw_devlink_sf_port_register(struct mlx5_eswitch *esw, struct devlink_port *dl_port,
 				      u16 vport_num, u32 controller, u32 sfnum);
 void mlx5_esw_devlink_sf_port_unregister(struct mlx5_eswitch *esw, u16 vport_num);
@@ -748,6 +743,9 @@ void mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch *master_esw,
 					      struct mlx5_eswitch *slave_esw);
 int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw);
 
+bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev);
+void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev);
+
 static inline int mlx5_eswitch_num_vfs(struct mlx5_eswitch *esw)
 {
 	if (mlx5_esw_allowed(esw))
@@ -761,6 +759,7 @@ mlx5_eswitch_get_slow_fdb(struct mlx5_eswitch *esw)
 {
 	return esw->fdb_table.offloads.slow_fdb;
 }
+
 #else  /* CONFIG_MLX5_ESWITCH */
 /* eswitch API stubs */
 static inline int  mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; }
@@ -805,6 +804,15 @@ mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw)
 {
 	return 0;
 }
+
+static inline bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev)
+{
+	return true;
+}
+
+static inline void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev)
+{
+}
 #endif /* CONFIG_MLX5_ESWITCH */
 
 #endif /* __MLX5_ESWITCH_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index 25a8076a77bf..69215ffb9999 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -73,7 +73,7 @@
 
 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
 
-static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
+static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
 	.max_fte = MLX5_ESW_VPORT_TBL_SIZE,
 	.max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
 	.flags = 0,
@@ -760,7 +760,6 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
 	kfree(dest);
 	return rule;
 err_chain_src_rewrite:
-	esw_put_dest_tables_loop(esw, attr, 0, i);
 	mlx5_esw_vporttbl_put(esw, &fwd_attr);
 err_get_fwd:
 	mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
@@ -803,7 +802,6 @@ __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
 	if (fwd_rule)  {
 		mlx5_esw_vporttbl_put(esw, &fwd_attr);
 		mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
-		esw_put_dest_tables_loop(esw, attr, 0, esw_attr->split_count);
 	} else {
 		if (split)
 			mlx5_esw_vporttbl_put(esw, &fwd_attr);
@@ -1374,14 +1372,11 @@ esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
 	struct mlx5_flow_table *nf_ft, *ft;
 	struct mlx5_chains_attr attr = {};
 	struct mlx5_fs_chains *chains;
-	u32 fdb_max;
 	int err;
 
-	fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
-
 	esw_init_chains_offload_flags(esw, &attr.flags);
 	attr.ns = MLX5_FLOW_NAMESPACE_FDB;
-	attr.max_ft_sz = fdb_max;
+	attr.fs_base_prio = FDB_TC_OFFLOAD;
 	attr.max_grp_num = esw->params.large_group_num;
 	attr.default_ft = miss_fdb;
 	attr.mapping = esw->offloads.reg_c0_obj_pool;
@@ -1392,6 +1387,7 @@ esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
 		esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
 		return err;
 	}
+	mlx5_chains_print_info(chains);
 
 	esw->fdb_table.offloads.esw_chains_priv = chains;
 
@@ -2941,28 +2937,6 @@ metadata_err:
 	return err;
 }
 
-int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable)
-{
-	int err = 0;
-
-	down_write(&esw->mode_lock);
-	if (mlx5_esw_is_fdb_created(esw)) {
-		err = -EBUSY;
-		goto done;
-	}
-	if (!mlx5_esw_vport_match_metadata_supported(esw)) {
-		err = -EOPNOTSUPP;
-		goto done;
-	}
-	if (enable)
-		esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
-	else
-		esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
-done:
-	up_write(&esw->mode_lock);
-	return err;
-}
-
 int
 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
 				     struct mlx5_vport *vport)
@@ -3588,6 +3562,47 @@ int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
 	return err;
 }
 
+bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev)
+{
+	struct devlink *devlink = priv_to_devlink(dev);
+	struct mlx5_eswitch *esw;
+
+	devl_lock(devlink);
+	esw = mlx5_devlink_eswitch_get(devlink);
+	if (IS_ERR(esw)) {
+		devl_unlock(devlink);
+		/* Failure means no eswitch => not possible to change encap */
+		return true;
+	}
+
+	down_write(&esw->mode_lock);
+	if (esw->mode != MLX5_ESWITCH_LEGACY &&
+	    esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
+		up_write(&esw->mode_lock);
+		devl_unlock(devlink);
+		return false;
+	}
+
+	esw->offloads.num_block_encap++;
+	up_write(&esw->mode_lock);
+	devl_unlock(devlink);
+	return true;
+}
+
+void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev)
+{
+	struct devlink *devlink = priv_to_devlink(dev);
+	struct mlx5_eswitch *esw;
+
+	esw = mlx5_devlink_eswitch_get(devlink);
+	if (IS_ERR(esw))
+		return;
+
+	down_write(&esw->mode_lock);
+	esw->offloads.num_block_encap--;
+	up_write(&esw->mode_lock);
+}
+
 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
 					enum devlink_eswitch_encap_mode encap,
 					struct netlink_ext_ack *extack)
@@ -3629,6 +3644,13 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
 		goto unlock;
 	}
 
+	if (esw->offloads.num_block_encap) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Can't set encapsulation when IPsec SA and/or policies are configured");
+		err = -EOPNOTSUPP;
+		goto unlock;
+	}
+
 	esw_destroy_offloads_fdb_tables(esw);
 
 	esw->offloads.encap = encap;
@@ -3782,14 +3804,12 @@ int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_p
 	if (err)
 		goto devlink_err;
 
-	mlx5_esw_vport_debugfs_create(esw, vport_num, true, sfnum);
 	err = mlx5_esw_offloads_rep_load(esw, vport_num);
 	if (err)
 		goto rep_err;
 	return 0;
 
 rep_err:
-	mlx5_esw_vport_debugfs_destroy(esw, vport_num);
 	mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
 devlink_err:
 	mlx5_esw_vport_disable(esw, vport_num);
@@ -3799,7 +3819,6 @@ devlink_err:
 void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num)
 {
 	mlx5_esw_offloads_rep_unload(esw, vport_num);
-	mlx5_esw_vport_debugfs_destroy(esw, vport_num);
 	mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
 	mlx5_esw_vport_disable(esw, vport_num);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c
index 3a9a6bb9158d..edd910258314 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c
@@ -210,18 +210,6 @@ static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw,
 	return (port_mask & port_value) == MLX5_VPORT_UPLINK;
 }
 
-static bool
-mlx5_eswitch_is_push_vlan_no_cap(struct mlx5_eswitch *esw,
-				 struct mlx5_flow_act *flow_act)
-{
-	if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH &&
-	    !(mlx5_fs_get_capabilities(esw->dev, MLX5_FLOW_NAMESPACE_FDB) &
-	      MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX))
-		return true;
-
-	return false;
-}
-
 bool
 mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw,
 			      struct mlx5_flow_attr *attr,
@@ -237,7 +225,10 @@ mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw,
 	    (!mlx5_eswitch_offload_is_uplink_port(esw, spec) && !esw_attr->int_port))
 		return false;
 
-	if (mlx5_eswitch_is_push_vlan_no_cap(esw, flow_act))
+	/* push vlan on RX */
+	if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH &&
+	    !(mlx5_fs_get_capabilities(esw->dev, MLX5_FLOW_NAMESPACE_FDB) &
+	      MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX))
 		return true;
 
 	/* hairpin */
@@ -261,31 +252,19 @@ mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw,
 	struct mlx5_flow_act term_tbl_act = {};
 	struct mlx5_flow_handle *rule = NULL;
 	bool term_table_created = false;
-	bool is_push_vlan_on_rx;
 	int num_vport_dests = 0;
 	int i, curr_dest;
 
-	is_push_vlan_on_rx = mlx5_eswitch_is_push_vlan_no_cap(esw, flow_act);
 	mlx5_eswitch_termtbl_actions_move(flow_act, &term_tbl_act);
 	term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 
 	for (i = 0; i < num_dest; i++) {
 		struct mlx5_termtbl_handle *tt;
-		bool hairpin = false;
 
 		/* only vport destinations can be terminated */
 		if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT)
 			continue;
 
-		if (attr->dests[num_vport_dests].rep &&
-		    attr->dests[num_vport_dests].rep->vport == MLX5_VPORT_UPLINK)
-			hairpin = true;
-
-		if (!is_push_vlan_on_rx && !hairpin) {
-			num_vport_dests++;
-			continue;
-		}
-
 		if (attr->dests[num_vport_dests].flags & MLX5_ESW_DEST_ENCAP) {
 			term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
 			term_tbl_act.pkt_reformat = attr->dests[num_vport_dests].pkt_reformat;
@@ -333,9 +312,6 @@ revert_changes:
 	for (curr_dest = 0; curr_dest < num_vport_dests; curr_dest++) {
 		struct mlx5_termtbl_handle *tt = attr->dests[curr_dest].termtbl;
 
-		if (!tt)
-			continue;
-
 		attr->dests[curr_dest].termtbl = NULL;
 
 		/* search for the destination associated with the
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
index 731acbe22dc7..19da02c41616 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
@@ -137,7 +137,7 @@
 #define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + KERNEL_RX_MACSEC_MIN_LEVEL + 1)
 
 #define KERNEL_TX_IPSEC_NUM_PRIOS  1
-#define KERNEL_TX_IPSEC_NUM_LEVELS 2
+#define KERNEL_TX_IPSEC_NUM_LEVELS 3
 #define KERNEL_TX_IPSEC_MIN_LEVEL        (KERNEL_TX_IPSEC_NUM_LEVELS)
 
 #define KERNEL_TX_MACSEC_NUM_PRIOS  1
@@ -1762,7 +1762,8 @@ static bool dest_is_valid(struct mlx5_flow_destination *dest,
 
 	if (ignore_level) {
 		if (ft->type != FS_FT_FDB &&
-		    ft->type != FS_FT_NIC_RX)
+		    ft->type != FS_FT_NIC_RX &&
+		    ft->type != FS_FT_NIC_TX)
 			return false;
 
 		if (dest->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&
@@ -2996,7 +2997,7 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering)
 		goto out_err;
 	}
 
-	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BR_OFFLOAD, 3);
+	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BR_OFFLOAD, 4);
 	if (IS_ERR(maj_prio)) {
 		err = PTR_ERR(maj_prio);
 		goto out_err;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
index 4c2dad9d7cfb..50022e7565f1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
@@ -167,7 +167,7 @@ static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev)
 		if (mlx5_health_wait_pci_up(dev))
 			mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n");
 		else
-			mlx5_load_one(dev);
+			mlx5_load_one(dev, true);
 		devlink_remote_reload_actions_performed(priv_to_devlink(dev), 0,
 							BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
 							BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE));
@@ -499,7 +499,7 @@ int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev)
 	err = fw_reset->ret;
 	if (test_and_clear_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags)) {
 		mlx5_unload_one_devl_locked(dev, false);
-		mlx5_load_one_devl_locked(dev, false);
+		mlx5_load_one_devl_locked(dev, true);
 	}
 out:
 	clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
index f9438d4e43ca..871c32dda66e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
@@ -42,6 +42,7 @@
 #include "lib/pci_vsc.h"
 #include "lib/tout.h"
 #include "diag/fw_tracer.h"
+#include "diag/reporter_vnic.h"
 
 enum {
 	MAX_MISSES			= 3,
@@ -325,6 +326,10 @@ int mlx5_health_wait_pci_up(struct mlx5_core_dev *dev)
 	while (sensor_pci_not_working(dev)) {
 		if (time_after(jiffies, end))
 			return -ETIMEDOUT;
+		if (test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) {
+			mlx5_core_warn(dev, "device is being removed, stop waiting for PCI\n");
+			return -ENODEV;
+		}
 		msleep(100);
 	}
 	return 0;
@@ -894,6 +899,7 @@ void mlx5_health_cleanup(struct mlx5_core_dev *dev)
 
 	cancel_delayed_work_sync(&health->update_fw_log_ts_work);
 	destroy_workqueue(health->wq);
+	mlx5_reporter_vnic_destroy(dev);
 	mlx5_fw_reporters_destroy(dev);
 }
 
@@ -903,6 +909,7 @@ int mlx5_health_init(struct mlx5_core_dev *dev)
 	char *name;
 
 	mlx5_fw_reporters_create(dev);
+	mlx5_reporter_vnic_create(dev);
 
 	health = &dev->priv.health;
 	name = kmalloc(64, GFP_KERNEL);
@@ -922,6 +929,7 @@ int mlx5_health_init(struct mlx5_core_dev *dev)
 	return 0;
 
 out_err:
+	mlx5_reporter_vnic_destroy(dev);
 	mlx5_fw_reporters_destroy(dev);
 	return -ENOMEM;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/irq_affinity.c b/drivers/net/ethernet/mellanox/mlx5/core/irq_affinity.c
index 380a208ab137..fa467335526e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/irq_affinity.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/irq_affinity.c
@@ -45,30 +45,28 @@ static int cpu_get_least_loaded(struct mlx5_irq_pool *pool,
 
 /* Creating an IRQ from irq_pool */
 static struct mlx5_irq *
-irq_pool_request_irq(struct mlx5_irq_pool *pool, const struct cpumask *req_mask)
+irq_pool_request_irq(struct mlx5_irq_pool *pool, struct irq_affinity_desc *af_desc)
 {
-	cpumask_var_t auto_mask;
-	struct mlx5_irq *irq;
+	struct irq_affinity_desc auto_desc = {};
 	u32 irq_index;
 	int err;
 
-	if (!zalloc_cpumask_var(&auto_mask, GFP_KERNEL))
-		return ERR_PTR(-ENOMEM);
 	err = xa_alloc(&pool->irqs, &irq_index, NULL, pool->xa_num_irqs, GFP_KERNEL);
 	if (err)
 		return ERR_PTR(err);
 	if (pool->irqs_per_cpu) {
-		if (cpumask_weight(req_mask) > 1)
+		if (cpumask_weight(&af_desc->mask) > 1)
 			/* if req_mask contain more then one CPU, set the least loadad CPU
 			 * of req_mask
 			 */
-			cpumask_set_cpu(cpu_get_least_loaded(pool, req_mask), auto_mask);
+			cpumask_set_cpu(cpu_get_least_loaded(pool, &af_desc->mask),
+					&auto_desc.mask);
 		else
-			cpu_get(pool, cpumask_first(req_mask));
+			cpu_get(pool, cpumask_first(&af_desc->mask));
 	}
-	irq = mlx5_irq_alloc(pool, irq_index, cpumask_empty(auto_mask) ? req_mask : auto_mask);
-	free_cpumask_var(auto_mask);
-	return irq;
+	return mlx5_irq_alloc(pool, irq_index,
+			      cpumask_empty(&auto_desc.mask) ? af_desc : &auto_desc,
+			      NULL);
 }
 
 /* Looking for the IRQ with the smallest refcount that fits req_mask.
@@ -115,22 +113,22 @@ irq_pool_find_least_loaded(struct mlx5_irq_pool *pool, const struct cpumask *req
 /**
  * mlx5_irq_affinity_request - request an IRQ according to the given mask.
  * @pool: IRQ pool to request from.
- * @req_mask: cpumask requested for this IRQ.
+ * @af_desc: affinity descriptor for this IRQ.
  *
  * This function returns a pointer to IRQ, or ERR_PTR in case of error.
  */
 struct mlx5_irq *
-mlx5_irq_affinity_request(struct mlx5_irq_pool *pool, const struct cpumask *req_mask)
+mlx5_irq_affinity_request(struct mlx5_irq_pool *pool, struct irq_affinity_desc *af_desc)
 {
 	struct mlx5_irq *least_loaded_irq, *new_irq;
 
 	mutex_lock(&pool->lock);
-	least_loaded_irq = irq_pool_find_least_loaded(pool, req_mask);
+	least_loaded_irq = irq_pool_find_least_loaded(pool, &af_desc->mask);
 	if (least_loaded_irq &&
 	    mlx5_irq_read_locked(least_loaded_irq) < pool->min_threshold)
 		goto out;
 	/* We didn't find an IRQ with less than min_thres, try to allocate a new IRQ */
-	new_irq = irq_pool_request_irq(pool, req_mask);
+	new_irq = irq_pool_request_irq(pool, af_desc);
 	if (IS_ERR(new_irq)) {
 		if (!least_loaded_irq) {
 			/* We failed to create an IRQ and we didn't find an IRQ */
@@ -194,32 +192,30 @@ int mlx5_irq_affinity_irqs_request_auto(struct mlx5_core_dev *dev, int nirqs,
 					struct mlx5_irq **irqs)
 {
 	struct mlx5_irq_pool *pool = mlx5_irq_pool_get(dev);
-	cpumask_var_t req_mask;
+	struct irq_affinity_desc af_desc = {};
 	struct mlx5_irq *irq;
 	int i = 0;
 
-	if (!zalloc_cpumask_var(&req_mask, GFP_KERNEL))
-		return -ENOMEM;
-	cpumask_copy(req_mask, cpu_online_mask);
+	af_desc.is_managed = 1;
+	cpumask_copy(&af_desc.mask, cpu_online_mask);
 	for (i = 0; i < nirqs; i++) {
 		if (mlx5_irq_pool_is_sf_pool(pool))
-			irq = mlx5_irq_affinity_request(pool, req_mask);
+			irq = mlx5_irq_affinity_request(pool, &af_desc);
 		else
 			/* In case SF pool doesn't exists, fallback to the PF IRQs.
 			 * The PF IRQs are already allocated and binded to CPU
 			 * at this point. Hence, only an index is needed.
 			 */
-			irq = mlx5_irq_request(dev, i, NULL);
+			irq = mlx5_irq_request(dev, i, NULL, NULL);
 		if (IS_ERR(irq))
 			break;
 		irqs[i] = irq;
-		cpumask_clear_cpu(cpumask_first(mlx5_irq_get_affinity_mask(irq)), req_mask);
+		cpumask_clear_cpu(cpumask_first(mlx5_irq_get_affinity_mask(irq)), &af_desc.mask);
 		mlx5_core_dbg(pool->dev, "IRQ %u mapped to cpu %*pbl, %u EQs on this irq\n",
 			      pci_irq_vector(dev->pdev, mlx5_irq_get_index(irq)),
 			      cpumask_pr_args(mlx5_irq_get_affinity_mask(irq)),
 			      mlx5_irq_read_locked(irq) / MLX5_EQ_REFS_PER_IRQ);
 	}
-	free_cpumask_var(req_mask);
 	if (!i)
 		return PTR_ERR(irq);
 	return i;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
index 4c9a40211059..932fbc843c69 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
@@ -39,7 +39,7 @@
 #include "clock.h"
 
 enum {
-	MLX5_CYCLES_SHIFT	= 23
+	MLX5_CYCLES_SHIFT	= 31
 };
 
 enum {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c
index 81ed91fee59b..db9df9798ffa 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c
@@ -14,10 +14,8 @@
 #define chains_lock(chains) ((chains)->lock)
 #define chains_ht(chains) ((chains)->chains_ht)
 #define prios_ht(chains) ((chains)->prios_ht)
-#define tc_default_ft(chains) ((chains)->tc_default_ft)
-#define tc_end_ft(chains) ((chains)->tc_end_ft)
-#define ns_to_chains_fs_prio(ns) ((ns) == MLX5_FLOW_NAMESPACE_FDB ? \
-				  FDB_TC_OFFLOAD : MLX5E_TC_PRIO)
+#define chains_default_ft(chains) ((chains)->chains_default_ft)
+#define chains_end_ft(chains) ((chains)->chains_end_ft)
 #define FT_TBL_SZ (64 * 1024)
 
 struct mlx5_fs_chains {
@@ -28,13 +26,15 @@ struct mlx5_fs_chains {
 	/* Protects above chains_ht and prios_ht */
 	struct mutex lock;
 
-	struct mlx5_flow_table *tc_default_ft;
-	struct mlx5_flow_table *tc_end_ft;
+	struct mlx5_flow_table *chains_default_ft;
+	struct mlx5_flow_table *chains_end_ft;
 	struct mapping_ctx *chains_mapping;
 
 	enum mlx5_flow_namespace_type ns;
 	u32 group_num;
 	u32 flags;
+	int fs_base_prio;
+	int fs_base_level;
 };
 
 struct fs_chain {
@@ -145,7 +145,7 @@ void
 mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
 		       struct mlx5_flow_table *ft)
 {
-	tc_end_ft(chains) = ft;
+	chains_end_ft(chains) = ft;
 }
 
 static struct mlx5_flow_table *
@@ -164,11 +164,11 @@ mlx5_chains_create_table(struct mlx5_fs_chains *chains,
 	sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? FT_TBL_SZ : POOL_NEXT_SIZE;
 	ft_attr.max_fte = sz;
 
-	/* We use tc_default_ft(chains) as the table's next_ft till
+	/* We use chains_default_ft(chains) as the table's next_ft till
 	 * ignore_flow_level is allowed on FT creation and not just for FTEs.
 	 * Instead caller should add an explicit miss rule if needed.
 	 */
-	ft_attr.next_ft = tc_default_ft(chains);
+	ft_attr.next_ft = chains_default_ft(chains);
 
 	/* The root table(chain 0, prio 1, level 0) is required to be
 	 * connected to the previous fs_core managed prio.
@@ -177,22 +177,22 @@ mlx5_chains_create_table(struct mlx5_fs_chains *chains,
 	 */
 	if (!mlx5_chains_ignore_flow_level_supported(chains) ||
 	    (chain == 0 && prio == 1 && level == 0)) {
-		ft_attr.level = level;
-		ft_attr.prio = prio - 1;
+		ft_attr.level = chains->fs_base_level;
+		ft_attr.prio = chains->fs_base_prio;
 		ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ?
 			mlx5_get_fdb_sub_ns(chains->dev, chain) :
 			mlx5_get_flow_namespace(chains->dev, chains->ns);
 	} else {
 		ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED;
-		ft_attr.prio = ns_to_chains_fs_prio(chains->ns);
+		ft_attr.prio = chains->fs_base_prio;
 		/* Firmware doesn't allow us to create another level 0 table,
-		 * so we create all unmanaged tables as level 1.
+		 * so we create all unmanaged tables as level 1 (base + 1).
 		 *
 		 * To connect them, we use explicit miss rules with
 		 * ignore_flow_level. Caller is responsible to create
 		 * these rules (if needed).
 		 */
-		ft_attr.level = 1;
+		ft_attr.level = chains->fs_base_level + 1;
 		ns = mlx5_get_flow_namespace(chains->dev, chains->ns);
 	}
 
@@ -220,7 +220,8 @@ create_chain_restore(struct fs_chain *chain)
 	int err;
 
 	if (chain->chain == mlx5_chains_get_nf_ft_chain(chains) ||
-	    !mlx5_chains_prios_supported(chains))
+	    !mlx5_chains_prios_supported(chains) ||
+	    !chains->chains_mapping)
 		return 0;
 
 	err = mlx5_chains_get_chain_mapping(chains, chain->chain, &index);
@@ -380,7 +381,7 @@ mlx5_chains_add_miss_rule(struct fs_chain *chain,
 	dest.type  = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	dest.ft = next_ft;
 
-	if (next_ft == tc_end_ft(chains) &&
+	if (chains->chains_mapping && next_ft == chains_end_ft(chains) &&
 	    chain->chain != mlx5_chains_get_nf_ft_chain(chains) &&
 	    mlx5_chains_prios_supported(chains)) {
 		act.modify_hdr = chain->miss_modify_hdr;
@@ -494,8 +495,8 @@ mlx5_chains_create_prio(struct mlx5_fs_chains *chains,
 
 	/* Default miss for each chain: */
 	next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
-		  tc_default_ft(chains) :
-		  tc_end_ft(chains);
+		  chains_default_ft(chains) :
+		  chains_end_ft(chains);
 	list_for_each(pos, &chain_s->prios_list) {
 		struct prio *p = list_entry(pos, struct prio, list);
 
@@ -681,7 +682,7 @@ err_get_prio:
 struct mlx5_flow_table *
 mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains)
 {
-	return tc_end_ft(chains);
+	return chains_end_ft(chains);
 }
 
 struct mlx5_flow_table *
@@ -718,48 +719,38 @@ mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
 static struct mlx5_fs_chains *
 mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
 {
-	struct mlx5_fs_chains *chains_priv;
-	u32 max_flow_counter;
+	struct mlx5_fs_chains *chains;
 	int err;
 
-	chains_priv = kzalloc(sizeof(*chains_priv), GFP_KERNEL);
-	if (!chains_priv)
+	chains = kzalloc(sizeof(*chains), GFP_KERNEL);
+	if (!chains)
 		return ERR_PTR(-ENOMEM);
 
-	max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
-			    MLX5_CAP_GEN(dev, max_flow_counter_15_0);
-
-	mlx5_core_dbg(dev,
-		      "Init flow table chains, max counters(%d), groups(%d), max flow table size(%d)\n",
-		      max_flow_counter, attr->max_grp_num, attr->max_ft_sz);
-
-	chains_priv->dev = dev;
-	chains_priv->flags = attr->flags;
-	chains_priv->ns = attr->ns;
-	chains_priv->group_num = attr->max_grp_num;
-	chains_priv->chains_mapping = attr->mapping;
-	tc_default_ft(chains_priv) = tc_end_ft(chains_priv) = attr->default_ft;
+	chains->dev = dev;
+	chains->flags = attr->flags;
+	chains->ns = attr->ns;
+	chains->group_num = attr->max_grp_num;
+	chains->chains_mapping = attr->mapping;
+	chains->fs_base_prio = attr->fs_base_prio;
+	chains->fs_base_level = attr->fs_base_level;
+	chains_default_ft(chains) = chains_end_ft(chains) = attr->default_ft;
 
-	mlx5_core_info(dev, "Supported tc offload range - chains: %u, prios: %u\n",
-		       mlx5_chains_get_chain_range(chains_priv),
-		       mlx5_chains_get_prio_range(chains_priv));
-
-	err = rhashtable_init(&chains_ht(chains_priv), &chain_params);
+	err = rhashtable_init(&chains_ht(chains), &chain_params);
 	if (err)
 		goto init_chains_ht_err;
 
-	err = rhashtable_init(&prios_ht(chains_priv), &prio_params);
+	err = rhashtable_init(&prios_ht(chains), &prio_params);
 	if (err)
 		goto init_prios_ht_err;
 
-	mutex_init(&chains_lock(chains_priv));
+	mutex_init(&chains_lock(chains));
 
-	return chains_priv;
+	return chains;
 
 init_prios_ht_err:
-	rhashtable_destroy(&chains_ht(chains_priv));
+	rhashtable_destroy(&chains_ht(chains));
 init_chains_ht_err:
-	kfree(chains_priv);
+	kfree(chains);
 	return ERR_PTR(err);
 }
 
@@ -808,3 +799,9 @@ mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains, u32 chain_mapping)
 
 	return mapping_remove(ctx, chain_mapping);
 }
+
+void
+mlx5_chains_print_info(struct mlx5_fs_chains *chains)
+{
+	mlx5_core_dbg(chains->dev, "Flow table chains groups(%d)\n", chains->group_num);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h
index d50bdb226cef..8972fe05723a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h
@@ -17,8 +17,9 @@ enum mlx5_chains_flags {
 
 struct mlx5_chains_attr {
 	enum mlx5_flow_namespace_type ns;
+	int fs_base_prio;
+	int fs_base_level;
 	u32 flags;
-	u32 max_ft_sz;
 	u32 max_grp_num;
 	struct mlx5_flow_table *default_ft;
 	struct mapping_ctx *mapping;
@@ -68,6 +69,8 @@ void mlx5_chains_destroy(struct mlx5_fs_chains *chains);
 void
 mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
 		       struct mlx5_flow_table *ft);
+void
+mlx5_chains_print_info(struct mlx5_fs_chains *chains);
 
 #else /* CONFIG_MLX5_CLS_ACT */
 
@@ -89,7 +92,9 @@ static inline struct mlx5_fs_chains *
 mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
 { return NULL; }
 static inline void
-mlx5_chains_destroy(struct mlx5_fs_chains *chains) {};
+mlx5_chains_destroy(struct mlx5_fs_chains *chains) {}
+static inline void
+mlx5_chains_print_info(struct mlx5_fs_chains *chains) {}
 
 #endif /* CONFIG_MLX5_CLS_ACT */
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index f1de152a6113..edc738e86cac 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -46,12 +46,10 @@
 #include <linux/kmod.h>
 #include <linux/mlx5/mlx5_ifc.h>
 #include <linux/mlx5/vport.h>
-#ifdef CONFIG_RFS_ACCEL
-#include <linux/cpu_rmap.h>
-#endif
 #include <linux/version.h>
 #include <net/devlink.h>
 #include "mlx5_core.h"
+#include "thermal.h"
 #include "lib/eq.h"
 #include "fs_core.h"
 #include "lib/mpfs.h"
@@ -102,15 +100,19 @@ enum {
 static struct mlx5_profile profile[] = {
 	[0] = {
 		.mask           = 0,
+		.num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
 	},
 	[1] = {
 		.mask		= MLX5_PROF_MASK_QP_SIZE,
 		.log_max_qp	= 12,
+		.num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
+
 	},
 	[2] = {
 		.mask		= MLX5_PROF_MASK_QP_SIZE |
 				  MLX5_PROF_MASK_MR_CACHE,
 		.log_max_qp	= LOG_MAX_SUPPORTED_QPS,
+		.num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
 		.mr_cache[0]	= {
 			.size	= 500,
 			.limit	= 250
@@ -176,6 +178,11 @@ static struct mlx5_profile profile[] = {
 			.limit	= 4
 		},
 	},
+	[3] = {
+		.mask		= MLX5_PROF_MASK_QP_SIZE,
+		.log_max_qp	= LOG_MAX_SUPPORTED_QPS,
+		.num_cmd_caches = 0,
+	},
 };
 
 static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili,
@@ -191,7 +198,7 @@ static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili,
 		if (!(fw_initializing >> 31))
 			break;
 		if (time_after(jiffies, end) ||
-		    test_and_clear_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) {
+		    test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) {
 			err = -EBUSY;
 			break;
 		}
@@ -710,7 +717,7 @@ static int handle_hca_cap_port_selection(struct mlx5_core_dev *dev,
 	       MLX5_ST_SZ_BYTES(port_selection_cap));
 	MLX5_SET(port_selection_cap, set_hca_cap, port_select_flow_table_bypass, 1);
 
-	err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MODE_PORT_SELECTION);
+	err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_PORT_SELECTION);
 
 	return err;
 }
@@ -917,7 +924,6 @@ static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev,
 	return 0;
 
 err_clr_master:
-	pci_clear_master(dev->pdev);
 	release_bar(dev->pdev);
 err_disable:
 	mlx5_pci_disable_device(dev);
@@ -932,7 +938,6 @@ static void mlx5_pci_close(struct mlx5_core_dev *dev)
 	 */
 	mlx5_drain_health_wq(dev);
 	iounmap(dev->iseg);
-	pci_clear_master(dev->pdev);
 	release_bar(dev->pdev);
 	mlx5_pci_disable_device(dev);
 }
@@ -1402,16 +1407,16 @@ int mlx5_init_one(struct mlx5_core_dev *dev)
 		goto function_teardown;
 	}
 
+	err = mlx5_devlink_params_register(priv_to_devlink(dev));
+	if (err)
+		goto err_devlink_params_reg;
+
 	err = mlx5_load(dev);
 	if (err)
 		goto err_load;
 
 	set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
 
-	err = mlx5_devlink_params_register(priv_to_devlink(dev));
-	if (err)
-		goto err_devlink_params_reg;
-
 	err = mlx5_register_device(dev);
 	if (err)
 		goto err_register;
@@ -1421,11 +1426,11 @@ int mlx5_init_one(struct mlx5_core_dev *dev)
 	return 0;
 
 err_register:
-	mlx5_devlink_params_unregister(priv_to_devlink(dev));
-err_devlink_params_reg:
 	clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
 	mlx5_unload(dev);
 err_load:
+	mlx5_devlink_params_unregister(priv_to_devlink(dev));
+err_devlink_params_reg:
 	mlx5_cleanup_once(dev);
 function_teardown:
 	mlx5_function_teardown(dev, true);
@@ -1444,7 +1449,6 @@ void mlx5_uninit_one(struct mlx5_core_dev *dev)
 	mutex_lock(&dev->intf_state_mutex);
 
 	mlx5_unregister_device(dev);
-	mlx5_devlink_params_unregister(priv_to_devlink(dev));
 
 	if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
 		mlx5_core_warn(dev, "%s: interface is down, NOP\n",
@@ -1455,6 +1459,7 @@ void mlx5_uninit_one(struct mlx5_core_dev *dev)
 
 	clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
 	mlx5_unload(dev);
+	mlx5_devlink_params_unregister(priv_to_devlink(dev));
 	mlx5_cleanup_once(dev);
 	mlx5_function_teardown(dev, true);
 out:
@@ -1509,13 +1514,13 @@ out:
 	return err;
 }
 
-int mlx5_load_one(struct mlx5_core_dev *dev)
+int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery)
 {
 	struct devlink *devlink = priv_to_devlink(dev);
 	int ret;
 
 	devl_lock(devlink);
-	ret = mlx5_load_one_devl_locked(dev, false);
+	ret = mlx5_load_one_devl_locked(dev, recovery);
 	devl_unlock(devlink);
 	return ret;
 }
@@ -1768,6 +1773,10 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (err)
 		dev_err(&pdev->dev, "mlx5_crdump_enable failed with error code %d\n", err);
 
+	err = mlx5_thermal_init(dev);
+	if (err)
+		dev_err(&pdev->dev, "mlx5_thermal_init failed with error code %d\n", err);
+
 	pci_save_state(pdev);
 	devlink_register(devlink);
 	return 0;
@@ -1796,6 +1805,7 @@ static void remove_one(struct pci_dev *pdev)
 	mlx5_drain_fw_reset(dev);
 	devlink_unregister(devlink);
 	mlx5_sriov_disable(pdev);
+	mlx5_thermal_uninit(dev);
 	mlx5_crdump_disable(dev);
 	mlx5_drain_health_wq(dev);
 	mlx5_uninit_one(dev);
@@ -1912,7 +1922,8 @@ static void mlx5_pci_resume(struct pci_dev *pdev)
 
 	mlx5_pci_trace(dev, "Enter, loading driver..\n");
 
-	err = mlx5_load_one(dev);
+	err = mlx5_load_one(dev, false);
+
 	if (!err)
 		devlink_health_reporter_state_update(dev->priv.health.fw_fatal_reporter,
 						     DEVLINK_HEALTH_REPORTER_STATE_HEALTHY);
@@ -2003,7 +2014,7 @@ static int mlx5_resume(struct pci_dev *pdev)
 {
 	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
 
-	return mlx5_load_one(dev);
+	return mlx5_load_one(dev, false);
 }
 
 static const struct pci_device_id mlx5_core_pci_table[] = {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index be0785f83083..1d879374acaa 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -142,6 +142,7 @@ enum mlx5_semaphore_space_address {
 };
 
 #define MLX5_DEFAULT_PROF       2
+#define MLX5_SF_PROF		3
 
 static inline int mlx5_flexible_inlen(struct mlx5_core_dev *dev, size_t fixed,
 				      size_t item_size, size_t num_items,
@@ -321,7 +322,7 @@ int mlx5_init_one(struct mlx5_core_dev *dev);
 void mlx5_uninit_one(struct mlx5_core_dev *dev);
 void mlx5_unload_one(struct mlx5_core_dev *dev, bool suspend);
 void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev, bool suspend);
-int mlx5_load_one(struct mlx5_core_dev *dev);
+int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery);
 int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery);
 
 int mlx5_vport_set_other_func_cap(struct mlx5_core_dev *dev, const void *hca_cap, u16 function_id,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h
index 23cb63fa4588..efd0c299c5c7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h
@@ -9,6 +9,7 @@
 #define MLX5_COMP_EQS_PER_SF 8
 
 struct mlx5_irq;
+struct cpu_rmap;
 
 int mlx5_irq_table_init(struct mlx5_core_dev *dev);
 void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev);
@@ -25,9 +26,10 @@ int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs);
 struct mlx5_irq *mlx5_ctrl_irq_request(struct mlx5_core_dev *dev);
 void mlx5_ctrl_irq_release(struct mlx5_irq *ctrl_irq);
 struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, u16 vecidx,
-				  struct cpumask *affinity);
+				  struct irq_affinity_desc *af_desc,
+				  struct cpu_rmap **rmap);
 int mlx5_irqs_request_vectors(struct mlx5_core_dev *dev, u16 *cpus, int nirqs,
-			      struct mlx5_irq **irqs);
+			      struct mlx5_irq **irqs, struct cpu_rmap **rmap);
 void mlx5_irqs_release_vectors(struct mlx5_irq **irqs, int nirqs);
 int mlx5_irq_attach_nb(struct mlx5_irq *irq, struct notifier_block *nb);
 int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb);
@@ -39,7 +41,7 @@ struct mlx5_irq_pool;
 int mlx5_irq_affinity_irqs_request_auto(struct mlx5_core_dev *dev, int nirqs,
 					struct mlx5_irq **irqs);
 struct mlx5_irq *mlx5_irq_affinity_request(struct mlx5_irq_pool *pool,
-					   const struct cpumask *req_mask);
+					   struct irq_affinity_desc *af_desc);
 void mlx5_irq_affinity_irqs_release(struct mlx5_core_dev *dev, struct mlx5_irq **irqs,
 				    int num_irqs);
 #else
@@ -50,7 +52,7 @@ static inline int mlx5_irq_affinity_irqs_request_auto(struct mlx5_core_dev *dev,
 }
 
 static inline struct mlx5_irq *
-mlx5_irq_affinity_request(struct mlx5_irq_pool *pool, const struct cpumask *req_mask)
+mlx5_irq_affinity_request(struct mlx5_irq_pool *pool, struct irq_affinity_desc *af_desc)
 {
 	return ERR_PTR(-EOPNOTSUPP);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
index 6bde18bcd42f..2245d3b2f393 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
 /* Copyright (c) 2019 Mellanox Technologies. */
 
+#include <linux/pci.h>
 #include <linux/interrupt.h>
 #include <linux/notifier.h>
 #include <linux/mlx5/driver.h>
@@ -9,6 +10,7 @@
 #include "mlx5_irq.h"
 #include "pci_irq.h"
 #include "lib/sf.h"
+#include "lib/eq.h"
 #ifdef CONFIG_RFS_ACCEL
 #include <linux/cpu_rmap.h>
 #endif
@@ -29,12 +31,11 @@ struct mlx5_irq {
 	char name[MLX5_MAX_IRQ_NAME];
 	struct mlx5_irq_pool *pool;
 	int refcount;
-	u32 index;
-	int irqn;
+	struct msi_map map;
 };
 
 struct mlx5_irq_table {
-	struct mlx5_irq_pool *pf_pool;
+	struct mlx5_irq_pool *pcif_pool;
 	struct mlx5_irq_pool *sf_ctrl_pool;
 	struct mlx5_irq_pool *sf_comp_pool;
 };
@@ -127,15 +128,26 @@ out:
 static void irq_release(struct mlx5_irq *irq)
 {
 	struct mlx5_irq_pool *pool = irq->pool;
+#ifdef CONFIG_RFS_ACCEL
+	struct cpu_rmap *rmap;
+#endif
 
-	xa_erase(&pool->irqs, irq->index);
-	/* free_irq requires that affinity_hint and rmap will be cleared
-	 * before calling it. This is why there is asymmetry with set_rmap
-	 * which should be called after alloc_irq but before request_irq.
+	xa_erase(&pool->irqs, irq->map.index);
+	/* free_irq requires that affinity_hint and rmap will be cleared before
+	 * calling it. To satisfy this requirement, we call
+	 * irq_cpu_rmap_remove() to remove the notifier
 	 */
-	irq_update_affinity_hint(irq->irqn, NULL);
+	irq_update_affinity_hint(irq->map.virq, NULL);
+#ifdef CONFIG_RFS_ACCEL
+	rmap = mlx5_eq_table_get_rmap(pool->dev);
+	if (rmap && irq->map.index)
+		irq_cpu_rmap_remove(rmap, irq->map.virq);
+#endif
+
 	free_cpumask_var(irq->mask);
-	free_irq(irq->irqn, &irq->nh);
+	free_irq(irq->map.virq, &irq->nh);
+	if (irq->map.index && pci_msix_can_alloc_dyn(pool->dev->pdev))
+		pci_msix_free_irq(pool->dev->pdev, irq->map);
 	kfree(irq);
 }
 
@@ -198,7 +210,7 @@ static void irq_set_name(struct mlx5_irq_pool *pool, char *name, int vecidx)
 		return;
 	}
 
-	if (vecidx == pool->xa_num_irqs.max) {
+	if (!vecidx) {
 		snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_async%d", vecidx);
 		return;
 	}
@@ -207,7 +219,8 @@ static void irq_set_name(struct mlx5_irq_pool *pool, char *name, int vecidx)
 }
 
 struct mlx5_irq *mlx5_irq_alloc(struct mlx5_irq_pool *pool, int i,
-				const struct cpumask *affinity)
+				struct irq_affinity_desc *af_desc,
+				struct cpu_rmap **rmap)
 {
 	struct mlx5_core_dev *dev = pool->dev;
 	char name[MLX5_MAX_IRQ_NAME];
@@ -217,7 +230,28 @@ struct mlx5_irq *mlx5_irq_alloc(struct mlx5_irq_pool *pool, int i,
 	irq = kzalloc(sizeof(*irq), GFP_KERNEL);
 	if (!irq)
 		return ERR_PTR(-ENOMEM);
-	irq->irqn = pci_irq_vector(dev->pdev, i);
+	if (!i || !pci_msix_can_alloc_dyn(dev->pdev)) {
+		/* The vector at index 0 was already allocated.
+		 * Just get the irq number. If dynamic irq is not supported
+		 * vectors have also been allocated.
+		 */
+		irq->map.virq = pci_irq_vector(dev->pdev, i);
+		irq->map.index = 0;
+	} else {
+		irq->map = pci_msix_alloc_irq_at(dev->pdev, MSI_ANY_INDEX, af_desc);
+		if (!irq->map.virq) {
+			err = irq->map.index;
+			goto err_alloc_irq;
+		}
+	}
+
+	if (i && rmap && *rmap) {
+#ifdef CONFIG_RFS_ACCEL
+		err = irq_cpu_rmap_add(*rmap, irq->map.virq);
+		if (err)
+			goto err_irq_rmap;
+#endif
+	}
 	if (!mlx5_irq_pool_is_sf_pool(pool))
 		irq_set_name(pool, name, i);
 	else
@@ -225,7 +259,7 @@ struct mlx5_irq *mlx5_irq_alloc(struct mlx5_irq_pool *pool, int i,
 	ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh);
 	snprintf(irq->name, MLX5_MAX_IRQ_NAME,
 		 "%s@pci:%s", name, pci_name(dev->pdev));
-	err = request_irq(irq->irqn, irq_int_handler, 0, irq->name,
+	err = request_irq(irq->map.virq, irq_int_handler, 0, irq->name,
 			  &irq->nh);
 	if (err) {
 		mlx5_core_err(dev, "Failed to request irq. err = %d\n", err);
@@ -236,26 +270,37 @@ struct mlx5_irq *mlx5_irq_alloc(struct mlx5_irq_pool *pool, int i,
 		err = -ENOMEM;
 		goto err_cpumask;
 	}
-	if (affinity) {
-		cpumask_copy(irq->mask, affinity);
-		irq_set_affinity_and_hint(irq->irqn, irq->mask);
+	if (af_desc) {
+		cpumask_copy(irq->mask, &af_desc->mask);
+		irq_set_affinity_and_hint(irq->map.virq, irq->mask);
 	}
 	irq->pool = pool;
 	irq->refcount = 1;
-	irq->index = i;
-	err = xa_err(xa_store(&pool->irqs, irq->index, irq, GFP_KERNEL));
+	irq->map.index = i;
+	err = xa_err(xa_store(&pool->irqs, irq->map.index, irq, GFP_KERNEL));
 	if (err) {
 		mlx5_core_err(dev, "Failed to alloc xa entry for irq(%u). err = %d\n",
-			      irq->index, err);
+			      irq->map.index, err);
 		goto err_xa;
 	}
 	return irq;
 err_xa:
-	irq_update_affinity_hint(irq->irqn, NULL);
+	if (af_desc)
+		irq_update_affinity_hint(irq->map.virq, NULL);
 	free_cpumask_var(irq->mask);
 err_cpumask:
-	free_irq(irq->irqn, &irq->nh);
+	free_irq(irq->map.virq, &irq->nh);
 err_req_irq:
+#ifdef CONFIG_RFS_ACCEL
+	if (i && rmap && *rmap) {
+		free_irq_cpu_rmap(*rmap);
+		*rmap = NULL;
+	}
+err_irq_rmap:
+#endif
+	if (i && pci_msix_can_alloc_dyn(dev->pdev))
+		pci_msix_free_irq(dev->pdev, irq->map);
+err_alloc_irq:
 	kfree(irq);
 	return ERR_PTR(err);
 }
@@ -292,7 +337,7 @@ struct cpumask *mlx5_irq_get_affinity_mask(struct mlx5_irq *irq)
 
 int mlx5_irq_get_index(struct mlx5_irq *irq)
 {
-	return irq->index;
+	return irq->map.index;
 }
 
 /* irq_pool API */
@@ -300,7 +345,8 @@ int mlx5_irq_get_index(struct mlx5_irq *irq)
 /* requesting an irq from a given pool according to given index */
 static struct mlx5_irq *
 irq_pool_request_vector(struct mlx5_irq_pool *pool, int vecidx,
-			struct cpumask *affinity)
+			struct irq_affinity_desc *af_desc,
+			struct cpu_rmap **rmap)
 {
 	struct mlx5_irq *irq;
 
@@ -310,7 +356,7 @@ irq_pool_request_vector(struct mlx5_irq_pool *pool, int vecidx,
 		mlx5_irq_get_locked(irq);
 		goto unlock;
 	}
-	irq = mlx5_irq_alloc(pool, vecidx, affinity);
+	irq = mlx5_irq_alloc(pool, vecidx, af_desc, rmap);
 unlock:
 	mutex_unlock(&pool->lock);
 	return irq;
@@ -337,7 +383,7 @@ struct mlx5_irq_pool *mlx5_irq_pool_get(struct mlx5_core_dev *dev)
 	/* In some configs, there won't be a pool of SFs IRQs. Hence, returning
 	 * the PF IRQs pool in case the SF pool doesn't exist.
 	 */
-	return pool ? pool : irq_table->pf_pool;
+	return pool ? pool : irq_table->pcif_pool;
 }
 
 static struct mlx5_irq_pool *ctrl_irq_pool_get(struct mlx5_core_dev *dev)
@@ -351,7 +397,7 @@ static struct mlx5_irq_pool *ctrl_irq_pool_get(struct mlx5_core_dev *dev)
 	/* In some configs, there won't be a pool of SFs IRQs. Hence, returning
 	 * the PF IRQs pool in case the SF pool doesn't exist.
 	 */
-	return pool ? pool : irq_table->pf_pool;
+	return pool ? pool : irq_table->pcif_pool;
 }
 
 /**
@@ -364,7 +410,7 @@ static void mlx5_irqs_release(struct mlx5_irq **irqs, int nirqs)
 	int i;
 
 	for (i = 0; i < nirqs; i++) {
-		synchronize_irq(irqs[i]->irqn);
+		synchronize_irq(irqs[i]->map.virq);
 		mlx5_irq_put(irqs[i]);
 	}
 }
@@ -387,26 +433,26 @@ void mlx5_ctrl_irq_release(struct mlx5_irq *ctrl_irq)
 struct mlx5_irq *mlx5_ctrl_irq_request(struct mlx5_core_dev *dev)
 {
 	struct mlx5_irq_pool *pool = ctrl_irq_pool_get(dev);
-	cpumask_var_t req_mask;
+	struct irq_affinity_desc af_desc;
 	struct mlx5_irq *irq;
 
-	if (!zalloc_cpumask_var(&req_mask, GFP_KERNEL))
-		return ERR_PTR(-ENOMEM);
-	cpumask_copy(req_mask, cpu_online_mask);
+	cpumask_copy(&af_desc.mask, cpu_online_mask);
+	af_desc.is_managed = false;
 	if (!mlx5_irq_pool_is_sf_pool(pool)) {
-		/* In case we are allocating a control IRQ for PF/VF */
+		/* In case we are allocating a control IRQ from a pci device's pool.
+		 * This can happen also for a SF if the SFs pool is empty.
+		 */
 		if (!pool->xa_num_irqs.max) {
-			cpumask_clear(req_mask);
+			cpumask_clear(&af_desc.mask);
 			/* In case we only have a single IRQ for PF/VF */
-			cpumask_set_cpu(cpumask_first(cpu_online_mask), req_mask);
+			cpumask_set_cpu(cpumask_first(cpu_online_mask), &af_desc.mask);
 		}
-		/* Allocate the IRQ in the last index of the pool */
-		irq = irq_pool_request_vector(pool, pool->xa_num_irqs.max, req_mask);
+		/* Allocate the IRQ in index 0. The vector was already allocated */
+		irq = irq_pool_request_vector(pool, 0, &af_desc, NULL);
 	} else {
-		irq = mlx5_irq_affinity_request(pool, req_mask);
+		irq = mlx5_irq_affinity_request(pool, &af_desc);
 	}
 
-	free_cpumask_var(req_mask);
 	return irq;
 }
 
@@ -415,28 +461,82 @@ struct mlx5_irq *mlx5_ctrl_irq_request(struct mlx5_core_dev *dev)
  * @dev: mlx5 device that requesting the IRQ.
  * @vecidx: vector index of the IRQ. This argument is ignore if affinity is
  * provided.
- * @affinity: cpumask requested for this IRQ.
+ * @af_desc: affinity descriptor for this IRQ.
+ * @rmap: pointer to reverse map pointer for completion interrupts
  *
  * This function returns a pointer to IRQ, or ERR_PTR in case of error.
  */
 struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, u16 vecidx,
-				  struct cpumask *affinity)
+				  struct irq_affinity_desc *af_desc,
+				  struct cpu_rmap **rmap)
 {
 	struct mlx5_irq_table *irq_table = mlx5_irq_table_get(dev);
 	struct mlx5_irq_pool *pool;
 	struct mlx5_irq *irq;
 
-	pool = irq_table->pf_pool;
-	irq = irq_pool_request_vector(pool, vecidx, affinity);
+	pool = irq_table->pcif_pool;
+	irq = irq_pool_request_vector(pool, vecidx, af_desc, rmap);
 	if (IS_ERR(irq))
 		return irq;
 	mlx5_core_dbg(dev, "irq %u mapped to cpu %*pbl, %u EQs on this irq\n",
-		      irq->irqn, cpumask_pr_args(affinity),
+		      irq->map.virq, cpumask_pr_args(&af_desc->mask),
 		      irq->refcount / MLX5_EQ_REFS_PER_IRQ);
 	return irq;
 }
 
 /**
+ * mlx5_msix_alloc - allocate msix interrupt
+ * @dev: mlx5 device from which to request
+ * @handler: interrupt handler
+ * @affdesc: affinity descriptor
+ * @name: interrupt name
+ *
+ * Returns: struct msi_map with result encoded.
+ * Note: the caller must make sure to release the irq by calling
+ *       mlx5_msix_free() if shutdown was initiated.
+ */
+struct msi_map mlx5_msix_alloc(struct mlx5_core_dev *dev,
+			       irqreturn_t (*handler)(int, void *),
+			       const struct irq_affinity_desc *affdesc,
+			       const char *name)
+{
+	struct msi_map map;
+	int err;
+
+	if (!dev->pdev) {
+		map.virq = 0;
+		map.index = -EINVAL;
+		return map;
+	}
+
+	map = pci_msix_alloc_irq_at(dev->pdev, MSI_ANY_INDEX, affdesc);
+	if (!map.virq)
+		return map;
+
+	err = request_irq(map.virq, handler, 0, name, NULL);
+	if (err) {
+		mlx5_core_warn(dev, "err %d\n", err);
+		pci_msix_free_irq(dev->pdev, map);
+		map.virq = 0;
+		map.index = -ENOMEM;
+	}
+	return map;
+}
+EXPORT_SYMBOL(mlx5_msix_alloc);
+
+/**
+ * mlx5_msix_free - free a previously allocated msix interrupt
+ * @dev: mlx5 device associated with interrupt
+ * @map: map previously returned by mlx5_msix_alloc()
+ */
+void mlx5_msix_free(struct mlx5_core_dev *dev, struct msi_map map)
+{
+	free_irq(map.virq, NULL);
+	pci_msix_free_irq(dev->pdev, map);
+}
+EXPORT_SYMBOL(mlx5_msix_free);
+
+/**
  * mlx5_irqs_release_vectors - release one or more IRQs back to the system.
  * @irqs: IRQs to be released.
  * @nirqs: number of IRQs to be released.
@@ -452,6 +552,7 @@ void mlx5_irqs_release_vectors(struct mlx5_irq **irqs, int nirqs)
  * @cpus: CPUs array for binding the IRQs
  * @nirqs: number of IRQs to request.
  * @irqs: an output array of IRQs pointers.
+ * @rmap: pointer to reverse map pointer for completion interrupts
  *
  * Each IRQ is bound to at most 1 CPU.
  * This function is requests nirqs IRQs, starting from @vecidx.
@@ -460,24 +561,22 @@ void mlx5_irqs_release_vectors(struct mlx5_irq **irqs, int nirqs)
  * @nirqs), if successful, or a negative error code in case of an error.
  */
 int mlx5_irqs_request_vectors(struct mlx5_core_dev *dev, u16 *cpus, int nirqs,
-			      struct mlx5_irq **irqs)
+			      struct mlx5_irq **irqs, struct cpu_rmap **rmap)
 {
-	cpumask_var_t req_mask;
+	struct irq_affinity_desc af_desc;
 	struct mlx5_irq *irq;
 	int i;
 
-	if (!zalloc_cpumask_var(&req_mask, GFP_KERNEL))
-		return -ENOMEM;
+	af_desc.is_managed = 1;
 	for (i = 0; i < nirqs; i++) {
-		cpumask_set_cpu(cpus[i], req_mask);
-		irq = mlx5_irq_request(dev, i, req_mask);
+		cpumask_set_cpu(cpus[i], &af_desc.mask);
+		irq = mlx5_irq_request(dev, i + 1, &af_desc, rmap);
 		if (IS_ERR(irq))
 			break;
-		cpumask_clear(req_mask);
+		cpumask_clear(&af_desc.mask);
 		irqs[i] = irq;
 	}
 
-	free_cpumask_var(req_mask);
 	return i ? i : PTR_ERR(irq);
 }
 
@@ -521,7 +620,7 @@ static void irq_pool_free(struct mlx5_irq_pool *pool)
 	kvfree(pool);
 }
 
-static int irq_pools_init(struct mlx5_core_dev *dev, int sf_vec, int pf_vec)
+static int irq_pools_init(struct mlx5_core_dev *dev, int sf_vec, int pcif_vec)
 {
 	struct mlx5_irq_table *table = dev->priv.irq_table;
 	int num_sf_ctrl_by_msix;
@@ -529,12 +628,12 @@ static int irq_pools_init(struct mlx5_core_dev *dev, int sf_vec, int pf_vec)
 	int num_sf_ctrl;
 	int err;
 
-	/* init pf_pool */
-	table->pf_pool = irq_pool_alloc(dev, 0, pf_vec, NULL,
-					MLX5_EQ_SHARE_IRQ_MIN_COMP,
-					MLX5_EQ_SHARE_IRQ_MAX_COMP);
-	if (IS_ERR(table->pf_pool))
-		return PTR_ERR(table->pf_pool);
+	/* init pcif_pool */
+	table->pcif_pool = irq_pool_alloc(dev, 0, pcif_vec, NULL,
+					  MLX5_EQ_SHARE_IRQ_MIN_COMP,
+					  MLX5_EQ_SHARE_IRQ_MAX_COMP);
+	if (IS_ERR(table->pcif_pool))
+		return PTR_ERR(table->pcif_pool);
 	if (!mlx5_sf_max_functions(dev))
 		return 0;
 	if (sf_vec < MLX5_IRQ_VEC_COMP_BASE_SF) {
@@ -548,7 +647,7 @@ static int irq_pools_init(struct mlx5_core_dev *dev, int sf_vec, int pf_vec)
 					  MLX5_SFS_PER_CTRL_IRQ);
 	num_sf_ctrl = min_t(int, num_sf_ctrl_by_msix, num_sf_ctrl_by_sfs);
 	num_sf_ctrl = min_t(int, MLX5_IRQ_CTRL_SF_MAX, num_sf_ctrl);
-	table->sf_ctrl_pool = irq_pool_alloc(dev, pf_vec, num_sf_ctrl,
+	table->sf_ctrl_pool = irq_pool_alloc(dev, pcif_vec, num_sf_ctrl,
 					     "mlx5_sf_ctrl",
 					     MLX5_EQ_SHARE_IRQ_MIN_CTRL,
 					     MLX5_EQ_SHARE_IRQ_MAX_CTRL);
@@ -557,7 +656,7 @@ static int irq_pools_init(struct mlx5_core_dev *dev, int sf_vec, int pf_vec)
 		goto err_pf;
 	}
 	/* init sf_comp_pool */
-	table->sf_comp_pool = irq_pool_alloc(dev, pf_vec + num_sf_ctrl,
+	table->sf_comp_pool = irq_pool_alloc(dev, pcif_vec + num_sf_ctrl,
 					     sf_vec - num_sf_ctrl, "mlx5_sf_comp",
 					     MLX5_EQ_SHARE_IRQ_MIN_COMP,
 					     MLX5_EQ_SHARE_IRQ_MAX_COMP);
@@ -579,7 +678,7 @@ err_irqs_per_cpu:
 err_sf_ctrl:
 	irq_pool_free(table->sf_ctrl_pool);
 err_pf:
-	irq_pool_free(table->pf_pool);
+	irq_pool_free(table->pcif_pool);
 	return err;
 }
 
@@ -589,7 +688,7 @@ static void irq_pools_destroy(struct mlx5_irq_table *table)
 		irq_pool_free(table->sf_comp_pool);
 		irq_pool_free(table->sf_ctrl_pool);
 	}
-	irq_pool_free(table->pf_pool);
+	irq_pool_free(table->pcif_pool);
 }
 
 /* irq_table API */
@@ -620,9 +719,9 @@ void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev)
 
 int mlx5_irq_table_get_num_comp(struct mlx5_irq_table *table)
 {
-	if (!table->pf_pool->xa_num_irqs.max)
+	if (!table->pcif_pool->xa_num_irqs.max)
 		return 1;
-	return table->pf_pool->xa_num_irqs.max - table->pf_pool->xa_num_irqs.min;
+	return table->pcif_pool->xa_num_irqs.max - table->pcif_pool->xa_num_irqs.min;
 }
 
 int mlx5_irq_table_create(struct mlx5_core_dev *dev)
@@ -631,26 +730,30 @@ int mlx5_irq_table_create(struct mlx5_core_dev *dev)
 		      MLX5_CAP_GEN(dev, max_num_eqs) :
 		      1 << MLX5_CAP_GEN(dev, log_max_eq);
 	int total_vec;
-	int pf_vec;
+	int pcif_vec;
+	int req_vec;
 	int err;
+	int n;
 
 	if (mlx5_core_is_sf(dev))
 		return 0;
 
-	pf_vec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 1;
-	pf_vec = min_t(int, pf_vec, num_eqs);
+	pcif_vec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 1;
+	pcif_vec = min_t(int, pcif_vec, num_eqs);
 
-	total_vec = pf_vec;
+	total_vec = pcif_vec;
 	if (mlx5_sf_max_functions(dev))
 		total_vec += MLX5_IRQ_CTRL_SF_MAX +
 			MLX5_COMP_EQS_PER_SF * mlx5_sf_max_functions(dev);
+	total_vec = min_t(int, total_vec, pci_msix_vec_count(dev->pdev));
+	pcif_vec = min_t(int, pcif_vec, pci_msix_vec_count(dev->pdev));
 
-	total_vec = pci_alloc_irq_vectors(dev->pdev, 1, total_vec, PCI_IRQ_MSIX);
-	if (total_vec < 0)
-		return total_vec;
-	pf_vec = min(pf_vec, total_vec);
+	req_vec = pci_msix_can_alloc_dyn(dev->pdev) ? 1 : total_vec;
+	n = pci_alloc_irq_vectors(dev->pdev, 1, req_vec, PCI_IRQ_MSIX);
+	if (n < 0)
+		return n;
 
-	err = irq_pools_init(dev, total_vec - pf_vec, pf_vec);
+	err = irq_pools_init(dev, total_vec - pcif_vec, pcif_vec);
 	if (err)
 		pci_free_irq_vectors(dev->pdev);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.h b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.h
index 5c7e68bee43a..d3a77a0ab848 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.h
@@ -12,6 +12,7 @@
 #define MLX5_EQ_REFS_PER_IRQ (2)
 
 struct mlx5_irq;
+struct cpu_rmap;
 
 struct mlx5_irq_pool {
 	char name[MLX5_MAX_IRQ_NAME - MLX5_MAX_IRQ_IDX_CHARS];
@@ -31,7 +32,8 @@ static inline bool mlx5_irq_pool_is_sf_pool(struct mlx5_irq_pool *pool)
 }
 
 struct mlx5_irq *mlx5_irq_alloc(struct mlx5_irq_pool *pool, int i,
-				const struct cpumask *affinity);
+				struct irq_affinity_desc *af_desc,
+				struct cpu_rmap **rmap);
 int mlx5_irq_get_locked(struct mlx5_irq *irq);
 int mlx5_irq_read_locked(struct mlx5_irq *irq);
 int mlx5_irq_put(struct mlx5_irq *irq);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c
index a1548e6bfb35..0daeb4b72cca 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/port.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c
@@ -1054,3 +1054,154 @@ out:
 	kfree(out);
 	return err;
 }
+
+/* speed in units of 1Mb */
+static const u32 mlx5e_link_speed[MLX5E_LINK_MODES_NUMBER] = {
+	[MLX5E_1000BASE_CX_SGMII] = 1000,
+	[MLX5E_1000BASE_KX]       = 1000,
+	[MLX5E_10GBASE_CX4]       = 10000,
+	[MLX5E_10GBASE_KX4]       = 10000,
+	[MLX5E_10GBASE_KR]        = 10000,
+	[MLX5E_20GBASE_KR2]       = 20000,
+	[MLX5E_40GBASE_CR4]       = 40000,
+	[MLX5E_40GBASE_KR4]       = 40000,
+	[MLX5E_56GBASE_R4]        = 56000,
+	[MLX5E_10GBASE_CR]        = 10000,
+	[MLX5E_10GBASE_SR]        = 10000,
+	[MLX5E_10GBASE_ER]        = 10000,
+	[MLX5E_40GBASE_SR4]       = 40000,
+	[MLX5E_40GBASE_LR4]       = 40000,
+	[MLX5E_50GBASE_SR2]       = 50000,
+	[MLX5E_100GBASE_CR4]      = 100000,
+	[MLX5E_100GBASE_SR4]      = 100000,
+	[MLX5E_100GBASE_KR4]      = 100000,
+	[MLX5E_100GBASE_LR4]      = 100000,
+	[MLX5E_100BASE_TX]        = 100,
+	[MLX5E_1000BASE_T]        = 1000,
+	[MLX5E_10GBASE_T]         = 10000,
+	[MLX5E_25GBASE_CR]        = 25000,
+	[MLX5E_25GBASE_KR]        = 25000,
+	[MLX5E_25GBASE_SR]        = 25000,
+	[MLX5E_50GBASE_CR2]       = 50000,
+	[MLX5E_50GBASE_KR2]       = 50000,
+};
+
+static const u32 mlx5e_ext_link_speed[MLX5E_EXT_LINK_MODES_NUMBER] = {
+	[MLX5E_SGMII_100M] = 100,
+	[MLX5E_1000BASE_X_SGMII] = 1000,
+	[MLX5E_5GBASE_R] = 5000,
+	[MLX5E_10GBASE_XFI_XAUI_1] = 10000,
+	[MLX5E_40GBASE_XLAUI_4_XLPPI_4] = 40000,
+	[MLX5E_25GAUI_1_25GBASE_CR_KR] = 25000,
+	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2] = 50000,
+	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR] = 50000,
+	[MLX5E_CAUI_4_100GBASE_CR4_KR4] = 100000,
+	[MLX5E_100GAUI_2_100GBASE_CR2_KR2] = 100000,
+	[MLX5E_200GAUI_4_200GBASE_CR4_KR4] = 200000,
+	[MLX5E_400GAUI_8] = 400000,
+	[MLX5E_100GAUI_1_100GBASE_CR_KR] = 100000,
+	[MLX5E_200GAUI_2_200GBASE_CR2_KR2] = 200000,
+	[MLX5E_400GAUI_4_400GBASE_CR4_KR4] = 400000,
+};
+
+int mlx5_port_query_eth_proto(struct mlx5_core_dev *dev, u8 port, bool ext,
+			      struct mlx5_port_eth_proto *eproto)
+{
+	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
+	int err;
+
+	if (!eproto)
+		return -EINVAL;
+
+	err = mlx5_query_port_ptys(dev, out, sizeof(out), MLX5_PTYS_EN, port);
+	if (err)
+		return err;
+
+	eproto->cap   = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
+					   eth_proto_capability);
+	eproto->admin = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_admin);
+	eproto->oper  = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);
+	return 0;
+}
+
+bool mlx5_ptys_ext_supported(struct mlx5_core_dev *mdev)
+{
+	struct mlx5_port_eth_proto eproto;
+	int err;
+
+	if (MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet))
+		return true;
+
+	err = mlx5_port_query_eth_proto(mdev, 1, true, &eproto);
+	if (err)
+		return false;
+
+	return !!eproto.cap;
+}
+
+static void mlx5e_port_get_speed_arr(struct mlx5_core_dev *mdev,
+				     const u32 **arr, u32 *size,
+				     bool force_legacy)
+{
+	bool ext = force_legacy ? false : mlx5_ptys_ext_supported(mdev);
+
+	*size = ext ? ARRAY_SIZE(mlx5e_ext_link_speed) :
+		      ARRAY_SIZE(mlx5e_link_speed);
+	*arr  = ext ? mlx5e_ext_link_speed : mlx5e_link_speed;
+}
+
+u32 mlx5_port_ptys2speed(struct mlx5_core_dev *mdev, u32 eth_proto_oper,
+			 bool force_legacy)
+{
+	unsigned long temp = eth_proto_oper;
+	const u32 *table;
+	u32 speed = 0;
+	u32 max_size;
+	int i;
+
+	mlx5e_port_get_speed_arr(mdev, &table, &max_size, force_legacy);
+	i = find_first_bit(&temp, max_size);
+	if (i < max_size)
+		speed = table[i];
+	return speed;
+}
+
+u32 mlx5_port_speed2linkmodes(struct mlx5_core_dev *mdev, u32 speed,
+			      bool force_legacy)
+{
+	u32 link_modes = 0;
+	const u32 *table;
+	u32 max_size;
+	int i;
+
+	mlx5e_port_get_speed_arr(mdev, &table, &max_size, force_legacy);
+	for (i = 0; i < max_size; ++i) {
+		if (table[i] == speed)
+			link_modes |= MLX5E_PROT_MASK(i);
+	}
+	return link_modes;
+}
+
+int mlx5_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed)
+{
+	struct mlx5_port_eth_proto eproto;
+	u32 max_speed = 0;
+	const u32 *table;
+	u32 max_size;
+	bool ext;
+	int err;
+	int i;
+
+	ext = mlx5_ptys_ext_supported(mdev);
+	err = mlx5_port_query_eth_proto(mdev, 1, ext, &eproto);
+	if (err)
+		return err;
+
+	mlx5e_port_get_speed_arr(mdev, &table, &max_size, false);
+	for (i = 0; i < max_size; ++i)
+		if (eproto.cap & MLX5E_PROT_MASK(i))
+			max_speed = max(max_speed, table[i]);
+
+	*speed = max_speed;
+	return 0;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
index a7377619ba6f..e2f26d0bc615 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
@@ -28,7 +28,7 @@ static int mlx5_sf_dev_probe(struct auxiliary_device *adev, const struct auxilia
 	mdev->priv.adev_idx = adev->id;
 	sf_dev->mdev = mdev;
 
-	err = mlx5_mdev_init(mdev, MLX5_DEFAULT_PROF);
+	err = mlx5_mdev_init(mdev, MLX5_SF_PROF);
 	if (err) {
 		mlx5_core_warn(mdev, "mlx5_mdev_init on err=%d\n", err);
 		goto mdev_err;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c
index ee104cf04392..0eb9a8d7f282 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c
@@ -819,14 +819,34 @@ int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher,
 		case DR_ACTION_TYP_TNL_L2_TO_L2:
 			break;
 		case DR_ACTION_TYP_TNL_L3_TO_L2:
-			attr.decap_index = action->rewrite->index;
-			attr.decap_actions = action->rewrite->num_of_actions;
-			attr.decap_with_vlan =
-				attr.decap_actions == WITH_VLAN_NUM_HW_ACTIONS;
+			if (action->rewrite->ptrn && action->rewrite->arg) {
+				attr.decap_index = mlx5dr_arg_get_obj_id(action->rewrite->arg);
+				attr.decap_actions = action->rewrite->ptrn->num_of_actions;
+				attr.decap_pat_idx = action->rewrite->ptrn->index;
+			} else {
+				attr.decap_index = action->rewrite->index;
+				attr.decap_actions = action->rewrite->num_of_actions;
+				attr.decap_with_vlan =
+					attr.decap_actions == WITH_VLAN_NUM_HW_ACTIONS;
+				attr.decap_pat_idx = MLX5DR_INVALID_PATTERN_INDEX;
+			}
 			break;
 		case DR_ACTION_TYP_MODIFY_HDR:
-			attr.modify_index = action->rewrite->index;
-			attr.modify_actions = action->rewrite->num_of_actions;
+			if (action->rewrite->single_action_opt) {
+				attr.modify_actions = action->rewrite->num_of_actions;
+				attr.single_modify_action = action->rewrite->data;
+			} else {
+				if (action->rewrite->ptrn && action->rewrite->arg) {
+					attr.modify_index =
+						mlx5dr_arg_get_obj_id(action->rewrite->arg);
+					attr.modify_actions = action->rewrite->ptrn->num_of_actions;
+					attr.modify_pat_idx = action->rewrite->ptrn->index;
+				} else {
+					attr.modify_index = action->rewrite->index;
+					attr.modify_actions = action->rewrite->num_of_actions;
+					attr.modify_pat_idx = MLX5DR_INVALID_PATTERN_INDEX;
+				}
+			}
 			if (action->rewrite->modify_ttl)
 				dr_action_modify_ttl_adjust(dmn, &attr, rx_rule,
 							    &recalc_cs_required);
@@ -1365,8 +1385,6 @@ out_err:
 	return -EINVAL;
 }
 
-#define ACTION_CACHE_LINE_SIZE 64
-
 static int
 dr_action_create_reformat_action(struct mlx5dr_domain *dmn,
 				 u8 reformat_param_0, u8 reformat_param_1,
@@ -1403,36 +1421,25 @@ dr_action_create_reformat_action(struct mlx5dr_domain *dmn,
 	}
 	case DR_ACTION_TYP_TNL_L3_TO_L2:
 	{
-		u8 hw_actions[ACTION_CACHE_LINE_SIZE] = {};
+		u8 hw_actions[DR_ACTION_CACHE_LINE_SIZE] = {};
 		int ret;
 
 		ret = mlx5dr_ste_set_action_decap_l3_list(dmn->ste_ctx,
 							  data, data_sz,
 							  hw_actions,
-							  ACTION_CACHE_LINE_SIZE,
+							  DR_ACTION_CACHE_LINE_SIZE,
 							  &action->rewrite->num_of_actions);
 		if (ret) {
 			mlx5dr_dbg(dmn, "Failed creating decap l3 action list\n");
 			return ret;
 		}
 
-		action->rewrite->chunk = mlx5dr_icm_alloc_chunk(dmn->action_icm_pool,
-								DR_CHUNK_SIZE_8);
-		if (!action->rewrite->chunk) {
-			mlx5dr_dbg(dmn, "Failed allocating modify header chunk\n");
-			return -ENOMEM;
-		}
-
-		action->rewrite->data = (void *)hw_actions;
-		action->rewrite->index = (mlx5dr_icm_pool_get_chunk_icm_addr
-					  (action->rewrite->chunk) -
-					 dmn->info.caps.hdr_modify_icm_addr) /
-					 ACTION_CACHE_LINE_SIZE;
+		action->rewrite->data = hw_actions;
+		action->rewrite->dmn = dmn;
 
-		ret = mlx5dr_send_postsend_action(dmn, action);
+		ret = mlx5dr_ste_alloc_modify_hdr(action);
 		if (ret) {
-			mlx5dr_dbg(dmn, "Writing decap l3 actions to ICM failed\n");
-			mlx5dr_icm_free_chunk(action->rewrite->chunk);
+			mlx5dr_dbg(dmn, "Failed preparing reformat data\n");
 			return ret;
 		}
 		return 0;
@@ -1963,7 +1970,6 @@ static int dr_action_create_modify_action(struct mlx5dr_domain *dmn,
 					  __be64 actions[],
 					  struct mlx5dr_action *action)
 {
-	struct mlx5dr_icm_chunk *chunk;
 	u32 max_hw_actions;
 	u32 num_hw_actions;
 	u32 num_sw_actions;
@@ -1980,15 +1986,9 @@ static int dr_action_create_modify_action(struct mlx5dr_domain *dmn,
 		return -EINVAL;
 	}
 
-	chunk = mlx5dr_icm_alloc_chunk(dmn->action_icm_pool, DR_CHUNK_SIZE_16);
-	if (!chunk)
-		return -ENOMEM;
-
 	hw_actions = kcalloc(1, max_hw_actions * DR_MODIFY_ACTION_SIZE, GFP_KERNEL);
-	if (!hw_actions) {
-		ret = -ENOMEM;
-		goto free_chunk;
-	}
+	if (!hw_actions)
+		return -ENOMEM;
 
 	ret = dr_actions_convert_modify_header(action,
 					       max_hw_actions,
@@ -2000,24 +2000,24 @@ static int dr_action_create_modify_action(struct mlx5dr_domain *dmn,
 	if (ret)
 		goto free_hw_actions;
 
-	action->rewrite->chunk = chunk;
 	action->rewrite->modify_ttl = modify_ttl;
 	action->rewrite->data = (u8 *)hw_actions;
 	action->rewrite->num_of_actions = num_hw_actions;
-	action->rewrite->index = (mlx5dr_icm_pool_get_chunk_icm_addr(chunk) -
-				  dmn->info.caps.hdr_modify_icm_addr) /
-				  ACTION_CACHE_LINE_SIZE;
 
-	ret = mlx5dr_send_postsend_action(dmn, action);
-	if (ret)
-		goto free_hw_actions;
+	if (num_hw_actions == 1 &&
+	    dmn->info.caps.sw_format_ver >= MLX5_STEERING_FORMAT_CONNECTX_6DX) {
+		action->rewrite->single_action_opt = true;
+	} else {
+		action->rewrite->single_action_opt = false;
+		ret = mlx5dr_ste_alloc_modify_hdr(action);
+		if (ret)
+			goto free_hw_actions;
+	}
 
 	return 0;
 
 free_hw_actions:
 	kfree(hw_actions);
-free_chunk:
-	mlx5dr_icm_free_chunk(chunk);
 	return ret;
 }
 
@@ -2162,7 +2162,8 @@ int mlx5dr_action_destroy(struct mlx5dr_action *action)
 		refcount_dec(&action->reformat->dmn->refcount);
 		break;
 	case DR_ACTION_TYP_TNL_L3_TO_L2:
-		mlx5dr_icm_free_chunk(action->rewrite->chunk);
+		mlx5dr_ste_free_modify_hdr(action);
+		kfree(action->rewrite->data);
 		refcount_dec(&action->rewrite->dmn->refcount);
 		break;
 	case DR_ACTION_TYP_L2_TO_TNL_L2:
@@ -2173,7 +2174,8 @@ int mlx5dr_action_destroy(struct mlx5dr_action *action)
 		refcount_dec(&action->reformat->dmn->refcount);
 		break;
 	case DR_ACTION_TYP_MODIFY_HDR:
-		mlx5dr_icm_free_chunk(action->rewrite->chunk);
+		if (!action->rewrite->single_action_opt)
+			mlx5dr_ste_free_modify_hdr(action);
 		kfree(action->rewrite->data);
 		refcount_dec(&action->rewrite->dmn->refcount);
 		break;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_arg.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_arg.c
new file mode 100644
index 000000000000..01ed6442095d
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_arg.c
@@ -0,0 +1,273 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+#include "dr_types.h"
+
+#define DR_ICM_MODIFY_HDR_GRANULARITY_4K 12
+
+/* modify-header arg pool */
+enum dr_arg_chunk_size {
+	DR_ARG_CHUNK_SIZE_1,
+	DR_ARG_CHUNK_SIZE_MIN = DR_ARG_CHUNK_SIZE_1, /* keep updated when changing */
+	DR_ARG_CHUNK_SIZE_2,
+	DR_ARG_CHUNK_SIZE_3,
+	DR_ARG_CHUNK_SIZE_4,
+	DR_ARG_CHUNK_SIZE_MAX,
+};
+
+/* argument pool area */
+struct dr_arg_pool {
+	enum dr_arg_chunk_size log_chunk_size;
+	struct mlx5dr_domain *dmn;
+	struct list_head free_list;
+	struct mutex mutex; /* protect arg pool */
+};
+
+struct mlx5dr_arg_mgr {
+	struct mlx5dr_domain *dmn;
+	struct dr_arg_pool *pools[DR_ARG_CHUNK_SIZE_MAX];
+};
+
+static int dr_arg_pool_alloc_objs(struct dr_arg_pool *pool)
+{
+	struct mlx5dr_arg_obj *arg_obj, *tmp_arg;
+	struct list_head cur_list;
+	u16 object_range;
+	int num_of_objects;
+	u32 obj_id = 0;
+	int i, ret;
+
+	INIT_LIST_HEAD(&cur_list);
+
+	object_range =
+		pool->dmn->info.caps.log_header_modify_argument_granularity;
+
+	object_range =
+		max_t(u32, pool->dmn->info.caps.log_header_modify_argument_granularity,
+		      DR_ICM_MODIFY_HDR_GRANULARITY_4K);
+	object_range =
+		min_t(u32, pool->dmn->info.caps.log_header_modify_argument_max_alloc,
+		      object_range);
+
+	if (pool->log_chunk_size > object_range) {
+		mlx5dr_err(pool->dmn, "Required chunk size (%d) is not supported\n",
+			   pool->log_chunk_size);
+		return -ENOMEM;
+	}
+
+	num_of_objects = (1 << (object_range - pool->log_chunk_size));
+	/* Only one devx object per range */
+	ret = mlx5dr_cmd_create_modify_header_arg(pool->dmn->mdev,
+						  object_range,
+						  pool->dmn->pdn,
+						  &obj_id);
+	if (ret) {
+		mlx5dr_err(pool->dmn, "failed allocating object with range: %d:\n",
+			   object_range);
+		return -EAGAIN;
+	}
+
+	for (i = 0; i < num_of_objects; i++) {
+		arg_obj = kzalloc(sizeof(*arg_obj), GFP_KERNEL);
+		if (!arg_obj) {
+			ret = -ENOMEM;
+			goto clean_arg_obj;
+		}
+
+		arg_obj->log_chunk_size = pool->log_chunk_size;
+
+		list_add_tail(&arg_obj->list_node, &cur_list);
+
+		arg_obj->obj_id = obj_id;
+		arg_obj->obj_offset = i * (1 << pool->log_chunk_size);
+	}
+	list_splice_tail_init(&cur_list, &pool->free_list);
+
+	return 0;
+
+clean_arg_obj:
+	mlx5dr_cmd_destroy_modify_header_arg(pool->dmn->mdev, obj_id);
+	list_for_each_entry_safe(arg_obj, tmp_arg, &cur_list, list_node) {
+		list_del(&arg_obj->list_node);
+		kfree(arg_obj);
+	}
+	return ret;
+}
+
+static struct mlx5dr_arg_obj *dr_arg_pool_get_arg_obj(struct dr_arg_pool *pool)
+{
+	struct mlx5dr_arg_obj *arg_obj = NULL;
+	int ret;
+
+	mutex_lock(&pool->mutex);
+	if (list_empty(&pool->free_list)) {
+		ret = dr_arg_pool_alloc_objs(pool);
+		if (ret)
+			goto out;
+	}
+
+	arg_obj = list_first_entry_or_null(&pool->free_list,
+					   struct mlx5dr_arg_obj,
+					   list_node);
+	WARN(!arg_obj, "couldn't get dr arg obj from pool");
+
+	if (arg_obj)
+		list_del_init(&arg_obj->list_node);
+
+out:
+	mutex_unlock(&pool->mutex);
+	return arg_obj;
+}
+
+static void dr_arg_pool_put_arg_obj(struct dr_arg_pool *pool,
+				    struct mlx5dr_arg_obj *arg_obj)
+{
+	mutex_lock(&pool->mutex);
+	list_add(&arg_obj->list_node, &pool->free_list);
+	mutex_unlock(&pool->mutex);
+}
+
+static struct dr_arg_pool *dr_arg_pool_create(struct mlx5dr_domain *dmn,
+					      enum dr_arg_chunk_size chunk_size)
+{
+	struct dr_arg_pool *pool;
+
+	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
+	if (!pool)
+		return NULL;
+
+	pool->dmn = dmn;
+
+	INIT_LIST_HEAD(&pool->free_list);
+	mutex_init(&pool->mutex);
+
+	pool->log_chunk_size = chunk_size;
+	if (dr_arg_pool_alloc_objs(pool))
+		goto free_pool;
+
+	return pool;
+
+free_pool:
+	kfree(pool);
+
+	return NULL;
+}
+
+static void dr_arg_pool_destroy(struct dr_arg_pool *pool)
+{
+	struct mlx5dr_arg_obj *arg_obj, *tmp_arg;
+
+	list_for_each_entry_safe(arg_obj, tmp_arg, &pool->free_list, list_node) {
+		list_del(&arg_obj->list_node);
+		if (!arg_obj->obj_offset) /* the first in range */
+			mlx5dr_cmd_destroy_modify_header_arg(pool->dmn->mdev, arg_obj->obj_id);
+		kfree(arg_obj);
+	}
+
+	mutex_destroy(&pool->mutex);
+	kfree(pool);
+}
+
+static enum dr_arg_chunk_size dr_arg_get_chunk_size(u16 num_of_actions)
+{
+	if (num_of_actions <= 8)
+		return DR_ARG_CHUNK_SIZE_1;
+	if (num_of_actions <= 16)
+		return DR_ARG_CHUNK_SIZE_2;
+	if (num_of_actions <= 32)
+		return DR_ARG_CHUNK_SIZE_3;
+	if (num_of_actions <= 64)
+		return DR_ARG_CHUNK_SIZE_4;
+
+	return DR_ARG_CHUNK_SIZE_MAX;
+}
+
+u32 mlx5dr_arg_get_obj_id(struct mlx5dr_arg_obj *arg_obj)
+{
+	return (arg_obj->obj_id + arg_obj->obj_offset);
+}
+
+struct mlx5dr_arg_obj *mlx5dr_arg_get_obj(struct mlx5dr_arg_mgr *mgr,
+					  u16 num_of_actions,
+					  u8 *data)
+{
+	u32 size = dr_arg_get_chunk_size(num_of_actions);
+	struct mlx5dr_arg_obj *arg_obj;
+	int ret;
+
+	if (size >= DR_ARG_CHUNK_SIZE_MAX)
+		return NULL;
+
+	arg_obj = dr_arg_pool_get_arg_obj(mgr->pools[size]);
+	if (!arg_obj) {
+		mlx5dr_err(mgr->dmn, "Failed allocating args object for modify header\n");
+		return NULL;
+	}
+
+	/* write it into the hw */
+	ret = mlx5dr_send_postsend_args(mgr->dmn,
+					mlx5dr_arg_get_obj_id(arg_obj),
+					num_of_actions, data);
+	if (ret) {
+		mlx5dr_err(mgr->dmn, "Failed writing args object\n");
+		goto put_obj;
+	}
+
+	return arg_obj;
+
+put_obj:
+	mlx5dr_arg_put_obj(mgr, arg_obj);
+	return NULL;
+}
+
+void mlx5dr_arg_put_obj(struct mlx5dr_arg_mgr *mgr,
+			struct mlx5dr_arg_obj *arg_obj)
+{
+	dr_arg_pool_put_arg_obj(mgr->pools[arg_obj->log_chunk_size], arg_obj);
+}
+
+struct mlx5dr_arg_mgr*
+mlx5dr_arg_mgr_create(struct mlx5dr_domain *dmn)
+{
+	struct mlx5dr_arg_mgr *pool_mgr;
+	int i;
+
+	if (!mlx5dr_domain_is_support_ptrn_arg(dmn))
+		return NULL;
+
+	pool_mgr = kzalloc(sizeof(*pool_mgr), GFP_KERNEL);
+	if (!pool_mgr)
+		return NULL;
+
+	pool_mgr->dmn = dmn;
+
+	for (i = 0; i < DR_ARG_CHUNK_SIZE_MAX; i++) {
+		pool_mgr->pools[i] = dr_arg_pool_create(dmn, i);
+		if (!pool_mgr->pools[i])
+			goto clean_pools;
+	}
+
+	return pool_mgr;
+
+clean_pools:
+	for (i--; i >= 0; i--)
+		dr_arg_pool_destroy(pool_mgr->pools[i]);
+
+	kfree(pool_mgr);
+	return NULL;
+}
+
+void mlx5dr_arg_mgr_destroy(struct mlx5dr_arg_mgr *mgr)
+{
+	struct dr_arg_pool **pools;
+	int i;
+
+	if (!mgr)
+		return;
+
+	pools = mgr->pools;
+	for (i = 0; i < DR_ARG_CHUNK_SIZE_MAX; i++)
+		dr_arg_pool_destroy(pools[i]);
+
+	kfree(mgr);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c
index 07b6a6dcb92f..3835ba3f4dda 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c
@@ -132,6 +132,17 @@ int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev,
 
 	caps->isolate_vl_tc = MLX5_CAP_GEN(mdev, isolate_vl_tc_new);
 
+	caps->support_modify_argument =
+		MLX5_CAP_GEN_64(mdev, general_obj_types) &
+		MLX5_GENERAL_OBJ_TYPES_CAP_HEADER_MODIFY_ARGUMENT;
+
+	if (caps->support_modify_argument) {
+		caps->log_header_modify_argument_granularity =
+			MLX5_CAP_GEN(mdev, log_header_modify_argument_granularity);
+		caps->log_header_modify_argument_max_alloc =
+			MLX5_CAP_GEN(mdev, log_header_modify_argument_max_alloc);
+	}
+
 	/* geneve_tlv_option_0_exist is the indication of
 	 * STE support for lookup type flex_parser_ok
 	 */
@@ -200,6 +211,12 @@ int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev,
 	caps->hdr_modify_icm_addr =
 		MLX5_CAP64_DEV_MEM(mdev, header_modify_sw_icm_start_address);
 
+	caps->log_modify_pattern_icm_size =
+		MLX5_CAP_DEV_MEM(mdev, log_header_modify_pattern_sw_icm_size);
+
+	caps->hdr_modify_pattern_icm_addr =
+		MLX5_CAP64_DEV_MEM(mdev, header_modify_pattern_sw_icm_start_address);
+
 	caps->roce_min_src_udp = MLX5_CAP_ROCE(mdev, r_roce_min_src_udp_port);
 
 	caps->is_ecpf = mlx5_core_is_ecpf_esw_manager(mdev);
@@ -676,6 +693,49 @@ int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num,
 	return 0;
 }
 
+int mlx5dr_cmd_create_modify_header_arg(struct mlx5_core_dev *dev,
+					u16 log_obj_range, u32 pd,
+					u32 *obj_id)
+{
+	u32 in[MLX5_ST_SZ_DW(create_modify_header_arg_in)] = {};
+	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
+	void *attr;
+	int ret;
+
+	attr = MLX5_ADDR_OF(create_modify_header_arg_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, attr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, attr, obj_type,
+		 MLX5_OBJ_TYPE_HEADER_MODIFY_ARGUMENT);
+	MLX5_SET(general_obj_in_cmd_hdr, attr,
+		 op_param.create.log_obj_range, log_obj_range);
+
+	attr = MLX5_ADDR_OF(create_modify_header_arg_in, in, arg);
+	MLX5_SET(modify_header_arg, attr, access_pd, pd);
+
+	ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (ret)
+		return ret;
+
+	*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return 0;
+}
+
+void mlx5dr_cmd_destroy_modify_header_arg(struct mlx5_core_dev *dev,
+					  u32 obj_id)
+{
+	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
+	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
+
+	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
+		 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
+		 MLX5_OBJ_TYPE_HEADER_MODIFY_ARGUMENT);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);
+
+	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+
 static int mlx5dr_cmd_set_extended_dest(struct mlx5_core_dev *dev,
 					struct mlx5dr_cmd_fte_info *fte,
 					bool *extended_dest)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c
index db81d881d38e..7e36e1062139 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c
@@ -4,6 +4,7 @@
 #include <linux/debugfs.h>
 #include <linux/kernel.h>
 #include <linux/seq_file.h>
+#include <linux/version.h>
 #include "dr_types.h"
 
 #define DR_DBG_PTR_TO_ID(p) ((u64)(uintptr_t)(p) & 0xFFFFFFFFULL)
@@ -140,10 +141,33 @@ dr_dump_rule_action_mem(struct seq_file *file, const u64 rule_id,
 			   action->flow_tag->flow_tag);
 		break;
 	case DR_ACTION_TYP_MODIFY_HDR:
-		seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n",
+	{
+		struct mlx5dr_ptrn_obj *ptrn = action->rewrite->ptrn;
+		struct mlx5dr_arg_obj *arg = action->rewrite->arg;
+		u8 *rewrite_data = action->rewrite->data;
+		bool ptrn_arg;
+		int i;
+
+		ptrn_arg = !action->rewrite->single_action_opt && ptrn && arg;
+
+		seq_printf(file, "%d,0x%llx,0x%llx,0x%x,%d,0x%x,0x%x,0x%x",
 			   DR_DUMP_REC_TYPE_ACTION_MODIFY_HDR, action_id,
-			   rule_id, action->rewrite->index);
+			   rule_id, action->rewrite->index,
+			   action->rewrite->single_action_opt,
+			   ptrn_arg ? action->rewrite->num_of_actions : 0,
+			   ptrn_arg ? ptrn->index : 0,
+			   ptrn_arg ? mlx5dr_arg_get_obj_id(arg) : 0);
+
+		if (ptrn_arg) {
+			for (i = 0; i < action->rewrite->num_of_actions; i++) {
+				seq_printf(file, ",0x%016llx",
+					   be64_to_cpu(((__be64 *)rewrite_data)[i]));
+			}
+		}
+
+		seq_puts(file, "\n");
 		break;
+	}
 	case DR_ACTION_TYP_VPORT:
 		seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n",
 			   DR_DUMP_REC_TYPE_ACTION_VPORT, action_id, rule_id,
@@ -157,7 +181,10 @@ dr_dump_rule_action_mem(struct seq_file *file, const u64 rule_id,
 	case DR_ACTION_TYP_TNL_L3_TO_L2:
 		seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n",
 			   DR_DUMP_REC_TYPE_ACTION_DECAP_L3, action_id,
-			   rule_id, action->rewrite->index);
+			   rule_id,
+			   (action->rewrite->ptrn && action->rewrite->arg) ?
+			   mlx5dr_arg_get_obj_id(action->rewrite->arg) :
+			   action->rewrite->index);
 		break;
 	case DR_ACTION_TYP_L2_TO_TNL_L2:
 		seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n",
@@ -606,9 +633,18 @@ dr_dump_domain(struct seq_file *file, struct mlx5dr_domain *dmn)
 	u64 domain_id = DR_DBG_PTR_TO_ID(dmn);
 	int ret;
 
-	seq_printf(file, "%d,0x%llx,%d,0%x,%d,%s\n", DR_DUMP_REC_TYPE_DOMAIN,
+	seq_printf(file, "%d,0x%llx,%d,0%x,%d,%u.%u.%u,%s,%d,%u,%u,%u\n",
+		   DR_DUMP_REC_TYPE_DOMAIN,
 		   domain_id, dmn->type, dmn->info.caps.gvmi,
-		   dmn->info.supp_sw_steering, pci_name(dmn->mdev->pdev));
+		   dmn->info.supp_sw_steering,
+		   /* package version */
+		   LINUX_VERSION_MAJOR, LINUX_VERSION_PATCHLEVEL,
+		   LINUX_VERSION_SUBLEVEL,
+		   pci_name(dmn->mdev->pdev),
+		   0, /* domain flags */
+		   dmn->num_buddies[DR_ICM_TYPE_STE],
+		   dmn->num_buddies[DR_ICM_TYPE_MODIFY_ACTION],
+		   dmn->num_buddies[DR_ICM_TYPE_MODIFY_HDR_PTRN]);
 
 	ret = dr_dump_domain_info(file, &dmn->info, domain_id);
 	if (ret < 0)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c
index 5b8bb2ca31e6..9a2dfe6ebe31 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c
@@ -10,6 +10,46 @@
 	 ((dmn)->info.caps.dmn_type##_sw_owner_v2 &&	\
 	  (dmn)->info.caps.sw_format_ver <= MLX5_STEERING_FORMAT_CONNECTX_7))
 
+bool mlx5dr_domain_is_support_ptrn_arg(struct mlx5dr_domain *dmn)
+{
+	return dmn->info.caps.sw_format_ver >= MLX5_STEERING_FORMAT_CONNECTX_6DX &&
+	       dmn->info.caps.support_modify_argument;
+}
+
+static int dr_domain_init_modify_header_resources(struct mlx5dr_domain *dmn)
+{
+	if (!mlx5dr_domain_is_support_ptrn_arg(dmn))
+		return 0;
+
+	dmn->ptrn_mgr = mlx5dr_ptrn_mgr_create(dmn);
+	if (!dmn->ptrn_mgr) {
+		mlx5dr_err(dmn, "Couldn't create ptrn_mgr\n");
+		return -ENOMEM;
+	}
+
+	/* create argument pool */
+	dmn->arg_mgr = mlx5dr_arg_mgr_create(dmn);
+	if (!dmn->arg_mgr) {
+		mlx5dr_err(dmn, "Couldn't create arg_mgr\n");
+		goto free_modify_header_pattern;
+	}
+
+	return 0;
+
+free_modify_header_pattern:
+	mlx5dr_ptrn_mgr_destroy(dmn->ptrn_mgr);
+	return -ENOMEM;
+}
+
+static void dr_domain_destroy_modify_header_resources(struct mlx5dr_domain *dmn)
+{
+	if (!mlx5dr_domain_is_support_ptrn_arg(dmn))
+		return;
+
+	mlx5dr_arg_mgr_destroy(dmn->arg_mgr);
+	mlx5dr_ptrn_mgr_destroy(dmn->ptrn_mgr);
+}
+
 static void dr_domain_init_csum_recalc_fts(struct mlx5dr_domain *dmn)
 {
 	/* Per vport cached FW FT for checksum recalculation, this
@@ -149,14 +189,22 @@ static int dr_domain_init_resources(struct mlx5dr_domain *dmn)
 		goto clean_uar;
 	}
 
+	ret = dr_domain_init_modify_header_resources(dmn);
+	if (ret) {
+		mlx5dr_err(dmn, "Couldn't create modify-header-resources\n");
+		goto clean_mem_resources;
+	}
+
 	ret = mlx5dr_send_ring_alloc(dmn);
 	if (ret) {
 		mlx5dr_err(dmn, "Couldn't create send-ring\n");
-		goto clean_mem_resources;
+		goto clean_modify_hdr;
 	}
 
 	return 0;
 
+clean_modify_hdr:
+	dr_domain_destroy_modify_header_resources(dmn);
 clean_mem_resources:
 	dr_domain_uninit_mem_resources(dmn);
 clean_uar:
@@ -170,6 +218,7 @@ clean_pd:
 static void dr_domain_uninit_resources(struct mlx5dr_domain *dmn)
 {
 	mlx5dr_send_ring_free(dmn, dmn->send_ring);
+	dr_domain_destroy_modify_header_resources(dmn);
 	dr_domain_uninit_mem_resources(dmn);
 	mlx5_put_uars_page(dmn->mdev, dmn->uar);
 	mlx5_core_dealloc_pd(dmn->mdev, dmn->pdn);
@@ -215,7 +264,7 @@ static int dr_domain_query_vport(struct mlx5dr_domain *dmn,
 	return 0;
 }
 
-static int dr_domain_query_esw_mngr(struct mlx5dr_domain *dmn)
+static int dr_domain_query_esw_mgr(struct mlx5dr_domain *dmn)
 {
 	return dr_domain_query_vport(dmn, 0, false,
 				     &dmn->info.caps.vports.esw_manager_caps);
@@ -321,7 +370,7 @@ static int dr_domain_query_fdb_caps(struct mlx5_core_dev *mdev,
 	 * vports (vport 0, VFs and SFs) will be queried dynamically.
 	 */
 
-	ret = dr_domain_query_esw_mngr(dmn);
+	ret = dr_domain_query_esw_mgr(dmn);
 	if (ret) {
 		mlx5dr_err(dmn, "Failed to query eswitch manager vport caps (err: %d)", ret);
 		goto free_vports_caps_xa;
@@ -435,6 +484,9 @@ mlx5dr_domain_create(struct mlx5_core_dev *mdev, enum mlx5dr_domain_type type)
 	dmn->info.max_log_action_icm_sz = DR_CHUNK_SIZE_4K;
 	dmn->info.max_log_sw_icm_sz = min_t(u32, DR_CHUNK_SIZE_1024K,
 					    dmn->info.caps.log_icm_size);
+	dmn->info.max_log_modify_hdr_pattern_icm_sz =
+		min_t(u32, DR_CHUNK_SIZE_4K,
+		      dmn->info.caps.log_modify_pattern_icm_size);
 
 	if (!dmn->info.supp_sw_steering) {
 		mlx5dr_err(dmn, "SW steering is not supported\n");
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c
index 3eb6719bc8eb..0b5af9f3f605 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c
@@ -4,7 +4,9 @@
 #include "dr_types.h"
 
 #define DR_ICM_MODIFY_HDR_ALIGN_BASE 64
-#define DR_ICM_POOL_HOT_MEMORY_FRACTION 4
+#define DR_ICM_POOL_STE_HOT_MEM_PERCENT 25
+#define DR_ICM_POOL_MODIFY_HDR_PTRN_HOT_MEM_PERCENT 50
+#define DR_ICM_POOL_MODIFY_ACTION_HOT_MEM_PERCENT 90
 
 struct mlx5dr_icm_hot_chunk {
 	struct mlx5dr_icm_buddy_mem *buddy_mem;
@@ -29,6 +31,8 @@ struct mlx5dr_icm_pool {
 	struct mlx5dr_icm_hot_chunk *hot_chunks_arr;
 	u32 hot_chunks_num;
 	u64 hot_memory_size;
+	/* hot memory size threshold for triggering sync */
+	u64 th;
 };
 
 struct mlx5dr_icm_dm {
@@ -107,9 +111,9 @@ static struct mlx5dr_icm_mr *
 dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool)
 {
 	struct mlx5_core_dev *mdev = pool->dmn->mdev;
-	enum mlx5_sw_icm_type dm_type;
+	enum mlx5_sw_icm_type dm_type = 0;
 	struct mlx5dr_icm_mr *icm_mr;
-	size_t log_align_base;
+	size_t log_align_base = 0;
 	int err;
 
 	icm_mr = kvzalloc(sizeof(*icm_mr), GFP_KERNEL);
@@ -121,14 +125,25 @@ dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool)
 	icm_mr->dm.length = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
 							       pool->icm_type);
 
-	if (pool->icm_type == DR_ICM_TYPE_STE) {
+	switch (pool->icm_type) {
+	case DR_ICM_TYPE_STE:
 		dm_type = MLX5_SW_ICM_TYPE_STEERING;
 		log_align_base = ilog2(icm_mr->dm.length);
-	} else {
+		break;
+	case DR_ICM_TYPE_MODIFY_ACTION:
 		dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY;
 		/* Align base is 64B */
 		log_align_base = ilog2(DR_ICM_MODIFY_HDR_ALIGN_BASE);
+		break;
+	case DR_ICM_TYPE_MODIFY_HDR_PTRN:
+		dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN;
+		/* Align base is 64B */
+		log_align_base = ilog2(DR_ICM_MODIFY_HDR_ALIGN_BASE);
+		break;
+	default:
+		WARN_ON(pool->icm_type);
 	}
+
 	icm_mr->dm.type = dm_type;
 
 	err = mlx5_dm_sw_icm_alloc(mdev, icm_mr->dm.type, icm_mr->dm.length,
@@ -273,6 +288,8 @@ static int dr_icm_buddy_create(struct mlx5dr_icm_pool *pool)
 	/* add it to the -start- of the list in order to search in it first */
 	list_add(&buddy->list_node, &pool->buddy_mem_list);
 
+	pool->dmn->num_buddies[pool->icm_type]++;
+
 	return 0;
 
 err_cleanup_buddy:
@@ -286,13 +303,17 @@ free_mr:
 
 static void dr_icm_buddy_destroy(struct mlx5dr_icm_buddy_mem *buddy)
 {
+	enum mlx5dr_icm_type icm_type = buddy->pool->icm_type;
+
 	dr_icm_pool_mr_destroy(buddy->icm_mr);
 
 	mlx5dr_buddy_cleanup(buddy);
 
-	if (buddy->pool->icm_type == DR_ICM_TYPE_STE)
+	if (icm_type == DR_ICM_TYPE_STE)
 		dr_icm_buddy_cleanup_ste_cache(buddy);
 
+	buddy->pool->dmn->num_buddies[icm_type]--;
+
 	kvfree(buddy);
 }
 
@@ -319,15 +340,7 @@ dr_icm_chunk_init(struct mlx5dr_icm_chunk *chunk,
 
 static bool dr_icm_pool_is_sync_required(struct mlx5dr_icm_pool *pool)
 {
-	int allow_hot_size;
-
-	/* sync when hot memory reaches a certain fraction of the pool size */
-	allow_hot_size =
-		mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
-						   pool->icm_type) /
-		DR_ICM_POOL_HOT_MEMORY_FRACTION;
-
-	return pool->hot_memory_size > allow_hot_size;
+	return pool->hot_memory_size > pool->th;
 }
 
 static void dr_icm_pool_clear_hot_chunks_arr(struct mlx5dr_icm_pool *pool)
@@ -492,14 +505,9 @@ void mlx5dr_icm_pool_free_htbl(struct mlx5dr_icm_pool *pool, struct mlx5dr_ste_h
 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn,
 					       enum mlx5dr_icm_type icm_type)
 {
-	u32 num_of_chunks, entry_size, max_hot_size;
-	enum mlx5dr_icm_chunk_size max_log_chunk_sz;
+	u32 num_of_chunks, entry_size;
 	struct mlx5dr_icm_pool *pool;
-
-	if (icm_type == DR_ICM_TYPE_STE)
-		max_log_chunk_sz = dmn->info.max_log_sw_icm_sz;
-	else
-		max_log_chunk_sz = dmn->info.max_log_action_icm_sz;
+	u32 max_hot_size = 0;
 
 	pool = kvzalloc(sizeof(*pool), GFP_KERNEL);
 	if (!pool)
@@ -507,20 +515,38 @@ struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn,
 
 	pool->dmn = dmn;
 	pool->icm_type = icm_type;
-	pool->max_log_chunk_sz = max_log_chunk_sz;
 	pool->chunks_kmem_cache = dmn->chunks_kmem_cache;
 
 	INIT_LIST_HEAD(&pool->buddy_mem_list);
-
 	mutex_init(&pool->mutex);
 
-	entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(pool->icm_type);
+	switch (icm_type) {
+	case DR_ICM_TYPE_STE:
+		pool->max_log_chunk_sz = dmn->info.max_log_sw_icm_sz;
+		max_hot_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
+								  pool->icm_type) *
+			       DR_ICM_POOL_STE_HOT_MEM_PERCENT / 100;
+		break;
+	case DR_ICM_TYPE_MODIFY_ACTION:
+		pool->max_log_chunk_sz = dmn->info.max_log_action_icm_sz;
+		max_hot_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
+								  pool->icm_type) *
+			       DR_ICM_POOL_MODIFY_ACTION_HOT_MEM_PERCENT / 100;
+		break;
+	case DR_ICM_TYPE_MODIFY_HDR_PTRN:
+		pool->max_log_chunk_sz = dmn->info.max_log_modify_hdr_pattern_icm_sz;
+		max_hot_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
+								  pool->icm_type) *
+			       DR_ICM_POOL_MODIFY_HDR_PTRN_HOT_MEM_PERCENT / 100;
+		break;
+	default:
+		WARN_ON(icm_type);
+	}
 
-	max_hot_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
-							  pool->icm_type) /
-		       DR_ICM_POOL_HOT_MEMORY_FRACTION;
+	entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(pool->icm_type);
 
 	num_of_chunks = DIV_ROUND_UP(max_hot_size, entry_size) + 1;
+	pool->th = max_hot_size;
 
 	pool->hot_chunks_arr = kvcalloc(num_of_chunks,
 					sizeof(struct mlx5dr_icm_hot_chunk),
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ptrn.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ptrn.c
new file mode 100644
index 000000000000..13e06a6a6b22
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ptrn.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+#include "dr_types.h"
+#include "mlx5_ifc_dr_ste_v1.h"
+
+enum dr_ptrn_modify_hdr_action_id {
+	DR_PTRN_MODIFY_HDR_ACTION_ID_NOP = 0x00,
+	DR_PTRN_MODIFY_HDR_ACTION_ID_COPY = 0x05,
+	DR_PTRN_MODIFY_HDR_ACTION_ID_SET = 0x06,
+	DR_PTRN_MODIFY_HDR_ACTION_ID_ADD = 0x07,
+	DR_PTRN_MODIFY_HDR_ACTION_ID_INSERT_INLINE = 0x0a,
+};
+
+struct mlx5dr_ptrn_mgr {
+	struct mlx5dr_domain *dmn;
+	struct mlx5dr_icm_pool *ptrn_icm_pool;
+	/* cache for modify_header ptrn */
+	struct list_head ptrn_list;
+	struct mutex modify_hdr_mutex; /* protect the pattern cache */
+};
+
+/* Cache structure and functions */
+static bool dr_ptrn_compare_modify_hdr(size_t cur_num_of_actions,
+				       __be64 cur_hw_actions[],
+				       size_t num_of_actions,
+				       __be64 hw_actions[])
+{
+	int i;
+
+	if (cur_num_of_actions != num_of_actions)
+		return false;
+
+	for (i = 0; i < num_of_actions; i++) {
+		u8 action_id =
+			MLX5_GET(ste_double_action_set_v1, &hw_actions[i], action_id);
+
+		if (action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_COPY) {
+			if (hw_actions[i] != cur_hw_actions[i])
+				return false;
+		} else {
+			if ((__force __be32)hw_actions[i] !=
+			    (__force __be32)cur_hw_actions[i])
+				return false;
+		}
+	}
+
+	return true;
+}
+
+static struct mlx5dr_ptrn_obj *
+dr_ptrn_find_cached_pattern(struct mlx5dr_ptrn_mgr *mgr,
+			    size_t num_of_actions,
+			    __be64 hw_actions[])
+{
+	struct mlx5dr_ptrn_obj *cached_pattern;
+	struct mlx5dr_ptrn_obj *tmp;
+
+	list_for_each_entry_safe(cached_pattern, tmp, &mgr->ptrn_list, list) {
+		if (dr_ptrn_compare_modify_hdr(cached_pattern->num_of_actions,
+					       (__be64 *)cached_pattern->data,
+					       num_of_actions,
+					       hw_actions)) {
+			/* Put this pattern in the head of the list,
+			 * as we will probably use it more.
+			 */
+			list_del_init(&cached_pattern->list);
+			list_add(&cached_pattern->list, &mgr->ptrn_list);
+			return cached_pattern;
+		}
+	}
+
+	return NULL;
+}
+
+static struct mlx5dr_ptrn_obj *
+dr_ptrn_alloc_pattern(struct mlx5dr_ptrn_mgr *mgr,
+		      u16 num_of_actions, u8 *data)
+{
+	struct mlx5dr_ptrn_obj *pattern;
+	struct mlx5dr_icm_chunk *chunk;
+	u32 chunk_size;
+	u32 index;
+
+	chunk_size = ilog2(num_of_actions);
+	/* HW modify action index granularity is at least 64B */
+	chunk_size = max_t(u32, chunk_size, DR_CHUNK_SIZE_8);
+
+	chunk = mlx5dr_icm_alloc_chunk(mgr->ptrn_icm_pool, chunk_size);
+	if (!chunk)
+		return NULL;
+
+	index = (mlx5dr_icm_pool_get_chunk_icm_addr(chunk) -
+		 mgr->dmn->info.caps.hdr_modify_pattern_icm_addr) /
+		DR_ACTION_CACHE_LINE_SIZE;
+
+	pattern = kzalloc(sizeof(*pattern), GFP_KERNEL);
+	if (!pattern)
+		goto free_chunk;
+
+	pattern->data = kzalloc(num_of_actions * DR_MODIFY_ACTION_SIZE *
+				sizeof(*pattern->data), GFP_KERNEL);
+	if (!pattern->data)
+		goto free_pattern;
+
+	memcpy(pattern->data, data, num_of_actions * DR_MODIFY_ACTION_SIZE);
+	pattern->chunk = chunk;
+	pattern->index = index;
+	pattern->num_of_actions = num_of_actions;
+
+	list_add(&pattern->list, &mgr->ptrn_list);
+	refcount_set(&pattern->refcount, 1);
+
+	return pattern;
+
+free_pattern:
+	kfree(pattern);
+free_chunk:
+	mlx5dr_icm_free_chunk(chunk);
+	return NULL;
+}
+
+static void
+dr_ptrn_free_pattern(struct mlx5dr_ptrn_obj *pattern)
+{
+	list_del(&pattern->list);
+	mlx5dr_icm_free_chunk(pattern->chunk);
+	kfree(pattern->data);
+	kfree(pattern);
+}
+
+struct mlx5dr_ptrn_obj *
+mlx5dr_ptrn_cache_get_pattern(struct mlx5dr_ptrn_mgr *mgr,
+			      u16 num_of_actions,
+			      u8 *data)
+{
+	struct mlx5dr_ptrn_obj *pattern;
+	u64 *hw_actions;
+	u8 action_id;
+	int i;
+
+	mutex_lock(&mgr->modify_hdr_mutex);
+	pattern = dr_ptrn_find_cached_pattern(mgr,
+					      num_of_actions,
+					      (__be64 *)data);
+	if (!pattern) {
+		/* Alloc and add new pattern to cache */
+		pattern = dr_ptrn_alloc_pattern(mgr, num_of_actions, data);
+		if (!pattern)
+			goto out_unlock;
+
+		hw_actions = (u64 *)pattern->data;
+		/* Here we mask the pattern data to create a valid pattern
+		 * since we do an OR operation between the arg and pattern
+		 */
+		for (i = 0; i < num_of_actions; i++) {
+			action_id = MLX5_GET(ste_double_action_set_v1, &hw_actions[i], action_id);
+
+			if (action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_SET ||
+			    action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_ADD ||
+			    action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_INSERT_INLINE)
+				MLX5_SET(ste_double_action_set_v1, &hw_actions[i], inline_data, 0);
+		}
+
+		if (mlx5dr_send_postsend_pattern(mgr->dmn, pattern->chunk,
+						 num_of_actions, pattern->data)) {
+			refcount_dec(&pattern->refcount);
+			goto free_pattern;
+		}
+	} else {
+		refcount_inc(&pattern->refcount);
+	}
+
+	mutex_unlock(&mgr->modify_hdr_mutex);
+
+	return pattern;
+
+free_pattern:
+	dr_ptrn_free_pattern(pattern);
+out_unlock:
+	mutex_unlock(&mgr->modify_hdr_mutex);
+	return NULL;
+}
+
+void
+mlx5dr_ptrn_cache_put_pattern(struct mlx5dr_ptrn_mgr *mgr,
+			      struct mlx5dr_ptrn_obj *pattern)
+{
+	mutex_lock(&mgr->modify_hdr_mutex);
+
+	if (refcount_dec_and_test(&pattern->refcount))
+		dr_ptrn_free_pattern(pattern);
+
+	mutex_unlock(&mgr->modify_hdr_mutex);
+}
+
+struct mlx5dr_ptrn_mgr *mlx5dr_ptrn_mgr_create(struct mlx5dr_domain *dmn)
+{
+	struct mlx5dr_ptrn_mgr *mgr;
+
+	if (!mlx5dr_domain_is_support_ptrn_arg(dmn))
+		return NULL;
+
+	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
+	if (!mgr)
+		return NULL;
+
+	mgr->dmn = dmn;
+	mgr->ptrn_icm_pool = mlx5dr_icm_pool_create(dmn, DR_ICM_TYPE_MODIFY_HDR_PTRN);
+	if (!mgr->ptrn_icm_pool) {
+		mlx5dr_err(dmn, "Couldn't get modify-header-pattern memory\n");
+		goto free_mgr;
+	}
+
+	INIT_LIST_HEAD(&mgr->ptrn_list);
+	return mgr;
+
+free_mgr:
+	kfree(mgr);
+	return NULL;
+}
+
+void mlx5dr_ptrn_mgr_destroy(struct mlx5dr_ptrn_mgr *mgr)
+{
+	struct mlx5dr_ptrn_obj *pattern;
+	struct mlx5dr_ptrn_obj *tmp;
+
+	if (!mgr)
+		return;
+
+	WARN_ON(!list_empty(&mgr->ptrn_list));
+
+	list_for_each_entry_safe(pattern, tmp, &mgr->ptrn_list, list) {
+		list_del(&pattern->list);
+		kfree(pattern->data);
+		kfree(pattern);
+	}
+
+	mlx5dr_icm_pool_destroy(mgr->ptrn_icm_pool);
+	kfree(mgr);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c
index fd2d31cdbcf9..4a5ae86e2b62 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c
@@ -18,7 +18,13 @@ struct dr_data_seg {
 	unsigned int send_flags;
 };
 
+enum send_info_type {
+	WRITE_ICM = 0,
+	GTA_ARG   = 1,
+};
+
 struct postsend_info {
+	enum send_info_type type;
 	struct dr_data_seg write;
 	struct dr_data_seg read;
 	u64 remote_addr;
@@ -261,9 +267,10 @@ static struct mlx5dr_qp *dr_create_rc_qp(struct mlx5_core_dev *mdev,
 
 	dr_qp->rq.pc = 0;
 	dr_qp->rq.cc = 0;
-	dr_qp->rq.wqe_cnt = 4;
+	dr_qp->rq.wqe_cnt = 256;
 	dr_qp->sq.pc = 0;
 	dr_qp->sq.cc = 0;
+	dr_qp->sq.head = 0;
 	dr_qp->sq.wqe_cnt = roundup_pow_of_two(attr->max_send_wr);
 
 	MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4);
@@ -362,39 +369,113 @@ static void dr_cmd_notify_hw(struct mlx5dr_qp *dr_qp, void *ctrl)
 	mlx5_write64(ctrl, dr_qp->uar->map + MLX5_BF_OFFSET);
 }
 
-static void dr_rdma_segments(struct mlx5dr_qp *dr_qp, u64 remote_addr,
-			     u32 rkey, struct dr_data_seg *data_seg,
-			     u32 opcode, bool notify_hw)
+static void
+dr_rdma_handle_flow_access_arg_segments(struct mlx5_wqe_ctrl_seg *wq_ctrl,
+					u32 remote_addr,
+					struct dr_data_seg *data_seg,
+					int *size)
 {
-	struct mlx5_wqe_raddr_seg *wq_raddr;
-	struct mlx5_wqe_ctrl_seg *wq_ctrl;
-	struct mlx5_wqe_data_seg *wq_dseg;
-	unsigned int size;
-	unsigned int idx;
+	struct mlx5_wqe_header_modify_argument_update_seg *wq_arg_seg;
+	struct mlx5_wqe_flow_update_ctrl_seg *wq_flow_seg;
 
-	size = sizeof(*wq_ctrl) / 16 + sizeof(*wq_dseg) / 16 +
-		sizeof(*wq_raddr) / 16;
+	wq_ctrl->general_id = cpu_to_be32(remote_addr);
+	wq_flow_seg = (void *)(wq_ctrl + 1);
 
-	idx = dr_qp->sq.pc & (dr_qp->sq.wqe_cnt - 1);
+	/* mlx5_wqe_flow_update_ctrl_seg - all reserved */
+	memset(wq_flow_seg, 0, sizeof(*wq_flow_seg));
+	wq_arg_seg = (void *)(wq_flow_seg + 1);
+
+	memcpy(wq_arg_seg->argument_list,
+	       (void *)(uintptr_t)data_seg->addr,
+	       data_seg->length);
+
+	*size = (sizeof(*wq_ctrl) +      /* WQE ctrl segment */
+		 sizeof(*wq_flow_seg) +  /* WQE flow update ctrl seg - reserved */
+		 sizeof(*wq_arg_seg)) /  /* WQE hdr modify arg seg - data */
+		MLX5_SEND_WQE_DS;
+}
+
+static void
+dr_rdma_handle_icm_write_segments(struct mlx5_wqe_ctrl_seg *wq_ctrl,
+				  u64 remote_addr,
+				  u32 rkey,
+				  struct dr_data_seg *data_seg,
+				  unsigned int *size)
+{
+	struct mlx5_wqe_raddr_seg *wq_raddr;
+	struct mlx5_wqe_data_seg *wq_dseg;
 
-	wq_ctrl = mlx5_wq_cyc_get_wqe(&dr_qp->wq.sq, idx);
-	wq_ctrl->imm = 0;
-	wq_ctrl->fm_ce_se = (data_seg->send_flags) ?
-		MLX5_WQE_CTRL_CQ_UPDATE : 0;
-	wq_ctrl->opmod_idx_opcode = cpu_to_be32(((dr_qp->sq.pc & 0xffff) << 8) |
-						opcode);
-	wq_ctrl->qpn_ds = cpu_to_be32(size | dr_qp->qpn << 8);
 	wq_raddr = (void *)(wq_ctrl + 1);
+
 	wq_raddr->raddr = cpu_to_be64(remote_addr);
 	wq_raddr->rkey = cpu_to_be32(rkey);
 	wq_raddr->reserved = 0;
 
 	wq_dseg = (void *)(wq_raddr + 1);
+
 	wq_dseg->byte_count = cpu_to_be32(data_seg->length);
 	wq_dseg->lkey = cpu_to_be32(data_seg->lkey);
 	wq_dseg->addr = cpu_to_be64(data_seg->addr);
 
-	dr_qp->sq.wqe_head[idx] = dr_qp->sq.pc++;
+	*size = (sizeof(*wq_ctrl) +    /* WQE ctrl segment */
+		 sizeof(*wq_dseg) +    /* WQE data segment */
+		 sizeof(*wq_raddr)) /  /* WQE remote addr segment */
+		MLX5_SEND_WQE_DS;
+}
+
+static void dr_set_ctrl_seg(struct mlx5_wqe_ctrl_seg *wq_ctrl,
+			    struct dr_data_seg *data_seg)
+{
+	wq_ctrl->signature = 0;
+	wq_ctrl->rsvd[0] = 0;
+	wq_ctrl->rsvd[1] = 0;
+	wq_ctrl->fm_ce_se = data_seg->send_flags & IB_SEND_SIGNALED ?
+				MLX5_WQE_CTRL_CQ_UPDATE : 0;
+	wq_ctrl->imm = 0;
+}
+
+static void dr_rdma_segments(struct mlx5dr_qp *dr_qp, u64 remote_addr,
+			     u32 rkey, struct dr_data_seg *data_seg,
+			     u32 opcode, bool notify_hw)
+{
+	struct mlx5_wqe_ctrl_seg *wq_ctrl;
+	int opcode_mod = 0;
+	unsigned int size;
+	unsigned int idx;
+
+	idx = dr_qp->sq.pc & (dr_qp->sq.wqe_cnt - 1);
+
+	wq_ctrl = mlx5_wq_cyc_get_wqe(&dr_qp->wq.sq, idx);
+	dr_set_ctrl_seg(wq_ctrl, data_seg);
+
+	switch (opcode) {
+	case MLX5_OPCODE_RDMA_READ:
+	case MLX5_OPCODE_RDMA_WRITE:
+		dr_rdma_handle_icm_write_segments(wq_ctrl, remote_addr,
+						  rkey, data_seg, &size);
+		break;
+	case MLX5_OPCODE_FLOW_TBL_ACCESS:
+		opcode_mod = MLX5_CMD_OP_MOD_UPDATE_HEADER_MODIFY_ARGUMENT;
+		dr_rdma_handle_flow_access_arg_segments(wq_ctrl, remote_addr,
+							data_seg, &size);
+		break;
+	default:
+		WARN(true, "illegal opcode %d", opcode);
+		return;
+	}
+
+	/* --------------------------------------------------------
+	 * |opcode_mod (8 bit)|wqe_index (16 bits)| opcod (8 bits)|
+	 * --------------------------------------------------------
+	 */
+	wq_ctrl->opmod_idx_opcode =
+		cpu_to_be32((opcode_mod << 24) |
+			    ((dr_qp->sq.pc & 0xffff) << 8) |
+			    opcode);
+	wq_ctrl->qpn_ds = cpu_to_be32(size | dr_qp->qpn << 8);
+
+	dr_qp->sq.pc += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
+	dr_qp->sq.wqe_head[idx] = dr_qp->sq.head++;
 
 	if (notify_hw)
 		dr_cmd_notify_hw(dr_qp, wq_ctrl);
@@ -402,10 +483,16 @@ static void dr_rdma_segments(struct mlx5dr_qp *dr_qp, u64 remote_addr,
 
 static void dr_post_send(struct mlx5dr_qp *dr_qp, struct postsend_info *send_info)
 {
-	dr_rdma_segments(dr_qp, send_info->remote_addr, send_info->rkey,
-			 &send_info->write, MLX5_OPCODE_RDMA_WRITE, false);
-	dr_rdma_segments(dr_qp, send_info->remote_addr, send_info->rkey,
-			 &send_info->read, MLX5_OPCODE_RDMA_READ, true);
+	if (send_info->type == WRITE_ICM) {
+		dr_rdma_segments(dr_qp, send_info->remote_addr, send_info->rkey,
+				 &send_info->write, MLX5_OPCODE_RDMA_WRITE, false);
+		dr_rdma_segments(dr_qp, send_info->remote_addr, send_info->rkey,
+				 &send_info->read, MLX5_OPCODE_RDMA_READ, true);
+	} else { /* GTA_ARG */
+		dr_rdma_segments(dr_qp, send_info->remote_addr, send_info->rkey,
+				 &send_info->write, MLX5_OPCODE_FLOW_TBL_ACCESS, true);
+	}
+
 }
 
 /**
@@ -471,24 +558,54 @@ static int dr_handle_pending_wc(struct mlx5dr_domain *dmn,
 		} else if (ne == 1) {
 			send_ring->pending_wqe -= send_ring->signal_th;
 		}
-	} while (is_drain && send_ring->pending_wqe);
+	} while (ne == 1 ||
+		 (is_drain && send_ring->pending_wqe  >= send_ring->signal_th));
 
 	return 0;
 }
 
-static void dr_fill_data_segs(struct mlx5dr_send_ring *send_ring,
-			      struct postsend_info *send_info)
+static void dr_fill_write_args_segs(struct mlx5dr_send_ring *send_ring,
+				    struct postsend_info *send_info)
 {
 	send_ring->pending_wqe++;
 
 	if (send_ring->pending_wqe % send_ring->signal_th == 0)
 		send_info->write.send_flags |= IB_SEND_SIGNALED;
+	else
+		send_info->write.send_flags = 0;
+}
+
+static void dr_fill_write_icm_segs(struct mlx5dr_domain *dmn,
+				   struct mlx5dr_send_ring *send_ring,
+				   struct postsend_info *send_info)
+{
+	u32 buff_offset;
+
+	if (send_info->write.length > dmn->info.max_inline_size) {
+		buff_offset = (send_ring->tx_head &
+			       (dmn->send_ring->signal_th - 1)) *
+			      send_ring->max_post_send_size;
+		/* Copy to ring mr */
+		memcpy(send_ring->buf + buff_offset,
+		       (void *)(uintptr_t)send_info->write.addr,
+		       send_info->write.length);
+		send_info->write.addr = (uintptr_t)send_ring->mr->dma_addr + buff_offset;
+		send_info->write.lkey = send_ring->mr->mkey;
+
+		send_ring->tx_head++;
+	}
+
+	send_ring->pending_wqe++;
+
+	if (send_ring->pending_wqe % send_ring->signal_th == 0)
+		send_info->write.send_flags |= IB_SEND_SIGNALED;
 
 	send_ring->pending_wqe++;
 	send_info->read.length = send_info->write.length;
-	/* Read into the same write area */
-	send_info->read.addr = (uintptr_t)send_info->write.addr;
-	send_info->read.lkey = send_ring->mr->mkey;
+
+	/* Read into dedicated sync buffer */
+	send_info->read.addr = (uintptr_t)send_ring->sync_mr->dma_addr;
+	send_info->read.lkey = send_ring->sync_mr->mkey;
 
 	if (send_ring->pending_wqe % send_ring->signal_th == 0)
 		send_info->read.send_flags = IB_SEND_SIGNALED;
@@ -496,11 +613,20 @@ static void dr_fill_data_segs(struct mlx5dr_send_ring *send_ring,
 		send_info->read.send_flags = 0;
 }
 
+static void dr_fill_data_segs(struct mlx5dr_domain *dmn,
+			      struct mlx5dr_send_ring *send_ring,
+			      struct postsend_info *send_info)
+{
+	if (send_info->type == WRITE_ICM)
+		dr_fill_write_icm_segs(dmn, send_ring, send_info);
+	else /* args */
+		dr_fill_write_args_segs(send_ring, send_info);
+}
+
 static int dr_postsend_icm_data(struct mlx5dr_domain *dmn,
 				struct postsend_info *send_info)
 {
 	struct mlx5dr_send_ring *send_ring = dmn->send_ring;
-	u32 buff_offset;
 	int ret;
 
 	if (unlikely(dmn->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR ||
@@ -517,20 +643,7 @@ static int dr_postsend_icm_data(struct mlx5dr_domain *dmn,
 	if (ret)
 		goto out_unlock;
 
-	if (send_info->write.length > dmn->info.max_inline_size) {
-		buff_offset = (send_ring->tx_head &
-			       (dmn->send_ring->signal_th - 1)) *
-			send_ring->max_post_send_size;
-		/* Copy to ring mr */
-		memcpy(send_ring->buf + buff_offset,
-		       (void *)(uintptr_t)send_info->write.addr,
-		       send_info->write.length);
-		send_info->write.addr = (uintptr_t)send_ring->mr->dma_addr + buff_offset;
-		send_info->write.lkey = send_ring->mr->mkey;
-	}
-
-	send_ring->tx_head++;
-	dr_fill_data_segs(send_ring, send_info);
+	dr_fill_data_segs(dmn, send_ring, send_info);
 	dr_post_send(send_ring->qp, send_info);
 
 out_unlock:
@@ -736,6 +849,59 @@ int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn,
 	return dr_postsend_icm_data(dmn, &send_info);
 }
 
+int mlx5dr_send_postsend_pattern(struct mlx5dr_domain *dmn,
+				 struct mlx5dr_icm_chunk *chunk,
+				 u16 num_of_actions,
+				 u8 *data)
+{
+	struct postsend_info send_info = {};
+	int ret;
+
+	send_info.write.addr = (uintptr_t)data;
+	send_info.write.length = num_of_actions * DR_MODIFY_ACTION_SIZE;
+	send_info.remote_addr = mlx5dr_icm_pool_get_chunk_mr_addr(chunk);
+	send_info.rkey = mlx5dr_icm_pool_get_chunk_rkey(chunk);
+
+	ret = dr_postsend_icm_data(dmn, &send_info);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+int mlx5dr_send_postsend_args(struct mlx5dr_domain *dmn, u64 arg_id,
+			      u16 num_of_actions, u8 *actions_data)
+{
+	int data_len, iter = 0, cur_sent;
+	u64 addr;
+	int ret;
+
+	addr = (uintptr_t)actions_data;
+	data_len = num_of_actions * DR_MODIFY_ACTION_SIZE;
+
+	do {
+		struct postsend_info send_info = {};
+
+		send_info.type = GTA_ARG;
+		send_info.write.addr = addr;
+		cur_sent = min_t(u32, data_len, DR_ACTION_CACHE_LINE_SIZE);
+		send_info.write.length = cur_sent;
+		send_info.write.lkey = 0;
+		send_info.remote_addr = arg_id + iter;
+
+		ret = dr_postsend_icm_data(dmn, &send_info);
+		if (ret)
+			goto out;
+
+		iter++;
+		addr += cur_sent;
+		data_len -= cur_sent;
+	} while (data_len > 0);
+
+out:
+	return ret;
+}
+
 static int dr_modify_qp_rst2init(struct mlx5_core_dev *mdev,
 				 struct mlx5dr_qp *dr_qp,
 				 int port)
@@ -1123,16 +1289,25 @@ int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn)
 		goto free_mem;
 	}
 
+	dmn->send_ring->sync_buff = kzalloc(dmn->send_ring->max_post_send_size,
+					    GFP_KERNEL);
+	if (!dmn->send_ring->sync_buff) {
+		ret = -ENOMEM;
+		goto clean_mr;
+	}
+
 	dmn->send_ring->sync_mr = dr_reg_mr(dmn->mdev,
 					    dmn->pdn, dmn->send_ring->sync_buff,
-					    MIN_READ_SYNC);
+					    dmn->send_ring->max_post_send_size);
 	if (!dmn->send_ring->sync_mr) {
 		ret = -ENOMEM;
-		goto clean_mr;
+		goto free_sync_mem;
 	}
 
 	return 0;
 
+free_sync_mem:
+	kfree(dmn->send_ring->sync_buff);
 clean_mr:
 	dr_dereg_mr(dmn->mdev, dmn->send_ring->mr);
 free_mem:
@@ -1155,6 +1330,7 @@ void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn,
 	dr_dereg_mr(dmn->mdev, send_ring->sync_mr);
 	dr_dereg_mr(dmn->mdev, send_ring->mr);
 	kfree(send_ring->buf);
+	kfree(send_ring->sync_buff);
 	kfree(send_ring);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c
index 1e15f605df6e..9413aaf51251 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c
@@ -633,6 +633,63 @@ int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx,
 						 used_hw_action_num);
 }
 
+static int
+dr_ste_alloc_modify_hdr_chunk(struct mlx5dr_action *action)
+{
+	struct mlx5dr_domain *dmn = action->rewrite->dmn;
+	u32 chunk_size;
+	int ret;
+
+	chunk_size = ilog2(roundup_pow_of_two(action->rewrite->num_of_actions));
+
+	/* HW modify action index granularity is at least 64B */
+	chunk_size = max_t(u32, chunk_size, DR_CHUNK_SIZE_8);
+
+	action->rewrite->chunk = mlx5dr_icm_alloc_chunk(dmn->action_icm_pool,
+							chunk_size);
+	if (!action->rewrite->chunk)
+		return -ENOMEM;
+
+	action->rewrite->index = (mlx5dr_icm_pool_get_chunk_icm_addr(action->rewrite->chunk) -
+				  dmn->info.caps.hdr_modify_icm_addr) /
+				 DR_ACTION_CACHE_LINE_SIZE;
+
+	ret = mlx5dr_send_postsend_action(action->rewrite->dmn, action);
+	if (ret)
+		goto free_chunk;
+
+	return 0;
+
+free_chunk:
+	mlx5dr_icm_free_chunk(action->rewrite->chunk);
+	return -ENOMEM;
+}
+
+static void dr_ste_free_modify_hdr_chunk(struct mlx5dr_action *action)
+{
+	mlx5dr_icm_free_chunk(action->rewrite->chunk);
+}
+
+int mlx5dr_ste_alloc_modify_hdr(struct mlx5dr_action *action)
+{
+	struct mlx5dr_domain *dmn = action->rewrite->dmn;
+
+	if (mlx5dr_domain_is_support_ptrn_arg(dmn))
+		return dmn->ste_ctx->alloc_modify_hdr_chunk(action);
+
+	return dr_ste_alloc_modify_hdr_chunk(action);
+}
+
+void mlx5dr_ste_free_modify_hdr(struct mlx5dr_action *action)
+{
+	struct mlx5dr_domain *dmn = action->rewrite->dmn;
+
+	if (mlx5dr_domain_is_support_ptrn_arg(dmn))
+		return dmn->ste_ctx->dealloc_modify_hdr_chunk(action);
+
+	return dr_ste_free_modify_hdr_chunk(action);
+}
+
 static int dr_ste_build_pre_check_spec(struct mlx5dr_domain *dmn,
 				       struct mlx5dr_match_spec *spec)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h
index 7075142bcfb6..54a6619c3ecb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h
@@ -195,6 +195,8 @@ struct mlx5dr_ste_ctx {
 					u8 *hw_action,
 					u32 hw_action_sz,
 					u16 *used_hw_action_num);
+	int (*alloc_modify_hdr_chunk)(struct mlx5dr_action *action);
+	void (*dealloc_modify_hdr_chunk)(struct mlx5dr_action *action);
 
 	/* Send */
 	void (*prepare_for_postsend)(u8 *hw_ste_p, u32 ste_size);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c
index 084145f18084..4c0704ad166b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c
@@ -495,21 +495,66 @@ static void dr_ste_v1_set_rx_decap(u8 *hw_ste_p, u8 *s_action)
 	dr_ste_v1_set_reparse(hw_ste_p);
 }
 
-static void dr_ste_v1_set_rewrite_actions(u8 *hw_ste_p,
-					  u8 *s_action,
-					  u16 num_of_actions,
-					  u32 re_write_index)
+static void dr_ste_v1_set_accelerated_rewrite_actions(u8 *hw_ste_p,
+						      u8 *d_action,
+						      u16 num_of_actions,
+						      u32 rewrite_pattern,
+						      u32 rewrite_args,
+						      u8 *action_data)
+{
+	if (action_data) {
+		memcpy(d_action, action_data, DR_MODIFY_ACTION_SIZE);
+	} else {
+		MLX5_SET(ste_double_action_accelerated_modify_action_list_v1, d_action,
+			 action_id, DR_STE_V1_ACTION_ID_ACCELERATED_LIST);
+		MLX5_SET(ste_double_action_accelerated_modify_action_list_v1, d_action,
+			 modify_actions_pattern_pointer, rewrite_pattern);
+		MLX5_SET(ste_double_action_accelerated_modify_action_list_v1, d_action,
+			 number_of_modify_actions, num_of_actions);
+		MLX5_SET(ste_double_action_accelerated_modify_action_list_v1, d_action,
+			 modify_actions_argument_pointer, rewrite_args);
+	}
+
+	dr_ste_v1_set_reparse(hw_ste_p);
+}
+
+static void dr_ste_v1_set_basic_rewrite_actions(u8 *hw_ste_p,
+						u8 *s_action,
+						u16 num_of_actions,
+						u32 rewrite_index)
 {
 	MLX5_SET(ste_single_action_modify_list_v1, s_action, action_id,
 		 DR_STE_V1_ACTION_ID_MODIFY_LIST);
 	MLX5_SET(ste_single_action_modify_list_v1, s_action, num_of_modify_actions,
 		 num_of_actions);
 	MLX5_SET(ste_single_action_modify_list_v1, s_action, modify_actions_ptr,
-		 re_write_index);
+		 rewrite_index);
 
 	dr_ste_v1_set_reparse(hw_ste_p);
 }
 
+static void dr_ste_v1_set_rewrite_actions(u8 *hw_ste_p,
+					  u8 *action,
+					  u16 num_of_actions,
+					  u32 rewrite_pattern,
+					  u32 rewrite_args,
+					  u8 *action_data)
+{
+	if (rewrite_pattern != MLX5DR_INVALID_PATTERN_INDEX)
+		return dr_ste_v1_set_accelerated_rewrite_actions(hw_ste_p,
+								 action,
+								 num_of_actions,
+								 rewrite_pattern,
+								 rewrite_args,
+								 action_data);
+
+	/* fall back to the code that doesn't support accelerated modify header */
+	return dr_ste_v1_set_basic_rewrite_actions(hw_ste_p,
+						   action,
+						   num_of_actions,
+						   rewrite_args);
+}
+
 static void dr_ste_v1_set_aso_flow_meter(u8 *d_action,
 					 u32 object_id,
 					 u32 offset,
@@ -604,9 +649,6 @@ void dr_ste_v1_set_actions_tx(struct mlx5dr_domain *dmn,
 			allow_modify_hdr = false;
 	}
 
-	if (action_type_set[DR_ACTION_TYP_CTR])
-		dr_ste_v1_set_counter_id(last_ste, attr->ctr_id);
-
 	if (action_type_set[DR_ACTION_TYP_MODIFY_HDR]) {
 		if (!allow_modify_hdr || action_sz < DR_STE_ACTION_DOUBLE_SZ) {
 			dr_ste_v1_arr_init_next_match(&last_ste, added_stes,
@@ -617,7 +659,9 @@ void dr_ste_v1_set_actions_tx(struct mlx5dr_domain *dmn,
 		}
 		dr_ste_v1_set_rewrite_actions(last_ste, action,
 					      attr->modify_actions,
-					      attr->modify_index);
+					      attr->modify_pat_idx,
+					      attr->modify_index,
+					      attr->single_modify_action);
 		action_sz -= DR_STE_ACTION_DOUBLE_SZ;
 		action += DR_STE_ACTION_DOUBLE_SZ;
 		allow_encap = false;
@@ -724,6 +768,10 @@ void dr_ste_v1_set_actions_tx(struct mlx5dr_domain *dmn,
 						  attr->range.max);
 	}
 
+	/* set counter ID on the last STE to adhere to DMFS behavior */
+	if (action_type_set[DR_ACTION_TYP_CTR])
+		dr_ste_v1_set_counter_id(last_ste, attr->ctr_id);
+
 	dr_ste_v1_set_hit_gvmi(last_ste, attr->hit_gvmi);
 	dr_ste_v1_set_hit_addr(last_ste, attr->final_icm_addr, 1);
 }
@@ -743,7 +791,9 @@ void dr_ste_v1_set_actions_rx(struct mlx5dr_domain *dmn,
 	if (action_type_set[DR_ACTION_TYP_TNL_L3_TO_L2]) {
 		dr_ste_v1_set_rewrite_actions(last_ste, action,
 					      attr->decap_actions,
-					      attr->decap_index);
+					      attr->decap_pat_idx,
+					      attr->decap_index,
+					      NULL);
 		action_sz -= DR_STE_ACTION_DOUBLE_SZ;
 		action += DR_STE_ACTION_DOUBLE_SZ;
 		allow_modify_hdr = false;
@@ -798,7 +848,9 @@ void dr_ste_v1_set_actions_rx(struct mlx5dr_domain *dmn,
 		}
 		dr_ste_v1_set_rewrite_actions(last_ste, action,
 					      attr->modify_actions,
-					      attr->modify_index);
+					      attr->modify_pat_idx,
+					      attr->modify_index,
+					      attr->single_modify_action);
 		action_sz -= DR_STE_ACTION_DOUBLE_SZ;
 		action += DR_STE_ACTION_DOUBLE_SZ;
 	}
@@ -2175,6 +2227,49 @@ dr_ste_v1_build_tnl_gtpu_flex_parser_1_init(struct mlx5dr_ste_build *sb,
 	sb->ste_build_tag_func = &dr_ste_v1_build_tnl_gtpu_flex_parser_1_tag;
 }
 
+int dr_ste_v1_alloc_modify_hdr_ptrn_arg(struct mlx5dr_action *action)
+{
+	struct mlx5dr_ptrn_mgr *ptrn_mgr;
+	int ret;
+
+	ptrn_mgr = action->rewrite->dmn->ptrn_mgr;
+	if (!ptrn_mgr)
+		return -EOPNOTSUPP;
+
+	action->rewrite->arg = mlx5dr_arg_get_obj(action->rewrite->dmn->arg_mgr,
+						  action->rewrite->num_of_actions,
+						  action->rewrite->data);
+	if (!action->rewrite->arg) {
+		mlx5dr_err(action->rewrite->dmn, "Failed allocating args for modify header\n");
+		return -EAGAIN;
+	}
+
+	action->rewrite->ptrn =
+		mlx5dr_ptrn_cache_get_pattern(ptrn_mgr,
+					      action->rewrite->num_of_actions,
+					      action->rewrite->data);
+	if (!action->rewrite->ptrn) {
+		mlx5dr_err(action->rewrite->dmn, "Failed to get pattern\n");
+		ret = -EAGAIN;
+		goto put_arg;
+	}
+
+	return 0;
+
+put_arg:
+	mlx5dr_arg_put_obj(action->rewrite->dmn->arg_mgr,
+			   action->rewrite->arg);
+	return ret;
+}
+
+void dr_ste_v1_free_modify_hdr_ptrn_arg(struct mlx5dr_action *action)
+{
+	mlx5dr_ptrn_cache_put_pattern(action->rewrite->dmn->ptrn_mgr,
+				      action->rewrite->ptrn);
+	mlx5dr_arg_put_obj(action->rewrite->dmn->arg_mgr,
+			   action->rewrite->arg);
+}
+
 static struct mlx5dr_ste_ctx ste_ctx_v1 = {
 	/* Builders */
 	.build_eth_l2_src_dst_init	= &dr_ste_v1_build_eth_l2_src_dst_init,
@@ -2231,6 +2326,9 @@ static struct mlx5dr_ste_ctx ste_ctx_v1 = {
 	.set_action_add			= &dr_ste_v1_set_action_add,
 	.set_action_copy		= &dr_ste_v1_set_action_copy,
 	.set_action_decap_l3_list	= &dr_ste_v1_set_action_decap_l3_list,
+	.alloc_modify_hdr_chunk		= &dr_ste_v1_alloc_modify_hdr_ptrn_arg,
+	.dealloc_modify_hdr_chunk	= &dr_ste_v1_free_modify_hdr_ptrn_arg,
+
 	/* Send */
 	.prepare_for_postsend		= &dr_ste_v1_prepare_for_postsend,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.h
index b5c0f0f8392f..e2fc69867088 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.h
@@ -31,6 +31,8 @@ void dr_ste_v1_set_action_copy(u8 *d_action, u8 dst_hw_field, u8 dst_shifter,
 			       u8 dst_len, u8 src_hw_field, u8 src_shifter);
 int dr_ste_v1_set_action_decap_l3_list(void *data, u32 data_sz, u8 *hw_action,
 				       u32 hw_action_sz, u16 *used_hw_action_num);
+int dr_ste_v1_alloc_modify_hdr_ptrn_arg(struct mlx5dr_action *action);
+void dr_ste_v1_free_modify_hdr_ptrn_arg(struct mlx5dr_action *action);
 void dr_ste_v1_build_eth_l2_src_dst_init(struct mlx5dr_ste_build *sb,
 					 struct mlx5dr_match_param *mask);
 void dr_ste_v1_build_eth_l3_ipv6_dst_init(struct mlx5dr_ste_build *sb,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v2.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v2.c
index cf1a3c9a1cf4..808b013cf48c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v2.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v2.c
@@ -221,6 +221,8 @@ static struct mlx5dr_ste_ctx ste_ctx_v2 = {
 	.set_action_add			= &dr_ste_v1_set_action_add,
 	.set_action_copy		= &dr_ste_v1_set_action_copy,
 	.set_action_decap_l3_list	= &dr_ste_v1_set_action_decap_l3_list,
+	.alloc_modify_hdr_chunk		= &dr_ste_v1_alloc_modify_hdr_ptrn_arg,
+	.dealloc_modify_hdr_chunk	= &dr_ste_v1_free_modify_hdr_ptrn_arg,
 
 	/* Send */
 	.prepare_for_postsend		= &dr_ste_v1_prepare_for_postsend,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h
index 2b769dcbd453..678a993ab053 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h
@@ -21,11 +21,16 @@
 #define DR_NUM_OF_FLEX_PARSERS 8
 #define DR_STE_MAX_FLEX_0_ID 3
 #define DR_STE_MAX_FLEX_1_ID 7
+#define DR_ACTION_CACHE_LINE_SIZE 64
 
 #define mlx5dr_err(dmn, arg...) mlx5_core_err((dmn)->mdev, ##arg)
 #define mlx5dr_info(dmn, arg...) mlx5_core_info((dmn)->mdev, ##arg)
 #define mlx5dr_dbg(dmn, arg...) mlx5_core_dbg((dmn)->mdev, ##arg)
 
+struct mlx5dr_ptrn_mgr;
+struct mlx5dr_arg_mgr;
+struct mlx5dr_arg_obj;
+
 static inline bool dr_is_flex_parser_0_id(u8 parser_id)
 {
 	return parser_id <= DR_STE_MAX_FLEX_0_ID;
@@ -66,6 +71,8 @@ enum mlx5dr_icm_chunk_size {
 enum mlx5dr_icm_type {
 	DR_ICM_TYPE_STE,
 	DR_ICM_TYPE_MODIFY_ACTION,
+	DR_ICM_TYPE_MODIFY_HDR_PTRN,
+	DR_ICM_TYPE_MAX,
 };
 
 static inline enum mlx5dr_icm_chunk_size
@@ -255,11 +262,15 @@ u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste);
 struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste);
 
 #define MLX5DR_MAX_VLANS 2
+#define MLX5DR_INVALID_PATTERN_INDEX 0xffffffff
 
 struct mlx5dr_ste_actions_attr {
 	u32	modify_index;
+	u32	modify_pat_idx;
 	u16	modify_actions;
+	u8	*single_modify_action;
 	u32	decap_index;
+	u32	decap_pat_idx;
 	u16	decap_actions;
 	u8	decap_with_vlan:1;
 	u64	final_icm_addr;
@@ -331,6 +342,8 @@ int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx,
 					u8 *hw_action,
 					u32 hw_action_sz,
 					u16 *used_hw_action_num);
+int mlx5dr_ste_alloc_modify_hdr(struct mlx5dr_action *action);
+void mlx5dr_ste_free_modify_hdr(struct mlx5dr_action *action);
 
 const struct mlx5dr_ste_action_modify_field *
 mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field);
@@ -861,6 +874,8 @@ struct mlx5dr_cmd_caps {
 	u64 esw_tx_drop_address;
 	u32 log_icm_size;
 	u64 hdr_modify_icm_addr;
+	u32 log_modify_pattern_icm_size;
+	u64 hdr_modify_pattern_icm_addr;
 	u32 flex_protocols;
 	u8 flex_parser_id_icmp_dw0;
 	u8 flex_parser_id_icmp_dw1;
@@ -888,6 +903,9 @@ struct mlx5dr_cmd_caps {
 	struct mlx5dr_vports vports;
 	bool prio_tag_required;
 	struct mlx5dr_roce_cap roce_caps;
+	u16 log_header_modify_argument_granularity;
+	u16 log_header_modify_argument_max_alloc;
+	bool support_modify_argument;
 	u8 is_ecpf:1;
 	u8 isolate_vl_tc:1;
 };
@@ -910,6 +928,7 @@ struct mlx5dr_domain_info {
 	u32 max_send_wr;
 	u32 max_log_sw_icm_sz;
 	u32 max_log_action_icm_sz;
+	u32 max_log_modify_hdr_pattern_icm_sz;
 	struct mlx5dr_domain_rx_tx rx;
 	struct mlx5dr_domain_rx_tx tx;
 	struct mlx5dr_cmd_caps caps;
@@ -928,6 +947,8 @@ struct mlx5dr_domain {
 	struct mlx5dr_send_info_pool *send_info_pool_tx;
 	struct kmem_cache *chunks_kmem_cache;
 	struct kmem_cache *htbls_kmem_cache;
+	struct mlx5dr_ptrn_mgr *ptrn_mgr;
+	struct mlx5dr_arg_mgr *arg_mgr;
 	struct mlx5dr_send_ring *send_ring;
 	struct mlx5dr_domain_info info;
 	struct xarray csum_fts_xa;
@@ -935,6 +956,8 @@ struct mlx5dr_domain {
 	struct list_head dbg_tbl_list;
 	struct mlx5dr_dbg_dump_info dump_info;
 	struct xarray definers_xa;
+	/* memory management statistics */
+	u32 num_buddies[DR_ICM_TYPE_MAX];
 };
 
 struct mlx5dr_table_rx_tx {
@@ -994,15 +1017,34 @@ struct mlx5dr_ste_action_modify_field {
 	u8 l4_type;
 };
 
+struct mlx5dr_ptrn_obj {
+	struct mlx5dr_icm_chunk *chunk;
+	u8 *data;
+	u16 num_of_actions;
+	u32 index;
+	refcount_t refcount;
+	struct list_head list;
+};
+
+struct mlx5dr_arg_obj {
+	u32 obj_id;
+	u32 obj_offset;
+	struct list_head list_node;
+	u32 log_chunk_size;
+};
+
 struct mlx5dr_action_rewrite {
 	struct mlx5dr_domain *dmn;
 	struct mlx5dr_icm_chunk *chunk;
 	u8 *data;
 	u16 num_of_actions;
 	u32 index;
+	u8 single_action_opt:1;
 	u8 allow_rx:1;
 	u8 allow_tx:1;
 	u8 modify_ttl:1;
+	struct mlx5dr_ptrn_obj *ptrn;
+	struct mlx5dr_arg_obj *arg;
 };
 
 struct mlx5dr_action_reformat {
@@ -1334,6 +1376,12 @@ struct mlx5dr_cmd_gid_attr {
 int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num,
 			 u16 index, struct mlx5dr_cmd_gid_attr *attr);
 
+int mlx5dr_cmd_create_modify_header_arg(struct mlx5_core_dev *dev,
+					u16 log_obj_range, u32 pd,
+					u32 *obj_id);
+void mlx5dr_cmd_destroy_modify_header_arg(struct mlx5_core_dev *dev,
+					  u32 obj_id);
+
 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn,
 					       enum mlx5dr_icm_type icm_type);
 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool);
@@ -1368,6 +1416,7 @@ struct mlx5dr_qp {
 	struct mlx5_wq_ctrl wq_ctrl;
 	u32 qpn;
 	struct {
+		unsigned int head;
 		unsigned int pc;
 		unsigned int cc;
 		unsigned int size;
@@ -1399,9 +1448,6 @@ struct mlx5dr_mr {
 	size_t size;
 };
 
-#define MAX_SEND_CQE		64
-#define MIN_READ_SYNC		64
-
 struct mlx5dr_send_ring {
 	struct mlx5dr_cq *cq;
 	struct mlx5dr_qp *qp;
@@ -1416,7 +1462,7 @@ struct mlx5dr_send_ring {
 	u32 tx_head;
 	void *buf;
 	u32 buf_size;
-	u8 sync_buff[MIN_READ_SYNC];
+	u8 *sync_buff;
 	struct mlx5dr_mr *sync_mr;
 	spinlock_t lock; /* Protect the data path of the send ring */
 	bool err_state; /* send_ring is not usable in err state */
@@ -1440,6 +1486,12 @@ int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn,
 					bool update_hw_ste);
 int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn,
 				struct mlx5dr_action *action);
+int mlx5dr_send_postsend_pattern(struct mlx5dr_domain *dmn,
+				 struct mlx5dr_icm_chunk *chunk,
+				 u16 num_of_actions,
+				 u8 *data);
+int mlx5dr_send_postsend_args(struct mlx5dr_domain *dmn, u64 arg_id,
+			      u16 num_of_actions, u8 *actions_data);
 
 int mlx5dr_send_info_pool_create(struct mlx5dr_domain *dmn);
 void mlx5dr_send_info_pool_destroy(struct mlx5dr_domain *dmn);
@@ -1526,4 +1578,20 @@ static inline bool mlx5dr_supp_match_ranges(struct mlx5_core_dev *dev)
 			(1ULL << MLX5_IFC_DEFINER_FORMAT_ID_SELECT));
 }
 
+bool mlx5dr_domain_is_support_ptrn_arg(struct mlx5dr_domain *dmn);
+struct mlx5dr_ptrn_mgr *mlx5dr_ptrn_mgr_create(struct mlx5dr_domain *dmn);
+void mlx5dr_ptrn_mgr_destroy(struct mlx5dr_ptrn_mgr *mgr);
+struct mlx5dr_ptrn_obj *mlx5dr_ptrn_cache_get_pattern(struct mlx5dr_ptrn_mgr *mgr,
+						      u16 num_of_actions, u8 *data);
+void mlx5dr_ptrn_cache_put_pattern(struct mlx5dr_ptrn_mgr *mgr,
+				   struct mlx5dr_ptrn_obj *pattern);
+struct mlx5dr_arg_mgr *mlx5dr_arg_mgr_create(struct mlx5dr_domain *dmn);
+void mlx5dr_arg_mgr_destroy(struct mlx5dr_arg_mgr *mgr);
+struct mlx5dr_arg_obj *mlx5dr_arg_get_obj(struct mlx5dr_arg_mgr *mgr,
+					  u16 num_of_actions,
+					  u8 *data);
+void mlx5dr_arg_put_obj(struct mlx5dr_arg_mgr *mgr,
+			struct mlx5dr_arg_obj *arg_obj);
+u32 mlx5dr_arg_get_obj_id(struct mlx5dr_arg_obj *arg_obj);
+
 #endif  /* _DR_TYPES_H_ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr_ste_v1.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr_ste_v1.h
index 790a17d6207f..ca3b0f1453a7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr_ste_v1.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr_ste_v1.h
@@ -100,7 +100,7 @@ struct mlx5_ifc_ste_double_action_insert_with_ptr_v1_bits {
 	u8         pointer[0x20];
 };
 
-struct mlx5_ifc_ste_double_action_modify_action_list_v1_bits {
+struct mlx5_ifc_ste_double_action_accelerated_modify_action_list_v1_bits {
 	u8         action_id[0x8];
 	u8         modify_actions_pattern_pointer[0x18];
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/thermal.c b/drivers/net/ethernet/mellanox/mlx5/core/thermal.c
new file mode 100644
index 000000000000..e47fa6fb836f
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/thermal.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/device.h>
+#include <linux/thermal.h>
+#include <linux/err.h>
+#include <linux/mlx5/driver.h>
+#include "mlx5_core.h"
+#include "thermal.h"
+
+#define MLX5_THERMAL_POLL_INT_MSEC	1000
+#define MLX5_THERMAL_NUM_TRIPS		0
+#define MLX5_THERMAL_ASIC_SENSOR_INDEX	0
+
+/* Bit string indicating the writeablility of trip points if any */
+#define MLX5_THERMAL_TRIP_MASK	(BIT(MLX5_THERMAL_NUM_TRIPS) - 1)
+
+struct mlx5_thermal {
+	struct mlx5_core_dev *mdev;
+	struct thermal_zone_device *tzdev;
+};
+
+static int mlx5_thermal_get_mtmp_temp(struct mlx5_core_dev *mdev, u32 id, int *p_temp)
+{
+	u32 mtmp_out[MLX5_ST_SZ_DW(mtmp_reg)] = {};
+	u32 mtmp_in[MLX5_ST_SZ_DW(mtmp_reg)] = {};
+	int err;
+
+	MLX5_SET(mtmp_reg, mtmp_in, sensor_index, id);
+
+	err = mlx5_core_access_reg(mdev, mtmp_in,  sizeof(mtmp_in),
+				   mtmp_out, sizeof(mtmp_out),
+				   MLX5_REG_MTMP, 0, 0);
+
+	if (err)
+		return err;
+
+	*p_temp = MLX5_GET(mtmp_reg, mtmp_out, temperature);
+
+	return 0;
+}
+
+static int mlx5_thermal_get_temp(struct thermal_zone_device *tzdev,
+				 int *p_temp)
+{
+	struct mlx5_thermal *thermal = tzdev->devdata;
+	struct mlx5_core_dev *mdev = thermal->mdev;
+	int err;
+
+	err = mlx5_thermal_get_mtmp_temp(mdev, MLX5_THERMAL_ASIC_SENSOR_INDEX, p_temp);
+
+	if (err)
+		return err;
+
+	/* The unit of temp returned is in 0.125 C. The thermal
+	 * framework expects the value in 0.001 C.
+	 */
+	*p_temp *= 125;
+
+	return 0;
+}
+
+static struct thermal_zone_device_ops mlx5_thermal_ops = {
+	.get_temp = mlx5_thermal_get_temp,
+};
+
+int mlx5_thermal_init(struct mlx5_core_dev *mdev)
+{
+	struct mlx5_thermal *thermal;
+	struct thermal_zone_device *tzd;
+	const char *data = "mlx5";
+
+	tzd = thermal_zone_get_zone_by_name(data);
+	if (!IS_ERR(tzd))
+		return 0;
+
+	thermal = kzalloc(sizeof(*thermal), GFP_KERNEL);
+	if (!thermal)
+		return -ENOMEM;
+
+	thermal->mdev = mdev;
+	thermal->tzdev = thermal_zone_device_register(data,
+						      MLX5_THERMAL_NUM_TRIPS,
+						      MLX5_THERMAL_TRIP_MASK,
+						      thermal,
+						      &mlx5_thermal_ops,
+						      NULL, 0, MLX5_THERMAL_POLL_INT_MSEC);
+	if (IS_ERR(thermal->tzdev)) {
+		dev_err(mdev->device, "Failed to register thermal zone device (%s) %ld\n",
+			data, PTR_ERR(thermal->tzdev));
+		kfree(thermal);
+		return -EINVAL;
+	}
+
+	mdev->thermal = thermal;
+	return 0;
+}
+
+void mlx5_thermal_uninit(struct mlx5_core_dev *mdev)
+{
+	if (!mdev->thermal)
+		return;
+
+	thermal_zone_device_unregister(mdev->thermal->tzdev);
+	kfree(mdev->thermal);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/thermal.h b/drivers/net/ethernet/mellanox/mlx5/core/thermal.h
new file mode 100644
index 000000000000..7d752c122192
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/thermal.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+ * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.
+ */
+#ifndef __MLX5_THERMAL_DRIVER_H
+#define __MLX5_THERMAL_DRIVER_H
+
+#if IS_ENABLED(CONFIG_THERMAL)
+int mlx5_thermal_init(struct mlx5_core_dev *mdev);
+void mlx5_thermal_uninit(struct mlx5_core_dev *mdev);
+#else
+static inline int mlx5_thermal_init(struct mlx5_core_dev *mdev)
+{
+	mdev->thermal = NULL;
+	return 0;
+}
+
+static inline void mlx5_thermal_uninit(struct mlx5_core_dev *mdev) { }
+#endif
+
+#endif /* __MLX5_THERMAL_DRIVER_H */
diff --git a/drivers/net/ethernet/mellanox/mlxfw/mlxfw_mfa2_tlv_multi.c b/drivers/net/ethernet/mellanox/mlxfw/mlxfw_mfa2_tlv_multi.c
index 017d68f1e123..972c571b4158 100644
--- a/drivers/net/ethernet/mellanox/mlxfw/mlxfw_mfa2_tlv_multi.c
+++ b/drivers/net/ethernet/mellanox/mlxfw/mlxfw_mfa2_tlv_multi.c
@@ -31,6 +31,8 @@ mlxfw_mfa2_tlv_next(const struct mlxfw_mfa2_file *mfa2_file,
 
 	if (tlv->type == MLXFW_MFA2_TLV_MULTI_PART) {
 		multi = mlxfw_mfa2_tlv_multi_get(mfa2_file, tlv);
+		if (!multi)
+			return NULL;
 		tlv_len = NLA_ALIGN(tlv_len + be16_to_cpu(multi->total_len));
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
index 09ed6e5fa6c3..70d7fff24fa2 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
@@ -19,6 +19,9 @@
 #define MLXSW_THERMAL_ASIC_TEMP_NORM	75000	/* 75C */
 #define MLXSW_THERMAL_ASIC_TEMP_HIGH	85000	/* 85C */
 #define MLXSW_THERMAL_ASIC_TEMP_HOT	105000	/* 105C */
+#define MLXSW_THERMAL_MODULE_TEMP_NORM	55000	/* 55C */
+#define MLXSW_THERMAL_MODULE_TEMP_HIGH	65000	/* 65C */
+#define MLXSW_THERMAL_MODULE_TEMP_HOT	80000	/* 80C */
 #define MLXSW_THERMAL_HYSTERESIS_TEMP	5000	/* 5C */
 #define MLXSW_THERMAL_MODULE_TEMP_SHIFT	(MLXSW_THERMAL_HYSTERESIS_TEMP * 2)
 #define MLXSW_THERMAL_MAX_STATE	10
@@ -30,12 +33,6 @@ static char * const mlxsw_thermal_external_allowed_cdev[] = {
 	"mlxreg_fan",
 };
 
-enum mlxsw_thermal_trips {
-	MLXSW_THERMAL_TEMP_TRIP_NORM,
-	MLXSW_THERMAL_TEMP_TRIP_HIGH,
-	MLXSW_THERMAL_TEMP_TRIP_HOT,
-};
-
 struct mlxsw_cooling_states {
 	int	min_state;
 	int	max_state;
@@ -59,6 +56,24 @@ static const struct thermal_trip default_thermal_trips[] = {
 	},
 };
 
+static const struct thermal_trip default_thermal_module_trips[] = {
+	{	/* In range - 0-40% PWM */
+		.type		= THERMAL_TRIP_ACTIVE,
+		.temperature	= MLXSW_THERMAL_MODULE_TEMP_NORM,
+		.hysteresis	= MLXSW_THERMAL_HYSTERESIS_TEMP,
+	},
+	{
+		/* In range - 40-100% PWM */
+		.type		= THERMAL_TRIP_ACTIVE,
+		.temperature	= MLXSW_THERMAL_MODULE_TEMP_HIGH,
+		.hysteresis	= MLXSW_THERMAL_HYSTERESIS_TEMP,
+	},
+	{	/* Warning */
+		.type		= THERMAL_TRIP_HOT,
+		.temperature	= MLXSW_THERMAL_MODULE_TEMP_HOT,
+	},
+};
+
 static const struct mlxsw_cooling_states default_cooling_states[] = {
 	{
 		.min_state	= 0,
@@ -140,67 +155,10 @@ static int mlxsw_get_cooling_device_idx(struct mlxsw_thermal *thermal,
 	return -ENODEV;
 }
 
-static void
-mlxsw_thermal_module_trips_reset(struct mlxsw_thermal_module *tz)
-{
-	tz->trips[MLXSW_THERMAL_TEMP_TRIP_NORM].temperature = 0;
-	tz->trips[MLXSW_THERMAL_TEMP_TRIP_HIGH].temperature = 0;
-	tz->trips[MLXSW_THERMAL_TEMP_TRIP_HOT].temperature = 0;
-}
-
-static int
-mlxsw_thermal_module_trips_update(struct device *dev, struct mlxsw_core *core,
-				  struct mlxsw_thermal_module *tz,
-				  int crit_temp, int emerg_temp)
-{
-	int err;
-
-	/* Do not try to query temperature thresholds directly from the module's
-	 * EEPROM if we got valid thresholds from MTMP.
-	 */
-	if (!emerg_temp || !crit_temp) {
-		err = mlxsw_env_module_temp_thresholds_get(core, tz->slot_index,
-							   tz->module,
-							   SFP_TEMP_HIGH_WARN,
-							   &crit_temp);
-		if (err)
-			return err;
-
-		err = mlxsw_env_module_temp_thresholds_get(core, tz->slot_index,
-							   tz->module,
-							   SFP_TEMP_HIGH_ALARM,
-							   &emerg_temp);
-		if (err)
-			return err;
-	}
-
-	if (crit_temp > emerg_temp) {
-		dev_warn(dev, "%s : Critical threshold %d is above emergency threshold %d\n",
-			 tz->tzdev->type, crit_temp, emerg_temp);
-		return 0;
-	}
-
-	/* According to the system thermal requirements, the thermal zones are
-	 * defined with three trip points. The critical and emergency
-	 * temperature thresholds, provided by QSFP module are set as "active"
-	 * and "hot" trip points, "normal" trip point is derived from "active"
-	 * by subtracting double hysteresis value.
-	 */
-	if (crit_temp >= MLXSW_THERMAL_MODULE_TEMP_SHIFT)
-		tz->trips[MLXSW_THERMAL_TEMP_TRIP_NORM].temperature = crit_temp -
-					MLXSW_THERMAL_MODULE_TEMP_SHIFT;
-	else
-		tz->trips[MLXSW_THERMAL_TEMP_TRIP_NORM].temperature = crit_temp;
-	tz->trips[MLXSW_THERMAL_TEMP_TRIP_HIGH].temperature = crit_temp;
-	tz->trips[MLXSW_THERMAL_TEMP_TRIP_HOT].temperature = emerg_temp;
-
-	return 0;
-}
-
 static int mlxsw_thermal_bind(struct thermal_zone_device *tzdev,
 			      struct thermal_cooling_device *cdev)
 {
-	struct mlxsw_thermal *thermal = tzdev->devdata;
+	struct mlxsw_thermal *thermal = thermal_zone_device_priv(tzdev);
 	struct device *dev = thermal->bus_info->dev;
 	int i, err;
 
@@ -226,7 +184,7 @@ static int mlxsw_thermal_bind(struct thermal_zone_device *tzdev,
 static int mlxsw_thermal_unbind(struct thermal_zone_device *tzdev,
 				struct thermal_cooling_device *cdev)
 {
-	struct mlxsw_thermal *thermal = tzdev->devdata;
+	struct mlxsw_thermal *thermal = thermal_zone_device_priv(tzdev);
 	struct device *dev = thermal->bus_info->dev;
 	int i;
 	int err;
@@ -248,7 +206,7 @@ static int mlxsw_thermal_unbind(struct thermal_zone_device *tzdev,
 static int mlxsw_thermal_get_temp(struct thermal_zone_device *tzdev,
 				  int *p_temp)
 {
-	struct mlxsw_thermal *thermal = tzdev->devdata;
+	struct mlxsw_thermal *thermal = thermal_zone_device_priv(tzdev);
 	struct device *dev = thermal->bus_info->dev;
 	char mtmp_pl[MLXSW_REG_MTMP_LEN];
 	int temp;
@@ -280,7 +238,7 @@ static struct thermal_zone_device_ops mlxsw_thermal_ops = {
 static int mlxsw_thermal_module_bind(struct thermal_zone_device *tzdev,
 				     struct thermal_cooling_device *cdev)
 {
-	struct mlxsw_thermal_module *tz = tzdev->devdata;
+	struct mlxsw_thermal_module *tz = thermal_zone_device_priv(tzdev);
 	struct mlxsw_thermal *thermal = tz->parent;
 	int i, j, err;
 
@@ -309,7 +267,7 @@ err_thermal_zone_bind_cooling_device:
 static int mlxsw_thermal_module_unbind(struct thermal_zone_device *tzdev,
 				       struct thermal_cooling_device *cdev)
 {
-	struct mlxsw_thermal_module *tz = tzdev->devdata;
+	struct mlxsw_thermal_module *tz = thermal_zone_device_priv(tzdev);
 	struct mlxsw_thermal *thermal = tz->parent;
 	int i;
 	int err;
@@ -325,59 +283,22 @@ static int mlxsw_thermal_module_unbind(struct thermal_zone_device *tzdev,
 	return err;
 }
 
-static void
-mlxsw_thermal_module_temp_and_thresholds_get(struct mlxsw_core *core,
-					     u8 slot_index, u16 sensor_index,
-					     int *p_temp, int *p_crit_temp,
-					     int *p_emerg_temp)
-{
-	char mtmp_pl[MLXSW_REG_MTMP_LEN];
-	int err;
-
-	/* Read module temperature and thresholds. */
-	mlxsw_reg_mtmp_pack(mtmp_pl, slot_index, sensor_index,
-			    false, false);
-	err = mlxsw_reg_query(core, MLXSW_REG(mtmp), mtmp_pl);
-	if (err) {
-		/* Set temperature and thresholds to zero to avoid passing
-		 * uninitialized data back to the caller.
-		 */
-		*p_temp = 0;
-		*p_crit_temp = 0;
-		*p_emerg_temp = 0;
-
-		return;
-	}
-	mlxsw_reg_mtmp_unpack(mtmp_pl, p_temp, NULL, p_crit_temp, p_emerg_temp,
-			      NULL);
-}
-
 static int mlxsw_thermal_module_temp_get(struct thermal_zone_device *tzdev,
 					 int *p_temp)
 {
-	struct mlxsw_thermal_module *tz = tzdev->devdata;
+	struct mlxsw_thermal_module *tz = thermal_zone_device_priv(tzdev);
 	struct mlxsw_thermal *thermal = tz->parent;
-	int temp, crit_temp, emerg_temp;
-	struct device *dev;
+	char mtmp_pl[MLXSW_REG_MTMP_LEN];
 	u16 sensor_index;
+	int err;
 
-	dev = thermal->bus_info->dev;
 	sensor_index = MLXSW_REG_MTMP_MODULE_INDEX_MIN + tz->module;
-
-	/* Read module temperature and thresholds. */
-	mlxsw_thermal_module_temp_and_thresholds_get(thermal->core,
-						     tz->slot_index,
-						     sensor_index, &temp,
-						     &crit_temp, &emerg_temp);
-	*p_temp = temp;
-
-	if (!temp)
-		return 0;
-
-	/* Update trip points. */
-	mlxsw_thermal_module_trips_update(dev, thermal->core, tz,
-					  crit_temp, emerg_temp);
-
+	mlxsw_reg_mtmp_pack(mtmp_pl, tz->slot_index, sensor_index,
+			    false, false);
+	err = mlxsw_reg_query(thermal->core, MLXSW_REG(mtmp), mtmp_pl);
+	if (err)
+		return err;
+	mlxsw_reg_mtmp_unpack(mtmp_pl, p_temp, NULL, NULL, NULL, NULL);
 	return 0;
 }
 
@@ -390,7 +311,7 @@ static struct thermal_zone_device_ops mlxsw_thermal_module_ops = {
 static int mlxsw_thermal_gearbox_temp_get(struct thermal_zone_device *tzdev,
 					  int *p_temp)
 {
-	struct mlxsw_thermal_module *tz = tzdev->devdata;
+	struct mlxsw_thermal_module *tz = thermal_zone_device_priv(tzdev);
 	struct mlxsw_thermal *thermal = tz->parent;
 	char mtmp_pl[MLXSW_REG_MTMP_LEN];
 	u16 index;
@@ -521,36 +442,26 @@ static void mlxsw_thermal_module_tz_fini(struct thermal_zone_device *tzdev)
 	thermal_zone_device_unregister(tzdev);
 }
 
-static int
+static void
 mlxsw_thermal_module_init(struct device *dev, struct mlxsw_core *core,
 			  struct mlxsw_thermal *thermal,
 			  struct mlxsw_thermal_area *area, u8 module)
 {
 	struct mlxsw_thermal_module *module_tz;
-	int dummy_temp, crit_temp, emerg_temp;
-	u16 sensor_index;
 
-	sensor_index = MLXSW_REG_MTMP_MODULE_INDEX_MIN + module;
 	module_tz = &area->tz_module_arr[module];
 	/* Skip if parent is already set (case of port split). */
 	if (module_tz->parent)
-		return 0;
+		return;
 	module_tz->module = module;
 	module_tz->slot_index = area->slot_index;
 	module_tz->parent = thermal;
-	memcpy(module_tz->trips, default_thermal_trips,
+	BUILD_BUG_ON(ARRAY_SIZE(default_thermal_module_trips) !=
+		     MLXSW_THERMAL_NUM_TRIPS);
+	memcpy(module_tz->trips, default_thermal_module_trips,
 	       sizeof(thermal->trips));
 	memcpy(module_tz->cooling_states, default_cooling_states,
 	       sizeof(thermal->cooling_states));
-	/* Initialize all trip point. */
-	mlxsw_thermal_module_trips_reset(module_tz);
-	/* Read module temperature and thresholds. */
-	mlxsw_thermal_module_temp_and_thresholds_get(core, area->slot_index,
-						     sensor_index, &dummy_temp,
-						     &crit_temp, &emerg_temp);
-	/* Update trip point according to the module data. */
-	return mlxsw_thermal_module_trips_update(dev, core, module_tz,
-						 crit_temp, emerg_temp);
 }
 
 static void mlxsw_thermal_module_fini(struct mlxsw_thermal_module *module_tz)
@@ -589,11 +500,8 @@ mlxsw_thermal_modules_init(struct device *dev, struct mlxsw_core *core,
 	if (!area->tz_module_arr)
 		return -ENOMEM;
 
-	for (i = 0; i < area->tz_module_num; i++) {
-		err = mlxsw_thermal_module_init(dev, core, thermal, area, i);
-		if (err)
-			goto err_thermal_module_init;
-	}
+	for (i = 0; i < area->tz_module_num; i++)
+		mlxsw_thermal_module_init(dev, core, thermal, area, i);
 
 	for (i = 0; i < area->tz_module_num; i++) {
 		module_tz = &area->tz_module_arr[i];
@@ -607,7 +515,6 @@ mlxsw_thermal_modules_init(struct device *dev, struct mlxsw_core *core,
 	return 0;
 
 err_thermal_module_tz_init:
-err_thermal_module_init:
 	for (i = area->tz_module_num - 1; i >= 0; i--)
 		mlxsw_thermal_module_fini(&area->tz_module_arr[i]);
 	kfree(area->tz_module_arr);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h
index 48dbfea0a2a1..7cdf0ce24f28 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h
@@ -26,7 +26,7 @@
 #define MLXSW_PCI_CIR_TIMEOUT_MSECS		1000
 
 #define MLXSW_PCI_SW_RESET_TIMEOUT_MSECS	900000
-#define MLXSW_PCI_SW_RESET_WAIT_MSECS		200
+#define MLXSW_PCI_SW_RESET_WAIT_MSECS		400
 #define MLXSW_PCI_FW_READY			0xA1844
 #define MLXSW_PCI_FW_READY_MASK			0xFFFF
 #define MLXSW_PCI_FW_READY_MAGIC		0x5E
diff --git a/drivers/net/ethernet/micrel/ksz884x.c b/drivers/net/ethernet/micrel/ksz884x.c
index e6acd1e7b263..c5aeeb964c17 100644
--- a/drivers/net/ethernet/micrel/ksz884x.c
+++ b/drivers/net/ethernet/micrel/ksz884x.c
@@ -1476,15 +1476,6 @@ static void hw_turn_on_intr(struct ksz_hw *hw, u32 bit)
 		hw_set_intr(hw, hw->intr_mask);
 }
 
-static inline void hw_ena_intr_bit(struct ksz_hw *hw, uint interrupt)
-{
-	u32 read_intr;
-
-	read_intr = readl(hw->io + KS884X_INTERRUPTS_ENABLE);
-	hw->intr_set = read_intr | interrupt;
-	writel(hw->intr_set, hw->io + KS884X_INTERRUPTS_ENABLE);
-}
-
 static inline void hw_read_intr(struct ksz_hw *hw, uint *status)
 {
 	*status = readl(hw->io + KS884X_INTERRUPTS_STATUS);
@@ -1854,29 +1845,6 @@ static void port_init_cnt(struct ksz_hw *hw, int port)
  */
 
 /**
- * port_chk - check port register bits
- * @hw: 	The hardware instance.
- * @port:	The port index.
- * @offset:	The offset of the port register.
- * @bits:	The data bits to check.
- *
- * This function checks whether the specified bits of the port register are set
- * or not.
- *
- * Return 0 if the bits are not set.
- */
-static int port_chk(struct ksz_hw *hw, int port, int offset, u16 bits)
-{
-	u32 addr;
-	u16 data;
-
-	PORT_CTRL_ADDR(port, addr);
-	addr += offset;
-	data = readw(hw->io + addr);
-	return (data & bits) == bits;
-}
-
-/**
  * port_cfg - set port register bits
  * @hw: 	The hardware instance.
  * @port:	The port index.
@@ -1903,53 +1871,6 @@ static void port_cfg(struct ksz_hw *hw, int port, int offset, u16 bits,
 }
 
 /**
- * port_chk_shift - check port bit
- * @hw: 	The hardware instance.
- * @port:	The port index.
- * @addr:	The offset of the register.
- * @shift:	Number of bits to shift.
- *
- * This function checks whether the specified port is set in the register or
- * not.
- *
- * Return 0 if the port is not set.
- */
-static int port_chk_shift(struct ksz_hw *hw, int port, u32 addr, int shift)
-{
-	u16 data;
-	u16 bit = 1 << port;
-
-	data = readw(hw->io + addr);
-	data >>= shift;
-	return (data & bit) == bit;
-}
-
-/**
- * port_cfg_shift - set port bit
- * @hw: 	The hardware instance.
- * @port:	The port index.
- * @addr:	The offset of the register.
- * @shift:	Number of bits to shift.
- * @set:	The flag indicating whether the port is to be set or not.
- *
- * This routine sets or resets the specified port in the register.
- */
-static void port_cfg_shift(struct ksz_hw *hw, int port, u32 addr, int shift,
-	int set)
-{
-	u16 data;
-	u16 bits = 1 << port;
-
-	data = readw(hw->io + addr);
-	bits <<= shift;
-	if (set)
-		data |= bits;
-	else
-		data &= ~bits;
-	writew(data, hw->io + addr);
-}
-
-/**
  * port_r8 - read byte from port register
  * @hw: 	The hardware instance.
  * @port:	The port index.
@@ -2051,12 +1972,6 @@ static inline void port_cfg_broad_storm(struct ksz_hw *hw, int p, int set)
 		KS8842_PORT_CTRL_1_OFFSET, PORT_BROADCAST_STORM, set);
 }
 
-static inline int port_chk_broad_storm(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_1_OFFSET, PORT_BROADCAST_STORM);
-}
-
 /* Driver set switch broadcast storm protection at 10% rate. */
 #define BROADCAST_STORM_PROTECTION_RATE	10
 
@@ -2209,102 +2124,6 @@ static inline void port_cfg_back_pressure(struct ksz_hw *hw, int p, int set)
 		KS8842_PORT_CTRL_2_OFFSET, PORT_BACK_PRESSURE, set);
 }
 
-static inline void port_cfg_force_flow_ctrl(struct ksz_hw *hw, int p, int set)
-{
-	port_cfg(hw, p,
-		KS8842_PORT_CTRL_2_OFFSET, PORT_FORCE_FLOW_CTRL, set);
-}
-
-static inline int port_chk_back_pressure(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_2_OFFSET, PORT_BACK_PRESSURE);
-}
-
-static inline int port_chk_force_flow_ctrl(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_2_OFFSET, PORT_FORCE_FLOW_CTRL);
-}
-
-/* Spanning Tree */
-
-static inline void port_cfg_rx(struct ksz_hw *hw, int p, int set)
-{
-	port_cfg(hw, p,
-		KS8842_PORT_CTRL_2_OFFSET, PORT_RX_ENABLE, set);
-}
-
-static inline void port_cfg_tx(struct ksz_hw *hw, int p, int set)
-{
-	port_cfg(hw, p,
-		KS8842_PORT_CTRL_2_OFFSET, PORT_TX_ENABLE, set);
-}
-
-static inline void sw_cfg_fast_aging(struct ksz_hw *hw, int set)
-{
-	sw_cfg(hw, KS8842_SWITCH_CTRL_1_OFFSET, SWITCH_FAST_AGING, set);
-}
-
-static inline void sw_flush_dyn_mac_table(struct ksz_hw *hw)
-{
-	if (!(hw->overrides & FAST_AGING)) {
-		sw_cfg_fast_aging(hw, 1);
-		mdelay(1);
-		sw_cfg_fast_aging(hw, 0);
-	}
-}
-
-/* VLAN */
-
-static inline void port_cfg_ins_tag(struct ksz_hw *hw, int p, int insert)
-{
-	port_cfg(hw, p,
-		KS8842_PORT_CTRL_1_OFFSET, PORT_INSERT_TAG, insert);
-}
-
-static inline void port_cfg_rmv_tag(struct ksz_hw *hw, int p, int remove)
-{
-	port_cfg(hw, p,
-		KS8842_PORT_CTRL_1_OFFSET, PORT_REMOVE_TAG, remove);
-}
-
-static inline int port_chk_ins_tag(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_1_OFFSET, PORT_INSERT_TAG);
-}
-
-static inline int port_chk_rmv_tag(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_1_OFFSET, PORT_REMOVE_TAG);
-}
-
-static inline void port_cfg_dis_non_vid(struct ksz_hw *hw, int p, int set)
-{
-	port_cfg(hw, p,
-		KS8842_PORT_CTRL_2_OFFSET, PORT_DISCARD_NON_VID, set);
-}
-
-static inline void port_cfg_in_filter(struct ksz_hw *hw, int p, int set)
-{
-	port_cfg(hw, p,
-		KS8842_PORT_CTRL_2_OFFSET, PORT_INGRESS_VLAN_FILTER, set);
-}
-
-static inline int port_chk_dis_non_vid(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_2_OFFSET, PORT_DISCARD_NON_VID);
-}
-
-static inline int port_chk_in_filter(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_2_OFFSET, PORT_INGRESS_VLAN_FILTER);
-}
-
 /* Mirroring */
 
 static inline void port_cfg_mirror_sniffer(struct ksz_hw *hw, int p, int set)
@@ -2342,28 +2161,6 @@ static void sw_init_mirror(struct ksz_hw *hw)
 	sw_cfg_mirror_rx_tx(hw, 0);
 }
 
-static inline void sw_cfg_unk_def_deliver(struct ksz_hw *hw, int set)
-{
-	sw_cfg(hw, KS8842_SWITCH_CTRL_7_OFFSET,
-		SWITCH_UNK_DEF_PORT_ENABLE, set);
-}
-
-static inline int sw_cfg_chk_unk_def_deliver(struct ksz_hw *hw)
-{
-	return sw_chk(hw, KS8842_SWITCH_CTRL_7_OFFSET,
-		SWITCH_UNK_DEF_PORT_ENABLE);
-}
-
-static inline void sw_cfg_unk_def_port(struct ksz_hw *hw, int port, int set)
-{
-	port_cfg_shift(hw, port, KS8842_SWITCH_CTRL_7_OFFSET, 0, set);
-}
-
-static inline int sw_chk_unk_def_port(struct ksz_hw *hw, int port)
-{
-	return port_chk_shift(hw, port, KS8842_SWITCH_CTRL_7_OFFSET, 0);
-}
-
 /* Priority */
 
 static inline void port_cfg_diffserv(struct ksz_hw *hw, int p, int set)
@@ -2390,30 +2187,6 @@ static inline void port_cfg_prio(struct ksz_hw *hw, int p, int set)
 		KS8842_PORT_CTRL_1_OFFSET, PORT_PRIO_QUEUE_ENABLE, set);
 }
 
-static inline int port_chk_diffserv(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_1_OFFSET, PORT_DIFFSERV_ENABLE);
-}
-
-static inline int port_chk_802_1p(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_1_OFFSET, PORT_802_1P_ENABLE);
-}
-
-static inline int port_chk_replace_vid(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_2_OFFSET, PORT_USER_PRIORITY_CEILING);
-}
-
-static inline int port_chk_prio(struct ksz_hw *hw, int p)
-{
-	return port_chk(hw, p,
-		KS8842_PORT_CTRL_1_OFFSET, PORT_PRIO_QUEUE_ENABLE);
-}
-
 /**
  * sw_dis_diffserv - disable switch DiffServ priority
  * @hw: 	The hardware instance.
@@ -2614,23 +2387,6 @@ static void sw_cfg_port_base_vlan(struct ksz_hw *hw, int port, u8 member)
 }
 
 /**
- * sw_get_addr - get the switch MAC address.
- * @hw: 	The hardware instance.
- * @mac_addr:	Buffer to store the MAC address.
- *
- * This function retrieves the MAC address of the switch.
- */
-static inline void sw_get_addr(struct ksz_hw *hw, u8 *mac_addr)
-{
-	int i;
-
-	for (i = 0; i < 6; i += 2) {
-		mac_addr[i] = readb(hw->io + KS8842_MAC_ADDR_0_OFFSET + i);
-		mac_addr[1 + i] = readb(hw->io + KS8842_MAC_ADDR_1_OFFSET + i);
-	}
-}
-
-/**
  * sw_set_addr - configure switch MAC address
  * @hw: 	The hardware instance.
  * @mac_addr:	The MAC address.
@@ -2828,56 +2584,6 @@ static inline void hw_w_phy_ctrl(struct ksz_hw *hw, int phy, u16 data)
 	writew(data, hw->io + phy + KS884X_PHY_CTRL_OFFSET);
 }
 
-static inline void hw_r_phy_link_stat(struct ksz_hw *hw, int phy, u16 *data)
-{
-	*data = readw(hw->io + phy + KS884X_PHY_STATUS_OFFSET);
-}
-
-static inline void hw_r_phy_auto_neg(struct ksz_hw *hw, int phy, u16 *data)
-{
-	*data = readw(hw->io + phy + KS884X_PHY_AUTO_NEG_OFFSET);
-}
-
-static inline void hw_w_phy_auto_neg(struct ksz_hw *hw, int phy, u16 data)
-{
-	writew(data, hw->io + phy + KS884X_PHY_AUTO_NEG_OFFSET);
-}
-
-static inline void hw_r_phy_rem_cap(struct ksz_hw *hw, int phy, u16 *data)
-{
-	*data = readw(hw->io + phy + KS884X_PHY_REMOTE_CAP_OFFSET);
-}
-
-static inline void hw_r_phy_crossover(struct ksz_hw *hw, int phy, u16 *data)
-{
-	*data = readw(hw->io + phy + KS884X_PHY_CTRL_OFFSET);
-}
-
-static inline void hw_w_phy_crossover(struct ksz_hw *hw, int phy, u16 data)
-{
-	writew(data, hw->io + phy + KS884X_PHY_CTRL_OFFSET);
-}
-
-static inline void hw_r_phy_polarity(struct ksz_hw *hw, int phy, u16 *data)
-{
-	*data = readw(hw->io + phy + KS884X_PHY_PHY_CTRL_OFFSET);
-}
-
-static inline void hw_w_phy_polarity(struct ksz_hw *hw, int phy, u16 data)
-{
-	writew(data, hw->io + phy + KS884X_PHY_PHY_CTRL_OFFSET);
-}
-
-static inline void hw_r_phy_link_md(struct ksz_hw *hw, int phy, u16 *data)
-{
-	*data = readw(hw->io + phy + KS884X_PHY_LINK_MD_OFFSET);
-}
-
-static inline void hw_w_phy_link_md(struct ksz_hw *hw, int phy, u16 data)
-{
-	writew(data, hw->io + phy + KS884X_PHY_LINK_MD_OFFSET);
-}
-
 /**
  * hw_r_phy - read data from PHY register
  * @hw: 	The hardware instance.
@@ -3213,7 +2919,6 @@ static void port_get_link_speed(struct ksz_port *port)
 	u8 remote;
 	int i;
 	int p;
-	int change = 0;
 
 	interrupt = hw_block_intr(hw);
 
@@ -3260,17 +2965,14 @@ static void port_get_link_speed(struct ksz_port *port)
 					port_cfg_back_pressure(hw, p,
 						(1 == info->duplex));
 				}
-				change |= 1 << i;
 				port_cfg_change(hw, port, info, status);
 			}
 			info->state = media_connected;
 		} else {
-			if (media_disconnected != info->state) {
-				change |= 1 << i;
-
-				/* Indicate the link just goes down. */
+			/* Indicate the link just goes down. */
+			if (media_disconnected != info->state)
 				hw->port_mib[p].link_down = 1;
-			}
+
 			info->state = media_disconnected;
 		}
 		hw->port_mib[p].state = (u8) info->state;
diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c
index 7e0871b631e4..957d96a91a8a 100644
--- a/drivers/net/ethernet/microchip/lan743x_main.c
+++ b/drivers/net/ethernet/microchip/lan743x_main.c
@@ -1466,7 +1466,6 @@ static void lan743x_phy_close(struct lan743x_adapter *adapter)
 
 	phy_stop(netdev->phydev);
 	phy_disconnect(netdev->phydev);
-	netdev->phydev = NULL;
 }
 
 static void lan743x_phy_interface_select(struct lan743x_adapter *adapter)
diff --git a/drivers/net/ethernet/microchip/lan966x/Kconfig b/drivers/net/ethernet/microchip/lan966x/Kconfig
index 8bcd60f17d6d..571e6d4da1e9 100644
--- a/drivers/net/ethernet/microchip/lan966x/Kconfig
+++ b/drivers/net/ethernet/microchip/lan966x/Kconfig
@@ -6,7 +6,6 @@ config LAN966X_SWITCH
 	depends on NET_SWITCHDEV
 	depends on BRIDGE || BRIDGE=n
 	select PHYLINK
-	select PACKING
 	select PAGE_POOL
 	select VCAP
 	help
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
index 55b484b10562..bd72fbc2220f 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
@@ -390,6 +390,7 @@ static void lan966x_fdma_stop_netdev(struct lan966x *lan966x)
 static void lan966x_fdma_tx_clear_buf(struct lan966x *lan966x, int weight)
 {
 	struct lan966x_tx *tx = &lan966x->tx;
+	struct lan966x_rx *rx = &lan966x->rx;
 	struct lan966x_tx_dcb_buf *dcb_buf;
 	struct xdp_frame_bulk bq;
 	struct lan966x_db *db;
@@ -432,7 +433,8 @@ static void lan966x_fdma_tx_clear_buf(struct lan966x *lan966x, int weight)
 			if (dcb_buf->xdp_ndo)
 				xdp_return_frame_bulk(dcb_buf->data.xdpf, &bq);
 			else
-				xdp_return_frame_rx_napi(dcb_buf->data.xdpf);
+				page_pool_recycle_direct(rx->page_pool,
+							 dcb_buf->data.page);
 		}
 
 		clear = true;
@@ -517,7 +519,7 @@ static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx,
 	if (likely(!(skb->dev->features & NETIF_F_RXFCS)))
 		skb_trim(skb, skb->len - ETH_FCS_LEN);
 
-	lan966x_ptp_rxtstamp(lan966x, skb, timestamp);
+	lan966x_ptp_rxtstamp(lan966x, skb, src_port, timestamp);
 	skb->protocol = eth_type_trans(skb, skb->dev);
 
 	if (lan966x->bridge_mask & BIT(src_port)) {
@@ -699,15 +701,14 @@ static void lan966x_fdma_tx_start(struct lan966x_tx *tx, int next_to_use)
 	tx->last_in_use = next_to_use;
 }
 
-int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
-			   struct xdp_frame *xdpf,
-			   struct page *page,
-			   bool dma_map)
+int lan966x_fdma_xmit_xdpf(struct lan966x_port *port, void *ptr, u32 len)
 {
 	struct lan966x *lan966x = port->lan966x;
 	struct lan966x_tx_dcb_buf *next_dcb_buf;
 	struct lan966x_tx *tx = &lan966x->tx;
+	struct xdp_frame *xdpf;
 	dma_addr_t dma_addr;
+	struct page *page;
 	int next_to_use;
 	__be32 *ifh;
 	int ret = 0;
@@ -722,8 +723,13 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
 		goto out;
 	}
 
+	/* Get the next buffer */
+	next_dcb_buf = &tx->dcbs_buf[next_to_use];
+
 	/* Generate new IFH */
-	if (dma_map) {
+	if (!len) {
+		xdpf = ptr;
+
 		if (xdpf->headroom < IFH_LEN_BYTES) {
 			ret = NETDEV_TX_OK;
 			goto out;
@@ -743,11 +749,16 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
 			goto out;
 		}
 
+		next_dcb_buf->data.xdpf = xdpf;
+		next_dcb_buf->len = xdpf->len + IFH_LEN_BYTES;
+
 		/* Setup next dcb */
 		lan966x_fdma_tx_setup_dcb(tx, next_to_use,
 					  xdpf->len + IFH_LEN_BYTES,
 					  dma_addr);
 	} else {
+		page = ptr;
+
 		ifh = page_address(page) + XDP_PACKET_HEADROOM;
 		memset(ifh, 0x0, sizeof(__be32) * IFH_LEN);
 		lan966x_ifh_set_bypass(ifh, 1);
@@ -756,21 +767,21 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
 		dma_addr = page_pool_get_dma_addr(page);
 		dma_sync_single_for_device(lan966x->dev,
 					   dma_addr + XDP_PACKET_HEADROOM,
-					   xdpf->len + IFH_LEN_BYTES,
+					   len + IFH_LEN_BYTES,
 					   DMA_TO_DEVICE);
 
+		next_dcb_buf->data.page = page;
+		next_dcb_buf->len = len + IFH_LEN_BYTES;
+
 		/* Setup next dcb */
 		lan966x_fdma_tx_setup_dcb(tx, next_to_use,
-					  xdpf->len + IFH_LEN_BYTES,
+					  len + IFH_LEN_BYTES,
 					  dma_addr + XDP_PACKET_HEADROOM);
 	}
 
 	/* Fill up the buffer */
-	next_dcb_buf = &tx->dcbs_buf[next_to_use];
 	next_dcb_buf->use_skb = false;
-	next_dcb_buf->data.xdpf = xdpf;
-	next_dcb_buf->xdp_ndo = dma_map;
-	next_dcb_buf->len = xdpf->len + IFH_LEN_BYTES;
+	next_dcb_buf->xdp_ndo = !len;
 	next_dcb_buf->dma_addr = dma_addr;
 	next_dcb_buf->used = true;
 	next_dcb_buf->ptp = false;
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c
index 685e8cd7658c..2b6e046e1d10 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c
@@ -7,7 +7,6 @@
 #include <linux/ip.h>
 #include <linux/of_platform.h>
 #include <linux/of_net.h>
-#include <linux/packing.h>
 #include <linux/phy/phy.h>
 #include <linux/reset.h>
 #include <net/addrconf.h>
@@ -305,46 +304,57 @@ err:
 	return NETDEV_TX_BUSY;
 }
 
+static void lan966x_ifh_set(u8 *ifh, size_t val, size_t pos, size_t length)
+{
+	int i = 0;
+
+	do {
+		u8 p = IFH_LEN_BYTES - (pos + i) / 8 - 1;
+		u8 v = val >> i & 0xff;
+
+		/* There is no need to check for limits of the array, as these
+		 * will never be written
+		 */
+		ifh[p] |= v << ((pos + i) % 8);
+		ifh[p - 1] |= v >> (8 - (pos + i) % 8);
+
+		i += 8;
+	} while (i < length);
+}
+
 void lan966x_ifh_set_bypass(void *ifh, u64 bypass)
 {
-	packing(ifh, &bypass, IFH_POS_BYPASS + IFH_WID_BYPASS - 1,
-		IFH_POS_BYPASS, IFH_LEN * 4, PACK, 0);
+	lan966x_ifh_set(ifh, bypass, IFH_POS_BYPASS, IFH_WID_BYPASS);
 }
 
-void lan966x_ifh_set_port(void *ifh, u64 bypass)
+void lan966x_ifh_set_port(void *ifh, u64 port)
 {
-	packing(ifh, &bypass, IFH_POS_DSTS + IFH_WID_DSTS - 1,
-		IFH_POS_DSTS, IFH_LEN * 4, PACK, 0);
+	lan966x_ifh_set(ifh, port, IFH_POS_DSTS, IFH_WID_DSTS);
 }
 
-static void lan966x_ifh_set_qos_class(void *ifh, u64 bypass)
+static void lan966x_ifh_set_qos_class(void *ifh, u64 qos)
 {
-	packing(ifh, &bypass, IFH_POS_QOS_CLASS + IFH_WID_QOS_CLASS - 1,
-		IFH_POS_QOS_CLASS, IFH_LEN * 4, PACK, 0);
+	lan966x_ifh_set(ifh, qos, IFH_POS_QOS_CLASS, IFH_WID_QOS_CLASS);
 }
 
-static void lan966x_ifh_set_ipv(void *ifh, u64 bypass)
+static void lan966x_ifh_set_ipv(void *ifh, u64 ipv)
 {
-	packing(ifh, &bypass, IFH_POS_IPV + IFH_WID_IPV - 1,
-		IFH_POS_IPV, IFH_LEN * 4, PACK, 0);
+	lan966x_ifh_set(ifh, ipv, IFH_POS_IPV, IFH_WID_IPV);
 }
 
 static void lan966x_ifh_set_vid(void *ifh, u64 vid)
 {
-	packing(ifh, &vid, IFH_POS_TCI + IFH_WID_TCI - 1,
-		IFH_POS_TCI, IFH_LEN * 4, PACK, 0);
+	lan966x_ifh_set(ifh, vid, IFH_POS_TCI, IFH_WID_TCI);
 }
 
 static void lan966x_ifh_set_rew_op(void *ifh, u64 rew_op)
 {
-	packing(ifh, &rew_op, IFH_POS_REW_CMD + IFH_WID_REW_CMD - 1,
-		IFH_POS_REW_CMD, IFH_LEN * 4, PACK, 0);
+	lan966x_ifh_set(ifh, rew_op, IFH_POS_REW_CMD, IFH_WID_REW_CMD);
 }
 
 static void lan966x_ifh_set_timestamp(void *ifh, u64 timestamp)
 {
-	packing(ifh, &timestamp, IFH_POS_TIMESTAMP + IFH_WID_TIMESTAMP - 1,
-		IFH_POS_TIMESTAMP, IFH_LEN * 4, PACK, 0);
+	lan966x_ifh_set(ifh, timestamp, IFH_POS_TIMESTAMP, IFH_WID_TIMESTAMP);
 }
 
 static netdev_tx_t lan966x_port_xmit(struct sk_buff *skb,
@@ -582,22 +592,38 @@ static int lan966x_rx_frame_word(struct lan966x *lan966x, u8 grp, u32 *rval)
 	}
 }
 
+static u64 lan966x_ifh_get(u8 *ifh, size_t pos, size_t length)
+{
+	u64 val = 0;
+	u8 v;
+
+	for (int i = 0; i < length ; i++) {
+		int j = pos + i;
+		int k = j % 8;
+
+		if (i == 0 || k == 0)
+			v = ifh[IFH_LEN_BYTES - (j / 8) - 1];
+
+		if (v & (1 << k))
+			val |= (1ULL << i);
+	}
+
+	return val;
+}
+
 void lan966x_ifh_get_src_port(void *ifh, u64 *src_port)
 {
-	packing(ifh, src_port, IFH_POS_SRCPORT + IFH_WID_SRCPORT - 1,
-		IFH_POS_SRCPORT, IFH_LEN * 4, UNPACK, 0);
+	*src_port = lan966x_ifh_get(ifh, IFH_POS_SRCPORT, IFH_WID_SRCPORT);
 }
 
 static void lan966x_ifh_get_len(void *ifh, u64 *len)
 {
-	packing(ifh, len, IFH_POS_LEN + IFH_WID_LEN - 1,
-		IFH_POS_LEN, IFH_LEN * 4, UNPACK, 0);
+	*len = lan966x_ifh_get(ifh, IFH_POS_LEN, IFH_WID_LEN);
 }
 
 void lan966x_ifh_get_timestamp(void *ifh, u64 *timestamp)
 {
-	packing(ifh, timestamp, IFH_POS_TIMESTAMP + IFH_WID_TIMESTAMP - 1,
-		IFH_POS_TIMESTAMP, IFH_LEN * 4, UNPACK, 0);
+	*timestamp = lan966x_ifh_get(ifh, IFH_POS_TIMESTAMP, IFH_WID_TIMESTAMP);
 }
 
 static irqreturn_t lan966x_xtr_irq_handler(int irq, void *args)
@@ -668,7 +694,7 @@ static irqreturn_t lan966x_xtr_irq_handler(int irq, void *args)
 			*buf = val;
 		}
 
-		lan966x_ptp_rxtstamp(lan966x, skb, timestamp);
+		lan966x_ptp_rxtstamp(lan966x, skb, src_port, timestamp);
 		skb->protocol = eth_type_trans(skb, dev);
 
 		if (lan966x->bridge_mask & BIT(src_port)) {
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
index 49f5159afbf3..c977c70abc3d 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
@@ -92,6 +92,11 @@
 #define SE_IDX_QUEUE			0  /* 0-79 : Queue scheduler elements */
 #define SE_IDX_PORT			80 /* 80-89 : Port schedular elements */
 
+#define LAN966X_VCAP_CID_IS1_L0 VCAP_CID_INGRESS_L0 /* IS1 lookup 0 */
+#define LAN966X_VCAP_CID_IS1_L1 VCAP_CID_INGRESS_L1 /* IS1 lookup 1 */
+#define LAN966X_VCAP_CID_IS1_L2 VCAP_CID_INGRESS_L2 /* IS1 lookup 2 */
+#define LAN966X_VCAP_CID_IS1_MAX (VCAP_CID_INGRESS_L3 - 1) /* IS1 Max */
+
 #define LAN966X_VCAP_CID_IS2_L0 VCAP_CID_INGRESS_STAGE2_L0 /* IS2 lookup 0 */
 #define LAN966X_VCAP_CID_IS2_L1 VCAP_CID_INGRESS_STAGE2_L1 /* IS2 lookup 1 */
 #define LAN966X_VCAP_CID_IS2_MAX (VCAP_CID_INGRESS_STAGE2_L2 - 1) /* IS2 Max */
@@ -139,6 +144,39 @@ enum vcap_is2_port_sel_ipv6 {
 	VCAP_IS2_PS_IPV6_MAC_ETYPE,
 };
 
+enum vcap_is1_port_sel_other {
+	VCAP_IS1_PS_OTHER_NORMAL,
+	VCAP_IS1_PS_OTHER_7TUPLE,
+	VCAP_IS1_PS_OTHER_DBL_VID,
+	VCAP_IS1_PS_OTHER_DMAC_VID,
+};
+
+enum vcap_is1_port_sel_ipv4 {
+	VCAP_IS1_PS_IPV4_NORMAL,
+	VCAP_IS1_PS_IPV4_7TUPLE,
+	VCAP_IS1_PS_IPV4_5TUPLE_IP4,
+	VCAP_IS1_PS_IPV4_DBL_VID,
+	VCAP_IS1_PS_IPV4_DMAC_VID,
+};
+
+enum vcap_is1_port_sel_ipv6 {
+	VCAP_IS1_PS_IPV6_NORMAL,
+	VCAP_IS1_PS_IPV6_7TUPLE,
+	VCAP_IS1_PS_IPV6_5TUPLE_IP4,
+	VCAP_IS1_PS_IPV6_NORMAL_IP6,
+	VCAP_IS1_PS_IPV6_5TUPLE_IP6,
+	VCAP_IS1_PS_IPV6_DBL_VID,
+	VCAP_IS1_PS_IPV6_DMAC_VID,
+};
+
+enum vcap_is1_port_sel_rt {
+	VCAP_IS1_PS_RT_NORMAL,
+	VCAP_IS1_PS_RT_7TUPLE,
+	VCAP_IS1_PS_RT_DBL_VID,
+	VCAP_IS1_PS_RT_DMAC_VID,
+	VCAP_IS1_PS_RT_FOLLOW_OTHER = 7,
+};
+
 struct lan966x_port;
 
 struct lan966x_db {
@@ -205,6 +243,7 @@ struct lan966x_tx_dcb_buf {
 	union {
 		struct sk_buff *skb;
 		struct xdp_frame *xdpf;
+		struct page *page;
 	} data;
 	u32 len;
 	u32 used : 1;
@@ -369,7 +408,8 @@ struct lan966x_port {
 	struct phy *serdes;
 	struct fwnode_handle *fwnode;
 
-	u8 ptp_cmd;
+	u8 ptp_tx_cmd;
+	bool ptp_rx_cmd;
 	u16 ts_id;
 	struct sk_buff_head tx_skbs;
 
@@ -489,7 +529,7 @@ void lan966x_ptp_deinit(struct lan966x *lan966x);
 int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, struct ifreq *ifr);
 int lan966x_ptp_hwtstamp_get(struct lan966x_port *port, struct ifreq *ifr);
 void lan966x_ptp_rxtstamp(struct lan966x *lan966x, struct sk_buff *skb,
-			  u64 timestamp);
+			  u64 src_port, u64 timestamp);
 int lan966x_ptp_txtstamp_request(struct lan966x_port *port,
 				 struct sk_buff *skb);
 void lan966x_ptp_txtstamp_release(struct lan966x_port *port,
@@ -502,10 +542,7 @@ int lan966x_ptp_setup_traps(struct lan966x_port *port, struct ifreq *ifr);
 int lan966x_ptp_del_traps(struct lan966x_port *port);
 
 int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev);
-int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
-			   struct xdp_frame *frame,
-			   struct page *page,
-			   bool dma_map);
+int lan966x_fdma_xmit_xdpf(struct lan966x_port *port, void *ptr, u32 len);
 int lan966x_fdma_change_mtu(struct lan966x *lan966x);
 void lan966x_fdma_netdev_init(struct lan966x *lan966x, struct net_device *dev);
 void lan966x_fdma_netdev_deinit(struct lan966x *lan966x, struct net_device *dev);
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_police.c b/drivers/net/ethernet/microchip/lan966x/lan966x_police.c
index 7d66fe75cd3b..7302df2300fd 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_police.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_police.c
@@ -49,8 +49,7 @@ static int lan966x_police_add(struct lan966x_port *port,
 	return 0;
 }
 
-static int lan966x_police_del(struct lan966x_port *port,
-			      u16 pol_idx)
+static void lan966x_police_del(struct lan966x_port *port, u16 pol_idx)
 {
 	struct lan966x *lan966x = port->lan966x;
 
@@ -67,8 +66,6 @@ static int lan966x_police_del(struct lan966x_port *port,
 	lan_wr(ANA_POL_PIR_CFG_PIR_RATE_SET(GENMASK(14, 0)) |
 	       ANA_POL_PIR_CFG_PIR_BURST_SET(0),
 	       lan966x, ANA_POL_PIR_CFG(pol_idx));
-
-	return 0;
 }
 
 static int lan966x_police_validate(struct lan966x_port *port,
@@ -186,7 +183,6 @@ int lan966x_police_port_del(struct lan966x_port *port,
 			    struct netlink_ext_ack *extack)
 {
 	struct lan966x *lan966x = port->lan966x;
-	int err;
 
 	if (port->tc.police_id != police_id) {
 		NL_SET_ERR_MSG_MOD(extack,
@@ -194,12 +190,7 @@ int lan966x_police_port_del(struct lan966x_port *port,
 		return -EINVAL;
 	}
 
-	err = lan966x_police_del(port, POL_IDX_PORT + port->chip_port);
-	if (err) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Failed to add policer to port");
-		return err;
-	}
+	lan966x_police_del(port, POL_IDX_PORT + port->chip_port);
 
 	lan_rmw(ANA_POL_CFG_PORT_POL_ENA_SET(0) |
 		ANA_POL_CFG_POL_ORDER_SET(POL_ORDER),
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c
index 931e37b9a0ad..266a21a2d124 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c
@@ -272,13 +272,13 @@ int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, struct ifreq *ifr)
 
 	switch (cfg.tx_type) {
 	case HWTSTAMP_TX_ON:
-		port->ptp_cmd = IFH_REW_OP_TWO_STEP_PTP;
+		port->ptp_tx_cmd = IFH_REW_OP_TWO_STEP_PTP;
 		break;
 	case HWTSTAMP_TX_ONESTEP_SYNC:
-		port->ptp_cmd = IFH_REW_OP_ONE_STEP_PTP;
+		port->ptp_tx_cmd = IFH_REW_OP_ONE_STEP_PTP;
 		break;
 	case HWTSTAMP_TX_OFF:
-		port->ptp_cmd = IFH_REW_OP_NOOP;
+		port->ptp_tx_cmd = IFH_REW_OP_NOOP;
 		break;
 	default:
 		return -ERANGE;
@@ -286,6 +286,7 @@ int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, struct ifreq *ifr)
 
 	switch (cfg.rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
+		port->ptp_rx_cmd = false;
 		break;
 	case HWTSTAMP_FILTER_ALL:
 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
@@ -301,6 +302,7 @@ int lan966x_ptp_hwtstamp_set(struct lan966x_port *port, struct ifreq *ifr)
 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 	case HWTSTAMP_FILTER_NTP_ALL:
+		port->ptp_rx_cmd = true;
 		cfg.rx_filter = HWTSTAMP_FILTER_ALL;
 		break;
 	default:
@@ -332,7 +334,7 @@ static int lan966x_ptp_classify(struct lan966x_port *port, struct sk_buff *skb)
 	u8 msgtype;
 	int type;
 
-	if (port->ptp_cmd == IFH_REW_OP_NOOP)
+	if (port->ptp_tx_cmd == IFH_REW_OP_NOOP)
 		return IFH_REW_OP_NOOP;
 
 	type = ptp_classify_raw(skb);
@@ -343,7 +345,7 @@ static int lan966x_ptp_classify(struct lan966x_port *port, struct sk_buff *skb)
 	if (!header)
 		return IFH_REW_OP_NOOP;
 
-	if (port->ptp_cmd == IFH_REW_OP_TWO_STEP_PTP)
+	if (port->ptp_tx_cmd == IFH_REW_OP_TWO_STEP_PTP)
 		return IFH_REW_OP_TWO_STEP_PTP;
 
 	/* If it is sync and run 1 step then set the correct operation,
@@ -1009,9 +1011,6 @@ static int lan966x_ptp_phc_init(struct lan966x *lan966x,
 	phc->index = index;
 	phc->lan966x = lan966x;
 
-	/* PTP Rx stamping is always enabled.  */
-	phc->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
-
 	return 0;
 }
 
@@ -1088,14 +1087,15 @@ void lan966x_ptp_deinit(struct lan966x *lan966x)
 }
 
 void lan966x_ptp_rxtstamp(struct lan966x *lan966x, struct sk_buff *skb,
-			  u64 timestamp)
+			  u64 src_port, u64 timestamp)
 {
 	struct skb_shared_hwtstamps *shhwtstamps;
 	struct lan966x_phc *phc;
 	struct timespec64 ts;
 	u64 full_ts_in_ns;
 
-	if (!lan966x->ptp)
+	if (!lan966x->ptp ||
+	    !lan966x->ports[src_port]->ptp_rx_cmd)
 		return;
 
 	phc = &lan966x->phc[LAN966X_PHC_PORT];
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h b/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h
index 9767b5a1c958..f99f88b5caa8 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h
@@ -316,6 +316,42 @@ enum lan966x_target {
 #define ANA_DROP_CFG_DROP_MC_SMAC_ENA_GET(x)\
 	FIELD_GET(ANA_DROP_CFG_DROP_MC_SMAC_ENA, x)
 
+/*      ANA:PORT:VCAP_CFG */
+#define ANA_VCAP_CFG(g)           __REG(TARGET_ANA, 0, 1, 28672, g, 9, 128, 12, 0, 1, 4)
+
+#define ANA_VCAP_CFG_S1_ENA                      BIT(14)
+#define ANA_VCAP_CFG_S1_ENA_SET(x)\
+	FIELD_PREP(ANA_VCAP_CFG_S1_ENA, x)
+#define ANA_VCAP_CFG_S1_ENA_GET(x)\
+	FIELD_GET(ANA_VCAP_CFG_S1_ENA, x)
+
+/*      ANA:PORT:VCAP_S1_KEY_CFG */
+#define ANA_VCAP_S1_CFG(g, r)     __REG(TARGET_ANA, 0, 1, 28672, g, 9, 128, 16, r, 3, 4)
+
+#define ANA_VCAP_S1_CFG_KEY_RT_CFG               GENMASK(11, 9)
+#define ANA_VCAP_S1_CFG_KEY_RT_CFG_SET(x)\
+	FIELD_PREP(ANA_VCAP_S1_CFG_KEY_RT_CFG, x)
+#define ANA_VCAP_S1_CFG_KEY_RT_CFG_GET(x)\
+	FIELD_GET(ANA_VCAP_S1_CFG_KEY_RT_CFG, x)
+
+#define ANA_VCAP_S1_CFG_KEY_IP6_CFG              GENMASK(8, 6)
+#define ANA_VCAP_S1_CFG_KEY_IP6_CFG_SET(x)\
+	FIELD_PREP(ANA_VCAP_S1_CFG_KEY_IP6_CFG, x)
+#define ANA_VCAP_S1_CFG_KEY_IP6_CFG_GET(x)\
+	FIELD_GET(ANA_VCAP_S1_CFG_KEY_IP6_CFG, x)
+
+#define ANA_VCAP_S1_CFG_KEY_IP4_CFG              GENMASK(5, 3)
+#define ANA_VCAP_S1_CFG_KEY_IP4_CFG_SET(x)\
+	FIELD_PREP(ANA_VCAP_S1_CFG_KEY_IP4_CFG, x)
+#define ANA_VCAP_S1_CFG_KEY_IP4_CFG_GET(x)\
+	FIELD_GET(ANA_VCAP_S1_CFG_KEY_IP4_CFG, x)
+
+#define ANA_VCAP_S1_CFG_KEY_OTHER_CFG            GENMASK(2, 0)
+#define ANA_VCAP_S1_CFG_KEY_OTHER_CFG_SET(x)\
+	FIELD_PREP(ANA_VCAP_S1_CFG_KEY_OTHER_CFG, x)
+#define ANA_VCAP_S1_CFG_KEY_OTHER_CFG_GET(x)\
+	FIELD_GET(ANA_VCAP_S1_CFG_KEY_OTHER_CFG, x)
+
 /*      ANA:PORT:VCAP_S2_CFG */
 #define ANA_VCAP_S2_CFG(g)        __REG(TARGET_ANA, 0, 1, 28672, g, 9, 128, 28, 0, 1, 4)
 
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c b/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c
index f960727ecaee..47b2f7579dd2 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_tc_flower.c
@@ -5,14 +5,34 @@
 #include "vcap_api_client.h"
 #include "vcap_tc.h"
 
-static bool lan966x_tc_is_known_etype(u16 etype)
+static bool lan966x_tc_is_known_etype(struct vcap_tc_flower_parse_usage *st,
+				      u16 etype)
 {
-	switch (etype) {
-	case ETH_P_ALL:
-	case ETH_P_ARP:
-	case ETH_P_IP:
-	case ETH_P_IPV6:
-		return true;
+	switch (st->admin->vtype) {
+	case VCAP_TYPE_IS1:
+		switch (etype) {
+		case ETH_P_ALL:
+		case ETH_P_ARP:
+		case ETH_P_IP:
+		case ETH_P_IPV6:
+			return true;
+		}
+		break;
+	case VCAP_TYPE_IS2:
+		switch (etype) {
+		case ETH_P_ALL:
+		case ETH_P_ARP:
+		case ETH_P_IP:
+		case ETH_P_IPV6:
+		case ETH_P_SNAP:
+		case ETH_P_802_2:
+			return true;
+		}
+		break;
+	default:
+		NL_SET_ERR_MSG_MOD(st->fco->common.extack,
+				   "VCAP type not supported");
+		return false;
 	}
 
 	return false;
@@ -69,7 +89,7 @@ lan966x_tc_flower_handler_basic_usage(struct vcap_tc_flower_parse_usage *st)
 	flow_rule_match_basic(st->frule, &match);
 	if (match.mask->n_proto) {
 		st->l3_proto = be16_to_cpu(match.key->n_proto);
-		if (!lan966x_tc_is_known_etype(st->l3_proto)) {
+		if (!lan966x_tc_is_known_etype(st, st->l3_proto)) {
 			err = vcap_rule_add_key_u32(st->vrule, VCAP_KF_ETYPE,
 						    st->l3_proto, ~0);
 			if (err)
@@ -79,18 +99,61 @@ lan966x_tc_flower_handler_basic_usage(struct vcap_tc_flower_parse_usage *st)
 						    VCAP_BIT_1);
 			if (err)
 				goto out;
+		} else if (st->l3_proto == ETH_P_IPV6 &&
+			   st->admin->vtype == VCAP_TYPE_IS1) {
+			/* Don't set any keys in this case */
+		} else if (st->l3_proto == ETH_P_SNAP &&
+			   st->admin->vtype == VCAP_TYPE_IS1) {
+			err = vcap_rule_add_key_bit(st->vrule,
+						    VCAP_KF_ETYPE_LEN_IS,
+						    VCAP_BIT_0);
+			if (err)
+				goto out;
+
+			err = vcap_rule_add_key_bit(st->vrule,
+						    VCAP_KF_IP_SNAP_IS,
+						    VCAP_BIT_1);
+			if (err)
+				goto out;
+		} else if (st->admin->vtype == VCAP_TYPE_IS1) {
+			err = vcap_rule_add_key_bit(st->vrule,
+						    VCAP_KF_ETYPE_LEN_IS,
+						    VCAP_BIT_1);
+			if (err)
+				goto out;
+
+			err = vcap_rule_add_key_u32(st->vrule, VCAP_KF_ETYPE,
+						    st->l3_proto, ~0);
+			if (err)
+				goto out;
 		}
 	}
 	if (match.mask->ip_proto) {
 		st->l4_proto = match.key->ip_proto;
 
 		if (st->l4_proto == IPPROTO_TCP) {
+			if (st->admin->vtype == VCAP_TYPE_IS1) {
+				err = vcap_rule_add_key_bit(st->vrule,
+							    VCAP_KF_TCP_UDP_IS,
+							    VCAP_BIT_1);
+				if (err)
+					goto out;
+			}
+
 			err = vcap_rule_add_key_bit(st->vrule,
 						    VCAP_KF_TCP_IS,
 						    VCAP_BIT_1);
 			if (err)
 				goto out;
 		} else if (st->l4_proto == IPPROTO_UDP) {
+			if (st->admin->vtype == VCAP_TYPE_IS1) {
+				err = vcap_rule_add_key_bit(st->vrule,
+							    VCAP_KF_TCP_UDP_IS,
+							    VCAP_BIT_1);
+				if (err)
+					goto out;
+			}
+
 			err = vcap_rule_add_key_bit(st->vrule,
 						    VCAP_KF_TCP_IS,
 						    VCAP_BIT_0);
@@ -113,11 +176,29 @@ out:
 }
 
 static int
+lan966x_tc_flower_handler_cvlan_usage(struct vcap_tc_flower_parse_usage *st)
+{
+	if (st->admin->vtype != VCAP_TYPE_IS1) {
+		NL_SET_ERR_MSG_MOD(st->fco->common.extack,
+				   "cvlan not supported in this VCAP");
+		return -EINVAL;
+	}
+
+	return vcap_tc_flower_handler_cvlan_usage(st);
+}
+
+static int
 lan966x_tc_flower_handler_vlan_usage(struct vcap_tc_flower_parse_usage *st)
 {
-	return vcap_tc_flower_handler_vlan_usage(st,
-						 VCAP_KF_8021Q_VID_CLS,
-						 VCAP_KF_8021Q_PCP_CLS);
+	enum vcap_key_field vid_key = VCAP_KF_8021Q_VID_CLS;
+	enum vcap_key_field pcp_key = VCAP_KF_8021Q_PCP_CLS;
+
+	if (st->admin->vtype == VCAP_TYPE_IS1) {
+		vid_key = VCAP_KF_8021Q_VID0;
+		pcp_key = VCAP_KF_8021Q_PCP0;
+	}
+
+	return vcap_tc_flower_handler_vlan_usage(st, vid_key, pcp_key);
 }
 
 static int
@@ -128,6 +209,7 @@ static int
 	[FLOW_DISSECTOR_KEY_CONTROL] = lan966x_tc_flower_handler_control_usage,
 	[FLOW_DISSECTOR_KEY_PORTS] = vcap_tc_flower_handler_portnum_usage,
 	[FLOW_DISSECTOR_KEY_BASIC] = lan966x_tc_flower_handler_basic_usage,
+	[FLOW_DISSECTOR_KEY_CVLAN] = lan966x_tc_flower_handler_cvlan_usage,
 	[FLOW_DISSECTOR_KEY_VLAN] = lan966x_tc_flower_handler_vlan_usage,
 	[FLOW_DISSECTOR_KEY_TCP] = vcap_tc_flower_handler_tcp_usage,
 	[FLOW_DISSECTOR_KEY_ARP] = vcap_tc_flower_handler_arp_usage,
@@ -143,6 +225,7 @@ static int lan966x_tc_flower_use_dissectors(struct flow_cls_offload *f,
 		.fco = f,
 		.vrule = vrule,
 		.l3_proto = ETH_P_ALL,
+		.admin = admin,
 	};
 	int err = 0;
 
@@ -221,6 +304,100 @@ static int lan966x_tc_flower_action_check(struct vcap_control *vctrl,
 	return 0;
 }
 
+/* Add the actionset that is the default for the VCAP type */
+static int lan966x_tc_set_actionset(struct vcap_admin *admin,
+				    struct vcap_rule *vrule)
+{
+	enum vcap_actionfield_set aset;
+	int err = 0;
+
+	switch (admin->vtype) {
+	case VCAP_TYPE_IS1:
+		aset = VCAP_AFS_S1;
+		break;
+	case VCAP_TYPE_IS2:
+		aset = VCAP_AFS_BASE_TYPE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Do not overwrite any current actionset */
+	if (vrule->actionset == VCAP_AFS_NO_VALUE)
+		err = vcap_set_rule_set_actionset(vrule, aset);
+
+	return err;
+}
+
+static int lan966x_tc_add_rule_link_target(struct vcap_admin *admin,
+					   struct vcap_rule *vrule,
+					   int target_cid)
+{
+	int link_val = target_cid % VCAP_CID_LOOKUP_SIZE;
+	int err;
+
+	if (!link_val)
+		return 0;
+
+	switch (admin->vtype) {
+	case VCAP_TYPE_IS1:
+		/* Choose IS1 specific NXT_IDX key (for chaining rules from IS1) */
+		err = vcap_rule_add_key_u32(vrule, VCAP_KF_LOOKUP_GEN_IDX_SEL,
+					    1, ~0);
+		if (err)
+			return err;
+
+		return vcap_rule_add_key_u32(vrule, VCAP_KF_LOOKUP_GEN_IDX,
+					     link_val, ~0);
+	case VCAP_TYPE_IS2:
+		/* Add IS2 specific PAG key (for chaining rules from IS1) */
+		return vcap_rule_add_key_u32(vrule, VCAP_KF_LOOKUP_PAG,
+					     link_val, ~0);
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int lan966x_tc_add_rule_link(struct vcap_control *vctrl,
+				    struct vcap_admin *admin,
+				    struct vcap_rule *vrule,
+				    struct flow_cls_offload *f,
+				    int to_cid)
+{
+	struct vcap_admin *to_admin = vcap_find_admin(vctrl, to_cid);
+	int diff, err = 0;
+
+	if (!to_admin) {
+		NL_SET_ERR_MSG_MOD(f->common.extack,
+				   "Unknown destination chain");
+		return -EINVAL;
+	}
+
+	diff = vcap_chain_offset(vctrl, f->common.chain_index, to_cid);
+	if (!diff)
+		return 0;
+
+	/* Between IS1 and IS2 the PAG value is used */
+	if (admin->vtype == VCAP_TYPE_IS1 && to_admin->vtype == VCAP_TYPE_IS2) {
+		/* This works for IS1->IS2 */
+		err = vcap_rule_add_action_u32(vrule, VCAP_AF_PAG_VAL, diff);
+		if (err)
+			return err;
+
+		err = vcap_rule_add_action_u32(vrule, VCAP_AF_PAG_OVERRIDE_MASK,
+					       0xff);
+		if (err)
+			return err;
+	} else {
+		NL_SET_ERR_MSG_MOD(f->common.extack,
+				   "Unsupported chain destination");
+		return -EOPNOTSUPP;
+	}
+
+	return err;
+}
+
 static int lan966x_tc_flower_add(struct lan966x_port *port,
 				 struct flow_cls_offload *f,
 				 struct vcap_admin *admin,
@@ -248,11 +425,23 @@ static int lan966x_tc_flower_add(struct lan966x_port *port,
 	if (err)
 		goto out;
 
+	err = lan966x_tc_add_rule_link_target(admin, vrule,
+					      f->common.chain_index);
+	if (err)
+		goto out;
+
 	frule = flow_cls_offload_flow_rule(f);
 
 	flow_action_for_each(idx, act, &frule->action) {
 		switch (act->id) {
 		case FLOW_ACTION_TRAP:
+			if (admin->vtype != VCAP_TYPE_IS2) {
+				NL_SET_ERR_MSG_MOD(f->common.extack,
+						   "Trap action not supported in this VCAP");
+				err = -EOPNOTSUPP;
+				goto out;
+			}
+
 			err = vcap_rule_add_action_bit(vrule,
 						       VCAP_AF_CPU_COPY_ENA,
 						       VCAP_BIT_1);
@@ -266,6 +455,16 @@ static int lan966x_tc_flower_add(struct lan966x_port *port,
 
 			break;
 		case FLOW_ACTION_GOTO:
+			err = lan966x_tc_set_actionset(admin, vrule);
+			if (err)
+				goto out;
+
+			err = lan966x_tc_add_rule_link(port->lan966x->vcap_ctrl,
+						       admin, vrule,
+						       f, act->chain_index);
+			if (err)
+				goto out;
+
 			break;
 		default:
 			NL_SET_ERR_MSG_MOD(f->common.extack,
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_ag_api.c b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_ag_api.c
index 928e711960e6..66400a082d02 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_ag_api.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_ag_api.c
@@ -6,6 +6,965 @@
 #include "lan966x_vcap_ag_api.h"
 
 /* keyfields */
+static const struct vcap_field is1_normal_keyfield[] = {
+	[VCAP_KF_TYPE] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 0,
+		.width = 1,
+	},
+	[VCAP_KF_LOOKUP_INDEX] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 1,
+		.width = 2,
+	},
+	[VCAP_KF_IF_IGR_PORT_MASK] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 3,
+		.width = 9,
+	},
+	[VCAP_KF_L2_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 12,
+		.width = 1,
+	},
+	[VCAP_KF_L2_BC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 13,
+		.width = 1,
+	},
+	[VCAP_KF_IP_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 14,
+		.width = 1,
+	},
+	[VCAP_KF_8021CB_R_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 15,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 16,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 17,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_TPID0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 18,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 19,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 31,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 32,
+		.width = 3,
+	},
+	[VCAP_KF_L2_SMAC] = {
+		.type = VCAP_FIELD_U48,
+		.offset = 35,
+		.width = 48,
+	},
+	[VCAP_KF_ETYPE_LEN_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 83,
+		.width = 1,
+	},
+	[VCAP_KF_ETYPE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 84,
+		.width = 16,
+	},
+	[VCAP_KF_IP_SNAP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 100,
+		.width = 1,
+	},
+	[VCAP_KF_IP4_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 101,
+		.width = 1,
+	},
+	[VCAP_KF_L3_FRAGMENT] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 102,
+		.width = 1,
+	},
+	[VCAP_KF_L3_FRAG_OFS_GT0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 103,
+		.width = 1,
+	},
+	[VCAP_KF_L3_OPTIONS_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 104,
+		.width = 1,
+	},
+	[VCAP_KF_L3_DSCP] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 105,
+		.width = 6,
+	},
+	[VCAP_KF_L3_IP4_SIP] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 111,
+		.width = 32,
+	},
+	[VCAP_KF_TCP_UDP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 143,
+		.width = 1,
+	},
+	[VCAP_KF_TCP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 144,
+		.width = 1,
+	},
+	[VCAP_KF_L4_SPORT] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 145,
+		.width = 16,
+	},
+	[VCAP_KF_L4_RNG] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 161,
+		.width = 8,
+	},
+};
+
+static const struct vcap_field is1_5tuple_ip4_keyfield[] = {
+	[VCAP_KF_TYPE] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 0,
+		.width = 1,
+	},
+	[VCAP_KF_LOOKUP_INDEX] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 1,
+		.width = 2,
+	},
+	[VCAP_KF_IF_IGR_PORT_MASK] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 3,
+		.width = 9,
+	},
+	[VCAP_KF_L2_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 12,
+		.width = 1,
+	},
+	[VCAP_KF_L2_BC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 13,
+		.width = 1,
+	},
+	[VCAP_KF_IP_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 14,
+		.width = 1,
+	},
+	[VCAP_KF_8021CB_R_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 15,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 16,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 17,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_TPID0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 18,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 19,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 31,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 32,
+		.width = 3,
+	},
+	[VCAP_KF_8021Q_TPID1] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 35,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID1] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 36,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI1] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 48,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP1] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 49,
+		.width = 3,
+	},
+	[VCAP_KF_IP4_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 52,
+		.width = 1,
+	},
+	[VCAP_KF_L3_FRAGMENT] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 53,
+		.width = 1,
+	},
+	[VCAP_KF_L3_FRAG_OFS_GT0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 54,
+		.width = 1,
+	},
+	[VCAP_KF_L3_OPTIONS_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 55,
+		.width = 1,
+	},
+	[VCAP_KF_L3_DSCP] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 56,
+		.width = 6,
+	},
+	[VCAP_KF_L3_IP4_DIP] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 62,
+		.width = 32,
+	},
+	[VCAP_KF_L3_IP4_SIP] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 94,
+		.width = 32,
+	},
+	[VCAP_KF_L3_IP_PROTO] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 126,
+		.width = 8,
+	},
+	[VCAP_KF_TCP_UDP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 134,
+		.width = 1,
+	},
+	[VCAP_KF_TCP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 135,
+		.width = 1,
+	},
+	[VCAP_KF_L4_RNG] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 136,
+		.width = 8,
+	},
+	[VCAP_KF_IP_PAYLOAD_5TUPLE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 144,
+		.width = 32,
+	},
+};
+
+static const struct vcap_field is1_normal_ip6_keyfield[] = {
+	[VCAP_KF_TYPE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 0,
+		.width = 2,
+	},
+	[VCAP_KF_LOOKUP_INDEX] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 2,
+		.width = 2,
+	},
+	[VCAP_KF_IF_IGR_PORT_MASK] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 4,
+		.width = 9,
+	},
+	[VCAP_KF_L2_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 13,
+		.width = 1,
+	},
+	[VCAP_KF_L2_BC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 14,
+		.width = 1,
+	},
+	[VCAP_KF_IP_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 15,
+		.width = 1,
+	},
+	[VCAP_KF_8021CB_R_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 16,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 17,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 18,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_TPID0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 19,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 20,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 32,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 33,
+		.width = 3,
+	},
+	[VCAP_KF_8021Q_TPID1] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 36,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID1] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 37,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI1] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 49,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP1] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 50,
+		.width = 3,
+	},
+	[VCAP_KF_L2_SMAC] = {
+		.type = VCAP_FIELD_U48,
+		.offset = 53,
+		.width = 48,
+	},
+	[VCAP_KF_L3_DSCP] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 101,
+		.width = 6,
+	},
+	[VCAP_KF_L3_IP6_SIP] = {
+		.type = VCAP_FIELD_U128,
+		.offset = 107,
+		.width = 128,
+	},
+	[VCAP_KF_L3_IP_PROTO] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 235,
+		.width = 8,
+	},
+	[VCAP_KF_TCP_UDP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 243,
+		.width = 1,
+	},
+	[VCAP_KF_L4_RNG] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 244,
+		.width = 8,
+	},
+	[VCAP_KF_IP_PAYLOAD_S1_IP6] = {
+		.type = VCAP_FIELD_U112,
+		.offset = 252,
+		.width = 112,
+	},
+};
+
+static const struct vcap_field is1_7tuple_keyfield[] = {
+	[VCAP_KF_TYPE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 0,
+		.width = 2,
+	},
+	[VCAP_KF_LOOKUP_INDEX] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 2,
+		.width = 2,
+	},
+	[VCAP_KF_IF_IGR_PORT_MASK] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 4,
+		.width = 9,
+	},
+	[VCAP_KF_L2_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 13,
+		.width = 1,
+	},
+	[VCAP_KF_L2_BC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 14,
+		.width = 1,
+	},
+	[VCAP_KF_IP_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 15,
+		.width = 1,
+	},
+	[VCAP_KF_8021CB_R_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 16,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 17,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 18,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_TPID0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 19,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 20,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 32,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 33,
+		.width = 3,
+	},
+	[VCAP_KF_8021Q_TPID1] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 36,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID1] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 37,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI1] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 49,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP1] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 50,
+		.width = 3,
+	},
+	[VCAP_KF_L2_DMAC] = {
+		.type = VCAP_FIELD_U48,
+		.offset = 53,
+		.width = 48,
+	},
+	[VCAP_KF_L2_SMAC] = {
+		.type = VCAP_FIELD_U48,
+		.offset = 101,
+		.width = 48,
+	},
+	[VCAP_KF_ETYPE_LEN_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 149,
+		.width = 1,
+	},
+	[VCAP_KF_ETYPE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 150,
+		.width = 16,
+	},
+	[VCAP_KF_IP_SNAP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 166,
+		.width = 1,
+	},
+	[VCAP_KF_IP4_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 167,
+		.width = 1,
+	},
+	[VCAP_KF_L3_FRAGMENT] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 168,
+		.width = 1,
+	},
+	[VCAP_KF_L3_FRAG_OFS_GT0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 169,
+		.width = 1,
+	},
+	[VCAP_KF_L3_OPTIONS_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 170,
+		.width = 1,
+	},
+	[VCAP_KF_L3_DSCP] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 171,
+		.width = 6,
+	},
+	[VCAP_KF_L3_IP6_DIP_MSB] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 177,
+		.width = 16,
+	},
+	[VCAP_KF_L3_IP6_DIP] = {
+		.type = VCAP_FIELD_U64,
+		.offset = 193,
+		.width = 64,
+	},
+	[VCAP_KF_L3_IP6_SIP_MSB] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 257,
+		.width = 16,
+	},
+	[VCAP_KF_L3_IP6_SIP] = {
+		.type = VCAP_FIELD_U64,
+		.offset = 273,
+		.width = 64,
+	},
+	[VCAP_KF_TCP_UDP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 337,
+		.width = 1,
+	},
+	[VCAP_KF_TCP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 338,
+		.width = 1,
+	},
+	[VCAP_KF_L4_SPORT] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 339,
+		.width = 16,
+	},
+	[VCAP_KF_L4_RNG] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 355,
+		.width = 8,
+	},
+};
+
+static const struct vcap_field is1_5tuple_ip6_keyfield[] = {
+	[VCAP_KF_TYPE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 0,
+		.width = 2,
+	},
+	[VCAP_KF_LOOKUP_INDEX] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 2,
+		.width = 2,
+	},
+	[VCAP_KF_IF_IGR_PORT_MASK] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 4,
+		.width = 9,
+	},
+	[VCAP_KF_L2_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 13,
+		.width = 1,
+	},
+	[VCAP_KF_L2_BC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 14,
+		.width = 1,
+	},
+	[VCAP_KF_IP_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 15,
+		.width = 1,
+	},
+	[VCAP_KF_8021CB_R_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 16,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 17,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 18,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_TPID0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 19,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 20,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 32,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 33,
+		.width = 3,
+	},
+	[VCAP_KF_8021Q_TPID1] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 36,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID1] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 37,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI1] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 49,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP1] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 50,
+		.width = 3,
+	},
+	[VCAP_KF_L3_DSCP] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 53,
+		.width = 6,
+	},
+	[VCAP_KF_L3_IP6_DIP] = {
+		.type = VCAP_FIELD_U128,
+		.offset = 59,
+		.width = 128,
+	},
+	[VCAP_KF_L3_IP6_SIP] = {
+		.type = VCAP_FIELD_U128,
+		.offset = 187,
+		.width = 128,
+	},
+	[VCAP_KF_L3_IP_PROTO] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 315,
+		.width = 8,
+	},
+	[VCAP_KF_TCP_UDP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 323,
+		.width = 1,
+	},
+	[VCAP_KF_L4_RNG] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 324,
+		.width = 8,
+	},
+	[VCAP_KF_IP_PAYLOAD_5TUPLE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 332,
+		.width = 32,
+	},
+};
+
+static const struct vcap_field is1_dbl_vid_keyfield[] = {
+	[VCAP_KF_TYPE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 0,
+		.width = 2,
+	},
+	[VCAP_KF_LOOKUP_INDEX] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 2,
+		.width = 2,
+	},
+	[VCAP_KF_IF_IGR_PORT_MASK] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 4,
+		.width = 9,
+	},
+	[VCAP_KF_L2_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 13,
+		.width = 1,
+	},
+	[VCAP_KF_L2_BC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 14,
+		.width = 1,
+	},
+	[VCAP_KF_IP_MC_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 15,
+		.width = 1,
+	},
+	[VCAP_KF_8021CB_R_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 16,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 17,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 18,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_TPID0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 19,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 20,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 32,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 33,
+		.width = 3,
+	},
+	[VCAP_KF_8021Q_TPID1] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 36,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID1] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 37,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI1] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 49,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP1] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 50,
+		.width = 3,
+	},
+	[VCAP_KF_ETYPE_LEN_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 53,
+		.width = 1,
+	},
+	[VCAP_KF_ETYPE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 54,
+		.width = 16,
+	},
+	[VCAP_KF_IP_SNAP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 70,
+		.width = 1,
+	},
+	[VCAP_KF_IP4_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 71,
+		.width = 1,
+	},
+	[VCAP_KF_L3_FRAGMENT] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 72,
+		.width = 1,
+	},
+	[VCAP_KF_L3_FRAG_OFS_GT0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 73,
+		.width = 1,
+	},
+	[VCAP_KF_L3_OPTIONS_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 74,
+		.width = 1,
+	},
+	[VCAP_KF_L3_DSCP] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 75,
+		.width = 6,
+	},
+	[VCAP_KF_TCP_UDP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 81,
+		.width = 1,
+	},
+	[VCAP_KF_TCP_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 82,
+		.width = 1,
+	},
+};
+
+static const struct vcap_field is1_rt_keyfield[] = {
+	[VCAP_KF_TYPE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 0,
+		.width = 2,
+	},
+	[VCAP_KF_LOOKUP_FIRST_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 2,
+		.width = 1,
+	},
+	[VCAP_KF_IF_IGR_PORT] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 3,
+		.width = 3,
+	},
+	[VCAP_KF_8021CB_R_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 6,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 7,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 8,
+		.width = 1,
+	},
+	[VCAP_KF_L2_MAC] = {
+		.type = VCAP_FIELD_U48,
+		.offset = 9,
+		.width = 48,
+	},
+	[VCAP_KF_RT_VLAN_IDX] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 57,
+		.width = 3,
+	},
+	[VCAP_KF_RT_TYPE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 60,
+		.width = 2,
+	},
+	[VCAP_KF_RT_FRMID] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 62,
+		.width = 32,
+	},
+};
+
+static const struct vcap_field is1_dmac_vid_keyfield[] = {
+	[VCAP_KF_TYPE] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 0,
+		.width = 2,
+	},
+	[VCAP_KF_LOOKUP_INDEX] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 2,
+		.width = 2,
+	},
+	[VCAP_KF_IF_IGR_PORT_MASK] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 4,
+		.width = 9,
+	},
+	[VCAP_KF_8021CB_R_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 13,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 14,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 15,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_TPID0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 16,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_VID0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 17,
+		.width = 12,
+	},
+	[VCAP_KF_8021Q_DEI0] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 29,
+		.width = 1,
+	},
+	[VCAP_KF_8021Q_PCP0] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 30,
+		.width = 3,
+	},
+	[VCAP_KF_L2_DMAC] = {
+		.type = VCAP_FIELD_U48,
+		.offset = 33,
+		.width = 48,
+	},
+};
+
 static const struct vcap_field is2_mac_etype_keyfield[] = {
 	[VCAP_KF_TYPE] = {
 		.type = VCAP_FIELD_U32,
@@ -1163,6 +2122,49 @@ static const struct vcap_field is2_smac_sip6_keyfield[] = {
 };
 
 /* keyfield_set */
+static const struct vcap_set is1_keyfield_set[] = {
+	[VCAP_KFS_NORMAL] = {
+		.type_id = 0,
+		.sw_per_item = 2,
+		.sw_cnt = 2,
+	},
+	[VCAP_KFS_5TUPLE_IP4] = {
+		.type_id = 1,
+		.sw_per_item = 2,
+		.sw_cnt = 2,
+	},
+	[VCAP_KFS_NORMAL_IP6] = {
+		.type_id = 0,
+		.sw_per_item = 4,
+		.sw_cnt = 1,
+	},
+	[VCAP_KFS_7TUPLE] = {
+		.type_id = 1,
+		.sw_per_item = 4,
+		.sw_cnt = 1,
+	},
+	[VCAP_KFS_5TUPLE_IP6] = {
+		.type_id = 2,
+		.sw_per_item = 4,
+		.sw_cnt = 1,
+	},
+	[VCAP_KFS_DBL_VID] = {
+		.type_id = 0,
+		.sw_per_item = 1,
+		.sw_cnt = 4,
+	},
+	[VCAP_KFS_RT] = {
+		.type_id = 1,
+		.sw_per_item = 1,
+		.sw_cnt = 4,
+	},
+	[VCAP_KFS_DMAC_VID] = {
+		.type_id = 2,
+		.sw_per_item = 1,
+		.sw_cnt = 4,
+	},
+};
+
 static const struct vcap_set is2_keyfield_set[] = {
 	[VCAP_KFS_MAC_ETYPE] = {
 		.type_id = 0,
@@ -1227,6 +2229,17 @@ static const struct vcap_set is2_keyfield_set[] = {
 };
 
 /* keyfield_set map */
+static const struct vcap_field *is1_keyfield_set_map[] = {
+	[VCAP_KFS_NORMAL] = is1_normal_keyfield,
+	[VCAP_KFS_5TUPLE_IP4] = is1_5tuple_ip4_keyfield,
+	[VCAP_KFS_NORMAL_IP6] = is1_normal_ip6_keyfield,
+	[VCAP_KFS_7TUPLE] = is1_7tuple_keyfield,
+	[VCAP_KFS_5TUPLE_IP6] = is1_5tuple_ip6_keyfield,
+	[VCAP_KFS_DBL_VID] = is1_dbl_vid_keyfield,
+	[VCAP_KFS_RT] = is1_rt_keyfield,
+	[VCAP_KFS_DMAC_VID] = is1_dmac_vid_keyfield,
+};
+
 static const struct vcap_field *is2_keyfield_set_map[] = {
 	[VCAP_KFS_MAC_ETYPE] = is2_mac_etype_keyfield,
 	[VCAP_KFS_MAC_LLC] = is2_mac_llc_keyfield,
@@ -1243,6 +2256,17 @@ static const struct vcap_field *is2_keyfield_set_map[] = {
 };
 
 /* keyfield_set map sizes */
+static int is1_keyfield_set_map_size[] = {
+	[VCAP_KFS_NORMAL] = ARRAY_SIZE(is1_normal_keyfield),
+	[VCAP_KFS_5TUPLE_IP4] = ARRAY_SIZE(is1_5tuple_ip4_keyfield),
+	[VCAP_KFS_NORMAL_IP6] = ARRAY_SIZE(is1_normal_ip6_keyfield),
+	[VCAP_KFS_7TUPLE] = ARRAY_SIZE(is1_7tuple_keyfield),
+	[VCAP_KFS_5TUPLE_IP6] = ARRAY_SIZE(is1_5tuple_ip6_keyfield),
+	[VCAP_KFS_DBL_VID] = ARRAY_SIZE(is1_dbl_vid_keyfield),
+	[VCAP_KFS_RT] = ARRAY_SIZE(is1_rt_keyfield),
+	[VCAP_KFS_DMAC_VID] = ARRAY_SIZE(is1_dmac_vid_keyfield),
+};
+
 static int is2_keyfield_set_map_size[] = {
 	[VCAP_KFS_MAC_ETYPE] = ARRAY_SIZE(is2_mac_etype_keyfield),
 	[VCAP_KFS_MAC_LLC] = ARRAY_SIZE(is2_mac_llc_keyfield),
@@ -1259,6 +2283,154 @@ static int is2_keyfield_set_map_size[] = {
 };
 
 /* actionfields */
+static const struct vcap_field is1_s1_actionfield[] = {
+	[VCAP_AF_TYPE] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 0,
+		.width = 1,
+	},
+	[VCAP_AF_DSCP_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 1,
+		.width = 1,
+	},
+	[VCAP_AF_DSCP_VAL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 2,
+		.width = 6,
+	},
+	[VCAP_AF_QOS_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 8,
+		.width = 1,
+	},
+	[VCAP_AF_QOS_VAL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 9,
+		.width = 3,
+	},
+	[VCAP_AF_DP_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 12,
+		.width = 1,
+	},
+	[VCAP_AF_DP_VAL] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 13,
+		.width = 1,
+	},
+	[VCAP_AF_PAG_OVERRIDE_MASK] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 14,
+		.width = 8,
+	},
+	[VCAP_AF_PAG_VAL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 22,
+		.width = 8,
+	},
+	[VCAP_AF_ISDX_REPLACE_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 30,
+		.width = 1,
+	},
+	[VCAP_AF_ISDX_ADD_VAL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 31,
+		.width = 8,
+	},
+	[VCAP_AF_VID_REPLACE_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 39,
+		.width = 1,
+	},
+	[VCAP_AF_VID_VAL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 40,
+		.width = 12,
+	},
+	[VCAP_AF_PCP_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 67,
+		.width = 1,
+	},
+	[VCAP_AF_PCP_VAL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 68,
+		.width = 3,
+	},
+	[VCAP_AF_DEI_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 71,
+		.width = 1,
+	},
+	[VCAP_AF_DEI_VAL] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 72,
+		.width = 1,
+	},
+	[VCAP_AF_VLAN_POP_CNT_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 73,
+		.width = 1,
+	},
+	[VCAP_AF_VLAN_POP_CNT] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 74,
+		.width = 2,
+	},
+	[VCAP_AF_CUSTOM_ACE_TYPE_ENA] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 76,
+		.width = 4,
+	},
+	[VCAP_AF_SFID_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 80,
+		.width = 1,
+	},
+	[VCAP_AF_SFID_VAL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 81,
+		.width = 8,
+	},
+	[VCAP_AF_SGID_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 89,
+		.width = 1,
+	},
+	[VCAP_AF_SGID_VAL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 90,
+		.width = 8,
+	},
+	[VCAP_AF_POLICE_ENA] = {
+		.type = VCAP_FIELD_BIT,
+		.offset = 98,
+		.width = 1,
+	},
+	[VCAP_AF_POLICE_IDX] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 99,
+		.width = 9,
+	},
+	[VCAP_AF_OAM_SEL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 108,
+		.width = 3,
+	},
+	[VCAP_AF_MRP_SEL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 111,
+		.width = 2,
+	},
+	[VCAP_AF_DLR_SEL] = {
+		.type = VCAP_FIELD_U32,
+		.offset = 113,
+		.width = 2,
+	},
+};
+
 static const struct vcap_field is2_base_type_actionfield[] = {
 	[VCAP_AF_HIT_ME_ONCE] = {
 		.type = VCAP_FIELD_BIT,
@@ -1351,6 +2523,14 @@ static const struct vcap_field is2_smac_sip_actionfield[] = {
 };
 
 /* actionfield_set */
+static const struct vcap_set is1_actionfield_set[] = {
+	[VCAP_AFS_S1] = {
+		.type_id = 0,
+		.sw_per_item = 1,
+		.sw_cnt = 4,
+	},
+};
+
 static const struct vcap_set is2_actionfield_set[] = {
 	[VCAP_AFS_BASE_TYPE] = {
 		.type_id = -1,
@@ -1365,18 +2545,73 @@ static const struct vcap_set is2_actionfield_set[] = {
 };
 
 /* actionfield_set map */
+static const struct vcap_field *is1_actionfield_set_map[] = {
+	[VCAP_AFS_S1] = is1_s1_actionfield,
+};
+
 static const struct vcap_field *is2_actionfield_set_map[] = {
 	[VCAP_AFS_BASE_TYPE] = is2_base_type_actionfield,
 	[VCAP_AFS_SMAC_SIP] = is2_smac_sip_actionfield,
 };
 
 /* actionfield_set map size */
+static int is1_actionfield_set_map_size[] = {
+	[VCAP_AFS_S1] = ARRAY_SIZE(is1_s1_actionfield),
+};
+
 static int is2_actionfield_set_map_size[] = {
 	[VCAP_AFS_BASE_TYPE] = ARRAY_SIZE(is2_base_type_actionfield),
 	[VCAP_AFS_SMAC_SIP] = ARRAY_SIZE(is2_smac_sip_actionfield),
 };
 
 /* Type Groups */
+static const struct vcap_typegroup is1_x4_keyfield_set_typegroups[] = {
+	{
+		.offset = 0,
+		.width = 3,
+		.value = 4,
+	},
+	{
+		.offset = 96,
+		.width = 1,
+		.value = 0,
+	},
+	{
+		.offset = 192,
+		.width = 2,
+		.value = 0,
+	},
+	{
+		.offset = 288,
+		.width = 1,
+		.value = 0,
+	},
+	{}
+};
+
+static const struct vcap_typegroup is1_x2_keyfield_set_typegroups[] = {
+	{
+		.offset = 0,
+		.width = 2,
+		.value = 2,
+	},
+	{
+		.offset = 96,
+		.width = 1,
+		.value = 0,
+	},
+	{}
+};
+
+static const struct vcap_typegroup is1_x1_keyfield_set_typegroups[] = {
+	{
+		.offset = 0,
+		.width = 1,
+		.value = 1,
+	},
+	{}
+};
+
 static const struct vcap_typegroup is2_x4_keyfield_set_typegroups[] = {
 	{
 		.offset = 0,
@@ -1424,6 +2659,13 @@ static const struct vcap_typegroup is2_x1_keyfield_set_typegroups[] = {
 	{}
 };
 
+static const struct vcap_typegroup *is1_keyfield_set_typegroups[] = {
+	[4] = is1_x4_keyfield_set_typegroups,
+	[2] = is1_x2_keyfield_set_typegroups,
+	[1] = is1_x1_keyfield_set_typegroups,
+	[5] = NULL,
+};
+
 static const struct vcap_typegroup *is2_keyfield_set_typegroups[] = {
 	[4] = is2_x4_keyfield_set_typegroups,
 	[2] = is2_x2_keyfield_set_typegroups,
@@ -1431,6 +2673,10 @@ static const struct vcap_typegroup *is2_keyfield_set_typegroups[] = {
 	[5] = NULL,
 };
 
+static const struct vcap_typegroup is1_x1_actionfield_set_typegroups[] = {
+	{}
+};
+
 static const struct vcap_typegroup is2_x2_actionfield_set_typegroups[] = {
 	{
 		.offset = 0,
@@ -1454,6 +2700,11 @@ static const struct vcap_typegroup is2_x1_actionfield_set_typegroups[] = {
 	{}
 };
 
+static const struct vcap_typegroup *is1_actionfield_set_typegroups[] = {
+	[1] = is1_x1_actionfield_set_typegroups,
+	[5] = NULL,
+};
+
 static const struct vcap_typegroup *is2_actionfield_set_typegroups[] = {
 	[2] = is2_x2_actionfield_set_typegroups,
 	[1] = is2_x1_actionfield_set_typegroups,
@@ -1463,16 +2714,33 @@ static const struct vcap_typegroup *is2_actionfield_set_typegroups[] = {
 /* Keyfieldset names */
 static const char * const vcap_keyfield_set_names[] = {
 	[VCAP_KFS_NO_VALUE]                      =  "(None)",
+	[VCAP_KFS_5TUPLE_IP4]                    =  "VCAP_KFS_5TUPLE_IP4",
+	[VCAP_KFS_5TUPLE_IP6]                    =  "VCAP_KFS_5TUPLE_IP6",
+	[VCAP_KFS_7TUPLE]                        =  "VCAP_KFS_7TUPLE",
 	[VCAP_KFS_ARP]                           =  "VCAP_KFS_ARP",
+	[VCAP_KFS_DBL_VID]                       =  "VCAP_KFS_DBL_VID",
+	[VCAP_KFS_DMAC_VID]                      =  "VCAP_KFS_DMAC_VID",
+	[VCAP_KFS_ETAG]                          =  "VCAP_KFS_ETAG",
 	[VCAP_KFS_IP4_OTHER]                     =  "VCAP_KFS_IP4_OTHER",
 	[VCAP_KFS_IP4_TCP_UDP]                   =  "VCAP_KFS_IP4_TCP_UDP",
+	[VCAP_KFS_IP4_VID]                       =  "VCAP_KFS_IP4_VID",
 	[VCAP_KFS_IP6_OTHER]                     =  "VCAP_KFS_IP6_OTHER",
 	[VCAP_KFS_IP6_STD]                       =  "VCAP_KFS_IP6_STD",
 	[VCAP_KFS_IP6_TCP_UDP]                   =  "VCAP_KFS_IP6_TCP_UDP",
+	[VCAP_KFS_IP6_VID]                       =  "VCAP_KFS_IP6_VID",
+	[VCAP_KFS_IP_7TUPLE]                     =  "VCAP_KFS_IP_7TUPLE",
+	[VCAP_KFS_ISDX]                          =  "VCAP_KFS_ISDX",
+	[VCAP_KFS_LL_FULL]                       =  "VCAP_KFS_LL_FULL",
 	[VCAP_KFS_MAC_ETYPE]                     =  "VCAP_KFS_MAC_ETYPE",
 	[VCAP_KFS_MAC_LLC]                       =  "VCAP_KFS_MAC_LLC",
 	[VCAP_KFS_MAC_SNAP]                      =  "VCAP_KFS_MAC_SNAP",
+	[VCAP_KFS_NORMAL]                        =  "VCAP_KFS_NORMAL",
+	[VCAP_KFS_NORMAL_5TUPLE_IP4]             =  "VCAP_KFS_NORMAL_5TUPLE_IP4",
+	[VCAP_KFS_NORMAL_7TUPLE]                 =  "VCAP_KFS_NORMAL_7TUPLE",
+	[VCAP_KFS_NORMAL_IP6]                    =  "VCAP_KFS_NORMAL_IP6",
 	[VCAP_KFS_OAM]                           =  "VCAP_KFS_OAM",
+	[VCAP_KFS_PURE_5TUPLE_IP4]               =  "VCAP_KFS_PURE_5TUPLE_IP4",
+	[VCAP_KFS_RT]                            =  "VCAP_KFS_RT",
 	[VCAP_KFS_SMAC_SIP4]                     =  "VCAP_KFS_SMAC_SIP4",
 	[VCAP_KFS_SMAC_SIP6]                     =  "VCAP_KFS_SMAC_SIP6",
 };
@@ -1481,16 +2749,42 @@ static const char * const vcap_keyfield_set_names[] = {
 static const char * const vcap_actionfield_set_names[] = {
 	[VCAP_AFS_NO_VALUE]                      =  "(None)",
 	[VCAP_AFS_BASE_TYPE]                     =  "VCAP_AFS_BASE_TYPE",
+	[VCAP_AFS_CLASSIFICATION]                =  "VCAP_AFS_CLASSIFICATION",
+	[VCAP_AFS_CLASS_REDUCED]                 =  "VCAP_AFS_CLASS_REDUCED",
+	[VCAP_AFS_FULL]                          =  "VCAP_AFS_FULL",
+	[VCAP_AFS_S1]                            =  "VCAP_AFS_S1",
 	[VCAP_AFS_SMAC_SIP]                      =  "VCAP_AFS_SMAC_SIP",
 };
 
 /* Keyfield names */
 static const char * const vcap_keyfield_names[] = {
 	[VCAP_KF_NO_VALUE]                       =  "(None)",
+	[VCAP_KF_8021BR_ECID_BASE]               =  "8021BR_ECID_BASE",
+	[VCAP_KF_8021BR_ECID_EXT]                =  "8021BR_ECID_EXT",
+	[VCAP_KF_8021BR_E_TAGGED]                =  "8021BR_E_TAGGED",
+	[VCAP_KF_8021BR_GRP]                     =  "8021BR_GRP",
+	[VCAP_KF_8021BR_IGR_ECID_BASE]           =  "8021BR_IGR_ECID_BASE",
+	[VCAP_KF_8021BR_IGR_ECID_EXT]            =  "8021BR_IGR_ECID_EXT",
+	[VCAP_KF_8021CB_R_TAGGED_IS]             =  "8021CB_R_TAGGED_IS",
+	[VCAP_KF_8021Q_DEI0]                     =  "8021Q_DEI0",
+	[VCAP_KF_8021Q_DEI1]                     =  "8021Q_DEI1",
+	[VCAP_KF_8021Q_DEI2]                     =  "8021Q_DEI2",
 	[VCAP_KF_8021Q_DEI_CLS]                  =  "8021Q_DEI_CLS",
+	[VCAP_KF_8021Q_PCP0]                     =  "8021Q_PCP0",
+	[VCAP_KF_8021Q_PCP1]                     =  "8021Q_PCP1",
+	[VCAP_KF_8021Q_PCP2]                     =  "8021Q_PCP2",
 	[VCAP_KF_8021Q_PCP_CLS]                  =  "8021Q_PCP_CLS",
+	[VCAP_KF_8021Q_TPID0]                    =  "8021Q_TPID0",
+	[VCAP_KF_8021Q_TPID1]                    =  "8021Q_TPID1",
+	[VCAP_KF_8021Q_TPID2]                    =  "8021Q_TPID2",
+	[VCAP_KF_8021Q_VID0]                     =  "8021Q_VID0",
+	[VCAP_KF_8021Q_VID1]                     =  "8021Q_VID1",
+	[VCAP_KF_8021Q_VID2]                     =  "8021Q_VID2",
 	[VCAP_KF_8021Q_VID_CLS]                  =  "8021Q_VID_CLS",
+	[VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS]       =  "8021Q_VLAN_DBL_TAGGED_IS",
 	[VCAP_KF_8021Q_VLAN_TAGGED_IS]           =  "8021Q_VLAN_TAGGED_IS",
+	[VCAP_KF_8021Q_VLAN_TAGS]                =  "8021Q_VLAN_TAGS",
+	[VCAP_KF_ACL_GRP_ID]                     =  "ACL_GRP_ID",
 	[VCAP_KF_ARP_ADDR_SPACE_OK_IS]           =  "ARP_ADDR_SPACE_OK_IS",
 	[VCAP_KF_ARP_LEN_OK_IS]                  =  "ARP_LEN_OK_IS",
 	[VCAP_KF_ARP_OPCODE]                     =  "ARP_OPCODE",
@@ -1498,32 +2792,57 @@ static const char * const vcap_keyfield_names[] = {
 	[VCAP_KF_ARP_PROTO_SPACE_OK_IS]          =  "ARP_PROTO_SPACE_OK_IS",
 	[VCAP_KF_ARP_SENDER_MATCH_IS]            =  "ARP_SENDER_MATCH_IS",
 	[VCAP_KF_ARP_TGT_MATCH_IS]               =  "ARP_TGT_MATCH_IS",
+	[VCAP_KF_COSID_CLS]                      =  "COSID_CLS",
+	[VCAP_KF_ES0_ISDX_KEY_ENA]               =  "ES0_ISDX_KEY_ENA",
 	[VCAP_KF_ETYPE]                          =  "ETYPE",
+	[VCAP_KF_ETYPE_LEN_IS]                   =  "ETYPE_LEN_IS",
 	[VCAP_KF_HOST_MATCH]                     =  "HOST_MATCH",
+	[VCAP_KF_IF_EGR_PORT_MASK]               =  "IF_EGR_PORT_MASK",
+	[VCAP_KF_IF_EGR_PORT_MASK_RNG]           =  "IF_EGR_PORT_MASK_RNG",
 	[VCAP_KF_IF_IGR_PORT]                    =  "IF_IGR_PORT",
 	[VCAP_KF_IF_IGR_PORT_MASK]               =  "IF_IGR_PORT_MASK",
+	[VCAP_KF_IF_IGR_PORT_MASK_L3]            =  "IF_IGR_PORT_MASK_L3",
+	[VCAP_KF_IF_IGR_PORT_MASK_RNG]           =  "IF_IGR_PORT_MASK_RNG",
+	[VCAP_KF_IF_IGR_PORT_MASK_SEL]           =  "IF_IGR_PORT_MASK_SEL",
+	[VCAP_KF_IF_IGR_PORT_SEL]                =  "IF_IGR_PORT_SEL",
 	[VCAP_KF_IP4_IS]                         =  "IP4_IS",
+	[VCAP_KF_IP_MC_IS]                       =  "IP_MC_IS",
+	[VCAP_KF_IP_PAYLOAD_5TUPLE]              =  "IP_PAYLOAD_5TUPLE",
+	[VCAP_KF_IP_PAYLOAD_S1_IP6]              =  "IP_PAYLOAD_S1_IP6",
+	[VCAP_KF_IP_SNAP_IS]                     =  "IP_SNAP_IS",
+	[VCAP_KF_ISDX_CLS]                       =  "ISDX_CLS",
 	[VCAP_KF_ISDX_GT0_IS]                    =  "ISDX_GT0_IS",
 	[VCAP_KF_L2_BC_IS]                       =  "L2_BC_IS",
 	[VCAP_KF_L2_DMAC]                        =  "L2_DMAC",
 	[VCAP_KF_L2_FRM_TYPE]                    =  "L2_FRM_TYPE",
+	[VCAP_KF_L2_FWD_IS]                      =  "L2_FWD_IS",
 	[VCAP_KF_L2_LLC]                         =  "L2_LLC",
+	[VCAP_KF_L2_MAC]                         =  "L2_MAC",
 	[VCAP_KF_L2_MC_IS]                       =  "L2_MC_IS",
 	[VCAP_KF_L2_PAYLOAD0]                    =  "L2_PAYLOAD0",
 	[VCAP_KF_L2_PAYLOAD1]                    =  "L2_PAYLOAD1",
 	[VCAP_KF_L2_PAYLOAD2]                    =  "L2_PAYLOAD2",
+	[VCAP_KF_L2_PAYLOAD_ETYPE]               =  "L2_PAYLOAD_ETYPE",
 	[VCAP_KF_L2_SMAC]                        =  "L2_SMAC",
 	[VCAP_KF_L2_SNAP]                        =  "L2_SNAP",
 	[VCAP_KF_L3_DIP_EQ_SIP_IS]               =  "L3_DIP_EQ_SIP_IS",
+	[VCAP_KF_L3_DPL_CLS]                     =  "L3_DPL_CLS",
+	[VCAP_KF_L3_DSCP]                        =  "L3_DSCP",
+	[VCAP_KF_L3_DST_IS]                      =  "L3_DST_IS",
 	[VCAP_KF_L3_FRAGMENT]                    =  "L3_FRAGMENT",
+	[VCAP_KF_L3_FRAGMENT_TYPE]               =  "L3_FRAGMENT_TYPE",
+	[VCAP_KF_L3_FRAG_INVLD_L4_LEN]           =  "L3_FRAG_INVLD_L4_LEN",
 	[VCAP_KF_L3_FRAG_OFS_GT0]                =  "L3_FRAG_OFS_GT0",
 	[VCAP_KF_L3_IP4_DIP]                     =  "L3_IP4_DIP",
 	[VCAP_KF_L3_IP4_SIP]                     =  "L3_IP4_SIP",
 	[VCAP_KF_L3_IP6_DIP]                     =  "L3_IP6_DIP",
+	[VCAP_KF_L3_IP6_DIP_MSB]                 =  "L3_IP6_DIP_MSB",
 	[VCAP_KF_L3_IP6_SIP]                     =  "L3_IP6_SIP",
+	[VCAP_KF_L3_IP6_SIP_MSB]                 =  "L3_IP6_SIP_MSB",
 	[VCAP_KF_L3_IP_PROTO]                    =  "L3_IP_PROTO",
 	[VCAP_KF_L3_OPTIONS_IS]                  =  "L3_OPTIONS_IS",
 	[VCAP_KF_L3_PAYLOAD]                     =  "L3_PAYLOAD",
+	[VCAP_KF_L3_RT_IS]                       =  "L3_RT_IS",
 	[VCAP_KF_L3_TOS]                         =  "L3_TOS",
 	[VCAP_KF_L3_TTL_GT0]                     =  "L3_TTL_GT0",
 	[VCAP_KF_L4_1588_DOM]                    =  "L4_1588_DOM",
@@ -1531,6 +2850,7 @@ static const char * const vcap_keyfield_names[] = {
 	[VCAP_KF_L4_ACK]                         =  "L4_ACK",
 	[VCAP_KF_L4_DPORT]                       =  "L4_DPORT",
 	[VCAP_KF_L4_FIN]                         =  "L4_FIN",
+	[VCAP_KF_L4_PAYLOAD]                     =  "L4_PAYLOAD",
 	[VCAP_KF_L4_PSH]                         =  "L4_PSH",
 	[VCAP_KF_L4_RNG]                         =  "L4_RNG",
 	[VCAP_KF_L4_RST]                         =  "L4_RST",
@@ -1540,7 +2860,11 @@ static const char * const vcap_keyfield_names[] = {
 	[VCAP_KF_L4_SYN]                         =  "L4_SYN",
 	[VCAP_KF_L4_URG]                         =  "L4_URG",
 	[VCAP_KF_LOOKUP_FIRST_IS]                =  "LOOKUP_FIRST_IS",
+	[VCAP_KF_LOOKUP_GEN_IDX]                 =  "LOOKUP_GEN_IDX",
+	[VCAP_KF_LOOKUP_GEN_IDX_SEL]             =  "LOOKUP_GEN_IDX_SEL",
+	[VCAP_KF_LOOKUP_INDEX]                   =  "LOOKUP_INDEX",
 	[VCAP_KF_LOOKUP_PAG]                     =  "LOOKUP_PAG",
+	[VCAP_KF_MIRROR_PROBE]                   =  "MIRROR_PROBE",
 	[VCAP_KF_OAM_CCM_CNTS_EQ0]               =  "OAM_CCM_CNTS_EQ0",
 	[VCAP_KF_OAM_DETECTED]                   =  "OAM_DETECTED",
 	[VCAP_KF_OAM_FLAGS]                      =  "OAM_FLAGS",
@@ -1549,7 +2873,12 @@ static const char * const vcap_keyfield_names[] = {
 	[VCAP_KF_OAM_OPCODE]                     =  "OAM_OPCODE",
 	[VCAP_KF_OAM_VER]                        =  "OAM_VER",
 	[VCAP_KF_OAM_Y1731_IS]                   =  "OAM_Y1731_IS",
+	[VCAP_KF_PROT_ACTIVE]                    =  "PROT_ACTIVE",
+	[VCAP_KF_RT_FRMID]                       =  "RT_FRMID",
+	[VCAP_KF_RT_TYPE]                        =  "RT_TYPE",
+	[VCAP_KF_RT_VLAN_IDX]                    =  "RT_VLAN_IDX",
 	[VCAP_KF_TCP_IS]                         =  "TCP_IS",
+	[VCAP_KF_TCP_UDP_IS]                     =  "TCP_UDP_IS",
 	[VCAP_KF_TYPE]                           =  "TYPE",
 };
 
@@ -1557,24 +2886,95 @@ static const char * const vcap_keyfield_names[] = {
 static const char * const vcap_actionfield_names[] = {
 	[VCAP_AF_NO_VALUE]                       =  "(None)",
 	[VCAP_AF_ACL_ID]                         =  "ACL_ID",
+	[VCAP_AF_CLS_VID_SEL]                    =  "CLS_VID_SEL",
+	[VCAP_AF_CNT_ID]                         =  "CNT_ID",
+	[VCAP_AF_COPY_PORT_NUM]                  =  "COPY_PORT_NUM",
+	[VCAP_AF_COPY_QUEUE_NUM]                 =  "COPY_QUEUE_NUM",
 	[VCAP_AF_CPU_COPY_ENA]                   =  "CPU_COPY_ENA",
 	[VCAP_AF_CPU_QUEUE_NUM]                  =  "CPU_QUEUE_NUM",
+	[VCAP_AF_CUSTOM_ACE_TYPE_ENA]            =  "CUSTOM_ACE_TYPE_ENA",
+	[VCAP_AF_DEI_ENA]                        =  "DEI_ENA",
+	[VCAP_AF_DEI_VAL]                        =  "DEI_VAL",
+	[VCAP_AF_DLR_SEL]                        =  "DLR_SEL",
+	[VCAP_AF_DP_ENA]                         =  "DP_ENA",
+	[VCAP_AF_DP_VAL]                         =  "DP_VAL",
+	[VCAP_AF_DSCP_ENA]                       =  "DSCP_ENA",
+	[VCAP_AF_DSCP_VAL]                       =  "DSCP_VAL",
+	[VCAP_AF_ES2_REW_CMD]                    =  "ES2_REW_CMD",
 	[VCAP_AF_FWD_KILL_ENA]                   =  "FWD_KILL_ENA",
+	[VCAP_AF_FWD_MODE]                       =  "FWD_MODE",
 	[VCAP_AF_HIT_ME_ONCE]                    =  "HIT_ME_ONCE",
 	[VCAP_AF_HOST_MATCH]                     =  "HOST_MATCH",
+	[VCAP_AF_IGNORE_PIPELINE_CTRL]           =  "IGNORE_PIPELINE_CTRL",
+	[VCAP_AF_INTR_ENA]                       =  "INTR_ENA",
+	[VCAP_AF_ISDX_ADD_REPLACE_SEL]           =  "ISDX_ADD_REPLACE_SEL",
+	[VCAP_AF_ISDX_ADD_VAL]                   =  "ISDX_ADD_VAL",
 	[VCAP_AF_ISDX_ENA]                       =  "ISDX_ENA",
+	[VCAP_AF_ISDX_REPLACE_ENA]               =  "ISDX_REPLACE_ENA",
+	[VCAP_AF_ISDX_VAL]                       =  "ISDX_VAL",
 	[VCAP_AF_LRN_DIS]                        =  "LRN_DIS",
+	[VCAP_AF_MAP_IDX]                        =  "MAP_IDX",
+	[VCAP_AF_MAP_KEY]                        =  "MAP_KEY",
+	[VCAP_AF_MAP_LOOKUP_SEL]                 =  "MAP_LOOKUP_SEL",
 	[VCAP_AF_MASK_MODE]                      =  "MASK_MODE",
+	[VCAP_AF_MATCH_ID]                       =  "MATCH_ID",
+	[VCAP_AF_MATCH_ID_MASK]                  =  "MATCH_ID_MASK",
 	[VCAP_AF_MIRROR_ENA]                     =  "MIRROR_ENA",
+	[VCAP_AF_MIRROR_PROBE]                   =  "MIRROR_PROBE",
+	[VCAP_AF_MIRROR_PROBE_ID]                =  "MIRROR_PROBE_ID",
+	[VCAP_AF_MRP_SEL]                        =  "MRP_SEL",
+	[VCAP_AF_NXT_IDX]                        =  "NXT_IDX",
+	[VCAP_AF_NXT_IDX_CTRL]                   =  "NXT_IDX_CTRL",
+	[VCAP_AF_OAM_SEL]                        =  "OAM_SEL",
+	[VCAP_AF_PAG_OVERRIDE_MASK]              =  "PAG_OVERRIDE_MASK",
+	[VCAP_AF_PAG_VAL]                        =  "PAG_VAL",
+	[VCAP_AF_PCP_ENA]                        =  "PCP_ENA",
+	[VCAP_AF_PCP_VAL]                        =  "PCP_VAL",
+	[VCAP_AF_PIPELINE_FORCE_ENA]             =  "PIPELINE_FORCE_ENA",
+	[VCAP_AF_PIPELINE_PT]                    =  "PIPELINE_PT",
 	[VCAP_AF_POLICE_ENA]                     =  "POLICE_ENA",
 	[VCAP_AF_POLICE_IDX]                     =  "POLICE_IDX",
+	[VCAP_AF_POLICE_REMARK]                  =  "POLICE_REMARK",
 	[VCAP_AF_POLICE_VCAP_ONLY]               =  "POLICE_VCAP_ONLY",
 	[VCAP_AF_PORT_MASK]                      =  "PORT_MASK",
+	[VCAP_AF_QOS_ENA]                        =  "QOS_ENA",
+	[VCAP_AF_QOS_VAL]                        =  "QOS_VAL",
 	[VCAP_AF_REW_OP]                         =  "REW_OP",
+	[VCAP_AF_RT_DIS]                         =  "RT_DIS",
+	[VCAP_AF_SFID_ENA]                       =  "SFID_ENA",
+	[VCAP_AF_SFID_VAL]                       =  "SFID_VAL",
+	[VCAP_AF_SGID_ENA]                       =  "SGID_ENA",
+	[VCAP_AF_SGID_VAL]                       =  "SGID_VAL",
+	[VCAP_AF_TYPE]                           =  "TYPE",
+	[VCAP_AF_VID_REPLACE_ENA]                =  "VID_REPLACE_ENA",
+	[VCAP_AF_VID_VAL]                        =  "VID_VAL",
+	[VCAP_AF_VLAN_POP_CNT]                   =  "VLAN_POP_CNT",
+	[VCAP_AF_VLAN_POP_CNT_ENA]               =  "VLAN_POP_CNT_ENA",
 };
 
 /* VCAPs */
 const struct vcap_info lan966x_vcaps[] = {
+	[VCAP_TYPE_IS1] = {
+		.name = "is1",
+		.rows = 192,
+		.sw_count = 4,
+		.sw_width = 96,
+		.sticky_width = 32,
+		.act_width = 123,
+		.default_cnt = 0,
+		.require_cnt_dis = 1,
+		.version = 1,
+		.keyfield_set = is1_keyfield_set,
+		.keyfield_set_size = ARRAY_SIZE(is1_keyfield_set),
+		.actionfield_set = is1_actionfield_set,
+		.actionfield_set_size = ARRAY_SIZE(is1_actionfield_set),
+		.keyfield_set_map = is1_keyfield_set_map,
+		.keyfield_set_map_size = is1_keyfield_set_map_size,
+		.actionfield_set_map = is1_actionfield_set_map,
+		.actionfield_set_map_size = is1_actionfield_set_map_size,
+		.keyfield_set_typegroups = is1_keyfield_set_typegroups,
+		.actionfield_set_typegroups = is1_actionfield_set_typegroups,
+	},
 	[VCAP_TYPE_IS2] = {
 		.name = "is2",
 		.rows = 64,
@@ -1600,7 +3000,7 @@ const struct vcap_info lan966x_vcaps[] = {
 
 const struct vcap_statistics lan966x_vcap_stats = {
 	.name = "lan966x",
-	.count = 1,
+	.count = 2,
 	.keyfield_set_names = vcap_keyfield_set_names,
 	.actionfield_set_names = vcap_actionfield_set_names,
 	.keyfield_names = vcap_keyfield_names,
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_debugfs.c b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_debugfs.c
index 7a0db58f5513..d90c08cfcf14 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_debugfs.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_debugfs.c
@@ -5,9 +5,124 @@
 #include "vcap_api.h"
 #include "vcap_api_client.h"
 
-static void lan966x_vcap_port_keys(struct lan966x_port *port,
-				   struct vcap_admin *admin,
-				   struct vcap_output_print *out)
+static void lan966x_vcap_is1_port_keys(struct lan966x_port *port,
+				       struct vcap_admin *admin,
+				       struct vcap_output_print *out)
+{
+	struct lan966x *lan966x = port->lan966x;
+	u32 val;
+
+	out->prf(out->dst, "  port[%d] (%s): ", port->chip_port,
+		 netdev_name(port->dev));
+
+	val = lan_rd(lan966x, ANA_VCAP_CFG(port->chip_port));
+	out->prf(out->dst, "\n    state: ");
+	if (ANA_VCAP_CFG_S1_ENA_GET(val))
+		out->prf(out->dst, "on");
+	else
+		out->prf(out->dst, "off");
+
+	for (int l = 0; l < admin->lookups; ++l) {
+		out->prf(out->dst, "\n    Lookup %d: ", l);
+
+		out->prf(out->dst, "\n      other: ");
+		switch (ANA_VCAP_S1_CFG_KEY_OTHER_CFG_GET(val)) {
+		case VCAP_IS1_PS_OTHER_NORMAL:
+			out->prf(out->dst, "normal");
+			break;
+		case VCAP_IS1_PS_OTHER_7TUPLE:
+			out->prf(out->dst, "7tuple");
+			break;
+		case VCAP_IS1_PS_OTHER_DBL_VID:
+			out->prf(out->dst, "dbl_vid");
+			break;
+		case VCAP_IS1_PS_OTHER_DMAC_VID:
+			out->prf(out->dst, "dmac_vid");
+			break;
+		default:
+			out->prf(out->dst, "-");
+			break;
+		}
+
+		out->prf(out->dst, "\n      ipv4: ");
+		switch (ANA_VCAP_S1_CFG_KEY_IP4_CFG_GET(val)) {
+		case VCAP_IS1_PS_IPV4_NORMAL:
+			out->prf(out->dst, "normal");
+			break;
+		case VCAP_IS1_PS_IPV4_7TUPLE:
+			out->prf(out->dst, "7tuple");
+			break;
+		case VCAP_IS1_PS_IPV4_5TUPLE_IP4:
+			out->prf(out->dst, "5tuple_ipv4");
+			break;
+		case VCAP_IS1_PS_IPV4_DBL_VID:
+			out->prf(out->dst, "dbl_vid");
+			break;
+		case VCAP_IS1_PS_IPV4_DMAC_VID:
+			out->prf(out->dst, "dmac_vid");
+			break;
+		default:
+			out->prf(out->dst, "-");
+			break;
+		}
+
+		out->prf(out->dst, "\n      ipv6: ");
+		switch (ANA_VCAP_S1_CFG_KEY_IP6_CFG_GET(val)) {
+		case VCAP_IS1_PS_IPV6_NORMAL:
+			out->prf(out->dst, "normal");
+			break;
+		case VCAP_IS1_PS_IPV6_7TUPLE:
+			out->prf(out->dst, "7tuple");
+			break;
+		case VCAP_IS1_PS_IPV6_5TUPLE_IP4:
+			out->prf(out->dst, "5tuple_ip4");
+			break;
+		case VCAP_IS1_PS_IPV6_NORMAL_IP6:
+			out->prf(out->dst, "normal_ip6");
+			break;
+		case VCAP_IS1_PS_IPV6_5TUPLE_IP6:
+			out->prf(out->dst, "5tuple_ip6");
+			break;
+		case VCAP_IS1_PS_IPV6_DBL_VID:
+			out->prf(out->dst, "dbl_vid");
+			break;
+		case VCAP_IS1_PS_IPV6_DMAC_VID:
+			out->prf(out->dst, "dmac_vid");
+			break;
+		default:
+			out->prf(out->dst, "-");
+			break;
+		}
+
+		out->prf(out->dst, "\n      rt: ");
+		switch (ANA_VCAP_S1_CFG_KEY_RT_CFG_GET(val)) {
+		case VCAP_IS1_PS_RT_NORMAL:
+			out->prf(out->dst, "normal");
+			break;
+		case VCAP_IS1_PS_RT_7TUPLE:
+			out->prf(out->dst, "7tuple");
+			break;
+		case VCAP_IS1_PS_RT_DBL_VID:
+			out->prf(out->dst, "dbl_vid");
+			break;
+		case VCAP_IS1_PS_RT_DMAC_VID:
+			out->prf(out->dst, "dmac_vid");
+			break;
+		case VCAP_IS1_PS_RT_FOLLOW_OTHER:
+			out->prf(out->dst, "follow_other");
+			break;
+		default:
+			out->prf(out->dst, "-");
+			break;
+		}
+	}
+
+	out->prf(out->dst, "\n");
+}
+
+static void lan966x_vcap_is2_port_keys(struct lan966x_port *port,
+				       struct vcap_admin *admin,
+				       struct vcap_output_print *out)
 {
 	struct lan966x *lan966x = port->lan966x;
 	u32 val;
@@ -88,7 +203,17 @@ int lan966x_vcap_port_info(struct net_device *dev,
 	vcap = &vctrl->vcaps[admin->vtype];
 
 	out->prf(out->dst, "%s:\n", vcap->name);
-	lan966x_vcap_port_keys(port, admin, out);
+	switch (admin->vtype) {
+	case VCAP_TYPE_IS2:
+		lan966x_vcap_is2_port_keys(port, admin, out);
+		break;
+	case VCAP_TYPE_IS1:
+		lan966x_vcap_is1_port_keys(port, admin, out);
+		break;
+	default:
+		out->prf(out->dst, "  no info\n");
+		break;
+	}
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c
index 68f9d69fd37b..7ea8e8633609 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_vcap_impl.c
@@ -8,6 +8,7 @@
 
 #define STREAMSIZE (64 * 4)
 
+#define LAN966X_IS1_LOOKUPS 3
 #define LAN966X_IS2_LOOKUPS 2
 
 static struct lan966x_vcap_inst {
@@ -20,6 +21,15 @@ static struct lan966x_vcap_inst {
 	bool ingress; /* is vcap in the ingress path */
 } lan966x_vcap_inst_cfg[] = {
 	{
+		.vtype = VCAP_TYPE_IS1, /* IS1-0 */
+		.tgt_inst = 1,
+		.lookups = LAN966X_IS1_LOOKUPS,
+		.first_cid = LAN966X_VCAP_CID_IS1_L0,
+		.last_cid = LAN966X_VCAP_CID_IS1_MAX,
+		.count = 768,
+		.ingress = true,
+	},
+	{
 		.vtype = VCAP_TYPE_IS2, /* IS2-0 */
 		.tgt_inst = 2,
 		.lookups = LAN966X_IS2_LOOKUPS,
@@ -72,7 +82,21 @@ static void __lan966x_vcap_range_init(struct lan966x *lan966x,
 	lan966x_vcap_wait_update(lan966x, admin->tgt_inst);
 }
 
-static int lan966x_vcap_cid_to_lookup(int cid)
+static int lan966x_vcap_is1_cid_to_lookup(int cid)
+{
+	int lookup = 0;
+
+	if (cid >= LAN966X_VCAP_CID_IS1_L1 &&
+	    cid < LAN966X_VCAP_CID_IS1_L2)
+		lookup = 1;
+	else if (cid >= LAN966X_VCAP_CID_IS1_L2 &&
+		 cid < LAN966X_VCAP_CID_IS1_MAX)
+		lookup = 2;
+
+	return lookup;
+}
+
+static int lan966x_vcap_is2_cid_to_lookup(int cid)
 {
 	if (cid >= LAN966X_VCAP_CID_IS2_L1 &&
 	    cid < LAN966X_VCAP_CID_IS2_MAX)
@@ -81,6 +105,67 @@ static int lan966x_vcap_cid_to_lookup(int cid)
 	return 0;
 }
 
+/* Return the list of keysets for the vcap port configuration */
+static int
+lan966x_vcap_is1_get_port_keysets(struct net_device *ndev, int lookup,
+				  struct vcap_keyset_list *keysetlist,
+				  u16 l3_proto)
+{
+	struct lan966x_port *port = netdev_priv(ndev);
+	struct lan966x *lan966x = port->lan966x;
+	u32 val;
+
+	val = lan_rd(lan966x, ANA_VCAP_S1_CFG(port->chip_port, lookup));
+
+	/* Collect all keysets for the port in a list */
+	if (l3_proto == ETH_P_ALL || l3_proto == ETH_P_IP) {
+		switch (ANA_VCAP_S1_CFG_KEY_IP4_CFG_GET(val)) {
+		case VCAP_IS1_PS_IPV4_7TUPLE:
+			vcap_keyset_list_add(keysetlist, VCAP_KFS_7TUPLE);
+			break;
+		case VCAP_IS1_PS_IPV4_5TUPLE_IP4:
+			vcap_keyset_list_add(keysetlist, VCAP_KFS_5TUPLE_IP4);
+			break;
+		case VCAP_IS1_PS_IPV4_NORMAL:
+			vcap_keyset_list_add(keysetlist, VCAP_KFS_NORMAL);
+			break;
+		}
+	}
+
+	if (l3_proto == ETH_P_ALL || l3_proto == ETH_P_IPV6) {
+		switch (ANA_VCAP_S1_CFG_KEY_IP6_CFG_GET(val)) {
+		case VCAP_IS1_PS_IPV6_NORMAL:
+		case VCAP_IS1_PS_IPV6_NORMAL_IP6:
+			vcap_keyset_list_add(keysetlist, VCAP_KFS_NORMAL);
+			vcap_keyset_list_add(keysetlist, VCAP_KFS_NORMAL_IP6);
+			break;
+		case VCAP_IS1_PS_IPV6_5TUPLE_IP6:
+			vcap_keyset_list_add(keysetlist, VCAP_KFS_5TUPLE_IP6);
+			break;
+		case VCAP_IS1_PS_IPV6_7TUPLE:
+			vcap_keyset_list_add(keysetlist, VCAP_KFS_7TUPLE);
+			break;
+		case VCAP_IS1_PS_IPV6_5TUPLE_IP4:
+			vcap_keyset_list_add(keysetlist, VCAP_KFS_5TUPLE_IP4);
+			break;
+		case VCAP_IS1_PS_IPV6_DMAC_VID:
+			vcap_keyset_list_add(keysetlist, VCAP_KFS_DMAC_VID);
+			break;
+		}
+	}
+
+	switch (ANA_VCAP_S1_CFG_KEY_OTHER_CFG_GET(val)) {
+	case VCAP_IS1_PS_OTHER_7TUPLE:
+		vcap_keyset_list_add(keysetlist, VCAP_KFS_7TUPLE);
+		break;
+	case VCAP_IS1_PS_OTHER_NORMAL:
+		vcap_keyset_list_add(keysetlist, VCAP_KFS_NORMAL);
+		break;
+	}
+
+	return 0;
+}
+
 static int
 lan966x_vcap_is2_get_port_keysets(struct net_device *dev, int lookup,
 				  struct vcap_keyset_list *keysetlist,
@@ -180,11 +265,26 @@ lan966x_vcap_validate_keyset(struct net_device *dev,
 	if (!kslist || kslist->cnt == 0)
 		return VCAP_KFS_NO_VALUE;
 
-	lookup = lan966x_vcap_cid_to_lookup(rule->vcap_chain_id);
 	keysetlist.max = ARRAY_SIZE(keysets);
 	keysetlist.keysets = keysets;
-	err = lan966x_vcap_is2_get_port_keysets(dev, lookup, &keysetlist,
-						l3_proto);
+
+	switch (admin->vtype) {
+	case VCAP_TYPE_IS1:
+		lookup = lan966x_vcap_is1_cid_to_lookup(rule->vcap_chain_id);
+		err = lan966x_vcap_is1_get_port_keysets(dev, lookup, &keysetlist,
+							l3_proto);
+		break;
+	case VCAP_TYPE_IS2:
+		lookup = lan966x_vcap_is2_cid_to_lookup(rule->vcap_chain_id);
+		err = lan966x_vcap_is2_get_port_keysets(dev, lookup, &keysetlist,
+							l3_proto);
+		break;
+	default:
+		pr_err("vcap type: %s not supported\n",
+		       lan966x_vcaps[admin->vtype].name);
+		return VCAP_KFS_NO_VALUE;
+	}
+
 	if (err)
 		return VCAP_KFS_NO_VALUE;
 
@@ -197,17 +297,32 @@ lan966x_vcap_validate_keyset(struct net_device *dev,
 	return VCAP_KFS_NO_VALUE;
 }
 
-static bool lan966x_vcap_is_first_chain(struct vcap_rule *rule)
+static bool lan966x_vcap_is2_is_first_chain(struct vcap_rule *rule)
 {
 	return (rule->vcap_chain_id >= LAN966X_VCAP_CID_IS2_L0 &&
 		rule->vcap_chain_id < LAN966X_VCAP_CID_IS2_L1);
 }
 
-static void lan966x_vcap_add_default_fields(struct net_device *dev,
-					    struct vcap_admin *admin,
-					    struct vcap_rule *rule)
+static void lan966x_vcap_is1_add_default_fields(struct lan966x_port *port,
+						struct vcap_admin *admin,
+						struct vcap_rule *rule)
+{
+	u32 value, mask;
+	u32 lookup;
+
+	if (vcap_rule_get_key_u32(rule, VCAP_KF_IF_IGR_PORT_MASK,
+				  &value, &mask))
+		vcap_rule_add_key_u32(rule, VCAP_KF_IF_IGR_PORT_MASK, 0,
+				      ~BIT(port->chip_port));
+
+	lookup = lan966x_vcap_is1_cid_to_lookup(rule->vcap_chain_id);
+	vcap_rule_add_key_u32(rule, VCAP_KF_LOOKUP_INDEX, lookup, 0x3);
+}
+
+static void lan966x_vcap_is2_add_default_fields(struct lan966x_port *port,
+						struct vcap_admin *admin,
+						struct vcap_rule *rule)
 {
-	struct lan966x_port *port = netdev_priv(dev);
 	u32 value, mask;
 
 	if (vcap_rule_get_key_u32(rule, VCAP_KF_IF_IGR_PORT_MASK,
@@ -215,7 +330,7 @@ static void lan966x_vcap_add_default_fields(struct net_device *dev,
 		vcap_rule_add_key_u32(rule, VCAP_KF_IF_IGR_PORT_MASK, 0,
 				      ~BIT(port->chip_port));
 
-	if (lan966x_vcap_is_first_chain(rule))
+	if (lan966x_vcap_is2_is_first_chain(rule))
 		vcap_rule_add_key_bit(rule, VCAP_KF_LOOKUP_FIRST_IS,
 				      VCAP_BIT_1);
 	else
@@ -223,6 +338,26 @@ static void lan966x_vcap_add_default_fields(struct net_device *dev,
 				      VCAP_BIT_0);
 }
 
+static void lan966x_vcap_add_default_fields(struct net_device *dev,
+					    struct vcap_admin *admin,
+					    struct vcap_rule *rule)
+{
+	struct lan966x_port *port = netdev_priv(dev);
+
+	switch (admin->vtype) {
+	case VCAP_TYPE_IS1:
+		lan966x_vcap_is1_add_default_fields(port, admin, rule);
+		break;
+	case VCAP_TYPE_IS2:
+		lan966x_vcap_is2_add_default_fields(port, admin, rule);
+		break;
+	default:
+		pr_err("vcap type: %s not supported\n",
+		       lan966x_vcaps[admin->vtype].name);
+		break;
+	}
+}
+
 static void lan966x_vcap_cache_erase(struct vcap_admin *admin)
 {
 	memset(admin->cache.keystream, 0, STREAMSIZE);
@@ -464,8 +599,37 @@ static void lan966x_vcap_block_init(struct lan966x *lan966x,
 static void lan966x_vcap_port_key_deselection(struct lan966x *lan966x,
 					      struct vcap_admin *admin)
 {
-	for (int p = 0; p < lan966x->num_phys_ports; ++p)
-		lan_wr(0, lan966x, ANA_VCAP_S2_CFG(p));
+	u32 val;
+
+	switch (admin->vtype) {
+	case VCAP_TYPE_IS1:
+		val = ANA_VCAP_S1_CFG_KEY_IP6_CFG_SET(VCAP_IS1_PS_IPV6_5TUPLE_IP6) |
+		      ANA_VCAP_S1_CFG_KEY_IP4_CFG_SET(VCAP_IS1_PS_IPV4_5TUPLE_IP4) |
+		      ANA_VCAP_S1_CFG_KEY_OTHER_CFG_SET(VCAP_IS1_PS_OTHER_NORMAL);
+
+		for (int p = 0; p < lan966x->num_phys_ports; ++p) {
+			if (!lan966x->ports[p])
+				continue;
+
+			for (int l = 0; l < LAN966X_IS1_LOOKUPS; ++l)
+				lan_wr(val, lan966x, ANA_VCAP_S1_CFG(p, l));
+
+			lan_rmw(ANA_VCAP_CFG_S1_ENA_SET(true),
+				ANA_VCAP_CFG_S1_ENA, lan966x,
+				ANA_VCAP_CFG(p));
+		}
+
+		break;
+	case VCAP_TYPE_IS2:
+		for (int p = 0; p < lan966x->num_phys_ports; ++p)
+			lan_wr(0, lan966x, ANA_VCAP_S2_CFG(p));
+
+		break;
+	default:
+		pr_err("vcap type: %s not supported\n",
+		       lan966x_vcaps[admin->vtype].name);
+		break;
+	}
 }
 
 int lan966x_vcap_init(struct lan966x *lan966x)
@@ -506,6 +670,10 @@ int lan966x_vcap_init(struct lan966x *lan966x)
 			lan_rmw(ANA_VCAP_S2_CFG_ENA_SET(true),
 				ANA_VCAP_S2_CFG_ENA, lan966x,
 				ANA_VCAP_S2_CFG(lan966x->ports[p]->chip_port));
+
+			lan_rmw(ANA_VCAP_CFG_S1_ENA_SET(true),
+				ANA_VCAP_CFG_S1_ENA, lan966x,
+				ANA_VCAP_CFG(lan966x->ports[p]->chip_port));
 		}
 	}
 
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c
index 2e6f486ec67d..9ee61db8690b 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c
@@ -62,7 +62,7 @@ int lan966x_xdp_xmit(struct net_device *dev,
 		struct xdp_frame *xdpf = frames[i];
 		int err;
 
-		err = lan966x_fdma_xmit_xdpf(port, xdpf, NULL, true);
+		err = lan966x_fdma_xmit_xdpf(port, xdpf, 0);
 		if (err)
 			break;
 
@@ -76,7 +76,6 @@ int lan966x_xdp_run(struct lan966x_port *port, struct page *page, u32 data_len)
 {
 	struct bpf_prog *xdp_prog = port->xdp_prog;
 	struct lan966x *lan966x = port->lan966x;
-	struct xdp_frame *xdpf;
 	struct xdp_buff xdp;
 	u32 act;
 
@@ -90,11 +89,8 @@ int lan966x_xdp_run(struct lan966x_port *port, struct page *page, u32 data_len)
 	case XDP_PASS:
 		return FDMA_PASS;
 	case XDP_TX:
-		xdpf = xdp_convert_buff_to_frame(&xdp);
-		if (!xdpf)
-			return FDMA_DROP;
-
-		return lan966x_fdma_xmit_xdpf(port, xdpf, page, false) ?
+		return lan966x_fdma_xmit_xdpf(port, page,
+					      data_len - IFH_LEN_BYTES) ?
 		       FDMA_DROP : FDMA_TX;
 	case XDP_REDIRECT:
 		if (xdp_do_redirect(port->dev, &xdp, xdp_prog))
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c
index 42b77ba9b572..a7edf524eedb 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c
@@ -282,6 +282,7 @@ static int sparx5_create_port(struct sparx5 *sparx5,
 	spx5_port->phylink_pcs.poll = true;
 	spx5_port->phylink_pcs.ops = &sparx5_phylink_pcs_ops;
 	spx5_port->is_mrouter = false;
+	INIT_LIST_HEAD(&spx5_port->tc_templates);
 	sparx5->ports[config->portno] = spx5_port;
 
 	err = sparx5_port_init(sparx5, spx5_port, &config->conf);
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
index 72e7928912eb..62c85463b634 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
@@ -192,6 +192,7 @@ struct sparx5_port {
 	u16 ts_id;
 	struct sk_buff_head tx_skbs;
 	bool is_mrouter;
+	struct list_head tc_templates; /* list of TC templates on this port */
 };
 
 enum sparx5_core_clockfreq {
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c b/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c
index b36819aafaca..3f87a5285a6d 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c
@@ -28,6 +28,14 @@ struct sparx5_multiple_rules {
 	struct sparx5_wildcard_rule rule[SPX5_MAX_RULE_SIZE];
 };
 
+struct sparx5_tc_flower_template {
+	struct list_head list; /* for insertion in the list of templates */
+	int cid; /* chain id */
+	enum vcap_keyfield_set orig; /* keyset used before the template */
+	enum vcap_keyfield_set keyset; /* new keyset used by template */
+	u16 l3_proto; /* protocol specified in the template */
+};
+
 static int
 sparx5_tc_flower_es0_tpid(struct vcap_tc_flower_parse_usage *st)
 {
@@ -382,7 +390,7 @@ static int sparx5_tc_select_protocol_keyset(struct net_device *ndev,
 	/* Find the keysets that the rule can use */
 	matches.keysets = keysets;
 	matches.max = ARRAY_SIZE(keysets);
-	if (vcap_rule_find_keysets(vrule, &matches) == 0)
+	if (!vcap_rule_find_keysets(vrule, &matches))
 		return -EINVAL;
 
 	/* Find the keysets that the port configuration supports */
@@ -996,6 +1004,73 @@ static int sparx5_tc_action_vlan_push(struct vcap_admin *admin,
 	return err;
 }
 
+/* Remove rule keys that may prevent templates from matching a keyset */
+static void sparx5_tc_flower_simplify_rule(struct vcap_admin *admin,
+					   struct vcap_rule *vrule,
+					   u16 l3_proto)
+{
+	switch (admin->vtype) {
+	case VCAP_TYPE_IS0:
+		vcap_rule_rem_key(vrule, VCAP_KF_ETYPE);
+		switch (l3_proto) {
+		case ETH_P_IP:
+			break;
+		case ETH_P_IPV6:
+			vcap_rule_rem_key(vrule, VCAP_KF_IP_SNAP_IS);
+			break;
+		default:
+			break;
+		}
+		break;
+	case VCAP_TYPE_ES2:
+		switch (l3_proto) {
+		case ETH_P_IP:
+			if (vrule->keyset == VCAP_KFS_IP4_OTHER)
+				vcap_rule_rem_key(vrule, VCAP_KF_TCP_IS);
+			break;
+		case ETH_P_IPV6:
+			if (vrule->keyset == VCAP_KFS_IP6_STD)
+				vcap_rule_rem_key(vrule, VCAP_KF_TCP_IS);
+			vcap_rule_rem_key(vrule, VCAP_KF_IP4_IS);
+			break;
+		default:
+			break;
+		}
+		break;
+	case VCAP_TYPE_IS2:
+		switch (l3_proto) {
+		case ETH_P_IP:
+		case ETH_P_IPV6:
+			vcap_rule_rem_key(vrule, VCAP_KF_IP4_IS);
+			break;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+}
+
+static bool sparx5_tc_flower_use_template(struct net_device *ndev,
+					  struct flow_cls_offload *fco,
+					  struct vcap_admin *admin,
+					  struct vcap_rule *vrule)
+{
+	struct sparx5_port *port = netdev_priv(ndev);
+	struct sparx5_tc_flower_template *ftp;
+
+	list_for_each_entry(ftp, &port->tc_templates, list) {
+		if (ftp->cid != fco->common.chain_index)
+			continue;
+
+		vcap_set_rule_set_keyset(vrule, ftp->keyset);
+		sparx5_tc_flower_simplify_rule(admin, vrule, ftp->l3_proto);
+		return true;
+	}
+	return false;
+}
+
 static int sparx5_tc_flower_replace(struct net_device *ndev,
 				    struct flow_cls_offload *fco,
 				    struct vcap_admin *admin,
@@ -1122,12 +1197,14 @@ static int sparx5_tc_flower_replace(struct net_device *ndev,
 			goto out;
 	}
 
-	err = sparx5_tc_select_protocol_keyset(ndev, vrule, admin,
-					       state.l3_proto, &multi);
-	if (err) {
-		NL_SET_ERR_MSG_MOD(fco->common.extack,
-				   "No matching port keyset for filter protocol and keys");
-		goto out;
+	if (!sparx5_tc_flower_use_template(ndev, fco, admin, vrule)) {
+		err = sparx5_tc_select_protocol_keyset(ndev, vrule, admin,
+						       state.l3_proto, &multi);
+		if (err) {
+			NL_SET_ERR_MSG_MOD(fco->common.extack,
+					   "No matching port keyset for filter protocol and keys");
+			goto out;
+		}
 	}
 
 	/* provide the l3 protocol to guide the keyset selection */
@@ -1259,6 +1336,120 @@ static int sparx5_tc_flower_stats(struct net_device *ndev,
 	return err;
 }
 
+static int sparx5_tc_flower_template_create(struct net_device *ndev,
+					    struct flow_cls_offload *fco,
+					    struct vcap_admin *admin)
+{
+	struct sparx5_port *port = netdev_priv(ndev);
+	struct vcap_tc_flower_parse_usage state = {
+		.fco = fco,
+		.l3_proto = ETH_P_ALL,
+		.admin = admin,
+	};
+	struct sparx5_tc_flower_template *ftp;
+	struct vcap_keyset_list kslist = {};
+	enum vcap_keyfield_set keysets[10];
+	struct vcap_control *vctrl;
+	struct vcap_rule *vrule;
+	int count, err;
+
+	if (admin->vtype == VCAP_TYPE_ES0) {
+		pr_err("%s:%d: %s\n", __func__, __LINE__,
+		       "VCAP does not support templates");
+		return -EINVAL;
+	}
+
+	count = vcap_admin_rule_count(admin, fco->common.chain_index);
+	if (count > 0) {
+		pr_err("%s:%d: %s\n", __func__, __LINE__,
+		       "Filters are already present");
+		return -EBUSY;
+	}
+
+	ftp = kzalloc(sizeof(*ftp), GFP_KERNEL);
+	if (!ftp)
+		return -ENOMEM;
+
+	ftp->cid = fco->common.chain_index;
+	ftp->orig = VCAP_KFS_NO_VALUE;
+	ftp->keyset = VCAP_KFS_NO_VALUE;
+
+	vctrl = port->sparx5->vcap_ctrl;
+	vrule = vcap_alloc_rule(vctrl, ndev, fco->common.chain_index,
+				VCAP_USER_TC, fco->common.prio, 0);
+	if (IS_ERR(vrule)) {
+		err = PTR_ERR(vrule);
+		goto err_rule;
+	}
+
+	state.vrule = vrule;
+	state.frule = flow_cls_offload_flow_rule(fco);
+	err = sparx5_tc_use_dissectors(&state, admin, vrule);
+	if (err) {
+		pr_err("%s:%d: key error: %d\n", __func__, __LINE__, err);
+		goto out;
+	}
+
+	ftp->l3_proto = state.l3_proto;
+
+	sparx5_tc_flower_simplify_rule(admin, vrule, state.l3_proto);
+
+	/* Find the keysets that the rule can use */
+	kslist.keysets = keysets;
+	kslist.max = ARRAY_SIZE(keysets);
+	if (!vcap_rule_find_keysets(vrule, &kslist)) {
+		pr_err("%s:%d: %s\n", __func__, __LINE__,
+		       "Could not find a suitable keyset");
+		err = -ENOENT;
+		goto out;
+	}
+
+	ftp->keyset = vcap_select_min_rule_keyset(vctrl, admin->vtype, &kslist);
+	kslist.cnt = 0;
+	sparx5_vcap_set_port_keyset(ndev, admin, fco->common.chain_index,
+				    state.l3_proto,
+				    ftp->keyset,
+				    &kslist);
+
+	if (kslist.cnt > 0)
+		ftp->orig = kslist.keysets[0];
+
+	/* Store new template */
+	list_add_tail(&ftp->list, &port->tc_templates);
+	vcap_free_rule(vrule);
+	return 0;
+
+out:
+	vcap_free_rule(vrule);
+err_rule:
+	kfree(ftp);
+	return err;
+}
+
+static int sparx5_tc_flower_template_destroy(struct net_device *ndev,
+					     struct flow_cls_offload *fco,
+					     struct vcap_admin *admin)
+{
+	struct sparx5_port *port = netdev_priv(ndev);
+	struct sparx5_tc_flower_template *ftp, *tmp;
+	int err = -ENOENT;
+
+	/* Rules using the template are removed by the tc framework */
+	list_for_each_entry_safe(ftp, tmp, &port->tc_templates, list) {
+		if (ftp->cid != fco->common.chain_index)
+			continue;
+
+		sparx5_vcap_set_port_keyset(ndev, admin,
+					    fco->common.chain_index,
+					    ftp->l3_proto, ftp->orig,
+					    NULL);
+		list_del(&ftp->list);
+		kfree(ftp);
+		break;
+	}
+	return err;
+}
+
 int sparx5_tc_flower(struct net_device *ndev, struct flow_cls_offload *fco,
 		     bool ingress)
 {
@@ -1282,6 +1473,10 @@ int sparx5_tc_flower(struct net_device *ndev, struct flow_cls_offload *fco,
 		return sparx5_tc_flower_destroy(ndev, fco, admin);
 	case FLOW_CLS_STATS:
 		return sparx5_tc_flower_stats(ndev, fco, admin);
+	case FLOW_CLS_TMPLT_CREATE:
+		return sparx5_tc_flower_template_create(ndev, fco, admin);
+	case FLOW_CLS_TMPLT_DESTROY:
+		return sparx5_tc_flower_template_destroy(ndev, fco, admin);
 	default:
 		return -EOPNOTSUPP;
 	}
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_debugfs.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_debugfs.c
index 07b472c84a47..12722f728ef7 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_debugfs.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_debugfs.c
@@ -198,7 +198,7 @@ static void sparx5_vcap_is2_port_keys(struct sparx5 *sparx5,
 			out->prf(out->dst, "ip6_std");
 			break;
 		case VCAP_IS2_PS_IPV6_MC_IP4_TCP_UDP_OTHER:
-			out->prf(out->dst, "ip4_tcp_udp ipv4_other");
+			out->prf(out->dst, "ip4_tcp_udp ip4_other");
 			break;
 		}
 		out->prf(out->dst, "\n      ipv6_uc: ");
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c
index d0d4e0385ac7..187efa1fc904 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c
@@ -1519,6 +1519,276 @@ static struct vcap_operations sparx5_vcap_ops = {
 	.port_info = sparx5_port_info,
 };
 
+static u32 sparx5_vcap_is0_keyset_to_etype_ps(enum vcap_keyfield_set keyset)
+{
+	switch (keyset) {
+	case VCAP_KFS_NORMAL_7TUPLE:
+		return VCAP_IS0_PS_ETYPE_NORMAL_7TUPLE;
+	case VCAP_KFS_NORMAL_5TUPLE_IP4:
+		return VCAP_IS0_PS_ETYPE_NORMAL_5TUPLE_IP4;
+	default:
+		return VCAP_IS0_PS_ETYPE_NORMAL_7TUPLE;
+	}
+}
+
+static void sparx5_vcap_is0_set_port_keyset(struct net_device *ndev, int lookup,
+					    enum vcap_keyfield_set keyset,
+					    int l3_proto)
+{
+	struct sparx5_port *port = netdev_priv(ndev);
+	struct sparx5 *sparx5 = port->sparx5;
+	int portno = port->portno;
+	u32 value;
+
+	switch (l3_proto) {
+	case ETH_P_IP:
+		value = sparx5_vcap_is0_keyset_to_etype_ps(keyset);
+		spx5_rmw(ANA_CL_ADV_CL_CFG_IP4_CLM_KEY_SEL_SET(value),
+			 ANA_CL_ADV_CL_CFG_IP4_CLM_KEY_SEL,
+			 sparx5,
+			 ANA_CL_ADV_CL_CFG(portno, lookup));
+		break;
+	case ETH_P_IPV6:
+		value = sparx5_vcap_is0_keyset_to_etype_ps(keyset);
+		spx5_rmw(ANA_CL_ADV_CL_CFG_IP6_CLM_KEY_SEL_SET(value),
+			 ANA_CL_ADV_CL_CFG_IP6_CLM_KEY_SEL,
+			 sparx5,
+			 ANA_CL_ADV_CL_CFG(portno, lookup));
+		break;
+	default:
+		value = sparx5_vcap_is0_keyset_to_etype_ps(keyset);
+		spx5_rmw(ANA_CL_ADV_CL_CFG_ETYPE_CLM_KEY_SEL_SET(value),
+			 ANA_CL_ADV_CL_CFG_ETYPE_CLM_KEY_SEL,
+			 sparx5,
+			 ANA_CL_ADV_CL_CFG(portno, lookup));
+		break;
+	}
+}
+
+static u32 sparx5_vcap_is2_keyset_to_arp_ps(enum vcap_keyfield_set keyset)
+{
+	switch (keyset) {
+	case VCAP_KFS_ARP:
+		return VCAP_IS2_PS_ARP_ARP;
+	default:
+		return VCAP_IS2_PS_ARP_MAC_ETYPE;
+	}
+}
+
+static u32 sparx5_vcap_is2_keyset_to_ipv4_ps(enum vcap_keyfield_set keyset)
+{
+	switch (keyset) {
+	case VCAP_KFS_MAC_ETYPE:
+		return VCAP_IS2_PS_IPV4_UC_MAC_ETYPE;
+	case VCAP_KFS_IP4_OTHER:
+	case VCAP_KFS_IP4_TCP_UDP:
+		return VCAP_IS2_PS_IPV4_UC_IP4_TCP_UDP_OTHER;
+	case VCAP_KFS_IP_7TUPLE:
+		return VCAP_IS2_PS_IPV4_UC_IP_7TUPLE;
+	default:
+		return VCAP_KFS_NO_VALUE;
+	}
+}
+
+static u32 sparx5_vcap_is2_keyset_to_ipv6_uc_ps(enum vcap_keyfield_set keyset)
+{
+	switch (keyset) {
+	case VCAP_KFS_MAC_ETYPE:
+		return VCAP_IS2_PS_IPV6_UC_MAC_ETYPE;
+	case VCAP_KFS_IP4_OTHER:
+	case VCAP_KFS_IP4_TCP_UDP:
+		return VCAP_IS2_PS_IPV6_UC_IP4_TCP_UDP_OTHER;
+	case VCAP_KFS_IP_7TUPLE:
+		return VCAP_IS2_PS_IPV6_UC_IP_7TUPLE;
+	default:
+		return VCAP_KFS_NO_VALUE;
+	}
+}
+
+static u32 sparx5_vcap_is2_keyset_to_ipv6_mc_ps(enum vcap_keyfield_set keyset)
+{
+	switch (keyset) {
+	case VCAP_KFS_MAC_ETYPE:
+		return VCAP_IS2_PS_IPV6_MC_MAC_ETYPE;
+	case VCAP_KFS_IP4_OTHER:
+	case VCAP_KFS_IP4_TCP_UDP:
+		return VCAP_IS2_PS_IPV6_MC_IP4_TCP_UDP_OTHER;
+	case VCAP_KFS_IP_7TUPLE:
+		return VCAP_IS2_PS_IPV6_MC_IP_7TUPLE;
+	default:
+		return VCAP_KFS_NO_VALUE;
+	}
+}
+
+static void sparx5_vcap_is2_set_port_keyset(struct net_device *ndev, int lookup,
+					    enum vcap_keyfield_set keyset,
+					    int l3_proto)
+{
+	struct sparx5_port *port = netdev_priv(ndev);
+	struct sparx5 *sparx5 = port->sparx5;
+	int portno = port->portno;
+	u32 value;
+
+	switch (l3_proto) {
+	case ETH_P_ARP:
+		value = sparx5_vcap_is2_keyset_to_arp_ps(keyset);
+		spx5_rmw(ANA_ACL_VCAP_S2_KEY_SEL_ARP_KEY_SEL_SET(value),
+			 ANA_ACL_VCAP_S2_KEY_SEL_ARP_KEY_SEL,
+			 sparx5,
+			 ANA_ACL_VCAP_S2_KEY_SEL(portno, lookup));
+		break;
+	case ETH_P_IP:
+		value = sparx5_vcap_is2_keyset_to_ipv4_ps(keyset);
+		spx5_rmw(ANA_ACL_VCAP_S2_KEY_SEL_IP4_UC_KEY_SEL_SET(value),
+			 ANA_ACL_VCAP_S2_KEY_SEL_IP4_UC_KEY_SEL,
+			 sparx5,
+			 ANA_ACL_VCAP_S2_KEY_SEL(portno, lookup));
+		spx5_rmw(ANA_ACL_VCAP_S2_KEY_SEL_IP4_MC_KEY_SEL_SET(value),
+			 ANA_ACL_VCAP_S2_KEY_SEL_IP4_MC_KEY_SEL,
+			 sparx5,
+			 ANA_ACL_VCAP_S2_KEY_SEL(portno, lookup));
+		break;
+	case ETH_P_IPV6:
+		value = sparx5_vcap_is2_keyset_to_ipv6_uc_ps(keyset);
+		spx5_rmw(ANA_ACL_VCAP_S2_KEY_SEL_IP6_UC_KEY_SEL_SET(value),
+			 ANA_ACL_VCAP_S2_KEY_SEL_IP6_UC_KEY_SEL,
+			 sparx5,
+			 ANA_ACL_VCAP_S2_KEY_SEL(portno, lookup));
+		value = sparx5_vcap_is2_keyset_to_ipv6_mc_ps(keyset);
+		spx5_rmw(ANA_ACL_VCAP_S2_KEY_SEL_IP6_MC_KEY_SEL_SET(value),
+			 ANA_ACL_VCAP_S2_KEY_SEL_IP6_MC_KEY_SEL,
+			 sparx5,
+			 ANA_ACL_VCAP_S2_KEY_SEL(portno, lookup));
+		break;
+	default:
+		value = VCAP_IS2_PS_NONETH_MAC_ETYPE;
+		spx5_rmw(ANA_ACL_VCAP_S2_KEY_SEL_NON_ETH_KEY_SEL_SET(value),
+			 ANA_ACL_VCAP_S2_KEY_SEL_NON_ETH_KEY_SEL,
+			 sparx5,
+			 ANA_ACL_VCAP_S2_KEY_SEL(portno, lookup));
+		break;
+	}
+}
+
+static u32 sparx5_vcap_es2_keyset_to_arp_ps(enum vcap_keyfield_set keyset)
+{
+	switch (keyset) {
+	case VCAP_KFS_ARP:
+		return VCAP_ES2_PS_ARP_ARP;
+	default:
+		return VCAP_ES2_PS_ARP_MAC_ETYPE;
+	}
+}
+
+static u32 sparx5_vcap_es2_keyset_to_ipv4_ps(enum vcap_keyfield_set keyset)
+{
+	switch (keyset) {
+	case VCAP_KFS_MAC_ETYPE:
+		return VCAP_ES2_PS_IPV4_MAC_ETYPE;
+	case VCAP_KFS_IP_7TUPLE:
+		return VCAP_ES2_PS_IPV4_IP_7TUPLE;
+	case VCAP_KFS_IP4_TCP_UDP:
+		return VCAP_ES2_PS_IPV4_IP4_TCP_UDP_OTHER;
+	case VCAP_KFS_IP4_OTHER:
+		return VCAP_ES2_PS_IPV4_IP4_OTHER;
+	default:
+		return VCAP_ES2_PS_IPV4_MAC_ETYPE;
+	}
+}
+
+static u32 sparx5_vcap_es2_keyset_to_ipv6_ps(enum vcap_keyfield_set keyset)
+{
+	switch (keyset) {
+	case VCAP_KFS_MAC_ETYPE:
+		return VCAP_ES2_PS_IPV6_MAC_ETYPE;
+	case VCAP_KFS_IP4_TCP_UDP:
+	case VCAP_KFS_IP4_OTHER:
+		return VCAP_ES2_PS_IPV6_IP4_DOWNGRADE;
+	case VCAP_KFS_IP_7TUPLE:
+		return VCAP_ES2_PS_IPV6_IP_7TUPLE;
+	case VCAP_KFS_IP6_STD:
+		return VCAP_ES2_PS_IPV6_IP6_STD;
+	default:
+		return VCAP_ES2_PS_IPV6_MAC_ETYPE;
+	}
+}
+
+static void sparx5_vcap_es2_set_port_keyset(struct net_device *ndev, int lookup,
+					    enum vcap_keyfield_set keyset,
+					    int l3_proto)
+{
+	struct sparx5_port *port = netdev_priv(ndev);
+	struct sparx5 *sparx5 = port->sparx5;
+	int portno = port->portno;
+	u32 value;
+
+	switch (l3_proto) {
+	case ETH_P_IP:
+		value = sparx5_vcap_es2_keyset_to_ipv4_ps(keyset);
+		spx5_rmw(EACL_VCAP_ES2_KEY_SEL_IP4_KEY_SEL_SET(value),
+			 EACL_VCAP_ES2_KEY_SEL_IP4_KEY_SEL,
+			 sparx5,
+			 EACL_VCAP_ES2_KEY_SEL(portno, lookup));
+		break;
+	case ETH_P_IPV6:
+		value = sparx5_vcap_es2_keyset_to_ipv6_ps(keyset);
+		spx5_rmw(EACL_VCAP_ES2_KEY_SEL_IP6_KEY_SEL_SET(value),
+			 EACL_VCAP_ES2_KEY_SEL_IP6_KEY_SEL,
+			 sparx5,
+			 EACL_VCAP_ES2_KEY_SEL(portno, lookup));
+		break;
+	case ETH_P_ARP:
+		value = sparx5_vcap_es2_keyset_to_arp_ps(keyset);
+		spx5_rmw(EACL_VCAP_ES2_KEY_SEL_ARP_KEY_SEL_SET(value),
+			 EACL_VCAP_ES2_KEY_SEL_ARP_KEY_SEL,
+			 sparx5,
+			 EACL_VCAP_ES2_KEY_SEL(portno, lookup));
+		break;
+	}
+}
+
+/* Change the port keyset for the lookup and protocol */
+void sparx5_vcap_set_port_keyset(struct net_device *ndev,
+				 struct vcap_admin *admin,
+				 int cid,
+				 u16 l3_proto,
+				 enum vcap_keyfield_set keyset,
+				 struct vcap_keyset_list *orig)
+{
+	struct sparx5_port *port;
+	int lookup;
+
+	switch (admin->vtype) {
+	case VCAP_TYPE_IS0:
+		lookup = sparx5_vcap_is0_cid_to_lookup(cid);
+		if (orig)
+			sparx5_vcap_is0_get_port_keysets(ndev, lookup, orig,
+							 l3_proto);
+		sparx5_vcap_is0_set_port_keyset(ndev, lookup, keyset, l3_proto);
+		break;
+	case VCAP_TYPE_IS2:
+		lookup = sparx5_vcap_is2_cid_to_lookup(cid);
+		if (orig)
+			sparx5_vcap_is2_get_port_keysets(ndev, lookup, orig,
+							 l3_proto);
+		sparx5_vcap_is2_set_port_keyset(ndev, lookup, keyset, l3_proto);
+		break;
+	case VCAP_TYPE_ES0:
+		break;
+	case VCAP_TYPE_ES2:
+		lookup = sparx5_vcap_es2_cid_to_lookup(cid);
+		if (orig)
+			sparx5_vcap_es2_get_port_keysets(ndev, lookup, orig,
+							 l3_proto);
+		sparx5_vcap_es2_set_port_keyset(ndev, lookup, keyset, l3_proto);
+		break;
+	default:
+		port = netdev_priv(ndev);
+		sparx5_vcap_type_err(port->sparx5, admin, __func__);
+		break;
+	}
+}
+
 /* Enable IS0 lookups per port and set the keyset generation */
 static void sparx5_vcap_is0_port_key_selection(struct sparx5 *sparx5,
 					       struct vcap_admin *admin)
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.h b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.h
index 3260ab5e3a82..2684d9199b05 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.h
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.h
@@ -195,6 +195,12 @@ int sparx5_vcap_get_port_keyset(struct net_device *ndev,
 				u16 l3_proto,
 				struct vcap_keyset_list *kslist);
 
+/* Change the port keyset for the lookup and protocol */
+void sparx5_vcap_set_port_keyset(struct net_device *ndev,
+				 struct vcap_admin *admin, int cid,
+				 u16 l3_proto, enum vcap_keyfield_set keyset,
+				 struct vcap_keyset_list *orig);
+
 /* Check if the ethertype is supported by the vcap port classification */
 bool sparx5_vcap_is_known_etype(struct vcap_admin *admin, u16 etype);
 
diff --git a/drivers/net/ethernet/microchip/vcap/vcap_ag_api.h b/drivers/net/ethernet/microchip/vcap/vcap_ag_api.h
index 0844fcaeee68..a556c4419986 100644
--- a/drivers/net/ethernet/microchip/vcap/vcap_ag_api.h
+++ b/drivers/net/ethernet/microchip/vcap/vcap_ag_api.h
@@ -3,8 +3,8 @@
  * Microchip VCAP API
  */
 
-/* This file is autogenerated by cml-utils 2023-02-10 11:15:56 +0100.
- * Commit ID: c30fb4bf0281cd4a7133bdab6682f9e43c872ada
+/* This file is autogenerated by cml-utils 2023-02-16 11:41:14 +0100.
+ * Commit ID: be85f176b3a151fa748dcaf97c8824a5c2e065f3
  */
 
 #ifndef __VCAP_AG_API__
@@ -14,6 +14,7 @@ enum vcap_type {
 	VCAP_TYPE_ES0,
 	VCAP_TYPE_ES2,
 	VCAP_TYPE_IS0,
+	VCAP_TYPE_IS1,
 	VCAP_TYPE_IS2,
 	VCAP_TYPE_MAX
 };
@@ -21,7 +22,12 @@ enum vcap_type {
 /* Keyfieldset names with origin information */
 enum vcap_keyfield_set {
 	VCAP_KFS_NO_VALUE,          /* initial value */
+	VCAP_KFS_5TUPLE_IP4,        /* lan966x is1 X2 */
+	VCAP_KFS_5TUPLE_IP6,        /* lan966x is1 X4 */
+	VCAP_KFS_7TUPLE,            /* lan966x is1 X4 */
 	VCAP_KFS_ARP,               /* sparx5 is2 X6, sparx5 es2 X6, lan966x is2 X2 */
+	VCAP_KFS_DBL_VID,           /* lan966x is1 X1 */
+	VCAP_KFS_DMAC_VID,          /* lan966x is1 X1 */
 	VCAP_KFS_ETAG,              /* sparx5 is0 X2 */
 	VCAP_KFS_IP4_OTHER,         /* sparx5 is2 X6, sparx5 es2 X6, lan966x is2 X2 */
 	VCAP_KFS_IP4_TCP_UDP,       /* sparx5 is2 X6, sparx5 es2 X6, lan966x is2 X2 */
@@ -36,10 +42,13 @@ enum vcap_keyfield_set {
 	VCAP_KFS_MAC_ETYPE,         /* sparx5 is2 X6, sparx5 es2 X6, lan966x is2 X2 */
 	VCAP_KFS_MAC_LLC,           /* lan966x is2 X2 */
 	VCAP_KFS_MAC_SNAP,          /* lan966x is2 X2 */
+	VCAP_KFS_NORMAL,            /* lan966x is1 X2 */
 	VCAP_KFS_NORMAL_5TUPLE_IP4,  /* sparx5 is0 X6 */
 	VCAP_KFS_NORMAL_7TUPLE,     /* sparx5 is0 X12 */
+	VCAP_KFS_NORMAL_IP6,        /* lan966x is1 X4 */
 	VCAP_KFS_OAM,               /* lan966x is2 X2 */
 	VCAP_KFS_PURE_5TUPLE_IP4,   /* sparx5 is0 X3 */
+	VCAP_KFS_RT,                /* lan966x is1 X1 */
 	VCAP_KFS_SMAC_SIP4,         /* lan966x is2 X1 */
 	VCAP_KFS_SMAC_SIP6,         /* lan966x is2 X2 */
 };
@@ -61,17 +70,20 @@ enum vcap_keyfield_set {
  *   Used by 802.1BR Bridge Port Extension in an E-Tag
  * VCAP_KF_8021BR_IGR_ECID_EXT: W8, sparx5: is0
  *   Used by 802.1BR Bridge Port Extension in an E-Tag
- * VCAP_KF_8021Q_DEI0: W1, sparx5: is0
+ * VCAP_KF_8021CB_R_TAGGED_IS: W1, lan966x: is1
+ *   Set if frame contains an RTAG: IEEE 802.1CB (FRER Redundancy tag, Ethertype
+ *   0xf1c1)
+ * VCAP_KF_8021Q_DEI0: W1, sparx5: is0, lan966x: is1
  *   First DEI in multiple vlan tags (outer tag or default port tag)
- * VCAP_KF_8021Q_DEI1: W1, sparx5: is0
+ * VCAP_KF_8021Q_DEI1: W1, sparx5: is0, lan966x: is1
  *   Second DEI in multiple vlan tags (inner tag)
  * VCAP_KF_8021Q_DEI2: W1, sparx5: is0
  *   Third DEI in multiple vlan tags (not always available)
  * VCAP_KF_8021Q_DEI_CLS: W1, sparx5: is2/es2, lan966x: is2
  *   Classified DEI
- * VCAP_KF_8021Q_PCP0: W3, sparx5: is0
+ * VCAP_KF_8021Q_PCP0: W3, sparx5: is0, lan966x: is1
  *   First PCP in multiple vlan tags (outer tag or default port tag)
- * VCAP_KF_8021Q_PCP1: W3, sparx5: is0
+ * VCAP_KF_8021Q_PCP1: W3, sparx5: is0, lan966x: is1
  *   Second PCP in multiple vlan tags (inner tag)
  * VCAP_KF_8021Q_PCP2: W3, sparx5: is0
  *   Third PCP in multiple vlan tags (not always available)
@@ -79,22 +91,24 @@ enum vcap_keyfield_set {
  *   Classified PCP
  * VCAP_KF_8021Q_TPID: W3, sparx5: es0
  *   TPID for outer tag: 0: Customer TPID 1: Service TPID (88A8 or programmable)
- * VCAP_KF_8021Q_TPID0: W3, sparx5: is0
+ * VCAP_KF_8021Q_TPID0: sparx5 is0 W3, lan966x is1 W1
  *   First TPIC in multiple vlan tags (outer tag or default port tag)
- * VCAP_KF_8021Q_TPID1: W3, sparx5: is0
+ * VCAP_KF_8021Q_TPID1: sparx5 is0 W3, lan966x is1 W1
  *   Second TPID in multiple vlan tags (inner tag)
  * VCAP_KF_8021Q_TPID2: W3, sparx5: is0
  *   Third TPID in multiple vlan tags (not always available)
- * VCAP_KF_8021Q_VID0: W12, sparx5: is0
+ * VCAP_KF_8021Q_VID0: W12, sparx5: is0, lan966x: is1
  *   First VID in multiple vlan tags (outer tag or default port tag)
- * VCAP_KF_8021Q_VID1: W12, sparx5: is0
+ * VCAP_KF_8021Q_VID1: W12, sparx5: is0, lan966x: is1
  *   Second VID in multiple vlan tags (inner tag)
  * VCAP_KF_8021Q_VID2: W12, sparx5: is0
  *   Third VID in multiple vlan tags (not always available)
  * VCAP_KF_8021Q_VID_CLS: sparx5 is2 W13, sparx5 es0 W13, sparx5 es2 W13,
  *   lan966x is2 W12
  *   Classified VID
- * VCAP_KF_8021Q_VLAN_TAGGED_IS: W1, sparx5: is2/es2, lan966x: is2
+ * VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS: W1, lan966x: is1
+ *   Set if frame has two or more Q-tags. Independent of port VLAN awareness
+ * VCAP_KF_8021Q_VLAN_TAGGED_IS: W1, sparx5: is2/es2, lan966x: is1/is2
  *   Sparx5: Set if frame was received with a VLAN tag, LAN966x: Set if frame has
  *   one or more Q-tags. Independent of port VLAN awareness
  * VCAP_KF_8021Q_VLAN_TAGS: W3, sparx5: is0
@@ -120,9 +134,9 @@ enum vcap_keyfield_set {
  *   Class of service
  * VCAP_KF_ES0_ISDX_KEY_ENA: W1, sparx5: es2
  *   The value taken from the IFH .FWD.ES0_ISDX_KEY_ENA
- * VCAP_KF_ETYPE: W16, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_ETYPE: W16, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Ethernet type
- * VCAP_KF_ETYPE_LEN_IS: W1, sparx5: is0/is2/es2
+ * VCAP_KF_ETYPE_LEN_IS: W1, sparx5: is0/is2/es2, lan966x: is1
  *   Set if frame has EtherType >= 0x600
  * VCAP_KF_HOST_MATCH: W1, lan966x: is2
  *   The action from the SMAC_SIP4 or SMAC_SIP6 lookups. Used for IP source
@@ -134,11 +148,12 @@ enum vcap_keyfield_set {
  *   CPU queue)
  * VCAP_KF_IF_EGR_PORT_NO: W7, sparx5: es0
  *   Egress port number
- * VCAP_KF_IF_IGR_PORT: sparx5 is0 W7, sparx5 es2 W9, lan966x is2 W4
+ * VCAP_KF_IF_IGR_PORT: sparx5 is0 W7, sparx5 es2 W9, lan966x is1 W3, lan966x
+ *   is2 W4
  *   Sparx5: Logical ingress port number retrieved from
  *   ANA_CL::PORT_ID_CFG.LPORT_NUM or ERLEG, LAN966x: ingress port nunmber
  * VCAP_KF_IF_IGR_PORT_MASK: sparx5 is0 W65, sparx5 is2 W32, sparx5 is2 W65,
- *   lan966x is2 W9
+ *   lan966x is1 W9, lan966x is2 W9
  *   Ingress port mask, one bit per port/erleg
  * VCAP_KF_IF_IGR_PORT_MASK_L3: W1, sparx5: is2
  *   If set, IF_IGR_PORT_MASK, IF_IGR_PORT_MASK_RNG, and IF_IGR_PORT_MASK_SEL are
@@ -151,24 +166,26 @@ enum vcap_keyfield_set {
  *   Mapping: 0: DEFAULT 1: LOOPBACK 2: MASQUERADE 3: CPU_VD
  * VCAP_KF_IF_IGR_PORT_SEL: W1, sparx5: es2
  *   Selector for IF_IGR_PORT: physical port number or ERLEG
- * VCAP_KF_IP4_IS: W1, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_IP4_IS: W1, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Set if frame has EtherType = 0x800 and IP version = 4
- * VCAP_KF_IP_MC_IS: W1, sparx5: is0
+ * VCAP_KF_IP_MC_IS: W1, sparx5: is0, lan966x: is1
  *   Set if frame is IPv4 frame and frame's destination MAC address is an IPv4
  *   multicast address (0x01005E0 /25). Set if frame is IPv6 frame and frame's
  *   destination MAC address is an IPv6 multicast address (0x3333/16).
- * VCAP_KF_IP_PAYLOAD_5TUPLE: W32, sparx5: is0
+ * VCAP_KF_IP_PAYLOAD_5TUPLE: W32, sparx5: is0, lan966x: is1
  *   Payload bytes after IP header
- * VCAP_KF_IP_SNAP_IS: W1, sparx5: is0
+ * VCAP_KF_IP_PAYLOAD_S1_IP6: W112, lan966x: is1
+ *   Payload after IPv6 header
+ * VCAP_KF_IP_SNAP_IS: W1, sparx5: is0, lan966x: is1
  *   Set if frame is IPv4, IPv6, or SNAP frame
  * VCAP_KF_ISDX_CLS: W12, sparx5: is2/es0/es2
  *   Classified ISDX
  * VCAP_KF_ISDX_GT0_IS: W1, sparx5: is2/es0/es2, lan966x: is2
  *   Set if classified ISDX > 0
- * VCAP_KF_L2_BC_IS: W1, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L2_BC_IS: W1, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Set if frame's destination MAC address is the broadcast address
  *   (FF-FF-FF-FF-FF-FF).
- * VCAP_KF_L2_DMAC: W48, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L2_DMAC: W48, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Destination MAC address
  * VCAP_KF_L2_FRM_TYPE: W4, lan966x: is2
  *   Frame subtype for specific EtherTypes (MRP, DLR)
@@ -176,7 +193,9 @@ enum vcap_keyfield_set {
  *   Set if the frame is allowed to be forwarded to front ports
  * VCAP_KF_L2_LLC: W40, lan966x: is2
  *   LLC header and data after up to two VLAN tags and the type/length field
- * VCAP_KF_L2_MC_IS: W1, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L2_MAC: W48, lan966x: is1
+ *   MAC address (FIRST=1: SMAC, FIRST=0: DMAC)
+ * VCAP_KF_L2_MC_IS: W1, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Set if frame's destination MAC address is a multicast address (bit 40 = 1).
  * VCAP_KF_L2_PAYLOAD0: W16, lan966x: is2
  *   Payload bytes 0-1 after the frame's EtherType
@@ -188,7 +207,7 @@ enum vcap_keyfield_set {
  *   specifically for PTP frames.
  * VCAP_KF_L2_PAYLOAD_ETYPE: W64, sparx5: is2/es2
  *   Byte 0-7 of L2 payload after Type/Len field and overloading for OAM
- * VCAP_KF_L2_SMAC: W48, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L2_SMAC: W48, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Source MAC address
  * VCAP_KF_L2_SNAP: W40, lan966x: is2
  *   SNAP header after LLC header (AA-AA-03)
@@ -196,32 +215,38 @@ enum vcap_keyfield_set {
  *   Set if Src IP matches Dst IP address
  * VCAP_KF_L3_DPL_CLS: W1, sparx5: es0/es2
  *   The frames drop precedence level
- * VCAP_KF_L3_DSCP: W6, sparx5: is0
+ * VCAP_KF_L3_DSCP: W6, sparx5: is0, lan966x: is1
  *   Frame's DSCP value
  * VCAP_KF_L3_DST_IS: W1, sparx5: is2
  *   Set if lookup is done for egress router leg
- * VCAP_KF_L3_FRAGMENT: W1, lan966x: is2
+ * VCAP_KF_L3_FRAGMENT: W1, lan966x: is1/is2
  *   Set if IPv4 frame is fragmented
  * VCAP_KF_L3_FRAGMENT_TYPE: W2, sparx5: is0/is2/es2
  *   L3 Fragmentation type (none, initial, suspicious, valid follow up)
  * VCAP_KF_L3_FRAG_INVLD_L4_LEN: W1, sparx5: is0/is2
  *   Set if frame's L4 length is less than ANA_CL:COMMON:CLM_FRAGMENT_CFG.L4_MIN_L
  *   EN
- * VCAP_KF_L3_FRAG_OFS_GT0: W1, lan966x: is2
+ * VCAP_KF_L3_FRAG_OFS_GT0: W1, lan966x: is1/is2
  *   Set if IPv4 frame is fragmented and it is not the first fragment
- * VCAP_KF_L3_IP4_DIP: W32, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L3_IP4_DIP: W32, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Destination IPv4 Address
- * VCAP_KF_L3_IP4_SIP: W32, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L3_IP4_SIP: W32, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Source IPv4 Address
- * VCAP_KF_L3_IP6_DIP: W128, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L3_IP6_DIP: sparx5 is0 W128, sparx5 is2 W128, sparx5 es2 W128,
+ *   lan966x is1 W64, lan966x is1 W128, lan966x is2 W128
  *   Sparx5: Full IPv6 DIP, LAN966x: Either Full IPv6 DIP or a subset depending on
  *   frame type
- * VCAP_KF_L3_IP6_SIP: W128, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L3_IP6_DIP_MSB: W16, lan966x: is1
+ *   MS 16bits of IPv6 DIP
+ * VCAP_KF_L3_IP6_SIP: sparx5 is0 W128, sparx5 is2 W128, sparx5 es2 W128,
+ *   lan966x is1 W128, lan966x is1 W64, lan966x is2 W128
  *   Sparx5: Full IPv6 SIP, LAN966x: Either Full IPv6 SIP or a subset depending on
  *   frame type
- * VCAP_KF_L3_IP_PROTO: W8, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L3_IP6_SIP_MSB: W16, lan966x: is1
+ *   MS 16bits of IPv6 DIP
+ * VCAP_KF_L3_IP_PROTO: W8, sparx5: is0/is2/es2, lan966x: is1/is2
  *   IPv4 frames: IP protocol. IPv6 frames: Next header, same as for IPV4
- * VCAP_KF_L3_OPTIONS_IS: W1, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L3_OPTIONS_IS: W1, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Set if IPv4 frame contains options (IP len > 5)
  * VCAP_KF_L3_PAYLOAD: sparx5 is2 W96, sparx5 is2 W40, sparx5 es2 W96, sparx5
  *   es2 W40, lan966x is2 W56
@@ -254,7 +279,8 @@ enum vcap_keyfield_set {
  * VCAP_KF_L4_PSH: W1, sparx5: is2/es2, lan966x: is2
  *   Sparx5: TCP flag PSH, LAN966x: TCP: TCP flag PSH. PTP over UDP: flagField bit
  *   1 (twoStepFlag)
- * VCAP_KF_L4_RNG: sparx5 is0 W8, sparx5 is2 W16, sparx5 es2 W16, lan966x is2 W8
+ * VCAP_KF_L4_RNG: sparx5 is0 W8, sparx5 is2 W16, sparx5 es2 W16, lan966x is1
+ *   W8, lan966x is2 W8
  *   Range checker bitmask (one for each range checker). Input into range checkers
  *   is taken from classified results (VID, DSCP) and frame (SPORT, DPORT, ETYPE,
  *   outer VID, inner VID)
@@ -264,7 +290,7 @@ enum vcap_keyfield_set {
  * VCAP_KF_L4_SEQUENCE_EQ0_IS: W1, sparx5: is2/es2, lan966x: is2
  *   Set if TCP sequence number is 0, LAN966x: Overlayed with PTP over UDP:
  *   messageType bit 0
- * VCAP_KF_L4_SPORT: W16, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_L4_SPORT: W16, sparx5: is0/is2/es2, lan966x: is1/is2
  *   TCP/UDP source port
  * VCAP_KF_L4_SPORT_EQ_DPORT_IS: W1, sparx5: is2/es2, lan966x: is2
  *   Set if UDP or TCP source port equals UDP or TCP destination port
@@ -274,13 +300,16 @@ enum vcap_keyfield_set {
  * VCAP_KF_L4_URG: W1, sparx5: is2/es2, lan966x: is2
  *   Sparx5: TCP flag URG, LAN966x: TCP: TCP flag URG. PTP over UDP: flagField bit
  *   7 (reserved)
- * VCAP_KF_LOOKUP_FIRST_IS: W1, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_LOOKUP_FIRST_IS: W1, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Selects between entries relevant for first and second lookup. Set for first
  *   lookup, cleared for second lookup.
  * VCAP_KF_LOOKUP_GEN_IDX: W12, sparx5: is0
  *   Generic index - for chaining CLM instances
  * VCAP_KF_LOOKUP_GEN_IDX_SEL: W2, sparx5: is0
  *   Select the mode of the Generic Index
+ * VCAP_KF_LOOKUP_INDEX: W2, lan966x: is1
+ *   0: First lookup, 1: Second lookup, 2: Third lookup, Similar to VCAP_KF_FIRST
+ *   but with extra info
  * VCAP_KF_LOOKUP_PAG: W8, sparx5: is2, lan966x: is2
  *   Classified Policy Association Group: chains rules from IS1/CLM to IS2
  * VCAP_KF_MIRROR_PROBE: W2, sparx5: es2
@@ -303,14 +332,22 @@ enum vcap_keyfield_set {
  *   Set if frame's EtherType = 0x8902
  * VCAP_KF_PROT_ACTIVE: W1, sparx5: es0/es2
  *   Protection is active
- * VCAP_KF_TCP_IS: W1, sparx5: is0/is2/es2, lan966x: is2
+ * VCAP_KF_RT_FRMID: W32, lan966x: is1
+ *   Profinet or OPC-UA FrameId
+ * VCAP_KF_RT_TYPE: W2, lan966x: is1
+ *   Encoding of frame's EtherType: 0: Other, 1: Profinet, 2: OPC-UA, 3: Custom
+ *   (ANA::RT_CUSTOM)
+ * VCAP_KF_RT_VLAN_IDX: W3, lan966x: is1
+ *   Real-time VLAN index from ANA::RT_VLAN_PCP
+ * VCAP_KF_TCP_IS: W1, sparx5: is0/is2/es2, lan966x: is1/is2
  *   Set if frame is IPv4 TCP frame (IP protocol = 6) or IPv6 TCP frames (Next
  *   header = 6)
- * VCAP_KF_TCP_UDP_IS: W1, sparx5: is0/is2/es2
+ * VCAP_KF_TCP_UDP_IS: W1, sparx5: is0/is2/es2, lan966x: is1
  *   Set if frame is IPv4/IPv6 TCP or UDP frame (IP protocol/next header equals 6
  *   or 17)
  * VCAP_KF_TYPE: sparx5 is0 W2, sparx5 is0 W1, sparx5 is2 W4, sparx5 is2 W2,
- *   sparx5 es0 W1, sparx5 es2 W3, lan966x is2 W4, lan966x is2 W2
+ *   sparx5 es0 W1, sparx5 es2 W3, lan966x is1 W1, lan966x is1 W2, lan966x is2 W4,
+ *   lan966x is2 W2
  *   Keyset type id - set by the API
  */
 
@@ -323,6 +360,7 @@ enum vcap_key_field {
 	VCAP_KF_8021BR_GRP,
 	VCAP_KF_8021BR_IGR_ECID_BASE,
 	VCAP_KF_8021BR_IGR_ECID_EXT,
+	VCAP_KF_8021CB_R_TAGGED_IS,
 	VCAP_KF_8021Q_DEI0,
 	VCAP_KF_8021Q_DEI1,
 	VCAP_KF_8021Q_DEI2,
@@ -339,6 +377,7 @@ enum vcap_key_field {
 	VCAP_KF_8021Q_VID1,
 	VCAP_KF_8021Q_VID2,
 	VCAP_KF_8021Q_VID_CLS,
+	VCAP_KF_8021Q_VLAN_DBL_TAGGED_IS,
 	VCAP_KF_8021Q_VLAN_TAGGED_IS,
 	VCAP_KF_8021Q_VLAN_TAGS,
 	VCAP_KF_ACL_GRP_ID,
@@ -366,6 +405,7 @@ enum vcap_key_field {
 	VCAP_KF_IP4_IS,
 	VCAP_KF_IP_MC_IS,
 	VCAP_KF_IP_PAYLOAD_5TUPLE,
+	VCAP_KF_IP_PAYLOAD_S1_IP6,
 	VCAP_KF_IP_SNAP_IS,
 	VCAP_KF_ISDX_CLS,
 	VCAP_KF_ISDX_GT0_IS,
@@ -374,6 +414,7 @@ enum vcap_key_field {
 	VCAP_KF_L2_FRM_TYPE,
 	VCAP_KF_L2_FWD_IS,
 	VCAP_KF_L2_LLC,
+	VCAP_KF_L2_MAC,
 	VCAP_KF_L2_MC_IS,
 	VCAP_KF_L2_PAYLOAD0,
 	VCAP_KF_L2_PAYLOAD1,
@@ -392,7 +433,9 @@ enum vcap_key_field {
 	VCAP_KF_L3_IP4_DIP,
 	VCAP_KF_L3_IP4_SIP,
 	VCAP_KF_L3_IP6_DIP,
+	VCAP_KF_L3_IP6_DIP_MSB,
 	VCAP_KF_L3_IP6_SIP,
+	VCAP_KF_L3_IP6_SIP_MSB,
 	VCAP_KF_L3_IP_PROTO,
 	VCAP_KF_L3_OPTIONS_IS,
 	VCAP_KF_L3_PAYLOAD,
@@ -416,6 +459,7 @@ enum vcap_key_field {
 	VCAP_KF_LOOKUP_FIRST_IS,
 	VCAP_KF_LOOKUP_GEN_IDX,
 	VCAP_KF_LOOKUP_GEN_IDX_SEL,
+	VCAP_KF_LOOKUP_INDEX,
 	VCAP_KF_LOOKUP_PAG,
 	VCAP_KF_MIRROR_PROBE,
 	VCAP_KF_OAM_CCM_CNTS_EQ0,
@@ -427,6 +471,9 @@ enum vcap_key_field {
 	VCAP_KF_OAM_VER,
 	VCAP_KF_OAM_Y1731_IS,
 	VCAP_KF_PROT_ACTIVE,
+	VCAP_KF_RT_FRMID,
+	VCAP_KF_RT_TYPE,
+	VCAP_KF_RT_VLAN_IDX,
 	VCAP_KF_TCP_IS,
 	VCAP_KF_TCP_UDP_IS,
 	VCAP_KF_TYPE,
@@ -440,6 +487,7 @@ enum vcap_actionfield_set {
 	VCAP_AFS_CLASS_REDUCED,     /* sparx5 is0 X1 */
 	VCAP_AFS_ES0,               /* sparx5 es0 X1 */
 	VCAP_AFS_FULL,              /* sparx5 is0 X3 */
+	VCAP_AFS_S1,                /* lan966x is1 X1 */
 	VCAP_AFS_SMAC_SIP,          /* lan966x is2 X1 */
 };
 
@@ -470,23 +518,31 @@ enum vcap_actionfield_set {
  *   CPU extraction queue. Used when FWD_SEL >0 and PIPELINE_ACT = XTR.
  * VCAP_AF_CPU_QUEUE_NUM: W3, sparx5: is2/es2, lan966x: is2
  *   CPU queue number. Used when CPU_COPY_ENA is set.
+ * VCAP_AF_CUSTOM_ACE_TYPE_ENA: W4, lan966x: is1
+ *   Enables use of custom keys in IS2. Bits 3:2 control second lookup in IS2
+ *   while bits 1:0 control first lookup. Encoding per lookup: 0: Disabled.  1:
+ *   Extract 40 bytes after position corresponding to the location of the IPv4
+ *   header and use as key.  2: Extract 40 bytes after SMAC and use as key
  * VCAP_AF_DEI_A_VAL: W1, sparx5: es0
  *   DEI used in ES0 tag A. See TAG_A_DEI_SEL.
  * VCAP_AF_DEI_B_VAL: W1, sparx5: es0
  *   DEI used in ES0 tag B. See TAG_B_DEI_SEL.
  * VCAP_AF_DEI_C_VAL: W1, sparx5: es0
  *   DEI used in ES0 tag C. See TAG_C_DEI_SEL.
- * VCAP_AF_DEI_ENA: W1, sparx5: is0
+ * VCAP_AF_DEI_ENA: W1, sparx5: is0, lan966x: is1
  *   If set, use DEI_VAL as classified DEI value. Otherwise, DEI from basic
  *   classification is used
- * VCAP_AF_DEI_VAL: W1, sparx5: is0
+ * VCAP_AF_DEI_VAL: W1, sparx5: is0, lan966x: is1
  *   See DEI_ENA
- * VCAP_AF_DP_ENA: W1, sparx5: is0
+ * VCAP_AF_DLR_SEL: W2, lan966x: is1
+ *   0: No changes to port-based selection in ANA:PORT:OAM_CFG.DLR_ENA.  1: Enable
+ *   DLR frame processing 2: Disable DLR processing
+ * VCAP_AF_DP_ENA: W1, sparx5: is0, lan966x: is1
  *   If set, use DP_VAL as classified drop precedence level. Otherwise, drop
  *   precedence level from basic classification is used.
- * VCAP_AF_DP_VAL: W2, sparx5: is0
+ * VCAP_AF_DP_VAL: sparx5 is0 W2, lan966x is1 W1
  *   See DP_ENA.
- * VCAP_AF_DSCP_ENA: W1, sparx5: is0
+ * VCAP_AF_DSCP_ENA: W1, sparx5: is0, lan966x: is1
  *   If set, use DSCP_VAL as classified DSCP value. Otherwise, DSCP value from
  *   basic classification is used.
  * VCAP_AF_DSCP_SEL: W3, sparx5: es0
@@ -495,7 +551,7 @@ enum vcap_actionfield_set {
  *   table 0, otherwise use DSCP_VAL. 5: Mapped using mapping table 1, otherwise
  *   use mapping table 0. 6: Mapped using mapping table 2, otherwise use DSCP_VAL.
  *   7: Mapped using mapping table 3, otherwise use mapping table 2
- * VCAP_AF_DSCP_VAL: W6, sparx5: is0/es0
+ * VCAP_AF_DSCP_VAL: W6, sparx5: is0/es0, lan966x: is1
  *   See DSCP_ENA.
  * VCAP_AF_ES2_REW_CMD: W3, sparx5: es2
  *   Command forwarded to REW: 0: No action. 1: SWAP MAC addresses. 2: Do L2CP
@@ -529,9 +585,16 @@ enum vcap_actionfield_set {
  * VCAP_AF_ISDX_ADD_REPLACE_SEL: W1, sparx5: is0
  *   Controls the classified ISDX. 0: New ISDX = old ISDX + ISDX_VAL. 1: New ISDX
  *   = ISDX_VAL.
+ * VCAP_AF_ISDX_ADD_VAL: W8, lan966x: is1
+ *   If ISDX_REPLACE_ENA is set, ISDX_ADD_VAL is used directly as the new ISDX.
+ *   Encoding: ISDX_REPLACE_ENA=0, ISDX_ADD_VAL=0: Disabled ISDX_EPLACE_ENA=0,
+ *   ISDX_ADD_VAL>0: Add value to classified ISDX. ISDX_REPLACE_ENA=1: Replace
+ *   with ISDX_ADD_VAL value.
  * VCAP_AF_ISDX_ENA: W1, lan966x: is2
  *   Setting this bit to 1 causes the classified ISDX to be set to the value of
  *   POLICE_IDX[8:0].
+ * VCAP_AF_ISDX_REPLACE_ENA: W1, lan966x: is1
+ *   If set, classified ISDX is set to ISDX_ADD_VAL.
  * VCAP_AF_ISDX_VAL: W12, sparx5: is0
  *   See isdx_add_replace_sel
  * VCAP_AF_LOOP_ENA: W1, sparx5: es0
@@ -572,14 +635,22 @@ enum vcap_actionfield_set {
  * VCAP_AF_MIRROR_PROBE_ID: W2, sparx5: es2
  *   Signals a mirror probe to be placed in the IFH. Only possible when FWD_MODE
  *   is copy. 0: No mirroring. 1-3: Use mirror probe 0-2.
+ * VCAP_AF_MRP_SEL: W2, lan966x: is1
+ *   0: No changes to port-based selection in ANA:PORT:OAM_CFG.MRP_ENA.  1: Enable
+ *   MRP frame processing 2: Disable MRP processing
  * VCAP_AF_NXT_IDX: W12, sparx5: is0
  *   Index used as part of key (field G_IDX) in the next lookup.
  * VCAP_AF_NXT_IDX_CTRL: W3, sparx5: is0
  *   Controls the generation of the G_IDX used in the VCAP CLM next lookup
- * VCAP_AF_PAG_OVERRIDE_MASK: W8, sparx5: is0
+ * VCAP_AF_OAM_SEL: W3, lan966x: is1
+ *   0: No changes to port-based selection in ANA:PORT:OAM_CFG.OAM_CFG 1: Enable
+ *   OAM frame processing for untagged frames 2: Enable OAM frame processing for
+ *   single frames 3: Enable OAM frame processing for double frames 4: Disable OAM
+ *   frame processing
+ * VCAP_AF_PAG_OVERRIDE_MASK: W8, sparx5: is0, lan966x: is1
  *   Bits set in this mask will override PAG_VAL from port profile. New PAG = (PAG
  *   (input) AND ~PAG_OVERRIDE_MASK) OR (PAG_VAL AND PAG_OVERRIDE_MASK)
- * VCAP_AF_PAG_VAL: W8, sparx5: is0
+ * VCAP_AF_PAG_VAL: W8, sparx5: is0, lan966x: is1
  *   See PAG_OVERRIDE_MASK.
  * VCAP_AF_PCP_A_VAL: W3, sparx5: es0
  *   PCP used in ES0 tag A. See TAG_A_PCP_SEL.
@@ -587,10 +658,10 @@ enum vcap_actionfield_set {
  *   PCP used in ES0 tag B. See TAG_B_PCP_SEL.
  * VCAP_AF_PCP_C_VAL: W3, sparx5: es0
  *   PCP used in ES0 tag C. See TAG_C_PCP_SEL.
- * VCAP_AF_PCP_ENA: W1, sparx5: is0
+ * VCAP_AF_PCP_ENA: W1, sparx5: is0, lan966x: is1
  *   If set, use PCP_VAL as classified PCP value. Otherwise, PCP from basic
  *   classification is used.
- * VCAP_AF_PCP_VAL: W3, sparx5: is0
+ * VCAP_AF_PCP_VAL: W3, sparx5: is0, lan966x: is1
  *   See PCP_ENA.
  * VCAP_AF_PIPELINE_ACT: W1, sparx5: es0
  *   Pipeline action when FWD_SEL > 0. 0: XTR. CPU_QU selects CPU extraction queue
@@ -600,11 +671,11 @@ enum vcap_actionfield_set {
  *   PIPELINE_PT == NONE. Overrules previous settings of pipeline point.
  * VCAP_AF_PIPELINE_PT: sparx5 is2 W5, sparx5 es0 W2
  *   Pipeline point used if PIPELINE_FORCE_ENA is set
- * VCAP_AF_POLICE_ENA: W1, sparx5: is2/es2, lan966x: is2
- *   Setting this bit to 1 causes frames that hit this action to be policed by the
- *   ACL policer specified in POLICE_IDX. Only applies to the first lookup.
- * VCAP_AF_POLICE_IDX: sparx5 is2 W6, sparx5 es2 W6, lan966x is2 W9
- *   Selects VCAP policer used when policing frames (POLICE_ENA)
+ * VCAP_AF_POLICE_ENA: W1, sparx5: is2/es2, lan966x: is1/is2
+ *   If set, POLICE_IDX is used to lookup ANA::POL.
+ * VCAP_AF_POLICE_IDX: sparx5 is2 W6, sparx5 es2 W6, lan966x is1 W9, lan966x is2
+ *   W9
+ *   Policer index.
  * VCAP_AF_POLICE_REMARK: W1, sparx5: es2
  *   If set, frames exceeding policer rates are marked as yellow but not
  *   discarded.
@@ -628,16 +699,24 @@ enum vcap_actionfield_set {
  *   port. 1: ES0 tag A: Push ES0 tag A. No port tag. 2: Force port tag: Always
  *   push port tag. No ES0 tag A. 3: Force untag: Never push port tag or ES0 tag
  *   A.
- * VCAP_AF_QOS_ENA: W1, sparx5: is0
+ * VCAP_AF_QOS_ENA: W1, sparx5: is0, lan966x: is1
  *   If set, use QOS_VAL as classified QoS class. Otherwise, QoS class from basic
  *   classification is used.
- * VCAP_AF_QOS_VAL: W3, sparx5: is0
+ * VCAP_AF_QOS_VAL: W3, sparx5: is0, lan966x: is1
  *   See QOS_ENA.
  * VCAP_AF_REW_OP: W16, lan966x: is2
  *   Rewriter operation command.
  * VCAP_AF_RT_DIS: W1, sparx5: is2
  *   If set, routing is disallowed. Only applies when IS_INNER_ACL is 0. See also
  *   IGR_ACL_ENA, EGR_ACL_ENA, and RLEG_STAT_IDX.
+ * VCAP_AF_SFID_ENA: W1, lan966x: is1
+ *   If set, SFID_VAL is used to lookup ANA::SFID.
+ * VCAP_AF_SFID_VAL: W8, lan966x: is1
+ *   Stream filter identifier.
+ * VCAP_AF_SGID_ENA: W1, lan966x: is1
+ *   If set, SGID_VAL is used to lookup ANA::SGID.
+ * VCAP_AF_SGID_VAL: W8, lan966x: is1
+ *   Stream gate identifier.
  * VCAP_AF_SWAP_MACS_ENA: W1, sparx5: es0
  *   This setting is only active when FWD_SEL = 1 or FWD_SEL = 2 and PIPELINE_ACT
  *   = LBK_ASM. 0: No action. 1: Swap MACs and clear bit 40 in new SMAC.
@@ -686,7 +765,7 @@ enum vcap_actionfield_set {
  * VCAP_AF_TAG_C_VID_SEL: W2, sparx5: es0
  *   Selects VID for ES0 tag C. The resulting VID is termed C-TAG.VID. 0:
  *   Classified VID. 1: VID_C_VAL. 2: IFH.ENCAP.GVID. 3: Reserved.
- * VCAP_AF_TYPE: W1, sparx5: is0
+ * VCAP_AF_TYPE: W1, sparx5: is0, lan966x: is1
  *   Actionset type id - Set by the API
  * VCAP_AF_UNTAG_VID_ENA: W1, sparx5: es0
  *   Controls insertion of tag C. Untag or insert mode can be selected. See
@@ -697,8 +776,19 @@ enum vcap_actionfield_set {
  *   VID used in ES0 tag B. See TAG_B_VID_SEL.
  * VCAP_AF_VID_C_VAL: W12, sparx5: es0
  *   VID used in ES0 tag C. See TAG_C_VID_SEL.
- * VCAP_AF_VID_VAL: W13, sparx5: is0
+ * VCAP_AF_VID_REPLACE_ENA: W1, lan966x: is1
+ *   Controls the classified VID: VID_REPLACE_ENA=0: Add VID_ADD_VAL to basic
+ *   classified VID and use result as new classified VID. VID_REPLACE_ENA = 1:
+ *   Replace basic classified VID with VID_VAL value and use as new classified
+ *   VID.
+ * VCAP_AF_VID_VAL: sparx5 is0 W13, lan966x is1 W12
  *   New VID Value
+ * VCAP_AF_VLAN_POP_CNT: W2, lan966x: is1
+ *   See VLAN_POP_CNT_ENA
+ * VCAP_AF_VLAN_POP_CNT_ENA: W1, lan966x: is1
+ *   If set, use VLAN_POP_CNT as the number of VLAN tags to pop from the incoming
+ *   frame. This number is used by the Rewriter. Otherwise, VLAN_POP_CNT from
+ *   ANA:PORT:VLAN_CFG.VLAN_POP_CNT is used
  */
 
 /* Actionfield names */
@@ -712,11 +802,13 @@ enum vcap_action_field {
 	VCAP_AF_CPU_COPY_ENA,
 	VCAP_AF_CPU_QU,
 	VCAP_AF_CPU_QUEUE_NUM,
+	VCAP_AF_CUSTOM_ACE_TYPE_ENA,
 	VCAP_AF_DEI_A_VAL,
 	VCAP_AF_DEI_B_VAL,
 	VCAP_AF_DEI_C_VAL,
 	VCAP_AF_DEI_ENA,
 	VCAP_AF_DEI_VAL,
+	VCAP_AF_DLR_SEL,
 	VCAP_AF_DP_ENA,
 	VCAP_AF_DP_VAL,
 	VCAP_AF_DSCP_ENA,
@@ -732,7 +824,9 @@ enum vcap_action_field {
 	VCAP_AF_IGNORE_PIPELINE_CTRL,
 	VCAP_AF_INTR_ENA,
 	VCAP_AF_ISDX_ADD_REPLACE_SEL,
+	VCAP_AF_ISDX_ADD_VAL,
 	VCAP_AF_ISDX_ENA,
+	VCAP_AF_ISDX_REPLACE_ENA,
 	VCAP_AF_ISDX_VAL,
 	VCAP_AF_LOOP_ENA,
 	VCAP_AF_LRN_DIS,
@@ -745,8 +839,10 @@ enum vcap_action_field {
 	VCAP_AF_MIRROR_ENA,
 	VCAP_AF_MIRROR_PROBE,
 	VCAP_AF_MIRROR_PROBE_ID,
+	VCAP_AF_MRP_SEL,
 	VCAP_AF_NXT_IDX,
 	VCAP_AF_NXT_IDX_CTRL,
+	VCAP_AF_OAM_SEL,
 	VCAP_AF_PAG_OVERRIDE_MASK,
 	VCAP_AF_PAG_VAL,
 	VCAP_AF_PCP_A_VAL,
@@ -770,6 +866,10 @@ enum vcap_action_field {
 	VCAP_AF_QOS_VAL,
 	VCAP_AF_REW_OP,
 	VCAP_AF_RT_DIS,
+	VCAP_AF_SFID_ENA,
+	VCAP_AF_SFID_VAL,
+	VCAP_AF_SGID_ENA,
+	VCAP_AF_SGID_VAL,
 	VCAP_AF_SWAP_MACS_ENA,
 	VCAP_AF_TAG_A_DEI_SEL,
 	VCAP_AF_TAG_A_PCP_SEL,
@@ -788,7 +888,10 @@ enum vcap_action_field {
 	VCAP_AF_VID_A_VAL,
 	VCAP_AF_VID_B_VAL,
 	VCAP_AF_VID_C_VAL,
+	VCAP_AF_VID_REPLACE_ENA,
 	VCAP_AF_VID_VAL,
+	VCAP_AF_VLAN_POP_CNT,
+	VCAP_AF_VLAN_POP_CNT_ENA,
 };
 
 #endif /* __VCAP_AG_API__ */
diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api.c b/drivers/net/ethernet/microchip/vcap/vcap_api.c
index 4847d0d99ec9..5675b0962bc3 100644
--- a/drivers/net/ethernet/microchip/vcap/vcap_api.c
+++ b/drivers/net/ethernet/microchip/vcap/vcap_api.c
@@ -976,6 +976,25 @@ int vcap_lookup_rule_by_cookie(struct vcap_control *vctrl, u64 cookie)
 }
 EXPORT_SYMBOL_GPL(vcap_lookup_rule_by_cookie);
 
+/* Get number of rules in a vcap instance lookup chain id range */
+int vcap_admin_rule_count(struct vcap_admin *admin, int cid)
+{
+	int max_cid = roundup(cid + 1, VCAP_CID_LOOKUP_SIZE);
+	int min_cid = rounddown(cid, VCAP_CID_LOOKUP_SIZE);
+	struct vcap_rule_internal *elem;
+	int count = 0;
+
+	list_for_each_entry(elem, &admin->rules, list) {
+		mutex_lock(&admin->lock);
+		if (elem->data.vcap_chain_id >= min_cid &&
+		    elem->data.vcap_chain_id < max_cid)
+			++count;
+		mutex_unlock(&admin->lock);
+	}
+	return count;
+}
+EXPORT_SYMBOL_GPL(vcap_admin_rule_count);
+
 /* Make a copy of the rule, shallow or full */
 static struct vcap_rule_internal *vcap_dup_rule(struct vcap_rule_internal *ri,
 						bool full)
@@ -3403,6 +3422,25 @@ int vcap_rule_mod_key_u32(struct vcap_rule *rule, enum vcap_key_field key,
 }
 EXPORT_SYMBOL_GPL(vcap_rule_mod_key_u32);
 
+/* Remove a key field with value and mask in the rule */
+int vcap_rule_rem_key(struct vcap_rule *rule, enum vcap_key_field key)
+{
+	struct vcap_rule_internal *ri = to_intrule(rule);
+	struct vcap_client_keyfield *field;
+
+	field = vcap_find_keyfield(rule, key);
+	if (!field) {
+		pr_err("%s:%d: key %s is not in the rule\n",
+		       __func__, __LINE__, vcap_keyfield_name(ri->vctrl, key));
+		return -EINVAL;
+	}
+	/* Deallocate the key field */
+	list_del(&field->ctrl.list);
+	kfree(field);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(vcap_rule_rem_key);
+
 static int vcap_rule_mod_action(struct vcap_rule *rule,
 				enum vcap_action_field action,
 				enum vcap_field_type ftype,
@@ -3475,6 +3513,29 @@ int vcap_filter_rule_keys(struct vcap_rule *rule,
 }
 EXPORT_SYMBOL_GPL(vcap_filter_rule_keys);
 
+/* Select the keyset from the list that results in the smallest rule size */
+enum vcap_keyfield_set
+vcap_select_min_rule_keyset(struct vcap_control *vctrl,
+			    enum vcap_type vtype,
+			    struct vcap_keyset_list *kslist)
+{
+	enum vcap_keyfield_set ret = VCAP_KFS_NO_VALUE;
+	const struct vcap_set *kset;
+	int max = 100, idx;
+
+	for (idx = 0; idx < kslist->cnt; ++idx) {
+		kset = vcap_keyfieldset(vctrl, vtype, kslist->keysets[idx]);
+		if (!kset)
+			continue;
+		if (kset->sw_per_item >= max)
+			continue;
+		max = kset->sw_per_item;
+		ret = kslist->keysets[idx];
+	}
+	return ret;
+}
+EXPORT_SYMBOL_GPL(vcap_select_min_rule_keyset);
+
 /* Make a full copy of an existing rule with a new rule id */
 struct vcap_rule *vcap_copy_rule(struct vcap_rule *erule)
 {
diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_client.h b/drivers/net/ethernet/microchip/vcap/vcap_api_client.h
index 417af9754bcc..d9d1f7c9d762 100644
--- a/drivers/net/ethernet/microchip/vcap/vcap_api_client.h
+++ b/drivers/net/ethernet/microchip/vcap/vcap_api_client.h
@@ -201,6 +201,9 @@ int vcap_rule_add_action_bit(struct vcap_rule *rule,
 int vcap_rule_add_action_u32(struct vcap_rule *rule,
 			     enum vcap_action_field action, u32 value);
 
+/* Get number of rules in a vcap instance lookup chain id range */
+int vcap_admin_rule_count(struct vcap_admin *admin, int cid);
+
 /* VCAP rule counter operations */
 int vcap_get_rule_count_by_cookie(struct vcap_control *vctrl,
 				  struct vcap_counter *ctr, u64 cookie);
@@ -269,6 +272,14 @@ int vcap_rule_mod_action_u32(struct vcap_rule *rule,
 int vcap_rule_get_key_u32(struct vcap_rule *rule, enum vcap_key_field key,
 			  u32 *value, u32 *mask);
 
+/* Remove a key field with value and mask in the rule */
+int vcap_rule_rem_key(struct vcap_rule *rule, enum vcap_key_field key);
+
+/* Select the keyset from the list that results in the smallest rule size */
+enum vcap_keyfield_set
+vcap_select_min_rule_keyset(struct vcap_control *vctrl, enum vcap_type vtype,
+			    struct vcap_keyset_list *kslist);
+
 struct vcap_client_actionfield *
 vcap_find_actionfield(struct vcap_rule *rule, enum vcap_action_field act);
 #endif /* __VCAP_API_CLIENT__ */
diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs_kunit.c b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs_kunit.c
index 0de3f677135a..b23c11b0647c 100644
--- a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs_kunit.c
+++ b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs_kunit.c
@@ -387,7 +387,7 @@ static const char * const test_admin_info_expect[] = {
 	"default_cnt: 73\n",
 	"require_cnt_dis: 0\n",
 	"version: 1\n",
-	"vtype: 3\n",
+	"vtype: 4\n",
 	"vinst: 0\n",
 	"ingress: 1\n",
 	"first_cid: 10000\n",
@@ -435,7 +435,7 @@ static const char * const test_admin_expect[] = {
 	"default_cnt: 73\n",
 	"require_cnt_dis: 0\n",
 	"version: 1\n",
-	"vtype: 3\n",
+	"vtype: 4\n",
 	"vinst: 0\n",
 	"ingress: 1\n",
 	"first_cid: 8000000\n",
diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c
index f9b8f372ec8a..8f3f78b68592 100644
--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c
+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c
@@ -1439,7 +1439,6 @@ free_gc:
 release_region:
 	pci_release_regions(pdev);
 disable_dev:
-	pci_clear_master(pdev);
 	pci_disable_device(pdev);
 	dev_err(&pdev->dev, "gdma probe failed: err = %d\n", err);
 	return err;
@@ -1458,7 +1457,6 @@ static void mana_gd_remove(struct pci_dev *pdev)
 	vfree(gc);
 
 	pci_release_regions(pdev);
-	pci_clear_master(pdev);
 	pci_disable_device(pdev);
 }
 
diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
index 3caea631229c..23b1521c0df9 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
@@ -133,12 +133,6 @@ out:
 	return act;
 }
 
-static unsigned int mana_xdp_fraglen(unsigned int len)
-{
-	return SKB_DATA_ALIGN(len) +
-	       SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
-}
-
 struct bpf_prog *mana_xdp_get(struct mana_port_context *apc)
 {
 	ASSERT_RTNL();
@@ -179,17 +173,18 @@ static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,
 {
 	struct mana_port_context *apc = netdev_priv(ndev);
 	struct bpf_prog *old_prog;
-	int buf_max;
+	struct gdma_context *gc;
+
+	gc = apc->ac->gdma_dev->gdma_context;
 
 	old_prog = mana_xdp_get(apc);
 
 	if (!old_prog && !prog)
 		return 0;
 
-	buf_max = XDP_PACKET_HEADROOM + mana_xdp_fraglen(ndev->mtu + ETH_HLEN);
-	if (prog && buf_max > PAGE_SIZE) {
-		netdev_err(ndev, "XDP: mtu:%u too large, buf_max:%u\n",
-			   ndev->mtu, buf_max);
+	if (prog && ndev->mtu > MANA_XDP_MTU_MAX) {
+		netdev_err(ndev, "XDP: mtu:%u too large, mtu_max:%lu\n",
+			   ndev->mtu, MANA_XDP_MTU_MAX);
 		NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large");
 
 		return -EOPNOTSUPP;
@@ -206,6 +201,11 @@ static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,
 	if (apc->port_is_up)
 		mana_chn_setxdp(apc, prog);
 
+	if (prog)
+		ndev->max_mtu = MANA_XDP_MTU_MAX;
+	else
+		ndev->max_mtu = gc->adapter_mtu - ETH_HLEN;
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index 6120f2b6684f..06d6292e09b3 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -156,6 +156,7 @@ netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 	struct mana_txq *txq;
 	struct mana_cq *cq;
 	int err, len;
+	u16 ihs;
 
 	if (unlikely(!apc->port_is_up))
 		goto tx_drop;
@@ -166,6 +167,7 @@ netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 	txq = &apc->tx_qp[txq_idx].txq;
 	gdma_sq = txq->gdma_sq;
 	cq = &apc->tx_qp[txq_idx].tx_cq;
+	tx_stats = &txq->stats;
 
 	pkg.tx_oob.s_oob.vcq_num = cq->gdma_id;
 	pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame;
@@ -179,10 +181,17 @@ netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 
 	pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt;
 
-	if (pkt_fmt == MANA_SHORT_PKT_FMT)
+	if (pkt_fmt == MANA_SHORT_PKT_FMT) {
 		pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob);
-	else
+		u64_stats_update_begin(&tx_stats->syncp);
+		tx_stats->short_pkt_fmt++;
+		u64_stats_update_end(&tx_stats->syncp);
+	} else {
 		pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob);
+		u64_stats_update_begin(&tx_stats->syncp);
+		tx_stats->long_pkt_fmt++;
+		u64_stats_update_end(&tx_stats->syncp);
+	}
 
 	pkg.wqe_req.inline_oob_data = &pkg.tx_oob;
 	pkg.wqe_req.flags = 0;
@@ -232,9 +241,35 @@ netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 						 &ipv6_hdr(skb)->daddr, 0,
 						 IPPROTO_TCP, 0);
 		}
+
+		if (skb->encapsulation) {
+			ihs = skb_inner_tcp_all_headers(skb);
+			u64_stats_update_begin(&tx_stats->syncp);
+			tx_stats->tso_inner_packets++;
+			tx_stats->tso_inner_bytes += skb->len - ihs;
+			u64_stats_update_end(&tx_stats->syncp);
+		} else {
+			if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
+				ihs = skb_transport_offset(skb) + sizeof(struct udphdr);
+			} else {
+				ihs = skb_tcp_all_headers(skb);
+				if (ipv6_has_hopopt_jumbo(skb))
+					ihs -= sizeof(struct hop_jumbo_hdr);
+			}
+
+			u64_stats_update_begin(&tx_stats->syncp);
+			tx_stats->tso_packets++;
+			tx_stats->tso_bytes += skb->len - ihs;
+			u64_stats_update_end(&tx_stats->syncp);
+		}
+
 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		csum_type = mana_checksum_info(skb);
 
+		u64_stats_update_begin(&tx_stats->syncp);
+		tx_stats->csum_partial++;
+		u64_stats_update_end(&tx_stats->syncp);
+
 		if (csum_type == IPPROTO_TCP) {
 			pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
 			pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
@@ -254,8 +289,12 @@ netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 		}
 	}
 
-	if (mana_map_skb(skb, apc, &pkg))
+	if (mana_map_skb(skb, apc, &pkg)) {
+		u64_stats_update_begin(&tx_stats->syncp);
+		tx_stats->mana_map_err++;
+		u64_stats_update_end(&tx_stats->syncp);
 		goto free_sgl_ptr;
+	}
 
 	skb_queue_tail(&txq->pending_skbs, skb);
 
@@ -388,6 +427,199 @@ static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb,
 	return txq;
 }
 
+/* Release pre-allocated RX buffers */
+static void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc)
+{
+	struct device *dev;
+	int i;
+
+	dev = mpc->ac->gdma_dev->gdma_context->dev;
+
+	if (!mpc->rxbufs_pre)
+		goto out1;
+
+	if (!mpc->das_pre)
+		goto out2;
+
+	while (mpc->rxbpre_total) {
+		i = --mpc->rxbpre_total;
+		dma_unmap_single(dev, mpc->das_pre[i], mpc->rxbpre_datasize,
+				 DMA_FROM_DEVICE);
+		put_page(virt_to_head_page(mpc->rxbufs_pre[i]));
+	}
+
+	kfree(mpc->das_pre);
+	mpc->das_pre = NULL;
+
+out2:
+	kfree(mpc->rxbufs_pre);
+	mpc->rxbufs_pre = NULL;
+
+out1:
+	mpc->rxbpre_datasize = 0;
+	mpc->rxbpre_alloc_size = 0;
+	mpc->rxbpre_headroom = 0;
+}
+
+/* Get a buffer from the pre-allocated RX buffers */
+static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da)
+{
+	struct net_device *ndev = rxq->ndev;
+	struct mana_port_context *mpc;
+	void *va;
+
+	mpc = netdev_priv(ndev);
+
+	if (!mpc->rxbufs_pre || !mpc->das_pre || !mpc->rxbpre_total) {
+		netdev_err(ndev, "No RX pre-allocated bufs\n");
+		return NULL;
+	}
+
+	/* Check sizes to catch unexpected coding error */
+	if (mpc->rxbpre_datasize != rxq->datasize) {
+		netdev_err(ndev, "rxbpre_datasize mismatch: %u: %u\n",
+			   mpc->rxbpre_datasize, rxq->datasize);
+		return NULL;
+	}
+
+	if (mpc->rxbpre_alloc_size != rxq->alloc_size) {
+		netdev_err(ndev, "rxbpre_alloc_size mismatch: %u: %u\n",
+			   mpc->rxbpre_alloc_size, rxq->alloc_size);
+		return NULL;
+	}
+
+	if (mpc->rxbpre_headroom != rxq->headroom) {
+		netdev_err(ndev, "rxbpre_headroom mismatch: %u: %u\n",
+			   mpc->rxbpre_headroom, rxq->headroom);
+		return NULL;
+	}
+
+	mpc->rxbpre_total--;
+
+	*da = mpc->das_pre[mpc->rxbpre_total];
+	va = mpc->rxbufs_pre[mpc->rxbpre_total];
+	mpc->rxbufs_pre[mpc->rxbpre_total] = NULL;
+
+	/* Deallocate the array after all buffers are gone */
+	if (!mpc->rxbpre_total)
+		mana_pre_dealloc_rxbufs(mpc);
+
+	return va;
+}
+
+/* Get RX buffer's data size, alloc size, XDP headroom based on MTU */
+static void mana_get_rxbuf_cfg(int mtu, u32 *datasize, u32 *alloc_size,
+			       u32 *headroom)
+{
+	if (mtu > MANA_XDP_MTU_MAX)
+		*headroom = 0; /* no support for XDP */
+	else
+		*headroom = XDP_PACKET_HEADROOM;
+
+	*alloc_size = mtu + MANA_RXBUF_PAD + *headroom;
+
+	*datasize = ALIGN(mtu + ETH_HLEN, MANA_RX_DATA_ALIGN);
+}
+
+static int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu)
+{
+	struct device *dev;
+	struct page *page;
+	dma_addr_t da;
+	int num_rxb;
+	void *va;
+	int i;
+
+	mana_get_rxbuf_cfg(new_mtu, &mpc->rxbpre_datasize,
+			   &mpc->rxbpre_alloc_size, &mpc->rxbpre_headroom);
+
+	dev = mpc->ac->gdma_dev->gdma_context->dev;
+
+	num_rxb = mpc->num_queues * RX_BUFFERS_PER_QUEUE;
+
+	WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n");
+	mpc->rxbufs_pre = kmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL);
+	if (!mpc->rxbufs_pre)
+		goto error;
+
+	mpc->das_pre = kmalloc_array(num_rxb, sizeof(dma_addr_t), GFP_KERNEL);
+	if (!mpc->das_pre)
+		goto error;
+
+	mpc->rxbpre_total = 0;
+
+	for (i = 0; i < num_rxb; i++) {
+		if (mpc->rxbpre_alloc_size > PAGE_SIZE) {
+			va = netdev_alloc_frag(mpc->rxbpre_alloc_size);
+			if (!va)
+				goto error;
+
+			page = virt_to_head_page(va);
+			/* Check if the frag falls back to single page */
+			if (compound_order(page) <
+			    get_order(mpc->rxbpre_alloc_size)) {
+				put_page(page);
+				goto error;
+			}
+		} else {
+			page = dev_alloc_page();
+			if (!page)
+				goto error;
+
+			va = page_to_virt(page);
+		}
+
+		da = dma_map_single(dev, va + mpc->rxbpre_headroom,
+				    mpc->rxbpre_datasize, DMA_FROM_DEVICE);
+		if (dma_mapping_error(dev, da)) {
+			put_page(virt_to_head_page(va));
+			goto error;
+		}
+
+		mpc->rxbufs_pre[i] = va;
+		mpc->das_pre[i] = da;
+		mpc->rxbpre_total = i + 1;
+	}
+
+	return 0;
+
+error:
+	mana_pre_dealloc_rxbufs(mpc);
+	return -ENOMEM;
+}
+
+static int mana_change_mtu(struct net_device *ndev, int new_mtu)
+{
+	struct mana_port_context *mpc = netdev_priv(ndev);
+	unsigned int old_mtu = ndev->mtu;
+	int err;
+
+	/* Pre-allocate buffers to prevent failure in mana_attach later */
+	err = mana_pre_alloc_rxbufs(mpc, new_mtu);
+	if (err) {
+		netdev_err(ndev, "Insufficient memory for new MTU\n");
+		return err;
+	}
+
+	err = mana_detach(ndev, false);
+	if (err) {
+		netdev_err(ndev, "mana_detach failed: %d\n", err);
+		goto out;
+	}
+
+	ndev->mtu = new_mtu;
+
+	err = mana_attach(ndev);
+	if (err) {
+		netdev_err(ndev, "mana_attach failed: %d\n", err);
+		ndev->mtu = old_mtu;
+	}
+
+out:
+	mana_pre_dealloc_rxbufs(mpc);
+	return err;
+}
+
 static const struct net_device_ops mana_devops = {
 	.ndo_open		= mana_open,
 	.ndo_stop		= mana_close,
@@ -397,6 +629,7 @@ static const struct net_device_ops mana_devops = {
 	.ndo_get_stats64	= mana_get_stats64,
 	.ndo_bpf		= mana_bpf,
 	.ndo_xdp_xmit		= mana_xdp_xmit,
+	.ndo_change_mtu		= mana_change_mtu,
 };
 
 static void mana_cleanup_port_context(struct mana_port_context *apc)
@@ -586,6 +819,9 @@ static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver,
 
 	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG,
 			     sizeof(req), sizeof(resp));
+
+	req.hdr.resp.msg_version = GDMA_MESSAGE_V2;
+
 	req.proto_major_ver = proto_major_ver;
 	req.proto_minor_ver = proto_minor_ver;
 	req.proto_micro_ver = proto_micro_ver;
@@ -608,6 +844,11 @@ static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver,
 
 	*max_num_vports = resp.max_num_vports;
 
+	if (resp.hdr.response.msg_version == GDMA_MESSAGE_V2)
+		gc->adapter_mtu = resp.adapter_mtu;
+	else
+		gc->adapter_mtu = ETH_FRAME_LEN;
+
 	return 0;
 }
 
@@ -1038,6 +1279,8 @@ static void mana_poll_tx_cq(struct mana_cq *cq)
 	if (comp_read < 1)
 		return;
 
+	apc->eth_stats.tx_cqes = comp_read;
+
 	for (i = 0; i < comp_read; i++) {
 		struct mana_tx_comp_oob *cqe_oob;
 
@@ -1064,6 +1307,7 @@ static void mana_poll_tx_cq(struct mana_cq *cq)
 		case CQE_TX_VLAN_TAGGING_VIOLATION:
 			WARN_ONCE(1, "TX: CQE error %d: ignored.\n",
 				  cqe_oob->cqe_hdr.cqe_type);
+			apc->eth_stats.tx_cqe_err++;
 			break;
 
 		default:
@@ -1072,6 +1316,7 @@ static void mana_poll_tx_cq(struct mana_cq *cq)
 			 */
 			WARN_ONCE(1, "TX: Unexpected CQE type %d: HW BUG?\n",
 				  cqe_oob->cqe_hdr.cqe_type);
+			apc->eth_stats.tx_cqe_unknown_type++;
 			return;
 		}
 
@@ -1118,6 +1363,8 @@ static void mana_poll_tx_cq(struct mana_cq *cq)
 		WARN_ON_ONCE(1);
 
 	cq->work_done = pkt_transmitted;
+
+	apc->eth_stats.tx_cqes -= pkt_transmitted;
 }
 
 static void mana_post_pkt_rxq(struct mana_rxq *rxq)
@@ -1140,10 +1387,10 @@ static void mana_post_pkt_rxq(struct mana_rxq *rxq)
 	WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1);
 }
 
-static struct sk_buff *mana_build_skb(void *buf_va, uint pkt_len,
-				      struct xdp_buff *xdp)
+static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va,
+				      uint pkt_len, struct xdp_buff *xdp)
 {
-	struct sk_buff *skb = build_skb(buf_va, PAGE_SIZE);
+	struct sk_buff *skb = napi_build_skb(buf_va, rxq->alloc_size);
 
 	if (!skb)
 		return NULL;
@@ -1151,11 +1398,12 @@ static struct sk_buff *mana_build_skb(void *buf_va, uint pkt_len,
 	if (xdp->data_hard_start) {
 		skb_reserve(skb, xdp->data - xdp->data_hard_start);
 		skb_put(skb, xdp->data_end - xdp->data);
-	} else {
-		skb_reserve(skb, XDP_PACKET_HEADROOM);
-		skb_put(skb, pkt_len);
+		return skb;
 	}
 
+	skb_reserve(skb, rxq->headroom);
+	skb_put(skb, pkt_len);
+
 	return skb;
 }
 
@@ -1188,7 +1436,7 @@ static void mana_rx_skb(void *buf_va, struct mana_rxcomp_oob *cqe,
 	if (act != XDP_PASS && act != XDP_TX)
 		goto drop_xdp;
 
-	skb = mana_build_skb(buf_va, pkt_len, &xdp);
+	skb = mana_build_skb(rxq, buf_va, pkt_len, &xdp);
 
 	if (!skb)
 		goto drop;
@@ -1237,14 +1485,77 @@ drop_xdp:
 	u64_stats_update_end(&rx_stats->syncp);
 
 drop:
-	WARN_ON_ONCE(rxq->xdp_save_page);
-	rxq->xdp_save_page = virt_to_page(buf_va);
+	WARN_ON_ONCE(rxq->xdp_save_va);
+	/* Save for reuse */
+	rxq->xdp_save_va = buf_va;
 
 	++ndev->stats.rx_dropped;
 
 	return;
 }
 
+static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev,
+			     dma_addr_t *da, bool is_napi)
+{
+	struct page *page;
+	void *va;
+
+	/* Reuse XDP dropped page if available */
+	if (rxq->xdp_save_va) {
+		va = rxq->xdp_save_va;
+		rxq->xdp_save_va = NULL;
+	} else if (rxq->alloc_size > PAGE_SIZE) {
+		if (is_napi)
+			va = napi_alloc_frag(rxq->alloc_size);
+		else
+			va = netdev_alloc_frag(rxq->alloc_size);
+
+		if (!va)
+			return NULL;
+
+		page = virt_to_head_page(va);
+		/* Check if the frag falls back to single page */
+		if (compound_order(page) < get_order(rxq->alloc_size)) {
+			put_page(page);
+			return NULL;
+		}
+	} else {
+		page = dev_alloc_page();
+		if (!page)
+			return NULL;
+
+		va = page_to_virt(page);
+	}
+
+	*da = dma_map_single(dev, va + rxq->headroom, rxq->datasize,
+			     DMA_FROM_DEVICE);
+	if (dma_mapping_error(dev, *da)) {
+		put_page(virt_to_head_page(va));
+		return NULL;
+	}
+
+	return va;
+}
+
+/* Allocate frag for rx buffer, and save the old buf */
+static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq,
+			       struct mana_recv_buf_oob *rxoob, void **old_buf)
+{
+	dma_addr_t da;
+	void *va;
+
+	va = mana_get_rxfrag(rxq, dev, &da, true);
+	if (!va)
+		return;
+
+	dma_unmap_single(dev, rxoob->sgl[0].address, rxq->datasize,
+			 DMA_FROM_DEVICE);
+	*old_buf = rxoob->buf_va;
+
+	rxoob->buf_va = va;
+	rxoob->sgl[0].address = da;
+}
+
 static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 				struct gdma_comp *cqe)
 {
@@ -1252,11 +1563,12 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 	struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context;
 	struct net_device *ndev = rxq->ndev;
 	struct mana_recv_buf_oob *rxbuf_oob;
+	struct mana_port_context *apc;
 	struct device *dev = gc->dev;
-	void *new_buf, *old_buf;
-	struct page *new_page;
+	void *old_buf = NULL;
 	u32 curr, pktlen;
-	dma_addr_t da;
+
+	apc = netdev_priv(ndev);
 
 	switch (oob->cqe_hdr.cqe_type) {
 	case CQE_RX_OKAY:
@@ -1270,6 +1582,7 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 
 	case CQE_RX_COALESCED_4:
 		netdev_err(ndev, "RX coalescing is unsupported\n");
+		apc->eth_stats.rx_coalesced_err++;
 		return;
 
 	case CQE_RX_OBJECT_FENCE:
@@ -1279,6 +1592,7 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 	default:
 		netdev_err(ndev, "Unknown RX CQE type = %d\n",
 			   oob->cqe_hdr.cqe_type);
+		apc->eth_stats.rx_cqe_unknown_type++;
 		return;
 	}
 
@@ -1295,40 +1609,11 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 	rxbuf_oob = &rxq->rx_oobs[curr];
 	WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
 
-	/* Reuse XDP dropped page if available */
-	if (rxq->xdp_save_page) {
-		new_page = rxq->xdp_save_page;
-		rxq->xdp_save_page = NULL;
-	} else {
-		new_page = alloc_page(GFP_ATOMIC);
-	}
-
-	if (new_page) {
-		da = dma_map_page(dev, new_page, XDP_PACKET_HEADROOM, rxq->datasize,
-				  DMA_FROM_DEVICE);
-
-		if (dma_mapping_error(dev, da)) {
-			__free_page(new_page);
-			new_page = NULL;
-		}
-	}
-
-	new_buf = new_page ? page_to_virt(new_page) : NULL;
-
-	if (new_buf) {
-		dma_unmap_page(dev, rxbuf_oob->buf_dma_addr, rxq->datasize,
-			       DMA_FROM_DEVICE);
-
-		old_buf = rxbuf_oob->buf_va;
-
-		/* refresh the rxbuf_oob with the new page */
-		rxbuf_oob->buf_va = new_buf;
-		rxbuf_oob->buf_dma_addr = da;
-		rxbuf_oob->sgl[0].address = rxbuf_oob->buf_dma_addr;
-	} else {
-		old_buf = NULL; /* drop the packet if no memory */
-	}
+	mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf);
 
+	/* Unsuccessful refill will have old_buf == NULL.
+	 * In this case, mana_rx_skb() will drop the packet.
+	 */
 	mana_rx_skb(old_buf, oob, rxq);
 
 drop:
@@ -1341,11 +1626,15 @@ static void mana_poll_rx_cq(struct mana_cq *cq)
 {
 	struct gdma_comp *comp = cq->gdma_comp_buf;
 	struct mana_rxq *rxq = cq->rxq;
+	struct mana_port_context *apc;
 	int comp_read, i;
 
+	apc = netdev_priv(rxq->ndev);
+
 	comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER);
 	WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER);
 
+	apc->eth_stats.rx_cqes = comp_read;
 	rxq->xdp_flush = false;
 
 	for (i = 0; i < comp_read; i++) {
@@ -1357,6 +1646,8 @@ static void mana_poll_rx_cq(struct mana_cq *cq)
 			return;
 
 		mana_process_rx_cqe(rxq, cq, &comp[i]);
+
+		apc->eth_stats.rx_cqes--;
 	}
 
 	if (rxq->xdp_flush)
@@ -1603,8 +1894,8 @@ static void mana_destroy_rxq(struct mana_port_context *apc,
 
 	mana_deinit_cq(apc, &rxq->rx_cq);
 
-	if (rxq->xdp_save_page)
-		__free_page(rxq->xdp_save_page);
+	if (rxq->xdp_save_va)
+		put_page(virt_to_head_page(rxq->xdp_save_va));
 
 	for (i = 0; i < rxq->num_rx_buf; i++) {
 		rx_oob = &rxq->rx_oobs[i];
@@ -1612,10 +1903,10 @@ static void mana_destroy_rxq(struct mana_port_context *apc,
 		if (!rx_oob->buf_va)
 			continue;
 
-		dma_unmap_page(dev, rx_oob->buf_dma_addr, rxq->datasize,
-			       DMA_FROM_DEVICE);
+		dma_unmap_single(dev, rx_oob->sgl[0].address,
+				 rx_oob->sgl[0].size, DMA_FROM_DEVICE);
 
-		free_page((unsigned long)rx_oob->buf_va);
+		put_page(virt_to_head_page(rx_oob->buf_va));
 		rx_oob->buf_va = NULL;
 	}
 
@@ -1625,6 +1916,30 @@ static void mana_destroy_rxq(struct mana_port_context *apc,
 	kfree(rxq);
 }
 
+static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key,
+			    struct mana_rxq *rxq, struct device *dev)
+{
+	struct mana_port_context *mpc = netdev_priv(rxq->ndev);
+	dma_addr_t da;
+	void *va;
+
+	if (mpc->rxbufs_pre)
+		va = mana_get_rxbuf_pre(rxq, &da);
+	else
+		va = mana_get_rxfrag(rxq, dev, &da, false);
+
+	if (!va)
+		return -ENOMEM;
+
+	rx_oob->buf_va = va;
+
+	rx_oob->sgl[0].address = da;
+	rx_oob->sgl[0].size = rxq->datasize;
+	rx_oob->sgl[0].mem_key = mem_key;
+
+	return 0;
+}
+
 #define MANA_WQE_HEADER_SIZE 16
 #define MANA_WQE_SGE_SIZE 16
 
@@ -1634,11 +1949,10 @@ static int mana_alloc_rx_wqe(struct mana_port_context *apc,
 	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
 	struct mana_recv_buf_oob *rx_oob;
 	struct device *dev = gc->dev;
-	struct page *page;
-	dma_addr_t da;
 	u32 buf_idx;
+	int ret;
 
-	WARN_ON(rxq->datasize == 0 || rxq->datasize > PAGE_SIZE);
+	WARN_ON(rxq->datasize == 0);
 
 	*rxq_size = 0;
 	*cq_size = 0;
@@ -1647,25 +1961,12 @@ static int mana_alloc_rx_wqe(struct mana_port_context *apc,
 		rx_oob = &rxq->rx_oobs[buf_idx];
 		memset(rx_oob, 0, sizeof(*rx_oob));
 
-		page = alloc_page(GFP_KERNEL);
-		if (!page)
-			return -ENOMEM;
-
-		da = dma_map_page(dev, page, XDP_PACKET_HEADROOM, rxq->datasize,
-				  DMA_FROM_DEVICE);
-
-		if (dma_mapping_error(dev, da)) {
-			__free_page(page);
-			return -ENOMEM;
-		}
-
-		rx_oob->buf_va = page_to_virt(page);
-		rx_oob->buf_dma_addr = da;
-
 		rx_oob->num_sge = 1;
-		rx_oob->sgl[0].address = rx_oob->buf_dma_addr;
-		rx_oob->sgl[0].size = rxq->datasize;
-		rx_oob->sgl[0].mem_key = apc->ac->gdma_dev->gpa_mkey;
+
+		ret = mana_fill_rx_oob(rx_oob, apc->ac->gdma_dev->gpa_mkey, rxq,
+				       dev);
+		if (ret)
+			return ret;
 
 		rx_oob->wqe_req.sgl = rx_oob->sgl;
 		rx_oob->wqe_req.num_sge = rx_oob->num_sge;
@@ -1724,9 +2025,11 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
 	rxq->ndev = ndev;
 	rxq->num_rx_buf = RX_BUFFERS_PER_QUEUE;
 	rxq->rxq_idx = rxq_idx;
-	rxq->datasize = ALIGN(MAX_FRAME_SIZE, 64);
 	rxq->rxobj = INVALID_MANA_HANDLE;
 
+	mana_get_rxbuf_cfg(ndev->mtu, &rxq->datasize, &rxq->alloc_size,
+			   &rxq->headroom);
+
 	err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size);
 	if (err)
 		goto out;
@@ -2138,8 +2441,8 @@ static int mana_probe_port(struct mana_context *ac, int port_idx,
 	ndev->netdev_ops = &mana_devops;
 	ndev->ethtool_ops = &mana_ethtool_ops;
 	ndev->mtu = ETH_DATA_LEN;
-	ndev->max_mtu = ndev->mtu;
-	ndev->min_mtu = ndev->mtu;
+	ndev->max_mtu = gc->adapter_mtu - ETH_HLEN;
+	ndev->min_mtu = ETH_MIN_MTU;
 	ndev->needed_headroom = MANA_HEADROOM;
 	ndev->dev_port = port_idx;
 	SET_NETDEV_DEV(ndev, gc->dev);
diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
index 5b776a33a817..a64c81410dc1 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
@@ -13,6 +13,15 @@ static const struct {
 } mana_eth_stats[] = {
 	{"stop_queue", offsetof(struct mana_ethtool_stats, stop_queue)},
 	{"wake_queue", offsetof(struct mana_ethtool_stats, wake_queue)},
+	{"tx_cqes", offsetof(struct mana_ethtool_stats, tx_cqes)},
+	{"tx_cq_err", offsetof(struct mana_ethtool_stats, tx_cqe_err)},
+	{"tx_cqe_unknown_type", offsetof(struct mana_ethtool_stats,
+					tx_cqe_unknown_type)},
+	{"rx_cqes", offsetof(struct mana_ethtool_stats, rx_cqes)},
+	{"rx_coalesced_err", offsetof(struct mana_ethtool_stats,
+					rx_coalesced_err)},
+	{"rx_cqe_unknown_type", offsetof(struct mana_ethtool_stats,
+					rx_cqe_unknown_type)},
 };
 
 static int mana_get_sset_count(struct net_device *ndev, int stringset)
@@ -23,7 +32,8 @@ static int mana_get_sset_count(struct net_device *ndev, int stringset)
 	if (stringset != ETH_SS_STATS)
 		return -EINVAL;
 
-	return ARRAY_SIZE(mana_eth_stats) + num_queues * 8;
+	return ARRAY_SIZE(mana_eth_stats) + num_queues *
+				(MANA_STATS_RX_COUNT + MANA_STATS_TX_COUNT);
 }
 
 static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
@@ -61,6 +71,22 @@ static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
 		p += ETH_GSTRING_LEN;
 		sprintf(p, "tx_%d_xdp_xmit", i);
 		p += ETH_GSTRING_LEN;
+		sprintf(p, "tx_%d_tso_packets", i);
+		p += ETH_GSTRING_LEN;
+		sprintf(p, "tx_%d_tso_bytes", i);
+		p += ETH_GSTRING_LEN;
+		sprintf(p, "tx_%d_tso_inner_packets", i);
+		p += ETH_GSTRING_LEN;
+		sprintf(p, "tx_%d_tso_inner_bytes", i);
+		p += ETH_GSTRING_LEN;
+		sprintf(p, "tx_%d_long_pkt_fmt", i);
+		p += ETH_GSTRING_LEN;
+		sprintf(p, "tx_%d_short_pkt_fmt", i);
+		p += ETH_GSTRING_LEN;
+		sprintf(p, "tx_%d_csum_partial", i);
+		p += ETH_GSTRING_LEN;
+		sprintf(p, "tx_%d_mana_map_err", i);
+		p += ETH_GSTRING_LEN;
 	}
 }
 
@@ -78,6 +104,14 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 	u64 xdp_xmit;
 	u64 xdp_drop;
 	u64 xdp_tx;
+	u64 tso_packets;
+	u64 tso_bytes;
+	u64 tso_inner_packets;
+	u64 tso_inner_bytes;
+	u64 long_pkt_fmt;
+	u64 short_pkt_fmt;
+	u64 csum_partial;
+	u64 mana_map_err;
 	int q, i = 0;
 
 	if (!apc->port_is_up)
@@ -113,11 +147,27 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 			packets = tx_stats->packets;
 			bytes = tx_stats->bytes;
 			xdp_xmit = tx_stats->xdp_xmit;
+			tso_packets = tx_stats->tso_packets;
+			tso_bytes = tx_stats->tso_bytes;
+			tso_inner_packets = tx_stats->tso_inner_packets;
+			tso_inner_bytes = tx_stats->tso_inner_bytes;
+			long_pkt_fmt = tx_stats->long_pkt_fmt;
+			short_pkt_fmt = tx_stats->short_pkt_fmt;
+			csum_partial = tx_stats->csum_partial;
+			mana_map_err = tx_stats->mana_map_err;
 		} while (u64_stats_fetch_retry(&tx_stats->syncp, start));
 
 		data[i++] = packets;
 		data[i++] = bytes;
 		data[i++] = xdp_xmit;
+		data[i++] = tso_packets;
+		data[i++] = tso_bytes;
+		data[i++] = tso_inner_packets;
+		data[i++] = tso_inner_bytes;
+		data[i++] = long_pkt_fmt;
+		data[i++] = short_pkt_fmt;
+		data[i++] = csum_partial;
+		data[i++] = mana_map_err;
 	}
 }
 
diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c
index 08acb7b89086..1f5f00b30441 100644
--- a/drivers/net/ethernet/mscc/ocelot.c
+++ b/drivers/net/ethernet/mscc/ocelot.c
@@ -7,6 +7,9 @@
 #include <linux/dsa/ocelot.h>
 #include <linux/if_bridge.h>
 #include <linux/iopoll.h>
+#include <linux/phy/phy.h>
+#include <net/pkt_sched.h>
+#include <soc/mscc/ocelot_hsio.h>
 #include <soc/mscc/ocelot_vcap.h>
 #include "ocelot.h"
 #include "ocelot_vcap.h"
@@ -211,6 +214,36 @@ static void ocelot_mact_init(struct ocelot *ocelot)
 	ocelot_write(ocelot, MACACCESS_CMD_INIT, ANA_TABLES_MACACCESS);
 }
 
+void ocelot_pll5_init(struct ocelot *ocelot)
+{
+	/* Configure PLL5. This will need a proper CCF driver
+	 * The values are coming from the VTSS API for Ocelot
+	 */
+	regmap_write(ocelot->targets[HSIO], HSIO_PLL5G_CFG4,
+		     HSIO_PLL5G_CFG4_IB_CTRL(0x7600) |
+		     HSIO_PLL5G_CFG4_IB_BIAS_CTRL(0x8));
+	regmap_write(ocelot->targets[HSIO], HSIO_PLL5G_CFG0,
+		     HSIO_PLL5G_CFG0_CORE_CLK_DIV(0x11) |
+		     HSIO_PLL5G_CFG0_CPU_CLK_DIV(2) |
+		     HSIO_PLL5G_CFG0_ENA_BIAS |
+		     HSIO_PLL5G_CFG0_ENA_VCO_BUF |
+		     HSIO_PLL5G_CFG0_ENA_CP1 |
+		     HSIO_PLL5G_CFG0_SELCPI(2) |
+		     HSIO_PLL5G_CFG0_LOOP_BW_RES(0xe) |
+		     HSIO_PLL5G_CFG0_SELBGV820(4) |
+		     HSIO_PLL5G_CFG0_DIV4 |
+		     HSIO_PLL5G_CFG0_ENA_CLKTREE |
+		     HSIO_PLL5G_CFG0_ENA_LANE);
+	regmap_write(ocelot->targets[HSIO], HSIO_PLL5G_CFG2,
+		     HSIO_PLL5G_CFG2_EN_RESET_FRQ_DET |
+		     HSIO_PLL5G_CFG2_EN_RESET_OVERRUN |
+		     HSIO_PLL5G_CFG2_GAIN_TEST(0x8) |
+		     HSIO_PLL5G_CFG2_ENA_AMPCTRL |
+		     HSIO_PLL5G_CFG2_PWD_AMPCTRL_N |
+		     HSIO_PLL5G_CFG2_AMPC_SEL(0x10));
+}
+EXPORT_SYMBOL(ocelot_pll5_init);
+
 static void ocelot_vcap_enable(struct ocelot *ocelot, int port)
 {
 	ocelot_write_gix(ocelot, ANA_PORT_VCAP_S2_CFG_S2_ENA |
@@ -778,6 +811,71 @@ static int ocelot_port_flush(struct ocelot *ocelot, int port)
 	return err;
 }
 
+int ocelot_port_configure_serdes(struct ocelot *ocelot, int port,
+				 struct device_node *portnp)
+{
+	struct ocelot_port *ocelot_port = ocelot->ports[port];
+	struct device *dev = ocelot->dev;
+	int err;
+
+	/* Ensure clock signals and speed are set on all QSGMII links */
+	if (ocelot_port->phy_mode == PHY_INTERFACE_MODE_QSGMII)
+		ocelot_port_rmwl(ocelot_port, 0,
+				 DEV_CLOCK_CFG_MAC_TX_RST |
+				 DEV_CLOCK_CFG_MAC_RX_RST,
+				 DEV_CLOCK_CFG);
+
+	if (ocelot_port->phy_mode != PHY_INTERFACE_MODE_INTERNAL) {
+		struct phy *serdes = of_phy_get(portnp, NULL);
+
+		if (IS_ERR(serdes)) {
+			err = PTR_ERR(serdes);
+			dev_err_probe(dev, err,
+				      "missing SerDes phys for port %d\n",
+				      port);
+			return err;
+		}
+
+		err = phy_set_mode_ext(serdes, PHY_MODE_ETHERNET,
+				       ocelot_port->phy_mode);
+		of_phy_put(serdes);
+		if (err) {
+			dev_err(dev, "Could not SerDes mode on port %d: %pe\n",
+				port, ERR_PTR(err));
+			return err;
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ocelot_port_configure_serdes);
+
+void ocelot_phylink_mac_config(struct ocelot *ocelot, int port,
+			       unsigned int link_an_mode,
+			       const struct phylink_link_state *state)
+{
+	struct ocelot_port *ocelot_port = ocelot->ports[port];
+
+	/* Disable HDX fast control */
+	ocelot_port_writel(ocelot_port, DEV_PORT_MISC_HDX_FAST_DIS,
+			   DEV_PORT_MISC);
+
+	/* SGMII only for now */
+	ocelot_port_writel(ocelot_port, PCS1G_MODE_CFG_SGMII_MODE_ENA,
+			   PCS1G_MODE_CFG);
+	ocelot_port_writel(ocelot_port, PCS1G_SD_CFG_SD_SEL, PCS1G_SD_CFG);
+
+	/* Enable PCS */
+	ocelot_port_writel(ocelot_port, PCS1G_CFG_PCS_ENA, PCS1G_CFG);
+
+	/* No aneg on SGMII */
+	ocelot_port_writel(ocelot_port, 0, PCS1G_ANEG_CFG);
+
+	/* No loopback */
+	ocelot_port_writel(ocelot_port, 0, PCS1G_LB_CFG);
+}
+EXPORT_SYMBOL_GPL(ocelot_phylink_mac_config);
+
 void ocelot_phylink_mac_link_down(struct ocelot *ocelot, int port,
 				  unsigned int link_an_mode,
 				  phy_interface_t interface,
@@ -908,7 +1006,12 @@ void ocelot_phylink_mac_link_up(struct ocelot *ocelot, int port,
 	 */
 	if (ocelot->ops->cut_through_fwd) {
 		mutex_lock(&ocelot->fwd_domain_lock);
-		ocelot->ops->cut_through_fwd(ocelot);
+		/* Workaround for hardware bug - FP doesn't work
+		 * at all link speeds for all PHY modes. The function
+		 * below also calls ocelot->ops->cut_through_fwd(),
+		 * so we don't need to do it twice.
+		 */
+		ocelot_port_update_active_preemptible_tcs(ocelot, port);
 		mutex_unlock(&ocelot->fwd_domain_lock);
 	}
 
@@ -2602,6 +2705,58 @@ void ocelot_port_mirror_del(struct ocelot *ocelot, int from, bool ingress)
 }
 EXPORT_SYMBOL_GPL(ocelot_port_mirror_del);
 
+static void ocelot_port_reset_mqprio(struct ocelot *ocelot, int port)
+{
+	struct net_device *dev = ocelot->ops->port_to_netdev(ocelot, port);
+
+	netdev_reset_tc(dev);
+	ocelot_port_change_fp(ocelot, port, 0);
+}
+
+int ocelot_port_mqprio(struct ocelot *ocelot, int port,
+		       struct tc_mqprio_qopt_offload *mqprio)
+{
+	struct net_device *dev = ocelot->ops->port_to_netdev(ocelot, port);
+	struct netlink_ext_ack *extack = mqprio->extack;
+	struct tc_mqprio_qopt *qopt = &mqprio->qopt;
+	int num_tc = qopt->num_tc;
+	int tc, err;
+
+	if (!num_tc) {
+		ocelot_port_reset_mqprio(ocelot, port);
+		return 0;
+	}
+
+	err = netdev_set_num_tc(dev, num_tc);
+	if (err)
+		return err;
+
+	for (tc = 0; tc < num_tc; tc++) {
+		if (qopt->count[tc] != 1) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Only one TXQ per TC supported");
+			return -EINVAL;
+		}
+
+		err = netdev_set_tc_queue(dev, tc, 1, qopt->offset[tc]);
+		if (err)
+			goto err_reset_tc;
+	}
+
+	err = netif_set_real_num_tx_queues(dev, num_tc);
+	if (err)
+		goto err_reset_tc;
+
+	ocelot_port_change_fp(ocelot, port, mqprio->preemptible_tcs);
+
+	return 0;
+
+err_reset_tc:
+	ocelot_port_reset_mqprio(ocelot, port);
+	return err;
+}
+EXPORT_SYMBOL_GPL(ocelot_port_mqprio);
+
 void ocelot_init_port(struct ocelot *ocelot, int port)
 {
 	struct ocelot_port *ocelot_port = ocelot->ports[port];
diff --git a/drivers/net/ethernet/mscc/ocelot.h b/drivers/net/ethernet/mscc/ocelot.h
index e9a0179448bf..87f2055c242c 100644
--- a/drivers/net/ethernet/mscc/ocelot.h
+++ b/drivers/net/ethernet/mscc/ocelot.h
@@ -74,6 +74,15 @@ struct ocelot_multicast {
 	struct ocelot_pgid *pgid;
 };
 
+static inline void ocelot_reg_to_target_addr(struct ocelot *ocelot,
+					     enum ocelot_reg reg,
+					     enum ocelot_target *target,
+					     u32 *addr)
+{
+	*target = reg >> TARGET_OFFSET;
+	*addr = ocelot->map[*target][reg & REG_MASK];
+}
+
 int ocelot_bridge_num_find(struct ocelot *ocelot,
 			   const struct net_device *bridge);
 
@@ -85,9 +94,6 @@ int ocelot_mact_forget(struct ocelot *ocelot,
 struct net_device *ocelot_port_to_netdev(struct ocelot *ocelot, int port);
 int ocelot_netdev_to_port(struct net_device *dev);
 
-u32 ocelot_port_readl(struct ocelot_port *port, u32 reg);
-void ocelot_port_writel(struct ocelot_port *port, u32 val, u32 reg);
-
 int ocelot_probe_port(struct ocelot *ocelot, int port, struct regmap *target,
 		      struct device_node *portnp);
 void ocelot_release_port(struct ocelot_port *ocelot_port);
@@ -110,6 +116,9 @@ int ocelot_stats_init(struct ocelot *ocelot);
 void ocelot_stats_deinit(struct ocelot *ocelot);
 
 int ocelot_mm_init(struct ocelot *ocelot);
+void ocelot_port_change_fp(struct ocelot *ocelot, int port,
+			   unsigned long preemptible_tcs);
+void ocelot_port_update_active_preemptible_tcs(struct ocelot *ocelot, int port);
 
 extern struct notifier_block ocelot_netdevice_nb;
 extern struct notifier_block ocelot_switchdev_nb;
diff --git a/drivers/net/ethernet/mscc/ocelot_io.c b/drivers/net/ethernet/mscc/ocelot_io.c
index 2067382d0ee1..3aa7dc29ebe1 100644
--- a/drivers/net/ethernet/mscc/ocelot_io.c
+++ b/drivers/net/ethernet/mscc/ocelot_io.c
@@ -10,57 +10,60 @@
 
 #include "ocelot.h"
 
-int __ocelot_bulk_read_ix(struct ocelot *ocelot, u32 reg, u32 offset, void *buf,
-			  int count)
+int __ocelot_bulk_read_ix(struct ocelot *ocelot, enum ocelot_reg reg,
+			  u32 offset, void *buf, int count)
 {
-	u16 target = reg >> TARGET_OFFSET;
+	enum ocelot_target target;
+	u32 addr;
 
+	ocelot_reg_to_target_addr(ocelot, reg, &target, &addr);
 	WARN_ON(!target);
 
-	return regmap_bulk_read(ocelot->targets[target],
-				ocelot->map[target][reg & REG_MASK] + offset,
+	return regmap_bulk_read(ocelot->targets[target], addr + offset,
 				buf, count);
 }
 EXPORT_SYMBOL_GPL(__ocelot_bulk_read_ix);
 
-u32 __ocelot_read_ix(struct ocelot *ocelot, u32 reg, u32 offset)
+u32 __ocelot_read_ix(struct ocelot *ocelot, enum ocelot_reg reg, u32 offset)
 {
-	u16 target = reg >> TARGET_OFFSET;
-	u32 val;
+	enum ocelot_target target;
+	u32 addr, val;
 
+	ocelot_reg_to_target_addr(ocelot, reg, &target, &addr);
 	WARN_ON(!target);
 
-	regmap_read(ocelot->targets[target],
-		    ocelot->map[target][reg & REG_MASK] + offset, &val);
+	regmap_read(ocelot->targets[target], addr + offset, &val);
 	return val;
 }
 EXPORT_SYMBOL_GPL(__ocelot_read_ix);
 
-void __ocelot_write_ix(struct ocelot *ocelot, u32 val, u32 reg, u32 offset)
+void __ocelot_write_ix(struct ocelot *ocelot, u32 val, enum ocelot_reg reg,
+		       u32 offset)
 {
-	u16 target = reg >> TARGET_OFFSET;
+	enum ocelot_target target;
+	u32 addr;
 
+	ocelot_reg_to_target_addr(ocelot, reg, &target, &addr);
 	WARN_ON(!target);
 
-	regmap_write(ocelot->targets[target],
-		     ocelot->map[target][reg & REG_MASK] + offset, val);
+	regmap_write(ocelot->targets[target], addr + offset, val);
 }
 EXPORT_SYMBOL_GPL(__ocelot_write_ix);
 
-void __ocelot_rmw_ix(struct ocelot *ocelot, u32 val, u32 mask, u32 reg,
-		     u32 offset)
+void __ocelot_rmw_ix(struct ocelot *ocelot, u32 val, u32 mask,
+		     enum ocelot_reg reg, u32 offset)
 {
-	u16 target = reg >> TARGET_OFFSET;
+	enum ocelot_target target;
+	u32 addr;
 
+	ocelot_reg_to_target_addr(ocelot, reg, &target, &addr);
 	WARN_ON(!target);
 
-	regmap_update_bits(ocelot->targets[target],
-			   ocelot->map[target][reg & REG_MASK] + offset,
-			   mask, val);
+	regmap_update_bits(ocelot->targets[target], addr + offset, mask, val);
 }
 EXPORT_SYMBOL_GPL(__ocelot_rmw_ix);
 
-u32 ocelot_port_readl(struct ocelot_port *port, u32 reg)
+u32 ocelot_port_readl(struct ocelot_port *port, enum ocelot_reg reg)
 {
 	struct ocelot *ocelot = port->ocelot;
 	u16 target = reg >> TARGET_OFFSET;
@@ -73,7 +76,7 @@ u32 ocelot_port_readl(struct ocelot_port *port, u32 reg)
 }
 EXPORT_SYMBOL_GPL(ocelot_port_readl);
 
-void ocelot_port_writel(struct ocelot_port *port, u32 val, u32 reg)
+void ocelot_port_writel(struct ocelot_port *port, u32 val, enum ocelot_reg reg)
 {
 	struct ocelot *ocelot = port->ocelot;
 	u16 target = reg >> TARGET_OFFSET;
@@ -84,7 +87,8 @@ void ocelot_port_writel(struct ocelot_port *port, u32 val, u32 reg)
 }
 EXPORT_SYMBOL_GPL(ocelot_port_writel);
 
-void ocelot_port_rmwl(struct ocelot_port *port, u32 val, u32 mask, u32 reg)
+void ocelot_port_rmwl(struct ocelot_port *port, u32 val, u32 mask,
+		      enum ocelot_reg reg)
 {
 	u32 cur = ocelot_port_readl(port, reg);
 
diff --git a/drivers/net/ethernet/mscc/ocelot_mm.c b/drivers/net/ethernet/mscc/ocelot_mm.c
index 0a8f21ae23f0..fb3145118d68 100644
--- a/drivers/net/ethernet/mscc/ocelot_mm.c
+++ b/drivers/net/ethernet/mscc/ocelot_mm.c
@@ -49,14 +49,68 @@ static enum ethtool_mm_verify_status ocelot_mm_verify_status(u32 val)
 	}
 }
 
-void ocelot_port_mm_irq(struct ocelot *ocelot, int port)
+void ocelot_port_update_active_preemptible_tcs(struct ocelot *ocelot, int port)
+{
+	struct ocelot_port *ocelot_port = ocelot->ports[port];
+	struct ocelot_mm_state *mm = &ocelot->mm[port];
+	u32 val = 0;
+
+	lockdep_assert_held(&ocelot->fwd_domain_lock);
+
+	/* Only commit preemptible TCs when MAC Merge is active.
+	 * On NXP LS1028A, when using QSGMII, the port hangs if transmitting
+	 * preemptible frames at any other link speed than gigabit, so avoid
+	 * preemption at lower speeds in this PHY mode.
+	 */
+	if ((ocelot_port->phy_mode != PHY_INTERFACE_MODE_QSGMII ||
+	     ocelot_port->speed == SPEED_1000) && mm->tx_active)
+		val = mm->preemptible_tcs;
+
+	/* Cut through switching doesn't work for preemptible priorities,
+	 * so first make sure it is disabled.
+	 */
+	mm->active_preemptible_tcs = val;
+	ocelot->ops->cut_through_fwd(ocelot);
+
+	dev_dbg(ocelot->dev,
+		"port %d %s/%s, MM TX %s, preemptible TCs 0x%x, active 0x%x\n",
+		port, phy_modes(ocelot_port->phy_mode),
+		phy_speed_to_str(ocelot_port->speed),
+		mm->tx_active ? "active" : "inactive", mm->preemptible_tcs,
+		mm->active_preemptible_tcs);
+
+	ocelot_rmw_rix(ocelot, QSYS_PREEMPTION_CFG_P_QUEUES(val),
+		       QSYS_PREEMPTION_CFG_P_QUEUES_M,
+		       QSYS_PREEMPTION_CFG, port);
+}
+
+void ocelot_port_change_fp(struct ocelot *ocelot, int port,
+			   unsigned long preemptible_tcs)
+{
+	struct ocelot_mm_state *mm = &ocelot->mm[port];
+
+	mutex_lock(&ocelot->fwd_domain_lock);
+
+	if (mm->preemptible_tcs == preemptible_tcs)
+		goto out_unlock;
+
+	mm->preemptible_tcs = preemptible_tcs;
+
+	ocelot_port_update_active_preemptible_tcs(ocelot, port);
+
+out_unlock:
+	mutex_unlock(&ocelot->fwd_domain_lock);
+}
+
+static void ocelot_mm_update_port_status(struct ocelot *ocelot, int port)
 {
 	struct ocelot_port *ocelot_port = ocelot->ports[port];
 	struct ocelot_mm_state *mm = &ocelot->mm[port];
 	enum ethtool_mm_verify_status verify_status;
-	u32 val;
+	u32 val, ack = 0;
 
-	mutex_lock(&mm->lock);
+	if (!mm->tx_enabled)
+		return;
 
 	val = ocelot_port_readl(ocelot_port, DEV_MM_STATUS);
 
@@ -73,25 +127,43 @@ void ocelot_port_mm_irq(struct ocelot *ocelot, int port)
 
 		dev_dbg(ocelot->dev, "Port %d TX preemption %s\n",
 			port, mm->tx_active ? "active" : "inactive");
+		ocelot_port_update_active_preemptible_tcs(ocelot, port);
+
+		ack |= DEV_MM_STAT_MM_STATUS_PRMPT_ACTIVE_STICKY;
 	}
 
 	if (val & DEV_MM_STAT_MM_STATUS_UNEXP_RX_PFRM_STICKY) {
 		dev_err(ocelot->dev,
 			"Unexpected P-frame received on port %d while verification was unsuccessful or not yet verified\n",
 			port);
+
+		ack |= DEV_MM_STAT_MM_STATUS_UNEXP_RX_PFRM_STICKY;
 	}
 
 	if (val & DEV_MM_STAT_MM_STATUS_UNEXP_TX_PFRM_STICKY) {
 		dev_err(ocelot->dev,
 			"Unexpected P-frame requested to be transmitted on port %d while verification was unsuccessful or not yet verified, or MM_TX_ENA=0\n",
 			port);
+
+		ack |= DEV_MM_STAT_MM_STATUS_UNEXP_TX_PFRM_STICKY;
 	}
 
-	ocelot_port_writel(ocelot_port, val, DEV_MM_STATUS);
+	if (ack)
+		ocelot_port_writel(ocelot_port, ack, DEV_MM_STATUS);
+}
 
-	mutex_unlock(&mm->lock);
+void ocelot_mm_irq(struct ocelot *ocelot)
+{
+	int port;
+
+	mutex_lock(&ocelot->fwd_domain_lock);
+
+	for (port = 0; port < ocelot->num_phys_ports; port++)
+		ocelot_mm_update_port_status(ocelot, port);
+
+	mutex_unlock(&ocelot->fwd_domain_lock);
 }
-EXPORT_SYMBOL_GPL(ocelot_port_mm_irq);
+EXPORT_SYMBOL_GPL(ocelot_mm_irq);
 
 int ocelot_port_set_mm(struct ocelot *ocelot, int port,
 		       struct ethtool_mm_cfg *cfg,
@@ -121,7 +193,7 @@ int ocelot_port_set_mm(struct ocelot *ocelot, int port,
 	if (!cfg->verify_enabled)
 		verify_disable = DEV_MM_CONFIG_VERIF_CONFIG_PRM_VERIFY_DIS;
 
-	mutex_lock(&mm->lock);
+	mutex_lock(&ocelot->fwd_domain_lock);
 
 	ocelot_port_rmwl(ocelot_port, mm_enable,
 			 DEV_MM_CONFIG_ENABLE_CONFIG_MM_TX_ENA |
@@ -140,7 +212,20 @@ int ocelot_port_set_mm(struct ocelot *ocelot, int port,
 		       QSYS_PREEMPTION_CFG,
 		       port);
 
-	mutex_unlock(&mm->lock);
+	/* The switch will emit an IRQ when TX is disabled, to notify that it
+	 * has become inactive. We optimize ocelot_mm_update_port_status() to
+	 * not bother processing MM IRQs at all for ports with TX disabled,
+	 * but we need to ACK this IRQ now, while mm->tx_enabled is still set,
+	 * otherwise we get an IRQ storm.
+	 */
+	if (mm->tx_enabled && !cfg->tx_enabled) {
+		ocelot_mm_update_port_status(ocelot, port);
+		WARN_ON(mm->tx_active);
+	}
+
+	mm->tx_enabled = cfg->tx_enabled;
+
+	mutex_unlock(&ocelot->fwd_domain_lock);
 
 	return 0;
 }
@@ -158,7 +243,7 @@ int ocelot_port_get_mm(struct ocelot *ocelot, int port,
 
 	mm = &ocelot->mm[port];
 
-	mutex_lock(&mm->lock);
+	mutex_lock(&ocelot->fwd_domain_lock);
 
 	val = ocelot_port_readl(ocelot_port, DEV_MM_ENABLE_CONFIG);
 	state->pmac_enabled = !!(val & DEV_MM_CONFIG_ENABLE_CONFIG_MM_RX_ENA);
@@ -174,10 +259,11 @@ int ocelot_port_get_mm(struct ocelot *ocelot, int port,
 	state->tx_min_frag_size = ethtool_mm_frag_size_add_to_min(add_frag_size);
 	state->rx_min_frag_size = ETH_ZLEN;
 
+	ocelot_mm_update_port_status(ocelot, port);
 	state->verify_status = mm->verify_status;
 	state->tx_active = mm->tx_active;
 
-	mutex_unlock(&mm->lock);
+	mutex_unlock(&ocelot->fwd_domain_lock);
 
 	return 0;
 }
@@ -201,7 +287,6 @@ int ocelot_mm_init(struct ocelot *ocelot)
 		u32 val;
 
 		mm = &ocelot->mm[port];
-		mutex_init(&mm->lock);
 		ocelot_port = ocelot->ports[port];
 
 		/* Update initial status variable for the
diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c
index ca4bde861397..21a87a3fc556 100644
--- a/drivers/net/ethernet/mscc/ocelot_net.c
+++ b/drivers/net/ethernet/mscc/ocelot_net.c
@@ -1675,25 +1675,10 @@ static void vsc7514_phylink_mac_config(struct phylink_config *config,
 {
 	struct net_device *ndev = to_net_dev(config->dev);
 	struct ocelot_port_private *priv = netdev_priv(ndev);
-	struct ocelot_port *ocelot_port = &priv->port;
-
-	/* Disable HDX fast control */
-	ocelot_port_writel(ocelot_port, DEV_PORT_MISC_HDX_FAST_DIS,
-			   DEV_PORT_MISC);
-
-	/* SGMII only for now */
-	ocelot_port_writel(ocelot_port, PCS1G_MODE_CFG_SGMII_MODE_ENA,
-			   PCS1G_MODE_CFG);
-	ocelot_port_writel(ocelot_port, PCS1G_SD_CFG_SD_SEL, PCS1G_SD_CFG);
-
-	/* Enable PCS */
-	ocelot_port_writel(ocelot_port, PCS1G_CFG_PCS_ENA, PCS1G_CFG);
-
-	/* No aneg on SGMII */
-	ocelot_port_writel(ocelot_port, 0, PCS1G_ANEG_CFG);
+	struct ocelot *ocelot = priv->port.ocelot;
+	int port = priv->port.index;
 
-	/* No loopback */
-	ocelot_port_writel(ocelot_port, 0, PCS1G_LB_CFG);
+	ocelot_phylink_mac_config(ocelot, port, link_an_mode, state);
 }
 
 static void vsc7514_phylink_mac_link_down(struct phylink_config *config,
@@ -1757,34 +1742,11 @@ static int ocelot_port_phylink_create(struct ocelot *ocelot, int port,
 		return -EINVAL;
 	}
 
-	/* Ensure clock signals and speed are set on all QSGMII links */
-	if (phy_mode == PHY_INTERFACE_MODE_QSGMII)
-		ocelot_port_rmwl(ocelot_port, 0,
-				 DEV_CLOCK_CFG_MAC_TX_RST |
-				 DEV_CLOCK_CFG_MAC_RX_RST,
-				 DEV_CLOCK_CFG);
-
 	ocelot_port->phy_mode = phy_mode;
 
-	if (phy_mode != PHY_INTERFACE_MODE_INTERNAL) {
-		struct phy *serdes = of_phy_get(portnp, NULL);
-
-		if (IS_ERR(serdes)) {
-			err = PTR_ERR(serdes);
-			dev_err_probe(dev, err,
-				      "missing SerDes phys for port %d\n",
-				      port);
-			return err;
-		}
-
-		err = phy_set_mode_ext(serdes, PHY_MODE_ETHERNET, phy_mode);
-		of_phy_put(serdes);
-		if (err) {
-			dev_err(dev, "Could not SerDes mode on port %d: %pe\n",
-				port, ERR_PTR(err));
-			return err;
-		}
-	}
+	err = ocelot_port_configure_serdes(ocelot, port, portnp);
+	if (err)
+		return err;
 
 	priv = container_of(ocelot_port, struct ocelot_port_private, port);
 
diff --git a/drivers/net/ethernet/mscc/ocelot_stats.c b/drivers/net/ethernet/mscc/ocelot_stats.c
index d0e6cd8dbe5c..5c55197c7327 100644
--- a/drivers/net/ethernet/mscc/ocelot_stats.c
+++ b/drivers/net/ethernet/mscc/ocelot_stats.c
@@ -145,7 +145,7 @@ enum ocelot_stat {
 };
 
 struct ocelot_stat_layout {
-	u32 reg;
+	enum ocelot_reg reg;
 	char name[ETH_GSTRING_LEN];
 };
 
@@ -257,7 +257,7 @@ struct ocelot_stat_layout {
 
 struct ocelot_stats_region {
 	struct list_head node;
-	u32 base;
+	enum ocelot_reg base;
 	enum ocelot_stat first_stat;
 	int count;
 	u32 *buf;
@@ -395,7 +395,7 @@ static void ocelot_check_stats_work(struct work_struct *work)
 void ocelot_get_strings(struct ocelot *ocelot, int port, u32 sset, u8 *data)
 {
 	const struct ocelot_stat_layout *layout;
-	int i;
+	enum ocelot_stat i;
 
 	if (sset != ETH_SS_STATS)
 		return;
@@ -442,7 +442,8 @@ out_unlock:
 int ocelot_get_sset_count(struct ocelot *ocelot, int port, int sset)
 {
 	const struct ocelot_stat_layout *layout;
-	int i, num_stats = 0;
+	enum ocelot_stat i;
+	int num_stats = 0;
 
 	if (sset != ETH_SS_STATS)
 		return -EOPNOTSUPP;
@@ -461,8 +462,8 @@ static void ocelot_port_ethtool_stats_cb(struct ocelot *ocelot, int port,
 					 void *priv)
 {
 	const struct ocelot_stat_layout *layout;
+	enum ocelot_stat i;
 	u64 *data = priv;
-	int i;
 
 	layout = ocelot_get_stats_layout(ocelot);
 
@@ -889,8 +890,8 @@ static int ocelot_prepare_stats_regions(struct ocelot *ocelot)
 {
 	struct ocelot_stats_region *region = NULL;
 	const struct ocelot_stat_layout *layout;
-	unsigned int last = 0;
-	int i;
+	enum ocelot_reg last = 0;
+	enum ocelot_stat i;
 
 	INIT_LIST_HEAD(&ocelot->stats_regions);
 
@@ -900,6 +901,17 @@ static int ocelot_prepare_stats_regions(struct ocelot *ocelot)
 		if (!layout[i].reg)
 			continue;
 
+		/* enum ocelot_stat must be kept sorted in the same order
+		 * as the addresses behind layout[i].reg in order to have
+		 * efficient bulking
+		 */
+		if (last) {
+			WARN(ocelot->map[SYS][last & REG_MASK] >= ocelot->map[SYS][layout[i].reg & REG_MASK],
+			     "reg 0x%x had address 0x%x but reg 0x%x has address 0x%x, bulking broken!",
+			     last, ocelot->map[SYS][last & REG_MASK],
+			     layout[i].reg, ocelot->map[SYS][layout[i].reg & REG_MASK]);
+		}
+
 		if (region && ocelot->map[SYS][layout[i].reg & REG_MASK] ==
 		    ocelot->map[SYS][last & REG_MASK] + 4) {
 			region->count++;
@@ -909,12 +921,6 @@ static int ocelot_prepare_stats_regions(struct ocelot *ocelot)
 			if (!region)
 				return -ENOMEM;
 
-			/* enum ocelot_stat must be kept sorted in the same
-			 * order as layout[i].reg in order to have efficient
-			 * bulking
-			 */
-			WARN_ON(last >= layout[i].reg);
-
 			region->base = layout[i].reg;
 			region->first_stat = i;
 			region->count = 1;
@@ -925,6 +931,15 @@ static int ocelot_prepare_stats_regions(struct ocelot *ocelot)
 	}
 
 	list_for_each_entry(region, &ocelot->stats_regions, node) {
+		enum ocelot_target target;
+		u32 addr;
+
+		ocelot_reg_to_target_addr(ocelot, region->base, &target,
+					  &addr);
+
+		dev_dbg(ocelot->dev,
+			"region of %d contiguous counters starting with SYS:STAT:CNT[0x%03x]\n",
+			region->count, addr / 4);
 		region->buf = devm_kcalloc(ocelot->dev, region->count,
 					   sizeof(*region->buf), GFP_KERNEL);
 		if (!region->buf)
@@ -972,4 +987,3 @@ void ocelot_stats_deinit(struct ocelot *ocelot)
 	cancel_delayed_work(&ocelot->stats_work);
 	destroy_workqueue(ocelot->stats_queue);
 }
-
diff --git a/drivers/net/ethernet/mscc/ocelot_vsc7514.c b/drivers/net/ethernet/mscc/ocelot_vsc7514.c
index 7388c3b0535c..97e90e2869d4 100644
--- a/drivers/net/ethernet/mscc/ocelot_vsc7514.c
+++ b/drivers/net/ethernet/mscc/ocelot_vsc7514.c
@@ -18,7 +18,6 @@
 
 #include <soc/mscc/ocelot.h>
 #include <soc/mscc/ocelot_vcap.h>
-#include <soc/mscc/ocelot_hsio.h>
 #include <soc/mscc/vsc7514_regs.h>
 #include "ocelot_fdma.h"
 #include "ocelot.h"
@@ -26,35 +25,6 @@
 #define VSC7514_VCAP_POLICER_BASE			128
 #define VSC7514_VCAP_POLICER_MAX			191
 
-static void ocelot_pll5_init(struct ocelot *ocelot)
-{
-	/* Configure PLL5. This will need a proper CCF driver
-	 * The values are coming from the VTSS API for Ocelot
-	 */
-	regmap_write(ocelot->targets[HSIO], HSIO_PLL5G_CFG4,
-		     HSIO_PLL5G_CFG4_IB_CTRL(0x7600) |
-		     HSIO_PLL5G_CFG4_IB_BIAS_CTRL(0x8));
-	regmap_write(ocelot->targets[HSIO], HSIO_PLL5G_CFG0,
-		     HSIO_PLL5G_CFG0_CORE_CLK_DIV(0x11) |
-		     HSIO_PLL5G_CFG0_CPU_CLK_DIV(2) |
-		     HSIO_PLL5G_CFG0_ENA_BIAS |
-		     HSIO_PLL5G_CFG0_ENA_VCO_BUF |
-		     HSIO_PLL5G_CFG0_ENA_CP1 |
-		     HSIO_PLL5G_CFG0_SELCPI(2) |
-		     HSIO_PLL5G_CFG0_LOOP_BW_RES(0xe) |
-		     HSIO_PLL5G_CFG0_SELBGV820(4) |
-		     HSIO_PLL5G_CFG0_DIV4 |
-		     HSIO_PLL5G_CFG0_ENA_CLKTREE |
-		     HSIO_PLL5G_CFG0_ENA_LANE);
-	regmap_write(ocelot->targets[HSIO], HSIO_PLL5G_CFG2,
-		     HSIO_PLL5G_CFG2_EN_RESET_FRQ_DET |
-		     HSIO_PLL5G_CFG2_EN_RESET_OVERRUN |
-		     HSIO_PLL5G_CFG2_GAIN_TEST(0x8) |
-		     HSIO_PLL5G_CFG2_ENA_AMPCTRL |
-		     HSIO_PLL5G_CFG2_PWD_AMPCTRL_N |
-		     HSIO_PLL5G_CFG2_AMPC_SEL(0x10));
-}
-
 static int ocelot_chip_init(struct ocelot *ocelot, const struct ocelot_ops *ops)
 {
 	int ret;
diff --git a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c
index c0dcce8ae437..b1f026b81dea 100644
--- a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c
+++ b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c
@@ -269,7 +269,7 @@ static void set_sha2_512hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
 static int nfp_net_xfrm_add_state(struct xfrm_state *x,
 				  struct netlink_ext_ack *extack)
 {
-	struct net_device *netdev = x->xso.dev;
+	struct net_device *netdev = x->xso.real_dev;
 	struct nfp_ipsec_cfg_mssg msg = {};
 	int i, key_len, trunc_len, err = 0;
 	struct nfp_ipsec_cfg_add_sa *cfg;
@@ -513,7 +513,7 @@ static void nfp_net_xfrm_del_state(struct xfrm_state *x)
 		.cmd = NFP_IPSEC_CFG_MSSG_INV_SA,
 		.sa_idx = x->xso.offload_handle - 1,
 	};
-	struct net_device *netdev = x->xso.dev;
+	struct net_device *netdev = x->xso.real_dev;
 	struct nfp_net *nn;
 	int err;
 
diff --git a/drivers/net/ethernet/netronome/nfp/flower/conntrack.c b/drivers/net/ethernet/netronome/nfp/flower/conntrack.c
index d23830b5bcb8..73032173ac4e 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/conntrack.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/conntrack.c
@@ -55,9 +55,21 @@ static void *get_hashentry(struct rhashtable *ht, void *key,
 
 bool is_pre_ct_flow(struct flow_cls_offload *flow)
 {
+	struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
+	struct flow_dissector *dissector = rule->match.dissector;
 	struct flow_action_entry *act;
+	struct flow_match_ct ct;
 	int i;
 
+	if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) {
+		flow_rule_match_ct(rule, &ct);
+		if (ct.key->ct_state)
+			return false;
+	}
+
+	if (flow->common.chain_index)
+		return false;
+
 	flow_action_for_each(i, act, &flow->rule->action) {
 		if (act->id == FLOW_ACTION_CT) {
 			/* The pre_ct rule only have the ct or ct nat action, cannot
@@ -82,24 +94,23 @@ bool is_post_ct_flow(struct flow_cls_offload *flow)
 	struct flow_match_ct ct;
 	int i;
 
-	/* post ct entry cannot contains any ct action except ct_clear. */
-	flow_action_for_each(i, act, &flow->rule->action) {
-		if (act->id == FLOW_ACTION_CT) {
-			/* ignore ct clear action. */
-			if (act->ct.action == TCA_CT_ACT_CLEAR) {
-				exist_ct_clear = true;
-				continue;
-			}
-
-			return false;
-		}
-	}
-
 	if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) {
 		flow_rule_match_ct(rule, &ct);
 		if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)
 			return true;
 	} else {
+		/* post ct entry cannot contains any ct action except ct_clear. */
+		flow_action_for_each(i, act, &flow->rule->action) {
+			if (act->id == FLOW_ACTION_CT) {
+				/* ignore ct clear action. */
+				if (act->ct.action == TCA_CT_ACT_CLEAR) {
+					exist_ct_clear = true;
+					continue;
+				}
+
+				return false;
+			}
+		}
 		/* when do nat with ct, the post ct entry ignore the ct status,
 		 * will match the nat field(sip/dip) instead. In this situation,
 		 * the flow chain index is not zero and contains ct clear action.
@@ -511,6 +522,21 @@ static int nfp_ct_check_vlan_merge(struct flow_action_entry *a_in,
 	return 0;
 }
 
+/* Extra check for multiple ct-zones merge
+ * currently surpport nft entries merge check in different zones
+ */
+static int nfp_ct_merge_extra_check(struct nfp_fl_ct_flow_entry *nft_entry,
+				    struct nfp_fl_ct_tc_merge *tc_m_entry)
+{
+	struct nfp_fl_nft_tc_merge *prev_nft_m_entry;
+	struct nfp_fl_ct_flow_entry *pre_ct_entry;
+
+	pre_ct_entry = tc_m_entry->pre_ct_parent;
+	prev_nft_m_entry = pre_ct_entry->prev_m_entries[pre_ct_entry->num_prev_m_entries - 1];
+
+	return nfp_ct_merge_check(prev_nft_m_entry->nft_parent, nft_entry);
+}
+
 static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
 				  struct nfp_fl_ct_flow_entry *post_ct_entry,
 				  struct nfp_fl_ct_flow_entry *nft_entry)
@@ -682,34 +708,34 @@ static void nfp_fl_get_csum_flag(struct flow_action_entry *a_in, u8 ip_proto, u3
 static int nfp_fl_merge_actions_offload(struct flow_rule **rules,
 					struct nfp_flower_priv *priv,
 					struct net_device *netdev,
-					struct nfp_fl_payload *flow_pay)
+					struct nfp_fl_payload *flow_pay,
+					int num_rules)
 {
 	enum flow_action_hw_stats tmp_stats = FLOW_ACTION_HW_STATS_DONT_CARE;
 	struct flow_action_entry *a_in;
-	int i, j, num_actions, id;
+	int i, j, id, num_actions = 0;
 	struct flow_rule *a_rule;
 	int err = 0, offset = 0;
 
-	num_actions = rules[CT_TYPE_PRE_CT]->action.num_entries +
-		      rules[CT_TYPE_NFT]->action.num_entries +
-		      rules[CT_TYPE_POST_CT]->action.num_entries;
+	for (i = 0; i < num_rules; i++)
+		num_actions += rules[i]->action.num_entries;
 
 	/* Add one action to make sure there is enough room to add an checksum action
 	 * when do nat.
 	 */
-	a_rule = flow_rule_alloc(num_actions + 1);
+	a_rule = flow_rule_alloc(num_actions + (num_rules / 2));
 	if (!a_rule)
 		return -ENOMEM;
 
-	/* Actions need a BASIC dissector. */
-	a_rule->match = rules[CT_TYPE_PRE_CT]->match;
 	/* post_ct entry have one action at least. */
-	if (rules[CT_TYPE_POST_CT]->action.num_entries != 0) {
-		tmp_stats = rules[CT_TYPE_POST_CT]->action.entries[0].hw_stats;
-	}
+	if (rules[num_rules - 1]->action.num_entries != 0)
+		tmp_stats = rules[num_rules - 1]->action.entries[0].hw_stats;
+
+	/* Actions need a BASIC dissector. */
+	a_rule->match = rules[0]->match;
 
 	/* Copy actions */
-	for (j = 0; j < _CT_TYPE_MAX; j++) {
+	for (j = 0; j < num_rules; j++) {
 		u32 csum_updated = 0;
 		u8 ip_proto = 0;
 
@@ -747,8 +773,9 @@ static int nfp_fl_merge_actions_offload(struct flow_rule **rules,
 				/* nft entry is generated by tc ct, which mangle action do not care
 				 * the stats, inherit the post entry stats to meet the
 				 * flow_action_hw_stats_check.
+				 * nft entry flow rules are at odd array index.
 				 */
-				if (j == CT_TYPE_NFT) {
+				if (j & 0x01) {
 					if (a_in->hw_stats == FLOW_ACTION_HW_STATS_DONT_CARE)
 						a_in->hw_stats = tmp_stats;
 					nfp_fl_get_csum_flag(a_in, ip_proto, &csum_updated);
@@ -784,32 +811,40 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 {
 	enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE;
 	struct nfp_fl_ct_zone_entry *zt = m_entry->zt;
+	struct flow_rule *rules[NFP_MAX_ENTRY_RULES];
+	struct nfp_fl_ct_flow_entry *pre_ct_entry;
 	struct nfp_fl_key_ls key_layer, tmp_layer;
 	struct nfp_flower_priv *priv = zt->priv;
 	u16 key_map[_FLOW_PAY_LAYERS_MAX];
 	struct nfp_fl_payload *flow_pay;
-
-	struct flow_rule *rules[_CT_TYPE_MAX];
 	u8 *key, *msk, *kdata, *mdata;
 	struct nfp_port *port = NULL;
+	int num_rules, err, i, j = 0;
 	struct net_device *netdev;
 	bool qinq_sup;
 	u32 port_id;
 	u16 offset;
-	int i, err;
 
 	netdev = m_entry->netdev;
 	qinq_sup = !!(priv->flower_ext_feats & NFP_FL_FEATS_VLAN_QINQ);
 
-	rules[CT_TYPE_PRE_CT] = m_entry->tc_m_parent->pre_ct_parent->rule;
-	rules[CT_TYPE_NFT] = m_entry->nft_parent->rule;
-	rules[CT_TYPE_POST_CT] = m_entry->tc_m_parent->post_ct_parent->rule;
+	pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
+	num_rules = pre_ct_entry->num_prev_m_entries * 2 + _CT_TYPE_MAX;
+
+	for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++) {
+		rules[j++] = pre_ct_entry->prev_m_entries[i]->tc_m_parent->pre_ct_parent->rule;
+		rules[j++] = pre_ct_entry->prev_m_entries[i]->nft_parent->rule;
+	}
+
+	rules[j++] = m_entry->tc_m_parent->pre_ct_parent->rule;
+	rules[j++] = m_entry->nft_parent->rule;
+	rules[j++] = m_entry->tc_m_parent->post_ct_parent->rule;
 
 	memset(&key_layer, 0, sizeof(struct nfp_fl_key_ls));
 	memset(&key_map, 0, sizeof(key_map));
 
 	/* Calculate the resultant key layer and size for offload */
-	for (i = 0; i < _CT_TYPE_MAX; i++) {
+	for (i = 0; i < num_rules; i++) {
 		err = nfp_flower_calculate_key_layers(priv->app,
 						      m_entry->netdev,
 						      &tmp_layer, rules[i],
@@ -875,7 +910,7 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 	 * that the layer is not present.
 	 */
 	if (!qinq_sup) {
-		for (i = 0; i < _CT_TYPE_MAX; i++) {
+		for (i = 0; i < num_rules; i++) {
 			offset = key_map[FLOW_PAY_META_TCI];
 			key = kdata + offset;
 			msk = mdata + offset;
@@ -889,7 +924,7 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 		offset = key_map[FLOW_PAY_MAC_MPLS];
 		key = kdata + offset;
 		msk = mdata + offset;
-		for (i = 0; i < _CT_TYPE_MAX; i++) {
+		for (i = 0; i < num_rules; i++) {
 			nfp_flower_compile_mac((struct nfp_flower_mac_mpls *)key,
 					       (struct nfp_flower_mac_mpls *)msk,
 					       rules[i]);
@@ -905,7 +940,7 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 		offset = key_map[FLOW_PAY_IPV4];
 		key = kdata + offset;
 		msk = mdata + offset;
-		for (i = 0; i < _CT_TYPE_MAX; i++) {
+		for (i = 0; i < num_rules; i++) {
 			nfp_flower_compile_ipv4((struct nfp_flower_ipv4 *)key,
 						(struct nfp_flower_ipv4 *)msk,
 						rules[i]);
@@ -916,7 +951,7 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 		offset = key_map[FLOW_PAY_IPV6];
 		key = kdata + offset;
 		msk = mdata + offset;
-		for (i = 0; i < _CT_TYPE_MAX; i++) {
+		for (i = 0; i < num_rules; i++) {
 			nfp_flower_compile_ipv6((struct nfp_flower_ipv6 *)key,
 						(struct nfp_flower_ipv6 *)msk,
 						rules[i]);
@@ -927,7 +962,7 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 		offset = key_map[FLOW_PAY_L4];
 		key = kdata + offset;
 		msk = mdata + offset;
-		for (i = 0; i < _CT_TYPE_MAX; i++) {
+		for (i = 0; i < num_rules; i++) {
 			nfp_flower_compile_tport((struct nfp_flower_tp_ports *)key,
 						 (struct nfp_flower_tp_ports *)msk,
 						 rules[i]);
@@ -938,7 +973,7 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 		offset = key_map[FLOW_PAY_QINQ];
 		key = kdata + offset;
 		msk = mdata + offset;
-		for (i = 0; i < _CT_TYPE_MAX; i++) {
+		for (i = 0; i < num_rules; i++) {
 			nfp_flower_compile_vlan((struct nfp_flower_vlan *)key,
 						(struct nfp_flower_vlan *)msk,
 						rules[i]);
@@ -954,7 +989,7 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 			struct nfp_ipv6_addr_entry *entry;
 			struct in6_addr *dst;
 
-			for (i = 0; i < _CT_TYPE_MAX; i++) {
+			for (i = 0; i < num_rules; i++) {
 				nfp_flower_compile_ipv6_gre_tun((void *)key,
 								(void *)msk, rules[i]);
 			}
@@ -971,7 +1006,7 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 		} else {
 			__be32 dst;
 
-			for (i = 0; i < _CT_TYPE_MAX; i++) {
+			for (i = 0; i < num_rules; i++) {
 				nfp_flower_compile_ipv4_gre_tun((void *)key,
 								(void *)msk, rules[i]);
 			}
@@ -995,7 +1030,7 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 			struct nfp_ipv6_addr_entry *entry;
 			struct in6_addr *dst;
 
-			for (i = 0; i < _CT_TYPE_MAX; i++) {
+			for (i = 0; i < num_rules; i++) {
 				nfp_flower_compile_ipv6_udp_tun((void *)key,
 								(void *)msk, rules[i]);
 			}
@@ -1012,7 +1047,7 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 		} else {
 			__be32 dst;
 
-			for (i = 0; i < _CT_TYPE_MAX; i++) {
+			for (i = 0; i < num_rules; i++) {
 				nfp_flower_compile_ipv4_udp_tun((void *)key,
 								(void *)msk, rules[i]);
 			}
@@ -1029,13 +1064,13 @@ static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 			offset = key_map[FLOW_PAY_GENEVE_OPT];
 			key = kdata + offset;
 			msk = mdata + offset;
-			for (i = 0; i < _CT_TYPE_MAX; i++)
+			for (i = 0; i < num_rules; i++)
 				nfp_flower_compile_geneve_opt(key, msk, rules[i]);
 		}
 	}
 
 	/* Merge actions into flow_pay */
-	err = nfp_fl_merge_actions_offload(rules, priv, netdev, flow_pay);
+	err = nfp_fl_merge_actions_offload(rules, priv, netdev, flow_pay, num_rules);
 	if (err)
 		goto ct_offload_err;
 
@@ -1168,6 +1203,12 @@ static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
 	if (err)
 		return err;
 
+	if (pre_ct_entry->num_prev_m_entries > 0) {
+		err = nfp_ct_merge_extra_check(nft_entry, tc_m_entry);
+		if (err)
+			return err;
+	}
+
 	/* Combine tc_merge and nft cookies for this cookie. */
 	new_cookie[0] = tc_m_entry->cookie[0];
 	new_cookie[1] = tc_m_entry->cookie[1];
@@ -1198,11 +1239,6 @@ static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
 	list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children);
 	list_add(&nft_m_entry->nft_flow_list, &nft_entry->children);
 
-	/* Generate offload structure and send to nfp */
-	err = nfp_fl_ct_add_offload(nft_m_entry);
-	if (err)
-		goto err_nft_ct_offload;
-
 	err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node,
 				     nfp_nft_ct_merge_params);
 	if (err)
@@ -1210,12 +1246,20 @@ static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
 
 	zt->nft_merge_count++;
 
+	if (post_ct_entry->goto_chain_index > 0)
+		return nfp_fl_create_new_pre_ct(nft_m_entry);
+
+	/* Generate offload structure and send to nfp */
+	err = nfp_fl_ct_add_offload(nft_m_entry);
+	if (err)
+		goto err_nft_ct_offload;
+
 	return err;
 
-err_nft_ct_merge_insert:
+err_nft_ct_offload:
 	nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie,
 			      nft_m_entry->netdev);
-err_nft_ct_offload:
+err_nft_ct_merge_insert:
 	list_del(&nft_m_entry->tc_merge_list);
 	list_del(&nft_m_entry->nft_flow_list);
 	kfree(nft_m_entry);
@@ -1243,7 +1287,7 @@ static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt,
 	/* Checks that the chain_index of the filter matches the
 	 * chain_index of the GOTO action.
 	 */
-	if (post_ct_entry->chain_index != pre_ct_entry->chain_index)
+	if (post_ct_entry->chain_index != pre_ct_entry->goto_chain_index)
 		return -EINVAL;
 
 	err = nfp_ct_merge_check(pre_ct_entry, post_ct_entry);
@@ -1461,7 +1505,7 @@ nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
 
 	entry->zt = zt;
 	entry->netdev = netdev;
-	entry->cookie = flow->cookie;
+	entry->cookie = flow->cookie > 0 ? flow->cookie : (unsigned long)entry;
 	entry->chain_index = flow->common.chain_index;
 	entry->tun_offset = NFP_FL_CT_NO_TUN;
 
@@ -1501,6 +1545,9 @@ nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
 
 	INIT_LIST_HEAD(&entry->children);
 
+	if (flow->cookie == 0)
+		return entry;
+
 	/* Now add a ct map entry to flower-priv */
 	map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie,
 			    nfp_ct_map_params, sizeof(*map));
@@ -1559,6 +1606,14 @@ static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry)
 	list_del(&m_entry->tc_merge_list);
 	list_del(&m_entry->nft_flow_list);
 
+	if (m_entry->next_pre_ct_entry) {
+		struct nfp_fl_ct_map_entry pre_ct_map_ent;
+
+		pre_ct_map_ent.ct_entry = m_entry->next_pre_ct_entry;
+		pre_ct_map_ent.cookie = 0;
+		nfp_fl_ct_del_flow(&pre_ct_map_ent);
+	}
+
 	kfree(m_entry);
 }
 
@@ -1656,6 +1711,22 @@ void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry)
 	kfree(entry);
 }
 
+static struct flow_action_entry *get_flow_act_ct(struct flow_rule *rule)
+{
+	struct flow_action_entry *act;
+	int i;
+
+	/* More than one ct action may be present in a flow rule,
+	 * Return the first one that is not a CT clear action
+	 */
+	flow_action_for_each(i, act, &rule->action) {
+		if (act->id == FLOW_ACTION_CT && act->ct.action != TCA_CT_ACT_CLEAR)
+			return act;
+	}
+
+	return NULL;
+}
+
 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
 					      enum flow_action_id act_id)
 {
@@ -1713,14 +1784,15 @@ nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry,
 int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
 			    struct net_device *netdev,
 			    struct flow_cls_offload *flow,
-			    struct netlink_ext_ack *extack)
+			    struct netlink_ext_ack *extack,
+			    struct nfp_fl_nft_tc_merge *m_entry)
 {
 	struct flow_action_entry *ct_act, *ct_goto;
 	struct nfp_fl_ct_flow_entry *ct_entry;
 	struct nfp_fl_ct_zone_entry *zt;
 	int err;
 
-	ct_act = get_flow_act(flow->rule, FLOW_ACTION_CT);
+	ct_act = get_flow_act_ct(flow->rule);
 	if (!ct_act) {
 		NL_SET_ERR_MSG_MOD(extack,
 				   "unsupported offload: Conntrack action empty in conntrack offload");
@@ -1756,7 +1828,22 @@ int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
 	if (IS_ERR(ct_entry))
 		return PTR_ERR(ct_entry);
 	ct_entry->type = CT_TYPE_PRE_CT;
-	ct_entry->chain_index = ct_goto->chain_index;
+	ct_entry->chain_index = flow->common.chain_index;
+	ct_entry->goto_chain_index = ct_goto->chain_index;
+
+	if (m_entry) {
+		struct nfp_fl_ct_flow_entry *pre_ct_entry;
+		int i;
+
+		pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
+		for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++)
+			ct_entry->prev_m_entries[i] = pre_ct_entry->prev_m_entries[i];
+		ct_entry->prev_m_entries[i++] = m_entry;
+		ct_entry->num_prev_m_entries = i;
+
+		m_entry->next_pre_ct_entry = ct_entry;
+	}
+
 	list_add(&ct_entry->list_node, &zt->pre_ct_list);
 	zt->pre_ct_count++;
 
@@ -1779,6 +1866,7 @@ int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
 	struct nfp_fl_ct_zone_entry *zt;
 	bool wildcarded = false;
 	struct flow_match_ct ct;
+	struct flow_action_entry *ct_goto;
 
 	flow_rule_match_ct(rule, &ct);
 	if (!ct.mask->ct_zone) {
@@ -1803,6 +1891,8 @@ int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
 
 	ct_entry->type = CT_TYPE_POST_CT;
 	ct_entry->chain_index = flow->common.chain_index;
+	ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO);
+	ct_entry->goto_chain_index = ct_goto ? ct_goto->chain_index : 0;
 	list_add(&ct_entry->list_node, &zt->post_ct_list);
 	zt->post_ct_count++;
 
@@ -1831,6 +1921,28 @@ int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
 	return 0;
 }
 
+int nfp_fl_create_new_pre_ct(struct nfp_fl_nft_tc_merge *m_entry)
+{
+	struct nfp_fl_ct_flow_entry *pre_ct_entry, *post_ct_entry;
+	struct flow_cls_offload new_pre_ct_flow;
+	int err;
+
+	pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
+	if (pre_ct_entry->num_prev_m_entries >= NFP_MAX_RECIRC_CT_ZONES - 1)
+		return -1;
+
+	post_ct_entry = m_entry->tc_m_parent->post_ct_parent;
+	memset(&new_pre_ct_flow, 0, sizeof(struct flow_cls_offload));
+	new_pre_ct_flow.rule = post_ct_entry->rule;
+	new_pre_ct_flow.common.chain_index = post_ct_entry->chain_index;
+
+	err = nfp_fl_ct_handle_pre_ct(pre_ct_entry->zt->priv,
+				      pre_ct_entry->netdev,
+				      &new_pre_ct_flow, NULL,
+				      m_entry);
+	return err;
+}
+
 static void
 nfp_fl_ct_sub_stats(struct nfp_fl_nft_tc_merge *nft_merge,
 		    enum ct_entry_type type, u64 *m_pkts,
@@ -1876,6 +1988,32 @@ nfp_fl_ct_sub_stats(struct nfp_fl_nft_tc_merge *nft_merge,
 				  0, priv->stats[ctx_id].used,
 				  FLOW_ACTION_HW_STATS_DELAYED);
 	}
+
+	/* Update previous pre_ct/post_ct/nft flow stats */
+	if (nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries > 0) {
+		struct nfp_fl_nft_tc_merge *tmp_nft_merge;
+		int i;
+
+		for (i = 0; i < nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries; i++) {
+			tmp_nft_merge = nft_merge->tc_m_parent->pre_ct_parent->prev_m_entries[i];
+			flow_stats_update(&tmp_nft_merge->tc_m_parent->pre_ct_parent->stats,
+					  priv->stats[ctx_id].bytes,
+					  priv->stats[ctx_id].pkts,
+					  0, priv->stats[ctx_id].used,
+					  FLOW_ACTION_HW_STATS_DELAYED);
+			flow_stats_update(&tmp_nft_merge->tc_m_parent->post_ct_parent->stats,
+					  priv->stats[ctx_id].bytes,
+					  priv->stats[ctx_id].pkts,
+					  0, priv->stats[ctx_id].used,
+					  FLOW_ACTION_HW_STATS_DELAYED);
+			flow_stats_update(&tmp_nft_merge->nft_parent->stats,
+					  priv->stats[ctx_id].bytes,
+					  priv->stats[ctx_id].pkts,
+					  0, priv->stats[ctx_id].used,
+					  FLOW_ACTION_HW_STATS_DELAYED);
+		}
+	}
+
 	/* Reset stats from the nfp */
 	priv->stats[ctx_id].pkts = 0;
 	priv->stats[ctx_id].bytes = 0;
@@ -2080,10 +2218,12 @@ int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent)
 	switch (ct_entry->type) {
 	case CT_TYPE_PRE_CT:
 		zt->pre_ct_count--;
-		rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
-				       nfp_ct_map_params);
+		if (ct_map_ent->cookie > 0)
+			rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
+					       nfp_ct_map_params);
 		nfp_fl_ct_clean_flow_entry(ct_entry);
-		kfree(ct_map_ent);
+		if (ct_map_ent->cookie > 0)
+			kfree(ct_map_ent);
 
 		if (!zt->pre_ct_count) {
 			zt->nft = NULL;
diff --git a/drivers/net/ethernet/netronome/nfp/flower/conntrack.h b/drivers/net/ethernet/netronome/nfp/flower/conntrack.h
index 762c0b36e269..c4ec78358033 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/conntrack.h
+++ b/drivers/net/ethernet/netronome/nfp/flower/conntrack.h
@@ -86,6 +86,9 @@ enum ct_entry_type {
 	_CT_TYPE_MAX,
 };
 
+#define NFP_MAX_RECIRC_CT_ZONES 4
+#define NFP_MAX_ENTRY_RULES  (NFP_MAX_RECIRC_CT_ZONES * 2 + 1)
+
 enum nfp_nfp_layer_name {
 	FLOW_PAY_META_TCI =    0,
 	FLOW_PAY_INPORT,
@@ -112,27 +115,33 @@ enum nfp_nfp_layer_name {
  * @cookie:	Flow cookie, same as original TC flow, used as key
  * @list_node:	Used by the list
  * @chain_index:	Chain index of the original flow
+ * @goto_chain_index:	goto chain index of the flow
  * @netdev:	netdev structure.
- * @type:	Type of pre-entry from enum ct_entry_type
  * @zt:		Reference to the zone table this belongs to
  * @children:	List of tc_merge flows this flow forms part of
  * @rule:	Reference to the original TC flow rule
  * @stats:	Used to cache stats for updating
+ * @prev_m_entries:	Array of all previous nft_tc_merge entries
+ * @num_prev_m_entries:	The number of all previous nft_tc_merge entries
  * @tun_offset: Used to indicate tunnel action offset in action list
  * @flags:	Used to indicate flow flag like NAT which used by merge.
+ * @type:	Type of ct-entry from enum ct_entry_type
  */
 struct nfp_fl_ct_flow_entry {
 	unsigned long cookie;
 	struct list_head list_node;
 	u32 chain_index;
-	enum ct_entry_type type;
+	u32 goto_chain_index;
 	struct net_device *netdev;
 	struct nfp_fl_ct_zone_entry *zt;
 	struct list_head children;
 	struct flow_rule *rule;
 	struct flow_stats stats;
+	struct nfp_fl_nft_tc_merge *prev_m_entries[NFP_MAX_RECIRC_CT_ZONES - 1];
+	u8 num_prev_m_entries;
 	u8 tun_offset;		// Set to NFP_FL_CT_NO_TUN if no tun
 	u8 flags;
+	u8 type;
 };
 
 /**
@@ -169,6 +178,7 @@ struct nfp_fl_ct_tc_merge {
  * @nft_parent:	The nft_entry parent
  * @tc_flower_cookie:	The cookie of the flow offloaded to the nfp
  * @flow_pay:	Reference to the offloaded flow struct
+ * @next_pre_ct_entry:	Reference to the next ct zone pre ct entry
  */
 struct nfp_fl_nft_tc_merge {
 	struct net_device *netdev;
@@ -181,6 +191,7 @@ struct nfp_fl_nft_tc_merge {
 	struct nfp_fl_ct_flow_entry *nft_parent;
 	unsigned long tc_flower_cookie;
 	struct nfp_fl_payload *flow_pay;
+	struct nfp_fl_ct_flow_entry *next_pre_ct_entry;
 };
 
 /**
@@ -204,6 +215,7 @@ bool is_post_ct_flow(struct flow_cls_offload *flow);
  * @netdev:	netdev structure.
  * @flow:	TC flower classifier offload structure.
  * @extack:	Extack pointer for errors
+ * @m_entry:previous nfp_fl_nft_tc_merge entry
  *
  * Adds a new entry to the relevant zone table and tries to
  * merge with other +trk+est entries and offload if possible.
@@ -213,7 +225,8 @@ bool is_post_ct_flow(struct flow_cls_offload *flow);
 int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
 			    struct net_device *netdev,
 			    struct flow_cls_offload *flow,
-			    struct netlink_ext_ack *extack);
+			    struct netlink_ext_ack *extack,
+			    struct nfp_fl_nft_tc_merge *m_entry);
 /**
  * nfp_fl_ct_handle_post_ct() - Handles +trk+est conntrack rules
  * @priv:	Pointer to app priv
@@ -232,6 +245,19 @@ int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
 			     struct netlink_ext_ack *extack);
 
 /**
+ * nfp_fl_create_new_pre_ct() - create next ct_zone -trk conntrack rules
+ * @m_entry:previous nfp_fl_nft_tc_merge entry
+ *
+ * Create a new pre_ct entry from previous nfp_fl_nft_tc_merge entry
+ * to the next relevant zone table. Try to merge with other +trk+est
+ * entries and offload if possible. The created new pre_ct entry is
+ * linked to the previous nfp_fl_nft_tc_merge entry.
+ *
+ * Return: negative value on error, 0 if configured successfully.
+ */
+int nfp_fl_create_new_pre_ct(struct nfp_fl_nft_tc_merge *m_entry);
+
+/**
  * nfp_fl_ct_clean_flow_entry() - Free a nfp_fl_ct_flow_entry
  * @entry:	Flow entry to cleanup
  */
diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c
index 8593cafa6368..18328eb7f5c3 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c
@@ -1344,7 +1344,7 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
 		port = nfp_port_from_netdev(netdev);
 
 	if (is_pre_ct_flow(flow))
-		return nfp_fl_ct_handle_pre_ct(priv, netdev, flow, extack);
+		return nfp_fl_ct_handle_pre_ct(priv, netdev, flow, extack, NULL);
 
 	if (is_post_ct_flow(flow))
 		return nfp_fl_ct_handle_post_ct(priv, netdev, flow, extack);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_hwmon.c b/drivers/net/ethernet/netronome/nfp/nfp_hwmon.c
index 5cabb1aa9c0c..0d6c59d6d4ae 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_hwmon.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_hwmon.c
@@ -115,7 +115,7 @@ static const struct hwmon_channel_info nfp_power = {
 	.config = nfp_power_config,
 };
 
-static const struct hwmon_channel_info *nfp_hwmon_info[] = {
+static const struct hwmon_channel_info * const nfp_hwmon_info[] = {
 	&nfp_chip,
 	&nfp_temp,
 	&nfp_power,
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.c b/drivers/net/ethernet/netronome/nfp/nfp_port.c
index 4f2308570dcf..54640bcb70fb 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_port.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_port.c
@@ -189,6 +189,7 @@ int nfp_port_init_phy_port(struct nfp_pf *pf, struct nfp_app *app,
 
 	port->eth_port = &pf->eth_tbl->ports[id];
 	port->eth_id = pf->eth_tbl->ports[id].index;
+	port->netdev->dev_port = id;
 	if (pf->mac_stats_mem)
 		port->eth_stats =
 			pf->mac_stats_mem + port->eth_id * NFP_MAC_STATS_SIZE;
diff --git a/drivers/net/ethernet/ni/nixge.c b/drivers/net/ethernet/ni/nixge.c
index 56e02cba0b8a..0fd156286d4d 100644
--- a/drivers/net/ethernet/ni/nixge.c
+++ b/drivers/net/ethernet/ni/nixge.c
@@ -1422,7 +1422,7 @@ static struct platform_driver nixge_driver = {
 	.remove		= nixge_remove,
 	.driver		= {
 		.name		= "nixge",
-		.of_match_table	= of_match_ptr(nixge_dt_ids),
+		.of_match_table	= nixge_dt_ids,
 	},
 };
 module_platform_driver(nixge_driver);
diff --git a/drivers/net/ethernet/pasemi/pasemi_mac.c b/drivers/net/ethernet/pasemi/pasemi_mac.c
index aaab590ef548..ed7dd0a04235 100644
--- a/drivers/net/ethernet/pasemi/pasemi_mac.c
+++ b/drivers/net/ethernet/pasemi/pasemi_mac.c
@@ -1423,7 +1423,7 @@ static void pasemi_mac_queue_csdesc(const struct sk_buff *skb,
 	write_dma_reg(PAS_DMA_TXCHAN_INCR(txring->chan.chno), 2);
 }
 
-static int pasemi_mac_start_tx(struct sk_buff *skb, struct net_device *dev)
+static netdev_tx_t pasemi_mac_start_tx(struct sk_buff *skb, struct net_device *dev)
 {
 	struct pasemi_mac * const mac = netdev_priv(dev);
 	struct pasemi_mac_txring * const txring = tx_ring(mac);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
index e508f8eb43bf..b8678da1cce5 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
@@ -392,7 +392,6 @@ static void ionic_remove(struct pci_dev *pdev)
 	ionic_port_reset(ionic);
 	ionic_reset(ionic);
 	ionic_dev_teardown(ionic);
-	pci_clear_master(pdev);
 	ionic_unmap_bars(ionic);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_phc.c b/drivers/net/ethernet/pensando/ionic/ionic_phc.c
index eac2f0e3576e..7505efdff8e9 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_phc.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_phc.c
@@ -579,11 +579,10 @@ void ionic_lif_alloc_phc(struct ionic_lif *lif)
 	diff |= diff >> 16;
 	diff |= diff >> 32;
 
-	/* constrain to the hardware bitmask, and use this as the bitmask */
+	/* constrain to the hardware bitmask */
 	diff &= phc->cc.mask;
-	phc->cc.mask = diff;
 
-	/* the wrap period is now defined by diff (or phc->cc.mask)
+	/* the wrap period is now defined by diff
 	 *
 	 * we will update the time basis at about 1/4 the wrap period, so
 	 * should not see a difference of more than +/- diff/4.
diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic.h b/drivers/net/ethernet/qlogic/netxen/netxen_nic.h
index f13fa7396aef..3d36d23df0c6 100644
--- a/drivers/net/ethernet/qlogic/netxen/netxen_nic.h
+++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic.h
@@ -854,7 +854,7 @@ typedef struct {
 	   The following is packed:
 	   - N cardrsp_rds_rings
 	   - N cardrs_sds_rings */
-	char data[0];
+	char data[];
 } nx_cardrsp_rx_ctx_t;
 
 #define SIZEOF_HOSTRQ_RX(HOSTRQ_RX, rds_rings, sds_rings)	\
diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
index de8d54b23f73..1d1e183d3a8b 100644
--- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
+++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
@@ -18,7 +18,6 @@
 #include <linux/ipv6.h>
 #include <linux/inetdevice.h>
 #include <linux/sysfs.h>
-#include <linux/aer.h>
 
 MODULE_DESCRIPTION("QLogic/NetXen (1/10) GbE Intelligent Ethernet Driver");
 MODULE_LICENSE("GPL");
@@ -1464,9 +1463,6 @@ netxen_nic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if ((err = pci_request_regions(pdev, netxen_nic_driver_name)))
 		goto err_out_disable_pdev;
 
-	if (NX_IS_REVISION_P3(pdev->revision))
-		pci_enable_pcie_error_reporting(pdev);
-
 	pci_set_master(pdev);
 
 	netdev = alloc_etherdev(sizeof(struct netxen_adapter));
@@ -1603,8 +1599,6 @@ err_out_free_netdev:
 	free_netdev(netdev);
 
 err_out_free_res:
-	if (NX_IS_REVISION_P3(pdev->revision))
-		pci_disable_pcie_error_reporting(pdev);
 	pci_release_regions(pdev);
 
 err_out_disable_pdev:
@@ -1659,10 +1653,8 @@ static void netxen_nic_remove(struct pci_dev *pdev)
 
 	netxen_release_firmware(adapter);
 
-	if (NX_IS_REVISION_P3(pdev->revision)) {
+	if (NX_IS_REVISION_P3(pdev->revision))
 		netxen_cleanup_minidump(adapter);
-		pci_disable_pcie_error_reporting(pdev);
-	}
 
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
@@ -1862,7 +1854,7 @@ netxen_tso_check(struct net_device *netdev,
 
 	if (protocol == cpu_to_be16(ETH_P_8021Q)) {
 
-		vh = (struct vlan_ethhdr *)skb->data;
+		vh = skb_vlan_eth_hdr(skb);
 		protocol = vh->h_vlan_encapsulated_proto;
 		flags = FLAGS_VLAN_TAGGED;
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.c b/drivers/net/ethernet/qlogic/qed/qed_ll2.c
index e5116a86cfbc..717a0b3f89bd 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_ll2.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.c
@@ -646,13 +646,13 @@ static int qed_ll2_lb_rxq_handler(struct qed_hwfn *p_hwfn,
 	struct qed_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
 	u16 packet_length = 0, parse_flags = 0, vlan = 0;
 	struct qed_ll2_rx_packet *p_pkt = NULL;
-	u32 num_ooo_add_to_peninsula = 0, cid;
 	union core_rx_cqe_union *cqe = NULL;
 	u16 cq_new_idx = 0, cq_old_idx = 0;
 	struct qed_ooo_buffer *p_buffer;
 	struct ooo_opaque *ooo_opq;
 	u8 placement_offset = 0;
 	u8 cqe_type;
+	u32 cid;
 
 	cq_new_idx = le16_to_cpu(*p_rx->p_fw_cons);
 	cq_old_idx = qed_chain_get_cons_idx(&p_rx->rcq_chain);
@@ -762,7 +762,6 @@ static int qed_ll2_lb_rxq_handler(struct qed_hwfn *p_hwfn,
 						   cid, ooo_opq->ooo_isle);
 				break;
 			case TCP_EVENT_ADD_PEN:
-				num_ooo_add_to_peninsula++;
 				qed_ooo_put_ready_buffer(p_hwfn,
 							 p_hwfn->p_ooo_info,
 							 p_buffer, true);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_main.c b/drivers/net/ethernet/qlogic/qed/qed_main.c
index c91898be7c03..f5af83342856 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_main.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_main.c
@@ -23,7 +23,6 @@
 #include <linux/qed/qed_if.h>
 #include <linux/qed/qed_ll2_if.h>
 #include <net/devlink.h>
-#include <linux/aer.h>
 #include <linux/phylink.h>
 
 #include "qed.h"
@@ -259,8 +258,6 @@ static void qed_free_pci(struct qed_dev *cdev)
 {
 	struct pci_dev *pdev = cdev->pdev;
 
-	pci_disable_pcie_error_reporting(pdev);
-
 	if (cdev->doorbells && cdev->db_size)
 		iounmap(cdev->doorbells);
 	if (cdev->regview)
@@ -366,12 +363,6 @@ static int qed_init_pci(struct qed_dev *cdev, struct pci_dev *pdev)
 		return -ENOMEM;
 	}
 
-	/* AER (Advanced Error reporting) configuration */
-	rc = pci_enable_pcie_error_reporting(pdev);
-	if (rc)
-		DP_VERBOSE(cdev, NETIF_MSG_DRV,
-			   "Failed to configure PCIe AER [%d]\n", rc);
-
 	return 0;
 
 err2:
diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h
index f90dcfe9ee68..f9931ecb7baa 100644
--- a/drivers/net/ethernet/qlogic/qede/qede.h
+++ b/drivers/net/ethernet/qlogic/qede/qede.h
@@ -6,8 +6,6 @@
 
 #ifndef _QEDE_H_
 #define _QEDE_H_
-#include <linux/compiler.h>
-#include <linux/version.h>
 #include <linux/workqueue.h>
 #include <linux/netdevice.h>
 #include <linux/interrupt.h>
diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
index 8034d812d5a0..374a86b875a3 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
@@ -4,7 +4,6 @@
  * Copyright (c) 2019-2020 Marvell International Ltd.
  */
 
-#include <linux/version.h>
 #include <linux/types.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index 261f982ca40d..4c6c685820e3 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -35,7 +35,6 @@
 #include <net/ip6_checksum.h>
 #include <linux/bitops.h>
 #include <linux/vmalloc.h>
-#include <linux/aer.h>
 #include "qede.h"
 #include "qede_ptp.h"
 
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
index 2fd5c6fdb500..bcef8ab715bf 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
@@ -8,7 +8,6 @@
 #include <linux/ipv6.h>
 #include <linux/ethtool.h>
 #include <linux/interrupt.h>
-#include <linux/aer.h>
 
 #include "qlcnic.h"
 #include "qlcnic_sriov.h"
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c
index 92930a055cbc..41894d154013 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c
@@ -318,7 +318,7 @@ static void qlcnic_send_filter(struct qlcnic_adapter *adapter,
 
 	if (adapter->flags & QLCNIC_VLAN_FILTERING) {
 		if (protocol == ETH_P_8021Q) {
-			vh = (struct vlan_ethhdr *)skb->data;
+			vh = skb_vlan_eth_hdr(skb);
 			vlan_id = ntohs(vh->h_vlan_TCI);
 		} else if (skb_vlan_tag_present(skb)) {
 			vlan_id = skb_vlan_tag_get(skb);
@@ -468,7 +468,7 @@ static int qlcnic_tx_pkt(struct qlcnic_adapter *adapter,
 	u32 producer = tx_ring->producer;
 
 	if (protocol == ETH_P_8021Q) {
-		vh = (struct vlan_ethhdr *)skb->data;
+		vh = skb_vlan_eth_hdr(skb);
 		flags = QLCNIC_FLAGS_VLAN_TAGGED;
 		vlan_tci = ntohs(vh->h_vlan_TCI);
 		protocol = ntohs(vh->h_vlan_encapsulated_proto);
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
index 44dac3c0908e..90df4a0909fa 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
@@ -12,7 +12,6 @@
 #include <net/ip.h>
 #include <linux/ipv6.h>
 #include <linux/inetdevice.h>
-#include <linux/aer.h>
 #include <linux/log2.h>
 #include <linux/pci.h>
 #include <net/vxlan.h>
@@ -2445,7 +2444,6 @@ qlcnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto err_out_disable_pdev;
 
 	pci_set_master(pdev);
-	pci_enable_pcie_error_reporting(pdev);
 
 	ahw = kzalloc(sizeof(struct qlcnic_hardware_context), GFP_KERNEL);
 	if (!ahw) {
@@ -2675,7 +2673,6 @@ err_out_free_hw_res:
 	kfree(ahw);
 
 err_out_free_res:
-	pci_disable_pcie_error_reporting(pdev);
 	pci_release_regions(pdev);
 
 err_out_disable_pdev:
@@ -2757,7 +2754,6 @@ static void qlcnic_remove(struct pci_dev *pdev)
 
 	qlcnic_release_firmware(adapter);
 
-	pci_disable_pcie_error_reporting(pdev);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
 
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c
index 5c2edb715d3e..74125188beb8 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c
@@ -12,7 +12,6 @@
 #include <linux/ipv6.h>
 #include <linux/inetdevice.h>
 #include <linux/sysfs.h>
-#include <linux/aer.h>
 #include <linux/log2.h>
 #ifdef CONFIG_QLCNIC_HWMON
 #include <linux/hwmon.h>
diff --git a/drivers/net/ethernet/qualcomm/Kconfig b/drivers/net/ethernet/qualcomm/Kconfig
index a4434eb38950..9210ff360fdc 100644
--- a/drivers/net/ethernet/qualcomm/Kconfig
+++ b/drivers/net/ethernet/qualcomm/Kconfig
@@ -52,6 +52,7 @@ config QCOM_EMAC
 	depends on HAS_DMA && HAS_IOMEM
 	select CRC32
 	select PHYLIB
+	select MDIO_DEVRES
 	help
 	  This driver supports the Qualcomm Technologies, Inc. Gigabit
 	  Ethernet Media Access Controller (EMAC). The controller
diff --git a/drivers/net/ethernet/qualcomm/qca_debug.c b/drivers/net/ethernet/qualcomm/qca_debug.c
index f62c39544e08..6f2fa2a42770 100644
--- a/drivers/net/ethernet/qualcomm/qca_debug.c
+++ b/drivers/net/ethernet/qualcomm/qca_debug.c
@@ -119,7 +119,7 @@ qcaspi_info_show(struct seq_file *s, void *what)
 	seq_printf(s, "SPI mode         : %x\n",
 		   qca->spi_dev->mode);
 	seq_printf(s, "SPI chip select  : %u\n",
-		   (unsigned int)qca->spi_dev->chip_select);
+		   (unsigned int)spi_get_chipselect(qca->spi_dev, 0));
 	seq_printf(s, "SPI legacy mode  : %u\n",
 		   (unsigned int)qca->legacy_mode);
 	seq_printf(s, "SPI burst length : %u\n",
diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c
index 45147a1016be..a7e376e7e689 100644
--- a/drivers/net/ethernet/realtek/r8169_main.c
+++ b/drivers/net/ethernet/realtek/r8169_main.c
@@ -30,6 +30,7 @@
 #include <linux/ipv6.h>
 #include <asm/unaligned.h>
 #include <net/ip6_checksum.h>
+#include <net/netdev_queues.h>
 
 #include "r8169.h"
 #include "r8169_firmware.h"
@@ -68,6 +69,8 @@
 #define NUM_RX_DESC	256	/* Number of Rx descriptor registers */
 #define R8169_TX_RING_BYTES	(NUM_TX_DESC * sizeof(struct TxDesc))
 #define R8169_RX_RING_BYTES	(NUM_RX_DESC * sizeof(struct RxDesc))
+#define R8169_TX_STOP_THRS	(MAX_SKB_FRAGS + 1)
+#define R8169_TX_START_THRS	(2 * R8169_TX_STOP_THRS)
 
 #define OCP_STD_PHY_BASE	0xa400
 
@@ -613,8 +616,13 @@ struct rtl8169_private {
 		struct work_struct work;
 	} wk;
 
+	spinlock_t config25_lock;
+	spinlock_t mac_ocp_lock;
+
+	spinlock_t cfg9346_usage_lock;
+	int cfg9346_usage_count;
+
 	unsigned supports_gmii:1;
-	unsigned aspm_manageable:1;
 	dma_addr_t counters_phys_addr;
 	struct rtl8169_counters *counters;
 	struct rtl8169_tc_offsets tc_offset;
@@ -661,12 +669,22 @@ static inline struct device *tp_to_dev(struct rtl8169_private *tp)
 
 static void rtl_lock_config_regs(struct rtl8169_private *tp)
 {
-	RTL_W8(tp, Cfg9346, Cfg9346_Lock);
+	unsigned long flags;
+
+	spin_lock_irqsave(&tp->cfg9346_usage_lock, flags);
+	if (!--tp->cfg9346_usage_count)
+		RTL_W8(tp, Cfg9346, Cfg9346_Lock);
+	spin_unlock_irqrestore(&tp->cfg9346_usage_lock, flags);
 }
 
 static void rtl_unlock_config_regs(struct rtl8169_private *tp)
 {
-	RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
+	unsigned long flags;
+
+	spin_lock_irqsave(&tp->cfg9346_usage_lock, flags);
+	if (!tp->cfg9346_usage_count++)
+		RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
+	spin_unlock_irqrestore(&tp->cfg9346_usage_lock, flags);
 }
 
 static void rtl_pci_commit(struct rtl8169_private *tp)
@@ -675,6 +693,28 @@ static void rtl_pci_commit(struct rtl8169_private *tp)
 	RTL_R8(tp, ChipCmd);
 }
 
+static void rtl_mod_config2(struct rtl8169_private *tp, u8 clear, u8 set)
+{
+	unsigned long flags;
+	u8 val;
+
+	spin_lock_irqsave(&tp->config25_lock, flags);
+	val = RTL_R8(tp, Config2);
+	RTL_W8(tp, Config2, (val & ~clear) | set);
+	spin_unlock_irqrestore(&tp->config25_lock, flags);
+}
+
+static void rtl_mod_config5(struct rtl8169_private *tp, u8 clear, u8 set)
+{
+	unsigned long flags;
+	u8 val;
+
+	spin_lock_irqsave(&tp->config25_lock, flags);
+	val = RTL_R8(tp, Config5);
+	RTL_W8(tp, Config5, (val & ~clear) | set);
+	spin_unlock_irqrestore(&tp->config25_lock, flags);
+}
+
 static bool rtl_is_8125(struct rtl8169_private *tp)
 {
 	return tp->mac_version >= RTL_GIGA_MAC_VER_61;
@@ -847,7 +887,7 @@ static int r8168_phy_ocp_read(struct rtl8169_private *tp, u32 reg)
 		(RTL_R32(tp, GPHY_OCP) & 0xffff) : -ETIMEDOUT;
 }
 
-static void r8168_mac_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
+static void __r8168_mac_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
 {
 	if (rtl_ocp_reg_failure(reg))
 		return;
@@ -855,7 +895,16 @@ static void r8168_mac_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
 	RTL_W32(tp, OCPDR, OCPAR_FLAG | (reg << 15) | data);
 }
 
-static u16 r8168_mac_ocp_read(struct rtl8169_private *tp, u32 reg)
+static void r8168_mac_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&tp->mac_ocp_lock, flags);
+	__r8168_mac_ocp_write(tp, reg, data);
+	spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
+}
+
+static u16 __r8168_mac_ocp_read(struct rtl8169_private *tp, u32 reg)
 {
 	if (rtl_ocp_reg_failure(reg))
 		return 0;
@@ -865,12 +914,28 @@ static u16 r8168_mac_ocp_read(struct rtl8169_private *tp, u32 reg)
 	return RTL_R32(tp, OCPDR);
 }
 
+static u16 r8168_mac_ocp_read(struct rtl8169_private *tp, u32 reg)
+{
+	unsigned long flags;
+	u16 val;
+
+	spin_lock_irqsave(&tp->mac_ocp_lock, flags);
+	val = __r8168_mac_ocp_read(tp, reg);
+	spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
+
+	return val;
+}
+
 static void r8168_mac_ocp_modify(struct rtl8169_private *tp, u32 reg, u16 mask,
 				 u16 set)
 {
-	u16 data = r8168_mac_ocp_read(tp, reg);
+	unsigned long flags;
+	u16 data;
 
-	r8168_mac_ocp_write(tp, reg, (data & ~mask) | set);
+	spin_lock_irqsave(&tp->mac_ocp_lock, flags);
+	data = __r8168_mac_ocp_read(tp, reg);
+	__r8168_mac_ocp_write(tp, reg, (data & ~mask) | set);
+	spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
 }
 
 /* Work around a hw issue with RTL8168g PHY, the quirk disables
@@ -1336,6 +1401,7 @@ static void __rtl8169_set_wol(struct rtl8169_private *tp, u32 wolopts)
 		{ WAKE_MAGIC, Config3, MagicPacket }
 	};
 	unsigned int i, tmp = ARRAY_SIZE(cfg);
+	unsigned long flags;
 	u8 options;
 
 	rtl_unlock_config_regs(tp);
@@ -1354,12 +1420,14 @@ static void __rtl8169_set_wol(struct rtl8169_private *tp, u32 wolopts)
 			r8168_mac_ocp_modify(tp, 0xc0b6, BIT(0), 0);
 	}
 
+	spin_lock_irqsave(&tp->config25_lock, flags);
 	for (i = 0; i < tmp; i++) {
 		options = RTL_R8(tp, cfg[i].reg) & ~cfg[i].mask;
 		if (wolopts & cfg[i].opt)
 			options |= cfg[i].mask;
 		RTL_W8(tp, cfg[i].reg, options);
 	}
+	spin_unlock_irqrestore(&tp->config25_lock, flags);
 
 	switch (tp->mac_version) {
 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
@@ -1371,10 +1439,10 @@ static void __rtl8169_set_wol(struct rtl8169_private *tp, u32 wolopts)
 	case RTL_GIGA_MAC_VER_34:
 	case RTL_GIGA_MAC_VER_37:
 	case RTL_GIGA_MAC_VER_39 ... RTL_GIGA_MAC_VER_63:
-		options = RTL_R8(tp, Config2) & ~PME_SIGNAL;
 		if (wolopts)
-			options |= PME_SIGNAL;
-		RTL_W8(tp, Config2, options);
+			rtl_mod_config2(tp, 0, PME_SIGNAL);
+		else
+			rtl_mod_config2(tp, PME_SIGNAL, 0);
 		break;
 	default:
 		break;
@@ -2675,10 +2743,12 @@ static void rtl_disable_exit_l1(struct rtl8169_private *tp)
 
 static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
 {
-	/* Don't enable ASPM in the chip if OS can't control ASPM */
-	if (enable && tp->aspm_manageable) {
-		RTL_W8(tp, Config5, RTL_R8(tp, Config5) | ASPM_en);
-		RTL_W8(tp, Config2, RTL_R8(tp, Config2) | ClkReqEn);
+	if (tp->mac_version < RTL_GIGA_MAC_VER_32)
+		return;
+
+	if (enable) {
+		rtl_mod_config5(tp, 0, ASPM_en);
+		rtl_mod_config2(tp, 0, ClkReqEn);
 
 		switch (tp->mac_version) {
 		case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
@@ -2701,11 +2771,9 @@ static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
 			break;
 		}
 
-		RTL_W8(tp, Config2, RTL_R8(tp, Config2) & ~ClkReqEn);
-		RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~ASPM_en);
+		rtl_mod_config2(tp, ClkReqEn, 0);
+		rtl_mod_config5(tp, ASPM_en, 0);
 	}
-
-	udelay(10);
 }
 
 static void rtl_set_fifo_size(struct rtl8169_private *tp, u16 rx_stat,
@@ -2863,7 +2931,7 @@ static void rtl_hw_start_8168e_1(struct rtl8169_private *tp)
 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | TXPLA_RST);
 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~TXPLA_RST);
 
-	RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~Spi_en);
+	rtl_mod_config5(tp, Spi_en, 0);
 }
 
 static void rtl_hw_start_8168e_2(struct rtl8169_private *tp)
@@ -2896,9 +2964,7 @@ static void rtl_hw_start_8168e_2(struct rtl8169_private *tp)
 
 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | PWM_EN);
-	RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~Spi_en);
-
-	rtl_hw_aspm_clkreq_enable(tp, true);
+	rtl_mod_config5(tp, Spi_en, 0);
 }
 
 static void rtl_hw_start_8168f(struct rtl8169_private *tp)
@@ -2919,7 +2985,7 @@ static void rtl_hw_start_8168f(struct rtl8169_private *tp)
 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | PWM_EN);
-	RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~Spi_en);
+	rtl_mod_config5(tp, Spi_en, 0);
 
 	rtl8168_config_eee_mac(tp);
 }
@@ -2989,11 +3055,7 @@ static void rtl_hw_start_8168g_1(struct rtl8169_private *tp)
 	};
 
 	rtl_hw_start_8168g(tp);
-
-	/* disable aspm and clock request before access ephy */
-	rtl_hw_aspm_clkreq_enable(tp, false);
 	rtl_ephy_init(tp, e_info_8168g_1);
-	rtl_hw_aspm_clkreq_enable(tp, true);
 }
 
 static void rtl_hw_start_8168g_2(struct rtl8169_private *tp)
@@ -3011,9 +3073,6 @@ static void rtl_hw_start_8168g_2(struct rtl8169_private *tp)
 	};
 
 	rtl_hw_start_8168g(tp);
-
-	/* disable aspm and clock request before access ephy */
-	rtl_hw_aspm_clkreq_enable(tp, false);
 	rtl_ephy_init(tp, e_info_8168g_2);
 }
 
@@ -3034,8 +3093,6 @@ static void rtl_hw_start_8411_2(struct rtl8169_private *tp)
 
 	rtl_hw_start_8168g(tp);
 
-	/* disable aspm and clock request before access ephy */
-	rtl_hw_aspm_clkreq_enable(tp, false);
 	rtl_ephy_init(tp, e_info_8411_2);
 
 	/* The following Realtek-provided magic fixes an issue with the RX unit
@@ -3173,8 +3230,6 @@ static void rtl_hw_start_8411_2(struct rtl8169_private *tp)
 	r8168_mac_ocp_write(tp, 0xFC32, 0x0C25);
 	r8168_mac_ocp_write(tp, 0xFC34, 0x00A9);
 	r8168_mac_ocp_write(tp, 0xFC36, 0x012D);
-
-	rtl_hw_aspm_clkreq_enable(tp, true);
 }
 
 static void rtl_hw_start_8168h_1(struct rtl8169_private *tp)
@@ -3189,8 +3244,6 @@ static void rtl_hw_start_8168h_1(struct rtl8169_private *tp)
 	};
 	int rg_saw_cnt;
 
-	/* disable aspm and clock request before access ephy */
-	rtl_hw_aspm_clkreq_enable(tp, false);
 	rtl_ephy_init(tp, e_info_8168h_1);
 
 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
@@ -3238,8 +3291,6 @@ static void rtl_hw_start_8168h_1(struct rtl8169_private *tp)
 	r8168_mac_ocp_write(tp, 0xe63e, 0x0000);
 	r8168_mac_ocp_write(tp, 0xc094, 0x0000);
 	r8168_mac_ocp_write(tp, 0xc09e, 0x0000);
-
-	rtl_hw_aspm_clkreq_enable(tp, true);
 }
 
 static void rtl_hw_start_8168ep(struct rtl8169_private *tp)
@@ -3278,8 +3329,6 @@ static void rtl_hw_start_8168ep_3(struct rtl8169_private *tp)
 		{ 0x1e, 0x0000,	0x2000 },
 	};
 
-	/* disable aspm and clock request before access ephy */
-	rtl_hw_aspm_clkreq_enable(tp, false);
 	rtl_ephy_init(tp, e_info_8168ep_3);
 
 	rtl_hw_start_8168ep(tp);
@@ -3290,8 +3339,6 @@ static void rtl_hw_start_8168ep_3(struct rtl8169_private *tp)
 	r8168_mac_ocp_modify(tp, 0xd3e2, 0x0fff, 0x0271);
 	r8168_mac_ocp_modify(tp, 0xd3e4, 0x00ff, 0x0000);
 	r8168_mac_ocp_modify(tp, 0xe860, 0x0000, 0x0080);
-
-	rtl_hw_aspm_clkreq_enable(tp, true);
 }
 
 static void rtl_hw_start_8117(struct rtl8169_private *tp)
@@ -3303,9 +3350,6 @@ static void rtl_hw_start_8117(struct rtl8169_private *tp)
 	int rg_saw_cnt;
 
 	rtl8168ep_stop_cmac(tp);
-
-	/* disable aspm and clock request before access ephy */
-	rtl_hw_aspm_clkreq_enable(tp, false);
 	rtl_ephy_init(tp, e_info_8117);
 
 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
@@ -3355,8 +3399,6 @@ static void rtl_hw_start_8117(struct rtl8169_private *tp)
 
 	/* firmware is for MAC only */
 	r8169_apply_firmware(tp);
-
-	rtl_hw_aspm_clkreq_enable(tp, true);
 }
 
 static void rtl_hw_start_8102e_1(struct rtl8169_private *tp)
@@ -3479,8 +3521,6 @@ static void rtl_hw_start_8402(struct rtl8169_private *tp)
 
 static void rtl_hw_start_8106(struct rtl8169_private *tp)
 {
-	rtl_hw_aspm_clkreq_enable(tp, false);
-
 	/* Force LAN exit from ASPM if Rx/Tx are not idle */
 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
 
@@ -3497,7 +3537,6 @@ static void rtl_hw_start_8106(struct rtl8169_private *tp)
 	rtl_eri_write(tp, 0x1b0, ERIAR_MASK_0011, 0x0000);
 
 	rtl_pcie_state_l2l3_disable(tp);
-	rtl_hw_aspm_clkreq_enable(tp, true);
 }
 
 DECLARE_RTL_COND(rtl_mac_ocp_e00e_cond)
@@ -3585,13 +3624,8 @@ static void rtl_hw_start_8125a_2(struct rtl8169_private *tp)
 	};
 
 	rtl_set_def_aspm_entry_latency(tp);
-
-	/* disable aspm and clock request before access ephy */
-	rtl_hw_aspm_clkreq_enable(tp, false);
 	rtl_ephy_init(tp, e_info_8125a_2);
-
 	rtl_hw_start_8125_common(tp);
-	rtl_hw_aspm_clkreq_enable(tp, true);
 }
 
 static void rtl_hw_start_8125b(struct rtl8169_private *tp)
@@ -3606,12 +3640,8 @@ static void rtl_hw_start_8125b(struct rtl8169_private *tp)
 	};
 
 	rtl_set_def_aspm_entry_latency(tp);
-	rtl_hw_aspm_clkreq_enable(tp, false);
-
 	rtl_ephy_init(tp, e_info_8125b);
 	rtl_hw_start_8125_common(tp);
-
-	rtl_hw_aspm_clkreq_enable(tp, true);
 }
 
 static void rtl_hw_config(struct rtl8169_private *tp)
@@ -3707,7 +3737,8 @@ static void rtl_hw_start_8169(struct rtl8169_private *tp)
 static void rtl_hw_start(struct  rtl8169_private *tp)
 {
 	rtl_unlock_config_regs(tp);
-
+	/* disable aspm and clock request before ephy access */
+	rtl_hw_aspm_clkreq_enable(tp, false);
 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
 
 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
@@ -3718,6 +3749,7 @@ static void rtl_hw_start(struct  rtl8169_private *tp)
 		rtl_hw_start_8168(tp);
 
 	rtl_enable_exit_l1(tp);
+	rtl_hw_aspm_clkreq_enable(tp, true);
 	rtl_set_rx_max_size(tp);
 	rtl_set_rx_tx_desc_registers(tp);
 	rtl_lock_config_regs(tp);
@@ -4133,13 +4165,9 @@ static bool rtl8169_tso_csum_v2(struct rtl8169_private *tp,
 	return true;
 }
 
-static bool rtl_tx_slots_avail(struct rtl8169_private *tp)
+static unsigned int rtl_tx_slots_avail(struct rtl8169_private *tp)
 {
-	unsigned int slots_avail = READ_ONCE(tp->dirty_tx) + NUM_TX_DESC
-					- READ_ONCE(tp->cur_tx);
-
-	/* A skbuff with nr_frags needs nr_frags+1 entries in the tx queue */
-	return slots_avail > MAX_SKB_FRAGS;
+	return READ_ONCE(tp->dirty_tx) + NUM_TX_DESC - READ_ONCE(tp->cur_tx);
 }
 
 /* Versions RTL8102e and from RTL8168c onwards support csum_v2 */
@@ -4216,27 +4244,10 @@ static netdev_tx_t rtl8169_start_xmit(struct sk_buff *skb,
 
 	WRITE_ONCE(tp->cur_tx, tp->cur_tx + frags + 1);
 
-	stop_queue = !rtl_tx_slots_avail(tp);
-	if (unlikely(stop_queue)) {
-		/* Avoid wrongly optimistic queue wake-up: rtl_tx thread must
-		 * not miss a ring update when it notices a stopped queue.
-		 */
-		smp_wmb();
-		netif_stop_queue(dev);
-		/* Sync with rtl_tx:
-		 * - publish queue status and cur_tx ring index (write barrier)
-		 * - refresh dirty_tx ring index (read barrier).
-		 * May the current thread have a pessimistic view of the ring
-		 * status and forget to wake up queue, a racing rtl_tx thread
-		 * can't.
-		 */
-		smp_mb__after_atomic();
-		if (rtl_tx_slots_avail(tp))
-			netif_start_queue(dev);
-		door_bell = true;
-	}
-
-	if (door_bell)
+	stop_queue = !netif_subqueue_maybe_stop(dev, 0, rtl_tx_slots_avail(tp),
+						R8169_TX_STOP_THRS,
+						R8169_TX_START_THRS);
+	if (door_bell || stop_queue)
 		rtl8169_doorbell(tp);
 
 	return NETDEV_TX_OK;
@@ -4360,19 +4371,12 @@ static void rtl_tx(struct net_device *dev, struct rtl8169_private *tp,
 	}
 
 	if (tp->dirty_tx != dirty_tx) {
-		netdev_completed_queue(dev, pkts_compl, bytes_compl);
 		dev_sw_netstats_tx_add(dev, pkts_compl, bytes_compl);
+		WRITE_ONCE(tp->dirty_tx, dirty_tx);
 
-		/* Sync with rtl8169_start_xmit:
-		 * - publish dirty_tx ring index (write barrier)
-		 * - refresh cur_tx ring index and queue status (read barrier)
-		 * May the current thread miss the stopped queue condition,
-		 * a racing xmit thread can only have a right view of the
-		 * ring status.
-		 */
-		smp_store_mb(tp->dirty_tx, dirty_tx);
-		if (netif_queue_stopped(dev) && rtl_tx_slots_avail(tp))
-			netif_wake_queue(dev);
+		netif_subqueue_completed_wake(dev, 0, pkts_compl, bytes_compl,
+					      rtl_tx_slots_avail(tp),
+					      R8169_TX_START_THRS);
 		/*
 		 * 8168 hack: TxPoll requests are lost when the Tx packets are
 		 * too close. Let's kick an extra TxPoll request when a burst
@@ -4510,6 +4514,10 @@ static irqreturn_t rtl8169_interrupt(int irq, void *dev_instance)
 	}
 
 	if (napi_schedule_prep(&tp->napi)) {
+		rtl_unlock_config_regs(tp);
+		rtl_hw_aspm_clkreq_enable(tp, false);
+		rtl_lock_config_regs(tp);
+
 		rtl_irq_disable(tp);
 		__napi_schedule(&tp->napi);
 	}
@@ -4569,9 +4577,14 @@ static int rtl8169_poll(struct napi_struct *napi, int budget)
 
 	work_done = rtl_rx(dev, tp, budget);
 
-	if (work_done < budget && napi_complete_done(napi, work_done))
+	if (work_done < budget && napi_complete_done(napi, work_done)) {
 		rtl_irq_enable(tp);
 
+		rtl_unlock_config_regs(tp);
+		rtl_hw_aspm_clkreq_enable(tp, true);
+		rtl_lock_config_regs(tp);
+	}
+
 	return work_done;
 }
 
@@ -5145,16 +5158,6 @@ done:
 	rtl_rar_set(tp, mac_addr);
 }
 
-/* register is set if system vendor successfully tested ASPM 1.2 */
-static bool rtl_aspm_is_safe(struct rtl8169_private *tp)
-{
-	if (tp->mac_version >= RTL_GIGA_MAC_VER_61 &&
-	    r8168_mac_ocp_read(tp, 0xc0b2) & 0xf)
-		return true;
-
-	return false;
-}
-
 static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	struct rtl8169_private *tp;
@@ -5176,6 +5179,10 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	tp->eee_adv = -1;
 	tp->ocp_base = OCP_STD_PHY_BASE;
 
+	spin_lock_init(&tp->cfg9346_usage_lock);
+	spin_lock_init(&tp->config25_lock);
+	spin_lock_init(&tp->mac_ocp_lock);
+
 	dev->tstats = devm_netdev_alloc_pcpu_stats(&pdev->dev,
 						   struct pcpu_sw_netstats);
 	if (!dev->tstats)
@@ -5222,19 +5229,6 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	tp->mac_version = chipset;
 
-	/* Disable ASPM L1 as that cause random device stop working
-	 * problems as well as full system hangs for some PCIe devices users.
-	 * Chips from RTL8168h partially have issues with L1.2, but seem
-	 * to work fine with L1 and L1.1.
-	 */
-	if (rtl_aspm_is_safe(tp))
-		rc = 0;
-	else if (tp->mac_version >= RTL_GIGA_MAC_VER_46)
-		rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2);
-	else
-		rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1);
-	tp->aspm_manageable = !rc;
-
 	tp->dash_type = rtl_check_dash(tp);
 
 	tp->cp_cmd = RTL_R16(tp, CPlusCmd) & CPCMD_MASK;
diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
index 894e2690c643..4d6b3b7d6abb 100644
--- a/drivers/net/ethernet/renesas/ravb_main.c
+++ b/drivers/net/ethernet/renesas/ravb_main.c
@@ -28,7 +28,6 @@
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
-#include <linux/sys_soc.h>
 #include <linux/reset.h>
 #include <linux/math64.h>
 
@@ -1390,11 +1389,6 @@ static void ravb_adjust_link(struct net_device *ndev)
 		phy_print_status(phydev);
 }
 
-static const struct soc_device_attribute r8a7795es10[] = {
-	{ .soc_id = "r8a7795", .revision = "ES1.0", },
-	{ /* sentinel */ }
-};
-
 /* PHY init function */
 static int ravb_phy_init(struct net_device *ndev)
 {
@@ -1434,15 +1428,6 @@ static int ravb_phy_init(struct net_device *ndev)
 		goto err_deregister_fixed_link;
 	}
 
-	/* This driver only support 10/100Mbit speeds on R-Car H3 ES1.0
-	 * at this time.
-	 */
-	if (soc_device_match(r8a7795es10)) {
-		phy_set_max_speed(phydev, SPEED_100);
-
-		netdev_info(ndev, "limited PHY to 100Mbit/s\n");
-	}
-
 	if (!info->half_duplex) {
 		/* 10BASE, Pause and Asym Pause is not supported */
 		phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT);
diff --git a/drivers/net/ethernet/renesas/rswitch.c b/drivers/net/ethernet/renesas/rswitch.c
index c4f93d24c6a4..29afaddb598d 100644
--- a/drivers/net/ethernet/renesas/rswitch.c
+++ b/drivers/net/ethernet/renesas/rswitch.c
@@ -1324,10 +1324,8 @@ out:
 
 static void rswitch_phy_device_deinit(struct rswitch_device *rdev)
 {
-	if (rdev->ndev->phydev) {
+	if (rdev->ndev->phydev)
 		phy_disconnect(rdev->ndev->phydev);
-		rdev->ndev->phydev = NULL;
-	}
 }
 
 static int rswitch_serdes_set_params(struct rswitch_device *rdev)
diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_platform.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_platform.c
index 926532466691..4e5526303f07 100644
--- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_platform.c
+++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_platform.c
@@ -229,7 +229,7 @@ static struct platform_driver sxgbe_platform_driver = {
 	.driver	= {
 		.name		= SXGBE_RESOURCE_NAME,
 		.pm		= &sxgbe_platform_pm_ops,
-		.of_match_table	= of_match_ptr(sxgbe_dt_ids),
+		.of_match_table	= sxgbe_dt_ids,
 	},
 };
 
diff --git a/drivers/net/ethernet/sfc/ef100.c b/drivers/net/ethernet/sfc/ef100.c
index 71aab3d0480f..6334992b0af4 100644
--- a/drivers/net/ethernet/sfc/ef100.c
+++ b/drivers/net/ethernet/sfc/ef100.c
@@ -11,7 +11,6 @@
 
 #include "net_driver.h"
 #include <linux/module.h>
-#include <linux/aer.h>
 #include "efx_common.h"
 #include "efx_channels.h"
 #include "io.h"
@@ -440,8 +439,6 @@ static void ef100_pci_remove(struct pci_dev *pci_dev)
 
 	pci_dbg(pci_dev, "shutdown successful\n");
 
-	pci_disable_pcie_error_reporting(pci_dev);
-
 	pci_set_drvdata(pci_dev, NULL);
 	efx_fini_struct(efx);
 	kfree(probe_data);
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 884d8d168862..a4f22d8e6ac7 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -18,7 +18,6 @@
 #include <linux/ethtool.h>
 #include <linux/topology.h>
 #include <linux/gfp.h>
-#include <linux/aer.h>
 #include <linux/interrupt.h>
 #include "net_driver.h"
 #include <net/gre.h>
@@ -541,7 +540,6 @@ int efx_net_open(struct net_device *net_dev)
 	else
 		efx->state = STATE_NET_UP;
 
-	efx_selftest_async_start(efx);
 	return 0;
 }
 
@@ -892,8 +890,6 @@ static void efx_pci_remove(struct pci_dev *pci_dev)
 	free_netdev(efx->net_dev);
 	probe_data = container_of(efx, struct efx_probe_data, efx);
 	kfree(probe_data);
-
-	pci_disable_pcie_error_reporting(pci_dev);
 };
 
 /* NIC VPD information
@@ -1123,8 +1119,6 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
 		netif_warn(efx, probe, efx->net_dev,
 			   "failed to create MTDs (%d)\n", rc);
 
-	(void)pci_enable_pcie_error_reporting(pci_dev);
-
 	if (efx->type->udp_tnl_push_ports)
 		efx->type->udp_tnl_push_ports(efx);
 
diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c
index cc30524c2fe4..361687de308d 100644
--- a/drivers/net/ethernet/sfc/efx_common.c
+++ b/drivers/net/ethernet/sfc/efx_common.c
@@ -544,6 +544,8 @@ void efx_start_all(struct efx_nic *efx)
 	/* Start the hardware monitor if there is one */
 	efx_start_monitor(efx);
 
+	efx_selftest_async_start(efx);
+
 	/* Link state detection is normally event-driven; we have
 	 * to poll now because we could have missed a change
 	 */
diff --git a/drivers/net/ethernet/sfc/falcon/efx.c b/drivers/net/ethernet/sfc/falcon/efx.c
index e151b0957751..e001f27085c6 100644
--- a/drivers/net/ethernet/sfc/falcon/efx.c
+++ b/drivers/net/ethernet/sfc/falcon/efx.c
@@ -17,7 +17,6 @@
 #include <linux/ethtool.h>
 #include <linux/topology.h>
 #include <linux/gfp.h>
-#include <linux/aer.h>
 #include <linux/interrupt.h>
 #include "net_driver.h"
 #include "efx.h"
@@ -2765,8 +2764,6 @@ static void ef4_pci_remove(struct pci_dev *pci_dev)
 
 	ef4_fini_struct(efx);
 	free_netdev(efx->net_dev);
-
-	pci_disable_pcie_error_reporting(pci_dev);
 };
 
 /* NIC VPD information
@@ -2927,12 +2924,6 @@ static int ef4_pci_probe(struct pci_dev *pci_dev,
 		netif_warn(efx, probe, efx->net_dev,
 			   "failed to create MTDs (%d)\n", rc);
 
-	rc = pci_enable_pcie_error_reporting(pci_dev);
-	if (rc && rc != -EINVAL)
-		netif_notice(efx, probe, efx->net_dev,
-			     "PCIE error reporting unavailable (%d).\n",
-			     rc);
-
 	return 0;
 
  fail4:
diff --git a/drivers/net/ethernet/sfc/mae.c b/drivers/net/ethernet/sfc/mae.c
index 2d32abe5f478..49706a7b94bf 100644
--- a/drivers/net/ethernet/sfc/mae.c
+++ b/drivers/net/ethernet/sfc/mae.c
@@ -241,6 +241,7 @@ static int efx_mae_get_basic_caps(struct efx_nic *efx, struct mae_caps *caps)
 	if (outlen < sizeof(outbuf))
 		return -EIO;
 	caps->match_field_count = MCDI_DWORD(outbuf, MAE_GET_CAPS_OUT_MATCH_FIELD_COUNT);
+	caps->encap_types = MCDI_DWORD(outbuf, MAE_GET_CAPS_OUT_ENCAP_TYPES_SUPPORTED);
 	caps->action_prios = MCDI_DWORD(outbuf, MAE_GET_CAPS_OUT_ACTION_PRIOS);
 	return 0;
 }
@@ -254,13 +255,23 @@ static int efx_mae_get_rule_fields(struct efx_nic *efx, u32 cmd,
 	size_t outlen;
 	int rc, i;
 
+	/* AR and OR caps MCDIs have identical layout, so we are using the
+	 * same code for both.
+	 */
+	BUILD_BUG_ON(MC_CMD_MAE_GET_AR_CAPS_OUT_LEN(MAE_NUM_FIELDS) <
+		     MC_CMD_MAE_GET_OR_CAPS_OUT_LEN(MAE_NUM_FIELDS));
 	BUILD_BUG_ON(MC_CMD_MAE_GET_AR_CAPS_IN_LEN);
+	BUILD_BUG_ON(MC_CMD_MAE_GET_OR_CAPS_IN_LEN);
 
 	rc = efx_mcdi_rpc(efx, cmd, NULL, 0, outbuf, sizeof(outbuf), &outlen);
 	if (rc)
 		return rc;
+	BUILD_BUG_ON(MC_CMD_MAE_GET_AR_CAPS_OUT_COUNT_OFST !=
+		     MC_CMD_MAE_GET_OR_CAPS_OUT_COUNT_OFST);
 	count = MCDI_DWORD(outbuf, MAE_GET_AR_CAPS_OUT_COUNT);
 	memset(field_support, MAE_FIELD_UNSUPPORTED, MAE_NUM_FIELDS);
+	BUILD_BUG_ON(MC_CMD_MAE_GET_AR_CAPS_OUT_FIELD_FLAGS_OFST !=
+		     MC_CMD_MAE_GET_OR_CAPS_OUT_FIELD_FLAGS_OFST);
 	caps = _MCDI_DWORD(outbuf, MAE_GET_AR_CAPS_OUT_FIELD_FLAGS);
 	/* We're only interested in the support status enum, not any other
 	 * flags, so just extract that from each entry.
@@ -278,8 +289,12 @@ int efx_mae_get_caps(struct efx_nic *efx, struct mae_caps *caps)
 	rc = efx_mae_get_basic_caps(efx, caps);
 	if (rc)
 		return rc;
-	return efx_mae_get_rule_fields(efx, MC_CMD_MAE_GET_AR_CAPS,
-				       caps->action_rule_fields);
+	rc = efx_mae_get_rule_fields(efx, MC_CMD_MAE_GET_AR_CAPS,
+				     caps->action_rule_fields);
+	if (rc)
+		return rc;
+	return efx_mae_get_rule_fields(efx, MC_CMD_MAE_GET_OR_CAPS,
+				       caps->outer_rule_fields);
 }
 
 /* Bit twiddling:
@@ -432,11 +447,86 @@ int efx_mae_match_check_caps(struct efx_nic *efx,
 	    CHECK_BIT(IP_FIRST_FRAG, ip_firstfrag) ||
 	    CHECK(RECIRC_ID, recirc_id))
 		return rc;
+	/* Matches on outer fields are done in a separate hardware table,
+	 * the Outer Rule table.  Thus the Action Rule merely does an
+	 * exact match on Outer Rule ID if any outer field matches are
+	 * present.  The exception is the VNI/VSID (enc_keyid), which is
+	 * available to the Action Rule match iff the Outer Rule matched
+	 * (and thus identified the encap protocol to use to extract it).
+	 */
+	if (efx_tc_match_is_encap(mask)) {
+		rc = efx_mae_match_check_cap_typ(
+				supported_fields[MAE_FIELD_OUTER_RULE_ID],
+				MASK_ONES);
+		if (rc) {
+			NL_SET_ERR_MSG_MOD(extack, "No support for encap rule ID matches");
+			return rc;
+		}
+		if (CHECK(ENC_VNET_ID, enc_keyid))
+			return rc;
+	} else if (mask->enc_keyid) {
+		NL_SET_ERR_MSG_MOD(extack, "Match on enc_keyid requires other encap fields");
+		return -EINVAL;
+	}
 	return 0;
 }
 #undef CHECK_BIT
 #undef CHECK
 
+#define CHECK(_mcdi)	({						       \
+	rc = efx_mae_match_check_cap_typ(supported_fields[MAE_FIELD_ ## _mcdi],\
+					 MASK_ONES);			       \
+	if (rc)								       \
+		NL_SET_ERR_MSG_FMT_MOD(extack,				       \
+				       "No support for field %s", #_mcdi);     \
+	rc;								       \
+})
+/* Checks that the fields needed for encap-rule matches are supported by the
+ * MAE.  All the fields are exact-match.
+ */
+int efx_mae_check_encap_match_caps(struct efx_nic *efx, bool ipv6,
+				   struct netlink_ext_ack *extack)
+{
+	u8 *supported_fields = efx->tc->caps->outer_rule_fields;
+	int rc;
+
+	if (CHECK(ENC_ETHER_TYPE))
+		return rc;
+	if (ipv6) {
+		if (CHECK(ENC_SRC_IP6) ||
+		    CHECK(ENC_DST_IP6))
+			return rc;
+	} else {
+		if (CHECK(ENC_SRC_IP4) ||
+		    CHECK(ENC_DST_IP4))
+			return rc;
+	}
+	if (CHECK(ENC_L4_DPORT) ||
+	    CHECK(ENC_IP_PROTO))
+		return rc;
+	return 0;
+}
+#undef CHECK
+
+int efx_mae_check_encap_type_supported(struct efx_nic *efx, enum efx_encap_type typ)
+{
+	unsigned int bit;
+
+	switch (typ & EFX_ENCAP_TYPES_MASK) {
+	case EFX_ENCAP_TYPE_VXLAN:
+		bit = MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_VXLAN_LBN;
+		break;
+	case EFX_ENCAP_TYPE_GENEVE:
+		bit = MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_GENEVE_LBN;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+	if (efx->tc->caps->encap_types & BIT(bit))
+		return 0;
+	return -EOPNOTSUPP;
+}
+
 int efx_mae_allocate_counter(struct efx_nic *efx, struct efx_tc_counter *cnt)
 {
 	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTER_ALLOC_OUT_LEN(1));
@@ -488,6 +578,20 @@ int efx_mae_free_counter(struct efx_nic *efx, struct efx_tc_counter *cnt)
 	return 0;
 }
 
+static int efx_mae_encap_type_to_mae_type(enum efx_encap_type type)
+{
+	switch (type & EFX_ENCAP_TYPES_MASK) {
+	case EFX_ENCAP_TYPE_NONE:
+		return MAE_MCDI_ENCAP_TYPE_NONE;
+	case EFX_ENCAP_TYPE_VXLAN:
+		return MAE_MCDI_ENCAP_TYPE_VXLAN;
+	case EFX_ENCAP_TYPE_GENEVE:
+		return MAE_MCDI_ENCAP_TYPE_GENEVE;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 int efx_mae_lookup_mport(struct efx_nic *efx, u32 vf_idx, u32 *id)
 {
 	struct ef100_nic_data *nic_data = efx->nic_data;
@@ -682,6 +786,11 @@ int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act)
 	size_t outlen;
 	int rc;
 
+	MCDI_POPULATE_DWORD_3(inbuf, MAE_ACTION_SET_ALLOC_IN_FLAGS,
+			      MAE_ACTION_SET_ALLOC_IN_VLAN_PUSH, act->vlan_push,
+			      MAE_ACTION_SET_ALLOC_IN_VLAN_POP, act->vlan_pop,
+			      MAE_ACTION_SET_ALLOC_IN_DECAP, act->decap);
+
 	MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_SRC_MAC_ID,
 		       MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL);
 	MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_DST_MAC_ID,
@@ -694,6 +803,18 @@ int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act)
 			       MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL);
 	MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_LIST_ID,
 		       MC_CMD_MAE_COUNTER_LIST_ALLOC_OUT_COUNTER_LIST_ID_NULL);
+	if (act->vlan_push) {
+		MCDI_SET_WORD_BE(inbuf, MAE_ACTION_SET_ALLOC_IN_VLAN0_TCI_BE,
+				 act->vlan_tci[0]);
+		MCDI_SET_WORD_BE(inbuf, MAE_ACTION_SET_ALLOC_IN_VLAN0_PROTO_BE,
+				 act->vlan_proto[0]);
+	}
+	if (act->vlan_push >= 2) {
+		MCDI_SET_WORD_BE(inbuf, MAE_ACTION_SET_ALLOC_IN_VLAN1_TCI_BE,
+				 act->vlan_tci[1]);
+		MCDI_SET_WORD_BE(inbuf, MAE_ACTION_SET_ALLOC_IN_VLAN1_PROTO_BE,
+				 act->vlan_proto[1]);
+	}
 	MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_ENCAP_HEADER_ID,
 		       MC_CMD_MAE_ENCAP_HEADER_ALLOC_OUT_ENCAP_HEADER_ID_NULL);
 	if (act->deliver)
@@ -829,6 +950,97 @@ int efx_mae_free_action_set_list(struct efx_nic *efx,
 	return 0;
 }
 
+int efx_mae_register_encap_match(struct efx_nic *efx,
+				 struct efx_tc_encap_match *encap)
+{
+	MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_OUTER_RULE_INSERT_IN_LEN(MAE_ENC_FIELD_PAIRS_LEN));
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_OUTER_RULE_INSERT_OUT_LEN);
+	MCDI_DECLARE_STRUCT_PTR(match_crit);
+	size_t outlen;
+	int rc;
+
+	rc = efx_mae_encap_type_to_mae_type(encap->tun_type);
+	if (rc < 0)
+		return rc;
+	match_crit = _MCDI_DWORD(inbuf, MAE_OUTER_RULE_INSERT_IN_FIELD_MATCH_CRITERIA);
+	/* The struct contains IP src and dst, and udp dport.
+	 * So we actually need to filter on IP src and dst, L4 dport, and
+	 * ipproto == udp.
+	 */
+	MCDI_SET_DWORD(inbuf, MAE_OUTER_RULE_INSERT_IN_ENCAP_TYPE, rc);
+#ifdef CONFIG_IPV6
+	if (encap->src_ip | encap->dst_ip) {
+#endif
+		MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_SRC_IP4_BE,
+					 encap->src_ip);
+		MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_SRC_IP4_BE_MASK,
+					 ~(__be32)0);
+		MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_DST_IP4_BE,
+					 encap->dst_ip);
+		MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_DST_IP4_BE_MASK,
+					 ~(__be32)0);
+		MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE,
+					htons(ETH_P_IP));
+#ifdef CONFIG_IPV6
+	} else {
+		memcpy(MCDI_STRUCT_PTR(match_crit, MAE_ENC_FIELD_PAIRS_ENC_SRC_IP6_BE),
+		       &encap->src_ip6, sizeof(encap->src_ip6));
+		memset(MCDI_STRUCT_PTR(match_crit, MAE_ENC_FIELD_PAIRS_ENC_SRC_IP6_BE_MASK),
+		       0xff, sizeof(encap->src_ip6));
+		memcpy(MCDI_STRUCT_PTR(match_crit, MAE_ENC_FIELD_PAIRS_ENC_DST_IP6_BE),
+		       &encap->dst_ip6, sizeof(encap->dst_ip6));
+		memset(MCDI_STRUCT_PTR(match_crit, MAE_ENC_FIELD_PAIRS_ENC_DST_IP6_BE_MASK),
+		       0xff, sizeof(encap->dst_ip6));
+		MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE,
+					htons(ETH_P_IPV6));
+	}
+#endif
+	MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE_MASK,
+				~(__be16)0);
+	MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_L4_DPORT_BE,
+				encap->udp_dport);
+	MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_L4_DPORT_BE_MASK,
+				~(__be16)0);
+	MCDI_STRUCT_SET_BYTE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_IP_PROTO, IPPROTO_UDP);
+	MCDI_STRUCT_SET_BYTE(match_crit, MAE_ENC_FIELD_PAIRS_ENC_IP_PROTO_MASK, ~0);
+	rc = efx_mcdi_rpc(efx, MC_CMD_MAE_OUTER_RULE_INSERT, inbuf,
+			  sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
+	if (rc)
+		return rc;
+	if (outlen < sizeof(outbuf))
+		return -EIO;
+	encap->fw_id = MCDI_DWORD(outbuf, MAE_OUTER_RULE_INSERT_OUT_OR_ID);
+	return 0;
+}
+
+int efx_mae_unregister_encap_match(struct efx_nic *efx,
+				   struct efx_tc_encap_match *encap)
+{
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_LEN(1));
+	MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_OUTER_RULE_REMOVE_IN_LEN(1));
+	size_t outlen;
+	int rc;
+
+	MCDI_SET_DWORD(inbuf, MAE_OUTER_RULE_REMOVE_IN_OR_ID, encap->fw_id);
+	rc = efx_mcdi_rpc(efx, MC_CMD_MAE_OUTER_RULE_REMOVE, inbuf,
+			  sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
+	if (rc)
+		return rc;
+	if (outlen < sizeof(outbuf))
+		return -EIO;
+	/* FW freed a different ID than we asked for, should also never happen.
+	 * Warn because it means we've now got a different idea to the FW of
+	 * what encap_mds exist, which could cause mayhem later.
+	 */
+	if (WARN_ON(MCDI_DWORD(outbuf, MAE_OUTER_RULE_REMOVE_OUT_REMOVED_OR_ID) != encap->fw_id))
+		return -EIO;
+	/* We're probably about to free @encap, but let's just make sure its
+	 * fw_id is blatted so that it won't look valid if it leaks out.
+	 */
+	encap->fw_id = MC_CMD_MAE_OUTER_RULE_INSERT_OUT_OUTER_RULE_ID_NULL;
+	return 0;
+}
+
 static int efx_mae_populate_match_criteria(MCDI_DECLARE_STRUCT_PTR(match_crit),
 					   const struct efx_tc_match *match)
 {
@@ -925,6 +1137,29 @@ static int efx_mae_populate_match_criteria(MCDI_DECLARE_STRUCT_PTR(match_crit),
 				match->value.tcp_flags);
 	MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_MASK,
 				match->mask.tcp_flags);
+	/* enc-keys are handled indirectly, through encap_match ID */
+	if (match->encap) {
+		MCDI_STRUCT_SET_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_OUTER_RULE_ID,
+				      match->encap->fw_id);
+		MCDI_STRUCT_SET_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_OUTER_RULE_ID_MASK,
+				      U32_MAX);
+		/* enc_keyid (VNI/VSID) is not part of the encap_match */
+		MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VNET_ID_BE,
+					 match->value.enc_keyid);
+		MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VNET_ID_BE_MASK,
+					 match->mask.enc_keyid);
+	} else if (WARN_ON_ONCE(match->mask.enc_src_ip) ||
+		   WARN_ON_ONCE(match->mask.enc_dst_ip) ||
+		   WARN_ON_ONCE(!ipv6_addr_any(&match->mask.enc_src_ip6)) ||
+		   WARN_ON_ONCE(!ipv6_addr_any(&match->mask.enc_dst_ip6)) ||
+		   WARN_ON_ONCE(match->mask.enc_ip_tos) ||
+		   WARN_ON_ONCE(match->mask.enc_ip_ttl) ||
+		   WARN_ON_ONCE(match->mask.enc_sport) ||
+		   WARN_ON_ONCE(match->mask.enc_dport) ||
+		   WARN_ON_ONCE(match->mask.enc_keyid)) {
+		/* No enc-keys should appear in a rule without an encap_match */
+		return -EOPNOTSUPP;
+	}
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/sfc/mae.h b/drivers/net/ethernet/sfc/mae.h
index bec293a06733..9226219491a0 100644
--- a/drivers/net/ethernet/sfc/mae.h
+++ b/drivers/net/ethernet/sfc/mae.h
@@ -70,8 +70,10 @@ void efx_mae_counters_grant_credits(struct work_struct *work);
 
 struct mae_caps {
 	u32 match_field_count;
+	u32 encap_types;
 	u32 action_prios;
 	u8 action_rule_fields[MAE_NUM_FIELDS];
+	u8 outer_rule_fields[MAE_NUM_FIELDS];
 };
 
 int efx_mae_get_caps(struct efx_nic *efx, struct mae_caps *caps);
@@ -79,6 +81,10 @@ int efx_mae_get_caps(struct efx_nic *efx, struct mae_caps *caps);
 int efx_mae_match_check_caps(struct efx_nic *efx,
 			     const struct efx_tc_match_fields *mask,
 			     struct netlink_ext_ack *extack);
+int efx_mae_check_encap_match_caps(struct efx_nic *efx, bool ipv6,
+				   struct netlink_ext_ack *extack);
+int efx_mae_check_encap_type_supported(struct efx_nic *efx,
+				       enum efx_encap_type typ);
 
 int efx_mae_allocate_counter(struct efx_nic *efx, struct efx_tc_counter *cnt);
 int efx_mae_free_counter(struct efx_nic *efx, struct efx_tc_counter *cnt);
@@ -91,6 +97,11 @@ int efx_mae_alloc_action_set_list(struct efx_nic *efx,
 int efx_mae_free_action_set_list(struct efx_nic *efx,
 				 struct efx_tc_action_set_list *acts);
 
+int efx_mae_register_encap_match(struct efx_nic *efx,
+				 struct efx_tc_encap_match *encap);
+int efx_mae_unregister_encap_match(struct efx_nic *efx,
+				   struct efx_tc_encap_match *encap);
+
 int efx_mae_insert_rule(struct efx_nic *efx, const struct efx_tc_match *match,
 			u32 prio, u32 acts_id, u32 *id);
 int efx_mae_delete_rule(struct efx_nic *efx, u32 id);
diff --git a/drivers/net/ethernet/sfc/mcdi.h b/drivers/net/ethernet/sfc/mcdi.h
index b139b76febff..454e9d51a4c2 100644
--- a/drivers/net/ethernet/sfc/mcdi.h
+++ b/drivers/net/ethernet/sfc/mcdi.h
@@ -233,6 +233,11 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev);
 	((void)BUILD_BUG_ON_ZERO(_field ## _LEN != 2),  \
 	le16_to_cpu(*(__force const __le16 *)MCDI_STRUCT_PTR(_buf, _field)))
 /* Write a 16-bit field defined in the protocol as being big-endian. */
+#define MCDI_SET_WORD_BE(_buf, _field, _value) do {			\
+	BUILD_BUG_ON(MC_CMD_ ## _field ## _LEN != 2);			\
+	BUILD_BUG_ON(MC_CMD_ ## _field ## _OFST & 1);			\
+	*(__force __be16 *)MCDI_PTR(_buf, _field) = (_value);		\
+	} while (0)
 #define MCDI_STRUCT_SET_WORD_BE(_buf, _field, _value) do {		\
 	BUILD_BUG_ON(_field ## _LEN != 2);				\
 	BUILD_BUG_ON(_field ## _OFST & 1);				\
diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c
index 9f07e1ba7780..0c40571133cb 100644
--- a/drivers/net/ethernet/sfc/ptp.c
+++ b/drivers/net/ethernet/sfc/ptp.c
@@ -33,6 +33,7 @@
 #include <linux/ip.h>
 #include <linux/udp.h>
 #include <linux/time.h>
+#include <linux/errno.h>
 #include <linux/ktime.h>
 #include <linux/module.h>
 #include <linux/pps_kernel.h>
@@ -74,6 +75,9 @@
 /* How long an unmatched event or packet can be held */
 #define PKT_EVENT_LIFETIME_MS		10
 
+/* How long unused unicast filters can be held */
+#define UCAST_FILTER_EXPIRY_JIFFIES	msecs_to_jiffies(30000)
+
 /* Offsets into PTP packet for identification.  These offsets are from the
  * start of the IP header, not the MAC header.  Note that neither PTP V1 nor
  * PTP V2 permit the use of IPV4 options.
@@ -118,8 +122,6 @@
 
 #define	PTP_MIN_LENGTH		63
 
-#define PTP_RXFILTERS_LEN	5
-
 #define PTP_ADDR_IPV4		0xe0000181	/* 224.0.1.129 */
 #define PTP_ADDR_IPV6		{0xff, 0x0e, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
 				0, 0x01, 0x81}	/* ff0e::181 */
@@ -214,6 +216,24 @@ struct efx_ptp_timeset {
 };
 
 /**
+ * struct efx_ptp_rxfilter - Filter for PTP packets
+ * @list: Node of the list where the filter is added
+ * @ether_type: Network protocol of the filter (ETHER_P_IP / ETHER_P_IPV6)
+ * @loc_port: UDP port of the filter (PTP_EVENT_PORT / PTP_GENERAL_PORT)
+ * @loc_host: IPv4/v6 address of the filter
+ * @expiry: time when the filter expires, in jiffies
+ * @handle: Handle ID for the MCDI filters table
+ */
+struct efx_ptp_rxfilter {
+	struct list_head list;
+	__be16 ether_type;
+	__be16 loc_port;
+	__be32 loc_host[4];
+	unsigned long expiry;
+	int handle;
+};
+
+/**
  * struct efx_ptp_data - Precision Time Protocol (PTP) state
  * @efx: The NIC context
  * @channel: The PTP channel (Siena only)
@@ -227,10 +247,11 @@ struct efx_ptp_timeset {
  * @rx_evts: Instantiated events (on evt_list and evt_free_list)
  * @workwq: Work queue for processing pending PTP operations
  * @work: Work task
+ * @cleanup_work: Work task for periodic cleanup
  * @reset_required: A serious error has occurred and the PTP task needs to be
  *                  reset (disable, enable).
- * @rxfilters: Receive filters when operating
- * @rxfilters_count: Num of installed rxfilters, should be == PTP_RXFILTERS_LEN
+ * @rxfilters_mcast: Receive filters for multicast PTP packets
+ * @rxfilters_ucast: Receive filters for unicast PTP packets
  * @config: Current timestamp configuration
  * @enabled: PTP operation enabled
  * @mode: Mode in which PTP operating (PTP version)
@@ -298,9 +319,10 @@ struct efx_ptp_data {
 	struct efx_ptp_event_rx rx_evts[MAX_RECEIVE_EVENTS];
 	struct workqueue_struct *workwq;
 	struct work_struct work;
+	struct delayed_work cleanup_work;
 	bool reset_required;
-	u32 rxfilters[PTP_RXFILTERS_LEN];
-	size_t rxfilters_count;
+	struct list_head rxfilters_mcast;
+	struct list_head rxfilters_ucast;
 	struct hwtstamp_config config;
 	bool enabled;
 	unsigned int mode;
@@ -358,6 +380,8 @@ static int efx_phc_settime(struct ptp_clock_info *ptp,
 			   const struct timespec64 *e_ts);
 static int efx_phc_enable(struct ptp_clock_info *ptp,
 			  struct ptp_clock_request *request, int on);
+static int efx_ptp_insert_unicast_filter(struct efx_nic *efx,
+					 struct sk_buff *skb);
 
 bool efx_ptp_use_mac_tx_timestamps(struct efx_nic *efx)
 {
@@ -1103,6 +1127,8 @@ static void efx_ptp_xmit_skb_queue(struct efx_nic *efx, struct sk_buff *skb)
 
 	tx_queue = efx_channel_get_tx_queue(ptp_data->channel, type);
 	if (tx_queue && tx_queue->timestamping) {
+		skb_get(skb);
+
 		/* This code invokes normal driver TX code which is always
 		 * protected from softirqs when called from generic TX code,
 		 * which in turn disables preemption. Look at __dev_queue_xmit
@@ -1126,6 +1152,13 @@ static void efx_ptp_xmit_skb_queue(struct efx_nic *efx, struct sk_buff *skb)
 		local_bh_disable();
 		efx_enqueue_skb(tx_queue, skb);
 		local_bh_enable();
+
+		/* We need to add the filters after enqueuing the packet.
+		 * Otherwise, there's high latency in sending back the
+		 * timestamp, causing ptp4l timeouts
+		 */
+		efx_ptp_insert_unicast_filter(efx, skb);
+		dev_consume_skb_any(skb);
 	} else {
 		WARN_ONCE(1, "PTP channel has no timestamped tx queue\n");
 		dev_kfree_skb_any(skb);
@@ -1135,11 +1168,11 @@ static void efx_ptp_xmit_skb_queue(struct efx_nic *efx, struct sk_buff *skb)
 /* Transmit a PTP packet, via the MCDI interface, to the wire. */
 static void efx_ptp_xmit_skb_mc(struct efx_nic *efx, struct sk_buff *skb)
 {
+	MCDI_DECLARE_BUF(txtime, MC_CMD_PTP_OUT_TRANSMIT_LEN);
 	struct efx_ptp_data *ptp_data = efx->ptp_data;
 	struct skb_shared_hwtstamps timestamps;
-	int rc = -EIO;
-	MCDI_DECLARE_BUF(txtime, MC_CMD_PTP_OUT_TRANSMIT_LEN);
 	size_t len;
+	int rc;
 
 	MCDI_SET_DWORD(ptp_data->txbuf, PTP_IN_OP, MC_CMD_PTP_OP_TRANSMIT);
 	MCDI_SET_DWORD(ptp_data->txbuf, PTP_IN_PERIPH_ID, 0);
@@ -1173,7 +1206,10 @@ static void efx_ptp_xmit_skb_mc(struct efx_nic *efx, struct sk_buff *skb)
 
 	skb_tstamp_tx(skb, &timestamps);
 
-	rc = 0;
+	/* Add the filters after sending back the timestamp to avoid delaying it
+	 * or ptp4l may timeout.
+	 */
+	efx_ptp_insert_unicast_filter(efx, skb);
 
 fail:
 	dev_kfree_skb_any(skb);
@@ -1289,15 +1325,37 @@ static inline void efx_ptp_process_rx(struct efx_nic *efx, struct sk_buff *skb)
 	local_bh_enable();
 }
 
-static void efx_ptp_remove_multicast_filters(struct efx_nic *efx)
+static struct efx_ptp_rxfilter *
+efx_ptp_find_filter(struct list_head *filter_list, struct efx_filter_spec *spec)
 {
-	struct efx_ptp_data *ptp = efx->ptp_data;
+	struct efx_ptp_rxfilter *rxfilter;
 
-	while (ptp->rxfilters_count) {
-		ptp->rxfilters_count--;
-		efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,
-					  ptp->rxfilters[ptp->rxfilters_count]);
+	list_for_each_entry(rxfilter, filter_list, list) {
+		if (rxfilter->ether_type == spec->ether_type &&
+		    rxfilter->loc_port == spec->loc_port &&
+		    !memcmp(rxfilter->loc_host, spec->loc_host, sizeof(spec->loc_host)))
+			return rxfilter;
 	}
+
+	return NULL;
+}
+
+static void efx_ptp_remove_one_filter(struct efx_nic *efx,
+				      struct efx_ptp_rxfilter *rxfilter)
+{
+	efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,
+				  rxfilter->handle);
+	list_del(&rxfilter->list);
+	kfree(rxfilter);
+}
+
+static void efx_ptp_remove_filters(struct efx_nic *efx,
+				   struct list_head *filter_list)
+{
+	struct efx_ptp_rxfilter *rxfilter, *tmp;
+
+	list_for_each_entry_safe(rxfilter, tmp, filter_list, list)
+		efx_ptp_remove_one_filter(efx, rxfilter);
 }
 
 static void efx_ptp_init_filter(struct efx_nic *efx,
@@ -1311,48 +1369,80 @@ static void efx_ptp_init_filter(struct efx_nic *efx,
 }
 
 static int efx_ptp_insert_filter(struct efx_nic *efx,
-				 struct efx_filter_spec *rxfilter)
+				 struct list_head *filter_list,
+				 struct efx_filter_spec *spec,
+				 unsigned long expiry)
 {
 	struct efx_ptp_data *ptp = efx->ptp_data;
+	struct efx_ptp_rxfilter *rxfilter;
+	int rc;
+
+	rxfilter = efx_ptp_find_filter(filter_list, spec);
+	if (rxfilter) {
+		rxfilter->expiry = expiry;
+		return 0;
+	}
+
+	rxfilter = kzalloc(sizeof(*rxfilter), GFP_KERNEL);
+	if (!rxfilter)
+		return -ENOMEM;
 
-	int rc = efx_filter_insert_filter(efx, rxfilter, true);
+	rc = efx_filter_insert_filter(efx, spec, true);
 	if (rc < 0)
-		return rc;
-	ptp->rxfilters[ptp->rxfilters_count] = rc;
-	ptp->rxfilters_count++;
+		goto fail;
+
+	rxfilter->handle = rc;
+	rxfilter->ether_type = spec->ether_type;
+	rxfilter->loc_port = spec->loc_port;
+	memcpy(rxfilter->loc_host, spec->loc_host, sizeof(spec->loc_host));
+	rxfilter->expiry = expiry;
+	list_add(&rxfilter->list, filter_list);
+
+	queue_delayed_work(ptp->workwq, &ptp->cleanup_work,
+			   UCAST_FILTER_EXPIRY_JIFFIES + 1);
+
 	return 0;
+
+fail:
+	kfree(rxfilter);
+	return rc;
 }
 
-static int efx_ptp_insert_ipv4_filter(struct efx_nic *efx, u16 port)
+static int efx_ptp_insert_ipv4_filter(struct efx_nic *efx,
+				      struct list_head *filter_list,
+				      __be32 addr, u16 port,
+				      unsigned long expiry)
 {
-	struct efx_filter_spec rxfilter;
+	struct efx_filter_spec spec;
 
-	efx_ptp_init_filter(efx, &rxfilter);
-	efx_filter_set_ipv4_local(&rxfilter, IPPROTO_UDP, htonl(PTP_ADDR_IPV4),
-				  htons(port));
-	return efx_ptp_insert_filter(efx, &rxfilter);
+	efx_ptp_init_filter(efx, &spec);
+	efx_filter_set_ipv4_local(&spec, IPPROTO_UDP, addr, htons(port));
+	return efx_ptp_insert_filter(efx, filter_list, &spec, expiry);
 }
 
-static int efx_ptp_insert_ipv6_filter(struct efx_nic *efx, u16 port)
+static int efx_ptp_insert_ipv6_filter(struct efx_nic *efx,
+				      struct list_head *filter_list,
+				      struct in6_addr *addr, u16 port,
+				      unsigned long expiry)
 {
-	const struct in6_addr addr = {{PTP_ADDR_IPV6}};
-	struct efx_filter_spec rxfilter;
+	struct efx_filter_spec spec;
 
-	efx_ptp_init_filter(efx, &rxfilter);
-	efx_filter_set_ipv6_local(&rxfilter, IPPROTO_UDP, &addr, htons(port));
-	return efx_ptp_insert_filter(efx, &rxfilter);
+	efx_ptp_init_filter(efx, &spec);
+	efx_filter_set_ipv6_local(&spec, IPPROTO_UDP, addr, htons(port));
+	return efx_ptp_insert_filter(efx, filter_list, &spec, expiry);
 }
 
-static int efx_ptp_insert_eth_filter(struct efx_nic *efx)
+static int efx_ptp_insert_eth_multicast_filter(struct efx_nic *efx)
 {
+	struct efx_ptp_data *ptp = efx->ptp_data;
 	const u8 addr[ETH_ALEN] = PTP_ADDR_ETHER;
-	struct efx_filter_spec rxfilter;
+	struct efx_filter_spec spec;
 
-	efx_ptp_init_filter(efx, &rxfilter);
-	efx_filter_set_eth_local(&rxfilter, EFX_FILTER_VID_UNSPEC, addr);
-	rxfilter.match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
-	rxfilter.ether_type = htons(ETH_P_1588);
-	return efx_ptp_insert_filter(efx, &rxfilter);
+	efx_ptp_init_filter(efx, &spec);
+	efx_filter_set_eth_local(&spec, EFX_FILTER_VID_UNSPEC, addr);
+	spec.match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
+	spec.ether_type = htons(ETH_P_1588);
+	return efx_ptp_insert_filter(efx, &ptp->rxfilters_mcast, &spec, 0);
 }
 
 static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)
@@ -1360,17 +1450,21 @@ static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)
 	struct efx_ptp_data *ptp = efx->ptp_data;
 	int rc;
 
-	if (!ptp->channel || ptp->rxfilters_count)
+	if (!ptp->channel || !list_empty(&ptp->rxfilters_mcast))
 		return 0;
 
 	/* Must filter on both event and general ports to ensure
 	 * that there is no packet re-ordering.
 	 */
-	rc = efx_ptp_insert_ipv4_filter(efx, PTP_EVENT_PORT);
+	rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_mcast,
+					htonl(PTP_ADDR_IPV4), PTP_EVENT_PORT,
+					0);
 	if (rc < 0)
 		goto fail;
 
-	rc = efx_ptp_insert_ipv4_filter(efx, PTP_GENERAL_PORT);
+	rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_mcast,
+					htonl(PTP_ADDR_IPV4), PTP_GENERAL_PORT,
+					0);
 	if (rc < 0)
 		goto fail;
 
@@ -1378,15 +1472,19 @@ static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)
 	 * PTP over IPv6 and Ethernet
 	 */
 	if (efx_ptp_use_mac_tx_timestamps(efx)) {
-		rc = efx_ptp_insert_ipv6_filter(efx, PTP_EVENT_PORT);
+		struct in6_addr ipv6_addr = {{PTP_ADDR_IPV6}};
+
+		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_mcast,
+						&ipv6_addr, PTP_EVENT_PORT, 0);
 		if (rc < 0)
 			goto fail;
 
-		rc = efx_ptp_insert_ipv6_filter(efx, PTP_GENERAL_PORT);
+		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_mcast,
+						&ipv6_addr, PTP_GENERAL_PORT, 0);
 		if (rc < 0)
 			goto fail;
 
-		rc = efx_ptp_insert_eth_filter(efx);
+		rc = efx_ptp_insert_eth_multicast_filter(efx);
 		if (rc < 0)
 			goto fail;
 	}
@@ -1394,7 +1492,64 @@ static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)
 	return 0;
 
 fail:
-	efx_ptp_remove_multicast_filters(efx);
+	efx_ptp_remove_filters(efx, &ptp->rxfilters_mcast);
+	return rc;
+}
+
+static bool efx_ptp_valid_unicast_event_pkt(struct sk_buff *skb)
+{
+	if (skb->protocol == htons(ETH_P_IP)) {
+		return ip_hdr(skb)->daddr != htonl(PTP_ADDR_IPV4) &&
+			ip_hdr(skb)->protocol == IPPROTO_UDP &&
+			udp_hdr(skb)->source == htons(PTP_EVENT_PORT);
+	} else if (skb->protocol == htons(ETH_P_IPV6)) {
+		struct in6_addr mcast_addr = {{PTP_ADDR_IPV6}};
+
+		return !ipv6_addr_equal(&ipv6_hdr(skb)->daddr, &mcast_addr) &&
+			ipv6_hdr(skb)->nexthdr == IPPROTO_UDP &&
+			udp_hdr(skb)->source == htons(PTP_EVENT_PORT);
+	}
+	return false;
+}
+
+static int efx_ptp_insert_unicast_filter(struct efx_nic *efx,
+					 struct sk_buff *skb)
+{
+	struct efx_ptp_data *ptp = efx->ptp_data;
+	unsigned long expiry;
+	int rc;
+
+	if (!efx_ptp_valid_unicast_event_pkt(skb))
+		return -EINVAL;
+
+	expiry = jiffies + UCAST_FILTER_EXPIRY_JIFFIES;
+
+	if (skb->protocol == htons(ETH_P_IP)) {
+		__be32 addr = ip_hdr(skb)->saddr;
+
+		rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_ucast,
+						addr, PTP_EVENT_PORT, expiry);
+		if (rc < 0)
+			goto out;
+
+		rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_ucast,
+						addr, PTP_GENERAL_PORT, expiry);
+	} else if (efx_ptp_use_mac_tx_timestamps(efx)) {
+		/* IPv6 PTP only supported by devices with MAC hw timestamp */
+		struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
+
+		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_ucast,
+						addr, PTP_EVENT_PORT, expiry);
+		if (rc < 0)
+			goto out;
+
+		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_ucast,
+						addr, PTP_GENERAL_PORT, expiry);
+	} else {
+		return -EOPNOTSUPP;
+	}
+
+out:
 	return rc;
 }
 
@@ -1419,7 +1574,7 @@ static int efx_ptp_start(struct efx_nic *efx)
 	return 0;
 
 fail:
-	efx_ptp_remove_multicast_filters(efx);
+	efx_ptp_remove_filters(efx, &ptp->rxfilters_mcast);
 	return rc;
 }
 
@@ -1435,7 +1590,8 @@ static int efx_ptp_stop(struct efx_nic *efx)
 
 	rc = efx_ptp_disable(efx);
 
-	efx_ptp_remove_multicast_filters(efx);
+	efx_ptp_remove_filters(efx, &ptp->rxfilters_mcast);
+	efx_ptp_remove_filters(efx, &ptp->rxfilters_ucast);
 
 	/* Make sure RX packets are really delivered */
 	efx_ptp_deliver_rx_queue(&efx->ptp_data->rxq);
@@ -1499,6 +1655,23 @@ static void efx_ptp_worker(struct work_struct *work)
 		efx_ptp_process_rx(efx, skb);
 }
 
+static void efx_ptp_cleanup_worker(struct work_struct *work)
+{
+	struct efx_ptp_data *ptp =
+		container_of(work, struct efx_ptp_data, cleanup_work.work);
+	struct efx_ptp_rxfilter *rxfilter, *tmp;
+
+	list_for_each_entry_safe(rxfilter, tmp, &ptp->rxfilters_ucast, list) {
+		if (time_is_before_jiffies(rxfilter->expiry))
+			efx_ptp_remove_one_filter(ptp->efx, rxfilter);
+	}
+
+	if (!list_empty(&ptp->rxfilters_ucast)) {
+		queue_delayed_work(ptp->workwq, &ptp->cleanup_work,
+				   UCAST_FILTER_EXPIRY_JIFFIES + 1);
+	}
+}
+
 static const struct ptp_clock_info efx_phc_clock_info = {
 	.owner		= THIS_MODULE,
 	.name		= "sfc",
@@ -1557,6 +1730,7 @@ int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel)
 	}
 
 	INIT_WORK(&ptp->work, efx_ptp_worker);
+	INIT_DELAYED_WORK(&ptp->cleanup_work, efx_ptp_cleanup_worker);
 	ptp->config.flags = 0;
 	ptp->config.tx_type = HWTSTAMP_TX_OFF;
 	ptp->config.rx_filter = HWTSTAMP_FILTER_NONE;
@@ -1566,6 +1740,9 @@ int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel)
 	for (pos = 0; pos < MAX_RECEIVE_EVENTS; pos++)
 		list_add(&ptp->rx_evts[pos].link, &ptp->evt_free_list);
 
+	INIT_LIST_HEAD(&ptp->rxfilters_mcast);
+	INIT_LIST_HEAD(&ptp->rxfilters_ucast);
+
 	/* Get the NIC PTP attributes and set up time conversions */
 	rc = efx_ptp_get_attributes(efx);
 	if (rc < 0)
@@ -1645,6 +1822,7 @@ void efx_ptp_remove(struct efx_nic *efx)
 	(void)efx_ptp_disable(efx);
 
 	cancel_work_sync(&efx->ptp_data->work);
+	cancel_delayed_work_sync(&efx->ptp_data->cleanup_work);
 	if (efx->ptp_data->pps_workwq)
 		cancel_work_sync(&efx->ptp_data->pps_work);
 
diff --git a/drivers/net/ethernet/sfc/siena/efx.c b/drivers/net/ethernet/sfc/siena/efx.c
index ef52ec71d197..8c557f6a183c 100644
--- a/drivers/net/ethernet/sfc/siena/efx.c
+++ b/drivers/net/ethernet/sfc/siena/efx.c
@@ -18,7 +18,6 @@
 #include <linux/ethtool.h>
 #include <linux/topology.h>
 #include <linux/gfp.h>
-#include <linux/aer.h>
 #include <linux/interrupt.h>
 #include "net_driver.h"
 #include <net/gre.h>
@@ -874,8 +873,6 @@ static void efx_pci_remove(struct pci_dev *pci_dev)
 
 	efx_siena_fini_struct(efx);
 	free_netdev(efx->net_dev);
-
-	pci_disable_pcie_error_reporting(pci_dev);
 };
 
 /* NIC VPD information
@@ -1094,8 +1091,6 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
 		netif_warn(efx, probe, efx->net_dev,
 			   "failed to create MTDs (%d)\n", rc);
 
-	(void)pci_enable_pcie_error_reporting(pci_dev);
-
 	if (efx->type->udp_tnl_push_ports)
 		efx->type->udp_tnl_push_ports(efx);
 
diff --git a/drivers/net/ethernet/sfc/tc.c b/drivers/net/ethernet/sfc/tc.c
index deeaab9ee761..0327639a628a 100644
--- a/drivers/net/ethernet/sfc/tc.c
+++ b/drivers/net/ethernet/sfc/tc.c
@@ -10,12 +10,24 @@
  */
 
 #include <net/pkt_cls.h>
+#include <net/vxlan.h>
+#include <net/geneve.h>
 #include "tc.h"
 #include "tc_bindings.h"
 #include "mae.h"
 #include "ef100_rep.h"
 #include "efx.h"
 
+static enum efx_encap_type efx_tc_indr_netdev_type(struct net_device *net_dev)
+{
+	if (netif_is_vxlan(net_dev))
+		return EFX_ENCAP_TYPE_VXLAN;
+	if (netif_is_geneve(net_dev))
+		return EFX_ENCAP_TYPE_GENEVE;
+
+	return EFX_ENCAP_TYPE_NONE;
+}
+
 #define EFX_EFV_PF	NULL
 /* Look up the representor information (efv) for a device.
  * May return NULL for the PF (us), or an error pointer for a device that
@@ -43,6 +55,20 @@ static struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx,
 	return efv;
 }
 
+/* Convert a driver-internal vport ID into an internal device (PF or VF) */
+static s64 efx_tc_flower_internal_mport(struct efx_nic *efx, struct efx_rep *efv)
+{
+	u32 mport;
+
+	if (IS_ERR(efv))
+		return PTR_ERR(efv);
+	if (!efv) /* device is PF (us) */
+		efx_mae_mport_uplink(efx, &mport);
+	else /* device is repr */
+		efx_mae_mport_mport(efx, efv->mport, &mport);
+	return mport;
+}
+
 /* Convert a driver-internal vport ID into an external device (wire or VF) */
 static s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv)
 {
@@ -57,6 +83,12 @@ static s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv
 	return mport;
 }
 
+static const struct rhashtable_params efx_tc_encap_match_ht_params = {
+	.key_len	= offsetof(struct efx_tc_encap_match, linkage),
+	.key_offset	= 0,
+	.head_offset	= offsetof(struct efx_tc_encap_match, linkage),
+};
+
 static const struct rhashtable_params efx_tc_match_action_ht_params = {
 	.key_len	= sizeof(unsigned long),
 	.key_offset	= offsetof(struct efx_tc_flow_rule, cookie),
@@ -66,7 +98,7 @@ static const struct rhashtable_params efx_tc_match_action_ht_params = {
 static void efx_tc_free_action_set(struct efx_nic *efx,
 				   struct efx_tc_action_set *act, bool in_hw)
 {
-	/* Failure paths calling this on the 'running action' set in_hw=false,
+	/* Failure paths calling this on the 'cursor' action set in_hw=false,
 	 * because if the alloc had succeeded we'd've put it in acts.list and
 	 * not still have it in act.
 	 */
@@ -100,15 +132,6 @@ static void efx_tc_free_action_set_list(struct efx_nic *efx,
 	/* Don't kfree, as acts is embedded inside a struct efx_tc_flow_rule */
 }
 
-static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule)
-{
-	efx_mae_delete_rule(efx, rule->fw_id);
-
-	/* Release entries in subsidiary tables */
-	efx_tc_free_action_set_list(efx, &rule->acts, true);
-	rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
-}
-
 static void efx_tc_flow_free(void *ptr, void *arg)
 {
 	struct efx_tc_flow_rule *rule = ptr;
@@ -193,6 +216,11 @@ static int efx_tc_flower_parse_match(struct efx_nic *efx,
 	      BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 	      BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 	      BIT(FLOW_DISSECTOR_KEY_PORTS) |
+	      BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
+	      BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
+	      BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
+	      BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) |
+	      BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) |
 	      BIT(FLOW_DISSECTOR_KEY_TCP) |
 	      BIT(FLOW_DISSECTOR_KEY_IP))) {
 		NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#x",
@@ -280,12 +308,228 @@ static int efx_tc_flower_parse_match(struct efx_nic *efx,
 	MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport);
 	MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport);
 	MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags);
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
+		struct flow_match_control fm;
 
+		flow_rule_match_enc_control(rule, &fm);
+		if (fm.mask->flags) {
+			NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on enc_control.flags %#x",
+					       fm.mask->flags);
+			return -EOPNOTSUPP;
+		}
+		if (!IS_ALL_ONES(fm.mask->addr_type)) {
+			NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)",
+					       fm.mask->addr_type,
+					       fm.key->addr_type);
+			return -EOPNOTSUPP;
+		}
+		switch (fm.key->addr_type) {
+		case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
+			MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
+					     src, enc_src_ip);
+			MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
+					     dst, enc_dst_ip);
+			break;
+#ifdef CONFIG_IPV6
+		case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
+			MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
+					     src, enc_src_ip6);
+			MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
+					     dst, enc_dst_ip6);
+			break;
+#endif
+		default:
+			NL_SET_ERR_MSG_FMT_MOD(extack,
+					       "Unsupported enc addr_type %u (supported are IPv4, IPv6)",
+					       fm.key->addr_type);
+			return -EOPNOTSUPP;
+		}
+		MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, tos, enc_ip_tos);
+		MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, ttl, enc_ip_ttl);
+		MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, src, enc_sport);
+		MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, dst, enc_dport);
+		MAP_ENC_KEY_AND_MASK(KEYID, enc_keyid, enc_keyid, keyid, enc_keyid);
+	} else if (dissector->used_keys &
+		   (BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
+		    BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
+		    BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
+		    BIT(FLOW_DISSECTOR_KEY_ENC_IP) |
+		    BIT(FLOW_DISSECTOR_KEY_ENC_PORTS))) {
+		NL_SET_ERR_MSG_FMT_MOD(extack, "Flower enc keys require enc_control (keys: %#x)",
+				       dissector->used_keys);
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int efx_tc_flower_record_encap_match(struct efx_nic *efx,
+					    struct efx_tc_match *match,
+					    enum efx_encap_type type,
+					    struct netlink_ext_ack *extack)
+{
+	struct efx_tc_encap_match *encap, *old;
+	bool ipv6 = false;
+	int rc;
+
+	/* We require that the socket-defining fields (IP addrs and UDP dest
+	 * port) are present and exact-match.  Other fields are currently not
+	 * allowed.  This meets what OVS will ask for, and means that we don't
+	 * need to handle difficult checks for overlapping matches as could
+	 * come up if we allowed masks or varying sets of match fields.
+	 */
+	if (match->mask.enc_dst_ip | match->mask.enc_src_ip) {
+		if (!IS_ALL_ONES(match->mask.enc_dst_ip)) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Egress encap match is not exact on dst IP address");
+			return -EOPNOTSUPP;
+		}
+		if (!IS_ALL_ONES(match->mask.enc_src_ip)) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Egress encap match is not exact on src IP address");
+			return -EOPNOTSUPP;
+		}
+#ifdef CONFIG_IPV6
+		if (!ipv6_addr_any(&match->mask.enc_dst_ip6) ||
+		    !ipv6_addr_any(&match->mask.enc_src_ip6)) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Egress encap match on both IPv4 and IPv6, don't understand");
+			return -EOPNOTSUPP;
+		}
+	} else {
+		ipv6 = true;
+		if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Egress encap match is not exact on dst IP address");
+			return -EOPNOTSUPP;
+		}
+		if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Egress encap match is not exact on src IP address");
+			return -EOPNOTSUPP;
+		}
+#endif
+	}
+	if (!IS_ALL_ONES(match->mask.enc_dport)) {
+		NL_SET_ERR_MSG_MOD(extack, "Egress encap match is not exact on dst UDP port");
+		return -EOPNOTSUPP;
+	}
+	if (match->mask.enc_sport) {
+		NL_SET_ERR_MSG_MOD(extack, "Egress encap match on src UDP port not supported");
+		return -EOPNOTSUPP;
+	}
+	if (match->mask.enc_ip_tos) {
+		NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP ToS not supported");
+		return -EOPNOTSUPP;
+	}
+	if (match->mask.enc_ip_ttl) {
+		NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP TTL not supported");
+		return -EOPNOTSUPP;
+	}
+
+	rc = efx_mae_check_encap_match_caps(efx, ipv6, extack);
+	if (rc) {
+		NL_SET_ERR_MSG_FMT_MOD(extack, "MAE hw reports no support for IPv%d encap matches",
+				       ipv6 ? 6 : 4);
+		return -EOPNOTSUPP;
+	}
+
+	encap = kzalloc(sizeof(*encap), GFP_USER);
+	if (!encap)
+		return -ENOMEM;
+	encap->src_ip = match->value.enc_src_ip;
+	encap->dst_ip = match->value.enc_dst_ip;
+#ifdef CONFIG_IPV6
+	encap->src_ip6 = match->value.enc_src_ip6;
+	encap->dst_ip6 = match->value.enc_dst_ip6;
+#endif
+	encap->udp_dport = match->value.enc_dport;
+	encap->tun_type = type;
+	old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht,
+						&encap->linkage,
+						efx_tc_encap_match_ht_params);
+	if (old) {
+		/* don't need our new entry */
+		kfree(encap);
+		if (old->tun_type != type) {
+			NL_SET_ERR_MSG_FMT_MOD(extack,
+					       "Egress encap match with conflicting tun_type %u != %u",
+					       old->tun_type, type);
+			return -EEXIST;
+		}
+		if (!refcount_inc_not_zero(&old->ref))
+			return -EAGAIN;
+		/* existing entry found */
+		encap = old;
+	} else {
+		rc = efx_mae_register_encap_match(efx, encap);
+		if (rc) {
+			NL_SET_ERR_MSG_MOD(extack, "Failed to record egress encap match in HW");
+			goto fail;
+		}
+		refcount_set(&encap->ref, 1);
+	}
+	match->encap = encap;
 	return 0;
+fail:
+	rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,
+			       efx_tc_encap_match_ht_params);
+	kfree(encap);
+	return rc;
+}
+
+static void efx_tc_flower_release_encap_match(struct efx_nic *efx,
+					      struct efx_tc_encap_match *encap)
+{
+	int rc;
+
+	if (!refcount_dec_and_test(&encap->ref))
+		return; /* still in use */
+
+	rc = efx_mae_unregister_encap_match(efx, encap);
+	if (rc)
+		/* Display message but carry on and remove entry from our
+		 * SW tables, because there's not much we can do about it.
+		 */
+		netif_err(efx, drv, efx->net_dev,
+			  "Failed to release encap match %#x, rc %d\n",
+			  encap->fw_id, rc);
+	rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,
+			       efx_tc_encap_match_ht_params);
+	kfree(encap);
+}
+
+static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule)
+{
+	efx_mae_delete_rule(efx, rule->fw_id);
+
+	/* Release entries in subsidiary tables */
+	efx_tc_free_action_set_list(efx, &rule->acts, true);
+	if (rule->match.encap)
+		efx_tc_flower_release_encap_match(efx, rule->match.encap);
+	rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
+}
+
+static const char *efx_tc_encap_type_name(enum efx_encap_type typ)
+{
+	switch (typ) {
+	case EFX_ENCAP_TYPE_NONE:
+		return "none";
+	case EFX_ENCAP_TYPE_VXLAN:
+		return "vxlan";
+	case EFX_ENCAP_TYPE_GENEVE:
+		return "geneve";
+	default:
+		pr_warn_once("Unknown efx_encap_type %d encountered\n", typ);
+		return "unknown";
+	}
 }
 
 /* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */
 enum efx_tc_action_order {
+	EFX_TC_AO_DECAP,
+	EFX_TC_AO_VLAN_POP,
+	EFX_TC_AO_VLAN_PUSH,
 	EFX_TC_AO_COUNT,
 	EFX_TC_AO_DELIVER
 };
@@ -294,6 +538,24 @@ static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act,
 					  enum efx_tc_action_order new)
 {
 	switch (new) {
+	case EFX_TC_AO_DECAP:
+		if (act->decap)
+			return false;
+		fallthrough;
+	case EFX_TC_AO_VLAN_POP:
+		if (act->vlan_pop >= 2)
+			return false;
+		/* If we've already pushed a VLAN, we can't then pop it;
+		 * the hardware would instead try to pop an existing VLAN
+		 * before pushing the new one.
+		 */
+		if (act->vlan_push)
+			return false;
+		fallthrough;
+	case EFX_TC_AO_VLAN_PUSH:
+		if (act->vlan_push >= 2)
+			return false;
+		fallthrough;
 	case EFX_TC_AO_COUNT:
 		if (act->count)
 			return false;
@@ -307,6 +569,286 @@ static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act,
 	}
 }
 
+static int efx_tc_flower_replace_foreign(struct efx_nic *efx,
+					 struct net_device *net_dev,
+					 struct flow_cls_offload *tc)
+{
+	struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
+	struct netlink_ext_ack *extack = tc->common.extack;
+	struct efx_tc_flow_rule *rule = NULL, *old = NULL;
+	struct efx_tc_action_set *act = NULL;
+	bool found = false, uplinked = false;
+	const struct flow_action_entry *fa;
+	struct efx_tc_match match;
+	struct efx_rep *to_efv;
+	s64 rc;
+	int i;
+
+	/* Parse match */
+	memset(&match, 0, sizeof(match));
+	rc = efx_tc_flower_parse_match(efx, fr, &match, NULL);
+	if (rc)
+		return rc;
+	/* The rule as given to us doesn't specify a source netdevice.
+	 * But, determining whether packets from a VF should match it is
+	 * complicated, so leave those to the software slowpath: qualify
+	 * the filter with source m-port == wire.
+	 */
+	rc = efx_tc_flower_external_mport(efx, EFX_EFV_PF);
+	if (rc < 0) {
+		NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter");
+		return rc;
+	}
+	match.value.ingress_port = rc;
+	match.mask.ingress_port = ~0;
+
+	if (tc->common.chain_index) {
+		NL_SET_ERR_MSG_MOD(extack, "No support for nonzero chain_index");
+		return -EOPNOTSUPP;
+	}
+	match.mask.recirc_id = 0xff;
+
+	flow_action_for_each(i, fa, &fr->action) {
+		switch (fa->id) {
+		case FLOW_ACTION_REDIRECT:
+		case FLOW_ACTION_MIRRED: /* mirred means mirror here */
+			to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
+			if (IS_ERR(to_efv))
+				continue;
+			found = true;
+			break;
+		default:
+			break;
+		}
+	}
+	if (!found) { /* We don't care. */
+		netif_dbg(efx, drv, efx->net_dev,
+			  "Ignoring foreign filter that doesn't egdev us\n");
+		rc = -EOPNOTSUPP;
+		goto release;
+	}
+
+	rc = efx_mae_match_check_caps(efx, &match.mask, NULL);
+	if (rc)
+		goto release;
+
+	if (efx_tc_match_is_encap(&match.mask)) {
+		enum efx_encap_type type;
+
+		type = efx_tc_indr_netdev_type(net_dev);
+		if (type == EFX_ENCAP_TYPE_NONE) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Egress encap match on unsupported tunnel device");
+			rc = -EOPNOTSUPP;
+			goto release;
+		}
+
+		rc = efx_mae_check_encap_type_supported(efx, type);
+		if (rc) {
+			NL_SET_ERR_MSG_FMT_MOD(extack,
+					       "Firmware reports no support for %s encap match",
+					       efx_tc_encap_type_name(type));
+			goto release;
+		}
+
+		rc = efx_tc_flower_record_encap_match(efx, &match, type,
+						      extack);
+		if (rc)
+			goto release;
+	} else {
+		/* This is not a tunnel decap rule, ignore it */
+		netif_dbg(efx, drv, efx->net_dev,
+			  "Ignoring foreign filter without encap match\n");
+		rc = -EOPNOTSUPP;
+		goto release;
+	}
+
+	rule = kzalloc(sizeof(*rule), GFP_USER);
+	if (!rule) {
+		rc = -ENOMEM;
+		goto release;
+	}
+	INIT_LIST_HEAD(&rule->acts.list);
+	rule->cookie = tc->cookie;
+	old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,
+						&rule->linkage,
+						efx_tc_match_action_ht_params);
+	if (old) {
+		netif_dbg(efx, drv, efx->net_dev,
+			  "Ignoring already-offloaded rule (cookie %lx)\n",
+			  tc->cookie);
+		rc = -EEXIST;
+		goto release;
+	}
+
+	act = kzalloc(sizeof(*act), GFP_USER);
+	if (!act) {
+		rc = -ENOMEM;
+		goto release;
+	}
+
+	/* Parse actions.  For foreign rules we only support decap & redirect.
+	 * See corresponding code in efx_tc_flower_replace() for theory of
+	 * operation & how 'act' cursor is used.
+	 */
+	flow_action_for_each(i, fa, &fr->action) {
+		struct efx_tc_action_set save;
+
+		switch (fa->id) {
+		case FLOW_ACTION_REDIRECT:
+		case FLOW_ACTION_MIRRED:
+			/* See corresponding code in efx_tc_flower_replace() for
+			 * long explanations of what's going on here.
+			 */
+			save = *act;
+			if (fa->hw_stats) {
+				struct efx_tc_counter_index *ctr;
+
+				if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
+					NL_SET_ERR_MSG_FMT_MOD(extack,
+							       "hw_stats_type %u not supported (only 'delayed')",
+							       fa->hw_stats);
+					rc = -EOPNOTSUPP;
+					goto release;
+				}
+				if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
+					rc = -EOPNOTSUPP;
+					goto release;
+				}
+
+				ctr = efx_tc_flower_get_counter_index(efx,
+								      tc->cookie,
+								      EFX_TC_COUNTER_TYPE_AR);
+				if (IS_ERR(ctr)) {
+					rc = PTR_ERR(ctr);
+					NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
+					goto release;
+				}
+				act->count = ctr;
+			}
+
+			if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
+				/* can't happen */
+				rc = -EOPNOTSUPP;
+				NL_SET_ERR_MSG_MOD(extack,
+						   "Deliver action violates action order (can't happen)");
+				goto release;
+			}
+			to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
+			/* PF implies egdev is us, in which case we really
+			 * want to deliver to the uplink (because this is an
+			 * ingress filter).  If we don't recognise the egdev
+			 * at all, then we'd better trap so SW can handle it.
+			 */
+			if (IS_ERR(to_efv))
+				to_efv = EFX_EFV_PF;
+			if (to_efv == EFX_EFV_PF) {
+				if (uplinked)
+					break;
+				uplinked = true;
+			}
+			rc = efx_tc_flower_internal_mport(efx, to_efv);
+			if (rc < 0) {
+				NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
+				goto release;
+			}
+			act->dest_mport = rc;
+			act->deliver = 1;
+			rc = efx_mae_alloc_action_set(efx, act);
+			if (rc) {
+				NL_SET_ERR_MSG_MOD(extack,
+						   "Failed to write action set to hw (mirred)");
+				goto release;
+			}
+			list_add_tail(&act->list, &rule->acts.list);
+			act = NULL;
+			if (fa->id == FLOW_ACTION_REDIRECT)
+				break; /* end of the line */
+			/* Mirror, so continue on with saved act */
+			act = kzalloc(sizeof(*act), GFP_USER);
+			if (!act) {
+				rc = -ENOMEM;
+				goto release;
+			}
+			*act = save;
+			break;
+		case FLOW_ACTION_TUNNEL_DECAP:
+			if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DECAP)) {
+				rc = -EINVAL;
+				NL_SET_ERR_MSG_MOD(extack, "Decap action violates action order");
+				goto release;
+			}
+			act->decap = 1;
+			/* If we previously delivered/trapped to uplink, now
+			 * that we've decapped we'll want another copy if we
+			 * try to deliver/trap to uplink again.
+			 */
+			uplinked = false;
+			break;
+		default:
+			NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
+					       fa->id);
+			rc = -EOPNOTSUPP;
+			goto release;
+		}
+	}
+
+	if (act) {
+		if (!uplinked) {
+			/* Not shot/redirected, so deliver to default dest (which is
+			 * the uplink, as this is an ingress filter)
+			 */
+			efx_mae_mport_uplink(efx, &act->dest_mport);
+			act->deliver = 1;
+		}
+		rc = efx_mae_alloc_action_set(efx, act);
+		if (rc) {
+			NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
+			goto release;
+		}
+		list_add_tail(&act->list, &rule->acts.list);
+		act = NULL; /* Prevent double-free in error path */
+	}
+
+	rule->match = match;
+
+	netif_dbg(efx, drv, efx->net_dev,
+		  "Successfully parsed foreign filter (cookie %lx)\n",
+		  tc->cookie);
+
+	rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
+	if (rc) {
+		NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
+		goto release;
+	}
+	rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
+				 rule->acts.fw_id, &rule->fw_id);
+	if (rc) {
+		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
+		goto release_acts;
+	}
+	return 0;
+
+release_acts:
+	efx_mae_free_action_set_list(efx, &rule->acts);
+release:
+	/* We failed to insert the rule, so free up any entries we created in
+	 * subsidiary tables.
+	 */
+	if (act)
+		efx_tc_free_action_set(efx, act, false);
+	if (rule) {
+		rhashtable_remove_fast(&efx->tc->match_action_ht,
+				       &rule->linkage,
+				       efx_tc_match_action_ht_params);
+		efx_tc_free_action_set_list(efx, &rule->acts, false);
+	}
+	kfree(rule);
+	if (match.encap)
+		efx_tc_flower_release_encap_match(efx, match.encap);
+	return rc;
+}
+
 static int efx_tc_flower_replace(struct efx_nic *efx,
 				 struct net_device *net_dev,
 				 struct flow_cls_offload *tc,
@@ -331,10 +873,8 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
 
 	from_efv = efx_tc_flower_lookup_efv(efx, net_dev);
 	if (IS_ERR(from_efv)) {
-		/* Might be a tunnel decap rule from an indirect block.
-		 * Support for those not implemented yet.
-		 */
-		return -EOPNOTSUPP;
+		/* Not from our PF or representors, so probably a tunnel dev */
+		return efx_tc_flower_replace_foreign(efx, net_dev, tc);
 	}
 
 	if (efv != from_efv) {
@@ -357,6 +897,11 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
 	rc = efx_tc_flower_parse_match(efx, fr, &match, extack);
 	if (rc)
 		return rc;
+	if (efx_tc_match_is_encap(&match.mask)) {
+		NL_SET_ERR_MSG_MOD(extack, "Ingress enc_key matches not supported");
+		rc = -EOPNOTSUPP;
+		goto release;
+	}
 
 	if (tc->common.chain_index) {
 		NL_SET_ERR_MSG_MOD(extack, "No support for nonzero chain_index");
@@ -391,8 +936,33 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
 		goto release;
 	}
 
+	/**
+	 * DOC: TC action translation
+	 *
+	 * Actions in TC are sequential and cumulative, with delivery actions
+	 * potentially anywhere in the order.  The EF100 MAE, however, takes
+	 * an 'action set list' consisting of 'action sets', each of which is
+	 * applied to the _original_ packet, and consists of a set of optional
+	 * actions in a fixed order with delivery at the end.
+	 * To translate between these two models, we maintain a 'cursor', @act,
+	 * which describes the cumulative effect of all the packet-mutating
+	 * actions encountered so far; on handling a delivery (mirred or drop)
+	 * action, once the action-set has been inserted into hardware, we
+	 * append @act to the action-set list (@rule->acts); if this is a pipe
+	 * action (mirred mirror) we then allocate a new @act with a copy of
+	 * the cursor state _before_ the delivery action, otherwise we set @act
+	 * to %NULL.
+	 * This ensures that every allocated action-set is either attached to
+	 * @rule->acts or pointed to by @act (and never both), and that only
+	 * those action-sets in @rule->acts exist in hardware.  Consequently,
+	 * in the failure path, @act only needs to be freed in memory, whereas
+	 * for @rule->acts we remove each action-set from hardware before
+	 * freeing it (efx_tc_free_action_set_list()), even if the action-set
+	 * list itself is not in hardware.
+	 */
 	flow_action_for_each(i, fa, &fr->action) {
 		struct efx_tc_action_set save;
+		u16 tci;
 
 		if (!act) {
 			/* more actions after a non-pipe action */
@@ -494,6 +1064,31 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
 			}
 			*act = save;
 			break;
+		case FLOW_ACTION_VLAN_POP:
+			if (act->vlan_push) {
+				act->vlan_push--;
+			} else if (efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_POP)) {
+				act->vlan_pop++;
+			} else {
+				NL_SET_ERR_MSG_MOD(extack,
+						   "More than two VLAN pops, or action order violated");
+				rc = -EINVAL;
+				goto release;
+			}
+			break;
+		case FLOW_ACTION_VLAN_PUSH:
+			if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_PUSH)) {
+				rc = -EINVAL;
+				NL_SET_ERR_MSG_MOD(extack,
+						   "More than two VLAN pushes, or action order violated");
+				goto release;
+			}
+			tci = fa->vlan.vid & VLAN_VID_MASK;
+			tci |= fa->vlan.prio << VLAN_PRIO_SHIFT;
+			act->vlan_tci[act->vlan_push] = cpu_to_be16(tci);
+			act->vlan_proto[act->vlan_push] = fa->vlan.proto;
+			act->vlan_push++;
+			break;
 		default:
 			NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
 					       fa->id);
@@ -847,6 +1442,18 @@ void efx_fini_tc(struct efx_nic *efx)
 	efx->tc->up = false;
 }
 
+/* At teardown time, all TC filter rules (and thus all resources they created)
+ * should already have been removed.  If we find any in our hashtables, make a
+ * cursory attempt to clean up the software side.
+ */
+static void efx_tc_encap_match_free(void *ptr, void *__unused)
+{
+	struct efx_tc_encap_match *encap = ptr;
+
+	WARN_ON(refcount_read(&encap->ref));
+	kfree(encap);
+}
+
 int efx_init_struct_tc(struct efx_nic *efx)
 {
 	int rc;
@@ -869,6 +1476,9 @@ int efx_init_struct_tc(struct efx_nic *efx)
 	rc = efx_tc_init_counters(efx);
 	if (rc < 0)
 		goto fail_counters;
+	rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params);
+	if (rc < 0)
+		goto fail_encap_match_ht;
 	rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params);
 	if (rc < 0)
 		goto fail_match_action_ht;
@@ -881,6 +1491,8 @@ int efx_init_struct_tc(struct efx_nic *efx)
 	efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type;
 	return 0;
 fail_match_action_ht:
+	rhashtable_destroy(&efx->tc->encap_match_ht);
+fail_encap_match_ht:
 	efx_tc_destroy_counters(efx);
 fail_counters:
 	mutex_destroy(&efx->tc->mutex);
@@ -903,6 +1515,8 @@ void efx_fini_struct_tc(struct efx_nic *efx)
 			     MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
 	rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free,
 				    efx);
+	rhashtable_free_and_destroy(&efx->tc->encap_match_ht,
+				    efx_tc_encap_match_free, NULL);
 	efx_tc_fini_counters(efx);
 	mutex_unlock(&efx->tc->mutex);
 	mutex_destroy(&efx->tc->mutex);
diff --git a/drivers/net/ethernet/sfc/tc.h b/drivers/net/ethernet/sfc/tc.h
index 418ce8c13a06..04cced6a2d39 100644
--- a/drivers/net/ethernet/sfc/tc.h
+++ b/drivers/net/ethernet/sfc/tc.h
@@ -18,8 +18,20 @@
 
 #define IS_ALL_ONES(v)	(!(typeof (v))~(v))
 
+#ifdef CONFIG_IPV6
+static inline bool efx_ipv6_addr_all_ones(struct in6_addr *addr)
+{
+	return !memchr_inv(addr, 0xff, sizeof(*addr));
+}
+#endif
+
 struct efx_tc_action_set {
+	u16 vlan_push:2;
+	u16 vlan_pop:2;
+	u16 decap:1;
 	u16 deliver:1;
+	__be16 vlan_tci[2]; /* TCIs for vlan_push */
+	__be16 vlan_proto[2]; /* Ethertypes for vlan_push */
 	struct efx_tc_counter_index *count;
 	u32 dest_mport;
 	u32 fw_id; /* index of this entry in firmware actions table */
@@ -44,11 +56,38 @@ struct efx_tc_match_fields {
 	/* L4 */
 	__be16 l4_sport, l4_dport; /* Ports (UDP, TCP) */
 	__be16 tcp_flags;
+	/* Encap.  The following are *outer* fields.  Note that there are no
+	 * outer eth (L2) fields; this is because TC doesn't have them.
+	 */
+	__be32 enc_src_ip, enc_dst_ip;
+	struct in6_addr enc_src_ip6, enc_dst_ip6;
+	u8 enc_ip_tos, enc_ip_ttl;
+	__be16 enc_sport, enc_dport;
+	__be32 enc_keyid; /* e.g. VNI, VSID */
+};
+
+static inline bool efx_tc_match_is_encap(const struct efx_tc_match_fields *mask)
+{
+	return mask->enc_src_ip || mask->enc_dst_ip ||
+	       !ipv6_addr_any(&mask->enc_src_ip6) ||
+	       !ipv6_addr_any(&mask->enc_dst_ip6) || mask->enc_ip_tos ||
+	       mask->enc_ip_ttl || mask->enc_sport || mask->enc_dport;
+}
+
+struct efx_tc_encap_match {
+	__be32 src_ip, dst_ip;
+	struct in6_addr src_ip6, dst_ip6;
+	__be16 udp_dport;
+	struct rhash_head linkage;
+	enum efx_encap_type tun_type;
+	refcount_t ref;
+	u32 fw_id; /* index of this entry in firmware encap match table */
 };
 
 struct efx_tc_match {
 	struct efx_tc_match_fields value;
 	struct efx_tc_match_fields mask;
+	struct efx_tc_encap_match *encap;
 };
 
 struct efx_tc_action_set_list {
@@ -78,6 +117,7 @@ enum efx_tc_rule_prios {
  * @mutex: Used to serialise operations on TC hashtables
  * @counter_ht: Hashtable of TC counters (FW IDs and counter values)
  * @counter_id_ht: Hashtable mapping TC counter cookies to counters
+ * @encap_match_ht: Hashtable of TC encap matches
  * @match_action_ht: Hashtable of TC match-action rules
  * @reps_mport_id: MAE port allocated for representor RX
  * @reps_filter_uc: VNIC filter for representor unicast RX (promisc)
@@ -101,6 +141,7 @@ struct efx_tc_state {
 	struct mutex mutex;
 	struct rhashtable counter_ht;
 	struct rhashtable counter_id_ht;
+	struct rhashtable encap_match_ht;
 	struct rhashtable match_action_ht;
 	u32 reps_mport_id, reps_mport_vport_id;
 	s32 reps_filter_uc, reps_filter_mc;
diff --git a/drivers/net/ethernet/sfc/tx_tso.c b/drivers/net/ethernet/sfc/tx_tso.c
index 898e5c61d908..d381d8164f07 100644
--- a/drivers/net/ethernet/sfc/tx_tso.c
+++ b/drivers/net/ethernet/sfc/tx_tso.c
@@ -147,7 +147,7 @@ static __be16 efx_tso_check_protocol(struct sk_buff *skb)
 	EFX_WARN_ON_ONCE_PARANOID(((struct ethhdr *)skb->data)->h_proto !=
 				  protocol);
 	if (protocol == htons(ETH_P_8021Q)) {
-		struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
+		struct vlan_ethhdr *veh = skb_vlan_eth_hdr(skb);
 
 		protocol = veh->h_vlan_encapsulated_proto;
 	}
diff --git a/drivers/net/ethernet/smsc/smc91x.c b/drivers/net/ethernet/smsc/smc91x.c
index 35e99bf0c401..032eccf8eb42 100644
--- a/drivers/net/ethernet/smsc/smc91x.c
+++ b/drivers/net/ethernet/smsc/smc91x.c
@@ -57,6 +57,7 @@ static const char version[] =
 #include <linux/kernel.h>
 #include <linux/sched.h>
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/errno.h>
@@ -69,7 +70,6 @@ static const char version[] =
 #include <linux/workqueue.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
-#include <linux/of_gpio.h>
 
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c
index a690d139e177..174dc8908b72 100644
--- a/drivers/net/ethernet/smsc/smsc911x.c
+++ b/drivers/net/ethernet/smsc/smsc911x.c
@@ -1016,7 +1016,7 @@ static void smsc911x_phy_adjust_link(struct net_device *dev)
 static int smsc911x_mii_probe(struct net_device *dev)
 {
 	struct smsc911x_data *pdata = netdev_priv(dev);
-	struct phy_device *phydev = NULL;
+	struct phy_device *phydev;
 	int ret;
 
 	/* find the first phy */
@@ -1744,7 +1744,6 @@ irq_stop_out:
 	free_irq(dev->irq, dev);
 mii_free_out:
 	phy_disconnect(dev->phydev);
-	dev->phydev = NULL;
 out:
 	pm_runtime_put(dev->dev.parent);
 	return retval;
@@ -1775,7 +1774,6 @@ static int smsc911x_stop(struct net_device *dev)
 	if (dev->phydev) {
 		phy_stop(dev->phydev);
 		phy_disconnect(dev->phydev);
-		dev->phydev = NULL;
 	}
 	netif_carrier_off(dev);
 	pm_runtime_put(dev->dev.parent);
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index f77511fe4e87..5f5a997f21f3 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -165,6 +165,18 @@ config DWMAC_SOCFPGA
 	  for the stmmac device driver. This driver is used for
 	  arria5 and cyclone5 FPGA SoCs.
 
+config DWMAC_STARFIVE
+	tristate "StarFive dwmac support"
+	depends on OF && (ARCH_STARFIVE || COMPILE_TEST)
+	select MFD_SYSCON
+	default m if ARCH_STARFIVE
+	help
+	  Support for ethernet controllers on StarFive RISC-V SoCs
+
+	  This selects the StarFive platform specific glue layer support for
+	  the stmmac device driver. This driver is used for StarFive JH7110
+	  ethernet controller.
+
 config DWMAC_STI
 	tristate "STi GMAC support"
 	default ARCH_STI
diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile
index 057e4bab5c08..8738fdbb4b2d 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Makefile
+++ b/drivers/net/ethernet/stmicro/stmmac/Makefile
@@ -23,6 +23,7 @@ obj-$(CONFIG_DWMAC_OXNAS)	+= dwmac-oxnas.o
 obj-$(CONFIG_DWMAC_QCOM_ETHQOS)	+= dwmac-qcom-ethqos.o
 obj-$(CONFIG_DWMAC_ROCKCHIP)	+= dwmac-rk.o
 obj-$(CONFIG_DWMAC_SOCFPGA)	+= dwmac-altr-socfpga.o
+obj-$(CONFIG_DWMAC_STARFIVE)	+= dwmac-starfive.o
 obj-$(CONFIG_DWMAC_STI)		+= dwmac-sti.o
 obj-$(CONFIG_DWMAC_STM32)	+= dwmac-stm32.o
 obj-$(CONFIG_DWMAC_SUNXI)	+= dwmac-sunxi.o
diff --git a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
index 2e8744ac6b91..fb55efd52240 100644
--- a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
+++ b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
@@ -14,9 +14,9 @@
 
 #include "stmmac.h"
 
-static int jumbo_frm(void *p, struct sk_buff *skb, int csum)
+static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb,
+		     int csum)
 {
-	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)p;
 	unsigned int nopaged_len = skb_headlen(skb);
 	struct stmmac_priv *priv = tx_q->priv_data;
 	unsigned int entry = tx_q->cur_tx;
@@ -125,9 +125,8 @@ static void init_dma_chain(void *des, dma_addr_t phy_addr,
 	}
 }
 
-static void refill_desc3(void *priv_ptr, struct dma_desc *p)
+static void refill_desc3(struct stmmac_rx_queue *rx_q, struct dma_desc *p)
 {
-	struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)priv_ptr;
 	struct stmmac_priv *priv = rx_q->priv_data;
 
 	if (priv->hwts_rx_en && !priv->extend_desc)
@@ -141,9 +140,8 @@ static void refill_desc3(void *priv_ptr, struct dma_desc *p)
 				      sizeof(struct dma_desc)));
 }
 
-static void clean_desc3(void *priv_ptr, struct dma_desc *p)
+static void clean_desc3(struct stmmac_tx_queue *tx_q, struct dma_desc *p)
 {
-	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)priv_ptr;
 	struct stmmac_priv *priv = tx_q->priv_data;
 	unsigned int entry = tx_q->dirty_tx;
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h
index 54bb072aeb2d..4ad692c4116c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/common.h
+++ b/drivers/net/ethernet/stmicro/stmmac/common.h
@@ -242,7 +242,7 @@ struct stmmac_safety_stats {
 
 #define SF_DMA_MODE 1		/* DMA STORE-AND-FORWARD Operation Mode */
 
-/* DAM HW feature register fields */
+/* DMA HW feature register fields */
 #define DMA_HW_FEAT_MIISEL	0x00000001	/* 10/100 Mbps Support */
 #define DMA_HW_FEAT_GMIISEL	0x00000002	/* 1000 Mbps Support */
 #define DMA_HW_FEAT_HDSEL	0x00000004	/* Half-Duplex Support */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c
index dfbaea06d108..9354bf419112 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c
@@ -20,18 +20,18 @@
 #define  GMAC_CONFIG_INTF_RGMII		(0x1 << 0)
 
 struct anarion_gmac {
-	uintptr_t ctl_block;
+	void __iomem *ctl_block;
 	uint32_t phy_intf_sel;
 };
 
 static uint32_t gmac_read_reg(struct anarion_gmac *gmac, uint8_t reg)
 {
-	return readl((void *)(gmac->ctl_block + reg));
+	return readl(gmac->ctl_block + reg);
 };
 
 static void gmac_write_reg(struct anarion_gmac *gmac, uint8_t reg, uint32_t val)
 {
-	writel(val, (void *)(gmac->ctl_block + reg));
+	writel(val, gmac->ctl_block + reg);
 }
 
 static int anarion_gmac_init(struct platform_device *pdev, void *priv)
@@ -68,16 +68,16 @@ static struct anarion_gmac *anarion_config_dt(struct platform_device *pdev)
 
 	ctl_block = devm_platform_ioremap_resource(pdev, 1);
 	if (IS_ERR(ctl_block)) {
-		dev_err(&pdev->dev, "Cannot get reset region (%ld)!\n",
-			PTR_ERR(ctl_block));
-		return ctl_block;
+		err = PTR_ERR(ctl_block);
+		dev_err(&pdev->dev, "Cannot get reset region (%d)!\n", err);
+		return ERR_PTR(err);
 	}
 
 	gmac = devm_kzalloc(&pdev->dev, sizeof(*gmac), GFP_KERNEL);
 	if (!gmac)
 		return ERR_PTR(-ENOMEM);
 
-	gmac->ctl_block = (uintptr_t)ctl_block;
+	gmac->ctl_block = ctl_block;
 
 	err = of_get_phy_mode(pdev->dev.of_node, &phy_mode);
 	if (err)
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-generic.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-generic.c
index 5e731a72cce8..ef8f3a940938 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-generic.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-generic.c
@@ -91,7 +91,7 @@ static struct platform_driver dwmac_generic_driver = {
 	.driver = {
 		.name           = STMMAC_RESOURCE_NAME,
 		.pm		= &stmmac_pltfr_pm_ops,
-		.of_match_table = of_match_ptr(dwmac_generic_match),
+		.of_match_table = dwmac_generic_match,
 	},
 };
 module_platform_driver(dwmac_generic_driver);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c
index 2a2be65d65a0..7c228bd0d099 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c
@@ -37,10 +37,15 @@
 #define MX93_GPR_ENET_QOS_INTF_SEL_RGMII	(0x1 << 1)
 #define MX93_GPR_ENET_QOS_CLK_GEN_EN		(0x1 << 0)
 
+#define DMA_BUS_MODE			0x00001000
+#define DMA_BUS_MODE_SFT_RESET		(0x1 << 0)
+#define RMII_RESET_SPEED		(0x3 << 14)
+
 struct imx_dwmac_ops {
 	u32 addr_width;
 	bool mac_rgmii_txclk_auto_adj;
 
+	int (*fix_soc_reset)(void *priv, void __iomem *ioaddr);
 	int (*set_intf_mode)(struct plat_stmmacenet_data *plat_dat);
 };
 
@@ -207,6 +212,25 @@ static void imx_dwmac_fix_speed(void *priv, unsigned int speed)
 		dev_err(dwmac->dev, "failed to set tx rate %lu\n", rate);
 }
 
+static int imx_dwmac_mx93_reset(void *priv, void __iomem *ioaddr)
+{
+	struct plat_stmmacenet_data *plat_dat = priv;
+	u32 value = readl(ioaddr + DMA_BUS_MODE);
+
+	/* DMA SW reset */
+	value |= DMA_BUS_MODE_SFT_RESET;
+	writel(value, ioaddr + DMA_BUS_MODE);
+
+	if (plat_dat->interface == PHY_INTERFACE_MODE_RMII) {
+		usleep_range(100, 200);
+		writel(RMII_RESET_SPEED, ioaddr + MAC_CTRL_REG);
+	}
+
+	return readl_poll_timeout(ioaddr + DMA_BUS_MODE, value,
+				 !(value & DMA_BUS_MODE_SFT_RESET),
+				 10000, 1000000);
+}
+
 static int
 imx_dwmac_parse_dt(struct imx_priv_data *dwmac, struct device *dev)
 {
@@ -304,6 +328,8 @@ static int imx_dwmac_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_dwmac_init;
 
+	dwmac->plat_dat->fix_soc_reset = dwmac->ops->fix_soc_reset;
+
 	ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res);
 	if (ret)
 		goto err_drv_probe;
@@ -337,6 +363,7 @@ static struct imx_dwmac_ops imx93_dwmac_data = {
 	.addr_width = 32,
 	.mac_rgmii_txclk_auto_adj = true,
 	.set_intf_mode = imx93_set_intf_mode,
+	.fix_soc_reset = imx_dwmac_mx93_reset,
 };
 
 static const struct of_device_id imx_dwmac_match[] = {
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
index e8b507f88fbc..f6754e3643f3 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
@@ -263,6 +263,11 @@ static int meson_axg_set_phy_mode(struct meson8b_dwmac *dwmac)
 	return 0;
 }
 
+static void meson8b_clk_disable_unprepare(void *data)
+{
+	clk_disable_unprepare(data);
+}
+
 static int meson8b_devm_clk_prepare_enable(struct meson8b_dwmac *dwmac,
 					   struct clk *clk)
 {
@@ -273,8 +278,7 @@ static int meson8b_devm_clk_prepare_enable(struct meson8b_dwmac *dwmac,
 		return ret;
 
 	return devm_add_action_or_reset(dwmac->dev,
-					(void(*)(void *))clk_disable_unprepare,
-					clk);
+					meson8b_clk_disable_unprepare, clk);
 }
 
 static int meson8b_init_rgmii_delays(struct meson8b_dwmac *dwmac)
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c
index 732774645c1a..16a8c361283b 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c
@@ -11,6 +11,7 @@
 
 #define RGMII_IO_MACRO_CONFIG		0x0
 #define SDCC_HC_REG_DLL_CONFIG		0x4
+#define SDCC_TEST_CTL			0x8
 #define SDCC_HC_REG_DDR_CONFIG		0xC
 #define SDCC_HC_REG_DLL_CONFIG2		0x10
 #define SDC4_STATUS			0x14
@@ -49,6 +50,7 @@
 #define SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY	GENMASK(26, 21)
 #define SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_CODE	GENMASK(29, 27)
 #define SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_EN	BIT(30)
+#define SDCC_DDR_CONFIG_TCXO_CYCLES_CNT		GENMASK(11, 9)
 #define SDCC_DDR_CONFIG_PRG_RCLK_DLY		GENMASK(8, 0)
 
 /* SDCC_HC_REG_DLL_CONFIG2 fields */
@@ -78,7 +80,9 @@ struct ethqos_emac_por {
 struct ethqos_emac_driver_data {
 	const struct ethqos_emac_por *por;
 	unsigned int num_por;
-	bool rgmii_config_looback_en;
+	bool rgmii_config_loopback_en;
+	bool has_emac3;
+	struct dwmac4_addrs dwmac4_addrs;
 };
 
 struct qcom_ethqos {
@@ -91,7 +95,8 @@ struct qcom_ethqos {
 
 	const struct ethqos_emac_por *por;
 	unsigned int num_por;
-	bool rgmii_config_looback_en;
+	bool rgmii_config_loopback_en;
+	bool has_emac3;
 };
 
 static int rgmii_readl(struct qcom_ethqos *ethqos, unsigned int offset)
@@ -183,7 +188,8 @@ static const struct ethqos_emac_por emac_v2_3_0_por[] = {
 static const struct ethqos_emac_driver_data emac_v2_3_0_data = {
 	.por = emac_v2_3_0_por,
 	.num_por = ARRAY_SIZE(emac_v2_3_0_por),
-	.rgmii_config_looback_en = true,
+	.rgmii_config_loopback_en = true,
+	.has_emac3 = false,
 };
 
 static const struct ethqos_emac_por emac_v2_1_0_por[] = {
@@ -198,7 +204,40 @@ static const struct ethqos_emac_por emac_v2_1_0_por[] = {
 static const struct ethqos_emac_driver_data emac_v2_1_0_data = {
 	.por = emac_v2_1_0_por,
 	.num_por = ARRAY_SIZE(emac_v2_1_0_por),
-	.rgmii_config_looback_en = false,
+	.rgmii_config_loopback_en = false,
+	.has_emac3 = false,
+};
+
+static const struct ethqos_emac_por emac_v3_0_0_por[] = {
+	{ .offset = RGMII_IO_MACRO_CONFIG,	.value = 0x40c01343 },
+	{ .offset = SDCC_HC_REG_DLL_CONFIG,	.value = 0x2004642c },
+	{ .offset = SDCC_HC_REG_DDR_CONFIG,	.value = 0x80040800 },
+	{ .offset = SDCC_HC_REG_DLL_CONFIG2,	.value = 0x00200000 },
+	{ .offset = SDCC_USR_CTL,		.value = 0x00010800 },
+	{ .offset = RGMII_IO_MACRO_CONFIG2,	.value = 0x00002060 },
+};
+
+static const struct ethqos_emac_driver_data emac_v3_0_0_data = {
+	.por = emac_v3_0_0_por,
+	.num_por = ARRAY_SIZE(emac_v3_0_0_por),
+	.rgmii_config_loopback_en = false,
+	.has_emac3 = true,
+	.dwmac4_addrs = {
+		.dma_chan = 0x00008100,
+		.dma_chan_offset = 0x1000,
+		.mtl_chan = 0x00008000,
+		.mtl_chan_offset = 0x1000,
+		.mtl_ets_ctrl = 0x00008010,
+		.mtl_ets_ctrl_offset = 0x1000,
+		.mtl_txq_weight = 0x00008018,
+		.mtl_txq_weight_offset = 0x1000,
+		.mtl_send_slp_cred = 0x0000801c,
+		.mtl_send_slp_cred_offset = 0x1000,
+		.mtl_high_cred = 0x00008020,
+		.mtl_high_cred_offset = 0x1000,
+		.mtl_low_cred = 0x00008024,
+		.mtl_low_cred_offset = 0x1000,
+	},
 };
 
 static int ethqos_dll_configure(struct qcom_ethqos *ethqos)
@@ -222,11 +261,13 @@ static int ethqos_dll_configure(struct qcom_ethqos *ethqos)
 	rgmii_updatel(ethqos, SDCC_DLL_CONFIG_DLL_EN,
 		      SDCC_DLL_CONFIG_DLL_EN, SDCC_HC_REG_DLL_CONFIG);
 
-	rgmii_updatel(ethqos, SDCC_DLL_MCLK_GATING_EN,
-		      0, SDCC_HC_REG_DLL_CONFIG);
+	if (!ethqos->has_emac3) {
+		rgmii_updatel(ethqos, SDCC_DLL_MCLK_GATING_EN,
+			      0, SDCC_HC_REG_DLL_CONFIG);
 
-	rgmii_updatel(ethqos, SDCC_DLL_CDR_FINE_PHASE,
-		      0, SDCC_HC_REG_DLL_CONFIG);
+		rgmii_updatel(ethqos, SDCC_DLL_CDR_FINE_PHASE,
+			      0, SDCC_HC_REG_DLL_CONFIG);
+	}
 
 	/* Wait for CK_OUT_EN clear */
 	do {
@@ -261,28 +302,48 @@ static int ethqos_dll_configure(struct qcom_ethqos *ethqos)
 	rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_DDR_CAL_EN,
 		      SDCC_DLL_CONFIG2_DDR_CAL_EN, SDCC_HC_REG_DLL_CONFIG2);
 
-	rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_DLL_CLOCK_DIS,
-		      0, SDCC_HC_REG_DLL_CONFIG2);
+	if (!ethqos->has_emac3) {
+		rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_DLL_CLOCK_DIS,
+			      0, SDCC_HC_REG_DLL_CONFIG2);
 
-	rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_MCLK_FREQ_CALC,
-		      0x1A << 10, SDCC_HC_REG_DLL_CONFIG2);
+		rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_MCLK_FREQ_CALC,
+			      0x1A << 10, SDCC_HC_REG_DLL_CONFIG2);
 
-	rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SEL,
-		      BIT(2), SDCC_HC_REG_DLL_CONFIG2);
+		rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SEL,
+			      BIT(2), SDCC_HC_REG_DLL_CONFIG2);
 
-	rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SW,
-		      SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SW,
-		      SDCC_HC_REG_DLL_CONFIG2);
+		rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SW,
+			      SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SW,
+			      SDCC_HC_REG_DLL_CONFIG2);
+	}
 
 	return 0;
 }
 
 static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos)
 {
+	int phase_shift;
+	int phy_mode;
+	int loopback;
+
+	/* Determine if the PHY adds a 2 ns TX delay or the MAC handles it */
+	phy_mode = device_get_phy_mode(&ethqos->pdev->dev);
+	if (phy_mode == PHY_INTERFACE_MODE_RGMII_ID ||
+	    phy_mode == PHY_INTERFACE_MODE_RGMII_TXID)
+		phase_shift = 0;
+	else
+		phase_shift = RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN;
+
 	/* Disable loopback mode */
 	rgmii_updatel(ethqos, RGMII_CONFIG2_TX_TO_RX_LOOPBACK_EN,
 		      0, RGMII_IO_MACRO_CONFIG2);
 
+	/* Determine if this platform wants loopback enabled after programming */
+	if (ethqos->rgmii_config_loopback_en)
+		loopback = RGMII_CONFIG_LOOPBACK_EN;
+	else
+		loopback = 0;
+
 	/* Select RGMII, write 0 to interface select */
 	rgmii_updatel(ethqos, RGMII_CONFIG_INTF_SEL,
 		      0, RGMII_IO_MACRO_CONFIG);
@@ -300,27 +361,32 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos)
 			      RGMII_CONFIG_PROG_SWAP, RGMII_IO_MACRO_CONFIG);
 		rgmii_updatel(ethqos, RGMII_CONFIG2_DATA_DIVIDE_CLK_SEL,
 			      0, RGMII_IO_MACRO_CONFIG2);
+
 		rgmii_updatel(ethqos, RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN,
-			      RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN,
-			      RGMII_IO_MACRO_CONFIG2);
+			      phase_shift, RGMII_IO_MACRO_CONFIG2);
 		rgmii_updatel(ethqos, RGMII_CONFIG2_RSVD_CONFIG15,
 			      0, RGMII_IO_MACRO_CONFIG2);
 		rgmii_updatel(ethqos, RGMII_CONFIG2_RX_PROG_SWAP,
 			      RGMII_CONFIG2_RX_PROG_SWAP,
 			      RGMII_IO_MACRO_CONFIG2);
 
-		/* Set PRG_RCLK_DLY to 57 for 1.8 ns delay */
-		rgmii_updatel(ethqos, SDCC_DDR_CONFIG_PRG_RCLK_DLY,
-			      57, SDCC_HC_REG_DDR_CONFIG);
+		/* PRG_RCLK_DLY = TCXO period * TCXO_CYCLES_CNT / 2 * RX delay ns,
+		 * in practice this becomes PRG_RCLK_DLY = 52 * 4 / 2 * RX delay ns
+		 */
+		if (ethqos->has_emac3) {
+			/* 0.9 ns */
+			rgmii_updatel(ethqos, SDCC_DDR_CONFIG_PRG_RCLK_DLY,
+				      115, SDCC_HC_REG_DDR_CONFIG);
+		} else {
+			/* 1.8 ns */
+			rgmii_updatel(ethqos, SDCC_DDR_CONFIG_PRG_RCLK_DLY,
+				      57, SDCC_HC_REG_DDR_CONFIG);
+		}
 		rgmii_updatel(ethqos, SDCC_DDR_CONFIG_PRG_DLY_EN,
 			      SDCC_DDR_CONFIG_PRG_DLY_EN,
 			      SDCC_HC_REG_DDR_CONFIG);
-		if (ethqos->rgmii_config_looback_en)
-			rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN,
-				      RGMII_CONFIG_LOOPBACK_EN, RGMII_IO_MACRO_CONFIG);
-		else
-			rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN,
-				      0, RGMII_IO_MACRO_CONFIG);
+		rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN,
+			      loopback, RGMII_IO_MACRO_CONFIG);
 		break;
 
 	case SPEED_100:
@@ -336,14 +402,20 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos)
 		rgmii_updatel(ethqos, RGMII_CONFIG2_DATA_DIVIDE_CLK_SEL,
 			      0, RGMII_IO_MACRO_CONFIG2);
 		rgmii_updatel(ethqos, RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN,
-			      RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN,
-			      RGMII_IO_MACRO_CONFIG2);
+			      phase_shift, RGMII_IO_MACRO_CONFIG2);
 		rgmii_updatel(ethqos, RGMII_CONFIG_MAX_SPD_PRG_2,
 			      BIT(6), RGMII_IO_MACRO_CONFIG);
 		rgmii_updatel(ethqos, RGMII_CONFIG2_RSVD_CONFIG15,
 			      0, RGMII_IO_MACRO_CONFIG2);
-		rgmii_updatel(ethqos, RGMII_CONFIG2_RX_PROG_SWAP,
-			      0, RGMII_IO_MACRO_CONFIG2);
+
+		if (ethqos->has_emac3)
+			rgmii_updatel(ethqos, RGMII_CONFIG2_RX_PROG_SWAP,
+				      RGMII_CONFIG2_RX_PROG_SWAP,
+				      RGMII_IO_MACRO_CONFIG2);
+		else
+			rgmii_updatel(ethqos, RGMII_CONFIG2_RX_PROG_SWAP,
+				      0, RGMII_IO_MACRO_CONFIG2);
+
 		/* Write 0x5 to PRG_RCLK_DLY_CODE */
 		rgmii_updatel(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_CODE,
 			      (BIT(29) | BIT(27)), SDCC_HC_REG_DDR_CONFIG);
@@ -353,13 +425,8 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos)
 		rgmii_updatel(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_EN,
 			      SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_EN,
 			      SDCC_HC_REG_DDR_CONFIG);
-		if (ethqos->rgmii_config_looback_en)
-			rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN,
-				      RGMII_CONFIG_LOOPBACK_EN, RGMII_IO_MACRO_CONFIG);
-		else
-			rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN,
-				      0, RGMII_IO_MACRO_CONFIG);
-
+		rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN,
+			      loopback, RGMII_IO_MACRO_CONFIG);
 		break;
 
 	case SPEED_10:
@@ -375,14 +442,19 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos)
 		rgmii_updatel(ethqos, RGMII_CONFIG2_DATA_DIVIDE_CLK_SEL,
 			      0, RGMII_IO_MACRO_CONFIG2);
 		rgmii_updatel(ethqos, RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN,
-			      0, RGMII_IO_MACRO_CONFIG2);
+			      phase_shift, RGMII_IO_MACRO_CONFIG2);
 		rgmii_updatel(ethqos, RGMII_CONFIG_MAX_SPD_PRG_9,
 			      BIT(12) | GENMASK(9, 8),
 			      RGMII_IO_MACRO_CONFIG);
 		rgmii_updatel(ethqos, RGMII_CONFIG2_RSVD_CONFIG15,
 			      0, RGMII_IO_MACRO_CONFIG2);
-		rgmii_updatel(ethqos, RGMII_CONFIG2_RX_PROG_SWAP,
-			      0, RGMII_IO_MACRO_CONFIG2);
+		if (ethqos->has_emac3)
+			rgmii_updatel(ethqos, RGMII_CONFIG2_RX_PROG_SWAP,
+				      RGMII_CONFIG2_RX_PROG_SWAP,
+				      RGMII_IO_MACRO_CONFIG2);
+		else
+			rgmii_updatel(ethqos, RGMII_CONFIG2_RX_PROG_SWAP,
+				      0, RGMII_IO_MACRO_CONFIG2);
 		/* Write 0x5 to PRG_RCLK_DLY_CODE */
 		rgmii_updatel(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_CODE,
 			      (BIT(29) | BIT(27)), SDCC_HC_REG_DDR_CONFIG);
@@ -393,7 +465,7 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos)
 			      SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_EN,
 			      SDCC_HC_REG_DDR_CONFIG);
 		rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN,
-			      RGMII_CONFIG_LOOPBACK_EN, RGMII_IO_MACRO_CONFIG);
+			      loopback, RGMII_IO_MACRO_CONFIG);
 		break;
 	default:
 		dev_err(&ethqos->pdev->dev,
@@ -425,6 +497,17 @@ static int ethqos_configure(struct qcom_ethqos *ethqos)
 	rgmii_updatel(ethqos, SDCC_DLL_CONFIG_PDN,
 		      SDCC_DLL_CONFIG_PDN, SDCC_HC_REG_DLL_CONFIG);
 
+	if (ethqos->has_emac3) {
+		if (ethqos->speed == SPEED_1000) {
+			rgmii_writel(ethqos, 0x1800000, SDCC_TEST_CTL);
+			rgmii_writel(ethqos, 0x2C010800, SDCC_USR_CTL);
+			rgmii_writel(ethqos, 0xA001, SDCC_HC_REG_DLL_CONFIG2);
+		} else {
+			rgmii_writel(ethqos, 0x40010800, SDCC_USR_CTL);
+			rgmii_writel(ethqos, 0xA001, SDCC_HC_REG_DLL_CONFIG2);
+		}
+	}
+
 	/* Clear DLL_RST */
 	rgmii_updatel(ethqos, SDCC_DLL_CONFIG_DLL_RST, 0,
 		      SDCC_HC_REG_DLL_CONFIG);
@@ -444,7 +527,9 @@ static int ethqos_configure(struct qcom_ethqos *ethqos)
 			      SDCC_HC_REG_DLL_CONFIG);
 
 		/* Set USR_CTL bit 26 with mask of 3 bits */
-		rgmii_updatel(ethqos, GENMASK(26, 24), BIT(26), SDCC_USR_CTL);
+		if (!ethqos->has_emac3)
+			rgmii_updatel(ethqos, GENMASK(26, 24), BIT(26),
+				      SDCC_USR_CTL);
 
 		/* wait for DLL LOCK */
 		do {
@@ -538,7 +623,8 @@ static int qcom_ethqos_probe(struct platform_device *pdev)
 	data = of_device_get_match_data(&pdev->dev);
 	ethqos->por = data->por;
 	ethqos->num_por = data->num_por;
-	ethqos->rgmii_config_looback_en = data->rgmii_config_looback_en;
+	ethqos->rgmii_config_loopback_en = data->rgmii_config_loopback_en;
+	ethqos->has_emac3 = data->has_emac3;
 
 	ethqos->rgmii_clk = devm_clk_get(&pdev->dev, "rgmii");
 	if (IS_ERR(ethqos->rgmii_clk)) {
@@ -558,6 +644,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev)
 	plat_dat->fix_mac_speed = ethqos_fix_mac_speed;
 	plat_dat->dump_debug_regs = rgmii_dump;
 	plat_dat->has_gmac4 = 1;
+	plat_dat->dwmac4_addrs = &data->dwmac4_addrs;
 	plat_dat->pmt = 1;
 	plat_dat->tso_en = of_property_read_bool(np, "snps,tso");
 	if (of_device_is_compatible(np, "qcom,qcs404-ethqos"))
@@ -595,6 +682,7 @@ static int qcom_ethqos_remove(struct platform_device *pdev)
 
 static const struct of_device_id qcom_ethqos_match[] = {
 	{ .compatible = "qcom,qcs404-ethqos", .data = &emac_v2_3_0_data},
+	{ .compatible = "qcom,sc8280xp-ethqos", .data = &emac_v3_0_0_data},
 	{ .compatible = "qcom,sm8150-ethqos", .data = &emac_v2_1_0_data},
 	{ }
 };
@@ -606,7 +694,7 @@ static struct platform_driver qcom_ethqos_driver = {
 	.driver = {
 		.name           = "qcom-ethqos",
 		.pm		= &stmmac_pltfr_pm_ops,
-		.of_match_table = of_match_ptr(qcom_ethqos_match),
+		.of_match_table = qcom_ethqos_match,
 	},
 };
 module_platform_driver(qcom_ethqos_driver);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
index 4b8fd11563e4..4ea31ccf24d0 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
@@ -39,6 +39,24 @@ struct rk_gmac_ops {
 	u32 regs[];
 };
 
+static const char * const rk_clocks[] = {
+	"aclk_mac", "pclk_mac", "mac_clk_tx", "clk_mac_speed",
+};
+
+static const char * const rk_rmii_clocks[] = {
+	"mac_clk_rx", "clk_mac_ref", "clk_mac_refout",
+};
+
+enum rk_clocks_index {
+	RK_ACLK_MAC = 0,
+	RK_PCLK_MAC,
+	RK_MAC_CLK_TX,
+	RK_CLK_MAC_SPEED,
+	RK_MAC_CLK_RX,
+	RK_CLK_MAC_REF,
+	RK_CLK_MAC_REFOUT,
+};
+
 struct rk_priv_data {
 	struct platform_device *pdev;
 	phy_interface_t phy_iface;
@@ -51,15 +69,9 @@ struct rk_priv_data {
 	bool clock_input;
 	bool integrated_phy;
 
+	struct clk_bulk_data *clks;
+	int num_clks;
 	struct clk *clk_mac;
-	struct clk *gmac_clkin;
-	struct clk *mac_clk_rx;
-	struct clk *mac_clk_tx;
-	struct clk *clk_mac_ref;
-	struct clk *clk_mac_refout;
-	struct clk *clk_mac_speed;
-	struct clk *aclk_mac;
-	struct clk *pclk_mac;
 	struct clk *clk_phy;
 
 	struct reset_control *phy_reset;
@@ -104,10 +116,11 @@ static void px30_set_to_rmii(struct rk_priv_data *bsp_priv)
 
 static void px30_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
 {
+	struct clk *clk_mac_speed = bsp_priv->clks[RK_CLK_MAC_SPEED].clk;
 	struct device *dev = &bsp_priv->pdev->dev;
 	int ret;
 
-	if (IS_ERR(bsp_priv->clk_mac_speed)) {
+	if (!clk_mac_speed) {
 		dev_err(dev, "%s: Missing clk_mac_speed clock\n", __func__);
 		return;
 	}
@@ -116,7 +129,7 @@ static void px30_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
 		regmap_write(bsp_priv->grf, PX30_GRF_GMAC_CON1,
 			     PX30_GMAC_SPEED_10M);
 
-		ret = clk_set_rate(bsp_priv->clk_mac_speed, 2500000);
+		ret = clk_set_rate(clk_mac_speed, 2500000);
 		if (ret)
 			dev_err(dev, "%s: set clk_mac_speed rate 2500000 failed: %d\n",
 				__func__, ret);
@@ -124,7 +137,7 @@ static void px30_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
 		regmap_write(bsp_priv->grf, PX30_GRF_GMAC_CON1,
 			     PX30_GMAC_SPEED_100M);
 
-		ret = clk_set_rate(bsp_priv->clk_mac_speed, 25000000);
+		ret = clk_set_rate(clk_mac_speed, 25000000);
 		if (ret)
 			dev_err(dev, "%s: set clk_mac_speed rate 25000000 failed: %d\n",
 				__func__, ret);
@@ -1066,6 +1079,7 @@ static void rk3568_set_to_rmii(struct rk_priv_data *bsp_priv)
 
 static void rk3568_set_gmac_speed(struct rk_priv_data *bsp_priv, int speed)
 {
+	struct clk *clk_mac_speed = bsp_priv->clks[RK_CLK_MAC_SPEED].clk;
 	struct device *dev = &bsp_priv->pdev->dev;
 	unsigned long rate;
 	int ret;
@@ -1085,7 +1099,7 @@ static void rk3568_set_gmac_speed(struct rk_priv_data *bsp_priv, int speed)
 		return;
 	}
 
-	ret = clk_set_rate(bsp_priv->clk_mac_speed, rate);
+	ret = clk_set_rate(clk_mac_speed, rate);
 	if (ret)
 		dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n",
 			__func__, rate, ret);
@@ -1371,6 +1385,7 @@ static void rv1126_set_to_rmii(struct rk_priv_data *bsp_priv)
 
 static void rv1126_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed)
 {
+	struct clk *clk_mac_speed = bsp_priv->clks[RK_CLK_MAC_SPEED].clk;
 	struct device *dev = &bsp_priv->pdev->dev;
 	unsigned long rate;
 	int ret;
@@ -1390,7 +1405,7 @@ static void rv1126_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed)
 		return;
 	}
 
-	ret = clk_set_rate(bsp_priv->clk_mac_speed, rate);
+	ret = clk_set_rate(clk_mac_speed, rate);
 	if (ret)
 		dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n",
 			__func__, rate, ret);
@@ -1398,6 +1413,7 @@ static void rv1126_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed)
 
 static void rv1126_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
 {
+	struct clk *clk_mac_speed = bsp_priv->clks[RK_CLK_MAC_SPEED].clk;
 	struct device *dev = &bsp_priv->pdev->dev;
 	unsigned long rate;
 	int ret;
@@ -1414,7 +1430,7 @@ static void rv1126_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
 		return;
 	}
 
-	ret = clk_set_rate(bsp_priv->clk_mac_speed, rate);
+	ret = clk_set_rate(clk_mac_speed, rate);
 	if (ret)
 		dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n",
 			__func__, rate, ret);
@@ -1475,68 +1491,50 @@ static int rk_gmac_clk_init(struct plat_stmmacenet_data *plat)
 {
 	struct rk_priv_data *bsp_priv = plat->bsp_priv;
 	struct device *dev = &bsp_priv->pdev->dev;
-	int ret;
+	int phy_iface = bsp_priv->phy_iface;
+	int i, j, ret;
 
 	bsp_priv->clk_enabled = false;
 
-	bsp_priv->mac_clk_rx = devm_clk_get(dev, "mac_clk_rx");
-	if (IS_ERR(bsp_priv->mac_clk_rx))
-		dev_err(dev, "cannot get clock %s\n",
-			"mac_clk_rx");
+	bsp_priv->num_clks = ARRAY_SIZE(rk_clocks);
+	if (phy_iface == PHY_INTERFACE_MODE_RMII)
+		bsp_priv->num_clks += ARRAY_SIZE(rk_rmii_clocks);
 
-	bsp_priv->mac_clk_tx = devm_clk_get(dev, "mac_clk_tx");
-	if (IS_ERR(bsp_priv->mac_clk_tx))
-		dev_err(dev, "cannot get clock %s\n",
-			"mac_clk_tx");
+	bsp_priv->clks = devm_kcalloc(dev, bsp_priv->num_clks,
+				      sizeof(*bsp_priv->clks), GFP_KERNEL);
+	if (!bsp_priv->clks)
+		return -ENOMEM;
 
-	bsp_priv->aclk_mac = devm_clk_get(dev, "aclk_mac");
-	if (IS_ERR(bsp_priv->aclk_mac))
-		dev_err(dev, "cannot get clock %s\n",
-			"aclk_mac");
+	for (i = 0; i < ARRAY_SIZE(rk_clocks); i++)
+		bsp_priv->clks[i].id = rk_clocks[i];
 
-	bsp_priv->pclk_mac = devm_clk_get(dev, "pclk_mac");
-	if (IS_ERR(bsp_priv->pclk_mac))
-		dev_err(dev, "cannot get clock %s\n",
-			"pclk_mac");
-
-	bsp_priv->clk_mac = devm_clk_get(dev, "stmmaceth");
-	if (IS_ERR(bsp_priv->clk_mac))
-		dev_err(dev, "cannot get clock %s\n",
-			"stmmaceth");
-
-	if (bsp_priv->phy_iface == PHY_INTERFACE_MODE_RMII) {
-		bsp_priv->clk_mac_ref = devm_clk_get(dev, "clk_mac_ref");
-		if (IS_ERR(bsp_priv->clk_mac_ref))
-			dev_err(dev, "cannot get clock %s\n",
-				"clk_mac_ref");
-
-		if (!bsp_priv->clock_input) {
-			bsp_priv->clk_mac_refout =
-				devm_clk_get(dev, "clk_mac_refout");
-			if (IS_ERR(bsp_priv->clk_mac_refout))
-				dev_err(dev, "cannot get clock %s\n",
-					"clk_mac_refout");
-		}
+	if (phy_iface == PHY_INTERFACE_MODE_RMII) {
+		for (j = 0; j < ARRAY_SIZE(rk_rmii_clocks); j++)
+			bsp_priv->clks[i++].id = rk_rmii_clocks[j];
 	}
 
-	bsp_priv->clk_mac_speed = devm_clk_get(dev, "clk_mac_speed");
-	if (IS_ERR(bsp_priv->clk_mac_speed))
-		dev_err(dev, "cannot get clock %s\n", "clk_mac_speed");
+	ret = devm_clk_bulk_get_optional(dev, bsp_priv->num_clks,
+					 bsp_priv->clks);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get clocks\n");
+
+	/* "stmmaceth" will be enabled by the core */
+	bsp_priv->clk_mac = devm_clk_get(dev, "stmmaceth");
+	ret = PTR_ERR_OR_ZERO(bsp_priv->clk_mac);
+	if (ret)
+		return dev_err_probe(dev, ret, "Cannot get stmmaceth clock\n");
 
 	if (bsp_priv->clock_input) {
 		dev_info(dev, "clock input from PHY\n");
-	} else {
-		if (bsp_priv->phy_iface == PHY_INTERFACE_MODE_RMII)
-			clk_set_rate(bsp_priv->clk_mac, 50000000);
+	} else if (phy_iface == PHY_INTERFACE_MODE_RMII) {
+		clk_set_rate(bsp_priv->clk_mac, 50000000);
 	}
 
 	if (plat->phy_node && bsp_priv->integrated_phy) {
 		bsp_priv->clk_phy = of_clk_get(plat->phy_node, 0);
-		if (IS_ERR(bsp_priv->clk_phy)) {
-			ret = PTR_ERR(bsp_priv->clk_phy);
-			dev_err(dev, "Cannot get PHY clock: %d\n", ret);
-			return -EINVAL;
-		}
+		ret = PTR_ERR_OR_ZERO(bsp_priv->clk_phy);
+		if (ret)
+			return dev_err_probe(dev, ret, "Cannot get PHY clock\n");
 		clk_set_rate(bsp_priv->clk_phy, 50000000);
 	}
 
@@ -1545,77 +1543,36 @@ static int rk_gmac_clk_init(struct plat_stmmacenet_data *plat)
 
 static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable)
 {
-	int phy_iface = bsp_priv->phy_iface;
+	int ret;
 
 	if (enable) {
 		if (!bsp_priv->clk_enabled) {
-			if (phy_iface == PHY_INTERFACE_MODE_RMII) {
-				if (!IS_ERR(bsp_priv->mac_clk_rx))
-					clk_prepare_enable(
-						bsp_priv->mac_clk_rx);
-
-				if (!IS_ERR(bsp_priv->clk_mac_ref))
-					clk_prepare_enable(
-						bsp_priv->clk_mac_ref);
-
-				if (!IS_ERR(bsp_priv->clk_mac_refout))
-					clk_prepare_enable(
-						bsp_priv->clk_mac_refout);
-			}
-
-			if (!IS_ERR(bsp_priv->clk_phy))
-				clk_prepare_enable(bsp_priv->clk_phy);
+			ret = clk_bulk_prepare_enable(bsp_priv->num_clks,
+						      bsp_priv->clks);
+			if (ret)
+				return ret;
 
-			if (!IS_ERR(bsp_priv->aclk_mac))
-				clk_prepare_enable(bsp_priv->aclk_mac);
-
-			if (!IS_ERR(bsp_priv->pclk_mac))
-				clk_prepare_enable(bsp_priv->pclk_mac);
-
-			if (!IS_ERR(bsp_priv->mac_clk_tx))
-				clk_prepare_enable(bsp_priv->mac_clk_tx);
-
-			if (!IS_ERR(bsp_priv->clk_mac_speed))
-				clk_prepare_enable(bsp_priv->clk_mac_speed);
+			ret = clk_prepare_enable(bsp_priv->clk_phy);
+			if (ret)
+				return ret;
 
 			if (bsp_priv->ops && bsp_priv->ops->set_clock_selection)
 				bsp_priv->ops->set_clock_selection(bsp_priv,
 					       bsp_priv->clock_input, true);
 
-			/**
-			 * if (!IS_ERR(bsp_priv->clk_mac))
-			 *	clk_prepare_enable(bsp_priv->clk_mac);
-			 */
 			mdelay(5);
 			bsp_priv->clk_enabled = true;
 		}
 	} else {
 		if (bsp_priv->clk_enabled) {
-			if (phy_iface == PHY_INTERFACE_MODE_RMII) {
-				clk_disable_unprepare(bsp_priv->mac_clk_rx);
-
-				clk_disable_unprepare(bsp_priv->clk_mac_ref);
-
-				clk_disable_unprepare(bsp_priv->clk_mac_refout);
-			}
-
+			clk_bulk_disable_unprepare(bsp_priv->num_clks,
+						   bsp_priv->clks);
 			clk_disable_unprepare(bsp_priv->clk_phy);
 
-			clk_disable_unprepare(bsp_priv->aclk_mac);
-
-			clk_disable_unprepare(bsp_priv->pclk_mac);
-
-			clk_disable_unprepare(bsp_priv->mac_clk_tx);
-
-			clk_disable_unprepare(bsp_priv->clk_mac_speed);
-
 			if (bsp_priv->ops && bsp_priv->ops->set_clock_selection)
 				bsp_priv->ops->set_clock_selection(bsp_priv,
 					      bsp_priv->clock_input, false);
-			/**
-			 * if (!IS_ERR(bsp_priv->clk_mac))
-			 *	clk_disable_unprepare(bsp_priv->clk_mac);
-			 */
+
 			bsp_priv->clk_enabled = false;
 		}
 	}
@@ -1629,9 +1586,6 @@ static int phy_power_on(struct rk_priv_data *bsp_priv, bool enable)
 	int ret;
 	struct device *dev = &bsp_priv->pdev->dev;
 
-	if (!ldo)
-		return 0;
-
 	if (enable) {
 		ret = regulator_enable(ldo);
 		if (ret)
@@ -1679,14 +1633,11 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev,
 		}
 	}
 
-	bsp_priv->regulator = devm_regulator_get_optional(dev, "phy");
+	bsp_priv->regulator = devm_regulator_get(dev, "phy");
 	if (IS_ERR(bsp_priv->regulator)) {
-		if (PTR_ERR(bsp_priv->regulator) == -EPROBE_DEFER) {
-			dev_err(dev, "phy regulator is not available yet, deferred probing\n");
-			return ERR_PTR(-EPROBE_DEFER);
-		}
-		dev_err(dev, "no regulator found\n");
-		bsp_priv->regulator = NULL;
+		ret = PTR_ERR(bsp_priv->regulator);
+		dev_err_probe(dev, ret, "failed to get phy regulator\n");
+		return ERR_PTR(ret);
 	}
 
 	ret = of_property_read_string(dev->of_node, "clock_in_out", &strings);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-starfive.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-starfive.c
new file mode 100644
index 000000000000..4f51a7889642
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-starfive.c
@@ -0,0 +1,171 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * StarFive DWMAC platform driver
+ *
+ * Copyright (C) 2021 Emil Renner Berthing <kernel@esmil.dk>
+ * Copyright (C) 2022 StarFive Technology Co., Ltd.
+ *
+ */
+
+#include <linux/mfd/syscon.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+
+#include "stmmac_platform.h"
+
+#define STARFIVE_DWMAC_PHY_INFT_RGMII	0x1
+#define STARFIVE_DWMAC_PHY_INFT_RMII	0x4
+#define STARFIVE_DWMAC_PHY_INFT_FIELD	0x7U
+
+struct starfive_dwmac {
+	struct device *dev;
+	struct clk *clk_tx;
+};
+
+static void starfive_dwmac_fix_mac_speed(void *priv, unsigned int speed)
+{
+	struct starfive_dwmac *dwmac = priv;
+	unsigned long rate;
+	int err;
+
+	rate = clk_get_rate(dwmac->clk_tx);
+
+	switch (speed) {
+	case SPEED_1000:
+		rate = 125000000;
+		break;
+	case SPEED_100:
+		rate = 25000000;
+		break;
+	case SPEED_10:
+		rate = 2500000;
+		break;
+	default:
+		dev_err(dwmac->dev, "invalid speed %u\n", speed);
+		break;
+	}
+
+	err = clk_set_rate(dwmac->clk_tx, rate);
+	if (err)
+		dev_err(dwmac->dev, "failed to set tx rate %lu\n", rate);
+}
+
+static int starfive_dwmac_set_mode(struct plat_stmmacenet_data *plat_dat)
+{
+	struct starfive_dwmac *dwmac = plat_dat->bsp_priv;
+	struct regmap *regmap;
+	unsigned int args[2];
+	unsigned int mode;
+	int err;
+
+	switch (plat_dat->interface) {
+	case PHY_INTERFACE_MODE_RMII:
+		mode = STARFIVE_DWMAC_PHY_INFT_RMII;
+		break;
+
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+		mode = STARFIVE_DWMAC_PHY_INFT_RGMII;
+		break;
+
+	default:
+		dev_err(dwmac->dev, "unsupported interface %d\n",
+			plat_dat->interface);
+		return -EINVAL;
+	}
+
+	regmap = syscon_regmap_lookup_by_phandle_args(dwmac->dev->of_node,
+						      "starfive,syscon",
+						      2, args);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dwmac->dev, PTR_ERR(regmap), "getting the regmap failed\n");
+
+	/* args[0]:offset  args[1]: shift */
+	err = regmap_update_bits(regmap, args[0],
+				 STARFIVE_DWMAC_PHY_INFT_FIELD << args[1],
+				 mode << args[1]);
+	if (err)
+		return dev_err_probe(dwmac->dev, err, "error setting phy mode\n");
+
+	return 0;
+}
+
+static int starfive_dwmac_probe(struct platform_device *pdev)
+{
+	struct plat_stmmacenet_data *plat_dat;
+	struct stmmac_resources stmmac_res;
+	struct starfive_dwmac *dwmac;
+	struct clk *clk_gtx;
+	int err;
+
+	err = stmmac_get_platform_resources(pdev, &stmmac_res);
+	if (err)
+		return dev_err_probe(&pdev->dev, err,
+				     "failed to get resources\n");
+
+	plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac);
+	if (IS_ERR(plat_dat))
+		return dev_err_probe(&pdev->dev, PTR_ERR(plat_dat),
+				     "dt configuration failed\n");
+
+	dwmac = devm_kzalloc(&pdev->dev, sizeof(*dwmac), GFP_KERNEL);
+	if (!dwmac)
+		return -ENOMEM;
+
+	dwmac->clk_tx = devm_clk_get_enabled(&pdev->dev, "tx");
+	if (IS_ERR(dwmac->clk_tx))
+		return dev_err_probe(&pdev->dev, PTR_ERR(dwmac->clk_tx),
+				     "error getting tx clock\n");
+
+	clk_gtx = devm_clk_get_enabled(&pdev->dev, "gtx");
+	if (IS_ERR(clk_gtx))
+		return dev_err_probe(&pdev->dev, PTR_ERR(clk_gtx),
+				     "error getting gtx clock\n");
+
+	/* Generally, the rgmii_tx clock is provided by the internal clock,
+	 * which needs to match the corresponding clock frequency according
+	 * to different speeds. If the rgmii_tx clock is provided by the
+	 * external rgmii_rxin, there is no need to configure the clock
+	 * internally, because rgmii_rxin will be adaptively adjusted.
+	 */
+	if (!device_property_read_bool(&pdev->dev, "starfive,tx-use-rgmii-clk"))
+		plat_dat->fix_mac_speed = starfive_dwmac_fix_mac_speed;
+
+	dwmac->dev = &pdev->dev;
+	plat_dat->bsp_priv = dwmac;
+	plat_dat->dma_cfg->dche = true;
+
+	err = starfive_dwmac_set_mode(plat_dat);
+	if (err)
+		return err;
+
+	err = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res);
+	if (err) {
+		stmmac_remove_config_dt(pdev, plat_dat);
+		return err;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id starfive_dwmac_match[] = {
+	{ .compatible = "starfive,jh7110-dwmac"	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, starfive_dwmac_match);
+
+static struct platform_driver starfive_dwmac_driver = {
+	.probe  = starfive_dwmac_probe,
+	.remove = stmmac_pltfr_remove,
+	.driver = {
+		.name = "starfive-dwmac",
+		.pm = &stmmac_pltfr_pm_ops,
+		.of_match_table = starfive_dwmac_match,
+	},
+};
+module_platform_driver(starfive_dwmac_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("StarFive DWMAC platform driver");
+MODULE_AUTHOR("Emil Renner Berthing <kernel@esmil.dk>");
+MODULE_AUTHOR("Samin Guo <samin.guo@starfivetech.com>");
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c
index be3b1ebc06ab..465ce66ef9c1 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c
@@ -35,7 +35,7 @@
 #define IS_PHY_IF_MODE_GBIT(iface)	(IS_PHY_IF_MODE_RGMII(iface) || \
 					 iface == PHY_INTERFACE_MODE_GMII)
 
-/* STiH4xx register definitions (STiH415/STiH416/STiH407/STiH410 families)
+/* STiH4xx register definitions (STiH407/STiH410 families)
  *
  * Below table summarizes the clock requirement and clock sources for
  * supported phy interface modes with link speeds.
@@ -75,27 +75,6 @@
 #define STIH4XX_ETH_SEL_INTERNAL_NOTEXT_PHYCLK	BIT(7)
 #define STIH4XX_ETH_SEL_TXCLK_NOT_CLK125	BIT(6)
 
-/* STiD127 register definitions
- *-----------------------
- * src	 |BIT(6)| BIT(7)|
- *-----------------------
- * MII   |  1	|   n/a	|
- *-----------------------
- * RMII  |  n/a	|   1	|
- * clkgen|	|	|
- *-----------------------
- * RMII  |  n/a	|   0	|
- * phyclk|	|	|
- *-----------------------
- * RGMII |  1	|  n/a	|
- * clkgen|	|	|
- *-----------------------
- */
-
-#define STID127_RETIME_SRC_MASK			GENMASK(7, 6)
-#define STID127_ETH_SEL_INTERNAL_NOTEXT_PHYCLK	BIT(7)
-#define STID127_ETH_SEL_INTERNAL_NOTEXT_TXCLK	BIT(6)
-
 #define ENMII_MASK	GENMASK(5, 5)
 #define ENMII		BIT(5)
 #define EN_MASK		GENMASK(1, 1)
@@ -194,36 +173,6 @@ static void stih4xx_fix_retime_src(void *priv, u32 spd)
 			   stih4xx_tx_retime_val[src]);
 }
 
-static void stid127_fix_retime_src(void *priv, u32 spd)
-{
-	struct sti_dwmac *dwmac = priv;
-	u32 reg = dwmac->ctrl_reg;
-	u32 freq = 0;
-	u32 val = 0;
-
-	if (dwmac->interface == PHY_INTERFACE_MODE_MII) {
-		val = STID127_ETH_SEL_INTERNAL_NOTEXT_TXCLK;
-	} else if (dwmac->interface == PHY_INTERFACE_MODE_RMII) {
-		if (!dwmac->ext_phyclk) {
-			val = STID127_ETH_SEL_INTERNAL_NOTEXT_PHYCLK;
-			freq = DWMAC_50MHZ;
-		}
-	} else if (IS_PHY_IF_MODE_RGMII(dwmac->interface)) {
-		val = STID127_ETH_SEL_INTERNAL_NOTEXT_TXCLK;
-		if (spd == SPEED_1000)
-			freq = DWMAC_125MHZ;
-		else if (spd == SPEED_100)
-			freq = DWMAC_25MHZ;
-		else if (spd == SPEED_10)
-			freq = DWMAC_2_5MHZ;
-	}
-
-	if (freq)
-		clk_set_rate(dwmac->clk, freq);
-
-	regmap_update_bits(dwmac->regmap, reg, STID127_RETIME_SRC_MASK, val);
-}
-
 static int sti_dwmac_set_mode(struct sti_dwmac *dwmac)
 {
 	struct regmap *regmap = dwmac->regmap;
@@ -408,14 +357,7 @@ static const struct sti_dwmac_of_data stih4xx_dwmac_data = {
 	.fix_retime_src = stih4xx_fix_retime_src,
 };
 
-static const struct sti_dwmac_of_data stid127_dwmac_data = {
-	.fix_retime_src = stid127_fix_retime_src,
-};
-
 static const struct of_device_id sti_dwmac_match[] = {
-	{ .compatible = "st,stih415-dwmac", .data = &stih4xx_dwmac_data},
-	{ .compatible = "st,stih416-dwmac", .data = &stih4xx_dwmac_data},
-	{ .compatible = "st,stid127-dwmac", .data = &stid127_dwmac_data},
 	{ .compatible = "st,stih407-dwmac", .data = &stih4xx_dwmac_data},
 	{ }
 };
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
index f834472599f7..c2c592ba0eb8 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
@@ -304,7 +304,8 @@ static void sun8i_dwmac_dma_init(void __iomem *ioaddr,
 	writel(0x1FFFFFF, ioaddr + EMAC_INT_STA);
 }
 
-static void sun8i_dwmac_dma_init_rx(void __iomem *ioaddr,
+static void sun8i_dwmac_dma_init_rx(struct stmmac_priv *priv,
+				    void __iomem *ioaddr,
 				    struct stmmac_dma_cfg *dma_cfg,
 				    dma_addr_t dma_rx_phy, u32 chan)
 {
@@ -312,7 +313,8 @@ static void sun8i_dwmac_dma_init_rx(void __iomem *ioaddr,
 	writel(lower_32_bits(dma_rx_phy), ioaddr + EMAC_RX_DESC_LIST);
 }
 
-static void sun8i_dwmac_dma_init_tx(void __iomem *ioaddr,
+static void sun8i_dwmac_dma_init_tx(struct stmmac_priv *priv,
+				    void __iomem *ioaddr,
 				    struct stmmac_dma_cfg *dma_cfg,
 				    dma_addr_t dma_tx_phy, u32 chan)
 {
@@ -324,7 +326,8 @@ static void sun8i_dwmac_dma_init_tx(void __iomem *ioaddr,
  * Called from stmmac_dma_ops->dump_regs
  * Used for ethtool
  */
-static void sun8i_dwmac_dump_regs(void __iomem *ioaddr, u32 *reg_space)
+static void sun8i_dwmac_dump_regs(struct stmmac_priv *priv,
+				  void __iomem *ioaddr, u32 *reg_space)
 {
 	int i;
 
@@ -352,7 +355,8 @@ static void sun8i_dwmac_dump_mac_regs(struct mac_device_info *hw,
 	}
 }
 
-static void sun8i_dwmac_enable_dma_irq(void __iomem *ioaddr, u32 chan,
+static void sun8i_dwmac_enable_dma_irq(struct stmmac_priv *priv,
+				       void __iomem *ioaddr, u32 chan,
 				       bool rx, bool tx)
 {
 	u32 value = readl(ioaddr + EMAC_INT_EN);
@@ -365,7 +369,8 @@ static void sun8i_dwmac_enable_dma_irq(void __iomem *ioaddr, u32 chan,
 	writel(value, ioaddr + EMAC_INT_EN);
 }
 
-static void sun8i_dwmac_disable_dma_irq(void __iomem *ioaddr, u32 chan,
+static void sun8i_dwmac_disable_dma_irq(struct stmmac_priv *priv,
+					void __iomem *ioaddr, u32 chan,
 					bool rx, bool tx)
 {
 	u32 value = readl(ioaddr + EMAC_INT_EN);
@@ -378,7 +383,8 @@ static void sun8i_dwmac_disable_dma_irq(void __iomem *ioaddr, u32 chan,
 	writel(value, ioaddr + EMAC_INT_EN);
 }
 
-static void sun8i_dwmac_dma_start_tx(void __iomem *ioaddr, u32 chan)
+static void sun8i_dwmac_dma_start_tx(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 chan)
 {
 	u32 v;
 
@@ -398,7 +404,8 @@ static void sun8i_dwmac_enable_dma_transmission(void __iomem *ioaddr)
 	writel(v, ioaddr + EMAC_TX_CTL1);
 }
 
-static void sun8i_dwmac_dma_stop_tx(void __iomem *ioaddr, u32 chan)
+static void sun8i_dwmac_dma_stop_tx(struct stmmac_priv *priv,
+				    void __iomem *ioaddr, u32 chan)
 {
 	u32 v;
 
@@ -407,7 +414,8 @@ static void sun8i_dwmac_dma_stop_tx(void __iomem *ioaddr, u32 chan)
 	writel(v, ioaddr + EMAC_TX_CTL1);
 }
 
-static void sun8i_dwmac_dma_start_rx(void __iomem *ioaddr, u32 chan)
+static void sun8i_dwmac_dma_start_rx(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 chan)
 {
 	u32 v;
 
@@ -417,7 +425,8 @@ static void sun8i_dwmac_dma_start_rx(void __iomem *ioaddr, u32 chan)
 	writel(v, ioaddr + EMAC_RX_CTL1);
 }
 
-static void sun8i_dwmac_dma_stop_rx(void __iomem *ioaddr, u32 chan)
+static void sun8i_dwmac_dma_stop_rx(struct stmmac_priv *priv,
+				    void __iomem *ioaddr, u32 chan)
 {
 	u32 v;
 
@@ -426,7 +435,8 @@ static void sun8i_dwmac_dma_stop_rx(void __iomem *ioaddr, u32 chan)
 	writel(v, ioaddr + EMAC_RX_CTL1);
 }
 
-static int sun8i_dwmac_dma_interrupt(void __iomem *ioaddr,
+static int sun8i_dwmac_dma_interrupt(struct stmmac_priv *priv,
+				     void __iomem *ioaddr,
 				     struct stmmac_extra_stats *x, u32 chan,
 				     u32 dir)
 {
@@ -492,7 +502,8 @@ static int sun8i_dwmac_dma_interrupt(void __iomem *ioaddr,
 	return ret;
 }
 
-static void sun8i_dwmac_dma_operation_mode_rx(void __iomem *ioaddr, int mode,
+static void sun8i_dwmac_dma_operation_mode_rx(struct stmmac_priv *priv,
+					      void __iomem *ioaddr, int mode,
 					      u32 channel, int fifosz, u8 qmode)
 {
 	u32 v;
@@ -515,7 +526,8 @@ static void sun8i_dwmac_dma_operation_mode_rx(void __iomem *ioaddr, int mode,
 	writel(v, ioaddr + EMAC_RX_CTL1);
 }
 
-static void sun8i_dwmac_dma_operation_mode_tx(void __iomem *ioaddr, int mode,
+static void sun8i_dwmac_dma_operation_mode_tx(struct stmmac_priv *priv,
+					      void __iomem *ioaddr, int mode,
 					      u32 channel, int fifosz, u8 qmode)
 {
 	u32 v;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
index 0e00dd83d027..3927609abc44 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
@@ -414,7 +414,8 @@ static void dwmac1000_get_adv_lp(void __iomem *ioaddr, struct rgmii_adv *adv)
 	dwmac_get_adv_lp(ioaddr, GMAC_PCS_BASE, adv);
 }
 
-static void dwmac1000_debug(void __iomem *ioaddr, struct stmmac_extra_stats *x,
+static void dwmac1000_debug(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    struct stmmac_extra_stats *x,
 			    u32 rx_queues, u32 tx_queues)
 {
 	u32 value = readl(ioaddr + GMAC_DEBUG);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c
index f5581db0ba9b..daf79cdbd3ec 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c
@@ -110,7 +110,8 @@ static void dwmac1000_dma_init(void __iomem *ioaddr,
 	writel(DMA_INTR_DEFAULT_MASK, ioaddr + DMA_INTR_ENA);
 }
 
-static void dwmac1000_dma_init_rx(void __iomem *ioaddr,
+static void dwmac1000_dma_init_rx(struct stmmac_priv *priv,
+				  void __iomem *ioaddr,
 				  struct stmmac_dma_cfg *dma_cfg,
 				  dma_addr_t dma_rx_phy, u32 chan)
 {
@@ -118,7 +119,8 @@ static void dwmac1000_dma_init_rx(void __iomem *ioaddr,
 	writel(lower_32_bits(dma_rx_phy), ioaddr + DMA_RCV_BASE_ADDR);
 }
 
-static void dwmac1000_dma_init_tx(void __iomem *ioaddr,
+static void dwmac1000_dma_init_tx(struct stmmac_priv *priv,
+				  void __iomem *ioaddr,
 				  struct stmmac_dma_cfg *dma_cfg,
 				  dma_addr_t dma_tx_phy, u32 chan)
 {
@@ -147,7 +149,8 @@ static u32 dwmac1000_configure_fc(u32 csr6, int rxfifosz)
 	return csr6;
 }
 
-static void dwmac1000_dma_operation_mode_rx(void __iomem *ioaddr, int mode,
+static void dwmac1000_dma_operation_mode_rx(struct stmmac_priv *priv,
+					    void __iomem *ioaddr, int mode,
 					    u32 channel, int fifosz, u8 qmode)
 {
 	u32 csr6 = readl(ioaddr + DMA_CONTROL);
@@ -175,7 +178,8 @@ static void dwmac1000_dma_operation_mode_rx(void __iomem *ioaddr, int mode,
 	writel(csr6, ioaddr + DMA_CONTROL);
 }
 
-static void dwmac1000_dma_operation_mode_tx(void __iomem *ioaddr, int mode,
+static void dwmac1000_dma_operation_mode_tx(struct stmmac_priv *priv,
+					    void __iomem *ioaddr, int mode,
 					    u32 channel, int fifosz, u8 qmode)
 {
 	u32 csr6 = readl(ioaddr + DMA_CONTROL);
@@ -208,7 +212,8 @@ static void dwmac1000_dma_operation_mode_tx(void __iomem *ioaddr, int mode,
 	writel(csr6, ioaddr + DMA_CONTROL);
 }
 
-static void dwmac1000_dump_dma_regs(void __iomem *ioaddr, u32 *reg_space)
+static void dwmac1000_dump_dma_regs(struct stmmac_priv *priv,
+				    void __iomem *ioaddr, u32 *reg_space)
 {
 	int i;
 
@@ -263,8 +268,8 @@ static int dwmac1000_get_hw_feature(void __iomem *ioaddr,
 	return 0;
 }
 
-static void dwmac1000_rx_watchdog(void __iomem *ioaddr, u32 riwt,
-				  u32 queue)
+static void dwmac1000_rx_watchdog(struct stmmac_priv *priv,
+				  void __iomem *ioaddr, u32 riwt, u32 queue)
 {
 	writel(riwt, ioaddr + DMA_RX_WATCHDOG);
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c
index 8f0d9bc7cab5..1c32b1788f02 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c
@@ -29,7 +29,7 @@ static void dwmac100_dma_init(void __iomem *ioaddr,
 	writel(DMA_INTR_DEFAULT_MASK, ioaddr + DMA_INTR_ENA);
 }
 
-static void dwmac100_dma_init_rx(void __iomem *ioaddr,
+static void dwmac100_dma_init_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
 				 struct stmmac_dma_cfg *dma_cfg,
 				 dma_addr_t dma_rx_phy, u32 chan)
 {
@@ -37,7 +37,7 @@ static void dwmac100_dma_init_rx(void __iomem *ioaddr,
 	writel(lower_32_bits(dma_rx_phy), ioaddr + DMA_RCV_BASE_ADDR);
 }
 
-static void dwmac100_dma_init_tx(void __iomem *ioaddr,
+static void dwmac100_dma_init_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
 				 struct stmmac_dma_cfg *dma_cfg,
 				 dma_addr_t dma_tx_phy, u32 chan)
 {
@@ -50,7 +50,8 @@ static void dwmac100_dma_init_tx(void __iomem *ioaddr,
  * The transmit threshold can be programmed by setting the TTC bits in the DMA
  * control register.
  */
-static void dwmac100_dma_operation_mode_tx(void __iomem *ioaddr, int mode,
+static void dwmac100_dma_operation_mode_tx(struct stmmac_priv *priv,
+					   void __iomem *ioaddr, int mode,
 					   u32 channel, int fifosz, u8 qmode)
 {
 	u32 csr6 = readl(ioaddr + DMA_CONTROL);
@@ -65,7 +66,8 @@ static void dwmac100_dma_operation_mode_tx(void __iomem *ioaddr, int mode,
 	writel(csr6, ioaddr + DMA_CONTROL);
 }
 
-static void dwmac100_dump_dma_regs(void __iomem *ioaddr, u32 *reg_space)
+static void dwmac100_dump_dma_regs(struct stmmac_priv *priv,
+				   void __iomem *ioaddr, u32 *reg_space)
 {
 	int i;
 
@@ -80,10 +82,10 @@ static void dwmac100_dump_dma_regs(void __iomem *ioaddr, u32 *reg_space)
 }
 
 /* DMA controller has two counters to track the number of the missed frames. */
-static void dwmac100_dma_diagnostic_fr(void *data, struct stmmac_extra_stats *x,
+static void dwmac100_dma_diagnostic_fr(struct net_device_stats *stats,
+				       struct stmmac_extra_stats *x,
 				       void __iomem *ioaddr)
 {
-	struct net_device_stats *stats = (struct net_device_stats *)data;
 	u32 csr8 = readl(ioaddr + DMA_MISSED_FRAME_CTR);
 
 	if (unlikely(csr8)) {
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
index ccd49346d3b3..4538f334df57 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
@@ -336,14 +336,25 @@ enum power_event {
 
 #define MTL_CHAN_BASE_ADDR		0x00000d00
 #define MTL_CHAN_BASE_OFFSET		0x40
-#define MTL_CHANX_BASE_ADDR(x)		(MTL_CHAN_BASE_ADDR + \
-					(x * MTL_CHAN_BASE_OFFSET))
 
-#define MTL_CHAN_TX_OP_MODE(x)		MTL_CHANX_BASE_ADDR(x)
-#define MTL_CHAN_TX_DEBUG(x)		(MTL_CHANX_BASE_ADDR(x) + 0x8)
-#define MTL_CHAN_INT_CTRL(x)		(MTL_CHANX_BASE_ADDR(x) + 0x2c)
-#define MTL_CHAN_RX_OP_MODE(x)		(MTL_CHANX_BASE_ADDR(x) + 0x30)
-#define MTL_CHAN_RX_DEBUG(x)		(MTL_CHANX_BASE_ADDR(x) + 0x38)
+static inline u32 mtl_chanx_base_addr(const struct dwmac4_addrs *addrs,
+				      const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_chan + (x * addrs->mtl_chan_offset);
+	else
+		addr = MTL_CHAN_BASE_ADDR + (x * MTL_CHAN_BASE_OFFSET);
+
+	return addr;
+}
+
+#define MTL_CHAN_TX_OP_MODE(addrs, x)	mtl_chanx_base_addr(addrs, x)
+#define MTL_CHAN_TX_DEBUG(addrs, x)	(mtl_chanx_base_addr(addrs, x) + 0x8)
+#define MTL_CHAN_INT_CTRL(addrs, x)	(mtl_chanx_base_addr(addrs, x) + 0x2c)
+#define MTL_CHAN_RX_OP_MODE(addrs, x)	(mtl_chanx_base_addr(addrs, x) + 0x30)
+#define MTL_CHAN_RX_DEBUG(addrs, x)	(mtl_chanx_base_addr(addrs, x) + 0x38)
 
 #define MTL_OP_MODE_RSF			BIT(5)
 #define MTL_OP_MODE_TXQEN_MASK		GENMASK(3, 2)
@@ -388,8 +399,19 @@ enum power_event {
 /* MTL ETS Control register */
 #define MTL_ETS_CTRL_BASE_ADDR		0x00000d10
 #define MTL_ETS_CTRL_BASE_OFFSET	0x40
-#define MTL_ETSX_CTRL_BASE_ADDR(x)	(MTL_ETS_CTRL_BASE_ADDR + \
-					((x) * MTL_ETS_CTRL_BASE_OFFSET))
+
+static inline u32 mtl_etsx_ctrl_base_addr(const struct dwmac4_addrs *addrs,
+					  const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_ets_ctrl + (x * addrs->mtl_ets_ctrl_offset);
+	else
+		addr = MTL_ETS_CTRL_BASE_ADDR + (x * MTL_ETS_CTRL_BASE_OFFSET);
+
+	return addr;
+}
 
 #define MTL_ETS_CTRL_CC			BIT(3)
 #define MTL_ETS_CTRL_AVALG		BIT(2)
@@ -397,31 +419,76 @@ enum power_event {
 /* MTL Queue Quantum Weight */
 #define MTL_TXQ_WEIGHT_BASE_ADDR	0x00000d18
 #define MTL_TXQ_WEIGHT_BASE_OFFSET	0x40
-#define MTL_TXQX_WEIGHT_BASE_ADDR(x)	(MTL_TXQ_WEIGHT_BASE_ADDR + \
-					((x) * MTL_TXQ_WEIGHT_BASE_OFFSET))
+
+static inline u32 mtl_txqx_weight_base_addr(const struct dwmac4_addrs *addrs,
+					    const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_txq_weight + (x * addrs->mtl_txq_weight_offset);
+	else
+		addr = MTL_TXQ_WEIGHT_BASE_ADDR + (x * MTL_TXQ_WEIGHT_BASE_OFFSET);
+
+	return addr;
+}
+
 #define MTL_TXQ_WEIGHT_ISCQW_MASK	GENMASK(20, 0)
 
 /* MTL sendSlopeCredit register */
 #define MTL_SEND_SLP_CRED_BASE_ADDR	0x00000d1c
 #define MTL_SEND_SLP_CRED_OFFSET	0x40
-#define MTL_SEND_SLP_CREDX_BASE_ADDR(x)	(MTL_SEND_SLP_CRED_BASE_ADDR + \
-					((x) * MTL_SEND_SLP_CRED_OFFSET))
+
+static inline u32 mtl_send_slp_credx_base_addr(const struct dwmac4_addrs *addrs,
+					       const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_send_slp_cred + (x * addrs->mtl_send_slp_cred_offset);
+	else
+		addr = MTL_SEND_SLP_CRED_BASE_ADDR + (x * MTL_SEND_SLP_CRED_OFFSET);
+
+	return addr;
+}
 
 #define MTL_SEND_SLP_CRED_SSC_MASK	GENMASK(13, 0)
 
 /* MTL hiCredit register */
 #define MTL_HIGH_CRED_BASE_ADDR		0x00000d20
 #define MTL_HIGH_CRED_OFFSET		0x40
-#define MTL_HIGH_CREDX_BASE_ADDR(x)	(MTL_HIGH_CRED_BASE_ADDR + \
-					((x) * MTL_HIGH_CRED_OFFSET))
+
+static inline u32 mtl_high_credx_base_addr(const struct dwmac4_addrs *addrs,
+					   const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_high_cred + (x * addrs->mtl_high_cred_offset);
+	else
+		addr = MTL_HIGH_CRED_BASE_ADDR + (x * MTL_HIGH_CRED_OFFSET);
+
+	return addr;
+}
 
 #define MTL_HIGH_CRED_HC_MASK		GENMASK(28, 0)
 
 /* MTL loCredit register */
 #define MTL_LOW_CRED_BASE_ADDR		0x00000d24
 #define MTL_LOW_CRED_OFFSET		0x40
-#define MTL_LOW_CREDX_BASE_ADDR(x)	(MTL_LOW_CRED_BASE_ADDR + \
-					((x) * MTL_LOW_CRED_OFFSET))
+
+static inline u32 mtl_low_credx_base_addr(const struct dwmac4_addrs *addrs,
+					  const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_low_cred + (x * addrs->mtl_low_cred_offset);
+	else
+		addr = MTL_LOW_CRED_BASE_ADDR + (x * MTL_LOW_CRED_OFFSET);
+
+	return addr;
+}
 
 #define MTL_HIGH_CRED_LC_MASK		GENMASK(28, 0)
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
index 36251ec2589c..afaec3fb9ab6 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
@@ -198,15 +198,18 @@ static void dwmac4_prog_mtl_tx_algorithms(struct mac_device_info *hw,
 	writel(value, ioaddr + MTL_OPERATION_MODE);
 }
 
-static void dwmac4_set_mtl_tx_queue_weight(struct mac_device_info *hw,
+static void dwmac4_set_mtl_tx_queue_weight(struct stmmac_priv *priv,
+					   struct mac_device_info *hw,
 					   u32 weight, u32 queue)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	void __iomem *ioaddr = hw->pcsr;
-	u32 value = readl(ioaddr + MTL_TXQX_WEIGHT_BASE_ADDR(queue));
+	u32 value = readl(ioaddr + mtl_txqx_weight_base_addr(dwmac4_addrs,
+							     queue));
 
 	value &= ~MTL_TXQ_WEIGHT_ISCQW_MASK;
 	value |= weight & MTL_TXQ_WEIGHT_ISCQW_MASK;
-	writel(value, ioaddr + MTL_TXQX_WEIGHT_BASE_ADDR(queue));
+	writel(value, ioaddr + mtl_txqx_weight_base_addr(dwmac4_addrs, queue));
 }
 
 static void dwmac4_map_mtl_dma(struct mac_device_info *hw, u32 queue, u32 chan)
@@ -227,10 +230,12 @@ static void dwmac4_map_mtl_dma(struct mac_device_info *hw, u32 queue, u32 chan)
 	}
 }
 
-static void dwmac4_config_cbs(struct mac_device_info *hw,
+static void dwmac4_config_cbs(struct stmmac_priv *priv,
+			      struct mac_device_info *hw,
 			      u32 send_slope, u32 idle_slope,
 			      u32 high_credit, u32 low_credit, u32 queue)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	void __iomem *ioaddr = hw->pcsr;
 	u32 value;
 
@@ -241,31 +246,33 @@ static void dwmac4_config_cbs(struct mac_device_info *hw,
 	pr_debug("\tlow_credit: 0x%08x\n", low_credit);
 
 	/* enable AV algorithm */
-	value = readl(ioaddr + MTL_ETSX_CTRL_BASE_ADDR(queue));
+	value = readl(ioaddr + mtl_etsx_ctrl_base_addr(dwmac4_addrs, queue));
 	value |= MTL_ETS_CTRL_AVALG;
 	value |= MTL_ETS_CTRL_CC;
-	writel(value, ioaddr + MTL_ETSX_CTRL_BASE_ADDR(queue));
+	writel(value, ioaddr + mtl_etsx_ctrl_base_addr(dwmac4_addrs, queue));
 
 	/* configure send slope */
-	value = readl(ioaddr + MTL_SEND_SLP_CREDX_BASE_ADDR(queue));
+	value = readl(ioaddr + mtl_send_slp_credx_base_addr(dwmac4_addrs,
+							    queue));
 	value &= ~MTL_SEND_SLP_CRED_SSC_MASK;
 	value |= send_slope & MTL_SEND_SLP_CRED_SSC_MASK;
-	writel(value, ioaddr + MTL_SEND_SLP_CREDX_BASE_ADDR(queue));
+	writel(value, ioaddr + mtl_send_slp_credx_base_addr(dwmac4_addrs,
+							    queue));
 
 	/* configure idle slope (same register as tx weight) */
-	dwmac4_set_mtl_tx_queue_weight(hw, idle_slope, queue);
+	dwmac4_set_mtl_tx_queue_weight(priv, hw, idle_slope, queue);
 
 	/* configure high credit */
-	value = readl(ioaddr + MTL_HIGH_CREDX_BASE_ADDR(queue));
+	value = readl(ioaddr + mtl_high_credx_base_addr(dwmac4_addrs, queue));
 	value &= ~MTL_HIGH_CRED_HC_MASK;
 	value |= high_credit & MTL_HIGH_CRED_HC_MASK;
-	writel(value, ioaddr + MTL_HIGH_CREDX_BASE_ADDR(queue));
+	writel(value, ioaddr + mtl_high_credx_base_addr(dwmac4_addrs, queue));
 
 	/* configure high credit */
-	value = readl(ioaddr + MTL_LOW_CREDX_BASE_ADDR(queue));
+	value = readl(ioaddr + mtl_low_credx_base_addr(dwmac4_addrs, queue));
 	value &= ~MTL_HIGH_CRED_LC_MASK;
 	value |= low_credit & MTL_HIGH_CRED_LC_MASK;
-	writel(value, ioaddr + MTL_LOW_CREDX_BASE_ADDR(queue));
+	writel(value, ioaddr + mtl_low_credx_base_addr(dwmac4_addrs, queue));
 }
 
 static void dwmac4_dump_regs(struct mac_device_info *hw, u32 *reg_space)
@@ -759,8 +766,10 @@ static void dwmac4_phystatus(void __iomem *ioaddr, struct stmmac_extra_stats *x)
 	}
 }
 
-static int dwmac4_irq_mtl_status(struct mac_device_info *hw, u32 chan)
+static int dwmac4_irq_mtl_status(struct stmmac_priv *priv,
+				 struct mac_device_info *hw, u32 chan)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	void __iomem *ioaddr = hw->pcsr;
 	u32 mtl_int_qx_status;
 	int ret = 0;
@@ -770,12 +779,13 @@ static int dwmac4_irq_mtl_status(struct mac_device_info *hw, u32 chan)
 	/* Check MTL Interrupt */
 	if (mtl_int_qx_status & MTL_INT_QX(chan)) {
 		/* read Queue x Interrupt status */
-		u32 status = readl(ioaddr + MTL_CHAN_INT_CTRL(chan));
+		u32 status = readl(ioaddr + MTL_CHAN_INT_CTRL(dwmac4_addrs,
+							      chan));
 
 		if (status & MTL_RX_OVERFLOW_INT) {
 			/*  clear Interrupt */
 			writel(status | MTL_RX_OVERFLOW_INT,
-			       ioaddr + MTL_CHAN_INT_CTRL(chan));
+			       ioaddr + MTL_CHAN_INT_CTRL(dwmac4_addrs, chan));
 			ret = CORE_IRQ_MTL_RX_OVERFLOW;
 		}
 	}
@@ -833,14 +843,16 @@ static int dwmac4_irq_status(struct mac_device_info *hw,
 	return ret;
 }
 
-static void dwmac4_debug(void __iomem *ioaddr, struct stmmac_extra_stats *x,
+static void dwmac4_debug(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 struct stmmac_extra_stats *x,
 			 u32 rx_queues, u32 tx_queues)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	u32 value;
 	u32 queue;
 
 	for (queue = 0; queue < tx_queues; queue++) {
-		value = readl(ioaddr + MTL_CHAN_TX_DEBUG(queue));
+		value = readl(ioaddr + MTL_CHAN_TX_DEBUG(dwmac4_addrs, queue));
 
 		if (value & MTL_DEBUG_TXSTSFSTS)
 			x->mtl_tx_status_fifo_full++;
@@ -865,7 +877,7 @@ static void dwmac4_debug(void __iomem *ioaddr, struct stmmac_extra_stats *x,
 	}
 
 	for (queue = 0; queue < rx_queues; queue++) {
-		value = readl(ioaddr + MTL_CHAN_RX_DEBUG(queue));
+		value = readl(ioaddr + MTL_CHAN_RX_DEBUG(dwmac4_addrs, queue));
 
 		if (value & MTL_DEBUG_RXFSTS_MASK) {
 			u32 rxfsts = (value & MTL_DEBUG_RXFSTS_MASK)
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
index 8cc80b1db4cb..6a011d8633e8 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
@@ -13,11 +13,11 @@
 #include "dwmac4.h"
 #include "dwmac4_descs.h"
 
-static int dwmac4_wrback_get_tx_status(void *data, struct stmmac_extra_stats *x,
+static int dwmac4_wrback_get_tx_status(struct net_device_stats *stats,
+				       struct stmmac_extra_stats *x,
 				       struct dma_desc *p,
 				       void __iomem *ioaddr)
 {
-	struct net_device_stats *stats = (struct net_device_stats *)data;
 	unsigned int tdes3;
 	int ret = tx_done;
 
@@ -73,10 +73,10 @@ static int dwmac4_wrback_get_tx_status(void *data, struct stmmac_extra_stats *x,
 	return ret;
 }
 
-static int dwmac4_wrback_get_rx_status(void *data, struct stmmac_extra_stats *x,
+static int dwmac4_wrback_get_rx_status(struct net_device_stats *stats,
+				       struct stmmac_extra_stats *x,
 				       struct dma_desc *p)
 {
-	struct net_device_stats *stats = (struct net_device_stats *)data;
 	unsigned int rdes1 = le32_to_cpu(p->des1);
 	unsigned int rdes2 = le32_to_cpu(p->des2);
 	unsigned int rdes3 = le32_to_cpu(p->des3);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
index d99fa028c646..84d3a8551b03 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
@@ -13,6 +13,7 @@
 #include <linux/io.h>
 #include "dwmac4.h"
 #include "dwmac4_dma.h"
+#include "stmmac.h"
 
 static void dwmac4_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi)
 {
@@ -68,77 +69,87 @@ static void dwmac4_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi)
 	writel(value, ioaddr + DMA_SYS_BUS_MODE);
 }
 
-static void dwmac4_dma_init_rx_chan(void __iomem *ioaddr,
+static void dwmac4_dma_init_rx_chan(struct stmmac_priv *priv,
+				    void __iomem *ioaddr,
 				    struct stmmac_dma_cfg *dma_cfg,
 				    dma_addr_t dma_rx_phy, u32 chan)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	u32 value;
 	u32 rxpbl = dma_cfg->rxpbl ?: dma_cfg->pbl;
 
-	value = readl(ioaddr + DMA_CHAN_RX_CONTROL(chan));
+	value = readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
 	value = value | (rxpbl << DMA_BUS_MODE_RPBL_SHIFT);
-	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
 
 	if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && likely(dma_cfg->eame))
 		writel(upper_32_bits(dma_rx_phy),
-		       ioaddr + DMA_CHAN_RX_BASE_ADDR_HI(chan));
+		       ioaddr + DMA_CHAN_RX_BASE_ADDR_HI(dwmac4_addrs, chan));
 
-	writel(lower_32_bits(dma_rx_phy), ioaddr + DMA_CHAN_RX_BASE_ADDR(chan));
+	writel(lower_32_bits(dma_rx_phy),
+	       ioaddr + DMA_CHAN_RX_BASE_ADDR(dwmac4_addrs, chan));
 }
 
-static void dwmac4_dma_init_tx_chan(void __iomem *ioaddr,
+static void dwmac4_dma_init_tx_chan(struct stmmac_priv *priv,
+				    void __iomem *ioaddr,
 				    struct stmmac_dma_cfg *dma_cfg,
 				    dma_addr_t dma_tx_phy, u32 chan)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	u32 value;
 	u32 txpbl = dma_cfg->txpbl ?: dma_cfg->pbl;
 
-	value = readl(ioaddr + DMA_CHAN_TX_CONTROL(chan));
+	value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 	value = value | (txpbl << DMA_BUS_MODE_PBL_SHIFT);
 
 	/* Enable OSP to get best performance */
 	value |= DMA_CONTROL_OSP;
 
-	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 
 	if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && likely(dma_cfg->eame))
 		writel(upper_32_bits(dma_tx_phy),
-		       ioaddr + DMA_CHAN_TX_BASE_ADDR_HI(chan));
+		       ioaddr + DMA_CHAN_TX_BASE_ADDR_HI(dwmac4_addrs, chan));
 
-	writel(lower_32_bits(dma_tx_phy), ioaddr + DMA_CHAN_TX_BASE_ADDR(chan));
+	writel(lower_32_bits(dma_tx_phy),
+	       ioaddr + DMA_CHAN_TX_BASE_ADDR(dwmac4_addrs, chan));
 }
 
-static void dwmac4_dma_init_channel(void __iomem *ioaddr,
+static void dwmac4_dma_init_channel(struct stmmac_priv *priv,
+				    void __iomem *ioaddr,
 				    struct stmmac_dma_cfg *dma_cfg, u32 chan)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	u32 value;
 
 	/* common channel control register config */
-	value = readl(ioaddr + DMA_CHAN_CONTROL(chan));
+	value = readl(ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
 	if (dma_cfg->pblx8)
 		value = value | DMA_BUS_MODE_PBL;
-	writel(value, ioaddr + DMA_CHAN_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
 
 	/* Mask interrupts by writing to CSR7 */
 	writel(DMA_CHAN_INTR_DEFAULT_MASK,
-	       ioaddr + DMA_CHAN_INTR_ENA(chan));
+	       ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 }
 
-static void dwmac410_dma_init_channel(void __iomem *ioaddr,
+static void dwmac410_dma_init_channel(struct stmmac_priv *priv,
+				      void __iomem *ioaddr,
 				      struct stmmac_dma_cfg *dma_cfg, u32 chan)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	u32 value;
 
 	/* common channel control register config */
-	value = readl(ioaddr + DMA_CHAN_CONTROL(chan));
+	value = readl(ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
 	if (dma_cfg->pblx8)
 		value = value | DMA_BUS_MODE_PBL;
 
-	writel(value, ioaddr + DMA_CHAN_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
 
 	/* Mask interrupts by writing to CSR7 */
 	writel(DMA_CHAN_INTR_DEFAULT_MASK_4_10,
-	       ioaddr + DMA_CHAN_INTR_ENA(chan));
+	       ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 }
 
 static void dwmac4_dma_init(void __iomem *ioaddr,
@@ -176,65 +187,78 @@ static void dwmac4_dma_init(void __iomem *ioaddr,
 
 }
 
-static void _dwmac4_dump_dma_regs(void __iomem *ioaddr, u32 channel,
+static void _dwmac4_dump_dma_regs(struct stmmac_priv *priv,
+				  void __iomem *ioaddr, u32 channel,
 				  u32 *reg_space)
 {
-	reg_space[DMA_CHAN_CONTROL(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_CONTROL(channel));
-	reg_space[DMA_CHAN_TX_CONTROL(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_TX_CONTROL(channel));
-	reg_space[DMA_CHAN_RX_CONTROL(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_RX_CONTROL(channel));
-	reg_space[DMA_CHAN_TX_BASE_ADDR(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_TX_BASE_ADDR(channel));
-	reg_space[DMA_CHAN_RX_BASE_ADDR(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_RX_BASE_ADDR(channel));
-	reg_space[DMA_CHAN_TX_END_ADDR(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_TX_END_ADDR(channel));
-	reg_space[DMA_CHAN_RX_END_ADDR(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_RX_END_ADDR(channel));
-	reg_space[DMA_CHAN_TX_RING_LEN(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_TX_RING_LEN(channel));
-	reg_space[DMA_CHAN_RX_RING_LEN(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_RX_RING_LEN(channel));
-	reg_space[DMA_CHAN_INTR_ENA(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_INTR_ENA(channel));
-	reg_space[DMA_CHAN_RX_WATCHDOG(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_RX_WATCHDOG(channel));
-	reg_space[DMA_CHAN_SLOT_CTRL_STATUS(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_SLOT_CTRL_STATUS(channel));
-	reg_space[DMA_CHAN_CUR_TX_DESC(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_CUR_TX_DESC(channel));
-	reg_space[DMA_CHAN_CUR_RX_DESC(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_CUR_RX_DESC(channel));
-	reg_space[DMA_CHAN_CUR_TX_BUF_ADDR(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_CUR_TX_BUF_ADDR(channel));
-	reg_space[DMA_CHAN_CUR_RX_BUF_ADDR(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_CUR_RX_BUF_ADDR(channel));
-	reg_space[DMA_CHAN_STATUS(channel) / 4] =
-		readl(ioaddr + DMA_CHAN_STATUS(channel));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	const struct dwmac4_addrs *default_addrs = NULL;
+
+	/* Purposely save the registers in the "normal" layout, regardless of
+	 * platform modifications, to keep reg_space size constant
+	 */
+	reg_space[DMA_CHAN_CONTROL(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_TX_CONTROL(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_CONTROL(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_TX_BASE_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_TX_BASE_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_BASE_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_BASE_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_TX_END_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_TX_END_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_END_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_END_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_TX_RING_LEN(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_TX_RING_LEN(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_RING_LEN(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_RING_LEN(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_INTR_ENA(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_WATCHDOG(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_WATCHDOG(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_SLOT_CTRL_STATUS(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_SLOT_CTRL_STATUS(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_CUR_TX_DESC(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CUR_TX_DESC(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_CUR_RX_DESC(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CUR_RX_DESC(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_CUR_TX_BUF_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CUR_TX_BUF_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_CUR_RX_BUF_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CUR_RX_BUF_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_STATUS(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_STATUS(dwmac4_addrs, channel));
 }
 
-static void dwmac4_dump_dma_regs(void __iomem *ioaddr, u32 *reg_space)
+static void dwmac4_dump_dma_regs(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 u32 *reg_space)
 {
 	int i;
 
 	for (i = 0; i < DMA_CHANNEL_NB_MAX; i++)
-		_dwmac4_dump_dma_regs(ioaddr, i, reg_space);
+		_dwmac4_dump_dma_regs(priv, ioaddr, i, reg_space);
 }
 
-static void dwmac4_rx_watchdog(void __iomem *ioaddr, u32 riwt, u32 queue)
+static void dwmac4_rx_watchdog(struct stmmac_priv *priv, void __iomem *ioaddr,
+			       u32 riwt, u32 queue)
 {
-	writel(riwt, ioaddr + DMA_CHAN_RX_WATCHDOG(queue));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	writel(riwt, ioaddr + DMA_CHAN_RX_WATCHDOG(dwmac4_addrs, queue));
 }
 
-static void dwmac4_dma_rx_chan_op_mode(void __iomem *ioaddr, int mode,
+static void dwmac4_dma_rx_chan_op_mode(struct stmmac_priv *priv,
+				       void __iomem *ioaddr, int mode,
 				       u32 channel, int fifosz, u8 qmode)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	unsigned int rqs = fifosz / 256 - 1;
 	u32 mtl_rx_op;
 
-	mtl_rx_op = readl(ioaddr + MTL_CHAN_RX_OP_MODE(channel));
+	mtl_rx_op = readl(ioaddr + MTL_CHAN_RX_OP_MODE(dwmac4_addrs, channel));
 
 	if (mode == SF_DMA_MODE) {
 		pr_debug("GMAC: enable RX store and forward mode\n");
@@ -292,13 +316,16 @@ static void dwmac4_dma_rx_chan_op_mode(void __iomem *ioaddr, int mode,
 		mtl_rx_op |= rfa << MTL_OP_MODE_RFA_SHIFT;
 	}
 
-	writel(mtl_rx_op, ioaddr + MTL_CHAN_RX_OP_MODE(channel));
+	writel(mtl_rx_op, ioaddr + MTL_CHAN_RX_OP_MODE(dwmac4_addrs, channel));
 }
 
-static void dwmac4_dma_tx_chan_op_mode(void __iomem *ioaddr, int mode,
+static void dwmac4_dma_tx_chan_op_mode(struct stmmac_priv *priv,
+				       void __iomem *ioaddr, int mode,
 				       u32 channel, int fifosz, u8 qmode)
 {
-	u32 mtl_tx_op = readl(ioaddr + MTL_CHAN_TX_OP_MODE(channel));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 mtl_tx_op = readl(ioaddr + MTL_CHAN_TX_OP_MODE(dwmac4_addrs,
+							   channel));
 	unsigned int tqs = fifosz / 256 - 1;
 
 	if (mode == SF_DMA_MODE) {
@@ -344,7 +371,7 @@ static void dwmac4_dma_tx_chan_op_mode(void __iomem *ioaddr, int mode,
 	mtl_tx_op &= ~MTL_OP_MODE_TQS_MASK;
 	mtl_tx_op |= tqs << MTL_OP_MODE_TQS_SHIFT;
 
-	writel(mtl_tx_op, ioaddr +  MTL_CHAN_TX_OP_MODE(channel));
+	writel(mtl_tx_op, ioaddr +  MTL_CHAN_TX_OP_MODE(dwmac4_addrs, channel));
 }
 
 static int dwmac4_get_hw_feature(void __iomem *ioaddr,
@@ -442,26 +469,31 @@ static int dwmac4_get_hw_feature(void __iomem *ioaddr,
 }
 
 /* Enable/disable TSO feature and set MSS */
-static void dwmac4_enable_tso(void __iomem *ioaddr, bool en, u32 chan)
+static void dwmac4_enable_tso(struct stmmac_priv *priv, void __iomem *ioaddr,
+			      bool en, u32 chan)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	u32 value;
 
 	if (en) {
 		/* enable TSO */
-		value = readl(ioaddr + DMA_CHAN_TX_CONTROL(chan));
+		value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 		writel(value | DMA_CONTROL_TSE,
-		       ioaddr + DMA_CHAN_TX_CONTROL(chan));
+		       ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 	} else {
 		/* enable TSO */
-		value = readl(ioaddr + DMA_CHAN_TX_CONTROL(chan));
+		value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 		writel(value & ~DMA_CONTROL_TSE,
-		       ioaddr + DMA_CHAN_TX_CONTROL(chan));
+		       ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 	}
 }
 
-static void dwmac4_qmode(void __iomem *ioaddr, u32 channel, u8 qmode)
+static void dwmac4_qmode(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 channel, u8 qmode)
 {
-	u32 mtl_tx_op = readl(ioaddr + MTL_CHAN_TX_OP_MODE(channel));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 mtl_tx_op = readl(ioaddr + MTL_CHAN_TX_OP_MODE(dwmac4_addrs,
+							   channel));
 
 	mtl_tx_op &= ~MTL_OP_MODE_TXQEN_MASK;
 	if (qmode != MTL_QUEUE_AVB)
@@ -469,47 +501,54 @@ static void dwmac4_qmode(void __iomem *ioaddr, u32 channel, u8 qmode)
 	else
 		mtl_tx_op |= MTL_OP_MODE_TXQEN_AV;
 
-	writel(mtl_tx_op, ioaddr +  MTL_CHAN_TX_OP_MODE(channel));
+	writel(mtl_tx_op, ioaddr +  MTL_CHAN_TX_OP_MODE(dwmac4_addrs, channel));
 }
 
-static void dwmac4_set_bfsize(void __iomem *ioaddr, int bfsize, u32 chan)
+static void dwmac4_set_bfsize(struct stmmac_priv *priv, void __iomem *ioaddr,
+			      int bfsize, u32 chan)
 {
-	u32 value = readl(ioaddr + DMA_CHAN_RX_CONTROL(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
 
 	value &= ~DMA_RBSZ_MASK;
 	value |= (bfsize << DMA_RBSZ_SHIFT) & DMA_RBSZ_MASK;
 
-	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
 }
 
-static void dwmac4_enable_sph(void __iomem *ioaddr, bool en, u32 chan)
+static void dwmac4_enable_sph(struct stmmac_priv *priv, void __iomem *ioaddr,
+			      bool en, u32 chan)
 {
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
 	u32 value = readl(ioaddr + GMAC_EXT_CONFIG);
 
 	value &= ~GMAC_CONFIG_HDSMS;
 	value |= GMAC_CONFIG_HDSMS_256; /* Segment max 256 bytes */
 	writel(value, ioaddr + GMAC_EXT_CONFIG);
 
-	value = readl(ioaddr + DMA_CHAN_CONTROL(chan));
+	value = readl(ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
 	if (en)
 		value |= DMA_CONTROL_SPH;
 	else
 		value &= ~DMA_CONTROL_SPH;
-	writel(value, ioaddr + DMA_CHAN_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
 }
 
-static int dwmac4_enable_tbs(void __iomem *ioaddr, bool en, u32 chan)
+static int dwmac4_enable_tbs(struct stmmac_priv *priv, void __iomem *ioaddr,
+			     bool en, u32 chan)
 {
-	u32 value = readl(ioaddr + DMA_CHAN_TX_CONTROL(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 
 	if (en)
 		value |= DMA_CONTROL_EDSE;
 	else
 		value &= ~DMA_CONTROL_EDSE;
 
-	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 
-	value = readl(ioaddr + DMA_CHAN_TX_CONTROL(chan)) & DMA_CONTROL_EDSE;
+	value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs,
+						   chan)) & DMA_CONTROL_EDSE;
 	if (en && !value)
 		return -EIO;
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h
index 9321879b599c..358e7dcb6a9a 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h
@@ -95,29 +95,41 @@
 /* Following DMA defines are chanels oriented */
 #define DMA_CHAN_BASE_ADDR		0x00001100
 #define DMA_CHAN_BASE_OFFSET		0x80
-#define DMA_CHANX_BASE_ADDR(x)		(DMA_CHAN_BASE_ADDR + \
-					(x * DMA_CHAN_BASE_OFFSET))
+
+static inline u32 dma_chanx_base_addr(const struct dwmac4_addrs *addrs,
+				      const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->dma_chan + (x * addrs->dma_chan_offset);
+	else
+		addr = DMA_CHAN_BASE_ADDR + (x * DMA_CHAN_BASE_OFFSET);
+
+	return addr;
+}
+
 #define DMA_CHAN_REG_NUMBER		17
 
-#define DMA_CHAN_CONTROL(x)		DMA_CHANX_BASE_ADDR(x)
-#define DMA_CHAN_TX_CONTROL(x)		(DMA_CHANX_BASE_ADDR(x) + 0x4)
-#define DMA_CHAN_RX_CONTROL(x)		(DMA_CHANX_BASE_ADDR(x) + 0x8)
-#define DMA_CHAN_TX_BASE_ADDR_HI(x)	(DMA_CHANX_BASE_ADDR(x) + 0x10)
-#define DMA_CHAN_TX_BASE_ADDR(x)	(DMA_CHANX_BASE_ADDR(x) + 0x14)
-#define DMA_CHAN_RX_BASE_ADDR_HI(x)	(DMA_CHANX_BASE_ADDR(x) + 0x18)
-#define DMA_CHAN_RX_BASE_ADDR(x)	(DMA_CHANX_BASE_ADDR(x) + 0x1c)
-#define DMA_CHAN_TX_END_ADDR(x)		(DMA_CHANX_BASE_ADDR(x) + 0x20)
-#define DMA_CHAN_RX_END_ADDR(x)		(DMA_CHANX_BASE_ADDR(x) + 0x28)
-#define DMA_CHAN_TX_RING_LEN(x)		(DMA_CHANX_BASE_ADDR(x) + 0x2c)
-#define DMA_CHAN_RX_RING_LEN(x)		(DMA_CHANX_BASE_ADDR(x) + 0x30)
-#define DMA_CHAN_INTR_ENA(x)		(DMA_CHANX_BASE_ADDR(x) + 0x34)
-#define DMA_CHAN_RX_WATCHDOG(x)		(DMA_CHANX_BASE_ADDR(x) + 0x38)
-#define DMA_CHAN_SLOT_CTRL_STATUS(x)	(DMA_CHANX_BASE_ADDR(x) + 0x3c)
-#define DMA_CHAN_CUR_TX_DESC(x)		(DMA_CHANX_BASE_ADDR(x) + 0x44)
-#define DMA_CHAN_CUR_RX_DESC(x)		(DMA_CHANX_BASE_ADDR(x) + 0x4c)
-#define DMA_CHAN_CUR_TX_BUF_ADDR(x)	(DMA_CHANX_BASE_ADDR(x) + 0x54)
-#define DMA_CHAN_CUR_RX_BUF_ADDR(x)	(DMA_CHANX_BASE_ADDR(x) + 0x5c)
-#define DMA_CHAN_STATUS(x)		(DMA_CHANX_BASE_ADDR(x) + 0x60)
+#define DMA_CHAN_CONTROL(addrs, x)	dma_chanx_base_addr(addrs, x)
+#define DMA_CHAN_TX_CONTROL(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x4)
+#define DMA_CHAN_RX_CONTROL(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x8)
+#define DMA_CHAN_TX_BASE_ADDR_HI(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x10)
+#define DMA_CHAN_TX_BASE_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x14)
+#define DMA_CHAN_RX_BASE_ADDR_HI(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x18)
+#define DMA_CHAN_RX_BASE_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x1c)
+#define DMA_CHAN_TX_END_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x20)
+#define DMA_CHAN_RX_END_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x28)
+#define DMA_CHAN_TX_RING_LEN(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x2c)
+#define DMA_CHAN_RX_RING_LEN(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x30)
+#define DMA_CHAN_INTR_ENA(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x34)
+#define DMA_CHAN_RX_WATCHDOG(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x38)
+#define DMA_CHAN_SLOT_CTRL_STATUS(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x3c)
+#define DMA_CHAN_CUR_TX_DESC(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x44)
+#define DMA_CHAN_CUR_RX_DESC(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x4c)
+#define DMA_CHAN_CUR_TX_BUF_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x54)
+#define DMA_CHAN_CUR_RX_BUF_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x5c)
+#define DMA_CHAN_STATUS(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x60)
 
 /* DMA Control X */
 #define DMA_CONTROL_SPH			BIT(24)
@@ -220,19 +232,31 @@
 #define DMA_CHAN0_DBG_STAT_RPS_SHIFT	8
 
 int dwmac4_dma_reset(void __iomem *ioaddr);
-void dwmac4_enable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx);
-void dwmac410_enable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx);
-void dwmac4_disable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx);
-void dwmac410_disable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx);
-void dwmac4_dma_start_tx(void __iomem *ioaddr, u32 chan);
-void dwmac4_dma_stop_tx(void __iomem *ioaddr, u32 chan);
-void dwmac4_dma_start_rx(void __iomem *ioaddr, u32 chan);
-void dwmac4_dma_stop_rx(void __iomem *ioaddr, u32 chan);
-int dwmac4_dma_interrupt(void __iomem *ioaddr,
+void dwmac4_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   u32 chan, bool rx, bool tx);
+void dwmac410_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			     u32 chan, bool rx, bool tx);
+void dwmac4_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 chan, bool rx, bool tx);
+void dwmac410_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			      u32 chan, bool rx, bool tx);
+void dwmac4_dma_start_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan);
+void dwmac4_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+void dwmac4_dma_start_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan);
+void dwmac4_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+int dwmac4_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr,
 			 struct stmmac_extra_stats *x, u32 chan, u32 dir);
-void dwmac4_set_rx_ring_len(void __iomem *ioaddr, u32 len, u32 chan);
-void dwmac4_set_tx_ring_len(void __iomem *ioaddr, u32 len, u32 chan);
-void dwmac4_set_rx_tail_ptr(void __iomem *ioaddr, u32 tail_ptr, u32 chan);
-void dwmac4_set_tx_tail_ptr(void __iomem *ioaddr, u32 tail_ptr, u32 chan);
+void dwmac4_set_rx_ring_len(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 len, u32 chan);
+void dwmac4_set_tx_ring_len(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 len, u32 chan);
+void dwmac4_set_rx_tail_ptr(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 tail_ptr, u32 chan);
+void dwmac4_set_tx_tail_ptr(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 tail_ptr, u32 chan);
 
 #endif /* __DWMAC4_DMA_H__ */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c
index d1c605777985..df41eac54058 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c
@@ -11,6 +11,7 @@
 #include "common.h"
 #include "dwmac4_dma.h"
 #include "dwmac4.h"
+#include "stmmac.h"
 
 int dwmac4_dma_reset(void __iomem *ioaddr)
 {
@@ -25,120 +26,151 @@ int dwmac4_dma_reset(void __iomem *ioaddr)
 				 10000, 1000000);
 }
 
-void dwmac4_set_rx_tail_ptr(void __iomem *ioaddr, u32 tail_ptr, u32 chan)
+void dwmac4_set_rx_tail_ptr(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 tail_ptr, u32 chan)
 {
-	writel(tail_ptr, ioaddr + DMA_CHAN_RX_END_ADDR(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	writel(tail_ptr, ioaddr + DMA_CHAN_RX_END_ADDR(dwmac4_addrs, chan));
 }
 
-void dwmac4_set_tx_tail_ptr(void __iomem *ioaddr, u32 tail_ptr, u32 chan)
+void dwmac4_set_tx_tail_ptr(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 tail_ptr, u32 chan)
 {
-	writel(tail_ptr, ioaddr + DMA_CHAN_TX_END_ADDR(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	writel(tail_ptr, ioaddr + DMA_CHAN_TX_END_ADDR(dwmac4_addrs, chan));
 }
 
-void dwmac4_dma_start_tx(void __iomem *ioaddr, u32 chan)
+void dwmac4_dma_start_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan)
 {
-	u32 value = readl(ioaddr + DMA_CHAN_TX_CONTROL(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 
 	value |= DMA_CONTROL_ST;
-	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 
 	value = readl(ioaddr + GMAC_CONFIG);
 	value |= GMAC_CONFIG_TE;
 	writel(value, ioaddr + GMAC_CONFIG);
 }
 
-void dwmac4_dma_stop_tx(void __iomem *ioaddr, u32 chan)
+void dwmac4_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan)
 {
-	u32 value = readl(ioaddr + DMA_CHAN_TX_CONTROL(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	u32 value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 
 	value &= ~DMA_CONTROL_ST;
-	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
 }
 
-void dwmac4_dma_start_rx(void __iomem *ioaddr, u32 chan)
+void dwmac4_dma_start_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan)
 {
-	u32 value = readl(ioaddr + DMA_CHAN_RX_CONTROL(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	u32 value = readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
 
 	value |= DMA_CONTROL_SR;
 
-	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
 
 	value = readl(ioaddr + GMAC_CONFIG);
 	value |= GMAC_CONFIG_RE;
 	writel(value, ioaddr + GMAC_CONFIG);
 }
 
-void dwmac4_dma_stop_rx(void __iomem *ioaddr, u32 chan)
+void dwmac4_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan)
 {
-	u32 value = readl(ioaddr + DMA_CHAN_RX_CONTROL(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
 
 	value &= ~DMA_CONTROL_SR;
-	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(chan));
+	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
 }
 
-void dwmac4_set_tx_ring_len(void __iomem *ioaddr, u32 len, u32 chan)
+void dwmac4_set_tx_ring_len(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 len, u32 chan)
 {
-	writel(len, ioaddr + DMA_CHAN_TX_RING_LEN(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	writel(len, ioaddr + DMA_CHAN_TX_RING_LEN(dwmac4_addrs, chan));
 }
 
-void dwmac4_set_rx_ring_len(void __iomem *ioaddr, u32 len, u32 chan)
+void dwmac4_set_rx_ring_len(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 len, u32 chan)
 {
-	writel(len, ioaddr + DMA_CHAN_RX_RING_LEN(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	writel(len, ioaddr + DMA_CHAN_RX_RING_LEN(dwmac4_addrs, chan));
 }
 
-void dwmac4_enable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx)
+void dwmac4_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   u32 chan, bool rx, bool tx)
 {
-	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 
 	if (rx)
 		value |= DMA_CHAN_INTR_DEFAULT_RX;
 	if (tx)
 		value |= DMA_CHAN_INTR_DEFAULT_TX;
 
-	writel(value, ioaddr + DMA_CHAN_INTR_ENA(chan));
+	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 }
 
-void dwmac410_enable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx)
+void dwmac410_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			     u32 chan, bool rx, bool tx)
 {
-	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 
 	if (rx)
 		value |= DMA_CHAN_INTR_DEFAULT_RX_4_10;
 	if (tx)
 		value |= DMA_CHAN_INTR_DEFAULT_TX_4_10;
 
-	writel(value, ioaddr + DMA_CHAN_INTR_ENA(chan));
+	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 }
 
-void dwmac4_disable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx)
+void dwmac4_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 chan, bool rx, bool tx)
 {
-	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 
 	if (rx)
 		value &= ~DMA_CHAN_INTR_DEFAULT_RX;
 	if (tx)
 		value &= ~DMA_CHAN_INTR_DEFAULT_TX;
 
-	writel(value, ioaddr + DMA_CHAN_INTR_ENA(chan));
+	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 }
 
-void dwmac410_disable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx)
+void dwmac410_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			      u32 chan, bool rx, bool tx)
 {
-	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 
 	if (rx)
 		value &= ~DMA_CHAN_INTR_DEFAULT_RX_4_10;
 	if (tx)
 		value &= ~DMA_CHAN_INTR_DEFAULT_TX_4_10;
 
-	writel(value, ioaddr + DMA_CHAN_INTR_ENA(chan));
+	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 }
 
-int dwmac4_dma_interrupt(void __iomem *ioaddr,
+int dwmac4_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr,
 			 struct stmmac_extra_stats *x, u32 chan, u32 dir)
 {
-	u32 intr_status = readl(ioaddr + DMA_CHAN_STATUS(chan));
-	u32 intr_en = readl(ioaddr + DMA_CHAN_INTR_ENA(chan));
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 intr_status = readl(ioaddr + DMA_CHAN_STATUS(dwmac4_addrs, chan));
+	u32 intr_en = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 	int ret = 0;
 
 	if (dir == DMA_DIR_RX)
@@ -183,7 +215,8 @@ int dwmac4_dma_interrupt(void __iomem *ioaddr,
 	if (unlikely(intr_status & DMA_CHAN_STATUS_ERI))
 		x->rx_early_irq++;
 
-	writel(intr_status & intr_en, ioaddr + DMA_CHAN_STATUS(chan));
+	writel(intr_status & intr_en,
+	       ioaddr + DMA_CHAN_STATUS(dwmac4_addrs, chan));
 	return ret;
 }
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h
index acd70b9a3173..72672391675f 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h
@@ -153,14 +153,20 @@
 #define NUM_DWMAC4_DMA_REGS	27
 
 void dwmac_enable_dma_transmission(void __iomem *ioaddr);
-void dwmac_enable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx);
-void dwmac_disable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx);
-void dwmac_dma_start_tx(void __iomem *ioaddr, u32 chan);
-void dwmac_dma_stop_tx(void __iomem *ioaddr, u32 chan);
-void dwmac_dma_start_rx(void __iomem *ioaddr, u32 chan);
-void dwmac_dma_stop_rx(void __iomem *ioaddr, u32 chan);
-int dwmac_dma_interrupt(void __iomem *ioaddr, struct stmmac_extra_stats *x,
-			u32 chan, u32 dir);
+void dwmac_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			  u32 chan, bool rx, bool tx);
+void dwmac_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   u32 chan, bool rx, bool tx);
+void dwmac_dma_start_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+void dwmac_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+		       u32 chan);
+void dwmac_dma_start_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+void dwmac_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+		       u32 chan);
+int dwmac_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr,
+			struct stmmac_extra_stats *x, u32 chan, u32 dir);
 int dwmac_dma_reset(void __iomem *ioaddr);
 
 #endif /* __DWMAC_DMA_H__ */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c
index 9b6138b11776..0b6f999a8305 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c
@@ -32,7 +32,8 @@ void dwmac_enable_dma_transmission(void __iomem *ioaddr)
 	writel(1, ioaddr + DMA_XMT_POLL_DEMAND);
 }
 
-void dwmac_enable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx)
+void dwmac_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			  u32 chan, bool rx, bool tx)
 {
 	u32 value = readl(ioaddr + DMA_INTR_ENA);
 
@@ -44,7 +45,8 @@ void dwmac_enable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx)
 	writel(value, ioaddr + DMA_INTR_ENA);
 }
 
-void dwmac_disable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx)
+void dwmac_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   u32 chan, bool rx, bool tx)
 {
 	u32 value = readl(ioaddr + DMA_INTR_ENA);
 
@@ -56,28 +58,30 @@ void dwmac_disable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx)
 	writel(value, ioaddr + DMA_INTR_ENA);
 }
 
-void dwmac_dma_start_tx(void __iomem *ioaddr, u32 chan)
+void dwmac_dma_start_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan)
 {
 	u32 value = readl(ioaddr + DMA_CONTROL);
 	value |= DMA_CONTROL_ST;
 	writel(value, ioaddr + DMA_CONTROL);
 }
 
-void dwmac_dma_stop_tx(void __iomem *ioaddr, u32 chan)
+void dwmac_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr, u32 chan)
 {
 	u32 value = readl(ioaddr + DMA_CONTROL);
 	value &= ~DMA_CONTROL_ST;
 	writel(value, ioaddr + DMA_CONTROL);
 }
 
-void dwmac_dma_start_rx(void __iomem *ioaddr, u32 chan)
+void dwmac_dma_start_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan)
 {
 	u32 value = readl(ioaddr + DMA_CONTROL);
 	value |= DMA_CONTROL_SR;
 	writel(value, ioaddr + DMA_CONTROL);
 }
 
-void dwmac_dma_stop_rx(void __iomem *ioaddr, u32 chan)
+void dwmac_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr, u32 chan)
 {
 	u32 value = readl(ioaddr + DMA_CONTROL);
 	value &= ~DMA_CONTROL_SR;
@@ -154,7 +158,7 @@ static void show_rx_process_state(unsigned int status)
 }
 #endif
 
-int dwmac_dma_interrupt(void __iomem *ioaddr,
+int dwmac_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr,
 			struct stmmac_extra_stats *x, u32 chan, u32 dir)
 {
 	int ret = 0;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c
index c6c4d7948fe5..a0c2ef8bb0ac 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c
@@ -187,7 +187,8 @@ static void dwxgmac2_prog_mtl_tx_algorithms(struct mac_device_info *hw,
 	}
 }
 
-static void dwxgmac2_set_mtl_tx_queue_weight(struct mac_device_info *hw,
+static void dwxgmac2_set_mtl_tx_queue_weight(struct stmmac_priv *priv,
+					     struct mac_device_info *hw,
 					     u32 weight, u32 queue)
 {
 	void __iomem *ioaddr = hw->pcsr;
@@ -212,7 +213,8 @@ static void dwxgmac2_map_mtl_to_dma(struct mac_device_info *hw, u32 queue,
 	writel(value, ioaddr + reg);
 }
 
-static void dwxgmac2_config_cbs(struct mac_device_info *hw,
+static void dwxgmac2_config_cbs(struct stmmac_priv *priv,
+				struct mac_device_info *hw,
 				u32 send_slope, u32 idle_slope,
 				u32 high_credit, u32 low_credit, u32 queue)
 {
@@ -276,7 +278,8 @@ static int dwxgmac2_host_irq_status(struct mac_device_info *hw,
 	return ret;
 }
 
-static int dwxgmac2_host_mtl_irq_status(struct mac_device_info *hw, u32 chan)
+static int dwxgmac2_host_mtl_irq_status(struct stmmac_priv *priv,
+					struct mac_device_info *hw, u32 chan)
 {
 	void __iomem *ioaddr = hw->pcsr;
 	int ret = 0;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
index b1f0c3984a09..13c347ee8be9 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
@@ -8,7 +8,8 @@
 #include "common.h"
 #include "dwxgmac2.h"
 
-static int dwxgmac2_get_tx_status(void *data, struct stmmac_extra_stats *x,
+static int dwxgmac2_get_tx_status(struct net_device_stats *stats,
+				  struct stmmac_extra_stats *x,
 				  struct dma_desc *p, void __iomem *ioaddr)
 {
 	unsigned int tdes3 = le32_to_cpu(p->des3);
@@ -22,7 +23,8 @@ static int dwxgmac2_get_tx_status(void *data, struct stmmac_extra_stats *x,
 	return ret;
 }
 
-static int dwxgmac2_get_rx_status(void *data, struct stmmac_extra_stats *x,
+static int dwxgmac2_get_rx_status(struct net_device_stats *stats,
+				  struct stmmac_extra_stats *x,
 				  struct dma_desc *p)
 {
 	unsigned int rdes3 = le32_to_cpu(p->des3);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c
index 5e98355f422b..dfd53264e036 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c
@@ -33,7 +33,8 @@ static void dwxgmac2_dma_init(void __iomem *ioaddr,
 	writel(value, ioaddr + XGMAC_DMA_SYSBUS_MODE);
 }
 
-static void dwxgmac2_dma_init_chan(void __iomem *ioaddr,
+static void dwxgmac2_dma_init_chan(struct stmmac_priv *priv,
+				   void __iomem *ioaddr,
 				   struct stmmac_dma_cfg *dma_cfg, u32 chan)
 {
 	u32 value = readl(ioaddr + XGMAC_DMA_CH_CONTROL(chan));
@@ -45,7 +46,8 @@ static void dwxgmac2_dma_init_chan(void __iomem *ioaddr,
 	writel(XGMAC_DMA_INT_DEFAULT_EN, ioaddr + XGMAC_DMA_CH_INT_EN(chan));
 }
 
-static void dwxgmac2_dma_init_rx_chan(void __iomem *ioaddr,
+static void dwxgmac2_dma_init_rx_chan(struct stmmac_priv *priv,
+				      void __iomem *ioaddr,
 				      struct stmmac_dma_cfg *dma_cfg,
 				      dma_addr_t phy, u32 chan)
 {
@@ -61,7 +63,8 @@ static void dwxgmac2_dma_init_rx_chan(void __iomem *ioaddr,
 	writel(lower_32_bits(phy), ioaddr + XGMAC_DMA_CH_RxDESC_LADDR(chan));
 }
 
-static void dwxgmac2_dma_init_tx_chan(void __iomem *ioaddr,
+static void dwxgmac2_dma_init_tx_chan(struct stmmac_priv *priv,
+				      void __iomem *ioaddr,
 				      struct stmmac_dma_cfg *dma_cfg,
 				      dma_addr_t phy, u32 chan)
 {
@@ -131,7 +134,8 @@ static void dwxgmac2_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi)
 	writel(XGMAC_RDPS, ioaddr + XGMAC_RX_EDMA_CTRL);
 }
 
-static void dwxgmac2_dma_dump_regs(void __iomem *ioaddr, u32 *reg_space)
+static void dwxgmac2_dma_dump_regs(struct stmmac_priv *priv,
+				   void __iomem *ioaddr, u32 *reg_space)
 {
 	int i;
 
@@ -139,8 +143,8 @@ static void dwxgmac2_dma_dump_regs(void __iomem *ioaddr, u32 *reg_space)
 		reg_space[i] = readl(ioaddr + i * 4);
 }
 
-static void dwxgmac2_dma_rx_mode(void __iomem *ioaddr, int mode,
-				 u32 channel, int fifosz, u8 qmode)
+static void dwxgmac2_dma_rx_mode(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 int mode, u32 channel, int fifosz, u8 qmode)
 {
 	u32 value = readl(ioaddr + XGMAC_MTL_RXQ_OPMODE(channel));
 	unsigned int rqs = fifosz / 256 - 1;
@@ -205,8 +209,8 @@ static void dwxgmac2_dma_rx_mode(void __iomem *ioaddr, int mode,
 	writel(value | XGMAC_RXOIE, ioaddr + XGMAC_MTL_QINTEN(channel));
 }
 
-static void dwxgmac2_dma_tx_mode(void __iomem *ioaddr, int mode,
-				 u32 channel, int fifosz, u8 qmode)
+static void dwxgmac2_dma_tx_mode(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 int mode, u32 channel, int fifosz, u8 qmode)
 {
 	u32 value = readl(ioaddr + XGMAC_MTL_TXQ_OPMODE(channel));
 	unsigned int tqs = fifosz / 256 - 1;
@@ -248,7 +252,8 @@ static void dwxgmac2_dma_tx_mode(void __iomem *ioaddr, int mode,
 	writel(value, ioaddr +  XGMAC_MTL_TXQ_OPMODE(channel));
 }
 
-static void dwxgmac2_enable_dma_irq(void __iomem *ioaddr, u32 chan,
+static void dwxgmac2_enable_dma_irq(struct stmmac_priv *priv,
+				    void __iomem *ioaddr, u32 chan,
 				    bool rx, bool tx)
 {
 	u32 value = readl(ioaddr + XGMAC_DMA_CH_INT_EN(chan));
@@ -261,7 +266,8 @@ static void dwxgmac2_enable_dma_irq(void __iomem *ioaddr, u32 chan,
 	writel(value, ioaddr + XGMAC_DMA_CH_INT_EN(chan));
 }
 
-static void dwxgmac2_disable_dma_irq(void __iomem *ioaddr, u32 chan,
+static void dwxgmac2_disable_dma_irq(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 chan,
 				     bool rx, bool tx)
 {
 	u32 value = readl(ioaddr + XGMAC_DMA_CH_INT_EN(chan));
@@ -274,7 +280,8 @@ static void dwxgmac2_disable_dma_irq(void __iomem *ioaddr, u32 chan,
 	writel(value, ioaddr + XGMAC_DMA_CH_INT_EN(chan));
 }
 
-static void dwxgmac2_dma_start_tx(void __iomem *ioaddr, u32 chan)
+static void dwxgmac2_dma_start_tx(struct stmmac_priv *priv,
+				  void __iomem *ioaddr, u32 chan)
 {
 	u32 value;
 
@@ -287,7 +294,8 @@ static void dwxgmac2_dma_start_tx(void __iomem *ioaddr, u32 chan)
 	writel(value, ioaddr + XGMAC_TX_CONFIG);
 }
 
-static void dwxgmac2_dma_stop_tx(void __iomem *ioaddr, u32 chan)
+static void dwxgmac2_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 u32 chan)
 {
 	u32 value;
 
@@ -300,7 +308,8 @@ static void dwxgmac2_dma_stop_tx(void __iomem *ioaddr, u32 chan)
 	writel(value, ioaddr + XGMAC_TX_CONFIG);
 }
 
-static void dwxgmac2_dma_start_rx(void __iomem *ioaddr, u32 chan)
+static void dwxgmac2_dma_start_rx(struct stmmac_priv *priv,
+				  void __iomem *ioaddr, u32 chan)
 {
 	u32 value;
 
@@ -313,7 +322,8 @@ static void dwxgmac2_dma_start_rx(void __iomem *ioaddr, u32 chan)
 	writel(value, ioaddr + XGMAC_RX_CONFIG);
 }
 
-static void dwxgmac2_dma_stop_rx(void __iomem *ioaddr, u32 chan)
+static void dwxgmac2_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 u32 chan)
 {
 	u32 value;
 
@@ -322,7 +332,8 @@ static void dwxgmac2_dma_stop_rx(void __iomem *ioaddr, u32 chan)
 	writel(value, ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan));
 }
 
-static int dwxgmac2_dma_interrupt(void __iomem *ioaddr,
+static int dwxgmac2_dma_interrupt(struct stmmac_priv *priv,
+				  void __iomem *ioaddr,
 				  struct stmmac_extra_stats *x, u32 chan,
 				  u32 dir)
 {
@@ -449,32 +460,38 @@ static int dwxgmac2_get_hw_feature(void __iomem *ioaddr,
 	return 0;
 }
 
-static void dwxgmac2_rx_watchdog(void __iomem *ioaddr, u32 riwt, u32 queue)
+static void dwxgmac2_rx_watchdog(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 u32 riwt, u32 queue)
 {
 	writel(riwt & XGMAC_RWT, ioaddr + XGMAC_DMA_CH_Rx_WATCHDOG(queue));
 }
 
-static void dwxgmac2_set_rx_ring_len(void __iomem *ioaddr, u32 len, u32 chan)
+static void dwxgmac2_set_rx_ring_len(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 len, u32 chan)
 {
 	writel(len, ioaddr + XGMAC_DMA_CH_RxDESC_RING_LEN(chan));
 }
 
-static void dwxgmac2_set_tx_ring_len(void __iomem *ioaddr, u32 len, u32 chan)
+static void dwxgmac2_set_tx_ring_len(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 len, u32 chan)
 {
 	writel(len, ioaddr + XGMAC_DMA_CH_TxDESC_RING_LEN(chan));
 }
 
-static void dwxgmac2_set_rx_tail_ptr(void __iomem *ioaddr, u32 ptr, u32 chan)
+static void dwxgmac2_set_rx_tail_ptr(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 ptr, u32 chan)
 {
 	writel(ptr, ioaddr + XGMAC_DMA_CH_RxDESC_TAIL_LPTR(chan));
 }
 
-static void dwxgmac2_set_tx_tail_ptr(void __iomem *ioaddr, u32 ptr, u32 chan)
+static void dwxgmac2_set_tx_tail_ptr(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 ptr, u32 chan)
 {
 	writel(ptr, ioaddr + XGMAC_DMA_CH_TxDESC_TAIL_LPTR(chan));
 }
 
-static void dwxgmac2_enable_tso(void __iomem *ioaddr, bool en, u32 chan)
+static void dwxgmac2_enable_tso(struct stmmac_priv *priv, void __iomem *ioaddr,
+				bool en, u32 chan)
 {
 	u32 value = readl(ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
 
@@ -486,7 +503,8 @@ static void dwxgmac2_enable_tso(void __iomem *ioaddr, bool en, u32 chan)
 	writel(value, ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
 }
 
-static void dwxgmac2_qmode(void __iomem *ioaddr, u32 channel, u8 qmode)
+static void dwxgmac2_qmode(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   u32 channel, u8 qmode)
 {
 	u32 value = readl(ioaddr + XGMAC_MTL_TXQ_OPMODE(channel));
 	u32 flow = readl(ioaddr + XGMAC_RX_FLOW_CTRL);
@@ -503,7 +521,8 @@ static void dwxgmac2_qmode(void __iomem *ioaddr, u32 channel, u8 qmode)
 	writel(value, ioaddr +  XGMAC_MTL_TXQ_OPMODE(channel));
 }
 
-static void dwxgmac2_set_bfsize(void __iomem *ioaddr, int bfsize, u32 chan)
+static void dwxgmac2_set_bfsize(struct stmmac_priv *priv, void __iomem *ioaddr,
+				int bfsize, u32 chan)
 {
 	u32 value;
 
@@ -513,7 +532,8 @@ static void dwxgmac2_set_bfsize(void __iomem *ioaddr, int bfsize, u32 chan)
 	writel(value, ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan));
 }
 
-static void dwxgmac2_enable_sph(void __iomem *ioaddr, bool en, u32 chan)
+static void dwxgmac2_enable_sph(struct stmmac_priv *priv, void __iomem *ioaddr,
+				bool en, u32 chan)
 {
 	u32 value = readl(ioaddr + XGMAC_RX_CONFIG);
 
@@ -529,7 +549,8 @@ static void dwxgmac2_enable_sph(void __iomem *ioaddr, bool en, u32 chan)
 	writel(value, ioaddr + XGMAC_DMA_CH_CONTROL(chan));
 }
 
-static int dwxgmac2_enable_tbs(void __iomem *ioaddr, bool en, u32 chan)
+static int dwxgmac2_enable_tbs(struct stmmac_priv *priv, void __iomem *ioaddr,
+			       bool en, u32 chan)
 {
 	u32 value = readl(ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
index 1bcbbd724fb5..a91d8f13a931 100644
--- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
@@ -12,10 +12,10 @@
 #include "common.h"
 #include "descs_com.h"
 
-static int enh_desc_get_tx_status(void *data, struct stmmac_extra_stats *x,
+static int enh_desc_get_tx_status(struct net_device_stats *stats,
+				  struct stmmac_extra_stats *x,
 				  struct dma_desc *p, void __iomem *ioaddr)
 {
-	struct net_device_stats *stats = (struct net_device_stats *)data;
 	unsigned int tdes0 = le32_to_cpu(p->des0);
 	int ret = tx_done;
 
@@ -117,7 +117,8 @@ static int enh_desc_coe_rdes0(int ipc_err, int type, int payload_err)
 	return ret;
 }
 
-static void enh_desc_get_ext_status(void *data, struct stmmac_extra_stats *x,
+static void enh_desc_get_ext_status(struct net_device_stats *stats,
+				    struct stmmac_extra_stats *x,
 				    struct dma_extended_desc *p)
 {
 	unsigned int rdes0 = le32_to_cpu(p->basic.des0);
@@ -181,10 +182,10 @@ static void enh_desc_get_ext_status(void *data, struct stmmac_extra_stats *x,
 	}
 }
 
-static int enh_desc_get_rx_status(void *data, struct stmmac_extra_stats *x,
+static int enh_desc_get_rx_status(struct net_device_stats *stats,
+				  struct stmmac_extra_stats *x,
 				  struct dma_desc *p)
 {
-	struct net_device_stats *stats = (struct net_device_stats *)data;
 	unsigned int rdes0 = le32_to_cpu(p->des0);
 	int ret = good_frame;
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.c b/drivers/net/ethernet/stmicro/stmmac/hwif.c
index bb7114f970f8..b8ba8f2d8041 100644
--- a/drivers/net/ethernet/stmicro/stmmac/hwif.c
+++ b/drivers/net/ethernet/stmicro/stmmac/hwif.c
@@ -87,6 +87,19 @@ static int stmmac_dwxlgmac_quirks(struct stmmac_priv *priv)
 	return 0;
 }
 
+int stmmac_reset(struct stmmac_priv *priv, void __iomem *ioaddr)
+{
+	struct plat_stmmacenet_data *plat = priv ? priv->plat : NULL;
+
+	if (!priv)
+		return -EINVAL;
+
+	if (plat && plat->fix_soc_reset)
+		return plat->fix_soc_reset(plat, ioaddr);
+
+	return stmmac_do_callback(priv, dma, reset, ioaddr);
+}
+
 static const struct stmmac_hwif_entry {
 	bool gmac;
 	bool gmac4;
diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h
index 16a7421715cb..6ee7cf07cfd7 100644
--- a/drivers/net/ethernet/stmicro/stmmac/hwif.h
+++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h
@@ -26,6 +26,7 @@
 })
 
 struct stmmac_extra_stats;
+struct stmmac_priv;
 struct stmmac_safety_stats;
 struct dma_desc;
 struct dma_extended_desc;
@@ -56,8 +57,9 @@ struct stmmac_desc_ops {
 	/* Last tx segment reports the transmit status */
 	int (*get_tx_ls)(struct dma_desc *p);
 	/* Return the transmit status looking at the TDES1 */
-	int (*tx_status)(void *data, struct stmmac_extra_stats *x,
-			struct dma_desc *p, void __iomem *ioaddr);
+	int (*tx_status)(struct net_device_stats *stats,
+			 struct stmmac_extra_stats *x,
+			 struct dma_desc *p, void __iomem *ioaddr);
 	/* Get the buffer size from the descriptor */
 	int (*get_tx_len)(struct dma_desc *p);
 	/* Handle extra events on specific interrupts hw dependent */
@@ -65,10 +67,12 @@ struct stmmac_desc_ops {
 	/* Get the receive frame size */
 	int (*get_rx_frame_len)(struct dma_desc *p, int rx_coe_type);
 	/* Return the reception status looking at the RDES1 */
-	int (*rx_status)(void *data, struct stmmac_extra_stats *x,
-			struct dma_desc *p);
-	void (*rx_extended_status)(void *data, struct stmmac_extra_stats *x,
-			struct dma_extended_desc *p);
+	int (*rx_status)(struct net_device_stats *stats,
+			 struct stmmac_extra_stats *x,
+			 struct dma_desc *p);
+	void (*rx_extended_status)(struct net_device_stats *stats,
+				   struct stmmac_extra_stats *x,
+				   struct dma_extended_desc *p);
 	/* Set tx timestamp enable bit */
 	void (*enable_tx_timestamp) (struct dma_desc *p);
 	/* get tx timestamp status */
@@ -168,110 +172,125 @@ struct stmmac_dma_ops {
 	int (*reset)(void __iomem *ioaddr);
 	void (*init)(void __iomem *ioaddr, struct stmmac_dma_cfg *dma_cfg,
 		     int atds);
-	void (*init_chan)(void __iomem *ioaddr,
+	void (*init_chan)(struct stmmac_priv *priv, void __iomem *ioaddr,
 			  struct stmmac_dma_cfg *dma_cfg, u32 chan);
-	void (*init_rx_chan)(void __iomem *ioaddr,
+	void (*init_rx_chan)(struct stmmac_priv *priv, void __iomem *ioaddr,
 			     struct stmmac_dma_cfg *dma_cfg,
 			     dma_addr_t phy, u32 chan);
-	void (*init_tx_chan)(void __iomem *ioaddr,
+	void (*init_tx_chan)(struct stmmac_priv *priv, void __iomem *ioaddr,
 			     struct stmmac_dma_cfg *dma_cfg,
 			     dma_addr_t phy, u32 chan);
 	/* Configure the AXI Bus Mode Register */
 	void (*axi)(void __iomem *ioaddr, struct stmmac_axi *axi);
 	/* Dump DMA registers */
-	void (*dump_regs)(void __iomem *ioaddr, u32 *reg_space);
-	void (*dma_rx_mode)(void __iomem *ioaddr, int mode, u32 channel,
-			    int fifosz, u8 qmode);
-	void (*dma_tx_mode)(void __iomem *ioaddr, int mode, u32 channel,
+	void (*dump_regs)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			  u32 *reg_space);
+	void (*dma_rx_mode)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    int mode, u32 channel,
 			    int fifosz, u8 qmode);
+	void (*dma_tx_mode)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    int mode, u32 channel, int fifosz, u8 qmode);
 	/* To track extra statistic (if supported) */
-	void (*dma_diagnostic_fr) (void *data, struct stmmac_extra_stats *x,
-				   void __iomem *ioaddr);
+	void (*dma_diagnostic_fr)(struct net_device_stats *stats,
+				  struct stmmac_extra_stats *x,
+				  void __iomem *ioaddr);
 	void (*enable_dma_transmission) (void __iomem *ioaddr);
-	void (*enable_dma_irq)(void __iomem *ioaddr, u32 chan,
-			       bool rx, bool tx);
-	void (*disable_dma_irq)(void __iomem *ioaddr, u32 chan,
-				bool rx, bool tx);
-	void (*start_tx)(void __iomem *ioaddr, u32 chan);
-	void (*stop_tx)(void __iomem *ioaddr, u32 chan);
-	void (*start_rx)(void __iomem *ioaddr, u32 chan);
-	void (*stop_rx)(void __iomem *ioaddr, u32 chan);
-	int (*dma_interrupt) (void __iomem *ioaddr,
-			      struct stmmac_extra_stats *x, u32 chan, u32 dir);
+	void (*enable_dma_irq)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			       u32 chan, bool rx, bool tx);
+	void (*disable_dma_irq)(struct stmmac_priv *priv, void __iomem *ioaddr,
+				u32 chan, bool rx, bool tx);
+	void (*start_tx)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan);
+	void (*stop_tx)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+	void (*start_rx)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan);
+	void (*stop_rx)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+	int (*dma_interrupt)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			     struct stmmac_extra_stats *x, u32 chan, u32 dir);
 	/* If supported then get the optional core features */
 	int (*get_hw_feature)(void __iomem *ioaddr,
 			      struct dma_features *dma_cap);
 	/* Program the HW RX Watchdog */
-	void (*rx_watchdog)(void __iomem *ioaddr, u32 riwt, u32 queue);
-	void (*set_tx_ring_len)(void __iomem *ioaddr, u32 len, u32 chan);
-	void (*set_rx_ring_len)(void __iomem *ioaddr, u32 len, u32 chan);
-	void (*set_rx_tail_ptr)(void __iomem *ioaddr, u32 tail_ptr, u32 chan);
-	void (*set_tx_tail_ptr)(void __iomem *ioaddr, u32 tail_ptr, u32 chan);
-	void (*enable_tso)(void __iomem *ioaddr, bool en, u32 chan);
-	void (*qmode)(void __iomem *ioaddr, u32 channel, u8 qmode);
-	void (*set_bfsize)(void __iomem *ioaddr, int bfsize, u32 chan);
-	void (*enable_sph)(void __iomem *ioaddr, bool en, u32 chan);
-	int (*enable_tbs)(void __iomem *ioaddr, bool en, u32 chan);
+	void (*rx_watchdog)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 riwt, u32 queue);
+	void (*set_tx_ring_len)(struct stmmac_priv *priv, void __iomem *ioaddr,
+				u32 len, u32 chan);
+	void (*set_rx_ring_len)(struct stmmac_priv *priv, void __iomem *ioaddr,
+				u32 len, u32 chan);
+	void (*set_rx_tail_ptr)(struct stmmac_priv *priv, void __iomem *ioaddr,
+				u32 tail_ptr, u32 chan);
+	void (*set_tx_tail_ptr)(struct stmmac_priv *priv, void __iomem *ioaddr,
+				u32 tail_ptr, u32 chan);
+	void (*enable_tso)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   bool en, u32 chan);
+	void (*qmode)(struct stmmac_priv *priv, void __iomem *ioaddr,
+		      u32 channel, u8 qmode);
+	void (*set_bfsize)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   int bfsize, u32 chan);
+	void (*enable_sph)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   bool en, u32 chan);
+	int (*enable_tbs)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			  bool en, u32 chan);
 };
 
-#define stmmac_reset(__priv, __args...) \
-	stmmac_do_callback(__priv, dma, reset, __args)
 #define stmmac_dma_init(__priv, __args...) \
 	stmmac_do_void_callback(__priv, dma, init, __args)
 #define stmmac_init_chan(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, init_chan, __args)
+	stmmac_do_void_callback(__priv, dma, init_chan, __priv, __args)
 #define stmmac_init_rx_chan(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, init_rx_chan, __args)
+	stmmac_do_void_callback(__priv, dma, init_rx_chan, __priv, __args)
 #define stmmac_init_tx_chan(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, init_tx_chan, __args)
+	stmmac_do_void_callback(__priv, dma, init_tx_chan, __priv, __args)
 #define stmmac_axi(__priv, __args...) \
 	stmmac_do_void_callback(__priv, dma, axi, __args)
 #define stmmac_dump_dma_regs(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, dump_regs, __args)
+	stmmac_do_void_callback(__priv, dma, dump_regs, __priv, __args)
 #define stmmac_dma_rx_mode(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, dma_rx_mode, __args)
+	stmmac_do_void_callback(__priv, dma, dma_rx_mode, __priv, __args)
 #define stmmac_dma_tx_mode(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, dma_tx_mode, __args)
+	stmmac_do_void_callback(__priv, dma, dma_tx_mode, __priv, __args)
 #define stmmac_dma_diagnostic_fr(__priv, __args...) \
 	stmmac_do_void_callback(__priv, dma, dma_diagnostic_fr, __args)
 #define stmmac_enable_dma_transmission(__priv, __args...) \
 	stmmac_do_void_callback(__priv, dma, enable_dma_transmission, __args)
 #define stmmac_enable_dma_irq(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, enable_dma_irq, __args)
+	stmmac_do_void_callback(__priv, dma, enable_dma_irq, __priv, __args)
 #define stmmac_disable_dma_irq(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, disable_dma_irq, __args)
+	stmmac_do_void_callback(__priv, dma, disable_dma_irq, __priv, __args)
 #define stmmac_start_tx(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, start_tx, __args)
+	stmmac_do_void_callback(__priv, dma, start_tx, __priv, __args)
 #define stmmac_stop_tx(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, stop_tx, __args)
+	stmmac_do_void_callback(__priv, dma, stop_tx, __priv, __args)
 #define stmmac_start_rx(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, start_rx, __args)
+	stmmac_do_void_callback(__priv, dma, start_rx, __priv, __args)
 #define stmmac_stop_rx(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, stop_rx, __args)
+	stmmac_do_void_callback(__priv, dma, stop_rx, __priv, __args)
 #define stmmac_dma_interrupt_status(__priv, __args...) \
-	stmmac_do_callback(__priv, dma, dma_interrupt, __args)
+	stmmac_do_callback(__priv, dma, dma_interrupt, __priv, __args)
 #define stmmac_get_hw_feature(__priv, __args...) \
 	stmmac_do_callback(__priv, dma, get_hw_feature, __args)
 #define stmmac_rx_watchdog(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, rx_watchdog, __args)
+	stmmac_do_void_callback(__priv, dma, rx_watchdog, __priv, __args)
 #define stmmac_set_tx_ring_len(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, set_tx_ring_len, __args)
+	stmmac_do_void_callback(__priv, dma, set_tx_ring_len, __priv, __args)
 #define stmmac_set_rx_ring_len(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, set_rx_ring_len, __args)
+	stmmac_do_void_callback(__priv, dma, set_rx_ring_len, __priv, __args)
 #define stmmac_set_rx_tail_ptr(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, set_rx_tail_ptr, __args)
+	stmmac_do_void_callback(__priv, dma, set_rx_tail_ptr, __priv, __args)
 #define stmmac_set_tx_tail_ptr(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, set_tx_tail_ptr, __args)
+	stmmac_do_void_callback(__priv, dma, set_tx_tail_ptr, __priv, __args)
 #define stmmac_enable_tso(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, enable_tso, __args)
+	stmmac_do_void_callback(__priv, dma, enable_tso, __priv, __args)
 #define stmmac_dma_qmode(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, qmode, __args)
+	stmmac_do_void_callback(__priv, dma, qmode, __priv, __args)
 #define stmmac_set_dma_bfsize(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, set_bfsize, __args)
+	stmmac_do_void_callback(__priv, dma, set_bfsize, __priv, __args)
 #define stmmac_enable_sph(__priv, __args...) \
-	stmmac_do_void_callback(__priv, dma, enable_sph, __args)
+	stmmac_do_void_callback(__priv, dma, enable_sph, __priv, __args)
 #define stmmac_enable_tbs(__priv, __args...) \
-	stmmac_do_callback(__priv, dma, enable_tbs, __args)
+	stmmac_do_callback(__priv, dma, enable_tbs, __priv, __args)
 
 struct mac_device_info;
 struct net_device;
@@ -303,21 +322,23 @@ struct stmmac_ops {
 	/* Program TX Algorithms */
 	void (*prog_mtl_tx_algorithms)(struct mac_device_info *hw, u32 tx_alg);
 	/* Set MTL TX queues weight */
-	void (*set_mtl_tx_queue_weight)(struct mac_device_info *hw,
+	void (*set_mtl_tx_queue_weight)(struct stmmac_priv *priv,
+					struct mac_device_info *hw,
 					u32 weight, u32 queue);
 	/* RX MTL queue to RX dma mapping */
 	void (*map_mtl_to_dma)(struct mac_device_info *hw, u32 queue, u32 chan);
 	/* Configure AV Algorithm */
-	void (*config_cbs)(struct mac_device_info *hw, u32 send_slope,
-			   u32 idle_slope, u32 high_credit, u32 low_credit,
-			   u32 queue);
+	void (*config_cbs)(struct stmmac_priv *priv, struct mac_device_info *hw,
+			   u32 send_slope, u32 idle_slope, u32 high_credit,
+			   u32 low_credit, u32 queue);
 	/* Dump MAC registers */
 	void (*dump_regs)(struct mac_device_info *hw, u32 *reg_space);
 	/* Handle extra events on specific interrupts hw dependent */
 	int (*host_irq_status)(struct mac_device_info *hw,
 			       struct stmmac_extra_stats *x);
 	/* Handle MTL interrupts */
-	int (*host_mtl_irq_status)(struct mac_device_info *hw, u32 chan);
+	int (*host_mtl_irq_status)(struct stmmac_priv *priv,
+				   struct mac_device_info *hw, u32 chan);
 	/* Multicast filter setting */
 	void (*set_filter)(struct mac_device_info *hw, struct net_device *dev);
 	/* Flow control setting */
@@ -337,8 +358,9 @@ struct stmmac_ops {
 	void (*set_eee_lpi_entry_timer)(struct mac_device_info *hw, int et);
 	void (*set_eee_timer)(struct mac_device_info *hw, int ls, int tw);
 	void (*set_eee_pls)(struct mac_device_info *hw, int link);
-	void (*debug)(void __iomem *ioaddr, struct stmmac_extra_stats *x,
-		      u32 rx_queues, u32 tx_queues);
+	void (*debug)(struct stmmac_priv *priv, void __iomem *ioaddr,
+		      struct stmmac_extra_stats *x, u32 rx_queues,
+		      u32 tx_queues);
 	/* PCS calls */
 	void (*pcs_ctrl_ane)(void __iomem *ioaddr, bool ane, bool srgmi_ral,
 			     bool loopback);
@@ -418,17 +440,17 @@ struct stmmac_ops {
 #define stmmac_prog_mtl_tx_algorithms(__priv, __args...) \
 	stmmac_do_void_callback(__priv, mac, prog_mtl_tx_algorithms, __args)
 #define stmmac_set_mtl_tx_queue_weight(__priv, __args...) \
-	stmmac_do_void_callback(__priv, mac, set_mtl_tx_queue_weight, __args)
+	stmmac_do_void_callback(__priv, mac, set_mtl_tx_queue_weight, __priv, __args)
 #define stmmac_map_mtl_to_dma(__priv, __args...) \
 	stmmac_do_void_callback(__priv, mac, map_mtl_to_dma, __args)
 #define stmmac_config_cbs(__priv, __args...) \
-	stmmac_do_void_callback(__priv, mac, config_cbs, __args)
+	stmmac_do_void_callback(__priv, mac, config_cbs, __priv, __args)
 #define stmmac_dump_mac_regs(__priv, __args...) \
 	stmmac_do_void_callback(__priv, mac, dump_regs, __args)
 #define stmmac_host_irq_status(__priv, __args...) \
 	stmmac_do_callback(__priv, mac, host_irq_status, __args)
 #define stmmac_host_mtl_irq_status(__priv, __args...) \
-	stmmac_do_callback(__priv, mac, host_mtl_irq_status, __args)
+	stmmac_do_callback(__priv, mac, host_mtl_irq_status, __priv, __args)
 #define stmmac_set_filter(__priv, __args...) \
 	stmmac_do_void_callback(__priv, mac, set_filter, __args)
 #define stmmac_flow_ctrl(__priv, __args...) \
@@ -450,11 +472,11 @@ struct stmmac_ops {
 #define stmmac_set_eee_pls(__priv, __args...) \
 	stmmac_do_void_callback(__priv, mac, set_eee_pls, __args)
 #define stmmac_mac_debug(__priv, __args...) \
-	stmmac_do_void_callback(__priv, mac, debug, __args)
+	stmmac_do_void_callback(__priv, mac, debug, __priv, __args)
 #define stmmac_pcs_ctrl_ane(__priv, __args...) \
 	stmmac_do_void_callback(__priv, mac, pcs_ctrl_ane, __args)
 #define stmmac_pcs_rane(__priv, __args...) \
-	stmmac_do_void_callback(__priv, mac, pcs_rane, __args)
+	stmmac_do_void_callback(__priv, mac, pcs_rane, __priv, __args)
 #define stmmac_pcs_get_adv_lp(__priv, __args...) \
 	stmmac_do_void_callback(__priv, mac, pcs_get_adv_lp, __args)
 #define stmmac_safety_feat_config(__priv, __args...) \
@@ -502,8 +524,6 @@ struct stmmac_ops {
 #define stmmac_fpe_irq_status(__priv, __args...) \
 	stmmac_do_callback(__priv, mac, fpe_irq_status, __args)
 
-struct stmmac_priv;
-
 /* PTP and HW Timer helpers */
 struct stmmac_hwtimestamp {
 	void (*config_hw_tstamping) (void __iomem *ioaddr, u32 data);
@@ -535,16 +555,20 @@ struct stmmac_hwtimestamp {
 #define stmmac_timestamp_interrupt(__priv, __args...) \
 	stmmac_do_void_callback(__priv, ptp, timestamp_interrupt, __args)
 
+struct stmmac_tx_queue;
+struct stmmac_rx_queue;
+
 /* Helpers to manage the descriptors for chain and ring modes */
 struct stmmac_mode_ops {
 	void (*init) (void *des, dma_addr_t phy_addr, unsigned int size,
 		      unsigned int extend_desc);
 	unsigned int (*is_jumbo_frm) (int len, int ehn_desc);
-	int (*jumbo_frm)(void *priv, struct sk_buff *skb, int csum);
+	int (*jumbo_frm)(struct stmmac_tx_queue *tx_q, struct sk_buff *skb,
+			 int csum);
 	int (*set_16kib_bfsize)(int mtu);
 	void (*init_desc3)(struct dma_desc *p);
-	void (*refill_desc3) (void *priv, struct dma_desc *p);
-	void (*clean_desc3) (void *priv, struct dma_desc *p);
+	void (*refill_desc3)(struct stmmac_rx_queue *rx_q, struct dma_desc *p);
+	void (*clean_desc3)(struct stmmac_tx_queue *tx_q, struct dma_desc *p);
 };
 
 #define stmmac_mode_init(__priv, __args...) \
@@ -640,6 +664,7 @@ extern const struct stmmac_mmc_ops dwxgmac_mmc_ops;
 #define GMAC_VERSION		0x00000020	/* GMAC CORE Version */
 #define GMAC4_VERSION		0x00000110	/* GMAC4+ CORE Version */
 
+int stmmac_reset(struct stmmac_priv *priv, void __iomem *ioaddr);
 int stmmac_hwif_init(struct stmmac_priv *priv);
 
 #endif /* __STMMAC_HWIF_H__ */
diff --git a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
index e3da4da242ee..350e6670a576 100644
--- a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
@@ -12,10 +12,10 @@
 #include "common.h"
 #include "descs_com.h"
 
-static int ndesc_get_tx_status(void *data, struct stmmac_extra_stats *x,
+static int ndesc_get_tx_status(struct net_device_stats *stats,
+			       struct stmmac_extra_stats *x,
 			       struct dma_desc *p, void __iomem *ioaddr)
 {
-	struct net_device_stats *stats = (struct net_device_stats *)data;
 	unsigned int tdes0 = le32_to_cpu(p->des0);
 	unsigned int tdes1 = le32_to_cpu(p->des1);
 	int ret = tx_done;
@@ -70,12 +70,12 @@ static int ndesc_get_tx_len(struct dma_desc *p)
  * and, if required, updates the multicast statistics.
  * In case of success, it returns good_frame because the GMAC device
  * is supposed to be able to compute the csum in HW. */
-static int ndesc_get_rx_status(void *data, struct stmmac_extra_stats *x,
+static int ndesc_get_rx_status(struct net_device_stats *stats,
+			       struct stmmac_extra_stats *x,
 			       struct dma_desc *p)
 {
 	int ret = good_frame;
 	unsigned int rdes0 = le32_to_cpu(p->des0);
-	struct net_device_stats *stats = (struct net_device_stats *)data;
 
 	if (unlikely(rdes0 & RDES0_OWN))
 		return dma_own;
diff --git a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
index 2b5b17d8b8a0..d218412ca832 100644
--- a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
+++ b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
@@ -14,9 +14,9 @@
 
 #include "stmmac.h"
 
-static int jumbo_frm(void *p, struct sk_buff *skb, int csum)
+static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb,
+		     int csum)
 {
-	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)p;
 	unsigned int nopaged_len = skb_headlen(skb);
 	struct stmmac_priv *priv = tx_q->priv_data;
 	unsigned int entry = tx_q->cur_tx;
@@ -101,9 +101,8 @@ static unsigned int is_jumbo_frm(int len, int enh_desc)
 	return ret;
 }
 
-static void refill_desc3(void *priv_ptr, struct dma_desc *p)
+static void refill_desc3(struct stmmac_rx_queue *rx_q, struct dma_desc *p)
 {
-	struct stmmac_rx_queue *rx_q = priv_ptr;
 	struct stmmac_priv *priv = rx_q->priv_data;
 
 	/* Fill DES3 in case of RING mode */
@@ -117,9 +116,8 @@ static void init_desc3(struct dma_desc *p)
 	p->des3 = cpu_to_le32(le32_to_cpu(p->des2) + BUF_SIZE_8KiB);
 }
 
-static void clean_desc3(void *priv_ptr, struct dma_desc *p)
+static void clean_desc3(struct stmmac_tx_queue *tx_q, struct dma_desc *p)
 {
-	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)priv_ptr;
 	struct stmmac_priv *priv = tx_q->priv_data;
 	unsigned int entry = tx_q->dirty_tx;
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
index 3d15e1e92e18..07ea5ab0a60b 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
@@ -92,6 +92,13 @@ struct stmmac_rx_buffer {
 	dma_addr_t sec_addr;
 };
 
+struct stmmac_xdp_buff {
+	struct xdp_buff xdp;
+	struct stmmac_priv *priv;
+	struct dma_desc *desc;
+	struct dma_desc *ndesc;
+};
+
 struct stmmac_rx_queue {
 	u32 rx_count_frames;
 	u32 queue_index;
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
index 35c8dd92d369..2ae73ab842d4 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
@@ -393,19 +393,10 @@ stmmac_ethtool_set_link_ksettings(struct net_device *dev,
 
 	if (priv->hw->pcs & STMMAC_PCS_RGMII ||
 	    priv->hw->pcs & STMMAC_PCS_SGMII) {
-		u32 mask = ADVERTISED_Autoneg | ADVERTISED_Pause;
-
 		/* Only support ANE */
 		if (cmd->base.autoneg != AUTONEG_ENABLE)
 			return -EINVAL;
 
-		mask &= (ADVERTISED_1000baseT_Half |
-			ADVERTISED_1000baseT_Full |
-			ADVERTISED_100baseT_Half |
-			ADVERTISED_100baseT_Full |
-			ADVERTISED_10baseT_Half |
-			ADVERTISED_10baseT_Full);
-
 		mutex_lock(&priv->lock);
 		stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);
 		mutex_unlock(&priv->lock);
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index d7fcab057032..0fca81507a77 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -1614,6 +1614,12 @@ static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv,
 	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
 	int i;
 
+	/* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes)
+	 * in struct xdp_buff_xsk to stash driver specific information. Thus,
+	 * use this macro to make sure no size violations.
+	 */
+	XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff);
+
 	for (i = 0; i < dma_conf->dma_rx_size; i++) {
 		struct stmmac_rx_buffer *buf;
 		dma_addr_t dma_addr;
@@ -4563,13 +4569,10 @@ dma_map_err:
 
 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
 {
-	struct vlan_ethhdr *veth;
-	__be16 vlan_proto;
+	struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb);
+	__be16 vlan_proto = veth->h_vlan_proto;
 	u16 vlanid;
 
-	veth = (struct vlan_ethhdr *)skb->data;
-	vlan_proto = veth->h_vlan_proto;
-
 	if ((vlan_proto == htons(ETH_P_8021Q) &&
 	     dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
 	    (vlan_proto == htons(ETH_P_8021AD) &&
@@ -4998,6 +5001,16 @@ static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
 	return ret;
 }
 
+static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp)
+{
+	/* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used
+	 * to represent incoming packet, whereas cb field in the same structure
+	 * is used to store driver specific info. Thus, struct stmmac_xdp_buff
+	 * is laid on top of xdp and cb fields of struct xdp_buff_xsk.
+	 */
+	return (struct stmmac_xdp_buff *)xdp;
+}
+
 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
 {
 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
@@ -5027,6 +5040,7 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
 	}
 	while (count < limit) {
 		struct stmmac_rx_buffer *buf;
+		struct stmmac_xdp_buff *ctx;
 		unsigned int buf1_len = 0;
 		struct dma_desc *np, *p;
 		int entry;
@@ -5112,6 +5126,11 @@ read_again:
 			goto read_again;
 		}
 
+		ctx = xsk_buff_to_stmmac_ctx(buf->xdp);
+		ctx->priv = priv;
+		ctx->desc = p;
+		ctx->ndesc = np;
+
 		/* XDP ZC Frame only support primary buffers for now */
 		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
 		len += buf1_len;
@@ -5190,7 +5209,7 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 	enum dma_data_direction dma_dir;
 	unsigned int desc_size;
 	struct sk_buff *skb = NULL;
-	struct xdp_buff xdp;
+	struct stmmac_xdp_buff ctx;
 	int xdp_status = 0;
 	int buf_sz;
 
@@ -5311,17 +5330,22 @@ read_again:
 			dma_sync_single_for_cpu(priv->device, buf->addr,
 						buf1_len, dma_dir);
 
-			xdp_init_buff(&xdp, buf_sz, &rx_q->xdp_rxq);
-			xdp_prepare_buff(&xdp, page_address(buf->page),
-					 buf->page_offset, buf1_len, false);
+			xdp_init_buff(&ctx.xdp, buf_sz, &rx_q->xdp_rxq);
+			xdp_prepare_buff(&ctx.xdp, page_address(buf->page),
+					 buf->page_offset, buf1_len, true);
 
-			pre_len = xdp.data_end - xdp.data_hard_start -
+			pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start -
 				  buf->page_offset;
-			skb = stmmac_xdp_run_prog(priv, &xdp);
+
+			ctx.priv = priv;
+			ctx.desc = p;
+			ctx.ndesc = np;
+
+			skb = stmmac_xdp_run_prog(priv, &ctx.xdp);
 			/* Due xdp_adjust_tail: DMA sync for_device
 			 * cover max len CPU touch
 			 */
-			sync_len = xdp.data_end - xdp.data_hard_start -
+			sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start -
 				   buf->page_offset;
 			sync_len = max(sync_len, pre_len);
 
@@ -5331,7 +5355,7 @@ read_again:
 
 				if (xdp_res & STMMAC_XDP_CONSUMED) {
 					page_pool_put_page(rx_q->page_pool,
-							   virt_to_head_page(xdp.data),
+							   virt_to_head_page(ctx.xdp.data),
 							   sync_len, true);
 					buf->page = NULL;
 					priv->dev->stats.rx_dropped++;
@@ -5359,7 +5383,7 @@ read_again:
 
 		if (!skb) {
 			/* XDP program may expand or reduce tail */
-			buf1_len = xdp.data_end - xdp.data;
+			buf1_len = ctx.xdp.data_end - ctx.xdp.data;
 
 			skb = napi_alloc_skb(&ch->rx_napi, buf1_len);
 			if (!skb) {
@@ -5369,7 +5393,7 @@ read_again:
 			}
 
 			/* XDP program may adjust header */
-			skb_copy_to_linear_data(skb, xdp.data, buf1_len);
+			skb_copy_to_linear_data(skb, ctx.xdp.data, buf1_len);
 			skb_put(skb, buf1_len);
 
 			/* Data payload copied into SKB, page ready for recycle */
@@ -6350,6 +6374,10 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid
 	bool is_double = false;
 	int ret;
 
+	ret = pm_runtime_resume_and_get(priv->device);
+	if (ret < 0)
+		return ret;
+
 	if (be16_to_cpu(proto) == ETH_P_8021AD)
 		is_double = true;
 
@@ -6357,16 +6385,18 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid
 	ret = stmmac_vlan_update(priv, is_double);
 	if (ret) {
 		clear_bit(vid, priv->active_vlans);
-		return ret;
+		goto err_pm_put;
 	}
 
 	if (priv->hw->num_vlan) {
 		ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
 		if (ret)
-			return ret;
+			goto err_pm_put;
 	}
+err_pm_put:
+	pm_runtime_put(priv->device);
 
-	return 0;
+	return ret;
 }
 
 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
@@ -7060,6 +7090,37 @@ void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable)
 	}
 }
 
+static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp)
+{
+	const struct stmmac_xdp_buff *ctx = (void *)_ctx;
+	struct dma_desc *desc_contains_ts = ctx->desc;
+	struct stmmac_priv *priv = ctx->priv;
+	struct dma_desc *ndesc = ctx->ndesc;
+	struct dma_desc *desc = ctx->desc;
+	u64 ns = 0;
+
+	if (!priv->hwts_rx_en)
+		return -ENODATA;
+
+	/* For GMAC4, the valid timestamp is from CTX next desc. */
+	if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
+		desc_contains_ts = ndesc;
+
+	/* Check if timestamp is available */
+	if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) {
+		stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns);
+		ns -= priv->plat->cdc_error_adj;
+		*timestamp = ns_to_ktime(ns);
+		return 0;
+	}
+
+	return -ENODATA;
+}
+
+static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = {
+	.xmo_rx_timestamp		= stmmac_xdp_rx_timestamp,
+};
+
 /**
  * stmmac_dvr_probe
  * @device: device pointer
@@ -7167,6 +7228,8 @@ int stmmac_dvr_probe(struct device *device,
 
 	ndev->netdev_ops = &stmmac_netdev_ops;
 
+	ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops;
+
 	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 			    NETIF_F_RXCSUM;
 	ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
@@ -7253,6 +7316,10 @@ int stmmac_dvr_probe(struct device *device,
 	if (priv->dma_cap.rssen && priv->plat->rss_en)
 		ndev->features |= NETIF_F_RXHASH;
 
+	ndev->vlan_features |= ndev->features;
+	/* TSO doesn't work on VLANs yet */
+	ndev->vlan_features &= ~NETIF_F_TSO;
+
 	/* MTU range: 46 - hw-specific max */
 	ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
 	if (priv->plat->has_xgmac)
@@ -7275,6 +7342,8 @@ int stmmac_dvr_probe(struct device *device,
 	if (flow_ctrl)
 		priv->flow_ctrl = FLOW_AUTO;	/* RX/TX pause on */
 
+	ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
+
 	/* Setup channels NAPI */
 	stmmac_napi_add(ndev);
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
index 21aaa2730ac8..6807c4c1a0a2 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
@@ -281,9 +281,8 @@ static int stmmac_mdio_read_c22(struct mii_bus *bus, int phyaddr, int phyreg)
 	value |= (phyreg << priv->hw->mii.reg_shift) & priv->hw->mii.reg_mask;
 	value |= (priv->clk_csr << priv->hw->mii.clk_csr_shift)
 		& priv->hw->mii.clk_csr_mask;
-	if (priv->plat->has_gmac4) {
+	if (priv->plat->has_gmac4)
 		value |= MII_GMAC4_READ;
-	}
 
 	data = stmmac_mdio_read(priv, data, value);
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
index 067a40fe0a23..eb0b2898daa3 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
@@ -519,7 +519,8 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac)
 	if (of_device_is_compatible(np, "snps,dwmac-4.00") ||
 	    of_device_is_compatible(np, "snps,dwmac-4.10a") ||
 	    of_device_is_compatible(np, "snps,dwmac-4.20a") ||
-	    of_device_is_compatible(np, "snps,dwmac-5.10a")) {
+	    of_device_is_compatible(np, "snps,dwmac-5.10a") ||
+	    of_device_is_compatible(np, "snps,dwmac-5.20")) {
 		plat->has_gmac4 = 1;
 		plat->has_gmac = 0;
 		plat->pmt = 1;
diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c
index b0c7ab74a82e..b93613cd1994 100644
--- a/drivers/net/ethernet/sun/sunhme.c
+++ b/drivers/net/ethernet/sun/sunhme.c
@@ -14,48 +14,44 @@
  *     argument : macaddr=0x00,0x10,0x20,0x30,0x40,0x50
  */
 
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/types.h>
+#include <linux/bitops.h>
+#include <linux/crc32.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/errno.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
 #include <linux/fcntl.h>
-#include <linux/interrupt.h>
-#include <linux/ioport.h>
 #include <linux/in.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-#include <linux/delay.h>
 #include <linux/init.h>
-#include <linux/ethtool.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/kernel.h>
 #include <linux/mii.h>
-#include <linux/crc32.h>
-#include <linux/random.h>
-#include <linux/errno.h>
+#include <linux/mm.h>
+#include <linux/module.h>
 #include <linux/netdevice.h>
-#include <linux/etherdevice.h>
+#include <linux/of_device.h>
+#include <linux/of.h>
+#include <linux/pci.h>
+#include <linux/random.h>
 #include <linux/skbuff.h>
-#include <linux/mm.h>
-#include <linux/bitops.h>
-#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
 
-#include <asm/io.h>
-#include <asm/dma.h>
 #include <asm/byteorder.h>
+#include <asm/dma.h>
+#include <asm/irq.h>
 
 #ifdef CONFIG_SPARC
-#include <linux/of.h>
-#include <linux/of_device.h>
+#include <asm/auxio.h>
 #include <asm/idprom.h>
 #include <asm/openprom.h>
 #include <asm/oplib.h>
 #include <asm/prom.h>
-#include <asm/auxio.h>
-#endif
-#include <linux/uaccess.h>
-
-#include <asm/irq.h>
-
-#ifdef CONFIG_PCI
-#include <linux/pci.h>
 #endif
 
 #include "sunhme.h"
@@ -589,8 +585,6 @@ no_response:
 	return 1;
 }
 
-static int happy_meal_init(struct happy_meal *hp);
-
 static int is_lucent_phy(struct happy_meal *hp)
 {
 	void __iomem *tregs = hp->tcvregs;
@@ -606,6 +600,124 @@ static int is_lucent_phy(struct happy_meal *hp)
 	return ret;
 }
 
+/* hp->happy_lock must be held */
+static void
+happy_meal_begin_auto_negotiation(struct happy_meal *hp,
+				  void __iomem *tregs,
+				  const struct ethtool_link_ksettings *ep)
+{
+	int timeout;
+
+	/* Read all of the registers we are interested in now. */
+	hp->sw_bmsr      = happy_meal_tcvr_read(hp, tregs, MII_BMSR);
+	hp->sw_bmcr      = happy_meal_tcvr_read(hp, tregs, MII_BMCR);
+	hp->sw_physid1   = happy_meal_tcvr_read(hp, tregs, MII_PHYSID1);
+	hp->sw_physid2   = happy_meal_tcvr_read(hp, tregs, MII_PHYSID2);
+
+	/* XXX Check BMSR_ANEGCAPABLE, should not be necessary though. */
+
+	hp->sw_advertise = happy_meal_tcvr_read(hp, tregs, MII_ADVERTISE);
+	if (!ep || ep->base.autoneg == AUTONEG_ENABLE) {
+		/* Advertise everything we can support. */
+		if (hp->sw_bmsr & BMSR_10HALF)
+			hp->sw_advertise |= (ADVERTISE_10HALF);
+		else
+			hp->sw_advertise &= ~(ADVERTISE_10HALF);
+
+		if (hp->sw_bmsr & BMSR_10FULL)
+			hp->sw_advertise |= (ADVERTISE_10FULL);
+		else
+			hp->sw_advertise &= ~(ADVERTISE_10FULL);
+		if (hp->sw_bmsr & BMSR_100HALF)
+			hp->sw_advertise |= (ADVERTISE_100HALF);
+		else
+			hp->sw_advertise &= ~(ADVERTISE_100HALF);
+		if (hp->sw_bmsr & BMSR_100FULL)
+			hp->sw_advertise |= (ADVERTISE_100FULL);
+		else
+			hp->sw_advertise &= ~(ADVERTISE_100FULL);
+		happy_meal_tcvr_write(hp, tregs, MII_ADVERTISE, hp->sw_advertise);
+
+		/* XXX Currently no Happy Meal cards I know off support 100BaseT4,
+		 * XXX and this is because the DP83840 does not support it, changes
+		 * XXX would need to be made to the tx/rx logic in the driver as well
+		 * XXX so I completely skip checking for it in the BMSR for now.
+		 */
+
+		ASD("Advertising [ %s%s%s%s]\n",
+		    hp->sw_advertise & ADVERTISE_10HALF ? "10H " : "",
+		    hp->sw_advertise & ADVERTISE_10FULL ? "10F " : "",
+		    hp->sw_advertise & ADVERTISE_100HALF ? "100H " : "",
+		    hp->sw_advertise & ADVERTISE_100FULL ? "100F " : "");
+
+		/* Enable Auto-Negotiation, this is usually on already... */
+		hp->sw_bmcr |= BMCR_ANENABLE;
+		happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr);
+
+		/* Restart it to make sure it is going. */
+		hp->sw_bmcr |= BMCR_ANRESTART;
+		happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr);
+
+		/* BMCR_ANRESTART self clears when the process has begun. */
+
+		timeout = 64;  /* More than enough. */
+		while (--timeout) {
+			hp->sw_bmcr = happy_meal_tcvr_read(hp, tregs, MII_BMCR);
+			if (!(hp->sw_bmcr & BMCR_ANRESTART))
+				break; /* got it. */
+			udelay(10);
+		}
+		if (!timeout) {
+			netdev_err(hp->dev,
+				   "Happy Meal would not start auto negotiation BMCR=0x%04x\n",
+				   hp->sw_bmcr);
+			netdev_notice(hp->dev,
+				      "Performing force link detection.\n");
+			goto force_link;
+		} else {
+			hp->timer_state = arbwait;
+		}
+	} else {
+force_link:
+		/* Force the link up, trying first a particular mode.
+		 * Either we are here at the request of ethtool or
+		 * because the Happy Meal would not start to autoneg.
+		 */
+
+		/* Disable auto-negotiation in BMCR, enable the duplex and
+		 * speed setting, init the timer state machine, and fire it off.
+		 */
+		if (!ep || ep->base.autoneg == AUTONEG_ENABLE) {
+			hp->sw_bmcr = BMCR_SPEED100;
+		} else {
+			if (ep->base.speed == SPEED_100)
+				hp->sw_bmcr = BMCR_SPEED100;
+			else
+				hp->sw_bmcr = 0;
+			if (ep->base.duplex == DUPLEX_FULL)
+				hp->sw_bmcr |= BMCR_FULLDPLX;
+		}
+		happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr);
+
+		if (!is_lucent_phy(hp)) {
+			/* OK, seems we need do disable the transceiver for the first
+			 * tick to make sure we get an accurate link state at the
+			 * second tick.
+			 */
+			hp->sw_csconfig = happy_meal_tcvr_read(hp, tregs,
+							       DP83840_CSCONFIG);
+			hp->sw_csconfig &= ~(CSCONFIG_TCVDISAB);
+			happy_meal_tcvr_write(hp, tregs, DP83840_CSCONFIG,
+					      hp->sw_csconfig);
+		}
+		hp->timer_state = ltrywait;
+	}
+
+	hp->timer_ticks = 0;
+	hp->happy_timer.expires = jiffies + (12 * HZ)/10;  /* 1.2 sec. */
+	add_timer(&hp->happy_timer);
+}
+
 static void happy_meal_timer(struct timer_list *t)
 {
 	struct happy_meal *hp = from_timer(hp, t, happy_timer);
@@ -743,12 +855,7 @@ static void happy_meal_timer(struct timer_list *t)
 					netdev_notice(hp->dev,
 						      "Link down, cable problem?\n");
 
-					ret = happy_meal_init(hp);
-					if (ret) {
-						/* ho hum... */
-						netdev_err(hp->dev,
-							   "Error, cannot re-init the Happy Meal.\n");
-					}
+					happy_meal_begin_auto_negotiation(hp, tregs, NULL);
 					goto out;
 				}
 				if (!is_lucent_phy(hp)) {
@@ -874,32 +981,6 @@ static void happy_meal_get_counters(struct happy_meal *hp, void __iomem *bregs)
 	hme_write32(hp, bregs + BMAC_LTCTR, 0);
 }
 
-/* hp->happy_lock must be held */
-static void happy_meal_poll_stop(struct happy_meal *hp, void __iomem *tregs)
-{
-	/* If polling disabled or not polling already, nothing to do. */
-	if ((hp->happy_flags & (HFLAG_POLLENABLE | HFLAG_POLL)) !=
-	   (HFLAG_POLLENABLE | HFLAG_POLL)) {
-		ASD("not polling, return\n");
-		return;
-	}
-
-	/* Shut up the MIF. */
-	ASD("were polling, mif ints off, polling off\n");
-	hme_write32(hp, tregs + TCVR_IMASK, 0xffff);
-
-	/* Turn off polling. */
-	hme_write32(hp, tregs + TCVR_CFG,
-		    hme_read32(hp, tregs + TCVR_CFG) & ~(TCV_CFG_PENABLE));
-
-	/* We are no longer polling. */
-	hp->happy_flags &= ~(HFLAG_POLL);
-
-	/* Let the bits set. */
-	udelay(200);
-	ASD("done\n");
-}
-
 /* Only Sun can take such nice parts and fuck up the programming interface
  * like this.  Good job guys...
  */
@@ -1004,57 +1085,26 @@ static int happy_meal_tcvr_reset(struct happy_meal *hp, void __iomem *tregs)
 static void happy_meal_transceiver_check(struct happy_meal *hp, void __iomem *tregs)
 {
 	unsigned long tconfig = hme_read32(hp, tregs + TCVR_CFG);
+	u32 reread = hme_read32(hp, tregs + TCVR_CFG);
 
 	ASD("tcfg=%08lx\n", tconfig);
-	if (hp->happy_flags & HFLAG_POLL) {
-		/* If we are polling, we must stop to get the transceiver type. */
-		if (hp->tcvr_type == internal) {
-			if (tconfig & TCV_CFG_MDIO1) {
-				happy_meal_poll_stop(hp, tregs);
-				hp->paddr = TCV_PADDR_ETX;
-				hp->tcvr_type = external;
-				tconfig &= ~(TCV_CFG_PENABLE);
-				tconfig |= TCV_CFG_PSELECT;
-				hme_write32(hp, tregs + TCVR_CFG, tconfig);
-				ASD("poll stop, internal->external\n");
-			}
-		} else {
-			if (hp->tcvr_type == external) {
-				if (!(hme_read32(hp, tregs + TCVR_STATUS) >> 16)) {
-					happy_meal_poll_stop(hp, tregs);
-					hp->paddr = TCV_PADDR_ITX;
-					hp->tcvr_type = internal;
-					hme_write32(hp, tregs + TCVR_CFG,
-						    hme_read32(hp, tregs + TCVR_CFG) &
-						    ~(TCV_CFG_PSELECT));
-					ASD("poll stop, external->internal\n");
-				}
-			} else {
-				ASD("polling, none\n");
-			}
-		}
+	if (reread & TCV_CFG_MDIO1) {
+		hme_write32(hp, tregs + TCVR_CFG, tconfig | TCV_CFG_PSELECT);
+		hp->paddr = TCV_PADDR_ETX;
+		hp->tcvr_type = external;
+		ASD("not polling, external\n");
 	} else {
-		u32 reread = hme_read32(hp, tregs + TCVR_CFG);
-
-		/* Else we can just work off of the MDIO bits. */
-		if (reread & TCV_CFG_MDIO1) {
-			hme_write32(hp, tregs + TCVR_CFG, tconfig | TCV_CFG_PSELECT);
-			hp->paddr = TCV_PADDR_ETX;
-			hp->tcvr_type = external;
-			ASD("not polling, external\n");
+		if (reread & TCV_CFG_MDIO0) {
+			hme_write32(hp, tregs + TCVR_CFG,
+				    tconfig & ~(TCV_CFG_PSELECT));
+			hp->paddr = TCV_PADDR_ITX;
+			hp->tcvr_type = internal;
+			ASD("not polling, internal\n");
 		} else {
-			if (reread & TCV_CFG_MDIO0) {
-				hme_write32(hp, tregs + TCVR_CFG,
-					    tconfig & ~(TCV_CFG_PSELECT));
-				hp->paddr = TCV_PADDR_ITX;
-				hp->tcvr_type = internal;
-				ASD("not polling, internal\n");
-			} else {
-				netdev_err(hp->dev,
-					   "Transceiver and a coke please.");
-				hp->tcvr_type = none; /* Grrr... */
-				ASD("not polling, none\n");
-			}
+			netdev_err(hp->dev,
+				   "Transceiver and a coke please.");
+			hp->tcvr_type = none; /* Grrr... */
+			ASD("not polling, none\n");
 		}
 	}
 }
@@ -1202,124 +1252,6 @@ static void happy_meal_init_rings(struct happy_meal *hp)
 }
 
 /* hp->happy_lock must be held */
-static void
-happy_meal_begin_auto_negotiation(struct happy_meal *hp,
-				  void __iomem *tregs,
-				  const struct ethtool_link_ksettings *ep)
-{
-	int timeout;
-
-	/* Read all of the registers we are interested in now. */
-	hp->sw_bmsr      = happy_meal_tcvr_read(hp, tregs, MII_BMSR);
-	hp->sw_bmcr      = happy_meal_tcvr_read(hp, tregs, MII_BMCR);
-	hp->sw_physid1   = happy_meal_tcvr_read(hp, tregs, MII_PHYSID1);
-	hp->sw_physid2   = happy_meal_tcvr_read(hp, tregs, MII_PHYSID2);
-
-	/* XXX Check BMSR_ANEGCAPABLE, should not be necessary though. */
-
-	hp->sw_advertise = happy_meal_tcvr_read(hp, tregs, MII_ADVERTISE);
-	if (!ep || ep->base.autoneg == AUTONEG_ENABLE) {
-		/* Advertise everything we can support. */
-		if (hp->sw_bmsr & BMSR_10HALF)
-			hp->sw_advertise |= (ADVERTISE_10HALF);
-		else
-			hp->sw_advertise &= ~(ADVERTISE_10HALF);
-
-		if (hp->sw_bmsr & BMSR_10FULL)
-			hp->sw_advertise |= (ADVERTISE_10FULL);
-		else
-			hp->sw_advertise &= ~(ADVERTISE_10FULL);
-		if (hp->sw_bmsr & BMSR_100HALF)
-			hp->sw_advertise |= (ADVERTISE_100HALF);
-		else
-			hp->sw_advertise &= ~(ADVERTISE_100HALF);
-		if (hp->sw_bmsr & BMSR_100FULL)
-			hp->sw_advertise |= (ADVERTISE_100FULL);
-		else
-			hp->sw_advertise &= ~(ADVERTISE_100FULL);
-		happy_meal_tcvr_write(hp, tregs, MII_ADVERTISE, hp->sw_advertise);
-
-		/* XXX Currently no Happy Meal cards I know off support 100BaseT4,
-		 * XXX and this is because the DP83840 does not support it, changes
-		 * XXX would need to be made to the tx/rx logic in the driver as well
-		 * XXX so I completely skip checking for it in the BMSR for now.
-		 */
-
-		ASD("Advertising [ %s%s%s%s]\n",
-		    hp->sw_advertise & ADVERTISE_10HALF ? "10H " : "",
-		    hp->sw_advertise & ADVERTISE_10FULL ? "10F " : "",
-		    hp->sw_advertise & ADVERTISE_100HALF ? "100H " : "",
-		    hp->sw_advertise & ADVERTISE_100FULL ? "100F " : "");
-
-		/* Enable Auto-Negotiation, this is usually on already... */
-		hp->sw_bmcr |= BMCR_ANENABLE;
-		happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr);
-
-		/* Restart it to make sure it is going. */
-		hp->sw_bmcr |= BMCR_ANRESTART;
-		happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr);
-
-		/* BMCR_ANRESTART self clears when the process has begun. */
-
-		timeout = 64;  /* More than enough. */
-		while (--timeout) {
-			hp->sw_bmcr = happy_meal_tcvr_read(hp, tregs, MII_BMCR);
-			if (!(hp->sw_bmcr & BMCR_ANRESTART))
-				break; /* got it. */
-			udelay(10);
-		}
-		if (!timeout) {
-			netdev_err(hp->dev,
-				   "Happy Meal would not start auto negotiation BMCR=0x%04x\n",
-				   hp->sw_bmcr);
-			netdev_notice(hp->dev,
-				      "Performing force link detection.\n");
-			goto force_link;
-		} else {
-			hp->timer_state = arbwait;
-		}
-	} else {
-force_link:
-		/* Force the link up, trying first a particular mode.
-		 * Either we are here at the request of ethtool or
-		 * because the Happy Meal would not start to autoneg.
-		 */
-
-		/* Disable auto-negotiation in BMCR, enable the duplex and
-		 * speed setting, init the timer state machine, and fire it off.
-		 */
-		if (!ep || ep->base.autoneg == AUTONEG_ENABLE) {
-			hp->sw_bmcr = BMCR_SPEED100;
-		} else {
-			if (ep->base.speed == SPEED_100)
-				hp->sw_bmcr = BMCR_SPEED100;
-			else
-				hp->sw_bmcr = 0;
-			if (ep->base.duplex == DUPLEX_FULL)
-				hp->sw_bmcr |= BMCR_FULLDPLX;
-		}
-		happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr);
-
-		if (!is_lucent_phy(hp)) {
-			/* OK, seems we need do disable the transceiver for the first
-			 * tick to make sure we get an accurate link state at the
-			 * second tick.
-			 */
-			hp->sw_csconfig = happy_meal_tcvr_read(hp, tregs,
-							       DP83840_CSCONFIG);
-			hp->sw_csconfig &= ~(CSCONFIG_TCVDISAB);
-			happy_meal_tcvr_write(hp, tregs, DP83840_CSCONFIG,
-					      hp->sw_csconfig);
-		}
-		hp->timer_state = ltrywait;
-	}
-
-	hp->timer_ticks = 0;
-	hp->happy_timer.expires = jiffies + (12 * HZ)/10;  /* 1.2 sec. */
-	add_timer(&hp->happy_timer);
-}
-
-/* hp->happy_lock must be held */
 static int happy_meal_init(struct happy_meal *hp)
 {
 	const unsigned char *e = &hp->dev->dev_addr[0];
@@ -1341,10 +1273,6 @@ static int happy_meal_init(struct happy_meal *hp)
 		happy_meal_get_counters(hp, bregs);
 	}
 
-	/* Stop polling. */
-	HMD("to happy_meal_poll_stop\n");
-	happy_meal_poll_stop(hp, tregs);
-
 	/* Stop transmitter and receiver. */
 	HMD("to happy_meal_stop\n");
 	happy_meal_stop(hp, gregs);
@@ -1353,11 +1281,6 @@ static int happy_meal_init(struct happy_meal *hp)
 	HMD("to happy_meal_init_rings\n");
 	happy_meal_init_rings(hp);
 
-	/* Shut up the MIF. */
-	HMD("Disable all MIF irqs (old[%08x])\n",
-	    hme_read32(hp, tregs + TCVR_IMASK));
-	hme_write32(hp, tregs + TCVR_IMASK, 0xffff);
-
 	/* See if we can enable the MIF frame on this card to speak to the DP83840. */
 	if (hp->happy_flags & HFLAG_FENABLE) {
 		HMD("use frame old[%08x]\n",
@@ -1612,7 +1535,6 @@ static void happy_meal_set_initial_advertisement(struct happy_meal *hp)
 	void __iomem *gregs	= hp->gregs;
 
 	happy_meal_stop(hp, gregs);
-	hme_write32(hp, tregs + TCVR_IMASK, 0xffff);
 	if (hp->happy_flags & HFLAG_FENABLE)
 		hme_write32(hp, tregs + TCVR_CFG,
 			    hme_read32(hp, tregs + TCVR_CFG) & ~(TCV_CFG_BENABLE));
@@ -1770,34 +1692,6 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status)
 }
 
 /* hp->happy_lock must be held */
-static void happy_meal_mif_interrupt(struct happy_meal *hp)
-{
-	void __iomem *tregs = hp->tcvregs;
-
-	netdev_info(hp->dev, "Link status change.\n");
-	hp->sw_bmcr = happy_meal_tcvr_read(hp, tregs, MII_BMCR);
-	hp->sw_lpa = happy_meal_tcvr_read(hp, tregs, MII_LPA);
-
-	/* Use the fastest transmission protocol possible. */
-	if (hp->sw_lpa & LPA_100FULL) {
-		netdev_info(hp->dev, "Switching to 100Mbps at full duplex.\n");
-		hp->sw_bmcr |= (BMCR_FULLDPLX | BMCR_SPEED100);
-	} else if (hp->sw_lpa & LPA_100HALF) {
-		netdev_info(hp->dev, "Switching to 100MBps at half duplex.\n");
-		hp->sw_bmcr |= BMCR_SPEED100;
-	} else if (hp->sw_lpa & LPA_10FULL) {
-		netdev_info(hp->dev, "Switching to 10MBps at full duplex.\n");
-		hp->sw_bmcr |= BMCR_FULLDPLX;
-	} else {
-		netdev_info(hp->dev, "Using 10Mbps at half duplex.\n");
-	}
-	happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr);
-
-	/* Finally stop polling and shut up the MIF. */
-	happy_meal_poll_stop(hp, tregs);
-}
-
-/* hp->happy_lock must be held */
 static void happy_meal_tx(struct happy_meal *hp)
 {
 	struct happy_meal_txd *txbase = &hp->happy_block->happy_meal_txd[0];
@@ -1972,6 +1866,8 @@ static irqreturn_t happy_meal_interrupt(int irq, void *dev_id)
 	u32 happy_status       = hme_read32(hp, hp->gregs + GREG_STAT);
 
 	HMD("status=%08x\n", happy_status);
+	if (!happy_status)
+		return IRQ_NONE;
 
 	spin_lock(&hp->happy_lock);
 
@@ -1980,9 +1876,6 @@ static irqreturn_t happy_meal_interrupt(int irq, void *dev_id)
 			goto out;
 	}
 
-	if (happy_status & GREG_STAT_MIFIRQ)
-		happy_meal_mif_interrupt(hp);
-
 	if (happy_status & GREG_STAT_TXALL)
 		happy_meal_tx(hp);
 
@@ -1996,66 +1889,16 @@ out:
 	return IRQ_HANDLED;
 }
 
-#ifdef CONFIG_SBUS
-static irqreturn_t quattro_sbus_interrupt(int irq, void *cookie)
-{
-	struct quattro *qp = (struct quattro *) cookie;
-	int i;
-
-	for (i = 0; i < 4; i++) {
-		struct net_device *dev = qp->happy_meals[i];
-		struct happy_meal *hp  = netdev_priv(dev);
-		u32 happy_status       = hme_read32(hp, hp->gregs + GREG_STAT);
-
-		HMD("status=%08x\n", happy_status);
-
-		if (!(happy_status & (GREG_STAT_ERRORS |
-				      GREG_STAT_MIFIRQ |
-				      GREG_STAT_TXALL |
-				      GREG_STAT_RXTOHOST)))
-			continue;
-
-		spin_lock(&hp->happy_lock);
-
-		if (happy_status & GREG_STAT_ERRORS)
-			if (happy_meal_is_not_so_happy(hp, happy_status))
-				goto next;
-
-		if (happy_status & GREG_STAT_MIFIRQ)
-			happy_meal_mif_interrupt(hp);
-
-		if (happy_status & GREG_STAT_TXALL)
-			happy_meal_tx(hp);
-
-		if (happy_status & GREG_STAT_RXTOHOST)
-			happy_meal_rx(hp, dev);
-
-	next:
-		spin_unlock(&hp->happy_lock);
-	}
-	HMD("done\n");
-
-	return IRQ_HANDLED;
-}
-#endif
-
 static int happy_meal_open(struct net_device *dev)
 {
 	struct happy_meal *hp = netdev_priv(dev);
 	int res;
 
-	/* On SBUS Quattro QFE cards, all hme interrupts are concentrated
-	 * into a single source which we register handling at probe time.
-	 */
-	if ((hp->happy_flags & (HFLAG_QUATTRO|HFLAG_PCI)) != HFLAG_QUATTRO) {
-		res = request_irq(hp->irq, happy_meal_interrupt, IRQF_SHARED,
-				  dev->name, dev);
-		if (res) {
-			HMD("EAGAIN\n");
-			netdev_err(dev, "Can't order irq %d to go.\n", hp->irq);
-
-			return -EAGAIN;
-		}
+	res = request_irq(hp->irq, happy_meal_interrupt, IRQF_SHARED,
+			  dev->name, dev);
+	if (res) {
+		netdev_err(dev, "Can't order irq %d to go.\n", hp->irq);
+		return res;
 	}
 
 	HMD("to happy_meal_init\n");
@@ -2064,7 +1907,7 @@ static int happy_meal_open(struct net_device *dev)
 	res = happy_meal_init(hp);
 	spin_unlock_irq(&hp->happy_lock);
 
-	if (res && ((hp->happy_flags & (HFLAG_QUATTRO|HFLAG_PCI)) != HFLAG_QUATTRO))
+	if (res)
 		free_irq(hp->irq, dev);
 	return res;
 }
@@ -2082,12 +1925,7 @@ static int happy_meal_close(struct net_device *dev)
 
 	spin_unlock_irq(&hp->happy_lock);
 
-	/* On Quattro QFE cards, all hme interrupts are concentrated
-	 * into a single source which we register handling at probe
-	 * time and never unregister.
-	 */
-	if ((hp->happy_flags & (HFLAG_QUATTRO|HFLAG_PCI)) != HFLAG_QUATTRO)
-		free_irq(hp->irq, dev);
+	free_irq(hp->irq, dev);
 
 	return 0;
 }
@@ -2420,59 +2258,6 @@ static struct quattro *quattro_sbus_find(struct platform_device *child)
 	platform_set_drvdata(op, qp);
 	return qp;
 }
-
-/* After all quattro cards have been probed, we call these functions
- * to register the IRQ handlers for the cards that have been
- * successfully probed and skip the cards that failed to initialize
- */
-static int __init quattro_sbus_register_irqs(void)
-{
-	struct quattro *qp;
-
-	for (qp = qfe_sbus_list; qp != NULL; qp = qp->next) {
-		struct platform_device *op = qp->quattro_dev;
-		int err, qfe_slot, skip = 0;
-
-		for (qfe_slot = 0; qfe_slot < 4; qfe_slot++) {
-			if (!qp->happy_meals[qfe_slot])
-				skip = 1;
-		}
-		if (skip)
-			continue;
-
-		err = request_irq(op->archdata.irqs[0],
-				  quattro_sbus_interrupt,
-				  IRQF_SHARED, "Quattro",
-				  qp);
-		if (err != 0) {
-			dev_err(&op->dev,
-				"Quattro HME: IRQ registration error %d.\n",
-				err);
-			return err;
-		}
-	}
-
-	return 0;
-}
-
-static void quattro_sbus_free_irqs(void)
-{
-	struct quattro *qp;
-
-	for (qp = qfe_sbus_list; qp != NULL; qp = qp->next) {
-		struct platform_device *op = qp->quattro_dev;
-		int qfe_slot, skip = 0;
-
-		for (qfe_slot = 0; qfe_slot < 4; qfe_slot++) {
-			if (!qp->happy_meals[qfe_slot])
-				skip = 1;
-		}
-		if (skip)
-			continue;
-
-		free_irq(op->archdata.irqs[0], qp);
-	}
-}
 #endif /* CONFIG_SBUS */
 
 #ifdef CONFIG_PCI
@@ -2520,6 +2305,184 @@ static const struct net_device_ops hme_netdev_ops = {
 	.ndo_validate_addr	= eth_validate_addr,
 };
 
+#ifdef CONFIG_PCI
+static int is_quattro_p(struct pci_dev *pdev)
+{
+	struct pci_dev *busdev = pdev->bus->self;
+	struct pci_dev *this_pdev;
+	int n_hmes;
+
+	if (!busdev || busdev->vendor != PCI_VENDOR_ID_DEC ||
+	    busdev->device != PCI_DEVICE_ID_DEC_21153)
+		return 0;
+
+	n_hmes = 0;
+	list_for_each_entry(this_pdev, &pdev->bus->devices, bus_list) {
+		if (this_pdev->vendor == PCI_VENDOR_ID_SUN &&
+		    this_pdev->device == PCI_DEVICE_ID_SUN_HAPPYMEAL)
+			n_hmes++;
+	}
+
+	if (n_hmes != 4)
+		return 0;
+
+	return 1;
+}
+
+/* Fetch MAC address from vital product data of PCI ROM. */
+static int find_eth_addr_in_vpd(void __iomem *rom_base, int len, int index, unsigned char *dev_addr)
+{
+	int this_offset;
+
+	for (this_offset = 0x20; this_offset < len; this_offset++) {
+		void __iomem *p = rom_base + this_offset;
+
+		if (readb(p + 0) != 0x90 ||
+		    readb(p + 1) != 0x00 ||
+		    readb(p + 2) != 0x09 ||
+		    readb(p + 3) != 0x4e ||
+		    readb(p + 4) != 0x41 ||
+		    readb(p + 5) != 0x06)
+			continue;
+
+		this_offset += 6;
+		p += 6;
+
+		if (index == 0) {
+			for (int i = 0; i < 6; i++)
+				dev_addr[i] = readb(p + i);
+			return 1;
+		}
+		index--;
+	}
+	return 0;
+}
+
+static void __maybe_unused get_hme_mac_nonsparc(struct pci_dev *pdev,
+						unsigned char *dev_addr)
+{
+	void __iomem *p;
+	size_t size;
+
+	p = pci_map_rom(pdev, &size);
+	if (p) {
+		int index = 0;
+		int found;
+
+		if (is_quattro_p(pdev))
+			index = PCI_SLOT(pdev->devfn);
+
+		found = readb(p) == 0x55 &&
+			readb(p + 1) == 0xaa &&
+			find_eth_addr_in_vpd(p, (64 * 1024), index, dev_addr);
+		pci_unmap_rom(pdev, p);
+		if (found)
+			return;
+	}
+
+	/* Sun MAC prefix then 3 random bytes. */
+	dev_addr[0] = 0x08;
+	dev_addr[1] = 0x00;
+	dev_addr[2] = 0x20;
+	get_random_bytes(&dev_addr[3], 3);
+}
+#endif
+
+static void happy_meal_addr_init(struct happy_meal *hp,
+				 struct device_node *dp, int qfe_slot)
+{
+	int i;
+
+	for (i = 0; i < 6; i++) {
+		if (macaddr[i] != 0)
+			break;
+	}
+
+	if (i < 6) { /* a mac address was given */
+		u8 addr[ETH_ALEN];
+
+		for (i = 0; i < 6; i++)
+			addr[i] = macaddr[i];
+		eth_hw_addr_set(hp->dev, addr);
+		macaddr[5]++;
+	} else {
+#ifdef CONFIG_SPARC
+		const unsigned char *addr;
+		int len;
+
+		/* If user did not specify a MAC address specifically, use
+		 * the Quattro local-mac-address property...
+		 */
+		if (qfe_slot != -1) {
+			addr = of_get_property(dp, "local-mac-address", &len);
+			if (addr && len == 6) {
+				eth_hw_addr_set(hp->dev, addr);
+				return;
+			}
+		}
+
+		eth_hw_addr_set(hp->dev, idprom->id_ethaddr);
+#else
+		u8 addr[ETH_ALEN];
+
+		get_hme_mac_nonsparc(hp->happy_dev, addr);
+		eth_hw_addr_set(hp->dev, addr);
+#endif
+	}
+}
+
+static int happy_meal_common_probe(struct happy_meal *hp,
+				   struct device_node *dp)
+{
+	struct net_device *dev = hp->dev;
+	int err;
+
+#ifdef CONFIG_SPARC
+	hp->hm_revision = of_getintprop_default(dp, "hm-rev", hp->hm_revision);
+#endif
+
+	/* Now enable the feature flags we can. */
+	if (hp->hm_revision == 0x20 || hp->hm_revision == 0x21)
+		hp->happy_flags |= HFLAG_20_21;
+	else if (hp->hm_revision != 0xa0)
+		hp->happy_flags |= HFLAG_NOT_A0;
+
+	hp->happy_block = dmam_alloc_coherent(hp->dma_dev, PAGE_SIZE,
+					      &hp->hblock_dvma, GFP_KERNEL);
+	if (!hp->happy_block)
+		return -ENOMEM;
+
+	/* Force check of the link first time we are brought up. */
+	hp->linkcheck = 0;
+
+	/* Force timer state to 'asleep' with count of zero. */
+	hp->timer_state = asleep;
+	hp->timer_ticks = 0;
+
+	timer_setup(&hp->happy_timer, happy_meal_timer, 0);
+
+	dev->netdev_ops = &hme_netdev_ops;
+	dev->watchdog_timeo = 5 * HZ;
+	dev->ethtool_ops = &hme_ethtool_ops;
+
+	/* Happy Meal can do it all... */
+	dev->hw_features = NETIF_F_SG | NETIF_F_HW_CSUM;
+	dev->features |= dev->hw_features | NETIF_F_RXCSUM;
+
+
+	/* Grrr, Happy Meal comes up by default not advertising
+	 * full duplex 100baseT capabilities, fix this.
+	 */
+	spin_lock_irq(&hp->happy_lock);
+	happy_meal_set_initial_advertisement(hp);
+	spin_unlock_irq(&hp->happy_lock);
+
+	err = devm_register_netdev(hp->dma_dev, dev);
+	if (err)
+		dev_err(hp->dma_dev, "Cannot register net device, aborting.\n");
+	return err;
+}
+
 #ifdef CONFIG_SBUS
 static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe)
 {
@@ -2527,152 +2490,92 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe)
 	struct quattro *qp = NULL;
 	struct happy_meal *hp;
 	struct net_device *dev;
-	int i, qfe_slot = -1;
-	u8 addr[ETH_ALEN];
-	int err = -ENODEV;
+	int qfe_slot = -1;
+	int err;
 
 	sbus_dp = op->dev.parent->of_node;
 
 	/* We can match PCI devices too, do not accept those here. */
 	if (!of_node_name_eq(sbus_dp, "sbus") && !of_node_name_eq(sbus_dp, "sbi"))
-		return err;
+		return -ENODEV;
 
 	if (is_qfe) {
 		qp = quattro_sbus_find(op);
 		if (qp == NULL)
-			goto err_out;
+			return -ENODEV;
 		for (qfe_slot = 0; qfe_slot < 4; qfe_slot++)
 			if (qp->happy_meals[qfe_slot] == NULL)
 				break;
 		if (qfe_slot == 4)
-			goto err_out;
+			return -ENODEV;
 	}
 
-	err = -ENOMEM;
-	dev = alloc_etherdev(sizeof(struct happy_meal));
+	dev = devm_alloc_etherdev(&op->dev, sizeof(struct happy_meal));
 	if (!dev)
-		goto err_out;
+		return -ENOMEM;
 	SET_NETDEV_DEV(dev, &op->dev);
 
-	/* If user did not specify a MAC address specifically, use
-	 * the Quattro local-mac-address property...
-	 */
-	for (i = 0; i < 6; i++) {
-		if (macaddr[i] != 0)
-			break;
-	}
-	if (i < 6) { /* a mac address was given */
-		for (i = 0; i < 6; i++)
-			addr[i] = macaddr[i];
-		eth_hw_addr_set(dev, addr);
-		macaddr[5]++;
-	} else {
-		const unsigned char *addr;
-		int len;
-
-		addr = of_get_property(dp, "local-mac-address", &len);
-
-		if (qfe_slot != -1 && addr && len == ETH_ALEN)
-			eth_hw_addr_set(dev, addr);
-		else
-			eth_hw_addr_set(dev, idprom->id_ethaddr);
-	}
-
 	hp = netdev_priv(dev);
-
+	hp->dev = dev;
 	hp->happy_dev = op;
 	hp->dma_dev = &op->dev;
+	happy_meal_addr_init(hp, dp, qfe_slot);
 
 	spin_lock_init(&hp->happy_lock);
 
-	err = -ENODEV;
 	if (qp != NULL) {
 		hp->qfe_parent = qp;
 		hp->qfe_ent = qfe_slot;
 		qp->happy_meals[qfe_slot] = dev;
 	}
 
-	hp->gregs = of_ioremap(&op->resource[0], 0,
-			       GREG_REG_SIZE, "HME Global Regs");
-	if (!hp->gregs) {
+	hp->gregs = devm_platform_ioremap_resource(op, 0);
+	if (IS_ERR(hp->gregs)) {
 		dev_err(&op->dev, "Cannot map global registers.\n");
-		goto err_out_free_netdev;
+		err = PTR_ERR(hp->gregs);
+		goto err_out_clear_quattro;
 	}
 
-	hp->etxregs = of_ioremap(&op->resource[1], 0,
-				 ETX_REG_SIZE, "HME TX Regs");
-	if (!hp->etxregs) {
+	hp->etxregs = devm_platform_ioremap_resource(op, 1);
+	if (IS_ERR(hp->etxregs)) {
 		dev_err(&op->dev, "Cannot map MAC TX registers.\n");
-		goto err_out_iounmap;
+		err = PTR_ERR(hp->etxregs);
+		goto err_out_clear_quattro;
 	}
 
-	hp->erxregs = of_ioremap(&op->resource[2], 0,
-				 ERX_REG_SIZE, "HME RX Regs");
-	if (!hp->erxregs) {
+	hp->erxregs = devm_platform_ioremap_resource(op, 2);
+	if (IS_ERR(hp->erxregs)) {
 		dev_err(&op->dev, "Cannot map MAC RX registers.\n");
-		goto err_out_iounmap;
+		err = PTR_ERR(hp->erxregs);
+		goto err_out_clear_quattro;
 	}
 
-	hp->bigmacregs = of_ioremap(&op->resource[3], 0,
-				    BMAC_REG_SIZE, "HME BIGMAC Regs");
-	if (!hp->bigmacregs) {
+	hp->bigmacregs = devm_platform_ioremap_resource(op, 3);
+	if (IS_ERR(hp->bigmacregs)) {
 		dev_err(&op->dev, "Cannot map BIGMAC registers.\n");
-		goto err_out_iounmap;
+		err = PTR_ERR(hp->bigmacregs);
+		goto err_out_clear_quattro;
 	}
 
-	hp->tcvregs = of_ioremap(&op->resource[4], 0,
-				 TCVR_REG_SIZE, "HME Tranceiver Regs");
-	if (!hp->tcvregs) {
+	hp->tcvregs = devm_platform_ioremap_resource(op, 4);
+	if (IS_ERR(hp->tcvregs)) {
 		dev_err(&op->dev, "Cannot map TCVR registers.\n");
-		goto err_out_iounmap;
+		err = PTR_ERR(hp->tcvregs);
+		goto err_out_clear_quattro;
 	}
 
-	hp->hm_revision = of_getintprop_default(dp, "hm-rev", 0xff);
-	if (hp->hm_revision == 0xff)
-		hp->hm_revision = 0xa0;
-
-	/* Now enable the feature flags we can. */
-	if (hp->hm_revision == 0x20 || hp->hm_revision == 0x21)
-		hp->happy_flags = HFLAG_20_21;
-	else if (hp->hm_revision != 0xa0)
-		hp->happy_flags = HFLAG_NOT_A0;
+	hp->hm_revision = 0xa0;
 
 	if (qp != NULL)
 		hp->happy_flags |= HFLAG_QUATTRO;
 
+	hp->irq = op->archdata.irqs[0];
+
 	/* Get the supported DVMA burst sizes from our Happy SBUS. */
 	hp->happy_bursts = of_getintprop_default(sbus_dp,
 						 "burst-sizes", 0x00);
 
-	hp->happy_block = dma_alloc_coherent(hp->dma_dev,
-					     PAGE_SIZE,
-					     &hp->hblock_dvma,
-					     GFP_ATOMIC);
-	err = -ENOMEM;
-	if (!hp->happy_block)
-		goto err_out_iounmap;
-
-	/* Force check of the link first time we are brought up. */
-	hp->linkcheck = 0;
-
-	/* Force timer state to 'asleep' with count of zero. */
-	hp->timer_state = asleep;
-	hp->timer_ticks = 0;
-
-	timer_setup(&hp->happy_timer, happy_meal_timer, 0);
-
-	hp->dev = dev;
-	dev->netdev_ops = &hme_netdev_ops;
-	dev->watchdog_timeo = 5*HZ;
-	dev->ethtool_ops = &hme_ethtool_ops;
-
-	/* Happy Meal can do it all... */
-	dev->hw_features = NETIF_F_SG | NETIF_F_HW_CSUM;
-	dev->features |= dev->hw_features | NETIF_F_RXCSUM;
-
-	hp->irq = op->archdata.irqs[0];
-
-#if defined(CONFIG_SBUS) && defined(CONFIG_PCI)
+#ifdef CONFIG_PCI
 	/* Hook up SBUS register/descriptor accessors. */
 	hp->read_desc32 = sbus_hme_read_desc32;
 	hp->write_txd = sbus_hme_write_txd;
@@ -2681,18 +2584,9 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe)
 	hp->write32 = sbus_hme_write32;
 #endif
 
-	/* Grrr, Happy Meal comes up by default not advertising
-	 * full duplex 100baseT capabilities, fix this.
-	 */
-	spin_lock_irq(&hp->happy_lock);
-	happy_meal_set_initial_advertisement(hp);
-	spin_unlock_irq(&hp->happy_lock);
-
-	err = register_netdev(hp->dev);
-	if (err) {
-		dev_err(&op->dev, "Cannot register net device, aborting.\n");
-		goto err_out_free_coherent;
-	}
+	err = happy_meal_common_probe(hp, dp);
+	if (err)
+		goto err_out_clear_quattro;
 
 	platform_set_drvdata(op, hp);
 
@@ -2706,135 +2600,26 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe)
 
 	return 0;
 
-err_out_free_coherent:
-	dma_free_coherent(hp->dma_dev,
-			  PAGE_SIZE,
-			  hp->happy_block,
-			  hp->hblock_dvma);
-
-err_out_iounmap:
-	if (hp->gregs)
-		of_iounmap(&op->resource[0], hp->gregs, GREG_REG_SIZE);
-	if (hp->etxregs)
-		of_iounmap(&op->resource[1], hp->etxregs, ETX_REG_SIZE);
-	if (hp->erxregs)
-		of_iounmap(&op->resource[2], hp->erxregs, ERX_REG_SIZE);
-	if (hp->bigmacregs)
-		of_iounmap(&op->resource[3], hp->bigmacregs, BMAC_REG_SIZE);
-	if (hp->tcvregs)
-		of_iounmap(&op->resource[4], hp->tcvregs, TCVR_REG_SIZE);
-
+err_out_clear_quattro:
 	if (qp)
 		qp->happy_meals[qfe_slot] = NULL;
-
-err_out_free_netdev:
-	free_netdev(dev);
-
-err_out:
 	return err;
 }
 #endif
 
 #ifdef CONFIG_PCI
-#ifndef CONFIG_SPARC
-static int is_quattro_p(struct pci_dev *pdev)
-{
-	struct pci_dev *busdev = pdev->bus->self;
-	struct pci_dev *this_pdev;
-	int n_hmes;
-
-	if (busdev == NULL ||
-	    busdev->vendor != PCI_VENDOR_ID_DEC ||
-	    busdev->device != PCI_DEVICE_ID_DEC_21153)
-		return 0;
-
-	n_hmes = 0;
-	list_for_each_entry(this_pdev, &pdev->bus->devices, bus_list) {
-		if (this_pdev->vendor == PCI_VENDOR_ID_SUN &&
-		    this_pdev->device == PCI_DEVICE_ID_SUN_HAPPYMEAL)
-			n_hmes++;
-	}
-
-	if (n_hmes != 4)
-		return 0;
-
-	return 1;
-}
-
-/* Fetch MAC address from vital product data of PCI ROM. */
-static int find_eth_addr_in_vpd(void __iomem *rom_base, int len, int index, unsigned char *dev_addr)
-{
-	int this_offset;
-
-	for (this_offset = 0x20; this_offset < len; this_offset++) {
-		void __iomem *p = rom_base + this_offset;
-
-		if (readb(p + 0) != 0x90 ||
-		    readb(p + 1) != 0x00 ||
-		    readb(p + 2) != 0x09 ||
-		    readb(p + 3) != 0x4e ||
-		    readb(p + 4) != 0x41 ||
-		    readb(p + 5) != 0x06)
-			continue;
-
-		this_offset += 6;
-		p += 6;
-
-		if (index == 0) {
-			int i;
-
-			for (i = 0; i < 6; i++)
-				dev_addr[i] = readb(p + i);
-			return 1;
-		}
-		index--;
-	}
-	return 0;
-}
-
-static void get_hme_mac_nonsparc(struct pci_dev *pdev, unsigned char *dev_addr)
-{
-	size_t size;
-	void __iomem *p = pci_map_rom(pdev, &size);
-
-	if (p) {
-		int index = 0;
-		int found;
-
-		if (is_quattro_p(pdev))
-			index = PCI_SLOT(pdev->devfn);
-
-		found = readb(p) == 0x55 &&
-			readb(p + 1) == 0xaa &&
-			find_eth_addr_in_vpd(p, (64 * 1024), index, dev_addr);
-		pci_unmap_rom(pdev, p);
-		if (found)
-			return;
-	}
-
-	/* Sun MAC prefix then 3 random bytes. */
-	dev_addr[0] = 0x08;
-	dev_addr[1] = 0x00;
-	dev_addr[2] = 0x20;
-	get_random_bytes(&dev_addr[3], 3);
-}
-#endif /* !(CONFIG_SPARC) */
-
 static int happy_meal_pci_probe(struct pci_dev *pdev,
 				const struct pci_device_id *ent)
 {
+	struct device_node *dp = NULL;
 	struct quattro *qp = NULL;
-#ifdef CONFIG_SPARC
-	struct device_node *dp;
-#endif
 	struct happy_meal *hp;
 	struct net_device *dev;
 	void __iomem *hpreg_base;
 	struct resource *hpreg_res;
-	int i, qfe_slot = -1;
 	char prom_name[64];
-	u8 addr[ETH_ALEN];
-	int err;
+	int qfe_slot = -1;
+	int err = -ENODEV;
 
 	/* Now make sure pci_dev cookie is there. */
 #ifdef CONFIG_SPARC
@@ -2849,33 +2634,29 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 
 	err = pcim_enable_device(pdev);
 	if (err)
-		goto err_out;
+		return err;
 	pci_set_master(pdev);
 
 	if (!strcmp(prom_name, "SUNW,qfe") || !strcmp(prom_name, "qfe")) {
 		qp = quattro_pci_find(pdev);
-		if (IS_ERR(qp)) {
-			err = PTR_ERR(qp);
-			goto err_out;
-		}
+		if (IS_ERR(qp))
+			return PTR_ERR(qp);
 
 		for (qfe_slot = 0; qfe_slot < 4; qfe_slot++)
 			if (!qp->happy_meals[qfe_slot])
 				break;
 
 		if (qfe_slot == 4)
-			goto err_out;
+			return -ENODEV;
 	}
 
 	dev = devm_alloc_etherdev(&pdev->dev, sizeof(struct happy_meal));
-	if (!dev) {
-		err = -ENOMEM;
-		goto err_out;
-	}
+	if (!dev)
+		return -ENOMEM;
 	SET_NETDEV_DEV(dev, &pdev->dev);
 
 	hp = netdev_priv(dev);
-
+	hp->dev = dev;
 	hp->happy_dev = pdev;
 	hp->dma_dev = &pdev->dev;
 
@@ -2911,35 +2692,7 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 		goto err_out_clear_quattro;
 	}
 
-	for (i = 0; i < 6; i++) {
-		if (macaddr[i] != 0)
-			break;
-	}
-	if (i < 6) { /* a mac address was given */
-		for (i = 0; i < 6; i++)
-			addr[i] = macaddr[i];
-		eth_hw_addr_set(dev, addr);
-		macaddr[5]++;
-	} else {
-#ifdef CONFIG_SPARC
-		const unsigned char *addr;
-		int len;
-
-		if (qfe_slot != -1 &&
-		    (addr = of_get_property(dp, "local-mac-address", &len))
-			!= NULL &&
-		    len == 6) {
-			eth_hw_addr_set(dev, addr);
-		} else {
-			eth_hw_addr_set(dev, idprom->id_ethaddr);
-		}
-#else
-		u8 addr[ETH_ALEN];
-
-		get_hme_mac_nonsparc(pdev, addr);
-		eth_hw_addr_set(dev, addr);
-#endif
-	}
+	happy_meal_addr_init(hp, dp, qfe_slot);
 
 	/* Layout registers. */
 	hp->gregs      = (hpreg_base + 0x0000UL);
@@ -2948,20 +2701,10 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 	hp->bigmacregs = (hpreg_base + 0x6000UL);
 	hp->tcvregs    = (hpreg_base + 0x7000UL);
 
-#ifdef CONFIG_SPARC
-	hp->hm_revision = of_getintprop_default(dp, "hm-rev", 0xff);
-	if (hp->hm_revision == 0xff)
+	if (IS_ENABLED(CONFIG_SPARC))
 		hp->hm_revision = 0xc0 | (pdev->revision & 0x0f);
-#else
-	/* works with this on non-sparc hosts */
-	hp->hm_revision = 0x20;
-#endif
-
-	/* Now enable the feature flags we can. */
-	if (hp->hm_revision == 0x20 || hp->hm_revision == 0x21)
-		hp->happy_flags = HFLAG_20_21;
-	else if (hp->hm_revision != 0xa0 && hp->hm_revision != 0xc0)
-		hp->happy_flags = HFLAG_NOT_A0;
+	else
+		hp->hm_revision = 0x20;
 
 	if (qp != NULL)
 		hp->happy_flags |= HFLAG_QUATTRO;
@@ -2973,31 +2716,9 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 	/* Assume PCI happy meals can handle all burst sizes. */
 	hp->happy_bursts = DMA_BURSTBITS;
 #endif
-
-	hp->happy_block = dmam_alloc_coherent(&pdev->dev, PAGE_SIZE,
-					      &hp->hblock_dvma, GFP_KERNEL);
-	if (!hp->happy_block) {
-		err = -ENOMEM;
-		goto err_out_clear_quattro;
-	}
-
-	hp->linkcheck = 0;
-	hp->timer_state = asleep;
-	hp->timer_ticks = 0;
-
-	timer_setup(&hp->happy_timer, happy_meal_timer, 0);
-
 	hp->irq = pdev->irq;
-	hp->dev = dev;
-	dev->netdev_ops = &hme_netdev_ops;
-	dev->watchdog_timeo = 5*HZ;
-	dev->ethtool_ops = &hme_ethtool_ops;
 
-	/* Happy Meal can do it all... */
-	dev->hw_features = NETIF_F_SG | NETIF_F_HW_CSUM;
-	dev->features |= dev->hw_features | NETIF_F_RXCSUM;
-
-#if defined(CONFIG_SBUS) && defined(CONFIG_PCI)
+#ifdef CONFIG_SBUS
 	/* Hook up PCI register/descriptor accessors. */
 	hp->read_desc32 = pci_hme_read_desc32;
 	hp->write_txd = pci_hme_write_txd;
@@ -3006,18 +2727,9 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 	hp->write32 = pci_hme_write32;
 #endif
 
-	/* Grrr, Happy Meal comes up by default not advertising
-	 * full duplex 100baseT capabilities, fix this.
-	 */
-	spin_lock_irq(&hp->happy_lock);
-	happy_meal_set_initial_advertisement(hp);
-	spin_unlock_irq(&hp->happy_lock);
-
-	err = devm_register_netdev(&pdev->dev, dev);
-	if (err) {
-		dev_err(&pdev->dev, "Cannot register net device, aborting.\n");
+	err = happy_meal_common_probe(hp, dp);
+	if (err)
 		goto err_out_clear_quattro;
-	}
 
 	pci_set_drvdata(pdev, hp);
 
@@ -3048,8 +2760,6 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 err_out_clear_quattro:
 	if (qp != NULL)
 		qp->happy_meals[qfe_slot] = NULL;
-
-err_out:
 	return err;
 }
 
@@ -3107,30 +2817,6 @@ static int hme_sbus_probe(struct platform_device *op)
 	return happy_meal_sbus_probe_one(op, is_qfe);
 }
 
-static int hme_sbus_remove(struct platform_device *op)
-{
-	struct happy_meal *hp = platform_get_drvdata(op);
-	struct net_device *net_dev = hp->dev;
-
-	unregister_netdev(net_dev);
-
-	/* XXX qfe parent interrupt... */
-
-	of_iounmap(&op->resource[0], hp->gregs, GREG_REG_SIZE);
-	of_iounmap(&op->resource[1], hp->etxregs, ETX_REG_SIZE);
-	of_iounmap(&op->resource[2], hp->erxregs, ERX_REG_SIZE);
-	of_iounmap(&op->resource[3], hp->bigmacregs, BMAC_REG_SIZE);
-	of_iounmap(&op->resource[4], hp->tcvregs, TCVR_REG_SIZE);
-	dma_free_coherent(hp->dma_dev,
-			  PAGE_SIZE,
-			  hp->happy_block,
-			  hp->hblock_dvma);
-
-	free_netdev(net_dev);
-
-	return 0;
-}
-
 static const struct of_device_id hme_sbus_match[] = {
 	{
 		.name = "SUNW,hme",
@@ -3154,24 +2840,16 @@ static struct platform_driver hme_sbus_driver = {
 		.of_match_table = hme_sbus_match,
 	},
 	.probe		= hme_sbus_probe,
-	.remove		= hme_sbus_remove,
 };
 
 static int __init happy_meal_sbus_init(void)
 {
-	int err;
-
-	err = platform_driver_register(&hme_sbus_driver);
-	if (!err)
-		err = quattro_sbus_register_irqs();
-
-	return err;
+	return platform_driver_register(&hme_sbus_driver);
 }
 
 static void happy_meal_sbus_exit(void)
 {
 	platform_driver_unregister(&hme_sbus_driver);
-	quattro_sbus_free_irqs();
 
 	while (qfe_sbus_list) {
 		struct quattro *qfe = qfe_sbus_list;
diff --git a/drivers/net/ethernet/sun/sunhme.h b/drivers/net/ethernet/sun/sunhme.h
index 9118c60c9426..258b4c7fe962 100644
--- a/drivers/net/ethernet/sun/sunhme.h
+++ b/drivers/net/ethernet/sun/sunhme.h
@@ -462,22 +462,20 @@ struct happy_meal {
 };
 
 /* Here are the happy flags. */
-#define HFLAG_POLL                0x00000001      /* We are doing MIF polling          */
 #define HFLAG_FENABLE             0x00000002      /* The MII frame is enabled          */
 #define HFLAG_LANCE               0x00000004      /* We are using lance-mode           */
 #define HFLAG_RXENABLE            0x00000008      /* Receiver is enabled               */
 #define HFLAG_AUTO                0x00000010      /* Using auto-negotiation, 0 = force */
 #define HFLAG_FULL                0x00000020      /* Full duplex enable                */
 #define HFLAG_MACFULL             0x00000040      /* Using full duplex in the MAC      */
-#define HFLAG_POLLENABLE          0x00000080      /* Actually try MIF polling          */
 #define HFLAG_RXCV                0x00000100      /* XXX RXCV ENABLE                   */
 #define HFLAG_INIT                0x00000200      /* Init called at least once         */
 #define HFLAG_LINKUP              0x00000400      /* 1 = Link is up                    */
 #define HFLAG_PCI                 0x00000800      /* PCI based Happy Meal              */
 #define HFLAG_QUATTRO		  0x00001000      /* On QFE/Quattro card	       */
 
-#define HFLAG_20_21  (HFLAG_POLLENABLE | HFLAG_FENABLE)
-#define HFLAG_NOT_A0 (HFLAG_POLLENABLE | HFLAG_FENABLE | HFLAG_LANCE | HFLAG_RXCV)
+#define HFLAG_20_21  HFLAG_FENABLE
+#define HFLAG_NOT_A0 (HFLAG_FENABLE | HFLAG_LANCE | HFLAG_RXCV)
 
 /* Support for QFE/Quattro cards. */
 struct quattro {
diff --git a/drivers/net/ethernet/sunplus/spl2sw_phy.c b/drivers/net/ethernet/sunplus/spl2sw_phy.c
index 404f508a54d4..6f899e48f51d 100644
--- a/drivers/net/ethernet/sunplus/spl2sw_phy.c
+++ b/drivers/net/ethernet/sunplus/spl2sw_phy.c
@@ -84,9 +84,7 @@ void spl2sw_phy_remove(struct spl2sw_common *comm)
 	for (i = 0; i < MAX_NETDEV_NUM; i++)
 		if (comm->ndev[i]) {
 			ndev = comm->ndev[i];
-			if (ndev) {
+			if (ndev)
 				phy_disconnect(ndev->phydev);
-				ndev->phydev = NULL;
-			}
 		}
 }
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
index bcea87b7151c..11cbcd9e2c72 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
@@ -76,6 +76,7 @@
 #define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2       0x31C
 
 #define AM65_CPSW_SGMII_CONTROL_REG		0x010
+#define AM65_CPSW_SGMII_MR_ADV_ABILITY_REG	0x018
 #define AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE	BIT(0)
 
 #define AM65_CPSW_CTL_VLAN_AWARE		BIT(1)
@@ -85,6 +86,7 @@
 
 /* AM65_CPSW_P0_REG_CTL */
 #define AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN	BIT(0)
+#define AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN	BIT(16)
 
 /* AM65_CPSW_PORT_REG_PRI_CTL */
 #define AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN	BIT(8)
@@ -384,8 +386,8 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
 	/* set base flow_id */
 	writel(common->rx_flow_id_base,
 	       host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET);
-	/* en tx crc offload */
-	writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN, host_p->port_base + AM65_CPSW_P0_REG_CTL);
+	writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN | AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN,
+	       host_p->port_base + AM65_CPSW_P0_REG_CTL);
 
 	am65_cpsw_nuss_set_p0_ptype(common);
 
@@ -427,6 +429,8 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
 	else
 		am65_cpsw_init_host_port_switch(common);
 
+	am65_cpsw_qos_tx_p0_rate_init(common);
+
 	for (i = 0; i < common->rx_chns.descs_num; i++) {
 		skb = __netdev_alloc_skb_ip_align(NULL,
 						  AM65_CPSW_MAX_PACKET_SIZE,
@@ -598,8 +602,12 @@ static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
 		goto runtime_put;
 	}
 
-	for (i = 0; i < common->tx_ch_num; i++)
-		netdev_tx_reset_queue(netdev_get_tx_queue(ndev, i));
+	for (i = 0; i < common->tx_ch_num; i++) {
+		struct netdev_queue *txq = netdev_get_tx_queue(ndev, i);
+
+		netdev_tx_reset_queue(txq);
+		txq->tx_maxrate =  common->tx_chns[i].rate_mbps;
+	}
 
 	ret = am65_cpsw_nuss_common_open(common);
 	if (ret)
@@ -1424,6 +1432,7 @@ static const struct net_device_ops am65_cpsw_nuss_netdev_ops = {
 	.ndo_vlan_rx_kill_vid	= am65_cpsw_nuss_ndo_slave_kill_vid,
 	.ndo_eth_ioctl		= am65_cpsw_nuss_ndo_slave_ioctl,
 	.ndo_setup_tc           = am65_cpsw_qos_ndo_setup_tc,
+	.ndo_set_tx_maxrate	= am65_cpsw_qos_ndo_tx_p0_set_maxrate,
 };
 
 static void am65_cpsw_disable_phy(struct phy *phy)
@@ -1466,15 +1475,13 @@ static void am65_cpsw_disable_serdes_phy(struct am65_cpsw_common *common)
 static int am65_cpsw_init_serdes_phy(struct device *dev, struct device_node *port_np,
 				     struct am65_cpsw_port *port)
 {
-	const char *name = "serdes-phy";
+	const char *name = "serdes";
 	struct phy *phy;
 	int ret;
 
-	phy = devm_of_phy_get(dev, port_np, name);
-	if (PTR_ERR(phy) == -ENODEV)
-		return 0;
-	if (IS_ERR(phy))
-		return PTR_ERR(phy);
+	phy = devm_of_phy_optional_get(dev, port_np, name);
+	if (IS_ERR_OR_NULL(phy))
+		return PTR_ERR_OR_ZERO(phy);
 
 	/* Serdes PHY exists. Store it. */
 	port->slave.serdes_phy = phy;
@@ -1498,9 +1505,26 @@ static void am65_cpsw_nuss_mac_config(struct phylink_config *config, unsigned in
 	struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
 	struct am65_cpsw_common *common = port->common;
 
-	if (common->pdata.extra_modes & BIT(state->interface))
+	if (common->pdata.extra_modes & BIT(state->interface)) {
+		if (state->interface == PHY_INTERFACE_MODE_SGMII) {
+			writel(ADVERTISE_SGMII,
+			       port->sgmii_base + AM65_CPSW_SGMII_MR_ADV_ABILITY_REG);
+			cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN);
+		} else {
+			cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN);
+		}
+
+		if (state->interface == PHY_INTERFACE_MODE_USXGMII) {
+			cpsw_sl_ctl_set(port->slave.mac_sl,
+					CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN);
+		} else {
+			cpsw_sl_ctl_clr(port->slave.mac_sl,
+					CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN);
+		}
+
 		writel(AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE,
 		       port->sgmii_base + AM65_CPSW_SGMII_CONTROL_REG);
+	}
 }
 
 static void am65_cpsw_nuss_mac_link_down(struct phylink_config *config, unsigned int mode,
@@ -1511,6 +1535,7 @@ static void am65_cpsw_nuss_mac_link_down(struct phylink_config *config, unsigned
 	struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
 	struct am65_cpsw_common *common = port->common;
 	struct net_device *ndev = port->ndev;
+	u32 mac_control;
 	int tmo;
 
 	/* disable forwarding */
@@ -1522,7 +1547,14 @@ static void am65_cpsw_nuss_mac_link_down(struct phylink_config *config, unsigned
 	dev_dbg(common->dev, "down msc_sl %08x tmo %d\n",
 		cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS), tmo);
 
-	cpsw_sl_ctl_reset(port->slave.mac_sl);
+	/* All the bits that am65_cpsw_nuss_mac_link_up() can possibly set */
+	mac_control = CPSW_SL_CTL_GMII_EN | CPSW_SL_CTL_GIG | CPSW_SL_CTL_IFCTL_A |
+		      CPSW_SL_CTL_FULLDUPLEX | CPSW_SL_CTL_RX_FLOW_EN | CPSW_SL_CTL_TX_FLOW_EN;
+	/* If interface mode is RGMII, CPSW_SL_CTL_EXT_EN might have been set for 10 Mbps */
+	if (phy_interface_mode_is_rgmii(interface))
+		mac_control |= CPSW_SL_CTL_EXT_EN;
+	/* Only clear those bits that can be set by am65_cpsw_nuss_mac_link_up() */
+	cpsw_sl_ctl_clr(port->slave.mac_sl, mac_control);
 
 	am65_cpsw_qos_link_down(ndev);
 	netif_tx_stop_all_queues(ndev);
@@ -1539,8 +1571,12 @@ static void am65_cpsw_nuss_mac_link_up(struct phylink_config *config, struct phy
 	u32 mac_control = CPSW_SL_CTL_GMII_EN;
 	struct net_device *ndev = port->ndev;
 
+	/* Bring the port out of idle state */
+	cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
+
 	if (speed == SPEED_1000)
 		mac_control |= CPSW_SL_CTL_GIG;
+	/* TODO: Verify whether in-band is necessary for 10 Mbps RGMII */
 	if (speed == SPEED_10 && phy_interface_mode_is_rgmii(interface))
 		/* Can be used with in band mode only */
 		mac_control |= CPSW_SL_CTL_EXT_EN;
@@ -1610,6 +1646,7 @@ void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common)
 
 	devm_remove_action(dev, am65_cpsw_nuss_free_tx_chns, common);
 
+	common->tx_ch_rate_msk = 0;
 	for (i = 0; i < common->tx_ch_num; i++) {
 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
 
@@ -2142,18 +2179,36 @@ am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx)
 	/* Configuring Phylink */
 	port->slave.phylink_config.dev = &port->ndev->dev;
 	port->slave.phylink_config.type = PHYLINK_NETDEV;
-	port->slave.phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000FD;
+	port->slave.phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 |
+						      MAC_1000FD | MAC_5000FD;
 	port->slave.phylink_config.mac_managed_pm = true; /* MAC does PM */
 
-	if (phy_interface_mode_is_rgmii(port->slave.phy_if)) {
+	switch (port->slave.phy_if) {
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
 		phy_interface_set_rgmii(port->slave.phylink_config.supported_interfaces);
-	} else if (port->slave.phy_if == PHY_INTERFACE_MODE_RMII) {
+		break;
+
+	case PHY_INTERFACE_MODE_RMII:
 		__set_bit(PHY_INTERFACE_MODE_RMII,
 			  port->slave.phylink_config.supported_interfaces);
-	} else if (common->pdata.extra_modes & BIT(port->slave.phy_if)) {
-		__set_bit(PHY_INTERFACE_MODE_QSGMII,
-			  port->slave.phylink_config.supported_interfaces);
-	} else {
+		break;
+
+	case PHY_INTERFACE_MODE_QSGMII:
+	case PHY_INTERFACE_MODE_SGMII:
+	case PHY_INTERFACE_MODE_USXGMII:
+		if (common->pdata.extra_modes & BIT(port->slave.phy_if)) {
+			__set_bit(port->slave.phy_if,
+				  port->slave.phylink_config.supported_interfaces);
+		} else {
+			dev_err(dev, "selected phy-mode is not supported\n");
+			return -EOPNOTSUPP;
+		}
+		break;
+
+	default:
 		dev_err(dev, "selected phy-mode is not supported\n");
 		return -EOPNOTSUPP;
 	}
@@ -2755,14 +2810,21 @@ static const struct am65_cpsw_pdata j7200_cpswxg_pdata = {
 	.quirks = 0,
 	.ale_dev_id = "am64-cpswxg",
 	.fdqring_mode = K3_RINGACC_RING_MODE_RING,
-	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII),
+	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII),
 };
 
 static const struct am65_cpsw_pdata j721e_cpswxg_pdata = {
 	.quirks = 0,
 	.ale_dev_id = "am64-cpswxg",
 	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
-	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII),
+	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII),
+};
+
+static const struct am65_cpsw_pdata j784s4_cpswxg_pdata = {
+	.quirks = 0,
+	.ale_dev_id = "am64-cpswxg",
+	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
+	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_USXGMII),
 };
 
 static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
@@ -2771,6 +2833,7 @@ static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
 	{ .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata},
 	{ .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata},
 	{ .compatible = "ti,j721e-cpswxg-nuss", .data = &j721e_cpswxg_pdata},
+	{ .compatible = "ti,j784s4-cpswxg-nuss", .data = &j784s4_cpswxg_pdata},
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.h b/drivers/net/ethernet/ti/am65-cpsw-nuss.h
index cad04662739c..bf40c88fbd9b 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.h
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.h
@@ -79,6 +79,7 @@ struct am65_cpsw_tx_chn {
 	u32 id;
 	u32 descs_num;
 	char tx_chn_name[128];
+	u32 rate_mbps;
 };
 
 struct am65_cpsw_rx_chn {
@@ -126,6 +127,7 @@ struct am65_cpsw_common {
 	int			usage_count; /* number of opened ports */
 	struct cpsw_ale		*ale;
 	int			tx_ch_num;
+	u32			tx_ch_rate_msk;
 	u32			rx_flow_id_base;
 
 	struct am65_cpsw_tx_chn	tx_chns[AM65_CPSW_MAX_TX_QUEUES];
diff --git a/drivers/net/ethernet/ti/am65-cpsw-qos.c b/drivers/net/ethernet/ti/am65-cpsw-qos.c
index 8dc2c3085dcf..3a908db6e5b2 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-qos.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-qos.c
@@ -19,6 +19,7 @@
 #define AM65_CPSW_PN_REG_CTL			0x004
 #define AM65_CPSW_PN_REG_FIFO_STATUS		0x050
 #define AM65_CPSW_PN_REG_EST_CTL		0x060
+#define AM65_CPSW_PN_REG_PRI_CIR(pri)		(0x140 + 4 * (pri))
 
 /* AM65_CPSW_REG_CTL register fields */
 #define AM65_CPSW_CTL_EST_EN			BIT(18)
@@ -819,3 +820,115 @@ void am65_cpsw_qos_link_down(struct net_device *ndev)
 
 	port->qos.link_speed = SPEED_UNKNOWN;
 }
+
+static u32
+am65_cpsw_qos_tx_rate_calc(u32 rate_mbps, unsigned long bus_freq)
+{
+	u32 ir;
+
+	bus_freq /= 1000000;
+	ir = DIV_ROUND_UP(((u64)rate_mbps * 32768),  bus_freq);
+	return ir;
+}
+
+static void
+am65_cpsw_qos_tx_p0_rate_apply(struct am65_cpsw_common *common,
+			       int tx_ch, u32 rate_mbps)
+{
+	struct am65_cpsw_host *host = am65_common_get_host(common);
+	u32 ch_cir;
+	int i;
+
+	ch_cir = am65_cpsw_qos_tx_rate_calc(rate_mbps, common->bus_freq);
+	writel(ch_cir, host->port_base + AM65_CPSW_PN_REG_PRI_CIR(tx_ch));
+
+	/* update rates for every port tx queues */
+	for (i = 0; i < common->port_num; i++) {
+		struct net_device *ndev = common->ports[i].ndev;
+
+		if (!ndev)
+			continue;
+		netdev_get_tx_queue(ndev, tx_ch)->tx_maxrate = rate_mbps;
+	}
+}
+
+int am65_cpsw_qos_ndo_tx_p0_set_maxrate(struct net_device *ndev,
+					int queue, u32 rate_mbps)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct am65_cpsw_common *common = port->common;
+	struct am65_cpsw_tx_chn *tx_chn;
+	u32 ch_rate, tx_ch_rate_msk_new;
+	u32 ch_msk = 0;
+	int ret;
+
+	dev_dbg(common->dev, "apply TX%d rate limiting %uMbps tx_rate_msk%x\n",
+		queue, rate_mbps, common->tx_ch_rate_msk);
+
+	if (common->pf_p0_rx_ptype_rrobin) {
+		dev_err(common->dev, "TX Rate Limiting failed - rrobin mode\n");
+		return -EINVAL;
+	}
+
+	ch_rate = netdev_get_tx_queue(ndev, queue)->tx_maxrate;
+	if (ch_rate == rate_mbps)
+		return 0;
+
+	ret = pm_runtime_get_sync(common->dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(common->dev);
+		return ret;
+	}
+	ret = 0;
+
+	tx_ch_rate_msk_new = common->tx_ch_rate_msk;
+	if (rate_mbps && !(tx_ch_rate_msk_new & BIT(queue))) {
+		tx_ch_rate_msk_new |= BIT(queue);
+		ch_msk = GENMASK(common->tx_ch_num - 1, queue);
+		ch_msk = tx_ch_rate_msk_new ^ ch_msk;
+	} else if (!rate_mbps) {
+		tx_ch_rate_msk_new &= ~BIT(queue);
+		ch_msk = queue ? GENMASK(queue - 1, 0) : 0;
+		ch_msk = tx_ch_rate_msk_new & ch_msk;
+	}
+
+	if (ch_msk) {
+		dev_err(common->dev, "TX rate limiting has to be enabled sequentially hi->lo tx_rate_msk:%x tx_rate_msk_new:%x\n",
+			common->tx_ch_rate_msk, tx_ch_rate_msk_new);
+		ret = -EINVAL;
+		goto exit_put;
+	}
+
+	tx_chn = &common->tx_chns[queue];
+	tx_chn->rate_mbps = rate_mbps;
+	common->tx_ch_rate_msk = tx_ch_rate_msk_new;
+
+	if (!common->usage_count)
+		/* will be applied on next netif up */
+		goto exit_put;
+
+	am65_cpsw_qos_tx_p0_rate_apply(common, queue, rate_mbps);
+
+exit_put:
+	pm_runtime_put(common->dev);
+	return ret;
+}
+
+void am65_cpsw_qos_tx_p0_rate_init(struct am65_cpsw_common *common)
+{
+	struct am65_cpsw_host *host = am65_common_get_host(common);
+	int tx_ch;
+
+	for (tx_ch = 0; tx_ch < common->tx_ch_num; tx_ch++) {
+		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[tx_ch];
+		u32 ch_cir;
+
+		if (!tx_chn->rate_mbps)
+			continue;
+
+		ch_cir = am65_cpsw_qos_tx_rate_calc(tx_chn->rate_mbps,
+						    common->bus_freq);
+		writel(ch_cir,
+		       host->port_base + AM65_CPSW_PN_REG_PRI_CIR(tx_ch));
+	}
+}
diff --git a/drivers/net/ethernet/ti/am65-cpsw-qos.h b/drivers/net/ethernet/ti/am65-cpsw-qos.h
index fb223b43b196..0cc2a3b3d7f9 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-qos.h
+++ b/drivers/net/ethernet/ti/am65-cpsw-qos.h
@@ -8,6 +8,8 @@
 #include <linux/netdevice.h>
 #include <net/pkt_sched.h>
 
+struct am65_cpsw_common;
+
 struct am65_cpsw_est {
 	int buf;
 	/* has to be the last one */
@@ -33,5 +35,7 @@ int am65_cpsw_qos_ndo_setup_tc(struct net_device *ndev, enum tc_setup_type type,
 			       void *type_data);
 void am65_cpsw_qos_link_up(struct net_device *ndev, int link_speed);
 void am65_cpsw_qos_link_down(struct net_device *ndev);
+int am65_cpsw_qos_ndo_tx_p0_set_maxrate(struct net_device *ndev, int queue, u32 rate_mbps);
+void am65_cpsw_qos_tx_p0_rate_init(struct am65_cpsw_common *common);
 
 #endif /* AM65_CPSW_QOS_H_ */
diff --git a/drivers/net/ethernet/ti/am65-cpts.c b/drivers/net/ethernet/ti/am65-cpts.c
index 8caf85acbb6a..c66618d91c28 100644
--- a/drivers/net/ethernet/ti/am65-cpts.c
+++ b/drivers/net/ethernet/ti/am65-cpts.c
@@ -175,6 +175,7 @@ struct am65_cpts {
 	u64 timestamp;
 	u32 genf_enable;
 	u32 hw_ts_enable;
+	u32 estf_enable;
 	struct sk_buff_head txq;
 	bool pps_enabled;
 	bool pps_present;
@@ -405,13 +406,13 @@ static irqreturn_t am65_cpts_interrupt(int irq, void *dev_id)
 static int am65_cpts_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
 {
 	struct am65_cpts *cpts = container_of(ptp, struct am65_cpts, ptp_info);
-	u32 pps_ctrl_val = 0, pps_ppm_hi = 0, pps_ppm_low = 0;
+	u32 estf_ctrl_val = 0, estf_ppm_hi = 0, estf_ppm_low = 0;
 	s32 ppb = scaled_ppm_to_ppb(scaled_ppm);
 	int pps_index = cpts->pps_genf_idx;
 	u64 adj_period, pps_adj_period;
 	u32 ctrl_val, ppm_hi, ppm_low;
 	unsigned long flags;
-	int neg_adj = 0;
+	int neg_adj = 0, i;
 
 	if (ppb < 0) {
 		neg_adj = 1;
@@ -441,19 +442,19 @@ static int am65_cpts_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
 	ppm_low = lower_32_bits(adj_period);
 
 	if (cpts->pps_enabled) {
-		pps_ctrl_val = am65_cpts_read32(cpts, genf[pps_index].control);
+		estf_ctrl_val = am65_cpts_read32(cpts, genf[pps_index].control);
 		if (neg_adj)
-			pps_ctrl_val &= ~BIT(1);
+			estf_ctrl_val &= ~BIT(1);
 		else
-			pps_ctrl_val |= BIT(1);
+			estf_ctrl_val |= BIT(1);
 
 		/* GenF PPM will do correction using cpts refclk tick which is
 		 * (cpts->ts_add_val + 1) ns, so GenF length PPM adj period
 		 * need to be corrected.
 		 */
 		pps_adj_period = adj_period * (cpts->ts_add_val + 1);
-		pps_ppm_hi = upper_32_bits(pps_adj_period) & 0x3FF;
-		pps_ppm_low = lower_32_bits(pps_adj_period);
+		estf_ppm_hi = upper_32_bits(pps_adj_period) & 0x3FF;
+		estf_ppm_low = lower_32_bits(pps_adj_period);
 	}
 
 	spin_lock_irqsave(&cpts->lock, flags);
@@ -471,11 +472,18 @@ static int am65_cpts_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
 	am65_cpts_write32(cpts, ppm_low, ts_ppm_low);
 
 	if (cpts->pps_enabled) {
-		am65_cpts_write32(cpts, pps_ctrl_val, genf[pps_index].control);
-		am65_cpts_write32(cpts, pps_ppm_hi, genf[pps_index].ppm_hi);
-		am65_cpts_write32(cpts, pps_ppm_low, genf[pps_index].ppm_low);
+		am65_cpts_write32(cpts, estf_ctrl_val, genf[pps_index].control);
+		am65_cpts_write32(cpts, estf_ppm_hi, genf[pps_index].ppm_hi);
+		am65_cpts_write32(cpts, estf_ppm_low, genf[pps_index].ppm_low);
 	}
 
+	for (i = 0; i < AM65_CPTS_ESTF_MAX_NUM; i++) {
+		if (cpts->estf_enable & BIT(i)) {
+			am65_cpts_write32(cpts, estf_ctrl_val, estf[i].control);
+			am65_cpts_write32(cpts, estf_ppm_hi, estf[i].ppm_hi);
+			am65_cpts_write32(cpts, estf_ppm_low, estf[i].ppm_low);
+		}
+	}
 	/* All GenF/EstF can be updated here the same way */
 	spin_unlock_irqrestore(&cpts->lock, flags);
 
@@ -596,6 +604,11 @@ int am65_cpts_estf_enable(struct am65_cpts *cpts, int idx,
 	am65_cpts_write32(cpts, val, estf[idx].comp_lo);
 	val = lower_32_bits(cycles);
 	am65_cpts_write32(cpts, val, estf[idx].length);
+	am65_cpts_write32(cpts, 0, estf[idx].control);
+	am65_cpts_write32(cpts, 0, estf[idx].ppm_hi);
+	am65_cpts_write32(cpts, 0, estf[idx].ppm_low);
+
+	cpts->estf_enable |= BIT(idx);
 
 	dev_dbg(cpts->dev, "%s: ESTF:%u enabled\n", __func__, idx);
 
@@ -606,6 +619,7 @@ EXPORT_SYMBOL_GPL(am65_cpts_estf_enable);
 void am65_cpts_estf_disable(struct am65_cpts *cpts, int idx)
 {
 	am65_cpts_write32(cpts, 0, estf[idx].length);
+	cpts->estf_enable &= ~BIT(idx);
 
 	dev_dbg(cpts->dev, "%s: ESTF:%u disabled\n", __func__, idx);
 }
diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c
index 1bb596a9d8a2..d829113c16ee 100644
--- a/drivers/net/ethernet/ti/netcp_core.c
+++ b/drivers/net/ethernet/ti/netcp_core.c
@@ -2081,8 +2081,8 @@ static int netcp_create_interface(struct netcp_device *netcp_device,
 	netcp->tx_pool_region_id = temp[1];
 
 	if (netcp->tx_pool_size < MAX_SKB_FRAGS) {
-		dev_err(dev, "tx-pool size too small, must be at least %ld\n",
-			MAX_SKB_FRAGS);
+		dev_err(dev, "tx-pool size too small, must be at least %u\n",
+			(unsigned int)MAX_SKB_FRAGS);
 		ret = -ENODEV;
 		goto quit;
 	}
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c
index 7db57f934a91..ca409b4054d0 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c
+++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c
@@ -4,6 +4,7 @@
 #include <linux/etherdevice.h>
 #include <linux/netdevice.h>
 #include <linux/if_ether.h>
+#include <linux/if_vlan.h>
 #include <linux/iopoll.h>
 #include <linux/pci.h>
 
@@ -1261,7 +1262,7 @@ static void wx_set_rx_buffer_len(struct wx *wx)
 	struct net_device *netdev = wx->netdev;
 	u32 mhadd, max_frame;
 
-	max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
+	max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
 	/* adjust max frame to be at least the size of a standard frame */
 	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
 		max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN);
@@ -1271,6 +1272,24 @@ static void wx_set_rx_buffer_len(struct wx *wx)
 		wr32(wx, WX_PSR_MAX_SZ, max_frame);
 }
 
+/**
+ * wx_change_mtu - Change the Maximum Transfer Unit
+ * @netdev: network interface device structure
+ * @new_mtu: new value for maximum frame size
+ *
+ * Returns 0 on success, negative on failure
+ **/
+int wx_change_mtu(struct net_device *netdev, int new_mtu)
+{
+	struct wx *wx = netdev_priv(netdev);
+
+	netdev->mtu = new_mtu;
+	wx_set_rx_buffer_len(wx);
+
+	return 0;
+}
+EXPORT_SYMBOL(wx_change_mtu);
+
 /* Disable the specified rx queue */
 void wx_disable_rx_queue(struct wx *wx, struct wx_ring *ring)
 {
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.h b/drivers/net/ethernet/wangxun/libwx/wx_hw.h
index 44dfd6ea442a..c173c56f0ab5 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_hw.h
+++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.h
@@ -23,6 +23,7 @@ void wx_flush_sw_mac_table(struct wx *wx);
 int wx_set_mac(struct net_device *netdev, void *p);
 void wx_disable_rx(struct wx *wx);
 void wx_set_rx_mode(struct net_device *netdev);
+int wx_change_mtu(struct net_device *netdev, int new_mtu);
 void wx_disable_rx_queue(struct wx *wx, struct wx_ring *ring);
 void wx_configure(struct wx *wx);
 int wx_disable_pcie_master(struct wx *wx);
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c
index eb89a274083e..1e8d8b7b0c62 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c
+++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c
@@ -1798,10 +1798,13 @@ static int wx_setup_rx_resources(struct wx_ring *rx_ring)
 	ret = wx_alloc_page_pool(rx_ring);
 	if (ret < 0) {
 		dev_err(rx_ring->dev, "Page pool creation failed: %d\n", ret);
-		goto err;
+		goto err_desc;
 	}
 
 	return 0;
+
+err_desc:
+	dma_free_coherent(dev, rx_ring->size, rx_ring->desc, rx_ring->dma);
 err:
 	kvfree(rx_ring->rx_buffer_info);
 	rx_ring->rx_buffer_info = NULL;
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h
index 97e2c1e13b80..32f952d93009 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_type.h
+++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h
@@ -7,11 +7,6 @@
 #include <linux/bitfield.h>
 #include <linux/netdevice.h>
 
-/* Vendor ID */
-#ifndef PCI_VENDOR_ID_WANGXUN
-#define PCI_VENDOR_ID_WANGXUN                   0x8088
-#endif
-
 #define WX_NCSI_SUP                             0x8000
 #define WX_NCSI_MASK                            0x8000
 #define WX_WOL_SUP                              0x4000
@@ -300,6 +295,8 @@
 #define WX_MAX_RXD                   8192
 #define WX_MAX_TXD                   8192
 
+#define WX_MAX_JUMBO_FRAME_SIZE      9432 /* max payload 9414 */
+
 /* Supported Rx Buffer Sizes */
 #define WX_RXBUFFER_256      256    /* Used for skb receive header */
 #define WX_RXBUFFER_2K       2048
diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
index 17412e5282de..df6b870aa871 100644
--- a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
+++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
@@ -6,10 +6,10 @@
 #include <linux/pci.h>
 #include <linux/netdevice.h>
 #include <linux/string.h>
-#include <linux/aer.h>
 #include <linux/etherdevice.h>
 #include <net/ip.h>
 #include <linux/phy.h>
+#include <linux/if_vlan.h>
 
 #include "../libwx/wx_type.h"
 #include "../libwx/wx_hw.h"
@@ -470,6 +470,7 @@ static void ngbe_shutdown(struct pci_dev *pdev)
 static const struct net_device_ops ngbe_netdev_ops = {
 	.ndo_open               = ngbe_open,
 	.ndo_stop               = ngbe_close,
+	.ndo_change_mtu         = wx_change_mtu,
 	.ndo_start_xmit         = wx_xmit_frame,
 	.ndo_set_rx_mode        = wx_set_rx_mode,
 	.ndo_validate_addr      = eth_validate_addr,
@@ -520,7 +521,6 @@ static int ngbe_probe(struct pci_dev *pdev,
 		goto err_pci_disable_dev;
 	}
 
-	pci_enable_pcie_error_reporting(pdev);
 	pci_set_master(pdev);
 
 	netdev = devm_alloc_etherdev_mqs(&pdev->dev,
@@ -562,7 +562,8 @@ static int ngbe_probe(struct pci_dev *pdev,
 	netdev->priv_flags |= IFF_SUPP_NOFCS;
 
 	netdev->min_mtu = ETH_MIN_MTU;
-	netdev->max_mtu = NGBE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
+	netdev->max_mtu = WX_MAX_JUMBO_FRAME_SIZE -
+			  (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
 
 	wx->bd_number = func_nums;
 	/* setup the private structure */
@@ -669,7 +670,6 @@ err_clear_interrupt_scheme:
 err_free_mac_table:
 	kfree(wx->mac_table);
 err_pci_release_regions:
-	pci_disable_pcie_error_reporting(pdev);
 	pci_release_selected_regions(pdev,
 				     pci_select_bars(pdev, IORESOURCE_MEM));
 err_pci_disable_dev:
@@ -698,7 +698,6 @@ static void ngbe_remove(struct pci_dev *pdev)
 
 	kfree(wx->mac_table);
 	wx_clear_interrupt_scheme(wx);
-	pci_disable_pcie_error_reporting(pdev);
 
 	pci_disable_device(pdev);
 }
diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h b/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h
index a2351349785e..373d5af628cd 100644
--- a/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h
+++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h
@@ -137,7 +137,6 @@ enum NGBE_MSCA_CMD_value {
 #define NGBE_RX_PB_SIZE				42
 #define NGBE_MC_TBL_SIZE			128
 #define NGBE_TDB_PB_SZ				(20 * 1024) /* 160KB Packet Buffer */
-#define NGBE_MAX_JUMBO_FRAME_SIZE		9432 /* max payload 9414 */
 
 /* TX/RX descriptor defines */
 #define NGBE_DEFAULT_TXD			512 /* default ring size */
diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
index a58ce5463686..5b8a121fb496 100644
--- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
+++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
@@ -6,9 +6,9 @@
 #include <linux/pci.h>
 #include <linux/netdevice.h>
 #include <linux/string.h>
-#include <linux/aer.h>
 #include <linux/etherdevice.h>
 #include <net/ip.h>
+#include <linux/if_vlan.h>
 
 #include "../libwx/wx_type.h"
 #include "../libwx/wx_lib.h"
@@ -488,6 +488,7 @@ static void txgbe_shutdown(struct pci_dev *pdev)
 static const struct net_device_ops txgbe_netdev_ops = {
 	.ndo_open               = txgbe_open,
 	.ndo_stop               = txgbe_close,
+	.ndo_change_mtu         = wx_change_mtu,
 	.ndo_start_xmit         = wx_xmit_frame,
 	.ndo_set_rx_mode        = wx_set_rx_mode,
 	.ndo_validate_addr      = eth_validate_addr,
@@ -539,7 +540,6 @@ static int txgbe_probe(struct pci_dev *pdev,
 		goto err_pci_disable_dev;
 	}
 
-	pci_enable_pcie_error_reporting(pdev);
 	pci_set_master(pdev);
 
 	netdev = devm_alloc_etherdev_mqs(&pdev->dev,
@@ -606,7 +606,8 @@ static int txgbe_probe(struct pci_dev *pdev,
 	netdev->priv_flags |= IFF_SUPP_NOFCS;
 
 	netdev->min_mtu = ETH_MIN_MTU;
-	netdev->max_mtu = TXGBE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
+	netdev->max_mtu = WX_MAX_JUMBO_FRAME_SIZE -
+			  (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
 
 	/* make sure the EEPROM is good */
 	err = txgbe_validate_eeprom_checksum(wx, NULL);
@@ -699,7 +700,6 @@ err_release_hw:
 err_free_mac_table:
 	kfree(wx->mac_table);
 err_pci_release_regions:
-	pci_disable_pcie_error_reporting(pdev);
 	pci_release_selected_regions(pdev,
 				     pci_select_bars(pdev, IORESOURCE_MEM));
 err_pci_disable_dev:
@@ -730,8 +730,6 @@ static void txgbe_remove(struct pci_dev *pdev)
 	kfree(wx->mac_table);
 	wx_clear_interrupt_scheme(wx);
 
-	pci_disable_pcie_error_reporting(pdev);
-
 	pci_disable_device(pdev);
 }
 
diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h
index 563ea51deca6..63a1c733718d 100644
--- a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h
+++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h
@@ -79,7 +79,6 @@
 #define TXGBE_SP_MC_TBL_SIZE    128
 #define TXGBE_SP_RX_PB_SIZE     512
 #define TXGBE_SP_TDB_PB_SZ      (160 * 1024) /* 160KB Packet Buffer */
-#define TXGBE_MAX_JUMBO_FRAME_SIZE      9432 /* max payload 9414 */
 
 /* TX/RX descriptor defines */
 #define TXGBE_DEFAULT_TXD               512
diff --git a/drivers/net/fddi/skfp/rmt.c b/drivers/net/fddi/skfp/rmt.c
index 37a89675dbeb..314623650a84 100644
--- a/drivers/net/fddi/skfp/rmt.c
+++ b/drivers/net/fddi/skfp/rmt.c
@@ -234,9 +234,9 @@ static void rmt_fsm(struct s_smc *smc, int cmd)
 		if (smc->r.rm_join) {
 			smc->r.sm_ma_avail = TRUE ;
 			if (smc->mib.m[MAC0].fddiMACMA_UnitdataEnable)
-			smc->mib.m[MAC0].fddiMACMA_UnitdataAvailable = TRUE ;
-				else
-			smc->mib.m[MAC0].fddiMACMA_UnitdataAvailable = FALSE ;
+				smc->mib.m[MAC0].fddiMACMA_UnitdataAvailable = TRUE;
+			else
+				smc->mib.m[MAC0].fddiMACMA_UnitdataAvailable = FALSE;
 		}
 		DB_RMTN(1, "RMT : RING UP");
 		RS_CLEAR(smc,RS_NORINGOP) ;
diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index 89ff7f8e8c7e..78f9d588f712 100644
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -365,13 +365,6 @@ static int geneve_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
 	if (unlikely(geneveh->ver != GENEVE_VER))
 		goto drop;
 
-	inner_proto = geneveh->proto_type;
-
-	if (unlikely((inner_proto != htons(ETH_P_TEB) &&
-		      inner_proto != htons(ETH_P_IP) &&
-		      inner_proto != htons(ETH_P_IPV6))))
-		goto drop;
-
 	gs = rcu_dereference_sk_user_data(sk);
 	if (!gs)
 		goto drop;
@@ -380,6 +373,8 @@ static int geneve_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
 	if (!geneve)
 		goto drop;
 
+	inner_proto = geneveh->proto_type;
+
 	if (unlikely((!geneve->cfg.inner_proto_inherit &&
 		      inner_proto != htons(ETH_P_TEB)))) {
 		geneve->dev->stats.rx_dropped++;
@@ -1426,7 +1421,7 @@ static int geneve_configure(struct net *net, struct net_device *dev,
 		dev->type = ARPHRD_NONE;
 		dev->hard_header_len = 0;
 		dev->addr_len = 0;
-		dev->flags = IFF_NOARP;
+		dev->flags = IFF_POINTOPOINT | IFF_NOARP;
 	}
 
 	err = register_netdevice(dev);
diff --git a/drivers/net/hamradio/Kconfig b/drivers/net/hamradio/Kconfig
index a9c44f08199d..a94c7bd5db2e 100644
--- a/drivers/net/hamradio/Kconfig
+++ b/drivers/net/hamradio/Kconfig
@@ -47,7 +47,7 @@ config BPQETHER
 
 config SCC
 	tristate "Z8530 SCC driver"
-	depends on ISA && AX25 && ISA_DMA_API
+	depends on ISA && AX25
 	help
 	  These cards are used to connect your Linux box to an amateur radio
 	  in order to communicate with other computers. If you want to use
diff --git a/drivers/net/ieee802154/adf7242.c b/drivers/net/ieee802154/adf7242.c
index 5cf218c674a5..f9972b8140f9 100644
--- a/drivers/net/ieee802154/adf7242.c
+++ b/drivers/net/ieee802154/adf7242.c
@@ -1336,9 +1336,8 @@ MODULE_DEVICE_TABLE(spi, adf7242_device_id);
 static struct spi_driver adf7242_driver = {
 	.id_table = adf7242_device_id,
 	.driver = {
-		   .of_match_table = of_match_ptr(adf7242_of_match),
+		   .of_match_table = adf7242_of_match,
 		   .name = "adf7242",
-		   .owner = THIS_MODULE,
 		   },
 	.probe = adf7242_probe,
 	.remove = adf7242_remove,
diff --git a/drivers/net/ieee802154/at86rf230.c b/drivers/net/ieee802154/at86rf230.c
index 62b984f84d9f..164c7f605af5 100644
--- a/drivers/net/ieee802154/at86rf230.c
+++ b/drivers/net/ieee802154/at86rf230.c
@@ -1662,7 +1662,7 @@ MODULE_DEVICE_TABLE(spi, at86rf230_device_id);
 static struct spi_driver at86rf230_driver = {
 	.id_table = at86rf230_device_id,
 	.driver = {
-		.of_match_table = of_match_ptr(at86rf230_of_match),
+		.of_match_table = at86rf230_of_match,
 		.name	= "at86rf230",
 	},
 	.probe      = at86rf230_probe,
diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c
index d0b5129439ed..0134ebac227a 100644
--- a/drivers/net/ieee802154/ca8210.c
+++ b/drivers/net/ieee802154/ca8210.c
@@ -51,6 +51,7 @@
 #include <linux/clk-provider.h>
 #include <linux/debugfs.h>
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/gpio.h>
 #include <linux/ieee802154.h>
 #include <linux/io.h>
@@ -2854,7 +2855,7 @@ static int ca8210_interrupt_init(struct spi_device *spi)
 	);
 	if (ret) {
 		dev_crit(&spi->dev, "request_irq %d failed\n", pdata->irq_id);
-		gpio_unexport(pdata->gpio_irq);
+		gpiod_unexport(gpio_to_desc(pdata->gpio_irq));
 		gpio_free(pdata->gpio_irq);
 	}
 
@@ -2968,7 +2969,7 @@ static int ca8210_test_interface_init(struct ca8210_priv *priv)
 		sizeof(node_name),
 		"ca8210@%d_%d",
 		priv->spi->master->bus_num,
-		priv->spi->chip_select
+		spi_get_chipselect(priv->spi, 0)
 	);
 
 	test->ca8210_dfs_spi_int = debugfs_create_file(
@@ -3178,8 +3179,7 @@ MODULE_DEVICE_TABLE(of, ca8210_of_ids);
 static struct spi_driver ca8210_spi_driver = {
 	.driver = {
 		.name =                 DRIVER_NAME,
-		.owner =                THIS_MODULE,
-		.of_match_table =       of_match_ptr(ca8210_of_ids),
+		.of_match_table =       ca8210_of_ids,
 	},
 	.probe  =                       ca8210_probe,
 	.remove =                       ca8210_remove
diff --git a/drivers/net/ieee802154/mcr20a.c b/drivers/net/ieee802154/mcr20a.c
index f53d185e0568..87abe3b46316 100644
--- a/drivers/net/ieee802154/mcr20a.c
+++ b/drivers/net/ieee802154/mcr20a.c
@@ -1352,7 +1352,7 @@ MODULE_DEVICE_TABLE(spi, mcr20a_device_id);
 static struct spi_driver mcr20a_driver = {
 	.id_table = mcr20a_device_id,
 	.driver = {
-		.of_match_table = of_match_ptr(mcr20a_of_match),
+		.of_match_table = mcr20a_of_match,
 		.name	= "mcr20a",
 	},
 	.probe      = mcr20a_probe,
diff --git a/drivers/net/ipa/Makefile b/drivers/net/ipa/Makefile
index cba199422f47..7293d5cc2b2b 100644
--- a/drivers/net/ipa/Makefile
+++ b/drivers/net/ipa/Makefile
@@ -2,10 +2,12 @@
 #
 # Makefile for the Qualcomm IPA driver.
 
-IPA_VERSIONS		:=	3.1 3.5.1 4.2 4.5 4.7 4.9 4.11
+IPA_REG_VERSIONS	:=	3.1 3.5.1 4.2 4.5 4.7 4.9 4.11 5.0
 
 # Some IPA versions can reuse another set of GSI register definitions.
-GSI_IPA_VERSIONS	:=	3.1 3.5.1 4.0 4.5 4.9 4.11
+GSI_REG_VERSIONS	:=	3.1 3.5.1 4.0 4.5 4.9 4.11 5.0
+
+IPA_DATA_VERSIONS	:=	3.1 3.5.1 4.2 4.5 4.7 4.9 4.11 5.0
 
 obj-$(CONFIG_QCOM_IPA)	+=	ipa.o
 
@@ -16,8 +18,8 @@ ipa-y			:=	ipa_main.o ipa_power.o ipa_reg.o ipa_mem.o \
 				ipa_resource.o ipa_qmi.o ipa_qmi_msg.o \
 				ipa_sysfs.o
 
-ipa-y			+=	$(GSI_IPA_VERSIONS:%=reg/gsi_reg-v%.o)
+ipa-y			+=	$(IPA_REG_VERSIONS:%=reg/ipa_reg-v%.o)
 
-ipa-y			+=	$(IPA_VERSIONS:%=reg/ipa_reg-v%.o)
+ipa-y			+=	$(GSI_REG_VERSIONS:%=reg/gsi_reg-v%.o)
 
-ipa-y			+=	$(IPA_VERSIONS:%=data/ipa_data-v%.o)
+ipa-y			+=	$(IPA_DATA_VERSIONS:%=data/ipa_data-v%.o)
diff --git a/drivers/net/ipa/data/ipa_data-v5.0.c b/drivers/net/ipa/data/ipa_data-v5.0.c
new file mode 100644
index 000000000000..4d8171dae4cd
--- /dev/null
+++ b/drivers/net/ipa/data/ipa_data-v5.0.c
@@ -0,0 +1,481 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* Copyright (C) 2023 Linaro Ltd. */
+
+#include <linux/log2.h>
+
+#include "../gsi.h"
+#include "../ipa_data.h"
+#include "../ipa_endpoint.h"
+#include "../ipa_mem.h"
+
+/** enum ipa_resource_type - IPA resource types for an SoC having IPA v5.0 */
+enum ipa_resource_type {
+	/* Source resource types; first must have value 0 */
+	IPA_RESOURCE_TYPE_SRC_PKT_CONTEXTS		= 0,
+	IPA_RESOURCE_TYPE_SRC_DESCRIPTOR_LISTS,
+	IPA_RESOURCE_TYPE_SRC_DESCRIPTOR_BUFF,
+	IPA_RESOURCE_TYPE_SRC_HPS_DMARS,
+	IPA_RESOURCE_TYPE_SRC_ACK_ENTRIES,
+
+	/* Destination resource types; first must have value 0 */
+	IPA_RESOURCE_TYPE_DST_DATA_SECTORS		= 0,
+	IPA_RESOURCE_TYPE_DST_DPS_DMARS,
+	IPA_RESOURCE_TYPE_DST_ULSO_SEGMENTS,
+};
+
+/* Resource groups used for an SoC having IPA v5.0 */
+enum ipa_rsrc_group_id {
+	/* Source resource group identifiers */
+	IPA_RSRC_GROUP_SRC_UL				= 0,
+	IPA_RSRC_GROUP_SRC_DL,
+	IPA_RSRC_GROUP_SRC_UNUSED_2,
+	IPA_RSRC_GROUP_SRC_UNUSED_3,
+	IPA_RSRC_GROUP_SRC_URLLC,
+	IPA_RSRC_GROUP_SRC_U_RX_QC,
+	IPA_RSRC_GROUP_SRC_COUNT,	/* Last in set; not a source group */
+
+	/* Destination resource group identifiers */
+	IPA_RSRC_GROUP_DST_UL				= 0,
+	IPA_RSRC_GROUP_DST_DL,
+	IPA_RSRC_GROUP_DST_DMA,
+	IPA_RSRC_GROUP_DST_QDSS,
+	IPA_RSRC_GROUP_DST_CV2X,
+	IPA_RSRC_GROUP_DST_UC,
+	IPA_RSRC_GROUP_DST_DRB_IP,
+	IPA_RSRC_GROUP_DST_COUNT,	/* Last; not a destination group */
+};
+
+/* QSB configuration data for an SoC having IPA v5.0 */
+static const struct ipa_qsb_data ipa_qsb_data[] = {
+	[IPA_QSB_MASTER_DDR] = {
+		.max_writes		= 0,
+		.max_reads		= 0,	/* no limit (hardware max) */
+		.max_reads_beats	= 0,
+	},
+	[IPA_QSB_MASTER_PCIE] = {
+		.max_writes		= 0,
+		.max_reads		= 0,	/* no limit (hardware max) */
+		.max_reads_beats	= 0,
+	},
+};
+
+/* Endpoint configuration data for an SoC having IPA v5.0 */
+static const struct ipa_gsi_endpoint_data ipa_gsi_endpoint_data[] = {
+	[IPA_ENDPOINT_AP_COMMAND_TX] = {
+		.ee_id		= GSI_EE_AP,
+		.channel_id	= 12,
+		.endpoint_id	= 14,
+		.toward_ipa	= true,
+		.channel = {
+			.tre_count	= 256,
+			.event_count	= 256,
+			.tlv_count	= 20,
+		},
+		.endpoint = {
+			.config = {
+				.resource_group	= IPA_RSRC_GROUP_SRC_UL,
+				.dma_mode	= true,
+				.dma_endpoint	= IPA_ENDPOINT_AP_LAN_RX,
+				.tx = {
+					.seq_type = IPA_SEQ_DMA,
+				},
+			},
+		},
+	},
+	[IPA_ENDPOINT_AP_LAN_RX] = {
+		.ee_id		= GSI_EE_AP,
+		.channel_id	= 13,
+		.endpoint_id	= 16,
+		.toward_ipa	= false,
+		.channel = {
+			.tre_count	= 256,
+			.event_count	= 256,
+			.tlv_count	= 9,
+		},
+		.endpoint = {
+			.config = {
+				.resource_group	= IPA_RSRC_GROUP_DST_UL,
+				.aggregation	= true,
+				.status_enable	= true,
+				.rx = {
+					.buffer_size	= 8192,
+					.pad_align	= ilog2(sizeof(u32)),
+					.aggr_time_limit = 500,
+				},
+			},
+		},
+	},
+	[IPA_ENDPOINT_AP_MODEM_TX] = {
+		.ee_id		= GSI_EE_AP,
+		.channel_id	= 11,
+		.endpoint_id	= 2,
+		.toward_ipa	= true,
+		.channel = {
+			.tre_count	= 512,
+			.event_count	= 512,
+			.tlv_count	= 25,
+		},
+		.endpoint = {
+			.filter_support	= true,
+			.config = {
+				.resource_group	= IPA_RSRC_GROUP_SRC_UL,
+				.checksum       = true,
+				.qmap		= true,
+				.status_enable	= true,
+				.tx = {
+					.seq_type = IPA_SEQ_2_PASS_SKIP_LAST_UC,
+					.status_endpoint =
+						IPA_ENDPOINT_MODEM_AP_RX,
+				},
+			},
+		},
+	},
+	[IPA_ENDPOINT_AP_MODEM_RX] = {
+		.ee_id		= GSI_EE_AP,
+		.channel_id	= 1,
+		.endpoint_id	= 23,
+		.toward_ipa	= false,
+		.channel = {
+			.tre_count	= 256,
+			.event_count	= 256,
+			.tlv_count	= 9,
+		},
+		.endpoint = {
+			.config = {
+				.resource_group	= IPA_RSRC_GROUP_DST_DL,
+				.checksum       = true,
+				.qmap		= true,
+				.aggregation	= true,
+				.rx = {
+					.buffer_size	= 8192,
+					.aggr_time_limit = 500,
+					.aggr_close_eof	= true,
+				},
+			},
+		},
+	},
+	[IPA_ENDPOINT_MODEM_AP_TX] = {
+		.ee_id		= GSI_EE_MODEM,
+		.channel_id	= 0,
+		.endpoint_id	= 12,
+		.toward_ipa	= true,
+		.endpoint = {
+			.filter_support	= true,
+		},
+	},
+	[IPA_ENDPOINT_MODEM_AP_RX] = {
+		.ee_id		= GSI_EE_MODEM,
+		.channel_id	= 7,
+		.endpoint_id	= 21,
+		.toward_ipa	= false,
+	},
+	[IPA_ENDPOINT_MODEM_DL_NLO_TX] = {
+		.ee_id		= GSI_EE_MODEM,
+		.channel_id	= 2,
+		.endpoint_id	= 15,
+		.toward_ipa	= true,
+		.endpoint = {
+			.filter_support	= true,
+		},
+	},
+};
+
+/* Source resource configuration data for an SoC having IPA v5.0 */
+static const struct ipa_resource ipa_resource_src[] = {
+	[IPA_RESOURCE_TYPE_SRC_PKT_CONTEXTS] = {
+		.limits[IPA_RSRC_GROUP_SRC_UL] = {
+			.min = 3,	.max = 9,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_DL] = {
+			.min = 4,	.max = 10,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_URLLC] = {
+			.min = 1,	.max = 63,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_U_RX_QC] = {
+			.min = 0,	.max = 63,
+		},
+	},
+	[IPA_RESOURCE_TYPE_SRC_DESCRIPTOR_LISTS] = {
+		.limits[IPA_RSRC_GROUP_SRC_UL] = {
+			.min = 9,	.max = 9,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_DL] = {
+			.min = 12,	.max = 12,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_URLLC] = {
+			.min = 10,	.max = 10,
+		},
+	},
+	[IPA_RESOURCE_TYPE_SRC_DESCRIPTOR_BUFF] = {
+		.limits[IPA_RSRC_GROUP_SRC_UL] = {
+			.min = 9,	.max = 9,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_DL] = {
+			.min = 24,	.max = 24,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_URLLC] = {
+			.min = 20,	.max = 20,
+		},
+	},
+	[IPA_RESOURCE_TYPE_SRC_HPS_DMARS] = {
+		.limits[IPA_RSRC_GROUP_SRC_UL] = {
+			.min = 0,	.max = 63,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_DL] = {
+			.min = 0,	.max = 63,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_URLLC] = {
+			.min = 1,	.max = 63,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_U_RX_QC] = {
+			.min = 0,	.max = 63,
+		},
+	},
+	[IPA_RESOURCE_TYPE_SRC_ACK_ENTRIES] = {
+		.limits[IPA_RSRC_GROUP_SRC_UL] = {
+			.min = 22,	.max = 22,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_DL] = {
+			.min = 16,	.max = 16,
+		},
+		.limits[IPA_RSRC_GROUP_SRC_URLLC] = {
+			.min = 16,	.max = 16,
+		},
+	},
+};
+
+/* Destination resource configuration data for an SoC having IPA v5.0 */
+static const struct ipa_resource ipa_resource_dst[] = {
+	[IPA_RESOURCE_TYPE_DST_DATA_SECTORS] = {
+		.limits[IPA_RSRC_GROUP_DST_UL] = {
+			.min = 6,	.max = 6,
+		},
+		.limits[IPA_RSRC_GROUP_DST_DL] = {
+			.min = 5,	.max = 5,
+		},
+		.limits[IPA_RSRC_GROUP_DST_DRB_IP] = {
+			.min = 39,	.max = 39,
+		},
+	},
+	[IPA_RESOURCE_TYPE_DST_DPS_DMARS] = {
+		.limits[IPA_RSRC_GROUP_DST_UL] = {
+			.min = 0,	.max = 3,
+		},
+		.limits[IPA_RSRC_GROUP_DST_DL] = {
+			.min = 0,	.max = 3,
+		},
+	},
+	[IPA_RESOURCE_TYPE_DST_ULSO_SEGMENTS] = {
+		.limits[IPA_RSRC_GROUP_DST_UL] = {
+			.min = 0,	.max = 63,
+		},
+		.limits[IPA_RSRC_GROUP_DST_DL] = {
+			.min = 0,	.max = 63,
+		},
+	},
+};
+
+/* Resource configuration data for an SoC having IPA v5.0 */
+static const struct ipa_resource_data ipa_resource_data = {
+	.rsrc_group_dst_count	= IPA_RSRC_GROUP_DST_COUNT,
+	.rsrc_group_src_count	= IPA_RSRC_GROUP_SRC_COUNT,
+	.resource_src_count	= ARRAY_SIZE(ipa_resource_src),
+	.resource_src		= ipa_resource_src,
+	.resource_dst_count	= ARRAY_SIZE(ipa_resource_dst),
+	.resource_dst		= ipa_resource_dst,
+};
+
+/* IPA-resident memory region data for an SoC having IPA v5.0 */
+static const struct ipa_mem ipa_mem_local_data[] = {
+	{
+		.id		= IPA_MEM_UC_EVENT_RING,
+		.offset		= 0x0000,
+		.size		= 0x1000,
+		.canary_count	= 0,
+	},
+	{
+		.id		= IPA_MEM_UC_SHARED,
+		.offset		= 0x1000,
+		.size		= 0x0080,
+		.canary_count	= 0,
+	},
+	{
+		.id		= IPA_MEM_UC_INFO,
+		.offset		= 0x1080,
+		.size		= 0x0200,
+		.canary_count	= 0,
+	},
+	{
+		.id		= IPA_MEM_V4_FILTER_HASHED,
+		.offset		= 0x1288,
+		.size		= 0x0078,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_V4_FILTER,
+		.offset		= 0x1308,
+		.size		= 0x0078,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_V6_FILTER_HASHED,
+		.offset		= 0x1388,
+		.size		= 0x0078,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_V6_FILTER,
+		.offset		= 0x1408,
+		.size		= 0x0078,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_V4_ROUTE_HASHED,
+		.offset		= 0x1488,
+		.size		= 0x0098,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_V4_ROUTE,
+		.offset		= 0x1528,
+		.size		= 0x0098,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_V6_ROUTE_HASHED,
+		.offset		= 0x15c8,
+		.size		= 0x0098,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_V6_ROUTE,
+		.offset		= 0x1668,
+		.size		= 0x0098,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_MODEM_HEADER,
+		.offset		= 0x1708,
+		.size		= 0x0240,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_AP_HEADER,
+		.offset		= 0x1948,
+		.size		= 0x01e0,
+		.canary_count	= 0,
+	},
+	{
+		.id		= IPA_MEM_MODEM_PROC_CTX,
+		.offset		= 0x1b40,
+		.size		= 0x0b20,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_AP_PROC_CTX,
+		.offset		= 0x2660,
+		.size		= 0x0200,
+		.canary_count	= 0,
+	},
+	{
+		.id		= IPA_MEM_STATS_QUOTA_MODEM,
+		.offset		= 0x2868,
+		.size		= 0x0060,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_STATS_QUOTA_AP,
+		.offset		= 0x28c8,
+		.size		= 0x0048,
+		.canary_count	= 0,
+	},
+	{
+		.id		= IPA_MEM_AP_V4_FILTER,
+		.offset		= 0x2918,
+		.size		= 0x0118,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_AP_V6_FILTER,
+		.offset		= 0x2aa0,
+		.size		= 0x0228,
+		.canary_count	= 0,
+	},
+	{
+		.id		= IPA_MEM_STATS_FILTER_ROUTE,
+		.offset		= 0x2cd0,
+		.size		= 0x0ba0,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_STATS_DROP,
+		.offset		= 0x3870,
+		.size		= 0x0020,
+		.canary_count	= 0,
+	},
+	{
+		.id		= IPA_MEM_MODEM,
+		.offset		= 0x3898,
+		.size		= 0x0d48,
+		.canary_count	= 2,
+	},
+	{
+		.id		= IPA_MEM_NAT_TABLE,
+		.offset		= 0x45e0,
+		.size		= 0x0900,
+		.canary_count	= 0,
+	},
+	{
+		.id		= IPA_MEM_PDN_CONFIG,
+		.offset		= 0x4ee8,
+		.size		= 0x0100,
+		.canary_count	= 2,
+	},
+};
+
+/* Memory configuration data for an SoC having IPA v5.0 */
+static const struct ipa_mem_data ipa_mem_data = {
+	.local_count	= ARRAY_SIZE(ipa_mem_local_data),
+	.local		= ipa_mem_local_data,
+	.imem_addr	= 0x14688000,
+	.imem_size	= 0x00003000,
+	.smem_id	= 497,
+	.smem_size	= 0x00009000,
+};
+
+/* Interconnect rates are in 1000 byte/second units */
+static const struct ipa_interconnect_data ipa_interconnect_data[] = {
+	{
+		.name			= "memory",
+		.peak_bandwidth		= 1900000,	/* 1.9 GBps */
+		.average_bandwidth	= 600000,	/* 600 MBps */
+	},
+	/* Average rate is unused for the next interconnect */
+	{
+		.name			= "config",
+		.peak_bandwidth		= 76800,	/* 76.8 MBps */
+		.average_bandwidth	= 0,		/* unused */
+	},
+};
+
+/* Clock and interconnect configuration data for an SoC having IPA v5.0 */
+static const struct ipa_power_data ipa_power_data = {
+	.core_clock_rate	= 120 * 1000 * 1000,	/* Hz */
+	.interconnect_count	= ARRAY_SIZE(ipa_interconnect_data),
+	.interconnect_data	= ipa_interconnect_data,
+};
+
+/* Configuration data for an SoC having IPA v5.0. */
+const struct ipa_data ipa_data_v5_0 = {
+	.version		= IPA_VERSION_5_0,
+	.qsb_count		= ARRAY_SIZE(ipa_qsb_data),
+	.qsb_data		= ipa_qsb_data,
+	.modem_route_count	= 11,
+	.endpoint_count		= ARRAY_SIZE(ipa_gsi_endpoint_data),
+	.endpoint_data		= ipa_gsi_endpoint_data,
+	.resource_data		= &ipa_resource_data,
+	.mem_data		= &ipa_mem_data,
+	.power_data		= &ipa_power_data,
+};
diff --git a/drivers/net/ipa/gsi.h b/drivers/net/ipa/gsi.h
index 50bc80cb167c..42063b227c18 100644
--- a/drivers/net/ipa/gsi.h
+++ b/drivers/net/ipa/gsi.h
@@ -16,8 +16,8 @@
 #include "ipa_version.h"
 
 /* Maximum number of channels and event rings supported by the driver */
-#define GSI_CHANNEL_COUNT_MAX	23
-#define GSI_EVT_RING_COUNT_MAX	24
+#define GSI_CHANNEL_COUNT_MAX	28
+#define GSI_EVT_RING_COUNT_MAX	28
 
 /* Maximum TLV FIFO size for a channel; 64 here is arbitrary (and high) */
 #define GSI_TLV_MAX		64
diff --git a/drivers/net/ipa/gsi_reg.c b/drivers/net/ipa/gsi_reg.c
index 1651fbad4bd5..c5458e28b12f 100644
--- a/drivers/net/ipa/gsi_reg.c
+++ b/drivers/net/ipa/gsi_reg.c
@@ -109,6 +109,9 @@ static const struct regs *gsi_regs(struct gsi *gsi)
 	case IPA_VERSION_4_11:
 		return &gsi_regs_v4_11;
 
+	case IPA_VERSION_5_0:
+		return &gsi_regs_v5_0;
+
 	default:
 		return NULL;
 	}
diff --git a/drivers/net/ipa/gsi_reg.h b/drivers/net/ipa/gsi_reg.h
index 48fde65fa2e8..cf046567f3fe 100644
--- a/drivers/net/ipa/gsi_reg.h
+++ b/drivers/net/ipa/gsi_reg.h
@@ -355,6 +355,7 @@ extern const struct regs gsi_regs_v4_0;
 extern const struct regs gsi_regs_v4_5;
 extern const struct regs gsi_regs_v4_9;
 extern const struct regs gsi_regs_v4_11;
+extern const struct regs gsi_regs_v5_0;
 
 /**
  * gsi_reg() - Return the structure describing a GSI register
diff --git a/drivers/net/ipa/ipa_data.h b/drivers/net/ipa/ipa_data.h
index 818e64114ed5..ce82b00fdc49 100644
--- a/drivers/net/ipa/ipa_data.h
+++ b/drivers/net/ipa/ipa_data.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2022 Linaro Ltd.
+ * Copyright (C) 2019-2023 Linaro Ltd.
  */
 #ifndef _IPA_DATA_H_
 #define _IPA_DATA_H_
@@ -249,5 +249,6 @@ extern const struct ipa_data ipa_data_v4_5;
 extern const struct ipa_data ipa_data_v4_7;
 extern const struct ipa_data ipa_data_v4_9;
 extern const struct ipa_data ipa_data_v4_11;
+extern const struct ipa_data ipa_data_v5_0;
 
 #endif /* _IPA_DATA_H_ */
diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c
index 4cc8d8d6bc9b..6a2f2fc2f501 100644
--- a/drivers/net/ipa/ipa_main.c
+++ b/drivers/net/ipa/ipa_main.c
@@ -285,7 +285,7 @@ static void ipa_hardware_config_comp(struct ipa *ipa)
 	} else if (ipa->version < IPA_VERSION_4_5) {
 		val |= reg_bit(reg, GSI_MULTI_AXI_MASTERS_DIS);
 	} else {
-		/* For IPA v4.5 FULL_FLUSH_WAIT_RS_CLOSURE_EN is 0 */
+		/* For IPA v4.5+ FULL_FLUSH_WAIT_RS_CLOSURE_EN is 0 */
 	}
 
 	val |= reg_bit(reg, GSI_MULTI_INORDER_RD_DIS);
@@ -684,6 +684,10 @@ static const struct of_device_id ipa_match[] = {
 		.compatible	= "qcom,sc7280-ipa",
 		.data		= &ipa_data_v4_11,
 	},
+	{
+		.compatible	= "qcom,sdx65-ipa",
+		.data		= &ipa_data_v5_0,
+	},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, ipa_match);
diff --git a/drivers/net/ipa/ipa_reg.c b/drivers/net/ipa/ipa_reg.c
index 3f475428dddd..818a84f7c42d 100644
--- a/drivers/net/ipa/ipa_reg.c
+++ b/drivers/net/ipa/ipa_reg.c
@@ -123,6 +123,8 @@ static const struct regs *ipa_regs(enum ipa_version version)
 		return &ipa_regs_v4_9;
 	case IPA_VERSION_4_11:
 		return &ipa_regs_v4_11;
+	case IPA_VERSION_5_0:
+		return &ipa_regs_v5_0;
 	default:
 		return NULL;
 	}
diff --git a/drivers/net/ipa/ipa_reg.h b/drivers/net/ipa/ipa_reg.h
index 7dd65d39333d..3ac48dea865b 100644
--- a/drivers/net/ipa/ipa_reg.h
+++ b/drivers/net/ipa/ipa_reg.h
@@ -636,6 +636,7 @@ extern const struct regs ipa_regs_v4_5;
 extern const struct regs ipa_regs_v4_7;
 extern const struct regs ipa_regs_v4_9;
 extern const struct regs ipa_regs_v4_11;
+extern const struct regs ipa_regs_v5_0;
 
 const struct reg *ipa_reg(struct ipa *ipa, enum ipa_reg_id reg_id);
 
diff --git a/drivers/net/ipa/ipa_sysfs.c b/drivers/net/ipa/ipa_sysfs.c
index 14bd2f903045..2ff09ce343b7 100644
--- a/drivers/net/ipa/ipa_sysfs.c
+++ b/drivers/net/ipa/ipa_sysfs.c
@@ -36,6 +36,8 @@ static const char *ipa_version_string(struct ipa *ipa)
 		return "4.9";
 	case IPA_VERSION_4_11:
 		return "4.11";
+	case IPA_VERSION_5_0:
+		return "5.0";
 	default:
 		return "0.0";	/* Won't happen (checked at probe time) */
 	}
diff --git a/drivers/net/ipa/reg/gsi_reg-v5.0.c b/drivers/net/ipa/reg/gsi_reg-v5.0.c
new file mode 100644
index 000000000000..d7b81a36d673
--- /dev/null
+++ b/drivers/net/ipa/reg/gsi_reg-v5.0.c
@@ -0,0 +1,317 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* Copyright (C) 2023 Linaro Ltd. */
+
+#include <linux/types.h>
+
+#include "../gsi.h"
+#include "../reg.h"
+#include "../gsi_reg.h"
+
+REG(INTER_EE_SRC_CH_IRQ_MSK, inter_ee_src_ch_irq_msk,
+    0x0000c01c + 0x1000 * GSI_EE_AP);
+
+REG(INTER_EE_SRC_EV_CH_IRQ_MSK, inter_ee_src_ev_ch_irq_msk,
+    0x0000c028 + 0x1000 * GSI_EE_AP);
+
+static const u32 reg_ch_c_cntxt_0_fmask[] = {
+	[CHTYPE_PROTOCOL]				= GENMASK(6, 0),
+	[CHTYPE_DIR]					= BIT(7),
+	[CH_EE]						= GENMASK(11, 8),
+	[CHID]						= GENMASK(19, 12),
+	[CHSTATE]					= GENMASK(23, 20),
+	[ELEMENT_SIZE]					= GENMASK(31, 24),
+};
+
+REG_STRIDE_FIELDS(CH_C_CNTXT_0, ch_c_cntxt_0,
+		  0x00014000 + 0x12000 * GSI_EE_AP, 0x80);
+
+static const u32 reg_ch_c_cntxt_1_fmask[] = {
+	[CH_R_LENGTH]					= GENMASK(23, 0),
+	[ERINDEX]					= GENMASK(31, 24),
+};
+
+REG_STRIDE_FIELDS(CH_C_CNTXT_1, ch_c_cntxt_1,
+		  0x00014004 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(CH_C_CNTXT_2, ch_c_cntxt_2, 0x00014008 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(CH_C_CNTXT_3, ch_c_cntxt_3, 0x0001400c + 0x12000 * GSI_EE_AP, 0x80);
+
+static const u32 reg_ch_c_qos_fmask[] = {
+	[WRR_WEIGHT]					= GENMASK(3, 0),
+						/* Bits 4-7 reserved */
+	[MAX_PREFETCH]					= BIT(8),
+	[USE_DB_ENG]					= BIT(9),
+	[PREFETCH_MODE]					= GENMASK(13, 10),
+						/* Bits 14-15 reserved */
+	[EMPTY_LVL_THRSHOLD]				= GENMASK(23, 16),
+	[DB_IN_BYTES]					= BIT(24),
+	[LOW_LATENCY_EN]				= BIT(25),
+						/* Bits 26-31 reserved */
+};
+
+REG_STRIDE_FIELDS(CH_C_QOS, ch_c_qos, 0x00014048 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(CH_C_SCRATCH_0, ch_c_scratch_0,
+	   0x0001404c + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(CH_C_SCRATCH_1, ch_c_scratch_1,
+	   0x00014050 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(CH_C_SCRATCH_2, ch_c_scratch_2,
+	   0x00014054 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(CH_C_SCRATCH_3, ch_c_scratch_3,
+	   0x00014058 + 0x12000 * GSI_EE_AP, 0x80);
+
+static const u32 reg_ev_ch_e_cntxt_0_fmask[] = {
+	[EV_CHTYPE]					= GENMASK(6, 0),
+	[EV_INTYPE]					= BIT(7),
+	[EV_EVCHID]					= GENMASK(15, 8),
+	[EV_EE]						= GENMASK(19, 16),
+	[EV_CHSTATE]					= GENMASK(23, 20),
+	[EV_ELEMENT_SIZE]				= GENMASK(31, 24),
+};
+
+REG_STRIDE_FIELDS(EV_CH_E_CNTXT_0, ev_ch_e_cntxt_0,
+		  0x0001c000 + 0x12000 * GSI_EE_AP, 0x80);
+
+static const u32 reg_ev_ch_e_cntxt_1_fmask[] = {
+	[R_LENGTH]					= GENMASK(19, 0),
+};
+
+REG_STRIDE_FIELDS(EV_CH_E_CNTXT_1, ev_ch_e_cntxt_1,
+		  0x0001c004 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(EV_CH_E_CNTXT_2, ev_ch_e_cntxt_2,
+	   0x0001c008 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(EV_CH_E_CNTXT_3, ev_ch_e_cntxt_3,
+	   0x0001c00c + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(EV_CH_E_CNTXT_4, ev_ch_e_cntxt_4,
+	   0x0001c010 + 0x12000 * GSI_EE_AP, 0x80);
+
+static const u32 reg_ev_ch_e_cntxt_8_fmask[] = {
+	[EV_MODT]					= GENMASK(15, 0),
+	[EV_MODC]					= GENMASK(23, 16),
+	[EV_MOD_CNT]					= GENMASK(31, 24),
+};
+
+REG_STRIDE_FIELDS(EV_CH_E_CNTXT_8, ev_ch_e_cntxt_8,
+		  0x0001c020 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(EV_CH_E_CNTXT_9, ev_ch_e_cntxt_9,
+	   0x0001c024 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(EV_CH_E_CNTXT_10, ev_ch_e_cntxt_10,
+	   0x0001c028 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(EV_CH_E_CNTXT_11, ev_ch_e_cntxt_11,
+	   0x0001c02c + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(EV_CH_E_CNTXT_12, ev_ch_e_cntxt_12,
+	   0x0001c030 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(EV_CH_E_CNTXT_13, ev_ch_e_cntxt_13,
+	   0x0001c034 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(EV_CH_E_SCRATCH_0, ev_ch_e_scratch_0,
+	   0x0001c048 + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(EV_CH_E_SCRATCH_1, ev_ch_e_scratch_1,
+	   0x0001c04c + 0x12000 * GSI_EE_AP, 0x80);
+
+REG_STRIDE(CH_C_DOORBELL_0, ch_c_doorbell_0,
+	   0x00024000 + 0x12000 * GSI_EE_AP, 0x08);
+
+REG_STRIDE(EV_CH_E_DOORBELL_0, ev_ch_e_doorbell_0,
+	   0x00024800 + 0x12000 * GSI_EE_AP, 0x08);
+
+static const u32 reg_gsi_status_fmask[] = {
+	[ENABLED]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+REG_FIELDS(GSI_STATUS, gsi_status, 0x00025000 + 0x12000 * GSI_EE_AP);
+
+static const u32 reg_ch_cmd_fmask[] = {
+	[CH_CHID]					= GENMASK(7, 0),
+						/* Bits 8-23 reserved */
+	[CH_OPCODE]					= GENMASK(31, 24),
+};
+
+REG_FIELDS(CH_CMD, ch_cmd, 0x00025008 + 0x12000 * GSI_EE_AP);
+
+static const u32 reg_ev_ch_cmd_fmask[] = {
+	[EV_CHID]					= GENMASK(7, 0),
+						/* Bits 8-23 reserved */
+	[EV_OPCODE]					= GENMASK(31, 24),
+};
+
+REG_FIELDS(EV_CH_CMD, ev_ch_cmd, 0x00025010 + 0x12000 * GSI_EE_AP);
+
+static const u32 reg_generic_cmd_fmask[] = {
+	[GENERIC_OPCODE]				= GENMASK(4, 0),
+	[GENERIC_CHID]					= GENMASK(9, 5),
+	[GENERIC_EE]					= GENMASK(13, 10),
+						/* Bits 14-31 reserved */
+};
+
+REG_FIELDS(GENERIC_CMD, generic_cmd, 0x00025018 + 0x12000 * GSI_EE_AP);
+
+static const u32 reg_hw_param_2_fmask[] = {
+	[NUM_CH_PER_EE]					= GENMASK(7, 0),
+	[IRAM_SIZE]					= GENMASK(12, 8),
+	[GSI_CH_PEND_TRANSLATE]				= BIT(13),
+	[GSI_CH_FULL_LOGIC]				= BIT(14),
+	[GSI_USE_SDMA]					= BIT(15),
+	[GSI_SDMA_N_INT]				= GENMASK(18, 16),
+	[GSI_SDMA_MAX_BURST]				= GENMASK(26, 19),
+	[GSI_SDMA_N_IOVEC]				= GENMASK(29, 27),
+	[GSI_USE_RD_WR_ENG]				= BIT(30),
+	[GSI_USE_INTER_EE]				= BIT(31),
+};
+
+REG_FIELDS(HW_PARAM_2, hw_param_2, 0x00025040 + 0x12000 * GSI_EE_AP);
+
+static const u32 reg_hw_param_4_fmask[] = {
+	[EV_PER_EE]					= GENMASK(7, 0),
+	[IRAM_PROTOCOL_COUNT]				= GENMASK(15, 8),
+						/* Bits 16-31 reserved */
+};
+
+REG_FIELDS(HW_PARAM_4, hw_param_4, 0x00025050 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_TYPE_IRQ, cntxt_type_irq, 0x00025080 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_TYPE_IRQ_MSK, cntxt_type_irq_msk, 0x00025088 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_SRC_CH_IRQ, cntxt_src_ch_irq, 0x00025090 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_SRC_CH_IRQ_MSK, cntxt_src_ch_irq_msk,
+    0x00025094 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_SRC_CH_IRQ_CLR, cntxt_src_ch_irq_clr,
+    0x00025098 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_SRC_EV_CH_IRQ, cntxt_src_ev_ch_irq, 0x0002509c + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_SRC_EV_CH_IRQ_MSK, cntxt_src_ev_ch_irq_msk,
+    0x000250a0 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_SRC_EV_CH_IRQ_CLR, cntxt_src_ev_ch_irq_clr,
+    0x000250a4 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_SRC_IEOB_IRQ, cntxt_src_ieob_irq, 0x000250a8 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_SRC_IEOB_IRQ_MSK, cntxt_src_ieob_irq_msk,
+    0x000250ac + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_SRC_IEOB_IRQ_CLR, cntxt_src_ieob_irq_clr,
+    0x000250b0 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_GLOB_IRQ_STTS, cntxt_glob_irq_stts, 0x00025200 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_GLOB_IRQ_EN, cntxt_glob_irq_en, 0x00025204 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_GLOB_IRQ_CLR, cntxt_glob_irq_clr, 0x00025208 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_GSI_IRQ_STTS, cntxt_gsi_irq_stts, 0x0002520c + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_GSI_IRQ_EN, cntxt_gsi_irq_en, 0x00025210 + 0x12000 * GSI_EE_AP);
+
+REG(CNTXT_GSI_IRQ_CLR, cntxt_gsi_irq_clr, 0x00025214 + 0x12000 * GSI_EE_AP);
+
+static const u32 reg_cntxt_intset_fmask[] = {
+	[INTYPE]					= BIT(0)
+						/* Bits 1-31 reserved */
+};
+
+REG_FIELDS(CNTXT_INTSET, cntxt_intset, 0x00025220 + 0x12000 * GSI_EE_AP);
+
+static const u32 reg_error_log_fmask[] = {
+	[ERR_ARG3]					= GENMASK(3, 0),
+	[ERR_ARG2]					= GENMASK(7, 4),
+	[ERR_ARG1]					= GENMASK(11, 8),
+	[ERR_CODE]					= GENMASK(15, 12),
+						/* Bits 16-18 reserved */
+	[ERR_VIRT_IDX]					= GENMASK(23, 19),
+	[ERR_TYPE]					= GENMASK(27, 24),
+	[ERR_EE]					= GENMASK(31, 28),
+};
+
+REG_FIELDS(ERROR_LOG, error_log, 0x00025240 + 0x12000 * GSI_EE_AP);
+
+REG(ERROR_LOG_CLR, error_log_clr, 0x00025244 + 0x12000 * GSI_EE_AP);
+
+static const u32 reg_cntxt_scratch_0_fmask[] = {
+	[INTER_EE_RESULT]				= GENMASK(2, 0),
+						/* Bits 3-4 reserved */
+	[GENERIC_EE_RESULT]				= GENMASK(7, 5),
+						/* Bits 8-31 reserved */
+};
+
+REG_FIELDS(CNTXT_SCRATCH_0, cntxt_scratch_0, 0x00025400 + 0x12000 * GSI_EE_AP);
+
+static const struct reg *reg_array[] = {
+	[INTER_EE_SRC_CH_IRQ_MSK]	= &reg_inter_ee_src_ch_irq_msk,
+	[INTER_EE_SRC_EV_CH_IRQ_MSK]	= &reg_inter_ee_src_ev_ch_irq_msk,
+	[CH_C_CNTXT_0]			= &reg_ch_c_cntxt_0,
+	[CH_C_CNTXT_1]			= &reg_ch_c_cntxt_1,
+	[CH_C_CNTXT_2]			= &reg_ch_c_cntxt_2,
+	[CH_C_CNTXT_3]			= &reg_ch_c_cntxt_3,
+	[CH_C_QOS]			= &reg_ch_c_qos,
+	[CH_C_SCRATCH_0]		= &reg_ch_c_scratch_0,
+	[CH_C_SCRATCH_1]		= &reg_ch_c_scratch_1,
+	[CH_C_SCRATCH_2]		= &reg_ch_c_scratch_2,
+	[CH_C_SCRATCH_3]		= &reg_ch_c_scratch_3,
+	[EV_CH_E_CNTXT_0]		= &reg_ev_ch_e_cntxt_0,
+	[EV_CH_E_CNTXT_1]		= &reg_ev_ch_e_cntxt_1,
+	[EV_CH_E_CNTXT_2]		= &reg_ev_ch_e_cntxt_2,
+	[EV_CH_E_CNTXT_3]		= &reg_ev_ch_e_cntxt_3,
+	[EV_CH_E_CNTXT_4]		= &reg_ev_ch_e_cntxt_4,
+	[EV_CH_E_CNTXT_8]		= &reg_ev_ch_e_cntxt_8,
+	[EV_CH_E_CNTXT_9]		= &reg_ev_ch_e_cntxt_9,
+	[EV_CH_E_CNTXT_10]		= &reg_ev_ch_e_cntxt_10,
+	[EV_CH_E_CNTXT_11]		= &reg_ev_ch_e_cntxt_11,
+	[EV_CH_E_CNTXT_12]		= &reg_ev_ch_e_cntxt_12,
+	[EV_CH_E_CNTXT_13]		= &reg_ev_ch_e_cntxt_13,
+	[EV_CH_E_SCRATCH_0]		= &reg_ev_ch_e_scratch_0,
+	[EV_CH_E_SCRATCH_1]		= &reg_ev_ch_e_scratch_1,
+	[CH_C_DOORBELL_0]		= &reg_ch_c_doorbell_0,
+	[EV_CH_E_DOORBELL_0]		= &reg_ev_ch_e_doorbell_0,
+	[GSI_STATUS]			= &reg_gsi_status,
+	[CH_CMD]			= &reg_ch_cmd,
+	[EV_CH_CMD]			= &reg_ev_ch_cmd,
+	[GENERIC_CMD]			= &reg_generic_cmd,
+	[HW_PARAM_2]			= &reg_hw_param_2,
+	[HW_PARAM_4]			= &reg_hw_param_4,
+	[CNTXT_TYPE_IRQ]		= &reg_cntxt_type_irq,
+	[CNTXT_TYPE_IRQ_MSK]		= &reg_cntxt_type_irq_msk,
+	[CNTXT_SRC_CH_IRQ]		= &reg_cntxt_src_ch_irq,
+	[CNTXT_SRC_CH_IRQ_MSK]		= &reg_cntxt_src_ch_irq_msk,
+	[CNTXT_SRC_CH_IRQ_CLR]		= &reg_cntxt_src_ch_irq_clr,
+	[CNTXT_SRC_EV_CH_IRQ]		= &reg_cntxt_src_ev_ch_irq,
+	[CNTXT_SRC_EV_CH_IRQ_MSK]	= &reg_cntxt_src_ev_ch_irq_msk,
+	[CNTXT_SRC_EV_CH_IRQ_CLR]	= &reg_cntxt_src_ev_ch_irq_clr,
+	[CNTXT_SRC_IEOB_IRQ]		= &reg_cntxt_src_ieob_irq,
+	[CNTXT_SRC_IEOB_IRQ_MSK]	= &reg_cntxt_src_ieob_irq_msk,
+	[CNTXT_SRC_IEOB_IRQ_CLR]	= &reg_cntxt_src_ieob_irq_clr,
+	[CNTXT_GLOB_IRQ_STTS]		= &reg_cntxt_glob_irq_stts,
+	[CNTXT_GLOB_IRQ_EN]		= &reg_cntxt_glob_irq_en,
+	[CNTXT_GLOB_IRQ_CLR]		= &reg_cntxt_glob_irq_clr,
+	[CNTXT_GSI_IRQ_STTS]		= &reg_cntxt_gsi_irq_stts,
+	[CNTXT_GSI_IRQ_EN]		= &reg_cntxt_gsi_irq_en,
+	[CNTXT_GSI_IRQ_CLR]		= &reg_cntxt_gsi_irq_clr,
+	[CNTXT_INTSET]			= &reg_cntxt_intset,
+	[ERROR_LOG]			= &reg_error_log,
+	[ERROR_LOG_CLR]			= &reg_error_log_clr,
+	[CNTXT_SCRATCH_0]		= &reg_cntxt_scratch_0,
+};
+
+const struct regs gsi_regs_v5_0 = {
+	.reg_count	= ARRAY_SIZE(reg_array),
+	.reg		= reg_array,
+};
diff --git a/drivers/net/ipa/reg/ipa_reg-v5.0.c b/drivers/net/ipa/reg/ipa_reg-v5.0.c
new file mode 100644
index 000000000000..95e0edff4170
--- /dev/null
+++ b/drivers/net/ipa/reg/ipa_reg-v5.0.c
@@ -0,0 +1,564 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* Copyright (C) 2023 Linaro Ltd. */
+
+#include <linux/types.h>
+
+#include "../ipa.h"
+#include "../ipa_reg.h"
+
+static const u32 reg_flavor_0_fmask[] = {
+	[MAX_PIPES]					= GENMASK(7, 0),
+	[MAX_CONS_PIPES]				= GENMASK(15, 8),
+	[MAX_PROD_PIPES]				= GENMASK(23, 16),
+	[PROD_LOWEST]					= GENMASK(31, 24),
+};
+
+REG_FIELDS(FLAVOR_0, flavor_0, 0x00000000);
+
+static const u32 reg_comp_cfg_fmask[] = {
+	[RAM_ARB_PRI_CLIENT_SAMP_FIX_DIS]		= BIT(0),
+	[GSI_SNOC_BYPASS_DIS]				= BIT(1),
+	[GEN_QMB_0_SNOC_BYPASS_DIS]			= BIT(2),
+	[GEN_QMB_1_SNOC_BYPASS_DIS]			= BIT(3),
+						/* Bit 4 reserved */
+	[IPA_QMB_SELECT_CONS_EN]			= BIT(5),
+	[IPA_QMB_SELECT_PROD_EN]			= BIT(6),
+	[GSI_MULTI_INORDER_RD_DIS]			= BIT(7),
+	[GSI_MULTI_INORDER_WR_DIS]			= BIT(8),
+	[GEN_QMB_0_MULTI_INORDER_RD_DIS]		= BIT(9),
+	[GEN_QMB_1_MULTI_INORDER_RD_DIS]		= BIT(10),
+	[GEN_QMB_0_MULTI_INORDER_WR_DIS]		= BIT(11),
+	[GEN_QMB_1_MULTI_INORDER_WR_DIS]		= BIT(12),
+	[GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS]		= BIT(13),
+	[GSI_SNOC_CNOC_LOOP_PROT_DISABLE]		= BIT(14),
+	[GSI_MULTI_AXI_MASTERS_DIS]			= BIT(15),
+	[IPA_QMB_SELECT_GLOBAL_EN]			= BIT(16),
+	[FULL_FLUSH_WAIT_RS_CLOSURE_EN]			= BIT(17),
+						/* Bit 18 reserved */
+	[QMB_RAM_RD_CACHE_DISABLE]			= BIT(19),
+	[GENQMB_AOOOWR]					= BIT(20),
+	[IF_OUT_OF_BUF_STOP_RESET_MASK_EN]		= BIT(21),
+	[ATOMIC_FETCHER_ARB_LOCK_DIS]			= GENMASK(27, 22),
+						/* Bits 28-29 reserved */
+	[GEN_QMB_1_DYNAMIC_ASIZE]			= BIT(30),
+	[GEN_QMB_0_DYNAMIC_ASIZE]			= BIT(31),
+};
+
+REG_FIELDS(COMP_CFG, comp_cfg, 0x0000002c);
+
+static const u32 reg_clkon_cfg_fmask[] = {
+	[CLKON_RX]					= BIT(0),
+	[CLKON_PROC]					= BIT(1),
+	[TX_WRAPPER]					= BIT(2),
+	[CLKON_MISC]					= BIT(3),
+	[RAM_ARB]					= BIT(4),
+	[FTCH_HPS]					= BIT(5),
+	[FTCH_DPS]					= BIT(6),
+	[CLKON_HPS]					= BIT(7),
+	[CLKON_DPS]					= BIT(8),
+	[RX_HPS_CMDQS]					= BIT(9),
+	[HPS_DPS_CMDQS]					= BIT(10),
+	[DPS_TX_CMDQS]					= BIT(11),
+	[RSRC_MNGR]					= BIT(12),
+	[CTX_HANDLER]					= BIT(13),
+	[ACK_MNGR]					= BIT(14),
+	[D_DCPH]					= BIT(15),
+	[H_DCPH]					= BIT(16),
+						/* Bit 17 reserved */
+	[NTF_TX_CMDQS]					= BIT(18),
+	[CLKON_TX_0]					= BIT(19),
+	[CLKON_TX_1]					= BIT(20),
+	[CLKON_FNR]					= BIT(21),
+	[QSB2AXI_CMDQ_L]				= BIT(22),
+	[AGGR_WRAPPER]					= BIT(23),
+	[RAM_SLAVEWAY]					= BIT(24),
+	[CLKON_QMB]					= BIT(25),
+	[WEIGHT_ARB]					= BIT(26),
+	[GSI_IF]					= BIT(27),
+	[CLKON_GLOBAL]					= BIT(28),
+	[GLOBAL_2X_CLK]					= BIT(29),
+	[DPL_FIFO]					= BIT(30),
+	[DRBIP]						= BIT(31),
+};
+
+REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000034);
+
+static const u32 reg_route_fmask[] = {
+	[ROUTE_DEF_PIPE]				= GENMASK(7, 0),
+	[ROUTE_FRAG_DEF_PIPE]				= GENMASK(15, 8),
+	[ROUTE_DEF_HDR_OFST]				= GENMASK(25, 16),
+	[ROUTE_DEF_HDR_TABLE]				= BIT(26),
+	[ROUTE_DEF_RETAIN_HDR]				= BIT(27),
+	[ROUTE_DIS]					= BIT(28),
+						/* Bits 29-31 reserved */
+};
+
+REG_FIELDS(ROUTE, route, 0x00000038);
+
+static const u32 reg_shared_mem_size_fmask[] = {
+	[MEM_SIZE]					= GENMASK(15, 0),
+	[MEM_BADDR]					= GENMASK(31, 16),
+};
+
+REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000040);
+
+static const u32 reg_qsb_max_writes_fmask[] = {
+	[GEN_QMB_0_MAX_WRITES]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_WRITES]				= GENMASK(7, 4),
+						/* Bits 8-31 reserved */
+};
+
+REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000054);
+
+static const u32 reg_qsb_max_reads_fmask[] = {
+	[GEN_QMB_0_MAX_READS]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_READS]				= GENMASK(7, 4),
+						/* Bits 8-15 reserved */
+	[GEN_QMB_0_MAX_READS_BEATS]			= GENMASK(23, 16),
+	[GEN_QMB_1_MAX_READS_BEATS]			= GENMASK(31, 24),
+};
+
+REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000058);
+
+/* Valid bits defined by ipa->available */
+
+REG_STRIDE(STATE_AGGR_ACTIVE, state_aggr_active, 0x00000100, 0x0004);
+
+static const u32 reg_filt_rout_cache_flush_fmask[] = {
+	[ROUTER_CACHE]					= BIT(0),
+						/* Bits 1-3 reserved */
+	[FILTER_CACHE]					= BIT(4),
+						/* Bits 5-31 reserved */
+};
+
+REG_FIELDS(FILT_ROUT_CACHE_FLUSH, filt_rout_cache_flush, 0x0000404);
+
+static const u32 reg_local_pkt_proc_cntxt_fmask[] = {
+	[IPA_BASE_ADDR]					= GENMASK(17, 0),
+						/* Bits 18-31 reserved */
+};
+
+/* Offset must be a multiple of 8 */
+REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x00000478);
+
+static const u32 reg_ipa_tx_cfg_fmask[] = {
+						/* Bits 0-1 reserved */
+	[PREFETCH_ALMOST_EMPTY_SIZE_TX0]		= GENMASK(5, 2),
+	[DMAW_SCND_OUTSD_PRED_THRESHOLD]		= GENMASK(9, 6),
+	[DMAW_SCND_OUTSD_PRED_EN]			= BIT(10),
+	[DMAW_MAX_BEATS_256_DIS]			= BIT(11),
+	[PA_MASK_EN]					= BIT(12),
+	[PREFETCH_ALMOST_EMPTY_SIZE_TX1]		= GENMASK(16, 13),
+	[DUAL_TX_ENABLE]				= BIT(17),
+	[SSPND_PA_NO_START_STATE]			= BIT(18),
+						/* Bit 19 reserved */
+	[HOLB_STICKY_DROP_EN]				= BIT(20),
+						/* Bits 21-31 reserved */
+};
+
+REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x00000488);
+
+static const u32 reg_idle_indication_cfg_fmask[] = {
+	[ENTER_IDLE_DEBOUNCE_THRESH]			= GENMASK(15, 0),
+	[CONST_NON_IDLE_ENABLE]				= BIT(16),
+						/* Bits 17-31 reserved */
+};
+
+REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x000004a8);
+
+static const u32 reg_qtime_timestamp_cfg_fmask[] = {
+	[DPL_TIMESTAMP_LSB]				= GENMASK(4, 0),
+						/* Bits 5-6 reserved */
+	[DPL_TIMESTAMP_SEL]				= BIT(7),
+	[TAG_TIMESTAMP_LSB]				= GENMASK(12, 8),
+						/* Bits 13-15 reserved */
+	[NAT_TIMESTAMP_LSB]				= GENMASK(20, 16),
+						/* Bits 21-31 reserved */
+};
+
+REG_FIELDS(QTIME_TIMESTAMP_CFG, qtime_timestamp_cfg, 0x000004ac);
+
+static const u32 reg_timers_xo_clk_div_cfg_fmask[] = {
+	[DIV_VALUE]					= GENMASK(8, 0),
+						/* Bits 9-30 reserved */
+	[DIV_ENABLE]					= BIT(31),
+};
+
+REG_FIELDS(TIMERS_XO_CLK_DIV_CFG, timers_xo_clk_div_cfg, 0x000004b0);
+
+static const u32 reg_timers_pulse_gran_cfg_fmask[] = {
+	[PULSE_GRAN_0]					= GENMASK(2, 0),
+	[PULSE_GRAN_1]					= GENMASK(5, 3),
+	[PULSE_GRAN_2]					= GENMASK(8, 6),
+	[PULSE_GRAN_3]					= GENMASK(11, 9),
+						/* Bits 12-31 reserved */
+};
+
+REG_FIELDS(TIMERS_PULSE_GRAN_CFG, timers_pulse_gran_cfg, 0x000004b4);
+
+static const u32 reg_src_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type,
+		  0x00000500, 0x0020);
+
+static const u32 reg_src_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type,
+		  0x00000504, 0x0020);
+
+static const u32 reg_src_rsrc_grp_45_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+REG_STRIDE_FIELDS(SRC_RSRC_GRP_45_RSRC_TYPE, src_rsrc_grp_45_rsrc_type,
+		  0x00000508, 0x0020);
+
+static const u32 reg_src_rsrc_grp_67_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+REG_STRIDE_FIELDS(SRC_RSRC_GRP_67_RSRC_TYPE, src_rsrc_grp_67_rsrc_type,
+		  0x0000050c, 0x0020);
+
+static const u32 reg_dst_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type,
+		  0x00000600, 0x0020);
+
+static const u32 reg_dst_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type,
+		  0x00000604, 0x0020);
+
+static const u32 reg_dst_rsrc_grp_45_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+REG_STRIDE_FIELDS(DST_RSRC_GRP_45_RSRC_TYPE, dst_rsrc_grp_45_rsrc_type,
+		  0x00000608, 0x0020);
+
+static const u32 reg_dst_rsrc_grp_67_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+REG_STRIDE_FIELDS(DST_RSRC_GRP_67_RSRC_TYPE, dst_rsrc_grp_67_rsrc_type,
+		  0x0000060c, 0x0020);
+
+/* Valid bits defined by ipa->available */
+
+REG_STRIDE(AGGR_FORCE_CLOSE, aggr_force_close, 0x000006b0, 0x0004);
+
+static const u32 reg_endp_init_cfg_fmask[] = {
+	[FRAG_OFFLOAD_EN]				= BIT(0),
+	[CS_OFFLOAD_EN]					= GENMASK(2, 1),
+	[CS_METADATA_HDR_OFFSET]			= GENMASK(6, 3),
+						/* Bit 7 reserved */
+	[CS_GEN_QMB_MASTER_SEL]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00001008, 0x0080);
+
+static const u32 reg_endp_init_nat_fmask[] = {
+	[NAT_EN]					= GENMASK(1, 0),
+						/* Bits 2-31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000100c, 0x0080);
+
+static const u32 reg_endp_init_hdr_fmask[] = {
+	[HDR_LEN]					= GENMASK(5, 0),
+	[HDR_OFST_METADATA_VALID]			= BIT(6),
+	[HDR_OFST_METADATA]				= GENMASK(12, 7),
+	[HDR_ADDITIONAL_CONST_LEN]			= GENMASK(18, 13),
+	[HDR_OFST_PKT_SIZE_VALID]			= BIT(19),
+	[HDR_OFST_PKT_SIZE]				= GENMASK(25, 20),
+						/* Bit 26 reserved */
+	[HDR_LEN_INC_DEAGG_HDR]				= BIT(27),
+	[HDR_LEN_MSB]					= GENMASK(29, 28),
+	[HDR_OFST_METADATA_MSB]				= GENMASK(31, 30),
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00001010, 0x0080);
+
+static const u32 reg_endp_init_hdr_ext_fmask[] = {
+	[HDR_ENDIANNESS]				= BIT(0),
+	[HDR_TOTAL_LEN_OR_PAD_VALID]			= BIT(1),
+	[HDR_TOTAL_LEN_OR_PAD]				= BIT(2),
+	[HDR_PAYLOAD_LEN_INC_PADDING]			= BIT(3),
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET]			= GENMASK(9, 4),
+	[HDR_PAD_TO_ALIGNMENT]				= GENMASK(13, 10),
+						/* Bits 14-15 reserved */
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB]		= GENMASK(17, 16),
+	[HDR_OFST_PKT_SIZE_MSB]				= GENMASK(19, 18),
+	[HDR_ADDITIONAL_CONST_LEN_MSB]			= GENMASK(21, 20),
+	[HDR_BYTES_TO_REMOVE_VALID]			= BIT(22),
+						/* Bit 23 reserved */
+	[HDR_BYTES_TO_REMOVE]				= GENMASK(31, 24),
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00001014, 0x0080);
+
+REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask,
+	   0x00001018, 0x0080);
+
+static const u32 reg_endp_init_mode_fmask[] = {
+	[ENDP_MODE]					= GENMASK(2, 0),
+	[DCPH_ENABLE]					= BIT(3),
+	[DEST_PIPE_INDEX]				= GENMASK(11, 4),
+	[BYTE_THRESHOLD]				= GENMASK(27, 12),
+	[PIPE_REPLICATION_EN]				= BIT(28),
+	[PAD_EN]					= BIT(29),
+	[DRBIP_ACL_ENABLE]				= BIT(30),
+						/* Bit 31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00001020, 0x0080);
+
+static const u32 reg_endp_init_aggr_fmask[] = {
+	[AGGR_EN]					= GENMASK(1, 0),
+	[AGGR_TYPE]					= GENMASK(4, 2),
+	[BYTE_LIMIT]					= GENMASK(10, 5),
+						/* Bit 11 reserved */
+	[TIME_LIMIT]					= GENMASK(16, 12),
+	[PKT_LIMIT]					= GENMASK(22, 17),
+	[SW_EOF_ACTIVE]					= BIT(23),
+	[FORCE_CLOSE]					= BIT(24),
+						/* Bit 25 reserved */
+	[HARD_BYTE_LIMIT_EN]				= BIT(26),
+	[AGGR_GRAN_SEL]					= BIT(27),
+						/* Bits 28-31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00001024, 0x0080);
+
+static const u32 reg_endp_init_hol_block_en_fmask[] = {
+	[HOL_BLOCK_EN]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en,
+		  0x0000102c, 0x0080);
+
+static const u32 reg_endp_init_hol_block_timer_fmask[] = {
+	[TIMER_LIMIT]					= GENMASK(4, 0),
+						/* Bits 5-7 reserved */
+	[TIMER_GRAN_SEL]				= GENMASK(9, 8),
+						/* Bits 10-31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer,
+		  0x00001030, 0x0080);
+
+static const u32 reg_endp_init_deaggr_fmask[] = {
+	[DEAGGR_HDR_LEN]				= GENMASK(5, 0),
+	[SYSPIPE_ERR_DETECTION]				= BIT(6),
+	[PACKET_OFFSET_VALID]				= BIT(7),
+	[PACKET_OFFSET_LOCATION]			= GENMASK(13, 8),
+	[IGNORE_MIN_PKT_ERR]				= BIT(14),
+						/* Bit 15 reserved */
+	[MAX_PACKET_LEN]				= GENMASK(31, 16),
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00001034, 0x0080);
+
+static const u32 reg_endp_init_rsrc_grp_fmask[] = {
+	[ENDP_RSRC_GRP]					= GENMASK(2, 0),
+						/* Bits 3-31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp, 0x00001038, 0x0080);
+
+static const u32 reg_endp_init_seq_fmask[] = {
+	[SEQ_TYPE]					= GENMASK(7, 0),
+						/* Bits 8-31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000103c, 0x0080);
+
+static const u32 reg_endp_status_fmask[] = {
+	[STATUS_EN]					= BIT(0),
+	[STATUS_ENDP]					= GENMASK(8, 1),
+	[STATUS_PKT_SUPPRESS]				= BIT(9),
+						/* Bits 10-31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00001040, 0x0080);
+
+static const u32 reg_endp_filter_cache_cfg_fmask[] = {
+	[CACHE_MSK_SRC_ID]				= BIT(0),
+	[CACHE_MSK_SRC_IP]				= BIT(1),
+	[CACHE_MSK_DST_IP]				= BIT(2),
+	[CACHE_MSK_SRC_PORT]				= BIT(3),
+	[CACHE_MSK_DST_PORT]				= BIT(4),
+	[CACHE_MSK_PROTOCOL]				= BIT(5),
+	[CACHE_MSK_METADATA]				= BIT(6),
+						/* Bits 7-31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_FILTER_CACHE_CFG, endp_filter_cache_cfg,
+		  0x0000105c, 0x0080);
+
+static const u32 reg_endp_router_cache_cfg_fmask[] = {
+	[CACHE_MSK_SRC_ID]				= BIT(0),
+	[CACHE_MSK_SRC_IP]				= BIT(1),
+	[CACHE_MSK_DST_IP]				= BIT(2),
+	[CACHE_MSK_SRC_PORT]				= BIT(3),
+	[CACHE_MSK_DST_PORT]				= BIT(4),
+	[CACHE_MSK_PROTOCOL]				= BIT(5),
+	[CACHE_MSK_METADATA]				= BIT(6),
+						/* Bits 7-31 reserved */
+};
+
+REG_STRIDE_FIELDS(ENDP_ROUTER_CACHE_CFG, endp_router_cache_cfg,
+		  0x00001070, 0x0080);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+REG(IPA_IRQ_STTS, ipa_irq_stts, 0x0000c008 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+REG(IPA_IRQ_EN, ipa_irq_en, 0x0000c00c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+REG(IPA_IRQ_CLR, ipa_irq_clr, 0x0000c010 + 0x1000 * GSI_EE_AP);
+
+static const u32 reg_ipa_irq_uc_fmask[] = {
+	[UC_INTR]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000c01c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+
+REG_STRIDE(IRQ_SUSPEND_INFO, irq_suspend_info,
+	   0x0000c030 + 0x1000 * GSI_EE_AP, 0x0004);
+
+/* Valid bits defined by ipa->available */
+
+REG_STRIDE(IRQ_SUSPEND_EN, irq_suspend_en,
+	   0x0000c050 + 0x1000 * GSI_EE_AP, 0x0004);
+
+/* Valid bits defined by ipa->available */
+
+REG_STRIDE(IRQ_SUSPEND_CLR, irq_suspend_clr,
+	   0x0000c070 + 0x1000 * GSI_EE_AP, 0x0004);
+
+static const struct reg *reg_array[] = {
+	[COMP_CFG]			= &reg_comp_cfg,
+	[CLKON_CFG]			= &reg_clkon_cfg,
+	[ROUTE]				= &reg_route,
+	[SHARED_MEM_SIZE]		= &reg_shared_mem_size,
+	[QSB_MAX_WRITES]		= &reg_qsb_max_writes,
+	[QSB_MAX_READS]			= &reg_qsb_max_reads,
+	[FILT_ROUT_CACHE_FLUSH]		= &reg_filt_rout_cache_flush,
+	[STATE_AGGR_ACTIVE]		= &reg_state_aggr_active,
+	[LOCAL_PKT_PROC_CNTXT]		= &reg_local_pkt_proc_cntxt,
+	[AGGR_FORCE_CLOSE]		= &reg_aggr_force_close,
+	[IPA_TX_CFG]			= &reg_ipa_tx_cfg,
+	[FLAVOR_0]			= &reg_flavor_0,
+	[IDLE_INDICATION_CFG]		= &reg_idle_indication_cfg,
+	[QTIME_TIMESTAMP_CFG]		= &reg_qtime_timestamp_cfg,
+	[TIMERS_XO_CLK_DIV_CFG]		= &reg_timers_xo_clk_div_cfg,
+	[TIMERS_PULSE_GRAN_CFG]		= &reg_timers_pulse_gran_cfg,
+	[SRC_RSRC_GRP_01_RSRC_TYPE]	= &reg_src_rsrc_grp_01_rsrc_type,
+	[SRC_RSRC_GRP_23_RSRC_TYPE]	= &reg_src_rsrc_grp_23_rsrc_type,
+	[SRC_RSRC_GRP_45_RSRC_TYPE]	= &reg_src_rsrc_grp_45_rsrc_type,
+	[SRC_RSRC_GRP_67_RSRC_TYPE]	= &reg_src_rsrc_grp_67_rsrc_type,
+	[DST_RSRC_GRP_01_RSRC_TYPE]	= &reg_dst_rsrc_grp_01_rsrc_type,
+	[DST_RSRC_GRP_23_RSRC_TYPE]	= &reg_dst_rsrc_grp_23_rsrc_type,
+	[DST_RSRC_GRP_45_RSRC_TYPE]	= &reg_dst_rsrc_grp_45_rsrc_type,
+	[DST_RSRC_GRP_67_RSRC_TYPE]	= &reg_dst_rsrc_grp_67_rsrc_type,
+	[ENDP_INIT_CFG]			= &reg_endp_init_cfg,
+	[ENDP_INIT_NAT]			= &reg_endp_init_nat,
+	[ENDP_INIT_HDR]			= &reg_endp_init_hdr,
+	[ENDP_INIT_HDR_EXT]		= &reg_endp_init_hdr_ext,
+	[ENDP_INIT_HDR_METADATA_MASK]	= &reg_endp_init_hdr_metadata_mask,
+	[ENDP_INIT_MODE]		= &reg_endp_init_mode,
+	[ENDP_INIT_AGGR]		= &reg_endp_init_aggr,
+	[ENDP_INIT_HOL_BLOCK_EN]	= &reg_endp_init_hol_block_en,
+	[ENDP_INIT_HOL_BLOCK_TIMER]	= &reg_endp_init_hol_block_timer,
+	[ENDP_INIT_DEAGGR]		= &reg_endp_init_deaggr,
+	[ENDP_INIT_RSRC_GRP]		= &reg_endp_init_rsrc_grp,
+	[ENDP_INIT_SEQ]			= &reg_endp_init_seq,
+	[ENDP_STATUS]			= &reg_endp_status,
+	[ENDP_FILTER_CACHE_CFG]		= &reg_endp_filter_cache_cfg,
+	[ENDP_ROUTER_CACHE_CFG]		= &reg_endp_router_cache_cfg,
+	[IPA_IRQ_STTS]			= &reg_ipa_irq_stts,
+	[IPA_IRQ_EN]			= &reg_ipa_irq_en,
+	[IPA_IRQ_CLR]			= &reg_ipa_irq_clr,
+	[IPA_IRQ_UC]			= &reg_ipa_irq_uc,
+	[IRQ_SUSPEND_INFO]		= &reg_irq_suspend_info,
+	[IRQ_SUSPEND_EN]		= &reg_irq_suspend_en,
+	[IRQ_SUSPEND_CLR]		= &reg_irq_suspend_clr,
+};
+
+const struct regs ipa_regs_v5_0 = {
+	.reg_count	= ARRAY_SIZE(reg_array),
+	.reg		= reg_array,
+};
diff --git a/drivers/net/ipvlan/ipvtap.c b/drivers/net/ipvlan/ipvtap.c
index dde272586e80..60944a4beada 100644
--- a/drivers/net/ipvlan/ipvtap.c
+++ b/drivers/net/ipvlan/ipvtap.c
@@ -38,7 +38,6 @@ static const void *ipvtap_net_namespace(const struct device *d)
 
 static struct class ipvtap_class = {
 	 .name = "ipvtap",
-	 .owner = THIS_MODULE,
 	 .ns_type = &net_ns_type_operations,
 	 .namespace = ipvtap_net_namespace,
 };
diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c
index 25616247d7a5..3427993f94f7 100644
--- a/drivers/net/macsec.c
+++ b/drivers/net/macsec.c
@@ -1021,8 +1021,12 @@ static enum rx_handler_result handle_not_macsec(struct sk_buff *skb)
 		 * the SecTAG, so we have to deduce which port to deliver to.
 		 */
 		if (macsec_is_offloaded(macsec) && netif_running(ndev)) {
-			if (md_dst && md_dst->type == METADATA_MACSEC &&
-			    (!find_rx_sc(&macsec->secy, md_dst->u.macsec_info.sci)))
+			struct macsec_rx_sc *rx_sc = NULL;
+
+			if (md_dst && md_dst->type == METADATA_MACSEC)
+				rx_sc = find_rx_sc(&macsec->secy, md_dst->u.macsec_info.sci);
+
+			if (md_dst && md_dst->type == METADATA_MACSEC && !rx_sc)
 				continue;
 
 			if (ether_addr_equal_64bits(hdr->h_dest,
@@ -1047,7 +1051,13 @@ static enum rx_handler_result handle_not_macsec(struct sk_buff *skb)
 					nskb->pkt_type = PACKET_MULTICAST;
 
 				__netif_rx(nskb);
+			} else if (rx_sc || ndev->flags & IFF_PROMISC) {
+				skb->dev = ndev;
+				skb->pkt_type = PACKET_HOST;
+				ret = RX_HANDLER_ANOTHER;
+				goto out;
 			}
+
 			continue;
 		}
 
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index 99a971929c8e..4a53debf9d7c 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -47,9 +47,11 @@ struct macvlan_port {
 	struct sk_buff_head	bc_queue;
 	struct work_struct	bc_work;
 	u32			bc_queue_len_used;
+	int			bc_cutoff;
 	u32			flags;
 	int			count;
 	struct hlist_head	vlan_source_hash[MACVLAN_HASH_SIZE];
+	DECLARE_BITMAP(bc_filter, MACVLAN_MC_FILTER_SZ);
 	DECLARE_BITMAP(mc_filter, MACVLAN_MC_FILTER_SZ);
 	unsigned char           perm_addr[ETH_ALEN];
 };
@@ -291,6 +293,31 @@ static void macvlan_broadcast(struct sk_buff *skb,
 	}
 }
 
+static void macvlan_multicast_rx(const struct macvlan_port *port,
+				 const struct macvlan_dev *src,
+				 struct sk_buff *skb)
+{
+	if (!src)
+		/* frame comes from an external address */
+		macvlan_broadcast(skb, port, NULL,
+				  MACVLAN_MODE_PRIVATE |
+				  MACVLAN_MODE_VEPA    |
+				  MACVLAN_MODE_PASSTHRU|
+				  MACVLAN_MODE_BRIDGE);
+	else if (src->mode == MACVLAN_MODE_VEPA)
+		/* flood to everyone except source */
+		macvlan_broadcast(skb, port, src->dev,
+				  MACVLAN_MODE_VEPA |
+				  MACVLAN_MODE_BRIDGE);
+	else
+		/*
+		 * flood only to VEPA ports, bridge ports
+		 * already saw the frame on the way out.
+		 */
+		macvlan_broadcast(skb, port, src->dev,
+				  MACVLAN_MODE_VEPA);
+}
+
 static void macvlan_process_broadcast(struct work_struct *w)
 {
 	struct macvlan_port *port = container_of(w, struct macvlan_port,
@@ -308,27 +335,7 @@ static void macvlan_process_broadcast(struct work_struct *w)
 		const struct macvlan_dev *src = MACVLAN_SKB_CB(skb)->src;
 
 		rcu_read_lock();
-
-		if (!src)
-			/* frame comes from an external address */
-			macvlan_broadcast(skb, port, NULL,
-					  MACVLAN_MODE_PRIVATE |
-					  MACVLAN_MODE_VEPA    |
-					  MACVLAN_MODE_PASSTHRU|
-					  MACVLAN_MODE_BRIDGE);
-		else if (src->mode == MACVLAN_MODE_VEPA)
-			/* flood to everyone except source */
-			macvlan_broadcast(skb, port, src->dev,
-					  MACVLAN_MODE_VEPA |
-					  MACVLAN_MODE_BRIDGE);
-		else
-			/*
-			 * flood only to VEPA ports, bridge ports
-			 * already saw the frame on the way out.
-			 */
-			macvlan_broadcast(skb, port, src->dev,
-					  MACVLAN_MODE_VEPA);
-
+		macvlan_multicast_rx(port, src, skb);
 		rcu_read_unlock();
 
 		if (src)
@@ -476,8 +483,10 @@ static rx_handler_result_t macvlan_handle_frame(struct sk_buff **pskb)
 		}
 
 		hash = mc_hash(NULL, eth->h_dest);
-		if (test_bit(hash, port->mc_filter))
+		if (test_bit(hash, port->bc_filter))
 			macvlan_broadcast_enqueue(port, src, skb);
+		else if (test_bit(hash, port->mc_filter))
+			macvlan_multicast_rx(port, src, skb);
 
 		return RX_HANDLER_PASS;
 	}
@@ -780,16 +789,19 @@ static void macvlan_change_rx_flags(struct net_device *dev, int change)
 
 static void macvlan_compute_filter(unsigned long *mc_filter,
 				   struct net_device *dev,
-				   struct macvlan_dev *vlan)
+				   struct macvlan_dev *vlan, int cutoff)
 {
 	if (dev->flags & (IFF_PROMISC | IFF_ALLMULTI)) {
 		bitmap_fill(mc_filter, MACVLAN_MC_FILTER_SZ);
 	} else {
-		struct netdev_hw_addr *ha;
 		DECLARE_BITMAP(filter, MACVLAN_MC_FILTER_SZ);
+		struct netdev_hw_addr *ha;
 
 		bitmap_zero(filter, MACVLAN_MC_FILTER_SZ);
 		netdev_for_each_mc_addr(ha, dev) {
+			if (!vlan && ha->synced <= cutoff)
+				continue;
+
 			__set_bit(mc_hash(vlan, ha->addr), filter);
 		}
 
@@ -799,11 +811,22 @@ static void macvlan_compute_filter(unsigned long *mc_filter,
 	}
 }
 
+static void macvlan_recompute_bc_filter(struct macvlan_dev *vlan)
+{
+	if (vlan->port->bc_cutoff < 0) {
+		bitmap_zero(vlan->port->bc_filter, MACVLAN_MC_FILTER_SZ);
+		return;
+	}
+
+	macvlan_compute_filter(vlan->port->bc_filter, vlan->lowerdev, NULL,
+			       vlan->port->bc_cutoff);
+}
+
 static void macvlan_set_mac_lists(struct net_device *dev)
 {
 	struct macvlan_dev *vlan = netdev_priv(dev);
 
-	macvlan_compute_filter(vlan->mc_filter, dev, vlan);
+	macvlan_compute_filter(vlan->mc_filter, dev, vlan, 0);
 
 	dev_uc_sync(vlan->lowerdev, dev);
 	dev_mc_sync(vlan->lowerdev, dev);
@@ -821,7 +844,18 @@ static void macvlan_set_mac_lists(struct net_device *dev)
 	 * The solution is to maintain a list of broadcast addresses like
 	 * we do for uc/mc, if you care.
 	 */
-	macvlan_compute_filter(vlan->port->mc_filter, vlan->lowerdev, NULL);
+	macvlan_compute_filter(vlan->port->mc_filter, vlan->lowerdev, NULL,
+			       0);
+	macvlan_recompute_bc_filter(vlan);
+}
+
+static void update_port_bc_cutoff(struct macvlan_dev *vlan, int cutoff)
+{
+	if (vlan->port->bc_cutoff == cutoff)
+		return;
+
+	vlan->port->bc_cutoff = cutoff;
+	macvlan_recompute_bc_filter(vlan);
 }
 
 static int macvlan_change_mtu(struct net_device *dev, int new_mtu)
@@ -1236,6 +1270,7 @@ static int macvlan_port_create(struct net_device *dev)
 		INIT_HLIST_HEAD(&port->vlan_source_hash[i]);
 
 	port->bc_queue_len_used = 0;
+	port->bc_cutoff = 1;
 	skb_queue_head_init(&port->bc_queue);
 	INIT_WORK(&port->bc_work, macvlan_process_broadcast);
 
@@ -1509,6 +1544,10 @@ int macvlan_common_newlink(struct net *src_net, struct net_device *dev,
 	if (data && data[IFLA_MACVLAN_BC_QUEUE_LEN])
 		vlan->bc_queue_len_req = nla_get_u32(data[IFLA_MACVLAN_BC_QUEUE_LEN]);
 
+	if (data && data[IFLA_MACVLAN_BC_CUTOFF])
+		update_port_bc_cutoff(
+			vlan, nla_get_s32(data[IFLA_MACVLAN_BC_CUTOFF]));
+
 	err = register_netdevice(dev);
 	if (err < 0)
 		goto destroy_macvlan_port;
@@ -1605,6 +1644,10 @@ static int macvlan_changelink(struct net_device *dev,
 		update_port_bc_queue_len(vlan->port);
 	}
 
+	if (data && data[IFLA_MACVLAN_BC_CUTOFF])
+		update_port_bc_cutoff(
+			vlan, nla_get_s32(data[IFLA_MACVLAN_BC_CUTOFF]));
+
 	if (set_mode)
 		vlan->mode = mode;
 	if (data && data[IFLA_MACVLAN_MACADDR_MODE]) {
@@ -1685,6 +1728,9 @@ static int macvlan_fill_info(struct sk_buff *skb,
 		goto nla_put_failure;
 	if (nla_put_u32(skb, IFLA_MACVLAN_BC_QUEUE_LEN_USED, port->bc_queue_len_used))
 		goto nla_put_failure;
+	if (port->bc_cutoff != 1 &&
+	    nla_put_s32(skb, IFLA_MACVLAN_BC_CUTOFF, port->bc_cutoff))
+		goto nla_put_failure;
 	return 0;
 
 nla_put_failure:
diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c
index 031344239f27..bddcc127812e 100644
--- a/drivers/net/macvtap.c
+++ b/drivers/net/macvtap.c
@@ -43,7 +43,6 @@ static const void *macvtap_net_namespace(const struct device *d)
 
 static struct class macvtap_class = {
 	.name = "macvtap",
-	.owner = THIS_MODULE,
 	.ns_type = &net_ns_type_operations,
 	.namespace = macvtap_net_namespace,
 };
diff --git a/drivers/net/mdio/Kconfig b/drivers/net/mdio/Kconfig
index 90309980686e..9ff2e6f22f3f 100644
--- a/drivers/net/mdio/Kconfig
+++ b/drivers/net/mdio/Kconfig
@@ -65,6 +65,7 @@ config MDIO_ASPEED
 	tristate "ASPEED MDIO bus controller"
 	depends on ARCH_ASPEED || COMPILE_TEST
 	depends on OF_MDIO && HAS_IOMEM
+	depends on MDIO_DEVRES
 	help
 	  This module provides a driver for the independent MDIO bus
 	  controllers found in the ASPEED AST2600 SoC. This is a driver for the
@@ -170,6 +171,7 @@ config MDIO_IPQ4019
 	tristate "Qualcomm IPQ4019 MDIO interface support"
 	depends on HAS_IOMEM && OF_MDIO
 	depends on COMMON_CLK
+	depends on MDIO_DEVRES
 	help
 	  This driver supports the MDIO interface found in Qualcomm
 	  IPQ40xx, IPQ60xx, IPQ807x and IPQ50xx series Soc-s.
@@ -178,6 +180,7 @@ config MDIO_IPQ8064
 	tristate "Qualcomm IPQ8064 MDIO interface support"
 	depends on HAS_IOMEM && OF_MDIO
 	depends on MFD_SYSCON
+	depends on MDIO_DEVRES
 	help
 	  This driver supports the MDIO interface found in the network
 	  interface units of the IPQ8064 SoC
diff --git a/drivers/net/mdio/of_mdio.c b/drivers/net/mdio/of_mdio.c
index 1e46e39f5f46..7eb32ebb846d 100644
--- a/drivers/net/mdio/of_mdio.c
+++ b/drivers/net/mdio/of_mdio.c
@@ -131,7 +131,7 @@ bool of_mdiobus_child_is_phy(struct device_node *child)
 		return true;
 	}
 
-	if (!of_find_property(child, "compatible", NULL))
+	if (!of_property_present(child, "compatible"))
 		return true;
 
 	return false;
@@ -205,7 +205,7 @@ int __of_mdiobus_register(struct mii_bus *mdio, struct device_node *np,
 	/* auto scan for PHYs with empty reg property */
 	for_each_available_child_of_node(np, child) {
 		/* Skip PHYs with reg property set */
-		if (of_find_property(child, "reg", NULL))
+		if (of_property_present(child, "reg"))
 			continue;
 
 		for (addr = 0; addr < PHY_MAX_ADDR; addr++) {
diff --git a/drivers/net/netdevsim/bus.c b/drivers/net/netdevsim/bus.c
index 0052968e881e..0787ad252dd9 100644
--- a/drivers/net/netdevsim/bus.c
+++ b/drivers/net/netdevsim/bus.c
@@ -132,7 +132,7 @@ static struct nsim_bus_dev *
 nsim_bus_dev_new(unsigned int id, unsigned int port_count, unsigned int num_queues);
 
 static ssize_t
-new_device_store(struct bus_type *bus, const char *buf, size_t count)
+new_device_store(const struct bus_type *bus, const char *buf, size_t count)
 {
 	unsigned int id, port_count, num_queues;
 	struct nsim_bus_dev *nsim_bus_dev;
@@ -186,7 +186,7 @@ static BUS_ATTR_WO(new_device);
 static void nsim_bus_dev_del(struct nsim_bus_dev *nsim_bus_dev);
 
 static ssize_t
-del_device_store(struct bus_type *bus, const char *buf, size_t count)
+del_device_store(const struct bus_type *bus, const char *buf, size_t count)
 {
 	struct nsim_bus_dev *nsim_bus_dev, *tmp;
 	unsigned int id;
diff --git a/drivers/net/pcs/Kconfig b/drivers/net/pcs/Kconfig
index 6e7e6c346a3e..7c34fb7cbf7b 100644
--- a/drivers/net/pcs/Kconfig
+++ b/drivers/net/pcs/Kconfig
@@ -18,6 +18,13 @@ config PCS_LYNX
 	  This module provides helpers to phylink for managing the Lynx PCS
 	  which is part of the Layerscape and QorIQ Ethernet SERDES.
 
+config PCS_MTK_LYNXI
+	tristate
+	select REGMAP
+	help
+	  This module provides helpers to phylink for managing the LynxI PCS
+	  which is part of MediaTek's SoC and Ethernet switch ICs.
+
 config PCS_RZN1_MIIC
 	tristate "Renesas RZ/N1 MII converter"
 	depends on OF && (ARCH_RZN1 || COMPILE_TEST)
diff --git a/drivers/net/pcs/Makefile b/drivers/net/pcs/Makefile
index 4c780d8f2e98..9b9afd6b1c22 100644
--- a/drivers/net/pcs/Makefile
+++ b/drivers/net/pcs/Makefile
@@ -5,5 +5,6 @@ pcs_xpcs-$(CONFIG_PCS_XPCS)	:= pcs-xpcs.o pcs-xpcs-nxp.o
 
 obj-$(CONFIG_PCS_XPCS)		+= pcs_xpcs.o
 obj-$(CONFIG_PCS_LYNX)		+= pcs-lynx.o
+obj-$(CONFIG_PCS_MTK_LYNXI)	+= pcs-mtk-lynxi.o
 obj-$(CONFIG_PCS_RZN1_MIIC)	+= pcs-rzn1-miic.o
 obj-$(CONFIG_PCS_ALTERA_TSE)	+= pcs-altera-tse.o
diff --git a/drivers/net/pcs/pcs-lynx.c b/drivers/net/pcs/pcs-lynx.c
index 3903f3baba2b..622c3de3f3a8 100644
--- a/drivers/net/pcs/pcs-lynx.c
+++ b/drivers/net/pcs/pcs-lynx.c
@@ -112,11 +112,11 @@ static void lynx_pcs_get_state(struct phylink_pcs *pcs,
 	}
 
 	dev_dbg(&lynx->mdio->dev,
-		"mode=%s/%s/%s link=%u an_enabled=%u an_complete=%u\n",
+		"mode=%s/%s/%s link=%u an_complete=%u\n",
 		phy_modes(state->interface),
 		phy_speed_to_str(state->speed),
 		phy_duplex_to_str(state->duplex),
-		state->link, state->an_enabled, state->an_complete);
+		state->link, state->an_complete);
 }
 
 static int lynx_pcs_config_giga(struct mdio_device *pcs, unsigned int mode,
diff --git a/drivers/net/pcs/pcs-mtk-lynxi.c b/drivers/net/pcs/pcs-mtk-lynxi.c
new file mode 100644
index 000000000000..888452325edc
--- /dev/null
+++ b/drivers/net/pcs/pcs-mtk-lynxi.c
@@ -0,0 +1,305 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018-2019 MediaTek Inc.
+/* A library for MediaTek SGMII circuit
+ *
+ * Author: Sean Wang <sean.wang@mediatek.com>
+ * Author: Alexander Couzens <lynxis@fe80.eu>
+ * Author: Daniel Golle <daniel@makrotopia.org>
+ *
+ */
+
+#include <linux/mdio.h>
+#include <linux/of.h>
+#include <linux/pcs/pcs-mtk-lynxi.h>
+#include <linux/phylink.h>
+#include <linux/regmap.h>
+
+/* SGMII subsystem config registers */
+/* BMCR (low 16) BMSR (high 16) */
+#define SGMSYS_PCS_CONTROL_1		0x0
+#define SGMII_BMCR			GENMASK(15, 0)
+#define SGMII_BMSR			GENMASK(31, 16)
+
+#define SGMSYS_PCS_DEVICE_ID		0x4
+#define SGMII_LYNXI_DEV_ID		0x4d544950
+
+#define SGMSYS_PCS_ADVERTISE		0x8
+#define SGMII_ADVERTISE			GENMASK(15, 0)
+#define SGMII_LPA			GENMASK(31, 16)
+
+#define SGMSYS_PCS_SCRATCH		0x14
+#define SGMII_DEV_VERSION		GENMASK(31, 16)
+
+/* Register to programmable link timer, the unit in 2 * 8ns */
+#define SGMSYS_PCS_LINK_TIMER		0x18
+#define SGMII_LINK_TIMER_MASK		GENMASK(19, 0)
+#define SGMII_LINK_TIMER_VAL(ns)	FIELD_PREP(SGMII_LINK_TIMER_MASK, \
+						   ((ns) / 2 / 8))
+
+/* Register to control remote fault */
+#define SGMSYS_SGMII_MODE		0x20
+#define SGMII_IF_MODE_SGMII		BIT(0)
+#define SGMII_SPEED_DUPLEX_AN		BIT(1)
+#define SGMII_SPEED_MASK		GENMASK(3, 2)
+#define SGMII_SPEED_10			FIELD_PREP(SGMII_SPEED_MASK, 0)
+#define SGMII_SPEED_100			FIELD_PREP(SGMII_SPEED_MASK, 1)
+#define SGMII_SPEED_1000		FIELD_PREP(SGMII_SPEED_MASK, 2)
+#define SGMII_DUPLEX_HALF		BIT(4)
+#define SGMII_REMOTE_FAULT_DIS		BIT(8)
+
+/* Register to reset SGMII design */
+#define SGMSYS_RESERVED_0		0x34
+#define SGMII_SW_RESET			BIT(0)
+
+/* Register to set SGMII speed, ANA RG_ Control Signals III */
+#define SGMII_PHY_SPEED_MASK		GENMASK(3, 2)
+#define SGMII_PHY_SPEED_1_25G		FIELD_PREP(SGMII_PHY_SPEED_MASK, 0)
+#define SGMII_PHY_SPEED_3_125G		FIELD_PREP(SGMII_PHY_SPEED_MASK, 1)
+
+/* Register to power up QPHY */
+#define SGMSYS_QPHY_PWR_STATE_CTRL	0xe8
+#define	SGMII_PHYA_PWD			BIT(4)
+
+/* Register to QPHY wrapper control */
+#define SGMSYS_QPHY_WRAP_CTRL		0xec
+#define SGMII_PN_SWAP_MASK		GENMASK(1, 0)
+#define SGMII_PN_SWAP_TX_RX		(BIT(0) | BIT(1))
+
+/* struct mtk_pcs_lynxi -  This structure holds each sgmii regmap andassociated
+ *                         data
+ * @regmap:                The register map pointing at the range used to setup
+ *                         SGMII modes
+ * @dev:                   Pointer to device owning the PCS
+ * @ana_rgc3:              The offset of register ANA_RGC3 relative to regmap
+ * @interface:             Currently configured interface mode
+ * @pcs:                   Phylink PCS structure
+ * @flags:                 Flags indicating hardware properties
+ */
+struct mtk_pcs_lynxi {
+	struct regmap		*regmap;
+	u32			ana_rgc3;
+	phy_interface_t		interface;
+	struct			phylink_pcs pcs;
+	u32			flags;
+};
+
+static struct mtk_pcs_lynxi *pcs_to_mtk_pcs_lynxi(struct phylink_pcs *pcs)
+{
+	return container_of(pcs, struct mtk_pcs_lynxi, pcs);
+}
+
+static void mtk_pcs_lynxi_get_state(struct phylink_pcs *pcs,
+				    struct phylink_link_state *state)
+{
+	struct mtk_pcs_lynxi *mpcs = pcs_to_mtk_pcs_lynxi(pcs);
+	unsigned int bm, adv;
+
+	/* Read the BMSR and LPA */
+	regmap_read(mpcs->regmap, SGMSYS_PCS_CONTROL_1, &bm);
+	regmap_read(mpcs->regmap, SGMSYS_PCS_ADVERTISE, &adv);
+
+	phylink_mii_c22_pcs_decode_state(state, FIELD_GET(SGMII_BMSR, bm),
+					 FIELD_GET(SGMII_LPA, adv));
+}
+
+static int mtk_pcs_lynxi_config(struct phylink_pcs *pcs, unsigned int mode,
+				phy_interface_t interface,
+				const unsigned long *advertising,
+				bool permit_pause_to_mac)
+{
+	struct mtk_pcs_lynxi *mpcs = pcs_to_mtk_pcs_lynxi(pcs);
+	bool mode_changed = false, changed, use_an;
+	unsigned int rgc3, sgm_mode, bmcr;
+	int advertise, link_timer;
+
+	advertise = phylink_mii_c22_pcs_encode_advertisement(interface,
+							     advertising);
+	if (advertise < 0)
+		return advertise;
+
+	/* Clearing IF_MODE_BIT0 switches the PCS to BASE-X mode, and
+	 * we assume that fixes it's speed at bitrate = line rate (in
+	 * other words, 1000Mbps or 2500Mbps).
+	 */
+	if (interface == PHY_INTERFACE_MODE_SGMII) {
+		sgm_mode = SGMII_IF_MODE_SGMII;
+		if (phylink_autoneg_inband(mode)) {
+			sgm_mode |= SGMII_REMOTE_FAULT_DIS |
+				    SGMII_SPEED_DUPLEX_AN;
+			use_an = true;
+		} else {
+			use_an = false;
+		}
+	} else if (phylink_autoneg_inband(mode)) {
+		/* 1000base-X or 2500base-X autoneg */
+		sgm_mode = SGMII_REMOTE_FAULT_DIS;
+		use_an = linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
+					   advertising);
+	} else {
+		/* 1000base-X or 2500base-X without autoneg */
+		sgm_mode = 0;
+		use_an = false;
+	}
+
+	if (use_an)
+		bmcr = BMCR_ANENABLE;
+	else
+		bmcr = 0;
+
+	if (mpcs->interface != interface) {
+		link_timer = phylink_get_link_timer_ns(interface);
+		if (link_timer < 0)
+			return link_timer;
+
+		/* PHYA power down */
+		regmap_set_bits(mpcs->regmap, SGMSYS_QPHY_PWR_STATE_CTRL,
+				SGMII_PHYA_PWD);
+
+		/* Reset SGMII PCS state */
+		regmap_set_bits(mpcs->regmap, SGMSYS_RESERVED_0,
+				SGMII_SW_RESET);
+
+		if (mpcs->flags & MTK_SGMII_FLAG_PN_SWAP)
+			regmap_update_bits(mpcs->regmap, SGMSYS_QPHY_WRAP_CTRL,
+					   SGMII_PN_SWAP_MASK,
+					   SGMII_PN_SWAP_TX_RX);
+
+		if (interface == PHY_INTERFACE_MODE_2500BASEX)
+			rgc3 = SGMII_PHY_SPEED_3_125G;
+		else
+			rgc3 = SGMII_PHY_SPEED_1_25G;
+
+		/* Configure the underlying interface speed */
+		regmap_update_bits(mpcs->regmap, mpcs->ana_rgc3,
+				   SGMII_PHY_SPEED_MASK, rgc3);
+
+		/* Setup the link timer */
+		regmap_write(mpcs->regmap, SGMSYS_PCS_LINK_TIMER,
+			     SGMII_LINK_TIMER_VAL(link_timer));
+
+		mpcs->interface = interface;
+		mode_changed = true;
+	}
+
+	/* Update the advertisement, noting whether it has changed */
+	regmap_update_bits_check(mpcs->regmap, SGMSYS_PCS_ADVERTISE,
+				 SGMII_ADVERTISE, advertise, &changed);
+
+	/* Update the sgmsys mode register */
+	regmap_update_bits(mpcs->regmap, SGMSYS_SGMII_MODE,
+			   SGMII_REMOTE_FAULT_DIS | SGMII_SPEED_DUPLEX_AN |
+			   SGMII_IF_MODE_SGMII, sgm_mode);
+
+	/* Update the BMCR */
+	regmap_update_bits(mpcs->regmap, SGMSYS_PCS_CONTROL_1,
+			   BMCR_ANENABLE, bmcr);
+
+	/* Release PHYA power down state
+	 * Only removing bit SGMII_PHYA_PWD isn't enough.
+	 * There are cases when the SGMII_PHYA_PWD register contains 0x9 which
+	 * prevents SGMII from working. The SGMII still shows link but no traffic
+	 * can flow. Writing 0x0 to the PHYA_PWD register fix the issue. 0x0 was
+	 * taken from a good working state of the SGMII interface.
+	 * Unknown how much the QPHY needs but it is racy without a sleep.
+	 * Tested on mt7622 & mt7986.
+	 */
+	usleep_range(50, 100);
+	regmap_write(mpcs->regmap, SGMSYS_QPHY_PWR_STATE_CTRL, 0);
+
+	return changed || mode_changed;
+}
+
+static void mtk_pcs_lynxi_restart_an(struct phylink_pcs *pcs)
+{
+	struct mtk_pcs_lynxi *mpcs = pcs_to_mtk_pcs_lynxi(pcs);
+
+	regmap_set_bits(mpcs->regmap, SGMSYS_PCS_CONTROL_1, BMCR_ANRESTART);
+}
+
+static void mtk_pcs_lynxi_link_up(struct phylink_pcs *pcs, unsigned int mode,
+				  phy_interface_t interface, int speed,
+				  int duplex)
+{
+	struct mtk_pcs_lynxi *mpcs = pcs_to_mtk_pcs_lynxi(pcs);
+	unsigned int sgm_mode;
+
+	if (!phylink_autoneg_inband(mode)) {
+		/* Force the speed and duplex setting */
+		if (speed == SPEED_10)
+			sgm_mode = SGMII_SPEED_10;
+		else if (speed == SPEED_100)
+			sgm_mode = SGMII_SPEED_100;
+		else
+			sgm_mode = SGMII_SPEED_1000;
+
+		if (duplex != DUPLEX_FULL)
+			sgm_mode |= SGMII_DUPLEX_HALF;
+
+		regmap_update_bits(mpcs->regmap, SGMSYS_SGMII_MODE,
+				   SGMII_DUPLEX_HALF | SGMII_SPEED_MASK,
+				   sgm_mode);
+	}
+}
+
+static const struct phylink_pcs_ops mtk_pcs_lynxi_ops = {
+	.pcs_get_state = mtk_pcs_lynxi_get_state,
+	.pcs_config = mtk_pcs_lynxi_config,
+	.pcs_an_restart = mtk_pcs_lynxi_restart_an,
+	.pcs_link_up = mtk_pcs_lynxi_link_up,
+};
+
+struct phylink_pcs *mtk_pcs_lynxi_create(struct device *dev,
+					 struct regmap *regmap, u32 ana_rgc3,
+					 u32 flags)
+{
+	struct mtk_pcs_lynxi *mpcs;
+	u32 id, ver;
+	int ret;
+
+	ret = regmap_read(regmap, SGMSYS_PCS_DEVICE_ID, &id);
+	if (ret < 0)
+		return NULL;
+
+	if (id != SGMII_LYNXI_DEV_ID) {
+		dev_err(dev, "unknown PCS device id %08x\n", id);
+		return NULL;
+	}
+
+	ret = regmap_read(regmap, SGMSYS_PCS_SCRATCH, &ver);
+	if (ret < 0)
+		return NULL;
+
+	ver = FIELD_GET(SGMII_DEV_VERSION, ver);
+	if (ver != 0x1) {
+		dev_err(dev, "unknown PCS device version %04x\n", ver);
+		return NULL;
+	}
+
+	dev_dbg(dev, "MediaTek LynxI SGMII PCS (id 0x%08x, ver 0x%04x)\n", id,
+		ver);
+
+	mpcs = kzalloc(sizeof(*mpcs), GFP_KERNEL);
+	if (!mpcs)
+		return NULL;
+
+	mpcs->ana_rgc3 = ana_rgc3;
+	mpcs->regmap = regmap;
+	mpcs->flags = flags;
+	mpcs->pcs.ops = &mtk_pcs_lynxi_ops;
+	mpcs->pcs.poll = true;
+	mpcs->interface = PHY_INTERFACE_MODE_NA;
+
+	return &mpcs->pcs;
+}
+EXPORT_SYMBOL(mtk_pcs_lynxi_create);
+
+void mtk_pcs_lynxi_destroy(struct phylink_pcs *pcs)
+{
+	if (!pcs)
+		return;
+
+	kfree(pcs_to_mtk_pcs_lynxi(pcs));
+}
+EXPORT_SYMBOL(mtk_pcs_lynxi_destroy);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/pcs/pcs-xpcs.c b/drivers/net/pcs/pcs-xpcs.c
index bc428a816719..539cd43eae8d 100644
--- a/drivers/net/pcs/pcs-xpcs.c
+++ b/drivers/net/pcs/pcs-xpcs.c
@@ -321,7 +321,7 @@ static int xpcs_read_fault_c73(struct dw_xpcs *xpcs,
 	return 0;
 }
 
-static int xpcs_read_link_c73(struct dw_xpcs *xpcs, bool an)
+static int xpcs_read_link_c73(struct dw_xpcs *xpcs)
 {
 	bool link = true;
 	int ret;
@@ -333,15 +333,6 @@ static int xpcs_read_link_c73(struct dw_xpcs *xpcs, bool an)
 	if (!(ret & MDIO_STAT1_LSTATUS))
 		link = false;
 
-	if (an) {
-		ret = xpcs_read(xpcs, MDIO_MMD_AN, MDIO_STAT1);
-		if (ret < 0)
-			return ret;
-
-		if (!(ret & MDIO_STAT1_LSTATUS))
-			link = false;
-	}
-
 	return link;
 }
 
@@ -932,10 +923,11 @@ static int xpcs_get_state_c73(struct dw_xpcs *xpcs,
 			      struct phylink_link_state *state,
 			      const struct xpcs_compat *compat)
 {
+	bool an_enabled;
 	int ret;
 
 	/* Link needs to be read first ... */
-	state->link = xpcs_read_link_c73(xpcs, state->an_enabled) > 0 ? 1 : 0;
+	state->link = xpcs_read_link_c73(xpcs) > 0 ? 1 : 0;
 
 	/* ... and then we check the faults. */
 	ret = xpcs_read_fault_c73(xpcs, state);
@@ -949,11 +941,13 @@ static int xpcs_get_state_c73(struct dw_xpcs *xpcs,
 		return xpcs_do_config(xpcs, state->interface, MLO_AN_INBAND, NULL);
 	}
 
-	if (state->an_enabled && xpcs_aneg_done_c73(xpcs, state, compat)) {
+	an_enabled = linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
+				       state->advertising);
+	if (an_enabled && xpcs_aneg_done_c73(xpcs, state, compat)) {
 		state->an_complete = true;
 		xpcs_read_lpa_c73(xpcs, state);
 		xpcs_resolve_lpa_c73(xpcs, state);
-	} else if (state->an_enabled) {
+	} else if (an_enabled) {
 		state->link = 0;
 	} else if (state->link) {
 		xpcs_resolve_pma(xpcs, state);
@@ -1008,7 +1002,8 @@ static int xpcs_get_state_c37_1000basex(struct dw_xpcs *xpcs,
 {
 	int lpa, bmsr;
 
-	if (state->an_enabled) {
+	if (linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
+			      state->advertising)) {
 		/* Reset link state */
 		state->link = false;
 
diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig
index 54874555c921..93b8efc79227 100644
--- a/drivers/net/phy/Kconfig
+++ b/drivers/net/phy/Kconfig
@@ -44,6 +44,12 @@ config LED_TRIGGER_PHY
 		<Speed in megabits>Mbps OR <Speed in gigabits>Gbps OR link
 		for any speed known to the PHY.
 
+config PHYLIB_LEDS
+	def_bool OF
+	depends on LEDS_CLASS=y || LEDS_CLASS=PHYLIB
+	help
+	  When LED class support is enabled, phylib can automatically
+	  probe LED setting from device tree.
 
 config FIXED_PHY
 	tristate "MDIO Bus/PHY emulation with fixed speed/link PHYs"
@@ -70,6 +76,7 @@ config AMD_PHY
 config MESON_GXL_PHY
 	tristate "Amlogic Meson GXL Internal PHY"
 	depends on ARCH_MESON || COMPILE_TEST
+	select SMSC_PHY
 	help
 	  Currently has a driver for the Amlogic Meson GXL Internal PHY
 
@@ -235,6 +242,11 @@ config MICREL_PHY
 	help
 	  Supports the KSZ9021, VSC8201, KS8001 PHYs.
 
+config MICROCHIP_T1S_PHY
+	tristate "Microchip 10BASE-T1S Ethernet PHY"
+	help
+	  Currently supports the LAN8670, LAN8671, LAN8672
+
 config MICROCHIP_PHY
 	tristate "Microchip PHYs"
 	help
@@ -264,6 +276,12 @@ config NATIONAL_PHY
 	help
 	  Currently supports the DP83865 PHY.
 
+config NXP_CBTX_PHY
+	tristate "NXP 100BASE-TX PHYs"
+	help
+	  Support the 100BASE-TX PHY integrated on the SJA1110 automotive
+	  switch family.
+
 config NXP_C45_TJA11XX_PHY
 	tristate "NXP C45 TJA11XX PHYs"
 	depends on PTP_1588_CLOCK_OPTIONAL
diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile
index b5138066ba04..f289ab16a1da 100644
--- a/drivers/net/phy/Makefile
+++ b/drivers/net/phy/Makefile
@@ -74,11 +74,13 @@ obj-$(CONFIG_MICREL_KS8995MA)	+= spi_ks8995.o
 obj-$(CONFIG_MICREL_PHY)	+= micrel.o
 obj-$(CONFIG_MICROCHIP_PHY)	+= microchip.o
 obj-$(CONFIG_MICROCHIP_T1_PHY)	+= microchip_t1.o
+obj-$(CONFIG_MICROCHIP_T1S_PHY) += microchip_t1s.o
 obj-$(CONFIG_MICROSEMI_PHY)	+= mscc/
 obj-$(CONFIG_MOTORCOMM_PHY)	+= motorcomm.o
 obj-$(CONFIG_NATIONAL_PHY)	+= national.o
 obj-$(CONFIG_NCN26000_PHY)	+= ncn26000.o
 obj-$(CONFIG_NXP_C45_TJA11XX_PHY)	+= nxp-c45-tja11xx.o
+obj-$(CONFIG_NXP_CBTX_PHY)	+= nxp-cbtx.o
 obj-$(CONFIG_NXP_TJA11XX_PHY)	+= nxp-tja11xx.o
 obj-$(CONFIG_QSEMI_PHY)		+= qsemi.o
 obj-$(CONFIG_REALTEK_PHY)	+= realtek.o
diff --git a/drivers/net/phy/aquantia_hwmon.c b/drivers/net/phy/aquantia_hwmon.c
index 19c4c280a6cd..0da451e46f69 100644
--- a/drivers/net/phy/aquantia_hwmon.c
+++ b/drivers/net/phy/aquantia_hwmon.c
@@ -210,7 +210,7 @@ static const struct hwmon_channel_info aqr_hwmon_temp = {
 	.config = aqr_hwmon_temp_config,
 };
 
-static const struct hwmon_channel_info *aqr_hwmon_info[] = {
+static const struct hwmon_channel_info * const aqr_hwmon_info[] = {
 	&aqr_hwmon_chip,
 	&aqr_hwmon_temp,
 	NULL,
diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c
index 22f4458274aa..656136628ffd 100644
--- a/drivers/net/phy/at803x.c
+++ b/drivers/net/phy/at803x.c
@@ -13,12 +13,11 @@
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
 #include <linux/ethtool_netlink.h>
-#include <linux/of_gpio.h>
 #include <linux/bitfield.h>
-#include <linux/gpio/consumer.h>
 #include <linux/regulator/of_regulator.h>
 #include <linux/regulator/driver.h>
 #include <linux/regulator/consumer.h>
+#include <linux/of.h>
 #include <linux/phylink.h>
 #include <linux/sfp.h>
 #include <dt-bindings/net/qca-ar803x.h>
diff --git a/drivers/net/phy/bcm54140.c b/drivers/net/phy/bcm54140.c
index d8f3024860dc..d43076592f81 100644
--- a/drivers/net/phy/bcm54140.c
+++ b/drivers/net/phy/bcm54140.c
@@ -364,7 +364,7 @@ static int bcm54140_hwmon_write(struct device *dev,
 	}
 }
 
-static const struct hwmon_channel_info *bcm54140_hwmon_info[] = {
+static const struct hwmon_channel_info * const bcm54140_hwmon_info[] = {
 	HWMON_CHANNEL_INFO(temp,
 			   HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX |
 			   HWMON_T_ALARM),
diff --git a/drivers/net/phy/bcm7xxx.c b/drivers/net/phy/bcm7xxx.c
index 75593e7d1118..06be71ecd2f8 100644
--- a/drivers/net/phy/bcm7xxx.c
+++ b/drivers/net/phy/bcm7xxx.c
@@ -45,7 +45,6 @@
 
 struct bcm7xxx_phy_priv {
 	u64	*stats;
-	struct clk *clk;
 };
 
 static int bcm7xxx_28nm_d0_afe_config_init(struct phy_device *phydev)
@@ -811,6 +810,7 @@ static void bcm7xxx_28nm_get_phy_stats(struct phy_device *phydev,
 static int bcm7xxx_28nm_probe(struct phy_device *phydev)
 {
 	struct bcm7xxx_phy_priv *priv;
+	struct clk *clk;
 	int ret = 0;
 
 	priv = devm_kzalloc(&phydev->mdio.dev, sizeof(*priv), GFP_KERNEL);
@@ -825,13 +825,9 @@ static int bcm7xxx_28nm_probe(struct phy_device *phydev)
 	if (!priv->stats)
 		return -ENOMEM;
 
-	priv->clk = devm_clk_get_optional(&phydev->mdio.dev, NULL);
-	if (IS_ERR(priv->clk))
-		return PTR_ERR(priv->clk);
-
-	ret = clk_prepare_enable(priv->clk);
-	if (ret)
-		return ret;
+	clk = devm_clk_get_optional_enabled(&phydev->mdio.dev, NULL);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
 
 	/* Dummy read to a register to workaround an issue upon reset where the
 	 * internal inverter may not allow the first MDIO transaction to pass
@@ -844,13 +840,6 @@ static int bcm7xxx_28nm_probe(struct phy_device *phydev)
 	return ret;
 }
 
-static void bcm7xxx_28nm_remove(struct phy_device *phydev)
-{
-	struct bcm7xxx_phy_priv *priv = phydev->priv;
-
-	clk_disable_unprepare(priv->clk);
-}
-
 #define BCM7XXX_28NM_GPHY(_oui, _name)					\
 {									\
 	.phy_id		= (_oui),					\
@@ -866,7 +855,6 @@ static void bcm7xxx_28nm_remove(struct phy_device *phydev)
 	.get_strings	= bcm_phy_get_strings,				\
 	.get_stats	= bcm7xxx_28nm_get_phy_stats,			\
 	.probe		= bcm7xxx_28nm_probe,				\
-	.remove		= bcm7xxx_28nm_remove,				\
 }
 
 #define BCM7XXX_28NM_EPHY(_oui, _name)					\
@@ -882,7 +870,6 @@ static void bcm7xxx_28nm_remove(struct phy_device *phydev)
 	.get_strings	= bcm_phy_get_strings,				\
 	.get_stats	= bcm7xxx_28nm_get_phy_stats,			\
 	.probe		= bcm7xxx_28nm_probe,				\
-	.remove		= bcm7xxx_28nm_remove,				\
 	.read_mmd	= bcm7xxx_28nm_ephy_read_mmd,			\
 	.write_mmd	= bcm7xxx_28nm_ephy_write_mmd,			\
 }
@@ -908,7 +895,6 @@ static void bcm7xxx_28nm_remove(struct phy_device *phydev)
 	/* PHY_BASIC_FEATURES */					\
 	.flags		= PHY_IS_INTERNAL,				\
 	.probe		= bcm7xxx_28nm_probe,				\
-	.remove		= bcm7xxx_28nm_remove,				\
 	.config_init	= bcm7xxx_16nm_ephy_config_init,		\
 	.config_aneg	= genphy_config_aneg,				\
 	.read_status	= genphy_read_status,				\
diff --git a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c
index 89cd821f1f46..d75f526a20a4 100644
--- a/drivers/net/phy/dp83867.c
+++ b/drivers/net/phy/dp83867.c
@@ -26,6 +26,8 @@
 #define MII_DP83867_MICR	0x12
 #define MII_DP83867_ISR		0x13
 #define DP83867_CFG2		0x14
+#define DP83867_LEDCR1		0x18
+#define DP83867_LEDCR2		0x19
 #define DP83867_CFG3		0x1e
 #define DP83867_CTRL		0x1f
 
@@ -150,6 +152,12 @@
 /* FLD_THR_CFG */
 #define DP83867_FLD_THR_CFG_ENERGY_LOST_THR_MASK	0x7
 
+#define DP83867_LED_COUNT	4
+
+/* LED_DRV bits */
+#define DP83867_LED_DRV_EN(x)	BIT((x) * 4)
+#define DP83867_LED_DRV_VAL(x)	BIT((x) * 4 + 1)
+
 enum {
 	DP83867_PORT_MIRROING_KEEP,
 	DP83867_PORT_MIRROING_EN,
@@ -468,8 +476,7 @@ static int dp83867_set_tunable(struct phy_device *phydev,
 
 static int dp83867_config_port_mirroring(struct phy_device *phydev)
 {
-	struct dp83867_private *dp83867 =
-		(struct dp83867_private *)phydev->priv;
+	struct dp83867_private *dp83867 = phydev->priv;
 
 	if (dp83867->port_mirroring == DP83867_PORT_MIRROING_EN)
 		phy_set_bits_mmd(phydev, DP83867_DEVADDR, DP83867_CFG4,
@@ -693,6 +700,30 @@ static int dp83867_of_init(struct phy_device *phydev)
 }
 #endif /* CONFIG_OF_MDIO */
 
+static int dp83867_suspend(struct phy_device *phydev)
+{
+	/* Disable PHY Interrupts */
+	if (phy_interrupt_is_valid(phydev)) {
+		phydev->interrupts = PHY_INTERRUPT_DISABLED;
+		dp83867_config_intr(phydev);
+	}
+
+	return genphy_suspend(phydev);
+}
+
+static int dp83867_resume(struct phy_device *phydev)
+{
+	/* Enable PHY Interrupts */
+	if (phy_interrupt_is_valid(phydev)) {
+		phydev->interrupts = PHY_INTERRUPT_ENABLED;
+		dp83867_config_intr(phydev);
+	}
+
+	genphy_resume(phydev);
+
+	return 0;
+}
+
 static int dp83867_probe(struct phy_device *phydev)
 {
 	struct dp83867_private *dp83867;
@@ -946,6 +977,27 @@ static int dp83867_loopback(struct phy_device *phydev, bool enable)
 			  enable ? BMCR_LOOPBACK : 0);
 }
 
+static int
+dp83867_led_brightness_set(struct phy_device *phydev,
+			   u8 index, enum led_brightness brightness)
+{
+	u32 val;
+
+	if (index >= DP83867_LED_COUNT)
+		return -EINVAL;
+
+	/* DRV_EN==1: output is DRV_VAL */
+	val = DP83867_LED_DRV_EN(index);
+
+	if (brightness)
+		val |= DP83867_LED_DRV_VAL(index);
+
+	return phy_modify(phydev, DP83867_LEDCR2,
+			  DP83867_LED_DRV_VAL(index) |
+			  DP83867_LED_DRV_EN(index),
+			  val);
+}
+
 static struct phy_driver dp83867_driver[] = {
 	{
 		.phy_id		= DP83867_PHY_ID,
@@ -968,11 +1020,13 @@ static struct phy_driver dp83867_driver[] = {
 		.config_intr	= dp83867_config_intr,
 		.handle_interrupt = dp83867_handle_interrupt,
 
-		.suspend	= genphy_suspend,
-		.resume		= genphy_resume,
+		.suspend	= dp83867_suspend,
+		.resume		= dp83867_resume,
 
 		.link_change_notify = dp83867_link_change_notify,
 		.set_loopback	= dp83867_loopback,
+
+		.led_brightness_set = dp83867_led_brightness_set,
 	},
 };
 module_phy_driver(dp83867_driver);
diff --git a/drivers/net/phy/marvell-88x2222.c b/drivers/net/phy/marvell-88x2222.c
index fd9ad4820192..f83cae64585d 100644
--- a/drivers/net/phy/marvell-88x2222.c
+++ b/drivers/net/phy/marvell-88x2222.c
@@ -487,7 +487,7 @@ static int mv2222_sfp_insert(void *upstream, const struct sfp_eeprom_id *id)
 
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(sfp_supported) = { 0, };
 
-	priv = (struct mv2222_data *)phydev->priv;
+	priv = phydev->priv;
 	dev = &phydev->mdio.dev;
 
 	sfp_parse_support(phydev->sfp_bus, id, sfp_supported, interfaces);
@@ -524,7 +524,7 @@ static void mv2222_sfp_remove(void *upstream)
 	struct phy_device *phydev = upstream;
 	struct mv2222_data *priv;
 
-	priv = (struct mv2222_data *)phydev->priv;
+	priv = phydev->priv;
 
 	priv->line_interface = PHY_INTERFACE_MODE_NA;
 	linkmode_zero(priv->supported);
diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c
index 63a3644d86c9..43b6cb725551 100644
--- a/drivers/net/phy/marvell.c
+++ b/drivers/net/phy/marvell.c
@@ -144,11 +144,15 @@
 /* WOL Event Interrupt Enable */
 #define MII_88E1318S_PHY_CSIER_WOL_EIE			BIT(7)
 
-/* LED Timer Control Register */
-#define MII_88E1318S_PHY_LED_TCR			0x12
-#define MII_88E1318S_PHY_LED_TCR_FORCE_INT		BIT(15)
-#define MII_88E1318S_PHY_LED_TCR_INTn_ENABLE		BIT(7)
-#define MII_88E1318S_PHY_LED_TCR_INT_ACTIVE_LOW		BIT(11)
+#define MII_88E1318S_PHY_LED_FUNC		0x10
+#define MII_88E1318S_PHY_LED_FUNC_OFF		(0x8)
+#define MII_88E1318S_PHY_LED_FUNC_ON		(0x9)
+#define MII_88E1318S_PHY_LED_FUNC_HI_Z		(0xa)
+#define MII_88E1318S_PHY_LED_FUNC_BLINK		(0xb)
+#define MII_88E1318S_PHY_LED_TCR		0x12
+#define MII_88E1318S_PHY_LED_TCR_FORCE_INT	BIT(15)
+#define MII_88E1318S_PHY_LED_TCR_INTn_ENABLE	BIT(7)
+#define MII_88E1318S_PHY_LED_TCR_INT_ACTIVE_LOW	BIT(11)
 
 /* Magic Packet MAC address registers */
 #define MII_88E1318S_PHY_MAGIC_PACKET_WORD2		0x17
@@ -2735,7 +2739,7 @@ static const struct hwmon_channel_info marvell_hwmon_temp = {
 	.config = marvell_hwmon_temp_config,
 };
 
-static const struct hwmon_channel_info *marvell_hwmon_info[] = {
+static const struct hwmon_channel_info * const marvell_hwmon_info[] = {
 	&marvell_hwmon_chip,
 	&marvell_hwmon_temp,
 	NULL
@@ -2832,6 +2836,63 @@ static int marvell_hwmon_probe(struct phy_device *phydev)
 }
 #endif
 
+static int m88e1318_led_brightness_set(struct phy_device *phydev,
+				       u8 index, enum led_brightness value)
+{
+	int reg;
+
+	reg = phy_read_paged(phydev, MII_MARVELL_LED_PAGE,
+			     MII_88E1318S_PHY_LED_FUNC);
+	if (reg < 0)
+		return reg;
+
+	switch (index) {
+	case 0:
+	case 1:
+	case 2:
+		reg &= ~(0xf << (4 * index));
+		if (value == LED_OFF)
+			reg |= MII_88E1318S_PHY_LED_FUNC_OFF << (4 * index);
+		else
+			reg |= MII_88E1318S_PHY_LED_FUNC_ON << (4 * index);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return phy_write_paged(phydev, MII_MARVELL_LED_PAGE,
+			       MII_88E1318S_PHY_LED_FUNC, reg);
+}
+
+static int m88e1318_led_blink_set(struct phy_device *phydev, u8 index,
+				  unsigned long *delay_on,
+				  unsigned long *delay_off)
+{
+	int reg;
+
+	reg = phy_read_paged(phydev, MII_MARVELL_LED_PAGE,
+			     MII_88E1318S_PHY_LED_FUNC);
+	if (reg < 0)
+		return reg;
+
+	switch (index) {
+	case 0:
+	case 1:
+	case 2:
+		reg &= ~(0xf << (4 * index));
+		reg |= MII_88E1318S_PHY_LED_FUNC_BLINK << (4 * index);
+		/* Reset default is 84ms */
+		*delay_on = 84 / 2;
+		*delay_off = 84 / 2;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return phy_write_paged(phydev, MII_MARVELL_LED_PAGE,
+			       MII_88E1318S_PHY_LED_FUNC, reg);
+}
+
 static int marvell_probe(struct phy_device *phydev)
 {
 	struct marvell_priv *priv;
@@ -3081,6 +3142,8 @@ static struct phy_driver marvell_drivers[] = {
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
+		.led_brightness_set = m88e1318_led_brightness_set,
+		.led_blink_set = m88e1318_led_blink_set,
 	},
 	{
 		.phy_id = MARVELL_PHY_ID_88E1145,
@@ -3187,6 +3250,8 @@ static struct phy_driver marvell_drivers[] = {
 		.cable_test_start = marvell_vct7_cable_test_start,
 		.cable_test_tdr_start = marvell_vct5_cable_test_tdr_start,
 		.cable_test_get_status = marvell_vct7_cable_test_get_status,
+		.led_brightness_set = m88e1318_led_brightness_set,
+		.led_blink_set = m88e1318_led_blink_set,
 	},
 	{
 		.phy_id = MARVELL_PHY_ID_88E1540,
@@ -3213,6 +3278,8 @@ static struct phy_driver marvell_drivers[] = {
 		.cable_test_start = marvell_vct7_cable_test_start,
 		.cable_test_tdr_start = marvell_vct5_cable_test_tdr_start,
 		.cable_test_get_status = marvell_vct7_cable_test_get_status,
+		.led_brightness_set = m88e1318_led_brightness_set,
+		.led_blink_set = m88e1318_led_blink_set,
 	},
 	{
 		.phy_id = MARVELL_PHY_ID_88E1545,
@@ -3239,6 +3306,8 @@ static struct phy_driver marvell_drivers[] = {
 		.cable_test_start = marvell_vct7_cable_test_start,
 		.cable_test_tdr_start = marvell_vct5_cable_test_tdr_start,
 		.cable_test_get_status = marvell_vct7_cable_test_get_status,
+		.led_brightness_set = m88e1318_led_brightness_set,
+		.led_blink_set = m88e1318_led_blink_set,
 	},
 	{
 		.phy_id = MARVELL_PHY_ID_88E3016,
@@ -3380,6 +3449,8 @@ static struct phy_driver marvell_drivers[] = {
 		.get_stats = marvell_get_stats,
 		.get_tunable = m88e1540_get_tunable,
 		.set_tunable = m88e1540_set_tunable,
+		.led_brightness_set = m88e1318_led_brightness_set,
+		.led_blink_set = m88e1318_led_blink_set,
 	},
 };
 
diff --git a/drivers/net/phy/marvell10g.c b/drivers/net/phy/marvell10g.c
index 383a9c9f36e5..55d9d7acc32e 100644
--- a/drivers/net/phy/marvell10g.c
+++ b/drivers/net/phy/marvell10g.c
@@ -243,7 +243,7 @@ static const struct hwmon_channel_info mv3310_hwmon_temp = {
 	.config = mv3310_hwmon_temp_config,
 };
 
-static const struct hwmon_channel_info *mv3310_hwmon_info[] = {
+static const struct hwmon_channel_info * const mv3310_hwmon_info[] = {
 	&mv3310_hwmon_chip,
 	&mv3310_hwmon_temp,
 	NULL,
diff --git a/drivers/net/phy/meson-gxl.c b/drivers/net/phy/meson-gxl.c
index a6015cd03bff..bb9b33b6bce2 100644
--- a/drivers/net/phy/meson-gxl.c
+++ b/drivers/net/phy/meson-gxl.c
@@ -13,6 +13,7 @@
 #include <linux/phy.h>
 #include <linux/netdevice.h>
 #include <linux/bitfield.h>
+#include <linux/smscphy.h>
 
 #define TSTCNTL		20
 #define  TSTCNTL_READ		BIT(15)
@@ -23,18 +24,6 @@
 #define  TSTCNTL_WRITE_ADDRESS	GENMASK(4, 0)
 #define TSTREAD1	21
 #define TSTWRITE	23
-#define INTSRC_FLAG	29
-#define  INTSRC_ANEG_PR		BIT(1)
-#define  INTSRC_PARALLEL_FAULT	BIT(2)
-#define  INTSRC_ANEG_LP_ACK	BIT(3)
-#define  INTSRC_LINK_DOWN	BIT(4)
-#define  INTSRC_REMOTE_FAULT	BIT(5)
-#define  INTSRC_ANEG_COMPLETE	BIT(6)
-#define  INTSRC_ENERGY_DETECT	BIT(7)
-#define INTSRC_MASK	30
-
-#define INT_SOURCES (INTSRC_LINK_DOWN | INTSRC_ANEG_COMPLETE | \
-		     INTSRC_ENERGY_DETECT)
 
 #define BANK_ANALOG_DSP		0
 #define BANK_WOL		1
@@ -195,59 +184,6 @@ read_status_continue:
 	return genphy_read_status(phydev);
 }
 
-static int meson_gxl_ack_interrupt(struct phy_device *phydev)
-{
-	int ret = phy_read(phydev, INTSRC_FLAG);
-
-	return ret < 0 ? ret : 0;
-}
-
-static int meson_gxl_config_intr(struct phy_device *phydev)
-{
-	int ret;
-
-	if (phydev->interrupts == PHY_INTERRUPT_ENABLED) {
-		/* Ack any pending IRQ */
-		ret = meson_gxl_ack_interrupt(phydev);
-		if (ret)
-			return ret;
-
-		ret = phy_write(phydev, INTSRC_MASK, INT_SOURCES);
-	} else {
-		ret = phy_write(phydev, INTSRC_MASK, 0);
-
-		/* Ack any pending IRQ */
-		ret = meson_gxl_ack_interrupt(phydev);
-	}
-
-	return ret;
-}
-
-static irqreturn_t meson_gxl_handle_interrupt(struct phy_device *phydev)
-{
-	int irq_status;
-
-	irq_status = phy_read(phydev, INTSRC_FLAG);
-	if (irq_status < 0) {
-		phy_error(phydev);
-		return IRQ_NONE;
-	}
-
-	irq_status &= INT_SOURCES;
-
-	if (irq_status == 0)
-		return IRQ_NONE;
-
-	/* Aneg-complete interrupt is used for link-up detection */
-	if (phydev->autoneg == AUTONEG_ENABLE &&
-	    irq_status == INTSRC_ENERGY_DETECT)
-		return IRQ_HANDLED;
-
-	phy_trigger_machine(phydev);
-
-	return IRQ_HANDLED;
-}
-
 static struct phy_driver meson_gxl_phy[] = {
 	{
 		PHY_ID_MATCH_EXACT(0x01814400),
@@ -257,8 +193,8 @@ static struct phy_driver meson_gxl_phy[] = {
 		.soft_reset     = genphy_soft_reset,
 		.config_init	= meson_gxl_config_init,
 		.read_status	= meson_gxl_read_status,
-		.config_intr	= meson_gxl_config_intr,
-		.handle_interrupt = meson_gxl_handle_interrupt,
+		.config_intr	= smsc_phy_config_intr,
+		.handle_interrupt = smsc_phy_handle_interrupt,
 		.suspend        = genphy_suspend,
 		.resume         = genphy_resume,
 		.read_mmd	= genphy_read_mmd_unsupported,
@@ -268,9 +204,16 @@ static struct phy_driver meson_gxl_phy[] = {
 		.name		= "Meson G12A Internal PHY",
 		/* PHY_BASIC_FEATURES */
 		.flags		= PHY_IS_INTERNAL,
+		.probe		= smsc_phy_probe,
+		.config_init	= smsc_phy_config_init,
 		.soft_reset     = genphy_soft_reset,
-		.config_intr	= meson_gxl_config_intr,
-		.handle_interrupt = meson_gxl_handle_interrupt,
+		.read_status	= lan87xx_read_status,
+		.config_intr	= smsc_phy_config_intr,
+		.handle_interrupt = smsc_phy_handle_interrupt,
+
+		.get_tunable	= smsc_phy_get_tunable,
+		.set_tunable	= smsc_phy_set_tunable,
+
 		.suspend        = genphy_suspend,
 		.resume         = genphy_resume,
 		.read_mmd	= genphy_read_mmd_unsupported,
diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c
index 4e884e4ba0ea..3f81bb8dac44 100644
--- a/drivers/net/phy/micrel.c
+++ b/drivers/net/phy/micrel.c
@@ -10,13 +10,13 @@
  * Copyright (c) 2014 Johan Hovold <johan@kernel.org>
  *
  * Support : Micrel Phys:
- *		Giga phys: ksz9021, ksz9031, ksz9131
+ *		Giga phys: ksz9021, ksz9031, ksz9131, lan8841, lan8814
  *		100/10 Phys : ksz8001, ksz8721, ksz8737, ksz8041
  *			   ksz8021, ksz8031, ksz8051,
  *			   ksz8081, ksz8091,
  *			   ksz8061,
  *		Switch : ksz8873, ksz886x
- *			 ksz9477
+ *			 ksz9477, lan8804
  */
 
 #include <linux/bitfield.h>
@@ -318,6 +318,7 @@ struct kszphy_ptp_priv {
 	struct ptp_clock_info ptp_clock_info;
 	/* Lock for ptp_clock */
 	struct mutex ptp_lock;
+	struct ptp_pin_desc *pin_config;
 };
 
 struct kszphy_priv {
@@ -435,11 +436,9 @@ static int kszphy_config_intr(struct phy_device *phydev)
 		if (err)
 			return err;
 
-		temp = KSZPHY_INTCS_ALL;
-		err = phy_write(phydev, MII_KSZPHY_INTCS, temp);
+		err = phy_write(phydev, MII_KSZPHY_INTCS, KSZPHY_INTCS_ALL);
 	} else {
-		temp = 0;
-		err = phy_write(phydev, MII_KSZPHY_INTCS, temp);
+		err = phy_write(phydev, MII_KSZPHY_INTCS, 0);
 		if (err)
 			return err;
 
@@ -3438,6 +3437,7 @@ static void lan8841_ptp_process_rx_ts(struct kszphy_ptp_priv *ptp_priv)
 #define LAN8841_PTP_INT_STS_PTP_TX_TS_INT	BIT(12)
 #define LAN8841_PTP_INT_STS_PTP_RX_TS_OVRFL_INT	BIT(9)
 #define LAN8841_PTP_INT_STS_PTP_RX_TS_INT	BIT(8)
+#define LAN8841_PTP_INT_STS_PTP_GPIO_CAP_INT	BIT(2)
 
 static void lan8841_ptp_flush_fifo(struct kszphy_ptp_priv *ptp_priv, bool egress)
 {
@@ -3452,6 +3452,67 @@ static void lan8841_ptp_flush_fifo(struct kszphy_ptp_priv *ptp_priv, bool egress
 	phy_read_mmd(phydev, 2, LAN8841_PTP_INT_STS);
 }
 
+#define LAN8841_PTP_GPIO_CAP_STS			506
+#define LAN8841_PTP_GPIO_SEL				327
+#define LAN8841_PTP_GPIO_SEL_GPIO_SEL(gpio)		((gpio) << 8)
+#define LAN8841_PTP_GPIO_RE_LTC_SEC_HI_CAP		498
+#define LAN8841_PTP_GPIO_RE_LTC_SEC_LO_CAP		499
+#define LAN8841_PTP_GPIO_RE_LTC_NS_HI_CAP		500
+#define LAN8841_PTP_GPIO_RE_LTC_NS_LO_CAP		501
+#define LAN8841_PTP_GPIO_FE_LTC_SEC_HI_CAP		502
+#define LAN8841_PTP_GPIO_FE_LTC_SEC_LO_CAP		503
+#define LAN8841_PTP_GPIO_FE_LTC_NS_HI_CAP		504
+#define LAN8841_PTP_GPIO_FE_LTC_NS_LO_CAP		505
+
+static void lan8841_gpio_process_cap(struct kszphy_ptp_priv *ptp_priv)
+{
+	struct phy_device *phydev = ptp_priv->phydev;
+	struct ptp_clock_event ptp_event = {0};
+	int pin, ret, tmp;
+	s32 sec, nsec;
+
+	pin = ptp_find_pin_unlocked(ptp_priv->ptp_clock, PTP_PF_EXTTS, 0);
+	if (pin == -1)
+		return;
+
+	tmp = phy_read_mmd(phydev, 2, LAN8841_PTP_GPIO_CAP_STS);
+	if (tmp < 0)
+		return;
+
+	ret = phy_write_mmd(phydev, 2, LAN8841_PTP_GPIO_SEL,
+			    LAN8841_PTP_GPIO_SEL_GPIO_SEL(pin));
+	if (ret)
+		return;
+
+	mutex_lock(&ptp_priv->ptp_lock);
+	if (tmp & BIT(pin)) {
+		sec = phy_read_mmd(phydev, 2, LAN8841_PTP_GPIO_RE_LTC_SEC_HI_CAP);
+		sec <<= 16;
+		sec |= phy_read_mmd(phydev, 2, LAN8841_PTP_GPIO_RE_LTC_SEC_LO_CAP);
+
+		nsec = phy_read_mmd(phydev, 2, LAN8841_PTP_GPIO_RE_LTC_NS_HI_CAP) & 0x3fff;
+		nsec <<= 16;
+		nsec |= phy_read_mmd(phydev, 2, LAN8841_PTP_GPIO_RE_LTC_NS_LO_CAP);
+	} else {
+		sec = phy_read_mmd(phydev, 2, LAN8841_PTP_GPIO_FE_LTC_SEC_HI_CAP);
+		sec <<= 16;
+		sec |= phy_read_mmd(phydev, 2, LAN8841_PTP_GPIO_FE_LTC_SEC_LO_CAP);
+
+		nsec = phy_read_mmd(phydev, 2, LAN8841_PTP_GPIO_FE_LTC_NS_HI_CAP) & 0x3fff;
+		nsec <<= 16;
+		nsec |= phy_read_mmd(phydev, 2, LAN8841_PTP_GPIO_FE_LTC_NS_LO_CAP);
+	}
+	mutex_unlock(&ptp_priv->ptp_lock);
+	ret = phy_write_mmd(phydev, 2, LAN8841_PTP_GPIO_SEL, 0);
+	if (ret)
+		return;
+
+	ptp_event.index = 0;
+	ptp_event.timestamp = ktime_set(sec, nsec);
+	ptp_event.type = PTP_CLOCK_EXTTS;
+	ptp_clock_event(ptp_priv->ptp_clock, &ptp_event);
+}
+
 static void lan8841_handle_ptp_interrupt(struct phy_device *phydev)
 {
 	struct kszphy_priv *priv = phydev->priv;
@@ -3460,12 +3521,16 @@ static void lan8841_handle_ptp_interrupt(struct phy_device *phydev)
 
 	do {
 		status = phy_read_mmd(phydev, 2, LAN8841_PTP_INT_STS);
+
 		if (status & LAN8841_PTP_INT_STS_PTP_TX_TS_INT)
 			lan8841_ptp_process_tx_ts(ptp_priv);
 
 		if (status & LAN8841_PTP_INT_STS_PTP_RX_TS_INT)
 			lan8841_ptp_process_rx_ts(ptp_priv);
 
+		if (status & LAN8841_PTP_INT_STS_PTP_GPIO_CAP_INT)
+			lan8841_gpio_process_cap(ptp_priv);
+
 		if (status & LAN8841_PTP_INT_STS_PTP_TX_TS_OVRFL_INT) {
 			lan8841_ptp_flush_fifo(ptp_priv, true);
 			skb_queue_purge(&ptp_priv->tx_queue);
@@ -3658,6 +3723,77 @@ static int lan8841_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0;
 }
 
+#define LAN8841_EVENT_A		0
+#define LAN8841_EVENT_B		1
+#define LAN8841_PTP_LTC_TARGET_SEC_HI(event)	((event) == LAN8841_EVENT_A ? 278 : 288)
+#define LAN8841_PTP_LTC_TARGET_SEC_LO(event)	((event) == LAN8841_EVENT_A ? 279 : 289)
+#define LAN8841_PTP_LTC_TARGET_NS_HI(event)	((event) == LAN8841_EVENT_A ? 280 : 290)
+#define LAN8841_PTP_LTC_TARGET_NS_LO(event)	((event) == LAN8841_EVENT_A ? 281 : 291)
+
+static int lan8841_ptp_set_target(struct kszphy_ptp_priv *ptp_priv, u8 event,
+				  s64 sec, u32 nsec)
+{
+	struct phy_device *phydev = ptp_priv->phydev;
+	int ret;
+
+	ret = phy_write_mmd(phydev, 2, LAN8841_PTP_LTC_TARGET_SEC_HI(event),
+			    upper_16_bits(sec));
+	if (ret)
+		return ret;
+
+	ret = phy_write_mmd(phydev, 2, LAN8841_PTP_LTC_TARGET_SEC_LO(event),
+			    lower_16_bits(sec));
+	if (ret)
+		return ret;
+
+	ret = phy_write_mmd(phydev, 2, LAN8841_PTP_LTC_TARGET_NS_HI(event) & 0x3fff,
+			    upper_16_bits(nsec));
+	if (ret)
+		return ret;
+
+	return phy_write_mmd(phydev, 2, LAN8841_PTP_LTC_TARGET_NS_LO(event),
+			    lower_16_bits(nsec));
+}
+
+#define LAN8841_BUFFER_TIME	2
+
+static int lan8841_ptp_update_target(struct kszphy_ptp_priv *ptp_priv,
+				     const struct timespec64 *ts)
+{
+	return lan8841_ptp_set_target(ptp_priv, LAN8841_EVENT_A,
+				      ts->tv_sec + LAN8841_BUFFER_TIME, 0);
+}
+
+#define LAN8841_PTP_LTC_TARGET_RELOAD_SEC_HI(event)	((event) == LAN8841_EVENT_A ? 282 : 292)
+#define LAN8841_PTP_LTC_TARGET_RELOAD_SEC_LO(event)	((event) == LAN8841_EVENT_A ? 283 : 293)
+#define LAN8841_PTP_LTC_TARGET_RELOAD_NS_HI(event)	((event) == LAN8841_EVENT_A ? 284 : 294)
+#define LAN8841_PTP_LTC_TARGET_RELOAD_NS_LO(event)	((event) == LAN8841_EVENT_A ? 285 : 295)
+
+static int lan8841_ptp_set_reload(struct kszphy_ptp_priv *ptp_priv, u8 event,
+				  s64 sec, u32 nsec)
+{
+	struct phy_device *phydev = ptp_priv->phydev;
+	int ret;
+
+	ret = phy_write_mmd(phydev, 2, LAN8841_PTP_LTC_TARGET_RELOAD_SEC_HI(event),
+			    upper_16_bits(sec));
+	if (ret)
+		return ret;
+
+	ret = phy_write_mmd(phydev, 2, LAN8841_PTP_LTC_TARGET_RELOAD_SEC_LO(event),
+			    lower_16_bits(sec));
+	if (ret)
+		return ret;
+
+	ret = phy_write_mmd(phydev, 2, LAN8841_PTP_LTC_TARGET_RELOAD_NS_HI(event) & 0x3fff,
+			    upper_16_bits(nsec));
+	if (ret)
+		return ret;
+
+	return phy_write_mmd(phydev, 2, LAN8841_PTP_LTC_TARGET_RELOAD_NS_LO(event),
+			     lower_16_bits(nsec));
+}
+
 #define LAN8841_PTP_LTC_SET_SEC_HI	262
 #define LAN8841_PTP_LTC_SET_SEC_MID	263
 #define LAN8841_PTP_LTC_SET_SEC_LO	264
@@ -3671,6 +3807,7 @@ static int lan8841_ptp_settime64(struct ptp_clock_info *ptp,
 	struct kszphy_ptp_priv *ptp_priv = container_of(ptp, struct kszphy_ptp_priv,
 							ptp_clock_info);
 	struct phy_device *phydev = ptp_priv->phydev;
+	int ret;
 
 	/* Set the value to be stored */
 	mutex_lock(&ptp_priv->ptp_lock);
@@ -3683,9 +3820,10 @@ static int lan8841_ptp_settime64(struct ptp_clock_info *ptp,
 	/* Set the command to load the LTC */
 	phy_write_mmd(phydev, 2, LAN8841_PTP_CMD_CTL,
 		      LAN8841_PTP_CMD_CTL_PTP_LTC_LOAD);
+	ret = lan8841_ptp_update_target(ptp_priv, ts);
 	mutex_unlock(&ptp_priv->ptp_lock);
 
-	return 0;
+	return ret;
 }
 
 #define LAN8841_PTP_LTC_RD_SEC_HI	358
@@ -3740,6 +3878,7 @@ static int lan8841_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
 	bool add = true;
 	u32 nsec;
 	s32 sec;
+	int ret;
 
 	/* The HW allows up to 15 sec to adjust the time, but here we limit to
 	 * 10 sec the adjustment. The reason is, in case the adjustment is 14
@@ -3803,7 +3942,13 @@ static int lan8841_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
 	}
 	mutex_unlock(&ptp_priv->ptp_lock);
 
-	return 0;
+	/* Update the target clock */
+	ptp->gettime64(ptp, &ts);
+	mutex_lock(&ptp_priv->ptp_lock);
+	ret = lan8841_ptp_update_target(ptp_priv, &ts);
+	mutex_unlock(&ptp_priv->ptp_lock);
+
+	return ret;
 }
 
 #define LAN8841_PTP_LTC_RATE_ADJ_HI		269
@@ -3839,6 +3984,387 @@ static int lan8841_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
 	return 0;
 }
 
+static int lan8841_ptp_verify(struct ptp_clock_info *ptp, unsigned int pin,
+			      enum ptp_pin_function func, unsigned int chan)
+{
+	switch (func) {
+	case PTP_PF_NONE:
+	case PTP_PF_PEROUT:
+	case PTP_PF_EXTTS:
+		break;
+	default:
+		return -1;
+	}
+
+	return 0;
+}
+
+#define LAN8841_PTP_GPIO_NUM	10
+#define LAN8841_GPIO_EN		128
+#define LAN8841_GPIO_DIR	129
+#define LAN8841_GPIO_BUF	130
+
+static int lan8841_ptp_perout_off(struct kszphy_ptp_priv *ptp_priv, int pin)
+{
+	struct phy_device *phydev = ptp_priv->phydev;
+	int ret;
+
+	ret = phy_clear_bits_mmd(phydev, 2, LAN8841_GPIO_EN, BIT(pin));
+	if (ret)
+		return ret;
+
+	ret = phy_clear_bits_mmd(phydev, 2, LAN8841_GPIO_DIR, BIT(pin));
+	if (ret)
+		return ret;
+
+	return phy_clear_bits_mmd(phydev, 2, LAN8841_GPIO_BUF, BIT(pin));
+}
+
+static int lan8841_ptp_perout_on(struct kszphy_ptp_priv *ptp_priv, int pin)
+{
+	struct phy_device *phydev = ptp_priv->phydev;
+	int ret;
+
+	ret = phy_set_bits_mmd(phydev, 2, LAN8841_GPIO_EN, BIT(pin));
+	if (ret)
+		return ret;
+
+	ret = phy_set_bits_mmd(phydev, 2, LAN8841_GPIO_DIR, BIT(pin));
+	if (ret)
+		return ret;
+
+	return phy_set_bits_mmd(phydev, 2, LAN8841_GPIO_BUF, BIT(pin));
+}
+
+#define LAN8841_GPIO_DATA_SEL1				131
+#define LAN8841_GPIO_DATA_SEL2				132
+#define LAN8841_GPIO_DATA_SEL_GPIO_DATA_SEL_EVENT_MASK	GENMASK(2, 0)
+#define LAN8841_GPIO_DATA_SEL_GPIO_DATA_SEL_EVENT_A	1
+#define LAN8841_GPIO_DATA_SEL_GPIO_DATA_SEL_EVENT_B	2
+#define LAN8841_PTP_GENERAL_CONFIG			257
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_POL_A	BIT(1)
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_POL_B	BIT(3)
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_A_MASK	GENMASK(7, 4)
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_B_MASK	GENMASK(11, 8)
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_A		4
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_B		7
+
+static int lan8841_ptp_remove_event(struct kszphy_ptp_priv *ptp_priv, int pin,
+				    u8 event)
+{
+	struct phy_device *phydev = ptp_priv->phydev;
+	u16 tmp;
+	int ret;
+
+	/* Now remove pin from the event. GPIO_DATA_SEL1 contains the GPIO
+	 * pins 0-4 while GPIO_DATA_SEL2 contains GPIO pins 5-9, therefore
+	 * depending on the pin, it requires to read a different register
+	 */
+	if (pin < 5) {
+		tmp = LAN8841_GPIO_DATA_SEL_GPIO_DATA_SEL_EVENT_MASK << (3 * pin);
+		ret = phy_clear_bits_mmd(phydev, 2, LAN8841_GPIO_DATA_SEL1, tmp);
+	} else {
+		tmp = LAN8841_GPIO_DATA_SEL_GPIO_DATA_SEL_EVENT_MASK << (3 * (pin - 5));
+		ret = phy_clear_bits_mmd(phydev, 2, LAN8841_GPIO_DATA_SEL2, tmp);
+	}
+	if (ret)
+		return ret;
+
+	/* Disable the event */
+	if (event == LAN8841_EVENT_A)
+		tmp = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_POL_A |
+		      LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_A_MASK;
+	else
+		tmp = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_POL_B |
+		      LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_B_MASK;
+	return phy_clear_bits_mmd(phydev, 2, LAN8841_GPIO_EN, tmp);
+}
+
+static int lan8841_ptp_enable_event(struct kszphy_ptp_priv *ptp_priv, int pin,
+				    u8 event, int pulse_width)
+{
+	struct phy_device *phydev = ptp_priv->phydev;
+	u16 tmp;
+	int ret;
+
+	/* Enable the event */
+	if (event == LAN8841_EVENT_A)
+		ret = phy_modify_mmd(phydev, 2, LAN8841_PTP_GENERAL_CONFIG,
+				     LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_POL_A |
+				     LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_A_MASK,
+				     LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_POL_A |
+				     pulse_width << LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_A);
+	else
+		ret = phy_modify_mmd(phydev, 2, LAN8841_PTP_GENERAL_CONFIG,
+				     LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_POL_B |
+				     LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_B_MASK,
+				     LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_POL_B |
+				     pulse_width << LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_B);
+	if (ret)
+		return ret;
+
+	/* Now connect the pin to the event. GPIO_DATA_SEL1 contains the GPIO
+	 * pins 0-4 while GPIO_DATA_SEL2 contains GPIO pins 5-9, therefore
+	 * depending on the pin, it requires to read a different register
+	 */
+	if (event == LAN8841_EVENT_A)
+		tmp = LAN8841_GPIO_DATA_SEL_GPIO_DATA_SEL_EVENT_A;
+	else
+		tmp = LAN8841_GPIO_DATA_SEL_GPIO_DATA_SEL_EVENT_B;
+
+	if (pin < 5)
+		ret = phy_set_bits_mmd(phydev, 2, LAN8841_GPIO_DATA_SEL1,
+				       tmp << (3 * pin));
+	else
+		ret = phy_set_bits_mmd(phydev, 2, LAN8841_GPIO_DATA_SEL2,
+				       tmp << (3 * (pin - 5)));
+
+	return ret;
+}
+
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_200MS	13
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_100MS	12
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_50MS	11
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_10MS	10
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_5MS	9
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_1MS	8
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_500US	7
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_100US	6
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_50US	5
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_10US	4
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_5US	3
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_1US	2
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_500NS	1
+#define LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_100NS	0
+
+static int lan8841_ptp_perout(struct ptp_clock_info *ptp,
+			      struct ptp_clock_request *rq, int on)
+{
+	struct kszphy_ptp_priv *ptp_priv = container_of(ptp, struct kszphy_ptp_priv,
+							ptp_clock_info);
+	struct phy_device *phydev = ptp_priv->phydev;
+	struct timespec64 ts_on, ts_period;
+	s64 on_nsec, period_nsec;
+	int pulse_width;
+	int pin;
+	int ret;
+
+	if (rq->perout.flags & ~PTP_PEROUT_DUTY_CYCLE)
+		return -EOPNOTSUPP;
+
+	pin = ptp_find_pin(ptp_priv->ptp_clock, PTP_PF_PEROUT, rq->perout.index);
+	if (pin == -1 || pin >= LAN8841_PTP_GPIO_NUM)
+		return -EINVAL;
+
+	if (!on) {
+		ret = lan8841_ptp_perout_off(ptp_priv, pin);
+		if (ret)
+			return ret;
+
+		return lan8841_ptp_remove_event(ptp_priv, LAN8841_EVENT_A, pin);
+	}
+
+	ts_on.tv_sec = rq->perout.on.sec;
+	ts_on.tv_nsec = rq->perout.on.nsec;
+	on_nsec = timespec64_to_ns(&ts_on);
+
+	ts_period.tv_sec = rq->perout.period.sec;
+	ts_period.tv_nsec = rq->perout.period.nsec;
+	period_nsec = timespec64_to_ns(&ts_period);
+
+	if (period_nsec < 200) {
+		pr_warn_ratelimited("%s: perout period too small, minimum is 200 nsec\n",
+				    phydev_name(phydev));
+		return -EOPNOTSUPP;
+	}
+
+	if (on_nsec >= period_nsec) {
+		pr_warn_ratelimited("%s: pulse width must be smaller than period\n",
+				    phydev_name(phydev));
+		return -EINVAL;
+	}
+
+	switch (on_nsec) {
+	case 200000000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_200MS;
+		break;
+	case 100000000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_100MS;
+		break;
+	case 50000000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_50MS;
+		break;
+	case 10000000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_10MS;
+		break;
+	case 5000000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_5MS;
+		break;
+	case 1000000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_1MS;
+		break;
+	case 500000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_500US;
+		break;
+	case 100000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_100US;
+		break;
+	case 50000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_50US;
+		break;
+	case 10000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_10US;
+		break;
+	case 5000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_5US;
+		break;
+	case 1000:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_1US;
+		break;
+	case 500:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_500NS;
+		break;
+	case 100:
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_100NS;
+		break;
+	default:
+		pr_warn_ratelimited("%s: Use default duty cycle of 100ns\n",
+				    phydev_name(phydev));
+		pulse_width = LAN8841_PTP_GENERAL_CONFIG_LTC_EVENT_100NS;
+		break;
+	}
+
+	mutex_lock(&ptp_priv->ptp_lock);
+	ret = lan8841_ptp_set_target(ptp_priv, LAN8841_EVENT_A, rq->perout.start.sec,
+				     rq->perout.start.nsec);
+	mutex_unlock(&ptp_priv->ptp_lock);
+	if (ret)
+		return ret;
+
+	ret = lan8841_ptp_set_reload(ptp_priv, LAN8841_EVENT_A, rq->perout.period.sec,
+				     rq->perout.period.nsec);
+	if (ret)
+		return ret;
+
+	ret = lan8841_ptp_enable_event(ptp_priv, pin, LAN8841_EVENT_A,
+				       pulse_width);
+	if (ret)
+		return ret;
+
+	ret = lan8841_ptp_perout_on(ptp_priv, pin);
+	if (ret)
+		lan8841_ptp_remove_event(ptp_priv, pin, LAN8841_EVENT_A);
+
+	return ret;
+}
+
+#define LAN8841_PTP_GPIO_CAP_EN			496
+#define LAN8841_PTP_GPIO_CAP_EN_GPIO_RE_CAPTURE_ENABLE(gpio)	(BIT(gpio))
+#define LAN8841_PTP_GPIO_CAP_EN_GPIO_FE_CAPTURE_ENABLE(gpio)	(BIT(gpio) << 8)
+#define LAN8841_PTP_INT_EN_PTP_GPIO_CAP_EN	BIT(2)
+
+static int lan8841_ptp_extts_on(struct kszphy_ptp_priv *ptp_priv, int pin,
+				u32 flags)
+{
+	struct phy_device *phydev = ptp_priv->phydev;
+	u16 tmp = 0;
+	int ret;
+
+	/* Set GPIO to be intput */
+	ret = phy_set_bits_mmd(phydev, 2, LAN8841_GPIO_EN, BIT(pin));
+	if (ret)
+		return ret;
+
+	ret = phy_clear_bits_mmd(phydev, 2, LAN8841_GPIO_BUF, BIT(pin));
+	if (ret)
+		return ret;
+
+	/* Enable capture on the edges of the pin */
+	if (flags & PTP_RISING_EDGE)
+		tmp |= LAN8841_PTP_GPIO_CAP_EN_GPIO_RE_CAPTURE_ENABLE(pin);
+	if (flags & PTP_FALLING_EDGE)
+		tmp |= LAN8841_PTP_GPIO_CAP_EN_GPIO_FE_CAPTURE_ENABLE(pin);
+	ret = phy_write_mmd(phydev, 2, LAN8841_PTP_GPIO_CAP_EN, tmp);
+	if (ret)
+		return ret;
+
+	/* Enable interrupt */
+	return phy_modify_mmd(phydev, 2, LAN8841_PTP_INT_EN,
+			      LAN8841_PTP_INT_EN_PTP_GPIO_CAP_EN,
+			      LAN8841_PTP_INT_EN_PTP_GPIO_CAP_EN);
+}
+
+static int lan8841_ptp_extts_off(struct kszphy_ptp_priv *ptp_priv, int pin)
+{
+	struct phy_device *phydev = ptp_priv->phydev;
+	int ret;
+
+	/* Set GPIO to be output */
+	ret = phy_clear_bits_mmd(phydev, 2, LAN8841_GPIO_EN, BIT(pin));
+	if (ret)
+		return ret;
+
+	ret = phy_clear_bits_mmd(phydev, 2, LAN8841_GPIO_BUF, BIT(pin));
+	if (ret)
+		return ret;
+
+	/* Disable capture on both of the edges */
+	ret = phy_modify_mmd(phydev, 2, LAN8841_PTP_GPIO_CAP_EN,
+			     LAN8841_PTP_GPIO_CAP_EN_GPIO_RE_CAPTURE_ENABLE(pin) |
+			     LAN8841_PTP_GPIO_CAP_EN_GPIO_FE_CAPTURE_ENABLE(pin),
+			     0);
+	if (ret)
+		return ret;
+
+	/* Disable interrupt */
+	return phy_modify_mmd(phydev, 2, LAN8841_PTP_INT_EN,
+			      LAN8841_PTP_INT_EN_PTP_GPIO_CAP_EN,
+			      0);
+}
+
+static int lan8841_ptp_extts(struct ptp_clock_info *ptp,
+			     struct ptp_clock_request *rq, int on)
+{
+	struct kszphy_ptp_priv *ptp_priv = container_of(ptp, struct kszphy_ptp_priv,
+							ptp_clock_info);
+	int pin;
+	int ret;
+
+	/* Reject requests with unsupported flags */
+	if (rq->extts.flags & ~(PTP_ENABLE_FEATURE |
+				PTP_EXTTS_EDGES |
+				PTP_STRICT_FLAGS))
+		return -EOPNOTSUPP;
+
+	pin = ptp_find_pin(ptp_priv->ptp_clock, PTP_PF_EXTTS, rq->extts.index);
+	if (pin == -1 || pin >= LAN8841_PTP_GPIO_NUM)
+		return -EINVAL;
+
+	mutex_lock(&ptp_priv->ptp_lock);
+	if (on)
+		ret = lan8841_ptp_extts_on(ptp_priv, pin, rq->extts.flags);
+	else
+		ret = lan8841_ptp_extts_off(ptp_priv, pin);
+	mutex_unlock(&ptp_priv->ptp_lock);
+
+	return ret;
+}
+
+static int lan8841_ptp_enable(struct ptp_clock_info *ptp,
+			      struct ptp_clock_request *rq, int on)
+{
+	switch (rq->type) {
+	case PTP_CLK_REQ_EXTTS:
+		return lan8841_ptp_extts(ptp, rq, on);
+	case PTP_CLK_REQ_PEROUT:
+		return lan8841_ptp_perout(ptp, rq, on);
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
 static struct ptp_clock_info lan8841_ptp_clock_info = {
 	.owner		= THIS_MODULE,
 	.name		= "lan8841 ptp",
@@ -3847,6 +4373,11 @@ static struct ptp_clock_info lan8841_ptp_clock_info = {
 	.settime64	= lan8841_ptp_settime64,
 	.adjtime	= lan8841_ptp_adjtime,
 	.adjfine	= lan8841_ptp_adjfine,
+	.verify         = lan8841_ptp_verify,
+	.enable         = lan8841_ptp_enable,
+	.n_per_out      = LAN8841_PTP_GPIO_NUM,
+	.n_ext_ts       = LAN8841_PTP_GPIO_NUM,
+	.n_pins         = LAN8841_PTP_GPIO_NUM,
 };
 
 #define LAN8841_OPERATION_MODE_STRAP_LOW_REGISTER 3
@@ -3874,7 +4405,23 @@ static int lan8841_probe(struct phy_device *phydev)
 	priv = phydev->priv;
 	ptp_priv = &priv->ptp_priv;
 
+	ptp_priv->pin_config = devm_kcalloc(&phydev->mdio.dev,
+					    LAN8841_PTP_GPIO_NUM,
+					    sizeof(*ptp_priv->pin_config),
+					    GFP_KERNEL);
+	if (!ptp_priv->pin_config)
+		return -ENOMEM;
+
+	for (int i = 0; i < LAN8841_PTP_GPIO_NUM; ++i) {
+		struct ptp_pin_desc *p = &ptp_priv->pin_config[i];
+
+		snprintf(p->name, sizeof(p->name), "pin%d", i);
+		p->index = i;
+		p->func = PTP_PF_NONE;
+	}
+
 	ptp_priv->ptp_clock_info = lan8841_ptp_clock_info;
+	ptp_priv->ptp_clock_info.pin_config = ptp_priv->pin_config;
 	ptp_priv->ptp_clock = ptp_clock_register(&ptp_priv->ptp_clock_info,
 						 &phydev->mdio.dev);
 	if (IS_ERR(ptp_priv->ptp_clock)) {
diff --git a/drivers/net/phy/microchip_t1s.c b/drivers/net/phy/microchip_t1s.c
new file mode 100644
index 000000000000..094967b3c111
--- /dev/null
+++ b/drivers/net/phy/microchip_t1s.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Driver for Microchip 10BASE-T1S LAN867X PHY
+ *
+ * Support: Microchip Phys:
+ *  lan8670, lan8671, lan8672
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/phy.h>
+
+#define PHY_ID_LAN867X 0x0007C160
+
+#define LAN867X_REG_IRQ_1_CTL 0x001C
+#define LAN867X_REG_IRQ_2_CTL 0x001D
+
+/* The arrays below are pulled from the following table from AN1699
+ * Access MMD Address Value Mask
+ * RMW 0x1F 0x00D0 0x0002 0x0E03
+ * RMW 0x1F 0x00D1 0x0000 0x0300
+ * RMW 0x1F 0x0084 0x3380 0xFFC0
+ * RMW 0x1F 0x0085 0x0006 0x000F
+ * RMW 0x1F 0x008A 0xC000 0xF800
+ * RMW 0x1F 0x0087 0x801C 0x801C
+ * RMW 0x1F 0x0088 0x033F 0x1FFF
+ * W   0x1F 0x008B 0x0404 ------
+ * RMW 0x1F 0x0080 0x0600 0x0600
+ * RMW 0x1F 0x00F1 0x2400 0x7F00
+ * RMW 0x1F 0x0096 0x2000 0x2000
+ * W   0x1F 0x0099 0x7F80 ------
+ */
+
+static const int lan867x_fixup_registers[12] = {
+	0x00D0, 0x00D1, 0x0084, 0x0085,
+	0x008A, 0x0087, 0x0088, 0x008B,
+	0x0080, 0x00F1, 0x0096, 0x0099,
+};
+
+static const int lan867x_fixup_values[12] = {
+	0x0002, 0x0000, 0x3380, 0x0006,
+	0xC000, 0x801C, 0x033F, 0x0404,
+	0x0600, 0x2400, 0x2000, 0x7F80,
+};
+
+static const int lan867x_fixup_masks[12] = {
+	0x0E03, 0x0300, 0xFFC0, 0x000F,
+	0xF800, 0x801C, 0x1FFF, 0xFFFF,
+	0x0600, 0x7F00, 0x2000, 0xFFFF,
+};
+
+static int lan867x_config_init(struct phy_device *phydev)
+{
+	/* HW quirk: Microchip states in the application note (AN1699) for the phy
+	 * that a set of read-modify-write (rmw) operations has to be performed
+	 * on a set of seemingly magic registers.
+	 * The result of these operations is just described as 'optimal performance'
+	 * Microchip gives no explanation as to what these mmd regs do,
+	 * in fact they are marked as reserved in the datasheet.
+	 * It is unclear if phy_modify_mmd would be safe to use or if a write
+	 * really has to happen to each register.
+	 * In order to exactly conform to what is stated in the AN phy_write_mmd is
+	 * used, which might then write the same value back as read + modified.
+	 */
+
+	int reg_value;
+	int err;
+	int reg;
+
+	/* Read-Modified Write Pseudocode (from AN1699)
+	 * current_val = read_register(mmd, addr) // Read current register value
+	 * new_val = current_val AND (NOT mask) // Clear bit fields to be written
+	 * new_val = new_val OR value // Set bits
+	 * write_register(mmd, addr, new_val) // Write back updated register value
+	 */
+	for (int i = 0; i < ARRAY_SIZE(lan867x_fixup_registers); i++) {
+		reg = lan867x_fixup_registers[i];
+		reg_value = phy_read_mmd(phydev, MDIO_MMD_VEND2, reg);
+		reg_value &= ~lan867x_fixup_masks[i];
+		reg_value |= lan867x_fixup_values[i];
+		err = phy_write_mmd(phydev, MDIO_MMD_VEND2, reg, reg_value);
+		if (err != 0)
+			return err;
+	}
+
+	/* None of the interrupts in the lan867x phy seem relevant.
+	 * Other phys inspect the link status and call phy_trigger_machine
+	 * in the interrupt handler.
+	 * This phy does not support link status, and thus has no interrupt
+	 * for it either.
+	 * So we'll just disable all interrupts on the chip.
+	 */
+	err = phy_write_mmd(phydev, MDIO_MMD_VEND2, LAN867X_REG_IRQ_1_CTL, 0xFFFF);
+	if (err != 0)
+		return err;
+	return phy_write_mmd(phydev, MDIO_MMD_VEND2, LAN867X_REG_IRQ_2_CTL, 0xFFFF);
+}
+
+static int lan867x_read_status(struct phy_device *phydev)
+{
+	/* The phy has some limitations, namely:
+	 *  - always reports link up
+	 *  - only supports 10MBit half duplex
+	 *  - does not support auto negotiate
+	 */
+	phydev->link = 1;
+	phydev->duplex = DUPLEX_HALF;
+	phydev->speed = SPEED_10;
+	phydev->autoneg = AUTONEG_DISABLE;
+
+	return 0;
+}
+
+static struct phy_driver lan867x_driver[] = {
+	{
+		PHY_ID_MATCH_MODEL(PHY_ID_LAN867X),
+		.name               = "LAN867X",
+		.features           = PHY_BASIC_T1S_P2MP_FEATURES,
+		.config_init        = lan867x_config_init,
+		.read_status        = lan867x_read_status,
+		.get_plca_cfg	    = genphy_c45_plca_get_cfg,
+		.set_plca_cfg	    = genphy_c45_plca_set_cfg,
+		.get_plca_status    = genphy_c45_plca_get_status,
+	}
+};
+
+module_phy_driver(lan867x_driver);
+
+static struct mdio_device_id __maybe_unused tbl[] = {
+	{ PHY_ID_MATCH_MODEL(PHY_ID_LAN867X) },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(mdio, tbl);
+
+MODULE_DESCRIPTION("Microchip 10BASE-T1S lan867x Phy driver");
+MODULE_AUTHOR("Ramón Nordin Rodriguez");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/phy/mxl-gpy.c b/drivers/net/phy/mxl-gpy.c
index e5972b4ef6e8..6301a9abfb95 100644
--- a/drivers/net/phy/mxl-gpy.c
+++ b/drivers/net/phy/mxl-gpy.c
@@ -107,6 +107,13 @@ struct gpy_priv {
 
 	u8 fw_major;
 	u8 fw_minor;
+
+	/* It takes 3 seconds to fully switch out of loopback mode before
+	 * it can safely re-enter loopback mode. Record the time when
+	 * loopback is disabled. Check and wait if necessary before loopback
+	 * is enabled.
+	 */
+	u64 lb_dis_to;
 };
 
 static const struct {
@@ -175,7 +182,7 @@ static umode_t gpy_hwmon_is_visible(const void *data,
 	return 0444;
 }
 
-static const struct hwmon_channel_info *gpy_hwmon_info[] = {
+static const struct hwmon_channel_info * const gpy_hwmon_info[] = {
 	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
 	NULL
 };
@@ -769,18 +776,34 @@ static void gpy_get_wol(struct phy_device *phydev,
 
 static int gpy_loopback(struct phy_device *phydev, bool enable)
 {
+	struct gpy_priv *priv = phydev->priv;
+	u16 set = 0;
 	int ret;
 
-	ret = phy_modify(phydev, MII_BMCR, BMCR_LOOPBACK,
-			 enable ? BMCR_LOOPBACK : 0);
-	if (!ret) {
-		/* It takes some time for PHY device to switch
-		 * into/out-of loopback mode.
+	if (enable) {
+		u64 now = get_jiffies_64();
+
+		/* wait until 3 seconds from last disable */
+		if (time_before64(now, priv->lb_dis_to))
+			msleep(jiffies64_to_msecs(priv->lb_dis_to - now));
+
+		set = BMCR_LOOPBACK;
+	}
+
+	ret = phy_modify(phydev, MII_BMCR, BMCR_LOOPBACK, set);
+	if (ret <= 0)
+		return ret;
+
+	if (enable) {
+		/* It takes some time for PHY device to switch into
+		 * loopback mode.
 		 */
 		msleep(100);
+	} else {
+		priv->lb_dis_to = get_jiffies_64() + HZ * 3;
 	}
 
-	return ret;
+	return 0;
 }
 
 static int gpy115_loopback(struct phy_device *phydev, bool enable)
diff --git a/drivers/net/phy/nxp-cbtx.c b/drivers/net/phy/nxp-cbtx.c
new file mode 100644
index 000000000000..145703f0a406
--- /dev/null
+++ b/drivers/net/phy/nxp-cbtx.c
@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Driver for 100BASE-TX PHY embedded into NXP SJA1110 switch
+ *
+ * Copyright 2022-2023 NXP
+ */
+
+#include <linux/kernel.h>
+#include <linux/mii.h>
+#include <linux/module.h>
+#include <linux/phy.h>
+
+#define PHY_ID_CBTX_SJA1110			0x001bb020
+
+/* Registers */
+#define  CBTX_MODE_CTRL_STAT			0x11
+#define  CBTX_PDOWN_CTRL			0x18
+#define  CBTX_RX_ERR_COUNTER			0x1a
+#define  CBTX_IRQ_STAT				0x1d
+#define  CBTX_IRQ_ENABLE			0x1e
+
+/* Fields */
+#define CBTX_MODE_CTRL_STAT_AUTO_MDIX_EN	BIT(7)
+#define CBTX_MODE_CTRL_STAT_MDIX_MODE		BIT(6)
+
+#define CBTX_PDOWN_CTL_TRUE_PDOWN		BIT(0)
+
+#define CBTX_IRQ_ENERGYON			BIT(7)
+#define CBTX_IRQ_AN_COMPLETE			BIT(6)
+#define CBTX_IRQ_REM_FAULT			BIT(5)
+#define CBTX_IRQ_LINK_DOWN			BIT(4)
+#define CBTX_IRQ_AN_LP_ACK			BIT(3)
+#define CBTX_IRQ_PARALLEL_DETECT_FAULT		BIT(2)
+#define CBTX_IRQ_AN_PAGE_RECV			BIT(1)
+
+static int cbtx_soft_reset(struct phy_device *phydev)
+{
+	int ret;
+
+	/* Can't soft reset unless we remove PHY from true power down mode */
+	ret = phy_clear_bits(phydev, CBTX_PDOWN_CTRL,
+			     CBTX_PDOWN_CTL_TRUE_PDOWN);
+	if (ret)
+		return ret;
+
+	return genphy_soft_reset(phydev);
+}
+
+static int cbtx_config_init(struct phy_device *phydev)
+{
+	/* Wait for cbtx_config_aneg() to kick in and apply this */
+	phydev->mdix_ctrl = ETH_TP_MDI_AUTO;
+
+	return 0;
+}
+
+static int cbtx_mdix_status(struct phy_device *phydev)
+{
+	int ret;
+
+	ret = phy_read(phydev, CBTX_MODE_CTRL_STAT);
+	if (ret < 0)
+		return ret;
+
+	if (ret & CBTX_MODE_CTRL_STAT_MDIX_MODE)
+		phydev->mdix = ETH_TP_MDI_X;
+	else
+		phydev->mdix = ETH_TP_MDI;
+
+	return 0;
+}
+
+static int cbtx_read_status(struct phy_device *phydev)
+{
+	int ret;
+
+	ret = cbtx_mdix_status(phydev);
+	if (ret)
+		return ret;
+
+	return genphy_read_status(phydev);
+}
+
+static int cbtx_mdix_config(struct phy_device *phydev)
+{
+	int ret;
+
+	switch (phydev->mdix_ctrl) {
+	case ETH_TP_MDI_AUTO:
+		return phy_set_bits(phydev, CBTX_MODE_CTRL_STAT,
+				    CBTX_MODE_CTRL_STAT_AUTO_MDIX_EN);
+	case ETH_TP_MDI:
+		ret = phy_clear_bits(phydev, CBTX_MODE_CTRL_STAT,
+				     CBTX_MODE_CTRL_STAT_AUTO_MDIX_EN);
+		if (ret)
+			return ret;
+
+		return phy_clear_bits(phydev, CBTX_MODE_CTRL_STAT,
+				      CBTX_MODE_CTRL_STAT_MDIX_MODE);
+	case ETH_TP_MDI_X:
+		ret = phy_clear_bits(phydev, CBTX_MODE_CTRL_STAT,
+				     CBTX_MODE_CTRL_STAT_AUTO_MDIX_EN);
+		if (ret)
+			return ret;
+
+		return phy_set_bits(phydev, CBTX_MODE_CTRL_STAT,
+				    CBTX_MODE_CTRL_STAT_MDIX_MODE);
+	}
+
+	return 0;
+}
+
+static int cbtx_config_aneg(struct phy_device *phydev)
+{
+	int ret;
+
+	ret = cbtx_mdix_config(phydev);
+	if (ret)
+		return ret;
+
+	return genphy_config_aneg(phydev);
+}
+
+static int cbtx_ack_interrupts(struct phy_device *phydev)
+{
+	return phy_read(phydev, CBTX_IRQ_STAT);
+}
+
+static int cbtx_config_intr(struct phy_device *phydev)
+{
+	int ret;
+
+	if (phydev->interrupts == PHY_INTERRUPT_ENABLED) {
+		ret = cbtx_ack_interrupts(phydev);
+		if (ret < 0)
+			return ret;
+
+		ret = phy_write(phydev, CBTX_IRQ_ENABLE, CBTX_IRQ_LINK_DOWN |
+				CBTX_IRQ_AN_COMPLETE | CBTX_IRQ_ENERGYON);
+		if (ret)
+			return ret;
+	} else {
+		ret = phy_write(phydev, CBTX_IRQ_ENABLE, 0);
+		if (ret)
+			return ret;
+
+		ret = cbtx_ack_interrupts(phydev);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static irqreturn_t cbtx_handle_interrupt(struct phy_device *phydev)
+{
+	int irq_stat, irq_enabled;
+
+	irq_stat = cbtx_ack_interrupts(phydev);
+	if (irq_stat < 0) {
+		phy_error(phydev);
+		return IRQ_NONE;
+	}
+
+	irq_enabled = phy_read(phydev, CBTX_IRQ_ENABLE);
+	if (irq_enabled < 0) {
+		phy_error(phydev);
+		return IRQ_NONE;
+	}
+
+	if (!(irq_enabled & irq_stat))
+		return IRQ_NONE;
+
+	phy_trigger_machine(phydev);
+
+	return IRQ_HANDLED;
+}
+
+static int cbtx_get_sset_count(struct phy_device *phydev)
+{
+	return 1;
+}
+
+static void cbtx_get_strings(struct phy_device *phydev, u8 *data)
+{
+	strncpy(data, "100btx_rx_err", ETH_GSTRING_LEN);
+}
+
+static void cbtx_get_stats(struct phy_device *phydev,
+			   struct ethtool_stats *stats, u64 *data)
+{
+	int ret;
+
+	ret = phy_read(phydev, CBTX_RX_ERR_COUNTER);
+	data[0] = (ret < 0) ? U64_MAX : ret;
+}
+
+static struct phy_driver cbtx_driver[] = {
+	{
+		PHY_ID_MATCH_MODEL(PHY_ID_CBTX_SJA1110),
+		.name			= "NXP CBTX (SJA1110)",
+		/* PHY_BASIC_FEATURES */
+		.soft_reset		= cbtx_soft_reset,
+		.config_init		= cbtx_config_init,
+		.suspend		= genphy_suspend,
+		.resume			= genphy_resume,
+		.config_intr		= cbtx_config_intr,
+		.handle_interrupt	= cbtx_handle_interrupt,
+		.read_status		= cbtx_read_status,
+		.config_aneg		= cbtx_config_aneg,
+		.get_sset_count		= cbtx_get_sset_count,
+		.get_strings		= cbtx_get_strings,
+		.get_stats		= cbtx_get_stats,
+	},
+};
+
+module_phy_driver(cbtx_driver);
+
+static struct mdio_device_id __maybe_unused cbtx_tbl[] = {
+	{ PHY_ID_MATCH_MODEL(PHY_ID_CBTX_SJA1110) },
+	{ },
+};
+
+MODULE_DEVICE_TABLE(mdio, cbtx_tbl);
+
+MODULE_AUTHOR("Vladimir Oltean <vladimir.oltean@nxp.com>");
+MODULE_DESCRIPTION("NXP CBTX PHY driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/phy/nxp-tja11xx.c b/drivers/net/phy/nxp-tja11xx.c
index ec91e671f8aa..b13e15310feb 100644
--- a/drivers/net/phy/nxp-tja11xx.c
+++ b/drivers/net/phy/nxp-tja11xx.c
@@ -477,7 +477,7 @@ static umode_t tja11xx_hwmon_is_visible(const void *data,
 	return 0;
 }
 
-static const struct hwmon_channel_info *tja11xx_hwmon_info[] = {
+static const struct hwmon_channel_info * const tja11xx_hwmon_info[] = {
 	HWMON_CHANNEL_INFO(in, HWMON_I_LCRIT_ALARM),
 	HWMON_CHANNEL_INFO(temp, HWMON_T_CRIT_ALARM),
 	NULL
diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c
index 99a07eb54c44..0c0df38cd1ab 100644
--- a/drivers/net/phy/phy.c
+++ b/drivers/net/phy/phy.c
@@ -1181,6 +1181,22 @@ void phy_stop_machine(struct phy_device *phydev)
 	mutex_unlock(&phydev->lock);
 }
 
+static void phy_process_error(struct phy_device *phydev)
+{
+	mutex_lock(&phydev->lock);
+	phydev->state = PHY_HALTED;
+	mutex_unlock(&phydev->lock);
+
+	phy_trigger_machine(phydev);
+}
+
+static void phy_error_precise(struct phy_device *phydev,
+			      const void *func, int err)
+{
+	WARN(1, "%pS: returned: %d\n", func, err);
+	phy_process_error(phydev);
+}
+
 /**
  * phy_error - enter HALTED state for this PHY device
  * @phydev: target phy_device struct
@@ -1193,12 +1209,7 @@ void phy_stop_machine(struct phy_device *phydev)
 void phy_error(struct phy_device *phydev)
 {
 	WARN_ON(1);
-
-	mutex_lock(&phydev->lock);
-	phydev->state = PHY_HALTED;
-	mutex_unlock(&phydev->lock);
-
-	phy_trigger_machine(phydev);
+	phy_process_error(phydev);
 }
 EXPORT_SYMBOL(phy_error);
 
@@ -1393,6 +1404,7 @@ void phy_state_machine(struct work_struct *work)
 	struct net_device *dev = phydev->attached_dev;
 	bool needs_aneg = false, do_suspend = false;
 	enum phy_state old_state;
+	const void *func = NULL;
 	bool finished = false;
 	int err = 0;
 
@@ -1411,6 +1423,7 @@ void phy_state_machine(struct work_struct *work)
 	case PHY_NOLINK:
 	case PHY_RUNNING:
 		err = phy_check_link_status(phydev);
+		func = &phy_check_link_status;
 		break;
 	case PHY_CABLETEST:
 		err = phydev->drv->cable_test_get_status(phydev, &finished);
@@ -1440,16 +1453,18 @@ void phy_state_machine(struct work_struct *work)
 
 	mutex_unlock(&phydev->lock);
 
-	if (needs_aneg)
+	if (needs_aneg) {
 		err = phy_start_aneg(phydev);
-	else if (do_suspend)
+		func = &phy_start_aneg;
+	} else if (do_suspend) {
 		phy_suspend(phydev);
+	}
 
 	if (err == -ENODEV)
 		return;
 
 	if (err < 0)
-		phy_error(phydev);
+		phy_error_precise(phydev, func, err);
 
 	phy_process_state_change(phydev, old_state);
 
diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
index 1de3e339b31a..17d0d0555a79 100644
--- a/drivers/net/phy/phy_device.c
+++ b/drivers/net/phy/phy_device.c
@@ -19,10 +19,12 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
+#include <linux/list.h>
 #include <linux/mdio.h>
 #include <linux/mii.h>
 #include <linux/mm.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/netdevice.h>
 #include <linux/phy.h>
 #include <linux/phy_led_triggers.h>
@@ -674,6 +676,7 @@ struct phy_device *phy_device_create(struct mii_bus *bus, int addr, u32 phy_id,
 	device_initialize(&mdiodev->dev);
 
 	dev->state = PHY_DOWN;
+	INIT_LIST_HEAD(&dev->leds);
 
 	mutex_init(&dev->lock);
 	INIT_DELAYED_WORK(&dev->state_queue, phy_state_machine);
@@ -2988,6 +2991,105 @@ static bool phy_drv_supports_irq(struct phy_driver *phydrv)
 	return phydrv->config_intr && phydrv->handle_interrupt;
 }
 
+static int phy_led_set_brightness(struct led_classdev *led_cdev,
+				  enum led_brightness value)
+{
+	struct phy_led *phyled = to_phy_led(led_cdev);
+	struct phy_device *phydev = phyled->phydev;
+	int err;
+
+	mutex_lock(&phydev->lock);
+	err = phydev->drv->led_brightness_set(phydev, phyled->index, value);
+	mutex_unlock(&phydev->lock);
+
+	return err;
+}
+
+static int phy_led_blink_set(struct led_classdev *led_cdev,
+			     unsigned long *delay_on,
+			     unsigned long *delay_off)
+{
+	struct phy_led *phyled = to_phy_led(led_cdev);
+	struct phy_device *phydev = phyled->phydev;
+	int err;
+
+	mutex_lock(&phydev->lock);
+	err = phydev->drv->led_blink_set(phydev, phyled->index,
+					 delay_on, delay_off);
+	mutex_unlock(&phydev->lock);
+
+	return err;
+}
+
+static int of_phy_led(struct phy_device *phydev,
+		      struct device_node *led)
+{
+	struct device *dev = &phydev->mdio.dev;
+	struct led_init_data init_data = {};
+	struct led_classdev *cdev;
+	struct phy_led *phyled;
+	u32 index;
+	int err;
+
+	phyled = devm_kzalloc(dev, sizeof(*phyled), GFP_KERNEL);
+	if (!phyled)
+		return -ENOMEM;
+
+	cdev = &phyled->led_cdev;
+	phyled->phydev = phydev;
+
+	err = of_property_read_u32(led, "reg", &index);
+	if (err)
+		return err;
+	if (index > U8_MAX)
+		return -EINVAL;
+
+	phyled->index = index;
+	if (phydev->drv->led_brightness_set)
+		cdev->brightness_set_blocking = phy_led_set_brightness;
+	if (phydev->drv->led_blink_set)
+		cdev->blink_set = phy_led_blink_set;
+	cdev->max_brightness = 1;
+	init_data.devicename = dev_name(&phydev->mdio.dev);
+	init_data.fwnode = of_fwnode_handle(led);
+	init_data.devname_mandatory = true;
+
+	err = devm_led_classdev_register_ext(dev, cdev, &init_data);
+	if (err)
+		return err;
+
+	list_add(&phyled->list, &phydev->leds);
+
+	return 0;
+}
+
+static int of_phy_leds(struct phy_device *phydev)
+{
+	struct device_node *node = phydev->mdio.dev.of_node;
+	struct device_node *leds, *led;
+	int err;
+
+	if (!IS_ENABLED(CONFIG_OF_MDIO))
+		return 0;
+
+	if (!node)
+		return 0;
+
+	leds = of_get_child_by_name(node, "leds");
+	if (!leds)
+		return 0;
+
+	for_each_available_child_of_node(leds, led) {
+		err = of_phy_led(phydev, led);
+		if (err) {
+			of_node_put(led);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
 /**
  * fwnode_mdio_find_device - Given a fwnode, find the mdio_device
  * @fwnode: pointer to the mdio_device's fwnode
@@ -3076,9 +3178,7 @@ EXPORT_SYMBOL_GPL(fwnode_get_phy_node);
  * phy_probe - probe and init a PHY device
  * @dev: device to probe and init
  *
- * Description: Take care of setting up the phy_device structure,
- *   set the state to READY (the driver's init function should
- *   set it to STARTING if needed).
+ * Take care of setting up the phy_device structure, set the state to READY.
  */
 static int phy_probe(struct device *dev)
 {
@@ -3185,6 +3285,12 @@ static int phy_probe(struct device *dev)
 	/* Set the state to READY by default */
 	phydev->state = PHY_READY;
 
+	/* Get the LEDs from the device tree, and instantiate standard
+	 * LEDs for them.
+	 */
+	if (IS_ENABLED(CONFIG_PHYLIB_LEDS))
+		err = of_phy_leds(phydev);
+
 out:
 	/* Re-assert the reset signal on error */
 	if (err)
diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c
index 30c166b33468..a4111f1be375 100644
--- a/drivers/net/phy/phylink.c
+++ b/drivers/net/phy/phylink.c
@@ -843,7 +843,6 @@ static int phylink_parse_mode(struct phylink *pl, struct fwnode_handle *fwnode)
 		phylink_set(pl->supported, Autoneg);
 		phylink_set(pl->supported, Asym_Pause);
 		phylink_set(pl->supported, Pause);
-		pl->link_config.an_enabled = true;
 		pl->cfg_link_an_mode = MLO_AN_INBAND;
 
 		switch (pl->link_config.interface) {
@@ -945,9 +944,6 @@ static int phylink_parse_mode(struct phylink *pl, struct fwnode_handle *fwnode)
 				    "failed to validate link configuration for in-band status\n");
 			return -EINVAL;
 		}
-
-		/* Check if MAC/PCS also supports Autoneg. */
-		pl->link_config.an_enabled = phylink_test(pl->supported, Autoneg);
 	}
 
 	return 0;
@@ -957,7 +953,8 @@ static void phylink_apply_manual_flow(struct phylink *pl,
 				      struct phylink_link_state *state)
 {
 	/* If autoneg is disabled, pause AN is also disabled */
-	if (!state->an_enabled)
+	if (!linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
+			       state->advertising))
 		state->pause &= ~MLO_PAUSE_AN;
 
 	/* Manual configuration of pause modes */
@@ -997,21 +994,22 @@ static void phylink_mac_config(struct phylink *pl,
 			       const struct phylink_link_state *state)
 {
 	phylink_dbg(pl,
-		    "%s: mode=%s/%s/%s/%s/%s adv=%*pb pause=%02x link=%u an=%u\n",
+		    "%s: mode=%s/%s/%s/%s/%s adv=%*pb pause=%02x link=%u\n",
 		    __func__, phylink_an_mode_str(pl->cur_link_an_mode),
 		    phy_modes(state->interface),
 		    phy_speed_to_str(state->speed),
 		    phy_duplex_to_str(state->duplex),
 		    phy_rate_matching_to_str(state->rate_matching),
 		    __ETHTOOL_LINK_MODE_MASK_NBITS, state->advertising,
-		    state->pause, state->link, state->an_enabled);
+		    state->pause, state->link);
 
 	pl->mac_ops->mac_config(pl->config, pl->cur_link_an_mode, state);
 }
 
 static void phylink_mac_pcs_an_restart(struct phylink *pl)
 {
-	if (pl->link_config.an_enabled &&
+	if (linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
+			      pl->link_config.advertising) &&
 	    phy_interface_mode_is_8023z(pl->link_config.interface) &&
 	    phylink_autoneg_inband(pl->cur_link_an_mode)) {
 		if (pl->pcs)
@@ -1138,9 +1136,9 @@ static void phylink_mac_pcs_get_state(struct phylink *pl,
 	linkmode_copy(state->advertising, pl->link_config.advertising);
 	linkmode_zero(state->lp_advertising);
 	state->interface = pl->link_config.interface;
-	state->an_enabled = pl->link_config.an_enabled;
 	state->rate_matching = pl->link_config.rate_matching;
-	if (state->an_enabled) {
+	if (linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
+			      state->advertising)) {
 		state->speed = SPEED_UNKNOWN;
 		state->duplex = DUPLEX_UNKNOWN;
 		state->pause = MLO_PAUSE_NONE;
@@ -1531,7 +1529,6 @@ struct phylink *phylink_create(struct phylink_config *config,
 	pl->link_config.pause = MLO_PAUSE_AN;
 	pl->link_config.speed = SPEED_UNKNOWN;
 	pl->link_config.duplex = DUPLEX_UNKNOWN;
-	pl->link_config.an_enabled = true;
 	pl->mac_ops = mac_ops;
 	__set_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state);
 	timer_setup(&pl->link_poll, phylink_fixed_poll, 0);
@@ -2155,8 +2152,9 @@ static void phylink_get_ksettings(const struct phylink_link_state *state,
 		kset->base.speed = state->speed;
 		kset->base.duplex = state->duplex;
 	}
-	kset->base.autoneg = state->an_enabled ? AUTONEG_ENABLE :
-				AUTONEG_DISABLE;
+	kset->base.autoneg = linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
+					       state->advertising) ?
+				AUTONEG_ENABLE : AUTONEG_DISABLE;
 }
 
 /**
@@ -2303,9 +2301,8 @@ int phylink_ethtool_ksettings_set(struct phylink *pl,
 	/* We have ruled out the case with a PHY attached, and the
 	 * fixed-link cases.  All that is left are in-band links.
 	 */
-	config.an_enabled = kset->base.autoneg == AUTONEG_ENABLE;
 	linkmode_mod_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, config.advertising,
-			 config.an_enabled);
+			 kset->base.autoneg == AUTONEG_ENABLE);
 
 	/* If this link is with an SFP, ensure that changes to advertised modes
 	 * also cause the associated interface to be selected such that the
@@ -2339,13 +2336,14 @@ int phylink_ethtool_ksettings_set(struct phylink *pl,
 	}
 
 	/* If autonegotiation is enabled, we must have an advertisement */
-	if (config.an_enabled && phylink_is_empty_linkmode(config.advertising))
+	if (linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
+			      config.advertising) &&
+	    phylink_is_empty_linkmode(config.advertising))
 		return -EINVAL;
 
 	mutex_lock(&pl->state_mutex);
 	pl->link_config.speed = config.speed;
 	pl->link_config.duplex = config.duplex;
-	pl->link_config.an_enabled = config.an_enabled;
 
 	if (pl->link_config.interface != config.interface) {
 		/* The interface changed, e.g. 1000base-X <-> 2500base-X */
@@ -2951,7 +2949,6 @@ static int phylink_sfp_config_phy(struct phylink *pl, u8 mode,
 	config.speed = SPEED_UNKNOWN;
 	config.duplex = DUPLEX_UNKNOWN;
 	config.pause = MLO_PAUSE_AN;
-	config.an_enabled = pl->link_config.an_enabled;
 
 	/* Ignore errors if we're expecting a PHY to attach later */
 	ret = phylink_validate(pl, support, &config);
@@ -3020,7 +3017,6 @@ static int phylink_sfp_config_optical(struct phylink *pl)
 	config.speed = SPEED_UNKNOWN;
 	config.duplex = DUPLEX_UNKNOWN;
 	config.pause = MLO_PAUSE_AN;
-	config.an_enabled = true;
 
 	/* For all the interfaces that are supported, reduce the sfp_support
 	 * mask to only those link modes that can be supported.
@@ -3319,7 +3315,8 @@ void phylink_mii_c22_pcs_decode_state(struct phylink_link_state *state,
 	/* If there is no link or autonegotiation is disabled, the LP advertisement
 	 * data is not meaningful, so don't go any further.
 	 */
-	if (!state->link || !state->an_enabled)
+	if (!state->link || !linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
+					       state->advertising))
 		return;
 
 	switch (state->interface) {
diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c
index 9fc50fcc8fc9..9372e5a4cadc 100644
--- a/drivers/net/phy/sfp-bus.c
+++ b/drivers/net/phy/sfp-bus.c
@@ -151,6 +151,10 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 	unsigned int br_min, br_nom, br_max;
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(modes) = { 0, };
 
+	phylink_set(modes, Autoneg);
+	phylink_set(modes, Pause);
+	phylink_set(modes, Asym_Pause);
+
 	/* Decode the bitrate information to MBd */
 	br_min = br_nom = br_max = 0;
 	if (id->base.br_nominal) {
@@ -329,10 +333,6 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 		bus->sfp_quirk->modes(id, modes, interfaces);
 
 	linkmode_or(support, support, modes);
-
-	phylink_set(support, Autoneg);
-	phylink_set(support, Pause);
-	phylink_set(support, Asym_Pause);
 }
 EXPORT_SYMBOL_GPL(sfp_parse_support);
 
diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c
index bf345032d450..89636dc71e48 100644
--- a/drivers/net/phy/sfp.c
+++ b/drivers/net/phy/sfp.c
@@ -261,6 +261,8 @@ struct sfp {
 	unsigned int module_power_mW;
 	unsigned int module_t_start_up;
 	unsigned int module_t_wait;
+
+	bool have_a2;
 	bool tx_fault_ignore;
 
 	const struct sfp_quirk *quirk;
@@ -364,6 +366,23 @@ static void sfp_quirk_2500basex(const struct sfp_eeprom_id *id,
 	__set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces);
 }
 
+static void sfp_quirk_disable_autoneg(const struct sfp_eeprom_id *id,
+				      unsigned long *modes,
+				      unsigned long *interfaces)
+{
+	linkmode_clear_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, modes);
+}
+
+static void sfp_quirk_oem_2_5g(const struct sfp_eeprom_id *id,
+			       unsigned long *modes,
+			       unsigned long *interfaces)
+{
+	/* Copper 2.5G SFP */
+	linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, modes);
+	__set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces);
+	sfp_quirk_disable_autoneg(id, modes, interfaces);
+}
+
 static void sfp_quirk_ubnt_uf_instant(const struct sfp_eeprom_id *id,
 				      unsigned long *modes,
 				      unsigned long *interfaces)
@@ -409,6 +428,7 @@ static const struct sfp_quirk sfp_quirks[] = {
 	SFP_QUIRK_M("UBNT", "UF-INSTANT", sfp_quirk_ubnt_uf_instant),
 
 	SFP_QUIRK_F("OEM", "SFP-10G-T", sfp_fixup_rollball_cc),
+	SFP_QUIRK_M("OEM", "SFP-2.5G-T", sfp_quirk_oem_2_5g),
 	SFP_QUIRK_F("OEM", "RTSFP-10", sfp_fixup_rollball_cc),
 	SFP_QUIRK_F("OEM", "RTSFP-10G", sfp_fixup_rollball_cc),
 	SFP_QUIRK_F("Turris", "RTSFP-10", sfp_fixup_rollball),
@@ -1368,7 +1388,7 @@ static const struct hwmon_ops sfp_hwmon_ops = {
 	.read_string = sfp_hwmon_read_string,
 };
 
-static const struct hwmon_channel_info *sfp_hwmon_info[] = {
+static const struct hwmon_channel_info * const sfp_hwmon_info[] = {
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_REGISTER_TZ),
 	HWMON_CHANNEL_INFO(in,
@@ -1463,20 +1483,10 @@ static void sfp_hwmon_probe(struct work_struct *work)
 
 static int sfp_hwmon_insert(struct sfp *sfp)
 {
-	if (sfp->id.ext.sff8472_compliance == SFP_SFF8472_COMPLIANCE_NONE)
-		return 0;
-
-	if (!(sfp->id.ext.diagmon & SFP_DIAGMON_DDM))
-		return 0;
-
-	if (sfp->id.ext.diagmon & SFP_DIAGMON_ADDRMODE)
-		/* This driver in general does not support address
-		 * change.
-		 */
-		return 0;
-
-	mod_delayed_work(system_wq, &sfp->hwmon_probe, 1);
-	sfp->hwmon_tries = R_PROBE_RETRY_SLOW;
+	if (sfp->have_a2 && sfp->id.ext.diagmon & SFP_DIAGMON_DDM) {
+		mod_delayed_work(system_wq, &sfp->hwmon_probe, 1);
+		sfp->hwmon_tries = R_PROBE_RETRY_SLOW;
+	}
 
 	return 0;
 }
@@ -1926,6 +1936,18 @@ static int sfp_cotsworks_fixup_check(struct sfp *sfp, struct sfp_eeprom_id *id)
 	return 0;
 }
 
+static int sfp_module_parse_sff8472(struct sfp *sfp)
+{
+	/* If the module requires address swap mode, warn about it */
+	if (sfp->id.ext.diagmon & SFP_DIAGMON_ADDRMODE)
+		dev_warn(sfp->dev,
+			 "module address swap to access page 0xA2 is not supported.\n");
+	else
+		sfp->have_a2 = true;
+
+	return 0;
+}
+
 static int sfp_sm_mod_probe(struct sfp *sfp, bool report)
 {
 	/* SFP module inserted - read I2C data */
@@ -2059,10 +2081,11 @@ static int sfp_sm_mod_probe(struct sfp *sfp, bool report)
 		return -EINVAL;
 	}
 
-	/* If the module requires address swap mode, warn about it */
-	if (sfp->id.ext.diagmon & SFP_DIAGMON_ADDRMODE)
-		dev_warn(sfp->dev,
-			 "module address swap to access page 0xA2 is not supported.\n");
+	if (sfp->id.ext.sff8472_compliance != SFP_SFF8472_COMPLIANCE_NONE) {
+		ret = sfp_module_parse_sff8472(sfp);
+		if (ret < 0)
+			return ret;
+	}
 
 	/* Parse the module power requirement */
 	ret = sfp_module_parse_power(sfp);
@@ -2109,6 +2132,7 @@ static void sfp_sm_mod_remove(struct sfp *sfp)
 
 	memset(&sfp->id, 0, sizeof(sfp->id));
 	sfp->module_power_mW = 0;
+	sfp->have_a2 = false;
 
 	dev_info(sfp->dev, "module removed\n");
 }
@@ -2289,7 +2313,11 @@ static void sfp_sm_main(struct sfp *sfp, unsigned int event)
 		    sfp->sm_dev_state != SFP_DEV_UP)
 			break;
 
-		if (!(sfp->id.ext.diagmon & SFP_DIAGMON_ADDRMODE))
+		/* Only use the soft state bits if we have access to the A2h
+		 * memory, which implies that we have some level of SFF-8472
+		 * compliance.
+		 */
+		if (sfp->have_a2)
 			sfp_soft_start_poll(sfp);
 
 		sfp_module_tx_enable(sfp);
diff --git a/drivers/net/phy/smsc.c b/drivers/net/phy/smsc.c
index df2c5435c5c4..692930750215 100644
--- a/drivers/net/phy/smsc.c
+++ b/drivers/net/phy/smsc.c
@@ -33,6 +33,10 @@
 #define SPECIAL_CTRL_STS_AMDIX_ENABLE_	0x4000
 #define SPECIAL_CTRL_STS_AMDIX_STATE_	0x2000
 
+#define EDPD_MAX_WAIT_DFLT_MS		640
+/* interval between phylib state machine runs in ms */
+#define PHY_STATE_MACH_MS		1000
+
 struct smsc_hw_stat {
 	const char *string;
 	u8 reg;
@@ -44,7 +48,9 @@ static struct smsc_hw_stat smsc_hw_stats[] = {
 };
 
 struct smsc_phy_priv {
-	bool energy_enable;
+	unsigned int edpd_enable:1;
+	unsigned int edpd_mode_set_by_user:1;
+	unsigned int edpd_max_wait_ms;
 };
 
 static int smsc_phy_ack_interrupt(struct phy_device *phydev)
@@ -54,7 +60,7 @@ static int smsc_phy_ack_interrupt(struct phy_device *phydev)
 	return rc < 0 ? rc : 0;
 }
 
-static int smsc_phy_config_intr(struct phy_device *phydev)
+int smsc_phy_config_intr(struct phy_device *phydev)
 {
 	int rc;
 
@@ -75,8 +81,21 @@ static int smsc_phy_config_intr(struct phy_device *phydev)
 
 	return rc < 0 ? rc : 0;
 }
+EXPORT_SYMBOL_GPL(smsc_phy_config_intr);
+
+static int smsc_phy_config_edpd(struct phy_device *phydev)
+{
+	struct smsc_phy_priv *priv = phydev->priv;
+
+	if (priv->edpd_enable)
+		return phy_set_bits(phydev, MII_LAN83C185_CTRL_STATUS,
+				    MII_LAN83C185_EDPWRDOWN);
+	else
+		return phy_clear_bits(phydev, MII_LAN83C185_CTRL_STATUS,
+				      MII_LAN83C185_EDPWRDOWN);
+}
 
-static irqreturn_t smsc_phy_handle_interrupt(struct phy_device *phydev)
+irqreturn_t smsc_phy_handle_interrupt(struct phy_device *phydev)
 {
 	int irq_status;
 
@@ -95,25 +114,22 @@ static irqreturn_t smsc_phy_handle_interrupt(struct phy_device *phydev)
 
 	return IRQ_HANDLED;
 }
+EXPORT_SYMBOL_GPL(smsc_phy_handle_interrupt);
 
-static int smsc_phy_config_init(struct phy_device *phydev)
+int smsc_phy_config_init(struct phy_device *phydev)
 {
 	struct smsc_phy_priv *priv = phydev->priv;
-	int rc;
 
-	if (!priv->energy_enable || phydev->irq != PHY_POLL)
+	if (!priv)
 		return 0;
 
-	rc = phy_read(phydev, MII_LAN83C185_CTRL_STATUS);
-
-	if (rc < 0)
-		return rc;
+	/* don't use EDPD in irq mode except overridden by user */
+	if (!priv->edpd_mode_set_by_user && phydev->irq != PHY_POLL)
+		priv->edpd_enable = false;
 
-	/* Enable energy detect mode for this SMSC Transceivers */
-	rc = phy_write(phydev, MII_LAN83C185_CTRL_STATUS,
-		       rc | MII_LAN83C185_EDPWRDOWN);
-	return rc;
+	return smsc_phy_config_edpd(phydev);
 }
+EXPORT_SYMBOL_GPL(smsc_phy_config_init);
 
 static int smsc_phy_reset(struct phy_device *phydev)
 {
@@ -170,18 +186,15 @@ static int lan87xx_config_aneg(struct phy_device *phydev)
 
 static int lan95xx_config_aneg_ext(struct phy_device *phydev)
 {
-	int rc;
-
-	if (phydev->phy_id != 0x0007c0f0) /* not (LAN9500A or LAN9505A) */
-		return lan87xx_config_aneg(phydev);
+	if (phydev->phy_id == 0x0007c0f0) { /* LAN9500A or LAN9505A */
+		/* Extend Manual AutoMDIX timer */
+		int rc = phy_set_bits(phydev, PHY_EDPD_CONFIG,
+				      PHY_EDPD_CONFIG_EXT_CROSSOVER_);
 
-	/* Extend Manual AutoMDIX timer */
-	rc = phy_read(phydev, PHY_EDPD_CONFIG);
-	if (rc < 0)
-		return rc;
+		if (rc < 0)
+			return rc;
+	}
 
-	rc |= PHY_EDPD_CONFIG_EXT_CROSSOVER_;
-	phy_write(phydev, PHY_EDPD_CONFIG, rc);
 	return lan87xx_config_aneg(phydev);
 }
 
@@ -196,7 +209,7 @@ static int lan95xx_config_aneg_ext(struct phy_device *phydev)
  * The workaround is only applicable to poll mode. Energy Detect Power-Down may
  * not be used in interrupt mode lest link change detection becomes unreliable.
  */
-static int lan87xx_read_status(struct phy_device *phydev)
+int lan87xx_read_status(struct phy_device *phydev)
 {
 	struct smsc_phy_priv *priv = phydev->priv;
 	int err;
@@ -205,9 +218,13 @@ static int lan87xx_read_status(struct phy_device *phydev)
 	if (err)
 		return err;
 
-	if (!phydev->link && priv->energy_enable && phydev->irq == PHY_POLL) {
+	if (!phydev->link && priv && priv->edpd_enable &&
+	    priv->edpd_max_wait_ms) {
+		unsigned int max_wait = priv->edpd_max_wait_ms * 1000;
+		int rc;
+
 		/* Disable EDPD to wake up PHY */
-		int rc = phy_read(phydev, MII_LAN83C185_CTRL_STATUS);
+		rc = phy_read(phydev, MII_LAN83C185_CTRL_STATUS);
 		if (rc < 0)
 			return rc;
 
@@ -221,7 +238,7 @@ static int lan87xx_read_status(struct phy_device *phydev)
 		 */
 		read_poll_timeout(phy_read, rc,
 				  rc & MII_LAN83C185_ENERGYON || rc < 0,
-				  10000, 640000, true, phydev,
+				  10000, max_wait, true, phydev,
 				  MII_LAN83C185_CTRL_STATUS);
 		if (rc < 0)
 			return rc;
@@ -239,6 +256,7 @@ static int lan87xx_read_status(struct phy_device *phydev)
 
 	return err;
 }
+EXPORT_SYMBOL_GPL(lan87xx_read_status);
 
 static int smsc_get_sset_count(struct phy_device *phydev)
 {
@@ -279,10 +297,82 @@ static void smsc_get_stats(struct phy_device *phydev,
 		data[i] = smsc_get_stat(phydev, i);
 }
 
-static int smsc_phy_probe(struct phy_device *phydev)
+static int smsc_phy_get_edpd(struct phy_device *phydev, u16 *edpd)
+{
+	struct smsc_phy_priv *priv = phydev->priv;
+
+	if (!priv)
+		return -EOPNOTSUPP;
+
+	if (!priv->edpd_enable)
+		*edpd = ETHTOOL_PHY_EDPD_DISABLE;
+	else if (!priv->edpd_max_wait_ms)
+		*edpd = ETHTOOL_PHY_EDPD_NO_TX;
+	else
+		*edpd = PHY_STATE_MACH_MS + priv->edpd_max_wait_ms;
+
+	return 0;
+}
+
+static int smsc_phy_set_edpd(struct phy_device *phydev, u16 edpd)
+{
+	struct smsc_phy_priv *priv = phydev->priv;
+
+	if (!priv)
+		return -EOPNOTSUPP;
+
+	switch (edpd) {
+	case ETHTOOL_PHY_EDPD_DISABLE:
+		priv->edpd_enable = false;
+		break;
+	case ETHTOOL_PHY_EDPD_NO_TX:
+		priv->edpd_enable = true;
+		priv->edpd_max_wait_ms = 0;
+		break;
+	case ETHTOOL_PHY_EDPD_DFLT_TX_MSECS:
+		edpd = PHY_STATE_MACH_MS + EDPD_MAX_WAIT_DFLT_MS;
+		fallthrough;
+	default:
+		if (phydev->irq != PHY_POLL)
+			return -EOPNOTSUPP;
+		if (edpd < PHY_STATE_MACH_MS || edpd > PHY_STATE_MACH_MS + 1000)
+			return -EINVAL;
+		priv->edpd_enable = true;
+		priv->edpd_max_wait_ms = edpd - PHY_STATE_MACH_MS;
+	}
+
+	priv->edpd_mode_set_by_user = true;
+
+	return smsc_phy_config_edpd(phydev);
+}
+
+int smsc_phy_get_tunable(struct phy_device *phydev,
+			 struct ethtool_tunable *tuna, void *data)
+{
+	switch (tuna->id) {
+	case ETHTOOL_PHY_EDPD:
+		return smsc_phy_get_edpd(phydev, data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+EXPORT_SYMBOL_GPL(smsc_phy_get_tunable);
+
+int smsc_phy_set_tunable(struct phy_device *phydev,
+			 struct ethtool_tunable *tuna, const void *data)
+{
+	switch (tuna->id) {
+	case ETHTOOL_PHY_EDPD:
+		return smsc_phy_set_edpd(phydev, *(u16 *)data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+EXPORT_SYMBOL_GPL(smsc_phy_set_tunable);
+
+int smsc_phy_probe(struct phy_device *phydev)
 {
 	struct device *dev = &phydev->mdio.dev;
-	struct device_node *of_node = dev->of_node;
 	struct smsc_phy_priv *priv;
 	struct clk *refclk;
 
@@ -290,10 +380,11 @@ static int smsc_phy_probe(struct phy_device *phydev)
 	if (!priv)
 		return -ENOMEM;
 
-	priv->energy_enable = true;
+	priv->edpd_enable = true;
+	priv->edpd_max_wait_ms = EDPD_MAX_WAIT_DFLT_MS;
 
-	if (of_property_read_bool(of_node, "smsc,disable-energy-detect"))
-		priv->energy_enable = false;
+	if (device_property_present(dev, "smsc,disable-energy-detect"))
+		priv->edpd_enable = false;
 
 	phydev->priv = priv;
 
@@ -305,6 +396,7 @@ static int smsc_phy_probe(struct phy_device *phydev)
 
 	return clk_set_rate(refclk, 50 * 1000 * 1000);
 }
+EXPORT_SYMBOL_GPL(smsc_phy_probe);
 
 static struct phy_driver smsc_phy_driver[] = {
 {
@@ -377,6 +469,9 @@ static struct phy_driver smsc_phy_driver[] = {
 	.get_strings	= smsc_get_strings,
 	.get_stats	= smsc_get_stats,
 
+	.get_tunable	= smsc_phy_get_tunable,
+	.set_tunable	= smsc_phy_set_tunable,
+
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
 }, {
@@ -421,6 +516,9 @@ static struct phy_driver smsc_phy_driver[] = {
 	.get_strings	= smsc_get_strings,
 	.get_stats	= smsc_get_stats,
 
+	.get_tunable	= smsc_phy_get_tunable,
+	.set_tunable	= smsc_phy_set_tunable,
+
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
 }, {
@@ -447,6 +545,9 @@ static struct phy_driver smsc_phy_driver[] = {
 	.get_strings	= smsc_get_strings,
 	.get_stats	= smsc_get_stats,
 
+	.get_tunable	= smsc_phy_get_tunable,
+	.set_tunable	= smsc_phy_set_tunable,
+
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
 }, {
@@ -477,6 +578,9 @@ static struct phy_driver smsc_phy_driver[] = {
 	.get_strings	= smsc_get_strings,
 	.get_stats	= smsc_get_stats,
 
+	.get_tunable	= smsc_phy_get_tunable,
+	.set_tunable	= smsc_phy_set_tunable,
+
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
 } };
diff --git a/drivers/net/phy/spi_ks8995.c b/drivers/net/phy/spi_ks8995.c
index d4202d40d47a..7196e927c2cd 100644
--- a/drivers/net/phy/spi_ks8995.c
+++ b/drivers/net/phy/spi_ks8995.c
@@ -491,7 +491,7 @@ static void ks8995_remove(struct spi_device *spi)
 static struct spi_driver ks8995_driver = {
 	.driver = {
 		.name	    = "spi-ks8995",
-		.of_match_table = of_match_ptr(ks8895_spi_of_match),
+		.of_match_table = ks8895_spi_of_match,
 	},
 	.probe	  = ks8995_probe,
 	.remove	  = ks8995_remove,
diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
index 1d71f5276241..a9beacd552cf 100644
--- a/drivers/net/ppp/ppp_generic.c
+++ b/drivers/net/ppp/ppp_generic.c
@@ -1394,7 +1394,7 @@ static int __init ppp_init(void)
 		goto out_net;
 	}
 
-	ppp_class = class_create(THIS_MODULE, "ppp");
+	ppp_class = class_create("ppp");
 	if (IS_ERR(ppp_class)) {
 		err = PTR_ERR(ppp_class);
 		goto out_chrdev;
diff --git a/drivers/net/rionet.c b/drivers/net/rionet.c
index fbcb9d05da64..4eececc94513 100644
--- a/drivers/net/rionet.c
+++ b/drivers/net/rionet.c
@@ -662,8 +662,7 @@ static int rionet_shutdown(struct notifier_block *nb, unsigned long code,
 	return NOTIFY_DONE;
 }
 
-static void rionet_remove_mport(struct device *dev,
-				struct class_interface *class_intf)
+static void rionet_remove_mport(struct device *dev)
 {
 	struct rio_mport *mport = to_rio_mport(dev);
 	struct net_device *ndev;
diff --git a/drivers/net/tap.c b/drivers/net/tap.c
index 8941aa199ea3..ce993cc75bf3 100644
--- a/drivers/net/tap.c
+++ b/drivers/net/tap.c
@@ -555,6 +555,9 @@ static int tap_open(struct inode *inode, struct file *file)
 		goto err_put;
 	}
 
+	/* tap groks IOCB_NOWAIT just fine, mark it as such */
+	file->f_mode |= FMODE_NOWAIT;
+
 	dev_put(tap->dev);
 
 	rtnl_unlock();
@@ -771,8 +774,12 @@ static ssize_t tap_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
 	struct file *file = iocb->ki_filp;
 	struct tap_queue *q = file->private_data;
+	int noblock = 0;
+
+	if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT))
+		noblock = 1;
 
-	return tap_get_user(q, NULL, from, file->f_flags & O_NONBLOCK);
+	return tap_get_user(q, NULL, from, noblock);
 }
 
 /* Put packet to the user space buffer */
@@ -888,8 +895,12 @@ static ssize_t tap_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	struct file *file = iocb->ki_filp;
 	struct tap_queue *q = file->private_data;
 	ssize_t len = iov_iter_count(to), ret;
+	int noblock = 0;
+
+	if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT))
+		noblock = 1;
 
-	ret = tap_do_read(q, to, file->f_flags & O_NONBLOCK, NULL);
+	ret = tap_do_read(q, to, noblock, NULL);
 	ret = min_t(ssize_t, ret, len);
 	if (ret > 0)
 		iocb->ki_pos = ret;
diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c
index 26ef3706445e..0c1e8970ee58 100644
--- a/drivers/net/thunderbolt/main.c
+++ b/drivers/net/thunderbolt/main.c
@@ -148,7 +148,7 @@ struct tbnet_ring {
 /**
  * struct tbnet - ThunderboltIP network driver private data
  * @svc: XDomain service the driver is bound to
- * @xd: XDomain the service blongs to
+ * @xd: XDomain the service belongs to
  * @handler: ThunderboltIP configuration protocol handler
  * @dev: Networking device
  * @napi: NAPI structure for Rx polling
@@ -764,7 +764,7 @@ static bool tbnet_check_frame(struct tbnet *net, const struct tbnet_frame *tf,
 	 */
 	if (net->skb && net->rx_hdr.frame_count) {
 		/* Check the frame count fits the count field */
-		if (frame_count != net->rx_hdr.frame_count) {
+		if (frame_count != le32_to_cpu(net->rx_hdr.frame_count)) {
 			net->stats.rx_length_errors++;
 			return false;
 		}
@@ -772,8 +772,8 @@ static bool tbnet_check_frame(struct tbnet *net, const struct tbnet_frame *tf,
 		/* Check the frame identifiers are incremented correctly,
 		 * and id is matching.
 		 */
-		if (frame_index != net->rx_hdr.frame_index + 1 ||
-		    frame_id != net->rx_hdr.frame_id) {
+		if (frame_index != le16_to_cpu(net->rx_hdr.frame_index) + 1 ||
+		    frame_id != le16_to_cpu(net->rx_hdr.frame_id)) {
 			net->stats.rx_missed_errors++;
 			return false;
 		}
@@ -873,11 +873,12 @@ static int tbnet_poll(struct napi_struct *napi, int budget)
 					TBNET_RX_PAGE_SIZE - hdr_size);
 		}
 
-		net->rx_hdr.frame_size = frame_size;
-		net->rx_hdr.frame_count = le32_to_cpu(hdr->frame_count);
-		net->rx_hdr.frame_index = le16_to_cpu(hdr->frame_index);
-		net->rx_hdr.frame_id = le16_to_cpu(hdr->frame_id);
-		last = net->rx_hdr.frame_index == net->rx_hdr.frame_count - 1;
+		net->rx_hdr.frame_size = hdr->frame_size;
+		net->rx_hdr.frame_count = hdr->frame_count;
+		net->rx_hdr.frame_index = hdr->frame_index;
+		net->rx_hdr.frame_id = hdr->frame_id;
+		last = le16_to_cpu(net->rx_hdr.frame_index) ==
+		       le32_to_cpu(net->rx_hdr.frame_count) - 1;
 
 		rx_packets++;
 		net->stats.rx_bytes += frame_size;
@@ -990,8 +991,10 @@ static bool tbnet_xmit_csum_and_map(struct tbnet *net, struct sk_buff *skb,
 {
 	struct thunderbolt_ip_frame_header *hdr = page_address(frames[0]->page);
 	struct device *dma_dev = tb_ring_dma_device(net->tx_ring.ring);
-	__wsum wsum = htonl(skb->len - skb_transport_offset(skb));
 	unsigned int i, len, offset = skb_transport_offset(skb);
+	/* Remove payload length from checksum */
+	u32 paylen = skb->len - skb_transport_offset(skb);
+	__wsum wsum = (__force __wsum)htonl(paylen);
 	__be16 protocol = skb->protocol;
 	void *data = skb->data;
 	void *dest = hdr + 1;
@@ -1027,7 +1030,7 @@ static bool tbnet_xmit_csum_and_map(struct tbnet *net, struct sk_buff *skb,
 	/* Data points on the beginning of packet.
 	 * Check is the checksum absolute place in the packet.
 	 * ipcso will update IP checksum.
-	 * tucso will update TCP/UPD checksum.
+	 * tucso will update TCP/UDP checksum.
 	 */
 	if (protocol == htons(ETH_P_IP)) {
 		__sum16 *ipcso = dest + ((void *)&(ip_hdr(skb)->check) - data);
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index ad653b32b2f0..d4d0a41a905a 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -1486,7 +1486,8 @@ static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
 	skb->truesize += skb->data_len;
 
 	for (i = 1; i < it->nr_segs; i++) {
-		size_t fragsz = it->iov[i].iov_len;
+		const struct iovec *iov = iter_iov(it);
+		size_t fragsz = iov->iov_len;
 		struct page *page;
 		void *frag;
 
@@ -3463,6 +3464,8 @@ static int tun_chr_open(struct inode *inode, struct file * file)
 
 	sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);
 
+	/* tun groks IOCB_NOWAIT just fine, mark it as such */
+	file->f_mode |= FMODE_NOWAIT;
 	return 0;
 }
 
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index e1b38fbf1dd9..dce9f9d63e04 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -26,6 +26,7 @@
 #include <linux/ptr_ring.h>
 #include <linux/bpf_trace.h>
 #include <linux/net_tstamp.h>
+#include <net/page_pool.h>
 
 #define DRV_NAME	"veth"
 #define DRV_VERSION	"1.0"
@@ -65,6 +66,7 @@ struct veth_rq {
 	bool			rx_notify_masked;
 	struct ptr_ring		xdp_ring;
 	struct xdp_rxq_info	xdp_rxq;
+	struct page_pool	*page_pool;
 };
 
 struct veth_priv {
@@ -155,6 +157,8 @@ static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
 			for (j = 0; j < VETH_TQ_STATS_LEN; j++)
 				ethtool_sprintf(&p, "tx_queue_%u_%.18s",
 						i, veth_tq_stats_desc[j].desc);
+
+		page_pool_ethtool_stats_get_strings(p);
 		break;
 	}
 }
@@ -165,7 +169,8 @@ static int veth_get_sset_count(struct net_device *dev, int sset)
 	case ETH_SS_STATS:
 		return ARRAY_SIZE(ethtool_stats_keys) +
 		       VETH_RQ_STATS_LEN * dev->real_num_rx_queues +
-		       VETH_TQ_STATS_LEN * dev->real_num_tx_queues;
+		       VETH_TQ_STATS_LEN * dev->real_num_tx_queues +
+		       page_pool_ethtool_stats_get_count();
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -176,7 +181,8 @@ static void veth_get_ethtool_stats(struct net_device *dev,
 {
 	struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
 	struct net_device *peer = rtnl_dereference(priv->peer);
-	int i, j, idx;
+	struct page_pool_stats pp_stats = {};
+	int i, j, idx, pp_idx;
 
 	data[0] = peer ? peer->ifindex : 0;
 	idx = 1;
@@ -195,9 +201,10 @@ static void veth_get_ethtool_stats(struct net_device *dev,
 		} while (u64_stats_fetch_retry(&rq_stats->syncp, start));
 		idx += VETH_RQ_STATS_LEN;
 	}
+	pp_idx = idx;
 
 	if (!peer)
-		return;
+		goto page_pool_stats;
 
 	rcv_priv = netdev_priv(peer);
 	for (i = 0; i < peer->real_num_rx_queues; i++) {
@@ -214,7 +221,16 @@ static void veth_get_ethtool_stats(struct net_device *dev,
 				data[tx_idx + j] += *(u64 *)(base + offset);
 			}
 		} while (u64_stats_fetch_retry(&rq_stats->syncp, start));
+		pp_idx = tx_idx + VETH_TQ_STATS_LEN;
+	}
+
+page_pool_stats:
+	for (i = 0; i < dev->real_num_rx_queues; i++) {
+		if (!priv->rq[i].page_pool)
+			continue;
+		page_pool_get_stats(priv->rq[i].page_pool, &pp_stats);
 	}
+	page_pool_ethtool_stats_get(&data[pp_idx], &pp_stats);
 }
 
 static void veth_get_channels(struct net_device *dev,
@@ -727,17 +743,20 @@ static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq,
 			goto drop;
 
 		/* Allocate skb head */
-		page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
+		page = page_pool_dev_alloc_pages(rq->page_pool);
 		if (!page)
 			goto drop;
 
 		nskb = build_skb(page_address(page), PAGE_SIZE);
 		if (!nskb) {
-			put_page(page);
+			page_pool_put_full_page(rq->page_pool, page, true);
 			goto drop;
 		}
 
 		skb_reserve(nskb, VETH_XDP_HEADROOM);
+		skb_copy_header(nskb, skb);
+		skb_mark_for_recycle(nskb);
+
 		size = min_t(u32, skb->len, max_head_size);
 		if (skb_copy_bits(skb, 0, nskb->data, size)) {
 			consume_skb(nskb);
@@ -745,7 +764,6 @@ static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq,
 		}
 		skb_put(nskb, size);
 
-		skb_copy_header(nskb, skb);
 		head_off = skb_headroom(nskb) - skb_headroom(skb);
 		skb_headers_offset_update(nskb, head_off);
 
@@ -754,7 +772,7 @@ static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq,
 		len = skb->len - off;
 
 		for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) {
-			page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
+			page = page_pool_dev_alloc_pages(rq->page_pool);
 			if (!page) {
 				consume_skb(nskb);
 				goto drop;
@@ -1002,12 +1020,38 @@ static int veth_poll(struct napi_struct *napi, int budget)
 	return done;
 }
 
+static int veth_create_page_pool(struct veth_rq *rq)
+{
+	struct page_pool_params pp_params = {
+		.order = 0,
+		.pool_size = VETH_RING_SIZE,
+		.nid = NUMA_NO_NODE,
+		.dev = &rq->dev->dev,
+	};
+
+	rq->page_pool = page_pool_create(&pp_params);
+	if (IS_ERR(rq->page_pool)) {
+		int err = PTR_ERR(rq->page_pool);
+
+		rq->page_pool = NULL;
+		return err;
+	}
+
+	return 0;
+}
+
 static int __veth_napi_enable_range(struct net_device *dev, int start, int end)
 {
 	struct veth_priv *priv = netdev_priv(dev);
 	int err, i;
 
 	for (i = start; i < end; i++) {
+		err = veth_create_page_pool(&priv->rq[i]);
+		if (err)
+			goto err_page_pool;
+	}
+
+	for (i = start; i < end; i++) {
 		struct veth_rq *rq = &priv->rq[i];
 
 		err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL);
@@ -1027,6 +1071,11 @@ static int __veth_napi_enable_range(struct net_device *dev, int start, int end)
 err_xdp_ring:
 	for (i--; i >= start; i--)
 		ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free);
+err_page_pool:
+	for (i = start; i < end; i++) {
+		page_pool_destroy(priv->rq[i].page_pool);
+		priv->rq[i].page_pool = NULL;
+	}
 
 	return err;
 }
@@ -1056,6 +1105,11 @@ static void veth_napi_del_range(struct net_device *dev, int start, int end)
 		rq->rx_notify_masked = false;
 		ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free);
 	}
+
+	for (i = start; i < end; i++) {
+		page_pool_destroy(priv->rq[i].page_pool);
+		priv->rq[i].page_pool = NULL;
+	}
 }
 
 static void veth_napi_del(struct net_device *dev)
@@ -1262,11 +1316,12 @@ static void veth_set_xdp_features(struct net_device *dev)
 
 	peer = rtnl_dereference(priv->peer);
 	if (peer && peer->real_num_tx_queues <= dev->real_num_rx_queues) {
+		struct veth_priv *priv_peer = netdev_priv(peer);
 		xdp_features_t val = NETDEV_XDP_ACT_BASIC |
 				     NETDEV_XDP_ACT_REDIRECT |
 				     NETDEV_XDP_ACT_RX_SG;
 
-		if (priv->_xdp_prog || veth_gro_requested(dev))
+		if (priv_peer->_xdp_prog || veth_gro_requested(peer))
 			val |= NETDEV_XDP_ACT_NDO_XMIT |
 			       NETDEV_XDP_ACT_NDO_XMIT_SG;
 		xdp_set_features_flag(dev, val);
@@ -1504,19 +1559,23 @@ static int veth_set_features(struct net_device *dev,
 {
 	netdev_features_t changed = features ^ dev->features;
 	struct veth_priv *priv = netdev_priv(dev);
+	struct net_device *peer;
 	int err;
 
 	if (!(changed & NETIF_F_GRO) || !(dev->flags & IFF_UP) || priv->_xdp_prog)
 		return 0;
 
+	peer = rtnl_dereference(priv->peer);
 	if (features & NETIF_F_GRO) {
 		err = veth_napi_enable(dev);
 		if (err)
 			return err;
 
-		xdp_features_set_redirect_target(dev, true);
+		if (peer)
+			xdp_features_set_redirect_target(peer, true);
 	} else {
-		xdp_features_clear_redirect_target(dev);
+		if (peer)
+			xdp_features_clear_redirect_target(peer);
 		veth_napi_del(dev);
 	}
 	return 0;
@@ -1598,13 +1657,13 @@ static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog,
 			peer->max_mtu = max_mtu;
 		}
 
-		xdp_features_set_redirect_target(dev, true);
+		xdp_features_set_redirect_target(peer, true);
 	}
 
 	if (old_prog) {
 		if (!prog) {
-			if (!veth_gro_requested(dev))
-				xdp_features_clear_redirect_target(dev);
+			if (peer && !veth_gro_requested(dev))
+				xdp_features_clear_redirect_target(peer);
 
 			if (dev->flags & IFF_UP)
 				veth_disable_xdp(dev);
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 2396c28c0122..8d8038538fc4 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -62,7 +62,8 @@ static const unsigned long guest_offloads[] = {
 	VIRTIO_NET_F_GUEST_UFO,
 	VIRTIO_NET_F_GUEST_CSUM,
 	VIRTIO_NET_F_GUEST_USO4,
-	VIRTIO_NET_F_GUEST_USO6
+	VIRTIO_NET_F_GUEST_USO6,
+	VIRTIO_NET_F_GUEST_HDRLEN
 };
 
 #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
@@ -814,8 +815,13 @@ static struct page *xdp_linearize_page(struct receive_queue *rq,
 				       int page_off,
 				       unsigned int *len)
 {
-	struct page *page = alloc_page(GFP_ATOMIC);
+	int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+	struct page *page;
+
+	if (page_off + *len + tailroom > PAGE_SIZE)
+		return NULL;
 
+	page = alloc_page(GFP_ATOMIC);
 	if (!page)
 		return NULL;
 
@@ -823,7 +829,6 @@ static struct page *xdp_linearize_page(struct receive_queue *rq,
 	page_off += *len;
 
 	while (--*num_buf) {
-		int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 		unsigned int buflen;
 		void *buf;
 		int off;
@@ -4232,7 +4237,8 @@ static struct virtio_device_id id_table[] = {
 	VIRTIO_NET_F_CTRL_MAC_ADDR, \
 	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
 	VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \
-	VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL
+	VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL, \
+	VIRTIO_NET_F_GUEST_HDRLEN
 
 static unsigned int features[] = {
 	VIRTNET_FEATURES,
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index da488cbb0542..f2b76ee866a4 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -1504,7 +1504,7 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
 				goto rcd_done;
 			}
 
-			if (rxDataRingUsed) {
+			if (rxDataRingUsed && adapter->rxdataring_enabled) {
 				size_t sz;
 
 				BUG_ON(rcd->len > rq->data_ring.desc_size);
diff --git a/drivers/net/vxlan/Makefile b/drivers/net/vxlan/Makefile
index d4c255499b72..91b8fec8b6cf 100644
--- a/drivers/net/vxlan/Makefile
+++ b/drivers/net/vxlan/Makefile
@@ -4,4 +4,4 @@
 
 obj-$(CONFIG_VXLAN) += vxlan.o
 
-vxlan-objs := vxlan_core.o vxlan_multicast.o vxlan_vnifilter.o
+vxlan-objs := vxlan_core.o vxlan_multicast.o vxlan_vnifilter.o vxlan_mdb.o
diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c
index b1b179effe2a..561fe1b314f5 100644
--- a/drivers/net/vxlan/vxlan_core.c
+++ b/drivers/net/vxlan/vxlan_core.c
@@ -71,53 +71,6 @@ static inline bool vxlan_collect_metadata(struct vxlan_sock *vs)
 	       ip_tunnel_collect_metadata();
 }
 
-#if IS_ENABLED(CONFIG_IPV6)
-static int vxlan_nla_get_addr(union vxlan_addr *ip, struct nlattr *nla)
-{
-	if (nla_len(nla) >= sizeof(struct in6_addr)) {
-		ip->sin6.sin6_addr = nla_get_in6_addr(nla);
-		ip->sa.sa_family = AF_INET6;
-		return 0;
-	} else if (nla_len(nla) >= sizeof(__be32)) {
-		ip->sin.sin_addr.s_addr = nla_get_in_addr(nla);
-		ip->sa.sa_family = AF_INET;
-		return 0;
-	} else {
-		return -EAFNOSUPPORT;
-	}
-}
-
-static int vxlan_nla_put_addr(struct sk_buff *skb, int attr,
-			      const union vxlan_addr *ip)
-{
-	if (ip->sa.sa_family == AF_INET6)
-		return nla_put_in6_addr(skb, attr, &ip->sin6.sin6_addr);
-	else
-		return nla_put_in_addr(skb, attr, ip->sin.sin_addr.s_addr);
-}
-
-#else /* !CONFIG_IPV6 */
-
-static int vxlan_nla_get_addr(union vxlan_addr *ip, struct nlattr *nla)
-{
-	if (nla_len(nla) >= sizeof(struct in6_addr)) {
-		return -EAFNOSUPPORT;
-	} else if (nla_len(nla) >= sizeof(__be32)) {
-		ip->sin.sin_addr.s_addr = nla_get_in_addr(nla);
-		ip->sa.sa_family = AF_INET;
-		return 0;
-	} else {
-		return -EAFNOSUPPORT;
-	}
-}
-
-static int vxlan_nla_put_addr(struct sk_buff *skb, int attr,
-			      const union vxlan_addr *ip)
-{
-	return nla_put_in_addr(skb, attr, ip->sin.sin_addr.s_addr);
-}
-#endif
-
 /* Find VXLAN socket based on network namespace, address family, UDP port,
  * enabled unshareable flags and socket device binding (see l3mdev with
  * non-default VRF).
@@ -1863,7 +1816,7 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
 		struct vxlan_fdb *f;
 		struct sk_buff	*reply;
 
-		if (!(n->nud_state & NUD_CONNECTED)) {
+		if (!(READ_ONCE(n->nud_state) & NUD_CONNECTED)) {
 			neigh_release(n);
 			goto out;
 		}
@@ -2027,7 +1980,7 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
 		struct vxlan_fdb *f;
 		struct sk_buff *reply;
 
-		if (!(n->nud_state & NUD_CONNECTED)) {
+		if (!(READ_ONCE(n->nud_state) & NUD_CONNECTED)) {
 			neigh_release(n);
 			goto out;
 		}
@@ -2140,28 +2093,7 @@ static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb)
 	return false;
 }
 
-static void vxlan_build_gbp_hdr(struct vxlanhdr *vxh, u32 vxflags,
-				struct vxlan_metadata *md)
-{
-	struct vxlanhdr_gbp *gbp;
-
-	if (!md->gbp)
-		return;
-
-	gbp = (struct vxlanhdr_gbp *)vxh;
-	vxh->vx_flags |= VXLAN_HF_GBP;
-
-	if (md->gbp & VXLAN_GBP_DONT_LEARN)
-		gbp->dont_learn = 1;
-
-	if (md->gbp & VXLAN_GBP_POLICY_APPLIED)
-		gbp->policy_applied = 1;
-
-	gbp->policy_id = htons(md->gbp & VXLAN_GBP_ID_MASK);
-}
-
-static int vxlan_build_gpe_hdr(struct vxlanhdr *vxh, u32 vxflags,
-			       __be16 protocol)
+static int vxlan_build_gpe_hdr(struct vxlanhdr *vxh, __be16 protocol)
 {
 	struct vxlanhdr_gpe *gpe = (struct vxlanhdr_gpe *)vxh;
 
@@ -2224,9 +2156,9 @@ static int vxlan_build_skb(struct sk_buff *skb, struct dst_entry *dst,
 	}
 
 	if (vxflags & VXLAN_F_GBP)
-		vxlan_build_gbp_hdr(vxh, vxflags, md);
+		vxlan_build_gbp_hdr(vxh, md);
 	if (vxflags & VXLAN_F_GPE) {
-		err = vxlan_build_gpe_hdr(vxh, vxflags, skb->protocol);
+		err = vxlan_build_gpe_hdr(vxh, skb->protocol);
 		if (err < 0)
 			return err;
 		inner_protocol = skb->protocol;
@@ -2442,9 +2374,8 @@ static int encap_bypass_if_local(struct sk_buff *skb, struct net_device *dev,
 	return 0;
 }
 
-static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
-			   __be32 default_vni, struct vxlan_rdst *rdst,
-			   bool did_rsc)
+void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
+		    __be32 default_vni, struct vxlan_rdst *rdst, bool did_rsc)
 {
 	struct dst_cache *dst_cache;
 	struct ip_tunnel_info *info;
@@ -2791,6 +2722,21 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev)
 #endif
 	}
 
+	if (vxlan->cfg.flags & VXLAN_F_MDB) {
+		struct vxlan_mdb_entry *mdb_entry;
+
+		rcu_read_lock();
+		mdb_entry = vxlan_mdb_entry_skb_get(vxlan, skb, vni);
+		if (mdb_entry) {
+			netdev_tx_t ret;
+
+			ret = vxlan_mdb_xmit(vxlan, mdb_entry, skb);
+			rcu_read_unlock();
+			return ret;
+		}
+		rcu_read_unlock();
+	}
+
 	eth = eth_hdr(skb);
 	f = vxlan_find_mac(vxlan, eth->h_dest, vni);
 	did_rsc = false;
@@ -2926,8 +2872,14 @@ static int vxlan_init(struct net_device *dev)
 	if (err)
 		goto err_free_percpu;
 
+	err = vxlan_mdb_init(vxlan);
+	if (err)
+		goto err_gro_cells_destroy;
+
 	return 0;
 
+err_gro_cells_destroy:
+	gro_cells_destroy(&vxlan->gro_cells);
 err_free_percpu:
 	free_percpu(dev->tstats);
 err_vnigroup_uninit:
@@ -2952,6 +2904,8 @@ static void vxlan_uninit(struct net_device *dev)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 
+	vxlan_mdb_fini(vxlan);
+
 	if (vxlan->cfg.flags & VXLAN_F_VNIFILTER)
 		vxlan_vnigroup_uninit(vxlan);
 
@@ -3108,6 +3062,9 @@ static const struct net_device_ops vxlan_netdev_ether_ops = {
 	.ndo_fdb_del		= vxlan_fdb_delete,
 	.ndo_fdb_dump		= vxlan_fdb_dump,
 	.ndo_fdb_get		= vxlan_fdb_get,
+	.ndo_mdb_add		= vxlan_mdb_add,
+	.ndo_mdb_del		= vxlan_mdb_del,
+	.ndo_mdb_dump		= vxlan_mdb_dump,
 	.ndo_fill_metadata_dst	= vxlan_fill_metadata_dst,
 };
 
diff --git a/drivers/net/vxlan/vxlan_mdb.c b/drivers/net/vxlan/vxlan_mdb.c
new file mode 100644
index 000000000000..5e041622261a
--- /dev/null
+++ b/drivers/net/vxlan/vxlan_mdb.c
@@ -0,0 +1,1462 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/if_bridge.h>
+#include <linux/in.h>
+#include <linux/list.h>
+#include <linux/netdevice.h>
+#include <linux/netlink.h>
+#include <linux/rhashtable.h>
+#include <linux/rhashtable-types.h>
+#include <linux/rtnetlink.h>
+#include <linux/skbuff.h>
+#include <linux/types.h>
+#include <net/netlink.h>
+#include <net/vxlan.h>
+
+#include "vxlan_private.h"
+
+struct vxlan_mdb_entry_key {
+	union vxlan_addr src;
+	union vxlan_addr dst;
+	__be32 vni;
+};
+
+struct vxlan_mdb_entry {
+	struct rhash_head rhnode;
+	struct list_head remotes;
+	struct vxlan_mdb_entry_key key;
+	struct hlist_node mdb_node;
+	struct rcu_head rcu;
+};
+
+#define VXLAN_MDB_REMOTE_F_BLOCKED	BIT(0)
+
+struct vxlan_mdb_remote {
+	struct list_head list;
+	struct vxlan_rdst __rcu *rd;
+	u8 flags;
+	u8 filter_mode;
+	u8 rt_protocol;
+	struct hlist_head src_list;
+	struct rcu_head rcu;
+};
+
+#define VXLAN_SGRP_F_DELETE	BIT(0)
+
+struct vxlan_mdb_src_entry {
+	struct hlist_node node;
+	union vxlan_addr addr;
+	u8 flags;
+};
+
+struct vxlan_mdb_dump_ctx {
+	long reserved;
+	long entry_idx;
+	long remote_idx;
+};
+
+struct vxlan_mdb_config_src_entry {
+	union vxlan_addr addr;
+	struct list_head node;
+};
+
+struct vxlan_mdb_config {
+	struct vxlan_dev *vxlan;
+	struct vxlan_mdb_entry_key group;
+	struct list_head src_list;
+	union vxlan_addr remote_ip;
+	u32 remote_ifindex;
+	__be32 remote_vni;
+	__be16 remote_port;
+	u16 nlflags;
+	u8 flags;
+	u8 filter_mode;
+	u8 rt_protocol;
+};
+
+static const struct rhashtable_params vxlan_mdb_rht_params = {
+	.head_offset = offsetof(struct vxlan_mdb_entry, rhnode),
+	.key_offset = offsetof(struct vxlan_mdb_entry, key),
+	.key_len = sizeof(struct vxlan_mdb_entry_key),
+	.automatic_shrinking = true,
+};
+
+static int __vxlan_mdb_add(const struct vxlan_mdb_config *cfg,
+			   struct netlink_ext_ack *extack);
+static int __vxlan_mdb_del(const struct vxlan_mdb_config *cfg,
+			   struct netlink_ext_ack *extack);
+
+static void vxlan_br_mdb_entry_fill(const struct vxlan_dev *vxlan,
+				    const struct vxlan_mdb_entry *mdb_entry,
+				    const struct vxlan_mdb_remote *remote,
+				    struct br_mdb_entry *e)
+{
+	const union vxlan_addr *dst = &mdb_entry->key.dst;
+
+	memset(e, 0, sizeof(*e));
+	e->ifindex = vxlan->dev->ifindex;
+	e->state = MDB_PERMANENT;
+
+	if (remote->flags & VXLAN_MDB_REMOTE_F_BLOCKED)
+		e->flags |= MDB_FLAGS_BLOCKED;
+
+	switch (dst->sa.sa_family) {
+	case AF_INET:
+		e->addr.u.ip4 = dst->sin.sin_addr.s_addr;
+		e->addr.proto = htons(ETH_P_IP);
+		break;
+#if IS_ENABLED(CONFIG_IPV6)
+	case AF_INET6:
+		e->addr.u.ip6 = dst->sin6.sin6_addr;
+		e->addr.proto = htons(ETH_P_IPV6);
+		break;
+#endif
+	}
+}
+
+static int vxlan_mdb_entry_info_fill_srcs(struct sk_buff *skb,
+					  const struct vxlan_mdb_remote *remote)
+{
+	struct vxlan_mdb_src_entry *ent;
+	struct nlattr *nest;
+
+	if (hlist_empty(&remote->src_list))
+		return 0;
+
+	nest = nla_nest_start(skb, MDBA_MDB_EATTR_SRC_LIST);
+	if (!nest)
+		return -EMSGSIZE;
+
+	hlist_for_each_entry(ent, &remote->src_list, node) {
+		struct nlattr *nest_ent;
+
+		nest_ent = nla_nest_start(skb, MDBA_MDB_SRCLIST_ENTRY);
+		if (!nest_ent)
+			goto out_cancel_err;
+
+		if (vxlan_nla_put_addr(skb, MDBA_MDB_SRCATTR_ADDRESS,
+				       &ent->addr) ||
+		    nla_put_u32(skb, MDBA_MDB_SRCATTR_TIMER, 0))
+			goto out_cancel_err;
+
+		nla_nest_end(skb, nest_ent);
+	}
+
+	nla_nest_end(skb, nest);
+
+	return 0;
+
+out_cancel_err:
+	nla_nest_cancel(skb, nest);
+	return -EMSGSIZE;
+}
+
+static int vxlan_mdb_entry_info_fill(const struct vxlan_dev *vxlan,
+				     struct sk_buff *skb,
+				     const struct vxlan_mdb_entry *mdb_entry,
+				     const struct vxlan_mdb_remote *remote)
+{
+	struct vxlan_rdst *rd = rtnl_dereference(remote->rd);
+	struct br_mdb_entry e;
+	struct nlattr *nest;
+
+	nest = nla_nest_start_noflag(skb, MDBA_MDB_ENTRY_INFO);
+	if (!nest)
+		return -EMSGSIZE;
+
+	vxlan_br_mdb_entry_fill(vxlan, mdb_entry, remote, &e);
+
+	if (nla_put_nohdr(skb, sizeof(e), &e) ||
+	    nla_put_u32(skb, MDBA_MDB_EATTR_TIMER, 0))
+		goto nest_err;
+
+	if (!vxlan_addr_any(&mdb_entry->key.src) &&
+	    vxlan_nla_put_addr(skb, MDBA_MDB_EATTR_SOURCE, &mdb_entry->key.src))
+		goto nest_err;
+
+	if (nla_put_u8(skb, MDBA_MDB_EATTR_RTPROT, remote->rt_protocol) ||
+	    nla_put_u8(skb, MDBA_MDB_EATTR_GROUP_MODE, remote->filter_mode) ||
+	    vxlan_mdb_entry_info_fill_srcs(skb, remote) ||
+	    vxlan_nla_put_addr(skb, MDBA_MDB_EATTR_DST, &rd->remote_ip))
+		goto nest_err;
+
+	if (rd->remote_port && rd->remote_port != vxlan->cfg.dst_port &&
+	    nla_put_u16(skb, MDBA_MDB_EATTR_DST_PORT,
+			be16_to_cpu(rd->remote_port)))
+		goto nest_err;
+
+	if (rd->remote_vni != vxlan->default_dst.remote_vni &&
+	    nla_put_u32(skb, MDBA_MDB_EATTR_VNI, be32_to_cpu(rd->remote_vni)))
+		goto nest_err;
+
+	if (rd->remote_ifindex &&
+	    nla_put_u32(skb, MDBA_MDB_EATTR_IFINDEX, rd->remote_ifindex))
+		goto nest_err;
+
+	if ((vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) &&
+	    mdb_entry->key.vni && nla_put_u32(skb, MDBA_MDB_EATTR_SRC_VNI,
+					      be32_to_cpu(mdb_entry->key.vni)))
+		goto nest_err;
+
+	nla_nest_end(skb, nest);
+
+	return 0;
+
+nest_err:
+	nla_nest_cancel(skb, nest);
+	return -EMSGSIZE;
+}
+
+static int vxlan_mdb_entry_fill(const struct vxlan_dev *vxlan,
+				struct sk_buff *skb,
+				struct vxlan_mdb_dump_ctx *ctx,
+				const struct vxlan_mdb_entry *mdb_entry)
+{
+	int remote_idx = 0, s_remote_idx = ctx->remote_idx;
+	struct vxlan_mdb_remote *remote;
+	struct nlattr *nest;
+	int err = 0;
+
+	nest = nla_nest_start_noflag(skb, MDBA_MDB_ENTRY);
+	if (!nest)
+		return -EMSGSIZE;
+
+	list_for_each_entry(remote, &mdb_entry->remotes, list) {
+		if (remote_idx < s_remote_idx)
+			goto skip;
+
+		err = vxlan_mdb_entry_info_fill(vxlan, skb, mdb_entry, remote);
+		if (err)
+			break;
+skip:
+		remote_idx++;
+	}
+
+	ctx->remote_idx = err ? remote_idx : 0;
+	nla_nest_end(skb, nest);
+	return err;
+}
+
+static int vxlan_mdb_fill(const struct vxlan_dev *vxlan, struct sk_buff *skb,
+			  struct vxlan_mdb_dump_ctx *ctx)
+{
+	int entry_idx = 0, s_entry_idx = ctx->entry_idx;
+	struct vxlan_mdb_entry *mdb_entry;
+	struct nlattr *nest;
+	int err = 0;
+
+	nest = nla_nest_start_noflag(skb, MDBA_MDB);
+	if (!nest)
+		return -EMSGSIZE;
+
+	hlist_for_each_entry(mdb_entry, &vxlan->mdb_list, mdb_node) {
+		if (entry_idx < s_entry_idx)
+			goto skip;
+
+		err = vxlan_mdb_entry_fill(vxlan, skb, ctx, mdb_entry);
+		if (err)
+			break;
+skip:
+		entry_idx++;
+	}
+
+	ctx->entry_idx = err ? entry_idx : 0;
+	nla_nest_end(skb, nest);
+	return err;
+}
+
+int vxlan_mdb_dump(struct net_device *dev, struct sk_buff *skb,
+		   struct netlink_callback *cb)
+{
+	struct vxlan_mdb_dump_ctx *ctx = (void *)cb->ctx;
+	struct vxlan_dev *vxlan = netdev_priv(dev);
+	struct br_port_msg *bpm;
+	struct nlmsghdr *nlh;
+	int err;
+
+	ASSERT_RTNL();
+
+	NL_ASSERT_DUMP_CTX_FITS(struct vxlan_mdb_dump_ctx);
+
+	nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+			cb->nlh->nlmsg_seq, RTM_NEWMDB, sizeof(*bpm),
+			NLM_F_MULTI);
+	if (!nlh)
+		return -EMSGSIZE;
+
+	bpm = nlmsg_data(nlh);
+	memset(bpm, 0, sizeof(*bpm));
+	bpm->family = AF_BRIDGE;
+	bpm->ifindex = dev->ifindex;
+
+	err = vxlan_mdb_fill(vxlan, skb, ctx);
+
+	nlmsg_end(skb, nlh);
+
+	cb->seq = vxlan->mdb_seq;
+	nl_dump_check_consistent(cb, nlh);
+
+	return err;
+}
+
+static const struct nla_policy
+vxlan_mdbe_src_list_entry_pol[MDBE_SRCATTR_MAX + 1] = {
+	[MDBE_SRCATTR_ADDRESS] = NLA_POLICY_RANGE(NLA_BINARY,
+						  sizeof(struct in_addr),
+						  sizeof(struct in6_addr)),
+};
+
+static const struct nla_policy
+vxlan_mdbe_src_list_pol[MDBE_SRC_LIST_MAX + 1] = {
+	[MDBE_SRC_LIST_ENTRY] = NLA_POLICY_NESTED(vxlan_mdbe_src_list_entry_pol),
+};
+
+static struct netlink_range_validation vni_range = {
+	.max = VXLAN_N_VID - 1,
+};
+
+static const struct nla_policy vxlan_mdbe_attrs_pol[MDBE_ATTR_MAX + 1] = {
+	[MDBE_ATTR_SOURCE] = NLA_POLICY_RANGE(NLA_BINARY,
+					      sizeof(struct in_addr),
+					      sizeof(struct in6_addr)),
+	[MDBE_ATTR_GROUP_MODE] = NLA_POLICY_RANGE(NLA_U8, MCAST_EXCLUDE,
+						  MCAST_INCLUDE),
+	[MDBE_ATTR_SRC_LIST] = NLA_POLICY_NESTED(vxlan_mdbe_src_list_pol),
+	[MDBE_ATTR_RTPROT] = NLA_POLICY_MIN(NLA_U8, RTPROT_STATIC),
+	[MDBE_ATTR_DST] = NLA_POLICY_RANGE(NLA_BINARY,
+					   sizeof(struct in_addr),
+					   sizeof(struct in6_addr)),
+	[MDBE_ATTR_DST_PORT] = { .type = NLA_U16 },
+	[MDBE_ATTR_VNI] = NLA_POLICY_FULL_RANGE(NLA_U32, &vni_range),
+	[MDBE_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 1),
+	[MDBE_ATTR_SRC_VNI] = NLA_POLICY_FULL_RANGE(NLA_U32, &vni_range),
+};
+
+static bool vxlan_mdb_is_valid_source(const struct nlattr *attr, __be16 proto,
+				      struct netlink_ext_ack *extack)
+{
+	switch (proto) {
+	case htons(ETH_P_IP):
+		if (nla_len(attr) != sizeof(struct in_addr)) {
+			NL_SET_ERR_MSG_MOD(extack, "IPv4 invalid source address length");
+			return false;
+		}
+		if (ipv4_is_multicast(nla_get_in_addr(attr))) {
+			NL_SET_ERR_MSG_MOD(extack, "IPv4 multicast source address is not allowed");
+			return false;
+		}
+		break;
+#if IS_ENABLED(CONFIG_IPV6)
+	case htons(ETH_P_IPV6): {
+		struct in6_addr src;
+
+		if (nla_len(attr) != sizeof(struct in6_addr)) {
+			NL_SET_ERR_MSG_MOD(extack, "IPv6 invalid source address length");
+			return false;
+		}
+		src = nla_get_in6_addr(attr);
+		if (ipv6_addr_is_multicast(&src)) {
+			NL_SET_ERR_MSG_MOD(extack, "IPv6 multicast source address is not allowed");
+			return false;
+		}
+		break;
+	}
+#endif
+	default:
+		NL_SET_ERR_MSG_MOD(extack, "Invalid protocol used with source address");
+		return false;
+	}
+
+	return true;
+}
+
+static void vxlan_mdb_config_group_set(struct vxlan_mdb_config *cfg,
+				       const struct br_mdb_entry *entry,
+				       const struct nlattr *source_attr)
+{
+	struct vxlan_mdb_entry_key *group = &cfg->group;
+
+	switch (entry->addr.proto) {
+	case htons(ETH_P_IP):
+		group->dst.sa.sa_family = AF_INET;
+		group->dst.sin.sin_addr.s_addr = entry->addr.u.ip4;
+		break;
+#if IS_ENABLED(CONFIG_IPV6)
+	case htons(ETH_P_IPV6):
+		group->dst.sa.sa_family = AF_INET6;
+		group->dst.sin6.sin6_addr = entry->addr.u.ip6;
+		break;
+#endif
+	}
+
+	if (source_attr)
+		vxlan_nla_get_addr(&group->src, source_attr);
+}
+
+static bool vxlan_mdb_is_star_g(const struct vxlan_mdb_entry_key *group)
+{
+	return !vxlan_addr_any(&group->dst) && vxlan_addr_any(&group->src);
+}
+
+static bool vxlan_mdb_is_sg(const struct vxlan_mdb_entry_key *group)
+{
+	return !vxlan_addr_any(&group->dst) && !vxlan_addr_any(&group->src);
+}
+
+static int vxlan_mdb_config_src_entry_init(struct vxlan_mdb_config *cfg,
+					   __be16 proto,
+					   const struct nlattr *src_entry,
+					   struct netlink_ext_ack *extack)
+{
+	struct nlattr *tb[MDBE_SRCATTR_MAX + 1];
+	struct vxlan_mdb_config_src_entry *src;
+	int err;
+
+	err = nla_parse_nested(tb, MDBE_SRCATTR_MAX, src_entry,
+			       vxlan_mdbe_src_list_entry_pol, extack);
+	if (err)
+		return err;
+
+	if (NL_REQ_ATTR_CHECK(extack, src_entry, tb, MDBE_SRCATTR_ADDRESS))
+		return -EINVAL;
+
+	if (!vxlan_mdb_is_valid_source(tb[MDBE_SRCATTR_ADDRESS], proto,
+				       extack))
+		return -EINVAL;
+
+	src = kzalloc(sizeof(*src), GFP_KERNEL);
+	if (!src)
+		return -ENOMEM;
+
+	err = vxlan_nla_get_addr(&src->addr, tb[MDBE_SRCATTR_ADDRESS]);
+	if (err)
+		goto err_free_src;
+
+	list_add_tail(&src->node, &cfg->src_list);
+
+	return 0;
+
+err_free_src:
+	kfree(src);
+	return err;
+}
+
+static void
+vxlan_mdb_config_src_entry_fini(struct vxlan_mdb_config_src_entry *src)
+{
+	list_del(&src->node);
+	kfree(src);
+}
+
+static int vxlan_mdb_config_src_list_init(struct vxlan_mdb_config *cfg,
+					  __be16 proto,
+					  const struct nlattr *src_list,
+					  struct netlink_ext_ack *extack)
+{
+	struct vxlan_mdb_config_src_entry *src, *tmp;
+	struct nlattr *src_entry;
+	int rem, err;
+
+	nla_for_each_nested(src_entry, src_list, rem) {
+		err = vxlan_mdb_config_src_entry_init(cfg, proto, src_entry,
+						      extack);
+		if (err)
+			goto err_src_entry_init;
+	}
+
+	return 0;
+
+err_src_entry_init:
+	list_for_each_entry_safe_reverse(src, tmp, &cfg->src_list, node)
+		vxlan_mdb_config_src_entry_fini(src);
+	return err;
+}
+
+static void vxlan_mdb_config_src_list_fini(struct vxlan_mdb_config *cfg)
+{
+	struct vxlan_mdb_config_src_entry *src, *tmp;
+
+	list_for_each_entry_safe_reverse(src, tmp, &cfg->src_list, node)
+		vxlan_mdb_config_src_entry_fini(src);
+}
+
+static int vxlan_mdb_config_attrs_init(struct vxlan_mdb_config *cfg,
+				       const struct br_mdb_entry *entry,
+				       const struct nlattr *set_attrs,
+				       struct netlink_ext_ack *extack)
+{
+	struct nlattr *mdbe_attrs[MDBE_ATTR_MAX + 1];
+	int err;
+
+	err = nla_parse_nested(mdbe_attrs, MDBE_ATTR_MAX, set_attrs,
+			       vxlan_mdbe_attrs_pol, extack);
+	if (err)
+		return err;
+
+	if (NL_REQ_ATTR_CHECK(extack, set_attrs, mdbe_attrs, MDBE_ATTR_DST)) {
+		NL_SET_ERR_MSG_MOD(extack, "Missing remote destination IP address");
+		return -EINVAL;
+	}
+
+	if (mdbe_attrs[MDBE_ATTR_SOURCE] &&
+	    !vxlan_mdb_is_valid_source(mdbe_attrs[MDBE_ATTR_SOURCE],
+				       entry->addr.proto, extack))
+		return -EINVAL;
+
+	vxlan_mdb_config_group_set(cfg, entry, mdbe_attrs[MDBE_ATTR_SOURCE]);
+
+	/* rtnetlink code only validates that IPv4 group address is
+	 * multicast.
+	 */
+	if (!vxlan_addr_is_multicast(&cfg->group.dst) &&
+	    !vxlan_addr_any(&cfg->group.dst)) {
+		NL_SET_ERR_MSG_MOD(extack, "Group address is not multicast");
+		return -EINVAL;
+	}
+
+	if (vxlan_addr_any(&cfg->group.dst) &&
+	    mdbe_attrs[MDBE_ATTR_SOURCE]) {
+		NL_SET_ERR_MSG_MOD(extack, "Source cannot be specified for the all-zeros entry");
+		return -EINVAL;
+	}
+
+	if (vxlan_mdb_is_sg(&cfg->group))
+		cfg->filter_mode = MCAST_INCLUDE;
+
+	if (mdbe_attrs[MDBE_ATTR_GROUP_MODE]) {
+		if (!vxlan_mdb_is_star_g(&cfg->group)) {
+			NL_SET_ERR_MSG_MOD(extack, "Filter mode can only be set for (*, G) entries");
+			return -EINVAL;
+		}
+		cfg->filter_mode = nla_get_u8(mdbe_attrs[MDBE_ATTR_GROUP_MODE]);
+	}
+
+	if (mdbe_attrs[MDBE_ATTR_SRC_LIST]) {
+		if (!vxlan_mdb_is_star_g(&cfg->group)) {
+			NL_SET_ERR_MSG_MOD(extack, "Source list can only be set for (*, G) entries");
+			return -EINVAL;
+		}
+		if (!mdbe_attrs[MDBE_ATTR_GROUP_MODE]) {
+			NL_SET_ERR_MSG_MOD(extack, "Source list cannot be set without filter mode");
+			return -EINVAL;
+		}
+		err = vxlan_mdb_config_src_list_init(cfg, entry->addr.proto,
+						     mdbe_attrs[MDBE_ATTR_SRC_LIST],
+						     extack);
+		if (err)
+			return err;
+	}
+
+	if (vxlan_mdb_is_star_g(&cfg->group) && list_empty(&cfg->src_list) &&
+	    cfg->filter_mode == MCAST_INCLUDE) {
+		NL_SET_ERR_MSG_MOD(extack, "Cannot add (*, G) INCLUDE with an empty source list");
+		return -EINVAL;
+	}
+
+	if (mdbe_attrs[MDBE_ATTR_RTPROT])
+		cfg->rt_protocol = nla_get_u8(mdbe_attrs[MDBE_ATTR_RTPROT]);
+
+	err = vxlan_nla_get_addr(&cfg->remote_ip, mdbe_attrs[MDBE_ATTR_DST]);
+	if (err) {
+		NL_SET_ERR_MSG_MOD(extack, "Invalid remote destination address");
+		goto err_src_list_fini;
+	}
+
+	if (mdbe_attrs[MDBE_ATTR_DST_PORT])
+		cfg->remote_port =
+			cpu_to_be16(nla_get_u16(mdbe_attrs[MDBE_ATTR_DST_PORT]));
+
+	if (mdbe_attrs[MDBE_ATTR_VNI])
+		cfg->remote_vni =
+			cpu_to_be32(nla_get_u32(mdbe_attrs[MDBE_ATTR_VNI]));
+
+	if (mdbe_attrs[MDBE_ATTR_IFINDEX]) {
+		cfg->remote_ifindex =
+			nla_get_s32(mdbe_attrs[MDBE_ATTR_IFINDEX]);
+		if (!__dev_get_by_index(cfg->vxlan->net, cfg->remote_ifindex)) {
+			NL_SET_ERR_MSG_MOD(extack, "Outgoing interface not found");
+			err = -EINVAL;
+			goto err_src_list_fini;
+		}
+	}
+
+	if (mdbe_attrs[MDBE_ATTR_SRC_VNI])
+		cfg->group.vni =
+			cpu_to_be32(nla_get_u32(mdbe_attrs[MDBE_ATTR_SRC_VNI]));
+
+	return 0;
+
+err_src_list_fini:
+	vxlan_mdb_config_src_list_fini(cfg);
+	return err;
+}
+
+static int vxlan_mdb_config_init(struct vxlan_mdb_config *cfg,
+				 struct net_device *dev, struct nlattr *tb[],
+				 u16 nlmsg_flags,
+				 struct netlink_ext_ack *extack)
+{
+	struct br_mdb_entry *entry = nla_data(tb[MDBA_SET_ENTRY]);
+	struct vxlan_dev *vxlan = netdev_priv(dev);
+
+	memset(cfg, 0, sizeof(*cfg));
+	cfg->vxlan = vxlan;
+	cfg->group.vni = vxlan->default_dst.remote_vni;
+	INIT_LIST_HEAD(&cfg->src_list);
+	cfg->nlflags = nlmsg_flags;
+	cfg->filter_mode = MCAST_EXCLUDE;
+	cfg->rt_protocol = RTPROT_STATIC;
+	cfg->remote_vni = vxlan->default_dst.remote_vni;
+	cfg->remote_port = vxlan->cfg.dst_port;
+
+	if (entry->ifindex != dev->ifindex) {
+		NL_SET_ERR_MSG_MOD(extack, "Port net device must be the VXLAN net device");
+		return -EINVAL;
+	}
+
+	/* State is not part of the entry key and can be ignored on deletion
+	 * requests.
+	 */
+	if ((nlmsg_flags & (NLM_F_CREATE | NLM_F_REPLACE)) &&
+	    entry->state != MDB_PERMANENT) {
+		NL_SET_ERR_MSG_MOD(extack, "MDB entry must be permanent");
+		return -EINVAL;
+	}
+
+	if (entry->flags) {
+		NL_SET_ERR_MSG_MOD(extack, "Invalid MDB entry flags");
+		return -EINVAL;
+	}
+
+	if (entry->vid) {
+		NL_SET_ERR_MSG_MOD(extack, "VID must not be specified");
+		return -EINVAL;
+	}
+
+	if (entry->addr.proto != htons(ETH_P_IP) &&
+	    entry->addr.proto != htons(ETH_P_IPV6)) {
+		NL_SET_ERR_MSG_MOD(extack, "Group address must be an IPv4 / IPv6 address");
+		return -EINVAL;
+	}
+
+	if (NL_REQ_ATTR_CHECK(extack, NULL, tb, MDBA_SET_ENTRY_ATTRS)) {
+		NL_SET_ERR_MSG_MOD(extack, "Missing MDBA_SET_ENTRY_ATTRS attribute");
+		return -EINVAL;
+	}
+
+	return vxlan_mdb_config_attrs_init(cfg, entry, tb[MDBA_SET_ENTRY_ATTRS],
+					   extack);
+}
+
+static void vxlan_mdb_config_fini(struct vxlan_mdb_config *cfg)
+{
+	vxlan_mdb_config_src_list_fini(cfg);
+}
+
+static struct vxlan_mdb_entry *
+vxlan_mdb_entry_lookup(struct vxlan_dev *vxlan,
+		       const struct vxlan_mdb_entry_key *group)
+{
+	return rhashtable_lookup_fast(&vxlan->mdb_tbl, group,
+				      vxlan_mdb_rht_params);
+}
+
+static struct vxlan_mdb_remote *
+vxlan_mdb_remote_lookup(const struct vxlan_mdb_entry *mdb_entry,
+			const union vxlan_addr *addr)
+{
+	struct vxlan_mdb_remote *remote;
+
+	list_for_each_entry(remote, &mdb_entry->remotes, list) {
+		struct vxlan_rdst *rd = rtnl_dereference(remote->rd);
+
+		if (vxlan_addr_equal(addr, &rd->remote_ip))
+			return remote;
+	}
+
+	return NULL;
+}
+
+static void vxlan_mdb_rdst_free(struct rcu_head *head)
+{
+	struct vxlan_rdst *rd = container_of(head, struct vxlan_rdst, rcu);
+
+	dst_cache_destroy(&rd->dst_cache);
+	kfree(rd);
+}
+
+static int vxlan_mdb_remote_rdst_init(const struct vxlan_mdb_config *cfg,
+				      struct vxlan_mdb_remote *remote)
+{
+	struct vxlan_rdst *rd;
+	int err;
+
+	rd = kzalloc(sizeof(*rd), GFP_KERNEL);
+	if (!rd)
+		return -ENOMEM;
+
+	err = dst_cache_init(&rd->dst_cache, GFP_KERNEL);
+	if (err)
+		goto err_free_rdst;
+
+	rd->remote_ip = cfg->remote_ip;
+	rd->remote_port = cfg->remote_port;
+	rd->remote_vni = cfg->remote_vni;
+	rd->remote_ifindex = cfg->remote_ifindex;
+	rcu_assign_pointer(remote->rd, rd);
+
+	return 0;
+
+err_free_rdst:
+	kfree(rd);
+	return err;
+}
+
+static void vxlan_mdb_remote_rdst_fini(struct vxlan_rdst *rd)
+{
+	call_rcu(&rd->rcu, vxlan_mdb_rdst_free);
+}
+
+static int vxlan_mdb_remote_init(const struct vxlan_mdb_config *cfg,
+				 struct vxlan_mdb_remote *remote)
+{
+	int err;
+
+	err = vxlan_mdb_remote_rdst_init(cfg, remote);
+	if (err)
+		return err;
+
+	remote->flags = cfg->flags;
+	remote->filter_mode = cfg->filter_mode;
+	remote->rt_protocol = cfg->rt_protocol;
+	INIT_HLIST_HEAD(&remote->src_list);
+
+	return 0;
+}
+
+static void vxlan_mdb_remote_fini(struct vxlan_dev *vxlan,
+				  struct vxlan_mdb_remote *remote)
+{
+	WARN_ON_ONCE(!hlist_empty(&remote->src_list));
+	vxlan_mdb_remote_rdst_fini(rtnl_dereference(remote->rd));
+}
+
+static struct vxlan_mdb_src_entry *
+vxlan_mdb_remote_src_entry_lookup(const struct vxlan_mdb_remote *remote,
+				  const union vxlan_addr *addr)
+{
+	struct vxlan_mdb_src_entry *ent;
+
+	hlist_for_each_entry(ent, &remote->src_list, node) {
+		if (vxlan_addr_equal(&ent->addr, addr))
+			return ent;
+	}
+
+	return NULL;
+}
+
+static struct vxlan_mdb_src_entry *
+vxlan_mdb_remote_src_entry_add(struct vxlan_mdb_remote *remote,
+			       const union vxlan_addr *addr)
+{
+	struct vxlan_mdb_src_entry *ent;
+
+	ent = kzalloc(sizeof(*ent), GFP_KERNEL);
+	if (!ent)
+		return NULL;
+
+	ent->addr = *addr;
+	hlist_add_head(&ent->node, &remote->src_list);
+
+	return ent;
+}
+
+static void
+vxlan_mdb_remote_src_entry_del(struct vxlan_mdb_src_entry *ent)
+{
+	hlist_del(&ent->node);
+	kfree(ent);
+}
+
+static int
+vxlan_mdb_remote_src_fwd_add(const struct vxlan_mdb_config *cfg,
+			     const union vxlan_addr *addr,
+			     struct netlink_ext_ack *extack)
+{
+	struct vxlan_mdb_config sg_cfg;
+
+	memset(&sg_cfg, 0, sizeof(sg_cfg));
+	sg_cfg.vxlan = cfg->vxlan;
+	sg_cfg.group.src = *addr;
+	sg_cfg.group.dst = cfg->group.dst;
+	sg_cfg.group.vni = cfg->group.vni;
+	INIT_LIST_HEAD(&sg_cfg.src_list);
+	sg_cfg.remote_ip = cfg->remote_ip;
+	sg_cfg.remote_ifindex = cfg->remote_ifindex;
+	sg_cfg.remote_vni = cfg->remote_vni;
+	sg_cfg.remote_port = cfg->remote_port;
+	sg_cfg.nlflags = cfg->nlflags;
+	sg_cfg.filter_mode = MCAST_INCLUDE;
+	if (cfg->filter_mode == MCAST_EXCLUDE)
+		sg_cfg.flags = VXLAN_MDB_REMOTE_F_BLOCKED;
+	sg_cfg.rt_protocol = cfg->rt_protocol;
+
+	return __vxlan_mdb_add(&sg_cfg, extack);
+}
+
+static void
+vxlan_mdb_remote_src_fwd_del(struct vxlan_dev *vxlan,
+			     const struct vxlan_mdb_entry_key *group,
+			     const struct vxlan_mdb_remote *remote,
+			     const union vxlan_addr *addr)
+{
+	struct vxlan_rdst *rd = rtnl_dereference(remote->rd);
+	struct vxlan_mdb_config sg_cfg;
+
+	memset(&sg_cfg, 0, sizeof(sg_cfg));
+	sg_cfg.vxlan = vxlan;
+	sg_cfg.group.src = *addr;
+	sg_cfg.group.dst = group->dst;
+	sg_cfg.group.vni = group->vni;
+	INIT_LIST_HEAD(&sg_cfg.src_list);
+	sg_cfg.remote_ip = rd->remote_ip;
+
+	__vxlan_mdb_del(&sg_cfg, NULL);
+}
+
+static int
+vxlan_mdb_remote_src_add(const struct vxlan_mdb_config *cfg,
+			 struct vxlan_mdb_remote *remote,
+			 const struct vxlan_mdb_config_src_entry *src,
+			 struct netlink_ext_ack *extack)
+{
+	struct vxlan_mdb_src_entry *ent;
+	int err;
+
+	ent = vxlan_mdb_remote_src_entry_lookup(remote, &src->addr);
+	if (!ent) {
+		ent = vxlan_mdb_remote_src_entry_add(remote, &src->addr);
+		if (!ent)
+			return -ENOMEM;
+	} else if (!(cfg->nlflags & NLM_F_REPLACE)) {
+		NL_SET_ERR_MSG_MOD(extack, "Source entry already exists");
+		return -EEXIST;
+	}
+
+	err = vxlan_mdb_remote_src_fwd_add(cfg, &ent->addr, extack);
+	if (err)
+		goto err_src_del;
+
+	/* Clear flags in case source entry was marked for deletion as part of
+	 * replace flow.
+	 */
+	ent->flags = 0;
+
+	return 0;
+
+err_src_del:
+	vxlan_mdb_remote_src_entry_del(ent);
+	return err;
+}
+
+static void vxlan_mdb_remote_src_del(struct vxlan_dev *vxlan,
+				     const struct vxlan_mdb_entry_key *group,
+				     const struct vxlan_mdb_remote *remote,
+				     struct vxlan_mdb_src_entry *ent)
+{
+	vxlan_mdb_remote_src_fwd_del(vxlan, group, remote, &ent->addr);
+	vxlan_mdb_remote_src_entry_del(ent);
+}
+
+static int vxlan_mdb_remote_srcs_add(const struct vxlan_mdb_config *cfg,
+				     struct vxlan_mdb_remote *remote,
+				     struct netlink_ext_ack *extack)
+{
+	struct vxlan_mdb_config_src_entry *src;
+	struct vxlan_mdb_src_entry *ent;
+	struct hlist_node *tmp;
+	int err;
+
+	list_for_each_entry(src, &cfg->src_list, node) {
+		err = vxlan_mdb_remote_src_add(cfg, remote, src, extack);
+		if (err)
+			goto err_src_del;
+	}
+
+	return 0;
+
+err_src_del:
+	hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node)
+		vxlan_mdb_remote_src_del(cfg->vxlan, &cfg->group, remote, ent);
+	return err;
+}
+
+static void vxlan_mdb_remote_srcs_del(struct vxlan_dev *vxlan,
+				      const struct vxlan_mdb_entry_key *group,
+				      struct vxlan_mdb_remote *remote)
+{
+	struct vxlan_mdb_src_entry *ent;
+	struct hlist_node *tmp;
+
+	hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node)
+		vxlan_mdb_remote_src_del(vxlan, group, remote, ent);
+}
+
+static size_t
+vxlan_mdb_nlmsg_src_list_size(const struct vxlan_mdb_entry_key *group,
+			      const struct vxlan_mdb_remote *remote)
+{
+	struct vxlan_mdb_src_entry *ent;
+	size_t nlmsg_size;
+
+	if (hlist_empty(&remote->src_list))
+		return 0;
+
+	/* MDBA_MDB_EATTR_SRC_LIST */
+	nlmsg_size = nla_total_size(0);
+
+	hlist_for_each_entry(ent, &remote->src_list, node) {
+			      /* MDBA_MDB_SRCLIST_ENTRY */
+		nlmsg_size += nla_total_size(0) +
+			      /* MDBA_MDB_SRCATTR_ADDRESS */
+			      nla_total_size(vxlan_addr_size(&group->dst)) +
+			      /* MDBA_MDB_SRCATTR_TIMER */
+			      nla_total_size(sizeof(u8));
+	}
+
+	return nlmsg_size;
+}
+
+static size_t vxlan_mdb_nlmsg_size(const struct vxlan_dev *vxlan,
+				   const struct vxlan_mdb_entry *mdb_entry,
+				   const struct vxlan_mdb_remote *remote)
+{
+	const struct vxlan_mdb_entry_key *group = &mdb_entry->key;
+	struct vxlan_rdst *rd = rtnl_dereference(remote->rd);
+	size_t nlmsg_size;
+
+	nlmsg_size = NLMSG_ALIGN(sizeof(struct br_port_msg)) +
+		     /* MDBA_MDB */
+		     nla_total_size(0) +
+		     /* MDBA_MDB_ENTRY */
+		     nla_total_size(0) +
+		     /* MDBA_MDB_ENTRY_INFO */
+		     nla_total_size(sizeof(struct br_mdb_entry)) +
+		     /* MDBA_MDB_EATTR_TIMER */
+		     nla_total_size(sizeof(u32));
+	/* MDBA_MDB_EATTR_SOURCE */
+	if (vxlan_mdb_is_sg(group))
+		nlmsg_size += nla_total_size(vxlan_addr_size(&group->dst));
+	/* MDBA_MDB_EATTR_RTPROT */
+	nlmsg_size += nla_total_size(sizeof(u8));
+	/* MDBA_MDB_EATTR_SRC_LIST */
+	nlmsg_size += vxlan_mdb_nlmsg_src_list_size(group, remote);
+	/* MDBA_MDB_EATTR_GROUP_MODE */
+	nlmsg_size += nla_total_size(sizeof(u8));
+	/* MDBA_MDB_EATTR_DST */
+	nlmsg_size += nla_total_size(vxlan_addr_size(&rd->remote_ip));
+	/* MDBA_MDB_EATTR_DST_PORT */
+	if (rd->remote_port && rd->remote_port != vxlan->cfg.dst_port)
+		nlmsg_size += nla_total_size(sizeof(u16));
+	/* MDBA_MDB_EATTR_VNI */
+	if (rd->remote_vni != vxlan->default_dst.remote_vni)
+		nlmsg_size += nla_total_size(sizeof(u32));
+	/* MDBA_MDB_EATTR_IFINDEX */
+	if (rd->remote_ifindex)
+		nlmsg_size += nla_total_size(sizeof(u32));
+	/* MDBA_MDB_EATTR_SRC_VNI */
+	if ((vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) && group->vni)
+		nlmsg_size += nla_total_size(sizeof(u32));
+
+	return nlmsg_size;
+}
+
+static int vxlan_mdb_nlmsg_fill(const struct vxlan_dev *vxlan,
+				struct sk_buff *skb,
+				const struct vxlan_mdb_entry *mdb_entry,
+				const struct vxlan_mdb_remote *remote,
+				int type)
+{
+	struct nlattr *mdb_nest, *mdb_entry_nest;
+	struct br_port_msg *bpm;
+	struct nlmsghdr *nlh;
+
+	nlh = nlmsg_put(skb, 0, 0, type, sizeof(*bpm), 0);
+	if (!nlh)
+		return -EMSGSIZE;
+
+	bpm = nlmsg_data(nlh);
+	memset(bpm, 0, sizeof(*bpm));
+	bpm->family  = AF_BRIDGE;
+	bpm->ifindex = vxlan->dev->ifindex;
+
+	mdb_nest = nla_nest_start_noflag(skb, MDBA_MDB);
+	if (!mdb_nest)
+		goto cancel;
+	mdb_entry_nest = nla_nest_start_noflag(skb, MDBA_MDB_ENTRY);
+	if (!mdb_entry_nest)
+		goto cancel;
+
+	if (vxlan_mdb_entry_info_fill(vxlan, skb, mdb_entry, remote))
+		goto cancel;
+
+	nla_nest_end(skb, mdb_entry_nest);
+	nla_nest_end(skb, mdb_nest);
+	nlmsg_end(skb, nlh);
+
+	return 0;
+
+cancel:
+	nlmsg_cancel(skb, nlh);
+	return -EMSGSIZE;
+}
+
+static void vxlan_mdb_remote_notify(const struct vxlan_dev *vxlan,
+				    const struct vxlan_mdb_entry *mdb_entry,
+				    const struct vxlan_mdb_remote *remote,
+				    int type)
+{
+	struct net *net = dev_net(vxlan->dev);
+	struct sk_buff *skb;
+	int err = -ENOBUFS;
+
+	skb = nlmsg_new(vxlan_mdb_nlmsg_size(vxlan, mdb_entry, remote),
+			GFP_KERNEL);
+	if (!skb)
+		goto errout;
+
+	err = vxlan_mdb_nlmsg_fill(vxlan, skb, mdb_entry, remote, type);
+	if (err) {
+		kfree_skb(skb);
+		goto errout;
+	}
+
+	rtnl_notify(skb, net, 0, RTNLGRP_MDB, NULL, GFP_KERNEL);
+	return;
+errout:
+	rtnl_set_sk_err(net, RTNLGRP_MDB, err);
+}
+
+static int
+vxlan_mdb_remote_srcs_replace(const struct vxlan_mdb_config *cfg,
+			      const struct vxlan_mdb_entry *mdb_entry,
+			      struct vxlan_mdb_remote *remote,
+			      struct netlink_ext_ack *extack)
+{
+	struct vxlan_dev *vxlan = cfg->vxlan;
+	struct vxlan_mdb_src_entry *ent;
+	struct hlist_node *tmp;
+	int err;
+
+	hlist_for_each_entry(ent, &remote->src_list, node)
+		ent->flags |= VXLAN_SGRP_F_DELETE;
+
+	err = vxlan_mdb_remote_srcs_add(cfg, remote, extack);
+	if (err)
+		goto err_clear_delete;
+
+	hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node) {
+		if (ent->flags & VXLAN_SGRP_F_DELETE)
+			vxlan_mdb_remote_src_del(vxlan, &mdb_entry->key, remote,
+						 ent);
+	}
+
+	return 0;
+
+err_clear_delete:
+	hlist_for_each_entry(ent, &remote->src_list, node)
+		ent->flags &= ~VXLAN_SGRP_F_DELETE;
+	return err;
+}
+
+static int vxlan_mdb_remote_replace(const struct vxlan_mdb_config *cfg,
+				    const struct vxlan_mdb_entry *mdb_entry,
+				    struct vxlan_mdb_remote *remote,
+				    struct netlink_ext_ack *extack)
+{
+	struct vxlan_rdst *new_rd, *old_rd = rtnl_dereference(remote->rd);
+	struct vxlan_dev *vxlan = cfg->vxlan;
+	int err;
+
+	err = vxlan_mdb_remote_rdst_init(cfg, remote);
+	if (err)
+		return err;
+	new_rd = rtnl_dereference(remote->rd);
+
+	err = vxlan_mdb_remote_srcs_replace(cfg, mdb_entry, remote, extack);
+	if (err)
+		goto err_rdst_reset;
+
+	WRITE_ONCE(remote->flags, cfg->flags);
+	WRITE_ONCE(remote->filter_mode, cfg->filter_mode);
+	remote->rt_protocol = cfg->rt_protocol;
+	vxlan_mdb_remote_notify(vxlan, mdb_entry, remote, RTM_NEWMDB);
+
+	vxlan_mdb_remote_rdst_fini(old_rd);
+
+	return 0;
+
+err_rdst_reset:
+	rcu_assign_pointer(remote->rd, old_rd);
+	vxlan_mdb_remote_rdst_fini(new_rd);
+	return err;
+}
+
+static int vxlan_mdb_remote_add(const struct vxlan_mdb_config *cfg,
+				struct vxlan_mdb_entry *mdb_entry,
+				struct netlink_ext_ack *extack)
+{
+	struct vxlan_mdb_remote *remote;
+	int err;
+
+	remote = vxlan_mdb_remote_lookup(mdb_entry, &cfg->remote_ip);
+	if (remote) {
+		if (!(cfg->nlflags & NLM_F_REPLACE)) {
+			NL_SET_ERR_MSG_MOD(extack, "Replace not specified and MDB remote entry already exists");
+			return -EEXIST;
+		}
+		return vxlan_mdb_remote_replace(cfg, mdb_entry, remote, extack);
+	}
+
+	if (!(cfg->nlflags & NLM_F_CREATE)) {
+		NL_SET_ERR_MSG_MOD(extack, "Create not specified and entry does not exist");
+		return -ENOENT;
+	}
+
+	remote = kzalloc(sizeof(*remote), GFP_KERNEL);
+	if (!remote)
+		return -ENOMEM;
+
+	err = vxlan_mdb_remote_init(cfg, remote);
+	if (err) {
+		NL_SET_ERR_MSG_MOD(extack, "Failed to initialize remote MDB entry");
+		goto err_free_remote;
+	}
+
+	err = vxlan_mdb_remote_srcs_add(cfg, remote, extack);
+	if (err)
+		goto err_remote_fini;
+
+	list_add_rcu(&remote->list, &mdb_entry->remotes);
+	vxlan_mdb_remote_notify(cfg->vxlan, mdb_entry, remote, RTM_NEWMDB);
+
+	return 0;
+
+err_remote_fini:
+	vxlan_mdb_remote_fini(cfg->vxlan, remote);
+err_free_remote:
+	kfree(remote);
+	return err;
+}
+
+static void vxlan_mdb_remote_del(struct vxlan_dev *vxlan,
+				 struct vxlan_mdb_entry *mdb_entry,
+				 struct vxlan_mdb_remote *remote)
+{
+	vxlan_mdb_remote_notify(vxlan, mdb_entry, remote, RTM_DELMDB);
+	list_del_rcu(&remote->list);
+	vxlan_mdb_remote_srcs_del(vxlan, &mdb_entry->key, remote);
+	vxlan_mdb_remote_fini(vxlan, remote);
+	kfree_rcu(remote, rcu);
+}
+
+static struct vxlan_mdb_entry *
+vxlan_mdb_entry_get(struct vxlan_dev *vxlan,
+		    const struct vxlan_mdb_entry_key *group)
+{
+	struct vxlan_mdb_entry *mdb_entry;
+	int err;
+
+	mdb_entry = vxlan_mdb_entry_lookup(vxlan, group);
+	if (mdb_entry)
+		return mdb_entry;
+
+	mdb_entry = kzalloc(sizeof(*mdb_entry), GFP_KERNEL);
+	if (!mdb_entry)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&mdb_entry->remotes);
+	memcpy(&mdb_entry->key, group, sizeof(mdb_entry->key));
+	hlist_add_head(&mdb_entry->mdb_node, &vxlan->mdb_list);
+
+	err = rhashtable_lookup_insert_fast(&vxlan->mdb_tbl,
+					    &mdb_entry->rhnode,
+					    vxlan_mdb_rht_params);
+	if (err)
+		goto err_free_entry;
+
+	if (hlist_is_singular_node(&mdb_entry->mdb_node, &vxlan->mdb_list))
+		vxlan->cfg.flags |= VXLAN_F_MDB;
+
+	return mdb_entry;
+
+err_free_entry:
+	hlist_del(&mdb_entry->mdb_node);
+	kfree(mdb_entry);
+	return ERR_PTR(err);
+}
+
+static void vxlan_mdb_entry_put(struct vxlan_dev *vxlan,
+				struct vxlan_mdb_entry *mdb_entry)
+{
+	if (!list_empty(&mdb_entry->remotes))
+		return;
+
+	if (hlist_is_singular_node(&mdb_entry->mdb_node, &vxlan->mdb_list))
+		vxlan->cfg.flags &= ~VXLAN_F_MDB;
+
+	rhashtable_remove_fast(&vxlan->mdb_tbl, &mdb_entry->rhnode,
+			       vxlan_mdb_rht_params);
+	hlist_del(&mdb_entry->mdb_node);
+	kfree_rcu(mdb_entry, rcu);
+}
+
+static int __vxlan_mdb_add(const struct vxlan_mdb_config *cfg,
+			   struct netlink_ext_ack *extack)
+{
+	struct vxlan_dev *vxlan = cfg->vxlan;
+	struct vxlan_mdb_entry *mdb_entry;
+	int err;
+
+	mdb_entry = vxlan_mdb_entry_get(vxlan, &cfg->group);
+	if (IS_ERR(mdb_entry))
+		return PTR_ERR(mdb_entry);
+
+	err = vxlan_mdb_remote_add(cfg, mdb_entry, extack);
+	if (err)
+		goto err_entry_put;
+
+	vxlan->mdb_seq++;
+
+	return 0;
+
+err_entry_put:
+	vxlan_mdb_entry_put(vxlan, mdb_entry);
+	return err;
+}
+
+static int __vxlan_mdb_del(const struct vxlan_mdb_config *cfg,
+			   struct netlink_ext_ack *extack)
+{
+	struct vxlan_dev *vxlan = cfg->vxlan;
+	struct vxlan_mdb_entry *mdb_entry;
+	struct vxlan_mdb_remote *remote;
+
+	mdb_entry = vxlan_mdb_entry_lookup(vxlan, &cfg->group);
+	if (!mdb_entry) {
+		NL_SET_ERR_MSG_MOD(extack, "Did not find MDB entry");
+		return -ENOENT;
+	}
+
+	remote = vxlan_mdb_remote_lookup(mdb_entry, &cfg->remote_ip);
+	if (!remote) {
+		NL_SET_ERR_MSG_MOD(extack, "Did not find MDB remote entry");
+		return -ENOENT;
+	}
+
+	vxlan_mdb_remote_del(vxlan, mdb_entry, remote);
+	vxlan_mdb_entry_put(vxlan, mdb_entry);
+
+	vxlan->mdb_seq++;
+
+	return 0;
+}
+
+int vxlan_mdb_add(struct net_device *dev, struct nlattr *tb[], u16 nlmsg_flags,
+		  struct netlink_ext_ack *extack)
+{
+	struct vxlan_mdb_config cfg;
+	int err;
+
+	ASSERT_RTNL();
+
+	err = vxlan_mdb_config_init(&cfg, dev, tb, nlmsg_flags, extack);
+	if (err)
+		return err;
+
+	err = __vxlan_mdb_add(&cfg, extack);
+
+	vxlan_mdb_config_fini(&cfg);
+	return err;
+}
+
+int vxlan_mdb_del(struct net_device *dev, struct nlattr *tb[],
+		  struct netlink_ext_ack *extack)
+{
+	struct vxlan_mdb_config cfg;
+	int err;
+
+	ASSERT_RTNL();
+
+	err = vxlan_mdb_config_init(&cfg, dev, tb, 0, extack);
+	if (err)
+		return err;
+
+	err = __vxlan_mdb_del(&cfg, extack);
+
+	vxlan_mdb_config_fini(&cfg);
+	return err;
+}
+
+struct vxlan_mdb_entry *vxlan_mdb_entry_skb_get(struct vxlan_dev *vxlan,
+						struct sk_buff *skb,
+						__be32 src_vni)
+{
+	struct vxlan_mdb_entry *mdb_entry;
+	struct vxlan_mdb_entry_key group;
+
+	if (!is_multicast_ether_addr(eth_hdr(skb)->h_dest) ||
+	    is_broadcast_ether_addr(eth_hdr(skb)->h_dest))
+		return NULL;
+
+	/* When not in collect metadata mode, 'src_vni' is zero, but MDB
+	 * entries are stored with the VNI of the VXLAN device.
+	 */
+	if (!(vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA))
+		src_vni = vxlan->default_dst.remote_vni;
+
+	memset(&group, 0, sizeof(group));
+	group.vni = src_vni;
+
+	switch (skb->protocol) {
+	case htons(ETH_P_IP):
+		if (!pskb_may_pull(skb, sizeof(struct iphdr)))
+			return NULL;
+		group.dst.sa.sa_family = AF_INET;
+		group.dst.sin.sin_addr.s_addr = ip_hdr(skb)->daddr;
+		group.src.sa.sa_family = AF_INET;
+		group.src.sin.sin_addr.s_addr = ip_hdr(skb)->saddr;
+		break;
+#if IS_ENABLED(CONFIG_IPV6)
+	case htons(ETH_P_IPV6):
+		if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
+			return NULL;
+		group.dst.sa.sa_family = AF_INET6;
+		group.dst.sin6.sin6_addr = ipv6_hdr(skb)->daddr;
+		group.src.sa.sa_family = AF_INET6;
+		group.src.sin6.sin6_addr = ipv6_hdr(skb)->saddr;
+		break;
+#endif
+	default:
+		return NULL;
+	}
+
+	mdb_entry = vxlan_mdb_entry_lookup(vxlan, &group);
+	if (mdb_entry)
+		return mdb_entry;
+
+	memset(&group.src, 0, sizeof(group.src));
+	mdb_entry = vxlan_mdb_entry_lookup(vxlan, &group);
+	if (mdb_entry)
+		return mdb_entry;
+
+	/* No (S, G) or (*, G) found. Look up the all-zeros entry, but only if
+	 * the destination IP address is not link-local multicast since we want
+	 * to transmit such traffic together with broadcast and unknown unicast
+	 * traffic.
+	 */
+	switch (skb->protocol) {
+	case htons(ETH_P_IP):
+		if (ipv4_is_local_multicast(group.dst.sin.sin_addr.s_addr))
+			return NULL;
+		group.dst.sin.sin_addr.s_addr = 0;
+		break;
+#if IS_ENABLED(CONFIG_IPV6)
+	case htons(ETH_P_IPV6):
+		if (ipv6_addr_type(&group.dst.sin6.sin6_addr) &
+		    IPV6_ADDR_LINKLOCAL)
+			return NULL;
+		memset(&group.dst.sin6.sin6_addr, 0,
+		       sizeof(group.dst.sin6.sin6_addr));
+		break;
+#endif
+	default:
+		return NULL;
+	}
+
+	return vxlan_mdb_entry_lookup(vxlan, &group);
+}
+
+netdev_tx_t vxlan_mdb_xmit(struct vxlan_dev *vxlan,
+			   const struct vxlan_mdb_entry *mdb_entry,
+			   struct sk_buff *skb)
+{
+	struct vxlan_mdb_remote *remote, *fremote = NULL;
+	__be32 src_vni = mdb_entry->key.vni;
+
+	list_for_each_entry_rcu(remote, &mdb_entry->remotes, list) {
+		struct sk_buff *skb1;
+
+		if ((vxlan_mdb_is_star_g(&mdb_entry->key) &&
+		     READ_ONCE(remote->filter_mode) == MCAST_INCLUDE) ||
+		    (READ_ONCE(remote->flags) & VXLAN_MDB_REMOTE_F_BLOCKED))
+			continue;
+
+		if (!fremote) {
+			fremote = remote;
+			continue;
+		}
+
+		skb1 = skb_clone(skb, GFP_ATOMIC);
+		if (skb1)
+			vxlan_xmit_one(skb1, vxlan->dev, src_vni,
+				       rcu_dereference(remote->rd), false);
+	}
+
+	if (fremote)
+		vxlan_xmit_one(skb, vxlan->dev, src_vni,
+			       rcu_dereference(fremote->rd), false);
+	else
+		kfree_skb(skb);
+
+	return NETDEV_TX_OK;
+}
+
+static void vxlan_mdb_check_empty(void *ptr, void *arg)
+{
+	WARN_ON_ONCE(1);
+}
+
+static void vxlan_mdb_remotes_flush(struct vxlan_dev *vxlan,
+				    struct vxlan_mdb_entry *mdb_entry)
+{
+	struct vxlan_mdb_remote *remote, *tmp;
+
+	list_for_each_entry_safe(remote, tmp, &mdb_entry->remotes, list)
+		vxlan_mdb_remote_del(vxlan, mdb_entry, remote);
+}
+
+static void vxlan_mdb_entries_flush(struct vxlan_dev *vxlan)
+{
+	struct vxlan_mdb_entry *mdb_entry;
+	struct hlist_node *tmp;
+
+	/* The removal of an entry cannot trigger the removal of another entry
+	 * since entries are always added to the head of the list.
+	 */
+	hlist_for_each_entry_safe(mdb_entry, tmp, &vxlan->mdb_list, mdb_node) {
+		vxlan_mdb_remotes_flush(vxlan, mdb_entry);
+		vxlan_mdb_entry_put(vxlan, mdb_entry);
+	}
+}
+
+int vxlan_mdb_init(struct vxlan_dev *vxlan)
+{
+	int err;
+
+	err = rhashtable_init(&vxlan->mdb_tbl, &vxlan_mdb_rht_params);
+	if (err)
+		return err;
+
+	INIT_HLIST_HEAD(&vxlan->mdb_list);
+
+	return 0;
+}
+
+void vxlan_mdb_fini(struct vxlan_dev *vxlan)
+{
+	vxlan_mdb_entries_flush(vxlan);
+	WARN_ON_ONCE(vxlan->cfg.flags & VXLAN_F_MDB);
+	rhashtable_free_and_destroy(&vxlan->mdb_tbl, vxlan_mdb_check_empty,
+				    NULL);
+}
diff --git a/drivers/net/vxlan/vxlan_private.h b/drivers/net/vxlan/vxlan_private.h
index 599c3b4fdd5e..817fa3075842 100644
--- a/drivers/net/vxlan/vxlan_private.h
+++ b/drivers/net/vxlan/vxlan_private.h
@@ -85,6 +85,39 @@ bool vxlan_addr_equal(const union vxlan_addr *a, const union vxlan_addr *b)
 		return a->sin.sin_addr.s_addr == b->sin.sin_addr.s_addr;
 }
 
+static inline int vxlan_nla_get_addr(union vxlan_addr *ip,
+				     const struct nlattr *nla)
+{
+	if (nla_len(nla) >= sizeof(struct in6_addr)) {
+		ip->sin6.sin6_addr = nla_get_in6_addr(nla);
+		ip->sa.sa_family = AF_INET6;
+		return 0;
+	} else if (nla_len(nla) >= sizeof(__be32)) {
+		ip->sin.sin_addr.s_addr = nla_get_in_addr(nla);
+		ip->sa.sa_family = AF_INET;
+		return 0;
+	} else {
+		return -EAFNOSUPPORT;
+	}
+}
+
+static inline int vxlan_nla_put_addr(struct sk_buff *skb, int attr,
+				     const union vxlan_addr *ip)
+{
+	if (ip->sa.sa_family == AF_INET6)
+		return nla_put_in6_addr(skb, attr, &ip->sin6.sin6_addr);
+	else
+		return nla_put_in_addr(skb, attr, ip->sin.sin_addr.s_addr);
+}
+
+static inline bool vxlan_addr_is_multicast(const union vxlan_addr *ip)
+{
+	if (ip->sa.sa_family == AF_INET6)
+		return ipv6_addr_is_multicast(&ip->sin6.sin6_addr);
+	else
+		return ipv4_is_multicast(ip->sin.sin_addr.s_addr);
+}
+
 #else /* !CONFIG_IPV6 */
 
 static inline
@@ -93,8 +126,41 @@ bool vxlan_addr_equal(const union vxlan_addr *a, const union vxlan_addr *b)
 	return a->sin.sin_addr.s_addr == b->sin.sin_addr.s_addr;
 }
 
+static inline int vxlan_nla_get_addr(union vxlan_addr *ip,
+				     const struct nlattr *nla)
+{
+	if (nla_len(nla) >= sizeof(struct in6_addr)) {
+		return -EAFNOSUPPORT;
+	} else if (nla_len(nla) >= sizeof(__be32)) {
+		ip->sin.sin_addr.s_addr = nla_get_in_addr(nla);
+		ip->sa.sa_family = AF_INET;
+		return 0;
+	} else {
+		return -EAFNOSUPPORT;
+	}
+}
+
+static inline int vxlan_nla_put_addr(struct sk_buff *skb, int attr,
+				     const union vxlan_addr *ip)
+{
+	return nla_put_in_addr(skb, attr, ip->sin.sin_addr.s_addr);
+}
+
+static inline bool vxlan_addr_is_multicast(const union vxlan_addr *ip)
+{
+	return ipv4_is_multicast(ip->sin.sin_addr.s_addr);
+}
+
 #endif
 
+static inline size_t vxlan_addr_size(const union vxlan_addr *ip)
+{
+	if (ip->sa.sa_family == AF_INET6)
+		return sizeof(struct in6_addr);
+	else
+		return sizeof(__be32);
+}
+
 static inline struct vxlan_vni_node *
 vxlan_vnifilter_lookup(struct vxlan_dev *vxlan, __be32 vni)
 {
@@ -127,6 +193,8 @@ int vxlan_fdb_update(struct vxlan_dev *vxlan,
 		     __be16 port, __be32 src_vni, __be32 vni,
 		     __u32 ifindex, __u16 ndm_flags, u32 nhid,
 		     bool swdev_notify, struct netlink_ext_ack *extack);
+void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
+		    __be32 default_vni, struct vxlan_rdst *rdst, bool did_rsc);
 int vxlan_vni_in_use(struct net *src_net, struct vxlan_dev *vxlan,
 		     struct vxlan_config *conf, __be32 vni);
 
@@ -159,4 +227,20 @@ int vxlan_igmp_join(struct vxlan_dev *vxlan, union vxlan_addr *rip,
 		    int rifindex);
 int vxlan_igmp_leave(struct vxlan_dev *vxlan, union vxlan_addr *rip,
 		     int rifindex);
+
+/* vxlan_mdb.c */
+int vxlan_mdb_dump(struct net_device *dev, struct sk_buff *skb,
+		   struct netlink_callback *cb);
+int vxlan_mdb_add(struct net_device *dev, struct nlattr *tb[], u16 nlmsg_flags,
+		  struct netlink_ext_ack *extack);
+int vxlan_mdb_del(struct net_device *dev, struct nlattr *tb[],
+		  struct netlink_ext_ack *extack);
+struct vxlan_mdb_entry *vxlan_mdb_entry_skb_get(struct vxlan_dev *vxlan,
+						struct sk_buff *skb,
+						__be32 src_vni);
+netdev_tx_t vxlan_mdb_xmit(struct vxlan_dev *vxlan,
+			   const struct vxlan_mdb_entry *mdb_entry,
+			   struct sk_buff *skb);
+int vxlan_mdb_init(struct vxlan_dev *vxlan);
+void vxlan_mdb_fini(struct vxlan_dev *vxlan);
 #endif
diff --git a/drivers/net/wan/slic_ds26522.c b/drivers/net/wan/slic_ds26522.c
index 6063552cea9b..8a51cfcff99e 100644
--- a/drivers/net/wan/slic_ds26522.c
+++ b/drivers/net/wan/slic_ds26522.c
@@ -211,7 +211,7 @@ static int slic_ds26522_probe(struct spi_device *spi)
 
 	ret = slic_ds26522_init_configure(spi);
 	if (ret == 0)
-		pr_info("DS26522 cs%d configured\n", spi->chip_select);
+		pr_info("DS26522 cs%d configured\n", spi_get_chipselect(spi, 0));
 
 	return ret;
 }
diff --git a/drivers/net/wireless/Kconfig b/drivers/net/wireless/Kconfig
index cb1c15012dd0..7555af5195ec 100644
--- a/drivers/net/wireless/Kconfig
+++ b/drivers/net/wireless/Kconfig
@@ -38,79 +38,8 @@ source "drivers/net/wireless/ti/Kconfig"
 source "drivers/net/wireless/zydas/Kconfig"
 source "drivers/net/wireless/quantenna/Kconfig"
 
-config PCMCIA_RAYCS
-	tristate "Aviator/Raytheon 2.4GHz wireless support"
-	depends on PCMCIA
-	select WIRELESS_EXT
-	select WEXT_SPY
-	select WEXT_PRIV
-	help
-	  Say Y here if you intend to attach an Aviator/Raytheon PCMCIA
-	  (PC-card) wireless Ethernet networking card to your computer.
-	  Please read the file
-	  <file:Documentation/networking/device_drivers/wifi/ray_cs.rst> for
-	  details.
-
-	  To compile this driver as a module, choose M here: the module will be
-	  called ray_cs.  If unsure, say N.
-
-config PCMCIA_WL3501
-	tristate "Planet WL3501 PCMCIA cards"
-	depends on CFG80211 && PCMCIA
-	select WIRELESS_EXT
-	select WEXT_SPY
-	help
-	  A driver for WL3501 PCMCIA 802.11 wireless cards made by Planet.
-	  It has basic support for Linux wireless extensions and initial
-	  micro support for ethtool.
-
-config MAC80211_HWSIM
-	tristate "Simulated radio testing tool for mac80211"
-	depends on MAC80211
-	help
-	  This driver is a developer testing tool that can be used to test
-	  IEEE 802.11 networking stack (mac80211) functionality. This is not
-	  needed for normal wireless LAN usage and is only for testing. See
-	  Documentation/networking/mac80211_hwsim for more information on how
-	  to use this tool.
-
-	  To compile this driver as a module, choose M here: the module will be
-	  called mac80211_hwsim.  If unsure, say N.
+source "drivers/net/wireless/legacy/Kconfig"
 
-config USB_NET_RNDIS_WLAN
-	tristate "Wireless RNDIS USB support"
-	depends on USB
-	depends on CFG80211
-	select USB_NET_DRIVERS
-	select USB_USBNET
-	select USB_NET_CDCETHER
-	select USB_NET_RNDIS_HOST
-	help
-	  This is a driver for wireless RNDIS devices.
-	  These are USB based adapters found in devices such as:
-
-	  Buffalo WLI-U2-KG125S
-	  U.S. Robotics USR5421
-	  Belkin F5D7051
-	  Linksys WUSB54GSv2
-	  Linksys WUSB54GSC
-	  Asus WL169gE
-	  Eminent EM4045
-	  BT Voyager 1055
-	  Linksys WUSB54GSv1
-	  U.S. Robotics USR5420
-	  BUFFALO WLI-USB-G54
-
-	  All of these devices are based on Broadcom 4320 chip which is the
-	  only wireless RNDIS chip known to date.
-
-	  If you choose to build a module, it'll be called rndis_wlan.
-
-config VIRT_WIFI
-	tristate "Wifi wrapper for ethernet drivers"
-	depends on CFG80211
-	help
-	  This option adds support for ethernet connections to appear as if they
-	  are wifi connections through a special rtnetlink device.
+source "drivers/net/wireless/virtual/Kconfig"
 
 endif # WLAN
diff --git a/drivers/net/wireless/Makefile b/drivers/net/wireless/Makefile
index a61cf6c90343..4d7374d567d1 100644
--- a/drivers/net/wireless/Makefile
+++ b/drivers/net/wireless/Makefile
@@ -23,12 +23,5 @@ obj-$(CONFIG_WLAN_VENDOR_ST) += st/
 obj-$(CONFIG_WLAN_VENDOR_TI) += ti/
 obj-$(CONFIG_WLAN_VENDOR_ZYDAS) += zydas/
 
-# 16-bit wireless PCMCIA client drivers
-obj-$(CONFIG_PCMCIA_RAYCS)	+= ray_cs.o
-obj-$(CONFIG_PCMCIA_WL3501)	+= wl3501_cs.o
-
-obj-$(CONFIG_USB_NET_RNDIS_WLAN)	+= rndis_wlan.o
-
-obj-$(CONFIG_MAC80211_HWSIM)	+= mac80211_hwsim.o
-
-obj-$(CONFIG_VIRT_WIFI)	+= virt_wifi.o
+obj-$(CONFIG_WLAN) += legacy/
+obj-$(CONFIG_WLAN) += virtual/
diff --git a/drivers/net/wireless/ath/ath.h b/drivers/net/wireless/ath/ath.h
index f083fb9038c3..f02a308a9ffc 100644
--- a/drivers/net/wireless/ath/ath.h
+++ b/drivers/net/wireless/ath/ath.h
@@ -96,11 +96,13 @@ struct ath_keyval {
 	u8 kv_type;
 	u8 kv_pad;
 	u16 kv_len;
-	u8 kv_val[16]; /* TK */
-	u8 kv_mic[8]; /* Michael MIC key */
-	u8 kv_txmic[8]; /* Michael MIC TX key (used only if the hardware
-			 * supports both MIC keys in the same key cache entry;
-			 * in that case, kv_mic is the RX key) */
+	struct_group(kv_values,
+		u8 kv_val[16]; /* TK */
+		u8 kv_mic[8]; /* Michael MIC key */
+		u8 kv_txmic[8]; /* Michael MIC TX key (used only if the hardware
+				 * supports both MIC keys in the same key cache entry;
+				 * in that case, kv_mic is the RX key) */
+	);
 };
 
 enum ath_cipher {
diff --git a/drivers/net/wireless/ath/ath10k/ce.c b/drivers/net/wireless/ath/ath10k/ce.c
index c2f3bd35c392..c27b8204718a 100644
--- a/drivers/net/wireless/ath/ath10k/ce.c
+++ b/drivers/net/wireless/ath/ath10k/ce.c
@@ -77,45 +77,6 @@ static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
 	return addr;
 }
 
-static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar,
-					 struct ath10k_ce_pipe *ce_state)
-{
-	u32 ce_id = ce_state->id;
-	u32 addr = 0;
-
-	switch (ce_id) {
-	case 1:
-		addr = 0x00032034;
-		break;
-	case 2:
-		addr = 0x00032038;
-		break;
-	case 5:
-		addr = 0x00032044;
-		break;
-	case 7:
-		addr = 0x0003204C;
-		break;
-	case 8:
-		addr = 0x00032050;
-		break;
-	case 9:
-		addr = 0x00032054;
-		break;
-	case 10:
-		addr = 0x00032058;
-		break;
-	case 11:
-		addr = 0x0003205C;
-		break;
-	default:
-		ath10k_warn(ar, "invalid CE id: %d", ce_id);
-		break;
-	}
-
-	return addr;
-}
-
 static inline unsigned int
 ath10k_set_ring_byte(unsigned int offset,
 		     struct ath10k_hw_ce_regs_addr_map *addr_map)
@@ -123,13 +84,6 @@ ath10k_set_ring_byte(unsigned int offset,
 	return ((offset << addr_map->lsb) & addr_map->mask);
 }
 
-static inline unsigned int
-ath10k_get_ring_byte(unsigned int offset,
-		     struct ath10k_hw_ce_regs_addr_map *addr_map)
-{
-	return ((offset & addr_map->mask) >> (addr_map->lsb));
-}
-
 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
 {
 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
@@ -438,19 +392,6 @@ static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
 			  host_ie_addr & ~(wm_regs->wm_mask));
 }
 
-static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
-					       u32 ce_ctrl_addr)
-{
-	struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
-
-	u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
-					    ar->hw_ce_regs->misc_ie_addr);
-
-	ath10k_ce_write32(ar,
-			  ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
-			  misc_ie_addr | misc_regs->err_mask);
-}
-
 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
 						u32 ce_ctrl_addr)
 {
diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c
index ec8d5b29bc72..7675858f069b 100644
--- a/drivers/net/wireless/ath/ath10k/mac.c
+++ b/drivers/net/wireless/ath/ath10k/mac.c
@@ -6030,7 +6030,6 @@ static void ath10k_configure_filter(struct ieee80211_hw *hw,
 
 	mutex_lock(&ar->conf_mutex);
 
-	changed_flags &= SUPPORTED_FILTERS;
 	*total_flags &= SUPPORTED_FILTERS;
 	ar->filter_flags = *total_flags;
 
diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c
index 728d607289c3..a7f44f6335fb 100644
--- a/drivers/net/wireless/ath/ath10k/pci.c
+++ b/drivers/net/wireless/ath/ath10k/pci.c
@@ -3406,15 +3406,12 @@ static int ath10k_pci_claim(struct ath10k *ar)
 	if (!ar_pci->mem) {
 		ath10k_err(ar, "failed to iomap BAR%d\n", BAR_NUM);
 		ret = -EIO;
-		goto err_master;
+		goto err_region;
 	}
 
 	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot pci_mem 0x%pK\n", ar_pci->mem);
 	return 0;
 
-err_master:
-	pci_clear_master(pdev);
-
 err_region:
 	pci_release_region(pdev, BAR_NUM);
 
@@ -3431,7 +3428,6 @@ static void ath10k_pci_release(struct ath10k *ar)
 
 	pci_iounmap(pdev, ar_pci->mem);
 	pci_release_region(pdev, BAR_NUM);
-	pci_clear_master(pdev);
 	pci_disable_device(pdev);
 }
 
diff --git a/drivers/net/wireless/ath/ath10k/qmi.c b/drivers/net/wireless/ath/ath10k/qmi.c
index 90f457b8e1fe..038c5903c0dc 100644
--- a/drivers/net/wireless/ath/ath10k/qmi.c
+++ b/drivers/net/wireless/ath/ath10k/qmi.c
@@ -33,7 +33,7 @@ static int ath10k_qmi_map_msa_permission(struct ath10k_qmi *qmi,
 {
 	struct qcom_scm_vmperm dst_perms[3];
 	struct ath10k *ar = qmi->ar;
-	unsigned int src_perms;
+	u64 src_perms;
 	u32 perm_count;
 	int ret;
 
@@ -65,7 +65,7 @@ static int ath10k_qmi_unmap_msa_permission(struct ath10k_qmi *qmi,
 {
 	struct qcom_scm_vmperm dst_perms;
 	struct ath10k *ar = qmi->ar;
-	unsigned int src_perms;
+	u64 src_perms;
 	int ret;
 
 	src_perms = BIT(QCOM_SCM_VMID_MSS_MSA) | BIT(QCOM_SCM_VMID_WLAN);
diff --git a/drivers/net/wireless/ath/ath10k/snoc.c b/drivers/net/wireless/ath/ath10k/snoc.c
index 9a82f0336d95..5128a452c65f 100644
--- a/drivers/net/wireless/ath/ath10k/snoc.c
+++ b/drivers/net/wireless/ath/ath10k/snoc.c
@@ -927,6 +927,7 @@ static int ath10k_snoc_hif_start(struct ath10k *ar)
 
 	bitmap_clear(ar_snoc->pending_ce_irqs, 0, CE_COUNT_MAX);
 
+	dev_set_threaded(&ar->napi_dev, true);
 	ath10k_core_napi_enable(ar);
 	ath10k_snoc_irq_enable(ar);
 	ath10k_snoc_rx_post(ar);
diff --git a/drivers/net/wireless/ath/ath11k/ahb.c b/drivers/net/wireless/ath/ath11k/ahb.c
index 920abce9053a..5cbba9a8b6ba 100644
--- a/drivers/net/wireless/ath/ath11k/ahb.c
+++ b/drivers/net/wireless/ath/ath11k/ahb.c
@@ -874,11 +874,11 @@ static int ath11k_ahb_setup_msi_resources(struct ath11k_base *ab)
 	ab->pci.msi.ep_base_data = int_prop + 32;
 
 	for (i = 0; i < ab->pci.msi.config->total_vectors; i++) {
-		res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
-		if (!res)
-			return -ENODEV;
+		ret = platform_get_irq(pdev, i);
+		if (ret < 0)
+			return ret;
 
-		ab->pci.msi.irqs[i] = res->start;
+		ab->pci.msi.irqs[i] = ret;
 	}
 
 	set_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags);
@@ -1078,6 +1078,12 @@ static int ath11k_ahb_fw_resource_deinit(struct ath11k_base *ab)
 	struct iommu_domain *iommu;
 	size_t unmapped_size;
 
+	/* Chipsets not requiring MSA would have not initialized
+	 * MSA resources, return success in such cases.
+	 */
+	if (!ab->hw_params.fixed_fw_mem)
+		return 0;
+
 	if (ab_ahb->fw.use_tz)
 		return 0;
 
@@ -1174,7 +1180,7 @@ static int ath11k_ahb_probe(struct platform_device *pdev)
 		 * to a new space for accessing them.
 		 */
 		ab->mem_ce = ioremap(ce_remap->base, ce_remap->size);
-		if (IS_ERR(ab->mem_ce)) {
+		if (!ab->mem_ce) {
 			dev_err(&pdev->dev, "ce ioremap error\n");
 			ret = -ENOMEM;
 			goto err_core_free;
diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c
index 75fdbe4ef83a..b1b90bd34d67 100644
--- a/drivers/net/wireless/ath/ath11k/core.c
+++ b/drivers/net/wireless/ath/ath11k/core.c
@@ -116,7 +116,6 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.tcl_ring_retry = true,
 		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
 		.smp2p_wow_exit = false,
-		.ftm_responder = true,
 	},
 	{
 		.hw_rev = ATH11K_HW_IPQ6018_HW10,
@@ -199,7 +198,6 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
 		.smp2p_wow_exit = false,
 		.support_fw_mac_sequence = false,
-		.ftm_responder = true,
 	},
 	{
 		.name = "qca6390 hw2.0",
@@ -284,7 +282,6 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
 		.smp2p_wow_exit = false,
 		.support_fw_mac_sequence = true,
-		.ftm_responder = false,
 	},
 	{
 		.name = "qcn9074 hw1.0",
@@ -366,7 +363,6 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
 		.smp2p_wow_exit = false,
 		.support_fw_mac_sequence = false,
-		.ftm_responder = true,
 	},
 	{
 		.name = "wcn6855 hw2.0",
@@ -451,7 +447,6 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
 		.smp2p_wow_exit = false,
 		.support_fw_mac_sequence = true,
-		.ftm_responder = false,
 	},
 	{
 		.name = "wcn6855 hw2.1",
@@ -534,7 +529,6 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
 		.smp2p_wow_exit = false,
 		.support_fw_mac_sequence = true,
-		.ftm_responder = false,
 	},
 	{
 		.name = "wcn6750 hw1.0",
@@ -599,7 +593,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.current_cc_support = true,
 		.dbr_debug_support = false,
 		.global_reset = false,
-		.bios_sar_capa = NULL,
+		.bios_sar_capa = &ath11k_hw_sar_capa_wcn6855,
 		.m3_fw_support = false,
 		.fixed_bdf_addr = false,
 		.fixed_mem_region = false,
@@ -615,7 +609,6 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.tx_ring_size = DP_TCL_DATA_RING_SIZE_WCN6750,
 		.smp2p_wow_exit = true,
 		.support_fw_mac_sequence = true,
-		.ftm_responder = false,
 	},
 	{
 		.hw_rev = ATH11K_HW_IPQ5018_HW10,
@@ -695,7 +688,6 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
 		.smp2p_wow_exit = false,
 		.support_fw_mac_sequence = false,
-		.ftm_responder = true,
 	},
 };
 
diff --git a/drivers/net/wireless/ath/ath11k/dbring.c b/drivers/net/wireless/ath/ath11k/dbring.c
index 2107ec05d14f..5536e8642331 100644
--- a/drivers/net/wireless/ath/ath11k/dbring.c
+++ b/drivers/net/wireless/ath/ath11k/dbring.c
@@ -26,13 +26,13 @@ int ath11k_dbring_validate_buffer(struct ath11k *ar, void *buffer, u32 size)
 static void ath11k_dbring_fill_magic_value(struct ath11k *ar,
 					   void *buffer, u32 size)
 {
-	u32 *temp;
-	int idx;
-
-	size = size >> 2;
+	/* memset32 function fills buffer payload with the ATH11K_DB_MAGIC_VALUE
+	 * and the variable size is expected to be the number of u32 values
+	 * to be stored, not the number of bytes.
+	 */
+	size = size / sizeof(u32);
 
-	for (idx = 0, temp = buffer; idx < size; idx++, temp++)
-		*temp++ = ATH11K_DB_MAGIC_VALUE;
+	memset32(buffer, ATH11K_DB_MAGIC_VALUE, size);
 }
 
 static int ath11k_dbring_bufs_replenish(struct ath11k *ar,
diff --git a/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h b/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h
index 2b97cbbd28cb..0bbd58a380de 100644
--- a/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h
+++ b/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h
@@ -143,7 +143,8 @@ enum htt_tx_pdev_underrun_enum {
 /* Bytes stored in little endian order */
 /* Length should be multiple of DWORD */
 struct htt_stats_string_tlv {
-	u32 data[0]; /* Can be variable length */
+	 /* Can be variable length */
+	DECLARE_FLEX_ARRAY(u32, data);
 } __packed;
 
 #define HTT_STATS_MAC_ID	GENMASK(7, 0)
@@ -205,27 +206,32 @@ struct htt_tx_pdev_stats_cmn_tlv {
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_pdev_stats_urrn_tlv_v {
-	u32 urrn_stats[0]; /* HTT_TX_PDEV_MAX_URRN_STATS */
+	/* HTT_TX_PDEV_MAX_URRN_STATS */
+	DECLARE_FLEX_ARRAY(u32, urrn_stats);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_pdev_stats_flush_tlv_v {
-	u32 flush_errs[0]; /* HTT_TX_PDEV_MAX_FLUSH_REASON_STATS */
+	/* HTT_TX_PDEV_MAX_FLUSH_REASON_STATS */
+	DECLARE_FLEX_ARRAY(u32, flush_errs);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_pdev_stats_sifs_tlv_v {
-	u32 sifs_status[0]; /* HTT_TX_PDEV_MAX_SIFS_BURST_STATS */
+	/* HTT_TX_PDEV_MAX_SIFS_BURST_STATS */
+	DECLARE_FLEX_ARRAY(u32, sifs_status);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_pdev_stats_phy_err_tlv_v {
-	u32  phy_errs[0]; /* HTT_TX_PDEV_MAX_PHY_ERR_STATS */
+	/* HTT_TX_PDEV_MAX_PHY_ERR_STATS */
+	DECLARE_FLEX_ARRAY(u32, phy_errs);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_pdev_stats_sifs_hist_tlv_v {
-	u32 sifs_hist_status[0]; /* HTT_TX_PDEV_SIFS_BURST_HIST_STATS */
+	/* HTT_TX_PDEV_SIFS_BURST_HIST_STATS */
+	DECLARE_FLEX_ARRAY(u32, sifs_hist_status);
 };
 
 struct htt_tx_pdev_stats_tx_ppdu_stats_tlv_v {
@@ -590,20 +596,20 @@ struct htt_tx_hwq_difs_latency_stats_tlv_v {
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_hwq_cmd_result_stats_tlv_v {
-	/* Histogram of sched cmd result */
-	u32 cmd_result[0]; /* HTT_TX_HWQ_MAX_CMD_RESULT_STATS */
+	/* Histogram of sched cmd result, HTT_TX_HWQ_MAX_CMD_RESULT_STATS */
+	DECLARE_FLEX_ARRAY(u32, cmd_result);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_hwq_cmd_stall_stats_tlv_v {
-	/* Histogram of various pause conitions */
-	u32 cmd_stall_status[0]; /* HTT_TX_HWQ_MAX_CMD_STALL_STATS */
+	/* Histogram of various pause conitions, HTT_TX_HWQ_MAX_CMD_STALL_STATS */
+	DECLARE_FLEX_ARRAY(u32, cmd_stall_status);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_hwq_fes_result_stats_tlv_v {
-	/* Histogram of number of user fes result */
-	u32 fes_result[0]; /* HTT_TX_HWQ_MAX_FES_RESULT_STATS */
+	/* Histogram of number of user fes result, HTT_TX_HWQ_MAX_FES_RESULT_STATS */
+	DECLARE_FLEX_ARRAY(u32, fes_result);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size
@@ -635,8 +641,8 @@ struct htt_tx_hwq_tried_mpdu_cnt_hist_tlv_v {
  * #define WAL_TXOP_USED_HISTOGRAM_INTERVAL 1000 ( 1 ms )
  */
 struct htt_tx_hwq_txop_used_cnt_hist_tlv_v {
-	/* Histogram of txop used cnt */
-	u32 txop_used_cnt_hist[0]; /* HTT_TX_HWQ_TXOP_USED_CNT_HIST */
+	/* Histogram of txop used cnt,  HTT_TX_HWQ_TXOP_USED_CNT_HIST */
+	DECLARE_FLEX_ARRAY(u32, txop_used_cnt_hist);
 };
 
 /* == TX SELFGEN STATS == */
@@ -804,17 +810,20 @@ struct htt_tx_pdev_mpdu_stats_tlv {
 /* == TX SCHED STATS == */
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_sched_txq_cmd_posted_tlv_v {
-	u32 sched_cmd_posted[0]; /* HTT_TX_PDEV_SCHED_TX_MODE_MAX */
+	/* HTT_TX_PDEV_SCHED_TX_MODE_MAX */
+	DECLARE_FLEX_ARRAY(u32, sched_cmd_posted);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_sched_txq_cmd_reaped_tlv_v {
-	u32 sched_cmd_reaped[0]; /* HTT_TX_PDEV_SCHED_TX_MODE_MAX */
+	/* HTT_TX_PDEV_SCHED_TX_MODE_MAX */
+	DECLARE_FLEX_ARRAY(u32, sched_cmd_reaped);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_sched_txq_sched_order_su_tlv_v {
-	u32 sched_order_su[0]; /* HTT_TX_PDEV_NUM_SCHED_ORDER_LOG */
+	/* HTT_TX_PDEV_NUM_SCHED_ORDER_LOG */
+	DECLARE_FLEX_ARRAY(u32, sched_order_su);
 };
 
 enum htt_sched_txq_sched_ineligibility_tlv_enum {
@@ -842,7 +851,7 @@ enum htt_sched_txq_sched_ineligibility_tlv_enum {
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_sched_txq_sched_ineligibility_tlv_v {
 	/* indexed by htt_sched_txq_sched_ineligibility_tlv_enum */
-	u32 sched_ineligibility[0];
+	DECLARE_FLEX_ARRAY(u32, sched_ineligibility);
 };
 
 #define	HTT_TX_PDEV_STATS_SCHED_PER_TXQ_MAC_ID	GENMASK(7, 0)
@@ -888,18 +897,20 @@ struct htt_stats_tx_sched_cmn_tlv {
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_tqm_gen_mpdu_stats_tlv_v {
-	u32 gen_mpdu_end_reason[0]; /* HTT_TX_TQM_MAX_GEN_MPDU_END_REASON */
+	/* HTT_TX_TQM_MAX_GEN_MPDU_END_REASON */
+	DECLARE_FLEX_ARRAY(u32, gen_mpdu_end_reason);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_tqm_list_mpdu_stats_tlv_v {
-	u32 list_mpdu_end_reason[0]; /* HTT_TX_TQM_MAX_LIST_MPDU_END_REASON */
+	 /* HTT_TX_TQM_MAX_LIST_MPDU_END_REASON */
+	DECLARE_FLEX_ARRAY(u32, list_mpdu_end_reason);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_tx_tqm_list_mpdu_cnt_tlv_v {
-	u32 list_mpdu_cnt_hist[0];
-			/* HTT_TX_TQM_MAX_LIST_MPDU_CNT_HISTOGRAM_BINS */
+	/* HTT_TX_TQM_MAX_LIST_MPDU_CNT_HISTOGRAM_BINS */
+	DECLARE_FLEX_ARRAY(u32, list_mpdu_cnt_hist);
 };
 
 struct htt_tx_tqm_pdev_stats_tlv_v {
@@ -1098,7 +1109,7 @@ struct htt_tx_de_compl_stats_tlv {
  *                               ENTRIES_PER_BIN_COUNT)
  */
 struct htt_tx_de_fw2wbm_ring_full_hist_tlv {
-	u32 fw2wbm_ring_full_hist[0];
+	DECLARE_FLEX_ARRAY(u32, fw2wbm_ring_full_hist);
 };
 
 struct htt_tx_de_cmn_stats_tlv {
@@ -1151,7 +1162,7 @@ struct htt_ring_if_cmn_tlv {
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_sfm_client_user_tlv_v {
 	/* Number of DWORDS used per user and per client */
-	u32 dwords_used_by_user_n[0];
+	DECLARE_FLEX_ARRAY(u32, dwords_used_by_user_n);
 };
 
 struct htt_sfm_client_tlv {
@@ -1436,12 +1447,14 @@ struct htt_rx_soc_fw_stats_tlv {
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_rx_soc_fw_refill_ring_empty_tlv_v {
-	u32 refill_ring_empty_cnt[0]; /* HTT_RX_STATS_REFILL_MAX_RING */
+	/* HTT_RX_STATS_REFILL_MAX_RING */
+	DECLARE_FLEX_ARRAY(u32, refill_ring_empty_cnt);
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_rx_soc_fw_refill_ring_num_refill_tlv_v {
-	u32 refill_ring_num_refill[0]; /* HTT_RX_STATS_REFILL_MAX_RING */
+	/* HTT_RX_STATS_REFILL_MAX_RING */
+	DECLARE_FLEX_ARRAY(u32, refill_ring_num_refill);
 };
 
 /* RXDMA error code from WBM released packets */
@@ -1473,7 +1486,7 @@ enum htt_rx_rxdma_error_code_enum {
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_rx_soc_fw_refill_ring_num_rxdma_err_tlv_v {
-	u32 rxdma_err[0]; /* HTT_RX_RXDMA_MAX_ERR_CODE */
+	DECLARE_FLEX_ARRAY(u32, rxdma_err); /* HTT_RX_RXDMA_MAX_ERR_CODE */
 };
 
 /* REO error code from WBM released packets */
@@ -1505,7 +1518,7 @@ enum htt_rx_reo_error_code_enum {
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_rx_soc_fw_refill_ring_num_reo_err_tlv_v {
-	u32 reo_err[0]; /* HTT_RX_REO_MAX_ERR_CODE */
+	DECLARE_FLEX_ARRAY(u32, reo_err); /* HTT_RX_REO_MAX_ERR_CODE */
 };
 
 /* == RX PDEV STATS == */
@@ -1622,13 +1635,13 @@ struct htt_rx_pdev_fw_stats_phy_err_tlv {
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_rx_pdev_fw_ring_mpdu_err_tlv_v {
 	/* Num error MPDU for each RxDMA error type  */
-	u32 fw_ring_mpdu_err[0]; /* HTT_RX_STATS_RXDMA_MAX_ERR */
+	DECLARE_FLEX_ARRAY(u32, fw_ring_mpdu_err); /* HTT_RX_STATS_RXDMA_MAX_ERR */
 };
 
 /* NOTE: Variable length TLV, use length spec to infer array size */
 struct htt_rx_pdev_fw_mpdu_drop_tlv_v {
 	/* Num MPDU dropped  */
-	u32 fw_mpdu_drop[0]; /* HTT_RX_STATS_FW_DROP_REASON_MAX */
+	DECLARE_FLEX_ARRAY(u32, fw_mpdu_drop); /* HTT_RX_STATS_FW_DROP_REASON_MAX */
 };
 
 #define HTT_PDEV_CCA_STATS_TX_FRAME_INFO_PRESENT               (0x1)
diff --git a/drivers/net/wireless/ath/ath11k/dp.c b/drivers/net/wireless/ath/ath11k/dp.c
index f5156a7fbdd7..d070bcb3fe24 100644
--- a/drivers/net/wireless/ath/ath11k/dp.c
+++ b/drivers/net/wireless/ath/ath11k/dp.c
@@ -36,6 +36,7 @@ void ath11k_dp_peer_cleanup(struct ath11k *ar, int vdev_id, const u8 *addr)
 	}
 
 	ath11k_peer_rx_tid_cleanup(ar, peer);
+	peer->dp_setup_done = false;
 	crypto_free_shash(peer->tfm_mmic);
 	spin_unlock_bh(&ab->base_lock);
 }
@@ -72,7 +73,8 @@ int ath11k_dp_peer_setup(struct ath11k *ar, int vdev_id, const u8 *addr)
 	ret = ath11k_peer_rx_frag_setup(ar, addr, vdev_id);
 	if (ret) {
 		ath11k_warn(ab, "failed to setup rx defrag context\n");
-		return ret;
+		tid--;
+		goto peer_clean;
 	}
 
 	/* TODO: Setup other peer specific resource used in data path */
diff --git a/drivers/net/wireless/ath/ath11k/dp.h b/drivers/net/wireless/ath/ath11k/dp.h
index be9eafc872b3..d04f78ab6b37 100644
--- a/drivers/net/wireless/ath/ath11k/dp.h
+++ b/drivers/net/wireless/ath/ath11k/dp.h
@@ -214,7 +214,7 @@ struct ath11k_pdev_dp {
 #define DP_REO_REINJECT_RING_SIZE	32
 #define DP_RX_RELEASE_RING_SIZE		1024
 #define DP_REO_EXCEPTION_RING_SIZE	128
-#define DP_REO_CMD_RING_SIZE		128
+#define DP_REO_CMD_RING_SIZE		256
 #define DP_REO_STATUS_RING_SIZE		2048
 #define DP_RXDMA_BUF_RING_SIZE		4096
 #define DP_RXDMA_REFILL_RING_SIZE	2048
@@ -303,12 +303,16 @@ struct ath11k_dp {
 
 #define HTT_TX_WBM_COMP_STATUS_OFFSET 8
 
+#define HTT_INVALID_PEER_ID	0xffff
+
 /* HTT tx completion is overlaid in wbm_release_ring */
 #define HTT_TX_WBM_COMP_INFO0_STATUS		GENMASK(12, 9)
 #define HTT_TX_WBM_COMP_INFO0_REINJECT_REASON	GENMASK(16, 13)
 #define HTT_TX_WBM_COMP_INFO0_REINJECT_REASON	GENMASK(16, 13)
 
 #define HTT_TX_WBM_COMP_INFO1_ACK_RSSI		GENMASK(31, 24)
+#define HTT_TX_WBM_COMP_INFO2_SW_PEER_ID	GENMASK(15, 0)
+#define HTT_TX_WBM_COMP_INFO2_VALID		BIT(21)
 
 struct htt_tx_wbm_completion {
 	u32 info0;
diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c
index b65a84a88264..f67ce62b2b48 100644
--- a/drivers/net/wireless/ath/ath11k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath11k/dp_rx.c
@@ -389,10 +389,10 @@ int ath11k_dp_rxbufs_replenish(struct ath11k_base *ab, int mac_id,
 			goto fail_free_skb;
 
 		spin_lock_bh(&rx_ring->idr_lock);
-		buf_id = idr_alloc(&rx_ring->bufs_idr, skb, 0,
-				   rx_ring->bufs_max * 3, GFP_ATOMIC);
+		buf_id = idr_alloc(&rx_ring->bufs_idr, skb, 1,
+				   (rx_ring->bufs_max * 3) + 1, GFP_ATOMIC);
 		spin_unlock_bh(&rx_ring->idr_lock);
-		if (buf_id < 0)
+		if (buf_id <= 0)
 			goto fail_dma_unmap;
 
 		desc = ath11k_hal_srng_src_get_next_entry(ab, srng);
@@ -435,7 +435,6 @@ fail_free_skb:
 static int ath11k_dp_rxdma_buf_ring_free(struct ath11k *ar,
 					 struct dp_rxdma_ring *rx_ring)
 {
-	struct ath11k_pdev_dp *dp = &ar->dp;
 	struct sk_buff *skb;
 	int buf_id;
 
@@ -453,28 +452,6 @@ static int ath11k_dp_rxdma_buf_ring_free(struct ath11k *ar,
 	idr_destroy(&rx_ring->bufs_idr);
 	spin_unlock_bh(&rx_ring->idr_lock);
 
-	/* if rxdma1_enable is false, mon_status_refill_ring
-	 * isn't setup, so don't clean.
-	 */
-	if (!ar->ab->hw_params.rxdma1_enable)
-		return 0;
-
-	rx_ring = &dp->rx_mon_status_refill_ring[0];
-
-	spin_lock_bh(&rx_ring->idr_lock);
-	idr_for_each_entry(&rx_ring->bufs_idr, skb, buf_id) {
-		idr_remove(&rx_ring->bufs_idr, buf_id);
-		/* XXX: Understand where internal driver does this dma_unmap
-		 * of rxdma_buffer.
-		 */
-		dma_unmap_single(ar->ab->dev, ATH11K_SKB_RXCB(skb)->paddr,
-				 skb->len + skb_tailroom(skb), DMA_BIDIRECTIONAL);
-		dev_kfree_skb_any(skb);
-	}
-
-	idr_destroy(&rx_ring->bufs_idr);
-	spin_unlock_bh(&rx_ring->idr_lock);
-
 	return 0;
 }
 
@@ -691,13 +668,18 @@ void ath11k_dp_reo_cmd_list_cleanup(struct ath11k_base *ab)
 	struct ath11k_dp *dp = &ab->dp;
 	struct dp_reo_cmd *cmd, *tmp;
 	struct dp_reo_cache_flush_elem *cmd_cache, *tmp_cache;
+	struct dp_rx_tid *rx_tid;
 
 	spin_lock_bh(&dp->reo_cmd_lock);
 	list_for_each_entry_safe(cmd, tmp, &dp->reo_cmd_list, list) {
 		list_del(&cmd->list);
-		dma_unmap_single(ab->dev, cmd->data.paddr,
-				 cmd->data.size, DMA_BIDIRECTIONAL);
-		kfree(cmd->data.vaddr);
+		rx_tid = &cmd->data;
+		if (rx_tid->vaddr) {
+			dma_unmap_single(ab->dev, rx_tid->paddr,
+					 rx_tid->size, DMA_BIDIRECTIONAL);
+			kfree(rx_tid->vaddr);
+			rx_tid->vaddr = NULL;
+		}
 		kfree(cmd);
 	}
 
@@ -705,9 +687,13 @@ void ath11k_dp_reo_cmd_list_cleanup(struct ath11k_base *ab)
 				 &dp->reo_cmd_cache_flush_list, list) {
 		list_del(&cmd_cache->list);
 		dp->reo_cmd_cache_flush_count--;
-		dma_unmap_single(ab->dev, cmd_cache->data.paddr,
-				 cmd_cache->data.size, DMA_BIDIRECTIONAL);
-		kfree(cmd_cache->data.vaddr);
+		rx_tid = &cmd_cache->data;
+		if (rx_tid->vaddr) {
+			dma_unmap_single(ab->dev, rx_tid->paddr,
+					 rx_tid->size, DMA_BIDIRECTIONAL);
+			kfree(rx_tid->vaddr);
+			rx_tid->vaddr = NULL;
+		}
 		kfree(cmd_cache);
 	}
 	spin_unlock_bh(&dp->reo_cmd_lock);
@@ -721,10 +707,12 @@ static void ath11k_dp_reo_cmd_free(struct ath11k_dp *dp, void *ctx,
 	if (status != HAL_REO_CMD_SUCCESS)
 		ath11k_warn(dp->ab, "failed to flush rx tid hw desc, tid %d status %d\n",
 			    rx_tid->tid, status);
-
-	dma_unmap_single(dp->ab->dev, rx_tid->paddr, rx_tid->size,
-			 DMA_BIDIRECTIONAL);
-	kfree(rx_tid->vaddr);
+	if (rx_tid->vaddr) {
+		dma_unmap_single(dp->ab->dev, rx_tid->paddr, rx_tid->size,
+				 DMA_BIDIRECTIONAL);
+		kfree(rx_tid->vaddr);
+		rx_tid->vaddr = NULL;
+	}
 }
 
 static void ath11k_dp_reo_cache_flush(struct ath11k_base *ab,
@@ -763,6 +751,7 @@ static void ath11k_dp_reo_cache_flush(struct ath11k_base *ab,
 		dma_unmap_single(ab->dev, rx_tid->paddr, rx_tid->size,
 				 DMA_BIDIRECTIONAL);
 		kfree(rx_tid->vaddr);
+		rx_tid->vaddr = NULL;
 	}
 }
 
@@ -815,6 +804,7 @@ free_desc:
 	dma_unmap_single(ab->dev, rx_tid->paddr, rx_tid->size,
 			 DMA_BIDIRECTIONAL);
 	kfree(rx_tid->vaddr);
+	rx_tid->vaddr = NULL;
 }
 
 void ath11k_peer_rx_tid_delete(struct ath11k *ar,
@@ -827,6 +817,8 @@ void ath11k_peer_rx_tid_delete(struct ath11k *ar,
 	if (!rx_tid->active)
 		return;
 
+	rx_tid->active = false;
+
 	cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS;
 	cmd.addr_lo = lower_32_bits(rx_tid->paddr);
 	cmd.addr_hi = upper_32_bits(rx_tid->paddr);
@@ -841,9 +833,11 @@ void ath11k_peer_rx_tid_delete(struct ath11k *ar,
 		dma_unmap_single(ar->ab->dev, rx_tid->paddr, rx_tid->size,
 				 DMA_BIDIRECTIONAL);
 		kfree(rx_tid->vaddr);
+		rx_tid->vaddr = NULL;
 	}
 
-	rx_tid->active = false;
+	rx_tid->paddr = 0;
+	rx_tid->size = 0;
 }
 
 static int ath11k_dp_rx_link_desc_return(struct ath11k_base *ab,
@@ -990,6 +984,7 @@ static void ath11k_dp_rx_tid_mem_free(struct ath11k_base *ab,
 	dma_unmap_single(ab->dev, rx_tid->paddr, rx_tid->size,
 			 DMA_BIDIRECTIONAL);
 	kfree(rx_tid->vaddr);
+	rx_tid->vaddr = NULL;
 
 	rx_tid->active = false;
 
@@ -1014,7 +1009,8 @@ int ath11k_peer_rx_tid_setup(struct ath11k *ar, const u8 *peer_mac, int vdev_id,
 
 	peer = ath11k_peer_find(ab, vdev_id, peer_mac);
 	if (!peer) {
-		ath11k_warn(ab, "failed to find the peer to set up rx tid\n");
+		ath11k_warn(ab, "failed to find the peer %pM to set up rx tid\n",
+			    peer_mac);
 		spin_unlock_bh(&ab->base_lock);
 		return -ENOENT;
 	}
@@ -1027,7 +1023,8 @@ int ath11k_peer_rx_tid_setup(struct ath11k *ar, const u8 *peer_mac, int vdev_id,
 						    ba_win_sz, ssn, true);
 		spin_unlock_bh(&ab->base_lock);
 		if (ret) {
-			ath11k_warn(ab, "failed to update reo for rx tid %d\n", tid);
+			ath11k_warn(ab, "failed to update reo for peer %pM rx tid %d\n: %d",
+				    peer_mac, tid, ret);
 			return ret;
 		}
 
@@ -1035,8 +1032,8 @@ int ath11k_peer_rx_tid_setup(struct ath11k *ar, const u8 *peer_mac, int vdev_id,
 							     peer_mac, paddr,
 							     tid, 1, ba_win_sz);
 		if (ret)
-			ath11k_warn(ab, "failed to send wmi command to update rx reorder queue, tid :%d (%d)\n",
-				    tid, ret);
+			ath11k_warn(ab, "failed to send wmi rx reorder queue for peer %pM tid %d: %d\n",
+				    peer_mac, tid, ret);
 		return ret;
 	}
 
@@ -1069,6 +1066,8 @@ int ath11k_peer_rx_tid_setup(struct ath11k *ar, const u8 *peer_mac, int vdev_id,
 	ret = dma_mapping_error(ab->dev, paddr);
 	if (ret) {
 		spin_unlock_bh(&ab->base_lock);
+		ath11k_warn(ab, "failed to setup dma map for peer %pM rx tid %d: %d\n",
+			    peer_mac, tid, ret);
 		goto err_mem_free;
 	}
 
@@ -1082,15 +1081,16 @@ int ath11k_peer_rx_tid_setup(struct ath11k *ar, const u8 *peer_mac, int vdev_id,
 	ret = ath11k_wmi_peer_rx_reorder_queue_setup(ar, vdev_id, peer_mac,
 						     paddr, tid, 1, ba_win_sz);
 	if (ret) {
-		ath11k_warn(ar->ab, "failed to setup rx reorder queue, tid :%d (%d)\n",
-			    tid, ret);
+		ath11k_warn(ar->ab, "failed to setup rx reorder queue for peer %pM tid %d: %d\n",
+			    peer_mac, tid, ret);
 		ath11k_dp_rx_tid_mem_free(ab, peer_mac, vdev_id, tid);
 	}
 
 	return ret;
 
 err_mem_free:
-	kfree(vaddr);
+	kfree(rx_tid->vaddr);
+	rx_tid->vaddr = NULL;
 
 	return ret;
 }
@@ -2665,6 +2665,9 @@ try_again:
 				   cookie);
 		mac_id = FIELD_GET(DP_RXDMA_BUF_COOKIE_PDEV_ID, cookie);
 
+		if (unlikely(buf_id == 0))
+			continue;
+
 		ar = ab->pdevs[mac_id].ar;
 		rx_ring = &ar->dp.rx_refill_buf_ring;
 		spin_lock_bh(&rx_ring->idr_lock);
@@ -3029,39 +3032,51 @@ static int ath11k_dp_rx_reap_mon_status_ring(struct ath11k_base *ab, int mac_id,
 
 			spin_lock_bh(&rx_ring->idr_lock);
 			skb = idr_find(&rx_ring->bufs_idr, buf_id);
+			spin_unlock_bh(&rx_ring->idr_lock);
+
 			if (!skb) {
 				ath11k_warn(ab, "rx monitor status with invalid buf_id %d\n",
 					    buf_id);
-				spin_unlock_bh(&rx_ring->idr_lock);
 				pmon->buf_state = DP_MON_STATUS_REPLINISH;
 				goto move_next;
 			}
 
-			idr_remove(&rx_ring->bufs_idr, buf_id);
-			spin_unlock_bh(&rx_ring->idr_lock);
-
 			rxcb = ATH11K_SKB_RXCB(skb);
 
-			dma_unmap_single(ab->dev, rxcb->paddr,
-					 skb->len + skb_tailroom(skb),
-					 DMA_FROM_DEVICE);
+			dma_sync_single_for_cpu(ab->dev, rxcb->paddr,
+						skb->len + skb_tailroom(skb),
+						DMA_FROM_DEVICE);
 
 			tlv = (struct hal_tlv_hdr *)skb->data;
 			if (FIELD_GET(HAL_TLV_HDR_TAG, tlv->tl) !=
 					HAL_RX_STATUS_BUFFER_DONE) {
-				ath11k_warn(ab, "mon status DONE not set %lx\n",
+				ath11k_warn(ab, "mon status DONE not set %lx, buf_id %d\n",
 					    FIELD_GET(HAL_TLV_HDR_TAG,
-						      tlv->tl));
-				dev_kfree_skb_any(skb);
+						      tlv->tl), buf_id);
+				/* If done status is missing, hold onto status
+				 * ring until status is done for this status
+				 * ring buffer.
+				 * Keep HP in mon_status_ring unchanged,
+				 * and break from here.
+				 * Check status for same buffer for next time
+				 */
 				pmon->buf_state = DP_MON_STATUS_NO_DMA;
-				goto move_next;
+				break;
 			}
 
+			spin_lock_bh(&rx_ring->idr_lock);
+			idr_remove(&rx_ring->bufs_idr, buf_id);
+			spin_unlock_bh(&rx_ring->idr_lock);
 			if (ab->hw_params.full_monitor_mode) {
 				ath11k_dp_rx_mon_update_status_buf_state(pmon, tlv);
 				if (paddr == pmon->mon_status_paddr)
 					pmon->buf_state = DP_MON_STATUS_MATCH;
 			}
+
+			dma_unmap_single(ab->dev, rxcb->paddr,
+					 skb->len + skb_tailroom(skb),
+					 DMA_FROM_DEVICE);
+
 			__skb_queue_tail(skb_list, skb);
 		} else {
 			pmon->buf_state = DP_MON_STATUS_REPLINISH;
@@ -3117,8 +3132,11 @@ int ath11k_peer_rx_frag_setup(struct ath11k *ar, const u8 *peer_mac, int vdev_id
 	int i;
 
 	tfm = crypto_alloc_shash("michael_mic", 0, 0);
-	if (IS_ERR(tfm))
+	if (IS_ERR(tfm)) {
+		ath11k_warn(ab, "failed to allocate michael_mic shash: %ld\n",
+			    PTR_ERR(tfm));
 		return PTR_ERR(tfm);
+	}
 
 	spin_lock_bh(&ab->base_lock);
 
@@ -3138,6 +3156,7 @@ int ath11k_peer_rx_frag_setup(struct ath11k *ar, const u8 *peer_mac, int vdev_id
 	}
 
 	peer->tfm_mmic = tfm;
+	peer->dp_setup_done = true;
 	spin_unlock_bh(&ab->base_lock);
 
 	return 0;
@@ -3583,6 +3602,13 @@ static int ath11k_dp_rx_frag_h_mpdu(struct ath11k *ar,
 		ret = -ENOENT;
 		goto out_unlock;
 	}
+	if (!peer->dp_setup_done) {
+		ath11k_warn(ab, "The peer %pM [%d] has uninitialized datapath\n",
+			    peer->addr, peer_id);
+		ret = -ENOENT;
+		goto out_unlock;
+	}
+
 	rx_tid = &peer->rx_tid[tid];
 
 	if ((!skb_queue_empty(&rx_tid->rx_frags) && seqno != rx_tid->cur_sn) ||
@@ -3598,7 +3624,7 @@ static int ath11k_dp_rx_frag_h_mpdu(struct ath11k *ar,
 		goto out_unlock;
 	}
 
-	if (frag_no > __fls(rx_tid->rx_frag_bitmap))
+	if (!rx_tid->rx_frag_bitmap || (frag_no > __fls(rx_tid->rx_frag_bitmap)))
 		__skb_queue_tail(&rx_tid->rx_frags, msdu);
 	else
 		ath11k_dp_rx_h_sort_frags(ar, &rx_tid->rx_frags, msdu);
diff --git a/drivers/net/wireless/ath/ath11k/dp_tx.c b/drivers/net/wireless/ath/ath11k/dp_tx.c
index 8afbba236935..08a28464eb7a 100644
--- a/drivers/net/wireless/ath/ath11k/dp_tx.c
+++ b/drivers/net/wireless/ath/ath11k/dp_tx.c
@@ -316,10 +316,12 @@ ath11k_dp_tx_htt_tx_complete_buf(struct ath11k_base *ab,
 				 struct dp_tx_ring *tx_ring,
 				 struct ath11k_dp_htt_wbm_tx_status *ts)
 {
+	struct ieee80211_tx_status status = { 0 };
 	struct sk_buff *msdu;
 	struct ieee80211_tx_info *info;
 	struct ath11k_skb_cb *skb_cb;
 	struct ath11k *ar;
+	struct ath11k_peer *peer;
 
 	spin_lock(&tx_ring->tx_idr_lock);
 	msdu = idr_remove(&tx_ring->txbuf_idr, ts->msdu_id);
@@ -341,6 +343,11 @@ ath11k_dp_tx_htt_tx_complete_buf(struct ath11k_base *ab,
 
 	dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE);
 
+	if (!skb_cb->vif) {
+		dev_kfree_skb_any(msdu);
+		return;
+	}
+
 	memset(&info->status, 0, sizeof(info->status));
 
 	if (ts->acked) {
@@ -355,7 +362,23 @@ ath11k_dp_tx_htt_tx_complete_buf(struct ath11k_base *ab,
 		}
 	}
 
-	ieee80211_tx_status(ar->hw, msdu);
+	spin_lock_bh(&ab->base_lock);
+	peer = ath11k_peer_find_by_id(ab, ts->peer_id);
+	if (!peer || !peer->sta) {
+		ath11k_dbg(ab, ATH11K_DBG_DATA,
+			   "dp_tx: failed to find the peer with peer_id %d\n",
+			    ts->peer_id);
+		spin_unlock_bh(&ab->base_lock);
+		dev_kfree_skb_any(msdu);
+		return;
+	}
+	spin_unlock_bh(&ab->base_lock);
+
+	status.sta = peer->sta;
+	status.info = info;
+	status.skb = msdu;
+
+	ieee80211_tx_status_ext(ar->hw, &status);
 }
 
 static void
@@ -379,7 +402,15 @@ ath11k_dp_tx_process_htt_tx_complete(struct ath11k_base *ab,
 		ts.msdu_id = msdu_id;
 		ts.ack_rssi = FIELD_GET(HTT_TX_WBM_COMP_INFO1_ACK_RSSI,
 					status_desc->info1);
+
+		if (FIELD_GET(HTT_TX_WBM_COMP_INFO2_VALID, status_desc->info2))
+			ts.peer_id = FIELD_GET(HTT_TX_WBM_COMP_INFO2_SW_PEER_ID,
+					       status_desc->info2);
+		else
+			ts.peer_id = HTT_INVALID_PEER_ID;
+
 		ath11k_dp_tx_htt_tx_complete_buf(ab, tx_ring, &ts);
+
 		break;
 	case HAL_WBM_REL_HTT_TX_COMP_STATUS_REINJ:
 	case HAL_WBM_REL_HTT_TX_COMP_STATUS_INSPECT:
diff --git a/drivers/net/wireless/ath/ath11k/dp_tx.h b/drivers/net/wireless/ath/ath11k/dp_tx.h
index e87d65bfbf06..68a21ea9b934 100644
--- a/drivers/net/wireless/ath/ath11k/dp_tx.h
+++ b/drivers/net/wireless/ath/ath11k/dp_tx.h
@@ -13,6 +13,7 @@ struct ath11k_dp_htt_wbm_tx_status {
 	u32 msdu_id;
 	bool acked;
 	int ack_rssi;
+	u16 peer_id;
 };
 
 void ath11k_dp_tx_update_txcompl(struct ath11k *ar, struct hal_tx_status *ts);
diff --git a/drivers/net/wireless/ath/ath11k/hal_rx.c b/drivers/net/wireless/ath/ath11k/hal_rx.c
index 7f39c6fb7408..bb1d40034aa8 100644
--- a/drivers/net/wireless/ath/ath11k/hal_rx.c
+++ b/drivers/net/wireless/ath/ath11k/hal_rx.c
@@ -865,6 +865,12 @@ ath11k_hal_rx_populate_mu_user_info(void *rx_tlv, struct hal_rx_mon_ppdu_info *p
 	ath11k_hal_rx_populate_byte_count(rx_tlv, ppdu_info, rx_user_status);
 }
 
+static u16 ath11k_hal_rx_mpduinfo_get_peerid(struct ath11k_base *ab,
+					     struct hal_rx_mpdu_info *mpdu_info)
+{
+	return ab->hw_params.hw_ops->mpdu_info_get_peerid(mpdu_info);
+}
+
 static enum hal_rx_mon_status
 ath11k_hal_rx_parse_mon_status_tlv(struct ath11k_base *ab,
 				   struct hal_rx_mon_ppdu_info *ppdu_info,
@@ -1023,7 +1029,7 @@ ath11k_hal_rx_parse_mon_status_tlv(struct ath11k_base *ab,
 		info1 = __le32_to_cpu(vht_sig->info1);
 
 		ppdu_info->ldpc = FIELD_GET(HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING,
-					    info0);
+					    info1);
 		ppdu_info->mcs = FIELD_GET(HAL_RX_VHT_SIG_A_INFO_INFO1_MCS,
 					   info1);
 		gi_setting = FIELD_GET(HAL_RX_VHT_SIG_A_INFO_INFO1_GI_SETTING,
@@ -1446,7 +1452,7 @@ ath11k_hal_rx_parse_mon_status_tlv(struct ath11k_base *ab,
 		 * PHYRX_OTHER_RECEIVE_INFO TLV.
 		 */
 		ppdu_info->rssi_comb =
-			FIELD_GET(HAL_RX_PHYRX_RSSI_LEGACY_INFO_INFO1_RSSI_COMB,
+			FIELD_GET(HAL_RX_PHYRX_RSSI_LEGACY_INFO_INFO0_RSSI_COMB,
 				  __le32_to_cpu(rssi->info0));
 
 		if (db2dbm) {
@@ -1459,9 +1465,11 @@ ath11k_hal_rx_parse_mon_status_tlv(struct ath11k_base *ab,
 		break;
 	}
 	case HAL_RX_MPDU_START: {
+		struct hal_rx_mpdu_info *mpdu_info =
+				(struct hal_rx_mpdu_info *)tlv_data;
 		u16 peer_id;
 
-		peer_id = ab->hw_params.hw_ops->mpdu_info_get_peerid(tlv_data);
+		peer_id = ath11k_hal_rx_mpduinfo_get_peerid(ab, mpdu_info);
 		if (peer_id)
 			ppdu_info->peer_id = peer_id;
 		break;
diff --git a/drivers/net/wireless/ath/ath11k/hal_rx.h b/drivers/net/wireless/ath/ath11k/hal_rx.h
index f6bae07abfd3..61bd8416c4fd 100644
--- a/drivers/net/wireless/ath/ath11k/hal_rx.h
+++ b/drivers/net/wireless/ath/ath11k/hal_rx.h
@@ -385,7 +385,7 @@ struct hal_rx_he_sig_b2_ofdma_info {
 	__le32 info0;
 } __packed;
 
-#define HAL_RX_PHYRX_RSSI_LEGACY_INFO_INFO1_RSSI_COMB	GENMASK(15, 8)
+#define HAL_RX_PHYRX_RSSI_LEGACY_INFO_INFO0_RSSI_COMB	GENMASK(15, 8)
 
 #define HAL_RX_PHYRX_RSSI_PREAMBLE_PRI20	GENMASK(7, 0)
 
@@ -405,7 +405,7 @@ struct hal_rx_phyrx_rssi_legacy_info {
 #define HAL_RX_MPDU_INFO_INFO0_PEERID_WCN6855	GENMASK(15, 0)
 #define HAL_RX_MPDU_INFO_INFO1_MPDU_LEN		GENMASK(13, 0)
 
-struct hal_rx_mpdu_info {
+struct hal_rx_mpdu_info_ipq8074 {
 	__le32 rsvd0;
 	__le32 info0;
 	__le32 rsvd1[11];
@@ -413,12 +413,28 @@ struct hal_rx_mpdu_info {
 	__le32 rsvd2[9];
 } __packed;
 
+struct hal_rx_mpdu_info_qcn9074 {
+	__le32 rsvd0[10];
+	__le32 info0;
+	__le32 rsvd1[2];
+	__le32 info1;
+	__le32 rsvd2[9];
+} __packed;
+
 struct hal_rx_mpdu_info_wcn6855 {
 	__le32 rsvd0[8];
 	__le32 info0;
 	__le32 rsvd1[14];
 } __packed;
 
+struct hal_rx_mpdu_info {
+	union {
+		struct hal_rx_mpdu_info_ipq8074 ipq8074;
+		struct hal_rx_mpdu_info_qcn9074 qcn9074;
+		struct hal_rx_mpdu_info_wcn6855 wcn6855;
+	} u;
+} __packed;
+
 #define HAL_RX_PPDU_END_DURATION	GENMASK(23, 0)
 struct hal_rx_ppdu_end_duration {
 	__le32 rsvd0[9];
diff --git a/drivers/net/wireless/ath/ath11k/hw.c b/drivers/net/wireless/ath/ath11k/hw.c
index ab8f0ccacc6b..eb995f9cf0fa 100644
--- a/drivers/net/wireless/ath/ath11k/hw.c
+++ b/drivers/net/wireless/ath/ath11k/hw.c
@@ -201,6 +201,7 @@ static void ath11k_init_wmi_config_ipq8074(struct ath11k_base *ab,
 	config->twt_ap_pdev_count = ab->num_radios;
 	config->twt_ap_sta_count = 1000;
 	config->flag1 |= WMI_RSRC_CFG_FLAG1_BSS_CHANNEL_INFO_64;
+	config->flag1 |= WMI_RSRC_CFG_FLAG1_ACK_RSSI;
 }
 
 static int ath11k_hw_mac_id_to_pdev_id_ipq8074(struct ath11k_hw_params *hw,
@@ -834,26 +835,35 @@ static void ath11k_hw_ipq5018_reo_setup(struct ath11k_base *ab)
 			   ring_hash_map);
 }
 
-static u16 ath11k_hw_ipq8074_mpdu_info_get_peerid(u8 *tlv_data)
+static u16
+ath11k_hw_ipq8074_mpdu_info_get_peerid(struct hal_rx_mpdu_info *mpdu_info)
 {
 	u16 peer_id = 0;
-	struct hal_rx_mpdu_info *mpdu_info =
-		(struct hal_rx_mpdu_info *)tlv_data;
 
 	peer_id = FIELD_GET(HAL_RX_MPDU_INFO_INFO0_PEERID,
-			    __le32_to_cpu(mpdu_info->info0));
+			    __le32_to_cpu(mpdu_info->u.ipq8074.info0));
 
 	return peer_id;
 }
 
-static u16 ath11k_hw_wcn6855_mpdu_info_get_peerid(u8 *tlv_data)
+static u16
+ath11k_hw_qcn9074_mpdu_info_get_peerid(struct hal_rx_mpdu_info *mpdu_info)
+{
+	u16 peer_id = 0;
+
+	peer_id = FIELD_GET(HAL_RX_MPDU_INFO_INFO0_PEERID,
+			    __le32_to_cpu(mpdu_info->u.qcn9074.info0));
+
+	return peer_id;
+}
+
+static u16
+ath11k_hw_wcn6855_mpdu_info_get_peerid(struct hal_rx_mpdu_info *mpdu_info)
 {
 	u16 peer_id = 0;
-	struct hal_rx_mpdu_info_wcn6855 *mpdu_info =
-		(struct hal_rx_mpdu_info_wcn6855 *)tlv_data;
 
 	peer_id = FIELD_GET(HAL_RX_MPDU_INFO_INFO0_PEERID_WCN6855,
-			    __le32_to_cpu(mpdu_info->info0));
+			    __le32_to_cpu(mpdu_info->u.wcn6855.info0));
 	return peer_id;
 }
 
@@ -1041,7 +1051,7 @@ const struct ath11k_hw_ops qcn9074_ops = {
 	.rx_desc_get_attention = ath11k_hw_qcn9074_rx_desc_get_attention,
 	.rx_desc_get_msdu_payload = ath11k_hw_qcn9074_rx_desc_get_msdu_payload,
 	.reo_setup = ath11k_hw_ipq8074_reo_setup,
-	.mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid,
+	.mpdu_info_get_peerid = ath11k_hw_qcn9074_mpdu_info_get_peerid,
 	.rx_desc_mac_addr2_valid = ath11k_hw_ipq9074_rx_desc_mac_addr2_valid,
 	.rx_desc_mpdu_start_addr2 = ath11k_hw_ipq9074_rx_desc_mpdu_start_addr2,
 	.get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector,
@@ -1223,6 +1233,7 @@ const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_ipq8074 = {
 		ATH11K_RX_WBM_REL_RING_MASK_0,
 	},
 	.reo_status = {
+		0, 0, 0,
 		ATH11K_REO_STATUS_RING_MASK_0,
 	},
 	.rxdma2host = {
diff --git a/drivers/net/wireless/ath/ath11k/hw.h b/drivers/net/wireless/ath/ath11k/hw.h
index 0be4e1232384..6a5dd2dbdb3a 100644
--- a/drivers/net/wireless/ath/ath11k/hw.h
+++ b/drivers/net/wireless/ath/ath11k/hw.h
@@ -224,7 +224,6 @@ struct ath11k_hw_params {
 	u32 tx_ring_size;
 	bool smp2p_wow_exit;
 	bool support_fw_mac_sequence;
-	bool ftm_responder;
 };
 
 struct ath11k_hw_ops {
@@ -264,7 +263,7 @@ struct ath11k_hw_ops {
 	struct rx_attention *(*rx_desc_get_attention)(struct hal_rx_desc *desc);
 	u8 *(*rx_desc_get_msdu_payload)(struct hal_rx_desc *desc);
 	void (*reo_setup)(struct ath11k_base *ab);
-	u16 (*mpdu_info_get_peerid)(u8 *tlv_data);
+	u16 (*mpdu_info_get_peerid)(struct hal_rx_mpdu_info *mpdu_info);
 	bool (*rx_desc_mac_addr2_valid)(struct hal_rx_desc *desc);
 	u8* (*rx_desc_mpdu_start_addr2)(struct hal_rx_desc *desc);
 	u32 (*get_ring_selector)(struct sk_buff *skb);
diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c
index 110a38cce0a7..1c93f1afccc5 100644
--- a/drivers/net/wireless/ath/ath11k/mac.c
+++ b/drivers/net/wireless/ath/ath11k/mac.c
@@ -2699,6 +2699,117 @@ static int ath11k_setup_peer_smps(struct ath11k *ar, struct ath11k_vif *arvif,
 					 ath11k_smps_map[smps]);
 }
 
+static bool ath11k_mac_set_he_txbf_conf(struct ath11k_vif *arvif)
+{
+	struct ath11k *ar = arvif->ar;
+	u32 param, value;
+	int ret;
+
+	if (!arvif->vif->bss_conf.he_support)
+		return true;
+
+	param = WMI_VDEV_PARAM_SET_HEMU_MODE;
+	value = 0;
+	if (arvif->vif->bss_conf.he_su_beamformer) {
+		value |= FIELD_PREP(HE_MODE_SU_TX_BFER, HE_SU_BFER_ENABLE);
+		if (arvif->vif->bss_conf.he_mu_beamformer &&
+		    arvif->vdev_type == WMI_VDEV_TYPE_AP)
+			value |= FIELD_PREP(HE_MODE_MU_TX_BFER, HE_MU_BFER_ENABLE);
+	}
+
+	if (arvif->vif->type != NL80211_IFTYPE_MESH_POINT) {
+		value |= FIELD_PREP(HE_MODE_DL_OFDMA, HE_DL_MUOFDMA_ENABLE) |
+			 FIELD_PREP(HE_MODE_UL_OFDMA, HE_UL_MUOFDMA_ENABLE);
+
+		if (arvif->vif->bss_conf.he_full_ul_mumimo)
+			value |= FIELD_PREP(HE_MODE_UL_MUMIMO, HE_UL_MUMIMO_ENABLE);
+
+		if (arvif->vif->bss_conf.he_su_beamformee)
+			value |= FIELD_PREP(HE_MODE_SU_TX_BFEE, HE_SU_BFEE_ENABLE);
+	}
+
+	ret = ath11k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, param, value);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to set vdev %d HE MU mode: %d\n",
+			    arvif->vdev_id, ret);
+		return false;
+	}
+
+	param = WMI_VDEV_PARAM_SET_HE_SOUNDING_MODE;
+	value =	FIELD_PREP(HE_VHT_SOUNDING_MODE, HE_VHT_SOUNDING_MODE_ENABLE) |
+		FIELD_PREP(HE_TRIG_NONTRIG_SOUNDING_MODE,
+			   HE_TRIG_NONTRIG_SOUNDING_MODE_ENABLE);
+	ret = ath11k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id,
+					    param, value);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to set vdev %d sounding mode: %d\n",
+			    arvif->vdev_id, ret);
+		return false;
+	}
+	return true;
+}
+
+static bool ath11k_mac_vif_recalc_sta_he_txbf(struct ath11k *ar,
+					      struct ieee80211_vif *vif,
+					      struct ieee80211_sta_he_cap *he_cap)
+{
+	struct ath11k_vif *arvif = (void *)vif->drv_priv;
+	struct ieee80211_he_cap_elem he_cap_elem = {0};
+	struct ieee80211_sta_he_cap *cap_band = NULL;
+	struct cfg80211_chan_def def;
+	u32 param = WMI_VDEV_PARAM_SET_HEMU_MODE;
+	u32 hemode = 0;
+	int ret;
+
+	if (!vif->bss_conf.he_support)
+		return true;
+
+	if (vif->type != NL80211_IFTYPE_STATION)
+		return false;
+
+	if (WARN_ON(ath11k_mac_vif_chan(vif, &def)))
+		return false;
+
+	if (def.chan->band == NL80211_BAND_2GHZ)
+		cap_band = &ar->mac.iftype[NL80211_BAND_2GHZ][vif->type].he_cap;
+	else
+		cap_band = &ar->mac.iftype[NL80211_BAND_5GHZ][vif->type].he_cap;
+
+	memcpy(&he_cap_elem, &cap_band->he_cap_elem, sizeof(he_cap_elem));
+
+	if (HECAP_PHY_SUBFME_GET(he_cap_elem.phy_cap_info)) {
+		if (HECAP_PHY_SUBFMR_GET(he_cap->he_cap_elem.phy_cap_info))
+			hemode |= FIELD_PREP(HE_MODE_SU_TX_BFEE, HE_SU_BFEE_ENABLE);
+		if (HECAP_PHY_MUBFMR_GET(he_cap->he_cap_elem.phy_cap_info))
+			hemode |= FIELD_PREP(HE_MODE_MU_TX_BFEE, HE_MU_BFEE_ENABLE);
+	}
+
+	if (vif->type != NL80211_IFTYPE_MESH_POINT) {
+		hemode |= FIELD_PREP(HE_MODE_DL_OFDMA, HE_DL_MUOFDMA_ENABLE) |
+			  FIELD_PREP(HE_MODE_UL_OFDMA, HE_UL_MUOFDMA_ENABLE);
+
+		if (HECAP_PHY_ULMUMIMO_GET(he_cap_elem.phy_cap_info))
+			if (HECAP_PHY_ULMUMIMO_GET(he_cap->he_cap_elem.phy_cap_info))
+				hemode |= FIELD_PREP(HE_MODE_UL_MUMIMO,
+						     HE_UL_MUMIMO_ENABLE);
+
+		if (FIELD_GET(HE_MODE_MU_TX_BFEE, hemode))
+			hemode |= FIELD_PREP(HE_MODE_SU_TX_BFEE, HE_SU_BFEE_ENABLE);
+
+		if (FIELD_GET(HE_MODE_MU_TX_BFER, hemode))
+			hemode |= FIELD_PREP(HE_MODE_SU_TX_BFER, HE_SU_BFER_ENABLE);
+	}
+
+	ret = ath11k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, param, hemode);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to submit vdev param txbf 0x%x: %d\n",
+			    hemode, ret);
+		return false;
+	}
+
+	return true;
+}
+
 static void ath11k_bss_assoc(struct ieee80211_hw *hw,
 			     struct ieee80211_vif *vif,
 			     struct ieee80211_bss_conf *bss_conf)
@@ -2709,6 +2820,7 @@ static void ath11k_bss_assoc(struct ieee80211_hw *hw,
 	struct ieee80211_sta *ap_sta;
 	struct ath11k_peer *peer;
 	bool is_auth = false;
+	struct ieee80211_sta_he_cap  he_cap;
 	int ret;
 
 	lockdep_assert_held(&ar->conf_mutex);
@@ -2726,6 +2838,9 @@ static void ath11k_bss_assoc(struct ieee80211_hw *hw,
 		return;
 	}
 
+	/* he_cap here is updated at assoc success for sta mode only */
+	he_cap  = ap_sta->deflink.he_cap;
+
 	ath11k_peer_assoc_prepare(ar, vif, ap_sta, &peer_arg, false);
 
 	rcu_read_unlock();
@@ -2753,6 +2868,12 @@ static void ath11k_bss_assoc(struct ieee80211_hw *hw,
 		return;
 	}
 
+	if (!ath11k_mac_vif_recalc_sta_he_txbf(ar, vif, &he_cap)) {
+		ath11k_warn(ar->ab, "failed to recalc he txbf for vdev %i on bss %pM\n",
+			    arvif->vdev_id, bss_conf->bssid);
+		return;
+	}
+
 	WARN_ON(arvif->is_up);
 
 	arvif->aid = vif->cfg.aid;
@@ -3202,6 +3323,8 @@ static void ath11k_mac_op_bss_info_changed(struct ieee80211_hw *hw,
 		ether_addr_copy(arvif->bssid, info->bssid);
 
 	if (changed & BSS_CHANGED_BEACON_ENABLED) {
+		if (info->enable_beacon)
+			ath11k_mac_set_he_txbf_conf(arvif);
 		ath11k_control_beaconing(arvif, info);
 
 		if (arvif->is_up && vif->bss_conf.he_support &&
@@ -3415,7 +3538,7 @@ static void ath11k_mac_op_bss_info_changed(struct ieee80211_hw *hw,
 
 	if (changed & BSS_CHANGED_FTM_RESPONDER &&
 	    arvif->ftm_responder != info->ftm_responder &&
-	    ar->ab->hw_params.ftm_responder &&
+	    test_bit(WMI_TLV_SERVICE_RTT, ar->ab->wmi_ab.svc_map) &&
 	    (vif->type == NL80211_IFTYPE_AP ||
 	     vif->type == NL80211_IFTYPE_MESH_POINT)) {
 		arvif->ftm_responder = info->ftm_responder;
@@ -3632,6 +3755,18 @@ static int ath11k_mac_op_hw_scan(struct ieee80211_hw *hw,
 	int i;
 	u32 scan_timeout;
 
+	/* Firmwares advertising the support of triggering 11D algorithm
+	 * on the scan results of a regular scan expects driver to send
+	 * WMI_11D_SCAN_START_CMDID before sending WMI_START_SCAN_CMDID.
+	 * With this feature, separate 11D scan can be avoided since
+	 * regdomain can be determined with the scan results of the
+	 * regular scan.
+	 */
+	if (ar->state_11d == ATH11K_11D_PREPARING &&
+	    test_bit(WMI_TLV_SERVICE_SUPPORT_11D_FOR_HOST_SCAN,
+		     ar->ab->wmi_ab.svc_map))
+		ath11k_mac_11d_scan_start(ar, arvif->vdev_id);
+
 	mutex_lock(&ar->conf_mutex);
 
 	spin_lock_bh(&ar->data_lock);
@@ -3696,8 +3831,29 @@ static int ath11k_mac_op_hw_scan(struct ieee80211_hw *hw,
 			goto exit;
 		}
 
-		for (i = 0; i < arg->num_chan; i++)
-			arg->chan_list[i] = req->channels[i]->center_freq;
+		for (i = 0; i < arg->num_chan; i++) {
+			if (test_bit(WMI_TLV_SERVICE_SCAN_CONFIG_PER_CHANNEL,
+				     ar->ab->wmi_ab.svc_map)) {
+				arg->chan_list[i] =
+					u32_encode_bits(req->channels[i]->center_freq,
+							WMI_SCAN_CONFIG_PER_CHANNEL_MASK);
+
+				/* If NL80211_SCAN_FLAG_COLOCATED_6GHZ is set in scan
+				 * flags, then scan all PSC channels in 6 GHz band and
+				 * those non-PSC channels where RNR IE is found during
+				 * the legacy 2.4/5 GHz scan.
+				 * If NL80211_SCAN_FLAG_COLOCATED_6GHZ is not set,
+				 * then all channels in 6 GHz will be scanned.
+				 */
+				if (req->channels[i]->band == NL80211_BAND_6GHZ &&
+				    req->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ &&
+				    !cfg80211_channel_is_psc(req->channels[i]))
+					arg->chan_list[i] |=
+						WMI_SCAN_CH_FLAG_SCAN_ONLY_IF_RNR_FOUND;
+			} else {
+				arg->chan_list[i] = req->channels[i]->center_freq;
+			}
+		}
 	}
 
 	if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) {
@@ -5360,6 +5516,43 @@ static __le16 ath11k_mac_setup_he_6ghz_cap(struct ath11k_pdev_cap *pcap,
 	return cpu_to_le16(bcap->he_6ghz_capa);
 }
 
+static void ath11k_mac_set_hemcsmap(struct ath11k *ar,
+				    struct ath11k_pdev_cap *cap,
+				    struct ieee80211_sta_he_cap *he_cap,
+				    int band)
+{
+	u16 txmcs_map, rxmcs_map;
+	u32 i;
+
+	rxmcs_map = 0;
+	txmcs_map = 0;
+	for (i = 0; i < 8; i++) {
+		if (i < ar->num_tx_chains &&
+		    (ar->cfg_tx_chainmask >> cap->tx_chain_mask_shift) & BIT(i))
+			txmcs_map |= IEEE80211_HE_MCS_SUPPORT_0_11 << (i * 2);
+		else
+			txmcs_map |= IEEE80211_HE_MCS_NOT_SUPPORTED << (i * 2);
+
+		if (i < ar->num_rx_chains &&
+		    (ar->cfg_rx_chainmask >> cap->tx_chain_mask_shift) & BIT(i))
+			rxmcs_map |= IEEE80211_HE_MCS_SUPPORT_0_11 << (i * 2);
+		else
+			rxmcs_map |= IEEE80211_HE_MCS_NOT_SUPPORTED << (i * 2);
+	}
+	he_cap->he_mcs_nss_supp.rx_mcs_80 =
+		cpu_to_le16(rxmcs_map & 0xffff);
+	he_cap->he_mcs_nss_supp.tx_mcs_80 =
+		cpu_to_le16(txmcs_map & 0xffff);
+	he_cap->he_mcs_nss_supp.rx_mcs_160 =
+		cpu_to_le16(rxmcs_map & 0xffff);
+	he_cap->he_mcs_nss_supp.tx_mcs_160 =
+		cpu_to_le16(txmcs_map & 0xffff);
+	he_cap->he_mcs_nss_supp.rx_mcs_80p80 =
+		cpu_to_le16(rxmcs_map & 0xffff);
+	he_cap->he_mcs_nss_supp.tx_mcs_80p80 =
+		cpu_to_le16(txmcs_map & 0xffff);
+}
+
 static int ath11k_mac_copy_he_cap(struct ath11k *ar,
 				  struct ath11k_pdev_cap *cap,
 				  struct ieee80211_sband_iftype_data *data,
@@ -5417,18 +5610,7 @@ static int ath11k_mac_copy_he_cap(struct ath11k *ar,
 			break;
 		}
 
-		he_cap->he_mcs_nss_supp.rx_mcs_80 =
-			cpu_to_le16(band_cap->he_mcs & 0xffff);
-		he_cap->he_mcs_nss_supp.tx_mcs_80 =
-			cpu_to_le16(band_cap->he_mcs & 0xffff);
-		he_cap->he_mcs_nss_supp.rx_mcs_160 =
-			cpu_to_le16((band_cap->he_mcs >> 16) & 0xffff);
-		he_cap->he_mcs_nss_supp.tx_mcs_160 =
-			cpu_to_le16((band_cap->he_mcs >> 16) & 0xffff);
-		he_cap->he_mcs_nss_supp.rx_mcs_80p80 =
-			cpu_to_le16((band_cap->he_mcs >> 16) & 0xffff);
-		he_cap->he_mcs_nss_supp.tx_mcs_80p80 =
-			cpu_to_le16((band_cap->he_mcs >> 16) & 0xffff);
+		ath11k_mac_set_hemcsmap(ar, cap, he_cap, band);
 
 		memset(he_cap->ppe_thres, 0, sizeof(he_cap->ppe_thres));
 		if (he_cap_elem->phy_cap_info[6] &
@@ -6026,69 +6208,6 @@ ath11k_mac_setup_vdev_create_params(struct ath11k_vif *arvif,
 	}
 }
 
-static u32
-ath11k_mac_prepare_he_mode(struct ath11k_pdev *pdev, u32 viftype)
-{
-	struct ath11k_pdev_cap *pdev_cap = &pdev->cap;
-	struct ath11k_band_cap *cap_band = NULL;
-	u32 *hecap_phy_ptr = NULL;
-	u32 hemode = 0;
-
-	if (pdev->cap.supported_bands & WMI_HOST_WLAN_2G_CAP)
-		cap_band = &pdev_cap->band[NL80211_BAND_2GHZ];
-	else
-		cap_band = &pdev_cap->band[NL80211_BAND_5GHZ];
-
-	hecap_phy_ptr = &cap_band->he_cap_phy_info[0];
-
-	hemode = FIELD_PREP(HE_MODE_SU_TX_BFEE, HE_SU_BFEE_ENABLE) |
-		 FIELD_PREP(HE_MODE_SU_TX_BFER, HECAP_PHY_SUBFMR_GET(hecap_phy_ptr)) |
-		 FIELD_PREP(HE_MODE_UL_MUMIMO, HECAP_PHY_ULMUMIMO_GET(hecap_phy_ptr));
-
-	/* TODO WDS and other modes */
-	if (viftype == NL80211_IFTYPE_AP) {
-		hemode |= FIELD_PREP(HE_MODE_MU_TX_BFER,
-			  HECAP_PHY_MUBFMR_GET(hecap_phy_ptr)) |
-			  FIELD_PREP(HE_MODE_DL_OFDMA, HE_DL_MUOFDMA_ENABLE) |
-			  FIELD_PREP(HE_MODE_UL_OFDMA, HE_UL_MUOFDMA_ENABLE);
-	} else {
-		hemode |= FIELD_PREP(HE_MODE_MU_TX_BFEE, HE_MU_BFEE_ENABLE);
-	}
-
-	return hemode;
-}
-
-static int ath11k_set_he_mu_sounding_mode(struct ath11k *ar,
-					  struct ath11k_vif *arvif)
-{
-	u32 param_id, param_value;
-	struct ath11k_base *ab = ar->ab;
-	int ret = 0;
-
-	param_id = WMI_VDEV_PARAM_SET_HEMU_MODE;
-	param_value = ath11k_mac_prepare_he_mode(ar->pdev, arvif->vif->type);
-	ret = ath11k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id,
-					    param_id, param_value);
-	if (ret) {
-		ath11k_warn(ab, "failed to set vdev %d HE MU mode: %d param_value %x\n",
-			    arvif->vdev_id, ret, param_value);
-		return ret;
-	}
-	param_id = WMI_VDEV_PARAM_SET_HE_SOUNDING_MODE;
-	param_value =
-		FIELD_PREP(HE_VHT_SOUNDING_MODE, HE_VHT_SOUNDING_MODE_ENABLE) |
-		FIELD_PREP(HE_TRIG_NONTRIG_SOUNDING_MODE,
-			   HE_TRIG_NONTRIG_SOUNDING_MODE_ENABLE);
-	ret = ath11k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id,
-					    param_id, param_value);
-	if (ret) {
-		ath11k_warn(ab, "failed to set vdev %d HE MU mode: %d\n",
-			    arvif->vdev_id, ret);
-		return ret;
-	}
-	return ret;
-}
-
 static void ath11k_mac_op_update_vif_offload(struct ieee80211_hw *hw,
 					     struct ieee80211_vif *vif)
 {
@@ -6562,6 +6681,11 @@ static void ath11k_mac_op_remove_interface(struct ieee80211_hw *hw,
 	ath11k_dbg(ab, ATH11K_DBG_MAC, "mac remove interface (vdev %d)\n",
 		   arvif->vdev_id);
 
+	ret = ath11k_spectral_vif_stop(arvif);
+	if (ret)
+		ath11k_warn(ab, "failed to stop spectral for vdev %i: %d\n",
+			    arvif->vdev_id, ret);
+
 	if (arvif->vdev_type == WMI_VDEV_TYPE_STA)
 		ath11k_mac_11d_scan_stop(ar);
 
@@ -6757,7 +6881,6 @@ ath11k_mac_vdev_start_restart(struct ath11k_vif *arvif,
 	struct ath11k_base *ab = ar->ab;
 	struct wmi_vdev_start_req_arg arg = {};
 	const struct cfg80211_chan_def *chandef = &ctx->def;
-	int he_support = arvif->vif->bss_conf.he_support;
 	int ret = 0;
 
 	lockdep_assert_held(&ar->conf_mutex);
@@ -6798,15 +6921,6 @@ ath11k_mac_vdev_start_restart(struct ath11k_vif *arvif,
 		spin_lock_bh(&ab->base_lock);
 		arg.regdomain = ar->ab->dfs_region;
 		spin_unlock_bh(&ab->base_lock);
-
-		if (he_support) {
-			ret = ath11k_set_he_mu_sounding_mode(ar, arvif);
-			if (ret) {
-				ath11k_warn(ar->ab, "failed to set he mode vdev %i\n",
-					    arg.vdev_id);
-				return ret;
-			}
-		}
 	}
 
 	arg.channel.passive |= !!(chandef->chan->flags & IEEE80211_CHAN_NO_IR);
@@ -9094,6 +9208,11 @@ static int __ath11k_mac_register(struct ath11k *ar)
 		goto err_free_if_combs;
 	}
 
+	if (test_bit(WMI_TLV_SERVICE_TX_DATA_MGMT_ACK_RSSI,
+		     ar->ab->wmi_ab.svc_map))
+		wiphy_ext_feature_set(ar->hw->wiphy,
+				      NL80211_EXT_FEATURE_ACK_SIGNAL_SUPPORT);
+
 	ar->hw->queues = ATH11K_HW_MAX_QUEUES;
 	ar->hw->wiphy->tx_queue_len = ATH11K_QUEUE_LEN;
 	ar->hw->offchannel_tx_hw_queue = ATH11K_HW_MAX_QUEUES - 1;
@@ -9128,7 +9247,7 @@ static int __ath11k_mac_register(struct ath11k *ar)
 	wiphy_ext_feature_set(ar->hw->wiphy,
 			      NL80211_EXT_FEATURE_SET_SCAN_DWELL);
 
-	if (ab->hw_params.ftm_responder)
+	if (test_bit(WMI_TLV_SERVICE_RTT, ar->ab->wmi_ab.svc_map))
 		wiphy_ext_feature_set(ar->hw->wiphy,
 				      NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER);
 
diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c
index 0aeef2948ff5..7b33731a50ee 100644
--- a/drivers/net/wireless/ath/ath11k/pci.c
+++ b/drivers/net/wireless/ath/ath11k/pci.c
@@ -540,7 +540,7 @@ static int ath11k_pci_claim(struct ath11k_pci *ab_pci, struct pci_dev *pdev)
 	if (!ab->mem) {
 		ath11k_err(ab, "failed to map pci bar %d\n", ATH11K_PCI_BAR_NUM);
 		ret = -EIO;
-		goto clear_master;
+		goto release_region;
 	}
 
 	ab->mem_ce = ab->mem;
@@ -548,8 +548,6 @@ static int ath11k_pci_claim(struct ath11k_pci *ab_pci, struct pci_dev *pdev)
 	ath11k_dbg(ab, ATH11K_DBG_BOOT, "boot pci_mem 0x%pK\n", ab->mem);
 	return 0;
 
-clear_master:
-	pci_clear_master(pdev);
 release_region:
 	pci_release_region(pdev, ATH11K_PCI_BAR_NUM);
 disable_device:
@@ -565,7 +563,6 @@ static void ath11k_pci_free_region(struct ath11k_pci *ab_pci)
 
 	pci_iounmap(pci_dev, ab->mem);
 	ab->mem = NULL;
-	pci_clear_master(pci_dev);
 	pci_release_region(pci_dev, ATH11K_PCI_BAR_NUM);
 	if (pci_is_enabled(pci_dev))
 		pci_disable_device(pci_dev);
@@ -1039,7 +1036,8 @@ module_exit(ath11k_pci_exit);
 MODULE_DESCRIPTION("Driver support for Qualcomm Technologies 802.11ax WLAN PCIe devices");
 MODULE_LICENSE("Dual BSD/GPL");
 
-/* QCA639x 2.0 firmware files */
-MODULE_FIRMWARE(ATH11K_FW_DIR "/QCA6390/hw2.0/" ATH11K_BOARD_API2_FILE);
-MODULE_FIRMWARE(ATH11K_FW_DIR "/QCA6390/hw2.0/" ATH11K_AMSS_FILE);
-MODULE_FIRMWARE(ATH11K_FW_DIR "/QCA6390/hw2.0/" ATH11K_M3_FILE);
+/* firmware files */
+MODULE_FIRMWARE(ATH11K_FW_DIR "/QCA6390/hw2.0/*");
+MODULE_FIRMWARE(ATH11K_FW_DIR "/QCN9074/hw1.0/*");
+MODULE_FIRMWARE(ATH11K_FW_DIR "/WCN6855/hw2.0/*");
+MODULE_FIRMWARE(ATH11K_FW_DIR "/WCN6855/hw2.1/*");
diff --git a/drivers/net/wireless/ath/ath11k/peer.c b/drivers/net/wireless/ath/ath11k/peer.c
index 1ae7af02c364..1380811827a8 100644
--- a/drivers/net/wireless/ath/ath11k/peer.c
+++ b/drivers/net/wireless/ath/ath11k/peer.c
@@ -382,22 +382,23 @@ int ath11k_peer_create(struct ath11k *ar, struct ath11k_vif *arvif,
 		return -ENOBUFS;
 	}
 
+	mutex_lock(&ar->ab->tbl_mtx_lock);
 	spin_lock_bh(&ar->ab->base_lock);
 	peer = ath11k_peer_find_by_addr(ar->ab, param->peer_addr);
 	if (peer) {
 		if (peer->vdev_id == param->vdev_id) {
 			spin_unlock_bh(&ar->ab->base_lock);
+			mutex_unlock(&ar->ab->tbl_mtx_lock);
 			return -EINVAL;
 		}
 
 		/* Assume sta is transitioning to another band.
 		 * Remove here the peer from rhash.
 		 */
-		mutex_lock(&ar->ab->tbl_mtx_lock);
 		ath11k_peer_rhash_delete(ar->ab, peer);
-		mutex_unlock(&ar->ab->tbl_mtx_lock);
 	}
 	spin_unlock_bh(&ar->ab->base_lock);
+	mutex_unlock(&ar->ab->tbl_mtx_lock);
 
 	ret = ath11k_wmi_send_peer_create_cmd(ar, param);
 	if (ret) {
diff --git a/drivers/net/wireless/ath/ath11k/peer.h b/drivers/net/wireless/ath/ath11k/peer.h
index 6dd17bafe3a0..9bd385d0a38c 100644
--- a/drivers/net/wireless/ath/ath11k/peer.h
+++ b/drivers/net/wireless/ath/ath11k/peer.h
@@ -35,6 +35,7 @@ struct ath11k_peer {
 	u16 sec_type;
 	u16 sec_type_grp;
 	bool is_authorized;
+	bool dp_setup_done;
 };
 
 void ath11k_peer_unmap_event(struct ath11k_base *ab, u16 peer_id);
diff --git a/drivers/net/wireless/ath/ath11k/reg.c b/drivers/net/wireless/ath/ath11k/reg.c
index 6fae4e61ede7..67443457f4da 100644
--- a/drivers/net/wireless/ath/ath11k/reg.c
+++ b/drivers/net/wireless/ath/ath11k/reg.c
@@ -613,13 +613,19 @@ ath11k_reg_build_regd(struct ath11k_base *ab,
 {
 	struct ieee80211_regdomain *tmp_regd, *default_regd, *new_regd = NULL;
 	struct cur_reg_rule *reg_rule;
-	u8 i = 0, j = 0;
+	u8 i = 0, j = 0, k = 0;
 	u8 num_rules;
 	u16 max_bw;
 	u32 flags;
 	char alpha2[3];
 
-	num_rules = reg_info->num_5g_reg_rules + reg_info->num_2g_reg_rules;
+	num_rules = reg_info->num_5ghz_reg_rules + reg_info->num_2ghz_reg_rules;
+
+	/* FIXME: Currently taking reg rules for 6 GHz only from Indoor AP mode list.
+	 * This can be updated after complete 6 GHz regulatory support is added.
+	 */
+	if (reg_info->is_ext_reg_event)
+		num_rules += reg_info->num_6ghz_rules_ap[WMI_REG_INDOOR_AP];
 
 	if (!num_rules)
 		goto ret;
@@ -640,24 +646,24 @@ ath11k_reg_build_regd(struct ath11k_base *ab,
 	tmp_regd->dfs_region = ath11k_map_fw_dfs_region(reg_info->dfs_region);
 
 	ath11k_dbg(ab, ATH11K_DBG_REG,
-		   "\r\nCountry %s, CFG Regdomain %s FW Regdomain %d, num_reg_rules %d\n",
+		   "Country %s, CFG Regdomain %s FW Regdomain %d, num_reg_rules %d\n",
 		   alpha2, ath11k_reg_get_regdom_str(tmp_regd->dfs_region),
 		   reg_info->dfs_region, num_rules);
 	/* Update reg_rules[] below. Firmware is expected to
-	 * send these rules in order(2G rules first and then 5G)
+	 * send these rules in order(2 GHz rules first and then 5 GHz)
 	 */
 	for (; i < num_rules; i++) {
-		if (reg_info->num_2g_reg_rules &&
-		    (i < reg_info->num_2g_reg_rules)) {
-			reg_rule = reg_info->reg_rules_2g_ptr + i;
+		if (reg_info->num_2ghz_reg_rules &&
+		    (i < reg_info->num_2ghz_reg_rules)) {
+			reg_rule = reg_info->reg_rules_2ghz_ptr + i;
 			max_bw = min_t(u16, reg_rule->max_bw,
-				       reg_info->max_bw_2g);
+				       reg_info->max_bw_2ghz);
 			flags = 0;
-		} else if (reg_info->num_5g_reg_rules &&
-			   (j < reg_info->num_5g_reg_rules)) {
-			reg_rule = reg_info->reg_rules_5g_ptr + j++;
+		} else if (reg_info->num_5ghz_reg_rules &&
+			   (j < reg_info->num_5ghz_reg_rules)) {
+			reg_rule = reg_info->reg_rules_5ghz_ptr + j++;
 			max_bw = min_t(u16, reg_rule->max_bw,
-				       reg_info->max_bw_5g);
+				       reg_info->max_bw_5ghz);
 
 			/* FW doesn't pass NL80211_RRF_AUTO_BW flag for
 			 * BW Auto correction, we can enable this by default
@@ -666,6 +672,14 @@ ath11k_reg_build_regd(struct ath11k_base *ab,
 			 * per other BW rule flags we pass from here
 			 */
 			flags = NL80211_RRF_AUTO_BW;
+		} else if (reg_info->is_ext_reg_event &&
+			   reg_info->num_6ghz_rules_ap[WMI_REG_INDOOR_AP] &&
+			   (k < reg_info->num_6ghz_rules_ap[WMI_REG_INDOOR_AP])) {
+			reg_rule = reg_info->reg_rules_6ghz_ap_ptr[WMI_REG_INDOOR_AP] +
+				   k++;
+			max_bw = min_t(u16, reg_rule->max_bw,
+				       reg_info->max_bw_6ghz_ap[WMI_REG_INDOOR_AP]);
+			flags = NL80211_RRF_AUTO_BW;
 		} else {
 			break;
 		}
@@ -693,12 +707,21 @@ ath11k_reg_build_regd(struct ath11k_base *ab,
 			continue;
 		}
 
-		ath11k_dbg(ab, ATH11K_DBG_REG,
-			   "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n",
-			   i + 1, reg_rule->start_freq, reg_rule->end_freq,
-			   max_bw, reg_rule->ant_gain, reg_rule->reg_power,
-			   tmp_regd->reg_rules[i].dfs_cac_ms,
-			   flags);
+		if (reg_info->is_ext_reg_event) {
+			ath11k_dbg(ab, ATH11K_DBG_REG,
+				   "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d) (%d, %d)\n",
+				   i + 1, reg_rule->start_freq, reg_rule->end_freq,
+				   max_bw, reg_rule->ant_gain, reg_rule->reg_power,
+				   tmp_regd->reg_rules[i].dfs_cac_ms, flags,
+				   reg_rule->psd_flag, reg_rule->psd_eirp);
+		} else {
+			ath11k_dbg(ab, ATH11K_DBG_REG,
+				   "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n",
+				   i + 1, reg_rule->start_freq, reg_rule->end_freq,
+				   max_bw, reg_rule->ant_gain, reg_rule->reg_power,
+				   tmp_regd->reg_rules[i].dfs_cac_ms,
+				   flags);
+		}
 	}
 
 	tmp_regd->n_reg_rules = i;
diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c
index b3a7d7bfe17c..d0b59bc2905a 100644
--- a/drivers/net/wireless/ath/ath11k/wmi.c
+++ b/drivers/net/wireless/ath/ath11k/wmi.c
@@ -82,6 +82,12 @@ struct wmi_tlv_fw_stats_parse {
 	bool chain_rssi_done;
 };
 
+struct wmi_tlv_mgmt_rx_parse {
+	const struct wmi_mgmt_rx_hdr *fixed;
+	const u8 *frame_buf;
+	bool frame_buf_done;
+};
+
 static const struct wmi_tlv_policy wmi_tlv_policies[] = {
 	[WMI_TAG_ARRAY_BYTE]
 		= { .min_len = 0 },
@@ -105,6 +111,8 @@ static const struct wmi_tlv_policy wmi_tlv_policies[] = {
 		= { .min_len = sizeof(struct wmi_vdev_stopped_event) },
 	[WMI_TAG_REG_CHAN_LIST_CC_EVENT]
 		= { .min_len = sizeof(struct wmi_reg_chan_list_cc_event) },
+	[WMI_TAG_REG_CHAN_LIST_CC_EXT_EVENT]
+		= { .min_len = sizeof(struct wmi_reg_chan_list_cc_ext_event) },
 	[WMI_TAG_MGMT_RX_HDR]
 		= { .min_len = sizeof(struct wmi_mgmt_rx_hdr) },
 	[WMI_TAG_MGMT_TX_COMPL_EVENT]
@@ -863,7 +871,8 @@ static void ath11k_wmi_put_wmi_channel(struct wmi_channel *chan,
 
 		chan->band_center_freq2 = arg->channel.band_center_freq1;
 
-	} else if (arg->channel.mode == MODE_11AC_VHT80_80) {
+	} else if ((arg->channel.mode == MODE_11AC_VHT80_80) ||
+		   (arg->channel.mode == MODE_11AX_HE80_80)) {
 		chan->band_center_freq2 = arg->channel.band_center_freq2;
 	} else {
 		chan->band_center_freq2 = 0;
@@ -2068,6 +2077,12 @@ void ath11k_wmi_start_scan_init(struct ath11k *ar,
 				  WMI_SCAN_EVENT_FOREIGN_CHAN |
 				  WMI_SCAN_EVENT_DEQUEUED;
 	arg->scan_flags |= WMI_SCAN_CHAN_STAT_EVENT;
+
+	if (test_bit(WMI_TLV_SERVICE_PASSIVE_SCAN_START_TIME_ENHANCE,
+		     ar->ab->wmi_ab.svc_map))
+		arg->scan_ctrl_flags_ext |=
+			WMI_SCAN_FLAG_EXT_PASSIVE_SCAN_START_TIME_ENHANCE;
+
 	arg->num_bssid = 1;
 
 	/* fill bssid_list[0] with 0xff, otherwise bssid and RA will be
@@ -2149,6 +2164,8 @@ ath11k_wmi_copy_scan_event_cntrl_flags(struct wmi_start_scan_cmd *cmd,
 	/* for adaptive scan mode using 3 bits (21 - 23 bits) */
 	WMI_SCAN_SET_DWELL_MODE(cmd->scan_ctrl_flags,
 				param->adaptive_dwell_time_mode);
+
+	cmd->scan_ctrl_flags_ext = param->scan_ctrl_flags_ext;
 }
 
 int ath11k_wmi_send_scan_start_cmd(struct ath11k *ar,
@@ -3966,6 +3983,10 @@ ath11k_wmi_copy_resource_config(struct wmi_resource_config *wmi_cfg,
 	wmi_cfg->sched_params = tg_cfg->sched_params;
 	wmi_cfg->twt_ap_pdev_count = tg_cfg->twt_ap_pdev_count;
 	wmi_cfg->twt_ap_sta_count = tg_cfg->twt_ap_sta_count;
+	wmi_cfg->host_service_flags &=
+		~(1 << WMI_CFG_HOST_SERVICE_FLAG_REG_CC_EXT);
+	wmi_cfg->host_service_flags |= (tg_cfg->is_reg_cc_ext_event_supported <<
+					WMI_CFG_HOST_SERVICE_FLAG_REG_CC_EXT);
 }
 
 static int ath11k_init_cmd_send(struct ath11k_pdev_wmi *wmi,
@@ -4184,6 +4205,10 @@ int ath11k_wmi_cmd_init(struct ath11k_base *ab)
 
 	ab->hw_params.hw_ops->wmi_init_config(ab, &config);
 
+	if (test_bit(WMI_TLV_SERVICE_REG_CC_EXT_EVENT_SUPPORT,
+		     ab->wmi_ab.svc_map))
+		config.is_reg_cc_ext_event_supported = 1;
+
 	memcpy(&wmi_sc->wlan_resource_config, &config, sizeof(config));
 
 	init_param.res_cfg = &wmi_sc->wlan_resource_config;
@@ -4907,6 +4932,26 @@ static int ath11k_pull_vdev_start_resp_tlv(struct ath11k_base *ab, struct sk_buf
 	return 0;
 }
 
+static void ath11k_print_reg_rule(struct ath11k_base *ab, const char *band,
+				  u32 num_reg_rules,
+				  struct cur_reg_rule *reg_rule_ptr)
+{
+	struct cur_reg_rule *reg_rule = reg_rule_ptr;
+	u32 count;
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI, "number of reg rules in %s band: %d\n",
+		   band, num_reg_rules);
+
+	for (count = 0; count < num_reg_rules; count++) {
+		ath11k_dbg(ab, ATH11K_DBG_WMI,
+			   "reg rule %d: (%d - %d @ %d) (%d, %d) (FLAGS %d)\n",
+			   count + 1, reg_rule->start_freq, reg_rule->end_freq,
+			   reg_rule->max_bw, reg_rule->ant_gain,
+			   reg_rule->reg_power, reg_rule->flags);
+		reg_rule++;
+	}
+}
+
 static struct cur_reg_rule
 *create_reg_rules_from_wmi(u32 num_reg_rules,
 			   struct wmi_regulatory_rule_struct *wmi_reg_rule)
@@ -4951,7 +4996,7 @@ static int ath11k_pull_reg_chan_list_update_ev(struct ath11k_base *ab,
 	const void **tb;
 	const struct wmi_reg_chan_list_cc_event *chan_list_event_hdr;
 	struct wmi_regulatory_rule_struct *wmi_reg_rule;
-	u32 num_2g_reg_rules, num_5g_reg_rules;
+	u32 num_2ghz_reg_rules, num_5ghz_reg_rules;
 	int ret;
 
 	ath11k_dbg(ab, ATH11K_DBG_WMI, "processing regulatory channel list\n");
@@ -4970,10 +5015,10 @@ static int ath11k_pull_reg_chan_list_update_ev(struct ath11k_base *ab,
 		return -EPROTO;
 	}
 
-	reg_info->num_2g_reg_rules = chan_list_event_hdr->num_2g_reg_rules;
-	reg_info->num_5g_reg_rules = chan_list_event_hdr->num_5g_reg_rules;
+	reg_info->num_2ghz_reg_rules = chan_list_event_hdr->num_2ghz_reg_rules;
+	reg_info->num_5ghz_reg_rules = chan_list_event_hdr->num_5ghz_reg_rules;
 
-	if (!(reg_info->num_2g_reg_rules + reg_info->num_5g_reg_rules)) {
+	if (!(reg_info->num_2ghz_reg_rules + reg_info->num_5ghz_reg_rules)) {
 		ath11k_warn(ab, "No regulatory rules available in the event info\n");
 		kfree(tb);
 		return -EINVAL;
@@ -4987,61 +5032,68 @@ static int ath11k_pull_reg_chan_list_update_ev(struct ath11k_base *ab,
 	reg_info->phy_id = chan_list_event_hdr->phy_id;
 	reg_info->ctry_code = chan_list_event_hdr->country_id;
 	reg_info->reg_dmn_pair = chan_list_event_hdr->domain_code;
-	if (chan_list_event_hdr->status_code == WMI_REG_SET_CC_STATUS_PASS)
-		reg_info->status_code = REG_SET_CC_STATUS_PASS;
-	else if (chan_list_event_hdr->status_code == WMI_REG_CURRENT_ALPHA2_NOT_FOUND)
-		reg_info->status_code = REG_CURRENT_ALPHA2_NOT_FOUND;
-	else if (chan_list_event_hdr->status_code == WMI_REG_INIT_ALPHA2_NOT_FOUND)
-		reg_info->status_code = REG_INIT_ALPHA2_NOT_FOUND;
-	else if (chan_list_event_hdr->status_code == WMI_REG_SET_CC_CHANGE_NOT_ALLOWED)
-		reg_info->status_code = REG_SET_CC_CHANGE_NOT_ALLOWED;
-	else if (chan_list_event_hdr->status_code == WMI_REG_SET_CC_STATUS_NO_MEMORY)
-		reg_info->status_code = REG_SET_CC_STATUS_NO_MEMORY;
-	else if (chan_list_event_hdr->status_code == WMI_REG_SET_CC_STATUS_FAIL)
-		reg_info->status_code = REG_SET_CC_STATUS_FAIL;
-
-	reg_info->min_bw_2g = chan_list_event_hdr->min_bw_2g;
-	reg_info->max_bw_2g = chan_list_event_hdr->max_bw_2g;
-	reg_info->min_bw_5g = chan_list_event_hdr->min_bw_5g;
-	reg_info->max_bw_5g = chan_list_event_hdr->max_bw_5g;
-
-	num_2g_reg_rules = reg_info->num_2g_reg_rules;
-	num_5g_reg_rules = reg_info->num_5g_reg_rules;
 
 	ath11k_dbg(ab, ATH11K_DBG_WMI,
-		   "%s:cc %s dsf %d BW: min_2g %d max_2g %d min_5g %d max_5g %d",
-		   __func__, reg_info->alpha2, reg_info->dfs_region,
-		   reg_info->min_bw_2g, reg_info->max_bw_2g,
-		   reg_info->min_bw_5g, reg_info->max_bw_5g);
+		   "status_code %s",
+		   ath11k_cc_status_to_str(reg_info->status_code));
+
+	reg_info->status_code =
+		ath11k_wmi_cc_setting_code_to_reg(chan_list_event_hdr->status_code);
+
+	reg_info->is_ext_reg_event = false;
+
+	reg_info->min_bw_2ghz = chan_list_event_hdr->min_bw_2ghz;
+	reg_info->max_bw_2ghz = chan_list_event_hdr->max_bw_2ghz;
+	reg_info->min_bw_5ghz = chan_list_event_hdr->min_bw_5ghz;
+	reg_info->max_bw_5ghz = chan_list_event_hdr->max_bw_5ghz;
+
+	num_2ghz_reg_rules = reg_info->num_2ghz_reg_rules;
+	num_5ghz_reg_rules = reg_info->num_5ghz_reg_rules;
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "cc %s dsf %d BW: min_2ghz %d max_2ghz %d min_5ghz %d max_5ghz %d",
+		   reg_info->alpha2, reg_info->dfs_region,
+		   reg_info->min_bw_2ghz, reg_info->max_bw_2ghz,
+		   reg_info->min_bw_5ghz, reg_info->max_bw_5ghz);
 
 	ath11k_dbg(ab, ATH11K_DBG_WMI,
-		   "%s: num_2g_reg_rules %d num_5g_reg_rules %d", __func__,
-		   num_2g_reg_rules, num_5g_reg_rules);
+		   "num_2ghz_reg_rules %d num_5ghz_reg_rules %d",
+		   num_2ghz_reg_rules, num_5ghz_reg_rules);
 
 	wmi_reg_rule =
 		(struct wmi_regulatory_rule_struct *)((u8 *)chan_list_event_hdr
 						+ sizeof(*chan_list_event_hdr)
 						+ sizeof(struct wmi_tlv));
 
-	if (num_2g_reg_rules) {
-		reg_info->reg_rules_2g_ptr = create_reg_rules_from_wmi(num_2g_reg_rules,
-								       wmi_reg_rule);
-		if (!reg_info->reg_rules_2g_ptr) {
+	if (num_2ghz_reg_rules) {
+		reg_info->reg_rules_2ghz_ptr =
+				create_reg_rules_from_wmi(num_2ghz_reg_rules,
+							  wmi_reg_rule);
+		if (!reg_info->reg_rules_2ghz_ptr) {
 			kfree(tb);
-			ath11k_warn(ab, "Unable to Allocate memory for 2g rules\n");
+			ath11k_warn(ab, "Unable to Allocate memory for 2 GHz rules\n");
 			return -ENOMEM;
 		}
+
+		ath11k_print_reg_rule(ab, "2 GHz",
+				      num_2ghz_reg_rules,
+				      reg_info->reg_rules_2ghz_ptr);
 	}
 
-	if (num_5g_reg_rules) {
-		wmi_reg_rule += num_2g_reg_rules;
-		reg_info->reg_rules_5g_ptr = create_reg_rules_from_wmi(num_5g_reg_rules,
-								       wmi_reg_rule);
-		if (!reg_info->reg_rules_5g_ptr) {
+	if (num_5ghz_reg_rules) {
+		wmi_reg_rule += num_2ghz_reg_rules;
+		reg_info->reg_rules_5ghz_ptr =
+				create_reg_rules_from_wmi(num_5ghz_reg_rules,
+							  wmi_reg_rule);
+		if (!reg_info->reg_rules_5ghz_ptr) {
 			kfree(tb);
-			ath11k_warn(ab, "Unable to Allocate memory for 5g rules\n");
+			ath11k_warn(ab, "Unable to Allocate memory for 5 GHz rules\n");
 			return -ENOMEM;
 		}
+
+		ath11k_print_reg_rule(ab, "5 GHz",
+				      num_5ghz_reg_rules,
+				      reg_info->reg_rules_5ghz_ptr);
 	}
 
 	ath11k_dbg(ab, ATH11K_DBG_WMI, "processed regulatory channel list\n");
@@ -5050,6 +5102,429 @@ static int ath11k_pull_reg_chan_list_update_ev(struct ath11k_base *ab,
 	return 0;
 }
 
+static struct cur_reg_rule
+*create_ext_reg_rules_from_wmi(u32 num_reg_rules,
+			       struct wmi_regulatory_ext_rule *wmi_reg_rule)
+{
+	struct cur_reg_rule *reg_rule_ptr;
+	u32 count;
+
+	reg_rule_ptr =  kcalloc(num_reg_rules, sizeof(*reg_rule_ptr), GFP_ATOMIC);
+
+	if (!reg_rule_ptr)
+		return NULL;
+
+	for (count = 0; count < num_reg_rules; count++) {
+		reg_rule_ptr[count].start_freq =
+			u32_get_bits(wmi_reg_rule[count].freq_info,
+				     REG_RULE_START_FREQ);
+		reg_rule_ptr[count].end_freq =
+			u32_get_bits(wmi_reg_rule[count].freq_info,
+				     REG_RULE_END_FREQ);
+		reg_rule_ptr[count].max_bw =
+			u32_get_bits(wmi_reg_rule[count].bw_pwr_info,
+				     REG_RULE_MAX_BW);
+		reg_rule_ptr[count].reg_power =
+			u32_get_bits(wmi_reg_rule[count].bw_pwr_info,
+				     REG_RULE_REG_PWR);
+		reg_rule_ptr[count].ant_gain =
+			u32_get_bits(wmi_reg_rule[count].bw_pwr_info,
+				     REG_RULE_ANT_GAIN);
+		reg_rule_ptr[count].flags =
+			u32_get_bits(wmi_reg_rule[count].flag_info,
+				     REG_RULE_FLAGS);
+		reg_rule_ptr[count].psd_flag =
+			u32_get_bits(wmi_reg_rule[count].psd_power_info,
+				     REG_RULE_PSD_INFO);
+		reg_rule_ptr[count].psd_eirp =
+			u32_get_bits(wmi_reg_rule[count].psd_power_info,
+				     REG_RULE_PSD_EIRP);
+	}
+
+	return reg_rule_ptr;
+}
+
+static u8
+ath11k_invalid_5ghz_reg_ext_rules_from_wmi(u32 num_reg_rules,
+					   const struct wmi_regulatory_ext_rule *rule)
+{
+	u8 num_invalid_5ghz_rules = 0;
+	u32 count, start_freq;
+
+	for (count = 0; count < num_reg_rules; count++) {
+		start_freq = u32_get_bits(rule[count].freq_info,
+					  REG_RULE_START_FREQ);
+
+		if (start_freq >= ATH11K_MIN_6G_FREQ)
+			num_invalid_5ghz_rules++;
+	}
+
+	return num_invalid_5ghz_rules;
+}
+
+static int ath11k_pull_reg_chan_list_ext_update_ev(struct ath11k_base *ab,
+						   struct sk_buff *skb,
+						   struct cur_regulatory_info *reg_info)
+{
+	const void **tb;
+	const struct wmi_reg_chan_list_cc_ext_event *ev;
+	struct wmi_regulatory_ext_rule *ext_wmi_reg_rule;
+	u32 num_2ghz_reg_rules, num_5ghz_reg_rules;
+	u32 num_6ghz_reg_rules_ap[WMI_REG_CURRENT_MAX_AP_TYPE];
+	u32 num_6ghz_client[WMI_REG_CURRENT_MAX_AP_TYPE][WMI_REG_MAX_CLIENT_TYPE];
+	u32 total_reg_rules = 0;
+	int ret, i, j, num_invalid_5ghz_ext_rules = 0;
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI, "processing regulatory ext channel list\n");
+
+	tb = ath11k_wmi_tlv_parse_alloc(ab, skb->data, skb->len, GFP_ATOMIC);
+	if (IS_ERR(tb)) {
+		ret = PTR_ERR(tb);
+		ath11k_warn(ab, "failed to parse tlv: %d\n", ret);
+		return ret;
+	}
+
+	ev = tb[WMI_TAG_REG_CHAN_LIST_CC_EXT_EVENT];
+	if (!ev) {
+		ath11k_warn(ab, "failed to fetch reg chan list ext update ev\n");
+		kfree(tb);
+		return -EPROTO;
+	}
+
+	reg_info->num_2ghz_reg_rules = ev->num_2ghz_reg_rules;
+	reg_info->num_5ghz_reg_rules = ev->num_5ghz_reg_rules;
+	reg_info->num_6ghz_rules_ap[WMI_REG_INDOOR_AP] =
+			ev->num_6ghz_reg_rules_ap_lpi;
+	reg_info->num_6ghz_rules_ap[WMI_REG_STANDARD_POWER_AP] =
+			ev->num_6ghz_reg_rules_ap_sp;
+	reg_info->num_6ghz_rules_ap[WMI_REG_VERY_LOW_POWER_AP] =
+			ev->num_6ghz_reg_rules_ap_vlp;
+
+	for (i = 0; i < WMI_REG_MAX_CLIENT_TYPE; i++) {
+		reg_info->num_6ghz_rules_client[WMI_REG_INDOOR_AP][i] =
+			ev->num_6ghz_reg_rules_client_lpi[i];
+		reg_info->num_6ghz_rules_client[WMI_REG_STANDARD_POWER_AP][i] =
+			ev->num_6ghz_reg_rules_client_sp[i];
+		reg_info->num_6ghz_rules_client[WMI_REG_VERY_LOW_POWER_AP][i] =
+			ev->num_6ghz_reg_rules_client_vlp[i];
+	}
+
+	num_2ghz_reg_rules = reg_info->num_2ghz_reg_rules;
+	num_5ghz_reg_rules = reg_info->num_5ghz_reg_rules;
+
+	total_reg_rules += num_2ghz_reg_rules;
+	total_reg_rules += num_5ghz_reg_rules;
+
+	if ((num_2ghz_reg_rules > MAX_REG_RULES) ||
+	    (num_5ghz_reg_rules > MAX_REG_RULES)) {
+		ath11k_warn(ab, "Num reg rules for 2.4 GHz/5 GHz exceeds max limit (num_2ghz_reg_rules: %d num_5ghz_reg_rules: %d max_rules: %d)\n",
+			    num_2ghz_reg_rules, num_5ghz_reg_rules, MAX_REG_RULES);
+		kfree(tb);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < WMI_REG_CURRENT_MAX_AP_TYPE; i++) {
+		num_6ghz_reg_rules_ap[i] = reg_info->num_6ghz_rules_ap[i];
+
+		if (num_6ghz_reg_rules_ap[i] > MAX_6GHZ_REG_RULES) {
+			ath11k_warn(ab, "Num 6 GHz reg rules for AP mode(%d) exceeds max limit (num_6ghz_reg_rules_ap: %d, max_rules: %d)\n",
+				    i, num_6ghz_reg_rules_ap[i], MAX_6GHZ_REG_RULES);
+			kfree(tb);
+			return -EINVAL;
+		}
+
+		total_reg_rules += num_6ghz_reg_rules_ap[i];
+	}
+
+	for (i = 0; i < WMI_REG_MAX_CLIENT_TYPE; i++) {
+		num_6ghz_client[WMI_REG_INDOOR_AP][i] =
+			reg_info->num_6ghz_rules_client[WMI_REG_INDOOR_AP][i];
+		total_reg_rules += num_6ghz_client[WMI_REG_INDOOR_AP][i];
+
+		num_6ghz_client[WMI_REG_STANDARD_POWER_AP][i] =
+			reg_info->num_6ghz_rules_client[WMI_REG_STANDARD_POWER_AP][i];
+		total_reg_rules += num_6ghz_client[WMI_REG_STANDARD_POWER_AP][i];
+
+		num_6ghz_client[WMI_REG_VERY_LOW_POWER_AP][i] =
+			reg_info->num_6ghz_rules_client[WMI_REG_VERY_LOW_POWER_AP][i];
+		total_reg_rules += num_6ghz_client[WMI_REG_VERY_LOW_POWER_AP][i];
+
+		if ((num_6ghz_client[WMI_REG_INDOOR_AP][i] > MAX_6GHZ_REG_RULES) ||
+		    (num_6ghz_client[WMI_REG_STANDARD_POWER_AP][i] >
+							     MAX_6GHZ_REG_RULES) ||
+		    (num_6ghz_client[WMI_REG_VERY_LOW_POWER_AP][i] >
+							     MAX_6GHZ_REG_RULES)) {
+			ath11k_warn(ab,
+				    "Num 6 GHz client reg rules exceeds max limit, for client(type: %d)\n",
+				    i);
+			kfree(tb);
+			return -EINVAL;
+		}
+	}
+
+	if (!total_reg_rules) {
+		ath11k_warn(ab, "No reg rules available\n");
+		kfree(tb);
+		return -EINVAL;
+	}
+
+	memcpy(reg_info->alpha2, &ev->alpha2, REG_ALPHA2_LEN);
+
+	reg_info->dfs_region = ev->dfs_region;
+	reg_info->phybitmap = ev->phybitmap;
+	reg_info->num_phy = ev->num_phy;
+	reg_info->phy_id = ev->phy_id;
+	reg_info->ctry_code = ev->country_id;
+	reg_info->reg_dmn_pair = ev->domain_code;
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "status_code %s",
+		   ath11k_cc_status_to_str(reg_info->status_code));
+
+	reg_info->status_code =
+		ath11k_wmi_cc_setting_code_to_reg(ev->status_code);
+
+	reg_info->is_ext_reg_event = true;
+
+	reg_info->min_bw_2ghz = ev->min_bw_2ghz;
+	reg_info->max_bw_2ghz = ev->max_bw_2ghz;
+	reg_info->min_bw_5ghz = ev->min_bw_5ghz;
+	reg_info->max_bw_5ghz = ev->max_bw_5ghz;
+
+	reg_info->min_bw_6ghz_ap[WMI_REG_INDOOR_AP] =
+			ev->min_bw_6ghz_ap_lpi;
+	reg_info->max_bw_6ghz_ap[WMI_REG_INDOOR_AP] =
+			ev->max_bw_6ghz_ap_lpi;
+	reg_info->min_bw_6ghz_ap[WMI_REG_STANDARD_POWER_AP] =
+			ev->min_bw_6ghz_ap_sp;
+	reg_info->max_bw_6ghz_ap[WMI_REG_STANDARD_POWER_AP] =
+			ev->max_bw_6ghz_ap_sp;
+	reg_info->min_bw_6ghz_ap[WMI_REG_VERY_LOW_POWER_AP] =
+			ev->min_bw_6ghz_ap_vlp;
+	reg_info->max_bw_6ghz_ap[WMI_REG_VERY_LOW_POWER_AP] =
+			ev->max_bw_6ghz_ap_vlp;
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "6 GHz AP BW: LPI (%d - %d), SP (%d - %d), VLP (%d - %d)\n",
+		   reg_info->min_bw_6ghz_ap[WMI_REG_INDOOR_AP],
+		   reg_info->max_bw_6ghz_ap[WMI_REG_INDOOR_AP],
+		   reg_info->min_bw_6ghz_ap[WMI_REG_STANDARD_POWER_AP],
+		   reg_info->max_bw_6ghz_ap[WMI_REG_STANDARD_POWER_AP],
+		   reg_info->min_bw_6ghz_ap[WMI_REG_VERY_LOW_POWER_AP],
+		   reg_info->max_bw_6ghz_ap[WMI_REG_VERY_LOW_POWER_AP]);
+
+	for (i = 0; i < WMI_REG_MAX_CLIENT_TYPE; i++) {
+		reg_info->min_bw_6ghz_client[WMI_REG_INDOOR_AP][i] =
+				ev->min_bw_6ghz_client_lpi[i];
+		reg_info->max_bw_6ghz_client[WMI_REG_INDOOR_AP][i] =
+				ev->max_bw_6ghz_client_lpi[i];
+		reg_info->min_bw_6ghz_client[WMI_REG_STANDARD_POWER_AP][i] =
+				ev->min_bw_6ghz_client_sp[i];
+		reg_info->max_bw_6ghz_client[WMI_REG_STANDARD_POWER_AP][i] =
+				ev->max_bw_6ghz_client_sp[i];
+		reg_info->min_bw_6ghz_client[WMI_REG_VERY_LOW_POWER_AP][i] =
+				ev->min_bw_6ghz_client_vlp[i];
+		reg_info->max_bw_6ghz_client[WMI_REG_VERY_LOW_POWER_AP][i] =
+				ev->max_bw_6ghz_client_vlp[i];
+
+		ath11k_dbg(ab, ATH11K_DBG_WMI,
+			   "6 GHz %s BW: LPI (%d - %d), SP (%d - %d), VLP (%d - %d)\n",
+			   ath11k_6ghz_client_type_to_str(i),
+			   reg_info->min_bw_6ghz_client[WMI_REG_INDOOR_AP][i],
+			   reg_info->max_bw_6ghz_client[WMI_REG_INDOOR_AP][i],
+			   reg_info->min_bw_6ghz_client[WMI_REG_STANDARD_POWER_AP][i],
+			   reg_info->max_bw_6ghz_client[WMI_REG_STANDARD_POWER_AP][i],
+			   reg_info->min_bw_6ghz_client[WMI_REG_VERY_LOW_POWER_AP][i],
+			   reg_info->max_bw_6ghz_client[WMI_REG_VERY_LOW_POWER_AP][i]);
+	}
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "cc_ext %s dsf %d BW: min_2ghz %d max_2ghz %d min_5ghz %d max_5ghz %d",
+		   reg_info->alpha2, reg_info->dfs_region,
+		   reg_info->min_bw_2ghz, reg_info->max_bw_2ghz,
+		   reg_info->min_bw_5ghz, reg_info->max_bw_5ghz);
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "num_2ghz_reg_rules %d num_5ghz_reg_rules %d",
+		   num_2ghz_reg_rules, num_5ghz_reg_rules);
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "num_6ghz_reg_rules_ap_lpi: %d num_6ghz_reg_rules_ap_sp: %d num_6ghz_reg_rules_ap_vlp: %d",
+		   num_6ghz_reg_rules_ap[WMI_REG_INDOOR_AP],
+		   num_6ghz_reg_rules_ap[WMI_REG_STANDARD_POWER_AP],
+		   num_6ghz_reg_rules_ap[WMI_REG_VERY_LOW_POWER_AP]);
+
+	j = WMI_REG_DEFAULT_CLIENT;
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "6 GHz Regular client: num_6ghz_reg_rules_lpi: %d num_6ghz_reg_rules_sp: %d num_6ghz_reg_rules_vlp: %d",
+		   num_6ghz_client[WMI_REG_INDOOR_AP][j],
+		   num_6ghz_client[WMI_REG_STANDARD_POWER_AP][j],
+		   num_6ghz_client[WMI_REG_VERY_LOW_POWER_AP][j]);
+
+	j = WMI_REG_SUBORDINATE_CLIENT;
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "6 GHz Subordinate client: num_6ghz_reg_rules_lpi: %d num_6ghz_reg_rules_sp: %d num_6ghz_reg_rules_vlp: %d",
+		   num_6ghz_client[WMI_REG_INDOOR_AP][j],
+		   num_6ghz_client[WMI_REG_STANDARD_POWER_AP][j],
+		   num_6ghz_client[WMI_REG_VERY_LOW_POWER_AP][j]);
+
+	ext_wmi_reg_rule =
+		(struct wmi_regulatory_ext_rule *)((u8 *)ev + sizeof(*ev) +
+						   sizeof(struct wmi_tlv));
+	if (num_2ghz_reg_rules) {
+		reg_info->reg_rules_2ghz_ptr =
+			create_ext_reg_rules_from_wmi(num_2ghz_reg_rules,
+						      ext_wmi_reg_rule);
+
+		if (!reg_info->reg_rules_2ghz_ptr) {
+			kfree(tb);
+			ath11k_warn(ab, "Unable to Allocate memory for 2 GHz rules\n");
+			return -ENOMEM;
+		}
+
+		ath11k_print_reg_rule(ab, "2 GHz",
+				      num_2ghz_reg_rules,
+				      reg_info->reg_rules_2ghz_ptr);
+	}
+
+	ext_wmi_reg_rule += num_2ghz_reg_rules;
+
+	/* Firmware might include 6 GHz reg rule in 5 GHz rule list
+	 * for few countries along with separate 6 GHz rule.
+	 * Having same 6 GHz reg rule in 5 GHz and 6 GHz rules list
+	 * causes intersect check to be true, and same rules will be
+	 * shown multiple times in iw cmd.
+	 * Hence, avoid parsing 6 GHz rule from 5 GHz reg rule list
+	 */
+	num_invalid_5ghz_ext_rules =
+		ath11k_invalid_5ghz_reg_ext_rules_from_wmi(num_5ghz_reg_rules,
+							   ext_wmi_reg_rule);
+
+	if (num_invalid_5ghz_ext_rules) {
+		ath11k_dbg(ab, ATH11K_DBG_WMI,
+			   "CC: %s 5 GHz reg rules number %d from fw, %d number of invalid 5 GHz rules",
+			   reg_info->alpha2, reg_info->num_5ghz_reg_rules,
+			   num_invalid_5ghz_ext_rules);
+
+		num_5ghz_reg_rules = num_5ghz_reg_rules - num_invalid_5ghz_ext_rules;
+		reg_info->num_5ghz_reg_rules = num_5ghz_reg_rules;
+	}
+
+	if (num_5ghz_reg_rules) {
+		reg_info->reg_rules_5ghz_ptr =
+			create_ext_reg_rules_from_wmi(num_5ghz_reg_rules,
+						      ext_wmi_reg_rule);
+
+		if (!reg_info->reg_rules_5ghz_ptr) {
+			kfree(tb);
+			ath11k_warn(ab, "Unable to Allocate memory for 5 GHz rules\n");
+			return -ENOMEM;
+		}
+
+		ath11k_print_reg_rule(ab, "5 GHz",
+				      num_5ghz_reg_rules,
+				      reg_info->reg_rules_5ghz_ptr);
+	}
+
+	/* We have adjusted the number of 5 GHz reg rules above. But still those
+	 * many rules needs to be adjusted in ext_wmi_reg_rule.
+	 *
+	 * NOTE: num_invalid_5ghz_ext_rules will be 0 for rest other cases.
+	 */
+	ext_wmi_reg_rule += (num_5ghz_reg_rules + num_invalid_5ghz_ext_rules);
+
+	for (i = 0; i < WMI_REG_CURRENT_MAX_AP_TYPE; i++) {
+		reg_info->reg_rules_6ghz_ap_ptr[i] =
+			create_ext_reg_rules_from_wmi(num_6ghz_reg_rules_ap[i],
+						      ext_wmi_reg_rule);
+
+		if (!reg_info->reg_rules_6ghz_ap_ptr[i]) {
+			kfree(tb);
+			ath11k_warn(ab, "Unable to Allocate memory for 6 GHz AP rules\n");
+			return -ENOMEM;
+		}
+
+		ath11k_print_reg_rule(ab, ath11k_6ghz_ap_type_to_str(i),
+				      num_6ghz_reg_rules_ap[i],
+				      reg_info->reg_rules_6ghz_ap_ptr[i]);
+
+		ext_wmi_reg_rule += num_6ghz_reg_rules_ap[i];
+	}
+
+	for (j = 0; j < WMI_REG_CURRENT_MAX_AP_TYPE; j++) {
+		ath11k_dbg(ab, ATH11K_DBG_WMI,
+			   "6 GHz AP type %s", ath11k_6ghz_ap_type_to_str(j));
+
+		for (i = 0; i < WMI_REG_MAX_CLIENT_TYPE; i++) {
+			reg_info->reg_rules_6ghz_client_ptr[j][i] =
+				create_ext_reg_rules_from_wmi(num_6ghz_client[j][i],
+							      ext_wmi_reg_rule);
+
+			if (!reg_info->reg_rules_6ghz_client_ptr[j][i]) {
+				kfree(tb);
+				ath11k_warn(ab, "Unable to Allocate memory for 6 GHz client rules\n");
+				return -ENOMEM;
+			}
+
+			ath11k_print_reg_rule(ab,
+					      ath11k_6ghz_client_type_to_str(i),
+					      num_6ghz_client[j][i],
+					      reg_info->reg_rules_6ghz_client_ptr[j][i]);
+
+			ext_wmi_reg_rule += num_6ghz_client[j][i];
+		}
+	}
+
+	reg_info->client_type = ev->client_type;
+	reg_info->rnr_tpe_usable = ev->rnr_tpe_usable;
+	reg_info->unspecified_ap_usable =
+			ev->unspecified_ap_usable;
+	reg_info->domain_code_6ghz_ap[WMI_REG_INDOOR_AP] =
+			ev->domain_code_6ghz_ap_lpi;
+	reg_info->domain_code_6ghz_ap[WMI_REG_STANDARD_POWER_AP] =
+			ev->domain_code_6ghz_ap_sp;
+	reg_info->domain_code_6ghz_ap[WMI_REG_VERY_LOW_POWER_AP] =
+			ev->domain_code_6ghz_ap_vlp;
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "6 GHz reg info client type %s rnr_tpe_usable %d unspecified_ap_usable %d AP sub domain: lpi %s, sp %s, vlp %s\n",
+		   ath11k_6ghz_client_type_to_str(reg_info->client_type),
+		   reg_info->rnr_tpe_usable,
+		   reg_info->unspecified_ap_usable,
+		   ath11k_sub_reg_6ghz_to_str(ev->domain_code_6ghz_ap_lpi),
+		   ath11k_sub_reg_6ghz_to_str(ev->domain_code_6ghz_ap_sp),
+		   ath11k_sub_reg_6ghz_to_str(ev->domain_code_6ghz_ap_vlp));
+
+	for (i = 0; i < WMI_REG_MAX_CLIENT_TYPE; i++) {
+		reg_info->domain_code_6ghz_client[WMI_REG_INDOOR_AP][i] =
+				ev->domain_code_6ghz_client_lpi[i];
+		reg_info->domain_code_6ghz_client[WMI_REG_STANDARD_POWER_AP][i] =
+				ev->domain_code_6ghz_client_sp[i];
+		reg_info->domain_code_6ghz_client[WMI_REG_VERY_LOW_POWER_AP][i] =
+				ev->domain_code_6ghz_client_vlp[i];
+
+		ath11k_dbg(ab, ATH11K_DBG_WMI,
+			   "6 GHz client type %s client sub domain: lpi %s, sp %s, vlp %s\n",
+			   ath11k_6ghz_client_type_to_str(i),
+			   ath11k_sub_reg_6ghz_to_str(ev->domain_code_6ghz_client_lpi[i]),
+			   ath11k_sub_reg_6ghz_to_str(ev->domain_code_6ghz_client_sp[i]),
+			   ath11k_sub_reg_6ghz_to_str(ev->domain_code_6ghz_client_vlp[i])
+			  );
+	}
+
+	reg_info->domain_code_6ghz_super_id = ev->domain_code_6ghz_super_id;
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "6 GHz client_type %s 6 GHz super domain %s",
+		   ath11k_6ghz_client_type_to_str(reg_info->client_type),
+		   ath11k_super_reg_6ghz_to_str(reg_info->domain_code_6ghz_super_id));
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI, "processed regulatory ext channel list\n");
+
+	kfree(tb);
+	return 0;
+}
+
 static int ath11k_pull_peer_del_resp_ev(struct ath11k_base *ab, struct sk_buff *skb,
 					struct wmi_peer_delete_resp_event *peer_del_resp)
 {
@@ -5165,28 +5640,49 @@ static int ath11k_pull_vdev_stopped_param_tlv(struct ath11k_base *ab, struct sk_
 	return 0;
 }
 
+static int ath11k_wmi_tlv_mgmt_rx_parse(struct ath11k_base *ab,
+					u16 tag, u16 len,
+					const void *ptr, void *data)
+{
+	struct wmi_tlv_mgmt_rx_parse *parse = data;
+
+	switch (tag) {
+	case WMI_TAG_MGMT_RX_HDR:
+		parse->fixed = ptr;
+		break;
+	case WMI_TAG_ARRAY_BYTE:
+		if (!parse->frame_buf_done) {
+			parse->frame_buf = ptr;
+			parse->frame_buf_done = true;
+		}
+		break;
+	}
+	return 0;
+}
+
 static int ath11k_pull_mgmt_rx_params_tlv(struct ath11k_base *ab,
 					  struct sk_buff *skb,
 					  struct mgmt_rx_event_params *hdr)
 {
-	const void **tb;
+	struct wmi_tlv_mgmt_rx_parse parse = { };
 	const struct wmi_mgmt_rx_hdr *ev;
 	const u8 *frame;
 	int ret;
 
-	tb = ath11k_wmi_tlv_parse_alloc(ab, skb->data, skb->len, GFP_ATOMIC);
-	if (IS_ERR(tb)) {
-		ret = PTR_ERR(tb);
-		ath11k_warn(ab, "failed to parse tlv: %d\n", ret);
+	ret = ath11k_wmi_tlv_iter(ab, skb->data, skb->len,
+				  ath11k_wmi_tlv_mgmt_rx_parse,
+				  &parse);
+	if (ret) {
+		ath11k_warn(ab, "failed to parse mgmt rx tlv %d\n",
+			    ret);
 		return ret;
 	}
 
-	ev = tb[WMI_TAG_MGMT_RX_HDR];
-	frame = tb[WMI_TAG_ARRAY_BYTE];
+	ev = parse.fixed;
+	frame = parse.frame_buf;
 
 	if (!ev || !frame) {
 		ath11k_warn(ab, "failed to fetch mgmt rx hdr");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -5205,7 +5701,6 @@ static int ath11k_pull_mgmt_rx_params_tlv(struct ath11k_base *ab,
 
 	if (skb->len < (frame - skb->data) + hdr->buf_len) {
 		ath11k_warn(ab, "invalid length in mgmt rx hdr ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -5217,12 +5712,11 @@ static int ath11k_pull_mgmt_rx_params_tlv(struct ath11k_base *ab,
 
 	ath11k_ce_byte_swap(skb->data, hdr->buf_len);
 
-	kfree(tb);
 	return 0;
 }
 
-static int wmi_process_mgmt_tx_comp(struct ath11k *ar, u32 desc_id,
-				    u32 status)
+static int wmi_process_mgmt_tx_comp(struct ath11k *ar,
+				    struct wmi_mgmt_tx_compl_event *tx_compl_param)
 {
 	struct sk_buff *msdu;
 	struct ieee80211_tx_info *info;
@@ -5230,24 +5724,29 @@ static int wmi_process_mgmt_tx_comp(struct ath11k *ar, u32 desc_id,
 	int num_mgmt;
 
 	spin_lock_bh(&ar->txmgmt_idr_lock);
-	msdu = idr_find(&ar->txmgmt_idr, desc_id);
+	msdu = idr_find(&ar->txmgmt_idr, tx_compl_param->desc_id);
 
 	if (!msdu) {
 		ath11k_warn(ar->ab, "received mgmt tx compl for invalid msdu_id: %d\n",
-			    desc_id);
+			    tx_compl_param->desc_id);
 		spin_unlock_bh(&ar->txmgmt_idr_lock);
 		return -ENOENT;
 	}
 
-	idr_remove(&ar->txmgmt_idr, desc_id);
+	idr_remove(&ar->txmgmt_idr, tx_compl_param->desc_id);
 	spin_unlock_bh(&ar->txmgmt_idr_lock);
 
 	skb_cb = ATH11K_SKB_CB(msdu);
 	dma_unmap_single(ar->ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE);
 
 	info = IEEE80211_SKB_CB(msdu);
-	if ((!(info->flags & IEEE80211_TX_CTL_NO_ACK)) && !status)
+	if ((!(info->flags & IEEE80211_TX_CTL_NO_ACK)) &&
+	    !tx_compl_param->status) {
 		info->flags |= IEEE80211_TX_STAT_ACK;
+		if (test_bit(WMI_TLV_SERVICE_TX_DATA_MGMT_ACK_RSSI,
+			     ar->ab->wmi_ab.svc_map))
+			info->status.ack_signal = tx_compl_param->ack_rssi;
+	}
 
 	ieee80211_tx_status_irqsafe(ar->hw, msdu);
 
@@ -5259,7 +5758,7 @@ static int wmi_process_mgmt_tx_comp(struct ath11k *ar, u32 desc_id,
 
 	ath11k_dbg(ar->ab, ATH11K_DBG_WMI,
 		   "wmi mgmt tx comp pending %d desc id %d\n",
-		   num_mgmt, desc_id);
+		   num_mgmt, tx_compl_param->desc_id);
 
 	if (!num_mgmt)
 		wake_up(&ar->txmgmt_empty_waitq);
@@ -5292,6 +5791,7 @@ static int ath11k_pull_mgmt_tx_compl_param_tlv(struct ath11k_base *ab,
 	param->pdev_id = ev->pdev_id;
 	param->desc_id = ev->desc_id;
 	param->status = ev->status;
+	param->ack_rssi = ev->ack_rssi;
 
 	kfree(tb);
 	return 0;
@@ -6491,12 +6991,14 @@ static bool ath11k_reg_is_world_alpha(char *alpha)
 	return false;
 }
 
-static int ath11k_reg_chan_list_event(struct ath11k_base *ab, struct sk_buff *skb)
+static int ath11k_reg_chan_list_event(struct ath11k_base *ab,
+				      struct sk_buff *skb,
+				      enum wmi_reg_chan_list_cmd_type id)
 {
 	struct cur_regulatory_info *reg_info = NULL;
 	struct ieee80211_regdomain *regd = NULL;
 	bool intersect = false;
-	int ret = 0, pdev_idx;
+	int ret = 0, pdev_idx, i, j;
 	struct ath11k *ar;
 
 	reg_info = kzalloc(sizeof(*reg_info), GFP_ATOMIC);
@@ -6505,7 +7007,11 @@ static int ath11k_reg_chan_list_event(struct ath11k_base *ab, struct sk_buff *sk
 		goto fallback;
 	}
 
-	ret = ath11k_pull_reg_chan_list_update_ev(ab, skb, reg_info);
+	if (id == WMI_REG_CHAN_LIST_CC_ID)
+		ret = ath11k_pull_reg_chan_list_update_ev(ab, skb, reg_info);
+	else
+		ret = ath11k_pull_reg_chan_list_ext_update_ev(ab, skb, reg_info);
+
 	if (ret) {
 		ath11k_warn(ab, "failed to extract regulatory info from received event\n");
 		goto fallback;
@@ -6605,8 +7111,16 @@ fallback:
 	WARN_ON(1);
 mem_free:
 	if (reg_info) {
-		kfree(reg_info->reg_rules_2g_ptr);
-		kfree(reg_info->reg_rules_5g_ptr);
+		kfree(reg_info->reg_rules_2ghz_ptr);
+		kfree(reg_info->reg_rules_5ghz_ptr);
+		if (reg_info->is_ext_reg_event) {
+			for (i = 0; i < WMI_REG_CURRENT_MAX_AP_TYPE; i++)
+				kfree(reg_info->reg_rules_6ghz_ap_ptr[i]);
+
+			for (j = 0; j < WMI_REG_CURRENT_MAX_AP_TYPE; j++)
+				for (i = 0; i < WMI_REG_MAX_CLIENT_TYPE; i++)
+					kfree(reg_info->reg_rules_6ghz_client_ptr[j][i]);
+		}
 		kfree(reg_info);
 	}
 	return ret;
@@ -7062,13 +7576,12 @@ static void ath11k_mgmt_tx_compl_event(struct ath11k_base *ab, struct sk_buff *s
 		goto exit;
 	}
 
-	wmi_process_mgmt_tx_comp(ar, tx_compl_param.desc_id,
-				 tx_compl_param.status);
+	wmi_process_mgmt_tx_comp(ar, &tx_compl_param);
 
 	ath11k_dbg(ab, ATH11K_DBG_MGMT,
-		   "mgmt tx compl ev pdev_id %d, desc_id %d, status %d",
+		   "mgmt tx compl ev pdev_id %d, desc_id %d, status %d ack_rssi %d",
 		   tx_compl_param.pdev_id, tx_compl_param.desc_id,
-		   tx_compl_param.status);
+		   tx_compl_param.status, tx_compl_param.ack_rssi);
 
 exit:
 	rcu_read_unlock();
@@ -8039,7 +8552,10 @@ static void ath11k_wmi_tlv_op_rx(struct ath11k_base *ab, struct sk_buff *skb)
 		ath11k_service_ready_ext2_event(ab, skb);
 		break;
 	case WMI_REG_CHAN_LIST_CC_EVENTID:
-		ath11k_reg_chan_list_event(ab, skb);
+		ath11k_reg_chan_list_event(ab, skb, WMI_REG_CHAN_LIST_CC_ID);
+		break;
+	case WMI_REG_CHAN_LIST_CC_EXT_EVENTID:
+		ath11k_reg_chan_list_event(ab, skb, WMI_REG_CHAN_LIST_CC_EXT_ID);
 		break;
 	case WMI_READY_EVENTID:
 		ath11k_ready_event(ab, skb);
diff --git a/drivers/net/wireless/ath/ath11k/wmi.h b/drivers/net/wireless/ath/ath11k/wmi.h
index 0a045af5419b..92fddb77669c 100644
--- a/drivers/net/wireless/ath/ath11k/wmi.h
+++ b/drivers/net/wireless/ath/ath11k/wmi.h
@@ -797,6 +797,7 @@ enum wmi_tlv_event_id {
 	WMI_RMC_NEW_LEADER_EVENTID = WMI_TLV_CMD(WMI_GRP_RMC),
 	WMI_REG_CHAN_LIST_CC_EVENTID = WMI_TLV_CMD(WMI_GRP_REGULATORY),
 	WMI_11D_NEW_COUNTRY_EVENTID,
+	WMI_REG_CHAN_LIST_CC_EXT_EVENTID,
 	WMI_NDI_CAP_RSP_EVENTID = WMI_TLV_CMD(WMI_GRP_PROTOTYPE),
 	WMI_NDP_INITIATOR_RSP_EVENTID,
 	WMI_NDP_RESPONDER_RSP_EVENTID,
@@ -1865,6 +1866,8 @@ enum wmi_tlv_tag {
 	WMI_TAG_PDEV_SRG_OBSS_BSSID_ENABLE_BITMAP_CMD,
 	WMI_TAG_PDEV_NON_SRG_OBSS_COLOR_ENABLE_BITMAP_CMD,
 	WMI_TAG_PDEV_NON_SRG_OBSS_BSSID_ENABLE_BITMAP_CMD,
+	WMI_TAG_REGULATORY_RULE_EXT_STRUCT = 0x3A9,
+	WMI_TAG_REG_CHAN_LIST_CC_EXT_EVENT,
 	WMI_TAG_PDEV_SET_BIOS_SAR_TABLE_CMD = 0x3D8,
 	WMI_TAG_PDEV_SET_BIOS_GEO_TABLE_CMD,
 	WMI_TAG_MAX
@@ -2093,10 +2096,14 @@ enum wmi_tlv_service {
 	WMI_TLV_SERVICE_EXT2_MSG = 220,
 	WMI_TLV_SERVICE_PEER_POWER_SAVE_DURATION_SUPPORT = 246,
 	WMI_TLV_SERVICE_SRG_SRP_SPATIAL_REUSE_SUPPORT = 249,
+	WMI_TLV_SERVICE_PASSIVE_SCAN_START_TIME_ENHANCE = 263,
 
 	/* The second 128 bits */
 	WMI_MAX_EXT_SERVICE = 256,
+	WMI_TLV_SERVICE_SCAN_CONFIG_PER_CHANNEL = 265,
+	WMI_TLV_SERVICE_REG_CC_EXT_EVENT_SUPPORT = 281,
 	WMI_TLV_SERVICE_BIOS_SAR_SUPPORT = 326,
+	WMI_TLV_SERVICE_SUPPORT_11D_FOR_HOST_SCAN = 357,
 
 	/* The third 128 bits */
 	WMI_MAX_EXT2_SERVICE = 384
@@ -2310,6 +2317,9 @@ struct wmi_init_cmd {
 } __packed;
 
 #define WMI_RSRC_CFG_FLAG1_BSS_CHANNEL_INFO_64 BIT(5)
+#define WMI_RSRC_CFG_FLAG1_ACK_RSSI BIT(18)
+
+#define WMI_CFG_HOST_SERVICE_FLAG_REG_CC_EXT 4
 
 struct wmi_resource_config {
 	u32 tlv_header;
@@ -2370,6 +2380,15 @@ struct wmi_resource_config {
 	u32 sched_params;
 	u32 twt_ap_pdev_count;
 	u32 twt_ap_sta_count;
+	u32 max_nlo_ssids;
+	u32 num_pkt_filters;
+	u32 num_max_sta_vdevs;
+	u32 max_bssid_indicator;
+	u32 ul_resp_config;
+	u32 msdu_flow_override_config0;
+	u32 msdu_flow_override_config1;
+	u32 flags2;
+	u32 host_service_flags;
 } __packed;
 
 struct wmi_service_ready_event {
@@ -2852,36 +2871,40 @@ struct rx_reorder_queue_remove_params {
 #define REG_RULE_MAX_BW				0x0000ffff
 #define REG_RULE_REG_PWR			0x00ff0000
 #define REG_RULE_ANT_GAIN			0xff000000
+#define REG_RULE_PSD_INFO			BIT(0)
+#define REG_RULE_PSD_EIRP			0xff0000
 
 #define WMI_VDEV_PARAM_TXBF_SU_TX_BFEE BIT(0)
 #define WMI_VDEV_PARAM_TXBF_MU_TX_BFEE BIT(1)
 #define WMI_VDEV_PARAM_TXBF_SU_TX_BFER BIT(2)
 #define WMI_VDEV_PARAM_TXBF_MU_TX_BFER BIT(3)
 
-#define HECAP_PHYDWORD_0	0
-#define HECAP_PHYDWORD_1	1
-#define HECAP_PHYDWORD_2	2
+#define HE_PHYCAP_BYTE_0	0
+#define HE_PHYCAP_BYTE_1	1
+#define HE_PHYCAP_BYTE_2	2
+#define HE_PHYCAP_BYTE_3	3
+#define HE_PHYCAP_BYTE_4	4
 
-#define HECAP_PHY_SU_BFER		BIT(31)
+#define HECAP_PHY_SU_BFER		BIT(7)
 #define HECAP_PHY_SU_BFEE		BIT(0)
 #define HECAP_PHY_MU_BFER		BIT(1)
-#define HECAP_PHY_UL_MUMIMO		BIT(22)
-#define HECAP_PHY_UL_MUOFDMA		BIT(23)
+#define HECAP_PHY_UL_MUMIMO		BIT(6)
+#define HECAP_PHY_UL_MUOFDMA		BIT(7)
 
 #define HECAP_PHY_SUBFMR_GET(hecap_phy) \
-	FIELD_GET(HECAP_PHY_SU_BFER, hecap_phy[HECAP_PHYDWORD_0])
+	FIELD_GET(HECAP_PHY_SU_BFER, hecap_phy[HE_PHYCAP_BYTE_3])
 
 #define HECAP_PHY_SUBFME_GET(hecap_phy) \
-	FIELD_GET(HECAP_PHY_SU_BFEE, hecap_phy[HECAP_PHYDWORD_1])
+	FIELD_GET(HECAP_PHY_SU_BFEE, hecap_phy[HE_PHYCAP_BYTE_4])
 
 #define HECAP_PHY_MUBFMR_GET(hecap_phy) \
-	FIELD_GET(HECAP_PHY_MU_BFER, hecap_phy[HECAP_PHYDWORD_1])
+	FIELD_GET(HECAP_PHY_MU_BFER, hecap_phy[HE_PHYCAP_BYTE_4])
 
 #define HECAP_PHY_ULMUMIMO_GET(hecap_phy) \
-	FIELD_GET(HECAP_PHY_UL_MUMIMO, hecap_phy[HECAP_PHYDWORD_0])
+	FIELD_GET(HECAP_PHY_UL_MUMIMO, hecap_phy[HE_PHYCAP_BYTE_2])
 
 #define HECAP_PHY_ULOFDMA_GET(hecap_phy) \
-	FIELD_GET(HECAP_PHY_UL_MUOFDMA, hecap_phy[HECAP_PHYDWORD_0])
+	FIELD_GET(HECAP_PHY_UL_MUOFDMA, hecap_phy[HE_PHYCAP_BYTE_2])
 
 #define HE_MODE_SU_TX_BFEE	BIT(0)
 #define HE_MODE_SU_TX_BFER	BIT(1)
@@ -2894,8 +2917,11 @@ struct rx_reorder_queue_remove_params {
 #define HE_DL_MUOFDMA_ENABLE	1
 #define HE_UL_MUOFDMA_ENABLE	1
 #define HE_DL_MUMIMO_ENABLE	1
+#define HE_UL_MUMIMO_ENABLE	1
 #define HE_MU_BFEE_ENABLE	1
 #define HE_SU_BFEE_ENABLE	1
+#define HE_MU_BFER_ENABLE	1
+#define HE_SU_BFER_ENABLE	1
 
 #define HE_VHT_SOUNDING_MODE_ENABLE		1
 #define HE_SU_MU_SOUNDING_MODE_ENABLE		1
@@ -3223,6 +3249,10 @@ struct  wmi_start_scan_cmd {
 
 #define WMI_SCAN_DWELL_MODE_MASK 0x00E00000
 #define WMI_SCAN_DWELL_MODE_SHIFT        21
+#define WMI_SCAN_FLAG_EXT_PASSIVE_SCAN_START_TIME_ENHANCE   0x00000800
+
+#define WMI_SCAN_CONFIG_PER_CHANNEL_MASK	GENMASK(19, 0)
+#define WMI_SCAN_CH_FLAG_SCAN_ONLY_IF_RNR_FOUND	BIT(20)
 
 enum {
 	WMI_SCAN_DWELL_MODE_DEFAULT      = 0,
@@ -3270,6 +3300,7 @@ struct scan_req_params {
 		};
 		u32 scan_events;
 	};
+	u32 scan_ctrl_flags_ext;
 	u32 dwell_time_active;
 	u32 dwell_time_active_2g;
 	u32 dwell_time_passive;
@@ -4040,6 +4071,7 @@ struct wmi_he_rate_set {
 
 #define MAX_REG_RULES 10
 #define REG_ALPHA2_LEN 2
+#define MAX_6GHZ_REG_RULES 5
 
 enum wmi_start_event_param {
 	WMI_VDEV_START_RESP_EVENT = 0,
@@ -4070,16 +4102,6 @@ enum wmi_vdev_start_resp_status_code {
 	WMI_VDEV_START_RESPONSE_INVALID_REGDOMAIN = 4,
 };
 
-;
-enum cc_setting_code {
-	REG_SET_CC_STATUS_PASS = 0,
-	REG_CURRENT_ALPHA2_NOT_FOUND = 1,
-	REG_INIT_ALPHA2_NOT_FOUND = 2,
-	REG_SET_CC_CHANGE_NOT_ALLOWED = 3,
-	REG_SET_CC_STATUS_NO_MEMORY = 4,
-	REG_SET_CC_STATUS_FAIL = 5,
-};
-
 /* Regaulatory Rule Flags Passed by FW */
 #define REGULATORY_CHAN_DISABLED     BIT(0)
 #define REGULATORY_CHAN_NO_IR        BIT(1)
@@ -4093,15 +4115,216 @@ enum cc_setting_code {
 #define REGULATORY_CHAN_NO_20MHZ     BIT(11)
 #define REGULATORY_CHAN_NO_10MHZ     BIT(12)
 
-enum {
+enum wmi_reg_chan_list_cmd_type {
+	WMI_REG_CHAN_LIST_CC_ID = 0,
+	WMI_REG_CHAN_LIST_CC_EXT_ID = 1,
+};
+
+enum wmi_reg_cc_setting_code {
 	WMI_REG_SET_CC_STATUS_PASS = 0,
 	WMI_REG_CURRENT_ALPHA2_NOT_FOUND = 1,
 	WMI_REG_INIT_ALPHA2_NOT_FOUND = 2,
 	WMI_REG_SET_CC_CHANGE_NOT_ALLOWED = 3,
 	WMI_REG_SET_CC_STATUS_NO_MEMORY = 4,
 	WMI_REG_SET_CC_STATUS_FAIL = 5,
+
+	/* add new setting code above, update in
+	 * @enum cc_setting_code as well.
+	 * Also handle it in ath11k_wmi_cc_setting_code_to_reg()
+	 */
+};
+
+enum cc_setting_code {
+	REG_SET_CC_STATUS_PASS = 0,
+	REG_CURRENT_ALPHA2_NOT_FOUND = 1,
+	REG_INIT_ALPHA2_NOT_FOUND = 2,
+	REG_SET_CC_CHANGE_NOT_ALLOWED = 3,
+	REG_SET_CC_STATUS_NO_MEMORY = 4,
+	REG_SET_CC_STATUS_FAIL = 5,
+
+	/* add new setting code above, update in
+	 * @enum wmi_reg_cc_setting_code as well.
+	 * Also handle it in ath11k_cc_status_to_str()
+	 */
+};
+
+static inline enum cc_setting_code
+ath11k_wmi_cc_setting_code_to_reg(enum wmi_reg_cc_setting_code status_code)
+{
+	switch (status_code) {
+	case WMI_REG_SET_CC_STATUS_PASS:
+		return REG_SET_CC_STATUS_PASS;
+	case WMI_REG_CURRENT_ALPHA2_NOT_FOUND:
+		return REG_CURRENT_ALPHA2_NOT_FOUND;
+	case WMI_REG_INIT_ALPHA2_NOT_FOUND:
+		return REG_INIT_ALPHA2_NOT_FOUND;
+	case WMI_REG_SET_CC_CHANGE_NOT_ALLOWED:
+		return REG_SET_CC_CHANGE_NOT_ALLOWED;
+	case WMI_REG_SET_CC_STATUS_NO_MEMORY:
+		return REG_SET_CC_STATUS_NO_MEMORY;
+	case WMI_REG_SET_CC_STATUS_FAIL:
+		return REG_SET_CC_STATUS_FAIL;
+	}
+
+	return REG_SET_CC_STATUS_FAIL;
+}
+
+static inline const char *ath11k_cc_status_to_str(enum cc_setting_code code)
+{
+	switch (code) {
+	case REG_SET_CC_STATUS_PASS:
+		return "REG_SET_CC_STATUS_PASS";
+	case REG_CURRENT_ALPHA2_NOT_FOUND:
+		return "REG_CURRENT_ALPHA2_NOT_FOUND";
+	case REG_INIT_ALPHA2_NOT_FOUND:
+		return "REG_INIT_ALPHA2_NOT_FOUND";
+	case REG_SET_CC_CHANGE_NOT_ALLOWED:
+		return "REG_SET_CC_CHANGE_NOT_ALLOWED";
+	case REG_SET_CC_STATUS_NO_MEMORY:
+		return "REG_SET_CC_STATUS_NO_MEMORY";
+	case REG_SET_CC_STATUS_FAIL:
+		return "REG_SET_CC_STATUS_FAIL";
+	}
+
+	return "Unknown CC status";
+}
+
+enum wmi_reg_6ghz_ap_type {
+	WMI_REG_INDOOR_AP = 0,
+	WMI_REG_STANDARD_POWER_AP = 1,
+	WMI_REG_VERY_LOW_POWER_AP = 2,
+
+	/* add AP type above, handle in ath11k_6ghz_ap_type_to_str()
+	 */
+	WMI_REG_CURRENT_MAX_AP_TYPE,
+	WMI_REG_MAX_AP_TYPE = 7,
 };
 
+static inline const char *
+ath11k_6ghz_ap_type_to_str(enum wmi_reg_6ghz_ap_type type)
+{
+	switch (type) {
+	case WMI_REG_INDOOR_AP:
+		return "INDOOR AP";
+	case WMI_REG_STANDARD_POWER_AP:
+		return "STANDARD POWER AP";
+	case WMI_REG_VERY_LOW_POWER_AP:
+		return "VERY LOW POWER AP";
+	case WMI_REG_CURRENT_MAX_AP_TYPE:
+		return "CURRENT_MAX_AP_TYPE";
+	case WMI_REG_MAX_AP_TYPE:
+		return "MAX_AP_TYPE";
+	}
+
+	return "unknown 6 GHz AP type";
+}
+
+enum wmi_reg_6ghz_client_type {
+	WMI_REG_DEFAULT_CLIENT = 0,
+	WMI_REG_SUBORDINATE_CLIENT = 1,
+	WMI_REG_MAX_CLIENT_TYPE = 2,
+
+	/* add client type above, handle it in
+	 * ath11k_6ghz_client_type_to_str()
+	 */
+};
+
+static inline const char *
+ath11k_6ghz_client_type_to_str(enum wmi_reg_6ghz_client_type type)
+{
+	switch (type) {
+	case WMI_REG_DEFAULT_CLIENT:
+		return "DEFAULT CLIENT";
+	case WMI_REG_SUBORDINATE_CLIENT:
+		return "SUBORDINATE CLIENT";
+	case WMI_REG_MAX_CLIENT_TYPE:
+		return "MAX_CLIENT_TYPE";
+	}
+
+	return "unknown 6 GHz client type";
+}
+
+enum reg_subdomains_6ghz {
+	EMPTY_6GHZ = 0x0,
+	FCC1_CLIENT_LPI_REGULAR_6GHZ = 0x01,
+	FCC1_CLIENT_SP_6GHZ = 0x02,
+	FCC1_AP_LPI_6GHZ = 0x03,
+	FCC1_CLIENT_LPI_SUBORDINATE = FCC1_AP_LPI_6GHZ,
+	FCC1_AP_SP_6GHZ = 0x04,
+	ETSI1_LPI_6GHZ = 0x10,
+	ETSI1_VLP_6GHZ = 0x11,
+	ETSI2_LPI_6GHZ = 0x12,
+	ETSI2_VLP_6GHZ = 0x13,
+	APL1_LPI_6GHZ = 0x20,
+	APL1_VLP_6GHZ = 0x21,
+
+	/* add sub-domain above, handle it in
+	 * ath11k_sub_reg_6ghz_to_str()
+	 */
+};
+
+static inline const char *
+ath11k_sub_reg_6ghz_to_str(enum reg_subdomains_6ghz sub_id)
+{
+	switch (sub_id) {
+	case EMPTY_6GHZ:
+		return "N/A";
+	case FCC1_CLIENT_LPI_REGULAR_6GHZ:
+		return "FCC1_CLIENT_LPI_REGULAR_6GHZ";
+	case FCC1_CLIENT_SP_6GHZ:
+		return "FCC1_CLIENT_SP_6GHZ";
+	case FCC1_AP_LPI_6GHZ:
+		return "FCC1_AP_LPI_6GHZ/FCC1_CLIENT_LPI_SUBORDINATE";
+	case FCC1_AP_SP_6GHZ:
+		return "FCC1_AP_SP_6GHZ";
+	case ETSI1_LPI_6GHZ:
+		return "ETSI1_LPI_6GHZ";
+	case ETSI1_VLP_6GHZ:
+		return "ETSI1_VLP_6GHZ";
+	case ETSI2_LPI_6GHZ:
+		return "ETSI2_LPI_6GHZ";
+	case ETSI2_VLP_6GHZ:
+		return "ETSI2_VLP_6GHZ";
+	case APL1_LPI_6GHZ:
+		return "APL1_LPI_6GHZ";
+	case APL1_VLP_6GHZ:
+		return "APL1_VLP_6GHZ";
+	}
+
+	return "unknown sub reg id";
+}
+
+enum reg_super_domain_6ghz {
+	FCC1_6GHZ = 0x01,
+	ETSI1_6GHZ = 0x02,
+	ETSI2_6GHZ = 0x03,
+	APL1_6GHZ = 0x04,
+	FCC1_6GHZ_CL = 0x05,
+
+	/* add super domain above, handle it in
+	 * ath11k_super_reg_6ghz_to_str()
+	 */
+};
+
+static inline const char *
+ath11k_super_reg_6ghz_to_str(enum reg_super_domain_6ghz domain_id)
+{
+	switch (domain_id) {
+	case FCC1_6GHZ:
+		return "FCC1_6GHZ";
+	case ETSI1_6GHZ:
+		return "ETSI1_6GHZ";
+	case ETSI2_6GHZ:
+		return "ETSI2_6GHZ";
+	case APL1_6GHZ:
+		return "APL1_6GHZ";
+	case FCC1_6GHZ_CL:
+		return "FCC1_6GHZ_CL";
+	}
+
+	return "unknown domain id";
+}
+
 struct cur_reg_rule {
 	u16 start_freq;
 	u16 end_freq;
@@ -4109,6 +4332,8 @@ struct cur_reg_rule {
 	u8 reg_power;
 	u8 ant_gain;
 	u16 flags;
+	bool psd_flag;
+	s8 psd_eirp;
 };
 
 struct cur_regulatory_info {
@@ -4120,14 +4345,30 @@ struct cur_regulatory_info {
 	u8 alpha2[REG_ALPHA2_LEN + 1];
 	u32 dfs_region;
 	u32 phybitmap;
-	u32 min_bw_2g;
-	u32 max_bw_2g;
-	u32 min_bw_5g;
-	u32 max_bw_5g;
-	u32 num_2g_reg_rules;
-	u32 num_5g_reg_rules;
-	struct cur_reg_rule *reg_rules_2g_ptr;
-	struct cur_reg_rule *reg_rules_5g_ptr;
+	u32 min_bw_2ghz;
+	u32 max_bw_2ghz;
+	u32 min_bw_5ghz;
+	u32 max_bw_5ghz;
+	u32 num_2ghz_reg_rules;
+	u32 num_5ghz_reg_rules;
+	struct cur_reg_rule *reg_rules_2ghz_ptr;
+	struct cur_reg_rule *reg_rules_5ghz_ptr;
+	bool is_ext_reg_event;
+	enum wmi_reg_6ghz_client_type client_type;
+	bool rnr_tpe_usable;
+	bool unspecified_ap_usable;
+	u8 domain_code_6ghz_ap[WMI_REG_CURRENT_MAX_AP_TYPE];
+	u8 domain_code_6ghz_client[WMI_REG_CURRENT_MAX_AP_TYPE][WMI_REG_MAX_CLIENT_TYPE];
+	u32 domain_code_6ghz_super_id;
+	u32 min_bw_6ghz_ap[WMI_REG_CURRENT_MAX_AP_TYPE];
+	u32 max_bw_6ghz_ap[WMI_REG_CURRENT_MAX_AP_TYPE];
+	u32 min_bw_6ghz_client[WMI_REG_CURRENT_MAX_AP_TYPE][WMI_REG_MAX_CLIENT_TYPE];
+	u32 max_bw_6ghz_client[WMI_REG_CURRENT_MAX_AP_TYPE][WMI_REG_MAX_CLIENT_TYPE];
+	u32 num_6ghz_rules_ap[WMI_REG_CURRENT_MAX_AP_TYPE];
+	u32 num_6ghz_rules_client[WMI_REG_CURRENT_MAX_AP_TYPE][WMI_REG_MAX_CLIENT_TYPE];
+	struct cur_reg_rule *reg_rules_6ghz_ap_ptr[WMI_REG_CURRENT_MAX_AP_TYPE];
+	struct cur_reg_rule *reg_rules_6ghz_client_ptr
+		[WMI_REG_CURRENT_MAX_AP_TYPE][WMI_REG_MAX_CLIENT_TYPE];
 };
 
 struct wmi_reg_chan_list_cc_event {
@@ -4139,12 +4380,12 @@ struct wmi_reg_chan_list_cc_event {
 	u32 domain_code;
 	u32 dfs_region;
 	u32 phybitmap;
-	u32 min_bw_2g;
-	u32 max_bw_2g;
-	u32 min_bw_5g;
-	u32 max_bw_5g;
-	u32 num_2g_reg_rules;
-	u32 num_5g_reg_rules;
+	u32 min_bw_2ghz;
+	u32 max_bw_2ghz;
+	u32 min_bw_5ghz;
+	u32 max_bw_5ghz;
+	u32 num_2ghz_reg_rules;
+	u32 num_5ghz_reg_rules;
 } __packed;
 
 struct wmi_regulatory_rule_struct {
@@ -4154,6 +4395,61 @@ struct wmi_regulatory_rule_struct {
 	u32  flag_info;
 };
 
+#define WMI_REG_CLIENT_MAX 4
+
+struct wmi_reg_chan_list_cc_ext_event {
+	u32 status_code;
+	u32 phy_id;
+	u32 alpha2;
+	u32 num_phy;
+	u32 country_id;
+	u32 domain_code;
+	u32 dfs_region;
+	u32 phybitmap;
+	u32 min_bw_2ghz;
+	u32 max_bw_2ghz;
+	u32 min_bw_5ghz;
+	u32 max_bw_5ghz;
+	u32 num_2ghz_reg_rules;
+	u32 num_5ghz_reg_rules;
+	u32 client_type;
+	u32 rnr_tpe_usable;
+	u32 unspecified_ap_usable;
+	u32 domain_code_6ghz_ap_lpi;
+	u32 domain_code_6ghz_ap_sp;
+	u32 domain_code_6ghz_ap_vlp;
+	u32 domain_code_6ghz_client_lpi[WMI_REG_CLIENT_MAX];
+	u32 domain_code_6ghz_client_sp[WMI_REG_CLIENT_MAX];
+	u32 domain_code_6ghz_client_vlp[WMI_REG_CLIENT_MAX];
+	u32 domain_code_6ghz_super_id;
+	u32 min_bw_6ghz_ap_sp;
+	u32 max_bw_6ghz_ap_sp;
+	u32 min_bw_6ghz_ap_lpi;
+	u32 max_bw_6ghz_ap_lpi;
+	u32 min_bw_6ghz_ap_vlp;
+	u32 max_bw_6ghz_ap_vlp;
+	u32 min_bw_6ghz_client_sp[WMI_REG_CLIENT_MAX];
+	u32 max_bw_6ghz_client_sp[WMI_REG_CLIENT_MAX];
+	u32 min_bw_6ghz_client_lpi[WMI_REG_CLIENT_MAX];
+	u32 max_bw_6ghz_client_lpi[WMI_REG_CLIENT_MAX];
+	u32 min_bw_6ghz_client_vlp[WMI_REG_CLIENT_MAX];
+	u32 max_bw_6ghz_client_vlp[WMI_REG_CLIENT_MAX];
+	u32 num_6ghz_reg_rules_ap_sp;
+	u32 num_6ghz_reg_rules_ap_lpi;
+	u32 num_6ghz_reg_rules_ap_vlp;
+	u32 num_6ghz_reg_rules_client_sp[WMI_REG_CLIENT_MAX];
+	u32 num_6ghz_reg_rules_client_lpi[WMI_REG_CLIENT_MAX];
+	u32 num_6ghz_reg_rules_client_vlp[WMI_REG_CLIENT_MAX];
+} __packed;
+
+struct wmi_regulatory_ext_rule {
+	u32 tlv_header;
+	u32 freq_info;
+	u32 bw_pwr_info;
+	u32 flag_info;
+	u32 psd_power_info;
+} __packed;
+
 struct wmi_vdev_delete_resp_event {
 	u32 vdev_id;
 } __packed;
@@ -4542,6 +4838,8 @@ struct wmi_mgmt_tx_compl_event {
 	u32 desc_id;
 	u32 status;
 	u32 pdev_id;
+	u32 ppdu_id;
+	u32 ack_rssi;
 } __packed;
 
 struct wmi_scan_event {
@@ -5347,6 +5645,7 @@ struct target_resource_config {
 	u32 sched_params;
 	u32 twt_ap_pdev_count;
 	u32 twt_ap_sta_count;
+	u8 is_reg_cc_ext_event_supported;
 };
 
 enum wmi_debug_log_param {
diff --git a/drivers/net/wireless/ath/ath12k/ce.c b/drivers/net/wireless/ath/ath12k/ce.c
index aed6987804bf..be0d669d31fc 100644
--- a/drivers/net/wireless/ath/ath12k/ce.c
+++ b/drivers/net/wireless/ath/ath12k/ce.c
@@ -946,7 +946,7 @@ int ath12k_ce_alloc_pipes(struct ath12k_base *ab)
 
 		ret = ath12k_ce_alloc_pipe(ab, i);
 		if (ret) {
-			/* Free any parial successful allocation */
+			/* Free any partial successful allocation */
 			ath12k_ce_free_pipes(ab);
 			return ret;
 		}
diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h
index a54ae74543c1..9439052a652e 100644
--- a/drivers/net/wireless/ath/ath12k/core.h
+++ b/drivers/net/wireless/ath/ath12k/core.h
@@ -395,6 +395,7 @@ struct ath12k_sta {
 	u8 rssi_comb;
 	struct ath12k_rx_peer_stats *rx_stats;
 	struct ath12k_wbm_tx_stats *wbm_tx_stats;
+	u32 bw_prev;
 };
 
 #define ATH12K_MIN_5G_FREQ 4150
@@ -691,7 +692,7 @@ struct ath12k_base {
 
 	/* Below regd's are protected by ab->data_lock */
 	/* This is the regd set for every radio
-	 * by the firmware during initializatin
+	 * by the firmware during initialization
 	 */
 	struct ieee80211_regdomain *default_regd[MAX_RADIOS];
 	/* This regd is set during dynamic country setting
diff --git a/drivers/net/wireless/ath/ath12k/dp.c b/drivers/net/wireless/ath/ath12k/dp.c
index eb0261500ab0..ae1645d0f42a 100644
--- a/drivers/net/wireless/ath/ath12k/dp.c
+++ b/drivers/net/wireless/ath/ath12k/dp.c
@@ -1429,7 +1429,7 @@ static int ath12k_dp_cc_init(struct ath12k_base *ab)
 		}
 
 		if (dp->spt_info[i].paddr & ATH12K_SPT_4K_ALIGN_CHECK) {
-			ath12k_warn(ab, "SPT allocated memoty is not 4K aligned");
+			ath12k_warn(ab, "SPT allocated memory is not 4K aligned");
 			ret = -EINVAL;
 			goto free;
 		}
@@ -1461,15 +1461,12 @@ static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab)
 	dp->reoq_lut.vaddr = dma_alloc_coherent(ab->dev,
 						DP_REOQ_LUT_SIZE,
 						&dp->reoq_lut.paddr,
-						GFP_KERNEL);
-
+						GFP_KERNEL | __GFP_ZERO);
 	if (!dp->reoq_lut.vaddr) {
 		ath12k_warn(ab, "failed to allocate memory for reoq table");
 		return -ENOMEM;
 	}
 
-	memset(dp->reoq_lut.vaddr, 0, DP_REOQ_LUT_SIZE);
-
 	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab),
 			   dp->reoq_lut.paddr);
 	return 0;
diff --git a/drivers/net/wireless/ath/ath12k/dp.h b/drivers/net/wireless/ath/ath12k/dp.h
index 36a876d7f61d..7c5dafce5a68 100644
--- a/drivers/net/wireless/ath/ath12k/dp.h
+++ b/drivers/net/wireless/ath/ath12k/dp.h
@@ -371,7 +371,7 @@ struct ath12k_dp {
 
 #define HTT_TX_WBM_COMP_STATUS_OFFSET 8
 
-/* HTT tx completion is overlayed in wbm_release_ring */
+/* HTT tx completion is overlaid in wbm_release_ring */
 #define HTT_TX_WBM_COMP_INFO0_STATUS		GENMASK(16, 13)
 #define HTT_TX_WBM_COMP_INFO1_REINJECT_REASON	GENMASK(3, 0)
 #define HTT_TX_WBM_COMP_INFO1_EXCEPTION_FRAME	BIT(4)
@@ -545,7 +545,7 @@ enum htt_srng_ring_id {
  *                     3'b010: 4 usec
  *                     3'b011: 8 usec (default)
  *                     3'b100: 16 usec
- *                     Others: Reserverd
+ *                     Others: Reserved
  *           b'19    - response_required:
  *                     Host needs HTT_T2H_MSG_TYPE_SRING_SETUP_DONE as response
  *           b'20:31 - reserved:  reserved for future use
@@ -1126,7 +1126,7 @@ struct htt_tx_ring_selection_cfg_cmd {
 	__le32 tlv_filter_mask_in1;
 	__le32 tlv_filter_mask_in2;
 	__le32 tlv_filter_mask_in3;
-	__le32 reserverd[3];
+	__le32 reserved[3];
 } __packed;
 
 #define HTT_TX_RING_TLV_FILTER_MGMT_DMA_LEN	GENMASK(3, 0)
diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.c b/drivers/net/wireless/ath/ath12k/dp_mon.c
index a214797c96a2..f1e57e98bdc6 100644
--- a/drivers/net/wireless/ath/ath12k/dp_mon.c
+++ b/drivers/net/wireless/ath/ath12k/dp_mon.c
@@ -529,12 +529,12 @@ static void ath12k_dp_mon_parse_he_sig_su(u8 *tlv_data,
 	case 3:
 		if (he_dcm && he_stbc) {
 			he_gi = HE_GI_0_8;
-					he_ltf = HE_LTF_4_X;
+			he_ltf = HE_LTF_4_X;
 		} else {
 			he_gi = HE_GI_3_2;
 			he_ltf = HE_LTF_4_X;
-			}
-			break;
+		}
+		break;
 	}
 	ppdu_info->gi = he_gi;
 	value = he_gi << HE_GI_SHIFT;
@@ -813,7 +813,7 @@ ath12k_dp_mon_rx_parse_status_tlv(struct ath12k_base *ab,
 		spin_unlock_bh(&buf_ring->idr_lock);
 
 		if (unlikely(!msdu)) {
-			ath12k_warn(ab, "montior destination with invalid buf_id %d\n",
+			ath12k_warn(ab, "monitor destination with invalid buf_id %d\n",
 				    buf_id);
 			return HAL_RX_MON_STATUS_PPDU_NOT_DONE;
 		}
@@ -1124,7 +1124,7 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct
 
 	/* PN for multicast packets are not validate in HW,
 	 * so skip 802.3 rx path
-	 * Also, fast_rx expectes the STA to be authorized, hence
+	 * Also, fast_rx expects the STA to be authorized, hence
 	 * eapol packets are sent in slow path.
 	 */
 	if (decap == DP_RX_DECAP_TYPE_ETHERNET2_DIX && !is_eapol_tkip &&
@@ -1268,7 +1268,8 @@ int ath12k_dp_mon_buf_replenish(struct ath12k_base *ab,
 	struct sk_buff *skb;
 	struct hal_srng *srng;
 	dma_addr_t paddr;
-	u32 cookie, buf_id;
+	u32 cookie;
+	int buf_id;
 
 	srng = &ab->hal.srng_list[buf_ring->refill_buf_ring.ring_id];
 	spin_lock_bh(&srng->lock);
@@ -1917,7 +1918,7 @@ ath12k_dp_mon_tx_parse_status_tlv(struct ath12k_base *ab,
 		spin_unlock_bh(&buf_ring->idr_lock);
 
 		if (unlikely(!msdu)) {
-			ath12k_warn(ab, "montior destination with invalid buf_id %d\n",
+			ath12k_warn(ab, "monitor destination with invalid buf_id %d\n",
 				    buf_id);
 			return DP_MON_TX_STATUS_PPDU_NOT_DONE;
 		}
@@ -2110,7 +2111,7 @@ int ath12k_dp_mon_srng_process(struct ath12k *ar, int mac_id, int *budget,
 		spin_unlock_bh(&buf_ring->idr_lock);
 
 		if (unlikely(!skb)) {
-			ath12k_warn(ab, "montior destination with invalid buf_id %d\n",
+			ath12k_warn(ab, "monitor destination with invalid buf_id %d\n",
 				    buf_id);
 			goto move_next;
 		}
@@ -2511,7 +2512,7 @@ int ath12k_dp_mon_rx_process_stats(struct ath12k *ar, int mac_id,
 		spin_unlock_bh(&buf_ring->idr_lock);
 
 		if (unlikely(!skb)) {
-			ath12k_warn(ab, "montior destination with invalid buf_id %d\n",
+			ath12k_warn(ab, "monitor destination with invalid buf_id %d\n",
 				    buf_id);
 			goto move_next;
 		}
diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.c b/drivers/net/wireless/ath/ath12k/dp_rx.c
index 83a43ad48c51..e78478a5b978 100644
--- a/drivers/net/wireless/ath/ath12k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath12k/dp_rx.c
@@ -196,7 +196,8 @@ static void ath12k_dp_rxdesc_set_msdu_len(struct ath12k_base *ab,
 static bool ath12k_dp_rx_h_is_mcbc(struct ath12k_base *ab,
 				   struct hal_rx_desc *desc)
 {
-	return ab->hw_params->hal_ops->rx_desc_is_mcbc(desc);
+	return (ath12k_dp_rx_h_first_msdu(ab, desc) &&
+		ab->hw_params->hal_ops->rx_desc_is_mcbc(desc));
 }
 
 static bool ath12k_dp_rxdesc_mac_addr2_valid(struct ath12k_base *ab,
@@ -2443,7 +2444,7 @@ static void ath12k_dp_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *nap
 
 	/* PN for multicast packets are not validate in HW,
 	 * so skip 802.3 rx path
-	 * Also, fast_rx expectes the STA to be authorized, hence
+	 * Also, fast_rx expects the STA to be authorized, hence
 	 * eapol packets are sent in slow path.
 	 */
 	if (decap == DP_RX_DECAP_TYPE_ETHERNET2_DIX && !is_eapol &&
@@ -2611,7 +2612,7 @@ try_again:
 		if (!desc_info) {
 			desc_info = ath12k_dp_get_rx_desc(ab, cookie);
 			if (!desc_info) {
-				ath12k_warn(ab, "Invalid cookie in manual desc retrival");
+				ath12k_warn(ab, "Invalid cookie in manual desc retrieval");
 				continue;
 			}
 		}
@@ -3047,10 +3048,14 @@ static int ath12k_dp_rx_h_defrag_reo_reinject(struct ath12k *ar,
 	reo_ent_ring->rx_mpdu_info.peer_meta_data =
 		reo_dest_ring->rx_mpdu_info.peer_meta_data;
 
-	reo_ent_ring->queue_addr_lo = cpu_to_le32(lower_32_bits(rx_tid->paddr));
-	reo_ent_ring->info0 = le32_encode_bits(upper_32_bits(rx_tid->paddr),
-					       HAL_REO_ENTR_RING_INFO0_QUEUE_ADDR_HI) |
-		le32_encode_bits(dst_ind, HAL_REO_ENTR_RING_INFO0_DEST_IND);
+	/* Firmware expects physical address to be filled in queue_addr_lo in
+	 * the MLO scenario and in case of non MLO peer meta data needs to be
+	 * filled.
+	 * TODO: Need to handle for MLO scenario.
+	 */
+	reo_ent_ring->queue_addr_lo = reo_dest_ring->rx_mpdu_info.peer_meta_data;
+	reo_ent_ring->info0 = le32_encode_bits(dst_ind,
+					       HAL_REO_ENTR_RING_INFO0_DEST_IND);
 
 	reo_ent_ring->info1 = le32_encode_bits(rx_tid->cur_sn,
 					       HAL_REO_ENTR_RING_INFO1_MPDU_SEQ_NUM);
@@ -3297,7 +3302,7 @@ ath12k_dp_process_rx_err_buf(struct ath12k *ar, struct hal_reo_dest_ring *desc,
 	if (!desc_info) {
 		desc_info = ath12k_dp_get_rx_desc(ab, cookie);
 		if (!desc_info) {
-			ath12k_warn(ab, "Invalid cookie in manual desc retrival");
+			ath12k_warn(ab, "Invalid cookie in manual desc retrieval");
 			return -EINVAL;
 		}
 	}
@@ -3494,11 +3499,14 @@ static int ath12k_dp_rx_h_null_q_desc(struct ath12k *ar, struct sk_buff *msdu,
 	msdu_len = ath12k_dp_rx_h_msdu_len(ab, desc);
 	peer_id = ath12k_dp_rx_h_peer_id(ab, desc);
 
+	spin_lock(&ab->base_lock);
 	if (!ath12k_peer_find_by_id(ab, peer_id)) {
+		spin_unlock(&ab->base_lock);
 		ath12k_dbg(ab, ATH12K_DBG_DATA, "invalid peer id received in wbm err pkt%d\n",
 			   peer_id);
 		return -EINVAL;
 	}
+	spin_unlock(&ab->base_lock);
 
 	if (!rxcb->is_frag && ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE)) {
 		/* First buffer will be freed by the caller, so deduct it's length */
@@ -3718,7 +3726,7 @@ int ath12k_dp_rx_process_wbm_err(struct ath12k_base *ab,
 		if (!desc_info) {
 			desc_info = ath12k_dp_get_rx_desc(ab, err_info.cookie);
 			if (!desc_info) {
-				ath12k_warn(ab, "Invalid cookie in manual desc retrival");
+				ath12k_warn(ab, "Invalid cookie in manual desc retrieval");
 				continue;
 			}
 		}
diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.c b/drivers/net/wireless/ath/ath12k/dp_tx.c
index 95294f35155c..d3c7c76d6b75 100644
--- a/drivers/net/wireless/ath/ath12k/dp_tx.c
+++ b/drivers/net/wireless/ath/ath12k/dp_tx.c
@@ -13,6 +13,10 @@ static enum hal_tcl_encap_type
 ath12k_dp_tx_get_encap_type(struct ath12k_vif *arvif, struct sk_buff *skb)
 {
 	struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb);
+	struct ath12k_base *ab = arvif->ar->ab;
+
+	if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags))
+		return HAL_TCL_ENCAP_TYPE_RAW;
 
 	if (tx_info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP)
 		return HAL_TCL_ENCAP_TYPE_ETHERNET;
@@ -270,7 +274,7 @@ tcl_ring_sel:
 					  skb_ext_desc->len, DMA_TO_DEVICE);
 		ret = dma_mapping_error(ab->dev, ti.paddr);
 		if (ret) {
-			kfree(skb_ext_desc);
+			kfree_skb(skb_ext_desc);
 			goto fail_unmap_dma;
 		}
 
diff --git a/drivers/net/wireless/ath/ath12k/hal.c b/drivers/net/wireless/ath/ath12k/hal.c
index 95d04819083f..0ec53afe9915 100644
--- a/drivers/net/wireless/ath/ath12k/hal.c
+++ b/drivers/net/wireless/ath/ath12k/hal.c
@@ -609,7 +609,7 @@ static int ath12k_hal_srng_create_config_qcn9274(struct ath12k_base *ab)
 		HAL_WBM0_RELEASE_RING_BASE_LSB(ab);
 	s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP;
 
-	/* Some LMAC rings are not accesed from the host:
+	/* Some LMAC rings are not accessed from the host:
 	 * RXDMA_BUG, RXDMA_DST, RXDMA_MONITOR_BUF, RXDMA_MONITOR_STATUS,
 	 * RXDMA_MONITOR_DST, RXDMA_MONITOR_DESC, RXDMA_DIR_BUF_SRC,
 	 * RXDMA_RX_MONITOR_BUF, TX_MONITOR_BUF, TX_MONITOR_DST, SW2RXDMA
diff --git a/drivers/net/wireless/ath/ath12k/hal.h b/drivers/net/wireless/ath/ath12k/hal.h
index dfbd8bce70e5..0d4fa12ea622 100644
--- a/drivers/net/wireless/ath/ath12k/hal.h
+++ b/drivers/net/wireless/ath/ath12k/hal.h
@@ -270,7 +270,7 @@ struct ath12k_base;
 #define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN		BIT(5)
 #define HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN		BIT(8)
 
-/* TCL ring feild mask and offset */
+/* TCL ring field mask and offset */
 #define HAL_TCL1_RING_BASE_MSB_RING_SIZE		GENMASK(27, 8)
 #define HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB	GENMASK(7, 0)
 #define HAL_TCL1_RING_ID_ENTRY_SIZE			GENMASK(7, 0)
@@ -296,7 +296,7 @@ struct ath12k_base;
 #define HAL_TCL1_RING_FIELD_DSCP_TID_MAP6		GENMASK(20, 18)
 #define HAL_TCL1_RING_FIELD_DSCP_TID_MAP7		GENMASK(23, 21)
 
-/* REO ring feild mask and offset */
+/* REO ring field mask and offset */
 #define HAL_REO1_RING_BASE_MSB_RING_SIZE		GENMASK(27, 8)
 #define HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB	GENMASK(7, 0)
 #define HAL_REO1_RING_ID_RING_ID			GENMASK(15, 8)
@@ -738,7 +738,7 @@ struct hal_srng {
 	} u;
 };
 
-/* Interrupt mitigation - Batch threshold in terms of numer of frames */
+/* Interrupt mitigation - Batch threshold in terms of number of frames */
 #define HAL_SRNG_INT_BATCH_THRESHOLD_TX 256
 #define HAL_SRNG_INT_BATCH_THRESHOLD_RX 128
 #define HAL_SRNG_INT_BATCH_THRESHOLD_OTHER 1
@@ -813,7 +813,7 @@ enum hal_rx_buf_return_buf_manager {
 #define HAL_REO_CMD_FLG_UNBLK_RESOURCE		BIT(7)
 #define HAL_REO_CMD_FLG_UNBLK_CACHE		BIT(8)
 
-/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO0_UPD_* feilds */
+/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO0_UPD_* fields */
 #define HAL_REO_CMD_UPD0_RX_QUEUE_NUM		BIT(8)
 #define HAL_REO_CMD_UPD0_VLD			BIT(9)
 #define HAL_REO_CMD_UPD0_ALDC			BIT(10)
@@ -838,7 +838,7 @@ enum hal_rx_buf_return_buf_manager {
 #define HAL_REO_CMD_UPD0_PN_VALID		BIT(29)
 #define HAL_REO_CMD_UPD0_PN			BIT(30)
 
-/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO1_* feilds */
+/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO1_* fields */
 #define HAL_REO_CMD_UPD1_VLD			BIT(16)
 #define HAL_REO_CMD_UPD1_ALDC			GENMASK(18, 17)
 #define HAL_REO_CMD_UPD1_DIS_DUP_DETECTION	BIT(19)
@@ -854,7 +854,7 @@ enum hal_rx_buf_return_buf_manager {
 #define HAL_REO_CMD_UPD1_PN_HANDLE_ENABLE	BIT(30)
 #define HAL_REO_CMD_UPD1_IGNORE_AMPDU_FLG	BIT(31)
 
-/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO2_* feilds */
+/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO2_* fields */
 #define HAL_REO_CMD_UPD2_SVLD			BIT(10)
 #define HAL_REO_CMD_UPD2_SSN			GENMASK(22, 11)
 #define HAL_REO_CMD_UPD2_SEQ_2K_ERR		BIT(23)
diff --git a/drivers/net/wireless/ath/ath12k/hal_desc.h b/drivers/net/wireless/ath/ath12k/hal_desc.h
index 2250ca2d19a3..6c17adc6d60b 100644
--- a/drivers/net/wireless/ath/ath12k/hal_desc.h
+++ b/drivers/net/wireless/ath/ath12k/hal_desc.h
@@ -706,7 +706,7 @@ struct rx_msdu_desc {
  *
  * msdu_continuation
  *		When set, this MSDU buffer was not able to hold the entire MSDU.
- *		The next buffer will therefor contain additional information
+ *		The next buffer will therefore contain additional information
  *		related to this MSDU.
  *
  * msdu_length
@@ -1294,7 +1294,7 @@ struct hal_tcl_data_cmd {
  *		link descriptor.
  *
  * tcl_cmd_type
- *		used to select the type of TCL Command decriptor
+ *		used to select the type of TCL Command descriptor
  *
  * desc_type
  *		Indicates the type of address provided in the buf_addr_info.
@@ -1408,7 +1408,7 @@ struct hal_tcl_data_cmd {
  * index_loop_override
  *		When set, address search and packet routing is forced to use
  *		'search_index' instead of following the register configuration
- *		seleced by Bank_id.
+ *		selected by Bank_id.
  *
  * ring_id
  *		The buffer pointer ring ID.
@@ -1990,7 +1990,7 @@ struct hal_wbm_release_ring {
  *	Producer: SW/TQM/RXDMA/REO/SWITCH
  *	Consumer: WBM/SW/FW
  *
- * HTT tx status is overlayed on wbm_release ring on 4-byte words 2, 3, 4 and 5
+ * HTT tx status is overlaid on wbm_release ring on 4-byte words 2, 3, 4 and 5
  * for software based completions.
  *
  * buf_addr_info
@@ -2552,7 +2552,7 @@ struct hal_reo_status_hdr {
  *		commands.
  *
  * execution_time (in us)
- *		The amount of time REO took to excecute the command. Note that
+ *		The amount of time REO took to execute the command. Note that
  *		this time does not include the duration of the command waiting
  *		in the command ring, before the execution started.
  *
diff --git a/drivers/net/wireless/ath/ath12k/hw.c b/drivers/net/wireless/ath/ath12k/hw.c
index 91d576fd4b0f..1ffac7e3deaa 100644
--- a/drivers/net/wireless/ath/ath12k/hw.c
+++ b/drivers/net/wireless/ath/ath12k/hw.c
@@ -944,7 +944,7 @@ static const struct ath12k_hw_params ath12k_hw_params[] = {
 		.interface_modes = BIT(NL80211_IFTYPE_STATION),
 		.supports_monitor = false,
 
-		.idle_ps = false,
+		.idle_ps = true,
 		.download_calib = false,
 		.supports_suspend = false,
 		.tcl_ring_retry = false,
diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c
index bf7e5b6977b2..ee792822b411 100644
--- a/drivers/net/wireless/ath/ath12k/mac.c
+++ b/drivers/net/wireless/ath/ath12k/mac.c
@@ -3220,10 +3220,11 @@ static void ath12k_sta_rc_update_wk(struct work_struct *wk)
 	enum nl80211_band band;
 	const u8 *ht_mcs_mask;
 	const u16 *vht_mcs_mask;
-	u32 changed, bw, nss, smps;
+	u32 changed, bw, nss, smps, bw_prev;
 	int err, num_vht_rates;
 	const struct cfg80211_bitrate_mask *mask;
 	struct ath12k_wmi_peer_assoc_arg peer_arg;
+	enum wmi_phy_mode peer_phymode;
 
 	arsta = container_of(wk, struct ath12k_sta, update_wk);
 	sta = container_of((void *)arsta, struct ieee80211_sta, drv_priv);
@@ -3243,6 +3244,7 @@ static void ath12k_sta_rc_update_wk(struct work_struct *wk)
 	arsta->changed = 0;
 
 	bw = arsta->bw;
+	bw_prev = arsta->bw_prev;
 	nss = arsta->nss;
 	smps = arsta->smps;
 
@@ -3255,11 +3257,53 @@ static void ath12k_sta_rc_update_wk(struct work_struct *wk)
 			   ath12k_mac_max_vht_nss(vht_mcs_mask)));
 
 	if (changed & IEEE80211_RC_BW_CHANGED) {
-		err = ath12k_wmi_set_peer_param(ar, sta->addr, arvif->vdev_id,
-						WMI_PEER_CHWIDTH, bw);
-		if (err)
-			ath12k_warn(ar->ab, "failed to update STA %pM peer bw %d: %d\n",
-				    sta->addr, bw, err);
+		ath12k_peer_assoc_h_phymode(ar, arvif->vif, sta, &peer_arg);
+		peer_phymode = peer_arg.peer_phymode;
+
+		if (bw > bw_prev) {
+			/* Phymode shows maximum supported channel width, if we
+			 * upgrade bandwidth then due to sanity check of firmware,
+			 * we have to send WMI_PEER_PHYMODE followed by
+			 * WMI_PEER_CHWIDTH
+			 */
+			ath12k_dbg(ar->ab, ATH12K_DBG_MAC, "mac bandwidth upgrade for sta %pM new %d old %d\n",
+				   sta->addr, bw, bw_prev);
+			err = ath12k_wmi_set_peer_param(ar, sta->addr,
+							arvif->vdev_id, WMI_PEER_PHYMODE,
+							peer_phymode);
+			if (err) {
+				ath12k_warn(ar->ab, "failed to update STA %pM to peer phymode %d: %d\n",
+					    sta->addr, peer_phymode, err);
+				goto err_rc_bw_changed;
+			}
+			err = ath12k_wmi_set_peer_param(ar, sta->addr,
+							arvif->vdev_id, WMI_PEER_CHWIDTH,
+							bw);
+			if (err)
+				ath12k_warn(ar->ab, "failed to update STA %pM to peer bandwidth %d: %d\n",
+					    sta->addr, bw, err);
+		} else {
+			/* When we downgrade bandwidth this will conflict with phymode
+			 * and cause to trigger firmware crash. In this case we send
+			 * WMI_PEER_CHWIDTH followed by WMI_PEER_PHYMODE
+			 */
+			ath12k_dbg(ar->ab, ATH12K_DBG_MAC, "mac bandwidth downgrade for sta %pM new %d old %d\n",
+				   sta->addr, bw, bw_prev);
+			err = ath12k_wmi_set_peer_param(ar, sta->addr,
+							arvif->vdev_id, WMI_PEER_CHWIDTH,
+							bw);
+			if (err) {
+				ath12k_warn(ar->ab, "failed to update STA %pM peer to bandwidth %d: %d\n",
+					    sta->addr, bw, err);
+				goto err_rc_bw_changed;
+			}
+			err = ath12k_wmi_set_peer_param(ar, sta->addr,
+							arvif->vdev_id, WMI_PEER_PHYMODE,
+							peer_phymode);
+			if (err)
+				ath12k_warn(ar->ab, "failed to update STA %pM to peer phymode %d: %d\n",
+					    sta->addr, peer_phymode, err);
+		}
 	}
 
 	if (changed & IEEE80211_RC_NSS_CHANGED) {
@@ -3321,7 +3365,7 @@ static void ath12k_sta_rc_update_wk(struct work_struct *wk)
 					    sta->addr, arvif->vdev_id);
 		}
 	}
-
+err_rc_bw_changed:
 	mutex_unlock(&ar->conf_mutex);
 }
 
@@ -3433,6 +3477,34 @@ exit:
 	return ret;
 }
 
+static u32 ath12k_mac_ieee80211_sta_bw_to_wmi(struct ath12k *ar,
+					      struct ieee80211_sta *sta)
+{
+	u32 bw = WMI_PEER_CHWIDTH_20MHZ;
+
+	switch (sta->deflink.bandwidth) {
+	case IEEE80211_STA_RX_BW_20:
+		bw = WMI_PEER_CHWIDTH_20MHZ;
+		break;
+	case IEEE80211_STA_RX_BW_40:
+		bw = WMI_PEER_CHWIDTH_40MHZ;
+		break;
+	case IEEE80211_STA_RX_BW_80:
+		bw = WMI_PEER_CHWIDTH_80MHZ;
+		break;
+	case IEEE80211_STA_RX_BW_160:
+		bw = WMI_PEER_CHWIDTH_160MHZ;
+		break;
+	default:
+		ath12k_warn(ar->ab, "Invalid bandwidth %d in rc update for %pM\n",
+			    sta->deflink.bandwidth, sta->addr);
+		bw = WMI_PEER_CHWIDTH_20MHZ;
+		break;
+	}
+
+	return bw;
+}
+
 static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
 				   struct ieee80211_vif *vif,
 				   struct ieee80211_sta *sta,
@@ -3498,6 +3570,13 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
 		if (ret)
 			ath12k_warn(ar->ab, "Failed to associate station: %pM\n",
 				    sta->addr);
+
+		spin_lock_bh(&ar->data_lock);
+
+		arsta->bw = ath12k_mac_ieee80211_sta_bw_to_wmi(ar, sta);
+		arsta->bw_prev = sta->deflink.bandwidth;
+
+		spin_unlock_bh(&ar->data_lock);
 	} else if (old_state == IEEE80211_STA_ASSOC &&
 		   new_state == IEEE80211_STA_AUTHORIZED) {
 		spin_lock_bh(&ar->ab->base_lock);
@@ -3607,28 +3686,8 @@ static void ath12k_mac_op_sta_rc_update(struct ieee80211_hw *hw,
 	spin_lock_bh(&ar->data_lock);
 
 	if (changed & IEEE80211_RC_BW_CHANGED) {
-		bw = WMI_PEER_CHWIDTH_20MHZ;
-
-		switch (sta->deflink.bandwidth) {
-		case IEEE80211_STA_RX_BW_20:
-			bw = WMI_PEER_CHWIDTH_20MHZ;
-			break;
-		case IEEE80211_STA_RX_BW_40:
-			bw = WMI_PEER_CHWIDTH_40MHZ;
-			break;
-		case IEEE80211_STA_RX_BW_80:
-			bw = WMI_PEER_CHWIDTH_80MHZ;
-			break;
-		case IEEE80211_STA_RX_BW_160:
-			bw = WMI_PEER_CHWIDTH_160MHZ;
-			break;
-		default:
-			ath12k_warn(ar->ab, "Invalid bandwidth %d in rc update for %pM\n",
-				    sta->deflink.bandwidth, sta->addr);
-			bw = WMI_PEER_CHWIDTH_20MHZ;
-			break;
-		}
-
+		bw = ath12k_mac_ieee80211_sta_bw_to_wmi(ar, sta);
+		arsta->bw_prev = arsta->bw;
 		arsta->bw = bw;
 	}
 
diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c
index ae7f6083c9fc..9f174daf324c 100644
--- a/drivers/net/wireless/ath/ath12k/pci.c
+++ b/drivers/net/wireless/ath/ath12k/pci.c
@@ -119,6 +119,30 @@ static const char *irq_name[ATH12K_IRQ_NUM_MAX] = {
 	"tcl2host-status-ring",
 };
 
+static int ath12k_pci_bus_wake_up(struct ath12k_base *ab)
+{
+	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);
+
+	return mhi_device_get_sync(ab_pci->mhi_ctrl->mhi_dev);
+}
+
+static void ath12k_pci_bus_release(struct ath12k_base *ab)
+{
+	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);
+
+	mhi_device_put(ab_pci->mhi_ctrl->mhi_dev);
+}
+
+static const struct ath12k_pci_ops ath12k_pci_ops_qcn9274 = {
+	.wakeup = NULL,
+	.release = NULL,
+};
+
+static const struct ath12k_pci_ops ath12k_pci_ops_wcn7850 = {
+	.wakeup = ath12k_pci_bus_wake_up,
+	.release = ath12k_pci_bus_release,
+};
+
 static void ath12k_pci_select_window(struct ath12k_pci *ab_pci, u32 offset)
 {
 	struct ath12k_base *ab = ab_pci->ab;
@@ -731,14 +755,12 @@ static int ath12k_pci_claim(struct ath12k_pci *ab_pci, struct pci_dev *pdev)
 	if (!ab->mem) {
 		ath12k_err(ab, "failed to map pci bar %d\n", ATH12K_PCI_BAR_NUM);
 		ret = -EIO;
-		goto clear_master;
+		goto release_region;
 	}
 
 	ath12k_dbg(ab, ATH12K_DBG_BOOT, "boot pci_mem 0x%pK\n", ab->mem);
 	return 0;
 
-clear_master:
-	pci_clear_master(pdev);
 release_region:
 	pci_release_region(pdev, ATH12K_PCI_BAR_NUM);
 disable_device:
@@ -754,7 +776,6 @@ static void ath12k_pci_free_region(struct ath12k_pci *ab_pci)
 
 	pci_iounmap(pci_dev, ab->mem);
 	ab->mem = NULL;
-	pci_clear_master(pci_dev);
 	pci_release_region(pci_dev, ATH12K_PCI_BAR_NUM);
 	if (pci_is_enabled(pci_dev))
 		pci_disable_device(pci_dev);
@@ -989,13 +1010,14 @@ u32 ath12k_pci_read32(struct ath12k_base *ab, u32 offset)
 {
 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);
 	u32 val, window_start;
+	int ret = 0;
 
 	/* for offset beyond BAR + 4K - 32, may
 	 * need to wakeup MHI to access.
 	 */
 	if (test_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags) &&
-	    offset >= ACCESS_ALWAYS_OFF)
-		mhi_device_get_sync(ab_pci->mhi_ctrl->mhi_dev);
+	    offset >= ACCESS_ALWAYS_OFF && ab_pci->pci_ops->wakeup)
+		ret = ab_pci->pci_ops->wakeup(ab);
 
 	if (offset < WINDOW_START) {
 		val = ioread32(ab->mem + offset);
@@ -1023,9 +1045,9 @@ u32 ath12k_pci_read32(struct ath12k_base *ab, u32 offset)
 	}
 
 	if (test_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags) &&
-	    offset >= ACCESS_ALWAYS_OFF)
-		mhi_device_put(ab_pci->mhi_ctrl->mhi_dev);
-
+	    offset >= ACCESS_ALWAYS_OFF && ab_pci->pci_ops->release &&
+	    !ret)
+		ab_pci->pci_ops->release(ab);
 	return val;
 }
 
@@ -1033,13 +1055,14 @@ void ath12k_pci_write32(struct ath12k_base *ab, u32 offset, u32 value)
 {
 	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);
 	u32 window_start;
+	int ret = 0;
 
 	/* for offset beyond BAR + 4K - 32, may
 	 * need to wakeup MHI to access.
 	 */
 	if (test_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags) &&
-	    offset >= ACCESS_ALWAYS_OFF)
-		mhi_device_get_sync(ab_pci->mhi_ctrl->mhi_dev);
+	    offset >= ACCESS_ALWAYS_OFF && ab_pci->pci_ops->wakeup)
+		ret = ab_pci->pci_ops->wakeup(ab);
 
 	if (offset < WINDOW_START) {
 		iowrite32(value, ab->mem + offset);
@@ -1067,8 +1090,9 @@ void ath12k_pci_write32(struct ath12k_base *ab, u32 offset, u32 value)
 	}
 
 	if (test_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags) &&
-	    offset >= ACCESS_ALWAYS_OFF)
-		mhi_device_put(ab_pci->mhi_ctrl->mhi_dev);
+	    offset >= ACCESS_ALWAYS_OFF && ab_pci->pci_ops->release &&
+	    !ret)
+		ab_pci->pci_ops->release(ab);
 }
 
 int ath12k_pci_power_up(struct ath12k_base *ab)
@@ -1182,6 +1206,7 @@ static int ath12k_pci_probe(struct pci_dev *pdev,
 	case QCN9274_DEVICE_ID:
 		ab_pci->msi_config = &ath12k_msi_config[0];
 		ab->static_window_map = true;
+		ab_pci->pci_ops = &ath12k_pci_ops_qcn9274;
 		ath12k_pci_read_hw_version(ab, &soc_hw_version_major,
 					   &soc_hw_version_minor);
 		switch (soc_hw_version_major) {
@@ -1195,13 +1220,15 @@ static int ath12k_pci_probe(struct pci_dev *pdev,
 			dev_err(&pdev->dev,
 				"Unknown hardware version found for QCN9274: 0x%x\n",
 				soc_hw_version_major);
-			return -EOPNOTSUPP;
+			ret = -EOPNOTSUPP;
+			goto err_pci_free_region;
 		}
 		break;
 	case WCN7850_DEVICE_ID:
 		ab_pci->msi_config = &ath12k_msi_config[0];
 		ab->static_window_map = false;
 		ab->hw_rev = ATH12K_HW_WCN7850_HW20;
+		ab_pci->pci_ops = &ath12k_pci_ops_wcn7850;
 		break;
 
 	default:
diff --git a/drivers/net/wireless/ath/ath12k/pci.h b/drivers/net/wireless/ath/ath12k/pci.h
index 0d9e40ab31f2..0f24fd9395cd 100644
--- a/drivers/net/wireless/ath/ath12k/pci.h
+++ b/drivers/net/wireless/ath/ath12k/pci.h
@@ -86,6 +86,11 @@ enum ath12k_pci_flags {
 	ATH12K_PCI_ASPM_RESTORE,
 };
 
+struct ath12k_pci_ops {
+	int (*wakeup)(struct ath12k_base *ab);
+	void (*release)(struct ath12k_base *ab);
+};
+
 struct ath12k_pci {
 	struct pci_dev *pdev;
 	struct ath12k_base *ab;
@@ -103,6 +108,7 @@ struct ath12k_pci {
 	/* enum ath12k_pci_flags */
 	unsigned long flags;
 	u16 link_ctl;
+	const struct ath12k_pci_ops *pci_ops;
 };
 
 static inline struct ath12k_pci *ath12k_pci_priv(struct ath12k_base *ab)
diff --git a/drivers/net/wireless/ath/ath12k/qmi.c b/drivers/net/wireless/ath/ath12k/qmi.c
index 979a63f2e2ab..03ba245fbee9 100644
--- a/drivers/net/wireless/ath/ath12k/qmi.c
+++ b/drivers/net/wireless/ath/ath12k/qmi.c
@@ -2991,7 +2991,7 @@ static void ath12k_qmi_driver_event_work(struct work_struct *work)
 		spin_unlock(&qmi->event_lock);
 
 		if (test_bit(ATH12K_FLAG_UNREGISTERING, &ab->dev_flags))
-			return;
+			goto skip;
 
 		switch (event->type) {
 		case ATH12K_QMI_EVENT_SERVER_ARRIVE:
@@ -3032,6 +3032,8 @@ static void ath12k_qmi_driver_event_work(struct work_struct *work)
 			ath12k_warn(ab, "invalid event type: %d", event->type);
 			break;
 		}
+
+skip:
 		kfree(event);
 		spin_lock(&qmi->event_lock);
 	}
diff --git a/drivers/net/wireless/ath/ath12k/rx_desc.h b/drivers/net/wireless/ath/ath12k/rx_desc.h
index 5feaff6450ad..f99556a253e5 100644
--- a/drivers/net/wireless/ath/ath12k/rx_desc.h
+++ b/drivers/net/wireless/ath/ath12k/rx_desc.h
@@ -1072,7 +1072,7 @@ struct rx_msdu_end_qcn9274 {
  *
  * l4_offset
  *		Depending upon mode bit, this field either indicates the
- *		L4 offset nin bytes from the start of RX_HEADER (only valid
+ *		L4 offset in bytes from the start of RX_HEADER (only valid
  *		if either ipv4_proto or ipv6_proto is set to 1) or indicates
  *		the offset in bytes to the start of TCP or UDP header from
  *		the start of the IP header after decapsulation (Only valid if
diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c
index f6df14149531..7ae0bb78b2b5 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.c
+++ b/drivers/net/wireless/ath/ath12k/wmi.c
@@ -494,7 +494,7 @@ ath12k_pull_mac_phy_cap_svc_ready_ext(struct ath12k_wmi_pdev *wmi_handle,
 
 	/* tx/rx chainmask reported from fw depends on the actual hw chains used,
 	 * For example, for 4x4 capable macphys, first 4 chains can be used for first
-	 * mac and the remaing 4 chains can be used for the second mac or vice-versa.
+	 * mac and the remaining 4 chains can be used for the second mac or vice-versa.
 	 * In this case, tx/rx chainmask 0xf will be advertised for first mac and 0xf0
 	 * will be advertised for second mac or vice-versa. Compute the shift value
 	 * for tx/rx chainmask which will be used to advertise supported ht/vht rates to
@@ -1743,7 +1743,7 @@ int ath12k_wmi_vdev_install_key(struct ath12k *ar,
 	int ret, len, key_len_aligned;
 
 	/* WMI_TAG_ARRAY_BYTE needs to be aligned with 4, the actual key
-	 * length is specifed in cmd->key_len.
+	 * length is specified in cmd->key_len.
 	 */
 	key_len_aligned = roundup(arg->key_len, 4);
 
@@ -2438,6 +2438,9 @@ int ath12k_wmi_send_scan_chan_list_cmd(struct ath12k *ar,
 			if (channel_arg->psc_channel)
 				chan_info->info |= cpu_to_le32(WMI_CHAN_INFO_PSC);
 
+			if (channel_arg->dfs_set)
+				chan_info->info |= cpu_to_le32(WMI_CHAN_INFO_DFS);
+
 			chan_info->info |= le32_encode_bits(channel_arg->phy_mode,
 							    WMI_CHAN_INFO_MODE);
 			*reg1 |= le32_encode_bits(channel_arg->minpower,
@@ -4934,6 +4937,9 @@ static int freq_to_idx(struct ath12k *ar, int freq)
 	int band, ch, idx = 0;
 
 	for (band = NL80211_BAND_2GHZ; band < NUM_NL80211_BANDS; band++) {
+		if (!ar->mac.sbands[band].channels)
+			continue;
+
 		sband = ar->hw->wiphy->bands[band];
 		if (!sband)
 			continue;
@@ -5995,7 +6001,7 @@ static void ath12k_service_available_event(struct ath12k_base *ab, struct sk_buf
 	}
 
 	/* TODO: Use wmi_service_segment_offset information to get the service
-	 * especially when more services are advertised in multiple sevice
+	 * especially when more services are advertised in multiple service
 	 * available events.
 	 */
 	for (i = 0, j = WMI_MAX_SERVICE;
diff --git a/drivers/net/wireless/ath/ath12k/wmi.h b/drivers/net/wireless/ath/ath12k/wmi.h
index 84e3fb918e43..08a8c9e0f59f 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.h
+++ b/drivers/net/wireless/ath/ath12k/wmi.h
@@ -4002,7 +4002,7 @@ struct ath12k_wmi_pdev_radar_event {
 } __packed;
 
 struct wmi_pdev_temperature_event {
-	/* temperature value in Celcius degree */
+	/* temperature value in Celsius degree */
 	a_sle32 temp;
 	__le32 pdev_id;
 } __packed;
@@ -4192,7 +4192,7 @@ enum wmi_sta_ps_param_tx_wake_threshold {
  */
 enum wmi_sta_ps_param_pspoll_count {
 	WMI_STA_PS_PSPOLL_COUNT_NO_MAX = 0,
-	/* Values greater than 0 indicate the maximum numer of PS-Poll frames
+	/* Values greater than 0 indicate the maximum number of PS-Poll frames
 	 * FW will send before waking up.
 	 */
 };
diff --git a/drivers/net/wireless/ath/ath5k/ahb.c b/drivers/net/wireless/ath/ath5k/ahb.c
index 2c9cec8b53d9..28a1e5eff204 100644
--- a/drivers/net/wireless/ath/ath5k/ahb.c
+++ b/drivers/net/wireless/ath/ath5k/ahb.c
@@ -113,15 +113,13 @@ static int ath_ahb_probe(struct platform_device *pdev)
 		goto err_out;
 	}
 
-	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (res == NULL) {
-		dev_err(&pdev->dev, "no IRQ resource found\n");
-		ret = -ENXIO;
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "no IRQ resource found: %d\n", irq);
+		ret = irq;
 		goto err_iounmap;
 	}
 
-	irq = res->start;
-
 	hw = ieee80211_alloc_hw(sizeof(struct ath5k_hw), &ath5k_hw_ops);
 	if (hw == NULL) {
 		dev_err(&pdev->dev, "no memory for ieee80211_hw\n");
diff --git a/drivers/net/wireless/ath/ath5k/eeprom.c b/drivers/net/wireless/ath/ath5k/eeprom.c
index d444b3d70ba2..58d3e86f6256 100644
--- a/drivers/net/wireless/ath/ath5k/eeprom.c
+++ b/drivers/net/wireless/ath/ath5k/eeprom.c
@@ -529,7 +529,7 @@ ath5k_eeprom_read_freq_list(struct ath5k_hw *ah, int *offset, int max,
 		ee->ee_n_piers[mode]++;
 
 		freq2 = (val >> 8) & 0xff;
-		if (!freq2)
+		if (!freq2 || i >= max)
 			break;
 
 		pc[i++].freq = ath5k_eeprom_bin2freq(ee,
diff --git a/drivers/net/wireless/ath/ath6kl/bmi.c b/drivers/net/wireless/ath/ath6kl/bmi.c
index bde5a10d470c..af98e871199d 100644
--- a/drivers/net/wireless/ath/ath6kl/bmi.c
+++ b/drivers/net/wireless/ath/ath6kl/bmi.c
@@ -246,7 +246,7 @@ int ath6kl_bmi_execute(struct ath6kl *ar, u32 addr, u32 *param)
 		return -EACCES;
 	}
 
-	size = sizeof(cid) + sizeof(addr) + sizeof(param);
+	size = sizeof(cid) + sizeof(addr) + sizeof(*param);
 	if (size > ar->bmi.max_cmd_size) {
 		WARN_ON(1);
 		return -EINVAL;
diff --git a/drivers/net/wireless/ath/ath6kl/htc_pipe.c b/drivers/net/wireless/ath/ath6kl/htc_pipe.c
index c68848819a52..9b88d96bfe96 100644
--- a/drivers/net/wireless/ath/ath6kl/htc_pipe.c
+++ b/drivers/net/wireless/ath/ath6kl/htc_pipe.c
@@ -960,8 +960,8 @@ static int ath6kl_htc_pipe_rx_complete(struct ath6kl *ar, struct sk_buff *skb,
 	 * Thus the possibility of ar->htc_target being NULL
 	 * via ath6kl_recv_complete -> ath6kl_usb_io_comp_work.
 	 */
-	if (WARN_ON_ONCE(!target)) {
-		ath6kl_err("Target not yet initialized\n");
+	if (!target) {
+		ath6kl_dbg(ATH6KL_DBG_HTC, "Target not yet initialized\n");
 		status = -EINVAL;
 		goto free_skb;
 	}
diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c
index f521dfa2f194..27ff1ca2631f 100644
--- a/drivers/net/wireless/ath/ath9k/hif_usb.c
+++ b/drivers/net/wireless/ath/ath9k/hif_usb.c
@@ -42,8 +42,6 @@ static const struct usb_device_id ath9k_hif_usb_ids[] = {
 
 	{ USB_DEVICE(0x0cf3, 0x7015),
 	  .driver_info = AR9287_USB },  /* Atheros */
-	{ USB_DEVICE(0x1668, 0x1200),
-	  .driver_info = AR9287_USB },  /* Verizon */
 
 	{ USB_DEVICE(0x0cf3, 0x7010),
 	  .driver_info = AR9280_USB },  /* Atheros */
@@ -534,6 +532,24 @@ static struct ath9k_htc_hif hif_usb = {
 	.send = hif_usb_send,
 };
 
+/* Need to free remain_skb allocated in ath9k_hif_usb_rx_stream
+ * in case ath9k_hif_usb_rx_stream wasn't called next time to
+ * process the buffer and subsequently free it.
+ */
+static void ath9k_hif_usb_free_rx_remain_skb(struct hif_device_usb *hif_dev)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&hif_dev->rx_lock, flags);
+	if (hif_dev->remain_skb) {
+		dev_kfree_skb_any(hif_dev->remain_skb);
+		hif_dev->remain_skb = NULL;
+		hif_dev->rx_remain_len = 0;
+		RX_STAT_INC(hif_dev, skb_dropped);
+	}
+	spin_unlock_irqrestore(&hif_dev->rx_lock, flags);
+}
+
 static void ath9k_hif_usb_rx_stream(struct hif_device_usb *hif_dev,
 				    struct sk_buff *skb)
 {
@@ -868,6 +884,7 @@ err:
 static void ath9k_hif_usb_dealloc_rx_urbs(struct hif_device_usb *hif_dev)
 {
 	usb_kill_anchored_urbs(&hif_dev->rx_submitted);
+	ath9k_hif_usb_free_rx_remain_skb(hif_dev);
 }
 
 static int ath9k_hif_usb_alloc_rx_urbs(struct hif_device_usb *hif_dev)
diff --git a/drivers/net/wireless/ath/ath9k/mci.c b/drivers/net/wireless/ath/ath9k/mci.c
index 3363fc4e8966..a0845002d6fe 100644
--- a/drivers/net/wireless/ath/ath9k/mci.c
+++ b/drivers/net/wireless/ath/ath9k/mci.c
@@ -646,9 +646,7 @@ void ath9k_mci_update_wlan_channels(struct ath_softc *sc, bool allow_all)
 	struct ath_hw *ah = sc->sc_ah;
 	struct ath9k_hw_mci *mci = &ah->btcoex_hw.mci;
 	struct ath9k_channel *chan = ah->curchan;
-	static const u32 channelmap[] = {
-		0x00000000, 0xffff0000, 0xffffffff, 0x7fffffff
-	};
+	u32 channelmap[] = {0x00000000, 0xffff0000, 0xffffffff, 0x7fffffff};
 	int i;
 	s16 chan_start, chan_end;
 	u16 wlan_chan;
diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c
index ef9a8e0b75e6..f6f2ab7a63ff 100644
--- a/drivers/net/wireless/ath/ath9k/xmit.c
+++ b/drivers/net/wireless/ath/ath9k/xmit.c
@@ -34,6 +34,12 @@
 #define NUM_SYMBOLS_PER_USEC(_usec) (_usec >> 2)
 #define NUM_SYMBOLS_PER_USEC_HALFGI(_usec) (((_usec*5)-4)/18)
 
+/* Shifts in ar5008_phy.c and ar9003_phy.c are equal for all revisions */
+#define ATH9K_PWRTBL_11NA_OFDM_SHIFT    0
+#define ATH9K_PWRTBL_11NG_OFDM_SHIFT    4
+#define ATH9K_PWRTBL_11NA_HT_SHIFT      8
+#define ATH9K_PWRTBL_11NG_HT_SHIFT      12
+
 
 static u16 bits_per_symbol[][2] = {
 	/* 20MHz 40MHz */
@@ -1169,13 +1175,14 @@ void ath_update_max_aggr_framelen(struct ath_softc *sc, int queue, int txop)
 }
 
 static u8 ath_get_rate_txpower(struct ath_softc *sc, struct ath_buf *bf,
-			       u8 rateidx, bool is_40, bool is_cck)
+			       u8 rateidx, bool is_40, bool is_cck, bool is_mcs)
 {
 	u8 max_power;
 	struct sk_buff *skb;
 	struct ath_frame_info *fi;
 	struct ieee80211_tx_info *info;
 	struct ath_hw *ah = sc->sc_ah;
+	bool is_2ghz, is_5ghz, use_stbc;
 
 	if (sc->tx99_state || !ah->tpc_enabled)
 		return MAX_RATE_POWER;
@@ -1184,6 +1191,19 @@ static u8 ath_get_rate_txpower(struct ath_softc *sc, struct ath_buf *bf,
 	fi = get_frame_info(skb);
 	info = IEEE80211_SKB_CB(skb);
 
+	is_2ghz = info->band == NL80211_BAND_2GHZ;
+	is_5ghz = info->band == NL80211_BAND_5GHZ;
+	use_stbc = is_mcs && rateidx < 8 && (info->flags &
+					     IEEE80211_TX_CTL_STBC);
+
+	if (is_mcs)
+		rateidx += is_5ghz ? ATH9K_PWRTBL_11NA_HT_SHIFT
+				   : ATH9K_PWRTBL_11NG_HT_SHIFT;
+	else if (is_2ghz && !is_cck)
+		rateidx += ATH9K_PWRTBL_11NG_OFDM_SHIFT;
+	else
+		rateidx += ATH9K_PWRTBL_11NA_OFDM_SHIFT;
+
 	if (!AR_SREV_9300_20_OR_LATER(ah)) {
 		int txpower = fi->tx_power;
 
@@ -1193,10 +1213,8 @@ static u8 ath_get_rate_txpower(struct ath_softc *sc, struct ath_buf *bf,
 			u16 eeprom_rev = ah->eep_ops->get_eeprom_rev(ah);
 
 			if (eeprom_rev >= AR5416_EEP_MINOR_VER_2) {
-				bool is_2ghz;
 				struct modal_eep_header *pmodal;
 
-				is_2ghz = info->band == NL80211_BAND_2GHZ;
 				pmodal = &eep->modalHeader[is_2ghz];
 				power_ht40delta = pmodal->ht40PowerIncForPdadc;
 			} else {
@@ -1229,7 +1247,7 @@ static u8 ath_get_rate_txpower(struct ath_softc *sc, struct ath_buf *bf,
 		if (!max_power && !AR_SREV_9280_20_OR_LATER(ah))
 			max_power = 1;
 	} else if (!bf->bf_state.bfs_paprd) {
-		if (rateidx < 8 && (info->flags & IEEE80211_TX_CTL_STBC))
+		if (use_stbc)
 			max_power = min_t(u8, ah->tx_power_stbc[rateidx],
 					  fi->tx_power);
 		else
@@ -1319,7 +1337,7 @@ static void ath_buf_set_rate(struct ath_softc *sc, struct ath_buf *bf,
 			}
 
 			info->txpower[i] = ath_get_rate_txpower(sc, bf, rix,
-								is_40, false);
+								is_40, false, true);
 			continue;
 		}
 
@@ -1350,7 +1368,7 @@ static void ath_buf_set_rate(struct ath_softc *sc, struct ath_buf *bf,
 
 		is_cck = IS_CCK_RATE(info->rates[i].Rate);
 		info->txpower[i] = ath_get_rate_txpower(sc, bf, rix, false,
-							is_cck);
+							is_cck, false);
 	}
 
 	/* For AR5416 - RTS cannot be followed by a frame larger than 8K */
diff --git a/drivers/net/wireless/ath/carl9170/cmd.c b/drivers/net/wireless/ath/carl9170/cmd.c
index f2b4f537e4c1..b8ed193c0195 100644
--- a/drivers/net/wireless/ath/carl9170/cmd.c
+++ b/drivers/net/wireless/ath/carl9170/cmd.c
@@ -120,7 +120,7 @@ struct carl9170_cmd *carl9170_cmd_buf(struct ar9170 *ar,
 {
 	struct carl9170_cmd *tmp;
 
-	tmp = kzalloc(sizeof(struct carl9170_cmd_head) + len, GFP_ATOMIC);
+	tmp = kzalloc(sizeof(*tmp), GFP_ATOMIC);
 	if (tmp) {
 		tmp->hdr.cmd = cmd;
 		tmp->hdr.len = len;
diff --git a/drivers/net/wireless/ath/carl9170/fwcmd.h b/drivers/net/wireless/ath/carl9170/fwcmd.h
index ff4b3b50250c..e5bcc364f088 100644
--- a/drivers/net/wireless/ath/carl9170/fwcmd.h
+++ b/drivers/net/wireless/ath/carl9170/fwcmd.h
@@ -320,9 +320,9 @@ struct carl9170_rsp {
 		struct carl9170_u32_list	rreg_res;
 		struct carl9170_u32_list	echo;
 #ifdef __CARL9170FW__
-		struct carl9170_tx_status	tx_status[0];
+		DECLARE_FLEX_ARRAY(struct carl9170_tx_status, tx_status);
 #endif /* __CARL9170FW__ */
-		struct _carl9170_tx_status	_tx_status[0];
+		DECLARE_FLEX_ARRAY(struct _carl9170_tx_status, _tx_status);
 		struct carl9170_gpio		gpio;
 		struct carl9170_tsf_rsp		tsf;
 		struct carl9170_psm		psm;
diff --git a/drivers/net/wireless/ath/key.c b/drivers/net/wireless/ath/key.c
index 61b59a804e30..b7b61d4f02ba 100644
--- a/drivers/net/wireless/ath/key.c
+++ b/drivers/net/wireless/ath/key.c
@@ -503,7 +503,7 @@ int ath_key_config(struct ath_common *common,
 
 	hk.kv_len = key->keylen;
 	if (key->keylen)
-		memcpy(hk.kv_val, key->key, key->keylen);
+		memcpy(&hk.kv_values, key->key, key->keylen);
 
 	if (!(key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) {
 		switch (vif->type) {
diff --git a/drivers/net/wireless/ath/wcn36xx/dxe.c b/drivers/net/wireless/ath/wcn36xx/dxe.c
index 4e9e13941c8f..9013f056eecb 100644
--- a/drivers/net/wireless/ath/wcn36xx/dxe.c
+++ b/drivers/net/wireless/ath/wcn36xx/dxe.c
@@ -112,8 +112,8 @@ int wcn36xx_dxe_alloc_ctl_blks(struct wcn36xx *wcn)
 	wcn->dxe_rx_l_ch.desc_num = WCN36XX_DXE_CH_DESC_NUMB_RX_L;
 	wcn->dxe_rx_h_ch.desc_num = WCN36XX_DXE_CH_DESC_NUMB_RX_H;
 
-	wcn->dxe_tx_l_ch.dxe_wq =  WCN36XX_DXE_WQ_TX_L;
-	wcn->dxe_tx_h_ch.dxe_wq =  WCN36XX_DXE_WQ_TX_H;
+	wcn->dxe_tx_l_ch.dxe_wq =  WCN36XX_DXE_WQ_TX_L(wcn);
+	wcn->dxe_tx_h_ch.dxe_wq =  WCN36XX_DXE_WQ_TX_H(wcn);
 
 	wcn->dxe_tx_l_ch.ctrl_bd = WCN36XX_DXE_CTRL_TX_L_BD;
 	wcn->dxe_tx_h_ch.ctrl_bd = WCN36XX_DXE_CTRL_TX_H_BD;
@@ -165,8 +165,9 @@ void wcn36xx_dxe_free_ctl_blks(struct wcn36xx *wcn)
 	wcn36xx_dxe_free_ctl_block(&wcn->dxe_rx_h_ch);
 }
 
-static int wcn36xx_dxe_init_descs(struct device *dev, struct wcn36xx_dxe_ch *wcn_ch)
+static int wcn36xx_dxe_init_descs(struct wcn36xx *wcn, struct wcn36xx_dxe_ch *wcn_ch)
 {
+	struct device *dev = wcn->dev;
 	struct wcn36xx_dxe_desc *cur_dxe = NULL;
 	struct wcn36xx_dxe_desc *prev_dxe = NULL;
 	struct wcn36xx_dxe_ctl *cur_ctl = NULL;
@@ -190,11 +191,11 @@ static int wcn36xx_dxe_init_descs(struct device *dev, struct wcn36xx_dxe_ch *wcn
 		switch (wcn_ch->ch_type) {
 		case WCN36XX_DXE_CH_TX_L:
 			cur_dxe->ctrl = WCN36XX_DXE_CTRL_TX_L;
-			cur_dxe->dst_addr_l = WCN36XX_DXE_WQ_TX_L;
+			cur_dxe->dst_addr_l = WCN36XX_DXE_WQ_TX_L(wcn);
 			break;
 		case WCN36XX_DXE_CH_TX_H:
 			cur_dxe->ctrl = WCN36XX_DXE_CTRL_TX_H;
-			cur_dxe->dst_addr_l = WCN36XX_DXE_WQ_TX_H;
+			cur_dxe->dst_addr_l = WCN36XX_DXE_WQ_TX_H(wcn);
 			break;
 		case WCN36XX_DXE_CH_RX_L:
 			cur_dxe->ctrl = WCN36XX_DXE_CTRL_RX_L;
@@ -914,7 +915,7 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
 	/***************************************/
 	/* Init descriptors for TX LOW channel */
 	/***************************************/
-	ret = wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_tx_l_ch);
+	ret = wcn36xx_dxe_init_descs(wcn, &wcn->dxe_tx_l_ch);
 	if (ret) {
 		dev_err(wcn->dev, "Error allocating descriptor\n");
 		return ret;
@@ -928,14 +929,14 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
 	/* Program DMA destination addr for TX LOW */
 	wcn36xx_dxe_write_register(wcn,
 		WCN36XX_DXE_CH_DEST_ADDR_TX_L,
-		WCN36XX_DXE_WQ_TX_L);
+		WCN36XX_DXE_WQ_TX_L(wcn));
 
 	wcn36xx_dxe_read_register(wcn, WCN36XX_DXE_REG_CH_EN, &reg_data);
 
 	/***************************************/
 	/* Init descriptors for TX HIGH channel */
 	/***************************************/
-	ret = wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_tx_h_ch);
+	ret = wcn36xx_dxe_init_descs(wcn, &wcn->dxe_tx_h_ch);
 	if (ret) {
 		dev_err(wcn->dev, "Error allocating descriptor\n");
 		goto out_err_txh_ch;
@@ -950,14 +951,14 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
 	/* Program DMA destination addr for TX HIGH */
 	wcn36xx_dxe_write_register(wcn,
 		WCN36XX_DXE_CH_DEST_ADDR_TX_H,
-		WCN36XX_DXE_WQ_TX_H);
+		WCN36XX_DXE_WQ_TX_H(wcn));
 
 	wcn36xx_dxe_read_register(wcn, WCN36XX_DXE_REG_CH_EN, &reg_data);
 
 	/***************************************/
 	/* Init descriptors for RX LOW channel */
 	/***************************************/
-	ret = wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_rx_l_ch);
+	ret = wcn36xx_dxe_init_descs(wcn, &wcn->dxe_rx_l_ch);
 	if (ret) {
 		dev_err(wcn->dev, "Error allocating descriptor\n");
 		goto out_err_rxl_ch;
@@ -988,7 +989,7 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
 	/***************************************/
 	/* Init descriptors for RX HIGH channel */
 	/***************************************/
-	ret = wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_rx_h_ch);
+	ret = wcn36xx_dxe_init_descs(wcn, &wcn->dxe_rx_h_ch);
 	if (ret) {
 		dev_err(wcn->dev, "Error allocating descriptor\n");
 		goto out_err_rxh_ch;
diff --git a/drivers/net/wireless/ath/wcn36xx/dxe.h b/drivers/net/wireless/ath/wcn36xx/dxe.h
index 26a31edf52e9..dd8c684a3ba7 100644
--- a/drivers/net/wireless/ath/wcn36xx/dxe.h
+++ b/drivers/net/wireless/ath/wcn36xx/dxe.h
@@ -135,8 +135,8 @@ H2H_TEST_RX_TX = DMA2
 	WCN36xx_DXE_CTRL_ENDIANNESS)
 
 /* TODO This must calculated properly but not hardcoded */
-#define WCN36XX_DXE_WQ_TX_L			0x17
-#define WCN36XX_DXE_WQ_TX_H			0x17
+#define WCN36XX_DXE_WQ_TX_L(wcn)    ((wcn)->is_pronto_v3 ? 0x6 : 0x17)
+#define WCN36XX_DXE_WQ_TX_H(wcn)    ((wcn)->is_pronto_v3 ? 0x6 : 0x17)
 #define WCN36XX_DXE_WQ_RX_L			0xB
 #define WCN36XX_DXE_WQ_RX_H			0x4
 
diff --git a/drivers/net/wireless/ath/wcn36xx/main.c b/drivers/net/wireless/ath/wcn36xx/main.c
index 3b79cc1c7c5b..8dbd115a393c 100644
--- a/drivers/net/wireless/ath/wcn36xx/main.c
+++ b/drivers/net/wireless/ath/wcn36xx/main.c
@@ -1508,6 +1508,7 @@ static int wcn36xx_platform_get_resources(struct wcn36xx *wcn,
 	}
 
 	wcn->is_pronto = !!of_device_is_compatible(mmio_node, "qcom,pronto");
+	wcn->is_pronto_v3 = !!of_device_is_compatible(mmio_node, "qcom,pronto-v3-pil");
 
 	/* Map the CCU memory */
 	index = of_property_match_string(mmio_node, "reg-names", "ccu");
diff --git a/drivers/net/wireless/ath/wcn36xx/smd.c b/drivers/net/wireless/ath/wcn36xx/smd.c
index 566f0b9c1584..17e1919d1cd8 100644
--- a/drivers/net/wireless/ath/wcn36xx/smd.c
+++ b/drivers/net/wireless/ath/wcn36xx/smd.c
@@ -475,8 +475,8 @@ out:
 
 #define PREPARE_HAL_BUF(send_buf, msg_body) \
 	do {							\
-		memset(send_buf, 0, msg_body.header.len);	\
-		memcpy(send_buf, &msg_body, sizeof(msg_body));	\
+		memcpy_and_pad(send_buf, msg_body.header.len,	\
+			       &msg_body, sizeof(msg_body), 0);	\
 	} while (0)						\
 
 #define PREPARE_HAL_PTT_MSG_BUF(send_buf, p_msg_body) \
diff --git a/drivers/net/wireless/ath/wcn36xx/wcn36xx.h b/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
index 9aa08b636d08..ff4a8e5d7209 100644
--- a/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
+++ b/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
@@ -217,6 +217,7 @@ struct wcn36xx {
 	u8			fw_major;
 	u32			fw_feat_caps[WCN36XX_HAL_CAPS_SIZE];
 	bool			is_pronto;
+	bool			is_pronto_v3;
 
 	/* extra byte for the NULL termination */
 	u8			crm_version[WCN36XX_HAL_VERSION_LENGTH + 1];
diff --git a/drivers/net/wireless/broadcom/b43legacy/dma.c b/drivers/net/wireless/broadcom/b43legacy/dma.c
index 6869f2bf1bae..60e41de72f29 100644
--- a/drivers/net/wireless/broadcom/b43legacy/dma.c
+++ b/drivers/net/wireless/broadcom/b43legacy/dma.c
@@ -127,14 +127,6 @@ static inline int next_slot(struct b43legacy_dmaring *ring, int slot)
 	return slot + 1;
 }
 
-static inline int prev_slot(struct b43legacy_dmaring *ring, int slot)
-{
-	B43legacy_WARN_ON(!(slot >= 0 && slot <= ring->nr_slots - 1));
-	if (slot == 0)
-		return ring->nr_slots - 1;
-	return slot - 1;
-}
-
 #ifdef CONFIG_B43LEGACY_DEBUG
 static void update_max_used_slots(struct b43legacy_dmaring *ring,
 				  int current_used_slots)
diff --git a/drivers/net/wireless/broadcom/b43legacy/radio.c b/drivers/net/wireless/broadcom/b43legacy/radio.c
index fdf78c10a05c..8d7eb89c1628 100644
--- a/drivers/net/wireless/broadcom/b43legacy/radio.c
+++ b/drivers/net/wireless/broadcom/b43legacy/radio.c
@@ -1709,23 +1709,6 @@ u16 b43legacy_radio_init2050(struct b43legacy_wldev *dev)
 	return ret;
 }
 
-static inline
-u16 freq_r3A_value(u16 frequency)
-{
-	u16 value;
-
-	if (frequency < 5091)
-		value = 0x0040;
-	else if (frequency < 5321)
-		value = 0x0000;
-	else if (frequency < 5806)
-		value = 0x0080;
-	else
-		value = 0x0040;
-
-	return value;
-}
-
 int b43legacy_radio_selectchannel(struct b43legacy_wldev *dev,
 				  u8 channel,
 				  int synthetic_pu_workaround)
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile
index 0e996cf24f88..dc6d27a36faa 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile
@@ -48,6 +48,8 @@ brcmfmac-$(CONFIG_OF) += \
 		of.o
 brcmfmac-$(CONFIG_DMI) += \
 		dmi.o
+brcmfmac-$(CONFIG_ACPI) += \
+		acpi.o
 
 ifeq ($(CONFIG_BRCMFMAC),m)
 obj-m += wcc/
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/acpi.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/acpi.c
new file mode 100644
index 000000000000..c4a54861bfb4
--- /dev/null
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/acpi.c
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: ISC
+/*
+ * Copyright The Asahi Linux Contributors
+ */
+
+#include <linux/acpi.h>
+#include "debug.h"
+#include "core.h"
+#include "common.h"
+
+void brcmf_acpi_probe(struct device *dev, enum brcmf_bus_type bus_type,
+		      struct brcmf_mp_device *settings)
+{
+	acpi_status status;
+	const union acpi_object *o;
+	struct acpi_buffer buf = {ACPI_ALLOCATE_BUFFER, NULL};
+	struct acpi_device *adev = ACPI_COMPANION(dev);
+
+	if (!adev)
+		return;
+
+	if (!ACPI_FAILURE(acpi_dev_get_property(adev, "module-instance",
+						ACPI_TYPE_STRING, &o))) {
+		brcmf_dbg(INFO, "ACPI module-instance=%s\n", o->string.pointer);
+		settings->board_type = devm_kasprintf(dev, GFP_KERNEL,
+						      "apple,%s",
+						      o->string.pointer);
+	} else {
+		brcmf_dbg(INFO, "No ACPI module-instance\n");
+		return;
+	}
+
+	status = acpi_evaluate_object(adev->handle, "RWCV", NULL, &buf);
+	o = buf.pointer;
+	if (!ACPI_FAILURE(status) && o && o->type == ACPI_TYPE_BUFFER &&
+	    o->buffer.length >= 2) {
+		char *antenna_sku = devm_kzalloc(dev, 3, GFP_KERNEL);
+
+		if (antenna_sku) {
+			memcpy(antenna_sku, o->buffer.pointer, 2);
+			brcmf_dbg(INFO, "ACPI RWCV data=%*phN antenna-sku=%s\n",
+				  (int)o->buffer.length, o->buffer.pointer,
+				  antenna_sku);
+			settings->antenna_sku = antenna_sku;
+		}
+
+		kfree(buf.pointer);
+	} else {
+		brcmf_dbg(INFO, "No ACPI antenna-sku\n");
+	}
+}
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
index 65d4799a5658..ff710b0b5071 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
@@ -965,6 +965,12 @@ out:
 		.driver_data = BRCMF_FWVENDOR_ ## fw_vend \
 	}
 
+#define CYW_SDIO_DEVICE(dev_id, fw_vend) \
+	{ \
+		SDIO_DEVICE(SDIO_VENDOR_ID_CYPRESS, dev_id), \
+		.driver_data = BRCMF_FWVENDOR_ ## fw_vend \
+	}
+
 /* devices we support, null terminated */
 static const struct sdio_device_id brcmf_sdmmc_ids[] = {
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43143, WCC),
@@ -979,6 +985,7 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = {
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4335_4339, WCC),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4339, WCC),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43430, WCC),
+	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43439, WCC),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4345, WCC),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43455, WCC),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4354, WCC),
@@ -986,9 +993,9 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = {
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4359, WCC),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_4373, CYW),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_43012, CYW),
-	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_43439, CYW),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_43752, CYW),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_89359, CYW),
+	CYW_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_43439, CYW),
 	{ /* end: all zeroes */ }
 };
 MODULE_DEVICE_TABLE(sdio, brcmf_sdmmc_ids);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h
index 501136e011b5..fe31051a9e11 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h
@@ -55,6 +55,7 @@ enum brcmf_bus_protocol_type {
 /* Firmware blobs that may be available */
 enum brcmf_blob_type {
 	BRCMF_BLOB_CLM,
+	BRCMF_BLOB_TXCAP,
 };
 
 struct brcmf_mp_device;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
index a9690ec4c850..de8a2e27f49c 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
@@ -1039,12 +1039,134 @@ void brcmf_set_mpc(struct brcmf_if *ifp, int mpc)
 	}
 }
 
+static void brcmf_scan_params_v2_to_v1(struct brcmf_scan_params_v2_le *params_v2_le,
+				       struct brcmf_scan_params_le *params_le)
+{
+	size_t params_size;
+	u32 ch;
+	int n_channels, n_ssids;
+
+	memcpy(&params_le->ssid_le, &params_v2_le->ssid_le,
+	       sizeof(params_le->ssid_le));
+	memcpy(&params_le->bssid, &params_v2_le->bssid,
+	       sizeof(params_le->bssid));
+
+	params_le->bss_type = params_v2_le->bss_type;
+	params_le->scan_type = le32_to_cpu(params_v2_le->scan_type);
+	params_le->nprobes = params_v2_le->nprobes;
+	params_le->active_time = params_v2_le->active_time;
+	params_le->passive_time = params_v2_le->passive_time;
+	params_le->home_time = params_v2_le->home_time;
+	params_le->channel_num = params_v2_le->channel_num;
+
+	ch = le32_to_cpu(params_v2_le->channel_num);
+	n_channels = ch & BRCMF_SCAN_PARAMS_COUNT_MASK;
+	n_ssids = ch >> BRCMF_SCAN_PARAMS_NSSID_SHIFT;
+
+	params_size = sizeof(u16) * n_channels;
+	if (n_ssids > 0) {
+		params_size = roundup(params_size, sizeof(u32));
+		params_size += sizeof(struct brcmf_ssid_le) * n_ssids;
+	}
+
+	memcpy(&params_le->channel_list[0],
+	       &params_v2_le->channel_list[0], params_size);
+}
+
+static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg,
+			     struct brcmf_scan_params_v2_le *params_le,
+			     struct cfg80211_scan_request *request)
+{
+	u32 n_ssids;
+	u32 n_channels;
+	s32 i;
+	s32 offset;
+	u16 chanspec;
+	char *ptr;
+	int length;
+	struct brcmf_ssid_le ssid_le;
+
+	eth_broadcast_addr(params_le->bssid);
+
+	length = BRCMF_SCAN_PARAMS_V2_FIXED_SIZE;
+
+	params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V2);
+	params_le->bss_type = DOT11_BSSTYPE_ANY;
+	params_le->scan_type = cpu_to_le32(BRCMF_SCANTYPE_ACTIVE);
+	params_le->channel_num = 0;
+	params_le->nprobes = cpu_to_le32(-1);
+	params_le->active_time = cpu_to_le32(-1);
+	params_le->passive_time = cpu_to_le32(-1);
+	params_le->home_time = cpu_to_le32(-1);
+	memset(&params_le->ssid_le, 0, sizeof(params_le->ssid_le));
+
+	/* Scan abort */
+	if (!request) {
+		length += sizeof(u16);
+		params_le->channel_num = cpu_to_le32(1);
+		params_le->channel_list[0] = cpu_to_le16(-1);
+		params_le->length = cpu_to_le16(length);
+		return;
+	}
+
+	n_ssids = request->n_ssids;
+	n_channels = request->n_channels;
+
+	/* Copy channel array if applicable */
+	brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n",
+		  n_channels);
+	if (n_channels > 0) {
+		length += roundup(sizeof(u16) * n_channels, sizeof(u32));
+		for (i = 0; i < n_channels; i++) {
+			chanspec = channel_to_chanspec(&cfg->d11inf,
+						       request->channels[i]);
+			brcmf_dbg(SCAN, "Chan : %d, Channel spec: %x\n",
+				  request->channels[i]->hw_value, chanspec);
+			params_le->channel_list[i] = cpu_to_le16(chanspec);
+		}
+	} else {
+		brcmf_dbg(SCAN, "Scanning all channels\n");
+	}
+
+	/* Copy ssid array if applicable */
+	brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids);
+	if (n_ssids > 0) {
+		offset = offsetof(struct brcmf_scan_params_v2_le, channel_list) +
+				n_channels * sizeof(u16);
+		offset = roundup(offset, sizeof(u32));
+		length += sizeof(ssid_le) * n_ssids,
+		ptr = (char *)params_le + offset;
+		for (i = 0; i < n_ssids; i++) {
+			memset(&ssid_le, 0, sizeof(ssid_le));
+			ssid_le.SSID_len =
+					cpu_to_le32(request->ssids[i].ssid_len);
+			memcpy(ssid_le.SSID, request->ssids[i].ssid,
+			       request->ssids[i].ssid_len);
+			if (!ssid_le.SSID_len)
+				brcmf_dbg(SCAN, "%d: Broadcast scan\n", i);
+			else
+				brcmf_dbg(SCAN, "%d: scan for  %.32s size=%d\n",
+					  i, ssid_le.SSID, ssid_le.SSID_len);
+			memcpy(ptr, &ssid_le, sizeof(ssid_le));
+			ptr += sizeof(ssid_le);
+		}
+	} else {
+		brcmf_dbg(SCAN, "Performing passive scan\n");
+		params_le->scan_type = cpu_to_le32(BRCMF_SCANTYPE_PASSIVE);
+	}
+	params_le->length = cpu_to_le16(length);
+	/* Adding mask to channel numbers */
+	params_le->channel_num =
+		cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) |
+			(n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK));
+}
+
 s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg,
 				struct brcmf_if *ifp, bool aborted,
 				bool fw_abort)
 {
 	struct brcmf_pub *drvr = cfg->pub;
-	struct brcmf_scan_params_le params_le;
+	struct brcmf_scan_params_v2_le params_v2_le;
 	struct cfg80211_scan_request *scan_request;
 	u64 reqid;
 	u32 bucket;
@@ -1063,20 +1185,23 @@ s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg,
 	if (fw_abort) {
 		/* Do a scan abort to stop the driver's scan engine */
 		brcmf_dbg(SCAN, "ABORT scan in firmware\n");
-		memset(&params_le, 0, sizeof(params_le));
-		eth_broadcast_addr(params_le.bssid);
-		params_le.bss_type = DOT11_BSSTYPE_ANY;
-		params_le.scan_type = 0;
-		params_le.channel_num = cpu_to_le32(1);
-		params_le.nprobes = cpu_to_le32(1);
-		params_le.active_time = cpu_to_le32(-1);
-		params_le.passive_time = cpu_to_le32(-1);
-		params_le.home_time = cpu_to_le32(-1);
-		/* Scan is aborted by setting channel_list[0] to -1 */
-		params_le.channel_list[0] = cpu_to_le16(-1);
+
+		brcmf_escan_prep(cfg, &params_v2_le, NULL);
+
 		/* E-Scan (or anyother type) can be aborted by SCAN */
-		err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN,
-					     &params_le, sizeof(params_le));
+		if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SCAN_V2)) {
+			err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN,
+						     &params_v2_le,
+						     sizeof(params_v2_le));
+		} else {
+			struct brcmf_scan_params_le params_le;
+
+			brcmf_scan_params_v2_to_v1(&params_v2_le, &params_le);
+			err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN,
+						     &params_le,
+						     sizeof(params_le));
+		}
+
 		if (err)
 			bphy_err(drvr, "Scan abort failed\n");
 	}
@@ -1295,83 +1420,13 @@ done:
 	return err;
 }
 
-static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg,
-			     struct brcmf_scan_params_le *params_le,
-			     struct cfg80211_scan_request *request)
-{
-	u32 n_ssids;
-	u32 n_channels;
-	s32 i;
-	s32 offset;
-	u16 chanspec;
-	char *ptr;
-	struct brcmf_ssid_le ssid_le;
-
-	eth_broadcast_addr(params_le->bssid);
-	params_le->bss_type = DOT11_BSSTYPE_ANY;
-	params_le->scan_type = BRCMF_SCANTYPE_ACTIVE;
-	params_le->channel_num = 0;
-	params_le->nprobes = cpu_to_le32(-1);
-	params_le->active_time = cpu_to_le32(-1);
-	params_le->passive_time = cpu_to_le32(-1);
-	params_le->home_time = cpu_to_le32(-1);
-	memset(&params_le->ssid_le, 0, sizeof(params_le->ssid_le));
-
-	n_ssids = request->n_ssids;
-	n_channels = request->n_channels;
-
-	/* Copy channel array if applicable */
-	brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n",
-		  n_channels);
-	if (n_channels > 0) {
-		for (i = 0; i < n_channels; i++) {
-			chanspec = channel_to_chanspec(&cfg->d11inf,
-						       request->channels[i]);
-			brcmf_dbg(SCAN, "Chan : %d, Channel spec: %x\n",
-				  request->channels[i]->hw_value, chanspec);
-			params_le->channel_list[i] = cpu_to_le16(chanspec);
-		}
-	} else {
-		brcmf_dbg(SCAN, "Scanning all channels\n");
-	}
-	/* Copy ssid array if applicable */
-	brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids);
-	if (n_ssids > 0) {
-		offset = offsetof(struct brcmf_scan_params_le, channel_list) +
-				n_channels * sizeof(u16);
-		offset = roundup(offset, sizeof(u32));
-		ptr = (char *)params_le + offset;
-		for (i = 0; i < n_ssids; i++) {
-			memset(&ssid_le, 0, sizeof(ssid_le));
-			ssid_le.SSID_len =
-					cpu_to_le32(request->ssids[i].ssid_len);
-			memcpy(ssid_le.SSID, request->ssids[i].ssid,
-			       request->ssids[i].ssid_len);
-			if (!ssid_le.SSID_len)
-				brcmf_dbg(SCAN, "%d: Broadcast scan\n", i);
-			else
-				brcmf_dbg(SCAN, "%d: scan for  %.32s size=%d\n",
-					  i, ssid_le.SSID, ssid_le.SSID_len);
-			memcpy(ptr, &ssid_le, sizeof(ssid_le));
-			ptr += sizeof(ssid_le);
-		}
-	} else {
-		brcmf_dbg(SCAN, "Performing passive scan\n");
-		params_le->scan_type = BRCMF_SCANTYPE_PASSIVE;
-	}
-	/* Adding mask to channel numbers */
-	params_le->channel_num =
-		cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) |
-			(n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK));
-}
-
 static s32
 brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp,
 		struct cfg80211_scan_request *request)
 {
 	struct brcmf_pub *drvr = cfg->pub;
-	s32 params_size = BRCMF_SCAN_PARAMS_FIXED_SIZE +
-			  offsetof(struct brcmf_escan_params_le, params_le);
+	s32 params_size = BRCMF_SCAN_PARAMS_V2_FIXED_SIZE +
+			  offsetof(struct brcmf_escan_params_le, params_v2_le);
 	struct brcmf_escan_params_le *params;
 	s32 err = 0;
 
@@ -1391,8 +1446,22 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp,
 		goto exit;
 	}
 	BUG_ON(params_size + sizeof("escan") >= BRCMF_DCMD_MEDLEN);
-	brcmf_escan_prep(cfg, &params->params_le, request);
-	params->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION);
+	brcmf_escan_prep(cfg, &params->params_v2_le, request);
+
+	params->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION_V2);
+
+	if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SCAN_V2)) {
+		struct brcmf_escan_params_le *params_v1;
+
+		params_size -= BRCMF_SCAN_PARAMS_V2_FIXED_SIZE;
+		params_size += BRCMF_SCAN_PARAMS_FIXED_SIZE;
+		params_v1 = kzalloc(params_size, GFP_KERNEL);
+		params_v1->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION);
+		brcmf_scan_params_v2_to_v1(&params->params_v2_le, &params_v1->params_le);
+		kfree(params);
+		params = params_v1;
+	}
+
 	params->action = cpu_to_le16(WL_ESCAN_ACTION_START);
 	params->sync_id = cpu_to_le16(0x1234);
 
@@ -1617,13 +1686,14 @@ static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk_data, u16 pmk_len)
 {
 	struct brcmf_pub *drvr = ifp->drvr;
 	struct brcmf_wsec_pmk_le pmk;
-	int i, err;
+	int err;
+
+	memset(&pmk, 0, sizeof(pmk));
 
-	/* convert to firmware key format */
-	pmk.key_len = cpu_to_le16(pmk_len << 1);
-	pmk.flags = cpu_to_le16(BRCMF_WSEC_PASSPHRASE);
-	for (i = 0; i < pmk_len; i++)
-		snprintf(&pmk.key[2 * i], 3, "%02x", pmk_data[i]);
+	/* pass pmk directly */
+	pmk.key_len = cpu_to_le16(pmk_len);
+	pmk.flags = cpu_to_le16(0);
+	memcpy(pmk.key, pmk_data, pmk_len);
 
 	/* store psk in firmware */
 	err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_WSEC_PMK,
@@ -4237,6 +4307,37 @@ exit:
 	return 0;
 }
 
+static s32
+brcmf_pmksa_v3_op(struct brcmf_if *ifp, struct cfg80211_pmksa *pmksa,
+		  bool alive)
+{
+	struct brcmf_pmk_op_v3_le *pmk_op;
+	int length = offsetof(struct brcmf_pmk_op_v3_le, pmk);
+	int ret;
+
+	pmk_op = kzalloc(sizeof(*pmk_op), GFP_KERNEL);
+	pmk_op->version = cpu_to_le16(BRCMF_PMKSA_VER_3);
+
+	if (!pmksa) {
+		/* Flush operation, operate on entire list */
+		pmk_op->count = cpu_to_le16(0);
+	} else {
+		/* Single PMK operation */
+		pmk_op->count = cpu_to_le16(1);
+		length += sizeof(struct brcmf_pmksa_v3);
+		memcpy(pmk_op->pmk[0].bssid, pmksa->bssid, ETH_ALEN);
+		memcpy(pmk_op->pmk[0].pmkid, pmksa->pmkid, WLAN_PMKID_LEN);
+		pmk_op->pmk[0].pmkid_len = WLAN_PMKID_LEN;
+		pmk_op->pmk[0].time_left = cpu_to_le32(alive ? BRCMF_PMKSA_NO_EXPIRY : 0);
+	}
+
+	pmk_op->length = cpu_to_le16(length);
+
+	ret = brcmf_fil_iovar_data_set(ifp, "pmkid_info", pmk_op, sizeof(*pmk_op));
+	kfree(pmk_op);
+	return ret;
+}
+
 static __used s32
 brcmf_update_pmklist(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp)
 {
@@ -4270,6 +4371,14 @@ brcmf_cfg80211_set_pmksa(struct wiphy *wiphy, struct net_device *ndev,
 	if (!check_vif_up(ifp->vif))
 		return -EIO;
 
+	brcmf_dbg(CONN, "set_pmksa - PMK bssid: %pM =\n", pmksa->bssid);
+	brcmf_dbg(CONN, "%*ph\n", WLAN_PMKID_LEN, pmksa->pmkid);
+
+	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PMKID_V3))
+		return brcmf_pmksa_v3_op(ifp, pmksa, true);
+
+	/* TODO: implement PMKID_V2 */
+
 	npmk = le32_to_cpu(cfg->pmk_list.npmk);
 	for (i = 0; i < npmk; i++)
 		if (!memcmp(pmksa->bssid, pmk[i].bssid, ETH_ALEN))
@@ -4286,9 +4395,6 @@ brcmf_cfg80211_set_pmksa(struct wiphy *wiphy, struct net_device *ndev,
 		return -EINVAL;
 	}
 
-	brcmf_dbg(CONN, "set_pmksa - PMK bssid: %pM =\n", pmk[npmk].bssid);
-	brcmf_dbg(CONN, "%*ph\n", WLAN_PMKID_LEN, pmk[npmk].pmkid);
-
 	err = brcmf_update_pmklist(cfg, ifp);
 
 	brcmf_dbg(TRACE, "Exit\n");
@@ -4312,6 +4418,11 @@ brcmf_cfg80211_del_pmksa(struct wiphy *wiphy, struct net_device *ndev,
 
 	brcmf_dbg(CONN, "del_pmksa - PMK bssid = %pM\n", pmksa->bssid);
 
+	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PMKID_V3))
+		return brcmf_pmksa_v3_op(ifp, pmksa, false);
+
+	/* TODO: implement PMKID_V2 */
+
 	npmk = le32_to_cpu(cfg->pmk_list.npmk);
 	for (i = 0; i < npmk; i++)
 		if (!memcmp(pmksa->bssid, pmk[i].bssid, ETH_ALEN))
@@ -4348,6 +4459,11 @@ brcmf_cfg80211_flush_pmksa(struct wiphy *wiphy, struct net_device *ndev)
 	if (!check_vif_up(ifp->vif))
 		return -EIO;
 
+	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PMKID_V3))
+		return brcmf_pmksa_v3_op(ifp, NULL, false);
+
+	/* TODO: implement PMKID_V2 */
+
 	memset(&cfg->pmk_list, 0, sizeof(cfg->pmk_list));
 	err = brcmf_update_pmklist(cfg, ifp);
 
@@ -6164,6 +6280,11 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg,
 		(struct brcmf_cfg80211_assoc_ielen_le *)cfg->extra_buf;
 	req_len = le32_to_cpu(assoc_info->req_len);
 	resp_len = le32_to_cpu(assoc_info->resp_len);
+	if (req_len > WL_EXTRA_BUF_MAX || resp_len > WL_EXTRA_BUF_MAX) {
+		bphy_err(drvr, "invalid lengths in assoc info: req %u resp %u\n",
+			 req_len, resp_len);
+		return -EINVAL;
+	}
 	if (req_len) {
 		err = brcmf_fil_iovar_data_get(ifp, "assoc_req_ies",
 					       cfg->extra_buf,
@@ -6489,18 +6610,20 @@ static s32 brcmf_notify_rssi(struct brcmf_if *ifp,
 {
 	struct brcmf_cfg80211_vif *vif = ifp->vif;
 	struct brcmf_rssi_be *info = data;
-	s32 rssi, snr, noise;
+	s32 rssi, snr = 0, noise = 0;
 	s32 low, high, last;
 
-	if (e->datalen < sizeof(*info)) {
+	if (e->datalen >= sizeof(*info)) {
+		rssi = be32_to_cpu(info->rssi);
+		snr = be32_to_cpu(info->snr);
+		noise = be32_to_cpu(info->noise);
+	} else if (e->datalen >= sizeof(rssi)) {
+		rssi = be32_to_cpu(*(__be32 *)data);
+	} else {
 		brcmf_err("insufficient RSSI event data\n");
 		return 0;
 	}
 
-	rssi = be32_to_cpu(info->rssi);
-	snr = be32_to_cpu(info->snr);
-	noise = be32_to_cpu(info->noise);
-
 	low = vif->cqm_rssi_low;
 	high = vif->cqm_rssi_high;
 	last = vif->cqm_rssi_last;
@@ -7763,6 +7886,7 @@ static bool brmcf_use_iso3166_ccode_fallback(struct brcmf_pub *drvr)
 	switch (drvr->bus_if->chip) {
 	case BRCM_CC_43430_CHIP_ID:
 	case BRCM_CC_4345_CHIP_ID:
+	case BRCM_CC_4356_CHIP_ID:
 	case BRCM_CC_43602_CHIP_ID:
 		return true;
 	default:
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
index 8073f31be27d..9f9bf08a70bb 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
@@ -212,8 +212,9 @@ struct sbsocramregs {
 #define	ARMCR4_TCBANB_MASK	0xf
 #define	ARMCR4_TCBANB_SHIFT	0
 
-#define	ARMCR4_BSZ_MASK		0x3f
+#define	ARMCR4_BSZ_MASK		0x7f
 #define	ARMCR4_BSZ_MULT		8192
+#define	ARMCR4_BLK_1K_MASK	0x200
 
 struct brcmf_core_priv {
 	struct brcmf_core pub;
@@ -684,6 +685,7 @@ static u32 brcmf_chip_tcm_ramsize(struct brcmf_core_priv *cr4)
 	u32 nbb;
 	u32 totb;
 	u32 bxinfo;
+	u32 blksize;
 	u32 idx;
 
 	corecap = brcmf_chip_core_read32(cr4, ARMCR4_CAP);
@@ -695,7 +697,11 @@ static u32 brcmf_chip_tcm_ramsize(struct brcmf_core_priv *cr4)
 	for (idx = 0; idx < totb; idx++) {
 		brcmf_chip_core_write32(cr4, ARMCR4_BANKIDX, idx);
 		bxinfo = brcmf_chip_core_read32(cr4, ARMCR4_BANKINFO);
-		memsize += ((bxinfo & ARMCR4_BSZ_MASK) + 1) * ARMCR4_BSZ_MULT;
+		blksize = ARMCR4_BSZ_MULT;
+		if (bxinfo & ARMCR4_BLK_1K_MASK)
+			blksize >>= 3;
+
+		memsize += ((bxinfo & ARMCR4_BSZ_MASK) + 1) * blksize;
 	}
 
 	return memsize;
@@ -737,6 +743,8 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci)
 		return 0x170000;
 	case BRCM_CC_4378_CHIP_ID:
 		return 0x352000;
+	case BRCM_CC_4387_CHIP_ID:
+		return 0x740000;
 	default:
 		brcmf_err("unknown chip: %s\n", ci->pub.name);
 		break;
@@ -1292,15 +1300,18 @@ static bool brcmf_chip_cm3_set_active(struct brcmf_chip_priv *chip)
 static inline void
 brcmf_chip_cr4_set_passive(struct brcmf_chip_priv *chip)
 {
+	int i;
 	struct brcmf_core *core;
 
 	brcmf_chip_disable_arm(chip, BCMA_CORE_ARM_CR4);
 
-	core = brcmf_chip_get_core(&chip->pub, BCMA_CORE_80211);
-	brcmf_chip_resetcore(core, D11_BCMA_IOCTL_PHYRESET |
-				   D11_BCMA_IOCTL_PHYCLOCKEN,
-			     D11_BCMA_IOCTL_PHYCLOCKEN,
-			     D11_BCMA_IOCTL_PHYCLOCKEN);
+	/* Disable the cores only and let the firmware enable them.
+	 * Releasing reset ourselves breaks BCM4387 in weird ways.
+	 */
+	for (i = 0; (core = brcmf_chip_get_d11core(&chip->pub, i)); i++)
+		brcmf_chip_coredisable(core, D11_BCMA_IOCTL_PHYRESET |
+				       D11_BCMA_IOCTL_PHYCLOCKEN,
+				       D11_BCMA_IOCTL_PHYCLOCKEN);
 }
 
 static bool brcmf_chip_cr4_set_active(struct brcmf_chip_priv *chip, u32 rstvec)
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
index f235beaddddb..a194b0e68eb5 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
@@ -101,7 +101,7 @@ void brcmf_c_set_joinpref_default(struct brcmf_if *ifp)
 
 static int brcmf_c_download(struct brcmf_if *ifp, u16 flag,
 			    struct brcmf_dload_data_le *dload_buf,
-			    u32 len)
+			    u32 len, const char *var)
 {
 	s32 err;
 
@@ -111,18 +111,18 @@ static int brcmf_c_download(struct brcmf_if *ifp, u16 flag,
 	dload_buf->len = cpu_to_le32(len);
 	dload_buf->crc = cpu_to_le32(0);
 
-	err = brcmf_fil_iovar_data_set(ifp, "clmload", dload_buf,
+	err = brcmf_fil_iovar_data_set(ifp, var, dload_buf,
 				       struct_size(dload_buf, data, len));
 
 	return err;
 }
 
-static int brcmf_c_process_clm_blob(struct brcmf_if *ifp)
+static int brcmf_c_download_blob(struct brcmf_if *ifp,
+				 const void *data, size_t size,
+				 const char *loadvar, const char *statvar)
 {
 	struct brcmf_pub *drvr = ifp->drvr;
-	struct brcmf_bus *bus = drvr->bus_if;
 	struct brcmf_dload_data_le *chunk_buf;
-	const struct firmware *clm = NULL;
 	u32 chunk_len;
 	u32 datalen;
 	u32 cumulative_len;
@@ -132,21 +132,14 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp)
 
 	brcmf_dbg(TRACE, "Enter\n");
 
-	err = brcmf_bus_get_blob(bus, &clm, BRCMF_BLOB_CLM);
-	if (err || !clm) {
-		brcmf_info("no clm_blob available (err=%d), device may have limited channels available\n",
-			   err);
-		return 0;
-	}
-
 	chunk_buf = kzalloc(struct_size(chunk_buf, data, MAX_CHUNK_LEN),
 			    GFP_KERNEL);
 	if (!chunk_buf) {
 		err = -ENOMEM;
-		goto done;
+		return -ENOMEM;
 	}
 
-	datalen = clm->size;
+	datalen = size;
 	cumulative_len = 0;
 	do {
 		if (datalen > MAX_CHUNK_LEN) {
@@ -155,9 +148,10 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp)
 			chunk_len = datalen;
 			dl_flag |= DL_END;
 		}
-		memcpy(chunk_buf->data, clm->data + cumulative_len, chunk_len);
+		memcpy(chunk_buf->data, data + cumulative_len, chunk_len);
 
-		err = brcmf_c_download(ifp, dl_flag, chunk_buf, chunk_len);
+		err = brcmf_c_download(ifp, dl_flag, chunk_buf, chunk_len,
+				       loadvar);
 
 		dl_flag &= ~DL_BEGIN;
 
@@ -166,20 +160,64 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp)
 	} while ((datalen > 0) && (err == 0));
 
 	if (err) {
-		bphy_err(drvr, "clmload (%zu byte file) failed (%d)\n",
-			 clm->size, err);
-		/* Retrieve clmload_status and print */
-		err = brcmf_fil_iovar_int_get(ifp, "clmload_status", &status);
+		bphy_err(drvr, "%s (%zu byte file) failed (%d)\n",
+			 loadvar, size, err);
+		/* Retrieve status and print */
+		err = brcmf_fil_iovar_int_get(ifp, statvar, &status);
 		if (err)
-			bphy_err(drvr, "get clmload_status failed (%d)\n", err);
+			bphy_err(drvr, "get %s failed (%d)\n", statvar, err);
 		else
-			brcmf_dbg(INFO, "clmload_status=%d\n", status);
+			brcmf_dbg(INFO, "%s=%d\n", statvar, status);
 		err = -EIO;
 	}
 
 	kfree(chunk_buf);
-done:
-	release_firmware(clm);
+	return err;
+}
+
+static int brcmf_c_process_clm_blob(struct brcmf_if *ifp)
+{
+	struct brcmf_pub *drvr = ifp->drvr;
+	struct brcmf_bus *bus = drvr->bus_if;
+	const struct firmware *fw = NULL;
+	s32 err;
+
+	brcmf_dbg(TRACE, "Enter\n");
+
+	err = brcmf_bus_get_blob(bus, &fw, BRCMF_BLOB_CLM);
+	if (err || !fw) {
+		brcmf_info("no clm_blob available (err=%d), device may have limited channels available\n",
+			   err);
+		return 0;
+	}
+
+	err = brcmf_c_download_blob(ifp, fw->data, fw->size,
+				    "clmload", "clmload_status");
+
+	release_firmware(fw);
+	return err;
+}
+
+static int brcmf_c_process_txcap_blob(struct brcmf_if *ifp)
+{
+	struct brcmf_pub *drvr = ifp->drvr;
+	struct brcmf_bus *bus = drvr->bus_if;
+	const struct firmware *fw = NULL;
+	s32 err;
+
+	brcmf_dbg(TRACE, "Enter\n");
+
+	err = brcmf_bus_get_blob(bus, &fw, BRCMF_BLOB_TXCAP);
+	if (err || !fw) {
+		brcmf_info("no txcap_blob available (err=%d)\n", err);
+		return 0;
+	}
+
+	brcmf_info("TxCap blob found, loading\n");
+	err = brcmf_c_download_blob(ifp, fw->data, fw->size,
+				    "txcapload", "txcapload_status");
+
+	release_firmware(fw);
 	return err;
 }
 
@@ -208,6 +246,23 @@ static const u8 brcmf_default_mac_address[ETH_ALEN] = {
 	0x00, 0x90, 0x4c, 0xc5, 0x12, 0x38
 };
 
+static int brcmf_c_process_cal_blob(struct brcmf_if *ifp)
+{
+	struct brcmf_pub *drvr = ifp->drvr;
+	struct brcmf_mp_device *settings = drvr->settings;
+	s32 err;
+
+	brcmf_dbg(TRACE, "Enter\n");
+
+	if (!settings->cal_blob || !settings->cal_size)
+		return 0;
+
+	brcmf_info("Calibration blob provided by platform, loading\n");
+	err = brcmf_c_download_blob(ifp, settings->cal_blob, settings->cal_size,
+				    "calload", "calload_status");
+	return err;
+}
+
 int brcmf_c_preinit_dcmds(struct brcmf_if *ifp)
 {
 	struct brcmf_pub *drvr = ifp->drvr;
@@ -291,6 +346,20 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp)
 		goto done;
 	}
 
+	/* Do TxCap downloading, if needed */
+	err = brcmf_c_process_txcap_blob(ifp);
+	if (err < 0) {
+		bphy_err(drvr, "download TxCap blob file failed, %d\n", err);
+		goto done;
+	}
+
+	/* Download external calibration blob, if available */
+	err = brcmf_c_process_cal_blob(ifp);
+	if (err < 0) {
+		bphy_err(drvr, "download calibration blob file failed, %d\n", err);
+		goto done;
+	}
+
 	/* query for 'ver' to get version info from firmware */
 	memset(buf, 0, sizeof(buf));
 	err = brcmf_fil_iovar_data_get(ifp, "ver", buf, sizeof(buf));
@@ -487,6 +556,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev,
 		/* No platform data for this device, try OF and DMI data */
 		brcmf_dmi_probe(settings, chip, chiprev);
 		brcmf_of_probe(dev, bus_type, settings);
+		brcmf_acpi_probe(dev, bus_type, settings);
 	}
 	return settings;
 }
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
index aa25abffcc7d..2be2986d2110 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
@@ -54,6 +54,8 @@ struct brcmf_mp_device {
 	const char	*board_type;
 	unsigned char	mac[ETH_ALEN];
 	const char	*antenna_sku;
+	const void	*cal_blob;
+	int		cal_size;
 	union {
 		struct brcmfmac_sdio_pd sdio;
 	} bus;
@@ -77,6 +79,15 @@ static inline void
 brcmf_dmi_probe(struct brcmf_mp_device *settings, u32 chip, u32 chiprev) {}
 #endif
 
+#ifdef CONFIG_ACPI
+void brcmf_acpi_probe(struct device *dev, enum brcmf_bus_type bus_type,
+		      struct brcmf_mp_device *settings);
+#else
+static inline void brcmf_acpi_probe(struct device *dev,
+				    enum brcmf_bus_type bus_type,
+				    struct brcmf_mp_device *settings) {}
+#endif
+
 u8 brcmf_map_prio_to_prec(void *cfg, u8 prio);
 
 u8 brcmf_map_prio_to_aci(void *cfg, u8 prio);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
index 10bac865d724..6d10c9efbe93 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
@@ -126,6 +126,53 @@ static void brcmf_feat_firmware_overrides(struct brcmf_pub *drv)
 	drv->feat_flags |= feat_flags;
 }
 
+struct brcmf_feat_wlcfeat {
+	u16 min_ver_major;
+	u16 min_ver_minor;
+	u32 feat_flags;
+};
+
+static const struct brcmf_feat_wlcfeat brcmf_feat_wlcfeat_map[] = {
+	{ 12, 0, BIT(BRCMF_FEAT_PMKID_V2) },
+	{ 13, 0, BIT(BRCMF_FEAT_PMKID_V3) },
+};
+
+static void brcmf_feat_wlc_version_overrides(struct brcmf_pub *drv)
+{
+	struct brcmf_if *ifp = brcmf_get_ifp(drv, 0);
+	const struct brcmf_feat_wlcfeat *e;
+	struct brcmf_wlc_version_le ver;
+	u32 feat_flags = 0;
+	int i, err, major, minor;
+
+	err = brcmf_fil_iovar_data_get(ifp, "wlc_ver", &ver, sizeof(ver));
+	if (err)
+		return;
+
+	major = le16_to_cpu(ver.wlc_ver_major);
+	minor = le16_to_cpu(ver.wlc_ver_minor);
+
+	brcmf_dbg(INFO, "WLC version: %d.%d\n", major, minor);
+
+	for (i = 0; i < ARRAY_SIZE(brcmf_feat_wlcfeat_map); i++) {
+		e = &brcmf_feat_wlcfeat_map[i];
+		if (major > e->min_ver_major ||
+		    (major == e->min_ver_major &&
+		     minor >= e->min_ver_minor)) {
+			feat_flags |= e->feat_flags;
+		}
+	}
+
+	if (!feat_flags)
+		return;
+
+	for (i = 0; i < BRCMF_FEAT_LAST; i++)
+		if (feat_flags & BIT(i))
+			brcmf_dbg(INFO, "enabling firmware feature: %s\n",
+				  brcmf_feat_names[i]);
+	drv->feat_flags |= feat_flags;
+}
+
 /**
  * brcmf_feat_iovar_int_get() - determine feature through iovar query.
  *
@@ -290,6 +337,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr)
 		ifp->drvr->feat_flags |= BIT(BRCMF_FEAT_SCAN_RANDOM_MAC);
 
 	brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_FWSUP, "sup_wpa");
+	brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_SCAN_V2, "scan_ver");
 
 	if (drvr->settings->feature_disable) {
 		brcmf_dbg(INFO, "Features: 0x%02x, disable: 0x%02x\n",
@@ -298,6 +346,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr)
 		ifp->drvr->feat_flags &= ~drvr->settings->feature_disable;
 	}
 
+	brcmf_feat_wlc_version_overrides(drvr);
 	brcmf_feat_firmware_overrides(drvr);
 
 	/* set chip related quirks */
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h
index f1b086a69d73..7f4f0b3e4a7b 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h
@@ -30,6 +30,7 @@
  * SAE: simultaneous authentication of equals
  * FWAUTH: Firmware authenticator
  * DUMP_OBSS: Firmware has capable to dump obss info to support ACS
+ * SCAN_V2: Version 2 scan params
  */
 #define BRCMF_FEAT_LIST \
 	BRCMF_FEAT_DEF(MBSS) \
@@ -53,7 +54,10 @@
 	BRCMF_FEAT_DEF(DOT11H) \
 	BRCMF_FEAT_DEF(SAE) \
 	BRCMF_FEAT_DEF(FWAUTH) \
-	BRCMF_FEAT_DEF(DUMP_OBSS)
+	BRCMF_FEAT_DEF(DUMP_OBSS) \
+	BRCMF_FEAT_DEF(SCAN_V2) \
+	BRCMF_FEAT_DEF(PMKID_V2) \
+	BRCMF_FEAT_DEF(PMKID_V3)
 
 /*
  * Quirks:
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h
index 04e1beedfd81..792adaf880b4 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h
@@ -48,6 +48,10 @@
 
 /* size of brcmf_scan_params not including variable length array */
 #define BRCMF_SCAN_PARAMS_FIXED_SIZE	64
+#define BRCMF_SCAN_PARAMS_V2_FIXED_SIZE	72
+
+/* version of brcmf_scan_params structure */
+#define BRCMF_SCAN_PARAMS_VERSION_V2	2
 
 /* masks for channel and ssid count */
 #define BRCMF_SCAN_PARAMS_COUNT_MASK	0x0000ffff
@@ -67,6 +71,7 @@
 #define BRCMF_PRIMARY_KEY		(1 << 1)
 #define DOT11_BSSTYPE_ANY		2
 #define BRCMF_ESCAN_REQ_VERSION		1
+#define BRCMF_ESCAN_REQ_VERSION_V2	2
 
 #define BRCMF_MAXRATES_IN_SET		16	/* max # of rates in rateset */
 
@@ -169,6 +174,10 @@
 
 #define BRCMF_HE_CAP_MCS_MAP_NSS_MAX	8
 
+#define BRCMF_PMKSA_VER_2		2
+#define BRCMF_PMKSA_VER_3		3
+#define BRCMF_PMKSA_NO_EXPIRY		0xffffffff
+
 /* MAX_CHUNK_LEN is the maximum length for data passing to firmware in each
  * ioctl. It is relatively small because firmware has small maximum size input
  * playload restriction for ioctls.
@@ -350,6 +359,12 @@ struct brcmf_ssid_le {
 	unsigned char SSID[IEEE80211_MAX_SSID_LEN];
 };
 
+/* Alternate SSID structure used in some places... */
+struct brcmf_ssid8_le {
+	u8 SSID_len;
+	unsigned char SSID[IEEE80211_MAX_SSID_LEN];
+};
+
 struct brcmf_scan_params_le {
 	struct brcmf_ssid_le ssid_le;	/* default: {0, ""} */
 	u8 bssid[ETH_ALEN];	/* default: bcast */
@@ -386,6 +401,45 @@ struct brcmf_scan_params_le {
 	__le16 channel_list[1];	/* list of chanspecs */
 };
 
+struct brcmf_scan_params_v2_le {
+	__le16 version;		/* structure version */
+	__le16 length;		/* structure length */
+	struct brcmf_ssid_le ssid_le;	/* default: {0, ""} */
+	u8 bssid[ETH_ALEN];	/* default: bcast */
+	s8 bss_type;		/* default: any,
+				 * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT
+				 */
+	u8 pad;
+	__le32 scan_type;	/* flags, 0 use default */
+	__le32 nprobes;		/* -1 use default, number of probes per channel */
+	__le32 active_time;	/* -1 use default, dwell time per channel for
+				 * active scanning
+				 */
+	__le32 passive_time;	/* -1 use default, dwell time per channel
+				 * for passive scanning
+				 */
+	__le32 home_time;	/* -1 use default, dwell time for the
+				 * home channel between channel scans
+				 */
+	__le32 channel_num;	/* count of channels and ssids that follow
+				 *
+				 * low half is count of channels in
+				 * channel_list, 0 means default (use all
+				 * available channels)
+				 *
+				 * high half is entries in struct brcmf_ssid
+				 * array that follows channel_list, aligned for
+				 * s32 (4 bytes) meaning an odd channel count
+				 * implies a 2-byte pad between end of
+				 * channel_list and first ssid
+				 *
+				 * if ssid count is zero, single ssid in the
+				 * fixed parameter portion is assumed, otherwise
+				 * ssid in the fixed portion is ignored
+				 */
+	__le16 channel_list[1];	/* list of chanspecs */
+};
+
 struct brcmf_scan_results {
 	u32 buflen;
 	u32 version;
@@ -397,7 +451,10 @@ struct brcmf_escan_params_le {
 	__le32 version;
 	__le16 action;
 	__le16 sync_id;
-	struct brcmf_scan_params_le params_le;
+	union {
+		struct brcmf_scan_params_le params_le;
+		struct brcmf_scan_params_v2_le params_v2_le;
+	};
 };
 
 struct brcmf_escan_result_le {
@@ -742,6 +799,31 @@ struct brcmf_rev_info_le {
 };
 
 /**
+ * struct brcmf_wlc_version_le - firmware revision info.
+ *
+ * @version: structure version.
+ * @length: structure length.
+ * @epi_ver_major: EPI major version
+ * @epi_ver_minor: EPI minor version
+ * @epi_ver_rc: EPI rc version
+ * @epi_ver_incr: EPI increment version
+ * @wlc_ver_major: WLC major version
+ * @wlc_ver_minor: WLC minor version
+ */
+struct brcmf_wlc_version_le {
+	__le16 version;
+	__le16 length;
+
+	__le16 epi_ver_major;
+	__le16 epi_ver_minor;
+	__le16 epi_ver_rc;
+	__le16 epi_ver_incr;
+
+	__le16 wlc_ver_major;
+	__le16 wlc_ver_minor;
+};
+
+/**
  * struct brcmf_assoclist_le - request assoc list.
  *
  * @count: indicates number of stations.
@@ -804,6 +886,51 @@ struct brcmf_pmksa {
 };
 
 /**
+ * struct brcmf_pmksa_v2 - PMK Security Association
+ *
+ * @length: Length of the structure.
+ * @bssid: The AP's BSSID.
+ * @pmkid: The PMK ID.
+ * @pmk: PMK material for FILS key derivation.
+ * @pmk_len: Length of PMK data.
+ * @ssid: The AP's SSID.
+ * @fils_cache_id: FILS cache identifier
+ */
+struct brcmf_pmksa_v2 {
+	__le16 length;
+	u8 bssid[ETH_ALEN];
+	u8 pmkid[WLAN_PMKID_LEN];
+	u8 pmk[WLAN_PMK_LEN_SUITE_B_192];
+	__le16 pmk_len;
+	struct brcmf_ssid8_le ssid;
+	u16 fils_cache_id;
+};
+
+/**
+ * struct brcmf_pmksa_v3 - PMK Security Association
+ *
+ * @bssid: The AP's BSSID.
+ * @pmkid: The PMK ID.
+ * @pmkid_len: The length of the PMK ID.
+ * @pmk: PMK material for FILS key derivation.
+ * @pmk_len: Length of PMK data.
+ * @fils_cache_id: FILS cache identifier
+ * @ssid: The AP's SSID.
+ * @time_left: Remaining time until expiry. 0 = expired, ~0 = no expiry.
+ */
+struct brcmf_pmksa_v3 {
+	u8 bssid[ETH_ALEN];
+	u8 pmkid[WLAN_PMKID_LEN];
+	u8 pmkid_len;
+	u8 pmk[WLAN_PMK_LEN_SUITE_B_192];
+	u8 pmk_len;
+	__le16 fils_cache_id;
+	u8 pad;
+	struct brcmf_ssid8_le ssid;
+	__le32 time_left;
+};
+
+/**
  * struct brcmf_pmk_list_le - List of pmksa's.
  *
  * @npmk: Number of pmksa's.
@@ -815,6 +942,34 @@ struct brcmf_pmk_list_le {
 };
 
 /**
+ * struct brcmf_pmk_list_v2_le - List of pmksa's.
+ *
+ * @version: Request version.
+ * @length: Length of this structure.
+ * @pmk: PMK SA information.
+ */
+struct brcmf_pmk_list_v2_le {
+	__le16 version;
+	__le16 length;
+	struct brcmf_pmksa_v2 pmk[BRCMF_MAXPMKID];
+};
+
+/**
+ * struct brcmf_pmk_op_v3_le - Operation on PMKSA list.
+ *
+ * @version: Request version.
+ * @length: Length of this structure.
+ * @pmk: PMK SA information.
+ */
+struct brcmf_pmk_op_v3_le {
+	__le16 version;
+	__le16 length;
+	__le16 count;
+	__le16 pad;
+	struct brcmf_pmksa_v3 pmk[BRCMF_MAXPMKID];
+};
+
+/**
  * struct brcmf_pno_param_le - PNO scan configuration parameters
  *
  * @version: PNO parameters version.
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c
index fdd0c9abc1a1..e406e11481a6 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c
@@ -86,6 +86,13 @@ void brcmf_of_probe(struct device *dev, enum brcmf_bus_type bus_type,
 	if (!of_property_read_string(np, "apple,antenna-sku", &prop))
 		settings->antenna_sku = prop;
 
+	/* The WLAN calibration blob is normally stored in SROM, but Apple
+	 * ARM64 platforms pass it via the DT instead.
+	 */
+	prop = of_get_property(np, "brcm,cal-blob", &settings->cal_size);
+	if (prop && settings->cal_size)
+		settings->cal_blob = prop;
+
 	/* Set board-type to the first string of the machine compatible prop */
 	root = of_find_node_by_path("/");
 	if (root && err) {
@@ -122,7 +129,7 @@ void brcmf_of_probe(struct device *dev, enum brcmf_bus_type bus_type,
 		sdio->drive_strength = val;
 
 	/* make sure there are interrupts defined in the node */
-	if (!of_find_property(np, "interrupts", NULL))
+	if (!of_property_present(np, "interrupts"))
 		return;
 
 	irq = irq_of_parse_and_map(np, 0);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
index a9b9b2dc62d4..59f3e9c5e139 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
@@ -15,6 +15,7 @@
 #include <linux/sched/signal.h>
 #include <linux/kthread.h>
 #include <linux/io.h>
+#include <linux/random.h>
 #include <asm/unaligned.h>
 
 #include <soc.h>
@@ -57,6 +58,7 @@ BRCMF_FW_CLM_DEF(4356, "brcmfmac4356-pcie");
 BRCMF_FW_CLM_DEF(43570, "brcmfmac43570-pcie");
 BRCMF_FW_DEF(4358, "brcmfmac4358-pcie");
 BRCMF_FW_DEF(4359, "brcmfmac4359-pcie");
+BRCMF_FW_DEF(4359C, "brcmfmac4359c-pcie");
 BRCMF_FW_CLM_DEF(4364B2, "brcmfmac4364b2-pcie");
 BRCMF_FW_CLM_DEF(4364B3, "brcmfmac4364b3-pcie");
 BRCMF_FW_DEF(4365B, "brcmfmac4365b-pcie");
@@ -66,6 +68,8 @@ BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie");
 BRCMF_FW_DEF(4371, "brcmfmac4371-pcie");
 BRCMF_FW_CLM_DEF(4377B3, "brcmfmac4377b3-pcie");
 BRCMF_FW_CLM_DEF(4378B1, "brcmfmac4378b1-pcie");
+BRCMF_FW_CLM_DEF(4378B3, "brcmfmac4378b3-pcie");
+BRCMF_FW_CLM_DEF(4387C2, "brcmfmac4387c2-pcie");
 
 /* firmware config files */
 MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.txt");
@@ -74,6 +78,7 @@ MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.txt");
 /* per-board firmware binaries */
 MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.bin");
 MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.clm_blob");
+MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.txcap_blob");
 
 static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602),
@@ -88,7 +93,8 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43569_CHIP_ID, 0xFFFFFFFF, 43570),
 	BRCMF_FW_ENTRY(BRCM_CC_43570_CHIP_ID, 0xFFFFFFFF, 43570),
 	BRCMF_FW_ENTRY(BRCM_CC_4358_CHIP_ID, 0xFFFFFFFF, 4358),
-	BRCMF_FW_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359),
+	BRCMF_FW_ENTRY(BRCM_CC_4359_CHIP_ID, 0x000001FF, 4359),
+	BRCMF_FW_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFE00, 4359C),
 	BRCMF_FW_ENTRY(BRCM_CC_4364_CHIP_ID, 0x0000000F, 4364B2), /* 3 */
 	BRCMF_FW_ENTRY(BRCM_CC_4364_CHIP_ID, 0xFFFFFFF0, 4364B3), /* 4 */
 	BRCMF_FW_ENTRY(BRCM_CC_4365_CHIP_ID, 0x0000000F, 4365B),
@@ -99,7 +105,9 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43666_CHIP_ID, 0xFFFFFFF0, 4366C),
 	BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371),
 	BRCMF_FW_ENTRY(BRCM_CC_4377_CHIP_ID, 0xFFFFFFFF, 4377B3), /* revision ID 4 */
-	BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0xFFFFFFFF, 4378B1), /* revision ID 3 */
+	BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0x0000000F, 4378B1), /* revision ID 3 */
+	BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0xFFFFFFE0, 4378B3), /* revision ID 5 */
+	BRCMF_FW_ENTRY(BRCM_CC_4387_CHIP_ID, 0xFFFFFFFF, 4387C2), /* revision ID 7 */
 };
 
 #define BRCMF_PCIE_FW_UP_TIMEOUT		5000 /* msec */
@@ -326,7 +334,9 @@ struct brcmf_pciedev_info {
 	char fw_name[BRCMF_FW_NAME_LEN];
 	char nvram_name[BRCMF_FW_NAME_LEN];
 	char clm_name[BRCMF_FW_NAME_LEN];
+	char txcap_name[BRCMF_FW_NAME_LEN];
 	const struct firmware *clm_fw;
+	const struct firmware *txcap_fw;
 	const struct brcmf_pcie_reginfo *reginfo;
 	void __iomem *regs;
 	void __iomem *tcm;
@@ -1517,6 +1527,10 @@ static int brcmf_pcie_get_blob(struct device *dev, const struct firmware **fw,
 		*fw = devinfo->clm_fw;
 		devinfo->clm_fw = NULL;
 		break;
+	case BRCMF_BLOB_TXCAP:
+		*fw = devinfo->txcap_fw;
+		devinfo->txcap_fw = NULL;
+		break;
 	default:
 		return -ENOENT;
 	}
@@ -1653,6 +1667,13 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo,
 	return 0;
 }
 
+struct brcmf_random_seed_footer {
+	__le32 length;
+	__le32 magic;
+};
+
+#define BRCMF_RANDOM_SEED_MAGIC		0xfeedc0de
+#define BRCMF_RANDOM_SEED_LENGTH	0x100
 
 static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo,
 					const struct firmware *fw, void *nvram,
@@ -1689,6 +1710,30 @@ static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo,
 			  nvram_len;
 		memcpy_toio(devinfo->tcm + address, nvram, nvram_len);
 		brcmf_fw_nvram_free(nvram);
+
+		if (devinfo->otp.valid) {
+			size_t rand_len = BRCMF_RANDOM_SEED_LENGTH;
+			struct brcmf_random_seed_footer footer = {
+				.length = cpu_to_le32(rand_len),
+				.magic = cpu_to_le32(BRCMF_RANDOM_SEED_MAGIC),
+			};
+			void *randbuf;
+
+			/* Some Apple chips/firmwares expect a buffer of random
+			 * data to be present before NVRAM
+			 */
+			brcmf_dbg(PCIE, "Download random seed\n");
+
+			address -= sizeof(footer);
+			memcpy_toio(devinfo->tcm + address, &footer,
+				    sizeof(footer));
+
+			address -= rand_len;
+			randbuf = kzalloc(rand_len, GFP_KERNEL);
+			get_random_bytes(randbuf, rand_len);
+			memcpy_toio(devinfo->tcm + address, randbuf, rand_len);
+			kfree(randbuf);
+		}
 	} else {
 		brcmf_dbg(PCIE, "No matching NVRAM file found %s\n",
 			  devinfo->nvram_name);
@@ -2016,6 +2061,11 @@ static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo)
 		base = 0x1120;
 		words = 0x170;
 		break;
+	case BRCM_CC_4387_CHIP_ID:
+		coreid = BCMA_CORE_GCI;
+		base = 0x113c;
+		words = 0x170;
+		break;
 	default:
 		/* OTP not supported on this chip */
 		return 0;
@@ -2073,6 +2123,7 @@ static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo)
 #define BRCMF_PCIE_FW_CODE	0
 #define BRCMF_PCIE_FW_NVRAM	1
 #define BRCMF_PCIE_FW_CLM	2
+#define BRCMF_PCIE_FW_TXCAP	3
 
 static void brcmf_pcie_setup(struct device *dev, int ret,
 			     struct brcmf_fw_request *fwreq)
@@ -2099,6 +2150,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret,
 	nvram = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.data;
 	nvram_len = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.len;
 	devinfo->clm_fw = fwreq->items[BRCMF_PCIE_FW_CLM].binary;
+	devinfo->txcap_fw = fwreq->items[BRCMF_PCIE_FW_TXCAP].binary;
 	kfree(fwreq);
 
 	ret = brcmf_chip_get_raminfo(devinfo->ci);
@@ -2180,6 +2232,7 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo)
 		{ ".bin", devinfo->fw_name },
 		{ ".txt", devinfo->nvram_name },
 		{ ".clm_blob", devinfo->clm_name },
+		{ ".txcap_blob", devinfo->txcap_name },
 	};
 
 	fwreq = brcmf_fw_alloc_request(devinfo->ci->chip, devinfo->ci->chiprev,
@@ -2194,6 +2247,8 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo)
 	fwreq->items[BRCMF_PCIE_FW_NVRAM].flags = BRCMF_FW_REQF_OPTIONAL;
 	fwreq->items[BRCMF_PCIE_FW_CLM].type = BRCMF_FW_TYPE_BINARY;
 	fwreq->items[BRCMF_PCIE_FW_CLM].flags = BRCMF_FW_REQF_OPTIONAL;
+	fwreq->items[BRCMF_PCIE_FW_TXCAP].type = BRCMF_FW_TYPE_BINARY;
+	fwreq->items[BRCMF_PCIE_FW_TXCAP].flags = BRCMF_FW_REQF_OPTIONAL;
 	/* NVRAM reserves PCI domain 0 for Broadcom's SDK faked bus */
 	fwreq->domain_nr = pci_domain_nr(devinfo->pdev->bus) + 1;
 	fwreq->bus_nr = devinfo->pdev->bus->number;
@@ -2491,6 +2546,7 @@ brcmf_pcie_remove(struct pci_dev *pdev)
 	brcmf_pcie_reset_device(devinfo);
 	brcmf_pcie_release_resource(devinfo);
 	release_firmware(devinfo->clm_fw);
+	release_firmware(devinfo->txcap_fw);
 
 	if (devinfo->ci)
 		brcmf_chip_detach(devinfo->ci);
@@ -2630,6 +2686,7 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = {
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_43596_DEVICE_ID, CYW),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4377_DEVICE_ID, WCC),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4378_DEVICE_ID, WCC),
+	BRCMF_PCIE_DEVICE(BRCM_PCIE_4387_DEVICE_ID, WCC),
 
 	{ /* end: all zeroes */ }
 };
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/ampdu.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/ampdu.c
index 2631eb7569eb..e24228e60027 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/ampdu.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/ampdu.c
@@ -845,7 +845,7 @@ brcms_c_ampdu_dotxstatus_complete(struct ampdu_info *ampdu, struct scb *scb,
 	u16 seq, start_seq = 0, bindex, index, mcl;
 	u8 mcs = 0;
 	bool ba_recd = false, ack_recd = false;
-	u8 suc_mpdu = 0, tot_mpdu = 0;
+	u8 tot_mpdu = 0;
 	uint supr_status;
 	bool retry = true;
 	u16 mimoantsel = 0;
@@ -975,7 +975,6 @@ brcms_c_ampdu_dotxstatus_complete(struct ampdu_info *ampdu, struct scb *scb,
 				ieee80211_tx_status_irqsafe(wlc->pub->ieee_hw,
 							    p);
 				ack_recd = true;
-				suc_mpdu++;
 			}
 		}
 		/* either retransmit or send bar if ack not recd */
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/led.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/led.c
index 9540a05247c2..89c8829528c2 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/led.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/led.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <net/mac80211.h>
 #include <linux/bcma/bcma_driver_chipcommon.h>
+#include <linux/gpio.h>
 #include <linux/gpio/driver.h>
 #include <linux/gpio/machine.h>
 #include <linux/gpio/consumer.h>
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c
index a8333e6adbda..0bd4e679a359 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c
@@ -1048,7 +1048,6 @@ static int ieee_hw_rate_init(struct ieee80211_hw *hw)
 	struct brcms_info *wl = hw->priv;
 	struct brcms_c_info *wlc = wl->wlc;
 	struct ieee80211_supported_band *band;
-	int has_5g = 0;
 	u16 phy_type;
 
 	hw->wiphy->bands[NL80211_BAND_2GHZ] = NULL;
@@ -1070,7 +1069,6 @@ static int ieee_hw_rate_init(struct ieee80211_hw *hw)
 
 	/* Assume all bands use the same phy.  True for 11n devices. */
 	if (wl->pub->_nbands > 1) {
-		has_5g++;
 		if (phy_type == PHY_TYPE_N || phy_type == PHY_TYPE_LCN) {
 			band = &wlc->bandstate[BAND_5G_INDEX]->band;
 			*band = brcms_band_5GHz_nphy_template;
diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
index 896615f57952..44684bf1b9ac 100644
--- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
+++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
@@ -54,6 +54,7 @@
 #define BRCM_CC_4371_CHIP_ID		0x4371
 #define BRCM_CC_4377_CHIP_ID		0x4377
 #define BRCM_CC_4378_CHIP_ID		0x4378
+#define BRCM_CC_4387_CHIP_ID		0x4387
 #define CY_CC_4373_CHIP_ID		0x4373
 #define CY_CC_43012_CHIP_ID		43012
 #define CY_CC_43439_CHIP_ID		43439
@@ -95,6 +96,7 @@
 #define BRCM_PCIE_43596_DEVICE_ID	0x4415
 #define BRCM_PCIE_4377_DEVICE_ID	0x4488
 #define BRCM_PCIE_4378_DEVICE_ID	0x4425
+#define BRCM_PCIE_4387_DEVICE_ID	0x4433
 
 /* brcmsmac IDs */
 #define BCM4313_D11N2G_ID	0x4727	/* 4313 802.11n 2.4G device */
diff --git a/drivers/net/wireless/cisco/Kconfig b/drivers/net/wireless/cisco/Kconfig
index 681bfc2d740a..b40ee25aca99 100644
--- a/drivers/net/wireless/cisco/Kconfig
+++ b/drivers/net/wireless/cisco/Kconfig
@@ -14,7 +14,7 @@ if WLAN_VENDOR_CISCO
 
 config AIRO
 	tristate "Cisco/Aironet 34X/35X/4500/4800 ISA and PCI cards"
-	depends on CFG80211 && ISA_DMA_API && (PCI || BROKEN)
+	depends on CFG80211 && (PCI || BROKEN)
 	select WIRELESS_EXT
 	select CRYPTO
 	select CRYPTO_SKCIPHER
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.c b/drivers/net/wireless/intel/ipw2x00/ipw2200.c
index d382f2017325..dfe0f74369e6 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2200.c
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.c
@@ -377,19 +377,6 @@ static inline u8 _ipw_read8(struct ipw_priv *ipw, unsigned long ofs)
 	_ipw_read8(ipw, ofs); \
 })
 
-/* 16-bit direct read (low 4K) */
-static inline u16 _ipw_read16(struct ipw_priv *ipw, unsigned long ofs)
-{
-	return readw(ipw->hw_base + ofs);
-}
-
-/* alias to 16-bit direct read (low 4K of SRAM/regs), with debug wrapper */
-#define ipw_read16(ipw, ofs) ({ \
-	IPW_DEBUG_IO("%s %d: read_direct16(0x%08X)\n", __FILE__, __LINE__, \
-			(u32)(ofs)); \
-	_ipw_read16(ipw, ofs); \
-})
-
 /* 32-bit direct read (low 4K) */
 static inline u32 _ipw_read32(struct ipw_priv *ipw, unsigned long ofs)
 {
@@ -1234,9 +1221,9 @@ static struct ipw_fw_error *ipw_alloc_error_log(struct ipw_priv *priv)
 	u32 base = ipw_read32(priv, IPW_ERROR_LOG);
 	u32 elem_len = ipw_read_reg32(priv, base);
 
-	error = kmalloc(sizeof(*error) +
-			sizeof(*error->elem) * elem_len +
-			sizeof(*error->log) * log_len, GFP_ATOMIC);
+	error = kmalloc(size_add(struct_size(error, elem, elem_len),
+				 array_size(sizeof(*error->log), log_len)),
+			GFP_ATOMIC);
 	if (!error) {
 		IPW_ERROR("Memory allocation for firmware error log "
 			  "failed.\n");
@@ -1247,7 +1234,6 @@ static struct ipw_fw_error *ipw_alloc_error_log(struct ipw_priv *priv)
 	error->config = priv->config;
 	error->elem_len = elem_len;
 	error->log_len = log_len;
-	error->elem = (struct ipw_error_elem *)error->payload;
 	error->log = (struct ipw_event *)(error->elem + elem_len);
 
 	ipw_capture_event_log(priv, log_len, error->log);
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.h b/drivers/net/wireless/intel/ipw2x00/ipw2200.h
index 09ddd21608d4..8ebf09121e17 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2200.h
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.h
@@ -1106,9 +1106,8 @@ struct ipw_fw_error {	 /* XXX */
 	u32 config;
 	u32 elem_len;
 	u32 log_len;
-	struct ipw_error_elem *elem;
 	struct ipw_event *log;
-	u8 payload[];
+	struct ipw_error_elem elem[];
 } __packed;
 
 #ifdef CONFIG_IPW2200_PROMISCUOUS
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
index 3bdd6774716d..b6f82510e980 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
@@ -10,7 +10,7 @@
 #include "fw/api/txq.h"
 
 /* Highest firmware API version supported */
-#define IWL_22000_UCODE_API_MAX	74
+#define IWL_22000_UCODE_API_MAX	78
 
 /* Lowest firmware API version supported */
 #define IWL_22000_UCODE_API_MIN	39
@@ -50,21 +50,35 @@
 #define IWL_MA_A_GF4_A_FW_PRE		"iwlwifi-ma-a0-gf4-a0-"
 #define IWL_MA_A_MR_A_FW_PRE		"iwlwifi-ma-a0-mr-a0-"
 #define IWL_MA_A_FM_A_FW_PRE		"iwlwifi-ma-a0-fm-a0-"
+#define IWL_MA_B_HR_B_FW_PRE		"iwlwifi-ma-b0-hr-b0-"
+#define IWL_MA_B_GF_A_FW_PRE		"iwlwifi-ma-b0-gf-a0-"
+#define IWL_MA_B_GF4_A_FW_PRE		"iwlwifi-ma-b0-gf4-a0-"
+#define IWL_MA_B_MR_A_FW_PRE		"iwlwifi-ma-b0-mr-a0-"
+#define IWL_MA_B_FM_A_FW_PRE		"iwlwifi-ma-b0-fm-a0-"
 #define IWL_SNJ_A_MR_A_FW_PRE		"iwlwifi-SoSnj-a0-mr-a0-"
+#define IWL_BZ_A_HR_A_FW_PRE		"iwlwifi-bz-a0-hr-b0-"
 #define IWL_BZ_A_HR_B_FW_PRE		"iwlwifi-bz-a0-hr-b0-"
 #define IWL_BZ_A_GF_A_FW_PRE		"iwlwifi-bz-a0-gf-a0-"
 #define IWL_BZ_A_GF4_A_FW_PRE		"iwlwifi-bz-a0-gf4-a0-"
 #define IWL_BZ_A_MR_A_FW_PRE		"iwlwifi-bz-a0-mr-a0-"
 #define IWL_BZ_A_FM_A_FW_PRE		"iwlwifi-bz-a0-fm-a0-"
 #define IWL_BZ_A_FM4_A_FW_PRE		"iwlwifi-bz-a0-fm4-a0-"
+#define IWL_BZ_A_FM_B_FW_PRE		"iwlwifi-bz-a0-fm-b0-"
+#define IWL_BZ_A_FM4_B_FW_PRE		"iwlwifi-bz-a0-fm4-b0-"
 #define IWL_GL_A_FM_A_FW_PRE		"iwlwifi-gl-a0-fm-a0-"
 #define IWL_GL_B_FM_B_FW_PRE		"iwlwifi-gl-b0-fm-b0-"
 #define IWL_BZ_Z_GF_A_FW_PRE		"iwlwifi-bz-z0-gf-a0-"
 #define IWL_BNJ_A_FM_A_FW_PRE		"iwlwifi-BzBnj-a0-fm-a0-"
 #define IWL_BNJ_A_FM4_A_FW_PRE		"iwlwifi-BzBnj-a0-fm4-a0-"
+#define IWL_BNJ_B_FM4_B_FW_PRE		"iwlwifi-BzBnj-b0-fm4-b0-"
 #define IWL_BNJ_A_GF_A_FW_PRE		"iwlwifi-BzBnj-a0-gf-a0-"
+#define IWL_BNJ_B_GF_A_FW_PRE		"iwlwifi-BzBnj-b0-gf-a0-"
 #define IWL_BNJ_A_GF4_A_FW_PRE		"iwlwifi-BzBnj-a0-gf4-a0-"
+#define IWL_BNJ_B_GF4_A_FW_PRE		"iwlwifi-BzBnj-b0-gf4-a0-"
+#define IWL_BNJ_A_HR_A_FW_PRE		"iwlwifi-BzBnj-a0-hr-b0-"
 #define IWL_BNJ_A_HR_B_FW_PRE		"iwlwifi-BzBnj-a0-hr-b0-"
+#define IWL_BNJ_B_HR_A_FW_PRE		"iwlwifi-BzBnj-b0-hr-b0-"
+#define IWL_BNJ_B_HR_B_FW_PRE		"iwlwifi-BzBnj-b0-hr-b0-"
 #define IWL_BNJ_B_FM_B_FW_PRE		"iwlwifi-BzBnj-b0-fm-b0-"
 
 
@@ -110,8 +124,20 @@
 	IWL_MA_A_MR_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_MA_A_FM_A_FW_MODULE_FIRMWARE(api)		\
 	IWL_MA_A_FM_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_MA_B_HR_B_FW_MODULE_FIRMWARE(api)		\
+	IWL_MA_B_HR_B_FW_PRE __stringify(api) ".ucode"
+#define IWL_MA_B_GF_A_FW_MODULE_FIRMWARE(api)		\
+	IWL_MA_B_GF_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_MA_B_GF4_A_FW_MODULE_FIRMWARE(api)		\
+	IWL_MA_B_GF4_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_MA_B_MR_A_FW_MODULE_FIRMWARE(api) \
+	IWL_MA_B_MR_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_MA_B_FM_A_FW_MODULE_FIRMWARE(api)		\
+	IWL_MA_B_FM_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_SNJ_A_MR_A_MODULE_FIRMWARE(api) \
 	IWL_SNJ_A_MR_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_BZ_A_HR_A_MODULE_FIRMWARE(api) \
+	IWL_BZ_A_HR_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_BZ_A_HR_B_MODULE_FIRMWARE(api) \
 	IWL_BZ_A_HR_B_FW_PRE __stringify(api) ".ucode"
 #define IWL_BZ_A_GF_A_MODULE_FIRMWARE(api) \
@@ -121,23 +147,39 @@
 #define IWL_BZ_A_MR_A_MODULE_FIRMWARE(api) \
 	IWL_BZ_A_MR_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_BZ_A_FM_A_MODULE_FIRMWARE(api) \
-		IWL_BZ_A_FM_A_FW_PRE __stringify(api) ".ucode"
+	IWL_BZ_A_FM_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_BZ_A_FM4_A_MODULE_FIRMWARE(api) \
-		IWL_BZ_A_FM4_A_FW_PRE __stringify(api) ".ucode"
+	IWL_BZ_A_FM4_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_BZ_A_FM_B_MODULE_FIRMWARE(api) \
+	IWL_BZ_A_FM_B_FW_PRE __stringify(api) ".ucode"
+#define IWL_BZ_A_FM4_B_MODULE_FIRMWARE(api) \
+	IWL_BZ_A_FM4_B_FW_PRE __stringify(api) ".ucode"
 #define IWL_GL_A_FM_A_MODULE_FIRMWARE(api) \
-		IWL_GL_A_FM_A_FW_PRE __stringify(api) ".ucode"
+	IWL_GL_A_FM_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_GL_B_FM_B_MODULE_FIRMWARE(api) \
-		IWL_GL_B_FM_B_FW_PRE __stringify(api) ".ucode"
+	IWL_GL_B_FM_B_FW_PRE __stringify(api) ".ucode"
 #define IWL_BNJ_A_FM_A_MODULE_FIRMWARE(api) \
 	IWL_BNJ_A_FM_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_BNJ_A_FM4_A_MODULE_FIRMWARE(api) \
 	IWL_BNJ_A_FM4_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_BNJ_B_FM4_B_MODULE_FIRMWARE(api) \
+	IWL_BNJ_B_FM4_B_FW_PRE __stringify(api) ".ucode"
 #define IWL_BNJ_A_GF_A_MODULE_FIRMWARE(api) \
 	IWL_BNJ_A_GF_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_BNJ_B_GF_A_MODULE_FIRMWARE(api) \
+	IWL_BNJ_B_GF_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_BNJ_A_GF4_A_MODULE_FIRMWARE(api) \
 	IWL_BNJ_A_GF4_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_BNJ_B_GF4_A_MODULE_FIRMWARE(api) \
+	IWL_BNJ_B_GF4_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_BNJ_A_HR_A_MODULE_FIRMWARE(api) \
+	IWL_BNJ_A_HR_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_BNJ_A_HR_B_MODULE_FIRMWARE(api) \
 	IWL_BNJ_A_HR_B_FW_PRE __stringify(api) ".ucode"
+#define IWL_BNJ_B_HR_A_MODULE_FIRMWARE(api) \
+	IWL_BNJ_B_HR_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_BNJ_B_HR_B_MODULE_FIRMWARE(api) \
+	IWL_BNJ_B_HR_B_FW_PRE __stringify(api) ".ucode"
 #define IWL_BNJ_B_FM_B_MODULE_FIRMWARE(api) \
 	IWL_BNJ_B_FM_B_FW_PRE __stringify(api) ".ucode"
 
@@ -278,7 +320,7 @@ static const struct iwl_ht_params iwl_gl_a_ht_params = {
 	.trans.gen2 = true,						\
 	.nvm_type = IWL_NVM_EXT,					\
 	.dbgc_supported = true,						\
-	.min_umac_error_event_table = 0x400000,				\
+	.min_umac_error_event_table = 0xD0000,				\
 	.d3_debug_data_base_addr = 0x401000,				\
 	.d3_debug_data_length = 60 * 1024,				\
 	.mon_smem_regs = {						\
@@ -864,6 +906,41 @@ const struct iwl_cfg iwl_cfg_ma_a0_ms_a0 = {
 	.num_rbds = IWL_NUM_RBDS_AX210_HE,
 };
 
+const struct iwl_cfg iwl_cfg_ma_b0_fm_a0 = {
+	.fw_name_pre = IWL_MA_B_FM_A_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_AX210,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
+const struct iwl_cfg iwl_cfg_ma_b0_hr_b0 = {
+	.fw_name_pre = IWL_MA_B_HR_B_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_AX210,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
+const struct iwl_cfg iwl_cfg_ma_b0_gf_a0 = {
+	.fw_name_pre = IWL_MA_B_GF_A_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_AX210,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
+const struct iwl_cfg iwl_cfg_ma_b0_gf4_a0 = {
+	.fw_name_pre = IWL_MA_B_GF4_A_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_AX210,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
+const struct iwl_cfg iwl_cfg_ma_b0_mr_a0 = {
+	.fw_name_pre = IWL_MA_B_MR_A_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_AX210,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
 const struct iwl_cfg iwl_cfg_so_a0_ms_a0 = {
 	.fw_name_pre = IWL_SO_A_MR_A_FW_PRE,
 	.uhb_supported = false,
@@ -910,6 +987,14 @@ const struct iwl_cfg iwl_cfg_quz_a0_hr_b0 = {
 	.num_rbds = IWL_NUM_RBDS_22000_HE,
 };
 
+const struct iwl_cfg iwl_cfg_bz_a0_hr_a0 = {
+	.fw_name_pre = IWL_BZ_A_HR_A_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.features = IWL_TX_CSUM_NETIF_FLAGS_BZ | NETIF_F_RXCSUM,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
 const struct iwl_cfg iwl_cfg_bz_a0_hr_b0 = {
 	.fw_name_pre = IWL_BZ_A_HR_B_FW_PRE,
 	.uhb_supported = true,
@@ -958,6 +1043,22 @@ const struct iwl_cfg iwl_cfg_bz_a0_fm4_a0 = {
 	.num_rbds = IWL_NUM_RBDS_AX210_HE,
 };
 
+const struct iwl_cfg iwl_cfg_bz_a0_fm_b0 = {
+	.fw_name_pre = IWL_BZ_A_FM_B_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.features = IWL_TX_CSUM_NETIF_FLAGS_BZ | NETIF_F_RXCSUM,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
+const struct iwl_cfg iwl_cfg_bz_a0_fm4_b0 = {
+	.fw_name_pre = IWL_BZ_A_FM4_B_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.features = IWL_TX_CSUM_NETIF_FLAGS_BZ | NETIF_F_RXCSUM,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
 const struct iwl_cfg iwl_cfg_gl_a0_fm_a0 = {
 	.fw_name_pre = IWL_GL_A_FM_A_FW_PRE,
 	.uhb_supported = true,
@@ -998,6 +1099,14 @@ const struct iwl_cfg iwl_cfg_bnj_a0_fm4_a0 = {
 	.num_rbds = IWL_NUM_RBDS_AX210_HE,
 };
 
+const struct iwl_cfg iwl_cfg_bnj_b0_fm4_b0 = {
+	.fw_name_pre = IWL_BNJ_B_FM4_B_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
 const struct iwl_cfg iwl_cfg_bnj_a0_gf_a0 = {
 	.fw_name_pre = IWL_BNJ_A_GF_A_FW_PRE,
 	.uhb_supported = true,
@@ -1006,6 +1115,14 @@ const struct iwl_cfg iwl_cfg_bnj_a0_gf_a0 = {
 	.num_rbds = IWL_NUM_RBDS_AX210_HE,
 };
 
+const struct iwl_cfg iwl_cfg_bnj_b0_gf_a0 = {
+	.fw_name_pre = IWL_BNJ_B_GF_A_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
 const struct iwl_cfg iwl_cfg_bnj_a0_gf4_a0 = {
 	.fw_name_pre = IWL_BNJ_A_GF4_A_FW_PRE,
 	.uhb_supported = true,
@@ -1014,6 +1131,22 @@ const struct iwl_cfg iwl_cfg_bnj_a0_gf4_a0 = {
 	.num_rbds = IWL_NUM_RBDS_AX210_HE,
 };
 
+const struct iwl_cfg iwl_cfg_bnj_b0_gf4_a0 = {
+	.fw_name_pre = IWL_BNJ_B_GF4_A_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
+const struct iwl_cfg iwl_cfg_bnj_a0_hr_a0 = {
+	.fw_name_pre = IWL_BNJ_A_HR_A_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
 const struct iwl_cfg iwl_cfg_bnj_a0_hr_b0 = {
 	.fw_name_pre = IWL_BNJ_A_HR_B_FW_PRE,
 	.uhb_supported = true,
@@ -1022,6 +1155,22 @@ const struct iwl_cfg iwl_cfg_bnj_a0_hr_b0 = {
 	.num_rbds = IWL_NUM_RBDS_AX210_HE,
 };
 
+const struct iwl_cfg iwl_cfg_bnj_b0_hr_a0 = {
+	.fw_name_pre = IWL_BNJ_B_HR_A_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
+const struct iwl_cfg iwl_cfg_bnj_b0_hr_b0 = {
+	.fw_name_pre = IWL_BNJ_B_HR_B_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
 const struct iwl_cfg iwl_cfg_bnj_b0_fm_b0 = {
 	.fw_name_pre = IWL_BNJ_B_FM_B_FW_PRE,
 	.uhb_supported = true,
@@ -1050,18 +1199,31 @@ MODULE_FIRMWARE(IWL_MA_A_GF_A_FW_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_MA_A_GF4_A_FW_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_MA_A_MR_A_FW_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_MA_A_FM_A_FW_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_MA_B_HR_B_FW_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_MA_B_GF_A_FW_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_MA_B_GF4_A_FW_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_MA_B_MR_A_FW_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_MA_B_FM_A_FW_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_SNJ_A_MR_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BZ_A_HR_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BZ_A_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BZ_A_GF_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BZ_A_GF4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BZ_A_MR_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BZ_A_FM_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BZ_A_FM_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_GL_A_FM_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BNJ_A_FM_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BNJ_A_FM4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BNJ_B_FM4_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BNJ_A_GF_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BNJ_B_GF_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BNJ_A_GF4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BNJ_B_GF4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BNJ_A_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BNJ_B_HR_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BNJ_B_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BZ_A_FM4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BZ_A_FM4_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_GL_B_FM_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BNJ_B_FM_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/sta.c b/drivers/net/wireless/intel/iwlwifi/dvm/sta.c
index cef43cf80620..8b01ab986cb1 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/sta.c
@@ -1081,6 +1081,7 @@ static int iwlagn_send_sta_key(struct iwl_priv *priv,
 {
 	__le16 key_flags;
 	struct iwl_addsta_cmd sta_cmd;
+	size_t to_copy;
 	int i;
 
 	spin_lock_bh(&priv->sta_lock);
@@ -1100,7 +1101,9 @@ static int iwlagn_send_sta_key(struct iwl_priv *priv,
 		sta_cmd.key.tkip_rx_tsc_byte2 = tkip_iv32;
 		for (i = 0; i < 5; i++)
 			sta_cmd.key.tkip_rx_ttak[i] = cpu_to_le16(tkip_p1k[i]);
-		memcpy(sta_cmd.key.key, keyconf->key, keyconf->keylen);
+		/* keyconf may contain MIC rx/tx keys which iwl does not use */
+		to_copy = min_t(size_t, sizeof(sta_cmd.key.key), keyconf->keylen);
+		memcpy(sta_cmd.key.key, keyconf->key, to_copy);
 		break;
 	case WLAN_CIPHER_SUITE_WEP104:
 		key_flags |= STA_KEY_FLG_KEY_SIZE_MSK;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
index a02e5a67b706..5f4a51310add 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
@@ -1006,8 +1006,10 @@ int iwl_acpi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 	union acpi_object *wifi_pkg, *data, *flags;
 	int i, j, ret, tbl_rev, num_sub_bands = 0;
 	int idx = 2;
+	u8 cmd_ver;
 
 	fwrt->ppag_flags = 0;
+	fwrt->ppag_table_valid = false;
 
 	data = iwl_acpi_get_object(fwrt->dev, ACPI_PPAG_METHOD);
 	if (IS_ERR(data))
@@ -1054,8 +1056,15 @@ read_table:
 	}
 
 	fwrt->ppag_flags = flags->integer.value & ACPI_PPAG_MASK;
-
-	if (!fwrt->ppag_flags) {
+	cmd_ver = iwl_fw_lookup_cmd_ver(fwrt->fw,
+					WIDE_ID(PHY_OPS_GROUP,
+						PER_PLATFORM_ANT_GAIN_CMD),
+					IWL_FW_CMD_VER_UNKNOWN);
+	if (cmd_ver == IWL_FW_CMD_VER_UNKNOWN) {
+		ret = -EINVAL;
+		goto out_free;
+	}
+	if (!fwrt->ppag_flags && cmd_ver <= 3) {
 		ret = 0;
 		goto out_free;
 	}
@@ -1076,21 +1085,22 @@ read_table:
 			}
 
 			fwrt->ppag_chains[i].subbands[j] = ent->integer.value;
-
+			/* from ver 4 the fw deals with out of range values */
+			if (cmd_ver >= 4)
+				continue;
 			if ((j == 0 &&
 				(fwrt->ppag_chains[i].subbands[j] > ACPI_PPAG_MAX_LB ||
 				 fwrt->ppag_chains[i].subbands[j] < ACPI_PPAG_MIN_LB)) ||
 				(j != 0 &&
 				(fwrt->ppag_chains[i].subbands[j] > ACPI_PPAG_MAX_HB ||
 				fwrt->ppag_chains[i].subbands[j] < ACPI_PPAG_MIN_HB))) {
-					fwrt->ppag_flags = 0;
 					ret = -EINVAL;
 					goto out_free;
 				}
 		}
 	}
 
-
+	fwrt->ppag_table_valid = true;
 	ret = 0;
 
 out_free:
@@ -1115,19 +1125,22 @@ int iwl_read_ppag_table(struct iwl_fw_runtime *fwrt, union iwl_ppag_table_cmd *c
                 IWL_DEBUG_RADIO(fwrt,
                                 "PPAG capability not supported by FW, command not sent.\n");
                 return -EINVAL;
-        }
-        if (!fwrt->ppag_flags) {
-                IWL_DEBUG_RADIO(fwrt, "PPAG not enabled, command not sent.\n");
-                return -EINVAL;
-        }
+	}
+
+	cmd_ver = iwl_fw_lookup_cmd_ver(fwrt->fw,
+					WIDE_ID(PHY_OPS_GROUP,
+						PER_PLATFORM_ANT_GAIN_CMD),
+					IWL_FW_CMD_VER_UNKNOWN);
+	if (!fwrt->ppag_table_valid || (cmd_ver <= 3 && !fwrt->ppag_flags)) {
+		IWL_DEBUG_RADIO(fwrt, "PPAG not enabled, command not sent.\n");
+		return -EINVAL;
+	}
 
         /* The 'flags' field is the same in v1 and in v2 so we can just
          * use v1 to access it.
          */
         cmd->v1.flags = cpu_to_le32(fwrt->ppag_flags);
-        cmd_ver = iwl_fw_lookup_cmd_ver(fwrt->fw,
-                                        WIDE_ID(PHY_OPS_GROUP, PER_PLATFORM_ANT_GAIN_CMD),
-                                        IWL_FW_CMD_VER_UNKNOWN);
+
 	if (cmd_ver == 1) {
                 num_sub_bands = IWL_NUM_SUB_BANDS_V1;
                 gain = cmd->v1.gain[0];
@@ -1138,7 +1151,7 @@ int iwl_read_ppag_table(struct iwl_fw_runtime *fwrt, union iwl_ppag_table_cmd *c
                                         fwrt->ppag_ver);
                         cmd->v1.flags &= cpu_to_le32(IWL_PPAG_ETSI_MASK);
 		}
-	} else if (cmd_ver == 2 || cmd_ver == 3) {
+	} else if (cmd_ver >= 2 && cmd_ver <= 4) {
                 num_sub_bands = IWL_NUM_SUB_BANDS_V2;
                 gain = cmd->v2.gain[0];
                 *cmd_size = sizeof(cmd->v2);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
index 28c87a480246..111d96cbde6f 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
@@ -266,6 +266,24 @@ enum iwl_legacy_cmds {
 	HOT_SPOT_CMD = 0x53,
 
 	/**
+	 * @WNM_80211V_TIMING_MEASUREMENT_NOTIFICATION: Time Sync
+	 *	measurement notification for TM/FTM. Sent on receipt of
+	 *	respective WNM action frame for TM protocol or public action
+	 *	frame for FTM protocol from peer device along with additional
+	 *	meta data specified in &struct iwl_time_msmt_notify
+	 */
+	WNM_80211V_TIMING_MEASUREMENT_NOTIFICATION = 0x67,
+
+	/**
+	 * @WNM_80211V_TIMING_MEASUREMENT_CONFIRM_NOTIFICATION: Time Sync
+	 *	measurement confirmation notification for TM/FTM. Sent on
+	 *	receipt of Ack from peer for previously Tx'ed TM/FTM
+	 *	action frame along with additional meta data specified in
+	 *	&struct iwl_time_msmt_cfm_notify
+	 */
+	WNM_80211V_TIMING_MEASUREMENT_CONFIRM_NOTIFICATION = 0x68,
+
+	/**
 	 * @SCAN_OFFLOAD_COMPLETE:
 	 * notification, &struct iwl_periodic_scan_complete
 	 */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h b/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h
index df0833890e55..8a613e150a02 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h
@@ -767,7 +767,7 @@ struct iwl_wowlan_status_v12 {
 } __packed; /* WOWLAN_STATUSES_RSP_API_S_VER_12 */
 
 /**
- * struct iwl_wowlan_info_notif - WoWLAN information notification
+ * struct iwl_wowlan_info_notif_v1 - WoWLAN information notification
  * @gtk: GTK data
  * @igtk: IGTK data
  * @replay_ctr: GTK rekey replay counter
@@ -785,7 +785,7 @@ struct iwl_wowlan_status_v12 {
  * @station_id: station id
  * @reserved2: reserved
  */
-struct iwl_wowlan_info_notif {
+struct iwl_wowlan_info_notif_v1 {
 	struct iwl_wowlan_gtk_status_v3 gtk[WOWLAN_GTK_KEYS_NUM];
 	struct iwl_wowlan_igtk_status igtk[WOWLAN_IGTK_KEYS_NUM];
 	__le64 replay_ctr;
@@ -804,6 +804,39 @@ struct iwl_wowlan_info_notif {
 } __packed; /* WOWLAN_INFO_NTFY_API_S_VER_1 */
 
 /**
+ * struct iwl_wowlan_info_notif - WoWLAN information notification
+ * @gtk: GTK data
+ * @igtk: IGTK data
+ * @replay_ctr: GTK rekey replay counter
+ * @pattern_number: number of the matched patterns
+ * @reserved1: reserved
+ * @qos_seq_ctr: QoS sequence counters to use next
+ * @wakeup_reasons: wakeup reasons, see &enum iwl_wowlan_wakeup_reason
+ * @num_of_gtk_rekeys: number of GTK rekeys
+ * @transmitted_ndps: number of transmitted neighbor discovery packets
+ * @received_beacons: number of received beacons
+ * @tid_tear_down: bit mask of tids whose BA sessions were closed
+ *	in suspend state
+ * @station_id: station id
+ * @reserved2: reserved
+ */
+struct iwl_wowlan_info_notif {
+	struct iwl_wowlan_gtk_status_v3 gtk[WOWLAN_GTK_KEYS_NUM];
+	struct iwl_wowlan_igtk_status igtk[WOWLAN_IGTK_KEYS_NUM];
+	__le64 replay_ctr;
+	__le16 pattern_number;
+	__le16 reserved1;
+	__le16 qos_seq_ctr[8];
+	__le32 wakeup_reasons;
+	__le32 num_of_gtk_rekeys;
+	__le32 transmitted_ndps;
+	__le32 received_beacons;
+	u8 tid_tear_down;
+	u8 station_id;
+	u8 reserved2[2];
+} __packed; /* WOWLAN_INFO_NTFY_API_S_VER_2 */
+
+/**
  * struct iwl_wowlan_wake_pkt_notif - WoWLAN wake packet notification
  * @wake_packet_length: wakeup packet length
  * @station_id: station id
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
index 8b38a0073077..6f59381b9f9a 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2020 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -27,6 +27,17 @@ enum iwl_data_path_subcmd_ids {
 	TRIGGER_RX_QUEUES_NOTIF_CMD = 0x2,
 
 	/**
+	 * @WNM_PLATFORM_PTM_REQUEST_CMD: &struct iwl_time_sync_cfg_cmd
+	 */
+	WNM_PLATFORM_PTM_REQUEST_CMD = 0x3,
+
+	/**
+	 * @WNM_80211V_TIMING_MEASUREMENT_CONFIG_CMD:
+	 *	&struct iwl_time_sync_cfg_cmd
+	 */
+	WNM_80211V_TIMING_MEASUREMENT_CONFIG_CMD = 0x4,
+
+	/**
 	 * @STA_HE_CTXT_CMD: &struct iwl_he_sta_context_cmd
 	 */
 	STA_HE_CTXT_CMD = 0x7,
@@ -146,6 +157,177 @@ enum iwl_channel_estimation_flags {
 	IWL_CHANNEL_ESTIMATION_COUNTER	= BIT(2),
 };
 
+enum iwl_time_sync_protocol_type {
+	IWL_TIME_SYNC_PROTOCOL_TM	= BIT(0),
+	IWL_TIME_SYNC_PROTOCOL_FTM	= BIT(1),
+}; /* WNM_TIMING_ENABLED_PROTOCOL_API_E_VER_1 */
+
+/**
+ * struct iwl_time_sync_cfg_cmd - TM/FTM time sync measurement configuration
+ *
+ * @protocols: The type of frames to raise notifications for. A bitmap
+ *	of @iwl_time_sync_protocol_type
+ * @peer_addr: peer address with which TM/FTM measurements are required
+ * @reserved: for alignment
+ */
+struct iwl_time_sync_cfg_cmd {
+	__le32 protocols;
+	u8 peer_addr[ETH_ALEN];
+	u8 reserved[2];
+} __packed; /* WNM_80211V_TIMING_MEASUREMENT_CONFIG_CMD_API_S_VER_1 */
+
+/**
+ * enum iwl_synced_time_operation - PTM request options
+ *
+ * @IWL_SYNCED_TIME_OPERATION_READ_ARTB: read only the ARTB time
+ * @IWL_SYNCED_TIME_OPERATION_READ_GP2: read only the GP2 time
+ * @IWL_SYNCED_TIME_OPERATION_READ_BOTH: latch the ARTB and GP2 clocks and
+ *	provide timestamps from both clocks for the same time point
+ */
+enum iwl_synced_time_operation {
+	IWL_SYNCED_TIME_OPERATION_READ_ARTB = 1,
+	IWL_SYNCED_TIME_OPERATION_READ_GP2,
+	IWL_SYNCED_TIME_OPERATION_READ_BOTH,
+};
+
+/**
+ * struct iwl_synced_time_cmd - request synced GP2/ARTB timestamps
+ *
+ * @operation: one of &enum iwl_synced_time_operation
+ */
+struct iwl_synced_time_cmd {
+	__le32 operation;
+} __packed; /* WNM_80211V_TIMING_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_synced_time_rsp - response to iwl_synced_time_cmd
+ *
+ * @operation: one of &enum iwl_synced_time_operation
+ * @platform_timestamp_hi: high DWORD of the ARTB clock timestamp in nanoseconds
+ * @platform_timestamp_lo: low DWORD of the ARTB clock timestamp in nanoseconds
+ * @gp2_timestamp_hi: high DWORD of the GP2 clock timestamp in 10's of
+ *	nanoseconds
+ * @gp2_timestamp_lo: low DWORD of the GP2 clock timestamp in 10's of
+ *	nanoseconds
+ */
+struct iwl_synced_time_rsp {
+	__le32 operation;
+	__le32 platform_timestamp_hi;
+	__le32 platform_timestamp_lo;
+	__le32 gp2_timestamp_hi;
+	__le32 gp2_timestamp_lo;
+} __packed; /* WNM_80211V_TIMING_RSP_API_S_VER_1 */
+
+/* PTP_CTX_MAX_DATA_SIZE_IN_API_D_VER_1 */
+#define PTP_CTX_MAX_DATA_SIZE   128
+
+/**
+ * struct iwl_time_msmt_ptp_ctx - Vendor specific information element
+ * to allow a space for flexibility for the userspace App
+ *
+ * @element_id: element id of vendor specific ie
+ * @length: length of vendor specific ie
+ * @reserved: for alignment
+ * @data: vendor specific data blob
+ */
+struct iwl_time_msmt_ptp_ctx {
+	/* Differentiate between FTM and TM specific Vendor IEs */
+	union {
+		struct {
+			u8 element_id;
+			u8 length;
+			__le16 reserved;
+			u8 data[PTP_CTX_MAX_DATA_SIZE];
+		} ftm; /* FTM specific vendor IE */
+		struct {
+			u8 element_id;
+			u8 length;
+			u8 data[PTP_CTX_MAX_DATA_SIZE];
+		} tm; /* TM specific vendor IE */
+	};
+} __packed /* PTP_CTX_VER_1 */;
+
+/**
+ * struct iwl_time_msmt_notify - Time Sync measurement notification
+ * for TM/FTM, along with additional meta data.
+ *
+ * @peer_addr: peer address
+ * @reserved: for alignment
+ * @dialog_token: measurement flow dialog token number
+ * @followup_dialog_token: Measurement flow previous dialog token number
+ * @t1_hi: high dword of t1-time of the Tx'ed action frame departure on
+ *	sender side in units of 10 nano seconds
+ * @t1_lo: low dword of t1-time of the Tx'ed action frame departure on
+ *	sender side in units of 10 nano seconds
+ * @t1_max_err: maximum t1-time error in units of 10 nano seconds
+ * @t4_hi: high dword of t4-time of the Rx'ed action frame's Ack arrival on
+ *	sender side in units of 10 nano seconds
+ * @t4_lo: low dword of t4-time of the Rx'ed action frame's Ack arrival on
+ *	sender side in units of 10 nano seconds
+ * @t4_max_err: maximum t4-time error in units of 10 nano seconds
+ * @t2_hi: high dword of t2-time of the Rx'ed action frame arrival on
+ *	receiver side in units of 10 nano seconds
+ * @t2_lo: low dword of t2-time of the Rx'ed action frame arrival on
+ *	receiver side in units of 10 nano seconds
+ * @t2_max_err: maximum t2-time error in units of 10 nano seconds
+ * @t3_hi: high dword of t3-time of the Tx'ed action frame's Ack departure on
+ *	receiver side in units of 10 nano seconds
+ * @t3_lo: low dword of t3-time of the Tx'ed action frame's Ack departure on
+ *	receiver side in units of 10 nano seconds
+ * @t3_max_err: maximum t3-time error in units of 10 nano seconds
+ * @ptp: vendor specific information element
+ */
+struct iwl_time_msmt_notify {
+	u8 peer_addr[ETH_ALEN];
+	u8 reserved[2];
+	__le32 dialog_token;
+	__le32 followup_dialog_token;
+	__le32 t1_hi;
+	__le32 t1_lo;
+	__le32 t1_max_err;
+	__le32 t4_hi;
+	__le32 t4_lo;
+	__le32 t4_max_err;
+	__le32 t2_hi;
+	__le32 t2_lo;
+	__le32 t2_max_err;
+	__le32 t3_hi;
+	__le32 t3_lo;
+	__le32 t3_max_err;
+	struct iwl_time_msmt_ptp_ctx ptp;
+} __packed; /* WNM_80211V_TIMING_MEASUREMENT_NTFY_API_S_VER_1 */
+
+/**
+ * struct iwl_time_msmt_cfm_notify - Time Sync measurement confirmation
+ * notification for TM/FTM. Sent on receipt of 802.11 Ack from peer for the
+ * Tx'ed TM/FTM measurement action frame.
+ *
+ * @peer_addr: peer address
+ * @reserved: for alignment
+ * @dialog_token: measurement flow dialog token number
+ * @t1_hi: high dword of t1-time of the Tx'ed action frame departure on
+ *	sender side in units of 10 nano seconds
+ * @t1_lo: low dword of t1-time of the Tx'ed action frame departure on
+ *	sender side in units of 10 nano seconds
+ * @t1_max_err: maximum t1-time error in units of 10 nano seconds
+ * @t4_hi: high dword of t4-time of the Rx'ed action frame's Ack arrival on
+ *	sender side in units of 10 nano seconds
+ * @t4_lo: low dword of t4-time of the Rx'ed action frame's Ack arrival on
+ *	sender side in units of 10 nano seconds
+ * @t4_max_err: maximum t4-time error in units of 10 nano seconds
+ */
+struct iwl_time_msmt_cfm_notify {
+	u8 peer_addr[ETH_ALEN];
+	u8 reserved[2];
+	__le32 dialog_token;
+	__le32 t1_hi;
+	__le32 t1_lo;
+	__le32 t1_max_err;
+	__le32 t4_hi;
+	__le32 t4_lo;
+	__le32 t4_max_err;
+} __packed; /* WNM_80211V_TIMING_MEASUREMENT_CONFIRM_NTFY_API_S_VER_1 */
+
 /**
  * struct iwl_channel_estimation_cfg - channel estimation reporting config
  */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h b/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h
index 0c555089e05f..8fef38139bf6 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h
@@ -43,6 +43,12 @@ enum iwl_debug_cmds {
 	 */
 	BUFFER_ALLOCATION = 0x8,
 	/**
+	 * @GET_TAS_STATUS:
+	 * sends command to fw to get TAS status
+	 * the response is &struct iwl_mvm_tas_status_resp
+	 */
+	GET_TAS_STATUS = 0xA,
+	/**
 	 * @FW_DUMP_COMPLETE_CMD:
 	 * sends command to fw once dump collection completed
 	 * &struct iwl_dbg_dump_complete_cmd
@@ -421,4 +427,94 @@ struct iwl_dbg_dump_complete_cmd {
 	__le32 tp_data;
 } __packed; /* FW_DUMP_COMPLETE_CMD_API_S_VER_1 */
 
+#define TAS_LMAC_BAND_HB       0
+#define TAS_LMAC_BAND_LB       1
+#define TAS_LMAC_BAND_UHB      2
+#define TAS_LMAC_BAND_INVALID  3
+
+/**
+ * struct iwl_mvm_tas_status_per_mac - tas status per lmac
+ * @static_status: tas statically enabled or disabled per lmac - TRUE/FALSE
+ * @static_dis_reason: TAS static disable reason, uses
+ *	&enum iwl_mvm_tas_statically_disabled_reason
+ * @dynamic_status: Current TAS  status. uses
+ *	&enum iwl_mvm_tas_dyna_status
+ * @near_disconnection: is TAS currently near disconnection per lmac? - TRUE/FALSE
+ * @max_reg_pwr_limit: Regulatory power limits in dBm
+ * @sar_limit: SAR limits per lmac in dBm
+ * @band: Band per lmac
+ * @reserved: reserved
+ */
+struct iwl_mvm_tas_status_per_mac {
+	u8 static_status;
+	u8 static_dis_reason;
+	u8 dynamic_status;
+	u8 near_disconnection;
+	__le16 max_reg_pwr_limit;
+	__le16 sar_limit;
+	u8 band;
+	u8 reserved[3];
+} __packed; /*DEBUG_GET_TAS_STATUS_PER_MAC_S_VER_1*/
+
+/**
+ * struct iwl_mvm_tas_status_resp - Response to GET_TAS_STATUS
+ * @tas_fw_version: TAS FW version
+ * @is_uhb_for_usa_enable: is UHB enabled in USA? - TRUE/FALSE
+ * @curr_mcc: current mcc
+ * @block_list: country block list
+ * @tas_status_mac: TAS status per lmac, uses
+ *	&struct iwl_mvm_tas_status_per_mac
+ * @in_dual_radio: is TAS in dual radio? - TRUE/FALSE
+ * @reserved: reserved
+ */
+struct iwl_mvm_tas_status_resp {
+	u8 tas_fw_version;
+	u8 is_uhb_for_usa_enable;
+	__le16 curr_mcc;
+	__le16 block_list[16];
+	struct iwl_mvm_tas_status_per_mac tas_status_mac[2];
+	u8 in_dual_radio;
+	u8 reserved[3];
+} __packed; /*DEBUG_GET_TAS_STATUS_RSP_API_S_VER_3*/
+
+/**
+ * enum iwl_mvm_tas_dyna_status - TAS current running status
+ * @TAS_DYNA_INACTIVE: TAS status is inactive
+ * @TAS_DYNA_INACTIVE_MVM_MODE: TAS is disabled due because FW is in MVM mode
+ *	or is in softap mode.
+ * @TAS_DYNA_INACTIVE_TRIGGER_MODE: TAS is disabled because FW is in
+ *	multi user trigger mode
+ * @TAS_DYNA_INACTIVE_BLOCK_LISTED: TAS is disabled because  current mcc
+ *	is blocklisted mcc
+ * @TAS_DYNA_INACTIVE_UHB_NON_US: TAS is disabled because current band is UHB
+ *	and current mcc is USA
+ * @TAS_DYNA_ACTIVE: TAS is currently active
+ * @TAS_DYNA_STATUS_MAX: TAS status max value
+ */
+enum iwl_mvm_tas_dyna_status {
+	TAS_DYNA_INACTIVE,
+	TAS_DYNA_INACTIVE_MVM_MODE,
+	TAS_DYNA_INACTIVE_TRIGGER_MODE,
+	TAS_DYNA_INACTIVE_BLOCK_LISTED,
+	TAS_DYNA_INACTIVE_UHB_NON_US,
+	TAS_DYNA_ACTIVE,
+
+	TAS_DYNA_STATUS_MAX,
+}; /*_TAS_DYNA_STATUS_E*/
+
+/**
+ * enum iwl_mvm_tas_statically_disabled_reason - TAS statically disabled reason
+ * @TAS_DISABLED_DUE_TO_BIOS: TAS is disabled because TAS is disabled in BIOS
+ * @TAS_DISABLED_DUE_TO_SAR_6DBM: TAS is disabled because SAR limit is less than 6 Dbm
+ * @TAS_DISABLED_REASON_INVALID: TAS disable reason is invalid
+ * @TAS_DISABLED_REASON_MAX: TAS disable reason max value
+ */
+enum iwl_mvm_tas_statically_disabled_reason {
+	TAS_DISABLED_DUE_TO_BIOS,
+	TAS_DISABLED_DUE_TO_SAR_6DBM,
+	TAS_DISABLED_REASON_INVALID,
+
+	TAS_DISABLED_REASON_MAX,
+}; /*_TAS_STATICALLY_DISABLED_REASON_E*/
+
 #endif /* __iwl_fw_api_debug_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
index 712532f17630..74f2efbad34e 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
@@ -1,12 +1,14 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2019, 2021 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2019, 2021-2022 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
 #ifndef __iwl_fw_api_mac_cfg_h__
 #define __iwl_fw_api_mac_cfg_h__
 
+#include "mac.h"
+
 /**
  * enum iwl_mac_conf_subcmd_ids - mac configuration command IDs
  */
@@ -31,7 +33,30 @@ enum iwl_mac_conf_subcmd_ids {
 	 * @CANCEL_CHANNEL_SWITCH_CMD: &struct iwl_cancel_channel_switch_cmd
 	 */
 	CANCEL_CHANNEL_SWITCH_CMD = 0x6,
-
+	/**
+	 * @MAC_CONFIG_CMD: &struct iwl_mac_config_cmd
+	 */
+	MAC_CONFIG_CMD = 0x8,
+	/**
+	 * @LINK_CONFIG_CMD: &struct iwl_link_config_cmd
+	 */
+	LINK_CONFIG_CMD = 0x9,
+	/**
+	 * @STA_CONFIG_CMD: &struct iwl_mvm_sta_cfg_cmd
+	 */
+	STA_CONFIG_CMD = 0xA,
+	/**
+	 * @AUX_STA_CMD: &struct iwl_mvm_aux_sta_cmd
+	 */
+	AUX_STA_CMD = 0xB,
+	/**
+	 * @STA_REMOVE_CMD: &struct iwl_mvm_remove_sta_cmd
+	 */
+	STA_REMOVE_CMD = 0xC,
+	/**
+	 * @STA_DISABLE_TX_CMD: &struct iwl_mvm_sta_disable_tx_cmd
+	 */
+	STA_DISABLE_TX_CMD = 0xD,
 	/**
 	 * @SESSION_PROTECTION_NOTIF: &struct iwl_mvm_session_prot_notif
 	 */
@@ -182,4 +207,393 @@ struct iwl_mac_low_latency_cmd {
 	__le16 reserved;
 } __packed; /* MAC_LOW_LATENCY_API_S_VER_1 */
 
+/**
+ * struct iwl_mac_client_data - configuration data for client MAC context
+ *
+ * @is_assoc: 1 for associated state, 0 otherwise
+ * @assoc_id: unique ID assigned by the AP during association
+ * @data_policy: see &enum iwl_mac_data_policy
+ * @ctwin: client traffic window in TU (period after TBTT when GO is present).
+ *	0 indicates that there is no CT window.
+ */
+struct iwl_mac_client_data {
+	__le32 is_assoc;
+	__le32 assoc_id;
+	__le32 data_policy;
+	__le32 ctwin;
+} __packed; /* MAC_CONTEXT_CONFIG_CLIENT_DATA_API_S_VER_1 */
+
+/**
+ * struct iwl_mac_p2p_dev_data  - configuration data for P2P device MAC context
+ *
+ * @is_disc_extended: if set to true, P2P Device discoverability is enabled on
+ *	other channels as well. This should be to true only in case that the
+ *	device is discoverable and there is an active GO. Note that setting this
+ *	field when not needed, will increase the number of interrupts and have
+ *	effect on the platform power, as this setting opens the Rx filters on
+ *	all macs.
+ */
+struct iwl_mac_p2p_dev_data {
+	__le32 is_disc_extended;
+} __packed; /* MAC_CONTEXT_CONFIG_P2P_DEV_DATA_API_S_VER_1 */
+
+/**
+ * enum iwl_mac_config_filter_flags - MAC context configuration filter flags
+ *
+ * @MAC_CFG_FILTER_PROMISC: accept all data frames
+ * @MAC_CFG_FILTER_ACCEPT_CONTROL_AND_MGMT: pass all management and
+ *	control frames to the host
+ * @MAC_CFG_FILTER_ACCEPT_GRP: accept multicast frames
+ * @MAC_CFG_FILTER_ACCEPT_BEACON: accept beacon frames
+ * @MAC_CFG_FILTER_ACCEPT_BCAST_PROBE_RESP: accept broadcast probe response
+ * @MAC_CFG_FILTER_ACCEPT_PROBE_REQ: accept probe requests
+ */
+enum iwl_mac_config_filter_flags {
+	MAC_CFG_FILTER_PROMISC			= BIT(0),
+	MAC_CFG_FILTER_ACCEPT_CONTROL_AND_MGMT	= BIT(1),
+	MAC_CFG_FILTER_ACCEPT_GRP		= BIT(2),
+	MAC_CFG_FILTER_ACCEPT_BEACON		= BIT(3),
+	MAC_CFG_FILTER_ACCEPT_BCAST_PROBE_RESP	= BIT(4),
+	MAC_CFG_FILTER_ACCEPT_PROBE_REQ		= BIT(5),
+}; /* MAC_FILTER_FLAGS_MASK_E_VER_1 */
+
+/**
+ * struct iwl_mac_config_cmd - command structure to configure MAC contexts in
+ *	MLD API
+ * ( MAC_CONTEXT_CONFIG_CMD = 0x8 )
+ *
+ * @id_and_color: ID and color of the MAC
+ * @action: action to perform, one of FW_CTXT_ACTION_*
+ * @mac_type: one of &enum iwl_mac_types
+ * @local_mld_addr: mld address
+ * @reserved_for_local_mld_addr: reserved
+ * @filter_flags: combination of &enum iwl_mac_config_filter_flags
+ * @he_support: does this MAC support HE
+ * @he_ap_support: HE AP enabled, "pseudo HE", no trigger frame handling
+ * @eht_support: does this MAC support EHT. Requires he_support
+ * @nic_not_ack_enabled: mark that the NIC doesn't support receiving
+ *	ACK-enabled AGG, (i.e. both BACK and non-BACK frames in single AGG).
+ *	If the NIC is not ACK_ENABLED it may use the EOF-bit in first non-0
+ *	len delim to determine if AGG or single.
+ * @client: client mac data
+ * @go_ibss: mac data for go or ibss
+ * @p2p_dev: mac data for p2p device
+ */
+struct iwl_mac_config_cmd {
+	/* COMMON_INDEX_HDR_API_S_VER_1 */
+	__le32 id_and_color;
+	__le32 action;
+	/* MAC_CONTEXT_TYPE_API_E */
+	__le32 mac_type;
+	u8 local_mld_addr[6];
+	__le16 reserved_for_local_mld_addr;
+	__le32 filter_flags;
+	__le16 he_support;
+	__le16 he_ap_support;
+	__le32 eht_support;
+	__le32 nic_not_ack_enabled;
+	/* MAC_CONTEXT_CONFIG_SPECIFIC_DATA_API_U_VER_1 */
+	union {
+		struct iwl_mac_client_data client;
+		struct iwl_mac_p2p_dev_data p2p_dev;
+	};
+} __packed; /* MAC_CONTEXT_CONFIG_CMD_API_S_VER_1 */
+
+/**
+ * enum iwl_link_ctx_modify_flags - indicate to the fw what fields are being
+ *	modified in &iwl_link_ctx_cfg_cmd
+ *
+ * @LINK_CONTEXT_MODIFY_ACTIVE: covers iwl_link_ctx_cfg_cmd::active
+ * @LINK_CONTEXT_MODIFY_RATES_INFO: covers iwl_link_ctx_cfg_cmd::cck_rates,
+ *	iwl_link_ctx_cfg_cmd::ofdm_rates,
+ *	iwl_link_ctx_cfg_cmd::cck_short_preamble,
+ *	iwl_link_ctx_cfg_cmd::short_slot
+ * @LINK_CONTEXT_MODIFY_PROTECT_FLAGS: covers
+ *	iwl_link_ctx_cfg_cmd::protection_flags
+ * @LINK_CONTEXT_MODIFY_QOS_PARAMS: covers iwl_link_ctx_cfg_cmd::qos_flags,
+ *	iwl_link_ctx_cfg_cmd::ac,
+ * @LINK_CONTEXT_MODIFY_BEACON_TIMING: covers iwl_link_ctx_cfg_cmd::bi,
+ *	iwl_link_ctx_cfg_cmd::dtim_interval,
+ *	iwl_link_ctx_cfg_cmd::dtim_time,
+ *	iwl_link_ctx_cfg_cmd::dtim_tsf,
+ *	iwl_link_ctx_cfg_cmd::assoc_beacon_arrive_time.
+ *	This flag can be set only once after assoc.
+ * @LINK_CONTEXT_MODIFY_HE_PARAMS: covers
+ *	iwl_link_ctx_cfg_cmd::htc_trig_based_pkt_ext
+ *	iwl_link_ctx_cfg_cmd::rand_alloc_ecwmin,
+ *	iwl_link_ctx_cfg_cmd::rand_alloc_ecwmax,
+ *	iwl_link_ctx_cfg_cmd::trig_based_txf,
+ *	iwl_link_ctx_cfg_cmd::bss_color,
+ *	iwl_link_ctx_cfg_cmd::ndp_fdbk_buff_th_exp,
+ *	iwl_link_ctx_cfg_cmd::ref_bssid_addr
+ *	iwl_link_ctx_cfg_cmd::bssid_index,
+ *	iwl_link_ctx_cfg_cmd::frame_time_rts_th.
+ *	This flag can be set any time.
+ * @LINK_CONTEXT_MODIFY_BSS_COLOR_DISABLE: covers
+ *	iwl_link_ctx_cfg_cmd::bss_color_disable
+ * @LINK_CONTEXT_MODIFY_EHT_PARAMS: covers iwl_link_ctx_cfg_cmd::puncture_mask.
+ *	This flag can be set only if the MAC that this link relates to has
+ *	eht_support set to true.
+ * @LINK_CONTEXT_MODIFY_ALL: set all above flags
+ */
+enum iwl_link_ctx_modify_flags {
+	LINK_CONTEXT_MODIFY_ACTIVE		= BIT(0),
+	LINK_CONTEXT_MODIFY_RATES_INFO		= BIT(1),
+	LINK_CONTEXT_MODIFY_PROTECT_FLAGS	= BIT(2),
+	LINK_CONTEXT_MODIFY_QOS_PARAMS		= BIT(3),
+	LINK_CONTEXT_MODIFY_BEACON_TIMING	= BIT(4),
+	LINK_CONTEXT_MODIFY_HE_PARAMS		= BIT(5),
+	LINK_CONTEXT_MODIFY_BSS_COLOR_DISABLE	= BIT(6),
+	LINK_CONTEXT_MODIFY_EHT_PARAMS		= BIT(7),
+	LINK_CONTEXT_MODIFY_ALL			= 0xff,
+}; /* LINK_CONTEXT_MODIFY_MASK_E_VER_1 */
+
+/**
+ * enum iwl_link_ctx_protection_flags - link protection flags
+ * @LINK_PROT_FLG_TGG_PROTECT: 11g protection when transmitting OFDM frames,
+ *	this will require CCK RTS/CTS2self.
+ *	RTS/CTS will protect full burst time.
+ * @LINK_PROT_FLG_HT_PROT: enable HT protection
+ * @LINK_PROT_FLG_FAT_PROT: protect 40 MHz transmissions
+ * @LINK_PROT_FLG_SELF_CTS_EN: allow CTS2self
+ */
+enum iwl_link_ctx_protection_flags {
+	LINK_PROT_FLG_TGG_PROTECT	= BIT(0),
+	LINK_PROT_FLG_HT_PROT		= BIT(1),
+	LINK_PROT_FLG_FAT_PROT		= BIT(2),
+	LINK_PROT_FLG_SELF_CTS_EN	= BIT(3),
+}; /* LINK_PROTECT_FLAGS_E_VER_1 */
+
+/**
+ * enum iwl_link_ctx_flags - link context flags
+ *
+ * @LINK_FLG_BSS_COLOR_DIS: BSS color disable, don't use the BSS
+ *	color for RX filter but use MAC header
+ *	enabled AGG, i.e. both BACK and non-BACK frames in a single AGG
+ * @LINK_FLG_MU_EDCA_CW: indicates that there is an element of MU EDCA
+ *	parameter set, i.e. the backoff counters for trig-based ACs
+ * @LINK_FLG_RU_2MHZ_BLOCK: indicates that 26-tone RU OFDMA transmission are
+ *      not allowed (as there are OBSS that might classify such transmissions as
+ *      radar pulses).
+ * @LINK_FLG_NDP_FEEDBACK_ENABLED: mark support for NDP feedback and change
+ *	of threshold
+ */
+enum iwl_link_ctx_flags {
+	LINK_FLG_BSS_COLOR_DIS		= BIT(0),
+	LINK_FLG_MU_EDCA_CW		= BIT(1),
+	LINK_FLG_RU_2MHZ_BLOCK		= BIT(2),
+	LINK_FLG_NDP_FEEDBACK_ENABLED	= BIT(3),
+}; /* LINK_CONTEXT_FLAG_E_VER_1 */
+
+/**
+ * struct iwl_link_config_cmd - command structure to configure the LINK context
+ *	in MLD API
+ * ( LINK_CONFIG_CMD =0x9 )
+ *
+ * @action: action to perform, one of FW_CTXT_ACTION_*
+ * @link_id: the id of the link that this cmd configures
+ * @mac_id: interface ID. Relevant only if action is FW_CTXT_ACTION_ADD
+ * @phy_id: PHY index. Can be changed only if the link was inactive
+ *	(and stays inactive). If the link is active (or becomes active),
+ *	this field is ignored.
+ * @local_link_addr: the links MAC address. Can be changed only if the link was
+ *	inactive (and stays inactive). If the link is active
+ *	(or becomes active), this field is ignored.
+ * @reserved_for_local_link_addr: reserved
+ * @modify_mask: from &enum iwl_link_ctx_modify_flags, selects what to change.
+ *	Relevant only if action is FW_CTXT_ACTION_MODIFY
+ * @active: indicates whether the link is active or not
+ * @listen_lmac: indicates whether the link should be allocated on the Listen
+ *	Lmac or on the Main Lmac. Cannot be changed on an active Link.
+ *	Relevant only for eSR.
+ * @cck_rates: basic rates available for CCK
+ * @ofdm_rates: basic rates available for OFDM
+ * @cck_short_preamble: 1 for enabling short preamble, 0 otherwise
+ * @short_slot: 1 for enabling short slots, 0 otherwise
+ * @protection_flags: combination of &enum iwl_link_ctx_protection_flags
+ * @qos_flags: from &enum iwl_mac_qos_flags
+ * @ac: one iwl_mac_qos configuration for each AC
+ * @htc_trig_based_pkt_ext: default PE in 4us units
+ * @rand_alloc_ecwmin: random CWmin = 2**ECWmin-1
+ * @rand_alloc_ecwmax: random CWmax = 2**ECWmax-1
+ * @ndp_fdbk_buff_th_exp: set exponent for the NDP feedback buffered threshold
+ * @trig_based_txf: MU EDCA Parameter set for the trigger based traffic queues
+ * @bi: beacon interval in TU, applicable only when associated
+ * @dtim_interval: DTIM interval in TU.
+ *	Relevant only for GO, otherwise this is offloaded.
+ * @puncture_mask: puncture mask for EHT
+ * @frame_time_rts_th: HE duration RTS threshold, in units of 32us
+ * @flags: a combination from &enum iwl_link_ctx_flags
+ * @flags_mask: what of %flags have changed. Also &enum iwl_link_ctx_flags
+ * Below fields are for multi-bssid:
+ * @ref_bssid_addr: reference BSSID used by the AP
+ * @reserved_for_ref_bssid_addr: reserved
+ * @bssid_index: index of the associated VAP
+ * @bss_color: 11ax AP ID that is used in the HE SIG-A to mark inter BSS frame
+ * @reserved: alignment
+ * @ibss_bssid_addr: bssid for ibss
+ * @reserved_for_ibss_bssid_addr: reserved
+ * @reserved1: reserved for future use
+ */
+struct iwl_link_config_cmd {
+	__le32 action;
+	__le32 link_id;
+	__le32 mac_id;
+	__le32 phy_id;
+	u8 local_link_addr[6];
+	__le16 reserved_for_local_link_addr;
+	__le32 modify_mask;
+	__le32 active;
+	__le32 listen_lmac;
+	__le32 cck_rates;
+	__le32 ofdm_rates;
+	__le32 cck_short_preamble;
+	__le32 short_slot;
+	__le32 protection_flags;
+	/* MAC_QOS_PARAM_API_S_VER_1 */
+	__le32 qos_flags;
+	struct iwl_ac_qos ac[AC_NUM + 1];
+	u8 htc_trig_based_pkt_ext;
+	u8 rand_alloc_ecwmin;
+	u8 rand_alloc_ecwmax;
+	u8 ndp_fdbk_buff_th_exp;
+	struct iwl_he_backoff_conf trig_based_txf[AC_NUM];
+	__le32 bi;
+	__le32 dtim_interval;
+	__le16 puncture_mask;
+	__le16 frame_time_rts_th;
+	__le32 flags;
+	__le32 flags_mask;
+	/* The below fields are for multi-bssid */
+	u8 ref_bssid_addr[6];
+	__le16 reserved_for_ref_bssid_addr;
+	u8 bssid_index;
+	u8 bss_color;
+	u8 reserved[2];
+	u8 ibss_bssid_addr[6];
+	__le16 reserved_for_ibss_bssid_addr;
+	__le32 reserved1[8];
+} __packed; /* LINK_CONTEXT_CONFIG_CMD_API_S_VER_1 */
+
+/* Currently FW supports link ids in the range 0-3 and can have
+ * at most two active links for each vif.
+ */
+#define IWL_MVM_FW_MAX_ACTIVE_LINKS_NUM 2
+#define IWL_MVM_FW_MAX_LINK_ID 3
+
+/**
+ * enum iwl_fw_sta_type - FW station types
+ * @STATION_TYPE_PEER: represents a peer - AP in BSS, a TDLS sta, a client in
+ *	P2P.
+ * @STATION_TYPE_BCAST_MGMT: The station used to send beacons and
+ *	probe responses. Also used for traffic injection in sniffer mode
+ * @STATION_TYPE_MCAST: the station used for BCAST / MCAST in GO. Will be
+ *	suspended / resumed at the right timing depending on the clients'
+ *	power save state and the DTIM timing
+ * @STATION_TYPE_AUX: aux sta. In the FW there is no need for a special type
+ *	for the aux sta, so this type is only for driver - internal use.
+ */
+enum iwl_fw_sta_type {
+	STATION_TYPE_PEER,
+	STATION_TYPE_BCAST_MGMT,
+	STATION_TYPE_MCAST,
+	STATION_TYPE_AUX,
+}; /* STATION_TYPE_E_VER_1 */
+
+/**
+ * struct iwl_mvm_sta_cfg_cmd - cmd structure to add a peer sta to the uCode's
+ *	station table
+ * ( STA_CONFIG_CMD = 0xA )
+ *
+ * @sta_id: index of station in uCode's station table
+ * @link_id: the id of the link that is used to communicate with this sta
+ * @peer_mld_address: the peers mld address
+ * @reserved_for_peer_mld_address: reserved
+ * @peer_link_address: the address of the link that is used to communicate
+ *	with this sta
+ * @reserved_for_peer_link_address: reserved
+ * @station_type: type of this station. See &enum iwl_fw_sta_type
+ * @assoc_id: for GO only
+ * @beamform_flags: beam forming controls
+ * @mfp: indicates whether the STA uses management frame protection or not.
+ * @mimo: indicates whether the sta uses mimo or not
+ * @mimo_protection: indicates whether the sta uses mimo protection or not
+ * @ack_enabled: indicates that the AP supports receiving ACK-
+ *	enabled AGG, i.e. both BACK and non-BACK frames in a single AGG
+ * @trig_rnd_alloc: indicates that trigger based random allocation
+ *	is enabled according to UORA element existence
+ * @tx_ampdu_spacing: minimum A-MPDU spacing:
+ *	4 - 2us density, 5 - 4us density, 6 - 8us density, 7 - 16us density
+ * @tx_ampdu_max_size: maximum A-MPDU length: 0 - 8K, 1 - 16K, 2 - 32K,
+ *	3 - 64K, 4 - 128K, 5 - 256K, 6 - 512K, 7 - 1024K.
+ * @sp_length: the size of the SP in actual number of frames
+ * @uapsd_acs:  4 LS bits are trigger enabled ACs, 4 MS bits are the deliver
+ *	enabled ACs.
+ * @pkt_ext: optional, exists according to PPE-present bit in the HE/EHT-PHY
+ *	capa
+ * @htc_flags: which features are supported in HTC
+ */
+struct iwl_mvm_sta_cfg_cmd {
+	__le32 sta_id;
+	__le32 link_id;
+	u8 peer_mld_address[ETH_ALEN];
+	__le16 reserved_for_peer_mld_address;
+	u8 peer_link_address[ETH_ALEN];
+	__le16 reserved_for_peer_link_address;
+	__le32 station_type;
+	__le32 assoc_id;
+	__le32 beamform_flags;
+	__le32 mfp;
+	__le32 mimo;
+	__le32 mimo_protection;
+	__le32 ack_enabled;
+	__le32 trig_rnd_alloc;
+	__le32 tx_ampdu_spacing;
+	__le32 tx_ampdu_max_size;
+	__le32 sp_length;
+	__le32 uapsd_acs;
+	struct iwl_he_pkt_ext_v2 pkt_ext;
+	__le32 htc_flags;
+} __packed; /* STA_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_mvm_aux_sta_cmd - command for AUX STA configuration
+ * ( AUX_STA_CMD = 0xB )
+ *
+ * @sta_id: index of aux sta to configure
+ * @lmac_id: ?
+ * @mac_addr: mac addr of the auxilary sta
+ * @reserved_for_mac_addr: reserved
+ */
+struct iwl_mvm_aux_sta_cmd {
+	__le32 sta_id;
+	__le32 lmac_id;
+	u8 mac_addr[ETH_ALEN];
+	__le16 reserved_for_mac_addr;
+
+} __packed; /* AUX_STA_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_mvm_remove_sta_cmd - a cmd structure to remove a sta added by
+ *	STA_CONFIG_CMD or AUX_STA_CONFIG_CMD
+ * ( STA_REMOVE_CMD = 0xC )
+ *
+ * @sta_id: index of station to remove
+ */
+struct iwl_mvm_remove_sta_cmd {
+	__le32 sta_id;
+} __packed; /* REMOVE_STA_API_S_VER_1 */
+
+/**
+ * struct iwl_mvm_sta_disable_tx_cmd - disable / re-enable tx to a sta
+ * ( STA_DISABLE_TX_CMD = 0xD )
+ *
+ * @sta_id: index of the station to disable tx to
+ * @disable: indicates if to disable or re-enable tx
+ */
+struct iwl_mvm_sta_disable_tx_cmd {
+	__le32 sta_id;
+	__le32 disable;
+} __packed; /* STA_DISABLE_TX_API_S_VER_1 */
+
 #endif /* __iwl_fw_api_mac_cfg_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
index ddacd5b45aea..c9a48fc5fac8 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
@@ -373,9 +373,6 @@ enum {
 
 /* Bit 4-5: (0) SISO, (1) MIMO2 (2) MIMO3 */
 #define RATE_VHT_MCS_RATE_CODE_MSK	0xf
-#define RATE_VHT_MCS_NSS_POS		4
-#define RATE_VHT_MCS_NSS_MSK		(3 << RATE_VHT_MCS_NSS_POS)
-#define RATE_VHT_MCS_MIMO2_MSK		BIT(RATE_VHT_MCS_NSS_POS)
 
 /*
  * Legacy OFDM rate format for bits 7:0
@@ -449,11 +446,16 @@ enum {
  *	1			2xLTF+0.8us
  *	2			2xLTF+1.6us
  *	3			4xLTF+3.2us
- * HE TRIG:
+ * HE-EHT TRIG:
  *	0			1xLTF+1.6us
  *	1			2xLTF+1.6us
  *	2			4xLTF+3.2us
  *	3			(does not occur)
+ * EHT MU:
+ *	0			2xLTF+0.8us
+ *	1			2xLTF+1.6us
+ *	2			4xLTF+0.8us
+ *	3			4xLTF+3.2us
  */
 #define RATE_MCS_HE_GI_LTF_POS		20
 #define RATE_MCS_HE_GI_LTF_MSK_V1		(3 << RATE_MCS_HE_GI_LTF_POS)
@@ -546,12 +548,17 @@ enum {
 /*
  * Bits 13-11: (0) 20MHz, (1) 40MHz, (2) 80MHz, (3) 160MHz, (4) 320MHz
  */
-#define RATE_MCS_CHAN_WIDTH_MSK			(0x7 << RATE_MCS_CHAN_WIDTH_POS)
-#define RATE_MCS_CHAN_WIDTH_20			(0 << RATE_MCS_CHAN_WIDTH_POS)
-#define RATE_MCS_CHAN_WIDTH_40			(1 << RATE_MCS_CHAN_WIDTH_POS)
-#define RATE_MCS_CHAN_WIDTH_80			(2 << RATE_MCS_CHAN_WIDTH_POS)
-#define RATE_MCS_CHAN_WIDTH_160			(3 << RATE_MCS_CHAN_WIDTH_POS)
-#define RATE_MCS_CHAN_WIDTH_320			(4 << RATE_MCS_CHAN_WIDTH_POS)
+#define RATE_MCS_CHAN_WIDTH_MSK		(0x7 << RATE_MCS_CHAN_WIDTH_POS)
+#define RATE_MCS_CHAN_WIDTH_20_VAL	0
+#define RATE_MCS_CHAN_WIDTH_20		(RATE_MCS_CHAN_WIDTH_20_VAL << RATE_MCS_CHAN_WIDTH_POS)
+#define RATE_MCS_CHAN_WIDTH_40_VAL	1
+#define RATE_MCS_CHAN_WIDTH_40		(RATE_MCS_CHAN_WIDTH_40_VAL << RATE_MCS_CHAN_WIDTH_POS)
+#define RATE_MCS_CHAN_WIDTH_80_VAL	2
+#define RATE_MCS_CHAN_WIDTH_80		(RATE_MCS_CHAN_WIDTH_80_VAL << RATE_MCS_CHAN_WIDTH_POS)
+#define RATE_MCS_CHAN_WIDTH_160_VAL	3
+#define RATE_MCS_CHAN_WIDTH_160		(RATE_MCS_CHAN_WIDTH_160_VAL << RATE_MCS_CHAN_WIDTH_POS)
+#define RATE_MCS_CHAN_WIDTH_320_VAL	4
+#define RATE_MCS_CHAN_WIDTH_320		(RATE_MCS_CHAN_WIDTH_320_VAL << RATE_MCS_CHAN_WIDTH_POS)
 
 /* Bit 15-14: Antenna selection:
  * Bit 14: Ant A active
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
index 1c4e84932058..fdd8b01f09e4 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
@@ -367,7 +367,8 @@ enum iwl_rx_phy_eht_data1 {
 	/* number of EHT-LTF symbols 0 - 1 EHT-LTF, 1 - 2 EHT-LTFs, 2 - 4 EHT-LTFs,
 	 * 3 - 6 EHT-LTFs, 4 - 8 EHT-LTFs */
 	IWL_RX_PHY_DATA1_EHT_SIG_LTF_NUM		= 0x000000e0,
-	IWL_RX_PHY_DATA1_EHT_RU_ALLOC			= 0x0000ff00,
+	IWL_RX_PHY_DATA1_EHT_B0				= 0x00000100,
+	IWL_RX_PHY_DATA1_EHT_RU_B1_B7_ALLOC		= 0x0000fe00,
 };
 
 /* goes into Metadata DW 7 */
@@ -413,7 +414,7 @@ enum iwl_rx_phy_eht_data2 {
 	/* OFDM_RX_VECTOR_COMMON_RU_ALLOC_0_OUT */
 	IWL_RX_PHY_DATA2_EHT_MU_EXT_RU_ALLOC_A1	= 0x000001ff,
 	IWL_RX_PHY_DATA2_EHT_MU_EXT_RU_ALLOC_A2	= 0x0003fe00,
-	IWL_RX_PHY_DATA2_EHT_MU_EXT_RU_ALLOC_A3	= 0x01fc0000,
+	IWL_RX_PHY_DATA2_EHT_MU_EXT_RU_ALLOC_B1	= 0x07fc0000,
 
 	/* info type: EHT-TB-EXT */
 	IWL_RX_PHY_DATA2_EHT_TB_EXT_TRIG_SIGA1	= 0xffffffff,
@@ -423,19 +424,18 @@ enum iwl_rx_phy_eht_data2 {
 enum iwl_rx_phy_eht_data3 {
 	/* info type: EHT-MU-EXT */
 	/* OFDM_RX_VECTOR_COMMON_RU_ALLOC_1_OUT */
-	IWL_RX_PHY_DATA3_EHT_MU_EXT_RU_ALLOC_B1	= 0x000001ff,
-	IWL_RX_PHY_DATA3_EHT_MU_EXT_RU_ALLOC_B2	= 0x0003fe00,
-	IWL_RX_PHY_DATA3_EHT_MU_EXT_RU_ALLOC_B3	= 0x01fc0000,
+	IWL_RX_PHY_DATA3_EHT_MU_EXT_RU_ALLOC_B2	= 0x000001ff,
+	IWL_RX_PHY_DATA3_EHT_MU_EXT_RU_ALLOC_C1	= 0x0003fe00,
+	IWL_RX_PHY_DATA3_EHT_MU_EXT_RU_ALLOC_C2	= 0x07fc0000,
 };
 
 /* goes into Metadata DW 4 */
 enum iwl_rx_phy_eht_data4 {
 	/* info type: EHT-MU-EXT */
 	/* OFDM_RX_VECTOR_COMMON_RU_ALLOC_2_OUT */
-	IWL_RX_PHY_DATA4_EHT_MU_EXT_RU_ALLOC_C1	= 0x000001ff,
-	IWL_RX_PHY_DATA4_EHT_MU_EXT_RU_ALLOC_C2	= 0x0003fe00,
-	IWL_RX_PHY_DATA4_EHT_MU_EXT_RU_ALLOC_C3	= 0x01fc0000,
-	IWL_RX_PHY_DATA4_EHT_MU_EXT_SIGB_MCS	= 0x18000000,
+	IWL_RX_PHY_DATA4_EHT_MU_EXT_RU_ALLOC_D1	= 0x000001ff,
+	IWL_RX_PHY_DATA4_EHT_MU_EXT_RU_ALLOC_D2	= 0x0003fe00,
+	IWL_RX_PHY_DATA4_EHT_MU_EXT_SIGB_MCS	= 0x000c0000,
 };
 
 /* goes into Metadata DW 16 */
@@ -673,22 +673,31 @@ struct iwl_rx_mpdu_desc {
 	 * @mac_phy_idx: MAC/PHY index
 	 */
 	u8 mac_phy_idx;
-	/* DW4 - carries csum data only when rpa_en == 1 */
-	/**
-	 * @raw_csum: raw checksum (alledgedly unreliable)
-	 */
-	__le16 raw_csum;
-
+	/* DW4 */
 	union {
+		struct {
+			/* carries csum data only when rpa_en == 1 */
+			/**
+			 * @raw_csum: raw checksum (alledgedly unreliable)
+			 */
+			__le16 raw_csum;
+
+			union {
+				/**
+				 * @l3l4_flags: &enum iwl_rx_l3l4_flags
+				 */
+				__le16 l3l4_flags;
+
+				/**
+				 * @phy_data4: depends on info type, see phy_data1
+				 */
+				__le16 phy_data4;
+			};
+		};
 		/**
-		 * @l3l4_flags: &enum iwl_rx_l3l4_flags
-		 */
-		__le16 l3l4_flags;
-
-		/**
-		 * @phy_data4: depends on info type, see phy_data1
+		 * @phy_eht_data4: depends on info type, see phy_data1
 		 */
-		__le16 phy_data4;
+		__le32 phy_eht_data4;
 	};
 	/* DW5 */
 	/**
@@ -725,7 +734,7 @@ struct iwl_rx_mpdu_desc {
 #define RX_NO_DATA_INFO_TYPE_RX_ERR	1
 #define RX_NO_DATA_INFO_TYPE_NDP	2
 #define RX_NO_DATA_INFO_TYPE_MU_UNMATCHED	3
-#define RX_NO_DATA_INFO_TYPE_HE_TB_UNMATCHED	4
+#define RX_NO_DATA_INFO_TYPE_TB_UNMATCHED	4
 
 #define RX_NO_DATA_INFO_ERR_POS		8
 #define RX_NO_DATA_INFO_ERR_MSK		(0xff << RX_NO_DATA_INFO_ERR_POS)
@@ -743,6 +752,35 @@ struct iwl_rx_mpdu_desc {
 #define RX_NO_DATA_RX_VEC0_VHT_NSTS_MSK	0x38000000
 #define RX_NO_DATA_RX_VEC2_EHT_NSTS_MSK	0x00f00000
 
+/* content of OFDM_RX_VECTOR_USIG_A1_OUT */
+enum iwl_rx_usig_a1 {
+	IWL_RX_USIG_A1_ENHANCED_WIFI_VER_ID	= 0x00000007,
+	IWL_RX_USIG_A1_BANDWIDTH		= 0x00000038,
+	IWL_RX_USIG_A1_UL_FLAG			= 0x00000040,
+	IWL_RX_USIG_A1_BSS_COLOR		= 0x00001f80,
+	IWL_RX_USIG_A1_TXOP_DURATION		= 0x000fe000,
+	IWL_RX_USIG_A1_DISREGARD		= 0x01f00000,
+	IWL_RX_USIG_A1_VALIDATE			= 0x02000000,
+	IWL_RX_USIG_A1_EHT_BW320_SLOT		= 0x04000000,
+	IWL_RX_USIG_A1_EHT_TYPE			= 0x18000000,
+	IWL_RX_USIG_A1_RDY			= 0x80000000,
+};
+
+/* content of OFDM_RX_VECTOR_USIG_A2_EHT_OUT */
+enum iwl_rx_usig_a2_eht {
+	IWL_RX_USIG_A2_EHT_PPDU_TYPE		= 0x00000003,
+	IWL_RX_USIG_A2_EHT_USIG2_VALIDATE_B2	= 0x00000004,
+	IWL_RX_USIG_A2_EHT_PUNC_CHANNEL		= 0x000000f8,
+	IWL_RX_USIG_A2_EHT_USIG2_VALIDATE_B8	= 0x00000100,
+	IWL_RX_USIG_A2_EHT_SIG_MCS		= 0x00000600,
+	IWL_RX_USIG_A2_EHT_SIG_SYM_NUM		= 0x0000f800,
+	IWL_RX_USIG_A2_EHT_TRIG_SPATIAL_REUSE_1 = 0x000f0000,
+	IWL_RX_USIG_A2_EHT_TRIG_SPATIAL_REUSE_2 = 0x00f00000,
+	IWL_RX_USIG_A2_EHT_TRIG_USIG2_DISREGARD	= 0x1f000000,
+	IWL_RX_USIG_A2_EHT_CRC_OK		= 0x40000000,
+	IWL_RX_USIG_A2_EHT_RDY			= 0x80000000,
+};
+
 /**
  * struct iwl_rx_no_data - RX no data descriptor
  * @info: 7:0 frame type, 15:8 RX error type
@@ -780,7 +818,7 @@ struct iwl_rx_no_data {
  * @rx_vec: DW-12:9 raw RX vectors from DSP according to modulation type.
  *	for VHT: OFDM_RX_VECTOR_SIGA1_OUT, OFDM_RX_VECTOR_SIGA2_OUT
  *	for HE: OFDM_RX_VECTOR_HE_SIGA1_OUT, OFDM_RX_VECTOR_HE_SIGA2_OUT
- *	for EHT: OFDM_RX_VECTOR_USIG_A1_OUT, OFDM_RX_VECTOR_USIG_A2_OUT,
+ *	for EHT: OFDM_RX_VECTOR_USIG_A1_OUT, OFDM_RX_VECTOR_USIG_A2_EHT_OUT,
  *	OFDM_RX_VECTOR_EHT_OUT, OFDM_RX_VECTOR_EHT_USER_FIELD_OUT
  */
 struct iwl_rx_no_data_ver_3 {
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
index 7ba0e3409199..ec96ba053a5c 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
@@ -709,10 +709,13 @@ enum iwl_umac_scan_general_flags_v2 {
  *     should be aware of a P2P GO operation on the 2GHz band.
  * @IWL_UMAC_SCAN_GEN_PARAMS_FLAGS2_RESPECT_P2P_GO_HB: scan event scheduling
  *     should be aware of a P2P GO operation on the 5GHz or 6GHz band.
+ * @IWL_UMAC_SCAN_GEN_PARAMS_FLAGS2_DONT_TOGGLE_ANT: don't toggle between
+ *     valid antennas, and use the same antenna as in previous scan
  */
 enum iwl_umac_scan_general_params_flags2 {
 	IWL_UMAC_SCAN_GEN_PARAMS_FLAGS2_RESPECT_P2P_GO_LB = BIT(0),
 	IWL_UMAC_SCAN_GEN_PARAMS_FLAGS2_RESPECT_P2P_GO_HB = BIT(1),
+	IWL_UMAC_SCAN_GEN_PARAMS_FLAGS2_DONT_TOGGLE_ANT   = BIT(2),
 };
 
 /**
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h
index ecc6706f66ed..97edf5477ba7 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h
@@ -800,7 +800,7 @@ enum iwl_mac_beacon_flags {
  *	is &enum iwl_mac_beacon_flags.
  * @short_ssid: Short SSID
  * @reserved: reserved
- * @template_id: currently equal to the mac context id of the coresponding mac.
+ * @link_id: the firmware id of the link that will use this beacon
  * @tim_idx: the offset of the tim IE in the beacon
  * @tim_size: the length of the tim IE
  * @ecsa_offset: offset to the ECSA IE if present
@@ -812,15 +812,17 @@ struct iwl_mac_beacon_cmd {
 	__le16 flags;
 	__le32 short_ssid;
 	__le32 reserved;
-	__le32 template_id;
+	__le32 link_id;
 	__le32 tim_idx;
 	__le32 tim_size;
 	__le32 ecsa_offset;
 	__le32 csa_offset;
 	struct ieee80211_hdr frame[];
 } __packed; /* BEACON_TEMPLATE_CMD_API_S_VER_10,
-	       BEACON_TEMPLATE_CMD_API_S_VER_11,
-	       BEACON_TEMPLATE_CMD_API_S_VER_12 */
+	     * BEACON_TEMPLATE_CMD_API_S_VER_11,
+	     * BEACON_TEMPLATE_CMD_API_S_VER_12,
+	     * BEACON_TEMPLATE_CMD_API_S_VER_13
+	     */
 
 struct iwl_beacon_notif {
 	struct iwl_mvm_tx_resp beacon_notify_hdr;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
index abf49022edbe..d9faaae01abd 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
@@ -1038,7 +1038,7 @@ iwl_dump_ini_prph_mac_iter(struct iwl_fw_runtime *fwrt,
 	range->range_data_size = reg->dev_addr.size;
 	for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4) {
 		prph_val = iwl_read_prph(fwrt->trans, addr + i);
-		if (prph_val == 0x5a5a5a5a)
+		if ((prph_val & ~0xf) == 0xa5a5a5a0)
 			return -EBUSY;
 		*val++ = cpu_to_le32(prph_val);
 	}
@@ -1388,13 +1388,13 @@ static void iwl_ini_get_rxf_data(struct iwl_fw_runtime *fwrt,
 	if (!data)
 		return;
 
+	memset(data, 0, sizeof(*data));
+
 	/* make sure only one bit is set in only one fid */
 	if (WARN_ONCE(hweight_long(fid1) + hweight_long(fid2) != 1,
 		      "fid1=%x, fid2=%x\n", fid1, fid2))
 		return;
 
-	memset(data, 0, sizeof(*data));
-
 	if (fid1) {
 		fifo_idx = ffs(fid1) - 1;
 		if (WARN_ONCE(fifo_idx >= MAX_NUM_LMAC, "fifo_idx=%d\n",
@@ -1562,7 +1562,7 @@ iwl_dump_ini_dbgi_sram_iter(struct iwl_fw_runtime *fwrt,
 		prph_data = iwl_read_prph_no_grab(fwrt->trans, (i % 2) ?
 					  DBGI_SRAM_TARGET_ACCESS_RDATA_MSB :
 					  DBGI_SRAM_TARGET_ACCESS_RDATA_LSB);
-		if (prph_data == 0x5a5a5a5a) {
+		if ((prph_data & ~0xf) == 0xa5a5a5a0) {
 			iwl_trans_release_nic_access(fwrt->trans);
 			return -EBUSY;
 		}
@@ -2320,6 +2320,36 @@ static u32 iwl_dump_ini_info(struct iwl_fw_runtime *fwrt,
 	return entry->size;
 }
 
+static u32 iwl_dump_ini_file_name_info(struct iwl_fw_runtime *fwrt,
+				       struct list_head *list)
+{
+	struct iwl_fw_ini_dump_entry *entry;
+	struct iwl_dump_file_name_info *tlv;
+	u32 len = strnlen(fwrt->trans->dbg.dump_file_name_ext,
+			  IWL_FW_INI_MAX_NAME);
+
+	if (!fwrt->trans->dbg.dump_file_name_ext_valid)
+		return 0;
+
+	entry = vzalloc(sizeof(*entry) + sizeof(*tlv) + len);
+	if (!entry)
+		return 0;
+
+	entry->size = sizeof(*tlv) + len;
+
+	tlv = (void *)entry->data;
+	tlv->type = cpu_to_le32(IWL_INI_DUMP_NAME_TYPE);
+	tlv->len = cpu_to_le32(len);
+	memcpy(tlv->data, fwrt->trans->dbg.dump_file_name_ext, len);
+
+	/* add the dump file name extension tlv to the list */
+	list_add_tail(&entry->list, list);
+
+	fwrt->trans->dbg.dump_file_name_ext_valid = false;
+
+	return entry->size;
+}
+
 static const struct iwl_dump_ini_mem_ops iwl_dump_ini_region_ops[] = {
 	[IWL_FW_INI_REGION_INVALID] = {},
 	[IWL_FW_INI_REGION_INTERNAL_BUFFER] = {
@@ -2495,8 +2525,10 @@ static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
 		size += iwl_dump_ini_mem(fwrt, list, &reg_data,
 					 &iwl_dump_ini_region_ops[IWL_FW_INI_REGION_DRAM_IMR]);
 
-	if (size)
+	if (size) {
+		size += iwl_dump_ini_file_name_info(fwrt, list);
 		size += iwl_dump_ini_info(fwrt, trigger, list);
+	}
 
 	return size;
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
index 43e997283db0..607e07ed2477 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
@@ -317,8 +317,10 @@ static void *iwl_dbgfs_fw_info_seq_next(struct seq_file *seq,
 	const struct iwl_fw *fw = priv->fwrt->fw;
 
 	*pos = ++state->pos;
-	if (*pos >= fw->ucode_capa.n_cmd_versions)
+	if (*pos >= fw->ucode_capa.n_cmd_versions) {
+		kfree(state);
 		return NULL;
+	}
 
 	return state;
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dump.c b/drivers/net/wireless/intel/iwlwifi/fw/dump.c
index 792f7fee1840..f86f7b4baa18 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/dump.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dump.c
@@ -14,6 +14,13 @@
 #include "iwl-csr.h"
 #include "pnvm.h"
 
+#define FW_ASSERT_LMAC_FATAL			0x70
+#define FW_ASSERT_LMAC2_FATAL			0x72
+#define FW_ASSERT_UMAC_FATAL			0x71
+#define UMAC_RT_NMI_LMAC2_FATAL			0x72
+#define RT_NMI_INTERRUPT_OTHER_LMAC_FATAL	0x73
+#define FW_ASSERT_NMI_UNKNOWN			0x84
+
 /*
  * Note: This structure is read from the device with IO accesses,
  * and the reading already does the endian conversion. As it is
@@ -96,6 +103,17 @@ struct iwl_umac_error_event_table {
 #define ERROR_START_OFFSET  (1 * sizeof(u32))
 #define ERROR_ELEM_SIZE     (7 * sizeof(u32))
 
+static bool iwl_fwrt_if_errorid_other_cpu(u32 err_id)
+{
+	err_id &= 0xFF;
+
+	if ((err_id >= FW_ASSERT_LMAC_FATAL &&
+	     err_id <= RT_NMI_INTERRUPT_OTHER_LMAC_FATAL) ||
+	    err_id == FW_ASSERT_NMI_UNKNOWN)
+		return  true;
+	return false;
+}
+
 static void iwl_fwrt_dump_umac_error_log(struct iwl_fw_runtime *fwrt)
 {
 	struct iwl_trans *trans = fwrt->trans;
@@ -113,6 +131,13 @@ static void iwl_fwrt_dump_umac_error_log(struct iwl_fw_runtime *fwrt)
 	if (table.valid)
 		fwrt->dump.umac_err_id = table.error_id;
 
+	if (!iwl_fwrt_if_errorid_other_cpu(fwrt->dump.umac_err_id) &&
+	    !fwrt->trans->dbg.dump_file_name_ext_valid) {
+		fwrt->trans->dbg.dump_file_name_ext_valid = true;
+		snprintf(fwrt->trans->dbg.dump_file_name_ext, IWL_FW_INI_MAX_NAME,
+			 "0x%x", fwrt->dump.umac_err_id);
+	}
+
 	if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
 		IWL_ERR(trans, "Start IWL Error Log Dump:\n");
 		IWL_ERR(trans, "Transport status: 0x%08lX, valid: %d\n",
@@ -189,6 +214,13 @@ static void iwl_fwrt_dump_lmac_error_log(struct iwl_fw_runtime *fwrt, u8 lmac_nu
 	if (table.valid)
 		fwrt->dump.lmac_err_id[lmac_num] = table.error_id;
 
+	if (!iwl_fwrt_if_errorid_other_cpu(fwrt->dump.lmac_err_id[lmac_num]) &&
+	    !fwrt->trans->dbg.dump_file_name_ext_valid) {
+		fwrt->trans->dbg.dump_file_name_ext_valid = true;
+		snprintf(fwrt->trans->dbg.dump_file_name_ext, IWL_FW_INI_MAX_NAME,
+			 "0x%x", fwrt->dump.lmac_err_id[lmac_num]);
+	}
+
 	if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
 		IWL_ERR(trans, "Start IWL Error Log Dump:\n");
 		IWL_ERR(trans, "Transport status: 0x%08lX, valid: %d\n",
@@ -274,6 +306,16 @@ static void iwl_fwrt_dump_tcm_error_log(struct iwl_fw_runtime *fwrt, int idx)
 
 	iwl_trans_read_mem_bytes(trans, base, &table, sizeof(table));
 
+	if (table.valid)
+		fwrt->dump.tcm_err_id[idx] = table.error_id;
+
+	if (!iwl_fwrt_if_errorid_other_cpu(fwrt->dump.tcm_err_id[idx]) &&
+	    !fwrt->trans->dbg.dump_file_name_ext_valid) {
+		fwrt->trans->dbg.dump_file_name_ext_valid = true;
+		snprintf(fwrt->trans->dbg.dump_file_name_ext, IWL_FW_INI_MAX_NAME,
+			 "0x%x", fwrt->dump.tcm_err_id[idx]);
+	}
+
 	IWL_ERR(fwrt, "TCM%d status:\n", idx + 1);
 	IWL_ERR(fwrt, "0x%08X | error ID\n", table.error_id);
 	IWL_ERR(fwrt, "0x%08X | tcm branchlink2\n", table.blink2);
@@ -337,6 +379,16 @@ static void iwl_fwrt_dump_rcm_error_log(struct iwl_fw_runtime *fwrt, int idx)
 
 	iwl_trans_read_mem_bytes(trans, base, &table, sizeof(table));
 
+	if (table.valid)
+		fwrt->dump.rcm_err_id[idx] = table.error_id;
+
+	if (!iwl_fwrt_if_errorid_other_cpu(fwrt->dump.rcm_err_id[idx]) &&
+	    !fwrt->trans->dbg.dump_file_name_ext_valid) {
+		fwrt->trans->dbg.dump_file_name_ext_valid = true;
+		snprintf(fwrt->trans->dbg.dump_file_name_ext, IWL_FW_INI_MAX_NAME,
+			 "0x%x", fwrt->dump.rcm_err_id[idx]);
+	}
+
 	IWL_ERR(fwrt, "RCM%d status:\n", idx + 1);
 	IWL_ERR(fwrt, "0x%08X | error ID\n", table.error_id);
 	IWL_ERR(fwrt, "0x%08X | rcm branchlink2\n", table.blink2);
@@ -432,6 +484,9 @@ static void iwl_fwrt_dump_fseq_regs(struct iwl_fw_runtime *fwrt)
 
 void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt)
 {
+	struct iwl_pc_data *pc_data;
+	u8 count;
+
 	if (!test_bit(STATUS_DEVICE_ENABLED, &fwrt->trans->status)) {
 		IWL_ERR(fwrt,
 			"DEVICE_ENABLED bit is not set. Aborting dump.\n");
@@ -444,10 +499,20 @@ void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt)
 	iwl_fwrt_dump_umac_error_log(fwrt);
 	iwl_fwrt_dump_tcm_error_log(fwrt, 0);
 	iwl_fwrt_dump_rcm_error_log(fwrt, 0);
-	iwl_fwrt_dump_tcm_error_log(fwrt, 1);
-	iwl_fwrt_dump_rcm_error_log(fwrt, 1);
+	if (fwrt->trans->dbg.tcm_error_event_table[1])
+		iwl_fwrt_dump_tcm_error_log(fwrt, 1);
+	if (fwrt->trans->dbg.rcm_error_event_table[1])
+		iwl_fwrt_dump_rcm_error_log(fwrt, 1);
 	iwl_fwrt_dump_iml_error_log(fwrt);
 	iwl_fwrt_dump_fseq_regs(fwrt);
+	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_22000) {
+		pc_data = fwrt->trans->dbg.pc_data;
+		for (count = 0; count < fwrt->trans->dbg.num_pc;
+		     count++, pc_data++)
+			IWL_ERR(fwrt, "%s: 0x%x\n",
+				pc_data->pc_name,
+				pc_data->pc_address);
+	}
 
 	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
 		u32 scratch = iwl_read32(fwrt->trans, CSR_FUNC_SCRATCH);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h b/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
index c62576e442bd..f5e08988dc7b 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2014, 2018-2021 Intel Corporation
+ * Copyright (C) 2014, 2018-2022 Intel Corporation
  * Copyright (C) 2014-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -76,6 +76,18 @@ struct iwl_fw_error_dump_data {
 } __packed;
 
 /**
+ * struct iwl_dump_file_name_info - data for dump file name addition
+ * @type: region type with reserved bits
+ * @len: the length of file name string to be added to dump file
+ * @data: the string need to be added to dump file
+ */
+struct iwl_dump_file_name_info {
+	__le32 type;
+	__le32 len;
+	__u8 data[];
+} __packed;
+
+/**
  * struct iwl_fw_error_dump_file - the layout of the header of the file
  * @barker: must be %IWL_FW_ERROR_DUMP_BARKER
  * @file_len: the length of all the file starting from %barker
@@ -231,6 +243,9 @@ struct iwl_fw_error_dump_mem {
 /* Use bit 31 as dump info type to avoid colliding with region types */
 #define IWL_INI_DUMP_INFO_TYPE BIT(31)
 
+/* Use bit 31 and bit 24 as dump name type to avoid colliding with region types */
+#define IWL_INI_DUMP_NAME_TYPE (BIT(31) | BIT(24))
+
 /**
  * struct iwl_fw_error_dump_data - data for one type
  * @type: &enum iwl_fw_ini_region_type
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h
index a7817d952022..cddf09d6be1c 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/file.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h
@@ -101,8 +101,10 @@ enum iwl_ucode_tlv_type {
 
 	IWL_UCODE_TLV_SEC_TABLE_ADDR		= 66,
 	IWL_UCODE_TLV_D3_KEK_KCK_ADDR		= 67,
+	IWL_UCODE_TLV_CURRENT_PC		= 68,
 
 	IWL_UCODE_TLV_FW_NUM_STATIONS		= IWL_UCODE_TLV_CONST_BASE + 0,
+	IWL_UCODE_TLV_FW_NUM_BEACONS		= IWL_UCODE_TLV_CONST_BASE + 2,
 
 	IWL_UCODE_TLV_TYPE_DEBUG_INFO		= IWL_UCODE_TLV_DEBUG_BASE + 0,
 	IWL_UCODE_TLV_TYPE_BUFFER_ALLOCATION	= IWL_UCODE_TLV_DEBUG_BASE + 1,
@@ -455,6 +457,11 @@ enum iwl_ucode_tlv_capa {
 	IWL_UCODE_TLV_CAPA_BIGTK_SUPPORT		= (__force iwl_ucode_tlv_capa_t)100,
 	IWL_UCODE_TLV_CAPA_DRAM_FRAG_SUPPORT		= (__force iwl_ucode_tlv_capa_t)104,
 	IWL_UCODE_TLV_CAPA_DUMP_COMPLETE_SUPPORT	= (__force iwl_ucode_tlv_capa_t)105,
+	IWL_UCODE_TLV_CAPA_SYNCED_TIME			= (__force iwl_ucode_tlv_capa_t)106,
+	IWL_UCODE_TLV_CAPA_TIME_SYNC_BOTH_FTM_TM        = (__force iwl_ucode_tlv_capa_t)108,
+	IWL_UCODE_TLV_CAPA_BIGTK_TX_SUPPORT		= (__force iwl_ucode_tlv_capa_t)109,
+	IWL_UCODE_TLV_CAPA_MLD_API_SUPPORT		= (__force iwl_ucode_tlv_capa_t)110,
+	IWL_UCODE_TLV_CAPA_SCAN_DONT_TOGGLE_ANT         = (__force iwl_ucode_tlv_capa_t)111,
 
 #ifdef __CHECKER__
 	/* sparse says it cannot increment the previous enum member */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/img.h b/drivers/net/wireless/intel/iwlwifi/fw/img.h
index f878ac508801..8d0d58d61892 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/img.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/img.h
@@ -51,6 +51,7 @@ struct iwl_ucode_capabilities {
 	u32 error_log_addr;
 	u32 error_log_size;
 	u32 num_stations;
+	u32 num_beacons;
 	unsigned long _api[BITS_TO_LONGS(NUM_IWL_UCODE_TLV_API)];
 	unsigned long _capa[BITS_TO_LONGS(NUM_IWL_UCODE_TLV_CAPA)];
 
@@ -182,10 +183,10 @@ struct iwl_dump_exclude {
  * @enhance_sensitivity_table: device can do enhanced sensitivity.
  * @init_evtlog_ptr: event log offset for init ucode.
  * @init_evtlog_size: event log size for init ucode.
- * @init_errlog_ptr: error log offfset for init ucode.
+ * @init_errlog_ptr: error log offset for init ucode.
  * @inst_evtlog_ptr: event log offset for runtime ucode.
  * @inst_evtlog_size: event log size for runtime ucode.
- * @inst_errlog_ptr: error log offfset for runtime ucode.
+ * @inst_errlog_ptr: error log offset for runtime ucode.
  * @type: firmware type (&enum iwl_fw_type)
  * @human_readable: human readable version
  *	we get the ALIVE from the uCode
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c b/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c
index b6d3ac6ed440..c6f2672fdc73 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright(c) 2020-2021 Intel Corporation
+ * Copyright(c) 2020-2022 Intel Corporation
  */
 
 #include "iwl-drv.h"
@@ -318,7 +318,6 @@ parse:
 	kfree(data);
 
 skip_parse:
-	data = NULL;
 	/* now try to get the reduce power table, if not loaded yet */
 	if (!trans->reduce_power_loaded) {
 		data = iwl_uefi_get_reduced_power(trans, &len);
@@ -329,19 +328,16 @@ skip_parse:
 			 * trying again over and over.
 			 */
 			trans->reduce_power_loaded = true;
-
-			goto skip_reduce_power;
+		} else {
+			ret = iwl_trans_set_reduce_power(trans, data, len);
+			if (ret)
+				IWL_DEBUG_FW(trans,
+					     "Failed to set reduce power table %d\n",
+					     ret);
+			kfree(data);
 		}
 	}
 
-	ret = iwl_trans_set_reduce_power(trans, data, len);
-	if (ret)
-		IWL_DEBUG_FW(trans,
-			     "Failed to set reduce power table %d\n",
-			     ret);
-	kfree(data);
-
-skip_reduce_power:
 	iwl_init_notification_wait(notif_wait, &pnvm_wait,
 				   ntf_cmds, ARRAY_SIZE(ntf_cmds),
 				   iwl_pnvm_complete_fn, trans);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/rs.c b/drivers/net/wireless/intel/iwlwifi/fw/rs.c
index e128d2e07f38..b09e68dbf5a9 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/rs.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/rs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2021 Intel Corporation
+ * Copyright (C) 2021-2022 Intel Corporation
  */
 
 #include <net/mac80211.h>
@@ -126,7 +126,7 @@ u32 iwl_new_rate_from_v1(u32 rate_v1)
 		   rate_v1 & RATE_MCS_HE_MSK_V1) {
 		rate_v2 |= rate_v1 & RATE_VHT_MCS_RATE_CODE_MSK;
 
-		rate_v2 |= rate_v1 & RATE_VHT_MCS_MIMO2_MSK;
+		rate_v2 |= rate_v1 & RATE_MCS_NSS_MSK;
 
 		if (rate_v1 & RATE_MCS_HE_MSK_V1) {
 			u32 he_type_bits = rate_v1 & RATE_MCS_HE_TYPE_MSK_V1;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
index d3cb1ae68a96..df689a9b7e2c 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
@@ -24,6 +24,8 @@ struct iwl_fw_runtime_ops {
 };
 
 #define MAX_NUM_LMAC 2
+#define MAX_NUM_TCM 2
+#define MAX_NUM_RCM 2
 struct iwl_fwrt_shared_mem_cfg {
 	int num_lmacs;
 	int num_txfifo_entries;
@@ -129,6 +131,8 @@ struct iwl_fw_runtime {
 		unsigned long non_collect_ts_start[IWL_FW_INI_TIME_POINT_NUM];
 		u32 *d3_debug_data;
 		u32 lmac_err_id[MAX_NUM_LMAC];
+		u32 tcm_err_id[MAX_NUM_TCM];
+		u32 rcm_err_id[MAX_NUM_RCM];
 		u32 umac_err_id;
 
 		struct iwl_txf_iter_data txf_iter_data;
@@ -161,6 +165,7 @@ struct iwl_fw_runtime {
 	struct iwl_ppag_chain ppag_chains[IWL_NUM_CHAIN_LIMITS];
 	u32 ppag_flags;
 	u32 ppag_ver;
+	bool ppag_table_valid;
 	struct iwl_sar_offset_mapping_cmd sgom_table;
 	bool sgom_enabled;
 	u8 reduced_power_flags;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/uefi.c b/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
index 0b6f694cf30d..01afea33c38c 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
@@ -222,7 +222,7 @@ void *iwl_uefi_get_reduced_power(struct iwl_trans *trans, size_t *len)
 		return ERR_PTR(-ENOMEM);
 
 	status = efi.get_variable(IWL_UEFI_REDUCED_POWER_NAME, &IWL_EFI_VAR_GUID,
-				  NULL, &package_size, data);
+				  NULL, &package_size, package);
 	if (status != EFI_SUCCESS) {
 		IWL_DEBUG_FW(trans,
 			     "Reduced Power UEFI variable not found 0x%lx (len %lu)\n",
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
index cfa5e1b3c3f6..411b7d4fcc9a 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
@@ -469,6 +469,7 @@ struct iwl_dev_info {
 	u16 mac_type;
 	u16 rf_type;
 	u8 mac_step;
+	u8 rf_step;
 	u8 rf_id;
 	u8 no_160;
 	u8 cores;
@@ -639,26 +640,40 @@ extern const struct iwl_cfg iwl_cfg_ma_a0_gf4_a0;
 extern const struct iwl_cfg iwl_cfg_ma_a0_mr_a0;
 extern const struct iwl_cfg iwl_cfg_ma_a0_ms_a0;
 extern const struct iwl_cfg iwl_cfg_ma_a0_fm_a0;
+extern const struct iwl_cfg iwl_cfg_ma_b0_hr_b0;
+extern const struct iwl_cfg iwl_cfg_ma_b0_gf_a0;
+extern const struct iwl_cfg iwl_cfg_ma_b0_gf4_a0;
+extern const struct iwl_cfg iwl_cfg_ma_b0_mr_a0;
+extern const struct iwl_cfg iwl_cfg_ma_b0_fm_a0;
 extern const struct iwl_cfg iwl_cfg_snj_a0_mr_a0;
 extern const struct iwl_cfg iwl_cfg_snj_a0_ms_a0;
 extern const struct iwl_cfg iwl_cfg_so_a0_hr_a0;
 extern const struct iwl_cfg iwl_cfg_so_a0_ms_a0;
 extern const struct iwl_cfg iwl_cfg_quz_a0_hr_b0;
+extern const struct iwl_cfg iwl_cfg_bz_a0_hr_a0;
 extern const struct iwl_cfg iwl_cfg_bz_a0_hr_b0;
 extern const struct iwl_cfg iwl_cfg_bz_a0_gf_a0;
 extern const struct iwl_cfg iwl_cfg_bz_a0_gf4_a0;
 extern const struct iwl_cfg iwl_cfg_bz_a0_mr_a0;
 extern const struct iwl_cfg iwl_cfg_bz_a0_fm_a0;
 extern const struct iwl_cfg iwl_cfg_bz_a0_fm4_a0;
+extern const struct iwl_cfg iwl_cfg_bz_a0_fm_b0;
+extern const struct iwl_cfg iwl_cfg_bz_a0_fm4_b0;
 extern const struct iwl_cfg iwl_cfg_gl_a0_fm_a0;
 extern const struct iwl_cfg iwl_cfg_gl_b0_fm_b0;
 extern const struct iwl_cfg iwl_cfg_bz_z0_gf_a0;
 extern const struct iwl_cfg iwl_cfg_bnj_a0_fm_a0;
 extern const struct iwl_cfg iwl_cfg_bnj_a0_fm4_a0;
 extern const struct iwl_cfg iwl_cfg_bnj_a0_gf_a0;
+extern const struct iwl_cfg iwl_cfg_bnj_b0_gf_a0;
 extern const struct iwl_cfg iwl_cfg_bnj_a0_gf4_a0;
+extern const struct iwl_cfg iwl_cfg_bnj_b0_gf4_a0;
+extern const struct iwl_cfg iwl_cfg_bnj_a0_hr_a0;
 extern const struct iwl_cfg iwl_cfg_bnj_a0_hr_b0;
+extern const struct iwl_cfg iwl_cfg_bnj_b0_hr_a0;
+extern const struct iwl_cfg iwl_cfg_bnj_b0_hr_b0;
 extern const struct iwl_cfg iwl_cfg_bnj_b0_fm_b0;
+extern const struct iwl_cfg iwl_cfg_bnj_b0_fm4_b0;
 #endif /* CONFIG_IWLMVM */
 
 #endif /* __IWL_CONFIG_H__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
index 3e1f011e93aa..587368a0ad4a 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
@@ -102,6 +102,8 @@
 #define CSR_LTR_LONG_VAL_AD_SNOOP_VAL		0x000003ff
 #define CSR_LTR_LONG_VAL_AD_SCALE_USEC		2
 
+#define CSR_LTR_LAST_MSG			(CSR_BASE + 0x0DC)
+
 /* GIO Chicken Bits (PCI Express bus link power management) */
 #define CSR_GIO_CHICKEN_BITS    (CSR_BASE+0x100)
 
@@ -309,6 +311,8 @@ enum {
 	SILICON_A_STEP = 0,
 	SILICON_B_STEP,
 	SILICON_C_STEP,
+	SILICON_D_STEP,
+	SILICON_E_STEP,
 	SILICON_Z_STEP = 0xf,
 };
 
@@ -348,6 +352,7 @@ enum {
 #define CSR_HW_RF_ID_TYPE_HRCDB		(0x00109F00)
 #define CSR_HW_RF_ID_TYPE_GF		(0x0010D000)
 #define CSR_HW_RF_ID_TYPE_GF4		(0x0010E000)
+#define CSR_HW_RF_ID_TYPE_MS		(0x00111000)
 
 /* HW_RF CHIP STEP  */
 #define CSR_HW_RF_STEP(_val) (((_val) >> 8) & 0xF)
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
index 48e7376a5fea..898d5dcf1012 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
@@ -138,6 +138,12 @@ static int iwl_dbg_tlv_alloc_buf_alloc(struct iwl_trans *trans,
 	    alloc_id != IWL_FW_INI_ALLOCATION_ID_DBGC1)
 		goto err;
 
+	if (buf_location == IWL_FW_INI_LOCATION_DRAM_PATH &&
+	    alloc->req_size == 0) {
+		IWL_ERR(trans, "WRT: Invalid DRAM buffer allocation requested size (0)\n");
+		return -EINVAL;
+	}
+
 	trans->dbg.fw_mon_cfg[alloc_id] = *alloc;
 
 	return 0;
@@ -350,9 +356,9 @@ void iwl_dbg_tlv_alloc(struct iwl_trans *trans, const struct iwl_ucode_tlv *tlv,
 
 	ret = dbg_tlv_alloc[tlv_idx](trans, tlv);
 	if (ret) {
-		IWL_ERR(trans,
-			"WRT: Failed to allocate TLV 0x%x, ret %d, (ext=%d)\n",
-			type, ret, ext);
+		IWL_WARN(trans,
+			 "WRT: Failed to allocate TLV 0x%x, ret %d, (ext=%d)\n",
+			 type, ret, ext);
 		goto out_err;
 	}
 
@@ -797,7 +803,7 @@ static void iwl_dbg_tlv_update_drams(struct iwl_fw_runtime *fwrt)
 		if (!ret)
 			dram_alloc = true;
 		else
-			IWL_WARN(fwrt,
+			IWL_INFO(fwrt,
 				 "WRT: Failed to set DRAM buffer for alloc id %d, ret=%d\n",
 				 i, ret);
 	}
@@ -1218,11 +1224,12 @@ iwl_dbg_tlv_tp_trigger(struct iwl_fw_runtime *fwrt, bool sync,
 		}
 
 		fwrt->trans->dbg.restart_required = FALSE;
-		IWL_DEBUG_INFO(fwrt, "WRT: tp %d, reset_fw %d\n",
-			       tp, dump_data.trig->reset_fw);
-		IWL_DEBUG_INFO(fwrt, "WRT: restart_required %d, last_tp_resetfw %d\n",
-			       fwrt->trans->dbg.restart_required,
-			       fwrt->trans->dbg.last_tp_resetfw);
+		IWL_DEBUG_FW(fwrt, "WRT: tp %d, reset_fw %d\n",
+			     tp, dump_data.trig->reset_fw);
+		IWL_DEBUG_FW(fwrt,
+			     "WRT: restart_required %d, last_tp_resetfw %d\n",
+			     fwrt->trans->dbg.restart_required,
+			     fwrt->trans->dbg.last_tp_resetfw);
 
 		if (fwrt->trans->trans_cfg->device_family ==
 		    IWL_DEVICE_FAMILY_9000) {
@@ -1235,18 +1242,19 @@ iwl_dbg_tlv_tp_trigger(struct iwl_fw_runtime *fwrt, bool sync,
 			IWL_DEBUG_FW(fwrt, "WRT: FW_ASSERT due to reset_fw_mode-no restart\n");
 		} else if (le32_to_cpu(dump_data.trig->reset_fw) ==
 			   IWL_FW_INI_RESET_FW_MODE_STOP_AND_RELOAD_FW) {
-			IWL_DEBUG_INFO(fwrt, "WRT: stop and reload firmware\n");
+			IWL_DEBUG_FW(fwrt, "WRT: stop and reload firmware\n");
 			fwrt->trans->dbg.restart_required = TRUE;
 		} else if (le32_to_cpu(dump_data.trig->reset_fw) ==
 			   IWL_FW_INI_RESET_FW_MODE_STOP_FW_ONLY) {
-			IWL_DEBUG_INFO(fwrt, "WRT: stop only and no reload firmware\n");
+			IWL_DEBUG_FW(fwrt,
+				     "WRT: stop only and no reload firmware\n");
 			fwrt->trans->dbg.restart_required = FALSE;
 			fwrt->trans->dbg.last_tp_resetfw =
 				le32_to_cpu(dump_data.trig->reset_fw);
 		} else if (le32_to_cpu(dump_data.trig->reset_fw) ==
 			   IWL_FW_INI_RESET_FW_MODE_NOTHING) {
-			IWL_DEBUG_INFO(fwrt,
-				       "WRT: nothing need to be done after debug collection\n");
+			IWL_DEBUG_FW(fwrt,
+				     "WRT: nothing need to be done after debug collection\n");
 		} else {
 			IWL_ERR(fwrt, "WRT: wrong resetfw %d\n",
 				le32_to_cpu(dump_data.trig->reset_fw));
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-debug.c b/drivers/net/wireless/intel/iwlwifi/iwl-debug.c
index ae4c2a3d63d5..3a3c13a41fc6 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-debug.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-debug.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2005-2011, 2021 Intel Corporation
+ * Copyright (C) 2005-2011, 2021-2022 Intel Corporation
  */
 #include <linux/device.h>
 #include <linux/interrupt.h>
@@ -57,6 +57,7 @@ void __iwl_err(struct device *dev, enum iwl_err_mode mode, const char *fmt, ...)
 	default:
 		break;
 	}
+	vaf.va = &args;
 	trace_iwlwifi_err(&vaf);
 	va_end(args);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c
index 999b7c652289..e46639b097f4 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c
@@ -12,6 +12,9 @@
 #include "iwl-trans.h"
 
 #define CREATE_TRACE_POINTS
+#ifdef CONFIG_CC_IS_GCC
+#pragma GCC diagnostic ignored "-Wsuggest-attribute=format"
+#endif
 #include "iwl-devtrace.h"
 
 EXPORT_TRACEPOINT_SYMBOL(iwlwifi_dev_ucode_event);
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
index 4c977ba9cd85..34feb4d29adc 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
@@ -127,6 +127,7 @@ static void iwl_dealloc_ucode(struct iwl_drv *drv)
 	kfree(drv->fw.iml);
 	kfree(drv->fw.ucode_capa.cmd_versions);
 	kfree(drv->fw.phy_integration_ver);
+	kfree(drv->trans->dbg.pc_data);
 
 	for (i = 0; i < IWL_UCODE_TYPE_MAX; i++)
 		iwl_free_fw_img(drv, drv->fw.img + i);
@@ -894,7 +895,7 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 				drv->fw.img[IWL_UCODE_WOWLAN].is_dual_cpus =
 					true;
 			} else if ((num_of_cpus > 2) || (num_of_cpus < 1)) {
-				IWL_ERR(drv, "Driver support upto 2 CPUs\n");
+				IWL_ERR(drv, "Driver support up to 2 CPUs\n");
 				return -EINVAL;
 			}
 			break;
@@ -1154,6 +1155,12 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 			capa->num_stations =
 				le32_to_cpup((const __le32 *)tlv_data);
 			break;
+		case IWL_UCODE_TLV_FW_NUM_BEACONS:
+			if (tlv_len != sizeof(u32))
+				goto invalid_tlv_len;
+			capa->num_beacons =
+				le32_to_cpup((const __le32 *)tlv_data);
+			break;
 		case IWL_UCODE_TLV_UMAC_DEBUG_ADDRS: {
 			const struct iwl_umac_debug_addrs *dbg_ptrs =
 				(const void *)tlv_data;
@@ -1232,6 +1239,14 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 			iwl_drv_set_dump_exclude(drv, tlv_type,
 						 tlv_data, tlv_len);
 			break;
+		case IWL_UCODE_TLV_CURRENT_PC:
+			if (tlv_len < sizeof(struct iwl_pc_data))
+				goto invalid_tlv_len;
+			drv->trans->dbg.num_pc =
+				tlv_len / sizeof(struct iwl_pc_data);
+			drv->trans->dbg.pc_data =
+				kmemdup(tlv_data, tlv_len, GFP_KERNEL);
+			break;
 		default:
 			IWL_DEBUG_INFO(drv, "unknown TLV: %d\n", tlv_type);
 			break;
@@ -1406,6 +1421,7 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
 			IWL_DEFAULT_STANDARD_PHY_CALIBRATE_TBL_SIZE;
 	fw->ucode_capa.n_scan_channels = IWL_DEFAULT_SCAN_CHANNELS;
 	fw->ucode_capa.num_stations = IWL_MVM_STATION_COUNT_MAX;
+	fw->ucode_capa.num_beacons = 1;
 	/* dump all fw memory areas by default */
 	fw->dbg.dump_mask = 0xffffffff;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h b/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h
index baa643386018..0e8ca761d24b 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h
@@ -47,13 +47,12 @@ struct iwl_nvm_data {
 	struct ieee80211_supported_band bands[NUM_NL80211_BANDS];
 
 	/*
-	 * iftype data for low (2.4 GHz) and high (5 and 6 GHz) bands,
-	 * we can use the same for 5 and 6 GHz bands because they have
-	 * the same data
+	 * iftype data for low (2.4 GHz) high (5 GHz) and uhb (6 GHz) bands
 	 */
 	struct {
 		struct ieee80211_sband_iftype_data low[2];
 		struct ieee80211_sband_iftype_data high[2];
+		struct ieee80211_sband_iftype_data uhb[2];
 	} iftd;
 
 	struct ieee80211_channel channels[];
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
index aa8e08487b52..7dcb1c3ab728 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
@@ -81,7 +81,7 @@ static const u16 iwl_nvm_channels[] = {
 	/* 2.4 GHz */
 	1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 	/* 5 GHz */
-	36, 40, 44 , 48, 52, 56, 60, 64,
+	36, 40, 44, 48, 52, 56, 60, 64,
 	100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 	149, 153, 157, 161, 165
 };
@@ -860,7 +860,10 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans,
 	/* Advertise an A-MPDU exponent extension based on
 	 * operating band
 	 */
-	if (sband->band != NL80211_BAND_2GHZ)
+	if (sband->band == NL80211_BAND_6GHZ && iftype_data->eht_cap.has_eht)
+		iftype_data->he_cap.he_cap_elem.mac_cap_info[3] |=
+			IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_2;
+	else if (sband->band != NL80211_BAND_2GHZ)
 		iftype_data->he_cap.he_cap_elem.mac_cap_info[3] |=
 			IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_1;
 	else
@@ -876,16 +879,13 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans,
 				       IEEE80211_EHT_MAC_CAP0_MAX_MPDU_LEN_MASK);
 		break;
 	case NL80211_BAND_6GHZ:
-		if (!is_ap || iwlwifi_mod_params.nvm_file)
-			iftype_data->eht_cap.eht_cap_elem.phy_cap_info[0] |=
-				IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ;
+		iftype_data->eht_cap.eht_cap_elem.phy_cap_info[0] |=
+			IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ;
 		fallthrough;
 	case NL80211_BAND_5GHZ:
 		iftype_data->he_cap.he_cap_elem.phy_cap_info[0] |=
-			IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G;
-		if (!is_ap || iwlwifi_mod_params.nvm_file)
-			iftype_data->he_cap.he_cap_elem.phy_cap_info[0] |=
-				IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G;
+			IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+			IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G;
 		break;
 	default:
 		WARN_ON(1);
@@ -938,6 +938,10 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans,
 		}
 	}
 
+	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210 && !is_ap)
+		iftype_data->he_cap.he_cap_elem.phy_cap_info[2] |=
+			IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO;
+
 	switch (CSR_HW_RFID_TYPE(trans->hw_rf_id)) {
 	case IWL_CFG_RF_TYPE_GF:
 	case IWL_CFG_RF_TYPE_MR:
@@ -999,15 +1003,18 @@ static void iwl_init_he_hw_capab(struct iwl_trans *trans,
 
 	BUILD_BUG_ON(sizeof(data->iftd.low) != sizeof(iwl_he_eht_capa));
 	BUILD_BUG_ON(sizeof(data->iftd.high) != sizeof(iwl_he_eht_capa));
+	BUILD_BUG_ON(sizeof(data->iftd.uhb) != sizeof(iwl_he_eht_capa));
 
 	switch (sband->band) {
 	case NL80211_BAND_2GHZ:
 		iftype_data = data->iftd.low;
 		break;
 	case NL80211_BAND_5GHZ:
-	case NL80211_BAND_6GHZ:
 		iftype_data = data->iftd.high;
 		break;
+	case NL80211_BAND_6GHZ:
+		iftype_data = data->iftd.uhb;
+		break;
 	default:
 		WARN_ON(1);
 		return;
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
index 62ce116d3783..0dfe00eae05d 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
@@ -350,6 +350,11 @@
 #define WFPM_OTP_CFG1_ADDR		0x00a03098
 #define WFPM_OTP_CFG1_IS_JACKET_BIT	BIT(4)
 #define WFPM_OTP_CFG1_IS_CDB_BIT	BIT(5)
+#define WFPM_OTP_BZ_BNJ_JACKET_BIT	5
+#define WFPM_OTP_BZ_BNJ_CDB_BIT		4
+#define WFPM_OTP_CFG1_IS_JACKET(_val)   (((_val) & 0x00000020) >> WFPM_OTP_BZ_BNJ_JACKET_BIT)
+#define WFPM_OTP_CFG1_IS_CDB(_val)      (((_val) & 0x00000010) >> WFPM_OTP_BZ_BNJ_CDB_BIT)
+
 
 #define WFPM_GP2			0xA030B4
 
@@ -445,6 +450,8 @@ enum {
 #define REG_CRF_ID_TYPE_GF_TC			0xF08
 #define REG_CRF_ID_TYPE_MR			0x810
 #define REG_CRF_ID_TYPE_FM			0x910
+#define REG_CRF_ID_TYPE_FMI			0x930
+#define REG_CRF_ID_TYPE_FMR			0x900
 
 #define HPM_DEBUG			0xA03440
 #define PERSISTENCE_BIT			BIT(12)
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
index 9aced3e44bc2..9f1228b5a384 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
@@ -748,6 +748,18 @@ struct iwl_imr_data {
 	__le64 imr_base_addr;
 };
 
+#define IWL_TRANS_CURRENT_PC_NAME_MAX_BYTES      32
+
+/**
+ * struct iwl_pc_data - program counter details
+ * @pc_name: cpu name
+ * @pc_address: cpu program counter
+ */
+struct iwl_pc_data {
+	u8  pc_name[IWL_TRANS_CURRENT_PC_NAME_MAX_BYTES];
+	u32 pc_address;
+};
+
 /**
  * struct iwl_trans_debug - transport debug related data
  *
@@ -775,6 +787,10 @@ struct iwl_imr_data {
  * @periodic_trig_list: periodic triggers list
  * @domains_bitmap: bitmap of active domains other than &IWL_FW_INI_DOMAIN_ALWAYS_ON
  * @ucode_preset: preset based on ucode
+ * @dump_file_name_ext: dump file name extension
+ * @dump_file_name_ext_valid: dump file name extension if valid or not
+ * @num_pc: number of program counter for cpu
+ * @pc_data: details of the program counter
  */
 struct iwl_trans_debug {
 	u8 n_dest_reg;
@@ -813,6 +829,10 @@ struct iwl_trans_debug {
 	bool restart_required;
 	u32 last_tp_resetfw;
 	struct iwl_imr_data imr_data;
+	u8 dump_file_name_ext[IWL_FW_INI_MAX_NAME];
+	bool dump_file_name_ext_valid;
+	u32 num_pc;
+	struct iwl_pc_data *pc_data;
 };
 
 struct iwl_dma_ptr {
@@ -977,7 +997,7 @@ struct iwl_trans_txqs {
  *	0 indicates that frag SKBs (NETIF_F_SG) aren't supported.
  * @hw_rf_id a u32 with the device RF ID
  * @hw_crf_id a u32 with the device CRF ID
- * @hw_cdb_id a u32 with the device CDB ID
+ * @hw_wfpm_id a u32 with the device wfpm ID
  * @hw_id: a u32 with the ID of the device / sub-device.
  *	Set during transport allocation.
  * @hw_id_str: a string with info about HW ID. Set during transport allocation.
@@ -1020,7 +1040,8 @@ struct iwl_trans {
 	u32 hw_rev_step;
 	u32 hw_rf_id;
 	u32 hw_crf_id;
-	u32 hw_cdb_id;
+	u32 hw_cnv_id;
+	u32 hw_wfpm_id;
 	u32 hw_id;
 	char hw_id_str[52];
 	u32 sku_id[3];
diff --git a/drivers/net/wireless/intel/iwlwifi/mei/iwl-mei.h b/drivers/net/wireless/intel/iwlwifi/mei/iwl-mei.h
index ae66192feefe..655d95d3a068 100644
--- a/drivers/net/wireless/intel/iwlwifi/mei/iwl-mei.h
+++ b/drivers/net/wireless/intel/iwlwifi/mei/iwl-mei.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (C) 2021 Intel Corporation
+ * Copyright (C) 2021 - 2022 Intel Corporation
  */
 
 #ifndef __iwl_mei_h__
@@ -301,7 +301,7 @@ struct iwl_mei_colloc_info {
 struct iwl_mei_ops {
 	void (*me_conn_status)(void *priv,
 			       const struct iwl_mei_conn_info *conn_info);
-	void (*rfkill)(void *priv, bool blocked);
+	void (*rfkill)(void *priv, bool blocked, bool csme_taking_ownership);
 	void (*roaming_forbidden)(void *priv, bool forbidden);
 	void (*sap_connected)(void *priv);
 	void (*nic_stolen)(void *priv);
diff --git a/drivers/net/wireless/intel/iwlwifi/mei/main.c b/drivers/net/wireless/intel/iwlwifi/mei/main.c
index 67dfb77fedf7..0a29fb013005 100644
--- a/drivers/net/wireless/intel/iwlwifi/mei/main.c
+++ b/drivers/net/wireless/intel/iwlwifi/mei/main.c
@@ -31,6 +31,11 @@ MODULE_LICENSE("GPL");
 #define MEI_WLAN_UUID UUID_LE(0x13280904, 0x7792, 0x4fcb, \
 			      0xa1, 0xaa, 0x5e, 0x70, 0xcb, 0xb1, 0xe8, 0x65)
 
+/* After CSME takes ownership, it won't release it for 60 seconds to avoid
+ * frequent ownership transitions.
+ */
+#define MEI_OWNERSHIP_RETAKE_TIMEOUT_MS	msecs_to_jiffies(60000)
+
 /*
  * Since iwlwifi calls iwlmei without any context, hold a pointer to the
  * mei_cl_device structure here.
@@ -156,6 +161,8 @@ struct iwl_mei_filters {
  *	accessed without the mutex.
  * @netdev_work: used to defer registering and unregistering of the netdev to
  *	avoid taking the rtnl lock in the SAP messages handlers.
+ * @ownership_dwork: used to re-ask for NIC ownership after ownership was taken
+ *	by CSME or when a previous ownership request failed.
  * @sap_seq_no: the sequence number for the SAP messages
  * @seq_no: the sequence number for the SAP messages
  * @dbgfs_dir: the debugfs dir entry
@@ -179,6 +186,7 @@ struct iwl_mei {
 	bool pldr_active;
 	spinlock_t data_q_lock;
 	struct work_struct netdev_work;
+	struct delayed_work ownership_dwork;
 
 	atomic_t sap_seq_no;
 	atomic_t seq_no;
@@ -716,7 +724,7 @@ iwl_mei_handle_conn_status(struct mei_cl_device *cldev,
 						     status->link_prot_state);
 	else
 		iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv,
-					  status->link_prot_state);
+					  status->link_prot_state, false);
 }
 
 static void iwl_mei_set_init_conf(struct iwl_mei *mei)
@@ -788,7 +796,7 @@ static void iwl_mei_handle_amt_state(struct mei_cl_device *cldev,
 	if (mei->amt_enabled)
 		iwl_mei_set_init_conf(mei);
 	else if (iwl_mei_cache.ops)
-		iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, false);
+		iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, false, false);
 
 	schedule_work(&mei->netdev_work);
 
@@ -829,10 +837,12 @@ static void iwl_mei_handle_csme_taking_ownership(struct mei_cl_device *cldev,
 		 */
 		mei->csme_taking_ownership = true;
 
-		iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, true);
+		iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, true, true);
 	} else {
 		iwl_mei_send_sap_msg(cldev,
 				     SAP_MSG_NOTIF_CSME_OWNERSHIP_CONFIRMED);
+		schedule_delayed_work(&mei->ownership_dwork,
+				      MEI_OWNERSHIP_RETAKE_TIMEOUT_MS);
 	}
 }
 
@@ -882,7 +892,7 @@ static void iwl_mei_handle_rx_host_own_req(struct mei_cl_device *cldev,
 
 	/* We can now start the connection, unblock rfkill */
 	if (iwl_mei_cache.ops)
-		iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, false);
+		iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, false, false);
 }
 
 static void iwl_mei_handle_pldr_ack(struct mei_cl_device *cldev,
@@ -1447,7 +1457,13 @@ int iwl_mei_get_ownership(void)
 
 	ret = wait_event_timeout(mei->get_ownership_wq,
 				 mei->got_ownership, HZ / 2);
-	return (!ret) ? -ETIMEDOUT : 0;
+	if (!ret) {
+		schedule_delayed_work(&mei->ownership_dwork,
+				      MEI_OWNERSHIP_RETAKE_TIMEOUT_MS);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
 out:
 	mutex_unlock(&iwl_mei_mutex);
 	return ret;
@@ -1738,6 +1754,8 @@ void iwl_mei_device_state(bool up)
 	iwl_mei_send_sap_msg(mei->cldev,
 			     SAP_MSG_NOTIF_CSME_OWNERSHIP_CONFIRMED);
 	mei->csme_taking_ownership = false;
+	schedule_delayed_work(&mei->ownership_dwork,
+			      MEI_OWNERSHIP_RETAKE_TIMEOUT_MS);
 out:
 	mutex_unlock(&iwl_mei_mutex);
 }
@@ -1773,7 +1791,8 @@ int iwl_mei_register(void *priv, const struct iwl_mei_ops *ops)
 		if (iwl_mei_is_connected()) {
 			if (mei->amt_enabled)
 				iwl_mei_send_sap_msg(mei->cldev,
-						     SAP_MSG_NOTIF_WIFIDR_UP);
+						     SAP_MSG_NOTIF_WIFIDR_UP,
+						     false);
 			ops->rfkill(priv, mei->link_prot_state);
 		}
 	}
@@ -1894,6 +1913,11 @@ static void iwl_mei_dbgfs_unregister(struct iwl_mei *mei) {}
 
 #endif /* CONFIG_DEBUG_FS */
 
+static void iwl_mei_ownership_dwork(struct work_struct *wk)
+{
+	iwl_mei_get_ownership();
+}
+
 #define ALLOC_SHARED_MEM_RETRY_MAX_NUM	3
 
 /*
@@ -1923,6 +1947,7 @@ static int iwl_mei_probe(struct mei_cl_device *cldev,
 	init_waitqueue_head(&mei->pldr_wq);
 	spin_lock_init(&mei->data_q_lock);
 	INIT_WORK(&mei->netdev_work, iwl_mei_netdev_work);
+	INIT_DELAYED_WORK(&mei->ownership_dwork, iwl_mei_ownership_dwork);
 
 	mei_cldev_set_drvdata(cldev, mei);
 	mei->cldev = cldev;
@@ -2087,7 +2112,7 @@ static void iwl_mei_remove(struct mei_cl_device *cldev)
 	spin_unlock_bh(&mei->data_q_lock);
 
 	if (iwl_mei_cache.ops)
-		iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, false);
+		iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, false, false);
 
 	/*
 	 * mei_cldev_disable will return only after all the MEI Rx is done.
@@ -2105,6 +2130,7 @@ static void iwl_mei_remove(struct mei_cl_device *cldev)
 	cancel_work_sync(&mei->send_csa_msg_wk);
 	cancel_delayed_work_sync(&mei->csa_throttle_end_wk);
 	cancel_work_sync(&mei->netdev_work);
+	cancel_delayed_work_sync(&mei->ownership_dwork);
 
 	/*
 	 * If someone waits for the ownership, let him know that we are going
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
index b28fcf0cf9cf..593fe28d89cf 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
@@ -7,7 +7,9 @@ iwlmvm-y += power.o coex.o
 iwlmvm-y += tt.o offloading.o tdls.o
 iwlmvm-y += ftm-responder.o ftm-initiator.o
 iwlmvm-y += rfi.o
-iwlmvm-y += mld-key.o
+iwlmvm-y += mld-key.o mld-mac.o link.o mld-sta.o mld-mac80211.o
+iwlmvm-y += ptp.o
+iwlmvm-y += time-sync.o
 iwlmvm-$(CONFIG_IWLWIFI_DEBUGFS) += debugfs.o debugfs-vif.o
 iwlmvm-$(CONFIG_IWLWIFI_LEDS) += led.o
 iwlmvm-$(CONFIG_PM) += d3.o
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/binding.c b/drivers/net/wireless/intel/iwlwifi/mvm/binding.c
index 0aac306304cb..ef50ccabcc73 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/binding.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/binding.c
@@ -2,6 +2,7 @@
 /*
  * Copyright (C) 2012-2014, 2020 Intel Corporation
  * Copyright (C) 2016 Intel Deutschland GmbH
+ * Copyright (C) 2022 Intel Corporation
  */
 #include <net/mac80211.h>
 #include "fw-api.h"
@@ -75,7 +76,7 @@ static void iwl_mvm_iface_iterator(void *_data, u8 *mac,
 	if (vif == data->ignore_vif)
 		return;
 
-	if (mvmvif->phy_ctxt != data->phyctxt)
+	if (mvmvif->deflink.phy_ctxt != data->phyctxt)
 		return;
 
 	if (WARN_ON_ONCE(data->idx >= MAX_MACS_IN_BINDING))
@@ -132,7 +133,7 @@ int iwl_mvm_binding_add_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-	if (WARN_ON_ONCE(!mvmvif->phy_ctxt))
+	if (WARN_ON_ONCE(!mvmvif->deflink.phy_ctxt))
 		return -EINVAL;
 
 	/*
@@ -142,7 +143,8 @@ int iwl_mvm_binding_add_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	if (iwl_mvm_sf_update(mvm, vif, false))
 		return -EINVAL;
 
-	return iwl_mvm_binding_update(mvm, vif, mvmvif->phy_ctxt, true);
+	return iwl_mvm_binding_update(mvm, vif, mvmvif->deflink.phy_ctxt,
+				      true);
 }
 
 int iwl_mvm_binding_remove_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
@@ -150,10 +152,11 @@ int iwl_mvm_binding_remove_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	int ret;
 
-	if (WARN_ON_ONCE(!mvmvif->phy_ctxt))
+	if (WARN_ON_ONCE(!mvmvif->deflink.phy_ctxt))
 		return -EINVAL;
 
-	ret = iwl_mvm_binding_update(mvm, vif, mvmvif->phy_ctxt, false);
+	ret = iwl_mvm_binding_update(mvm, vif, mvmvif->deflink.phy_ctxt,
+				     false);
 
 	if (!ret)
 		if (iwl_mvm_sf_update(mvm, vif, true))
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/coex.c b/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
index ee3c8a786199..5a5b1128e75c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
@@ -194,7 +194,7 @@ static int iwl_mvm_bt_coex_reduced_txp(struct iwl_mvm *mvm, u8 sta_id,
 	if (mvmsta->bt_reduced_txpower == enable)
 		return 0;
 
-	value = mvmsta->sta_id;
+	value = mvmsta->deflink.sta_id;
 
 	if (enable)
 		value |= BT_REDUCED_TX_POWER_BIT;
@@ -257,33 +257,35 @@ static void iwl_mvm_bt_coex_tcm_based_ci(struct iwl_mvm *mvm,
 	swap(data->primary, data->secondary);
 }
 
-/* must be called under rcu_read_lock */
-static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
-				      struct ieee80211_vif *vif)
+static void iwl_mvm_bt_notif_per_link(struct iwl_mvm *mvm,
+				      struct ieee80211_vif *vif,
+				      struct iwl_bt_iterator_data *data,
+				      unsigned int link_id)
 {
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_bt_iterator_data *data = _data;
-	struct iwl_mvm *mvm = data->mvm;
-	struct ieee80211_chanctx_conf *chanctx_conf;
 	/* default smps_mode is AUTOMATIC - only used for client modes */
 	enum ieee80211_smps_mode smps_mode = IEEE80211_SMPS_AUTOMATIC;
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	u32 bt_activity_grading, min_ag_for_static_smps;
+	struct ieee80211_chanctx_conf *chanctx_conf;
+	struct iwl_mvm_vif_link_info *link_info;
+	struct ieee80211_bss_conf *link_conf;
 	int ave_rssi;
 
 	lockdep_assert_held(&mvm->mutex);
 
-	switch (vif->type) {
-	case NL80211_IFTYPE_STATION:
-		break;
-	case NL80211_IFTYPE_AP:
-		if (!mvmvif->ap_ibss_active)
-			return;
-		break;
-	default:
+	link_info = mvmvif->link[link_id];
+	if (!link_info)
 		return;
-	}
 
-	chanctx_conf = rcu_dereference(vif->bss_conf.chanctx_conf);
+	link_conf = rcu_dereference(vif->link_conf[link_id]);
+	/* This can happen due to races: if we receive the notification
+	 * and have the mutex held, while mac80211 is stuck on our mutex
+	 * in the middle of removing the link.
+	 */
+	if (!link_conf)
+		return;
+
+	chanctx_conf = rcu_dereference(link_conf->chanctx_conf);
 
 	/* If channel context is invalid or not on 2.4GHz .. */
 	if ((!chanctx_conf ||
@@ -291,9 +293,10 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 		if (vif->type == NL80211_IFTYPE_STATION) {
 			/* ... relax constraints and disable rssi events */
 			iwl_mvm_update_smps(mvm, vif, IWL_MVM_SMPS_REQ_BT_COEX,
-					    smps_mode);
-			iwl_mvm_bt_coex_reduced_txp(mvm, mvmvif->ap_sta_id,
+					    smps_mode, link_id);
+			iwl_mvm_bt_coex_reduced_txp(mvm, link_info->ap_sta_id,
 						    false);
+			/* FIXME: should this be per link? */
 			iwl_mvm_bt_coex_enable_rssi_event(mvm, vif, false, 0);
 		}
 		return;
@@ -314,17 +317,18 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 	if (!vif->cfg.assoc)
 		smps_mode = IEEE80211_SMPS_AUTOMATIC;
 
-	if (mvmvif->phy_ctxt &&
-	    (mvm->last_bt_notif.rrc_status & BIT(mvmvif->phy_ctxt->id)))
+	if (link_info->phy_ctxt &&
+	    (mvm->last_bt_notif.rrc_status & BIT(link_info->phy_ctxt->id)))
 		smps_mode = IEEE80211_SMPS_AUTOMATIC;
 
 	IWL_DEBUG_COEX(data->mvm,
-		       "mac %d: bt_activity_grading %d smps_req %d\n",
-		       mvmvif->id, bt_activity_grading, smps_mode);
+		       "mac %d link %d: bt_activity_grading %d smps_req %d\n",
+		       mvmvif->id, link_info->fw_link_id,
+		       bt_activity_grading, smps_mode);
 
 	if (vif->type == NL80211_IFTYPE_STATION)
 		iwl_mvm_update_smps(mvm, vif, IWL_MVM_SMPS_REQ_BT_COEX,
-				    smps_mode);
+				    smps_mode, link_id);
 
 	/* low latency is always primary */
 	if (iwl_mvm_vif_low_latency(mvmvif)) {
@@ -353,6 +357,7 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 			data->secondary = chanctx_conf;
 		}
 
+		/* FIXME: TCM load per interface? or need something per link? */
 		if (data->primary == chanctx_conf)
 			data->primary_load = mvm->tcm.result.load[mvmvif->id];
 		else if (data->secondary == chanctx_conf)
@@ -370,6 +375,7 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 		/* if secondary is not NULL, it might be a GO */
 		data->secondary = chanctx_conf;
 
+	/* FIXME: TCM load per interface? or need something per link? */
 	if (data->primary == chanctx_conf)
 		data->primary_load = mvm->tcm.result.load[mvmvif->id];
 	else if (data->secondary == chanctx_conf)
@@ -384,7 +390,8 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 	if (iwl_get_coex_type(mvm, vif) == BT_COEX_LOOSE_LUT ||
 	    mvm->cfg->bt_shared_single_ant || !vif->cfg.assoc ||
 	    le32_to_cpu(mvm->last_bt_notif.bt_activity_grading) == BT_OFF) {
-		iwl_mvm_bt_coex_reduced_txp(mvm, mvmvif->ap_sta_id, false);
+		iwl_mvm_bt_coex_reduced_txp(mvm, link_info->ap_sta_id, false);
+		/* FIXME: should this be per link? */
 		iwl_mvm_bt_coex_enable_rssi_event(mvm, vif, false, 0);
 		return;
 	}
@@ -396,10 +403,12 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 	if (!ave_rssi)
 		ave_rssi = -100;
 	if (ave_rssi > -IWL_MVM_BT_COEX_EN_RED_TXP_THRESH) {
-		if (iwl_mvm_bt_coex_reduced_txp(mvm, mvmvif->ap_sta_id, true))
+		if (iwl_mvm_bt_coex_reduced_txp(mvm, link_info->ap_sta_id,
+						true))
 			IWL_ERR(mvm, "Couldn't send BT_CONFIG cmd\n");
 	} else if (ave_rssi < -IWL_MVM_BT_COEX_DIS_RED_TXP_THRESH) {
-		if (iwl_mvm_bt_coex_reduced_txp(mvm, mvmvif->ap_sta_id, false))
+		if (iwl_mvm_bt_coex_reduced_txp(mvm, link_info->ap_sta_id,
+						false))
 			IWL_ERR(mvm, "Couldn't send BT_CONFIG cmd\n");
 	}
 
@@ -407,6 +416,32 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 	iwl_mvm_bt_coex_enable_rssi_event(mvm, vif, true, ave_rssi);
 }
 
+/* must be called under rcu_read_lock */
+static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
+				      struct ieee80211_vif *vif)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_bt_iterator_data *data = _data;
+	struct iwl_mvm *mvm = data->mvm;
+	unsigned int link_id;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	switch (vif->type) {
+	case NL80211_IFTYPE_STATION:
+		break;
+	case NL80211_IFTYPE_AP:
+		if (!mvmvif->ap_ibss_active)
+			return;
+		break;
+	default:
+		return;
+	}
+
+	for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++)
+		iwl_mvm_bt_notif_per_link(mvm, vif, data, link_id);
+}
+
 static void iwl_mvm_bt_coex_notif_handle(struct iwl_mvm *mvm)
 {
 	struct iwl_bt_iterator_data data = {
@@ -521,7 +556,7 @@ void iwl_mvm_bt_rssi_event(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	 * Rssi update while not associated - can happen since the statistics
 	 * are handled asynchronously
 	 */
-	if (mvmvif->ap_sta_id == IWL_MVM_INVALID_STA)
+	if (mvmvif->deflink.ap_sta_id == IWL_MVM_INVALID_STA)
 		return;
 
 	/* No BT - reports should be disabled */
@@ -537,10 +572,13 @@ void iwl_mvm_bt_rssi_event(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	 */
 	if (rssi_event == RSSI_EVENT_LOW || mvm->cfg->bt_shared_single_ant ||
 	    iwl_get_coex_type(mvm, vif) == BT_COEX_LOOSE_LUT)
-		ret = iwl_mvm_bt_coex_reduced_txp(mvm, mvmvif->ap_sta_id,
+		ret = iwl_mvm_bt_coex_reduced_txp(mvm,
+						  mvmvif->deflink.ap_sta_id,
 						  false);
 	else
-		ret = iwl_mvm_bt_coex_reduced_txp(mvm, mvmvif->ap_sta_id, true);
+		ret = iwl_mvm_bt_coex_reduced_txp(mvm,
+						  mvmvif->deflink.ap_sta_id,
+						  true);
 
 	if (ret)
 		IWL_ERR(mvm, "couldn't send BT_CONFIG HCMD upon RSSI event\n");
@@ -554,7 +592,7 @@ u16 iwl_mvm_coex_agg_time_limit(struct iwl_mvm *mvm,
 {
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(mvmsta->vif);
-	struct iwl_mvm_phy_ctxt *phy_ctxt = mvmvif->phy_ctxt;
+	struct iwl_mvm_phy_ctxt *phy_ctxt = mvmvif->deflink.phy_ctxt;
 	enum iwl_bt_coex_lut_type lut_type;
 
 	if (mvm->last_bt_notif.ttc_status & BIT(phy_ctxt->id))
@@ -578,7 +616,7 @@ bool iwl_mvm_bt_coex_is_mimo_allowed(struct iwl_mvm *mvm,
 {
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(mvmsta->vif);
-	struct iwl_mvm_phy_ctxt *phy_ctxt = mvmvif->phy_ctxt;
+	struct iwl_mvm_phy_ctxt *phy_ctxt = mvmvif->deflink.phy_ctxt;
 	enum iwl_bt_coex_lut_type lut_type;
 
 	if (mvm->last_bt_notif.ttc_status & BIT(phy_ctxt->id))
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
index c5ad34b063df..37aa4676dc94 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
@@ -470,7 +470,7 @@ static int iwl_mvm_wowlan_config_rsc_tsc(struct iwl_mvm *mvm,
 		for (i = 0; i < ARRAY_SIZE(data.rsc->mcast_key_id_map); i++)
 			data.rsc->mcast_key_id_map[i] =
 				IWL_MCAST_KEY_MAP_INVALID;
-		data.rsc->sta_id = cpu_to_le32(mvmvif->ap_sta_id);
+		data.rsc->sta_id = cpu_to_le32(mvmvif->deflink.ap_sta_id);
 
 		ieee80211_iter_keys(mvm->hw, vif,
 				    iwl_mvm_wowlan_get_rsc_v5_data,
@@ -493,7 +493,8 @@ static int iwl_mvm_wowlan_config_rsc_tsc(struct iwl_mvm *mvm,
 
 		if (ver == 4) {
 			size = sizeof(*data.rsc_tsc);
-			data.rsc_tsc->sta_id = cpu_to_le32(mvmvif->ap_sta_id);
+			data.rsc_tsc->sta_id =
+				cpu_to_le32(mvmvif->deflink.ap_sta_id);
 		} else {
 			/* ver == 2 || ver == IWL_FW_CMD_VER_UNKNOWN */
 			size = sizeof(data.rsc_tsc->params);
@@ -563,6 +564,7 @@ static void iwl_mvm_wowlan_get_tkip_data(struct ieee80211_hw *hw,
 		}
 
 		for (i = 0; i < IWL_NUM_RSC; i++) {
+			ieee80211_get_key_rx_seq(key, i, &seq);
 			/* wrapping isn't allowed, AP must rekey */
 			if (seq.tkip.iv32 > cur_rx_iv32)
 				cur_rx_iv32 = seq.tkip.iv32;
@@ -691,7 +693,7 @@ static int iwl_mvm_send_patterns(struct iwl_mvm *mvm,
 
 	pattern_cmd->n_patterns = wowlan->n_patterns;
 	if (ver >= 3)
-		pattern_cmd->sta_id = mvmvif->ap_sta_id;
+		pattern_cmd->sta_id = mvmvif->deflink.ap_sta_id;
 
 	for (i = 0; i < wowlan->n_patterns; i++) {
 		int mask_len = DIV_ROUND_UP(wowlan->patterns[i].pattern_len, 8);
@@ -732,7 +734,7 @@ static int iwl_mvm_d3_reprogram(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		return -EINVAL;
 
 	/* add back the PHY */
-	if (WARN_ON(!mvmvif->phy_ctxt))
+	if (WARN_ON(!mvmvif->deflink.phy_ctxt))
 		return -EINVAL;
 
 	rcu_read_lock();
@@ -746,7 +748,7 @@ static int iwl_mvm_d3_reprogram(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	chains_dynamic = ctx->rx_chains_dynamic;
 	rcu_read_unlock();
 
-	ret = iwl_mvm_phy_ctxt_add(mvm, mvmvif->phy_ctxt, &chandef,
+	ret = iwl_mvm_phy_ctxt_add(mvm, mvmvif->deflink.phy_ctxt, &chandef,
 				   chains_static, chains_dynamic);
 	if (ret)
 		return ret;
@@ -763,12 +765,12 @@ static int iwl_mvm_d3_reprogram(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	/* add back binding - XXX refactor? */
 	binding_cmd.id_and_color =
-		cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->phy_ctxt->id,
-						mvmvif->phy_ctxt->color));
+		cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->deflink.phy_ctxt->id,
+						mvmvif->deflink.phy_ctxt->color));
 	binding_cmd.action = cpu_to_le32(FW_CTXT_ACTION_ADD);
 	binding_cmd.phy =
-		cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->phy_ctxt->id,
-						mvmvif->phy_ctxt->color));
+		cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->deflink.phy_ctxt->id,
+						mvmvif->deflink.phy_ctxt->color));
 	binding_cmd.macs[0] = cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->id,
 							      mvmvif->color));
 	for (i = 1; i < MAX_MACS_IN_BINDING; i++)
@@ -791,7 +793,8 @@ static int iwl_mvm_d3_reprogram(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	ret = iwl_mvm_sta_send_to_fw(mvm, ap_sta, false, 0);
 	if (ret)
 		return ret;
-	rcu_assign_pointer(mvm->fw_id_to_mac_id[mvmvif->ap_sta_id], ap_sta);
+	rcu_assign_pointer(mvm->fw_id_to_mac_id[mvmvif->deflink.ap_sta_id],
+			   ap_sta);
 
 	ret = iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL);
 	if (ret)
@@ -800,8 +803,8 @@ static int iwl_mvm_d3_reprogram(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	/* and some quota */
 	quota = iwl_mvm_quota_cmd_get_quota(mvm, &quota_cmd, 0);
 	quota->id_and_color =
-		cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->phy_ctxt->id,
-						mvmvif->phy_ctxt->color));
+		cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->deflink.phy_ctxt->id,
+						mvmvif->deflink.phy_ctxt->color));
 	quota->quota = cpu_to_le32(IWL_MVM_MAX_QUOTA);
 	quota->max_duration = cpu_to_le32(IWL_MVM_MAX_QUOTA);
 
@@ -1027,7 +1030,7 @@ static int iwl_mvm_wowlan_config_key_params(struct iwl_mvm *mvm,
 		if (ver == 2) {
 			size = sizeof(tkip_data.tkip);
 			tkip_data.tkip.sta_id =
-				cpu_to_le32(mvmvif->ap_sta_id);
+				cpu_to_le32(mvmvif->deflink.ap_sta_id);
 		} else if (ver == 1 || ver == IWL_FW_CMD_VER_UNKNOWN) {
 			size = sizeof(struct iwl_wowlan_tkip_params_cmd_ver_1);
 		} else {
@@ -1076,7 +1079,7 @@ static int iwl_mvm_wowlan_config_key_params(struct iwl_mvm *mvm,
 		kek_kck_cmd.kek_len = cpu_to_le16(mvmvif->rekey_data.kek_len);
 		kek_kck_cmd.replay_ctr = mvmvif->rekey_data.replay_ctr;
 		kek_kck_cmd.akm = cpu_to_le32(mvmvif->rekey_data.akm);
-		kek_kck_cmd.sta_id = cpu_to_le32(mvmvif->ap_sta_id);
+		kek_kck_cmd.sta_id = cpu_to_le32(mvmvif->deflink.ap_sta_id);
 
 		if (cmd_ver == 4) {
 			cmd_size = sizeof(struct iwl_wowlan_kek_kck_material_cmd_v4);
@@ -1269,7 +1272,7 @@ static int __iwl_mvm_suspend(struct ieee80211_hw *hw,
 
 	mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-	if (mvmvif->ap_sta_id == IWL_MVM_INVALID_STA) {
+	if (mvmvif->deflink.ap_sta_id == IWL_MVM_INVALID_STA) {
 		/* if we're not associated, this must be netdetect */
 		if (!wowlan->nd_config) {
 			ret = 1;
@@ -1285,10 +1288,10 @@ static int __iwl_mvm_suspend(struct ieee80211_hw *hw,
 	} else {
 		struct iwl_wowlan_config_cmd wowlan_config_cmd = {};
 
-		wowlan_config_cmd.sta_id = mvmvif->ap_sta_id;
+		wowlan_config_cmd.sta_id = mvmvif->deflink.ap_sta_id;
 
 		ap_sta = rcu_dereference_protected(
-			mvm->fw_id_to_mac_id[mvmvif->ap_sta_id],
+			mvm->fw_id_to_mac_id[mvmvif->deflink.ap_sta_id],
 			lockdep_is_held(&mvm->mutex));
 		if (IS_ERR_OR_NULL(ap_sta)) {
 			ret = -EINVAL;
@@ -2015,6 +2018,12 @@ static void iwl_mvm_parse_wowlan_info_notif(struct iwl_mvm *mvm,
 {
 	u32 i;
 
+	if (!data) {
+		IWL_ERR(mvm, "iwl_wowlan_info_notif data is NULL\n");
+		status = NULL;
+		return;
+	}
+
 	if (len < sizeof(*data)) {
 		IWL_ERR(mvm, "Invalid WoWLAN info notification!\n");
 		status = NULL;
@@ -2575,7 +2584,8 @@ iwl_mvm_choose_query_wakeup_reasons(struct iwl_mvm *mvm,
 	/* if FW uses status notification, status shouldn't be NULL here */
 	if (!d3_data->status) {
 		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-		u8 sta_id = mvm->net_detect ? IWL_MVM_INVALID_STA : mvmvif->ap_sta_id;
+		u8 sta_id = mvm->net_detect ? IWL_MVM_INVALID_STA :
+					      mvmvif->deflink.ap_sta_id;
 
 		d3_data->status = iwl_mvm_send_wowlan_get_status(mvm, sta_id);
 	}
@@ -2702,10 +2712,15 @@ static bool iwl_mvm_wait_d3_notif(struct iwl_notif_wait_data *notif_wait,
 	struct iwl_d3_data *d3_data = data;
 	u32 len;
 	int ret;
+	int wowlan_info_ver = iwl_fw_lookup_notif_ver(mvm->fw,
+						      PROT_OFFLOAD_GROUP,
+						      WOWLAN_INFO_NOTIFICATION,
+						      IWL_FW_CMD_VER_UNKNOWN);
+
 
 	switch (WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)) {
 	case WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_INFO_NOTIFICATION): {
-		struct iwl_wowlan_info_notif *notif = (void *)pkt->data;
+		struct iwl_wowlan_info_notif *notif;
 
 		if (d3_data->notif_received & IWL_D3_NOTIF_WOWLAN_INFO) {
 			/* We might get two notifications due to dual bss */
@@ -2714,10 +2729,32 @@ static bool iwl_mvm_wait_d3_notif(struct iwl_notif_wait_data *notif_wait,
 			break;
 		}
 
+		if (wowlan_info_ver < 2) {
+			struct iwl_wowlan_info_notif_v1 *notif_v1 = (void *)pkt->data;
+
+			notif = kmemdup(notif_v1,
+					offsetofend(struct iwl_wowlan_info_notif,
+						    received_beacons),
+					GFP_ATOMIC);
+
+			if (!notif)
+				return false;
+
+			notif->tid_tear_down = notif_v1->tid_tear_down;
+			notif->station_id = notif_v1->station_id;
+
+		} else {
+			notif = (void *)pkt->data;
+		}
+
 		d3_data->notif_received |= IWL_D3_NOTIF_WOWLAN_INFO;
 		len = iwl_rx_packet_payload_len(pkt);
 		iwl_mvm_parse_wowlan_info_notif(mvm, notif, d3_data->status,
 						len);
+
+		if (wowlan_info_ver < 2)
+			kfree(notif);
+
 		if (d3_data->status &&
 		    d3_data->status->wakeup_reasons & IWL_WOWLAN_WAKEUP_REASON_HAS_WAKEUP_PKT)
 			/* We are supposed to get also wake packet notif */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
index 3779ac040ba0..3613b1fdc5d9 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
@@ -179,7 +179,7 @@ static ssize_t iwl_dbgfs_mac_params_read(struct file *file,
 
 	mutex_lock(&mvm->mutex);
 
-	ap_sta_id = mvmvif->ap_sta_id;
+	ap_sta_id = mvmvif->deflink.ap_sta_id;
 
 	switch (ieee80211_vif_type_p2p(vif)) {
 	case NL80211_IFTYPE_ADHOC:
@@ -211,14 +211,14 @@ static ssize_t iwl_dbgfs_mac_params_read(struct file *file,
 	pos += scnprintf(buf+pos, bufsz-pos, "Load: %d\n",
 			 mvm->tcm.result.load[mvmvif->id]);
 	pos += scnprintf(buf+pos, bufsz-pos, "QoS:\n");
-	for (i = 0; i < ARRAY_SIZE(mvmvif->queue_params); i++)
+	for (i = 0; i < ARRAY_SIZE(mvmvif->deflink.queue_params); i++)
 		pos += scnprintf(buf+pos, bufsz-pos,
 				 "\t%d: txop:%d - cw_min:%d - cw_max = %d - aifs = %d upasd = %d\n",
-				 i, mvmvif->queue_params[i].txop,
-				 mvmvif->queue_params[i].cw_min,
-				 mvmvif->queue_params[i].cw_max,
-				 mvmvif->queue_params[i].aifs,
-				 mvmvif->queue_params[i].uapsd);
+				 i, mvmvif->deflink.queue_params[i].txop,
+				 mvmvif->deflink.queue_params[i].cw_min,
+				 mvmvif->deflink.queue_params[i].cw_max,
+				 mvmvif->deflink.queue_params[i].aifs,
+				 mvmvif->deflink.queue_params[i].uapsd);
 
 	if (vif->type == NL80211_IFTYPE_STATION &&
 	    ap_sta_id != IWL_MVM_INVALID_STA) {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
index 85b99316d029..84a488538427 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2023 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -8,6 +8,7 @@
 #include <linux/err.h>
 #include <linux/ieee80211.h>
 #include <linux/netdevice.h>
+#include <linux/dmi.h>
 
 #include "mvm.h"
 #include "sta.h"
@@ -15,6 +16,7 @@
 #include "debugfs.h"
 #include "iwl-modparams.h"
 #include "fw/error-dump.h"
+#include "fw/api/phy-ctxt.h"
 
 static ssize_t iwl_dbgfs_ctdp_budget_read(struct file *file,
 					  char __user *user_buf,
@@ -214,9 +216,9 @@ static ssize_t iwl_dbgfs_set_nic_temperature_read(struct file *file,
 	int pos;
 
 	if (!mvm->temperature_test)
-		pos = scnprintf(buf , sizeof(buf), "disabled\n");
+		pos = scnprintf(buf, sizeof(buf), "disabled\n");
 	else
-		pos = scnprintf(buf , sizeof(buf), "%d\n", mvm->temperature);
+		pos = scnprintf(buf, sizeof(buf), "%d\n", mvm->temperature);
 
 	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
 }
@@ -261,7 +263,7 @@ static ssize_t iwl_dbgfs_set_nic_temperature_write(struct iwl_mvm *mvm,
 		mvm->temperature = temperature;
 	}
 	IWL_DEBUG_TEMP(mvm, "%sabling debug set temperature (temp = %d)\n",
-		       mvm->temperature_test ? "En" : "Dis" ,
+		       mvm->temperature_test ? "En" : "Dis",
 		       mvm->temperature);
 	/* handle the temperature change */
 	iwl_mvm_tt_handler(mvm);
@@ -291,7 +293,7 @@ static ssize_t iwl_dbgfs_nic_temp_read(struct file *file,
 	if (ret)
 		return -EIO;
 
-	pos = scnprintf(buf , sizeof(buf), "%d\n", temp);
+	pos = scnprintf(buf, sizeof(buf), "%d\n", temp);
 
 	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
 }
@@ -338,6 +340,26 @@ static ssize_t iwl_dbgfs_sar_geo_profile_read(struct file *file,
 
 	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
 }
+
+static ssize_t iwl_dbgfs_wifi_6e_enable_read(struct file *file,
+					     char __user *user_buf,
+					     size_t count, loff_t *ppos)
+{
+	struct iwl_mvm *mvm = file->private_data;
+	int err, pos;
+	char buf[12];
+	u32 value;
+
+	err = iwl_acpi_get_dsm_u32(mvm->fwrt.dev, 0,
+				   DSM_FUNC_ENABLE_6E,
+				   &iwl_guid, &value);
+	if (err)
+		return err;
+
+	pos = sprintf(buf, "0x%08x\n", value);
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
+}
 #endif
 
 static ssize_t iwl_dbgfs_stations_read(struct file *file, char __user *user_buf,
@@ -374,7 +396,7 @@ static ssize_t iwl_dbgfs_rs_data_read(struct file *file, char __user *user_buf,
 {
 	struct ieee80211_sta *sta = file->private_data;
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_lq_sta_rs_fw *lq_sta = &mvmsta->lq_sta.rs_fw;
+	struct iwl_lq_sta_rs_fw *lq_sta = &mvmsta->deflink.lq_sta.rs_fw;
 	struct iwl_mvm *mvm = lq_sta->pers.drv;
 	static const size_t bufsz = 2048;
 	char *buff;
@@ -714,6 +736,190 @@ static ssize_t iwl_dbgfs_fw_ver_read(struct file *file, char __user *user_buf,
 	return ret;
 }
 
+static ssize_t iwl_dbgfs_tas_get_status_read(struct file *file,
+					     char __user *user_buf,
+					     size_t count, loff_t *ppos)
+{
+	struct iwl_mvm *mvm = file->private_data;
+	struct iwl_mvm_tas_status_resp tas_rsp;
+	struct iwl_mvm_tas_status_resp *rsp = &tas_rsp;
+	static const size_t bufsz = 1024;
+	char *buff, *pos, *endpos;
+	const char * const tas_dis_reason[TAS_DISABLED_REASON_MAX] = {
+		[TAS_DISABLED_DUE_TO_BIOS] =
+			"Due To BIOS",
+		[TAS_DISABLED_DUE_TO_SAR_6DBM] =
+			"Due To SAR Limit Less Than 6 dBm",
+		[TAS_DISABLED_REASON_INVALID] =
+			"N/A",
+	};
+	const char * const tas_current_status[TAS_DYNA_STATUS_MAX] = {
+		[TAS_DYNA_INACTIVE] = "INACTIVE",
+		[TAS_DYNA_INACTIVE_MVM_MODE] =
+			"inactive due to mvm mode",
+		[TAS_DYNA_INACTIVE_TRIGGER_MODE] =
+			"inactive due to trigger mode",
+		[TAS_DYNA_INACTIVE_BLOCK_LISTED] =
+			"inactive due to block listed",
+		[TAS_DYNA_INACTIVE_UHB_NON_US] =
+			"inactive due to uhb non US",
+		[TAS_DYNA_ACTIVE] = "ACTIVE",
+	};
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(DEBUG_GROUP, GET_TAS_STATUS),
+		.flags = CMD_WANT_SKB,
+		.len = { 0, },
+		.data = { NULL, },
+	};
+	int ret, i, tmp;
+	bool tas_enabled = false;
+	unsigned long dyn_status;
+
+	if (!iwl_mvm_firmware_running(mvm))
+		return -ENODEV;
+
+	mutex_lock(&mvm->mutex);
+	ret = iwl_mvm_send_cmd(mvm, &hcmd);
+	mutex_unlock(&mvm->mutex);
+	if (ret < 0)
+		return ret;
+
+	buff = kzalloc(bufsz, GFP_KERNEL);
+	if (!buff)
+		return -ENOMEM;
+	pos = buff;
+	endpos = pos + bufsz;
+
+	rsp = (void *)hcmd.resp_pkt->data;
+
+	pos += scnprintf(pos, endpos - pos, "TAS Conclusion:\n");
+	for (i = 0; i < rsp->in_dual_radio + 1; i++) {
+		if (rsp->tas_status_mac[i].band != TAS_LMAC_BAND_INVALID &&
+		    rsp->tas_status_mac[i].dynamic_status & BIT(TAS_DYNA_ACTIVE)) {
+			pos += scnprintf(pos, endpos - pos, "\tON for ");
+			switch (rsp->tas_status_mac[i].band) {
+			case TAS_LMAC_BAND_HB:
+				pos += scnprintf(pos, endpos - pos, "HB\n");
+				break;
+			case TAS_LMAC_BAND_LB:
+				pos += scnprintf(pos, endpos - pos, "LB\n");
+				break;
+			case TAS_LMAC_BAND_UHB:
+				pos += scnprintf(pos, endpos - pos, "UHB\n");
+				break;
+			case TAS_LMAC_BAND_INVALID:
+				pos += scnprintf(pos, endpos - pos,
+						 "INVALID BAND\n");
+				break;
+			default:
+				pos += scnprintf(pos, endpos - pos,
+						 "Unsupported band (%d)\n",
+						 rsp->tas_status_mac[i].band);
+				goto out;
+			}
+			tas_enabled = true;
+		}
+	}
+	if (!tas_enabled)
+		pos += scnprintf(pos, endpos - pos, "\tOFF\n");
+
+	pos += scnprintf(pos, endpos - pos, "TAS Report\n");
+	pos += scnprintf(pos, endpos - pos, "TAS FW version: %d\n",
+			 rsp->tas_fw_version);
+	pos += scnprintf(pos, endpos - pos, "Is UHB enabled for USA?: %s\n",
+			 rsp->is_uhb_for_usa_enable ? "True" : "False");
+	pos += scnprintf(pos, endpos - pos, "Current MCC: 0x%x\n",
+			 le16_to_cpu(rsp->curr_mcc));
+
+	pos += scnprintf(pos, endpos - pos, "Block list entries:");
+	for (i = 0; i < APCI_WTAS_BLACK_LIST_MAX; i++)
+		pos += scnprintf(pos, endpos - pos, " 0x%x",
+				 le16_to_cpu(rsp->block_list[i]));
+
+	pos += scnprintf(pos, endpos - pos, "\nOEM name: %s\n",
+			 dmi_get_system_info(DMI_SYS_VENDOR));
+	pos += scnprintf(pos, endpos - pos, "\tVendor In Approved List: %s\n",
+			 iwl_mvm_is_vendor_in_approved_list() ? "YES" : "NO");
+	pos += scnprintf(pos, endpos - pos,
+			 "\tDo TAS Support Dual Radio?: %s\n",
+			 rsp->in_dual_radio ? "TRUE" : "FALSE");
+
+	for (i = 0; i < rsp->in_dual_radio + 1; i++) {
+		if (rsp->tas_status_mac[i].static_status == 0) {
+			pos += scnprintf(pos, endpos - pos,
+					 "Static status: disabled\n");
+			pos += scnprintf(pos, endpos - pos,
+					 "Static disabled reason: %s (0)\n",
+					 tas_dis_reason[0]);
+			goto out;
+		}
+
+		pos += scnprintf(pos, endpos - pos, "TAS status for ");
+		switch (rsp->tas_status_mac[i].band) {
+		case TAS_LMAC_BAND_HB:
+			pos += scnprintf(pos, endpos - pos, "High band\n");
+			break;
+		case TAS_LMAC_BAND_LB:
+			pos += scnprintf(pos, endpos - pos, "Low band\n");
+			break;
+		case TAS_LMAC_BAND_UHB:
+			pos += scnprintf(pos, endpos - pos,
+					 "Ultra high band\n");
+			break;
+		case TAS_LMAC_BAND_INVALID:
+			pos += scnprintf(pos, endpos - pos,
+					 "INVALID band\n");
+			break;
+		default:
+			pos += scnprintf(pos, endpos - pos,
+					 "Unsupported band (%d)\n",
+					 rsp->tas_status_mac[i].band);
+			goto out;
+		}
+		pos += scnprintf(pos, endpos - pos, "Static status: %sabled\n",
+				 rsp->tas_status_mac[i].static_status ?
+				 "En" : "Dis");
+		pos += scnprintf(pos, endpos - pos,
+				 "\tStatic Disabled Reason: ");
+		if (rsp->tas_status_mac[i].static_dis_reason < TAS_DISABLED_REASON_MAX)
+			pos += scnprintf(pos, endpos - pos, "%s (%d)\n",
+					 tas_dis_reason[rsp->tas_status_mac[i].static_dis_reason],
+					 rsp->tas_status_mac[i].static_dis_reason);
+		else
+			pos += scnprintf(pos, endpos - pos,
+					 "unsupported value (%d)\n",
+					 rsp->tas_status_mac[i].static_dis_reason);
+
+		pos += scnprintf(pos, endpos - pos, "Dynamic status:\n");
+		dyn_status = (rsp->tas_status_mac[i].dynamic_status);
+		for_each_set_bit(tmp, &dyn_status, sizeof(dyn_status)) {
+			if (tmp >= 0 && tmp < TAS_DYNA_STATUS_MAX)
+				pos += scnprintf(pos, endpos - pos,
+						 "\t%s (%d)\n",
+						 tas_current_status[tmp], tmp);
+		}
+
+		pos += scnprintf(pos, endpos - pos,
+				 "Is near disconnection?: %s\n",
+				 rsp->tas_status_mac[i].near_disconnection ?
+				 "True" : "False");
+		tmp = le16_to_cpu(rsp->tas_status_mac[i].max_reg_pwr_limit);
+		pos += scnprintf(pos, endpos - pos,
+				 "Max. regulatory pwr limit (dBm): %d.%03d\n",
+				 tmp / 8, 125 * (tmp % 8));
+		tmp = le16_to_cpu(rsp->tas_status_mac[i].sar_limit);
+		pos += scnprintf(pos, endpos - pos,
+				 "SAR limit (dBm): %d.%03d\n",
+				 tmp / 8, 125 * (tmp % 8));
+	}
+
+out:
+	ret = simple_read_from_buffer(user_buf, count, ppos, buff, pos - buff);
+	kfree(buff);
+	iwl_free_resp(&hcmd);
+	return ret;
+}
+
 static ssize_t iwl_dbgfs_phy_integration_ver_read(struct file *file,
 						  char __user *user_buf,
 						  size_t count, loff_t *ppos)
@@ -1202,6 +1408,7 @@ static int _iwl_dbgfs_inject_beacon_ie(struct iwl_mvm *mvm, char *bin, int len)
 	struct sk_buff *beacon;
 	struct ieee80211_tx_info *info;
 	struct iwl_mac_beacon_cmd beacon_cmd = {};
+	unsigned int link_id;
 	u8 rate;
 	int i;
 
@@ -1250,17 +1457,24 @@ static int _iwl_dbgfs_inject_beacon_ie(struct iwl_mvm *mvm, char *bin, int len)
 	info = IEEE80211_SKB_CB(beacon);
 	rate = iwl_mvm_mac_ctxt_get_beacon_rate(mvm, info, vif);
 
-	beacon_cmd.flags =
-		cpu_to_le16(iwl_mvm_mac_ctxt_get_beacon_flags(mvm->fw, rate));
-	beacon_cmd.byte_cnt = cpu_to_le16((u16)beacon->len);
-	beacon_cmd.template_id = cpu_to_le32((u32)mvmvif->id);
+	for_each_mvm_vif_valid_link(mvmvif, link_id) {
+		beacon_cmd.flags =
+			cpu_to_le16(iwl_mvm_mac_ctxt_get_beacon_flags(mvm->fw,
+								      rate));
+		beacon_cmd.byte_cnt = cpu_to_le16((u16)beacon->len);
+		if (iwl_fw_lookup_cmd_ver(mvm->fw, BEACON_TEMPLATE_CMD, 0) > 12)
+			beacon_cmd.link_id =
+				cpu_to_le32(mvmvif->link[link_id]->fw_link_id);
+		else
+			beacon_cmd.link_id = cpu_to_le32((u32)mvmvif->id);
 
-	iwl_mvm_mac_ctxt_set_tim(mvm, &beacon_cmd.tim_idx,
-				 &beacon_cmd.tim_size,
-				 beacon->data, beacon->len);
+		iwl_mvm_mac_ctxt_set_tim(mvm, &beacon_cmd.tim_idx,
+					 &beacon_cmd.tim_size,
+					 beacon->data, beacon->len);
 
-	iwl_mvm_mac_ctxt_send_beacon_cmd(mvm, beacon, &beacon_cmd,
-					 sizeof(beacon_cmd));
+		iwl_mvm_mac_ctxt_send_beacon_cmd(mvm, beacon, &beacon_cmd,
+						 sizeof(beacon_cmd));
+	}
 	mutex_unlock(&mvm->mutex);
 
 	dev_kfree_skb(beacon);
@@ -1685,6 +1899,7 @@ MVM_DEBUGFS_READ_FILE_OPS(fw_rx_stats);
 MVM_DEBUGFS_READ_FILE_OPS(drv_rx_stats);
 MVM_DEBUGFS_READ_FILE_OPS(fw_ver);
 MVM_DEBUGFS_READ_FILE_OPS(phy_integration_ver);
+MVM_DEBUGFS_READ_FILE_OPS(tas_get_status);
 MVM_DEBUGFS_WRITE_FILE_OPS(fw_restart, 10);
 MVM_DEBUGFS_WRITE_FILE_OPS(fw_nmi, 10);
 MVM_DEBUGFS_WRITE_FILE_OPS(bt_tx_prio, 10);
@@ -1703,6 +1918,7 @@ MVM_DEBUGFS_READ_FILE_OPS(uapsd_noagg_bssids);
 
 #ifdef CONFIG_ACPI
 MVM_DEBUGFS_READ_FILE_OPS(sar_geo_profile);
+MVM_DEBUGFS_READ_FILE_OPS(wifi_6e_enable);
 #endif
 
 MVM_DEBUGFS_READ_WRITE_STA_FILE_OPS(amsdu_len, 16);
@@ -1745,6 +1961,11 @@ static ssize_t iwl_dbgfs_mem_read(struct file *file, char __user *user_buf,
 	if (ret < 0)
 		return ret;
 
+	if (iwl_rx_packet_payload_len(hcmd.resp_pkt) < sizeof(*rsp)) {
+		ret = -EIO;
+		goto out;
+	}
+
 	rsp = (void *)hcmd.resp_pkt->data;
 	if (le32_to_cpu(rsp->status) != DEBUG_MEM_STATUS_SUCCESS) {
 		ret = -ENXIO;
@@ -1821,6 +2042,11 @@ static ssize_t iwl_dbgfs_mem_write(struct file *file,
 	if (ret < 0)
 		return ret;
 
+	if (iwl_rx_packet_payload_len(hcmd.resp_pkt) < sizeof(*rsp)) {
+		ret = -EIO;
+		goto out;
+	}
+
 	rsp = (void *)hcmd.resp_pkt->data;
 	if (rsp->status != DEBUG_MEM_STATUS_SUCCESS) {
 		ret = -ENXIO;
@@ -1894,8 +2120,10 @@ void iwl_mvm_dbgfs_register(struct iwl_mvm *mvm)
 
 	if (mvm->fw->phy_integration_ver)
 		MVM_DEBUGFS_ADD_FILE(phy_integration_ver, mvm->debugfs_dir, 0400);
+	MVM_DEBUGFS_ADD_FILE(tas_get_status, mvm->debugfs_dir, 0400);
 #ifdef CONFIG_ACPI
 	MVM_DEBUGFS_ADD_FILE(sar_geo_profile, mvm->debugfs_dir, 0400);
+	MVM_DEBUGFS_ADD_FILE(wifi_6e_enable, mvm->debugfs_dir, 0400);
 #endif
 	MVM_DEBUGFS_ADD_FILE(he_sniffer_params, mvm->debugfs_dir, 0600);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
index 05f3136b1c43..3963a0d4ed04 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
@@ -25,6 +25,10 @@ struct iwl_mvm_smooth_entry {
 	u64 host_time;
 };
 
+enum iwl_mvm_pasn_flags {
+	IWL_MVM_PASN_FLAG_HAS_HLTK = BIT(0),
+};
+
 struct iwl_mvm_ftm_pasn_entry {
 	struct list_head list;
 	u8 addr[ETH_ALEN];
@@ -33,6 +37,7 @@ struct iwl_mvm_ftm_pasn_entry {
 	u8 cipher;
 	u8 tx_pn[IEEE80211_CCMP_PN_LEN];
 	u8 rx_pn[IEEE80211_CCMP_PN_LEN];
+	u32 flags;
 };
 
 int iwl_mvm_ftm_add_pasn_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
@@ -73,20 +78,30 @@ int iwl_mvm_ftm_add_pasn_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		struct ieee80211_sta *sta;
 
 		rcu_read_lock();
-		sta = rcu_dereference(mvm->fw_id_to_mac_id[mvmvif->ap_sta_id]);
+		sta = rcu_dereference(mvm->fw_id_to_mac_id[mvmvif->deflink.ap_sta_id]);
 		if (!IS_ERR_OR_NULL(sta) && sta->mfp)
 			expected_tk_len = 0;
 		rcu_read_unlock();
 	}
 
-	if (tk_len != expected_tk_len || hltk_len != sizeof(pasn->hltk)) {
+	if (tk_len != expected_tk_len ||
+	    (hltk_len && hltk_len != sizeof(pasn->hltk))) {
 		IWL_ERR(mvm, "Invalid key length: tk_len=%u hltk_len=%u\n",
 			tk_len, hltk_len);
 		goto out;
 	}
 
+	if (!expected_tk_len && !hltk_len) {
+		IWL_ERR(mvm, "TK and HLTK not set\n");
+		goto out;
+	}
+
 	memcpy(pasn->addr, addr, sizeof(pasn->addr));
-	memcpy(pasn->hltk, hltk, sizeof(pasn->hltk));
+
+	if (hltk_len) {
+		memcpy(pasn->hltk, hltk, sizeof(pasn->hltk));
+		pasn->flags |= IWL_MVM_PASN_FLAG_HAS_HLTK;
+	}
 
 	if (tk && tk_len)
 		memcpy(pasn->tk, tk, sizeof(pasn->tk));
@@ -510,13 +525,13 @@ iwl_mvm_ftm_put_target(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 		rcu_read_lock();
 
-		sta = rcu_dereference(mvm->fw_id_to_mac_id[mvmvif->ap_sta_id]);
+		sta = rcu_dereference(mvm->fw_id_to_mac_id[mvmvif->deflink.ap_sta_id]);
 		if (sta->mfp && (peer->ftm.trigger_based || peer->ftm.non_trigger_based))
 			FTM_PUT_FLAG(PMF);
 
 		rcu_read_unlock();
 
-		target->sta_id = mvmvif->ap_sta_id;
+		target->sta_id = mvmvif->deflink.ap_sta_id;
 	} else {
 		target->sta_id = IWL_MVM_INVALID_STA;
 	}
@@ -691,7 +706,11 @@ iwl_mvm_ftm_set_secured_ranging(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			continue;
 
 		target->cipher = entry->cipher;
-		memcpy(target->hltk, entry->hltk, sizeof(target->hltk));
+
+		if (entry->flags & IWL_MVM_PASN_FLAG_HAS_HLTK)
+			memcpy(target->hltk, entry->hltk, sizeof(target->hltk));
+		else
+			memset(target->hltk, 0, sizeof(target->hltk));
 
 		if (vif->cfg.assoc &&
 		    !memcmp(vif->bss_conf.bssid, target->bssid,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-responder.c b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-responder.c
index e862d1b43f21..1b6fb73ddfc7 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-responder.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-responder.c
@@ -119,7 +119,7 @@ iwl_mvm_ftm_responder_cmd(struct iwl_mvm *mvm,
 			cpu_to_le32(IWL_TOF_RESPONDER_CMD_VALID_CHAN_INFO |
 				    IWL_TOF_RESPONDER_CMD_VALID_BSSID |
 				    IWL_TOF_RESPONDER_CMD_VALID_STA_ID),
-		.sta_id = mvmvif->bcast_sta.sta_id,
+		.sta_id = mvmvif->deflink.bcast_sta.sta_id,
 	};
 	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id, 6);
 	int err;
@@ -317,6 +317,8 @@ int iwl_mvm_ftm_respoder_add_pasn_sta(struct iwl_mvm *mvm,
 		.addr = addr,
 		.hltk = hltk,
 	};
+	struct iwl_mvm_pasn_hltk_data *hltk_data_ptr = NULL;
+
 	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw,
 					   WIDE_ID(LOCATION_GROUP, TOF_RESPONDER_DYN_CONFIG_CMD),
 					   2);
@@ -328,12 +330,21 @@ int iwl_mvm_ftm_respoder_add_pasn_sta(struct iwl_mvm *mvm,
 		return -ENOTSUPP;
 	}
 
-	hltk_data.cipher = iwl_mvm_cipher_to_location_cipher(cipher);
-	if (hltk_data.cipher == IWL_LOCATION_CIPHER_INVALID) {
-		IWL_ERR(mvm, "invalid cipher: %u\n", cipher);
+	if ((!hltk || !hltk_len) && (!tk || !tk_len)) {
+		IWL_ERR(mvm, "TK and HLTK not set\n");
 		return -EINVAL;
 	}
 
+	if (hltk && hltk_len) {
+		hltk_data.cipher = iwl_mvm_cipher_to_location_cipher(cipher);
+		if (hltk_data.cipher == IWL_LOCATION_CIPHER_INVALID) {
+			IWL_ERR(mvm, "invalid cipher: %u\n", cipher);
+			return -EINVAL;
+		}
+
+		hltk_data_ptr = &hltk_data;
+	}
+
 	if (tk && tk_len) {
 		sta = kzalloc(sizeof(*sta), GFP_KERNEL);
 		if (!sta)
@@ -350,7 +361,7 @@ int iwl_mvm_ftm_respoder_add_pasn_sta(struct iwl_mvm *mvm,
 		list_add_tail(&sta->list, &mvm->resp_pasn_list);
 	}
 
-	ret = iwl_mvm_ftm_responder_dyn_cfg_v3(mvm, vif, NULL, &hltk_data);
+	ret = iwl_mvm_ftm_responder_dyn_cfg_v3(mvm, vif, NULL, hltk_data_ptr);
 	if (ret && sta)
 		iwl_mvm_resp_del_pasn_sta(mvm, vif, sta);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
index 0c6b49fcb00d..b35c96cf7ad2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
@@ -21,6 +21,7 @@
 #include "iwl-phy-db.h"
 #include "iwl-modparams.h"
 #include "iwl-nvm-parse.h"
+#include "time-sync.h"
 
 #define MVM_UCODE_ALIVE_TIMEOUT	(HZ)
 #define MVM_UCODE_CALIB_TIMEOUT	(2 * HZ)
@@ -122,8 +123,6 @@ static bool iwl_alive_fn(struct iwl_notif_wait_data *notif_wait,
 	u32 version = iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP,
 					      UCODE_ALIVE_NTFY, 0);
 	u32 i;
-	struct iwl_trans *trans = mvm->trans;
-	enum iwl_device_family device_family = trans->trans_cfg->device_family;
 
 
 	if (version == 6) {
@@ -233,8 +232,7 @@ static bool iwl_alive_fn(struct iwl_notif_wait_data *notif_wait,
 
 	if (umac_error_table) {
 		if (umac_error_table >=
-		    mvm->trans->cfg->min_umac_error_event_table ||
-		    device_family >= IWL_DEVICE_FAMILY_BZ) {
+		    mvm->trans->cfg->min_umac_error_event_table) {
 			iwl_fw_umac_set_alive_err_table(mvm->trans,
 							umac_error_table);
 		} else {
@@ -323,6 +321,8 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 	static const u16 alive_cmd[] = { UCODE_ALIVE_NTFY };
 	bool run_in_rfkill =
 		ucode_type == IWL_UCODE_INIT || iwl_mvm_has_unified_ucode(mvm);
+	u8 count;
+	struct iwl_pc_data *pc_data;
 
 	if (ucode_type == IWL_UCODE_REGULAR &&
 	    iwl_fw_dbg_conf_usniffer(mvm->fw, FW_DBG_START_FROM_ALIVE) &&
@@ -395,6 +395,14 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 
 		/* LMAC/UMAC PC info */
 		if (trans->trans_cfg->device_family >=
+					IWL_DEVICE_FAMILY_22000) {
+			pc_data = trans->dbg.pc_data;
+			for (count = 0; count < trans->dbg.num_pc;
+			     count++, pc_data++)
+				IWL_ERR(mvm, "%s: 0x%x\n",
+					pc_data->pc_name,
+					pc_data->pc_address);
+		} else if (trans->trans_cfg->device_family >=
 					IWL_DEVICE_FAMILY_9000) {
 			IWL_ERR(mvm, "UMAC PC: 0x%x\n",
 				iwl_read_umac_prph(trans,
@@ -469,111 +477,6 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 	return 0;
 }
 
-static int iwl_run_unified_mvm_ucode(struct iwl_mvm *mvm)
-{
-	struct iwl_notification_wait init_wait;
-	struct iwl_nvm_access_complete_cmd nvm_complete = {};
-	struct iwl_init_extended_cfg_cmd init_cfg = {
-		.init_flags = cpu_to_le32(BIT(IWL_INIT_NVM)),
-	};
-	static const u16 init_complete[] = {
-		INIT_COMPLETE_NOTIF,
-	};
-	int ret;
-
-	if (mvm->trans->cfg->tx_with_siso_diversity)
-		init_cfg.init_flags |= cpu_to_le32(BIT(IWL_INIT_PHY));
-
-	lockdep_assert_held(&mvm->mutex);
-
-	mvm->rfkill_safe_init_done = false;
-
-	iwl_init_notification_wait(&mvm->notif_wait,
-				   &init_wait,
-				   init_complete,
-				   ARRAY_SIZE(init_complete),
-				   iwl_wait_init_complete,
-				   NULL);
-
-	iwl_dbg_tlv_time_point(&mvm->fwrt, IWL_FW_INI_TIME_POINT_EARLY, NULL);
-
-	/* Will also start the device */
-	ret = iwl_mvm_load_ucode_wait_alive(mvm, IWL_UCODE_REGULAR);
-	if (ret) {
-		IWL_ERR(mvm, "Failed to start RT ucode: %d\n", ret);
-		goto error;
-	}
-	iwl_dbg_tlv_time_point(&mvm->fwrt, IWL_FW_INI_TIME_POINT_AFTER_ALIVE,
-			       NULL);
-
-	/* Send init config command to mark that we are sending NVM access
-	 * commands
-	 */
-	ret = iwl_mvm_send_cmd_pdu(mvm, WIDE_ID(SYSTEM_GROUP,
-						INIT_EXTENDED_CFG_CMD),
-				   CMD_SEND_IN_RFKILL,
-				   sizeof(init_cfg), &init_cfg);
-	if (ret) {
-		IWL_ERR(mvm, "Failed to run init config command: %d\n",
-			ret);
-		goto error;
-	}
-
-	/* Load NVM to NIC if needed */
-	if (mvm->nvm_file_name) {
-		ret = iwl_read_external_nvm(mvm->trans, mvm->nvm_file_name,
-					    mvm->nvm_sections);
-		if (ret)
-			goto error;
-		ret = iwl_mvm_load_nvm_to_nic(mvm);
-		if (ret)
-			goto error;
-	}
-
-	if (IWL_MVM_PARSE_NVM && !mvm->nvm_data) {
-		ret = iwl_nvm_init(mvm);
-		if (ret) {
-			IWL_ERR(mvm, "Failed to read NVM: %d\n", ret);
-			goto error;
-		}
-	}
-
-	ret = iwl_mvm_send_cmd_pdu(mvm, WIDE_ID(REGULATORY_AND_NVM_GROUP,
-						NVM_ACCESS_COMPLETE),
-				   CMD_SEND_IN_RFKILL,
-				   sizeof(nvm_complete), &nvm_complete);
-	if (ret) {
-		IWL_ERR(mvm, "Failed to run complete NVM access: %d\n",
-			ret);
-		goto error;
-	}
-
-	/* We wait for the INIT complete notification */
-	ret = iwl_wait_notification(&mvm->notif_wait, &init_wait,
-				    MVM_UCODE_ALIVE_TIMEOUT);
-	if (ret)
-		return ret;
-
-	/* Read the NVM only at driver load time, no need to do this twice */
-	if (!IWL_MVM_PARSE_NVM && !mvm->nvm_data) {
-		mvm->nvm_data = iwl_get_nvm(mvm->trans, mvm->fw);
-		if (IS_ERR(mvm->nvm_data)) {
-			ret = PTR_ERR(mvm->nvm_data);
-			mvm->nvm_data = NULL;
-			IWL_ERR(mvm, "Failed to read NVM: %d\n", ret);
-			return ret;
-		}
-	}
-
-	mvm->rfkill_safe_init_done = true;
-
-	return 0;
-
-error:
-	iwl_remove_notification(&mvm->notif_wait, &init_wait);
-	return ret;
-}
-
 #ifdef CONFIG_ACPI
 static void iwl_mvm_phy_filter_init(struct iwl_mvm *mvm,
 				    struct iwl_phy_specific_cfg *phy_filters)
@@ -700,6 +603,118 @@ static int iwl_send_phy_cfg_cmd(struct iwl_mvm *mvm)
 	return iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, cmd_size, &phy_cfg_cmd);
 }
 
+static int iwl_run_unified_mvm_ucode(struct iwl_mvm *mvm)
+{
+	struct iwl_notification_wait init_wait;
+	struct iwl_nvm_access_complete_cmd nvm_complete = {};
+	struct iwl_init_extended_cfg_cmd init_cfg = {
+		.init_flags = cpu_to_le32(BIT(IWL_INIT_NVM)),
+	};
+	static const u16 init_complete[] = {
+		INIT_COMPLETE_NOTIF,
+	};
+	int ret;
+
+	if (mvm->trans->cfg->tx_with_siso_diversity)
+		init_cfg.init_flags |= cpu_to_le32(BIT(IWL_INIT_PHY));
+
+	lockdep_assert_held(&mvm->mutex);
+
+	mvm->rfkill_safe_init_done = false;
+
+	iwl_init_notification_wait(&mvm->notif_wait,
+				   &init_wait,
+				   init_complete,
+				   ARRAY_SIZE(init_complete),
+				   iwl_wait_init_complete,
+				   NULL);
+
+	iwl_dbg_tlv_time_point(&mvm->fwrt, IWL_FW_INI_TIME_POINT_EARLY, NULL);
+
+	/* Will also start the device */
+	ret = iwl_mvm_load_ucode_wait_alive(mvm, IWL_UCODE_REGULAR);
+	if (ret) {
+		IWL_ERR(mvm, "Failed to start RT ucode: %d\n", ret);
+		goto error;
+	}
+	iwl_dbg_tlv_time_point(&mvm->fwrt, IWL_FW_INI_TIME_POINT_AFTER_ALIVE,
+			       NULL);
+
+	/* Send init config command to mark that we are sending NVM access
+	 * commands
+	 */
+	ret = iwl_mvm_send_cmd_pdu(mvm, WIDE_ID(SYSTEM_GROUP,
+						INIT_EXTENDED_CFG_CMD),
+				   CMD_SEND_IN_RFKILL,
+				   sizeof(init_cfg), &init_cfg);
+	if (ret) {
+		IWL_ERR(mvm, "Failed to run init config command: %d\n",
+			ret);
+		goto error;
+	}
+
+	/* Load NVM to NIC if needed */
+	if (mvm->nvm_file_name) {
+		ret = iwl_read_external_nvm(mvm->trans, mvm->nvm_file_name,
+					    mvm->nvm_sections);
+		if (ret)
+			goto error;
+		ret = iwl_mvm_load_nvm_to_nic(mvm);
+		if (ret)
+			goto error;
+	}
+
+	if (IWL_MVM_PARSE_NVM && !mvm->nvm_data) {
+		ret = iwl_nvm_init(mvm);
+		if (ret) {
+			IWL_ERR(mvm, "Failed to read NVM: %d\n", ret);
+			goto error;
+		}
+	}
+
+	ret = iwl_mvm_send_cmd_pdu(mvm, WIDE_ID(REGULATORY_AND_NVM_GROUP,
+						NVM_ACCESS_COMPLETE),
+				   CMD_SEND_IN_RFKILL,
+				   sizeof(nvm_complete), &nvm_complete);
+	if (ret) {
+		IWL_ERR(mvm, "Failed to run complete NVM access: %d\n",
+			ret);
+		goto error;
+	}
+
+	ret = iwl_send_phy_cfg_cmd(mvm);
+	if (ret) {
+		IWL_ERR(mvm, "Failed to run PHY configuration: %d\n",
+			ret);
+		goto error;
+	}
+
+	/* We wait for the INIT complete notification */
+	ret = iwl_wait_notification(&mvm->notif_wait, &init_wait,
+				    MVM_UCODE_ALIVE_TIMEOUT);
+	if (ret)
+		return ret;
+
+	/* Read the NVM only at driver load time, no need to do this twice */
+	if (!IWL_MVM_PARSE_NVM && !mvm->nvm_data) {
+		mvm->nvm_data = iwl_get_nvm(mvm->trans, mvm->fw);
+		if (IS_ERR(mvm->nvm_data)) {
+			ret = PTR_ERR(mvm->nvm_data);
+			mvm->nvm_data = NULL;
+			IWL_ERR(mvm, "Failed to read NVM: %d\n", ret);
+			return ret;
+		}
+	}
+
+	mvm->rfkill_safe_init_done = true;
+
+	return 0;
+
+error:
+	iwl_remove_notification(&mvm->notif_wait, &init_wait);
+	return ret;
+}
+
 int iwl_run_init_mvm_ucode(struct iwl_mvm *mvm)
 {
 	struct iwl_notification_wait calib_wait;
@@ -1040,7 +1055,7 @@ int iwl_mvm_ppag_send_cmd(struct iwl_mvm *mvm)
 
 	ret = iwl_read_ppag_table(&mvm->fwrt, &cmd, &cmd_size);
 	/* Not supporting PPAG table is a valid scenario */
-	if(ret < 0)
+	if (ret < 0)
 		return 0;
 
 	IWL_DEBUG_RADIO(mvm, "Sending PER_PLATFORM_ANT_GAIN_CMD\n");
@@ -1084,11 +1099,21 @@ static const struct dmi_system_id dmi_tas_approved_list[] = {
 			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
 		},
 	},
+	{ .ident = "MSFT",
+	  .matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Microsoft Corporation"),
+		},
+	},
 
 	/* keep last */
 	{}
 };
 
+bool iwl_mvm_is_vendor_in_approved_list(void)
+{
+	return dmi_check_system(dmi_tas_approved_list);
+}
+
 static bool iwl_mvm_add_to_tas_block_list(__le32 *list, __le32 *le_size, unsigned int mcc)
 {
 	int i;
@@ -1368,6 +1393,11 @@ static void iwl_mvm_lari_cfg(struct iwl_mvm *mvm)
 {
 }
 
+bool iwl_mvm_is_vendor_in_approved_list(void)
+{
+	return false;
+}
+
 static u8 iwl_mvm_eval_dsm_rfi(struct iwl_mvm *mvm)
 {
 	return DSM_VALUE_RFI_DISABLE;
@@ -1525,12 +1555,11 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 		ret = iwl_send_phy_db_data(mvm->phy_db);
 		if (ret)
 			goto error;
+		ret = iwl_send_phy_cfg_cmd(mvm);
+		if (ret)
+			goto error;
 	}
 
-	ret = iwl_send_phy_cfg_cmd(mvm);
-	if (ret)
-		goto error;
-
 	ret = iwl_mvm_send_bt_init_conf(mvm);
 	if (ret)
 		goto error;
@@ -1557,8 +1586,12 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 	}
 
 	/* init the fw <-> mac80211 STA mapping */
-	for (i = 0; i < mvm->fw->ucode_capa.num_stations; i++)
+	for (i = 0; i < mvm->fw->ucode_capa.num_stations; i++) {
 		RCU_INIT_POINTER(mvm->fw_id_to_mac_id[i], NULL);
+		RCU_INIT_POINTER(mvm->fw_id_to_link_sta[i], NULL);
+	}
+
+	memset(&mvm->fw_link_ids_map, 0, sizeof(mvm->fw_link_ids_map));
 
 	mvm->tdls_cs.peer.sta_id = IWL_MVM_INVALID_STA;
 
@@ -1664,8 +1697,15 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 			goto error;
 	}
 
-	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
+	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
 		iwl_mvm_send_recovery_cmd(mvm, ERROR_RECOVERY_UPDATE_DB);
+		iwl_mvm_time_sync_config(mvm, mvm->time_sync.peer_addr,
+					 IWL_TIME_SYNC_PROTOCOL_TM |
+					 IWL_TIME_SYNC_PROTOCOL_FTM);
+	}
+
+	if (!mvm->ptp_data.ptp_clock)
+		iwl_mvm_ptp_init(mvm);
 
 	if (iwl_acpi_get_eckv(mvm->dev, &mvm->ext_clock_valid))
 		IWL_DEBUG_INFO(mvm, "ECKV table doesn't exist in BIOS\n");
@@ -1687,8 +1727,6 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 	iwl_mvm_tas_init(mvm);
 	iwl_mvm_leds_sync(mvm);
 
-	iwl_mvm_ftm_initiator_smooth_config(mvm);
-
 	if (fw_has_capa(&mvm->fw->ucode_capa,
 			IWL_UCODE_TLV_CAPA_RFIM_SUPPORT)) {
 		if (iwl_mvm_eval_dsm_rfi(mvm) == DSM_VALUE_RFI_ENABLE)
@@ -1735,8 +1773,10 @@ int iwl_mvm_load_d3_fw(struct iwl_mvm *mvm)
 		goto error;
 
 	/* init the fw <-> mac80211 STA mapping */
-	for (i = 0; i < mvm->fw->ucode_capa.num_stations; i++)
+	for (i = 0; i < mvm->fw->ucode_capa.num_stations; i++) {
 		RCU_INIT_POINTER(mvm->fw_id_to_mac_id[i], NULL);
+		RCU_INIT_POINTER(mvm->fw_id_to_link_sta[i], NULL);
+	}
 
 	if (iwl_fw_lookup_cmd_ver(mvm->fw, ADD_STA, 0) < 12) {
 		/*
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/link.c b/drivers/net/wireless/intel/iwlwifi/mvm/link.c
new file mode 100644
index 000000000000..eb828de40a3c
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/link.c
@@ -0,0 +1,294 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2022 - 2023 Intel Corporation
+ */
+#include "mvm.h"
+#include "time-event.h"
+
+static u32 iwl_mvm_get_free_fw_link_id(struct iwl_mvm *mvm,
+				       struct iwl_mvm_vif *mvm_vif)
+{
+	u32 link_id;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	link_id = ffz(mvm->fw_link_ids_map);
+
+	/* this case can happen if there're deactivated but not removed links */
+	if (link_id > IWL_MVM_FW_MAX_LINK_ID)
+		return IWL_MVM_FW_LINK_ID_INVALID;
+
+	mvm->fw_link_ids_map |= BIT(link_id);
+	return link_id;
+}
+
+static void iwl_mvm_release_fw_link_id(struct iwl_mvm *mvm, u32 link_id)
+{
+	lockdep_assert_held(&mvm->mutex);
+
+	if (!WARN_ON(link_id > IWL_MVM_FW_MAX_LINK_ID))
+		mvm->fw_link_ids_map &= ~BIT(link_id);
+}
+
+static int iwl_mvm_link_cmd_send(struct iwl_mvm *mvm,
+				 struct iwl_link_config_cmd *cmd,
+				 enum iwl_ctxt_action action)
+{
+	int ret;
+
+	cmd->action = cpu_to_le32(action);
+	ret = iwl_mvm_send_cmd_pdu(mvm,
+				   WIDE_ID(MAC_CONF_GROUP, LINK_CONFIG_CMD), 0,
+				   sizeof(*cmd), cmd);
+	if (ret)
+		IWL_ERR(mvm, "Failed to send LINK_CONFIG_CMD (action:%d): %d\n",
+			action, ret);
+	return ret;
+}
+
+int iwl_mvm_add_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+		     struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	unsigned int link_id = link_conf->link_id;
+	struct iwl_mvm_vif_link_info *link_info = mvmvif->link[link_id];
+	struct iwl_link_config_cmd cmd = {};
+	struct iwl_mvm_phy_ctxt *phyctxt;
+
+	if (WARN_ON_ONCE(!link_info))
+		return -EINVAL;
+
+	if (link_info->fw_link_id == IWL_MVM_FW_LINK_ID_INVALID) {
+		link_info->fw_link_id = iwl_mvm_get_free_fw_link_id(mvm,
+								    mvmvif);
+		if (link_info->fw_link_id == IWL_MVM_FW_LINK_ID_INVALID)
+			return -EINVAL;
+	}
+
+	/* Update SF - Disable if needed. if this fails, SF might still be on
+	 * while many macs are bound, which is forbidden - so fail the binding.
+	 */
+	if (iwl_mvm_sf_update(mvm, vif, false))
+		return -EINVAL;
+
+	cmd.link_id = cpu_to_le32(link_info->fw_link_id);
+	cmd.mac_id = cpu_to_le32(mvmvif->id);
+	/* P2P-Device already has a valid PHY context during add */
+	phyctxt = link_info->phy_ctxt;
+	if (phyctxt)
+		cmd.phy_id = cpu_to_le32(phyctxt->id);
+	else
+		cmd.phy_id = cpu_to_le32(FW_CTXT_INVALID);
+
+	memcpy(cmd.local_link_addr, link_conf->addr, ETH_ALEN);
+
+	if (vif->type == NL80211_IFTYPE_ADHOC && link_conf->bssid)
+		memcpy(cmd.ibss_bssid_addr, link_conf->bssid, ETH_ALEN);
+
+	return iwl_mvm_link_cmd_send(mvm, &cmd, FW_CTXT_ACTION_ADD);
+}
+
+int iwl_mvm_link_changed(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			 struct ieee80211_bss_conf *link_conf,
+			 u32 changes, bool active)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	unsigned int link_id = link_conf->link_id;
+	struct iwl_mvm_vif_link_info *link_info = mvmvif->link[link_id];
+	struct iwl_mvm_phy_ctxt *phyctxt;
+	struct iwl_link_config_cmd cmd = {};
+	u32 ht_flag, flags = 0, flags_mask = 0;
+	int ret;
+
+	if (WARN_ON_ONCE(!link_info ||
+			 link_info->fw_link_id == IWL_MVM_FW_LINK_ID_INVALID))
+		return -EINVAL;
+
+	if (changes & LINK_CONTEXT_MODIFY_ACTIVE) {
+		/* When activating a link, phy context should be valid;
+		 * when deactivating a link, it also should be valid since
+		 * the link was active before. So, do nothing in this case.
+		 * Since a link is added first with FW_CTXT_INVALID, then we
+		 * can get here in case it's removed before it was activated.
+		 */
+		if (!link_info->phy_ctxt)
+			return 0;
+
+		/* check there aren't too many active links */
+		if (!link_info->active && active) {
+			int i, count = 0;
+
+			/* link with phy_ctxt is active in FW */
+			for_each_mvm_vif_valid_link(mvmvif, i)
+				if (mvmvif->link[i]->phy_ctxt)
+					count++;
+
+			/* FIXME: IWL_MVM_FW_MAX_ACTIVE_LINKS_NUM should be
+			 * defined per HW
+			 */
+			if (count >= IWL_MVM_FW_MAX_ACTIVE_LINKS_NUM)
+				return -EINVAL;
+		}
+
+		/* Catch early if driver tries to activate or deactivate a link
+		 * twice.
+		 */
+		WARN_ON_ONCE(active == link_info->active);
+
+		/* When deactivating a link session protection should
+		 * be stopped
+		 */
+		if (!active && vif->type == NL80211_IFTYPE_STATION)
+			iwl_mvm_stop_session_protection(mvm, vif);
+	}
+
+	cmd.link_id = cpu_to_le32(link_info->fw_link_id);
+
+	/* The phy_id, link address and listen_lmac can be modified only until
+	 * the link becomes active, otherwise they will be ignored.
+	 */
+	phyctxt = link_info->phy_ctxt;
+	if (phyctxt)
+		cmd.phy_id = cpu_to_le32(phyctxt->id);
+	else
+		cmd.phy_id = cpu_to_le32(FW_CTXT_INVALID);
+	cmd.mac_id = cpu_to_le32(mvmvif->id);
+
+	memcpy(cmd.local_link_addr, link_conf->addr, ETH_ALEN);
+
+	cmd.active = cpu_to_le32(active);
+
+	if (vif->type == NL80211_IFTYPE_ADHOC && link_conf->bssid)
+		memcpy(cmd.ibss_bssid_addr, link_conf->bssid, ETH_ALEN);
+
+	/* TODO: set a value to cmd.listen_lmac when system requiremens
+	 * will define it
+	 */
+
+	iwl_mvm_set_fw_basic_rates(mvm, vif, link_conf,
+				   &cmd.cck_rates, &cmd.ofdm_rates);
+
+	cmd.cck_short_preamble = cpu_to_le32(link_conf->use_short_preamble);
+	cmd.short_slot = cpu_to_le32(link_conf->use_short_slot);
+
+	/* The fw does not distinguish between ht and fat */
+	ht_flag = LINK_PROT_FLG_HT_PROT | LINK_PROT_FLG_FAT_PROT;
+	iwl_mvm_set_fw_protection_flags(mvm, vif, link_conf,
+					&cmd.protection_flags,
+					ht_flag, LINK_PROT_FLG_TGG_PROTECT);
+
+	iwl_mvm_set_fw_qos_params(mvm, vif, link_conf, &cmd.ac[0],
+				  &cmd.qos_flags);
+
+
+	cmd.bi = cpu_to_le32(link_conf->beacon_int);
+	cmd.dtim_interval = cpu_to_le32(link_conf->beacon_int *
+					link_conf->dtim_period);
+
+	if (!link_conf->he_support || iwlwifi_mod_params.disable_11ax ||
+	    (vif->type == NL80211_IFTYPE_STATION && !vif->cfg.assoc)) {
+		changes &= ~LINK_CONTEXT_MODIFY_HE_PARAMS;
+		goto send_cmd;
+	}
+
+	cmd.htc_trig_based_pkt_ext = link_conf->htc_trig_based_pkt_ext;
+
+	if (link_conf->uora_exists) {
+		cmd.rand_alloc_ecwmin =
+			link_conf->uora_ocw_range & 0x7;
+		cmd.rand_alloc_ecwmax =
+			(link_conf->uora_ocw_range >> 3) & 0x7;
+	}
+
+	/* TODO  how to set ndp_fdbk_buff_th_exp? */
+
+	if (iwl_mvm_set_fw_mu_edca_params(mvm, mvmvif,
+					  &cmd.trig_based_txf[0])) {
+		flags |= LINK_FLG_MU_EDCA_CW;
+		flags_mask |= LINK_FLG_MU_EDCA_CW;
+	}
+
+	if (link_conf->eht_puncturing && !iwlwifi_mod_params.disable_11be)
+		cmd.puncture_mask = cpu_to_le16(link_conf->eht_puncturing);
+	else
+		/* This flag can be set only if the MAC has eht support */
+		changes &= ~LINK_CONTEXT_MODIFY_EHT_PARAMS;
+
+	cmd.bss_color = link_conf->he_bss_color.color;
+
+	if (!link_conf->he_bss_color.enabled) {
+		flags |= LINK_FLG_BSS_COLOR_DIS;
+		flags_mask |= LINK_FLG_BSS_COLOR_DIS;
+	}
+
+	cmd.frame_time_rts_th = cpu_to_le16(link_conf->frame_time_rts_th);
+
+	/* Block 26-tone RU OFDMA transmissions */
+	if (link_info->he_ru_2mhz_block) {
+		flags |= LINK_FLG_RU_2MHZ_BLOCK;
+		flags_mask |= LINK_FLG_RU_2MHZ_BLOCK;
+	}
+
+	if (link_conf->nontransmitted) {
+		ether_addr_copy(cmd.ref_bssid_addr,
+				link_conf->transmitter_bssid);
+		cmd.bssid_index = link_conf->bssid_index;
+	}
+
+send_cmd:
+	cmd.modify_mask = cpu_to_le32(changes);
+	cmd.flags = cpu_to_le32(flags);
+	cmd.flags_mask = cpu_to_le32(flags_mask);
+
+	ret = iwl_mvm_link_cmd_send(mvm, &cmd, FW_CTXT_ACTION_MODIFY);
+	if (!ret && (changes & LINK_CONTEXT_MODIFY_ACTIVE))
+		link_info->active = active;
+
+	return ret;
+}
+
+int iwl_mvm_remove_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	unsigned int link_id = link_conf->link_id;
+	struct iwl_mvm_vif_link_info *link_info = mvmvif->link[link_id];
+	struct iwl_link_config_cmd cmd = {};
+	int ret;
+
+	if (WARN_ON(!link_info ||
+		    link_info->fw_link_id == IWL_MVM_FW_LINK_ID_INVALID))
+		return -EINVAL;
+
+	cmd.link_id = cpu_to_le32(link_info->fw_link_id);
+	iwl_mvm_release_fw_link_id(mvm, link_info->fw_link_id);
+	link_info->fw_link_id = IWL_MVM_FW_LINK_ID_INVALID;
+
+	ret = iwl_mvm_link_cmd_send(mvm, &cmd, FW_CTXT_ACTION_REMOVE);
+
+	if (!ret)
+		if (iwl_mvm_sf_update(mvm, vif, true))
+			IWL_ERR(mvm, "Failed to update SF state\n");
+
+	return ret;
+}
+
+/* link should be deactivated before removal, so in most cases we need to
+ * perform these two operations together
+ */
+int iwl_mvm_disable_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			 struct ieee80211_bss_conf *link_conf)
+{
+	int ret;
+
+	ret = iwl_mvm_link_changed(mvm, vif, link_conf,
+				   LINK_CONTEXT_MODIFY_ACTIVE, false);
+	if (ret)
+		return ret;
+
+	ret = iwl_mvm_remove_link(mvm, vif, link_conf);
+	if (ret)
+		return ret;
+
+	return ret;
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
index aa791dbc3066..cc90f2884cff 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
@@ -225,16 +225,20 @@ int iwl_mvm_mac_ctxt_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	 * that we should share it with another interface.
 	 */
 
-	/* Currently, MAC ID 0 should be used only for the managed/IBSS vif */
-	switch (vif->type) {
-	case NL80211_IFTYPE_ADHOC:
-		break;
-	case NL80211_IFTYPE_STATION:
-		if (!vif->p2p)
+	/* MAC ID 0 should be used only for the managed/IBSS vif with non-MLO
+	 * FW API
+	 */
+	if (!mvm->mld_api_is_used) {
+		switch (vif->type) {
+		case NL80211_IFTYPE_ADHOC:
 			break;
-		fallthrough;
-	default:
-		__clear_bit(0, data.available_mac_ids);
+		case NL80211_IFTYPE_STATION:
+			if (!vif->p2p)
+				break;
+			fallthrough;
+		default:
+			__clear_bit(0, data.available_mac_ids);
+		}
 	}
 
 	ieee80211_iterate_active_interfaces_atomic(
@@ -293,15 +297,15 @@ int iwl_mvm_mac_ctxt_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 		 * For TVQM this will be overwritten later with the FW assigned
 		 * queue value (when queue is enabled).
 		 */
-		mvmvif->cab_queue = IWL_MVM_DQA_GCAST_QUEUE;
+		mvmvif->deflink.cab_queue = IWL_MVM_DQA_GCAST_QUEUE;
 	}
 
-	mvmvif->bcast_sta.sta_id = IWL_MVM_INVALID_STA;
-	mvmvif->mcast_sta.sta_id = IWL_MVM_INVALID_STA;
-	mvmvif->ap_sta_id = IWL_MVM_INVALID_STA;
+	mvmvif->deflink.bcast_sta.sta_id = IWL_MVM_INVALID_STA;
+	mvmvif->deflink.mcast_sta.sta_id = IWL_MVM_INVALID_STA;
+	mvmvif->deflink.ap_sta_id = IWL_MVM_INVALID_STA;
 
 	for (i = 0; i < NUM_IWL_MVM_SMPS_REQ; i++)
-		mvmvif->smps_requests[i] = IEEE80211_SMPS_AUTOMATIC;
+		mvmvif->deflink.smps_requests[i] = IEEE80211_SMPS_AUTOMATIC;
 
 	return 0;
 
@@ -396,15 +400,46 @@ static void iwl_mvm_ack_rates(struct iwl_mvm *mvm,
 	*ofdm_rates = ofdm;
 }
 
-static void iwl_mvm_mac_ctxt_set_ht_flags(struct iwl_mvm *mvm,
-					 struct ieee80211_vif *vif,
-					 struct iwl_mac_ctx_cmd *cmd)
+void iwl_mvm_set_fw_basic_rates(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+				struct ieee80211_bss_conf *link_conf,
+				__le32 *cck_rates, __le32 *ofdm_rates)
+{
+	struct ieee80211_chanctx_conf *chanctx;
+	u8 cck_ack_rates = 0, ofdm_ack_rates = 0;
+
+	rcu_read_lock();
+	chanctx = rcu_dereference(link_conf->chanctx_conf);
+	iwl_mvm_ack_rates(mvm, vif, chanctx ? chanctx->def.chan->band
+					    : NL80211_BAND_2GHZ,
+			  &cck_ack_rates, &ofdm_ack_rates);
+
+	rcu_read_unlock();
+
+	*cck_rates = cpu_to_le32((u32)cck_ack_rates);
+	*ofdm_rates = cpu_to_le32((u32)ofdm_ack_rates);
+}
+
+void iwl_mvm_set_fw_protection_flags(struct iwl_mvm *mvm,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_bss_conf *link_conf,
+				     __le32 *protection_flags, u32 ht_flag,
+				     u32 tgg_flag)
 {
 	/* for both sta and ap, ht_operation_mode hold the protection_mode */
-	u8 protection_mode = vif->bss_conf.ht_operation_mode &
+	u8 protection_mode = link_conf->ht_operation_mode &
 				 IEEE80211_HT_OP_MODE_PROTECTION;
-	/* The fw does not distinguish between ht and fat */
-	u32 ht_flag = MAC_PROT_FLG_HT_PROT | MAC_PROT_FLG_FAT_PROT;
+	bool ht_enabled = !!(link_conf->ht_operation_mode &
+			     IEEE80211_HT_OP_MODE_PROTECTION);
+
+	if (link_conf->use_cts_prot)
+		*protection_flags |= cpu_to_le32(tgg_flag);
+
+	IWL_DEBUG_RATE(mvm, "use_cts_prot %d, ht_operation_mode %d\n",
+		       link_conf->use_cts_prot,
+		       link_conf->ht_operation_mode);
+
+	if (!ht_enabled)
+		return;
 
 	IWL_DEBUG_RATE(mvm, "protection mode set to %d\n", protection_mode);
 	/*
@@ -416,12 +451,12 @@ static void iwl_mvm_mac_ctxt_set_ht_flags(struct iwl_mvm *mvm,
 		break;
 	case IEEE80211_HT_OP_MODE_PROTECTION_NONMEMBER:
 	case IEEE80211_HT_OP_MODE_PROTECTION_NONHT_MIXED:
-		cmd->protection_flags |= cpu_to_le32(ht_flag);
+		*protection_flags |= cpu_to_le32(ht_flag);
 		break;
 	case IEEE80211_HT_OP_MODE_PROTECTION_20MHZ:
 		/* Protect when channel wider than 20MHz */
-		if (vif->bss_conf.chandef.width > NL80211_CHAN_WIDTH_20)
-			cmd->protection_flags |= cpu_to_le32(ht_flag);
+		if (link_conf->chandef.width > NL80211_CHAN_WIDTH_20)
+			*protection_flags |= cpu_to_le32(ht_flag);
 		break;
 	default:
 		IWL_ERR(mvm, "Illegal protection mode %d\n",
@@ -430,46 +465,77 @@ static void iwl_mvm_mac_ctxt_set_ht_flags(struct iwl_mvm *mvm,
 	}
 }
 
-static void iwl_mvm_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
-					struct ieee80211_vif *vif,
-					struct iwl_mac_ctx_cmd *cmd,
-					const u8 *bssid_override,
-					u32 action)
+void iwl_mvm_set_fw_qos_params(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			       struct ieee80211_bss_conf *link_conf,
+			       struct iwl_ac_qos *ac, __le32 *qos_flags)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct ieee80211_chanctx_conf *chanctx;
-	bool ht_enabled = !!(vif->bss_conf.ht_operation_mode &
-			     IEEE80211_HT_OP_MODE_PROTECTION);
-	u8 cck_ack_rates, ofdm_ack_rates;
-	const u8 *bssid = bssid_override ?: vif->bss_conf.bssid;
 	int i;
 
-	cmd->id_and_color = cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->id,
-							    mvmvif->color));
-	cmd->action = cpu_to_le32(action);
+	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
+		u8 txf = iwl_mvm_mac_ac_to_tx_fifo(mvm, i);
+		u8 ucode_ac = iwl_mvm_mac80211_ac_to_ucode_ac(i);
+
+		ac[ucode_ac].cw_min =
+			cpu_to_le16(mvmvif->deflink.queue_params[i].cw_min);
+		ac[ucode_ac].cw_max =
+			cpu_to_le16(mvmvif->deflink.queue_params[i].cw_max);
+		ac[ucode_ac].edca_txop =
+			cpu_to_le16(mvmvif->deflink.queue_params[i].txop * 32);
+		ac[ucode_ac].aifsn = mvmvif->deflink.queue_params[i].aifs;
+		ac[ucode_ac].fifos_mask = BIT(txf);
+	}
+
+	if (link_conf->qos)
+		*qos_flags |= cpu_to_le32(MAC_QOS_FLG_UPDATE_EDCA);
+
+	if (link_conf->chandef.width != NL80211_CHAN_WIDTH_20_NOHT)
+		*qos_flags |= cpu_to_le32(MAC_QOS_FLG_TGN);
+}
+
+int iwl_mvm_get_mac_type(struct ieee80211_vif *vif)
+{
+	u32 mac_type = FW_MAC_TYPE_BSS_STA;
 
 	switch (vif->type) {
 	case NL80211_IFTYPE_STATION:
 		if (vif->p2p)
-			cmd->mac_type = cpu_to_le32(FW_MAC_TYPE_P2P_STA);
+			mac_type = FW_MAC_TYPE_P2P_STA;
 		else
-			cmd->mac_type = cpu_to_le32(FW_MAC_TYPE_BSS_STA);
+			mac_type = FW_MAC_TYPE_BSS_STA;
 		break;
 	case NL80211_IFTYPE_AP:
-		cmd->mac_type = cpu_to_le32(FW_MAC_TYPE_GO);
+		mac_type = FW_MAC_TYPE_GO;
 		break;
 	case NL80211_IFTYPE_MONITOR:
-		cmd->mac_type = cpu_to_le32(FW_MAC_TYPE_LISTENER);
+		mac_type = FW_MAC_TYPE_LISTENER;
 		break;
 	case NL80211_IFTYPE_P2P_DEVICE:
-		cmd->mac_type = cpu_to_le32(FW_MAC_TYPE_P2P_DEVICE);
+		mac_type = FW_MAC_TYPE_P2P_DEVICE;
 		break;
 	case NL80211_IFTYPE_ADHOC:
-		cmd->mac_type = cpu_to_le32(FW_MAC_TYPE_IBSS);
+		mac_type = FW_MAC_TYPE_IBSS;
 		break;
 	default:
 		WARN_ON_ONCE(1);
 	}
+	return mac_type;
+}
+
+static void iwl_mvm_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
+					struct ieee80211_vif *vif,
+					struct iwl_mac_ctx_cmd *cmd,
+					const u8 *bssid_override,
+					u32 action)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	const u8 *bssid = bssid_override ?: vif->bss_conf.bssid;
+	u32 ht_flag;
+
+	cmd->id_and_color = cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->id,
+							    mvmvif->color));
+	cmd->action = cpu_to_le32(action);
+	cmd->mac_type = cpu_to_le32(iwl_mvm_get_mac_type(vif));
 
 	cmd->tsf_id = cpu_to_le32(mvmvif->tsf_id);
 
@@ -480,15 +546,8 @@ static void iwl_mvm_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
 	else
 		eth_broadcast_addr(cmd->bssid_addr);
 
-	rcu_read_lock();
-	chanctx = rcu_dereference(vif->bss_conf.chanctx_conf);
-	iwl_mvm_ack_rates(mvm, vif, chanctx ? chanctx->def.chan->band
-					    : NL80211_BAND_2GHZ,
-			  &cck_ack_rates, &ofdm_ack_rates);
-	rcu_read_unlock();
-
-	cmd->cck_rates = cpu_to_le32((u32)cck_ack_rates);
-	cmd->ofdm_rates = cpu_to_le32((u32)ofdm_ack_rates);
+	iwl_mvm_set_fw_basic_rates(mvm, vif, &vif->bss_conf, &cmd->cck_rates,
+				   &cmd->ofdm_rates);
 
 	cmd->cck_short_preamble =
 		cpu_to_le32(vif->bss_conf.use_short_preamble ?
@@ -499,33 +558,14 @@ static void iwl_mvm_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
 
 	cmd->filter_flags = 0;
 
-	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
-		u8 txf = iwl_mvm_mac_ac_to_tx_fifo(mvm, i);
-		u8 ucode_ac = iwl_mvm_mac80211_ac_to_ucode_ac(i);
+	iwl_mvm_set_fw_qos_params(mvm, vif, &vif->bss_conf, &cmd->ac[0],
+				  &cmd->qos_flags);
 
-		cmd->ac[ucode_ac].cw_min =
-			cpu_to_le16(mvmvif->queue_params[i].cw_min);
-		cmd->ac[ucode_ac].cw_max =
-			cpu_to_le16(mvmvif->queue_params[i].cw_max);
-		cmd->ac[ucode_ac].edca_txop =
-			cpu_to_le16(mvmvif->queue_params[i].txop * 32);
-		cmd->ac[ucode_ac].aifsn = mvmvif->queue_params[i].aifs;
-		cmd->ac[ucode_ac].fifos_mask = BIT(txf);
-	}
-
-	if (vif->bss_conf.qos)
-		cmd->qos_flags |= cpu_to_le32(MAC_QOS_FLG_UPDATE_EDCA);
-
-	if (vif->bss_conf.use_cts_prot)
-		cmd->protection_flags |= cpu_to_le32(MAC_PROT_FLG_TGG_PROTECT);
-
-	IWL_DEBUG_RATE(mvm, "use_cts_prot %d, ht_operation_mode %d\n",
-		       vif->bss_conf.use_cts_prot,
-		       vif->bss_conf.ht_operation_mode);
-	if (vif->bss_conf.chandef.width != NL80211_CHAN_WIDTH_20_NOHT)
-		cmd->qos_flags |= cpu_to_le32(MAC_QOS_FLG_TGN);
-	if (ht_enabled)
-		iwl_mvm_mac_ctxt_set_ht_flags(mvm, vif, cmd);
+	/* The fw does not distinguish between ht and fat */
+	ht_flag = MAC_PROT_FLG_HT_PROT | MAC_PROT_FLG_FAT_PROT;
+	iwl_mvm_set_fw_protection_flags(mvm, vif, &vif->bss_conf,
+					&cmd->protection_flags,
+					ht_flag, MAC_PROT_FLG_TGG_PROTECT);
 }
 
 static int iwl_mvm_mac_ctxt_send_cmd(struct iwl_mvm *mvm,
@@ -534,11 +574,76 @@ static int iwl_mvm_mac_ctxt_send_cmd(struct iwl_mvm *mvm,
 	int ret = iwl_mvm_send_cmd_pdu(mvm, MAC_CONTEXT_CMD, 0,
 				       sizeof(*cmd), cmd);
 	if (ret)
-		IWL_ERR(mvm, "Failed to send MAC context (action:%d): %d\n",
+		IWL_ERR(mvm, "Failed to send MAC_CONTEXT_CMD (action:%d): %d\n",
 			le32_to_cpu(cmd->action), ret);
 	return ret;
 }
 
+void iwl_mvm_set_fw_dtim_tbtt(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link_conf,
+			      __le64 *dtim_tsf, __le32 *dtim_time,
+			      __le32 *assoc_beacon_arrive_time)
+{
+	u32 dtim_offs;
+
+	/*
+	 * The DTIM count counts down, so when it is N that means N
+	 * more beacon intervals happen until the DTIM TBTT. Therefore
+	 * add this to the current time. If that ends up being in the
+	 * future, the firmware will handle it.
+	 *
+	 * Also note that the system_timestamp (which we get here as
+	 * "sync_device_ts") and TSF timestamp aren't at exactly the
+	 * same offset in the frame -- the TSF is at the first symbol
+	 * of the TSF, the system timestamp is at signal acquisition
+	 * time. This means there's an offset between them of at most
+	 * a few hundred microseconds (24 * 8 bits + PLCP time gives
+	 * 384us in the longest case), this is currently not relevant
+	 * as the firmware wakes up around 2ms before the TBTT.
+	 */
+	dtim_offs = link_conf->sync_dtim_count *
+			link_conf->beacon_int;
+	/* convert TU to usecs */
+	dtim_offs *= 1024;
+
+	*dtim_tsf =
+		cpu_to_le64(link_conf->sync_tsf + dtim_offs);
+	*dtim_time =
+		cpu_to_le32(link_conf->sync_device_ts + dtim_offs);
+	*assoc_beacon_arrive_time =
+		cpu_to_le32(link_conf->sync_device_ts);
+
+	IWL_DEBUG_INFO(mvm, "DTIM TBTT is 0x%llx/0x%x, offset %d\n",
+		       le64_to_cpu(*dtim_tsf),
+		       le32_to_cpu(*dtim_time),
+		       dtim_offs);
+}
+
+__le32 iwl_mvm_mac_ctxt_cmd_p2p_sta_get_oppps_ctwin(struct iwl_mvm *mvm,
+						    struct ieee80211_vif *vif)
+{
+	struct ieee80211_p2p_noa_attr *noa =
+		&vif->bss_conf.p2p_noa_attr;
+
+	return cpu_to_le32(noa->oppps_ctwindow &
+			IEEE80211_P2P_OPPPS_CTWINDOW_MASK);
+}
+
+__le32 iwl_mvm_mac_ctxt_cmd_sta_get_twt_policy(struct iwl_mvm *mvm,
+					       struct ieee80211_vif *vif)
+{
+	__le32 twt_policy = cpu_to_le32(0);
+
+	if (vif->bss_conf.twt_requester && IWL_MVM_USE_TWT)
+		twt_policy |= cpu_to_le32(TWT_SUPPORTED);
+	if (vif->bss_conf.twt_protected)
+		twt_policy |= cpu_to_le32(PROTECTED_TWT_SUPPORTED);
+	if (vif->bss_conf.twt_broadcast)
+		twt_policy |= cpu_to_le32(BROADCAST_TWT_SUPPORTED);
+
+	return twt_policy;
+}
+
 static int iwl_mvm_mac_ctxt_cmd_sta(struct iwl_mvm *mvm,
 				    struct ieee80211_vif *vif,
 				    u32 action, bool force_assoc_off,
@@ -559,11 +664,9 @@ static int iwl_mvm_mac_ctxt_cmd_sta(struct iwl_mvm *mvm,
 	cmd.filter_flags |= cpu_to_le32(MAC_FILTER_ACCEPT_GRP);
 
 	if (vif->p2p) {
-		struct ieee80211_p2p_noa_attr *noa =
-			&vif->bss_conf.p2p_noa_attr;
+		cmd.p2p_sta.ctwin =
+			iwl_mvm_mac_ctxt_cmd_p2p_sta_get_oppps_ctwin(mvm, vif);
 
-		cmd.p2p_sta.ctwin = cpu_to_le32(noa->oppps_ctwindow &
-					IEEE80211_P2P_OPPPS_CTWINDOW_MASK);
 		ctxt_sta = &cmd.p2p_sta.sta;
 	} else {
 		ctxt_sta = &cmd.sta;
@@ -573,39 +676,11 @@ static int iwl_mvm_mac_ctxt_cmd_sta(struct iwl_mvm *mvm,
 	if (vif->cfg.assoc && vif->bss_conf.dtim_period &&
 	    !force_assoc_off) {
 		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-		u32 dtim_offs;
 
-		/*
-		 * The DTIM count counts down, so when it is N that means N
-		 * more beacon intervals happen until the DTIM TBTT. Therefore
-		 * add this to the current time. If that ends up being in the
-		 * future, the firmware will handle it.
-		 *
-		 * Also note that the system_timestamp (which we get here as
-		 * "sync_device_ts") and TSF timestamp aren't at exactly the
-		 * same offset in the frame -- the TSF is at the first symbol
-		 * of the TSF, the system timestamp is at signal acquisition
-		 * time. This means there's an offset between them of at most
-		 * a few hundred microseconds (24 * 8 bits + PLCP time gives
-		 * 384us in the longest case), this is currently not relevant
-		 * as the firmware wakes up around 2ms before the TBTT.
-		 */
-		dtim_offs = vif->bss_conf.sync_dtim_count *
-				vif->bss_conf.beacon_int;
-		/* convert TU to usecs */
-		dtim_offs *= 1024;
-
-		ctxt_sta->dtim_tsf =
-			cpu_to_le64(vif->bss_conf.sync_tsf + dtim_offs);
-		ctxt_sta->dtim_time =
-			cpu_to_le32(vif->bss_conf.sync_device_ts + dtim_offs);
-		ctxt_sta->assoc_beacon_arrive_time =
-			cpu_to_le32(vif->bss_conf.sync_device_ts);
-
-		IWL_DEBUG_INFO(mvm, "DTIM TBTT is 0x%llx/0x%x, offset %d\n",
-			       le64_to_cpu(ctxt_sta->dtim_tsf),
-			       le32_to_cpu(ctxt_sta->dtim_time),
-			       dtim_offs);
+		iwl_mvm_set_fw_dtim_tbtt(mvm, vif, &vif->bss_conf,
+					 &ctxt_sta->dtim_tsf,
+					 &ctxt_sta->dtim_time,
+					 &ctxt_sta->assoc_beacon_arrive_time);
 
 		ctxt_sta->is_assoc = cpu_to_le32(1);
 
@@ -635,14 +710,8 @@ static int iwl_mvm_mac_ctxt_cmd_sta(struct iwl_mvm *mvm,
 
 	if (vif->bss_conf.he_support && !iwlwifi_mod_params.disable_11ax) {
 		cmd.filter_flags |= cpu_to_le32(MAC_FILTER_IN_11AX);
-		if (vif->bss_conf.twt_requester && IWL_MVM_USE_TWT)
-			ctxt_sta->data_policy |= cpu_to_le32(TWT_SUPPORTED);
-		if (vif->bss_conf.twt_protected)
-			ctxt_sta->data_policy |=
-				cpu_to_le32(PROTECTED_TWT_SUPPORTED);
-		if (vif->bss_conf.twt_broadcast)
-			ctxt_sta->data_policy |=
-				cpu_to_le32(BROADCAST_TWT_SUPPORTED);
+		ctxt_sta->data_policy |=
+			iwl_mvm_mac_ctxt_cmd_sta_get_twt_policy(mvm, vif);
 	}
 
 
@@ -654,7 +723,7 @@ static int iwl_mvm_mac_ctxt_cmd_listener(struct iwl_mvm *mvm,
 					 u32 action)
 {
 	struct iwl_mac_ctx_cmd cmd = {};
-	u32 tfd_queue_msk = BIT(mvm->snif_queue);
+	u32 tfd_queue_msk = 0;
 	int ret;
 
 	WARN_ON(vif->type != NL80211_IFTYPE_MONITOR);
@@ -669,6 +738,14 @@ static int iwl_mvm_mac_ctxt_cmd_listener(struct iwl_mvm *mvm,
 				       MAC_FILTER_ACCEPT_GRP);
 	ieee80211_hw_set(mvm->hw, RX_INCLUDES_FCS);
 
+	/*
+	 * the queue mask is only relevant for old TX API, and
+	 * mvm->snif_queue isn't set here (it's still set to
+	 * IWL_MVM_INVALID_QUEUE so the BIT() of it is UB)
+	 */
+	if (!iwl_mvm_has_new_tx_api(mvm))
+		tfd_queue_msk = BIT(mvm->snif_queue);
+
 	/* Allocate sniffer station */
 	ret = iwl_mvm_allocate_int_sta(mvm, &mvm->snif_sta, tfd_queue_msk,
 				       vif->type, IWL_STA_GENERAL_PURPOSE);
@@ -716,20 +793,11 @@ static void iwl_mvm_go_iterator(void *_data, u8 *mac, struct ieee80211_vif *vif)
 		data->go_active = true;
 }
 
-static int iwl_mvm_mac_ctxt_cmd_p2p_device(struct iwl_mvm *mvm,
-					   struct ieee80211_vif *vif,
-					   u32 action)
+__le32 iwl_mac_ctxt_p2p_dev_has_extended_disc(struct iwl_mvm *mvm,
+					      struct ieee80211_vif *vif)
 {
-	struct iwl_mac_ctx_cmd cmd = {};
 	struct iwl_mvm_go_iterator_data data = {};
 
-	WARN_ON(vif->type != NL80211_IFTYPE_P2P_DEVICE);
-
-	iwl_mvm_mac_ctxt_cmd_common(mvm, vif, &cmd, NULL, action);
-
-	/* Override the filter flags to accept only probe requests */
-	cmd.filter_flags = cpu_to_le32(MAC_FILTER_IN_PROBE_REQUEST);
-
 	/*
 	 * This flag should be set to true when the P2P Device is
 	 * discoverable and there is at least another active P2P GO. Settings
@@ -742,7 +810,25 @@ static int iwl_mvm_mac_ctxt_cmd_p2p_device(struct iwl_mvm *mvm,
 		mvm->hw, IEEE80211_IFACE_ITER_RESUME_ALL,
 		iwl_mvm_go_iterator, &data);
 
-	cmd.p2p_dev.is_disc_extended = cpu_to_le32(data.go_active ? 1 : 0);
+	return cpu_to_le32(data.go_active ? 1 : 0);
+}
+
+static int iwl_mvm_mac_ctxt_cmd_p2p_device(struct iwl_mvm *mvm,
+					   struct ieee80211_vif *vif,
+					   u32 action)
+{
+	struct iwl_mac_ctx_cmd cmd = {};
+
+	WARN_ON(vif->type != NL80211_IFTYPE_P2P_DEVICE);
+
+	iwl_mvm_mac_ctxt_cmd_common(mvm, vif, &cmd, NULL, action);
+
+	cmd.p2p_dev.is_disc_extended =
+		iwl_mac_ctxt_p2p_dev_has_extended_disc(mvm, vif);
+
+	/* Override the filter flags to accept only probe requests */
+	cmd.filter_flags = cpu_to_le32(MAC_FILTER_IN_PROBE_REQUEST);
+
 	return iwl_mvm_mac_ctxt_send_cmd(mvm, &cmd);
 }
 
@@ -788,17 +874,44 @@ static u32 iwl_mvm_find_ie_offset(u8 *beacon, u8 eid, u32 frame_size)
 	return ie - beacon;
 }
 
-static u8 iwl_mvm_mac_ctxt_get_lowest_rate(struct iwl_mvm *mvm,
-					   struct ieee80211_tx_info *info,
-					   struct ieee80211_vif *vif)
+u8 iwl_mvm_mac_ctxt_get_lowest_rate(struct iwl_mvm *mvm,
+				    struct ieee80211_tx_info *info,
+				    struct ieee80211_vif *vif)
 {
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct ieee80211_supported_band *sband;
 	unsigned long basic = vif->bss_conf.basic_rates;
 	u16 lowest_cck = IWL_RATE_COUNT, lowest_ofdm = IWL_RATE_COUNT;
+	u32 link_id = u32_get_bits(info->control.flags,
+				   IEEE80211_TX_CTRL_MLO_LINK);
+	u8 band = info->band;
 	u8 rate;
 	u32 i;
 
-	sband = mvm->hw->wiphy->bands[info->band];
+	if (link_id == IEEE80211_LINK_UNSPECIFIED && vif->valid_links) {
+		for (i = 0; i < ARRAY_SIZE(mvmvif->link); i++) {
+			if (!mvmvif->link[i])
+				continue;
+			/* shouldn't do this when >1 link is active */
+			WARN_ON_ONCE(link_id != IEEE80211_LINK_UNSPECIFIED);
+			link_id = i;
+		}
+	}
+
+	if (link_id < IEEE80211_LINK_UNSPECIFIED) {
+		struct ieee80211_bss_conf *link_conf;
+
+		rcu_read_lock();
+		link_conf = rcu_dereference(vif->link_conf[link_id]);
+		if (link_conf) {
+			basic = link_conf->basic_rates;
+			if (link_conf->chandef.chan)
+				band = link_conf->chandef.chan->band;
+		}
+		rcu_read_unlock();
+	}
+
+	sband = mvm->hw->wiphy->bands[band];
 	for_each_set_bit(i, &basic, BITS_PER_LONG) {
 		u16 hw = sband->bitrates[i].hw_value;
 
@@ -810,7 +923,9 @@ static u8 iwl_mvm_mac_ctxt_get_lowest_rate(struct iwl_mvm *mvm,
 		}
 	}
 
-	if (info->band == NL80211_BAND_2GHZ && !vif->p2p) {
+	if (band == NL80211_BAND_2GHZ && !vif->p2p &&
+	    vif->type != NL80211_IFTYPE_P2P_DEVICE &&
+	    !(info->flags & IEEE80211_TX_CTL_NO_CCK_RATE)) {
 		if (lowest_cck != IWL_RATE_COUNT)
 			rate = lowest_cck;
 		else if (lowest_ofdm != IWL_RATE_COUNT)
@@ -870,7 +985,7 @@ static void iwl_mvm_mac_ctxt_set_tx(struct iwl_mvm *mvm,
 
 	/* Set up TX command fields */
 	tx->len = cpu_to_le16((u16)beacon->len);
-	tx->sta_id = mvmvif->bcast_sta.sta_id;
+	tx->sta_id = mvmvif->deflink.bcast_sta.sta_id;
 	tx->life_time = cpu_to_le32(TX_CMD_LIFE_TIME_INFINITE);
 	tx_flags = TX_CMD_FLG_SEQ_CTL | TX_CMD_FLG_TSF;
 	tx_flags |=
@@ -965,7 +1080,8 @@ static int iwl_mvm_mac_ctxt_send_beacon_v7(struct iwl_mvm *mvm,
 
 static int iwl_mvm_mac_ctxt_send_beacon_v9(struct iwl_mvm *mvm,
 					   struct ieee80211_vif *vif,
-					   struct sk_buff *beacon)
+					   struct sk_buff *beacon,
+					   struct ieee80211_bss_conf *link_conf)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(beacon);
@@ -978,7 +1094,7 @@ static int iwl_mvm_mac_ctxt_send_beacon_v9(struct iwl_mvm *mvm,
 
 	/* Enable FILS on PSC channels only */
 	rcu_read_lock();
-	ctx = rcu_dereference(vif->bss_conf.chanctx_conf);
+	ctx = rcu_dereference(link_conf->chanctx_conf);
 	channel = ieee80211_frequency_to_channel(ctx->def.chan->center_freq);
 	WARN_ON(channel == 0);
 	if (cfg80211_channel_is_psc(ctx->def.chan) &&
@@ -995,7 +1111,11 @@ static int iwl_mvm_mac_ctxt_send_beacon_v9(struct iwl_mvm *mvm,
 
 	beacon_cmd.flags = cpu_to_le16(flags);
 	beacon_cmd.byte_cnt = cpu_to_le16((u16)beacon->len);
-	beacon_cmd.template_id = cpu_to_le32((u32)mvmvif->id);
+	if (iwl_fw_lookup_cmd_ver(mvm->fw, BEACON_TEMPLATE_CMD, 0) > 12)
+		beacon_cmd.link_id =
+			cpu_to_le32(mvmvif->link[link_conf->link_id]->fw_link_id);
+	else
+		beacon_cmd.link_id = cpu_to_le32((u32)mvmvif->id);
 
 	if (vif->type == NL80211_IFTYPE_AP)
 		iwl_mvm_mac_ctxt_set_tim(mvm, &beacon_cmd.tim_idx,
@@ -1015,9 +1135,10 @@ static int iwl_mvm_mac_ctxt_send_beacon_v9(struct iwl_mvm *mvm,
 						sizeof(beacon_cmd));
 }
 
-int iwl_mvm_mac_ctxt_send_beacon(struct iwl_mvm *mvm,
-				 struct ieee80211_vif *vif,
-				 struct sk_buff *beacon)
+static int iwl_mvm_mac_ctxt_send_beacon(struct iwl_mvm *mvm,
+					struct ieee80211_vif *vif,
+					struct sk_buff *beacon,
+					struct ieee80211_bss_conf *link_conf)
 {
 	if (WARN_ON(!beacon))
 		return -EINVAL;
@@ -1031,14 +1152,16 @@ int iwl_mvm_mac_ctxt_send_beacon(struct iwl_mvm *mvm,
 
 	if (fw_has_api(&mvm->fw->ucode_capa,
 		       IWL_UCODE_TLV_API_NEW_BEACON_TEMPLATE))
-		return iwl_mvm_mac_ctxt_send_beacon_v9(mvm, vif, beacon);
+		return iwl_mvm_mac_ctxt_send_beacon_v9(mvm, vif, beacon,
+						       link_conf);
 
 	return iwl_mvm_mac_ctxt_send_beacon_v7(mvm, vif, beacon);
 }
 
 /* The beacon template for the AP/GO/IBSS has changed and needs update */
 int iwl_mvm_mac_ctxt_beacon_changed(struct iwl_mvm *mvm,
-				    struct ieee80211_vif *vif)
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_bss_conf *link_conf)
 {
 	struct sk_buff *beacon;
 	int ret;
@@ -1046,7 +1169,8 @@ int iwl_mvm_mac_ctxt_beacon_changed(struct iwl_mvm *mvm,
 	WARN_ON(vif->type != NL80211_IFTYPE_AP &&
 		vif->type != NL80211_IFTYPE_ADHOC);
 
-	beacon = ieee80211_beacon_get_template(mvm->hw, vif, NULL, 0);
+	beacon = ieee80211_beacon_get_template(mvm->hw, vif, NULL,
+					       link_conf->link_id);
 	if (!beacon)
 		return -ENOMEM;
 
@@ -1057,7 +1181,7 @@ int iwl_mvm_mac_ctxt_beacon_changed(struct iwl_mvm *mvm,
 	}
 #endif
 
-	ret = iwl_mvm_mac_ctxt_send_beacon(mvm, vif, beacon);
+	ret = iwl_mvm_mac_ctxt_send_beacon(mvm, vif, beacon, link_conf);
 	dev_kfree_skb(beacon);
 	return ret;
 }
@@ -1087,6 +1211,30 @@ static void iwl_mvm_mac_ap_iterator(void *_data, u8 *mac,
 }
 
 /*
+ * Fill the filter flags for mac context of type AP or P2P GO.
+ */
+void iwl_mvm_mac_ctxt_cmd_ap_set_filter_flags(struct iwl_mvm *mvm,
+					      struct iwl_mvm_vif *mvmvif,
+					      __le32 *filter_flags,
+					      int accept_probe_req_flag,
+					      int accept_beacon_flag)
+{
+	/*
+	 * in AP mode, pass probe requests and beacons from other APs
+	 * (needed for ht protection); when there're no any associated
+	 * station don't ask FW to pass beacons to prevent unnecessary
+	 * wake-ups.
+	 */
+	*filter_flags |= cpu_to_le32(accept_probe_req_flag);
+	if (mvmvif->ap_assoc_sta_count || !mvm->drop_bcn_ap_mode) {
+		*filter_flags |= cpu_to_le32(accept_beacon_flag);
+		IWL_DEBUG_HC(mvm, "Asking FW to pass beacons\n");
+	} else {
+		IWL_DEBUG_HC(mvm, "No need to receive beacons\n");
+	}
+}
+
+/*
  * Fill the specific data for mac context of type AP of P2P GO
  */
 static void iwl_mvm_mac_ctxt_cmd_fill_ap(struct iwl_mvm *mvm,
@@ -1105,19 +1253,10 @@ static void iwl_mvm_mac_ctxt_cmd_fill_ap(struct iwl_mvm *mvm,
 	/* in AP mode, the MCAST FIFO takes the EDCA params from VO */
 	cmd->ac[IWL_MVM_TX_FIFO_VO].fifos_mask |= BIT(IWL_MVM_TX_FIFO_MCAST);
 
-	/*
-	 * in AP mode, pass probe requests and beacons from other APs
-	 * (needed for ht protection); when there're no any associated
-	 * station don't ask FW to pass beacons to prevent unnecessary
-	 * wake-ups.
-	 */
-	cmd->filter_flags |= cpu_to_le32(MAC_FILTER_IN_PROBE_REQUEST);
-	if (mvmvif->ap_assoc_sta_count || !mvm->drop_bcn_ap_mode) {
-		cmd->filter_flags |= cpu_to_le32(MAC_FILTER_IN_BEACON);
-		IWL_DEBUG_HC(mvm, "Asking FW to pass beacons\n");
-	} else {
-		IWL_DEBUG_HC(mvm, "No need to receive beacons\n");
-	}
+	iwl_mvm_mac_ctxt_cmd_ap_set_filter_flags(mvm, mvmvif,
+						 &cmd->filter_flags,
+						 MAC_FILTER_IN_PROBE_REQUEST,
+						 MAC_FILTER_IN_BEACON);
 
 	ctxt_ap->bi = cpu_to_le32(vif->bss_conf.beacon_int);
 	ctxt_ap->dtim_interval = cpu_to_le32(vif->bss_conf.beacon_int *
@@ -1125,7 +1264,7 @@ static void iwl_mvm_mac_ctxt_cmd_fill_ap(struct iwl_mvm *mvm,
 
 	if (!fw_has_api(&mvm->fw->ucode_capa,
 			IWL_UCODE_TLV_API_STA_TYPE))
-		ctxt_ap->mcast_qid = cpu_to_le32(mvmvif->cab_queue);
+		ctxt_ap->mcast_qid = cpu_to_le32(mvmvif->deflink.cab_queue);
 
 	/*
 	 * Only set the beacon time when the MAC is being added, when we
@@ -1279,12 +1418,9 @@ int iwl_mvm_mac_ctxt_remove(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 							   mvmvif->color));
 	cmd.action = cpu_to_le32(FW_CTXT_ACTION_REMOVE);
 
-	ret = iwl_mvm_send_cmd_pdu(mvm, MAC_CONTEXT_CMD, 0,
-				   sizeof(cmd), &cmd);
-	if (ret) {
-		IWL_ERR(mvm, "Failed to remove MAC context: %d\n", ret);
+	ret = iwl_mvm_mac_ctxt_send_cmd(mvm, &cmd);
+	if (ret)
 		return ret;
-	}
 
 	mvmvif->uploaded = false;
 
@@ -1312,7 +1448,8 @@ static void iwl_mvm_csa_count_down(struct iwl_mvm *mvm,
 	if (!ieee80211_beacon_cntdwn_is_complete(csa_vif)) {
 		int c = ieee80211_beacon_update_cntdwn(csa_vif);
 
-		iwl_mvm_mac_ctxt_beacon_changed(mvm, csa_vif);
+		iwl_mvm_mac_ctxt_beacon_changed(mvm, csa_vif,
+						&csa_vif->bss_conf);
 		if (csa_vif->p2p &&
 		    !iwl_mvm_te_scheduled(&mvmvif->time_event_data) && gp2 &&
 		    tx_success) {
@@ -1420,6 +1557,7 @@ void iwl_mvm_rx_missed_beacons_notif(struct iwl_mvm *mvm,
 	struct ieee80211_vif *vif;
 	u32 id = le32_to_cpu(mb->mac_id);
 	union iwl_dbg_tlv_tp_data tp_data = { .fw_pkt = pkt };
+	u32 mac_type;
 
 	IWL_DEBUG_INFO(mvm,
 		       "missed bcn mac_id=%u, consecutive=%u (%u, %u, %u)\n",
@@ -1435,6 +1573,14 @@ void iwl_mvm_rx_missed_beacons_notif(struct iwl_mvm *mvm,
 	if (!vif)
 		goto out;
 
+	mac_type = iwl_mvm_get_mac_type(vif);
+
+	IWL_DEBUG_INFO(mvm, "missed beacon mac_type=%u,\n", mac_type);
+
+	mvm->trans->dbg.dump_file_name_ext_valid = true;
+	snprintf(mvm->trans->dbg.dump_file_name_ext, IWL_FW_INI_MAX_NAME,
+		 "MacId_%d_MacType_%d", id, mac_type);
+
 	rx_missed_bcon = le32_to_cpu(mb->consec_missed_beacons);
 	rx_missed_bcon_since_rx =
 		le32_to_cpu(mb->consec_missed_beacons_since_last_rx);
@@ -1568,9 +1714,9 @@ void iwl_mvm_probe_resp_data_notif(struct iwl_mvm *mvm,
 	    sizeof(struct ieee80211_p2p_noa_desc) + 2)
 		new_data->noa_len -= sizeof(struct ieee80211_p2p_noa_desc);
 
-	old_data = rcu_dereference_protected(mvmvif->probe_resp_data,
-					lockdep_is_held(&mvmvif->mvm->mutex));
-	rcu_assign_pointer(mvmvif->probe_resp_data, new_data);
+	old_data = rcu_dereference_protected(mvmvif->deflink.probe_resp_data,
+					     lockdep_is_held(&mvmvif->mvm->mutex));
+	rcu_assign_pointer(mvmvif->deflink.probe_resp_data, new_data);
 
 	if (old_data)
 		kfree_rcu(old_data, rcu_head);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
index b55b1b17f4d1..0f01b62357c6 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
@@ -28,6 +28,7 @@
 #include "fw/error-dump.h"
 #include "iwl-prph.h"
 #include "iwl-nvm-parse.h"
+#include "time-sync.h"
 
 static const struct ieee80211_iface_limit iwl_mvm_limits[] = {
 	{
@@ -222,23 +223,46 @@ int iwl_mvm_init_fw_regd(struct iwl_mvm *mvm)
 	return ret;
 }
 
+/* Each capability added here should also be add to tm_if_types_ext_capa_sta */
 static const u8 he_if_types_ext_capa_sta[] = {
 	 [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING,
 	 [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT,
-	 [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF,
+	 [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF |
+	       WLAN_EXT_CAPA8_MAX_MSDU_IN_AMSDU_LSB,
+	 [8] = WLAN_EXT_CAPA9_MAX_MSDU_IN_AMSDU_MSB,
 };
 
-static const struct wiphy_iftype_ext_capab he_iftypes_ext_capa[] = {
+static const u8 tm_if_types_ext_capa_sta[] = {
+	 [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING,
+	 [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT |
+	       WLAN_EXT_CAPA3_TIMING_MEASUREMENT_SUPPORT,
+	 [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF |
+	       WLAN_EXT_CAPA8_MAX_MSDU_IN_AMSDU_LSB,
+	 [8] = WLAN_EXT_CAPA9_MAX_MSDU_IN_AMSDU_MSB,
+	 [9] = WLAN_EXT_CAPA10_TWT_REQUESTER_SUPPORT,
+};
+
+/* Additional interface types for which extended capabilities are
+ * specified separately
+ */
+static const struct wiphy_iftype_ext_capab add_iftypes_ext_capa[] = {
 	{
 		.iftype = NL80211_IFTYPE_STATION,
 		.extended_capabilities = he_if_types_ext_capa_sta,
 		.extended_capabilities_mask = he_if_types_ext_capa_sta,
 		.extended_capabilities_len = sizeof(he_if_types_ext_capa_sta),
 	},
+	{
+		.iftype = NL80211_IFTYPE_STATION,
+		.extended_capabilities = tm_if_types_ext_capa_sta,
+		.extended_capabilities_mask = tm_if_types_ext_capa_sta,
+		.extended_capabilities_len = sizeof(tm_if_types_ext_capa_sta),
+		/* relevant only if EHT is supported */
+		.eml_capabilities = IEEE80211_EML_CAP_EMLSR_SUPP,
+	},
 };
 
-static int
-iwl_mvm_op_get_antenna(struct ieee80211_hw *hw, u32 *tx_ant, u32 *rx_ant)
+int iwl_mvm_op_get_antenna(struct ieee80211_hw *hw, u32 *tx_ant, u32 *rx_ant)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	*tx_ant = iwl_mvm_get_valid_tx_ant(mvm);
@@ -260,6 +284,8 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	bool unified = fw_has_capa(&mvm->fw->ucode_capa,
 				   IWL_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG);
 #endif
+	u32 sec_key_id = WIDE_ID(DATA_PATH_GROUP, SEC_KEY_CMD);
+	u8 sec_key_ver = iwl_fw_lookup_cmd_ver(mvm->fw, sec_key_id, 0);
 
 	/* Tell mac80211 our characteristics */
 	ieee80211_hw_set(hw, SIGNAL_DBM);
@@ -269,17 +295,33 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	ieee80211_hw_set(hw, SUPPORTS_PS);
 	ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS);
 	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
-	ieee80211_hw_set(hw, TIMING_BEACON_ONLY);
 	ieee80211_hw_set(hw, CONNECTION_MONITOR);
 	ieee80211_hw_set(hw, CHANCTX_STA_CSA);
 	ieee80211_hw_set(hw, SUPPORT_FAST_XMIT);
 	ieee80211_hw_set(hw, SUPPORTS_CLONED_SKBS);
 	ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU);
 	ieee80211_hw_set(hw, NEEDS_UNIQUE_STA_ADDR);
-	ieee80211_hw_set(hw, DEAUTH_NEED_MGD_TX_PREP);
 	ieee80211_hw_set(hw, SUPPORTS_VHT_EXT_NSS_BW);
 	ieee80211_hw_set(hw, BUFF_MMPDU_TXQ);
 	ieee80211_hw_set(hw, STA_MMPDU_TXQ);
+
+	/* Set this early since we need to have it for the check below */
+	if (mvm->mld_api_is_used &&
+	    mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+		hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_MLO;
+
+	/* With MLD FW API, it tracks timing by itself,
+	 * no need for any timing from the host
+	 */
+	if (!mvm->mld_api_is_used)
+		ieee80211_hw_set(hw, TIMING_BEACON_ONLY);
+
+	/* We should probably have this, but mac80211
+	 * currently doesn't support it for MLO.
+	 */
+	if (!(hw->wiphy->flags & WIPHY_FLAG_SUPPORTS_MLO))
+		ieee80211_hw_set(hw, DEAUTH_NEED_MGD_TX_PREP);
+
 	/*
 	 * On older devices, enabling TX A-MSDU occasionally leads to
 	 * something getting messed up, the command read from the FIFO
@@ -376,6 +418,8 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 
 	wiphy_ext_feature_set(hw->wiphy,
 			      NL80211_EXT_FEATURE_BEACON_RATE_LEGACY);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT);
 
 	if (fw_has_capa(&mvm->fw->ucode_capa,
 			IWL_UCODE_TLV_CAPA_FTM_CALIBRATED)) {
@@ -384,11 +428,20 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 		hw->wiphy->pmsr_capa = &iwl_mvm_pmsr_capa;
 	}
 
-	if (fw_has_capa(&mvm->fw->ucode_capa,
-			IWL_UCODE_TLV_CAPA_BIGTK_SUPPORT))
+	if (sec_key_ver &&
+	    fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_BIGTK_TX_SUPPORT))
+		wiphy_ext_feature_set(hw->wiphy,
+				      NL80211_EXT_FEATURE_BEACON_PROTECTION);
+	else if (fw_has_capa(&mvm->fw->ucode_capa,
+			     IWL_UCODE_TLV_CAPA_BIGTK_SUPPORT))
 		wiphy_ext_feature_set(hw->wiphy,
 				      NL80211_EXT_FEATURE_BEACON_PROTECTION_CLIENT);
 
+	if (fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_TIME_SYNC_BOTH_FTM_TM))
+		hw->wiphy->hw_timestamp_max_peers = 1;
+
 	ieee80211_hw_set(hw, SINGLE_SCAN_ON_ALL_BANDS);
 	hw->wiphy->features |=
 		NL80211_FEATURE_SCHED_SCAN_RANDOM_MAC_ADDR |
@@ -562,16 +615,34 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 					      NL80211_EXT_FEATURE_OCE_PROBE_REQ_DEFERRAL_SUPPRESSION);
 	}
 
+	hw->wiphy->iftype_ext_capab = NULL;
+	hw->wiphy->num_iftype_ext_capab = 0;
+
 	if (mvm->nvm_data->sku_cap_11ax_enable &&
 	    !iwlwifi_mod_params.disable_11ax) {
-		hw->wiphy->iftype_ext_capab = he_iftypes_ext_capa;
+		hw->wiphy->iftype_ext_capab = add_iftypes_ext_capa;
 		hw->wiphy->num_iftype_ext_capab =
-			ARRAY_SIZE(he_iftypes_ext_capa);
+			ARRAY_SIZE(add_iftypes_ext_capa) - 1;
 
 		ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID);
 		ieee80211_hw_set(hw, SUPPORTS_ONLY_HE_MULTI_BSSID);
 	}
 
+	if (iwl_fw_lookup_cmd_ver(mvm->fw,
+				  WIDE_ID(DATA_PATH_GROUP,
+					  WNM_80211V_TIMING_MEASUREMENT_CONFIG_CMD),
+				  IWL_FW_CMD_VER_UNKNOWN) >= 1) {
+		IWL_DEBUG_INFO(mvm->trans, "Timing measurement supported\n");
+
+		if (!hw->wiphy->iftype_ext_capab) {
+			hw->wiphy->num_iftype_ext_capab = 1;
+			hw->wiphy->iftype_ext_capab = add_iftypes_ext_capa +
+				ARRAY_SIZE(add_iftypes_ext_capa) - 1;
+		} else {
+			hw->wiphy->iftype_ext_capab = add_iftypes_ext_capa + 1;
+		}
+	}
+
 	mvm->rts_threshold = IEEE80211_MAX_RTS_THRESHOLD;
 
 #ifdef CONFIG_PM_SLEEP
@@ -653,9 +724,8 @@ static void iwl_mvm_tx_skb(struct iwl_mvm *mvm, struct sk_buff *skb,
 	ieee80211_free_txskb(mvm->hw, skb);
 }
 
-static void iwl_mvm_mac_tx(struct ieee80211_hw *hw,
-			   struct ieee80211_tx_control *control,
-			   struct sk_buff *skb)
+void iwl_mvm_mac_tx(struct ieee80211_hw *hw,
+		    struct ieee80211_tx_control *control, struct sk_buff *skb)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct ieee80211_sta *sta = control->sta;
@@ -663,6 +733,9 @@ static void iwl_mvm_mac_tx(struct ieee80211_hw *hw,
 	struct ieee80211_hdr *hdr = (void *)skb->data;
 	bool offchannel = IEEE80211_SKB_CB(skb)->flags &
 		IEEE80211_TX_CTL_TX_OFFCHAN;
+	u32 link_id = u32_get_bits(info->control.flags,
+				   IEEE80211_TX_CTRL_MLO_LINK);
+	struct ieee80211_sta *tmp_sta = sta;
 
 	if (iwl_mvm_is_radio_killed(mvm)) {
 		IWL_DEBUG_DROP(mvm, "Dropping - RF/CT KILL\n");
@@ -686,7 +759,7 @@ static void iwl_mvm_mac_tx(struct ieee80211_hw *hw,
 	    !offchannel) {
 		struct iwl_mvm_vif *mvmvif =
 			iwl_mvm_vif_from_mac80211(info->control.vif);
-		u8 ap_sta_id = READ_ONCE(mvmvif->ap_sta_id);
+		u8 ap_sta_id = READ_ONCE(mvmvif->deflink.ap_sta_id);
 
 		if (ap_sta_id < mvm->fw->ucode_capa.num_stations) {
 			/* mac80211 holds rcu read lock */
@@ -696,6 +769,25 @@ static void iwl_mvm_mac_tx(struct ieee80211_hw *hw,
 		}
 	}
 
+	if (tmp_sta && !sta && link_id != IEEE80211_LINK_UNSPECIFIED &&
+	    !ieee80211_is_probe_resp(hdr->frame_control)) {
+		/* translate MLD addresses to LINK addresses */
+		struct ieee80211_link_sta *link_sta =
+			rcu_dereference(tmp_sta->link[link_id]);
+		struct ieee80211_bss_conf *link_conf =
+			rcu_dereference(info->control.vif->link_conf[link_id]);
+		struct ieee80211_mgmt *mgmt;
+
+		if (WARN_ON(!link_sta || !link_conf))
+			goto drop;
+
+		/* if sta is NULL, the frame is a management frame */
+		mgmt = (void *)hdr;
+		memcpy(mgmt->da, link_sta->addr, ETH_ALEN);
+		memcpy(mgmt->sa, link_conf->addr, ETH_ALEN);
+		memcpy(mgmt->bssid, link_conf->bssid, ETH_ALEN);
+	}
+
 	iwl_mvm_tx_skb(mvm, skb, sta);
 	return;
  drop:
@@ -754,8 +846,8 @@ void iwl_mvm_mac_itxq_xmit(struct ieee80211_hw *hw, struct ieee80211_txq *txq)
 	rcu_read_unlock();
 }
 
-static void iwl_mvm_mac_wake_tx_queue(struct ieee80211_hw *hw,
-				      struct ieee80211_txq *txq)
+void iwl_mvm_mac_wake_tx_queue(struct ieee80211_hw *hw,
+			       struct ieee80211_txq *txq)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_txq *mvmtxq = iwl_mvm_txq_from_mac80211(txq);
@@ -833,9 +925,9 @@ iwl_mvm_ampdu_check_trigger(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	}
 }
 
-static int iwl_mvm_mac_ampdu_action(struct ieee80211_hw *hw,
-				    struct ieee80211_vif *vif,
-				    struct ieee80211_ampdu_params *params)
+int iwl_mvm_mac_ampdu_action(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_ampdu_params *params)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int ret;
@@ -857,8 +949,8 @@ static int iwl_mvm_mac_ampdu_action(struct ieee80211_hw *hw,
 
 	switch (action) {
 	case IEEE80211_AMPDU_RX_START:
-		if (iwl_mvm_vif_from_mac80211(vif)->ap_sta_id ==
-				iwl_mvm_sta_from_mac80211(sta)->sta_id) {
+		if (iwl_mvm_vif_from_mac80211(vif)->deflink.ap_sta_id ==
+		    iwl_mvm_sta_from_mac80211(sta)->deflink.sta_id) {
 			struct iwl_mvm_vif *mvmvif;
 			u16 macid = iwl_mvm_vif_from_mac80211(vif)->id;
 			struct iwl_mvm_tcm_mac *mdata = &mvm->tcm.data[macid];
@@ -921,17 +1013,29 @@ static void iwl_mvm_cleanup_iterator(void *data, u8 *mac,
 {
 	struct iwl_mvm *mvm = data;
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_probe_resp_data *probe_data;
+	unsigned int link_id;
 
 	mvmvif->uploaded = false;
-	mvmvif->ap_sta_id = IWL_MVM_INVALID_STA;
 
 	spin_lock_bh(&mvm->time_event_lock);
 	iwl_mvm_te_clear_data(mvm, &mvmvif->time_event_data);
 	spin_unlock_bh(&mvm->time_event_lock);
 
-	mvmvif->phy_ctxt = NULL;
 	memset(&mvmvif->bf_data, 0, sizeof(mvmvif->bf_data));
-	memset(&mvmvif->probe_resp_data, 0, sizeof(mvmvif->probe_resp_data));
+
+	for_each_mvm_vif_valid_link(mvmvif, link_id) {
+		mvmvif->link[link_id]->ap_sta_id = IWL_MVM_INVALID_STA;
+		mvmvif->link[link_id]->fw_link_id = IWL_MVM_FW_LINK_ID_INVALID;
+		mvmvif->link[link_id]->phy_ctxt = NULL;
+		mvmvif->link[link_id]->active = 0;
+	}
+
+	probe_data = rcu_dereference_protected(mvmvif->deflink.probe_resp_data,
+					       lockdep_is_held(&mvm->mutex));
+	if (probe_data)
+		kfree_rcu(probe_data, rcu_head);
+	RCU_INIT_POINTER(mvmvif->deflink.probe_resp_data, NULL);
 }
 
 static void iwl_mvm_restart_cleanup(struct iwl_mvm *mvm)
@@ -968,6 +1072,9 @@ static void iwl_mvm_restart_cleanup(struct iwl_mvm *mvm)
 	mvm->rx_ba_sessions = 0;
 	mvm->fwrt.dump.conf = FW_DBG_INVALID;
 	mvm->monitor_on = false;
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	mvm->beacon_inject_active = false;
+#endif
 
 	/* keep statistics ticking */
 	iwl_mvm_accu_radio_stats(mvm);
@@ -1030,7 +1137,7 @@ int __iwl_mvm_mac_start(struct iwl_mvm *mvm)
 	return ret;
 }
 
-static int iwl_mvm_mac_start(struct ieee80211_hw *hw)
+int iwl_mvm_mac_start(struct ieee80211_hw *hw)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int ret;
@@ -1099,9 +1206,8 @@ static void iwl_mvm_restart_complete(struct iwl_mvm *mvm)
 	mutex_unlock(&mvm->mutex);
 }
 
-static void
-iwl_mvm_mac_reconfig_complete(struct ieee80211_hw *hw,
-			      enum ieee80211_reconfig_type reconfig_type)
+void iwl_mvm_mac_reconfig_complete(struct ieee80211_hw *hw,
+				   enum ieee80211_reconfig_type reconfig_type)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -1163,7 +1269,7 @@ void __iwl_mvm_mac_stop(struct iwl_mvm *mvm)
 	}
 }
 
-static void iwl_mvm_mac_stop(struct ieee80211_hw *hw)
+void iwl_mvm_mac_stop(struct ieee80211_hw *hw)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -1202,7 +1308,7 @@ static void iwl_mvm_mac_stop(struct ieee80211_hw *hw)
 	cancel_work_sync(&mvm->async_handlers_wk);
 }
 
-static struct iwl_mvm_phy_ctxt *iwl_mvm_get_free_phy_ctxt(struct iwl_mvm *mvm)
+struct iwl_mvm_phy_ctxt *iwl_mvm_get_free_phy_ctxt(struct iwl_mvm *mvm)
 {
 	u16 i;
 
@@ -1216,8 +1322,8 @@ static struct iwl_mvm_phy_ctxt *iwl_mvm_get_free_phy_ctxt(struct iwl_mvm *mvm)
 	return NULL;
 }
 
-static int iwl_mvm_set_tx_power(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
-				s16 tx_power)
+int iwl_mvm_set_tx_power(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			 s16 tx_power)
 {
 	u32 cmd_id = REDUCE_TX_POWER_CMD;
 	int len;
@@ -1252,8 +1358,8 @@ static int iwl_mvm_set_tx_power(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	return iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, len, &cmd);
 }
 
-static int iwl_mvm_post_channel_switch(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif)
+int iwl_mvm_post_channel_switch(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
@@ -1266,7 +1372,7 @@ static int iwl_mvm_post_channel_switch(struct ieee80211_hw *hw,
 
 		mvmvif->csa_bcn_pending = false;
 		mvmsta = iwl_mvm_sta_from_staid_protected(mvm,
-							  mvmvif->ap_sta_id);
+							  mvmvif->deflink.ap_sta_id);
 
 		if (WARN_ON(!mvmsta)) {
 			ret = -EIO;
@@ -1274,8 +1380,10 @@ static int iwl_mvm_post_channel_switch(struct ieee80211_hw *hw,
 		}
 
 		iwl_mvm_sta_modify_disable_tx(mvm, mvmsta, false);
-
-		iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL);
+		if (mvm->mld_api_is_used)
+			iwl_mvm_mld_mac_ctxt_changed(mvm, vif, false);
+		else
+			iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL);
 
 		if (!fw_has_capa(&mvm->fw->ucode_capa,
 				 IWL_UCODE_TLV_CAPA_CHANNEL_SWITCH_CMD)) {
@@ -1299,8 +1407,8 @@ out_unlock:
 	return ret;
 }
 
-static void iwl_mvm_abort_channel_switch(struct ieee80211_hw *hw,
-					 struct ieee80211_vif *vif)
+void iwl_mvm_abort_channel_switch(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
@@ -1336,7 +1444,7 @@ static void iwl_mvm_abort_channel_switch(struct ieee80211_hw *hw,
 	iwl_mvm_post_channel_switch(hw, vif);
 }
 
-static void iwl_mvm_channel_switch_disconnect_wk(struct work_struct *wk)
+void iwl_mvm_channel_switch_disconnect_wk(struct work_struct *wk)
 {
 	struct iwl_mvm_vif *mvmvif;
 	struct ieee80211_vif *vif;
@@ -1348,15 +1456,47 @@ static void iwl_mvm_channel_switch_disconnect_wk(struct work_struct *wk)
 	ieee80211_chswitch_done(vif, false);
 }
 
-static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
-				     struct ieee80211_vif *vif)
+static u8
+iwl_mvm_chandef_get_primary_80(struct cfg80211_chan_def *chandef)
+{
+	int data_start;
+	int control_start;
+	int bw;
+
+	if (chandef->width == NL80211_CHAN_WIDTH_320)
+		bw = 320;
+	else if (chandef->width == NL80211_CHAN_WIDTH_160)
+		bw = 160;
+	else
+		return 0;
+
+	/* data is bw wide so the start is half the width */
+	data_start = chandef->center_freq1 - bw / 2;
+	/* control is 20Mhz width */
+	control_start = chandef->chan->center_freq - 10;
+
+	return (control_start - data_start) / 80;
+}
+
+/*
+ * Returns true if addding the interface is done
+ * (either with success or failure)
+ *
+ * FIXME: remove this again and merge it in
+ */
+static bool iwl_mvm_mac_add_interface_common(struct iwl_mvm *mvm,
+					     struct ieee80211_hw *hw,
+					     struct ieee80211_vif *vif,
+					     int *ret)
 {
-	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
 
 	mvmvif->mvm = mvm;
-	RCU_INIT_POINTER(mvmvif->probe_resp_data, NULL);
+
+	/* the first link always points to the default one */
+	mvmvif->link[0] = &mvmvif->deflink;
 
 	/*
 	 * Not much to do here. The stack will not allow interface
@@ -1364,17 +1504,15 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 	 * don't really have to check the types.
 	 */
 
-	mutex_lock(&mvm->mutex);
-
 	/* make sure that beacon statistics don't go backwards with FW reset */
 	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
-		mvmvif->beacon_stats.accu_num_beacons +=
-			mvmvif->beacon_stats.num_beacons;
+		mvmvif->deflink.beacon_stats.accu_num_beacons +=
+			mvmvif->deflink.beacon_stats.num_beacons;
 
 	/* Allocate resources for the MAC context, and add it to the fw  */
-	ret = iwl_mvm_mac_ctxt_init(mvm, vif);
-	if (ret)
-		goto out_unlock;
+	*ret = iwl_mvm_mac_ctxt_init(mvm, vif);
+	if (*ret)
+		return true;
 
 	rcu_assign_pointer(mvm->vif_id_to_mac[mvmvif->id], vif);
 
@@ -1391,28 +1529,51 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 	 */
 	if (vif->type == NL80211_IFTYPE_AP ||
 	    vif->type == NL80211_IFTYPE_ADHOC) {
-		ret = iwl_mvm_alloc_bcast_sta(mvm, vif);
-		if (ret) {
-			IWL_ERR(mvm, "Failed to allocate bcast sta\n");
-			goto out_unlock;
-		}
-
-		/*
-		 * Only queue for this station is the mcast queue,
-		 * which shouldn't be in TFD mask anyway
-		 */
-		ret = iwl_mvm_allocate_int_sta(mvm, &mvmvif->mcast_sta,
-					       0, vif->type,
-					       IWL_STA_MULTICAST);
-		if (ret)
-			goto out_unlock;
-
 		iwl_mvm_vif_dbgfs_register(mvm, vif);
-		goto out_unlock;
+		return true;
 	}
 
 	mvmvif->features |= hw->netdev_features;
 
+	return false;
+}
+
+static int iwl_mvm_alloc_bcast_mcast_sta(struct iwl_mvm *mvm,
+					 struct ieee80211_vif *vif)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	ret = iwl_mvm_alloc_bcast_sta(mvm, vif);
+	if (ret) {
+		IWL_ERR(mvm, "Failed to allocate bcast sta\n");
+		return ret;
+	}
+
+	/*
+	 * Only queue for this station is the mcast queue,
+	 * which shouldn't be in TFD mask anyway
+	 */
+	return iwl_mvm_allocate_int_sta(mvm, &mvmvif->deflink.mcast_sta, 0,
+					vif->type,
+					IWL_STA_MULTICAST);
+}
+
+static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int ret;
+
+	mutex_lock(&mvm->mutex);
+
+	/* Common for MLD and non-MLD API */
+	if (iwl_mvm_mac_add_interface_common(mvm, hw, vif, &ret))
+		goto out;
+
 	ret = iwl_mvm_mac_ctxt_add(mvm, vif);
 	if (ret)
 		goto out_unlock;
@@ -1440,13 +1601,13 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 	 */
 	if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
 
-		mvmvif->phy_ctxt = iwl_mvm_get_free_phy_ctxt(mvm);
-		if (!mvmvif->phy_ctxt) {
+		mvmvif->deflink.phy_ctxt = iwl_mvm_get_free_phy_ctxt(mvm);
+		if (!mvmvif->deflink.phy_ctxt) {
 			ret = -ENOSPC;
 			goto out_free_bf;
 		}
 
-		iwl_mvm_phy_ctxt_ref(mvm, mvmvif->phy_ctxt);
+		iwl_mvm_phy_ctxt_ref(mvm, mvmvif->deflink.phy_ctxt);
 		ret = iwl_mvm_binding_add_vif(mvm, vif);
 		if (ret)
 			goto out_unref_phy;
@@ -1464,8 +1625,11 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 	INIT_DELAYED_WORK(&mvmvif->csa_work,
 			  iwl_mvm_channel_switch_disconnect_wk);
 
-	if (vif->type == NL80211_IFTYPE_MONITOR)
+	if (vif->type == NL80211_IFTYPE_MONITOR) {
 		mvm->monitor_on = true;
+		mvm->monitor_p80 =
+			iwl_mvm_chandef_get_primary_80(&vif->bss_conf.chandef);
+	}
 
 	iwl_mvm_vif_dbgfs_register(mvm, vif);
 
@@ -1477,12 +1641,17 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 		mvm->csme_vif = vif;
 	}
 
+out:
+	if (!ret && (vif->type == NL80211_IFTYPE_AP ||
+		     vif->type == NL80211_IFTYPE_ADHOC))
+		ret = iwl_mvm_alloc_bcast_mcast_sta(mvm, vif);
+
 	goto out_unlock;
 
  out_unbind:
 	iwl_mvm_binding_remove_vif(mvm, vif);
  out_unref_phy:
-	iwl_mvm_phy_ctxt_unref(mvm, mvmvif->phy_ctxt);
+	iwl_mvm_phy_ctxt_unref(mvm, mvmvif->deflink.phy_ctxt);
  out_free_bf:
 	if (mvm->bf_allowed_vif == mvmvif) {
 		mvm->bf_allowed_vif = NULL;
@@ -1490,7 +1659,7 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 				       IEEE80211_VIF_SUPPORTS_CQM_RSSI);
 	}
  out_remove_mac:
-	mvmvif->phy_ctxt = NULL;
+	mvmvif->deflink.phy_ctxt = NULL;
 	iwl_mvm_mac_ctxt_remove(mvm, vif);
  out_unlock:
 	mutex_unlock(&mvm->mutex);
@@ -1498,8 +1667,8 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 	return ret;
 }
 
-static void iwl_mvm_prepare_mac_removal(struct iwl_mvm *mvm,
-					struct ieee80211_vif *vif)
+void iwl_mvm_prepare_mac_removal(struct iwl_mvm *mvm,
+				 struct ieee80211_vif *vif)
 {
 	if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
 		/*
@@ -1511,8 +1680,12 @@ static void iwl_mvm_prepare_mac_removal(struct iwl_mvm *mvm,
 	}
 }
 
-static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw,
-					 struct ieee80211_vif *vif)
+/* This function is doing the common part of removing the interface for
+ * both - MLD and non-MLD modes. Returns true if removing the interface
+ * is done
+ */
+static bool iwl_mvm_mac_remove_interface_common(struct ieee80211_hw *hw,
+						struct ieee80211_vif *vif)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
@@ -1531,9 +1704,9 @@ static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw,
 		mvm->csme_vif = NULL;
 	}
 
-	probe_data = rcu_dereference_protected(mvmvif->probe_resp_data,
+	probe_data = rcu_dereference_protected(mvmvif->deflink.probe_resp_data,
 					       lockdep_is_held(&mvm->mutex));
-	RCU_INIT_POINTER(mvmvif->probe_resp_data, NULL);
+	RCU_INIT_POINTER(mvmvif->deflink.probe_resp_data, NULL);
 	if (probe_data)
 		kfree_rcu(probe_data, rcu_head);
 
@@ -1560,20 +1733,30 @@ static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw,
 			mvm->noa_duration = 0;
 		}
 #endif
-		iwl_mvm_dealloc_int_sta(mvm, &mvmvif->mcast_sta);
-		iwl_mvm_dealloc_bcast_sta(mvm, vif);
-		goto out_release;
+		return true;
 	}
 
+	iwl_mvm_power_update_mac(mvm);
+	return false;
+}
+
+static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw,
+					 struct ieee80211_vif *vif)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+
+	if (iwl_mvm_mac_remove_interface_common(hw, vif))
+		goto out;
+
 	if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
 		mvm->p2p_device_vif = NULL;
 		iwl_mvm_rm_p2p_bcast_sta(mvm, vif);
 		iwl_mvm_binding_remove_vif(mvm, vif);
-		iwl_mvm_phy_ctxt_unref(mvm, mvmvif->phy_ctxt);
-		mvmvif->phy_ctxt = NULL;
+		iwl_mvm_phy_ctxt_unref(mvm, mvmvif->deflink.phy_ctxt);
+		mvmvif->deflink.phy_ctxt = NULL;
 	}
 
-	iwl_mvm_power_update_mac(mvm);
 	iwl_mvm_mac_ctxt_remove(mvm, vif);
 
 	RCU_INIT_POINTER(mvm->vif_id_to_mac[mvmvif->id], NULL);
@@ -1581,13 +1764,14 @@ static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw,
 	if (vif->type == NL80211_IFTYPE_MONITOR)
 		mvm->monitor_on = false;
 
-out_release:
-	mutex_unlock(&mvm->mutex);
-}
+out:
+	if (vif->type == NL80211_IFTYPE_AP ||
+	    vif->type == NL80211_IFTYPE_ADHOC) {
+		iwl_mvm_dealloc_int_sta(mvm, &mvmvif->deflink.mcast_sta);
+		iwl_mvm_dealloc_bcast_sta(mvm, vif);
+	}
 
-static int iwl_mvm_mac_config(struct ieee80211_hw *hw, u32 changed)
-{
-	return 0;
+	mutex_unlock(&mvm->mutex);
 }
 
 struct iwl_mvm_mc_iter_data {
@@ -1661,8 +1845,8 @@ static void iwl_mvm_recalc_multicast(struct iwl_mvm *mvm)
 		IWL_ERR(mvm, "Failed to synchronize multicast groups update\n");
 }
 
-static u64 iwl_mvm_prepare_multicast(struct ieee80211_hw *hw,
-				     struct netdev_hw_addr_list *mc_list)
+u64 iwl_mvm_prepare_multicast(struct ieee80211_hw *hw,
+			      struct netdev_hw_addr_list *mc_list)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mcast_filter_cmd *cmd;
@@ -1698,10 +1882,9 @@ static u64 iwl_mvm_prepare_multicast(struct ieee80211_hw *hw,
 	return (u64)(unsigned long)cmd;
 }
 
-static void iwl_mvm_configure_filter(struct ieee80211_hw *hw,
-				     unsigned int changed_flags,
-				     unsigned int *total_flags,
-				     u64 multicast)
+void iwl_mvm_configure_filter(struct ieee80211_hw *hw,
+			      unsigned int changed_flags,
+			      unsigned int *total_flags, u64 multicast)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mcast_filter_cmd *cmd = (void *)(unsigned long)multicast;
@@ -1748,8 +1931,7 @@ static void iwl_mvm_config_iface_filter(struct ieee80211_hw *hw,
 	mutex_unlock(&mvm->mutex);
 }
 
-static int iwl_mvm_update_mu_groups(struct iwl_mvm *mvm,
-				    struct ieee80211_vif *vif)
+int iwl_mvm_update_mu_groups(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
 	struct iwl_mu_group_mgmt_cmd cmd = {};
 
@@ -1874,12 +2056,13 @@ set_thresholds:
 }
 
 static void iwl_mvm_set_pkt_ext_from_he_ppe(struct iwl_mvm *mvm,
-					    struct ieee80211_sta *sta,
+					    struct ieee80211_link_sta *link_sta,
 					    struct iwl_he_pkt_ext_v2 *pkt_ext,
 					    bool inheritance)
 {
-	u8 nss = (sta->deflink.he_cap.ppe_thres[0] & IEEE80211_PPE_THRES_NSS_MASK) + 1;
-	u8 *ppe = &sta->deflink.he_cap.ppe_thres[0];
+	u8 nss = (link_sta->he_cap.ppe_thres[0] &
+		  IEEE80211_PPE_THRES_NSS_MASK) + 1;
+	u8 *ppe = &link_sta->he_cap.ppe_thres[0];
 	u8 ru_index_bitmap =
 		u8_get_bits(*ppe,
 			    IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK);
@@ -1890,9 +2073,9 @@ static void iwl_mvm_set_pkt_ext_from_he_ppe(struct iwl_mvm *mvm,
 			  inheritance);
 }
 
-static void iwl_mvm_set_pkt_ext_from_nominal_padding(struct iwl_he_pkt_ext_v2 *pkt_ext,
-						     u8 nominal_padding,
-						     u32 *flags)
+static int
+iwl_mvm_set_pkt_ext_from_nominal_padding(struct iwl_he_pkt_ext_v2 *pkt_ext,
+					 u8 nominal_padding)
 {
 	int low_th = -1;
 	int high_th = -1;
@@ -1915,21 +2098,22 @@ static void iwl_mvm_set_pkt_ext_from_nominal_padding(struct iwl_he_pkt_ext_v2 *p
 		break;
 	}
 
+	if (low_th < 0 || high_th < 0)
+		return -EINVAL;
+
 	/* Set the PPE thresholds accordingly */
-	if (low_th >= 0 && high_th >= 0) {
-		for (i = 0; i < MAX_HE_SUPP_NSS; i++) {
-			u8 bw;
+	for (i = 0; i < MAX_HE_SUPP_NSS; i++) {
+		u8 bw;
 
-			for (bw = 0;
-			     bw < ARRAY_SIZE(pkt_ext->pkt_ext_qam_th[i]);
-			     bw++) {
-				pkt_ext->pkt_ext_qam_th[i][bw][0] = low_th;
-				pkt_ext->pkt_ext_qam_th[i][bw][1] = high_th;
-			}
+		for (bw = 0;
+			bw < ARRAY_SIZE(pkt_ext->pkt_ext_qam_th[i]);
+			bw++) {
+			pkt_ext->pkt_ext_qam_th[i][bw][0] = low_th;
+			pkt_ext->pkt_ext_qam_th[i][bw][1] = high_th;
 		}
-
-		*flags |= STA_CTXT_HE_PACKET_EXT;
 	}
+
+	return 0;
 }
 
 static void iwl_mvm_get_optimal_ppe_info(struct iwl_he_pkt_ext_v2 *pkt_ext,
@@ -1957,6 +2141,190 @@ static void iwl_mvm_get_optimal_ppe_info(struct iwl_he_pkt_ext_v2 *pkt_ext,
 	}
 }
 
+/* Set the pkt_ext field according to PPE Thresholds element */
+int iwl_mvm_set_sta_pkt_ext(struct iwl_mvm *mvm,
+			    struct ieee80211_link_sta *link_sta,
+			    struct iwl_he_pkt_ext_v2 *pkt_ext)
+{
+	u8 nominal_padding;
+	int i, ret = 0;
+
+	if (WARN_ON(!link_sta))
+		return -EINVAL;
+
+	/* Initialize the PPE thresholds to "None" (7), as described in Table
+	 * 9-262ac of 80211.ax/D3.0.
+	 */
+	memset(pkt_ext, IWL_HE_PKT_EXT_NONE,
+	       sizeof(struct iwl_he_pkt_ext_v2));
+
+	if (link_sta->eht_cap.has_eht) {
+		nominal_padding =
+			u8_get_bits(link_sta->eht_cap.eht_cap_elem.phy_cap_info[5],
+				    IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK);
+
+		/* If PPE Thresholds exists, parse them into a FW-familiar
+		 * format.
+		 */
+		if (link_sta->eht_cap.eht_cap_elem.phy_cap_info[5] &
+		    IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT) {
+			u8 nss = (link_sta->eht_cap.eht_ppe_thres[0] &
+				IEEE80211_EHT_PPE_THRES_NSS_MASK) + 1;
+			u8 *ppe = &link_sta->eht_cap.eht_ppe_thres[0];
+			u8 ru_index_bitmap =
+				u16_get_bits(*ppe,
+					     IEEE80211_EHT_PPE_THRES_RU_INDEX_BITMASK_MASK);
+			 /* Starting after PPE header */
+			u8 ppe_pos_bit = IEEE80211_EHT_PPE_THRES_INFO_HEADER_SIZE;
+
+			iwl_mvm_parse_ppe(mvm, pkt_ext, nss, ru_index_bitmap,
+					  ppe, ppe_pos_bit, true);
+		/* EHT PPE Thresholds doesn't exist - set the API according to
+		 * HE PPE Tresholds
+		 */
+		} else if (link_sta->he_cap.he_cap_elem.phy_cap_info[6] &
+			   IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
+			/* Even though HE Capabilities IE doesn't contain PPE
+			 * Thresholds for BW 320Mhz, thresholds for this BW will
+			 * be filled in with the same values as 160Mhz, due to
+			 * the inheritance, as required.
+			 */
+			iwl_mvm_set_pkt_ext_from_he_ppe(mvm, link_sta, pkt_ext,
+							true);
+
+			/* According to the requirements, for MCSs 12-13 the
+			 * maximum value between HE PPE Threshold and Common
+			 * Nominal Packet Padding needs to be taken
+			 */
+			iwl_mvm_get_optimal_ppe_info(pkt_ext, nominal_padding);
+
+		/* if PPE Thresholds doesn't present in both EHT IE and HE IE -
+		 * take the Thresholds from Common Nominal Packet Padding field
+		 */
+		} else {
+			ret = iwl_mvm_set_pkt_ext_from_nominal_padding(pkt_ext,
+								       nominal_padding);
+		}
+	} else if (link_sta->he_cap.has_he) {
+		/* If PPE Thresholds exist, parse them into a FW-familiar format. */
+		if (link_sta->he_cap.he_cap_elem.phy_cap_info[6] &
+			IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
+			iwl_mvm_set_pkt_ext_from_he_ppe(mvm, link_sta, pkt_ext,
+							false);
+		/* PPE Thresholds doesn't exist - set the API PPE values
+		 * according to Common Nominal Packet Padding field.
+		 */
+		} else {
+			nominal_padding =
+				u8_get_bits(link_sta->he_cap.he_cap_elem.phy_cap_info[9],
+					    IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK);
+			if (nominal_padding != IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED)
+				ret = iwl_mvm_set_pkt_ext_from_nominal_padding(pkt_ext,
+									       nominal_padding);
+		}
+	}
+
+	for (i = 0; i < MAX_HE_SUPP_NSS; i++) {
+		int bw;
+
+		for (bw = 0;
+		     bw < ARRAY_SIZE(*pkt_ext->pkt_ext_qam_th[i]);
+		     bw++) {
+			u8 *qam_th =
+				&pkt_ext->pkt_ext_qam_th[i][bw][0];
+
+			IWL_DEBUG_HT(mvm,
+				     "PPE table: nss[%d] bw[%d] PPET8 = %d, PPET16 = %d\n",
+				     i, bw, qam_th[0], qam_th[1]);
+		}
+	}
+	return ret;
+}
+
+/*
+ * This function sets the MU EDCA parameters ans returns whether MU EDCA
+ * is enabled or not
+ */
+bool iwl_mvm_set_fw_mu_edca_params(struct iwl_mvm *mvm,
+				   struct iwl_mvm_vif *mvmvif,
+				   struct iwl_he_backoff_conf *trig_based_txf)
+{
+	int i;
+	/* Mark MU EDCA as enabled, unless none detected on some AC */
+	bool mu_edca_enabled = true;
+
+	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
+		struct ieee80211_he_mu_edca_param_ac_rec *mu_edca =
+			&mvmvif->deflink.queue_params[i].mu_edca_param_rec;
+		u8 ac = iwl_mvm_mac80211_ac_to_ucode_ac(i);
+
+		if (!mvmvif->deflink.queue_params[i].mu_edca) {
+			mu_edca_enabled = false;
+			break;
+		}
+
+		trig_based_txf[ac].cwmin =
+			cpu_to_le16(mu_edca->ecw_min_max & 0xf);
+		trig_based_txf[ac].cwmax =
+			cpu_to_le16((mu_edca->ecw_min_max & 0xf0) >> 4);
+		trig_based_txf[ac].aifsn =
+			cpu_to_le16(mu_edca->aifsn & 0xf);
+		trig_based_txf[ac].mu_time =
+			cpu_to_le16(mu_edca->mu_edca_timer);
+	}
+
+	return mu_edca_enabled;
+}
+
+bool iwl_mvm_is_nic_ack_enabled(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
+{
+	const struct ieee80211_supported_band *sband;
+	const struct ieee80211_sta_he_cap *own_he_cap = NULL;
+
+	/* This capability is the same for all bands,
+	 * so take it from one of them.
+	 */
+	sband = mvm->hw->wiphy->bands[NL80211_BAND_2GHZ];
+	own_he_cap = ieee80211_get_he_iftype_cap(sband,
+						 ieee80211_vif_type_p2p(vif));
+
+	return (own_he_cap && (own_he_cap->he_cap_elem.mac_cap_info[2] &
+			       IEEE80211_HE_MAC_CAP2_ACK_EN));
+}
+
+__le32 iwl_mvm_get_sta_htc_flags(struct ieee80211_sta *sta,
+				 struct ieee80211_link_sta *link_sta)
+{
+	u8 *mac_cap_info =
+		&link_sta->he_cap.he_cap_elem.mac_cap_info[0];
+	__le32 htc_flags = 0;
+
+	if (mac_cap_info[0] & IEEE80211_HE_MAC_CAP0_HTC_HE)
+		htc_flags |= cpu_to_le32(IWL_HE_HTC_SUPPORT);
+	if ((mac_cap_info[1] & IEEE80211_HE_MAC_CAP1_LINK_ADAPTATION) ||
+	    (mac_cap_info[2] & IEEE80211_HE_MAC_CAP2_LINK_ADAPTATION)) {
+		u8 link_adap =
+			((mac_cap_info[2] &
+			  IEEE80211_HE_MAC_CAP2_LINK_ADAPTATION) << 1) +
+			 (mac_cap_info[1] &
+			  IEEE80211_HE_MAC_CAP1_LINK_ADAPTATION);
+
+		if (link_adap == 2)
+			htc_flags |=
+				cpu_to_le32(IWL_HE_HTC_LINK_ADAP_UNSOLICITED);
+		else if (link_adap == 3)
+			htc_flags |= cpu_to_le32(IWL_HE_HTC_LINK_ADAP_BOTH);
+	}
+	if (mac_cap_info[2] & IEEE80211_HE_MAC_CAP2_BSR)
+		htc_flags |= cpu_to_le32(IWL_HE_HTC_BSR_SUPP);
+	if (mac_cap_info[3] & IEEE80211_HE_MAC_CAP3_OMI_CONTROL)
+		htc_flags |= cpu_to_le32(IWL_HE_HTC_OMI_SUPP);
+	if (mac_cap_info[4] & IEEE80211_HE_MAC_CAP4_BQR)
+		htc_flags |= cpu_to_le32(IWL_HE_HTC_BQR_SUPP);
+
+	return htc_flags;
+}
+
 static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm,
 			       struct ieee80211_vif *vif, u8 sta_id)
 {
@@ -1976,11 +2344,7 @@ static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm,
 	struct ieee80211_sta *sta;
 	u32 flags;
 	int i;
-	const struct ieee80211_sta_he_cap *own_he_cap = NULL;
-	struct ieee80211_chanctx_conf *chanctx_conf;
-	const struct ieee80211_supported_band *sband;
 	void *cmd;
-	u8 nominal_padding;
 
 	if (!fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_MBSSID_HE))
 		ver = 1;
@@ -2006,16 +2370,6 @@ static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm,
 
 	rcu_read_lock();
 
-	chanctx_conf = rcu_dereference(vif->bss_conf.chanctx_conf);
-	if (WARN_ON(!chanctx_conf)) {
-		rcu_read_unlock();
-		return;
-	}
-
-	sband = mvm->hw->wiphy->bands[chanctx_conf->def.chan->band];
-	own_he_cap = ieee80211_get_he_iftype_cap(sband,
-						 ieee80211_vif_type_p2p(vif));
-
 	sta = rcu_dereference(mvm->fw_id_to_mac_id[sta_ctxt_cmd.sta_id]);
 	if (IS_ERR_OR_NULL(sta)) {
 		rcu_read_unlock();
@@ -2031,136 +2385,15 @@ static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm,
 	flags = 0;
 
 	/* Block 26-tone RU OFDMA transmissions */
-	if (mvmvif->he_ru_2mhz_block)
+	if (mvmvif->deflink.he_ru_2mhz_block)
 		flags |= STA_CTXT_HE_RU_2MHZ_BLOCK;
 
 	/* HTC flags */
-	if (sta->deflink.he_cap.he_cap_elem.mac_cap_info[0] &
-	    IEEE80211_HE_MAC_CAP0_HTC_HE)
-		sta_ctxt_cmd.htc_flags |= cpu_to_le32(IWL_HE_HTC_SUPPORT);
-	if ((sta->deflink.he_cap.he_cap_elem.mac_cap_info[1] &
-	      IEEE80211_HE_MAC_CAP1_LINK_ADAPTATION) ||
-	    (sta->deflink.he_cap.he_cap_elem.mac_cap_info[2] &
-	      IEEE80211_HE_MAC_CAP2_LINK_ADAPTATION)) {
-		u8 link_adap =
-			((sta->deflink.he_cap.he_cap_elem.mac_cap_info[2] &
-			  IEEE80211_HE_MAC_CAP2_LINK_ADAPTATION) << 1) +
-			 (sta->deflink.he_cap.he_cap_elem.mac_cap_info[1] &
-			  IEEE80211_HE_MAC_CAP1_LINK_ADAPTATION);
+	sta_ctxt_cmd.htc_flags = iwl_mvm_get_sta_htc_flags(sta, &sta->deflink);
 
-		if (link_adap == 2)
-			sta_ctxt_cmd.htc_flags |=
-				cpu_to_le32(IWL_HE_HTC_LINK_ADAP_UNSOLICITED);
-		else if (link_adap == 3)
-			sta_ctxt_cmd.htc_flags |=
-				cpu_to_le32(IWL_HE_HTC_LINK_ADAP_BOTH);
-	}
-	if (sta->deflink.he_cap.he_cap_elem.mac_cap_info[2] & IEEE80211_HE_MAC_CAP2_BSR)
-		sta_ctxt_cmd.htc_flags |= cpu_to_le32(IWL_HE_HTC_BSR_SUPP);
-	if (sta->deflink.he_cap.he_cap_elem.mac_cap_info[3] &
-	    IEEE80211_HE_MAC_CAP3_OMI_CONTROL)
-		sta_ctxt_cmd.htc_flags |= cpu_to_le32(IWL_HE_HTC_OMI_SUPP);
-	if (sta->deflink.he_cap.he_cap_elem.mac_cap_info[4] & IEEE80211_HE_MAC_CAP4_BQR)
-		sta_ctxt_cmd.htc_flags |= cpu_to_le32(IWL_HE_HTC_BQR_SUPP);
-
-	/*
-	 * Initialize the PPE thresholds to "None" (7), as described in Table
-	 * 9-262ac of 80211.ax/D3.0.
-	 */
-	memset(&sta_ctxt_cmd.pkt_ext, IWL_HE_PKT_EXT_NONE,
-	       sizeof(sta_ctxt_cmd.pkt_ext));
-
-	if (sta->deflink.eht_cap.has_eht) {
-		nominal_padding =
-			u8_get_bits(sta->deflink.eht_cap.eht_cap_elem.phy_cap_info[5],
-				    IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK);
-
-		/* If PPE Thresholds exists, parse them into a FW-familiar format. */
-		if (sta->deflink.eht_cap.eht_cap_elem.phy_cap_info[5] &
-		    IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT) {
-			u8 nss = (sta->deflink.eht_cap.eht_ppe_thres[0] &
-				IEEE80211_EHT_PPE_THRES_NSS_MASK) + 1;
-			u8 *ppe = &sta->deflink.eht_cap.eht_ppe_thres[0];
-			u8 ru_index_bitmap =
-				u16_get_bits(*ppe,
-					     IEEE80211_EHT_PPE_THRES_RU_INDEX_BITMASK_MASK);
-			 /* Starting after PPE header */
-			u8 ppe_pos_bit = IEEE80211_EHT_PPE_THRES_INFO_HEADER_SIZE;
-
-			iwl_mvm_parse_ppe(mvm,
-					  &sta_ctxt_cmd.pkt_ext,
-					  nss, ru_index_bitmap, ppe,
-					  ppe_pos_bit, true);
-			flags |= STA_CTXT_HE_PACKET_EXT;
-		/* EHT PPE Thresholds doesn't exist - set the API according to HE PPE Tresholds*/
-		} else if (sta->deflink.he_cap.he_cap_elem.phy_cap_info[6] &
-			   IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
-			struct iwl_he_pkt_ext_v2 *pkt_ext =
-				&sta_ctxt_cmd.pkt_ext;
-
-			/*
-			* Even though HE Capabilities IE doesn't contain PPE
-			* Thresholds for BW 320Mhz, thresholds for this BW will
-			* be filled in with the same values as 160Mhz, due to
-			* the inheritance, as required.
-			*/
-			iwl_mvm_set_pkt_ext_from_he_ppe(mvm, sta, pkt_ext,
-							true);
-
-			/*
-			* According to the requirements, for MCSs 12-13 the maximum value between
-			* HE PPE Threshold and Common Nominal Packet Padding needs to be taken
-			*/
-			iwl_mvm_get_optimal_ppe_info(pkt_ext, nominal_padding);
-
-			flags |= STA_CTXT_HE_PACKET_EXT;
-
-		/*
-		* if PPE Thresholds doesn't present in both EHT IE and HE IE -
-		* take the Thresholds from Common Nominal Packet Padding field
-		*/
-		} else {
-			iwl_mvm_set_pkt_ext_from_nominal_padding(&sta_ctxt_cmd.pkt_ext,
-								 nominal_padding,
-								 &flags);
-		}
-	} else if (sta->deflink.he_cap.has_he) {
-		/* If PPE Thresholds exist, parse them into a FW-familiar format. */
-		if (sta->deflink.he_cap.he_cap_elem.phy_cap_info[6] &
-			IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
-			iwl_mvm_set_pkt_ext_from_he_ppe(mvm, sta,
-							&sta_ctxt_cmd.pkt_ext,
-							false);
-			flags |= STA_CTXT_HE_PACKET_EXT;
-		/*
-		* PPE Thresholds doesn't exist - set the API PPE values
-		* according to Common Nominal Packet Padding field.
-		*/
-		} else {
-			nominal_padding =
-				u8_get_bits(sta->deflink.he_cap.he_cap_elem.phy_cap_info[9],
-					    IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK);
-			if (nominal_padding != IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED)
-				iwl_mvm_set_pkt_ext_from_nominal_padding(&sta_ctxt_cmd.pkt_ext,
-									 nominal_padding,
-									 &flags);
-		}
-	}
-
-	for (i = 0; i < MAX_HE_SUPP_NSS; i++) {
-		int bw;
-
-		for (bw = 0;
-		     bw < ARRAY_SIZE(sta_ctxt_cmd.pkt_ext.pkt_ext_qam_th[i]);
-		     bw++) {
-			u8 *qam_th =
-				&sta_ctxt_cmd.pkt_ext.pkt_ext_qam_th[i][bw][0];
-
-			IWL_DEBUG_HT(mvm,
-				     "PPE table: nss[%d] bw[%d] PPET8 = %d, PPET16 = %d\n",
-				     i, bw, qam_th[0], qam_th[1]);
-		}
-	}
+	/* PPE Thresholds */
+	if (!iwl_mvm_set_sta_pkt_ext(mvm, &sta->deflink, &sta_ctxt_cmd.pkt_ext))
+		flags |= STA_CTXT_HE_PACKET_EXT;
 
 	if (sta->deflink.he_cap.he_cap_elem.mac_cap_info[2] &
 	    IEEE80211_HE_MAC_CAP2_32BIT_BA_BITMAP)
@@ -2172,28 +2405,9 @@ static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm,
 
 	rcu_read_unlock();
 
-	/* Mark MU EDCA as enabled, unless none detected on some AC */
-	flags |= STA_CTXT_HE_MU_EDCA_CW;
-	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
-		struct ieee80211_he_mu_edca_param_ac_rec *mu_edca =
-			&mvmvif->queue_params[i].mu_edca_param_rec;
-		u8 ac = iwl_mvm_mac80211_ac_to_ucode_ac(i);
-
-		if (!mvmvif->queue_params[i].mu_edca) {
-			flags &= ~STA_CTXT_HE_MU_EDCA_CW;
-			break;
-		}
-
-		sta_ctxt_cmd.trig_based_txf[ac].cwmin =
-			cpu_to_le16(mu_edca->ecw_min_max & 0xf);
-		sta_ctxt_cmd.trig_based_txf[ac].cwmax =
-			cpu_to_le16((mu_edca->ecw_min_max & 0xf0) >> 4);
-		sta_ctxt_cmd.trig_based_txf[ac].aifsn =
-			cpu_to_le16(mu_edca->aifsn);
-		sta_ctxt_cmd.trig_based_txf[ac].mu_time =
-			cpu_to_le16(mu_edca->mu_edca_timer);
-	}
-
+	if (iwl_mvm_set_fw_mu_edca_params(mvm, mvmvif,
+					  &sta_ctxt_cmd.trig_based_txf[0]))
+		flags |= STA_CTXT_HE_MU_EDCA_CW;
 
 	if (vif->bss_conf.uora_exists) {
 		flags |= STA_CTXT_HE_TRIG_RND_ALLOC;
@@ -2204,8 +2418,7 @@ static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm,
 			(vif->bss_conf.uora_ocw_range >> 3) & 0x7;
 	}
 
-	if (own_he_cap && !(own_he_cap->he_cap_elem.mac_cap_info[2] &
-			    IEEE80211_HE_MAC_CAP2_ACK_EN))
+	if (!iwl_mvm_is_nic_ack_enabled(mvm, vif))
 		flags |= STA_CTXT_HE_NIC_NOT_ACK_ENABLED;
 
 	if (vif->bss_conf.nontransmitted) {
@@ -2265,9 +2478,8 @@ static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm,
 		IWL_ERR(mvm, "Failed to config FW to work HE!\n");
 }
 
-static void iwl_mvm_protect_assoc(struct iwl_mvm *mvm,
-				  struct ieee80211_vif *vif,
-				  u32 duration_override)
+void iwl_mvm_protect_assoc(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			   u32 duration_override)
 {
 	u32 duration = IWL_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS;
 	u32 min_duration = IWL_MVM_TE_SESSION_PROTECTION_MIN_TIME_MS;
@@ -2293,6 +2505,87 @@ static void iwl_mvm_protect_assoc(struct iwl_mvm *mvm,
 					min_duration, 500, false);
 }
 
+/* Handle association common part to MLD and non-MLD modes */
+void iwl_mvm_bss_info_changed_station_assoc(struct iwl_mvm *mvm,
+					    struct ieee80211_vif *vif,
+					    u64 changes)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int ret;
+
+	/* The firmware tracks the MU-MIMO group on its own.
+	 * However, on HW restart we should restore this data.
+	 */
+	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status) &&
+	    (changes & BSS_CHANGED_MU_GROUPS) && vif->bss_conf.mu_mimo_owner) {
+		ret = iwl_mvm_update_mu_groups(mvm, vif);
+		if (ret)
+			IWL_ERR(mvm,
+				"failed to update VHT MU_MIMO groups\n");
+	}
+
+	iwl_mvm_recalc_multicast(mvm);
+
+	/* reset rssi values */
+	mvmvif->bf_data.ave_beacon_signal = 0;
+
+	iwl_mvm_bt_coex_vif_change(mvm);
+	iwl_mvm_update_smps_on_active_links(mvm, vif, IWL_MVM_SMPS_REQ_TT,
+					    IEEE80211_SMPS_AUTOMATIC);
+	if (fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_UMAC_SCAN))
+		iwl_mvm_config_scan(mvm);
+}
+
+/* Execute the common part for MLD and non-MLD modes */
+void
+iwl_mvm_bss_info_changed_station_common(struct iwl_mvm *mvm,
+					struct ieee80211_vif *vif,
+					struct ieee80211_bss_conf *link_conf,
+					u64 changes)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int ret;
+
+	if (changes & BSS_CHANGED_BEACON_INFO) {
+		/* We received a beacon from the associated AP so
+		 * remove the session protection.
+		 */
+		iwl_mvm_stop_session_protection(mvm, vif);
+
+		iwl_mvm_sf_update(mvm, vif, false);
+		WARN_ON(iwl_mvm_enable_beacon_filter(mvm, vif, 0));
+	}
+
+	if (changes & (BSS_CHANGED_PS | BSS_CHANGED_P2P_PS | BSS_CHANGED_QOS |
+		       /* Send power command on every beacon change,
+			* because we may have not enabled beacon abort yet.
+			*/
+		       BSS_CHANGED_BEACON_INFO)) {
+		ret = iwl_mvm_power_update_mac(mvm);
+		if (ret)
+			IWL_ERR(mvm, "failed to update power mode\n");
+	}
+
+	if (changes & BSS_CHANGED_CQM) {
+		IWL_DEBUG_MAC80211(mvm, "cqm info_changed\n");
+		/* reset cqm events tracking */
+		mvmvif->bf_data.last_cqm_event = 0;
+		if (mvmvif->bf_data.bf_enabled) {
+			/* FIXME: need to update per link when FW API will
+			 * support it
+			 */
+			ret = iwl_mvm_enable_beacon_filter(mvm, vif, 0);
+			if (ret)
+				IWL_ERR(mvm,
+					"failed to update CQM thresholds\n");
+		}
+	}
+
+	if (changes & BSS_CHANGED_BANDWIDTH)
+		iwl_mvm_update_link_smps(vif, link_conf);
+}
+
 static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm,
 					     struct ieee80211_vif *vif,
 					     struct ieee80211_bss_conf *bss_conf,
@@ -2311,7 +2604,7 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm,
 		     !iwlwifi_mod_params.disable_11ax) ||
 		    (vif->bss_conf.eht_support &&
 		     !iwlwifi_mod_params.disable_11be))
-			iwl_mvm_cfg_he_sta(mvm, vif, mvmvif->ap_sta_id);
+			iwl_mvm_cfg_he_sta(mvm, vif, mvmvif->deflink.ap_sta_id);
 
 		iwl_mvm_mac_ctxt_recalc_tsf_id(mvm, vif);
 	}
@@ -2323,7 +2616,7 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm,
 	      !iwlwifi_mod_params.disable_11ax) ||
 	     (vif->bss_conf.eht_support &&
 	      !iwlwifi_mod_params.disable_11be)))
-		iwl_mvm_cfg_he_sta(mvm, vif, mvmvif->ap_sta_id);
+		iwl_mvm_cfg_he_sta(mvm, vif, mvmvif->deflink.ap_sta_id);
 
 	/*
 	 * If we're not associated yet, take the (new) BSSID before associating
@@ -2332,22 +2625,22 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm,
 	 * branch for disassociation below.
 	 */
 	if (changes & BSS_CHANGED_BSSID && !mvmvif->associated)
-		memcpy(mvmvif->bssid, bss_conf->bssid, ETH_ALEN);
+		memcpy(mvmvif->deflink.bssid, bss_conf->bssid, ETH_ALEN);
 
-	ret = iwl_mvm_mac_ctxt_changed(mvm, vif, false, mvmvif->bssid);
+	ret = iwl_mvm_mac_ctxt_changed(mvm, vif, false, mvmvif->deflink.bssid);
 	if (ret)
 		IWL_ERR(mvm, "failed to update MAC %pM\n", vif->addr);
 
 	/* after sending it once, adopt mac80211 data */
-	memcpy(mvmvif->bssid, bss_conf->bssid, ETH_ALEN);
+	memcpy(mvmvif->deflink.bssid, bss_conf->bssid, ETH_ALEN);
 	mvmvif->associated = vif->cfg.assoc;
 
 	if (changes & BSS_CHANGED_ASSOC) {
 		if (vif->cfg.assoc) {
 			/* clear statistics to get clean beacon counter */
 			iwl_mvm_request_statistics(mvm, true);
-			memset(&mvmvif->beacon_stats, 0,
-			       sizeof(mvmvif->beacon_stats));
+			memset(&mvmvif->deflink.beacon_stats, 0,
+			       sizeof(mvmvif->deflink.beacon_stats));
 
 			/* add quota for this interface */
 			ret = iwl_mvm_update_quotas(mvm, true, NULL);
@@ -2402,9 +2695,9 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm,
 			if (vif->p2p) {
 				iwl_mvm_update_smps(mvm, vif,
 						    IWL_MVM_SMPS_REQ_PROT,
-						    IEEE80211_SMPS_DYNAMIC);
+						    IEEE80211_SMPS_DYNAMIC, 0);
 			}
-		} else if (mvmvif->ap_sta_id != IWL_MVM_INVALID_STA) {
+		} else if (mvmvif->deflink.ap_sta_id != IWL_MVM_INVALID_STA) {
 			iwl_mvm_mei_host_disassociated(mvm);
 			/*
 			 * If update fails - SF might be running in associated
@@ -2427,19 +2720,20 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm,
 			if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART,
 				      &mvm->status)) {
 				/* first remove remaining keys */
-				iwl_mvm_sec_key_remove_ap(mvm, vif);
+				iwl_mvm_sec_key_remove_ap(mvm, vif,
+							  &mvmvif->deflink, 0);
 
 				/*
 				 * Remove AP station now that
 				 * the MAC is unassoc
 				 */
 				ret = iwl_mvm_rm_sta_id(mvm, vif,
-							mvmvif->ap_sta_id);
+							mvmvif->deflink.ap_sta_id);
 				if (ret)
 					IWL_ERR(mvm,
 						"failed to remove AP station\n");
 
-				mvmvif->ap_sta_id = IWL_MVM_INVALID_STA;
+				mvmvif->deflink.ap_sta_id = IWL_MVM_INVALID_STA;
 			}
 
 			/* remove quota for this interface */
@@ -2455,67 +2749,52 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm,
 					vif->addr);
 		}
 
-		/*
-		 * The firmware tracks the MU-MIMO group on its own.
-		 * However, on HW restart we should restore this data.
-		 */
-		if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status) &&
-		    (changes & BSS_CHANGED_MU_GROUPS) && vif->bss_conf.mu_mimo_owner) {
-			ret = iwl_mvm_update_mu_groups(mvm, vif);
-			if (ret)
-				IWL_ERR(mvm,
-					"failed to update VHT MU_MIMO groups\n");
-		}
+		iwl_mvm_bss_info_changed_station_assoc(mvm, vif, changes);
+	}
 
-		iwl_mvm_recalc_multicast(mvm);
+	iwl_mvm_bss_info_changed_station_common(mvm, vif, &vif->bss_conf,
+						changes);
+}
 
-		/* reset rssi values */
-		mvmvif->bf_data.ave_beacon_signal = 0;
+bool iwl_mvm_start_ap_ibss_common(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  int *ret)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int i;
 
-		iwl_mvm_bt_coex_vif_change(mvm);
-		iwl_mvm_update_smps(mvm, vif, IWL_MVM_SMPS_REQ_TT,
-				    IEEE80211_SMPS_AUTOMATIC);
-		if (fw_has_capa(&mvm->fw->ucode_capa,
-				IWL_UCODE_TLV_CAPA_UMAC_SCAN))
-			iwl_mvm_config_scan(mvm);
-	}
+	lockdep_assert_held(&mvm->mutex);
 
-	if (changes & BSS_CHANGED_BEACON_INFO) {
-		/*
-		 * We received a beacon from the associated AP so
-		 * remove the session protection.
-		 */
-		iwl_mvm_stop_session_protection(mvm, vif);
+	mvmvif->ap_assoc_sta_count = 0;
 
-		iwl_mvm_sf_update(mvm, vif, false);
-		WARN_ON(iwl_mvm_enable_beacon_filter(mvm, vif, 0));
-	}
+	/* must be set before quota calculations */
+	mvmvif->ap_ibss_active = true;
 
-	if (changes & (BSS_CHANGED_PS | BSS_CHANGED_P2P_PS | BSS_CHANGED_QOS |
-		       /*
-			* Send power command on every beacon change,
-			* because we may have not enabled beacon abort yet.
-			*/
-		       BSS_CHANGED_BEACON_INFO)) {
-		ret = iwl_mvm_power_update_mac(mvm);
-		if (ret)
-			IWL_ERR(mvm, "failed to update power mode\n");
+	/* send all the early keys to the device now */
+	for (i = 0; i < ARRAY_SIZE(mvmvif->ap_early_keys); i++) {
+		struct ieee80211_key_conf *key = mvmvif->ap_early_keys[i];
+
+		if (!key)
+			continue;
+
+		mvmvif->ap_early_keys[i] = NULL;
+
+		*ret = __iwl_mvm_mac_set_key(hw, SET_KEY, vif, NULL, key);
+		if (*ret)
+			return true;
 	}
 
-	if (changes & BSS_CHANGED_CQM) {
-		IWL_DEBUG_MAC80211(mvm, "cqm info_changed\n");
-		/* reset cqm events tracking */
-		mvmvif->bf_data.last_cqm_event = 0;
-		if (mvmvif->bf_data.bf_enabled) {
-			ret = iwl_mvm_enable_beacon_filter(mvm, vif, 0);
-			if (ret)
-				IWL_ERR(mvm,
-					"failed to update CQM thresholds\n");
-		}
+	if (vif->type == NL80211_IFTYPE_AP && !vif->p2p) {
+		iwl_mvm_vif_set_low_latency(mvmvif, true,
+					    LOW_LATENCY_VIF_TYPE);
+		iwl_mvm_send_low_latency_cmd(mvm, true, mvmvif->id);
 	}
 
-	if (changes & BSS_CHANGED_BANDWIDTH)
-		iwl_mvm_apply_fw_smps_request(vif);
+	/* power updated needs to be done before quotas */
+	iwl_mvm_power_update_mac(mvm);
+
+	return false;
 }
 
 static int iwl_mvm_start_ap_ibss(struct ieee80211_hw *hw,
@@ -2524,15 +2803,10 @@ static int iwl_mvm_start_ap_ibss(struct ieee80211_hw *hw,
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	int ret, i;
+	int ret;
 
 	mutex_lock(&mvm->mutex);
 
-	/* Send the beacon template */
-	ret = iwl_mvm_mac_ctxt_beacon_changed(mvm, vif);
-	if (ret)
-		goto out_unlock;
-
 	/*
 	 * Re-calculate the tsf id, as the leader-follower relations depend on
 	 * the beacon interval, which was not known when the AP interface
@@ -2541,12 +2815,31 @@ static int iwl_mvm_start_ap_ibss(struct ieee80211_hw *hw,
 	if (vif->type == NL80211_IFTYPE_AP)
 		iwl_mvm_mac_ctxt_recalc_tsf_id(mvm, vif);
 
-	mvmvif->ap_assoc_sta_count = 0;
+	/* For older devices need to send beacon template before adding mac
+	 * context. For the newer, the beacon is a resource that belongs to a
+	 * MAC, so need to send beacon template after adding the mac.
+	 */
+	if (mvm->trans->trans_cfg->device_family > IWL_DEVICE_FAMILY_22000) {
+		/* Add the mac context */
+		ret = iwl_mvm_mac_ctxt_add(mvm, vif);
+		if (ret)
+			goto out_unlock;
 
-	/* Add the mac context */
-	ret = iwl_mvm_mac_ctxt_add(mvm, vif);
-	if (ret)
-		goto out_unlock;
+		/* Send the beacon template */
+		ret = iwl_mvm_mac_ctxt_beacon_changed(mvm, vif, link_conf);
+		if (ret)
+			goto out_unlock;
+	} else {
+		/* Send the beacon template */
+		ret = iwl_mvm_mac_ctxt_beacon_changed(mvm, vif, link_conf);
+		if (ret)
+			goto out_unlock;
+
+		/* Add the mac context */
+		ret = iwl_mvm_mac_ctxt_add(mvm, vif);
+		if (ret)
+			goto out_unlock;
+	}
 
 	/* Perform the binding */
 	ret = iwl_mvm_binding_add_vif(mvm, vif);
@@ -2588,35 +2881,12 @@ static int iwl_mvm_start_ap_ibss(struct ieee80211_hw *hw,
 		}
 	}
 
-	/* must be set before quota calculations */
-	mvmvif->ap_ibss_active = true;
-
-	/* send all the early keys to the device now */
-	for (i = 0; i < ARRAY_SIZE(mvmvif->ap_early_keys); i++) {
-		struct ieee80211_key_conf *key = mvmvif->ap_early_keys[i];
-
-		if (!key)
-			continue;
-
-		mvmvif->ap_early_keys[i] = NULL;
-
-		ret = __iwl_mvm_mac_set_key(hw, SET_KEY, vif, NULL, key);
-		if (ret)
-			goto out_quota_failed;
-	}
-
-	if (vif->type == NL80211_IFTYPE_AP && !vif->p2p) {
-		iwl_mvm_vif_set_low_latency(mvmvif, true,
-					    LOW_LATENCY_VIF_TYPE);
-		iwl_mvm_send_low_latency_cmd(mvm, true, mvmvif->id);
-	}
-
-	/* power updated needs to be done before quotas */
-	iwl_mvm_power_update_mac(mvm);
+	if (iwl_mvm_start_ap_ibss_common(hw, vif, &ret))
+		goto out_failed;
 
 	ret = iwl_mvm_update_quotas(mvm, false, NULL);
 	if (ret)
-		goto out_quota_failed;
+		goto out_failed;
 
 	/* Need to update the P2P Device MAC (only GO, IBSS is single vif) */
 	if (vif->p2p && mvm->p2p_device_vif)
@@ -2632,7 +2902,7 @@ static int iwl_mvm_start_ap_ibss(struct ieee80211_hw *hw,
 
 	goto out_unlock;
 
-out_quota_failed:
+out_failed:
 	iwl_mvm_power_update_mac(mvm);
 	mvmvif->ap_ibss_active = false;
 	iwl_mvm_send_rm_bcast_sta(mvm, vif);
@@ -2659,16 +2929,15 @@ static int iwl_mvm_start_ibss(struct ieee80211_hw *hw,
 	return iwl_mvm_start_ap_ibss(hw, vif, &vif->bss_conf);
 }
 
-static void iwl_mvm_stop_ap_ibss(struct ieee80211_hw *hw,
-				 struct ieee80211_vif *vif,
-				 struct ieee80211_bss_conf *link_conf)
+/* Common part for MLD and non-MLD ops */
+void iwl_mvm_stop_ap_ibss_common(struct iwl_mvm *mvm,
+				 struct ieee80211_vif *vif)
 {
-	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-	iwl_mvm_prepare_mac_removal(mvm, vif);
+	lockdep_assert_held(&mvm->mutex);
 
-	mutex_lock(&mvm->mutex);
+	iwl_mvm_prepare_mac_removal(mvm, vif);
 
 	/* Handle AP stop while in CSA */
 	if (rcu_access_pointer(mvm->csa_vif) == vif) {
@@ -2693,6 +2962,17 @@ static void iwl_mvm_stop_ap_ibss(struct ieee80211_hw *hw,
 	}
 
 	iwl_mvm_bt_coex_vif_change(mvm);
+}
+
+static void iwl_mvm_stop_ap_ibss(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+
+	mutex_lock(&mvm->mutex);
+
+	iwl_mvm_stop_ap_ibss_common(mvm, vif);
 
 	/* Need to update the P2P Device MAC (only GO, IBSS is single vif) */
 	if (vif->p2p && mvm->p2p_device_vif)
@@ -2756,7 +3036,7 @@ iwl_mvm_bss_info_changed_ap_ibss(struct iwl_mvm *mvm,
 
 	/* Need to send a new beacon template to the FW */
 	if (changes & BSS_CHANGED_BEACON &&
-	    iwl_mvm_mac_ctxt_beacon_changed(mvm, vif))
+	    iwl_mvm_mac_ctxt_beacon_changed(mvm, vif, &vif->bss_conf))
 		IWL_WARN(mvm, "Failed updating beacon data\n");
 
 	if (changes & BSS_CHANGED_FTM_RESPONDER) {
@@ -2774,6 +3054,22 @@ static void iwl_mvm_bss_info_changed(struct ieee80211_hw *hw,
 				     struct ieee80211_bss_conf *bss_conf,
 				     u64 changes)
 {
+	struct iwl_mvm_bss_info_changed_ops callbacks = {
+		.bss_info_changed_sta = iwl_mvm_bss_info_changed_station,
+		.bss_info_changed_ap_ibss = iwl_mvm_bss_info_changed_ap_ibss,
+	};
+
+	iwl_mvm_bss_info_changed_common(hw, vif, bss_conf, &callbacks,
+					changes);
+}
+
+void
+iwl_mvm_bss_info_changed_common(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_bss_conf *bss_conf,
+				struct iwl_mvm_bss_info_changed_ops *callbacks,
+				u64 changes)
+{
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
 	mutex_lock(&mvm->mutex);
@@ -2783,11 +3079,12 @@ static void iwl_mvm_bss_info_changed(struct ieee80211_hw *hw,
 
 	switch (vif->type) {
 	case NL80211_IFTYPE_STATION:
-		iwl_mvm_bss_info_changed_station(mvm, vif, bss_conf, changes);
+		callbacks->bss_info_changed_sta(mvm, vif, bss_conf, changes);
 		break;
 	case NL80211_IFTYPE_AP:
 	case NL80211_IFTYPE_ADHOC:
-		iwl_mvm_bss_info_changed_ap_ibss(mvm, vif, bss_conf, changes);
+		callbacks->bss_info_changed_ap_ibss(mvm, vif, bss_conf,
+						    changes);
 		break;
 	case NL80211_IFTYPE_MONITOR:
 		if (changes & BSS_CHANGED_MU_GROUPS)
@@ -2807,9 +3104,8 @@ static void iwl_mvm_bss_info_changed(struct ieee80211_hw *hw,
 	mutex_unlock(&mvm->mutex);
 }
 
-static int iwl_mvm_mac_hw_scan(struct ieee80211_hw *hw,
-			       struct ieee80211_vif *vif,
-			       struct ieee80211_scan_request *hw_req)
+int iwl_mvm_mac_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			struct ieee80211_scan_request *hw_req)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int ret;
@@ -2825,8 +3121,8 @@ static int iwl_mvm_mac_hw_scan(struct ieee80211_hw *hw,
 	return ret;
 }
 
-static void iwl_mvm_mac_cancel_hw_scan(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif)
+void iwl_mvm_mac_cancel_hw_scan(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -2845,7 +3141,7 @@ static void iwl_mvm_mac_cancel_hw_scan(struct ieee80211_hw *hw,
 	mutex_unlock(&mvm->mutex);
 }
 
-static void
+void
 iwl_mvm_mac_allow_buffered_frames(struct ieee80211_hw *hw,
 				  struct ieee80211_sta *sta, u16 tids,
 				  int num_frames,
@@ -2860,7 +3156,7 @@ iwl_mvm_mac_allow_buffered_frames(struct ieee80211_hw *hw,
 					  tids, more_data, false);
 }
 
-static void
+void
 iwl_mvm_mac_release_buffered_frames(struct ieee80211_hw *hw,
 				    struct ieee80211_sta *sta, u16 tids,
 				    int num_frames,
@@ -2921,7 +3217,7 @@ static void __iwl_mvm_mac_sta_notify(struct ieee80211_hw *hw,
 		 */
 		break;
 	case STA_NOTIFY_AWAKE:
-		if (WARN_ON(mvmsta->sta_id == IWL_MVM_INVALID_STA))
+		if (WARN_ON(mvmsta->deflink.sta_id == IWL_MVM_INVALID_STA))
 			break;
 
 		if (txqs)
@@ -2934,10 +3230,8 @@ static void __iwl_mvm_mac_sta_notify(struct ieee80211_hw *hw,
 	spin_unlock_bh(&mvmsta->lock);
 }
 
-static void iwl_mvm_mac_sta_notify(struct ieee80211_hw *hw,
-				   struct ieee80211_vif *vif,
-				   enum sta_notify_cmd cmd,
-				   struct ieee80211_sta *sta)
+void iwl_mvm_mac_sta_notify(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			    enum sta_notify_cmd cmd, struct ieee80211_sta *sta)
 {
 	__iwl_mvm_mac_sta_notify(hw, cmd, sta);
 }
@@ -2995,12 +3289,13 @@ void iwl_mvm_sta_pm_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	rcu_read_unlock();
 }
 
-static void iwl_mvm_sta_pre_rcu_remove(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif,
-				       struct ieee80211_sta *sta)
+void iwl_mvm_sta_pre_rcu_remove(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_sta *sta)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	unsigned int link_id;
 
 	/*
 	 * This is called before mac80211 does RCU synchronisation,
@@ -3009,12 +3304,26 @@ static void iwl_mvm_sta_pre_rcu_remove(struct ieee80211_hw *hw,
 	 * be able to find the station this way, and we don't rely
 	 * on further RCU synchronisation after the sta_state()
 	 * callback deleted the station.
+	 * Since there's mvm->mutex here, no need to have RCU lock for
+	 * mvm_sta->link access.
 	 */
 	mutex_lock(&mvm->mutex);
-	if (sta == rcu_access_pointer(mvm->fw_id_to_mac_id[mvm_sta->sta_id]))
-		rcu_assign_pointer(mvm->fw_id_to_mac_id[mvm_sta->sta_id],
-				   ERR_PTR(-ENOENT));
+	for (link_id = 0; link_id < ARRAY_SIZE(mvm_sta->link); link_id++) {
+		struct iwl_mvm_link_sta *link_sta;
+		u32 sta_id;
 
+		if (!mvm_sta->link[link_id])
+			continue;
+
+		link_sta = rcu_dereference_protected(mvm_sta->link[link_id],
+						     lockdep_is_held(&mvm->mutex));
+		sta_id = link_sta->sta_id;
+		if (sta == rcu_access_pointer(mvm->fw_id_to_mac_id[sta_id])) {
+			RCU_INIT_POINTER(mvm->fw_id_to_mac_id[sta_id],
+					 ERR_PTR(-ENOENT));
+			RCU_INIT_POINTER(mvm->fw_id_to_link_sta[sta_id], NULL);
+		}
+	}
 	mutex_unlock(&mvm->mutex);
 }
 
@@ -3107,20 +3416,27 @@ static void iwl_mvm_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy,
 	rcu_read_unlock();
 }
 
-static void iwl_mvm_check_he_obss_narrow_bw_ru(struct ieee80211_hw *hw,
-					       struct ieee80211_vif *vif)
+static void
+iwl_mvm_check_he_obss_narrow_bw_ru(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif,
+				   unsigned int link_id,
+				   struct ieee80211_bss_conf *link_conf)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_he_obss_narrow_bw_ru_data iter_data = {
 		.tolerated = true,
 	};
 
-	if (!(vif->bss_conf.chandef.chan->flags & IEEE80211_CHAN_RADAR)) {
-		mvmvif->he_ru_2mhz_block = false;
+	if (WARN_ON_ONCE(!link_conf->chandef.chan ||
+			 !mvmvif->link[link_id]))
+		return;
+
+	if (!(link_conf->chandef.chan->flags & IEEE80211_CHAN_RADAR)) {
+		mvmvif->link[link_id]->he_ru_2mhz_block = false;
 		return;
 	}
 
-	cfg80211_bss_iter(hw->wiphy, &vif->bss_conf.chandef,
+	cfg80211_bss_iter(hw->wiphy, &link_conf->chandef,
 			  iwl_mvm_check_he_obss_narrow_bw_ru_iter,
 			  &iter_data);
 
@@ -3128,7 +3444,7 @@ static void iwl_mvm_check_he_obss_narrow_bw_ru(struct ieee80211_hw *hw,
 	 * If there is at least one AP on radar channel that cannot
 	 * tolerate 26-tone RU UL OFDMA transmissions using HE TB PPDU.
 	 */
-	mvmvif->he_ru_2mhz_block = !iter_data.tolerated;
+	mvmvif->link[link_id]->he_ru_2mhz_block = !iter_data.tolerated;
 }
 
 static void iwl_mvm_reset_cca_40mhz_workaround(struct iwl_mvm *mvm,
@@ -3172,7 +3488,6 @@ static void iwl_mvm_mei_host_associated(struct iwl_mvm *mvm,
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mei_conn_info conn_info = {
 		.ssid_len = vif->cfg.ssid_len,
-		.channel = vif->bss_conf.chandef.chan->hw_value,
 	};
 
 	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
@@ -3181,6 +3496,12 @@ static void iwl_mvm_mei_host_associated(struct iwl_mvm *mvm,
 	if (!mvm->mei_registered)
 		return;
 
+	/* FIXME: MEI needs to be updated for MLO */
+	if (!vif->bss_conf.chandef.chan)
+		return;
+
+	conn_info.channel = vif->bss_conf.chandef.chan->hw_value;
+
 	switch (mvm_sta->pairwise_cipher) {
 	case WLAN_CIPHER_SUITE_TKIP:
 		conn_info.pairwise_cipher = IWL_MEI_CIPHER_TKIP;
@@ -3230,24 +3551,317 @@ static void iwl_mvm_mei_host_associated(struct iwl_mvm *mvm,
 #endif
 }
 
+static int iwl_mvm_mac_ctxt_changed_wrapper(struct iwl_mvm *mvm,
+					    struct ieee80211_vif *vif,
+					    bool force_assoc_off)
+{
+	return iwl_mvm_mac_ctxt_changed(mvm, vif, force_assoc_off, NULL);
+}
+
 static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 				 struct ieee80211_vif *vif,
 				 struct ieee80211_sta *sta,
 				 enum ieee80211_sta_state old_state,
 				 enum ieee80211_sta_state new_state)
 {
+	struct iwl_mvm_sta_state_ops callbacks = {
+		.add_sta = iwl_mvm_add_sta,
+		.update_sta = iwl_mvm_update_sta,
+		.rm_sta = iwl_mvm_rm_sta,
+		.mac_ctxt_changed = iwl_mvm_mac_ctxt_changed_wrapper,
+	};
+
+	return iwl_mvm_mac_sta_state_common(hw, vif, sta, old_state, new_state,
+					    &callbacks);
+}
+
+/* FIXME: temporary making two assumptions in all sta handling functions:
+ *	(1) when setting sta state, the link exists and protected
+ *	(2) if a link is valid in sta then it's valid in vif (can
+ *	use same index in the link array)
+ */
+static void iwl_mvm_rs_rate_init_all_links(struct iwl_mvm *mvm,
+					   struct ieee80211_vif *vif,
+					   struct ieee80211_sta *sta)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	unsigned int link_id;
+
+	for_each_mvm_vif_valid_link(mvmvif, link_id) {
+		struct ieee80211_bss_conf *conf =
+			link_conf_dereference_check(vif, link_id);
+		struct ieee80211_link_sta *link_sta =
+			link_sta_dereference_check(sta, link_id);
+
+		if (!conf || !link_sta || !mvmvif->link[link_id]->phy_ctxt)
+			continue;
+
+		iwl_mvm_rs_rate_init(mvm, vif, sta, conf, link_sta,
+				     mvmvif->link[link_id]->phy_ctxt->channel->band);
+	}
+}
+
+#define IWL_MVM_MIN_BEACON_INTERVAL_TU 16
+
+static bool iwl_mvm_vif_conf_from_sta(struct iwl_mvm *mvm,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_sta *sta)
+{
+	unsigned int i;
+
+	/* Beacon interval check - firmware will crash if the beacon
+	 * interval is less than 16. We can't avoid connecting at all,
+	 * so refuse the station state change, this will cause mac80211
+	 * to abandon attempts to connect to this AP, and eventually
+	 * wpa_s will blocklist the AP...
+	 */
+
+	for_each_set_bit(i, (unsigned long *)&sta->valid_links,
+			 IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_link_sta *link_sta =
+			link_sta_dereference_protected(sta, i);
+		struct ieee80211_bss_conf *link_conf =
+			link_conf_dereference_protected(vif, i);
+
+		if (!link_conf || !link_sta)
+			continue;
+
+		if (link_conf->beacon_int < IWL_MVM_MIN_BEACON_INTERVAL_TU) {
+			IWL_ERR(mvm,
+				"Beacon interval %d for AP %pM is too small\n",
+				link_conf->beacon_int, link_sta->addr);
+			return false;
+		}
+
+		link_conf->he_support = link_sta->he_cap.has_he;
+	}
+
+	return true;
+}
+
+static void iwl_mvm_vif_set_he_support(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_sta *sta,
+				       bool is_sta)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	unsigned int i;
+
+	for_each_set_bit(i, (unsigned long *)&sta->valid_links,
+			 IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_link_sta *link_sta =
+			link_sta_dereference_protected(sta, i);
+		struct ieee80211_bss_conf *link_conf =
+			link_conf_dereference_protected(vif, i);
+
+		if (!link_conf || !link_sta || !mvmvif->link[i])
+			continue;
+
+		link_conf->he_support = link_sta->he_cap.has_he;
+
+		if (is_sta) {
+			mvmvif->link[i]->he_ru_2mhz_block = false;
+			if (link_sta->he_cap.has_he)
+				iwl_mvm_check_he_obss_narrow_bw_ru(hw, vif, i,
+								   link_conf);
+		}
+	}
+}
+
+static int
+iwl_mvm_sta_state_notexist_to_none(struct iwl_mvm *mvm,
+				   struct ieee80211_vif *vif,
+				   struct ieee80211_sta *sta,
+				   struct iwl_mvm_sta_state_ops *callbacks)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	unsigned int i;
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (vif->type == NL80211_IFTYPE_STATION &&
+	    !iwl_mvm_vif_conf_from_sta(mvm, vif, sta))
+		return -EINVAL;
+
+	if (sta->tdls &&
+	    (vif->p2p ||
+	     iwl_mvm_tdls_sta_count(mvm, NULL) == IWL_MVM_TDLS_STA_COUNT ||
+	     iwl_mvm_phy_ctx_count(mvm) > 1)) {
+		IWL_DEBUG_MAC80211(mvm, "refusing TDLS sta\n");
+		return -EBUSY;
+	}
+
+	ret = callbacks->add_sta(mvm, vif, sta);
+	if (sta->tdls && ret == 0) {
+		iwl_mvm_recalc_tdls_state(mvm, vif, true);
+		iwl_mvm_tdls_check_trigger(mvm, vif, sta->addr,
+					   NL80211_TDLS_SETUP);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sta->link); i++) {
+		struct ieee80211_link_sta *link_sta;
+
+		link_sta = link_sta_dereference_protected(sta, i);
+		if (!link_sta)
+			continue;
+
+		link_sta->agg.max_rc_amsdu_len = 1;
+	}
+	ieee80211_sta_recalc_aggregates(sta);
+
+	if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls)
+		mvmvif->ap_sta = sta;
+
+	return 0;
+}
+
+static int
+iwl_mvm_sta_state_auth_to_assoc(struct ieee80211_hw *hw,
+				struct iwl_mvm *mvm,
+				struct ieee80211_vif *vif,
+				struct ieee80211_sta *sta,
+				struct iwl_mvm_sta_state_ops *callbacks)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	unsigned int i;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (vif->type == NL80211_IFTYPE_AP) {
+		iwl_mvm_vif_set_he_support(hw, vif, sta, false);
+		mvmvif->ap_assoc_sta_count++;
+		callbacks->mac_ctxt_changed(mvm, vif, false);
+
+		/* since the below is not for MLD API, it's ok to use
+		 * the default bss_conf
+		 */
+		if (!mvm->mld_api_is_used &&
+		    ((vif->bss_conf.he_support &&
+		      !iwlwifi_mod_params.disable_11ax) ||
+		    (vif->bss_conf.eht_support &&
+		     !iwlwifi_mod_params.disable_11be)))
+			iwl_mvm_cfg_he_sta(mvm, vif, mvm_sta->deflink.sta_id);
+	} else if (vif->type == NL80211_IFTYPE_STATION) {
+		iwl_mvm_vif_set_he_support(hw, vif, sta, true);
+
+		callbacks->mac_ctxt_changed(mvm, vif, false);
+
+		if (!mvm->mld_api_is_used)
+			goto out;
+
+		for_each_set_bit(i, (unsigned long *)&sta->valid_links,
+				 IEEE80211_MLD_MAX_NUM_LINKS) {
+			struct ieee80211_bss_conf *link_conf =
+				link_conf_dereference_protected(vif, i);
+
+			if (WARN_ON(!link_conf))
+				return -EINVAL;
+			if (!mvmvif->link[i])
+				continue;
+
+			iwl_mvm_link_changed(mvm, vif, link_conf,
+					     LINK_CONTEXT_MODIFY_ALL &
+					     ~LINK_CONTEXT_MODIFY_ACTIVE,
+					     true);
+		}
+	}
+
+out:
+	iwl_mvm_rs_rate_init_all_links(mvm, vif, sta);
+
+	return callbacks->update_sta(mvm, vif, sta);
+}
+
+static int
+iwl_mvm_sta_state_assoc_to_authorized(struct iwl_mvm *mvm,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_sta *sta,
+				      struct iwl_mvm_sta_state_ops *callbacks)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+
+	lockdep_assert_held(&mvm->mutex);
+
+	/* we don't support TDLS during DCM */
+	if (iwl_mvm_phy_ctx_count(mvm) > 1)
+		iwl_mvm_teardown_tdls_peers(mvm);
+
+	if (sta->tdls) {
+		iwl_mvm_tdls_check_trigger(mvm, vif, sta->addr,
+					   NL80211_TDLS_ENABLE_LINK);
+	} else {
+		/* enable beacon filtering */
+		WARN_ON(iwl_mvm_enable_beacon_filter(mvm, vif, 0));
+
+		mvmvif->authorized = 1;
+
+		callbacks->mac_ctxt_changed(mvm, vif, false);
+		iwl_mvm_mei_host_associated(mvm, vif, mvm_sta);
+	}
+
+	mvm_sta->authorized = true;
+
+	iwl_mvm_rs_rate_init_all_links(mvm, vif, sta);
+
+	return 0;
+}
+
+static int
+iwl_mvm_sta_state_authorized_to_assoc(struct iwl_mvm *mvm,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_sta *sta,
+				      struct iwl_mvm_sta_state_ops *callbacks)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	mvmsta->authorized = false;
+
+	/* once we move into assoc state, need to update rate scale to
+	 * disable using wide bandwidth
+	 */
+	iwl_mvm_rs_rate_init_all_links(mvm, vif, sta);
+
+	if (!sta->tdls) {
+		/* Set this but don't call iwl_mvm_mac_ctxt_changed()
+		 * yet to avoid sending high prio again for a little
+		 * time.
+		 */
+		mvmvif->authorized = 0;
+
+		/* disable beacon filtering */
+		ret = iwl_mvm_disable_beacon_filter(mvm, vif, 0);
+		WARN_ON(ret &&
+			!test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED,
+				  &mvm->status));
+	}
+
+	return 0;
+}
+
+/* Common part for MLD and non-MLD modes */
+int iwl_mvm_mac_sta_state_common(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_sta *sta,
+				 enum ieee80211_sta_state old_state,
+				 enum ieee80211_sta_state new_state,
+				 struct iwl_mvm_sta_state_ops *callbacks)
+{
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	unsigned int link_id;
 	int ret;
 
 	IWL_DEBUG_MAC80211(mvm, "station %pM state change %d->%d\n",
 			   sta->addr, old_state, new_state);
 
-	/* this would be a mac80211 bug ... but don't crash */
-	if (WARN_ON_ONCE(!mvmvif->phy_ctxt))
-		return test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status) ? 0 : -EINVAL;
-
 	/*
 	 * If we are in a STA removal flow and in DQA mode:
 	 *
@@ -3278,48 +3892,25 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 	}
 
 	mutex_lock(&mvm->mutex);
+
+	/* this would be a mac80211 bug ... but don't crash */
+	for_each_mvm_vif_valid_link(mvmvif, link_id) {
+		if (WARN_ON_ONCE(!mvmvif->link[link_id]->phy_ctxt)) {
+			mutex_unlock(&mvm->mutex);
+			return test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED,
+					&mvm->status) ? 0 : -EINVAL;
+		}
+	}
+
 	/* track whether or not the station is associated */
 	mvm_sta->sta_state = new_state;
 
 	if (old_state == IEEE80211_STA_NOTEXIST &&
 	    new_state == IEEE80211_STA_NONE) {
-		/*
-		 * Firmware bug - it'll crash if the beacon interval is less
-		 * than 16. We can't avoid connecting at all, so refuse the
-		 * station state change, this will cause mac80211 to abandon
-		 * attempts to connect to this AP, and eventually wpa_s will
-		 * blocklist the AP...
-		 */
-		if (vif->type == NL80211_IFTYPE_STATION &&
-		    vif->bss_conf.beacon_int < 16) {
-			IWL_ERR(mvm,
-				"AP %pM beacon interval is %d, refusing due to firmware bug!\n",
-				sta->addr, vif->bss_conf.beacon_int);
-			ret = -EINVAL;
+		ret = iwl_mvm_sta_state_notexist_to_none(mvm, vif, sta,
+							 callbacks);
+		if (ret < 0)
 			goto out_unlock;
-		}
-
-		if (vif->type == NL80211_IFTYPE_STATION)
-			vif->bss_conf.he_support = sta->deflink.he_cap.has_he;
-
-		if (sta->tdls &&
-		    (vif->p2p ||
-		     iwl_mvm_tdls_sta_count(mvm, NULL) ==
-						IWL_MVM_TDLS_STA_COUNT ||
-		     iwl_mvm_phy_ctx_count(mvm) > 1)) {
-			IWL_DEBUG_MAC80211(mvm, "refusing TDLS sta\n");
-			ret = -EBUSY;
-			goto out_unlock;
-		}
-
-		ret = iwl_mvm_add_sta(mvm, vif, sta);
-		if (sta->tdls && ret == 0) {
-			iwl_mvm_recalc_tdls_state(mvm, vif, true);
-			iwl_mvm_tdls_check_trigger(mvm, vif, sta->addr,
-						   NL80211_TDLS_SETUP);
-		}
-
-		sta->deflink.agg.max_rc_amsdu_len = 1;
 	} else if (old_state == IEEE80211_STA_NONE &&
 		   new_state == IEEE80211_STA_AUTH) {
 		/*
@@ -3331,85 +3922,21 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 		ret = 0;
 	} else if (old_state == IEEE80211_STA_AUTH &&
 		   new_state == IEEE80211_STA_ASSOC) {
-		if (vif->type == NL80211_IFTYPE_AP) {
-			vif->bss_conf.he_support = sta->deflink.he_cap.has_he;
-			mvmvif->ap_assoc_sta_count++;
-			iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL);
-			if ((vif->bss_conf.he_support &&
-			     !iwlwifi_mod_params.disable_11ax) ||
-			    (vif->bss_conf.eht_support &&
-			     !iwlwifi_mod_params.disable_11be))
-				iwl_mvm_cfg_he_sta(mvm, vif, mvm_sta->sta_id);
-		} else if (vif->type == NL80211_IFTYPE_STATION) {
-			vif->bss_conf.he_support = sta->deflink.he_cap.has_he;
-
-			mvmvif->he_ru_2mhz_block = false;
-			if (sta->deflink.he_cap.has_he)
-				iwl_mvm_check_he_obss_narrow_bw_ru(hw, vif);
-
-			iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL);
-		}
-
-		iwl_mvm_rs_rate_init(mvm, sta, mvmvif->phy_ctxt->channel->band,
-				     false);
-		ret = iwl_mvm_update_sta(mvm, vif, sta);
+		ret = iwl_mvm_sta_state_auth_to_assoc(hw, mvm, vif, sta,
+						      callbacks);
 	} else if (old_state == IEEE80211_STA_ASSOC &&
 		   new_state == IEEE80211_STA_AUTHORIZED) {
-		ret = 0;
-
-		/* we don't support TDLS during DCM */
-		if (iwl_mvm_phy_ctx_count(mvm) > 1)
-			iwl_mvm_teardown_tdls_peers(mvm);
-
-		if (sta->tdls) {
-			iwl_mvm_tdls_check_trigger(mvm, vif, sta->addr,
-						   NL80211_TDLS_ENABLE_LINK);
-		} else {
-			/* enable beacon filtering */
-			WARN_ON(iwl_mvm_enable_beacon_filter(mvm, vif, 0));
-
-			mvmvif->authorized = 1;
-
-			/*
-			 * Now that the station is authorized, i.e., keys were already
-			 * installed, need to indicate to the FW that
-			 * multicast data frames can be forwarded to the driver
-			 */
-			iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL);
-			iwl_mvm_mei_host_associated(mvm, vif, mvm_sta);
-		}
-
-		iwl_mvm_rs_rate_init(mvm, sta, mvmvif->phy_ctxt->channel->band,
-				     true);
+		ret = iwl_mvm_sta_state_assoc_to_authorized(mvm, vif, sta,
+							    callbacks);
 	} else if (old_state == IEEE80211_STA_AUTHORIZED &&
 		   new_state == IEEE80211_STA_ASSOC) {
-		/* once we move into assoc state, need to update rate scale to
-		 * disable using wide bandwidth
-		 */
-		iwl_mvm_rs_rate_init(mvm, sta, mvmvif->phy_ctxt->channel->band,
-				     false);
-		if (!sta->tdls) {
-			/* Multicast data frames are no longer allowed */
-			iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL);
-
-			/*
-			 * Set this after the above iwl_mvm_mac_ctxt_changed()
-			 * to avoid sending high prio again for a little time.
-			 */
-			mvmvif->authorized = 0;
-
-			/* disable beacon filtering */
-			ret = iwl_mvm_disable_beacon_filter(mvm, vif, 0);
-			WARN_ON(ret &&
-				!test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED,
-					  &mvm->status));
-		}
-		ret = 0;
+		ret = iwl_mvm_sta_state_authorized_to_assoc(mvm, vif, sta,
+							    callbacks);
 	} else if (old_state == IEEE80211_STA_ASSOC &&
 		   new_state == IEEE80211_STA_AUTH) {
 		if (vif->type == NL80211_IFTYPE_AP) {
 			mvmvif->ap_assoc_sta_count--;
-			iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL);
+			callbacks->mac_ctxt_changed(mvm, vif, false);
 		} else if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls)
 			iwl_mvm_stop_session_protection(mvm, vif);
 		ret = 0;
@@ -3418,9 +3945,11 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 		ret = 0;
 	} else if (old_state == IEEE80211_STA_NONE &&
 		   new_state == IEEE80211_STA_NOTEXIST) {
-		if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls)
+		if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) {
 			iwl_mvm_stop_session_protection(mvm, vif);
-		ret = iwl_mvm_rm_sta(mvm, vif, sta);
+			mvmvif->ap_sta = NULL;
+		}
+		ret = callbacks->rm_sta(mvm, vif, sta);
 		if (sta->tdls) {
 			iwl_mvm_recalc_tdls_state(mvm, vif, false);
 			iwl_mvm_tdls_check_trigger(mvm, vif, sta->addr,
@@ -3449,7 +3978,7 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 	return ret;
 }
 
-static int iwl_mvm_mac_set_rts_threshold(struct ieee80211_hw *hw, u32 value)
+int iwl_mvm_mac_set_rts_threshold(struct ieee80211_hw *hw, u32 value)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -3458,18 +3987,15 @@ static int iwl_mvm_mac_set_rts_threshold(struct ieee80211_hw *hw, u32 value)
 	return 0;
 }
 
-static void iwl_mvm_sta_rc_update(struct ieee80211_hw *hw,
-				  struct ieee80211_vif *vif,
-				  struct ieee80211_sta *sta, u32 changed)
+void iwl_mvm_sta_rc_update(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			   struct ieee80211_sta *sta, u32 changed)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
 	if (changed & (IEEE80211_RC_BW_CHANGED |
 		       IEEE80211_RC_SUPP_RATES_CHANGED |
 		       IEEE80211_RC_NSS_CHANGED))
-		iwl_mvm_rs_rate_init(mvm, sta, mvmvif->phy_ctxt->channel->band,
-				     true);
+		iwl_mvm_rs_rate_init_all_links(mvm, vif, sta);
 
 	if (vif->type == NL80211_IFTYPE_STATION &&
 	    changed & IEEE80211_RC_NSS_CHANGED)
@@ -3484,7 +4010,7 @@ static int iwl_mvm_mac_conf_tx(struct ieee80211_hw *hw,
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-	mvmvif->queue_params[ac] = *params;
+	mvmvif->deflink.queue_params[ac] = *params;
 
 	/*
 	 * No need to update right away, we'll get BSS_CHANGED_QOS
@@ -3501,9 +4027,9 @@ static int iwl_mvm_mac_conf_tx(struct ieee80211_hw *hw,
 	return 0;
 }
 
-static void iwl_mvm_mac_mgd_prepare_tx(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif,
-				       struct ieee80211_prep_tx_info *info)
+void iwl_mvm_mac_mgd_prepare_tx(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_prep_tx_info *info)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -3512,9 +4038,9 @@ static void iwl_mvm_mac_mgd_prepare_tx(struct ieee80211_hw *hw,
 	mutex_unlock(&mvm->mutex);
 }
 
-static void iwl_mvm_mac_mgd_complete_tx(struct ieee80211_hw *hw,
-					struct ieee80211_vif *vif,
-					struct ieee80211_prep_tx_info *info)
+void iwl_mvm_mac_mgd_complete_tx(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_prep_tx_info *info)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -3527,10 +4053,10 @@ static void iwl_mvm_mac_mgd_complete_tx(struct ieee80211_hw *hw,
 	mutex_unlock(&mvm->mutex);
 }
 
-static int iwl_mvm_mac_sched_scan_start(struct ieee80211_hw *hw,
-					struct ieee80211_vif *vif,
-					struct cfg80211_sched_scan_request *req,
-					struct ieee80211_scan_ies *ies)
+int iwl_mvm_mac_sched_scan_start(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct cfg80211_sched_scan_request *req,
+				 struct ieee80211_scan_ies *ies)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -3550,8 +4076,8 @@ out:
 	return ret;
 }
 
-static int iwl_mvm_mac_sched_scan_stop(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif)
+int iwl_mvm_mac_sched_scan_stop(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int ret;
@@ -3587,7 +4113,7 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_sta *mvmsta = NULL;
-	struct iwl_mvm_key_pn *ptk_pn;
+	struct iwl_mvm_key_pn *ptk_pn = NULL;
 	int keyidx = key->keyidx;
 	u32 sec_key_id = WIDE_ID(DATA_PATH_GROUP, SEC_KEY_CMD);
 	u8 sec_key_ver = iwl_fw_lookup_cmd_ver(mvm->fw, sec_key_id, 0);
@@ -3634,7 +4160,8 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 
 	switch (cmd) {
 	case SET_KEY:
-		if (keyidx == 6 || keyidx == 7)
+		if (vif->type == NL80211_IFTYPE_STATION &&
+		    (keyidx == 6 || keyidx == 7))
 			rcu_assign_pointer(mvmvif->bcn_prot.keys[keyidx - 6],
 					   key);
 
@@ -3645,10 +4172,14 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 			 * on IBSS they're per-station and because we're lazy
 			 * we don't support them for RX, so do the same.
 			 * CMAC/GMAC in AP/IBSS modes must be done in software.
+			 *
+			 * Except, of course, beacon protection - it must be
+			 * offloaded since we just set a beacon template.
 			 */
-			if (key->cipher == WLAN_CIPHER_SUITE_AES_CMAC ||
-			    key->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_128 ||
-			    key->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_256) {
+			if (keyidx < 6 &&
+			    (key->cipher == WLAN_CIPHER_SUITE_AES_CMAC ||
+			     key->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_128 ||
+			     key->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_256)) {
 				ret = -EOPNOTSUPP;
 				break;
 			}
@@ -3729,7 +4260,8 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 		if (mvmsta && key->flags & IEEE80211_KEY_FLAG_PAIRWISE)
 			mvmsta->pairwise_cipher = key->cipher;
 
-		IWL_DEBUG_MAC80211(mvm, "set hwcrypto key\n");
+		IWL_DEBUG_MAC80211(mvm, "set hwcrypto key (sta:%pM, id:%d)\n",
+				   sta ? sta->addr : NULL, key->keyidx);
 
 		if (sec_key_ver)
 			ret = iwl_mvm_sec_key_add(mvm, vif, sta, key);
@@ -3739,6 +4271,10 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 		if (ret) {
 			IWL_WARN(mvm, "set key failed\n");
 			key->hw_key_idx = STA_KEY_IDX_INVALID;
+			if (ptk_pn) {
+				RCU_INIT_POINTER(mvmsta->ptk_pn[keyidx], NULL);
+				kfree(ptk_pn);
+			}
 			/*
 			 * can't add key for RX, but we don't need it
 			 * in the device for TX so still return 0,
@@ -3753,7 +4289,8 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 
 		break;
 	case DISABLE_KEY:
-		if (keyidx == 6 || keyidx == 7)
+		if (vif->type == NL80211_IFTYPE_STATION &&
+		    (keyidx == 6 || keyidx == 7))
 			RCU_INIT_POINTER(mvmvif->bcn_prot.keys[keyidx - 6],
 					 NULL);
 
@@ -3800,11 +4337,9 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 	return ret;
 }
 
-static int iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
-			       enum set_key_cmd cmd,
-			       struct ieee80211_vif *vif,
-			       struct ieee80211_sta *sta,
-			       struct ieee80211_key_conf *key)
+int iwl_mvm_mac_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+			struct ieee80211_vif *vif, struct ieee80211_sta *sta,
+			struct ieee80211_key_conf *key)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int ret;
@@ -3816,11 +4351,11 @@ static int iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 	return ret;
 }
 
-static void iwl_mvm_mac_update_tkip_key(struct ieee80211_hw *hw,
-					struct ieee80211_vif *vif,
-					struct ieee80211_key_conf *keyconf,
-					struct ieee80211_sta *sta,
-					u32 iv32, u16 *phase1key)
+void iwl_mvm_mac_update_tkip_key(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_key_conf *keyconf,
+				 struct ieee80211_sta *sta,
+				 u32 iv32, u16 *phase1key)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -3991,18 +4526,79 @@ static int iwl_mvm_send_aux_roc_cmd(struct iwl_mvm *mvm,
 	return res;
 }
 
+static int iwl_mvm_add_aux_sta_for_hs20(struct iwl_mvm *mvm, u32 lmac_id)
+{
+	int ret = 0;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (!fw_has_capa(&mvm->fw->ucode_capa,
+			 IWL_UCODE_TLV_CAPA_HOTSPOT_SUPPORT)) {
+		IWL_ERR(mvm, "hotspot not supported\n");
+		return -EINVAL;
+	}
+
+	if (iwl_fw_lookup_cmd_ver(mvm->fw, ADD_STA, 0) >= 12) {
+		ret = iwl_mvm_add_aux_sta(mvm, lmac_id);
+		WARN(ret, "Failed to allocate aux station");
+	}
+
+	return ret;
+}
+
+static int iwl_mvm_roc_switch_binding(struct iwl_mvm *mvm,
+				      struct ieee80211_vif *vif,
+				      struct iwl_mvm_phy_ctxt *new_phy_ctxt)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int ret = 0;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	/* Unbind the P2P_DEVICE from the current PHY context,
+	 * and if the PHY context is not used remove it.
+	 */
+	ret = iwl_mvm_binding_remove_vif(mvm, vif);
+	if (WARN(ret, "Failed unbinding P2P_DEVICE\n"))
+		return ret;
+
+	iwl_mvm_phy_ctxt_unref(mvm, mvmvif->deflink.phy_ctxt);
+
+	/* Bind the P2P_DEVICE to the current PHY Context */
+	mvmvif->deflink.phy_ctxt = new_phy_ctxt;
+
+	ret = iwl_mvm_binding_add_vif(mvm, vif);
+	WARN(ret, "Failed binding P2P_DEVICE\n");
+	return ret;
+}
+
 static int iwl_mvm_roc(struct ieee80211_hw *hw,
 		       struct ieee80211_vif *vif,
 		       struct ieee80211_channel *channel,
 		       int duration,
 		       enum ieee80211_roc_type type)
 {
+	struct iwl_mvm_roc_ops ops = {
+		.add_aux_sta_for_hs20 = iwl_mvm_add_aux_sta_for_hs20,
+		.switch_phy_ctxt = iwl_mvm_roc_switch_binding,
+	};
+
+	return iwl_mvm_roc_common(hw, vif, channel, duration, type, &ops);
+}
+
+/* Execute the common part for MLD and non-MLD modes */
+int iwl_mvm_roc_common(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		       struct ieee80211_channel *channel, int duration,
+		       enum ieee80211_roc_type type,
+		       struct iwl_mvm_roc_ops *ops)
+{
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct cfg80211_chan_def chandef;
 	struct iwl_mvm_phy_ctxt *phy_ctxt;
 	bool band_change_removal;
 	int ret, i;
+	u32 lmac_id;
 
 	IWL_DEBUG_MAC80211(mvm, "enter (%d, %d, %d)\n", channel->hw_value,
 			   duration, type);
@@ -4017,25 +4613,13 @@ static int iwl_mvm_roc(struct ieee80211_hw *hw,
 
 	switch (vif->type) {
 	case NL80211_IFTYPE_STATION:
-		if (fw_has_capa(&mvm->fw->ucode_capa,
-				IWL_UCODE_TLV_CAPA_HOTSPOT_SUPPORT)) {
-			/* Use aux roc framework (HS20) */
-			if (iwl_fw_lookup_cmd_ver(mvm->fw, ADD_STA, 0) >= 12) {
-				u32 lmac_id;
-
-				lmac_id = iwl_mvm_get_lmac_id(mvm->fw,
-							      channel->band);
-				ret = iwl_mvm_add_aux_sta(mvm, lmac_id);
-				if (WARN(ret,
-					 "Failed to allocate aux station"))
-					goto out_unlock;
-			}
+		lmac_id = iwl_mvm_get_lmac_id(mvm->fw, channel->band);
+
+		/* Use aux roc framework (HS20) */
+		ret = ops->add_aux_sta_for_hs20(mvm, lmac_id);
+		if (!ret)
 			ret = iwl_mvm_send_aux_roc_cmd(mvm, channel,
 						       vif, duration);
-			goto out_unlock;
-		}
-		IWL_ERR(mvm, "hotspot not supported\n");
-		ret = -EINVAL;
 		goto out_unlock;
 	case NL80211_IFTYPE_P2P_DEVICE:
 		/* handle below */
@@ -4048,34 +4632,21 @@ static int iwl_mvm_roc(struct ieee80211_hw *hw,
 
 	for (i = 0; i < NUM_PHY_CTX; i++) {
 		phy_ctxt = &mvm->phy_ctxts[i];
-		if (phy_ctxt->ref == 0 || mvmvif->phy_ctxt == phy_ctxt)
+		if (phy_ctxt->ref == 0 || mvmvif->deflink.phy_ctxt == phy_ctxt)
 			continue;
 
 		if (phy_ctxt->ref && channel == phy_ctxt->channel) {
-			/*
-			 * Unbind the P2P_DEVICE from the current PHY context,
-			 * and if the PHY context is not used remove it.
-			 */
-			ret = iwl_mvm_binding_remove_vif(mvm, vif);
-			if (WARN(ret, "Failed unbinding P2P_DEVICE\n"))
-				goto out_unlock;
-
-			iwl_mvm_phy_ctxt_unref(mvm, mvmvif->phy_ctxt);
-
-			/* Bind the P2P_DEVICE to the current PHY Context */
-			mvmvif->phy_ctxt = phy_ctxt;
-
-			ret = iwl_mvm_binding_add_vif(mvm, vif);
-			if (WARN(ret, "Failed binding P2P_DEVICE\n"))
+			ret = ops->switch_phy_ctxt(mvm, vif, phy_ctxt);
+			if (ret)
 				goto out_unlock;
 
-			iwl_mvm_phy_ctxt_ref(mvm, mvmvif->phy_ctxt);
+			iwl_mvm_phy_ctxt_ref(mvm, mvmvif->deflink.phy_ctxt);
 			goto schedule_time_event;
 		}
 	}
 
 	/* Need to update the PHY context only if the ROC channel changed */
-	if (channel == mvmvif->phy_ctxt->channel)
+	if (channel == mvmvif->deflink.phy_ctxt->channel)
 		goto schedule_time_event;
 
 	cfg80211_chandef_create(&chandef, channel, NL80211_CHAN_NO_HT);
@@ -4089,14 +4660,14 @@ static int iwl_mvm_roc(struct ieee80211_hw *hw,
 	band_change_removal =
 		fw_has_capa(&mvm->fw->ucode_capa,
 			    IWL_UCODE_TLV_CAPA_BINDING_CDB_SUPPORT) &&
-		mvmvif->phy_ctxt->channel->band != chandef.chan->band;
+		mvmvif->deflink.phy_ctxt->channel->band != chandef.chan->band;
 
-	if (mvmvif->phy_ctxt->ref == 1 && !band_change_removal) {
+	if (mvmvif->deflink.phy_ctxt->ref == 1 && !band_change_removal) {
 		/*
 		 * Change the PHY context configuration as it is currently
 		 * referenced only by the P2P Device MAC (and we can modify it)
 		 */
-		ret = iwl_mvm_phy_ctxt_changed(mvm, mvmvif->phy_ctxt,
+		ret = iwl_mvm_phy_ctxt_changed(mvm, mvmvif->deflink.phy_ctxt,
 					       &chandef, 1, 1);
 		if (ret)
 			goto out_unlock;
@@ -4119,21 +4690,11 @@ static int iwl_mvm_roc(struct ieee80211_hw *hw,
 			goto out_unlock;
 		}
 
-		/* Unbind the P2P_DEVICE from the current PHY context */
-		ret = iwl_mvm_binding_remove_vif(mvm, vif);
-		if (WARN(ret, "Failed unbinding P2P_DEVICE\n"))
-			goto out_unlock;
-
-		iwl_mvm_phy_ctxt_unref(mvm, mvmvif->phy_ctxt);
-
-		/* Bind the P2P_DEVICE to the new allocated PHY context */
-		mvmvif->phy_ctxt = phy_ctxt;
-
-		ret = iwl_mvm_binding_add_vif(mvm, vif);
-		if (WARN(ret, "Failed binding P2P_DEVICE\n"))
+		ret = ops->switch_phy_ctxt(mvm, vif, phy_ctxt);
+		if (ret)
 			goto out_unlock;
 
-		iwl_mvm_phy_ctxt_ref(mvm, mvmvif->phy_ctxt);
+		iwl_mvm_phy_ctxt_ref(mvm, mvmvif->deflink.phy_ctxt);
 	}
 
 schedule_time_event:
@@ -4146,8 +4707,8 @@ out_unlock:
 	return ret;
 }
 
-static int iwl_mvm_cancel_roc(struct ieee80211_hw *hw,
-			      struct ieee80211_vif *vif)
+int iwl_mvm_cancel_roc(struct ieee80211_hw *hw,
+		       struct ieee80211_vif *vif)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -4224,8 +4785,8 @@ out:
 	return ret;
 }
 
-static int iwl_mvm_add_chanctx(struct ieee80211_hw *hw,
-			       struct ieee80211_chanctx_conf *ctx)
+int iwl_mvm_add_chanctx(struct ieee80211_hw *hw,
+			struct ieee80211_chanctx_conf *ctx)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int ret;
@@ -4248,8 +4809,8 @@ static void __iwl_mvm_remove_chanctx(struct iwl_mvm *mvm,
 	iwl_mvm_phy_ctxt_unref(mvm, phy_ctxt);
 }
 
-static void iwl_mvm_remove_chanctx(struct ieee80211_hw *hw,
-				   struct ieee80211_chanctx_conf *ctx)
+void iwl_mvm_remove_chanctx(struct ieee80211_hw *hw,
+			    struct ieee80211_chanctx_conf *ctx)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -4258,9 +4819,8 @@ static void iwl_mvm_remove_chanctx(struct ieee80211_hw *hw,
 	mutex_unlock(&mvm->mutex);
 }
 
-static void iwl_mvm_change_chanctx(struct ieee80211_hw *hw,
-				   struct ieee80211_chanctx_conf *ctx,
-				   u32 changed)
+void iwl_mvm_change_chanctx(struct ieee80211_hw *hw,
+			    struct ieee80211_chanctx_conf *ctx, u32 changed)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	u16 *phy_ctxt_id = (u16 *)ctx->drv_priv;
@@ -4299,19 +4859,25 @@ out_unlock:
 	mutex_unlock(&mvm->mutex);
 }
 
-static int __iwl_mvm_assign_vif_chanctx(struct iwl_mvm *mvm,
-					struct ieee80211_vif *vif,
-					struct ieee80211_chanctx_conf *ctx,
-					bool switching_chanctx)
+/*
+ * This function executes the common part for MLD and non-MLD modes.
+ *
+ * Returns true if we're done assigning the chanctx
+ * (either on failure or success)
+ */
+static bool
+__iwl_mvm_assign_vif_chanctx_common(struct iwl_mvm *mvm,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_chanctx_conf *ctx,
+				    bool switching_chanctx, int *ret)
 {
 	u16 *phy_ctxt_id = (u16 *)ctx->drv_priv;
 	struct iwl_mvm_phy_ctxt *phy_ctxt = &mvm->phy_ctxts[*phy_ctxt_id];
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	int ret;
 
 	lockdep_assert_held(&mvm->mutex);
 
-	mvmvif->phy_ctxt = phy_ctxt;
+	mvmvif->deflink.phy_ctxt = phy_ctxt;
 
 	switch (vif->type) {
 	case NL80211_IFTYPE_AP:
@@ -4326,19 +4892,36 @@ static int __iwl_mvm_assign_vif_chanctx(struct iwl_mvm *mvm,
 		 * The AP binding flow is handled as part of the start_ap flow
 		 * (in bss_info_changed), similarly for IBSS.
 		 */
-		ret = 0;
-		goto out;
+		*ret = 0;
+		return true;
 	case NL80211_IFTYPE_STATION:
-		mvmvif->csa_bcn_pending = false;
 		break;
 	case NL80211_IFTYPE_MONITOR:
 		/* always disable PS when a monitor interface is active */
 		mvmvif->ps_disabled = true;
 		break;
 	default:
-		ret = -EINVAL;
-		goto out;
+		*ret = -EINVAL;
+		return true;
 	}
+	return false;
+}
+
+static int __iwl_mvm_assign_vif_chanctx(struct iwl_mvm *mvm,
+					struct ieee80211_vif *vif,
+					struct ieee80211_bss_conf *link_conf,
+					struct ieee80211_chanctx_conf *ctx,
+					bool switching_chanctx)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int ret;
+
+	if (WARN_ON(!link_conf))
+		return -EINVAL;
+
+	if (__iwl_mvm_assign_vif_chanctx_common(mvm, vif, ctx,
+						switching_chanctx, &ret))
+		goto out;
 
 	ret = iwl_mvm_binding_add_vif(mvm, vif);
 	if (ret)
@@ -4372,7 +4955,12 @@ static int __iwl_mvm_assign_vif_chanctx(struct iwl_mvm *mvm,
 		iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL);
 	}
 
-	if (switching_chanctx && vif->type == NL80211_IFTYPE_STATION) {
+	if (vif->type == NL80211_IFTYPE_STATION) {
+		if (!switching_chanctx) {
+			mvmvif->csa_bcn_pending = false;
+			goto out;
+		}
+
 		mvmvif->csa_bcn_pending = true;
 
 		if (!fw_has_capa(&mvm->fw->ucode_capa,
@@ -4397,9 +4985,10 @@ out_remove_binding:
 	iwl_mvm_power_update_mac(mvm);
 out:
 	if (ret)
-		mvmvif->phy_ctxt = NULL;
+		mvmvif->deflink.phy_ctxt = NULL;
 	return ret;
 }
+
 static int iwl_mvm_assign_vif_chanctx(struct ieee80211_hw *hw,
 				      struct ieee80211_vif *vif,
 				      struct ieee80211_bss_conf *link_conf,
@@ -4409,35 +4998,39 @@ static int iwl_mvm_assign_vif_chanctx(struct ieee80211_hw *hw,
 	int ret;
 
 	mutex_lock(&mvm->mutex);
-	ret = __iwl_mvm_assign_vif_chanctx(mvm, vif, ctx, false);
+	ret = __iwl_mvm_assign_vif_chanctx(mvm, vif, link_conf, ctx, false);
 	mutex_unlock(&mvm->mutex);
 
 	return ret;
 }
 
-static void __iwl_mvm_unassign_vif_chanctx(struct iwl_mvm *mvm,
-					   struct ieee80211_vif *vif,
-					   struct ieee80211_chanctx_conf *ctx,
-					   bool switching_chanctx)
+/*
+ * This function executes the common part for MLD and non-MLD modes.
+ *
+ * Returns if chanctx unassign chanctx is done
+ * (either on failure or success)
+ */
+static bool __iwl_mvm_unassign_vif_chanctx_common(struct iwl_mvm *mvm,
+						  struct ieee80211_vif *vif,
+						  bool switching_chanctx)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct ieee80211_vif *disabled_vif = NULL;
 
 	lockdep_assert_held(&mvm->mutex);
-	iwl_mvm_remove_time_event(mvm, mvmvif, &mvmvif->time_event_data);
+	iwl_mvm_remove_time_event(mvm, mvmvif,
+				  &mvmvif->time_event_data);
 
 	switch (vif->type) {
 	case NL80211_IFTYPE_ADHOC:
-		goto out;
+		return true;
 	case NL80211_IFTYPE_MONITOR:
 		mvmvif->monitor_active = false;
 		mvmvif->ps_disabled = false;
-		iwl_mvm_rm_snif_sta(mvm, vif);
 		break;
 	case NL80211_IFTYPE_AP:
 		/* This part is triggered only during CSA */
 		if (!switching_chanctx || !mvmvif->ap_ibss_active)
-			goto out;
+			return true;
 
 		mvmvif->csa_countdown = false;
 
@@ -4449,18 +5042,33 @@ static void __iwl_mvm_unassign_vif_chanctx(struct iwl_mvm *mvm,
 
 		mvmvif->ap_ibss_active = false;
 		break;
-	case NL80211_IFTYPE_STATION:
-		if (!switching_chanctx)
-			break;
+	default:
+		break;
+	}
+	return false;
+}
 
-		disabled_vif = vif;
+static void __iwl_mvm_unassign_vif_chanctx(struct iwl_mvm *mvm,
+					   struct ieee80211_vif *vif,
+					   struct ieee80211_bss_conf *link_conf,
+					   struct ieee80211_chanctx_conf *ctx,
+					   bool switching_chanctx)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct ieee80211_vif *disabled_vif = NULL;
+
+	if (__iwl_mvm_unassign_vif_chanctx_common(mvm, vif, switching_chanctx))
+		goto out;
 
+	if (vif->type == NL80211_IFTYPE_MONITOR)
+		iwl_mvm_rm_snif_sta(mvm, vif);
+
+
+	if (vif->type == NL80211_IFTYPE_STATION && switching_chanctx) {
+		disabled_vif = vif;
 		if (!fw_has_capa(&mvm->fw->ucode_capa,
 				 IWL_UCODE_TLV_CAPA_CHANNEL_SWITCH_CMD))
 			iwl_mvm_mac_ctxt_changed(mvm, vif, true, NULL);
-		break;
-	default:
-		break;
 	}
 
 	iwl_mvm_update_quotas(mvm, false, disabled_vif);
@@ -4470,7 +5078,7 @@ out:
 	if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_CHANNEL_SWITCH_CMD) &&
 	    switching_chanctx)
 		return;
-	mvmvif->phy_ctxt = NULL;
+	mvmvif->deflink.phy_ctxt = NULL;
 	iwl_mvm_power_update_mac(mvm);
 }
 
@@ -4482,18 +5090,20 @@ static void iwl_mvm_unassign_vif_chanctx(struct ieee80211_hw *hw,
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
 	mutex_lock(&mvm->mutex);
-	__iwl_mvm_unassign_vif_chanctx(mvm, vif, ctx, false);
+	__iwl_mvm_unassign_vif_chanctx(mvm, vif, link_conf, ctx, false);
 	mutex_unlock(&mvm->mutex);
 }
 
 static int
 iwl_mvm_switch_vif_chanctx_swap(struct iwl_mvm *mvm,
-				struct ieee80211_vif_chanctx_switch *vifs)
+				struct ieee80211_vif_chanctx_switch *vifs,
+				struct iwl_mvm_switch_vif_chanctx_ops *ops)
 {
 	int ret;
 
 	mutex_lock(&mvm->mutex);
-	__iwl_mvm_unassign_vif_chanctx(mvm, vifs[0].vif, vifs[0].old_ctx, true);
+	ops->__unassign_vif_chanctx(mvm, vifs[0].vif, vifs[0].link_conf,
+				    vifs[0].old_ctx, true);
 	__iwl_mvm_remove_chanctx(mvm, vifs[0].old_ctx);
 
 	ret = __iwl_mvm_add_chanctx(mvm, vifs[0].new_ctx);
@@ -4502,8 +5112,8 @@ iwl_mvm_switch_vif_chanctx_swap(struct iwl_mvm *mvm,
 		goto out_reassign;
 	}
 
-	ret = __iwl_mvm_assign_vif_chanctx(mvm, vifs[0].vif, vifs[0].new_ctx,
-					   true);
+	ret = ops->__assign_vif_chanctx(mvm, vifs[0].vif, vifs[0].link_conf,
+					vifs[0].new_ctx, true);
 	if (ret) {
 		IWL_ERR(mvm,
 			"failed to assign new_ctx during channel switch\n");
@@ -4525,8 +5135,8 @@ out_reassign:
 		goto out_restart;
 	}
 
-	if (__iwl_mvm_assign_vif_chanctx(mvm, vifs[0].vif, vifs[0].old_ctx,
-					 true)) {
+	if (ops->__assign_vif_chanctx(mvm, vifs[0].vif, vifs[0].link_conf,
+				      vifs[0].old_ctx, true)) {
 		IWL_ERR(mvm, "failed to reassign old_ctx after failure.\n");
 		goto out_restart;
 	}
@@ -4545,15 +5155,17 @@ out:
 
 static int
 iwl_mvm_switch_vif_chanctx_reassign(struct iwl_mvm *mvm,
-				    struct ieee80211_vif_chanctx_switch *vifs)
+				    struct ieee80211_vif_chanctx_switch *vifs,
+				    struct iwl_mvm_switch_vif_chanctx_ops *ops)
 {
 	int ret;
 
 	mutex_lock(&mvm->mutex);
-	__iwl_mvm_unassign_vif_chanctx(mvm, vifs[0].vif, vifs[0].old_ctx, true);
+	ops->__unassign_vif_chanctx(mvm, vifs[0].vif, vifs[0].link_conf,
+				    vifs[0].old_ctx, true);
 
-	ret = __iwl_mvm_assign_vif_chanctx(mvm, vifs[0].vif, vifs[0].new_ctx,
-					   true);
+	ret = ops->__assign_vif_chanctx(mvm, vifs[0].vif, vifs[0].link_conf,
+					vifs[0].new_ctx, true);
 	if (ret) {
 		IWL_ERR(mvm,
 			"failed to assign new_ctx during channel switch\n");
@@ -4563,8 +5175,8 @@ iwl_mvm_switch_vif_chanctx_reassign(struct iwl_mvm *mvm,
 	goto out;
 
 out_reassign:
-	if (__iwl_mvm_assign_vif_chanctx(mvm, vifs[0].vif, vifs[0].old_ctx,
-					 true)) {
+	if (ops->__assign_vif_chanctx(mvm, vifs[0].vif, vifs[0].link_conf,
+				      vifs[0].old_ctx, true)) {
 		IWL_ERR(mvm, "failed to reassign old_ctx after failure.\n");
 		goto out_restart;
 	}
@@ -4581,10 +5193,13 @@ out:
 	return ret;
 }
 
-static int iwl_mvm_switch_vif_chanctx(struct ieee80211_hw *hw,
-				      struct ieee80211_vif_chanctx_switch *vifs,
-				      int n_vifs,
-				      enum ieee80211_chanctx_switch_mode mode)
+/* Execute the common part for both MLD and non-MLD modes */
+int
+iwl_mvm_switch_vif_chanctx_common(struct ieee80211_hw *hw,
+				  struct ieee80211_vif_chanctx_switch *vifs,
+				  int n_vifs,
+				  enum ieee80211_chanctx_switch_mode mode,
+				  struct iwl_mvm_switch_vif_chanctx_ops *ops)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int ret;
@@ -4595,10 +5210,10 @@ static int iwl_mvm_switch_vif_chanctx(struct ieee80211_hw *hw,
 
 	switch (mode) {
 	case CHANCTX_SWMODE_SWAP_CONTEXTS:
-		ret = iwl_mvm_switch_vif_chanctx_swap(mvm, vifs);
+		ret = iwl_mvm_switch_vif_chanctx_swap(mvm, vifs, ops);
 		break;
 	case CHANCTX_SWMODE_REASSIGN_VIF:
-		ret = iwl_mvm_switch_vif_chanctx_reassign(mvm, vifs);
+		ret = iwl_mvm_switch_vif_chanctx_reassign(mvm, vifs, ops);
 		break;
 	default:
 		ret = -EOPNOTSUPP;
@@ -4608,16 +5223,28 @@ static int iwl_mvm_switch_vif_chanctx(struct ieee80211_hw *hw,
 	return ret;
 }
 
-static int iwl_mvm_tx_last_beacon(struct ieee80211_hw *hw)
+static int iwl_mvm_switch_vif_chanctx(struct ieee80211_hw *hw,
+				      struct ieee80211_vif_chanctx_switch *vifs,
+				      int n_vifs,
+				      enum ieee80211_chanctx_switch_mode mode)
+{
+	struct iwl_mvm_switch_vif_chanctx_ops ops = {
+		.__assign_vif_chanctx = __iwl_mvm_assign_vif_chanctx,
+		.__unassign_vif_chanctx = __iwl_mvm_unassign_vif_chanctx,
+	};
+
+	return iwl_mvm_switch_vif_chanctx_common(hw, vifs, n_vifs, mode, &ops);
+}
+
+int iwl_mvm_tx_last_beacon(struct ieee80211_hw *hw)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
 	return mvm->ibss_manager;
 }
 
-static int iwl_mvm_set_tim(struct ieee80211_hw *hw,
-			   struct ieee80211_sta *sta,
-			   bool set)
+int iwl_mvm_set_tim(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
+		    bool set)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
@@ -4627,7 +5254,8 @@ static int iwl_mvm_set_tim(struct ieee80211_hw *hw,
 		return -EINVAL;
 	}
 
-	return iwl_mvm_mac_ctxt_beacon_changed(mvm, mvm_sta->vif);
+	return iwl_mvm_mac_ctxt_beacon_changed(mvm, mvm_sta->vif,
+					       &mvm_sta->vif->bss_conf);
 }
 
 #ifdef CONFIG_NL80211_TESTMODE
@@ -4683,9 +5311,9 @@ static int __iwl_mvm_mac_testmode_cmd(struct iwl_mvm *mvm,
 	return -EOPNOTSUPP;
 }
 
-static int iwl_mvm_mac_testmode_cmd(struct ieee80211_hw *hw,
-				    struct ieee80211_vif *vif,
-				    void *data, int len)
+int iwl_mvm_mac_testmode_cmd(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     void *data, int len)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int err;
@@ -4698,9 +5326,8 @@ static int iwl_mvm_mac_testmode_cmd(struct ieee80211_hw *hw,
 }
 #endif
 
-static void iwl_mvm_channel_switch(struct ieee80211_hw *hw,
-				   struct ieee80211_vif *vif,
-				   struct ieee80211_channel_switch *chsw)
+void iwl_mvm_channel_switch(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			    struct ieee80211_channel_switch *chsw)
 {
 	/* By implementing this operation, we prevent mac80211 from
 	 * starting its own channel switch timer, so that we can call
@@ -4775,9 +5402,9 @@ static int iwl_mvm_old_pre_chan_sw_sta(struct iwl_mvm *mvm,
 }
 
 #define IWL_MAX_CSA_BLOCK_TX 1500
-static int iwl_mvm_pre_channel_switch(struct ieee80211_hw *hw,
-				      struct ieee80211_vif *vif,
-				      struct ieee80211_channel_switch *chsw)
+int iwl_mvm_pre_channel_switch(struct ieee80211_hw *hw,
+			       struct ieee80211_vif *vif,
+			       struct ieee80211_channel_switch *chsw)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct ieee80211_vif *csa_vif;
@@ -4890,9 +5517,9 @@ out_unlock:
 	return ret;
 }
 
-static void iwl_mvm_channel_switch_rx_beacon(struct ieee80211_hw *hw,
-					     struct ieee80211_vif *vif,
-					     struct ieee80211_channel_switch *chsw)
+void iwl_mvm_channel_switch_rx_beacon(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_channel_switch *chsw)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
@@ -4977,13 +5604,14 @@ static void iwl_mvm_flush_no_vif(struct iwl_mvm *mvm, u32 queues, bool drop)
 	mutex_unlock(&mvm->mutex);
 }
 
-static void iwl_mvm_mac_flush(struct ieee80211_hw *hw,
-			      struct ieee80211_vif *vif, u32 queues, bool drop)
+void iwl_mvm_mac_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		       u32 queues, bool drop)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif;
 	struct iwl_mvm_sta *mvmsta;
 	struct ieee80211_sta *sta;
+	bool ap_sta_done = false;
 	int i;
 	u32 msk = 0;
 
@@ -5012,16 +5640,23 @@ static void iwl_mvm_mac_flush(struct ieee80211_hw *hw,
 		if (mvmsta->vif != vif)
 			continue;
 
+		if (sta == mvmvif->ap_sta) {
+			if (ap_sta_done)
+				continue;
+			ap_sta_done = true;
+		}
+
 		/* make sure only TDLS peers or the AP are flushed */
-		WARN_ON(i != mvmvif->ap_sta_id && !sta->tdls);
+		WARN_ON_ONCE(sta != mvmvif->ap_sta && !sta->tdls);
 
 		if (drop) {
 			if (iwl_mvm_flush_sta(mvm, mvmsta, false))
 				IWL_ERR(mvm, "flush request fail\n");
 		} else {
-			msk |= mvmsta->tfd_queue_msk;
 			if (iwl_mvm_has_new_tx_api(mvm))
 				iwl_mvm_wait_sta_queues_empty(mvm, mvmsta);
+			else /* only used for !iwl_mvm_has_new_tx_api() below */
+				msk |= mvmsta->tfd_queue_msk;
 		}
 	}
 
@@ -5034,8 +5669,8 @@ static void iwl_mvm_mac_flush(struct ieee80211_hw *hw,
 		iwl_trans_wait_tx_queues_empty(mvm->trans, msk);
 }
 
-static int iwl_mvm_mac_get_survey(struct ieee80211_hw *hw, int idx,
-				  struct survey_info *survey)
+int iwl_mvm_mac_get_survey(struct ieee80211_hw *hw, int idx,
+			   struct survey_info *survey)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int ret;
@@ -5219,22 +5854,22 @@ static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo)
 	}
 }
 
-static void iwl_mvm_mac_sta_statistics(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif,
-				       struct ieee80211_sta *sta,
-				       struct station_info *sinfo)
+void iwl_mvm_mac_sta_statistics(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_sta *sta,
+				struct station_info *sinfo)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 
-	if (mvmsta->avg_energy) {
-		sinfo->signal_avg = -(s8)mvmsta->avg_energy;
+	if (mvmsta->deflink.avg_energy) {
+		sinfo->signal_avg = -(s8)mvmsta->deflink.avg_energy;
 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG);
 	}
 
 	if (iwl_mvm_has_tlc_offload(mvm)) {
-		struct iwl_lq_sta_rs_fw *lq_sta = &mvmsta->lq_sta.rs_fw;
+		struct iwl_lq_sta_rs_fw *lq_sta = &mvmsta->deflink.lq_sta.rs_fw;
 
 		iwl_mvm_set_sta_rate(lq_sta->last_rate_n_flags, &sinfo->txrate);
 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
@@ -5249,18 +5884,19 @@ static void iwl_mvm_mac_sta_statistics(struct ieee80211_hw *hw,
 
 	mutex_lock(&mvm->mutex);
 
-	if (mvmvif->ap_sta_id != mvmsta->sta_id)
+	if (mvmvif->deflink.ap_sta_id != mvmsta->deflink.sta_id)
 		goto unlock;
 
 	if (iwl_mvm_request_statistics(mvm, false))
 		goto unlock;
 
-	sinfo->rx_beacon = mvmvif->beacon_stats.num_beacons +
-			   mvmvif->beacon_stats.accu_num_beacons;
+	sinfo->rx_beacon = mvmvif->deflink.beacon_stats.num_beacons +
+			   mvmvif->deflink.beacon_stats.accu_num_beacons;
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_BEACON_RX);
-	if (mvmvif->beacon_stats.avg_signal) {
+	if (mvmvif->deflink.beacon_stats.avg_signal) {
 		/* firmware only reports a value after RXing a few beacons */
-		sinfo->rx_beacon_signal_avg = mvmvif->beacon_stats.avg_signal;
+		sinfo->rx_beacon_signal_avg =
+			mvmvif->deflink.beacon_stats.avg_signal;
 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_BEACON_SIGNAL_AVG);
 	}
  unlock:
@@ -5362,9 +5998,9 @@ static void iwl_mvm_event_bar_rx_callback(struct iwl_mvm *mvm,
 				event->u.ba.ssn);
 }
 
-static void iwl_mvm_mac_event_callback(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif,
-				       const struct ieee80211_event *event)
+void iwl_mvm_mac_event_callback(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				const struct ieee80211_event *event)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -5446,7 +6082,7 @@ out:
 	}
 }
 
-static void iwl_mvm_sync_rx_queues(struct ieee80211_hw *hw)
+void iwl_mvm_sync_rx_queues(struct ieee80211_hw *hw)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -5455,7 +6091,7 @@ static void iwl_mvm_sync_rx_queues(struct ieee80211_hw *hw)
 	mutex_unlock(&mvm->mutex);
 }
 
-static int
+int
 iwl_mvm_mac_get_ftm_responder_stats(struct ieee80211_hw *hw,
 				    struct ieee80211_vif *vif,
 				    struct cfg80211_ftm_responder_stats *stats)
@@ -5484,9 +6120,8 @@ iwl_mvm_mac_get_ftm_responder_stats(struct ieee80211_hw *hw,
 	return 0;
 }
 
-static int iwl_mvm_start_pmsr(struct ieee80211_hw *hw,
-			      struct ieee80211_vif *vif,
-			      struct cfg80211_pmsr_request *request)
+int iwl_mvm_start_pmsr(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		       struct cfg80211_pmsr_request *request)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	int ret;
@@ -5498,9 +6133,8 @@ static int iwl_mvm_start_pmsr(struct ieee80211_hw *hw,
 	return ret;
 }
 
-static void iwl_mvm_abort_pmsr(struct ieee80211_hw *hw,
-			       struct ieee80211_vif *vif,
-			       struct cfg80211_pmsr_request *request)
+void iwl_mvm_abort_pmsr(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			struct cfg80211_pmsr_request *request)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
@@ -5525,7 +6159,7 @@ static bool iwl_mvm_mac_can_aggregate(struct ieee80211_hw *hw,
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 
-	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+	if (iwl_mvm_has_new_tx_csum(mvm))
 		return iwl_mvm_tx_csum_bz(mvm, head, true) ==
 		       iwl_mvm_tx_csum_bz(mvm, skb, true);
 
@@ -5539,6 +6173,29 @@ static bool iwl_mvm_mac_can_aggregate(struct ieee80211_hw *hw,
 	return iwl_mvm_can_hw_csum(skb) == iwl_mvm_can_hw_csum(head);
 }
 
+int iwl_mvm_set_hw_timestamp(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct cfg80211_set_hw_timestamp *hwts)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	u32 protocols = 0;
+	int ret;
+
+	/* HW timestamping is only supported for a specific station */
+	if (!hwts->macaddr)
+		return -EOPNOTSUPP;
+
+	if (hwts->enable)
+		protocols =
+			IWL_TIME_SYNC_PROTOCOL_TM | IWL_TIME_SYNC_PROTOCOL_FTM;
+
+	mutex_lock(&mvm->mutex);
+	ret = iwl_mvm_time_sync_config(mvm, hwts->macaddr, protocols);
+	mutex_unlock(&mvm->mutex);
+
+	return ret;
+}
+
 const struct ieee80211_ops iwl_mvm_hw_ops = {
 	.tx = iwl_mvm_mac_tx,
 	.wake_tx_queue = iwl_mvm_mac_wake_tx_queue,
@@ -5627,4 +6284,5 @@ const struct ieee80211_ops iwl_mvm_hw_ops = {
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 	.sta_add_debugfs = iwl_mvm_sta_add_debugfs,
 #endif
+	.set_hw_timestamp = iwl_mvm_set_hw_timestamp,
 };
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mld-key.c b/drivers/net/wireless/intel/iwlwifi/mvm/mld-key.c
index e27c893502f7..8853821b3716 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mld-key.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mld-key.c
@@ -14,23 +14,41 @@ static u32 iwl_mvm_get_sec_sta_mask(struct iwl_mvm *mvm,
 				    struct ieee80211_key_conf *keyconf)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_vif_link_info *link_info = &mvmvif->deflink;
 
-	if (vif->type == NL80211_IFTYPE_AP &&
-	    !(keyconf->flags & IEEE80211_KEY_FLAG_PAIRWISE))
-		return BIT(mvmvif->mcast_sta.sta_id);
-
-	if (sta) {
-		struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
+	lockdep_assert_held(&mvm->mutex);
 
-		return BIT(mvmsta->sta_id);
+	if (keyconf->link_id >= 0) {
+		link_info = mvmvif->link[keyconf->link_id];
+		if (!link_info)
+			return 0;
 	}
 
-	if (vif->type == NL80211_IFTYPE_STATION &&
-	    mvmvif->ap_sta_id != IWL_MVM_INVALID_STA)
-		return BIT(mvmvif->ap_sta_id);
+	/* AP group keys are per link and should be on the mcast STA */
+	if (vif->type == NL80211_IFTYPE_AP &&
+	    !(keyconf->flags & IEEE80211_KEY_FLAG_PAIRWISE))
+		return BIT(link_info->mcast_sta.sta_id);
+
+	/* for client mode use the AP STA also for group keys */
+	if (!sta && vif->type == NL80211_IFTYPE_STATION)
+		sta = mvmvif->ap_sta;
+
+	/* During remove the STA was removed and the group keys come later
+	 * (which sounds like a bad sequence, but remember that to mac80211 the
+	 * group keys have no sta pointer), so we don't have a STA now.
+	 * Since this happens for group keys only, just use the link_info as
+	 * the group keys are per link; make sure that is the case by checking
+	 * we do have a link_id or are not doing MLO.
+	 * Of course the same can be done during add as well, but we must do
+	 * it during remove, since we don't have the mvmvif->ap_sta pointer.
+	 */
+	if (!sta && (keyconf->link_id >= 0 || !vif->valid_links))
+		return BIT(link_info->ap_sta_id);
+
+	/* STA should be non-NULL now, but iwl_mvm_sta_fw_id_mask() checks */
 
-	/* invalid */
-	return 0;
+	/* pass link_id to filter by it if not -1 (GTK on client) */
+	return iwl_mvm_sta_fw_id_mask(mvm, sta, keyconf->link_id);
 }
 
 static u32 iwl_mvm_get_sec_flags(struct iwl_mvm *mvm,
@@ -41,6 +59,8 @@ static u32 iwl_mvm_get_sec_flags(struct iwl_mvm *mvm,
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	u32 flags = 0;
 
+	lockdep_assert_held(&mvm->mutex);
+
 	if (!(keyconf->flags & IEEE80211_KEY_FLAG_PAIRWISE))
 		flags |= IWL_SEC_KEY_FLAG_MCAST_KEY;
 
@@ -68,22 +88,68 @@ static u32 iwl_mvm_get_sec_flags(struct iwl_mvm *mvm,
 		break;
 	}
 
-	rcu_read_lock();
-	if (!sta && vif->type == NL80211_IFTYPE_STATION &&
-	    mvmvif->ap_sta_id != IWL_MVM_INVALID_STA) {
-		u8 sta_id = mvmvif->ap_sta_id;
-
-		sta = rcu_dereference_check(mvm->fw_id_to_mac_id[sta_id],
-					    lockdep_is_held(&mvm->mutex));
-	}
+	if (!sta && vif->type == NL80211_IFTYPE_STATION)
+		sta = mvmvif->ap_sta;
 
 	if (!IS_ERR_OR_NULL(sta) && sta->mfp)
 		flags |= IWL_SEC_KEY_FLAG_MFP;
-	rcu_read_unlock();
 
 	return flags;
 }
 
+struct iwl_mvm_sta_key_update_data {
+	struct ieee80211_sta *sta;
+	u32 old_sta_mask;
+	u32 new_sta_mask;
+	int err;
+};
+
+static void iwl_mvm_mld_update_sta_key(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_sta *sta,
+				       struct ieee80211_key_conf *key,
+				       void *_data)
+{
+	u32 cmd_id = WIDE_ID(DATA_PATH_GROUP, SEC_KEY_CMD);
+	struct iwl_mvm_sta_key_update_data *data = _data;
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct iwl_sec_key_cmd cmd = {
+		.action = cpu_to_le32(FW_CTXT_ACTION_MODIFY),
+		.u.modify.old_sta_mask = cpu_to_le32(data->old_sta_mask),
+		.u.modify.new_sta_mask = cpu_to_le32(data->new_sta_mask),
+		.u.modify.key_id = cpu_to_le32(key->keyidx),
+		.u.modify.key_flags =
+			cpu_to_le32(iwl_mvm_get_sec_flags(mvm, vif, sta, key)),
+	};
+	int err;
+
+	/* only need to do this for pairwise keys (link_id == -1) */
+	if (sta != data->sta || key->link_id >= 0)
+		return;
+
+	err = iwl_mvm_send_cmd_pdu(mvm, cmd_id, CMD_ASYNC, sizeof(cmd), &cmd);
+
+	if (err)
+		data->err = err;
+}
+
+int iwl_mvm_mld_update_sta_keys(struct iwl_mvm *mvm,
+				struct ieee80211_vif *vif,
+				struct ieee80211_sta *sta,
+				u32 old_sta_mask,
+				u32 new_sta_mask)
+{
+	struct iwl_mvm_sta_key_update_data data = {
+		.sta = sta,
+		.old_sta_mask = old_sta_mask,
+		.new_sta_mask = new_sta_mask,
+	};
+
+	ieee80211_iter_keys_rcu(mvm->hw, vif, iwl_mvm_mld_update_sta_key,
+				&data);
+	return data.err;
+}
+
 static int __iwl_mvm_sec_key_del(struct iwl_mvm *mvm, u32 sta_mask,
 				 u32 key_flags, u32 keyidx, u32 flags)
 {
@@ -118,6 +184,9 @@ int iwl_mvm_sec_key_add(struct iwl_mvm *mvm,
 	if (WARN_ON(keyconf->keylen > sizeof(cmd.u.add.key)))
 		return -EINVAL;
 
+	if (WARN_ON(!sta_mask))
+		return -EINVAL;
+
 	if (keyconf->cipher == WLAN_CIPHER_SUITE_WEP40 ||
 	    keyconf->cipher == WLAN_CIPHER_SUITE_WEP104)
 		memcpy(cmd.u.add.key + IWL_SEC_WEP_KEY_OFFSET, keyconf->key,
@@ -164,6 +233,9 @@ static int _iwl_mvm_sec_key_del(struct iwl_mvm *mvm,
 	u32 key_flags = iwl_mvm_get_sec_flags(mvm, vif, sta, keyconf);
 	int ret;
 
+	if (WARN_ON(!sta_mask))
+		return -EINVAL;
+
 	ret = __iwl_mvm_sec_key_del(mvm, sta_mask, key_flags, keyconf->keyidx,
 				    flags);
 	if (ret)
@@ -195,6 +267,7 @@ static void iwl_mvm_sec_key_remove_ap_iter(struct ieee80211_hw *hw,
 					   void *data)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	unsigned int link_id = (uintptr_t)data;
 
 	if (key->hw_key_idx == STA_KEY_IDX_INVALID)
 		return;
@@ -202,19 +275,23 @@ static void iwl_mvm_sec_key_remove_ap_iter(struct ieee80211_hw *hw,
 	if (sta)
 		return;
 
+	if (key->link_id >= 0 && key->link_id != link_id)
+		return;
+
 	_iwl_mvm_sec_key_del(mvm, vif, NULL, key, CMD_ASYNC);
 	key->hw_key_idx = STA_KEY_IDX_INVALID;
 }
 
 void iwl_mvm_sec_key_remove_ap(struct iwl_mvm *mvm,
-			       struct ieee80211_vif *vif)
+			       struct ieee80211_vif *vif,
+			       struct iwl_mvm_vif_link_info *link,
+			       unsigned int link_id)
 {
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	u32 sec_key_id = WIDE_ID(DATA_PATH_GROUP, SEC_KEY_CMD);
 	u8 sec_key_ver = iwl_fw_lookup_cmd_ver(mvm->fw, sec_key_id, 0);
 
-	if (WARN_ON(vif->type != NL80211_IFTYPE_STATION ||
-		    mvmvif->ap_sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON_ONCE(vif->type != NL80211_IFTYPE_STATION ||
+			 link->ap_sta_id == IWL_MVM_INVALID_STA))
 		return;
 
 	if (!sec_key_ver)
@@ -222,5 +299,5 @@ void iwl_mvm_sec_key_remove_ap(struct iwl_mvm *mvm,
 
 	ieee80211_iter_keys_rcu(mvm->hw, vif,
 				iwl_mvm_sec_key_remove_ap_iter,
-				NULL);
+				(void *)(uintptr_t)link_id);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac.c b/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac.c
new file mode 100644
index 000000000000..1717fb52dc12
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac.c
@@ -0,0 +1,309 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2022 Intel Corporation
+ */
+#include "mvm.h"
+
+static void iwl_mvm_mld_set_he_support(struct iwl_mvm *mvm,
+				       struct ieee80211_vif *vif,
+				       struct iwl_mac_config_cmd *cmd)
+{
+	if (vif->type == NL80211_IFTYPE_AP)
+		cmd->he_ap_support = cpu_to_le16(1);
+	else
+		cmd->he_support = cpu_to_le16(1);
+}
+
+static void iwl_mvm_mld_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
+					    struct ieee80211_vif *vif,
+					    struct iwl_mac_config_cmd *cmd,
+					    u32 action)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct ieee80211_bss_conf *link_conf;
+	unsigned int link_id;
+
+	cmd->id_and_color = cpu_to_le32(mvmvif->id);
+	cmd->action = cpu_to_le32(action);
+
+	cmd->mac_type = cpu_to_le32(iwl_mvm_get_mac_type(vif));
+
+	memcpy(cmd->local_mld_addr, vif->addr, ETH_ALEN);
+
+	cmd->he_support = 0;
+	cmd->eht_support = 0;
+
+	/* should be set by specific context type handler */
+	cmd->filter_flags = 0;
+
+	cmd->nic_not_ack_enabled =
+		cpu_to_le32(!iwl_mvm_is_nic_ack_enabled(mvm, vif));
+
+	if (iwlwifi_mod_params.disable_11ax)
+		return;
+
+	/* If we have MLO enabled, then the firmware needs to enable
+	 * address translation for the station(s) we add. That depends
+	 * on having EHT enabled in firmware, which in turn depends on
+	 * mac80211 in the code below.
+	 * However, mac80211 doesn't enable HE/EHT until it has parsed
+	 * the association response successfully, so just skip all that
+	 * and enable both when we have MLO.
+	 */
+	if (vif->valid_links) {
+		iwl_mvm_mld_set_he_support(mvm, vif, cmd);
+		cmd->eht_support = cpu_to_le32(1);
+		return;
+	}
+
+	rcu_read_lock();
+	for (link_id = 0; link_id < ARRAY_SIZE((vif)->link_conf); link_id++) {
+		link_conf = rcu_dereference(vif->link_conf[link_id]);
+		if (!link_conf)
+			continue;
+
+		if (link_conf->he_support)
+			iwl_mvm_mld_set_he_support(mvm, vif, cmd);
+
+		/* it's not reasonable to have EHT without HE and FW API doesn't
+		 * support it. Ignore EHT in this case.
+		 */
+		if (!link_conf->he_support && link_conf->eht_support)
+			continue;
+
+		if (link_conf->eht_support) {
+			cmd->eht_support = cpu_to_le32(1);
+			break;
+		}
+	}
+	rcu_read_unlock();
+}
+
+static int iwl_mvm_mld_mac_ctxt_send_cmd(struct iwl_mvm *mvm,
+					 struct iwl_mac_config_cmd *cmd)
+{
+	int ret = iwl_mvm_send_cmd_pdu(mvm,
+				       WIDE_ID(MAC_CONF_GROUP, MAC_CONFIG_CMD),
+				       0, sizeof(*cmd), cmd);
+	if (ret)
+		IWL_ERR(mvm, "Failed to send MAC_CONFIG_CMD (action:%d): %d\n",
+			le32_to_cpu(cmd->action), ret);
+	return ret;
+}
+
+static int iwl_mvm_mld_mac_ctxt_cmd_sta(struct iwl_mvm *mvm,
+					struct ieee80211_vif *vif,
+					u32 action, bool force_assoc_off)
+{
+	struct iwl_mac_config_cmd cmd = {};
+
+	WARN_ON(vif->type != NL80211_IFTYPE_STATION);
+
+	/* Fill the common data for all mac context types */
+	iwl_mvm_mld_mac_ctxt_cmd_common(mvm, vif, &cmd, action);
+
+	/*
+	 * We always want to hear MCAST frames, if we're not authorized yet,
+	 * we'll drop them.
+	 */
+	cmd.filter_flags |= cpu_to_le32(MAC_CFG_FILTER_ACCEPT_GRP);
+
+	if (vif->p2p)
+		cmd.client.ctwin =
+			iwl_mvm_mac_ctxt_cmd_p2p_sta_get_oppps_ctwin(mvm, vif);
+
+	if (vif->cfg.assoc && !force_assoc_off) {
+		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+
+		cmd.client.is_assoc = cpu_to_le32(1);
+
+		if (!mvmvif->authorized &&
+		    fw_has_capa(&mvm->fw->ucode_capa,
+				IWL_UCODE_TLV_CAPA_COEX_HIGH_PRIO))
+			cmd.client.data_policy |=
+				cpu_to_le32(COEX_HIGH_PRIORITY_ENABLE);
+
+	} else {
+		cmd.client.is_assoc = cpu_to_le32(0);
+
+		/* Allow beacons to pass through as long as we are not
+		 * associated, or we do not have dtim period information.
+		 */
+		cmd.filter_flags |= cpu_to_le32(MAC_CFG_FILTER_ACCEPT_BEACON);
+	}
+
+	cmd.client.assoc_id = cpu_to_le32(vif->cfg.aid);
+
+	if (vif->probe_req_reg && vif->cfg.assoc && vif->p2p)
+		cmd.filter_flags |= cpu_to_le32(MAC_CFG_FILTER_ACCEPT_PROBE_REQ);
+
+	if (vif->bss_conf.he_support && !iwlwifi_mod_params.disable_11ax)
+		cmd.client.data_policy |=
+			iwl_mvm_mac_ctxt_cmd_sta_get_twt_policy(mvm, vif);
+
+	return iwl_mvm_mld_mac_ctxt_send_cmd(mvm, &cmd);
+}
+
+static int iwl_mvm_mld_mac_ctxt_cmd_listener(struct iwl_mvm *mvm,
+					     struct ieee80211_vif *vif,
+					     u32 action)
+{
+	struct iwl_mac_config_cmd cmd = {};
+
+	WARN_ON(vif->type != NL80211_IFTYPE_MONITOR);
+
+	iwl_mvm_mld_mac_ctxt_cmd_common(mvm, vif, &cmd, action);
+
+	cmd.filter_flags = cpu_to_le32(MAC_CFG_FILTER_PROMISC |
+				       MAC_FILTER_IN_CONTROL_AND_MGMT |
+				       MAC_CFG_FILTER_ACCEPT_BEACON |
+				       MAC_CFG_FILTER_ACCEPT_PROBE_REQ |
+				       MAC_CFG_FILTER_ACCEPT_GRP);
+
+	return iwl_mvm_mld_mac_ctxt_send_cmd(mvm, &cmd);
+}
+
+static int iwl_mvm_mld_mac_ctxt_cmd_ibss(struct iwl_mvm *mvm,
+					 struct ieee80211_vif *vif,
+					 u32 action)
+{
+	struct iwl_mac_config_cmd cmd = {};
+
+	WARN_ON(vif->type != NL80211_IFTYPE_ADHOC);
+
+	iwl_mvm_mld_mac_ctxt_cmd_common(mvm, vif, &cmd, action);
+
+	cmd.filter_flags = cpu_to_le32(MAC_CFG_FILTER_ACCEPT_BEACON |
+				       MAC_CFG_FILTER_ACCEPT_PROBE_REQ |
+				       MAC_CFG_FILTER_ACCEPT_GRP);
+
+	return iwl_mvm_mld_mac_ctxt_send_cmd(mvm, &cmd);
+}
+
+static int iwl_mvm_mld_mac_ctxt_cmd_p2p_device(struct iwl_mvm *mvm,
+					       struct ieee80211_vif *vif,
+					       u32 action)
+{
+	struct iwl_mac_config_cmd cmd = {};
+
+	WARN_ON(vif->type != NL80211_IFTYPE_P2P_DEVICE);
+
+	iwl_mvm_mld_mac_ctxt_cmd_common(mvm, vif, &cmd, action);
+
+	cmd.p2p_dev.is_disc_extended =
+		iwl_mac_ctxt_p2p_dev_has_extended_disc(mvm, vif);
+
+	/* Override the filter flags to accept only probe requests */
+	cmd.filter_flags = cpu_to_le32(MAC_CFG_FILTER_ACCEPT_PROBE_REQ);
+
+	return iwl_mvm_mld_mac_ctxt_send_cmd(mvm, &cmd);
+}
+
+static int iwl_mvm_mld_mac_ctxt_cmd_ap_go(struct iwl_mvm *mvm,
+					  struct ieee80211_vif *vif,
+					  u32 action)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mac_config_cmd cmd = {};
+
+	WARN_ON(vif->type != NL80211_IFTYPE_AP);
+
+	/* Fill the common data for all mac context types */
+	iwl_mvm_mld_mac_ctxt_cmd_common(mvm, vif, &cmd, action);
+
+	iwl_mvm_mac_ctxt_cmd_ap_set_filter_flags(mvm, mvmvif,
+						 &cmd.filter_flags,
+						 MAC_CFG_FILTER_ACCEPT_PROBE_REQ,
+						 MAC_CFG_FILTER_ACCEPT_BEACON);
+
+	return iwl_mvm_mld_mac_ctxt_send_cmd(mvm, &cmd);
+}
+
+static int iwl_mvm_mld_mac_ctx_send(struct iwl_mvm *mvm,
+				    struct ieee80211_vif *vif,
+				    u32 action, bool force_assoc_off)
+{
+	switch (vif->type) {
+	case NL80211_IFTYPE_STATION:
+		return iwl_mvm_mld_mac_ctxt_cmd_sta(mvm, vif, action,
+						    force_assoc_off);
+	case NL80211_IFTYPE_AP:
+		return iwl_mvm_mld_mac_ctxt_cmd_ap_go(mvm, vif, action);
+	case NL80211_IFTYPE_MONITOR:
+		return iwl_mvm_mld_mac_ctxt_cmd_listener(mvm, vif, action);
+	case NL80211_IFTYPE_P2P_DEVICE:
+		return iwl_mvm_mld_mac_ctxt_cmd_p2p_device(mvm, vif, action);
+	case NL80211_IFTYPE_ADHOC:
+		return iwl_mvm_mld_mac_ctxt_cmd_ibss(mvm, vif, action);
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+int iwl_mvm_mld_mac_ctxt_add(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int ret;
+
+	if (WARN_ON_ONCE(vif->type == NL80211_IFTYPE_NAN))
+		return -EOPNOTSUPP;
+
+	if (WARN_ONCE(mvmvif->uploaded, "Adding active MAC %pM/%d\n",
+		      vif->addr, ieee80211_vif_type_p2p(vif)))
+		return -EIO;
+
+	ret = iwl_mvm_mld_mac_ctx_send(mvm, vif, FW_CTXT_ACTION_ADD,
+				       true);
+	if (ret)
+		return ret;
+
+	/* will only do anything at resume from D3 time */
+	iwl_mvm_set_last_nonqos_seq(mvm, vif);
+
+	mvmvif->uploaded = true;
+	return 0;
+}
+
+int iwl_mvm_mld_mac_ctxt_changed(struct iwl_mvm *mvm,
+				 struct ieee80211_vif *vif,
+				 bool force_assoc_off)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+
+	if (WARN_ON_ONCE(vif->type == NL80211_IFTYPE_NAN))
+		return -EOPNOTSUPP;
+
+	if (WARN_ONCE(!mvmvif->uploaded, "Changing inactive MAC %pM/%d\n",
+		      vif->addr, ieee80211_vif_type_p2p(vif)))
+		return -EIO;
+
+	return iwl_mvm_mld_mac_ctx_send(mvm, vif, FW_CTXT_ACTION_MODIFY,
+					force_assoc_off);
+}
+
+int iwl_mvm_mld_mac_ctxt_remove(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mac_config_cmd cmd = {
+		.action = cpu_to_le32(FW_CTXT_ACTION_REMOVE),
+		.id_and_color = cpu_to_le32(mvmvif->id),
+	};
+	int ret;
+
+	if (WARN_ON_ONCE(vif->type == NL80211_IFTYPE_NAN))
+		return -EOPNOTSUPP;
+
+	if (WARN_ONCE(!mvmvif->uploaded, "Removing inactive MAC %pM/%d\n",
+		      vif->addr, ieee80211_vif_type_p2p(vif)))
+		return -EIO;
+
+	ret = iwl_mvm_mld_mac_ctxt_send_cmd(mvm, &cmd);
+	if (ret)
+		return ret;
+
+	mvmvif->uploaded = false;
+
+	return 0;
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c
new file mode 100644
index 000000000000..fbc2d5ed1006
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c
@@ -0,0 +1,1101 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2022 Intel Corporation
+ */
+#include "mvm.h"
+
+static int iwl_mvm_mld_mac_add_interface(struct ieee80211_hw *hw,
+					 struct ieee80211_vif *vif)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int ret;
+
+	mutex_lock(&mvm->mutex);
+
+	mvmvif->mvm = mvm;
+
+	/* Not much to do here. The stack will not allow interface
+	 * types or combinations that we didn't advertise, so we
+	 * don't really have to check the types.
+	 */
+
+	/* make sure that beacon statistics don't go backwards with FW reset */
+	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
+		mvmvif->deflink.beacon_stats.accu_num_beacons +=
+			mvmvif->deflink.beacon_stats.num_beacons;
+
+	/* Allocate resources for the MAC context, and add it to the fw  */
+	ret = iwl_mvm_mac_ctxt_init(mvm, vif);
+	if (ret)
+		goto out_unlock;
+
+	rcu_assign_pointer(mvm->vif_id_to_mac[mvmvif->id], vif);
+
+	mvmvif->features |= hw->netdev_features;
+
+	/* reset deflink MLO parameters */
+	mvmvif->deflink.fw_link_id = IWL_MVM_FW_LINK_ID_INVALID;
+	mvmvif->deflink.active = 0;
+	/* the first link always points to the default one */
+	mvmvif->link[0] = &mvmvif->deflink;
+
+	ret = iwl_mvm_mld_mac_ctxt_add(mvm, vif);
+	if (ret)
+		goto out_unlock;
+
+	/* beacon filtering */
+	ret = iwl_mvm_disable_beacon_filter(mvm, vif, 0);
+	if (ret)
+		goto out_remove_mac;
+
+	if (!mvm->bf_allowed_vif &&
+	    vif->type == NL80211_IFTYPE_STATION && !vif->p2p) {
+		mvm->bf_allowed_vif = mvmvif;
+		vif->driver_flags |= IEEE80211_VIF_BEACON_FILTER |
+				     IEEE80211_VIF_SUPPORTS_CQM_RSSI;
+	}
+
+	/*
+	 * P2P_DEVICE interface does not have a channel context assigned to it,
+	 * so a dedicated PHY context is allocated to it and the corresponding
+	 * MAC context is bound to it at this stage.
+	 */
+	if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
+		mvmvif->deflink.phy_ctxt = iwl_mvm_get_free_phy_ctxt(mvm);
+		if (!mvmvif->deflink.phy_ctxt) {
+			ret = -ENOSPC;
+			goto out_free_bf;
+		}
+
+		iwl_mvm_phy_ctxt_ref(mvm, mvmvif->deflink.phy_ctxt);
+		ret = iwl_mvm_add_link(mvm, vif, &vif->bss_conf);
+		if (ret)
+			goto out_unref_phy;
+
+		ret = iwl_mvm_link_changed(mvm, vif, &vif->bss_conf,
+					   LINK_CONTEXT_MODIFY_ACTIVE |
+					   LINK_CONTEXT_MODIFY_RATES_INFO,
+					   true);
+		if (ret)
+			goto out_remove_link;
+
+		ret = iwl_mvm_mld_add_bcast_sta(mvm, vif, &vif->bss_conf);
+		if (ret)
+			goto out_remove_link;
+
+		/* Save a pointer to p2p device vif, so it can later be used to
+		 * update the p2p device MAC when a GO is started/stopped
+		 */
+		mvm->p2p_device_vif = vif;
+	} else {
+		ret = iwl_mvm_add_link(mvm, vif, &vif->bss_conf);
+		if (ret)
+			goto out_free_bf;
+	}
+
+	ret = iwl_mvm_power_update_mac(mvm);
+	if (ret)
+		goto out_free_bf;
+
+	iwl_mvm_tcm_add_vif(mvm, vif);
+	INIT_DELAYED_WORK(&mvmvif->csa_work,
+			  iwl_mvm_channel_switch_disconnect_wk);
+
+	if (vif->type == NL80211_IFTYPE_MONITOR) {
+		mvm->monitor_on = true;
+		ieee80211_hw_set(mvm->hw, RX_INCLUDES_FCS);
+	}
+
+	iwl_mvm_vif_dbgfs_register(mvm, vif);
+
+	if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status) &&
+	    vif->type == NL80211_IFTYPE_STATION && !vif->p2p &&
+	    !mvm->csme_vif && mvm->mei_registered) {
+		iwl_mei_set_nic_info(vif->addr, mvm->nvm_data->hw_addr);
+		iwl_mei_set_netdev(ieee80211_vif_to_wdev(vif)->netdev);
+		mvm->csme_vif = vif;
+	}
+
+	goto out_unlock;
+
+ out_remove_link:
+	iwl_mvm_disable_link(mvm, vif, &vif->bss_conf);
+ out_unref_phy:
+	iwl_mvm_phy_ctxt_unref(mvm, mvmvif->deflink.phy_ctxt);
+ out_free_bf:
+	if (mvm->bf_allowed_vif == mvmvif) {
+		mvm->bf_allowed_vif = NULL;
+		vif->driver_flags &= ~(IEEE80211_VIF_BEACON_FILTER |
+				       IEEE80211_VIF_SUPPORTS_CQM_RSSI);
+	}
+ out_remove_mac:
+	mvmvif->deflink.phy_ctxt = NULL;
+	mvmvif->link[0] = NULL;
+	iwl_mvm_mld_mac_ctxt_remove(mvm, vif);
+ out_unlock:
+	mutex_unlock(&mvm->mutex);
+
+	return ret;
+}
+
+static void iwl_mvm_mld_mac_remove_interface(struct ieee80211_hw *hw,
+					     struct ieee80211_vif *vif)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_probe_resp_data *probe_data;
+
+	iwl_mvm_prepare_mac_removal(mvm, vif);
+
+	if (!(vif->type == NL80211_IFTYPE_AP ||
+	      vif->type == NL80211_IFTYPE_ADHOC))
+		iwl_mvm_tcm_rm_vif(mvm, vif);
+
+	mutex_lock(&mvm->mutex);
+
+	if (vif == mvm->csme_vif) {
+		iwl_mei_set_netdev(NULL);
+		mvm->csme_vif = NULL;
+	}
+
+	if (mvm->bf_allowed_vif == mvmvif) {
+		mvm->bf_allowed_vif = NULL;
+		vif->driver_flags &= ~(IEEE80211_VIF_BEACON_FILTER |
+				       IEEE80211_VIF_SUPPORTS_CQM_RSSI);
+	}
+
+	if (vif->bss_conf.ftm_responder)
+		memset(&mvm->ftm_resp_stats, 0, sizeof(mvm->ftm_resp_stats));
+
+	iwl_mvm_vif_dbgfs_clean(mvm, vif);
+
+	/* For AP/GO interface, the tear down of the resources allocated to the
+	 * interface is be handled as part of the stop_ap flow.
+	 */
+	if (vif->type == NL80211_IFTYPE_AP ||
+	    vif->type == NL80211_IFTYPE_ADHOC) {
+#ifdef CONFIG_NL80211_TESTMODE
+		if (vif == mvm->noa_vif) {
+			mvm->noa_vif = NULL;
+			mvm->noa_duration = 0;
+		}
+#endif
+	}
+
+	iwl_mvm_power_update_mac(mvm);
+
+	if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
+		mvm->p2p_device_vif = NULL;
+
+		/* P2P device uses only one link */
+		iwl_mvm_mld_rm_bcast_sta(mvm, vif, &vif->bss_conf);
+		iwl_mvm_disable_link(mvm, vif, &vif->bss_conf);
+		iwl_mvm_phy_ctxt_unref(mvm, mvmvif->deflink.phy_ctxt);
+		mvmvif->deflink.phy_ctxt = NULL;
+	} else {
+		iwl_mvm_disable_link(mvm, vif, &vif->bss_conf);
+	}
+
+	iwl_mvm_mld_mac_ctxt_remove(mvm, vif);
+
+	RCU_INIT_POINTER(mvm->vif_id_to_mac[mvmvif->id], NULL);
+
+	probe_data = rcu_dereference_protected(mvmvif->deflink.probe_resp_data,
+					       lockdep_is_held(&mvm->mutex));
+	RCU_INIT_POINTER(mvmvif->deflink.probe_resp_data, NULL);
+	if (probe_data)
+		kfree_rcu(probe_data, rcu_head);
+
+	if (vif->type == NL80211_IFTYPE_MONITOR) {
+		mvm->monitor_on = false;
+		__clear_bit(IEEE80211_HW_RX_INCLUDES_FCS, mvm->hw->flags);
+	}
+
+	mutex_unlock(&mvm->mutex);
+}
+
+static int
+__iwl_mvm_mld_assign_vif_chanctx(struct iwl_mvm *mvm,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_bss_conf *link_conf,
+				 struct ieee80211_chanctx_conf *ctx,
+				 bool switching_chanctx)
+{
+	u16 *phy_ctxt_id = (u16 *)ctx->drv_priv;
+	struct iwl_mvm_phy_ctxt *phy_ctxt = &mvm->phy_ctxts[*phy_ctxt_id];
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	unsigned int link_id = link_conf->link_id;
+	int ret;
+
+	if (WARN_ON_ONCE(!mvmvif->link[link_id]))
+		return -EINVAL;
+
+	/* mac parameters such as HE support can change at this stage
+	 * For sta, need first to configure correct state from drv_sta_state
+	 * and only after that update mac config.
+	 */
+	if (vif->type == NL80211_IFTYPE_AP) {
+		ret = iwl_mvm_mld_mac_ctxt_changed(mvm, vif, false);
+		if (ret) {
+			IWL_ERR(mvm, "failed to update MAC %pM\n", vif->addr);
+			return -EINVAL;
+		}
+	}
+
+	mvmvif->link[link_id]->phy_ctxt = phy_ctxt;
+
+	if (switching_chanctx) {
+		/* reactivate if we turned this off during channel switch */
+		if (vif->type == NL80211_IFTYPE_AP)
+			mvmvif->ap_ibss_active = true;
+	}
+
+	/* send it first with phy context ID */
+	ret = iwl_mvm_link_changed(mvm, vif, link_conf, 0, false);
+	if (ret)
+		goto out;
+
+	/* Initialize rate control for the AP station, since we might be
+	 * doing a link switch here - we cannot initialize it before since
+	 * this needs the phy context assigned (and in FW?), and we cannot
+	 * do it later because it needs to be initialized as soon as we're
+	 * able to TX on the link, i.e. when active.
+	 *
+	 * Firmware restart isn't quite correct yet for MLO, but we don't
+	 * need to do it in that case anyway since it will happen from the
+	 * normal station state callback.
+	 */
+	if (mvmvif->ap_sta &&
+	    !test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
+		struct ieee80211_link_sta *link_sta;
+
+		rcu_read_lock();
+		link_sta = rcu_dereference(mvmvif->ap_sta->link[link_id]);
+
+		if (!WARN_ON_ONCE(!link_sta))
+			iwl_mvm_rs_rate_init(mvm, vif, mvmvif->ap_sta,
+					     link_conf, link_sta,
+					     phy_ctxt->channel->band);
+		rcu_read_unlock();
+	}
+
+	/* then activate */
+	ret = iwl_mvm_link_changed(mvm, vif, link_conf,
+				   LINK_CONTEXT_MODIFY_ACTIVE |
+				   LINK_CONTEXT_MODIFY_RATES_INFO,
+				   true);
+	if (ret)
+		goto out;
+
+	/*
+	 * Power state must be updated before quotas,
+	 * otherwise fw will complain.
+	 */
+	iwl_mvm_power_update_mac(mvm);
+
+	if (vif->type == NL80211_IFTYPE_MONITOR) {
+		ret = iwl_mvm_mld_add_snif_sta(mvm, vif, link_conf);
+		if (ret)
+			goto deactivate;
+	}
+
+	return 0;
+
+deactivate:
+	iwl_mvm_link_changed(mvm, vif, link_conf, LINK_CONTEXT_MODIFY_ACTIVE,
+			     false);
+out:
+	mvmvif->link[link_id]->phy_ctxt = NULL;
+	iwl_mvm_power_update_mac(mvm);
+	return ret;
+}
+
+static int iwl_mvm_mld_assign_vif_chanctx(struct ieee80211_hw *hw,
+					  struct ieee80211_vif *vif,
+					  struct ieee80211_bss_conf *link_conf,
+					  struct ieee80211_chanctx_conf *ctx)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	int ret;
+
+	mutex_lock(&mvm->mutex);
+	ret = __iwl_mvm_mld_assign_vif_chanctx(mvm, vif, link_conf, ctx, false);
+	mutex_unlock(&mvm->mutex);
+
+	return ret;
+}
+
+static void
+__iwl_mvm_mld_unassign_vif_chanctx(struct iwl_mvm *mvm,
+				   struct ieee80211_vif *vif,
+				   struct ieee80211_bss_conf *link_conf,
+				   struct ieee80211_chanctx_conf *ctx,
+				   bool switching_chanctx)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	unsigned int link_id = link_conf->link_id;
+
+	/* shouldn't happen, but verify link_id is valid before accessing */
+	if (WARN_ON_ONCE(!mvmvif->link[link_id]))
+		return;
+
+	if (vif->type == NL80211_IFTYPE_AP && switching_chanctx) {
+		mvmvif->csa_countdown = false;
+
+		/* Set CS bit on all the stations */
+		iwl_mvm_modify_all_sta_disable_tx(mvm, mvmvif, true);
+
+		/* Save blocked iface, the timeout is set on the next beacon */
+		rcu_assign_pointer(mvm->csa_tx_blocked_vif, vif);
+
+		mvmvif->ap_ibss_active = false;
+	}
+
+	if (vif->type == NL80211_IFTYPE_MONITOR)
+		iwl_mvm_mld_rm_snif_sta(mvm, vif);
+
+	iwl_mvm_link_changed(mvm, vif, link_conf,
+			     LINK_CONTEXT_MODIFY_ACTIVE, false);
+
+	if (switching_chanctx)
+		return;
+	mvmvif->link[link_id]->phy_ctxt = NULL;
+	iwl_mvm_power_update_mac(mvm);
+}
+
+static void iwl_mvm_mld_unassign_vif_chanctx(struct ieee80211_hw *hw,
+					     struct ieee80211_vif *vif,
+					     struct ieee80211_bss_conf *link_conf,
+					     struct ieee80211_chanctx_conf *ctx)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+
+	mutex_lock(&mvm->mutex);
+	__iwl_mvm_mld_unassign_vif_chanctx(mvm, vif, link_conf, ctx, false);
+	mutex_unlock(&mvm->mutex);
+}
+
+static int iwl_mvm_mld_start_ap_ibss(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int ret;
+
+	mutex_lock(&mvm->mutex);
+	/* Send the beacon template */
+	ret = iwl_mvm_mac_ctxt_beacon_changed(mvm, vif, link_conf);
+	if (ret)
+		goto out_unlock;
+
+	/* the link should be already activated when assigning chan context */
+	ret = iwl_mvm_link_changed(mvm, vif, link_conf,
+				   LINK_CONTEXT_MODIFY_ALL &
+				   ~LINK_CONTEXT_MODIFY_ACTIVE,
+				   true);
+	if (ret)
+		goto out_unlock;
+
+	ret = iwl_mvm_mld_add_mcast_sta(mvm, vif, link_conf);
+	if (ret)
+		goto out_unlock;
+
+	/* Send the bcast station. At this stage the TBTT and DTIM time
+	 * events are added and applied to the scheduler
+	 */
+	ret = iwl_mvm_mld_add_bcast_sta(mvm, vif, link_conf);
+	if (ret)
+		goto out_rm_mcast;
+
+	if (iwl_mvm_start_ap_ibss_common(hw, vif, &ret))
+		goto out_failed;
+
+	/* Need to update the P2P Device MAC (only GO, IBSS is single vif) */
+	if (vif->p2p && mvm->p2p_device_vif)
+		iwl_mvm_mld_mac_ctxt_changed(mvm, mvm->p2p_device_vif, false);
+
+	iwl_mvm_bt_coex_vif_change(mvm);
+
+	/* we don't support TDLS during DCM */
+	if (iwl_mvm_phy_ctx_count(mvm) > 1)
+		iwl_mvm_teardown_tdls_peers(mvm);
+
+	iwl_mvm_ftm_restart_responder(mvm, vif);
+
+	goto out_unlock;
+
+out_failed:
+	iwl_mvm_power_update_mac(mvm);
+	mvmvif->ap_ibss_active = false;
+	iwl_mvm_mld_rm_bcast_sta(mvm, vif, link_conf);
+out_rm_mcast:
+	iwl_mvm_mld_rm_mcast_sta(mvm, vif, link_conf);
+out_unlock:
+	mutex_unlock(&mvm->mutex);
+	return ret;
+}
+
+static int iwl_mvm_mld_start_ap(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_bss_conf *link_conf)
+{
+	return iwl_mvm_mld_start_ap_ibss(hw, vif, link_conf);
+}
+
+static int iwl_mvm_mld_start_ibss(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif)
+{
+	return iwl_mvm_mld_start_ap_ibss(hw, vif, &vif->bss_conf);
+}
+
+static void iwl_mvm_mld_stop_ap_ibss(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+
+	mutex_lock(&mvm->mutex);
+
+	iwl_mvm_stop_ap_ibss_common(mvm, vif);
+
+	/* Need to update the P2P Device MAC (only GO, IBSS is single vif) */
+	if (vif->p2p && mvm->p2p_device_vif)
+		iwl_mvm_mld_mac_ctxt_changed(mvm, mvm->p2p_device_vif, false);
+
+	iwl_mvm_ftm_responder_clear(mvm, vif);
+
+	iwl_mvm_mld_rm_bcast_sta(mvm, vif, link_conf);
+	iwl_mvm_mld_rm_mcast_sta(mvm, vif, link_conf);
+
+	iwl_mvm_power_update_mac(mvm);
+	mutex_unlock(&mvm->mutex);
+}
+
+static void iwl_mvm_mld_stop_ap(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_bss_conf *link_conf)
+{
+	iwl_mvm_mld_stop_ap_ibss(hw, vif, link_conf);
+}
+
+static void iwl_mvm_mld_stop_ibss(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif)
+{
+	iwl_mvm_mld_stop_ap_ibss(hw, vif, &vif->bss_conf);
+}
+
+static int iwl_mvm_mld_mac_sta_state(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_sta *sta,
+				     enum ieee80211_sta_state old_state,
+				     enum ieee80211_sta_state new_state)
+{
+	struct iwl_mvm_sta_state_ops callbacks = {
+		.add_sta = iwl_mvm_mld_add_sta,
+		.update_sta = iwl_mvm_mld_update_sta,
+		.rm_sta = iwl_mvm_mld_rm_sta,
+		.mac_ctxt_changed = iwl_mvm_mld_mac_ctxt_changed,
+	};
+
+	return iwl_mvm_mac_sta_state_common(hw, vif, sta, old_state, new_state,
+					    &callbacks);
+}
+
+static void
+iwl_mvm_mld_link_info_changed_station(struct iwl_mvm *mvm,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_bss_conf *link_conf,
+				      u64 changes)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	bool has_he, has_eht;
+	u32 link_changes = 0;
+	int ret;
+
+	if (WARN_ON_ONCE(!mvmvif->link[link_conf->link_id]))
+		return;
+
+	has_he = link_conf->he_support && !iwlwifi_mod_params.disable_11ax;
+	has_eht = link_conf->eht_support && !iwlwifi_mod_params.disable_11be;
+
+	/* Update EDCA params */
+	if (changes & BSS_CHANGED_QOS && vif->cfg.assoc && link_conf->qos)
+		link_changes |= LINK_CONTEXT_MODIFY_QOS_PARAMS;
+
+	if (changes & BSS_CHANGED_ERP_SLOT)
+		link_changes |= LINK_CONTEXT_MODIFY_RATES_INFO;
+
+	if (vif->cfg.assoc && (has_he || has_eht)) {
+		IWL_DEBUG_MAC80211(mvm, "Associated in HE mode\n");
+		link_changes |= LINK_CONTEXT_MODIFY_HE_PARAMS;
+	}
+
+	/* Update EHT Puncturing info */
+	if (changes & BSS_CHANGED_EHT_PUNCTURING && vif->cfg.assoc && has_eht)
+		link_changes |= LINK_CONTEXT_MODIFY_EHT_PARAMS;
+
+	if (link_changes) {
+		ret = iwl_mvm_link_changed(mvm, vif, link_conf, link_changes,
+					   true);
+		if (ret)
+			IWL_ERR(mvm, "failed to update link\n");
+	}
+
+	ret = iwl_mvm_mld_mac_ctxt_changed(mvm, vif, false);
+	if (ret)
+		IWL_ERR(mvm, "failed to update MAC %pM\n", vif->addr);
+
+	memcpy(mvmvif->link[link_conf->link_id]->bssid, link_conf->bssid,
+	       ETH_ALEN);
+
+	iwl_mvm_bss_info_changed_station_common(mvm, vif, link_conf, changes);
+}
+
+static bool iwl_mvm_mld_vif_have_valid_ap_sta(struct iwl_mvm_vif *mvmvif)
+{
+	int i;
+
+	for_each_mvm_vif_valid_link(mvmvif, i) {
+		if (mvmvif->link[i]->ap_sta_id != IWL_MVM_INVALID_STA)
+			return true;
+	}
+
+	return false;
+}
+
+static void iwl_mvm_mld_vif_delete_all_stas(struct iwl_mvm *mvm,
+					    struct ieee80211_vif *vif)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int i, ret;
+
+	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
+		return;
+
+	for_each_mvm_vif_valid_link(mvmvif, i) {
+		struct iwl_mvm_vif_link_info *link = mvmvif->link[i];
+
+		if (!link)
+			continue;
+
+		iwl_mvm_sec_key_remove_ap(mvm, vif, link, i);
+		ret = iwl_mvm_mld_rm_sta_id(mvm, link->ap_sta_id);
+		if (ret)
+			IWL_ERR(mvm, "failed to remove AP station\n");
+
+		link->ap_sta_id = IWL_MVM_INVALID_STA;
+	}
+}
+
+static void iwl_mvm_mld_vif_cfg_changed_station(struct iwl_mvm *mvm,
+						struct ieee80211_vif *vif,
+						u64 changes)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct ieee80211_bss_conf *link_conf;
+	bool protect = false;
+	unsigned int i;
+	int ret;
+
+	/* This might get called without active links during the
+	 * chanctx switch, but we don't care about it anyway.
+	 */
+	if (changes == BSS_CHANGED_IDLE)
+		return;
+
+	ret = iwl_mvm_mld_mac_ctxt_changed(mvm, vif, false);
+	if (ret)
+		IWL_ERR(mvm, "failed to update MAC %pM\n", vif->addr);
+
+	mvmvif->associated = vif->cfg.assoc;
+
+	if (!(changes & BSS_CHANGED_ASSOC))
+		return;
+
+	if (vif->cfg.assoc) {
+		/* clear statistics to get clean beacon counter */
+		iwl_mvm_request_statistics(mvm, true);
+		iwl_mvm_sf_update(mvm, vif, false);
+		iwl_mvm_power_vif_assoc(mvm, vif);
+
+		for_each_mvm_vif_valid_link(mvmvif, i) {
+			memset(&mvmvif->link[i]->beacon_stats, 0,
+			       sizeof(mvmvif->link[i]->beacon_stats));
+
+			if (vif->p2p) {
+				iwl_mvm_update_smps(mvm, vif,
+						    IWL_MVM_SMPS_REQ_PROT,
+						    IEEE80211_SMPS_DYNAMIC, i);
+			}
+
+			rcu_read_lock();
+			link_conf = rcu_dereference(vif->link_conf[i]);
+			if (link_conf && !link_conf->dtim_period)
+				protect = true;
+			rcu_read_unlock();
+		}
+
+		if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status) &&
+		    protect) {
+			/* If we're not restarting and still haven't
+			 * heard a beacon (dtim period unknown) then
+			 * make sure we still have enough minimum time
+			 * remaining in the time event, since the auth
+			 * might actually have taken quite a while
+			 * (especially for SAE) and so the remaining
+			 * time could be small without us having heard
+			 * a beacon yet.
+			 */
+			iwl_mvm_protect_assoc(mvm, vif, 0);
+		}
+
+		iwl_mvm_sf_update(mvm, vif, false);
+
+		/* FIXME: need to decide about misbehaving AP handling */
+		iwl_mvm_power_vif_assoc(mvm, vif);
+	} else if (iwl_mvm_mld_vif_have_valid_ap_sta(mvmvif)) {
+		iwl_mvm_mei_host_disassociated(mvm);
+
+		/* If update fails - SF might be running in associated
+		 * mode while disassociated - which is forbidden.
+		 */
+		ret = iwl_mvm_sf_update(mvm, vif, false);
+		WARN_ONCE(ret &&
+			  !test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED,
+				    &mvm->status),
+			  "Failed to update SF upon disassociation\n");
+
+		/* If we get an assert during the connection (after the
+		 * station has been added, but before the vif is set
+		 * to associated), mac80211 will re-add the station and
+		 * then configure the vif. Since the vif is not
+		 * associated, we would remove the station here and
+		 * this would fail the recovery.
+		 */
+		iwl_mvm_mld_vif_delete_all_stas(mvm, vif);
+	}
+
+	iwl_mvm_bss_info_changed_station_assoc(mvm, vif, changes);
+}
+
+static void
+iwl_mvm_mld_link_info_changed_ap_ibss(struct iwl_mvm *mvm,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_bss_conf *link_conf,
+				      u64 changes)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	u32 link_changes = LINK_CONTEXT_MODIFY_PROTECT_FLAGS |
+			   LINK_CONTEXT_MODIFY_QOS_PARAMS;
+
+	/* Changes will be applied when the AP/IBSS is started */
+	if (!mvmvif->ap_ibss_active)
+		return;
+
+	if (link_conf->he_support)
+		link_changes |= LINK_CONTEXT_MODIFY_HE_PARAMS;
+
+	if (changes & (BSS_CHANGED_ERP_CTS_PROT | BSS_CHANGED_HT |
+		       BSS_CHANGED_BANDWIDTH | BSS_CHANGED_QOS |
+		       BSS_CHANGED_HE_BSS_COLOR) &&
+		       iwl_mvm_link_changed(mvm, vif, link_conf,
+					    link_changes, true))
+		IWL_ERR(mvm, "failed to update MAC %pM\n", vif->addr);
+
+	/* Need to send a new beacon template to the FW */
+	if (changes & BSS_CHANGED_BEACON &&
+	    iwl_mvm_mac_ctxt_beacon_changed(mvm, vif, link_conf))
+		IWL_WARN(mvm, "Failed updating beacon data\n");
+
+	/* FIXME: need to decide if we need FTM responder per link */
+	if (changes & BSS_CHANGED_FTM_RESPONDER) {
+		int ret = iwl_mvm_ftm_start_responder(mvm, vif);
+
+		if (ret)
+			IWL_WARN(mvm, "Failed to enable FTM responder (%d)\n",
+				 ret);
+	}
+}
+
+static void iwl_mvm_mld_link_info_changed(struct ieee80211_hw *hw,
+					  struct ieee80211_vif *vif,
+					  struct ieee80211_bss_conf *link_conf,
+					  u64 changes)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+
+	mutex_lock(&mvm->mutex);
+
+	switch (vif->type) {
+	case NL80211_IFTYPE_STATION:
+		iwl_mvm_mld_link_info_changed_station(mvm, vif, link_conf,
+						      changes);
+		break;
+	case NL80211_IFTYPE_AP:
+	case NL80211_IFTYPE_ADHOC:
+		iwl_mvm_mld_link_info_changed_ap_ibss(mvm, vif, link_conf,
+						      changes);
+		break;
+	case NL80211_IFTYPE_MONITOR:
+		if (changes & BSS_CHANGED_MU_GROUPS)
+			iwl_mvm_update_mu_groups(mvm, vif);
+		break;
+	default:
+		/* shouldn't happen */
+		WARN_ON_ONCE(1);
+	}
+
+	if (changes & BSS_CHANGED_TXPOWER) {
+		IWL_DEBUG_CALIB(mvm, "Changing TX Power to %d dBm\n",
+				link_conf->txpower);
+		iwl_mvm_set_tx_power(mvm, vif, link_conf->txpower);
+	}
+
+	mutex_unlock(&mvm->mutex);
+}
+
+static void iwl_mvm_mld_vif_cfg_changed(struct ieee80211_hw *hw,
+					struct ieee80211_vif *vif,
+					u64 changes)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+
+	mutex_lock(&mvm->mutex);
+
+	if (changes & BSS_CHANGED_IDLE && !vif->cfg.idle)
+		iwl_mvm_scan_stop(mvm, IWL_MVM_SCAN_SCHED, true);
+
+	if (vif->type == NL80211_IFTYPE_STATION)
+		iwl_mvm_mld_vif_cfg_changed_station(mvm, vif, changes);
+
+	mutex_unlock(&mvm->mutex);
+}
+
+static int
+iwl_mvm_mld_switch_vif_chanctx(struct ieee80211_hw *hw,
+			       struct ieee80211_vif_chanctx_switch *vifs,
+			       int n_vifs,
+			       enum ieee80211_chanctx_switch_mode mode)
+{
+	struct iwl_mvm_switch_vif_chanctx_ops ops = {
+		.__assign_vif_chanctx = __iwl_mvm_mld_assign_vif_chanctx,
+		.__unassign_vif_chanctx = __iwl_mvm_mld_unassign_vif_chanctx,
+	};
+
+	return iwl_mvm_switch_vif_chanctx_common(hw, vifs, n_vifs, mode, &ops);
+}
+
+static void iwl_mvm_mld_config_iface_filter(struct ieee80211_hw *hw,
+					    struct ieee80211_vif *vif,
+					    unsigned int filter_flags,
+					    unsigned int changed_flags)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+
+	/* We support only filter for probe requests */
+	if (!(changed_flags & FIF_PROBE_REQ))
+		return;
+
+	/* Supported only for p2p client interfaces */
+	if (vif->type != NL80211_IFTYPE_STATION || !vif->cfg.assoc ||
+	    !vif->p2p)
+		return;
+
+	mutex_lock(&mvm->mutex);
+	iwl_mvm_mld_mac_ctxt_changed(mvm, vif, false);
+	mutex_unlock(&mvm->mutex);
+}
+
+static int
+iwl_mvm_mld_mac_conf_tx(struct ieee80211_hw *hw,
+			struct ieee80211_vif *vif,
+			unsigned int link_id, u16 ac,
+			const struct ieee80211_tx_queue_params *params)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+
+	mvmvif->deflink.queue_params[ac] = *params;
+
+	/* No need to update right away, we'll get BSS_CHANGED_QOS
+	 * The exception is P2P_DEVICE interface which needs immediate update.
+	 */
+	if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
+		int ret;
+
+		mutex_lock(&mvm->mutex);
+		ret = iwl_mvm_link_changed(mvm, vif, &vif->bss_conf,
+					   LINK_CONTEXT_MODIFY_QOS_PARAMS,
+					   true);
+		mutex_unlock(&mvm->mutex);
+		return ret;
+	}
+	return 0;
+}
+
+static int iwl_mvm_link_switch_phy_ctx(struct iwl_mvm *mvm,
+				       struct ieee80211_vif *vif,
+				       struct iwl_mvm_phy_ctxt *new_phy_ctxt)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	int ret = 0;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	/* Inorder to change the phy_ctx of a link, the link needs to be
+	 * inactive. Therefore, first deactivate the link, then change its
+	 * phy_ctx, and then activate it again.
+	 */
+	ret = iwl_mvm_link_changed(mvm, vif, &vif->bss_conf,
+				   LINK_CONTEXT_MODIFY_ACTIVE, false);
+	if (WARN(ret, "Failed to deactivate link\n"))
+		return ret;
+
+	iwl_mvm_phy_ctxt_unref(mvm, mvmvif->deflink.phy_ctxt);
+
+	mvmvif->deflink.phy_ctxt = new_phy_ctxt;
+
+	ret = iwl_mvm_link_changed(mvm, vif, &vif->bss_conf, 0, false);
+	if (WARN(ret, "Failed to deactivate link\n"))
+		return ret;
+
+	ret = iwl_mvm_link_changed(mvm, vif, &vif->bss_conf,
+				   LINK_CONTEXT_MODIFY_ACTIVE, true);
+	WARN(ret, "Failed binding P2P_DEVICE\n");
+	return ret;
+}
+
+static int iwl_mvm_mld_roc(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			   struct ieee80211_channel *channel, int duration,
+			   enum ieee80211_roc_type type)
+{
+	struct iwl_mvm_roc_ops ops = {
+		.add_aux_sta_for_hs20 = iwl_mvm_mld_add_aux_sta,
+		.switch_phy_ctxt = iwl_mvm_link_switch_phy_ctx,
+	};
+
+	return iwl_mvm_roc_common(hw, vif, channel, duration, type, &ops);
+}
+
+static int
+iwl_mvm_mld_change_vif_links(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     u16 old_links, u16 new_links,
+			     struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS])
+{
+	struct iwl_mvm_vif_link_info *new_link[IEEE80211_MLD_MAX_NUM_LINKS] = {};
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	u16 removed = old_links & ~new_links;
+	u16 added = new_links & ~old_links;
+	int err, i;
+
+	if (hweight16(new_links) > 2) {
+		return -EOPNOTSUPP;
+	} else if (hweight16(new_links) > 1) {
+		unsigned int n_active = 0;
+
+		for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
+			struct ieee80211_bss_conf *link_conf;
+
+			link_conf = link_conf_dereference_protected(vif, i);
+			if (link_conf &&
+			    rcu_access_pointer(link_conf->chanctx_conf))
+				n_active++;
+		}
+
+		if (vif->type == NL80211_IFTYPE_AP &&
+		    n_active > mvm->fw->ucode_capa.num_beacons)
+			return -EOPNOTSUPP;
+		else if (n_active > 1)
+			return -EOPNOTSUPP;
+	}
+
+	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
+		int r;
+
+		if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
+			break;
+
+		if (!(added & BIT(i)))
+			continue;
+		new_link[i] = kzalloc(sizeof(*new_link[i]), GFP_KERNEL);
+		if (!new_link[i]) {
+			err = -ENOMEM;
+			goto free;
+		}
+
+		new_link[i]->bcast_sta.sta_id = IWL_MVM_INVALID_STA;
+		new_link[i]->mcast_sta.sta_id = IWL_MVM_INVALID_STA;
+		new_link[i]->ap_sta_id = IWL_MVM_INVALID_STA;
+		new_link[i]->fw_link_id = IWL_MVM_FW_LINK_ID_INVALID;
+
+		for (r = 0; r < NUM_IWL_MVM_SMPS_REQ; r++)
+			new_link[i]->smps_requests[r] =
+				IEEE80211_SMPS_AUTOMATIC;
+	}
+
+	mutex_lock(&mvm->mutex);
+
+	if (old_links == 0) {
+		err = iwl_mvm_disable_link(mvm, vif, &vif->bss_conf);
+		if (err)
+			goto out_err;
+		mvmvif->link[0] = NULL;
+	}
+
+	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
+		if (removed & BIT(i)) {
+			struct ieee80211_bss_conf *link_conf = old[i];
+
+			err = iwl_mvm_disable_link(mvm, vif, link_conf);
+			if (err)
+				goto out_err;
+			kfree(mvmvif->link[i]);
+			mvmvif->link[i] = NULL;
+		}
+
+		if (added & BIT(i)) {
+			struct ieee80211_bss_conf *link_conf;
+
+			link_conf = link_conf_dereference_protected(vif, i);
+			if (WARN_ON(!link_conf))
+				continue;
+
+			if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART,
+				      &mvm->status))
+				mvmvif->link[i] = new_link[i];
+			new_link[i] = NULL;
+			err = iwl_mvm_add_link(mvm, vif, link_conf);
+			if (err)
+				goto out_err;
+		}
+	}
+
+	err = 0;
+	if (new_links == 0) {
+		mvmvif->link[0] = &mvmvif->deflink;
+		err = iwl_mvm_add_link(mvm, vif, &vif->bss_conf);
+	}
+
+out_err:
+	/* we really don't have a good way to roll back here ... */
+	mutex_unlock(&mvm->mutex);
+
+free:
+	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++)
+		kfree(new_link[i]);
+	return err;
+}
+
+static int
+iwl_mvm_mld_change_sta_links(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_sta *sta,
+			     u16 old_links, u16 new_links)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+	int ret;
+
+	mutex_lock(&mvm->mutex);
+	ret = iwl_mvm_mld_update_sta_links(mvm, vif, sta, old_links, new_links);
+	mutex_unlock(&mvm->mutex);
+
+	return ret;
+}
+
+const struct ieee80211_ops iwl_mvm_mld_hw_ops = {
+	.tx = iwl_mvm_mac_tx,
+	.wake_tx_queue = iwl_mvm_mac_wake_tx_queue,
+	.ampdu_action = iwl_mvm_mac_ampdu_action,
+	.get_antenna = iwl_mvm_op_get_antenna,
+	.start = iwl_mvm_mac_start,
+	.reconfig_complete = iwl_mvm_mac_reconfig_complete,
+	.stop = iwl_mvm_mac_stop,
+	.add_interface = iwl_mvm_mld_mac_add_interface,
+	.remove_interface = iwl_mvm_mld_mac_remove_interface,
+	.config = iwl_mvm_mac_config,
+	.prepare_multicast = iwl_mvm_prepare_multicast,
+	.configure_filter = iwl_mvm_configure_filter,
+	.config_iface_filter = iwl_mvm_mld_config_iface_filter,
+	.link_info_changed = iwl_mvm_mld_link_info_changed,
+	.vif_cfg_changed = iwl_mvm_mld_vif_cfg_changed,
+	.hw_scan = iwl_mvm_mac_hw_scan,
+	.cancel_hw_scan = iwl_mvm_mac_cancel_hw_scan,
+	.sta_pre_rcu_remove = iwl_mvm_sta_pre_rcu_remove,
+	.sta_state = iwl_mvm_mld_mac_sta_state,
+	.sta_notify = iwl_mvm_mac_sta_notify,
+	.allow_buffered_frames = iwl_mvm_mac_allow_buffered_frames,
+	.release_buffered_frames = iwl_mvm_mac_release_buffered_frames,
+	.set_rts_threshold = iwl_mvm_mac_set_rts_threshold,
+	.sta_rc_update = iwl_mvm_sta_rc_update,
+	.conf_tx = iwl_mvm_mld_mac_conf_tx,
+	.mgd_prepare_tx = iwl_mvm_mac_mgd_prepare_tx,
+	.mgd_complete_tx = iwl_mvm_mac_mgd_complete_tx,
+	.mgd_protect_tdls_discover = iwl_mvm_mac_mgd_protect_tdls_discover,
+	.flush = iwl_mvm_mac_flush,
+	.sched_scan_start = iwl_mvm_mac_sched_scan_start,
+	.sched_scan_stop = iwl_mvm_mac_sched_scan_stop,
+	.set_key = iwl_mvm_mac_set_key,
+	.update_tkip_key = iwl_mvm_mac_update_tkip_key,
+	.remain_on_channel = iwl_mvm_mld_roc,
+	.cancel_remain_on_channel = iwl_mvm_cancel_roc,
+	.add_chanctx = iwl_mvm_add_chanctx,
+	.remove_chanctx = iwl_mvm_remove_chanctx,
+	.change_chanctx = iwl_mvm_change_chanctx,
+	.assign_vif_chanctx = iwl_mvm_mld_assign_vif_chanctx,
+	.unassign_vif_chanctx = iwl_mvm_mld_unassign_vif_chanctx,
+	.switch_vif_chanctx = iwl_mvm_mld_switch_vif_chanctx,
+
+	.start_ap = iwl_mvm_mld_start_ap,
+	.stop_ap = iwl_mvm_mld_stop_ap,
+	.join_ibss = iwl_mvm_mld_start_ibss,
+	.leave_ibss = iwl_mvm_mld_stop_ibss,
+
+	.tx_last_beacon = iwl_mvm_tx_last_beacon,
+
+	.set_tim = iwl_mvm_set_tim,
+
+	.channel_switch = iwl_mvm_channel_switch,
+	.pre_channel_switch = iwl_mvm_pre_channel_switch,
+	.post_channel_switch = iwl_mvm_post_channel_switch,
+	.abort_channel_switch = iwl_mvm_abort_channel_switch,
+	.channel_switch_rx_beacon = iwl_mvm_channel_switch_rx_beacon,
+
+	.tdls_channel_switch = iwl_mvm_tdls_channel_switch,
+	.tdls_cancel_channel_switch = iwl_mvm_tdls_cancel_channel_switch,
+	.tdls_recv_channel_switch = iwl_mvm_tdls_recv_channel_switch,
+
+	.event_callback = iwl_mvm_mac_event_callback,
+
+	.sync_rx_queues = iwl_mvm_sync_rx_queues,
+
+	CFG80211_TESTMODE_CMD(iwl_mvm_mac_testmode_cmd)
+
+#ifdef CONFIG_PM_SLEEP
+	/* look at d3.c */
+	.suspend = iwl_mvm_suspend,
+	.resume = iwl_mvm_resume,
+	.set_wakeup = iwl_mvm_set_wakeup,
+	.set_rekey_data = iwl_mvm_set_rekey_data,
+#if IS_ENABLED(CONFIG_IPV6)
+	.ipv6_addr_change = iwl_mvm_ipv6_addr_change,
+#endif
+	.set_default_unicast_key = iwl_mvm_set_default_unicast_key,
+#endif
+	.get_survey = iwl_mvm_mac_get_survey,
+	.sta_statistics = iwl_mvm_mac_sta_statistics,
+	.get_ftm_responder_stats = iwl_mvm_mac_get_ftm_responder_stats,
+	.start_pmsr = iwl_mvm_start_pmsr,
+	.abort_pmsr = iwl_mvm_abort_pmsr,
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	.sta_add_debugfs = iwl_mvm_sta_add_debugfs,
+#endif
+	.set_hw_timestamp = iwl_mvm_set_hw_timestamp,
+
+	.change_vif_links = iwl_mvm_mld_change_vif_links,
+	.change_sta_links = iwl_mvm_mld_change_sta_links,
+};
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c
new file mode 100644
index 000000000000..0bfdf4462755
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c
@@ -0,0 +1,1167 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2022 Intel Corporation
+ */
+#include "mvm.h"
+#include "time-sync.h"
+#include "sta.h"
+
+u32 iwl_mvm_sta_fw_id_mask(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
+			   int filter_link_id)
+{
+	struct iwl_mvm_sta *mvmsta;
+	unsigned int link_id;
+	u32 result = 0;
+
+	if (!sta)
+		return 0;
+
+	mvmsta = iwl_mvm_sta_from_mac80211(sta);
+
+	/* it's easy when the STA is not an MLD */
+	if (!sta->valid_links)
+		return BIT(mvmsta->deflink.sta_id);
+
+	/* but if it is an MLD, get the mask of all the FW STAs it has ... */
+	for (link_id = 0; link_id < ARRAY_SIZE(mvmsta->link); link_id++) {
+		struct iwl_mvm_link_sta *link_sta;
+
+		/* unless we have a specific link in mind */
+		if (filter_link_id >= 0 && link_id != filter_link_id)
+			continue;
+
+		link_sta =
+			rcu_dereference_check(mvmsta->link[link_id],
+					      lockdep_is_held(&mvm->mutex));
+		if (!link_sta)
+			continue;
+
+		result |= BIT(link_sta->sta_id);
+	}
+
+	return result;
+}
+
+static int iwl_mvm_mld_send_sta_cmd(struct iwl_mvm *mvm,
+				    struct iwl_mvm_sta_cfg_cmd *cmd)
+{
+	int ret = iwl_mvm_send_cmd_pdu(mvm,
+				       WIDE_ID(MAC_CONF_GROUP, STA_CONFIG_CMD),
+				       0, sizeof(*cmd), cmd);
+	if (ret)
+		IWL_ERR(mvm, "STA_CONFIG_CMD send failed, ret=0x%x\n", ret);
+	return ret;
+}
+
+/*
+ * Add an internal station to the FW table
+ */
+static int iwl_mvm_mld_add_int_sta_to_fw(struct iwl_mvm *mvm,
+					 struct iwl_mvm_int_sta *sta,
+					 const u8 *addr, int link_id)
+{
+	struct iwl_mvm_sta_cfg_cmd cmd;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.sta_id = cpu_to_le32((u8)sta->sta_id);
+
+	cmd.link_id = cpu_to_le32(link_id);
+
+	cmd.station_type = cpu_to_le32(sta->type);
+
+	if (addr) {
+		memcpy(cmd.peer_mld_address, addr, ETH_ALEN);
+		memcpy(cmd.peer_link_address, addr, ETH_ALEN);
+	}
+
+	return iwl_mvm_mld_send_sta_cmd(mvm, &cmd);
+}
+
+/*
+ * Remove a station from the FW table. Before sending the command to remove
+ * the station validate that the station is indeed known to the driver (sanity
+ * only).
+ */
+static int iwl_mvm_mld_rm_sta_from_fw(struct iwl_mvm *mvm, u32 sta_id)
+{
+	struct iwl_mvm_remove_sta_cmd rm_sta_cmd = {
+		.sta_id = cpu_to_le32(sta_id),
+	};
+	int ret;
+
+	/* Note: internal stations are marked as error values */
+	if (!rcu_access_pointer(mvm->fw_id_to_mac_id[sta_id])) {
+		IWL_ERR(mvm, "Invalid station id %d\n", sta_id);
+		return -EINVAL;
+	}
+
+	ret = iwl_mvm_send_cmd_pdu(mvm, WIDE_ID(MAC_CONF_GROUP, STA_REMOVE_CMD),
+				   0, sizeof(rm_sta_cmd), &rm_sta_cmd);
+	if (ret) {
+		IWL_ERR(mvm, "Failed to remove station. Id=%d\n", sta_id);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int iwl_mvm_add_aux_sta_to_fw(struct iwl_mvm *mvm,
+				     struct iwl_mvm_int_sta *sta,
+				     u32 lmac_id)
+{
+	int ret;
+
+	struct iwl_mvm_aux_sta_cmd cmd = {
+		.sta_id = cpu_to_le32(sta->sta_id),
+		.lmac_id = cpu_to_le32(lmac_id),
+	};
+
+	ret = iwl_mvm_send_cmd_pdu(mvm, WIDE_ID(MAC_CONF_GROUP, AUX_STA_CMD),
+				   0, sizeof(cmd), &cmd);
+	if (ret)
+		IWL_ERR(mvm, "Failed to send AUX_STA_CMD\n");
+	return ret;
+}
+
+/*
+ * Adds an internal sta to the FW table with its queues
+ */
+static int iwl_mvm_mld_add_int_sta_with_queue(struct iwl_mvm *mvm,
+					      struct iwl_mvm_int_sta *sta,
+					      const u8 *addr, int link_id,
+					      u16 *queue, u8 tid,
+					      unsigned int *_wdg_timeout)
+{
+	int ret, txq;
+	unsigned int wdg_timeout = _wdg_timeout ? *_wdg_timeout :
+		mvm->trans->trans_cfg->base_params->wd_timeout;
+
+	if (WARN_ON_ONCE(sta->sta_id == IWL_MVM_INVALID_STA))
+		return -ENOSPC;
+
+	if (sta->type == STATION_TYPE_AUX)
+		ret = iwl_mvm_add_aux_sta_to_fw(mvm, sta, link_id);
+	else
+		ret = iwl_mvm_mld_add_int_sta_to_fw(mvm, sta, addr, link_id);
+	if (ret)
+		return ret;
+
+	/*
+	 * For 22000 firmware and on we cannot add queue to a station unknown
+	 * to firmware so enable queue here - after the station was added
+	 */
+	txq = iwl_mvm_tvqm_enable_txq(mvm, NULL, sta->sta_id, tid,
+				      wdg_timeout);
+	if (txq < 0) {
+		iwl_mvm_mld_rm_sta_from_fw(mvm, sta->sta_id);
+		return txq;
+	}
+	*queue = txq;
+
+	return 0;
+}
+
+/*
+ * Adds a new int sta: allocate it in the driver, add it to the FW table,
+ * and add its queues.
+ */
+static int iwl_mvm_mld_add_int_sta(struct iwl_mvm *mvm,
+				   struct iwl_mvm_int_sta *int_sta, u16 *queue,
+				   enum nl80211_iftype iftype,
+				   enum iwl_fw_sta_type sta_type,
+				   int link_id, const u8 *addr, u8 tid,
+				   unsigned int *wdg_timeout)
+{
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	/* qmask argument is not used in the new tx api, send a don't care */
+	ret = iwl_mvm_allocate_int_sta(mvm, int_sta, 0, iftype,
+				       sta_type);
+	if (ret)
+		return ret;
+
+	ret = iwl_mvm_mld_add_int_sta_with_queue(mvm, int_sta, addr, link_id,
+						 queue, tid, wdg_timeout);
+	if (ret) {
+		iwl_mvm_dealloc_int_sta(mvm, int_sta);
+		return ret;
+	}
+
+	return 0;
+}
+
+/* Allocate a new station entry for the broadcast station to the given vif,
+ * and send it to the FW.
+ * Note that each P2P mac should have its own broadcast station.
+ */
+int iwl_mvm_mld_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_vif_link_info *mvm_link =
+		mvmvif->link[link_conf->link_id];
+	struct iwl_mvm_int_sta *bsta = &mvm_link->bcast_sta;
+	static const u8 _baddr[] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
+	const u8 *baddr = _baddr;
+	unsigned int wdg_timeout =
+		iwl_mvm_get_wd_timeout(mvm, vif, false, false);
+	u16 *queue;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (vif->type == NL80211_IFTYPE_ADHOC)
+		baddr = link_conf->bssid;
+
+	if (vif->type == NL80211_IFTYPE_AP ||
+	    vif->type == NL80211_IFTYPE_ADHOC) {
+		queue = &mvm_link->mgmt_queue;
+	} else if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
+		queue = &mvm->p2p_dev_queue;
+	} else {
+		WARN(1, "Missing required TXQ for adding bcast STA\n");
+		return -EINVAL;
+	}
+
+	return iwl_mvm_mld_add_int_sta(mvm, bsta, queue,
+				       ieee80211_vif_type_p2p(vif),
+				       STATION_TYPE_BCAST_MGMT,
+				       mvm_link->fw_link_id, baddr,
+				       IWL_MAX_TID_COUNT, &wdg_timeout);
+}
+
+/* Allocate a new station entry for the broadcast station to the given vif,
+ * and send it to the FW.
+ * Note that each AP/GO mac should have its own multicast station.
+ */
+int iwl_mvm_mld_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_vif_link_info *mvm_link =
+		mvmvif->link[link_conf->link_id];
+	struct iwl_mvm_int_sta *msta = &mvm_link->mcast_sta;
+	static const u8 _maddr[] = {0x03, 0x00, 0x00, 0x00, 0x00, 0x00};
+	const u8 *maddr = _maddr;
+	unsigned int timeout = iwl_mvm_get_wd_timeout(mvm, vif, false, false);
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_AP &&
+		    vif->type != NL80211_IFTYPE_ADHOC))
+		return -EOPNOTSUPP;
+
+	/* In IBSS, ieee80211_check_queues() sets the cab_queue to be
+	 * invalid, so make sure we use the queue we want.
+	 * Note that this is done here as we want to avoid making DQA
+	 * changes in mac80211 layer.
+	 */
+	if (vif->type == NL80211_IFTYPE_ADHOC)
+		mvm_link->cab_queue = IWL_MVM_DQA_GCAST_QUEUE;
+
+	return iwl_mvm_mld_add_int_sta(mvm, msta, &mvm_link->cab_queue,
+				       vif->type, STATION_TYPE_MCAST,
+				       mvm_link->fw_link_id, maddr, 0,
+				       &timeout);
+}
+
+/* Allocate a new station entry for the sniffer station to the given vif,
+ * and send it to the FW.
+ */
+int iwl_mvm_mld_add_snif_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_vif_link_info *mvm_link =
+		mvmvif->link[link_conf->link_id];
+
+	lockdep_assert_held(&mvm->mutex);
+
+	return iwl_mvm_mld_add_int_sta(mvm, &mvm->snif_sta, &mvm->snif_queue,
+				       vif->type, STATION_TYPE_BCAST_MGMT,
+				       mvm_link->fw_link_id, NULL,
+				       IWL_MAX_TID_COUNT, NULL);
+}
+
+int iwl_mvm_mld_add_aux_sta(struct iwl_mvm *mvm, u32 lmac_id)
+{
+	lockdep_assert_held(&mvm->mutex);
+
+	/* In CDB NICs we need to specify which lmac to use for aux activity;
+	 * use the link_id argument place to send lmac_id to the function.
+	 */
+	return iwl_mvm_mld_add_int_sta(mvm, &mvm->aux_sta, &mvm->aux_queue,
+				       NL80211_IFTYPE_UNSPECIFIED,
+				       STATION_TYPE_AUX, lmac_id, NULL,
+				       IWL_MAX_TID_COUNT, NULL);
+}
+
+static int iwl_mvm_mld_disable_txq(struct iwl_mvm *mvm, u32 sta_mask,
+				   u16 *queueptr, u8 tid)
+{
+	int queue = *queueptr;
+	int ret = 0;
+
+	if (tid == IWL_MAX_TID_COUNT)
+		tid = IWL_MGMT_TID;
+
+	if (mvm->sta_remove_requires_queue_remove) {
+		u32 cmd_id = WIDE_ID(DATA_PATH_GROUP,
+				     SCD_QUEUE_CONFIG_CMD);
+		struct iwl_scd_queue_cfg_cmd remove_cmd = {
+			.operation = cpu_to_le32(IWL_SCD_QUEUE_REMOVE),
+			.u.remove.tid = cpu_to_le32(tid),
+			.u.remove.sta_mask = cpu_to_le32(sta_mask),
+		};
+
+		ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0,
+					   sizeof(remove_cmd),
+					   &remove_cmd);
+	}
+
+	iwl_trans_txq_free(mvm->trans, queue);
+	*queueptr = IWL_MVM_INVALID_QUEUE;
+
+	return ret;
+}
+
+/* Removes a sta from the FW table, disable its queues, and dealloc it
+ */
+static int iwl_mvm_mld_rm_int_sta(struct iwl_mvm *mvm,
+				  struct iwl_mvm_int_sta *int_sta,
+				  bool flush, u8 tid, u16 *queuptr)
+{
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (WARN_ON_ONCE(int_sta->sta_id == IWL_MVM_INVALID_STA))
+		return -EINVAL;
+
+	if (flush)
+		iwl_mvm_flush_sta(mvm, int_sta, true);
+
+	iwl_mvm_mld_disable_txq(mvm, BIT(int_sta->sta_id), queuptr, tid);
+
+	ret = iwl_mvm_mld_rm_sta_from_fw(mvm, int_sta->sta_id);
+	if (ret)
+		IWL_WARN(mvm, "Failed sending remove station\n");
+
+	iwl_mvm_dealloc_int_sta(mvm, int_sta);
+
+	return ret;
+}
+
+int iwl_mvm_mld_rm_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_vif_link_info *link = mvmvif->link[link_conf->link_id];
+	u16 *queueptr;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	switch (vif->type) {
+	case NL80211_IFTYPE_AP:
+	case NL80211_IFTYPE_ADHOC:
+		queueptr = &link->mgmt_queue;
+		break;
+	case NL80211_IFTYPE_P2P_DEVICE:
+		queueptr = &mvm->p2p_dev_queue;
+		break;
+	default:
+		WARN(1, "Can't free bcast queue on vif type %d\n",
+		     vif->type);
+		return -EINVAL;
+	}
+
+	return iwl_mvm_mld_rm_int_sta(mvm, &link->bcast_sta,
+				      true, IWL_MAX_TID_COUNT, queueptr);
+}
+
+/* Send the FW a request to remove the station from it's internal data
+ * structures, and in addition remove it from the local data structure.
+ */
+int iwl_mvm_mld_rm_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_vif_link_info *link = mvmvif->link[link_conf->link_id];
+
+	lockdep_assert_held(&mvm->mutex);
+
+	return iwl_mvm_mld_rm_int_sta(mvm, &link->mcast_sta, true, 0,
+				      &link->cab_queue);
+}
+
+int iwl_mvm_mld_rm_snif_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
+{
+	lockdep_assert_held(&mvm->mutex);
+
+	return iwl_mvm_mld_rm_int_sta(mvm, &mvm->snif_sta, false,
+				      IWL_MAX_TID_COUNT, &mvm->snif_queue);
+}
+
+int iwl_mvm_mld_rm_aux_sta(struct iwl_mvm *mvm)
+{
+	lockdep_assert_held(&mvm->mutex);
+
+	return iwl_mvm_mld_rm_int_sta(mvm, &mvm->aux_sta, false,
+				      IWL_MAX_TID_COUNT, &mvm->aux_queue);
+}
+
+/* send a cfg sta command to add/update a sta in firmware */
+static int iwl_mvm_mld_cfg_sta(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
+			       struct ieee80211_vif *vif,
+			       struct ieee80211_link_sta *link_sta,
+			       struct ieee80211_bss_conf *link_conf,
+			       struct iwl_mvm_link_sta *mvm_link_sta)
+{
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_mvm_vif *mvm_vif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_vif_link_info *link_info =
+					mvm_vif->link[link_conf->link_id];
+	struct iwl_mvm_sta_cfg_cmd cmd = {
+		.sta_id = cpu_to_le32(mvm_link_sta->sta_id),
+		.station_type = cpu_to_le32(mvm_sta->sta_type),
+	};
+	u32 agg_size = 0, mpdu_dens = 0;
+
+	/* when adding sta, link should exist in FW */
+	if (WARN_ON(link_info->fw_link_id == IWL_MVM_FW_LINK_ID_INVALID))
+		return -EINVAL;
+
+	cmd.link_id = cpu_to_le32(link_info->fw_link_id);
+
+	memcpy(&cmd.peer_mld_address, sta->addr, ETH_ALEN);
+	memcpy(&cmd.peer_link_address, link_sta->addr, ETH_ALEN);
+
+	if (mvm_sta->sta_state >= IEEE80211_STA_ASSOC)
+		cmd.assoc_id = cpu_to_le32(sta->aid);
+
+	switch (link_sta->rx_nss) {
+	case 1:
+		cmd.mimo = cpu_to_le32(0);
+		break;
+	case 2 ... 8:
+		cmd.mimo = cpu_to_le32(1);
+		break;
+	}
+
+	switch (sta->deflink.smps_mode) {
+	case IEEE80211_SMPS_AUTOMATIC:
+	case IEEE80211_SMPS_NUM_MODES:
+		WARN_ON(1);
+		break;
+	case IEEE80211_SMPS_STATIC:
+		/* override NSS */
+		cmd.mimo = cpu_to_le32(0);
+		break;
+	case IEEE80211_SMPS_DYNAMIC:
+		cmd.mimo_protection = cpu_to_le32(1);
+		break;
+	case IEEE80211_SMPS_OFF:
+		/* nothing */
+		break;
+	}
+
+	mpdu_dens = iwl_mvm_get_sta_ampdu_dens(link_sta, link_conf, &agg_size);
+	cmd.tx_ampdu_spacing = cpu_to_le32(mpdu_dens);
+	cmd.tx_ampdu_max_size = cpu_to_le32(agg_size);
+
+	if (sta->wme) {
+		cmd.sp_length =
+			cpu_to_le32(sta->max_sp ? sta->max_sp * 2 : 128);
+		cmd.uapsd_acs = cpu_to_le32(iwl_mvm_get_sta_uapsd_acs(sta));
+	}
+
+	if (link_sta->he_cap.has_he) {
+		cmd.trig_rnd_alloc =
+			cpu_to_le32(link_conf->uora_exists ? 1 : 0);
+
+		/* PPE Thresholds */
+		iwl_mvm_set_sta_pkt_ext(mvm, link_sta, &cmd.pkt_ext);
+
+		/* HTC flags */
+		cmd.htc_flags = iwl_mvm_get_sta_htc_flags(sta, link_sta);
+
+		if (link_sta->he_cap.he_cap_elem.mac_cap_info[2] &
+		    IEEE80211_HE_MAC_CAP2_ACK_EN)
+			cmd.ack_enabled = cpu_to_le32(1);
+	}
+
+	return iwl_mvm_mld_send_sta_cmd(mvm, &cmd);
+}
+
+static void iwl_mvm_mld_free_sta_link(struct iwl_mvm *mvm,
+				      struct iwl_mvm_sta *mvm_sta,
+				      struct iwl_mvm_link_sta *mvm_sta_link,
+				      unsigned int link_id,
+				      bool is_in_fw)
+{
+	RCU_INIT_POINTER(mvm->fw_id_to_mac_id[mvm_sta_link->sta_id],
+			 is_in_fw ? ERR_PTR(-EINVAL) : NULL);
+	RCU_INIT_POINTER(mvm->fw_id_to_link_sta[mvm_sta_link->sta_id], NULL);
+	RCU_INIT_POINTER(mvm_sta->link[link_id], NULL);
+
+	if (mvm_sta_link != &mvm_sta->deflink)
+		kfree_rcu(mvm_sta_link, rcu_head);
+}
+
+static void iwl_mvm_mld_sta_rm_all_sta_links(struct iwl_mvm *mvm,
+					     struct iwl_mvm_sta *mvm_sta)
+{
+	unsigned int link_id;
+
+	for (link_id = 0; link_id < ARRAY_SIZE(mvm_sta->link); link_id++) {
+		struct iwl_mvm_link_sta *link =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+
+		if (!link)
+			continue;
+
+		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, link, link_id, false);
+	}
+}
+
+static int iwl_mvm_mld_alloc_sta_link(struct iwl_mvm *mvm,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_sta *sta,
+				      unsigned int link_id)
+{
+	struct ieee80211_link_sta *link_sta =
+		link_sta_dereference_protected(sta, link_id);
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_mvm_link_sta *link;
+	u32 sta_id = iwl_mvm_find_free_sta_id(mvm,
+					  ieee80211_vif_type_p2p(vif));
+
+	if (sta_id == IWL_MVM_INVALID_STA)
+		return -ENOSPC;
+
+	if (rcu_access_pointer(sta->link[link_id]) == &sta->deflink) {
+		link = &mvm_sta->deflink;
+	} else {
+		link = kzalloc(sizeof(*link), GFP_KERNEL);
+		if (!link)
+			return -ENOMEM;
+	}
+
+	link->sta_id = sta_id;
+	rcu_assign_pointer(mvm_sta->link[link_id], link);
+	rcu_assign_pointer(mvm->fw_id_to_mac_id[link->sta_id], sta);
+	rcu_assign_pointer(mvm->fw_id_to_link_sta[link->sta_id],
+			   link_sta);
+
+	return 0;
+}
+
+/* allocate all the links of a sta, called when the station is first added */
+static int iwl_mvm_mld_alloc_sta_links(struct iwl_mvm *mvm,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_sta *sta)
+{
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	unsigned int link_id;
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	for (link_id = 0; link_id < ARRAY_SIZE(sta->link); link_id++) {
+		if (!rcu_access_pointer(sta->link[link_id]) ||
+		    mvm_sta->link[link_id])
+			continue;
+
+		ret = iwl_mvm_mld_alloc_sta_link(mvm, vif, sta, link_id);
+		if (ret)
+			goto err;
+	}
+
+	return 0;
+
+err:
+	iwl_mvm_mld_sta_rm_all_sta_links(mvm, mvm_sta);
+	return ret;
+}
+
+static void iwl_mvm_mld_set_ap_sta_id(struct ieee80211_sta *sta,
+				      struct iwl_mvm_vif_link_info *vif_link,
+				      struct iwl_mvm_link_sta *sta_link)
+{
+	if (!sta->tdls) {
+		WARN_ON(vif_link->ap_sta_id != IWL_MVM_INVALID_STA);
+		vif_link->ap_sta_id = sta_link->sta_id;
+	} else {
+		WARN_ON(vif_link->ap_sta_id == IWL_MVM_INVALID_STA);
+	}
+}
+
+/* FIXME: consider waiting for mac80211 to add the STA instead of allocating
+ * queues here
+ */
+static int iwl_mvm_alloc_sta_after_restart(struct iwl_mvm *mvm,
+					   struct ieee80211_vif *vif,
+					   struct ieee80211_sta *sta)
+{
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
+	/* no active link found */
+	int ret = -EINVAL;
+	int sta_id;
+
+	/* First add an empty station since allocating a queue requires
+	 * a valid station. Since we need a link_id to allocate a station,
+	 * pick up the first valid one.
+	 */
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct iwl_mvm_vif_link_info *mvm_link;
+		struct ieee80211_bss_conf *link_conf =
+			link_conf_dereference_protected(vif, link_id);
+		struct iwl_mvm_link_sta *mvm_link_sta =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+
+		if (!link_conf)
+			continue;
+
+		mvm_link = mvmvif->link[link_conf->link_id];
+
+		if (!mvm_link || !mvm_link_sta)
+			continue;
+
+		sta_id = mvm_link_sta->sta_id;
+		ret = iwl_mvm_mld_cfg_sta(mvm, sta, vif, link_sta,
+					  link_conf, mvm_link_sta);
+		if (ret)
+			return ret;
+
+		rcu_assign_pointer(mvm->fw_id_to_mac_id[sta_id], sta);
+		rcu_assign_pointer(mvm->fw_id_to_link_sta[sta_id], link_sta);
+		ret = 0;
+	}
+
+	iwl_mvm_realloc_queues_after_restart(mvm, sta);
+
+	return ret;
+}
+
+int iwl_mvm_mld_add_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			struct ieee80211_sta *sta)
+{
+	struct iwl_mvm_vif *mvm_vif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	unsigned long link_sta_added_to_fw = 0;
+	struct ieee80211_link_sta *link_sta;
+	int ret = 0;
+	unsigned int link_id;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
+		ret = iwl_mvm_mld_alloc_sta_links(mvm, vif, sta);
+		if (ret)
+			return ret;
+	}
+
+	spin_lock_init(&mvm_sta->lock);
+
+	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
+		ret = iwl_mvm_alloc_sta_after_restart(mvm, vif, sta);
+	else
+		ret = iwl_mvm_sta_init(mvm, vif, sta, IWL_MVM_INVALID_STA,
+				       STATION_TYPE_PEER);
+	if (ret)
+		goto err;
+
+	/* at this stage sta link pointers are already allocated */
+	ret = iwl_mvm_mld_update_sta(mvm, vif, sta);
+
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct ieee80211_bss_conf *link_conf =
+			link_conf_dereference_protected(vif, link_id);
+		struct iwl_mvm_link_sta *mvm_link_sta =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+
+		if (WARN_ON(!link_conf || !mvm_link_sta))
+			goto err;
+
+		ret = iwl_mvm_mld_cfg_sta(mvm, sta, vif, link_sta, link_conf,
+					  mvm_link_sta);
+		if (ret)
+			goto err;
+
+		link_sta_added_to_fw |= BIT(link_id);
+
+		if (vif->type == NL80211_IFTYPE_STATION)
+			iwl_mvm_mld_set_ap_sta_id(sta, mvm_vif->link[link_id],
+						  mvm_link_sta);
+	}
+
+	return 0;
+
+err:
+	/* remove all already allocated stations in FW */
+	for_each_set_bit(link_id, &link_sta_added_to_fw,
+			 IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct iwl_mvm_link_sta *mvm_link_sta =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+
+		iwl_mvm_mld_rm_sta_from_fw(mvm, mvm_link_sta->sta_id);
+	}
+
+	/* free all sta resources in the driver */
+	iwl_mvm_mld_sta_rm_all_sta_links(mvm, mvm_sta);
+	return ret;
+}
+
+int iwl_mvm_mld_update_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			   struct ieee80211_sta *sta)
+{
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
+	int ret = 0;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct ieee80211_bss_conf *link_conf =
+			link_conf_dereference_protected(vif, link_id);
+		struct iwl_mvm_link_sta *mvm_link_sta =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+
+		if (WARN_ON(!link_conf || !mvm_link_sta))
+			return -EINVAL;
+
+		ret = iwl_mvm_mld_cfg_sta(mvm, sta, vif, link_sta, link_conf,
+					  mvm_link_sta);
+
+		if (ret) {
+			IWL_ERR(mvm, "Failed to update sta link %d\n", link_id);
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static void iwl_mvm_mld_disable_sta_queues(struct iwl_mvm *mvm,
+					   struct ieee80211_vif *vif,
+					   struct ieee80211_sta *sta)
+{
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	u32 sta_mask = iwl_mvm_sta_fw_id_mask(mvm, sta, -1);
+	int i;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	for (i = 0; i < ARRAY_SIZE(mvm_sta->tid_data); i++) {
+		if (mvm_sta->tid_data[i].txq_id == IWL_MVM_INVALID_QUEUE)
+			continue;
+
+		iwl_mvm_mld_disable_txq(mvm, sta_mask,
+					&mvm_sta->tid_data[i].txq_id, i);
+		mvm_sta->tid_data[i].txq_id = IWL_MVM_INVALID_QUEUE;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sta->txq); i++) {
+		struct iwl_mvm_txq *mvmtxq =
+			iwl_mvm_txq_from_mac80211(sta->txq[i]);
+
+		mvmtxq->txq_id = IWL_MVM_INVALID_QUEUE;
+	}
+}
+
+int iwl_mvm_mld_rm_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+		       struct ieee80211_sta *sta)
+{
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	kfree(mvm_sta->dup_data);
+
+	/* flush its queues here since we are freeing mvm_sta */
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct iwl_mvm_link_sta *mvm_link_sta =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+
+		if (WARN_ON(!mvm_link_sta))
+			return -EINVAL;
+
+		ret = iwl_mvm_flush_sta_tids(mvm, mvm_link_sta->sta_id,
+					     0xffff);
+		if (ret)
+			return ret;
+	}
+
+	ret = iwl_mvm_wait_sta_queues_empty(mvm, mvm_sta);
+	if (ret)
+		return ret;
+
+	iwl_mvm_mld_disable_sta_queues(mvm, vif, sta);
+
+	for_each_sta_active_link(vif, sta, link_sta, link_id) {
+		struct iwl_mvm_link_sta *mvm_link_sta =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+		bool stay_in_fw;
+
+		stay_in_fw = iwl_mvm_sta_del(mvm, vif, sta, link_sta, &ret);
+		if (ret)
+			break;
+
+		if (!stay_in_fw)
+			ret = iwl_mvm_mld_rm_sta_from_fw(mvm,
+							 mvm_link_sta->sta_id);
+
+		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_link_sta,
+					  link_id, stay_in_fw);
+	}
+
+	return ret;
+}
+
+int iwl_mvm_mld_rm_sta_id(struct iwl_mvm *mvm, u8 sta_id)
+{
+	int ret = iwl_mvm_mld_rm_sta_from_fw(mvm, sta_id);
+
+	lockdep_assert_held(&mvm->mutex);
+
+	RCU_INIT_POINTER(mvm->fw_id_to_mac_id[sta_id], NULL);
+	RCU_INIT_POINTER(mvm->fw_id_to_link_sta[sta_id], NULL);
+	return ret;
+}
+
+void iwl_mvm_mld_sta_modify_disable_tx(struct iwl_mvm *mvm,
+				       struct iwl_mvm_sta *mvmsta,
+				       bool disable)
+{
+	struct iwl_mvm_sta_disable_tx_cmd cmd;
+	int ret;
+
+	cmd.sta_id = cpu_to_le32(mvmsta->deflink.sta_id);
+	cmd.disable = cpu_to_le32(disable);
+
+	ret = iwl_mvm_send_cmd_pdu(mvm,
+				   WIDE_ID(MAC_CONF_GROUP, STA_DISABLE_TX_CMD),
+				   CMD_ASYNC, sizeof(cmd), &cmd);
+	if (ret)
+		IWL_ERR(mvm,
+			"Failed to send STA_DISABLE_TX_CMD command (%d)\n",
+			ret);
+}
+
+void iwl_mvm_mld_sta_modify_disable_tx_ap(struct iwl_mvm *mvm,
+					  struct ieee80211_sta *sta,
+					  bool disable)
+{
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+
+	spin_lock_bh(&mvm_sta->lock);
+
+	if (mvm_sta->disable_tx == disable) {
+		spin_unlock_bh(&mvm_sta->lock);
+		return;
+	}
+
+	iwl_mvm_mld_sta_modify_disable_tx(mvm, mvm_sta, disable);
+
+	spin_unlock_bh(&mvm_sta->lock);
+}
+
+void iwl_mvm_mld_modify_all_sta_disable_tx(struct iwl_mvm *mvm,
+					   struct iwl_mvm_vif *mvmvif,
+					   bool disable)
+{
+	struct ieee80211_sta *sta;
+	struct iwl_mvm_sta *mvm_sta;
+	int i;
+
+	rcu_read_lock();
+
+	/* Block/unblock all the stations of the given mvmvif */
+	for (i = 0; i < mvm->fw->ucode_capa.num_stations; i++) {
+		sta = rcu_dereference(mvm->fw_id_to_mac_id[i]);
+		if (IS_ERR_OR_NULL(sta))
+			continue;
+
+		mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+		if (mvm_sta->mac_id_n_color !=
+		    FW_CMD_ID_AND_COLOR(mvmvif->id, mvmvif->color))
+			continue;
+
+		iwl_mvm_mld_sta_modify_disable_tx(mvm, mvm_sta, disable);
+	}
+
+	rcu_read_unlock();
+}
+
+static int iwl_mvm_mld_update_sta_queues(struct iwl_mvm *mvm,
+					 struct ieee80211_sta *sta,
+					 u32 old_sta_mask,
+					 u32 new_sta_mask)
+{
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_scd_queue_cfg_cmd cmd = {
+		.operation = cpu_to_le32(IWL_SCD_QUEUE_MODIFY),
+		.u.modify.old_sta_mask = cpu_to_le32(old_sta_mask),
+		.u.modify.new_sta_mask = cpu_to_le32(new_sta_mask),
+	};
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(DATA_PATH_GROUP, SCD_QUEUE_CONFIG_CMD),
+		.len[0] = sizeof(cmd),
+		.data[0] = &cmd
+	};
+	int tid;
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	for (tid = 0; tid <= IWL_MAX_TID_COUNT; tid++) {
+		struct iwl_mvm_tid_data *tid_data = &mvm_sta->tid_data[tid];
+		int txq_id = tid_data->txq_id;
+
+		if (txq_id == IWL_MVM_INVALID_QUEUE)
+			continue;
+
+		if (tid == IWL_MAX_TID_COUNT)
+			cmd.u.modify.tid = cpu_to_le32(IWL_MGMT_TID);
+		else
+			cmd.u.modify.tid = cpu_to_le32(tid);
+
+		ret = iwl_mvm_send_cmd(mvm, &hcmd);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int iwl_mvm_mld_update_sta_baids(struct iwl_mvm *mvm,
+					u32 old_sta_mask,
+					u32 new_sta_mask)
+{
+	struct iwl_rx_baid_cfg_cmd cmd = {
+		.action = cpu_to_le32(IWL_RX_BAID_ACTION_MODIFY),
+		.modify.old_sta_id_mask = cpu_to_le32(old_sta_mask),
+		.modify.new_sta_id_mask = cpu_to_le32(new_sta_mask),
+	};
+	u32 cmd_id = WIDE_ID(DATA_PATH_GROUP, RX_BAID_ALLOCATION_CONFIG_CMD);
+	int baid;
+
+	BUILD_BUG_ON(sizeof(struct iwl_rx_baid_cfg_resp) != sizeof(baid));
+
+	for (baid = 0; baid < ARRAY_SIZE(mvm->baid_map); baid++) {
+		struct iwl_mvm_baid_data *data;
+		int ret;
+
+		data = rcu_dereference_protected(mvm->baid_map[baid],
+						 lockdep_is_held(&mvm->mutex));
+		if (!data)
+			continue;
+
+		if (!(data->sta_mask & old_sta_mask))
+			continue;
+
+		WARN_ONCE(data->sta_mask != old_sta_mask,
+			  "BAID data for %d corrupted - expected 0x%x found 0x%x\n",
+			  baid, old_sta_mask, data->sta_mask);
+
+		cmd.modify.tid = cpu_to_le32(data->tid);
+
+		ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, sizeof(cmd), &cmd);
+		data->sta_mask = new_sta_mask;
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int iwl_mvm_mld_update_sta_resources(struct iwl_mvm *mvm,
+					    struct ieee80211_vif *vif,
+					    struct ieee80211_sta *sta,
+					    u32 old_sta_mask,
+					    u32 new_sta_mask)
+{
+	int ret;
+
+	ret = iwl_mvm_mld_update_sta_queues(mvm, sta,
+					    old_sta_mask,
+					    new_sta_mask);
+	if (ret)
+		return ret;
+
+	ret = iwl_mvm_mld_update_sta_keys(mvm, vif, sta,
+					  old_sta_mask,
+					  new_sta_mask);
+	if (ret)
+		return ret;
+
+	return iwl_mvm_mld_update_sta_baids(mvm, old_sta_mask, new_sta_mask);
+}
+
+int iwl_mvm_mld_update_sta_links(struct iwl_mvm *mvm,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_sta *sta,
+				 u16 old_links, u16 new_links)
+{
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_mvm_vif *mvm_vif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_link_sta *mvm_sta_link;
+	struct iwl_mvm_vif_link_info *mvm_vif_link;
+	unsigned long links_to_add = ~old_links & new_links;
+	unsigned long links_to_rem = old_links & ~new_links;
+	unsigned long old_links_long = old_links;
+	u32 current_sta_mask = 0, sta_mask_added = 0, sta_mask_to_rem = 0;
+	unsigned long link_sta_added_to_fw = 0, link_sta_allocated = 0;
+	unsigned int link_id;
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	for_each_set_bit(link_id, &old_links_long,
+			 IEEE80211_MLD_MAX_NUM_LINKS) {
+		mvm_sta_link =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+
+		if (WARN_ON(!mvm_sta_link)) {
+			ret = -EINVAL;
+			goto err;
+		}
+
+		current_sta_mask |= BIT(mvm_sta_link->sta_id);
+		if (links_to_rem & BIT(link_id))
+			sta_mask_to_rem |= BIT(mvm_sta_link->sta_id);
+	}
+
+	if (sta_mask_to_rem) {
+		ret = iwl_mvm_mld_update_sta_resources(mvm, vif, sta,
+						       current_sta_mask,
+						       current_sta_mask &
+							~sta_mask_to_rem);
+		if (WARN_ON(ret))
+			goto err;
+
+		current_sta_mask &= ~sta_mask_to_rem;
+	}
+
+	for_each_set_bit(link_id, &links_to_rem, IEEE80211_MLD_MAX_NUM_LINKS) {
+		mvm_sta_link =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+		mvm_vif_link = mvm_vif->link[link_id];
+
+		if (WARN_ON(!mvm_sta_link || !mvm_vif_link)) {
+			ret = -EINVAL;
+			goto err;
+		}
+
+		ret = iwl_mvm_mld_rm_sta_from_fw(mvm, mvm_sta_link->sta_id);
+		if (WARN_ON(ret))
+			goto err;
+
+		if (vif->type == NL80211_IFTYPE_STATION)
+			mvm_vif_link->ap_sta_id = IWL_MVM_INVALID_STA;
+
+		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_sta_link, link_id,
+					  false);
+	}
+
+	for_each_set_bit(link_id, &links_to_add, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_bss_conf *link_conf =
+			link_conf_dereference_protected(vif, link_id);
+		struct ieee80211_link_sta *link_sta =
+			link_sta_dereference_protected(sta, link_id);
+		mvm_vif_link = mvm_vif->link[link_id];
+
+		if (WARN_ON(!mvm_vif_link || !link_conf || !link_sta ||
+			    mvm_sta->link[link_id])) {
+			ret = -EINVAL;
+			goto err;
+		}
+
+		ret = iwl_mvm_mld_alloc_sta_link(mvm, vif, sta, link_id);
+		if (WARN_ON(ret))
+			goto err;
+
+		link_sta->agg.max_rc_amsdu_len = 1;
+		ieee80211_sta_recalc_aggregates(sta);
+
+		mvm_sta_link =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+
+		if (WARN_ON(!mvm_sta_link)) {
+			ret = -EINVAL;
+			goto err;
+		}
+
+		if (vif->type == NL80211_IFTYPE_STATION)
+			iwl_mvm_mld_set_ap_sta_id(sta, mvm_vif_link,
+						  mvm_sta_link);
+
+		link_sta_allocated |= BIT(link_id);
+
+		sta_mask_added |= BIT(mvm_sta_link->sta_id);
+
+		ret = iwl_mvm_mld_cfg_sta(mvm, sta, vif, link_sta, link_conf,
+					  mvm_sta_link);
+		if (WARN_ON(ret))
+			goto err;
+
+		link_sta_added_to_fw |= BIT(link_id);
+
+		iwl_mvm_rs_add_sta_link(mvm, mvm_sta_link);
+	}
+
+	if (sta_mask_added) {
+		ret = iwl_mvm_mld_update_sta_resources(mvm, vif, sta,
+						       current_sta_mask,
+						       current_sta_mask |
+							sta_mask_added);
+		if (WARN_ON(ret))
+			goto err;
+	}
+
+	return 0;
+
+err:
+	/* remove all already allocated stations in FW */
+	for_each_set_bit(link_id, &link_sta_added_to_fw,
+			 IEEE80211_MLD_MAX_NUM_LINKS) {
+		mvm_sta_link =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+
+		iwl_mvm_mld_rm_sta_from_fw(mvm, mvm_sta_link->sta_id);
+	}
+
+	/* remove all already allocated station links in driver */
+	for_each_set_bit(link_id, &link_sta_allocated,
+			 IEEE80211_MLD_MAX_NUM_LINKS) {
+		mvm_sta_link =
+			rcu_dereference_protected(mvm_sta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+
+		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_sta_link, link_id,
+					  false);
+	}
+
+	return ret;
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index f307c345dfa0..6e7470d3a826 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -16,6 +16,8 @@
 #include <linux/thermal.h>
 #endif
 
+#include <linux/ptp_clock_kernel.h>
+
 #include <linux/ktime.h>
 
 #include "iwl-op-mode.h"
@@ -71,7 +73,11 @@
 /* offchannel queue towards mac80211 */
 #define IWL_MVM_OFFCHANNEL_QUEUE 0
 
+/* invalid value for FW link id */
+#define IWL_MVM_FW_LINK_ID_INVALID 0xff
+
 extern const struct ieee80211_ops iwl_mvm_hw_ops;
+extern const struct ieee80211_ops iwl_mvm_mld_hw_ops;
 
 /**
  * struct iwl_mvm_mod_params - module parameters for iwlmvm
@@ -278,11 +284,60 @@ struct iwl_probe_resp_data {
 };
 
 /**
+ * struct iwl_mvm_vif_link_info - per link data in Virtual Interface
+ * @ap_sta_id: the sta_id of the AP - valid only if VIF type is STA
+ * @fw_link_id: the id of the link according to the FW API
+ * @bssid: BSSID for this (client) interface
+ * @bcast_sta: station used for broadcast packets. Used by the following
+ *	vifs: P2P_DEVICE, GO and AP.
+ * @beacon_stats: beacon statistics, containing the # of received beacons,
+ *	# of received beacons accumulated over FW restart, and the current
+ *	average signal of beacons retrieved from the firmware
+ * @smps_requests: the SMPS requests of different parts of the driver,
+ *	combined on update to yield the overall request to mac80211.
+ * @probe_resp_data: data from FW notification to store NOA and CSA related
+ *	data to be inserted into probe response.
+ * @he_ru_2mhz_block: 26-tone RU OFDMA transmissions should be blocked
+ * @queue_params: QoS params for this MAC
+ * @mgmt_queue: queue number for unbufferable management frames
+ */
+struct iwl_mvm_vif_link_info {
+	u8 bssid[ETH_ALEN];
+	u8 ap_sta_id;
+	u8 fw_link_id;
+
+	struct iwl_mvm_int_sta bcast_sta;
+	struct iwl_mvm_int_sta mcast_sta;
+
+	struct {
+		u32 num_beacons, accu_num_beacons;
+		u8 avg_signal;
+	} beacon_stats;
+
+	enum ieee80211_smps_mode smps_requests[NUM_IWL_MVM_SMPS_REQ];
+	struct iwl_probe_resp_data __rcu *probe_resp_data;
+
+	bool he_ru_2mhz_block;
+	bool active;
+
+	u16 cab_queue;
+	/* Assigned while mac80211 has the link in a channel context,
+	 * or, for P2P Device, while it exists.
+	 */
+	struct iwl_mvm_phy_ctxt *phy_ctxt;
+	/* QoS data from mac80211, need to store this here
+	 * as mac80211 has a separate callback but we need
+	 * to have the data for the MAC context
+	 */
+	struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS];
+
+	u16 mgmt_queue;
+};
+
+/**
  * struct iwl_mvm_vif - data per Virtual Interface, it is a MAC context
  * @id: between 0 and 3
  * @color: to solve races upon MAC addition and removal
- * @ap_sta_id: the sta_id of the AP - valid only if VIF type is STA
- * @bssid: BSSID for this (client) interface
  * @associated: indicates that we're currently associated, used only for
  *	managing the firmware state in iwl_mvm_bss_info_changed_station()
  * @ap_assoc_sta_count: count of stations associated to us - valid only
@@ -290,7 +345,7 @@ struct iwl_probe_resp_data {
  * @uploaded: indicates the MAC context has been added to the device
  * @ap_ibss_active: indicates that AP/IBSS is configured and that the interface
  *	should get quota etc.
- * @pm_enabled - Indicate if MAC power management is allowed
+ * @pm_enabled - indicate if MAC power management is allowed
  * @monitor_active: indicates that monitor context is configured, and that the
  *	interface should get quota etc.
  * @low_latency: bit flags for low latency
@@ -299,68 +354,31 @@ struct iwl_probe_resp_data {
  *	as a result from low_latency bit flags and takes force into account.
  * @authorized: indicates the AP station was set to authorized
  * @ps_disabled: indicates that this interface requires PS to be disabled
- * @queue_params: QoS params for this MAC
- * @bcast_sta: station used for broadcast packets. Used by the following
- *  vifs: P2P_DEVICE, GO and AP.
- * @beacon_skb: the skb used to hold the AP/GO beacon template
- * @smps_requests: the SMPS requests of different parts of the driver,
- *	combined on update to yield the overall request to mac80211.
- * @beacon_stats: beacon statistics, containing the # of received beacons,
- *	# of received beacons accumulated over FW restart, and the current
- *	average signal of beacons retrieved from the firmware
+ * @csa_countdown: indicates that CSA countdown may be started
  * @csa_failed: CSA failed to schedule time event, report an error later
+ * @csa_bcn_pending: indicates that we are waiting for a beacon on a new channel
  * @features: hw features active for this vif
- * @probe_resp_data: data from FW notification to store NOA and CSA related
- *	data to be inserted into probe response.
  */
 struct iwl_mvm_vif {
 	struct iwl_mvm *mvm;
 	u16 id;
 	u16 color;
-	u8 ap_sta_id;
 
-	u8 bssid[ETH_ALEN];
 	bool associated;
 	u8 ap_assoc_sta_count;
-
-	u16 cab_queue;
-
 	bool uploaded;
 	bool ap_ibss_active;
 	bool pm_enabled;
 	bool monitor_active;
+
 	u8 low_latency: 6;
 	u8 low_latency_actual: 1;
+
 	u8 authorized:1;
 	bool ps_disabled;
-	struct iwl_mvm_vif_bf_data bf_data;
-
-	struct {
-		u32 num_beacons, accu_num_beacons;
-		u8 avg_signal;
-	} beacon_stats;
 
 	u32 ap_beacon_time;
-
-	enum iwl_tsf_id tsf_id;
-
-	/*
-	 * QoS data from mac80211, need to store this here
-	 * as mac80211 has a separate callback but we need
-	 * to have the data for the MAC context
-	 */
-	struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS];
-	struct iwl_mvm_time_event_data time_event_data;
-	struct iwl_mvm_time_event_data hs_time_event_data;
-
-	struct iwl_mvm_int_sta bcast_sta;
-	struct iwl_mvm_int_sta mcast_sta;
-
-	/*
-	 * Assigned while mac80211 has the interface in a channel context,
-	 * or, for P2P Device, while it exists.
-	 */
-	struct iwl_mvm_phy_ctxt *phy_ctxt;
+	struct iwl_mvm_vif_bf_data bf_data;
 
 #ifdef CONFIG_PM
 	/* WoWLAN GTK rekey data */
@@ -396,40 +414,45 @@ struct iwl_mvm_vif {
 	int dbgfs_quota_min;
 #endif
 
-	enum ieee80211_smps_mode smps_requests[NUM_IWL_MVM_SMPS_REQ];
-
 	/* FW identified misbehaving AP */
 	u8 uapsd_misbehaving_bssid[ETH_ALEN];
-
 	struct delayed_work uapsd_nonagg_detected_wk;
 
-	/* Indicates that CSA countdown may be started */
 	bool csa_countdown;
 	bool csa_failed;
+	bool csa_bcn_pending;
 	u16 csa_target_freq;
 	u16 csa_count;
 	u16 csa_misbehave;
 	struct delayed_work csa_work;
 
-	/* Indicates that we are waiting for a beacon on a new channel */
-	bool csa_bcn_pending;
+	enum iwl_tsf_id tsf_id;
+
+	struct iwl_mvm_time_event_data time_event_data;
+	struct iwl_mvm_time_event_data hs_time_event_data;
 
 	/* TCP Checksum Offload */
 	netdev_features_t features;
 
-	struct iwl_probe_resp_data __rcu *probe_resp_data;
+	struct ieee80211_sta *ap_sta;
 
 	/* we can only have 2 GTK + 2 IGTK active at a time */
 	struct ieee80211_key_conf *ap_early_keys[4];
 
-	/* 26-tone RU OFDMA transmissions should be blocked */
-	bool he_ru_2mhz_block;
-
 	struct {
 		struct ieee80211_key_conf __rcu *keys[2];
 	} bcn_prot;
+
+	struct iwl_mvm_vif_link_info deflink;
+	struct iwl_mvm_vif_link_info *link[IEEE80211_MLD_MAX_NUM_LINKS];
 };
 
+#define for_each_mvm_vif_valid_link(mvm_vif, link_id)			\
+	for (link_id = 0;						\
+	     link_id < ARRAY_SIZE((mvm_vif)->link);			\
+	     link_id++)							\
+		if ((mvm_vif)->link[link_id])
+
 static inline struct iwl_mvm_vif *
 iwl_mvm_vif_from_mac80211(struct ieee80211_vif *vif)
 {
@@ -657,7 +680,7 @@ __aligned(roundup_pow_of_two(sizeof(struct _iwl_mvm_reorder_buf_entry)))
 
 /**
  * struct iwl_mvm_baid_data - BA session data
- * @sta_id: station id
+ * @sta_mask: current station mask for the BAID
  * @tid: tid of the session
  * @baid baid of the session
  * @timeout: the timeout set in the addba request
@@ -671,7 +694,7 @@ __aligned(roundup_pow_of_two(sizeof(struct _iwl_mvm_reorder_buf_entry)))
  */
 struct iwl_mvm_baid_data {
 	struct rcu_head rcu_head;
-	u8 sta_id;
+	u32 sta_mask;
 	u8 tid;
 	u8 baid;
 	u16 timeout;
@@ -772,6 +795,43 @@ struct iwl_mvm_dqa_txq_info {
 	enum iwl_mvm_queue_status status;
 };
 
+struct ptp_data {
+	struct ptp_clock *ptp_clock;
+	struct ptp_clock_info ptp_clock_info;
+
+	struct delayed_work dwork;
+
+	/* The last GP2 reading from the hw */
+	u32 last_gp2;
+
+	/* number of wraparounds since scale_update_adj_time_ns */
+	u32 wrap_counter;
+
+	/* GP2 time when the scale was last updated */
+	u32 scale_update_gp2;
+
+	/* Adjusted time when the scale was last updated in nanoseconds */
+	u64 scale_update_adj_time_ns;
+
+	/* clock frequency offset, scaled to 65536000000 */
+	u64 scaled_freq;
+
+	/* Delta between hardware clock and ptp clock in nanoseconds */
+	s64 delta;
+};
+
+struct iwl_time_sync_data {
+	struct sk_buff_head frame_list;
+	u8 peer_addr[ETH_ALEN];
+	bool active;
+};
+
+struct iwl_mei_scan_filter {
+	bool is_mei_limited_scan;
+	struct sk_buff_head scan_res;
+	struct work_struct scan_work;
+};
+
 struct iwl_mvm {
 	/* for logger access */
 	struct device *dev;
@@ -857,6 +917,8 @@ struct iwl_mvm {
 	/* data related to data path */
 	struct iwl_rx_phy_info last_phy_info;
 	struct ieee80211_sta __rcu *fw_id_to_mac_id[IWL_MVM_STATION_COUNT_MAX];
+	struct ieee80211_link_sta __rcu *fw_id_to_link_sta[IWL_MVM_STATION_COUNT_MAX];
+	unsigned long fw_link_ids_map;
 	u8 rx_ba_sessions;
 
 	/* configured by mac80211 */
@@ -1083,6 +1145,8 @@ struct iwl_mvm {
 
 	struct list_head resp_pasn_list;
 
+	struct ptp_data ptp_data;
+
 	struct {
 		u8 range_resp;
 	} cmd_ver;
@@ -1100,6 +1164,11 @@ struct iwl_mvm {
 
 	/* does a monitor vif exist (only one can exist hence bool) */
 	bool monitor_on;
+	/*
+	 * primary channel position relative to he whole bandwidth,
+	 * in steps of 80 MHz
+	 */
+	u8 monitor_p80;
 
 	/* sniffer data to include in radiotap */
 	__le16 cur_aid;
@@ -1109,8 +1178,13 @@ struct iwl_mvm {
 	unsigned long last_reset_or_resume_time_jiffies;
 
 	bool sta_remove_requires_queue_remove;
+	bool mld_api_is_used;
 
 	bool pldr_sync;
+
+	struct iwl_time_sync_data time_sync;
+
+	struct iwl_mei_scan_filter mei_scan_filter;
 };
 
 /* Extract MVM priv from op_mode and _hw */
@@ -1310,7 +1384,7 @@ static inline bool iwl_mvm_is_csum_supported(struct iwl_mvm *mvm)
 {
 	return fw_has_capa(&mvm->fw->ucode_capa,
 			   IWL_UCODE_TLV_CAPA_CSUM_SUPPORT) &&
-               !IWL_MVM_HW_CSUM_DISABLE;
+		!IWL_MVM_HW_CSUM_DISABLE;
 }
 
 static inline bool iwl_mvm_is_mplut_supported(struct iwl_mvm *mvm)
@@ -1335,6 +1409,12 @@ static inline bool iwl_mvm_has_new_rx_api(struct iwl_mvm *mvm)
 			   IWL_UCODE_TLV_CAPA_MULTI_QUEUE_RX_SUPPORT);
 }
 
+static inline bool iwl_mvm_has_mld_api(const struct iwl_fw *fw)
+{
+	return fw_has_capa(&fw->ucode_capa,
+			   IWL_UCODE_TLV_CAPA_MLD_API_SUPPORT);
+}
+
 static inline bool iwl_mvm_has_new_tx_api(struct iwl_mvm *mvm)
 {
 	/* TODO - replace with TLV once defined */
@@ -1436,6 +1516,19 @@ static inline bool iwl_mvm_is_ctdp_supported(struct iwl_mvm *mvm)
 			   IWL_UCODE_TLV_CAPA_CTDP_SUPPORT);
 }
 
+static inline bool iwl_mvm_has_new_tx_csum(struct iwl_mvm *mvm)
+{
+	if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
+		return false;
+
+	if (mvm->trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_BZ &&
+	    CSR_HW_REV_TYPE(mvm->trans->hw_rev) == IWL_CFG_MAC_TYPE_GL &&
+	    mvm->trans->hw_rev_step <= SILICON_B_STEP)
+		return false;
+
+	return true;
+}
+
 extern const u8 iwl_mvm_ac_to_tx_fifo[];
 extern const u8 iwl_mvm_ac_to_gen2_tx_fifo[];
 
@@ -1476,6 +1569,7 @@ void iwl_mvm_hwrate_to_tx_rate_v1(u32 rate_n_flags,
 				  struct ieee80211_tx_rate *r);
 u8 iwl_mvm_mac80211_idx_to_hwrate(const struct iwl_fw *fw, int rate_idx);
 u8 iwl_mvm_mac80211_ac_to_ucode_ac(enum ieee80211_ac_numbers ac);
+bool iwl_mvm_is_nic_ack_enabled(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 
 static inline void iwl_mvm_dump_nic_error_log(struct iwl_mvm *mvm)
 {
@@ -1523,6 +1617,17 @@ int iwl_mvm_flush_tx_path(struct iwl_mvm *mvm, u32 tfd_msk);
 int iwl_mvm_flush_sta(struct iwl_mvm *mvm, void *sta, bool internal);
 int iwl_mvm_flush_sta_tids(struct iwl_mvm *mvm, u32 sta_id, u16 tids);
 
+/* Utils to extract sta related data */
+__le32 iwl_mvm_get_sta_htc_flags(struct ieee80211_sta *sta,
+				 struct ieee80211_link_sta *link_sta);
+u8 iwl_mvm_get_sta_uapsd_acs(struct ieee80211_sta *sta);
+u32 iwl_mvm_get_sta_ampdu_dens(struct ieee80211_link_sta *link_sta,
+			       struct ieee80211_bss_conf *link_conf,
+			       u32 *_agg_size);
+int iwl_mvm_set_sta_pkt_ext(struct iwl_mvm *mvm,
+			    struct ieee80211_link_sta *link_sta,
+			    struct iwl_he_pkt_ext_v2 *pkt_ext);
+
 void iwl_mvm_async_handlers_purge(struct iwl_mvm *mvm);
 
 static inline void iwl_mvm_set_tx_cmd_ccmp(struct ieee80211_tx_info *info,
@@ -1622,6 +1727,7 @@ void iwl_mvm_rx_shared_mem_cfg_notif(struct iwl_mvm *mvm,
 				     struct iwl_rx_cmd_buffer *rxb);
 
 /* MVM PHY */
+struct iwl_mvm_phy_ctxt *iwl_mvm_get_free_phy_ctxt(struct iwl_mvm *mvm);
 int iwl_mvm_phy_ctxt_add(struct iwl_mvm *mvm, struct iwl_mvm_phy_ctxt *ctxt,
 			 struct cfg80211_chan_def *chandef,
 			 u8 chains_static, u8 chains_dynamic);
@@ -1637,22 +1743,60 @@ u8 iwl_mvm_get_channel_width(struct cfg80211_chan_def *chandef);
 u8 iwl_mvm_get_ctrl_pos(struct cfg80211_chan_def *chandef);
 
 /* MAC (virtual interface) programming */
+
+void iwl_mvm_prepare_mac_removal(struct iwl_mvm *mvm,
+				 struct ieee80211_vif *vif);
+void iwl_mvm_set_fw_basic_rates(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+				struct ieee80211_bss_conf *link_conf,
+				__le32 *cck_rates, __le32 *ofdm_rates);
+void iwl_mvm_set_fw_protection_flags(struct iwl_mvm *mvm,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_bss_conf *link_conf,
+				     __le32 *protection_flags, u32 ht_flag,
+				     u32 tgg_flag);
+void iwl_mvm_set_fw_qos_params(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			       struct ieee80211_bss_conf *link_conf,
+			       struct iwl_ac_qos *ac, __le32 *qos_flags);
+bool iwl_mvm_set_fw_mu_edca_params(struct iwl_mvm *mvm,
+				   struct iwl_mvm_vif *mvmvif,
+				   struct iwl_he_backoff_conf *trig_based_txf);
+void iwl_mvm_set_fw_dtim_tbtt(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link_conf,
+			      __le64 *dtim_tsf, __le32 *dtim_time,
+			      __le32 *assoc_beacon_arrive_time);
+__le32 iwl_mac_ctxt_p2p_dev_has_extended_disc(struct iwl_mvm *mvm,
+					      struct ieee80211_vif *vif);
+void iwl_mvm_mac_ctxt_cmd_ap_set_filter_flags(struct iwl_mvm *mvm,
+					      struct iwl_mvm_vif *mvmvif,
+					      __le32 *filter_flags,
+					      int accept_probe_req_flag,
+					      int accept_beacon_flag);
+int iwl_mvm_get_mac_type(struct ieee80211_vif *vif);
+__le32 iwl_mvm_mac_ctxt_cmd_p2p_sta_get_oppps_ctwin(struct iwl_mvm *mvm,
+						    struct ieee80211_vif *vif);
+__le32 iwl_mvm_mac_ctxt_cmd_sta_get_twt_policy(struct iwl_mvm *mvm,
+					       struct ieee80211_vif *vif);
+int iwl_mvm_mld_mac_ctxt_add(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
+int iwl_mvm_mld_mac_ctxt_changed(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+				 bool force_assoc_off);
+int iwl_mvm_mld_mac_ctxt_remove(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_mac_ctxt_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_mac_ctxt_add(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_mac_ctxt_changed(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			     bool force_assoc_off, const u8 *bssid_override);
 int iwl_mvm_mac_ctxt_remove(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_mac_ctxt_beacon_changed(struct iwl_mvm *mvm,
-				    struct ieee80211_vif *vif);
-int iwl_mvm_mac_ctxt_send_beacon(struct iwl_mvm *mvm,
-				 struct ieee80211_vif *vif,
-				 struct sk_buff *beacon);
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_bss_conf *link_conf);
 int iwl_mvm_mac_ctxt_send_beacon_cmd(struct iwl_mvm *mvm,
 				     struct sk_buff *beacon,
 				     void *data, int len);
 u8 iwl_mvm_mac_ctxt_get_beacon_rate(struct iwl_mvm *mvm,
 				    struct ieee80211_tx_info *info,
 				    struct ieee80211_vif *vif);
+u8 iwl_mvm_mac_ctxt_get_lowest_rate(struct iwl_mvm *mvm,
+				    struct ieee80211_tx_info *info,
+				    struct ieee80211_vif *vif);
 u16 iwl_mvm_mac_ctxt_get_beacon_flags(const struct iwl_fw *fw,
 				      u8 rate_idx);
 void iwl_mvm_mac_ctxt_set_tim(struct iwl_mvm *mvm,
@@ -1683,6 +1827,93 @@ void iwl_mvm_channel_switch_error_notif(struct iwl_mvm *mvm,
 int iwl_mvm_binding_add_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_binding_remove_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 
+/* Links */
+int iwl_mvm_add_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+		     struct ieee80211_bss_conf *link_conf);
+int iwl_mvm_link_changed(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			 struct ieee80211_bss_conf *link_conf,
+			 u32 changes, bool active);
+int iwl_mvm_remove_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			struct ieee80211_bss_conf *link_conf);
+int iwl_mvm_disable_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			 struct ieee80211_bss_conf *link_conf);
+
+/* AP and IBSS */
+bool iwl_mvm_start_ap_ibss_common(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif, int *ret);
+void iwl_mvm_stop_ap_ibss_common(struct iwl_mvm *mvm,
+				 struct ieee80211_vif *vif);
+
+/* BSS Info */
+/**
+ * struct iwl_mvm_bss_info_changed_ops - callbacks for the bss_info_changed()
+ *
+ * Since the only difference between both MLD and
+ * non-MLD versions of bss_info_changed() is these function calls,
+ * each version will send its specific function calls to
+ * %iwl_mvm_bss_info_changed_common().
+ *
+ * @bss_info_changed_sta: pointer to the function that handles changes
+ *	in bss_info in sta mode
+ * @bss_info_changed_ap_ibss: pointer to the function that handles changes
+ *	in bss_info in ap and ibss modes
+ */
+struct iwl_mvm_bss_info_changed_ops {
+	void (*bss_info_changed_sta)(struct iwl_mvm *mvm,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_bss_conf *bss_conf,
+				     u64 changes);
+	void (*bss_info_changed_ap_ibss)(struct iwl_mvm *mvm,
+					 struct ieee80211_vif *vif,
+					 struct ieee80211_bss_conf *bss_conf,
+					 u64 changes);
+};
+
+void
+iwl_mvm_bss_info_changed_common(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_bss_conf *bss_conf,
+				struct iwl_mvm_bss_info_changed_ops *callbacks,
+				u64 changes);
+void
+iwl_mvm_bss_info_changed_station_common(struct iwl_mvm *mvm,
+					struct ieee80211_vif *vif,
+					struct ieee80211_bss_conf *link_conf,
+					u64 changes);
+void iwl_mvm_bss_info_changed_station_assoc(struct iwl_mvm *mvm,
+					    struct ieee80211_vif *vif,
+					    u64 changes);
+
+/* ROC */
+/**
+ * struct iwl_mvm_roc_ops - callbacks for the remain_on_channel()
+ *
+ * Since the only difference between both MLD and
+ * non-MLD versions of remain_on_channel() is these function calls,
+ * each version will send its specific function calls to
+ * %iwl_mvm_roc_common().
+ *
+ * @add_aux_sta_for_hs20: pointer to the function that adds an aux sta
+ *	for Hot Spot 2.0
+ * @switch_phy_ctxt: pointer to the function that switches a vif from one
+ *	phy_ctx to another
+ */
+struct iwl_mvm_roc_ops {
+	int (*add_aux_sta_for_hs20)(struct iwl_mvm *mvm, u32 lmac_id);
+	int (*switch_phy_ctxt)(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			       struct iwl_mvm_phy_ctxt *new_phy_ctxt);
+};
+
+int iwl_mvm_roc_common(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		       struct ieee80211_channel *channel, int duration,
+		       enum ieee80211_roc_type type,
+		       struct iwl_mvm_roc_ops *ops);
+int iwl_mvm_cancel_roc(struct ieee80211_hw *hw,
+		       struct ieee80211_vif *vif);
+/*Session Protection */
+void iwl_mvm_protect_assoc(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			   u32 duration_override);
+
 /* Quota management */
 static inline size_t iwl_mvm_quota_cmd_size(struct iwl_mvm *mvm)
 {
@@ -1862,10 +2093,17 @@ int iwl_mvm_disable_beacon_filter(struct iwl_mvm *mvm,
 /* SMPS */
 void iwl_mvm_update_smps(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 				enum iwl_mvm_smps_type_request req_type,
-				enum ieee80211_smps_mode smps_request);
+				enum ieee80211_smps_mode smps_request,
+				unsigned int link_id);
+void
+iwl_mvm_update_smps_on_active_links(struct iwl_mvm *mvm,
+				    struct ieee80211_vif *vif,
+				    enum iwl_mvm_smps_type_request req_type,
+				    enum ieee80211_smps_mode smps_request);
 bool iwl_mvm_rx_diversity_allowed(struct iwl_mvm *mvm,
 				  struct iwl_mvm_phy_ctxt *ctxt);
-void iwl_mvm_apply_fw_smps_request(struct ieee80211_vif *vif);
+void iwl_mvm_update_link_smps(struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link_conf);
 
 /* Low latency */
 int iwl_mvm_update_low_latency(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
@@ -2074,7 +2312,11 @@ void iwl_mvm_event_frame_timeout_callback(struct iwl_mvm *mvm,
 					  struct ieee80211_vif *vif,
 					  const struct ieee80211_sta *sta,
 					  u16 tid);
+void iwl_mvm_mei_scan_filter_init(struct iwl_mei_scan_filter *mei_scan_filter);
 
+void iwl_mvm_ptp_init(struct iwl_mvm *mvm);
+void iwl_mvm_ptp_remove(struct iwl_mvm *mvm);
+u64 iwl_mvm_ptp_get_adj_time(struct iwl_mvm *mvm, u64 base_time);
 int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b);
 int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm);
 int iwl_mvm_ppag_send_cmd(struct iwl_mvm *mvm);
@@ -2096,7 +2338,14 @@ int iwl_mvm_sec_key_del(struct iwl_mvm *mvm,
 			struct ieee80211_sta *sta,
 			struct ieee80211_key_conf *keyconf);
 void iwl_mvm_sec_key_remove_ap(struct iwl_mvm *mvm,
-			       struct ieee80211_vif *vif);
+			       struct ieee80211_vif *vif,
+			       struct iwl_mvm_vif_link_info *link,
+			       unsigned int link_id);
+int iwl_mvm_mld_update_sta_keys(struct iwl_mvm *mvm,
+				struct ieee80211_vif *vif,
+				struct ieee80211_sta *sta,
+				u32 old_sta_mask,
+				u32 new_sta_mask);
 
 int iwl_rfi_send_config_cmd(struct iwl_mvm *mvm,
 			    struct iwl_rfi_lut_entry *rfi_table);
@@ -2119,6 +2368,45 @@ static inline u8 iwl_mvm_phy_band_from_nl80211(enum nl80211_band band)
 	}
 }
 
+/* Channel Switch */
+void iwl_mvm_channel_switch_disconnect_wk(struct work_struct *wk);
+int iwl_mvm_post_channel_switch(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif);
+
+/* Channel Context */
+/**
+ * struct iwl_mvm_switch_vif_chanctx_ops - callbacks for switch_vif_chanctx()
+ *
+ * Since the only difference between both MLD and
+ * non-MLD versions of switch_vif_chanctx() is these function calls,
+ * each version will send its specific function calls to
+ * %iwl_mvm_switch_vif_chanctx_common().
+ *
+ * @__assign_vif_chanctx: pointer to the function that assigns a chanctx to
+ *	a given vif
+ * @__unassign_vif_chanctx: pointer to the function that unassigns a chanctx to
+ *	a given vif
+ */
+struct iwl_mvm_switch_vif_chanctx_ops {
+	int (*__assign_vif_chanctx)(struct iwl_mvm *mvm,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_bss_conf *link_conf,
+				    struct ieee80211_chanctx_conf *ctx,
+				    bool switching_chanctx);
+	void (*__unassign_vif_chanctx)(struct iwl_mvm *mvm,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_bss_conf *link_conf,
+				       struct ieee80211_chanctx_conf *ctx,
+				       bool switching_chanctx);
+};
+
+int
+iwl_mvm_switch_vif_chanctx_common(struct ieee80211_hw *hw,
+				  struct ieee80211_vif_chanctx_switch *vifs,
+				  int n_vifs,
+				  enum ieee80211_chanctx_switch_mode mode,
+				  struct iwl_mvm_switch_vif_chanctx_ops *ops);
+
 /* Channel info utils */
 static inline bool iwl_mvm_has_ultra_hb_channel(struct iwl_mvm *mvm)
 {
@@ -2236,8 +2524,147 @@ static inline void iwl_mvm_mei_set_sw_rfkill_state(struct iwl_mvm *mvm)
 					 sw_rfkill);
 }
 
+static inline bool iwl_mvm_mei_filter_scan(struct iwl_mvm *mvm,
+					   struct sk_buff *skb)
+{
+	struct ieee80211_mgmt *mgmt = (void *)skb->data;
+
+	if (mvm->mei_scan_filter.is_mei_limited_scan &&
+	    (ieee80211_is_probe_resp(mgmt->frame_control) ||
+	     ieee80211_is_beacon(mgmt->frame_control))) {
+		skb_queue_tail(&mvm->mei_scan_filter.scan_res, skb);
+		schedule_work(&mvm->mei_scan_filter.scan_work);
+		return true;
+	}
+
+	return false;
+}
+
 void iwl_mvm_send_roaming_forbidden_event(struct iwl_mvm *mvm,
 					  struct ieee80211_vif *vif,
 					  bool forbidden);
+bool iwl_mvm_is_vendor_in_approved_list(void);
+
+/* Callbacks for ieee80211_ops */
+void iwl_mvm_mac_tx(struct ieee80211_hw *hw,
+		    struct ieee80211_tx_control *control, struct sk_buff *skb);
+void iwl_mvm_mac_wake_tx_queue(struct ieee80211_hw *hw,
+			       struct ieee80211_txq *txq);
+
+int iwl_mvm_mac_ampdu_action(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_ampdu_params *params);
+int iwl_mvm_op_get_antenna(struct ieee80211_hw *hw, u32 *tx_ant, u32 *rx_ant);
+int iwl_mvm_mac_start(struct ieee80211_hw *hw);
+void iwl_mvm_mac_reconfig_complete(struct ieee80211_hw *hw,
+				   enum ieee80211_reconfig_type reconfig_type);
+void iwl_mvm_mac_stop(struct ieee80211_hw *hw);
+static inline int iwl_mvm_mac_config(struct ieee80211_hw *hw, u32 changed)
+{
+	return 0;
+}
+
+u64 iwl_mvm_prepare_multicast(struct ieee80211_hw *hw,
+			      struct netdev_hw_addr_list *mc_list);
 
+void iwl_mvm_configure_filter(struct ieee80211_hw *hw,
+			      unsigned int changed_flags,
+			      unsigned int *total_flags, u64 multicast);
+int iwl_mvm_mac_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			struct ieee80211_scan_request *hw_req);
+void iwl_mvm_mac_cancel_hw_scan(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif);
+void iwl_mvm_sta_pre_rcu_remove(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_sta *sta);
+void iwl_mvm_mac_sta_notify(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			    enum sta_notify_cmd cmd,
+			    struct ieee80211_sta *sta);
+void
+iwl_mvm_mac_allow_buffered_frames(struct ieee80211_hw *hw,
+				  struct ieee80211_sta *sta, u16 tids,
+				  int num_frames,
+				  enum ieee80211_frame_release_type reason,
+				  bool more_data);
+void
+iwl_mvm_mac_release_buffered_frames(struct ieee80211_hw *hw,
+				    struct ieee80211_sta *sta, u16 tids,
+				    int num_frames,
+				    enum ieee80211_frame_release_type reason,
+				    bool more_data);
+int iwl_mvm_mac_set_rts_threshold(struct ieee80211_hw *hw, u32 value);
+void iwl_mvm_sta_rc_update(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			   struct ieee80211_sta *sta, u32 changed);
+void iwl_mvm_mac_mgd_prepare_tx(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_prep_tx_info *info);
+void iwl_mvm_mac_mgd_complete_tx(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_prep_tx_info *info);
+void iwl_mvm_mac_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		       u32 queues, bool drop);
+int iwl_mvm_mac_sched_scan_start(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct cfg80211_sched_scan_request *req,
+				 struct ieee80211_scan_ies *ies);
+int iwl_mvm_mac_sched_scan_stop(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif);
+int iwl_mvm_mac_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+			struct ieee80211_vif *vif, struct ieee80211_sta *sta,
+			struct ieee80211_key_conf *key);
+void iwl_mvm_mac_update_tkip_key(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_key_conf *keyconf,
+				 struct ieee80211_sta *sta,
+				 u32 iv32, u16 *phase1key);
+int iwl_mvm_add_chanctx(struct ieee80211_hw *hw,
+			struct ieee80211_chanctx_conf *ctx);
+void iwl_mvm_remove_chanctx(struct ieee80211_hw *hw,
+			    struct ieee80211_chanctx_conf *ctx);
+void iwl_mvm_change_chanctx(struct ieee80211_hw *hw,
+			    struct ieee80211_chanctx_conf *ctx, u32 changed);
+int iwl_mvm_tx_last_beacon(struct ieee80211_hw *hw);
+int iwl_mvm_set_tim(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
+		    bool set);
+void iwl_mvm_channel_switch(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			    struct ieee80211_channel_switch *chsw);
+int iwl_mvm_pre_channel_switch(struct ieee80211_hw *hw,
+			       struct ieee80211_vif *vif,
+			       struct ieee80211_channel_switch *chsw);
+void iwl_mvm_abort_channel_switch(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif);
+void iwl_mvm_channel_switch_rx_beacon(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_channel_switch *chsw);
+void iwl_mvm_mac_event_callback(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				const struct ieee80211_event *event);
+void iwl_mvm_sync_rx_queues(struct ieee80211_hw *hw);
+int iwl_mvm_mac_testmode_cmd(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     void *data, int len);
+int iwl_mvm_mac_get_survey(struct ieee80211_hw *hw, int idx,
+			   struct survey_info *survey);
+void iwl_mvm_mac_sta_statistics(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_sta *sta,
+				struct station_info *sinfo);
+int
+iwl_mvm_mac_get_ftm_responder_stats(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif,
+				    struct cfg80211_ftm_responder_stats *stats);
+int iwl_mvm_start_pmsr(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		       struct cfg80211_pmsr_request *request);
+void iwl_mvm_abort_pmsr(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			struct cfg80211_pmsr_request *request);
+
+bool iwl_mvm_have_links_same_channel(struct iwl_mvm_vif *vif1,
+				     struct iwl_mvm_vif *vif2);
+bool iwl_mvm_vif_is_active(struct iwl_mvm_vif *mvmvif);
+int iwl_mvm_set_tx_power(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			 s16 tx_power);
+int iwl_mvm_set_hw_timestamp(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct cfg80211_set_hw_timestamp *hwts);
+int iwl_mvm_update_mu_groups(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 #endif /* __IWL_MVM_H__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
index 9711841bb456..32625bfacaae 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
@@ -29,6 +29,7 @@
 #include "fw-api.h"
 #include "fw/acpi.h"
 #include "fw/uefi.h"
+#include "time-sync.h"
 
 #define DRV_DESCRIPTION	"The new Intel(R) wireless AGN driver for Linux"
 MODULE_DESCRIPTION(DRV_DESCRIPTION);
@@ -208,24 +209,37 @@ static void iwl_mvm_rx_monitor_notif(struct iwl_mvm *mvm,
 	ieee80211_disconnect(vif, true);
 }
 
-void iwl_mvm_apply_fw_smps_request(struct ieee80211_vif *vif)
+void iwl_mvm_update_link_smps(struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link_conf)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm *mvm = mvmvif->mvm;
 	enum ieee80211_smps_mode mode = IEEE80211_SMPS_AUTOMATIC;
 
+	if (!link_conf)
+		return;
+
 	if (mvm->fw_static_smps_request &&
-	    vif->bss_conf.chandef.width == NL80211_CHAN_WIDTH_160 &&
-	    vif->bss_conf.he_support)
+	    link_conf->chandef.width == NL80211_CHAN_WIDTH_160 &&
+	    link_conf->he_support)
 		mode = IEEE80211_SMPS_STATIC;
 
-	iwl_mvm_update_smps(mvm, vif, IWL_MVM_SMPS_REQ_FW, mode);
+	iwl_mvm_update_smps(mvm, vif, IWL_MVM_SMPS_REQ_FW, mode,
+			    link_conf->link_id);
 }
 
 static void iwl_mvm_intf_dual_chain_req(void *data, u8 *mac,
 					struct ieee80211_vif *vif)
 {
-	iwl_mvm_apply_fw_smps_request(vif);
+	struct ieee80211_bss_conf *link_conf;
+	unsigned int link_id;
+
+	rcu_read_lock();
+
+	for_each_vif_active_link(vif, link_conf, link_id)
+		iwl_mvm_update_link_smps(vif, link_conf);
+
+	rcu_read_unlock();
 }
 
 static void iwl_mvm_rx_thermal_dual_chain_req(struct iwl_mvm *mvm,
@@ -404,6 +418,15 @@ static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 	RX_HANDLER_GRP(SYSTEM_GROUP, RFI_DEACTIVATE_NOTIF,
 		       iwl_rfi_deactivate_notif_handler, RX_HANDLER_ASYNC_UNLOCKED,
 		       struct iwl_rfi_deactivate_notif),
+
+	RX_HANDLER_GRP(LEGACY_GROUP,
+		       WNM_80211V_TIMING_MEASUREMENT_NOTIFICATION,
+		       iwl_mvm_time_sync_msmt_event, RX_HANDLER_SYNC,
+		       struct iwl_time_msmt_notify),
+	RX_HANDLER_GRP(LEGACY_GROUP,
+		       WNM_80211V_TIMING_MEASUREMENT_CONFIRM_NOTIFICATION,
+		       iwl_mvm_time_sync_msmt_confirm_event, RX_HANDLER_SYNC,
+		       struct iwl_time_msmt_cfm_notify),
 };
 #undef RX_HANDLER
 #undef RX_HANDLER_GRP
@@ -449,6 +472,8 @@ static const struct iwl_hcmd_names iwl_mvm_legacy_names[] = {
 	HCMD_NAME(SCAN_OFFLOAD_PROFILES_QUERY_CMD),
 	HCMD_NAME(BT_COEX_UPDATE_REDUCED_TXP),
 	HCMD_NAME(BT_COEX_CI),
+	HCMD_NAME(WNM_80211V_TIMING_MEASUREMENT_NOTIFICATION),
+	HCMD_NAME(WNM_80211V_TIMING_MEASUREMENT_CONFIRM_NOTIFICATION),
 	HCMD_NAME(PHY_CONFIGURATION_CMD),
 	HCMD_NAME(CALIB_RES_NOTIF_PHY_DB),
 	HCMD_NAME(PHY_DB_CMD),
@@ -521,6 +546,12 @@ static const struct iwl_hcmd_names iwl_mvm_system_names[] = {
 static const struct iwl_hcmd_names iwl_mvm_mac_conf_names[] = {
 	HCMD_NAME(CHANNEL_SWITCH_TIME_EVENT_CMD),
 	HCMD_NAME(SESSION_PROTECTION_CMD),
+	HCMD_NAME(MAC_CONFIG_CMD),
+	HCMD_NAME(LINK_CONFIG_CMD),
+	HCMD_NAME(STA_CONFIG_CMD),
+	HCMD_NAME(AUX_STA_CMD),
+	HCMD_NAME(STA_REMOVE_CMD),
+	HCMD_NAME(STA_DISABLE_TX_CMD),
 	HCMD_NAME(SESSION_PROTECTION_NOTIF),
 	HCMD_NAME(CHANNEL_SWITCH_START_NOTIF),
 };
@@ -989,10 +1020,14 @@ static void iwl_mvm_me_conn_status(void *priv, const struct iwl_mei_conn_info *c
 		kfree_rcu(prev_conn_info, rcu_head);
 }
 
-static void iwl_mvm_mei_rfkill(void *priv, bool blocked)
+static void iwl_mvm_mei_rfkill(void *priv, bool blocked,
+			       bool csme_taking_ownership)
 {
 	struct iwl_mvm *mvm = priv;
 
+	if (blocked && !csme_taking_ownership)
+		return;
+
 	mvm->mei_rfkill_blocked = blocked;
 	if (!mvm->hw_registered)
 		return;
@@ -1098,6 +1133,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	 ********************************/
 	hw = ieee80211_alloc_hw(sizeof(struct iwl_op_mode) +
 				sizeof(struct iwl_mvm),
+				iwl_mvm_has_mld_api(fw) ? &iwl_mvm_mld_hw_ops :
 				&iwl_mvm_hw_ops);
 	if (!hw)
 		return NULL;
@@ -1278,6 +1314,8 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	mvm->sta_remove_requires_queue_remove =
 		trans_cfg.queue_alloc_cmd_ver > 0;
 
+	mvm->mld_api_is_used = iwl_mvm_has_mld_api(mvm->fw);
+
 	/* Configure transport layer */
 	iwl_trans_configure(mvm->trans, &trans_cfg);
 
@@ -1333,10 +1371,16 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	else
 		memset(&mvm->rx_stats, 0, sizeof(struct mvm_statistics_rx));
 
+	iwl_mvm_ftm_initiator_smooth_config(mvm);
+
+	iwl_mvm_init_time_sync(&mvm->time_sync);
+
 	mvm->debugfs_dir = dbgfs_dir;
 
 	mvm->mei_registered = !iwl_mei_register(mvm, &mei_ops);
 
+	iwl_mvm_mei_scan_filter_init(&mvm->mei_scan_filter);
+
 	if (iwl_mvm_start_get_nvm(mvm)) {
 		/*
 		 * Getting NVM failed while CSME is the owner, but we are
@@ -1430,6 +1474,8 @@ static void iwl_op_mode_mvm_stop(struct iwl_op_mode *op_mode)
 	kfree(mvm->error_recovery_buf);
 	mvm->error_recovery_buf = NULL;
 
+	iwl_mvm_ptp_remove(mvm);
+
 	iwl_trans_op_mode_leave(mvm->trans);
 
 	iwl_phy_db_free(mvm->phy_db);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
index 06f4203fb989..3ab6fb83a175 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
@@ -382,12 +382,12 @@ static void iwl_mvm_binding_iterator(void *_data, u8 *mac,
 	unsigned long *data = _data;
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-	if (!mvmvif->phy_ctxt)
+	if (!mvmvif->deflink.phy_ctxt)
 		return;
 
 	if (vif->type == NL80211_IFTYPE_STATION ||
 	    vif->type == NL80211_IFTYPE_AP)
-		__set_bit(mvmvif->phy_ctxt->id, data);
+		__set_bit(mvmvif->deflink.phy_ctxt->id, data);
 }
 
 int iwl_mvm_phy_ctx_count(struct iwl_mvm *mvm)
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/power.c b/drivers/net/wireless/intel/iwlwifi/mvm/power.c
index f5744162d0d8..ac1dae52556f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/power.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/power.c
@@ -150,7 +150,7 @@ static void iwl_mvm_power_configure_uapsd(struct iwl_mvm *mvm,
 #endif
 
 	for (ac = IEEE80211_AC_VO; ac <= IEEE80211_AC_BK; ac++) {
-		if (!mvmvif->queue_params[ac].uapsd)
+		if (!mvmvif->deflink.queue_params[ac].uapsd)
 			continue;
 
 		if (!test_bit(IWL_MVM_STATUS_IN_D3, &mvm->status))
@@ -160,7 +160,7 @@ static void iwl_mvm_power_configure_uapsd(struct iwl_mvm *mvm,
 		cmd->uapsd_ac_flags |= BIT(ac);
 
 		/* QNDP TID - the highest TID with no admission control */
-		if (!tid_found && !mvmvif->queue_params[ac].acm) {
+		if (!tid_found && !mvmvif->deflink.queue_params[ac].acm) {
 			tid_found = true;
 			switch (ac) {
 			case IEEE80211_AC_VO:
@@ -279,18 +279,25 @@ static bool iwl_mvm_power_allow_uapsd(struct iwl_mvm *mvm,
 static bool iwl_mvm_power_is_radar(struct ieee80211_vif *vif)
 {
 	struct ieee80211_chanctx_conf *chanctx_conf;
-	struct ieee80211_channel *chan;
+	struct ieee80211_bss_conf *link_conf;
 	bool radar_detect = false;
+	unsigned int link_id;
 
 	rcu_read_lock();
-	chanctx_conf = rcu_dereference(vif->bss_conf.chanctx_conf);
-	WARN_ON(!chanctx_conf);
-	if (chanctx_conf) {
-		chan = chanctx_conf->def.chan;
-		radar_detect = chan->flags & IEEE80211_CHAN_RADAR;
+	for_each_vif_active_link(vif, link_conf, link_id) {
+		chanctx_conf = rcu_dereference(link_conf->chanctx_conf);
+		/* this happens on link switching, just ignore inactive ones */
+		if (!chanctx_conf)
+			continue;
+
+		radar_detect = !!(chanctx_conf->def.chan->flags &
+				  IEEE80211_CHAN_RADAR);
+		if (radar_detect)
+			goto out;
 	}
-	rcu_read_unlock();
 
+out:
+	rcu_read_unlock();
 	return radar_detect;
 }
 
@@ -509,8 +516,9 @@ static void iwl_mvm_power_uapsd_misbehav_ap_iterator(void *_data, u8 *mac,
 	/* The ap_sta_id is not expected to change during current association
 	 * so no explicit protection is needed
 	 */
-	if (mvmvif->ap_sta_id == *ap_sta_id)
-		memcpy(mvmvif->uapsd_misbehaving_bssid, vif->bss_conf.bssid,
+	if (mvmvif->deflink.ap_sta_id == *ap_sta_id)
+		memcpy(mvmvif->uapsd_misbehaving_bssid,
+		       vif->bss_conf.bssid,
 		       ETH_ALEN);
 }
 
@@ -552,7 +560,7 @@ static void iwl_mvm_power_ps_disabled_iterator(void *_data, u8* mac,
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	bool *disable_ps = _data;
 
-	if (mvmvif->phy_ctxt && mvmvif->phy_ctxt->id < NUM_PHY_CTX)
+	if (iwl_mvm_vif_is_active(mvmvif))
 		*disable_ps |= mvmvif->ps_disabled;
 }
 
@@ -561,11 +569,13 @@ static void iwl_mvm_power_get_vifs_iterator(void *_data, u8 *mac,
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_power_vifs *power_iterator = _data;
-	bool active = mvmvif->phy_ctxt && mvmvif->phy_ctxt->id < NUM_PHY_CTX;
+	bool active;
 
 	if (!mvmvif->uploaded)
 		return;
 
+	active = iwl_mvm_vif_is_active(mvmvif);
+
 	switch (ieee80211_vif_type_p2p(vif)) {
 	case NL80211_IFTYPE_P2P_DEVICE:
 		break;
@@ -649,11 +659,12 @@ static void iwl_mvm_power_set_pm(struct iwl_mvm *mvm,
 	}
 
 	if (vifs->bss_active && vifs->p2p_active)
-		client_same_channel = (bss_mvmvif->phy_ctxt->id ==
-				       p2p_mvmvif->phy_ctxt->id);
+		client_same_channel =
+			iwl_mvm_have_links_same_channel(bss_mvmvif, p2p_mvmvif);
+
 	if (vifs->bss_active && vifs->ap_active)
-		ap_same_channel = (bss_mvmvif->phy_ctxt->id ==
-				   ap_mvmvif->phy_ctxt->id);
+		ap_same_channel =
+			iwl_mvm_have_links_same_channel(bss_mvmvif, ap_mvmvif);
 
 	/* clients are not stand alone: enable PM if DCM */
 	if (!(client_same_channel || ap_same_channel)) {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
new file mode 100644
index 000000000000..e89259de6f4c
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
@@ -0,0 +1,326 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2021 - 2023 Intel Corporation
+ */
+
+#include "mvm.h"
+#include "iwl-debug.h"
+#include <linux/timekeeping.h>
+#include <linux/math64.h>
+
+#define IWL_PTP_GP2_WRAP	0x100000000ULL
+#define IWL_PTP_WRAP_TIME	(3600 * HZ)
+
+/* The scaled_ppm parameter is ppm (parts per million) with a 16-bit fractional
+ * part, which means that a value of 1 in one of those fields actually means
+ * 2^-16 ppm, and 2^16=65536 is 1 ppm.
+ */
+#define SCALE_FACTOR	65536000000ULL
+#define IWL_PTP_WRAP_THRESHOLD_USEC	(5000)
+
+#define IWL_PTP_GET_CROSS_TS_NUM	5
+
+static void iwl_mvm_ptp_update_new_read(struct iwl_mvm *mvm, u32 gp2)
+{
+	/* If the difference is above the threshold, assume it's a wraparound.
+	 * Otherwise assume it's an old read and ignore it.
+	 */
+	if (gp2 < mvm->ptp_data.last_gp2 &&
+	    mvm->ptp_data.last_gp2 - gp2 < IWL_PTP_WRAP_THRESHOLD_USEC) {
+		IWL_DEBUG_INFO(mvm,
+			       "PTP: ignore old read (gp2=%u, last_gp2=%u)\n",
+			       gp2, mvm->ptp_data.last_gp2);
+		return;
+	}
+
+	if (gp2 < mvm->ptp_data.last_gp2) {
+		mvm->ptp_data.wrap_counter++;
+		IWL_DEBUG_INFO(mvm,
+			       "PTP: wraparound detected (new counter=%u)\n",
+			       mvm->ptp_data.wrap_counter);
+	}
+
+	mvm->ptp_data.last_gp2 = gp2;
+	schedule_delayed_work(&mvm->ptp_data.dwork, IWL_PTP_WRAP_TIME);
+}
+
+u64 iwl_mvm_ptp_get_adj_time(struct iwl_mvm *mvm, u64 base_time_ns)
+{
+	struct ptp_data *data = &mvm->ptp_data;
+	u64 last_gp2_ns = mvm->ptp_data.scale_update_gp2 * NSEC_PER_USEC;
+	u64 res;
+	u64 diff;
+
+	iwl_mvm_ptp_update_new_read(mvm,
+				    div64_u64(base_time_ns, NSEC_PER_USEC));
+
+	IWL_DEBUG_INFO(mvm, "base_time_ns=%llu, wrap_counter=%u\n",
+		       (unsigned long long)base_time_ns, data->wrap_counter);
+
+	base_time_ns = base_time_ns +
+		(data->wrap_counter * IWL_PTP_GP2_WRAP * NSEC_PER_USEC);
+
+	/* It is possible that a GP2 timestamp was received from fw before the
+	 * last scale update. Since we don't know how to scale - ignore it.
+	 */
+	if (base_time_ns < last_gp2_ns) {
+		IWL_DEBUG_INFO(mvm, "Time before scale update - ignore\n");
+		return 0;
+	}
+
+	diff = base_time_ns - last_gp2_ns;
+	IWL_DEBUG_INFO(mvm, "diff ns=%llu\n", (unsigned long long)diff);
+
+	diff = mul_u64_u64_div_u64(diff, data->scaled_freq,
+				   SCALE_FACTOR);
+	IWL_DEBUG_INFO(mvm, "scaled diff ns=%llu\n", (unsigned long long)diff);
+
+	res = data->scale_update_adj_time_ns + data->delta + diff;
+
+	IWL_DEBUG_INFO(mvm, "base=%llu delta=%lld adj=%llu\n",
+		       (unsigned long long)base_time_ns, (long long)data->delta,
+		       (unsigned long long)res);
+	return res;
+}
+
+static int
+iwl_mvm_get_crosstimestamp_fw(struct iwl_mvm *mvm, u32 *gp2, u64 *sys_time)
+{
+	struct iwl_synced_time_cmd synced_time_cmd = {
+		.operation = cpu_to_le32(IWL_SYNCED_TIME_OPERATION_READ_BOTH)
+	};
+	struct iwl_host_cmd cmd = {
+		.id = WIDE_ID(DATA_PATH_GROUP, WNM_PLATFORM_PTM_REQUEST_CMD),
+		.flags = CMD_WANT_SKB,
+		.data[0] = &synced_time_cmd,
+		.len[0] = sizeof(synced_time_cmd),
+	};
+	struct iwl_synced_time_rsp *resp;
+	struct iwl_rx_packet *pkt;
+	int ret;
+	u64 gp2_10ns;
+
+	ret = iwl_mvm_send_cmd(mvm, &cmd);
+	if (ret)
+		return ret;
+
+	pkt = cmd.resp_pkt;
+
+	if (iwl_rx_packet_payload_len(pkt) != sizeof(*resp)) {
+		IWL_ERR(mvm, "PTP: Invalid command response\n");
+		iwl_free_resp(&cmd);
+		return -EIO;
+	}
+
+	resp = (void *)pkt->data;
+
+	gp2_10ns = (u64)le32_to_cpu(resp->gp2_timestamp_hi) << 32 |
+		le32_to_cpu(resp->gp2_timestamp_lo);
+	*gp2 = div_u64(gp2_10ns, 100);
+
+	*sys_time = (u64)le32_to_cpu(resp->platform_timestamp_hi) << 32 |
+		le32_to_cpu(resp->platform_timestamp_lo);
+
+	return ret;
+}
+
+static void iwl_mvm_phc_get_crosstimestamp_loop(struct iwl_mvm *mvm,
+						ktime_t *sys_time, u32 *gp2)
+{
+	u64 diff = 0, new_diff;
+	u64 tmp_sys_time;
+	u32 tmp_gp2;
+	int i;
+
+	for (i = 0; i < IWL_PTP_GET_CROSS_TS_NUM; i++) {
+		iwl_mvm_get_sync_time(mvm, CLOCK_REALTIME, &tmp_gp2, NULL,
+				      &tmp_sys_time);
+		new_diff = tmp_sys_time - ((u64)tmp_gp2 * NSEC_PER_USEC);
+		if (!diff || new_diff < diff) {
+			*sys_time = tmp_sys_time;
+			*gp2 = tmp_gp2;
+			diff = new_diff;
+			IWL_DEBUG_INFO(mvm, "PTP: new times: gp2=%u sys=%lld\n",
+				       *gp2, *sys_time);
+		}
+	}
+}
+
+static int
+iwl_mvm_phc_get_crosstimestamp(struct ptp_clock_info *ptp,
+			       struct system_device_crosststamp *xtstamp)
+{
+	struct iwl_mvm *mvm = container_of(ptp, struct iwl_mvm,
+					   ptp_data.ptp_clock_info);
+	int ret = 0;
+	/* Raw value read from GP2 register in usec */
+	u32 gp2;
+	/* GP2 value in ns*/
+	s64 gp2_ns;
+	/* System (wall) time */
+	ktime_t sys_time;
+
+	memset(xtstamp, 0, sizeof(struct system_device_crosststamp));
+
+	if (!mvm->ptp_data.ptp_clock) {
+		IWL_ERR(mvm, "No PHC clock registered\n");
+		return -ENODEV;
+	}
+
+	mutex_lock(&mvm->mutex);
+	if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_SYNCED_TIME)) {
+		ret = iwl_mvm_get_crosstimestamp_fw(mvm, &gp2, &sys_time);
+
+		if (ret)
+			goto out;
+	} else {
+		iwl_mvm_phc_get_crosstimestamp_loop(mvm, &sys_time, &gp2);
+	}
+
+	gp2_ns = iwl_mvm_ptp_get_adj_time(mvm, (u64)gp2 * NSEC_PER_USEC);
+
+	IWL_INFO(mvm, "Got Sync Time: GP2:%u, last_GP2: %u, GP2_ns: %lld, sys_time: %lld\n",
+		 gp2, mvm->ptp_data.last_gp2, gp2_ns, (s64)sys_time);
+
+	/* System monotonic raw time is not used */
+	xtstamp->device = (ktime_t)gp2_ns;
+	xtstamp->sys_realtime = sys_time;
+
+out:
+	mutex_unlock(&mvm->mutex);
+	return ret;
+}
+
+static void iwl_mvm_ptp_work(struct work_struct *wk)
+{
+	struct iwl_mvm *mvm = container_of(wk, struct iwl_mvm,
+					   ptp_data.dwork.work);
+	u32 gp2;
+
+	mutex_lock(&mvm->mutex);
+	gp2 = iwl_mvm_get_systime(mvm);
+	iwl_mvm_ptp_update_new_read(mvm, gp2);
+	mutex_unlock(&mvm->mutex);
+}
+
+static int iwl_mvm_ptp_gettime(struct ptp_clock_info *ptp,
+			       struct timespec64 *ts)
+{
+	struct iwl_mvm *mvm = container_of(ptp, struct iwl_mvm,
+					   ptp_data.ptp_clock_info);
+	u64 gp2;
+	u64 ns;
+
+	mutex_lock(&mvm->mutex);
+	gp2 = iwl_mvm_get_systime(mvm);
+	ns = iwl_mvm_ptp_get_adj_time(mvm, gp2 * NSEC_PER_USEC);
+	mutex_unlock(&mvm->mutex);
+
+	*ts = ns_to_timespec64(ns);
+	return 0;
+}
+
+static int iwl_mvm_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
+{
+	struct iwl_mvm *mvm = container_of(ptp, struct iwl_mvm,
+					   ptp_data.ptp_clock_info);
+	struct ptp_data *data = container_of(ptp, struct ptp_data,
+					     ptp_clock_info);
+
+	mutex_lock(&mvm->mutex);
+	data->delta += delta;
+	IWL_DEBUG_INFO(mvm, "delta=%lld, new delta=%lld\n", (long long)delta,
+		       (long long)data->delta);
+	mutex_unlock(&mvm->mutex);
+	return 0;
+}
+
+static int iwl_mvm_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
+{
+	struct iwl_mvm *mvm = container_of(ptp, struct iwl_mvm,
+					   ptp_data.ptp_clock_info);
+	struct ptp_data *data = &mvm->ptp_data;
+	u32 gp2;
+
+	mutex_lock(&mvm->mutex);
+
+	/* Must call _iwl_mvm_ptp_get_adj_time() before updating
+	 * data->scale_update_gp2 or data->scaled_freq since
+	 * scale_update_adj_time_ns should reflect the previous scaled_freq.
+	 */
+	gp2 = iwl_mvm_get_systime(mvm);
+	data->scale_update_adj_time_ns =
+		iwl_mvm_ptp_get_adj_time(mvm, gp2 * NSEC_PER_USEC);
+	data->scale_update_gp2 = gp2;
+	data->wrap_counter = 0;
+	data->delta = 0;
+
+	data->scaled_freq = SCALE_FACTOR + scaled_ppm;
+	IWL_DEBUG_INFO(mvm, "adjfine: scaled_ppm=%ld new=%llu\n",
+		       scaled_ppm, (unsigned long long)data->scaled_freq);
+
+	mutex_unlock(&mvm->mutex);
+	return 0;
+}
+
+/* iwl_mvm_ptp_init - initialize PTP for devices which support it.
+ * @mvm: internal mvm structure, see &struct iwl_mvm.
+ *
+ * Performs the required steps for enabling PTP support.
+ */
+void iwl_mvm_ptp_init(struct iwl_mvm *mvm)
+{
+	/* Warn if the interface already has a ptp_clock defined */
+	if (WARN_ON(mvm->ptp_data.ptp_clock))
+		return;
+
+	mvm->ptp_data.ptp_clock_info.owner = THIS_MODULE;
+	mvm->ptp_data.ptp_clock_info.max_adj = 0x7fffffff;
+	mvm->ptp_data.ptp_clock_info.getcrosststamp =
+					iwl_mvm_phc_get_crosstimestamp;
+	mvm->ptp_data.ptp_clock_info.adjfine = iwl_mvm_ptp_adjfine;
+	mvm->ptp_data.ptp_clock_info.adjtime = iwl_mvm_ptp_adjtime;
+	mvm->ptp_data.ptp_clock_info.gettime64 = iwl_mvm_ptp_gettime;
+	mvm->ptp_data.scaled_freq = SCALE_FACTOR;
+
+	/* Give a short 'friendly name' to identify the PHC clock */
+	snprintf(mvm->ptp_data.ptp_clock_info.name,
+		 sizeof(mvm->ptp_data.ptp_clock_info.name),
+		 "%s", "iwlwifi-PTP");
+
+	INIT_DELAYED_WORK(&mvm->ptp_data.dwork, iwl_mvm_ptp_work);
+
+	mvm->ptp_data.ptp_clock =
+		ptp_clock_register(&mvm->ptp_data.ptp_clock_info, mvm->dev);
+
+	if (IS_ERR(mvm->ptp_data.ptp_clock)) {
+		IWL_ERR(mvm, "Failed to register PHC clock (%ld)\n",
+			PTR_ERR(mvm->ptp_data.ptp_clock));
+		mvm->ptp_data.ptp_clock = NULL;
+	} else if (mvm->ptp_data.ptp_clock) {
+		IWL_INFO(mvm, "Registered PHC clock: %s, with index: %d\n",
+			 mvm->ptp_data.ptp_clock_info.name,
+			 ptp_clock_index(mvm->ptp_data.ptp_clock));
+	}
+}
+
+/* iwl_mvm_ptp_remove - disable PTP device.
+ * @mvm: internal mvm structure, see &struct iwl_mvm.
+ *
+ * Disable PTP support.
+ */
+void iwl_mvm_ptp_remove(struct iwl_mvm *mvm)
+{
+	if (mvm->ptp_data.ptp_clock) {
+		IWL_INFO(mvm, "Unregistering PHC clock: %s, with index: %d\n",
+			 mvm->ptp_data.ptp_clock_info.name,
+			 ptp_clock_index(mvm->ptp_data.ptp_clock));
+
+		ptp_clock_unregister(mvm->ptp_data.ptp_clock);
+		mvm->ptp_data.ptp_clock = NULL;
+		memset(&mvm->ptp_data.ptp_clock_info, 0,
+		       sizeof(mvm->ptp_data.ptp_clock_info));
+		mvm->ptp_data.last_gp2 = 0;
+		cancel_delayed_work_sync(&mvm->ptp_data.dwork);
+	}
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/quota.c b/drivers/net/wireless/intel/iwlwifi/mvm/quota.c
index cea1a34f9130..aad2614af9ad 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/quota.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/quota.c
@@ -33,11 +33,11 @@ static void iwl_mvm_quota_iterator(void *_data, u8 *mac,
 	if (vif == data->disabled_vif)
 		return;
 
-	if (!mvmvif->phy_ctxt)
+	if (!mvmvif->deflink.phy_ctxt)
 		return;
 
 	/* currently, PHY ID == binding ID */
-	id = mvmvif->phy_ctxt->id;
+	id = mvmvif->deflink.phy_ctxt->id;
 
 	/* need at least one binding per PHY */
 	BUILD_BUG_ON(NUM_PHY_CTX > MAX_BINDINGS);
@@ -67,9 +67,10 @@ static void iwl_mvm_quota_iterator(void *_data, u8 *mac,
 	}
 
 	if (data->colors[id] < 0)
-		data->colors[id] = mvmvif->phy_ctxt->color;
+		data->colors[id] = mvmvif->deflink.phy_ctxt->color;
 	else
-		WARN_ON_ONCE(data->colors[id] != mvmvif->phy_ctxt->color);
+		WARN_ON_ONCE(data->colors[id] !=
+			     mvmvif->deflink.phy_ctxt->color);
 
 	data->n_interfaces[id]++;
 
@@ -99,7 +100,7 @@ static void iwl_mvm_adjust_quota_for_noa(struct iwl_mvm *mvm,
 	if (!mvmvif->ap_ibss_active)
 		return;
 
-	phy_id = mvmvif->phy_ctxt->id;
+	phy_id = mvmvif->deflink.phy_ctxt->id;
 	beacon_int = mvm->noa_vif->bss_conf.beacon_int;
 
 	for (i = 0; i < MAX_BINDINGS; i++) {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
index f30eeab5505b..c3a00bfbeef2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
@@ -9,9 +9,9 @@
 #include "iwl-op-mode.h"
 #include "mvm.h"
 
-static u8 rs_fw_bw_from_sta_bw(const struct ieee80211_sta *sta)
+static u8 rs_fw_bw_from_sta_bw(const struct ieee80211_link_sta *link_sta)
 {
-	switch (sta->deflink.bandwidth) {
+	switch (link_sta->bandwidth) {
 	case IEEE80211_STA_RX_BW_320:
 		return IWL_TLC_MNG_CH_WIDTH_320MHZ;
 	case IEEE80211_STA_RX_BW_160:
@@ -38,11 +38,11 @@ static u8 rs_fw_set_active_chains(u8 chains)
 	return fw_chains;
 }
 
-static u8 rs_fw_sgi_cw_support(struct ieee80211_sta *sta)
+static u8 rs_fw_sgi_cw_support(struct ieee80211_link_sta *link_sta)
 {
-	struct ieee80211_sta_ht_cap *ht_cap = &sta->deflink.ht_cap;
-	struct ieee80211_sta_vht_cap *vht_cap = &sta->deflink.vht_cap;
-	struct ieee80211_sta_he_cap *he_cap = &sta->deflink.he_cap;
+	struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
+	struct ieee80211_sta_vht_cap *vht_cap = &link_sta->vht_cap;
+	struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
 	u8 supp = 0;
 
 	if (he_cap->has_he)
@@ -61,12 +61,14 @@ static u8 rs_fw_sgi_cw_support(struct ieee80211_sta *sta)
 }
 
 static u16 rs_fw_get_config_flags(struct iwl_mvm *mvm,
-				  struct ieee80211_sta *sta,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_link_sta *link_sta,
 				  struct ieee80211_supported_band *sband)
 {
-	struct ieee80211_sta_ht_cap *ht_cap = &sta->deflink.ht_cap;
-	struct ieee80211_sta_vht_cap *vht_cap = &sta->deflink.vht_cap;
-	struct ieee80211_sta_he_cap *he_cap = &sta->deflink.he_cap;
+	struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
+	struct ieee80211_sta_vht_cap *vht_cap = &link_sta->vht_cap;
+	struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
+	const struct ieee80211_sta_he_cap *sband_he_cap;
 	bool vht_ena = vht_cap->vht_supported;
 	u16 flags = 0;
 
@@ -92,17 +94,19 @@ static u16 rs_fw_get_config_flags(struct iwl_mvm *mvm,
 	    IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD))
 		flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
 
-	if (sband->iftype_data && sband->iftype_data->he_cap.has_he &&
-	    !(sband->iftype_data->he_cap.he_cap_elem.phy_cap_info[1] &
-	     IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD))
+	sband_he_cap = ieee80211_get_he_iftype_cap(sband,
+						   ieee80211_vif_type_p2p(vif));
+	if (sband_he_cap &&
+	    !(sband_he_cap->he_cap_elem.phy_cap_info[1] &
+			IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD))
 		flags &= ~IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
 
 	if (he_cap->has_he &&
 	    (he_cap->he_cap_elem.phy_cap_info[3] &
 	     IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_MASK &&
-	     sband->iftype_data &&
-	     sband->iftype_data->he_cap.he_cap_elem.phy_cap_info[3] &
-	     IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_MASK))
+	     sband_he_cap &&
+	     sband_he_cap->he_cap_elem.phy_cap_info[3] &
+			IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_MASK))
 		flags |= IWL_TLC_MNG_CFG_FLAGS_HE_DCM_NSS_1_MSK;
 
 	return flags;
@@ -132,20 +136,20 @@ int rs_fw_vht_highest_rx_mcs_index(const struct ieee80211_sta_vht_cap *vht_cap,
 }
 
 static void
-rs_fw_vht_set_enabled_rates(const struct ieee80211_sta *sta,
+rs_fw_vht_set_enabled_rates(const struct ieee80211_link_sta *link_sta,
 			    const struct ieee80211_sta_vht_cap *vht_cap,
 			    struct iwl_tlc_config_cmd_v4 *cmd)
 {
 	u16 supp;
 	int i, highest_mcs;
-	u8 max_nss = sta->deflink.rx_nss;
+	u8 max_nss = link_sta->rx_nss;
 	struct ieee80211_vht_cap ieee_vht_cap = {
 		.vht_cap_info = cpu_to_le32(vht_cap->cap),
 		.supp_mcs = vht_cap->vht_mcs,
 	};
 
 	/* the station support only a single receive chain */
-	if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC)
+	if (link_sta->smps_mode == IEEE80211_SMPS_STATIC)
 		max_nss = 1;
 
 	for (i = 0; i < max_nss && i < IWL_TLC_NSS_MAX; i++) {
@@ -156,7 +160,7 @@ rs_fw_vht_set_enabled_rates(const struct ieee80211_sta *sta,
 			continue;
 
 		supp = BIT(highest_mcs + 1) - 1;
-		if (sta->deflink.bandwidth == IEEE80211_STA_RX_BW_20)
+		if (link_sta->bandwidth == IEEE80211_STA_RX_BW_20)
 			supp &= ~BIT(IWL_TLC_MNG_HT_RATE_MCS9);
 
 		cmd->ht_rates[i][IWL_TLC_MCS_PER_BW_80] = cpu_to_le16(supp);
@@ -165,7 +169,7 @@ rs_fw_vht_set_enabled_rates(const struct ieee80211_sta *sta,
 		 * configuration is supported - only for MCS 0 since we already
 		 * decoded the MCS bits anyway ourselves.
 		 */
-		if (sta->deflink.bandwidth == IEEE80211_STA_RX_BW_160 &&
+		if (link_sta->bandwidth == IEEE80211_STA_RX_BW_160 &&
 		    ieee80211_get_vht_max_nss(&ieee_vht_cap,
 					      IEEE80211_VHT_CHANWIDTH_160MHZ,
 					      0, true, nss) >= nss)
@@ -192,11 +196,11 @@ static u16 rs_fw_he_ieee80211_mcs_to_rs_mcs(u16 mcs)
 }
 
 static void
-rs_fw_he_set_enabled_rates(const struct ieee80211_sta *sta,
+rs_fw_he_set_enabled_rates(const struct ieee80211_link_sta *link_sta,
 			   struct ieee80211_supported_band *sband,
 			   struct iwl_tlc_config_cmd_v4 *cmd)
 {
-	const struct ieee80211_sta_he_cap *he_cap = &sta->deflink.he_cap;
+	const struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
 	u16 mcs_160 = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_160);
 	u16 mcs_80 = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_80);
 	u16 tx_mcs_80 =
@@ -204,10 +208,10 @@ rs_fw_he_set_enabled_rates(const struct ieee80211_sta *sta,
 	u16 tx_mcs_160 =
 		le16_to_cpu(sband->iftype_data->he_cap.he_mcs_nss_supp.tx_mcs_160);
 	int i;
-	u8 nss = sta->deflink.rx_nss;
+	u8 nss = link_sta->rx_nss;
 
 	/* the station support only a single receive chain */
-	if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC)
+	if (link_sta->smps_mode == IEEE80211_SMPS_STATIC)
 		nss = 1;
 
 	for (i = 0; i < nss && i < IWL_TLC_NSS_MAX; i++) {
@@ -282,13 +286,15 @@ rs_fw_rs_mcs2eht_mcs(enum IWL_TLC_MCS_PER_BW bw,
 	}
 }
 
-static void rs_fw_eht_set_enabled_rates(const struct ieee80211_sta *sta,
-					struct ieee80211_supported_band *sband,
-					struct iwl_tlc_config_cmd_v4 *cmd)
+static void
+rs_fw_eht_set_enabled_rates(struct ieee80211_vif *vif,
+			    const struct ieee80211_link_sta *link_sta,
+			    struct ieee80211_supported_band *sband,
+			    struct iwl_tlc_config_cmd_v4 *cmd)
 {
 	/* peer RX mcs capa */
 	const struct ieee80211_eht_mcs_nss_supp *eht_rx_mcs =
-		&sta->deflink.eht_cap.eht_mcs_nss_supp;
+		&link_sta->eht_cap.eht_mcs_nss_supp;
 	/* our TX mcs capa */
 	const struct ieee80211_eht_mcs_nss_supp *eht_tx_mcs =
 		&sband->iftype_data->eht_cap.eht_mcs_nss_supp;
@@ -298,7 +304,8 @@ static void rs_fw_eht_set_enabled_rates(const struct ieee80211_sta *sta,
 	struct ieee80211_eht_mcs_nss_supp_20mhz_only mcs_tx_20;
 
 	/* peer is 20Mhz only */
-	if (!(sta->deflink.he_cap.he_cap_elem.phy_cap_info[0] &
+	if (vif->type == NL80211_IFTYPE_AP &&
+	    !(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
 	      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
 		mcs_rx_20 = eht_rx_mcs->only_20mhz;
 	} else {
@@ -337,10 +344,14 @@ static void rs_fw_eht_set_enabled_rates(const struct ieee80211_sta *sta,
 		const struct ieee80211_eht_mcs_nss_supp_bw *mcs_tx =
 			rs_fw_rs_mcs2eht_mcs(bw, eht_tx_mcs);
 
-		/* got unsuppored index for bw */
+		/* got unsupported index for bw */
 		if (!mcs_rx || !mcs_tx)
 			continue;
 
+		/* break out if we don't support the bandwidth */
+		if (cmd->max_ch_width < (bw + IWL_TLC_MNG_CH_WIDTH_80MHZ))
+			break;
+
 		rs_fw_set_eht_mcs_nss(cmd->ht_rates, bw,
 				      MAX_NSS_MCS(9, mcs_rx, mcs_tx), GENMASK(9, 0));
 		rs_fw_set_eht_mcs_nss(cmd->ht_rates, bw,
@@ -350,25 +361,26 @@ static void rs_fw_eht_set_enabled_rates(const struct ieee80211_sta *sta,
 	}
 
 	/* the station support only a single receive chain */
-	if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC ||
-	    sta->deflink.rx_nss < 2)
+	if (link_sta->smps_mode == IEEE80211_SMPS_STATIC ||
+	    link_sta->rx_nss < 2)
 		memset(cmd->ht_rates[IWL_TLC_NSS_2], 0,
 		       sizeof(cmd->ht_rates[IWL_TLC_NSS_2]));
 }
 
-static void rs_fw_set_supp_rates(struct ieee80211_sta *sta,
+static void rs_fw_set_supp_rates(struct ieee80211_vif *vif,
+				 struct ieee80211_link_sta *link_sta,
 				 struct ieee80211_supported_band *sband,
 				 struct iwl_tlc_config_cmd_v4 *cmd)
 {
 	int i;
 	u16 supp = 0;
 	unsigned long tmp; /* must be unsigned long for for_each_set_bit */
-	const struct ieee80211_sta_ht_cap *ht_cap = &sta->deflink.ht_cap;
-	const struct ieee80211_sta_vht_cap *vht_cap = &sta->deflink.vht_cap;
-	const struct ieee80211_sta_he_cap *he_cap = &sta->deflink.he_cap;
+	const struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
+	const struct ieee80211_sta_vht_cap *vht_cap = &link_sta->vht_cap;
+	const struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
 
 	/* non HT rates */
-	tmp = sta->deflink.supp_rates[sband->band];
+	tmp = link_sta->supp_rates[sband->band];
 	for_each_set_bit(i, &tmp, BITS_PER_LONG)
 		supp |= BIT(sband->bitrates[i].hw_value);
 
@@ -376,22 +388,22 @@ static void rs_fw_set_supp_rates(struct ieee80211_sta *sta,
 	cmd->mode = IWL_TLC_MNG_MODE_NON_HT;
 
 	/* HT/VHT rates */
-	if (sta->deflink.eht_cap.has_eht) {
+	if (link_sta->eht_cap.has_eht) {
 		cmd->mode = IWL_TLC_MNG_MODE_EHT;
-		rs_fw_eht_set_enabled_rates(sta, sband, cmd);
+		rs_fw_eht_set_enabled_rates(vif, link_sta, sband, cmd);
 	} else if (he_cap->has_he) {
 		cmd->mode = IWL_TLC_MNG_MODE_HE;
-		rs_fw_he_set_enabled_rates(sta, sband, cmd);
+		rs_fw_he_set_enabled_rates(link_sta, sband, cmd);
 	} else if (vht_cap->vht_supported) {
 		cmd->mode = IWL_TLC_MNG_MODE_VHT;
-		rs_fw_vht_set_enabled_rates(sta, vht_cap, cmd);
+		rs_fw_vht_set_enabled_rates(link_sta, vht_cap, cmd);
 	} else if (ht_cap->ht_supported) {
 		cmd->mode = IWL_TLC_MNG_MODE_HT;
 		cmd->ht_rates[IWL_TLC_NSS_1][IWL_TLC_MCS_PER_BW_80] =
 			cpu_to_le16(ht_cap->mcs.rx_mask[0]);
 
 		/* the station support only a single receive chain */
-		if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC)
+		if (link_sta->smps_mode == IEEE80211_SMPS_STATIC)
 			cmd->ht_rates[IWL_TLC_NSS_2][IWL_TLC_MCS_PER_BW_80] =
 				0;
 		else
@@ -406,15 +418,18 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_tlc_update_notif *notif;
 	struct ieee80211_sta *sta;
+	struct ieee80211_link_sta *link_sta;
 	struct iwl_mvm_sta *mvmsta;
+	struct iwl_mvm_link_sta *mvm_link_sta;
 	struct iwl_lq_sta_rs_fw *lq_sta;
 	u32 flags;
 
 	rcu_read_lock();
 
 	notif = (void *)pkt->data;
+	link_sta = rcu_dereference(mvm->fw_id_to_link_sta[notif->sta_id]);
 	sta = rcu_dereference(mvm->fw_id_to_mac_id[notif->sta_id]);
-	if (IS_ERR_OR_NULL(sta)) {
+	if (IS_ERR_OR_NULL(sta) || !link_sta) {
 		/* can happen in remove station flow where mvm removed internally
 		 * the station before removing from FW
 		 */
@@ -434,7 +449,14 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
 
 	flags = le32_to_cpu(notif->flags);
 
-	lq_sta = &mvmsta->lq_sta.rs_fw;
+	mvm_link_sta = rcu_dereference(mvmsta->link[link_sta->link_id]);
+	if (!mvm_link_sta) {
+		IWL_DEBUG_RATE(mvm,
+			       "Invalid mvmsta RCU pointer for link (%d) of  sta id (%d) in TLC notification\n",
+			       link_sta->link_id, notif->sta_id);
+		goto out;
+	}
+	lq_sta = &mvm_link_sta->lq_sta.rs_fw;
 
 	if (flags & IWL_TLC_NOTIF_FLAG_RATE) {
 		char pretty_rate[100];
@@ -461,9 +483,9 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
 		u16 size = le32_to_cpu(notif->amsdu_size);
 		int i;
 
-		if (sta->deflink.agg.max_amsdu_len < size) {
+		if (link_sta->agg.max_amsdu_len < size) {
 			/*
-			 * In debug sta->deflink.agg.max_amsdu_len < size
+			 * In debug link_sta->agg.max_amsdu_len < size
 			 * so also check with orig_amsdu_len which holds the
 			 * original data before debugfs changed the value
 			 */
@@ -473,18 +495,18 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
 
 		mvmsta->amsdu_enabled = le32_to_cpu(notif->amsdu_enabled);
 		mvmsta->max_amsdu_len = size;
-		sta->deflink.agg.max_rc_amsdu_len = mvmsta->max_amsdu_len;
+		link_sta->agg.max_rc_amsdu_len = mvmsta->max_amsdu_len;
 
 		for (i = 0; i < IWL_MAX_TID_COUNT; i++) {
 			if (mvmsta->amsdu_enabled & BIT(i))
-				sta->deflink.agg.max_tid_amsdu_len[i] =
+				link_sta->agg.max_tid_amsdu_len[i] =
 					iwl_mvm_max_amsdu_size(mvm, sta, i);
 			else
 				/*
 				 * Not so elegant, but this will effectively
 				 * prevent AMSDU on this TID
 				 */
-				sta->deflink.agg.max_tid_amsdu_len[i] = 1;
+				link_sta->agg.max_tid_amsdu_len[i] = 1;
 		}
 
 		IWL_DEBUG_RATE(mvm,
@@ -496,14 +518,18 @@ out:
 	rcu_read_unlock();
 }
 
-u16 rs_fw_get_max_amsdu_len(struct ieee80211_sta *sta)
+u16 rs_fw_get_max_amsdu_len(struct ieee80211_sta *sta,
+			    struct ieee80211_bss_conf *link_conf,
+			    struct ieee80211_link_sta *link_sta)
 {
-	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	const struct ieee80211_sta_vht_cap *vht_cap = &sta->deflink.vht_cap;
-	const struct ieee80211_sta_ht_cap *ht_cap = &sta->deflink.ht_cap;
+	const struct ieee80211_sta_vht_cap *vht_cap = &link_sta->vht_cap;
+	const struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
+
+	if (WARN_ON_ONCE(!link_conf->chandef.chan))
+		return IEEE80211_MAX_MPDU_LEN_VHT_3895;
 
-	if (mvmsta->vif->bss_conf.chandef.chan->band == NL80211_BAND_6GHZ) {
-		switch (le16_get_bits(sta->deflink.he_6ghz_capa.capa,
+	if (link_conf->chandef.chan->band == NL80211_BAND_6GHZ) {
+		switch (le16_get_bits(link_sta->he_6ghz_capa.capa,
 				      IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN)) {
 		case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454:
 			return IEEE80211_MAX_MPDU_LEN_VHT_11454;
@@ -538,34 +564,52 @@ u16 rs_fw_get_max_amsdu_len(struct ieee80211_sta *sta)
 	return 0;
 }
 
-void rs_fw_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
-		     enum nl80211_band band, bool update)
+void iwl_mvm_rs_fw_rate_init(struct iwl_mvm *mvm,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_sta *sta,
+			     struct ieee80211_bss_conf *link_conf,
+			     struct ieee80211_link_sta *link_sta,
+			     enum nl80211_band band)
 {
 	struct ieee80211_hw *hw = mvm->hw;
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_lq_sta_rs_fw *lq_sta = &mvmsta->lq_sta.rs_fw;
 	u32 cmd_id = WIDE_ID(DATA_PATH_GROUP, TLC_MNG_CONFIG_CMD);
 	struct ieee80211_supported_band *sband = hw->wiphy->bands[band];
-	u16 max_amsdu_len = rs_fw_get_max_amsdu_len(sta);
+	u16 max_amsdu_len = rs_fw_get_max_amsdu_len(sta, link_conf, link_sta);
+	struct iwl_mvm_link_sta *mvm_link_sta;
+	struct iwl_lq_sta_rs_fw *lq_sta;
 	struct iwl_tlc_config_cmd_v4 cfg_cmd = {
-		.sta_id = mvmsta->sta_id,
-		.max_ch_width = update ?
-			rs_fw_bw_from_sta_bw(sta) : RATE_MCS_CHAN_WIDTH_20,
-		.flags = cpu_to_le16(rs_fw_get_config_flags(mvm, sta, sband)),
+		.max_ch_width = mvmsta->authorized ?
+			rs_fw_bw_from_sta_bw(link_sta) : IWL_TLC_MNG_CH_WIDTH_20MHZ,
+		.flags = cpu_to_le16(rs_fw_get_config_flags(mvm, vif, link_sta,
+							    sband)),
 		.chains = rs_fw_set_active_chains(iwl_mvm_get_valid_tx_ant(mvm)),
-		.sgi_ch_width_supp = rs_fw_sgi_cw_support(sta),
+		.sgi_ch_width_supp = rs_fw_sgi_cw_support(link_sta),
 		.max_mpdu_len = iwl_mvm_is_csum_supported(mvm) ?
 				cpu_to_le16(max_amsdu_len) : 0,
 	};
-	int ret;
+	unsigned int link_id = link_conf->link_id;
 	int cmd_ver;
+	int ret;
+
+	rcu_read_lock();
+	mvm_link_sta = rcu_dereference(mvmsta->link[link_id]);
+	if (WARN_ON_ONCE(!mvm_link_sta)) {
+		rcu_read_unlock();
+		return;
+	}
+
+	cfg_cmd.sta_id = mvm_link_sta->sta_id;
 
+	lq_sta = &mvm_link_sta->lq_sta.rs_fw;
 	memset(lq_sta, 0, offsetof(typeof(*lq_sta), pers));
 
+	rcu_read_unlock();
+
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 	iwl_mvm_reset_frame_stats(mvm);
 #endif
-	rs_fw_set_supp_rates(sta, sband, &cfg_cmd);
+	rs_fw_set_supp_rates(vif, link_sta, sband, &cfg_cmd);
 
 	/*
 	 * since TLC offload works with one mode we can assume
@@ -635,16 +679,18 @@ int rs_fw_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 	return 0;
 }
 
-void iwl_mvm_rs_add_sta(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta)
+void iwl_mvm_rs_add_sta_link(struct iwl_mvm *mvm,
+			     struct iwl_mvm_link_sta *link_sta)
 {
-	struct iwl_lq_sta_rs_fw *lq_sta = &mvmsta->lq_sta.rs_fw;
+	struct iwl_lq_sta_rs_fw *lq_sta;
 
-	IWL_DEBUG_RATE(mvm, "create station rate scale window\n");
+	lq_sta = &link_sta->lq_sta.rs_fw;
 
 	lq_sta->pers.drv = mvm;
-	lq_sta->pers.sta_id = mvmsta->sta_id;
+	lq_sta->pers.sta_id = link_sta->sta_id;
 	lq_sta->pers.chains = 0;
-	memset(lq_sta->pers.chain_signal, 0, sizeof(lq_sta->pers.chain_signal));
+	memset(lq_sta->pers.chain_signal, 0,
+	       sizeof(lq_sta->pers.chain_signal));
 	lq_sta->pers.last_rssi = S8_MIN;
 	lq_sta->last_rate_n_flags = 0;
 
@@ -652,3 +698,20 @@ void iwl_mvm_rs_add_sta(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta)
 	lq_sta->pers.dbg_fixed_rate = 0;
 #endif
 }
+
+void iwl_mvm_rs_add_sta(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta)
+{
+	unsigned int link_id;
+
+	IWL_DEBUG_RATE(mvm, "create station rate scale window\n");
+
+	for (link_id = 0; link_id < ARRAY_SIZE(mvmsta->link); link_id++) {
+		struct iwl_mvm_link_sta *link =
+			rcu_dereference_protected(mvmsta->link[link_id],
+						  lockdep_is_held(&mvm->mutex));
+		if (!link)
+			continue;
+
+		iwl_mvm_rs_add_sta_link(mvm, link);
+	}
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
index 0b50b816684a..a4c1e3bf4ff1 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /******************************************************************************
  *
- * Copyright(c) 2005 - 2014, 2018 - 2021 Intel Corporation. All rights reserved.
+ * Copyright(c) 2005 - 2014, 2018 - 2022 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
  *****************************************************************************/
@@ -512,10 +512,10 @@ static char *rs_pretty_rate(const struct rs_rate *rate)
 		 (rate->index <= IWL_RATE_MCS_9_INDEX))
 		rate_str = ht_vht_rates[rate->index];
 	else
-		rate_str = "BAD_RATE";
+		rate_str = NULL;
 
 	sprintf(buf, "(%s|%s|%s)", rs_pretty_lq_type(rate->type),
-		iwl_rs_pretty_ant(rate->ant), rate_str);
+		iwl_rs_pretty_ant(rate->ant), rate_str ?: "BAD_RATE");
 	return buf;
 }
 
@@ -754,7 +754,7 @@ static int rs_collect_tlc_data(struct iwl_mvm *mvm,
 		return -EINVAL;
 
 	if (tbl->column != RS_COLUMN_INVALID) {
-		struct lq_sta_pers *pers = &mvmsta->lq_sta.rs_drv.pers;
+		struct lq_sta_pers *pers = &mvmsta->deflink.lq_sta.rs_drv.pers;
 
 		pers->tx_stats[tbl->column][scale_index].total += attempts;
 		pers->tx_stats[tbl->column][scale_index].success += successes;
@@ -895,8 +895,7 @@ static int rs_rate_from_ucode_rate(const u32 ucode_rate,
 			WARN_ON_ONCE(1);
 		}
 	} else if (ucode_rate & RATE_MCS_VHT_MSK_V1) {
-		nss = ((ucode_rate & RATE_VHT_MCS_NSS_MSK) >>
-		       RATE_VHT_MCS_NSS_POS) + 1;
+		nss = FIELD_GET(RATE_MCS_NSS_MSK, ucode_rate) + 1;
 
 		if (nss == 1) {
 			rate->type = LQ_VHT_SISO;
@@ -910,8 +909,7 @@ static int rs_rate_from_ucode_rate(const u32 ucode_rate,
 			WARN_ON_ONCE(1);
 		}
 	} else if (ucode_rate & RATE_MCS_HE_MSK_V1) {
-		nss = ((ucode_rate & RATE_VHT_MCS_NSS_MSK) >>
-		      RATE_VHT_MCS_NSS_POS) + 1;
+		nss = FIELD_GET(RATE_MCS_NSS_MSK, ucode_rate) + 1;
 
 		if (nss == 1) {
 			rate->type = LQ_HE_SISO;
@@ -1489,9 +1487,11 @@ static void rs_set_amsdu_len(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 			     enum rs_action scale_action)
 {
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
+	struct ieee80211_bss_conf *bss_conf = &mvmsta->vif->bss_conf;
 	int i;
 
-	sta->deflink.agg.max_amsdu_len = rs_fw_get_max_amsdu_len(sta);
+	sta->deflink.agg.max_amsdu_len =
+		rs_fw_get_max_amsdu_len(sta, bss_conf, &sta->deflink);
 
 	/*
 	 * In case TLC offload is not active amsdu_enabled is either 0xFFFF
@@ -1504,7 +1504,7 @@ static void rs_set_amsdu_len(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	else
 		mvmsta->amsdu_enabled = 0xFFFF;
 
-	if (mvmsta->vif->bss_conf.he_support &&
+	if (bss_conf->he_support &&
 	    !iwlwifi_mod_params.disable_11ax)
 		mvmsta->max_amsdu_len = sta->deflink.agg.max_amsdu_len;
 	else
@@ -2601,7 +2601,7 @@ void rs_update_last_rssi(struct iwl_mvm *mvm,
 			 struct iwl_mvm_sta *mvmsta,
 			 struct ieee80211_rx_status *rx_status)
 {
-	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta.rs_drv;
+	struct iwl_lq_sta *lq_sta = &mvmsta->deflink.lq_sta.rs_drv;
 	int i;
 
 	lq_sta->pers.chains = rx_status->chains;
@@ -2684,7 +2684,6 @@ static void rs_drv_get_rate(void *mvm_r, struct ieee80211_sta *sta,
 		/* if vif isn't initialized mvm doesn't know about
 		 * this station, so don't do anything with the it
 		 */
-		sta = NULL;
 		mvm_sta = NULL;
 	}
 
@@ -2714,7 +2713,7 @@ static void *rs_drv_alloc_sta(void *mvm_rate, struct ieee80211_sta *sta,
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_op_mode *op_mode = (struct iwl_op_mode *)mvm_rate;
 	struct iwl_mvm *mvm  = IWL_OP_MODE_GET_MVM(op_mode);
-	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta.rs_drv;
+	struct iwl_lq_sta *lq_sta = &mvmsta->deflink.lq_sta.rs_drv;
 
 	IWL_DEBUG_RATE(mvm, "create station rate scale window\n");
 
@@ -2885,8 +2884,7 @@ void iwl_mvm_update_frame_stats(struct iwl_mvm *mvm, u32 rate, bool agg)
 		nss = ((rate & RATE_HT_MCS_NSS_MSK_V1) >> RATE_HT_MCS_NSS_POS_V1) + 1;
 	} else if (rate & RATE_MCS_VHT_MSK_V1) {
 		mvm->drv_rx_stats.vht_frames++;
-		nss = ((rate & RATE_VHT_MCS_NSS_MSK) >>
-		       RATE_VHT_MCS_NSS_POS) + 1;
+		nss = FIELD_GET(RATE_MCS_NSS_MSK, rate) + 1;
 	} else {
 		mvm->drv_rx_stats.legacy_frames++;
 	}
@@ -2921,18 +2919,18 @@ static void rs_drv_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	struct ieee80211_sta_ht_cap *ht_cap = &sta->deflink.ht_cap;
 	struct ieee80211_sta_vht_cap *vht_cap = &sta->deflink.vht_cap;
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta.rs_drv;
+	struct iwl_lq_sta *lq_sta = &mvmsta->deflink.lq_sta.rs_drv;
 	struct ieee80211_supported_band *sband;
 	unsigned long supp; /* must be unsigned long for for_each_set_bit */
 
-	lockdep_assert_held(&mvmsta->lq_sta.rs_drv.pers.lock);
+	lockdep_assert_held(&mvmsta->deflink.lq_sta.rs_drv.pers.lock);
 
 	/* clear all non-persistent lq data */
 	memset(lq_sta, 0, offsetof(typeof(*lq_sta), pers));
 
 	sband = hw->wiphy->bands[band];
 
-	lq_sta->lq.sta_id = mvmsta->sta_id;
+	lq_sta->lq.sta_id = mvmsta->deflink.sta_id;
 	mvmsta->amsdu_enabled = 0;
 	mvmsta->max_amsdu_len = sta->cur->max_amsdu_len;
 
@@ -2944,7 +2942,7 @@ static void rs_drv_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 
 	IWL_DEBUG_RATE(mvm,
 		       "LQ: *** rate scale station global init for station %d ***\n",
-		       mvmsta->sta_id);
+		       mvmsta->deflink.sta_id);
 	/* TODO: what is a good starting rate for STA? About middle? Maybe not
 	 * the lowest or the highest rate.. Could consider using RSSI from
 	 * previous packets? Need to have IEEE 802.1X auth succeed immediately
@@ -3006,17 +3004,20 @@ static void rs_drv_rate_update(void *mvm_r,
 			       void *priv_sta, u32 changed)
 {
 	struct iwl_op_mode *op_mode = mvm_r;
+	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_mvm *mvm __maybe_unused = IWL_OP_MODE_GET_MVM(op_mode);
 	u8 tid;
 
-	if (!iwl_mvm_sta_from_mac80211(sta)->vif)
+	if (!mvmsta->vif)
 		return;
 
 	/* Stop any ongoing aggregations as rs starts off assuming no agg */
 	for (tid = 0; tid < IWL_MAX_TID_COUNT; tid++)
 		ieee80211_stop_tx_ba_session(sta, tid);
 
-	iwl_mvm_rs_rate_init(mvm, sta, sband->band, true);
+	iwl_mvm_rs_rate_init(mvm, mvmsta->vif, sta,
+			     &mvmsta->vif->bss_conf, &sta->deflink,
+			     sband->band);
 }
 
 static void __iwl_mvm_rs_tx_status(struct iwl_mvm *mvm,
@@ -3036,7 +3037,7 @@ static void __iwl_mvm_rs_tx_status(struct iwl_mvm *mvm,
 	u8 lq_color = RS_DRV_DATA_LQ_COLOR_GET(tlc_info);
 	u32 tx_resp_hwrate = (uintptr_t)info->status.status_driver_data[1];
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta.rs_drv;
+	struct iwl_lq_sta *lq_sta = &mvmsta->deflink.lq_sta.rs_drv;
 
 	if (!lq_sta->pers.drv) {
 		IWL_DEBUG_RATE(mvm, "Rate scaling not initialized yet.\n");
@@ -3260,11 +3261,11 @@ void iwl_mvm_rs_tx_status(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	/* If it's locked we are in middle of init flow
 	 * just wait for next tx status to update the lq_sta data
 	 */
-	if (!spin_trylock(&mvmsta->lq_sta.rs_drv.pers.lock))
+	if (!spin_trylock(&mvmsta->deflink.lq_sta.rs_drv.pers.lock))
 		return;
 
 	__iwl_mvm_rs_tx_status(mvm, sta, tid, info, ndp);
-	spin_unlock(&mvmsta->lq_sta.rs_drv.pers.lock);
+	spin_unlock(&mvmsta->deflink.lq_sta.rs_drv.pers.lock);
 }
 
 #ifdef CONFIG_MAC80211_DEBUGFS
@@ -3440,7 +3441,7 @@ static void rs_bfer_active_iter(void *_data,
 {
 	struct rs_bfer_active_iter_data *data = _data;
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_lq_cmd *lq_cmd = &mvmsta->lq_sta.rs_drv.lq;
+	struct iwl_lq_cmd *lq_cmd = &mvmsta->deflink.lq_sta.rs_drv.lq;
 	u32 ss_params = le32_to_cpu(lq_cmd->ss_params);
 
 	if (sta == data->exclude_sta)
@@ -3471,7 +3472,8 @@ static int rs_bfer_priority(struct iwl_mvm_sta *sta)
 		prio = 1;
 		break;
 	default:
-		WARN_ONCE(true, "viftype %d sta_id %d", viftype, sta->sta_id);
+		WARN_ONCE(true, "viftype %d sta_id %d", viftype,
+			  sta->deflink.sta_id);
 		prio = -1;
 	}
 
@@ -3548,12 +3550,12 @@ static void rs_set_lq_ss_params(struct iwl_mvm *mvm,
 	}
 
 	IWL_DEBUG_RATE(mvm, "Found existing sta %d with BFER activated\n",
-		       bfer_mvmsta->sta_id);
+		       bfer_mvmsta->deflink.sta_id);
 
 	/* Disallow BFER on another STA if active and we're a higher priority */
 	if (rs_bfer_priority_cmp(mvmsta, bfer_mvmsta) > 0) {
 		struct iwl_lq_cmd *bfersta_lq_cmd =
-			&bfer_mvmsta->lq_sta.rs_drv.lq;
+			&bfer_mvmsta->deflink.lq_sta.rs_drv.lq;
 		u32 bfersta_ss_params = le32_to_cpu(bfersta_lq_cmd->ss_params);
 
 		bfersta_ss_params &= ~LQ_SS_BFER_ALLOWED;
@@ -3563,7 +3565,7 @@ static void rs_set_lq_ss_params(struct iwl_mvm *mvm,
 		ss_params |= LQ_SS_BFER_ALLOWED;
 		IWL_DEBUG_RATE(mvm,
 			       "Lower priority BFER sta found (%d). Switch BFER\n",
-			       bfer_mvmsta->sta_id);
+			       bfer_mvmsta->deflink.sta_id);
 	}
 out:
 	lq_cmd->ss_params = cpu_to_le32(ss_params);
@@ -3605,7 +3607,7 @@ static void rs_fill_lq_cmd(struct iwl_mvm *mvm,
 	    num_of_ant(initial_rate->ant) == 1)
 		lq_cmd->single_stream_ant_msk = initial_rate->ant;
 
-	lq_cmd->agg_frame_cnt_limit = mvmsta->max_agg_bufsize;
+	lq_cmd->agg_frame_cnt_limit = lq_sta->pers.max_agg_bufsize;
 
 	/*
 	 * In case of low latency, tell the firmware to leave a frame in the
@@ -3665,8 +3667,7 @@ int rs_pretty_print_rate_v1(char *buf, int bufsz, const u32 rate)
 	if (rate & RATE_MCS_VHT_MSK_V1) {
 		type = "VHT";
 		mcs = rate & RATE_VHT_MCS_RATE_CODE_MSK;
-		nss = ((rate & RATE_VHT_MCS_NSS_MSK)
-		       >> RATE_VHT_MCS_NSS_POS) + 1;
+		nss = FIELD_GET(RATE_MCS_NSS_MSK, rate) + 1;
 	} else if (rate & RATE_MCS_HT_MSK_V1) {
 		type = "HT";
 		mcs = rate & RATE_HT_MCS_INDEX_MSK_V1;
@@ -3675,8 +3676,7 @@ int rs_pretty_print_rate_v1(char *buf, int bufsz, const u32 rate)
 	} else if (rate & RATE_MCS_HE_MSK_V1) {
 		type = "HE";
 		mcs = rate & RATE_VHT_MCS_RATE_CODE_MSK;
-		nss = ((rate & RATE_VHT_MCS_NSS_MSK)
-		       >> RATE_VHT_MCS_NSS_POS) + 1;
+		nss = FIELD_GET(RATE_MCS_NSS_MSK, rate) + 1;
 	} else {
 		type = "Unknown"; /* shouldn't happen */
 	}
@@ -3750,7 +3750,7 @@ static ssize_t rs_sta_dbgfs_scale_table_read(struct file *file,
 
 	struct iwl_lq_sta *lq_sta = file->private_data;
 	struct iwl_mvm_sta *mvmsta =
-		container_of(lq_sta, struct iwl_mvm_sta, lq_sta.rs_drv);
+		container_of(lq_sta, struct iwl_mvm_sta, deflink.lq_sta.rs_drv);
 	struct iwl_mvm *mvm;
 	struct iwl_scale_tbl_info *tbl = &(lq_sta->lq_info[lq_sta->active_tbl]);
 	struct rs_rate *rate = &tbl->rate;
@@ -4051,7 +4051,8 @@ static void rs_drv_add_sta_debugfs(void *mvm, void *priv_sta,
 	struct iwl_lq_sta *lq_sta = priv_sta;
 	struct iwl_mvm_sta *mvmsta;
 
-	mvmsta = container_of(lq_sta, struct iwl_mvm_sta, lq_sta.rs_drv);
+	mvmsta = container_of(lq_sta, struct iwl_mvm_sta,
+			      deflink.lq_sta.rs_drv);
 
 	if (!mvmsta->vif)
 		return;
@@ -4100,17 +4101,22 @@ static const struct rate_control_ops rs_mvm_ops_drv = {
 	.capa = RATE_CTRL_CAPA_VHT_EXT_NSS_BW,
 };
 
-void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
-			  enum nl80211_band band, bool update)
+void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_sta *sta,
+			  struct ieee80211_bss_conf *link_conf,
+			  struct ieee80211_link_sta *link_sta,
+			  enum nl80211_band band)
 {
 	if (iwl_mvm_has_tlc_offload(mvm)) {
-		rs_fw_rate_init(mvm, sta, band, update);
+		iwl_mvm_rs_fw_rate_init(mvm, vif, sta, link_conf,
+					link_sta, band);
 	} else {
 		struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 
-		spin_lock(&mvmsta->lq_sta.rs_drv.pers.lock);
+		spin_lock(&mvmsta->deflink.lq_sta.rs_drv.pers.lock);
 		rs_drv_rate_init(mvm, sta, band);
-		spin_unlock(&mvmsta->lq_sta.rs_drv.pers.lock);
+		spin_unlock(&mvmsta->deflink.lq_sta.rs_drv.pers.lock);
 	}
 }
 
@@ -4127,7 +4133,7 @@ void iwl_mvm_rate_control_unregister(void)
 static int rs_drv_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 				bool enable)
 {
-	struct iwl_lq_cmd *lq = &mvmsta->lq_sta.rs_drv.lq;
+	struct iwl_lq_cmd *lq = &mvmsta->deflink.lq_sta.rs_drv.lq;
 
 	lockdep_assert_held(&mvm->mutex);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs.h b/drivers/net/wireless/intel/iwlwifi/mvm/rs.h
index b7bc8c1b2dda..1ca375a5cf6b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs.h
@@ -1,10 +1,9 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /******************************************************************************
  *
- * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2017 Intel Deutschland GmbH
- * Copyright(c) 2018 - 2019 Intel Corporation
+ * Copyright (C) 2003 - 2014, 2018 - 2022 Intel Corporation
  *****************************************************************************/
 
 #ifndef __rs_h__
@@ -204,6 +203,7 @@ struct rs_rate {
 /**
  * struct iwl_lq_sta_rs_fw - rate and related statistics for RS in FW
  * @last_rate_n_flags: last rate reported by FW
+ * @max_agg_bufsize: the maximal size of the AGG buffer for this station
  * @sta_id: the id of the station
 #ifdef CONFIG_MAC80211_DEBUGFS
  * @dbg_fixed_rate: for debug, use fixed rate if not 0
@@ -353,6 +353,7 @@ struct iwl_lq_sta {
 
 	/* last tx rate_n_flags */
 	u32 last_rate_n_flags;
+
 	/* packets destined for this STA are aggregated */
 	u8 is_agg;
 
@@ -371,6 +372,7 @@ struct iwl_lq_sta {
 		u8 chains;
 		s8 chain_signal[IEEE80211_MAX_CHAINS];
 		s8 last_rssi;
+		u16 max_agg_bufsize;
 		struct rs_rate_stats tx_stats[RS_COLUMN_COUNT][IWL_RATE_COUNT];
 		struct iwl_mvm *drv;
 		spinlock_t lock; /* for races in reinit/update table */
@@ -392,8 +394,12 @@ struct iwl_lq_sta {
 				   ((_c) << RS_DRV_DATA_LQ_COLOR_POS)))
 
 /* Initialize station's rate scaling information after adding station */
-void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
-			  enum nl80211_band band, bool init);
+void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_sta *sta,
+			  struct ieee80211_bss_conf *link_conf,
+			  struct ieee80211_link_sta *link_sta,
+			  enum nl80211_band band);
 
 /* Notify RS about Tx status */
 void iwl_mvm_rs_tx_status(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
@@ -428,13 +434,24 @@ int iwl_mvm_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 void iwl_mvm_reset_frame_stats(struct iwl_mvm *mvm);
 #endif
 
+struct iwl_mvm_link_sta;
+
 void iwl_mvm_rs_add_sta(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta);
-void rs_fw_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
-		     enum nl80211_band band, bool update);
+void iwl_mvm_rs_add_sta_link(struct iwl_mvm *mvm,
+			     struct iwl_mvm_link_sta *link_sta);
+
+void iwl_mvm_rs_fw_rate_init(struct iwl_mvm *mvm,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_sta *sta,
+			     struct ieee80211_bss_conf *link_conf,
+			     struct ieee80211_link_sta *link_sta,
+			     enum nl80211_band band);
 int rs_fw_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 			bool enable);
 void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
 			      struct iwl_rx_cmd_buffer *rxb);
 
-u16 rs_fw_get_max_amsdu_len(struct ieee80211_sta *sta);
+u16 rs_fw_get_max_amsdu_len(struct ieee80211_sta *sta,
+			    struct ieee80211_bss_conf *link_conf,
+			    struct ieee80211_link_sta *link_sta);
 #endif /* __rs__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
index 49ca1e168fc5..b38b24246675 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
@@ -190,7 +190,7 @@ static u32 iwl_mvm_set_mac80211_rx_flag(struct iwl_mvm *mvm,
 	default:
 		/* Expected in monitor (not having the keys) */
 		if (!mvm->monitor_on)
-			IWL_ERR(mvm, "Unhandled alg: 0x%x\n", rx_pkt_status);
+			IWL_WARN(mvm, "Unhandled alg: 0x%x\n", rx_pkt_status);
 	}
 
 	return 0;
@@ -237,11 +237,11 @@ static void iwl_mvm_rx_handle_tcm(struct iwl_mvm *mvm,
 
 	if (mdata->opened_rx_ba_sessions ||
 	    mdata->uapsd_nonagg_detect.detected ||
-	    (!mvmvif->queue_params[IEEE80211_AC_VO].uapsd &&
-	     !mvmvif->queue_params[IEEE80211_AC_VI].uapsd &&
-	     !mvmvif->queue_params[IEEE80211_AC_BE].uapsd &&
-	     !mvmvif->queue_params[IEEE80211_AC_BK].uapsd) ||
-	    mvmsta->sta_id != mvmvif->ap_sta_id)
+	    (!mvmvif->deflink.queue_params[IEEE80211_AC_VO].uapsd &&
+	     !mvmvif->deflink.queue_params[IEEE80211_AC_VI].uapsd &&
+	     !mvmvif->deflink.queue_params[IEEE80211_AC_BE].uapsd &&
+	     !mvmvif->deflink.queue_params[IEEE80211_AC_BK].uapsd) ||
+	    mvmsta->deflink.sta_id != mvmvif->deflink.ap_sta_id)
 		return;
 
 	if (rate_n_flags & RATE_MCS_HT_MSK_V1) {
@@ -253,8 +253,7 @@ static void iwl_mvm_rx_handle_tcm(struct iwl_mvm *mvm,
 				ARRAY_SIZE(thresh_tpt)))
 			return;
 		thr = thresh_tpt[rate_n_flags & RATE_VHT_MCS_RATE_CODE_MSK];
-		thr *= 1 + ((rate_n_flags & RATE_VHT_MCS_NSS_MSK) >>
-					RATE_VHT_MCS_NSS_POS);
+		thr *= 1 + FIELD_GET(RATE_MCS_NSS_MSK, rate_n_flags);
 	}
 
 	thr <<= ((rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK_V1) >>
@@ -384,9 +383,10 @@ void iwl_mvm_rx_rx_mpdu(struct iwl_mvm *mvm, struct napi_struct *napi,
 		 * Don't even try to decrypt a MCAST frame that was received
 		 * before the managed vif is authorized, we'd fail anyway.
 		 */
-		if (vif->type == NL80211_IFTYPE_STATION &&
+		if (is_multicast_ether_addr(hdr->addr1) &&
+		    vif->type == NL80211_IFTYPE_STATION &&
 		    !mvmvif->authorized &&
-		    is_multicast_ether_addr(hdr->addr1)) {
+		    ieee80211_has_protected(hdr->frame_control)) {
 			IWL_DEBUG_DROP(mvm, "MCAST before the vif is authorized\n");
 			kfree_skb(skb);
 			rcu_read_unlock();
@@ -500,8 +500,7 @@ void iwl_mvm_rx_rx_mpdu(struct iwl_mvm *mvm, struct napi_struct *napi,
 		u8 stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >>
 				RATE_MCS_STBC_POS;
 		rx_status->nss =
-			((rate_n_flags & RATE_VHT_MCS_NSS_MSK) >>
-						RATE_VHT_MCS_NSS_POS) + 1;
+			FIELD_GET(RATE_MCS_NSS_MSK, rate_n_flags) + 1;
 		rx_status->rate_idx = rate_n_flags & RATE_VHT_MCS_RATE_CODE_MSK;
 		rx_status->encoding = RX_ENC_VHT;
 		rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
@@ -630,9 +629,9 @@ static void iwl_mvm_stat_iterator(void *_data, u8 *mac,
 	 * data copied into the "data" struct, but rather the data from
 	 * the notification directly.
 	 */
-	mvmvif->beacon_stats.num_beacons =
+	mvmvif->deflink.beacon_stats.num_beacons =
 		le32_to_cpu(data->beacon_counter[vif_id]);
-	mvmvif->beacon_stats.avg_signal =
+	mvmvif->deflink.beacon_stats.avg_signal =
 		-data->beacon_average_energy[vif_id];
 
 	if (mvmvif->id != id)
@@ -645,8 +644,8 @@ static void iwl_mvm_stat_iterator(void *_data, u8 *mac,
 	 * request to clear statistics
 	 */
 	if (le32_to_cpu(data->flags) & IWL_STATISTICS_REPLY_FLG_CLEAR)
-		mvmvif->beacon_stats.accu_num_beacons +=
-			mvmvif->beacon_stats.num_beacons;
+		mvmvif->deflink.beacon_stats.accu_num_beacons +=
+			mvmvif->deflink.beacon_stats.num_beacons;
 
 	iwl_mvm_update_vif_sig(vif, sig);
 }
@@ -668,17 +667,17 @@ static void iwl_mvm_stat_iterator_all_macs(void *_data, u8 *mac,
 
 	mac_stats = &data->per_mac_stats[vif_id];
 
-	mvmvif->beacon_stats.num_beacons =
+	mvmvif->deflink.beacon_stats.num_beacons =
 		le32_to_cpu(mac_stats->beacon_counter);
-	mvmvif->beacon_stats.avg_signal =
+	mvmvif->deflink.beacon_stats.avg_signal =
 		-le32_to_cpu(mac_stats->beacon_average_energy);
 
 	/* make sure that beacon statistics don't go backwards with TCM
 	 * request to clear statistics
 	 */
 	if (le32_to_cpu(data->flags) & IWL_STATISTICS_REPLY_FLG_CLEAR)
-		mvmvif->beacon_stats.accu_num_beacons +=
-			mvmvif->beacon_stats.num_beacons;
+		mvmvif->deflink.beacon_stats.accu_num_beacons +=
+			mvmvif->deflink.beacon_stats.num_beacons;
 
 	sig = -le32_to_cpu(mac_stats->beacon_filter_average_energy);
 	iwl_mvm_update_vif_sig(vif, sig);
@@ -714,14 +713,14 @@ static void iwl_mvm_stats_energy_iter(void *_data,
 {
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	u8 *energy = _data;
-	u32 sta_id = mvmsta->sta_id;
+	u32 sta_id = mvmsta->deflink.sta_id;
 
 	if (WARN_ONCE(sta_id >= IWL_MVM_STATION_COUNT_MAX, "sta_id %d >= %d",
 		      sta_id, IWL_MVM_STATION_COUNT_MAX))
 		return;
 
 	if (energy[sta_id])
-		mvmsta->avg_energy = energy[sta_id];
+		mvmsta->deflink.avg_energy = energy[sta_id];
 
 }
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
index 549dbe0be223..e1d02c260e69 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2023 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -9,6 +9,7 @@
 #include "iwl-trans.h"
 #include "mvm.h"
 #include "fw-api.h"
+#include "time-sync.h"
 
 static inline int iwl_mvm_check_pn(struct iwl_mvm *mvm, struct sk_buff *skb,
 				   int queue, struct ieee80211_sta *sta)
@@ -105,28 +106,12 @@ static int iwl_mvm_create_skb(struct iwl_mvm *mvm, struct sk_buff *skb,
 
 	/*
 	 * For non monitor interface strip the bytes the RADA might not have
-	 * removed. As monitor interface cannot exist with other interfaces
-	 * this removal is safe.
+	 * removed (it might be disabled, e.g. for mgmt frames). As a monitor
+	 * interface cannot exist with other interfaces, this removal is safe
+	 * and sufficient, in monitor mode there's no decryption being done.
 	 */
-	if (mic_crc_len && !ieee80211_hw_check(mvm->hw, RX_INCLUDES_FCS)) {
-		u32 pkt_flags = le32_to_cpu(pkt->len_n_flags);
-
-		/*
-		 * If RADA was not enabled then decryption was not performed so
-		 * the MIC cannot be removed.
-		 */
-		if (!(pkt_flags & FH_RSCSR_RADA_EN)) {
-			if (WARN_ON(crypt_len > mic_crc_len))
-				return -EINVAL;
-
-			mic_crc_len -= crypt_len;
-		}
-
-		if (WARN_ON(mic_crc_len > len))
-			return -EINVAL;
-
+	if (len > mic_crc_len && !ieee80211_hw_check(mvm->hw, RX_INCLUDES_FCS))
 		len -= mic_crc_len;
-	}
 
 	/* If frame is small enough to fit in skb->head, pull it completely.
 	 * If not, only pull ieee80211_hdr (including crypto if present, and
@@ -171,8 +156,7 @@ static int iwl_mvm_create_skb(struct iwl_mvm *mvm, struct sk_buff *skb,
 	 * Starting from Bz hardware, it calculates starting directly after
 	 * the MAC header, so that matches mac80211's expectation.
 	 */
-	if (skb->ip_summed == CHECKSUM_COMPLETE &&
-	    mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ) {
+	if (skb->ip_summed == CHECKSUM_COMPLETE) {
 		struct {
 			u8 hdr[6];
 			__be16 type;
@@ -187,7 +171,7 @@ static int iwl_mvm_create_skb(struct iwl_mvm *mvm, struct sk_buff *skb,
 			      shdr->type != htons(ETH_P_PAE) &&
 			      shdr->type != htons(ETH_P_TDLS))))
 			skb->ip_summed = CHECKSUM_NONE;
-		else
+		else if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
 			/* mac80211 assumes full CSUM including SNAP header */
 			skb_postpush_rcsum(skb, shdr, sizeof(*shdr));
 	}
@@ -205,49 +189,69 @@ static int iwl_mvm_create_skb(struct iwl_mvm *mvm, struct sk_buff *skb,
 	return 0;
 }
 
+/* put a TLV on the skb and return data pointer
+ *
+ * Also pad to 4 the len and zero out all data part
+ */
+static void *
+iwl_mvm_radiotap_put_tlv(struct sk_buff *skb, u16 type, u16 len)
+{
+	struct ieee80211_radiotap_tlv *tlv;
+
+	tlv = skb_put(skb, sizeof(*tlv));
+	tlv->type = cpu_to_le16(type);
+	tlv->len = cpu_to_le16(len);
+	return skb_put_zero(skb, ALIGN(len, 4));
+}
+
 static void iwl_mvm_add_rtap_sniffer_config(struct iwl_mvm *mvm,
 					    struct sk_buff *skb)
 {
 	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
-	struct ieee80211_vendor_radiotap *radiotap;
-	const int size = sizeof(*radiotap) + sizeof(__le16);
+	struct ieee80211_radiotap_vendor_content *radiotap;
+	const u16 vendor_data_len = sizeof(mvm->cur_aid);
 
 	if (!mvm->cur_aid)
 		return;
 
-	/* ensure alignment */
-	BUILD_BUG_ON((size + 2) % 4);
+	radiotap = iwl_mvm_radiotap_put_tlv(skb,
+					    IEEE80211_RADIOTAP_VENDOR_NAMESPACE,
+					    sizeof(*radiotap) + vendor_data_len);
 
-	radiotap = skb_put(skb, size + 2);
-	radiotap->align = 1;
 	/* Intel OUI */
 	radiotap->oui[0] = 0xf6;
 	radiotap->oui[1] = 0x54;
 	radiotap->oui[2] = 0x25;
 	/* radiotap sniffer config sub-namespace */
-	radiotap->subns = 1;
-	radiotap->present = 0x1;
-	radiotap->len = size - sizeof(*radiotap);
-	radiotap->pad = 2;
+	radiotap->oui_subtype = 1;
+	radiotap->vendor_type = 0;
 
 	/* fill the data now */
 	memcpy(radiotap->data, &mvm->cur_aid, sizeof(mvm->cur_aid));
-	/* and clear the padding */
-	memset(radiotap->data + sizeof(__le16), 0, radiotap->pad);
 
-	rx_status->flag |= RX_FLAG_RADIOTAP_VENDOR_DATA;
+	rx_status->flag |= RX_FLAG_RADIOTAP_TLV_AT_END;
 }
 
 /* iwl_mvm_pass_packet_to_mac80211 - passes the packet for mac80211 */
 static void iwl_mvm_pass_packet_to_mac80211(struct iwl_mvm *mvm,
 					    struct napi_struct *napi,
 					    struct sk_buff *skb, int queue,
-					    struct ieee80211_sta *sta)
+					    struct ieee80211_sta *sta,
+					    struct ieee80211_link_sta *link_sta)
 {
-	if (iwl_mvm_check_pn(mvm, skb, queue, sta))
+	if (unlikely(iwl_mvm_check_pn(mvm, skb, queue, sta))) {
 		kfree_skb(skb);
-	else
-		ieee80211_rx_napi(mvm->hw, sta, skb, napi);
+		return;
+	}
+
+	if (sta && sta->valid_links && link_sta) {
+		struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
+
+		rx_status->link_valid = 1;
+		rx_status->link_id = link_sta->link_id;
+	}
+
+	ieee80211_rx_napi(mvm->hw, sta, skb, napi);
 }
 
 static void iwl_mvm_get_signal_strength(struct iwl_mvm *mvm,
@@ -391,9 +395,7 @@ static int iwl_mvm_rx_crypto(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		if (!(status & IWL_RX_MPDU_STATUS_MIC_OK))
 			return -1;
 
-		stats->flag |= RX_FLAG_DECRYPTED;
-		if (pkt_flags & FH_RSCSR_RADA_EN)
-			stats->flag |= RX_FLAG_MIC_STRIPPED;
+		stats->flag |= RX_FLAG_DECRYPTED | RX_FLAG_MIC_STRIPPED;
 		*crypt_len = IEEE80211_CCMP_HDR_LEN;
 		return 0;
 	case IWL_RX_MPDU_STATUS_SEC_TKIP:
@@ -443,7 +445,7 @@ static int iwl_mvm_rx_crypto(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		 */
 		if (!is_multicast_ether_addr(hdr->addr1) &&
 		    !mvm->monitor_on && net_ratelimit())
-			IWL_ERR(mvm, "Unhandled alg: 0x%x\n", status);
+			IWL_WARN(mvm, "Unhandled alg: 0x%x\n", status);
 	}
 
 	return 0;
@@ -620,7 +622,7 @@ static void iwl_mvm_release_frames(struct iwl_mvm *mvm,
 		while ((skb = __skb_dequeue(skb_list))) {
 			iwl_mvm_pass_packet_to_mac80211(mvm, napi, skb,
 							reorder_buf->queue,
-							sta);
+							sta, NULL /* FIXME */);
 			reorder_buf->num_stored--;
 		}
 	}
@@ -685,7 +687,7 @@ void iwl_mvm_reorder_timer_expired(struct timer_list *t)
 	if (expired) {
 		struct ieee80211_sta *sta;
 		struct iwl_mvm_sta *mvmsta;
-		u8 sta_id = baid_data->sta_id;
+		u8 sta_id = ffs(baid_data->sta_mask) - 1;
 
 		rcu_read_lock();
 		sta = rcu_dereference(buf->mvm->fw_id_to_mac_id[sta_id]);
@@ -720,6 +722,7 @@ static void iwl_mvm_del_ba(struct iwl_mvm *mvm, int queue,
 	struct ieee80211_sta *sta;
 	struct iwl_mvm_reorder_buffer *reorder_buf;
 	u8 baid = data->baid;
+	u32 sta_id;
 
 	if (WARN_ONCE(baid >= IWL_MAX_BAID, "invalid BAID: %x\n", baid))
 		return;
@@ -730,7 +733,9 @@ static void iwl_mvm_del_ba(struct iwl_mvm *mvm, int queue,
 	if (WARN_ON_ONCE(!ba_data))
 		goto out;
 
-	sta = rcu_dereference(mvm->fw_id_to_mac_id[ba_data->sta_id]);
+	/* pick any STA ID to find the pointer */
+	sta_id = ffs(ba_data->sta_mask) - 1;
+	sta = rcu_dereference(mvm->fw_id_to_mac_id[sta_id]);
 	if (WARN_ON_ONCE(IS_ERR_OR_NULL(sta)))
 		goto out;
 
@@ -757,6 +762,7 @@ static void iwl_mvm_release_frames_from_notif(struct iwl_mvm *mvm,
 	struct ieee80211_sta *sta;
 	struct iwl_mvm_reorder_buffer *reorder_buf;
 	struct iwl_mvm_baid_data *ba_data;
+	u32 sta_id;
 
 	IWL_DEBUG_HT(mvm, "Frame release notification for BAID %u, NSSN %d\n",
 		     baid, nssn);
@@ -774,7 +780,9 @@ static void iwl_mvm_release_frames_from_notif(struct iwl_mvm *mvm,
 		goto out;
 	}
 
-	sta = rcu_dereference(mvm->fw_id_to_mac_id[ba_data->sta_id]);
+	/* pick any STA ID to find the pointer */
+	sta_id = ffs(ba_data->sta_mask) - 1;
+	sta = rcu_dereference(mvm->fw_id_to_mac_id[sta_id]);
 	if (WARN_ON_ONCE(IS_ERR_OR_NULL(sta)))
 		goto out;
 
@@ -915,7 +923,6 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 {
 	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_hdr *hdr = (void *)skb_mac_header(skb);
-	struct iwl_mvm_sta *mvm_sta;
 	struct iwl_mvm_baid_data *baid_data;
 	struct iwl_mvm_reorder_buffer *buffer;
 	struct sk_buff *tail;
@@ -927,6 +934,7 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 	u8 sub_frame_idx = desc->amsdu_info &
 			   IWL_RX_MPDU_AMSDU_SUBFRAME_IDX_MASK;
 	struct iwl_mvm_reorder_buf_entry *entries;
+	u32 sta_mask;
 	int index;
 	u16 nssn, sn;
 	u8 baid;
@@ -949,8 +957,6 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 		      "Got valid BAID without a valid station assigned\n"))
 		return false;
 
-	mvm_sta = iwl_mvm_sta_from_mac80211(sta);
-
 	/* not a data packet or a bar */
 	if (!ieee80211_is_back_req(hdr->frame_control) &&
 	    (!ieee80211_is_data_qos(hdr->frame_control) ||
@@ -968,10 +974,14 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 		return false;
 	}
 
-	if (WARN(tid != baid_data->tid || mvm_sta->sta_id != baid_data->sta_id,
-		 "baid 0x%x is mapped to sta:%d tid:%d, but was received for sta:%d tid:%d\n",
-		 baid, baid_data->sta_id, baid_data->tid, mvm_sta->sta_id,
-		 tid))
+	rcu_read_lock();
+	sta_mask = iwl_mvm_sta_fw_id_mask(mvm, sta, -1);
+	rcu_read_unlock();
+
+	if (WARN(tid != baid_data->tid ||
+		 !(sta_mask & baid_data->sta_mask),
+		 "baid 0x%x is mapped to sta_mask:0x%x tid:%d, but was received for sta_mask:0x%x tid:%d\n",
+		 baid, baid_data->sta_mask, baid_data->tid, sta_mask, tid))
 		return false;
 
 	nssn = reorder & IWL_RX_MPDU_REORDER_NSSN_MASK;
@@ -1167,8 +1177,11 @@ static void iwl_mvm_flip_address(u8 *addr)
 
 struct iwl_mvm_rx_phy_data {
 	enum iwl_rx_phy_info_type info_type;
-	__le32 d0, d1, d2, d3;
+	__le32 d0, d1, d2, d3, eht_d4, d5;
 	__le16 d4;
+	bool with_data;
+	bool first_subframe;
+	__le32 rx_vec[4];
 
 	u32 rate_n_flags;
 	u32 gp2_on_air_rise;
@@ -1446,6 +1459,528 @@ static void iwl_mvm_decode_he_phy_data(struct iwl_mvm *mvm,
 	}
 }
 
+#define LE32_DEC_ENC(value, dec_bits, enc_bits) \
+	le32_encode_bits(le32_get_bits(value, dec_bits), enc_bits)
+
+#define IWL_MVM_ENC_USIG_VALUE_MASK(usig, in_value, dec_bits, enc_bits) do { \
+	typeof(enc_bits) _enc_bits = enc_bits; \
+	typeof(usig) _usig = usig; \
+	(_usig)->mask |= cpu_to_le32(_enc_bits); \
+	(_usig)->value |= LE32_DEC_ENC(in_value, dec_bits, _enc_bits); \
+} while (0)
+
+#define __IWL_MVM_ENC_EHT_RU(rt_data, rt_ru, fw_data, fw_ru) \
+	eht->data[(rt_data)] |= \
+		(cpu_to_le32 \
+		 (IEEE80211_RADIOTAP_EHT_DATA ## rt_data ## _RU_ALLOC_CC_ ## rt_ru ## _KNOWN) | \
+		 LE32_DEC_ENC(data ## fw_data, \
+			      IWL_RX_PHY_DATA ## fw_data ## _EHT_MU_EXT_RU_ALLOC_ ## fw_ru, \
+			      IEEE80211_RADIOTAP_EHT_DATA ## rt_data ## _RU_ALLOC_CC_ ## rt_ru))
+
+#define _IWL_MVM_ENC_EHT_RU(rt_data, rt_ru, fw_data, fw_ru)	\
+	__IWL_MVM_ENC_EHT_RU(rt_data, rt_ru, fw_data, fw_ru)
+
+#define IEEE80211_RADIOTAP_RU_DATA_1_1_1	1
+#define IEEE80211_RADIOTAP_RU_DATA_2_1_1	2
+#define IEEE80211_RADIOTAP_RU_DATA_1_1_2	2
+#define IEEE80211_RADIOTAP_RU_DATA_2_1_2	2
+#define IEEE80211_RADIOTAP_RU_DATA_1_2_1	3
+#define IEEE80211_RADIOTAP_RU_DATA_2_2_1	3
+#define IEEE80211_RADIOTAP_RU_DATA_1_2_2	3
+#define IEEE80211_RADIOTAP_RU_DATA_2_2_2	4
+
+#define IWL_RX_RU_DATA_A1			2
+#define IWL_RX_RU_DATA_A2			2
+#define IWL_RX_RU_DATA_B1			2
+#define IWL_RX_RU_DATA_B2			3
+#define IWL_RX_RU_DATA_C1			3
+#define IWL_RX_RU_DATA_C2			3
+#define IWL_RX_RU_DATA_D1			4
+#define IWL_RX_RU_DATA_D2			4
+
+#define IWL_MVM_ENC_EHT_RU(rt_ru, fw_ru)				\
+	_IWL_MVM_ENC_EHT_RU(IEEE80211_RADIOTAP_RU_DATA_ ## rt_ru,	\
+			    rt_ru,					\
+			    IWL_RX_RU_DATA_ ## fw_ru,			\
+			    fw_ru)
+
+static void iwl_mvm_decode_eht_ext_mu(struct iwl_mvm *mvm,
+				      struct iwl_mvm_rx_phy_data *phy_data,
+				      struct ieee80211_rx_status *rx_status,
+				      struct ieee80211_radiotap_eht *eht,
+				      struct ieee80211_radiotap_eht_usig *usig)
+{
+	if (phy_data->with_data) {
+		__le32 data1 = phy_data->d1;
+		__le32 data2 = phy_data->d2;
+		__le32 data3 = phy_data->d3;
+		__le32 data4 = phy_data->eht_d4;
+		__le32 data5 = phy_data->d5;
+		u32 phy_bw = phy_data->rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK;
+
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, data5,
+					    IWL_RX_PHY_DATA5_EHT_TYPE_AND_COMP,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, data5,
+					    IWL_RX_PHY_DATA5_EHT_MU_PUNC_CH_CODE,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B3_B7_PUNCTURED_INFO);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, data4,
+					    IWL_RX_PHY_DATA4_EHT_MU_EXT_SIGB_MCS,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS);
+		IWL_MVM_ENC_USIG_VALUE_MASK
+			(usig, data1, IWL_RX_PHY_DATA1_EHT_MU_NUM_SIG_SYM_USIGA2,
+			 IEEE80211_RADIOTAP_EHT_USIG2_MU_B11_B15_EHT_SIG_SYMBOLS);
+
+		eht->user_info[0] |=
+			cpu_to_le32(IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN) |
+			LE32_DEC_ENC(data5, IWL_RX_PHY_DATA5_EHT_MU_STA_ID_USR,
+				     IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID);
+
+		eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_NR_NON_OFDMA_USERS_M);
+		eht->data[7] |= LE32_DEC_ENC
+			(data5, IWL_RX_PHY_DATA5_EHT_MU_NUM_USR_NON_OFDMA,
+			 IEEE80211_RADIOTAP_EHT_DATA7_NUM_OF_NON_OFDMA_USERS);
+
+		/*
+		 * Hardware labels the content channels/RU allocation values
+		 * as follows:
+		 *           Content Channel 1		Content Channel 2
+		 *   20 MHz: A1
+		 *   40 MHz: A1				B1
+		 *   80 MHz: A1 C1			B1 D1
+		 *  160 MHz: A1 C1 A2 C2		B1 D1 B2 D2
+		 *  320 MHz: A1 C1 A2 C2 A3 C3 A4 C4	B1 D1 B2 D2 B3 D3 B4 D4
+		 *
+		 * However firmware can only give us A1-D2, so the higher
+		 * frequencies are missing.
+		 */
+
+		switch (phy_bw) {
+		case RATE_MCS_CHAN_WIDTH_320:
+			/* additional values are missing in RX metadata */
+		case RATE_MCS_CHAN_WIDTH_160:
+			/* content channel 1 */
+			IWL_MVM_ENC_EHT_RU(1_2_1, A2);
+			IWL_MVM_ENC_EHT_RU(1_2_2, C2);
+			/* content channel 2 */
+			IWL_MVM_ENC_EHT_RU(2_2_1, B2);
+			IWL_MVM_ENC_EHT_RU(2_2_2, D2);
+			fallthrough;
+		case RATE_MCS_CHAN_WIDTH_80:
+			/* content channel 1 */
+			IWL_MVM_ENC_EHT_RU(1_1_2, C1);
+			/* content channel 2 */
+			IWL_MVM_ENC_EHT_RU(2_1_2, D1);
+			fallthrough;
+		case RATE_MCS_CHAN_WIDTH_40:
+			/* content channel 2 */
+			IWL_MVM_ENC_EHT_RU(2_1_1, B1);
+			fallthrough;
+		case RATE_MCS_CHAN_WIDTH_20:
+			IWL_MVM_ENC_EHT_RU(1_1_1, A1);
+			break;
+		}
+	} else {
+		__le32 usig_a1 = phy_data->rx_vec[0];
+		__le32 usig_a2 = phy_data->rx_vec[1];
+
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a1,
+					    IWL_RX_USIG_A1_DISREGARD,
+					    IEEE80211_RADIOTAP_EHT_USIG1_MU_B20_B24_DISREGARD);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a1,
+					    IWL_RX_USIG_A1_VALIDATE,
+					    IEEE80211_RADIOTAP_EHT_USIG1_MU_B25_VALIDATE);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_PPDU_TYPE,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_USIG2_VALIDATE_B2,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B2_VALIDATE);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_PUNC_CHANNEL,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B3_B7_PUNCTURED_INFO);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_USIG2_VALIDATE_B8,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B8_VALIDATE);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_SIG_MCS,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS);
+		IWL_MVM_ENC_USIG_VALUE_MASK
+			(usig, usig_a2, IWL_RX_USIG_A2_EHT_SIG_SYM_NUM,
+			 IEEE80211_RADIOTAP_EHT_USIG2_MU_B11_B15_EHT_SIG_SYMBOLS);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_CRC_OK,
+					    IEEE80211_RADIOTAP_EHT_USIG2_MU_B16_B19_CRC);
+	}
+}
+
+static void iwl_mvm_decode_eht_ext_tb(struct iwl_mvm *mvm,
+				      struct iwl_mvm_rx_phy_data *phy_data,
+				      struct ieee80211_rx_status *rx_status,
+				      struct ieee80211_radiotap_eht *eht,
+				      struct ieee80211_radiotap_eht_usig *usig)
+{
+	if (phy_data->with_data) {
+		__le32 data5 = phy_data->d5;
+
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, data5,
+					    IWL_RX_PHY_DATA5_EHT_TYPE_AND_COMP,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, data5,
+					    IWL_RX_PHY_DATA5_EHT_TB_SPATIAL_REUSE1,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B3_B6_SPATIAL_REUSE_1);
+
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, data5,
+					    IWL_RX_PHY_DATA5_EHT_TB_SPATIAL_REUSE2,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B7_B10_SPATIAL_REUSE_2);
+	} else {
+		__le32 usig_a1 = phy_data->rx_vec[0];
+		__le32 usig_a2 = phy_data->rx_vec[1];
+
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a1,
+					    IWL_RX_USIG_A1_DISREGARD,
+					    IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_PPDU_TYPE,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_USIG2_VALIDATE_B2,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B2_VALIDATE);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_TRIG_SPATIAL_REUSE_1,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B3_B6_SPATIAL_REUSE_1);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_TRIG_SPATIAL_REUSE_2,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B7_B10_SPATIAL_REUSE_2);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_TRIG_USIG2_DISREGARD,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B11_B15_DISREGARD);
+		IWL_MVM_ENC_USIG_VALUE_MASK(usig, usig_a2,
+					    IWL_RX_USIG_A2_EHT_CRC_OK,
+					    IEEE80211_RADIOTAP_EHT_USIG2_TB_B16_B19_CRC);
+	}
+}
+
+static void iwl_mvm_decode_eht_ru(struct iwl_mvm *mvm,
+				  struct ieee80211_rx_status *rx_status,
+				  struct ieee80211_radiotap_eht *eht)
+{
+	u32 ru = le32_get_bits(eht->data[8],
+			       IEEE80211_RADIOTAP_EHT_DATA8_RU_ALLOC_TB_FMT_B7_B1);
+	enum nl80211_eht_ru_alloc nl_ru;
+
+	/* Using D1.5 Table 9-53a - Encoding of PS160 and RU Allocation subfields
+	 * in an EHT variant User Info field
+	 */
+
+	switch (ru) {
+	case 0 ... 36:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_26;
+		break;
+	case 37 ... 52:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_52;
+		break;
+	case 53 ... 60:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_106;
+		break;
+	case 61 ... 64:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_242;
+		break;
+	case 65 ... 66:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_484;
+		break;
+	case 67:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_996;
+		break;
+	case 68:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_2x996;
+		break;
+	case 69:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_4x996;
+		break;
+	case 70 ... 81:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_52P26;
+		break;
+	case 82 ... 89:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_106P26;
+		break;
+	case 90 ... 93:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_484P242;
+		break;
+	case 94 ... 95:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_996P484;
+		break;
+	case 96 ... 99:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_996P484P242;
+		break;
+	case 100 ... 103:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_2x996P484;
+		break;
+	case 104:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_3x996;
+		break;
+	case 105 ... 106:
+		nl_ru = NL80211_RATE_INFO_EHT_RU_ALLOC_3x996P484;
+		break;
+	default:
+		return;
+	}
+
+	rx_status->bw = RATE_INFO_BW_EHT_RU;
+	rx_status->eht.ru = nl_ru;
+}
+
+static void iwl_mvm_decode_eht_phy_data(struct iwl_mvm *mvm,
+					struct iwl_mvm_rx_phy_data *phy_data,
+					struct ieee80211_rx_status *rx_status,
+					struct ieee80211_radiotap_eht *eht,
+					struct ieee80211_radiotap_eht_usig *usig)
+
+{
+	__le32 data0 = phy_data->d0;
+	__le32 data1 = phy_data->d1;
+	__le32 usig_a1 = phy_data->rx_vec[0];
+	u8 info_type = phy_data->info_type;
+
+	/* Not in EHT range */
+	if (info_type < IWL_RX_PHY_INFO_TYPE_EHT_MU ||
+	    info_type > IWL_RX_PHY_INFO_TYPE_EHT_TB_EXT)
+		return;
+
+	usig->common |= cpu_to_le32
+		(IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL_KNOWN |
+		 IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR_KNOWN);
+	if (phy_data->with_data) {
+		usig->common |= LE32_DEC_ENC(data0,
+					     IWL_RX_PHY_DATA0_EHT_UPLINK,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL);
+		usig->common |= LE32_DEC_ENC(data0,
+					     IWL_RX_PHY_DATA0_EHT_BSS_COLOR_MASK,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR);
+	} else {
+		usig->common |= LE32_DEC_ENC(usig_a1,
+					     IWL_RX_USIG_A1_UL_FLAG,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL);
+		usig->common |= LE32_DEC_ENC(usig_a1,
+					     IWL_RX_USIG_A1_BSS_COLOR,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR);
+	}
+
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_SPATIAL_REUSE);
+	eht->data[0] |= LE32_DEC_ENC(data0,
+				     IWL_RX_PHY_DATA0_ETH_SPATIAL_REUSE_MASK,
+				     IEEE80211_RADIOTAP_EHT_DATA0_SPATIAL_REUSE);
+
+	/* All RU allocating size/index is in TB format */
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_RU_ALLOC_TB_FMT);
+	eht->data[8] |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_PS160,
+				     IEEE80211_RADIOTAP_EHT_DATA8_RU_ALLOC_TB_FMT_PS_160);
+	eht->data[8] |= LE32_DEC_ENC(data1, IWL_RX_PHY_DATA1_EHT_B0,
+				     IEEE80211_RADIOTAP_EHT_DATA8_RU_ALLOC_TB_FMT_B0);
+	eht->data[8] |= LE32_DEC_ENC(data1, IWL_RX_PHY_DATA1_EHT_RU_B1_B7_ALLOC,
+				     IEEE80211_RADIOTAP_EHT_DATA8_RU_ALLOC_TB_FMT_B7_B1);
+
+	iwl_mvm_decode_eht_ru(mvm, rx_status, eht);
+
+	/* We only get here in case of IWL_RX_MPDU_PHY_TSF_OVERLOAD is set
+	 * which is on only in case of monitor mode so no need to check monitor
+	 * mode
+	 */
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_PRIMARY_80);
+	eht->data[1] |=
+		le32_encode_bits(mvm->monitor_p80,
+				 IEEE80211_RADIOTAP_EHT_DATA1_PRIMARY_80);
+
+	usig->common |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP_KNOWN);
+	if (phy_data->with_data)
+		usig->common |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_TXOP_DUR_MASK,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP);
+	else
+		usig->common |= LE32_DEC_ENC(usig_a1, IWL_RX_USIG_A1_TXOP_DURATION,
+					     IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP);
+
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_LDPC_EXTRA_SYM_OM);
+	eht->data[0] |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_LDPC_EXT_SYM,
+				     IEEE80211_RADIOTAP_EHT_DATA0_LDPC_EXTRA_SYM_OM);
+
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_PRE_PADD_FACOR_OM);
+	eht->data[0] |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_PRE_FEC_PAD_MASK,
+				    IEEE80211_RADIOTAP_EHT_DATA0_PRE_PADD_FACOR_OM);
+
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_PE_DISAMBIGUITY_OM);
+	eht->data[0] |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_PE_DISAMBIG,
+				     IEEE80211_RADIOTAP_EHT_DATA0_PE_DISAMBIGUITY_OM);
+
+	/* TODO: what about IWL_RX_PHY_DATA0_EHT_BW320_SLOT */
+
+	if (!le32_get_bits(data0, IWL_RX_PHY_DATA0_EHT_SIGA_CRC_OK))
+		usig->common |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_USIG_COMMON_BAD_USIG_CRC);
+
+	usig->common |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER_KNOWN);
+	usig->common |= LE32_DEC_ENC(data0, IWL_RX_PHY_DATA0_EHT_PHY_VER,
+				     IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER);
+
+	/*
+	 * TODO: what about TB - IWL_RX_PHY_DATA1_EHT_TB_PILOT_TYPE,
+	 *			 IWL_RX_PHY_DATA1_EHT_TB_LOW_SS
+	 */
+
+	eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF);
+	eht->data[0] |= LE32_DEC_ENC(data1, IWL_RX_PHY_DATA1_EHT_SIG_LTF_NUM,
+				     IEEE80211_RADIOTAP_EHT_DATA0_EHT_LTF);
+
+	if (info_type == IWL_RX_PHY_INFO_TYPE_EHT_TB_EXT ||
+	    info_type == IWL_RX_PHY_INFO_TYPE_EHT_TB)
+		iwl_mvm_decode_eht_ext_tb(mvm, phy_data, rx_status, eht, usig);
+
+	if (info_type == IWL_RX_PHY_INFO_TYPE_EHT_MU_EXT ||
+	    info_type == IWL_RX_PHY_INFO_TYPE_EHT_MU)
+		iwl_mvm_decode_eht_ext_mu(mvm, phy_data, rx_status, eht, usig);
+}
+
+static void iwl_mvm_rx_eht(struct iwl_mvm *mvm, struct sk_buff *skb,
+			   struct iwl_mvm_rx_phy_data *phy_data,
+			   int queue)
+{
+	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
+
+	struct ieee80211_radiotap_eht *eht;
+	struct ieee80211_radiotap_eht_usig *usig;
+	size_t eht_len = sizeof(*eht);
+
+	u32 rate_n_flags = phy_data->rate_n_flags;
+	u32 he_type = rate_n_flags & RATE_MCS_HE_TYPE_MSK;
+	/* EHT and HE have the same valus for LTF */
+	u8 ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_UNKNOWN;
+	u16 phy_info = phy_data->phy_info;
+	u32 bw;
+
+	/* u32 for 1 user_info */
+	if (phy_data->with_data)
+		eht_len += sizeof(u32);
+
+	eht = iwl_mvm_radiotap_put_tlv(skb, IEEE80211_RADIOTAP_EHT, eht_len);
+
+	usig = iwl_mvm_radiotap_put_tlv(skb, IEEE80211_RADIOTAP_EHT_USIG,
+					sizeof(*usig));
+	rx_status->flag |= RX_FLAG_RADIOTAP_TLV_AT_END;
+	usig->common |=
+		cpu_to_le32(IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW_KNOWN);
+
+	/* specific handling for 320MHz */
+	bw = FIELD_GET(RATE_MCS_CHAN_WIDTH_MSK, rate_n_flags);
+	if (bw == RATE_MCS_CHAN_WIDTH_320_VAL)
+		bw += FIELD_GET(IWL_RX_PHY_DATA0_EHT_BW320_SLOT,
+				le32_to_cpu(phy_data->d0));
+
+	usig->common |= cpu_to_le32
+		(FIELD_PREP(IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW, bw));
+
+	/* report the AMPDU-EOF bit on single frames */
+	if (!queue && !(phy_info & IWL_RX_MPDU_PHY_AMPDU)) {
+		rx_status->flag |= RX_FLAG_AMPDU_DETAILS;
+		rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT_KNOWN;
+		if (phy_data->d0 & cpu_to_le32(IWL_RX_PHY_DATA0_EHT_DELIM_EOF))
+			rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT;
+	}
+
+	/* update aggregation data for monitor sake on default queue */
+	if (!queue && (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD) &&
+	    (phy_info & IWL_RX_MPDU_PHY_AMPDU) && phy_data->first_subframe) {
+		rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT_KNOWN;
+		if (phy_data->d0 & cpu_to_le32(IWL_RX_PHY_DATA0_EHT_DELIM_EOF))
+			rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT;
+	}
+
+	if (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD)
+		iwl_mvm_decode_eht_phy_data(mvm, phy_data, rx_status, eht, usig);
+
+#define CHECK_TYPE(F)							\
+	BUILD_BUG_ON(IEEE80211_RADIOTAP_HE_DATA1_FORMAT_ ## F !=	\
+		     (RATE_MCS_HE_TYPE_ ## F >> RATE_MCS_HE_TYPE_POS))
+
+	CHECK_TYPE(SU);
+	CHECK_TYPE(EXT_SU);
+	CHECK_TYPE(MU);
+	CHECK_TYPE(TRIG);
+
+	switch (FIELD_GET(RATE_MCS_HE_GI_LTF_MSK, rate_n_flags)) {
+	case 0:
+		if (he_type == RATE_MCS_HE_TYPE_TRIG) {
+			rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_1_6;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_1X;
+		} else {
+			rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_0_8;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_2X;
+		}
+		break;
+	case 1:
+		rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_1_6;
+		ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_2X;
+		break;
+	case 2:
+		ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X;
+		if (he_type == RATE_MCS_HE_TYPE_TRIG)
+			rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_3_2;
+		else
+			rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_0_8;
+		break;
+	case 3:
+		if (he_type != RATE_MCS_HE_TYPE_TRIG) {
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X;
+			rx_status->eht.gi = NL80211_RATE_INFO_EHT_GI_3_2;
+		}
+		break;
+	default:
+		/* nothing here */
+		break;
+	}
+
+	if (ltf != IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_UNKNOWN) {
+		eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_GI);
+		eht->data[0] |= cpu_to_le32
+			(FIELD_PREP(IEEE80211_RADIOTAP_EHT_DATA0_LTF,
+				    ltf) |
+			 FIELD_PREP(IEEE80211_RADIOTAP_EHT_DATA0_GI,
+				    rx_status->eht.gi));
+	}
+
+
+	if (!phy_data->with_data) {
+		eht->known |= cpu_to_le32(IEEE80211_RADIOTAP_EHT_KNOWN_NSS_S |
+					  IEEE80211_RADIOTAP_EHT_KNOWN_BEAMFORMED_S);
+		eht->data[7] |=
+			le32_encode_bits(le32_get_bits(phy_data->rx_vec[2],
+						       RX_NO_DATA_RX_VEC2_EHT_NSTS_MSK),
+					 IEEE80211_RADIOTAP_EHT_DATA7_NSS_S);
+		if (rate_n_flags & RATE_MCS_BF_MSK)
+			eht->data[7] |=
+				cpu_to_le32(IEEE80211_RADIOTAP_EHT_DATA7_BEAMFORMED_S);
+	} else {
+		eht->user_info[0] |=
+			cpu_to_le32(IEEE80211_RADIOTAP_EHT_USER_INFO_MCS_KNOWN |
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_CODING_KNOWN |
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_KNOWN_O |
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_KNOWN_O |
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_DATA_FOR_USER);
+
+		if (rate_n_flags & RATE_MCS_BF_MSK)
+			eht->user_info[0] |=
+				cpu_to_le32(IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_O);
+
+		if (rate_n_flags & RATE_MCS_LDPC_MSK)
+			eht->user_info[0] |=
+				cpu_to_le32(IEEE80211_RADIOTAP_EHT_USER_INFO_CODING);
+
+		eht->user_info[0] |= cpu_to_le32
+			(FIELD_PREP(IEEE80211_RADIOTAP_EHT_USER_INFO_MCS,
+				    FIELD_GET(RATE_VHT_MCS_RATE_CODE_MSK,
+					      rate_n_flags)) |
+			 FIELD_PREP(IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_O,
+				    FIELD_GET(RATE_MCS_NSS_MSK, rate_n_flags)));
+	}
+}
+
 static void iwl_mvm_rx_he(struct iwl_mvm *mvm, struct sk_buff *skb,
 			  struct iwl_mvm_rx_phy_data *phy_data,
 			  int queue)
@@ -1497,15 +2032,10 @@ static void iwl_mvm_rx_he(struct iwl_mvm *mvm, struct sk_buff *skb,
 
 	/* update aggregation data for monitor sake on default queue */
 	if (!queue && (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD) &&
-	    (phy_info & IWL_RX_MPDU_PHY_AMPDU)) {
-		bool toggle_bit = phy_info & IWL_RX_MPDU_PHY_AMPDU_TOGGLE;
-
-		/* toggle is switched whenever new aggregation starts */
-		if (toggle_bit != mvm->ampdu_toggle) {
-			rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT_KNOWN;
-			if (phy_data->d0 & cpu_to_le32(IWL_RX_PHY_DATA0_HE_DELIM_EOF))
-				rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT;
-		}
+	    (phy_info & IWL_RX_MPDU_PHY_AMPDU) && phy_data->first_subframe) {
+		rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT_KNOWN;
+		if (phy_data->d0 & cpu_to_le32(IWL_RX_PHY_DATA0_EHT_DELIM_EOF))
+			rx_status->flag |= RX_FLAG_AMPDU_EOF_BIT;
 	}
 
 	if (he_type == RATE_MCS_HE_TYPE_EXT_SU &&
@@ -1593,6 +2123,10 @@ static void iwl_mvm_decode_lsig(struct sk_buff *skb,
 	case IWL_RX_PHY_INFO_TYPE_HE_MU:
 	case IWL_RX_PHY_INFO_TYPE_HE_MU_EXT:
 	case IWL_RX_PHY_INFO_TYPE_HE_TB:
+	case IWL_RX_PHY_INFO_TYPE_EHT_MU:
+	case IWL_RX_PHY_INFO_TYPE_EHT_TB:
+	case IWL_RX_PHY_INFO_TYPE_EHT_MU_EXT:
+	case IWL_RX_PHY_INFO_TYPE_EHT_TB_EXT:
 		lsig = skb_put(skb, sizeof(*lsig));
 		lsig->data1 = cpu_to_le16(IEEE80211_RADIOTAP_LSIG_DATA1_LENGTH_KNOWN);
 		lsig->data2 = le16_encode_bits(le32_get_bits(phy_data->d1,
@@ -1689,6 +2223,10 @@ static void iwl_mvm_rx_fill_status(struct iwl_mvm *mvm,
 	iwl_mvm_get_signal_strength(mvm, rx_status, rate_n_flags,
 				    phy_data->energy_a, phy_data->energy_b);
 
+	/* using TLV format and must be after all fixed len fields */
+	if (format == RATE_MCS_EHT_MSK)
+		iwl_mvm_rx_eht(mvm, skb, phy_data, queue);
+
 	if (unlikely(mvm->monitor_on))
 		iwl_mvm_add_rtap_sniffer_config(mvm, skb);
 
@@ -1758,6 +2296,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 	u32 len;
 	u32 pkt_len = iwl_rx_packet_payload_len(pkt);
 	struct ieee80211_sta *sta = NULL;
+	struct ieee80211_link_sta *link_sta = NULL;
 	struct sk_buff *skb;
 	u8 crypt_len = 0;
 	size_t desc_size;
@@ -1788,6 +2327,8 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		phy_data.d1 = desc->v3.phy_data1;
 		phy_data.d2 = desc->v3.phy_data2;
 		phy_data.d3 = desc->v3.phy_data3;
+		phy_data.eht_d4 = desc->phy_eht_data4;
+		phy_data.d5 = desc->v3.phy_data5;
 	} else {
 		phy_data.rate_n_flags = le32_to_cpu(desc->v1.rate_n_flags);
 		phy_data.channel = desc->v1.channel;
@@ -1817,6 +2358,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		return;
 	}
 
+	phy_data.with_data = true;
 	phy_data.phy_info = le16_to_cpu(desc->phy_info);
 	phy_data.d4 = desc->phy_data4;
 
@@ -1897,6 +2439,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 			if (mvm->ampdu_ref == 0)
 				mvm->ampdu_ref++;
 			mvm->ampdu_toggle = toggle_bit;
+			phy_data.first_subframe = true;
 		}
 		rx_status->ampdu_reference = mvm->ampdu_ref;
 	}
@@ -1910,6 +2453,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 			sta = rcu_dereference(mvm->fw_id_to_mac_id[id]);
 			if (IS_ERR(sta))
 				sta = NULL;
+			link_sta = rcu_dereference(mvm->fw_id_to_link_sta[id]);
 		}
 	} else if (!is_multicast_ether_addr(hdr->addr2)) {
 		/*
@@ -2043,9 +2587,11 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		goto out;
 	}
 
-	if (!iwl_mvm_reorder(mvm, napi, queue, sta, skb, desc))
-		iwl_mvm_pass_packet_to_mac80211(mvm, napi, skb, queue,
-						sta);
+	if (!iwl_mvm_reorder(mvm, napi, queue, sta, skb, desc) &&
+	    likely(!iwl_mvm_time_sync_frame(mvm, skb, hdr->addr2)) &&
+	    likely(!iwl_mvm_mei_filter_scan(mvm, skb)))
+		iwl_mvm_pass_packet_to_mac80211(mvm, napi, skb, queue, sta,
+						link_sta);
 out:
 	rcu_read_unlock();
 }
@@ -2079,6 +2625,7 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 	phy_data.energy_a = u32_get_bits(rssi, RX_NO_DATA_CHAIN_A_MSK);
 	phy_data.energy_b = u32_get_bits(rssi, RX_NO_DATA_CHAIN_B_MSK);
 	phy_data.channel = u32_get_bits(rssi, RX_NO_DATA_CHANNEL_MSK);
+	phy_data.with_data = false;
 
 	if (iwl_fw_lookup_notif_ver(mvm->fw, DATA_PATH_GROUP,
 				    RX_NO_DATA_NOTIF, 0) < 2) {
@@ -2097,6 +2644,7 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 		    sizeof(struct iwl_rx_no_data_ver_3)))
 		/* invalid len for ver 3 */
 			return;
+		memcpy(phy_data.rx_vec, desc->rx_vec, sizeof(phy_data.rx_vec));
 	} else {
 		if (format == RATE_MCS_EHT_MSK)
 			/* no support for EHT before version 3 API */
@@ -2123,7 +2671,7 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 			IEEE80211_RADIOTAP_ZERO_LEN_PSDU_SOUNDING;
 		break;
 	case RX_NO_DATA_INFO_TYPE_MU_UNMATCHED:
-	case RX_NO_DATA_INFO_TYPE_HE_TB_UNMATCHED:
+	case RX_NO_DATA_INFO_TYPE_TB_UNMATCHED:
 		rx_status->zero_length_psdu_type =
 			IEEE80211_RADIOTAP_ZERO_LEN_PSDU_NOT_CAPTURED;
 		break;
@@ -2142,11 +2690,8 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 	 *
 	 * We mark it as mac header, for upper layers to know where
 	 * all radio tap header ends.
-	 *
-	 * Since data doesn't move data while putting data on skb and that is
-	 * the only way we use, data + len is the next place that hdr would be put
 	 */
-	skb_set_mac_header(skb, skb->len);
+	skb_reset_mac_header(skb);
 
 	/*
 	 * Override the nss from the rx_vec since the rate_n_flags has
@@ -2220,9 +2765,10 @@ void iwl_mvm_rx_bar_frame_release(struct iwl_mvm *mvm, struct napi_struct *napi,
 		goto out;
 	}
 
-	if (WARN(tid != baid_data->tid || sta_id != baid_data->sta_id,
-		 "baid 0x%x is mapped to sta:%d tid:%d, but BAR release received for sta:%d tid:%d\n",
-		 baid, baid_data->sta_id, baid_data->tid, sta_id,
+	if (WARN(tid != baid_data->tid || sta_id > IWL_MVM_STATION_COUNT_MAX ||
+		 !(baid_data->sta_mask & BIT(sta_id)),
+		 "baid 0x%x is mapped to sta_mask:0x%x tid:%d, but BAR release received for sta:%d tid:%d\n",
+		 baid, baid_data->sta_mask, baid_data->tid, sta_id,
 		 tid))
 		goto out;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
index acd8803dbcdd..175615755d9d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
@@ -45,6 +45,9 @@
 /* minimal number of 2GHz and 5GHz channels in the regular scan request */
 #define IWL_MVM_6GHZ_PASSIVE_SCAN_MIN_CHANS 4
 
+/* Number of iterations on the channel for mei filtered scan */
+#define IWL_MEI_SCAN_NUM_ITER	5U
+
 struct iwl_mvm_scan_timing_params {
 	u32 suspend_time;
 	u32 max_out_time;
@@ -98,6 +101,7 @@ struct iwl_mvm_scan_params {
 	bool scan_6ghz;
 	bool enable_6ghz_passive;
 	bool respect_p2p_go, respect_p2p_go_hb;
+	u8 bssid[ETH_ALEN] __aligned(2);
 };
 
 static inline void *iwl_mvm_get_scan_req_umac_data(struct iwl_mvm *mvm)
@@ -193,8 +197,9 @@ static void iwl_mvm_scan_iterator(void *_data, u8 *mac,
 	struct iwl_mvm_scan_iter_data *data = _data;
 	struct iwl_mvm_vif *curr_mvmvif;
 
-	if (vif->type != NL80211_IFTYPE_P2P_DEVICE && mvmvif->phy_ctxt &&
-	    mvmvif->phy_ctxt->id < NUM_PHY_CTX)
+	if (vif->type != NL80211_IFTYPE_P2P_DEVICE &&
+	    mvmvif->deflink.phy_ctxt &&
+	    mvmvif->deflink.phy_ctxt->id < NUM_PHY_CTX)
 		data->global_cnt += 1;
 
 	if (!data->current_vif || vif == data->current_vif)
@@ -203,8 +208,8 @@ static void iwl_mvm_scan_iterator(void *_data, u8 *mac,
 	curr_mvmvif = iwl_mvm_vif_from_mac80211(data->current_vif);
 
 	if (vif->type == NL80211_IFTYPE_AP && vif->p2p &&
-	    mvmvif->phy_ctxt && curr_mvmvif->phy_ctxt &&
-	    mvmvif->phy_ctxt->id != curr_mvmvif->phy_ctxt->id)
+	    mvmvif->deflink.phy_ctxt && curr_mvmvif->deflink.phy_ctxt &&
+	    mvmvif->deflink.phy_ctxt->id != curr_mvmvif->deflink.phy_ctxt->id)
 		data->is_dcm_with_p2p_go = true;
 }
 
@@ -239,8 +244,9 @@ iwl_mvm_scan_type _iwl_mvm_get_scan_type(struct iwl_mvm *mvm,
 		 * set all scan requests as fast-balance scan
 		 */
 		if (vif && vif->type == NL80211_IFTYPE_STATION &&
-		    vif->bss_conf.dtim_period < 220 &&
-		    data.is_dcm_with_p2p_go)
+		    data.is_dcm_with_p2p_go &&
+		    ((vif->bss_conf.beacon_int *
+		      vif->bss_conf.dtim_period) < 220))
 			return IWL_SCAN_TYPE_FAST_BALANCE;
 	}
 
@@ -758,7 +764,7 @@ iwl_mvm_build_scan_probe(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	frame->frame_control = cpu_to_le16(IEEE80211_STYPE_PROBE_REQ);
 	eth_broadcast_addr(frame->da);
-	eth_broadcast_addr(frame->bssid);
+	ether_addr_copy(frame->bssid, params->bssid);
 	frame->seq_ctrl = 0;
 
 	pos = frame->u.probe_req.variable;
@@ -2080,6 +2086,11 @@ static u8 iwl_mvm_scan_umac_flags2(struct iwl_mvm *mvm,
 				IWL_UMAC_SCAN_GEN_PARAMS_FLAGS2_RESPECT_P2P_GO_HB;
 	}
 
+	if (params->scan_6ghz &&
+	    fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_SCAN_DONT_TOGGLE_ANT))
+		flags |= IWL_UMAC_SCAN_GEN_PARAMS_FLAGS2_DONT_TOGGLE_ANT;
+
 	return flags;
 }
 
@@ -2291,7 +2302,7 @@ iwl_mvm_scan_umac_fill_general_p_v11(struct iwl_mvm *mvm,
 
 	iwl_mvm_scan_umac_dwell_v11(mvm, gp, params);
 
-	IWL_DEBUG_SCAN(mvm, "Gerenal: flags=0x%x, flags2=0x%x\n",
+	IWL_DEBUG_SCAN(mvm, "General: flags=0x%x, flags2=0x%x\n",
 		       gen_flags, gen_flags2);
 
 	gp->flags = cpu_to_le16(gen_flags);
@@ -2616,6 +2627,89 @@ static const struct iwl_scan_umac_handler iwl_scan_umac_handlers[] = {
 	IWL_SCAN_UMAC_HANDLER(12),
 };
 
+static void iwl_mvm_mei_scan_work(struct work_struct *wk)
+{
+	struct iwl_mei_scan_filter *scan_filter =
+		container_of(wk, struct iwl_mei_scan_filter, scan_work);
+	struct iwl_mvm *mvm =
+		container_of(scan_filter, struct iwl_mvm, mei_scan_filter);
+	struct iwl_mvm_csme_conn_info *info;
+	struct sk_buff *skb;
+	u8 bssid[ETH_ALEN];
+
+	mutex_lock(&mvm->mutex);
+	info = iwl_mvm_get_csme_conn_info(mvm);
+	memcpy(bssid, info->conn_info.bssid, ETH_ALEN);
+	mutex_unlock(&mvm->mutex);
+
+	while ((skb = skb_dequeue(&scan_filter->scan_res))) {
+		struct ieee80211_mgmt *mgmt = (void *)skb->data;
+
+		if (!memcmp(mgmt->bssid, bssid, ETH_ALEN))
+			ieee80211_rx_irqsafe(mvm->hw, skb);
+		else
+			kfree_skb(skb);
+	}
+}
+
+void iwl_mvm_mei_scan_filter_init(struct iwl_mei_scan_filter *mei_scan_filter)
+{
+	skb_queue_head_init(&mei_scan_filter->scan_res);
+	INIT_WORK(&mei_scan_filter->scan_work, iwl_mvm_mei_scan_work);
+}
+
+/* In case CSME is connected and has link protection set, this function will
+ * override the scan request to scan only the associated channel and only for
+ * the associated SSID.
+ */
+static void iwl_mvm_mei_limited_scan(struct iwl_mvm *mvm,
+				     struct iwl_mvm_scan_params *params)
+{
+	struct iwl_mvm_csme_conn_info *info = iwl_mvm_get_csme_conn_info(mvm);
+	struct iwl_mei_conn_info *conn_info;
+	struct ieee80211_channel *chan;
+	int scan_iters, i;
+
+	if (!info) {
+		IWL_DEBUG_SCAN(mvm, "mei_limited_scan: no connection info\n");
+		return;
+	}
+
+	conn_info = &info->conn_info;
+	if (!info->conn_info.lp_state || !info->conn_info.ssid_len)
+		return;
+
+	if (!params->n_channels || !params->n_ssids)
+		return;
+
+	mvm->mei_scan_filter.is_mei_limited_scan = true;
+
+	chan = ieee80211_get_channel(mvm->hw->wiphy,
+				     ieee80211_channel_to_frequency(conn_info->channel,
+								    conn_info->band));
+	if (!chan) {
+		IWL_DEBUG_SCAN(mvm,
+			       "Failed to get CSME channel (chan=%u band=%u)\n",
+			       conn_info->channel, conn_info->band);
+		return;
+	}
+
+	/* The mei filtered scan must find the AP, otherwise CSME will
+	 * take the NIC ownership. Add several iterations on the channel to
+	 * make the scan more robust.
+	 */
+	scan_iters = min(IWL_MEI_SCAN_NUM_ITER, params->n_channels);
+	params->n_channels = scan_iters;
+	for (i = 0; i < scan_iters; i++)
+		params->channels[i] = chan;
+
+	IWL_DEBUG_SCAN(mvm, "Mei scan: num iterations=%u\n", scan_iters);
+
+	params->n_ssids = 1;
+	params->ssids[0].ssid_len = conn_info->ssid_len;
+	memcpy(params->ssids[0].ssid, conn_info->ssid, conn_info->ssid_len);
+}
+
 static int iwl_mvm_build_scan_cmd(struct iwl_mvm *mvm,
 				  struct ieee80211_vif *vif,
 				  struct iwl_host_cmd *hcmd,
@@ -2628,6 +2722,8 @@ static int iwl_mvm_build_scan_cmd(struct iwl_mvm *mvm,
 	lockdep_assert_held(&mvm->mutex);
 	memset(mvm->scan_cmd, 0, mvm->scan_cmd_size);
 
+	iwl_mvm_mei_limited_scan(mvm, params);
+
 	if (!fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_UMAC_SCAN)) {
 		hcmd->id = SCAN_OFFLOAD_REQUEST_CMD;
 
@@ -2676,11 +2772,23 @@ static void iwl_mvm_scan_respect_p2p_go_iter(void *_data, u8 *mac,
 	if (vif == data->current_vif)
 		return;
 
-	if (vif->type == NL80211_IFTYPE_AP && vif->p2p &&
-	    mvmvif->phy_ctxt->id < NUM_PHY_CTX &&
-	    (data->band == NUM_NL80211_BANDS ||
-	     mvmvif->phy_ctxt->channel->band == data->band))
-		data->p2p_go = true;
+	if (vif->type == NL80211_IFTYPE_AP && vif->p2p) {
+		u32 link_id;
+
+		for (link_id = 0;
+		     link_id < ARRAY_SIZE(mvmvif->link);
+		     link_id++) {
+			struct iwl_mvm_vif_link_info *link =
+				mvmvif->link[link_id];
+
+			if (link && link->phy_ctxt->id < NUM_PHY_CTX &&
+			    (data->band == NUM_NL80211_BANDS ||
+			     link->phy_ctxt->channel->band == data->band)) {
+				data->p2p_go = true;
+				break;
+			}
+		}
+	}
 }
 
 static bool _iwl_mvm_get_respect_p2p_go(struct iwl_mvm *mvm,
@@ -2782,6 +2890,7 @@ int iwl_mvm_reg_scan_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	params.pass_all = true;
 	params.n_match_sets = 0;
 	params.match_sets = NULL;
+	ether_addr_copy(params.bssid, req->bssid);
 
 	params.scan_plans = &scan_plan;
 	params.n_scan_plans = 1;
@@ -2875,6 +2984,7 @@ int iwl_mvm_sched_scan_start(struct iwl_mvm *mvm,
 	params.pass_all =  iwl_mvm_scan_pass_all(mvm, req);
 	params.n_match_sets = req->n_match_sets;
 	params.match_sets = req->match_sets;
+	eth_broadcast_addr(params.bssid);
 	if (!req->n_scan_plans)
 		return -EINVAL;
 
@@ -2970,6 +3080,8 @@ void iwl_mvm_rx_umac_scan_complete_notif(struct iwl_mvm *mvm,
 	u32 uid = __le32_to_cpu(notif->uid);
 	bool aborted = (notif->status == IWL_SCAN_OFFLOAD_ABORTED);
 
+	mvm->mei_scan_filter.is_mei_limited_scan = false;
+
 	if (WARN_ON(!(mvm->scan_uid_status[uid] & mvm->scan_status)))
 		return;
 
@@ -2980,7 +3092,7 @@ void iwl_mvm_rx_umac_scan_complete_notif(struct iwl_mvm *mvm,
 			.scan_start_tsf = mvm->scan_start,
 		};
 
-		memcpy(info.tsf_bssid, mvm->scan_vif->bssid, ETH_ALEN);
+		memcpy(info.tsf_bssid, mvm->scan_vif->deflink.bssid, ETH_ALEN);
 		ieee80211_scan_completed(mvm->hw, &info);
 		mvm->scan_vif = NULL;
 		cancel_delayed_work(&mvm->scan_timeout_dwork);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sf.c b/drivers/net/wireless/intel/iwlwifi/mvm/sf.c
index 1f4ac1e93cee..98f330fcf678 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sf.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sf.c
@@ -8,7 +8,7 @@
 /* For counting bound interfaces */
 struct iwl_mvm_active_iface_iterator_data {
 	struct ieee80211_vif *ignore_vif;
-	u8 sta_vif_ap_sta_id;
+	struct ieee80211_sta *sta_vif_ap_sta;
 	enum iwl_sf_state sta_vif_state;
 	u32 num_active_macs;
 };
@@ -23,14 +23,14 @@ static void iwl_mvm_bound_iface_iterator(void *_data, u8 *mac,
 	struct iwl_mvm_active_iface_iterator_data *data = _data;
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-	if (vif == data->ignore_vif || !mvmvif->phy_ctxt ||
+	if (vif == data->ignore_vif || !mvmvif->deflink.phy_ctxt ||
 	    vif->type == NL80211_IFTYPE_P2P_DEVICE)
 		return;
 
 	data->num_active_macs++;
 
 	if (vif->type == NL80211_IFTYPE_STATION) {
-		data->sta_vif_ap_sta_id = mvmvif->ap_sta_id;
+		data->sta_vif_ap_sta = mvmvif->ap_sta;
 		if (vif->cfg.assoc)
 			data->sta_vif_state = SF_FULL_ON;
 		else
@@ -98,6 +98,10 @@ static void iwl_mvm_fill_sf_command(struct iwl_mvm *mvm,
 				    struct ieee80211_sta *sta)
 {
 	int i, j, watermark;
+	u8 max_rx_nss = 0;
+	bool is_legacy = true;
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
 
 	sf_cmd->watermark[SF_LONG_DELAY_ON] = cpu_to_le32(SF_W_MARK_SCAN);
 
@@ -106,10 +110,25 @@ static void iwl_mvm_fill_sf_command(struct iwl_mvm *mvm,
 	 * capabilities of the AP station, and choose the watermark accordingly.
 	 */
 	if (sta) {
-		if (sta->deflink.ht_cap.ht_supported ||
-		    sta->deflink.vht_cap.vht_supported ||
-		    sta->deflink.he_cap.has_he) {
-			switch (sta->deflink.rx_nss) {
+		/* find the maximal NSS number among all links (if relevant) */
+		rcu_read_lock();
+		for (link_id = 0; link_id < ARRAY_SIZE(sta->link); link_id++) {
+			link_sta = rcu_dereference(sta->link[link_id]);
+			if (!link_sta)
+				continue;
+
+			if (link_sta->ht_cap.ht_supported ||
+			    link_sta->vht_cap.vht_supported ||
+			    link_sta->eht_cap.has_eht ||
+			    link_sta->he_cap.has_he) {
+				is_legacy = false;
+				max_rx_nss = max(max_rx_nss, link_sta->rx_nss);
+			}
+		}
+		rcu_read_unlock();
+
+		if (!is_legacy) {
+			switch (max_rx_nss) {
 			case 1:
 				watermark = SF_W_MARK_SISO;
 				break;
@@ -151,16 +170,14 @@ static void iwl_mvm_fill_sf_command(struct iwl_mvm *mvm,
 		memcpy(sf_cmd->full_on_timeouts, sf_full_timeout_def,
 		       sizeof(sf_full_timeout_def));
 	}
-
 }
 
-static int iwl_mvm_sf_config(struct iwl_mvm *mvm, u8 sta_id,
+static int iwl_mvm_sf_config(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 			     enum iwl_sf_state new_state)
 {
 	struct iwl_sf_cfg_cmd sf_cmd = {
 		.state = cpu_to_le32(new_state),
 	};
-	struct ieee80211_sta *sta;
 	int ret = 0;
 
 	if (mvm->cfg->disable_dummy_notification)
@@ -178,20 +195,12 @@ static int iwl_mvm_sf_config(struct iwl_mvm *mvm, u8 sta_id,
 		iwl_mvm_fill_sf_command(mvm, &sf_cmd, NULL);
 		break;
 	case SF_FULL_ON:
-		if (sta_id == IWL_MVM_INVALID_STA) {
+		if (!sta) {
 			IWL_ERR(mvm,
 				"No station: Cannot switch SF to FULL_ON\n");
 			return -EINVAL;
 		}
-		rcu_read_lock();
-		sta = rcu_dereference(mvm->fw_id_to_mac_id[sta_id]);
-		if (IS_ERR_OR_NULL(sta)) {
-			IWL_ERR(mvm, "Invalid station id\n");
-			rcu_read_unlock();
-			return -EINVAL;
-		}
 		iwl_mvm_fill_sf_command(mvm, &sf_cmd, sta);
-		rcu_read_unlock();
 		break;
 	case SF_INIT_OFF:
 		iwl_mvm_fill_sf_command(mvm, &sf_cmd, NULL);
@@ -219,13 +228,12 @@ int iwl_mvm_sf_update(struct iwl_mvm *mvm, struct ieee80211_vif *changed_vif,
 		      bool remove_vif)
 {
 	enum iwl_sf_state new_state;
-	u8 sta_id = IWL_MVM_INVALID_STA;
 	struct iwl_mvm_vif *mvmvif = NULL;
 	struct iwl_mvm_active_iface_iterator_data data = {
 		.ignore_vif = changed_vif,
 		.sta_vif_state = SF_UNINIT,
-		.sta_vif_ap_sta_id = IWL_MVM_INVALID_STA,
 	};
+	struct ieee80211_sta *sta = NULL;
 
 	/*
 	 * Ignore the call if we are in HW Restart flow, or if the handled
@@ -255,7 +263,7 @@ int iwl_mvm_sf_update(struct iwl_mvm *mvm, struct ieee80211_vif *changed_vif,
 			 * and we filled the relevant data during iteration
 			 */
 			new_state = data.sta_vif_state;
-			sta_id = data.sta_vif_ap_sta_id;
+			sta = data.sta_vif_ap_sta;
 		} else {
 			if (WARN_ON(!changed_vif))
 				return -EINVAL;
@@ -264,7 +272,7 @@ int iwl_mvm_sf_update(struct iwl_mvm *mvm, struct ieee80211_vif *changed_vif,
 			} else if (changed_vif->cfg.assoc &&
 				   changed_vif->bss_conf.dtim_period) {
 				mvmvif = iwl_mvm_vif_from_mac80211(changed_vif);
-				sta_id = mvmvif->ap_sta_id;
+				sta = mvmvif->ap_sta;
 				new_state = SF_FULL_ON;
 			} else {
 				new_state = SF_INIT_OFF;
@@ -275,5 +283,6 @@ int iwl_mvm_sf_update(struct iwl_mvm *mvm, struct ieee80211_vif *changed_vif,
 		/* If there are multiple active macs - change to SF_UNINIT */
 		new_state = SF_UNINIT;
 	}
-	return iwl_mvm_sf_config(mvm, sta_id, new_state);
+
+	return iwl_mvm_sf_config(mvm, sta, new_state);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
index 9caae77995ca..5469d634e289 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
@@ -24,8 +24,7 @@ static inline int iwl_mvm_add_sta_cmd_size(struct iwl_mvm *mvm)
 		return sizeof(struct iwl_mvm_add_sta_cmd_v7);
 }
 
-static int iwl_mvm_find_free_sta_id(struct iwl_mvm *mvm,
-				    enum nl80211_iftype iftype)
+int iwl_mvm_find_free_sta_id(struct iwl_mvm *mvm, enum nl80211_iftype iftype)
 {
 	int sta_id;
 	u32 reserved_ids = 0;
@@ -51,13 +50,79 @@ static int iwl_mvm_find_free_sta_id(struct iwl_mvm *mvm,
 	return IWL_MVM_INVALID_STA;
 }
 
+/* Calculate the ampdu density and max size */
+u32 iwl_mvm_get_sta_ampdu_dens(struct ieee80211_link_sta *link_sta,
+			       struct ieee80211_bss_conf *link_conf,
+			       u32 *_agg_size)
+{
+	u32 agg_size = 0, mpdu_dens = 0;
+
+	if (WARN_ON(!link_sta))
+		return 0;
+
+	if (link_sta->ht_cap.ht_supported)
+		mpdu_dens = link_sta->ht_cap.ampdu_density;
+
+	if (link_conf->chandef.chan->band ==
+	    NL80211_BAND_6GHZ) {
+		mpdu_dens = le16_get_bits(link_sta->he_6ghz_capa.capa,
+					  IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START);
+		agg_size = le16_get_bits(link_sta->he_6ghz_capa.capa,
+					 IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP);
+	} else if (link_sta->vht_cap.vht_supported) {
+		agg_size = link_sta->vht_cap.cap &
+			IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK;
+		agg_size >>=
+			IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT;
+	} else if (link_sta->ht_cap.ht_supported) {
+		agg_size = link_sta->ht_cap.ampdu_factor;
+	}
+
+	/* D6.0 10.12.2 A-MPDU length limit rules
+	 * A STA indicates the maximum length of the A-MPDU preEOF padding
+	 * that it can receive in an HE PPDU in the Maximum A-MPDU Length
+	 * Exponent field in its HT Capabilities, VHT Capabilities,
+	 * and HE 6 GHz Band Capabilities elements (if present) and the
+	 * Maximum AMPDU Length Exponent Extension field in its HE
+	 * Capabilities element
+	 */
+	if (link_sta->he_cap.has_he)
+		agg_size +=
+			u8_get_bits(link_sta->he_cap.he_cap_elem.mac_cap_info[3],
+				    IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_MASK);
+
+	/* Limit to max A-MPDU supported by FW */
+	if (agg_size > (STA_FLG_MAX_AGG_SIZE_4M >> STA_FLG_MAX_AGG_SIZE_SHIFT))
+		agg_size = (STA_FLG_MAX_AGG_SIZE_4M >>
+			    STA_FLG_MAX_AGG_SIZE_SHIFT);
+
+	*_agg_size = agg_size;
+	return mpdu_dens;
+}
+
+u8 iwl_mvm_get_sta_uapsd_acs(struct ieee80211_sta *sta)
+{
+	u8 uapsd_acs = 0;
+
+	if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BK)
+		uapsd_acs |= BIT(AC_BK);
+	if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BE)
+		uapsd_acs |= BIT(AC_BE);
+	if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VI)
+		uapsd_acs |= BIT(AC_VI);
+	if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VO)
+		uapsd_acs |= BIT(AC_VO);
+
+	return uapsd_acs | uapsd_acs << 4;
+}
+
 /* send station add/update command to firmware */
 int iwl_mvm_sta_send_to_fw(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 			   bool update, unsigned int flags)
 {
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_mvm_add_sta_cmd add_sta_cmd = {
-		.sta_id = mvm_sta->sta_id,
+		.sta_id = mvm_sta->deflink.sta_id,
 		.mac_id_n_color = cpu_to_le32(mvm_sta->mac_id_n_color),
 		.add_modify = update ? 1 : 0,
 		.station_flags_msk = cpu_to_le32(STA_FLG_FAT_EN_MSK |
@@ -134,68 +199,26 @@ int iwl_mvm_sta_send_to_fw(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		break;
 	}
 
-	if (sta->deflink.ht_cap.ht_supported) {
-		add_sta_cmd.station_flags_msk |=
-			cpu_to_le32(STA_FLG_MAX_AGG_SIZE_MSK |
-				    STA_FLG_AGG_MPDU_DENS_MSK);
-
-		mpdu_dens = sta->deflink.ht_cap.ampdu_density;
-	}
-
-	if (mvm_sta->vif->bss_conf.chandef.chan->band == NL80211_BAND_6GHZ) {
+	if (sta->deflink.ht_cap.ht_supported ||
+	    mvm_sta->vif->bss_conf.chandef.chan->band == NL80211_BAND_6GHZ)
 		add_sta_cmd.station_flags_msk |=
 			cpu_to_le32(STA_FLG_MAX_AGG_SIZE_MSK |
 				    STA_FLG_AGG_MPDU_DENS_MSK);
 
-		mpdu_dens = le16_get_bits(sta->deflink.he_6ghz_capa.capa,
-					  IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START);
-		agg_size = le16_get_bits(sta->deflink.he_6ghz_capa.capa,
-					 IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP);
-	} else if (sta->deflink.vht_cap.vht_supported) {
-		agg_size = sta->deflink.vht_cap.cap &
-			IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK;
-		agg_size >>=
-			IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT;
-	} else if (sta->deflink.ht_cap.ht_supported) {
-		agg_size = sta->deflink.ht_cap.ampdu_factor;
-	}
-
-	/* D6.0 10.12.2 A-MPDU length limit rules
-	 * A STA indicates the maximum length of the A-MPDU preEOF padding
-	 * that it can receive in an HE PPDU in the Maximum A-MPDU Length
-	 * Exponent field in its HT Capabilities, VHT Capabilities,
-	 * and HE 6 GHz Band Capabilities elements (if present) and the
-	 * Maximum AMPDU Length Exponent Extension field in its HE
-	 * Capabilities element
-	 */
-	if (sta->deflink.he_cap.has_he)
-		agg_size += u8_get_bits(sta->deflink.he_cap.he_cap_elem.mac_cap_info[3],
-					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_MASK);
-
-	/* Limit to max A-MPDU supported by FW */
-	if (agg_size > (STA_FLG_MAX_AGG_SIZE_4M >> STA_FLG_MAX_AGG_SIZE_SHIFT))
-		agg_size = (STA_FLG_MAX_AGG_SIZE_4M >>
-			    STA_FLG_MAX_AGG_SIZE_SHIFT);
-
+	mpdu_dens = iwl_mvm_get_sta_ampdu_dens(&sta->deflink,
+					       &mvm_sta->vif->bss_conf,
+					       &agg_size);
 	add_sta_cmd.station_flags |=
 		cpu_to_le32(agg_size << STA_FLG_MAX_AGG_SIZE_SHIFT);
 	add_sta_cmd.station_flags |=
 		cpu_to_le32(mpdu_dens << STA_FLG_AGG_MPDU_DENS_SHIFT);
+
 	if (mvm_sta->sta_state >= IEEE80211_STA_ASSOC)
 		add_sta_cmd.assoc_id = cpu_to_le16(sta->aid);
 
 	if (sta->wme) {
 		add_sta_cmd.modify_mask |= STA_MODIFY_UAPSD_ACS;
-
-		if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BK)
-			add_sta_cmd.uapsd_acs |= BIT(AC_BK);
-		if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BE)
-			add_sta_cmd.uapsd_acs |= BIT(AC_BE);
-		if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VI)
-			add_sta_cmd.uapsd_acs |= BIT(AC_VI);
-		if (sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VO)
-			add_sta_cmd.uapsd_acs |= BIT(AC_VO);
-		add_sta_cmd.uapsd_acs |= add_sta_cmd.uapsd_acs << 4;
+		add_sta_cmd.uapsd_acs = iwl_mvm_get_sta_uapsd_acs(sta);
 		add_sta_cmd.sp_length = sta->max_sp ? sta->max_sp * 2 : 128;
 	}
 
@@ -228,6 +251,7 @@ static void iwl_mvm_rx_agg_session_expired(struct timer_list *t)
 	struct ieee80211_sta *sta;
 	struct iwl_mvm_sta *mvm_sta;
 	unsigned long timeout;
+	unsigned int sta_id;
 
 	rcu_read_lock();
 
@@ -246,7 +270,8 @@ static void iwl_mvm_rx_agg_session_expired(struct timer_list *t)
 	}
 
 	/* Timer expired */
-	sta = rcu_dereference(ba_data->mvm->fw_id_to_mac_id[ba_data->sta_id]);
+	sta_id = ffs(ba_data->sta_mask) - 1; /* don't care which one */
+	sta = rcu_dereference(ba_data->mvm->fw_id_to_mac_id[sta_id]);
 
 	/*
 	 * sta should be valid unless the following happens:
@@ -296,7 +321,7 @@ static int iwl_mvm_invalidate_sta_queue(struct iwl_mvm *mvm, int queue,
 	mvmsta->tid_disable_agg |= disable_agg_tids;
 
 	cmd.mac_id_n_color = cpu_to_le32(mvmsta->mac_id_n_color);
-	cmd.sta_id = mvmsta->sta_id;
+	cmd.sta_id = mvmsta->deflink.sta_id;
 	cmd.add_modify = STA_MODE_MODIFY;
 	cmd.modify_mask = STA_MODIFY_QUEUES;
 	if (disable_agg_tids)
@@ -333,10 +358,14 @@ static int iwl_mvm_disable_txq(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 					     SCD_QUEUE_CONFIG_CMD);
 			struct iwl_scd_queue_cfg_cmd remove_cmd = {
 				.operation = cpu_to_le32(IWL_SCD_QUEUE_REMOVE),
-				.u.remove.tid = cpu_to_le32(tid),
 				.u.remove.sta_mask = cpu_to_le32(BIT(sta_id)),
 			};
 
+			if (tid == IWL_MAX_TID_COUNT)
+				tid = IWL_MGMT_TID;
+
+			remove_cmd.u.remove.tid = cpu_to_le32(tid);
+
 			ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0,
 						   sizeof(remove_cmd),
 						   &remove_cmd);
@@ -772,32 +801,52 @@ static int iwl_mvm_find_free_queue(struct iwl_mvm *mvm, u8 sta_id,
 	return -ENOSPC;
 }
 
-static int iwl_mvm_tvqm_enable_txq(struct iwl_mvm *mvm,
-				   u8 sta_id, u8 tid, unsigned int timeout)
+static int iwl_mvm_get_queue_size(struct ieee80211_sta *sta)
+{
+	int max_size = IWL_DEFAULT_QUEUE_SIZE;
+	unsigned int link_id;
+
+	/* this queue isn't used for traffic (cab_queue) */
+	if (!sta)
+		return IWL_MGMT_QUEUE_SIZE;
+
+	rcu_read_lock();
+
+	for (link_id = 0; link_id < ARRAY_SIZE(sta->link); link_id++) {
+		struct ieee80211_link_sta *link =
+			rcu_dereference(sta->link[link_id]);
+
+		if (!link)
+			continue;
+
+		/* support for 1k ba size */
+		if (link->eht_cap.has_eht &&
+		    max_size < IWL_DEFAULT_QUEUE_SIZE_EHT)
+			max_size = IWL_DEFAULT_QUEUE_SIZE_EHT;
+
+		/* support for 256 ba size */
+		if (link->he_cap.has_he &&
+		    max_size < IWL_DEFAULT_QUEUE_SIZE_HE)
+			max_size = IWL_DEFAULT_QUEUE_SIZE_HE;
+	}
+
+	rcu_read_unlock();
+	return max_size;
+}
+
+int iwl_mvm_tvqm_enable_txq(struct iwl_mvm *mvm,
+			    struct ieee80211_sta *sta,
+			    u8 sta_id, u8 tid, unsigned int timeout)
 {
 	int queue, size;
+	u32 sta_mask = 0;
 
 	if (tid == IWL_MAX_TID_COUNT) {
 		tid = IWL_MGMT_TID;
 		size = max_t(u32, IWL_MGMT_QUEUE_SIZE,
 			     mvm->trans->cfg->min_txq_size);
 	} else {
-		struct ieee80211_sta *sta;
-
-		rcu_read_lock();
-		sta = rcu_dereference(mvm->fw_id_to_mac_id[sta_id]);
-
-		/* this queue isn't used for traffic (cab_queue) */
-		if (IS_ERR_OR_NULL(sta)) {
-			size = IWL_MGMT_QUEUE_SIZE;
-		} else if (sta->deflink.he_cap.has_he) {
-			/* support for 256 ba size */
-			size = IWL_DEFAULT_QUEUE_SIZE_HE;
-		} else {
-			size = IWL_DEFAULT_QUEUE_SIZE;
-		}
-
-		rcu_read_unlock();
+		size = iwl_mvm_get_queue_size(sta);
 	}
 
 	/* take the min with bc tbl entries allowed */
@@ -806,22 +855,45 @@ static int iwl_mvm_tvqm_enable_txq(struct iwl_mvm *mvm,
 	/* size needs to be power of 2 values for calculating read/write pointers */
 	size = rounddown_pow_of_two(size);
 
+	if (sta) {
+		struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
+		unsigned int link_id;
+
+		for (link_id = 0;
+		     link_id < ARRAY_SIZE(mvmsta->link);
+		     link_id++) {
+			struct iwl_mvm_link_sta *link =
+				rcu_dereference_protected(mvmsta->link[link_id],
+							  lockdep_is_held(&mvm->mutex));
+
+			if (!link)
+				continue;
+
+			sta_mask |= BIT(link->sta_id);
+		}
+	} else {
+		sta_mask |= BIT(sta_id);
+	}
+
+	if (!sta_mask)
+		return -EINVAL;
+
 	do {
-		queue = iwl_trans_txq_alloc(mvm->trans, 0, BIT(sta_id),
+		queue = iwl_trans_txq_alloc(mvm->trans, 0, sta_mask,
 					    tid, size, timeout);
 
 		if (queue < 0)
 			IWL_DEBUG_TX_QUEUES(mvm,
-					    "Failed allocating TXQ of size %d for sta %d tid %d, ret: %d\n",
-					    size, sta_id, tid, queue);
+					    "Failed allocating TXQ of size %d for sta mask %x tid %d, ret: %d\n",
+					    size, sta_mask, tid, queue);
 		size /= 2;
 	} while (queue < 0 && size >= 16);
 
 	if (queue < 0)
 		return queue;
 
-	IWL_DEBUG_TX_QUEUES(mvm, "Enabling TXQ #%d for sta %d tid %d\n",
-			    queue, sta_id, tid);
+	IWL_DEBUG_TX_QUEUES(mvm, "Enabling TXQ #%d for sta mask 0x%x tid %d\n",
+			    queue, sta_mask, tid);
 
 	return queue;
 }
@@ -841,14 +913,15 @@ static int iwl_mvm_sta_alloc_queue_tvqm(struct iwl_mvm *mvm,
 
 	IWL_DEBUG_TX_QUEUES(mvm,
 			    "Allocating queue for sta %d on tid %d\n",
-			    mvmsta->sta_id, tid);
-	queue = iwl_mvm_tvqm_enable_txq(mvm, mvmsta->sta_id, tid, wdg_timeout);
+			    mvmsta->deflink.sta_id, tid);
+	queue = iwl_mvm_tvqm_enable_txq(mvm, sta, mvmsta->deflink.sta_id,
+					tid, wdg_timeout);
 	if (queue < 0)
 		return queue;
 
 	mvmtxq->txq_id = queue;
 	mvm->tvqm_info[queue].txq_tid = tid;
-	mvm->tvqm_info[queue].sta_id = mvmsta->sta_id;
+	mvm->tvqm_info[queue].sta_id = mvmsta->deflink.sta_id;
 
 	IWL_DEBUG_TX_QUEUES(mvm, "Allocated queue is %d\n", queue);
 
@@ -1033,7 +1106,7 @@ static void iwl_mvm_unshare_queue(struct iwl_mvm *mvm, int queue)
 		mvmsta->tid_disable_agg &= ~BIT(tid);
 
 		cmd.mac_id_n_color = cpu_to_le32(mvmsta->mac_id_n_color);
-		cmd.sta_id = mvmsta->sta_id;
+		cmd.sta_id = mvmsta->deflink.sta_id;
 		cmd.add_modify = STA_MODE_MODIFY;
 		cmd.modify_mask = STA_MODIFY_TID_DISABLE_TX;
 		cmd.tfd_queue_msk = cpu_to_le32(mvmsta->tfd_queue_msk);
@@ -1258,7 +1331,7 @@ static int iwl_mvm_sta_alloc_queue(struct iwl_mvm *mvm,
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_trans_txq_scd_cfg cfg = {
 		.fifo = iwl_mvm_mac_ac_to_tx_fifo(mvm, ac),
-		.sta_id = mvmsta->sta_id,
+		.sta_id = mvmsta->deflink.sta_id,
 		.tid = tid,
 		.frame_limit = IWL_FRAME_LIMIT,
 	};
@@ -1284,7 +1357,7 @@ static int iwl_mvm_sta_alloc_queue(struct iwl_mvm *mvm,
 	spin_unlock_bh(&mvmsta->lock);
 
 	if (tid == IWL_MAX_TID_COUNT) {
-		queue = iwl_mvm_find_free_queue(mvm, mvmsta->sta_id,
+		queue = iwl_mvm_find_free_queue(mvm, mvmsta->deflink.sta_id,
 						IWL_MVM_DQA_MIN_MGMT_QUEUE,
 						IWL_MVM_DQA_MAX_MGMT_QUEUE);
 		if (queue >= IWL_MVM_DQA_MIN_MGMT_QUEUE)
@@ -1303,12 +1376,12 @@ static int iwl_mvm_sta_alloc_queue(struct iwl_mvm *mvm,
 	}
 
 	if (queue < 0)
-		queue = iwl_mvm_find_free_queue(mvm, mvmsta->sta_id,
+		queue = iwl_mvm_find_free_queue(mvm, mvmsta->deflink.sta_id,
 						IWL_MVM_DQA_MIN_DATA_QUEUE,
 						IWL_MVM_DQA_MAX_DATA_QUEUE);
 	if (queue < 0) {
 		/* try harder - perhaps kill an inactive queue */
-		queue = iwl_mvm_inactivity_check(mvm, mvmsta->sta_id);
+		queue = iwl_mvm_inactivity_check(mvm, mvmsta->deflink.sta_id);
 	}
 
 	/* No free queue - we'll have to share */
@@ -1348,7 +1421,7 @@ static int iwl_mvm_sta_alloc_queue(struct iwl_mvm *mvm,
 	IWL_DEBUG_TX_QUEUES(mvm,
 			    "Allocating %squeue #%d to sta %d on tid %d\n",
 			    shared_queue ? "shared " : "", queue,
-			    mvmsta->sta_id, tid);
+			    mvmsta->deflink.sta_id, tid);
 
 	if (shared_queue) {
 		/* Disable any open aggs on this queue */
@@ -1415,7 +1488,7 @@ static int iwl_mvm_sta_alloc_queue(struct iwl_mvm *mvm,
 
 out_err:
 	queue_tmp = queue;
-	iwl_mvm_disable_txq(mvm, sta, mvmsta->sta_id, &queue_tmp, tid);
+	iwl_mvm_disable_txq(mvm, sta, mvmsta->deflink.sta_id, &queue_tmp, tid);
 
 	return ret;
 }
@@ -1494,12 +1567,12 @@ static int iwl_mvm_reserve_sta_stream(struct iwl_mvm *mvm,
 	     IWL_MVM_QUEUE_FREE))
 		queue = IWL_MVM_DQA_BSS_CLIENT_QUEUE;
 	else
-		queue = iwl_mvm_find_free_queue(mvm, mvmsta->sta_id,
+		queue = iwl_mvm_find_free_queue(mvm, mvmsta->deflink.sta_id,
 						IWL_MVM_DQA_MIN_DATA_QUEUE,
 						IWL_MVM_DQA_MAX_DATA_QUEUE);
 	if (queue < 0) {
 		/* try again - this time kick out a queue if needed */
-		queue = iwl_mvm_inactivity_check(mvm, mvmsta->sta_id);
+		queue = iwl_mvm_inactivity_check(mvm, mvmsta->deflink.sta_id);
 		if (queue < 0) {
 			IWL_ERR(mvm, "No available queues for new station\n");
 			return -ENOSPC;
@@ -1510,7 +1583,7 @@ static int iwl_mvm_reserve_sta_stream(struct iwl_mvm *mvm,
 	mvmsta->reserved_queue = queue;
 
 	IWL_DEBUG_TX_QUEUES(mvm, "Reserving data queue #%d for sta_id %d\n",
-			    queue, mvmsta->sta_id);
+			    queue, mvmsta->deflink.sta_id);
 
 	return 0;
 }
@@ -1522,15 +1595,15 @@ static int iwl_mvm_reserve_sta_stream(struct iwl_mvm *mvm,
  *
  * Note that re-enabling aggregations isn't done in this function.
  */
-static void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm,
-						 struct ieee80211_sta *sta)
+void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm,
+					  struct ieee80211_sta *sta)
 {
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 	unsigned int wdg =
 		iwl_mvm_get_wd_timeout(mvm, mvm_sta->vif, false, false);
 	int i;
 	struct iwl_trans_txq_scd_cfg cfg = {
-		.sta_id = mvm_sta->sta_id,
+		.sta_id = mvm_sta->deflink.sta_id,
 		.frame_limit = IWL_FRAME_LIMIT,
 	};
 
@@ -1552,8 +1625,9 @@ static void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm,
 		if (iwl_mvm_has_new_tx_api(mvm)) {
 			IWL_DEBUG_TX_QUEUES(mvm,
 					    "Re-mapping sta %d tid %d\n",
-					    mvm_sta->sta_id, i);
-			txq_id = iwl_mvm_tvqm_enable_txq(mvm, mvm_sta->sta_id,
+					    mvm_sta->deflink.sta_id, i);
+			txq_id = iwl_mvm_tvqm_enable_txq(mvm, sta,
+							 mvm_sta->deflink.sta_id,
 							 i, wdg);
 			/*
 			 * on failures, just set it to IWL_MVM_INVALID_QUEUE
@@ -1582,7 +1656,8 @@ static void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm,
 
 			IWL_DEBUG_TX_QUEUES(mvm,
 					    "Re-mapping sta %d tid %d to queue %d\n",
-					    mvm_sta->sta_id, i, txq_id);
+					    mvm_sta->deflink.sta_id, i,
+					    txq_id);
 
 			iwl_mvm_enable_txq(mvm, sta, txq_id, seq, &cfg, wdg);
 			mvm->queue_info[txq_id].status = IWL_MVM_QUEUE_READY;
@@ -1640,74 +1715,45 @@ static int iwl_mvm_add_int_sta_common(struct iwl_mvm *mvm,
 	return ret;
 }
 
-int iwl_mvm_add_sta(struct iwl_mvm *mvm,
-		    struct ieee80211_vif *vif,
-		    struct ieee80211_sta *sta)
+/* Initialize driver data of a new sta */
+int iwl_mvm_sta_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+		     struct ieee80211_sta *sta, int sta_id, u8 sta_type)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_mvm_rxq_dup_data *dup_data;
-	int i, ret, sta_id;
-	bool sta_update = false;
-	unsigned int sta_flags = 0;
+	int i, ret = 0;
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
-		sta_id = iwl_mvm_find_free_sta_id(mvm,
-						  ieee80211_vif_type_p2p(vif));
-	else
-		sta_id = mvm_sta->sta_id;
-
-	if (sta_id == IWL_MVM_INVALID_STA)
-		return -ENOSPC;
-
-	spin_lock_init(&mvm_sta->lock);
+	mvm_sta->mac_id_n_color = FW_CMD_ID_AND_COLOR(mvmvif->id,
+						      mvmvif->color);
+	mvm_sta->vif = vif;
 
-	/* if this is a HW restart re-alloc existing queues */
-	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
-		struct iwl_mvm_int_sta tmp_sta = {
-			.sta_id = sta_id,
-			.type = mvm_sta->sta_type,
-		};
+	/* for MLD sta_id(s) should be allocated for each link before calling
+	 * this function
+	 */
+	if (!mvm->mld_api_is_used) {
+		if (WARN_ON(sta_id == IWL_MVM_INVALID_STA))
+			return -EINVAL;
 
-		/*
-		 * First add an empty station since allocating
-		 * a queue requires a valid station
-		 */
-		ret = iwl_mvm_add_int_sta_common(mvm, &tmp_sta, sta->addr,
-						 mvmvif->id, mvmvif->color);
-		if (ret)
-			goto err;
+		mvm_sta->deflink.sta_id = sta_id;
+		rcu_assign_pointer(mvm_sta->link[0], &mvm_sta->deflink);
 
-		iwl_mvm_realloc_queues_after_restart(mvm, sta);
-		sta_update = true;
-		sta_flags = iwl_mvm_has_new_tx_api(mvm) ? 0 : STA_MODIFY_QUEUES;
-		goto update_fw;
+		if (!mvm->trans->trans_cfg->gen2)
+			mvm_sta->deflink.lq_sta.rs_drv.pers.max_agg_bufsize =
+				LINK_QUAL_AGG_FRAME_LIMIT_DEF;
+		else
+			mvm_sta->deflink.lq_sta.rs_drv.pers.max_agg_bufsize =
+				LINK_QUAL_AGG_FRAME_LIMIT_GEN2_DEF;
 	}
 
-	mvm_sta->sta_id = sta_id;
-	mvm_sta->mac_id_n_color = FW_CMD_ID_AND_COLOR(mvmvif->id,
-						      mvmvif->color);
-	mvm_sta->vif = vif;
-	if (!mvm->trans->trans_cfg->gen2)
-		mvm_sta->max_agg_bufsize = LINK_QUAL_AGG_FRAME_LIMIT_DEF;
-	else
-		mvm_sta->max_agg_bufsize = LINK_QUAL_AGG_FRAME_LIMIT_GEN2_DEF;
-	mvm_sta->tx_protection = 0;
 	mvm_sta->tt_tx_protection = false;
-	mvm_sta->sta_type = sta->tdls ? IWL_STA_TDLS_LINK : IWL_STA_LINK;
+	mvm_sta->sta_type = sta_type;
 
-	/* HW restart, don't assume the memory has been zeroed */
 	mvm_sta->tid_disable_agg = 0xffff; /* No aggs at first */
-	mvm_sta->tfd_queue_msk = 0;
 
-	/* for HW restart - reset everything but the sequence number */
 	for (i = 0; i <= IWL_MAX_TID_COUNT; i++) {
-		u16 seq = mvm_sta->tid_data[i].seq_number;
-		memset(&mvm_sta->tid_data[i], 0, sizeof(mvm_sta->tid_data[i]));
-		mvm_sta->tid_data[i].seq_number = seq;
-
 		/*
 		 * Mark all queues for this STA as unallocated and defer TX
 		 * frames until the queue is allocated
@@ -1724,10 +1770,7 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 		atomic_set(&mvmtxq->tx_request, 0);
 	}
 
-	mvm_sta->agg_tids = 0;
-
-	if (iwl_mvm_has_new_rx_api(mvm) &&
-	    !test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
+	if (iwl_mvm_has_new_rx_api(mvm)) {
 		int q;
 
 		dup_data = kcalloc(mvm->trans->num_rx_queues,
@@ -1753,7 +1796,7 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 		ret = iwl_mvm_reserve_sta_stream(mvm, sta,
 						 ieee80211_vif_type_p2p(vif));
 		if (ret)
-			goto err;
+			return ret;
 	}
 
 	/*
@@ -1763,10 +1806,60 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 	if (iwl_mvm_has_tlc_offload(mvm))
 		iwl_mvm_rs_add_sta(mvm, mvm_sta);
 	else
-		spin_lock_init(&mvm_sta->lq_sta.rs_drv.pers.lock);
+		spin_lock_init(&mvm_sta->deflink.lq_sta.rs_drv.pers.lock);
 
 	iwl_mvm_toggle_tx_ant(mvm, &mvm_sta->tx_ant);
 
+	return 0;
+}
+
+int iwl_mvm_add_sta(struct iwl_mvm *mvm,
+		    struct ieee80211_vif *vif,
+		    struct ieee80211_sta *sta)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	int ret, sta_id;
+	bool sta_update = false;
+	unsigned int sta_flags = 0;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
+		sta_id = iwl_mvm_find_free_sta_id(mvm,
+						  ieee80211_vif_type_p2p(vif));
+	else
+		sta_id = mvm_sta->deflink.sta_id;
+
+	if (sta_id == IWL_MVM_INVALID_STA)
+		return -ENOSPC;
+
+	spin_lock_init(&mvm_sta->lock);
+
+	/* if this is a HW restart re-alloc existing queues */
+	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
+		struct iwl_mvm_int_sta tmp_sta = {
+			.sta_id = sta_id,
+			.type = mvm_sta->sta_type,
+		};
+
+		/* First add an empty station since allocating
+		 * a queue requires a valid station
+		 */
+		ret = iwl_mvm_add_int_sta_common(mvm, &tmp_sta, sta->addr,
+						 mvmvif->id, mvmvif->color);
+		if (ret)
+			goto err;
+
+		iwl_mvm_realloc_queues_after_restart(mvm, sta);
+		sta_update = true;
+		sta_flags = iwl_mvm_has_new_tx_api(mvm) ? 0 : STA_MODIFY_QUEUES;
+		goto update_fw;
+	}
+
+	ret = iwl_mvm_sta_init(mvm, vif, sta, sta_id,
+			       sta->tdls ? IWL_STA_TDLS_LINK : IWL_STA_LINK);
+
 update_fw:
 	ret = iwl_mvm_sta_send_to_fw(mvm, sta, sta_update, sta_flags);
 	if (ret)
@@ -1774,10 +1867,10 @@ update_fw:
 
 	if (vif->type == NL80211_IFTYPE_STATION) {
 		if (!sta->tdls) {
-			WARN_ON(mvmvif->ap_sta_id != IWL_MVM_INVALID_STA);
-			mvmvif->ap_sta_id = sta_id;
+			WARN_ON(mvmvif->deflink.ap_sta_id != IWL_MVM_INVALID_STA);
+			mvmvif->deflink.ap_sta_id = sta_id;
 		} else {
-			WARN_ON(mvmvif->ap_sta_id == IWL_MVM_INVALID_STA);
+			WARN_ON(mvmvif->deflink.ap_sta_id == IWL_MVM_INVALID_STA);
 		}
 	}
 
@@ -1799,7 +1892,7 @@ int iwl_mvm_drain_sta(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 	lockdep_assert_held(&mvm->mutex);
 
 	cmd.mac_id_n_color = cpu_to_le32(mvmsta->mac_id_n_color);
-	cmd.sta_id = mvmsta->sta_id;
+	cmd.sta_id = mvmsta->deflink.sta_id;
 	cmd.add_modify = STA_MODE_MODIFY;
 	cmd.station_flags = drain ? cpu_to_le32(STA_FLG_DRAIN_FLOW) : 0;
 	cmd.station_flags_msk = cpu_to_le32(STA_FLG_DRAIN_FLOW);
@@ -1814,12 +1907,12 @@ int iwl_mvm_drain_sta(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 	switch (status & IWL_ADD_STA_STATUS_MASK) {
 	case ADD_STA_SUCCESS:
 		IWL_DEBUG_INFO(mvm, "Frames for staid %d will drained in fw\n",
-			       mvmsta->sta_id);
+			       mvmsta->deflink.sta_id);
 		break;
 	default:
 		ret = -EIO;
 		IWL_ERR(mvm, "Couldn't drain frames for staid %d\n",
-			mvmsta->sta_id);
+			mvmsta->deflink.sta_id);
 		break;
 	}
 
@@ -1871,7 +1964,7 @@ static void iwl_mvm_disable_sta_queues(struct iwl_mvm *mvm,
 		if (mvm_sta->tid_data[i].txq_id == IWL_MVM_INVALID_QUEUE)
 			continue;
 
-		iwl_mvm_disable_txq(mvm, sta, mvm_sta->sta_id,
+		iwl_mvm_disable_txq(mvm, sta, mvm_sta->deflink.sta_id,
 				    &mvm_sta->tid_data[i].txq_id, i);
 		mvm_sta->tid_data[i].txq_id = IWL_MVM_INVALID_QUEUE;
 	}
@@ -1912,42 +2005,27 @@ int iwl_mvm_wait_sta_queues_empty(struct iwl_mvm *mvm,
 	return 0;
 }
 
-int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
-		   struct ieee80211_vif *vif,
-		   struct ieee80211_sta *sta)
+/* Execute the common part for both MLD and non-MLD modes.
+ * Returns if we're done with removing the station, either
+ * with error or success
+ */
+bool iwl_mvm_sta_del(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+		     struct ieee80211_sta *sta,
+		     struct ieee80211_link_sta *link_sta, int *ret)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm_vif_link_info *mvm_link =
+		mvmvif->link[link_sta->link_id];
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
-	u8 sta_id = mvm_sta->sta_id;
-	int ret;
+	struct iwl_mvm_link_sta *mvm_link_sta;
+	u8 sta_id;
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (iwl_mvm_has_new_rx_api(mvm))
-		kfree(mvm_sta->dup_data);
-
-	ret = iwl_mvm_drain_sta(mvm, mvm_sta, true);
-	if (ret)
-		return ret;
-
-	/* flush its queues here since we are freeing mvm_sta */
-	ret = iwl_mvm_flush_sta(mvm, mvm_sta, false);
-	if (ret)
-		return ret;
-	if (iwl_mvm_has_new_tx_api(mvm)) {
-		ret = iwl_mvm_wait_sta_queues_empty(mvm, mvm_sta);
-	} else {
-		u32 q_mask = mvm_sta->tfd_queue_msk;
-
-		ret = iwl_trans_wait_tx_queues_empty(mvm->trans,
-						     q_mask);
-	}
-	if (ret)
-		return ret;
-
-	ret = iwl_mvm_drain_sta(mvm, mvm_sta, false);
-
-	iwl_mvm_disable_sta_queues(mvm, vif, sta);
+	mvm_link_sta =
+		rcu_dereference_protected(mvm_sta->link[link_sta->link_id],
+					  lockdep_is_held(&mvm->mutex));
+	sta_id = mvm_link_sta->sta_id;
 
 	/* If there is a TXQ still marked as reserved - free it */
 	if (mvm_sta->reserved_queue != IEEE80211_INVAL_HW_QUEUE) {
@@ -1963,23 +2041,24 @@ int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
 		if (WARN((*status != IWL_MVM_QUEUE_RESERVED) &&
 			 (*status != IWL_MVM_QUEUE_FREE),
 			 "sta_id %d reserved txq %d status %d",
-			 sta_id, reserved_txq, *status))
-			return -EINVAL;
+			 sta_id, reserved_txq, *status)) {
+			*ret = -EINVAL;
+			return true;
+		}
 
 		*status = IWL_MVM_QUEUE_FREE;
 	}
 
-	if (vif->type == NL80211_IFTYPE_STATION &&
-	    mvmvif->ap_sta_id == sta_id) {
+	if (vif->type == NL80211_IFTYPE_STATION) {
 		/* if associated - we can't remove the AP STA now */
 		if (vif->cfg.assoc)
-			return ret;
+			return true;
 
 		/* first remove remaining keys */
-		iwl_mvm_sec_key_remove_ap(mvm, vif);
+		iwl_mvm_sec_key_remove_ap(mvm, vif, mvm_link, 0);
 
 		/* unassoc - go ahead - remove the AP STA now */
-		mvmvif->ap_sta_id = IWL_MVM_INVALID_STA;
+		mvm_link->ap_sta_id = IWL_MVM_INVALID_STA;
 	}
 
 	/*
@@ -1998,8 +2077,49 @@ int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
 	spin_lock_bh(&mvm_sta->lock);
 	spin_unlock_bh(&mvm_sta->lock);
 
-	ret = iwl_mvm_rm_sta_common(mvm, mvm_sta->sta_id);
-	RCU_INIT_POINTER(mvm->fw_id_to_mac_id[mvm_sta->sta_id], NULL);
+	return false;
+}
+
+int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
+		   struct ieee80211_vif *vif,
+		   struct ieee80211_sta *sta)
+{
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (iwl_mvm_has_new_rx_api(mvm))
+		kfree(mvm_sta->dup_data);
+
+	ret = iwl_mvm_drain_sta(mvm, mvm_sta, true);
+	if (ret)
+		return ret;
+
+	/* flush its queues here since we are freeing mvm_sta */
+	ret = iwl_mvm_flush_sta(mvm, mvm_sta, false);
+	if (ret)
+		return ret;
+	if (iwl_mvm_has_new_tx_api(mvm)) {
+		ret = iwl_mvm_wait_sta_queues_empty(mvm, mvm_sta);
+	} else {
+		u32 q_mask = mvm_sta->tfd_queue_msk;
+
+		ret = iwl_trans_wait_tx_queues_empty(mvm->trans,
+						     q_mask);
+	}
+	if (ret)
+		return ret;
+
+	ret = iwl_mvm_drain_sta(mvm, mvm_sta, false);
+
+	iwl_mvm_disable_sta_queues(mvm, vif, sta);
+
+	if (iwl_mvm_sta_del(mvm, vif, sta, &sta->deflink, &ret))
+		return ret;
+
+	ret = iwl_mvm_rm_sta_common(mvm, mvm_sta->deflink.sta_id);
+	RCU_INIT_POINTER(mvm->fw_id_to_mac_id[mvm_sta->deflink.sta_id], NULL);
 
 	return ret;
 }
@@ -2019,7 +2139,7 @@ int iwl_mvm_rm_sta_id(struct iwl_mvm *mvm,
 int iwl_mvm_allocate_int_sta(struct iwl_mvm *mvm,
 			     struct iwl_mvm_int_sta *sta,
 			     u32 qmask, enum nl80211_iftype iftype,
-			     enum iwl_sta_type type)
+			     u8 type)
 {
 	if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status) ||
 	    sta->sta_id == IWL_MVM_INVALID_STA) {
@@ -2032,7 +2152,7 @@ int iwl_mvm_allocate_int_sta(struct iwl_mvm *mvm,
 	sta->type = type;
 
 	/* put a non-NULL value so iterating over the stations won't stop */
-	rcu_assign_pointer(mvm->fw_id_to_mac_id[sta->sta_id], ERR_PTR(-EINVAL));
+	RCU_INIT_POINTER(mvm->fw_id_to_mac_id[sta->sta_id], ERR_PTR(-EINVAL));
 	return 0;
 }
 
@@ -2068,7 +2188,7 @@ static int iwl_mvm_enable_aux_snif_queue_tvqm(struct iwl_mvm *mvm, u8 sta_id)
 
 	WARN_ON(!iwl_mvm_has_new_tx_api(mvm));
 
-	return iwl_mvm_tvqm_enable_txq(mvm, sta_id, IWL_MAX_TID_COUNT,
+	return iwl_mvm_tvqm_enable_txq(mvm, NULL, sta_id, IWL_MAX_TID_COUNT,
 				       wdg_timeout);
 }
 
@@ -2113,11 +2233,13 @@ static int iwl_mvm_add_int_sta_with_queue(struct iwl_mvm *mvm, int macidx,
 int iwl_mvm_add_aux_sta(struct iwl_mvm *mvm, u32 lmac_id)
 {
 	int ret;
+	u32 qmask = mvm->aux_queue == IWL_MVM_INVALID_QUEUE ? 0 :
+		BIT(mvm->aux_queue);
 
 	lockdep_assert_held(&mvm->mutex);
 
 	/* Allocate aux station and assign to it the aux queue */
-	ret = iwl_mvm_allocate_int_sta(mvm, &mvm->aux_sta, BIT(mvm->aux_queue),
+	ret = iwl_mvm_allocate_int_sta(mvm, &mvm->aux_sta, qmask,
 				       NL80211_IFTYPE_UNSPECIFIED,
 				       IWL_STA_AUX_ACTIVITY);
 	if (ret)
@@ -2203,7 +2325,7 @@ void iwl_mvm_dealloc_snif_sta(struct iwl_mvm *mvm)
 int iwl_mvm_send_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_mvm_int_sta *bsta = &mvmvif->bcast_sta;
+	struct iwl_mvm_int_sta *bsta = &mvmvif->deflink.bcast_sta;
 	static const u8 _baddr[] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
 	const u8 *baddr = _baddr;
 	int queue;
@@ -2212,7 +2334,7 @@ int iwl_mvm_send_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 		iwl_mvm_get_wd_timeout(mvm, vif, false, false);
 	struct iwl_trans_txq_scd_cfg cfg = {
 		.fifo = IWL_MVM_TX_FIFO_VO,
-		.sta_id = mvmvif->bcast_sta.sta_id,
+		.sta_id = mvmvif->deflink.bcast_sta.sta_id,
 		.tid = IWL_MAX_TID_COUNT,
 		.aggregate = false,
 		.frame_limit = IWL_FRAME_LIMIT,
@@ -2252,7 +2374,7 @@ int iwl_mvm_send_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	 * to firmware so enable queue here - after the station was added
 	 */
 	if (iwl_mvm_has_new_tx_api(mvm)) {
-		queue = iwl_mvm_tvqm_enable_txq(mvm, bsta->sta_id,
+		queue = iwl_mvm_tvqm_enable_txq(mvm, NULL, bsta->sta_id,
 						IWL_MAX_TID_COUNT,
 						wdg_timeout);
 		if (queue < 0) {
@@ -2261,24 +2383,32 @@ int iwl_mvm_send_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 		}
 
 		if (vif->type == NL80211_IFTYPE_AP ||
-		    vif->type == NL80211_IFTYPE_ADHOC)
+		    vif->type == NL80211_IFTYPE_ADHOC) {
+			/* for queue management */
 			mvm->probe_queue = queue;
-		else if (vif->type == NL80211_IFTYPE_P2P_DEVICE)
+			/* for use in TX */
+			mvmvif->deflink.mgmt_queue = queue;
+		} else if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
 			mvm->p2p_dev_queue = queue;
+		}
+	} else if (vif->type == NL80211_IFTYPE_AP ||
+		   vif->type == NL80211_IFTYPE_ADHOC) {
+		/* set it for use in TX */
+		mvmvif->deflink.mgmt_queue = mvm->probe_queue;
 	}
 
 	return 0;
 }
 
-static void iwl_mvm_free_bcast_sta_queues(struct iwl_mvm *mvm,
-					  struct ieee80211_vif *vif)
+void iwl_mvm_free_bcast_sta_queues(struct iwl_mvm *mvm,
+				   struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	u16 *queueptr, queue;
 
 	lockdep_assert_held(&mvm->mutex);
 
-	iwl_mvm_flush_sta(mvm, &mvmvif->bcast_sta, true);
+	iwl_mvm_flush_sta(mvm, &mvmvif->deflink.bcast_sta, true);
 
 	switch (vif->type) {
 	case NL80211_IFTYPE_AP:
@@ -2295,13 +2425,17 @@ static void iwl_mvm_free_bcast_sta_queues(struct iwl_mvm *mvm,
 	}
 
 	queue = *queueptr;
-	iwl_mvm_disable_txq(mvm, NULL, mvmvif->bcast_sta.sta_id,
+	iwl_mvm_disable_txq(mvm, NULL, mvmvif->deflink.bcast_sta.sta_id,
 			    queueptr, IWL_MAX_TID_COUNT);
+
+	if (vif->type == NL80211_IFTYPE_AP || vif->type == NL80211_IFTYPE_ADHOC)
+		mvmvif->deflink.mgmt_queue = mvm->probe_queue;
+
 	if (iwl_mvm_has_new_tx_api(mvm))
 		return;
 
-	WARN_ON(!(mvmvif->bcast_sta.tfd_queue_msk & BIT(queue)));
-	mvmvif->bcast_sta.tfd_queue_msk &= ~BIT(queue);
+	WARN_ON(!(mvmvif->deflink.bcast_sta.tfd_queue_msk & BIT(queue)));
+	mvmvif->deflink.bcast_sta.tfd_queue_msk &= ~BIT(queue);
 }
 
 /* Send the FW a request to remove the station from it's internal data
@@ -2315,7 +2449,7 @@ int iwl_mvm_send_rm_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 
 	iwl_mvm_free_bcast_sta_queues(mvm, vif);
 
-	ret = iwl_mvm_rm_sta_common(mvm, mvmvif->bcast_sta.sta_id);
+	ret = iwl_mvm_rm_sta_common(mvm, mvmvif->deflink.bcast_sta.sta_id);
 	if (ret)
 		IWL_WARN(mvm, "Failed sending remove station\n");
 	return ret;
@@ -2327,7 +2461,7 @@ int iwl_mvm_alloc_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 
 	lockdep_assert_held(&mvm->mutex);
 
-	return iwl_mvm_allocate_int_sta(mvm, &mvmvif->bcast_sta, 0,
+	return iwl_mvm_allocate_int_sta(mvm, &mvmvif->deflink.bcast_sta, 0,
 					ieee80211_vif_type_p2p(vif),
 					IWL_STA_GENERAL_PURPOSE);
 }
@@ -2342,7 +2476,7 @@ int iwl_mvm_alloc_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 int iwl_mvm_add_p2p_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_mvm_int_sta *bsta = &mvmvif->bcast_sta;
+	struct iwl_mvm_int_sta *bsta = &mvmvif->deflink.bcast_sta;
 	int ret;
 
 	lockdep_assert_held(&mvm->mutex);
@@ -2363,7 +2497,7 @@ void iwl_mvm_dealloc_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-	iwl_mvm_dealloc_int_sta(mvm, &mvmvif->bcast_sta);
+	iwl_mvm_dealloc_int_sta(mvm, &mvmvif->deflink.bcast_sta);
 }
 
 /*
@@ -2394,7 +2528,7 @@ int iwl_mvm_rm_p2p_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_mvm_int_sta *msta = &mvmvif->mcast_sta;
+	struct iwl_mvm_int_sta *msta = &mvmvif->deflink.mcast_sta;
 	static const u8 _maddr[] = {0x03, 0x00, 0x00, 0x00, 0x00, 0x00};
 	const u8 *maddr = _maddr;
 	struct iwl_trans_txq_scd_cfg cfg = {
@@ -2421,7 +2555,7 @@ int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	 * changes in mac80211 layer.
 	 */
 	if (vif->type == NL80211_IFTYPE_ADHOC)
-		mvmvif->cab_queue = IWL_MVM_DQA_GCAST_QUEUE;
+		mvmvif->deflink.cab_queue = IWL_MVM_DQA_GCAST_QUEUE;
 
 	/*
 	 * While in previous FWs we had to exclude cab queue from TFD queue
@@ -2429,9 +2563,10 @@ int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	 */
 	if (!iwl_mvm_has_new_tx_api(mvm) &&
 	    fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_STA_TYPE)) {
-		iwl_mvm_enable_txq(mvm, NULL, mvmvif->cab_queue, 0, &cfg,
+		iwl_mvm_enable_txq(mvm, NULL, mvmvif->deflink.cab_queue, 0,
+				   &cfg,
 				   timeout);
-		msta->tfd_queue_msk |= BIT(mvmvif->cab_queue);
+		msta->tfd_queue_msk |= BIT(mvmvif->deflink.cab_queue);
 	}
 	ret = iwl_mvm_add_int_sta_common(mvm, msta, maddr,
 					 mvmvif->id, mvmvif->color);
@@ -2446,17 +2581,17 @@ int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	 * tfd_queue_mask.
 	 */
 	if (iwl_mvm_has_new_tx_api(mvm)) {
-		int queue = iwl_mvm_tvqm_enable_txq(mvm, msta->sta_id,
-						    0,
-						    timeout);
+		int queue = iwl_mvm_tvqm_enable_txq(mvm, NULL, msta->sta_id,
+						    0, timeout);
 		if (queue < 0) {
 			ret = queue;
 			goto err;
 		}
-		mvmvif->cab_queue = queue;
+		mvmvif->deflink.cab_queue = queue;
 	} else if (!fw_has_api(&mvm->fw->ucode_capa,
 			       IWL_UCODE_TLV_API_STA_TYPE))
-		iwl_mvm_enable_txq(mvm, NULL, mvmvif->cab_queue, 0, &cfg,
+		iwl_mvm_enable_txq(mvm, NULL, mvmvif->deflink.cab_queue, 0,
+				   &cfg,
 				   timeout);
 
 	return 0;
@@ -2529,12 +2664,12 @@ int iwl_mvm_rm_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 
 	lockdep_assert_held(&mvm->mutex);
 
-	iwl_mvm_flush_sta(mvm, &mvmvif->mcast_sta, true);
+	iwl_mvm_flush_sta(mvm, &mvmvif->deflink.mcast_sta, true);
 
-	iwl_mvm_disable_txq(mvm, NULL, mvmvif->mcast_sta.sta_id,
-			    &mvmvif->cab_queue, 0);
+	iwl_mvm_disable_txq(mvm, NULL, mvmvif->deflink.mcast_sta.sta_id,
+			    &mvmvif->deflink.cab_queue, 0);
 
-	ret = iwl_mvm_rm_sta_common(mvm, mvmvif->mcast_sta.sta_id);
+	ret = iwl_mvm_rm_sta_common(mvm, mvmvif->deflink.mcast_sta.sta_id);
 	if (ret)
 		IWL_WARN(mvm, "Failed sending remove station\n");
 
@@ -2623,13 +2758,14 @@ static void iwl_mvm_init_reorder_buffer(struct iwl_mvm *mvm,
 }
 
 static int iwl_mvm_fw_baid_op_sta(struct iwl_mvm *mvm,
-				  struct iwl_mvm_sta *mvm_sta,
+				  struct ieee80211_sta *sta,
 				  bool start, int tid, u16 ssn,
 				  u16 buf_size)
 {
+	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_mvm_add_sta_cmd cmd = {
 		.mac_id_n_color = cpu_to_le32(mvm_sta->mac_id_n_color),
-		.sta_id = mvm_sta->sta_id,
+		.sta_id = mvm_sta->deflink.sta_id,
 		.add_modify = STA_MODE_MODIFY,
 	};
 	u32 status;
@@ -2671,7 +2807,7 @@ static int iwl_mvm_fw_baid_op_sta(struct iwl_mvm *mvm,
 }
 
 static int iwl_mvm_fw_baid_op_cmd(struct iwl_mvm *mvm,
-				  struct iwl_mvm_sta *mvm_sta,
+				  struct ieee80211_sta *sta,
 				  bool start, int tid, u16 ssn,
 				  u16 buf_size, int baid)
 {
@@ -2685,7 +2821,8 @@ static int iwl_mvm_fw_baid_op_cmd(struct iwl_mvm *mvm,
 	BUILD_BUG_ON(sizeof(struct iwl_rx_baid_cfg_resp) != sizeof(baid));
 
 	if (start) {
-		cmd.alloc.sta_id_mask = cpu_to_le32(BIT(mvm_sta->sta_id));
+		cmd.alloc.sta_id_mask =
+			cpu_to_le32(iwl_mvm_sta_fw_id_mask(mvm, sta, -1));
 		cmd.alloc.tid = tid;
 		cmd.alloc.ssn = cpu_to_le16(ssn);
 		cmd.alloc.win_size = cpu_to_le16(buf_size);
@@ -2694,7 +2831,8 @@ static int iwl_mvm_fw_baid_op_cmd(struct iwl_mvm *mvm,
 		cmd.remove_v1.baid = cpu_to_le32(baid);
 		BUILD_BUG_ON(sizeof(cmd.remove_v1) > sizeof(cmd.remove));
 	} else {
-		cmd.remove.sta_id_mask = cpu_to_le32(BIT(mvm_sta->sta_id));
+		cmd.remove.sta_id_mask =
+			cpu_to_le32(iwl_mvm_sta_fw_id_mask(mvm, sta, -1));
 		cmd.remove.tid = cpu_to_le32(tid);
 	}
 
@@ -2717,16 +2855,16 @@ static int iwl_mvm_fw_baid_op_cmd(struct iwl_mvm *mvm,
 	return baid;
 }
 
-static int iwl_mvm_fw_baid_op(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvm_sta,
+static int iwl_mvm_fw_baid_op(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 			      bool start, int tid, u16 ssn, u16 buf_size,
 			      int baid)
 {
 	if (fw_has_capa(&mvm->fw->ucode_capa,
 			IWL_UCODE_TLV_CAPA_BAID_ML_SUPPORT))
-		return iwl_mvm_fw_baid_op_cmd(mvm, mvm_sta, start,
+		return iwl_mvm_fw_baid_op_cmd(mvm, sta, start,
 					      tid, ssn, buf_size, baid);
 
-	return iwl_mvm_fw_baid_op_sta(mvm, mvm_sta, start,
+	return iwl_mvm_fw_baid_op_sta(mvm, sta, start,
 				      tid, ssn, buf_size);
 }
 
@@ -2796,7 +2934,7 @@ int iwl_mvm_sta_rx_agg(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 
 	/* Don't send command to remove (start=0) BAID during restart */
 	if (start || !test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
-		baid = iwl_mvm_fw_baid_op(mvm, mvm_sta, start, tid, ssn, buf_size,
+		baid = iwl_mvm_fw_baid_op(mvm, sta, start, tid, ssn, buf_size,
 					  baid);
 
 	if (baid < 0) {
@@ -2818,7 +2956,7 @@ int iwl_mvm_sta_rx_agg(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 			    iwl_mvm_rx_agg_session_expired, 0);
 		baid_data->mvm = mvm;
 		baid_data->tid = tid;
-		baid_data->sta_id = mvm_sta->sta_id;
+		baid_data->sta_mask = iwl_mvm_sta_fw_id_mask(mvm, sta, -1);
 
 		mvm_sta->tid_to_baid[tid] = baid;
 		if (timeout)
@@ -2833,7 +2971,7 @@ int iwl_mvm_sta_rx_agg(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		 * RX is being processed in parallel
 		 */
 		IWL_DEBUG_HT(mvm, "Sta %d(%d) is assigned to BAID %d\n",
-			     mvm_sta->sta_id, tid, baid);
+			     mvm_sta->deflink.sta_id, tid, baid);
 		WARN_ON(rcu_access_pointer(mvm->baid_map[baid]));
 		rcu_assign_pointer(mvm->baid_map[baid], baid_data);
 	} else  {
@@ -2895,7 +3033,7 @@ int iwl_mvm_sta_tx_agg(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	}
 
 	cmd.mac_id_n_color = cpu_to_le32(mvm_sta->mac_id_n_color);
-	cmd.sta_id = mvm_sta->sta_id;
+	cmd.sta_id = mvm_sta->deflink.sta_id;
 	cmd.add_modify = STA_MODE_MODIFY;
 	if (!iwl_mvm_has_new_tx_api(mvm))
 		cmd.modify_mask = STA_MODIFY_QUEUES;
@@ -2987,7 +3125,7 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	 */
 	txq_id = mvmsta->tid_data[tid].txq_id;
 	if (txq_id == IWL_MVM_INVALID_QUEUE) {
-		ret = iwl_mvm_find_free_queue(mvm, mvmsta->sta_id,
+		ret = iwl_mvm_find_free_queue(mvm, mvmsta->deflink.sta_id,
 					      IWL_MVM_DQA_MIN_DATA_QUEUE,
 					      IWL_MVM_DQA_MAX_DATA_QUEUE);
 		if (ret < 0) {
@@ -3025,7 +3163,8 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	IWL_DEBUG_TX_QUEUES(mvm,
 			    "Start AGG: sta %d tid %d queue %d - ssn = %d, next_recl = %d\n",
-			    mvmsta->sta_id, tid, txq_id, tid_data->ssn,
+			    mvmsta->deflink.sta_id, tid, txq_id,
+			    tid_data->ssn,
 			    tid_data->next_reclaimed);
 
 	/*
@@ -3064,7 +3203,7 @@ int iwl_mvm_sta_tx_agg_oper(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	u16 ssn;
 
 	struct iwl_trans_txq_scd_cfg cfg = {
-		.sta_id = mvmsta->sta_id,
+		.sta_id = mvmsta->deflink.sta_id,
 		.tid = tid,
 		.frame_limit = buf_size,
 		.aggregate = true,
@@ -3136,7 +3275,7 @@ int iwl_mvm_sta_tx_agg_oper(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		}
 
 		ret = iwl_mvm_reconfig_scd(mvm, queue, cfg.fifo,
-					   mvmsta->sta_id, tid,
+					   mvmsta->deflink.sta_id, tid,
 					   buf_size, ssn);
 		if (ret) {
 			IWL_ERR(mvm,
@@ -3167,14 +3306,16 @@ out:
 	 * for each station. Therefore, use the minimum of all the
 	 * aggregation sessions and our default value.
 	 */
-	mvmsta->max_agg_bufsize =
-		min(mvmsta->max_agg_bufsize, buf_size);
-	mvmsta->lq_sta.rs_drv.lq.agg_frame_cnt_limit = mvmsta->max_agg_bufsize;
+	mvmsta->deflink.lq_sta.rs_drv.pers.max_agg_bufsize =
+		min(mvmsta->deflink.lq_sta.rs_drv.pers.max_agg_bufsize,
+		    buf_size);
+	mvmsta->deflink.lq_sta.rs_drv.lq.agg_frame_cnt_limit =
+		mvmsta->deflink.lq_sta.rs_drv.pers.max_agg_bufsize;
 
 	IWL_DEBUG_HT(mvm, "Tx aggregation enabled on ra = %pM tid = %d\n",
 		     sta->addr, tid);
 
-	return iwl_mvm_send_lq_cmd(mvm, &mvmsta->lq_sta.rs_drv.lq);
+	return iwl_mvm_send_lq_cmd(mvm, &mvmsta->deflink.lq_sta.rs_drv.lq);
 }
 
 static void iwl_mvm_unreserve_agg_queue(struct iwl_mvm *mvm,
@@ -3223,7 +3364,8 @@ int iwl_mvm_sta_tx_agg_stop(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	txq_id = tid_data->txq_id;
 
 	IWL_DEBUG_TX_QUEUES(mvm, "Stop AGG: sta %d tid %d q %d state %d\n",
-			    mvmsta->sta_id, tid, txq_id, tid_data->state);
+			    mvmsta->deflink.sta_id, tid, txq_id,
+			    tid_data->state);
 
 	mvmsta->agg_tids &= ~BIT(tid);
 
@@ -3262,7 +3404,7 @@ int iwl_mvm_sta_tx_agg_stop(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	default:
 		IWL_ERR(mvm,
 			"Stopping AGG while state not ON or starting for %d on %d (%d)\n",
-			mvmsta->sta_id, tid, tid_data->state);
+			mvmsta->deflink.sta_id, tid, tid_data->state);
 		IWL_ERR(mvm,
 			"\ttid_data->txq_id = %d\n", tid_data->txq_id);
 		err = -EINVAL;
@@ -3288,7 +3430,8 @@ int iwl_mvm_sta_tx_agg_flush(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	spin_lock_bh(&mvmsta->lock);
 	txq_id = tid_data->txq_id;
 	IWL_DEBUG_TX_QUEUES(mvm, "Flush AGG: sta %d tid %d q %d state %d\n",
-			    mvmsta->sta_id, tid, txq_id, tid_data->state);
+			    mvmsta->deflink.sta_id, tid, txq_id,
+			    tid_data->state);
 	old_state = tid_data->state;
 	tid_data->state = IWL_AGG_OFF;
 	mvmsta->agg_tids &= ~BIT(tid);
@@ -3300,7 +3443,7 @@ int iwl_mvm_sta_tx_agg_flush(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		iwl_mvm_drain_sta(mvm, mvmsta, true);
 
 		if (iwl_mvm_has_new_tx_api(mvm)) {
-			if (iwl_mvm_flush_sta_tids(mvm, mvmsta->sta_id,
+			if (iwl_mvm_flush_sta_tids(mvm, mvmsta->deflink.sta_id,
 						   BIT(tid)))
 				IWL_ERR(mvm, "Couldn't flush the AGG queue\n");
 			iwl_trans_wait_txq_empty(mvm->trans, txq_id);
@@ -3360,8 +3503,8 @@ static struct iwl_mvm_sta *iwl_mvm_get_key_sta(struct iwl_mvm *mvm,
 	 * station ID, then use AP's station ID.
 	 */
 	if (vif->type == NL80211_IFTYPE_STATION &&
-	    mvmvif->ap_sta_id != IWL_MVM_INVALID_STA) {
-		u8 sta_id = mvmvif->ap_sta_id;
+	    mvmvif->deflink.ap_sta_id != IWL_MVM_INVALID_STA) {
+		u8 sta_id = mvmvif->deflink.ap_sta_id;
 
 		sta = rcu_dereference_check(mvm->fw_id_to_mac_id[sta_id],
 					    lockdep_is_held(&mvm->mutex));
@@ -3638,8 +3781,8 @@ static inline u8 *iwl_mvm_get_mac_addr(struct iwl_mvm *mvm,
 		return sta->addr;
 
 	if (vif->type == NL80211_IFTYPE_STATION &&
-	    mvmvif->ap_sta_id != IWL_MVM_INVALID_STA) {
-		u8 sta_id = mvmvif->ap_sta_id;
+	    mvmvif->deflink.ap_sta_id != IWL_MVM_INVALID_STA) {
+		u8 sta_id = mvmvif->deflink.ap_sta_id;
 		sta = rcu_dereference_protected(mvm->fw_id_to_mac_id[sta_id],
 						lockdep_is_held(&mvm->mutex));
 		return sta->addr;
@@ -3665,13 +3808,13 @@ static int __iwl_mvm_set_sta_key(struct iwl_mvm *mvm,
 	if (sta) {
 		struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 
-		sta_id = mvm_sta->sta_id;
+		sta_id = mvm_sta->deflink.sta_id;
 		mfp = sta->mfp;
 	} else if (vif->type == NL80211_IFTYPE_AP &&
 		   !(keyconf->flags & IEEE80211_KEY_FLAG_PAIRWISE)) {
 		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-		sta_id = mvmvif->mcast_sta.sta_id;
+		sta_id = mvmvif->deflink.mcast_sta.sta_id;
 	} else {
 		IWL_ERR(mvm, "Failed to find station id\n");
 		return -EINVAL;
@@ -3714,7 +3857,7 @@ int iwl_mvm_set_sta_key(struct iwl_mvm *mvm,
 			IWL_ERR(mvm, "Failed to find station\n");
 			return -EINVAL;
 		}
-		sta_id = mvm_sta->sta_id;
+		sta_id = mvm_sta->deflink.sta_id;
 
 		/*
 		 * It is possible that the 'sta' parameter is NULL, and thus
@@ -3736,7 +3879,7 @@ int iwl_mvm_set_sta_key(struct iwl_mvm *mvm,
 	} else {
 		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
-		sta_id = mvmvif->mcast_sta.sta_id;
+		sta_id = mvmvif->deflink.mcast_sta.sta_id;
 	}
 
 	if (keyconf->cipher == WLAN_CIPHER_SUITE_AES_CMAC ||
@@ -3809,9 +3952,9 @@ int iwl_mvm_remove_sta_key(struct iwl_mvm *mvm,
 	/* Get the station from the mvm local station table */
 	mvm_sta = iwl_mvm_get_key_sta(mvm, vif, sta);
 	if (mvm_sta)
-		sta_id = mvm_sta->sta_id;
+		sta_id = mvm_sta->deflink.sta_id;
 	else if (!sta && vif->type == NL80211_IFTYPE_AP && mcast)
-		sta_id = iwl_mvm_vif_from_mac80211(vif)->mcast_sta.sta_id;
+		sta_id = iwl_mvm_vif_from_mac80211(vif)->deflink.mcast_sta.sta_id;
 
 
 	IWL_DEBUG_WEP(mvm, "mvm remove dynamic key: idx=%d sta=%d\n",
@@ -3867,7 +4010,7 @@ void iwl_mvm_update_tkip_key(struct iwl_mvm *mvm,
 	mvm_sta = iwl_mvm_get_key_sta(mvm, vif, sta);
 	if (WARN_ON_ONCE(!mvm_sta))
 		goto unlock;
-	iwl_mvm_send_sta_key(mvm, mvm_sta->sta_id, keyconf, mcast,
+	iwl_mvm_send_sta_key(mvm, mvm_sta->deflink.sta_id, keyconf, mcast,
 			     iv32, phase1key, CMD_ASYNC, keyconf->hw_key_idx,
 			     mfp);
 
@@ -3881,7 +4024,7 @@ void iwl_mvm_sta_modify_ps_wake(struct iwl_mvm *mvm,
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_mvm_add_sta_cmd cmd = {
 		.add_modify = STA_MODE_MODIFY,
-		.sta_id = mvmsta->sta_id,
+		.sta_id = mvmsta->deflink.sta_id,
 		.station_flags_msk = cpu_to_le32(STA_FLG_PS),
 		.mac_id_n_color = cpu_to_le32(mvmsta->mac_id_n_color),
 	};
@@ -3902,7 +4045,7 @@ void iwl_mvm_sta_modify_sleep_tx_count(struct iwl_mvm *mvm,
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_mvm_add_sta_cmd cmd = {
 		.add_modify = STA_MODE_MODIFY,
-		.sta_id = mvmsta->sta_id,
+		.sta_id = mvmsta->deflink.sta_id,
 		.modify_mask = STA_MODIFY_SLEEPING_STA_TX_COUNT,
 		.sleep_tx_count = cpu_to_le16(cnt),
 		.mac_id_n_color = cpu_to_le32(mvmsta->mac_id_n_color),
@@ -3995,17 +4138,23 @@ void iwl_mvm_rx_eosp_notif(struct iwl_mvm *mvm,
 }
 
 void iwl_mvm_sta_modify_disable_tx(struct iwl_mvm *mvm,
-				   struct iwl_mvm_sta *mvmsta, bool disable)
+				   struct iwl_mvm_sta *mvmsta,
+				   bool disable)
 {
 	struct iwl_mvm_add_sta_cmd cmd = {
 		.add_modify = STA_MODE_MODIFY,
-		.sta_id = mvmsta->sta_id,
+		.sta_id = mvmsta->deflink.sta_id,
 		.station_flags = disable ? cpu_to_le32(STA_FLG_DISABLE_TX) : 0,
 		.station_flags_msk = cpu_to_le32(STA_FLG_DISABLE_TX),
 		.mac_id_n_color = cpu_to_le32(mvmsta->mac_id_n_color),
 	};
 	int ret;
 
+	if (mvm->mld_api_is_used) {
+		iwl_mvm_mld_sta_modify_disable_tx(mvm, mvmsta, disable);
+		return;
+	}
+
 	ret = iwl_mvm_send_cmd_pdu(mvm, ADD_STA, CMD_ASYNC,
 				   iwl_mvm_add_sta_cmd_size(mvm), &cmd);
 	if (ret)
@@ -4018,6 +4167,11 @@ void iwl_mvm_sta_modify_disable_tx_ap(struct iwl_mvm *mvm,
 {
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 
+	if (mvm->mld_api_is_used) {
+		iwl_mvm_mld_sta_modify_disable_tx_ap(mvm, sta, disable);
+		return;
+	}
+
 	spin_lock_bh(&mvm_sta->lock);
 
 	if (mvm_sta->disable_tx == disable) {
@@ -4068,6 +4222,11 @@ void iwl_mvm_modify_all_sta_disable_tx(struct iwl_mvm *mvm,
 	struct iwl_mvm_sta *mvm_sta;
 	int i;
 
+	if (mvm->mld_api_is_used) {
+		iwl_mvm_mld_modify_all_sta_disable_tx(mvm, mvmvif, disable);
+		return;
+	}
+
 	rcu_read_lock();
 
 	/* Block/unblock all the stations of the given mvmvif */
@@ -4090,17 +4249,19 @@ void iwl_mvm_modify_all_sta_disable_tx(struct iwl_mvm *mvm,
 		return;
 
 	/* Need to block/unblock also multicast station */
-	if (mvmvif->mcast_sta.sta_id != IWL_MVM_INVALID_STA)
+	if (mvmvif->deflink.mcast_sta.sta_id != IWL_MVM_INVALID_STA)
 		iwl_mvm_int_sta_modify_disable_tx(mvm, mvmvif,
-						  &mvmvif->mcast_sta, disable);
+						  &mvmvif->deflink.mcast_sta,
+						  disable);
 
 	/*
 	 * Only unblock the broadcast station (FW blocks it for immediate
 	 * quiet, not the driver)
 	 */
-	if (!disable && mvmvif->bcast_sta.sta_id != IWL_MVM_INVALID_STA)
+	if (!disable && mvmvif->deflink.bcast_sta.sta_id != IWL_MVM_INVALID_STA)
 		iwl_mvm_int_sta_modify_disable_tx(mvm, mvmvif,
-						  &mvmvif->bcast_sta, disable);
+						  &mvmvif->deflink.bcast_sta,
+						  disable);
 }
 
 void iwl_mvm_csa_client_absent(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
@@ -4110,7 +4271,7 @@ void iwl_mvm_csa_client_absent(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 
 	rcu_read_lock();
 
-	mvmsta = iwl_mvm_sta_from_staid_rcu(mvm, mvmvif->ap_sta_id);
+	mvmsta = iwl_mvm_sta_from_staid_rcu(mvm, mvmvif->deflink.ap_sta_id);
 
 	if (mvmsta)
 		iwl_mvm_sta_modify_disable_tx(mvm, mvmsta, true);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
index f1a4fc3e4038..a61d4f88125f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2021 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2016 Intel Deutschland GmbH
  */
@@ -331,14 +331,32 @@ struct iwl_mvm_rxq_dup_data {
 } ____cacheline_aligned_in_smp;
 
 /**
+ * struct iwl_mvm_link_sta - link specific parameters of a station
+ * @rcu_head: used for freeing the data
+ * @sta_id: the index of the station in the fw
+ * @lq_sta: holds rate scaling data, either for the case when RS is done in
+ *	the driver - %rs_drv or in the FW - %rs_fw.
+ * @avg_energy: energy as reported by FW statistics notification
+ */
+struct iwl_mvm_link_sta {
+	struct rcu_head rcu_head;
+	u32 sta_id;
+	union {
+		struct iwl_lq_sta_rs_fw rs_fw;
+		struct iwl_lq_sta rs_drv;
+	} lq_sta;
+
+	u8 avg_energy;
+};
+
+/**
  * struct iwl_mvm_sta - representation of a station in the driver
- * @sta_id: the index of the station in the fw (will be replaced by id_n_color)
  * @tfd_queue_msk: the tfd queues used by the station
  * @mac_id_n_color: the MAC context this station is linked to
  * @tid_disable_agg: bitmap: if bit(tid) is set, the fw won't send ampdus for
  *	tid.
- * @max_agg_bufsize: the maximal size of the AGG buffer for this station
  * @sta_type: station type
+ * @authorized: indicates station is authorized
  * @sta_state: station state according to enum %ieee80211_sta_state
  * @bt_reduced_txpower: is reduced tx power enabled for this station
  * @next_status_eosp: the next reclaimed packet is a PS-Poll response and
@@ -347,8 +365,6 @@ struct iwl_mvm_rxq_dup_data {
  * and from Tx response flow, it needs a spinlock.
  * @tid_data: per tid data + mgmt. Look at %iwl_mvm_tid_data.
  * @tid_to_baid: a simple map of TID to baid
- * @lq_sta: holds rate scaling data, either for the case when RS is done in
- *	the driver - %rs_drv or in the FW - %rs_fw.
  * @reserved_queue: the queue reserved for this STA for DQA purposes
  *	Every STA has is given one reserved queue to allow it to operate. If no
  *	such queue can be guaranteed, the STA addition will fail.
@@ -374,6 +390,12 @@ struct iwl_mvm_rxq_dup_data {
  *	used during connection establishment (e.g. for the 4 way handshake
  *	exchange).
  * @pairwise_cipher: used to feed iwlmei upon authorization
+ * @deflink: the default link station, for non-MLO STA, all link specific data
+ *	is accessed via deflink (or link[0]). For MLO, it will hold data of the
+ *	first added link STA.
+ * @link: per link sta entries. For non-MLO only link[0] holds data. For MLO,
+ *	link[0] points to deflink and link[link_id] is allocated when new link
+ *	sta is added.
  *
  * When mac80211 creates a station it reserves some space (hw->sta_data_size)
  * in the structure for use by driver. This structure is placed in that
@@ -381,22 +403,17 @@ struct iwl_mvm_rxq_dup_data {
  *
  */
 struct iwl_mvm_sta {
-	u32 sta_id;
 	u32 tfd_queue_msk;
 	u32 mac_id_n_color;
 	u16 tid_disable_agg;
-	u16 max_agg_bufsize;
-	enum iwl_sta_type sta_type;
+	u8 sta_type;
 	enum ieee80211_sta_state sta_state;
 	bool bt_reduced_txpower;
 	bool next_status_eosp;
+	bool authorized;
 	spinlock_t lock;
 	struct iwl_mvm_tid_data tid_data[IWL_MAX_TID_COUNT + 1];
 	u8 tid_to_baid[IWL_MAX_TID_COUNT];
-	union {
-		struct iwl_lq_sta_rs_fw rs_fw;
-		struct iwl_lq_sta rs_drv;
-	} lq_sta;
 	struct ieee80211_vif *vif;
 	struct iwl_mvm_key_pn __rcu *ptk_pn[4];
 	struct iwl_mvm_rxq_dup_data *dup_data;
@@ -414,9 +431,11 @@ struct iwl_mvm_sta {
 	bool sleeping;
 	u8 agg_tids;
 	u8 sleep_tx_count;
-	u8 avg_energy;
 	u8 tx_ant;
 	u32 pairwise_cipher;
+
+	struct iwl_mvm_link_sta deflink;
+	struct iwl_mvm_link_sta __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS];
 };
 
 u16 iwl_mvm_tid_queued(struct iwl_mvm *mvm, struct iwl_mvm_tid_data *tid_data);
@@ -436,7 +455,7 @@ iwl_mvm_sta_from_mac80211(struct ieee80211_sta *sta)
  */
 struct iwl_mvm_int_sta {
 	u32 sta_id;
-	enum iwl_sta_type type;
+	u8 type;
 	u32 tfd_queue_msk;
 };
 
@@ -452,6 +471,9 @@ struct iwl_mvm_int_sta {
  */
 int iwl_mvm_sta_send_to_fw(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 			   bool update, unsigned int flags);
+int iwl_mvm_find_free_sta_id(struct iwl_mvm *mvm, enum nl80211_iftype iftype);
+int iwl_mvm_sta_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+		     struct ieee80211_sta *sta, int sta_id, u8 sta_type);
 int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 		    struct ieee80211_vif *vif,
 		    struct ieee80211_sta *sta);
@@ -463,8 +485,13 @@ static inline int iwl_mvm_update_sta(struct iwl_mvm *mvm,
 	return iwl_mvm_sta_send_to_fw(mvm, sta, true, 0);
 }
 
+void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm,
+					  struct ieee80211_sta *sta);
 int iwl_mvm_wait_sta_queues_empty(struct iwl_mvm *mvm,
 				  struct iwl_mvm_sta *mvm_sta);
+bool iwl_mvm_sta_del(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+		     struct ieee80211_sta *sta,
+		     struct ieee80211_link_sta *link_sta, int *ret);
 int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
 		   struct ieee80211_vif *vif,
 		   struct ieee80211_sta *sta);
@@ -510,6 +537,8 @@ int iwl_mvm_add_aux_sta(struct iwl_mvm *mvm, u32 lmac_id);
 int iwl_mvm_rm_aux_sta(struct iwl_mvm *mvm);
 
 int iwl_mvm_alloc_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
+void iwl_mvm_free_bcast_sta_queues(struct iwl_mvm *mvm,
+				   struct ieee80211_vif *vif);
 int iwl_mvm_send_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_add_p2p_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_send_rm_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
@@ -519,7 +548,7 @@ int iwl_mvm_rm_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_allocate_int_sta(struct iwl_mvm *mvm,
 			     struct iwl_mvm_int_sta *sta,
 				    u32 qmask, enum nl80211_iftype iftype,
-				    enum iwl_sta_type type);
+				    u8 type);
 void iwl_mvm_dealloc_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 void iwl_mvm_dealloc_int_sta(struct iwl_mvm *mvm, struct iwl_mvm_int_sta *sta);
 int iwl_mvm_add_snif_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
@@ -543,6 +572,7 @@ void iwl_mvm_sta_modify_disable_tx_ap(struct iwl_mvm *mvm,
 void iwl_mvm_modify_all_sta_disable_tx(struct iwl_mvm *mvm,
 				       struct iwl_mvm_vif *mvmvif,
 				       bool disable);
+
 void iwl_mvm_csa_client_absent(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 void iwl_mvm_add_new_dqa_stream_wk(struct work_struct *wk);
 int iwl_mvm_add_pasn_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
@@ -551,4 +581,80 @@ int iwl_mvm_add_pasn_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 void iwl_mvm_cancel_channel_switch(struct iwl_mvm *mvm,
 				   struct ieee80211_vif *vif,
 				   u32 mac_id);
+/* Queues */
+int iwl_mvm_tvqm_enable_txq(struct iwl_mvm *mvm,
+			    struct ieee80211_sta *sta,
+			    u8 sta_id, u8 tid, unsigned int timeout);
+
+/* Sta state */
+/**
+ * struct iwl_mvm_sta_state_ops - callbacks for the sta_state() ops
+ *
+ * Since the only difference between both MLD and
+ * non-MLD versions of sta_state() is these function calls,
+ * each version will send its specific function calls to
+ * %iwl_mvm_mac_sta_state_common().
+ *
+ * @add_sta: pointer to the function that adds a new sta
+ * @update_sta: pointer to the function that updates a sta
+ * @rm_sta: pointer to the functions that removes a sta
+ * @mac_ctxt_changed: pointer to the function that handles a change in mac ctxt
+ */
+struct iwl_mvm_sta_state_ops {
+	int (*add_sta)(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+		       struct ieee80211_sta *sta);
+	int (*update_sta)(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			  struct ieee80211_sta *sta);
+	int (*rm_sta)(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+		      struct ieee80211_sta *sta);
+	int (*mac_ctxt_changed)(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+				bool force_assoc_off);
+};
+
+int iwl_mvm_mac_sta_state_common(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_sta *sta,
+				 enum ieee80211_sta_state old_state,
+				 enum ieee80211_sta_state new_state,
+				 struct iwl_mvm_sta_state_ops *callbacks);
+
+/* New MLD STA related APIs */
+/* STA */
+int iwl_mvm_mld_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link_conf);
+int iwl_mvm_mld_add_snif_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf);
+int iwl_mvm_mld_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			      struct ieee80211_bss_conf *link_conf);
+int iwl_mvm_mld_add_aux_sta(struct iwl_mvm *mvm, u32 lmac_id);
+int iwl_mvm_mld_rm_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf);
+int iwl_mvm_mld_rm_snif_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
+int iwl_mvm_mld_rm_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf);
+int iwl_mvm_mld_rm_aux_sta(struct iwl_mvm *mvm);
+int iwl_mvm_mld_add_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			struct ieee80211_sta *sta);
+int iwl_mvm_mld_update_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+			   struct ieee80211_sta *sta);
+int iwl_mvm_mld_rm_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
+		       struct ieee80211_sta *sta);
+int iwl_mvm_mld_rm_sta_id(struct iwl_mvm *mvm, u8 sta_id);
+int iwl_mvm_mld_update_sta_links(struct iwl_mvm *mvm,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_sta *sta,
+				 u16 old_links, u16 new_links);
+u32 iwl_mvm_sta_fw_id_mask(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
+			   int filter_link_id);
+
+/* Queues */
+void iwl_mvm_mld_modify_all_sta_disable_tx(struct iwl_mvm *mvm,
+					   struct iwl_mvm_vif *mvmvif,
+					   bool disable);
+void iwl_mvm_mld_sta_modify_disable_tx(struct iwl_mvm *mvm,
+				       struct iwl_mvm_sta *mvm_sta,
+				       bool disable);
+void iwl_mvm_mld_sta_modify_disable_tx_ap(struct iwl_mvm *mvm,
+					  struct ieee80211_sta *sta,
+					  bool disable);
 #endif /* __sta_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c b/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c
index 674dd137fb9f..dae6f2a1aad9 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c
@@ -369,7 +369,7 @@ iwl_mvm_tdls_config_channel_switch(struct iwl_mvm *mvm,
 		goto out;
 	}
 	mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	cmd.peer_sta_id = cpu_to_le32(mvmsta->sta_id);
+	cmd.peer_sta_id = cpu_to_le32(mvmsta->deflink.sta_id);
 
 	if (!chandef) {
 		if (mvm->tdls_cs.state == IWL_MVM_TDLS_SW_REQ_SENT &&
@@ -414,7 +414,7 @@ iwl_mvm_tdls_config_channel_switch(struct iwl_mvm *mvm,
 	}
 
 	iwl_mvm_set_tx_cmd(mvm, skb, &tail->frame.tx_cmd, info,
-			   mvmsta->sta_id);
+			   mvmsta->deflink.sta_id);
 
 	iwl_mvm_set_tx_cmd_rate(mvm, &tail->frame.tx_cmd, info, sta,
 				hdr->frame_control);
@@ -431,7 +431,7 @@ iwl_mvm_tdls_config_channel_switch(struct iwl_mvm *mvm,
 
 	/* channel switch has started, update state */
 	if (type != TDLS_MOVE_CH) {
-		mvm->tdls_cs.cur_sta_id = mvmsta->sta_id;
+		mvm->tdls_cs.cur_sta_id = mvmsta->deflink.sta_id;
 		iwl_mvm_tdls_update_cs_state(mvm,
 					     type == TDLS_SEND_CHAN_SW_REQ ?
 					     IWL_MVM_TDLS_SW_REQ_SENT :
@@ -541,7 +541,7 @@ iwl_mvm_tdls_channel_switch(struct ieee80211_hw *hw,
 	}
 
 	mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	mvm->tdls_cs.peer.sta_id = mvmsta->sta_id;
+	mvm->tdls_cs.peer.sta_id = mvmsta->deflink.sta_id;
 	mvm->tdls_cs.peer.chandef = *chandef;
 	mvm->tdls_cs.peer.initiator = sta->tdls_initiator;
 	mvm->tdls_cs.peer.op_class = oper_class;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
index e403a240a82f..6b7b6250f1bb 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
@@ -79,7 +79,8 @@ void iwl_mvm_roc_done_wk(struct work_struct *wk)
 
 		if (!WARN_ON(!mvm->p2p_device_vif)) {
 			mvmvif = iwl_mvm_vif_from_mac80211(mvm->p2p_device_vif);
-			iwl_mvm_flush_sta(mvm, &mvmvif->bcast_sta, true);
+			iwl_mvm_flush_sta(mvm, &mvmvif->deflink.bcast_sta,
+					  true);
 		}
 	}
 
@@ -94,6 +95,11 @@ void iwl_mvm_roc_done_wk(struct work_struct *wk)
 		/* do the same in case of hot spot 2.0 */
 		iwl_mvm_flush_sta(mvm, &mvm->aux_sta, true);
 
+		if (mvm->mld_api_is_used) {
+			iwl_mvm_mld_rm_aux_sta(mvm);
+			goto out_unlock;
+		}
+
 		/* In newer version of this command an aux station is added only
 		 * in cases of dedicated tx queue and need to be removed in end
 		 * of use */
@@ -101,6 +107,7 @@ void iwl_mvm_roc_done_wk(struct work_struct *wk)
 			iwl_mvm_rm_aux_sta(mvm);
 	}
 
+out_unlock:
 	mutex_unlock(&mvm->mutex);
 }
 
@@ -170,7 +177,8 @@ static bool iwl_mvm_te_check_disconnect(struct iwl_mvm *mvm,
 		struct iwl_mvm_sta *mvmsta;
 
 		rcu_read_lock();
-		mvmsta = iwl_mvm_sta_from_staid_rcu(mvm, mvmvif->ap_sta_id);
+		mvmsta = iwl_mvm_sta_from_staid_rcu(mvm,
+						    mvmvif->deflink.ap_sta_id);
 		if (!WARN_ON(!mvmsta))
 			iwl_mvm_sta_modify_disable_tx(mvm, mvmsta, false);
 		rcu_read_unlock();
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/time-sync.c b/drivers/net/wireless/intel/iwlwifi/mvm/time-sync.c
new file mode 100644
index 000000000000..edae3e24192b
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/time-sync.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2022 Intel Corporation
+ */
+
+#include "mvm.h"
+#include "time-sync.h"
+#include <linux/ieee80211.h>
+
+void iwl_mvm_init_time_sync(struct iwl_time_sync_data *data)
+{
+	skb_queue_head_init(&data->frame_list);
+}
+
+static bool iwl_mvm_is_skb_match(struct sk_buff *skb, u8 *addr, u8 dialog_token)
+{
+	struct ieee80211_mgmt *mgmt = (void *)skb->data;
+	u8 skb_dialog_token;
+
+	if (ieee80211_is_timing_measurement(skb))
+		skb_dialog_token = mgmt->u.action.u.wnm_timing_msr.dialog_token;
+	else
+		skb_dialog_token = mgmt->u.action.u.ftm.dialog_token;
+
+	if ((ether_addr_equal(mgmt->sa, addr) ||
+	     ether_addr_equal(mgmt->da, addr)) &&
+	    skb_dialog_token == dialog_token)
+		return true;
+
+	return false;
+}
+
+static struct sk_buff *iwl_mvm_time_sync_find_skb(struct iwl_mvm *mvm, u8 *addr,
+						  u8 dialog_token)
+{
+	struct sk_buff *skb;
+
+	/* The queue is expected to have only one SKB. If there are other SKBs
+	 * in the queue, they did not get a time sync notification and are
+	 * probably obsolete by now, so drop them.
+	 */
+	while ((skb = skb_dequeue(&mvm->time_sync.frame_list))) {
+		if (iwl_mvm_is_skb_match(skb, addr, dialog_token))
+			break;
+
+		kfree_skb(skb);
+		skb = NULL;
+	}
+
+	return skb;
+}
+
+static u64 iwl_mvm_get_64_bit(__le32 high, __le32 low)
+{
+	return ((u64)le32_to_cpu(high) << 32) | le32_to_cpu(low);
+}
+
+void iwl_mvm_time_sync_msmt_event(struct iwl_mvm *mvm,
+				  struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	struct iwl_time_msmt_notify *notif = (void *)pkt->data;
+	struct ieee80211_rx_status *rx_status;
+	struct skb_shared_hwtstamps *shwt;
+	u64 ts_10ns;
+	struct sk_buff *skb =
+		iwl_mvm_time_sync_find_skb(mvm, notif->peer_addr,
+					   le32_to_cpu(notif->dialog_token));
+	u64 adj_time;
+
+	if (!skb) {
+		IWL_DEBUG_INFO(mvm, "Time sync event but no pending skb\n");
+		return;
+	}
+
+	ts_10ns = iwl_mvm_get_64_bit(notif->t2_hi, notif->t2_lo);
+	adj_time = iwl_mvm_ptp_get_adj_time(mvm, ts_10ns * 10);
+	shwt = skb_hwtstamps(skb);
+	shwt->hwtstamp = ktime_set(0, adj_time);
+
+	ts_10ns = iwl_mvm_get_64_bit(notif->t3_hi, notif->t3_lo);
+	adj_time = iwl_mvm_ptp_get_adj_time(mvm, ts_10ns * 10);
+	rx_status = IEEE80211_SKB_RXCB(skb);
+	rx_status->ack_tx_hwtstamp = ktime_set(0, adj_time);
+
+	IWL_DEBUG_INFO(mvm,
+		       "Time sync: RX event - report frame t2=%llu t3=%llu\n",
+		       ktime_to_ns(shwt->hwtstamp),
+		       ktime_to_ns(rx_status->ack_tx_hwtstamp));
+	ieee80211_rx_napi(mvm->hw, NULL, skb, NULL);
+}
+
+void iwl_mvm_time_sync_msmt_confirm_event(struct iwl_mvm *mvm,
+					  struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	struct iwl_time_msmt_cfm_notify *notif = (void *)pkt->data;
+	struct ieee80211_tx_status status = {};
+	struct skb_shared_hwtstamps *shwt;
+	u64 ts_10ns, adj_time;
+
+	status.skb =
+		iwl_mvm_time_sync_find_skb(mvm, notif->peer_addr,
+					   le32_to_cpu(notif->dialog_token));
+
+	if (!status.skb) {
+		IWL_DEBUG_INFO(mvm, "Time sync confirm but no pending skb\n");
+		return;
+	}
+
+	ts_10ns = iwl_mvm_get_64_bit(notif->t1_hi, notif->t1_lo);
+	adj_time = iwl_mvm_ptp_get_adj_time(mvm, ts_10ns * 10);
+	shwt = skb_hwtstamps(status.skb);
+	shwt->hwtstamp = ktime_set(0, adj_time);
+
+	ts_10ns = iwl_mvm_get_64_bit(notif->t4_hi, notif->t4_lo);
+	adj_time = iwl_mvm_ptp_get_adj_time(mvm, ts_10ns * 10);
+	status.info = IEEE80211_SKB_CB(status.skb);
+	status.ack_hwtstamp = ktime_set(0, adj_time);
+
+	IWL_DEBUG_INFO(mvm,
+		       "Time sync: TX event - report frame t1=%llu t4=%llu\n",
+		       ktime_to_ns(shwt->hwtstamp),
+		       ktime_to_ns(status.ack_hwtstamp));
+	ieee80211_tx_status_ext(mvm->hw, &status);
+}
+
+int iwl_mvm_time_sync_config(struct iwl_mvm *mvm, const u8 *addr, u32 protocols)
+{
+	struct iwl_time_sync_cfg_cmd cmd = {};
+	int err;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (!fw_has_capa(&mvm->fw->ucode_capa,
+			 IWL_UCODE_TLV_CAPA_TIME_SYNC_BOTH_FTM_TM))
+		return -EINVAL;
+
+	/* The fw only supports one peer. We do allow reconfiguration of the
+	 * same peer for cases of fw reset etc.
+	 */
+	if (mvm->time_sync.active &&
+	    !ether_addr_equal(addr, mvm->time_sync.peer_addr)) {
+		IWL_DEBUG_INFO(mvm, "Time sync: reject config for peer: %pM\n",
+			       addr);
+		return -ENOBUFS;
+	}
+
+	if (protocols & ~(IWL_TIME_SYNC_PROTOCOL_TM |
+			  IWL_TIME_SYNC_PROTOCOL_FTM))
+		return -EINVAL;
+
+	cmd.protocols = cpu_to_le32(protocols);
+
+	ether_addr_copy(cmd.peer_addr, addr);
+
+	err = iwl_mvm_send_cmd_pdu(mvm,
+				   WIDE_ID(DATA_PATH_GROUP,
+					   WNM_80211V_TIMING_MEASUREMENT_CONFIG_CMD),
+				   0, sizeof(cmd), &cmd);
+	if (err) {
+		IWL_ERR(mvm, "Failed to send time sync cfg cmd: %d\n", err);
+	} else {
+		mvm->time_sync.active = protocols != 0;
+		ether_addr_copy(mvm->time_sync.peer_addr, addr);
+		IWL_DEBUG_INFO(mvm, "Time sync: set peer addr=%pM\n", addr);
+	}
+
+	if (!mvm->time_sync.active)
+		skb_queue_purge(&mvm->time_sync.frame_list);
+
+	return err;
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/time-sync.h b/drivers/net/wireless/intel/iwlwifi/mvm/time-sync.h
new file mode 100644
index 000000000000..2cfd0fb5e781
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/time-sync.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2022 Intel Corporation
+ */
+#ifndef __TIME_SYNC_H__
+#define __TIME_SYNC_H__
+
+#include "mvm.h"
+#include <linux/ieee80211.h>
+
+void iwl_mvm_init_time_sync(struct iwl_time_sync_data *data);
+void iwl_mvm_time_sync_msmt_event(struct iwl_mvm *mvm,
+				  struct iwl_rx_cmd_buffer *rxb);
+void iwl_mvm_time_sync_msmt_confirm_event(struct iwl_mvm *mvm,
+					  struct iwl_rx_cmd_buffer *rxb);
+int iwl_mvm_time_sync_config(struct iwl_mvm *mvm, const u8 *addr,
+			     u32 protocols);
+
+static inline
+bool iwl_mvm_time_sync_frame(struct iwl_mvm *mvm, struct sk_buff *skb, u8 *addr)
+{
+	if (ether_addr_equal(mvm->time_sync.peer_addr, addr) &&
+	    (ieee80211_is_timing_measurement(skb) || ieee80211_is_ftm(skb))) {
+		skb_queue_tail(&mvm->time_sync.frame_list, skb);
+		return true;
+	}
+
+	return false;
+}
+#endif
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
index 232c200af38f..157e96fa23c1 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2019-2021 Intel Corporation
+ * Copyright (C) 2012-2014, 2019-2022 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2016 Intel Deutschland GmbH
  */
@@ -334,7 +334,7 @@ static void iwl_mvm_tt_smps_iterator(void *_data, u8 *mac,
 	if (vif->type != NL80211_IFTYPE_STATION)
 		return;
 
-	iwl_mvm_update_smps(mvm, vif, IWL_MVM_SMPS_REQ_TT, smps_mode);
+	iwl_mvm_update_smps(mvm, vif, IWL_MVM_SMPS_REQ_TT, smps_mode, 0);
 }
 
 static void iwl_mvm_tt_tx_protection(struct iwl_mvm *mvm, bool enable)
@@ -615,7 +615,7 @@ send:
 static int iwl_mvm_tzone_get_temp(struct thermal_zone_device *device,
 				  int *temperature)
 {
-	struct iwl_mvm *mvm = (struct iwl_mvm *)device->devdata;
+	struct iwl_mvm *mvm = thermal_zone_device_priv(device);
 	int ret;
 	int temp;
 
@@ -641,7 +641,7 @@ out:
 static int iwl_mvm_tzone_set_trip_temp(struct thermal_zone_device *device,
 				       int trip, int temp)
 {
-	struct iwl_mvm *mvm = (struct iwl_mvm *)device->devdata;
+	struct iwl_mvm *mvm = thermal_zone_device_priv(device);
 	struct iwl_mvm_thermal_device *tzone;
 	int ret;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
index 9813d7fa1800..10d7178f1071 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
@@ -14,6 +14,7 @@
 #include "iwl-eeprom-parse.h"
 #include "mvm.h"
 #include "sta.h"
+#include "time-sync.h"
 
 static void
 iwl_mvm_bar_check_trigger(struct iwl_mvm *mvm, const u8 *addr,
@@ -183,10 +184,7 @@ static u32 iwl_mvm_tx_csum(struct iwl_mvm *mvm, struct sk_buff *skb,
 			   struct ieee80211_tx_info *info,
 			   bool amsdu)
 {
-	if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ ||
-	    (mvm->trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_BZ &&
-	     CSR_HW_REV_TYPE(mvm->trans->hw_rev) == IWL_CFG_MAC_TYPE_GL &&
-	     mvm->trans->hw_rev_step == SILICON_A_STEP))
+	if (!iwl_mvm_has_new_tx_csum(mvm))
 		return iwl_mvm_tx_csum_pre_bz(mvm, skb, info, amsdu);
 	return iwl_mvm_tx_csum_bz(mvm, skb, amsdu);
 }
@@ -331,22 +329,23 @@ static u32 iwl_mvm_get_tx_rate(struct iwl_mvm *mvm,
 			  sta ? iwl_mvm_sta_from_mac80211(sta)->sta_state : -1);
 
 		rate_idx = info->control.rates[0].idx;
+
+		/* For non 2 GHZ band, remap mac80211 rate indices into driver
+		 * indices.
+		 */
+		if (info->band != NL80211_BAND_2GHZ ||
+		    (info->flags & IEEE80211_TX_CTL_NO_CCK_RATE))
+			rate_idx += IWL_FIRST_OFDM_RATE;
+
+		/* For 2.4 GHZ band, check that there is no need to remap */
+		BUILD_BUG_ON(IWL_FIRST_CCK_RATE != 0);
 	}
 
 	/* if the rate isn't a well known legacy rate, take the lowest one */
 	if (rate_idx < 0 || rate_idx >= IWL_RATE_COUNT_LEGACY)
-		rate_idx = rate_lowest_index(
-				&mvm->nvm_data->bands[info->band], sta);
-
-	/*
-	 * For non 2 GHZ band, remap mac80211 rate
-	 * indices into driver indices
-	 */
-	if (info->band != NL80211_BAND_2GHZ)
-		rate_idx += IWL_FIRST_OFDM_RATE;
-
-	/* For 2.4 GHZ band, check that there is no need to remap */
-	BUILD_BUG_ON(IWL_FIRST_CCK_RATE != 0);
+		rate_idx = iwl_mvm_mac_ctxt_get_lowest_rate(mvm,
+							    info,
+							    info->control.vif);
 
 	/* Get PLCP rate for tx_cmd->rate_n_flags */
 	rate_plcp = iwl_mvm_mac80211_idx_to_hwrate(mvm->fw, rate_idx);
@@ -603,11 +602,11 @@ static void iwl_mvm_skb_prepare_status(struct sk_buff *skb,
 }
 
 static int iwl_mvm_get_ctrl_vif_queue(struct iwl_mvm *mvm,
+				      struct iwl_mvm_vif_link_info *link,
 				      struct ieee80211_tx_info *info,
-				      struct ieee80211_hdr *hdr)
+				      struct sk_buff *skb)
 {
-	struct iwl_mvm_vif *mvmvif =
-		iwl_mvm_vif_from_mac80211(info->control.vif);
+	struct ieee80211_hdr *hdr = (void *)skb->data;
 	__le16 fc = hdr->frame_control;
 
 	switch (info->control.vif->type) {
@@ -624,17 +623,17 @@ static int iwl_mvm_get_ctrl_vif_queue(struct iwl_mvm *mvm,
 		 * reason 7 ("Class 3 frame received from nonassociated STA").
 		 */
 		if (ieee80211_is_mgmt(fc) &&
-		    (!ieee80211_is_bufferable_mmpdu(fc) ||
+		    (!ieee80211_is_bufferable_mmpdu(skb) ||
 		     ieee80211_is_deauth(fc) || ieee80211_is_disassoc(fc)))
-			return mvm->probe_queue;
+			return link->mgmt_queue;
 
 		if (!ieee80211_has_order(fc) && !ieee80211_is_probe_req(fc) &&
 		    is_multicast_ether_addr(hdr->addr1))
-			return mvmvif->cab_queue;
+			return link->cab_queue;
 
 		WARN_ONCE(info->control.vif->type != NL80211_IFTYPE_ADHOC,
 			  "fc=0x%02x", le16_to_cpu(fc));
-		return mvm->probe_queue;
+		return link->mgmt_queue;
 	case NL80211_IFTYPE_P2P_DEVICE:
 		if (ieee80211_is_mgmt(fc))
 			return mvm->p2p_dev_queue;
@@ -667,7 +666,7 @@ static void iwl_mvm_probe_resp_set_noa(struct iwl_mvm *mvm,
 
 	rcu_read_lock();
 
-	resp_data = rcu_dereference(mvmvif->probe_resp_data);
+	resp_data = rcu_dereference(mvmvif->deflink.probe_resp_data);
 	if (!resp_data)
 		goto out;
 
@@ -738,12 +737,26 @@ int iwl_mvm_tx_skb_non_sta(struct iwl_mvm *mvm, struct sk_buff *skb)
 		if (info.control.vif->type == NL80211_IFTYPE_P2P_DEVICE ||
 		    info.control.vif->type == NL80211_IFTYPE_AP ||
 		    info.control.vif->type == NL80211_IFTYPE_ADHOC) {
+			u32 link_id = u32_get_bits(info.control.flags,
+						   IEEE80211_TX_CTRL_MLO_LINK);
+			struct iwl_mvm_vif_link_info *link;
+
+			if (link_id == IEEE80211_LINK_UNSPECIFIED) {
+				if (info.control.vif->active_links)
+					link_id = ffs(info.control.vif->active_links) - 1;
+				else
+					link_id = 0;
+			}
+
+			link = mvmvif->link[link_id];
+
 			if (!ieee80211_is_data(hdr->frame_control))
-				sta_id = mvmvif->bcast_sta.sta_id;
+				sta_id = link->bcast_sta.sta_id;
 			else
-				sta_id = mvmvif->mcast_sta.sta_id;
+				sta_id = link->mcast_sta.sta_id;
 
-			queue = iwl_mvm_get_ctrl_vif_queue(mvm, &info, hdr);
+			queue = iwl_mvm_get_ctrl_vif_queue(mvm, link, &info,
+							   skb);
 		} else if (info.control.vif->type == NL80211_IFTYPE_MONITOR) {
 			queue = mvm->snif_queue;
 			sta_id = mvm->snif_sta.sta_id;
@@ -791,10 +804,11 @@ unsigned int iwl_mvm_max_amsdu_size(struct iwl_mvm *mvm,
 				    struct ieee80211_sta *sta, unsigned int tid)
 {
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	enum nl80211_band band = mvmsta->vif->bss_conf.chandef.chan->band;
 	u8 ac = tid_to_mac80211_ac[tid];
+	enum nl80211_band band;
 	unsigned int txf;
-	int lmac = iwl_mvm_get_lmac_id(mvm->fw, band);
+	unsigned int val;
+	int lmac;
 
 	/* For HE redirect to trigger based fifos */
 	if (sta->deflink.he_cap.has_he && !WARN_ON(!iwl_mvm_has_new_tx_api(mvm)))
@@ -808,7 +822,37 @@ unsigned int iwl_mvm_max_amsdu_size(struct iwl_mvm *mvm,
 	 * We also want to have the start of the next packet inside the
 	 * fifo to be able to send bursts.
 	 */
-	return min_t(unsigned int, mvmsta->max_amsdu_len,
+	val = mvmsta->max_amsdu_len;
+
+	if (hweight16(sta->valid_links) <= 1) {
+		if (sta->valid_links) {
+			struct ieee80211_bss_conf *link_conf;
+			unsigned int link = ffs(sta->valid_links) - 1;
+
+			rcu_read_lock();
+			link_conf = rcu_dereference(mvmsta->vif->link_conf[link]);
+			if (WARN_ON(!link_conf))
+				band = NL80211_BAND_2GHZ;
+			else
+				band = link_conf->chandef.chan->band;
+			rcu_read_unlock();
+		} else {
+			band = mvmsta->vif->bss_conf.chandef.chan->band;
+		}
+
+		lmac = iwl_mvm_get_lmac_id(mvm->fw, band);
+	} else if (fw_has_capa(&mvm->fw->ucode_capa,
+			       IWL_UCODE_TLV_CAPA_CDB_SUPPORT)) {
+		/* for real MLO restrict to both LMACs if they exist */
+		lmac = IWL_LMAC_5G_INDEX;
+		val = min_t(unsigned int, val,
+			    mvm->fwrt.smem_cfg.lmac[lmac].txfifo_size[txf] - 256);
+		lmac = IWL_LMAC_24G_INDEX;
+	} else {
+		lmac = IWL_LMAC_24G_INDEX;
+	}
+
+	return min_t(unsigned int, val,
 		     mvm->fwrt.smem_cfg.lmac[lmac].txfifo_size[txf] - 256);
 }
 
@@ -1083,7 +1127,7 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 	if (WARN_ON_ONCE(!mvmsta))
 		return -1;
 
-	if (WARN_ON_ONCE(mvmsta->sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON_ONCE(mvmsta->deflink.sta_id == IWL_MVM_INVALID_STA))
 		return -1;
 
 	if (unlikely(ieee80211_is_any_nullfunc(fc)) && sta->deflink.he_cap.has_he)
@@ -1093,7 +1137,7 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 		iwl_mvm_probe_resp_set_noa(mvm, skb);
 
 	dev_cmd = iwl_mvm_set_tx_params(mvm, skb, info, hdrlen,
-					sta, mvmsta->sta_id);
+					sta, mvmsta->deflink.sta_id);
 	if (!dev_cmd)
 		goto drop;
 
@@ -1169,7 +1213,7 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 	}
 
 	IWL_DEBUG_TX(mvm, "TX to [%d|%d] Q:%d - seq: 0x%x len %d\n",
-		     mvmsta->sta_id, tid, txq_id,
+		     mvmsta->deflink.sta_id, tid, txq_id,
 		     IEEE80211_SEQ_TO_SN(seq_number), skb->len);
 
 	/* From now on, we cannot access info->control */
@@ -1204,7 +1248,8 @@ drop_unlock_sta:
 	iwl_trans_free_tx_cmd(mvm->trans, dev_cmd);
 	spin_unlock(&mvmsta->lock);
 drop:
-	IWL_DEBUG_TX(mvm, "TX to [%d|%d] dropped\n", mvmsta->sta_id, tid);
+	IWL_DEBUG_TX(mvm, "TX to [%d|%d] dropped\n", mvmsta->deflink.sta_id,
+		     tid);
 	return -1;
 }
 
@@ -1221,7 +1266,7 @@ int iwl_mvm_tx_skb_sta(struct iwl_mvm *mvm, struct sk_buff *skb,
 	if (WARN_ON_ONCE(!mvmsta))
 		return -1;
 
-	if (WARN_ON_ONCE(mvmsta->sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON_ONCE(mvmsta->deflink.sta_id == IWL_MVM_INVALID_STA))
 		return -1;
 
 	memcpy(&info, skb->cb, sizeof(info));
@@ -1241,8 +1286,7 @@ int iwl_mvm_tx_skb_sta(struct iwl_mvm *mvm, struct sk_buff *skb,
 	if (ret)
 		return ret;
 
-	if (WARN_ON(skb_queue_empty(&mpdus_skbs)))
-		return ret;
+	WARN_ON(skb_queue_empty(&mpdus_skbs));
 
 	while (!skb_queue_empty(&mpdus_skbs)) {
 		skb = __skb_dequeue(&mpdus_skbs);
@@ -1396,8 +1440,8 @@ void iwl_mvm_hwrate_to_tx_rate(u32 rate_n_flags,
 		r->idx = rate;
 	} else if (format ==  RATE_MCS_VHT_MSK) {
 		ieee80211_rate_set_vht(r, rate,
-				       ((rate_n_flags & RATE_MCS_NSS_MSK) >>
-					RATE_MCS_NSS_POS) + 1);
+				       FIELD_GET(RATE_MCS_NSS_MSK,
+						 rate_n_flags) + 1);
 		r->flags |= IEEE80211_TX_RC_VHT_MCS;
 	} else if (format == RATE_MCS_HE_MSK) {
 		/* mac80211 cannot do this without ieee80211_tx_status_ext()
@@ -1428,8 +1472,7 @@ void iwl_mvm_hwrate_to_tx_rate_v1(u32 rate_n_flags,
 	} else if (rate_n_flags & RATE_MCS_VHT_MSK_V1) {
 		ieee80211_rate_set_vht(
 			r, rate_n_flags & RATE_VHT_MCS_RATE_CODE_MSK,
-			((rate_n_flags & RATE_VHT_MCS_NSS_MSK) >>
-						RATE_VHT_MCS_NSS_POS) + 1);
+			FIELD_GET(RATE_MCS_NSS_MSK, rate_n_flags) + 1);
 		r->flags |= IEEE80211_TX_RC_VHT_MCS;
 	} else {
 		r->idx = iwl_mvm_legacy_rate_to_mac80211_idx(rate_n_flags,
@@ -1644,7 +1687,8 @@ static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 		info->status.status_driver_data[0] =
 			RS_DRV_DATA_PACK(lq_color, tx_resp->reduced_tpc);
 
-		ieee80211_tx_status(mvm->hw, skb);
+		if (likely(!iwl_mvm_time_sync_frame(mvm, skb, hdr->addr1)))
+			ieee80211_tx_status(mvm->hw, skb);
 	}
 
 	/* This is an aggregation queue or might become one, so we use
@@ -1974,9 +2018,11 @@ static void iwl_mvm_tx_reclaim(struct iwl_mvm *mvm, int sta_id, int tid,
 	 * possible (i.e. first MPDU in the aggregation wasn't acked)
 	 * Still it's important to update RS about sent vs. acked.
 	 */
-	if (!is_flush && skb_queue_empty(&reclaimed_skbs)) {
+	if (!is_flush && skb_queue_empty(&reclaimed_skbs) &&
+	    !iwl_mvm_has_tlc_offload(mvm)) {
 		struct ieee80211_chanctx_conf *chanctx_conf = NULL;
 
+		/* no TLC offload, so non-MLD mode */
 		if (mvmsta->vif)
 			chanctx_conf =
 				rcu_dereference(mvmsta->vif->bss_conf.chanctx_conf);
@@ -1987,11 +2033,8 @@ static void iwl_mvm_tx_reclaim(struct iwl_mvm *mvm, int sta_id, int tid,
 		tx_info->band = chanctx_conf->def.chan->band;
 		iwl_mvm_hwrate_to_tx_status(mvm->fw, rate, tx_info);
 
-		if (!iwl_mvm_has_tlc_offload(mvm)) {
-			IWL_DEBUG_TX_REPLY(mvm,
-					   "No reclaim. Update rs directly\n");
-			iwl_mvm_rs_tx_status(mvm, sta, tid, tx_info, false);
-		}
+		IWL_DEBUG_TX_REPLY(mvm, "No reclaim. Update rs directly\n");
+		iwl_mvm_rs_tx_status(mvm, sta, tid, tx_info, false);
 	}
 
 out:
@@ -2229,17 +2272,22 @@ free_rsp:
 
 int iwl_mvm_flush_sta(struct iwl_mvm *mvm, void *sta, bool internal)
 {
-	struct iwl_mvm_int_sta *int_sta = sta;
-	struct iwl_mvm_sta *mvm_sta = sta;
+	u32 sta_id, tfd_queue_msk;
 
-	BUILD_BUG_ON(offsetof(struct iwl_mvm_int_sta, sta_id) !=
-		     offsetof(struct iwl_mvm_sta, sta_id));
+	if (internal) {
+		struct iwl_mvm_int_sta *int_sta = sta;
 
-	if (iwl_mvm_has_new_tx_api(mvm))
-		return iwl_mvm_flush_sta_tids(mvm, mvm_sta->sta_id, 0xffff);
+		sta_id = int_sta->sta_id;
+		tfd_queue_msk = int_sta->tfd_queue_msk;
+	} else {
+		struct iwl_mvm_sta *mvm_sta = sta;
 
-	if (internal)
-		return iwl_mvm_flush_tx_path(mvm, int_sta->tfd_queue_msk);
+		sta_id = mvm_sta->deflink.sta_id;
+		tfd_queue_msk = mvm_sta->tfd_queue_msk;
+	}
+
+	if (iwl_mvm_has_new_tx_api(mvm))
+		return iwl_mvm_flush_sta_tids(mvm, sta_id, 0xffff);
 
-	return iwl_mvm_flush_tx_path(mvm, mvm_sta->tfd_queue_msk);
+	return iwl_mvm_flush_tx_path(mvm, tfd_queue_msk);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
index 14b2de65bd84..af31b09c3966 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
@@ -272,13 +272,15 @@ int iwl_mvm_send_lq_cmd(struct iwl_mvm *mvm, struct iwl_lq_cmd *lq)
  * @vif: Pointer to the ieee80211_vif structure
  * @req_type: The part of the driver who call for a change.
  * @smps_request: The request to change the SMPS mode.
+ * @link_id: for MLO link_id, otherwise 0 (deflink)
  *
  * Get a requst to change the SMPS mode,
  * and change it according to all other requests in the driver.
  */
 void iwl_mvm_update_smps(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			 enum iwl_mvm_smps_type_request req_type,
-			 enum ieee80211_smps_mode smps_request)
+			 enum ieee80211_smps_mode smps_request,
+			 unsigned int link_id)
 {
 	struct iwl_mvm_vif *mvmvif;
 	enum ieee80211_smps_mode smps_mode = IEEE80211_SMPS_AUTOMATIC;
@@ -294,17 +296,38 @@ void iwl_mvm_update_smps(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		return;
 
 	mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	mvmvif->smps_requests[req_type] = smps_request;
+
+	if (WARN_ON_ONCE(!mvmvif->link[link_id]))
+		return;
+
+	mvmvif->link[link_id]->smps_requests[req_type] = smps_request;
 	for (i = 0; i < NUM_IWL_MVM_SMPS_REQ; i++) {
-		if (mvmvif->smps_requests[i] == IEEE80211_SMPS_STATIC) {
+		if (mvmvif->link[link_id]->smps_requests[i] ==
+		    IEEE80211_SMPS_STATIC) {
 			smps_mode = IEEE80211_SMPS_STATIC;
 			break;
 		}
-		if (mvmvif->smps_requests[i] == IEEE80211_SMPS_DYNAMIC)
+		if (mvmvif->link[link_id]->smps_requests[i] ==
+		    IEEE80211_SMPS_DYNAMIC)
 			smps_mode = IEEE80211_SMPS_DYNAMIC;
 	}
 
-	ieee80211_request_smps(vif, 0, smps_mode);
+	ieee80211_request_smps(vif, link_id, smps_mode);
+}
+
+void iwl_mvm_update_smps_on_active_links(struct iwl_mvm *mvm,
+					 struct ieee80211_vif *vif,
+					 enum iwl_mvm_smps_type_request req_type,
+					 enum ieee80211_smps_mode smps_request)
+{
+	struct ieee80211_bss_conf *link_conf;
+	unsigned int link_id;
+
+	rcu_read_lock();
+	for_each_vif_active_link(vif, link_conf, link_id)
+		iwl_mvm_update_smps(mvm, vif, req_type, smps_request,
+				    link_id);
+	rcu_read_unlock();
 }
 
 static bool iwl_wait_stats_complete(struct iwl_notif_wait_data *notif_wait,
@@ -392,12 +415,12 @@ static void iwl_mvm_diversity_iter(void *_data, u8 *mac,
 	struct iwl_mvm_diversity_iter_data *data = _data;
 	int i;
 
-	if (mvmvif->phy_ctxt != data->ctxt)
+	if (mvmvif->deflink.phy_ctxt != data->ctxt)
 		return;
 
 	for (i = 0; i < NUM_IWL_MVM_SMPS_REQ; i++) {
-		if (mvmvif->smps_requests[i] == IEEE80211_SMPS_STATIC ||
-		    mvmvif->smps_requests[i] == IEEE80211_SMPS_DYNAMIC) {
+		if (mvmvif->deflink.smps_requests[i] == IEEE80211_SMPS_STATIC ||
+		    mvmvif->deflink.smps_requests[i] == IEEE80211_SMPS_DYNAMIC) {
 			data->result = false;
 			break;
 		}
@@ -495,10 +518,10 @@ static void iwl_mvm_ll_iter(void *_data, u8 *mac, struct ieee80211_vif *vif)
 	if (iwl_mvm_vif_low_latency(mvmvif)) {
 		result->result = true;
 
-		if (!mvmvif->phy_ctxt)
+		if (!mvmvif->deflink.phy_ctxt)
 			return;
 
-		band = mvmvif->phy_ctxt->channel->band;
+		band = mvmvif->deflink.phy_ctxt->channel->band;
 		result->result_per_band[band] = true;
 	}
 }
@@ -819,10 +842,10 @@ static void iwl_mvm_uapsd_agg_disconnect(struct iwl_mvm *mvm,
 	if (!vif->cfg.assoc)
 		return;
 
-	if (!mvmvif->queue_params[IEEE80211_AC_VO].uapsd &&
-	    !mvmvif->queue_params[IEEE80211_AC_VI].uapsd &&
-	    !mvmvif->queue_params[IEEE80211_AC_BE].uapsd &&
-	    !mvmvif->queue_params[IEEE80211_AC_BK].uapsd)
+	if (!mvmvif->deflink.queue_params[IEEE80211_AC_VO].uapsd &&
+	    !mvmvif->deflink.queue_params[IEEE80211_AC_VI].uapsd &&
+	    !mvmvif->deflink.queue_params[IEEE80211_AC_BE].uapsd &&
+	    !mvmvif->deflink.queue_params[IEEE80211_AC_BK].uapsd)
 		return;
 
 	if (mvm->tcm.data[mvmvif->id].uapsd_nonagg_detect.detected)
@@ -831,7 +854,8 @@ static void iwl_mvm_uapsd_agg_disconnect(struct iwl_mvm *mvm,
 	mvm->tcm.data[mvmvif->id].uapsd_nonagg_detect.detected = true;
 	IWL_INFO(mvm,
 		 "detected AP should do aggregation but isn't, likely due to U-APSD\n");
-	schedule_delayed_work(&mvmvif->uapsd_nonagg_detected_wk, 15 * HZ);
+	schedule_delayed_work(&mvmvif->uapsd_nonagg_detected_wk,
+			      15 * HZ);
 }
 
 static void iwl_mvm_check_uapsd_agg_expected_tpt(struct iwl_mvm *mvm,
@@ -883,10 +907,10 @@ static void iwl_mvm_tcm_iterator(void *_data, u8 *mac,
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	u32 *band = _data;
 
-	if (!mvmvif->phy_ctxt)
+	if (!mvmvif->deflink.phy_ctxt)
 		return;
 
-	band[mvmvif->id] = mvmvif->phy_ctxt->channel->band;
+	band[mvmvif->id] = mvmvif->deflink.phy_ctxt->channel->band;
 }
 
 static unsigned long iwl_mvm_calc_tcm_stats(struct iwl_mvm *mvm,
@@ -1137,3 +1161,36 @@ void iwl_mvm_get_sync_time(struct iwl_mvm *mvm, int clock_type,
 		iwl_mvm_power_update_device(mvm);
 	}
 }
+
+/* Find if at least two links from different vifs use same channel
+ * FIXME: consider having a refcount array in struct iwl_mvm_vif for
+ * used phy_ctxt ids.
+ */
+bool iwl_mvm_have_links_same_channel(struct iwl_mvm_vif *vif1,
+				     struct iwl_mvm_vif *vif2)
+{
+	unsigned int i, j;
+
+	for_each_mvm_vif_valid_link(vif1, i) {
+		for_each_mvm_vif_valid_link(vif2, j) {
+			if (vif1->link[i]->phy_ctxt == vif2->link[j]->phy_ctxt)
+				return true;
+		}
+	}
+
+	return false;
+}
+
+bool iwl_mvm_vif_is_active(struct iwl_mvm_vif *mvmvif)
+{
+	unsigned int i;
+
+	/* FIXME: can it fail when phy_ctxt is assigned? */
+	for_each_mvm_vif_valid_link(mvmvif, i) {
+		if (mvmvif->link[i]->phy_ctxt &&
+		    mvmvif->link[i]->phy_ctxt->id < NUM_PHY_CTX)
+			return true;
+	}
+
+	return false;
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index 99768d6a6032..dba112394838 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -504,6 +504,7 @@ static const struct pci_device_id iwl_hw_card_ids[] = {
 
 /* Bz devices */
 	{IWL_PCI_DEVICE(0x2727, PCI_ANY_ID, iwl_bz_trans_cfg)},
+	{IWL_PCI_DEVICE(0x272b, PCI_ANY_ID, iwl_bz_trans_cfg)},
 	{IWL_PCI_DEVICE(0xA840, PCI_ANY_ID, iwl_bz_trans_cfg)},
 	{IWL_PCI_DEVICE(0x7740, PCI_ANY_ID, iwl_bz_trans_cfg)},
 #endif /* CONFIG_IWLMVM */
@@ -513,16 +514,17 @@ static const struct pci_device_id iwl_hw_card_ids[] = {
 MODULE_DEVICE_TABLE(pci, iwl_hw_card_ids);
 
 #define _IWL_DEV_INFO(_device, _subdevice, _mac_type, _mac_step, _rf_type, \
-		      _rf_id, _no_160, _cores, _cdb, _jacket, _cfg, _name) \
-	{ .device = (_device), .subdevice = (_subdevice), .cfg = &(_cfg),  \
-	  .name = _name, .mac_type = _mac_type, .rf_type = _rf_type,	   \
-	  .no_160 = _no_160, .cores = _cores, .rf_id = _rf_id,		   \
+		      _rf_id, _rf_step, _no_160, _cores, _cdb, _jacket, _cfg, \
+		      _name) \
+	{ .device = (_device), .subdevice = (_subdevice), .cfg = &(_cfg), \
+	  .name = _name, .mac_type = _mac_type, .rf_type = _rf_type, .rf_step = _rf_step, \
+	  .no_160 = _no_160, .cores = _cores, .rf_id = _rf_id, \
 	  .mac_step = _mac_step, .cdb = _cdb, .jacket = _jacket }
 
 #define IWL_DEV_INFO(_device, _subdevice, _cfg, _name) \
-	_IWL_DEV_INFO(_device, _subdevice, IWL_CFG_ANY, IWL_CFG_ANY,	   \
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY,  \
-		      IWL_CFG_ANY, IWL_CFG_ANY, _cfg, _name)
+	_IWL_DEV_INFO(_device, _subdevice, IWL_CFG_ANY, IWL_CFG_ANY, \
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY, \
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY, _cfg, _name)
 
 static const struct iwl_dev_info iwl_dev_info_table[] = {
 #if IS_ENABLED(CONFIG_IWLMVM)
@@ -565,7 +567,6 @@ static const struct iwl_dev_info iwl_dev_info_table[] = {
 	IWL_DEV_INFO(0x43F0, 0x1652, killer1650i_2ax_cfg_qu_b0_hr_b0, iwl_ax201_killer_1650i_name),
 	IWL_DEV_INFO(0x43F0, 0x2074, iwl_ax201_cfg_qu_hr, NULL),
 	IWL_DEV_INFO(0x43F0, 0x4070, iwl_ax201_cfg_qu_hr, NULL),
-	IWL_DEV_INFO(0x43F0, 0x1651, killer1650s_2ax_cfg_qu_b0_hr_b0, iwl_ax201_killer_1650s_name),
 	IWL_DEV_INFO(0xA0F0, 0x0070, iwl_ax201_cfg_qu_hr, NULL),
 	IWL_DEV_INFO(0xA0F0, 0x0074, iwl_ax201_cfg_qu_hr, NULL),
 	IWL_DEV_INFO(0xA0F0, 0x0078, iwl_ax201_cfg_qu_hr, NULL),
@@ -694,87 +695,87 @@ static const struct iwl_dev_info iwl_dev_info_table[] = {
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_2ac_cfg_soc, iwl9461_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_2ac_cfg_soc, iwl9461_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_2ac_cfg_soc, iwl9462_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_2ac_cfg_soc, iwl9462_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_2ac_cfg_soc, iwl9560_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_PU, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_2ac_cfg_soc, iwl9560_name),
 
 	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_PNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9260_2ac_cfg, iwl9461_160_name),
 	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_PNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9260_2ac_cfg, iwl9461_name),
 	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_PNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9260_2ac_cfg, iwl9462_160_name),
 	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_PNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9260_2ac_cfg, iwl9462_name),
 
 	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT_GNSS, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9260_2ac_cfg, iwl9270_160_name),
 	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT_GNSS, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9260_2ac_cfg, iwl9270_name),
 
 	_IWL_DEV_INFO(0x271B, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH1, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_TH1, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9260_2ac_cfg, iwl9162_160_name),
 	_IWL_DEV_INFO(0x271B, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH1, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_TH1, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9260_2ac_cfg, iwl9162_name),
 
 	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9260_2ac_cfg, iwl9260_160_name),
 	_IWL_DEV_INFO(0x2526, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_TH, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_TH, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9260_2ac_cfg, iwl9260_name),
 
@@ -782,176 +783,176 @@ static const struct iwl_dev_info iwl_dev_info_table[] = {
 	/* Qu B step */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_b0_jf_b0_cfg, iwl9461_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_b0_jf_b0_cfg, iwl9461_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_b0_jf_b0_cfg, iwl9462_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_b0_jf_b0_cfg, iwl9462_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_b0_jf_b0_cfg, iwl9560_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_b0_jf_b0_cfg, iwl9560_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1551,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_b0_jf_b0_cfg, iwl9560_killer_1550s_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1552,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_b0_jf_b0_cfg, iwl9560_killer_1550i_name),
 
 	/* Qu C step */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_c0_jf_b0_cfg, iwl9461_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_c0_jf_b0_cfg, iwl9461_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_c0_jf_b0_cfg, iwl9462_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_c0_jf_b0_cfg, iwl9462_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_c0_jf_b0_cfg, iwl9560_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_c0_jf_b0_cfg, iwl9560_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1551,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_c0_jf_b0_cfg, iwl9560_killer_1550s_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1552,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qu_c0_jf_b0_cfg, iwl9560_killer_1550i_name),
 
 	/* QuZ */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_quz_a0_jf_b0_cfg, iwl9461_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_quz_a0_jf_b0_cfg, iwl9461_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_quz_a0_jf_b0_cfg, iwl9462_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_quz_a0_jf_b0_cfg, iwl9462_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_quz_a0_jf_b0_cfg, iwl9560_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_quz_a0_jf_b0_cfg, iwl9560_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1551,
 		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_quz_a0_jf_b0_cfg, iwl9560_killer_1550s_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1552,
 		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_quz_a0_jf_b0_cfg, iwl9560_killer_1550i_name),
 
 	/* QnJ */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qnj_b0_jf_b0_cfg, iwl9461_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qnj_b0_jf_b0_cfg, iwl9461_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qnj_b0_jf_b0_cfg, iwl9462_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qnj_b0_jf_b0_cfg, iwl9462_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qnj_b0_jf_b0_cfg, iwl9560_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qnj_b0_jf_b0_cfg, iwl9560_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1551,
 		      IWL_CFG_MAC_TYPE_QNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qnj_b0_jf_b0_cfg, iwl9560_killer_1550s_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, 0x1552,
 		      IWL_CFG_MAC_TYPE_QNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl9560_qnj_b0_jf_b0_cfg, iwl9560_killer_1550i_name),
 
@@ -959,367 +960,408 @@ static const struct iwl_dev_info iwl_dev_info_table[] = {
 	/* Qu B step */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_qu_b0_hr1_b0, iwl_ax101_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_qu_b0_hr_b0, iwl_ax203_name),
 
 	/* Qu C step */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_qu_c0_hr1_b0, iwl_ax101_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_qu_c0_hr_b0, iwl_ax203_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QU, SILICON_C_STEP,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_qu_c0_hr_b0, iwl_ax201_name),
 
 	/* QuZ */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QUZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_quz_a0_hr1_b0, iwl_ax101_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QUZ, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_quz_a0_hr_b0, iwl_ax203_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_QUZ, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      iwl_cfg_quz_a0_hr_b0, iwl_ax201_name),
 
 /* QnJ with Hr */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_QNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_qnj_b0_hr_b0_cfg, iwl_ax201_name),
 
 /* SnJ with Jf */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_snj_a0_jf_b0, iwl9461_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_snj_a0_jf_b0, iwl9461_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_snj_a0_jf_b0, iwl9462_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_snj_a0_jf_b0, iwl9462_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_snj_a0_jf_b0, iwl9560_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_snj_a0_jf_b0, iwl9560_name),
 
 /* SnJ with Hr */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_snj_hr_b0, iwl_ax101_name),
 
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_snj_hr_b0, iwl_ax201_name),
 
 /* Ma */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_MA, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_MA, SILICON_A_STEP,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_ma_a0_hr_b0, iwl_ax201_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_MA, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_MA, SILICON_A_STEP,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_ma_a0_gf_a0, iwl_ax211_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_MA, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_MA, SILICON_A_STEP,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_ANY,
 		      iwl_cfg_ma_a0_gf4_a0, iwl_ax211_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_MA, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_MR, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_MA, SILICON_A_STEP,
+		      IWL_CFG_RF_TYPE_MR, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_ma_a0_mr_a0, iwl_ax221_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_MA, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_MA, SILICON_A_STEP,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_ma_a0_fm_a0, iwl_ax231_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_MR, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_MR, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_snj_a0_mr_a0, iwl_ax221_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_MA, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      iwl_cfg_ma_b0_hr_b0, iwl_ax201_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_MA, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      iwl_cfg_ma_b0_gf_a0, iwl_ax211_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_MA, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_ANY,
+		      iwl_cfg_ma_b0_gf4_a0, iwl_ax211_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_MA, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_MR, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      iwl_cfg_ma_b0_mr_a0, iwl_ax221_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_MA, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      iwl_cfg_ma_b0_fm_a0, iwl_ax231_name),
 
 /* So with Hr */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_so_a0_hr_a0, iwl_ax203_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_so_a0_hr_a0, iwl_ax101_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_so_a0_hr_a0, iwl_ax201_name),
 
 /* So-F with Hr */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_so_a0_hr_a0, iwl_ax203_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY,
-		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_NO_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_so_a0_hr_a0, iwl_ax101_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_so_a0_hr_a0, iwl_ax201_name),
 
 /* So-F with Gf */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_ANY,
 		      iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_name),
 
 /* Bz */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      iwl_cfg_bz_a0_hr_a0, iwl_bz_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_bz_a0_hr_b0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_bz_a0_gf_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_ANY,
 		      iwl_cfg_bz_a0_gf4_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_MR, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_MR, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_bz_a0_mr_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_BZ, SILICON_A_STEP,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, SILICON_A_STEP,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_bz_a0_fm_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, SILICON_A_STEP,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_NO_JACKET,
 		      iwl_cfg_bz_a0_fm4_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, SILICON_B_STEP,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_IS_JACKET,
+		      iwl_cfg_bz_a0_fm_b0, iwl_bz_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, SILICON_B_STEP,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_IS_JACKET,
+		      iwl_cfg_bz_a0_fm4_b0, iwl_bz_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_GL, SILICON_A_STEP,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, SILICON_A_STEP,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_NO_JACKET,
 		      iwl_cfg_gl_a0_fm_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, SILICON_B_STEP,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_NO_JACKET,
 		      iwl_cfg_gl_b0_fm_b0, iwl_bz_name),
 
 /* BZ Z step */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_BZ, SILICON_Z_STEP,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_bz_z0_gf_a0, iwl_bz_name),
 
 /* BNJ */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_GL, SILICON_A_STEP,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, SILICON_A_STEP,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_IS_JACKET,
 		      iwl_cfg_bnj_a0_fm_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, SILICON_B_STEP,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_IS_JACKET,
 		      iwl_cfg_bnj_b0_fm_b0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_GL, SILICON_A_STEP,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, SILICON_A_STEP,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_IS_JACKET,
 		      iwl_cfg_bnj_a0_fm4_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, SILICON_B_STEP,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_IS_JACKET,
+		      iwl_cfg_bnj_b0_fm4_b0, iwl_bz_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_IS_JACKET,
 		      iwl_cfg_bnj_a0_gf_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
-		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_IS_JACKET,
-		      iwl_cfg_bnj_a0_gf4_a0, iwl_bz_name),
-	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_IS_JACKET,
-		      iwl_cfg_bnj_a0_hr_b0, iwl_bz_name),
-
-/* SoF with JF2 */
+		      iwl_cfg_bnj_b0_gf_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9560_160_name),
+		      IWL_CFG_MAC_TYPE_GL, SILICON_A_STEP,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_IS_JACKET,
+		      iwl_cfg_bnj_a0_gf4_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9560_name),
-
-/* SoF with JF */
+		      IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_IS_JACKET,
+		      iwl_cfg_bnj_b0_gf4_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9461_160_name),
+		      IWL_CFG_MAC_TYPE_GL, SILICON_A_STEP,
+		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      iwl_cfg_bnj_a0_hr_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
-		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9462_160_name),
+		      IWL_CFG_MAC_TYPE_GL, SILICON_A_STEP,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      iwl_cfg_bnj_a0_hr_b0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9461_name),
+		      IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_HR1, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      iwl_cfg_bnj_b0_hr_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
-		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
-		      iwlax210_2ax_cfg_so_jf_b0, iwl9462_name),
+		      IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_HR2, IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
+		      iwl_cfg_bnj_b0_hr_b0, iwl_bz_name),
 
 /* SoF with JF2 */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9560_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9560_name),
 
 /* SoF with JF */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9461_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9462_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9461_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9462_name),
 
 /* So with GF */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax211_2ax_cfg_so_gf_a0, iwl_ax211_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_GF, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_ANY,
 		      iwlax411_2ax_cfg_so_gf4_a0, iwl_ax411_name),
 
 /* So with JF2 */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9560_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF,
+		      IWL_CFG_RF_TYPE_JF2, IWL_CFG_RF_ID_JF, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9560_name),
 
 /* So with JF */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9461_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9462_160_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9461_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV,
+		      IWL_CFG_RF_TYPE_JF1, IWL_CFG_RF_ID_JF1_DIV, IWL_CFG_ANY,
 		      IWL_CFG_NO_160, IWL_CFG_CORES_BT, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwlax210_2ax_cfg_so_jf_b0, iwl9462_name),
 
@@ -1327,22 +1369,22 @@ static const struct iwl_dev_info iwl_dev_info_table[] = {
 /* For now we use the same FW as MR, but this will change in the future. */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SO, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_MS, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_MS, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_so_a0_ms_a0, iwl_ax204_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SOF, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_MS, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_MS, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_so_a0_ms_a0, iwl_ax204_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_MA, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_MS, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_MS, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_ma_a0_ms_a0, iwl_ax204_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_SNJ, IWL_CFG_ANY,
-		      IWL_CFG_RF_TYPE_MS, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_MS, IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_160, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_snj_a0_ms_a0, iwl_ax204_name)
 
@@ -1377,8 +1419,16 @@ static int get_crf_id(struct iwl_trans *iwl_trans)
 	/* Read crf info */
 	iwl_trans->hw_crf_id = iwl_read_prph_no_grab(iwl_trans, sd_reg_ver_addr);
 
+	/* Read cnv info */
+	iwl_trans->hw_cnv_id =
+		iwl_read_prph_no_grab(iwl_trans, CNVI_AUX_MISC_CHIP);
+
 	/* Read cdb info (also contains the jacket info if needed in the future */
-	iwl_trans->hw_cdb_id = iwl_read_umac_prph_no_grab(iwl_trans, WFPM_OTP_CFG1_ADDR);
+	iwl_trans->hw_wfpm_id =
+		iwl_read_umac_prph_no_grab(iwl_trans, WFPM_OTP_CFG1_ADDR);
+	IWL_INFO(iwl_trans, "Detected crf-id 0x%x, cnv-id 0x%x wfpm id 0x%x\n",
+		 iwl_trans->hw_crf_id, iwl_trans->hw_cnv_id,
+		 iwl_trans->hw_wfpm_id);
 
 	iwl_trans_release_nic_access(iwl_trans);
 
@@ -1394,7 +1444,11 @@ static int map_crf_id(struct iwl_trans *iwl_trans)
 {
 	int ret = 0;
 	u32 val = iwl_trans->hw_crf_id;
-	u32 cdb = iwl_trans->hw_cdb_id;
+	u32 step_id = REG_CRF_ID_STEP(val);
+	u32 slave_id = REG_CRF_ID_SLAVE(val);
+	u32 jacket_id_cnv  = REG_CRF_ID_SLAVE(iwl_trans->hw_cnv_id);
+	u32 jacket_id_wfpm  = WFPM_OTP_CFG1_IS_JACKET(iwl_trans->hw_wfpm_id);
+	u32 cdb_id_wfpm  = WFPM_OTP_CFG1_IS_CDB(iwl_trans->hw_wfpm_id);
 
 	/* Map between crf id to rf id */
 	switch (REG_CRF_ID_TYPE(val)) {
@@ -1404,9 +1458,12 @@ static int map_crf_id(struct iwl_trans *iwl_trans)
 	case REG_CRF_ID_TYPE_JF_2:
 		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_JF2 << 12);
 		break;
-	case REG_CRF_ID_TYPE_HR_NONE_CDB:
+	case REG_CRF_ID_TYPE_HR_NONE_CDB_1X1:
 		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_HR1 << 12);
 		break;
+	case REG_CRF_ID_TYPE_HR_NONE_CDB:
+		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_HR2 << 12);
+		break;
 	case REG_CRF_ID_TYPE_HR_CDB:
 		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_HR2 << 12);
 		break;
@@ -1416,27 +1473,43 @@ static int map_crf_id(struct iwl_trans *iwl_trans)
 	case REG_CRF_ID_TYPE_MR:
 		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_MR << 12);
 		break;
-		case REG_CRF_ID_TYPE_FM:
-			iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_FM << 12);
-			break;
+	case REG_CRF_ID_TYPE_FM:
+	case REG_CRF_ID_TYPE_FMI:
+	case REG_CRF_ID_TYPE_FMR:
+		iwl_trans->hw_rf_id = (IWL_CFG_RF_TYPE_FM << 12);
+		break;
 	default:
 		ret = -EIO;
 		IWL_ERR(iwl_trans,
-			"Can find a correct rfid for crf id 0x%x\n",
+			"Can't find a correct rfid for crf id 0x%x\n",
 			REG_CRF_ID_TYPE(val));
 		goto out;
 
 	}
 
+	/* Set Step-id */
+	iwl_trans->hw_rf_id |= (step_id << 8);
+
 	/* Set CDB capabilities */
-	if (cdb & BIT(4)) {
+	if (cdb_id_wfpm || slave_id) {
 		iwl_trans->hw_rf_id += BIT(28);
 		IWL_INFO(iwl_trans, "Adding cdb to rf id\n");
 	}
 
-	IWL_INFO(iwl_trans, "Detected RF 0x%x from crf id 0x%x\n",
-		 iwl_trans->hw_rf_id, REG_CRF_ID_TYPE(val));
+	/* Set Jacket capabilities */
+	if (jacket_id_wfpm || jacket_id_cnv) {
+		iwl_trans->hw_rf_id += BIT(29);
+		IWL_INFO(iwl_trans, "Adding jacket to rf id\n");
+	}
 
+	IWL_INFO(iwl_trans,
+		 "Detected rf-type 0x%x step-id 0x%x slave-id 0x%x from crf id 0x%x\n",
+		 REG_CRF_ID_TYPE(val), step_id, slave_id, iwl_trans->hw_rf_id);
+	IWL_INFO(iwl_trans,
+		 "Detected cdb-id 0x%x jacket-id 0x%x from wfpm id 0x%x\n",
+		 cdb_id_wfpm, jacket_id_wfpm, iwl_trans->hw_wfpm_id);
+	IWL_INFO(iwl_trans, "Detected jacket-id 0x%x from cnvi id 0x%x\n",
+		 jacket_id_cnv, iwl_trans->hw_cnv_id);
 
 out:
 	return ret;
@@ -1447,8 +1520,8 @@ out:
 
 static const struct iwl_dev_info *
 iwl_pci_find_dev_info(u16 device, u16 subsystem_device,
-		      u16 mac_type, u8 mac_step,
-		      u16 rf_type, u8 cdb, u8 jacket, u8 rf_id, u8 no_160, u8 cores)
+		      u16 mac_type, u8 mac_step, u16 rf_type, u8 cdb,
+		      u8 jacket, u8 rf_id, u8 no_160, u8 cores, u8 rf_step)
 {
 	int num_devices = ARRAY_SIZE(iwl_dev_info_table);
 	int i;
@@ -1499,6 +1572,10 @@ iwl_pci_find_dev_info(u16 device, u16 subsystem_device,
 		    dev_info->cores != cores)
 			continue;
 
+		if (dev_info->rf_step != (u8)IWL_CFG_ANY &&
+		    dev_info->rf_step != rf_step)
+			continue;
+
 		return dev_info;
 	}
 
@@ -1570,6 +1647,10 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto out_free_trans;
 	}
 
+	IWL_INFO(iwl_trans, "PCI dev %04x/%04x, rev=0x%x, rfid=0x%x\n",
+		 pdev->device, pdev->subsystem_device,
+		 iwl_trans->hw_rev, iwl_trans->hw_rf_id);
+
 	dev_info = iwl_pci_find_dev_info(pdev->device, pdev->subsystem_device,
 					 CSR_HW_REV_TYPE(iwl_trans->hw_rev),
 					 iwl_trans->hw_rev_step,
@@ -1578,8 +1659,8 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 					 CSR_HW_RFID_IS_JACKET(iwl_trans->hw_rf_id),
 					 IWL_SUBDEVICE_RF_ID(pdev->subsystem_device),
 					 IWL_SUBDEVICE_NO_160(pdev->subsystem_device),
-					 IWL_SUBDEVICE_CORES(pdev->subsystem_device));
-
+					 IWL_SUBDEVICE_CORES(pdev->subsystem_device),
+					 CSR_HW_RFID_STEP(iwl_trans->hw_rf_id));
 	if (dev_info) {
 		iwl_trans->cfg = dev_info->cfg;
 		iwl_trans->name = dev_info->name;
@@ -1699,6 +1780,9 @@ static void iwl_pci_remove(struct pci_dev *pdev)
 {
 	struct iwl_trans *trans = pci_get_drvdata(pdev);
 
+	if (!trans)
+		return;
+
 	iwl_drv_stop(trans->drv);
 
 	iwl_trans_pcie_free(trans);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
index f7e4f868363d..69b95ad5993b 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
@@ -497,6 +497,7 @@ int iwl_pcie_rx_stop(struct iwl_trans *trans);
 void iwl_pcie_rx_free(struct iwl_trans *trans);
 void iwl_pcie_free_rbs_pool(struct iwl_trans *trans);
 void iwl_pcie_rx_init_rxb_lists(struct iwl_rxq *rxq);
+void iwl_pcie_rx_napi_sync(struct iwl_trans *trans);
 void iwl_pcie_rxq_alloc_rbs(struct iwl_trans *trans, gfp_t priority,
 			    struct iwl_rxq *rxq);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
index 9c9f87fe8377..0d7890f99a5f 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2003-2014, 2018-2022 Intel Corporation
+ * Copyright (C) 2003-2014, 2018-2023 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -1053,6 +1053,22 @@ static int iwl_pcie_napi_poll_msix(struct napi_struct *napi, int budget)
 	return ret;
 }
 
+void iwl_pcie_rx_napi_sync(struct iwl_trans *trans)
+{
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+	int i;
+
+	if (unlikely(!trans_pcie->rxq))
+		return;
+
+	for (i = 0; i < trans->num_rx_queues; i++) {
+		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
+
+		if (rxq && rxq->napi.poll)
+			napi_synchronize(&rxq->napi);
+	}
+}
+
 static int _iwl_pcie_rx_init(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
index 94f40c4d2421..73b395841ca8 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2021 Intel Corporation
+ * Copyright (C) 2018-2022 Intel Corporation
  */
 #include "iwl-trans.h"
 #include "iwl-prph.h"
@@ -156,6 +156,7 @@ void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans)
 	if (test_and_clear_bit(STATUS_DEVICE_ENABLED, &trans->status)) {
 		IWL_DEBUG_INFO(trans,
 			       "DEVICE_ENABLED bit was set and is now cleared\n");
+		iwl_pcie_rx_napi_sync(trans);
 		iwl_txq_gen2_tx_free(trans);
 		iwl_pcie_rx_stop(trans);
 	}
@@ -277,6 +278,9 @@ static void iwl_pcie_get_rf_name(struct iwl_trans *trans)
 	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_HRCDB):
 		pos = scnprintf(buf, buflen, "HRCDB");
 		break;
+	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_MS):
+		pos = scnprintf(buf, buflen, "MS");
+		break;
 	default:
 		return;
 	}
@@ -347,7 +351,7 @@ void iwl_trans_pcie_gen2_fw_alive(struct iwl_trans *trans, u32 scd_addr)
 	mutex_unlock(&trans_pcie->mutex);
 }
 
-static void iwl_pcie_set_ltr(struct iwl_trans *trans)
+static bool iwl_pcie_set_ltr(struct iwl_trans *trans)
 {
 	u32 ltr_val = CSR_LTR_LONG_VAL_AD_NO_SNOOP_REQ |
 		      u32_encode_bits(CSR_LTR_LONG_VAL_AD_SCALE_USEC,
@@ -368,18 +372,77 @@ static void iwl_pcie_set_ltr(struct iwl_trans *trans)
 	     trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_22000) &&
 	    !trans->trans_cfg->integrated) {
 		iwl_write32(trans, CSR_LTR_LONG_VAL_AD, ltr_val);
-	} else if (trans->trans_cfg->integrated &&
-		   trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_22000) {
+		return true;
+	}
+
+	if (trans->trans_cfg->integrated &&
+	    trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_22000) {
 		iwl_write_prph(trans, HPM_MAC_LTR_CSR, HPM_MAC_LRT_ENABLE_ALL);
 		iwl_write_prph(trans, HPM_UMAC_LTR, ltr_val);
+		return true;
 	}
+
+	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_AX210) {
+		/* First clear the interrupt, just in case */
+		iwl_write32(trans, CSR_MSIX_HW_INT_CAUSES_AD,
+			    MSIX_HW_INT_CAUSES_REG_IML);
+		/* In this case, unfortunately the same ROM bug exists in the
+		 * device (not setting LTR correctly), but we don't have control
+		 * over the settings from the host due to some hardware security
+		 * features. The only workaround we've been able to come up with
+		 * so far is to try to keep the CPU and device busy by polling
+		 * it and the IML (image loader) completed interrupt.
+		 */
+		return false;
+	}
+
+	/* nothing needs to be done on other devices */
+	return true;
+}
+
+static void iwl_pcie_spin_for_iml(struct iwl_trans *trans)
+{
+/* in practice, this seems to complete in around 20-30ms at most, wait 100 */
+#define IML_WAIT_TIMEOUT	(HZ / 10)
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+	unsigned long end_time = jiffies + IML_WAIT_TIMEOUT;
+	u32 value, loops = 0;
+	bool irq = false;
+
+	if (WARN_ON(!trans_pcie->iml))
+		return;
+
+	value = iwl_read32(trans, CSR_LTR_LAST_MSG);
+	IWL_DEBUG_INFO(trans, "Polling for IML load - CSR_LTR_LAST_MSG=0x%x\n",
+		       value);
+
+	while (time_before(jiffies, end_time)) {
+		if (iwl_read32(trans, CSR_MSIX_HW_INT_CAUSES_AD) &
+				MSIX_HW_INT_CAUSES_REG_IML) {
+			irq = true;
+			break;
+		}
+		/* Keep the CPU and device busy. */
+		value = iwl_read32(trans, CSR_LTR_LAST_MSG);
+		loops++;
+	}
+
+	IWL_DEBUG_INFO(trans,
+		       "Polled for IML load: irq=%d, loops=%d, CSR_LTR_LAST_MSG=0x%x\n",
+		       irq, loops, value);
+
+	/* We don't fail here even if we timed out - maybe we get lucky and the
+	 * interrupt comes in later (and we get alive from firmware) and then
+	 * we're all happy - but if not we'll fail on alive timeout or get some
+	 * other error out.
+	 */
 }
 
 int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
 				 const struct fw_img *fw, bool run_in_rfkill)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	bool hw_rfkill;
+	bool hw_rfkill, keep_ram_busy;
 	int ret;
 
 	/* This may fail if AMT took ownership of the device */
@@ -440,7 +503,7 @@ int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
 	if (ret)
 		goto out;
 
-	iwl_pcie_set_ltr(trans);
+	keep_ram_busy = !iwl_pcie_set_ltr(trans);
 
 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
 		iwl_write32(trans, CSR_FUNC_SCRATCH, CSR_FUNC_SCRATCH_INIT_VALUE);
@@ -452,6 +515,9 @@ int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
 		iwl_write_prph(trans, UREG_CPU_INIT_RUN, 1);
 	}
 
+	if (keep_ram_busy)
+		iwl_pcie_spin_for_iml(trans);
+
 	/* re-check RF-Kill state since we may have missed the interrupt */
 	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
 	if (hw_rfkill && !run_in_rfkill)
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
index 0a9af1ad1f20..b281850fbf7a 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
@@ -599,7 +599,6 @@ static int iwl_pcie_set_hw_ready(struct iwl_trans *trans)
 int iwl_pcie_prepare_card_hw(struct iwl_trans *trans)
 {
 	int ret;
-	int t = 0;
 	int iter;
 
 	IWL_DEBUG_INFO(trans, "iwl_trans_prepare_card_hw enter\n");
@@ -616,6 +615,8 @@ int iwl_pcie_prepare_card_hw(struct iwl_trans *trans)
 	usleep_range(1000, 2000);
 
 	for (iter = 0; iter < 10; iter++) {
+		int t = 0;
+
 		/* If HW is not ready, prepare the conditions to check again */
 		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
 			    CSR_HW_IF_CONFIG_REG_PREPARE);
@@ -1260,6 +1261,7 @@ static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans)
 	if (test_and_clear_bit(STATUS_DEVICE_ENABLED, &trans->status)) {
 		IWL_DEBUG_INFO(trans,
 			       "DEVICE_ENABLED bit was set and is now cleared\n");
+		iwl_pcie_rx_napi_sync(trans);
 		iwl_pcie_tx_stop(trans);
 		iwl_pcie_rx_stop(trans);
 
@@ -1522,19 +1524,16 @@ static int iwl_pcie_d3_handshake(struct iwl_trans *trans, bool suspend)
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	int ret;
 
-	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_AX210) {
+	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_AX210)
 		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
 				    suspend ? UREG_DOORBELL_TO_ISR6_SUSPEND :
 					      UREG_DOORBELL_TO_ISR6_RESUME);
-	} else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+	else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
 		iwl_write32(trans, CSR_IPC_SLEEP_CONTROL,
 			    suspend ? CSR_IPC_SLEEP_CONTROL_SUSPEND :
 				      CSR_IPC_SLEEP_CONTROL_RESUME);
-		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
-				    UREG_DOORBELL_TO_ISR6_SLEEP_CTRL);
-	} else {
+	else
 		return 0;
-	}
 
 	ret = wait_event_timeout(trans_pcie->sx_waitq,
 				 trans_pcie->sx_complete, 2 * HZ);
@@ -2863,7 +2862,7 @@ static bool iwl_write_to_user_buf(char __user *user_buf, ssize_t count,
 				  void *buf, ssize_t *size,
 				  ssize_t *bytes_copied)
 {
-	int buf_size_left = count - *bytes_copied;
+	ssize_t buf_size_left = count - *bytes_copied;
 
 	buf_size_left = buf_size_left - (buf_size_left % sizeof(u32));
 	if (*size > buf_size_left)
diff --git a/drivers/net/wireless/intel/iwlwifi/queue/tx.c b/drivers/net/wireless/intel/iwlwifi/queue/tx.c
index 726185d6fab8..d1c39c214f95 100644
--- a/drivers/net/wireless/intel/iwlwifi/queue/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/queue/tx.c
@@ -1554,14 +1554,18 @@ void iwl_txq_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 		     struct sk_buff_head *skbs)
 {
 	struct iwl_txq *txq = trans->txqs.txq[txq_id];
-	int tfd_num = iwl_txq_get_cmd_index(txq, ssn);
-	int read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr);
-	int last_to_free;
+	int tfd_num, read_ptr, last_to_free;
 
 	/* This function is not meant to release cmd queue*/
 	if (WARN_ON(txq_id == trans->txqs.cmd.q_id))
 		return;
 
+	if (WARN_ON(!txq))
+		return;
+
+	tfd_num = iwl_txq_get_cmd_index(txq, ssn);
+	read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr);
+
 	spin_lock_bh(&txq->lock);
 
 	if (!test_bit(txq_id, trans->txqs.queue_used)) {
diff --git a/drivers/net/wireless/legacy/Kconfig b/drivers/net/wireless/legacy/Kconfig
new file mode 100644
index 000000000000..3a5275941212
--- /dev/null
+++ b/drivers/net/wireless/legacy/Kconfig
@@ -0,0 +1,55 @@
+config PCMCIA_RAYCS
+	tristate "Aviator/Raytheon 2.4GHz wireless support"
+	depends on PCMCIA
+	select WIRELESS_EXT
+	select WEXT_SPY
+	select WEXT_PRIV
+	help
+	  Say Y here if you intend to attach an Aviator/Raytheon PCMCIA
+	  (PC-card) wireless Ethernet networking card to your computer.
+	  Please read the file
+	  <file:Documentation/networking/device_drivers/wifi/ray_cs.rst> for
+	  details.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called ray_cs.  If unsure, say N.
+
+config PCMCIA_WL3501
+	tristate "Planet WL3501 PCMCIA cards"
+	depends on CFG80211 && PCMCIA
+	select WIRELESS_EXT
+	select WEXT_SPY
+	help
+	  A driver for WL3501 PCMCIA 802.11 wireless cards made by Planet.
+	  It has basic support for Linux wireless extensions and initial
+	  micro support for ethtool.
+
+config USB_NET_RNDIS_WLAN
+	tristate "Wireless RNDIS USB support"
+	depends on USB
+	depends on CFG80211
+	select USB_NET_DRIVERS
+	select USB_USBNET
+	select USB_NET_CDCETHER
+	select USB_NET_RNDIS_HOST
+	help
+	  This is a driver for wireless RNDIS devices.
+	  These are USB based adapters found in devices such as:
+
+	  Buffalo WLI-U2-KG125S
+	  U.S. Robotics USR5421
+	  Belkin F5D7051
+	  Linksys WUSB54GSv2
+	  Linksys WUSB54GSC
+	  Asus WL169gE
+	  Eminent EM4045
+	  BT Voyager 1055
+	  Linksys WUSB54GSv1
+	  U.S. Robotics USR5420
+	  BUFFALO WLI-USB-G54
+
+	  All of these devices are based on Broadcom 4320 chip which is the
+	  only wireless RNDIS chip known to date.
+
+	  If you choose to build a module, it'll be called rndis_wlan.
+
diff --git a/drivers/net/wireless/legacy/Makefile b/drivers/net/wireless/legacy/Makefile
new file mode 100644
index 000000000000..36878f080bfc
--- /dev/null
+++ b/drivers/net/wireless/legacy/Makefile
@@ -0,0 +1,6 @@
+# 16-bit wireless PCMCIA client drivers
+obj-$(CONFIG_PCMCIA_RAYCS)	+= ray_cs.o
+obj-$(CONFIG_PCMCIA_WL3501)	+= wl3501_cs.o
+
+obj-$(CONFIG_USB_NET_RNDIS_WLAN)	+= rndis_wlan.o
+
diff --git a/drivers/net/wireless/ray_cs.c b/drivers/net/wireless/legacy/ray_cs.c
index 1f57a0055bbd..1f57a0055bbd 100644
--- a/drivers/net/wireless/ray_cs.c
+++ b/drivers/net/wireless/legacy/ray_cs.c
diff --git a/drivers/net/wireless/ray_cs.h b/drivers/net/wireless/legacy/ray_cs.h
index 0609d8625019..0609d8625019 100644
--- a/drivers/net/wireless/ray_cs.h
+++ b/drivers/net/wireless/legacy/ray_cs.h
diff --git a/drivers/net/wireless/rayctl.h b/drivers/net/wireless/legacy/rayctl.h
index 2b0f332043d7..2b0f332043d7 100644
--- a/drivers/net/wireless/rayctl.h
+++ b/drivers/net/wireless/legacy/rayctl.h
diff --git a/drivers/net/wireless/rndis_wlan.c b/drivers/net/wireless/legacy/rndis_wlan.c
index bf72e5fd39cf..712038d46bdb 100644
--- a/drivers/net/wireless/rndis_wlan.c
+++ b/drivers/net/wireless/legacy/rndis_wlan.c
@@ -209,7 +209,7 @@ struct ndis_80211_status_indication {
 	union {
 		__le32					media_stream_mode;
 		__le32					radio_status;
-		struct ndis_80211_auth_request		auth_request[0];
+		DECLARE_FLEX_ARRAY(struct ndis_80211_auth_request, auth_request);
 		struct ndis_80211_pmkid_cand_list	cand_list;
 	} u;
 } __packed;
@@ -1972,7 +1972,7 @@ static bool rndis_bss_info_update(struct usbnet *usbdev,
 
 	if (bssid_len < sizeof(struct ndis_80211_bssid_ex) +
 			sizeof(struct ndis_80211_fixed_ies))
-		return NULL;
+		return false;
 
 	fixed = (struct ndis_80211_fixed_ies *)bssid->ies;
 
@@ -1981,13 +1981,13 @@ static bool rndis_bss_info_update(struct usbnet *usbdev,
 					(int)le32_to_cpu(bssid->ie_length));
 	ie_len -= sizeof(struct ndis_80211_fixed_ies);
 	if (ie_len < 0)
-		return NULL;
+		return false;
 
 	/* extract data for cfg80211_inform_bss */
 	channel = ieee80211_get_channel(priv->wdev.wiphy,
 			KHZ_TO_MHZ(le32_to_cpu(bssid->config.ds_config)));
 	if (!channel)
-		return NULL;
+		return false;
 
 	signal = level_to_qual(le32_to_cpu(bssid->rssi));
 	timestamp = le64_to_cpu(*(__le64 *)fixed->timestamp);
diff --git a/drivers/net/wireless/wl3501.h b/drivers/net/wireless/legacy/wl3501.h
index 91f276dd22a1..91f276dd22a1 100644
--- a/drivers/net/wireless/wl3501.h
+++ b/drivers/net/wireless/legacy/wl3501.h
diff --git a/drivers/net/wireless/wl3501_cs.c b/drivers/net/wireless/legacy/wl3501_cs.c
index 7fb2f9513476..7fb2f9513476 100644
--- a/drivers/net/wireless/wl3501_cs.c
+++ b/drivers/net/wireless/legacy/wl3501_cs.c
diff --git a/drivers/net/wireless/marvell/libertas/if_spi.c b/drivers/net/wireless/marvell/libertas/if_spi.c
index ff1c7ec8c450..1225fc0e3352 100644
--- a/drivers/net/wireless/marvell/libertas/if_spi.c
+++ b/drivers/net/wireless/marvell/libertas/if_spi.c
@@ -1051,7 +1051,7 @@ static int if_spi_init_card(struct if_spi_card *card)
 				"spi->max_speed_hz=%d\n",
 				card->card_id, card->card_rev,
 				card->spi->master->bus_num,
-				card->spi->chip_select,
+				spi_get_chipselect(card->spi, 0),
 				card->spi->max_speed_hz);
 		err = if_spi_prog_helper_firmware(card, helper);
 		if (err)
diff --git a/drivers/net/wireless/marvell/mwifiex/11h.c b/drivers/net/wireless/marvell/mwifiex/11h.c
index b0c40a776a2e..2ea03725f188 100644
--- a/drivers/net/wireless/marvell/mwifiex/11h.c
+++ b/drivers/net/wireless/marvell/mwifiex/11h.c
@@ -195,7 +195,6 @@ int mwifiex_11h_handle_chanrpt_ready(struct mwifiex_private *priv,
 {
 	struct host_cmd_ds_chan_rpt_event *rpt_event;
 	struct mwifiex_ie_types_chan_rpt_data *rpt;
-	u8 *evt_buf;
 	u16 event_len, tlv_len;
 
 	rpt_event = (void *)(skb->data + sizeof(u32));
@@ -208,8 +207,6 @@ int mwifiex_11h_handle_chanrpt_ready(struct mwifiex_private *priv,
 		return -1;
 	}
 
-	evt_buf = (void *)&rpt_event->tlvbuf;
-
 	while (event_len >= sizeof(struct mwifiex_ie_types_header)) {
 		rpt = (void *)&rpt_event->tlvbuf;
 		tlv_len = le16_to_cpu(rpt->header.len);
@@ -231,7 +228,6 @@ int mwifiex_11h_handle_chanrpt_ready(struct mwifiex_private *priv,
 			break;
 		}
 
-		evt_buf += (tlv_len + sizeof(rpt->header));
 		event_len -= (tlv_len + sizeof(rpt->header));
 	}
 
diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c
index da281cd1d36f..465190ebaf1c 100644
--- a/drivers/net/wireless/mediatek/mt76/dma.c
+++ b/drivers/net/wireless/mediatek/mt76/dma.c
@@ -402,8 +402,8 @@ mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
 		*info = le32_to_cpu(desc->info);
 
 	if (mt76_queue_is_wed_rx(q)) {
-		u32 token = FIELD_GET(MT_DMA_CTL_TOKEN,
-				      le32_to_cpu(desc->buf1));
+		u32 buf1 = le32_to_cpu(desc->buf1);
+		u32 token = FIELD_GET(MT_DMA_CTL_TOKEN, buf1);
 		struct mt76_txwi_cache *t = mt76_rx_token_release(dev, token);
 
 		if (!t)
@@ -424,6 +424,8 @@ mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
 
 			*drop = !!(ctrl & (MT_DMA_CTL_TO_HOST_A |
 					   MT_DMA_CTL_DROP));
+
+			*drop |= !!(buf1 & MT_DMA_CTL_WO_DROP);
 		}
 	} else {
 		buf = e->buf;
@@ -576,7 +578,9 @@ free:
 free_skb:
 	status.skb = tx_info.skb;
 	hw = mt76_tx_status_get_hw(dev, tx_info.skb);
+	spin_lock_bh(&dev->rx_lock);
 	ieee80211_tx_status_ext(hw, &status);
+	spin_unlock_bh(&dev->rx_lock);
 
 	return ret;
 }
@@ -849,7 +853,7 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
 		    !(dev->drv->rx_check(dev, data, len)))
 			goto free_frag;
 
-		skb = build_skb(data, q->buf_size);
+		skb = napi_build_skb(data, q->buf_size);
 		if (!skb)
 			goto free_frag;
 
diff --git a/drivers/net/wireless/mediatek/mt76/dma.h b/drivers/net/wireless/mediatek/mt76/dma.h
index 4b9bc7f462b8..1b090d78cd05 100644
--- a/drivers/net/wireless/mediatek/mt76/dma.h
+++ b/drivers/net/wireless/mediatek/mt76/dma.h
@@ -19,6 +19,7 @@
 #define MT_DMA_CTL_TO_HOST_A		BIT(12)
 #define MT_DMA_CTL_DROP			BIT(14)
 #define MT_DMA_CTL_TOKEN		GENMASK(31, 16)
+#define MT_DMA_CTL_WO_DROP		BIT(8)
 
 #define MT_DMA_PPE_CPU_REASON		GENMASK(15, 11)
 #define MT_DMA_PPE_ENTRY		GENMASK(30, 16)
diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c
index 34abf70f44af..467afef98ba2 100644
--- a/drivers/net/wireless/mediatek/mt76/mac80211.c
+++ b/drivers/net/wireless/mediatek/mt76/mac80211.c
@@ -418,7 +418,8 @@ mt76_phy_init(struct mt76_phy *phy, struct ieee80211_hw *hw)
 	SET_IEEE80211_DEV(hw, dev->dev);
 	SET_IEEE80211_PERM_ADDR(hw, phy->macaddr);
 
-	wiphy->features |= NL80211_FEATURE_ACTIVE_MONITOR;
+	wiphy->features |= NL80211_FEATURE_ACTIVE_MONITOR |
+			   NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE;
 	wiphy->flags |= WIPHY_FLAG_HAS_CHANNEL_SWITCH |
 			WIPHY_FLAG_SUPPORTS_TDLS |
 			WIPHY_FLAG_AP_UAPSD;
@@ -1066,9 +1067,14 @@ mt76_rx_convert(struct mt76_dev *dev, struct sk_buff *skb,
 	status->enc_flags = mstat.enc_flags;
 	status->encoding = mstat.encoding;
 	status->bw = mstat.bw;
-	status->he_ru = mstat.he_ru;
-	status->he_gi = mstat.he_gi;
-	status->he_dcm = mstat.he_dcm;
+	if (status->encoding == RX_ENC_EHT) {
+		status->eht.ru = mstat.eht.ru;
+		status->eht.gi = mstat.eht.gi;
+	} else {
+		status->he_ru = mstat.he_ru;
+		status->he_gi = mstat.he_gi;
+		status->he_dcm = mstat.he_dcm;
+	}
 	status->rate_idx = mstat.rate_idx;
 	status->nss = mstat.nss;
 	status->band = mstat.band;
@@ -1303,7 +1309,8 @@ mt76_check_sta(struct mt76_dev *dev, struct sk_buff *skb)
 	if (ps)
 		set_bit(MT_WCID_FLAG_PS, &wcid->flags);
 
-	dev->drv->sta_ps(dev, sta, ps);
+	if (dev->drv->sta_ps)
+		dev->drv->sta_ps(dev, sta, ps);
 
 	if (!ps)
 		clear_bit(MT_WCID_FLAG_PS, &wcid->flags);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h
index 183b0fc5a2d4..6b07b8fafec2 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76.h
@@ -621,12 +621,22 @@ struct mt76_rx_status {
 	u16 freq;
 	u32 flag;
 	u8 enc_flags;
-	u8 encoding:2, bw:3, he_ru:3;
-	u8 he_gi:2, he_dcm:1;
+	u8 encoding:3, bw:4;
+	union {
+		struct {
+			u8 he_ru:3;
+			u8 he_gi:2;
+			u8 he_dcm:1;
+		};
+		struct {
+			u8 ru:4;
+			u8 gi:2;
+		} eht;
+	};
+
 	u8 amsdu:1, first_amsdu:1, last_amsdu:1;
 	u8 rate_idx;
-	u8 nss;
-	u8 band;
+	u8 nss:5, band:3;
 	s8 signal;
 	u8 chains;
 	s8 chain_signal[IEEE80211_MAX_CHAINS];
@@ -778,6 +788,7 @@ struct mt76_dev {
 	spinlock_t rx_lock;
 	struct napi_struct napi[__MT_RXQ_MAX];
 	struct sk_buff_head rx_skb[__MT_RXQ_MAX];
+	struct tasklet_struct irq_tasklet;
 
 	struct list_head txwi_cache;
 	struct list_head rxwi_cache;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/mac.c b/drivers/net/wireless/mediatek/mt76/mt7603/mac.c
index 70a7f84af028..12e0af52082a 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7603/mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7603/mac.c
@@ -1279,8 +1279,11 @@ void mt7603_mac_add_txs(struct mt7603_dev *dev, void *data)
 	if (wcidx >= MT7603_WTBL_STA || !sta)
 		goto out;
 
-	if (mt7603_fill_txs(dev, msta, &info, txs_data))
+	if (mt7603_fill_txs(dev, msta, &info, txs_data)) {
+		spin_lock_bh(&dev->mt76.rx_lock);
 		ieee80211_tx_status_noskb(mt76_hw(dev), sta, &info);
+		spin_unlock_bh(&dev->mt76.rx_lock);
+	}
 
 out:
 	rcu_read_unlock();
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/dma.c b/drivers/net/wireless/mediatek/mt76/mt7615/dma.c
index f1914431ff7f..0ce01ccc5dce 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/dma.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/dma.c
@@ -76,7 +76,8 @@ static int mt7615_poll_tx(struct napi_struct *napi, int budget)
 
 	mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[MT_MCUQ_WM], false);
 	if (napi_complete(napi))
-		mt7615_irq_enable(dev, mt7615_tx_mcu_int_mask(dev));
+		mt76_connac_irq_enable(&dev->mt76,
+				       mt7615_tx_mcu_int_mask(dev));
 
 	mt76_connac_pm_unref(&dev->mphy, &dev->pm);
 
@@ -297,7 +298,7 @@ int mt7615_dma_init(struct mt7615_dev *dev)
 	else
 	    mask |= MT_INT_MCU_CMD;
 
-	mt7615_irq_enable(dev, mask);
+	mt76_connac_irq_enable(&dev->mt76, mask);
 
 	mt7615_dma_start(dev);
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.c
index 6dbaaf95ee38..68e88224b8b1 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.c
@@ -47,6 +47,9 @@ static int mt7615_efuse_init(struct mt7615_dev *dev, u32 base)
 	void *buf;
 	u32 val;
 
+	if (is_mt7663(&dev->mt76))
+		len = MT7663_EEPROM_SIZE;
+
 	val = mt76_rr(dev, base + MT_EFUSE_BASE_CTRL);
 	if (val & MT_EFUSE_BASE_CTRL_EMPTY)
 		return 0;
@@ -72,6 +75,8 @@ static int mt7615_eeprom_load(struct mt7615_dev *dev, u32 addr)
 {
 	int ret;
 
+	BUILD_BUG_ON(MT7615_EEPROM_FULL_SIZE < MT7663_EEPROM_SIZE);
+
 	ret = mt76_eeprom_init(&dev->mt76, MT7615_EEPROM_FULL_SIZE);
 	if (ret < 0)
 		return ret;
@@ -336,7 +341,7 @@ int mt7615_eeprom_init(struct mt7615_dev *dev, u32 addr)
 	ret = mt7615_check_eeprom(&dev->mt76);
 	if (ret && dev->mt76.otp.data) {
 		memcpy(dev->mt76.eeprom.data, dev->mt76.otp.data,
-		       MT7615_EEPROM_SIZE);
+		       dev->mt76.otp.size);
 	} else {
 		dev->flash_eeprom = true;
 		mt7615_cal_free_data(dev);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.h
index a024dee10362..a67fbb90f5b3 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/eeprom.h
@@ -46,7 +46,7 @@ enum mt7615_eeprom_field {
 
 	MT7615_EE_MAX =				0x3bf,
 	MT7622_EE_MAX =				0x3db,
-	MT7663_EE_MAX =				0x400,
+	MT7663_EE_MAX =				0x600,
 };
 
 #define MT_EE_RATE_POWER_MASK			GENMASK(5, 0)
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/init.c b/drivers/net/wireless/mediatek/mt76/mt7615/init.c
index 5fa6f097ec30..621e69f07e3c 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/init.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/init.c
@@ -396,6 +396,7 @@ mt7615_init_wiphy(struct ieee80211_hw *hw)
 
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_SET_SCAN_DWELL);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_VHT_IBSS);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
 
 	ieee80211_hw_set(hw, SINGLE_SCAN_ON_ALL_BANDS);
 	ieee80211_hw_set(hw, TX_STATUS_NO_AMPDU_LEN);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c
index 51a968a6afdc..da1d17b73a25 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c
@@ -655,11 +655,6 @@ static int mt7615_mac_fill_rx(struct mt7615_dev *dev, struct sk_buff *skb)
 	return 0;
 }
 
-void mt7615_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps)
-{
-}
-EXPORT_SYMBOL_GPL(mt7615_sta_ps);
-
 static u16
 mt7615_mac_tx_rate_val(struct mt7615_dev *dev,
 		       struct mt76_phy *mphy,
@@ -1530,8 +1525,11 @@ static void mt7615_mac_add_txs(struct mt7615_dev *dev, void *data)
 	if (wcid->phy_idx && dev->mt76.phys[MT_BAND1])
 		mphy = dev->mt76.phys[MT_BAND1];
 
-	if (mt7615_fill_txs(dev, msta, &info, txs_data))
+	if (mt7615_fill_txs(dev, msta, &info, txs_data)) {
+		spin_lock_bh(&dev->mt76.rx_lock);
 		ieee80211_tx_status_noskb(mphy->hw, sta, &info);
+		spin_unlock_bh(&dev->mt76.rx_lock);
+	}
 
 out:
 	rcu_read_unlock();
@@ -2352,7 +2350,7 @@ void mt7615_coredump_work(struct work_struct *work)
 			break;
 
 		skb_pull(skb, sizeof(struct mt7615_mcu_rxd));
-		if (data + skb->len - dump > MT76_CONNAC_COREDUMP_SZ) {
+		if (!dump || data + skb->len - dump > MT76_CONNAC_COREDUMP_SZ) {
 			dev_kfree_skb(skb);
 			continue;
 		}
@@ -2362,6 +2360,8 @@ void mt7615_coredump_work(struct work_struct *work)
 
 		dev_kfree_skb(skb);
 	}
-	dev_coredumpv(dev->mt76.dev, dump, MT76_CONNAC_COREDUMP_SZ,
-		      GFP_KERNEL);
+
+	if (dump)
+		dev_coredumpv(dev->mt76.dev, dump, MT76_CONNAC_COREDUMP_SZ,
+			      GFP_KERNEL);
 }
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.h b/drivers/net/wireless/mediatek/mt76/mt7615/mac.h
index 880c9f74a7f1..d08fbe64c262 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.h
@@ -19,18 +19,6 @@
 #define MT_RXD0_NORMAL_GROUP_3		BIT(27)
 #define MT_RXD0_NORMAL_GROUP_4		BIT(28)
 
-enum rx_pkt_type {
-	PKT_TYPE_TXS,
-	PKT_TYPE_TXRXV,
-	PKT_TYPE_NORMAL,
-	PKT_TYPE_RX_DUP_RFB,
-	PKT_TYPE_RX_TMR,
-	PKT_TYPE_RETRIEVE,
-	PKT_TYPE_TXRX_NOTIFY,
-	PKT_TYPE_RX_EVENT,
-	PKT_TYPE_NORMAL_MCU,
-};
-
 #define MT_RXD1_NORMAL_BSSID		GENMASK(31, 26)
 #define MT_RXD1_NORMAL_PAYLOAD_FORMAT	GENMASK(25, 24)
 #define MT_RXD1_FIRST_AMSDU_FRAME	GENMASK(1, 0)
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c
index eea398c79a98..8d745c9730c7 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c
@@ -163,16 +163,16 @@ int mt7615_mcu_parse_response(struct mt76_dev *mdev, int cmd,
 		   cmd == MCU_UNI_CMD(HIF_CTRL) ||
 		   cmd == MCU_UNI_CMD(OFFLOAD) ||
 		   cmd == MCU_UNI_CMD(SUSPEND)) {
-		struct mt7615_mcu_uni_event *event;
+		struct mt76_connac_mcu_uni_event *event;
 
 		skb_pull(skb, sizeof(*rxd));
-		event = (struct mt7615_mcu_uni_event *)skb->data;
+		event = (struct mt76_connac_mcu_uni_event *)skb->data;
 		ret = le32_to_cpu(event->status);
 	} else if (cmd == MCU_CE_QUERY(REG_READ)) {
-		struct mt7615_mcu_reg_event *event;
+		struct mt76_connac_mcu_reg_event *event;
 
 		skb_pull(skb, sizeof(*rxd));
-		event = (struct mt7615_mcu_reg_event *)skb->data;
+		event = (struct mt76_connac_mcu_reg_event *)skb->data;
 		ret = (int)le32_to_cpu(event->val);
 	}
 
@@ -861,7 +861,8 @@ mt7615_mcu_wtbl_sta_add(struct mt7615_phy *phy, struct ieee80211_vif *vif,
 		else
 			mvif->sta_added = true;
 	}
-	mt76_connac_mcu_sta_basic_tlv(sskb, vif, sta, enable, new_entry);
+	mt76_connac_mcu_sta_basic_tlv(&dev->mt76, sskb, vif, sta, enable,
+				      new_entry);
 	if (enable && sta)
 		mt76_connac_mcu_sta_tlv(phy->mt76, sskb, sta, vif, 0,
 					MT76_STA_INFO_STATE_ASSOC);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.h
index 615956acc6b5..8e9604be0792 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.h
@@ -206,17 +206,6 @@ enum {
 	MCU_ATE_SET_TX_POWER_CONTROL = 0x15,
 };
 
-struct mt7615_mcu_uni_event {
-	u8 cid;
-	u8 pad[3];
-	__le32 status; /* 0: success, others: fail */
-} __packed;
-
-struct mt7615_mcu_reg_event {
-	__le32 reg;
-	__le32 val;
-} __packed;
-
 struct mt7615_roc_tlv {
 	u8 bss_idx;
 	u8 token;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7615/mmio.c
index 83173efb56dc..ac036a072439 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/mmio.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/mmio.c
@@ -66,9 +66,7 @@ const u32 mt7663e_reg_map[] = {
 static void
 mt7615_rx_poll_complete(struct mt76_dev *mdev, enum mt76_rxq_id q)
 {
-	struct mt7615_dev *dev = container_of(mdev, struct mt7615_dev, mt76);
-
-	mt7615_irq_enable(dev, MT_INT_RX_DONE(q));
+	mt76_connac_irq_enable(mdev, MT_INT_RX_DONE(q));
 }
 
 static irqreturn_t mt7615_irq_handler(int irq, void *dev_instance)
@@ -80,14 +78,14 @@ static irqreturn_t mt7615_irq_handler(int irq, void *dev_instance)
 	if (!test_bit(MT76_STATE_INITIALIZED, &dev->mphy.state))
 		return IRQ_NONE;
 
-	tasklet_schedule(&dev->irq_tasklet);
+	tasklet_schedule(&dev->mt76.irq_tasklet);
 
 	return IRQ_HANDLED;
 }
 
 static void mt7615_irq_tasklet(struct tasklet_struct *t)
 {
-	struct mt7615_dev *dev = from_tasklet(dev, t, irq_tasklet);
+	struct mt7615_dev *dev = from_tasklet(dev, t, mt76.irq_tasklet);
 	u32 intr, mask = 0, tx_mcu_mask = mt7615_tx_mcu_int_mask(dev);
 	u32 mcu_int;
 
@@ -181,7 +179,6 @@ int mt7615_mmio_probe(struct device *pdev, void __iomem *mem_base,
 		.rx_check = mt7615_rx_check,
 		.rx_skb = mt7615_queue_rx_skb,
 		.rx_poll_complete = mt7615_rx_poll_complete,
-		.sta_ps = mt7615_sta_ps,
 		.sta_add = mt7615_mac_sta_add,
 		.sta_remove = mt7615_mac_sta_remove,
 		.update_survey = mt7615_update_channel,
@@ -202,7 +199,7 @@ int mt7615_mmio_probe(struct device *pdev, void __iomem *mem_base,
 
 	dev = container_of(mdev, struct mt7615_dev, mt76);
 	mt76_mmio_init(&dev->mt76, mem_base);
-	tasklet_setup(&dev->irq_tasklet, mt7615_irq_tasklet);
+	tasklet_setup(&mdev->irq_tasklet, mt7615_irq_tasklet);
 
 	dev->reg_map = map;
 	dev->ops = ops;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h b/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h
index 9e58f6924493..582d1b5b7cb3 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h
@@ -51,6 +51,7 @@
 #define MT7663_FIRMWARE_N9		"mediatek/mt7663_n9_rebb.bin"
 
 #define MT7615_EEPROM_SIZE		1024
+#define MT7663_EEPROM_SIZE		1536
 #define MT7615_TOKEN_SIZE		4096
 
 #define MT_FRAC_SCALE		12
@@ -245,8 +246,6 @@ struct mt7615_dev {
 	};
 
 	const struct mt76_bus_ops *bus_ops;
-	struct tasklet_struct irq_tasklet;
-
 	struct mt7615_phy phy;
 	u64 omac_mask;
 
@@ -412,13 +411,6 @@ void mt7615_mcu_rx_event(struct mt7615_dev *dev, struct sk_buff *skb);
 int mt7615_mcu_rdd_send_pattern(struct mt7615_dev *dev);
 int mt7615_mcu_fw_log_2_host(struct mt7615_dev *dev, u8 ctrl);
 
-static inline void mt7615_irq_enable(struct mt7615_dev *dev, u32 mask)
-{
-	mt76_set_irq_mask(&dev->mt76, 0, 0, mask);
-
-	tasklet_schedule(&dev->irq_tasklet);
-}
-
 static inline bool mt7615_firmware_offload(struct mt7615_dev *dev)
 {
 	return dev->fw_ver > MT7615_FIRMWARE_V2;
@@ -518,7 +510,6 @@ void mt7615_tx_token_put(struct mt7615_dev *dev);
 bool mt7615_rx_check(struct mt76_dev *mdev, void *data, int len);
 void mt7615_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
 			 struct sk_buff *skb, u32 *info);
-void mt7615_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps);
 int mt7615_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif,
 		       struct ieee80211_sta *sta);
 void mt7615_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif,
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/pci.c b/drivers/net/wireless/mediatek/mt76/mt7615/pci.c
index b808248943ea..9f43e673518b 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/pci.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/pci.c
@@ -94,7 +94,7 @@ static int mt7615_pci_suspend(struct pci_dev *pdev, pm_message_t state)
 	mt76_for_each_q_rx(mdev, i) {
 		napi_disable(&mdev->napi[i]);
 	}
-	tasklet_kill(&dev->irq_tasklet);
+	tasklet_kill(&mdev->irq_tasklet);
 
 	mt7615_dma_reset(dev);
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c b/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c
index 0680e002b981..f607eee3fb47 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/pci_init.c
@@ -122,7 +122,7 @@ void mt7615_unregister_device(struct mt7615_dev *dev)
 
 	mt7615_tx_token_put(dev);
 	mt7615_dma_cleanup(dev);
-	tasklet_disable(&dev->irq_tasklet);
+	tasklet_disable(&dev->mt76.irq_tasklet);
 
 	mt76_free_device(&dev->mt76);
 }
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c b/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c
index 304212f5f8da..fc547a0031ea 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c
@@ -84,7 +84,6 @@ static int mt7663s_probe(struct sdio_func *func,
 		.tx_status_data = mt7663_usb_sdio_tx_status_data,
 		.rx_skb = mt7615_queue_rx_skb,
 		.rx_check = mt7615_rx_check,
-		.sta_ps = mt7615_sta_ps,
 		.sta_add = mt7615_mac_sta_add,
 		.sta_remove = mt7615_mac_sta_remove,
 		.update_survey = mt7615_update_channel,
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/usb.c b/drivers/net/wireless/mediatek/mt76/mt7615/usb.c
index f2d651d7adff..04963b9f7498 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/usb.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/usb.c
@@ -120,7 +120,6 @@ static int mt7663u_probe(struct usb_interface *usb_intf,
 		.tx_status_data = mt7663_usb_sdio_tx_status_data,
 		.rx_skb = mt7615_queue_rx_skb,
 		.rx_check = mt7615_rx_check,
-		.sta_ps = mt7615_sta_ps,
 		.sta_add = mt7615_mac_sta_add,
 		.sta_remove = mt7615_mac_sta_remove,
 		.update_survey = mt7615_update_channel,
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac.h
index b339c50bff20..15653b274f83 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac.h
@@ -6,6 +6,20 @@
 
 #include "mt76.h"
 
+enum rx_pkt_type {
+	PKT_TYPE_TXS,
+	PKT_TYPE_TXRXV,
+	PKT_TYPE_NORMAL,
+	PKT_TYPE_RX_DUP_RFB,
+	PKT_TYPE_RX_TMR,
+	PKT_TYPE_RETRIEVE,
+	PKT_TYPE_TXRX_NOTIFY,
+	PKT_TYPE_RX_EVENT,
+	PKT_TYPE_NORMAL_MCU,
+	PKT_TYPE_RX_FW_MONITOR	= 0x0c,
+	PKT_TYPE_TXRX_NOTIFY_V0	= 0x18,
+};
+
 #define MT76_CONNAC_SCAN_IE_LEN			600
 #define MT76_CONNAC_MAX_NUM_SCHED_SCAN_INTERVAL	 10
 #define MT76_CONNAC_MAX_TIME_SCHED_SCAN_INTERVAL U16_MAX
@@ -279,6 +293,12 @@ static inline u8 mt76_connac_spe_idx(u8 antenna_mask)
 	return ant_to_spe[antenna_mask];
 }
 
+static inline void mt76_connac_irq_enable(struct mt76_dev *dev, u32 mask)
+{
+	mt76_set_irq_mask(dev, 0, 0, mask);
+	tasklet_schedule(&dev->irq_tasklet);
+}
+
 int mt76_connac_pm_wake(struct mt76_phy *phy, struct mt76_connac_pm *pm);
 void mt76_connac_power_save_sched(struct mt76_phy *phy,
 				  struct mt76_connac_pm *pm);
@@ -353,6 +373,7 @@ mt76_connac_mutex_release(struct mt76_dev *dev, struct mt76_connac_pm *pm)
 	mutex_unlock(&dev->mutex);
 }
 
+void mt76_connac_gen_ppe_thresh(u8 *he_ppet, int nss);
 int mt76_connac_init_tx_queues(struct mt76_phy *phy, int idx, int n_desc,
 			       int ring_base, u32 flags);
 void mt76_connac_write_hw_txp(struct mt76_dev *dev,
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h
index f33171bcd343..a5ec0f631385 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h
@@ -32,6 +32,16 @@ enum {
 	MT_LMAC_PSMP0,
 };
 
+#define MT_TX_FREE_MSDU_CNT		GENMASK(9, 0)
+#define MT_TX_FREE_WLAN_ID		GENMASK(23, 14)
+#define MT_TX_FREE_LATENCY		GENMASK(12, 0)
+/* 0: success, others: dropped */
+#define MT_TX_FREE_STATUS		GENMASK(14, 13)
+#define MT_TX_FREE_MSDU_ID		GENMASK(30, 16)
+#define MT_TX_FREE_PAIR			BIT(31)
+/* will support this field in further revision */
+#define MT_TX_FREE_RATE			GENMASK(13, 0)
+
 #define MT_TXD0_Q_IDX			GENMASK(31, 25)
 #define MT_TXD0_PKT_FMT			GENMASK(24, 23)
 #define MT_TXD0_ETH_TYPE_OFFSET		GENMASK(22, 16)
@@ -166,6 +176,15 @@ enum {
 
 #define MT_TXS7_MPDU_RETRY_CNT		GENMASK(31, 23)
 
+/* RXD DW0 */
+#define MT_RXD0_LENGTH			GENMASK(15, 0)
+#define MT_RXD0_PKT_FLAG                GENMASK(19, 16)
+#define MT_RXD0_PKT_TYPE		GENMASK(31, 27)
+
+#define MT_RXD0_NORMAL_ETH_TYPE_OFS	GENMASK(22, 16)
+#define MT_RXD0_NORMAL_IP_SUM		BIT(23)
+#define MT_RXD0_NORMAL_UDP_TCP_SUM	BIT(24)
+
 /* RXD DW1 */
 #define MT_RXD1_NORMAL_WLAN_IDX		GENMASK(9, 0)
 #define MT_RXD1_NORMAL_GROUP_1		BIT(11)
@@ -308,6 +327,9 @@ enum {
 #define MT_CRXV_FOE_HI		GENMASK(6, 0)
 #define MT_CRXV_FOE_SHIFT	13
 
+#define MT_CT_PARSE_LEN			72
+#define MT_CT_DMA_BUF_NUM		2
+
 #define MT_CT_INFO_APPLY_TXD		BIT(0)
 #define MT_CT_INFO_COPY_HOST_TXD_ALL	BIT(1)
 #define MT_CT_INFO_MGMT_FRAME		BIT(2)
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c
index aed4ee95fb2e..ee0fbfcd07d6 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c
@@ -9,6 +9,27 @@
 #define HE_PREP(f, m, v)	le16_encode_bits(le32_get_bits(v, MT_CRXV_HE_##m),\
 						 IEEE80211_RADIOTAP_HE_##f)
 
+void mt76_connac_gen_ppe_thresh(u8 *he_ppet, int nss)
+{
+	static const u8 ppet16_ppet8_ru3_ru0[] = { 0x1c, 0xc7, 0x71 };
+	u8 i, ppet_bits, ppet_size, ru_bit_mask = 0x7; /* HE80 */
+
+	he_ppet[0] = FIELD_PREP(IEEE80211_PPE_THRES_NSS_MASK, nss - 1) |
+		     FIELD_PREP(IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK,
+				ru_bit_mask);
+
+	ppet_bits = IEEE80211_PPE_THRES_INFO_PPET_SIZE *
+		    nss * hweight8(ru_bit_mask) * 2;
+	ppet_size = DIV_ROUND_UP(ppet_bits, 8);
+
+	for (i = 0; i < ppet_size - 1; i++)
+		he_ppet[i + 1] = ppet16_ppet8_ru3_ru0[i % 3];
+
+	he_ppet[i + 1] = ppet16_ppet8_ru3_ru0[i % 3] &
+			 (0xff >> (8 - (ppet_bits - 1) % 8));
+}
+EXPORT_SYMBOL_GPL(mt76_connac_gen_ppe_thresh);
+
 int mt76_connac_pm_wake(struct mt76_phy *phy, struct mt76_connac_pm *pm)
 {
 	struct mt76_dev *dev = phy->dev;
@@ -267,11 +288,29 @@ int mt76_connac_init_tx_queues(struct mt76_phy *phy, int idx, int n_desc,
 }
 EXPORT_SYMBOL_GPL(mt76_connac_init_tx_queues);
 
+#define __bitrate_mask_check(_mcs, _mode)				\
+({									\
+	u8 i = 0;							\
+	for (nss = 0; i < ARRAY_SIZE(mask->control[band]._mcs); i++) {	\
+		if (!mask->control[band]._mcs[i])			\
+			continue;					\
+		if (hweight16(mask->control[band]._mcs[i]) == 1) {	\
+			mode = MT_PHY_TYPE_##_mode;			\
+			rateidx = ffs(mask->control[band]._mcs[i]) - 1;	\
+			if (mode == MT_PHY_TYPE_HT)			\
+				rateidx += 8 * i;			\
+			else						\
+				nss = i + 1;				\
+			goto out;					\
+		}							\
+	}								\
+})
+
 u16 mt76_connac2_mac_tx_rate_val(struct mt76_phy *mphy,
 				 struct ieee80211_vif *vif,
 				 bool beacon, bool mcast)
 {
-	u8 mode = 0, band = mphy->chandef.chan->band;
+	u8 nss = 0, mode = 0, band = mphy->chandef.chan->band;
 	int rateidx = 0, mcast_rate;
 
 	if (!vif)
@@ -286,19 +325,12 @@ u16 mt76_connac2_mac_tx_rate_val(struct mt76_phy *mphy,
 		struct cfg80211_bitrate_mask *mask;
 
 		mask = &vif->bss_conf.beacon_tx_rate;
-		if (hweight16(mask->control[band].he_mcs[0]) == 1) {
-			rateidx = ffs(mask->control[band].he_mcs[0]) - 1;
-			mode = MT_PHY_TYPE_HE_SU;
-			goto out;
-		} else if (hweight16(mask->control[band].vht_mcs[0]) == 1) {
-			rateidx = ffs(mask->control[band].vht_mcs[0]) - 1;
-			mode = MT_PHY_TYPE_VHT;
-			goto out;
-		} else if (hweight8(mask->control[band].ht_mcs[0]) == 1) {
-			rateidx = ffs(mask->control[band].ht_mcs[0]) - 1;
-			mode = MT_PHY_TYPE_HT;
-			goto out;
-		} else if (hweight32(mask->control[band].legacy) == 1) {
+
+		__bitrate_mask_check(he_mcs, HE_SU);
+		__bitrate_mask_check(vht_mcs, VHT);
+		__bitrate_mask_check(ht_mcs, HT);
+
+		if (hweight32(mask->control[band].legacy) == 1) {
 			rateidx = ffs(mask->control[band].legacy) - 1;
 			goto legacy;
 		}
@@ -314,9 +346,9 @@ legacy:
 	rateidx = mt76_calculate_default_rate(mphy, rateidx);
 	mode = rateidx >> 8;
 	rateidx &= GENMASK(7, 0);
-
 out:
-	return FIELD_PREP(MT_TX_RATE_IDX, rateidx) |
+	return FIELD_PREP(MT_TX_RATE_NSS, nss) |
+	       FIELD_PREP(MT_TX_RATE_IDX, rateidx) |
 	       FIELD_PREP(MT_TX_RATE_MODE, mode);
 }
 EXPORT_SYMBOL_GPL(mt76_connac2_mac_tx_rate_val);
@@ -537,7 +569,8 @@ void mt76_connac2_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi,
 	if (txwi[2] & cpu_to_le32(MT_TXD2_FIX_RATE)) {
 		/* Fixed rata is available just for 802.11 txd */
 		struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
-		bool multicast = is_multicast_ether_addr(hdr->addr1);
+		bool multicast = ieee80211_is_data(hdr->frame_control) &&
+				 is_multicast_ether_addr(hdr->addr1);
 		u16 rate = mt76_connac2_mac_tx_rate_val(mphy, vif, beacon,
 							multicast);
 		u32 val = MT_TXD6_FIXED_BW;
@@ -582,6 +615,17 @@ bool mt76_connac2_mac_fill_txs(struct mt76_dev *dev, struct mt76_wcid *wcid,
 			le32_get_bits(txs_data[6], MT_TXS6_MPDU_FAIL_CNT);
 		stats->tx_retries +=
 			le32_get_bits(txs_data[7], MT_TXS7_MPDU_RETRY_CNT);
+
+		if (wcid->sta) {
+			struct ieee80211_sta *sta;
+			u8 tid;
+
+			sta = container_of((void *)wcid, struct ieee80211_sta,
+					   drv_priv);
+			tid = FIELD_GET(MT_TXS0_TID, txs);
+
+			ieee80211_refresh_tx_agg_session_timer(sta, tid);
+		}
 	}
 
 	txrate = FIELD_GET(MT_TXS0_TX_RATE, txs);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
index 008ece1b16f8..0f0a519f956f 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
@@ -363,7 +363,7 @@ void mt76_connac_mcu_bss_omac_tlv(struct sk_buff *skb,
 }
 EXPORT_SYMBOL_GPL(mt76_connac_mcu_bss_omac_tlv);
 
-void mt76_connac_mcu_sta_basic_tlv(struct sk_buff *skb,
+void mt76_connac_mcu_sta_basic_tlv(struct mt76_dev *dev, struct sk_buff *skb,
 				   struct ieee80211_vif *vif,
 				   struct ieee80211_sta *sta,
 				   bool enable, bool newly)
@@ -394,7 +394,7 @@ void mt76_connac_mcu_sta_basic_tlv(struct sk_buff *skb,
 	switch (vif->type) {
 	case NL80211_IFTYPE_MESH_POINT:
 	case NL80211_IFTYPE_AP:
-		if (vif->p2p)
+		if (vif->p2p && !is_mt7921(dev))
 			conn_type = CONNECTION_P2P_GC;
 		else
 			conn_type = CONNECTION_INFRA_STA;
@@ -402,7 +402,7 @@ void mt76_connac_mcu_sta_basic_tlv(struct sk_buff *skb,
 		basic->aid = cpu_to_le16(sta->aid);
 		break;
 	case NL80211_IFTYPE_STATION:
-		if (vif->p2p)
+		if (vif->p2p && !is_mt7921(dev))
 			conn_type = CONNECTION_P2P_GO;
 		else
 			conn_type = CONNECTION_INFRA_AP;
@@ -1029,7 +1029,7 @@ int mt76_connac_mcu_sta_cmd(struct mt76_phy *phy,
 		return PTR_ERR(skb);
 
 	if (info->sta || !info->offload_fw)
-		mt76_connac_mcu_sta_basic_tlv(skb, info->vif, info->sta,
+		mt76_connac_mcu_sta_basic_tlv(dev, skb, info->vif, info->sta,
 					      info->enable, info->newly);
 	if (info->sta && info->enable)
 		mt76_connac_mcu_sta_tlv(phy, skb, info->sta,
@@ -1678,8 +1678,16 @@ int mt76_connac_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 	req->channel_min_dwell_time = cpu_to_le16(duration);
 	req->channel_dwell_time = cpu_to_le16(duration);
 
-	req->channels_num = min_t(u8, sreq->n_channels, 32);
-	req->ext_channels_num = min_t(u8, ext_channels_num, 32);
+	if (sreq->n_channels == 0 || sreq->n_channels > 64) {
+		req->channel_type = 0;
+		req->channels_num = 0;
+		req->ext_channels_num = 0;
+	} else {
+		req->channel_type = 4;
+		req->channels_num = min_t(u8, sreq->n_channels, 32);
+		req->ext_channels_num = min_t(u8, ext_channels_num, 32);
+	}
+
 	for (i = 0; i < req->channels_num + req->ext_channels_num; i++) {
 		if (i >= 32)
 			chan = &req->ext_channels[i - 32];
@@ -1699,7 +1707,6 @@ int mt76_connac_mcu_hw_scan(struct mt76_phy *phy, struct ieee80211_vif *vif,
 		}
 		chan->channel_num = scan_list[i]->hw_value;
 	}
-	req->channel_type = sreq->n_channels ? 4 : 0;
 
 	if (sreq->ie_len > 0) {
 		memcpy(req->ies, sreq->ie, sreq->ie_len);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h
index a5e6ee4daf92..ca1ce97a6d2f 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h
@@ -127,7 +127,7 @@ struct mt76_connac2_mcu_rxd {
 	u8 rsv1[2];
 	u8 s2d_index;
 
-	u8 tlv[0];
+	u8 tlv[];
 };
 
 struct mt76_connac2_patch_hdr {
@@ -967,9 +967,6 @@ enum {
 	DEV_INFO_MAX_NUM
 };
 
-#define MCU_UNI_CMD_EVENT                       BIT(1)
-#define MCU_UNI_CMD_UNSOLICITED_EVENT           BIT(2)
-
 /* event table */
 enum {
 	MCU_EVENT_TARGET_ADDRESS_LEN = 0x01,
@@ -1224,6 +1221,7 @@ enum {
 	MCU_UNI_CMD_VOW = 0x37,
 	MCU_UNI_CMD_RRO = 0x57,
 	MCU_UNI_CMD_OFFCH_SCAN_CTRL = 0x58,
+	MCU_UNI_CMD_ASSERT_DUMP = 0x6f,
 };
 
 enum {
@@ -1692,6 +1690,17 @@ struct mt76_connac_config {
 	u8 data[320];
 } __packed;
 
+struct mt76_connac_mcu_uni_event {
+	u8 cid;
+	u8 pad[3];
+	__le32 status; /* 0: success, others: fail */
+} __packed;
+
+struct mt76_connac_mcu_reg_event {
+	__le32 reg;
+	__le32 val;
+} __packed;
+
 static inline enum mcu_cipher_type
 mt76_connac_mcu_get_cipher(int cipher)
 {
@@ -1779,7 +1788,7 @@ mt76_connac_mcu_add_tlv(struct sk_buff *skb, int tag, int len)
 
 int mt76_connac_mcu_set_channel_domain(struct mt76_phy *phy);
 int mt76_connac_mcu_set_vif_ps(struct mt76_dev *dev, struct ieee80211_vif *vif);
-void mt76_connac_mcu_sta_basic_tlv(struct sk_buff *skb,
+void mt76_connac_mcu_sta_basic_tlv(struct mt76_dev *dev, struct sk_buff *skb,
 				   struct ieee80211_vif *vif,
 				   struct ieee80211_sta *sta, bool enable,
 				   bool newly);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c
index d3f74473e6fb..3e41d809ade3 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c
@@ -631,8 +631,11 @@ void mt76x02_send_tx_status(struct mt76x02_dev *dev,
 
 	mt76_tx_status_unlock(mdev, &list);
 
-	if (!status.skb)
+	if (!status.skb) {
+		spin_lock_bh(&dev->mt76.rx_lock);
 		ieee80211_tx_status_ext(mt76_hw(dev), &status);
+		spin_unlock_bh(&dev->mt76.rx_lock);
+	}
 
 	if (!len)
 		goto out;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c
index 5a46813a59ea..879884ead660 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c
@@ -958,10 +958,10 @@ mt7915_xmit_queues_show(struct seq_file *file, void *data)
 
 DEFINE_SHOW_ATTRIBUTE(mt7915_xmit_queues);
 
-#define mt7915_txpower_puts(prefix, rate)					\
+#define mt7915_txpower_puts(rate)						\
 ({										\
-	len += scnprintf(buf + len, sz - len, "%-16s:", #prefix " (tmac)");	\
-	for (i = 0; i < mt7915_sku_group_len[rate]; i++, offs++)		\
+	len += scnprintf(buf + len, sz - len, "%-16s:", #rate " (TMAC)");	\
+	for (i = 0; i < mt7915_sku_group_len[SKU_##rate]; i++, offs++)		\
 		len += scnprintf(buf + len, sz - len, " %6d", txpwr[offs]);	\
 	len += scnprintf(buf + len, sz - len, "\n");				\
 })
@@ -1004,41 +1004,41 @@ mt7915_rate_txpower_get(struct file *file, char __user *user_buf,
 			 phy != &dev->phy, phy->mt76->chandef.chan->hw_value);
 	len += scnprintf(buf + len, sz - len, "%-16s  %6s %6s %6s %6s\n",
 			 " ", "1m", "2m", "5m", "11m");
-	mt7915_txpower_puts(CCK, SKU_CCK);
+	mt7915_txpower_puts(CCK);
 
 	len += scnprintf(buf + len, sz - len,
 			 "%-16s  %6s %6s %6s %6s %6s %6s %6s %6s\n",
 			 " ", "6m", "9m", "12m", "18m", "24m", "36m", "48m",
 			 "54m");
-	mt7915_txpower_puts(OFDM, SKU_OFDM);
+	mt7915_txpower_puts(OFDM);
 
 	len += scnprintf(buf + len, sz - len,
 			 "%-16s  %6s %6s %6s %6s %6s %6s %6s %6s\n",
 			 " ", "mcs0", "mcs1", "mcs2", "mcs3", "mcs4",
 			 "mcs5", "mcs6", "mcs7");
-	mt7915_txpower_puts(HT20, SKU_HT_BW20);
+	mt7915_txpower_puts(HT_BW20);
 
 	len += scnprintf(buf + len, sz - len,
 			 "%-16s  %6s %6s %6s %6s %6s %6s %6s %6s %6s\n",
 			 " ", "mcs0", "mcs1", "mcs2", "mcs3", "mcs4", "mcs5",
 			 "mcs6", "mcs7", "mcs32");
-	mt7915_txpower_puts(HT40, SKU_HT_BW40);
+	mt7915_txpower_puts(HT_BW40);
 
 	len += scnprintf(buf + len, sz - len,
 			 "%-16s  %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s\n",
 			 " ", "mcs0", "mcs1", "mcs2", "mcs3", "mcs4", "mcs5",
 			 "mcs6", "mcs7", "mcs8", "mcs9", "mcs10", "mcs11");
-	mt7915_txpower_puts(VHT20, SKU_VHT_BW20);
-	mt7915_txpower_puts(VHT40, SKU_VHT_BW40);
-	mt7915_txpower_puts(VHT80, SKU_VHT_BW80);
-	mt7915_txpower_puts(VHT160, SKU_VHT_BW160);
-	mt7915_txpower_puts(HE26, SKU_HE_RU26);
-	mt7915_txpower_puts(HE52, SKU_HE_RU52);
-	mt7915_txpower_puts(HE106, SKU_HE_RU106);
-	mt7915_txpower_puts(HE242, SKU_HE_RU242);
-	mt7915_txpower_puts(HE484, SKU_HE_RU484);
-	mt7915_txpower_puts(HE996, SKU_HE_RU996);
-	mt7915_txpower_puts(HE996x2, SKU_HE_RU2x996);
+	mt7915_txpower_puts(VHT_BW20);
+	mt7915_txpower_puts(VHT_BW40);
+	mt7915_txpower_puts(VHT_BW80);
+	mt7915_txpower_puts(VHT_BW160);
+	mt7915_txpower_puts(HE_RU26);
+	mt7915_txpower_puts(HE_RU52);
+	mt7915_txpower_puts(HE_RU106);
+	mt7915_txpower_puts(HE_RU242);
+	mt7915_txpower_puts(HE_RU484);
+	mt7915_txpower_puts(HE_RU996);
+	mt7915_txpower_puts(HE_RU2x996);
 
 	reg = is_mt7915(&dev->mt76) ? MT_WF_PHY_TPC_CTRL_STAT(band) :
 	      MT_WF_PHY_TPC_CTRL_STAT_MT7916(band);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/dma.c b/drivers/net/wireless/mediatek/mt76/mt7915/dma.c
index abe17dac9996..43a5456d4b97 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/dma.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/dma.c
@@ -87,8 +87,14 @@ static void mt7915_dma_config(struct mt7915_dev *dev)
 				   MT7916_RXQ_BAND0);
 			RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA0, MT_INT_WED_RX_DONE_WA_MT7916,
 				   MT7916_RXQ_MCU_WA);
-			RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_WED_RX_DONE_BAND1_MT7916,
-				   MT7916_RXQ_BAND1);
+			if (dev->hif2)
+				RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0,
+					   MT_INT_RX_DONE_BAND1_MT7916,
+					   MT7916_RXQ_BAND1);
+			else
+				RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0,
+					   MT_INT_WED_RX_DONE_BAND1_MT7916,
+					   MT7916_RXQ_BAND1);
 			RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA0, MT_INT_WED_RX_DONE_WA_MAIN_MT7916,
 				   MT7916_RXQ_MCU_WA_MAIN);
 			TXQ_CONFIG(0, WFDMA0, MT_INT_WED_TX_DONE_BAND0,
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c
index 5e288116b1b0..ac2049f49bb3 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c
@@ -89,6 +89,7 @@ static ssize_t mt7915_thermal_temp_store(struct device *dev,
 	     val < phy->throttle_temp[MT7915_CRIT_TEMP_IDX])) {
 		dev_err(phy->dev->mt76.dev,
 			"temp1_max shall be greater than temp1_crit.");
+		mutex_unlock(&phy->dev->mt76.mutex);
 		return -EINVAL;
 	}
 
@@ -202,6 +203,10 @@ static int mt7915_thermal_init(struct mt7915_phy *phy)
 			phy->cdev = cdev;
 	}
 
+	/* initialize critical/maximum high temperature */
+	phy->throttle_temp[MT7915_CRIT_TEMP_IDX] = MT7915_CRIT_TEMP;
+	phy->throttle_temp[MT7915_MAX_TEMP_IDX] = MT7915_MAX_TEMP;
+
 	if (!IS_REACHABLE(CONFIG_HWMON))
 		return 0;
 
@@ -210,10 +215,6 @@ static int mt7915_thermal_init(struct mt7915_phy *phy)
 	if (IS_ERR(hwmon))
 		return PTR_ERR(hwmon);
 
-	/* initialize critical/maximum high temperature */
-	phy->throttle_temp[MT7915_CRIT_TEMP_IDX] = MT7915_CRIT_TEMP;
-	phy->throttle_temp[MT7915_MAX_TEMP_IDX] = MT7915_MAX_TEMP;
-
 	return 0;
 }
 
@@ -368,6 +369,7 @@ mt7915_init_wiphy(struct mt7915_phy *phy)
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_UNSOL_BCAST_PROBE_RESP);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_FILS_DISCOVERY);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_ACK_SIGNAL_SUPPORT);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
 
 	if (!is_mt7915(&dev->mt76))
 		wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_STA_TX_PWR);
@@ -930,27 +932,6 @@ mt7915_set_stream_he_txbf_caps(struct mt7915_phy *phy,
 	}
 }
 
-static void
-mt7915_gen_ppe_thresh(u8 *he_ppet, int nss)
-{
-	u8 i, ppet_bits, ppet_size, ru_bit_mask = 0x7; /* HE80 */
-	static const u8 ppet16_ppet8_ru3_ru0[] = {0x1c, 0xc7, 0x71};
-
-	he_ppet[0] = FIELD_PREP(IEEE80211_PPE_THRES_NSS_MASK, nss - 1) |
-		     FIELD_PREP(IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK,
-				ru_bit_mask);
-
-	ppet_bits = IEEE80211_PPE_THRES_INFO_PPET_SIZE *
-		    nss * hweight8(ru_bit_mask) * 2;
-	ppet_size = DIV_ROUND_UP(ppet_bits, 8);
-
-	for (i = 0; i < ppet_size - 1; i++)
-		he_ppet[i + 1] = ppet16_ppet8_ru3_ru0[i % 3];
-
-	he_ppet[i + 1] = ppet16_ppet8_ru3_ru0[i % 3] &
-			 (0xff >> (8 - (ppet_bits - 1) % 8));
-}
-
 static int
 mt7915_init_he_caps(struct mt7915_phy *phy, enum nl80211_band band,
 		    struct ieee80211_sband_iftype_data *data)
@@ -1100,7 +1081,7 @@ mt7915_init_he_caps(struct mt7915_phy *phy, enum nl80211_band band,
 		memset(he_cap->ppe_thres, 0, sizeof(he_cap->ppe_thres));
 		if (he_cap_elem->phy_cap_info[6] &
 		    IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
-			mt7915_gen_ppe_thresh(he_cap->ppe_thres, nss);
+			mt76_connac_gen_ppe_thresh(he_cap->ppe_thres, nss);
 		} else {
 			he_cap_elem->phy_cap_info[9] |=
 				u8_encode_bits(IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US,
@@ -1179,7 +1160,7 @@ static void mt7915_stop_hardware(struct mt7915_dev *dev)
 	mt7915_mcu_exit(dev);
 	mt7915_tx_token_put(dev);
 	mt7915_dma_cleanup(dev);
-	tasklet_disable(&dev->irq_tasklet);
+	tasklet_disable(&dev->mt76.irq_tasklet);
 
 	if (is_mt7986(&dev->mt76))
 		mt7986_wmac_disable(dev);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c
index 97ca55d283fb..7df8d95fc3fb 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c
@@ -73,10 +73,6 @@ static struct mt76_wcid *mt7915_rx_get_wcid(struct mt7915_dev *dev,
 	return &sta->vif->sta.wcid;
 }
 
-void mt7915_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps)
-{
-}
-
 bool mt7915_mac_wtbl_update(struct mt7915_dev *dev, int idx, u32 mask)
 {
 	mt76_rmw(dev, MT_WTBL_UPDATE, MT_WTBL_UPDATE_WLAN_IDX,
@@ -1627,7 +1623,7 @@ void mt7915_mac_reset_work(struct work_struct *work)
 	}
 	local_bh_enable();
 
-	tasklet_schedule(&dev->irq_tasklet);
+	tasklet_schedule(&dev->mt76.irq_tasklet);
 
 	mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_RESET_DONE);
 	mt7915_wait_reset_state(dev, MT_MCU_CMD_NORMAL_STATE);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.h b/drivers/net/wireless/mediatek/mt76/mt7915/mac.h
index 6fa9c79f3e5f..ce94f87e2042 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.h
@@ -6,43 +6,12 @@
 
 #include "../mt76_connac2_mac.h"
 
-#define MT_CT_PARSE_LEN			72
-#define MT_CT_DMA_BUF_NUM		2
-
-#define MT_RXD0_LENGTH			GENMASK(15, 0)
-#define MT_RXD0_PKT_TYPE		GENMASK(31, 27)
-
-#define MT_RXD0_NORMAL_ETH_TYPE_OFS	GENMASK(22, 16)
-#define MT_RXD0_NORMAL_IP_SUM		BIT(23)
-#define MT_RXD0_NORMAL_UDP_TCP_SUM	BIT(24)
-
-enum rx_pkt_type {
-	PKT_TYPE_TXS,
-	PKT_TYPE_TXRXV,
-	PKT_TYPE_NORMAL,
-	PKT_TYPE_RX_DUP_RFB,
-	PKT_TYPE_RX_TMR,
-	PKT_TYPE_RETRIEVE,
-	PKT_TYPE_TXRX_NOTIFY,
-	PKT_TYPE_RX_EVENT,
-	PKT_TYPE_RX_FW_MONITOR = 0x0c,
-	PKT_TYPE_TXRX_NOTIFY_V0 = 0x18,
-};
-
 #define MT_TX_FREE_VER			GENMASK(18, 16)
-#define MT_TX_FREE_MSDU_CNT		GENMASK(9, 0)
-#define MT_TX_FREE_MSDU_CNT_V0	GENMASK(6, 0)
-#define MT_TX_FREE_WLAN_ID		GENMASK(23, 14)
-#define MT_TX_FREE_LATENCY		GENMASK(12, 0)
+#define MT_TX_FREE_MSDU_CNT_V0		GENMASK(6, 0)
 /* 0: success, others: dropped */
-#define MT_TX_FREE_MSDU_ID		GENMASK(30, 16)
-#define MT_TX_FREE_PAIR			BIT(31)
 #define MT_TX_FREE_MPDU_HEADER		BIT(30)
 #define MT_TX_FREE_MSDU_ID_V3		GENMASK(14, 0)
 
-/* will support this field in further revision */
-#define MT_TX_FREE_RATE			GENMASK(13, 0)
-
 #define MT_TXS5_F0_FINAL_MPDU		BIT(31)
 #define MT_TXS5_F0_QOS			BIT(30)
 #define MT_TXS5_F0_TX_COUNT		GENMASK(29, 25)
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c
index 784191ec4802..1b361199c061 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c
@@ -269,7 +269,6 @@ static int mt7915_add_interface(struct ieee80211_hw *hw,
 	vif->offload_flags |= IEEE80211_OFFLOAD_ENCAP_4ADDR;
 
 	mt7915_init_bitrate_mask(vif);
-	memset(&mvif->cap, -1, sizeof(mvif->cap));
 
 	mt7915_mcu_add_bss_info(phy, vif, true);
 	mt7915_mcu_add_sta(dev, vif, NULL, true);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
index 5545a8bdf1d0..9fcb22fa1f97 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
@@ -706,7 +706,6 @@ static void
 mt7915_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta,
 		      struct ieee80211_vif *vif)
 {
-	struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv;
 	struct ieee80211_he_cap_elem *elem = &sta->deflink.he_cap.he_cap_elem;
 	struct ieee80211_he_mcs_nss_supp mcs_map;
 	struct sta_rec_he *he;
@@ -740,7 +739,7 @@ mt7915_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta,
 	     IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_RU_MAPPING_IN_5G))
 		cap |= STA_REC_HE_CAP_BW20_RU242_SUPPORT;
 
-	if (mvif->cap.he_ldpc &&
+	if (vif->bss_conf.he_ldpc &&
 	    (elem->phy_cap_info[1] &
 	     IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD))
 		cap |= STA_REC_HE_CAP_LDPC;
@@ -849,7 +848,6 @@ static void
 mt7915_mcu_sta_muru_tlv(struct mt7915_dev *dev, struct sk_buff *skb,
 			struct ieee80211_sta *sta, struct ieee80211_vif *vif)
 {
-	struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv;
 	struct ieee80211_he_cap_elem *elem = &sta->deflink.he_cap.he_cap_elem;
 	struct sta_rec_muru *muru;
 	struct tlv *tlv;
@@ -862,9 +860,9 @@ mt7915_mcu_sta_muru_tlv(struct mt7915_dev *dev, struct sk_buff *skb,
 
 	muru = (struct sta_rec_muru *)tlv;
 
-	muru->cfg.mimo_dl_en = mvif->cap.he_mu_ebfer ||
-			       mvif->cap.vht_mu_ebfer ||
-			       mvif->cap.vht_mu_ebfee;
+	muru->cfg.mimo_dl_en = vif->bss_conf.he_mu_beamformer ||
+			       vif->bss_conf.vht_mu_beamformer ||
+			       vif->bss_conf.vht_mu_beamformee;
 	if (!is_mt7915(&dev->mt76))
 		muru->cfg.mimo_ul_en = true;
 	muru->cfg.ofdma_dl_en = true;
@@ -997,8 +995,8 @@ mt7915_mcu_sta_wtbl_tlv(struct mt7915_dev *dev, struct sk_buff *skb,
 	mt76_connac_mcu_wtbl_hdr_trans_tlv(skb, vif, wcid, tlv, wtbl_hdr);
 	if (sta)
 		mt76_connac_mcu_wtbl_ht_tlv(&dev->mt76, skb, sta, tlv,
-					    wtbl_hdr, mvif->cap.ht_ldpc,
-					    mvif->cap.vht_ldpc);
+					    wtbl_hdr, vif->bss_conf.ht_ldpc,
+					    vif->bss_conf.vht_ldpc);
 
 	return 0;
 }
@@ -1007,7 +1005,6 @@ static inline bool
 mt7915_is_ebf_supported(struct mt7915_phy *phy, struct ieee80211_vif *vif,
 			struct ieee80211_sta *sta, bool bfee)
 {
-	struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv;
 	int tx_ant = hweight8(phy->mt76->chainmask) - 1;
 
 	if (vif->type != NL80211_IFTYPE_STATION &&
@@ -1021,10 +1018,10 @@ mt7915_is_ebf_supported(struct mt7915_phy *phy, struct ieee80211_vif *vif,
 		struct ieee80211_he_cap_elem *pe = &sta->deflink.he_cap.he_cap_elem;
 
 		if (bfee)
-			return mvif->cap.he_su_ebfee &&
+			return vif->bss_conf.he_su_beamformee &&
 			       HE_PHY(CAP3_SU_BEAMFORMER, pe->phy_cap_info[3]);
 		else
-			return mvif->cap.he_su_ebfer &&
+			return vif->bss_conf.he_su_beamformer &&
 			       HE_PHY(CAP4_SU_BEAMFORMEE, pe->phy_cap_info[4]);
 	}
 
@@ -1032,10 +1029,10 @@ mt7915_is_ebf_supported(struct mt7915_phy *phy, struct ieee80211_vif *vif,
 		u32 cap = sta->deflink.vht_cap.cap;
 
 		if (bfee)
-			return mvif->cap.vht_su_ebfee &&
+			return vif->bss_conf.vht_su_beamformee &&
 			       (cap & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE);
 		else
-			return mvif->cap.vht_su_ebfer &&
+			return vif->bss_conf.vht_su_beamformer &&
 			       (cap & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE);
 	}
 
@@ -1530,7 +1527,7 @@ mt7915_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7915_dev *dev,
 			cap |= STA_CAP_TX_STBC;
 		if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_RX_STBC)
 			cap |= STA_CAP_RX_STBC;
-		if (mvif->cap.ht_ldpc &&
+		if (vif->bss_conf.ht_ldpc &&
 		    (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_LDPC_CODING))
 			cap |= STA_CAP_LDPC;
 
@@ -1556,7 +1553,7 @@ mt7915_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7915_dev *dev,
 			cap |= STA_CAP_VHT_TX_STBC;
 		if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_1)
 			cap |= STA_CAP_VHT_RX_STBC;
-		if (mvif->cap.vht_ldpc &&
+		if (vif->bss_conf.vht_ldpc &&
 		    (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXLDPC))
 			cap |= STA_CAP_VHT_LDPC;
 
@@ -1657,8 +1654,8 @@ int mt7915_mcu_add_sta(struct mt7915_dev *dev, struct ieee80211_vif *vif,
 		return PTR_ERR(skb);
 
 	/* starec basic */
-	mt76_connac_mcu_sta_basic_tlv(skb, vif, sta, enable,
-			!rcu_access_pointer(dev->mt76.wcid[msta->wcid.idx]));
+	mt76_connac_mcu_sta_basic_tlv(&dev->mt76, skb, vif, sta, enable,
+				      !rcu_access_pointer(dev->mt76.wcid[msta->wcid.idx]));
 	if (!enable)
 		goto out;
 
@@ -1876,84 +1873,6 @@ mt7915_mcu_beacon_cont(struct mt7915_dev *dev, struct ieee80211_vif *vif,
 }
 
 static void
-mt7915_mcu_beacon_check_caps(struct mt7915_phy *phy, struct ieee80211_vif *vif,
-			     struct sk_buff *skb)
-{
-	struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv;
-	struct mt7915_vif_cap *vc = &mvif->cap;
-	const struct ieee80211_he_cap_elem *he;
-	const struct ieee80211_vht_cap *vht;
-	const struct ieee80211_ht_cap *ht;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)skb->data;
-	const u8 *ie;
-	u32 len, bc;
-
-	/* Check missing configuration options to allow AP mode in mac80211
-	 * to remain in sync with hostapd settings, and get a subset of
-	 * beacon and hardware capabilities.
-	 */
-	if (WARN_ON_ONCE(skb->len <= (mgmt->u.beacon.variable - skb->data)))
-		return;
-
-	memset(vc, 0, sizeof(*vc));
-
-	len = skb->len - (mgmt->u.beacon.variable - skb->data);
-
-	ie = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY, mgmt->u.beacon.variable,
-			      len);
-	if (ie && ie[1] >= sizeof(*ht)) {
-		ht = (void *)(ie + 2);
-		vc->ht_ldpc = !!(le16_to_cpu(ht->cap_info) &
-				 IEEE80211_HT_CAP_LDPC_CODING);
-	}
-
-	ie = cfg80211_find_ie(WLAN_EID_VHT_CAPABILITY, mgmt->u.beacon.variable,
-			      len);
-	if (ie && ie[1] >= sizeof(*vht)) {
-		u32 pc = phy->mt76->sband_5g.sband.vht_cap.cap;
-
-		vht = (void *)(ie + 2);
-		bc = le32_to_cpu(vht->vht_cap_info);
-
-		vc->vht_ldpc = !!(bc & IEEE80211_VHT_CAP_RXLDPC);
-		vc->vht_su_ebfer =
-			(bc & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE) &&
-			(pc & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE);
-		vc->vht_su_ebfee =
-			(bc & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE) &&
-			(pc & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE);
-		vc->vht_mu_ebfer =
-			(bc & IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE) &&
-			(pc & IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE);
-		vc->vht_mu_ebfee =
-			(bc & IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE) &&
-			(pc & IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE);
-	}
-
-	ie = cfg80211_find_ext_ie(WLAN_EID_EXT_HE_CAPABILITY,
-				  mgmt->u.beacon.variable, len);
-	if (ie && ie[1] >= sizeof(*he) + 1) {
-		const struct ieee80211_sta_he_cap *pc =
-			mt76_connac_get_he_phy_cap(phy->mt76, vif);
-		const struct ieee80211_he_cap_elem *pe = &pc->he_cap_elem;
-
-		he = (void *)(ie + 3);
-
-		vc->he_ldpc =
-			HE_PHY(CAP1_LDPC_CODING_IN_PAYLOAD, pe->phy_cap_info[1]);
-		vc->he_su_ebfer =
-			HE_PHY(CAP3_SU_BEAMFORMER, he->phy_cap_info[3]) &&
-			HE_PHY(CAP3_SU_BEAMFORMER, pe->phy_cap_info[3]);
-		vc->he_su_ebfee =
-			HE_PHY(CAP4_SU_BEAMFORMEE, he->phy_cap_info[4]) &&
-			HE_PHY(CAP4_SU_BEAMFORMEE, pe->phy_cap_info[4]);
-		vc->he_mu_ebfer =
-			HE_PHY(CAP4_MU_BEAMFORMER, he->phy_cap_info[4]) &&
-			HE_PHY(CAP4_MU_BEAMFORMER, pe->phy_cap_info[4]);
-	}
-}
-
-static void
 mt7915_mcu_beacon_inband_discov(struct mt7915_dev *dev, struct ieee80211_vif *vif,
 				struct sk_buff *rskb, struct bss_info_bcn *bcn,
 				u32 changed)
@@ -2063,8 +1982,6 @@ int mt7915_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	info = IEEE80211_SKB_CB(skb);
 	info->hw_queue = FIELD_PREP(MT_TX_HW_QUEUE_PHY, ext_phy);
 
-	mt7915_mcu_beacon_check_caps(phy, vif, skb);
-
 	mt7915_mcu_beacon_cntdwn(vif, rskb, skb, bcn, &offs);
 	mt7915_mcu_beacon_mbss(rskb, skb, vif, bcn, &offs);
 	mt7915_mcu_beacon_cont(dev, vif, rskb, skb, bcn, &offs);
@@ -2370,7 +2287,9 @@ int mt7915_mcu_init_firmware(struct mt7915_dev *dev)
 	if (ret)
 		return ret;
 
-	if (mtk_wed_device_active(&dev->mt76.mmio.wed) && is_mt7915(&dev->mt76))
+	if ((mtk_wed_device_active(&dev->mt76.mmio.wed) &&
+	     is_mt7915(&dev->mt76)) ||
+	    !mtk_wed_get_rx_capa(&dev->mt76.mmio.wed))
 		mt7915_mcu_wa_cmd(dev, MCU_WA_PARAM_CMD(CAPABILITY), 0, 0, 0);
 
 	ret = mt7915_mcu_set_mwds(dev, 1);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c
index 225a19604d3e..45f3558bf31c 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c
@@ -916,7 +916,7 @@ static void mt7915_rx_poll_complete(struct mt76_dev *mdev,
 /* TODO: support 2/4/6/8 MSI-X vectors */
 static void mt7915_irq_tasklet(struct tasklet_struct *t)
 {
-	struct mt7915_dev *dev = from_tasklet(dev, t, irq_tasklet);
+	struct mt7915_dev *dev = from_tasklet(dev, t, mt76.irq_tasklet);
 	struct mtk_wed_device *wed = &dev->mt76.mmio.wed;
 	u32 intr, intr1, mask;
 
@@ -989,18 +989,18 @@ irqreturn_t mt7915_irq_handler(int irq, void *dev_instance)
 	struct mt7915_dev *dev = dev_instance;
 	struct mtk_wed_device *wed = &dev->mt76.mmio.wed;
 
-	if (mtk_wed_device_active(wed)) {
+	if (mtk_wed_device_active(wed))
 		mtk_wed_device_irq_set_mask(wed, 0);
-	} else {
+	else
 		mt76_wr(dev, MT_INT_MASK_CSR, 0);
-		if (dev->hif2)
-			mt76_wr(dev, MT_INT1_MASK_CSR, 0);
-	}
+
+	if (dev->hif2)
+		mt76_wr(dev, MT_INT1_MASK_CSR, 0);
 
 	if (!test_bit(MT76_STATE_INITIALIZED, &dev->mphy.state))
 		return IRQ_NONE;
 
-	tasklet_schedule(&dev->irq_tasklet);
+	tasklet_schedule(&dev->mt76.irq_tasklet);
 
 	return IRQ_HANDLED;
 }
@@ -1022,7 +1022,6 @@ struct mt7915_dev *mt7915_mmio_probe(struct device *pdev,
 		.rx_skb = mt7915_queue_rx_skb,
 		.rx_check = mt7915_rx_check,
 		.rx_poll_complete = mt7915_rx_poll_complete,
-		.sta_ps = mt7915_sta_ps,
 		.sta_add = mt7915_mac_sta_add,
 		.sta_remove = mt7915_mac_sta_remove,
 		.update_survey = mt7915_update_channel,
@@ -1041,7 +1040,7 @@ struct mt7915_dev *mt7915_mmio_probe(struct device *pdev,
 	if (ret)
 		goto error;
 
-	tasklet_setup(&dev->irq_tasklet, mt7915_irq_tasklet);
+	tasklet_setup(&mdev->irq_tasklet, mt7915_irq_tasklet);
 
 	return dev;
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h
index 3cbfb9b6a305..b3ead3530740 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h
@@ -147,23 +147,9 @@ struct mt7915_sta {
 	} twt;
 };
 
-struct mt7915_vif_cap {
-	bool ht_ldpc:1;
-	bool vht_ldpc:1;
-	bool he_ldpc:1;
-	bool vht_su_ebfer:1;
-	bool vht_su_ebfee:1;
-	bool vht_mu_ebfer:1;
-	bool vht_mu_ebfee:1;
-	bool he_su_ebfer:1;
-	bool he_su_ebfee:1;
-	bool he_mu_ebfer:1;
-};
-
 struct mt7915_vif {
 	struct mt76_vif mt76; /* must be first */
 
-	struct mt7915_vif_cap cap;
 	struct mt7915_sta sta;
 	struct mt7915_phy *phy;
 
@@ -308,7 +294,6 @@ struct mt7915_dev {
 	u32 wfdma_mask;
 
 	const struct mt76_bus_ops *bus_ops;
-	struct tasklet_struct irq_tasklet;
 	struct mt7915_phy phy;
 
 	/* monitor rx chain configured channel */
@@ -581,7 +566,7 @@ static inline void mt7915_irq_enable(struct mt7915_dev *dev, u32 mask)
 	else
 		mt76_set_irq_mask(&dev->mt76, 0, 0, mask);
 
-	tasklet_schedule(&dev->irq_tasklet);
+	tasklet_schedule(&dev->mt76.irq_tasklet);
 }
 
 static inline void mt7915_irq_disable(struct mt7915_dev *dev, u32 mask)
@@ -631,7 +616,6 @@ void mt7915_tx_token_put(struct mt7915_dev *dev);
 void mt7915_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
 			 struct sk_buff *skb, u32 *info);
 bool mt7915_rx_check(struct mt76_dev *mdev, void *data, int len);
-void mt7915_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps);
 void mt7915_stats_work(struct work_struct *work);
 int mt76_dfs_start_rdd(struct mt7915_dev *dev, bool force);
 int mt7915_dfs_init_radar_detector(struct mt7915_phy *phy);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c
index 2ac0a0f2859c..32c137066e7f 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c
@@ -1239,6 +1239,8 @@ static const struct of_device_id mt7986_wmac_of_match[] = {
 	{},
 };
 
+MODULE_DEVICE_TABLE(of, mt7986_wmac_of_match);
+
 struct platform_driver mt7986_wmac_driver = {
 	.driver = {
 		.name = "mt7986-wmac",
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.h b/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.h
index 35268b0890ad..6f2c4a572572 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.h
@@ -24,7 +24,7 @@ struct mt7921_asar_dyn {
 	u8 names[4];
 	u8 enable;
 	u8 nr_tbl;
-	struct mt7921_asar_dyn_limit tbl[0];
+	DECLARE_FLEX_ARRAY(struct mt7921_asar_dyn_limit, tbl);
 } __packed;
 
 struct mt7921_asar_dyn_limit_v2 {
@@ -37,7 +37,7 @@ struct mt7921_asar_dyn_v2 {
 	u8 enable;
 	u8 rsvd;
 	u8 nr_tbl;
-	struct mt7921_asar_dyn_limit_v2 tbl[0];
+	DECLARE_FLEX_ARRAY(struct mt7921_asar_dyn_limit_v2, tbl);
 } __packed;
 
 struct mt7921_asar_geo_band {
@@ -55,7 +55,7 @@ struct mt7921_asar_geo {
 	u8 names[4];
 	u8 version;
 	u8 nr_tbl;
-	struct mt7921_asar_geo_limit tbl[0];
+	DECLARE_FLEX_ARRAY(struct mt7921_asar_geo_limit, tbl);
 } __packed;
 
 struct mt7921_asar_geo_limit_v2 {
@@ -69,7 +69,7 @@ struct mt7921_asar_geo_v2 {
 	u8 version;
 	u8 rsvd;
 	u8 nr_tbl;
-	struct mt7921_asar_geo_limit_v2 tbl[0];
+	DECLARE_FLEX_ARRAY(struct mt7921_asar_geo_limit_v2, tbl);
 } __packed;
 
 struct mt7921_asar_cl {
@@ -85,7 +85,7 @@ struct mt7921_asar_fg {
 	u8 rsvd;
 	u8 nr_flag;
 	u8 rsvd1;
-	u8 flag[0];
+	u8 flag[];
 } __packed;
 
 struct mt7921_acpi_sar {
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c
index 29d8883268f6..d6b6edba2fec 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/debugfs.c
@@ -2,7 +2,6 @@
 /* Copyright (C) 2020 MediaTek Inc. */
 
 #include "mt7921.h"
-#include "eeprom.h"
 
 static int
 mt7921_reg_set(void *data, u64 val)
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/dma.c b/drivers/net/wireless/mediatek/mt76/mt7921/dma.c
index d1f10f6d9adc..f0a80c2b476a 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/dma.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/dma.c
@@ -3,7 +3,7 @@
 
 #include "mt7921.h"
 #include "../dma.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 
 static int mt7921_poll_tx(struct napi_struct *napi, int budget)
 {
@@ -19,7 +19,7 @@ static int mt7921_poll_tx(struct napi_struct *napi, int budget)
 
 	mt76_connac_tx_cleanup(&dev->mt76);
 	if (napi_complete(napi))
-		mt7921_irq_enable(dev, MT_INT_TX_DONE_ALL);
+		mt76_connac_irq_enable(&dev->mt76, MT_INT_TX_DONE_ALL);
 	mt76_connac_pm_unref(&dev->mphy, &dev->pm);
 
 	return 0;
@@ -66,6 +66,24 @@ static void mt7921_dma_prefetch(struct mt7921_dev *dev)
 
 static int mt7921_dma_disable(struct mt7921_dev *dev, bool force)
 {
+	/* disable WFDMA0 */
+	mt76_clear(dev, MT_WFDMA0_GLO_CFG,
+		   MT_WFDMA0_GLO_CFG_TX_DMA_EN | MT_WFDMA0_GLO_CFG_RX_DMA_EN |
+		   MT_WFDMA0_GLO_CFG_CSR_DISP_BASE_PTR_CHAIN_EN |
+		   MT_WFDMA0_GLO_CFG_OMIT_TX_INFO |
+		   MT_WFDMA0_GLO_CFG_OMIT_RX_INFO |
+		   MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2);
+
+	if (!mt76_poll_msec_tick(dev, MT_WFDMA0_GLO_CFG,
+				 MT_WFDMA0_GLO_CFG_TX_DMA_BUSY |
+				 MT_WFDMA0_GLO_CFG_RX_DMA_BUSY, 0, 100, 1))
+		return -ETIMEDOUT;
+
+	/* disable dmashdl */
+	mt76_clear(dev, MT_WFDMA0_GLO_CFG_EXT0,
+		   MT_WFDMA0_CSR_TX_DMASHDL_ENABLE);
+	mt76_set(dev, MT_DMASHDL_SW_CONTROL, MT_DMASHDL_DMASHDL_BYPASS);
+
 	if (force) {
 		/* reset */
 		mt76_clear(dev, MT_WFDMA0_RST,
@@ -77,24 +95,6 @@ static int mt7921_dma_disable(struct mt7921_dev *dev, bool force)
 			 MT_WFDMA0_RST_LOGIC_RST);
 	}
 
-	/* disable dmashdl */
-	mt76_clear(dev, MT_WFDMA0_GLO_CFG_EXT0,
-		   MT_WFDMA0_CSR_TX_DMASHDL_ENABLE);
-	mt76_set(dev, MT_DMASHDL_SW_CONTROL, MT_DMASHDL_DMASHDL_BYPASS);
-
-	/* disable WFDMA0 */
-	mt76_clear(dev, MT_WFDMA0_GLO_CFG,
-		   MT_WFDMA0_GLO_CFG_TX_DMA_EN | MT_WFDMA0_GLO_CFG_RX_DMA_EN |
-		   MT_WFDMA0_GLO_CFG_CSR_DISP_BASE_PTR_CHAIN_EN |
-		   MT_WFDMA0_GLO_CFG_OMIT_TX_INFO |
-		   MT_WFDMA0_GLO_CFG_OMIT_RX_INFO |
-		   MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2);
-
-	if (!mt76_poll(dev, MT_WFDMA0_GLO_CFG,
-		       MT_WFDMA0_GLO_CFG_TX_DMA_BUSY |
-		       MT_WFDMA0_GLO_CFG_RX_DMA_BUSY, 0, 1000))
-		return -ETIMEDOUT;
-
 	return 0;
 }
 
@@ -123,9 +123,9 @@ static int mt7921_dma_enable(struct mt7921_dev *dev)
 	mt76_set(dev, MT_WFDMA_DUMMY_CR, MT_WFDMA_NEED_REINIT);
 
 	/* enable interrupts for TX/RX rings */
-	mt7921_irq_enable(dev,
-			  MT_INT_RX_DONE_ALL | MT_INT_TX_DONE_ALL |
-			  MT_INT_MCU_CMD);
+	mt76_connac_irq_enable(&dev->mt76,
+			       MT_INT_RX_DONE_ALL | MT_INT_TX_DONE_ALL |
+			       MT_INT_MCU_CMD);
 	mt76_set(dev, MT_MCU2HOST_SW_INT_ENA, MT_MCU_CMD_WAKE_RX_PCIE);
 
 	return 0;
@@ -301,6 +301,10 @@ void mt7921_dma_cleanup(struct mt7921_dev *dev)
 		   MT_WFDMA0_GLO_CFG_OMIT_RX_INFO |
 		   MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2);
 
+	mt76_poll_msec_tick(dev, MT_WFDMA0_GLO_CFG,
+			    MT_WFDMA0_GLO_CFG_TX_DMA_BUSY |
+			    MT_WFDMA0_GLO_CFG_RX_DMA_BUSY, 0, 100, 1);
+
 	/* reset */
 	mt76_clear(dev, MT_WFDMA0_RST,
 		   MT_WFDMA0_RST_DMASHDL_ALL_RST |
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h
deleted file mode 100644
index 4b647278eb30..000000000000
--- a/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/* SPDX-License-Identifier: ISC */
-/* Copyright (C) 2020 MediaTek Inc. */
-
-#ifndef __MT7921_EEPROM_H
-#define __MT7921_EEPROM_H
-
-#include "mt7921.h"
-
-enum mt7921_eeprom_field {
-	MT_EE_CHIP_ID =		0x000,
-	MT_EE_VERSION =		0x002,
-	MT_EE_MAC_ADDR =	0x004,
-	MT_EE_WIFI_CONF =	0x07c,
-	MT_EE_HW_TYPE =		0x55b,
-	__MT_EE_MAX =		0x9ff
-};
-
-#define MT_EE_WIFI_CONF_TX_MASK			BIT(0)
-#define MT_EE_WIFI_CONF_BAND_SEL		GENMASK(3, 2)
-
-#define MT_EE_HW_TYPE_ENCAP			BIT(0)
-
-enum mt7921_eeprom_band {
-	MT_EE_NA,
-	MT_EE_5GHZ,
-	MT_EE_2GHZ,
-	MT_EE_DUAL_BAND,
-};
-
-#endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/init.c b/drivers/net/wireless/mediatek/mt76/mt7921/init.c
index cc94531185da..bf1da9fddfab 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/init.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/init.c
@@ -4,9 +4,8 @@
 #include <linux/etherdevice.h>
 #include <linux/firmware.h>
 #include "mt7921.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 #include "mcu.h"
-#include "eeprom.h"
 
 static const struct ieee80211_iface_limit if_limits[] = {
 	{
@@ -32,11 +31,13 @@ static const struct ieee80211_iface_combination if_comb[] = {
 static const struct ieee80211_iface_limit if_limits_chanctx[] = {
 	{
 		.max = 2,
-		.types = BIT(NL80211_IFTYPE_STATION),
+		.types = BIT(NL80211_IFTYPE_STATION) |
+			 BIT(NL80211_IFTYPE_P2P_CLIENT)
 	},
 	{
 		.max = 1,
-		.types = BIT(NL80211_IFTYPE_AP),
+		.types = BIT(NL80211_IFTYPE_AP) |
+			 BIT(NL80211_IFTYPE_P2P_GO)
 	}
 };
 
@@ -100,7 +101,9 @@ mt7921_init_wiphy(struct ieee80211_hw *hw)
 	wiphy->flags &= ~(WIPHY_FLAG_IBSS_RSN | WIPHY_FLAG_4ADDR_AP |
 			  WIPHY_FLAG_4ADDR_STATION);
 	wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) |
-				 BIT(NL80211_IFTYPE_AP);
+				 BIT(NL80211_IFTYPE_AP) |
+				 BIT(NL80211_IFTYPE_P2P_CLIENT) |
+				 BIT(NL80211_IFTYPE_P2P_GO);
 	wiphy->max_remain_on_channel_duration = 5000;
 	wiphy->max_scan_ie_len = MT76_CONNAC_SCAN_IE_LEN;
 	wiphy->max_scan_ssids = 4;
@@ -121,6 +124,7 @@ mt7921_init_wiphy(struct ieee80211_hw *hw)
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BEACON_RATE_VHT);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_BEACON_RATE_HE);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_ACK_SIGNAL_SUPPORT);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
 
 	ieee80211_hw_set(hw, SINGLE_SCAN_ON_ALL_BANDS);
 	ieee80211_hw_set(hw, HAS_RATE_CONTROL);
@@ -169,7 +173,8 @@ mt7921_mac_init_band(struct mt7921_dev *dev, u8 band)
 	mt76_rmw(dev, MT_WTBLOFF_TOP_RSCR(band), mask, set);
 }
 
-u8 mt7921_check_offload_capability(struct device *dev, const char *fw_wm)
+static u8
+mt7921_get_offload_capability(struct device *dev, const char *fw_wm)
 {
 	const struct mt76_connac2_fw_trailer *hdr;
 	struct mt7921_realease_info *rel_info;
@@ -223,7 +228,31 @@ out:
 
 	return offload_caps;
 }
-EXPORT_SYMBOL_GPL(mt7921_check_offload_capability);
+
+struct ieee80211_ops *
+mt7921_get_mac80211_ops(struct device *dev, void *drv_data, u8 *fw_features)
+{
+	struct ieee80211_ops *ops;
+
+	ops = devm_kmemdup(dev, &mt7921_ops, sizeof(mt7921_ops), GFP_KERNEL);
+	if (!ops)
+		return NULL;
+
+	*fw_features = mt7921_get_offload_capability(dev, drv_data);
+	if (!(*fw_features & MT7921_FW_CAP_CNM)) {
+		ops->remain_on_channel = NULL;
+		ops->cancel_remain_on_channel = NULL;
+		ops->add_chanctx = NULL;
+		ops->remove_chanctx = NULL;
+		ops->change_chanctx = NULL;
+		ops->assign_vif_chanctx = NULL;
+		ops->unassign_vif_chanctx = NULL;
+		ops->mgd_prepare_tx = NULL;
+		ops->mgd_complete_tx = NULL;
+	}
+	return ops;
+}
+EXPORT_SYMBOL_GPL(mt7921_get_mac80211_ops);
 
 int mt7921_mac_init(struct mt7921_dev *dev)
 {
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
index 557c20190c2b..1675bf520481 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
@@ -6,9 +6,20 @@
 #include <linux/timekeeping.h>
 #include "mt7921.h"
 #include "../dma.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 #include "mcu.h"
 
+#define MT_WTBL_TXRX_CAP_RATE_OFFSET	7
+#define MT_WTBL_TXRX_RATE_G2_HE		24
+#define MT_WTBL_TXRX_RATE_G2		12
+
+#define MT_WTBL_AC0_CTT_OFFSET		20
+
+static u32 mt7921_mac_wtbl_lmac_addr(int idx, u8 offset)
+{
+	return MT_WTBL_LMAC_OFFS(idx, 0) + offset * 4;
+}
+
 static struct mt76_wcid *mt7921_rx_get_wcid(struct mt7921_dev *dev,
 					    u16 idx, bool unicast)
 {
@@ -32,11 +43,6 @@ static struct mt76_wcid *mt7921_rx_get_wcid(struct mt7921_dev *dev,
 	return &sta->vif->sta.wcid;
 }
 
-void mt7921_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps)
-{
-}
-EXPORT_SYMBOL_GPL(mt7921_sta_ps);
-
 bool mt7921_mac_wtbl_update(struct mt7921_dev *dev, int idx, u32 mask)
 {
 	mt76_rmw(dev, MT_WTBL_UPDATE, MT_WTBL_UPDATE_WLAN_IDX,
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.h b/drivers/net/wireless/mediatek/mt76/mt7921/mac.h
deleted file mode 100644
index 8afec600364f..000000000000
--- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/* SPDX-License-Identifier: ISC */
-/* Copyright (C) 2020 MediaTek Inc. */
-
-#ifndef __MT7921_MAC_H
-#define __MT7921_MAC_H
-
-#include "../mt76_connac2_mac.h"
-
-#define MT_CT_PARSE_LEN			72
-#define MT_CT_DMA_BUF_NUM		2
-
-#define MT_RXD0_LENGTH			GENMASK(15, 0)
-#define MT_RXD0_PKT_FLAG                GENMASK(19, 16)
-#define MT_RXD0_PKT_TYPE		GENMASK(31, 27)
-
-#define MT_RXD0_NORMAL_ETH_TYPE_OFS	GENMASK(22, 16)
-#define MT_RXD0_NORMAL_IP_SUM		BIT(23)
-#define MT_RXD0_NORMAL_UDP_TCP_SUM	BIT(24)
-
-enum rx_pkt_type {
-	PKT_TYPE_TXS,
-	PKT_TYPE_TXRXV,
-	PKT_TYPE_NORMAL,
-	PKT_TYPE_RX_DUP_RFB,
-	PKT_TYPE_RX_TMR,
-	PKT_TYPE_RETRIEVE,
-	PKT_TYPE_TXRX_NOTIFY,
-	PKT_TYPE_RX_EVENT,
-	PKT_TYPE_NORMAL_MCU,
-};
-
-#define MT_TX_FREE_MSDU_CNT		GENMASK(9, 0)
-#define MT_TX_FREE_WLAN_ID		GENMASK(23, 14)
-#define MT_TX_FREE_LATENCY		GENMASK(12, 0)
-/* 0: success, others: dropped */
-#define MT_TX_FREE_STATUS		GENMASK(14, 13)
-#define MT_TX_FREE_MSDU_ID		GENMASK(30, 16)
-#define MT_TX_FREE_PAIR			BIT(31)
-/* will support this field in further revision */
-#define MT_TX_FREE_RATE			GENMASK(13, 0)
-
-#define MT_WTBL_TXRX_CAP_RATE_OFFSET	7
-#define MT_WTBL_TXRX_RATE_G2_HE		24
-#define MT_WTBL_TXRX_RATE_G2		12
-
-#define MT_WTBL_AC0_CTT_OFFSET		20
-
-static inline u32 mt7921_mac_wtbl_lmac_addr(int idx, u8 offset)
-{
-	return MT_WTBL_LMAC_OFFS(idx, 0) + offset * 4;
-}
-
-#endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c
index 42933a6b7334..3b6adb29cbef 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c
@@ -9,27 +9,6 @@
 #include "mt7921.h"
 #include "mcu.h"
 
-static void
-mt7921_gen_ppe_thresh(u8 *he_ppet, int nss)
-{
-	u8 i, ppet_bits, ppet_size, ru_bit_mask = 0x7; /* HE80 */
-	static const u8 ppet16_ppet8_ru3_ru0[] = {0x1c, 0xc7, 0x71};
-
-	he_ppet[0] = FIELD_PREP(IEEE80211_PPE_THRES_NSS_MASK, nss - 1) |
-		     FIELD_PREP(IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK,
-				ru_bit_mask);
-
-	ppet_bits = IEEE80211_PPE_THRES_INFO_PPET_SIZE *
-		    nss * hweight8(ru_bit_mask) * 2;
-	ppet_size = DIV_ROUND_UP(ppet_bits, 8);
-
-	for (i = 0; i < ppet_size - 1; i++)
-		he_ppet[i + 1] = ppet16_ppet8_ru3_ru0[i % 3];
-
-	he_ppet[i + 1] = ppet16_ppet8_ru3_ru0[i % 3] &
-			 (0xff >> (8 - (ppet_bits - 1) % 8));
-}
-
 static int
 mt7921_init_he_caps(struct mt7921_phy *phy, enum nl80211_band band,
 		    struct ieee80211_sband_iftype_data *data)
@@ -168,7 +147,7 @@ mt7921_init_he_caps(struct mt7921_phy *phy, enum nl80211_band band,
 		memset(he_cap->ppe_thres, 0, sizeof(he_cap->ppe_thres));
 		if (he_cap_elem->phy_cap_info[6] &
 		    IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
-			mt7921_gen_ppe_thresh(he_cap->ppe_thres, nss);
+			mt76_connac_gen_ppe_thresh(he_cap->ppe_thres, nss);
 		} else {
 			he_cap_elem->phy_cap_info[9] |=
 				u8_encode_bits(IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US,
@@ -702,10 +681,25 @@ static void mt7921_configure_filter(struct ieee80211_hw *hw,
 				    unsigned int *total_flags,
 				    u64 multicast)
 {
+#define MT7921_FILTER_FCSFAIL    BIT(2)
+#define MT7921_FILTER_CONTROL    BIT(5)
+#define MT7921_FILTER_OTHER_BSS  BIT(6)
+#define MT7921_FILTER_ENABLE     BIT(31)
+
 	struct mt7921_dev *dev = mt7921_hw_dev(hw);
+	u32 flags = MT7921_FILTER_ENABLE;
+
+#define MT7921_FILTER(_fif, _type) do {			\
+		if (*total_flags & (_fif))		\
+			flags |= MT7921_FILTER_##_type;	\
+	} while (0)
+
+	MT7921_FILTER(FIF_FCSFAIL, FCSFAIL);
+	MT7921_FILTER(FIF_CONTROL, CONTROL);
+	MT7921_FILTER(FIF_OTHER_BSS, OTHER_BSS);
 
 	mt7921_mutex_acquire(dev);
-	mt7921_mcu_set_rxfilter(dev, *total_flags, 0, 0);
+	mt7921_mcu_set_rxfilter(dev, flags, 0, 0);
 	mt7921_mutex_release(dev);
 
 	*total_flags &= (FIF_OTHER_BSS | FIF_FCSFAIL | FIF_CONTROL);
@@ -1694,7 +1688,7 @@ static void mt7921_ctx_iter(void *priv, u8 *mac,
 	if (ctx != mvif->ctx)
 		return;
 
-	if (vif->type & NL80211_IFTYPE_MONITOR)
+	if (vif->type == NL80211_IFTYPE_MONITOR)
 		mt7921_mcu_config_sniffer(mvif, ctx);
 	else
 		mt76_connac_mcu_uni_set_chctx(mvif->phy->mt76, &mvif->mt76, ctx);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
index c5e7ad06f877..c69ce6df4956 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
@@ -5,9 +5,8 @@
 #include <linux/firmware.h>
 #include "mt7921.h"
 #include "mt7921_trace.h"
-#include "eeprom.h"
 #include "mcu.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 
 #define MT_STA_BFER			BIT(0)
 #define MT_STA_BFEE			BIT(1)
@@ -16,24 +15,6 @@ static bool mt7921_disable_clc;
 module_param_named(disable_clc, mt7921_disable_clc, bool, 0644);
 MODULE_PARM_DESC(disable_clc, "disable CLC support");
 
-static int
-mt7921_mcu_parse_eeprom(struct mt76_dev *dev, struct sk_buff *skb)
-{
-	struct mt7921_mcu_eeprom_info *res;
-	u8 *buf;
-
-	if (!skb)
-		return -EINVAL;
-
-	skb_pull(skb, sizeof(struct mt76_connac2_mcu_rxd));
-
-	res = (struct mt7921_mcu_eeprom_info *)skb->data;
-	buf = dev->eeprom.data + le32_to_cpu(res->addr);
-	memcpy(buf, res->data, 16);
-
-	return 0;
-}
-
 int mt7921_mcu_parse_response(struct mt76_dev *mdev, int cmd,
 			      struct sk_buff *skb, int seq)
 {
@@ -60,27 +41,25 @@ int mt7921_mcu_parse_response(struct mt76_dev *mdev, int cmd,
 	} else if (cmd == MCU_EXT_CMD(THERMAL_CTRL)) {
 		skb_pull(skb, sizeof(*rxd) + 4);
 		ret = le32_to_cpu(*(__le32 *)skb->data);
-	} else if (cmd == MCU_EXT_CMD(EFUSE_ACCESS)) {
-		ret = mt7921_mcu_parse_eeprom(mdev, skb);
 	} else if (cmd == MCU_UNI_CMD(DEV_INFO_UPDATE) ||
 		   cmd == MCU_UNI_CMD(BSS_INFO_UPDATE) ||
 		   cmd == MCU_UNI_CMD(STA_REC_UPDATE) ||
 		   cmd == MCU_UNI_CMD(HIF_CTRL) ||
 		   cmd == MCU_UNI_CMD(OFFLOAD) ||
 		   cmd == MCU_UNI_CMD(SUSPEND)) {
-		struct mt7921_mcu_uni_event *event;
+		struct mt76_connac_mcu_uni_event *event;
 
 		skb_pull(skb, sizeof(*rxd));
-		event = (struct mt7921_mcu_uni_event *)skb->data;
+		event = (struct mt76_connac_mcu_uni_event *)skb->data;
 		ret = le32_to_cpu(event->status);
 		/* skip invalid event */
 		if (mcu_cmd != event->cid)
 			ret = -EAGAIN;
 	} else if (cmd == MCU_CE_QUERY(REG_READ)) {
-		struct mt7921_mcu_reg_event *event;
+		struct mt76_connac_mcu_reg_event *event;
 
 		skb_pull(skb, sizeof(*rxd));
-		event = (struct mt7921_mcu_reg_event *)skb->data;
+		event = (struct mt76_connac_mcu_reg_event *)skb->data;
 		ret = (int)le32_to_cpu(event->val);
 	} else {
 		skb_pull(skb, sizeof(struct mt76_connac2_mcu_rxd));
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h
index 96dc870fd35e..9b0aa3b70f0e 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h
@@ -50,22 +50,11 @@ struct mt7921_mcu_eeprom_info {
 #define MT_RA_RATE_DCM_EN		BIT(4)
 #define MT_RA_RATE_BW			GENMASK(14, 13)
 
-struct mt7921_mcu_uni_event {
-	u8 cid;
-	u8 pad[3];
-	__le32 status; /* 0: success, others: fail */
-} __packed;
-
 enum {
 	MT_EBF = BIT(0),	/* explicit beamforming */
 	MT_IBF = BIT(1)		/* implicit beamforming */
 };
 
-struct mt7921_mcu_reg_event {
-	__le32 reg;
-	__le32 val;
-} __packed;
-
 struct mt7921_mcu_ant_id_config {
 	u8 ant_id[4];
 } __packed;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h
index 1af70dac723b..149acb1662d5 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h
@@ -266,6 +266,17 @@ struct mt7921_phy {
 	bool roc_grant;
 };
 
+enum mt7921_eeprom_field {
+	MT_EE_CHIP_ID =		0x000,
+	MT_EE_VERSION =		0x002,
+	MT_EE_MAC_ADDR =	0x004,
+	MT_EE_WIFI_CONF =	0x07c,
+	MT_EE_HW_TYPE =		0x55b,
+	__MT_EE_MAX =		0x9ff
+};
+
+#define MT_EE_HW_TYPE_ENCAP			BIT(0)
+
 #define mt7921_init_reset(dev)		((dev)->hif_ops->init_reset(dev))
 #define mt7921_dev_reset(dev)		((dev)->hif_ops->reset(dev))
 #define mt7921_mcu_init(dev)		((dev)->hif_ops->mcu_init(dev))
@@ -287,7 +298,6 @@ struct mt7921_dev {
 
 	const struct mt76_bus_ops *bus_ops;
 	struct mt7921_phy phy;
-	struct tasklet_struct irq_tasklet;
 
 	struct work_struct reset_work;
 	bool hw_full_reset:1;
@@ -391,13 +401,6 @@ void mt7921_mcu_rx_event(struct mt7921_dev *dev, struct sk_buff *skb);
 int mt7921_mcu_set_rxfilter(struct mt7921_dev *dev, u32 fif,
 			    u8 bit_op, u32 bit_map);
 
-static inline void mt7921_irq_enable(struct mt7921_dev *dev, u32 mask)
-{
-	mt76_set_irq_mask(&dev->mt76, 0, 0, mask);
-
-	tasklet_schedule(&dev->irq_tasklet);
-}
-
 static inline u32
 mt7921_reg_map_l1(struct mt7921_dev *dev, u32 addr)
 {
@@ -478,7 +481,6 @@ void mt7921_tx_token_put(struct mt7921_dev *dev);
 bool mt7921_rx_check(struct mt76_dev *mdev, void *data, int len);
 void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
 			 struct sk_buff *skb, u32 *info);
-void mt7921_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps);
 void mt7921_stats_work(struct work_struct *work);
 void mt7921_set_stream_he_caps(struct mt7921_phy *phy);
 void mt7921_update_channel(struct mt76_phy *mphy);
@@ -593,5 +595,6 @@ int mt7921_mcu_set_roc(struct mt7921_phy *phy, struct mt7921_vif *vif,
 		       enum mt7921_roc_req type, u8 token_id);
 int mt7921_mcu_abort_roc(struct mt7921_phy *phy, struct mt7921_vif *vif,
 			 u8 token_id);
-u8 mt7921_check_offload_capability(struct device *dev, const char *fw_wm);
+struct ieee80211_ops *mt7921_get_mac80211_ops(struct device *dev,
+					      void *drv_data, u8 *fw_features);
 #endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c
index 5c23c827abe4..ddb1fa4ee01d 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c
@@ -8,7 +8,7 @@
 #include <linux/pci.h>
 
 #include "mt7921.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 #include "mcu.h"
 #include "../trace.h"
 
@@ -31,14 +31,12 @@ MODULE_PARM_DESC(disable_aspm, "disable PCI ASPM support");
 static void
 mt7921_rx_poll_complete(struct mt76_dev *mdev, enum mt76_rxq_id q)
 {
-	struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
-
 	if (q == MT_RXQ_MAIN)
-		mt7921_irq_enable(dev, MT_INT_RX_DONE_DATA);
+		mt76_connac_irq_enable(mdev, MT_INT_RX_DONE_DATA);
 	else if (q == MT_RXQ_MCU_WA)
-		mt7921_irq_enable(dev, MT_INT_RX_DONE_WM2);
+		mt76_connac_irq_enable(mdev, MT_INT_RX_DONE_WM2);
 	else
-		mt7921_irq_enable(dev, MT_INT_RX_DONE_WM);
+		mt76_connac_irq_enable(mdev, MT_INT_RX_DONE_WM);
 }
 
 static irqreturn_t mt7921_irq_handler(int irq, void *dev_instance)
@@ -50,7 +48,7 @@ static irqreturn_t mt7921_irq_handler(int irq, void *dev_instance)
 	if (!test_bit(MT76_STATE_INITIALIZED, &dev->mphy.state))
 		return IRQ_NONE;
 
-	tasklet_schedule(&dev->irq_tasklet);
+	tasklet_schedule(&dev->mt76.irq_tasklet);
 
 	return IRQ_HANDLED;
 }
@@ -115,14 +113,15 @@ static void mt7921e_unregister_device(struct mt7921_dev *dev)
 		napi_disable(&dev->mt76.napi[i]);
 	cancel_delayed_work_sync(&pm->ps_work);
 	cancel_work_sync(&pm->wake_work);
+	cancel_work_sync(&dev->reset_work);
 
 	mt7921_tx_token_put(dev);
-	mt7921_mcu_drv_pmctrl(dev);
+	__mt7921_mcu_drv_pmctrl(dev);
 	mt7921_dma_cleanup(dev);
 	mt7921_wfsys_reset(dev);
 	skb_queue_purge(&dev->mt76.mcu.res_q);
 
-	tasklet_disable(&dev->irq_tasklet);
+	tasklet_disable(&dev->mt76.irq_tasklet);
 }
 
 static u32 __mt7921_reg_addr(struct mt7921_dev *dev, u32 addr)
@@ -243,7 +242,6 @@ static int mt7921_pci_probe(struct pci_dev *pdev,
 		.rx_check = mt7921_rx_check,
 		.rx_skb = mt7921_queue_rx_skb,
 		.rx_poll_complete = mt7921_rx_poll_complete,
-		.sta_ps = mt7921_sta_ps,
 		.sta_add = mt7921_mac_sta_add,
 		.sta_assoc = mt7921_mac_sta_assoc,
 		.sta_remove = mt7921_mac_sta_remove,
@@ -256,13 +254,13 @@ static int mt7921_pci_probe(struct pci_dev *pdev,
 		.drv_own = mt7921e_mcu_drv_pmctrl,
 		.fw_own = mt7921e_mcu_fw_pmctrl,
 	};
-
 	struct ieee80211_ops *ops;
 	struct mt76_bus_ops *bus_ops;
 	struct mt7921_dev *dev;
 	struct mt76_dev *mdev;
 	u8 features;
 	int ret;
+	u16 cmd;
 
 	ret = pcim_enable_device(pdev);
 	if (ret)
@@ -272,6 +270,11 @@ static int mt7921_pci_probe(struct pci_dev *pdev,
 	if (ret)
 		return ret;
 
+	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
+	if (!(cmd & PCI_COMMAND_MEMORY)) {
+		cmd |= PCI_COMMAND_MEMORY;
+		pci_write_config_word(pdev, PCI_COMMAND, cmd);
+	}
 	pci_set_master(pdev);
 
 	ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
@@ -285,27 +288,13 @@ static int mt7921_pci_probe(struct pci_dev *pdev,
 	if (mt7921_disable_aspm)
 		mt76_pci_disable_aspm(pdev);
 
-	features = mt7921_check_offload_capability(&pdev->dev, (const char *)
-						   id->driver_data);
-	ops = devm_kmemdup(&pdev->dev, &mt7921_ops, sizeof(mt7921_ops),
-			   GFP_KERNEL);
+	ops = mt7921_get_mac80211_ops(&pdev->dev, (void *)id->driver_data,
+				      &features);
 	if (!ops) {
 		ret = -ENOMEM;
 		goto err_free_pci_vec;
 	}
 
-	if (!(features & MT7921_FW_CAP_CNM)) {
-		ops->remain_on_channel = NULL;
-		ops->cancel_remain_on_channel = NULL;
-		ops->add_chanctx = NULL;
-		ops->remove_chanctx = NULL;
-		ops->change_chanctx = NULL;
-		ops->assign_vif_chanctx = NULL;
-		ops->unassign_vif_chanctx = NULL;
-		ops->mgd_prepare_tx = NULL;
-		ops->mgd_complete_tx = NULL;
-	}
-
 	mdev = mt76_alloc_device(&pdev->dev, sizeof(*dev), ops, &drv_ops);
 	if (!mdev) {
 		ret = -ENOMEM;
@@ -318,7 +307,7 @@ static int mt7921_pci_probe(struct pci_dev *pdev,
 	dev->fw_features = features;
 	dev->hif_ops = &mt7921_pcie_ops;
 	mt76_mmio_init(&dev->mt76, pcim_iomap_table(pdev)[0]);
-	tasklet_init(&dev->irq_tasklet, mt7921_irq_tasklet, (unsigned long)dev);
+	tasklet_init(&mdev->irq_tasklet, mt7921_irq_tasklet, (unsigned long)dev);
 
 	dev->phy.dev = dev;
 	dev->phy.mt76 = &dev->mt76.phy;
@@ -430,7 +419,7 @@ static int mt7921_pci_suspend(struct device *device)
 	mt76_wr(dev, MT_WFDMA0_HOST_INT_ENA, 0);
 	mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0x0);
 	synchronize_irq(pdev->irq);
-	tasklet_kill(&dev->irq_tasklet);
+	tasklet_kill(&mdev->irq_tasklet);
 
 	err = mt7921_mcu_fw_pmctrl(dev);
 	if (err)
@@ -474,8 +463,9 @@ static int mt7921_pci_resume(struct device *device)
 
 	/* enable interrupt */
 	mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff);
-	mt7921_irq_enable(dev, MT_INT_RX_DONE_ALL | MT_INT_TX_DONE_ALL |
-			  MT_INT_MCU_CMD);
+	mt76_connac_irq_enable(&dev->mt76,
+			       MT_INT_RX_DONE_ALL | MT_INT_TX_DONE_ALL |
+			       MT_INT_MCU_CMD);
 	mt76_set(dev, MT_MCU2HOST_SW_INT_ENA, MT_MCU_CMD_WAKE_RX_PCIE);
 
 	/* put dma enabled */
@@ -509,17 +499,7 @@ failed:
 
 static void mt7921_pci_shutdown(struct pci_dev *pdev)
 {
-	struct mt76_dev *mdev = pci_get_drvdata(pdev);
-	struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
-	struct mt76_connac_pm *pm = &dev->pm;
-
-	cancel_delayed_work_sync(&pm->ps_work);
-	cancel_work_sync(&pm->wake_work);
-
-	/* chip cleanup before reboot */
-	mt7921_mcu_drv_pmctrl(dev);
-	mt7921_dma_cleanup(dev);
-	mt7921_wfsys_reset(dev);
+	mt7921_pci_remove(pdev);
 }
 
 static DEFINE_SIMPLE_DEV_PM_OPS(mt7921_pm_ops, mt7921_pci_suspend, mt7921_pci_resume);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c
index 8dd60408b117..6053a2556c20 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c
@@ -3,7 +3,7 @@
 
 #include "mt7921.h"
 #include "../dma.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 
 int mt7921e_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr,
 			   enum mt76_txq_id qid, struct mt76_wcid *wcid,
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c b/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c
index 8ce4252b8ae7..a77a309c0d60 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c
@@ -13,7 +13,7 @@
 
 #include "mt7921.h"
 #include "../sdio.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 #include "mcu.h"
 
 static const struct sdio_device_id mt7921s_table[] = {
@@ -99,7 +99,6 @@ static int mt7921s_probe(struct sdio_func *func,
 		.tx_status_data = mt7921_usb_sdio_tx_status_data,
 		.rx_skb = mt7921_queue_rx_skb,
 		.rx_check = mt7921_rx_check,
-		.sta_ps = mt7921_sta_ps,
 		.sta_add = mt7921_mac_sta_add,
 		.sta_assoc = mt7921_mac_sta_assoc,
 		.sta_remove = mt7921_mac_sta_remove,
@@ -122,33 +121,17 @@ static int mt7921s_probe(struct sdio_func *func,
 		.drv_own = mt7921s_mcu_drv_pmctrl,
 		.fw_own = mt7921s_mcu_fw_pmctrl,
 	};
-
 	struct ieee80211_ops *ops;
 	struct mt7921_dev *dev;
 	struct mt76_dev *mdev;
 	u8 features;
 	int ret;
 
-	features = mt7921_check_offload_capability(&func->dev, (const char *)
-						   id->driver_data);
-
-	ops = devm_kmemdup(&func->dev, &mt7921_ops, sizeof(mt7921_ops),
-			   GFP_KERNEL);
+	ops = mt7921_get_mac80211_ops(&func->dev, (void *)id->driver_data,
+				      &features);
 	if (!ops)
 		return -ENOMEM;
 
-	if (!(features & MT7921_FW_CAP_CNM)) {
-		ops->remain_on_channel = NULL;
-		ops->cancel_remain_on_channel = NULL;
-		ops->add_chanctx = NULL;
-		ops->remove_chanctx = NULL;
-		ops->change_chanctx = NULL;
-		ops->assign_vif_chanctx = NULL;
-		ops->unassign_vif_chanctx = NULL;
-		ops->mgd_prepare_tx = NULL;
-		ops->mgd_complete_tx = NULL;
-	}
-
 	mdev = mt76_alloc_device(&func->dev, sizeof(*dev), ops, &drv_ops);
 	if (!mdev)
 		return -ENOMEM;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mac.c
index 1b3adb3d91e8..cff9925c41ea 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mac.c
@@ -4,7 +4,7 @@
 #include <linux/iopoll.h>
 #include <linux/mmc/sdio_func.h>
 #include "mt7921.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 #include "../sdio.h"
 
 static void mt7921s_enable_irq(struct mt76_dev *dev)
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c
index 5c1489766d9f..177679ce1c80 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c
@@ -8,7 +8,7 @@
 
 #include "mt7921.h"
 #include "../sdio.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 #include "mcu.h"
 #include "regs.h"
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/usb.c b/drivers/net/wireless/mediatek/mt76/mt7921/usb.c
index 8fef09ed29c9..1f302c430339 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/usb.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/usb.c
@@ -10,7 +10,7 @@
 
 #include "mt7921.h"
 #include "mcu.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 
 static const struct usb_device_id mt7921u_device_table[] = {
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x0e8d, 0x7961, 0xff, 0xff, 0xff),
@@ -18,6 +18,9 @@ static const struct usb_device_id mt7921u_device_table[] = {
 	/* Comfast CF-952AX */
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x3574, 0x6211, 0xff, 0xff, 0xff),
 		.driver_info = (kernel_ulong_t)MT7921_FIRMWARE_WM },
+	/* Netgear, Inc. [A8000,AXE3000] */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0846, 0x9060, 0xff, 0xff, 0xff),
+		.driver_info = (kernel_ulong_t)MT7921_FIRMWARE_WM },
 	{ },
 };
 
@@ -183,7 +186,6 @@ static int mt7921u_probe(struct usb_interface *usb_intf,
 		.tx_status_data = mt7921_usb_sdio_tx_status_data,
 		.rx_skb = mt7921_queue_rx_skb,
 		.rx_check = mt7921_rx_check,
-		.sta_ps = mt7921_sta_ps,
 		.sta_add = mt7921_mac_sta_add,
 		.sta_assoc = mt7921_mac_sta_assoc,
 		.sta_remove = mt7921_mac_sta_remove,
@@ -210,27 +212,12 @@ static int mt7921u_probe(struct usb_interface *usb_intf,
 	u8 features;
 	int ret;
 
-	features = mt7921_check_offload_capability(&usb_intf->dev, (const char *)
-						   id->driver_info);
-	ops = devm_kmemdup(&usb_intf->dev, &mt7921_ops, sizeof(mt7921_ops),
-			   GFP_KERNEL);
+	ops = mt7921_get_mac80211_ops(&usb_intf->dev, (void *)id->driver_info,
+				      &features);
 	if (!ops)
 		return -ENOMEM;
 
-	if (!(features & MT7921_FW_CAP_CNM)) {
-		ops->remain_on_channel = NULL;
-		ops->cancel_remain_on_channel = NULL;
-		ops->add_chanctx = NULL;
-		ops->remove_chanctx = NULL;
-		ops->change_chanctx = NULL;
-		ops->assign_vif_chanctx = NULL;
-		ops->unassign_vif_chanctx = NULL;
-		ops->mgd_prepare_tx = NULL;
-		ops->mgd_complete_tx = NULL;
-	}
-
 	ops->stop = mt7921u_stop;
-
 	mdev = mt76_alloc_device(&usb_intf->dev, sizeof(*dev), ops, &drv_ops);
 	if (!mdev)
 		return -ENOMEM;
@@ -272,7 +259,7 @@ static int mt7921u_probe(struct usb_interface *usb_intf,
 
 	ret = mt7921u_dma_init(dev, false);
 	if (ret)
-		return ret;
+		goto error;
 
 	hw = mt76_hw(dev);
 	/* check hw sg support in order to enable AMSDU */
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/usb_mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/usb_mac.c
index efbd3954c883..50eb6e7fd6b5 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/usb_mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/usb_mac.c
@@ -10,7 +10,7 @@
 
 #include "mt7921.h"
 #include "mcu.h"
-#include "mac.h"
+#include "../mt76_connac2_mac.h"
 
 static u32 mt7921u_uhw_rr(struct mt76_dev *dev, u32 addr)
 {
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/Kconfig b/drivers/net/wireless/mediatek/mt76/mt7996/Kconfig
index 79fb47a73c91..1afa2f662e47 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/Kconfig
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/Kconfig
@@ -2,6 +2,7 @@
 config MT7996E
 	tristate "MediaTek MT7996 (PCIe) support"
 	select MT76_CONNAC_LIB
+	select WANT_DEV_COREDUMP
 	select RELAY
 	depends on MAC80211
 	depends on PCI
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/Makefile b/drivers/net/wireless/mediatek/mt76/mt7996/Makefile
index bcb9a3c53149..07c8b555c1ac 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/Makefile
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/Makefile
@@ -4,3 +4,5 @@ obj-$(CONFIG_MT7996E) += mt7996e.o
 
 mt7996e-y := pci.o init.o dma.o eeprom.o main.o mcu.o mac.o \
 	     debugfs.o mmio.o
+
+mt7996e-$(CONFIG_DEV_COREDUMP) += coredump.o
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/coredump.c b/drivers/net/wireless/mediatek/mt76/mt7996/coredump.c
new file mode 100644
index 000000000000..ccab0d7b9be4
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/coredump.c
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: ISC
+/* Copyright (C) 2023 MediaTek Inc. */
+
+#include <linux/devcoredump.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/utsname.h>
+#include "coredump.h"
+
+static bool coredump_memdump;
+module_param(coredump_memdump, bool, 0644);
+MODULE_PARM_DESC(coredump_memdump, "Optional ability to dump firmware memory");
+
+static const struct mt7996_mem_region mt7996_mem_regions[] = {
+	{
+		.start = 0x00800000,
+		.len = 0x0004ffff,
+		.name = "ULM0",
+	},
+	{
+		.start = 0x00900000,
+		.len = 0x00037fff,
+		.name = "ULM1",
+	},
+	{
+		.start = 0x02200000,
+		.len = 0x0003ffff,
+		.name = "ULM2",
+	},
+	{
+		.start = 0x00400000,
+		.len = 0x00067fff,
+		.name = "SRAM",
+	},
+	{
+		.start = 0xe0000000,
+		.len = 0x0015ffff,
+		.name = "CRAM0",
+	},
+	{
+		.start = 0xe0160000,
+		.len = 0x0011bfff,
+		.name = "CRAM1",
+	},
+};
+
+const struct mt7996_mem_region*
+mt7996_coredump_get_mem_layout(struct mt7996_dev *dev, u32 *num)
+{
+	switch (mt76_chip(&dev->mt76)) {
+	case 0x7990:
+	case 0x7991:
+		*num = ARRAY_SIZE(mt7996_mem_regions);
+		return &mt7996_mem_regions[0];
+	default:
+		return NULL;
+	}
+}
+
+static int mt7996_coredump_get_mem_size(struct mt7996_dev *dev)
+{
+	const struct mt7996_mem_region *mem_region;
+	size_t size = 0;
+	u32 num;
+	int i;
+
+	mem_region = mt7996_coredump_get_mem_layout(dev, &num);
+	if (!mem_region)
+		return 0;
+
+	for (i = 0; i < num; i++) {
+		size += mem_region->len;
+		mem_region++;
+	}
+
+	/* reserve space for the headers */
+	size += num * sizeof(struct mt7996_mem_hdr);
+	/* make sure it is aligned 4 bytes for debug message print out */
+	size = ALIGN(size, 4);
+
+	return size;
+}
+
+struct mt7996_crash_data *mt7996_coredump_new(struct mt7996_dev *dev)
+{
+	struct mt7996_crash_data *crash_data = dev->coredump.crash_data;
+
+	lockdep_assert_held(&dev->dump_mutex);
+
+	if (coredump_memdump &&
+	    !mt76_poll_msec(dev, MT_FW_DUMP_STATE, 0x3, 0x2, 500))
+		return NULL;
+
+	guid_gen(&crash_data->guid);
+	ktime_get_real_ts64(&crash_data->timestamp);
+
+	return crash_data;
+}
+
+static void
+mt7996_coredump_fw_state(struct mt7996_dev *dev, struct mt7996_coredump *dump,
+			 bool *exception)
+{
+	u32 count;
+
+	count = mt76_rr(dev, MT_FW_ASSERT_CNT);
+
+	/* normal mode: driver can manually trigger assert for detail info */
+	if (!count)
+		strscpy(dump->fw_state, "normal", sizeof(dump->fw_state));
+	else
+		strscpy(dump->fw_state, "exception", sizeof(dump->fw_state));
+
+	*exception = !!count;
+}
+
+static void
+mt7996_coredump_fw_stack(struct mt7996_dev *dev, struct mt7996_coredump *dump,
+			 bool exception)
+{
+	u32 oldest, i, idx;
+
+	strscpy(dump->pc_current, "program counter", sizeof(dump->pc_current));
+
+	/* 0: WM PC log output */
+	mt76_wr(dev, MT_CONN_DBG_CTL_OUT_SEL, 0);
+	/* choose 33th PC log buffer to read current PC index */
+	mt76_wr(dev, MT_CONN_DBG_CTL_PC_LOG_SEL, 0x3f);
+
+	/* read current PC */
+	dump->pc_stack[0] = mt76_rr(dev, MT_CONN_DBG_CTL_PC_LOG);
+
+	/* stop call stack record */
+	if (!exception) {
+		mt76_clear(dev, MT_MCU_WM_EXCP_PC_CTRL, BIT(0));
+		mt76_clear(dev, MT_MCU_WM_EXCP_LR_CTRL, BIT(0));
+	}
+
+	oldest = (u32)mt76_get_field(dev, MT_MCU_WM_EXCP_PC_CTRL,
+				     GENMASK(20, 16)) + 2;
+	for (i = 0; i < 16; i++) {
+		idx = ((oldest + 2 * i + 1) % 32);
+		dump->pc_stack[i + 1] =
+			mt76_rr(dev, MT_MCU_WM_EXCP_PC_LOG + idx * 4);
+	}
+
+	oldest = (u32)mt76_get_field(dev, MT_MCU_WM_EXCP_LR_CTRL,
+				     GENMASK(20, 16)) + 2;
+	for (i = 0; i < 16; i++) {
+		idx = ((oldest + 2 * i + 1) % 32);
+		dump->lr_stack[i] =
+			mt76_rr(dev, MT_MCU_WM_EXCP_LR_LOG + idx * 4);
+	}
+
+	/* start call stack record */
+	if (!exception) {
+		mt76_set(dev, MT_MCU_WM_EXCP_PC_CTRL, BIT(0));
+		mt76_set(dev, MT_MCU_WM_EXCP_LR_CTRL, BIT(0));
+	}
+}
+
+static struct mt7996_coredump *mt7996_coredump_build(struct mt7996_dev *dev)
+{
+	struct mt7996_crash_data *crash_data = dev->coredump.crash_data;
+	struct mt7996_coredump *dump;
+	struct mt7996_coredump_mem *dump_mem;
+	size_t len, sofar = 0, hdr_len = sizeof(*dump);
+	unsigned char *buf;
+	bool exception;
+
+	len = hdr_len;
+
+	if (coredump_memdump && crash_data->memdump_buf_len)
+		len += sizeof(*dump_mem) + crash_data->memdump_buf_len;
+
+	sofar += hdr_len;
+
+	/* this is going to get big when we start dumping memory and such,
+	 * so go ahead and use vmalloc.
+	 */
+	buf = vzalloc(len);
+	if (!buf)
+		return NULL;
+
+	mutex_lock(&dev->dump_mutex);
+
+	dump = (struct mt7996_coredump *)(buf);
+	dump->len = len;
+
+	/* plain text */
+	strscpy(dump->magic, "mt76-crash-dump", sizeof(dump->magic));
+	strscpy(dump->kernel, init_utsname()->release, sizeof(dump->kernel));
+	strscpy(dump->fw_ver, dev->mt76.hw->wiphy->fw_version,
+		sizeof(dump->fw_ver));
+
+	guid_copy(&dump->guid, &crash_data->guid);
+	dump->tv_sec = crash_data->timestamp.tv_sec;
+	dump->tv_nsec = crash_data->timestamp.tv_nsec;
+	dump->device_id = mt76_chip(&dev->mt76);
+
+	mt7996_coredump_fw_state(dev, dump, &exception);
+	mt7996_coredump_fw_stack(dev, dump, exception);
+
+	/* gather memory content */
+	dump_mem = (struct mt7996_coredump_mem *)(buf + sofar);
+	dump_mem->len = crash_data->memdump_buf_len;
+	if (coredump_memdump && crash_data->memdump_buf_len)
+		memcpy(dump_mem->data, crash_data->memdump_buf,
+		       crash_data->memdump_buf_len);
+
+	mutex_unlock(&dev->dump_mutex);
+
+	return dump;
+}
+
+int mt7996_coredump_submit(struct mt7996_dev *dev)
+{
+	struct mt7996_coredump *dump;
+
+	dump = mt7996_coredump_build(dev);
+	if (!dump) {
+		dev_warn(dev->mt76.dev, "no crash dump data found\n");
+		return -ENODATA;
+	}
+
+	dev_coredumpv(dev->mt76.dev, dump, dump->len, GFP_KERNEL);
+
+	return 0;
+}
+
+int mt7996_coredump_register(struct mt7996_dev *dev)
+{
+	struct mt7996_crash_data *crash_data;
+
+	crash_data = vzalloc(sizeof(*dev->coredump.crash_data));
+	if (!crash_data)
+		return -ENOMEM;
+
+	dev->coredump.crash_data = crash_data;
+
+	if (coredump_memdump) {
+		crash_data->memdump_buf_len = mt7996_coredump_get_mem_size(dev);
+		if (!crash_data->memdump_buf_len)
+			/* no memory content */
+			return 0;
+
+		crash_data->memdump_buf = vzalloc(crash_data->memdump_buf_len);
+		if (!crash_data->memdump_buf) {
+			vfree(crash_data);
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
+void mt7996_coredump_unregister(struct mt7996_dev *dev)
+{
+	if (dev->coredump.crash_data->memdump_buf) {
+		vfree(dev->coredump.crash_data->memdump_buf);
+		dev->coredump.crash_data->memdump_buf = NULL;
+		dev->coredump.crash_data->memdump_buf_len = 0;
+	}
+
+	vfree(dev->coredump.crash_data);
+	dev->coredump.crash_data = NULL;
+}
+
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/coredump.h b/drivers/net/wireless/mediatek/mt76/mt7996/coredump.h
new file mode 100644
index 000000000000..af2ba219b1b5
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/coredump.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: ISC */
+/* Copyright (C) 2023 MediaTek Inc. */
+
+#ifndef _COREDUMP_H_
+#define _COREDUMP_H_
+
+#include "mt7996.h"
+
+struct mt7996_coredump {
+	char magic[16];
+
+	u32 len;
+
+	guid_t guid;
+
+	/* time-of-day stamp */
+	u64 tv_sec;
+	/* time-of-day stamp, nano-seconds */
+	u64 tv_nsec;
+	/* kernel version */
+	char kernel[64];
+	/* firmware version */
+	char fw_ver[ETHTOOL_FWVERS_LEN];
+
+	u32 device_id;
+
+	/* exception state */
+	char fw_state[12];
+
+	/* program counters */
+	char pc_current[16];
+	u32 pc_stack[17];
+	/* link registers */
+	u32 lr_stack[16];
+
+	/* memory content */
+	u8 data[];
+} __packed;
+
+struct mt7996_coredump_mem {
+	u32 len;
+	u8 data[];
+} __packed;
+
+struct mt7996_mem_hdr {
+	u32 start;
+	u32 len;
+	u8 data[];
+};
+
+struct mt7996_mem_region {
+	u32 start;
+	size_t len;
+
+	const char *name;
+};
+
+#ifdef CONFIG_DEV_COREDUMP
+
+const struct mt7996_mem_region *
+mt7996_coredump_get_mem_layout(struct mt7996_dev *dev, u32 *num);
+struct mt7996_crash_data *mt7996_coredump_new(struct mt7996_dev *dev);
+int mt7996_coredump_submit(struct mt7996_dev *dev);
+int mt7996_coredump_register(struct mt7996_dev *dev);
+void mt7996_coredump_unregister(struct mt7996_dev *dev);
+
+#else /* CONFIG_DEV_COREDUMP */
+
+static inline const struct mt7996_mem_region *
+mt7996_coredump_get_mem_layout(struct mt7996_dev *dev, u32 *num)
+{
+	return NULL;
+}
+
+static inline int mt7996_coredump_submit(struct mt7996_dev *dev)
+{
+	return 0;
+}
+
+static inline struct
+mt7996_crash_data *mt7996_coredump_new(struct mt7996_dev *dev)
+{
+	return NULL;
+}
+
+static inline int mt7996_coredump_register(struct mt7996_dev *dev)
+{
+	return 0;
+}
+
+static inline void mt7996_coredump_unregister(struct mt7996_dev *dev)
+{
+}
+
+#endif /* CONFIG_DEV_COREDUMP */
+
+#endif /* _COREDUMP_H_ */
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c
index 9c5e9ac1c335..513ab4ba41c9 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c
@@ -48,12 +48,12 @@ DEFINE_DEBUGFS_ATTRIBUTE(fops_implicit_txbf, mt7996_implicit_txbf_get,
 
 /* test knob of system error recovery */
 static ssize_t
-mt7996_fw_ser_set(struct file *file, const char __user *user_buf,
-		  size_t count, loff_t *ppos)
+mt7996_sys_recovery_set(struct file *file, const char __user *user_buf,
+			size_t count, loff_t *ppos)
 {
 	struct mt7996_phy *phy = file->private_data;
 	struct mt7996_dev *dev = phy->dev;
-	u8 band_idx = phy->mt76->band_idx;
+	bool band = phy->mt76->band_idx;
 	char buf[16];
 	int ret = 0;
 	u16 val;
@@ -73,17 +73,47 @@ mt7996_fw_ser_set(struct file *file, const char __user *user_buf,
 		return -EINVAL;
 
 	switch (val) {
-	case SER_SET_RECOVER_L1:
-	case SER_SET_RECOVER_L2:
-	case SER_SET_RECOVER_L3_RX_ABORT:
-	case SER_SET_RECOVER_L3_TX_ABORT:
-	case SER_SET_RECOVER_L3_TX_DISABLE:
-	case SER_SET_RECOVER_L3_BF:
-		ret = mt7996_mcu_set_ser(dev, SER_ENABLE, BIT(val), band_idx);
+	/*
+	 * 0: grab firmware current SER state.
+	 * 1: trigger & enable system error L1 recovery.
+	 * 2: trigger & enable system error L2 recovery.
+	 * 3: trigger & enable system error L3 rx abort.
+	 * 4: trigger & enable system error L3 tx abort
+	 * 5: trigger & enable system error L3 tx disable.
+	 * 6: trigger & enable system error L3 bf recovery.
+	 * 7: trigger & enable system error L4 mdp recovery.
+	 * 8: trigger & enable system error full recovery.
+	 * 9: trigger firmware crash.
+	 */
+	case UNI_CMD_SER_QUERY:
+		ret = mt7996_mcu_set_ser(dev, UNI_CMD_SER_QUERY, 0, band);
+		break;
+	case UNI_CMD_SER_SET_RECOVER_L1:
+	case UNI_CMD_SER_SET_RECOVER_L2:
+	case UNI_CMD_SER_SET_RECOVER_L3_RX_ABORT:
+	case UNI_CMD_SER_SET_RECOVER_L3_TX_ABORT:
+	case UNI_CMD_SER_SET_RECOVER_L3_TX_DISABLE:
+	case UNI_CMD_SER_SET_RECOVER_L3_BF:
+	case UNI_CMD_SER_SET_RECOVER_L4_MDP:
+		ret = mt7996_mcu_set_ser(dev, UNI_CMD_SER_SET, BIT(val), band);
 		if (ret)
 			return ret;
 
-		ret = mt7996_mcu_set_ser(dev, SER_RECOVER, val, band_idx);
+		ret = mt7996_mcu_set_ser(dev, UNI_CMD_SER_TRIGGER, val, band);
+		break;
+
+	/* enable full chip reset */
+	case UNI_CMD_SER_SET_RECOVER_FULL:
+		mt76_set(dev, MT_WFDMA0_MCU_HOST_INT_ENA, MT_MCU_CMD_WDT_MASK);
+		dev->recovery.state |= MT_MCU_CMD_WDT_MASK;
+		mt7996_reset(dev);
+		break;
+
+	/* WARNING: trigger firmware crash */
+	case UNI_CMD_SER_SET_SYSTEM_ASSERT:
+		ret = mt7996_mcu_trigger_assert(dev);
+		if (ret)
+			return ret;
 		break;
 	default:
 		break;
@@ -92,9 +122,97 @@ mt7996_fw_ser_set(struct file *file, const char __user *user_buf,
 	return ret ? ret : count;
 }
 
-static const struct file_operations mt7996_fw_ser_ops = {
-	.write = mt7996_fw_ser_set,
-	/* TODO: ser read */
+static ssize_t
+mt7996_sys_recovery_get(struct file *file, char __user *user_buf,
+			size_t count, loff_t *ppos)
+{
+	struct mt7996_phy *phy = file->private_data;
+	struct mt7996_dev *dev = phy->dev;
+	char *buff;
+	int desc = 0;
+	ssize_t ret;
+	static const size_t bufsz = 1024;
+
+	buff = kmalloc(bufsz, GFP_KERNEL);
+	if (!buff)
+		return -ENOMEM;
+
+	/* HELP */
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "Please echo the correct value ...\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "0: grab firmware transient SER state\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "1: trigger system error L1 recovery\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "2: trigger system error L2 recovery\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "3: trigger system error L3 rx abort\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "4: trigger system error L3 tx abort\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "5: trigger system error L3 tx disable\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "6: trigger system error L3 bf recovery\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "7: trigger system error L4 mdp recovery\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "8: trigger system error full recovery\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "9: trigger firmware crash\n");
+
+	/* SER statistics */
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "\nlet's dump firmware SER statistics...\n");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_STATUS        = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_SER_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_PLE_ERR       = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_PLE_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_PLE_ERR_1     = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_PLE1_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_PLE_ERR_AMSDU = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_PLE_AMSDU_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_PSE_ERR       = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_PSE_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_PSE_ERR_1     = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_PSE1_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_LMAC_WISR6_B0 = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_LAMC_WISR6_BN0_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_LMAC_WISR6_B1 = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_LAMC_WISR6_BN1_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_LMAC_WISR6_B2 = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_LAMC_WISR6_BN2_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_LMAC_WISR7_B0 = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_LAMC_WISR7_BN0_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_LMAC_WISR7_B1 = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_LAMC_WISR7_BN1_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "::E  R , SER_LMAC_WISR7_B2 = 0x%08x\n",
+			  mt76_rr(dev, MT_SWDEF_LAMC_WISR7_BN2_STATS));
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "\nSYS_RESET_COUNT: WM %d, WA %d\n",
+			  dev->recovery.wm_reset_count,
+			  dev->recovery.wa_reset_count);
+
+	ret = simple_read_from_buffer(user_buf, count, ppos, buff, desc);
+	kfree(buff);
+	return ret;
+}
+
+static const struct file_operations mt7996_sys_recovery_ops = {
+	.write = mt7996_sys_recovery_set,
+	.read = mt7996_sys_recovery_get,
 	.open = simple_open,
 	.llseek = default_llseek,
 };
@@ -674,6 +792,8 @@ int mt7996_init_debugfs(struct mt7996_phy *phy)
 	debugfs_create_file("xmit-queues", 0400, dir, phy,
 			    &mt7996_xmit_queues_fops);
 	debugfs_create_file("tx_stats", 0400, dir, phy, &mt7996_tx_stats_fops);
+	debugfs_create_file("sys_recovery", 0600, dir, phy,
+			    &mt7996_sys_recovery_ops);
 	debugfs_create_file("fw_debug_wm", 0600, dir, dev, &fops_fw_debug_wm);
 	debugfs_create_file("fw_debug_wa", 0600, dir, dev, &fops_fw_debug_wa);
 	debugfs_create_file("fw_debug_bin", 0600, dir, dev, &fops_fw_debug_bin);
@@ -684,7 +804,6 @@ int mt7996_init_debugfs(struct mt7996_phy *phy)
 			    &fops_implicit_txbf);
 	debugfs_create_devm_seqfile(dev->mt76.dev, "twt_stats", dir,
 				    mt7996_twt_stats);
-	debugfs_create_file("fw_ser", 0600, dir, phy, &mt7996_fw_ser_ops);
 	debugfs_create_file("rf_regval", 0600, dir, dev, &fops_rf_regval);
 
 	if (phy->mt76->cap.has_5ghz) {
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c
index c09fe4274935..534143465d9b 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c
@@ -352,6 +352,70 @@ int mt7996_dma_init(struct mt7996_dev *dev)
 	return 0;
 }
 
+void mt7996_dma_reset(struct mt7996_dev *dev, bool force)
+{
+	struct mt76_phy *phy2 = dev->mt76.phys[MT_BAND1];
+	struct mt76_phy *phy3 = dev->mt76.phys[MT_BAND2];
+	u32 hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0);
+	int i;
+
+	mt76_clear(dev, MT_WFDMA0_GLO_CFG,
+		   MT_WFDMA0_GLO_CFG_TX_DMA_EN |
+		   MT_WFDMA0_GLO_CFG_RX_DMA_EN);
+
+	if (dev->hif2)
+		mt76_clear(dev, MT_WFDMA0_GLO_CFG + hif1_ofs,
+			   MT_WFDMA0_GLO_CFG_TX_DMA_EN |
+			   MT_WFDMA0_GLO_CFG_RX_DMA_EN);
+
+	usleep_range(1000, 2000);
+
+	for (i = 0; i < __MT_TXQ_MAX; i++) {
+		mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[i], true);
+		if (phy2)
+			mt76_queue_tx_cleanup(dev, phy2->q_tx[i], true);
+		if (phy3)
+			mt76_queue_tx_cleanup(dev, phy3->q_tx[i], true);
+	}
+
+	for (i = 0; i < __MT_MCUQ_MAX; i++)
+		mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[i], true);
+
+	mt76_for_each_q_rx(&dev->mt76, i)
+		mt76_queue_rx_cleanup(dev, &dev->mt76.q_rx[i]);
+
+	mt76_tx_status_check(&dev->mt76, true);
+
+	/* reset wfsys */
+	if (force)
+		mt7996_wfsys_reset(dev);
+
+	mt7996_dma_disable(dev, force);
+
+	/* reset hw queues */
+	for (i = 0; i < __MT_TXQ_MAX; i++) {
+		mt76_queue_reset(dev, dev->mphy.q_tx[i]);
+		if (phy2)
+			mt76_queue_reset(dev, phy2->q_tx[i]);
+		if (phy3)
+			mt76_queue_reset(dev, phy3->q_tx[i]);
+	}
+
+	for (i = 0; i < __MT_MCUQ_MAX; i++)
+		mt76_queue_reset(dev, dev->mt76.q_mcu[i]);
+
+	mt76_for_each_q_rx(&dev->mt76, i) {
+		mt76_queue_reset(dev, &dev->mt76.q_rx[i]);
+	}
+
+	mt76_tx_status_check(&dev->mt76, true);
+
+	mt76_for_each_q_rx(&dev->mt76, i)
+		mt76_queue_rx_reset(dev, i);
+
+	mt7996_dma_enable(dev);
+}
+
 void mt7996_dma_cleanup(struct mt7996_dev *dev)
 {
 	mt7996_dma_disable(dev, true);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c
index 2e48c5a40f81..544b6c6f1ea3 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c
@@ -138,10 +138,6 @@ static int mt7996_eeprom_parse_band_config(struct mt7996_phy *phy)
 	case MT_EE_BAND_SEL_6GHZ:
 		phy->mt76->cap.has_6ghz = true;
 		break;
-	case MT_EE_BAND_SEL_5GHZ_6GHZ:
-		phy->mt76->cap.has_5ghz = true;
-		phy->mt76->cap.has_6ghz = true;
-		break;
 	default:
 		ret = -EINVAL;
 		break;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h
index 8da599e0abea..0c749774f6b1 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h
@@ -31,11 +31,11 @@ enum mt7996_eeprom_field {
 #define MT_EE_WIFI_CONF2_BAND_SEL		GENMASK(2, 0)
 
 #define MT_EE_WIFI_CONF1_TX_PATH_BAND0		GENMASK(5, 3)
-#define MT_EE_WIFI_CONF2_TX_PATH_BAND1		GENMASK(5, 3)
-#define MT_EE_WIFI_CONF2_TX_PATH_BAND2		GENMASK(2, 0)
+#define MT_EE_WIFI_CONF2_TX_PATH_BAND1		GENMASK(2, 0)
+#define MT_EE_WIFI_CONF2_TX_PATH_BAND2		GENMASK(5, 3)
 #define MT_EE_WIFI_CONF4_STREAM_NUM_BAND0	GENMASK(5, 3)
-#define MT_EE_WIFI_CONF5_STREAM_NUM_BAND1	GENMASK(5, 3)
-#define MT_EE_WIFI_CONF5_STREAM_NUM_BAND2	GENMASK(2, 0)
+#define MT_EE_WIFI_CONF5_STREAM_NUM_BAND1	GENMASK(2, 0)
+#define MT_EE_WIFI_CONF5_STREAM_NUM_BAND2	GENMASK(5, 3)
 
 #define MT_EE_RATE_DELTA_MASK			GENMASK(5, 0)
 #define MT_EE_RATE_DELTA_SIGN			BIT(6)
@@ -46,7 +46,6 @@ enum mt7996_eeprom_band {
 	MT_EE_BAND_SEL_2GHZ,
 	MT_EE_BAND_SEL_5GHZ,
 	MT_EE_BAND_SEL_6GHZ,
-	MT_EE_BAND_SEL_5GHZ_6GHZ,
 };
 
 static inline int
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/init.c b/drivers/net/wireless/mediatek/mt76/mt7996/init.c
index 946da93eed32..f1b48cdda58f 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/init.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/init.c
@@ -8,6 +8,7 @@
 #include "mt7996.h"
 #include "mac.h"
 #include "mcu.h"
+#include "coredump.h"
 #include "eeprom.h"
 
 static const struct ieee80211_iface_limit if_limits[] = {
@@ -99,9 +100,8 @@ static void mt7996_led_set_brightness(struct led_classdev *led_cdev,
 		mt7996_led_set_config(led_cdev, 0xff, 0);
 }
 
-static void
-mt7996_init_txpower(struct mt7996_dev *dev,
-		    struct ieee80211_supported_band *sband)
+void mt7996_init_txpower(struct mt7996_dev *dev,
+			 struct ieee80211_supported_band *sband)
 {
 	int i, nss = hweight8(dev->mphy.antenna_mask);
 	int nss_delta = mt76_tx_power_nss_delta(nss);
@@ -182,6 +182,7 @@ mt7996_init_wiphy(struct ieee80211_hw *hw)
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_UNSOL_BCAST_PROBE_RESP);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_FILS_DISCOVERY);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_ACK_SIGNAL_SUPPORT);
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
 
 	if (!mdev->dev->of_node ||
 	    !of_property_read_bool(mdev->dev->of_node,
@@ -196,10 +197,13 @@ mt7996_init_wiphy(struct ieee80211_hw *hw)
 
 	hw->max_tx_fragments = 4;
 
-	if (phy->mt76->cap.has_2ghz)
+	if (phy->mt76->cap.has_2ghz) {
 		phy->mt76->sband_2g.sband.ht_cap.cap |=
 			IEEE80211_HT_CAP_LDPC_CODING |
 			IEEE80211_HT_CAP_MAX_AMSDU;
+		phy->mt76->sband_2g.sband.ht_cap.ampdu_density =
+			IEEE80211_HT_MPDU_DENSITY_2;
+	}
 
 	if (phy->mt76->cap.has_5ghz) {
 		phy->mt76->sband_5g.sband.ht_cap.cap |=
@@ -211,6 +215,8 @@ mt7996_init_wiphy(struct ieee80211_hw *hw)
 			IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK |
 			IEEE80211_VHT_CAP_SHORT_GI_160 |
 			IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ;
+		phy->mt76->sband_5g.sband.ht_cap.ampdu_density =
+			IEEE80211_HT_MPDU_DENSITY_1;
 	}
 
 	mt76_set_stream_caps(phy->mt76, true);
@@ -250,7 +256,21 @@ mt7996_mac_init_band(struct mt7996_dev *dev, u8 band)
 	mt76_rmw(dev, MT_WTBLOFF_RSCR(band), mask, set);
 }
 
-static void mt7996_mac_init(struct mt7996_dev *dev)
+static void mt7996_mac_init_basic_rates(struct mt7996_dev *dev)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(mt76_rates); i++) {
+		u16 rate = mt76_rates[i].hw_value;
+		u16 idx = MT7996_BASIC_RATES_TBL + i;
+
+		rate = FIELD_PREP(MT_TX_RATE_MODE, rate >> 8) |
+		       FIELD_PREP(MT_TX_RATE_IDX, rate & GENMASK(7, 0));
+		mt7996_mac_set_fixed_rate_table(dev, idx, rate);
+	}
+}
+
+void mt7996_mac_init(struct mt7996_dev *dev)
 {
 #define HIF_TXD_V2_1	4
 	int i;
@@ -282,9 +302,11 @@ static void mt7996_mac_init(struct mt7996_dev *dev)
 
 	for (i = MT_BAND0; i <= MT_BAND2; i++)
 		mt7996_mac_init_band(dev, i);
+
+	mt7996_mac_init_basic_rates(dev);
 }
 
-static int mt7996_txbf_init(struct mt7996_dev *dev)
+int mt7996_txbf_init(struct mt7996_dev *dev)
 {
 	int ret;
 
@@ -553,27 +575,6 @@ mt7996_set_stream_he_txbf_caps(struct mt7996_phy *phy,
 }
 
 static void
-mt7996_gen_ppe_thresh(u8 *he_ppet, int nss)
-{
-	u8 i, ppet_bits, ppet_size, ru_bit_mask = 0x7; /* HE80 */
-	static const u8 ppet16_ppet8_ru3_ru0[] = {0x1c, 0xc7, 0x71};
-
-	he_ppet[0] = FIELD_PREP(IEEE80211_PPE_THRES_NSS_MASK, nss - 1) |
-		     FIELD_PREP(IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK,
-				ru_bit_mask);
-
-	ppet_bits = IEEE80211_PPE_THRES_INFO_PPET_SIZE *
-		    nss * hweight8(ru_bit_mask) * 2;
-	ppet_size = DIV_ROUND_UP(ppet_bits, 8);
-
-	for (i = 0; i < ppet_size - 1; i++)
-		he_ppet[i + 1] = ppet16_ppet8_ru3_ru0[i % 3];
-
-	he_ppet[i + 1] = ppet16_ppet8_ru3_ru0[i % 3] &
-			 (0xff >> (8 - (ppet_bits - 1) % 8));
-}
-
-static void
 mt7996_init_he_caps(struct mt7996_phy *phy, enum nl80211_band band,
 		    struct ieee80211_sband_iftype_data *data,
 		    enum nl80211_iftype iftype)
@@ -678,7 +679,7 @@ mt7996_init_he_caps(struct mt7996_phy *phy, enum nl80211_band band,
 	memset(he_cap->ppe_thres, 0, sizeof(he_cap->ppe_thres));
 	if (he_cap_elem->phy_cap_info[6] &
 	    IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) {
-		mt7996_gen_ppe_thresh(he_cap->ppe_thres, nss);
+		mt76_connac_gen_ppe_thresh(he_cap->ppe_thres, nss);
 	} else {
 		he_cap_elem->phy_cap_info[9] |=
 			u8_encode_bits(IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US,
@@ -689,7 +690,7 @@ mt7996_init_he_caps(struct mt7996_phy *phy, enum nl80211_band band,
 		u16 cap = IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS |
 			  IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS;
 
-		cap |= u16_encode_bits(IEEE80211_HT_MPDU_DENSITY_2,
+		cap |= u16_encode_bits(IEEE80211_HT_MPDU_DENSITY_0_5,
 				       IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START) |
 		       u16_encode_bits(IEEE80211_VHT_MAX_AMPDU_1024K,
 				       IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP) |
@@ -858,6 +859,8 @@ int mt7996_register_device(struct mt7996_dev *dev)
 
 	init_waitqueue_head(&dev->reset_wait);
 	INIT_WORK(&dev->reset_work, mt7996_mac_reset_work);
+	INIT_WORK(&dev->dump_work, mt7996_mac_dump_work);
+	mutex_init(&dev->dump_mutex);
 
 	ret = mt7996_init_hardware(dev);
 	if (ret)
@@ -886,18 +889,25 @@ int mt7996_register_device(struct mt7996_dev *dev)
 	if (ret)
 		return ret;
 
-	return mt7996_init_debugfs(&dev->phy);
+	dev->recovery.hw_init_done = true;
+
+	ret = mt7996_init_debugfs(&dev->phy);
+	if (ret)
+		return ret;
+
+	return mt7996_coredump_register(dev);
 }
 
 void mt7996_unregister_device(struct mt7996_dev *dev)
 {
 	mt7996_unregister_phy(mt7996_phy3(dev), MT_BAND2);
 	mt7996_unregister_phy(mt7996_phy2(dev), MT_BAND1);
+	mt7996_coredump_unregister(dev);
 	mt76_unregister_device(&dev->mt76);
 	mt7996_mcu_exit(dev);
 	mt7996_tx_token_put(dev);
 	mt7996_dma_cleanup(dev);
-	tasklet_disable(&dev->irq_tasklet);
+	tasklet_disable(&dev->mt76.irq_tasklet);
 
 	mt76_free_device(&dev->mt76);
 }
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c
index c9a9f0e31771..130eb7b4fd91 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c
@@ -5,6 +5,7 @@
 
 #include <linux/etherdevice.h>
 #include <linux/timekeeping.h>
+#include "coredump.h"
 #include "mt7996.h"
 #include "../dma.h"
 #include "mac.h"
@@ -78,10 +79,6 @@ static struct mt76_wcid *mt7996_rx_get_wcid(struct mt7996_dev *dev,
 	return &sta->vif->sta.wcid;
 }
 
-void mt7996_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps)
-{
-}
-
 bool mt7996_mac_wtbl_update(struct mt7996_dev *dev, int idx, u32 mask)
 {
 	mt76_rmw(dev, MT_WTBL_UPDATE, MT_WTBL_UPDATE_WLAN_IDX,
@@ -255,17 +252,25 @@ void mt7996_mac_enable_rtscts(struct mt7996_dev *dev,
 		mt76_clear(dev, addr, BIT(5));
 }
 
+void mt7996_mac_set_fixed_rate_table(struct mt7996_dev *dev,
+				     u8 tbl_idx, u16 rate_idx)
+{
+	u32 ctrl = MT_WTBL_ITCR_WR | MT_WTBL_ITCR_EXEC | tbl_idx;
+
+	mt76_wr(dev, MT_WTBL_ITDR0, rate_idx);
+	/* use wtbl spe idx */
+	mt76_wr(dev, MT_WTBL_ITDR1, MT_WTBL_SPE_IDX_SEL);
+	mt76_wr(dev, MT_WTBL_ITCR, ctrl);
+}
+
 static void
 mt7996_mac_decode_he_radiotap_ru(struct mt76_rx_status *status,
 				 struct ieee80211_radiotap_he *he,
 				 __le32 *rxv)
 {
-	u32 ru_h, ru_l;
-	u8 ru, offs = 0;
+	u32 ru, offs = 0;
 
-	ru_l = le32_get_bits(rxv[0], MT_PRXV_HE_RU_ALLOC_L);
-	ru_h = le32_get_bits(rxv[1], MT_PRXV_HE_RU_ALLOC_H);
-	ru = (u8)(ru_l | ru_h << 4);
+	ru = le32_get_bits(rxv[0], MT_PRXV_HE_RU_ALLOC);
 
 	status->bw = RATE_INFO_BW_HE_RU;
 
@@ -330,18 +335,23 @@ mt7996_mac_decode_he_mu_radiotap(struct sk_buff *skb, __le32 *rxv)
 
 	he_mu->flags2 |= MU_PREP(FLAGS2_BW_FROM_SIG_A_BW, status->bw) |
 			 MU_PREP(FLAGS2_SIG_B_SYMS_USERS,
-				 le32_get_bits(rxv[2], MT_CRXV_HE_NUM_USER));
+				 le32_get_bits(rxv[4], MT_CRXV_HE_NUM_USER));
 
-	he_mu->ru_ch1[0] = le32_get_bits(rxv[3], MT_CRXV_HE_RU0);
+	he_mu->ru_ch1[0] = le32_get_bits(rxv[16], MT_CRXV_HE_RU0) & 0xff;
 
 	if (status->bw >= RATE_INFO_BW_40) {
 		he_mu->flags1 |= HE_BITS(MU_FLAGS1_CH2_RU_KNOWN);
-		he_mu->ru_ch2[0] = le32_get_bits(rxv[3], MT_CRXV_HE_RU1);
+		he_mu->ru_ch2[0] = le32_get_bits(rxv[16], MT_CRXV_HE_RU1) & 0xff;
 	}
 
 	if (status->bw >= RATE_INFO_BW_80) {
-		he_mu->ru_ch1[1] = le32_get_bits(rxv[3], MT_CRXV_HE_RU2);
-		he_mu->ru_ch2[1] = le32_get_bits(rxv[3], MT_CRXV_HE_RU3);
+		u32 ru_h, ru_l;
+
+		he_mu->ru_ch1[1] = le32_get_bits(rxv[16], MT_CRXV_HE_RU2) & 0xff;
+
+		ru_l = le32_get_bits(rxv[16], MT_CRXV_HE_RU3_L);
+		ru_h = le32_get_bits(rxv[17], MT_CRXV_HE_RU3_H) & 0x7;
+		he_mu->ru_ch2[1] = (u8)(ru_l | ru_h << 4);
 	}
 }
 
@@ -364,23 +374,23 @@ mt7996_mac_decode_he_radiotap(struct sk_buff *skb, __le32 *rxv, u8 mode)
 			 HE_BITS(DATA2_TXOP_KNOWN),
 	};
 	struct ieee80211_radiotap_he *he = NULL;
-	u32 ltf_size = le32_get_bits(rxv[2], MT_CRXV_HE_LTF_SIZE) + 1;
+	u32 ltf_size = le32_get_bits(rxv[4], MT_CRXV_HE_LTF_SIZE) + 1;
 
 	status->flag |= RX_FLAG_RADIOTAP_HE;
 
 	he = skb_push(skb, sizeof(known));
 	memcpy(he, &known, sizeof(known));
 
-	he->data3 = HE_PREP(DATA3_BSS_COLOR, BSS_COLOR, rxv[14]) |
-		    HE_PREP(DATA3_LDPC_XSYMSEG, LDPC_EXT_SYM, rxv[2]);
-	he->data4 = HE_PREP(DATA4_SU_MU_SPTL_REUSE, SR_MASK, rxv[11]);
-	he->data5 = HE_PREP(DATA5_PE_DISAMBIG, PE_DISAMBIG, rxv[2]) |
+	he->data3 = HE_PREP(DATA3_BSS_COLOR, BSS_COLOR, rxv[9]) |
+		    HE_PREP(DATA3_LDPC_XSYMSEG, LDPC_EXT_SYM, rxv[4]);
+	he->data4 = HE_PREP(DATA4_SU_MU_SPTL_REUSE, SR_MASK, rxv[13]);
+	he->data5 = HE_PREP(DATA5_PE_DISAMBIG, PE_DISAMBIG, rxv[5]) |
 		    le16_encode_bits(ltf_size,
 				     IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE);
 	if (le32_to_cpu(rxv[0]) & MT_PRXV_TXBF)
 		he->data5 |= HE_BITS(DATA5_TXBF);
-	he->data6 = HE_PREP(DATA6_TXOP, TXOP_DUR, rxv[14]) |
-		    HE_PREP(DATA6_DOPPLER, DOPPLER, rxv[14]);
+	he->data6 = HE_PREP(DATA6_TXOP, TXOP_DUR, rxv[9]) |
+		    HE_PREP(DATA6_DOPPLER, DOPPLER, rxv[9]);
 
 	switch (mode) {
 	case MT_PHY_TYPE_HE_SU:
@@ -389,22 +399,22 @@ mt7996_mac_decode_he_radiotap(struct sk_buff *skb, __le32 *rxv, u8 mode)
 			     HE_BITS(DATA1_BEAM_CHANGE_KNOWN) |
 			     HE_BITS(DATA1_BW_RU_ALLOC_KNOWN);
 
-		he->data3 |= HE_PREP(DATA3_BEAM_CHANGE, BEAM_CHNG, rxv[14]) |
-			     HE_PREP(DATA3_UL_DL, UPLINK, rxv[2]);
+		he->data3 |= HE_PREP(DATA3_BEAM_CHANGE, BEAM_CHNG, rxv[8]) |
+			     HE_PREP(DATA3_UL_DL, UPLINK, rxv[5]);
 		break;
 	case MT_PHY_TYPE_HE_EXT_SU:
 		he->data1 |= HE_BITS(DATA1_FORMAT_EXT_SU) |
 			     HE_BITS(DATA1_UL_DL_KNOWN) |
 			     HE_BITS(DATA1_BW_RU_ALLOC_KNOWN);
 
-		he->data3 |= HE_PREP(DATA3_UL_DL, UPLINK, rxv[2]);
+		he->data3 |= HE_PREP(DATA3_UL_DL, UPLINK, rxv[5]);
 		break;
 	case MT_PHY_TYPE_HE_MU:
 		he->data1 |= HE_BITS(DATA1_FORMAT_MU) |
 			     HE_BITS(DATA1_UL_DL_KNOWN);
 
-		he->data3 |= HE_PREP(DATA3_UL_DL, UPLINK, rxv[2]);
-		he->data4 |= HE_PREP(DATA4_MU_STA_ID, MU_AID, rxv[7]);
+		he->data3 |= HE_PREP(DATA3_UL_DL, UPLINK, rxv[5]);
+		he->data4 |= HE_PREP(DATA4_MU_STA_ID, MU_AID, rxv[8]);
 
 		mt7996_mac_decode_he_radiotap_ru(status, he, rxv);
 		mt7996_mac_decode_he_mu_radiotap(skb, rxv);
@@ -415,10 +425,10 @@ mt7996_mac_decode_he_radiotap(struct sk_buff *skb, __le32 *rxv, u8 mode)
 			     HE_BITS(DATA1_SPTL_REUSE3_KNOWN) |
 			     HE_BITS(DATA1_SPTL_REUSE4_KNOWN);
 
-		he->data4 |= HE_PREP(DATA4_TB_SPTL_REUSE1, SR_MASK, rxv[11]) |
-			     HE_PREP(DATA4_TB_SPTL_REUSE2, SR1_MASK, rxv[11]) |
-			     HE_PREP(DATA4_TB_SPTL_REUSE3, SR2_MASK, rxv[11]) |
-			     HE_PREP(DATA4_TB_SPTL_REUSE4, SR3_MASK, rxv[11]);
+		he->data4 |= HE_PREP(DATA4_TB_SPTL_REUSE1, SR_MASK, rxv[13]) |
+			     HE_PREP(DATA4_TB_SPTL_REUSE2, SR1_MASK, rxv[13]) |
+			     HE_PREP(DATA4_TB_SPTL_REUSE3, SR2_MASK, rxv[13]) |
+			     HE_PREP(DATA4_TB_SPTL_REUSE4, SR3_MASK, rxv[13]);
 
 		mt7996_mac_decode_he_radiotap_ru(status, he, rxv);
 		break;
@@ -570,11 +580,12 @@ mt7996_mac_fill_rx_rate(struct mt7996_dev *dev,
 	case MT_PHY_TYPE_EHT_SU:
 	case MT_PHY_TYPE_EHT_TRIG:
 	case MT_PHY_TYPE_EHT_MU:
-		/* TODO: currently report rx rate with HE rate */
 		status->nss = nss;
-		status->encoding = RX_ENC_HE;
-		bw = min_t(int, bw, IEEE80211_STA_RX_BW_160);
-		i = min_t(int, i & 0xf, 11);
+		status->encoding = RX_ENC_EHT;
+		i &= GENMASK(3, 0);
+
+		if (gi <= NL80211_RATE_INFO_EHT_GI_3_2)
+			status->eht.gi = gi;
 		break;
 	default:
 		return -EINVAL;
@@ -630,6 +641,8 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, struct sk_buff *skb)
 	u32 rxd4 = le32_to_cpu(rxd[4]);
 	u32 csum_mask = MT_RXD0_NORMAL_IP_SUM | MT_RXD0_NORMAL_UDP_TCP_SUM;
 	u32 csum_status = *(u32 *)skb->cb;
+	u32 mesh_mask = MT_RXD0_MESH | MT_RXD0_MHCP;
+	bool is_mesh = (rxd0 & mesh_mask) == mesh_mask;
 	bool unicast, insert_ccmp_hdr = false;
 	u8 remove_pad, amsdu_info, band_idx;
 	u8 mode = 0, qos_ctl = 0;
@@ -821,19 +834,16 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, struct sk_buff *skb)
 		int pad_start = 0;
 
 		skb_pull(skb, hdr_gap);
-		if (!hdr_trans && status->amsdu) {
+		if (!hdr_trans && status->amsdu && !(ieee80211_has_a4(fc) && is_mesh)) {
 			pad_start = ieee80211_get_hdrlen_from_skb(skb);
-		} else if (hdr_trans && (rxd2 & MT_RXD2_NORMAL_HDR_TRANS_ERROR)) {
+		} else if (hdr_trans && (rxd2 & MT_RXD2_NORMAL_HDR_TRANS_ERROR) &&
+			   get_unaligned_be16(skb->data + pad_start) == ETH_P_8021Q) {
 			/* When header translation failure is indicated,
 			 * the hardware will insert an extra 2-byte field
 			 * containing the data length after the protocol
 			 * type field.
 			 */
-			pad_start = 12;
-			if (get_unaligned_be16(skb->data + pad_start) == ETH_P_8021Q)
-				pad_start += 4;
-			else
-				pad_start = 0;
+			pad_start = 16;
 		}
 
 		if (pad_start) {
@@ -854,8 +864,17 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, struct sk_buff *skb)
 		hdr = mt76_skb_get_hdr(skb);
 		fc = hdr->frame_control;
 		if (ieee80211_is_data_qos(fc)) {
+			u8 *qos = ieee80211_get_qos_ctl(hdr);
+
 			seq_ctrl = le16_to_cpu(hdr->seq_ctrl);
-			qos_ctl = *ieee80211_get_qos_ctl(hdr);
+			qos_ctl = *qos;
+
+			/* Mesh DA/SA/Length will be stripped after hardware
+			 * de-amsdu, so here needs to clear amsdu present bit
+			 * to mark it as a normal mesh frame.
+			 */
+			if (ieee80211_has_a4(fc) && is_mesh && status->amsdu)
+				*qos &= ~IEEE80211_QOS_CTL_A_MSDU_PRESENT;
 		}
 	} else {
 		status->flag |= RX_FLAG_8023;
@@ -979,12 +998,13 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi,
 }
 
 void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi,
-			   struct sk_buff *skb, struct mt76_wcid *wcid, int pid,
-			   struct ieee80211_key_conf *key, u32 changed)
+			   struct sk_buff *skb, struct mt76_wcid *wcid,
+			   struct ieee80211_key_conf *key, int pid,
+			   enum mt76_txq_id qid, u32 changed)
 {
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
 	struct ieee80211_vif *vif = info->control.vif;
-	struct mt76_phy *mphy = &dev->mphy;
+	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	u8 band_idx = (info->hw_queue & MT_TX_HW_QUEUE_PHY) >> 2;
 	u8 p_fmt, q_idx, omac_idx = 0, wmm_idx = 0;
 	bool is_8023 = info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP;
@@ -996,22 +1016,18 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi,
 					 BSS_CHANGED_FILS_DISCOVERY));
 
 	if (vif) {
-		struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-
 		omac_idx = mvif->mt76.omac_idx;
 		wmm_idx = mvif->mt76.wmm_idx;
 		band_idx = mvif->mt76.band_idx;
 	}
 
-	mphy = mt76_dev_phy(&dev->mt76, band_idx);
-
 	if (inband_disc) {
 		p_fmt = MT_TX_TYPE_FW;
 		q_idx = MT_LMAC_ALTX0;
 	} else if (beacon) {
 		p_fmt = MT_TX_TYPE_FW;
 		q_idx = MT_LMAC_BCN0;
-	} else if (skb_get_queue_mapping(skb) >= MT_TXQ_PSD) {
+	} else if (qid >= MT_TXQ_PSD) {
 		p_fmt = MT_TX_TYPE_CT;
 		q_idx = MT_LMAC_ALTX0;
 	} else {
@@ -1062,18 +1078,17 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi,
 		mt7996_mac_write_txwi_80211(dev, txwi, skb, key);
 
 	if (txwi[1] & cpu_to_le32(MT_TXD1_FIXED_RATE)) {
-		/* Fixed rata is available just for 802.11 txd */
 		struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
-		bool multicast = is_multicast_ether_addr(hdr->addr1);
-		u16 rate = mt76_connac2_mac_tx_rate_val(mphy, vif, beacon,
-							multicast);
+		bool mcast = ieee80211_is_data(hdr->frame_control) &&
+			     is_multicast_ether_addr(hdr->addr1);
+		u8 idx = mvif->basic_rates_idx;
 
-		/* fix to bw 20 */
-		val = MT_TXD6_FIXED_BW |
-		      FIELD_PREP(MT_TXD6_BW, 0) |
-		      FIELD_PREP(MT_TXD6_TX_RATE, rate);
+		if (mcast && mvif->mcast_rates_idx)
+			idx = mvif->mcast_rates_idx;
+		else if (beacon && mvif->beacon_rates_idx)
+			idx = mvif->beacon_rates_idx;
 
-		txwi[6] |= cpu_to_le32(val);
+		txwi[6] |= FIELD_PREP(MT_TXD6_TX_RATE, idx);
 		txwi[3] |= cpu_to_le32(MT_TXD3_BA_DISABLE);
 	}
 }
@@ -1117,11 +1132,8 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr,
 		return id;
 
 	pid = mt76_tx_status_skb_add(mdev, wcid, tx_info->skb);
-	memset(txwi_ptr, 0, MT_TXD_SIZE);
-	/* Transmit non qos data by 802.11 header and need to fill txd by host*/
-	if (!is_8023 || pid >= MT_PACKET_ID_FIRST)
-		mt7996_mac_write_txwi(dev, txwi_ptr, tx_info->skb, wcid, pid,
-				      key, 0);
+	mt7996_mac_write_txwi(dev, txwi_ptr, tx_info->skb, wcid, key,
+			      pid, qid, 0);
 
 	txp = (struct mt76_connac_txp_common *)(txwi + MT_TXD_SIZE);
 	for (i = 0; i < nbuf; i++) {
@@ -1130,10 +1142,8 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr,
 	}
 	txp->fw.nbuf = nbuf;
 
-	txp->fw.flags = cpu_to_le16(MT_CT_INFO_FROM_HOST);
-
-	if (!is_8023 || pid >= MT_PACKET_ID_FIRST)
-		txp->fw.flags |= cpu_to_le16(MT_CT_INFO_APPLY_TXD);
+	txp->fw.flags =
+		cpu_to_le16(MT_CT_INFO_FROM_HOST | MT_CT_INFO_APPLY_TXD);
 
 	if (!key)
 		txp->fw.flags |= cpu_to_le16(MT_CT_INFO_NONE_CIPHER_FRAME);
@@ -1704,7 +1714,7 @@ mt7996_wait_reset_state(struct mt7996_dev *dev, u32 state)
 	bool ret;
 
 	ret = wait_event_timeout(dev->reset_wait,
-				 (READ_ONCE(dev->reset_state) & state),
+				 (READ_ONCE(dev->recovery.state) & state),
 				 MT7996_RESET_TIMEOUT);
 
 	WARN(!ret, "Timeout waiting for MCU reset state %x\n", state);
@@ -1753,68 +1763,207 @@ mt7996_update_beacons(struct mt7996_dev *dev)
 					    mt7996_update_vif_beacon, phy3->hw);
 }
 
-static void
-mt7996_dma_reset(struct mt7996_dev *dev)
+void mt7996_tx_token_put(struct mt7996_dev *dev)
 {
-	struct mt76_phy *phy2 = dev->mt76.phys[MT_BAND1];
-	struct mt76_phy *phy3 = dev->mt76.phys[MT_BAND2];
-	u32 hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0);
-	int i;
+	struct mt76_txwi_cache *txwi;
+	int id;
 
-	mt76_clear(dev, MT_WFDMA0_GLO_CFG,
-		   MT_WFDMA0_GLO_CFG_TX_DMA_EN |
-		   MT_WFDMA0_GLO_CFG_RX_DMA_EN);
+	spin_lock_bh(&dev->mt76.token_lock);
+	idr_for_each_entry(&dev->mt76.token, txwi, id) {
+		mt7996_txwi_free(dev, txwi, NULL, NULL);
+		dev->mt76.token_count--;
+	}
+	spin_unlock_bh(&dev->mt76.token_lock);
+	idr_destroy(&dev->mt76.token);
+}
 
-	if (dev->hif2)
-		mt76_clear(dev, MT_WFDMA0_GLO_CFG + hif1_ofs,
-			   MT_WFDMA0_GLO_CFG_TX_DMA_EN |
-			   MT_WFDMA0_GLO_CFG_RX_DMA_EN);
+static int
+mt7996_mac_restart(struct mt7996_dev *dev)
+{
+	struct mt7996_phy *phy2, *phy3;
+	struct mt76_dev *mdev = &dev->mt76;
+	int i, ret;
 
-	usleep_range(1000, 2000);
+	phy2 = mt7996_phy2(dev);
+	phy3 = mt7996_phy3(dev);
 
-	for (i = 0; i < __MT_TXQ_MAX; i++) {
-		mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[i], true);
-		if (phy2)
-			mt76_queue_tx_cleanup(dev, phy2->q_tx[i], true);
-		if (phy3)
-			mt76_queue_tx_cleanup(dev, phy3->q_tx[i], true);
+	if (dev->hif2) {
+		mt76_wr(dev, MT_INT1_MASK_CSR, 0x0);
+		mt76_wr(dev, MT_INT1_SOURCE_CSR, ~0);
 	}
 
-	for (i = 0; i < __MT_MCUQ_MAX; i++)
-		mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[i], true);
+	if (dev_is_pci(mdev->dev)) {
+		mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0x0);
+		if (dev->hif2)
+			mt76_wr(dev, MT_PCIE1_MAC_INT_ENABLE, 0x0);
+	}
 
-	mt76_for_each_q_rx(&dev->mt76, i)
-		mt76_queue_rx_reset(dev, i);
+	set_bit(MT76_RESET, &dev->mphy.state);
+	set_bit(MT76_MCU_RESET, &dev->mphy.state);
+	wake_up(&dev->mt76.mcu.wait);
+	if (phy2) {
+		set_bit(MT76_RESET, &phy2->mt76->state);
+		set_bit(MT76_MCU_RESET, &phy2->mt76->state);
+	}
+	if (phy3) {
+		set_bit(MT76_RESET, &phy3->mt76->state);
+		set_bit(MT76_MCU_RESET, &phy3->mt76->state);
+	}
+
+	/* lock/unlock all queues to ensure that no tx is pending */
+	mt76_txq_schedule_all(&dev->mphy);
+	if (phy2)
+		mt76_txq_schedule_all(phy2->mt76);
+	if (phy3)
+		mt76_txq_schedule_all(phy3->mt76);
+
+	/* disable all tx/rx napi */
+	mt76_worker_disable(&dev->mt76.tx_worker);
+	mt76_for_each_q_rx(mdev, i) {
+		if (mdev->q_rx[i].ndesc)
+			napi_disable(&dev->mt76.napi[i]);
+	}
+	napi_disable(&dev->mt76.tx_napi);
+
+	/* token reinit */
+	mt7996_tx_token_put(dev);
+	idr_init(&dev->mt76.token);
+
+	mt7996_dma_reset(dev, true);
 
-	mt76_tx_status_check(&dev->mt76, true);
+	local_bh_disable();
+	mt76_for_each_q_rx(mdev, i) {
+		if (mdev->q_rx[i].ndesc) {
+			napi_enable(&dev->mt76.napi[i]);
+			napi_schedule(&dev->mt76.napi[i]);
+		}
+	}
+	local_bh_enable();
+	clear_bit(MT76_MCU_RESET, &dev->mphy.state);
+	clear_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state);
+
+	mt76_wr(dev, MT_INT_MASK_CSR, dev->mt76.mmio.irqmask);
+	mt76_wr(dev, MT_INT_SOURCE_CSR, ~0);
+	if (dev->hif2) {
+		mt76_wr(dev, MT_INT1_MASK_CSR, dev->mt76.mmio.irqmask);
+		mt76_wr(dev, MT_INT1_SOURCE_CSR, ~0);
+	}
+	if (dev_is_pci(mdev->dev)) {
+		mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff);
+		if (dev->hif2)
+			mt76_wr(dev, MT_PCIE1_MAC_INT_ENABLE, 0xff);
+	}
+
+	/* load firmware */
+	ret = mt7996_mcu_init_firmware(dev);
+	if (ret)
+		goto out;
 
-	/* re-init prefetch settings after reset */
-	mt7996_dma_prefetch(dev);
+	/* set the necessary init items */
+	ret = mt7996_mcu_set_eeprom(dev);
+	if (ret)
+		goto out;
 
-	mt76_set(dev, MT_WFDMA0_GLO_CFG,
-		 MT_WFDMA0_GLO_CFG_TX_DMA_EN | MT_WFDMA0_GLO_CFG_RX_DMA_EN);
+	mt7996_mac_init(dev);
+	mt7996_init_txpower(dev, &dev->mphy.sband_2g.sband);
+	mt7996_init_txpower(dev, &dev->mphy.sband_5g.sband);
+	mt7996_init_txpower(dev, &dev->mphy.sband_6g.sband);
+	ret = mt7996_txbf_init(dev);
 
-	if (dev->hif2)
-		mt76_set(dev, MT_WFDMA0_GLO_CFG + hif1_ofs,
-			 MT_WFDMA0_GLO_CFG_TX_DMA_EN |
-			 MT_WFDMA0_GLO_CFG_RX_DMA_EN);
+	if (test_bit(MT76_STATE_RUNNING, &dev->mphy.state)) {
+		ret = mt7996_run(dev->mphy.hw);
+		if (ret)
+			goto out;
+	}
+
+	if (phy2 && test_bit(MT76_STATE_RUNNING, &phy2->mt76->state)) {
+		ret = mt7996_run(phy2->mt76->hw);
+		if (ret)
+			goto out;
+	}
+
+	if (phy3 && test_bit(MT76_STATE_RUNNING, &phy3->mt76->state)) {
+		ret = mt7996_run(phy3->mt76->hw);
+		if (ret)
+			goto out;
+	}
+
+out:
+	/* reset done */
+	clear_bit(MT76_RESET, &dev->mphy.state);
+	if (phy2)
+		clear_bit(MT76_RESET, &phy2->mt76->state);
+	if (phy3)
+		clear_bit(MT76_RESET, &phy3->mt76->state);
+
+	local_bh_disable();
+	napi_enable(&dev->mt76.tx_napi);
+	napi_schedule(&dev->mt76.tx_napi);
+	local_bh_enable();
+
+	mt76_worker_enable(&dev->mt76.tx_worker);
+	return ret;
 }
 
-void mt7996_tx_token_put(struct mt7996_dev *dev)
+static void
+mt7996_mac_full_reset(struct mt7996_dev *dev)
 {
-	struct mt76_txwi_cache *txwi;
-	int id;
+	struct mt7996_phy *phy2, *phy3;
+	int i;
 
-	spin_lock_bh(&dev->mt76.token_lock);
-	idr_for_each_entry(&dev->mt76.token, txwi, id) {
-		mt7996_txwi_free(dev, txwi, NULL, NULL);
-		dev->mt76.token_count--;
+	phy2 = mt7996_phy2(dev);
+	phy3 = mt7996_phy3(dev);
+	dev->recovery.hw_full_reset = true;
+
+	wake_up(&dev->mt76.mcu.wait);
+	ieee80211_stop_queues(mt76_hw(dev));
+	if (phy2)
+		ieee80211_stop_queues(phy2->mt76->hw);
+	if (phy3)
+		ieee80211_stop_queues(phy3->mt76->hw);
+
+	cancel_delayed_work_sync(&dev->mphy.mac_work);
+	if (phy2)
+		cancel_delayed_work_sync(&phy2->mt76->mac_work);
+	if (phy3)
+		cancel_delayed_work_sync(&phy3->mt76->mac_work);
+
+	mutex_lock(&dev->mt76.mutex);
+	for (i = 0; i < 10; i++) {
+		if (!mt7996_mac_restart(dev))
+			break;
 	}
-	spin_unlock_bh(&dev->mt76.token_lock);
-	idr_destroy(&dev->mt76.token);
+	mutex_unlock(&dev->mt76.mutex);
+
+	if (i == 10)
+		dev_err(dev->mt76.dev, "chip full reset failed\n");
+
+	ieee80211_restart_hw(mt76_hw(dev));
+	if (phy2)
+		ieee80211_restart_hw(phy2->mt76->hw);
+	if (phy3)
+		ieee80211_restart_hw(phy3->mt76->hw);
+
+	ieee80211_wake_queues(mt76_hw(dev));
+	if (phy2)
+		ieee80211_wake_queues(phy2->mt76->hw);
+	if (phy3)
+		ieee80211_wake_queues(phy3->mt76->hw);
+
+	dev->recovery.hw_full_reset = false;
+	ieee80211_queue_delayed_work(mt76_hw(dev),
+				     &dev->mphy.mac_work,
+				     MT7996_WATCHDOG_TIME);
+	if (phy2)
+		ieee80211_queue_delayed_work(phy2->mt76->hw,
+					     &phy2->mt76->mac_work,
+					     MT7996_WATCHDOG_TIME);
+	if (phy3)
+		ieee80211_queue_delayed_work(phy3->mt76->hw,
+					     &phy3->mt76->mac_work,
+					     MT7996_WATCHDOG_TIME);
 }
 
-/* system error recovery */
 void mt7996_mac_reset_work(struct work_struct *work)
 {
 	struct mt7996_phy *phy2, *phy3;
@@ -1825,9 +1974,36 @@ void mt7996_mac_reset_work(struct work_struct *work)
 	phy2 = mt7996_phy2(dev);
 	phy3 = mt7996_phy3(dev);
 
-	if (!(READ_ONCE(dev->reset_state) & MT_MCU_CMD_STOP_DMA))
+	/* chip full reset */
+	if (dev->recovery.restart) {
+		/* disable WA/WM WDT */
+		mt76_clear(dev, MT_WFDMA0_MCU_HOST_INT_ENA,
+			   MT_MCU_CMD_WDT_MASK);
+
+		if (READ_ONCE(dev->recovery.state) & MT_MCU_CMD_WA_WDT)
+			dev->recovery.wa_reset_count++;
+		else
+			dev->recovery.wm_reset_count++;
+
+		mt7996_mac_full_reset(dev);
+
+		/* enable mcu irq */
+		mt7996_irq_enable(dev, MT_INT_MCU_CMD);
+		mt7996_irq_disable(dev, 0);
+
+		/* enable WA/WM WDT */
+		mt76_set(dev, MT_WFDMA0_MCU_HOST_INT_ENA, MT_MCU_CMD_WDT_MASK);
+
+		dev->recovery.state = MT_MCU_CMD_NORMAL_STATE;
+		dev->recovery.restart = false;
+		return;
+	}
+
+	if (!(READ_ONCE(dev->recovery.state) & MT_MCU_CMD_STOP_DMA))
 		return;
 
+	dev_info(dev->mt76.dev,"\n%s L1 SER recovery start.",
+		 wiphy_name(dev->mt76.hw->wiphy));
 	ieee80211_stop_queues(mt76_hw(dev));
 	if (phy2)
 		ieee80211_stop_queues(phy2->mt76->hw);
@@ -1856,7 +2032,7 @@ void mt7996_mac_reset_work(struct work_struct *work)
 	mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_DMA_STOPPED);
 
 	if (mt7996_wait_reset_state(dev, MT_MCU_CMD_RESET_DONE)) {
-		mt7996_dma_reset(dev);
+		mt7996_dma_reset(dev, false);
 
 		mt7996_tx_token_put(dev);
 		idr_init(&dev->mt76.token);
@@ -1879,7 +2055,7 @@ void mt7996_mac_reset_work(struct work_struct *work)
 	}
 	local_bh_enable();
 
-	tasklet_schedule(&dev->irq_tasklet);
+	tasklet_schedule(&dev->mt76.irq_tasklet);
 
 	mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_RESET_DONE);
 	mt7996_wait_reset_state(dev, MT_MCU_CMD_NORMAL_STATE);
@@ -1911,6 +2087,101 @@ void mt7996_mac_reset_work(struct work_struct *work)
 		ieee80211_queue_delayed_work(phy3->mt76->hw,
 					     &phy3->mt76->mac_work,
 					     MT7996_WATCHDOG_TIME);
+	dev_info(dev->mt76.dev,"\n%s L1 SER recovery completed.",
+		 wiphy_name(dev->mt76.hw->wiphy));
+}
+
+/* firmware coredump */
+void mt7996_mac_dump_work(struct work_struct *work)
+{
+	const struct mt7996_mem_region *mem_region;
+	struct mt7996_crash_data *crash_data;
+	struct mt7996_dev *dev;
+	struct mt7996_mem_hdr *hdr;
+	size_t buf_len;
+	int i;
+	u32 num;
+	u8 *buf;
+
+	dev = container_of(work, struct mt7996_dev, dump_work);
+
+	mutex_lock(&dev->dump_mutex);
+
+	crash_data = mt7996_coredump_new(dev);
+	if (!crash_data) {
+		mutex_unlock(&dev->dump_mutex);
+		goto skip_coredump;
+	}
+
+	mem_region = mt7996_coredump_get_mem_layout(dev, &num);
+	if (!mem_region || !crash_data->memdump_buf_len) {
+		mutex_unlock(&dev->dump_mutex);
+		goto skip_memdump;
+	}
+
+	buf = crash_data->memdump_buf;
+	buf_len = crash_data->memdump_buf_len;
+
+	/* dumping memory content... */
+	memset(buf, 0, buf_len);
+	for (i = 0; i < num; i++) {
+		if (mem_region->len > buf_len) {
+			dev_warn(dev->mt76.dev, "%s len %zu is too large\n",
+				 mem_region->name, mem_region->len);
+			break;
+		}
+
+		/* reserve space for the header */
+		hdr = (void *)buf;
+		buf += sizeof(*hdr);
+		buf_len -= sizeof(*hdr);
+
+		mt7996_memcpy_fromio(dev, buf, mem_region->start,
+				     mem_region->len);
+
+		hdr->start = mem_region->start;
+		hdr->len = mem_region->len;
+
+		if (!mem_region->len)
+			/* note: the header remains, just with zero length */
+			break;
+
+		buf += mem_region->len;
+		buf_len -= mem_region->len;
+
+		mem_region++;
+	}
+
+	mutex_unlock(&dev->dump_mutex);
+
+skip_memdump:
+	mt7996_coredump_submit(dev);
+skip_coredump:
+	queue_work(dev->mt76.wq, &dev->reset_work);
+}
+
+void mt7996_reset(struct mt7996_dev *dev)
+{
+	if (!dev->recovery.hw_init_done)
+		return;
+
+	if (dev->recovery.hw_full_reset)
+		return;
+
+	/* wm/wa exception: do full recovery */
+	if (READ_ONCE(dev->recovery.state) & MT_MCU_CMD_WDT_MASK) {
+		dev->recovery.restart = true;
+		dev_info(dev->mt76.dev,
+			 "%s indicated firmware crash, attempting recovery\n",
+			 wiphy_name(dev->mt76.hw->wiphy));
+
+		mt7996_irq_disable(dev, MT_INT_MCU_CMD);
+		queue_work(dev->mt76.wq, &dev->dump_work);
+		return;
+	}
+
+	queue_work(dev->mt76.wq, &dev->reset_work);
+	wake_up(&dev->reset_wait);
 }
 
 void mt7996_mac_update_stats(struct mt7996_phy *phy)
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.h b/drivers/net/wireless/mediatek/mt76/mt7996/mac.h
index 27184cbac619..bc4e6c55373e 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.h
@@ -12,6 +12,8 @@
 #define MT_RXD0_LENGTH			GENMASK(15, 0)
 #define MT_RXD0_PKT_TYPE		GENMASK(31, 27)
 
+#define MT_RXD0_MESH			BIT(18)
+#define MT_RXD0_MHCP			BIT(19)
 #define MT_RXD0_NORMAL_ETH_TYPE_OFS	GENMASK(22, 16)
 #define MT_RXD0_NORMAL_IP_SUM		BIT(23)
 #define MT_RXD0_NORMAL_UDP_TCP_SUM	BIT(24)
@@ -20,18 +22,6 @@
 #define MT_RXD0_SW_PKT_TYPE_MAP		0x380F
 #define MT_RXD0_SW_PKT_TYPE_FRAME	0x3801
 
-enum rx_pkt_type {
-	PKT_TYPE_TXS,
-	PKT_TYPE_TXRXV,
-	PKT_TYPE_NORMAL,
-	PKT_TYPE_RX_DUP_RFB,
-	PKT_TYPE_RX_TMR,
-	PKT_TYPE_RETRIEVE,
-	PKT_TYPE_TXRX_NOTIFY,
-	PKT_TYPE_RX_EVENT,
-	PKT_TYPE_RX_FW_MONITOR = 0x0c,
-};
-
 /* RXD DW1 */
 #define MT_RXD1_NORMAL_WLAN_IDX		GENMASK(11, 0)
 #define MT_RXD1_NORMAL_GROUP_1		BIT(16)
@@ -102,8 +92,7 @@ enum rx_pkt_type {
 #define MT_PRXV_NSTS			GENMASK(10, 7)
 #define MT_PRXV_TXBF			BIT(11)
 #define MT_PRXV_HT_AD_CODE		BIT(12)
-#define MT_PRXV_HE_RU_ALLOC_L		GENMASK(31, 28)
-#define MT_PRXV_HE_RU_ALLOC_H		GENMASK(3, 0)
+#define MT_PRXV_HE_RU_ALLOC		GENMASK(30, 22)
 #define MT_PRXV_RCPI3			GENMASK(31, 24)
 #define MT_PRXV_RCPI2			GENMASK(23, 16)
 #define MT_PRXV_RCPI1			GENMASK(15, 8)
@@ -113,34 +102,32 @@ enum rx_pkt_type {
 #define MT_PRXV_TX_MODE			GENMASK(14, 11)
 #define MT_PRXV_FRAME_MODE		GENMASK(2, 0)
 #define MT_PRXV_DCM			BIT(5)
-#define MT_PRXV_NUM_RX			BIT(8, 6)
 
 /* C-RXV */
-#define MT_CRXV_HT_STBC			GENMASK(1, 0)
-#define MT_CRXV_TX_MODE			GENMASK(7, 4)
-#define MT_CRXV_FRAME_MODE		GENMASK(10, 8)
-#define MT_CRXV_HT_SHORT_GI		GENMASK(14, 13)
-#define MT_CRXV_HE_LTF_SIZE		GENMASK(18, 17)
-#define MT_CRXV_HE_LDPC_EXT_SYM		BIT(20)
-#define MT_CRXV_HE_PE_DISAMBIG		BIT(23)
-#define MT_CRXV_HE_NUM_USER		GENMASK(30, 24)
-#define MT_CRXV_HE_UPLINK		BIT(31)
-#define MT_CRXV_HE_RU0			GENMASK(7, 0)
-#define MT_CRXV_HE_RU1			GENMASK(15, 8)
-#define MT_CRXV_HE_RU2			GENMASK(23, 16)
-#define MT_CRXV_HE_RU3			GENMASK(31, 24)
-
-#define MT_CRXV_HE_MU_AID		GENMASK(30, 20)
+#define MT_CRXV_HE_NUM_USER		GENMASK(26, 20)
+#define MT_CRXV_HE_LTF_SIZE		GENMASK(28, 27)
+#define MT_CRXV_HE_LDPC_EXT_SYM		BIT(30)
+
+#define MT_CRXV_HE_PE_DISAMBIG		BIT(1)
+#define MT_CRXV_HE_UPLINK		BIT(2)
+
+#define MT_CRXV_HE_MU_AID		GENMASK(27, 17)
+#define MT_CRXV_HE_BEAM_CHNG		BIT(29)
+
+#define MT_CRXV_HE_DOPPLER		BIT(0)
+#define MT_CRXV_HE_BSS_COLOR		GENMASK(15, 10)
+#define MT_CRXV_HE_TXOP_DUR		GENMASK(19, 17)
 
 #define MT_CRXV_HE_SR_MASK		GENMASK(11, 8)
 #define MT_CRXV_HE_SR1_MASK		GENMASK(16, 12)
 #define MT_CRXV_HE_SR2_MASK             GENMASK(20, 17)
 #define MT_CRXV_HE_SR3_MASK             GENMASK(24, 21)
 
-#define MT_CRXV_HE_BSS_COLOR		GENMASK(5, 0)
-#define MT_CRXV_HE_TXOP_DUR		GENMASK(12, 6)
-#define MT_CRXV_HE_BEAM_CHNG		BIT(13)
-#define MT_CRXV_HE_DOPPLER		BIT(16)
+#define MT_CRXV_HE_RU0			GENMASK(8, 0)
+#define MT_CRXV_HE_RU1			GENMASK(17, 9)
+#define MT_CRXV_HE_RU2			GENMASK(26, 18)
+#define MT_CRXV_HE_RU3_L		GENMASK(31, 27)
+#define MT_CRXV_HE_RU3_H		GENMASK(3, 0)
 
 enum tx_header_format {
 	MT_HDR_FORMAT_802_3,
@@ -239,14 +226,11 @@ enum tx_mgnt_type {
 
 #define MT_TXD6_TX_SRC			GENMASK(31, 30)
 #define MT_TXD6_VTA			BIT(28)
-#define MT_TXD6_FIXED_BW		BIT(25)
-#define MT_TXD6_BW			GENMASK(24, 22)
+#define MT_TXD6_BW			GENMASK(25, 22)
 #define MT_TXD6_TX_RATE			GENMASK(21, 16)
 #define MT_TXD6_TIMESTAMP_OFS_EN	BIT(15)
 #define MT_TXD6_TIMESTAMP_OFS_IDX	GENMASK(14, 10)
 #define MT_TXD6_MSDU_CNT		GENMASK(9, 4)
-#define MT_TXD6_SPE_ID_IDX		BIT(10)
-#define MT_TXD6_ANT_ID			GENMASK(7, 4)
 #define MT_TXD6_DIS_MAT			BIT(3)
 #define MT_TXD6_DAS			BIT(2)
 #define MT_TXD6_AMSDU_CAP		BIT(1)
@@ -260,7 +244,7 @@ enum tx_mgnt_type {
 #define MT_TXD7_UDP_TCP_SUM		BIT(15)
 #define MT_TXD7_TX_TIME			GENMASK(9, 0)
 
-#define MT_TX_RATE_STBC			BIT(13)
+#define MT_TX_RATE_STBC			BIT(14)
 #define MT_TX_RATE_NSS			GENMASK(13, 10)
 #define MT_TX_RATE_MODE			GENMASK(9, 6)
 #define MT_TX_RATE_SU_EXT_TONE		BIT(5)
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/main.c b/drivers/net/wireless/mediatek/mt76/mt7996/main.c
index 1ba22d147949..f306e9c50ea3 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/main.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/main.c
@@ -5,6 +5,7 @@
 
 #include "mt7996.h"
 #include "mcu.h"
+#include "mac.h"
 
 static bool mt7996_dev_running(struct mt7996_dev *dev)
 {
@@ -22,17 +23,13 @@ static bool mt7996_dev_running(struct mt7996_dev *dev)
 	return phy && test_bit(MT76_STATE_RUNNING, &phy->mt76->state);
 }
 
-static int mt7996_start(struct ieee80211_hw *hw)
+int mt7996_run(struct ieee80211_hw *hw)
 {
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 	struct mt7996_phy *phy = mt7996_hw_phy(hw);
 	bool running;
 	int ret;
 
-	flush_work(&dev->init_work);
-
-	mutex_lock(&dev->mt76.mutex);
-
 	running = mt7996_dev_running(dev);
 	if (!running) {
 		ret = mt7996_mcu_set_hdr_trans(dev, true);
@@ -52,10 +49,6 @@ static int mt7996_start(struct ieee80211_hw *hw)
 
 	set_bit(MT76_STATE_RUNNING, &phy->mt76->state);
 
-	ieee80211_iterate_interfaces(dev->mt76.hw,
-				     IEEE80211_IFACE_ITER_RESUME_ALL,
-				     mt7996_mcu_set_pm, dev->mt76.hw);
-
 	ieee80211_queue_delayed_work(hw, &phy->mt76->mac_work,
 				     MT7996_WATCHDOG_TIME);
 
@@ -63,6 +56,18 @@ static int mt7996_start(struct ieee80211_hw *hw)
 		mt7996_mac_reset_counters(phy);
 
 out:
+	return ret;
+}
+
+static int mt7996_start(struct ieee80211_hw *hw)
+{
+	struct mt7996_dev *dev = mt7996_hw_dev(hw);
+	int ret;
+
+	flush_work(&dev->init_work);
+
+	mutex_lock(&dev->mt76.mutex);
+	ret = mt7996_run(hw);
 	mutex_unlock(&dev->mt76.mutex);
 
 	return ret;
@@ -79,10 +84,6 @@ static void mt7996_stop(struct ieee80211_hw *hw)
 
 	clear_bit(MT76_STATE_RUNNING, &phy->mt76->state);
 
-	ieee80211_iterate_interfaces(dev->mt76.hw,
-				     IEEE80211_IFACE_ITER_RESUME_ALL,
-				     mt7996_mcu_set_pm, dev->mt76.hw);
-
 	mutex_unlock(&dev->mt76.mutex);
 }
 
@@ -219,8 +220,12 @@ static int mt7996_add_interface(struct ieee80211_hw *hw,
 		vif->offload_flags = 0;
 	vif->offload_flags |= IEEE80211_OFFLOAD_ENCAP_4ADDR;
 
+	if (phy->mt76->chandef.chan->band != NL80211_BAND_2GHZ)
+		mvif->basic_rates_idx = MT7996_BASIC_RATES_TBL + 4;
+	else
+		mvif->basic_rates_idx = MT7996_BASIC_RATES_TBL;
+
 	mt7996_init_bitrate_mask(vif);
-	memset(&mvif->cap, -1, sizeof(mvif->cap));
 
 	mt7996_mcu_add_bss_info(phy, vif, true);
 	mt7996_mcu_add_sta(dev, vif, NULL, true);
@@ -496,11 +501,41 @@ mt7996_update_bss_color(struct ieee80211_hw *hw,
 	}
 }
 
+static u8
+mt7996_get_rates_table(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		       bool beacon, bool mcast)
+{
+	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
+	struct mt76_phy *mphy = hw->priv;
+	u16 rate;
+	u8 i, idx, ht;
+
+	rate = mt76_connac2_mac_tx_rate_val(mphy, vif, beacon, mcast);
+	ht = FIELD_GET(MT_TX_RATE_MODE, rate) > MT_PHY_TYPE_OFDM;
+
+	if (beacon && ht) {
+		struct mt7996_dev *dev = mt7996_hw_dev(hw);
+
+		/* must odd index */
+		idx = MT7996_BEACON_RATES_TBL + 2 * (mvif->mt76.idx % 20);
+		mt7996_mac_set_fixed_rate_table(dev, idx, rate);
+		return idx;
+	}
+
+	idx = FIELD_GET(MT_TX_RATE_IDX, rate);
+	for (i = 0; i < ARRAY_SIZE(mt76_rates); i++)
+		if ((mt76_rates[i].hw_value & GENMASK(7, 0)) == idx)
+			return MT7996_BASIC_RATES_TBL + i;
+
+	return mvif->basic_rates_idx;
+}
+
 static void mt7996_bss_info_changed(struct ieee80211_hw *hw,
 				    struct ieee80211_vif *vif,
 				    struct ieee80211_bss_conf *info,
 				    u64 changed)
 {
+	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	struct mt7996_phy *phy = mt7996_hw_phy(hw);
 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
 
@@ -532,6 +567,14 @@ static void mt7996_bss_info_changed(struct ieee80211_hw *hw,
 		}
 	}
 
+	if (changed & BSS_CHANGED_MCAST_RATE)
+		mvif->mcast_rates_idx =
+			mt7996_get_rates_table(hw, vif, false, true);
+
+	if (changed & BSS_CHANGED_BASIC_RATES)
+		mvif->basic_rates_idx =
+			mt7996_get_rates_table(hw, vif, false, false);
+
 	if (changed & BSS_CHANGED_BEACON_ENABLED && info->enable_beacon) {
 		mt7996_mcu_add_bss_info(phy, vif, true);
 		mt7996_mcu_add_sta(dev, vif, NULL, true);
@@ -548,8 +591,12 @@ static void mt7996_bss_info_changed(struct ieee80211_hw *hw,
 		mt7996_update_bss_color(hw, vif, &info->he_bss_color);
 
 	if (changed & (BSS_CHANGED_BEACON |
-		       BSS_CHANGED_BEACON_ENABLED))
+		       BSS_CHANGED_BEACON_ENABLED)) {
+		mvif->beacon_rates_idx =
+			mt7996_get_rates_table(hw, vif, true, false);
+
 		mt7996_mcu_add_beacon(hw, vif, info->enable_beacon);
+	}
 
 	if (changed & BSS_CHANGED_UNSOL_BCAST_PROBE_RESP ||
 	    changed & BSS_CHANGED_FILS_DISCOVERY)
@@ -891,6 +938,7 @@ mt7996_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
 	mt7996_set_stream_vht_txbf_caps(phy);
 	mt7996_set_stream_he_eht_caps(phy);
 
+	/* TODO: update bmc_wtbl spe_idx when antenna changes */
 	mutex_unlock(&dev->mt76.mutex);
 
 	return 0;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c
index dbe30832fd88..88e2f9d0e513 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c
@@ -422,7 +422,8 @@ mt7996_mcu_ie_countdown(struct mt7996_dev *dev, struct sk_buff *skb)
 	if (hdr->band && dev->mt76.phys[hdr->band])
 		mphy = dev->mt76.phys[hdr->band];
 
-	tail = skb->data + le16_to_cpu(rxd->len);
+	tail = skb->data + skb->len;
+	data += sizeof(struct header);
 	while (data + sizeof(struct tlv) < tail && le16_to_cpu(tlv->len)) {
 		switch (le16_to_cpu(tlv->tag)) {
 		case UNI_EVENT_IE_COUNTDOWN_CSA:
@@ -596,25 +597,24 @@ mt7996_mcu_bss_he_tlv(struct sk_buff *skb, struct ieee80211_vif *vif,
 }
 
 static void
-mt7996_mcu_bss_bmc_tlv(struct sk_buff *skb, struct mt7996_phy *phy)
+mt7996_mcu_bss_bmc_tlv(struct sk_buff *skb, struct ieee80211_vif *vif,
+		       struct mt7996_phy *phy)
 {
+	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	struct bss_rate_tlv *bmc;
 	struct cfg80211_chan_def *chandef = &phy->mt76->chandef;
 	enum nl80211_band band = chandef->chan->band;
 	struct tlv *tlv;
+	u8 idx = mvif->mcast_rates_idx ?
+		 mvif->mcast_rates_idx : mvif->basic_rates_idx;
 
 	tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_RATE, sizeof(*bmc));
 
 	bmc = (struct bss_rate_tlv *)tlv;
-	if (band == NL80211_BAND_2GHZ) {
-		bmc->short_preamble = true;
-	} else {
-		bmc->bc_trans = cpu_to_le16(0x8080);
-		bmc->mc_trans = cpu_to_le16(0x8080);
-		bmc->bc_fixed_rate = 1;
-		bmc->mc_fixed_rate = 1;
-		bmc->short_preamble = 1;
-	}
+
+	bmc->short_preamble = (band == NL80211_BAND_2GHZ);
+	bmc->bc_fixed_rate = idx;
+	bmc->mc_fixed_rate = idx;
 }
 
 static void
@@ -822,7 +822,7 @@ int mt7996_mcu_add_bss_info(struct mt7996_phy *phy,
 
 	if (enable) {
 		mt7996_mcu_bss_rfch_tlv(skb, vif, phy);
-		mt7996_mcu_bss_bmc_tlv(skb, phy);
+		mt7996_mcu_bss_bmc_tlv(skb, vif, phy);
 		mt7996_mcu_bss_ra_tlv(skb, vif, phy);
 		mt7996_mcu_bss_txcmd_tlv(skb, true);
 
@@ -1022,6 +1022,7 @@ mt7996_mcu_sta_amsdu_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
 	struct tlv *tlv;
 
 	if (vif->type != NL80211_IFTYPE_STATION &&
+	    vif->type != NL80211_IFTYPE_MESH_POINT &&
 	    vif->type != NL80211_IFTYPE_AP)
 		return;
 
@@ -1053,7 +1054,6 @@ static inline bool
 mt7996_is_ebf_supported(struct mt7996_phy *phy, struct ieee80211_vif *vif,
 			struct ieee80211_sta *sta, bool bfee)
 {
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
 	int sts = hweight16(phy->mt76->chainmask);
 
 	if (vif->type != NL80211_IFTYPE_STATION &&
@@ -1068,10 +1068,10 @@ mt7996_is_ebf_supported(struct mt7996_phy *phy, struct ieee80211_vif *vif,
 		struct ieee80211_eht_cap_elem_fixed *pe = &pc->eht_cap_elem;
 
 		if (bfee)
-			return mvif->cap.eht_su_ebfee &&
+			return vif->bss_conf.eht_su_beamformee &&
 			       EHT_PHY(CAP0_SU_BEAMFORMEE, pe->phy_cap_info[0]);
 		else
-			return mvif->cap.eht_su_ebfer &&
+			return vif->bss_conf.eht_su_beamformer &&
 			       EHT_PHY(CAP0_SU_BEAMFORMER, pe->phy_cap_info[0]);
 	}
 
@@ -1079,10 +1079,10 @@ mt7996_is_ebf_supported(struct mt7996_phy *phy, struct ieee80211_vif *vif,
 		struct ieee80211_he_cap_elem *pe = &sta->deflink.he_cap.he_cap_elem;
 
 		if (bfee)
-			return mvif->cap.he_su_ebfee &&
+			return vif->bss_conf.he_su_beamformee &&
 			       HE_PHY(CAP3_SU_BEAMFORMER, pe->phy_cap_info[3]);
 		else
-			return mvif->cap.he_su_ebfer &&
+			return vif->bss_conf.he_su_beamformer &&
 			       HE_PHY(CAP4_SU_BEAMFORMEE, pe->phy_cap_info[4]);
 	}
 
@@ -1090,10 +1090,10 @@ mt7996_is_ebf_supported(struct mt7996_phy *phy, struct ieee80211_vif *vif,
 		u32 cap = sta->deflink.vht_cap.cap;
 
 		if (bfee)
-			return mvif->cap.vht_su_ebfee &&
+			return vif->bss_conf.vht_su_beamformee &&
 			       (cap & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE);
 		else
-			return mvif->cap.vht_su_ebfer &&
+			return vif->bss_conf.vht_su_beamformer &&
 			       (cap & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE);
 	}
 
@@ -1471,6 +1471,12 @@ mt7996_mcu_sta_hdr_trans_tlv(struct mt7996_dev *dev, struct sk_buff *skb,
 		hdr_trans->to_ds = true;
 		hdr_trans->from_ds = true;
 	}
+
+	if (vif->type == NL80211_IFTYPE_MESH_POINT) {
+		hdr_trans->to_ds = true;
+		hdr_trans->from_ds = true;
+		hdr_trans->mesh = true;
+	}
 }
 
 static enum mcu_mmps_mode
@@ -1572,7 +1578,7 @@ mt7996_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7996_dev *dev,
 			cap |= STA_CAP_TX_STBC;
 		if (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_RX_STBC)
 			cap |= STA_CAP_RX_STBC;
-		if (mvif->cap.ht_ldpc &&
+		if (vif->bss_conf.ht_ldpc &&
 		    (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_LDPC_CODING))
 			cap |= STA_CAP_LDPC;
 
@@ -1598,7 +1604,7 @@ mt7996_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7996_dev *dev,
 			cap |= STA_CAP_VHT_TX_STBC;
 		if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_1)
 			cap |= STA_CAP_VHT_RX_STBC;
-		if (mvif->cap.vht_ldpc &&
+		if (vif->bss_conf.vht_ldpc &&
 		    (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXLDPC))
 			cap |= STA_CAP_VHT_LDPC;
 
@@ -1694,8 +1700,8 @@ int mt7996_mcu_add_sta(struct mt7996_dev *dev, struct ieee80211_vif *vif,
 		return PTR_ERR(skb);
 
 	/* starec basic */
-	mt76_connac_mcu_sta_basic_tlv(skb, vif, sta, enable,
-			!rcu_access_pointer(dev->mt76.wcid[msta->wcid.idx]));
+	mt76_connac_mcu_sta_basic_tlv(&dev->mt76, skb, vif, sta, enable,
+				      !rcu_access_pointer(dev->mt76.wcid[msta->wcid.idx]));
 	if (!enable)
 		goto out;
 
@@ -1906,105 +1912,10 @@ mt7996_mcu_beacon_cont(struct mt7996_dev *dev, struct ieee80211_vif *vif,
 	}
 
 	buf = (u8 *)bcn + sizeof(*bcn) - MAX_BEACON_SIZE;
-	mt7996_mac_write_txwi(dev, (__le32 *)buf, skb, wcid, 0, NULL,
+	mt7996_mac_write_txwi(dev, (__le32 *)buf, skb, wcid, NULL, 0, 0,
 			      BSS_CHANGED_BEACON);
-	memcpy(buf + MT_TXD_SIZE, skb->data, skb->len);
-}
 
-static void
-mt7996_mcu_beacon_check_caps(struct mt7996_phy *phy, struct ieee80211_vif *vif,
-			     struct sk_buff *skb)
-{
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct mt7996_vif_cap *vc = &mvif->cap;
-	const struct ieee80211_eht_cap_elem_fixed *eht;
-	const struct ieee80211_he_cap_elem *he;
-	const struct ieee80211_vht_cap *vht;
-	const struct ieee80211_ht_cap *ht;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)skb->data;
-	const u8 *ie;
-	u32 len, bc;
-
-	/* Check missing configuration options to allow AP mode in mac80211
-	 * to remain in sync with hostapd settings, and get a subset of
-	 * beacon and hardware capabilities.
-	 */
-	if (WARN_ON_ONCE(skb->len <= (mgmt->u.beacon.variable - skb->data)))
-		return;
-
-	memset(vc, 0, sizeof(*vc));
-
-	len = skb->len - (mgmt->u.beacon.variable - skb->data);
-
-	ie = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY, mgmt->u.beacon.variable,
-			      len);
-	if (ie && ie[1] >= sizeof(*ht)) {
-		ht = (void *)(ie + 2);
-		vc->ht_ldpc |= !!(le16_to_cpu(ht->cap_info) &
-				  IEEE80211_HT_CAP_LDPC_CODING);
-	}
-
-	ie = cfg80211_find_ie(WLAN_EID_VHT_CAPABILITY, mgmt->u.beacon.variable,
-			      len);
-	if (ie && ie[1] >= sizeof(*vht)) {
-		u32 pc = phy->mt76->sband_5g.sband.vht_cap.cap;
-
-		vht = (void *)(ie + 2);
-		bc = le32_to_cpu(vht->vht_cap_info);
-
-		vc->vht_ldpc |= !!(bc & IEEE80211_VHT_CAP_RXLDPC);
-		vc->vht_su_ebfer =
-			(bc & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE) &&
-			(pc & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE);
-		vc->vht_su_ebfee =
-			(bc & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE) &&
-			(pc & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE);
-		vc->vht_mu_ebfer =
-			(bc & IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE) &&
-			(pc & IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE);
-		vc->vht_mu_ebfee =
-			(bc & IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE) &&
-			(pc & IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE);
-	}
-
-	ie = cfg80211_find_ext_ie(WLAN_EID_EXT_HE_CAPABILITY,
-				  mgmt->u.beacon.variable, len);
-	if (ie && ie[1] >= sizeof(*he) + 1) {
-		const struct ieee80211_sta_he_cap *pc =
-			mt76_connac_get_he_phy_cap(phy->mt76, vif);
-		const struct ieee80211_he_cap_elem *pe = &pc->he_cap_elem;
-
-		he = (void *)(ie + 3);
-
-		vc->he_ldpc =
-			HE_PHY(CAP1_LDPC_CODING_IN_PAYLOAD, pe->phy_cap_info[1]);
-		vc->he_su_ebfer =
-			HE_PHY(CAP3_SU_BEAMFORMER, he->phy_cap_info[3]) &&
-			HE_PHY(CAP3_SU_BEAMFORMER, pe->phy_cap_info[3]);
-		vc->he_su_ebfee =
-			HE_PHY(CAP4_SU_BEAMFORMEE, he->phy_cap_info[4]) &&
-			HE_PHY(CAP4_SU_BEAMFORMEE, pe->phy_cap_info[4]);
-		vc->he_mu_ebfer =
-			HE_PHY(CAP4_MU_BEAMFORMER, he->phy_cap_info[4]) &&
-			HE_PHY(CAP4_MU_BEAMFORMER, pe->phy_cap_info[4]);
-	}
-
-	ie = cfg80211_find_ext_ie(WLAN_EID_EXT_EHT_CAPABILITY,
-				  mgmt->u.beacon.variable, len);
-	if (ie && ie[1] >= sizeof(*eht) + 1) {
-		const struct ieee80211_sta_eht_cap *pc =
-			mt76_connac_get_eht_phy_cap(phy->mt76, vif);
-		const struct ieee80211_eht_cap_elem_fixed *pe = &pc->eht_cap_elem;
-
-		eht = (void *)(ie + 3);
-
-		vc->eht_su_ebfer =
-			EHT_PHY(CAP0_SU_BEAMFORMER, eht->phy_cap_info[0]) &&
-			EHT_PHY(CAP0_SU_BEAMFORMER, pe->phy_cap_info[0]);
-		vc->eht_su_ebfee =
-			EHT_PHY(CAP0_SU_BEAMFORMEE, eht->phy_cap_info[0]) &&
-			EHT_PHY(CAP0_SU_BEAMFORMEE, pe->phy_cap_info[0]);
-	}
+	memcpy(buf + MT_TXD_SIZE, skb->data, skb->len);
 }
 
 int mt7996_mcu_add_beacon(struct ieee80211_hw *hw,
@@ -2045,8 +1956,6 @@ int mt7996_mcu_add_beacon(struct ieee80211_hw *hw,
 	info = IEEE80211_SKB_CB(skb);
 	info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->mt76->band_idx);
 
-	mt7996_mcu_beacon_check_caps(phy, vif, skb);
-
 	mt7996_mcu_beacon_cont(dev, vif, rskb, skb, bcn, &offs);
 	/* TODO: subtag - 11v MBSSID */
 	mt7996_mcu_beacon_cntdwn(vif, rskb, skb, &offs);
@@ -2115,8 +2024,7 @@ int mt7996_mcu_beacon_inband_discov(struct mt7996_dev *dev,
 
 	buf = (u8 *)tlv + sizeof(*discov) - MAX_INBAND_FRAME_SIZE;
 
-	mt7996_mac_write_txwi(dev, (__le32 *)buf, skb, wcid, 0, NULL,
-			      changed);
+	mt7996_mac_write_txwi(dev, (__le32 *)buf, skb, wcid, NULL, 0, 0, changed);
 
 	memcpy(buf + MT_TXD_SIZE, skb->data, skb->len);
 
@@ -2523,17 +2431,10 @@ mt7996_mcu_init_rx_airtime(struct mt7996_dev *dev)
 				     MCU_WM_UNI_CMD(VOW), true);
 }
 
-int mt7996_mcu_init(struct mt7996_dev *dev)
+int mt7996_mcu_init_firmware(struct mt7996_dev *dev)
 {
-	static const struct mt76_mcu_ops mt7996_mcu_ops = {
-		.headroom = sizeof(struct mt76_connac2_mcu_txd), /* reuse */
-		.mcu_skb_send_msg = mt7996_mcu_send_message,
-		.mcu_parse_response = mt7996_mcu_parse_response,
-	};
 	int ret;
 
-	dev->mt76.mcu_ops = &mt7996_mcu_ops;
-
 	/* force firmware operation mode into normal state,
 	 * which should be set before firmware download stage.
 	 */
@@ -2574,6 +2475,19 @@ int mt7996_mcu_init(struct mt7996_dev *dev)
 				 MCU_WA_PARAM_RED, 0, 0);
 }
 
+int mt7996_mcu_init(struct mt7996_dev *dev)
+{
+	static const struct mt76_mcu_ops mt7996_mcu_ops = {
+		.headroom = sizeof(struct mt76_connac2_mcu_txd), /* reuse */
+		.mcu_skb_send_msg = mt7996_mcu_send_message,
+		.mcu_parse_response = mt7996_mcu_parse_response,
+	};
+
+	dev->mt76.mcu_ops = &mt7996_mcu_ops;
+
+	return mt7996_mcu_init_firmware(dev);
+}
+
 void mt7996_mcu_exit(struct mt7996_dev *dev)
 {
 	mt7996_mcu_restart(&dev->mt76);
@@ -3133,7 +3047,7 @@ int mt7996_mcu_get_chip_config(struct mt7996_dev *dev, u32 *cap)
 			break;
 		default:
 			break;
-		};
+		}
 
 		buf += le16_to_cpu(tlv->len);
 	}
@@ -3576,32 +3490,6 @@ int mt7996_mcu_twt_agrt_update(struct mt7996_dev *dev,
 				 &req, sizeof(req), true);
 }
 
-void mt7996_mcu_set_pm(void *priv, u8 *mac, struct ieee80211_vif *vif)
-{
-#define EXIT_PM_STATE	0
-#define ENTER_PM_STATE	1
-	struct ieee80211_hw *hw = priv;
-	struct mt7996_dev *dev = mt7996_hw_dev(hw);
-	struct mt7996_phy *phy = mt7996_hw_phy(hw);
-	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
-	struct bss_power_save *ps;
-	struct sk_buff *skb;
-	struct tlv *tlv;
-	bool running = test_bit(MT76_STATE_RUNNING, &phy->mt76->state);
-
-	skb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &mvif->mt76,
-					 MT7996_BSS_UPDATE_MAX_SIZE);
-	if (IS_ERR(skb))
-		return;
-
-	tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_PS, sizeof(*ps));
-	ps = (struct bss_power_save *)tlv;
-	ps->profile = running ? EXIT_PM_STATE : ENTER_PM_STATE;
-
-	mt76_mcu_skb_send_msg(&dev->mt76, skb,
-			      MCU_WMWA_UNI_CMD(BSS_INFO_UPDATE), true);
-}
-
 int mt7996_mcu_set_rts_thresh(struct mt7996_phy *phy, u32 val)
 {
 	struct {
@@ -3733,6 +3621,22 @@ int mt7996_mcu_rf_regval(struct mt7996_dev *dev, u32 regidx, u32 *val, bool set)
 	return 0;
 }
 
+int mt7996_mcu_trigger_assert(struct mt7996_dev *dev)
+{
+	struct {
+		__le16 tag;
+		__le16 len;
+		u8 enable;
+		u8 rsv[3];
+	} __packed req = {
+		.len = cpu_to_le16(sizeof(req) - 4),
+		.enable = true,
+	};
+
+	return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(ASSERT_DUMP),
+				 &req, sizeof(req), false);
+}
+
 int mt7996_mcu_set_rro(struct mt7996_dev *dev, u16 tag, u8 val)
 {
 	struct {
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h
index dd0c5ac52703..d7075a4d0667 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h
@@ -396,7 +396,7 @@ struct sta_rec_hdr_trans {
 	u8 from_ds;
 	u8 to_ds;
 	u8 dis_rx_hdr_tran;
-	u8 rsv;
+	u8 mesh;
 } __packed;
 
 struct hdr_trans_en {
@@ -648,23 +648,21 @@ enum {
 };
 
 enum {
-	UNI_CMD_SER_QUERY = 0x0,
-	UNI_CMD_SER_SET = 0x2,
-	UNI_CMD_SER_TRIGGER = 0x3,
-};
-
-enum {
-	SER_QUERY,
+	UNI_CMD_SER_QUERY,
 	/* recovery */
-	SER_SET_RECOVER_L1,
-	SER_SET_RECOVER_L2,
-	SER_SET_RECOVER_L3_RX_ABORT,
-	SER_SET_RECOVER_L3_TX_ABORT,
-	SER_SET_RECOVER_L3_TX_DISABLE,
-	SER_SET_RECOVER_L3_BF,
+	UNI_CMD_SER_SET_RECOVER_L1,
+	UNI_CMD_SER_SET_RECOVER_L2,
+	UNI_CMD_SER_SET_RECOVER_L3_RX_ABORT,
+	UNI_CMD_SER_SET_RECOVER_L3_TX_ABORT,
+	UNI_CMD_SER_SET_RECOVER_L3_TX_DISABLE,
+	UNI_CMD_SER_SET_RECOVER_L3_BF,
+	UNI_CMD_SER_SET_RECOVER_L4_MDP,
+	UNI_CMD_SER_SET_RECOVER_FULL,
+	UNI_CMD_SER_SET_SYSTEM_ASSERT,
 	/* action */
-	SER_ENABLE = 2,
-	SER_RECOVER
+	UNI_CMD_SER_ENABLE = 1,
+	UNI_CMD_SER_SET,
+	UNI_CMD_SER_TRIGGER
 };
 
 enum {
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c
index 902370a2a639..3a591a7b47ae 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c
@@ -162,6 +162,14 @@ static u32 __mt7996_reg_addr(struct mt7996_dev *dev, u32 addr)
 	return mt7996_reg_map_l2(dev, addr);
 }
 
+void mt7996_memcpy_fromio(struct mt7996_dev *dev, void *buf, u32 offset,
+			  size_t len)
+{
+	u32 addr = __mt7996_reg_addr(dev, offset);
+
+	memcpy_fromio(buf, dev->mt76.mmio.regs + addr, len);
+}
+
 static u32 mt7996_rr(struct mt76_dev *mdev, u32 offset)
 {
 	struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76);
@@ -251,7 +259,7 @@ static void mt7996_rx_poll_complete(struct mt76_dev *mdev,
 /* TODO: support 2/4/6/8 MSI-X vectors */
 static void mt7996_irq_tasklet(struct tasklet_struct *t)
 {
-	struct mt7996_dev *dev = from_tasklet(dev, t, irq_tasklet);
+	struct mt7996_dev *dev = from_tasklet(dev, t, mt76.irq_tasklet);
 	u32 i, intr, mask, intr1;
 
 	mt76_wr(dev, MT_INT_MASK_CSR, 0);
@@ -289,10 +297,9 @@ static void mt7996_irq_tasklet(struct tasklet_struct *t)
 		u32 val = mt76_rr(dev, MT_MCU_CMD);
 
 		mt76_wr(dev, MT_MCU_CMD, val);
-		if (val & MT_MCU_CMD_ERROR_MASK) {
-			dev->reset_state = val;
-			ieee80211_queue_work(mt76_hw(dev), &dev->reset_work);
-			wake_up(&dev->reset_wait);
+		if (val & (MT_MCU_CMD_ERROR_MASK | MT_MCU_CMD_WDT_MASK)) {
+			dev->recovery.state = val;
+			mt7996_reset(dev);
 		}
 	}
 }
@@ -308,7 +315,7 @@ irqreturn_t mt7996_irq_handler(int irq, void *dev_instance)
 	if (!test_bit(MT76_STATE_INITIALIZED, &dev->mphy.state))
 		return IRQ_NONE;
 
-	tasklet_schedule(&dev->irq_tasklet);
+	tasklet_schedule(&dev->mt76.irq_tasklet);
 
 	return IRQ_HANDLED;
 }
@@ -320,6 +327,7 @@ struct mt7996_dev *mt7996_mmio_probe(struct device *pdev,
 		/* txwi_size = txd size + txp size */
 		.txwi_size = MT_TXD_SIZE + sizeof(struct mt76_connac_fw_txp),
 		.drv_flags = MT_DRV_TXWI_NO_FREE |
+			     MT_DRV_AMSDU_OFFLOAD |
 			     MT_DRV_HW_MGMT_TXQ,
 		.survey_flags = SURVEY_INFO_TIME_TX |
 				SURVEY_INFO_TIME_RX |
@@ -330,7 +338,6 @@ struct mt7996_dev *mt7996_mmio_probe(struct device *pdev,
 		.rx_skb = mt7996_queue_rx_skb,
 		.rx_check = mt7996_rx_check,
 		.rx_poll_complete = mt7996_rx_poll_complete,
-		.sta_ps = mt7996_sta_ps,
 		.sta_add = mt7996_mac_sta_add,
 		.sta_remove = mt7996_mac_sta_remove,
 		.update_survey = mt7996_update_channel,
@@ -349,7 +356,7 @@ struct mt7996_dev *mt7996_mmio_probe(struct device *pdev,
 	if (ret)
 		goto error;
 
-	tasklet_setup(&dev->irq_tasklet, mt7996_irq_tasklet);
+	tasklet_setup(&mdev->irq_tasklet, mt7996_irq_tasklet);
 
 	mt76_wr(dev, MT_INT_MASK_CSR, 0);
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h
index 018dfd2b36b0..4d7dcb95a620 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h
@@ -43,6 +43,10 @@
 #define MT7996_MAX_STA_TWT_AGRT		8
 #define MT7996_MAX_QUEUE		(__MT_RXQ_MAX +	__MT_MCUQ_MAX + 3)
 
+/* NOTE: used to map mt76_rates. idx may change if firmware expands table */
+#define MT7996_BASIC_RATES_TBL		11
+#define MT7996_BEACON_RATES_TBL		25
+
 struct mt7996_vif;
 struct mt7996_sta;
 struct mt7996_dfs_pulse;
@@ -112,30 +116,18 @@ struct mt7996_sta {
 	} twt;
 };
 
-struct mt7996_vif_cap {
-	bool ht_ldpc:1;
-	bool vht_ldpc:1;
-	bool he_ldpc:1;
-	bool vht_su_ebfer:1;
-	bool vht_su_ebfee:1;
-	bool vht_mu_ebfer:1;
-	bool vht_mu_ebfee:1;
-	bool he_su_ebfer:1;
-	bool he_su_ebfee:1;
-	bool he_mu_ebfer:1;
-	bool eht_su_ebfer:1;
-	bool eht_su_ebfee:1;
-};
-
 struct mt7996_vif {
 	struct mt76_vif mt76; /* must be first */
 
-	struct mt7996_vif_cap cap;
 	struct mt7996_sta sta;
 	struct mt7996_phy *phy;
 
 	struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS];
 	struct cfg80211_bitrate_mask bitrate_mask;
+
+	u8 basic_rates_idx;
+	u8 mcast_rates_idx;
+	u8 beacon_rates_idx;
 };
 
 /* per-phy stats.  */
@@ -192,6 +184,15 @@ struct mib_stats {
 	u32 tx_amsdu_cnt;
 };
 
+/* crash-dump */
+struct mt7996_crash_data {
+	guid_t guid;
+	struct timespec64 timestamp;
+
+	u8 *memdump_buf;
+	size_t memdump_buf_len;
+};
+
 struct mt7996_hif {
 	struct list_head list;
 
@@ -238,7 +239,6 @@ struct mt7996_dev {
 	u32 q_wfdma_mask;
 
 	const struct mt76_bus_ops *bus_ops;
-	struct tasklet_struct irq_tasklet;
 	struct mt7996_phy phy;
 
 	/* monitor rx chain configured channel */
@@ -251,9 +251,25 @@ struct mt7996_dev {
 
 	struct work_struct init_work;
 	struct work_struct rc_work;
+	struct work_struct dump_work;
 	struct work_struct reset_work;
 	wait_queue_head_t reset_wait;
-	u32 reset_state;
+	struct {
+		u32 state;
+		u32 wa_reset_count;
+		u32 wm_reset_count;
+		bool hw_full_reset:1;
+		bool hw_init_done:1;
+		bool restart:1;
+	} recovery;
+
+	/* protects coredump data */
+	struct mutex dump_mutex;
+#ifdef CONFIG_DEV_COREDUMP
+	struct {
+		struct mt7996_crash_data *crash_data;
+	} coredump;
+#endif
 
 	struct list_head sta_rc_list;
 	struct list_head sta_poll_list;
@@ -386,9 +402,16 @@ int mt7996_eeprom_get_target_power(struct mt7996_dev *dev,
 				   struct ieee80211_channel *chan);
 s8 mt7996_eeprom_get_power_delta(struct mt7996_dev *dev, int band);
 int mt7996_dma_init(struct mt7996_dev *dev);
+void mt7996_dma_reset(struct mt7996_dev *dev, bool force);
 void mt7996_dma_prefetch(struct mt7996_dev *dev);
 void mt7996_dma_cleanup(struct mt7996_dev *dev);
+void mt7996_init_txpower(struct mt7996_dev *dev,
+			 struct ieee80211_supported_band *sband);
+int mt7996_txbf_init(struct mt7996_dev *dev);
+void mt7996_reset(struct mt7996_dev *dev);
+int mt7996_run(struct ieee80211_hw *hw);
 int mt7996_mcu_init(struct mt7996_dev *dev);
+int mt7996_mcu_init_firmware(struct mt7996_dev *dev);
 int mt7996_mcu_twt_agrt_update(struct mt7996_dev *dev,
 			       struct mt7996_vif *mvif,
 			       struct mt7996_twt_flow *flow,
@@ -432,7 +455,6 @@ int mt7996_mcu_set_pulse_th(struct mt7996_dev *dev,
 int mt7996_mcu_set_radar_th(struct mt7996_dev *dev, int index,
 			    const struct mt7996_dfs_pattern *pattern);
 int mt7996_mcu_set_radio_en(struct mt7996_phy *phy, bool enable);
-void mt7996_mcu_set_pm(void *priv, u8 *mac, struct ieee80211_vif *vif);
 int mt7996_mcu_set_rts_thresh(struct mt7996_phy *phy, u32 val);
 int mt7996_mcu_get_chan_mib_info(struct mt7996_phy *phy, bool chan_switch);
 int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 index,
@@ -445,6 +467,7 @@ int mt7996_mcu_set_rro(struct mt7996_dev *dev, u16 tag, u8 val);
 int mt7996_mcu_wa_cmd(struct mt7996_dev *dev, int cmd, u32 a1, u32 a2, u32 a3);
 int mt7996_mcu_fw_log_2_host(struct mt7996_dev *dev, u8 type, u8 ctrl);
 int mt7996_mcu_fw_dbg_ctrl(struct mt7996_dev *dev, u32 module, u8 level);
+int mt7996_mcu_trigger_assert(struct mt7996_dev *dev);
 void mt7996_mcu_rx_event(struct mt7996_dev *dev, struct sk_buff *skb);
 void mt7996_mcu_exit(struct mt7996_dev *dev);
 
@@ -468,7 +491,7 @@ static inline void mt7996_irq_enable(struct mt7996_dev *dev, u32 mask)
 	else
 		mt76_set_irq_mask(&dev->mt76, 0, 0, mask);
 
-	tasklet_schedule(&dev->irq_tasklet);
+	tasklet_schedule(&dev->mt76.irq_tasklet);
 }
 
 static inline void mt7996_irq_disable(struct mt7996_dev *dev, u32 mask)
@@ -479,6 +502,10 @@ static inline void mt7996_irq_disable(struct mt7996_dev *dev, u32 mask)
 		mt76_set_irq_mask(&dev->mt76, MT_INT_MASK_CSR, mask, 0);
 }
 
+void mt7996_memcpy_fromio(struct mt7996_dev *dev, void *buf, u32 offset,
+			  size_t len);
+
+void mt7996_mac_init(struct mt7996_dev *dev);
 u32 mt7996_mac_wtbl_lmac_addr(struct mt7996_dev *dev, u16 wcid, u8 dw);
 bool mt7996_mac_wtbl_update(struct mt7996_dev *dev, int idx, u32 mask);
 void mt7996_mac_reset_counters(struct mt7996_phy *phy);
@@ -486,9 +513,12 @@ void mt7996_mac_cca_stats_reset(struct mt7996_phy *phy);
 void mt7996_mac_enable_nf(struct mt7996_dev *dev, u8 band);
 void mt7996_mac_enable_rtscts(struct mt7996_dev *dev,
 			      struct ieee80211_vif *vif, bool enable);
+void mt7996_mac_set_fixed_rate_table(struct mt7996_dev *dev,
+				     u8 tbl_idx, u16 rate_idx);
 void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi,
-			   struct sk_buff *skb, struct mt76_wcid *wcid, int pid,
-			   struct ieee80211_key_conf *key, u32 changed);
+			   struct sk_buff *skb, struct mt76_wcid *wcid,
+			   struct ieee80211_key_conf *key, int pid,
+			   enum mt76_txq_id qid, u32 changed);
 void mt7996_mac_set_timing(struct mt7996_phy *phy);
 int mt7996_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif,
 		       struct ieee80211_sta *sta);
@@ -496,6 +526,7 @@ void mt7996_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif,
 			   struct ieee80211_sta *sta);
 void mt7996_mac_work(struct work_struct *work);
 void mt7996_mac_reset_work(struct work_struct *work);
+void mt7996_mac_dump_work(struct work_struct *work);
 void mt7996_mac_sta_rc_work(struct work_struct *work);
 void mt7996_mac_update_stats(struct mt7996_phy *phy);
 void mt7996_mac_twt_teardown_flow(struct mt7996_dev *dev,
@@ -512,7 +543,6 @@ void mt7996_tx_token_put(struct mt7996_dev *dev);
 void mt7996_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
 			 struct sk_buff *skb, u32 *info);
 bool mt7996_rx_check(struct mt76_dev *mdev, void *data, int len);
-void mt7996_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps);
 void mt7996_stats_work(struct work_struct *work);
 int mt76_dfs_start_rdd(struct mt7996_dev *dev, bool force);
 int mt7996_dfs_init_radar_detector(struct mt7996_phy *phy);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h
index 7a28cae34e34..d1d3d154195d 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h
@@ -228,6 +228,13 @@ enum base_rev {
 #define MT_WTBL_UPDATE_ADM_COUNT_CLEAR		BIT(14)
 #define MT_WTBL_UPDATE_BUSY			BIT(31)
 
+#define MT_WTBL_ITCR				MT_WTBLON_TOP(0x3b0)
+#define MT_WTBL_ITCR_WR				BIT(16)
+#define MT_WTBL_ITCR_EXEC			BIT(31)
+#define MT_WTBL_ITDR0				MT_WTBLON_TOP(0x3b8)
+#define MT_WTBL_ITDR1				MT_WTBLON_TOP(0x3bc)
+#define MT_WTBL_SPE_IDX_SEL			BIT(6)
+
 /* WTBL */
 #define MT_WTBL_BASE				0x820d8000
 #define MT_WTBL_LMAC_ID				GENMASK(14, 8)
@@ -317,6 +324,8 @@ enum base_rev {
 #define MT_WFDMA0_RX_INT_PCIE_SEL		MT_WFDMA0(0x154)
 #define MT_WFDMA0_RX_INT_SEL_RING3		BIT(3)
 
+#define MT_WFDMA0_MCU_HOST_INT_ENA		MT_WFDMA0(0x1f4)
+
 #define MT_WFDMA0_GLO_CFG			MT_WFDMA0(0x208)
 #define MT_WFDMA0_GLO_CFG_TX_DMA_EN		BIT(0)
 #define MT_WFDMA0_GLO_CFG_RX_DMA_EN		BIT(2)
@@ -444,6 +453,10 @@ enum base_rev {
 #define MT_MCU_CMD_NORMAL_STATE			BIT(5)
 #define MT_MCU_CMD_ERROR_MASK			GENMASK(5, 1)
 
+#define MT_MCU_CMD_WA_WDT			BIT(31)
+#define MT_MCU_CMD_WM_WDT			BIT(30)
+#define MT_MCU_CMD_WDT_MASK			GENMASK(31, 30)
+
 /* l1/l2 remap */
 #define MT_HIF_REMAP_L1				0x155024
 #define MT_HIF_REMAP_L1_MASK			GENMASK(31, 16)
@@ -468,9 +481,28 @@ enum base_rev {
 #define MT_INFRA_MCU_END			0x7c3fffff
 
 /* FW MODE SYNC */
-#define MT_SWDEF_MODE				0x9143c
+#define MT_FW_ASSERT_CNT			0x02208274
+#define MT_FW_DUMP_STATE			0x02209e90
+
+#define MT_SWDEF_BASE				0x00401400
+
+#define MT_SWDEF(ofs)				(MT_SWDEF_BASE + (ofs))
+#define MT_SWDEF_MODE				MT_SWDEF(0x3c)
 #define MT_SWDEF_NORMAL_MODE			0
 
+#define MT_SWDEF_SER_STATS			MT_SWDEF(0x040)
+#define MT_SWDEF_PLE_STATS			MT_SWDEF(0x044)
+#define MT_SWDEF_PLE1_STATS			MT_SWDEF(0x048)
+#define MT_SWDEF_PLE_AMSDU_STATS		MT_SWDEF(0x04c)
+#define MT_SWDEF_PSE_STATS			MT_SWDEF(0x050)
+#define MT_SWDEF_PSE1_STATS			MT_SWDEF(0x054)
+#define MT_SWDEF_LAMC_WISR6_BN0_STATS		MT_SWDEF(0x058)
+#define MT_SWDEF_LAMC_WISR6_BN1_STATS		MT_SWDEF(0x05c)
+#define MT_SWDEF_LAMC_WISR6_BN2_STATS		MT_SWDEF(0x060)
+#define MT_SWDEF_LAMC_WISR7_BN0_STATS		MT_SWDEF(0x064)
+#define MT_SWDEF_LAMC_WISR7_BN1_STATS		MT_SWDEF(0x068)
+#define MT_SWDEF_LAMC_WISR7_BN2_STATS		MT_SWDEF(0x06c)
+
 /* LED */
 #define MT_LED_TOP_BASE				0x18013000
 #define MT_LED_PHYS(_n)				(MT_LED_TOP_BASE + (_n))
@@ -486,6 +518,13 @@ enum base_rev {
 
 #define MT_LED_EN(_n)				MT_LED_PHYS(0x40 + ((_n) * 4))
 
+/* CONN DBG */
+#define MT_CONN_DBG_CTL_BASE			0x18023000
+#define MT_CONN_DBG_CTL(ofs)			(MT_CONN_DBG_CTL_BASE + (ofs))
+#define MT_CONN_DBG_CTL_OUT_SEL			MT_CONN_DBG_CTL(0x604)
+#define MT_CONN_DBG_CTL_PC_LOG_SEL		MT_CONN_DBG_CTL(0x60c)
+#define MT_CONN_DBG_CTL_PC_LOG			MT_CONN_DBG_CTL(0x610)
+
 #define MT_LED_GPIO_MUX2			0x70005058 /* GPIO 18 */
 #define MT_LED_GPIO_MUX3			0x7000505C /* GPIO 26 */
 #define MT_LED_GPIO_SEL_MASK			GENMASK(11, 8)
@@ -506,7 +545,7 @@ enum base_rev {
 #define MT_TOP_MISC_FW_STATE			GENMASK(2, 0)
 
 #define MT_HW_REV				0x70010204
-#define MT_WF_SUBSYS_RST			0x70002600
+#define MT_WF_SUBSYS_RST			0x70028600
 
 /* PCIE MAC */
 #define MT_PCIE_MAC_BASE			0x74030000
@@ -539,4 +578,12 @@ enum base_rev {
 #define MT_WF_PHYRX_CSD_BAND_RXTD12_IRPI_SW_CLR_ONLY	BIT(18)
 #define MT_WF_PHYRX_CSD_BAND_RXTD12_IRPI_SW_CLR		BIT(29)
 
+/* CONN MCU EXCP CON */
+#define MT_MCU_WM_EXCP_BASE			0x89050000
+#define MT_MCU_WM_EXCP(ofs)			(MT_MCU_WM_EXCP_BASE + (ofs))
+#define MT_MCU_WM_EXCP_PC_CTRL			MT_MCU_WM_EXCP(0x100)
+#define MT_MCU_WM_EXCP_PC_LOG			MT_MCU_WM_EXCP(0x104)
+#define MT_MCU_WM_EXCP_LR_CTRL			MT_MCU_WM_EXCP(0x200)
+#define MT_MCU_WM_EXCP_LR_LOG			MT_MCU_WM_EXCP(0x204)
+
 #endif
diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c
index 1f309d05380a..72b3ec715e47 100644
--- a/drivers/net/wireless/mediatek/mt76/tx.c
+++ b/drivers/net/wireless/mediatek/mt76/tx.c
@@ -77,7 +77,9 @@ mt76_tx_status_unlock(struct mt76_dev *dev, struct sk_buff_head *list)
 		}
 
 		hw = mt76_tx_status_get_hw(dev, skb);
+		spin_lock_bh(&dev->rx_lock);
 		ieee80211_tx_status_ext(hw, &status);
+		spin_unlock_bh(&dev->rx_lock);
 	}
 	rcu_read_unlock();
 }
@@ -263,7 +265,9 @@ void __mt76_tx_complete_skb(struct mt76_dev *dev, u16 wcid_idx, struct sk_buff *
 	if (cb->pktid < MT_PACKET_ID_FIRST) {
 		hw = mt76_tx_status_get_hw(dev, skb);
 		status.sta = wcid_to_sta(wcid);
+		spin_lock_bh(&dev->rx_lock);
 		ieee80211_tx_status_ext(hw, &status);
+		spin_unlock_bh(&dev->rx_lock);
 		goto out;
 	}
 
@@ -330,7 +334,7 @@ mt76_tx(struct mt76_phy *phy, struct ieee80211_sta *sta,
 	if ((dev->drv->drv_flags & MT_DRV_HW_MGMT_TXQ) &&
 	    !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) &&
 	    !ieee80211_is_data(hdr->frame_control) &&
-	    !ieee80211_is_bufferable_mmpdu(hdr->frame_control)) {
+	    !ieee80211_is_bufferable_mmpdu(skb)) {
 		qid = MT_TXQ_PSD;
 	}
 
diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.c b/drivers/net/wireless/quantenna/qtnfmac/commands.c
index b1b73478d89b..68ae9c7ea95a 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/commands.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/commands.c
@@ -1325,9 +1325,10 @@ static int qtnf_cmd_band_fill_iftype(const u8 *data,
 	struct ieee80211_sband_iftype_data *iftype_data;
 	const struct qlink_tlv_iftype_data *tlv =
 		(const struct qlink_tlv_iftype_data *)data;
-	size_t payload_len = tlv->n_iftype_data * sizeof(*tlv->iftype_data) +
-		sizeof(*tlv) -
-		sizeof(struct qlink_tlv_hdr);
+	size_t payload_len;
+
+	payload_len = struct_size(tlv, iftype_data, tlv->n_iftype_data);
+	payload_len = size_sub(payload_len, sizeof(struct qlink_tlv_hdr));
 
 	if (tlv->hdr.len != cpu_to_le16(payload_len)) {
 		pr_err("bad IFTYPE_DATA TLV len %u\n", tlv->hdr.len);
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
index 3a035afcf7f9..9a9cfd0ce402 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
@@ -1091,6 +1091,7 @@ static void rt2x00lib_remove_hw(struct rt2x00_dev *rt2x00dev)
 	}
 
 	kfree(rt2x00dev->spec.channels_info);
+	kfree(rt2x00dev->chan_survey);
 }
 
 static const struct ieee80211_tpt_blink rt2x00_tpt_blink[] = {
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/Kconfig b/drivers/net/wireless/realtek/rtl8xxxu/Kconfig
index 2eed20b0988c..82bcaf44a65f 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/Kconfig
+++ b/drivers/net/wireless/realtek/rtl8xxxu/Kconfig
@@ -11,7 +11,7 @@ config RTL8XXXU
 	  parts written to utilize the Linux mac80211 stack.
 	  The driver is known to work with a number of RTL8723AU,
 	  RL8188CU, RTL8188RU, RTL8191CU, RTL8192CU, RTL8723BU, RTL8192EU,
-	  RTL8188FU, and RTL8188EU devices.
+	  RTL8188FU, RTL8188EU, and RTL8710BU (aka RTL8188GU) devices.
 
 	  This driver is under development and has a limited feature
 	  set. In particular it does not yet support 40MHz channels
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/Makefile b/drivers/net/wireless/realtek/rtl8xxxu/Makefile
index 0cb58fb30228..1bf083c15dcd 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/Makefile
+++ b/drivers/net/wireless/realtek/rtl8xxxu/Makefile
@@ -3,4 +3,4 @@ obj-$(CONFIG_RTL8XXXU)	+= rtl8xxxu.o
 
 rtl8xxxu-y	:= rtl8xxxu_core.o rtl8xxxu_8192e.o rtl8xxxu_8723b.o \
 		   rtl8xxxu_8723a.o rtl8xxxu_8192c.o rtl8xxxu_8188f.o \
-		   rtl8xxxu_8188e.o
+		   rtl8xxxu_8188e.o rtl8xxxu_8710b.o
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
index c8cee4a24755..8eafbf1cee71 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
@@ -27,7 +27,7 @@
 #define RTL8XXXU_MAX_REG_POLL		500
 #define	USB_INTR_CONTENT_LENGTH		56
 
-#define RTL8XXXU_OUT_ENDPOINTS		4
+#define RTL8XXXU_OUT_ENDPOINTS		6
 
 #define REALTEK_USB_READ		0xc0
 #define REALTEK_USB_WRITE		0x40
@@ -103,7 +103,8 @@ enum rtl8xxxu_rtl_chip {
 	RTL8822B = 0x8822b,
 	RTL8703B = 0x8703b,
 	RTL8195A = 0x8195a,
-	RTL8188F = 0x8188f
+	RTL8188F = 0x8188f,
+	RTL8710B = 0x8710b,
 };
 
 enum rtl8xxxu_rx_type {
@@ -618,6 +619,265 @@ struct rtl8723au_phy_stats {
 #endif
 };
 
+struct jaguar2_phy_stats_type0 {
+	/* DW0 */
+	u8		page_num;
+	u8		pwdb;
+#ifdef __LITTLE_ENDIAN
+	u8		gain: 6;
+	u8		rsvd_0: 1;
+	u8		trsw: 1;
+#else
+	u8		trsw: 1;
+	u8		rsvd_0: 1;
+	u8		gain: 6;
+#endif
+	u8		rsvd_1;
+
+	/* DW1 */
+	u8		rsvd_2;
+#ifdef __LITTLE_ENDIAN
+	u8		rxsc: 4;
+	u8		agc_table: 4;
+#else
+	u8		agc_table: 4;
+	u8		rxsc: 4;
+#endif
+	u8		channel;
+	u8		band;
+
+	/* DW2 */
+	u16		length;
+#ifdef __LITTLE_ENDIAN
+	u8		antidx_a: 3;
+	u8		antidx_b: 3;
+	u8		rsvd_3: 2;
+	u8		antidx_c: 3;
+	u8		antidx_d: 3;
+	u8		rsvd_4:2;
+#else
+	u8		rsvd_3: 2;
+	u8		antidx_b: 3;
+	u8		antidx_a: 3;
+	u8		rsvd_4:2;
+	u8		antidx_d: 3;
+	u8		antidx_c: 3;
+#endif
+
+	/* DW3 */
+	u8		signal_quality;
+#ifdef __LITTLE_ENDIAN
+	u8		vga:5;
+	u8		lna_l:3;
+	u8		bb_power:6;
+	u8		rsvd_9:1;
+	u8		lna_h:1;
+#else
+	u8		lna_l:3;
+	u8		vga:5;
+	u8		lna_h:1;
+	u8		rsvd_9:1;
+	u8		bb_power:6;
+#endif
+	u8		rsvd_5;
+
+	/* DW4 */
+	u32		rsvd_6;
+
+	/* DW5 */
+	u32		rsvd_7;
+
+	/* DW6 */
+	u32		rsvd_8;
+} __packed;
+
+struct jaguar2_phy_stats_type1 {
+	/* DW0 and DW1 */
+	u8		page_num;
+	u8		pwdb[4];
+#ifdef __LITTLE_ENDIAN
+	u8		l_rxsc: 4;
+	u8		ht_rxsc: 4;
+#else
+	u8		ht_rxsc: 4;
+	u8		l_rxsc: 4;
+#endif
+	u8		channel;
+#ifdef __LITTLE_ENDIAN
+	u8		band: 2;
+	u8		rsvd_0: 1;
+	u8		hw_antsw_occu: 1;
+	u8		gnt_bt: 1;
+	u8		ldpc: 1;
+	u8		stbc: 1;
+	u8		beamformed: 1;
+#else
+	u8		beamformed: 1;
+	u8		stbc: 1;
+	u8		ldpc: 1;
+	u8		gnt_bt: 1;
+	u8		hw_antsw_occu: 1;
+	u8		rsvd_0: 1;
+	u8		band: 2;
+#endif
+
+	/* DW2 */
+	u16		lsig_length;
+#ifdef __LITTLE_ENDIAN
+	u8		antidx_a: 3;
+	u8		antidx_b: 3;
+	u8		rsvd_1: 2;
+	u8		antidx_c: 3;
+	u8		antidx_d: 3;
+	u8		rsvd_2: 2;
+#else
+	u8		rsvd_1: 2;
+	u8		antidx_b: 3;
+	u8		antidx_a: 3;
+	u8		rsvd_2: 2;
+	u8		antidx_d: 3;
+	u8		antidx_c: 3;
+#endif
+
+	/* DW3 */
+	u8		paid;
+#ifdef __LITTLE_ENDIAN
+	u8		paid_msb: 1;
+	u8		gid: 6;
+	u8		rsvd_3: 1;
+#else
+	u8		rsvd_3: 1;
+	u8		gid: 6;
+	u8		paid_msb: 1;
+#endif
+	u8		intf_pos;
+#ifdef __LITTLE_ENDIAN
+	u8		intf_pos_msb: 1;
+	u8		rsvd_4: 2;
+	u8		nb_intf_flag: 1;
+	u8		rf_mode: 2;
+	u8		rsvd_5: 2;
+#else
+	u8		rsvd_5: 2;
+	u8		rf_mode: 2;
+	u8		nb_intf_flag: 1;
+	u8		rsvd_4: 2;
+	u8		intf_pos_msb: 1;
+#endif
+
+	/* DW4 */
+	s8		rxevm[4];			/* s(8,1) */
+
+	/* DW5 */
+	s8		cfo_tail[4];			/* s(8,7) */
+
+	/* DW6 */
+	s8		rxsnr[4];			/* s(8,1) */
+} __packed;
+
+struct jaguar2_phy_stats_type2 {
+	/* DW0 ane DW1 */
+	u8		page_num;
+	u8		pwdb[4];
+#ifdef __LITTLE_ENDIAN
+	u8		l_rxsc: 4;
+	u8		ht_rxsc: 4;
+#else
+	u8		ht_rxsc: 4;
+	u8		l_rxsc: 4;
+#endif
+	u8		channel;
+#ifdef __LITTLE_ENDIAN
+	u8		band: 2;
+	u8		rsvd_0: 1;
+	u8		hw_antsw_occu: 1;
+	u8		gnt_bt: 1;
+	u8		ldpc: 1;
+	u8		stbc: 1;
+	u8		beamformed: 1;
+#else
+	u8		beamformed: 1;
+	u8		stbc: 1;
+	u8		ldpc: 1;
+	u8		gnt_bt: 1;
+	u8		hw_antsw_occu: 1;
+	u8		rsvd_0: 1;
+	u8		band: 2;
+#endif
+
+	/* DW2 */
+#ifdef __LITTLE_ENDIAN
+	u8		shift_l_map: 6;
+	u8		rsvd_1: 2;
+#else
+	u8		rsvd_1: 2;
+	u8		shift_l_map: 6;
+#endif
+	u8		cnt_pw2cca;
+#ifdef __LITTLE_ENDIAN
+	u8		agc_table_a: 4;
+	u8		agc_table_b: 4;
+	u8		agc_table_c: 4;
+	u8		agc_table_d: 4;
+#else
+	u8		agc_table_b: 4;
+	u8		agc_table_a: 4;
+	u8		agc_table_d: 4;
+	u8		agc_table_c: 4;
+#endif
+
+	/* DW3 ~ DW6*/
+	u8		cnt_cca2agc_rdy;
+#ifdef __LITTLE_ENDIAN
+	u8		gain_a: 6;
+	u8		rsvd_2: 1;
+	u8		trsw_a: 1;
+	u8		gain_b: 6;
+	u8		rsvd_3: 1;
+	u8		trsw_b: 1;
+	u8		gain_c: 6;
+	u8		rsvd_4: 1;
+	u8		trsw_c: 1;
+	u8		gain_d: 6;
+	u8		rsvd_5: 1;
+	u8		trsw_d: 1;
+	u8		aagc_step_a: 2;
+	u8		aagc_step_b: 2;
+	u8		aagc_step_c: 2;
+	u8		aagc_step_d: 2;
+#else
+	u8		trsw_a: 1;
+	u8		rsvd_2: 1;
+	u8		gain_a: 6;
+	u8		trsw_b: 1;
+	u8		rsvd_3: 1;
+	u8		gain_b: 6;
+	u8		trsw_c: 1;
+	u8		rsvd_4: 1;
+	u8		gain_c: 6;
+	u8		trsw_d: 1;
+	u8		rsvd_5: 1;
+	u8		gain_d: 6;
+	u8		aagc_step_d: 2;
+	u8		aagc_step_c: 2;
+	u8		aagc_step_b: 2;
+	u8		aagc_step_a: 2;
+#endif
+	u8		ht_aagc_gain[4];
+	u8		dagc_gain[4];
+#ifdef __LITTLE_ENDIAN
+	u8		counter: 6;
+	u8		rsvd_6: 2;
+	u8		syn_count: 5;
+	u8		rsvd_7:3;
+#else
+	u8		rsvd_6: 2;
+	u8		counter: 6;
+	u8		rsvd_7:3;
+	u8		syn_count: 5;
+#endif
+} __packed;
+
 /*
  * Regs to backup
  */
@@ -963,6 +1223,29 @@ struct rtl8188eu_efuse {
 	u8 res12[0xc3];
 } __packed;
 
+struct rtl8710bu_efuse {
+	__le16 rtl_id;
+	u8 res0[0x1e];
+	struct rtl8188fu_efuse_tx_power tx_power_index_A;	/* 0x20 */
+	u8 res1[0x9c];			/* 0x2c */
+	u8 channel_plan;		/* 0xc8 */
+	u8 xtal_k;			/* 0xc9 */
+	u8 thermal_meter;		/* 0xca */
+	u8 res2[0x4f];
+	u8 mac_addr[ETH_ALEN];		/* 0x11a */
+	u8 res3[0x11];
+	u8 rf_board_option;		/* 0x131 */
+	u8 res4[2];
+	u8 eeprom_version;		/* 0x134 */
+	u8 eeprom_customer_id;		/* 0x135 */
+	u8 res5[5];
+	u8 country_code;		/* 0x13b */
+	u8 res6[0x84];
+	u8 vid[2];			/* 0x1c0 */
+	u8 pid[2];			/* 0x1c2 */
+	u8 res7[0x3c];
+} __packed;
+
 struct rtl8xxxu_reg8val {
 	u16 reg;
 	u8 val;
@@ -1486,6 +1769,7 @@ struct rtl8xxxu_priv {
 	struct rtl8723au_idx ht20_tx_power_diff[RTL8723B_TX_COUNT];
 	struct rtl8723au_idx ht40_tx_power_diff[RTL8723B_TX_COUNT];
 	struct rtl8xxxu_power_base *power_base;
+	u8 package_type;
 	u32 chip_cut:4;
 	u32 rom_rev:4;
 	u32 is_multi_func:1;
@@ -1505,6 +1789,7 @@ struct rtl8xxxu_priv {
 	u32 ep_tx_low_queue:1;
 	u32 rx_buf_aggregation:1;
 	u32 cck_agc_report_type:1;
+	u32 cck_new_agc:1;
 	u8 default_crystal_cap;
 	unsigned int pipe_interrupt;
 	unsigned int pipe_in;
@@ -1522,6 +1807,8 @@ struct rtl8xxxu_priv {
 	int nr_out_eps;
 
 	struct mutex h2c_mutex;
+	/* Protect the indirect register accesses of RTL8710BU. */
+	struct mutex syson_indirect_access_mutex;
 
 	struct usb_anchor rx_anchor;
 	struct usb_anchor tx_anchor;
@@ -1542,6 +1829,7 @@ struct rtl8xxxu_priv {
 		struct rtl8192eu_efuse efuse8192eu;
 		struct rtl8188fu_efuse efuse8188fu;
 		struct rtl8188eu_efuse efuse8188eu;
+		struct rtl8710bu_efuse efuse8710bu;
 	} efuse_wifi;
 	u32 adda_backup[RTL8XXXU_ADDA_REGS];
 	u32 mac_backup[RTL8XXXU_MAC_REGS];
@@ -1586,6 +1874,7 @@ struct rtl8xxxu_tx_urb {
 
 struct rtl8xxxu_fileops {
 	int (*identify_chip) (struct rtl8xxxu_priv *priv);
+	int (*read_efuse) (struct rtl8xxxu_priv *priv);
 	int (*parse_efuse) (struct rtl8xxxu_priv *priv);
 	int (*load_firmware) (struct rtl8xxxu_priv *priv);
 	int (*power_on) (struct rtl8xxxu_priv *priv);
@@ -1599,6 +1888,11 @@ struct rtl8xxxu_fileops {
 	void (*phy_iq_calibrate) (struct rtl8xxxu_priv *priv);
 	void (*config_channel) (struct ieee80211_hw *hw);
 	int (*parse_rx_desc) (struct rtl8xxxu_priv *priv, struct sk_buff *skb);
+	void (*parse_phystats) (struct rtl8xxxu_priv *priv,
+				struct ieee80211_rx_status *rx_status,
+				struct rtl8723au_phy_stats *phy_stats,
+				u32 rxmcs, struct ieee80211_hdr *hdr,
+				bool crc_icv_err);
 	void (*init_aggregation) (struct rtl8xxxu_priv *priv);
 	void (*init_statistics) (struct rtl8xxxu_priv *priv);
 	void (*init_burst) (struct rtl8xxxu_priv *priv);
@@ -1618,7 +1912,7 @@ struct rtl8xxxu_fileops {
 			     bool short_preamble, bool ampdu_enable,
 			     u32 rts_rate);
 	void (*set_crystal_cap) (struct rtl8xxxu_priv *priv, u8 crystal_cap);
-	s8 (*cck_rssi) (struct rtl8xxxu_priv *priv, u8 cck_agc_rpt);
+	s8 (*cck_rssi) (struct rtl8xxxu_priv *priv, struct rtl8723au_phy_stats *phy_stats);
 	int (*led_classdev_brightness_set) (struct led_classdev *led_cdev,
 					    enum led_brightness brightness);
 	int writeN_block_size;
@@ -1629,6 +1923,11 @@ struct rtl8xxxu_fileops {
 	u8 has_tx_report:1;
 	u8 gen2_thermal_meter:1;
 	u8 needs_full_init:1;
+	u8 init_reg_rxfltmap:1;
+	u8 init_reg_pkt_life_time:1;
+	u8 init_reg_hmtfr:1;
+	u8 ampdu_max_time;
+	u8 ustime_tsf_edca;
 	u32 adda_1t_init;
 	u32 adda_1t_path_on;
 	u32 adda_2t_path_on_a;
@@ -1654,10 +1953,22 @@ u32 rtl8xxxu_read32(struct rtl8xxxu_priv *priv, u16 addr);
 int rtl8xxxu_write8(struct rtl8xxxu_priv *priv, u16 addr, u8 val);
 int rtl8xxxu_write16(struct rtl8xxxu_priv *priv, u16 addr, u16 val);
 int rtl8xxxu_write32(struct rtl8xxxu_priv *priv, u16 addr, u32 val);
+int rtl8xxxu_write8_set(struct rtl8xxxu_priv *priv, u16 addr, u8 bits);
+int rtl8xxxu_write8_clear(struct rtl8xxxu_priv *priv, u16 addr, u8 bits);
+int rtl8xxxu_write16_set(struct rtl8xxxu_priv *priv, u16 addr, u16 bits);
+int rtl8xxxu_write16_clear(struct rtl8xxxu_priv *priv, u16 addr, u16 bits);
+int rtl8xxxu_write32_set(struct rtl8xxxu_priv *priv, u16 addr, u32 bits);
+int rtl8xxxu_write32_clear(struct rtl8xxxu_priv *priv, u16 addr, u32 bits);
+int rtl8xxxu_write32_mask(struct rtl8xxxu_priv *priv, u16 addr,
+			  u32 mask, u32 val);
+
 u32 rtl8xxxu_read_rfreg(struct rtl8xxxu_priv *priv,
 			enum rtl8xxxu_rfpath path, u8 reg);
 int rtl8xxxu_write_rfreg(struct rtl8xxxu_priv *priv,
 			 enum rtl8xxxu_rfpath path, u8 reg, u32 data);
+int rtl8xxxu_write_rfreg_mask(struct rtl8xxxu_priv *priv,
+			      enum rtl8xxxu_rfpath path, u8 reg,
+			      u32 mask, u32 val);
 void rtl8xxxu_save_regs(struct rtl8xxxu_priv *priv, const u32 *regs,
 			u32 *backup, int count);
 void rtl8xxxu_restore_regs(struct rtl8xxxu_priv *priv, const u32 *regs,
@@ -1687,10 +1998,12 @@ void rtl8xxxu_identify_vendor_2bits(struct rtl8xxxu_priv *priv, u32 vendor);
 void rtl8xxxu_config_endpoints_sie(struct rtl8xxxu_priv *priv);
 int rtl8xxxu_config_endpoints_no_sie(struct rtl8xxxu_priv *priv);
 int rtl8xxxu_read_efuse8(struct rtl8xxxu_priv *priv, u16 offset, u8 *data);
+int rtl8xxxu_read_efuse(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_reset_8051(struct rtl8xxxu_priv *priv);
 int rtl8xxxu_auto_llt_table(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_gen2_prepare_calibrate(struct rtl8xxxu_priv *priv, u8 start);
 void rtl8723a_phy_lc_calibrate(struct rtl8xxxu_priv *priv);
+void rtl8188f_phy_lc_calibrate(struct rtl8xxxu_priv *priv);
 int rtl8xxxu_flush_fifo(struct rtl8xxxu_priv *priv);
 int rtl8xxxu_gen2_h2c_cmd(struct rtl8xxxu_priv *priv,
 			  struct h2c_cmd *h2c, int len);
@@ -1724,6 +2037,16 @@ void rtl8xxxu_gen2_disable_rf(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_init_burst(struct rtl8xxxu_priv *priv);
 int rtl8xxxu_parse_rxdesc16(struct rtl8xxxu_priv *priv, struct sk_buff *skb);
 int rtl8xxxu_parse_rxdesc24(struct rtl8xxxu_priv *priv, struct sk_buff *skb);
+void rtl8723au_rx_parse_phystats(struct rtl8xxxu_priv *priv,
+				 struct ieee80211_rx_status *rx_status,
+				 struct rtl8723au_phy_stats *phy_stats,
+				 u32 rxmcs, struct ieee80211_hdr *hdr,
+				 bool crc_icv_err);
+void jaguar2_rx_parse_phystats(struct rtl8xxxu_priv *priv,
+			       struct ieee80211_rx_status *rx_status,
+			       struct rtl8723au_phy_stats *phy_stats,
+			       u32 rxmcs, struct ieee80211_hdr *hdr,
+			       bool crc_icv_err);
 int rtl8xxxu_gen2_channel_to_group(int channel);
 bool rtl8xxxu_simularity_compare(struct rtl8xxxu_priv *priv,
 				 int result[][8], int c1, int c2);
@@ -1749,12 +2072,13 @@ void rtl8723bu_set_ps_tdma(struct rtl8xxxu_priv *priv,
 void rtl8723bu_phy_init_antenna_selection(struct rtl8xxxu_priv *priv);
 void rtl8723a_set_crystal_cap(struct rtl8xxxu_priv *priv, u8 crystal_cap);
 void rtl8188f_set_crystal_cap(struct rtl8xxxu_priv *priv, u8 crystal_cap);
-s8 rtl8723a_cck_rssi(struct rtl8xxxu_priv *priv, u8 cck_agc_rpt);
+s8 rtl8723a_cck_rssi(struct rtl8xxxu_priv *priv, struct rtl8723au_phy_stats *phy_stats);
 void rtl8xxxu_update_ra_report(struct rtl8xxxu_ra_report *rarpt,
 			       u8 rate, u8 sgi, u8 bw);
 void rtl8188e_ra_info_init_all(struct rtl8xxxu_ra_info *ra);
 void rtl8188e_handle_ra_tx_report2(struct rtl8xxxu_priv *priv, struct sk_buff *skb);
 
+extern struct rtl8xxxu_fileops rtl8710bu_fops;
 extern struct rtl8xxxu_fileops rtl8188fu_fops;
 extern struct rtl8xxxu_fileops rtl8188eu_fops;
 extern struct rtl8xxxu_fileops rtl8192cu_fops;
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8188e.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8188e.c
index a99ddb41cd24..8986783ae8fa 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8188e.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8188e.c
@@ -568,10 +568,6 @@ static int rtl8188eu_parse_efuse(struct rtl8xxxu_priv *priv)
 
 	priv->default_crystal_cap = efuse->xtal_k & 0x3f;
 
-	dev_info(&priv->udev->dev, "Vendor: %.7s\n", efuse->vendor_name);
-	dev_info(&priv->udev->dev, "Product: %.11s\n", efuse->device_name);
-	dev_info(&priv->udev->dev, "Serial: %.11s\n", efuse->serial);
-
 	return 0;
 }
 
@@ -1326,13 +1322,14 @@ static void rtl8188e_usb_quirks(struct rtl8xxxu_priv *priv)
 	rtl8xxxu_write8(priv, REG_EARLY_MODE_CONTROL_8188E + 3, 0x01);
 }
 
-static s8 rtl8188e_cck_rssi(struct rtl8xxxu_priv *priv, u8 cck_agc_rpt)
+static s8 rtl8188e_cck_rssi(struct rtl8xxxu_priv *priv, struct rtl8723au_phy_stats *phy_stats)
 {
 	/* only use lna 0/1/2/3/7 */
 	static const s8 lna_gain_table_0[8] = {17, -1, -13, -29, -32, -35, -38, -41};
 	/* only use lna 3/7 */
 	static const s8 lna_gain_table_1[8] = {29, 20, 12, 3, -6, -15, -24, -33};
 
+	u8 cck_agc_rpt = phy_stats->cck_agc_rpt_ofdm_cfosho_a;
 	s8 rx_pwr_all = 0x00;
 	u8 vga_idx, lna_idx;
 	s8 lna_gain = 0;
@@ -1699,6 +1696,12 @@ void rtl8188e_handle_ra_tx_report2(struct rtl8xxxu_priv *priv, struct sk_buff *s
 
 	dev_dbg(dev, "%s: len: %d items: %d\n", __func__, tx_rpt_len, items);
 
+	/* We only use macid 0, so only the first item is relevant.
+	 * AP mode will use more of them if it's ever implemented.
+	 */
+	if (!priv->vif || priv->vif->type == NL80211_IFTYPE_STATION)
+		items = 1;
+
 	for (macid = 0; macid < items; macid++) {
 		valid = false;
 
@@ -1741,12 +1744,6 @@ void rtl8188e_handle_ra_tx_report2(struct rtl8xxxu_priv *priv, struct sk_buff *s
 			min_rpt_time = ra->rpt_time;
 
 		rpt += TX_RPT2_ITEM_SIZE;
-
-		/*
-		 * We only use macid 0, so only the first item is relevant.
-		 * AP mode will use more of them if it's ever implemented.
-		 */
-		break;
 	}
 
 	if (min_rpt_time != ra->pre_min_rpt_time) {
@@ -1856,6 +1853,7 @@ struct rtl8xxxu_fileops rtl8188eu_fops = {
 	.load_firmware = rtl8188eu_load_firmware,
 	.power_on = rtl8188eu_power_on,
 	.power_off = rtl8188eu_power_off,
+	.read_efuse = rtl8xxxu_read_efuse,
 	.reset_8051 = rtl8188eu_reset_8051,
 	.llt_init = rtl8xxxu_init_llt_table,
 	.init_phy_bb = rtl8188eu_init_phy_bb,
@@ -1864,6 +1862,7 @@ struct rtl8xxxu_fileops rtl8188eu_fops = {
 	.phy_iq_calibrate = rtl8188eu_phy_iq_calibrate,
 	.config_channel = rtl8188eu_config_channel,
 	.parse_rx_desc = rtl8xxxu_parse_rxdesc16,
+	.parse_phystats = rtl8723au_rx_parse_phystats,
 	.init_aggregation = rtl8188eu_init_aggregation,
 	.enable_rf = rtl8188e_enable_rf,
 	.disable_rf = rtl8188e_disable_rf,
@@ -1880,6 +1879,7 @@ struct rtl8xxxu_fileops rtl8188eu_fops = {
 	.rx_desc_size = sizeof(struct rtl8xxxu_rxdesc16),
 	.tx_desc_size = sizeof(struct rtl8xxxu_txdesc32),
 	.has_tx_report = 1,
+	.init_reg_pkt_life_time = 1,
 	.gen2_thermal_meter = 1,
 	.adda_1t_init = 0x0b1b25a0,
 	.adda_1t_path_on = 0x0bdb25a0,
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8188f.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8188f.c
index af6e2c8a5025..dbdfd7787465 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8188f.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8188f.c
@@ -734,9 +734,6 @@ static int rtl8188fu_parse_efuse(struct rtl8xxxu_priv *priv)
 
 	priv->default_crystal_cap = efuse->xtal_k & 0x3f;
 
-	dev_info(&priv->udev->dev, "Vendor: %.7s\n", efuse->vendor_name);
-	dev_info(&priv->udev->dev, "Product: %.7s\n", efuse->device_name);
-
 	return 0;
 }
 
@@ -791,7 +788,7 @@ static int rtl8188fu_init_phy_rf(struct rtl8xxxu_priv *priv)
 	return ret;
 }
 
-static void rtl8188f_phy_lc_calibrate(struct rtl8xxxu_priv *priv)
+void rtl8188f_phy_lc_calibrate(struct rtl8xxxu_priv *priv)
 {
 	u32 val32;
 	u32 rf_amode, lstf;
@@ -1677,8 +1674,9 @@ void rtl8188f_set_crystal_cap(struct rtl8xxxu_priv *priv, u8 crystal_cap)
 	cfo->crystal_cap = crystal_cap;
 }
 
-static s8 rtl8188f_cck_rssi(struct rtl8xxxu_priv *priv, u8 cck_agc_rpt)
+static s8 rtl8188f_cck_rssi(struct rtl8xxxu_priv *priv, struct rtl8723au_phy_stats *phy_stats)
 {
+	u8 cck_agc_rpt = phy_stats->cck_agc_rpt_ofdm_cfosho_a;
 	s8 rx_pwr_all = 0x00;
 	u8 vga_idx, lna_idx;
 
@@ -1714,6 +1712,7 @@ struct rtl8xxxu_fileops rtl8188fu_fops = {
 	.load_firmware = rtl8188fu_load_firmware,
 	.power_on = rtl8188fu_power_on,
 	.power_off = rtl8188fu_power_off,
+	.read_efuse = rtl8xxxu_read_efuse,
 	.reset_8051 = rtl8xxxu_reset_8051,
 	.llt_init = rtl8xxxu_auto_llt_table,
 	.init_phy_bb = rtl8188fu_init_phy_bb,
@@ -1723,6 +1722,7 @@ struct rtl8xxxu_fileops rtl8188fu_fops = {
 	.phy_iq_calibrate = rtl8188fu_phy_iq_calibrate,
 	.config_channel = rtl8188fu_config_channel,
 	.parse_rx_desc = rtl8xxxu_parse_rxdesc24,
+	.parse_phystats = rtl8723au_rx_parse_phystats,
 	.init_aggregation = rtl8188fu_init_aggregation,
 	.init_statistics = rtl8188fu_init_statistics,
 	.init_burst = rtl8xxxu_init_burst,
@@ -1743,6 +1743,11 @@ struct rtl8xxxu_fileops rtl8188fu_fops = {
 	.has_tx_report = 1,
 	.gen2_thermal_meter = 1,
 	.needs_full_init = 1,
+	.init_reg_rxfltmap = 1,
+	.init_reg_pkt_life_time = 1,
+	.init_reg_hmtfr = 1,
+	.ampdu_max_time = 0x70,
+	.ustime_tsf_edca = 0x28,
 	.adda_1t_init = 0x03c00014,
 	.adda_1t_path_on = 0x03c00014,
 	.trxff_boundary = 0x3f7f,
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192c.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192c.c
index e61d65c3579b..b30a9a513cb8 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192c.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192c.c
@@ -441,11 +441,6 @@ static int rtl8192cu_parse_efuse(struct rtl8xxxu_priv *priv)
 	       efuse->ht20_max_power_offset,
 	       sizeof(efuse->ht20_max_power_offset));
 
-	dev_info(&priv->udev->dev, "Vendor: %.7s\n",
-		 efuse->vendor_name);
-	dev_info(&priv->udev->dev, "Product: %.20s\n",
-		 efuse->device_name);
-
 	priv->power_base = &rtl8192c_power_base;
 
 	if (efuse->rf_regulatory & 0x20) {
@@ -594,6 +589,7 @@ struct rtl8xxxu_fileops rtl8192cu_fops = {
 	.load_firmware = rtl8192cu_load_firmware,
 	.power_on = rtl8192cu_power_on,
 	.power_off = rtl8xxxu_power_off,
+	.read_efuse = rtl8xxxu_read_efuse,
 	.reset_8051 = rtl8xxxu_reset_8051,
 	.llt_init = rtl8xxxu_init_llt_table,
 	.init_phy_bb = rtl8xxxu_gen1_init_phy_bb,
@@ -602,6 +598,7 @@ struct rtl8xxxu_fileops rtl8192cu_fops = {
 	.phy_iq_calibrate = rtl8xxxu_gen1_phy_iq_calibrate,
 	.config_channel = rtl8xxxu_gen1_config_channel,
 	.parse_rx_desc = rtl8xxxu_parse_rxdesc16,
+	.parse_phystats = rtl8723au_rx_parse_phystats,
 	.init_aggregation = rtl8xxxu_gen1_init_aggregation,
 	.enable_rf = rtl8xxxu_gen1_enable_rf,
 	.disable_rf = rtl8xxxu_gen1_disable_rf,
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c
index 5cfc00237f42..fcc2926ea938 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c
@@ -601,43 +601,9 @@ rtl8192e_set_tx_power(struct rtl8xxxu_priv *priv, int channel, bool ht40)
 	}
 }
 
-static void rtl8192eu_log_next_device_info(struct rtl8xxxu_priv *priv,
-					   char *record_name,
-					   char *device_info,
-					   unsigned int *record_offset)
-{
-	char *record = device_info + *record_offset;
-
-	/* A record is [ total length | 0x03 | value ] */
-	unsigned char l = record[0];
-
-	/*
-	 * The whole device info section seems to be 80 characters, make sure
-	 * we don't read further.
-	 */
-	if (*record_offset + l > 80) {
-		dev_warn(&priv->udev->dev,
-			 "invalid record length %d while parsing \"%s\" at offset %u.\n",
-			 l, record_name, *record_offset);
-		return;
-	}
-
-	if (l >= 2) {
-		char value[80];
-
-		memcpy(value, &record[2], l - 2);
-		value[l - 2] = '\0';
-		dev_info(&priv->udev->dev, "%s: %s\n", record_name, value);
-		*record_offset = *record_offset + l;
-	} else {
-		dev_info(&priv->udev->dev, "%s not available.\n", record_name);
-	}
-}
-
 static int rtl8192eu_parse_efuse(struct rtl8xxxu_priv *priv)
 {
 	struct rtl8192eu_efuse *efuse = &priv->efuse_wifi.efuse8192eu;
-	unsigned int record_offset;
 	int i;
 
 	if (efuse->rtl_id != cpu_to_le16(0x8129))
@@ -684,26 +650,6 @@ static int rtl8192eu_parse_efuse(struct rtl8xxxu_priv *priv)
 
 	priv->default_crystal_cap = priv->efuse_wifi.efuse8192eu.xtal_k & 0x3f;
 
-	/*
-	 * device_info section seems to be laid out as records
-	 * [ total length | 0x03 | value ] so:
-	 * - vendor length + 2
-	 * - 0x03
-	 * - vendor string (not null terminated)
-	 * - product length + 2
-	 * - 0x03
-	 * - product string (not null terminated)
-	 * Then there is one or 2 0x00 on all the 4 devices I own or found
-	 * dumped online.
-	 * As previous version of the code handled an optional serial
-	 * string, I now assume there may be a third record if the
-	 * length is not 0.
-	 */
-	record_offset = 0;
-	rtl8192eu_log_next_device_info(priv, "Vendor", efuse->device_info, &record_offset);
-	rtl8192eu_log_next_device_info(priv, "Product", efuse->device_info, &record_offset);
-	rtl8192eu_log_next_device_info(priv, "Serial", efuse->device_info, &record_offset);
-
 	return 0;
 }
 
@@ -1742,11 +1688,12 @@ static void rtl8192e_enable_rf(struct rtl8xxxu_priv *priv)
 	rtl8xxxu_write8(priv, REG_TXPAUSE, 0x00);
 }
 
-static s8 rtl8192e_cck_rssi(struct rtl8xxxu_priv *priv, u8 cck_agc_rpt)
+static s8 rtl8192e_cck_rssi(struct rtl8xxxu_priv *priv, struct rtl8723au_phy_stats *phy_stats)
 {
 	static const s8 lna_gain_table_0[8] = {15, 9, -10, -21, -23, -27, -43, -44};
 	static const s8 lna_gain_table_1[8] = {24, 18, 13, -4, -11, -18, -31, -36};
 
+	u8 cck_agc_rpt = phy_stats->cck_agc_rpt_ofdm_cfosho_a;
 	s8 rx_pwr_all = 0x00;
 	u8 vga_idx, lna_idx;
 	s8 lna_gain = 0;
@@ -1793,6 +1740,7 @@ struct rtl8xxxu_fileops rtl8192eu_fops = {
 	.load_firmware = rtl8192eu_load_firmware,
 	.power_on = rtl8192eu_power_on,
 	.power_off = rtl8192eu_power_off,
+	.read_efuse = rtl8xxxu_read_efuse,
 	.reset_8051 = rtl8xxxu_reset_8051,
 	.llt_init = rtl8xxxu_auto_llt_table,
 	.init_phy_bb = rtl8192eu_init_phy_bb,
@@ -1801,6 +1749,7 @@ struct rtl8xxxu_fileops rtl8192eu_fops = {
 	.phy_iq_calibrate = rtl8192eu_phy_iq_calibrate,
 	.config_channel = rtl8xxxu_gen2_config_channel,
 	.parse_rx_desc = rtl8xxxu_parse_rxdesc24,
+	.parse_phystats = rtl8723au_rx_parse_phystats,
 	.enable_rf = rtl8192e_enable_rf,
 	.disable_rf = rtl8xxxu_gen2_disable_rf,
 	.usb_quirks = rtl8xxxu_gen2_usb_quirks,
@@ -1817,6 +1766,7 @@ struct rtl8xxxu_fileops rtl8192eu_fops = {
 	.rx_desc_size = sizeof(struct rtl8xxxu_rxdesc24),
 	.has_s0s1 = 0,
 	.gen2_thermal_meter = 1,
+	.needs_full_init = 1,
 	.adda_1t_init = 0x0fc01616,
 	.adda_1t_path_on = 0x0fc01616,
 	.adda_2t_path_on_a = 0x0fc01616,
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8710b.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8710b.c
new file mode 100644
index 000000000000..22d4704dd31e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8710b.c
@@ -0,0 +1,1887 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * RTL8XXXU mac80211 USB driver - 8710bu aka 8188gu specific subdriver
+ *
+ * Copyright (c) 2023 Bitterblue Smith <rtl8821cerfe2@gmail.com>
+ *
+ * Portions copied from existing rtl8xxxu code:
+ * Copyright (c) 2014 - 2017 Jes Sorensen <Jes.Sorensen@gmail.com>
+ *
+ * Portions, notably calibration code:
+ * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/list.h>
+#include <linux/usb.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/wireless.h>
+#include <linux/firmware.h>
+#include <linux/moduleparam.h>
+#include <net/mac80211.h>
+#include "rtl8xxxu.h"
+#include "rtl8xxxu_regs.h"
+
+static const struct rtl8xxxu_reg8val rtl8710b_mac_init_table[] = {
+	{0x421, 0x0F}, {0x428, 0x0A}, {0x429, 0x10}, {0x430, 0x00},
+	{0x431, 0x00}, {0x432, 0x00}, {0x433, 0x01}, {0x434, 0x04},
+	{0x435, 0x05}, {0x436, 0x07}, {0x437, 0x08}, {0x43C, 0x04},
+	{0x43D, 0x05}, {0x43E, 0x07}, {0x43F, 0x08}, {0x440, 0x5D},
+	{0x441, 0x01}, {0x442, 0x00}, {0x444, 0x10}, {0x445, 0x00},
+	{0x446, 0x00}, {0x447, 0x00}, {0x448, 0x00}, {0x449, 0xF0},
+	{0x44A, 0x0F}, {0x44B, 0x3E}, {0x44C, 0x10}, {0x44D, 0x00},
+	{0x44E, 0x00}, {0x44F, 0x00}, {0x450, 0x00}, {0x451, 0xF0},
+	{0x452, 0x0F}, {0x453, 0x00}, {0x456, 0x5E}, {0x460, 0x66},
+	{0x461, 0x66}, {0x4C8, 0xFF}, {0x4C9, 0x08}, {0x4CC, 0xFF},
+	{0x4CD, 0xFF}, {0x4CE, 0x01}, {0x500, 0x26}, {0x501, 0xA2},
+	{0x502, 0x2F}, {0x503, 0x00}, {0x504, 0x28}, {0x505, 0xA3},
+	{0x506, 0x5E}, {0x507, 0x00}, {0x508, 0x2B}, {0x509, 0xA4},
+	{0x50A, 0x5E}, {0x50B, 0x00}, {0x50C, 0x4F}, {0x50D, 0xA4},
+	{0x50E, 0x00}, {0x50F, 0x00}, {0x512, 0x1C}, {0x514, 0x0A},
+	{0x516, 0x0A}, {0x525, 0x4F}, {0x550, 0x10}, {0x551, 0x10},
+	{0x559, 0x02}, {0x55C, 0x28}, {0x55D, 0xFF}, {0x605, 0x30},
+	{0x608, 0x0E}, {0x609, 0x2A}, {0x620, 0xFF}, {0x621, 0xFF},
+	{0x622, 0xFF}, {0x623, 0xFF}, {0x624, 0xFF}, {0x625, 0xFF},
+	{0x626, 0xFF}, {0x627, 0xFF}, {0x638, 0x28}, {0x63C, 0x0A},
+	{0x63D, 0x0A}, {0x63E, 0x0C}, {0x63F, 0x0C}, {0x640, 0x40},
+	{0x642, 0x40}, {0x643, 0x00}, {0x652, 0xC8}, {0x66A, 0xB0},
+	{0x66E, 0x05}, {0x700, 0x21}, {0x701, 0x43}, {0x702, 0x65},
+	{0x703, 0x87}, {0x708, 0x21}, {0x709, 0x43}, {0x70A, 0x65},
+	{0x70B, 0x87},
+	{0xffff, 0xff},
+};
+
+/* If updating the phy init tables, also update rtl8710b_revise_cck_tx_psf(). */
+static const struct rtl8xxxu_reg32val rtl8710bu_qfn48m_u_phy_init_table[] = {
+	{0x800, 0x80045700}, {0x804, 0x00000001},
+	{0x808, 0x00FC8000}, {0x80C, 0x0000000A},
+	{0x810, 0x10001331}, {0x814, 0x020C3D10},
+	{0x818, 0x00200385}, {0x81C, 0x00000000},
+	{0x820, 0x01000100}, {0x824, 0x00390204},
+	{0x828, 0x00000000}, {0x82C, 0x00000000},
+	{0x830, 0x00000000}, {0x834, 0x00000000},
+	{0x838, 0x00000000}, {0x83C, 0x00000000},
+	{0x840, 0x00010000}, {0x844, 0x00000000},
+	{0x848, 0x00000000}, {0x84C, 0x00000000},
+	{0x850, 0x00030000}, {0x854, 0x00000000},
+	{0x858, 0x7E1A569A}, {0x85C, 0x569A569A},
+	{0x860, 0x00000130}, {0x864, 0x20000000},
+	{0x868, 0x00000000}, {0x86C, 0x27272700},
+	{0x870, 0x00050000}, {0x874, 0x25005000},
+	{0x878, 0x00000808}, {0x87C, 0x004F0201},
+	{0x880, 0xB0000B1E}, {0x884, 0x00000007},
+	{0x888, 0x00000000}, {0x88C, 0xCCC400C0},
+	{0x890, 0x00000800}, {0x894, 0xFFFFFFFE},
+	{0x898, 0x40302010}, {0x89C, 0x00706050},
+	{0x900, 0x00000000}, {0x904, 0x00000023},
+	{0x908, 0x00000000}, {0x90C, 0x81121111},
+	{0x910, 0x00000402}, {0x914, 0x00000201},
+	{0x920, 0x18C6318C}, {0x924, 0x0000018C},
+	{0x948, 0x99000000}, {0x94C, 0x00000010},
+	{0x950, 0x00003000}, {0x954, 0x5A880000},
+	{0x958, 0x4BC6D87A}, {0x95C, 0x04EB9B79},
+	{0x96C, 0x00000003}, {0x970, 0x00000000},
+	{0x974, 0x00000000}, {0x978, 0x00000000},
+	{0x97C, 0x13000000}, {0x980, 0x00000000},
+	{0xA00, 0x00D046C8}, {0xA04, 0x80FF800C},
+	{0xA08, 0x84838300}, {0xA0C, 0x2E20100F},
+	{0xA10, 0x9500BB78}, {0xA14, 0x1114D028},
+	{0xA18, 0x00881117}, {0xA1C, 0x89140F00},
+	{0xA20, 0xE82C0001}, {0xA24, 0x64B80C1C},
+	{0xA28, 0x00008810}, {0xA2C, 0x00D30000},
+	{0xA70, 0x101FBF00}, {0xA74, 0x00000007},
+	{0xA78, 0x00000900}, {0xA7C, 0x225B0606},
+	{0xA80, 0x218075B1}, {0xA84, 0x00200000},
+	{0xA88, 0x040C0000}, {0xA8C, 0x12345678},
+	{0xA90, 0xABCDEF00}, {0xA94, 0x001B1B89},
+	{0xA98, 0x00000000}, {0xA9C, 0x80020000},
+	{0xAA0, 0x00000000}, {0xAA4, 0x0000000C},
+	{0xAA8, 0xCA110058}, {0xAAC, 0x01235667},
+	{0xAB0, 0x00000000}, {0xAB4, 0x20201402},
+	{0xB2C, 0x00000000}, {0xC00, 0x48071D40},
+	{0xC04, 0x03A05611}, {0xC08, 0x000000E4},
+	{0xC0C, 0x6C6C6C6C}, {0xC10, 0x18800000},
+	{0xC14, 0x40000100}, {0xC18, 0x08800000},
+	{0xC1C, 0x40000100}, {0xC20, 0x00000000},
+	{0xC24, 0x00000000}, {0xC28, 0x00000000},
+	{0xC2C, 0x00000000}, {0xC30, 0x69E9AC4A},
+	{0xC34, 0x31000040}, {0xC38, 0x21688080},
+	{0xC3C, 0x0000170C}, {0xC40, 0x1F78403F},
+	{0xC44, 0x00010036}, {0xC48, 0xEC020107},
+	{0xC4C, 0x007F037F}, {0xC50, 0x69553420},
+	{0xC54, 0x43BC0094}, {0xC58, 0x00013169},
+	{0xC5C, 0x00250492}, {0xC60, 0x00280A00},
+	{0xC64, 0x7112848B}, {0xC68, 0x47C074FF},
+	{0xC6C, 0x00000036}, {0xC70, 0x2C7F000D},
+	{0xC74, 0x020600DB}, {0xC78, 0x0000001F},
+	{0xC7C, 0x00B91612}, {0xC80, 0x390000E4},
+	{0xC84, 0x11F60000}, {0xC88, 0x1051B75F},
+	{0xC8C, 0x20200109}, {0xC90, 0x00091521},
+	{0xC94, 0x00000000}, {0xC98, 0x00121820},
+	{0xC9C, 0x00007F7F}, {0xCA0, 0x00011000},
+	{0xCA4, 0x800000A0}, {0xCA8, 0x84E6C606},
+	{0xCAC, 0x00000060}, {0xCB0, 0x00000000},
+	{0xCB4, 0x00000000}, {0xCB8, 0x00000000},
+	{0xCBC, 0x28000000}, {0xCC0, 0x1051B75F},
+	{0xCC4, 0x00000109}, {0xCC8, 0x000442D6},
+	{0xCCC, 0x00000000}, {0xCD0, 0x000001C8},
+	{0xCD4, 0x001C8000}, {0xCD8, 0x00000100},
+	{0xCDC, 0x40100000}, {0xCE0, 0x00222220},
+	{0xCE4, 0x10000000}, {0xCE8, 0x37644302},
+	{0xCEC, 0x2F97D40C}, {0xD00, 0x04030740},
+	{0xD04, 0x40020401}, {0xD08, 0x0000907F},
+	{0xD0C, 0x20010201}, {0xD10, 0xA0633333},
+	{0xD14, 0x3333BC53}, {0xD18, 0x7A8F5B6F},
+	{0xD2C, 0xCB979975}, {0xD30, 0x00000000},
+	{0xD34, 0x40608000}, {0xD38, 0x88000000},
+	{0xD3C, 0xC0127353}, {0xD40, 0x00000000},
+	{0xD44, 0x00000000}, {0xD48, 0x00000000},
+	{0xD4C, 0x00000000}, {0xD50, 0x00006528},
+	{0xD54, 0x00000000}, {0xD58, 0x00000282},
+	{0xD5C, 0x30032064}, {0xD60, 0x4653DE68},
+	{0xD64, 0x04518A3C}, {0xD68, 0x00002101},
+	{0xE00, 0x2D2D2D2D}, {0xE04, 0x2D2D2D2D},
+	{0xE08, 0x0390272D}, {0xE10, 0x2D2D2D2D},
+	{0xE14, 0x2D2D2D2D}, {0xE18, 0x2D2D2D2D},
+	{0xE1C, 0x2D2D2D2D}, {0xE28, 0x00000000},
+	{0xE30, 0x1000DC1F}, {0xE34, 0x10008C1F},
+	{0xE38, 0x02140102}, {0xE3C, 0x681604C2},
+	{0xE40, 0x01007C00}, {0xE44, 0x01004800},
+	{0xE48, 0xFB000000}, {0xE4C, 0x000028D1},
+	{0xE50, 0x1000DC1F}, {0xE54, 0x10008C1F},
+	{0xE58, 0x02140102}, {0xE5C, 0x28160D05},
+	{0xE60, 0x0000C008}, {0xE68, 0x001B25A4},
+	{0xE64, 0x281600A0}, {0xE6C, 0x01C00010},
+	{0xE70, 0x01C00010}, {0xE74, 0x02000010},
+	{0xE78, 0x02000010}, {0xE7C, 0x02000010},
+	{0xE80, 0x02000010}, {0xE84, 0x01C00010},
+	{0xE88, 0x02000010}, {0xE8C, 0x01C00010},
+	{0xED0, 0x01C00010}, {0xED4, 0x01C00010},
+	{0xED8, 0x01C00010}, {0xEDC, 0x00000010},
+	{0xEE0, 0x00000010}, {0xEEC, 0x03C00010},
+	{0xF14, 0x00000003}, {0xF00, 0x00100300},
+	{0xF08, 0x0000800B}, {0xF0C, 0x0000F007},
+	{0xF10, 0x0000A487}, {0xF1C, 0x80000064},
+	{0xF38, 0x00030155}, {0xF3C, 0x0000003A},
+	{0xF4C, 0x13000000}, {0xF50, 0x00000000},
+	{0xF18, 0x00000000},
+	{0xffff, 0xffffffff},
+};
+
+/* If updating the phy init tables, also update rtl8710b_revise_cck_tx_psf(). */
+static const struct rtl8xxxu_reg32val rtl8710bu_qfn48m_s_phy_init_table[] = {
+	{0x800, 0x80045700}, {0x804, 0x00000001},
+	{0x808, 0x00FC8000}, {0x80C, 0x0000000A},
+	{0x810, 0x10001331}, {0x814, 0x020C3D10},
+	{0x818, 0x00200385}, {0x81C, 0x00000000},
+	{0x820, 0x01000100}, {0x824, 0x00390204},
+	{0x828, 0x00000000}, {0x82C, 0x00000000},
+	{0x830, 0x00000000}, {0x834, 0x00000000},
+	{0x838, 0x00000000}, {0x83C, 0x00000000},
+	{0x840, 0x00010000}, {0x844, 0x00000000},
+	{0x848, 0x00000000}, {0x84C, 0x00000000},
+	{0x850, 0x00030000}, {0x854, 0x00000000},
+	{0x858, 0x7E1A569A}, {0x85C, 0x569A569A},
+	{0x860, 0x00000130}, {0x864, 0x20000000},
+	{0x868, 0x00000000}, {0x86C, 0x27272700},
+	{0x870, 0x00050000}, {0x874, 0x25005000},
+	{0x878, 0x00000808}, {0x87C, 0x004F0201},
+	{0x880, 0xB0000B1E}, {0x884, 0x00000007},
+	{0x888, 0x00000000}, {0x88C, 0xCCC400C0},
+	{0x890, 0x00000800}, {0x894, 0xFFFFFFFE},
+	{0x898, 0x40302010}, {0x89C, 0x00706050},
+	{0x900, 0x00000000}, {0x904, 0x00000023},
+	{0x908, 0x00000000}, {0x90C, 0x81121111},
+	{0x910, 0x00000402}, {0x914, 0x00000201},
+	{0x920, 0x18C6318C}, {0x924, 0x0000018C},
+	{0x948, 0x99000000}, {0x94C, 0x00000010},
+	{0x950, 0x00003000}, {0x954, 0x5A880000},
+	{0x958, 0x4BC6D87A}, {0x95C, 0x04EB9B79},
+	{0x96C, 0x00000003}, {0x970, 0x00000000},
+	{0x974, 0x00000000}, {0x978, 0x00000000},
+	{0x97C, 0x13000000}, {0x980, 0x00000000},
+	{0xA00, 0x00D046C8}, {0xA04, 0x80FF800C},
+	{0xA08, 0x84838300}, {0xA0C, 0x2A20100F},
+	{0xA10, 0x9500BB78}, {0xA14, 0x1114D028},
+	{0xA18, 0x00881117}, {0xA1C, 0x89140F00},
+	{0xA20, 0xE82C0001}, {0xA24, 0x64B80C1C},
+	{0xA28, 0x00008810}, {0xA2C, 0x00D30000},
+	{0xA70, 0x101FBF00}, {0xA74, 0x00000007},
+	{0xA78, 0x00000900}, {0xA7C, 0x225B0606},
+	{0xA80, 0x218075B1}, {0xA84, 0x00200000},
+	{0xA88, 0x040C0000}, {0xA8C, 0x12345678},
+	{0xA90, 0xABCDEF00}, {0xA94, 0x001B1B89},
+	{0xA98, 0x00000000}, {0xA9C, 0x80020000},
+	{0xAA0, 0x00000000}, {0xAA4, 0x0000000C},
+	{0xAA8, 0xCA110058}, {0xAAC, 0x01235667},
+	{0xAB0, 0x00000000}, {0xAB4, 0x20201402},
+	{0xB2C, 0x00000000}, {0xC00, 0x48071D40},
+	{0xC04, 0x03A05611}, {0xC08, 0x000000E4},
+	{0xC0C, 0x6C6C6C6C}, {0xC10, 0x18800000},
+	{0xC14, 0x40000100}, {0xC18, 0x08800000},
+	{0xC1C, 0x40000100}, {0xC20, 0x00000000},
+	{0xC24, 0x00000000}, {0xC28, 0x00000000},
+	{0xC2C, 0x00000000}, {0xC30, 0x69E9AC4A},
+	{0xC34, 0x31000040}, {0xC38, 0x21688080},
+	{0xC3C, 0x0000170C}, {0xC40, 0x1F78403F},
+	{0xC44, 0x00010036}, {0xC48, 0xEC020107},
+	{0xC4C, 0x007F037F}, {0xC50, 0x69553420},
+	{0xC54, 0x43BC0094}, {0xC58, 0x00013169},
+	{0xC5C, 0x00250492}, {0xC60, 0x00280A00},
+	{0xC64, 0x7112848B}, {0xC68, 0x47C074FF},
+	{0xC6C, 0x00000036}, {0xC70, 0x2C7F000D},
+	{0xC74, 0x020600DB}, {0xC78, 0x0000001F},
+	{0xC7C, 0x00B91612}, {0xC80, 0x390000E4},
+	{0xC84, 0x11F60000}, {0xC88, 0x1051B75F},
+	{0xC8C, 0x20200109}, {0xC90, 0x00091521},
+	{0xC94, 0x00000000}, {0xC98, 0x00121820},
+	{0xC9C, 0x00007F7F}, {0xCA0, 0x00011000},
+	{0xCA4, 0x800000A0}, {0xCA8, 0x84E6C606},
+	{0xCAC, 0x00000060}, {0xCB0, 0x00000000},
+	{0xCB4, 0x00000000}, {0xCB8, 0x00000000},
+	{0xCBC, 0x28000000}, {0xCC0, 0x1051B75F},
+	{0xCC4, 0x00000109}, {0xCC8, 0x000442D6},
+	{0xCCC, 0x00000000}, {0xCD0, 0x000001C8},
+	{0xCD4, 0x001C8000}, {0xCD8, 0x00000100},
+	{0xCDC, 0x40100000}, {0xCE0, 0x00222220},
+	{0xCE4, 0x10000000}, {0xCE8, 0x37644302},
+	{0xCEC, 0x2F97D40C}, {0xD00, 0x04030740},
+	{0xD04, 0x40020401}, {0xD08, 0x0000907F},
+	{0xD0C, 0x20010201}, {0xD10, 0xA0633333},
+	{0xD14, 0x3333BC53}, {0xD18, 0x7A8F5B6F},
+	{0xD2C, 0xCB979975}, {0xD30, 0x00000000},
+	{0xD34, 0x40608000}, {0xD38, 0x88000000},
+	{0xD3C, 0xC0127353}, {0xD40, 0x00000000},
+	{0xD44, 0x00000000}, {0xD48, 0x00000000},
+	{0xD4C, 0x00000000}, {0xD50, 0x00006528},
+	{0xD54, 0x00000000}, {0xD58, 0x00000282},
+	{0xD5C, 0x30032064}, {0xD60, 0x4653DE68},
+	{0xD64, 0x04518A3C}, {0xD68, 0x00002101},
+	{0xE00, 0x2D2D2D2D}, {0xE04, 0x2D2D2D2D},
+	{0xE08, 0x0390272D}, {0xE10, 0x2D2D2D2D},
+	{0xE14, 0x2D2D2D2D}, {0xE18, 0x2D2D2D2D},
+	{0xE1C, 0x2D2D2D2D}, {0xE28, 0x00000000},
+	{0xE30, 0x1000DC1F}, {0xE34, 0x10008C1F},
+	{0xE38, 0x02140102}, {0xE3C, 0x681604C2},
+	{0xE40, 0x01007C00}, {0xE44, 0x01004800},
+	{0xE48, 0xFB000000}, {0xE4C, 0x000028D1},
+	{0xE50, 0x1000DC1F}, {0xE54, 0x10008C1F},
+	{0xE58, 0x02140102}, {0xE5C, 0x28160D05},
+	{0xE60, 0x0000C008}, {0xE68, 0x001B25A4},
+	{0xE64, 0x281600A0}, {0xE6C, 0x01C00010},
+	{0xE70, 0x01C00010}, {0xE74, 0x02000010},
+	{0xE78, 0x02000010}, {0xE7C, 0x02000010},
+	{0xE80, 0x02000010}, {0xE84, 0x01C00010},
+	{0xE88, 0x02000010}, {0xE8C, 0x01C00010},
+	{0xED0, 0x01C00010}, {0xED4, 0x01C00010},
+	{0xED8, 0x01C00010}, {0xEDC, 0x00000010},
+	{0xEE0, 0x00000010}, {0xEEC, 0x03C00010},
+	{0xF14, 0x00000003}, {0xF00, 0x00100300},
+	{0xF08, 0x0000800B}, {0xF0C, 0x0000F007},
+	{0xF10, 0x0000A487}, {0xF1C, 0x80000064},
+	{0xF38, 0x00030155}, {0xF3C, 0x0000003A},
+	{0xF4C, 0x13000000}, {0xF50, 0x00000000},
+	{0xF18, 0x00000000},
+	{0xffff, 0xffffffff},
+};
+
+static const struct rtl8xxxu_reg32val rtl8710b_agc_table[] = {
+	{0xC78, 0xFC000001}, {0xC78, 0xFB010001},
+	{0xC78, 0xFA020001}, {0xC78, 0xF9030001},
+	{0xC78, 0xF8040001}, {0xC78, 0xF7050001},
+	{0xC78, 0xF6060001}, {0xC78, 0xF5070001},
+	{0xC78, 0xF4080001}, {0xC78, 0xF3090001},
+	{0xC78, 0xF20A0001}, {0xC78, 0xF10B0001},
+	{0xC78, 0xF00C0001}, {0xC78, 0xEF0D0001},
+	{0xC78, 0xEE0E0001}, {0xC78, 0xED0F0001},
+	{0xC78, 0xEC100001}, {0xC78, 0xEB110001},
+	{0xC78, 0xEA120001}, {0xC78, 0xE9130001},
+	{0xC78, 0xE8140001}, {0xC78, 0xE7150001},
+	{0xC78, 0xE6160001}, {0xC78, 0xE5170001},
+	{0xC78, 0xE4180001}, {0xC78, 0xE3190001},
+	{0xC78, 0xE21A0001}, {0xC78, 0xE11B0001},
+	{0xC78, 0xE01C0001}, {0xC78, 0xC31D0001},
+	{0xC78, 0xC21E0001}, {0xC78, 0xC11F0001},
+	{0xC78, 0xC0200001}, {0xC78, 0xA3210001},
+	{0xC78, 0xA2220001}, {0xC78, 0xA1230001},
+	{0xC78, 0xA0240001}, {0xC78, 0x86250001},
+	{0xC78, 0x85260001}, {0xC78, 0x84270001},
+	{0xC78, 0x83280001}, {0xC78, 0x82290001},
+	{0xC78, 0x812A0001}, {0xC78, 0x802B0001},
+	{0xC78, 0x632C0001}, {0xC78, 0x622D0001},
+	{0xC78, 0x612E0001}, {0xC78, 0x602F0001},
+	{0xC78, 0x42300001}, {0xC78, 0x41310001},
+	{0xC78, 0x40320001}, {0xC78, 0x23330001},
+	{0xC78, 0x22340001}, {0xC78, 0x21350001},
+	{0xC78, 0x20360001}, {0xC78, 0x02370001},
+	{0xC78, 0x01380001}, {0xC78, 0x00390001},
+	{0xC78, 0x003A0001}, {0xC78, 0x003B0001},
+	{0xC78, 0x003C0001}, {0xC78, 0x003D0001},
+	{0xC78, 0x003E0001}, {0xC78, 0x003F0001},
+	{0xC78, 0xF7400001}, {0xC78, 0xF7410001},
+	{0xC78, 0xF7420001}, {0xC78, 0xF7430001},
+	{0xC78, 0xF7440001}, {0xC78, 0xF7450001},
+	{0xC78, 0xF7460001}, {0xC78, 0xF7470001},
+	{0xC78, 0xF7480001}, {0xC78, 0xF6490001},
+	{0xC78, 0xF34A0001}, {0xC78, 0xF24B0001},
+	{0xC78, 0xF14C0001}, {0xC78, 0xF04D0001},
+	{0xC78, 0xD14E0001}, {0xC78, 0xD04F0001},
+	{0xC78, 0xB5500001}, {0xC78, 0xB4510001},
+	{0xC78, 0xB3520001}, {0xC78, 0xB2530001},
+	{0xC78, 0xB1540001}, {0xC78, 0xB0550001},
+	{0xC78, 0xAF560001}, {0xC78, 0xAE570001},
+	{0xC78, 0xAD580001}, {0xC78, 0xAC590001},
+	{0xC78, 0xAB5A0001}, {0xC78, 0xAA5B0001},
+	{0xC78, 0xA95C0001}, {0xC78, 0xA85D0001},
+	{0xC78, 0xA75E0001}, {0xC78, 0xA65F0001},
+	{0xC78, 0xA5600001}, {0xC78, 0xA4610001},
+	{0xC78, 0xA3620001}, {0xC78, 0xA2630001},
+	{0xC78, 0xA1640001}, {0xC78, 0xA0650001},
+	{0xC78, 0x87660001}, {0xC78, 0x86670001},
+	{0xC78, 0x85680001}, {0xC78, 0x84690001},
+	{0xC78, 0x836A0001}, {0xC78, 0x826B0001},
+	{0xC78, 0x816C0001}, {0xC78, 0x806D0001},
+	{0xC78, 0x636E0001}, {0xC78, 0x626F0001},
+	{0xC78, 0x61700001}, {0xC78, 0x60710001},
+	{0xC78, 0x42720001}, {0xC78, 0x41730001},
+	{0xC78, 0x40740001}, {0xC78, 0x23750001},
+	{0xC78, 0x22760001}, {0xC78, 0x21770001},
+	{0xC78, 0x20780001}, {0xC78, 0x03790001},
+	{0xC78, 0x027A0001}, {0xC78, 0x017B0001},
+	{0xC78, 0x007C0001}, {0xC78, 0x007D0001},
+	{0xC78, 0x007E0001}, {0xC78, 0x007F0001},
+	{0xC50, 0x69553422}, {0xC50, 0x69553420},
+	{0xffff, 0xffffffff}
+};
+
+static const struct rtl8xxxu_rfregval rtl8710bu_qfn48m_u_radioa_init_table[] = {
+	{0x00, 0x00030000}, {0x08, 0x00008400},
+	{0x17, 0x00000000}, {0x18, 0x00000C01},
+	{0x19, 0x000739D2}, {0x1C, 0x00000C4C},
+	{0x1B, 0x00000C6C}, {0x1E, 0x00080009},
+	{0x1F, 0x00000880}, {0x2F, 0x0001A060},
+	{0x3F, 0x00015000}, {0x42, 0x000060C0},
+	{0x57, 0x000D0000}, {0x58, 0x000C0160},
+	{0x67, 0x00001552}, {0x83, 0x00000000},
+	{0xB0, 0x000FF9F0}, {0xB1, 0x00010018},
+	{0xB2, 0x00054C00}, {0xB4, 0x0004486B},
+	{0xB5, 0x0000112A}, {0xB6, 0x0000053E},
+	{0xB7, 0x00014408}, {0xB8, 0x00010200},
+	{0xB9, 0x00080801}, {0xBA, 0x00040001},
+	{0xBB, 0x00000400}, {0xBF, 0x000C0000},
+	{0xC2, 0x00002400}, {0xC3, 0x00000009},
+	{0xC4, 0x00040C91}, {0xC5, 0x00099999},
+	{0xC6, 0x000000A3}, {0xC7, 0x00088820},
+	{0xC8, 0x00076C06}, {0xC9, 0x00000000},
+	{0xCA, 0x00080000}, {0xDF, 0x00000180},
+	{0xEF, 0x000001A8}, {0x3D, 0x00000003},
+	{0x3D, 0x00080003}, {0x51, 0x000F1E69},
+	{0x52, 0x000FBF6C}, {0x53, 0x0000032F},
+	{0x54, 0x00055007}, {0x56, 0x000517F0},
+	{0x35, 0x000000F4}, {0x35, 0x00000179},
+	{0x35, 0x000002F4}, {0x36, 0x00000BF8},
+	{0x36, 0x00008BF8}, {0x36, 0x00010BF8},
+	{0x36, 0x00018BF8}, {0x18, 0x00000C01},
+	{0x5A, 0x00048000}, {0x5A, 0x00048000},
+	{0x34, 0x0000ADF5}, {0x34, 0x00009DF2},
+	{0x34, 0x00008DEF}, {0x34, 0x00007DEC},
+	{0x34, 0x00006DE9}, {0x34, 0x00005CEC},
+	{0x34, 0x00004CE9}, {0x34, 0x00003C6C},
+	{0x34, 0x00002C69}, {0x34, 0x0000106E},
+	{0x34, 0x0000006B}, {0x84, 0x00048000},
+	{0x87, 0x00000065}, {0x8E, 0x00065540},
+	{0xDF, 0x00000110}, {0x86, 0x0000002A},
+	{0x8F, 0x00088000}, {0x81, 0x0003FD80},
+	{0xEF, 0x00082000}, {0x3B, 0x000F0F00},
+	{0x3B, 0x000E0E00}, {0x3B, 0x000DFE00},
+	{0x3B, 0x000C0D00}, {0x3B, 0x000B0C00},
+	{0x3B, 0x000A0500}, {0x3B, 0x00090400},
+	{0x3B, 0x00080000}, {0x3B, 0x00070F00},
+	{0x3B, 0x00060E00}, {0x3B, 0x00050A00},
+	{0x3B, 0x00040D00}, {0x3B, 0x00030C00},
+	{0x3B, 0x00020500}, {0x3B, 0x00010400},
+	{0x3B, 0x00000000}, {0xEF, 0x00080000},
+	{0xEF, 0x00088000}, {0x3B, 0x00000170},
+	{0x3B, 0x000C0030}, {0xEF, 0x00080000},
+	{0xEF, 0x00080000}, {0x30, 0x00010000},
+	{0x31, 0x0000000F}, {0x32, 0x00047EFE},
+	{0xEF, 0x00000000}, {0x00, 0x00010159},
+	{0x18, 0x0000FC01}, {0xFE, 0x00000000},
+	{0x00, 0x00033D95},
+	{0xff, 0xffffffff}
+};
+
+static const struct rtl8xxxu_rfregval rtl8710bu_qfn48m_s_radioa_init_table[] = {
+	{0x00, 0x00030000}, {0x08, 0x00008400},
+	{0x17, 0x00000000}, {0x18, 0x00000C01},
+	{0x19, 0x000739D2}, {0x1C, 0x00000C4C},
+	{0x1B, 0x00000C6C}, {0x1E, 0x00080009},
+	{0x1F, 0x00000880}, {0x2F, 0x0001A060},
+	{0x3F, 0x00015000}, {0x42, 0x000060C0},
+	{0x57, 0x000D0000}, {0x58, 0x000C0160},
+	{0x67, 0x00001552}, {0x83, 0x00000000},
+	{0xB0, 0x000FF9F0}, {0xB1, 0x00010018},
+	{0xB2, 0x00054C00}, {0xB4, 0x0004486B},
+	{0xB5, 0x0000112A}, {0xB6, 0x0000053E},
+	{0xB7, 0x00014408}, {0xB8, 0x00010200},
+	{0xB9, 0x00080801}, {0xBA, 0x00040001},
+	{0xBB, 0x00000400}, {0xBF, 0x000C0000},
+	{0xC2, 0x00002400}, {0xC3, 0x00000009},
+	{0xC4, 0x00040C91}, {0xC5, 0x00099999},
+	{0xC6, 0x000000A3}, {0xC7, 0x00088820},
+	{0xC8, 0x00076C06}, {0xC9, 0x00000000},
+	{0xCA, 0x00080000}, {0xDF, 0x00000180},
+	{0xEF, 0x000001A8}, {0x3D, 0x00000003},
+	{0x3D, 0x00080003}, {0x51, 0x000F1E69},
+	{0x52, 0x000FBF6C}, {0x53, 0x0000032F},
+	{0x54, 0x00055007}, {0x56, 0x000517F0},
+	{0x35, 0x000000F4}, {0x35, 0x00000179},
+	{0x35, 0x000002F4}, {0x36, 0x00000BF8},
+	{0x36, 0x00008BF8}, {0x36, 0x00010BF8},
+	{0x36, 0x00018BF8}, {0x18, 0x00000C01},
+	{0x5A, 0x00048000}, {0x5A, 0x00048000},
+	{0x34, 0x0000ADF5}, {0x34, 0x00009DF2},
+	{0x34, 0x00008DEF}, {0x34, 0x00007DEC},
+	{0x34, 0x00006DE9}, {0x34, 0x00005CEC},
+	{0x34, 0x00004CE9}, {0x34, 0x00003C6C},
+	{0x34, 0x00002C69}, {0x34, 0x0000106E},
+	{0x34, 0x0000006B}, {0x84, 0x00048000},
+	{0x87, 0x00000065}, {0x8E, 0x00065540},
+	{0xDF, 0x00000110}, {0x86, 0x0000002A},
+	{0x8F, 0x00088000}, {0x81, 0x0003FD80},
+	{0xEF, 0x00082000}, {0x3B, 0x000F0F00},
+	{0x3B, 0x000E0E00}, {0x3B, 0x000DFE00},
+	{0x3B, 0x000C0D00}, {0x3B, 0x000B0C00},
+	{0x3B, 0x000A0500}, {0x3B, 0x00090400},
+	{0x3B, 0x00080000}, {0x3B, 0x00070F00},
+	{0x3B, 0x00060E00}, {0x3B, 0x00050A00},
+	{0x3B, 0x00040D00}, {0x3B, 0x00030C00},
+	{0x3B, 0x00020500}, {0x3B, 0x00010400},
+	{0x3B, 0x00000000}, {0xEF, 0x00080000},
+	{0xEF, 0x00088000}, {0x3B, 0x000000B0},
+	{0x3B, 0x000C0030}, {0xEF, 0x00080000},
+	{0xEF, 0x00080000}, {0x30, 0x00010000},
+	{0x31, 0x0000000F}, {0x32, 0x00047EFE},
+	{0xEF, 0x00000000}, {0x00, 0x00010159},
+	{0x18, 0x0000FC01}, {0xFE, 0x00000000},
+	{0x00, 0x00033D95},
+	{0xff, 0xffffffff}
+};
+
+static u32 rtl8710b_indirect_read32(struct rtl8xxxu_priv *priv, u32 addr)
+{
+	struct device *dev = &priv->udev->dev;
+	u32 val32, value = 0xffffffff;
+	u8 polling_count = 0xff;
+
+	if (!IS_ALIGNED(addr, 4)) {
+		dev_warn(dev, "%s: Aborting because 0x%x is not a multiple of 4.\n",
+			 __func__, addr);
+		return value;
+	}
+
+	mutex_lock(&priv->syson_indirect_access_mutex);
+
+	rtl8xxxu_write32(priv, REG_USB_HOST_INDIRECT_ADDR_8710B, addr);
+	rtl8xxxu_write32(priv, REG_EFUSE_INDIRECT_CTRL_8710B, NORMAL_REG_READ_OFFSET);
+
+	do
+		val32 = rtl8xxxu_read32(priv, REG_EFUSE_INDIRECT_CTRL_8710B);
+	while ((val32 & BIT(31)) && (--polling_count > 0));
+
+	if (polling_count == 0)
+		dev_warn(dev, "%s: Failed to read from 0x%x, 0x806c = 0x%x\n",
+			 __func__, addr, val32);
+	else
+		value = rtl8xxxu_read32(priv, REG_USB_HOST_INDIRECT_DATA_8710B);
+
+	mutex_unlock(&priv->syson_indirect_access_mutex);
+
+	if (rtl8xxxu_debug & RTL8XXXU_DEBUG_REG_READ)
+		dev_info(dev, "%s(%04x) = 0x%08x\n", __func__, addr, value);
+
+	return value;
+}
+
+static void rtl8710b_indirect_write32(struct rtl8xxxu_priv *priv, u32 addr, u32 val)
+{
+	struct device *dev = &priv->udev->dev;
+	u8 polling_count = 0xff;
+	u32 val32;
+
+	if (!IS_ALIGNED(addr, 4)) {
+		dev_warn(dev, "%s: Aborting because 0x%x is not a multiple of 4.\n",
+			 __func__, addr);
+		return;
+	}
+
+	mutex_lock(&priv->syson_indirect_access_mutex);
+
+	rtl8xxxu_write32(priv, REG_USB_HOST_INDIRECT_ADDR_8710B, addr);
+	rtl8xxxu_write32(priv, REG_USB_HOST_INDIRECT_DATA_8710B, val);
+	rtl8xxxu_write32(priv, REG_EFUSE_INDIRECT_CTRL_8710B, NORMAL_REG_WRITE_OFFSET);
+
+	do
+		val32 = rtl8xxxu_read32(priv, REG_EFUSE_INDIRECT_CTRL_8710B);
+	while ((val32 & BIT(31)) && (--polling_count > 0));
+
+	if (polling_count == 0)
+		dev_warn(dev, "%s: Failed to write 0x%x to 0x%x, 0x806c = 0x%x\n",
+			 __func__, val, addr, val32);
+
+	mutex_unlock(&priv->syson_indirect_access_mutex);
+
+	if (rtl8xxxu_debug & RTL8XXXU_DEBUG_REG_WRITE)
+		dev_info(dev, "%s(%04x) = 0x%08x\n", __func__, addr, val);
+}
+
+static u32 rtl8710b_read_syson_reg(struct rtl8xxxu_priv *priv, u32 addr)
+{
+	return rtl8710b_indirect_read32(priv, addr | SYSON_REG_BASE_ADDR_8710B);
+}
+
+static void rtl8710b_write_syson_reg(struct rtl8xxxu_priv *priv, u32 addr, u32 val)
+{
+	rtl8710b_indirect_write32(priv, addr | SYSON_REG_BASE_ADDR_8710B, val);
+}
+
+static int rtl8710b_read_efuse8(struct rtl8xxxu_priv *priv, u16 offset, u8 *data)
+{
+	u32 val32;
+	int i;
+
+	/* Write Address */
+	rtl8xxxu_write32(priv, REG_USB_HOST_INDIRECT_ADDR_8710B, offset);
+
+	rtl8xxxu_write32(priv, REG_EFUSE_INDIRECT_CTRL_8710B, EFUSE_READ_OFFSET);
+
+	/* Poll for data read */
+	val32 = rtl8xxxu_read32(priv, REG_EFUSE_INDIRECT_CTRL_8710B);
+	for (i = 0; i < RTL8XXXU_MAX_REG_POLL; i++) {
+		val32 = rtl8xxxu_read32(priv, REG_EFUSE_INDIRECT_CTRL_8710B);
+		if (!(val32 & BIT(31)))
+			break;
+	}
+
+	if (i == RTL8XXXU_MAX_REG_POLL)
+		return -EIO;
+
+	val32 = rtl8xxxu_read32(priv, REG_USB_HOST_INDIRECT_DATA_8710B);
+
+	*data = val32 & 0xff;
+	return 0;
+}
+
+#define EEPROM_PACKAGE_TYPE_8710B	0xF8
+#define PACKAGE_QFN48M_U		0xee
+#define PACKAGE_QFN48M_S		0xfe
+
+static int rtl8710bu_identify_chip(struct rtl8xxxu_priv *priv)
+{
+	struct device *dev = &priv->udev->dev;
+	u32 cfg0, cfg2, vendor;
+	u8 package_type = 0x7; /* a nonsense value */
+
+	sprintf(priv->chip_name, "8710BU");
+	priv->rtl_chip = RTL8710B;
+	priv->rf_paths = 1;
+	priv->rx_paths = 1;
+	priv->tx_paths = 1;
+	priv->has_wifi = 1;
+
+	cfg0 = rtl8710b_read_syson_reg(priv, REG_SYS_SYSTEM_CFG0_8710B);
+	priv->chip_cut = cfg0 & 0xf;
+
+	if (cfg0 & BIT(16)) {
+		dev_info(dev, "%s: Unsupported test chip\n", __func__);
+		return -EOPNOTSUPP;
+	}
+
+	vendor = u32_get_bits(cfg0, 0xc0);
+
+	/* SMIC and TSMC are swapped compared to rtl8xxxu_identify_vendor_2bits */
+	switch (vendor) {
+	case 0:
+		sprintf(priv->chip_vendor, "SMIC");
+		priv->vendor_smic = 1;
+		break;
+	case 1:
+		sprintf(priv->chip_vendor, "TSMC");
+		break;
+	case 2:
+		sprintf(priv->chip_vendor, "UMC");
+		priv->vendor_umc = 1;
+		break;
+	default:
+		sprintf(priv->chip_vendor, "unknown");
+		break;
+	}
+
+	rtl8710b_read_efuse8(priv, EEPROM_PACKAGE_TYPE_8710B, &package_type);
+
+	if (package_type == 0xff) {
+		dev_warn(dev, "Package type is undefined. Assuming it based on the vendor.\n");
+
+		if (priv->vendor_umc) {
+			package_type = PACKAGE_QFN48M_U;
+		} else if (priv->vendor_smic) {
+			package_type = PACKAGE_QFN48M_S;
+		} else {
+			dev_warn(dev, "The vendor is neither UMC nor SMIC. Assuming the package type is QFN48M_U.\n");
+
+			/*
+			 * In this case the vendor driver doesn't set
+			 * the package type to anything, which is the
+			 * same as setting it to PACKAGE_DEFAULT (0).
+			 */
+			package_type = PACKAGE_QFN48M_U;
+		}
+	} else if (package_type != PACKAGE_QFN48M_S &&
+		   package_type != PACKAGE_QFN48M_U) {
+		dev_warn(dev, "Failed to read the package type. Assuming it's the default QFN48M_U.\n");
+
+		/*
+		 * In this case the vendor driver actually sets it to
+		 * PACKAGE_DEFAULT, but that selects the same values
+		 * from the init tables as PACKAGE_QFN48M_U.
+		 */
+		package_type = PACKAGE_QFN48M_U;
+	}
+
+	priv->package_type = package_type;
+
+	dev_dbg(dev, "Package type: 0x%x\n", package_type);
+
+	cfg2 = rtl8710b_read_syson_reg(priv, REG_SYS_SYSTEM_CFG2_8710B);
+	priv->rom_rev = cfg2 & 0xf;
+
+	return rtl8xxxu_config_endpoints_no_sie(priv);
+}
+
+static void rtl8710b_revise_cck_tx_psf(struct rtl8xxxu_priv *priv, u8 channel)
+{
+	if (channel == 13) {
+		/* Normal values */
+		rtl8xxxu_write32(priv, REG_CCK0_TX_FILTER2, 0x64B80C1C);
+		rtl8xxxu_write32(priv, REG_CCK0_DEBUG_PORT, 0x00008810);
+		rtl8xxxu_write32(priv, REG_CCK0_TX_FILTER3, 0x01235667);
+		/* Special value for channel 13 */
+		rtl8xxxu_write32(priv, REG_CCK0_TX_FILTER1, 0xd1d80001);
+	} else if (channel == 14) {
+		/* Special values for channel 14 */
+		rtl8xxxu_write32(priv, REG_CCK0_TX_FILTER2, 0x0000B81C);
+		rtl8xxxu_write32(priv, REG_CCK0_DEBUG_PORT, 0x00000000);
+		rtl8xxxu_write32(priv, REG_CCK0_TX_FILTER3, 0x00003667);
+		/* Normal value */
+		rtl8xxxu_write32(priv, REG_CCK0_TX_FILTER1, 0xE82C0001);
+	} else {
+		/* Restore normal values from the phy init table */
+		rtl8xxxu_write32(priv, REG_CCK0_TX_FILTER2, 0x64B80C1C);
+		rtl8xxxu_write32(priv, REG_CCK0_DEBUG_PORT, 0x00008810);
+		rtl8xxxu_write32(priv, REG_CCK0_TX_FILTER3, 0x01235667);
+		rtl8xxxu_write32(priv, REG_CCK0_TX_FILTER1, 0xE82C0001);
+	}
+}
+
+static void rtl8710bu_config_channel(struct ieee80211_hw *hw)
+{
+	struct rtl8xxxu_priv *priv = hw->priv;
+	bool ht40 = conf_is_ht40(&hw->conf);
+	u8 channel, subchannel = 0;
+	bool sec_ch_above = 0;
+	u32 val32;
+	u16 val16;
+
+	channel = (u8)hw->conf.chandef.chan->hw_value;
+
+	if (conf_is_ht40_plus(&hw->conf)) {
+		sec_ch_above = 1;
+		channel += 2;
+		subchannel = 2;
+	} else if (conf_is_ht40_minus(&hw->conf)) {
+		sec_ch_above = 0;
+		channel -= 2;
+		subchannel = 1;
+	}
+
+	/* Set channel */
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_MODE_AG);
+	u32p_replace_bits(&val32, channel, MODE_AG_CHANNEL_MASK);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_MODE_AG, val32);
+
+	rtl8710b_revise_cck_tx_psf(priv, channel);
+
+	/* Set bandwidth mode */
+	val16 = rtl8xxxu_read16(priv, REG_WMAC_TRXPTCL_CTL);
+	val16 &= ~WMAC_TRXPTCL_CTL_BW_MASK;
+	if (ht40)
+		val16 |= WMAC_TRXPTCL_CTL_BW_40;
+	rtl8xxxu_write16(priv, REG_WMAC_TRXPTCL_CTL, val16);
+
+	rtl8xxxu_write8(priv, REG_DATA_SUBCHANNEL, subchannel);
+
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_RF_MODE);
+	u32p_replace_bits(&val32, ht40, FPGA_RF_MODE);
+	rtl8xxxu_write32(priv, REG_FPGA0_RF_MODE, val32);
+
+	val32 = rtl8xxxu_read32(priv, REG_FPGA1_RF_MODE);
+	u32p_replace_bits(&val32, ht40, FPGA_RF_MODE);
+	rtl8xxxu_write32(priv, REG_FPGA1_RF_MODE, val32);
+
+	if (ht40) {
+		/* Set Control channel to upper or lower. */
+		val32 = rtl8xxxu_read32(priv, REG_CCK0_SYSTEM);
+		u32p_replace_bits(&val32, !sec_ch_above, CCK0_SIDEBAND);
+		rtl8xxxu_write32(priv, REG_CCK0_SYSTEM, val32);
+	}
+
+	/* RXADC CLK */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_RF_MODE);
+	val32 |= GENMASK(10, 8);
+	rtl8xxxu_write32(priv, REG_FPGA0_RF_MODE, val32);
+
+	/* TXDAC CLK */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_RF_MODE);
+	val32 |= BIT(14) | BIT(12);
+	val32 &= ~BIT(13);
+	rtl8xxxu_write32(priv, REG_FPGA0_RF_MODE, val32);
+
+	/* small BW */
+	val32 = rtl8xxxu_read32(priv, REG_OFDM0_TX_PSDO_NOISE_WEIGHT);
+	val32 &= ~GENMASK(31, 30);
+	rtl8xxxu_write32(priv, REG_OFDM0_TX_PSDO_NOISE_WEIGHT, val32);
+
+	/* adc buffer clk */
+	val32 = rtl8xxxu_read32(priv, REG_OFDM0_TX_PSDO_NOISE_WEIGHT);
+	val32 &= ~BIT(29);
+	val32 |= BIT(28);
+	rtl8xxxu_write32(priv, REG_OFDM0_TX_PSDO_NOISE_WEIGHT, val32);
+
+	/* adc buffer clk */
+	val32 = rtl8xxxu_read32(priv, REG_OFDM0_XA_RX_AFE);
+	val32 &= ~BIT(29);
+	val32 |= BIT(28);
+	rtl8xxxu_write32(priv, REG_OFDM0_XA_RX_AFE, val32);
+
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_XB_RF_INT_OE);
+	val32 &= ~BIT(30);
+	val32 |= BIT(29);
+	rtl8xxxu_write32(priv, REG_FPGA0_XB_RF_INT_OE, val32);
+
+	if (ht40) {
+		val32 = rtl8xxxu_read32(priv, REG_OFDM_RX_DFIR);
+		val32 &= ~BIT(19);
+		rtl8xxxu_write32(priv, REG_OFDM_RX_DFIR, val32);
+
+		val32 = rtl8xxxu_read32(priv, REG_OFDM_RX_DFIR);
+		val32 &= ~GENMASK(23, 20);
+		rtl8xxxu_write32(priv, REG_OFDM_RX_DFIR, val32);
+
+		val32 = rtl8xxxu_read32(priv, REG_OFDM_RX_DFIR);
+		val32 &= ~GENMASK(27, 24);
+		rtl8xxxu_write32(priv, REG_OFDM_RX_DFIR, val32);
+
+		/* RF TRX_BW */
+		val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_MODE_AG);
+		val32 &= ~MODE_AG_BW_MASK;
+		val32 |= MODE_AG_BW_40MHZ_8723B;
+		rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_MODE_AG, val32);
+	} else {
+		val32 = rtl8xxxu_read32(priv, REG_OFDM_RX_DFIR);
+		val32 |= BIT(19);
+		rtl8xxxu_write32(priv, REG_OFDM_RX_DFIR, val32);
+
+		val32 = rtl8xxxu_read32(priv, REG_OFDM_RX_DFIR);
+		val32 &= ~GENMASK(23, 20);
+		val32 |= BIT(23);
+		rtl8xxxu_write32(priv, REG_OFDM_RX_DFIR, val32);
+
+		val32 = rtl8xxxu_read32(priv, REG_OFDM_RX_DFIR);
+		val32 &= ~GENMASK(27, 24);
+		val32 |= BIT(27) | BIT(25);
+		rtl8xxxu_write32(priv, REG_OFDM_RX_DFIR, val32);
+
+		/* RF TRX_BW */
+		val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_MODE_AG);
+		val32 &= ~MODE_AG_BW_MASK;
+		val32 |= MODE_AG_BW_20MHZ_8723B;
+		rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_MODE_AG, val32);
+	}
+}
+
+static void rtl8710bu_init_aggregation(struct rtl8xxxu_priv *priv)
+{
+	u32 agg_rx;
+	u8 agg_ctrl;
+
+	/* RX aggregation */
+	agg_ctrl = rtl8xxxu_read8(priv, REG_TRXDMA_CTRL);
+	agg_ctrl &= ~TRXDMA_CTRL_RXDMA_AGG_EN;
+
+	agg_rx = rtl8xxxu_read32(priv, REG_RXDMA_AGG_PG_TH);
+	agg_rx &= ~RXDMA_USB_AGG_ENABLE;
+	agg_rx &= ~0xFF0F; /* reset agg size and timeout */
+
+	rtl8xxxu_write8(priv, REG_TRXDMA_CTRL, agg_ctrl);
+	rtl8xxxu_write32(priv, REG_RXDMA_AGG_PG_TH, agg_rx);
+}
+
+static void rtl8710bu_init_statistics(struct rtl8xxxu_priv *priv)
+{
+	u32 val32;
+
+	/* Time duration for NHM unit: 4us, 0xc350=200ms */
+	rtl8xxxu_write16(priv, REG_NHM_TIMER_8723B + 2, 0xc350);
+	rtl8xxxu_write16(priv, REG_NHM_TH9_TH10_8723B + 2, 0xffff);
+	rtl8xxxu_write32(priv, REG_NHM_TH3_TO_TH0_8723B, 0xffffff50);
+	rtl8xxxu_write32(priv, REG_NHM_TH7_TO_TH4_8723B, 0xffffffff);
+
+	/* TH8 */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	val32 |= 0xff;
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+	/* Enable CCK */
+	val32 = rtl8xxxu_read32(priv, REG_NHM_TH9_TH10_8723B);
+	val32 &= ~(BIT(8) | BIT(9) | BIT(10));
+	val32 |= BIT(8);
+	rtl8xxxu_write32(priv, REG_NHM_TH9_TH10_8723B, val32);
+
+	/* Max power amongst all RX antennas */
+	val32 = rtl8xxxu_read32(priv, REG_OFDM0_FA_RSTC);
+	val32 |= BIT(7);
+	rtl8xxxu_write32(priv, REG_OFDM0_FA_RSTC, val32);
+}
+
+static int rtl8710b_read_efuse(struct rtl8xxxu_priv *priv)
+{
+	struct device *dev = &priv->udev->dev;
+	u8 val8, word_mask, header, extheader;
+	u16 efuse_addr, offset;
+	int i, ret = 0;
+	u32 val32;
+
+	val32 = rtl8710b_read_syson_reg(priv, REG_SYS_EEPROM_CTRL0_8710B);
+	priv->boot_eeprom = u32_get_bits(val32, EEPROM_BOOT);
+	priv->has_eeprom = u32_get_bits(val32, EEPROM_ENABLE);
+
+	/* Default value is 0xff */
+	memset(priv->efuse_wifi.raw, 0xff, EFUSE_MAP_LEN);
+
+	efuse_addr = 0;
+	while (efuse_addr < EFUSE_REAL_CONTENT_LEN_8723A) {
+		u16 map_addr;
+
+		ret = rtl8710b_read_efuse8(priv, efuse_addr++, &header);
+		if (ret || header == 0xff)
+			goto exit;
+
+		if ((header & 0x1f) == 0x0f) {	/* extended header */
+			offset = (header & 0xe0) >> 5;
+
+			ret = rtl8710b_read_efuse8(priv, efuse_addr++, &extheader);
+			if (ret)
+				goto exit;
+
+			/* All words disabled */
+			if ((extheader & 0x0f) == 0x0f)
+				continue;
+
+			offset |= ((extheader & 0xf0) >> 1);
+			word_mask = extheader & 0x0f;
+		} else {
+			offset = (header >> 4) & 0x0f;
+			word_mask = header & 0x0f;
+		}
+
+		/* Get word enable value from PG header */
+
+		/* We have 8 bits to indicate validity */
+		map_addr = offset * 8;
+		for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++) {
+			/* Check word enable condition in the section */
+			if (word_mask & BIT(i)) {
+				map_addr += 2;
+				continue;
+			}
+
+			ret = rtl8710b_read_efuse8(priv, efuse_addr++, &val8);
+			if (ret)
+				goto exit;
+			if (map_addr >= EFUSE_MAP_LEN - 1) {
+				dev_warn(dev, "%s: Illegal map_addr (%04x), efuse corrupt!\n",
+					 __func__, map_addr);
+				ret = -EINVAL;
+				goto exit;
+			}
+			priv->efuse_wifi.raw[map_addr++] = val8;
+
+			ret = rtl8710b_read_efuse8(priv, efuse_addr++, &val8);
+			if (ret)
+				goto exit;
+			priv->efuse_wifi.raw[map_addr++] = val8;
+		}
+	}
+
+exit:
+
+	return ret;
+}
+
+static int rtl8710bu_parse_efuse(struct rtl8xxxu_priv *priv)
+{
+	struct rtl8710bu_efuse *efuse = &priv->efuse_wifi.efuse8710bu;
+
+	if (efuse->rtl_id != cpu_to_le16(0x8195))
+		return -EINVAL;
+
+	ether_addr_copy(priv->mac_addr, efuse->mac_addr);
+
+	memcpy(priv->cck_tx_power_index_A, efuse->tx_power_index_A.cck_base,
+	       sizeof(efuse->tx_power_index_A.cck_base));
+
+	memcpy(priv->ht40_1s_tx_power_index_A,
+	       efuse->tx_power_index_A.ht40_base,
+	       sizeof(efuse->tx_power_index_A.ht40_base));
+
+	priv->ofdm_tx_power_diff[0].a = efuse->tx_power_index_A.ht20_ofdm_1s_diff.a;
+	priv->ht20_tx_power_diff[0].a = efuse->tx_power_index_A.ht20_ofdm_1s_diff.b;
+
+	priv->default_crystal_cap = efuse->xtal_k & 0x3f;
+
+	return 0;
+}
+
+static int rtl8710bu_load_firmware(struct rtl8xxxu_priv *priv)
+{
+	if (priv->vendor_smic) {
+		return rtl8xxxu_load_firmware(priv, "rtlwifi/rtl8710bufw_SMIC.bin");
+	} else if (priv->vendor_umc) {
+		return rtl8xxxu_load_firmware(priv, "rtlwifi/rtl8710bufw_UMC.bin");
+	} else {
+		dev_err(&priv->udev->dev, "We have no suitable firmware for this chip.\n");
+		return -1;
+	}
+}
+
+static void rtl8710bu_init_phy_bb(struct rtl8xxxu_priv *priv)
+{
+	const struct rtl8xxxu_reg32val *phy_init_table;
+	u32 val32;
+
+	/* Enable BB and RF */
+	val32 = rtl8xxxu_read32(priv, REG_SYS_FUNC_8710B);
+	val32 |= GENMASK(17, 16) | GENMASK(26, 24);
+	rtl8xxxu_write32(priv, REG_SYS_FUNC_8710B, val32);
+
+	if (priv->package_type == PACKAGE_QFN48M_U)
+		phy_init_table = rtl8710bu_qfn48m_u_phy_init_table;
+	else
+		phy_init_table = rtl8710bu_qfn48m_s_phy_init_table;
+
+	rtl8xxxu_init_phy_regs(priv, phy_init_table);
+
+	rtl8xxxu_init_phy_regs(priv, rtl8710b_agc_table);
+}
+
+static int rtl8710bu_init_phy_rf(struct rtl8xxxu_priv *priv)
+{
+	const struct rtl8xxxu_rfregval *radioa_init_table;
+
+	if (priv->package_type == PACKAGE_QFN48M_U)
+		radioa_init_table = rtl8710bu_qfn48m_u_radioa_init_table;
+	else
+		radioa_init_table = rtl8710bu_qfn48m_s_radioa_init_table;
+
+	return rtl8xxxu_init_phy_rf(priv, radioa_init_table, RF_A);
+}
+
+static int rtl8710bu_iqk_path_a(struct rtl8xxxu_priv *priv, u32 *lok_result)
+{
+	u32 reg_eac, reg_e94, reg_e9c, val32, path_sel_bb;
+	int result = 0;
+
+	path_sel_bb = rtl8xxxu_read32(priv, REG_S0S1_PATH_SWITCH);
+
+	rtl8xxxu_write32(priv, REG_S0S1_PATH_SWITCH, 0x99000000);
+
+	/*
+	 * Leave IQK mode
+	 */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	u32p_replace_bits(&val32, 0, 0xffffff00);
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+	/*
+	 * Enable path A PA in TX IQK mode
+	 */
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_WE_LUT);
+	val32 |= 0x80000;
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_WE_LUT, val32);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_RCK_OS, 0x20000);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_TXPA_G1, 0x0000f);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_TXPA_G2, 0x07ff7);
+
+	/* PA,PAD gain adjust */
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF);
+	val32 |= BIT(11);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF, val32);
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_56);
+	u32p_replace_bits(&val32, 0x1ed, 0x00fff);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_56, val32);
+
+	/* enter IQK mode */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	u32p_replace_bits(&val32, 0x808000, 0xffffff00);
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+	/* path-A IQK setting */
+	rtl8xxxu_write32(priv, REG_TX_IQK_TONE_A, 0x18008c1c);
+	rtl8xxxu_write32(priv, REG_RX_IQK_TONE_A, 0x38008c1c);
+
+	rtl8xxxu_write32(priv, REG_TX_IQK_PI_A, 0x821403ff);
+	rtl8xxxu_write32(priv, REG_RX_IQK_PI_A, 0x28160c06);
+
+	/* LO calibration setting */
+	rtl8xxxu_write32(priv, REG_IQK_AGC_RSP, 0x02002911);
+
+	/* One shot, path A LOK & IQK */
+	rtl8xxxu_write32(priv, REG_IQK_AGC_PTS, 0xfa000000);
+	rtl8xxxu_write32(priv, REG_IQK_AGC_PTS, 0xf8000000);
+
+	mdelay(10);
+
+	rtl8xxxu_write32(priv, REG_S0S1_PATH_SWITCH, path_sel_bb);
+
+	/*
+	 * Leave IQK mode
+	 */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	u32p_replace_bits(&val32, 0, 0xffffff00);
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF);
+	val32 &= ~BIT(11);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF, val32);
+
+	/* save LOK result */
+	*lok_result = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_TXM_IDAC);
+
+	/* Check failed */
+	reg_eac = rtl8xxxu_read32(priv, REG_RX_POWER_AFTER_IQK_A_2);
+	reg_e94 = rtl8xxxu_read32(priv, REG_TX_POWER_BEFORE_IQK_A);
+	reg_e9c = rtl8xxxu_read32(priv, REG_TX_POWER_AFTER_IQK_A);
+
+	if (!(reg_eac & BIT(28)) &&
+	    ((reg_e94 & 0x03ff0000) != 0x01420000) &&
+	    ((reg_e9c & 0x03ff0000) != 0x00420000))
+		result |= 0x01;
+
+	return result;
+}
+
+static int rtl8710bu_rx_iqk_path_a(struct rtl8xxxu_priv *priv, u32 lok_result)
+{
+	u32 reg_ea4, reg_eac, reg_e94, reg_e9c, val32, path_sel_bb, tmp;
+	int result = 0;
+
+	path_sel_bb = rtl8xxxu_read32(priv, REG_S0S1_PATH_SWITCH);
+
+	rtl8xxxu_write32(priv, REG_S0S1_PATH_SWITCH, 0x99000000);
+
+	/*
+	 * Leave IQK mode
+	 */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	u32p_replace_bits(&val32, 0, 0xffffff00);
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+	/* modify RXIQK mode table */
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_WE_LUT);
+	val32 |= 0x80000;
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_WE_LUT, val32);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_RCK_OS, 0x30000);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_TXPA_G1, 0x0000f);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_TXPA_G2, 0xf1173);
+
+	/* PA,PAD gain adjust */
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF);
+	val32 |= BIT(11);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF, val32);
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_56);
+	u32p_replace_bits(&val32, 0xf, 0x003e0);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_56, val32);
+
+	/*
+	 * Enter IQK mode
+	 */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	u32p_replace_bits(&val32, 0x808000, 0xffffff00);
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+	/* path-A IQK setting */
+	rtl8xxxu_write32(priv, REG_TX_IQK_TONE_A, 0x18008c1c);
+	rtl8xxxu_write32(priv, REG_RX_IQK_TONE_A, 0x38008c1c);
+
+	rtl8xxxu_write32(priv, REG_TX_IQK_PI_A, 0x8216129f);
+	rtl8xxxu_write32(priv, REG_RX_IQK_PI_A, 0x28160c00);
+
+	/*
+	 * Tx IQK setting
+	 */
+	rtl8xxxu_write32(priv, REG_TX_IQK, 0x01007c00);
+	rtl8xxxu_write32(priv, REG_RX_IQK, 0x01004800);
+
+	/* LO calibration setting */
+	rtl8xxxu_write32(priv, REG_IQK_AGC_RSP, 0x0046a911);
+
+	/* One shot, path A LOK & IQK */
+	rtl8xxxu_write32(priv, REG_IQK_AGC_PTS, 0xf9000000);
+	rtl8xxxu_write32(priv, REG_IQK_AGC_PTS, 0xf8000000);
+
+	mdelay(10);
+
+	/* Check failed */
+	reg_eac = rtl8xxxu_read32(priv, REG_RX_POWER_AFTER_IQK_A_2);
+	reg_e94 = rtl8xxxu_read32(priv, REG_TX_POWER_BEFORE_IQK_A);
+	reg_e9c = rtl8xxxu_read32(priv, REG_TX_POWER_AFTER_IQK_A);
+
+	if (!(reg_eac & BIT(28)) &&
+	    ((reg_e94 & 0x03ff0000) != 0x01420000) &&
+	    ((reg_e9c & 0x03ff0000) != 0x00420000)) {
+		result |= 0x01;
+	} else { /* If TX not OK, ignore RX */
+
+		/* reload RF path */
+		rtl8xxxu_write32(priv, REG_S0S1_PATH_SWITCH, path_sel_bb);
+
+		/*
+		 * Leave IQK mode
+		 */
+		val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+		u32p_replace_bits(&val32, 0, 0xffffff00);
+		rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+		val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF);
+		val32 &= ~BIT(11);
+		rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF, val32);
+
+		return result;
+	}
+
+	val32 = 0x80007c00 | (reg_e94 & 0x3ff0000) | ((reg_e9c & 0x3ff0000) >> 16);
+	rtl8xxxu_write32(priv, REG_TX_IQK, val32);
+
+	/*
+	 * Modify RX IQK mode table
+	 */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	u32p_replace_bits(&val32, 0, 0xffffff00);
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_WE_LUT);
+	val32 |= 0x80000;
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_WE_LUT, val32);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_RCK_OS, 0x30000);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_TXPA_G1, 0x0000f);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_TXPA_G2, 0xf7ff2);
+
+	/*
+	 * PA, PAD setting
+	 */
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF);
+	val32 |= BIT(11);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF, val32);
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_56);
+	u32p_replace_bits(&val32, 0x2a, 0x00fff);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_56, val32);
+
+	/*
+	 * Enter IQK mode
+	 */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	u32p_replace_bits(&val32, 0x808000, 0xffffff00);
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+	/*
+	 * RX IQK setting
+	 */
+	rtl8xxxu_write32(priv, REG_RX_IQK, 0x01004800);
+
+	/* path-A IQK setting */
+	rtl8xxxu_write32(priv, REG_TX_IQK_TONE_A, 0x38008c1c);
+	rtl8xxxu_write32(priv, REG_RX_IQK_TONE_A, 0x18008c1c);
+
+	rtl8xxxu_write32(priv, REG_RX_IQK_PI_A, 0x2816169f);
+
+	/* LO calibration setting */
+	rtl8xxxu_write32(priv, REG_IQK_AGC_RSP, 0x0046a911);
+
+	/* One shot, path A LOK & IQK */
+	rtl8xxxu_write32(priv, REG_IQK_AGC_PTS, 0xf9000000);
+	rtl8xxxu_write32(priv, REG_IQK_AGC_PTS, 0xf8000000);
+
+	mdelay(10);
+
+	/* reload RF path */
+	rtl8xxxu_write32(priv, REG_S0S1_PATH_SWITCH, path_sel_bb);
+
+	/*
+	 * Leave IQK mode
+	 */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	u32p_replace_bits(&val32, 0, 0xffffff00);
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+	val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF);
+	val32 &= ~BIT(11);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_UNKNOWN_DF, val32);
+
+	/* reload LOK value */
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_TXM_IDAC, lok_result);
+
+	/* Check failed */
+	reg_eac = rtl8xxxu_read32(priv, REG_RX_POWER_AFTER_IQK_A_2);
+	reg_ea4 = rtl8xxxu_read32(priv, REG_RX_POWER_BEFORE_IQK_A_2);
+
+	tmp = (reg_eac & 0x03ff0000) >> 16;
+	if ((tmp & 0x200) > 0)
+		tmp = 0x400 - tmp;
+
+	if (!(reg_eac & BIT(27)) &&
+	    ((reg_ea4 & 0x03ff0000) != 0x01320000) &&
+	    ((reg_eac & 0x03ff0000) != 0x00360000) &&
+	    (((reg_ea4 & 0x03ff0000) >> 16) < 0x11a) &&
+	    (((reg_ea4 & 0x03ff0000) >> 16) > 0xe6) &&
+	    (tmp < 0x1a))
+		result |= 0x02;
+
+	return result;
+}
+
+static void rtl8710bu_phy_iqcalibrate(struct rtl8xxxu_priv *priv,
+				      int result[][8], int t)
+{
+	struct device *dev = &priv->udev->dev;
+	u32 i, val32, rx_initial_gain, lok_result;
+	u32 path_sel_bb, path_sel_rf;
+	int path_a_ok;
+	int retry = 2;
+	static const u32 adda_regs[RTL8XXXU_ADDA_REGS] = {
+		REG_FPGA0_XCD_SWITCH_CTRL, REG_BLUETOOTH,
+		REG_RX_WAIT_CCA, REG_TX_CCK_RFON,
+		REG_TX_CCK_BBON, REG_TX_OFDM_RFON,
+		REG_TX_OFDM_BBON, REG_TX_TO_RX,
+		REG_TX_TO_TX, REG_RX_CCK,
+		REG_RX_OFDM, REG_RX_WAIT_RIFS,
+		REG_RX_TO_RX, REG_STANDBY,
+		REG_SLEEP, REG_PMPD_ANAEN
+	};
+	static const u32 iqk_mac_regs[RTL8XXXU_MAC_REGS] = {
+		REG_TXPAUSE, REG_BEACON_CTRL,
+		REG_BEACON_CTRL_1, REG_GPIO_MUXCFG
+	};
+	static const u32 iqk_bb_regs[RTL8XXXU_BB_REGS] = {
+		REG_OFDM0_TRX_PATH_ENABLE, REG_OFDM0_TR_MUX_PAR,
+		REG_FPGA0_XCD_RF_SW_CTRL, REG_CONFIG_ANT_A, REG_CONFIG_ANT_B,
+		REG_FPGA0_XAB_RF_SW_CTRL, REG_FPGA0_XA_RF_INT_OE,
+		REG_FPGA0_XB_RF_INT_OE, REG_CCK0_AFE_SETTING
+	};
+
+	/*
+	 * Note: IQ calibration must be performed after loading
+	 *       PHY_REG.txt , and radio_a, radio_b.txt
+	 */
+
+	rx_initial_gain = rtl8xxxu_read32(priv, REG_OFDM0_XA_AGC_CORE1);
+
+	if (t == 0) {
+		/* Save ADDA parameters, turn Path A ADDA on */
+		rtl8xxxu_save_regs(priv, adda_regs, priv->adda_backup,
+				   RTL8XXXU_ADDA_REGS);
+		rtl8xxxu_save_mac_regs(priv, iqk_mac_regs, priv->mac_backup);
+		rtl8xxxu_save_regs(priv, iqk_bb_regs,
+				   priv->bb_backup, RTL8XXXU_BB_REGS);
+	}
+
+	rtl8xxxu_path_adda_on(priv, adda_regs, true);
+
+	if (t == 0) {
+		val32 = rtl8xxxu_read32(priv, REG_FPGA0_XA_HSSI_PARM1);
+		priv->pi_enabled = u32_get_bits(val32, FPGA0_HSSI_PARM1_PI);
+	}
+
+	if (!priv->pi_enabled) {
+		/* Switch BB to PI mode to do IQ Calibration */
+		rtl8xxxu_write32(priv, REG_FPGA0_XA_HSSI_PARM1, 0x01000100);
+		rtl8xxxu_write32(priv, REG_FPGA0_XB_HSSI_PARM1, 0x01000100);
+	}
+
+	/* MAC settings */
+	val32 = rtl8xxxu_read32(priv, REG_TX_PTCL_CTRL);
+	val32 |= 0x00ff0000;
+	rtl8xxxu_write32(priv, REG_TX_PTCL_CTRL, val32);
+
+	/* save RF path */
+	path_sel_bb = rtl8xxxu_read32(priv, REG_S0S1_PATH_SWITCH);
+	path_sel_rf = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_S0S1);
+
+	/* BB setting */
+	val32 = rtl8xxxu_read32(priv, REG_CCK0_AFE_SETTING);
+	val32 |= 0x0f000000;
+	rtl8xxxu_write32(priv, REG_CCK0_AFE_SETTING, val32);
+	rtl8xxxu_write32(priv, REG_RX_WAIT_CCA, 0x03c00010);
+	rtl8xxxu_write32(priv, REG_OFDM0_TRX_PATH_ENABLE, 0x03a05601);
+	rtl8xxxu_write32(priv, REG_OFDM0_TR_MUX_PAR, 0x000800e4);
+	rtl8xxxu_write32(priv, REG_FPGA0_XCD_RF_SW_CTRL, 0x25204000);
+
+	/* IQ calibration setting */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	u32p_replace_bits(&val32, 0x808000, 0xffffff00);
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+	rtl8xxxu_write32(priv, REG_TX_IQK, 0x01007c00);
+	rtl8xxxu_write32(priv, REG_RX_IQK, 0x01004800);
+
+	for (i = 0; i < retry; i++) {
+		path_a_ok = rtl8710bu_iqk_path_a(priv, &lok_result);
+
+		if (path_a_ok == 0x01) {
+			val32 = rtl8xxxu_read32(priv, REG_TX_POWER_BEFORE_IQK_A);
+			result[t][0] = (val32 >> 16) & 0x3ff;
+
+			val32 = rtl8xxxu_read32(priv, REG_TX_POWER_AFTER_IQK_A);
+			result[t][1] = (val32 >> 16) & 0x3ff;
+			break;
+		} else {
+			result[t][0] = 0x100;
+			result[t][1] = 0x0;
+		}
+	}
+
+	for (i = 0; i < retry; i++) {
+		path_a_ok = rtl8710bu_rx_iqk_path_a(priv, lok_result);
+
+		if (path_a_ok == 0x03) {
+			val32 = rtl8xxxu_read32(priv, REG_RX_POWER_BEFORE_IQK_A_2);
+			result[t][2] = (val32 >> 16) & 0x3ff;
+
+			val32 = rtl8xxxu_read32(priv, REG_RX_POWER_AFTER_IQK_A_2);
+			result[t][3] = (val32 >> 16) & 0x3ff;
+			break;
+		} else {
+			result[t][2] = 0x100;
+			result[t][3] = 0x0;
+		}
+	}
+
+	if (!path_a_ok)
+		dev_warn(dev, "%s: Path A IQK failed!\n", __func__);
+
+	/* Back to BB mode, load original value */
+	val32 = rtl8xxxu_read32(priv, REG_FPGA0_IQK);
+	u32p_replace_bits(&val32, 0, 0xffffff00);
+	rtl8xxxu_write32(priv, REG_FPGA0_IQK, val32);
+
+	if (t == 0)
+		return;
+
+	/* Reload ADDA power saving parameters */
+	rtl8xxxu_restore_regs(priv, adda_regs, priv->adda_backup, RTL8XXXU_ADDA_REGS);
+
+	/* Reload MAC parameters */
+	rtl8xxxu_restore_mac_regs(priv, iqk_mac_regs, priv->mac_backup);
+
+	/* Reload BB parameters */
+	rtl8xxxu_restore_regs(priv, iqk_bb_regs, priv->bb_backup, RTL8XXXU_BB_REGS);
+
+	/* Reload RF path */
+	rtl8xxxu_write32(priv, REG_S0S1_PATH_SWITCH, path_sel_bb);
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_S0S1, path_sel_rf);
+
+	/* Restore RX initial gain */
+	val32 = rtl8xxxu_read32(priv, REG_OFDM0_XA_AGC_CORE1);
+	u32p_replace_bits(&val32, 0x50, 0x000000ff);
+	rtl8xxxu_write32(priv, REG_OFDM0_XA_AGC_CORE1, val32);
+	val32 = rtl8xxxu_read32(priv, REG_OFDM0_XA_AGC_CORE1);
+	u32p_replace_bits(&val32, rx_initial_gain & 0xff, 0x000000ff);
+	rtl8xxxu_write32(priv, REG_OFDM0_XA_AGC_CORE1, val32);
+
+	/* Load 0xe30 IQC default value */
+	rtl8xxxu_write32(priv, REG_TX_IQK_TONE_A, 0x01008c00);
+	rtl8xxxu_write32(priv, REG_RX_IQK_TONE_A, 0x01008c00);
+}
+
+static void rtl8710bu_phy_iq_calibrate(struct rtl8xxxu_priv *priv)
+{
+	struct device *dev = &priv->udev->dev;
+	int result[4][8]; /* last is final result */
+	int i, candidate;
+	bool path_a_ok;
+	s32 reg_e94, reg_e9c, reg_ea4, reg_eac;
+	s32 reg_tmp = 0;
+	bool simu;
+	u32 path_sel_bb;
+
+	/* Save RF path */
+	path_sel_bb = rtl8xxxu_read32(priv, REG_S0S1_PATH_SWITCH);
+
+	memset(result, 0, sizeof(result));
+	candidate = -1;
+
+	path_a_ok = false;
+
+	for (i = 0; i < 3; i++) {
+		rtl8710bu_phy_iqcalibrate(priv, result, i);
+
+		if (i == 1) {
+			simu = rtl8xxxu_gen2_simularity_compare(priv, result, 0, 1);
+			if (simu) {
+				candidate = 0;
+				break;
+			}
+		}
+
+		if (i == 2) {
+			simu = rtl8xxxu_gen2_simularity_compare(priv, result, 0, 2);
+			if (simu) {
+				candidate = 0;
+				break;
+			}
+
+			simu = rtl8xxxu_gen2_simularity_compare(priv, result, 1, 2);
+			if (simu) {
+				candidate = 1;
+			} else {
+				for (i = 0; i < 8; i++)
+					reg_tmp += result[3][i];
+
+				if (reg_tmp)
+					candidate = 3;
+				else
+					candidate = -1;
+			}
+		}
+	}
+
+	if (candidate >= 0) {
+		reg_e94 = result[candidate][0];
+		reg_e9c = result[candidate][1];
+		reg_ea4 = result[candidate][2];
+		reg_eac = result[candidate][3];
+
+		dev_dbg(dev, "%s: candidate is %x\n", __func__, candidate);
+		dev_dbg(dev, "%s: e94=%x e9c=%x ea4=%x eac=%x\n",
+			__func__, reg_e94, reg_e9c, reg_ea4, reg_eac);
+
+		path_a_ok = true;
+
+		if (reg_e94)
+			rtl8xxxu_fill_iqk_matrix_a(priv, path_a_ok, result,
+						   candidate, (reg_ea4 == 0));
+	}
+
+	rtl8xxxu_save_regs(priv, rtl8xxxu_iqk_phy_iq_bb_reg,
+			   priv->bb_recovery_backup, RTL8XXXU_BB_REGS);
+
+	rtl8xxxu_write32(priv, REG_S0S1_PATH_SWITCH, path_sel_bb);
+}
+
+static int rtl8710b_emu_to_active(struct rtl8xxxu_priv *priv)
+{
+	u8 val8;
+	int count, ret = 0;
+
+	/* AFE power mode selection: 1: LDO mode, 0: Power-cut mode */
+	val8 = rtl8xxxu_read8(priv, 0x5d);
+	val8 &= ~BIT(0);
+	rtl8xxxu_write8(priv, 0x5d, val8);
+
+	val8 = rtl8xxxu_read8(priv, REG_SYS_FUNC_8710B);
+	val8 |= BIT(0);
+	rtl8xxxu_write8(priv, REG_SYS_FUNC_8710B, val8);
+
+	rtl8xxxu_write8(priv, 0x56, 0x0e);
+
+	val8 = rtl8xxxu_read8(priv, 0x20);
+	val8 |= BIT(0);
+	rtl8xxxu_write8(priv, 0x20, val8);
+
+	for (count = RTL8XXXU_MAX_REG_POLL; count; count--) {
+		val8 = rtl8xxxu_read8(priv, 0x20);
+		if (!(val8 & BIT(0)))
+			break;
+
+		udelay(10);
+	}
+
+	if (!count)
+		ret = -EBUSY;
+
+	return ret;
+}
+
+static int rtl8710bu_active_to_emu(struct rtl8xxxu_priv *priv)
+{
+	u8 val8;
+	u32 val32;
+	int count, ret = 0;
+
+	/* Turn off RF */
+	val32 = rtl8xxxu_read32(priv, REG_SYS_FUNC_8710B);
+	val32 &= ~GENMASK(26, 24);
+	rtl8xxxu_write32(priv, REG_SYS_FUNC_8710B, val32);
+
+	/* BB reset */
+	val32 = rtl8xxxu_read32(priv, REG_SYS_FUNC_8710B);
+	val32 &= ~GENMASK(17, 16);
+	rtl8xxxu_write32(priv, REG_SYS_FUNC_8710B, val32);
+
+	/* Turn off MAC by HW state machine */
+	val8 = rtl8xxxu_read8(priv, 0x20);
+	val8 |= BIT(1);
+	rtl8xxxu_write8(priv, 0x20, val8);
+
+	for (count = RTL8XXXU_MAX_REG_POLL; count; count--) {
+		val8 = rtl8xxxu_read8(priv, 0x20);
+		if ((val8 & BIT(1)) == 0) {
+			ret = 0;
+			break;
+		}
+		udelay(10);
+	}
+
+	if (!count)
+		ret = -EBUSY;
+
+	return ret;
+}
+
+static int rtl8710bu_active_to_lps(struct rtl8xxxu_priv *priv)
+{
+	struct device *dev = &priv->udev->dev;
+	u8 val8;
+	u16 val16;
+	u32 val32;
+	int retry, retval;
+
+	/* Tx Pause */
+	rtl8xxxu_write8(priv, REG_TXPAUSE, 0xff);
+
+	retry = 100;
+	retval = -EBUSY;
+	/*
+	 * Poll 32 bit wide REG_SCH_TX_CMD for 0x00000000 to ensure no TX is pending.
+	 */
+	do {
+		val32 = rtl8xxxu_read32(priv, REG_SCH_TX_CMD);
+		if (!val32) {
+			retval = 0;
+			break;
+		}
+		udelay(10);
+	} while (retry--);
+
+	if (!retry) {
+		dev_warn(dev, "Failed to flush TX queue\n");
+		retval = -EBUSY;
+		return retval;
+	}
+
+	/* Disable CCK and OFDM, clock gated */
+	val8 = rtl8xxxu_read8(priv, REG_SYS_FUNC);
+	val8 &= ~SYS_FUNC_BBRSTB;
+	rtl8xxxu_write8(priv, REG_SYS_FUNC, val8);
+
+	udelay(2);
+
+	/* Whole BB is reset */
+	val8 = rtl8xxxu_read8(priv, REG_SYS_FUNC);
+	val8 &= ~SYS_FUNC_BB_GLB_RSTN;
+	rtl8xxxu_write8(priv, REG_SYS_FUNC, val8);
+
+	/* Reset MAC TRX */
+	val16 = rtl8xxxu_read16(priv, REG_CR);
+	val16 &= 0xff00;
+	val16 |= CR_HCI_RXDMA_ENABLE | CR_HCI_TXDMA_ENABLE;
+	val16 &= ~CR_SECURITY_ENABLE;
+	rtl8xxxu_write16(priv, REG_CR, val16);
+
+	/* Respond TxOK to scheduler */
+	val8 = rtl8xxxu_read8(priv, REG_DUAL_TSF_RST);
+	val8 |= DUAL_TSF_TX_OK;
+	rtl8xxxu_write8(priv, REG_DUAL_TSF_RST, val8);
+
+	return retval;
+}
+
+static int rtl8710bu_power_on(struct rtl8xxxu_priv *priv)
+{
+	u32 val32;
+	u16 val16;
+	u8 val8;
+	int ret;
+
+	rtl8xxxu_write8(priv, REG_USB_ACCESS_TIMEOUT, 0x80);
+
+	val8 = rtl8xxxu_read8(priv, REG_SYS_ISO_CTRL);
+	val8 &= ~BIT(5);
+	rtl8xxxu_write8(priv, REG_SYS_ISO_CTRL, val8);
+
+	val8 = rtl8xxxu_read8(priv, REG_SYS_FUNC_8710B);
+	val8 |= BIT(0);
+	rtl8xxxu_write8(priv, REG_SYS_FUNC_8710B, val8);
+
+	val8 = rtl8xxxu_read8(priv, 0x20);
+	val8 |= BIT(0);
+	rtl8xxxu_write8(priv, 0x20, val8);
+
+	rtl8xxxu_write8(priv, REG_AFE_CTRL_8710B, 0);
+
+	val8 = rtl8xxxu_read8(priv, REG_WL_STATUS_8710B);
+	val8 |= BIT(1);
+	rtl8xxxu_write8(priv, REG_WL_STATUS_8710B, val8);
+
+	ret = rtl8710b_emu_to_active(priv);
+	if (ret)
+		return ret;
+
+	rtl8xxxu_write16(priv, REG_CR, 0);
+
+	val16 = rtl8xxxu_read16(priv, REG_CR);
+
+	val16 |= CR_HCI_TXDMA_ENABLE | CR_HCI_RXDMA_ENABLE |
+		 CR_TXDMA_ENABLE | CR_RXDMA_ENABLE |
+		 CR_PROTOCOL_ENABLE | CR_SCHEDULE_ENABLE |
+		 CR_SECURITY_ENABLE | CR_CALTIMER_ENABLE;
+	rtl8xxxu_write16(priv, REG_CR, val16);
+
+	/* Enable hardware sequence number. */
+	val8 = rtl8xxxu_read8(priv, REG_HWSEQ_CTRL);
+	val8 |= 0x7f;
+	rtl8xxxu_write8(priv, REG_HWSEQ_CTRL, val8);
+
+	udelay(2);
+
+	/*
+	 * Technically the rest was in the rtl8710bu_hal_init function,
+	 * not the power_on function, but it's fine because we only
+	 * call power_on from init_device.
+	 */
+
+	val8 = rtl8xxxu_read8(priv, 0xfef9);
+	val8 &= ~BIT(0);
+	rtl8xxxu_write8(priv, 0xfef9, val8);
+
+	/* Clear the 0x40000138[5] to prevent CM4 Suspend */
+	val32 = rtl8710b_read_syson_reg(priv, 0x138);
+	val32 &= ~BIT(5);
+	rtl8710b_write_syson_reg(priv, 0x138, val32);
+
+	return ret;
+}
+
+static void rtl8710bu_power_off(struct rtl8xxxu_priv *priv)
+{
+	u32 val32;
+	u8 val8;
+
+	rtl8xxxu_flush_fifo(priv);
+
+	rtl8xxxu_write32(priv, REG_HISR0_8710B, 0xffffffff);
+	rtl8xxxu_write32(priv, REG_HIMR0_8710B, 0x0);
+
+	/* Set the 0x40000138[5] to allow CM4 Suspend */
+	val32 = rtl8710b_read_syson_reg(priv, 0x138);
+	val32 |= BIT(5);
+	rtl8710b_write_syson_reg(priv, 0x138, val32);
+
+	/* Stop rx */
+	rtl8xxxu_write8(priv, REG_CR, 0x00);
+
+	rtl8710bu_active_to_lps(priv);
+
+	/* Reset MCU ? */
+	val8 = rtl8xxxu_read8(priv, REG_8051FW_CTRL_V1_8710B + 3);
+	val8 &= ~BIT(0);
+	rtl8xxxu_write8(priv, REG_8051FW_CTRL_V1_8710B + 3, val8);
+
+	/* Reset MCU ready status */
+	rtl8xxxu_write8(priv, REG_8051FW_CTRL_V1_8710B, 0x00);
+
+	rtl8710bu_active_to_emu(priv);
+}
+
+static void rtl8710b_reset_8051(struct rtl8xxxu_priv *priv)
+{
+	u8 val8;
+
+	val8 = rtl8xxxu_read8(priv, REG_8051FW_CTRL_V1_8710B + 3);
+	val8 &= ~BIT(0);
+	rtl8xxxu_write8(priv, REG_8051FW_CTRL_V1_8710B + 3, val8);
+
+	udelay(50);
+
+	val8 = rtl8xxxu_read8(priv, REG_8051FW_CTRL_V1_8710B + 3);
+	val8 |= BIT(0);
+	rtl8xxxu_write8(priv, REG_8051FW_CTRL_V1_8710B + 3, val8);
+}
+
+static void rtl8710b_enable_rf(struct rtl8xxxu_priv *priv)
+{
+	u32 val32;
+
+	rtl8xxxu_write8(priv, REG_RF_CTRL, RF_ENABLE | RF_RSTB | RF_SDMRSTB);
+
+	val32 = rtl8xxxu_read32(priv, REG_OFDM0_TRX_PATH_ENABLE);
+	val32 &= ~(OFDM_RF_PATH_RX_MASK | OFDM_RF_PATH_TX_MASK);
+	val32 |= OFDM_RF_PATH_RX_A | OFDM_RF_PATH_TX_A;
+	rtl8xxxu_write32(priv, REG_OFDM0_TRX_PATH_ENABLE, val32);
+
+	rtl8xxxu_write8(priv, REG_TXPAUSE, 0x00);
+}
+
+static void rtl8710b_disable_rf(struct rtl8xxxu_priv *priv)
+{
+	u32 val32;
+
+	val32 = rtl8xxxu_read32(priv, REG_OFDM0_TRX_PATH_ENABLE);
+	val32 &= ~OFDM_RF_PATH_TX_MASK;
+	rtl8xxxu_write32(priv, REG_OFDM0_TRX_PATH_ENABLE, val32);
+
+	/* Power down RF module */
+	rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_AC, 0);
+}
+
+static void rtl8710b_usb_quirks(struct rtl8xxxu_priv *priv)
+{
+	u16 val16;
+
+	rtl8xxxu_gen2_usb_quirks(priv);
+
+	val16 = rtl8xxxu_read16(priv, REG_CR);
+	val16 |= (CR_MAC_TX_ENABLE | CR_MAC_RX_ENABLE);
+	rtl8xxxu_write16(priv, REG_CR, val16);
+}
+
+#define XTAL1	GENMASK(29, 24)
+#define XTAL0	GENMASK(23, 18)
+
+static void rtl8710b_set_crystal_cap(struct rtl8xxxu_priv *priv, u8 crystal_cap)
+{
+	struct rtl8xxxu_cfo_tracking *cfo = &priv->cfo_tracking;
+	u32 val32;
+
+	if (crystal_cap == cfo->crystal_cap)
+		return;
+
+	val32 = rtl8710b_read_syson_reg(priv, REG_SYS_XTAL_CTRL0_8710B);
+
+	dev_dbg(&priv->udev->dev,
+		"%s: Adjusting crystal cap from 0x%x (actually 0x%x 0x%x) to 0x%x\n",
+		__func__,
+		cfo->crystal_cap,
+		u32_get_bits(val32, XTAL1),
+		u32_get_bits(val32, XTAL0),
+		crystal_cap);
+
+	u32p_replace_bits(&val32, crystal_cap, XTAL1);
+	u32p_replace_bits(&val32, crystal_cap, XTAL0);
+	rtl8710b_write_syson_reg(priv, REG_SYS_XTAL_CTRL0_8710B, val32);
+
+	cfo->crystal_cap = crystal_cap;
+}
+
+static s8 rtl8710b_cck_rssi(struct rtl8xxxu_priv *priv, struct rtl8723au_phy_stats *phy_stats)
+{
+	struct jaguar2_phy_stats_type0 *phy_stats0 = (struct jaguar2_phy_stats_type0 *)phy_stats;
+	u8 lna_idx = (phy_stats0->lna_h << 3) | phy_stats0->lna_l;
+	u8 vga_idx = phy_stats0->vga;
+	s8 rx_pwr_all = 0x00;
+
+	switch (lna_idx) {
+	case 7:
+		rx_pwr_all = -52 - (2 * vga_idx);
+		break;
+	case 6:
+		rx_pwr_all = -42 - (2 * vga_idx);
+		break;
+	case 5:
+		rx_pwr_all = -36 - (2 * vga_idx);
+		break;
+	case 3:
+		rx_pwr_all = -12 - (2 * vga_idx);
+		break;
+	case 2:
+		rx_pwr_all = 0 - (2 * vga_idx);
+		break;
+	default:
+		rx_pwr_all = 0;
+		break;
+	}
+
+	return rx_pwr_all;
+}
+
+struct rtl8xxxu_fileops rtl8710bu_fops = {
+	.identify_chip = rtl8710bu_identify_chip,
+	.parse_efuse = rtl8710bu_parse_efuse,
+	.load_firmware = rtl8710bu_load_firmware,
+	.power_on = rtl8710bu_power_on,
+	.power_off = rtl8710bu_power_off,
+	.read_efuse = rtl8710b_read_efuse,
+	.reset_8051 = rtl8710b_reset_8051,
+	.llt_init = rtl8xxxu_auto_llt_table,
+	.init_phy_bb = rtl8710bu_init_phy_bb,
+	.init_phy_rf = rtl8710bu_init_phy_rf,
+	.phy_lc_calibrate = rtl8188f_phy_lc_calibrate,
+	.phy_iq_calibrate = rtl8710bu_phy_iq_calibrate,
+	.config_channel = rtl8710bu_config_channel,
+	.parse_rx_desc = rtl8xxxu_parse_rxdesc24,
+	.parse_phystats = jaguar2_rx_parse_phystats,
+	.init_aggregation = rtl8710bu_init_aggregation,
+	.init_statistics = rtl8710bu_init_statistics,
+	.init_burst = rtl8xxxu_init_burst,
+	.enable_rf = rtl8710b_enable_rf,
+	.disable_rf = rtl8710b_disable_rf,
+	.usb_quirks = rtl8710b_usb_quirks,
+	.set_tx_power = rtl8188f_set_tx_power,
+	.update_rate_mask = rtl8xxxu_gen2_update_rate_mask,
+	.report_connect = rtl8xxxu_gen2_report_connect,
+	.report_rssi = rtl8xxxu_gen2_report_rssi,
+	.fill_txdesc = rtl8xxxu_fill_txdesc_v2,
+	.set_crystal_cap = rtl8710b_set_crystal_cap,
+	.cck_rssi = rtl8710b_cck_rssi,
+	.writeN_block_size = 4,
+	.rx_desc_size = sizeof(struct rtl8xxxu_rxdesc24),
+	.tx_desc_size = sizeof(struct rtl8xxxu_txdesc40),
+	.has_tx_report = 1,
+	.gen2_thermal_meter = 1,
+	.needs_full_init = 1,
+	.init_reg_rxfltmap = 1,
+	.init_reg_pkt_life_time = 1,
+	.init_reg_hmtfr = 1,
+	.ampdu_max_time = 0x5e,
+	/*
+	 * The RTL8710BU vendor driver uses 0x50 here and it works fine,
+	 * but in rtl8xxxu 0x50 causes slow upload and random packet loss. Why?
+	 */
+	.ustime_tsf_edca = 0x28,
+	.adda_1t_init = 0x03c00016,
+	.adda_1t_path_on = 0x03c00016,
+	.trxff_boundary = 0x3f7f,
+	.pbp_rx = PBP_PAGE_SIZE_256,
+	.pbp_tx = PBP_PAGE_SIZE_256,
+	.mactable = rtl8710b_mac_init_table,
+	.total_page_num = TX_TOTAL_PAGE_NUM_8723B,
+	.page_num_hi = TX_PAGE_NUM_HI_PQ_8723B,
+	.page_num_lo = TX_PAGE_NUM_LO_PQ_8723B,
+	.page_num_norm = TX_PAGE_NUM_NORM_PQ_8723B,
+};
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723a.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723a.c
index 5e7b58d395ba..15a30e496221 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723a.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723a.c
@@ -222,10 +222,6 @@ static int rtl8723au_parse_efuse(struct rtl8xxxu_priv *priv)
 
 	priv->power_base = &rtl8723a_power_base;
 
-	dev_info(&priv->udev->dev, "Vendor: %.7s\n",
-		 efuse->vendor_name);
-	dev_info(&priv->udev->dev, "Product: %.41s\n",
-		 efuse->device_name);
 	return 0;
 }
 
@@ -435,8 +431,9 @@ void rtl8723a_set_crystal_cap(struct rtl8xxxu_priv *priv, u8 crystal_cap)
 	cfo->crystal_cap = crystal_cap;
 }
 
-s8 rtl8723a_cck_rssi(struct rtl8xxxu_priv *priv, u8 cck_agc_rpt)
+s8 rtl8723a_cck_rssi(struct rtl8xxxu_priv *priv, struct rtl8723au_phy_stats *phy_stats)
 {
+	u8 cck_agc_rpt = phy_stats->cck_agc_rpt_ofdm_cfosho_a;
 	s8 rx_pwr_all = 0x00;
 
 	switch (cck_agc_rpt & 0xc0) {
@@ -487,6 +484,7 @@ struct rtl8xxxu_fileops rtl8723au_fops = {
 	.load_firmware = rtl8723au_load_firmware,
 	.power_on = rtl8723au_power_on,
 	.power_off = rtl8xxxu_power_off,
+	.read_efuse = rtl8xxxu_read_efuse,
 	.reset_8051 = rtl8xxxu_reset_8051,
 	.llt_init = rtl8xxxu_init_llt_table,
 	.init_phy_bb = rtl8xxxu_gen1_init_phy_bb,
@@ -495,6 +493,7 @@ struct rtl8xxxu_fileops rtl8723au_fops = {
 	.phy_iq_calibrate = rtl8xxxu_gen1_phy_iq_calibrate,
 	.config_channel = rtl8xxxu_gen1_config_channel,
 	.parse_rx_desc = rtl8xxxu_parse_rxdesc16,
+	.parse_phystats = rtl8723au_rx_parse_phystats,
 	.init_aggregation = rtl8xxxu_gen1_init_aggregation,
 	.enable_rf = rtl8xxxu_gen1_enable_rf,
 	.disable_rf = rtl8xxxu_gen1_disable_rf,
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c
index 21613d60dc22..abc56c7de6f7 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c
@@ -494,9 +494,6 @@ static int rtl8723bu_parse_efuse(struct rtl8xxxu_priv *priv)
 
 	priv->default_crystal_cap = priv->efuse_wifi.efuse8723bu.xtal_k & 0x3f;
 
-	dev_info(&priv->udev->dev, "Vendor: %.7s\n", efuse->vendor_name);
-	dev_info(&priv->udev->dev, "Product: %.41s\n", efuse->device_name);
-
 	return 0;
 }
 
@@ -1675,8 +1672,9 @@ static void rtl8723bu_init_statistics(struct rtl8xxxu_priv *priv)
 	rtl8xxxu_write32(priv, REG_OFDM0_FA_RSTC, val32);
 }
 
-static s8 rtl8723b_cck_rssi(struct rtl8xxxu_priv *priv, u8 cck_agc_rpt)
+static s8 rtl8723b_cck_rssi(struct rtl8xxxu_priv *priv, struct rtl8723au_phy_stats *phy_stats)
 {
+	u8 cck_agc_rpt = phy_stats->cck_agc_rpt_ofdm_cfosho_a;
 	s8 rx_pwr_all = 0x00;
 	u8 vga_idx, lna_idx;
 
@@ -1709,6 +1707,7 @@ struct rtl8xxxu_fileops rtl8723bu_fops = {
 	.load_firmware = rtl8723bu_load_firmware,
 	.power_on = rtl8723bu_power_on,
 	.power_off = rtl8723bu_power_off,
+	.read_efuse = rtl8xxxu_read_efuse,
 	.reset_8051 = rtl8723bu_reset_8051,
 	.llt_init = rtl8xxxu_auto_llt_table,
 	.init_phy_bb = rtl8723bu_init_phy_bb,
@@ -1718,6 +1717,7 @@ struct rtl8xxxu_fileops rtl8723bu_fops = {
 	.phy_iq_calibrate = rtl8723bu_phy_iq_calibrate,
 	.config_channel = rtl8xxxu_gen2_config_channel,
 	.parse_rx_desc = rtl8xxxu_parse_rxdesc24,
+	.parse_phystats = rtl8723au_rx_parse_phystats,
 	.init_aggregation = rtl8723bu_init_aggregation,
 	.init_statistics = rtl8723bu_init_statistics,
 	.init_burst = rtl8xxxu_init_burst,
@@ -1738,6 +1738,9 @@ struct rtl8xxxu_fileops rtl8723bu_fops = {
 	.has_tx_report = 1,
 	.gen2_thermal_meter = 1,
 	.needs_full_init = 1,
+	.init_reg_hmtfr = 1,
+	.ampdu_max_time = 0x5e,
+	.ustime_tsf_edca = 0x50,
 	.adda_1t_init = 0x01c00014,
 	.adda_1t_path_on = 0x01c00014,
 	.adda_2t_path_on_a = 0x01c00014,
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
index 620a5cc2bfdd..fd8c8c6d53d6 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
@@ -54,6 +54,8 @@ MODULE_FIRMWARE("rtlwifi/rtl8192eu_nic.bin");
 MODULE_FIRMWARE("rtlwifi/rtl8723bu_nic.bin");
 MODULE_FIRMWARE("rtlwifi/rtl8723bu_bt.bin");
 MODULE_FIRMWARE("rtlwifi/rtl8188fufw.bin");
+MODULE_FIRMWARE("rtlwifi/rtl8710bufw_SMIC.bin");
+MODULE_FIRMWARE("rtlwifi/rtl8710bufw_UMC.bin");
 
 module_param_named(debug, rtl8xxxu_debug, int, 0600);
 MODULE_PARM_DESC(debug, "Set debug mask");
@@ -654,6 +656,9 @@ u8 rtl8xxxu_read8(struct rtl8xxxu_priv *priv, u16 addr)
 	int len;
 	u8 data;
 
+	if (priv->rtl_chip == RTL8710B && addr <= 0xff)
+		addr |= 0x8000;
+
 	mutex_lock(&priv->usb_buf_mutex);
 	len = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
 			      REALTEK_USB_CMD_REQ, REALTEK_USB_READ,
@@ -674,6 +679,9 @@ u16 rtl8xxxu_read16(struct rtl8xxxu_priv *priv, u16 addr)
 	int len;
 	u16 data;
 
+	if (priv->rtl_chip == RTL8710B && addr <= 0xff)
+		addr |= 0x8000;
+
 	mutex_lock(&priv->usb_buf_mutex);
 	len = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
 			      REALTEK_USB_CMD_REQ, REALTEK_USB_READ,
@@ -694,6 +702,9 @@ u32 rtl8xxxu_read32(struct rtl8xxxu_priv *priv, u16 addr)
 	int len;
 	u32 data;
 
+	if (priv->rtl_chip == RTL8710B && addr <= 0xff)
+		addr |= 0x8000;
+
 	mutex_lock(&priv->usb_buf_mutex);
 	len = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
 			      REALTEK_USB_CMD_REQ, REALTEK_USB_READ,
@@ -713,6 +724,9 @@ int rtl8xxxu_write8(struct rtl8xxxu_priv *priv, u16 addr, u8 val)
 	struct usb_device *udev = priv->udev;
 	int ret;
 
+	if (priv->rtl_chip == RTL8710B && addr <= 0xff)
+		addr |= 0x8000;
+
 	mutex_lock(&priv->usb_buf_mutex);
 	priv->usb_buf.val8 = val;
 	ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
@@ -733,6 +747,9 @@ int rtl8xxxu_write16(struct rtl8xxxu_priv *priv, u16 addr, u16 val)
 	struct usb_device *udev = priv->udev;
 	int ret;
 
+	if (priv->rtl_chip == RTL8710B && addr <= 0xff)
+		addr |= 0x8000;
+
 	mutex_lock(&priv->usb_buf_mutex);
 	priv->usb_buf.val16 = cpu_to_le16(val);
 	ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
@@ -752,6 +769,9 @@ int rtl8xxxu_write32(struct rtl8xxxu_priv *priv, u16 addr, u32 val)
 	struct usb_device *udev = priv->udev;
 	int ret;
 
+	if (priv->rtl_chip == RTL8710B && addr <= 0xff)
+		addr |= 0x8000;
+
 	mutex_lock(&priv->usb_buf_mutex);
 	priv->usb_buf.val32 = cpu_to_le32(val);
 	ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
@@ -766,6 +786,85 @@ int rtl8xxxu_write32(struct rtl8xxxu_priv *priv, u16 addr, u32 val)
 	return ret;
 }
 
+int rtl8xxxu_write8_set(struct rtl8xxxu_priv *priv, u16 addr, u8 bits)
+{
+	u8 val8;
+
+	val8 = rtl8xxxu_read8(priv, addr);
+	val8 |= bits;
+	return rtl8xxxu_write8(priv, addr, val8);
+}
+
+int rtl8xxxu_write8_clear(struct rtl8xxxu_priv *priv, u16 addr, u8 bits)
+{
+	u8 val8;
+
+	val8 = rtl8xxxu_read8(priv, addr);
+	val8 &= ~bits;
+	return rtl8xxxu_write8(priv, addr, val8);
+}
+
+int rtl8xxxu_write16_set(struct rtl8xxxu_priv *priv, u16 addr, u16 bits)
+{
+	u16 val16;
+
+	val16 = rtl8xxxu_read16(priv, addr);
+	val16 |= bits;
+	return rtl8xxxu_write16(priv, addr, val16);
+}
+
+int rtl8xxxu_write16_clear(struct rtl8xxxu_priv *priv, u16 addr, u16 bits)
+{
+	u16 val16;
+
+	val16 = rtl8xxxu_read16(priv, addr);
+	val16 &= ~bits;
+	return rtl8xxxu_write16(priv, addr, val16);
+}
+
+int rtl8xxxu_write32_set(struct rtl8xxxu_priv *priv, u16 addr, u32 bits)
+{
+	u32 val32;
+
+	val32 = rtl8xxxu_read32(priv, addr);
+	val32 |= bits;
+	return rtl8xxxu_write32(priv, addr, val32);
+}
+
+int rtl8xxxu_write32_clear(struct rtl8xxxu_priv *priv, u16 addr, u32 bits)
+{
+	u32 val32;
+
+	val32 = rtl8xxxu_read32(priv, addr);
+	val32 &= ~bits;
+	return rtl8xxxu_write32(priv, addr, val32);
+}
+
+int rtl8xxxu_write32_mask(struct rtl8xxxu_priv *priv, u16 addr,
+			  u32 mask, u32 val)
+{
+	u32 orig, new, shift;
+
+	shift = __ffs(mask);
+
+	orig = rtl8xxxu_read32(priv, addr);
+	new = (orig & ~mask) | ((val << shift) & mask);
+	return rtl8xxxu_write32(priv, addr, new);
+}
+
+int rtl8xxxu_write_rfreg_mask(struct rtl8xxxu_priv *priv,
+			      enum rtl8xxxu_rfpath path, u8 reg,
+			      u32 mask, u32 val)
+{
+	u32 orig, new, shift;
+
+	shift = __ffs(mask);
+
+	orig = rtl8xxxu_read_rfreg(priv, path, reg);
+	new = (orig & ~mask) | ((val << shift) & mask);
+	return rtl8xxxu_write_rfreg(priv, path, reg, new);
+}
+
 static int
 rtl8xxxu_writeN(struct rtl8xxxu_priv *priv, u16 addr, u8 *buf, u16 len)
 {
@@ -1575,11 +1674,7 @@ rtl8xxxu_set_spec_sifs(struct rtl8xxxu_priv *priv, u16 cck, u16 ofdm)
 static void rtl8xxxu_print_chipinfo(struct rtl8xxxu_priv *priv)
 {
 	struct device *dev = &priv->udev->dev;
-	char cut = '?';
-
-	/* Currently always true: chip_cut is 4 bits. */
-	if (priv->chip_cut <= 15)
-		cut = 'A' + priv->chip_cut;
+	char cut = 'A' + priv->chip_cut;
 
 	dev_info(dev,
 		 "RTL%s rev %c (%s) romver %d, %iT%iR, TX queues %i, WiFi=%i, BT=%i, GPS=%i, HI PA=%i\n",
@@ -1647,6 +1742,8 @@ int rtl8xxxu_config_endpoints_no_sie(struct rtl8xxxu_priv *priv)
 	struct device *dev = &priv->udev->dev;
 
 	switch (priv->nr_out_eps) {
+	case 6:
+	case 5:
 	case 4:
 	case 3:
 		priv->ep_tx_low_queue = 1;
@@ -1703,7 +1800,7 @@ rtl8xxxu_read_efuse8(struct rtl8xxxu_priv *priv, u16 offset, u8 *data)
 	return 0;
 }
 
-static int rtl8xxxu_read_efuse(struct rtl8xxxu_priv *priv)
+int rtl8xxxu_read_efuse(struct rtl8xxxu_priv *priv)
 {
 	struct device *dev = &priv->udev->dev;
 	int i, ret = 0;
@@ -1849,12 +1946,18 @@ void rtl8xxxu_reset_8051(struct rtl8xxxu_priv *priv)
 static int rtl8xxxu_start_firmware(struct rtl8xxxu_priv *priv)
 {
 	struct device *dev = &priv->udev->dev;
+	u16 reg_mcu_fw_dl;
 	int ret = 0, i;
 	u32 val32;
 
+	if (priv->rtl_chip == RTL8710B)
+		reg_mcu_fw_dl = REG_8051FW_CTRL_V1_8710B;
+	else
+		reg_mcu_fw_dl = REG_MCU_FW_DL;
+
 	/* Poll checksum report */
 	for (i = 0; i < RTL8XXXU_FIRMWARE_POLL_MAX; i++) {
-		val32 = rtl8xxxu_read32(priv, REG_MCU_FW_DL);
+		val32 = rtl8xxxu_read32(priv, reg_mcu_fw_dl);
 		if (val32 & MCU_FW_DL_CSUM_REPORT)
 			break;
 	}
@@ -1865,10 +1968,10 @@ static int rtl8xxxu_start_firmware(struct rtl8xxxu_priv *priv)
 		goto exit;
 	}
 
-	val32 = rtl8xxxu_read32(priv, REG_MCU_FW_DL);
+	val32 = rtl8xxxu_read32(priv, reg_mcu_fw_dl);
 	val32 |= MCU_FW_DL_READY;
 	val32 &= ~MCU_WINT_INIT_READY;
-	rtl8xxxu_write32(priv, REG_MCU_FW_DL, val32);
+	rtl8xxxu_write32(priv, reg_mcu_fw_dl, val32);
 
 	/*
 	 * Reset the 8051 in order for the firmware to start running,
@@ -1878,7 +1981,7 @@ static int rtl8xxxu_start_firmware(struct rtl8xxxu_priv *priv)
 
 	/* Wait for firmware to become ready */
 	for (i = 0; i < RTL8XXXU_FIRMWARE_POLL_MAX; i++) {
-		val32 = rtl8xxxu_read32(priv, REG_MCU_FW_DL);
+		val32 = rtl8xxxu_read32(priv, reg_mcu_fw_dl);
 		if (val32 & MCU_WINT_INIT_READY)
 			break;
 
@@ -1894,7 +1997,7 @@ static int rtl8xxxu_start_firmware(struct rtl8xxxu_priv *priv)
 	/*
 	 * Init H2C command
 	 */
-	if (priv->rtl_chip == RTL8723B || priv->rtl_chip == RTL8188F)
+	if (priv->fops->init_reg_hmtfr)
 		rtl8xxxu_write8(priv, REG_HMTFR, 0x0f);
 exit:
 	return ret;
@@ -1903,42 +2006,56 @@ exit:
 static int rtl8xxxu_download_firmware(struct rtl8xxxu_priv *priv)
 {
 	int pages, remainder, i, ret;
+	u16 reg_mcu_fw_dl;
 	u8 val8;
 	u16 val16;
 	u32 val32;
 	u8 *fwptr;
 
-	val8 = rtl8xxxu_read8(priv, REG_SYS_FUNC + 1);
-	val8 |= 4;
-	rtl8xxxu_write8(priv, REG_SYS_FUNC + 1, val8);
+	if (priv->rtl_chip == RTL8710B) {
+		reg_mcu_fw_dl = REG_8051FW_CTRL_V1_8710B;
+	} else {
+		reg_mcu_fw_dl = REG_MCU_FW_DL;
 
-	/* 8051 enable */
-	val16 = rtl8xxxu_read16(priv, REG_SYS_FUNC);
-	val16 |= SYS_FUNC_CPU_ENABLE;
-	rtl8xxxu_write16(priv, REG_SYS_FUNC, val16);
+		val8 = rtl8xxxu_read8(priv, REG_SYS_FUNC + 1);
+		val8 |= 4;
+		rtl8xxxu_write8(priv, REG_SYS_FUNC + 1, val8);
+
+		/* 8051 enable */
+		val16 = rtl8xxxu_read16(priv, REG_SYS_FUNC);
+		val16 |= SYS_FUNC_CPU_ENABLE;
+		rtl8xxxu_write16(priv, REG_SYS_FUNC, val16);
+	}
 
-	val8 = rtl8xxxu_read8(priv, REG_MCU_FW_DL);
+	val8 = rtl8xxxu_read8(priv, reg_mcu_fw_dl);
 	if (val8 & MCU_FW_RAM_SEL) {
 		dev_info(&priv->udev->dev,
 			 "Firmware is already running, resetting the MCU.\n");
-		rtl8xxxu_write8(priv, REG_MCU_FW_DL, 0x00);
+		rtl8xxxu_write8(priv, reg_mcu_fw_dl, 0x00);
 		priv->fops->reset_8051(priv);
 	}
 
 	/* MCU firmware download enable */
-	val8 = rtl8xxxu_read8(priv, REG_MCU_FW_DL);
+	val8 = rtl8xxxu_read8(priv, reg_mcu_fw_dl);
 	val8 |= MCU_FW_DL_ENABLE;
-	rtl8xxxu_write8(priv, REG_MCU_FW_DL, val8);
+	rtl8xxxu_write8(priv, reg_mcu_fw_dl, val8);
 
 	/* 8051 reset */
-	val32 = rtl8xxxu_read32(priv, REG_MCU_FW_DL);
+	val32 = rtl8xxxu_read32(priv, reg_mcu_fw_dl);
 	val32 &= ~BIT(19);
-	rtl8xxxu_write32(priv, REG_MCU_FW_DL, val32);
+	rtl8xxxu_write32(priv, reg_mcu_fw_dl, val32);
+
+	if (priv->rtl_chip == RTL8710B) {
+		/* We must set 0x8090[8]=1 before download FW. */
+		val8 = rtl8xxxu_read8(priv, reg_mcu_fw_dl + 1);
+		val8 |= BIT(0);
+		rtl8xxxu_write8(priv, reg_mcu_fw_dl + 1, val8);
+	}
 
 	/* Reset firmware download checksum */
-	val8 = rtl8xxxu_read8(priv, REG_MCU_FW_DL);
+	val8 = rtl8xxxu_read8(priv, reg_mcu_fw_dl);
 	val8 |= MCU_FW_DL_CSUM_REPORT;
-	rtl8xxxu_write8(priv, REG_MCU_FW_DL, val8);
+	rtl8xxxu_write8(priv, reg_mcu_fw_dl, val8);
 
 	pages = priv->fw_size / RTL_FW_PAGE_SIZE;
 	remainder = priv->fw_size % RTL_FW_PAGE_SIZE;
@@ -1946,9 +2063,9 @@ static int rtl8xxxu_download_firmware(struct rtl8xxxu_priv *priv)
 	fwptr = priv->fw_data->data;
 
 	for (i = 0; i < pages; i++) {
-		val8 = rtl8xxxu_read8(priv, REG_MCU_FW_DL + 2) & 0xF8;
+		val8 = rtl8xxxu_read8(priv, reg_mcu_fw_dl + 2) & 0xF8;
 		val8 |= i;
-		rtl8xxxu_write8(priv, REG_MCU_FW_DL + 2, val8);
+		rtl8xxxu_write8(priv, reg_mcu_fw_dl + 2, val8);
 
 		ret = rtl8xxxu_writeN(priv, REG_FW_START_ADDRESS,
 				      fwptr, RTL_FW_PAGE_SIZE);
@@ -1961,9 +2078,9 @@ static int rtl8xxxu_download_firmware(struct rtl8xxxu_priv *priv)
 	}
 
 	if (remainder) {
-		val8 = rtl8xxxu_read8(priv, REG_MCU_FW_DL + 2) & 0xF8;
+		val8 = rtl8xxxu_read8(priv, reg_mcu_fw_dl + 2) & 0xF8;
 		val8 |= i;
-		rtl8xxxu_write8(priv, REG_MCU_FW_DL + 2, val8);
+		rtl8xxxu_write8(priv, reg_mcu_fw_dl + 2, val8);
 		ret = rtl8xxxu_writeN(priv, REG_FW_START_ADDRESS,
 				      fwptr, remainder);
 		if (ret != remainder) {
@@ -1975,9 +2092,9 @@ static int rtl8xxxu_download_firmware(struct rtl8xxxu_priv *priv)
 	ret = 0;
 fw_abort:
 	/* MCU firmware download disable */
-	val16 = rtl8xxxu_read16(priv, REG_MCU_FW_DL);
+	val16 = rtl8xxxu_read16(priv, reg_mcu_fw_dl);
 	val16 &= ~MCU_FW_DL_ENABLE;
-	rtl8xxxu_write16(priv, REG_MCU_FW_DL, val16);
+	rtl8xxxu_write16(priv, reg_mcu_fw_dl, val16);
 
 	return ret;
 }
@@ -2017,6 +2134,7 @@ int rtl8xxxu_load_firmware(struct rtl8xxxu_priv *priv, const char *fw_name)
 	case 0x5300:
 	case 0x2300:
 	case 0x88f0:
+	case 0x10b0:
 		break;
 	default:
 		ret = -EINVAL;
@@ -3827,24 +3945,17 @@ void rtl8xxxu_init_burst(struct rtl8xxxu_priv *priv)
 	rtl8xxxu_write8(priv, REG_HT_SINGLE_AMPDU_8723B, val8);
 
 	rtl8xxxu_write16(priv, REG_MAX_AGGR_NUM, 0x0c14);
-	if (priv->rtl_chip == RTL8723B)
-		val8 = 0x5e;
-	else if (priv->rtl_chip == RTL8188F)
-		val8 = 0x70; /* 0x5e would make it very slow */
-	rtl8xxxu_write8(priv, REG_AMPDU_MAX_TIME_8723B, val8);
+	rtl8xxxu_write8(priv, REG_AMPDU_MAX_TIME_8723B,
+			priv->fops->ampdu_max_time);
 	rtl8xxxu_write32(priv, REG_AGGLEN_LMT, 0xffffffff);
 	rtl8xxxu_write8(priv, REG_RX_PKT_LIMIT, 0x18);
 	rtl8xxxu_write8(priv, REG_PIFS, 0x00);
-	if (priv->rtl_chip == RTL8188F) {
+	if (priv->rtl_chip == RTL8188F || priv->rtl_chip == RTL8710B) {
 		rtl8xxxu_write8(priv, REG_FWHW_TXQ_CTRL, FWHW_TXQ_CTRL_AMPDU_RETRY);
 		rtl8xxxu_write32(priv, REG_FAST_EDCA_CTRL, 0x03086666);
 	}
-	if (priv->rtl_chip == RTL8723B)
-		val8 = 0x50;
-	else if (priv->rtl_chip == RTL8188F)
-		val8 = 0x28; /* 0x50 would make the upload slow */
-	rtl8xxxu_write8(priv, REG_USTIME_TSF_8723B, val8);
-	rtl8xxxu_write8(priv, REG_USTIME_EDCA, val8);
+	rtl8xxxu_write8(priv, REG_USTIME_TSF_8723B, priv->fops->ustime_tsf_edca);
+	rtl8xxxu_write8(priv, REG_USTIME_EDCA, priv->fops->ustime_tsf_edca);
 
 	/* to prevent mac is reseted by bus. */
 	val8 = rtl8xxxu_read8(priv, REG_RSV_CTRL);
@@ -3927,7 +4038,7 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 
 	/* RFSW Control - clear bit 14 ?? */
 	if (priv->rtl_chip != RTL8723B && priv->rtl_chip != RTL8192E &&
-	    priv->rtl_chip != RTL8188E)
+	    priv->rtl_chip != RTL8188E && priv->rtl_chip != RTL8710B)
 		rtl8xxxu_write32(priv, REG_FPGA0_TX_INFO, 0x00000003);
 
 	val32 = FPGA0_RF_TRSW | FPGA0_RF_TRSWB | FPGA0_RF_ANTSW |
@@ -3940,7 +4051,8 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	rtl8xxxu_write32(priv, REG_FPGA0_XAB_RF_SW_CTRL, val32);
 
 	/* 0x860[6:5]= 00 - why? - this sets antenna B */
-	if (priv->rtl_chip != RTL8192E && priv->rtl_chip != RTL8188E)
+	if (priv->rtl_chip != RTL8192E && priv->rtl_chip != RTL8188E &&
+	    priv->rtl_chip != RTL8710B)
 		rtl8xxxu_write32(priv, REG_FPGA0_XA_RF_INT_OE, 0x66f60210);
 
 	if (!macpower) {
@@ -4013,10 +4125,14 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 			val8 &= 0xf8;
 			rtl8xxxu_write8(priv, 0xa3, val8);
 		}
+
+		if (priv->rtl_chip == RTL8710B)
+			rtl8xxxu_write8(priv, REG_EARLY_MODE_CONTROL_8710B, 0);
 	}
 
 	/*
-	 * Unit in 8 bytes, not obvious what it is used for
+	 * Unit in 8 bytes.
+	 * Get Rx PHY status in order to report RSSI and others.
 	 */
 	rtl8xxxu_write8(priv, REG_RX_DRVINFO_SZ, 4);
 
@@ -4035,6 +4151,8 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 		val8 = rtl8xxxu_read8(priv, REG_USB_SPECIAL_OPTION);
 		val8 |= USB_SPEC_INT_BULK_SELECT;
 		rtl8xxxu_write8(priv, REG_USB_SPECIAL_OPTION, val8);
+	} else if (priv->rtl_chip == RTL8710B) {
+		rtl8xxxu_write32(priv, REG_HIMR0_8710B, 0);
 	} else {
 		/*
 		 * Enable all interrupts - not obvious USB needs to do this
@@ -4054,7 +4172,7 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 		RCR_APPEND_PHYSTAT | RCR_APPEND_ICV | RCR_APPEND_MIC;
 	rtl8xxxu_write32(priv, REG_RCR, val32);
 
-	if (priv->rtl_chip == RTL8188F) {
+	if (fops->init_reg_rxfltmap) {
 		/* Accept all data frames */
 		rtl8xxxu_write16(priv, REG_RXFLTMAP2, 0xffff);
 
@@ -4123,7 +4241,7 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	val16 = BEACON_DISABLE_TSF_UPDATE | (BEACON_DISABLE_TSF_UPDATE << 8);
 	rtl8xxxu_write16(priv, REG_BEACON_CTRL, val16);
 	rtl8xxxu_write16(priv, REG_TBTT_PROHIBIT, 0x6404);
-	if (priv->rtl_chip != RTL8188F)
+	if (priv->rtl_chip != RTL8188F && priv->rtl_chip != RTL8710B)
 		/* Firmware will control REG_DRVERLYINT when power saving is enable, */
 		/* so don't set this register on STA mode. */
 		rtl8xxxu_write8(priv, REG_DRIVER_EARLY_INT, DRIVER_EARLY_INT_TIME);
@@ -4133,14 +4251,13 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	/*
 	 * Initialize burst parameters
 	 */
-
 	if (priv->fops->init_burst)
 		priv->fops->init_burst(priv);
 
 	if (fops->init_aggregation)
 		fops->init_aggregation(priv);
 
-	if (priv->rtl_chip == RTL8188F || priv->rtl_chip == RTL8188E) {
+	if (fops->init_reg_pkt_life_time) {
 		rtl8xxxu_write16(priv, REG_PKT_VO_VI_LIFE_TIME, 0x0400); /* unit: 256us. 256ms */
 		rtl8xxxu_write16(priv, REG_PKT_BE_BK_LIFE_TIME, 0x0400); /* unit: 256us. 256ms */
 	}
@@ -4163,7 +4280,8 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	fops->set_tx_power(priv, 1, false);
 
 	/* Let the 8051 take control of antenna setting */
-	if (priv->rtl_chip != RTL8192E && priv->rtl_chip != RTL8188F) {
+	if (priv->rtl_chip != RTL8192E && priv->rtl_chip != RTL8188F &&
+	    priv->rtl_chip != RTL8710B) {
 		val8 = rtl8xxxu_read8(priv, REG_LEDCFG2);
 		val8 |= LEDCFG2_DPDT_SELECT;
 		rtl8xxxu_write8(priv, REG_LEDCFG2, val8);
@@ -4174,7 +4292,7 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	/* Disable BAR - not sure if this has any effect on USB */
 	rtl8xxxu_write32(priv, REG_BAR_MODE_CTRL, 0x0201ffff);
 
-	if (priv->rtl_chip != RTL8188F && priv->rtl_chip != RTL8188E)
+	if (priv->rtl_chip != RTL8188F && priv->rtl_chip != RTL8188E && priv->rtl_chip != RTL8710B)
 		rtl8xxxu_write16(priv, REG_FAST_EDCA_CTRL, 0);
 
 	if (fops->init_statistics)
@@ -4213,7 +4331,7 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	 * This should enable thermal meter
 	 */
 	if (fops->gen2_thermal_meter) {
-		if (priv->rtl_chip == RTL8188F) {
+		if (priv->rtl_chip == RTL8188F || priv->rtl_chip == RTL8710B) {
 			val32 = rtl8xxxu_read_rfreg(priv, RF_A, RF6052_REG_T_METER_8723B);
 			val32 |= 0x30000;
 			rtl8xxxu_write_rfreg(priv, RF_A, RF6052_REG_T_METER_8723B, val32);
@@ -4285,6 +4403,24 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 		rtl8xxxu_write32(priv, REG_AGC_RPT, val32);
 	}
 
+	if (priv->rtl_chip == RTL8710B) {
+		/*
+		 * 0x76D[5:4] is Port0,Port1 Enable Bit.
+		 * This is only for 8710B, 2b'00 for MP and 2b'11 for Normal Driver
+		 */
+		val8 = rtl8xxxu_read8(priv, REG_PORT_CONTROL_8710B);
+		val8 |= BIT(5) | BIT(4);
+		rtl8xxxu_write8(priv, REG_PORT_CONTROL_8710B, val8);
+
+		/* Set 0x5c[8] and [2:0] = 1, LDO mode */
+		val32 = rtl8xxxu_read32(priv, REG_WL_RF_PSS_8710B);
+		val32 |= 0x107;
+		rtl8xxxu_write32(priv, REG_WL_RF_PSS_8710B, val32);
+	}
+
+	val32 = rtl8xxxu_read32(priv, 0xa9c);
+	priv->cck_new_agc = u32_get_bits(val32, BIT(17));
+
 	/* Initialise the center frequency offset tracking */
 	if (priv->fops->set_crystal_cap) {
 		val32 = rtl8xxxu_read32(priv, REG_OFDM1_CFO_TRACKING);
@@ -5374,6 +5510,10 @@ static void rtl8xxxu_tx(struct ieee80211_hw *hw,
 
 	rtl8xxxu_calc_tx_desc_csum(tx_desc);
 
+	/* avoid zero checksum make tx hang */
+	if (priv->rtl_chip == RTL8710B)
+		tx_desc->csum = ~tx_desc->csum;
+
 	usb_fill_bulk_urb(&tx_urb->urb, priv->udev, priv->pipe_out[queue],
 			  skb->data, skb->len, rtl8xxxu_tx_complete, skb);
 
@@ -5389,11 +5529,11 @@ error:
 	dev_kfree_skb(skb);
 }
 
-static void rtl8xxxu_rx_parse_phystats(struct rtl8xxxu_priv *priv,
-				       struct ieee80211_rx_status *rx_status,
-				       struct rtl8723au_phy_stats *phy_stats,
-				       u32 rxmcs, struct ieee80211_hdr *hdr,
-				       bool crc_icv_err)
+void rtl8723au_rx_parse_phystats(struct rtl8xxxu_priv *priv,
+				 struct ieee80211_rx_status *rx_status,
+				 struct rtl8723au_phy_stats *phy_stats,
+				 u32 rxmcs, struct ieee80211_hdr *hdr,
+				 bool crc_icv_err)
 {
 	if (phy_stats->sgi_en)
 		rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
@@ -5402,9 +5542,7 @@ static void rtl8xxxu_rx_parse_phystats(struct rtl8xxxu_priv *priv,
 		/*
 		 * Handle PHY stats for CCK rates
 		 */
-		u8 cck_agc_rpt = phy_stats->cck_agc_rpt_ofdm_cfosho_a;
-
-		rx_status->signal = priv->fops->cck_rssi(priv, cck_agc_rpt);
+		rx_status->signal = priv->fops->cck_rssi(priv, phy_stats);
 	} else {
 		bool parse_cfo = priv->fops->set_crystal_cap &&
 				 priv->vif &&
@@ -5426,6 +5564,96 @@ static void rtl8xxxu_rx_parse_phystats(struct rtl8xxxu_priv *priv,
 	}
 }
 
+static void jaguar2_rx_parse_phystats_type0(struct rtl8xxxu_priv *priv,
+					    struct ieee80211_rx_status *rx_status,
+					    struct jaguar2_phy_stats_type0 *phy_stats0,
+					    u32 rxmcs, struct ieee80211_hdr *hdr,
+					    bool crc_icv_err)
+{
+	s8 rx_power = phy_stats0->pwdb - 110;
+
+	if (!priv->cck_new_agc)
+		rx_power = priv->fops->cck_rssi(priv, (struct rtl8723au_phy_stats *)phy_stats0);
+
+	rx_status->signal = rx_power;
+}
+
+static void jaguar2_rx_parse_phystats_type1(struct rtl8xxxu_priv *priv,
+					    struct ieee80211_rx_status *rx_status,
+					    struct jaguar2_phy_stats_type1 *phy_stats1,
+					    u32 rxmcs, struct ieee80211_hdr *hdr,
+					    bool crc_icv_err)
+{
+	bool parse_cfo = priv->fops->set_crystal_cap &&
+			 priv->vif &&
+			 priv->vif->type == NL80211_IFTYPE_STATION &&
+			 priv->vif->cfg.assoc &&
+			 !crc_icv_err &&
+			 !ieee80211_is_ctl(hdr->frame_control) &&
+			 ether_addr_equal(priv->vif->bss_conf.bssid, hdr->addr2);
+	u8 pwdb_max = 0;
+	int rx_path;
+
+	if (parse_cfo) {
+		/* Only path-A and path-B have CFO tail and short CFO */
+		priv->cfo_tracking.cfo_tail[RF_A] = phy_stats1->cfo_tail[RF_A];
+		priv->cfo_tracking.cfo_tail[RF_B] = phy_stats1->cfo_tail[RF_B];
+
+		priv->cfo_tracking.packet_count++;
+	}
+
+	for (rx_path = 0; rx_path < priv->rx_paths; rx_path++)
+		pwdb_max = max(pwdb_max, phy_stats1->pwdb[rx_path]);
+
+	rx_status->signal = pwdb_max - 110;
+}
+
+static void jaguar2_rx_parse_phystats_type2(struct rtl8xxxu_priv *priv,
+					    struct ieee80211_rx_status *rx_status,
+					    struct jaguar2_phy_stats_type2 *phy_stats2,
+					    u32 rxmcs, struct ieee80211_hdr *hdr,
+					    bool crc_icv_err)
+{
+	u8 pwdb_max = 0;
+	int rx_path;
+
+	for (rx_path = 0; rx_path < priv->rx_paths; rx_path++)
+		pwdb_max = max(pwdb_max, phy_stats2->pwdb[rx_path]);
+
+	rx_status->signal = pwdb_max - 110;
+}
+
+void jaguar2_rx_parse_phystats(struct rtl8xxxu_priv *priv,
+			       struct ieee80211_rx_status *rx_status,
+			       struct rtl8723au_phy_stats *phy_stats,
+			       u32 rxmcs, struct ieee80211_hdr *hdr,
+			       bool crc_icv_err)
+{
+	struct jaguar2_phy_stats_type0 *phy_stats0 = (struct jaguar2_phy_stats_type0 *)phy_stats;
+	struct jaguar2_phy_stats_type1 *phy_stats1 = (struct jaguar2_phy_stats_type1 *)phy_stats;
+	struct jaguar2_phy_stats_type2 *phy_stats2 = (struct jaguar2_phy_stats_type2 *)phy_stats;
+
+	switch (phy_stats0->page_num) {
+	case 0:
+		/* CCK */
+		jaguar2_rx_parse_phystats_type0(priv, rx_status, phy_stats0,
+						rxmcs, hdr, crc_icv_err);
+		break;
+	case 1:
+		/* OFDM */
+		jaguar2_rx_parse_phystats_type1(priv, rx_status, phy_stats1,
+						rxmcs, hdr, crc_icv_err);
+		break;
+	case 2:
+		/* Also OFDM but different (how?) */
+		jaguar2_rx_parse_phystats_type2(priv, rx_status, phy_stats2,
+						rxmcs, hdr, crc_icv_err);
+		break;
+	default:
+		return;
+	}
+}
+
 static void rtl8xxxu_free_rx_resources(struct rtl8xxxu_priv *priv)
 {
 	struct rtl8xxxu_rx_urb *rx_urb, *tmp;
@@ -5924,7 +6152,7 @@ int rtl8xxxu_parse_rxdesc16(struct rtl8xxxu_priv *priv, struct sk_buff *skb)
 			skb_trim(skb, pkt_len);
 
 			if (rx_desc->phy_stats)
-				rtl8xxxu_rx_parse_phystats(
+				priv->fops->parse_phystats(
 					priv, rx_status, phy_stats,
 					rx_desc->rxmcs,
 					(struct ieee80211_hdr *)skb->data,
@@ -5999,7 +6227,7 @@ int rtl8xxxu_parse_rxdesc24(struct rtl8xxxu_priv *priv, struct sk_buff *skb)
 	}
 
 	if (rx_desc->phy_stats)
-		rtl8xxxu_rx_parse_phystats(priv, rx_status, phy_stats,
+		priv->fops->parse_phystats(priv, rx_status, phy_stats,
 					   rx_desc->rxmcs, (struct ieee80211_hdr *)skb->data,
 					   rx_desc->crc32 || rx_desc->icverr);
 
@@ -6806,10 +7034,8 @@ exit:
 	rtl8xxxu_write16(priv, REG_RXFLTMAP2, 0xffff);
 	rtl8xxxu_write16(priv, REG_RXFLTMAP0, 0xffff);
 
-	if (priv->rtl_chip == RTL8188E)
-		rtl8xxxu_write32(priv, REG_OFDM0_XA_AGC_CORE1, 0x6955341e);
-	else
-		rtl8xxxu_write32(priv, REG_OFDM0_XA_AGC_CORE1, 0x6954341e);
+	rtl8xxxu_write32_mask(priv, REG_OFDM0_XA_AGC_CORE1,
+			      OFDM0_X_AGC_CORE1_IGI_MASK, 0x1e);
 
 	return ret;
 
@@ -7011,12 +7237,13 @@ static int rtl8xxxu_probe(struct usb_interface *interface,
 		case 0x818b:
 		case 0xf179:
 		case 0x8179:
+		case 0xb711:
 			untested = 0;
 			break;
 		}
 		break;
 	case 0x7392:
-		if (id->idProduct == 0x7811 || id->idProduct == 0xa611)
+		if (id->idProduct == 0x7811 || id->idProduct == 0xa611 || id->idProduct == 0xb811)
 			untested = 0;
 		break;
 	case 0x050d:
@@ -7059,6 +7286,7 @@ static int rtl8xxxu_probe(struct usb_interface *interface,
 	priv->udev = udev;
 	priv->fops = (struct rtl8xxxu_fileops *)id->driver_info;
 	mutex_init(&priv->usb_buf_mutex);
+	mutex_init(&priv->syson_indirect_access_mutex);
 	mutex_init(&priv->h2c_mutex);
 	INIT_LIST_HEAD(&priv->tx_urb_free_list);
 	spin_lock_init(&priv->tx_urb_lock);
@@ -7088,7 +7316,7 @@ static int rtl8xxxu_probe(struct usb_interface *interface,
 	else
 		INIT_WORK(&priv->c2hcmd_work, rtl8xxxu_c2hcmd_callback);
 
-	ret = rtl8xxxu_read_efuse(priv);
+	ret = priv->fops->read_efuse(priv);
 	if (ret) {
 		dev_err(&udev->dev, "Fatal - failed to read EFuse\n");
 		goto err_set_intfdata;
@@ -7178,6 +7406,7 @@ err_set_intfdata:
 
 	kfree(priv->fw_data);
 	mutex_destroy(&priv->usb_buf_mutex);
+	mutex_destroy(&priv->syson_indirect_access_mutex);
 	mutex_destroy(&priv->h2c_mutex);
 
 	ieee80211_free_hw(hw);
@@ -7207,6 +7436,7 @@ static void rtl8xxxu_disconnect(struct usb_interface *interface)
 
 	kfree(priv->fw_data);
 	mutex_destroy(&priv->usb_buf_mutex);
+	mutex_destroy(&priv->syson_indirect_access_mutex);
 	mutex_destroy(&priv->h2c_mutex);
 
 	if (priv->udev->state != USB_STATE_NOTATTACHED) {
@@ -7287,6 +7517,12 @@ static const struct usb_device_id dev_table[] = {
 /* Rosewill USB-N150 Nano */
 {USB_DEVICE_AND_INTERFACE_INFO(USB_VENDOR_ID_REALTEK, 0xffef, 0xff, 0xff, 0xff),
 	.driver_info = (unsigned long)&rtl8188eu_fops},
+/* RTL8710BU aka RTL8188GU (not to be confused with RTL8188GTVU) */
+{USB_DEVICE_AND_INTERFACE_INFO(USB_VENDOR_ID_REALTEK, 0xb711, 0xff, 0xff, 0xff),
+	.driver_info = (unsigned long)&rtl8710bu_fops},
+/* TOTOLINK N150UA V5 / N150UA-B */
+{USB_DEVICE_AND_INTERFACE_INFO(USB_VENDOR_ID_REALTEK, 0x2005, 0xff, 0xff, 0xff),
+	.driver_info = (unsigned long)&rtl8710bu_fops},
 #ifdef CONFIG_RTL8XXXU_UNTESTED
 /* Still supported by rtlwifi */
 {USB_DEVICE_AND_INTERFACE_INFO(USB_VENDOR_ID_REALTEK, 0x8176, 0xff, 0xff, 0xff),
@@ -7455,24 +7691,6 @@ static struct usb_driver rtl8xxxu_driver = {
 	.disable_hub_initiated_lpm = 1,
 };
 
-static int __init rtl8xxxu_module_init(void)
-{
-	int res;
-
-	res = usb_register(&rtl8xxxu_driver);
-	if (res < 0)
-		pr_err(DRIVER_NAME ": usb_register() failed (%i)\n", res);
-
-	return res;
-}
-
-static void __exit rtl8xxxu_module_exit(void)
-{
-	usb_deregister(&rtl8xxxu_driver);
-}
-
-
 MODULE_DEVICE_TABLE(usb, dev_table);
 
-module_init(rtl8xxxu_module_init);
-module_exit(rtl8xxxu_module_exit);
+module_usb_driver(rtl8xxxu_driver);
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_regs.h b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_regs.h
index 5849fa4e1566..4dffbab494c3 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_regs.h
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_regs.h
@@ -490,6 +490,8 @@
 #define  RXDMA_PRO_DMA_BURST_CNT	GENMASK(3, 2)	/* Set to 0x3. */
 #define  RXDMA_PRO_DMA_BURST_SIZE	GENMASK(5, 4)	/* Set to 0x1. */
 
+#define REG_EARLY_MODE_CONTROL_8710B	0x02bc
+
 #define REG_RF_BB_CMD_ADDR		0x02c0
 #define REG_RF_BB_CMD_DATA		0x02c4
 
@@ -845,6 +847,7 @@
 #define REG_BT_CONTROL_8723BU		0x0764
 #define  BT_CONTROL_BT_GRANT		BIT(12)
 
+#define REG_PORT_CONTROL_8710B		0x076d
 #define REG_WLAN_ACT_CONTROL_8723B	0x076e
 
 #define REG_FPGA0_RF_MODE		0x0800
@@ -1004,8 +1007,12 @@
 #define  CCK_PD_TYPE1_LV3_TH		0xdd
 #define  CCK_PD_TYPE1_LV4_TH		0xed
 
+#define REG_CCK0_TX_FILTER1		0x0a20
+#define REG_CCK0_TX_FILTER2		0x0a24
+#define REG_CCK0_DEBUG_PORT		0x0a28	/* debug port and Tx filter3 */
 #define REG_AGC_RPT			0xa80
 #define  AGC_RPT_CCK			BIT(7)
+#define REG_CCK0_TX_FILTER3		0x0aac
 
 #define REG_CONFIG_ANT_A		0x0b68
 #define REG_CONFIG_ANT_B		0x0b6c
@@ -1179,6 +1186,8 @@
 						    Unavailable */
 #define  USB_HIMR_ROK			BIT(0)	/*  Receive DMA OK Interrupt */
 
+#define REG_USB_ACCESS_TIMEOUT		0xfe4c
+
 #define REG_USB_SPECIAL_OPTION		0xfe55
 #define  USB_SPEC_USB_AGG_ENABLE	BIT(3)	/* Enable USB aggregation */
 #define  USB_SPEC_INT_BULK_SELECT	BIT(4)	/* Use interrupt endpoint to
@@ -1204,6 +1213,41 @@
 #define REG_NORMAL_SIE_MAC_ADDR		0xfe70	/* 0xfe70 - 0xfe75 */
 #define REG_NORMAL_SIE_STRING		0xfe80	/* 0xfe80 - 0xfedf */
 
+/*
+ * 8710B register addresses between 0x00 and 0xff must have 0x8000
+ * added to them. We take care of that in the rtl8xxxu_read{8,16,32}
+ * and rtl8xxxu_write{8,16,32} functions.
+ */
+#define REG_SYS_FUNC_8710B		0x0004
+#define REG_AFE_CTRL_8710B		0x0050
+#define REG_WL_RF_PSS_8710B		0x005c
+#define REG_EFUSE_INDIRECT_CTRL_8710B	0x006c
+#define  NORMAL_REG_READ_OFFSET		0x83000000
+#define  NORMAL_REG_WRITE_OFFSET	0x84000000
+#define  EFUSE_READ_OFFSET		0x85000000
+#define  EFUSE_WRITE_OFFSET		0x86000000
+#define REG_HIMR0_8710B			0x0080
+#define REG_HISR0_8710B			0x0084
+/*
+ * 8710B uses this instead of REG_MCU_FW_DL, but at least bits
+ * 0-7 have the same meaning.
+ */
+#define REG_8051FW_CTRL_V1_8710B	0x0090
+#define REG_USB_HOST_INDIRECT_DATA_8710B	0x009c
+#define REG_WL_STATUS_8710B		0x00f0
+#define REG_USB_HOST_INDIRECT_ADDR_8710B	0x00f8
+
+/*
+ * 8710B registers which must be accessed through rtl8710b_read_syson_reg
+ * and rtl8710b_write_syson_reg.
+ */
+#define SYSON_REG_BASE_ADDR_8710B	0x40000000
+#define REG_SYS_XTAL_CTRL0_8710B	0x060
+#define REG_SYS_EEPROM_CTRL0_8710B	0x0e0
+#define REG_SYS_SYSTEM_CFG0_8710B	0x1f0
+#define REG_SYS_SYSTEM_CFG1_8710B	0x1f4
+#define REG_SYS_SYSTEM_CFG2_8710B	0x1f8
+
 /* RF6052 registers */
 #define RF6052_REG_AC			0x00
 #define RF6052_REG_IQADJ_G1		0x01
diff --git a/drivers/net/wireless/realtek/rtlwifi/debug.c b/drivers/net/wireless/realtek/rtlwifi/debug.c
index 0b1bc04cb6ad..9eb26dfe4ca9 100644
--- a/drivers/net/wireless/realtek/rtlwifi/debug.c
+++ b/drivers/net/wireless/realtek/rtlwifi/debug.c
@@ -278,8 +278,8 @@ static ssize_t rtl_debugfs_set_write_reg(struct file *filp,
 
 	tmp_len = (count > sizeof(tmp) - 1 ? sizeof(tmp) - 1 : count);
 
-	if (!buffer || copy_from_user(tmp, buffer, tmp_len))
-		return count;
+	if (copy_from_user(tmp, buffer, tmp_len))
+		return -EFAULT;
 
 	tmp[tmp_len] = '\0';
 
@@ -287,7 +287,7 @@ static ssize_t rtl_debugfs_set_write_reg(struct file *filp,
 	num = sscanf(tmp, "%x %x %x", &addr, &val, &len);
 
 	if (num !=  3)
-		return count;
+		return -EINVAL;
 
 	switch (len) {
 	case 1:
@@ -375,8 +375,8 @@ static ssize_t rtl_debugfs_set_write_rfreg(struct file *filp,
 
 	tmp_len = (count > sizeof(tmp) - 1 ? sizeof(tmp) - 1 : count);
 
-	if (!buffer || copy_from_user(tmp, buffer, tmp_len))
-		return count;
+	if (copy_from_user(tmp, buffer, tmp_len))
+		return -EFAULT;
 
 	tmp[tmp_len] = '\0';
 
@@ -386,7 +386,7 @@ static ssize_t rtl_debugfs_set_write_rfreg(struct file *filp,
 	if (num != 4) {
 		rtl_dbg(rtlpriv, COMP_ERR, DBG_DMESG,
 			"Format is <path> <addr> <mask> <data>\n");
-		return count;
+		return -EINVAL;
 	}
 
 	rtl_set_rfreg(hw, path, addr, bitmask, data);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
index b9c62640d2cb..dc480323c9cb 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
@@ -1428,7 +1428,9 @@ static void _rtl92ce_read_txpower_info_from_hwpg(struct ieee80211_hw *hw,
 
 	for (rf_path = 0; rf_path < 2; rf_path++) {
 		for (i = 0; i < 3; i++) {
-			if (!autoload_fail) {
+			if (!autoload_fail &&
+			    hwinfo[EEPROM_TXPOWERCCK + rf_path * 3 + i] != 0xff &&
+			    hwinfo[EEPROM_TXPOWERHT40_1S + rf_path * 3 + i] != 0xff) {
 				rtlefuse->
 				    eeprom_chnlarea_txpwr_cck[rf_path][i] =
 				    hwinfo[EEPROM_TXPOWERCCK + rf_path * 3 + i];
@@ -1448,7 +1450,8 @@ static void _rtl92ce_read_txpower_info_from_hwpg(struct ieee80211_hw *hw,
 	}
 
 	for (i = 0; i < 3; i++) {
-		if (!autoload_fail)
+		if (!autoload_fail &&
+		    hwinfo[EEPROM_TXPOWERHT40_2SDIFF + i] != 0xff)
 			tempval = hwinfo[EEPROM_TXPOWERHT40_2SDIFF + i];
 		else
 			tempval = EEPROM_DEFAULT_HT40_2SDIFF;
@@ -1518,7 +1521,9 @@ static void _rtl92ce_read_txpower_info_from_hwpg(struct ieee80211_hw *hw,
 	}
 
 	for (i = 0; i < 3; i++) {
-		if (!autoload_fail) {
+		if (!autoload_fail &&
+		    hwinfo[EEPROM_TXPWR_GROUP + i] != 0xff &&
+		    hwinfo[EEPROM_TXPWR_GROUP + 3 + i] != 0xff) {
 			rtlefuse->eeprom_pwrlimit_ht40[i] =
 			    hwinfo[EEPROM_TXPWR_GROUP + i];
 			rtlefuse->eeprom_pwrlimit_ht20[i] =
@@ -1563,7 +1568,8 @@ static void _rtl92ce_read_txpower_info_from_hwpg(struct ieee80211_hw *hw,
 	for (i = 0; i < 14; i++) {
 		index = rtl92c_get_chnl_group((u8)i);
 
-		if (!autoload_fail)
+		if (!autoload_fail &&
+		    hwinfo[EEPROM_TXPOWERHT20DIFF + index] != 0xff)
 			tempval = hwinfo[EEPROM_TXPOWERHT20DIFF + index];
 		else
 			tempval = EEPROM_DEFAULT_HT20_DIFF;
@@ -1580,7 +1586,8 @@ static void _rtl92ce_read_txpower_info_from_hwpg(struct ieee80211_hw *hw,
 
 		index = rtl92c_get_chnl_group((u8)i);
 
-		if (!autoload_fail)
+		if (!autoload_fail &&
+		    hwinfo[EEPROM_TXPOWER_OFDMDIFF + index] != 0xff)
 			tempval = hwinfo[EEPROM_TXPOWER_OFDMDIFF + index];
 		else
 			tempval = EEPROM_DEFAULT_LEGACYHTTXPOWERDIFF;
@@ -1610,14 +1617,16 @@ static void _rtl92ce_read_txpower_info_from_hwpg(struct ieee80211_hw *hw,
 			"RF-B Legacy to HT40 Diff[%d] = 0x%x\n",
 			i, rtlefuse->txpwr_legacyhtdiff[RF90_PATH_B][i]);
 
-	if (!autoload_fail)
+	if (!autoload_fail && hwinfo[RF_OPTION1] != 0xff)
 		rtlefuse->eeprom_regulatory = (hwinfo[RF_OPTION1] & 0x7);
 	else
 		rtlefuse->eeprom_regulatory = 0;
 	RTPRINT(rtlpriv, FINIT, INIT_TXPOWER,
 		"eeprom_regulatory = 0x%x\n", rtlefuse->eeprom_regulatory);
 
-	if (!autoload_fail) {
+	if (!autoload_fail &&
+	    hwinfo[EEPROM_TSSI_A] != 0xff &&
+	    hwinfo[EEPROM_TSSI_B] != 0xff) {
 		rtlefuse->eeprom_tssi[RF90_PATH_A] = hwinfo[EEPROM_TSSI_A];
 		rtlefuse->eeprom_tssi[RF90_PATH_B] = hwinfo[EEPROM_TSSI_B];
 	} else {
@@ -1628,7 +1637,7 @@ static void _rtl92ce_read_txpower_info_from_hwpg(struct ieee80211_hw *hw,
 		rtlefuse->eeprom_tssi[RF90_PATH_A],
 		rtlefuse->eeprom_tssi[RF90_PATH_B]);
 
-	if (!autoload_fail)
+	if (!autoload_fail && hwinfo[EEPROM_THERMAL_METER] != 0xff)
 		tempval = hwinfo[EEPROM_THERMAL_METER];
 	else
 		tempval = EEPROM_DEFAULT_THERMALMETER;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
index 2aecb2583f75..df1e36fbc348 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
@@ -1047,7 +1047,6 @@ static int _rtl92de_set_media_status(struct ieee80211_hw *hw,
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	u8 bt_msr = rtl_read_byte(rtlpriv, MSR);
 	enum led_ctl_mode ledaction = LED_CTL_NO_LINK;
-	u8 bcnfunc_enable;
 
 	bt_msr &= 0xfc;
 
@@ -1064,31 +1063,26 @@ static int _rtl92de_set_media_status(struct ieee80211_hw *hw,
 			"Set HW_VAR_MEDIA_STATUS: No such media status(%x)\n",
 			type);
 	}
-	bcnfunc_enable = rtl_read_byte(rtlpriv, REG_BCN_CTRL);
 	switch (type) {
 	case NL80211_IFTYPE_UNSPECIFIED:
 		bt_msr |= MSR_NOLINK;
 		ledaction = LED_CTL_LINK;
-		bcnfunc_enable &= 0xF7;
 		rtl_dbg(rtlpriv, COMP_INIT, DBG_TRACE,
 			"Set Network type to NO LINK!\n");
 		break;
 	case NL80211_IFTYPE_ADHOC:
 		bt_msr |= MSR_ADHOC;
-		bcnfunc_enable |= 0x08;
 		rtl_dbg(rtlpriv, COMP_INIT, DBG_TRACE,
 			"Set Network type to Ad Hoc!\n");
 		break;
 	case NL80211_IFTYPE_STATION:
 		bt_msr |= MSR_INFRA;
 		ledaction = LED_CTL_LINK;
-		bcnfunc_enable &= 0xF7;
 		rtl_dbg(rtlpriv, COMP_INIT, DBG_TRACE,
 			"Set Network type to STA!\n");
 		break;
 	case NL80211_IFTYPE_AP:
 		bt_msr |= MSR_AP;
-		bcnfunc_enable |= 0x08;
 		rtl_dbg(rtlpriv, COMP_INIT, DBG_TRACE,
 			"Set Network type to AP!\n");
 		break;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
index bd0b7e365edb..a8b5bf45b1bb 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
@@ -1552,8 +1552,6 @@ void rtl92se_set_beacon_related_registers(struct ieee80211_hw *hw)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
-	u16 bcntime_cfg = 0;
-	u16 bcn_cw = 6, bcn_ifs = 0xf;
 	u16 atim_window = 2;
 
 	/* ATIM Window (in unit of TU). */
@@ -1576,13 +1574,6 @@ void rtl92se_set_beacon_related_registers(struct ieee80211_hw *hw)
 	 * other ad hoc STA */
 	rtl_write_byte(rtlpriv, BCN_ERR_THRESH, 100);
 
-	/* Beacon Time Configuration */
-	if (mac->opmode == NL80211_IFTYPE_ADHOC)
-		bcntime_cfg |= (bcn_cw << BCN_TCFG_CW_SHIFT);
-
-	/* TODO: bcn_ifs may required to be changed on ASIC */
-	bcntime_cfg |= bcn_ifs << BCN_TCFG_IFS;
-
 	/*for beacon changed */
 	rtl92s_phy_set_beacon_hwreg(hw, mac->beacon_interval);
 }
diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h
index 31f9e9e5c680..082af216760f 100644
--- a/drivers/net/wireless/realtek/rtlwifi/wifi.h
+++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h
@@ -2831,7 +2831,7 @@ struct rtl_priv {
 	 * beyond  this structure like:
 	 * rtl_pci_priv or rtl_usb_priv
 	 */
-	u8 priv[0] __aligned(sizeof(void *));
+	u8 priv[] __aligned(sizeof(void *));
 };
 
 #define rtl_priv(hw)		(((struct rtl_priv *)(hw)->priv))
diff --git a/drivers/net/wireless/realtek/rtw88/Kconfig b/drivers/net/wireless/realtek/rtw88/Kconfig
index 651ab56d9c6b..29eb2f8e0eb7 100644
--- a/drivers/net/wireless/realtek/rtw88/Kconfig
+++ b/drivers/net/wireless/realtek/rtw88/Kconfig
@@ -16,6 +16,9 @@ config RTW88_CORE
 config RTW88_PCI
 	tristate
 
+config RTW88_SDIO
+	tristate
+
 config RTW88_USB
 	tristate
 
@@ -42,6 +45,17 @@ config RTW88_8822BE
 
 	  802.11ac PCIe wireless network adapter
 
+config RTW88_8822BS
+	tristate "Realtek 8822BS SDIO wireless network adapter"
+	depends on MMC
+	select RTW88_CORE
+	select RTW88_SDIO
+	select RTW88_8822B
+	help
+	  Select this option will enable support for 8822BS chipset
+
+	  802.11ac SDIO wireless network adapter
+
 config RTW88_8822BU
 	tristate "Realtek 8822BU USB wireless network adapter"
 	depends on USB
@@ -64,6 +78,17 @@ config RTW88_8822CE
 
 	  802.11ac PCIe wireless network adapter
 
+config RTW88_8822CS
+	tristate "Realtek 8822CS SDIO wireless network adapter"
+	depends on MMC
+	select RTW88_CORE
+	select RTW88_SDIO
+	select RTW88_8822C
+	help
+	  Select this option will enable support for 8822CS chipset
+
+	  802.11ac SDIO wireless network adapter
+
 config RTW88_8822CU
 	tristate "Realtek 8822CU USB wireless network adapter"
 	depends on USB
@@ -108,6 +133,17 @@ config RTW88_8821CE
 
 	  802.11ac PCIe wireless network adapter
 
+config RTW88_8821CS
+	tristate "Realtek 8821CS SDIO wireless network adapter"
+	depends on MMC
+	select RTW88_CORE
+	select RTW88_SDIO
+	select RTW88_8821C
+	help
+	  Select this option will enable support for 8821CS chipset
+
+	  802.11ac SDIO wireless network adapter
+
 config RTW88_8821CU
 	tristate "Realtek 8821CU USB wireless network adapter"
 	depends on USB
diff --git a/drivers/net/wireless/realtek/rtw88/Makefile b/drivers/net/wireless/realtek/rtw88/Makefile
index fe7293ee87b4..82979b30ae8d 100644
--- a/drivers/net/wireless/realtek/rtw88/Makefile
+++ b/drivers/net/wireless/realtek/rtw88/Makefile
@@ -26,6 +26,9 @@ rtw88_8822b-objs		:= rtw8822b.o rtw8822b_table.o
 obj-$(CONFIG_RTW88_8822BE)	+= rtw88_8822be.o
 rtw88_8822be-objs		:= rtw8822be.o
 
+obj-$(CONFIG_RTW88_8822BS)	+= rtw88_8822bs.o
+rtw88_8822bs-objs		:= rtw8822bs.o
+
 obj-$(CONFIG_RTW88_8822BU)	+= rtw88_8822bu.o
 rtw88_8822bu-objs		:= rtw8822bu.o
 
@@ -35,6 +38,9 @@ rtw88_8822c-objs		:= rtw8822c.o rtw8822c_table.o
 obj-$(CONFIG_RTW88_8822CE)	+= rtw88_8822ce.o
 rtw88_8822ce-objs		:= rtw8822ce.o
 
+obj-$(CONFIG_RTW88_8822CS)	+= rtw88_8822cs.o
+rtw88_8822cs-objs		:= rtw8822cs.o
+
 obj-$(CONFIG_RTW88_8822CU)	+= rtw88_8822cu.o
 rtw88_8822cu-objs		:= rtw8822cu.o
 
@@ -53,11 +59,17 @@ rtw88_8821c-objs		:= rtw8821c.o rtw8821c_table.o
 obj-$(CONFIG_RTW88_8821CE)	+= rtw88_8821ce.o
 rtw88_8821ce-objs		:= rtw8821ce.o
 
+obj-$(CONFIG_RTW88_8821CS)	+= rtw88_8821cs.o
+rtw88_8821cs-objs		:= rtw8821cs.o
+
 obj-$(CONFIG_RTW88_8821CU)	+= rtw88_8821cu.o
 rtw88_8821cu-objs		:= rtw8821cu.o
 
 obj-$(CONFIG_RTW88_PCI)		+= rtw88_pci.o
 rtw88_pci-objs			:= pci.o
 
+obj-$(CONFIG_RTW88_SDIO)	+= rtw88_sdio.o
+rtw88_sdio-objs			:= sdio.o
+
 obj-$(CONFIG_RTW88_USB)		+= rtw88_usb.o
 rtw88_usb-objs			:= usb.o
diff --git a/drivers/net/wireless/realtek/rtw88/debug.h b/drivers/net/wireless/realtek/rtw88/debug.h
index 066792dd96af..a9149c6c2b48 100644
--- a/drivers/net/wireless/realtek/rtw88/debug.h
+++ b/drivers/net/wireless/realtek/rtw88/debug.h
@@ -24,6 +24,7 @@ enum rtw_debug_mask {
 	RTW_DBG_ADAPTIVITY	= 0x00008000,
 	RTW_DBG_HW_SCAN		= 0x00010000,
 	RTW_DBG_STATE		= 0x00020000,
+	RTW_DBG_SDIO		= 0x00040000,
 
 	RTW_DBG_ALL		= 0xffffffff
 };
diff --git a/drivers/net/wireless/realtek/rtw88/fw.c b/drivers/net/wireless/realtek/rtw88/fw.c
index 82295ac6402e..2a8ccc8a7f60 100644
--- a/drivers/net/wireless/realtek/rtw88/fw.c
+++ b/drivers/net/wireless/realtek/rtw88/fw.c
@@ -1393,6 +1393,10 @@ static void rtw_build_rsvd_page_iter(void *data, u8 *mac,
 	struct rtw_vif *rtwvif = (struct rtw_vif *)vif->drv_priv;
 	struct rtw_rsvd_page *rsvd_pkt;
 
+	/* AP not yet started, don't gather its rsvd pages */
+	if (vif->type == NL80211_IFTYPE_AP && !rtwdev->ap_active)
+		return;
+
 	list_for_each_entry(rsvd_pkt, &rtwvif->rsvd_page_list, vif_list) {
 		if (rsvd_pkt->type == RSVD_BEACON)
 			list_add(&rsvd_pkt->build_list,
@@ -1614,6 +1618,7 @@ void rtw_fw_update_beacon_work(struct work_struct *work)
 
 	mutex_lock(&rtwdev->mutex);
 	rtw_fw_download_rsvd_page(rtwdev);
+	rtw_send_rsvd_page_h2c(rtwdev);
 	mutex_unlock(&rtwdev->mutex);
 }
 
@@ -2155,11 +2160,19 @@ int rtw_hw_scan_offload(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
 	}
 	rtw_fw_set_scan_offload(rtwdev, &cs_option, rtwvif, &chan_list);
 out:
+	if (rtwdev->ap_active) {
+		ret = rtw_download_beacon(rtwdev);
+		if (ret)
+			rtw_err(rtwdev, "HW scan download beacon failed\n");
+	}
+
 	return ret;
 }
 
-void rtw_hw_scan_abort(struct rtw_dev *rtwdev, struct ieee80211_vif *vif)
+void rtw_hw_scan_abort(struct rtw_dev *rtwdev)
 {
+	struct ieee80211_vif *vif = rtwdev->scan_info.scanning_vif;
+
 	if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_SCAN_OFFLOAD))
 		return;
 
@@ -2244,6 +2257,7 @@ void rtw_hw_scan_chan_switch(struct rtw_dev *rtwdev, struct sk_buff *skb)
 		if (rtw_is_op_chan(rtwdev, chan)) {
 			rtw_store_op_chan(rtwdev, false);
 			ieee80211_wake_queues(rtwdev->hw);
+			rtw_core_enable_beacon(rtwdev, true);
 		}
 	} else if (id == RTW_SCAN_NOTIFY_ID_PRESWITCH) {
 		if (IS_CH_5G_BAND(chan)) {
@@ -2262,8 +2276,10 @@ void rtw_hw_scan_chan_switch(struct rtw_dev *rtwdev, struct sk_buff *skb)
 		 * if next channel is non-op channel.
 		 */
 		if (!rtw_is_op_chan(rtwdev, chan) &&
-		    rtw_is_op_chan(rtwdev, hal->current_channel))
+		    rtw_is_op_chan(rtwdev, hal->current_channel)) {
+			rtw_core_enable_beacon(rtwdev, false);
 			ieee80211_stop_queues(rtwdev->hw);
+		}
 	}
 
 	rtw_dbg(rtwdev, RTW_DBG_HW_SCAN,
diff --git a/drivers/net/wireless/realtek/rtw88/fw.h b/drivers/net/wireless/realtek/rtw88/fw.h
index 0a386e6d6e0d..397cbc3f6af6 100644
--- a/drivers/net/wireless/realtek/rtw88/fw.h
+++ b/drivers/net/wireless/realtek/rtw88/fw.h
@@ -868,5 +868,5 @@ int rtw_hw_scan_offload(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
 			bool enable);
 void rtw_hw_scan_status_report(struct rtw_dev *rtwdev, struct sk_buff *skb);
 void rtw_hw_scan_chan_switch(struct rtw_dev *rtwdev, struct sk_buff *skb);
-void rtw_hw_scan_abort(struct rtw_dev *rtwdev, struct ieee80211_vif *vif);
+void rtw_hw_scan_abort(struct rtw_dev *rtwdev);
 #endif
diff --git a/drivers/net/wireless/realtek/rtw88/mac.c b/drivers/net/wireless/realtek/rtw88/mac.c
index dae64901bac5..a168f36c38ec 100644
--- a/drivers/net/wireless/realtek/rtw88/mac.c
+++ b/drivers/net/wireless/realtek/rtw88/mac.c
@@ -7,6 +7,7 @@
 #include "reg.h"
 #include "fw.h"
 #include "debug.h"
+#include "sdio.h"
 
 void rtw_set_channel_mac(struct rtw_dev *rtwdev, u8 channel, u8 bw,
 			 u8 primary_ch_idx)
@@ -60,6 +61,7 @@ EXPORT_SYMBOL(rtw_set_channel_mac);
 
 static int rtw_mac_pre_system_cfg(struct rtw_dev *rtwdev)
 {
+	unsigned int retry;
 	u32 value32;
 	u8 value8;
 
@@ -77,6 +79,28 @@ static int rtw_mac_pre_system_cfg(struct rtw_dev *rtwdev)
 	case RTW_HCI_TYPE_PCIE:
 		rtw_write32_set(rtwdev, REG_HCI_OPT_CTRL, BIT_USB_SUS_DIS);
 		break;
+	case RTW_HCI_TYPE_SDIO:
+		rtw_write8_clr(rtwdev, REG_SDIO_HSUS_CTRL, BIT_HCI_SUS_REQ);
+
+		for (retry = 0; retry < RTW_PWR_POLLING_CNT; retry++) {
+			if (rtw_read8(rtwdev, REG_SDIO_HSUS_CTRL) & BIT_HCI_RESUME_RDY)
+				break;
+
+			usleep_range(10, 50);
+		}
+
+		if (retry == RTW_PWR_POLLING_CNT) {
+			rtw_err(rtwdev, "failed to poll REG_SDIO_HSUS_CTRL[1]");
+			return -ETIMEDOUT;
+		}
+
+		if (rtw_sdio_is_sdio30_supported(rtwdev))
+			rtw_write8_set(rtwdev, REG_HCI_OPT_CTRL + 2,
+				       BIT_SDIO_PAD_E5 >> 16);
+		else
+			rtw_write8_clr(rtwdev, REG_HCI_OPT_CTRL + 2,
+				       BIT_SDIO_PAD_E5 >> 16);
+		break;
 	case RTW_HCI_TYPE_USB:
 		break;
 	default:
@@ -222,6 +246,9 @@ static int rtw_pwr_seq_parser(struct rtw_dev *rtwdev,
 	case RTW_HCI_TYPE_USB:
 		intf_mask = RTW_PWR_INTF_USB_MSK;
 		break;
+	case RTW_HCI_TYPE_SDIO:
+		intf_mask = RTW_PWR_INTF_SDIO_MSK;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -233,7 +260,7 @@ static int rtw_pwr_seq_parser(struct rtw_dev *rtwdev,
 
 		ret = rtw_sub_pwr_seq_parser(rtwdev, intf_mask, cut_mask, cmd);
 		if (ret)
-			return -EBUSY;
+			return ret;
 
 		idx++;
 	} while (1);
@@ -245,8 +272,10 @@ static int rtw_mac_power_switch(struct rtw_dev *rtwdev, bool pwr_on)
 {
 	const struct rtw_chip_info *chip = rtwdev->chip;
 	const struct rtw_pwr_seq_cmd **pwr_seq;
+	u32 imr = 0;
 	u8 rpwm;
 	bool cur_pwr;
+	int ret;
 
 	if (rtw_chip_wcpu_11ac(rtwdev)) {
 		rpwm = rtw_read8(rtwdev, rtwdev->hci.rpwm_addr);
@@ -269,16 +298,24 @@ static int rtw_mac_power_switch(struct rtw_dev *rtwdev, bool pwr_on)
 	if (pwr_on == cur_pwr)
 		return -EALREADY;
 
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_SDIO) {
+		imr = rtw_read32(rtwdev, REG_SDIO_HIMR);
+		rtw_write32(rtwdev, REG_SDIO_HIMR, 0);
+	}
+
+	if (!pwr_on)
+		clear_bit(RTW_FLAG_POWERON, rtwdev->flags);
+
 	pwr_seq = pwr_on ? chip->pwr_on_seq : chip->pwr_off_seq;
-	if (rtw_pwr_seq_parser(rtwdev, pwr_seq))
-		return -EINVAL;
+	ret = rtw_pwr_seq_parser(rtwdev, pwr_seq);
 
-	if (pwr_on)
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_SDIO)
+		rtw_write32(rtwdev, REG_SDIO_HIMR, imr);
+
+	if (!ret && pwr_on)
 		set_bit(RTW_FLAG_POWERON, rtwdev->flags);
-	else
-		clear_bit(RTW_FLAG_POWERON, rtwdev->flags);
 
-	return 0;
+	return ret;
 }
 
 static int __rtw_mac_init_system_cfg(struct rtw_dev *rtwdev)
@@ -449,6 +486,9 @@ static void download_firmware_reg_backup(struct rtw_dev *rtwdev,
 	rtw_write16(rtwdev, REG_FIFOPAGE_INFO_1, 0x200);
 	rtw_write32(rtwdev, REG_RQPN_CTRL_2, bckp[bckp_idx - 1].val);
 
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_SDIO)
+		rtw_read32(rtwdev, REG_SDIO_FREE_TXPG);
+
 	/* Disable beacon related functions */
 	tmp = rtw_read8(rtwdev, REG_BCN_CTRL);
 	bckp[bckp_idx].len = 1;
@@ -1040,6 +1080,9 @@ static int txdma_queue_mapping(struct rtw_dev *rtwdev)
 		else
 			return -EINVAL;
 		break;
+	case RTW_HCI_TYPE_SDIO:
+		rqpn = &chip->rqpn_table[0];
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -1058,8 +1101,12 @@ static int txdma_queue_mapping(struct rtw_dev *rtwdev)
 	if (rtw_chip_wcpu_11ac(rtwdev))
 		rtw_write32(rtwdev, REG_H2CQ_CSR, BIT_H2CQ_FULL);
 
-	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB)
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_SDIO) {
+		rtw_read32(rtwdev, REG_SDIO_FREE_TXPG);
+		rtw_write32(rtwdev, REG_SDIO_TX_CTRL, 0);
+	} else if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB) {
 		rtw_write8_set(rtwdev, REG_TXDMA_PQ_MAP, BIT_RXDMA_ARBBW_EN);
+	}
 
 	return 0;
 }
@@ -1072,7 +1119,7 @@ static int set_trx_fifo_info(struct rtw_dev *rtwdev)
 	u8 csi_buf_pg_num = chip->csi_buf_pg_num;
 
 	/* config rsvd page num */
-	fifo->rsvd_drv_pg_num = 8;
+	fifo->rsvd_drv_pg_num = chip->rsvd_drv_pg_num;
 	fifo->txff_pg_num = chip->txff_size >> 7;
 	if (rtw_chip_wcpu_11n(rtwdev))
 		fifo->rsvd_pg_num = fifo->rsvd_drv_pg_num;
@@ -1202,6 +1249,9 @@ static int priority_queue_cfg(struct rtw_dev *rtwdev)
 		else
 			return -EINVAL;
 		break;
+	case RTW_HCI_TYPE_SDIO:
+		pg_tbl = &chip->page_table[0];
+		break;
 	default:
 		return -EINVAL;
 	}
diff --git a/drivers/net/wireless/realtek/rtw88/mac.h b/drivers/net/wireless/realtek/rtw88/mac.h
index 3172aa5ac4de..58c3dccc14bb 100644
--- a/drivers/net/wireless/realtek/rtw88/mac.h
+++ b/drivers/net/wireless/realtek/rtw88/mac.h
@@ -7,7 +7,6 @@
 
 #define RTW_HW_PORT_NUM		5
 #define cut_version_to_mask(cut) (0x1 << ((cut) + 1))
-#define SDIO_LOCAL_OFFSET	0x10250000
 #define DDMA_POLLING_COUNT	1000
 #define C2H_PKT_BUF		256
 #define REPORT_BUF		128
diff --git a/drivers/net/wireless/realtek/rtw88/mac80211.c b/drivers/net/wireless/realtek/rtw88/mac80211.c
index 3b92ac611d3f..7aa6edad0d01 100644
--- a/drivers/net/wireless/realtek/rtw88/mac80211.c
+++ b/drivers/net/wireless/realtek/rtw88/mac80211.c
@@ -155,25 +155,30 @@ static int rtw_ops_add_interface(struct ieee80211_hw *hw,
 	struct rtw_vif *rtwvif = (struct rtw_vif *)vif->drv_priv;
 	enum rtw_net_type net_type;
 	u32 config = 0;
-	u8 port = 0;
+	u8 port;
 	u8 bcn_ctrl = 0;
 
 	if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_BCN_FILTER))
 		vif->driver_flags |= IEEE80211_VIF_BEACON_FILTER |
 				     IEEE80211_VIF_SUPPORTS_CQM_RSSI;
-	rtwvif->port = port;
 	rtwvif->stats.tx_unicast = 0;
 	rtwvif->stats.rx_unicast = 0;
 	rtwvif->stats.tx_cnt = 0;
 	rtwvif->stats.rx_cnt = 0;
 	rtwvif->scan_req = NULL;
 	memset(&rtwvif->bfee, 0, sizeof(struct rtw_bfee));
-	rtwvif->conf = &rtw_vif_port[port];
 	rtw_txq_init(rtwdev, vif->txq);
 	INIT_LIST_HEAD(&rtwvif->rsvd_page_list);
 
 	mutex_lock(&rtwdev->mutex);
 
+	port = find_first_zero_bit(rtwdev->hw_port, RTW_PORT_NUM);
+	if (port >= RTW_PORT_NUM)
+		return -EINVAL;
+	set_bit(port, rtwdev->hw_port);
+
+	rtwvif->port = port;
+	rtwvif->conf = &rtw_vif_port[port];
 	rtw_leave_lps_deep(rtwdev);
 
 	switch (vif->type) {
@@ -195,6 +200,7 @@ static int rtw_ops_add_interface(struct ieee80211_hw *hw,
 		break;
 	default:
 		WARN_ON(1);
+		clear_bit(rtwvif->port, rtwdev->hw_port);
 		mutex_unlock(&rtwdev->mutex);
 		return -EINVAL;
 	}
@@ -206,6 +212,7 @@ static int rtw_ops_add_interface(struct ieee80211_hw *hw,
 	rtwvif->bcn_ctrl = bcn_ctrl;
 	config |= PORT_SET_BCN_CTRL;
 	rtw_vif_port_config(rtwdev, rtwvif, config);
+	rtw_core_port_switch(rtwdev, vif);
 
 	mutex_unlock(&rtwdev->mutex);
 
@@ -236,6 +243,7 @@ static void rtw_ops_remove_interface(struct ieee80211_hw *hw,
 	rtwvif->bcn_ctrl = 0;
 	config |= PORT_SET_BCN_CTRL;
 	rtw_vif_port_config(rtwdev, rtwvif, config);
+	clear_bit(rtwvif->port, rtwdev->hw_port);
 
 	mutex_unlock(&rtwdev->mutex);
 }
@@ -385,7 +393,8 @@ static void rtw_ops_bss_info_changed(struct ieee80211_hw *hw,
 			 * when disconnected by peer
 			 */
 			if (test_bit(RTW_FLAG_SCANNING, rtwdev->flags))
-				rtw_hw_scan_abort(rtwdev, vif);
+				rtw_hw_scan_abort(rtwdev);
+
 		}
 
 		config |= PORT_SET_NET_TYPE;
@@ -395,7 +404,7 @@ static void rtw_ops_bss_info_changed(struct ieee80211_hw *hw,
 	if (changed & BSS_CHANGED_BSSID) {
 		ether_addr_copy(rtwvif->bssid, conf->bssid);
 		config |= PORT_SET_BSSID;
-		if (is_zero_ether_addr(rtwvif->bssid))
+		if (!rtw_core_check_sta_active(rtwdev))
 			rtw_clear_op_chan(rtwdev);
 		else
 			rtw_store_op_chan(rtwdev, true);
@@ -409,6 +418,7 @@ static void rtw_ops_bss_info_changed(struct ieee80211_hw *hw,
 	if (changed & BSS_CHANGED_BEACON) {
 		rtw_set_dtim_period(rtwdev, conf->dtim_period);
 		rtw_fw_download_rsvd_page(rtwdev);
+		rtw_send_rsvd_page_h2c(rtwdev);
 	}
 
 	if (changed & BSS_CHANGED_BEACON_ENABLED) {
@@ -441,12 +451,27 @@ static int rtw_ops_start_ap(struct ieee80211_hw *hw,
 	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	mutex_lock(&rtwdev->mutex);
+	rtwdev->ap_active = true;
+	rtw_store_op_chan(rtwdev, true);
 	chip->ops->phy_calibration(rtwdev);
 	mutex_unlock(&rtwdev->mutex);
 
 	return 0;
 }
 
+static void rtw_ops_stop_ap(struct ieee80211_hw *hw,
+			    struct ieee80211_vif *vif,
+			    struct ieee80211_bss_conf *link_conf)
+{
+	struct rtw_dev *rtwdev = hw->priv;
+
+	mutex_lock(&rtwdev->mutex);
+	rtwdev->ap_active = false;
+	if (!rtw_core_check_sta_active(rtwdev))
+		rtw_clear_op_chan(rtwdev);
+	mutex_unlock(&rtwdev->mutex);
+}
+
 static int rtw_ops_conf_tx(struct ieee80211_hw *hw,
 			   struct ieee80211_vif *vif,
 			   unsigned int link_id, u16 ac,
@@ -849,7 +874,7 @@ static int rtw_ops_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	rtw_hw_scan_start(rtwdev, vif, req);
 	ret = rtw_hw_scan_offload(rtwdev, vif, true);
 	if (ret) {
-		rtw_hw_scan_abort(rtwdev, vif);
+		rtw_hw_scan_abort(rtwdev);
 		rtw_err(rtwdev, "HW scan failed with status: %d\n", ret);
 	}
 	mutex_unlock(&rtwdev->mutex);
@@ -869,7 +894,7 @@ static void rtw_ops_cancel_hw_scan(struct ieee80211_hw *hw,
 		return;
 
 	mutex_lock(&rtwdev->mutex);
-	rtw_hw_scan_abort(rtwdev, vif);
+	rtw_hw_scan_abort(rtwdev);
 	mutex_unlock(&rtwdev->mutex);
 }
 
@@ -908,6 +933,7 @@ const struct ieee80211_ops rtw_ops = {
 	.configure_filter	= rtw_ops_configure_filter,
 	.bss_info_changed	= rtw_ops_bss_info_changed,
 	.start_ap		= rtw_ops_start_ap,
+	.stop_ap		= rtw_ops_stop_ap,
 	.conf_tx		= rtw_ops_conf_tx,
 	.sta_add		= rtw_ops_sta_add,
 	.sta_remove		= rtw_ops_sta_remove,
diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c
index b2e78737bd5d..5bf6b4581557 100644
--- a/drivers/net/wireless/realtek/rtw88/main.c
+++ b/drivers/net/wireless/realtek/rtw88/main.c
@@ -18,6 +18,7 @@
 #include "debug.h"
 #include "bf.h"
 #include "sar.h"
+#include "sdio.h"
 
 bool rtw_disable_lps_deep_mode;
 EXPORT_SYMBOL(rtw_disable_lps_deep_mode);
@@ -102,6 +103,26 @@ static struct ieee80211_rate rtw_ratetable[] = {
 	{.bitrate = 540, .hw_value = 0x0b,},
 };
 
+static const struct ieee80211_iface_limit rtw_iface_limits[] = {
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_STATION),
+	},
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_AP),
+	}
+};
+
+static const struct ieee80211_iface_combination rtw_iface_combs[] = {
+	{
+		.limits = rtw_iface_limits,
+		.n_limits = ARRAY_SIZE(rtw_iface_limits),
+		.max_interfaces = 2,
+		.num_different_channels = 1,
+	}
+};
+
 u16 rtw_desc_to_bitrate(u8 desc_rate)
 {
 	struct ieee80211_rate rate;
@@ -256,7 +277,7 @@ static void rtw_watch_dog_work(struct work_struct *work)
 	 * threshold.
 	 */
 	if (rtwdev->ps_enabled && data.rtwvif && !ps_active &&
-	    !rtwdev->beacon_loss)
+	    !rtwdev->beacon_loss && !rtwdev->ap_active)
 		rtw_enter_lps(rtwdev, data.rtwvif->port);
 
 	rtwdev->watch_dog_cnt++;
@@ -609,6 +630,7 @@ free:
 	rcu_read_unlock();
 	rtw_iterate_stas_atomic(rtwdev, rtw_reset_sta_iter, rtwdev);
 	rtw_iterate_vifs_atomic(rtwdev, rtw_reset_vif_iter, rtwdev);
+	bitmap_zero(rtwdev->hw_port, RTW_PORT_NUM);
 	rtw_enter_ips(rtwdev);
 }
 
@@ -828,6 +850,9 @@ void rtw_set_channel(struct rtw_dev *rtwdev)
 
 	rtw_update_channel(rtwdev, center_chan, primary_chan, band, bandwidth);
 
+	if (rtwdev->scan_info.op_chan)
+		rtw_store_op_chan(rtwdev, true);
+
 	chip->ops->set_channel(rtwdev, center_chan, bandwidth,
 			       hal->current_primary_channel_index);
 
@@ -1785,6 +1810,10 @@ static int rtw_chip_parameter_setup(struct rtw_dev *rtwdev)
 		rtwdev->hci.rpwm_addr = 0x03d9;
 		rtwdev->hci.cpwm_addr = 0x03da;
 		break;
+	case RTW_HCI_TYPE_SDIO:
+		rtwdev->hci.rpwm_addr = REG_SDIO_HRPWM1;
+		rtwdev->hci.cpwm_addr = REG_SDIO_HCPWM1_V2;
+		break;
 	case RTW_HCI_TYPE_USB:
 		rtwdev->hci.rpwm_addr = 0xfe58;
 		rtwdev->hci.cpwm_addr = 0xfe57;
@@ -1979,7 +2008,7 @@ static int rtw_chip_board_info_setup(struct rtw_dev *rtwdev)
 	if (!rfe_def)
 		return -ENODEV;
 
-	rtw_phy_setup_phy_cond(rtwdev, 0);
+	rtw_phy_setup_phy_cond(rtwdev, hal->pkg_type);
 
 	rtw_phy_init_tx_power(rtwdev);
 	if (rfe_def->agc_btg_tbl)
@@ -2158,9 +2187,11 @@ int rtw_register_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw)
 	int max_tx_headroom = 0;
 	int ret;
 
-	/* TODO: USB & SDIO may need extra room? */
 	max_tx_headroom = rtwdev->chip->tx_pkt_desc_sz;
 
+	if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_SDIO)
+		max_tx_headroom += RTW_SDIO_DATA_PTR_ALIGN;
+
 	hw->extra_tx_headroom = max_tx_headroom;
 	hw->queues = IEEE80211_NUM_ACS;
 	hw->txq_data_size = sizeof(struct rtw_txq);
@@ -2194,6 +2225,11 @@ int rtw_register_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw)
 	hw->wiphy->max_scan_ssids = RTW_SCAN_MAX_SSIDS;
 	hw->wiphy->max_scan_ie_len = rtw_get_max_scan_ie_len(rtwdev);
 
+	if (rtwdev->chip->id == RTW_CHIP_TYPE_8822C) {
+		hw->wiphy->iface_combinations = rtw_iface_combs;
+		hw->wiphy->n_iface_combinations = ARRAY_SIZE(rtw_iface_combs);
+	}
+
 	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
 	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_SCAN_RANDOM_SN);
 	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_SET_SCAN_DWELL);
@@ -2243,6 +2279,121 @@ void rtw_unregister_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw)
 }
 EXPORT_SYMBOL(rtw_unregister_hw);
 
+static
+void rtw_swap_reg_nbytes(struct rtw_dev *rtwdev, const struct rtw_hw_reg *reg1,
+			 const struct rtw_hw_reg *reg2, u8 nbytes)
+{
+	u8 i;
+
+	for (i = 0; i < nbytes; i++) {
+		u8 v1 = rtw_read8(rtwdev, reg1->addr + i);
+		u8 v2 = rtw_read8(rtwdev, reg2->addr + i);
+
+		rtw_write8(rtwdev, reg1->addr + i, v2);
+		rtw_write8(rtwdev, reg2->addr + i, v1);
+	}
+}
+
+static
+void rtw_swap_reg_mask(struct rtw_dev *rtwdev, const struct rtw_hw_reg *reg1,
+		       const struct rtw_hw_reg *reg2)
+{
+	u32 v1, v2;
+
+	v1 = rtw_read32_mask(rtwdev, reg1->addr, reg1->mask);
+	v2 = rtw_read32_mask(rtwdev, reg2->addr, reg2->mask);
+	rtw_write32_mask(rtwdev, reg2->addr, reg2->mask, v1);
+	rtw_write32_mask(rtwdev, reg1->addr, reg1->mask, v2);
+}
+
+struct rtw_iter_port_switch_data {
+	struct rtw_dev *rtwdev;
+	struct rtw_vif *rtwvif_ap;
+};
+
+static void rtw_port_switch_iter(void *data, u8 *mac, struct ieee80211_vif *vif)
+{
+	struct rtw_iter_port_switch_data *iter_data = data;
+	struct rtw_dev *rtwdev = iter_data->rtwdev;
+	struct rtw_vif *rtwvif_target = (struct rtw_vif *)vif->drv_priv;
+	struct rtw_vif *rtwvif_ap = iter_data->rtwvif_ap;
+	const struct rtw_hw_reg *reg1, *reg2;
+
+	if (rtwvif_target->port != RTW_PORT_0)
+		return;
+
+	rtw_dbg(rtwdev, RTW_DBG_STATE, "AP port switch from %d -> %d\n",
+		rtwvif_ap->port, rtwvif_target->port);
+
+	reg1 = &rtwvif_ap->conf->net_type;
+	reg2 = &rtwvif_target->conf->net_type;
+	rtw_swap_reg_mask(rtwdev, reg1, reg2);
+
+	reg1 = &rtwvif_ap->conf->mac_addr;
+	reg2 = &rtwvif_target->conf->mac_addr;
+	rtw_swap_reg_nbytes(rtwdev, reg1, reg2, ETH_ALEN);
+
+	reg1 = &rtwvif_ap->conf->bssid;
+	reg2 = &rtwvif_target->conf->bssid;
+	rtw_swap_reg_nbytes(rtwdev, reg1, reg2, ETH_ALEN);
+
+	reg1 = &rtwvif_ap->conf->bcn_ctrl;
+	reg2 = &rtwvif_target->conf->bcn_ctrl;
+	rtw_swap_reg_nbytes(rtwdev, reg1, reg2, 1);
+
+	swap(rtwvif_target->port, rtwvif_ap->port);
+	swap(rtwvif_target->conf, rtwvif_ap->conf);
+}
+
+void rtw_core_port_switch(struct rtw_dev *rtwdev, struct ieee80211_vif *vif)
+{
+	struct rtw_vif *rtwvif = (struct rtw_vif *)vif->drv_priv;
+	struct rtw_iter_port_switch_data iter_data;
+
+	if (vif->type != NL80211_IFTYPE_AP || rtwvif->port == RTW_PORT_0)
+		return;
+
+	iter_data.rtwdev = rtwdev;
+	iter_data.rtwvif_ap = rtwvif;
+	rtw_iterate_vifs(rtwdev, rtw_port_switch_iter, &iter_data);
+}
+
+static void rtw_check_sta_active_iter(void *data, u8 *mac,
+				      struct ieee80211_vif *vif)
+{
+	struct rtw_vif *rtwvif = (struct rtw_vif *)vif->drv_priv;
+	bool *active = data;
+
+	if (*active)
+		return;
+
+	if (vif->type != NL80211_IFTYPE_STATION)
+		return;
+
+	if (vif->cfg.assoc || !is_zero_ether_addr(rtwvif->bssid))
+		*active = true;
+}
+
+bool rtw_core_check_sta_active(struct rtw_dev *rtwdev)
+{
+	bool sta_active = false;
+
+	rtw_iterate_vifs(rtwdev, rtw_check_sta_active_iter, &sta_active);
+
+	return rtwdev->ap_active || sta_active;
+}
+
+void rtw_core_enable_beacon(struct rtw_dev *rtwdev, bool enable)
+{
+	if (!rtwdev->ap_active)
+		return;
+
+	if (enable)
+		rtw_write32_set(rtwdev, REG_BCN_CTRL, BIT_EN_BCN_FUNCTION);
+	else
+		rtw_write32_clr(rtwdev, REG_BCN_CTRL, BIT_EN_BCN_FUNCTION);
+}
+
 MODULE_AUTHOR("Realtek Corporation");
 MODULE_DESCRIPTION("Realtek 802.11ac wireless core module");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h
index d4a53d556745..a563285e90ed 100644
--- a/drivers/net/wireless/realtek/rtw88/main.h
+++ b/drivers/net/wireless/realtek/rtw88/main.h
@@ -88,7 +88,7 @@ enum rtw_supported_band {
 	RTW_BAND_60G = BIT(NL80211_BAND_60GHZ),
 };
 
-/* now, support upto 80M bw */
+/* now, support up to 80M bw */
 #define RTW_MAX_CHANNEL_WIDTH RTW_CHANNEL_WIDTH_80
 
 enum rtw_bandwidth {
@@ -395,6 +395,15 @@ enum rtw_snr {
 	RTW_SNR_NUM
 };
 
+enum rtw_port {
+	RTW_PORT_0 = 0,
+	RTW_PORT_1 = 1,
+	RTW_PORT_2 = 2,
+	RTW_PORT_3 = 3,
+	RTW_PORT_4 = 4,
+	RTW_PORT_NUM
+};
+
 enum rtw_wow_flags {
 	RTW_WOW_FLAG_EN_MAGIC_PKT,
 	RTW_WOW_FLAG_EN_REKEY_PKT,
@@ -1168,6 +1177,7 @@ struct rtw_chip_info {
 	u32 txff_size;
 	u32 rxff_size;
 	u32 fw_rxff_size;
+	u16 rsvd_drv_pg_num;
 	u8 band;
 	u8 page_size;
 	u8 csi_buf_pg_num;
@@ -1871,7 +1881,7 @@ enum rtw_sar_bands {
 	RTW_SAR_BAND_NR,
 };
 
-/* the union is reserved for other knids of SAR sources
+/* the union is reserved for other kinds of SAR sources
  * which might not re-use same format with array common.
  */
 union rtw_sar_cfg {
@@ -1890,7 +1900,9 @@ struct rtw_hal {
 	u8 cut_version;
 	u8 mp_chip;
 	u8 oem_id;
+	u8 pkg_type;
 	struct rtw_phy_cond phy_cond;
+	bool rfe_btg;
 
 	u8 ps_mode;
 	u8 current_channel;
@@ -2020,7 +2032,7 @@ struct rtw_dev {
 	struct rtw_tx_report tx_report;
 
 	struct {
-		/* incicate the mail box to use with fw */
+		/* indicate the mail box to use with fw */
 		u8 last_box_num;
 		u32 seq;
 	} h2c;
@@ -2036,6 +2048,7 @@ struct rtw_dev {
 	u8 sta_cnt;
 	u32 rts_threshold;
 
+	DECLARE_BITMAP(hw_port, RTW_PORT_NUM);
 	DECLARE_BITMAP(mac_id_map, RTW_MAX_MAC_ID_NUM);
 	DECLARE_BITMAP(flags, NUM_OF_RTW_FLAGS);
 
@@ -2047,6 +2060,7 @@ struct rtw_dev {
 
 	bool need_rfk;
 	struct completion fw_scan_density;
+	bool ap_active;
 
 	/* hci related data, must be last */
 	u8 priv[] __aligned(sizeof(void *));
@@ -2188,4 +2202,7 @@ void rtw_set_txrx_1ss(struct rtw_dev *rtwdev, bool config_1ss);
 void rtw_update_channel(struct rtw_dev *rtwdev, u8 center_channel,
 			u8 primary_channel, enum rtw_supported_band band,
 			enum rtw_bandwidth bandwidth);
+void rtw_core_port_switch(struct rtw_dev *rtwdev, struct ieee80211_vif *vif);
+bool rtw_core_check_sta_active(struct rtw_dev *rtwdev);
+void rtw_core_enable_beacon(struct rtw_dev *rtwdev, bool enable);
 #endif
diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c
index b4bd831c9845..672ddde80816 100644
--- a/drivers/net/wireless/realtek/rtw88/pci.c
+++ b/drivers/net/wireless/realtek/rtw88/pci.c
@@ -89,13 +89,6 @@ static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val)
 	writel(val, rtwpci->mmap + addr);
 }
 
-static inline void *rtw_pci_get_tx_desc(struct rtw_pci_tx_ring *tx_ring, u8 idx)
-{
-	int offset = tx_ring->r.desc_size * idx;
-
-	return tx_ring->r.head + offset;
-}
-
 static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev,
 				      struct rtw_pci_tx_ring *tx_ring)
 {
@@ -1552,7 +1545,6 @@ static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev)
 
 static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev)
 {
-	pci_clear_master(pdev);
 	pci_disable_device(pdev);
 }
 
diff --git a/drivers/net/wireless/realtek/rtw88/reg.h b/drivers/net/wireless/realtek/rtw88/reg.h
index 8852b24d6c2a..2a2ae2081f34 100644
--- a/drivers/net/wireless/realtek/rtw88/reg.h
+++ b/drivers/net/wireless/realtek/rtw88/reg.h
@@ -87,6 +87,7 @@
 #define BIT_LTE_MUX_CTRL_PATH	BIT(26)
 #define REG_HCI_OPT_CTRL	0x0074
 #define BIT_USB_SUS_DIS		BIT(8)
+#define BIT_SDIO_PAD_E5		BIT(18)
 
 #define REG_AFE_CTRL_4		0x0078
 #define BIT_CK320M_AFE_EN	BIT(4)
@@ -185,6 +186,9 @@
 	(((x) & BIT_MASK_TXDMA_VIQ_MAP) << BIT_SHIFT_TXDMA_VIQ_MAP)
 #define REG_TXDMA_PQ_MAP	0x010C
 #define BIT_RXDMA_ARBBW_EN	BIT(0)
+#define BIT_RXSHFT_EN		BIT(1)
+#define BIT_RXDMA_AGG_EN	BIT(2)
+#define BIT_TXDMA_BW_EN		BIT(3)
 #define BIT_SHIFT_TXDMA_BEQ_MAP	8
 #define BIT_MASK_TXDMA_BEQ_MAP	0x3
 #define BIT_TXDMA_BEQ_MAP(x)                                                   \
@@ -283,10 +287,18 @@
 #define REG_H2C_TAIL		0x0248
 #define REG_H2C_READ_ADDR	0x024C
 #define REG_H2C_INFO		0x0254
+#define REG_RXDMA_AGG_PG_TH	0x0280
+#define BIT_RXDMA_AGG_PG_TH	GENMASK(7, 0)
+#define BIT_DMA_AGG_TO_V1	GENMASK(15, 8)
+#define BIT_EN_PRE_CALC		BIT(29)
 #define REG_RXPKT_NUM		0x0284
 #define BIT_RXDMA_REQ		BIT(19)
 #define BIT_RW_RELEASE		BIT(18)
 #define BIT_RXDMA_IDLE		BIT(17)
+#define REG_RXDMA_STATUS	0x0288
+#define REG_RXDMA_DPR		0x028C
+#define REG_RXDMA_MODE		0x0290
+#define BIT_DMA_MODE		BIT(1)
 #define REG_RXPKTNUM		0x02B0
 
 #define REG_INT_MIG		0x0304
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723d.c b/drivers/net/wireless/realtek/rtw88/rtw8723d.c
index 2d2f768bae2e..06e7454c9ca6 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8723d.c
+++ b/drivers/net/wireless/realtek/rtw88/rtw8723d.c
@@ -2743,6 +2743,7 @@ const struct rtw_chip_info rtw8723d_hw_spec = {
 	.ptct_efuse_size = 96 + 1,
 	.txff_size = 32768,
 	.rxff_size = 16384,
+	.rsvd_drv_pg_num = 8,
 	.txgi_factor = 1,
 	.is_pwr_by_rate_dec = true,
 	.max_power_index = 0x3f,
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.c b/drivers/net/wireless/realtek/rtw88/rtw8821c.c
index 17f800f6efbd..adf224618a2a 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8821c.c
+++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.c
@@ -32,6 +32,12 @@ static void rtw8821cu_efuse_parsing(struct rtw_efuse *efuse,
 	ether_addr_copy(efuse->addr, map->u.mac_addr);
 }
 
+static void rtw8821cs_efuse_parsing(struct rtw_efuse *efuse,
+				    struct rtw8821c_efuse *map)
+{
+	ether_addr_copy(efuse->addr, map->s.mac_addr);
+}
+
 enum rtw8821ce_rf_set {
 	SWITCH_TO_BTG,
 	SWITCH_TO_WLG,
@@ -41,13 +47,14 @@ enum rtw8821ce_rf_set {
 
 static int rtw8821c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 {
+	struct rtw_hal *hal = &rtwdev->hal;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 	struct rtw8821c_efuse *map;
 	int i;
 
 	map = (struct rtw8821c_efuse *)log_map;
 
-	efuse->rfe_option = map->rfe_option;
+	efuse->rfe_option = map->rfe_option & 0x1f;
 	efuse->rf_board_option = map->rf_board_option;
 	efuse->crystal_cap = map->xtal_k;
 	efuse->pa_type_2g = map->pa_type;
@@ -64,6 +71,19 @@ static int rtw8821c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 	efuse->tx_bb_swing_setting_2g = map->tx_bb_swing_setting_2g;
 	efuse->tx_bb_swing_setting_5g = map->tx_bb_swing_setting_5g;
 
+	hal->pkg_type = map->rfe_option & BIT(5) ? 1 : 0;
+
+	switch (efuse->rfe_option) {
+	case 0x2:
+	case 0x4:
+	case 0x7:
+	case 0xa:
+	case 0xc:
+	case 0xf:
+		hal->rfe_btg = true;
+		break;
+	}
+
 	for (i = 0; i < 4; i++)
 		efuse->txpwr_idx_table[i] = map->txpwr_idx_table[i];
 
@@ -77,6 +97,9 @@ static int rtw8821c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 	case RTW_HCI_TYPE_USB:
 		rtw8821cu_efuse_parsing(efuse, map);
 		break;
+	case RTW_HCI_TYPE_SDIO:
+		rtw8821cs_efuse_parsing(efuse, map);
+		break;
 	default:
 		/* unsupported now */
 		return -ENOTSUPP;
@@ -286,6 +309,7 @@ static void rtw8821c_switch_rf_set(struct rtw_dev *rtwdev, u8 rf_set)
 
 static void rtw8821c_set_channel_rf(struct rtw_dev *rtwdev, u8 channel, u8 bw)
 {
+	struct rtw_hal *hal = &rtwdev->hal;
 	u32 rf_reg18;
 
 	rf_reg18 = rtw_read_rf(rtwdev, RF_PATH_A, 0x18, RFREG_MASK);
@@ -317,11 +341,10 @@ static void rtw8821c_set_channel_rf(struct rtw_dev *rtwdev, u8 channel, u8 bw)
 	}
 
 	if (channel <= 14) {
-		if (rtwdev->efuse.rfe_option == 0)
-			rtw8821c_switch_rf_set(rtwdev, SWITCH_TO_WLG);
-		else if (rtwdev->efuse.rfe_option == 2 ||
-			 rtwdev->efuse.rfe_option == 4)
+		if (hal->rfe_btg)
 			rtw8821c_switch_rf_set(rtwdev, SWITCH_TO_BTG);
+		else
+			rtw8821c_switch_rf_set(rtwdev, SWITCH_TO_WLG);
 		rtw_write_rf(rtwdev, RF_PATH_A, RF_LUTDBG, BIT(6), 0x1);
 		rtw_write_rf(rtwdev, RF_PATH_A, 0x64, 0xf, 0xf);
 	} else {
@@ -1537,7 +1560,6 @@ static const struct rtw_rfe_def rtw8821c_rfe_defs[] = {
 	[2] = RTW_DEF_RFE_EXT(8821c, 0, 0, 2),
 	[4] = RTW_DEF_RFE_EXT(8821c, 0, 0, 2),
 	[6] = RTW_DEF_RFE(8821c, 0, 0),
-	[34] = RTW_DEF_RFE(8821c, 0, 0),
 };
 
 static struct rtw_hw_reg rtw8821c_dig[] = {
@@ -1911,6 +1933,7 @@ const struct rtw_chip_info rtw8821c_hw_spec = {
 	.ptct_efuse_size = 96,
 	.txff_size = 65536,
 	.rxff_size = 16384,
+	.rsvd_drv_pg_num = 8,
 	.txgi_factor = 1,
 	.is_pwr_by_rate_dec = true,
 	.max_power_index = 0x3f,
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.h b/drivers/net/wireless/realtek/rtw88/rtw8821c.h
index 1c81260f3a54..fcff31688c45 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8821c.h
+++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.h
@@ -65,6 +65,11 @@ struct rtw8821ce_efuse {
 	u8 res7;
 };
 
+struct rtw8821cs_efuse {
+	u8 res4[0x4a];			/* 0xd0 */
+	u8 mac_addr[ETH_ALEN];		/* 0x11a */
+} __packed;
+
 struct rtw8821c_efuse {
 	__le16 rtl_id;
 	u8 res0[0x0e];
@@ -94,6 +99,7 @@ struct rtw8821c_efuse {
 	union {
 		struct rtw8821ce_efuse e;
 		struct rtw8821cu_efuse u;
+		struct rtw8821cs_efuse s;
 	};
 };
 
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821cs.c b/drivers/net/wireless/realtek/rtw88/rtw8821cs.c
new file mode 100644
index 000000000000..a359413369a4
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw88/rtw8821cs.c
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ */
+
+#include <linux/mmc/sdio_func.h>
+#include <linux/mmc/sdio_ids.h>
+#include <linux/module.h>
+#include "main.h"
+#include "rtw8821c.h"
+#include "sdio.h"
+
+static const struct sdio_device_id rtw_8821cs_id_table[] =  {
+	{
+		SDIO_DEVICE(SDIO_VENDOR_ID_REALTEK,
+			    SDIO_DEVICE_ID_REALTEK_RTW8821CS),
+		.driver_data = (kernel_ulong_t)&rtw8821c_hw_spec,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(sdio, rtw_8821cs_id_table);
+
+static struct sdio_driver rtw_8821cs_driver = {
+	.name = "rtw_8821cs",
+	.probe = rtw_sdio_probe,
+	.remove = rtw_sdio_remove,
+	.id_table = rtw_8821cs_id_table,
+	.drv = {
+		.pm = &rtw_sdio_pm_ops,
+		.shutdown = rtw_sdio_shutdown,
+	}
+};
+module_sdio_driver(rtw_8821cs_driver);
+
+MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8821cs driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c
index 74dfb89b2c94..3017a9760da8 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c
+++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c
@@ -32,6 +32,12 @@ static void rtw8822bu_efuse_parsing(struct rtw_efuse *efuse,
 	ether_addr_copy(efuse->addr, map->u.mac_addr);
 }
 
+static void rtw8822bs_efuse_parsing(struct rtw_efuse *efuse,
+				    struct rtw8822b_efuse *map)
+{
+	ether_addr_copy(efuse->addr, map->s.mac_addr);
+}
+
 static int rtw8822b_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 {
 	struct rtw_efuse *efuse = &rtwdev->efuse;
@@ -65,6 +71,9 @@ static int rtw8822b_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 	case RTW_HCI_TYPE_USB:
 		rtw8822bu_efuse_parsing(efuse, map);
 		break;
+	case RTW_HCI_TYPE_SDIO:
+		rtw8822bs_efuse_parsing(efuse, map);
+		break;
 	default:
 		/* unsupported now */
 		return -ENOTSUPP;
@@ -2531,6 +2540,7 @@ const struct rtw_chip_info rtw8822b_hw_spec = {
 	.txff_size = 262144,
 	.rxff_size = 24576,
 	.fw_rxff_size = 12288,
+	.rsvd_drv_pg_num = 8,
 	.txgi_factor = 1,
 	.is_pwr_by_rate_dec = true,
 	.max_power_index = 0x3f,
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.h b/drivers/net/wireless/realtek/rtw88/rtw8822b.h
index 01d3644e0c94..2dc3a6660f06 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8822b.h
+++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.h
@@ -65,6 +65,11 @@ struct rtw8822be_efuse {
 	u8 res7;
 };
 
+struct rtw8822bs_efuse {
+	u8 res4[0x4a];			/* 0xd0 */
+	u8 mac_addr[ETH_ALEN];		/* 0x11a */
+} __packed;
+
 struct rtw8822b_efuse {
 	__le16 rtl_id;
 	u8 res0[0x0e];
@@ -92,8 +97,9 @@ struct rtw8822b_efuse {
 	u8 country_code[2];
 	u8 res[3];
 	union {
-		struct rtw8822bu_efuse u;
 		struct rtw8822be_efuse e;
+		struct rtw8822bu_efuse u;
+		struct rtw8822bs_efuse s;
 	};
 };
 
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822bs.c b/drivers/net/wireless/realtek/rtw88/rtw8822bs.c
new file mode 100644
index 000000000000..31d8645f83bd
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw88/rtw8822bs.c
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) Jernej Skrabec <jernej.skrabec@gmail.com>
+ */
+
+#include <linux/mmc/sdio_func.h>
+#include <linux/mmc/sdio_ids.h>
+#include <linux/module.h>
+#include "main.h"
+#include "rtw8822b.h"
+#include "sdio.h"
+
+static const struct sdio_device_id rtw_8822bs_id_table[] =  {
+	{
+		SDIO_DEVICE(SDIO_VENDOR_ID_REALTEK,
+			    SDIO_DEVICE_ID_REALTEK_RTW8822BS),
+		.driver_data = (kernel_ulong_t)&rtw8822b_hw_spec,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(sdio, rtw_8822bs_id_table);
+
+static struct sdio_driver rtw_8822bs_driver = {
+	.name = "rtw_8822bs",
+	.probe = rtw_sdio_probe,
+	.remove = rtw_sdio_remove,
+	.id_table = rtw_8822bs_id_table,
+	.drv = {
+		.pm = &rtw_sdio_pm_ops,
+		.shutdown = rtw_sdio_shutdown,
+	}
+};
+module_sdio_driver(rtw_8822bs_driver);
+
+MODULE_AUTHOR("Jernej Skrabec <jernej.skrabec@gmail.com>");
+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8822bs driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822c.c b/drivers/net/wireless/realtek/rtw88/rtw8822c.c
index 964e27887fe2..cd965edc29ce 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8822c.c
+++ b/drivers/net/wireless/realtek/rtw88/rtw8822c.c
@@ -35,6 +35,12 @@ static void rtw8822cu_efuse_parsing(struct rtw_efuse *efuse,
 	ether_addr_copy(efuse->addr, map->u.mac_addr);
 }
 
+static void rtw8822cs_efuse_parsing(struct rtw_efuse *efuse,
+				    struct rtw8822c_efuse *map)
+{
+	ether_addr_copy(efuse->addr, map->s.mac_addr);
+}
+
 static int rtw8822c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 {
 	struct rtw_efuse *efuse = &rtwdev->efuse;
@@ -67,6 +73,9 @@ static int rtw8822c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map)
 	case RTW_HCI_TYPE_USB:
 		rtw8822cu_efuse_parsing(efuse, map);
 		break;
+	case RTW_HCI_TYPE_SDIO:
+		rtw8822cs_efuse_parsing(efuse, map);
+		break;
 	default:
 		/* unsupported now */
 		return -ENOTSUPP;
@@ -5349,6 +5358,7 @@ const struct rtw_chip_info rtw8822c_hw_spec = {
 	.txff_size = 262144,
 	.rxff_size = 24576,
 	.fw_rxff_size = 12288,
+	.rsvd_drv_pg_num = 16,
 	.txgi_factor = 2,
 	.is_pwr_by_rate_dec = false,
 	.max_power_index = 0x7f,
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822c.h b/drivers/net/wireless/realtek/rtw88/rtw8822c.h
index 479d5d769c52..1bc0e7f5d6bb 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8822c.h
+++ b/drivers/net/wireless/realtek/rtw88/rtw8822c.h
@@ -16,6 +16,11 @@ struct rtw8822cu_efuse {
 	u8 res2[0x3d];
 };
 
+struct rtw8822cs_efuse {
+	u8 res0[0x4a];			/* 0x120 */
+	u8 mac_addr[ETH_ALEN];		/* 0x16a */
+} __packed;
+
 struct rtw8822ce_efuse {
 	u8 mac_addr[ETH_ALEN];		/* 0x120 */
 	u8 vender_id[2];
@@ -91,8 +96,9 @@ struct rtw8822c_efuse {
 	u8 res9;
 	u8 res10[0x42];
 	union {
-		struct rtw8822cu_efuse u;
 		struct rtw8822ce_efuse e;
+		struct rtw8822cu_efuse u;
+		struct rtw8822cs_efuse s;
 	};
 };
 
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822cs.c b/drivers/net/wireless/realtek/rtw88/rtw8822cs.c
new file mode 100644
index 000000000000..975e81c824f2
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw88/rtw8822cs.c
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ */
+
+#include <linux/mmc/sdio_func.h>
+#include <linux/mmc/sdio_ids.h>
+#include <linux/module.h>
+#include "main.h"
+#include "rtw8822c.h"
+#include "sdio.h"
+
+static const struct sdio_device_id rtw_8822cs_id_table[] =  {
+	{
+		SDIO_DEVICE(SDIO_VENDOR_ID_REALTEK,
+			    SDIO_DEVICE_ID_REALTEK_RTW8822CS),
+		.driver_data = (kernel_ulong_t)&rtw8822c_hw_spec,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(sdio, rtw_8822cs_id_table);
+
+static struct sdio_driver rtw_8822cs_driver = {
+	.name = "rtw_8822cs",
+	.probe = rtw_sdio_probe,
+	.remove = rtw_sdio_remove,
+	.id_table = rtw_8822cs_id_table,
+	.drv = {
+		.pm = &rtw_sdio_pm_ops,
+		.shutdown = rtw_sdio_shutdown,
+	}
+};
+module_sdio_driver(rtw_8822cs_driver);
+
+MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8822cs driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw88/sdio.c b/drivers/net/wireless/realtek/rtw88/sdio.c
new file mode 100644
index 000000000000..af0459a79899
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw88/sdio.c
@@ -0,0 +1,1394 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright (C) 2021 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ * Copyright (C) 2021 Jernej Skrabec <jernej.skrabec@gmail.com>
+ *
+ * Based on rtw88/pci.c:
+ *   Copyright(c) 2018-2019  Realtek Corporation
+ */
+
+#include <linux/module.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/sdio_func.h>
+#include "main.h"
+#include "debug.h"
+#include "fw.h"
+#include "ps.h"
+#include "reg.h"
+#include "rx.h"
+#include "sdio.h"
+#include "tx.h"
+
+#define RTW_SDIO_INDIRECT_RW_RETRIES			50
+
+static bool rtw_sdio_is_bus_addr(u32 addr)
+{
+	return !!(addr & RTW_SDIO_BUS_MSK);
+}
+
+static bool rtw_sdio_bus_claim_needed(struct rtw_sdio *rtwsdio)
+{
+	return !rtwsdio->irq_thread ||
+	       rtwsdio->irq_thread != current;
+}
+
+static u32 rtw_sdio_to_bus_offset(struct rtw_dev *rtwdev, u32 addr)
+{
+	switch (addr & RTW_SDIO_BUS_MSK) {
+	case WLAN_IOREG_OFFSET:
+		addr &= WLAN_IOREG_REG_MSK;
+		addr |= FIELD_PREP(REG_SDIO_CMD_ADDR_MSK,
+				   REG_SDIO_CMD_ADDR_MAC_REG);
+		break;
+	case SDIO_LOCAL_OFFSET:
+		addr &= SDIO_LOCAL_REG_MSK;
+		addr |= FIELD_PREP(REG_SDIO_CMD_ADDR_MSK,
+				   REG_SDIO_CMD_ADDR_SDIO_REG);
+		break;
+	default:
+		rtw_warn(rtwdev, "Cannot convert addr 0x%08x to bus offset",
+			 addr);
+	}
+
+	return addr;
+}
+
+static bool rtw_sdio_use_memcpy_io(struct rtw_dev *rtwdev, u32 addr,
+				   u8 alignment)
+{
+	return IS_ALIGNED(addr, alignment) &&
+	       test_bit(RTW_FLAG_POWERON, rtwdev->flags);
+}
+
+static void rtw_sdio_writel(struct rtw_dev *rtwdev, u32 val, u32 addr,
+			    int *err_ret)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	u8 buf[4];
+	int i;
+
+	if (rtw_sdio_use_memcpy_io(rtwdev, addr, 4)) {
+		sdio_writel(rtwsdio->sdio_func, val, addr, err_ret);
+		return;
+	}
+
+	*(__le32 *)buf = cpu_to_le32(val);
+
+	for (i = 0; i < 4; i++) {
+		sdio_writeb(rtwsdio->sdio_func, buf[i], addr + i, err_ret);
+		if (*err_ret)
+			return;
+	}
+}
+
+static void rtw_sdio_writew(struct rtw_dev *rtwdev, u16 val, u32 addr,
+			    int *err_ret)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	u8 buf[2];
+	int i;
+
+	if (rtw_sdio_use_memcpy_io(rtwdev, addr, 2)) {
+		sdio_writew(rtwsdio->sdio_func, val, addr, err_ret);
+		return;
+	}
+
+	*(__le16 *)buf = cpu_to_le16(val);
+
+	for (i = 0; i < 2; i++) {
+		sdio_writeb(rtwsdio->sdio_func, buf[i], addr + i, err_ret);
+		if (*err_ret)
+			return;
+	}
+}
+
+static u32 rtw_sdio_readl(struct rtw_dev *rtwdev, u32 addr, int *err_ret)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	u8 buf[4];
+	int i;
+
+	if (rtw_sdio_use_memcpy_io(rtwdev, addr, 4))
+		return sdio_readl(rtwsdio->sdio_func, addr, err_ret);
+
+	for (i = 0; i < 4; i++) {
+		buf[i] = sdio_readb(rtwsdio->sdio_func, addr + i, err_ret);
+		if (*err_ret)
+			return 0;
+	}
+
+	return le32_to_cpu(*(__le32 *)buf);
+}
+
+static u16 rtw_sdio_readw(struct rtw_dev *rtwdev, u32 addr, int *err_ret)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	u8 buf[2];
+	int i;
+
+	if (rtw_sdio_use_memcpy_io(rtwdev, addr, 2))
+		return sdio_readw(rtwsdio->sdio_func, addr, err_ret);
+
+	for (i = 0; i < 2; i++) {
+		buf[i] = sdio_readb(rtwsdio->sdio_func, addr + i, err_ret);
+		if (*err_ret)
+			return 0;
+	}
+
+	return le16_to_cpu(*(__le16 *)buf);
+}
+
+static u32 rtw_sdio_to_io_address(struct rtw_dev *rtwdev, u32 addr,
+				  bool direct)
+{
+	if (!direct)
+		return addr;
+
+	if (!rtw_sdio_is_bus_addr(addr))
+		addr |= WLAN_IOREG_OFFSET;
+
+	return rtw_sdio_to_bus_offset(rtwdev, addr);
+}
+
+static bool rtw_sdio_use_direct_io(struct rtw_dev *rtwdev, u32 addr)
+{
+	return !rtw_sdio_is_sdio30_supported(rtwdev) ||
+		rtw_sdio_is_bus_addr(addr);
+}
+
+static int rtw_sdio_indirect_reg_cfg(struct rtw_dev *rtwdev, u32 addr, u32 cfg)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	unsigned int retry;
+	u32 reg_cfg;
+	int ret;
+	u8 tmp;
+
+	reg_cfg = rtw_sdio_to_bus_offset(rtwdev, REG_SDIO_INDIRECT_REG_CFG);
+
+	rtw_sdio_writel(rtwdev, addr | cfg | BIT_SDIO_INDIRECT_REG_CFG_UNK20,
+			reg_cfg, &ret);
+	if (ret)
+		return ret;
+
+	for (retry = 0; retry < RTW_SDIO_INDIRECT_RW_RETRIES; retry++) {
+		tmp = sdio_readb(rtwsdio->sdio_func, reg_cfg + 2, &ret);
+		if (!ret && (tmp & BIT(4)))
+			return 0;
+	}
+
+	return -ETIMEDOUT;
+}
+
+static u8 rtw_sdio_indirect_read8(struct rtw_dev *rtwdev, u32 addr,
+				  int *err_ret)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	u32 reg_data;
+
+	*err_ret = rtw_sdio_indirect_reg_cfg(rtwdev, addr,
+					     BIT_SDIO_INDIRECT_REG_CFG_READ);
+	if (*err_ret)
+		return 0;
+
+	reg_data = rtw_sdio_to_bus_offset(rtwdev, REG_SDIO_INDIRECT_REG_DATA);
+	return sdio_readb(rtwsdio->sdio_func, reg_data, err_ret);
+}
+
+static int rtw_sdio_indirect_read_bytes(struct rtw_dev *rtwdev, u32 addr,
+					u8 *buf, int count)
+{
+	int i, ret = 0;
+
+	for (i = 0; i < count; i++) {
+		buf[i] = rtw_sdio_indirect_read8(rtwdev, addr + i, &ret);
+		if (ret)
+			break;
+	}
+
+	return ret;
+}
+
+static u16 rtw_sdio_indirect_read16(struct rtw_dev *rtwdev, u32 addr,
+				    int *err_ret)
+{
+	u32 reg_data;
+	u8 buf[2];
+
+	if (!IS_ALIGNED(addr, 2)) {
+		*err_ret = rtw_sdio_indirect_read_bytes(rtwdev, addr, buf, 2);
+		if (*err_ret)
+			return 0;
+
+		return le16_to_cpu(*(__le16 *)buf);
+	}
+
+	*err_ret = rtw_sdio_indirect_reg_cfg(rtwdev, addr,
+					     BIT_SDIO_INDIRECT_REG_CFG_READ);
+	if (*err_ret)
+		return 0;
+
+	reg_data = rtw_sdio_to_bus_offset(rtwdev, REG_SDIO_INDIRECT_REG_DATA);
+	return rtw_sdio_readw(rtwdev, reg_data, err_ret);
+}
+
+static u32 rtw_sdio_indirect_read32(struct rtw_dev *rtwdev, u32 addr,
+				    int *err_ret)
+{
+	u32 reg_data;
+	u8 buf[4];
+
+	if (!IS_ALIGNED(addr, 4)) {
+		*err_ret = rtw_sdio_indirect_read_bytes(rtwdev, addr, buf, 4);
+		if (*err_ret)
+			return 0;
+
+		return le32_to_cpu(*(__le32 *)buf);
+	}
+
+	*err_ret = rtw_sdio_indirect_reg_cfg(rtwdev, addr,
+					     BIT_SDIO_INDIRECT_REG_CFG_READ);
+	if (*err_ret)
+		return 0;
+
+	reg_data = rtw_sdio_to_bus_offset(rtwdev, REG_SDIO_INDIRECT_REG_DATA);
+	return rtw_sdio_readl(rtwdev, reg_data, err_ret);
+}
+
+static u8 rtw_sdio_read8(struct rtw_dev *rtwdev, u32 addr)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	bool direct, bus_claim;
+	int ret;
+	u8 val;
+
+	direct = rtw_sdio_use_direct_io(rtwdev, addr);
+	addr = rtw_sdio_to_io_address(rtwdev, addr, direct);
+	bus_claim = rtw_sdio_bus_claim_needed(rtwsdio);
+
+	if (bus_claim)
+		sdio_claim_host(rtwsdio->sdio_func);
+
+	if (direct)
+		val = sdio_readb(rtwsdio->sdio_func, addr, &ret);
+	else
+		val = rtw_sdio_indirect_read8(rtwdev, addr, &ret);
+
+	if (bus_claim)
+		sdio_release_host(rtwsdio->sdio_func);
+
+	if (ret)
+		rtw_warn(rtwdev, "sdio read8 failed (0x%x): %d", addr, ret);
+
+	return val;
+}
+
+static u16 rtw_sdio_read16(struct rtw_dev *rtwdev, u32 addr)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	bool direct, bus_claim;
+	int ret;
+	u16 val;
+
+	direct = rtw_sdio_use_direct_io(rtwdev, addr);
+	addr = rtw_sdio_to_io_address(rtwdev, addr, direct);
+	bus_claim = rtw_sdio_bus_claim_needed(rtwsdio);
+
+	if (bus_claim)
+		sdio_claim_host(rtwsdio->sdio_func);
+
+	if (direct)
+		val = rtw_sdio_readw(rtwdev, addr, &ret);
+	else
+		val = rtw_sdio_indirect_read16(rtwdev, addr, &ret);
+
+	if (bus_claim)
+		sdio_release_host(rtwsdio->sdio_func);
+
+	if (ret)
+		rtw_warn(rtwdev, "sdio read16 failed (0x%x): %d", addr, ret);
+
+	return val;
+}
+
+static u32 rtw_sdio_read32(struct rtw_dev *rtwdev, u32 addr)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	bool direct, bus_claim;
+	u32 val;
+	int ret;
+
+	direct = rtw_sdio_use_direct_io(rtwdev, addr);
+	addr = rtw_sdio_to_io_address(rtwdev, addr, direct);
+	bus_claim = rtw_sdio_bus_claim_needed(rtwsdio);
+
+	if (bus_claim)
+		sdio_claim_host(rtwsdio->sdio_func);
+
+	if (direct)
+		val = rtw_sdio_readl(rtwdev, addr, &ret);
+	else
+		val = rtw_sdio_indirect_read32(rtwdev, addr, &ret);
+
+	if (bus_claim)
+		sdio_release_host(rtwsdio->sdio_func);
+
+	if (ret)
+		rtw_warn(rtwdev, "sdio read32 failed (0x%x): %d", addr, ret);
+
+	return val;
+}
+
+static void rtw_sdio_indirect_write8(struct rtw_dev *rtwdev, u8 val, u32 addr,
+				     int *err_ret)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	u32 reg_data;
+
+	reg_data = rtw_sdio_to_bus_offset(rtwdev, REG_SDIO_INDIRECT_REG_DATA);
+	sdio_writeb(rtwsdio->sdio_func, val, reg_data, err_ret);
+	if (*err_ret)
+		return;
+
+	*err_ret = rtw_sdio_indirect_reg_cfg(rtwdev, addr,
+					     BIT_SDIO_INDIRECT_REG_CFG_WRITE);
+}
+
+static void rtw_sdio_indirect_write16(struct rtw_dev *rtwdev, u16 val, u32 addr,
+				      int *err_ret)
+{
+	u32 reg_data;
+
+	if (!IS_ALIGNED(addr, 2)) {
+		addr = rtw_sdio_to_io_address(rtwdev, addr, true);
+		rtw_sdio_writew(rtwdev, val, addr, err_ret);
+		return;
+	}
+
+	reg_data = rtw_sdio_to_bus_offset(rtwdev, REG_SDIO_INDIRECT_REG_DATA);
+	rtw_sdio_writew(rtwdev, val, reg_data, err_ret);
+	if (*err_ret)
+		return;
+
+	*err_ret = rtw_sdio_indirect_reg_cfg(rtwdev, addr,
+					     BIT_SDIO_INDIRECT_REG_CFG_WRITE |
+					     BIT_SDIO_INDIRECT_REG_CFG_WORD);
+}
+
+static void rtw_sdio_indirect_write32(struct rtw_dev *rtwdev, u32 val,
+				      u32 addr, int *err_ret)
+{
+	u32 reg_data;
+
+	if (!IS_ALIGNED(addr, 4)) {
+		addr = rtw_sdio_to_io_address(rtwdev, addr, true);
+		rtw_sdio_writel(rtwdev, val, addr, err_ret);
+		return;
+	}
+
+	reg_data = rtw_sdio_to_bus_offset(rtwdev, REG_SDIO_INDIRECT_REG_DATA);
+	rtw_sdio_writel(rtwdev, val, reg_data, err_ret);
+
+	*err_ret = rtw_sdio_indirect_reg_cfg(rtwdev, addr,
+					     BIT_SDIO_INDIRECT_REG_CFG_WRITE |
+					     BIT_SDIO_INDIRECT_REG_CFG_DWORD);
+}
+
+static void rtw_sdio_write8(struct rtw_dev *rtwdev, u32 addr, u8 val)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	bool direct, bus_claim;
+	int ret;
+
+	direct = rtw_sdio_use_direct_io(rtwdev, addr);
+	addr = rtw_sdio_to_io_address(rtwdev, addr, direct);
+	bus_claim = rtw_sdio_bus_claim_needed(rtwsdio);
+
+	if (bus_claim)
+		sdio_claim_host(rtwsdio->sdio_func);
+
+	if (direct)
+		sdio_writeb(rtwsdio->sdio_func, val, addr, &ret);
+	else
+		rtw_sdio_indirect_write8(rtwdev, val, addr, &ret);
+
+	if (bus_claim)
+		sdio_release_host(rtwsdio->sdio_func);
+
+	if (ret)
+		rtw_warn(rtwdev, "sdio write8 failed (0x%x): %d", addr, ret);
+}
+
+static void rtw_sdio_write16(struct rtw_dev *rtwdev, u32 addr, u16 val)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	bool direct, bus_claim;
+	int ret;
+
+	direct = rtw_sdio_use_direct_io(rtwdev, addr);
+	addr = rtw_sdio_to_io_address(rtwdev, addr, direct);
+	bus_claim = rtw_sdio_bus_claim_needed(rtwsdio);
+
+	if (bus_claim)
+		sdio_claim_host(rtwsdio->sdio_func);
+
+	if (direct)
+		rtw_sdio_writew(rtwdev, val, addr, &ret);
+	else
+		rtw_sdio_indirect_write16(rtwdev, val, addr, &ret);
+
+	if (bus_claim)
+		sdio_release_host(rtwsdio->sdio_func);
+
+	if (ret)
+		rtw_warn(rtwdev, "sdio write16 failed (0x%x): %d", addr, ret);
+}
+
+static void rtw_sdio_write32(struct rtw_dev *rtwdev, u32 addr, u32 val)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	bool direct, bus_claim;
+	int ret;
+
+	direct = rtw_sdio_use_direct_io(rtwdev, addr);
+	addr = rtw_sdio_to_io_address(rtwdev, addr, direct);
+	bus_claim = rtw_sdio_bus_claim_needed(rtwsdio);
+
+	if (bus_claim)
+		sdio_claim_host(rtwsdio->sdio_func);
+
+	if (direct)
+		rtw_sdio_writel(rtwdev, val, addr, &ret);
+	else
+		rtw_sdio_indirect_write32(rtwdev, val, addr, &ret);
+
+	if (bus_claim)
+		sdio_release_host(rtwsdio->sdio_func);
+
+	if (ret)
+		rtw_warn(rtwdev, "sdio write32 failed (0x%x): %d", addr, ret);
+}
+
+static u32 rtw_sdio_get_tx_addr(struct rtw_dev *rtwdev, size_t size,
+				enum rtw_tx_queue_type queue)
+{
+	u32 txaddr;
+
+	switch (queue) {
+	case RTW_TX_QUEUE_BCN:
+	case RTW_TX_QUEUE_H2C:
+	case RTW_TX_QUEUE_HI0:
+		txaddr = FIELD_PREP(REG_SDIO_CMD_ADDR_MSK,
+				    REG_SDIO_CMD_ADDR_TXFF_HIGH);
+		break;
+	case RTW_TX_QUEUE_VI:
+	case RTW_TX_QUEUE_VO:
+		txaddr = FIELD_PREP(REG_SDIO_CMD_ADDR_MSK,
+				    REG_SDIO_CMD_ADDR_TXFF_NORMAL);
+		break;
+	case RTW_TX_QUEUE_BE:
+	case RTW_TX_QUEUE_BK:
+		txaddr = FIELD_PREP(REG_SDIO_CMD_ADDR_MSK,
+				    REG_SDIO_CMD_ADDR_TXFF_LOW);
+		break;
+	case RTW_TX_QUEUE_MGMT:
+		txaddr = FIELD_PREP(REG_SDIO_CMD_ADDR_MSK,
+				    REG_SDIO_CMD_ADDR_TXFF_EXTRA);
+		break;
+	default:
+		rtw_warn(rtwdev, "Unsupported queue for TX addr: 0x%02x\n",
+			 queue);
+		return 0;
+	}
+
+	txaddr += DIV_ROUND_UP(size, 4);
+
+	return txaddr;
+};
+
+static int rtw_sdio_read_port(struct rtw_dev *rtwdev, u8 *buf, size_t count)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	bool bus_claim = rtw_sdio_bus_claim_needed(rtwsdio);
+	u32 rxaddr = rtwsdio->rx_addr++;
+	int ret;
+
+	if (bus_claim)
+		sdio_claim_host(rtwsdio->sdio_func);
+
+	ret = sdio_memcpy_fromio(rtwsdio->sdio_func, buf,
+				 RTW_SDIO_ADDR_RX_RX0FF_GEN(rxaddr), count);
+	if (ret)
+		rtw_warn(rtwdev,
+			 "Failed to read %zu byte(s) from SDIO port 0x%08x",
+			 count, rxaddr);
+
+	if (bus_claim)
+		sdio_release_host(rtwsdio->sdio_func);
+
+	return ret;
+}
+
+static int rtw_sdio_check_free_txpg(struct rtw_dev *rtwdev, u8 queue,
+				    size_t count)
+{
+	unsigned int pages_free, pages_needed;
+
+	if (rtw_chip_wcpu_11n(rtwdev)) {
+		u32 free_txpg;
+
+		free_txpg = rtw_sdio_read32(rtwdev, REG_SDIO_FREE_TXPG);
+
+		switch (queue) {
+		case RTW_TX_QUEUE_BCN:
+		case RTW_TX_QUEUE_H2C:
+		case RTW_TX_QUEUE_HI0:
+		case RTW_TX_QUEUE_MGMT:
+			/* high */
+			pages_free = free_txpg & 0xff;
+			break;
+		case RTW_TX_QUEUE_VI:
+		case RTW_TX_QUEUE_VO:
+			/* normal */
+			pages_free = (free_txpg >> 8) & 0xff;
+			break;
+		case RTW_TX_QUEUE_BE:
+		case RTW_TX_QUEUE_BK:
+			/* low */
+			pages_free = (free_txpg >> 16) & 0xff;
+			break;
+		default:
+			rtw_warn(rtwdev, "Unknown mapping for queue %u\n", queue);
+			return -EINVAL;
+		}
+
+		/* add the pages from the public queue */
+		pages_free += (free_txpg >> 24) & 0xff;
+	} else {
+		u32 free_txpg[3];
+
+		free_txpg[0] = rtw_sdio_read32(rtwdev, REG_SDIO_FREE_TXPG);
+		free_txpg[1] = rtw_sdio_read32(rtwdev, REG_SDIO_FREE_TXPG + 4);
+		free_txpg[2] = rtw_sdio_read32(rtwdev, REG_SDIO_FREE_TXPG + 8);
+
+		switch (queue) {
+		case RTW_TX_QUEUE_BCN:
+		case RTW_TX_QUEUE_H2C:
+		case RTW_TX_QUEUE_HI0:
+			/* high */
+			pages_free = free_txpg[0] & 0xfff;
+			break;
+		case RTW_TX_QUEUE_VI:
+		case RTW_TX_QUEUE_VO:
+			/* normal */
+			pages_free = (free_txpg[0] >> 16) & 0xfff;
+			break;
+		case RTW_TX_QUEUE_BE:
+		case RTW_TX_QUEUE_BK:
+			/* low */
+			pages_free = free_txpg[1] & 0xfff;
+			break;
+		case RTW_TX_QUEUE_MGMT:
+			/* extra */
+			pages_free = free_txpg[2] & 0xfff;
+			break;
+		default:
+			rtw_warn(rtwdev, "Unknown mapping for queue %u\n", queue);
+			return -EINVAL;
+		}
+
+		/* add the pages from the public queue */
+		pages_free += (free_txpg[1] >> 16) & 0xfff;
+	}
+
+	pages_needed = DIV_ROUND_UP(count, rtwdev->chip->page_size);
+
+	if (pages_needed > pages_free) {
+		rtw_dbg(rtwdev, RTW_DBG_SDIO,
+			"Not enough free pages (%u needed, %u free) in queue %u for %zu bytes\n",
+			pages_needed, pages_free, queue, count);
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+static int rtw_sdio_write_port(struct rtw_dev *rtwdev, struct sk_buff *skb,
+			       enum rtw_tx_queue_type queue)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	bool bus_claim;
+	size_t txsize;
+	u32 txaddr;
+	int ret;
+
+	txaddr = rtw_sdio_get_tx_addr(rtwdev, skb->len, queue);
+	if (!txaddr)
+		return -EINVAL;
+
+	txsize = sdio_align_size(rtwsdio->sdio_func, skb->len);
+
+	ret = rtw_sdio_check_free_txpg(rtwdev, queue, txsize);
+	if (ret)
+		return ret;
+
+	if (!IS_ALIGNED((unsigned long)skb->data, RTW_SDIO_DATA_PTR_ALIGN))
+		rtw_warn(rtwdev, "Got unaligned SKB in %s() for queue %u\n",
+			 __func__, queue);
+
+	bus_claim = rtw_sdio_bus_claim_needed(rtwsdio);
+
+	if (bus_claim)
+		sdio_claim_host(rtwsdio->sdio_func);
+
+	ret = sdio_memcpy_toio(rtwsdio->sdio_func, txaddr, skb->data, txsize);
+
+	if (bus_claim)
+		sdio_release_host(rtwsdio->sdio_func);
+
+	if (ret)
+		rtw_warn(rtwdev,
+			 "Failed to write %zu byte(s) to SDIO port 0x%08x",
+			 txsize, txaddr);
+
+	return ret;
+}
+
+static void rtw_sdio_init(struct rtw_dev *rtwdev)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+
+	rtwsdio->irq_mask = REG_SDIO_HIMR_RX_REQUEST | REG_SDIO_HIMR_CPWM1;
+}
+
+static void rtw_sdio_enable_rx_aggregation(struct rtw_dev *rtwdev)
+{
+	u8 size, timeout;
+
+	if (rtw_chip_wcpu_11n(rtwdev)) {
+		size = 0x6;
+		timeout = 0x6;
+	} else {
+		size = 0xff;
+		timeout = 0x1;
+	}
+
+	/* Make the firmware honor the size limit configured below */
+	rtw_write32_set(rtwdev, REG_RXDMA_AGG_PG_TH, BIT_EN_PRE_CALC);
+
+	rtw_write8_set(rtwdev, REG_TXDMA_PQ_MAP, BIT_RXDMA_AGG_EN);
+
+	rtw_write16(rtwdev, REG_RXDMA_AGG_PG_TH,
+		    FIELD_PREP(BIT_RXDMA_AGG_PG_TH, size) |
+		    FIELD_PREP(BIT_DMA_AGG_TO_V1, timeout));
+
+	rtw_write8_set(rtwdev, REG_RXDMA_MODE, BIT_DMA_MODE);
+}
+
+static void rtw_sdio_enable_interrupt(struct rtw_dev *rtwdev)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+
+	rtw_write32(rtwdev, REG_SDIO_HIMR, rtwsdio->irq_mask);
+}
+
+static void rtw_sdio_disable_interrupt(struct rtw_dev *rtwdev)
+{
+	rtw_write32(rtwdev, REG_SDIO_HIMR, 0x0);
+}
+
+static u8 rtw_sdio_get_tx_qsel(struct rtw_dev *rtwdev, struct sk_buff *skb,
+			       u8 queue)
+{
+	switch (queue) {
+	case RTW_TX_QUEUE_BCN:
+		return TX_DESC_QSEL_BEACON;
+	case RTW_TX_QUEUE_H2C:
+		return TX_DESC_QSEL_H2C;
+	case RTW_TX_QUEUE_MGMT:
+		if (rtw_chip_wcpu_11n(rtwdev))
+			return TX_DESC_QSEL_HIGH;
+		else
+			return TX_DESC_QSEL_MGMT;
+	case RTW_TX_QUEUE_HI0:
+		return TX_DESC_QSEL_HIGH;
+	default:
+		return skb->priority;
+	}
+}
+
+static int rtw_sdio_setup(struct rtw_dev *rtwdev)
+{
+	/* nothing to do */
+	return 0;
+}
+
+static int rtw_sdio_start(struct rtw_dev *rtwdev)
+{
+	rtw_sdio_enable_rx_aggregation(rtwdev);
+	rtw_sdio_enable_interrupt(rtwdev);
+
+	return 0;
+}
+
+static void rtw_sdio_stop(struct rtw_dev *rtwdev)
+{
+	rtw_sdio_disable_interrupt(rtwdev);
+}
+
+static void rtw_sdio_deep_ps_enter(struct rtw_dev *rtwdev)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	bool tx_empty = true;
+	u8 queue;
+
+	if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) {
+		/* Deep PS state is not allowed to TX-DMA */
+		for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) {
+			/* BCN queue is rsvd page, does not have DMA interrupt
+			 * H2C queue is managed by firmware
+			 */
+			if (queue == RTW_TX_QUEUE_BCN ||
+			    queue == RTW_TX_QUEUE_H2C)
+				continue;
+
+			/* check if there is any skb DMAing */
+			if (skb_queue_len(&rtwsdio->tx_queue[queue])) {
+				tx_empty = false;
+				break;
+			}
+		}
+	}
+
+	if (!tx_empty) {
+		rtw_dbg(rtwdev, RTW_DBG_PS,
+			"TX path not empty, cannot enter deep power save state\n");
+		return;
+	}
+
+	set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags);
+	rtw_power_mode_change(rtwdev, true);
+}
+
+static void rtw_sdio_deep_ps_leave(struct rtw_dev *rtwdev)
+{
+	if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
+		rtw_power_mode_change(rtwdev, false);
+}
+
+static void rtw_sdio_deep_ps(struct rtw_dev *rtwdev, bool enter)
+{
+	if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
+		rtw_sdio_deep_ps_enter(rtwdev);
+
+	if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
+		rtw_sdio_deep_ps_leave(rtwdev);
+}
+
+static void rtw_sdio_tx_kick_off(struct rtw_dev *rtwdev)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+
+	queue_work(rtwsdio->txwq, &rtwsdio->tx_handler_data->work);
+}
+
+static void rtw_sdio_link_ps(struct rtw_dev *rtwdev, bool enter)
+{
+	/* nothing to do */
+}
+
+static void rtw_sdio_interface_cfg(struct rtw_dev *rtwdev)
+{
+	u32 val;
+
+	rtw_read32(rtwdev, REG_SDIO_FREE_TXPG);
+
+	val = rtw_read32(rtwdev, REG_SDIO_TX_CTRL);
+	val &= 0xfff8;
+	rtw_write32(rtwdev, REG_SDIO_TX_CTRL, val);
+}
+
+static struct rtw_sdio_tx_data *rtw_sdio_get_tx_data(struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+
+	BUILD_BUG_ON(sizeof(struct rtw_sdio_tx_data) >
+		     sizeof(info->status.status_driver_data));
+
+	return (struct rtw_sdio_tx_data *)info->status.status_driver_data;
+}
+
+static void rtw_sdio_tx_skb_prepare(struct rtw_dev *rtwdev,
+				    struct rtw_tx_pkt_info *pkt_info,
+				    struct sk_buff *skb,
+				    enum rtw_tx_queue_type queue)
+{
+	const struct rtw_chip_info *chip = rtwdev->chip;
+	unsigned long data_addr, aligned_addr;
+	size_t offset;
+	u8 *pkt_desc;
+
+	pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz);
+
+	data_addr = (unsigned long)pkt_desc;
+	aligned_addr = ALIGN(data_addr, RTW_SDIO_DATA_PTR_ALIGN);
+
+	if (data_addr != aligned_addr) {
+		/* Ensure that the start of the pkt_desc is always aligned at
+		 * RTW_SDIO_DATA_PTR_ALIGN.
+		 */
+		offset = RTW_SDIO_DATA_PTR_ALIGN - (aligned_addr - data_addr);
+
+		pkt_desc = skb_push(skb, offset);
+
+		/* By inserting padding to align the start of the pkt_desc we
+		 * need to inform the firmware that the actual data starts at
+		 * a different offset than normal.
+		 */
+		pkt_info->offset += offset;
+	}
+
+	memset(pkt_desc, 0, chip->tx_pkt_desc_sz);
+
+	pkt_info->qsel = rtw_sdio_get_tx_qsel(rtwdev, skb, queue);
+
+	rtw_tx_fill_tx_desc(pkt_info, skb);
+	rtw_tx_fill_txdesc_checksum(rtwdev, pkt_info, pkt_desc);
+}
+
+static int rtw_sdio_write_data(struct rtw_dev *rtwdev,
+			       struct rtw_tx_pkt_info *pkt_info,
+			       struct sk_buff *skb,
+			       enum rtw_tx_queue_type queue)
+{
+	int ret;
+
+	rtw_sdio_tx_skb_prepare(rtwdev, pkt_info, skb, queue);
+
+	ret = rtw_sdio_write_port(rtwdev, skb, queue);
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
+static int rtw_sdio_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf,
+					 u32 size)
+{
+	struct rtw_tx_pkt_info pkt_info = {};
+	struct sk_buff *skb;
+
+	skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size);
+	if (!skb)
+		return -ENOMEM;
+
+	return rtw_sdio_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN);
+}
+
+static int rtw_sdio_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size)
+{
+	struct rtw_tx_pkt_info pkt_info = {};
+	struct sk_buff *skb;
+
+	skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size);
+	if (!skb)
+		return -ENOMEM;
+
+	return rtw_sdio_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C);
+}
+
+static int rtw_sdio_tx_write(struct rtw_dev *rtwdev,
+			     struct rtw_tx_pkt_info *pkt_info,
+			     struct sk_buff *skb)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	enum rtw_tx_queue_type queue = rtw_tx_queue_mapping(skb);
+	struct rtw_sdio_tx_data *tx_data;
+
+	rtw_sdio_tx_skb_prepare(rtwdev, pkt_info, skb, queue);
+
+	tx_data = rtw_sdio_get_tx_data(skb);
+	tx_data->sn = pkt_info->sn;
+
+	skb_queue_tail(&rtwsdio->tx_queue[queue], skb);
+
+	return 0;
+}
+
+static void rtw_sdio_tx_err_isr(struct rtw_dev *rtwdev)
+{
+	u32 val = rtw_read32(rtwdev, REG_TXDMA_STATUS);
+
+	rtw_write32(rtwdev, REG_TXDMA_STATUS, val);
+}
+
+static void rtw_sdio_rx_skb(struct rtw_dev *rtwdev, struct sk_buff *skb,
+			    u32 pkt_offset, struct rtw_rx_pkt_stat *pkt_stat,
+			    struct ieee80211_rx_status *rx_status)
+{
+	*IEEE80211_SKB_RXCB(skb) = *rx_status;
+
+	if (pkt_stat->is_c2h) {
+		skb_put(skb, pkt_stat->pkt_len + pkt_offset);
+		rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, skb);
+		return;
+	}
+
+	skb_put(skb, pkt_stat->pkt_len);
+	skb_reserve(skb, pkt_offset);
+
+	rtw_rx_stats(rtwdev, pkt_stat->vif, skb);
+
+	ieee80211_rx_irqsafe(rtwdev->hw, skb);
+}
+
+static void rtw_sdio_rxfifo_recv(struct rtw_dev *rtwdev, u32 rx_len)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	const struct rtw_chip_info *chip = rtwdev->chip;
+	u32 pkt_desc_sz = chip->rx_pkt_desc_sz;
+	struct ieee80211_rx_status rx_status;
+	struct rtw_rx_pkt_stat pkt_stat;
+	struct sk_buff *skb, *split_skb;
+	u32 pkt_offset, curr_pkt_len;
+	size_t bufsz;
+	u8 *rx_desc;
+	int ret;
+
+	bufsz = sdio_align_size(rtwsdio->sdio_func, rx_len);
+
+	skb = dev_alloc_skb(bufsz);
+	if (!skb)
+		return;
+
+	ret = rtw_sdio_read_port(rtwdev, skb->data, bufsz);
+	if (ret) {
+		dev_kfree_skb_any(skb);
+		return;
+	}
+
+	while (true) {
+		rx_desc = skb->data;
+		chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat,
+					 &rx_status);
+		pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz +
+			     pkt_stat.shift;
+
+		curr_pkt_len = ALIGN(pkt_offset + pkt_stat.pkt_len,
+				     RTW_SDIO_DATA_PTR_ALIGN);
+
+		if ((curr_pkt_len + pkt_desc_sz) >= rx_len) {
+			/* Use the original skb (with it's adjusted offset)
+			 * when processing the last (or even the only) entry to
+			 * have it's memory freed automatically.
+			 */
+			rtw_sdio_rx_skb(rtwdev, skb, pkt_offset, &pkt_stat,
+					&rx_status);
+			break;
+		}
+
+		split_skb = dev_alloc_skb(curr_pkt_len);
+		if (!split_skb) {
+			rtw_sdio_rx_skb(rtwdev, skb, pkt_offset, &pkt_stat,
+					&rx_status);
+			break;
+		}
+
+		skb_copy_header(split_skb, skb);
+		memcpy(split_skb->data, skb->data, curr_pkt_len);
+
+		rtw_sdio_rx_skb(rtwdev, split_skb, pkt_offset, &pkt_stat,
+				&rx_status);
+
+		/* Move to the start of the next RX descriptor */
+		skb_reserve(skb, curr_pkt_len);
+		rx_len -= curr_pkt_len;
+	}
+}
+
+static void rtw_sdio_rx_isr(struct rtw_dev *rtwdev)
+{
+	u32 rx_len, total_rx_bytes = 0;
+
+	while (total_rx_bytes < SZ_64K) {
+		if (rtw_chip_wcpu_11n(rtwdev))
+			rx_len = rtw_read16(rtwdev, REG_SDIO_RX0_REQ_LEN);
+		else
+			rx_len = rtw_read32(rtwdev, REG_SDIO_RX0_REQ_LEN);
+
+		if (!rx_len)
+			break;
+
+		rtw_sdio_rxfifo_recv(rtwdev, rx_len);
+
+		total_rx_bytes += rx_len;
+	}
+}
+
+static void rtw_sdio_handle_interrupt(struct sdio_func *sdio_func)
+{
+	struct ieee80211_hw *hw = sdio_get_drvdata(sdio_func);
+	struct rtw_sdio *rtwsdio;
+	struct rtw_dev *rtwdev;
+	u32 hisr;
+
+	rtwdev = hw->priv;
+	rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+
+	rtwsdio->irq_thread = current;
+
+	hisr = rtw_read32(rtwdev, REG_SDIO_HISR);
+
+	if (hisr & REG_SDIO_HISR_TXERR)
+		rtw_sdio_tx_err_isr(rtwdev);
+	if (hisr & REG_SDIO_HISR_RX_REQUEST) {
+		hisr &= ~REG_SDIO_HISR_RX_REQUEST;
+		rtw_sdio_rx_isr(rtwdev);
+	}
+
+	rtw_write32(rtwdev, REG_SDIO_HISR, hisr);
+
+	rtwsdio->irq_thread = NULL;
+}
+
+static int __maybe_unused rtw_sdio_suspend(struct device *dev)
+{
+	struct sdio_func *func = dev_to_sdio_func(dev);
+	struct ieee80211_hw *hw = dev_get_drvdata(dev);
+	struct rtw_dev *rtwdev = hw->priv;
+	int ret;
+
+	ret = sdio_set_host_pm_flags(func, MMC_PM_KEEP_POWER);
+	if (ret)
+		rtw_err(rtwdev, "Failed to host PM flag MMC_PM_KEEP_POWER");
+
+	return ret;
+}
+
+static int __maybe_unused rtw_sdio_resume(struct device *dev)
+{
+	return 0;
+}
+
+SIMPLE_DEV_PM_OPS(rtw_sdio_pm_ops, rtw_sdio_suspend, rtw_sdio_resume);
+EXPORT_SYMBOL(rtw_sdio_pm_ops);
+
+static int rtw_sdio_claim(struct rtw_dev *rtwdev, struct sdio_func *sdio_func)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	int ret;
+
+	sdio_claim_host(sdio_func);
+
+	ret = sdio_enable_func(sdio_func);
+	if (ret) {
+		rtw_err(rtwdev, "Failed to enable SDIO func");
+		goto err_release_host;
+	}
+
+	ret = sdio_set_block_size(sdio_func, RTW_SDIO_BLOCK_SIZE);
+	if (ret) {
+		rtw_err(rtwdev, "Failed to set SDIO block size to 512");
+		goto err_disable_func;
+	}
+
+	rtwsdio->sdio_func = sdio_func;
+
+	rtwsdio->sdio3_bus_mode = mmc_card_uhs(sdio_func->card);
+
+	sdio_set_drvdata(sdio_func, rtwdev->hw);
+	SET_IEEE80211_DEV(rtwdev->hw, &sdio_func->dev);
+
+	sdio_release_host(sdio_func);
+
+	return 0;
+
+err_disable_func:
+	sdio_disable_func(sdio_func);
+err_release_host:
+	sdio_release_host(sdio_func);
+	return ret;
+}
+
+static void rtw_sdio_declaim(struct rtw_dev *rtwdev,
+			     struct sdio_func *sdio_func)
+{
+	sdio_claim_host(sdio_func);
+	sdio_disable_func(sdio_func);
+	sdio_release_host(sdio_func);
+}
+
+static struct rtw_hci_ops rtw_sdio_ops = {
+	.tx_write = rtw_sdio_tx_write,
+	.tx_kick_off = rtw_sdio_tx_kick_off,
+	.setup = rtw_sdio_setup,
+	.start = rtw_sdio_start,
+	.stop = rtw_sdio_stop,
+	.deep_ps = rtw_sdio_deep_ps,
+	.link_ps = rtw_sdio_link_ps,
+	.interface_cfg = rtw_sdio_interface_cfg,
+
+	.read8 = rtw_sdio_read8,
+	.read16 = rtw_sdio_read16,
+	.read32 = rtw_sdio_read32,
+	.write8 = rtw_sdio_write8,
+	.write16 = rtw_sdio_write16,
+	.write32 = rtw_sdio_write32,
+	.write_data_rsvd_page = rtw_sdio_write_data_rsvd_page,
+	.write_data_h2c = rtw_sdio_write_data_h2c,
+};
+
+static int rtw_sdio_request_irq(struct rtw_dev *rtwdev,
+				struct sdio_func *sdio_func)
+{
+	int ret;
+
+	sdio_claim_host(sdio_func);
+	ret = sdio_claim_irq(sdio_func, &rtw_sdio_handle_interrupt);
+	sdio_release_host(sdio_func);
+
+	if (ret) {
+		rtw_err(rtwdev, "failed to claim SDIO IRQ");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void rtw_sdio_indicate_tx_status(struct rtw_dev *rtwdev,
+					struct sk_buff *skb)
+{
+	struct rtw_sdio_tx_data *tx_data = rtw_sdio_get_tx_data(skb);
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_hw *hw = rtwdev->hw;
+
+	/* enqueue to wait for tx report */
+	if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) {
+		rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn);
+		return;
+	}
+
+	/* always ACK for others, then they won't be marked as drop */
+	ieee80211_tx_info_clear_status(info);
+	if (info->flags & IEEE80211_TX_CTL_NO_ACK)
+		info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
+	else
+		info->flags |= IEEE80211_TX_STAT_ACK;
+
+	ieee80211_tx_status_irqsafe(hw, skb);
+}
+
+static void rtw_sdio_process_tx_queue(struct rtw_dev *rtwdev,
+				      enum rtw_tx_queue_type queue)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	struct sk_buff *skb;
+	int ret;
+
+	skb = skb_dequeue(&rtwsdio->tx_queue[queue]);
+	if (!skb)
+		return;
+
+	ret = rtw_sdio_write_port(rtwdev, skb, queue);
+	if (ret) {
+		skb_queue_head(&rtwsdio->tx_queue[queue], skb);
+		return;
+	}
+
+	if (queue <= RTW_TX_QUEUE_VO)
+		rtw_sdio_indicate_tx_status(rtwdev, skb);
+	else
+		dev_kfree_skb_any(skb);
+}
+
+static void rtw_sdio_tx_handler(struct work_struct *work)
+{
+	struct rtw_sdio_work_data *work_data =
+		container_of(work, struct rtw_sdio_work_data, work);
+	struct rtw_sdio *rtwsdio;
+	struct rtw_dev *rtwdev;
+	int limit, queue;
+
+	rtwdev = work_data->rtwdev;
+	rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+
+	if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))
+		rtw_sdio_deep_ps_leave(rtwdev);
+
+	for (queue = RTK_MAX_TX_QUEUE_NUM - 1; queue >= 0; queue--) {
+		for (limit = 0; limit < 1000; limit++) {
+			rtw_sdio_process_tx_queue(rtwdev, queue);
+
+			if (skb_queue_empty(&rtwsdio->tx_queue[queue]))
+				break;
+		}
+	}
+}
+
+static void rtw_sdio_free_irq(struct rtw_dev *rtwdev,
+			      struct sdio_func *sdio_func)
+{
+	sdio_claim_host(sdio_func);
+	sdio_release_irq(sdio_func);
+	sdio_release_host(sdio_func);
+}
+
+static int rtw_sdio_init_tx(struct rtw_dev *rtwdev)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	int i;
+
+	rtwsdio->txwq = create_singlethread_workqueue("rtw88_sdio: tx wq");
+	if (!rtwsdio->txwq) {
+		rtw_err(rtwdev, "failed to create TX work queue\n");
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++)
+		skb_queue_head_init(&rtwsdio->tx_queue[i]);
+	rtwsdio->tx_handler_data = kmalloc(sizeof(*rtwsdio->tx_handler_data),
+					   GFP_KERNEL);
+	if (!rtwsdio->tx_handler_data)
+		goto err_destroy_wq;
+
+	rtwsdio->tx_handler_data->rtwdev = rtwdev;
+	INIT_WORK(&rtwsdio->tx_handler_data->work, rtw_sdio_tx_handler);
+
+	return 0;
+
+err_destroy_wq:
+	destroy_workqueue(rtwsdio->txwq);
+	return -ENOMEM;
+}
+
+static void rtw_sdio_deinit_tx(struct rtw_dev *rtwdev)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+	int i;
+
+	for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++)
+		skb_queue_purge(&rtwsdio->tx_queue[i]);
+
+	flush_workqueue(rtwsdio->txwq);
+	destroy_workqueue(rtwsdio->txwq);
+	kfree(rtwsdio->tx_handler_data);
+}
+
+int rtw_sdio_probe(struct sdio_func *sdio_func,
+		   const struct sdio_device_id *id)
+{
+	struct ieee80211_hw *hw;
+	struct rtw_dev *rtwdev;
+	int drv_data_size;
+	int ret;
+
+	drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_sdio);
+	hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops);
+	if (!hw) {
+		dev_err(&sdio_func->dev, "failed to allocate hw");
+		return -ENOMEM;
+	}
+
+	rtwdev = hw->priv;
+	rtwdev->hw = hw;
+	rtwdev->dev = &sdio_func->dev;
+	rtwdev->chip = (struct rtw_chip_info *)id->driver_data;
+	rtwdev->hci.ops = &rtw_sdio_ops;
+	rtwdev->hci.type = RTW_HCI_TYPE_SDIO;
+
+	ret = rtw_core_init(rtwdev);
+	if (ret)
+		goto err_release_hw;
+
+	rtw_dbg(rtwdev, RTW_DBG_SDIO,
+		"rtw88 SDIO probe: vendor=0x%04x device=%04x class=%02x",
+		id->vendor, id->device, id->class);
+
+	ret = rtw_sdio_claim(rtwdev, sdio_func);
+	if (ret) {
+		rtw_err(rtwdev, "failed to claim SDIO device");
+		goto err_deinit_core;
+	}
+
+	rtw_sdio_init(rtwdev);
+
+	ret = rtw_sdio_init_tx(rtwdev);
+	if (ret) {
+		rtw_err(rtwdev, "failed to init SDIO TX queue\n");
+		goto err_sdio_declaim;
+	}
+
+	ret = rtw_chip_info_setup(rtwdev);
+	if (ret) {
+		rtw_err(rtwdev, "failed to setup chip information");
+		goto err_destroy_txwq;
+	}
+
+	ret = rtw_sdio_request_irq(rtwdev, sdio_func);
+	if (ret)
+		goto err_destroy_txwq;
+
+	ret = rtw_register_hw(rtwdev, hw);
+	if (ret) {
+		rtw_err(rtwdev, "failed to register hw");
+		goto err_free_irq;
+	}
+
+	return 0;
+
+err_free_irq:
+	rtw_sdio_free_irq(rtwdev, sdio_func);
+err_destroy_txwq:
+	rtw_sdio_deinit_tx(rtwdev);
+err_sdio_declaim:
+	rtw_sdio_declaim(rtwdev, sdio_func);
+err_deinit_core:
+	rtw_core_deinit(rtwdev);
+err_release_hw:
+	ieee80211_free_hw(hw);
+
+	return ret;
+}
+EXPORT_SYMBOL(rtw_sdio_probe);
+
+void rtw_sdio_remove(struct sdio_func *sdio_func)
+{
+	struct ieee80211_hw *hw = sdio_get_drvdata(sdio_func);
+	struct rtw_dev *rtwdev;
+
+	if (!hw)
+		return;
+
+	rtwdev = hw->priv;
+
+	rtw_unregister_hw(rtwdev, hw);
+	rtw_sdio_disable_interrupt(rtwdev);
+	rtw_sdio_free_irq(rtwdev, sdio_func);
+	rtw_sdio_declaim(rtwdev, sdio_func);
+	rtw_sdio_deinit_tx(rtwdev);
+	rtw_core_deinit(rtwdev);
+	ieee80211_free_hw(hw);
+}
+EXPORT_SYMBOL(rtw_sdio_remove);
+
+void rtw_sdio_shutdown(struct device *dev)
+{
+	struct sdio_func *sdio_func = dev_to_sdio_func(dev);
+	const struct rtw_chip_info *chip;
+	struct ieee80211_hw *hw;
+	struct rtw_dev *rtwdev;
+
+	hw = sdio_get_drvdata(sdio_func);
+	if (!hw)
+		return;
+
+	rtwdev = hw->priv;
+	chip = rtwdev->chip;
+
+	if (chip->ops->shutdown)
+		chip->ops->shutdown(rtwdev);
+}
+EXPORT_SYMBOL(rtw_sdio_shutdown);
+
+MODULE_AUTHOR("Martin Blumenstingl");
+MODULE_AUTHOR("Jernej Skrabec");
+MODULE_DESCRIPTION("Realtek 802.11ac wireless SDIO driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw88/sdio.h b/drivers/net/wireless/realtek/rtw88/sdio.h
new file mode 100644
index 000000000000..3c659ed180f0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw88/sdio.h
@@ -0,0 +1,178 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/* Copyright (C) 2021 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ * Copyright (C) 2021 Jernej Skrabec <jernej.skrabec@gmail.com>
+ */
+
+#ifndef __REG_SDIO_H_
+#define __REG_SDIO_H_
+
+/* I/O bus domain address mapping */
+#define SDIO_LOCAL_OFFSET			0x10250000
+#define WLAN_IOREG_OFFSET			0x10260000
+#define FIRMWARE_FIFO_OFFSET			0x10270000
+#define TX_HIQ_OFFSET				0x10310000
+#define TX_MIQ_OFFSET				0x10320000
+#define TX_LOQ_OFFSET				0x10330000
+#define TX_EPQ_OFFSET				0x10350000
+#define RX_RX0FF_OFFSET				0x10340000
+
+#define RTW_SDIO_BUS_MSK			0xffff0000
+#define SDIO_LOCAL_REG_MSK			0x00000fff
+#define WLAN_IOREG_REG_MSK			0x0000ffff
+
+/* SDIO Tx Control */
+#define REG_SDIO_TX_CTRL			(SDIO_LOCAL_OFFSET + 0x0000)
+
+/*SDIO status timeout*/
+#define REG_SDIO_TIMEOUT			(SDIO_LOCAL_OFFSET + 0x0002)
+
+/* SDIO Host Interrupt Mask */
+#define REG_SDIO_HIMR				(SDIO_LOCAL_OFFSET + 0x0014)
+#define REG_SDIO_HIMR_RX_REQUEST		BIT(0)
+#define REG_SDIO_HIMR_AVAL			BIT(1)
+#define REG_SDIO_HIMR_TXERR			BIT(2)
+#define REG_SDIO_HIMR_RXERR			BIT(3)
+#define REG_SDIO_HIMR_TXFOVW			BIT(4)
+#define REG_SDIO_HIMR_RXFOVW			BIT(5)
+#define REG_SDIO_HIMR_TXBCNOK			BIT(6)
+#define REG_SDIO_HIMR_TXBCNERR			BIT(7)
+#define REG_SDIO_HIMR_BCNERLY_INT		BIT(16)
+#define REG_SDIO_HIMR_C2HCMD			BIT(17)
+#define REG_SDIO_HIMR_CPWM1			BIT(18)
+#define REG_SDIO_HIMR_CPWM2			BIT(19)
+#define REG_SDIO_HIMR_HSISR_IND			BIT(20)
+#define REG_SDIO_HIMR_GTINT3_IND		BIT(21)
+#define REG_SDIO_HIMR_GTINT4_IND		BIT(22)
+#define REG_SDIO_HIMR_PSTIMEOUT			BIT(23)
+#define REG_SDIO_HIMR_OCPINT			BIT(24)
+#define REG_SDIO_HIMR_ATIMEND			BIT(25)
+#define REG_SDIO_HIMR_ATIMEND_E			BIT(26)
+#define REG_SDIO_HIMR_CTWEND			BIT(27)
+/* the following two are RTL8188 SDIO Specific */
+#define REG_SDIO_HIMR_MCU_ERR			BIT(28)
+#define REG_SDIO_HIMR_TSF_BIT32_TOGGLE		BIT(29)
+
+/* SDIO Host Interrupt Service Routine */
+#define REG_SDIO_HISR				(SDIO_LOCAL_OFFSET + 0x0018)
+#define REG_SDIO_HISR_RX_REQUEST		BIT(0)
+#define REG_SDIO_HISR_AVAL			BIT(1)
+#define REG_SDIO_HISR_TXERR			BIT(2)
+#define REG_SDIO_HISR_RXERR			BIT(3)
+#define REG_SDIO_HISR_TXFOVW			BIT(4)
+#define REG_SDIO_HISR_RXFOVW			BIT(5)
+#define REG_SDIO_HISR_TXBCNOK			BIT(6)
+#define REG_SDIO_HISR_TXBCNERR			BIT(7)
+#define REG_SDIO_HISR_BCNERLY_INT		BIT(16)
+#define REG_SDIO_HISR_C2HCMD			BIT(17)
+#define REG_SDIO_HISR_CPWM1			BIT(18)
+#define REG_SDIO_HISR_CPWM2			BIT(19)
+#define REG_SDIO_HISR_HSISR_IND			BIT(20)
+#define REG_SDIO_HISR_GTINT3_IND		BIT(21)
+#define REG_SDIO_HISR_GTINT4_IND		BIT(22)
+#define REG_SDIO_HISR_PSTIMEOUT			BIT(23)
+#define REG_SDIO_HISR_OCPINT			BIT(24)
+#define REG_SDIO_HISR_ATIMEND			BIT(25)
+#define REG_SDIO_HISR_ATIMEND_E			BIT(26)
+#define REG_SDIO_HISR_CTWEND			BIT(27)
+/* the following two are RTL8188 SDIO Specific */
+#define REG_SDIO_HISR_MCU_ERR			BIT(28)
+#define REG_SDIO_HISR_TSF_BIT32_TOGGLE		BIT(29)
+
+/* HCI Current Power Mode */
+#define REG_SDIO_HCPWM				(SDIO_LOCAL_OFFSET + 0x0019)
+/* RXDMA Request Length */
+#define REG_SDIO_RX0_REQ_LEN			(SDIO_LOCAL_OFFSET + 0x001C)
+/* OQT Free Page */
+#define REG_SDIO_OQT_FREE_PG			(SDIO_LOCAL_OFFSET + 0x001E)
+/* Free Tx Buffer Page */
+#define REG_SDIO_FREE_TXPG			(SDIO_LOCAL_OFFSET + 0x0020)
+/* HCI Current Power Mode 1 */
+#define REG_SDIO_HCPWM1				(SDIO_LOCAL_OFFSET + 0x0024)
+/* HCI Current Power Mode 2 */
+#define REG_SDIO_HCPWM2				(SDIO_LOCAL_OFFSET + 0x0026)
+/* Free Tx Page Sequence */
+#define REG_SDIO_FREE_TXPG_SEQ			(SDIO_LOCAL_OFFSET + 0x0028)
+/* HTSF Information */
+#define REG_SDIO_HTSFR_INFO			(SDIO_LOCAL_OFFSET + 0x0030)
+#define REG_SDIO_HCPWM1_V2			(SDIO_LOCAL_OFFSET + 0x0038)
+/* H2C */
+#define REG_SDIO_H2C				(SDIO_LOCAL_OFFSET + 0x0060)
+/* HCI Request Power Mode 1 */
+#define REG_SDIO_HRPWM1				(SDIO_LOCAL_OFFSET + 0x0080)
+/* HCI Request Power Mode 2 */
+#define REG_SDIO_HRPWM2				(SDIO_LOCAL_OFFSET + 0x0082)
+/* HCI Power Save Clock */
+#define REG_SDIO_HPS_CLKR			(SDIO_LOCAL_OFFSET + 0x0084)
+/* SDIO HCI Suspend Control */
+#define REG_SDIO_HSUS_CTRL			(SDIO_LOCAL_OFFSET + 0x0086)
+#define BIT_HCI_SUS_REQ				BIT(0)
+#define BIT_HCI_RESUME_RDY			BIT(1)
+/* SDIO Host Extension Interrupt Mask Always */
+#define REG_SDIO_HIMR_ON			(SDIO_LOCAL_OFFSET + 0x0090)
+/* SDIO Host Extension Interrupt Status Always */
+#define REG_SDIO_HISR_ON			(SDIO_LOCAL_OFFSET + 0x0091)
+
+#define REG_SDIO_INDIRECT_REG_CFG		(SDIO_LOCAL_OFFSET + 0x0040)
+#define BIT_SDIO_INDIRECT_REG_CFG_WORD		BIT(16)
+#define BIT_SDIO_INDIRECT_REG_CFG_DWORD		BIT(17)
+#define BIT_SDIO_INDIRECT_REG_CFG_WRITE		BIT(18)
+#define BIT_SDIO_INDIRECT_REG_CFG_READ		BIT(19)
+#define BIT_SDIO_INDIRECT_REG_CFG_UNK20		BIT(20)
+#define REG_SDIO_INDIRECT_REG_DATA		(SDIO_LOCAL_OFFSET + 0x0044)
+
+/* Sdio Address for SDIO Local Reg, TRX FIFO, MAC Reg */
+#define REG_SDIO_CMD_ADDR_MSK			GENMASK(16, 13)
+#define REG_SDIO_CMD_ADDR_SDIO_REG		0
+#define REG_SDIO_CMD_ADDR_MAC_REG		8
+#define REG_SDIO_CMD_ADDR_TXFF_HIGH		4
+#define REG_SDIO_CMD_ADDR_TXFF_LOW		6
+#define REG_SDIO_CMD_ADDR_TXFF_NORMAL		5
+#define REG_SDIO_CMD_ADDR_TXFF_EXTRA		7
+#define REG_SDIO_CMD_ADDR_RXFF			7
+
+#define RTW_SDIO_BLOCK_SIZE			512
+#define RTW_SDIO_ADDR_RX_RX0FF_GEN(_id)		(0x0e000 | ((_id) & 0x3))
+
+#define RTW_SDIO_DATA_PTR_ALIGN			8
+
+struct sdio_func;
+struct sdio_device_id;
+
+struct rtw_sdio_tx_data {
+	u8 sn;
+};
+
+struct rtw_sdio_work_data {
+	struct work_struct work;
+	struct rtw_dev *rtwdev;
+};
+
+struct rtw_sdio {
+	struct sdio_func *sdio_func;
+
+	u32 irq_mask;
+	u8 rx_addr;
+	bool sdio3_bus_mode;
+
+	void *irq_thread;
+
+	struct workqueue_struct *txwq;
+	struct rtw_sdio_work_data *tx_handler_data;
+	struct sk_buff_head tx_queue[RTK_MAX_TX_QUEUE_NUM];
+};
+
+extern const struct dev_pm_ops rtw_sdio_pm_ops;
+
+int rtw_sdio_probe(struct sdio_func *sdio_func,
+		   const struct sdio_device_id *id);
+void rtw_sdio_remove(struct sdio_func *sdio_func);
+void rtw_sdio_shutdown(struct device *dev);
+
+static inline bool rtw_sdio_is_sdio30_supported(struct rtw_dev *rtwdev)
+{
+	struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv;
+
+	return rtwsdio->sdio3_bus_mode;
+}
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtw88/usb.c b/drivers/net/wireless/realtek/rtw88/usb.c
index 2a8336b1847a..44a5fafb9905 100644
--- a/drivers/net/wireless/realtek/rtw88/usb.c
+++ b/drivers/net/wireless/realtek/rtw88/usb.c
@@ -118,6 +118,22 @@ static void rtw_usb_write32(struct rtw_dev *rtwdev, u32 addr, u32 val)
 	rtw_usb_write(rtwdev, addr, val, 4);
 }
 
+static int dma_mapping_to_ep(enum rtw_dma_mapping dma_mapping)
+{
+	switch (dma_mapping) {
+	case RTW_DMA_MAPPING_HIGH:
+		return 0;
+	case RTW_DMA_MAPPING_NORMAL:
+		return 1;
+	case RTW_DMA_MAPPING_LOW:
+		return 2;
+	case RTW_DMA_MAPPING_EXTRA:
+		return 3;
+	default:
+		return -EINVAL;
+	}
+}
+
 static int rtw_usb_parse(struct rtw_dev *rtwdev,
 			 struct usb_interface *interface)
 {
@@ -129,6 +145,8 @@ static int rtw_usb_parse(struct rtw_dev *rtwdev,
 	int num_out_pipes = 0;
 	int i;
 	u8 num;
+	const struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_rqpn *rqpn;
 
 	for (i = 0; i < interface_desc->bNumEndpoints; i++) {
 		endpoint = &host_interface->endpoint[i].desc;
@@ -183,31 +201,34 @@ static int rtw_usb_parse(struct rtw_dev *rtwdev,
 
 	rtwdev->hci.bulkout_num = num_out_pipes;
 
-	switch (num_out_pipes) {
-	case 4:
-	case 3:
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID0] = 2;
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID1] = 2;
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID2] = 2;
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID3] = 2;
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID4] = 1;
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID5] = 1;
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID6] = 0;
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID7] = 0;
-		break;
-	case 2:
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID0] = 1;
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID1] = 1;
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID2] = 1;
-		rtwusb->qsel_to_ep[TX_DESC_QSEL_TID3] = 1;
-		break;
-	case 1:
-		break;
-	default:
-		rtw_err(rtwdev, "failed to get out_pipes(%d)\n", num_out_pipes);
+	if (num_out_pipes < 1 || num_out_pipes > 4) {
+		rtw_err(rtwdev, "invalid number of endpoints %d\n", num_out_pipes);
 		return -EINVAL;
 	}
 
+	rqpn = &chip->rqpn_table[num_out_pipes];
+
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID0] = dma_mapping_to_ep(rqpn->dma_map_be);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID1] = dma_mapping_to_ep(rqpn->dma_map_bk);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID2] = dma_mapping_to_ep(rqpn->dma_map_bk);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID3] = dma_mapping_to_ep(rqpn->dma_map_be);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID4] = dma_mapping_to_ep(rqpn->dma_map_vi);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID5] = dma_mapping_to_ep(rqpn->dma_map_vi);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID6] = dma_mapping_to_ep(rqpn->dma_map_vo);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID7] = dma_mapping_to_ep(rqpn->dma_map_vo);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID8] = -EINVAL;
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID9] = -EINVAL;
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID10] = -EINVAL;
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID11] = -EINVAL;
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID12] = -EINVAL;
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID13] = -EINVAL;
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID14] = -EINVAL;
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_TID15] = -EINVAL;
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_BEACON] = dma_mapping_to_ep(rqpn->dma_map_hi);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_HIGH] = dma_mapping_to_ep(rqpn->dma_map_hi);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_MGMT] = dma_mapping_to_ep(rqpn->dma_map_mg);
+	rtwusb->qsel_to_ep[TX_DESC_QSEL_H2C] = dma_mapping_to_ep(rqpn->dma_map_hi);
+
 	return 0;
 }
 
@@ -250,7 +271,7 @@ static void rtw_usb_write_port_tx_complete(struct urb *urb)
 static int qsel_to_ep(struct rtw_usb *rtwusb, unsigned int qsel)
 {
 	if (qsel >= ARRAY_SIZE(rtwusb->qsel_to_ep))
-		return 0;
+		return -EINVAL;
 
 	return rtwusb->qsel_to_ep[qsel];
 }
@@ -265,6 +286,9 @@ static int rtw_usb_write_port(struct rtw_dev *rtwdev, u8 qsel, struct sk_buff *s
 	int ret;
 	int ep = qsel_to_ep(rtwusb, qsel);
 
+	if (ep < 0)
+		return ep;
+
 	pipe = usb_sndbulkpipe(usbd, rtwusb->out_ep[ep]);
 	urb = usb_alloc_urb(0, GFP_ATOMIC);
 	if (!urb)
@@ -780,6 +804,7 @@ static void rtw_usb_intf_deinit(struct rtw_dev *rtwdev,
 	struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev);
 
 	usb_put_dev(rtwusb->udev);
+	kfree(rtwusb->usb_data);
 	usb_set_intfdata(intf, NULL);
 }
 
@@ -808,7 +833,7 @@ int rtw_usb_probe(struct usb_interface *intf, const struct usb_device_id *id)
 
 	ret = rtw_usb_alloc_rx_bufs(rtwusb);
 	if (ret)
-		return ret;
+		goto err_release_hw;
 
 	ret = rtw_core_init(rtwdev);
 	if (ret)
diff --git a/drivers/net/wireless/realtek/rtw89/chan.c b/drivers/net/wireless/realtek/rtw89/chan.c
index 90596806bc93..4663db4ce2f6 100644
--- a/drivers/net/wireless/realtek/rtw89/chan.c
+++ b/drivers/net/wireless/realtek/rtw89/chan.c
@@ -141,6 +141,38 @@ void rtw89_config_entity_chandef(struct rtw89_dev *rtwdev,
 	__rtw89_config_entity_chandef(rtwdev, idx, chandef, true);
 }
 
+void rtw89_config_roc_chandef(struct rtw89_dev *rtwdev,
+			      enum rtw89_sub_entity_idx idx,
+			      const struct cfg80211_chan_def *chandef)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	enum rtw89_sub_entity_idx cur;
+
+	if (chandef) {
+		cur = atomic_cmpxchg(&hal->roc_entity_idx,
+				     RTW89_SUB_ENTITY_IDLE, idx);
+		if (cur != RTW89_SUB_ENTITY_IDLE) {
+			rtw89_debug(rtwdev, RTW89_DBG_TXRX,
+				    "ROC still processing on entity %d\n", idx);
+			return;
+		}
+
+		hal->roc_chandef = *chandef;
+	} else {
+		cur = atomic_cmpxchg(&hal->roc_entity_idx, idx,
+				     RTW89_SUB_ENTITY_IDLE);
+		if (cur == idx)
+			return;
+
+		if (cur == RTW89_SUB_ENTITY_IDLE)
+			rtw89_debug(rtwdev, RTW89_DBG_TXRX,
+				    "ROC already finished on entity %d\n", idx);
+		else
+			rtw89_debug(rtwdev, RTW89_DBG_TXRX,
+				    "ROC is processing on entity %d\n", cur);
+	}
+}
+
 static void rtw89_config_default_chandef(struct rtw89_dev *rtwdev)
 {
 	struct cfg80211_chan_def chandef = {0};
@@ -154,6 +186,7 @@ void rtw89_entity_init(struct rtw89_dev *rtwdev)
 	struct rtw89_hal *hal = &rtwdev->hal;
 
 	bitmap_zero(hal->entity_map, NUM_OF_RTW89_SUB_ENTITY);
+	atomic_set(&hal->roc_entity_idx, RTW89_SUB_ENTITY_IDLE);
 	rtw89_config_default_chandef(rtwdev);
 }
 
@@ -229,6 +262,8 @@ void rtw89_chanctx_ops_remove(struct rtw89_dev *rtwdev,
 			rtwvif->sub_entity_idx = RTW89_SUB_ENTITY_0;
 	}
 
+	atomic_cmpxchg(&hal->roc_entity_idx, roll, RTW89_SUB_ENTITY_0);
+
 	drop = roll;
 
 out:
diff --git a/drivers/net/wireless/realtek/rtw89/chan.h b/drivers/net/wireless/realtek/rtw89/chan.h
index ecbd4503bead..bdf369db5041 100644
--- a/drivers/net/wireless/realtek/rtw89/chan.h
+++ b/drivers/net/wireless/realtek/rtw89/chan.h
@@ -45,6 +45,9 @@ bool rtw89_assign_entity_chan(struct rtw89_dev *rtwdev,
 void rtw89_config_entity_chandef(struct rtw89_dev *rtwdev,
 				 enum rtw89_sub_entity_idx idx,
 				 const struct cfg80211_chan_def *chandef);
+void rtw89_config_roc_chandef(struct rtw89_dev *rtwdev,
+			      enum rtw89_sub_entity_idx idx,
+			      const struct cfg80211_chan_def *chandef);
 void rtw89_entity_init(struct rtw89_dev *rtwdev);
 enum rtw89_entity_mode rtw89_entity_recalc(struct rtw89_dev *rtwdev);
 int rtw89_chanctx_ops_add(struct rtw89_dev *rtwdev,
diff --git a/drivers/net/wireless/realtek/rtw89/coex.c b/drivers/net/wireless/realtek/rtw89/coex.c
index bcf483cafd20..acb3fac0c96d 100644
--- a/drivers/net/wireless/realtek/rtw89/coex.c
+++ b/drivers/net/wireless/realtek/rtw89/coex.c
@@ -9,7 +9,7 @@
 #include "ps.h"
 #include "reg.h"
 
-#define RTW89_COEX_VERSION 0x07000013
+#define RTW89_COEX_VERSION 0x07000113
 #define FCXDEF_STEP 50 /* MUST <= FCXMAX_STEP and match with wl fw*/
 
 enum btc_fbtc_tdma_template {
@@ -148,6 +148,13 @@ static const struct rtw89_btc_ver rtw89_btc_ver_defs[] = {
 	 .fwlrole = 1,   .frptmap = 2,    .fcxctrl = 1,
 	 .info_buf = 1280, .max_role_num = 5,
 	},
+	{RTL8852B, RTW89_FW_VER_CODE(0, 29, 29, 0),
+	 .fcxbtcrpt = 105, .fcxtdma = 3,  .fcxslots = 1, .fcxcysta = 5,
+	 .fcxstep = 3,   .fcxnullsta = 2, .fcxmreg = 2,  .fcxgpiodbg = 1,
+	 .fcxbtver = 1,  .fcxbtscan = 2,  .fcxbtafh = 2, .fcxbtdevinfo = 1,
+	 .fwlrole = 1,   .frptmap = 3,    .fcxctrl = 1,
+	 .info_buf = 1800, .max_role_num = 6,
+	},
 	{RTL8852B, RTW89_FW_VER_CODE(0, 29, 14, 0),
 	 .fcxbtcrpt = 5, .fcxtdma = 3,    .fcxslots = 1, .fcxcysta = 4,
 	 .fcxstep = 3,   .fcxnullsta = 2, .fcxmreg = 1,  .fcxgpiodbg = 1,
@@ -226,7 +233,6 @@ struct rtw89_btc_btf_set_slot_table {
 	u8 buf[];
 } __packed;
 
-#define BTF_SET_MON_REG_VER 1
 struct rtw89_btc_btf_set_mon_reg {
 	u8 fver;
 	u8 reg_num;
@@ -734,6 +740,7 @@ static void _reset_btc_var(struct rtw89_dev *rtwdev, u8 type)
 
 #define BTC_RPT_HDR_SIZE 3
 #define BTC_CHK_WLSLOT_DRIFT_MAX 15
+#define BTC_CHK_BTSLOT_DRIFT_MAX 15
 #define BTC_CHK_HANG_MAX 3
 
 static void _chk_btc_err(struct rtw89_dev *rtwdev, u8 type, u32 cnt)
@@ -748,62 +755,76 @@ static void _chk_btc_err(struct rtw89_dev *rtwdev, u8 type, u32 cnt)
 		    __func__, type, cnt);
 
 	switch (type) {
-	case BTC_DCNT_RPT_FREEZE:
+	case BTC_DCNT_RPT_HANG:
 		if (dm->cnt_dm[BTC_DCNT_RPT] == cnt && btc->fwinfo.rpt_en_map)
-			dm->cnt_dm[BTC_DCNT_RPT_FREEZE]++;
+			dm->cnt_dm[BTC_DCNT_RPT_HANG]++;
 		else
-			dm->cnt_dm[BTC_DCNT_RPT_FREEZE] = 0;
+			dm->cnt_dm[BTC_DCNT_RPT_HANG] = 0;
 
-		if (dm->cnt_dm[BTC_DCNT_RPT_FREEZE] >= BTC_CHK_HANG_MAX)
+		if (dm->cnt_dm[BTC_DCNT_RPT_HANG] >= BTC_CHK_HANG_MAX)
 			dm->error.map.wl_fw_hang = true;
 		else
 			dm->error.map.wl_fw_hang = false;
 
 		dm->cnt_dm[BTC_DCNT_RPT] = cnt;
 		break;
-	case BTC_DCNT_CYCLE_FREEZE:
+	case BTC_DCNT_CYCLE_HANG:
 		if (dm->cnt_dm[BTC_DCNT_CYCLE] == cnt &&
 		    (dm->tdma_now.type != CXTDMA_OFF ||
 		     dm->tdma_now.ext_ctrl == CXECTL_EXT))
-			dm->cnt_dm[BTC_DCNT_CYCLE_FREEZE]++;
+			dm->cnt_dm[BTC_DCNT_CYCLE_HANG]++;
 		else
-			dm->cnt_dm[BTC_DCNT_CYCLE_FREEZE] = 0;
+			dm->cnt_dm[BTC_DCNT_CYCLE_HANG] = 0;
 
-		if (dm->cnt_dm[BTC_DCNT_CYCLE_FREEZE] >= BTC_CHK_HANG_MAX)
+		if (dm->cnt_dm[BTC_DCNT_CYCLE_HANG] >= BTC_CHK_HANG_MAX)
 			dm->error.map.cycle_hang = true;
 		else
 			dm->error.map.cycle_hang = false;
 
 		dm->cnt_dm[BTC_DCNT_CYCLE] = cnt;
 		break;
-	case BTC_DCNT_W1_FREEZE:
+	case BTC_DCNT_W1_HANG:
 		if (dm->cnt_dm[BTC_DCNT_W1] == cnt &&
 		    dm->tdma_now.type != CXTDMA_OFF)
-			dm->cnt_dm[BTC_DCNT_W1_FREEZE]++;
+			dm->cnt_dm[BTC_DCNT_W1_HANG]++;
 		else
-			dm->cnt_dm[BTC_DCNT_W1_FREEZE] = 0;
+			dm->cnt_dm[BTC_DCNT_W1_HANG] = 0;
 
-		if (dm->cnt_dm[BTC_DCNT_W1_FREEZE] >= BTC_CHK_HANG_MAX)
+		if (dm->cnt_dm[BTC_DCNT_W1_HANG] >= BTC_CHK_HANG_MAX)
 			dm->error.map.w1_hang = true;
 		else
 			dm->error.map.w1_hang = false;
 
 		dm->cnt_dm[BTC_DCNT_W1] = cnt;
 		break;
-	case BTC_DCNT_B1_FREEZE:
+	case BTC_DCNT_B1_HANG:
 		if (dm->cnt_dm[BTC_DCNT_B1] == cnt &&
 		    dm->tdma_now.type != CXTDMA_OFF)
-			dm->cnt_dm[BTC_DCNT_B1_FREEZE]++;
+			dm->cnt_dm[BTC_DCNT_B1_HANG]++;
 		else
-			dm->cnt_dm[BTC_DCNT_B1_FREEZE] = 0;
+			dm->cnt_dm[BTC_DCNT_B1_HANG] = 0;
 
-		if (dm->cnt_dm[BTC_DCNT_B1_FREEZE] >= BTC_CHK_HANG_MAX)
+		if (dm->cnt_dm[BTC_DCNT_B1_HANG] >= BTC_CHK_HANG_MAX)
 			dm->error.map.b1_hang = true;
 		else
 			dm->error.map.b1_hang = false;
 
 		dm->cnt_dm[BTC_DCNT_B1] = cnt;
 		break;
+	case BTC_DCNT_E2G_HANG:
+		if (dm->cnt_dm[BTC_DCNT_E2G] == cnt &&
+		    dm->tdma_now.ext_ctrl == CXECTL_EXT)
+			dm->cnt_dm[BTC_DCNT_E2G_HANG]++;
+		else
+			dm->cnt_dm[BTC_DCNT_E2G_HANG] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_E2G_HANG] >= BTC_CHK_HANG_MAX)
+			dm->error.map.wl_e2g_hang = true;
+		else
+			dm->error.map.wl_e2g_hang = false;
+
+		dm->cnt_dm[BTC_DCNT_E2G] = cnt;
+		break;
 	case BTC_DCNT_TDMA_NONSYNC:
 		if (cnt != 0) /* if tdma not sync between drv/fw  */
 			dm->cnt_dm[BTC_DCNT_TDMA_NONSYNC]++;
@@ -822,23 +843,23 @@ static void _chk_btc_err(struct rtw89_dev *rtwdev, u8 type, u32 cnt)
 			dm->cnt_dm[BTC_DCNT_SLOT_NONSYNC] = 0;
 
 		if (dm->cnt_dm[BTC_DCNT_SLOT_NONSYNC] >= BTC_CHK_HANG_MAX)
-			dm->error.map.tdma_no_sync = true;
+			dm->error.map.slot_no_sync = true;
 		else
-			dm->error.map.tdma_no_sync = false;
+			dm->error.map.slot_no_sync = false;
 		break;
-	case BTC_DCNT_BTCNT_FREEZE:
+	case BTC_DCNT_BTCNT_HANG:
 		cnt = cx->cnt_bt[BTC_BCNT_HIPRI_RX] +
 		      cx->cnt_bt[BTC_BCNT_HIPRI_TX] +
 		      cx->cnt_bt[BTC_BCNT_LOPRI_RX] +
 		      cx->cnt_bt[BTC_BCNT_LOPRI_TX];
 
 		if (cnt == 0)
-			dm->cnt_dm[BTC_DCNT_BTCNT_FREEZE]++;
+			dm->cnt_dm[BTC_DCNT_BTCNT_HANG]++;
 		else
-			dm->cnt_dm[BTC_DCNT_BTCNT_FREEZE] = 0;
+			dm->cnt_dm[BTC_DCNT_BTCNT_HANG] = 0;
 
-		if ((dm->cnt_dm[BTC_DCNT_BTCNT_FREEZE] >= BTC_CHK_HANG_MAX &&
-		     bt->enable.now) || (!dm->cnt_dm[BTC_DCNT_BTCNT_FREEZE] &&
+		if ((dm->cnt_dm[BTC_DCNT_BTCNT_HANG] >= BTC_CHK_HANG_MAX &&
+		     bt->enable.now) || (!dm->cnt_dm[BTC_DCNT_BTCNT_HANG] &&
 		     !bt->enable.now))
 			_update_bt_scbd(rtwdev, false);
 		break;
@@ -853,6 +874,18 @@ static void _chk_btc_err(struct rtw89_dev *rtwdev, u8 type, u32 cnt)
 		else
 			dm->error.map.wl_slot_drift = false;
 		break;
+	case BTC_DCNT_BT_SLOT_DRIFT:
+		if (cnt >= BTC_CHK_BTSLOT_DRIFT_MAX)
+			dm->cnt_dm[BTC_DCNT_BT_SLOT_DRIFT]++;
+		else
+			dm->cnt_dm[BTC_DCNT_BT_SLOT_DRIFT] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_BT_SLOT_DRIFT] >= BTC_CHK_HANG_MAX)
+			dm->error.map.bt_slot_drift = true;
+		else
+			dm->error.map.bt_slot_drift = false;
+
+		break;
 	}
 }
 
@@ -864,13 +897,15 @@ static void _update_bt_report(struct rtw89_dev *rtwdev, u8 rpt_type, u8 *pfinfo)
 	struct rtw89_btc_bt_link_info *bt_linfo = &bt->link_info;
 	struct rtw89_btc_bt_a2dp_desc *a2dp = &bt_linfo->a2dp_desc;
 	struct rtw89_btc_fbtc_btver *pver = NULL;
-	struct rtw89_btc_fbtc_btscan *pscan = NULL;
+	struct rtw89_btc_fbtc_btscan_v1 *pscan_v1;
+	struct rtw89_btc_fbtc_btscan_v2 *pscan_v2;
 	struct rtw89_btc_fbtc_btafh *pafh_v1 = NULL;
 	struct rtw89_btc_fbtc_btafh_v2 *pafh_v2 = NULL;
 	struct rtw89_btc_fbtc_btdevinfo *pdev = NULL;
+	bool scan_update = true;
+	int i;
 
 	pver = (struct rtw89_btc_fbtc_btver *)pfinfo;
-	pscan = (struct rtw89_btc_fbtc_btscan *)pfinfo;
 	pdev = (struct rtw89_btc_fbtc_btdevinfo *)pfinfo;
 
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
@@ -884,7 +919,26 @@ static void _update_bt_report(struct rtw89_dev *rtwdev, u8 rpt_type, u8 *pfinfo)
 		bt->feature = le32_to_cpu(pver->feature);
 		break;
 	case BTC_RPT_TYPE_BT_SCAN:
-		memcpy(bt->scan_info, pscan->scan, BTC_SCAN_MAX1);
+		if (ver->fcxbtscan == 1) {
+			pscan_v1 = (struct rtw89_btc_fbtc_btscan_v1 *)pfinfo;
+			for (i = 0; i < BTC_SCAN_MAX1; i++) {
+				bt->scan_info_v1[i] = pscan_v1->scan[i];
+				if (bt->scan_info_v1[i].win == 0 &&
+				    bt->scan_info_v1[i].intvl == 0)
+					scan_update = false;
+			}
+		} else if (ver->fcxbtscan == 2) {
+			pscan_v2 = (struct rtw89_btc_fbtc_btscan_v2 *)pfinfo;
+			for (i = 0; i < CXSCAN_MAX; i++) {
+				bt->scan_info_v2[i] = pscan_v2->para[i];
+				if ((pscan_v2->type & BIT(i)) &&
+				    pscan_v2->para[i].win == 0 &&
+				    pscan_v2->para[i].intvl == 0)
+					scan_update = false;
+			}
+		}
+		if (scan_update)
+			bt->scan_info_update = 1;
 		break;
 	case BTC_RPT_TYPE_BT_AFH:
 		if (ver->fcxbtafh == 2) {
@@ -940,8 +994,8 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 	void *rpt_content = NULL, *pfinfo = NULL;
 	u8 rpt_type = 0;
 	u16 wl_slot_set = 0, wl_slot_real = 0;
-	u32 trace_step = btc->ctrl.trace_step, rpt_len = 0, diff_t;
-	u32 cnt_leak_slot = 0, bt_slot_real = 0, cnt_rx_imr = 0;
+	u32 trace_step = btc->ctrl.trace_step, rpt_len = 0, diff_t = 0;
+	u32 cnt_leak_slot, bt_slot_real, bt_slot_set, cnt_rx_imr;
 	u8 i;
 
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
@@ -975,6 +1029,11 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		} else if (ver->fcxbtcrpt == 5) {
 			pfinfo = &pfwinfo->rpt_ctrl.finfo.v5;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo.v5);
+		} else if (ver->fcxbtcrpt == 105) {
+			pfinfo = &pfwinfo->rpt_ctrl.finfo.v105;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo.v105);
+			pcinfo->req_fver = 5;
+			break;
 		} else {
 			goto err;
 		}
@@ -1014,6 +1073,10 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo.v4;
 			pcysta->v4 = pfwinfo->rpt_fbtc_cysta.finfo.v4;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo.v4);
+		} else if (ver->fcxcysta == 5) {
+			pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo.v5;
+			pcysta->v5 = pfwinfo->rpt_fbtc_cysta.finfo.v5;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo.v5);
 		} else {
 			goto err;
 		}
@@ -1039,7 +1102,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 	case BTC_RPT_TYPE_NULLSTA:
 		pcinfo = &pfwinfo->rpt_fbtc_nullsta.cinfo;
 		if (ver->fcxnullsta == 1) {
-			pfinfo = &pfwinfo->rpt_fbtc_nullsta.finfo;
+			pfinfo = &pfwinfo->rpt_fbtc_nullsta.finfo.v1;
 			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_nullsta.finfo.v1);
 		} else if (ver->fcxnullsta == 2) {
 			pfinfo = &pfwinfo->rpt_fbtc_nullsta.finfo.v2;
@@ -1051,8 +1114,15 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		break;
 	case BTC_RPT_TYPE_MREG:
 		pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo;
-		pfinfo = &pfwinfo->rpt_fbtc_mregval.finfo;
-		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_mregval.finfo);
+		if (ver->fcxmreg == 1) {
+			pfinfo = &pfwinfo->rpt_fbtc_mregval.finfo.v1;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_mregval.finfo.v1);
+		} else if (ver->fcxmreg == 2) {
+			pfinfo = &pfwinfo->rpt_fbtc_mregval.finfo.v2;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_mregval.finfo.v2);
+		} else {
+			goto err;
+		}
 		pcinfo->req_fver = ver->fcxmreg;
 		break;
 	case BTC_RPT_TYPE_GPIO_DBG:
@@ -1069,8 +1139,13 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		break;
 	case BTC_RPT_TYPE_BT_SCAN:
 		pcinfo = &pfwinfo->rpt_fbtc_btscan.cinfo;
-		pfinfo = &pfwinfo->rpt_fbtc_btscan.finfo;
-		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btscan.finfo);
+		if (ver->fcxbtscan == 1) {
+			pfinfo = &pfwinfo->rpt_fbtc_btscan.finfo.v1;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btscan.finfo.v1);
+		} else if (ver->fcxbtscan == 2) {
+			pfinfo = &pfwinfo->rpt_fbtc_btscan.finfo.v2;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btscan.finfo.v2);
+		}
 		pcinfo->req_fver = ver->fcxbtscan;
 		break;
 	case BTC_RPT_TYPE_BT_AFH:
@@ -1129,14 +1204,14 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			wl->ver_info.fw = prpt->v1.wl_fw_ver;
 			dm->wl_fw_cx_offload = !!prpt->v1.wl_fw_cx_offload;
 
-			_chk_btc_err(rtwdev, BTC_DCNT_RPT_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_RPT_HANG,
 				     pfwinfo->event[BTF_EVNT_RPT]);
 
 			/* To avoid I/O if WL LPS or power-off */
 			if (wl->status.map.lps != BTC_LPS_RF_OFF &&
 			    !wl->status.map.rf_off) {
 				rtwdev->chip->ops->btc_update_bt_cnt(rtwdev);
-				_chk_btc_err(rtwdev, BTC_DCNT_BTCNT_FREEZE, 0);
+				_chk_btc_err(rtwdev, BTC_DCNT_BTCNT_HANG, 0);
 
 				btc->cx.cnt_bt[BTC_BCNT_POLUT] =
 					rtw89_mac_get_plt_cnt(rtwdev,
@@ -1164,8 +1239,8 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			btc->cx.cnt_bt[BTC_BCNT_POLUT] =
 				le32_to_cpu(prpt->v4.bt_cnt[BTC_BCNT_POLLUTED]);
 
-			_chk_btc_err(rtwdev, BTC_DCNT_BTCNT_FREEZE, 0);
-			_chk_btc_err(rtwdev, BTC_DCNT_RPT_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_BTCNT_HANG, 0);
+			_chk_btc_err(rtwdev, BTC_DCNT_RPT_HANG,
 				     pfwinfo->event[BTF_EVNT_RPT]);
 
 			if (le32_to_cpu(prpt->v4.bt_cnt[BTC_BCNT_RFK_TIMEOUT]) > 0)
@@ -1196,8 +1271,35 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			btc->cx.cnt_bt[BTC_BCNT_POLUT] =
 				le16_to_cpu(prpt->v5.bt_cnt[BTC_BCNT_POLLUTED]);
 
-			_chk_btc_err(rtwdev, BTC_DCNT_BTCNT_FREEZE, 0);
-			_chk_btc_err(rtwdev, BTC_DCNT_RPT_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_BTCNT_HANG, 0);
+			_chk_btc_err(rtwdev, BTC_DCNT_RPT_HANG,
+				     pfwinfo->event[BTF_EVNT_RPT]);
+
+			dm->error.map.bt_rfk_timeout = bt->rfk_info.map.timeout;
+		} else if (ver->fcxbtcrpt == 105) {
+			prpt->v105 = pfwinfo->rpt_ctrl.finfo.v105;
+			pfwinfo->rpt_en_map = le32_to_cpu(prpt->v105.rpt_info.en);
+			wl->ver_info.fw_coex = le32_to_cpu(prpt->v105.rpt_info.cx_ver);
+			wl->ver_info.fw = le32_to_cpu(prpt->v105.rpt_info.fw_ver);
+			dm->wl_fw_cx_offload = 0;
+
+			for (i = RTW89_PHY_0; i < RTW89_PHY_MAX; i++)
+				memcpy(&dm->gnt.band[i], &prpt->v105.gnt_val[i][0],
+				       sizeof(dm->gnt.band[i]));
+
+			btc->cx.cnt_bt[BTC_BCNT_HIPRI_TX] =
+				le16_to_cpu(prpt->v105.bt_cnt[BTC_BCNT_HI_TX_V105]);
+			btc->cx.cnt_bt[BTC_BCNT_HIPRI_RX] =
+				le16_to_cpu(prpt->v105.bt_cnt[BTC_BCNT_HI_RX_V105]);
+			btc->cx.cnt_bt[BTC_BCNT_LOPRI_TX] =
+				le16_to_cpu(prpt->v105.bt_cnt[BTC_BCNT_LO_TX_V105]);
+			btc->cx.cnt_bt[BTC_BCNT_LOPRI_RX] =
+				le16_to_cpu(prpt->v105.bt_cnt[BTC_BCNT_LO_RX_V105]);
+			btc->cx.cnt_bt[BTC_BCNT_POLUT] =
+				le16_to_cpu(prpt->v105.bt_cnt[BTC_BCNT_POLLUTED_V105]);
+
+			_chk_btc_err(rtwdev, BTC_DCNT_BTCNT_HANG, 0);
+			_chk_btc_err(rtwdev, BTC_DCNT_RPT_HANG,
 				     pfwinfo->event[BTF_EVNT_RPT]);
 
 			dm->error.map.bt_rfk_timeout = bt->rfk_info.map.timeout;
@@ -1258,11 +1360,11 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 					     BTC_DCNT_WL_SLOT_DRIFT, diff_t);
 			}
 
-			_chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_W1_HANG,
 				     le32_to_cpu(pcysta->v2.slot_cnt[CXST_W1]));
-			_chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_W1_HANG,
 				     le32_to_cpu(pcysta->v2.slot_cnt[CXST_B1]));
-			_chk_btc_err(rtwdev, BTC_DCNT_CYCLE_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_CYCLE_HANG,
 				     le16_to_cpu(pcysta->v2.cycles));
 		} else if (ver->fcxcysta == 3) {
 			if (le16_to_cpu(pcysta->v3.cycles) < BTC_CYSTA_CHK_PERIOD)
@@ -1299,11 +1401,11 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 				}
 			}
 
-			_chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_W1_HANG,
 				     le32_to_cpu(pcysta->v3.slot_cnt[CXST_W1]));
-			_chk_btc_err(rtwdev, BTC_DCNT_B1_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_B1_HANG,
 				     le32_to_cpu(pcysta->v3.slot_cnt[CXST_B1]));
-			_chk_btc_err(rtwdev, BTC_DCNT_CYCLE_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_CYCLE_HANG,
 				     le16_to_cpu(pcysta->v3.cycles));
 		} else if (ver->fcxcysta == 4) {
 			if (le16_to_cpu(pcysta->v4.cycles) < BTC_CYSTA_CHK_PERIOD)
@@ -1341,12 +1443,60 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 				}
 			}
 
-			_chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_W1_HANG,
 				     le16_to_cpu(pcysta->v4.slot_cnt[CXST_W1]));
-			_chk_btc_err(rtwdev, BTC_DCNT_B1_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_B1_HANG,
 				     le16_to_cpu(pcysta->v4.slot_cnt[CXST_B1]));
-			_chk_btc_err(rtwdev, BTC_DCNT_CYCLE_FREEZE,
+			_chk_btc_err(rtwdev, BTC_DCNT_CYCLE_HANG,
 				     le16_to_cpu(pcysta->v4.cycles));
+		} else if (ver->fcxcysta == 5) {
+			if (dm->fddt_train == BTC_FDDT_ENABLE)
+				break;
+			cnt_leak_slot = le16_to_cpu(pcysta->v5.slot_cnt[CXST_LK]);
+			cnt_rx_imr = le32_to_cpu(pcysta->v5.leak_slot.cnt_rximr);
+
+			/* Check Leak-AP */
+			if (cnt_leak_slot != 0 && cnt_rx_imr != 0 &&
+			    dm->tdma_now.rxflctrl) {
+				if (le16_to_cpu(pcysta->v5.cycles) >= BTC_CYSTA_CHK_PERIOD &&
+				    cnt_leak_slot < BTC_LEAK_AP_TH * cnt_rx_imr)
+					dm->leak_ap = 1;
+			}
+
+			/* Check diff time between real WL slot and W1 slot */
+			if (dm->tdma_now.type == CXTDMA_OFF) {
+				wl_slot_set = le16_to_cpu(dm->slot_now[CXST_W1].dur);
+				wl_slot_real = le16_to_cpu(pcysta->v5.cycle_time.tavg[CXT_WL]);
+
+				if (wl_slot_real > wl_slot_set)
+					diff_t = wl_slot_real - wl_slot_set;
+				else
+					diff_t = wl_slot_set - wl_slot_real;
+			}
+			_chk_btc_err(rtwdev, BTC_DCNT_WL_SLOT_DRIFT, diff_t);
+
+			/* Check diff time between real BT slot and EBT/E5G slot */
+			bt_slot_set = btc->bt_req_len;
+			bt_slot_real = le16_to_cpu(pcysta->v5.cycle_time.tavg[CXT_BT]);
+			diff_t = 0;
+			if (dm->tdma_now.type == CXTDMA_OFF &&
+			    dm->tdma_now.ext_ctrl == CXECTL_EXT &&
+			    bt_slot_set != 0) {
+				if (bt_slot_set > bt_slot_real)
+					diff_t = bt_slot_set - bt_slot_real;
+				else
+					diff_t = bt_slot_real - bt_slot_set;
+			}
+
+			_chk_btc_err(rtwdev, BTC_DCNT_BT_SLOT_DRIFT, diff_t);
+			_chk_btc_err(rtwdev, BTC_DCNT_E2G_HANG,
+				     le16_to_cpu(pcysta->v5.slot_cnt[CXST_E2G]));
+			_chk_btc_err(rtwdev, BTC_DCNT_W1_HANG,
+				     le16_to_cpu(pcysta->v5.slot_cnt[CXST_W1]));
+			_chk_btc_err(rtwdev, BTC_DCNT_B1_HANG,
+				     le16_to_cpu(pcysta->v5.slot_cnt[CXST_B1]));
+			_chk_btc_err(rtwdev, BTC_DCNT_CYCLE_HANG,
+				     le16_to_cpu(pcysta->v5.cycles));
 		} else {
 			goto err;
 		}
@@ -1630,10 +1780,14 @@ static void rtw89_btc_fw_en_rpt(struct rtw89_dev *rtwdev,
 				u32 rpt_map, bool rpt_state)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_smap *wl_smap = &btc->cx.wl.status.map;
 	struct rtw89_btc_btf_fwinfo *fwinfo = &btc->fwinfo;
 	struct rtw89_btc_btf_set_report r = {0};
 	u32 val, bit_map;
 
+	if ((wl_smap->rf_off || wl_smap->lps != BTC_LPS_OFF) && rpt_state != 0)
+		return;
+
 	bit_map = rtw89_btc_fw_rpt_ver(rtwdev, rpt_map);
 
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
@@ -1682,18 +1836,26 @@ static void rtw89_btc_fw_set_slots(struct rtw89_dev *rtwdev, u8 num,
 static void btc_fw_set_monreg(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
+	const struct rtw89_btc_ver *ver = rtwdev->btc.ver;
 	struct rtw89_btc_btf_set_mon_reg *monreg = NULL;
-	u8 n, *ptr = NULL, ulen;
+	u8 n, *ptr = NULL, ulen, cxmreg_max;
 	u16 sz = 0;
 
 	n = chip->mon_reg_num;
-
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
 		    "[BTC], %s(): mon_reg_num=%d\n", __func__, n);
-	if (n > CXMREG_MAX) {
+
+	if (ver->fcxmreg == 1)
+		cxmreg_max = CXMREG_MAX;
+	else if (ver->fcxmreg == 2)
+		cxmreg_max = CXMREG_MAX_V2;
+	else
+		return;
+
+	if (n > cxmreg_max) {
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
 			    "[BTC], %s(): mon reg count %d > %d\n",
-			    __func__, n, CXMREG_MAX);
+			    __func__, n, cxmreg_max);
 		return;
 	}
 
@@ -1703,7 +1865,7 @@ static void btc_fw_set_monreg(struct rtw89_dev *rtwdev)
 	if (!monreg)
 		return;
 
-	monreg->fver = BTF_SET_MON_REG_VER;
+	monreg->fver = ver->fcxmreg;
 	monreg->reg_num = n;
 	ptr = &monreg->buf[0];
 	memcpy(ptr, chip->mon_reg, n * ulen);
@@ -1782,6 +1944,9 @@ static void _fw_set_drv_info(struct rtw89_dev *rtwdev, u8 type)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	const struct rtw89_btc_ver *ver = btc->ver;
+	struct rtw89_btc_dm *dm = &btc->dm;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_rf_trx_para rf_para = dm->rf_trx_para;
 
 	switch (type) {
 	case CXDRVINFO_INIT:
@@ -1792,10 +1957,25 @@ static void _fw_set_drv_info(struct rtw89_dev *rtwdev, u8 type)
 			rtw89_fw_h2c_cxdrv_role(rtwdev);
 		else if (ver->fwlrole == 1)
 			rtw89_fw_h2c_cxdrv_role_v1(rtwdev);
+		else if (ver->fwlrole == 2)
+			rtw89_fw_h2c_cxdrv_role_v2(rtwdev);
 		break;
 	case CXDRVINFO_CTRL:
 		rtw89_fw_h2c_cxdrv_ctrl(rtwdev);
 		break;
+	case CXDRVINFO_TRX:
+		dm->trx_info.tx_power = u32_get_bits(rf_para.wl_tx_power,
+						     RTW89_BTC_WL_DEF_TX_PWR);
+		dm->trx_info.rx_gain = u32_get_bits(rf_para.wl_rx_gain,
+						    RTW89_BTC_WL_DEF_TX_PWR);
+		dm->trx_info.bt_tx_power = u32_get_bits(rf_para.bt_tx_power,
+							RTW89_BTC_WL_DEF_TX_PWR);
+		dm->trx_info.bt_rx_gain = u32_get_bits(rf_para.bt_rx_gain,
+						       RTW89_BTC_WL_DEF_TX_PWR);
+		dm->trx_info.cn = wl->cn_report;
+		dm->trx_info.nhm = wl->nhm.pwr;
+		rtw89_fw_h2c_cxdrv_trx(rtwdev);
+		break;
 	case CXDRVINFO_RFK:
 		rtw89_fw_h2c_cxdrv_rfk(rtwdev);
 		break;
@@ -2086,8 +2266,10 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_bt_link_info *b = &bt->link_info;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
 	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
+	struct rtw89_btc_wl_role_info_v2 *wl_rinfo_v2 = &wl->role_info_v2;
 	struct rtw89_btc_wl_active_role *r;
 	struct rtw89_btc_wl_active_role_v1 *r1;
+	struct rtw89_btc_wl_active_role_v2 *r2;
 	u8 en = 0, i, ch = 0, bw = 0;
 	u8 mode, connect_cnt;
 
@@ -2097,9 +2279,14 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
 	if (ver->fwlrole == 0) {
 		mode = wl_rinfo->link_mode;
 		connect_cnt = wl_rinfo->connect_cnt;
-	} else {
+	} else if (ver->fwlrole == 1) {
 		mode = wl_rinfo_v1->link_mode;
 		connect_cnt = wl_rinfo_v1->connect_cnt;
+	} else if (ver->fwlrole == 2) {
+		mode = wl_rinfo_v2->link_mode;
+		connect_cnt = wl_rinfo_v2->connect_cnt;
+	} else {
+		return;
 	}
 
 	if (wl->status.map.rf_off || bt->whql_test ||
@@ -2112,6 +2299,7 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
 		for (i = 0; i < RTW89_PORT_NUM; i++) {
 			r = &wl_rinfo->active_role[i];
 			r1 = &wl_rinfo_v1->active_role_v1[i];
+			r2 = &wl_rinfo_v2->active_role_v2[i];
 
 			if (ver->fwlrole == 0 &&
 			    (r->role == RTW89_WIFI_ROLE_P2P_GO ||
@@ -2125,6 +2313,12 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
 				ch = r1->ch;
 				bw = r1->bw;
 				break;
+			} else if (ver->fwlrole == 2 &&
+				   (r2->role == RTW89_WIFI_ROLE_P2P_GO ||
+				    r2->role == RTW89_WIFI_ROLE_P2P_CLIENT)) {
+				ch = r2->ch;
+				bw = r2->bw;
+				break;
 			}
 		}
 	} else {
@@ -2133,6 +2327,7 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
 		for (i = 0; i < RTW89_PORT_NUM; i++) {
 			r = &wl_rinfo->active_role[i];
 			r1 = &wl_rinfo_v1->active_role_v1[i];
+			r2 = &wl_rinfo_v2->active_role_v2[i];
 
 			if (ver->fwlrole == 0 &&
 			    r->connected && r->band == RTW89_BAND_2G) {
@@ -2144,6 +2339,11 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
 				ch = r1->ch;
 				bw = r1->bw;
 				break;
+			} else if (ver->fwlrole == 2 &&
+				   r2->connected && r2->band == RTW89_BAND_2G) {
+				ch = r2->ch;
+				bw = r2->bw;
+				break;
 			}
 		}
 	}
@@ -3598,6 +3798,7 @@ static void _set_btg_ctrl(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
 	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
+	struct rtw89_btc_wl_role_info_v2 *wl_rinfo_v2 = &wl->role_info_v2;
 	struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
 	bool is_btg;
 	u8 mode;
@@ -3607,8 +3808,12 @@ static void _set_btg_ctrl(struct rtw89_dev *rtwdev)
 
 	if (ver->fwlrole == 0)
 		mode = wl_rinfo->link_mode;
-	else
+	else if (ver->fwlrole == 1)
 		mode = wl_rinfo_v1->link_mode;
+	else if (ver->fwlrole == 2)
+		mode = wl_rinfo_v2->link_mode;
+	else
+		return;
 
 	/* notify halbb ignore GNT_BT or not for WL BB Rx-AGC control */
 	if (mode == BTC_WLINK_5G) /* always 0 if 5G */
@@ -3709,6 +3914,7 @@ static void _set_wl_tx_limit(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_bt_hid_desc *hid = &b->hid_desc;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
 	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
+	struct rtw89_btc_wl_role_info_v2 *wl_rinfo_v2 = &wl->role_info_v2;
 	struct rtw89_txtime_data data = {.rtwdev = rtwdev};
 	u8 mode;
 	u8 tx_retry;
@@ -3721,8 +3927,12 @@ static void _set_wl_tx_limit(struct rtw89_dev *rtwdev)
 
 	if (ver->fwlrole == 0)
 		mode = wl_rinfo->link_mode;
-	else
+	else if (ver->fwlrole == 1)
 		mode = wl_rinfo_v1->link_mode;
+	else if (ver->fwlrole == 2)
+		mode = wl_rinfo_v2->link_mode;
+	else
+		return;
 
 	if (btc->dm.freerun || btc->ctrl.igno_bt || b->profile_cnt.now == 0 ||
 	    mode == BTC_WLINK_5G || mode == BTC_WLINK_NOLINK) {
@@ -3772,14 +3982,19 @@ static void _set_bt_rx_agc(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
 	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
+	struct rtw89_btc_wl_role_info_v2 *wl_rinfo_v2 = &wl->role_info_v2;
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
 	bool bt_hi_lna_rx = false;
 	u8 mode;
 
 	if (ver->fwlrole == 0)
 		mode = wl_rinfo->link_mode;
-	else
+	else if (ver->fwlrole == 1)
 		mode = wl_rinfo_v1->link_mode;
+	else if (ver->fwlrole == 2)
+		mode = wl_rinfo_v2->link_mode;
+	else
+		return;
 
 	if (mode != BTC_WLINK_NOLINK && btc->dm.wl_btg_rx)
 		bt_hi_lna_rx = true;
@@ -4052,6 +4267,68 @@ static void _action_wl_2g_scc_v1(struct rtw89_dev *rtwdev)
 	_set_policy(rtwdev, policy_type, BTC_ACT_WL_2G_SCC);
 }
 
+static void _action_wl_2g_scc_v2(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
+	struct rtw89_btc_dm *dm = &btc->dm;
+	struct rtw89_btc_wl_role_info_v2 *wl_rinfo = &wl->role_info_v2;
+	u16 policy_type = BTC_CXP_OFF_BT;
+	u32 dur;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_DEDICATED) {
+		policy_type = BTC_CXP_OFF_EQ0;
+	} else {
+		/* shared-antenna */
+		switch (wl_rinfo->mrole_type) {
+		case BTC_WLMROLE_STA_GC:
+			dm->wl_scc.null_role1 = RTW89_WIFI_ROLE_STATION;
+			dm->wl_scc.null_role2 = RTW89_WIFI_ROLE_P2P_CLIENT;
+			dm->wl_scc.ebt_null = 0; /* no ext-slot-control */
+			_action_by_bt(rtwdev);
+			return;
+		case BTC_WLMROLE_STA_STA:
+			dm->wl_scc.null_role1 = RTW89_WIFI_ROLE_STATION;
+			dm->wl_scc.null_role2 = RTW89_WIFI_ROLE_STATION;
+			dm->wl_scc.ebt_null = 0; /* no ext-slot-control */
+			_action_by_bt(rtwdev);
+			return;
+		case BTC_WLMROLE_STA_GC_NOA:
+		case BTC_WLMROLE_STA_GO:
+		case BTC_WLMROLE_STA_GO_NOA:
+			dm->wl_scc.null_role1 = RTW89_WIFI_ROLE_STATION;
+			dm->wl_scc.null_role2 = RTW89_WIFI_ROLE_NONE;
+			dur = wl_rinfo->mrole_noa_duration;
+
+			if (wl->status.map._4way) {
+				dm->wl_scc.ebt_null = 0;
+				policy_type = BTC_CXP_OFFE_WL;
+			} else if (bt->link_info.status.map.connect == 0) {
+				dm->wl_scc.ebt_null = 0;
+				policy_type = BTC_CXP_OFFE_2GISOB;
+			} else if (bt->link_info.a2dp_desc.exist &&
+				   dur < btc->bt_req_len) {
+				dm->wl_scc.ebt_null = 1; /* tx null at EBT */
+				policy_type = BTC_CXP_OFFE_2GBWMIXB2;
+			} else if (bt->link_info.a2dp_desc.exist ||
+				   bt->link_info.pan_desc.exist) {
+				dm->wl_scc.ebt_null = 1; /* tx null at EBT */
+				policy_type = BTC_CXP_OFFE_2GBWISOB;
+			} else {
+				dm->wl_scc.ebt_null = 0;
+				policy_type = BTC_CXP_OFFE_2GBWISOB;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+	_set_ant(rtwdev, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(rtwdev, policy_type, BTC_ACT_WL_2G_SCC);
+}
+
 static void _action_wl_2g_ap(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -4491,6 +4768,156 @@ static void _update_wl_info_v1(struct rtw89_dev *rtwdev)
 	_fw_set_drv_info(rtwdev, CXDRVINFO_ROLE);
 }
 
+static void _update_wl_info_v2(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_wl_link_info *wl_linfo = wl->link_info;
+	struct rtw89_btc_wl_role_info_v2 *wl_rinfo = &wl->role_info_v2;
+	struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
+	u8 cnt_connect = 0, cnt_connecting = 0, cnt_active = 0;
+	u8 cnt_2g = 0, cnt_5g = 0, phy;
+	u32 wl_2g_ch[2] = {}, wl_5g_ch[2] = {};
+	bool b2g = false, b5g = false, client_joined = false;
+	u8 i;
+
+	memset(wl_rinfo, 0, sizeof(*wl_rinfo));
+
+	for (i = 0; i < RTW89_PORT_NUM; i++) {
+		if (!wl_linfo[i].active)
+			continue;
+
+		cnt_active++;
+		wl_rinfo->active_role_v2[cnt_active - 1].role = wl_linfo[i].role;
+		wl_rinfo->active_role_v2[cnt_active - 1].pid = wl_linfo[i].pid;
+		wl_rinfo->active_role_v2[cnt_active - 1].phy = wl_linfo[i].phy;
+		wl_rinfo->active_role_v2[cnt_active - 1].band = wl_linfo[i].band;
+		wl_rinfo->active_role_v2[cnt_active - 1].noa = (u8)wl_linfo[i].noa;
+		wl_rinfo->active_role_v2[cnt_active - 1].connected = 0;
+
+		wl->port_id[wl_linfo[i].role] = wl_linfo[i].pid;
+
+		phy = wl_linfo[i].phy;
+
+		if (rtwdev->dbcc_en && phy < RTW89_PHY_MAX) {
+			wl_dinfo->role[phy] = wl_linfo[i].role;
+			wl_dinfo->op_band[phy] = wl_linfo[i].band;
+			_update_dbcc_band(rtwdev, phy);
+			_fw_set_drv_info(rtwdev, CXDRVINFO_DBCC);
+		}
+
+		if (wl_linfo[i].connected == MLME_NO_LINK) {
+			continue;
+		} else if (wl_linfo[i].connected == MLME_LINKING) {
+			cnt_connecting++;
+		} else {
+			cnt_connect++;
+			if ((wl_linfo[i].role == RTW89_WIFI_ROLE_P2P_GO ||
+			     wl_linfo[i].role == RTW89_WIFI_ROLE_AP) &&
+			     wl_linfo[i].client_cnt > 1)
+				client_joined = true;
+		}
+
+		wl_rinfo->role_map.val |= BIT(wl_linfo[i].role);
+		wl_rinfo->active_role_v2[cnt_active - 1].ch = wl_linfo[i].ch;
+		wl_rinfo->active_role_v2[cnt_active - 1].bw = wl_linfo[i].bw;
+		wl_rinfo->active_role_v2[cnt_active - 1].connected = 1;
+
+		/* only care 2 roles + BT coex */
+		if (wl_linfo[i].band != RTW89_BAND_2G) {
+			if (cnt_5g <= ARRAY_SIZE(wl_5g_ch) - 1)
+				wl_5g_ch[cnt_5g] = wl_linfo[i].ch;
+			cnt_5g++;
+			b5g = true;
+		} else {
+			if (cnt_2g <= ARRAY_SIZE(wl_2g_ch) - 1)
+				wl_2g_ch[cnt_2g] = wl_linfo[i].ch;
+			cnt_2g++;
+			b2g = true;
+		}
+	}
+
+	wl_rinfo->connect_cnt = cnt_connect;
+
+	/* Be careful to change the following sequence!! */
+	if (cnt_connect == 0) {
+		wl_rinfo->link_mode = BTC_WLINK_NOLINK;
+		wl_rinfo->role_map.role.none = 1;
+	} else if (!b2g && b5g) {
+		wl_rinfo->link_mode = BTC_WLINK_5G;
+	} else if (wl_rinfo->role_map.role.nan) {
+		wl_rinfo->link_mode = BTC_WLINK_2G_NAN;
+	} else if (cnt_connect > BTC_TDMA_WLROLE_MAX) {
+		wl_rinfo->link_mode = BTC_WLINK_OTHER;
+	} else  if (b2g && b5g && cnt_connect == 2) {
+		if (rtwdev->dbcc_en) {
+			switch (wl_dinfo->role[RTW89_PHY_0]) {
+			case RTW89_WIFI_ROLE_STATION:
+				wl_rinfo->link_mode = BTC_WLINK_2G_STA;
+				break;
+			case RTW89_WIFI_ROLE_P2P_GO:
+				wl_rinfo->link_mode = BTC_WLINK_2G_GO;
+				break;
+			case RTW89_WIFI_ROLE_P2P_CLIENT:
+				wl_rinfo->link_mode = BTC_WLINK_2G_GC;
+				break;
+			case RTW89_WIFI_ROLE_AP:
+				wl_rinfo->link_mode = BTC_WLINK_2G_AP;
+				break;
+			default:
+				wl_rinfo->link_mode = BTC_WLINK_OTHER;
+				break;
+			}
+		} else {
+			wl_rinfo->link_mode = BTC_WLINK_25G_MCC;
+		}
+	} else if (!b5g && cnt_connect == 2) {
+		if (wl_rinfo->role_map.role.station &&
+		    (wl_rinfo->role_map.role.p2p_go ||
+		    wl_rinfo->role_map.role.p2p_gc ||
+		    wl_rinfo->role_map.role.ap)) {
+			if (wl_2g_ch[0] == wl_2g_ch[1])
+				wl_rinfo->link_mode = BTC_WLINK_2G_SCC;
+			else
+				wl_rinfo->link_mode = BTC_WLINK_2G_MCC;
+		} else {
+			wl_rinfo->link_mode = BTC_WLINK_2G_MCC;
+		}
+	} else if (!b5g && cnt_connect == 1) {
+		if (wl_rinfo->role_map.role.station)
+			wl_rinfo->link_mode = BTC_WLINK_2G_STA;
+		else if (wl_rinfo->role_map.role.ap)
+			wl_rinfo->link_mode = BTC_WLINK_2G_AP;
+		else if (wl_rinfo->role_map.role.p2p_go)
+			wl_rinfo->link_mode = BTC_WLINK_2G_GO;
+		else if (wl_rinfo->role_map.role.p2p_gc)
+			wl_rinfo->link_mode = BTC_WLINK_2G_GC;
+		else
+			wl_rinfo->link_mode = BTC_WLINK_OTHER;
+	}
+
+	/* if no client_joined, don't care P2P-GO/AP role */
+	if (wl_rinfo->role_map.role.p2p_go || wl_rinfo->role_map.role.ap) {
+		if (!client_joined) {
+			if (wl_rinfo->link_mode == BTC_WLINK_2G_SCC ||
+			    wl_rinfo->link_mode == BTC_WLINK_2G_MCC) {
+				wl_rinfo->link_mode = BTC_WLINK_2G_STA;
+				wl_rinfo->connect_cnt = 1;
+			} else if (wl_rinfo->link_mode == BTC_WLINK_2G_GO ||
+				 wl_rinfo->link_mode == BTC_WLINK_2G_AP) {
+				wl_rinfo->link_mode = BTC_WLINK_NOLINK;
+				wl_rinfo->connect_cnt = 0;
+			}
+		}
+	}
+
+	rtw89_debug(rtwdev, RTW89_DBG_BTC,
+		    "[BTC], cnt_connect = %d, connecting = %d, link_mode = %d\n",
+		    cnt_connect, cnt_connecting, wl_rinfo->link_mode);
+
+	_fw_set_drv_info(rtwdev, CXDRVINFO_ROLE);
+}
+
 #define BTC_CHK_HANG_MAX 3
 #define BTC_SCB_INV_VALUE GENMASK(31, 0)
 
@@ -4578,7 +5005,7 @@ static void _update_bt_scbd(struct rtw89_dev *rtwdev, bool only_update)
 	}
 
 	if (!(val & BTC_BSCB_ON) ||
-	    btc->dm.cnt_dm[BTC_DCNT_BTCNT_FREEZE] >= BTC_CHK_HANG_MAX)
+	    btc->dm.cnt_dm[BTC_DCNT_BTCNT_HANG] >= BTC_CHK_HANG_MAX)
 		bt->enable.now = 0;
 	else
 		bt->enable.now = 1;
@@ -4649,6 +5076,7 @@ void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason)
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
 	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
+	struct rtw89_btc_wl_role_info_v2 *wl_rinfo_v2 = &wl->role_info_v2;
 	u8 mode;
 
 	lockdep_assert_held(&rtwdev->mutex);
@@ -4659,8 +5087,12 @@ void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason)
 
 	if (ver->fwlrole == 0)
 		mode = wl_rinfo->link_mode;
-	else
+	else if (ver->fwlrole == 1)
 		mode = wl_rinfo_v1->link_mode;
+	else if (ver->fwlrole == 2)
+		mode = wl_rinfo_v2->link_mode;
+	else
+		return;
 
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): reason=%d, mode=%d\n",
 		    __func__, reason, mode);
@@ -4702,6 +5134,7 @@ void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason)
 	}
 
 	dm->cnt_dm[BTC_DCNT_RUN]++;
+	dm->fddt_train = BTC_FDDT_DISABLE;
 
 	if (btc->ctrl.always_freerun) {
 		_action_freerun(rtwdev);
@@ -4785,6 +5218,8 @@ void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason)
 			_action_wl_2g_scc(rtwdev);
 		else if (ver->fwlrole == 1)
 			_action_wl_2g_scc_v1(rtwdev);
+		else if (ver->fwlrole == 2)
+			_action_wl_2g_scc_v2(rtwdev);
 		break;
 	case BTC_WLINK_2G_MCC:
 		bt->scan_rx_low_pri = true;
@@ -5078,6 +5513,8 @@ void rtw89_btc_ntfy_icmp_packet_work(struct work_struct *work)
 	mutex_unlock(&rtwdev->mutex);
 }
 
+#define BT_PROFILE_PROTOCOL_MASK GENMASK(7, 4)
+
 static void _update_bt_info(struct rtw89_dev *rtwdev, u8 *buf, u32 len)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
@@ -5134,6 +5571,7 @@ static void _update_bt_info(struct rtw89_dev *rtwdev, u8 *buf, u32 len)
 	a2dp->exist = btinfo.lb2.a2dp;
 	b->profile_cnt.now += (u8)a2dp->exist;
 	pan->active = btinfo.lb2.pan;
+	btc->dm.trx_info.bt_profile = u32_get_bits(btinfo.val, BT_PROFILE_PROTOCOL_MASK);
 
 	/* parse raw info low-Byte3 */
 	btinfo.val = bt->raw_info[BTC_BTINFO_L3];
@@ -5150,6 +5588,7 @@ static void _update_bt_info(struct rtw89_dev *rtwdev, u8 *buf, u32 len)
 	btinfo.val = bt->raw_info[BTC_BTINFO_H0];
 	/* raw val is dBm unit, translate from -100~ 0dBm to 0~100%*/
 	b->rssi = chip->ops->btc_get_bt_rssi(rtwdev, btinfo.hb0.rssi);
+	btc->dm.trx_info.bt_rssi = b->rssi;
 
 	/* parse raw info high-Byte1 */
 	btinfo.val = bt->raw_info[BTC_BTINFO_H1];
@@ -5290,8 +5729,10 @@ void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif
 	memcpy(wlinfo, &r, sizeof(*wlinfo));
 	if (ver->fwlrole == 0)
 		_update_wl_info(rtwdev);
-	else
+	else if (ver->fwlrole == 1)
 		_update_wl_info_v1(rtwdev);
+	else if (ver->fwlrole == 2)
+		_update_wl_info_v2(rtwdev);
 
 	if (wlinfo->role == RTW89_WIFI_ROLE_STATION &&
 	    wlinfo->connected == MLME_NO_LINK)
@@ -5330,6 +5771,11 @@ void rtw89_btc_ntfy_radio_state(struct rtw89_dev *rtwdev, enum btc_rfctrl rf_sta
 		wl->status.map.lps = BTC_LPS_RF_OFF;
 		wl->status.map.busy = 0;
 		break;
+	case BTC_RFCTRL_LPS_WL_ON: /* LPS-Protocol (RFon) */
+		wl->status.map.rf_off = 0;
+		wl->status.map.lps = BTC_LPS_RF_ON;
+		wl->status.map.busy = 0;
+		break;
 	case BTC_RFCTRL_WL_ON:
 	default:
 		wl->status.map.rf_off = 0;
@@ -5347,9 +5793,12 @@ void rtw89_btc_ntfy_radio_state(struct rtw89_dev *rtwdev, enum btc_rfctrl rf_sta
 		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_ALL, false);
 		if (rf_state == BTC_RFCTRL_WL_OFF)
 			_write_scbd(rtwdev, BTC_WSCB_ALL, false);
+		else if (rf_state == BTC_RFCTRL_LPS_WL_ON &&
+			 wl->status.map.lps_pre != BTC_LPS_OFF)
+			_update_bt_scbd(rtwdev, true);
 	}
 
-	btc->dm.cnt_dm[BTC_DCNT_BTCNT_FREEZE] = 0;
+	btc->dm.cnt_dm[BTC_DCNT_BTCNT_HANG] = 0;
 	if (wl->status.map.lps_pre == BTC_LPS_OFF &&
 	    wl->status.map.lps_pre != wl->status.map.lps)
 		btc->dm.tdma_instant_excute = 1;
@@ -5357,7 +5806,7 @@ void rtw89_btc_ntfy_radio_state(struct rtw89_dev *rtwdev, enum btc_rfctrl rf_sta
 		btc->dm.tdma_instant_excute = 0;
 
 	_run_coex(rtwdev, BTC_RSN_NTFY_RADIO_STATE);
-
+	btc->dm.tdma_instant_excute = 0;
 	wl->status.map.rf_off_pre = wl->status.map.rf_off;
 	wl->status.map.lps_pre = wl->status.map.lps;
 }
@@ -5481,6 +5930,8 @@ static void rtw89_btc_ntfy_wl_sta_iter(void *data, struct ieee80211_sta *sta)
 				(struct rtw89_btc_wl_sta_iter_data *)data;
 	struct rtw89_dev *rtwdev = iter_data->rtwdev;
 	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_dm *dm = &btc->dm;
+	const struct rtw89_btc_ver *ver = btc->ver;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_wl_link_info *link_info = NULL;
 	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
@@ -5488,6 +5939,8 @@ static void rtw89_btc_ntfy_wl_sta_iter(void *data, struct ieee80211_sta *sta)
 	struct rtw89_vif *rtwvif = rtwsta->rtwvif;
 	struct rtw89_traffic_stats *stats = &rtwvif->stats;
 	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_btc_wl_role_info *r;
+	struct rtw89_btc_wl_role_info_v1 *r1;
 	u32 last_tx_rate, last_rx_rate;
 	u16 last_tx_lvl, last_rx_lvl;
 	u8 port = rtwvif->port;
@@ -5564,10 +6017,33 @@ static void rtw89_btc_ntfy_wl_sta_iter(void *data, struct ieee80211_sta *sta)
 	link_info_t->tx_rate = rtwsta->ra_report.hw_rate;
 	link_info_t->rx_rate = rtwsta->rx_hw_rate;
 
-	wl->role_info.active_role[port].tx_lvl = (u16)stats->tx_tfc_lv;
-	wl->role_info.active_role[port].rx_lvl = (u16)stats->rx_tfc_lv;
-	wl->role_info.active_role[port].tx_rate = rtwsta->ra_report.hw_rate;
-	wl->role_info.active_role[port].rx_rate = rtwsta->rx_hw_rate;
+	if (link_info->role == RTW89_WIFI_ROLE_STATION ||
+	    link_info->role == RTW89_WIFI_ROLE_P2P_CLIENT) {
+		dm->trx_info.tx_rate = link_info_t->tx_rate;
+		dm->trx_info.rx_rate = link_info_t->rx_rate;
+	}
+
+	if (ver->fwlrole == 0) {
+		r = &wl->role_info;
+		r->active_role[port].tx_lvl = stats->tx_tfc_lv;
+		r->active_role[port].rx_lvl = stats->rx_tfc_lv;
+		r->active_role[port].tx_rate = rtwsta->ra_report.hw_rate;
+		r->active_role[port].rx_rate = rtwsta->rx_hw_rate;
+	} else if (ver->fwlrole == 1) {
+		r1 = &wl->role_info_v1;
+		r1->active_role_v1[port].tx_lvl = stats->tx_tfc_lv;
+		r1->active_role_v1[port].rx_lvl = stats->rx_tfc_lv;
+		r1->active_role_v1[port].tx_rate = rtwsta->ra_report.hw_rate;
+		r1->active_role_v1[port].rx_rate = rtwsta->rx_hw_rate;
+	} else if (ver->fwlrole == 2) {
+		dm->trx_info.tx_lvl = stats->tx_tfc_lv;
+		dm->trx_info.rx_lvl = stats->rx_tfc_lv;
+		dm->trx_info.tx_rate = rtwsta->ra_report.hw_rate;
+		dm->trx_info.rx_rate = rtwsta->rx_hw_rate;
+	}
+
+	dm->trx_info.tx_tp = link_info_t->tx_throughput;
+	dm->trx_info.rx_tp = link_info_t->rx_throughput;
 
 	if (is_sta_change)
 		iter_data->is_sta_change = true;
@@ -5581,6 +6057,7 @@ static void rtw89_btc_ntfy_wl_sta_iter(void *data, struct ieee80211_sta *sta)
 void rtw89_btc_ntfy_wl_sta(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_dm *dm = &btc->dm;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_wl_sta_iter_data data = {.rtwdev = rtwdev};
 	u8 i;
@@ -5599,6 +6076,9 @@ void rtw89_btc_ntfy_wl_sta(struct rtw89_dev *rtwdev)
 		}
 	}
 
+	if (dm->trx_info.wl_rssi != wl->rssi_level)
+		dm->trx_info.wl_rssi = wl->rssi_level;
+
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): busy=%d\n",
 		    __func__, !!wl->status.map.busy);
 
@@ -5700,11 +6180,6 @@ static void _show_cx_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 	seq_printf(m, " %-15s : Coex:%d.%d.%d(branch:%d), ",
 		   "[coex_version]", ver_main, ver_sub, ver_hotfix, id_branch);
 
-	if (dm->wl_fw_cx_offload != BTC_CX_FW_OFFLOAD)
-		dm->error.map.offload_mismatch = true;
-	else
-		dm->error.map.offload_mismatch = false;
-
 	ver_main = FIELD_GET(GENMASK(31, 24), wl->ver_info.fw_coex);
 	ver_sub = FIELD_GET(GENMASK(23, 16), wl->ver_info.fw_coex);
 	ver_hotfix = FIELD_GET(GENMASK(15, 8), wl->ver_info.fw_coex);
@@ -5816,6 +6291,7 @@ static void _show_wl_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_wl_info *wl = &cx->wl;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
 	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
+	struct rtw89_btc_wl_role_info_v2 *wl_rinfo_v2 = &wl->role_info_v2;
 	u8 mode;
 
 	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_WL))
@@ -5825,8 +6301,12 @@ static void _show_wl_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 
 	if (ver->fwlrole == 0)
 		mode = wl_rinfo->link_mode;
-	else
+	else if (ver->fwlrole == 1)
 		mode = wl_rinfo_v1->link_mode;
+	else if (ver->fwlrole == 2)
+		mode = wl_rinfo_v2->link_mode;
+	else
+		return;
 
 	seq_printf(m, " %-15s : link_mode:%d, ", "[status]", mode);
 
@@ -5996,11 +6476,40 @@ static void _show_bt_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 		   cx->cnt_bt[BTC_BCNT_INFOSAME]);
 
 	seq_printf(m,
-		   " %-15s : Hi-rx = %d, Hi-tx = %d, Lo-rx = %d, Lo-tx = %d (bt_polut_wl_tx = %d)\n",
+		   " %-15s : Hi-rx = %d, Hi-tx = %d, Lo-rx = %d, Lo-tx = %d (bt_polut_wl_tx = %d)",
 		   "[trx_req_cnt]", cx->cnt_bt[BTC_BCNT_HIPRI_RX],
 		   cx->cnt_bt[BTC_BCNT_HIPRI_TX], cx->cnt_bt[BTC_BCNT_LOPRI_RX],
 		   cx->cnt_bt[BTC_BCNT_LOPRI_TX], cx->cnt_bt[BTC_BCNT_POLUT]);
 
+	if (!bt->scan_info_update) {
+		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_SCAN_INFO, true);
+		seq_puts(m, "\n");
+	} else {
+		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_SCAN_INFO, false);
+		if (ver->fcxbtscan == 1) {
+			seq_printf(m,
+				   "(INQ:%d-%d/PAGE:%d-%d/LE:%d-%d/INIT:%d-%d)",
+				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INQ].win),
+				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INQ].intvl),
+				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_PAGE].win),
+				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_PAGE].intvl),
+				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_BLE].win),
+				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_BLE].intvl),
+				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INIT].win),
+				   le16_to_cpu(bt->scan_info_v1[BTC_SCAN_INIT].intvl));
+		} else if (ver->fcxbtscan == 2) {
+			seq_printf(m,
+				   "(BG:%d-%d/INIT:%d-%d/LE:%d-%d)",
+				   le16_to_cpu(bt->scan_info_v2[CXSCAN_BG].win),
+				   le16_to_cpu(bt->scan_info_v2[CXSCAN_BG].intvl),
+				   le16_to_cpu(bt->scan_info_v2[CXSCAN_INIT].win),
+				   le16_to_cpu(bt->scan_info_v2[CXSCAN_INIT].intvl),
+				   le16_to_cpu(bt->scan_info_v2[CXSCAN_LE].win),
+				   le16_to_cpu(bt->scan_info_v2[CXSCAN_LE].intvl));
+		}
+		seq_puts(m, "\n");
+	}
+
 	if (bt->enable.now && bt->ver_info.fw == 0)
 		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_BT_VER_INFO, true);
 	else
@@ -6322,6 +6831,10 @@ static void _show_error(struct rtw89_dev *rtwdev, struct seq_file *m)
 		pcysta->v4 = pfwinfo->rpt_fbtc_cysta.finfo.v4;
 		except_cnt = pcysta->v4.except_cnt;
 		exception_map = le32_to_cpu(pcysta->v4.except_map);
+	} else if (ver->fcxcysta == 5) {
+		pcysta->v5 = pfwinfo->rpt_fbtc_cysta.finfo.v5;
+		except_cnt = pcysta->v5.except_cnt;
+		exception_map = le32_to_cpu(pcysta->v5.except_map);
 	} else {
 		return;
 	}
@@ -6810,6 +7323,137 @@ static void _show_fbtc_cysta_v4(struct rtw89_dev *rtwdev, struct seq_file *m)
 	}
 }
 
+static void _show_fbtc_cysta_v5(struct rtw89_dev *rtwdev, struct seq_file *m)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_bt_a2dp_desc *a2dp = &btc->cx.bt.link_info.a2dp_desc;
+	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct rtw89_btc_dm *dm = &btc->dm;
+	struct rtw89_btc_fbtc_a2dp_trx_stat_v4 *a2dp_trx;
+	struct rtw89_btc_fbtc_cysta_v5 *pcysta;
+	struct rtw89_btc_rpt_cmn_info *pcinfo;
+	u8 i, cnt = 0, slot_pair, divide_cnt;
+	u16 cycle, c_begin, c_end, store_index;
+
+	pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo;
+	if (!pcinfo->valid)
+		return;
+
+	pcysta = &pfwinfo->rpt_fbtc_cysta.finfo.v5;
+	seq_printf(m,
+		   " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]",
+		   "[cycle_cnt]",
+		   le16_to_cpu(pcysta->cycles),
+		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]),
+		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]),
+		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]),
+		   le16_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK]));
+
+	for (i = 0; i < CXST_MAX; i++) {
+		if (!le16_to_cpu(pcysta->slot_cnt[i]))
+			continue;
+
+		seq_printf(m, ", %s:%d", id_to_slot(i),
+			   le16_to_cpu(pcysta->slot_cnt[i]));
+	}
+
+	if (dm->tdma_now.rxflctrl)
+		seq_printf(m, ", leak_rx:%d",
+			   le32_to_cpu(pcysta->leak_slot.cnt_rximr));
+
+	if (pcysta->collision_cnt)
+		seq_printf(m, ", collision:%d", pcysta->collision_cnt);
+
+	if (le16_to_cpu(pcysta->skip_cnt))
+		seq_printf(m, ", skip:%d",
+			   le16_to_cpu(pcysta->skip_cnt));
+
+	seq_puts(m, "\n");
+
+	seq_printf(m, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
+		   "[cycle_time]",
+		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]),
+		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]),
+		   le16_to_cpu(pcysta->leak_slot.tavg) / 1000,
+		   le16_to_cpu(pcysta->leak_slot.tavg) % 1000);
+	seq_printf(m,
+		   ", max_t[wl:%d/bt:%d/lk:%d.%03d]\n",
+		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]),
+		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]),
+		   le16_to_cpu(pcysta->leak_slot.tmax) / 1000,
+		   le16_to_cpu(pcysta->leak_slot.tmax) % 1000);
+
+	cycle = le16_to_cpu(pcysta->cycles);
+	if (cycle <= 1)
+		return;
+
+	/* 1 cycle record 1 wl-slot and 1 bt-slot */
+	slot_pair = BTC_CYCLE_SLOT_MAX / 2;
+
+	if (cycle <= slot_pair)
+		c_begin = 1;
+	else
+		c_begin = cycle - slot_pair + 1;
+
+	c_end = cycle;
+
+	if (a2dp->exist)
+		divide_cnt = 3;
+	else
+		divide_cnt = BTC_CYCLE_SLOT_MAX / 4;
+
+	if (c_begin > c_end)
+		return;
+
+	for (cycle = c_begin; cycle <= c_end; cycle++) {
+		cnt++;
+		store_index = ((cycle - 1) % slot_pair) * 2;
+
+		if (cnt % divide_cnt == 1)
+			seq_printf(m, " %-15s : ", "[cycle_step]");
+
+		seq_printf(m, "->b%02d",
+			   le16_to_cpu(pcysta->slot_step_time[store_index]));
+		if (a2dp->exist) {
+			a2dp_trx = &pcysta->a2dp_trx[store_index];
+			seq_printf(m, "(%d/%d/%dM/%d/%d/%d)",
+				   a2dp_trx->empty_cnt,
+				   a2dp_trx->retry_cnt,
+				   a2dp_trx->tx_rate ? 3 : 2,
+				   a2dp_trx->tx_cnt,
+				   a2dp_trx->ack_cnt,
+				   a2dp_trx->nack_cnt);
+		}
+		seq_printf(m, "->w%02d",
+			   le16_to_cpu(pcysta->slot_step_time[store_index + 1]));
+		if (a2dp->exist) {
+			a2dp_trx = &pcysta->a2dp_trx[store_index + 1];
+			seq_printf(m, "(%d/%d/%dM/%d/%d/%d)",
+				   a2dp_trx->empty_cnt,
+				   a2dp_trx->retry_cnt,
+				   a2dp_trx->tx_rate ? 3 : 2,
+				   a2dp_trx->tx_cnt,
+				   a2dp_trx->ack_cnt,
+				   a2dp_trx->nack_cnt);
+		}
+		if (cnt % divide_cnt == 0 || cnt == c_end)
+			seq_puts(m, "\n");
+	}
+
+	if (a2dp->exist) {
+		seq_printf(m, " %-15s : a2dp_ept:%d, a2dp_late:%d",
+			   "[a2dp_t_sta]",
+			   le16_to_cpu(pcysta->a2dp_ept.cnt),
+			   le16_to_cpu(pcysta->a2dp_ept.cnt_timeout));
+
+		seq_printf(m, ", avg_t:%d, max_t:%d",
+			   le16_to_cpu(pcysta->a2dp_ept.tavg),
+			   le16_to_cpu(pcysta->a2dp_ept.tmax));
+
+		seq_puts(m, "\n");
+	}
+}
+
 static void _show_fbtc_nullsta(struct rtw89_dev *rtwdev, struct seq_file *m)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -7014,6 +7658,8 @@ static void _show_fw_dm_msg(struct rtw89_dev *rtwdev, struct seq_file *m)
 		_show_fbtc_cysta_v3(rtwdev, m);
 	else if (ver->fcxcysta == 4)
 		_show_fbtc_cysta_v4(rtwdev, m);
+	else if (ver->fcxcysta == 5)
+		_show_fbtc_cysta_v5(rtwdev, m);
 
 	_show_fbtc_nullsta(rtwdev, m);
 
@@ -7065,13 +7711,117 @@ static void _get_gnt(struct rtw89_dev *rtwdev, struct rtw89_mac_ax_coex_gnt *gnt
 	}
 }
 
-static void _show_mreg(struct rtw89_dev *rtwdev, struct seq_file *m)
+static void _show_mreg_v1(struct rtw89_dev *rtwdev, struct seq_file *m)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
+	struct rtw89_btc_fbtc_mreg_val_v1 *pmreg = NULL;
+	struct rtw89_btc_fbtc_gpio_dbg *gdbg = NULL;
+	struct rtw89_btc_cx *cx = &btc->cx;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
+	struct rtw89_mac_ax_coex_gnt gnt_cfg = {};
+	struct rtw89_mac_ax_gnt gnt;
+	u8 i = 0, type = 0, cnt = 0;
+	u32 val, offset;
+
+	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_MREG))
+		return;
+
+	seq_puts(m, "========== [HW Status] ==========\n");
+
+	seq_printf(m,
+		   " %-15s : WL->BT:0x%08x(cnt:%d), BT->WL:0x%08x(total:%d, bt_update:%d)\n",
+		   "[scoreboard]", wl->scbd, cx->cnt_wl[BTC_WCNT_SCBDUPDATE],
+		   bt->scbd, cx->cnt_bt[BTC_BCNT_SCBDREAD],
+		   cx->cnt_bt[BTC_BCNT_SCBDUPDATE]);
+
+	/* To avoid I/O if WL LPS or power-off  */
+	if (!wl->status.map.lps && !wl->status.map.rf_off) {
+		btc->dm.pta_owner = rtw89_mac_get_ctrl_path(rtwdev);
+
+		_get_gnt(rtwdev, &gnt_cfg);
+		gnt = gnt_cfg.band[0];
+		seq_printf(m,
+			   " %-15s : pta_owner:%s, phy-0[gnt_wl:%s-%d/gnt_bt:%s-%d], ",
+			   "[gnt_status]",
+			   chip->chip_id == RTL8852C ? "HW" :
+			   btc->dm.pta_owner == BTC_CTRL_BY_WL ? "WL" : "BT",
+			   gnt.gnt_wl_sw_en ? "SW" : "HW", gnt.gnt_wl,
+			   gnt.gnt_bt_sw_en ? "SW" : "HW", gnt.gnt_bt);
+
+		gnt = gnt_cfg.band[1];
+		seq_printf(m, "phy-1[gnt_wl:%s-%d/gnt_bt:%s-%d]\n",
+			   gnt.gnt_wl_sw_en ? "SW" : "HW",
+			   gnt.gnt_wl,
+			   gnt.gnt_bt_sw_en ? "SW" : "HW",
+			   gnt.gnt_bt);
+	}
+	pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo;
+	if (!pcinfo->valid) {
+		rtw89_debug(rtwdev, RTW89_DBG_BTC,
+			    "[BTC], %s(): stop due rpt_fbtc_mregval.cinfo\n",
+			    __func__);
+		return;
+	}
+
+	pmreg = &pfwinfo->rpt_fbtc_mregval.finfo.v1;
+	rtw89_debug(rtwdev, RTW89_DBG_BTC,
+		    "[BTC], %s(): rpt_fbtc_mregval reg_num = %d\n",
+		    __func__, pmreg->reg_num);
+
+	for (i = 0; i < pmreg->reg_num; i++) {
+		type = (u8)le16_to_cpu(chip->mon_reg[i].type);
+		offset = le32_to_cpu(chip->mon_reg[i].offset);
+		val = le32_to_cpu(pmreg->mreg_val[i]);
+
+		if (cnt % 6 == 0)
+			seq_printf(m, " %-15s : %d_0x%04x=0x%08x",
+				   "[reg]", (u32)type, offset, val);
+		else
+			seq_printf(m, ", %d_0x%04x=0x%08x", (u32)type,
+				   offset, val);
+		if (cnt % 6 == 5)
+			seq_puts(m, "\n");
+		cnt++;
+
+		if (i >= pmreg->reg_num)
+			seq_puts(m, "\n");
+	}
+
+	pcinfo = &pfwinfo->rpt_fbtc_gpio_dbg.cinfo;
+	if (!pcinfo->valid) {
+		rtw89_debug(rtwdev, RTW89_DBG_BTC,
+			    "[BTC], %s(): stop due rpt_fbtc_gpio_dbg.cinfo\n",
+			    __func__);
+		seq_puts(m, "\n");
+		return;
+	}
+
+	gdbg = &pfwinfo->rpt_fbtc_gpio_dbg.finfo;
+	if (!gdbg->en_map)
+		return;
+
+	seq_printf(m, " %-15s : enable_map:0x%08x",
+		   "[gpio_dbg]", gdbg->en_map);
+
+	for (i = 0; i < BTC_DBG_MAX1; i++) {
+		if (!(gdbg->en_map & BIT(i)))
+			continue;
+		seq_printf(m, ", %d->GPIO%d", (u32)i, gdbg->gpio_map[i]);
+	}
+	seq_puts(m, "\n");
+}
+
+static void _show_mreg_v2(struct rtw89_dev *rtwdev, struct seq_file *m)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
 	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
-	struct rtw89_btc_fbtc_mreg_val *pmreg = NULL;
+	struct rtw89_btc_fbtc_mreg_val_v2 *pmreg = NULL;
 	struct rtw89_btc_fbtc_gpio_dbg *gdbg = NULL;
 	struct rtw89_btc_cx *cx = &btc->cx;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
@@ -7121,7 +7871,7 @@ static void _show_mreg(struct rtw89_dev *rtwdev, struct seq_file *m)
 		return;
 	}
 
-	pmreg = &pfwinfo->rpt_fbtc_mregval.finfo;
+	pmreg = &pfwinfo->rpt_fbtc_mregval.finfo.v2;
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
 		    "[BTC], %s(): rpt_fbtc_mregval reg_num = %d\n",
 		    __func__, pmreg->reg_num);
@@ -7150,6 +7900,7 @@ static void _show_mreg(struct rtw89_dev *rtwdev, struct seq_file *m)
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
 			    "[BTC], %s(): stop due rpt_fbtc_gpio_dbg.cinfo\n",
 			    __func__);
+		seq_puts(m, "\n");
 		return;
 	}
 
@@ -7453,8 +8204,122 @@ static void _show_summary_v5(struct rtw89_dev *rtwdev, struct seq_file *m)
 			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_on),
 			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_off));
 	} else {
+		seq_printf(m,
+			   " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d",
+			   "[summary]", pfwinfo->cnt_h2c,
+			   pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h);
+	}
+
+	if (!pcinfo->valid || pfwinfo->len_mismch || pfwinfo->fver_mismch ||
+	    pfwinfo->err[BTFRE_EXCEPTION]) {
 		seq_puts(m, "\n");
 		seq_printf(m,
+			   " %-15s : WL FW rpt error!![rpt_ctrl_valid:%d/len:"
+			   "0x%x/ver:0x%x/ex:%d/lps=%d/rf_off=%d]",
+			   "[ERROR]", pcinfo->valid, pfwinfo->len_mismch,
+			   pfwinfo->fver_mismch, pfwinfo->err[BTFRE_EXCEPTION],
+			   wl->status.map.lps, wl->status.map.rf_off);
+	}
+
+	for (i = 0; i < BTC_NCNT_NUM; i++)
+		cnt_sum += dm->cnt_notify[i];
+
+	seq_puts(m, "\n");
+	seq_printf(m,
+		   " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
+		   "[notify_cnt]",
+		   cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
+		   cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+
+	seq_printf(m,
+		   "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d",
+		   cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
+		   cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
+		   cnt[BTC_NCNT_WL_STA]);
+
+	seq_puts(m, "\n");
+	seq_printf(m,
+		   " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ",
+		   "[notify_cnt]",
+		   cnt[BTC_NCNT_SCAN_START], cnt[BTC_NCNT_SCAN_FINISH],
+		   cnt[BTC_NCNT_SWITCH_BAND], cnt[BTC_NCNT_SPECIAL_PACKET]);
+
+	seq_printf(m,
+		   "timer=%d, control=%d, customerize=%d",
+		   cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL],
+		   cnt[BTC_NCNT_CUSTOMERIZE]);
+}
+
+static void _show_summary_v105(struct rtw89_dev *rtwdev, struct seq_file *m)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct rtw89_btc_fbtc_rpt_ctrl_v105 *prptctrl;
+	struct rtw89_btc_rpt_cmn_info *pcinfo;
+	struct rtw89_btc_cx *cx = &btc->cx;
+	struct rtw89_btc_dm *dm = &btc->dm;
+	struct rtw89_btc_wl_info *wl = &cx->wl;
+	u32 cnt_sum = 0, *cnt = btc->dm.cnt_notify;
+	u8 i;
+
+	if (!(dm->coex_info_map & BTC_COEX_INFO_SUMMARY))
+		return;
+
+	seq_puts(m, "========== [Statistics] ==========\n");
+
+	pcinfo = &pfwinfo->rpt_ctrl.cinfo;
+	if (pcinfo->valid && !wl->status.map.lps && !wl->status.map.rf_off) {
+		prptctrl = &pfwinfo->rpt_ctrl.finfo.v105;
+
+		seq_printf(m,
+			   " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d, len:%d), ",
+			   "[summary]", pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
+			   le16_to_cpu(prptctrl->rpt_info.cnt_h2c),
+			   pfwinfo->cnt_c2h,
+			   le16_to_cpu(prptctrl->rpt_info.cnt_c2h),
+			   le16_to_cpu(prptctrl->rpt_info.len_c2h));
+
+		seq_printf(m,
+			   "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x",
+			   pfwinfo->event[BTF_EVNT_RPT],
+			   le16_to_cpu(prptctrl->rpt_info.cnt),
+			   le32_to_cpu(prptctrl->rpt_info.en));
+
+		if (dm->error.map.wl_fw_hang)
+			seq_puts(m, " (WL FW Hang!!)");
+		seq_puts(m, "\n");
+		seq_printf(m,
+			   " %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
+			   "[mailbox]",
+			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
+			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
+			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
+
+		seq_printf(m,
+			   "A2DP_empty:%d(stop:%d, tx:%d, ack:%d, nack:%d)\n",
+			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
+			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
+			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
+			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
+			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
+
+		seq_printf(m,
+			   " %-15s : wl_rfk[req:%d/go:%d/reject:%d/tout:%d]",
+			   "[RFK/LPS]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
+			   cx->cnt_wl[BTC_WCNT_RFK_GO],
+			   cx->cnt_wl[BTC_WCNT_RFK_REJECT],
+			   cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]);
+
+		seq_printf(m,
+			   ", bt_rfk[req:%d]",
+			   le16_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]));
+
+		seq_printf(m,
+			   ", AOAC[RF_on:%d/RF_off:%d]",
+			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_on),
+			   le16_to_cpu(prptctrl->rpt_info.cnt_aoac_rf_off));
+	} else {
+		seq_printf(m,
 			   " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d",
 			   "[summary]", pfwinfo->cnt_h2c,
 			   pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h);
@@ -7532,13 +8397,20 @@ void rtw89_btc_dump_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 	_show_bt_info(rtwdev, m);
 	_show_dm_info(rtwdev, m);
 	_show_fw_dm_msg(rtwdev, m);
-	_show_mreg(rtwdev, m);
+
+	if (ver->fcxmreg == 1)
+		_show_mreg_v1(rtwdev, m);
+	else if (ver->fcxmreg == 2)
+		_show_mreg_v2(rtwdev, m);
+
 	if (ver->fcxbtcrpt == 1)
 		_show_summary_v1(rtwdev, m);
 	else if (ver->fcxbtcrpt == 4)
 		_show_summary_v4(rtwdev, m);
 	else if (ver->fcxbtcrpt == 5)
 		_show_summary_v5(rtwdev, m);
+	else if (ver->fcxbtcrpt == 105)
+		_show_summary_v105(rtwdev, m);
 }
 
 void rtw89_coex_recognize_ver(struct rtw89_dev *rtwdev)
diff --git a/drivers/net/wireless/realtek/rtw89/coex.h b/drivers/net/wireless/realtek/rtw89/coex.h
index 401fb55df82b..f16421cb30ef 100644
--- a/drivers/net/wireless/realtek/rtw89/coex.h
+++ b/drivers/net/wireless/realtek/rtw89/coex.h
@@ -66,6 +66,11 @@ enum btc_rssi_st {
 	BTC_RSSI_ST_MAX
 };
 
+enum btc_fddt_en {
+	BTC_FDDT_DISABLE,
+	BTC_FDDT_ENABLE,
+};
+
 #define	BTC_RSSI_HIGH(_rssi_) \
 	({typeof(_rssi_) __rssi = (_rssi_); \
 	  ((__rssi == BTC_RSSI_ST_HIGH || \
@@ -126,6 +131,7 @@ enum btc_role_state {
 enum btc_rfctrl {
 	BTC_RFCTRL_WL_OFF,
 	BTC_RFCTRL_WL_ON,
+	BTC_RFCTRL_LPS_WL_ON,
 	BTC_RFCTRL_FW_CTRL,
 	BTC_RFCTRL_MAX
 };
diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c
index f09361bc4a4d..7fc0a26a4d73 100644
--- a/drivers/net/wireless/realtek/rtw89/core.c
+++ b/drivers/net/wireless/realtek/rtw89/core.c
@@ -156,6 +156,28 @@ static struct ieee80211_rate rtw89_bitrates[] = {
 	{ .bitrate = 540, .hw_value = 0x0b, },
 };
 
+static const struct ieee80211_iface_limit rtw89_iface_limits[] = {
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_STATION),
+	},
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_P2P_CLIENT) |
+			 BIT(NL80211_IFTYPE_P2P_GO) |
+			 BIT(NL80211_IFTYPE_AP),
+	},
+};
+
+static const struct ieee80211_iface_combination rtw89_iface_combs[] = {
+	{
+		.limits = rtw89_iface_limits,
+		.n_limits = ARRAY_SIZE(rtw89_iface_limits),
+		.max_interfaces = 2,
+		.num_different_channels = 1,
+	}
+};
+
 bool rtw89_ra_report_to_bitrate(struct rtw89_dev *rtwdev, u8 rpt_rate, u16 *bitrate)
 {
 	struct ieee80211_rate rate;
@@ -360,6 +382,15 @@ void rtw89_set_channel(struct rtw89_dev *rtwdev)
 	rtw89_set_entity_state(rtwdev, true);
 }
 
+void rtw89_get_channel(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
+		       struct rtw89_chan *chan)
+{
+	const struct cfg80211_chan_def *chandef;
+
+	chandef = rtw89_chandef_get(rtwdev, rtwvif->sub_entity_idx);
+	rtw89_get_channel_params(chandef, chan);
+}
+
 static enum rtw89_core_tx_type
 rtw89_core_get_tx_type(struct rtw89_dev *rtwdev,
 		       struct sk_buff *skb)
@@ -686,6 +717,33 @@ desc_bk:
 	desc_info->bk = true;
 }
 
+static u16 rtw89_core_get_data_rate(struct rtw89_dev *rtwdev,
+				    struct rtw89_core_tx_request *tx_req)
+{
+	struct ieee80211_vif *vif = tx_req->vif;
+	struct ieee80211_sta *sta = tx_req->sta;
+	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
+	struct rtw89_phy_rate_pattern *rate_pattern = &rtwvif->rate_pattern;
+	enum rtw89_sub_entity_idx idx = rtwvif->sub_entity_idx;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, idx);
+	u16 lowest_rate;
+
+	if (rate_pattern->enable)
+		return rate_pattern->rate;
+
+	if (vif->p2p)
+		lowest_rate = RTW89_HW_RATE_OFDM6;
+	else if (chan->band_type == RTW89_BAND_2G)
+		lowest_rate = RTW89_HW_RATE_CCK1;
+	else
+		lowest_rate = RTW89_HW_RATE_OFDM6;
+
+	if (!sta || !sta->deflink.supp_rates[chan->band_type])
+		return lowest_rate;
+
+	return __ffs(sta->deflink.supp_rates[chan->band_type]) + lowest_rate;
+}
+
 static void
 rtw89_core_tx_update_data_info(struct rtw89_dev *rtwdev,
 			       struct rtw89_core_tx_request *tx_req)
@@ -694,8 +752,6 @@ rtw89_core_tx_update_data_info(struct rtw89_dev *rtwdev,
 	struct ieee80211_sta *sta = tx_req->sta;
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	struct rtw89_sta *rtwsta = sta_to_rtwsta_safe(sta);
-	struct rtw89_phy_rate_pattern *rate_pattern = &rtwvif->rate_pattern;
-	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
 	struct sk_buff *skb = tx_req->skb;
 	u8 tid, tid_indicate;
@@ -719,14 +775,7 @@ rtw89_core_tx_update_data_info(struct rtw89_dev *rtwdev,
 	if (IEEE80211_SKB_CB(skb)->control.hw_key)
 		rtw89_core_tx_update_sec_key(rtwdev, tx_req);
 
-	if (vif->p2p)
-		desc_info->data_retry_lowest_rate = RTW89_HW_RATE_OFDM6;
-	else if (rate_pattern->enable)
-		desc_info->data_retry_lowest_rate = rate_pattern->rate;
-	else if (chan->band_type == RTW89_BAND_2G)
-		desc_info->data_retry_lowest_rate = RTW89_HW_RATE_CCK1;
-	else
-		desc_info->data_retry_lowest_rate = RTW89_HW_RATE_OFDM6;
+	desc_info->data_retry_lowest_rate = rtw89_core_get_data_rate(rtwdev, tx_req);
 }
 
 static enum btc_pkt_type
@@ -849,6 +898,37 @@ void rtw89_core_tx_kick_off(struct rtw89_dev *rtwdev, u8 qsel)
 	rtw89_hci_tx_kick_off(rtwdev, ch_dma);
 }
 
+int rtw89_core_tx_kick_off_and_wait(struct rtw89_dev *rtwdev, struct sk_buff *skb,
+				    int qsel, unsigned int timeout)
+{
+	struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
+	struct rtw89_tx_wait_info *wait;
+	unsigned long time_left;
+	int ret = 0;
+
+	wait = kzalloc(sizeof(*wait), GFP_KERNEL);
+	if (!wait) {
+		rtw89_core_tx_kick_off(rtwdev, qsel);
+		return 0;
+	}
+
+	init_completion(&wait->completion);
+	rcu_assign_pointer(skb_data->wait, wait);
+
+	rtw89_core_tx_kick_off(rtwdev, qsel);
+	time_left = wait_for_completion_timeout(&wait->completion,
+						msecs_to_jiffies(timeout));
+	if (time_left == 0)
+		ret = -ETIMEDOUT;
+	else if (!wait->tx_done)
+		ret = -EAGAIN;
+
+	rcu_assign_pointer(skb_data->wait, NULL);
+	kfree_rcu(wait, rcu_head);
+
+	return ret;
+}
+
 int rtw89_h2c_tx(struct rtw89_dev *rtwdev,
 		 struct sk_buff *skb, bool fwdl)
 {
@@ -1202,6 +1282,10 @@ static void rtw89_core_parse_phy_status_ie01(struct rtw89_dev *rtwdev, u8 *addr,
 	phy_ppdu->chan_idx = RTW89_GET_PHY_STS_IE01_CH_IDX(addr);
 	if (phy_ppdu->rate < RTW89_HW_RATE_OFDM6)
 		return;
+
+	if (!phy_ppdu->to_self)
+		return;
+
 	/* sign conversion for S(12,2) */
 	if (rtwdev->chip->cfo_src_fd)
 		cfo = sign_extend32(RTW89_GET_PHY_STS_IE01_FD_CFO(addr), 11);
@@ -1266,9 +1350,6 @@ static int rtw89_core_rx_parse_phy_sts(struct rtw89_dev *rtwdev,
 	if (phy_ppdu->ie < RTW89_CCK_PKT)
 		return -EINVAL;
 
-	if (!phy_ppdu->to_self)
-		return 0;
-
 	pos = (u8 *)phy_ppdu->buf + PHY_STS_HDR_LEN;
 	end = (u8 *)phy_ppdu->buf + phy_ppdu->len;
 	while (pos < end) {
@@ -1400,6 +1481,34 @@ static void rtw89_stats_trigger_frame(struct rtw89_dev *rtwdev,
 	}
 }
 
+static void rtw89_core_cancel_6ghz_probe_tx(struct rtw89_dev *rtwdev,
+					    struct sk_buff *skb)
+{
+	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
+	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)skb->data;
+	struct list_head *pkt_list = rtwdev->scan_info.pkt_list;
+	struct rtw89_pktofld_info *info;
+	const u8 *ies = mgmt->u.beacon.variable, *ssid_ie;
+
+	if (rx_status->band != NL80211_BAND_6GHZ)
+		return;
+
+	ssid_ie = cfg80211_find_ie(WLAN_EID_SSID, ies, skb->len);
+
+	list_for_each_entry(info, &pkt_list[NL80211_BAND_6GHZ], list) {
+		if (ether_addr_equal(info->bssid, mgmt->bssid)) {
+			rtw89_fw_h2c_del_pkt_offload(rtwdev, info->id);
+			continue;
+		}
+
+		if (!ssid_ie || ssid_ie[1] != info->ssid_len || info->ssid_len == 0)
+			continue;
+
+		if (memcmp(&ssid_ie[2], info->ssid, info->ssid_len) == 0)
+			rtw89_fw_h2c_del_pkt_offload(rtwdev, info->id);
+	}
+}
+
 static void rtw89_vif_rx_stats_iter(void *data, u8 *mac,
 				    struct ieee80211_vif *vif)
 {
@@ -1410,8 +1519,14 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac,
 	struct rtw89_rx_desc_info *desc_info = iter_data->desc_info;
 	struct sk_buff *skb = iter_data->skb;
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+	struct rtw89_rx_phy_ppdu *phy_ppdu = iter_data->phy_ppdu;
 	const u8 *bssid = iter_data->bssid;
 
+	if (rtwdev->scanning &&
+	    (ieee80211_is_beacon(hdr->frame_control) ||
+	     ieee80211_is_probe_resp(hdr->frame_control)))
+		rtw89_core_cancel_6ghz_probe_tx(rtwdev, skb);
+
 	if (!vif->bss_conf.bssid)
 		return;
 
@@ -1423,8 +1538,11 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac,
 	if (!ether_addr_equal(vif->bss_conf.bssid, bssid))
 		return;
 
-	if (ieee80211_is_beacon(hdr->frame_control))
+	if (ieee80211_is_beacon(hdr->frame_control)) {
+		if (vif->type == NL80211_IFTYPE_STATION)
+			rtw89_fw_h2c_rssi_offload(rtwdev, phy_ppdu);
 		pkt_stat->beacon_nr++;
+	}
 
 	if (!ether_addr_equal(vif->addr, hdr->addr1))
 		return;
@@ -1927,6 +2045,18 @@ static void rtw89_core_free_sta_pending_forbid_ba(struct rtw89_dev *rtwdev,
 	spin_unlock_bh(&rtwdev->ba_lock);
 }
 
+static void rtw89_core_free_sta_pending_roc_tx(struct rtw89_dev *rtwdev,
+					       struct ieee80211_sta *sta)
+{
+	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
+	struct sk_buff *skb, *tmp;
+
+	skb_queue_walk_safe(&rtwsta->roc_queue, skb, tmp) {
+		skb_unlink(skb, &rtwsta->roc_queue);
+		dev_kfree_skb_any(skb);
+	}
+}
+
 static void rtw89_core_stop_tx_ba_session(struct rtw89_dev *rtwdev,
 					  struct rtw89_txq *rtwtxq)
 {
@@ -2066,6 +2196,7 @@ static void rtw89_core_txq_schedule(struct rtw89_dev *rtwdev, u8 ac, bool *reinv
 {
 	struct ieee80211_hw *hw = rtwdev->hw;
 	struct ieee80211_txq *txq;
+	struct rtw89_vif *rtwvif;
 	struct rtw89_txq *rtwtxq;
 	unsigned long frame_cnt;
 	unsigned long byte_cnt;
@@ -2075,6 +2206,12 @@ static void rtw89_core_txq_schedule(struct rtw89_dev *rtwdev, u8 ac, bool *reinv
 	ieee80211_txq_schedule_start(hw, ac);
 	while ((txq = ieee80211_next_txq(hw, ac))) {
 		rtwtxq = (struct rtw89_txq *)txq->drv_priv;
+		rtwvif = (struct rtw89_vif *)txq->vif->drv_priv;
+
+		if (rtwvif->offchan) {
+			ieee80211_return_txq(hw, txq, true);
+			continue;
+		}
 		tx_resource = rtw89_check_and_reclaim_tx_resource(rtwdev, txq->tid);
 		sched_txq = false;
 
@@ -2101,8 +2238,7 @@ static void rtw89_ips_work(struct work_struct *work)
 	struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev,
 						ips_work);
 	mutex_lock(&rtwdev->mutex);
-	if (rtwdev->hw->conf.flags & IEEE80211_CONF_IDLE)
-		rtw89_enter_ips(rtwdev);
+	rtw89_enter_ips_by_hwflags(rtwdev);
 	mutex_unlock(&rtwdev->mutex);
 }
 
@@ -2143,6 +2279,187 @@ static void rtw89_forbid_ba_work(struct work_struct *w)
 	spin_unlock_bh(&rtwdev->ba_lock);
 }
 
+static void rtw89_core_sta_pending_tx_iter(void *data,
+					   struct ieee80211_sta *sta)
+{
+	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
+	struct rtw89_vif *rtwvif_target = data, *rtwvif = rtwsta->rtwvif;
+	struct rtw89_dev *rtwdev = rtwvif->rtwdev;
+	struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif);
+	struct sk_buff *skb, *tmp;
+	int qsel, ret;
+
+	if (rtwvif->sub_entity_idx != rtwvif_target->sub_entity_idx)
+		return;
+
+	if (skb_queue_len(&rtwsta->roc_queue) == 0)
+		return;
+
+	skb_queue_walk_safe(&rtwsta->roc_queue, skb, tmp) {
+		skb_unlink(skb, &rtwsta->roc_queue);
+
+		ret = rtw89_core_tx_write(rtwdev, vif, sta, skb, &qsel);
+		if (ret) {
+			rtw89_warn(rtwdev, "pending tx failed with %d\n", ret);
+			dev_kfree_skb_any(skb);
+		} else {
+			rtw89_core_tx_kick_off(rtwdev, qsel);
+		}
+	}
+}
+
+static void rtw89_core_handle_sta_pending_tx(struct rtw89_dev *rtwdev,
+					     struct rtw89_vif *rtwvif)
+{
+	ieee80211_iterate_stations_atomic(rtwdev->hw,
+					  rtw89_core_sta_pending_tx_iter,
+					  rtwvif);
+}
+
+static int rtw89_core_send_nullfunc(struct rtw89_dev *rtwdev,
+				    struct rtw89_vif *rtwvif, bool qos, bool ps)
+{
+	struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif);
+	struct ieee80211_sta *sta;
+	struct ieee80211_hdr *hdr;
+	struct sk_buff *skb;
+	int ret, qsel;
+
+	if (vif->type != NL80211_IFTYPE_STATION || !vif->cfg.assoc)
+		return 0;
+
+	rcu_read_lock();
+	sta = ieee80211_find_sta(vif, vif->bss_conf.bssid);
+	if (!sta) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	skb = ieee80211_nullfunc_get(rtwdev->hw, vif, -1, qos);
+	if (!skb) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	hdr = (struct ieee80211_hdr *)skb->data;
+	if (ps)
+		hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
+
+	ret = rtw89_core_tx_write(rtwdev, vif, sta, skb, &qsel);
+	if (ret) {
+		rtw89_warn(rtwdev, "nullfunc transmit failed: %d\n", ret);
+		dev_kfree_skb_any(skb);
+		goto out;
+	}
+
+	rcu_read_unlock();
+
+	return rtw89_core_tx_kick_off_and_wait(rtwdev, skb, qsel,
+					       RTW89_ROC_TX_TIMEOUT);
+out:
+	rcu_read_unlock();
+
+	return ret;
+}
+
+void rtw89_roc_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
+{
+	struct ieee80211_hw *hw = rtwdev->hw;
+	struct rtw89_roc *roc = &rtwvif->roc;
+	struct cfg80211_chan_def roc_chan;
+	struct rtw89_vif *tmp;
+	int ret;
+
+	lockdep_assert_held(&rtwdev->mutex);
+
+	ieee80211_queue_delayed_work(hw, &rtwvif->roc.roc_work,
+				     msecs_to_jiffies(rtwvif->roc.duration));
+
+	rtw89_leave_ips_by_hwflags(rtwdev);
+	rtw89_leave_lps(rtwdev);
+
+	ret = rtw89_core_send_nullfunc(rtwdev, rtwvif, true, true);
+	if (ret)
+		rtw89_debug(rtwdev, RTW89_DBG_TXRX,
+			    "roc send null-1 failed: %d\n", ret);
+
+	rtw89_for_each_rtwvif(rtwdev, tmp)
+		if (tmp->sub_entity_idx == rtwvif->sub_entity_idx)
+			tmp->offchan = true;
+
+	cfg80211_chandef_create(&roc_chan, &roc->chan, NL80211_CHAN_NO_HT);
+	rtw89_config_roc_chandef(rtwdev, rtwvif->sub_entity_idx, &roc_chan);
+	rtw89_set_channel(rtwdev);
+	rtw89_write32_clr(rtwdev,
+			  rtw89_mac_reg_by_idx(R_AX_RX_FLTR_OPT, RTW89_MAC_0),
+			  B_AX_A_UC_CAM_MATCH | B_AX_A_BC_CAM_MATCH);
+
+	ieee80211_ready_on_channel(hw);
+}
+
+void rtw89_roc_end(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
+{
+	struct ieee80211_hw *hw = rtwdev->hw;
+	struct rtw89_roc *roc = &rtwvif->roc;
+	struct rtw89_vif *tmp;
+	int ret;
+
+	lockdep_assert_held(&rtwdev->mutex);
+
+	ieee80211_remain_on_channel_expired(hw);
+
+	rtw89_leave_ips_by_hwflags(rtwdev);
+	rtw89_leave_lps(rtwdev);
+
+	rtw89_write32_mask(rtwdev,
+			   rtw89_mac_reg_by_idx(R_AX_RX_FLTR_OPT, RTW89_MAC_0),
+			   B_AX_RX_FLTR_CFG_MASK,
+			   rtwdev->hal.rx_fltr);
+
+	roc->state = RTW89_ROC_IDLE;
+	rtw89_config_roc_chandef(rtwdev, rtwvif->sub_entity_idx, NULL);
+	rtw89_set_channel(rtwdev);
+	ret = rtw89_core_send_nullfunc(rtwdev, rtwvif, true, false);
+	if (ret)
+		rtw89_debug(rtwdev, RTW89_DBG_TXRX,
+			    "roc send null-0 failed: %d\n", ret);
+
+	rtw89_for_each_rtwvif(rtwdev, tmp)
+		if (tmp->sub_entity_idx == rtwvif->sub_entity_idx)
+			tmp->offchan = false;
+
+	rtw89_core_handle_sta_pending_tx(rtwdev, rtwvif);
+	queue_work(rtwdev->txq_wq, &rtwdev->txq_work);
+
+	if (hw->conf.flags & IEEE80211_CONF_IDLE)
+		ieee80211_queue_delayed_work(hw, &roc->roc_work,
+					     RTW89_ROC_IDLE_TIMEOUT);
+}
+
+void rtw89_roc_work(struct work_struct *work)
+{
+	struct rtw89_vif *rtwvif = container_of(work, struct rtw89_vif,
+						roc.roc_work.work);
+	struct rtw89_dev *rtwdev = rtwvif->rtwdev;
+	struct rtw89_roc *roc = &rtwvif->roc;
+
+	mutex_lock(&rtwdev->mutex);
+
+	switch (roc->state) {
+	case RTW89_ROC_IDLE:
+		rtw89_enter_ips_by_hwflags(rtwdev);
+		break;
+	case RTW89_ROC_MGMT:
+	case RTW89_ROC_NORMAL:
+		rtw89_roc_end(rtwdev, rtwvif);
+		break;
+	default:
+		break;
+	}
+
+	mutex_unlock(&rtwdev->mutex);
+}
+
 static enum rtw89_tfc_lv rtw89_get_traffic_level(struct rtw89_dev *rtwdev,
 						 u32 throughput, u64 cnt)
 {
@@ -2199,8 +2516,10 @@ static bool rtw89_traffic_stats_track(struct rtw89_dev *rtwdev)
 	bool tfc_changed;
 
 	tfc_changed = rtw89_traffic_stats_calc(rtwdev, &rtwdev->stats);
-	rtw89_for_each_rtwvif(rtwdev, rtwvif)
+	rtw89_for_each_rtwvif(rtwdev, rtwvif) {
 		rtw89_traffic_stats_calc(rtwdev, &rtwvif->stats);
+		rtw89_fw_h2c_tp_offload(rtwdev, rtwvif);
+	}
 
 	return tfc_changed;
 }
@@ -2212,9 +2531,15 @@ static void rtw89_vif_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwv
 	    rtwvif->tdls_peer)
 		return;
 
+	if (rtwdev->total_sta_assoc > 1)
+		return;
+
+	if (rtwvif->offchan)
+		return;
+
 	if (rtwvif->stats.tx_tfc_lv == RTW89_TFC_IDLE &&
 	    rtwvif->stats.rx_tfc_lv == RTW89_TFC_IDLE)
-		rtw89_enter_lps(rtwdev, rtwvif);
+		rtw89_enter_lps(rtwdev, rtwvif, true);
 }
 
 static void rtw89_enter_lps_track(struct rtw89_dev *rtwdev)
@@ -2441,6 +2766,7 @@ int rtw89_core_sta_add(struct rtw89_dev *rtwdev,
 	rtwsta->rtwvif = rtwvif;
 	rtwsta->prev_rssi = 0;
 	INIT_LIST_HEAD(&rtwsta->ba_cam_list);
+	skb_queue_head_init(&rtwsta->roc_queue);
 
 	for (i = 0; i < ARRAY_SIZE(sta->txq); i++)
 		rtw89_core_txq_init(rtwdev, sta->txq[i]);
@@ -2487,6 +2813,9 @@ int rtw89_core_sta_disassoc(struct rtw89_dev *rtwdev,
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
 
+	if (vif->type == NL80211_IFTYPE_STATION)
+		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, vif, false);
+
 	rtwdev->total_sta_assoc--;
 	if (sta->tdls)
 		rtwvif->tdls_peer--;
@@ -2507,6 +2836,8 @@ int rtw89_core_sta_disconnect(struct rtw89_dev *rtwdev,
 	rtw89_mac_bf_disassoc(rtwdev, vif, sta);
 	rtw89_core_free_sta_pending_ba(rtwdev, sta);
 	rtw89_core_free_sta_pending_forbid_ba(rtwdev, sta);
+	rtw89_core_free_sta_pending_roc_tx(rtwdev, sta);
+
 	if (vif->type == NL80211_IFTYPE_AP || sta->tdls)
 		rtw89_cam_deinit_addr_cam(rtwdev, &rtwsta->addr_cam);
 	if (sta->tdls)
@@ -3128,7 +3459,6 @@ void rtw89_core_stop(struct rtw89_dev *rtwdev)
 int rtw89_core_init(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
-	int ret;
 	u8 band;
 
 	INIT_LIST_HEAD(&rtwdev->ba_list);
@@ -3162,6 +3492,8 @@ int rtw89_core_init(struct rtw89_dev *rtwdev)
 
 	INIT_WORK(&rtwdev->c2h_work, rtw89_fw_c2h_work);
 	INIT_WORK(&rtwdev->ips_work, rtw89_ips_work);
+	INIT_WORK(&rtwdev->load_firmware_work, rtw89_load_firmware_work);
+
 	skb_queue_head_init(&rtwdev->c2h_queue);
 	rtw89_core_ppdu_sts_init(rtwdev);
 	rtw89_traffic_stats_init(rtwdev, &rtwdev->stats);
@@ -3173,12 +3505,10 @@ int rtw89_core_init(struct rtw89_dev *rtwdev)
 	INIT_WORK(&btc->dhcp_notify_work, rtw89_btc_ntfy_dhcp_packet_work);
 	INIT_WORK(&btc->icmp_notify_work, rtw89_btc_ntfy_icmp_packet_work);
 
-	ret = rtw89_load_firmware(rtwdev);
-	if (ret) {
-		rtw89_warn(rtwdev, "no firmware loaded\n");
-		destroy_workqueue(rtwdev->txq_wq);
-		return ret;
-	}
+	init_completion(&rtwdev->fw.req.completion);
+
+	schedule_work(&rtwdev->load_firmware_work);
+
 	rtw89_ser_init(rtwdev);
 	rtw89_entity_init(rtwdev);
 
@@ -3205,13 +3535,14 @@ void rtw89_core_scan_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 
 	rtwdev->scanning = true;
 	rtw89_leave_lps(rtwdev);
-	if (hw_scan && (rtwdev->hw->conf.flags & IEEE80211_CONF_IDLE))
-		rtw89_leave_ips(rtwdev);
+	if (hw_scan)
+		rtw89_leave_ips_by_hwflags(rtwdev);
 
 	ether_addr_copy(rtwvif->mac_addr, mac_addr);
 	rtw89_btc_ntfy_scan_start(rtwdev, RTW89_PHY_0, chan->band_type);
 	rtw89_chip_rfk_scan(rtwdev, true);
 	rtw89_hci_recalc_int_mit(rtwdev);
+	rtw89_phy_config_edcca(rtwdev, true);
 
 	rtw89_fw_h2c_cam(rtwdev, rtwvif, NULL, mac_addr);
 }
@@ -3229,6 +3560,7 @@ void rtw89_core_scan_complete(struct rtw89_dev *rtwdev,
 
 	rtw89_chip_rfk_scan(rtwdev, false);
 	rtw89_btc_ntfy_scan_finish(rtwdev, RTW89_PHY_0);
+	rtw89_phy_config_edcca(rtwdev, false);
 
 	rtwdev->scanning = false;
 	rtwdev->dig.bypass_dig = true;
@@ -3239,6 +3571,8 @@ void rtw89_core_scan_complete(struct rtw89_dev *rtwdev,
 static void rtw89_read_chip_ver(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
+	int ret;
+	u8 val;
 	u8 cv;
 
 	cv = rtw89_read32_mask(rtwdev, R_AX_SYS_CFG1, B_AX_CHIP_VER_MASK);
@@ -3250,6 +3584,14 @@ static void rtw89_read_chip_ver(struct rtw89_dev *rtwdev)
 	}
 
 	rtwdev->hal.cv = cv;
+
+	if (chip->chip_id == RTL8852B || chip->chip_id == RTL8851B) {
+		ret = rtw89_mac_read_xtal_si(rtwdev, XTAL_SI_CV, &val);
+		if (!ret)
+			return;
+
+		rtwdev->hal.acv = u8_get_bits(val, XTAL_SI_ACV_MASK);
+	}
 }
 
 static void rtw89_core_setup_phycap(struct rtw89_dev *rtwdev)
@@ -3261,6 +3603,28 @@ static void rtw89_core_setup_phycap(struct rtw89_dev *rtwdev)
 		rtwdev->chip->chip_id == RTL8852A && rtwdev->hal.cv <= CHIP_CBV;
 }
 
+static void rtw89_core_setup_rfe_parms(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	const struct rtw89_rfe_parms_conf *conf = chip->rfe_parms_conf;
+	struct rtw89_efuse *efuse = &rtwdev->efuse;
+	u8 rfe_type = efuse->rfe_type;
+
+	if (!conf)
+		goto out;
+
+	while (conf->rfe_parms) {
+		if (rfe_type == conf->rfe_type) {
+			rtwdev->rfe_parms = conf->rfe_parms;
+			return;
+		}
+		conf++;
+	}
+
+out:
+	rtwdev->rfe_parms = chip->dflt_parms;
+}
+
 static int rtw89_chip_efuse_info_setup(struct rtw89_dev *rtwdev)
 {
 	int ret;
@@ -3282,6 +3646,7 @@ static int rtw89_chip_efuse_info_setup(struct rtw89_dev *rtwdev)
 		return ret;
 
 	rtw89_core_setup_phycap(rtwdev);
+	rtw89_core_setup_rfe_parms(rtwdev);
 
 	rtw89_mac_pwr_off(rtwdev);
 
@@ -3361,6 +3726,8 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	ieee80211_hw_set(hw, SINGLE_SCAN_ON_ALL_BANDS);
 	ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID);
 	ieee80211_hw_set(hw, WANT_MONITOR_VIF);
+	if (RTW89_CHK_FW_FEATURE(BEACON_FILTER, &rtwdev->fw))
+		ieee80211_hw_set(hw, CONNECTION_MONITOR);
 
 	hw->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) |
 				     BIT(NL80211_IFTYPE_AP) |
@@ -3372,7 +3739,7 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 
 	hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS |
 			    WIPHY_FLAG_TDLS_EXTERNAL_SETUP |
-			    WIPHY_FLAG_AP_UAPSD;
+			    WIPHY_FLAG_AP_UAPSD | WIPHY_FLAG_SPLIT_SCAN_6GHZ;
 	hw->wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR;
 
 	hw->wiphy->max_scan_ssids = RTW89_SCANOFLD_MAX_SSID;
@@ -3386,6 +3753,7 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	hw->wiphy->tid_config_support.peer |= BIT(NL80211_TID_CONFIG_ATTR_AMPDU_CTRL);
 	hw->wiphy->tid_config_support.vif |= BIT(NL80211_TID_CONFIG_ATTR_AMSDU_CTRL);
 	hw->wiphy->tid_config_support.peer |= BIT(NL80211_TID_CONFIG_ATTR_AMSDU_CTRL);
+	hw->wiphy->max_remain_on_channel_duration = 1000;
 
 	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
 
@@ -3401,18 +3769,22 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	ret = ieee80211_register_hw(hw);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to register hw\n");
-		goto err;
+		goto err_free_supported_band;
 	}
 
 	ret = rtw89_regd_init(rtwdev, rtw89_regd_notifier);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to init regd\n");
-		goto err;
+		goto err_unregister_hw;
 	}
 
 	return 0;
 
-err:
+err_unregister_hw:
+	ieee80211_unregister_hw(hw);
+err_free_supported_band:
+	rtw89_core_clr_supported_band(rtwdev);
+
 	return ret;
 }
 
@@ -3450,22 +3822,24 @@ struct rtw89_dev *rtw89_alloc_ieee80211_hw(struct device *device,
 					   u32 bus_data_size,
 					   const struct rtw89_chip_info *chip)
 {
+	struct rtw89_fw_info early_fw = {};
 	const struct firmware *firmware;
 	struct ieee80211_hw *hw;
 	struct rtw89_dev *rtwdev;
 	struct ieee80211_ops *ops;
 	u32 driver_data_size;
-	u32 early_feat_map = 0;
+	int fw_format = -1;
 	bool no_chanctx;
 
-	firmware = rtw89_early_fw_feature_recognize(device, chip, &early_feat_map);
+	firmware = rtw89_early_fw_feature_recognize(device, chip, &early_fw, &fw_format);
 
 	ops = kmemdup(&rtw89_ops, sizeof(rtw89_ops), GFP_KERNEL);
 	if (!ops)
 		goto err;
 
 	no_chanctx = chip->support_chanctx_num == 0 ||
-		     !(early_feat_map & BIT(RTW89_FW_FEATURE_SCAN_OFFLOAD));
+		     !RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD, &early_fw) ||
+		     !RTW89_CHK_FW_FEATURE(BEACON_FILTER, &early_fw);
 
 	if (no_chanctx) {
 		ops->add_chanctx = NULL;
@@ -3473,6 +3847,8 @@ struct rtw89_dev *rtw89_alloc_ieee80211_hw(struct device *device,
 		ops->change_chanctx = NULL;
 		ops->assign_vif_chanctx = NULL;
 		ops->unassign_vif_chanctx = NULL;
+		ops->remain_on_channel = NULL;
+		ops->cancel_remain_on_channel = NULL;
 	}
 
 	driver_data_size = sizeof(struct rtw89_dev) + bus_data_size;
@@ -3480,12 +3856,16 @@ struct rtw89_dev *rtw89_alloc_ieee80211_hw(struct device *device,
 	if (!hw)
 		goto err;
 
+	hw->wiphy->iface_combinations = rtw89_iface_combs;
+	hw->wiphy->n_iface_combinations = ARRAY_SIZE(rtw89_iface_combs);
+
 	rtwdev = hw->priv;
 	rtwdev->hw = hw;
 	rtwdev->dev = device;
 	rtwdev->ops = ops;
 	rtwdev->chip = chip;
-	rtwdev->fw.firmware = firmware;
+	rtwdev->fw.req.firmware = firmware;
+	rtwdev->fw.fw_format = fw_format;
 
 	rtw89_debug(rtwdev, RTW89_DBG_FW, "probe driver %s chanctx\n",
 		    no_chanctx ? "without" : "with");
@@ -3502,7 +3882,7 @@ EXPORT_SYMBOL(rtw89_alloc_ieee80211_hw);
 void rtw89_free_ieee80211_hw(struct rtw89_dev *rtwdev)
 {
 	kfree(rtwdev->ops);
-	release_firmware(rtwdev->fw.firmware);
+	release_firmware(rtwdev->fw.req.firmware);
 	ieee80211_free_hw(rtwdev->hw);
 }
 EXPORT_SYMBOL(rtw89_free_ieee80211_hw);
diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h
index 41365ffb7e5e..6df386a38fb4 100644
--- a/drivers/net/wireless/realtek/rtw89/core.h
+++ b/drivers/net/wireless/realtek/rtw89/core.h
@@ -108,6 +108,7 @@ enum rtw89_core_chip_id {
 	RTL8852A,
 	RTL8852B,
 	RTL8852C,
+	RTL8851B,
 };
 
 enum rtw89_cv {
@@ -569,6 +570,7 @@ enum rtw89_sub_entity_idx {
 	RTW89_SUB_ENTITY_0 = 0,
 
 	NUM_OF_RTW89_SUB_ENTITY,
+	RTW89_SUB_ENTITY_IDLE = NUM_OF_RTW89_SUB_ENTITY,
 };
 
 enum rtw89_rf_path {
@@ -883,20 +885,24 @@ enum rtw89_btc_dcnt {
 	BTC_DCNT_RUN = 0x0,
 	BTC_DCNT_CX_RUNINFO,
 	BTC_DCNT_RPT,
-	BTC_DCNT_RPT_FREEZE,
+	BTC_DCNT_RPT_HANG,
 	BTC_DCNT_CYCLE,
-	BTC_DCNT_CYCLE_FREEZE,
+	BTC_DCNT_CYCLE_HANG,
 	BTC_DCNT_W1,
-	BTC_DCNT_W1_FREEZE,
+	BTC_DCNT_W1_HANG,
 	BTC_DCNT_B1,
-	BTC_DCNT_B1_FREEZE,
+	BTC_DCNT_B1_HANG,
 	BTC_DCNT_TDMA_NONSYNC,
 	BTC_DCNT_SLOT_NONSYNC,
-	BTC_DCNT_BTCNT_FREEZE,
+	BTC_DCNT_BTCNT_HANG,
 	BTC_DCNT_WL_SLOT_DRIFT,
-	BTC_DCNT_BT_SLOT_DRIFT,
 	BTC_DCNT_WL_STA_LAST,
-	BTC_DCNT_NUM,
+	BTC_DCNT_BT_SLOT_DRIFT,
+	BTC_DCNT_BT_SLOT_FLOOD,
+	BTC_DCNT_FDDT_TRIG,
+	BTC_DCNT_E2G,
+	BTC_DCNT_E2G_HANG,
+	BTC_DCNT_NUM
 };
 
 enum rtw89_btc_wl_state_cnt {
@@ -953,6 +959,8 @@ struct rtw89_btc_ant_info {
 
 	u8 single_pos: 1;/* Single antenna at S0 or S1 */
 	u8 diversity: 1;
+	u8 btg_pos: 2;
+	u8 stream_cnt: 4;
 };
 
 enum rtw89_tfc_dir {
@@ -1176,6 +1184,22 @@ struct rtw89_btc_wl_active_role_v1 {
 	u32 noa_duration; /* ms */
 };
 
+struct rtw89_btc_wl_active_role_v2 {
+	u8 connected: 1;
+	u8 pid: 3;
+	u8 phy: 1;
+	u8 noa: 1;
+	u8 band: 2;
+
+	u8 client_ps: 1;
+	u8 bw: 7;
+
+	u8 role;
+	u8 ch;
+
+	u32 noa_duration; /* ms */
+};
+
 struct rtw89_btc_wl_role_info_bpos {
 	u16 none: 1;
 	u16 station: 1;
@@ -1224,6 +1248,21 @@ struct rtw89_btc_wl_role_info_v1 { /* struct size must be n*4 bytes */
 	u32 rsvd: 27;
 };
 
+struct rtw89_btc_wl_role_info_v2 { /* struct size must be n*4 bytes */
+	u8 connect_cnt;
+	u8 link_mode;
+	union rtw89_btc_wl_role_info_map role_map;
+	struct rtw89_btc_wl_active_role_v2 active_role_v2[RTW89_PORT_NUM];
+	u32 mrole_type; /* btc_wl_mrole_type */
+	u32 mrole_noa_duration; /* ms */
+
+	u32 dbcc_en: 1;
+	u32 dbcc_chg: 1;
+	u32 dbcc_2g_phy: 2; /* which phy operate in 2G, HW_PHY_0 or HW_PHY_1 */
+	u32 link_mode_chg: 1;
+	u32 rsvd: 27;
+};
+
 struct rtw89_btc_wl_ver_info {
 	u32 fw_coex; /* match with which coex_ver */
 	u32 fw;
@@ -1302,15 +1341,22 @@ struct rtw89_btc_dm_emap {
 	u32 pta_owner: 1;
 	u32 wl_rfk_timeout: 1;
 	u32 bt_rfk_timeout: 1;
-
 	u32 wl_fw_hang: 1;
-	u32 offload_mismatch: 1;
 	u32 cycle_hang: 1;
 	u32 w1_hang: 1;
-
 	u32 b1_hang: 1;
 	u32 tdma_no_sync: 1;
+	u32 slot_no_sync: 1;
 	u32 wl_slot_drift: 1;
+	u32 bt_slot_drift: 1;
+	u32 role_num_mismatch: 1;
+	u32 null1_tx_late: 1;
+	u32 bt_afh_conflict: 1;
+	u32 bt_leafh_conflict: 1;
+	u32 bt_slot_flood: 1;
+	u32 wl_e2g_hang: 1;
+	u32 wl_ver_mismatch: 1;
+	u32 bt_ver_mismatch: 1;
 };
 
 union rtw89_btc_dm_error_map {
@@ -1325,6 +1371,22 @@ struct rtw89_btc_rf_para {
 	u32 rx_gain_perpkt;
 };
 
+struct rtw89_btc_wl_nhm {
+	u8 instant_wl_nhm_dbm;
+	u8 instant_wl_nhm_per_mhz;
+	u16 valid_record_times;
+	s8 record_pwr[16];
+	u8 record_ratio[16];
+	s8 pwr; /* dbm_per_MHz  */
+	u8 ratio;
+	u8 current_status;
+	u8 refresh;
+	bool start_flag;
+	u8 last_ccx_rpt_stamp;
+	s8 pwr_max;
+	s8 pwr_min;
+};
+
 struct rtw89_btc_wl_info {
 	struct rtw89_btc_wl_link_info link_info[RTW89_PORT_NUM];
 	struct rtw89_btc_wl_rfk_info rfk_info;
@@ -1332,13 +1394,16 @@ struct rtw89_btc_wl_info {
 	struct rtw89_btc_wl_afh_info afh_info;
 	struct rtw89_btc_wl_role_info role_info;
 	struct rtw89_btc_wl_role_info_v1 role_info_v1;
+	struct rtw89_btc_wl_role_info_v2 role_info_v2;
 	struct rtw89_btc_wl_scan_info scan_info;
 	struct rtw89_btc_wl_dbcc_info dbcc_info;
 	struct rtw89_btc_rf_para rf_para;
+	struct rtw89_btc_wl_nhm nhm;
 	union rtw89_btc_wl_state_map status;
 
 	u8 port_id[RTW89_WIFI_ROLE_MLME_MAX];
 	u8 rssi_level;
+	u8 cn_report;
 
 	bool scbd_change;
 	u32 scbd;
@@ -1352,8 +1417,9 @@ struct rtw89_btc_module {
 	u8 bt_solo: 1;
 	u8 bt_pos: 1;
 	u8 switch_type: 1;
+	u8 wa_type: 3;
 
-	u8 rsvd;
+	u8 kt_ver_adie;
 };
 
 #define RTW89_BTC_DM_MAXSTEP 30
@@ -1384,14 +1450,6 @@ struct rtw89_btc_wl_tx_limit_para {
 	u16 tx_retry;
 };
 
-struct rtw89_btc_bt_scan_info {
-	u16 win;
-	u16 intvl;
-	u32 enable: 1;
-	u32 interlace: 1;
-	u32 rsvd: 30;
-};
-
 enum rtw89_btc_bt_scan_type {
 	BTC_SCAN_INQ	= 0,
 	BTC_SCAN_PAGE,
@@ -1402,9 +1460,50 @@ enum rtw89_btc_bt_scan_type {
 	BTC_SCAN_MAX1,
 };
 
+enum rtw89_btc_ble_scan_type {
+	CXSCAN_BG = 0,
+	CXSCAN_INIT,
+	CXSCAN_LE,
+	CXSCAN_MAX
+};
+
+#define RTW89_BTC_BTC_SCAN_V1_FLAG_ENABLE BIT(0)
+#define RTW89_BTC_BTC_SCAN_V1_FLAG_INTERLACE BIT(1)
+
+struct rtw89_btc_bt_scan_info_v1 {
+	__le16 win;
+	__le16 intvl;
+	__le32 flags;
+} __packed;
+
+struct rtw89_btc_bt_scan_info_v2 {
+	__le16 win;
+	__le16 intvl;
+} __packed;
+
+struct rtw89_btc_fbtc_btscan_v1 {
+	u8 fver; /* btc_ver::fcxbtscan */
+	u8 rsvd;
+	__le16 rsvd2;
+	struct rtw89_btc_bt_scan_info_v1 scan[BTC_SCAN_MAX1];
+} __packed;
+
+struct rtw89_btc_fbtc_btscan_v2 {
+	u8 fver; /* btc_ver::fcxbtscan */
+	u8 type;
+	__le16 rsvd2;
+	struct rtw89_btc_bt_scan_info_v2 para[CXSCAN_MAX];
+} __packed;
+
+union rtw89_btc_fbtc_btscan {
+	struct rtw89_btc_fbtc_btscan_v1 v1;
+	struct rtw89_btc_fbtc_btscan_v2 v2;
+};
+
 struct rtw89_btc_bt_info {
 	struct rtw89_btc_bt_link_info link_info;
-	struct rtw89_btc_bt_scan_info scan_info[BTC_SCAN_MAX1];
+	struct rtw89_btc_bt_scan_info_v1 scan_info_v1[BTC_SCAN_MAX1];
+	struct rtw89_btc_bt_scan_info_v2 scan_info_v2[CXSCAN_MAX];
 	struct rtw89_btc_bt_ver_info ver_info;
 	struct rtw89_btc_bool_sta_chg enable;
 	struct rtw89_btc_bool_sta_chg inq_pag;
@@ -1427,7 +1526,8 @@ struct rtw89_btc_bt_info {
 	u32 run_patch_code: 1;
 	u32 hi_lna_rx: 1;
 	u32 scan_rx_low_pri: 1;
-	u32 rsvd: 21;
+	u32 scan_info_update: 1;
+	u32 rsvd: 20;
 };
 
 struct rtw89_btc_cx {
@@ -1463,6 +1563,7 @@ union rtw89_btc_fbtc_tdma_le32 {
 };
 
 #define CXMREG_MAX 30
+#define CXMREG_MAX_V2 20
 #define FCXMAX_STEP 255 /*STEP trace record cnt, Max:65535, default:255*/
 #define BTC_CYCLE_SLOT_MAX 48 /* must be even number, non-zero */
 
@@ -1480,6 +1581,16 @@ enum rtw89_btc_bt_sta_counter {
 	BTC_BCNT_STA_MAX
 };
 
+enum rtw89_btc_bt_sta_counter_v105 {
+	BTC_BCNT_RFK_REQ_V105 = 0,
+	BTC_BCNT_HI_TX_V105 = 1,
+	BTC_BCNT_HI_RX_V105 = 2,
+	BTC_BCNT_LO_TX_V105 = 3,
+	BTC_BCNT_LO_RX_V105 = 4,
+	BTC_BCNT_POLLUTED_V105 = 5,
+	BTC_BCNT_STA_MAX_V105
+};
+
 struct rtw89_btc_fbtc_rpt_ctrl_v1 {
 	u16 fver; /* btc_ver::fcxbtcrpt */
 	u16 rpt_cnt; /* tmr counters */
@@ -1570,10 +1681,23 @@ struct rtw89_btc_fbtc_rpt_ctrl_v5 {
 	struct rtw89_btc_fbtc_rpt_ctrl_bt_mailbox bt_mbx_info;
 } __packed;
 
+struct rtw89_btc_fbtc_rpt_ctrl_v105 {
+	u8 fver;
+	u8 rsvd;
+	__le16 rsvd1;
+
+	u8 gnt_val[RTW89_PHY_MAX][4];
+	__le16 bt_cnt[BTC_BCNT_STA_MAX_V105];
+
+	struct rtw89_btc_fbtc_rpt_ctrl_info_v5 rpt_info;
+	struct rtw89_btc_fbtc_rpt_ctrl_bt_mailbox bt_mbx_info;
+} __packed;
+
 union rtw89_btc_fbtc_rpt_ctrl_ver_info {
 	struct rtw89_btc_fbtc_rpt_ctrl_v1 v1;
 	struct rtw89_btc_fbtc_rpt_ctrl_v4 v4;
 	struct rtw89_btc_fbtc_rpt_ctrl_v5 v5;
+	struct rtw89_btc_fbtc_rpt_ctrl_v105 v105;
 };
 
 enum rtw89_fbtc_ext_ctrl_type {
@@ -1689,13 +1813,25 @@ struct rtw89_btc_fbtc_gpio_dbg {
 	u8 gpio_map[BTC_DBG_MAX1]; /*the debug signals to GPIO-Position */
 } __packed;
 
-struct rtw89_btc_fbtc_mreg_val {
+struct rtw89_btc_fbtc_mreg_val_v1 {
 	u8 fver; /* btc_ver::fcxmreg */
 	u8 reg_num;
 	__le16 rsvd;
 	__le32 mreg_val[CXMREG_MAX];
 } __packed;
 
+struct rtw89_btc_fbtc_mreg_val_v2 {
+	u8 fver; /* btc_ver::fcxmreg */
+	u8 reg_num;
+	__le16 rsvd;
+	__le32 mreg_val[CXMREG_MAX_V2];
+} __packed;
+
+union rtw89_btc_fbtc_mreg_val {
+	struct rtw89_btc_fbtc_mreg_val_v1 v1;
+	struct rtw89_btc_fbtc_mreg_val_v2 v2;
+};
+
 #define RTW89_DEF_FBTC_MREG(__type, __bytes, __offset) \
 	{ .type = cpu_to_le16(__type), .bytes = cpu_to_le16(__bytes), \
 	  .offset = cpu_to_le32(__offset), }
@@ -1786,6 +1922,11 @@ struct rtw89_btc_fbtc_cycle_time_info {
 	__le16 tmaxdiff[CXT_MAX]; /* max wl-wl bt-bt cycle diff time */
 } __packed;
 
+struct rtw89_btc_fbtc_cycle_time_info_v5 {
+	__le16 tavg[CXT_MAX]; /* avg wl/bt cycle time */
+	__le16 tmax[CXT_MAX]; /* max wl/bt cycle time */
+} __packed;
+
 struct rtw89_btc_fbtc_a2dp_trx_stat {
 	u8 empty_cnt;
 	u8 retry_cnt;
@@ -1842,6 +1983,21 @@ struct rtw89_btc_fbtc_cycle_fddt_info {
 #define RTW89_BTC_FDDT_CELL_TRAIN_STATE GENMASK(3, 0)
 #define RTW89_BTC_FDDT_CELL_TRAIN_PHASE GENMASK(7, 4)
 
+struct rtw89_btc_fbtc_cycle_fddt_info_v5 {
+	__le16 train_cycle;
+	__le16 tp;
+
+	s8 tx_power; /* absolute Tx power (dBm), 0xff-> no BTC control */
+	s8 bt_tx_power; /* decrease Tx power (dB) */
+	s8 bt_rx_gain;  /* LNA constrain level */
+	u8 no_empty_cnt;
+
+	u8 rssi; /* [7:4] -> bt_rssi_level, [3:0]-> wl_rssi_level */
+	u8 cn; /* condition_num */
+	u8 train_status; /* [7:4]-> train-state, [3:0]-> train-phase */
+	u8 train_result; /* refer to enum btc_fddt_check_map */
+} __packed;
+
 struct rtw89_btc_fbtc_fddt_cell_status {
 	s8 wl_tx_pwr;
 	s8 bt_tx_pwr;
@@ -1849,6 +2005,12 @@ struct rtw89_btc_fbtc_fddt_cell_status {
 	u8 state_phase; /* [0:3] train state, [4:7] train phase */
 } __packed;
 
+struct rtw89_btc_fbtc_fddt_cell_status_v5 {
+	s8 wl_tx_pwr;
+	s8 bt_tx_pwr;
+	s8 bt_rx_gain;
+} __packed;
+
 struct rtw89_btc_fbtc_cysta_v3 { /* statistics for cycles */
 	u8 fver;
 	u8 rsvd;
@@ -1894,10 +2056,35 @@ struct rtw89_btc_fbtc_cysta_v4 { /* statistics for cycles */
 	__le32 except_map;
 } __packed;
 
+struct rtw89_btc_fbtc_cysta_v5 { /* statistics for cycles */
+	u8 fver;
+	u8 rsvd;
+	u8 collision_cnt; /* counter for event/timer occur at the same time */
+	u8 except_cnt;
+	u8 wl_rx_err_ratio[BTC_CYCLE_SLOT_MAX];
+
+	__le16 skip_cnt;
+	__le16 cycles; /* total cycle number */
+
+	__le16 slot_step_time[BTC_CYCLE_SLOT_MAX]; /* record the wl/bt slot time */
+	__le16 slot_cnt[CXST_MAX]; /* slot count */
+	__le16 bcn_cnt[CXBCN_MAX];
+	struct rtw89_btc_fbtc_cycle_time_info_v5 cycle_time;
+	struct rtw89_btc_fbtc_cycle_leak_info leak_slot;
+	struct rtw89_btc_fbtc_cycle_a2dp_empty_info a2dp_ept;
+	struct rtw89_btc_fbtc_a2dp_trx_stat_v4 a2dp_trx[BTC_CYCLE_SLOT_MAX];
+	struct rtw89_btc_fbtc_cycle_fddt_info_v5 fddt_trx[BTC_CYCLE_SLOT_MAX];
+	struct rtw89_btc_fbtc_fddt_cell_status_v5 fddt_cells[FDD_TRAIN_WL_DIRECTION]
+							    [FDD_TRAIN_WL_RSSI_LEVEL]
+							    [FDD_TRAIN_BT_RSSI_LEVEL];
+	__le32 except_map;
+} __packed;
+
 union rtw89_btc_fbtc_cysta_info {
 	struct rtw89_btc_fbtc_cysta_v2 v2;
 	struct rtw89_btc_fbtc_cysta_v3 v3;
 	struct rtw89_btc_fbtc_cysta_v4 v4;
+	struct rtw89_btc_fbtc_cysta_v5 v5;
 };
 
 struct rtw89_btc_fbtc_cynullsta_v1 { /* cycle null statistics */
@@ -1932,13 +2119,6 @@ struct rtw89_btc_fbtc_btver {
 	__le32 feature;
 } __packed;
 
-struct rtw89_btc_fbtc_btscan {
-	u8 fver; /* btc_ver::fcxbtscan */
-	u8 rsvd;
-	__le16 rsvd2;
-	u8 scan[6];
-} __packed;
-
 struct rtw89_btc_fbtc_btafh {
 	u8 fver; /* btc_ver::fcxbtafh */
 	u8 rsvd;
@@ -1976,6 +2156,30 @@ struct rtw89_btc_rf_trx_para {
 	u8 bt_rx_gain;  /* LNA constrain level */
 };
 
+struct rtw89_btc_trx_info {
+	u8 tx_lvl;
+	u8 rx_lvl;
+	u8 wl_rssi;
+	u8 bt_rssi;
+
+	s8 tx_power; /* absolute Tx power (dBm), 0xff-> no BTC control */
+	s8 rx_gain;  /* rx gain table index (TBD.) */
+	s8 bt_tx_power; /* decrease Tx power (dB) */
+	s8 bt_rx_gain;  /* LNA constrain level */
+
+	u8 cn; /* condition_num */
+	s8 nhm;
+	u8 bt_profile;
+	u8 rsvd2;
+
+	u16 tx_rate;
+	u16 rx_rate;
+
+	u32 tx_tp;
+	u32 rx_tp;
+	u32 rx_err_ratio;
+};
+
 struct rtw89_btc_dm {
 	struct rtw89_btc_fbtc_slot slot[CXST_MAX];
 	struct rtw89_btc_fbtc_slot slot_now[CXST_MAX];
@@ -1987,6 +2191,7 @@ struct rtw89_btc_dm {
 	struct rtw89_btc_wl_tx_limit_para wl_tx_limit;
 	struct rtw89_btc_dm_step dm_step;
 	struct rtw89_btc_wl_scc_ctrl wl_scc;
+	struct rtw89_btc_trx_info trx_info;
 	union rtw89_btc_dm_error_map error;
 	u32 cnt_dm[BTC_DCNT_NUM];
 	u32 cnt_notify[BTC_NCNT_NUM];
@@ -1997,6 +2202,7 @@ struct rtw89_btc_dm {
 	u32 wl_only: 1;
 	u32 wl_fw_cx_offload: 1;
 	u32 freerun: 1;
+	u32 fddt_train: 1;
 	u32 wl_ps_ctrl: 2;
 	u32 wl_mimo_ps: 1;
 	u32 leak_ap: 1;
@@ -2008,12 +2214,13 @@ struct rtw89_btc_dm {
 	u32 wl_stb_chg: 1;
 	u32 pta_owner: 1;
 	u32 tdma_instant_excute: 1;
-	u32 rsvd: 1;
 
 	u16 slot_dur[CXST_MAX];
 
 	u8 run_reason;
 	u8 run_action;
+
+	u8 wl_lna2: 1;
 };
 
 struct rtw89_btc_ctrl {
@@ -2117,7 +2324,7 @@ struct rtw89_btc_rpt_fbtc_nullsta {
 
 struct rtw89_btc_rpt_fbtc_mreg {
 	struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */
-	struct rtw89_btc_fbtc_mreg_val finfo; /* info from fw */
+	union rtw89_btc_fbtc_mreg_val finfo; /* info from fw */
 };
 
 struct rtw89_btc_rpt_fbtc_gpio_dbg {
@@ -2132,7 +2339,7 @@ struct rtw89_btc_rpt_fbtc_btver {
 
 struct rtw89_btc_rpt_fbtc_btscan {
 	struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */
-	struct rtw89_btc_fbtc_btscan finfo; /* info from fw */
+	union rtw89_btc_fbtc_btscan finfo; /* info from fw */
 };
 
 struct rtw89_btc_rpt_fbtc_btafh {
@@ -2395,6 +2602,7 @@ struct rtw89_sta {
 	struct rtw89_addr_cam_entry addr_cam; /* AP mode or TDLS peer only */
 	struct rtw89_bssid_cam_entry bssid_cam; /* TDLS peer only */
 	struct list_head ba_cam_list;
+	struct sk_buff_head roc_queue;
 
 	bool use_cfg_mask;
 	struct cfg80211_bitrate_mask mask;
@@ -2421,11 +2629,39 @@ struct rtw89_phy_rate_pattern {
 	bool enable;
 };
 
+struct rtw89_tx_wait_info {
+	struct rcu_head rcu_head;
+	struct completion completion;
+	bool tx_done;
+};
+
+struct rtw89_tx_skb_data {
+	struct rtw89_tx_wait_info __rcu *wait;
+	u8 hci_priv[];
+};
+
+#define RTW89_ROC_IDLE_TIMEOUT 500
+#define RTW89_ROC_TX_TIMEOUT 30
+enum rtw89_roc_state {
+	RTW89_ROC_IDLE,
+	RTW89_ROC_NORMAL,
+	RTW89_ROC_MGMT,
+};
+
+struct rtw89_roc {
+	struct ieee80211_channel chan;
+	struct delayed_work roc_work;
+	enum ieee80211_roc_type type;
+	enum rtw89_roc_state state;
+	int duration;
+};
+
 #define RTW89_P2P_MAX_NOA_NUM 2
 
 struct rtw89_vif {
 	struct list_head list;
 	struct rtw89_dev *rtwdev;
+	struct rtw89_roc roc;
 	enum rtw89_sub_entity_idx sub_entity_idx;
 
 	u8 mac_id;
@@ -2441,6 +2677,7 @@ struct rtw89_vif {
 	u8 bcn_hit_cond;
 	u8 hit_rule;
 	u8 last_noa_nr;
+	bool offchan;
 	bool trigger;
 	bool lsig_txop;
 	u8 tgt_ind;
@@ -2757,6 +2994,41 @@ struct rtw89_txpwr_table {
 		     const struct rtw89_txpwr_table *tbl);
 };
 
+struct rtw89_txpwr_rule_2ghz {
+	const s8 (*lmt)[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
+		       [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
+		       [RTW89_REGD_NUM][RTW89_2G_CH_NUM];
+	const s8 (*lmt_ru)[RTW89_RU_NUM][RTW89_NTX_NUM]
+			  [RTW89_REGD_NUM][RTW89_2G_CH_NUM];
+};
+
+struct rtw89_txpwr_rule_5ghz {
+	const s8 (*lmt)[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
+		       [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
+		       [RTW89_REGD_NUM][RTW89_5G_CH_NUM];
+	const s8 (*lmt_ru)[RTW89_RU_NUM][RTW89_NTX_NUM]
+			  [RTW89_REGD_NUM][RTW89_5G_CH_NUM];
+};
+
+struct rtw89_txpwr_rule_6ghz {
+	const s8 (*lmt)[RTW89_6G_BW_NUM][RTW89_NTX_NUM]
+		       [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
+		       [RTW89_REGD_NUM][RTW89_6G_CH_NUM];
+	const s8 (*lmt_ru)[RTW89_RU_NUM][RTW89_NTX_NUM]
+			  [RTW89_REGD_NUM][RTW89_6G_CH_NUM];
+};
+
+struct rtw89_rfe_parms {
+	struct rtw89_txpwr_rule_2ghz rule_2ghz;
+	struct rtw89_txpwr_rule_5ghz rule_5ghz;
+	struct rtw89_txpwr_rule_6ghz rule_6ghz;
+};
+
+struct rtw89_rfe_parms_conf {
+	const struct rtw89_rfe_parms *rfe_parms;
+	u8 rfe_type;
+};
+
 struct rtw89_page_regs {
 	u32 hci_fc_ctrl;
 	u32 ch_page_ctrl;
@@ -2847,7 +3119,8 @@ struct rtw89_phy_ul_tb_info {
 struct rtw89_chip_info {
 	enum rtw89_core_chip_id chip_id;
 	const struct rtw89_chip_ops *ops;
-	const char *fw_name;
+	const char *fw_basename;
+	u8 fw_format_max;
 	bool try_ce_fw;
 	u32 fifo_size;
 	u32 dle_scc_rsvd_size;
@@ -2893,21 +3166,10 @@ struct rtw89_chip_info {
 	const struct rtw89_phy_dig_gain_table *dig_table;
 	const struct rtw89_dig_regs *dig_regs;
 	const struct rtw89_phy_tssi_dbw_table *tssi_dbw_table;
-	const s8 (*txpwr_lmt_2g)[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
-				[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
-				[RTW89_REGD_NUM][RTW89_2G_CH_NUM];
-	const s8 (*txpwr_lmt_5g)[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
-				[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
-				[RTW89_REGD_NUM][RTW89_5G_CH_NUM];
-	const s8 (*txpwr_lmt_6g)[RTW89_6G_BW_NUM][RTW89_NTX_NUM]
-				[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
-				[RTW89_REGD_NUM][RTW89_6G_CH_NUM];
-	const s8 (*txpwr_lmt_ru_2g)[RTW89_RU_NUM][RTW89_NTX_NUM]
-				   [RTW89_REGD_NUM][RTW89_2G_CH_NUM];
-	const s8 (*txpwr_lmt_ru_5g)[RTW89_RU_NUM][RTW89_NTX_NUM]
-				   [RTW89_REGD_NUM][RTW89_5G_CH_NUM];
-	const s8 (*txpwr_lmt_ru_6g)[RTW89_RU_NUM][RTW89_NTX_NUM]
-				   [RTW89_REGD_NUM][RTW89_6G_CH_NUM];
+
+	/* NULL if no rfe-specific, or a null-terminated array by rfe_parms */
+	const struct rtw89_rfe_parms_conf *rfe_parms_conf;
+	const struct rtw89_rfe_parms *dflt_parms;
 
 	u8 txpwr_factor_rf;
 	u8 txpwr_factor_mac;
@@ -2938,16 +3200,20 @@ struct rtw89_chip_info {
 	u32 txwd_body_size;
 	u32 h2c_ctrl_reg;
 	const u32 *h2c_regs;
+	struct rtw89_reg_def h2c_counter_reg;
 	u32 c2h_ctrl_reg;
 	const u32 *c2h_regs;
+	struct rtw89_reg_def c2h_counter_reg;
 	const struct rtw89_page_regs *page_regs;
 	bool cfo_src_fd;
+	bool cfo_hw_comp;
 	const struct rtw89_reg_def *dcfo_comp;
 	u8 dcfo_comp_sft;
 	const struct rtw89_imr_info *imr_info;
 	const struct rtw89_rrsr_cfgs *rrsr_cfgs;
 	u32 bss_clr_map_reg;
 	u32 dma_ch_mask;
+	u32 edcca_lvl_reg;
 	const struct wiphy_wowlan_support *wowlan_stub;
 };
 
@@ -3023,9 +3289,10 @@ enum rtw89_fw_feature {
 	RTW89_FW_FEATURE_SCAN_OFFLOAD,
 	RTW89_FW_FEATURE_TX_WAKE,
 	RTW89_FW_FEATURE_CRASH_TRIGGER,
-	RTW89_FW_FEATURE_PACKET_DROP,
+	RTW89_FW_FEATURE_NO_PACKET_DROP,
 	RTW89_FW_FEATURE_NO_DEEP_PS,
 	RTW89_FW_FEATURE_NO_LPS_PG,
+	RTW89_FW_FEATURE_BEACON_FILTER,
 };
 
 struct rtw89_fw_suit {
@@ -3060,12 +3327,18 @@ struct rtw89_fw_suit {
 			  GET_FW_HDR_SUBVERSION(fw_hdr),	\
 			  GET_FW_HDR_SUBINDEX(fw_hdr))
 
-struct rtw89_fw_info {
+struct rtw89_fw_req_info {
 	const struct firmware *firmware;
-	struct rtw89_dev *rtwdev;
 	struct completion completion;
+};
+
+struct rtw89_fw_info {
+	struct rtw89_fw_req_info req;
+	int fw_format;
 	u8 h2c_seq;
 	u8 rec_seq;
+	u8 h2c_counter;
+	u8 c2h_counter;
 	struct rtw89_fw_suit normal;
 	struct rtw89_fw_suit wowlan;
 	bool fw_log_enable;
@@ -3143,6 +3416,7 @@ struct rtw89_sub_entity {
 struct rtw89_hal {
 	u32 rx_fltr;
 	u8 cv;
+	u8 acv;
 	u32 sw_amsdu_max_size;
 	u32 antenna_tx;
 	u32 antenna_rx;
@@ -3151,12 +3425,16 @@ struct rtw89_hal {
 	bool tx_path_diversity;
 	bool support_cckpd;
 	bool support_igi;
+	atomic_t roc_entity_idx;
 
 	DECLARE_BITMAP(entity_map, NUM_OF_RTW89_SUB_ENTITY);
 	struct rtw89_sub_entity sub[NUM_OF_RTW89_SUB_ENTITY];
+	struct cfg80211_chan_def roc_chandef;
 
 	bool entity_active;
 	enum rtw89_entity_mode entity_mode;
+
+	u32 edcca_bak;
 };
 
 #define RTW89_MAX_MAC_ID_NUM 128
@@ -3168,6 +3446,7 @@ enum rtw89_flags {
 	RTW89_FLAG_RUNNING,
 	RTW89_FLAG_BFEE_MON,
 	RTW89_FLAG_BFEE_EN,
+	RTW89_FLAG_BFEE_TIMER_KEEP,
 	RTW89_FLAG_NAPI_RUNNING,
 	RTW89_FLAG_LEISURE_PS,
 	RTW89_FLAG_LOW_POWER_MODE,
@@ -3411,6 +3690,8 @@ struct rtw89_cfo_tracking_info {
 	s32 cfo_avg_pre;
 	s32 cfo_avg[CFO_TRACK_MAX_USER];
 	s32 pre_cfo_avg[CFO_TRACK_MAX_USER];
+	s32 dcfo_avg;
+	s32 dcfo_avg_pre;
 	u32 packet_count;
 	u32 packet_count_pre;
 	s32 residual_cfo_acc;
@@ -3655,10 +3936,7 @@ struct rtw89_early_h2c {
 struct rtw89_hw_scan_info {
 	struct ieee80211_vif *scanning_vif;
 	struct list_head pkt_list[NUM_NL80211_BANDS];
-	u8 op_pri_ch;
-	u8 op_chan;
-	u8 op_bw;
-	u8 op_band;
+	struct rtw89_chan op_chan;
 	u32 last_chan_idx;
 };
 
@@ -3743,6 +4021,7 @@ struct rtw89_dev {
 	struct rtw89_hw_scan_info scan_info;
 	const struct rtw89_chip_info *chip;
 	const struct rtw89_pci_info *pci_info;
+	const struct rtw89_rfe_parms *rfe_parms;
 	struct rtw89_hal hal;
 	struct rtw89_mcc_info mcc;
 	struct rtw89_mac_info mac;
@@ -3774,6 +4053,7 @@ struct rtw89_dev {
 	struct sk_buff_head c2h_queue;
 	struct work_struct c2h_work;
 	struct work_struct ips_work;
+	struct work_struct load_firmware_work;
 
 	struct list_head early_h2c_list;
 
@@ -3813,6 +4093,7 @@ struct rtw89_dev {
 	struct delayed_work coex_rfk_chk_work;
 	struct delayed_work cfo_track_work;
 	struct delayed_work forbid_ba_work;
+	struct delayed_work roc_work;
 	struct rtw89_ppdu_sts_info ppdu_sts;
 	u8 total_sta_assoc;
 	bool scanning;
@@ -3968,6 +4249,14 @@ static inline void rtw89_hci_clear(struct rtw89_dev *rtwdev, struct pci_dev *pde
 		rtwdev->hci.ops->clear(rtwdev, pdev);
 }
 
+static inline
+struct rtw89_tx_skb_data *RTW89_TX_SKB_CB(struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+
+	return (struct rtw89_tx_skb_data *)info->status.status_driver_data;
+}
+
 static inline u8 rtw89_read8(struct rtw89_dev *rtwdev, u32 addr)
 {
 	return rtwdev->hci.ops->read8(rtwdev, addr);
@@ -4320,6 +4609,10 @@ const struct cfg80211_chan_def *rtw89_chandef_get(struct rtw89_dev *rtwdev,
 						  enum rtw89_sub_entity_idx idx)
 {
 	struct rtw89_hal *hal = &rtwdev->hal;
+	enum rtw89_sub_entity_idx roc_idx = atomic_read(&hal->roc_entity_idx);
+
+	if (roc_idx == idx)
+		return &hal->roc_chandef;
 
 	return &hal->sub[idx].chandef;
 }
@@ -4611,11 +4904,32 @@ static inline struct sk_buff *rtw89_alloc_skb_for_rx(struct rtw89_dev *rtwdev,
 	return dev_alloc_skb(length);
 }
 
+static inline void rtw89_core_tx_wait_complete(struct rtw89_dev *rtwdev,
+					       struct rtw89_tx_skb_data *skb_data,
+					       bool tx_done)
+{
+	struct rtw89_tx_wait_info *wait;
+
+	rcu_read_lock();
+
+	wait = rcu_dereference(skb_data->wait);
+	if (!wait)
+		goto out;
+
+	wait->tx_done = tx_done;
+	complete(&wait->completion);
+
+out:
+	rcu_read_unlock();
+}
+
 int rtw89_core_tx_write(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 			struct ieee80211_sta *sta, struct sk_buff *skb, int *qsel);
 int rtw89_h2c_tx(struct rtw89_dev *rtwdev,
 		 struct sk_buff *skb, bool fwdl);
 void rtw89_core_tx_kick_off(struct rtw89_dev *rtwdev, u8 qsel);
+int rtw89_core_tx_kick_off_and_wait(struct rtw89_dev *rtwdev, struct sk_buff *skb,
+				    int qsel, unsigned int timeout);
 void rtw89_core_fill_txdesc(struct rtw89_dev *rtwdev,
 			    struct rtw89_tx_desc_info *desc_info,
 			    void *txdesc);
@@ -4664,6 +4978,8 @@ void rtw89_free_ieee80211_hw(struct rtw89_dev *rtwdev);
 void rtw89_core_set_chip_txpwr(struct rtw89_dev *rtwdev);
 void rtw89_get_default_chandef(struct cfg80211_chan_def *chandef);
 void rtw89_set_channel(struct rtw89_dev *rtwdev);
+void rtw89_get_channel(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
+		       struct rtw89_chan *chan);
 u8 rtw89_core_acquire_bit_map(unsigned long *addr, unsigned long size);
 void rtw89_core_release_bit_map(unsigned long *addr, u8 bit);
 void rtw89_core_release_all_bits_map(unsigned long *addr, unsigned int nbits);
@@ -4685,6 +5001,9 @@ void rtw89_complete_cond(struct rtw89_wait_info *wait, unsigned int cond,
 int rtw89_core_start(struct rtw89_dev *rtwdev);
 void rtw89_core_stop(struct rtw89_dev *rtwdev);
 void rtw89_core_update_beacon_work(struct work_struct *work);
+void rtw89_roc_work(struct work_struct *work);
+void rtw89_roc_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
+void rtw89_roc_end(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
 void rtw89_core_scan_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 			   const u8 *mac_addr, bool hw_scan);
 void rtw89_core_scan_complete(struct rtw89_dev *rtwdev,
diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c
index 0e0e1483c099..1e5b7a998716 100644
--- a/drivers/net/wireless/realtek/rtw89/debug.c
+++ b/drivers/net/wireless/realtek/rtw89/debug.c
@@ -3069,18 +3069,13 @@ static int rtw89_dbg_trigger_ctrl_error(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_cpuio_ctrl ctrl_para = {0};
 	u16 pkt_id;
+	int ret;
 
 	rtw89_leave_ps_mode(rtwdev);
 
-	pkt_id = rtw89_mac_dle_buf_req(rtwdev, 0x20, true);
-	switch (pkt_id) {
-	case 0xffff:
-		return -ETIMEDOUT;
-	case 0xfff:
-		return -ENOMEM;
-	default:
-		break;
-	}
+	ret = rtw89_mac_dle_buf_req(rtwdev, 0x20, true, &pkt_id);
+	if (ret)
+		return ret;
 
 	/* intentionally, enqueue two pkt, but has only one pkt id */
 	ctrl_para.cmd_type = CPUIO_OP_CMD_ENQ_TO_HEAD;
diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c
index 0b73dc2e9ad7..b9b675bf9d05 100644
--- a/drivers/net/wireless/realtek/rtw89/fw.c
+++ b/drivers/net/wireless/realtek/rtw89/fw.c
@@ -155,8 +155,9 @@ int rtw89_mfw_recognize(struct rtw89_dev *rtwdev, enum rtw89_fw_type type,
 			struct rtw89_fw_suit *fw_suit, bool nowarn)
 {
 	struct rtw89_fw_info *fw_info = &rtwdev->fw;
-	const u8 *mfw = fw_info->firmware->data;
-	u32 mfw_len = fw_info->firmware->size;
+	const struct firmware *firmware = fw_info->req.firmware;
+	const u8 *mfw = firmware->data;
+	u32 mfw_len = firmware->size;
 	const struct rtw89_mfw_hdr *mfw_hdr = (const struct rtw89_mfw_hdr *)mfw;
 	const struct rtw89_mfw_info *mfw_info;
 	int i;
@@ -235,6 +236,7 @@ static bool __fw_feat_cond_ ## __cond(u32 suit_ver_code, u32 comp_ver_code) \
 
 __DEF_FW_FEAT_COND(ge, >=); /* greater or equal */
 __DEF_FW_FEAT_COND(le, <=); /* less or equal */
+__DEF_FW_FEAT_COND(lt, <); /* less than */
 
 struct __fw_feat_cfg {
 	enum rtw89_core_chip_id chip_id;
@@ -256,47 +258,60 @@ static const struct __fw_feat_cfg fw_feat_tbl[] = {
 	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 35, 0, SCAN_OFFLOAD),
 	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 35, 0, TX_WAKE),
 	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 36, 0, CRASH_TRIGGER),
-	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 38, 0, PACKET_DROP),
+	__CFG_FW_FEAT(RTL8852A, lt, 0, 13, 38, 0, NO_PACKET_DROP),
 	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 26, 0, NO_LPS_PG),
-	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 20, 0, PACKET_DROP),
+	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 26, 0, TX_WAKE),
+	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 29, 0, CRASH_TRIGGER),
+	__CFG_FW_FEAT(RTL8852B, ge, 0, 29, 29, 0, SCAN_OFFLOAD),
 	__CFG_FW_FEAT(RTL8852C, le, 0, 27, 33, 0, NO_DEEP_PS),
 	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 34, 0, TX_WAKE),
 	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 36, 0, SCAN_OFFLOAD),
 	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 40, 0, CRASH_TRIGGER),
+	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 56, 10, BEACON_FILTER),
 };
 
+static void rtw89_fw_iterate_feature_cfg(struct rtw89_fw_info *fw,
+					 const struct rtw89_chip_info *chip,
+					 u32 ver_code)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(fw_feat_tbl); i++) {
+		const struct __fw_feat_cfg *ent = &fw_feat_tbl[i];
+
+		if (chip->chip_id != ent->chip_id)
+			continue;
+
+		if (ent->cond(ver_code, ent->ver_code))
+			RTW89_SET_FW_FEATURE(ent->feature, fw);
+	}
+}
+
 static void rtw89_fw_recognize_features(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
-	const struct __fw_feat_cfg *ent;
 	const struct rtw89_fw_suit *fw_suit;
 	u32 suit_ver_code;
-	int i;
 
 	fw_suit = rtw89_fw_suit_get(rtwdev, RTW89_FW_NORMAL);
 	suit_ver_code = RTW89_FW_SUIT_VER_CODE(fw_suit);
 
-	for (i = 0; i < ARRAY_SIZE(fw_feat_tbl); i++) {
-		ent = &fw_feat_tbl[i];
-		if (chip->chip_id != ent->chip_id)
-			continue;
-
-		if (ent->cond(suit_ver_code, ent->ver_code))
-			RTW89_SET_FW_FEATURE(ent->feature, &rtwdev->fw);
-	}
+	rtw89_fw_iterate_feature_cfg(&rtwdev->fw, chip, suit_ver_code);
 }
 
 const struct firmware *
 rtw89_early_fw_feature_recognize(struct device *device,
 				 const struct rtw89_chip_info *chip,
-				 u32 *early_feat_map)
+				 struct rtw89_fw_info *early_fw,
+				 int *used_fw_format)
 {
 	union rtw89_compat_fw_hdr buf = {};
 	const struct firmware *firmware;
 	bool full_req = false;
+	char fw_name[64];
+	int fw_format;
 	u32 ver_code;
 	int ret;
-	int i;
 
 	/* If SECURITY_LOADPIN_ENFORCE is enabled, reading partial files will
 	 * be denied (-EPERM). Then, we don't get right firmware things as
@@ -305,12 +320,22 @@ rtw89_early_fw_feature_recognize(struct device *device,
 	if (IS_ENABLED(CONFIG_SECURITY_LOADPIN_ENFORCE))
 		full_req = true;
 
-	if (full_req)
-		ret = request_firmware(&firmware, chip->fw_name, device);
-	else
-		ret = request_partial_firmware_into_buf(&firmware, chip->fw_name,
-							device, &buf, sizeof(buf),
-							0);
+	for (fw_format = chip->fw_format_max; fw_format >= 0; fw_format--) {
+		rtw89_fw_get_filename(fw_name, sizeof(fw_name),
+				      chip->fw_basename, fw_format);
+
+		if (full_req)
+			ret = request_firmware(&firmware, fw_name, device);
+		else
+			ret = request_partial_firmware_into_buf(&firmware, fw_name,
+								device, &buf, sizeof(buf),
+								0);
+		if (!ret) {
+			dev_info(device, "loaded firmware %s\n", fw_name);
+			*used_fw_format = fw_format;
+			break;
+		}
+	}
 
 	if (ret) {
 		dev_err(device, "failed to early request firmware: %d\n", ret);
@@ -325,15 +350,7 @@ rtw89_early_fw_feature_recognize(struct device *device,
 	if (!ver_code)
 		goto out;
 
-	for (i = 0; i < ARRAY_SIZE(fw_feat_tbl); i++) {
-		const struct __fw_feat_cfg *ent = &fw_feat_tbl[i];
-
-		if (chip->chip_id != ent->chip_id)
-			continue;
-
-		if (ent->cond(ver_code, ent->ver_code))
-			*early_feat_map |= BIT(ent->feature);
-	}
+	rtw89_fw_iterate_feature_cfg(early_fw, chip, ver_code);
 
 out:
 	if (full_req)
@@ -612,6 +629,8 @@ int rtw89_fw_download(struct rtw89_dev *rtwdev, enum rtw89_fw_type type)
 
 	fw_info->h2c_seq = 0;
 	fw_info->rec_seq = 0;
+	fw_info->h2c_counter = 0;
+	fw_info->c2h_counter = 0;
 	rtwdev->mac.rpwm_seq_num = RPWM_SEQ_NUM_MAX;
 	rtwdev->mac.cpwm_seq_num = CPWM_SEQ_NUM_MAX;
 
@@ -626,67 +645,62 @@ int rtw89_wait_firmware_completion(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_fw_info *fw = &rtwdev->fw;
 
-	wait_for_completion(&fw->completion);
-	if (!fw->firmware)
+	wait_for_completion(&fw->req.completion);
+	if (!fw->req.firmware)
 		return -EINVAL;
 
 	return 0;
 }
 
-static void rtw89_load_firmware_cb(const struct firmware *firmware, void *context)
+static int rtw89_load_firmware_req(struct rtw89_dev *rtwdev,
+				   struct rtw89_fw_req_info *req,
+				   const char *fw_name, bool nowarn)
 {
-	struct rtw89_fw_info *fw = context;
-	struct rtw89_dev *rtwdev = fw->rtwdev;
-
-	if (!firmware || !firmware->data) {
-		rtw89_err(rtwdev, "failed to request firmware\n");
-		complete_all(&fw->completion);
-		return;
-	}
-
-	fw->firmware = firmware;
-	complete_all(&fw->completion);
-}
-
-int rtw89_load_firmware(struct rtw89_dev *rtwdev)
-{
-	struct rtw89_fw_info *fw = &rtwdev->fw;
-	const char *fw_name = rtwdev->chip->fw_name;
 	int ret;
 
-	fw->rtwdev = rtwdev;
-	init_completion(&fw->completion);
-
-	if (fw->firmware) {
+	if (req->firmware) {
 		rtw89_debug(rtwdev, RTW89_DBG_FW,
 			    "full firmware has been early requested\n");
-		complete_all(&fw->completion);
+		complete_all(&req->completion);
 		return 0;
 	}
 
-	ret = request_firmware_nowait(THIS_MODULE, true, fw_name, rtwdev->dev,
-				      GFP_KERNEL, fw, rtw89_load_firmware_cb);
-	if (ret) {
-		rtw89_err(rtwdev, "failed to async firmware request\n");
-		return ret;
-	}
+	if (nowarn)
+		ret = firmware_request_nowarn(&req->firmware, fw_name, rtwdev->dev);
+	else
+		ret = request_firmware(&req->firmware, fw_name, rtwdev->dev);
 
-	return 0;
+	complete_all(&req->completion);
+
+	return ret;
+}
+
+void rtw89_load_firmware_work(struct work_struct *work)
+{
+	struct rtw89_dev *rtwdev =
+		container_of(work, struct rtw89_dev, load_firmware_work);
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	char fw_name[64];
+
+	rtw89_fw_get_filename(fw_name, sizeof(fw_name),
+			      chip->fw_basename, rtwdev->fw.fw_format);
+
+	rtw89_load_firmware_req(rtwdev, &rtwdev->fw.req, fw_name, false);
 }
 
 void rtw89_unload_firmware(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_fw_info *fw = &rtwdev->fw;
 
-	rtw89_wait_firmware_completion(rtwdev);
+	cancel_work_sync(&rtwdev->load_firmware_work);
 
-	if (fw->firmware) {
-		release_firmware(fw->firmware);
+	if (fw->req.firmware) {
+		release_firmware(fw->req.firmware);
 
 		/* assign NULL back in case rtw89_free_ieee80211_hw()
 		 * try to release the same one again.
 		 */
-		fw->firmware = NULL;
+		fw->req.firmware = NULL;
 	}
 }
 
@@ -1147,9 +1161,18 @@ fail:
 static void __rtw89_fw_h2c_set_tx_path(struct rtw89_dev *rtwdev,
 				       struct sk_buff *skb)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_hal *hal = &rtwdev->hal;
-	u8 ntx_path = hal->antenna_tx ? hal->antenna_tx : RF_B;
-	u8 map_b = hal->antenna_tx == RF_AB ? 1 : 0;
+	u8 ntx_path;
+	u8 map_b;
+
+	if (chip->rf_path_num == 1) {
+		ntx_path = RF_A;
+		map_b = 0;
+	} else {
+		ntx_path = hal->antenna_tx ? hal->antenna_tx : RF_B;
+		map_b = hal->antenna_tx == RF_AB ? 1 : 0;
+	}
 
 	SET_CMC_TBL_NTX_PATH_EN(skb->data, ntx_path);
 	SET_CMC_TBL_PATH_MAP_A(skb->data, 0);
@@ -1732,6 +1755,147 @@ fail:
 	return ret;
 }
 
+int rtw89_fw_h2c_set_bcn_fltr_cfg(struct rtw89_dev *rtwdev,
+				  struct ieee80211_vif *vif,
+				  bool connect)
+{
+	struct rtw89_vif *rtwvif = vif_to_rtwvif_safe(vif);
+	struct ieee80211_bss_conf *bss_conf = vif ? &vif->bss_conf : NULL;
+	struct rtw89_h2c_bcnfltr *h2c;
+	u32 len = sizeof(*h2c);
+	struct sk_buff *skb;
+	int ret;
+
+	if (!RTW89_CHK_FW_FEATURE(BEACON_FILTER, &rtwdev->fw))
+		return -EINVAL;
+
+	if (!rtwvif || !bss_conf || rtwvif->net_type != RTW89_NET_TYPE_INFRA)
+		return -EINVAL;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c bcn filter\n");
+		return -ENOMEM;
+	}
+
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_bcnfltr *)skb->data;
+
+	h2c->w0 = le32_encode_bits(connect, RTW89_H2C_BCNFLTR_W0_MON_RSSI) |
+		  le32_encode_bits(connect, RTW89_H2C_BCNFLTR_W0_MON_BCN) |
+		  le32_encode_bits(connect, RTW89_H2C_BCNFLTR_W0_MON_EN) |
+		  le32_encode_bits(RTW89_BCN_FLTR_OFFLOAD_MODE_DEFAULT,
+				   RTW89_H2C_BCNFLTR_W0_MODE) |
+		  le32_encode_bits(RTW89_BCN_LOSS_CNT, RTW89_H2C_BCNFLTR_W0_BCN_LOSS_CNT) |
+		  le32_encode_bits(bss_conf->cqm_rssi_hyst, RTW89_H2C_BCNFLTR_W0_RSSI_HYST) |
+		  le32_encode_bits(bss_conf->cqm_rssi_thold + MAX_RSSI,
+				   RTW89_H2C_BCNFLTR_W0_RSSI_THRESHOLD) |
+		  le32_encode_bits(rtwvif->mac_id, RTW89_H2C_BCNFLTR_W0_MAC_ID);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC, H2C_CL_MAC_FW_OFLD,
+			      H2C_FUNC_CFG_BCNFLTR, 0, 1, len);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
+int rtw89_fw_h2c_rssi_offload(struct rtw89_dev *rtwdev,
+			      struct rtw89_rx_phy_ppdu *phy_ppdu)
+{
+	struct rtw89_h2c_ofld_rssi *h2c;
+	u32 len = sizeof(*h2c);
+	struct sk_buff *skb;
+	s8 rssi;
+	int ret;
+
+	if (!RTW89_CHK_FW_FEATURE(BEACON_FILTER, &rtwdev->fw))
+		return -EINVAL;
+
+	if (!phy_ppdu)
+		return -EINVAL;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c rssi\n");
+		return -ENOMEM;
+	}
+
+	rssi = phy_ppdu->rssi_avg >> RSSI_FACTOR;
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_ofld_rssi *)skb->data;
+
+	h2c->w0 = le32_encode_bits(phy_ppdu->mac_id, RTW89_H2C_OFLD_RSSI_W0_MACID) |
+		  le32_encode_bits(1, RTW89_H2C_OFLD_RSSI_W0_NUM);
+	h2c->w1 = le32_encode_bits(rssi, RTW89_H2C_OFLD_RSSI_W1_VAL);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC, H2C_CL_MAC_FW_OFLD,
+			      H2C_FUNC_OFLD_RSSI, 0, 1, len);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
+int rtw89_fw_h2c_tp_offload(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
+{
+	struct rtw89_traffic_stats *stats = &rtwvif->stats;
+	struct rtw89_h2c_ofld *h2c;
+	u32 len = sizeof(*h2c);
+	struct sk_buff *skb;
+	int ret;
+
+	if (rtwvif->net_type != RTW89_NET_TYPE_INFRA)
+		return -EINVAL;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c tp\n");
+		return -ENOMEM;
+	}
+
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_ofld *)skb->data;
+
+	h2c->w0 = le32_encode_bits(rtwvif->mac_id, RTW89_H2C_OFLD_W0_MAC_ID) |
+		  le32_encode_bits(stats->tx_throughput, RTW89_H2C_OFLD_W0_TX_TP) |
+		  le32_encode_bits(stats->rx_throughput, RTW89_H2C_OFLD_W0_RX_TP);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC, H2C_CL_MAC_FW_OFLD,
+			      H2C_FUNC_OFLD_TP, 0, 1, len);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
 #define H2C_RA_LEN 16
 int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi)
 {
@@ -1801,8 +1965,6 @@ fail:
 	return ret;
 }
 
-#define H2C_LEN_CXDRVHDR 2
-#define H2C_LEN_CXDRVINFO_INIT (12 + H2C_LEN_CXDRVHDR)
 int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -1810,44 +1972,52 @@ int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_init_info *init_info = &dm->init_info;
 	struct rtw89_btc_module *module = &init_info->module;
 	struct rtw89_btc_ant_info *ant = &module->ant;
+	struct rtw89_h2c_cxinit *h2c;
+	u32 len = sizeof(*h2c);
 	struct sk_buff *skb;
-	u8 *cmd;
 	int ret;
 
-	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_INIT);
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
 	if (!skb) {
 		rtw89_err(rtwdev, "failed to alloc skb for h2c cxdrv_init\n");
 		return -ENOMEM;
 	}
-	skb_put(skb, H2C_LEN_CXDRVINFO_INIT);
-	cmd = skb->data;
-
-	RTW89_SET_FWCMD_CXHDR_TYPE(cmd, CXDRVINFO_INIT);
-	RTW89_SET_FWCMD_CXHDR_LEN(cmd, H2C_LEN_CXDRVINFO_INIT - H2C_LEN_CXDRVHDR);
-
-	RTW89_SET_FWCMD_CXINIT_ANT_TYPE(cmd, ant->type);
-	RTW89_SET_FWCMD_CXINIT_ANT_NUM(cmd, ant->num);
-	RTW89_SET_FWCMD_CXINIT_ANT_ISO(cmd, ant->isolation);
-	RTW89_SET_FWCMD_CXINIT_ANT_POS(cmd, ant->single_pos);
-	RTW89_SET_FWCMD_CXINIT_ANT_DIVERSITY(cmd, ant->diversity);
-
-	RTW89_SET_FWCMD_CXINIT_MOD_RFE(cmd, module->rfe_type);
-	RTW89_SET_FWCMD_CXINIT_MOD_CV(cmd, module->cv);
-	RTW89_SET_FWCMD_CXINIT_MOD_BT_SOLO(cmd, module->bt_solo);
-	RTW89_SET_FWCMD_CXINIT_MOD_BT_POS(cmd, module->bt_pos);
-	RTW89_SET_FWCMD_CXINIT_MOD_SW_TYPE(cmd, module->switch_type);
-
-	RTW89_SET_FWCMD_CXINIT_WL_GCH(cmd, init_info->wl_guard_ch);
-	RTW89_SET_FWCMD_CXINIT_WL_ONLY(cmd, init_info->wl_only);
-	RTW89_SET_FWCMD_CXINIT_WL_INITOK(cmd, init_info->wl_init_ok);
-	RTW89_SET_FWCMD_CXINIT_DBCC_EN(cmd, init_info->dbcc_en);
-	RTW89_SET_FWCMD_CXINIT_CX_OTHER(cmd, init_info->cx_other);
-	RTW89_SET_FWCMD_CXINIT_BT_ONLY(cmd, init_info->bt_only);
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_cxinit *)skb->data;
+
+	h2c->hdr.type = CXDRVINFO_INIT;
+	h2c->hdr.len = len - H2C_LEN_CXDRVHDR;
+
+	h2c->ant_type = ant->type;
+	h2c->ant_num = ant->num;
+	h2c->ant_iso = ant->isolation;
+	h2c->ant_info =
+		u8_encode_bits(ant->single_pos, RTW89_H2C_CXINIT_ANT_INFO_POS) |
+		u8_encode_bits(ant->diversity, RTW89_H2C_CXINIT_ANT_INFO_DIVERSITY) |
+		u8_encode_bits(ant->btg_pos, RTW89_H2C_CXINIT_ANT_INFO_BTG_POS) |
+		u8_encode_bits(ant->stream_cnt, RTW89_H2C_CXINIT_ANT_INFO_STREAM_CNT);
+
+	h2c->mod_rfe = module->rfe_type;
+	h2c->mod_cv = module->cv;
+	h2c->mod_info =
+		u8_encode_bits(module->bt_solo, RTW89_H2C_CXINIT_MOD_INFO_BT_SOLO) |
+		u8_encode_bits(module->bt_pos, RTW89_H2C_CXINIT_MOD_INFO_BT_POS) |
+		u8_encode_bits(module->switch_type, RTW89_H2C_CXINIT_MOD_INFO_SW_TYPE) |
+		u8_encode_bits(module->wa_type, RTW89_H2C_CXINIT_MOD_INFO_WA_TYPE);
+	h2c->mod_adie_kt = module->kt_ver_adie;
+	h2c->wl_gch = init_info->wl_guard_ch;
+
+	h2c->info =
+		u8_encode_bits(init_info->wl_only, RTW89_H2C_CXINIT_INFO_WL_ONLY) |
+		u8_encode_bits(init_info->wl_init_ok, RTW89_H2C_CXINIT_INFO_WL_INITOK) |
+		u8_encode_bits(init_info->dbcc_en, RTW89_H2C_CXINIT_INFO_DBCC_EN) |
+		u8_encode_bits(init_info->cx_other, RTW89_H2C_CXINIT_INFO_CX_OTHER) |
+		u8_encode_bits(init_info->bt_only, RTW89_H2C_CXINIT_INFO_BT_ONLY);
 
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_OUTSRC, BTFC_SET,
 			      SET_DRV_INFO, 0, 0,
-			      H2C_LEN_CXDRVINFO_INIT);
+			      len);
 
 	ret = rtw89_h2c_tx(rtwdev, skb, false);
 	if (ret) {
@@ -2035,6 +2205,92 @@ fail:
 	return ret;
 }
 
+#define H2C_LEN_CXDRVINFO_ROLE_SIZE_V2(max_role_num) \
+	(4 + 8 * (max_role_num) + H2C_LEN_CXDRVINFO_ROLE_DBCC_LEN + H2C_LEN_CXDRVHDR)
+
+int rtw89_fw_h2c_cxdrv_role_v2(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	const struct rtw89_btc_ver *ver = btc->ver;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_wl_role_info_v2 *role_info = &wl->role_info_v2;
+	struct rtw89_btc_wl_role_info_bpos *bpos = &role_info->role_map.role;
+	struct rtw89_btc_wl_active_role_v2 *active = role_info->active_role_v2;
+	struct sk_buff *skb;
+	u32 len;
+	u8 *cmd, offset;
+	int ret;
+	int i;
+
+	len = H2C_LEN_CXDRVINFO_ROLE_SIZE_V2(ver->max_role_num);
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c cxdrv_role\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, len);
+	cmd = skb->data;
+
+	RTW89_SET_FWCMD_CXHDR_TYPE(cmd, CXDRVINFO_ROLE);
+	RTW89_SET_FWCMD_CXHDR_LEN(cmd, len - H2C_LEN_CXDRVHDR);
+
+	RTW89_SET_FWCMD_CXROLE_CONNECT_CNT(cmd, role_info->connect_cnt);
+	RTW89_SET_FWCMD_CXROLE_LINK_MODE(cmd, role_info->link_mode);
+
+	RTW89_SET_FWCMD_CXROLE_ROLE_NONE(cmd, bpos->none);
+	RTW89_SET_FWCMD_CXROLE_ROLE_STA(cmd, bpos->station);
+	RTW89_SET_FWCMD_CXROLE_ROLE_AP(cmd, bpos->ap);
+	RTW89_SET_FWCMD_CXROLE_ROLE_VAP(cmd, bpos->vap);
+	RTW89_SET_FWCMD_CXROLE_ROLE_ADHOC(cmd, bpos->adhoc);
+	RTW89_SET_FWCMD_CXROLE_ROLE_ADHOC_MASTER(cmd, bpos->adhoc_master);
+	RTW89_SET_FWCMD_CXROLE_ROLE_MESH(cmd, bpos->mesh);
+	RTW89_SET_FWCMD_CXROLE_ROLE_MONITOR(cmd, bpos->moniter);
+	RTW89_SET_FWCMD_CXROLE_ROLE_P2P_DEV(cmd, bpos->p2p_device);
+	RTW89_SET_FWCMD_CXROLE_ROLE_P2P_GC(cmd, bpos->p2p_gc);
+	RTW89_SET_FWCMD_CXROLE_ROLE_P2P_GO(cmd, bpos->p2p_go);
+	RTW89_SET_FWCMD_CXROLE_ROLE_NAN(cmd, bpos->nan);
+
+	offset = PORT_DATA_OFFSET;
+	for (i = 0; i < RTW89_PORT_NUM; i++, active++) {
+		RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED_V2(cmd, active->connected, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_PID_V2(cmd, active->pid, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_PHY_V2(cmd, active->phy, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_NOA_V2(cmd, active->noa, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_BAND_V2(cmd, active->band, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS_V2(cmd, active->client_ps, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_BW_V2(cmd, active->bw, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_ROLE_V2(cmd, active->role, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_CH_V2(cmd, active->ch, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_NOA_DUR_V2(cmd, active->noa_duration, i, offset);
+	}
+
+	offset = len - H2C_LEN_CXDRVINFO_ROLE_DBCC_LEN;
+	RTW89_SET_FWCMD_CXROLE_MROLE_TYPE(cmd, role_info->mrole_type, offset);
+	RTW89_SET_FWCMD_CXROLE_MROLE_NOA(cmd, role_info->mrole_noa_duration, offset);
+	RTW89_SET_FWCMD_CXROLE_DBCC_EN(cmd, role_info->dbcc_en, offset);
+	RTW89_SET_FWCMD_CXROLE_DBCC_CHG(cmd, role_info->dbcc_chg, offset);
+	RTW89_SET_FWCMD_CXROLE_DBCC_2G_PHY(cmd, role_info->dbcc_2g_phy, offset);
+	RTW89_SET_FWCMD_CXROLE_LINK_MODE_CHG(cmd, role_info->link_mode_chg, offset);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_OUTSRC, BTFC_SET,
+			      SET_DRV_INFO, 0, 0,
+			      len);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
 #define H2C_LEN_CXDRVINFO_CTRL (4 + H2C_LEN_CXDRVHDR)
 int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev)
 {
@@ -2080,6 +2336,62 @@ fail:
 	return ret;
 }
 
+#define H2C_LEN_CXDRVINFO_TRX (28 + H2C_LEN_CXDRVHDR)
+int rtw89_fw_h2c_cxdrv_trx(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_trx_info *trx = &btc->dm.trx_info;
+	struct sk_buff *skb;
+	u8 *cmd;
+	int ret;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_TRX);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c cxdrv_trx\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, H2C_LEN_CXDRVINFO_TRX);
+	cmd = skb->data;
+
+	RTW89_SET_FWCMD_CXHDR_TYPE(cmd, CXDRVINFO_TRX);
+	RTW89_SET_FWCMD_CXHDR_LEN(cmd, H2C_LEN_CXDRVINFO_TRX - H2C_LEN_CXDRVHDR);
+
+	RTW89_SET_FWCMD_CXTRX_TXLV(cmd, trx->tx_lvl);
+	RTW89_SET_FWCMD_CXTRX_RXLV(cmd, trx->rx_lvl);
+	RTW89_SET_FWCMD_CXTRX_WLRSSI(cmd, trx->wl_rssi);
+	RTW89_SET_FWCMD_CXTRX_BTRSSI(cmd, trx->bt_rssi);
+	RTW89_SET_FWCMD_CXTRX_TXPWR(cmd, trx->tx_power);
+	RTW89_SET_FWCMD_CXTRX_RXGAIN(cmd, trx->rx_gain);
+	RTW89_SET_FWCMD_CXTRX_BTTXPWR(cmd, trx->bt_tx_power);
+	RTW89_SET_FWCMD_CXTRX_BTRXGAIN(cmd, trx->bt_rx_gain);
+	RTW89_SET_FWCMD_CXTRX_CN(cmd, trx->cn);
+	RTW89_SET_FWCMD_CXTRX_NHM(cmd, trx->nhm);
+	RTW89_SET_FWCMD_CXTRX_BTPROFILE(cmd, trx->bt_profile);
+	RTW89_SET_FWCMD_CXTRX_RSVD2(cmd, trx->rsvd2);
+	RTW89_SET_FWCMD_CXTRX_TXRATE(cmd, trx->tx_rate);
+	RTW89_SET_FWCMD_CXTRX_RXRATE(cmd, trx->rx_rate);
+	RTW89_SET_FWCMD_CXTRX_TXTP(cmd, trx->tx_tp);
+	RTW89_SET_FWCMD_CXTRX_RXTP(cmd, trx->rx_tp);
+	RTW89_SET_FWCMD_CXTRX_RXERRRA(cmd, trx->rx_err_ratio);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_OUTSRC, BTFC_SET,
+			      SET_DRV_INFO, 0, 0,
+			      H2C_LEN_CXDRVINFO_TRX);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
 #define H2C_LEN_CXDRVINFO_RFK (4 + H2C_LEN_CXDRVHDR)
 int rtw89_fw_h2c_cxdrv_rfk(struct rtw89_dev *rtwdev)
 {
@@ -2275,46 +2587,51 @@ fail:
 	return ret;
 }
 
-#define H2C_LEN_SCAN_OFFLOAD 28
 int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev,
 			      struct rtw89_scan_option *option,
 			      struct rtw89_vif *rtwvif)
 {
-	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	struct rtw89_chan *op = &rtwdev->scan_info.op_chan;
+	struct rtw89_h2c_scanofld *h2c;
+	u32 len = sizeof(*h2c);
 	struct sk_buff *skb;
-	u8 *cmd;
 	int ret;
 
-	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_SCAN_OFFLOAD);
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
 	if (!skb) {
 		rtw89_err(rtwdev, "failed to alloc skb for h2c scan offload\n");
 		return -ENOMEM;
 	}
-	skb_put(skb, H2C_LEN_SCAN_OFFLOAD);
-	cmd = skb->data;
+	skb_put(skb, len);
+	h2c = (struct rtw89_h2c_scanofld *)skb->data;
+
+	h2c->w0 = le32_encode_bits(rtwvif->mac_id, RTW89_H2C_SCANOFLD_W0_MACID) |
+		  le32_encode_bits(rtwvif->port, RTW89_H2C_SCANOFLD_W0_PORT_ID) |
+		  le32_encode_bits(RTW89_PHY_0, RTW89_H2C_SCANOFLD_W0_BAND) |
+		  le32_encode_bits(option->enable, RTW89_H2C_SCANOFLD_W0_OPERATION);
+
+	h2c->w1 = le32_encode_bits(true, RTW89_H2C_SCANOFLD_W1_NOTIFY_END) |
+		  le32_encode_bits(option->target_ch_mode,
+				   RTW89_H2C_SCANOFLD_W1_TARGET_CH_MODE) |
+		  le32_encode_bits(RTW89_SCAN_IMMEDIATE,
+				   RTW89_H2C_SCANOFLD_W1_START_MODE) |
+		  le32_encode_bits(RTW89_SCAN_ONCE, RTW89_H2C_SCANOFLD_W1_SCAN_TYPE);
 
-	RTW89_SET_FWCMD_SCANOFLD_MACID(cmd, rtwvif->mac_id);
-	RTW89_SET_FWCMD_SCANOFLD_PORT_ID(cmd, rtwvif->port);
-	RTW89_SET_FWCMD_SCANOFLD_BAND(cmd, RTW89_PHY_0);
-	RTW89_SET_FWCMD_SCANOFLD_OPERATION(cmd, option->enable);
-	RTW89_SET_FWCMD_SCANOFLD_NOTIFY_END(cmd, true);
-	RTW89_SET_FWCMD_SCANOFLD_TARGET_CH_MODE(cmd, option->target_ch_mode);
-	RTW89_SET_FWCMD_SCANOFLD_START_MODE(cmd, RTW89_SCAN_IMMEDIATE);
-	RTW89_SET_FWCMD_SCANOFLD_SCAN_TYPE(cmd, RTW89_SCAN_ONCE);
 	if (option->target_ch_mode) {
-		RTW89_SET_FWCMD_SCANOFLD_TARGET_CH_BW(cmd, scan_info->op_bw);
-		RTW89_SET_FWCMD_SCANOFLD_TARGET_PRI_CH(cmd,
-						       scan_info->op_pri_ch);
-		RTW89_SET_FWCMD_SCANOFLD_TARGET_CENTRAL_CH(cmd,
-							   scan_info->op_chan);
-		RTW89_SET_FWCMD_SCANOFLD_TARGET_CH_BAND(cmd,
-							scan_info->op_band);
+		h2c->w1 |= le32_encode_bits(op->band_width,
+					    RTW89_H2C_SCANOFLD_W1_TARGET_CH_BW) |
+			   le32_encode_bits(op->primary_channel,
+					    RTW89_H2C_SCANOFLD_W1_TARGET_PRI_CH) |
+			   le32_encode_bits(op->channel,
+					    RTW89_H2C_SCANOFLD_W1_TARGET_CENTRAL_CH);
+		h2c->w0 |= le32_encode_bits(op->band_type,
+					    RTW89_H2C_SCANOFLD_W0_TARGET_CH_BAND);
 	}
 
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_MAC, H2C_CL_MAC_FW_OFLD,
 			      H2C_FUNC_SCANOFLD, 1, 1,
-			      H2C_LEN_SCAN_OFFLOAD);
+			      len);
 
 	ret = rtw89_h2c_tx(rtwdev, skb, false);
 	if (ret) {
@@ -2579,6 +2896,7 @@ static int rtw89_fw_write_h2c_reg(struct rtw89_dev *rtwdev,
 				  struct rtw89_mac_h2c_info *info)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_fw_info *fw_info = &rtwdev->fw;
 	const u32 *h2c_reg = chip->h2c_regs;
 	u8 i, val, len;
 	int ret;
@@ -2598,6 +2916,9 @@ static int rtw89_fw_write_h2c_reg(struct rtw89_dev *rtwdev,
 	for (i = 0; i < RTW89_H2CREG_MAX; i++)
 		rtw89_write32(rtwdev, h2c_reg[i], info->h2creg[i]);
 
+	fw_info->h2c_counter++;
+	rtw89_write8_mask(rtwdev, chip->h2c_counter_reg.addr,
+			  chip->h2c_counter_reg.mask, fw_info->h2c_counter);
 	rtw89_write8(rtwdev, chip->h2c_ctrl_reg, B_AX_H2CREG_TRIGGER);
 
 	return 0;
@@ -2607,6 +2928,7 @@ static int rtw89_fw_read_c2h_reg(struct rtw89_dev *rtwdev,
 				 struct rtw89_mac_c2h_info *info)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_fw_info *fw_info = &rtwdev->fw;
 	const u32 *c2h_reg = chip->c2h_regs;
 	u32 ret;
 	u8 i, val;
@@ -2630,6 +2952,10 @@ static int rtw89_fw_read_c2h_reg(struct rtw89_dev *rtwdev,
 	info->content_len = (RTW89_GET_C2H_HDR_LEN(*info->c2hreg) << 2) -
 				RTW89_C2HREG_HDR_LEN;
 
+	fw_info->c2h_counter++;
+	rtw89_write8_mask(rtwdev, chip->c2h_counter_reg.addr,
+			  chip->c2h_counter_reg.mask, fw_info->c2h_counter);
+
 	return 0;
 }
 
@@ -2702,9 +3028,29 @@ static void rtw89_release_pkt_list(struct rtw89_dev *rtwdev)
 	}
 }
 
+static bool rtw89_is_6ghz_wildcard_probe_req(struct rtw89_dev *rtwdev,
+					     struct rtw89_vif *rtwvif,
+					     struct rtw89_pktofld_info *info,
+					     enum nl80211_band band, u8 ssid_idx)
+{
+	struct cfg80211_scan_request *req = rtwvif->scan_req;
+
+	if (band != NL80211_BAND_6GHZ)
+		return false;
+
+	if (req->ssids[ssid_idx].ssid_len) {
+		memcpy(info->ssid, req->ssids[ssid_idx].ssid,
+		       req->ssids[ssid_idx].ssid_len);
+		info->ssid_len = req->ssids[ssid_idx].ssid_len;
+		return false;
+	} else {
+		return true;
+	}
+}
+
 static int rtw89_append_probe_req_ie(struct rtw89_dev *rtwdev,
 				     struct rtw89_vif *rtwvif,
-				     struct sk_buff *skb)
+				     struct sk_buff *skb, u8 ssid_idx)
 {
 	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
 	struct ieee80211_scan_ies *ies = rtwvif->scan_ies;
@@ -2732,6 +3078,13 @@ static int rtw89_append_probe_req_ie(struct rtw89_dev *rtwdev,
 			goto out;
 		}
 
+		if (rtw89_is_6ghz_wildcard_probe_req(rtwdev, rtwvif, info, band,
+						     ssid_idx)) {
+			kfree_skb(new);
+			kfree(info);
+			goto out;
+		}
+
 		ret = rtw89_fw_h2c_add_pkt_offload(rtwdev, &info->id, new);
 		if (ret) {
 			kfree_skb(new);
@@ -2762,7 +3115,7 @@ static int rtw89_hw_scan_update_probe_req(struct rtw89_dev *rtwdev,
 		if (!skb)
 			return -ENOMEM;
 
-		ret = rtw89_append_probe_req_ie(rtwdev, rtwvif, skb);
+		ret = rtw89_append_probe_req_ie(rtwdev, rtwvif, skb, i);
 		kfree_skb(skb);
 
 		if (ret)
@@ -2772,6 +3125,77 @@ static int rtw89_hw_scan_update_probe_req(struct rtw89_dev *rtwdev,
 	return 0;
 }
 
+static int rtw89_update_6ghz_rnr_chan(struct rtw89_dev *rtwdev,
+				      struct cfg80211_scan_request *req,
+				      struct rtw89_mac_chinfo *ch_info)
+{
+	struct ieee80211_vif *vif = rtwdev->scan_info.scanning_vif;
+	struct list_head *pkt_list = rtwdev->scan_info.pkt_list;
+	struct rtw89_vif *rtwvif = vif_to_rtwvif_safe(vif);
+	struct ieee80211_scan_ies *ies = rtwvif->scan_ies;
+	struct cfg80211_scan_6ghz_params *params;
+	struct rtw89_pktofld_info *info, *tmp;
+	struct ieee80211_hdr *hdr;
+	struct sk_buff *skb;
+	bool found;
+	int ret = 0;
+	u8 i;
+
+	if (!req->n_6ghz_params)
+		return 0;
+
+	for (i = 0; i < req->n_6ghz_params; i++) {
+		params = &req->scan_6ghz_params[i];
+
+		if (req->channels[params->channel_idx]->hw_value !=
+		    ch_info->pri_ch)
+			continue;
+
+		found = false;
+		list_for_each_entry(tmp, &pkt_list[NL80211_BAND_6GHZ], list) {
+			if (ether_addr_equal(tmp->bssid, params->bssid)) {
+				found = true;
+				break;
+			}
+		}
+		if (found)
+			continue;
+
+		skb = ieee80211_probereq_get(rtwdev->hw, rtwvif->mac_addr,
+					     NULL, 0, req->ie_len);
+		skb_put_data(skb, ies->ies[NL80211_BAND_6GHZ], ies->len[NL80211_BAND_6GHZ]);
+		skb_put_data(skb, ies->common_ies, ies->common_ie_len);
+		hdr = (struct ieee80211_hdr *)skb->data;
+		ether_addr_copy(hdr->addr3, params->bssid);
+
+		info = kzalloc(sizeof(*info), GFP_KERNEL);
+		if (!info) {
+			ret = -ENOMEM;
+			kfree_skb(skb);
+			goto out;
+		}
+
+		ret = rtw89_fw_h2c_add_pkt_offload(rtwdev, &info->id, skb);
+		if (ret) {
+			kfree_skb(skb);
+			kfree(info);
+			goto out;
+		}
+
+		ether_addr_copy(info->bssid, params->bssid);
+		info->channel_6ghz = req->channels[params->channel_idx]->hw_value;
+		list_add_tail(&info->list, &rtwdev->scan_info.pkt_list[NL80211_BAND_6GHZ]);
+
+		ch_info->tx_pkt = true;
+		ch_info->period = RTW89_CHANNEL_TIME_6G + RTW89_DWELL_TIME_6G;
+
+		kfree_skb(skb);
+	}
+
+out:
+	return ret;
+}
+
 static void rtw89_hw_scan_add_chan(struct rtw89_dev *rtwdev, int chan_type,
 				   int ssid_num,
 				   struct rtw89_mac_chinfo *ch_info)
@@ -2780,8 +3204,10 @@ static void rtw89_hw_scan_add_chan(struct rtw89_dev *rtwdev, int chan_type,
 	struct ieee80211_vif *vif = rtwdev->scan_info.scanning_vif;
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	struct cfg80211_scan_request *req = rtwvif->scan_req;
+	struct rtw89_chan *op = &rtwdev->scan_info.op_chan;
 	struct rtw89_pktofld_info *info;
 	u8 band, probe_count = 0;
+	int ret;
 
 	ch_info->notify_action = RTW89_SCANOFLD_DEBUG_MASK;
 	ch_info->dfs_ch = chan_type == RTW89_CHAN_DFS;
@@ -2793,33 +3219,39 @@ static void rtw89_hw_scan_add_chan(struct rtw89_dev *rtwdev, int chan_type,
 	ch_info->pause_data = false;
 	ch_info->probe_id = RTW89_SCANOFLD_PKT_NONE;
 
+	if (ch_info->ch_band == RTW89_BAND_6G) {
+		if ((ssid_num == 1 && req->ssids[0].ssid_len == 0) ||
+		    !ch_info->is_psc) {
+			ch_info->tx_pkt = false;
+			if (!req->duration_mandatory)
+				ch_info->period -= RTW89_DWELL_TIME_6G;
+		}
+	}
+
+	ret = rtw89_update_6ghz_rnr_chan(rtwdev, req, ch_info);
+	if (ret)
+		rtw89_warn(rtwdev, "RNR fails: %d\n", ret);
+
 	if (ssid_num) {
-		ch_info->num_pkt = ssid_num;
 		band = rtw89_hw_to_nl80211_band(ch_info->ch_band);
 
 		list_for_each_entry(info, &scan_info->pkt_list[band], list) {
-			ch_info->pkt_id[probe_count] = info->id;
-			if (++probe_count >= ssid_num)
+			if (info->channel_6ghz &&
+			    ch_info->pri_ch != info->channel_6ghz)
+				continue;
+			ch_info->pkt_id[probe_count++] = info->id;
+			if (probe_count >= RTW89_SCANOFLD_MAX_SSID)
 				break;
 		}
-		if (probe_count != ssid_num)
-			rtw89_err(rtwdev, "SSID num differs from list len\n");
-	}
-
-	if (ch_info->ch_band == RTW89_BAND_6G) {
-		if (ssid_num == 1 && req->ssids[0].ssid_len == 0) {
-			ch_info->tx_pkt = false;
-			if (!req->duration_mandatory)
-				ch_info->period -= RTW89_DWELL_TIME_6G;
-		}
+		ch_info->num_pkt = probe_count;
 	}
 
 	switch (chan_type) {
 	case RTW89_CHAN_OPERATE:
-		ch_info->central_ch = scan_info->op_chan;
-		ch_info->pri_ch = scan_info->op_pri_ch;
-		ch_info->ch_band = scan_info->op_band;
-		ch_info->bw = scan_info->op_bw;
+		ch_info->central_ch = op->channel;
+		ch_info->pri_ch = op->primary_channel;
+		ch_info->ch_band = op->band_type;
+		ch_info->bw = op->band_width;
 		ch_info->tx_null = true;
 		ch_info->num_pkt = 0;
 		break;
@@ -2837,7 +3269,7 @@ static void rtw89_hw_scan_add_chan(struct rtw89_dev *rtwdev, int chan_type,
 }
 
 static int rtw89_hw_scan_add_chan_list(struct rtw89_dev *rtwdev,
-				       struct rtw89_vif *rtwvif)
+				       struct rtw89_vif *rtwvif, bool connected)
 {
 	struct cfg80211_scan_request *req = rtwvif->scan_req;
 	struct rtw89_mac_chinfo	*ch_info, *tmp;
@@ -2872,6 +3304,7 @@ static int rtw89_hw_scan_add_chan_list(struct rtw89_dev *rtwdev,
 		ch_info->central_ch = channel->hw_value;
 		ch_info->pri_ch = channel->hw_value;
 		ch_info->rand_seq_num = random_seq;
+		ch_info->is_psc = cfg80211_channel_is_psc(channel);
 
 		if (channel->flags &
 		    (IEEE80211_CHAN_RADAR | IEEE80211_CHAN_NO_IR))
@@ -2880,7 +3313,7 @@ static int rtw89_hw_scan_add_chan_list(struct rtw89_dev *rtwdev,
 			type = RTW89_CHAN_ACTIVE;
 		rtw89_hw_scan_add_chan(rtwdev, type, req->n_ssids, ch_info);
 
-		if (rtwvif->net_type != RTW89_NET_TYPE_NO_LINK &&
+		if (connected &&
 		    off_chan_time + ch_info->period > RTW89_OFF_CHAN_TIME) {
 			tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);
 			if (!tmp) {
@@ -2913,7 +3346,7 @@ out:
 }
 
 static int rtw89_hw_scan_prehandle(struct rtw89_dev *rtwdev,
-				   struct rtw89_vif *rtwvif)
+				   struct rtw89_vif *rtwvif, bool connected)
 {
 	int ret;
 
@@ -2922,7 +3355,7 @@ static int rtw89_hw_scan_prehandle(struct rtw89_dev *rtwdev,
 		rtw89_err(rtwdev, "Update probe request failed\n");
 		goto out;
 	}
-	ret = rtw89_hw_scan_add_chan_list(rtwdev, rtwvif);
+	ret = rtw89_hw_scan_add_chan_list(rtwdev, rtwvif, connected);
 out:
 	return ret;
 }
@@ -2935,6 +3368,7 @@ void rtw89_hw_scan_start(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 	u32 rx_fltr = rtwdev->hal.rx_fltr;
 	u8 mac_addr[ETH_ALEN];
 
+	rtw89_get_channel(rtwdev, rtwvif, &rtwdev->scan_info.op_chan);
 	rtwdev->scan_info.scanning_vif = vif;
 	rtwdev->scan_info.last_chan_idx = 0;
 	rtwvif->scan_ies = &scan_req->ies;
@@ -2960,6 +3394,7 @@ void rtw89_hw_scan_start(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 			    bool aborted)
 {
+	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
 	struct cfg80211_scan_info info = {
 		.aborted = aborted,
 	};
@@ -2981,11 +3416,9 @@ void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 	rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	rtwvif->scan_req = NULL;
 	rtwvif->scan_ies = NULL;
-	rtwdev->scan_info.last_chan_idx = 0;
-	rtwdev->scan_info.scanning_vif = NULL;
+	scan_info->last_chan_idx = 0;
+	scan_info->scanning_vif = NULL;
 
-	if (rtwvif->net_type != RTW89_NET_TYPE_NO_LINK)
-		rtw89_store_op_chan(rtwdev, false);
 	rtw89_set_channel(rtwdev);
 }
 
@@ -3000,16 +3433,19 @@ int rtw89_hw_scan_offload(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 {
 	struct rtw89_scan_option opt = {0};
 	struct rtw89_vif *rtwvif;
+	bool connected;
 	int ret = 0;
 
 	rtwvif = vif ? (struct rtw89_vif *)vif->drv_priv : NULL;
 	if (!rtwvif)
 		return -EINVAL;
 
+	/* This variable implies connected or during attempt to connect */
+	connected = !is_zero_ether_addr(rtwvif->bssid);
 	opt.enable = enable;
-	opt.target_ch_mode = rtwvif->net_type != RTW89_NET_TYPE_NO_LINK;
+	opt.target_ch_mode = connected;
 	if (enable) {
-		ret = rtw89_hw_scan_prehandle(rtwdev, rtwvif);
+		ret = rtw89_hw_scan_prehandle(rtwdev, rtwvif, connected);
 		if (ret)
 			goto out;
 	}
@@ -3018,24 +3454,6 @@ out:
 	return ret;
 }
 
-void rtw89_store_op_chan(struct rtw89_dev *rtwdev, bool backup)
-{
-	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
-	const struct rtw89_chan *cur = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
-	struct rtw89_chan new;
-
-	if (backup) {
-		scan_info->op_pri_ch = cur->primary_channel;
-		scan_info->op_chan = cur->channel;
-		scan_info->op_bw = cur->band_width;
-		scan_info->op_band = cur->band_type;
-	} else {
-		rtw89_chan_create(&new, scan_info->op_chan, scan_info->op_pri_ch,
-				  scan_info->op_band, scan_info->op_bw);
-		rtw89_assign_entity_chan(rtwdev, RTW89_SUB_ENTITY_0, &new);
-	}
-}
-
 #define H2C_FW_CPU_EXCEPTION_LEN 4
 #define H2C_FW_CPU_EXCEPTION_TYPE_DEF 0x5566
 int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev)
diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h
index cae07e325326..675f85c41471 100644
--- a/drivers/net/wireless/realtek/rtw89/fw.h
+++ b/drivers/net/wireless/realtek/rtw89/fw.h
@@ -162,6 +162,27 @@ enum rtw89_p2pps_action {
 	RTW89_P2P_ACT_TERMINATE = 3,
 };
 
+enum rtw89_bcn_fltr_offload_mode {
+	RTW89_BCN_FLTR_OFFLOAD_MODE_0 = 0,
+	RTW89_BCN_FLTR_OFFLOAD_MODE_1,
+	RTW89_BCN_FLTR_OFFLOAD_MODE_2,
+	RTW89_BCN_FLTR_OFFLOAD_MODE_3,
+
+	RTW89_BCN_FLTR_OFFLOAD_MODE_DEFAULT = RTW89_BCN_FLTR_OFFLOAD_MODE_0,
+};
+
+enum rtw89_bcn_fltr_type {
+	RTW89_BCN_FLTR_BEACON_LOSS,
+	RTW89_BCN_FLTR_RSSI,
+	RTW89_BCN_FLTR_NOTIFY,
+};
+
+enum rtw89_bcn_fltr_rssi_event {
+	RTW89_BCN_FLTR_RSSI_NOT_CHANGED,
+	RTW89_BCN_FLTR_RSSI_HIGH,
+	RTW89_BCN_FLTR_RSSI_LOW,
+};
+
 #define FWDL_SECTION_MAX_NUM 10
 #define FWDL_SECTION_CHKSUM_LEN	8
 #define FWDL_SECTION_PER_PKT_LEN 2020
@@ -216,6 +237,8 @@ struct rtw89_h2creg_sch_tx_en {
 #define RTW89_SCAN_LIST_LIMIT \
 		((RTW89_H2C_MAX_SIZE / RTW89_MAC_CHINFO_SIZE) - RTW89_SCAN_LIST_GUARD)
 
+#define RTW89_BCN_LOSS_CNT 10
+
 struct rtw89_mac_chinfo {
 	u8 period;
 	u8 dwell_time;
@@ -237,6 +260,7 @@ struct rtw89_mac_chinfo {
 	u16 tx_pwr_idx;
 	u8 rsvd1;
 	struct list_head list;
+	bool is_psc;
 };
 
 struct rtw89_scan_option {
@@ -247,6 +271,12 @@ struct rtw89_scan_option {
 struct rtw89_pktofld_info {
 	struct list_head list;
 	u8 id;
+
+	/* Below fields are for 6 GHz RNR use only */
+	u8 ssid[IEEE80211_MAX_SSID_LEN];
+	u8 ssid_len;
+	u8 bssid[ETH_ALEN];
+	u16 channel_6ghz;
 };
 
 static inline void RTW89_SET_FWCMD_RA_IS_DIS(void *cmd, u32 val)
@@ -2145,6 +2175,7 @@ enum rtw89_btc_cxdrvinfo {
 	CXDRVINFO_RUN,
 	CXDRVINFO_CTRL,
 	CXDRVINFO_SCAN,
+	CXDRVINFO_TRX,  /* WL traffic to WL fw */
 	CXDRVINFO_MAX,
 };
 
@@ -2166,85 +2197,44 @@ static inline void RTW89_SET_FWCMD_CXHDR_LEN(void *cmd, u8 val)
 	u8p_replace_bits((u8 *)(cmd) + 1, val, GENMASK(7, 0));
 }
 
-static inline void RTW89_SET_FWCMD_CXINIT_ANT_TYPE(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 2, val, GENMASK(7, 0));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_ANT_NUM(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 3, val, GENMASK(7, 0));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_ANT_ISO(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 4, val, GENMASK(7, 0));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_ANT_POS(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 5, val, BIT(0));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_ANT_DIVERSITY(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 5, val, BIT(1));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_MOD_RFE(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 6, val, GENMASK(7, 0));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_MOD_CV(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 7, val, GENMASK(7, 0));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_MOD_BT_SOLO(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 8, val, BIT(0));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_MOD_BT_POS(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 8, val, BIT(1));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_MOD_SW_TYPE(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 8, val, BIT(2));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_WL_GCH(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 10, val, GENMASK(7, 0));
-}
+struct rtw89_h2c_cxhdr {
+	u8 type;
+	u8 len;
+} __packed;
 
-static inline void RTW89_SET_FWCMD_CXINIT_WL_ONLY(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 11, val, BIT(0));
-}
+#define H2C_LEN_CXDRVHDR sizeof(struct rtw89_h2c_cxhdr)
+
+struct rtw89_h2c_cxinit {
+	struct rtw89_h2c_cxhdr hdr;
+	u8 ant_type;
+	u8 ant_num;
+	u8 ant_iso;
+	u8 ant_info;
+	u8 mod_rfe;
+	u8 mod_cv;
+	u8 mod_info;
+	u8 mod_adie_kt;
+	u8 wl_gch;
+	u8 info;
+	u8 rsvd;
+	u8 rsvd1;
+} __packed;
 
-static inline void RTW89_SET_FWCMD_CXINIT_WL_INITOK(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 11, val, BIT(1));
-}
+#define RTW89_H2C_CXINIT_ANT_INFO_POS BIT(0)
+#define RTW89_H2C_CXINIT_ANT_INFO_DIVERSITY BIT(1)
+#define RTW89_H2C_CXINIT_ANT_INFO_BTG_POS GENMASK(3, 2)
+#define RTW89_H2C_CXINIT_ANT_INFO_STREAM_CNT GENMASK(7, 4)
 
-static inline void RTW89_SET_FWCMD_CXINIT_DBCC_EN(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 11, val, BIT(2));
-}
+#define RTW89_H2C_CXINIT_MOD_INFO_BT_SOLO BIT(0)
+#define RTW89_H2C_CXINIT_MOD_INFO_BT_POS BIT(1)
+#define RTW89_H2C_CXINIT_MOD_INFO_SW_TYPE BIT(2)
+#define RTW89_H2C_CXINIT_MOD_INFO_WA_TYPE GENMASK(5, 3)
 
-static inline void RTW89_SET_FWCMD_CXINIT_CX_OTHER(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 11, val, BIT(3));
-}
-
-static inline void RTW89_SET_FWCMD_CXINIT_BT_ONLY(void *cmd, u8 val)
-{
-	u8p_replace_bits((u8 *)(cmd) + 11, val, BIT(4));
-}
+#define RTW89_H2C_CXINIT_INFO_WL_ONLY BIT(0)
+#define RTW89_H2C_CXINIT_INFO_WL_INITOK BIT(1)
+#define RTW89_H2C_CXINIT_INFO_DBCC_EN BIT(2)
+#define RTW89_H2C_CXINIT_INFO_CX_OTHER BIT(3)
+#define RTW89_H2C_CXINIT_INFO_BT_ONLY BIT(4)
 
 static inline void RTW89_SET_FWCMD_CXROLE_CONNECT_CNT(void *cmd, u8 val)
 {
@@ -2386,6 +2376,56 @@ static inline void RTW89_SET_FWCMD_CXROLE_ACT_NOA_DUR(void *cmd, u32 val, int n,
 	le32p_replace_bits((__le32 *)((u8 *)cmd + (20 + (12 + offset) * n)), val, GENMASK(31, 0));
 }
 
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED_V2(void *cmd, u8 val, int n, u8 offset)
+{
+	u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, BIT(0));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_PID_V2(void *cmd, u8 val, int n, u8 offset)
+{
+	u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, GENMASK(3, 1));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_PHY_V2(void *cmd, u8 val, int n, u8 offset)
+{
+	u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, BIT(4));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_NOA_V2(void *cmd, u8 val, int n, u8 offset)
+{
+	u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, BIT(5));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_BAND_V2(void *cmd, u8 val, int n, u8 offset)
+{
+	u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, GENMASK(7, 6));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS_V2(void *cmd, u8 val, int n, u8 offset)
+{
+	u8p_replace_bits((u8 *)cmd + (7 + (12 + offset) * n), val, BIT(0));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_BW_V2(void *cmd, u8 val, int n, u8 offset)
+{
+	u8p_replace_bits((u8 *)cmd + (7 + (12 + offset) * n), val, GENMASK(7, 1));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_ROLE_V2(void *cmd, u8 val, int n, u8 offset)
+{
+	u8p_replace_bits((u8 *)cmd + (8 + (12 + offset) * n), val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_CH_V2(void *cmd, u8 val, int n, u8 offset)
+{
+	u8p_replace_bits((u8 *)cmd + (9 + (12 + offset) * n), val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_NOA_DUR_V2(void *cmd, u32 val, int n, u8 offset)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + (10 + (12 + offset) * n)), val, GENMASK(31, 0));
+}
+
 static inline void RTW89_SET_FWCMD_CXROLE_MROLE_TYPE(void *cmd, u32 val, u8 offset)
 {
 	le32p_replace_bits((__le32 *)((u8 *)cmd + offset), val, GENMASK(31, 0));
@@ -2436,6 +2476,91 @@ static inline void RTW89_SET_FWCMD_CXCTRL_TRACE_STEP(void *cmd, u32 val)
 	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 2), val, GENMASK(18, 3));
 }
 
+static inline void RTW89_SET_FWCMD_CXTRX_TXLV(void *cmd, u8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 2, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_RXLV(void *cmd, u8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 3, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_WLRSSI(void *cmd, u8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 4, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_BTRSSI(void *cmd, u8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 5, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_TXPWR(void *cmd, s8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 6, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_RXGAIN(void *cmd, s8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 7, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_BTTXPWR(void *cmd, s8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 8, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_BTRXGAIN(void *cmd, s8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 9, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_CN(void *cmd, u8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 10, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_NHM(void *cmd, s8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 11, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_BTPROFILE(void *cmd, u8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 12, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_RSVD2(void *cmd, u8 val)
+{
+	u8p_replace_bits((u8 *)cmd + 13, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_TXRATE(void *cmd, u16 val)
+{
+	le16p_replace_bits((__le16 *)((u8 *)cmd + 14), val, GENMASK(15, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_RXRATE(void *cmd, u16 val)
+{
+	le16p_replace_bits((__le16 *)((u8 *)cmd + 16), val, GENMASK(15, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_TXTP(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + 18), val, GENMASK(31, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_RXTP(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + 22), val, GENMASK(31, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXTRX_RXERRRA(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + 26), val, GENMASK(31, 0));
+}
+
 static inline void RTW89_SET_FWCMD_CXRFK_STATE(void *cmd, u32 val)
 {
 	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 2), val, GENMASK(1, 0));
@@ -2606,96 +2731,32 @@ static inline void RTW89_SET_FWCMD_CHINFO_POWER_IDX(void *cmd, u32 val)
 	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 16), val, GENMASK(15, 0));
 }
 
-static inline void RTW89_SET_FWCMD_SCANOFLD_MACID(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd)), val, GENMASK(7, 0));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_NORM_CY(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd)), val, GENMASK(15, 8));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_PORT_ID(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd)), val, GENMASK(18, 16));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_BAND(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd)), val, BIT(19));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_OPERATION(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd)), val, GENMASK(21, 20));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_TARGET_CH_BAND(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd)), val, GENMASK(23, 22));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_NOTIFY_END(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 4), val, BIT(0));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_TARGET_CH_MODE(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 4), val, BIT(1));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_START_MODE(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 4), val, BIT(2));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_SCAN_TYPE(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 4), val, GENMASK(4, 3));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_TARGET_CH_BW(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 4), val, GENMASK(7, 5));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_TARGET_PRI_CH(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 4), val, GENMASK(15, 8));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_TARGET_CENTRAL_CH(void *cmd,
-							      u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 4), val, GENMASK(23, 16));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_PROBE_REQ_PKT_ID(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 4), val, GENMASK(31, 24));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_NORM_PD(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 8), val, GENMASK(15, 0));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_SLOW_PD(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 8), val, GENMASK(23, 16));
-}
-
-static inline void RTW89_SET_FWCMD_SCANOFLD_TSF_HIGH(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 12), val, GENMASK(31, 0));
-}
+struct rtw89_h2c_scanofld {
+	__le32 w0;
+	__le32 w1;
+	__le32 w2;
+	__le32 tsf_high;
+	__le32 tsf_low;
+	__le32 w5;
+	__le32 w6;
+} __packed;
 
-static inline void RTW89_SET_FWCMD_SCANOFLD_TSF_SLOW(void *cmd, u32 val)
-{
-	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 16), val, GENMASK(31, 0));
-}
+#define RTW89_H2C_SCANOFLD_W0_MACID GENMASK(7, 0)
+#define RTW89_H2C_SCANOFLD_W0_NORM_CY GENMASK(15, 8)
+#define RTW89_H2C_SCANOFLD_W0_PORT_ID GENMASK(18, 16)
+#define RTW89_H2C_SCANOFLD_W0_BAND BIT(19)
+#define RTW89_H2C_SCANOFLD_W0_OPERATION GENMASK(21, 20)
+#define RTW89_H2C_SCANOFLD_W0_TARGET_CH_BAND GENMASK(23, 22)
+#define RTW89_H2C_SCANOFLD_W1_NOTIFY_END BIT(0)
+#define RTW89_H2C_SCANOFLD_W1_TARGET_CH_MODE BIT(1)
+#define RTW89_H2C_SCANOFLD_W1_START_MODE BIT(2)
+#define RTW89_H2C_SCANOFLD_W1_SCAN_TYPE GENMASK(4, 3)
+#define RTW89_H2C_SCANOFLD_W1_TARGET_CH_BW GENMASK(7, 5)
+#define RTW89_H2C_SCANOFLD_W1_TARGET_PRI_CH GENMASK(15, 8)
+#define RTW89_H2C_SCANOFLD_W1_TARGET_CENTRAL_CH GENMASK(23, 16)
+#define RTW89_H2C_SCANOFLD_W1_PROBE_REQ_PKT_ID GENMASK(31, 24)
+#define RTW89_H2C_SCANOFLD_W2_NORM_PD GENMASK(15, 0)
+#define RTW89_H2C_SCANOFLD_W2_SLOW_PD GENMASK(23, 16)
 
 static inline void RTW89_SET_FWCMD_P2P_MACID(void *cmd, u32 val)
 {
@@ -3174,6 +3235,17 @@ static inline struct rtw89_fw_c2h_attr *RTW89_SKB_C2H_CB(struct sk_buff *skb)
 #define RTW89_GET_MAC_C2H_REV_ACK_H2C_SEQ(c2h) \
 	le32_get_bits(*((const __le32 *)(c2h) + 2), GENMASK(23, 16))
 
+struct rtw89_c2h_mac_bcnfltr_rpt {
+	__le32 w0;
+	__le32 w1;
+	__le32 w2;
+} __packed;
+
+#define RTW89_C2H_MAC_BCNFLTR_RPT_W2_MACID GENMASK(7, 0)
+#define RTW89_C2H_MAC_BCNFLTR_RPT_W2_TYPE GENMASK(9, 8)
+#define RTW89_C2H_MAC_BCNFLTR_RPT_W2_EVENT GENMASK(11, 10)
+#define RTW89_C2H_MAC_BCNFLTR_RPT_W2_MA GENMASK(23, 16)
+
 #define RTW89_GET_PHY_C2H_RA_RPT_MACID(c2h) \
 	le32_get_bits(*((const __le32 *)(c2h) + 2), GENMASK(15, 0))
 #define RTW89_GET_PHY_C2H_RA_RPT_RETRY_RATIO(c2h) \
@@ -3267,6 +3339,36 @@ static_assert(sizeof(struct rtw89_mac_mcc_tsf_rpt) <= RTW89_COMPLETION_BUF_SIZE)
 #define RTW89_GET_MAC_C2H_MCC_STATUS_RPT_TSF_HIGH(c2h) \
 	le32_get_bits(*((const __le32 *)(c2h) + 4), GENMASK(31, 0))
 
+struct rtw89_h2c_bcnfltr {
+	__le32 w0;
+} __packed;
+
+#define RTW89_H2C_BCNFLTR_W0_MON_RSSI BIT(0)
+#define RTW89_H2C_BCNFLTR_W0_MON_BCN BIT(1)
+#define RTW89_H2C_BCNFLTR_W0_MON_EN BIT(2)
+#define RTW89_H2C_BCNFLTR_W0_MODE GENMASK(4, 3)
+#define RTW89_H2C_BCNFLTR_W0_BCN_LOSS_CNT GENMASK(11, 8)
+#define RTW89_H2C_BCNFLTR_W0_RSSI_HYST GENMASK(15, 12)
+#define RTW89_H2C_BCNFLTR_W0_RSSI_THRESHOLD GENMASK(23, 16)
+#define RTW89_H2C_BCNFLTR_W0_MAC_ID GENMASK(31, 24)
+
+struct rtw89_h2c_ofld_rssi {
+	__le32 w0;
+	__le32 w1;
+} __packed;
+
+#define RTW89_H2C_OFLD_RSSI_W0_MACID GENMASK(7, 0)
+#define RTW89_H2C_OFLD_RSSI_W0_NUM GENMASK(15, 8)
+#define RTW89_H2C_OFLD_RSSI_W1_VAL GENMASK(7, 0)
+
+struct rtw89_h2c_ofld {
+	__le32 w0;
+} __packed;
+
+#define RTW89_H2C_OFLD_W0_MAC_ID GENMASK(7, 0)
+#define RTW89_H2C_OFLD_W0_TX_TP GENMASK(17, 8)
+#define RTW89_H2C_OFLD_W0_RX_TP GENMASK(27, 18)
+
 #define RTW89_FW_HDR_SIZE 32
 #define RTW89_FW_SECTION_HDR_SIZE 16
 
@@ -3316,6 +3418,15 @@ static inline u32 rtw89_compat_fw_hdr_ver_code(const void *fw_buf)
 		return RTW89_FW_HDR_VER_CODE(&compat->fw_hdr);
 }
 
+static inline void rtw89_fw_get_filename(char *buf, size_t size,
+					 const char *fw_basename, int fw_format)
+{
+	if (fw_format <= 0)
+		snprintf(buf, size, "%s.bin", fw_basename);
+	else
+		snprintf(buf, size, "%s-%d.bin", fw_basename, fw_format);
+}
+
 #define RTW89_H2C_RF_PAGE_SIZE 500
 #define RTW89_H2C_RF_PAGE_NUM 3
 struct rtw89_fw_h2c_rf_reg_info {
@@ -3394,6 +3505,9 @@ struct rtw89_fw_h2c_rf_reg_info {
 #define H2C_FUNC_ADD_SCANOFLD_CH	0x16
 #define H2C_FUNC_SCANOFLD		0x17
 #define H2C_FUNC_PKT_DROP		0x1b
+#define H2C_FUNC_CFG_BCNFLTR		0x1e
+#define H2C_FUNC_OFLD_RSSI		0x1f
+#define H2C_FUNC_OFLD_TP		0x20
 
 /* CLASS 10 - Security CAM */
 #define H2C_CL_MAC_SEC_CAM		0xa
@@ -3457,9 +3571,10 @@ int rtw89_fw_recognize(struct rtw89_dev *rtwdev);
 const struct firmware *
 rtw89_early_fw_feature_recognize(struct device *device,
 				 const struct rtw89_chip_info *chip,
-				 u32 *early_feat_map);
+				 struct rtw89_fw_info *early_fw,
+				 int *used_fw_format);
 int rtw89_fw_download(struct rtw89_dev *rtwdev, enum rtw89_fw_type type);
-int rtw89_load_firmware(struct rtw89_dev *rtwdev);
+void rtw89_load_firmware_work(struct work_struct *work);
 void rtw89_unload_firmware(struct rtw89_dev *rtwdev);
 int rtw89_wait_firmware_completion(struct rtw89_dev *rtwdev);
 void rtw89_h2c_pkt_set_hdr(struct rtw89_dev *rtwdev, struct sk_buff *skb,
@@ -3494,11 +3609,19 @@ int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp,
 int rtw89_fw_h2c_set_edca(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 			  u8 ac, u32 val);
 int rtw89_fw_h2c_set_ofld_cfg(struct rtw89_dev *rtwdev);
+int rtw89_fw_h2c_set_bcn_fltr_cfg(struct rtw89_dev *rtwdev,
+				  struct ieee80211_vif *vif,
+				  bool connect);
+int rtw89_fw_h2c_rssi_offload(struct rtw89_dev *rtwdev,
+			      struct rtw89_rx_phy_ppdu *phy_ppdu);
+int rtw89_fw_h2c_tp_offload(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
 int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi);
 int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_cxdrv_role_v1(struct rtw89_dev *rtwdev);
+int rtw89_fw_h2c_cxdrv_role_v2(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev);
+int rtw89_fw_h2c_cxdrv_trx(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_cxdrv_rfk(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_del_pkt_offload(struct rtw89_dev *rtwdev, u8 id);
 int rtw89_fw_h2c_add_pkt_offload(struct rtw89_dev *rtwdev, u8 *id,
@@ -3536,7 +3659,6 @@ int rtw89_fw_msg_reg(struct rtw89_dev *rtwdev,
 		     struct rtw89_mac_c2h_info *c2h_info);
 int rtw89_fw_h2c_fw_log(struct rtw89_dev *rtwdev, bool enable);
 void rtw89_fw_st_dbg_dump(struct rtw89_dev *rtwdev);
-void rtw89_store_op_chan(struct rtw89_dev *rtwdev, bool backup);
 void rtw89_hw_scan_start(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 			 struct ieee80211_scan_request *req);
 void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c
index 2e2a2b6eab09..b8019cfc11b2 100644
--- a/drivers/net/wireless/realtek/rtw89/mac.c
+++ b/drivers/net/wireless/realtek/rtw89/mac.c
@@ -1473,6 +1473,8 @@ const struct rtw89_mac_size_set rtw89_mac_size = {
 	.ple_qt58 = {147, 0, 16, 20, 157, 13, 229, 0, 172, 14, 24, 0,},
 	/* 8852A PCIE WOW */
 	.ple_qt_52a_wow = {264, 0, 32, 20, 64, 13, 1005, 0, 64, 128, 120,},
+	/* 8852B PCIE WOW */
+	.ple_qt_52b_wow = {147, 0, 16, 20, 157, 13, 133, 0, 172, 14, 24, 0,},
 };
 EXPORT_SYMBOL(rtw89_mac_size);
 
@@ -1584,12 +1586,15 @@ static void dle_func_en(struct rtw89_dev *rtwdev, bool enable)
 
 static void dle_clk_en(struct rtw89_dev *rtwdev, bool enable)
 {
-	if (enable)
-		rtw89_write32_set(rtwdev, R_AX_DMAC_CLK_EN,
-				  B_AX_DLE_WDE_CLK_EN | B_AX_DLE_PLE_CLK_EN);
-	else
-		rtw89_write32_clr(rtwdev, R_AX_DMAC_CLK_EN,
-				  B_AX_DLE_WDE_CLK_EN | B_AX_DLE_PLE_CLK_EN);
+	u32 val = B_AX_DLE_WDE_CLK_EN | B_AX_DLE_PLE_CLK_EN;
+
+	if (enable) {
+		if (rtwdev->chip->chip_id == RTL8851B)
+			val |= B_AX_AXIDMA_CLK_EN;
+		rtw89_write32_set(rtwdev, R_AX_DMAC_CLK_EN, val);
+	} else {
+		rtw89_write32_clr(rtwdev, R_AX_DMAC_CLK_EN, val);
+	}
 }
 
 static int dle_mix_cfg(struct rtw89_dev *rtwdev, const struct rtw89_dle_mem *cfg)
@@ -1854,7 +1859,8 @@ static int preload_init(struct rtw89_dev *rtwdev, enum rtw89_mac_idx mac_idx,
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 
-	if (chip->chip_id == RTL8852A || chip->chip_id == RTL8852B || !is_qta_poh(rtwdev))
+	if (chip->chip_id == RTL8852A || chip->chip_id == RTL8852B ||
+	    chip->chip_id == RTL8851B || !is_qta_poh(rtwdev))
 		return 0;
 
 	return preload_init_set(rtwdev, mac_idx, mode);
@@ -1890,7 +1896,8 @@ static void _patch_ss2f_path(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 
-	if (chip->chip_id == RTL8852A || chip->chip_id == RTL8852B)
+	if (chip->chip_id == RTL8852A || chip->chip_id == RTL8852B ||
+	    chip->chip_id == RTL8851B)
 		return;
 
 	rtw89_write32_mask(rtwdev, R_AX_SS2FINFO_PATH, B_AX_SS_DEST_QUEUE_MASK,
@@ -1959,7 +1966,8 @@ static int sec_eng_init(struct rtw89_dev *rtwdev)
 	/* init TX encryption */
 	val |= (B_AX_SEC_TX_ENC | B_AX_SEC_RX_DEC);
 	val |= (B_AX_MC_DEC | B_AX_BC_DEC);
-	if (chip->chip_id == RTL8852A || chip->chip_id == RTL8852B)
+	if (chip->chip_id == RTL8852A || chip->chip_id == RTL8852B ||
+	    chip->chip_id == RTL8851B)
 		val &= ~B_AX_TX_PARTIAL_MODE;
 	rtw89_write32(rtwdev, R_AX_SEC_ENG_CTRL, val);
 
@@ -2065,7 +2073,7 @@ static int scheduler_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 		rtw89_write32_mask(rtwdev, reg, B_AX_SIFS_MACTXEN_T1_MASK,
 				   SIFS_MACTXEN_T1);
 
-	if (rtwdev->chip->chip_id == RTL8852B) {
+	if (rtwdev->chip->chip_id == RTL8852B || rtwdev->chip->chip_id == RTL8851B) {
 		reg = rtw89_mac_reg_by_idx(R_AX_SCH_EXT_CTRL, mac_idx);
 		rtw89_write32_set(rtwdev, reg, B_AX_PORT_RST_TSF_ADV);
 	}
@@ -2805,7 +2813,7 @@ int rtw89_mac_resume_sch_tx_v1(struct rtw89_dev *rtwdev, u8 mac_idx, u32 tx_en)
 }
 EXPORT_SYMBOL(rtw89_mac_resume_sch_tx_v1);
 
-u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd)
+int rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd, u16 *pkt_id)
 {
 	u32 val, reg;
 	int ret;
@@ -2820,9 +2828,13 @@ u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd)
 	ret = read_poll_timeout(rtw89_read32, val, val & B_AX_WD_BUF_STAT_DONE,
 				1, 2000, false, rtwdev, reg);
 	if (ret)
-		return 0xffff;
+		return ret;
+
+	*pkt_id = FIELD_GET(B_AX_WD_BUF_STAT_PKTID_MASK, val);
+	if (*pkt_id == S_WD_BUF_STAT_PKTID_INVALID)
+		return -ENOENT;
 
-	return FIELD_GET(B_AX_WD_BUF_STAT_PKTID_MASK, val);
+	return 0;
 }
 
 int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev,
@@ -2899,10 +2911,10 @@ static int dle_quota_change(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode)
 
 	dle_quota_cfg(rtwdev, cfg, INVALID_QT_WCPU);
 
-	pkt_id = rtw89_mac_dle_buf_req(rtwdev, 0x20, true);
-	if (pkt_id == 0xffff) {
+	ret = rtw89_mac_dle_buf_req(rtwdev, 0x20, true, &pkt_id);
+	if (ret) {
 		rtw89_err(rtwdev, "[ERR]WDE DLE buf req\n");
-		return -ENOMEM;
+		return ret;
 	}
 
 	ctrl_para.cmd_type = CPUIO_OP_CMD_ENQ_TO_HEAD;
@@ -2917,10 +2929,10 @@ static int dle_quota_change(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode)
 		return -EFAULT;
 	}
 
-	pkt_id = rtw89_mac_dle_buf_req(rtwdev, 0x20, false);
-	if (pkt_id == 0xffff) {
+	ret = rtw89_mac_dle_buf_req(rtwdev, 0x20, false, &pkt_id);
+	if (ret) {
 		rtw89_err(rtwdev, "[ERR]PLE DLE buf req\n");
-		return -ENOMEM;
+		return ret;
 	}
 
 	ctrl_para.cmd_type = CPUIO_OP_CMD_ENQ_TO_HEAD;
@@ -3364,8 +3376,15 @@ static int rtw89_mac_trx_init(struct rtw89_dev *rtwdev)
 
 static void rtw89_disable_fw_watchdog(struct rtw89_dev *rtwdev)
 {
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 	u32 val32;
 
+	if (chip_id == RTL8852B || chip_id == RTL8851B) {
+		rtw89_write32_clr(rtwdev, R_AX_PLATFORM_ENABLE, B_AX_APB_WRAP_EN);
+		rtw89_write32_set(rtwdev, R_AX_PLATFORM_ENABLE, B_AX_APB_WRAP_EN);
+		return;
+	}
+
 	rtw89_mac_mem_write(rtwdev, R_AX_WDT_CTRL,
 			    WDT_CTRL_ALL_DIS, RTW89_MAC_MEM_CPU_LOCAL);
 
@@ -3398,6 +3417,8 @@ int rtw89_mac_enable_cpu(struct rtw89_dev *rtwdev, u8 boot_reason, bool dlfw)
 	if (rtw89_read32(rtwdev, R_AX_PLATFORM_ENABLE) & B_AX_WCPU_EN)
 		return -EFAULT;
 
+	rtw89_write32(rtwdev, R_AX_UDM1, 0);
+	rtw89_write32(rtwdev, R_AX_UDM2, 0);
 	rtw89_write32(rtwdev, R_AX_HALT_H2C_CTRL, 0);
 	rtw89_write32(rtwdev, R_AX_HALT_C2H_CTRL, 0);
 	rtw89_write32(rtwdev, R_AX_HALT_H2C, 0);
@@ -3448,7 +3469,10 @@ static int rtw89_mac_dmac_pre_init(struct rtw89_dev *rtwdev)
 		      B_AX_PKT_BUF_EN;
 	rtw89_write32(rtwdev, R_AX_DMAC_FUNC_EN, val);
 
-	val = B_AX_DISPATCHER_CLK_EN;
+	if (chip_id == RTL8851B)
+		val = B_AX_DISPATCHER_CLK_EN | B_AX_AXIDMA_CLK_EN;
+	else
+		val = B_AX_DISPATCHER_CLK_EN;
 	rtw89_write32(rtwdev, R_AX_DMAC_CLK_EN, val);
 
 	if (chip_id != RTL8852C)
@@ -4174,9 +4198,9 @@ rtw89_mac_c2h_macid_pause(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len
 
 static bool rtw89_is_op_chan(struct rtw89_dev *rtwdev, u8 band, u8 channel)
 {
-	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
+	const struct rtw89_chan *op = &rtwdev->scan_info.op_chan;
 
-	return band == scan_info->op_band && channel == scan_info->op_pri_ch;
+	return band == op->band_type && channel == op->primary_channel;
 }
 
 static void
@@ -4191,6 +4215,9 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h,
 	u16 chan;
 	int ret;
 
+	if (!rtwvif)
+		return;
+
 	tx_fail = RTW89_GET_MAC_C2H_SCANOFLD_TX_FAIL(c2h->data);
 	status = RTW89_GET_MAC_C2H_SCANOFLD_STATUS(c2h->data);
 	chan = RTW89_GET_MAC_C2H_SCANOFLD_PRI_CH(c2h->data);
@@ -4223,11 +4250,15 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h,
 		}
 		break;
 	case RTW89_SCAN_ENTER_CH_NOTIFY:
-		rtw89_chan_create(&new, chan, chan, band, RTW89_CHANNEL_WIDTH_20);
-		rtw89_assign_entity_chan(rtwdev, RTW89_SUB_ENTITY_0, &new);
 		if (rtw89_is_op_chan(rtwdev, band, chan)) {
-			rtw89_store_op_chan(rtwdev, false);
+			rtw89_assign_entity_chan(rtwdev, rtwvif->sub_entity_idx,
+						 &rtwdev->scan_info.op_chan);
 			ieee80211_wake_queues(rtwdev->hw);
+		} else {
+			rtw89_chan_create(&new, chan, chan, band,
+					  RTW89_CHANNEL_WIDTH_20);
+			rtw89_assign_entity_chan(rtwdev, rtwvif->sub_entity_idx,
+						 &new);
 		}
 		break;
 	default:
@@ -4236,6 +4267,64 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h,
 }
 
 static void
+rtw89_mac_bcn_fltr_rpt(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
+		       struct sk_buff *skb)
+{
+	struct ieee80211_vif *vif = rtwvif_to_vif_safe(rtwvif);
+	enum nl80211_cqm_rssi_threshold_event nl_event;
+	const struct rtw89_c2h_mac_bcnfltr_rpt *c2h =
+		(const struct rtw89_c2h_mac_bcnfltr_rpt *)skb->data;
+	u8 type, event, mac_id;
+	s8 sig;
+
+	type = le32_get_bits(c2h->w2, RTW89_C2H_MAC_BCNFLTR_RPT_W2_TYPE);
+	sig = le32_get_bits(c2h->w2, RTW89_C2H_MAC_BCNFLTR_RPT_W2_MA) - MAX_RSSI;
+	event = le32_get_bits(c2h->w2, RTW89_C2H_MAC_BCNFLTR_RPT_W2_EVENT);
+	mac_id = le32_get_bits(c2h->w2, RTW89_C2H_MAC_BCNFLTR_RPT_W2_MACID);
+
+	if (mac_id != rtwvif->mac_id)
+		return;
+
+	rtw89_debug(rtwdev, RTW89_DBG_FW,
+		    "C2H bcnfltr rpt macid: %d, type: %d, ma: %d, event: %d\n",
+		    mac_id, type, sig, event);
+
+	switch (type) {
+	case RTW89_BCN_FLTR_BEACON_LOSS:
+		if (!rtwdev->scanning && !rtwvif->offchan)
+			ieee80211_connection_loss(vif);
+		else
+			rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, vif, true);
+		return;
+	case RTW89_BCN_FLTR_NOTIFY:
+		nl_event = NL80211_CQM_RSSI_THRESHOLD_EVENT_HIGH;
+		break;
+	case RTW89_BCN_FLTR_RSSI:
+		if (event == RTW89_BCN_FLTR_RSSI_LOW)
+			nl_event = NL80211_CQM_RSSI_THRESHOLD_EVENT_LOW;
+		else if (event == RTW89_BCN_FLTR_RSSI_HIGH)
+			nl_event = NL80211_CQM_RSSI_THRESHOLD_EVENT_HIGH;
+		else
+			return;
+		break;
+	default:
+		return;
+	}
+
+	ieee80211_cqm_rssi_notify(vif, nl_event, sig, GFP_KERNEL);
+}
+
+static void
+rtw89_mac_c2h_bcn_fltr_rpt(struct rtw89_dev *rtwdev, struct sk_buff *c2h,
+			   u32 len)
+{
+	struct rtw89_vif *rtwvif;
+
+	rtw89_for_each_rtwvif(rtwdev, rtwvif)
+		rtw89_mac_bcn_fltr_rpt(rtwdev, rtwvif, c2h);
+}
+
+static void
 rtw89_mac_c2h_rec_ack(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len)
 {
 	rtw89_debug(rtwdev, RTW89_DBG_FW,
@@ -4455,6 +4544,7 @@ void (* const rtw89_mac_c2h_ofld_handler[])(struct rtw89_dev *rtwdev,
 	[RTW89_MAC_C2H_FUNC_MACID_PAUSE] = rtw89_mac_c2h_macid_pause,
 	[RTW89_MAC_C2H_FUNC_SCANOFLD_RSP] = rtw89_mac_c2h_scanofld_rsp,
 	[RTW89_MAC_C2H_FUNC_TSF32_TOGL_RPT] = rtw89_mac_c2h_tsf32_toggle_rpt,
+	[RTW89_MAC_C2H_FUNC_BCNFLTR_RPT] = rtw89_mac_c2h_bcn_fltr_rpt,
 };
 
 static
@@ -4627,11 +4717,13 @@ int rtw89_mac_coex_init(struct rtw89_dev *rtwdev, const struct rtw89_mac_ax_coex
 	int ret;
 
 	rtw89_write8_set(rtwdev, R_AX_GPIO_MUXCFG, B_AX_ENBT);
-	rtw89_write8_set(rtwdev, R_AX_BTC_FUNC_EN, B_AX_PTA_WL_TX_EN);
+	if (rtwdev->chip->chip_id != RTL8851B)
+		rtw89_write8_set(rtwdev, R_AX_BTC_FUNC_EN, B_AX_PTA_WL_TX_EN);
 	rtw89_write8_set(rtwdev, R_AX_BT_COEX_CFG_2 + 1, B_AX_GNT_BT_POLARITY >> 8);
 	rtw89_write8_set(rtwdev, R_AX_CSR_MODE, B_AX_STATIS_BT_EN | B_AX_WL_ACT_MSK);
 	rtw89_write8_set(rtwdev, R_AX_CSR_MODE + 2, B_AX_BT_CNT_RST >> 16);
-	rtw89_write8_clr(rtwdev, R_AX_TRXPTCL_RESP_0 + 3, B_AX_RSP_CHK_BTCCA >> 24);
+	if (rtwdev->chip->chip_id != RTL8851B)
+		rtw89_write8_clr(rtwdev, R_AX_TRXPTCL_RESP_0 + 3, B_AX_RSP_CHK_BTCCA >> 24);
 
 	val16 = rtw89_read16(rtwdev, R_AX_CCA_CFG_0);
 	val16 = (val16 | B_AX_BTCCA_EN) & ~B_AX_BTCCA_BRK_TXOP_EN;
@@ -4931,6 +5023,24 @@ u16 rtw89_mac_get_plt_cnt(struct rtw89_dev *rtwdev, u8 band)
 	return cnt;
 }
 
+static void rtw89_mac_bfee_standby_timer(struct rtw89_dev *rtwdev, u8 mac_idx,
+					 bool keep)
+{
+	u32 reg;
+
+	rtw89_debug(rtwdev, RTW89_DBG_BF, "set bfee standby_timer to %d\n", keep);
+	reg = rtw89_mac_reg_by_idx(R_AX_BFMEE_RESP_OPTION, mac_idx);
+	if (keep) {
+		set_bit(RTW89_FLAG_BFEE_TIMER_KEEP, rtwdev->flags);
+		rtw89_write32_mask(rtwdev, reg, B_AX_BFMEE_BFRP_RX_STANDBY_TIMER_MASK,
+				   BFRP_RX_STANDBY_TIMER_KEEP);
+	} else {
+		clear_bit(RTW89_FLAG_BFEE_TIMER_KEEP, rtwdev->flags);
+		rtw89_write32_mask(rtwdev, reg, B_AX_BFMEE_BFRP_RX_STANDBY_TIMER_MASK,
+				   BFRP_RX_STANDBY_TIMER_RELEASE);
+	}
+}
+
 static void rtw89_mac_bfee_ctrl(struct rtw89_dev *rtwdev, u8 mac_idx, bool en)
 {
 	u32 reg;
@@ -4967,9 +5077,9 @@ static int rtw89_mac_init_bfee(struct rtw89_dev *rtwdev, u8 mac_idx)
 	rtw89_write32(rtwdev, reg, CSI_RRSC_BMAP);
 
 	reg = rtw89_mac_reg_by_idx(R_AX_BFMEE_RESP_OPTION, mac_idx);
-	val32 = FIELD_PREP(B_AX_BFMEE_BFRP_RX_STANDBY_TIMER_MASK, BFRP_RX_STANDBY_TIMER);
-	val32 |= FIELD_PREP(B_AX_BFMEE_NDP_RX_STANDBY_TIMER_MASK, NDP_RX_STANDBY_TIMER);
+	val32 = FIELD_PREP(B_AX_BFMEE_NDP_RX_STANDBY_TIMER_MASK, NDP_RX_STANDBY_TIMER);
 	rtw89_write32(rtwdev, reg, val32);
+	rtw89_mac_bfee_standby_timer(rtwdev, mac_idx, true);
 	rtw89_mac_bfee_ctrl(rtwdev, mac_idx, true);
 
 	reg = rtw89_mac_reg_by_idx(R_AX_TRXPTCL_RESP_CSI_CTRL_0, mac_idx);
@@ -5181,6 +5291,19 @@ void _rtw89_mac_bf_monitor_track(struct rtw89_dev *rtwdev)
 	struct rtw89_vif *rtwvif;
 	bool en = stats->tx_tfc_lv <= stats->rx_tfc_lv;
 	bool old = test_bit(RTW89_FLAG_BFEE_EN, rtwdev->flags);
+	bool keep_timer = true;
+	bool old_keep_timer;
+
+	old_keep_timer = test_bit(RTW89_FLAG_BFEE_TIMER_KEEP, rtwdev->flags);
+
+	if (stats->tx_tfc_lv <= RTW89_TFC_LOW && stats->rx_tfc_lv <= RTW89_TFC_LOW)
+		keep_timer = false;
+
+	if (keep_timer != old_keep_timer) {
+		rtw89_for_each_rtwvif(rtwdev, rtwvif)
+			rtw89_mac_bfee_standby_timer(rtwdev, rtwvif->mac_idx,
+						     keep_timer);
+	}
 
 	if (en == old)
 		return;
@@ -5426,7 +5549,7 @@ int rtw89_mac_ptk_drop_by_band_and_wait(struct rtw89_dev *rtwdev,
 	for (i = 0; i < try_cnt; i++) {
 		ret = read_poll_timeout(mac_is_txq_empty, empty, empty, 50,
 					50000, false, rtwdev);
-		if (ret)
+		if (ret && !RTW89_CHK_FW_FEATURE(NO_PACKET_DROP, &rtwdev->fw))
 			rtw89_fw_h2c_pkt_drop(rtwdev, &params);
 		else
 			return 0;
diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h
index 8064d3953d7f..a8d9847ef0b4 100644
--- a/drivers/net/wireless/realtek/rtw89/mac.h
+++ b/drivers/net/wireless/realtek/rtw89/mac.h
@@ -359,6 +359,7 @@ enum rtw89_mac_c2h_ofld_func {
 	RTW89_MAC_C2H_FUNC_MACID_PAUSE,
 	RTW89_MAC_C2H_FUNC_TSF32_TOGL_RPT = 0x6,
 	RTW89_MAC_C2H_FUNC_SCANOFLD_RSP = 0x9,
+	RTW89_MAC_C2H_FUNC_BCNFLTR_RPT = 0xd,
 	RTW89_MAC_C2H_FUNC_OFLD_MAX,
 };
 
@@ -815,6 +816,7 @@ struct rtw89_mac_size_set {
 	const struct rtw89_ple_quota ple_qt47;
 	const struct rtw89_ple_quota ple_qt58;
 	const struct rtw89_ple_quota ple_qt_52a_wow;
+	const struct rtw89_ple_quota ple_qt_52b_wow;
 };
 
 extern const struct rtw89_mac_size_set rtw89_mac_size;
@@ -1116,6 +1118,7 @@ enum rtw89_mac_xtal_si_offset {
 	XTAL_SI_XTAL_XMD_4 = 0x26,
 #define XTAL_SI_LPS_CAP		GENMASK(3, 0)
 	XTAL_SI_CV = 0x41,
+#define XTAL_SI_ACV_MASK	GENMASK(3, 0)
 	XTAL_SI_LOW_ADDR = 0x62,
 #define XTAL_SI_LOW_ADDR_MASK	GENMASK(7, 0)
 	XTAL_SI_CTRL = 0x63,
@@ -1146,7 +1149,7 @@ enum rtw89_mac_xtal_si_offset {
 int rtw89_mac_write_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 val, u8 mask);
 int rtw89_mac_read_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 *val);
 void rtw89_mac_pkt_drop_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
-u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd);
+int rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd, u16 *pkt_id);
 int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev,
 			struct rtw89_cpuio_ctrl *ctrl_para, bool wd);
 int rtw89_mac_typ_fltr_opt(struct rtw89_dev *rtwdev,
diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c
index d43281f7335b..ee4588b61b8f 100644
--- a/drivers/net/wireless/realtek/rtw89/mac80211.c
+++ b/drivers/net/wireless/realtek/rtw89/mac80211.c
@@ -23,9 +23,19 @@ static void rtw89_ops_tx(struct ieee80211_hw *hw,
 	struct rtw89_dev *rtwdev = hw->priv;
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
 	struct ieee80211_vif *vif = info->control.vif;
+	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	struct ieee80211_sta *sta = control->sta;
+	u32 flags = IEEE80211_SKB_CB(skb)->flags;
 	int ret, qsel;
 
+	if (rtwvif->offchan && !(flags & IEEE80211_TX_CTL_TX_OFFCHAN) && sta) {
+		struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
+
+		rtw89_debug(rtwdev, RTW89_DBG_TXRX, "ops_tx during offchan\n");
+		skb_queue_tail(&rtwsta->roc_queue, skb);
+		return;
+	}
+
 	ret = rtw89_core_tx_write(rtwdev, vif, sta, skb, &qsel);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to transmit skb: %d\n", ret);
@@ -95,7 +105,8 @@ static int rtw89_ops_config(struct ieee80211_hw *hw, u32 changed)
 	}
 
 	if ((changed & IEEE80211_CONF_CHANGE_IDLE) &&
-	    (hw->conf.flags & IEEE80211_CONF_IDLE))
+	    (hw->conf.flags & IEEE80211_CONF_IDLE) &&
+	    !rtwdev->scanning)
 		rtw89_enter_ips(rtwdev);
 
 	mutex_unlock(&rtwdev->mutex);
@@ -114,9 +125,19 @@ static int rtw89_ops_add_interface(struct ieee80211_hw *hw,
 		    vif->addr, vif->type, vif->p2p);
 
 	mutex_lock(&rtwdev->mutex);
+
+	rtw89_leave_ips_by_hwflags(rtwdev);
+
+	if (RTW89_CHK_FW_FEATURE(BEACON_FILTER, &rtwdev->fw))
+		vif->driver_flags |= IEEE80211_VIF_BEACON_FILTER |
+				     IEEE80211_VIF_SUPPORTS_CQM_RSSI;
+
 	rtwvif->rtwdev = rtwdev;
+	rtwvif->roc.state = RTW89_ROC_IDLE;
+	rtwvif->offchan = false;
 	list_add_tail(&rtwvif->list, &rtwdev->rtwvifs_list);
 	INIT_WORK(&rtwvif->update_beacon_work, rtw89_core_update_beacon_work);
+	INIT_DELAYED_WORK(&rtwvif->roc.roc_work, rtw89_roc_work);
 	rtw89_leave_ps_mode(rtwdev);
 
 	rtw89_traffic_stats_init(rtwdev, &rtwvif->stats);
@@ -163,6 +184,7 @@ static void rtw89_ops_remove_interface(struct ieee80211_hw *hw,
 		    vif->addr, vif->type, vif->p2p);
 
 	cancel_work_sync(&rtwvif->update_beacon_work);
+	cancel_delayed_work_sync(&rtwvif->roc.roc_work);
 
 	mutex_lock(&rtwdev->mutex);
 	rtw89_leave_ps_mode(rtwdev);
@@ -170,6 +192,8 @@ static void rtw89_ops_remove_interface(struct ieee80211_hw *hw,
 	rtw89_mac_remove_vif(rtwdev, rtwvif);
 	rtw89_core_release_bit_map(rtwdev->hw_port, rtwvif->port);
 	list_del_init(&rtwvif->list);
+	rtw89_enter_ips_by_hwflags(rtwdev);
+
 	mutex_unlock(&rtwdev->mutex);
 }
 
@@ -394,7 +418,6 @@ static void rtw89_ops_bss_info_changed(struct ieee80211_hw *hw,
 			rtw89_chip_cfg_txpwr_ul_tb_offset(rtwdev, vif);
 			rtw89_mac_port_update(rtwdev, rtwvif);
 			rtw89_mac_set_he_obss_narrow_bw_ru(rtwdev, vif);
-			rtw89_store_op_chan(rtwdev, true);
 		} else {
 			/* Abort ongoing scan if cancel_scan isn't issued
 			 * when disconnected by peer
@@ -425,6 +448,9 @@ static void rtw89_ops_bss_info_changed(struct ieee80211_hw *hw,
 	if (changed & BSS_CHANGED_P2P_PS)
 		rtw89_process_p2p_ps(rtwdev, vif);
 
+	if (changed & BSS_CHANGED_CQM)
+		rtw89_fw_h2c_set_bcn_fltr_cfg(rtwdev, vif, true);
+
 	mutex_unlock(&rtwdev->mutex);
 }
 
@@ -676,7 +702,7 @@ static void rtw89_ops_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	rtw89_leave_lps(rtwdev);
 	rtw89_hci_flush_queues(rtwdev, queues, drop);
 
-	if (drop && RTW89_CHK_FW_FEATURE(PACKET_DROP, &rtwdev->fw))
+	if (drop && !RTW89_CHK_FW_FEATURE(NO_PACKET_DROP, &rtwdev->fw))
 		__rtw89_drop_packets(rtwdev, vif);
 	else
 		rtw89_mac_flush_txq(rtwdev, queues, drop);
@@ -795,12 +821,13 @@ static int rtw89_ops_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			     struct ieee80211_scan_request *req)
 {
 	struct rtw89_dev *rtwdev = hw->priv;
+	struct rtw89_vif *rtwvif = vif_to_rtwvif_safe(vif);
 	int ret = 0;
 
 	if (!RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD, &rtwdev->fw))
 		return 1;
 
-	if (rtwdev->scanning)
+	if (rtwdev->scanning || rtwvif->offchan)
 		return -EBUSY;
 
 	mutex_lock(&rtwdev->mutex);
@@ -903,6 +930,63 @@ static void rtw89_ops_unassign_vif_chanctx(struct ieee80211_hw *hw,
 	mutex_unlock(&rtwdev->mutex);
 }
 
+static int rtw89_ops_remain_on_channel(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_channel *chan,
+				       int duration,
+				       enum ieee80211_roc_type type)
+{
+	struct rtw89_dev *rtwdev = hw->priv;
+	struct rtw89_vif *rtwvif = vif_to_rtwvif_safe(vif);
+	struct rtw89_roc *roc = &rtwvif->roc;
+
+	if (!vif)
+		return -EINVAL;
+
+	mutex_lock(&rtwdev->mutex);
+
+	if (roc->state != RTW89_ROC_IDLE) {
+		mutex_unlock(&rtwdev->mutex);
+		return -EBUSY;
+	}
+
+	if (rtwdev->scanning)
+		rtw89_hw_scan_abort(rtwdev, vif);
+
+	if (type == IEEE80211_ROC_TYPE_MGMT_TX)
+		roc->state = RTW89_ROC_MGMT;
+	else
+		roc->state = RTW89_ROC_NORMAL;
+
+	roc->duration = duration;
+	roc->chan = *chan;
+	roc->type = type;
+
+	rtw89_roc_start(rtwdev, rtwvif);
+
+	mutex_unlock(&rtwdev->mutex);
+
+	return 0;
+}
+
+static int rtw89_ops_cancel_remain_on_channel(struct ieee80211_hw *hw,
+					      struct ieee80211_vif *vif)
+{
+	struct rtw89_dev *rtwdev = hw->priv;
+	struct rtw89_vif *rtwvif = vif_to_rtwvif_safe(vif);
+
+	if (!rtwvif)
+		return -EINVAL;
+
+	cancel_delayed_work_sync(&rtwvif->roc.roc_work);
+
+	mutex_lock(&rtwdev->mutex);
+	rtw89_roc_end(rtwdev, rtwvif);
+	mutex_unlock(&rtwdev->mutex);
+
+	return 0;
+}
+
 static void rtw89_set_tid_config_iter(void *data, struct ieee80211_sta *sta)
 {
 	struct cfg80211_tid_config *tid_config = data;
@@ -1014,6 +1098,8 @@ const struct ieee80211_ops rtw89_ops = {
 	.change_chanctx		= rtw89_ops_change_chanctx,
 	.assign_vif_chanctx	= rtw89_ops_assign_vif_chanctx,
 	.unassign_vif_chanctx	= rtw89_ops_unassign_vif_chanctx,
+	.remain_on_channel		= rtw89_ops_remain_on_channel,
+	.cancel_remain_on_channel	= rtw89_ops_cancel_remain_on_channel,
 	.set_sar_specs		= rtw89_ops_set_sar_specs,
 	.sta_rc_update		= rtw89_ops_sta_rc_update,
 	.set_tid_config		= rtw89_ops_set_tid_config,
diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c
index ec8bb5f10482..70b4754667c9 100644
--- a/drivers/net/wireless/realtek/rtw89/pci.c
+++ b/drivers/net/wireless/realtek/rtw89/pci.c
@@ -364,8 +364,11 @@ static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,
 				struct rtw89_pci_tx_ring *tx_ring,
 				struct sk_buff *skb, u8 tx_status)
 {
+	struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
 	struct ieee80211_tx_info *info;
 
+	rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_status == RTW89_TX_DONE);
+
 	info = IEEE80211_SKB_CB(skb);
 	ieee80211_tx_info_clear_status(info);
 
@@ -1203,6 +1206,7 @@ static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
 	struct pci_dev *pdev = rtwpci->pdev;
 	struct sk_buff *skb = tx_req->skb;
 	struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
+	struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
 	bool en_wd_info = desc_info->en_wd_info;
 	u32 txwd_len;
 	u32 txwp_len;
@@ -1218,6 +1222,7 @@ static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
 	}
 
 	tx_data->dma = dma;
+	rcu_assign_pointer(skb_data->wait, NULL);
 
 	txwp_len = sizeof(*txwp_info);
 	txwd_len = chip->txwd_body_size;
@@ -1912,9 +1917,10 @@ __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate
 
 static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
 {
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 	int ret;
 
-	if (rtwdev->chip->chip_id != RTL8852B)
+	if (chip_id != RTL8852B && chip_id != RTL8851B)
 		return 0;
 
 	ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
@@ -1924,13 +1930,14 @@ static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
 
 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
 {
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 	enum rtw89_pcie_phy phy_rate;
 	u16 val16, mgn_set, div_set, tar;
 	u8 val8, bdr_ori;
 	bool l1_flag = false;
 	int ret = 0;
 
-	if (rtwdev->chip->chip_id != RTL8852B)
+	if (chip_id != RTL8852B && chip_id != RTL8851B)
 		return 0;
 
 	ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8);
@@ -2107,7 +2114,9 @@ static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)
 
 static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)
 {
-	if (rtwdev->chip->chip_id != RTL8852A && rtwdev->chip->chip_id != RTL8852B)
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
+
+	if (chip_id != RTL8852A && chip_id != RTL8852B && chip_id != RTL8851B)
 		return;
 
 	rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN);
@@ -2135,7 +2144,9 @@ static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
 
 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
 {
-	if (rtwdev->chip->chip_id != RTL8852A && rtwdev->chip->chip_id != RTL8852B)
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
+
+	if (chip_id != RTL8852A && chip_id != RTL8852B && chip_id != RTL8851B)
 		return;
 
 	rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
@@ -2143,8 +2154,9 @@ static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
 
 static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
 {
-	if (rtwdev->chip->chip_id == RTL8852A ||
-	    rtwdev->chip->chip_id == RTL8852B) {
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
+
+	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
 		rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
 		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
@@ -2157,7 +2169,9 @@ static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
 
 static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)
 {
-	if (rtwdev->chip->chip_id != RTL8852B)
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
+
+	if (chip_id != RTL8852B && chip_id != RTL8851B)
 		return 0;
 
 	return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK,
@@ -2694,7 +2708,6 @@ static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,
 static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,
 				     struct pci_dev *pdev)
 {
-	pci_clear_master(pdev);
 	pci_disable_device(pdev);
 }
 
@@ -3398,7 +3411,7 @@ static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
 	if (ret)
 		rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
 
-	if (chip_id == RTL8852A || chip_id == RTL8852B) {
+	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
 		if (enable)
 			ret = rtw89_pci_config_byte_set(rtwdev,
 							RTW89_PCIE_L1_CTRL,
@@ -3443,7 +3456,7 @@ static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
 	if (ret)
 		rtw89_err(rtwdev, "failed to read ASPM Delay\n");
 
-	if (chip_id == RTL8852A || chip_id == RTL8852B) {
+	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
 		if (enable)
 			ret = rtw89_pci_config_byte_set(rtwdev,
 							RTW89_PCIE_L1_CTRL,
@@ -3523,7 +3536,7 @@ static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 	int ret;
 
-	if (chip_id == RTL8852A || chip_id == RTL8852B) {
+	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
 		if (enable)
 			ret = rtw89_pci_config_byte_set(rtwdev,
 							RTW89_PCIE_TIMER_CTRL,
@@ -3722,7 +3735,7 @@ static int __maybe_unused rtw89_pci_suspend(struct device *dev)
 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
-	if (chip_id == RTL8852A || chip_id == RTL8852B) {
+	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
 		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
 		rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
@@ -3756,7 +3769,7 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev)
 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
-	if (chip_id == RTL8852A || chip_id == RTL8852B) {
+	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
 		rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
 		rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
@@ -3874,25 +3887,26 @@ int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	rtw89_pci_link_cfg(rtwdev);
 	rtw89_pci_l1ss_cfg(rtwdev);
 
-	ret = rtw89_core_register(rtwdev);
-	if (ret) {
-		rtw89_err(rtwdev, "failed to register core\n");
-		goto err_clear_resource;
-	}
-
 	rtw89_core_napi_init(rtwdev);
 
 	ret = rtw89_pci_request_irq(rtwdev, pdev);
 	if (ret) {
 		rtw89_err(rtwdev, "failed to request pci irq\n");
-		goto err_unregister;
+		goto err_deinit_napi;
+	}
+
+	ret = rtw89_core_register(rtwdev);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to register core\n");
+		goto err_free_irq;
 	}
 
 	return 0;
 
-err_unregister:
+err_free_irq:
+	rtw89_pci_free_irq(rtwdev, pdev);
+err_deinit_napi:
 	rtw89_core_napi_deinit(rtwdev);
-	rtw89_core_unregister(rtwdev);
 err_clear_resource:
 	rtw89_pci_clear_resource(rtwdev, pdev);
 err_declaim_pci:
diff --git a/drivers/net/wireless/realtek/rtw89/pci.h b/drivers/net/wireless/realtek/rtw89/pci.h
index 1e19740db8c5..0e4bd210b100 100644
--- a/drivers/net/wireless/realtek/rtw89/pci.h
+++ b/drivers/net/wireless/realtek/rtw89/pci.h
@@ -1004,9 +1004,9 @@ rtw89_pci_rxbd_increase(struct rtw89_pci_rx_ring *rx_ring, u32 cnt)
 
 static inline struct rtw89_pci_tx_data *RTW89_PCI_TX_SKB_CB(struct sk_buff *skb)
 {
-	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct rtw89_tx_skb_data *data = RTW89_TX_SKB_CB(skb);
 
-	return (struct rtw89_pci_tx_data *)info->status.status_driver_data;
+	return (struct rtw89_pci_tx_data *)data->hci_priv;
 }
 
 static inline struct rtw89_pci_tx_bd_32 *
diff --git a/drivers/net/wireless/realtek/rtw89/phy.c b/drivers/net/wireless/realtek/rtw89/phy.c
index d9f61ba3d176..c7e906123416 100644
--- a/drivers/net/wireless/realtek/rtw89/phy.c
+++ b/drivers/net/wireless/realtek/rtw89/phy.c
@@ -1617,29 +1617,35 @@ static u8 rtw89_channel_to_idx(struct rtw89_dev *rtwdev, u8 band, u8 channel)
 s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, u8 band,
 			      u8 bw, u8 ntx, u8 rs, u8 bf, u8 ch)
 {
-	const struct rtw89_chip_info *chip = rtwdev->chip;
+	const struct rtw89_rfe_parms *rfe_parms = rtwdev->rfe_parms;
+	const struct rtw89_txpwr_rule_2ghz *rule_2ghz = &rfe_parms->rule_2ghz;
+	const struct rtw89_txpwr_rule_5ghz *rule_5ghz = &rfe_parms->rule_5ghz;
+	const struct rtw89_txpwr_rule_6ghz *rule_6ghz = &rfe_parms->rule_6ghz;
 	u8 ch_idx = rtw89_channel_to_idx(rtwdev, band, ch);
 	u8 regd = rtw89_regd_get(rtwdev, band);
 	s8 lmt = 0, sar;
 
 	switch (band) {
 	case RTW89_BAND_2G:
-		lmt = (*chip->txpwr_lmt_2g)[bw][ntx][rs][bf][regd][ch_idx];
-		if (!lmt)
-			lmt = (*chip->txpwr_lmt_2g)[bw][ntx][rs][bf]
-						   [RTW89_WW][ch_idx];
+		lmt = (*rule_2ghz->lmt)[bw][ntx][rs][bf][regd][ch_idx];
+		if (lmt)
+			break;
+
+		lmt = (*rule_2ghz->lmt)[bw][ntx][rs][bf][RTW89_WW][ch_idx];
 		break;
 	case RTW89_BAND_5G:
-		lmt = (*chip->txpwr_lmt_5g)[bw][ntx][rs][bf][regd][ch_idx];
-		if (!lmt)
-			lmt = (*chip->txpwr_lmt_5g)[bw][ntx][rs][bf]
-						   [RTW89_WW][ch_idx];
+		lmt = (*rule_5ghz->lmt)[bw][ntx][rs][bf][regd][ch_idx];
+		if (lmt)
+			break;
+
+		lmt = (*rule_5ghz->lmt)[bw][ntx][rs][bf][RTW89_WW][ch_idx];
 		break;
 	case RTW89_BAND_6G:
-		lmt = (*chip->txpwr_lmt_6g)[bw][ntx][rs][bf][regd][ch_idx];
-		if (!lmt)
-			lmt = (*chip->txpwr_lmt_6g)[bw][ntx][rs][bf]
-						   [RTW89_WW][ch_idx];
+		lmt = (*rule_6ghz->lmt)[bw][ntx][rs][bf][regd][ch_idx];
+		if (lmt)
+			break;
+
+		lmt = (*rule_6ghz->lmt)[bw][ntx][rs][bf][RTW89_WW][ch_idx];
 		break;
 	default:
 		rtw89_warn(rtwdev, "unknown band type: %d\n", band);
@@ -1862,29 +1868,35 @@ void rtw89_phy_fill_txpwr_limit(struct rtw89_dev *rtwdev,
 static s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev, u8 band,
 					u8 ru, u8 ntx, u8 ch)
 {
-	const struct rtw89_chip_info *chip = rtwdev->chip;
+	const struct rtw89_rfe_parms *rfe_parms = rtwdev->rfe_parms;
+	const struct rtw89_txpwr_rule_2ghz *rule_2ghz = &rfe_parms->rule_2ghz;
+	const struct rtw89_txpwr_rule_5ghz *rule_5ghz = &rfe_parms->rule_5ghz;
+	const struct rtw89_txpwr_rule_6ghz *rule_6ghz = &rfe_parms->rule_6ghz;
 	u8 ch_idx = rtw89_channel_to_idx(rtwdev, band, ch);
 	u8 regd = rtw89_regd_get(rtwdev, band);
 	s8 lmt_ru = 0, sar;
 
 	switch (band) {
 	case RTW89_BAND_2G:
-		lmt_ru = (*chip->txpwr_lmt_ru_2g)[ru][ntx][regd][ch_idx];
-		if (!lmt_ru)
-			lmt_ru = (*chip->txpwr_lmt_ru_2g)[ru][ntx]
-							 [RTW89_WW][ch_idx];
+		lmt_ru = (*rule_2ghz->lmt_ru)[ru][ntx][regd][ch_idx];
+		if (lmt_ru)
+			break;
+
+		lmt_ru = (*rule_2ghz->lmt_ru)[ru][ntx][RTW89_WW][ch_idx];
 		break;
 	case RTW89_BAND_5G:
-		lmt_ru = (*chip->txpwr_lmt_ru_5g)[ru][ntx][regd][ch_idx];
-		if (!lmt_ru)
-			lmt_ru = (*chip->txpwr_lmt_ru_5g)[ru][ntx]
-							 [RTW89_WW][ch_idx];
+		lmt_ru = (*rule_5ghz->lmt_ru)[ru][ntx][regd][ch_idx];
+		if (lmt_ru)
+			break;
+
+		lmt_ru = (*rule_5ghz->lmt_ru)[ru][ntx][RTW89_WW][ch_idx];
 		break;
 	case RTW89_BAND_6G:
-		lmt_ru = (*chip->txpwr_lmt_ru_6g)[ru][ntx][regd][ch_idx];
-		if (!lmt_ru)
-			lmt_ru = (*chip->txpwr_lmt_ru_6g)[ru][ntx]
-							 [RTW89_WW][ch_idx];
+		lmt_ru = (*rule_6ghz->lmt_ru)[ru][ntx][regd][ch_idx];
+		if (lmt_ru)
+			break;
+
+		lmt_ru = (*rule_6ghz->lmt_ru)[ru][ntx][RTW89_WW][ch_idx];
 		break;
 	default:
 		rtw89_warn(rtwdev, "unknown band type: %d\n", band);
@@ -2405,7 +2417,6 @@ static void rtw89_dcfo_comp(struct rtw89_dev *rtwdev, s32 curr_cfo)
 	bool is_linked = rtwdev->total_sta_assoc > 0;
 	s32 cfo_avg_312;
 	s32 dcfo_comp_val;
-	u8 dcfo_comp_sft = rtwdev->chip->dcfo_comp_sft;
 	int sign;
 
 	if (!is_linked) {
@@ -2418,8 +2429,8 @@ static void rtw89_dcfo_comp(struct rtw89_dev *rtwdev, s32 curr_cfo)
 		return;
 	dcfo_comp_val = rtw89_phy_read32_mask(rtwdev, R_DCFO, B_DCFO);
 	sign = curr_cfo > 0 ? 1 : -1;
-	cfo_avg_312 = (curr_cfo << dcfo_comp_sft) / 5 + sign * dcfo_comp_val;
-	rtw89_debug(rtwdev, RTW89_DBG_CFO, "DCFO: avg_cfo=%d\n", cfo_avg_312);
+	cfo_avg_312 = curr_cfo / 625 + sign * dcfo_comp_val;
+	rtw89_debug(rtwdev, RTW89_DBG_CFO, "avg_cfo_312=%d step\n", cfo_avg_312);
 	if (rtwdev->chip->chip_id == RTL8852A && rtwdev->hal.cv == CHIP_CBV)
 		cfo_avg_312 = -cfo_avg_312;
 	rtw89_phy_set_phy_regs(rtwdev, dcfo_comp->addr, dcfo_comp->mask,
@@ -2428,9 +2439,16 @@ static void rtw89_dcfo_comp(struct rtw89_dev *rtwdev, s32 curr_cfo)
 
 static void rtw89_dcfo_comp_init(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
 	rtw89_phy_set_phy_regs(rtwdev, R_DCFO_OPT, B_DCFO_OPT_EN, 1);
 	rtw89_phy_set_phy_regs(rtwdev, R_DCFO_WEIGHT, B_DCFO_WEIGHT_MSK, 8);
-	rtw89_write32_clr(rtwdev, R_AX_PWR_UL_CTRL2, B_AX_PWR_UL_CFO_MASK);
+
+	if (chip->cfo_hw_comp)
+		rtw89_write32_mask(rtwdev, R_AX_PWR_UL_CTRL2,
+				   B_AX_PWR_UL_CFO_MASK, 0x6);
+	else
+		rtw89_write32_clr(rtwdev, R_AX_PWR_UL_CTRL2, B_AX_PWR_UL_CFO_MASK);
 }
 
 static void rtw89_phy_cfo_init(struct rtw89_dev *rtwdev)
@@ -2500,6 +2518,7 @@ static void rtw89_phy_cfo_crystal_cap_adjust(struct rtw89_dev *rtwdev,
 
 static s32 rtw89_phy_average_cfo_calc(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_cfo_tracking_info *cfo = &rtwdev->cfo_tracking;
 	s32 cfo_khz_all = 0;
 	s32 cfo_cnt_all = 0;
@@ -2516,6 +2535,8 @@ static s32 rtw89_phy_average_cfo_calc(struct rtw89_dev *rtwdev)
 		cfo_cnt_all += cfo->cfo_cnt[i];
 		cfo_all_avg = phy_div(cfo_khz_all, cfo_cnt_all);
 		cfo->pre_cfo_avg[i] = cfo->cfo_avg[i];
+		cfo->dcfo_avg = phy_div(cfo_khz_all << chip->dcfo_comp_sft,
+					cfo_cnt_all);
 	}
 	rtw89_debug(rtwdev, RTW89_DBG_CFO,
 		    "CFO track for macid = %d\n", i);
@@ -2642,7 +2663,9 @@ static void rtw89_phy_cfo_dm(struct rtw89_dev *rtwdev)
 	s32 new_cfo = 0;
 	bool x_cap_update = false;
 	u8 pre_x_cap = cfo->crystal_cap;
+	u8 dcfo_comp_sft = rtwdev->chip->dcfo_comp_sft;
 
+	cfo->dcfo_avg = 0;
 	rtw89_debug(rtwdev, RTW89_DBG_CFO, "CFO:total_sta_assoc=%d\n",
 		    rtwdev->total_sta_assoc);
 	if (rtwdev->total_sta_assoc == 0) {
@@ -2684,18 +2707,19 @@ static void rtw89_phy_cfo_dm(struct rtw89_dev *rtwdev)
 
 	rtw89_phy_cfo_crystal_cap_adjust(rtwdev, new_cfo);
 	cfo->cfo_avg_pre = new_cfo;
+	cfo->dcfo_avg_pre = cfo->dcfo_avg;
 	x_cap_update =  cfo->crystal_cap != pre_x_cap;
 	rtw89_debug(rtwdev, RTW89_DBG_CFO, "Xcap_up=%d\n", x_cap_update);
 	rtw89_debug(rtwdev, RTW89_DBG_CFO, "Xcap: D:%x C:%x->%x, ofst=%d\n",
 		    cfo->def_x_cap, pre_x_cap, cfo->crystal_cap,
 		    cfo->x_cap_ofst);
 	if (x_cap_update) {
-		if (new_cfo > 0)
-			new_cfo -= CFO_SW_COMP_FINE_TUNE;
+		if (cfo->dcfo_avg > 0)
+			cfo->dcfo_avg -= CFO_SW_COMP_FINE_TUNE << dcfo_comp_sft;
 		else
-			new_cfo += CFO_SW_COMP_FINE_TUNE;
+			cfo->dcfo_avg += CFO_SW_COMP_FINE_TUNE << dcfo_comp_sft;
 	}
-	rtw89_dcfo_comp(rtwdev, new_cfo);
+	rtw89_dcfo_comp(rtwdev, cfo->dcfo_avg);
 	rtw89_phy_cfo_statistics_reset(rtwdev);
 }
 
@@ -4294,3 +4318,94 @@ void rtw89_phy_tssi_ctrl_set_bandedge_cfg(struct rtw89_dev *rtwdev,
 					      data[RTW89_TSSI_SBW20]);
 }
 EXPORT_SYMBOL(rtw89_phy_tssi_ctrl_set_bandedge_cfg);
+
+static
+const u8 rtw89_ch_base_table[16] = {1, 0xff,
+				    36, 100, 132, 149, 0xff,
+				    1, 33, 65, 97, 129, 161, 193, 225, 0xff};
+#define RTW89_CH_BASE_IDX_2G		0
+#define RTW89_CH_BASE_IDX_5G_FIRST	2
+#define RTW89_CH_BASE_IDX_5G_LAST	5
+#define RTW89_CH_BASE_IDX_6G_FIRST	7
+#define RTW89_CH_BASE_IDX_6G_LAST	14
+
+#define RTW89_CH_BASE_IDX_MASK		GENMASK(7, 4)
+#define RTW89_CH_OFFSET_MASK		GENMASK(3, 0)
+
+u8 rtw89_encode_chan_idx(struct rtw89_dev *rtwdev, u8 central_ch, u8 band)
+{
+	u8 chan_idx;
+	u8 last, first;
+	u8 idx;
+
+	switch (band) {
+	case RTW89_BAND_2G:
+		chan_idx = FIELD_PREP(RTW89_CH_BASE_IDX_MASK, RTW89_CH_BASE_IDX_2G) |
+			   FIELD_PREP(RTW89_CH_OFFSET_MASK, central_ch);
+		return chan_idx;
+	case RTW89_BAND_5G:
+		first = RTW89_CH_BASE_IDX_5G_FIRST;
+		last = RTW89_CH_BASE_IDX_5G_LAST;
+		break;
+	case RTW89_BAND_6G:
+		first = RTW89_CH_BASE_IDX_6G_FIRST;
+		last = RTW89_CH_BASE_IDX_6G_LAST;
+		break;
+	default:
+		rtw89_warn(rtwdev, "Unsupported band %d\n", band);
+		return 0;
+	}
+
+	for (idx = last; idx >= first; idx--)
+		if (central_ch >= rtw89_ch_base_table[idx])
+			break;
+
+	if (idx < first) {
+		rtw89_warn(rtwdev, "Unknown band %d channel %d\n", band, central_ch);
+		return 0;
+	}
+
+	chan_idx = FIELD_PREP(RTW89_CH_BASE_IDX_MASK, idx) |
+		   FIELD_PREP(RTW89_CH_OFFSET_MASK,
+			      (central_ch - rtw89_ch_base_table[idx]) >> 1);
+	return chan_idx;
+}
+EXPORT_SYMBOL(rtw89_encode_chan_idx);
+
+void rtw89_decode_chan_idx(struct rtw89_dev *rtwdev, u8 chan_idx,
+			   u8 *ch, enum nl80211_band *band)
+{
+	u8 idx, offset;
+
+	idx = FIELD_GET(RTW89_CH_BASE_IDX_MASK, chan_idx);
+	offset = FIELD_GET(RTW89_CH_OFFSET_MASK, chan_idx);
+
+	if (idx == RTW89_CH_BASE_IDX_2G) {
+		*band = NL80211_BAND_2GHZ;
+		*ch = offset;
+		return;
+	}
+
+	*band = idx <= RTW89_CH_BASE_IDX_5G_LAST ? NL80211_BAND_5GHZ : NL80211_BAND_6GHZ;
+	*ch = rtw89_ch_base_table[idx] + (offset << 1);
+}
+EXPORT_SYMBOL(rtw89_decode_chan_idx);
+
+#define EDCCA_DEFAULT 249
+void rtw89_phy_config_edcca(struct rtw89_dev *rtwdev, bool scan)
+{
+	u32 reg = rtwdev->chip->edcca_lvl_reg;
+	struct rtw89_hal *hal = &rtwdev->hal;
+	u32 val;
+
+	if (scan) {
+		hal->edcca_bak = rtw89_phy_read32(rtwdev, reg);
+		val = hal->edcca_bak;
+		u32p_replace_bits(&val, EDCCA_DEFAULT, B_SEG0R_EDCCA_LVL_A_MSK);
+		u32p_replace_bits(&val, EDCCA_DEFAULT, B_SEG0R_EDCCA_LVL_P_MSK);
+		u32p_replace_bits(&val, EDCCA_DEFAULT, B_SEG0R_PPDU_LVL_MSK);
+		rtw89_phy_write32(rtwdev, reg, val);
+	} else {
+		rtw89_phy_write32(rtwdev, reg, hal->edcca_bak);
+	}
+}
diff --git a/drivers/net/wireless/realtek/rtw89/phy.h b/drivers/net/wireless/realtek/rtw89/phy.h
index 21233f094644..7535867d0f48 100644
--- a/drivers/net/wireless/realtek/rtw89/phy.h
+++ b/drivers/net/wireless/realtek/rtw89/phy.h
@@ -555,5 +555,9 @@ void rtw89_phy_tssi_ctrl_set_bandedge_cfg(struct rtw89_dev *rtwdev,
 					  enum rtw89_tssi_bandedge_cfg bandedge_cfg);
 void rtw89_phy_ul_tb_assoc(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
 void rtw89_phy_ul_tb_ctrl_track(struct rtw89_dev *rtwdev);
+u8 rtw89_encode_chan_idx(struct rtw89_dev *rtwdev, u8 central_ch, u8 band);
+void rtw89_decode_chan_idx(struct rtw89_dev *rtwdev, u8 chan_idx,
+			   u8 *ch, enum nl80211_band *band);
+void rtw89_phy_config_edcca(struct rtw89_dev *rtwdev, bool scan);
 
 #endif
diff --git a/drivers/net/wireless/realtek/rtw89/ps.c b/drivers/net/wireless/realtek/rtw89/ps.c
index 40498812205e..fa94335f699a 100644
--- a/drivers/net/wireless/realtek/rtw89/ps.c
+++ b/drivers/net/wireless/realtek/rtw89/ps.c
@@ -114,7 +114,8 @@ void rtw89_leave_ps_mode(struct rtw89_dev *rtwdev)
 	__rtw89_leave_ps_mode(rtwdev);
 }
 
-void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
+void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
+		     bool ps_mode)
 {
 	lockdep_assert_held(&rtwdev->mutex);
 
@@ -122,7 +123,8 @@ void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 		return;
 
 	__rtw89_enter_lps(rtwdev, rtwvif->mac_id);
-	__rtw89_enter_ps_mode(rtwdev, rtwvif);
+	if (ps_mode)
+		__rtw89_enter_ps_mode(rtwdev, rtwvif);
 }
 
 static void rtw89_leave_lps_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
@@ -155,6 +157,9 @@ void rtw89_enter_ips(struct rtw89_dev *rtwdev)
 
 	set_bit(RTW89_FLAG_INACTIVE_PS, rtwdev->flags);
 
+	if (!test_bit(RTW89_FLAG_POWERON, rtwdev->flags))
+		return;
+
 	rtw89_for_each_rtwvif(rtwdev, rtwvif)
 		rtw89_mac_vif_deinit(rtwdev, rtwvif);
 
@@ -166,6 +171,9 @@ void rtw89_leave_ips(struct rtw89_dev *rtwdev)
 	struct rtw89_vif *rtwvif;
 	int ret;
 
+	if (test_bit(RTW89_FLAG_POWERON, rtwdev->flags))
+		return;
+
 	ret = rtw89_core_start(rtwdev);
 	if (ret)
 		rtw89_err(rtwdev, "failed to leave idle state\n");
diff --git a/drivers/net/wireless/realtek/rtw89/ps.h b/drivers/net/wireless/realtek/rtw89/ps.h
index 6ac1f7ea5339..73c008db0426 100644
--- a/drivers/net/wireless/realtek/rtw89/ps.h
+++ b/drivers/net/wireless/realtek/rtw89/ps.h
@@ -5,7 +5,8 @@
 #ifndef __RTW89_PS_H_
 #define __RTW89_PS_H_
 
-void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
+void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
+		     bool ps_mode);
 void rtw89_leave_lps(struct rtw89_dev *rtwdev);
 void __rtw89_leave_ps_mode(struct rtw89_dev *rtwdev);
 void __rtw89_enter_ps_mode(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
@@ -15,4 +16,20 @@ void rtw89_leave_ips(struct rtw89_dev *rtwdev);
 void rtw89_set_coex_ctrl_lps(struct rtw89_dev *rtwdev, bool btc_ctrl);
 void rtw89_process_p2p_ps(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif);
 
+static inline void rtw89_leave_ips_by_hwflags(struct rtw89_dev *rtwdev)
+{
+	struct ieee80211_hw *hw = rtwdev->hw;
+
+	if (hw->conf.flags & IEEE80211_CONF_IDLE)
+		rtw89_leave_ips(rtwdev);
+}
+
+static inline void rtw89_enter_ips_by_hwflags(struct rtw89_dev *rtwdev)
+{
+	struct ieee80211_hw *hw = rtwdev->hw;
+
+	if (hw->conf.flags & IEEE80211_CONF_IDLE)
+		rtw89_enter_ips(rtwdev);
+}
+
 #endif
diff --git a/drivers/net/wireless/realtek/rtw89/reg.h b/drivers/net/wireless/realtek/rtw89/reg.h
index 600257909df2..266e4231b5f3 100644
--- a/drivers/net/wireless/realtek/rtw89/reg.h
+++ b/drivers/net/wireless/realtek/rtw89/reg.h
@@ -129,6 +129,7 @@
 
 #define R_AX_PLATFORM_ENABLE 0x0088
 #define B_AX_AXIDMA_EN BIT(3)
+#define B_AX_APB_WRAP_EN BIT(2)
 #define B_AX_WCPU_EN BIT(1)
 #define B_AX_PLATFORM_EN BIT(0)
 
@@ -207,6 +208,11 @@
 
 #define R_AX_UDM0 0x01F0
 #define R_AX_UDM1 0x01F4
+#define B_AX_UDM1_MASK GENMASK(31, 16)
+#define B_AX_UDM1_HALMAC_C2H_ENQ_CNT_MASK GENMASK(15, 12)
+#define B_AX_UDM1_HALMAC_H2C_DEQ_CNT_MASK GENMASK(11, 8)
+#define B_AX_UDM1_WCPU_C2H_ENQ_CNT_MASK GENMASK(7, 4)
+#define B_AX_UDM1_WCPU_H2C_DEQ_CNT_MASK GENMASK(3, 0)
 #define R_AX_UDM2 0x01F8
 #define R_AX_UDM3 0x01FC
 
@@ -483,6 +489,7 @@
 #define B_AX_DISPATCHER_CLK_EN BIT(18)
 #define B_AX_BBRPT_CLK_EN BIT(17)
 #define B_AX_MAC_SEC_CLK_EN BIT(16)
+#define B_AX_AXIDMA_CLK_EN BIT(9)
 
 #define PCI_LTR_IDLE_TIMER_1US 0
 #define PCI_LTR_IDLE_TIMER_10US 1
@@ -1581,6 +1588,7 @@
 #define R_AX_PL_BUF_STATUS 0x9824
 #define B_AX_WD_BUF_STAT_DONE BIT(31)
 #define B_AX_WD_BUF_STAT_PKTID_MASK GENMASK(11, 0)
+#define S_WD_BUF_STAT_PKTID_INVALID GENMASK(11, 0)
 
 #define R_AX_WD_CPUQ_OP_0 0x9810
 #define R_AX_PL_CPUQ_OP_0 0x9830
@@ -3056,6 +3064,8 @@
 #define R_AX_BFMEE_RESP_OPTION_C1 0xED80
 #define B_AX_BFMEE_NDP_RX_STANDBY_TIMER_MASK GENMASK(31, 24)
 #define B_AX_BFMEE_BFRP_RX_STANDBY_TIMER_MASK GENMASK(23, 20)
+#define BFRP_RX_STANDBY_TIMER_KEEP 0x0
+#define BFRP_RX_STANDBY_TIMER_RELEASE 0x1
 #define B_AX_MU_BFRPTSEG_SEL_MASK GENMASK(18, 17)
 #define B_AX_BFMEE_NDP_RXSTDBY_SEL BIT(16)
 #define BFRP_RX_STANDBY_TIMER		0x0
@@ -4273,6 +4283,11 @@
 #define B_PKT_POP_EN BIT(8)
 #define R_SEG0R_PD 0x481C
 #define R_SEG0R_PD_V1 0x4860
+#define R_SEG0R_EDCCA_LVL 0x4840
+#define R_SEG0R_EDCCA_LVL_V1 0x4884
+#define B_SEG0R_PPDU_LVL_MSK GENMASK(31, 24)
+#define B_SEG0R_EDCCA_LVL_P_MSK GENMASK(15, 8)
+#define B_SEG0R_EDCCA_LVL_A_MSK GENMASK(7, 0)
 #define B_SEG0R_PD_SPATIAL_REUSE_EN_MSK_V1 BIT(30)
 #define B_SEG0R_PD_SPATIAL_REUSE_EN_MSK BIT(29)
 #define B_SEG0R_PD_LOWER_BOUND_MSK GENMASK(10, 6)
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8851b_rfk_table.c b/drivers/net/wireless/realtek/rtw89/rtw8851b_rfk_table.c
new file mode 100644
index 000000000000..0abf7978ccab
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw89/rtw8851b_rfk_table.c
@@ -0,0 +1,534 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2022-2023  Realtek Corporation
+ */
+
+#include "rtw8851b_rfk_table.h"
+
+static const struct rtw89_reg5_def rtw8851b_dadck_setup_defs[] = {
+	RTW89_DECL_RFK_WM(0xc210, 0x003fc000, 0x80),
+	RTW89_DECL_RFK_WM(0xc224, 0x003fc000, 0x80),
+	RTW89_DECL_RFK_WM(0xc0f8, 0x30000000, 0x3),
+	RTW89_DECL_RFK_WM(0x12b8, BIT(30), 0x1),
+	RTW89_DECL_RFK_WM(0x030c, 0x1f000000, 0x1f),
+	RTW89_DECL_RFK_WM(0x032c, 0xc0000000, 0x0),
+	RTW89_DECL_RFK_WM(0x032c, BIT(22), 0x0),
+	RTW89_DECL_RFK_WM(0x032c, BIT(22), 0x1),
+	RTW89_DECL_RFK_WM(0x032c, BIT(16), 0x0),
+	RTW89_DECL_RFK_WM(0x032c, BIT(20), 0x1),
+	RTW89_DECL_RFK_WM(0x030c, 0x0f000000, 0x3),
+	RTW89_DECL_RFK_WM(0xc0f4, BIT(2), 0x0),
+	RTW89_DECL_RFK_WM(0xc0f4, BIT(4), 0x0),
+	RTW89_DECL_RFK_WM(0xc0f4, BIT(11), 0x1),
+	RTW89_DECL_RFK_WM(0xc0f4, BIT(11), 0x0),
+	RTW89_DECL_RFK_DELAY(1),
+	RTW89_DECL_RFK_WM(0xc0f4, 0x300, 0x1),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_dadck_setup_defs);
+
+static const struct rtw89_reg5_def rtw8851b_dadck_post_defs[] = {
+	RTW89_DECL_RFK_WM(0x032c, BIT(16), 0x1),
+	RTW89_DECL_RFK_WM(0x032c, BIT(20), 0x0),
+	RTW89_DECL_RFK_WM(0x030c, 0x1f000000, 0xc),
+	RTW89_DECL_RFK_WM(0x032c, 0xc0000000, 0x1),
+	RTW89_DECL_RFK_WM(0x12b8, BIT(30), 0x0),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_dadck_post_defs);
+
+static const struct rtw89_reg5_def rtw8851b_dack_s0_1_defs[] = {
+	RTW89_DECL_RFK_WM(0x12a0, BIT(15), 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x7000, 0x3),
+	RTW89_DECL_RFK_WM(0x12b8, BIT(30), 0x1),
+	RTW89_DECL_RFK_WM(0x030c, BIT(28), 0x1),
+	RTW89_DECL_RFK_WM(0x032c, 0x80000000, 0x0),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_dack_s0_1_defs);
+
+static const struct rtw89_reg5_def rtw8851b_dack_s0_2_defs[] = {
+	RTW89_DECL_RFK_WM(0xc004, BIT(0), 0x0),
+	RTW89_DECL_RFK_WM(0x12a0, BIT(15), 0x0),
+	RTW89_DECL_RFK_WM(0x12a0, 0x7000, 0x7),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_dack_s0_2_defs);
+
+static const struct rtw89_reg5_def rtw8851b_dack_manual_off_defs[] = {
+	RTW89_DECL_RFK_WM(0xc0f8, 0x30000000, 0x0),
+	RTW89_DECL_RFK_WM(0xc210, BIT(0), 0x0),
+	RTW89_DECL_RFK_WM(0xc224, BIT(0), 0x0),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_dack_manual_off_defs);
+
+static const struct rtw89_reg5_def rtw8851b_iqk_rxclk_80_defs[] = {
+	RTW89_DECL_RFK_WM(0x20fc, 0xffff0000, 0x0101),
+	RTW89_DECL_RFK_WM(0x5670, 0x00002000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00080000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00070000, 0x2),
+	RTW89_DECL_RFK_WM(0x5670, 0x60000000, 0x1),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x00000780, 0x8),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x00007800, 0x2),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x0c000000, 0x2),
+	RTW89_DECL_RFK_WM(0xc0d8, 0x000001e0, 0x5),
+	RTW89_DECL_RFK_WM(0xc0c4, 0x003e0000, 0xf),
+	RTW89_DECL_RFK_WM(0xc0ec, 0x00006000, 0x0),
+	RTW89_DECL_RFK_WM(0x12b8, 0x40000000, 0x1),
+	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x0f),
+	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x03),
+	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0001),
+	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0041),
+	RTW89_DECL_RFK_WM(0x20fc, 0xffff0000, 0x1101),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_rxclk_80_defs);
+
+static const struct rtw89_reg5_def rtw8851b_iqk_rxclk_others_defs[] = {
+	RTW89_DECL_RFK_WM(0x20fc, 0xffff0000, 0x0101),
+	RTW89_DECL_RFK_WM(0x5670, 0x00002000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00080000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00070000, 0x2),
+	RTW89_DECL_RFK_WM(0x5670, 0x60000000, 0x0),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x00000780, 0x8),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x00007800, 0x2),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x0c000000, 0x2),
+	RTW89_DECL_RFK_WM(0xc0d8, 0x000001e0, 0x5),
+	RTW89_DECL_RFK_WM(0xc0c4, 0x003e0000, 0xf),
+	RTW89_DECL_RFK_WM(0xc0ec, 0x00006000, 0x2),
+	RTW89_DECL_RFK_WM(0x12b8, 0x40000000, 0x1),
+	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x0f),
+	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x03),
+	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0001),
+	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0041),
+	RTW89_DECL_RFK_WM(0x20fc, 0xffff0000, 0x1101),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_rxclk_others_defs);
+
+static const struct rtw89_reg5_def rtw8851b_iqk_txk_2ghz_defs[] = {
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x51, 0x80000, 0x0),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x51, 0x00800, 0x0),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x52, 0x00800, 0x0),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x55, 0x0001f, 0x4),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0xef, 0x00004, 0x1),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x00, 0xffff0, 0x403e),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x11, 0x00003, 0x0),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x11, 0x00070, 0x6),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x11, 0x1f000, 0x10),
+	RTW89_DECL_RFK_DELAY(1),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_txk_2ghz_defs);
+
+static const struct rtw89_reg5_def rtw8851b_iqk_txk_5ghz_defs[] = {
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x60, 0x00007, 0x0),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x55, 0x0001f, 0x4),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0xef, 0x00004, 0x1),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x00, 0xffff0, 0x403e),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x11, 0x00003, 0x0),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x11, 0x00070, 0x7),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x11, 0x1f000, 0x7),
+	RTW89_DECL_RFK_DELAY(1),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_txk_5ghz_defs);
+
+static const struct rtw89_reg5_def rtw8851b_iqk_afebb_restore_defs[] = {
+	RTW89_DECL_RFK_WM(0x12b8, 0x40000000, 0x0),
+	RTW89_DECL_RFK_WM(0x20fc, 0x00010000, 0x1),
+	RTW89_DECL_RFK_WM(0x20fc, 0x00100000, 0x0),
+	RTW89_DECL_RFK_WM(0x20fc, 0x01000000, 0x1),
+	RTW89_DECL_RFK_WM(0x20fc, 0x10000000, 0x0),
+	RTW89_DECL_RFK_WM(0x5670, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x12a0, 0x000ff000, 0x00),
+	RTW89_DECL_RFK_WM(0x20fc, 0x00010000, 0x0),
+	RTW89_DECL_RFK_WM(0x20fc, 0x01000000, 0x0),
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x10005, 0x00001, 0x1),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_afebb_restore_defs);
+
+static const struct rtw89_reg5_def rtw8851b_iqk_macbb_defs[] = {
+	RTW89_DECL_RFK_WRF(RF_PATH_A, 0x10005, 0x00001, 0x0),
+	RTW89_DECL_RFK_WM(0x20fc, 0x00010000, 0x1),
+	RTW89_DECL_RFK_WM(0x20fc, 0x00100000, 0x0),
+	RTW89_DECL_RFK_WM(0x20fc, 0x01000000, 0x1),
+	RTW89_DECL_RFK_WM(0x20fc, 0x10000000, 0x0),
+	RTW89_DECL_RFK_WM(0x5670, MASKDWORD, 0xf801fffd),
+	RTW89_DECL_RFK_WM(0x5670, 0x00004000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00008000, 0x1),
+	RTW89_DECL_RFK_WM(0x5670, 0x80000000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00007000, 0x7),
+	RTW89_DECL_RFK_WM(0x5670, 0x00002000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00080000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00070000, 0x3),
+	RTW89_DECL_RFK_WM(0x5670, 0x60000000, 0x2),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x00000780, 0x9),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x00007800, 0x1),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x0c000000, 0x0),
+	RTW89_DECL_RFK_WM(0xc0d8, 0x000001e0, 0x3),
+	RTW89_DECL_RFK_WM(0xc0c4, 0x003e0000, 0xa),
+	RTW89_DECL_RFK_WM(0xc0ec, 0x00006000, 0x0),
+	RTW89_DECL_RFK_WM(0xc0e8, 0x00000040, 0x1),
+	RTW89_DECL_RFK_WM(0x12b8, 0x40000000, 0x1),
+	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x1f),
+	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x13),
+	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0001),
+	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0041),
+	RTW89_DECL_RFK_WM(0x20fc, 0x00100000, 0x1),
+	RTW89_DECL_RFK_WM(0x20fc, 0x10000000, 0x1),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_macbb_defs);
+
+static const struct rtw89_reg5_def rtw8851b_iqk_bb_afe_defs[] = {
+	RTW89_DECL_RFK_WM(0x5670, 0x00004000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00008000, 0x1),
+	RTW89_DECL_RFK_WM(0x5670, 0x80000000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00007000, 0x7),
+	RTW89_DECL_RFK_WM(0x5670, 0x00002000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00080000, 0x1),
+	RTW89_DECL_RFK_WM(0x12a0, 0x00070000, 0x3),
+	RTW89_DECL_RFK_WM(0x5670, 0x60000000, 0x2),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x00000780, 0x9),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x00007800, 0x1),
+	RTW89_DECL_RFK_WM(0xc0d4, 0x0c000000, 0x0),
+	RTW89_DECL_RFK_WM(0xc0d8, 0x000001e0, 0x3),
+	RTW89_DECL_RFK_WM(0xc0c4, 0x003e0000, 0xa),
+	RTW89_DECL_RFK_WM(0xc0ec, 0x00006000, 0x0),
+	RTW89_DECL_RFK_WM(0xc0e8, 0x00000040, 0x1),
+	RTW89_DECL_RFK_WM(0x12b8, 0x40000000, 0x1),
+	RTW89_DECL_RFK_WM(0x030c, MASKBYTE3, 0x1f),
+	RTW89_DECL_RFK_WM(0x030c, MASKBYTE3, 0x13),
+	RTW89_DECL_RFK_WM(0x032c, MASKHWORD, 0x0001),
+	RTW89_DECL_RFK_WM(0x032c, MASKHWORD, 0x0041),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_iqk_bb_afe_defs);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_sys_defs[] = {
+	RTW89_DECL_RFK_WM(0x12bc, 0x000ffff0, 0xb5b5),
+	RTW89_DECL_RFK_WM(0x32bc, 0x000ffff0, 0xb5b5),
+	RTW89_DECL_RFK_WM(0x0300, 0xff000000, 0x16),
+	RTW89_DECL_RFK_WM(0x0304, 0x0000ffff, 0x1f19),
+	RTW89_DECL_RFK_WM(0x0308, 0xff000000, 0x1c),
+	RTW89_DECL_RFK_WM(0x0314, 0xffff0000, 0x2041),
+	RTW89_DECL_RFK_WM(0x0318, 0xffffffff, 0x20012041),
+	RTW89_DECL_RFK_WM(0x0324, 0xffff0000, 0x2001),
+	RTW89_DECL_RFK_WM(0x0020, 0x00006000, 0x3),
+	RTW89_DECL_RFK_WM(0x0024, 0x00006000, 0x3),
+	RTW89_DECL_RFK_WM(0x0704, 0xffff0000, 0x601e),
+	RTW89_DECL_RFK_WM(0x2704, 0xffff0000, 0x601e),
+	RTW89_DECL_RFK_WM(0x0700, 0xf0000000, 0x4),
+	RTW89_DECL_RFK_WM(0x2700, 0xf0000000, 0x4),
+	RTW89_DECL_RFK_WM(0x0650, 0x3c000000, 0x0),
+	RTW89_DECL_RFK_WM(0x2650, 0x3c000000, 0x0),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_sys_defs);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_sys_a_defs_2g[] = {
+	RTW89_DECL_RFK_WM(0x120c, 0x000000ff, 0x33),
+	RTW89_DECL_RFK_WM(0x12c0, 0x0ff00000, 0x33),
+	RTW89_DECL_RFK_WM(0x58f8, 0x40000000, 0x1),
+	RTW89_DECL_RFK_WM(0x5814, 0x20000000, 0x0),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_sys_a_defs_2g);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_sys_a_defs_5g[] = {
+	RTW89_DECL_RFK_WM(0x120c, 0x000000ff, 0x44),
+	RTW89_DECL_RFK_WM(0x12c0, 0x0ff00000, 0x44),
+	RTW89_DECL_RFK_WM(0x58f8, 0x40000000, 0x0),
+	RTW89_DECL_RFK_WM(0x5814, 0x20000000, 0x0),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_sys_a_defs_5g);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_init_txpwr_defs_a[] = {
+	RTW89_DECL_RFK_WM(0x566c, 0x00001000, 0x0),
+	RTW89_DECL_RFK_WM(0x5800, 0xffffffff, 0x003f807f),
+	RTW89_DECL_RFK_WM(0x580c, 0x0000007f, 0x40),
+	RTW89_DECL_RFK_WM(0x580c, 0x0fffff00, 0x00040),
+	RTW89_DECL_RFK_WM(0x5810, 0xffffffff, 0x59010000),
+	RTW89_DECL_RFK_WM(0x5814, 0x01ffffff, 0x026d000),
+	RTW89_DECL_RFK_WM(0x5814, 0xf8000000, 0x00),
+	RTW89_DECL_RFK_WM(0x5818, 0x00ffffff, 0x2c18e8),
+	RTW89_DECL_RFK_WM(0x5818, 0x07000000, 0x0),
+	RTW89_DECL_RFK_WM(0x5818, 0xf0000000, 0x0),
+	RTW89_DECL_RFK_WM(0x581c, 0x3fffffff, 0x3dc80280),
+	RTW89_DECL_RFK_WM(0x5820, 0xffffffff, 0x00000080),
+	RTW89_DECL_RFK_WM(0x58e8, 0x0000003f, 0x04),
+	RTW89_DECL_RFK_WM(0x580c, 0x10000000, 0x1),
+	RTW89_DECL_RFK_WM(0x580c, 0x40000000, 0x1),
+	RTW89_DECL_RFK_WM(0x5834, 0x3fffffff, 0x000115f2),
+	RTW89_DECL_RFK_WM(0x5838, 0x7fffffff, 0x0000121),
+	RTW89_DECL_RFK_WM(0x5854, 0x3fffffff, 0x000115f2),
+	RTW89_DECL_RFK_WM(0x5858, 0x7fffffff, 0x0000121),
+	RTW89_DECL_RFK_WM(0x5860, 0x80000000, 0x0),
+	RTW89_DECL_RFK_WM(0x5864, 0x07ffffff, 0x00801ff),
+	RTW89_DECL_RFK_WM(0x5898, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x589c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x58a4, 0x000000ff, 0x16),
+	RTW89_DECL_RFK_WM(0x58b0, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x58b4, 0x7fffffff, 0x0a002000),
+	RTW89_DECL_RFK_WM(0x58b8, 0x7fffffff, 0x00007628),
+	RTW89_DECL_RFK_WM(0x58bc, 0x07ffffff, 0x7a7807f),
+	RTW89_DECL_RFK_WM(0x58c0, 0xfffe0000, 0x003f),
+	RTW89_DECL_RFK_WM(0x58c4, 0xffffffff, 0x0003ffff),
+	RTW89_DECL_RFK_WM(0x58c8, 0x00ffffff, 0x000000),
+	RTW89_DECL_RFK_WM(0x58c8, 0xf0000000, 0x0),
+	RTW89_DECL_RFK_WM(0x58cc, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x58d0, 0x07ffffff, 0x2008101),
+	RTW89_DECL_RFK_WM(0x58d4, 0x000000ff, 0x00),
+	RTW89_DECL_RFK_WM(0x58d4, 0x0003fe00, 0x0ff),
+	RTW89_DECL_RFK_WM(0x58d4, 0x07fc0000, 0x100),
+	RTW89_DECL_RFK_WM(0x58d8, 0xffffffff, 0x8008016c),
+	RTW89_DECL_RFK_WM(0x58dc, 0x0001ffff, 0x0807f),
+	RTW89_DECL_RFK_WM(0x58dc, 0xfff00000, 0x800),
+	RTW89_DECL_RFK_WM(0x58f0, 0x0003ffff, 0x001ff),
+	RTW89_DECL_RFK_WM(0x58f4, 0x000fffff, 0x00000),
+	RTW89_DECL_RFK_WM(0x58f8, 0x000fffff, 0x00000),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_init_txpwr_defs_a);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_init_txpwr_he_tb_defs_a[] = {
+	RTW89_DECL_RFK_WM(0x58a0, MASKDWORD, 0x000000fe),
+	RTW89_DECL_RFK_WM(0x58e4, 0x0000007f, 0x1f),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_init_txpwr_he_tb_defs_a);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_dck_defs_a[] = {
+	RTW89_DECL_RFK_WM(0x580c, 0x0fff0000, 0x000),
+	RTW89_DECL_RFK_WM(0x5814, 0x00001000, 0x1),
+	RTW89_DECL_RFK_WM(0x5814, 0x00002000, 0x1),
+	RTW89_DECL_RFK_WM(0x5814, 0x00004000, 0x1),
+	RTW89_DECL_RFK_WM(0x5814, 0x00038000, 0x3),
+	RTW89_DECL_RFK_WM(0x5814, 0x003c0000, 0x5),
+	RTW89_DECL_RFK_WM(0x5814, 0x18000000, 0x0),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_dck_defs_a);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_dac_gain_defs_a[] = {
+	RTW89_DECL_RFK_WM(0x58b0, 0x00000fff, 0x000),
+	RTW89_DECL_RFK_WM(0x5a00, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a04, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a08, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a0c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a10, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a14, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a18, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a1c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a20, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a24, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a28, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a2c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a30, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a34, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a38, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a3c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a40, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a44, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a48, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a4c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a50, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a54, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a58, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a5c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a60, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a64, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a68, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a6c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a70, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a74, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a78, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a7c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a80, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a84, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a88, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a8c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a90, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a94, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a98, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5a9c, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5aa0, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5aa4, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5aa8, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5aac, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5ab0, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5ab4, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5ab8, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5abc, MASKDWORD, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5ac0, MASKDWORD, 0x00000000),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_dac_gain_defs_a);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_slope_a_defs_2g[] = {
+	RTW89_DECL_RFK_WM(0x5608, 0x07ffffff, 0x0201008),
+	RTW89_DECL_RFK_WM(0x560c, 0x07ffffff, 0x0201008),
+	RTW89_DECL_RFK_WM(0x5610, 0x07ffffff, 0x0200e08),
+	RTW89_DECL_RFK_WM(0x5614, 0x07ffffff, 0x0201008),
+	RTW89_DECL_RFK_WM(0x5618, 0x07ffffff, 0x0201008),
+	RTW89_DECL_RFK_WM(0x561c, 0x000001ff, 0x007),
+	RTW89_DECL_RFK_WM(0x561c, 0xffff0000, 0x0808),
+	RTW89_DECL_RFK_WM(0x5620, 0xffffffff, 0x08080808),
+	RTW89_DECL_RFK_WM(0x5624, 0xffffffff, 0x08080808),
+	RTW89_DECL_RFK_WM(0x5628, 0xffffffff, 0x08080808),
+	RTW89_DECL_RFK_WM(0x562c, 0x0000ffff, 0x0808),
+	RTW89_DECL_RFK_WM(0x581c, 0x00100000, 0x1),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_slope_a_defs_2g);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_slope_a_defs_5g[] = {
+	RTW89_DECL_RFK_WM(0x5608, 0x07ffffff, 0x0201008),
+	RTW89_DECL_RFK_WM(0x560c, 0x07ffffff, 0x0341a08),
+	RTW89_DECL_RFK_WM(0x5610, 0x07ffffff, 0x0201417),
+	RTW89_DECL_RFK_WM(0x5614, 0x07ffffff, 0x0201008),
+	RTW89_DECL_RFK_WM(0x5618, 0x07ffffff, 0x0201008),
+	RTW89_DECL_RFK_WM(0x561c, 0x000001ff, 0x008),
+	RTW89_DECL_RFK_WM(0x561c, 0xffff0000, 0x0808),
+	RTW89_DECL_RFK_WM(0x5620, 0xffffffff, 0x0e0e0808),
+	RTW89_DECL_RFK_WM(0x5624, 0xffffffff, 0x08080d18),
+	RTW89_DECL_RFK_WM(0x5628, 0xffffffff, 0x08080808),
+	RTW89_DECL_RFK_WM(0x562c, 0x0000ffff, 0x0808),
+	RTW89_DECL_RFK_WM(0x581c, 0x00100000, 0x1),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_slope_a_defs_5g);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_align_a_2g_defs[] = {
+	RTW89_DECL_RFK_WM(0x5604, 0x80000000, 0x1),
+	RTW89_DECL_RFK_WM(0x5600, 0x3fffffff, 0x000000),
+	RTW89_DECL_RFK_WM(0x5604, 0x003fffff, 0x2d2400),
+	RTW89_DECL_RFK_WM(0x5630, 0x3fffffff, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5634, 0x000003ff, 0x000),
+	RTW89_DECL_RFK_WM(0x5634, 0x000ffc00, 0x000),
+	RTW89_DECL_RFK_WM(0x5634, 0x3ff00000, 0x3fa),
+	RTW89_DECL_RFK_WM(0x5638, 0x000003ff, 0x02e),
+	RTW89_DECL_RFK_WM(0x5638, 0x000ffc00, 0x09c),
+	RTW89_DECL_RFK_WM(0x563c, 0x3fffffff, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5640, 0x3fffffff, 0x3fb00000),
+	RTW89_DECL_RFK_WM(0x5644, 0x000003ff, 0x02f),
+	RTW89_DECL_RFK_WM(0x5644, 0x000ffc00, 0x09c),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_align_a_2g_defs);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_align_a_5g_defs[] = {
+	RTW89_DECL_RFK_WM(0x5604, 0x80000000, 0x1),
+	RTW89_DECL_RFK_WM(0x5600, 0x3fffffff, 0x000000),
+	RTW89_DECL_RFK_WM(0x5604, 0x003fffff, 0x3b2d24),
+	RTW89_DECL_RFK_WM(0x5630, 0x3fffffff, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5634, 0x000003ff, 0x000),
+	RTW89_DECL_RFK_WM(0x5634, 0x000ffc00, 0x3cb),
+	RTW89_DECL_RFK_WM(0x5634, 0x3ff00000, 0x030),
+	RTW89_DECL_RFK_WM(0x5638, 0x000003ff, 0x73),
+	RTW89_DECL_RFK_WM(0x5638, 0x000ffc00, 0xd4),
+	RTW89_DECL_RFK_WM(0x563c, 0x3fffffff, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5640, 0x3fffffff, 0x00000000),
+	RTW89_DECL_RFK_WM(0x5644, 0x000fffff, 0x00000),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_align_a_5g_defs);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_slope_defs_a[] = {
+	RTW89_DECL_RFK_WM(0x5820, 0x80000000, 0x0),
+	RTW89_DECL_RFK_WM(0x5818, 0x10000000, 0x0),
+	RTW89_DECL_RFK_WM(0x5814, 0x00000800, 0x1),
+	RTW89_DECL_RFK_WM(0x581c, 0x20000000, 0x1),
+	RTW89_DECL_RFK_WM(0x5820, 0x0000f000, 0xf),
+	RTW89_DECL_RFK_WM(0x581c, 0x000003ff, 0x280),
+	RTW89_DECL_RFK_WM(0x581c, 0x000ffc00, 0x200),
+	RTW89_DECL_RFK_WM(0x58b8, 0x007f0000, 0x00),
+	RTW89_DECL_RFK_WM(0x58b8, 0x7f000000, 0x00),
+	RTW89_DECL_RFK_WM(0x58b4, 0x7f000000, 0x0a),
+	RTW89_DECL_RFK_WM(0x58b8, 0x0000007f, 0x28),
+	RTW89_DECL_RFK_WM(0x58b8, 0x00007f00, 0x76),
+	RTW89_DECL_RFK_WM(0x5810, 0x20000000, 0x0),
+	RTW89_DECL_RFK_WM(0x580c, 0x10000000, 0x1),
+	RTW89_DECL_RFK_WM(0x580c, 0x40000000, 0x1),
+	RTW89_DECL_RFK_WM(0x5834, 0x0003ffff, 0x115f2),
+	RTW89_DECL_RFK_WM(0x5834, 0x3ffc0000, 0x000),
+	RTW89_DECL_RFK_WM(0x5838, 0x00000fff, 0x121),
+	RTW89_DECL_RFK_WM(0x5838, 0x003ff000, 0x000),
+	RTW89_DECL_RFK_WM(0x5854, 0x0003ffff, 0x115f2),
+	RTW89_DECL_RFK_WM(0x5854, 0x3ffc0000, 0x000),
+	RTW89_DECL_RFK_WM(0x5858, 0x00000fff, 0x121),
+	RTW89_DECL_RFK_WM(0x5858, 0x003ff000, 0x000),
+	RTW89_DECL_RFK_WM(0x5824, 0x0003ffff, 0x115f2),
+	RTW89_DECL_RFK_WM(0x5824, 0x3ffc0000, 0x000),
+	RTW89_DECL_RFK_WM(0x5828, 0x00000fff, 0x121),
+	RTW89_DECL_RFK_WM(0x5828, 0x003ff000, 0x000),
+	RTW89_DECL_RFK_WM(0x582c, 0x0003ffff, 0x115f2),
+	RTW89_DECL_RFK_WM(0x582c, 0x3ffc0000, 0x000),
+	RTW89_DECL_RFK_WM(0x5830, 0x00000fff, 0x121),
+	RTW89_DECL_RFK_WM(0x5830, 0x003ff000, 0x000),
+	RTW89_DECL_RFK_WM(0x583c, 0x0003ffff, 0x115f2),
+	RTW89_DECL_RFK_WM(0x583c, 0x3ffc0000, 0x000),
+	RTW89_DECL_RFK_WM(0x5840, 0x00000fff, 0x121),
+	RTW89_DECL_RFK_WM(0x5840, 0x003ff000, 0x000),
+	RTW89_DECL_RFK_WM(0x5844, 0x0003ffff, 0x115f2),
+	RTW89_DECL_RFK_WM(0x5844, 0x3ffc0000, 0x000),
+	RTW89_DECL_RFK_WM(0x5848, 0x00000fff, 0x121),
+	RTW89_DECL_RFK_WM(0x5848, 0x003ff000, 0x000),
+	RTW89_DECL_RFK_WM(0x584c, 0x0003ffff, 0x115f2),
+	RTW89_DECL_RFK_WM(0x584c, 0x3ffc0000, 0x000),
+	RTW89_DECL_RFK_WM(0x5850, 0x00000fff, 0x121),
+	RTW89_DECL_RFK_WM(0x5850, 0x003ff000, 0x000),
+	RTW89_DECL_RFK_WM(0x585c, 0x0003ffff, 0x115f2),
+	RTW89_DECL_RFK_WM(0x585c, 0x3ffc0000, 0x000),
+	RTW89_DECL_RFK_WM(0x5860, 0x00000fff, 0x121),
+	RTW89_DECL_RFK_WM(0x5860, 0x003ff000, 0x000),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_slope_defs_a);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_track_defs_a[] = {
+	RTW89_DECL_RFK_WM(0x5820, 0x80000000, 0x0),
+	RTW89_DECL_RFK_WM(0x5818, 0x10000000, 0x0),
+	RTW89_DECL_RFK_WM(0x5814, 0x00000800, 0x0),
+	RTW89_DECL_RFK_WM(0x581c, 0x20000000, 0x1),
+	RTW89_DECL_RFK_WM(0x5864, 0x000003ff, 0x1ff),
+	RTW89_DECL_RFK_WM(0x5864, 0x000ffc00, 0x200),
+	RTW89_DECL_RFK_WM(0x5820, 0x00000fff, 0x080),
+	RTW89_DECL_RFK_WM(0x5814, 0x01000000, 0x0),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_track_defs_a);
+
+static const struct rtw89_reg5_def rtw8851b_tssi_mv_avg_defs_a[] = {
+	RTW89_DECL_RFK_WM(0x58e4, 0x00003800, 0x1),
+	RTW89_DECL_RFK_WM(0x58e4, 0x00004000, 0x0),
+	RTW89_DECL_RFK_WM(0x58e4, 0x00008000, 0x1),
+	RTW89_DECL_RFK_WM(0x58e4, 0x000f0000, 0x0),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_tssi_mv_avg_defs_a);
+
+static const struct rtw89_reg5_def rtw8851b_nctl_post_defs[] = {
+	RTW89_DECL_RFK_WM(0x5864, 0x18000000, 0x3),
+	RTW89_DECL_RFK_WM(0x7864, 0x18000000, 0x3),
+	RTW89_DECL_RFK_WM(0x030c, 0xff000000, 0x13),
+	RTW89_DECL_RFK_WM(0x032c, 0xffff0000, 0x0041),
+	RTW89_DECL_RFK_WM(0x12b8, 0x10000000, 0x1),
+	RTW89_DECL_RFK_WM(0x2008, 0x01ffffff, 0x00fffff),
+	RTW89_DECL_RFK_WM(0x0c60, 0x00000003, 0x3),
+	RTW89_DECL_RFK_WM(0x0c6c, 0x00000001, 0x1),
+	RTW89_DECL_RFK_WM(0x58ac, 0x08000000, 0x1),
+	RTW89_DECL_RFK_WM(0x78ac, 0x08000000, 0x1),
+	RTW89_DECL_RFK_WM(0x0730, 0x00003800, 0x7),
+	RTW89_DECL_RFK_WM(0x2730, 0x00003800, 0x7),
+	RTW89_DECL_RFK_WM(0x0c7c, 0x00e00000, 0x1),
+	RTW89_DECL_RFK_WM(0x58c0, 0x0001ffff, 0x00000),
+	RTW89_DECL_RFK_WM(0x78c0, 0x0001ffff, 0x00000),
+	RTW89_DECL_RFK_WM(0x58fc, 0x3f000000, 0x00),
+	RTW89_DECL_RFK_WM(0x78fc, 0x3f000000, 0x00),
+};
+
+RTW89_DECLARE_RFK_TBL(rtw8851b_nctl_post_defs);
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8851b_rfk_table.h b/drivers/net/wireless/realtek/rtw89/rtw8851b_rfk_table.h
new file mode 100644
index 000000000000..febfbecb691c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw89/rtw8851b_rfk_table.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/* Copyright(c) 2022-2023  Realtek Corporation
+ */
+
+#ifndef __RTW89_8851B_RFK_TABLE_H__
+#define __RTW89_8851B_RFK_TABLE_H__
+
+#include "phy.h"
+
+extern const struct rtw89_rfk_tbl rtw8851b_dadck_setup_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_dadck_post_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_dack_s0_1_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_dack_s0_2_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_dack_manual_off_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_iqk_rxclk_80_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_iqk_rxclk_others_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_iqk_txk_2ghz_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_iqk_txk_5ghz_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_iqk_afebb_restore_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_iqk_bb_afe_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_iqk_macbb_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_sys_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_sys_a_defs_2g_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_sys_a_defs_5g_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_init_txpwr_defs_a_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_init_txpwr_he_tb_defs_a_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_dck_defs_a_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_dac_gain_defs_a_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_slope_a_defs_2g_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_slope_a_defs_5g_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_align_a_2g_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_align_a_5g_defs_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_slope_defs_a_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_track_defs_a_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_tssi_mv_avg_defs_a_tbl;
+extern const struct rtw89_rfk_tbl rtw8851b_nctl_post_defs_tbl;
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8851b_table.c b/drivers/net/wireless/realtek/rtw89/rtw8851b_table.c
new file mode 100644
index 000000000000..bb724140df4f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw89/rtw8851b_table.c
@@ -0,0 +1,14824 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2022-2023  Realtek Corporation
+ */
+
+#include "phy.h"
+#include "reg.h"
+#include "rtw8851b_table.h"
+
+static const struct rtw89_reg2_def rtw89_8851b_phy_bb_regs[] = {
+	{0x704, 0x601E0500},
+	{0x4000, 0x00000000},
+	{0x4004, 0xCA014000},
+	{0x4008, 0xC751D4F0},
+	{0x400C, 0x44511475},
+	{0x4010, 0x00000000},
+	{0x4014, 0x00000000},
+	{0x47BC, 0x00000380},
+	{0x4018, 0x4F4C084B},
+	{0x401C, 0x084A4E52},
+	{0x4020, 0x4D504E4B},
+	{0x4024, 0x4F4C0849},
+	{0x4028, 0x08484C50},
+	{0x402C, 0x4C50504C},
+	{0x4030, 0x5454084A},
+	{0x4034, 0x084B5654},
+	{0x4038, 0x6A6C605A},
+	{0x403C, 0x4C4C084C},
+	{0x4040, 0x084B4E4D},
+	{0x4044, 0x4E4C4B4B},
+	{0x4048, 0x4B4B084A},
+	{0x404C, 0x084A4E4C},
+	{0x4050, 0x514F4C4A},
+	{0x4054, 0x524E084A},
+	{0x4058, 0x084A5154},
+	{0x405C, 0x53555554},
+	{0x4060, 0x45450845},
+	{0x4064, 0x08454144},
+	{0x4068, 0x40434445},
+	{0x406C, 0x44450845},
+	{0x4070, 0x08444043},
+	{0x4074, 0x42434444},
+	{0x4078, 0x46450844},
+	{0x407C, 0x08444843},
+	{0x4080, 0x4B4E4A47},
+	{0x4084, 0x4F4C084B},
+	{0x4088, 0x084A4E52},
+	{0x408C, 0x4D504E4B},
+	{0x4090, 0x4F4C0849},
+	{0x4094, 0x08484C50},
+	{0x4098, 0x4C50504C},
+	{0x409C, 0x5454084A},
+	{0x40A0, 0x084B5654},
+	{0x40A4, 0x6A6C605A},
+	{0x40A8, 0x4C4C084C},
+	{0x40AC, 0x084B4E4D},
+	{0x40B0, 0x4E4C4B4B},
+	{0x40B4, 0x4B4B084A},
+	{0x40B8, 0x084A4E4C},
+	{0x40BC, 0x514F4C4A},
+	{0x40C0, 0x524E084A},
+	{0x40C4, 0x084A5154},
+	{0x40C8, 0x53555554},
+	{0x40CC, 0x45450845},
+	{0x40D0, 0x08454144},
+	{0x40D4, 0x40434445},
+	{0x40D8, 0x44450845},
+	{0x40DC, 0x08444043},
+	{0x40E0, 0x42434444},
+	{0x40E4, 0x46450844},
+	{0x40E8, 0x08444843},
+	{0x40EC, 0x4B4E4A47},
+	{0x40F0, 0x00000000},
+	{0x4A38, 0x00000000},
+	{0x40F4, 0x00000006},
+	{0x40F8, 0x00000000},
+	{0x40FC, 0x8C30C30C},
+	{0x4100, 0x4C30C30C},
+	{0x4104, 0x0C30C30C},
+	{0x4108, 0x0C30C30C},
+	{0x410C, 0x0C30C30C},
+	{0x4110, 0x0C30C30C},
+	{0x4114, 0x28A28A28},
+	{0x4118, 0x28A28A28},
+	{0x411C, 0x28A28A28},
+	{0x4120, 0x28A28A28},
+	{0x4124, 0x28A28A28},
+	{0x4128, 0x28A28A28},
+	{0x412C, 0x06666666},
+	{0x4130, 0x33333333},
+	{0x4134, 0x33333333},
+	{0x4138, 0x33333333},
+	{0x413C, 0x00000031},
+	{0x4140, 0x5100600A},
+	{0x4144, 0x18363113},
+	{0x4148, 0x1D976DDC},
+	{0x414C, 0x1C072DD7},
+	{0x4150, 0x1127CDF4},
+	{0x4154, 0x1E37BDF1},
+	{0x4158, 0x1FB7F1D6},
+	{0x415C, 0x1EA7DDF9},
+	{0x4160, 0x1FE445DD},
+	{0x4164, 0x1F97F1FE},
+	{0x4168, 0x1FF781ED},
+	{0x416C, 0x1FA7F5FE},
+	{0x4170, 0x1E07B913},
+	{0x4174, 0x1FD7FDFF},
+	{0x4178, 0x1E17B9FA},
+	{0x417C, 0x19A66914},
+	{0x4180, 0x10F65598},
+	{0x4184, 0x14A5A111},
+	{0x4188, 0x1D3765DB},
+	{0x418C, 0x17C685CA},
+	{0x4190, 0x1107C5F3},
+	{0x4194, 0x1B5785EB},
+	{0x4198, 0x1F97ED8F},
+	{0x419C, 0x1BC7A5F3},
+	{0x41A0, 0x1FE43595},
+	{0x41A4, 0x1EB7D9FC},
+	{0x41A8, 0x1FE65DBE},
+	{0x41AC, 0x1EC7D9FC},
+	{0x41B0, 0x1976FCFF},
+	{0x41B4, 0x1F77F5FF},
+	{0x41B8, 0x1976FDEC},
+	{0x41BC, 0x198664EF},
+	{0x41C0, 0x11062D93},
+	{0x41C4, 0x10C4E910},
+	{0x41C8, 0x1CA759DB},
+	{0x41CC, 0x1335A9B5},
+	{0x41D0, 0x1097B9F3},
+	{0x41D4, 0x17B72DE1},
+	{0x41D8, 0x1F67ED42},
+	{0x41DC, 0x18074DE9},
+	{0x41E0, 0x1FD40547},
+	{0x41E4, 0x1D57ADF9},
+	{0x41E8, 0x1FE52182},
+	{0x41EC, 0x1D67B1F9},
+	{0x41F0, 0x14860CE1},
+	{0x41F4, 0x1EC7E9FE},
+	{0x41F8, 0x14860DD6},
+	{0x41FC, 0x195664C7},
+	{0x4200, 0x0005E58A},
+	{0x4204, 0x00000000},
+	{0x4208, 0x00000000},
+	{0x420C, 0x7A000000},
+	{0x4210, 0x0F9F3D7A},
+	{0x4214, 0x0040817C},
+	{0x4218, 0x00E10204},
+	{0x421C, 0x227D94CD},
+	{0x4220, 0x08028A28},
+	{0x4224, 0x00000200},
+	{0x4228, 0x04688000},
+	{0x47C0, 0x00000001},
+	{0x4A48, 0x00000002},
+	{0x4B04, 0x00000000},
+	{0x4B08, 0x00000000},
+	{0x422C, 0x0060B002},
+	{0x4230, 0x9A8249A8},
+	{0x4234, 0x26A1469E},
+	{0x4238, 0x2099A824},
+	{0x423C, 0x2359461C},
+	{0x4240, 0x1631A675},
+	{0x4244, 0x2C6B1D63},
+	{0x4248, 0x0000000E},
+	{0x424C, 0x00000001},
+	{0x4250, 0x00000001},
+	{0x4254, 0x00000000},
+	{0x4258, 0x00000000},
+	{0x425C, 0x00000000},
+	{0x4260, 0x0020000C},
+	{0x4A30, 0x00000000},
+	{0x4264, 0x00000000},
+	{0x4268, 0x00000000},
+	{0x426C, 0x0418317C},
+	{0x4270, 0x2B33135C},
+	{0x4274, 0x00000002},
+	{0x4278, 0x00000000},
+	{0x427C, 0x00000000},
+	{0x4280, 0x00000000},
+	{0x4284, 0x00000000},
+	{0x4288, 0x00000000},
+	{0x428C, 0x00000000},
+	{0x4290, 0x00000000},
+	{0x4294, 0x00000000},
+	{0x4298, 0x00000000},
+	{0x429C, 0x84026000},
+	{0x42A0, 0x0051AC20},
+	{0x4A24, 0x0010C040},
+	{0x42A4, 0x02024008},
+	{0x42A8, 0x00000000},
+	{0x42AC, 0x00000000},
+	{0x42B0, 0x22CE803C},
+	{0x42B4, 0xD8000000},
+	{0x42B8, 0x596FD67E},
+	{0x42BC, 0x7D67D67D},
+	{0x42C0, 0x7D67D65B},
+	{0x42C4, 0x28029F59},
+	{0x42C8, 0x00280280},
+	{0x4AF4, 0x00000000},
+	{0x42CC, 0x00000000},
+	{0x42D0, 0x00000000},
+	{0x42D4, 0x00000003},
+	{0x4AF8, 0x00280000},
+	{0x42D8, 0x00000001},
+	{0x42DC, 0x69AEC800},
+	{0x42E0, 0x8B4CD3D1},
+	{0x42E4, 0xC514534F},
+	{0x42E8, 0x85145145},
+	{0x42EC, 0x45145145},
+	{0x42F0, 0x05145145},
+	{0x42F4, 0x05145145},
+	{0x42F8, 0x05145145},
+	{0x42FC, 0x17659145},
+	{0x4300, 0x176DD5D9},
+	{0x4304, 0x0F65765B},
+	{0x4308, 0x0F3CF3CF},
+	{0x430C, 0x0F3CF3CF},
+	{0x4310, 0x0F3CF3CF},
+	{0x4314, 0x0F3CF3CF},
+	{0x4318, 0x0F3CF3CF},
+	{0x431C, 0x0F3CF3CF},
+	{0x4320, 0x0F3CF3CF},
+	{0x4324, 0x0F44F351},
+	{0x4328, 0x192D7547},
+	{0x432C, 0x0F5CF5CF},
+	{0x4330, 0x051593D9},
+	{0x4334, 0x05145145},
+	{0x4338, 0x05145145},
+	{0x433C, 0x05145145},
+	{0x4340, 0x05145145},
+	{0x4344, 0x05145145},
+	{0x4348, 0x19545145},
+	{0x434C, 0x1B65B5DB},
+	{0x4350, 0x1965965B},
+	{0x4354, 0x0F3CF3CF},
+	{0x4358, 0x0F3CF3CF},
+	{0x435C, 0x0F3CF1CF},
+	{0x4360, 0x0F3CF3CF},
+	{0x4364, 0x0F3CF3CF},
+	{0x4368, 0x0F3CF3CF},
+	{0x436C, 0x0F3CF3CF},
+	{0x4370, 0x0934D2CF},
+	{0x4374, 0x112CB3CF},
+	{0x4378, 0x9777A777},
+	{0x437C, 0xBB7BAC95},
+	{0x4380, 0xB667B889},
+	{0x4384, 0x7B9B8899},
+	{0x4388, 0x7A5567C8},
+	{0x438C, 0x2278CCCC},
+	{0x4390, 0x7C222222},
+	{0x4394, 0x0000029B},
+	{0x4398, 0x001CCCCC},
+	{0x4AAC, 0xCCCCC88C},
+	{0x4AB0, 0x0000AACC},
+	{0x439C, 0x00000000},
+	{0x43A0, 0x00000008},
+	{0x43A4, 0x00000000},
+	{0x43A8, 0x00000000},
+	{0x43AC, 0x00000000},
+	{0x43B0, 0x10000000},
+	{0x43B4, 0x00401001},
+	{0x43B8, 0x00061003},
+	{0x43BC, 0x000024D8},
+	{0x43C0, 0x00000000},
+	{0x43C4, 0x10000020},
+	{0x43C8, 0x20000200},
+	{0x43CC, 0x00000000},
+	{0x43D0, 0x04000000},
+	{0x43D4, 0x44000100},
+	{0x43D8, 0x60804060},
+	{0x43DC, 0x44204210},
+	{0x43E0, 0x82108082},
+	{0x43E4, 0x82108402},
+	{0x43E8, 0xC8082108},
+	{0x43EC, 0xC8202084},
+	{0x43F0, 0x44208208},
+	{0x43F4, 0x84108204},
+	{0x43F8, 0xD0108104},
+	{0x43FC, 0xF8210108},
+	{0x4400, 0x6431E930},
+	{0x4404, 0x02309468},
+	{0x4408, 0x10C61C22},
+	{0x440C, 0x02109469},
+	{0x4410, 0x10C61C22},
+	{0x4414, 0x00041049},
+	{0x4A4C, 0x00060581},
+	{0x4418, 0x00000000},
+	{0x441C, 0x00000000},
+	{0x4420, 0xEC000000},
+	{0x4424, 0xB0200020},
+	{0x4428, 0x00001FF0},
+	{0x4AC8, 0x00000001},
+	{0x4B0C, 0x00000000},
+	{0x4CDC, 0x00000000},
+	{0x442C, 0x00000000},
+	{0x4430, 0x00000000},
+	{0x4434, 0x00000000},
+	{0x4438, 0x00000000},
+	{0x443C, 0x190642D0},
+	{0x4440, 0xA80668A0},
+	{0x4444, 0x60900820},
+	{0x4448, 0x9F28518C},
+	{0x444C, 0x32488A62},
+	{0x4450, 0x9C6E36DC},
+	{0x4454, 0x0000F52B},
+	{0x4A34, 0x00000007},
+	{0x4CE0, 0x68120000},
+	{0x4CE4, 0x1A0681E0},
+	{0x4CE8, 0x94060180},
+	{0x4CEC, 0x000603FF},
+	{0x4CF0, 0xA0502000},
+	{0x4CF4, 0x00001000},
+	{0x4D00, 0x00000044},
+	{0x4B14, 0x00000000},
+	{0x4458, 0x00000000},
+	{0x445C, 0x4801442E},
+	{0x4460, 0x0051A0FA},
+	{0x4B18, 0x0000011F},
+	{0x4B1C, 0x0000011F},
+	{0x4464, 0x00000000},
+	{0x4468, 0x00000000},
+	{0x446C, 0x00000000},
+	{0x4470, 0x00000000},
+	{0x4474, 0x00000000},
+	{0x4478, 0x00000000},
+	{0x447C, 0x00000000},
+	{0x4480, 0x2A0A6040},
+	{0x4484, 0x0A0A6829},
+	{0x4488, 0x00000004},
+	{0x448C, 0x00000000},
+	{0x4490, 0x80000000},
+	{0x4494, 0x10000000},
+	{0x4498, 0xE0000000},
+	{0x4A28, 0x000ED877},
+	{0x4AB4, 0x00000000},
+	{0x4B20, 0x00000000},
+	{0x4B24, 0x00000000},
+	{0x4B28, 0x00000000},
+	{0x4B2C, 0x00000000},
+	{0x449C, 0x0000001E},
+	{0x44A0, 0x02B2C394},
+	{0x44A4, 0x00000400},
+	{0x4A2C, 0x0050240E},
+	{0x4B30, 0x7FFFFD20},
+	{0x4B34, 0x920823FF},
+	{0x4B38, 0x7FFFFFFF},
+	{0x4B3C, 0x01773773},
+	{0x44A8, 0x00000001},
+	{0x44AC, 0x000190C0},
+	{0x44B0, 0x00000000},
+	{0x44B4, 0x00000000},
+	{0x44B8, 0x00000000},
+	{0x44BC, 0x00000000},
+	{0x44C0, 0x00000000},
+	{0x44C4, 0x00000000},
+	{0x44C8, 0x00000000},
+	{0x44CC, 0x00000000},
+	{0x44D0, 0x00000000},
+	{0x44D4, 0x00000000},
+	{0x44D8, 0x00000000},
+	{0x44DC, 0x00000000},
+	{0x44E0, 0x00000000},
+	{0x44E4, 0x00000000},
+	{0x44E8, 0x00000000},
+	{0x44EC, 0x00000000},
+	{0x44F0, 0x00000000},
+	{0x44F4, 0x00000000},
+	{0x44F8, 0x00000000},
+	{0x44FC, 0x00000000},
+	{0x4500, 0x00000000},
+	{0x4504, 0x00000000},
+	{0x4508, 0x00000000},
+	{0x450C, 0x00000000},
+	{0x4510, 0x00000000},
+	{0x4514, 0x00000000},
+	{0x4518, 0x00000000},
+	{0x451C, 0x00000000},
+	{0x4520, 0x00000000},
+	{0x4524, 0x00000000},
+	{0x4528, 0x00000000},
+	{0x452C, 0x00000000},
+	{0x4530, 0x4E830171},
+	{0x4534, 0x00000870},
+	{0x4538, 0x000000FF},
+	{0x453C, 0x00000000},
+	{0x4540, 0x00000000},
+	{0x4544, 0x00000000},
+	{0x4548, 0x00000000},
+	{0x454C, 0x00000000},
+	{0x4550, 0x00000000},
+	{0x4554, 0x00000000},
+	{0x4558, 0x00000000},
+	{0x455C, 0x00000000},
+	{0x4560, 0x40000000},
+	{0x4564, 0x40000000},
+	{0x4568, 0x00000000},
+	{0x456C, 0x20000000},
+	{0x4570, 0x04F040BB},
+	{0x4574, 0x000E53FF},
+	{0x4578, 0x000205CB},
+	{0x457C, 0x00200000},
+	{0x4580, 0x00000040},
+	{0x4584, 0x00000000},
+	{0x4588, 0x00000017},
+	{0x458C, 0x30000000},
+	{0x4590, 0x00000000},
+	{0x4594, 0x00000000},
+	{0x4598, 0x00000001},
+	{0x459C, 0x0003FE00},
+	{0x45A0, 0x00000086},
+	{0x45A4, 0x00000000},
+	{0x45A8, 0xC00001C0},
+	{0x45AC, 0x78038000},
+	{0x45B0, 0x8000004A},
+	{0x45B4, 0x04094800},
+	{0x45B8, 0x00280002},
+	{0x45BC, 0x06748790},
+	{0x45C0, 0x80000000},
+	{0x45C4, 0x00000000},
+	{0x45C8, 0x00000000},
+	{0x45CC, 0x00558670},
+	{0x45D0, 0x002883F0},
+	{0x45D4, 0x00090120},
+	{0x45D8, 0x00000000},
+	{0x45E0, 0xA3A6D3C4},
+	{0x45E4, 0xAB27B126},
+	{0x45E8, 0x00006778},
+	{0x45F4, 0x000001B5},
+	{0x45EC, 0x11110F0A},
+	{0x45F0, 0x00000003},
+	{0x4A0C, 0x0000000A},
+	{0x45F8, 0x0058BC3F},
+	{0x45FC, 0x00000003},
+	{0x462C, 0x00000020},
+	{0x4600, 0x000003D9},
+	{0x45F0, 0x00000004},
+	{0x4604, 0x002B1CB0},
+	{0x4A50, 0xC0000000},
+	{0x4A54, 0x00001000},
+	{0x4A58, 0x00000000},
+	{0x4A18, 0x00000024},
+	{0x4608, 0x00000001},
+	{0x460C, 0x00000000},
+	{0x4A10, 0x00000001},
+	{0x4610, 0x00000001},
+	{0x4614, 0x16E5298F},
+	{0x4618, 0x18C6294A},
+	{0x461C, 0x0E06318A},
+	{0x4620, 0x0E539CE5},
+	{0x4624, 0x00019287},
+	{0x4A14, 0x000000BF},
+	{0x4628, 0x00000001},
+	{0x4630, 0x000001AA},
+	{0x4A18, 0x00001900},
+	{0x4A1C, 0x000002A6},
+	{0x4634, 0x000000A3},
+	{0x4A20, 0x00000086},
+	{0x4638, 0x00045656},
+	{0x49F8, 0x00000000},
+	{0x463C, 0x00000000},
+	{0x4640, 0x00000000},
+	{0x4644, 0x00C8CC00},
+	{0x4648, 0xC400B6B6},
+	{0x464C, 0xDC400FC0},
+	{0x4A44, 0x00000000},
+	{0x4A8C, 0x00000110},
+	{0x4BC4, 0x00000001},
+	{0x4650, 0x08882550},
+	{0x4654, 0x08CC2660},
+	{0x4658, 0x09102660},
+	{0x465C, 0x00000154},
+	{0x45DC, 0xC39E38E8},
+	{0x4660, 0x452607E6},
+	{0x4664, 0x6750DC65},
+	{0x4668, 0xF3F0F1ED},
+	{0x466C, 0x30141506},
+	{0x4670, 0x2C2B2B2B},
+	{0x4674, 0x2C2C2C2C},
+	{0x4678, 0xDDB738E8},
+	{0x467C, 0x543618FB},
+	{0x4680, 0x4F31DC6F},
+	{0x4684, 0xFBEBDA00},
+	{0x4688, 0x1A10FF04},
+	{0x468C, 0x282A3000},
+	{0x4690, 0x2A29292A},
+	{0x4694, 0x04FA2A2A},
+	{0x4698, 0xEE0F04D1},
+	{0x469C, 0x99E91436},
+	{0x46A0, 0x0701E79E},
+	{0x46A4, 0x08D77CFF},
+	{0x46A8, 0x321AFF14},
+	{0x46AC, 0x60313447},
+	{0x46B0, 0x63666666},
+	{0x46B4, 0x35374425},
+	{0x46B8, 0x35883042},
+	{0x46BC, 0x5177C252},
+	{0x4720, 0x7FFFFD63},
+	{0x4724, 0xB58D11FF},
+	{0x4728, 0x47FFFFFF},
+	{0x472C, 0x0E7893B6},
+	{0x4730, 0xE0391201},
+	{0x4734, 0x00000020},
+	{0x4738, 0x8325C500},
+	{0x473C, 0x00000B7F},
+	{0x46C0, 0x00000000},
+	{0x46C4, 0x00000000},
+	{0x46C8, 0x00000219},
+	{0x4BDC, 0x00002020},
+	{0x46CC, 0x00000000},
+	{0x46D0, 0x00000000},
+	{0x4A3C, 0x00000002},
+	{0x46D4, 0x00000001},
+	{0x46D8, 0x00000001},
+	{0x46DC, 0x00000000},
+	{0x46E0, 0x00000000},
+	{0x46E4, 0x00000151},
+	{0x46E8, 0x00000498},
+	{0x46EC, 0x00000498},
+	{0x46F0, 0x00000000},
+	{0x46F4, 0x00000000},
+	{0x46F8, 0x00001146},
+	{0x46FC, 0x00000000},
+	{0x4700, 0x00000000},
+	{0x4704, 0x00C8CC00},
+	{0x4708, 0xC400B6B6},
+	{0x470C, 0xDC400FC0},
+	{0x4A90, 0x00000110},
+	{0x4B10, 0x00000000},
+	{0x4BE0, 0x00000001},
+	{0x4710, 0x08882550},
+	{0x4714, 0x08CC2660},
+	{0x4718, 0x09102660},
+	{0x471C, 0x00000154},
+	{0x4740, 0xC69F38E8},
+	{0x4744, 0x462709E9},
+	{0x4748, 0x6750DC67},
+	{0x474C, 0xF3F0F1ED},
+	{0x4750, 0x30141506},
+	{0x4754, 0x2C2B2B2B},
+	{0x4758, 0x2C2C2C2C},
+	{0x475C, 0xE0B738E8},
+	{0x4760, 0x52381BFE},
+	{0x4764, 0x5031DC6C},
+	{0x4768, 0xFBEBDA00},
+	{0x476C, 0x1A10FF04},
+	{0x4770, 0x282A3000},
+	{0x4774, 0x2A29292A},
+	{0x4778, 0x04FA2A2A},
+	{0x477C, 0xEE0F04D1},
+	{0x47C4, 0x00000000},
+	{0x47C8, 0xA32103FE},
+	{0x47CC, 0xB20A5328},
+	{0x47D0, 0xC686314F},
+	{0x47D4, 0x000004D7},
+	{0x4BFC, 0x00000000},
+	{0x4C00, 0x0C442416},
+	{0x4C04, 0x00000000},
+	{0x47D8, 0x009B902A},
+	{0x47DC, 0x009B902A},
+	{0x47E0, 0x98682C18},
+	{0x47E4, 0x6318C4C1},
+	{0x47E8, 0x6248C631},
+	{0x47EC, 0x922A8253},
+	{0x47F0, 0x00000005},
+	{0x47F4, 0x00001759},
+	{0x47F8, 0x4BB01800},
+	{0x47FC, 0x831408BE},
+	{0x4A84, 0x000000E9},
+	{0x4C08, 0x0F801404},
+	{0x4C0C, 0x00A2B404},
+	{0x4800, 0x9ABBCACB},
+	{0x4804, 0x56867578},
+	{0x4808, 0xBCCBBB13},
+	{0x480C, 0x7889989B},
+	{0x4810, 0xBBB0F455},
+	{0x4814, 0x777BBBBB},
+	{0x4818, 0x15277777},
+	{0x481C, 0x27039CE9},
+	{0x4820, 0x42424432},
+	{0x4824, 0x36058342},
+	{0x4828, 0x00000006},
+	{0x482C, 0x00000005},
+	{0x4830, 0x00000005},
+	{0x4834, 0xC7013016},
+	{0x4838, 0x84413016},
+	{0x483C, 0x84413016},
+	{0x4840, 0x8C413016},
+	{0x4844, 0x8C40B028},
+	{0x4848, 0x3140B028},
+	{0x484C, 0x2940B028},
+	{0x4850, 0x8440B028},
+	{0x4854, 0x2318C610},
+	{0x4858, 0x45344753},
+	{0x485C, 0x236A6A88},
+	{0x4860, 0xAC8DF814},
+	{0x4864, 0x08877ACB},
+	{0x4868, 0x000107AA},
+	{0x4A94, 0x00000000},
+	{0x486C, 0xBCEB4A14},
+	{0x4870, 0x000A3A4A},
+	{0x4874, 0xBCEB4A14},
+	{0x4878, 0x000A3A4A},
+	{0x487C, 0xBCBDBD85},
+	{0x4880, 0x0CABB99A},
+	{0x4884, 0x38384242},
+	{0x4888, 0x0086102E},
+	{0x488C, 0xCA24C82A},
+	{0x4AFC, 0x00000000},
+	{0x4C14, 0x0000349D},
+	{0x4CF8, 0x00000007},
+	{0x4890, 0x00008A62},
+	{0x4894, 0x00000008},
+	{0x4958, 0x80040000},
+	{0x495C, 0x80040000},
+	{0x4960, 0xFE800000},
+	{0x4964, 0x834C0000},
+	{0x4968, 0x00000000},
+	{0x496C, 0x00000000},
+	{0x4970, 0x00000000},
+	{0x4974, 0x00000000},
+	{0x4978, 0x00000000},
+	{0x497C, 0x00000000},
+	{0x4980, 0x40000000},
+	{0x4984, 0x00000000},
+	{0x4988, 0x00000000},
+	{0x498C, 0x00000000},
+	{0x4990, 0x00000000},
+	{0x4994, 0x04065800},
+	{0x4998, 0x02004080},
+	{0x499C, 0x0E1E3E05},
+	{0x49A0, 0x0A163068},
+	{0x49A4, 0x00206040},
+	{0x49A8, 0x02020202},
+	{0x49AC, 0x00002020},
+	{0x49B0, 0xF8F8F418},
+	{0x49B4, 0xF8E8F8F8},
+	{0x49B8, 0xF80808E8},
+	{0x4A00, 0xF8F8FA00},
+	{0x4A04, 0xFAFAFAF8},
+	{0x4A08, 0xFAFAFAFA},
+	{0x49BC, 0x00000000},
+	{0x49C0, 0x800C562D},
+	{0x49C4, 0x00000101},
+	{0x49C8, 0x00000000},
+	{0x49CC, 0x00000000},
+	{0x49D0, 0x00000000},
+	{0x49D4, 0x00000000},
+	{0x49D8, 0x00000000},
+	{0x49DC, 0x00000000},
+	{0x49E0, 0x00000000},
+	{0x49E4, 0x00000000},
+	{0x49E8, 0x00000000},
+	{0x49EC, 0x00000000},
+	{0x4C28, 0x00000000},
+	{0x4C2C, 0x00000000},
+	{0x4C30, 0x00000000},
+	{0x4C34, 0x00000000},
+	{0x4C38, 0x00000000},
+	{0x4C3C, 0x00000000},
+	{0x4C40, 0x00000000},
+	{0x4C44, 0x01C0C832},
+	{0x4C48, 0x03207032},
+	{0x4C4C, 0x0320701C},
+	{0x4C50, 0x03207032},
+	{0x4C54, 0x01C0C81C},
+	{0x4C58, 0x00A0281C},
+	{0x4C5C, 0x0320C80A},
+	{0x4C60, 0x00A0C832},
+	{0x4C64, 0x01C0C832},
+	{0x4C68, 0x03207032},
+	{0x4C6C, 0x0320701C},
+	{0x4C70, 0x03207032},
+	{0x4C74, 0x01C0C81C},
+	{0x4C78, 0x00A0281C},
+	{0x4C7C, 0x0321A80A},
+	{0x4C80, 0x0320C86A},
+	{0x4C84, 0x12B02832},
+	{0x4C88, 0x12B3292B},
+	{0x4C8C, 0x0CA4ACCA},
+	{0x4C90, 0x12B4AC6A},
+	{0x4C94, 0x0CA4ACCA},
+	{0x4C98, 0x06A3292B},
+	{0x4C9C, 0x06A0280A},
+	{0x4CA0, 0x0CA0286A},
+	{0x4CA4, 0x0CA1A8CA},
+	{0x4CA8, 0x06A3286A},
+	{0x4CAC, 0x0000000A},
+	{0x4CB0, 0x01209C27},
+	{0x4CB4, 0x02704800},
+	{0x4CB8, 0x02704812},
+	{0x4CBC, 0x00004827},
+	{0x4CC0, 0x01209C12},
+	{0x4CC4, 0x00000012},
+	{0x4CC8, 0x02718000},
+	{0x4CCC, 0x02709C60},
+	{0x4CD0, 0x00000027},
+	{0x4CD4, 0x00000000},
+	{0x4CD8, 0x0000014A},
+	{0x994, 0x00000010},
+	{0x904, 0x00000005},
+	{0x708, 0x00000000},
+	{0x884, 0x0043F01D},
+	{0x710, 0xEF810000},
+	{0x718, 0x1333233F},
+	{0x604, 0x041E1E1E},
+	{0x714, 0x00010000},
+	{0x586C, 0x000000F0},
+	{0x586C, 0x000000E0},
+	{0x586C, 0x000000D0},
+	{0x586C, 0x000000C0},
+	{0x586C, 0x000000B0},
+	{0x586C, 0x000000A0},
+	{0x586C, 0x00000090},
+	{0x586C, 0x00000080},
+	{0x586C, 0x00000070},
+	{0x586C, 0x00000060},
+	{0x586C, 0x00000050},
+	{0x586C, 0x00000040},
+	{0x586C, 0x00000030},
+	{0x586C, 0x00000020},
+	{0x586C, 0x00000010},
+	{0x586C, 0x00000000},
+	{0xC0D4, 0xABA41460},
+	{0xC0D8, 0xC43A7E87},
+	{0xC0DC, 0x30C194B8},
+	{0xC0E0, 0x75008138},
+	{0xC0E4, 0x0000272B},
+	{0xC0E8, 0x000A0C81},
+	{0xC0EC, 0x00030003},
+	{0xC0F0, 0x00000024},
+	{0xC0C4, 0x005E3A00},
+	{0xC004, 0x45800000},
+	{0xC024, 0x45800000},
+	{0x334, 0xFFFFFFFF},
+	{0x33C, 0x55000000},
+	{0x340, 0x00005555},
+	{0x724, 0x00111200},
+	{0x5868, 0xA9550000},
+	{0x5870, 0x33221100},
+	{0x5874, 0x77665544},
+	{0x5878, 0xBBAA9988},
+	{0x587C, 0xFFEEDDCC},
+	{0x5880, 0x76543210},
+	{0x5884, 0xFEDCBA98},
+	{0x5888, 0x00000000},
+	{0x588C, 0x00000000},
+	{0x5894, 0x00000008},
+	{0x650, 0x00200888},
+	{0x710, 0xF3810000},
+	{0x020, 0x0000F381},
+	{0x024, 0x0000F381},
+	{0x000, 0xC580801E},
+	{0x980, 0x10002250},
+	{0x988, 0x3C3C4107},
+	{0x994, 0x00000010},
+	{0x000, 0x0580801F},
+	{0x240C, 0x00000000},
+	{0x640, 0x210A141E},
+	{0x640, 0x2114141E},
+	{0x640, 0x2114141E},
+	{0x644, 0x3414283C},
+	{0x644, 0x3425283C},
+	{0x644, 0x3426283C},
+	{0x2640, 0x140A141E},
+	{0x2640, 0x1414141E},
+	{0x2640, 0x1414141E},
+	{0x2644, 0x3414283C},
+	{0x2644, 0x3425283C},
+	{0x2644, 0x3425183C},
+	{0x2300, 0x02748790},
+	{0x2304, 0x00558670},
+	{0x2308, 0x002883F0},
+	{0x230C, 0x00090120},
+	{0x2310, 0x00000000},
+	{0x2314, 0x06000000},
+	{0x2318, 0x00000000},
+	{0x231C, 0x00000000},
+	{0x2320, 0x03020100},
+	{0x2324, 0x07060504},
+	{0x2328, 0x0B0A0908},
+	{0x232C, 0x0F0E0D0C},
+	{0x2330, 0x13121110},
+	{0x2334, 0x17161514},
+	{0x2338, 0x0C700022},
+	{0x233C, 0x0A0529D0},
+	{0x2340, 0x000529D0},
+	{0x2344, 0x0006318A},
+	{0x2348, 0xB7E6318A},
+	{0x234C, 0x80039C00},
+	{0x2350, 0x80039C00},
+	{0x2354, 0x0005298F},
+	{0x2358, 0x0015296E},
+	{0x235C, 0x0C07FC31},
+	{0x2360, 0x0219AAAE},
+	{0x2364, 0xE4F624C3},
+	{0x2368, 0x53626F15},
+	{0x236C, 0x48000000},
+	{0x2370, 0x48000000},
+	{0x2374, 0x07540000},
+	{0x2378, 0x202401B9},
+	{0x237C, 0x00F7000E},
+	{0x2380, 0x0F0A1111},
+	{0x2384, 0x30D9000F},
+	{0x2388, 0x0200EA02},
+	{0x238C, 0x003CB061},
+	{0x2390, 0x69C00000},
+	{0x2394, 0x00000000},
+	{0x2398, 0x000000F0},
+	{0x239C, 0x0001FFFF},
+	{0x23A0, 0x00C80064},
+	{0x23A4, 0x0190012C},
+	{0x23A8, 0x001917BE},
+	{0x23AC, 0x0B30880C},
+	{0x23B0, 0x9281CE00},
+	{0x23B4, 0x7F027C00},
+	{0x704, 0x601E0502},
+	{0x5600, 0x00000000},
+	{0x5604, 0x802D2721},
+	{0x5610, 0x00201020},
+	{0x5618, 0x00801008},
+	{0x5624, 0x0808081E},
+	{0x562C, 0x0000081D},
+	{0x5634, 0x3D2EE000},
+	{0x5638, 0x0001AC42},
+	{0x5640, 0x3D6EF000},
+	{0x5644, 0x0001AC3E},
+	{0x566C, 0x00210005},
+	{0x5680, 0x20500010},
+	{0x5684, 0x00020001},
+	{0x56A0, 0x0034C000},
+	{0x56BC, 0x04000000},
+	{0x56C0, 0x00000688},
+	{0x56C4, 0x00000010},
+	{0x56C8, 0x0E800400},
+	{0x56CC, 0x01E400FF},
+	{0x5800, 0x003F807F},
+	{0x5810, 0x59008400},
+	{0x5814, 0x201AF000},
+	{0x5818, 0x182C18E8},
+	{0x581C, 0x3DD80280},
+	{0x5820, 0x80000080},
+	{0x5828, 0x023F8121},
+	{0x5830, 0x023F8121},
+	{0x5838, 0x003F8121},
+	{0x5840, 0x023F8121},
+	{0x5848, 0x023F8121},
+	{0x5850, 0x023F8121},
+	{0x5858, 0x003F7121},
+	{0x5860, 0x023F7121},
+	{0x5864, 0x1A1801FF},
+	{0x5868, 0xA9A90002},
+	{0x5880, 0x77777777},
+	{0x5884, 0x77777777},
+	{0x5894, 0x01080604},
+	{0x5898, 0x00000000},
+	{0x589C, 0x00000000},
+	{0x58A0, 0x000000FE},
+	{0x58B0, 0x00000800},
+	{0x58BC, 0x07A7807F},
+	{0x58C0, 0x007E0000},
+	{0x58C4, 0x0003FFFF},
+	{0x58D4, 0x7401FE00},
+	{0x58D8, 0x8008016C},
+	{0x58DC, 0xC000807F},
+	{0x58E4, 0x3000881F},
+	{0x58E8, 0x00000003},
+	{0x58F0, 0x400401FF},
+	{0x58F4, 0x80000000},
+	{0x58F8, 0xC0000000},
+	{0x58FC, 0x00000000},
+	{0x700, 0x40000030},
+	{0x704, 0x601E0502},
+	{0x704, 0x601E0500},
+	{0x704, 0x601E0502},
+	{0x20FC, 0x00000000},
+	{0x20F8, 0x00000000},
+	{0x20F0, 0x00000000},
+	{0x9C0, 0x00000001},
+	{0x9C0, 0x00000000},
+	{0x9C0, 0x00000001},
+	{0x9C0, 0x00000000},
+	{0x4AE8, 0x00000744},
+	{0x4AD4, 0x00000040},
+	{0x4AE4, 0x0079E99E},
+	{0x4BC8, 0xFBD5B89F},
+	{0x4BCC, 0x99563918},
+	{0x4BD0, 0x12EED5B8},
+	{0x4BD4, 0x6F7D542F},
+	{0x4BD8, 0x0000001D},
+	{0x300, 0xF30CE31C},
+	{0x304, 0x13EF1F19},
+	{0x308, 0x0C0CF3F3},
+	{0x30C, 0x0CE30C0C},
+	{0x310, 0x80496000},
+	{0x314, 0x0041E000},
+	{0x318, 0x20022042},
+	{0x31C, 0x20448009},
+	{0x320, 0x00010031},
+	{0x324, 0xE000E000},
+	{0x328, 0xE000E000},
+	{0x32C, 0xE0008049},
+	{0x12BC, 0x10104041},
+	{0x12C0, 0x13311111},
+	{0x12E4, 0x30D52A68},
+	{0x010, 0x0005FFFF},
+	{0x028, 0x0000F381},
+	{0x02C, 0x0000F381},
+	{0x620, 0x00141230},
+	{0x70C, 0x00000020},
+	{0x720, 0x20000000},
+	{0x730, 0x00000002},
+	{0x738, 0x004100C0},
+	{0x73C, 0x00000002},
+	{0x748, 0x01000002},
+	{0x74C, 0x00000001},
+	{0xA08, 0x00007800},
+	{0xC14, 0x25010000},
+	{0xC3C, 0x2840E1BF},
+	{0xC40, 0x00000000},
+	{0xC44, 0x00000007},
+	{0xC48, 0x410E4000},
+	{0xC54, 0x1EE14368},
+	{0xC58, 0x41000000},
+	{0xC5C, 0x80558000},
+	{0xC60, 0x017FFFF2},
+	{0xC64, 0x0010A130},
+	{0xC68, 0x90000050},
+	{0xC6C, 0x10201021},
+	{0xC70, 0x071B0660},
+	{0xC74, 0x00000000},
+	{0xC78, 0x80000000},
+	{0xC7C, 0x0020BFE0},
+	{0xC88, 0xC2AC8000},
+	{0xC8C, 0x02F2FC08},
+	{0xD00, 0x77777777},
+	{0xD04, 0xBBBBBBBB},
+	{0xD08, 0xBBBBBBBB},
+	{0xD0C, 0x000B2070},
+	{0xD10, 0x20110FFF},
+	{0xD18, 0x50009800},
+	{0xD20, 0x01900000},
+	{0xD30, 0x03FF8000},
+	{0xD40, 0xF64FA0F7},
+	{0xD44, 0x0401463F},
+	{0xD48, 0x0003FF7F},
+	{0xD4C, 0x00000000},
+	{0xD50, 0xF64FA0F7},
+	{0xD54, 0x04100437},
+	{0xD58, 0x0000FF7F},
+	{0xD5C, 0x00000000},
+	{0xD60, 0x00000000},
+	{0xD64, 0x00000000},
+	{0xD70, 0x00000015},
+	{0xD78, 0x00000001},
+	{0xD7C, 0x001D050E},
+	{0xD80, 0x00000100},
+	{0xD84, 0x00006607},
+	{0xD90, 0x000003FF},
+	{0xD94, 0x00000000},
+	{0xD98, 0x0000003F},
+	{0xD9C, 0x00000000},
+	{0xDA0, 0x000003FE},
+	{0xDA4, 0x00000000},
+	{0xDA8, 0x0000003F},
+	{0xDAC, 0x00000000},
+	{0xDD4, 0x00000000},
+	{0x1010, 0x00000000},
+	{0x2000, 0x50BBBF04},
+	{0x2008, 0x000FFFFF},
+	{0x5800, 0x03FF807F},
+	{0x5804, 0x04237040},
+	{0x5808, 0x04237040},
+	{0x5818, 0x082C1800},
+	{0x624, 0x0101030A},
+	{0x241C, 0x00000001},
+	{0xC0F8, 0x00000001},
+	{0x35C, 0x000004C4},
+	{0x1200, 0x00010142},
+	{0x120C, 0x00012233},
+	{0x1210, 0x8049E304},
+	{0x12A0, 0x49107056},
+	{0x12A8, 0x33337025},
+	{0x12AC, 0x12333121},
+	{0x12B8, 0x30020000},
+	{0x0F0, 0x00000001},
+	{0x0F4, 0x00000011},
+	{0x0F8, 0x20230307},
+};
+
+static const struct rtw89_reg2_def rtw89_8851b_phy_bb_reg_gain[] = {
+	{0xF00100FF, 0x00000000},
+	{0xF00200FF, 0x00000001},
+	{0xF00300FF, 0x00000002},
+	{0xF00400FF, 0x00000003},
+	{0xF00500FF, 0x00000004},
+	{0xF00600FF, 0x00000005},
+	{0x800100ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x000, 0x13F6D7B6},
+	{0x001, 0x00725132},
+	{0x002, 0x00005A38},
+	{0x900200ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x000, 0x13F6D7B6},
+	{0x001, 0x00725132},
+	{0x002, 0x00005A38},
+	{0x900300ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x000, 0x13F6D7B6},
+	{0x001, 0x00725132},
+	{0x002, 0x00005A38},
+	{0x900400ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x000, 0x19FADCBC},
+	{0x001, 0x007A5A3A},
+	{0x002, 0x00005838},
+	{0x900500ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x000, 0x19FADCBC},
+	{0x001, 0x007A5A3A},
+	{0x002, 0x00005838},
+	{0x900600ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x000, 0x19FADCBC},
+	{0x001, 0x007A5A3A},
+	{0x002, 0x00005838},
+	{0xA0000000, 0x00000000},
+	{0x000, 0x13F6D7B6},
+	{0x001, 0x00725132},
+	{0x002, 0x00005A38},
+	{0xB0000000, 0x00000000},
+	{0x800100ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x100, 0x1BFEE0B7},
+	{0x101, 0x006C5238},
+	{0x102, 0x00005031},
+	{0x900200ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x100, 0x1BFEE0B7},
+	{0x101, 0x006C5238},
+	{0x102, 0x00005031},
+	{0x900300ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x100, 0x1BFEE0B7},
+	{0x101, 0x006C5238},
+	{0x102, 0x00005031},
+	{0x900400ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x100, 0x1BFEE0B7},
+	{0x101, 0x006C5238},
+	{0x102, 0x00005031},
+	{0x900500ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x100, 0x1BFEE0B7},
+	{0x101, 0x006C5238},
+	{0x102, 0x00005031},
+	{0x900600ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x100, 0x1BFEE0B7},
+	{0x101, 0x006C5238},
+	{0x102, 0x00005031},
+	{0xA0000000, 0x00000000},
+	{0x100, 0x1BFEE0B7},
+	{0x101, 0x006C5238},
+	{0x102, 0x00005031},
+	{0xB0000000, 0x00000000},
+	{0x800100ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10000, 0x19F8D8C1},
+	{0x10001, 0x006F4F31},
+	{0x10002, 0x00006F58},
+	{0x900200ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10000, 0x19F8D8C1},
+	{0x10001, 0x006F4F31},
+	{0x10002, 0x00006F58},
+	{0x900300ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10000, 0x19F8D8C1},
+	{0x10001, 0x006F4F31},
+	{0x10002, 0x00006F58},
+	{0x900400ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10000, 0x1DF8DAC1},
+	{0x10001, 0x00755437},
+	{0x10002, 0x00007058},
+	{0x900500ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10000, 0x1DF8DAC1},
+	{0x10001, 0x00755437},
+	{0x10002, 0x00007058},
+	{0x900600ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10000, 0x1DF8DAC1},
+	{0x10001, 0x00755437},
+	{0x10002, 0x00007058},
+	{0xA0000000, 0x00000000},
+	{0x10000, 0x19F8D8C1},
+	{0x10001, 0x006F4F31},
+	{0x10002, 0x00006F58},
+	{0xB0000000, 0x00000000},
+	{0x800100ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10100, 0x09E9C69F},
+	{0x10101, 0x00674627},
+	{0x10102, 0x00006750},
+	{0x900200ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10100, 0x09E9C69F},
+	{0x10101, 0x00674627},
+	{0x10102, 0x00006750},
+	{0x900300ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10100, 0x09E9C69F},
+	{0x10101, 0x00674627},
+	{0x10102, 0x00006750},
+	{0x900400ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10100, 0x09E9C69F},
+	{0x10101, 0x00674627},
+	{0x10102, 0x00006750},
+	{0x900500ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10100, 0x09E9C69F},
+	{0x10101, 0x00674627},
+	{0x10102, 0x00006750},
+	{0x900600ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10100, 0x09E9C69F},
+	{0x10101, 0x00674627},
+	{0x10102, 0x00006750},
+	{0xA0000000, 0x00000000},
+	{0x10100, 0x09E9C69F},
+	{0x10101, 0x00674627},
+	{0x10102, 0x00006750},
+	{0xB0000000, 0x00000000},
+	{0x800100ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20000, 0x1AF0D2B8},
+	{0x20001, 0x00755334},
+	{0x20002, 0x00006F58},
+	{0x900200ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20000, 0x1AF0D2B8},
+	{0x20001, 0x00755334},
+	{0x20002, 0x00006F58},
+	{0x900300ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20000, 0x1AF0D2B8},
+	{0x20001, 0x00755334},
+	{0x20002, 0x00006F58},
+	{0x900400ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20000, 0x1D00E2C8},
+	{0x20001, 0x00775336},
+	{0x20002, 0x00006D58},
+	{0x900500ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20000, 0x1D00E2C8},
+	{0x20001, 0x00775336},
+	{0x20002, 0x00006D58},
+	{0x900600ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20000, 0x1D00E2C8},
+	{0x20001, 0x00775336},
+	{0x20002, 0x00006D58},
+	{0xA0000000, 0x00000000},
+	{0x20000, 0x1AF0D2B8},
+	{0x20001, 0x00755334},
+	{0x20002, 0x00006F58},
+	{0xB0000000, 0x00000000},
+	{0x800100ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20100, 0x07E9C6A0},
+	{0x20101, 0x00674728},
+	{0x20102, 0x00006850},
+	{0x900200ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20100, 0x07E9C6A0},
+	{0x20101, 0x00674728},
+	{0x20102, 0x00006850},
+	{0x900300ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20100, 0x07E9C6A0},
+	{0x20101, 0x00674728},
+	{0x20102, 0x00006850},
+	{0x900400ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20100, 0x07E9C6A0},
+	{0x20101, 0x00674728},
+	{0x20102, 0x00006850},
+	{0x900500ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20100, 0x07E9C6A0},
+	{0x20101, 0x00674728},
+	{0x20102, 0x00006850},
+	{0x900600ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x20100, 0x07E9C6A0},
+	{0x20101, 0x00674728},
+	{0x20102, 0x00006850},
+	{0xA0000000, 0x00000000},
+	{0x20100, 0x07E9C6A0},
+	{0x20101, 0x00674728},
+	{0x20102, 0x00006850},
+	{0xB0000000, 0x00000000},
+	{0x800100ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30000, 0x15EED2B6},
+	{0x30001, 0x006F4D2F},
+	{0x30002, 0x00006F58},
+	{0x900200ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30000, 0x15EED2B6},
+	{0x30001, 0x006F4D2F},
+	{0x30002, 0x00006F58},
+	{0x900300ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30000, 0x15EED2B6},
+	{0x30001, 0x006F4D2F},
+	{0x30002, 0x00006F58},
+	{0x900400ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30000, 0x1F00E2C6},
+	{0x30001, 0x00795739},
+	{0x30002, 0x00006F58},
+	{0x900500ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30000, 0x1F00E2C6},
+	{0x30001, 0x00795739},
+	{0x30002, 0x00006F58},
+	{0x900600ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30000, 0x1F00E2C6},
+	{0x30001, 0x00795739},
+	{0x30002, 0x00006F58},
+	{0xA0000000, 0x00000000},
+	{0x30000, 0x15EED2B6},
+	{0x30001, 0x006F4D2F},
+	{0x30002, 0x00006F58},
+	{0xB0000000, 0x00000000},
+	{0x800100ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30100, 0x06E9C69F},
+	{0x30101, 0x00654527},
+	{0x30102, 0x00006750},
+	{0x900200ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30100, 0x06E9C69F},
+	{0x30101, 0x00654527},
+	{0x30102, 0x00006750},
+	{0x900300ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30100, 0x06E9C69F},
+	{0x30101, 0x00654527},
+	{0x30102, 0x00006750},
+	{0x900400ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30100, 0x06E9C69F},
+	{0x30101, 0x00654527},
+	{0x30102, 0x00006750},
+	{0x900500ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30100, 0x06E9C69F},
+	{0x30101, 0x00654527},
+	{0x30102, 0x00006750},
+	{0x900600ff, 0x00000000}, {0x40000000, 0x00000000},
+	{0x30100, 0x06E9C69F},
+	{0x30101, 0x00654527},
+	{0x30102, 0x00006750},
+	{0xA0000000, 0x00000000},
+	{0x30100, 0x06E9C69F},
+	{0x30101, 0x00654527},
+	{0x30102, 0x00006750},
+	{0xB0000000, 0x00000000},
+	{0x1000000, 0x000000F4},
+	{0x1000010, 0x000000F8},
+	{0x1000011, 0x0000F8F8},
+	{0x1000100, 0x000000F8},
+	{0x1000110, 0x00000000},
+	{0x1000111, 0x00000000},
+	{0x1010000, 0x000000F4},
+	{0x1010010, 0x000000F8},
+	{0x1010011, 0x0000F8F8},
+	{0x1010020, 0x000000F8},
+	{0x1010021, 0x0808E8E8},
+	{0x1010029, 0x0000F8F8},
+	{0x1010100, 0x000000F4},
+	{0x1010110, 0x000000F8},
+	{0x1010111, 0x0000F8F8},
+	{0x1010120, 0x000000F8},
+	{0x1010121, 0x0808E8E8},
+	{0x1010129, 0x0000F8F8},
+	{0x1020000, 0x000000F4},
+	{0x1020010, 0x000000F8},
+	{0x1020011, 0x0000F8F8},
+	{0x1020020, 0x000000F8},
+	{0x1020021, 0x0808E8E8},
+	{0x1020029, 0x0000F8F8},
+	{0x1020100, 0x000000F4},
+	{0x1020110, 0x000000F8},
+	{0x1020111, 0x0000F8F8},
+	{0x1020120, 0x000000F8},
+	{0x1020121, 0x0808E8E8},
+	{0x1020129, 0x0000F8F8},
+	{0x1030000, 0x000000F4},
+	{0x1030010, 0x000000F8},
+	{0x1030011, 0x0000F8F8},
+	{0x1030020, 0x000000F8},
+	{0x1030021, 0x0808E8E8},
+	{0x1030029, 0x0000F8F8},
+	{0x1030100, 0x000000F4},
+	{0x1030110, 0x000000F8},
+	{0x1030111, 0x0000F8F8},
+	{0x1030120, 0x000000F8},
+	{0x1030121, 0x0808E8E8},
+	{0x1030129, 0x0000F8F8},
+};
+
+static const struct rtw89_reg2_def rtw89_8851b_phy_radioa_regs[] = {
+	{0xF0010000, 0x00000000},
+	{0xF0020000, 0x00000001},
+	{0xF0030000, 0x00000002},
+	{0x000, 0x00030000},
+	{0x018, 0x00013124},
+	{0x0EF, 0x00080000},
+	{0x033, 0x00000008},
+	{0x03E, 0x00000110},
+	{0x03F, 0x0000D39C},
+	{0x033, 0x0000000C},
+	{0x03E, 0x00000110},
+	{0x03F, 0x0000F79E},
+	{0x0EF, 0x00000000},
+	{0x01B, 0x00003A40},
+	{0x08F, 0x000C170E},
+	{0x08E, 0x00005160},
+	{0x002, 0x00000600},
+	{0x0EE, 0x00000002},
+	{0x033, 0x00000002},
+	{0x03F, 0x0000003F},
+	{0x033, 0x00000003},
+	{0x03F, 0x0000003F},
+	{0x033, 0x00000004},
+	{0x03F, 0x0000003F},
+	{0x033, 0x00000005},
+	{0x03F, 0x0000003F},
+	{0x033, 0x00000006},
+	{0x03F, 0x0000003F},
+	{0x033, 0x00000007},
+	{0x03F, 0x0000003F},
+	{0x033, 0x00000008},
+	{0x03F, 0x0000003F},
+	{0x033, 0x0000000C},
+	{0x03F, 0x0000003F},
+	{0x033, 0x0000000D},
+	{0x03F, 0x0000003F},
+	{0x033, 0x0000000E},
+	{0x03F, 0x0000003F},
+	{0x0EE, 0x00000000},
+	{0x0EF, 0x00004000},
+	{0x033, 0x00000007},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00000707},
+	{0x033, 0x00000006},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00000704},
+	{0x033, 0x00000005},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00020500},
+	{0x033, 0x00000004},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00010404},
+	{0x033, 0x00000003},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00099B04},
+	{0x033, 0x00000002},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00092B04},
+	{0x033, 0x00000001},
+	{0x03E, 0x00000000},
+	{0x03F, 0x000B3204},
+	{0x033, 0x00000000},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00003000},
+	{0x033, 0x00000017},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00000787},
+	{0x033, 0x00000016},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00000784},
+	{0x033, 0x00000015},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00020580},
+	{0x033, 0x00000014},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00010484},
+	{0x033, 0x00000013},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00099B84},
+	{0x033, 0x00000012},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00092B84},
+	{0x033, 0x00000011},
+	{0x03E, 0x00000000},
+	{0x03F, 0x000B3284},
+	{0x033, 0x00000010},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00003080},
+	{0x0EF, 0x00000000},
+	{0x0EE, 0x00000010},
+	{0x033, 0x00000006},
+	{0x03F, 0x00000003},
+	{0x033, 0x00000007},
+	{0x03F, 0x00000003},
+	{0x0EE, 0x00000000},
+	{0x0EF, 0x00001000},
+	{0x033, 0x00000000},
+	{0x03F, 0x00000034},
+	{0x033, 0x00000001},
+	{0x03F, 0x00000037},
+	{0x033, 0x00000002},
+	{0x03F, 0x00000034},
+	{0x033, 0x00000003},
+	{0x03F, 0x00000024},
+	{0x033, 0x00000004},
+	{0x03F, 0x00000037},
+	{0x033, 0x00000005},
+	{0x03F, 0x00000027},
+	{0x0EF, 0x00000000},
+	{0x0EC, 0x00000400},
+	{0x033, 0x00000001},
+	{0x03F, 0x00000022},
+	{0x033, 0x00000003},
+	{0x03F, 0x00000022},
+	{0x033, 0x00000009},
+	{0x03F, 0x00000022},
+	{0x0EC, 0x00000000},
+	{0x0EC, 0x00000004},
+	{0x033, 0x00000000},
+	{0x03F, 0x000000AE},
+	{0x033, 0x00000001},
+	{0x03F, 0x0000008C},
+	{0x033, 0x00000002},
+	{0x03F, 0x0000006A},
+	{0x033, 0x00000003},
+	{0x03F, 0x00000048},
+	{0x033, 0x00000004},
+	{0x03F, 0x00000026},
+	{0x033, 0x00000005},
+	{0x03F, 0x00000004},
+	{0x033, 0x00000006},
+	{0x03F, 0x00000002},
+	{0x033, 0x00000007},
+	{0x03F, 0x00000000},
+	{0x0EC, 0x00000000},
+	{0x0EF, 0x00008000},
+	{0x033, 0x00000007},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001FB0},
+	{0x033, 0x00000006},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001FB0},
+	{0x033, 0x00000005},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001DB0},
+	{0x033, 0x00000004},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001CB0},
+	{0x033, 0x00000003},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001BB0},
+	{0x033, 0x00000002},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001AB0},
+	{0x033, 0x00000001},
+	{0x03E, 0x00000003},
+	{0x03F, 0x0000D9BC},
+	{0x033, 0x00000000},
+	{0x03E, 0x00000003},
+	{0x03F, 0x0000D4BC},
+	{0x033, 0x00000017},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001FB0},
+	{0x033, 0x00000016},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001FB0},
+	{0x033, 0x00000015},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001DB0},
+	{0x033, 0x00000014},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001CB0},
+	{0x033, 0x00000013},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001BB0},
+	{0x033, 0x00000012},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001AB0},
+	{0x033, 0x00000011},
+	{0x03E, 0x00000003},
+	{0x03F, 0x0000D9BC},
+	{0x033, 0x00000010},
+	{0x03E, 0x00000003},
+	{0x03F, 0x0000D4BC},
+	{0x033, 0x00000027},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001FB0},
+	{0x033, 0x00000026},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001FB0},
+	{0x033, 0x00000025},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001DB0},
+	{0x033, 0x00000024},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001CB0},
+	{0x033, 0x00000023},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001BB0},
+	{0x033, 0x00000022},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001AB0},
+	{0x033, 0x00000021},
+	{0x03E, 0x00000003},
+	{0x03F, 0x0000D9BC},
+	{0x033, 0x00000020},
+	{0x03E, 0x00000003},
+	{0x03F, 0x0000D4BC},
+	{0x033, 0x0000000E},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001FB0},
+	{0x033, 0x0000000D},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001DB0},
+	{0x033, 0x0000000C},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001CB0},
+	{0x033, 0x0000000B},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00091BB0},
+	{0x033, 0x0000000A},
+	{0x03E, 0x00000003},
+	{0x03F, 0x000A9AB0},
+	{0x033, 0x00000009},
+	{0x03E, 0x00000003},
+	{0x03F, 0x000BD9BC},
+	{0x033, 0x00000008},
+	{0x03E, 0x00000003},
+	{0x03F, 0x0009D4BC},
+	{0x033, 0x0000001E},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001FB0},
+	{0x033, 0x0000001D},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001DB0},
+	{0x033, 0x0000001C},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001CB0},
+	{0x033, 0x0000001B},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00091BB0},
+	{0x033, 0x0000001A},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00099AB0},
+	{0x033, 0x00000019},
+	{0x03E, 0x00000003},
+	{0x03F, 0x000AD9BC},
+	{0x033, 0x00000018},
+	{0x03E, 0x00000003},
+	{0x03F, 0x0009D4BC},
+	{0x033, 0x0000002E},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001FB0},
+	{0x033, 0x0000002D},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001DB0},
+	{0x033, 0x0000002C},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001CB0},
+	{0x033, 0x0000002B},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00001BB0},
+	{0x033, 0x0000002A},
+	{0x03E, 0x00000003},
+	{0x03F, 0x00009AB0},
+	{0x033, 0x00000029},
+	{0x03E, 0x00000003},
+	{0x03F, 0x0009D9BC},
+	{0x033, 0x00000028},
+	{0x03E, 0x00000003},
+	{0x03F, 0x0000D4BC},
+	{0x0EF, 0x00000000},
+	{0x0EF, 0x00002000},
+	{0x033, 0x00000000},
+	{0x03F, 0x00000005},
+	{0x033, 0x00000001},
+	{0x03F, 0x00000004},
+	{0x033, 0x00000002},
+	{0x03F, 0x00000004},
+	{0x033, 0x00000004},
+	{0x03F, 0x0000000C},
+	{0x033, 0x00000005},
+	{0x03F, 0x00000003},
+	{0x033, 0x00000006},
+	{0x03F, 0x00000003},
+	{0x0EF, 0x00000000},
+	{0x06C, 0x00038085},
+	{0x06D, 0x00000D6B},
+	{0x06E, 0x0001FB89},
+	{0x06F, 0x00097B99},
+	{0x069, 0x00008040},
+	{0x0EF, 0x00000200},
+	{0x033, 0x00000004},
+	{0x03F, 0x000008FF},
+	{0x033, 0x00000005},
+	{0x03F, 0x000004F2},
+	{0x033, 0x00000006},
+	{0x03F, 0x00000217},
+	{0x033, 0x00000007},
+	{0x03F, 0x00000131},
+	{0x0EF, 0x00000000},
+	{0x0EF, 0x00000400},
+	{0x033, 0x00000004},
+	{0x03F, 0x000004F7},
+	{0x033, 0x00000005},
+	{0x03F, 0x000004F7},
+	{0x033, 0x00000006},
+	{0x03F, 0x000004F2},
+	{0x033, 0x00000007},
+	{0x03F, 0x00000117},
+	{0x0EF, 0x00000000},
+	{0x043, 0x00005000},
+	{0x036, 0x000147D0},
+	{0x0B0, 0x0008677C},
+	{0x0B1, 0x00012920},
+	{0x0BB, 0x000EF000},
+	{0x0CB, 0x000A9594},
+	{0x0CC, 0x000C36D2},
+	{0x0CD, 0x00024923},
+	{0x0CE, 0x00020180},
+	{0x0CF, 0x00000000},
+	{0x0D5, 0x0006E27A},
+	{0x0D8, 0x00000044},
+	{0x0D9, 0x00000007},
+	{0x0DD, 0x00000020},
+	{0x0E3, 0x0000002C},
+	{0x0B7, 0x0000000C},
+	{0x0E1, 0x000080C0},
+	{0x0E4, 0x00000380},
+	{0x0ED, 0x00002000},
+	{0x033, 0x00000001},
+	{0x03D, 0x000A6094},
+	{0x03E, 0x00003449},
+	{0x03F, 0x00000001},
+	{0x033, 0x00000003},
+	{0x03D, 0x000AA094},
+	{0x03E, 0x00003449},
+	{0x03F, 0x00000001},
+	{0x0ED, 0x00000000},
+	{0x0ED, 0x00000100},
+	{0x033, 0x00000000},
+	{0x03F, 0x0000007F},
+	{0x033, 0x00000001},
+	{0x03F, 0x0000007F},
+	{0x033, 0x00000002},
+	{0x03F, 0x0000007F},
+	{0x033, 0x00000003},
+	{0x03F, 0x0000007F},
+	{0x033, 0x00000004},
+	{0x03F, 0x0000007F},
+	{0x033, 0x00000005},
+	{0x03F, 0x0000007F},
+	{0x033, 0x00000007},
+	{0x03F, 0x0000007F},
+	{0x033, 0x00000008},
+	{0x03F, 0x0000007F},
+	{0x033, 0x00000009},
+	{0x03F, 0x0000007F},
+	{0x0ED, 0x00000000},
+	{0x0ED, 0x00000080},
+	{0x033, 0x00000000},
+	{0x03E, 0x000007E1},
+	{0x03F, 0x0001F87F},
+	{0x033, 0x00000010},
+	{0x03E, 0x000007E1},
+	{0x03F, 0x0001F87F},
+	{0x033, 0x00000030},
+	{0x03E, 0x000007E1},
+	{0x03F, 0x0001F87F},
+	{0x033, 0x00000040},
+	{0x03E, 0x000007E1},
+	{0x03F, 0x0001F87F},
+	{0x033, 0x00000050},
+	{0x03E, 0x000007E1},
+	{0x03F, 0x0001F87F},
+	{0x033, 0x00000070},
+	{0x03E, 0x000007E1},
+	{0x03F, 0x0001F87F},
+	{0x0ED, 0x00000000},
+	{0x0ED, 0x00000004},
+	{0x033, 0x00000000},
+	{0x03F, 0x00008420},
+	{0x0ED, 0x00000000},
+	{0x018, 0x00011108},
+	{0x0B9, 0x00000000},
+	{0x0B9, 0x00000000},
+	{0x0B9, 0x00000200},
+	{0x0FF, 0x00000000},
+	{0x0FF, 0x00000000},
+	{0x0FF, 0x00000000},
+	{0x0FF, 0x00000000},
+	{0x0FF, 0x00000000},
+	{0x0FF, 0x00000000},
+	{0x0FF, 0x00000000},
+	{0x0FF, 0x00000000},
+	{0x0FF, 0x00000000},
+	{0x0FF, 0x00000000},
+	{0x0B9, 0x00000000},
+	{0x018, 0x00013124},
+	{0x05A, 0x0006808F},
+	{0x0ED, 0x00000008},
+	{0x033, 0x00000001},
+	{0x03F, 0x0000000F},
+	{0x0ED, 0x00000000},
+	{0x000, 0x00020000},
+	{0x018, 0x00010124},
+	{0x0EE, 0x00000800},
+	{0x033, 0x00000004},
+	{0x03F, 0x00000002},
+	{0x033, 0x00000005},
+	{0x03F, 0x00000003},
+	{0x033, 0x00000006},
+	{0x03F, 0x00000006},
+	{0x033, 0x00000007},
+	{0x03F, 0x00000007},
+	{0x0EE, 0x00000000},
+	{0x0EE, 0x00001000},
+	{0x033, 0x00000008},
+	{0x03F, 0x00000000},
+	{0x033, 0x00000009},
+	{0x03F, 0x00000001},
+	{0x033, 0x0000000A},
+	{0x03F, 0x00000003},
+	{0x033, 0x0000000B},
+	{0x03F, 0x00000103},
+	{0x033, 0x0000000C},
+	{0x03F, 0x00000107},
+	{0x033, 0x0000000D},
+	{0x03F, 0x00000207},
+	{0x033, 0x0000000E},
+	{0x03F, 0x00000307},
+	{0x033, 0x0000000F},
+	{0x03F, 0x00000307},
+	{0x0EE, 0x00000000},
+	{0x0EE, 0x00000200},
+	{0x033, 0x00000004},
+	{0x03F, 0x00000000},
+	{0x033, 0x00000005},
+	{0x03F, 0x00000001},
+	{0x033, 0x00000006},
+	{0x03F, 0x00000002},
+	{0x033, 0x00000007},
+	{0x03F, 0x00000003},
+	{0x0EE, 0x00000000},
+	{0x011, 0x00014062},
+	{0x0EF, 0x00000010},
+	{0x033, 0x00000001},
+	{0x03F, 0x00000DF3},
+	{0x033, 0x00000002},
+	{0x03F, 0x00000DF3},
+	{0x033, 0x00000003},
+	{0x03F, 0x00000A83},
+	{0x033, 0x00000004},
+	{0x03F, 0x00000A83},
+	{0x033, 0x00000005},
+	{0x03F, 0x00000643},
+	{0x033, 0x00000006},
+	{0x03F, 0x00000643},
+	{0x0EF, 0x00000000},
+	{0x0EF, 0x00000100},
+	{0x033, 0x00000001},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000002},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000003},
+	{0x03F, 0x0001B5A9},
+	{0x033, 0x00000004},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000005},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000006},
+	{0x03F, 0x0001B589},
+	{0x033, 0x00000007},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000008},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000009},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x0000000A},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x0000000B},
+	{0x03F, 0x0001B5A9},
+	{0x033, 0x0000000C},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x0000000D},
+	{0x03F, 0x0001B5A9},
+	{0x033, 0x0000000E},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x0000000F},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000010},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000011},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000012},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000013},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000014},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000015},
+	{0x03F, 0x0001B589},
+	{0x033, 0x00000016},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000017},
+	{0x03F, 0x0001B5A9},
+	{0x033, 0x00000018},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000019},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x0000001A},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x0000001B},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x0000001C},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x0000001D},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x0000001E},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x0000001F},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000020},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000021},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000022},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000023},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000024},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000025},
+	{0x03F, 0x0001B5A8},
+	{0x033, 0x00000022},
+	{0x03F, 0x0001B5A8},
+	{0x0EF, 0x00000000},
+	{0x0EF, 0x00000040},
+	{0x033, 0x00000001},
+	{0x03F, 0x000002C5},
+	{0x033, 0x00000002},
+	{0x03F, 0x000002C5},
+	{0x033, 0x00000003},
+	{0x03F, 0x000002C5},
+	{0x033, 0x00000004},
+	{0x03F, 0x000002C5},
+	{0x033, 0x00000005},
+	{0x03F, 0x000002C5},
+	{0x033, 0x00000006},
+	{0x03F, 0x000002C5},
+	{0x033, 0x00000007},
+	{0x03F, 0x000002C5},
+	{0x033, 0x00000008},
+	{0x03F, 0x000002C5},
+	{0x033, 0x00000009},
+	{0x03F, 0x000002C5},
+	{0x033, 0x0000000A},
+	{0x03F, 0x000002C5},
+	{0x033, 0x0000000B},
+	{0x03F, 0x000002C5},
+	{0x0EF, 0x00000000},
+	{0x059, 0x00050033},
+	{0x061, 0x0005F48A},
+	{0x062, 0x00077435},
+	{0x063, 0x000F80A2},
+	{0x065, 0x00018F22},
+	{0x067, 0x00008060},
+	{0x07E, 0x0009780B},
+	{0x0EE, 0x00000004},
+	{0x033, 0x0000000B},
+	{0x03F, 0x0000000B},
+	{0x033, 0x0000000C},
+	{0x03F, 0x00000012},
+	{0x033, 0x0000000D},
+	{0x03F, 0x00000019},
+	{0x033, 0x0000000F},
+	{0x03F, 0x0000000B},
+	{0x033, 0x00000010},
+	{0x03F, 0x00000012},
+	{0x033, 0x00000011},
+	{0x03F, 0x00000019},
+	{0x03F, 0x00000000},
+	{0x0EE, 0x00000000},
+	{0x0EE, 0x00000800},
+	{0x033, 0x00000000},
+	{0x03F, 0x00000001},
+	{0x033, 0x00000001},
+	{0x03F, 0x00000002},
+	{0x033, 0x00000002},
+	{0x03F, 0x00000003},
+	{0x033, 0x00000003},
+	{0x03F, 0x00000007},
+	{0x0EE, 0x00000000},
+	{0x0EE, 0x00001000},
+	{0x033, 0x00000000},
+	{0x03F, 0x00003000},
+	{0x033, 0x00000001},
+	{0x03F, 0x00000000},
+	{0x033, 0x00000002},
+	{0x03F, 0x00000001},
+	{0x033, 0x00000003},
+	{0x03F, 0x00000003},
+	{0x033, 0x00000004},
+	{0x03F, 0x00000007},
+	{0x033, 0x00000005},
+	{0x03F, 0x0000000F},
+	{0x033, 0x00000006},
+	{0x03F, 0x0000010F},
+	{0x033, 0x00000007},
+	{0x03F, 0x0000030F},
+	{0x0EE, 0x00000000},
+	{0x0EE, 0x00000200},
+	{0x033, 0x00000000},
+	{0x03F, 0x00000004},
+	{0x033, 0x00000001},
+	{0x03F, 0x00000005},
+	{0x033, 0x00000002},
+	{0x03F, 0x00000006},
+	{0x033, 0x00000003},
+	{0x03F, 0x00000007},
+	{0x0EE, 0x00000000},
+	{0x0EF, 0x00000080},
+	{0x033, 0x00000004},
+	{0x03E, 0x0000001D},
+	{0x03F, 0x0001A241},
+	{0x033, 0x00000005},
+	{0x03E, 0x0000001D},
+	{0x03F, 0x0001A241},
+	{0x033, 0x00000006},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000007},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000008},
+	{0x03E, 0x0000001D},
+	{0x03F, 0x0001A241},
+	{0x033, 0x00000009},
+	{0x03E, 0x0001A241},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000000A},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000000B},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000000C},
+	{0x03E, 0x0000001D},
+	{0x03F, 0x0001A241},
+	{0x033, 0x0000000D},
+	{0x03E, 0x0000001D},
+	{0x03F, 0x0001A241},
+	{0x033, 0x0000000E},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000000F},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000010},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x000199C1},
+	{0x033, 0x00000011},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x000199C1},
+	{0x033, 0x00000012},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000013},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000014},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x000199C1},
+	{0x033, 0x00000015},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x000199C1},
+	{0x033, 0x00000016},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000017},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000018},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x000199C1},
+	{0x033, 0x00000019},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x000199C1},
+	{0x033, 0x0000001A},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000001B},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000001C},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x000199C1},
+	{0x033, 0x0000001D},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x000199C1},
+	{0x033, 0x0000001E},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000001F},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000020},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x000199C1},
+	{0x033, 0x00000021},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x000199C1},
+	{0x033, 0x00000022},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000023},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000024},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x0001E141},
+	{0x033, 0x00000025},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x0001E141},
+	{0x033, 0x00000026},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000027},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000028},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x0001E141},
+	{0x033, 0x00000029},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x0001E141},
+	{0x033, 0x0000002A},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000002B},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000002C},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x0001E141},
+	{0x033, 0x0000002D},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x0001E141},
+	{0x033, 0x0000002E},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000002F},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000030},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x0001E141},
+	{0x033, 0x00000031},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x0001E141},
+	{0x033, 0x00000032},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000033},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000034},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x0001E141},
+	{0x033, 0x00000035},
+	{0x03E, 0x0000001C},
+	{0x03F, 0x0001E141},
+	{0x033, 0x00000036},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000037},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000038},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x00000039},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C1},
+	{0x033, 0x0000003A},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C3},
+	{0x033, 0x0000003B},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C3},
+	{0x033, 0x0000003C},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C3},
+	{0x033, 0x0000003D},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C3},
+	{0x033, 0x0000003E},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C3},
+	{0x033, 0x0000003F},
+	{0x03E, 0x0000001B},
+	{0x03F, 0x0001C3C3},
+	{0x0EF, 0x00000000},
+	{0x051, 0x0003D368},
+	{0x052, 0x000A3338},
+	{0x053, 0x000688AF},
+	{0x054, 0x00012C04},
+	{0x058, 0x00084221},
+	{0x05B, 0x000EB000},
+	{0x100EE, 0x00002000},
+	{0x10030, 0x000000F9},
+	{0x10030, 0x000004F6},
+	{0x10030, 0x000008F3},
+	{0x10030, 0x00000CF0},
+	{0x10030, 0x000010ED},
+	{0x10030, 0x000014EA},
+	{0x10030, 0x000018E7},
+	{0x10030, 0x00001CE4},
+	{0x10030, 0x000020E1},
+	{0x10030, 0x000024A4},
+	{0x10030, 0x000028A1},
+	{0x10030, 0x00002C9E},
+	{0x10030, 0x0000309B},
+	{0x10030, 0x0000341E},
+	{0x10030, 0x0000381B},
+	{0x10030, 0x00003C18},
+	{0x10030, 0x00004015},
+	{0x10030, 0x000200BC},
+	{0x10030, 0x000204B9},
+	{0x10030, 0x000208B6},
+	{0x10030, 0x00020CB3},
+	{0x10030, 0x000210B0},
+	{0x10030, 0x000214AD},
+	{0x10030, 0x0002186C},
+	{0x10030, 0x00021C69},
+	{0x10030, 0x00022066},
+	{0x10030, 0x00022426},
+	{0x10030, 0x00022823},
+	{0x10030, 0x00022C20},
+	{0x10030, 0x0002301D},
+	{0x10030, 0x0002341A},
+	{0x10030, 0x00023817},
+	{0x10030, 0x00023C14},
+	{0x10030, 0x00024011},
+	{0x10030, 0x000280BC},
+	{0x10030, 0x000284B9},
+	{0x10030, 0x000288B6},
+	{0x10030, 0x00028CB3},
+	{0x10030, 0x000290B0},
+	{0x10030, 0x000294AD},
+	{0x10030, 0x0002986C},
+	{0x10030, 0x00029C69},
+	{0x10030, 0x0002A066},
+	{0x10030, 0x0002A426},
+	{0x10030, 0x0002A823},
+	{0x10030, 0x0002AC20},
+	{0x10030, 0x0002B01D},
+	{0x10030, 0x0002B41A},
+	{0x10030, 0x0002B817},
+	{0x10030, 0x0002BC14},
+	{0x10030, 0x0002C011},
+	{0x10030, 0x000300BC},
+	{0x10030, 0x000304B9},
+	{0x10030, 0x000308B6},
+	{0x10030, 0x00030CB3},
+	{0x10030, 0x000310B0},
+	{0x10030, 0x000314AD},
+	{0x10030, 0x0003186C},
+	{0x10030, 0x00031C69},
+	{0x10030, 0x00032066},
+	{0x10030, 0x00032426},
+	{0x10030, 0x00032823},
+	{0x10030, 0x00032C20},
+	{0x10030, 0x0003301D},
+	{0x10030, 0x0003341A},
+	{0x10030, 0x00033817},
+	{0x10030, 0x00033C14},
+	{0x10030, 0x00034011},
+	{0x100EE, 0x00000000},
+	{0x100EE, 0x00004000},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DDD},
+	{0x10030, 0x000211D7},
+	{0x10030, 0x000215D1},
+	{0x10030, 0x00021919},
+	{0x10030, 0x00021D13},
+	{0x10030, 0x000220D9},
+	{0x10030, 0x000224D3},
+	{0x10030, 0x00022899},
+	{0x10030, 0x00022C93},
+	{0x10030, 0x00023059},
+	{0x10030, 0x00023453},
+	{0x10030, 0x00023819},
+	{0x10030, 0x00023C13},
+	{0x10030, 0x0002400D},
+	{0x10030, 0x00024407},
+	{0x10030, 0x000281EF},
+	{0x10030, 0x000285E9},
+	{0x10030, 0x000289E3},
+	{0x10030, 0x00028DDD},
+	{0x10030, 0x000291D7},
+	{0x10030, 0x000295D1},
+	{0x10030, 0x00029919},
+	{0x10030, 0x00029D13},
+	{0x10030, 0x0002A0D9},
+	{0x10030, 0x0002A4D3},
+	{0x10030, 0x0002A899},
+	{0x10030, 0x0002AC93},
+	{0x10030, 0x0002B059},
+	{0x10030, 0x0002B453},
+	{0x10030, 0x0002B819},
+	{0x10030, 0x0002BC13},
+	{0x10030, 0x0002C00D},
+	{0x10030, 0x0002C407},
+	{0x10030, 0x000301EF},
+	{0x10030, 0x000305E9},
+	{0x10030, 0x000309E3},
+	{0x10030, 0x00030DDD},
+	{0x10030, 0x000311D7},
+	{0x10030, 0x000315D1},
+	{0x10030, 0x00031919},
+	{0x10030, 0x00031D13},
+	{0x10030, 0x000320D9},
+	{0x10030, 0x000324D3},
+	{0x10030, 0x00032899},
+	{0x10030, 0x00032C93},
+	{0x10030, 0x00033059},
+	{0x10030, 0x00033453},
+	{0x10030, 0x00033819},
+	{0x10030, 0x00033C13},
+	{0x10030, 0x0003400D},
+	{0x10030, 0x00034407},
+	{0x100EE, 0x00000000},
+	{0x100EE, 0x00004000},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011A5},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001965},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002125},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x000028E5},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030A5},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003865},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004025},
+	{0x10030, 0x0000441F},
+	{0x100EE, 0x00000000},
+	{0x0EF, 0x00000008},
+	{0x033, 0x00000000},
+	{0x03F, 0x00000004},
+	{0x0EF, 0x00000000},
+	{0x005, 0x00000001},
+	{0x10005, 0x00000001},
+	{0x0FE, 0x00000022},
+};
+
+static const struct rtw89_reg2_def rtw89_8851b_phy_nctl_regs[] = {
+	{0x8000, 0x00000008},
+	{0x8008, 0x00000000},
+	{0x8004, 0xe8862b66},
+	{0x800c, 0x78000000},
+	{0x8010, 0x88015000},
+	{0x8014, 0x80010100},
+	{0x8018, 0x10010100},
+	{0x801c, 0xa210bc00},
+	{0x8020, 0x000403e0},
+	{0x8024, 0x00072160},
+	{0x8028, 0x00180e00},
+	{0x8030, 0x400000c0},
+	{0x8034, 0x11000830},
+	{0x8038, 0x40000000},
+	{0x803c, 0x00000008},
+	{0x8040, 0x00000046},
+	{0x8044, 0x0010001f},
+	{0x8048, 0x00000003},
+	{0x804c, 0x420840e0},
+	{0x8050, 0xce08cce0},
+	{0x8054, 0x420840e0},
+	{0x8058, 0xce08cce0},
+	{0x805c, 0x150c0b02},
+	{0x8060, 0x150c0b02},
+	{0x8064, 0x2aa00047},
+	{0x8074, 0x80000000},
+	{0x807c, 0x000000ee},
+	{0x8088, 0x80000000},
+	{0x808c, 0x00000000},
+	{0x80b0, 0x00000000},
+	{0x80cc, 0x00000000},
+	{0x80d0, 0x00000000},
+	{0x80ec, 0x00000002},
+	{0x8098, 0x0000ff00},
+	{0x8070, 0x00e80000},
+	{0x80b0, 0xffe00fff},
+	{0x809c, 0x0000001f},
+	{0x80b8, 0x00002000},
+	{0x80bc, 0x00050033},
+	{0xa400, 0x00000000},
+	{0xa404, 0x00000180},
+	{0xa408, 0x000001af},
+	{0xa40c, 0x000001e3},
+	{0xa410, 0x00000220},
+	{0xa414, 0x00000262},
+	{0xa418, 0x000002ac},
+	{0xa41c, 0x0000035e},
+	{0xa420, 0x000003c7},
+	{0xa424, 0x0000043d},
+	{0xa428, 0x000004c1},
+	{0xa42c, 0x00000556},
+	{0xa430, 0x000005fc},
+	{0xa434, 0x000006b7},
+	{0xa438, 0x00000789},
+	{0xa43c, 0x00000875},
+	{0xa440, 0x0000011f},
+	{0x8104, 0x00000000},
+	{0x810c, 0x00000000},
+	{0x8110, 0x00000000},
+	{0x8114, 0x00000000},
+	{0x8120, 0x10010000},
+	{0x8124, 0x00000000},
+	{0x8128, 0x00000200},
+	{0x812c, 0x0000c000},
+	{0x8130, 0x40000000},
+	{0x8138, 0x40000000},
+	{0x813c, 0x40000000},
+	{0x8140, 0x00000000},
+	{0x8144, 0x0b040b03},
+	{0x8148, 0x07020b04},
+	{0x814c, 0x07020b04},
+	{0x8150, 0xe4e40000},
+	{0x8158, 0xffffffff},
+	{0x815c, 0xffffffff},
+	{0x8160, 0xffffffff},
+	{0x8164, 0xffffffff},
+	{0x8168, 0xffffffff},
+	{0x816c, 0x1fffffff},
+	{0x81cc, 0x00000000},
+	{0x81dc, 0x00000002},
+	{0x81e0, 0x00000000},
+	{0x81e4, 0x00000001},
+	{0x81a0, 0x00000000},
+	{0x81ac, 0x3fc20400},
+	{0x81b0, 0x3f914100},
+	{0x81bc, 0x0000005b},
+	{0x81c0, 0x0000005b},
+	{0x81b4, 0x01e0f078},
+	{0x81b8, 0x01e0f078},
+	{0x81f0, 0x0000f078},
+	{0x81d8, 0x00000001},
+	{0x9500, 0x00000000},
+	{0x9504, 0x00000000},
+	{0x9508, 0x00000000},
+	{0x950c, 0x00000000},
+	{0x9510, 0x00000000},
+	{0x9514, 0x00000000},
+	{0x9518, 0x00000000},
+	{0x951c, 0x00000000},
+	{0x9520, 0x00000000},
+	{0x9524, 0x00000000},
+	{0x9528, 0x00000000},
+	{0x952c, 0x00000000},
+	{0x9530, 0x00000000},
+	{0x9534, 0x00000000},
+	{0x9538, 0x00000000},
+	{0x953c, 0x00000000},
+	{0x9540, 0x04000000},
+	{0x9544, 0x00000000},
+	{0x9548, 0x00000000},
+	{0x954c, 0x00000000},
+	{0x9550, 0x00000000},
+	{0x9554, 0x00000000},
+	{0x9558, 0x00000000},
+	{0x955c, 0x00000000},
+	{0x9560, 0x00000000},
+	{0x9564, 0x00000000},
+	{0x9568, 0x00000000},
+	{0x956c, 0x00000000},
+	{0x9570, 0x00000000},
+	{0x9574, 0x00000000},
+	{0x9578, 0x00000000},
+	{0x957c, 0x00000000},
+	{0x9580, 0x00000000},
+	{0x9584, 0x04000000},
+	{0x9588, 0x00000000},
+	{0x958c, 0x00000000},
+	{0x9590, 0x00000000},
+	{0x9594, 0x00000000},
+	{0x9598, 0x00000000},
+	{0x959c, 0x00000000},
+	{0x95a0, 0x00000000},
+	{0x95a4, 0x00000000},
+	{0x95a8, 0x00000000},
+	{0x95ac, 0x00000000},
+	{0x95b0, 0x00000000},
+	{0x95b4, 0x00000000},
+	{0x95b8, 0x00000000},
+	{0x95bc, 0x00000000},
+	{0x95c0, 0x00000000},
+	{0x95c4, 0x00000000},
+	{0x95c8, 0x04000000},
+	{0x95cc, 0x00000000},
+	{0x95d0, 0x00000000},
+	{0x95d4, 0x00000000},
+	{0x95d8, 0x00000000},
+	{0x95dc, 0x00000000},
+	{0x95e0, 0x00000000},
+	{0x95e4, 0x00000000},
+	{0x95e8, 0x00000000},
+	{0x95ec, 0x00000000},
+	{0x95f0, 0x00000000},
+	{0x95f4, 0x00000000},
+	{0x95f8, 0x00000000},
+	{0x95fc, 0x00000000},
+	{0x9600, 0x00000000},
+	{0x9604, 0x00000000},
+	{0x9608, 0x00000000},
+	{0x960c, 0x04000000},
+	{0x9610, 0x00000000},
+	{0x9614, 0x00000000},
+	{0x9618, 0x00000000},
+	{0x961c, 0x00000000},
+	{0x9620, 0x00000000},
+	{0x9624, 0x00000000},
+	{0x9628, 0x00000000},
+	{0x962c, 0x00000000},
+	{0x9630, 0x00000000},
+	{0x9634, 0x00000000},
+	{0x9638, 0x00000000},
+	{0x963c, 0x00000000},
+	{0x9640, 0x00000000},
+	{0x9644, 0x00000000},
+	{0x9648, 0x00000000},
+	{0x964c, 0x00000000},
+	{0x9650, 0x04000000},
+	{0x9654, 0x00000000},
+	{0x9658, 0x00000000},
+	{0x965c, 0x00000000},
+	{0x9660, 0x00000000},
+	{0x9664, 0x00000000},
+	{0x9668, 0x00000000},
+	{0x966c, 0x00000000},
+	{0x9670, 0x00000000},
+	{0x9674, 0x00000000},
+	{0x9678, 0x00000000},
+	{0x967c, 0x00000000},
+	{0x9680, 0x00000000},
+	{0x9684, 0x00000000},
+	{0x9688, 0x00000000},
+	{0x968c, 0x00000000},
+	{0x9690, 0x00000000},
+	{0x9694, 0x04000000},
+	{0x9698, 0x00000000},
+	{0x969c, 0x00000000},
+	{0x96a0, 0x00000000},
+	{0x96a4, 0x00000000},
+	{0x96a8, 0x00000000},
+	{0x96ac, 0x00000000},
+	{0x96b0, 0x00000000},
+	{0x96b4, 0x00000000},
+	{0x96b8, 0x00000000},
+	{0x96bc, 0x00000000},
+	{0x96c0, 0x00000000},
+	{0x96c4, 0x00000000},
+	{0x96c8, 0x00000000},
+	{0x96cc, 0x00000000},
+	{0x96d0, 0x00000000},
+	{0x96d4, 0x00000000},
+	{0x96d8, 0x04000000},
+	{0x96dc, 0x00000000},
+	{0x96e0, 0x00000000},
+	{0x96e4, 0x00000000},
+	{0x96e8, 0x00000000},
+	{0x96ec, 0x00000000},
+	{0x96f0, 0x00000000},
+	{0x96f4, 0x00000000},
+	{0x96f8, 0x00000000},
+	{0x96fc, 0x00000000},
+	{0x9700, 0x00000000},
+	{0x9704, 0x00000000},
+	{0x9708, 0x00000000},
+	{0x970c, 0x00000000},
+	{0x9710, 0x00000000},
+	{0x9714, 0x00000000},
+	{0x9718, 0x00000000},
+	{0x971c, 0x04000000},
+	{0x9720, 0x00000000},
+	{0x9724, 0x00000000},
+	{0x9728, 0x00000000},
+	{0x972c, 0x00000000},
+	{0x9730, 0x00000000},
+	{0x9734, 0x00000000},
+	{0x9738, 0x00000000},
+	{0x973c, 0x00000000},
+	{0x9740, 0x00000000},
+	{0x9744, 0x00000000},
+	{0x9748, 0x00000000},
+	{0x974c, 0x00000000},
+	{0x9750, 0x00000000},
+	{0x9754, 0x00000000},
+	{0x9758, 0x00000000},
+	{0x975c, 0x00000000},
+	{0x9760, 0x04000000},
+	{0x9764, 0x00000000},
+	{0x9768, 0x00000000},
+	{0x976c, 0x00000000},
+	{0x9770, 0x00000000},
+	{0x9774, 0x00000000},
+	{0x9778, 0x00000000},
+	{0x977c, 0x00000000},
+	{0x9780, 0x00000000},
+	{0x9784, 0x00000000},
+	{0x9788, 0x00000000},
+	{0x978c, 0x00000000},
+	{0x9790, 0x00000000},
+	{0x9794, 0x00000000},
+	{0x9798, 0x00000000},
+	{0x979c, 0x00000000},
+	{0x97a0, 0x00000000},
+	{0x97a4, 0x04000000},
+	{0x97a8, 0x00000000},
+	{0x97ac, 0x00000000},
+	{0x97b0, 0x00000000},
+	{0x97b4, 0x00000000},
+	{0x97b8, 0x00000000},
+	{0x97bc, 0x00000000},
+	{0x97c0, 0x00000000},
+	{0x97c4, 0x00000000},
+	{0x97c8, 0x00000000},
+	{0x97cc, 0x00000000},
+	{0x97d0, 0x00000000},
+	{0x97d4, 0x00000000},
+	{0x97d8, 0x00000000},
+	{0x97dc, 0x00000000},
+	{0x97e0, 0x00000000},
+	{0x97e4, 0x00000000},
+	{0x97e8, 0x04000000},
+	{0x97ec, 0x00000000},
+	{0x97f0, 0x00000000},
+	{0x97f4, 0x00000000},
+	{0x97f8, 0x00000000},
+	{0x97fc, 0x00000000},
+	{0x9800, 0x00000000},
+	{0x9804, 0x00000000},
+	{0x9808, 0x00000000},
+	{0x980c, 0x00000000},
+	{0x9810, 0x00000000},
+	{0x9814, 0x00000000},
+	{0x9818, 0x00000000},
+	{0x981c, 0x00000000},
+	{0x9820, 0x00000000},
+	{0x9824, 0x00000000},
+	{0x9828, 0x00000000},
+	{0x982c, 0x04000000},
+	{0x81d8, 0x00000000},
+	{0xb104, 0x2b251f19},
+	{0xb108, 0x433d3731},
+	{0xb10c, 0x5b554f49},
+	{0xb110, 0x736d6761},
+	{0xb114, 0x7f7f7f79},
+	{0xb118, 0x120f7f7f},
+	{0xb11c, 0x1e1b1815},
+	{0xb120, 0x2a272421},
+	{0xb124, 0x3633302d},
+	{0xb128, 0x3f3f3c39},
+	{0xb12c, 0x3f3f3f3f},
+	{0x8088, 0x00000110},
+	{0x8000, 0x00000008},
+	{0x8080, 0x00000005},
+	{0x8500, 0x80000008},
+	{0x8504, 0x43000004},
+	{0x8508, 0x4b044a00},
+	{0x850c, 0x40098604},
+	{0x8510, 0x0004e01f},
+	{0x8514, 0x74104b00},
+	{0x8518, 0x000021e0},
+	{0x851c, 0x74301658},
+	{0x8520, 0x43800004},
+	{0x8524, 0x4c000007},
+	{0x8528, 0x43000004},
+	{0x852c, 0x56030007},
+	{0x8530, 0x57000004},
+	{0x8534, 0x400042fe},
+	{0x8538, 0x50554200},
+	{0x853c, 0xb4183000},
+	{0x8540, 0xe537a50f},
+	{0x8544, 0xf12bf02b},
+	{0x8548, 0xf32bf22b},
+	{0x854c, 0xf62bf42b},
+	{0x8550, 0xf82bf72b},
+	{0x8554, 0xfa2bf92b},
+	{0x8558, 0xfd2bfc2b},
+	{0x855c, 0xe537fe2b},
+	{0x8560, 0xf12af02a},
+	{0x8564, 0xf32af22a},
+	{0x8568, 0xf52af42a},
+	{0x856c, 0x000bf62a},
+	{0x8570, 0xf028a511},
+	{0x8574, 0xf228f128},
+	{0x8578, 0xf428f328},
+	{0x857c, 0xf628f528},
+	{0x8580, 0xf828f728},
+	{0x8584, 0xfa28f928},
+	{0x8588, 0xfc28fb28},
+	{0x858c, 0xfe28fd28},
+	{0x8590, 0xf028ff28},
+	{0x8594, 0xf228f128},
+	{0x8598, 0x30750001},
+	{0x859c, 0x30753075},
+	{0x85a0, 0x30b63097},
+	{0x85a4, 0x30be30bb},
+	{0x85a8, 0x30d930cc},
+	{0x85ac, 0x316d30e6},
+	{0x85b0, 0x3189317f},
+	{0x85b4, 0x31d23193},
+	{0x85b8, 0x31e43210},
+	{0x85bc, 0x31e831dd},
+	{0x85c0, 0x322831e1},
+	{0x85c4, 0x323c3232},
+	{0x85c8, 0x32503246},
+	{0x85cc, 0x3264325a},
+	{0x85d0, 0x3278326e},
+	{0x85d4, 0x32983285},
+	{0x85d8, 0x32aa32a6},
+	{0x85dc, 0x330b32f3},
+	{0x85e0, 0x333f330c},
+	{0x85e4, 0x334c3341},
+	{0x85e8, 0xe35e0001},
+	{0x85ec, 0x20887410},
+	{0x85f0, 0x140f0200},
+	{0x85f4, 0x02002098},
+	{0x85f8, 0x7430140f},
+	{0x85fc, 0x5b10e39c},
+	{0x8600, 0x20807410},
+	{0x8604, 0x140f0000},
+	{0x8608, 0x56015507},
+	{0x860c, 0x7410e382},
+	{0x8610, 0x02002088},
+	{0x8614, 0x5517140f},
+	{0x8618, 0xe34ee382},
+	{0x861c, 0x468e7508},
+	{0x8620, 0xe0ace38c},
+	{0x8624, 0x5500f0e2},
+	{0x8628, 0x5501e37e},
+	{0x862c, 0x5b10f1de},
+	{0x8630, 0x20907410},
+	{0x8634, 0x140f0000},
+	{0x8638, 0xe3825507},
+	{0x863c, 0x20987410},
+	{0x8640, 0x140f0200},
+	{0x8644, 0xe3825517},
+	{0x8648, 0x46967509},
+	{0x864c, 0xe0ace38c},
+	{0x8650, 0xe37e5500},
+	{0x8654, 0x00015501},
+	{0x8658, 0x4d000007},
+	{0x865c, 0x74200004},
+	{0x8660, 0x57005710},
+	{0x8664, 0x9700140f},
+	{0x8668, 0x00017430},
+	{0x866c, 0xe39ce35e},
+	{0x8670, 0xe52a0bbd},
+	{0x8674, 0xe36a0001},
+	{0x8678, 0x0001e3c4},
+	{0x867c, 0x55005b30},
+	{0x8680, 0x46500005},
+	{0x8684, 0x74000004},
+	{0x8688, 0x1658e37e},
+	{0x868c, 0x74305501},
+	{0x8690, 0x46100005},
+	{0x8694, 0x00010004},
+	{0x8698, 0x30f8e35e},
+	{0x869c, 0xe52a0023},
+	{0x86a0, 0x54ed0002},
+	{0x86a4, 0x00230baa},
+	{0x86a8, 0x0002e52a},
+	{0x86ac, 0xe356e3e4},
+	{0x86b0, 0xe35e0001},
+	{0x86b4, 0x002230f3},
+	{0x86b8, 0x0002e52a},
+	{0x86bc, 0x0baa54ec},
+	{0x86c0, 0xe52a0022},
+	{0x86c4, 0xe3e40002},
+	{0x86c8, 0x0001e356},
+	{0x86cc, 0x0baae35e},
+	{0x86d0, 0xe3e430ec},
+	{0x86d4, 0x0001e356},
+	{0x86d8, 0x6d0f6c67},
+	{0x86dc, 0xe52ae39c},
+	{0x86e0, 0xe39c6c8b},
+	{0x86e4, 0x0bace52a},
+	{0x86e8, 0x6d0f6cb3},
+	{0x86ec, 0xe52ae39c},
+	{0x86f0, 0x6cdb0bad},
+	{0x86f4, 0xe39c6d0f},
+	{0x86f8, 0x6cf5e52a},
+	{0x86fc, 0xe39c6d0f},
+	{0x8700, 0x6c0be52a},
+	{0x8704, 0xe39c6d00},
+	{0x8708, 0x6c25e52a},
+	{0x870c, 0xe52ae39c},
+	{0x8710, 0x6c4df8c6},
+	{0x8714, 0xe52ae39c},
+	{0x8718, 0x6c75f9cf},
+	{0x871c, 0xe52ae39c},
+	{0x8720, 0xe39c6c99},
+	{0x8724, 0xfad6e52a},
+	{0x8728, 0x21e87410},
+	{0x872c, 0x6e670009},
+	{0x8730, 0xe3c46f0f},
+	{0x8734, 0x7410e52f},
+	{0x8738, 0x000b21e8},
+	{0x873c, 0xe3c46e8b},
+	{0x8740, 0x7410e52f},
+	{0x8744, 0x000d21e8},
+	{0x8748, 0x6f0f6eb3},
+	{0x874c, 0xe52fe3c4},
+	{0x8750, 0xfe07ff08},
+	{0x8754, 0x21e87410},
+	{0x8758, 0x6ec7000e},
+	{0x875c, 0xe52fe3c4},
+	{0x8760, 0x21e87410},
+	{0x8764, 0x6edb000f},
+	{0x8768, 0xe3c46f0f},
+	{0x876c, 0x7410e52f},
+	{0x8770, 0x001021e8},
+	{0x8774, 0xe3c46eef},
+	{0x8778, 0xff03e52f},
+	{0x877c, 0xe52ffe02},
+	{0x8780, 0x21e87410},
+	{0x8784, 0x6e110013},
+	{0x8788, 0xe3c46f00},
+	{0x878c, 0xff03e52f},
+	{0x8790, 0xe52ffe02},
+	{0x8794, 0x21e87410},
+	{0x8798, 0x6e250014},
+	{0x879c, 0xe52fe3c4},
+	{0x87a0, 0xff08fc24},
+	{0x87a4, 0x7410fe07},
+	{0x87a8, 0x001521e8},
+	{0x87ac, 0xe3c46e39},
+	{0x87b0, 0x7410e52f},
+	{0x87b4, 0x001621e8},
+	{0x87b8, 0xe3c46e4d},
+	{0x87bc, 0xfd27e52f},
+	{0x87c0, 0x21e87410},
+	{0x87c4, 0x6e750018},
+	{0x87c8, 0xe52fe3c4},
+	{0x87cc, 0x21e87410},
+	{0x87d0, 0x6e99001a},
+	{0x87d4, 0xe52fe3c4},
+	{0x87d8, 0xe36afe24},
+	{0x87dc, 0x63404380},
+	{0x87e0, 0x43006880},
+	{0x87e4, 0x31300bac},
+	{0x87e8, 0xe52f0022},
+	{0x87ec, 0x54ec0002},
+	{0x87f0, 0x00220baa},
+	{0x87f4, 0x0002e52f},
+	{0x87f8, 0xe362e3e4},
+	{0x87fc, 0xe36a0001},
+	{0x8800, 0x63404380},
+	{0x8804, 0x43006881},
+	{0x8808, 0x31210baa},
+	{0x880c, 0xe362e3e4},
+	{0x8810, 0xe36a0001},
+	{0x8814, 0x63414380},
+	{0x8818, 0x43006882},
+	{0x881c, 0x31140baa},
+	{0x8820, 0xe362e3e4},
+	{0x8824, 0x00040001},
+	{0x8828, 0x000742fc},
+	{0x882c, 0x00046001},
+	{0x8830, 0x00074200},
+	{0x8834, 0x62006220},
+	{0x8838, 0x55010004},
+	{0x883c, 0x66055b40},
+	{0x8840, 0x62000007},
+	{0x8844, 0xe40e6300},
+	{0x8848, 0x09000004},
+	{0x884c, 0x0b400a01},
+	{0x8850, 0x0e010d00},
+	{0x8854, 0x00040032},
+	{0x8858, 0x42fb950b},
+	{0x885c, 0x4d040007},
+	{0x8860, 0x42000004},
+	{0x8864, 0x00074380},
+	{0x8868, 0x00044d01},
+	{0x886c, 0x00074300},
+	{0x8870, 0x05a30562},
+	{0x8874, 0xe40e961f},
+	{0x8878, 0xe37e0004},
+	{0x887c, 0x06a20007},
+	{0x8880, 0xe40e07a3},
+	{0x8884, 0xe37e0004},
+	{0x8888, 0x0002e3fe},
+	{0x888c, 0x4380e406},
+	{0x8890, 0x4d000007},
+	{0x8894, 0x43000004},
+	{0x8898, 0x000742fe},
+	{0x889c, 0x00044d00},
+	{0x88a0, 0x00014200},
+	{0x88a4, 0x42fc0004},
+	{0x88a8, 0x60030007},
+	{0x88ac, 0x42000004},
+	{0x88b0, 0x00073199},
+	{0x88b4, 0x07a306a2},
+	{0x88b8, 0xe1eb31c5},
+	{0x88bc, 0xe1fee1f9},
+	{0x88c0, 0xe1eb0001},
+	{0x88c4, 0x0001e1fe},
+	{0x88c8, 0xe1f9e1f2},
+	{0x88cc, 0x0001e1fe},
+	{0x88d0, 0xe1fee1f2},
+	{0x88d4, 0x00040001},
+	{0x88d8, 0x000742fc},
+	{0x88dc, 0x00046003},
+	{0x88e0, 0x00014200},
+	{0x88e4, 0x42fc0004},
+	{0x88e8, 0x60010007},
+	{0x88ec, 0x42000004},
+	{0x88f0, 0x00070001},
+	{0x88f4, 0x62006220},
+	{0x88f8, 0x0001e406},
+	{0x88fc, 0x63000007},
+	{0x8900, 0x09000004},
+	{0x8904, 0x0e010a00},
+	{0x8908, 0x00070032},
+	{0x890c, 0xe40e06a2},
+	{0x8910, 0x0002e41a},
+	{0x8914, 0x000742fe},
+	{0x8918, 0x00044d00},
+	{0x891c, 0x00014200},
+	{0x8920, 0x77000005},
+	{0x8924, 0x52000007},
+	{0x8928, 0x42fe0004},
+	{0x892c, 0x60000007},
+	{0x8930, 0x42000004},
+	{0x8934, 0x60004380},
+	{0x8938, 0x62016100},
+	{0x893c, 0x68046310},
+	{0x8940, 0x41000005},
+	{0x8944, 0x00075500},
+	{0x8948, 0x00045c02},
+	{0x894c, 0x00014300},
+	{0x8950, 0x6c060005},
+	{0x8954, 0xe2aae298},
+	{0x8958, 0xe42ae285},
+	{0x895c, 0xe432e2f3},
+	{0x8960, 0x0001e30c},
+	{0x8964, 0x0005e285},
+	{0x8968, 0xe2986c06},
+	{0x896c, 0xe42ae4a9},
+	{0x8970, 0xe432e2f3},
+	{0x8974, 0x0001e30c},
+	{0x8978, 0x6c000005},
+	{0x897c, 0xe2aae298},
+	{0x8980, 0xe445e285},
+	{0x8984, 0xe44de2f3},
+	{0x8988, 0x0001e30c},
+	{0x898c, 0x0005e285},
+	{0x8990, 0xe2986c00},
+	{0x8994, 0xe445e4a9},
+	{0x8998, 0xe44de2f3},
+	{0x899c, 0x0001e30c},
+	{0x89a0, 0x6c040005},
+	{0x89a4, 0xe2aae298},
+	{0x89a8, 0xe460e285},
+	{0x89ac, 0xe468e2f3},
+	{0x89b0, 0x0001e30c},
+	{0x89b4, 0x0005e285},
+	{0x89b8, 0xe2986c04},
+	{0x89bc, 0xe460e4a9},
+	{0x89c0, 0xe468e2f3},
+	{0x89c4, 0x0001e30c},
+	{0x89c8, 0x6c020005},
+	{0x89cc, 0xe2aae298},
+	{0x89d0, 0xe47be285},
+	{0x89d4, 0xe483e2f3},
+	{0x89d8, 0x0001e30c},
+	{0x89dc, 0x0005e285},
+	{0x89e0, 0xe2986c02},
+	{0x89e4, 0xe47be4a9},
+	{0x89e8, 0xe483e2f3},
+	{0x89ec, 0x0001e30c},
+	{0x89f0, 0x43800004},
+	{0x89f4, 0x610a6008},
+	{0x89f8, 0x63ce6200},
+	{0x89fc, 0x60800006},
+	{0x8a00, 0x00047f00},
+	{0x8a04, 0xe4e04300},
+	{0x8a08, 0x00070001},
+	{0x8a0c, 0x4d015500},
+	{0x8a10, 0x74200004},
+	{0x8a14, 0x57107711},
+	{0x8a18, 0x140f5700},
+	{0x8a1c, 0x00077430},
+	{0x8a20, 0x00044d00},
+	{0x8a24, 0x00074380},
+	{0x8a28, 0x00047200},
+	{0x8a2c, 0x00014300},
+	{0x8a30, 0x74200004},
+	{0x8a34, 0x77000005},
+	{0x8a38, 0x73887e07},
+	{0x8a3c, 0x8f007380},
+	{0x8a40, 0x0004140f},
+	{0x8a44, 0x00057430},
+	{0x8a48, 0x00017300},
+	{0x8a4c, 0x0005e496},
+	{0x8a50, 0x00017300},
+	{0x8a54, 0x43800004},
+	{0x8a58, 0x0006b103},
+	{0x8a5c, 0x91037cdb},
+	{0x8a60, 0x40db0007},
+	{0x8a64, 0x43000004},
+	{0x8a68, 0x0005e496},
+	{0x8a6c, 0x00067380},
+	{0x8a70, 0x60025d01},
+	{0x8a74, 0xe4ba6200},
+	{0x8a78, 0x73000005},
+	{0x8a7c, 0x76080007},
+	{0x8a80, 0x00047578},
+	{0x8a84, 0x00074380},
+	{0x8a88, 0x5e005e01},
+	{0x8a8c, 0x0006140a},
+	{0x8a90, 0x7f006380},
+	{0x8a94, 0x00076080},
+	{0x8a98, 0x4e204c3f},
+	{0x8a9c, 0x73047280},
+	{0x8aa0, 0x140a7300},
+	{0x8aa4, 0x00044d20},
+	{0x8aa8, 0x00064300},
+	{0x8aac, 0x00077402},
+	{0x8ab0, 0x40004001},
+	{0x8ab4, 0x0006ab00},
+	{0x8ab8, 0x00077404},
+	{0x8abc, 0x40004001},
+	{0x8ac0, 0x140aab00},
+	{0x8ac4, 0x43800004},
+	{0x8ac8, 0x52800007},
+	{0x8acc, 0x140a5200},
+	{0x8ad0, 0x4d004c00},
+	{0x8ad4, 0x00064e00},
+	{0x8ad8, 0x63006080},
+	{0x8adc, 0x43000004},
+	{0x8ae0, 0x76000007},
+	{0x8ae4, 0x00040001},
+	{0x8ae8, 0xb1034380},
+	{0x8aec, 0x7cdb0006},
+	{0x8af0, 0x00079103},
+	{0x8af4, 0x000440db},
+	{0x8af8, 0xe4964300},
+	{0x8afc, 0xe4ba7e03},
+	{0x8b00, 0x43800004},
+	{0x8b04, 0x0006b103},
+	{0x8b08, 0x91037c5b},
+	{0x8b0c, 0x405b0007},
+	{0x8b10, 0x43000004},
+	{0x8b14, 0x00010001},
+	{0x8b18, 0x43800004},
+	{0x8b1c, 0x4e200007},
+	{0x8b20, 0x63800006},
+	{0x8b24, 0x5f807cdb},
+	{0x8b28, 0x43000004},
+	{0x8b2c, 0x76080007},
+	{0x8b30, 0x00057560},
+	{0x8b34, 0x00047380},
+	{0x8b38, 0x0005420e},
+	{0x8b3c, 0x14c86c01},
+	{0x8b40, 0x6c001432},
+	{0x8b44, 0x42000004},
+	{0x8b48, 0x43800004},
+	{0x8b4c, 0x5f000006},
+	{0x8b50, 0x73010007},
+	{0x8b54, 0x00047300},
+	{0x8b58, 0x0007420f},
+	{0x8b5c, 0x52005280},
+	{0x8b60, 0x0004140a},
+	{0x8b64, 0x00064200},
+	{0x8b68, 0x7c5b6300},
+	{0x8b6c, 0x4e000007},
+	{0x8b70, 0x43000004},
+	{0x8b74, 0x73000005},
+	{0x8b78, 0x76000007},
+	{0x8b7c, 0xe4c30001},
+	{0x8b80, 0x00040001},
+	{0x8b84, 0x60004380},
+	{0x8b88, 0x62016100},
+	{0x8b8c, 0x00066310},
+	{0x8b90, 0x00046000},
+	{0x8b94, 0x00014300},
+	{0x8b98, 0x0001e4e0},
+	{0x8b9c, 0x4e004f02},
+	{0x8ba0, 0x52015302},
+	{0x8ba4, 0x140f0001},
+	{0x8ba8, 0x00019700},
+	{0x8bac, 0x65014380},
+	{0x8bb0, 0x79007800},
+	{0x8bb4, 0x7b407a00},
+	{0x8bb8, 0x00014300},
+	{0x8bbc, 0x65004380},
+	{0x8bc0, 0x00014300},
+	{0x8bc4, 0x64014380},
+	{0x8bc8, 0x7d007c00},
+	{0x8bcc, 0x7f407e00},
+	{0x8bd0, 0x00014300},
+	{0x8bd4, 0x64004380},
+	{0x8bd8, 0x00014300},
+	{0x8bdc, 0x7b004380},
+	{0x8be0, 0x79007a04},
+	{0x8be4, 0x43007802},
+	{0x8be8, 0x33825509},
+	{0x8bec, 0x43800001},
+	{0x8bf0, 0x7a007b40},
+	{0x8bf4, 0x55194300},
+	{0x8bf8, 0x00013382},
+	{0x8bfc, 0x74007401},
+	{0x8c00, 0x00018e00},
+	{0x8c04, 0x52300007},
+	{0x8c08, 0x74310004},
+	{0x8c0c, 0x8e007430},
+	{0x8c10, 0x52200007},
+	{0x8c14, 0x00010004},
+	{0x8c18, 0x57005702},
+	{0x8c1c, 0x00018e00},
+	{0x8c20, 0x57425740},
+	{0x8c24, 0x8e005740},
+	{0x8c28, 0x00015700},
+	{0x8c2c, 0x561042ef},
+	{0x8c30, 0x42005600},
+	{0x8c34, 0x00018c00},
+	{0x8c38, 0xe3a75b20},
+	{0x8c3c, 0x54005480},
+	{0x8c40, 0x54005481},
+	{0x8c44, 0x54005482},
+	{0x8c48, 0xbf1ae3ac},
+	{0x8c4c, 0xe36e300b},
+	{0x8c50, 0xe390e377},
+	{0x8c54, 0x0001e523},
+	{0x8c58, 0x54c054bf},
+	{0x8c5c, 0x54c154a3},
+	{0x8c60, 0x4c1854a4},
+	{0x8c64, 0xbf091402},
+	{0x8c68, 0x54a454c2},
+	{0x8c6c, 0xbf051402},
+	{0x8c70, 0x54a354c1},
+	{0x8c74, 0xbf011402},
+	{0x8c78, 0x54dfe534},
+	{0x8c7c, 0x54bf0001},
+	{0x8c80, 0x050a54e5},
+	{0x8c84, 0x000154df},
+	{0x8c88, 0x00071657},
+	{0x8c8c, 0x00044c80},
+	{0x8c90, 0x43807430},
+	{0x8c94, 0x7e007f40},
+	{0x8c98, 0x7c027d00},
+	{0x8c9c, 0x5b404300},
+	{0x8ca0, 0x5c015501},
+	{0x8ca4, 0x5480e396},
+	{0x8ca8, 0x54815400},
+	{0x8cac, 0x54825400},
+	{0x8cb0, 0x00075400},
+	{0x8cb4, 0x00044c00},
+	{0x8cb8, 0xe3ac7410},
+	{0x8cbc, 0x300bbfe1},
+	{0x8cc0, 0x56005610},
+	{0x8cc4, 0x00018c00},
+	{0x8cc8, 0x57005704},
+	{0x8ccc, 0xa7038e00},
+	{0x8cd0, 0x33f0aff7},
+	{0x8cd4, 0xaf034019},
+	{0x8cd8, 0x33f0402b},
+	{0x8cdc, 0x33df402b},
+	{0x8ce0, 0x57005708},
+	{0x8ce4, 0x57818e00},
+	{0x8ce8, 0x8e005780},
+	{0x8cec, 0x00074380},
+	{0x8cf0, 0x5c005c01},
+	{0x8cf4, 0x00041403},
+	{0x8cf8, 0x00014300},
+	{0x8cfc, 0x0007427f},
+	{0x8d00, 0x62006280},
+	{0x8d04, 0x00049200},
+	{0x8d08, 0x00014200},
+	{0x8d0c, 0x0007427f},
+	{0x8d10, 0x63146394},
+	{0x8d14, 0x00049200},
+	{0x8d18, 0x00014200},
+	{0x8d1c, 0x42fe0004},
+	{0x8d20, 0x4d010007},
+	{0x8d24, 0x42000004},
+	{0x8d28, 0x140f7420},
+	{0x8d2c, 0x57005710},
+	{0x8d30, 0x0001141f},
+	{0x8d34, 0x42fe0004},
+	{0x8d38, 0x4d010007},
+	{0x8d3c, 0x42000004},
+	{0x8d40, 0x140f7420},
+	{0x8d44, 0x000742bf},
+	{0x8d48, 0x62006240},
+	{0x8d4c, 0x0004141f},
+	{0x8d50, 0x00014200},
+	{0x8d54, 0x5d060006},
+	{0x8d58, 0x61046003},
+	{0x8d5c, 0x00056201},
+	{0x8d60, 0x00017310},
+	{0x8d64, 0x43800004},
+	{0x8d68, 0x5e010007},
+	{0x8d6c, 0x140a5e00},
+	{0x8d70, 0x0006b103},
+	{0x8d74, 0x91037f07},
+	{0x8d78, 0x43070007},
+	{0x8d7c, 0x5c000006},
+	{0x8d80, 0x5e035d02},
+	{0x8d84, 0x43000004},
+	{0x8d88, 0x00060001},
+	{0x8d8c, 0x60005d04},
+	{0x8d90, 0x62016104},
+	{0x8d94, 0x73100005},
+	{0x8d98, 0x00040001},
+	{0x8d9c, 0x00074380},
+	{0x8da0, 0x5e005e01},
+	{0x8da4, 0xb103140a},
+	{0x8da8, 0x7fc60006},
+	{0x8dac, 0x00079103},
+	{0x8db0, 0x000643c6},
+	{0x8db4, 0x5d025c00},
+	{0x8db8, 0x00045e03},
+	{0x8dbc, 0x00014300},
+	{0x8dc0, 0x5d040006},
+	{0x8dc4, 0x61046000},
+	{0x8dc8, 0x00056201},
+	{0x8dcc, 0x00017310},
+	{0x8dd0, 0x43800004},
+	{0x8dd4, 0x5e010007},
+	{0x8dd8, 0x140a5e00},
+	{0x8ddc, 0x0006b103},
+	{0x8de0, 0x91037fc6},
+	{0x8de4, 0x43c60007},
+	{0x8de8, 0x5c000006},
+	{0x8dec, 0x5e035d02},
+	{0x8df0, 0x43000004},
+	{0x8df4, 0x00060001},
+	{0x8df8, 0x60025d00},
+	{0x8dfc, 0x62016100},
+	{0x8e00, 0x73000005},
+	{0x8e04, 0x00040001},
+	{0x8e08, 0x00074380},
+	{0x8e0c, 0x5e005e01},
+	{0x8e10, 0xb103140a},
+	{0x8e14, 0x7fc00006},
+	{0x8e18, 0x00079103},
+	{0x8e1c, 0x000643c0},
+	{0x8e20, 0x5d025c00},
+	{0x8e24, 0x00045e03},
+	{0x8e28, 0x00014300},
+	{0x8e2c, 0x7e020005},
+	{0x8e30, 0x42f70004},
+	{0x8e34, 0x6c080005},
+	{0x8e38, 0x42700004},
+	{0x8e3c, 0x73810005},
+	{0x8e40, 0x93007380},
+	{0x8e44, 0x42f70004},
+	{0x8e48, 0x6c000005},
+	{0x8e4c, 0x42000004},
+	{0x8e50, 0x00040001},
+	{0x8e54, 0x00074380},
+	{0x8e58, 0x73007304},
+	{0x8e5c, 0x72401405},
+	{0x8e60, 0x43000004},
+	{0x8e64, 0x74040006},
+	{0x8e68, 0x40010007},
+	{0x8e6c, 0xab004000},
+	{0x8e70, 0x0001140f},
+	{0x8e74, 0x140ae517},
+	{0x8e78, 0x140ae4c3},
+	{0x8e7c, 0x0001e51e},
+	{0x8e80, 0xe4c3e517},
+	{0x8e84, 0x00040001},
+	{0x8e88, 0x00047410},
+	{0x8e8c, 0x42f04380},
+	{0x8e90, 0x62080007},
+	{0x8e94, 0x24206301},
+	{0x8e98, 0x14c80000},
+	{0x8e9c, 0x00002428},
+	{0x8ea0, 0x1a4215f4},
+	{0x8ea4, 0x6300000b},
+	{0x8ea8, 0x42000004},
+	{0x8eac, 0x74304300},
+	{0x8eb0, 0x4380140f},
+	{0x8eb4, 0x73080007},
+	{0x8eb8, 0x00047300},
+	{0x8ebc, 0x00014300},
+	{0x8ec0, 0x4bf00007},
+	{0x8ec4, 0x490b4a8f},
+	{0x8ec8, 0x4a8e48f1},
+	{0x8ecc, 0x48a5490a},
+	{0x8ed0, 0x49094a8d},
+	{0x8ed4, 0x4a8c487d},
+	{0x8ed8, 0x48754908},
+	{0x8edc, 0x49074a8b},
+	{0x8ee0, 0x4a8a4889},
+	{0x8ee4, 0x48b74906},
+	{0x8ee8, 0x49054a89},
+	{0x8eec, 0x4a8848fc},
+	{0x8ef0, 0x48564905},
+	{0x8ef4, 0x49044a87},
+	{0x8ef8, 0x4a8648c1},
+	{0x8efc, 0x483d4904},
+	{0x8f00, 0x49034a85},
+	{0x8f04, 0x4a8448c7},
+	{0x8f08, 0x485e4903},
+	{0x8f0c, 0x49024a83},
+	{0x8f10, 0x4a8248ac},
+	{0x8f14, 0x48624902},
+	{0x8f18, 0x49024a81},
+	{0x8f1c, 0x4a804820},
+	{0x8f20, 0x48004900},
+	{0x8f24, 0x49014a90},
+	{0x8f28, 0x4a10481f},
+	{0x8f2c, 0x00060001},
+	{0x8f30, 0x5f005f80},
+	{0x8f34, 0x00059900},
+	{0x8f38, 0x00017300},
+	{0x8f3c, 0x63800006},
+	{0x8f40, 0x98006300},
+	{0x8f44, 0x549f0001},
+	{0x8f48, 0x5c015400},
+	{0x8f4c, 0x540054df},
+	{0x8f50, 0x00015c02},
+	{0x8f54, 0x07145c01},
+	{0x8f58, 0x5c025400},
+	{0x8f5c, 0x5c020001},
+	{0x8f60, 0x54000714},
+	{0x8f64, 0x00015c01},
+	{0x8f68, 0x4c184c98},
+	{0x8f6c, 0x00080001},
+	{0x8f70, 0x5c020004},
+	{0x8f74, 0x09017430},
+	{0x8f78, 0x0ba60c01},
+	{0x8f7c, 0x77800005},
+	{0x8f80, 0x52200007},
+	{0x8f84, 0x43800004},
+	{0x8f88, 0x610a6008},
+	{0x8f8c, 0x63c26200},
+	{0x8f90, 0x5c000007},
+	{0x8f94, 0x43000004},
+	{0x8f98, 0x00000001},
+	{0x8080, 0x00000004},
+	{0x8080, 0x00000000},
+	{0x8088, 0x00000000},
+};
+
+static const struct rtw89_txpwr_byrate_cfg rtw89_8851b_txpwr_byrate[] = {
+	{ 0, 0, 0, 0, 4, 0x50505050, },
+	{ 0, 0, 1, 0, 4, 0x54585858, },
+	{ 0, 0, 1, 4, 4, 0x44484c50, },
+	{ 0, 0, 2, 0, 4, 0x50545858, },
+	{ 0, 0, 2, 4, 4, 0x4044484c, },
+	{ 0, 0, 2, 8, 4, 0x3034383c, },
+	{ 0, 0, 3, 0, 4, 0x50505050, },
+	{ 0, 1, 2, 0, 4, 0x50545858, },
+	{ 0, 1, 2, 4, 4, 0x4044484c, },
+	{ 0, 1, 2, 8, 4, 0x3034383c, },
+	{ 0, 1, 3, 0, 4, 0x50505050, },
+	{ 0, 0, 4, 1, 4, 0x00000000, },
+	{ 0, 0, 4, 0, 1, 0x00000000, },
+	{ 1, 0, 1, 0, 4, 0x58585858, },
+	{ 1, 0, 1, 4, 4, 0x484c5054, },
+	{ 1, 0, 2, 0, 4, 0x54585858, },
+	{ 1, 0, 2, 4, 4, 0x44484c50, },
+	{ 1, 0, 2, 8, 4, 0x34383c40, },
+	{ 1, 0, 3, 0, 4, 0x40404040, },
+	{ 1, 1, 2, 0, 4, 0x54585858, },
+	{ 1, 1, 2, 4, 4, 0x44484c50, },
+	{ 1, 1, 2, 8, 4, 0x34383c40, },
+	{ 1, 1, 3, 0, 4, 0x48484848, },
+	{ 1, 0, 4, 0, 4, 0x00000000, },
+	{ 2, 0, 1, 0, 4, 0x40404040, },
+	{ 2, 0, 1, 4, 4, 0x383c4040, },
+	{ 2, 0, 2, 0, 4, 0x40404040, },
+	{ 2, 0, 2, 4, 4, 0x34383c40, },
+	{ 2, 0, 2, 8, 4, 0x24282c30, },
+	{ 2, 0, 3, 0, 4, 0x40404040, },
+	{ 2, 1, 2, 0, 4, 0x40404040, },
+	{ 2, 1, 2, 4, 4, 0x34383c40, },
+	{ 2, 1, 2, 8, 4, 0x24282c30, },
+	{ 2, 1, 3, 0, 4, 0x40404040, },
+	{ 2, 0, 4, 0, 4, 0x00000000, },
+};
+
+static const s8 _txpwr_track_delta_swingidx_5ga_n[][DELTA_SWINGIDX_SIZE] = {
+	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0,
+	 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1},
+	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1,
+	 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4},
+};
+
+static const s8 _txpwr_track_delta_swingidx_5ga_p[][DELTA_SWINGIDX_SIZE] = {
+	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1,
+	 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+	{0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,
+	 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2},
+};
+
+static const s8 _txpwr_track_delta_swingidx_2ga_n[] = {
+	0, 0, 0, 0, -1, -1, -1, -2, -2, -2, -2, -3, -3, -3, -3, -3,
+	 -4, -4, -4, -4, -4, -5, -5, -5, -5, -5, -5, -6, -6, -6
+};
+
+static const s8 _txpwr_track_delta_swingidx_2ga_p[] = {
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4
+};
+
+static const s8 _txpwr_track_delta_swingidx_2g_cck_a_n[] = {
+	0, 0, 0, 0, -1, -1, -1, -2, -2, -2, -2, -3, -3, -3, -3, -3,
+	 -4, -4, -4, -4, -4, -5, -5, -5, -5, -5, -5, -6, -6, -6
+};
+
+static const s8 _txpwr_track_delta_swingidx_2g_cck_a_p[] = {
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4
+};
+
+const u8 rtw89_8851b_tx_shape[RTW89_BAND_MAX][RTW89_RS_TX_SHAPE_NUM]
+			     [RTW89_REGD_NUM] = {
+	[0][0][RTW89_ACMA] = 0,
+	[0][0][RTW89_CN] = 0,
+	[0][0][RTW89_ETSI] = 0,
+	[0][0][RTW89_FCC] = 1,
+	[0][0][RTW89_IC] = 1,
+	[0][0][RTW89_KCC] = 0,
+	[0][0][RTW89_MKK] = 0,
+	[0][0][RTW89_UK] = 0,
+	[0][1][RTW89_ACMA] = 0,
+	[0][1][RTW89_CN] = 0,
+	[0][1][RTW89_ETSI] = 0,
+	[0][1][RTW89_FCC] = 3,
+	[0][1][RTW89_IC] = 3,
+	[0][1][RTW89_KCC] = 0,
+	[0][1][RTW89_MKK] = 0,
+	[0][1][RTW89_UK] = 0,
+	[1][1][RTW89_ACMA] = 0,
+	[1][1][RTW89_CN] = 0,
+	[1][1][RTW89_ETSI] = 0,
+	[1][1][RTW89_FCC] = 3,
+	[1][1][RTW89_IC] = 3,
+	[1][1][RTW89_KCC] = 0,
+	[1][1][RTW89_MKK] = 0,
+	[1][1][RTW89_UK] = 0,
+};
+
+static
+const s8 rtw89_8851b_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
+				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
+				 [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
+	[0][0][0][0][RTW89_WW][0] = 58,
+	[0][0][0][0][RTW89_WW][1] = 58,
+	[0][0][0][0][RTW89_WW][2] = 58,
+	[0][0][0][0][RTW89_WW][3] = 58,
+	[0][0][0][0][RTW89_WW][4] = 58,
+	[0][0][0][0][RTW89_WW][5] = 58,
+	[0][0][0][0][RTW89_WW][6] = 58,
+	[0][0][0][0][RTW89_WW][7] = 58,
+	[0][0][0][0][RTW89_WW][8] = 58,
+	[0][0][0][0][RTW89_WW][9] = 58,
+	[0][0][0][0][RTW89_WW][10] = 58,
+	[0][0][0][0][RTW89_WW][11] = 58,
+	[0][0][0][0][RTW89_WW][12] = 52,
+	[0][0][0][0][RTW89_WW][13] = 76,
+	[0][1][0][0][RTW89_WW][0] = 0,
+	[0][1][0][0][RTW89_WW][1] = 0,
+	[0][1][0][0][RTW89_WW][2] = 0,
+	[0][1][0][0][RTW89_WW][3] = 0,
+	[0][1][0][0][RTW89_WW][4] = 0,
+	[0][1][0][0][RTW89_WW][5] = 0,
+	[0][1][0][0][RTW89_WW][6] = 0,
+	[0][1][0][0][RTW89_WW][7] = 0,
+	[0][1][0][0][RTW89_WW][8] = 0,
+	[0][1][0][0][RTW89_WW][9] = 0,
+	[0][1][0][0][RTW89_WW][10] = 0,
+	[0][1][0][0][RTW89_WW][11] = 0,
+	[0][1][0][0][RTW89_WW][12] = 0,
+	[0][1][0][0][RTW89_WW][13] = 0,
+	[1][0][0][0][RTW89_WW][0] = 0,
+	[1][0][0][0][RTW89_WW][1] = 0,
+	[1][0][0][0][RTW89_WW][2] = 58,
+	[1][0][0][0][RTW89_WW][3] = 58,
+	[1][0][0][0][RTW89_WW][4] = 58,
+	[1][0][0][0][RTW89_WW][5] = 58,
+	[1][0][0][0][RTW89_WW][6] = 58,
+	[1][0][0][0][RTW89_WW][7] = 58,
+	[1][0][0][0][RTW89_WW][8] = 58,
+	[1][0][0][0][RTW89_WW][9] = 58,
+	[1][0][0][0][RTW89_WW][10] = 58,
+	[1][0][0][0][RTW89_WW][11] = 0,
+	[1][0][0][0][RTW89_WW][12] = 0,
+	[1][0][0][0][RTW89_WW][13] = 0,
+	[1][1][0][0][RTW89_WW][0] = 0,
+	[1][1][0][0][RTW89_WW][1] = 0,
+	[1][1][0][0][RTW89_WW][2] = 0,
+	[1][1][0][0][RTW89_WW][3] = 0,
+	[1][1][0][0][RTW89_WW][4] = 0,
+	[1][1][0][0][RTW89_WW][5] = 0,
+	[1][1][0][0][RTW89_WW][6] = 0,
+	[1][1][0][0][RTW89_WW][7] = 0,
+	[1][1][0][0][RTW89_WW][8] = 0,
+	[1][1][0][0][RTW89_WW][9] = 0,
+	[1][1][0][0][RTW89_WW][10] = 0,
+	[1][1][0][0][RTW89_WW][11] = 0,
+	[1][1][0][0][RTW89_WW][12] = 0,
+	[1][1][0][0][RTW89_WW][13] = 0,
+	[0][0][1][0][RTW89_WW][0] = 58,
+	[0][0][1][0][RTW89_WW][1] = 60,
+	[0][0][1][0][RTW89_WW][2] = 60,
+	[0][0][1][0][RTW89_WW][3] = 60,
+	[0][0][1][0][RTW89_WW][4] = 60,
+	[0][0][1][0][RTW89_WW][5] = 60,
+	[0][0][1][0][RTW89_WW][6] = 60,
+	[0][0][1][0][RTW89_WW][7] = 60,
+	[0][0][1][0][RTW89_WW][8] = 60,
+	[0][0][1][0][RTW89_WW][9] = 60,
+	[0][0][1][0][RTW89_WW][10] = 60,
+	[0][0][1][0][RTW89_WW][11] = 60,
+	[0][0][1][0][RTW89_WW][12] = 58,
+	[0][0][1][0][RTW89_WW][13] = 0,
+	[0][1][1][0][RTW89_WW][0] = 0,
+	[0][1][1][0][RTW89_WW][1] = 0,
+	[0][1][1][0][RTW89_WW][2] = 0,
+	[0][1][1][0][RTW89_WW][3] = 0,
+	[0][1][1][0][RTW89_WW][4] = 0,
+	[0][1][1][0][RTW89_WW][5] = 0,
+	[0][1][1][0][RTW89_WW][6] = 0,
+	[0][1][1][0][RTW89_WW][7] = 0,
+	[0][1][1][0][RTW89_WW][8] = 0,
+	[0][1][1][0][RTW89_WW][9] = 0,
+	[0][1][1][0][RTW89_WW][10] = 0,
+	[0][1][1][0][RTW89_WW][11] = 0,
+	[0][1][1][0][RTW89_WW][12] = 0,
+	[0][1][1][0][RTW89_WW][13] = 0,
+	[0][0][2][0][RTW89_WW][0] = 60,
+	[0][0][2][0][RTW89_WW][1] = 60,
+	[0][0][2][0][RTW89_WW][2] = 60,
+	[0][0][2][0][RTW89_WW][3] = 60,
+	[0][0][2][0][RTW89_WW][4] = 60,
+	[0][0][2][0][RTW89_WW][5] = 60,
+	[0][0][2][0][RTW89_WW][6] = 60,
+	[0][0][2][0][RTW89_WW][7] = 60,
+	[0][0][2][0][RTW89_WW][8] = 60,
+	[0][0][2][0][RTW89_WW][9] = 60,
+	[0][0][2][0][RTW89_WW][10] = 60,
+	[0][0][2][0][RTW89_WW][11] = 60,
+	[0][0][2][0][RTW89_WW][12] = 60,
+	[0][0][2][0][RTW89_WW][13] = 0,
+	[0][1][2][0][RTW89_WW][0] = 0,
+	[0][1][2][0][RTW89_WW][1] = 0,
+	[0][1][2][0][RTW89_WW][2] = 0,
+	[0][1][2][0][RTW89_WW][3] = 0,
+	[0][1][2][0][RTW89_WW][4] = 0,
+	[0][1][2][0][RTW89_WW][5] = 0,
+	[0][1][2][0][RTW89_WW][6] = 0,
+	[0][1][2][0][RTW89_WW][7] = 0,
+	[0][1][2][0][RTW89_WW][8] = 0,
+	[0][1][2][0][RTW89_WW][9] = 0,
+	[0][1][2][0][RTW89_WW][10] = 0,
+	[0][1][2][0][RTW89_WW][11] = 0,
+	[0][1][2][0][RTW89_WW][12] = 0,
+	[0][1][2][0][RTW89_WW][13] = 0,
+	[0][1][2][1][RTW89_WW][0] = 0,
+	[0][1][2][1][RTW89_WW][1] = 0,
+	[0][1][2][1][RTW89_WW][2] = 0,
+	[0][1][2][1][RTW89_WW][3] = 0,
+	[0][1][2][1][RTW89_WW][4] = 0,
+	[0][1][2][1][RTW89_WW][5] = 0,
+	[0][1][2][1][RTW89_WW][6] = 0,
+	[0][1][2][1][RTW89_WW][7] = 0,
+	[0][1][2][1][RTW89_WW][8] = 0,
+	[0][1][2][1][RTW89_WW][9] = 0,
+	[0][1][2][1][RTW89_WW][10] = 0,
+	[0][1][2][1][RTW89_WW][11] = 0,
+	[0][1][2][1][RTW89_WW][12] = 0,
+	[0][1][2][1][RTW89_WW][13] = 0,
+	[1][0][2][0][RTW89_WW][0] = 0,
+	[1][0][2][0][RTW89_WW][1] = 0,
+	[1][0][2][0][RTW89_WW][2] = 58,
+	[1][0][2][0][RTW89_WW][3] = 58,
+	[1][0][2][0][RTW89_WW][4] = 58,
+	[1][0][2][0][RTW89_WW][5] = 58,
+	[1][0][2][0][RTW89_WW][6] = 58,
+	[1][0][2][0][RTW89_WW][7] = 58,
+	[1][0][2][0][RTW89_WW][8] = 58,
+	[1][0][2][0][RTW89_WW][9] = 58,
+	[1][0][2][0][RTW89_WW][10] = 58,
+	[1][0][2][0][RTW89_WW][11] = 0,
+	[1][0][2][0][RTW89_WW][12] = 0,
+	[1][0][2][0][RTW89_WW][13] = 0,
+	[1][1][2][0][RTW89_WW][0] = 0,
+	[1][1][2][0][RTW89_WW][1] = 0,
+	[1][1][2][0][RTW89_WW][2] = 0,
+	[1][1][2][0][RTW89_WW][3] = 0,
+	[1][1][2][0][RTW89_WW][4] = 0,
+	[1][1][2][0][RTW89_WW][5] = 0,
+	[1][1][2][0][RTW89_WW][6] = 0,
+	[1][1][2][0][RTW89_WW][7] = 0,
+	[1][1][2][0][RTW89_WW][8] = 0,
+	[1][1][2][0][RTW89_WW][9] = 0,
+	[1][1][2][0][RTW89_WW][10] = 0,
+	[1][1][2][0][RTW89_WW][11] = 0,
+	[1][1][2][0][RTW89_WW][12] = 0,
+	[1][1][2][0][RTW89_WW][13] = 0,
+	[1][1][2][1][RTW89_WW][0] = 0,
+	[1][1][2][1][RTW89_WW][1] = 0,
+	[1][1][2][1][RTW89_WW][2] = 0,
+	[1][1][2][1][RTW89_WW][3] = 0,
+	[1][1][2][1][RTW89_WW][4] = 0,
+	[1][1][2][1][RTW89_WW][5] = 0,
+	[1][1][2][1][RTW89_WW][6] = 0,
+	[1][1][2][1][RTW89_WW][7] = 0,
+	[1][1][2][1][RTW89_WW][8] = 0,
+	[1][1][2][1][RTW89_WW][9] = 0,
+	[1][1][2][1][RTW89_WW][10] = 0,
+	[1][1][2][1][RTW89_WW][11] = 0,
+	[1][1][2][1][RTW89_WW][12] = 0,
+	[1][1][2][1][RTW89_WW][13] = 0,
+	[0][0][0][0][RTW89_FCC][0] = 84,
+	[0][0][0][0][RTW89_ETSI][0] = 58,
+	[0][0][0][0][RTW89_MKK][0] = 68,
+	[0][0][0][0][RTW89_IC][0] = 84,
+	[0][0][0][0][RTW89_KCC][0] = 68,
+	[0][0][0][0][RTW89_ACMA][0] = 58,
+	[0][0][0][0][RTW89_CN][0] = 60,
+	[0][0][0][0][RTW89_UK][0] = 58,
+	[0][0][0][0][RTW89_FCC][1] = 84,
+	[0][0][0][0][RTW89_ETSI][1] = 58,
+	[0][0][0][0][RTW89_MKK][1] = 68,
+	[0][0][0][0][RTW89_IC][1] = 84,
+	[0][0][0][0][RTW89_KCC][1] = 68,
+	[0][0][0][0][RTW89_ACMA][1] = 58,
+	[0][0][0][0][RTW89_CN][1] = 60,
+	[0][0][0][0][RTW89_UK][1] = 58,
+	[0][0][0][0][RTW89_FCC][2] = 84,
+	[0][0][0][0][RTW89_ETSI][2] = 58,
+	[0][0][0][0][RTW89_MKK][2] = 68,
+	[0][0][0][0][RTW89_IC][2] = 84,
+	[0][0][0][0][RTW89_KCC][2] = 68,
+	[0][0][0][0][RTW89_ACMA][2] = 58,
+	[0][0][0][0][RTW89_CN][2] = 60,
+	[0][0][0][0][RTW89_UK][2] = 58,
+	[0][0][0][0][RTW89_FCC][3] = 84,
+	[0][0][0][0][RTW89_ETSI][3] = 58,
+	[0][0][0][0][RTW89_MKK][3] = 68,
+	[0][0][0][0][RTW89_IC][3] = 84,
+	[0][0][0][0][RTW89_KCC][3] = 68,
+	[0][0][0][0][RTW89_ACMA][3] = 58,
+	[0][0][0][0][RTW89_CN][3] = 60,
+	[0][0][0][0][RTW89_UK][3] = 58,
+	[0][0][0][0][RTW89_FCC][4] = 84,
+	[0][0][0][0][RTW89_ETSI][4] = 58,
+	[0][0][0][0][RTW89_MKK][4] = 68,
+	[0][0][0][0][RTW89_IC][4] = 84,
+	[0][0][0][0][RTW89_KCC][4] = 68,
+	[0][0][0][0][RTW89_ACMA][4] = 58,
+	[0][0][0][0][RTW89_CN][4] = 60,
+	[0][0][0][0][RTW89_UK][4] = 58,
+	[0][0][0][0][RTW89_FCC][5] = 84,
+	[0][0][0][0][RTW89_ETSI][5] = 58,
+	[0][0][0][0][RTW89_MKK][5] = 68,
+	[0][0][0][0][RTW89_IC][5] = 84,
+	[0][0][0][0][RTW89_KCC][5] = 68,
+	[0][0][0][0][RTW89_ACMA][5] = 58,
+	[0][0][0][0][RTW89_CN][5] = 60,
+	[0][0][0][0][RTW89_UK][5] = 58,
+	[0][0][0][0][RTW89_FCC][6] = 84,
+	[0][0][0][0][RTW89_ETSI][6] = 58,
+	[0][0][0][0][RTW89_MKK][6] = 68,
+	[0][0][0][0][RTW89_IC][6] = 84,
+	[0][0][0][0][RTW89_KCC][6] = 68,
+	[0][0][0][0][RTW89_ACMA][6] = 58,
+	[0][0][0][0][RTW89_CN][6] = 60,
+	[0][0][0][0][RTW89_UK][6] = 58,
+	[0][0][0][0][RTW89_FCC][7] = 84,
+	[0][0][0][0][RTW89_ETSI][7] = 58,
+	[0][0][0][0][RTW89_MKK][7] = 68,
+	[0][0][0][0][RTW89_IC][7] = 84,
+	[0][0][0][0][RTW89_KCC][7] = 68,
+	[0][0][0][0][RTW89_ACMA][7] = 58,
+	[0][0][0][0][RTW89_CN][7] = 60,
+	[0][0][0][0][RTW89_UK][7] = 58,
+	[0][0][0][0][RTW89_FCC][8] = 84,
+	[0][0][0][0][RTW89_ETSI][8] = 58,
+	[0][0][0][0][RTW89_MKK][8] = 68,
+	[0][0][0][0][RTW89_IC][8] = 84,
+	[0][0][0][0][RTW89_KCC][8] = 68,
+	[0][0][0][0][RTW89_ACMA][8] = 58,
+	[0][0][0][0][RTW89_CN][8] = 60,
+	[0][0][0][0][RTW89_UK][8] = 58,
+	[0][0][0][0][RTW89_FCC][9] = 84,
+	[0][0][0][0][RTW89_ETSI][9] = 58,
+	[0][0][0][0][RTW89_MKK][9] = 68,
+	[0][0][0][0][RTW89_IC][9] = 84,
+	[0][0][0][0][RTW89_KCC][9] = 68,
+	[0][0][0][0][RTW89_ACMA][9] = 58,
+	[0][0][0][0][RTW89_CN][9] = 60,
+	[0][0][0][0][RTW89_UK][9] = 58,
+	[0][0][0][0][RTW89_FCC][10] = 82,
+	[0][0][0][0][RTW89_ETSI][10] = 58,
+	[0][0][0][0][RTW89_MKK][10] = 68,
+	[0][0][0][0][RTW89_IC][10] = 82,
+	[0][0][0][0][RTW89_KCC][10] = 68,
+	[0][0][0][0][RTW89_ACMA][10] = 58,
+	[0][0][0][0][RTW89_CN][10] = 60,
+	[0][0][0][0][RTW89_UK][10] = 58,
+	[0][0][0][0][RTW89_FCC][11] = 62,
+	[0][0][0][0][RTW89_ETSI][11] = 58,
+	[0][0][0][0][RTW89_MKK][11] = 68,
+	[0][0][0][0][RTW89_IC][11] = 62,
+	[0][0][0][0][RTW89_KCC][11] = 68,
+	[0][0][0][0][RTW89_ACMA][11] = 58,
+	[0][0][0][0][RTW89_CN][11] = 60,
+	[0][0][0][0][RTW89_UK][11] = 58,
+	[0][0][0][0][RTW89_FCC][12] = 52,
+	[0][0][0][0][RTW89_ETSI][12] = 58,
+	[0][0][0][0][RTW89_MKK][12] = 68,
+	[0][0][0][0][RTW89_IC][12] = 52,
+	[0][0][0][0][RTW89_KCC][12] = 68,
+	[0][0][0][0][RTW89_ACMA][12] = 58,
+	[0][0][0][0][RTW89_CN][12] = 60,
+	[0][0][0][0][RTW89_UK][12] = 58,
+	[0][0][0][0][RTW89_FCC][13] = 127,
+	[0][0][0][0][RTW89_ETSI][13] = 127,
+	[0][0][0][0][RTW89_MKK][13] = 76,
+	[0][0][0][0][RTW89_IC][13] = 127,
+	[0][0][0][0][RTW89_KCC][13] = 127,
+	[0][0][0][0][RTW89_ACMA][13] = 127,
+	[0][0][0][0][RTW89_CN][13] = 127,
+	[0][0][0][0][RTW89_UK][13] = 127,
+	[0][1][0][0][RTW89_FCC][0] = 127,
+	[0][1][0][0][RTW89_ETSI][0] = 127,
+	[0][1][0][0][RTW89_MKK][0] = 127,
+	[0][1][0][0][RTW89_IC][0] = 127,
+	[0][1][0][0][RTW89_KCC][0] = 127,
+	[0][1][0][0][RTW89_ACMA][0] = 127,
+	[0][1][0][0][RTW89_CN][0] = 127,
+	[0][1][0][0][RTW89_UK][0] = 127,
+	[0][1][0][0][RTW89_FCC][1] = 127,
+	[0][1][0][0][RTW89_ETSI][1] = 127,
+	[0][1][0][0][RTW89_MKK][1] = 127,
+	[0][1][0][0][RTW89_IC][1] = 127,
+	[0][1][0][0][RTW89_KCC][1] = 127,
+	[0][1][0][0][RTW89_ACMA][1] = 127,
+	[0][1][0][0][RTW89_CN][1] = 127,
+	[0][1][0][0][RTW89_UK][1] = 127,
+	[0][1][0][0][RTW89_FCC][2] = 127,
+	[0][1][0][0][RTW89_ETSI][2] = 127,
+	[0][1][0][0][RTW89_MKK][2] = 127,
+	[0][1][0][0][RTW89_IC][2] = 127,
+	[0][1][0][0][RTW89_KCC][2] = 127,
+	[0][1][0][0][RTW89_ACMA][2] = 127,
+	[0][1][0][0][RTW89_CN][2] = 127,
+	[0][1][0][0][RTW89_UK][2] = 127,
+	[0][1][0][0][RTW89_FCC][3] = 127,
+	[0][1][0][0][RTW89_ETSI][3] = 127,
+	[0][1][0][0][RTW89_MKK][3] = 127,
+	[0][1][0][0][RTW89_IC][3] = 127,
+	[0][1][0][0][RTW89_KCC][3] = 127,
+	[0][1][0][0][RTW89_ACMA][3] = 127,
+	[0][1][0][0][RTW89_CN][3] = 127,
+	[0][1][0][0][RTW89_UK][3] = 127,
+	[0][1][0][0][RTW89_FCC][4] = 127,
+	[0][1][0][0][RTW89_ETSI][4] = 127,
+	[0][1][0][0][RTW89_MKK][4] = 127,
+	[0][1][0][0][RTW89_IC][4] = 127,
+	[0][1][0][0][RTW89_KCC][4] = 127,
+	[0][1][0][0][RTW89_ACMA][4] = 127,
+	[0][1][0][0][RTW89_CN][4] = 127,
+	[0][1][0][0][RTW89_UK][4] = 127,
+	[0][1][0][0][RTW89_FCC][5] = 127,
+	[0][1][0][0][RTW89_ETSI][5] = 127,
+	[0][1][0][0][RTW89_MKK][5] = 127,
+	[0][1][0][0][RTW89_IC][5] = 127,
+	[0][1][0][0][RTW89_KCC][5] = 127,
+	[0][1][0][0][RTW89_ACMA][5] = 127,
+	[0][1][0][0][RTW89_CN][5] = 127,
+	[0][1][0][0][RTW89_UK][5] = 127,
+	[0][1][0][0][RTW89_FCC][6] = 127,
+	[0][1][0][0][RTW89_ETSI][6] = 127,
+	[0][1][0][0][RTW89_MKK][6] = 127,
+	[0][1][0][0][RTW89_IC][6] = 127,
+	[0][1][0][0][RTW89_KCC][6] = 127,
+	[0][1][0][0][RTW89_ACMA][6] = 127,
+	[0][1][0][0][RTW89_CN][6] = 127,
+	[0][1][0][0][RTW89_UK][6] = 127,
+	[0][1][0][0][RTW89_FCC][7] = 127,
+	[0][1][0][0][RTW89_ETSI][7] = 127,
+	[0][1][0][0][RTW89_MKK][7] = 127,
+	[0][1][0][0][RTW89_IC][7] = 127,
+	[0][1][0][0][RTW89_KCC][7] = 127,
+	[0][1][0][0][RTW89_ACMA][7] = 127,
+	[0][1][0][0][RTW89_CN][7] = 127,
+	[0][1][0][0][RTW89_UK][7] = 127,
+	[0][1][0][0][RTW89_FCC][8] = 127,
+	[0][1][0][0][RTW89_ETSI][8] = 127,
+	[0][1][0][0][RTW89_MKK][8] = 127,
+	[0][1][0][0][RTW89_IC][8] = 127,
+	[0][1][0][0][RTW89_KCC][8] = 127,
+	[0][1][0][0][RTW89_ACMA][8] = 127,
+	[0][1][0][0][RTW89_CN][8] = 127,
+	[0][1][0][0][RTW89_UK][8] = 127,
+	[0][1][0][0][RTW89_FCC][9] = 127,
+	[0][1][0][0][RTW89_ETSI][9] = 127,
+	[0][1][0][0][RTW89_MKK][9] = 127,
+	[0][1][0][0][RTW89_IC][9] = 127,
+	[0][1][0][0][RTW89_KCC][9] = 127,
+	[0][1][0][0][RTW89_ACMA][9] = 127,
+	[0][1][0][0][RTW89_CN][9] = 127,
+	[0][1][0][0][RTW89_UK][9] = 127,
+	[0][1][0][0][RTW89_FCC][10] = 127,
+	[0][1][0][0][RTW89_ETSI][10] = 127,
+	[0][1][0][0][RTW89_MKK][10] = 127,
+	[0][1][0][0][RTW89_IC][10] = 127,
+	[0][1][0][0][RTW89_KCC][10] = 127,
+	[0][1][0][0][RTW89_ACMA][10] = 127,
+	[0][1][0][0][RTW89_CN][10] = 127,
+	[0][1][0][0][RTW89_UK][10] = 127,
+	[0][1][0][0][RTW89_FCC][11] = 127,
+	[0][1][0][0][RTW89_ETSI][11] = 127,
+	[0][1][0][0][RTW89_MKK][11] = 127,
+	[0][1][0][0][RTW89_IC][11] = 127,
+	[0][1][0][0][RTW89_KCC][11] = 127,
+	[0][1][0][0][RTW89_ACMA][11] = 127,
+	[0][1][0][0][RTW89_CN][11] = 127,
+	[0][1][0][0][RTW89_UK][11] = 127,
+	[0][1][0][0][RTW89_FCC][12] = 127,
+	[0][1][0][0][RTW89_ETSI][12] = 127,
+	[0][1][0][0][RTW89_MKK][12] = 127,
+	[0][1][0][0][RTW89_IC][12] = 127,
+	[0][1][0][0][RTW89_KCC][12] = 127,
+	[0][1][0][0][RTW89_ACMA][12] = 127,
+	[0][1][0][0][RTW89_CN][12] = 127,
+	[0][1][0][0][RTW89_UK][12] = 127,
+	[0][1][0][0][RTW89_FCC][13] = 127,
+	[0][1][0][0][RTW89_ETSI][13] = 127,
+	[0][1][0][0][RTW89_MKK][13] = 127,
+	[0][1][0][0][RTW89_IC][13] = 127,
+	[0][1][0][0][RTW89_KCC][13] = 127,
+	[0][1][0][0][RTW89_ACMA][13] = 127,
+	[0][1][0][0][RTW89_CN][13] = 127,
+	[0][1][0][0][RTW89_UK][13] = 127,
+	[1][0][0][0][RTW89_FCC][0] = 127,
+	[1][0][0][0][RTW89_ETSI][0] = 127,
+	[1][0][0][0][RTW89_MKK][0] = 127,
+	[1][0][0][0][RTW89_IC][0] = 127,
+	[1][0][0][0][RTW89_KCC][0] = 127,
+	[1][0][0][0][RTW89_ACMA][0] = 127,
+	[1][0][0][0][RTW89_CN][0] = 127,
+	[1][0][0][0][RTW89_UK][0] = 127,
+	[1][0][0][0][RTW89_FCC][1] = 127,
+	[1][0][0][0][RTW89_ETSI][1] = 127,
+	[1][0][0][0][RTW89_MKK][1] = 127,
+	[1][0][0][0][RTW89_IC][1] = 127,
+	[1][0][0][0][RTW89_KCC][1] = 127,
+	[1][0][0][0][RTW89_ACMA][1] = 127,
+	[1][0][0][0][RTW89_CN][1] = 127,
+	[1][0][0][0][RTW89_UK][1] = 127,
+	[1][0][0][0][RTW89_FCC][2] = 127,
+	[1][0][0][0][RTW89_ETSI][2] = 58,
+	[1][0][0][0][RTW89_MKK][2] = 70,
+	[1][0][0][0][RTW89_IC][2] = 127,
+	[1][0][0][0][RTW89_KCC][2] = 68,
+	[1][0][0][0][RTW89_ACMA][2] = 58,
+	[1][0][0][0][RTW89_CN][2] = 60,
+	[1][0][0][0][RTW89_UK][2] = 58,
+	[1][0][0][0][RTW89_FCC][3] = 127,
+	[1][0][0][0][RTW89_ETSI][3] = 58,
+	[1][0][0][0][RTW89_MKK][3] = 76,
+	[1][0][0][0][RTW89_IC][3] = 127,
+	[1][0][0][0][RTW89_KCC][3] = 68,
+	[1][0][0][0][RTW89_ACMA][3] = 58,
+	[1][0][0][0][RTW89_CN][3] = 60,
+	[1][0][0][0][RTW89_UK][3] = 58,
+	[1][0][0][0][RTW89_FCC][4] = 127,
+	[1][0][0][0][RTW89_ETSI][4] = 58,
+	[1][0][0][0][RTW89_MKK][4] = 76,
+	[1][0][0][0][RTW89_IC][4] = 127,
+	[1][0][0][0][RTW89_KCC][4] = 68,
+	[1][0][0][0][RTW89_ACMA][4] = 58,
+	[1][0][0][0][RTW89_CN][4] = 60,
+	[1][0][0][0][RTW89_UK][4] = 58,
+	[1][0][0][0][RTW89_FCC][5] = 127,
+	[1][0][0][0][RTW89_ETSI][5] = 58,
+	[1][0][0][0][RTW89_MKK][5] = 76,
+	[1][0][0][0][RTW89_IC][5] = 127,
+	[1][0][0][0][RTW89_KCC][5] = 68,
+	[1][0][0][0][RTW89_ACMA][5] = 58,
+	[1][0][0][0][RTW89_CN][5] = 60,
+	[1][0][0][0][RTW89_UK][5] = 58,
+	[1][0][0][0][RTW89_FCC][6] = 127,
+	[1][0][0][0][RTW89_ETSI][6] = 58,
+	[1][0][0][0][RTW89_MKK][6] = 76,
+	[1][0][0][0][RTW89_IC][6] = 127,
+	[1][0][0][0][RTW89_KCC][6] = 68,
+	[1][0][0][0][RTW89_ACMA][6] = 58,
+	[1][0][0][0][RTW89_CN][6] = 60,
+	[1][0][0][0][RTW89_UK][6] = 58,
+	[1][0][0][0][RTW89_FCC][7] = 127,
+	[1][0][0][0][RTW89_ETSI][7] = 58,
+	[1][0][0][0][RTW89_MKK][7] = 76,
+	[1][0][0][0][RTW89_IC][7] = 127,
+	[1][0][0][0][RTW89_KCC][7] = 68,
+	[1][0][0][0][RTW89_ACMA][7] = 58,
+	[1][0][0][0][RTW89_CN][7] = 60,
+	[1][0][0][0][RTW89_UK][7] = 58,
+	[1][0][0][0][RTW89_FCC][8] = 127,
+	[1][0][0][0][RTW89_ETSI][8] = 58,
+	[1][0][0][0][RTW89_MKK][8] = 76,
+	[1][0][0][0][RTW89_IC][8] = 127,
+	[1][0][0][0][RTW89_KCC][8] = 68,
+	[1][0][0][0][RTW89_ACMA][8] = 58,
+	[1][0][0][0][RTW89_CN][8] = 60,
+	[1][0][0][0][RTW89_UK][8] = 58,
+	[1][0][0][0][RTW89_FCC][9] = 127,
+	[1][0][0][0][RTW89_ETSI][9] = 58,
+	[1][0][0][0][RTW89_MKK][9] = 76,
+	[1][0][0][0][RTW89_IC][9] = 127,
+	[1][0][0][0][RTW89_KCC][9] = 68,
+	[1][0][0][0][RTW89_ACMA][9] = 58,
+	[1][0][0][0][RTW89_CN][9] = 60,
+	[1][0][0][0][RTW89_UK][9] = 58,
+	[1][0][0][0][RTW89_FCC][10] = 127,
+	[1][0][0][0][RTW89_ETSI][10] = 58,
+	[1][0][0][0][RTW89_MKK][10] = 66,
+	[1][0][0][0][RTW89_IC][10] = 127,
+	[1][0][0][0][RTW89_KCC][10] = 68,
+	[1][0][0][0][RTW89_ACMA][10] = 58,
+	[1][0][0][0][RTW89_CN][10] = 60,
+	[1][0][0][0][RTW89_UK][10] = 58,
+	[1][0][0][0][RTW89_FCC][11] = 127,
+	[1][0][0][0][RTW89_ETSI][11] = 127,
+	[1][0][0][0][RTW89_MKK][11] = 127,
+	[1][0][0][0][RTW89_IC][11] = 127,
+	[1][0][0][0][RTW89_KCC][11] = 127,
+	[1][0][0][0][RTW89_ACMA][11] = 127,
+	[1][0][0][0][RTW89_CN][11] = 127,
+	[1][0][0][0][RTW89_UK][11] = 127,
+	[1][0][0][0][RTW89_FCC][12] = 127,
+	[1][0][0][0][RTW89_ETSI][12] = 127,
+	[1][0][0][0][RTW89_MKK][12] = 127,
+	[1][0][0][0][RTW89_IC][12] = 127,
+	[1][0][0][0][RTW89_KCC][12] = 127,
+	[1][0][0][0][RTW89_ACMA][12] = 127,
+	[1][0][0][0][RTW89_CN][12] = 127,
+	[1][0][0][0][RTW89_UK][12] = 127,
+	[1][0][0][0][RTW89_FCC][13] = 127,
+	[1][0][0][0][RTW89_ETSI][13] = 127,
+	[1][0][0][0][RTW89_MKK][13] = 127,
+	[1][0][0][0][RTW89_IC][13] = 127,
+	[1][0][0][0][RTW89_KCC][13] = 127,
+	[1][0][0][0][RTW89_ACMA][13] = 127,
+	[1][0][0][0][RTW89_CN][13] = 127,
+	[1][0][0][0][RTW89_UK][13] = 127,
+	[1][1][0][0][RTW89_FCC][0] = 127,
+	[1][1][0][0][RTW89_ETSI][0] = 127,
+	[1][1][0][0][RTW89_MKK][0] = 127,
+	[1][1][0][0][RTW89_IC][0] = 127,
+	[1][1][0][0][RTW89_KCC][0] = 127,
+	[1][1][0][0][RTW89_ACMA][0] = 127,
+	[1][1][0][0][RTW89_CN][0] = 127,
+	[1][1][0][0][RTW89_UK][0] = 127,
+	[1][1][0][0][RTW89_FCC][1] = 127,
+	[1][1][0][0][RTW89_ETSI][1] = 127,
+	[1][1][0][0][RTW89_MKK][1] = 127,
+	[1][1][0][0][RTW89_IC][1] = 127,
+	[1][1][0][0][RTW89_KCC][1] = 127,
+	[1][1][0][0][RTW89_ACMA][1] = 127,
+	[1][1][0][0][RTW89_CN][1] = 127,
+	[1][1][0][0][RTW89_UK][1] = 127,
+	[1][1][0][0][RTW89_FCC][2] = 127,
+	[1][1][0][0][RTW89_ETSI][2] = 127,
+	[1][1][0][0][RTW89_MKK][2] = 127,
+	[1][1][0][0][RTW89_IC][2] = 127,
+	[1][1][0][0][RTW89_KCC][2] = 127,
+	[1][1][0][0][RTW89_ACMA][2] = 127,
+	[1][1][0][0][RTW89_CN][2] = 127,
+	[1][1][0][0][RTW89_UK][2] = 127,
+	[1][1][0][0][RTW89_FCC][3] = 127,
+	[1][1][0][0][RTW89_ETSI][3] = 127,
+	[1][1][0][0][RTW89_MKK][3] = 127,
+	[1][1][0][0][RTW89_IC][3] = 127,
+	[1][1][0][0][RTW89_KCC][3] = 127,
+	[1][1][0][0][RTW89_ACMA][3] = 127,
+	[1][1][0][0][RTW89_CN][3] = 127,
+	[1][1][0][0][RTW89_UK][3] = 127,
+	[1][1][0][0][RTW89_FCC][4] = 127,
+	[1][1][0][0][RTW89_ETSI][4] = 127,
+	[1][1][0][0][RTW89_MKK][4] = 127,
+	[1][1][0][0][RTW89_IC][4] = 127,
+	[1][1][0][0][RTW89_KCC][4] = 127,
+	[1][1][0][0][RTW89_ACMA][4] = 127,
+	[1][1][0][0][RTW89_CN][4] = 127,
+	[1][1][0][0][RTW89_UK][4] = 127,
+	[1][1][0][0][RTW89_FCC][5] = 127,
+	[1][1][0][0][RTW89_ETSI][5] = 127,
+	[1][1][0][0][RTW89_MKK][5] = 127,
+	[1][1][0][0][RTW89_IC][5] = 127,
+	[1][1][0][0][RTW89_KCC][5] = 127,
+	[1][1][0][0][RTW89_ACMA][5] = 127,
+	[1][1][0][0][RTW89_CN][5] = 127,
+	[1][1][0][0][RTW89_UK][5] = 127,
+	[1][1][0][0][RTW89_FCC][6] = 127,
+	[1][1][0][0][RTW89_ETSI][6] = 127,
+	[1][1][0][0][RTW89_MKK][6] = 127,
+	[1][1][0][0][RTW89_IC][6] = 127,
+	[1][1][0][0][RTW89_KCC][6] = 127,
+	[1][1][0][0][RTW89_ACMA][6] = 127,
+	[1][1][0][0][RTW89_CN][6] = 127,
+	[1][1][0][0][RTW89_UK][6] = 127,
+	[1][1][0][0][RTW89_FCC][7] = 127,
+	[1][1][0][0][RTW89_ETSI][7] = 127,
+	[1][1][0][0][RTW89_MKK][7] = 127,
+	[1][1][0][0][RTW89_IC][7] = 127,
+	[1][1][0][0][RTW89_KCC][7] = 127,
+	[1][1][0][0][RTW89_ACMA][7] = 127,
+	[1][1][0][0][RTW89_CN][7] = 127,
+	[1][1][0][0][RTW89_UK][7] = 127,
+	[1][1][0][0][RTW89_FCC][8] = 127,
+	[1][1][0][0][RTW89_ETSI][8] = 127,
+	[1][1][0][0][RTW89_MKK][8] = 127,
+	[1][1][0][0][RTW89_IC][8] = 127,
+	[1][1][0][0][RTW89_KCC][8] = 127,
+	[1][1][0][0][RTW89_ACMA][8] = 127,
+	[1][1][0][0][RTW89_CN][8] = 127,
+	[1][1][0][0][RTW89_UK][8] = 127,
+	[1][1][0][0][RTW89_FCC][9] = 127,
+	[1][1][0][0][RTW89_ETSI][9] = 127,
+	[1][1][0][0][RTW89_MKK][9] = 127,
+	[1][1][0][0][RTW89_IC][9] = 127,
+	[1][1][0][0][RTW89_KCC][9] = 127,
+	[1][1][0][0][RTW89_ACMA][9] = 127,
+	[1][1][0][0][RTW89_CN][9] = 127,
+	[1][1][0][0][RTW89_UK][9] = 127,
+	[1][1][0][0][RTW89_FCC][10] = 127,
+	[1][1][0][0][RTW89_ETSI][10] = 127,
+	[1][1][0][0][RTW89_MKK][10] = 127,
+	[1][1][0][0][RTW89_IC][10] = 127,
+	[1][1][0][0][RTW89_KCC][10] = 127,
+	[1][1][0][0][RTW89_ACMA][10] = 127,
+	[1][1][0][0][RTW89_CN][10] = 127,
+	[1][1][0][0][RTW89_UK][10] = 127,
+	[1][1][0][0][RTW89_FCC][11] = 127,
+	[1][1][0][0][RTW89_ETSI][11] = 127,
+	[1][1][0][0][RTW89_MKK][11] = 127,
+	[1][1][0][0][RTW89_IC][11] = 127,
+	[1][1][0][0][RTW89_KCC][11] = 127,
+	[1][1][0][0][RTW89_ACMA][11] = 127,
+	[1][1][0][0][RTW89_CN][11] = 127,
+	[1][1][0][0][RTW89_UK][11] = 127,
+	[1][1][0][0][RTW89_FCC][12] = 127,
+	[1][1][0][0][RTW89_ETSI][12] = 127,
+	[1][1][0][0][RTW89_MKK][12] = 127,
+	[1][1][0][0][RTW89_IC][12] = 127,
+	[1][1][0][0][RTW89_KCC][12] = 127,
+	[1][1][0][0][RTW89_ACMA][12] = 127,
+	[1][1][0][0][RTW89_CN][12] = 127,
+	[1][1][0][0][RTW89_UK][12] = 127,
+	[1][1][0][0][RTW89_FCC][13] = 127,
+	[1][1][0][0][RTW89_ETSI][13] = 127,
+	[1][1][0][0][RTW89_MKK][13] = 127,
+	[1][1][0][0][RTW89_IC][13] = 127,
+	[1][1][0][0][RTW89_KCC][13] = 127,
+	[1][1][0][0][RTW89_ACMA][13] = 127,
+	[1][1][0][0][RTW89_CN][13] = 127,
+	[1][1][0][0][RTW89_UK][13] = 127,
+	[0][0][1][0][RTW89_FCC][0] = 80,
+	[0][0][1][0][RTW89_ETSI][0] = 58,
+	[0][0][1][0][RTW89_MKK][0] = 72,
+	[0][0][1][0][RTW89_IC][0] = 80,
+	[0][0][1][0][RTW89_KCC][0] = 78,
+	[0][0][1][0][RTW89_ACMA][0] = 58,
+	[0][0][1][0][RTW89_CN][0] = 60,
+	[0][0][1][0][RTW89_UK][0] = 58,
+	[0][0][1][0][RTW89_FCC][1] = 80,
+	[0][0][1][0][RTW89_ETSI][1] = 60,
+	[0][0][1][0][RTW89_MKK][1] = 74,
+	[0][0][1][0][RTW89_IC][1] = 80,
+	[0][0][1][0][RTW89_KCC][1] = 78,
+	[0][0][1][0][RTW89_ACMA][1] = 60,
+	[0][0][1][0][RTW89_CN][1] = 60,
+	[0][0][1][0][RTW89_UK][1] = 60,
+	[0][0][1][0][RTW89_FCC][2] = 84,
+	[0][0][1][0][RTW89_ETSI][2] = 60,
+	[0][0][1][0][RTW89_MKK][2] = 74,
+	[0][0][1][0][RTW89_IC][2] = 84,
+	[0][0][1][0][RTW89_KCC][2] = 78,
+	[0][0][1][0][RTW89_ACMA][2] = 60,
+	[0][0][1][0][RTW89_CN][2] = 60,
+	[0][0][1][0][RTW89_UK][2] = 60,
+	[0][0][1][0][RTW89_FCC][3] = 84,
+	[0][0][1][0][RTW89_ETSI][3] = 60,
+	[0][0][1][0][RTW89_MKK][3] = 74,
+	[0][0][1][0][RTW89_IC][3] = 84,
+	[0][0][1][0][RTW89_KCC][3] = 78,
+	[0][0][1][0][RTW89_ACMA][3] = 60,
+	[0][0][1][0][RTW89_CN][3] = 60,
+	[0][0][1][0][RTW89_UK][3] = 60,
+	[0][0][1][0][RTW89_FCC][4] = 84,
+	[0][0][1][0][RTW89_ETSI][4] = 60,
+	[0][0][1][0][RTW89_MKK][4] = 74,
+	[0][0][1][0][RTW89_IC][4] = 84,
+	[0][0][1][0][RTW89_KCC][4] = 76,
+	[0][0][1][0][RTW89_ACMA][4] = 60,
+	[0][0][1][0][RTW89_CN][4] = 60,
+	[0][0][1][0][RTW89_UK][4] = 60,
+	[0][0][1][0][RTW89_FCC][5] = 84,
+	[0][0][1][0][RTW89_ETSI][5] = 60,
+	[0][0][1][0][RTW89_MKK][5] = 74,
+	[0][0][1][0][RTW89_IC][5] = 84,
+	[0][0][1][0][RTW89_KCC][5] = 76,
+	[0][0][1][0][RTW89_ACMA][5] = 60,
+	[0][0][1][0][RTW89_CN][5] = 60,
+	[0][0][1][0][RTW89_UK][5] = 60,
+	[0][0][1][0][RTW89_FCC][6] = 84,
+	[0][0][1][0][RTW89_ETSI][6] = 60,
+	[0][0][1][0][RTW89_MKK][6] = 74,
+	[0][0][1][0][RTW89_IC][6] = 84,
+	[0][0][1][0][RTW89_KCC][6] = 76,
+	[0][0][1][0][RTW89_ACMA][6] = 60,
+	[0][0][1][0][RTW89_CN][6] = 60,
+	[0][0][1][0][RTW89_UK][6] = 60,
+	[0][0][1][0][RTW89_FCC][7] = 84,
+	[0][0][1][0][RTW89_ETSI][7] = 60,
+	[0][0][1][0][RTW89_MKK][7] = 74,
+	[0][0][1][0][RTW89_IC][7] = 84,
+	[0][0][1][0][RTW89_KCC][7] = 76,
+	[0][0][1][0][RTW89_ACMA][7] = 60,
+	[0][0][1][0][RTW89_CN][7] = 60,
+	[0][0][1][0][RTW89_UK][7] = 60,
+	[0][0][1][0][RTW89_FCC][8] = 80,
+	[0][0][1][0][RTW89_ETSI][8] = 60,
+	[0][0][1][0][RTW89_MKK][8] = 74,
+	[0][0][1][0][RTW89_IC][8] = 80,
+	[0][0][1][0][RTW89_KCC][8] = 76,
+	[0][0][1][0][RTW89_ACMA][8] = 60,
+	[0][0][1][0][RTW89_CN][8] = 60,
+	[0][0][1][0][RTW89_UK][8] = 60,
+	[0][0][1][0][RTW89_FCC][9] = 76,
+	[0][0][1][0][RTW89_ETSI][9] = 60,
+	[0][0][1][0][RTW89_MKK][9] = 74,
+	[0][0][1][0][RTW89_IC][9] = 76,
+	[0][0][1][0][RTW89_KCC][9] = 74,
+	[0][0][1][0][RTW89_ACMA][9] = 60,
+	[0][0][1][0][RTW89_CN][9] = 60,
+	[0][0][1][0][RTW89_UK][9] = 60,
+	[0][0][1][0][RTW89_FCC][10] = 76,
+	[0][0][1][0][RTW89_ETSI][10] = 60,
+	[0][0][1][0][RTW89_MKK][10] = 74,
+	[0][0][1][0][RTW89_IC][10] = 76,
+	[0][0][1][0][RTW89_KCC][10] = 74,
+	[0][0][1][0][RTW89_ACMA][10] = 60,
+	[0][0][1][0][RTW89_CN][10] = 60,
+	[0][0][1][0][RTW89_UK][10] = 60,
+	[0][0][1][0][RTW89_FCC][11] = 68,
+	[0][0][1][0][RTW89_ETSI][11] = 60,
+	[0][0][1][0][RTW89_MKK][11] = 74,
+	[0][0][1][0][RTW89_IC][11] = 68,
+	[0][0][1][0][RTW89_KCC][11] = 74,
+	[0][0][1][0][RTW89_ACMA][11] = 60,
+	[0][0][1][0][RTW89_CN][11] = 60,
+	[0][0][1][0][RTW89_UK][11] = 60,
+	[0][0][1][0][RTW89_FCC][12] = 64,
+	[0][0][1][0][RTW89_ETSI][12] = 58,
+	[0][0][1][0][RTW89_MKK][12] = 70,
+	[0][0][1][0][RTW89_IC][12] = 64,
+	[0][0][1][0][RTW89_KCC][12] = 74,
+	[0][0][1][0][RTW89_ACMA][12] = 58,
+	[0][0][1][0][RTW89_CN][12] = 60,
+	[0][0][1][0][RTW89_UK][12] = 58,
+	[0][0][1][0][RTW89_FCC][13] = 127,
+	[0][0][1][0][RTW89_ETSI][13] = 127,
+	[0][0][1][0][RTW89_MKK][13] = 127,
+	[0][0][1][0][RTW89_IC][13] = 127,
+	[0][0][1][0][RTW89_KCC][13] = 127,
+	[0][0][1][0][RTW89_ACMA][13] = 127,
+	[0][0][1][0][RTW89_CN][13] = 127,
+	[0][0][1][0][RTW89_UK][13] = 127,
+	[0][1][1][0][RTW89_FCC][0] = 127,
+	[0][1][1][0][RTW89_ETSI][0] = 127,
+	[0][1][1][0][RTW89_MKK][0] = 127,
+	[0][1][1][0][RTW89_IC][0] = 127,
+	[0][1][1][0][RTW89_KCC][0] = 127,
+	[0][1][1][0][RTW89_ACMA][0] = 127,
+	[0][1][1][0][RTW89_CN][0] = 127,
+	[0][1][1][0][RTW89_UK][0] = 127,
+	[0][1][1][0][RTW89_FCC][1] = 127,
+	[0][1][1][0][RTW89_ETSI][1] = 127,
+	[0][1][1][0][RTW89_MKK][1] = 127,
+	[0][1][1][0][RTW89_IC][1] = 127,
+	[0][1][1][0][RTW89_KCC][1] = 127,
+	[0][1][1][0][RTW89_ACMA][1] = 127,
+	[0][1][1][0][RTW89_CN][1] = 127,
+	[0][1][1][0][RTW89_UK][1] = 127,
+	[0][1][1][0][RTW89_FCC][2] = 127,
+	[0][1][1][0][RTW89_ETSI][2] = 127,
+	[0][1][1][0][RTW89_MKK][2] = 127,
+	[0][1][1][0][RTW89_IC][2] = 127,
+	[0][1][1][0][RTW89_KCC][2] = 127,
+	[0][1][1][0][RTW89_ACMA][2] = 127,
+	[0][1][1][0][RTW89_CN][2] = 127,
+	[0][1][1][0][RTW89_UK][2] = 127,
+	[0][1][1][0][RTW89_FCC][3] = 127,
+	[0][1][1][0][RTW89_ETSI][3] = 127,
+	[0][1][1][0][RTW89_MKK][3] = 127,
+	[0][1][1][0][RTW89_IC][3] = 127,
+	[0][1][1][0][RTW89_KCC][3] = 127,
+	[0][1][1][0][RTW89_ACMA][3] = 127,
+	[0][1][1][0][RTW89_CN][3] = 127,
+	[0][1][1][0][RTW89_UK][3] = 127,
+	[0][1][1][0][RTW89_FCC][4] = 127,
+	[0][1][1][0][RTW89_ETSI][4] = 127,
+	[0][1][1][0][RTW89_MKK][4] = 127,
+	[0][1][1][0][RTW89_IC][4] = 127,
+	[0][1][1][0][RTW89_KCC][4] = 127,
+	[0][1][1][0][RTW89_ACMA][4] = 127,
+	[0][1][1][0][RTW89_CN][4] = 127,
+	[0][1][1][0][RTW89_UK][4] = 127,
+	[0][1][1][0][RTW89_FCC][5] = 127,
+	[0][1][1][0][RTW89_ETSI][5] = 127,
+	[0][1][1][0][RTW89_MKK][5] = 127,
+	[0][1][1][0][RTW89_IC][5] = 127,
+	[0][1][1][0][RTW89_KCC][5] = 127,
+	[0][1][1][0][RTW89_ACMA][5] = 127,
+	[0][1][1][0][RTW89_CN][5] = 127,
+	[0][1][1][0][RTW89_UK][5] = 127,
+	[0][1][1][0][RTW89_FCC][6] = 127,
+	[0][1][1][0][RTW89_ETSI][6] = 127,
+	[0][1][1][0][RTW89_MKK][6] = 127,
+	[0][1][1][0][RTW89_IC][6] = 127,
+	[0][1][1][0][RTW89_KCC][6] = 127,
+	[0][1][1][0][RTW89_ACMA][6] = 127,
+	[0][1][1][0][RTW89_CN][6] = 127,
+	[0][1][1][0][RTW89_UK][6] = 127,
+	[0][1][1][0][RTW89_FCC][7] = 127,
+	[0][1][1][0][RTW89_ETSI][7] = 127,
+	[0][1][1][0][RTW89_MKK][7] = 127,
+	[0][1][1][0][RTW89_IC][7] = 127,
+	[0][1][1][0][RTW89_KCC][7] = 127,
+	[0][1][1][0][RTW89_ACMA][7] = 127,
+	[0][1][1][0][RTW89_CN][7] = 127,
+	[0][1][1][0][RTW89_UK][7] = 127,
+	[0][1][1][0][RTW89_FCC][8] = 127,
+	[0][1][1][0][RTW89_ETSI][8] = 127,
+	[0][1][1][0][RTW89_MKK][8] = 127,
+	[0][1][1][0][RTW89_IC][8] = 127,
+	[0][1][1][0][RTW89_KCC][8] = 127,
+	[0][1][1][0][RTW89_ACMA][8] = 127,
+	[0][1][1][0][RTW89_CN][8] = 127,
+	[0][1][1][0][RTW89_UK][8] = 127,
+	[0][1][1][0][RTW89_FCC][9] = 127,
+	[0][1][1][0][RTW89_ETSI][9] = 127,
+	[0][1][1][0][RTW89_MKK][9] = 127,
+	[0][1][1][0][RTW89_IC][9] = 127,
+	[0][1][1][0][RTW89_KCC][9] = 127,
+	[0][1][1][0][RTW89_ACMA][9] = 127,
+	[0][1][1][0][RTW89_CN][9] = 127,
+	[0][1][1][0][RTW89_UK][9] = 127,
+	[0][1][1][0][RTW89_FCC][10] = 127,
+	[0][1][1][0][RTW89_ETSI][10] = 127,
+	[0][1][1][0][RTW89_MKK][10] = 127,
+	[0][1][1][0][RTW89_IC][10] = 127,
+	[0][1][1][0][RTW89_KCC][10] = 127,
+	[0][1][1][0][RTW89_ACMA][10] = 127,
+	[0][1][1][0][RTW89_CN][10] = 127,
+	[0][1][1][0][RTW89_UK][10] = 127,
+	[0][1][1][0][RTW89_FCC][11] = 127,
+	[0][1][1][0][RTW89_ETSI][11] = 127,
+	[0][1][1][0][RTW89_MKK][11] = 127,
+	[0][1][1][0][RTW89_IC][11] = 127,
+	[0][1][1][0][RTW89_KCC][11] = 127,
+	[0][1][1][0][RTW89_ACMA][11] = 127,
+	[0][1][1][0][RTW89_CN][11] = 127,
+	[0][1][1][0][RTW89_UK][11] = 127,
+	[0][1][1][0][RTW89_FCC][12] = 127,
+	[0][1][1][0][RTW89_ETSI][12] = 127,
+	[0][1][1][0][RTW89_MKK][12] = 127,
+	[0][1][1][0][RTW89_IC][12] = 127,
+	[0][1][1][0][RTW89_KCC][12] = 127,
+	[0][1][1][0][RTW89_ACMA][12] = 127,
+	[0][1][1][0][RTW89_CN][12] = 127,
+	[0][1][1][0][RTW89_UK][12] = 127,
+	[0][1][1][0][RTW89_FCC][13] = 127,
+	[0][1][1][0][RTW89_ETSI][13] = 127,
+	[0][1][1][0][RTW89_MKK][13] = 127,
+	[0][1][1][0][RTW89_IC][13] = 127,
+	[0][1][1][0][RTW89_KCC][13] = 127,
+	[0][1][1][0][RTW89_ACMA][13] = 127,
+	[0][1][1][0][RTW89_CN][13] = 127,
+	[0][1][1][0][RTW89_UK][13] = 127,
+	[0][0][2][0][RTW89_FCC][0] = 78,
+	[0][0][2][0][RTW89_ETSI][0] = 60,
+	[0][0][2][0][RTW89_MKK][0] = 72,
+	[0][0][2][0][RTW89_IC][0] = 78,
+	[0][0][2][0][RTW89_KCC][0] = 78,
+	[0][0][2][0][RTW89_ACMA][0] = 60,
+	[0][0][2][0][RTW89_CN][0] = 60,
+	[0][0][2][0][RTW89_UK][0] = 60,
+	[0][0][2][0][RTW89_FCC][1] = 78,
+	[0][0][2][0][RTW89_ETSI][1] = 60,
+	[0][0][2][0][RTW89_MKK][1] = 78,
+	[0][0][2][0][RTW89_IC][1] = 78,
+	[0][0][2][0][RTW89_KCC][1] = 78,
+	[0][0][2][0][RTW89_ACMA][1] = 60,
+	[0][0][2][0][RTW89_CN][1] = 60,
+	[0][0][2][0][RTW89_UK][1] = 60,
+	[0][0][2][0][RTW89_FCC][2] = 82,
+	[0][0][2][0][RTW89_ETSI][2] = 60,
+	[0][0][2][0][RTW89_MKK][2] = 78,
+	[0][0][2][0][RTW89_IC][2] = 82,
+	[0][0][2][0][RTW89_KCC][2] = 78,
+	[0][0][2][0][RTW89_ACMA][2] = 60,
+	[0][0][2][0][RTW89_CN][2] = 60,
+	[0][0][2][0][RTW89_UK][2] = 60,
+	[0][0][2][0][RTW89_FCC][3] = 82,
+	[0][0][2][0][RTW89_ETSI][3] = 60,
+	[0][0][2][0][RTW89_MKK][3] = 78,
+	[0][0][2][0][RTW89_IC][3] = 82,
+	[0][0][2][0][RTW89_KCC][3] = 78,
+	[0][0][2][0][RTW89_ACMA][3] = 60,
+	[0][0][2][0][RTW89_CN][3] = 60,
+	[0][0][2][0][RTW89_UK][3] = 60,
+	[0][0][2][0][RTW89_FCC][4] = 82,
+	[0][0][2][0][RTW89_ETSI][4] = 60,
+	[0][0][2][0][RTW89_MKK][4] = 78,
+	[0][0][2][0][RTW89_IC][4] = 82,
+	[0][0][2][0][RTW89_KCC][4] = 78,
+	[0][0][2][0][RTW89_ACMA][4] = 60,
+	[0][0][2][0][RTW89_CN][4] = 60,
+	[0][0][2][0][RTW89_UK][4] = 60,
+	[0][0][2][0][RTW89_FCC][5] = 82,
+	[0][0][2][0][RTW89_ETSI][5] = 60,
+	[0][0][2][0][RTW89_MKK][5] = 78,
+	[0][0][2][0][RTW89_IC][5] = 82,
+	[0][0][2][0][RTW89_KCC][5] = 78,
+	[0][0][2][0][RTW89_ACMA][5] = 60,
+	[0][0][2][0][RTW89_CN][5] = 60,
+	[0][0][2][0][RTW89_UK][5] = 60,
+	[0][0][2][0][RTW89_FCC][6] = 82,
+	[0][0][2][0][RTW89_ETSI][6] = 60,
+	[0][0][2][0][RTW89_MKK][6] = 78,
+	[0][0][2][0][RTW89_IC][6] = 82,
+	[0][0][2][0][RTW89_KCC][6] = 78,
+	[0][0][2][0][RTW89_ACMA][6] = 60,
+	[0][0][2][0][RTW89_CN][6] = 60,
+	[0][0][2][0][RTW89_UK][6] = 60,
+	[0][0][2][0][RTW89_FCC][7] = 82,
+	[0][0][2][0][RTW89_ETSI][7] = 60,
+	[0][0][2][0][RTW89_MKK][7] = 78,
+	[0][0][2][0][RTW89_IC][7] = 82,
+	[0][0][2][0][RTW89_KCC][7] = 78,
+	[0][0][2][0][RTW89_ACMA][7] = 60,
+	[0][0][2][0][RTW89_CN][7] = 60,
+	[0][0][2][0][RTW89_UK][7] = 60,
+	[0][0][2][0][RTW89_FCC][8] = 80,
+	[0][0][2][0][RTW89_ETSI][8] = 60,
+	[0][0][2][0][RTW89_MKK][8] = 78,
+	[0][0][2][0][RTW89_IC][8] = 80,
+	[0][0][2][0][RTW89_KCC][8] = 78,
+	[0][0][2][0][RTW89_ACMA][8] = 60,
+	[0][0][2][0][RTW89_CN][8] = 60,
+	[0][0][2][0][RTW89_UK][8] = 60,
+	[0][0][2][0][RTW89_FCC][9] = 76,
+	[0][0][2][0][RTW89_ETSI][9] = 60,
+	[0][0][2][0][RTW89_MKK][9] = 78,
+	[0][0][2][0][RTW89_IC][9] = 76,
+	[0][0][2][0][RTW89_KCC][9] = 78,
+	[0][0][2][0][RTW89_ACMA][9] = 60,
+	[0][0][2][0][RTW89_CN][9] = 60,
+	[0][0][2][0][RTW89_UK][9] = 60,
+	[0][0][2][0][RTW89_FCC][10] = 76,
+	[0][0][2][0][RTW89_ETSI][10] = 60,
+	[0][0][2][0][RTW89_MKK][10] = 78,
+	[0][0][2][0][RTW89_IC][10] = 76,
+	[0][0][2][0][RTW89_KCC][10] = 78,
+	[0][0][2][0][RTW89_ACMA][10] = 60,
+	[0][0][2][0][RTW89_CN][10] = 60,
+	[0][0][2][0][RTW89_UK][10] = 60,
+	[0][0][2][0][RTW89_FCC][11] = 70,
+	[0][0][2][0][RTW89_ETSI][11] = 60,
+	[0][0][2][0][RTW89_MKK][11] = 78,
+	[0][0][2][0][RTW89_IC][11] = 70,
+	[0][0][2][0][RTW89_KCC][11] = 78,
+	[0][0][2][0][RTW89_ACMA][11] = 60,
+	[0][0][2][0][RTW89_CN][11] = 60,
+	[0][0][2][0][RTW89_UK][11] = 60,
+	[0][0][2][0][RTW89_FCC][12] = 70,
+	[0][0][2][0][RTW89_ETSI][12] = 60,
+	[0][0][2][0][RTW89_MKK][12] = 70,
+	[0][0][2][0][RTW89_IC][12] = 70,
+	[0][0][2][0][RTW89_KCC][12] = 78,
+	[0][0][2][0][RTW89_ACMA][12] = 60,
+	[0][0][2][0][RTW89_CN][12] = 60,
+	[0][0][2][0][RTW89_UK][12] = 60,
+	[0][0][2][0][RTW89_FCC][13] = 127,
+	[0][0][2][0][RTW89_ETSI][13] = 127,
+	[0][0][2][0][RTW89_MKK][13] = 127,
+	[0][0][2][0][RTW89_IC][13] = 127,
+	[0][0][2][0][RTW89_KCC][13] = 127,
+	[0][0][2][0][RTW89_ACMA][13] = 127,
+	[0][0][2][0][RTW89_CN][13] = 127,
+	[0][0][2][0][RTW89_UK][13] = 127,
+	[0][1][2][0][RTW89_FCC][0] = 127,
+	[0][1][2][0][RTW89_ETSI][0] = 127,
+	[0][1][2][0][RTW89_MKK][0] = 127,
+	[0][1][2][0][RTW89_IC][0] = 127,
+	[0][1][2][0][RTW89_KCC][0] = 127,
+	[0][1][2][0][RTW89_ACMA][0] = 127,
+	[0][1][2][0][RTW89_CN][0] = 127,
+	[0][1][2][0][RTW89_UK][0] = 127,
+	[0][1][2][0][RTW89_FCC][1] = 127,
+	[0][1][2][0][RTW89_ETSI][1] = 127,
+	[0][1][2][0][RTW89_MKK][1] = 127,
+	[0][1][2][0][RTW89_IC][1] = 127,
+	[0][1][2][0][RTW89_KCC][1] = 127,
+	[0][1][2][0][RTW89_ACMA][1] = 127,
+	[0][1][2][0][RTW89_CN][1] = 127,
+	[0][1][2][0][RTW89_UK][1] = 127,
+	[0][1][2][0][RTW89_FCC][2] = 127,
+	[0][1][2][0][RTW89_ETSI][2] = 127,
+	[0][1][2][0][RTW89_MKK][2] = 127,
+	[0][1][2][0][RTW89_IC][2] = 127,
+	[0][1][2][0][RTW89_KCC][2] = 127,
+	[0][1][2][0][RTW89_ACMA][2] = 127,
+	[0][1][2][0][RTW89_CN][2] = 127,
+	[0][1][2][0][RTW89_UK][2] = 127,
+	[0][1][2][0][RTW89_FCC][3] = 127,
+	[0][1][2][0][RTW89_ETSI][3] = 127,
+	[0][1][2][0][RTW89_MKK][3] = 127,
+	[0][1][2][0][RTW89_IC][3] = 127,
+	[0][1][2][0][RTW89_KCC][3] = 127,
+	[0][1][2][0][RTW89_ACMA][3] = 127,
+	[0][1][2][0][RTW89_CN][3] = 127,
+	[0][1][2][0][RTW89_UK][3] = 127,
+	[0][1][2][0][RTW89_FCC][4] = 127,
+	[0][1][2][0][RTW89_ETSI][4] = 127,
+	[0][1][2][0][RTW89_MKK][4] = 127,
+	[0][1][2][0][RTW89_IC][4] = 127,
+	[0][1][2][0][RTW89_KCC][4] = 127,
+	[0][1][2][0][RTW89_ACMA][4] = 127,
+	[0][1][2][0][RTW89_CN][4] = 127,
+	[0][1][2][0][RTW89_UK][4] = 127,
+	[0][1][2][0][RTW89_FCC][5] = 127,
+	[0][1][2][0][RTW89_ETSI][5] = 127,
+	[0][1][2][0][RTW89_MKK][5] = 127,
+	[0][1][2][0][RTW89_IC][5] = 127,
+	[0][1][2][0][RTW89_KCC][5] = 127,
+	[0][1][2][0][RTW89_ACMA][5] = 127,
+	[0][1][2][0][RTW89_CN][5] = 127,
+	[0][1][2][0][RTW89_UK][5] = 127,
+	[0][1][2][0][RTW89_FCC][6] = 127,
+	[0][1][2][0][RTW89_ETSI][6] = 127,
+	[0][1][2][0][RTW89_MKK][6] = 127,
+	[0][1][2][0][RTW89_IC][6] = 127,
+	[0][1][2][0][RTW89_KCC][6] = 127,
+	[0][1][2][0][RTW89_ACMA][6] = 127,
+	[0][1][2][0][RTW89_CN][6] = 127,
+	[0][1][2][0][RTW89_UK][6] = 127,
+	[0][1][2][0][RTW89_FCC][7] = 127,
+	[0][1][2][0][RTW89_ETSI][7] = 127,
+	[0][1][2][0][RTW89_MKK][7] = 127,
+	[0][1][2][0][RTW89_IC][7] = 127,
+	[0][1][2][0][RTW89_KCC][7] = 127,
+	[0][1][2][0][RTW89_ACMA][7] = 127,
+	[0][1][2][0][RTW89_CN][7] = 127,
+	[0][1][2][0][RTW89_UK][7] = 127,
+	[0][1][2][0][RTW89_FCC][8] = 127,
+	[0][1][2][0][RTW89_ETSI][8] = 127,
+	[0][1][2][0][RTW89_MKK][8] = 127,
+	[0][1][2][0][RTW89_IC][8] = 127,
+	[0][1][2][0][RTW89_KCC][8] = 127,
+	[0][1][2][0][RTW89_ACMA][8] = 127,
+	[0][1][2][0][RTW89_CN][8] = 127,
+	[0][1][2][0][RTW89_UK][8] = 127,
+	[0][1][2][0][RTW89_FCC][9] = 127,
+	[0][1][2][0][RTW89_ETSI][9] = 127,
+	[0][1][2][0][RTW89_MKK][9] = 127,
+	[0][1][2][0][RTW89_IC][9] = 127,
+	[0][1][2][0][RTW89_KCC][9] = 127,
+	[0][1][2][0][RTW89_ACMA][9] = 127,
+	[0][1][2][0][RTW89_CN][9] = 127,
+	[0][1][2][0][RTW89_UK][9] = 127,
+	[0][1][2][0][RTW89_FCC][10] = 127,
+	[0][1][2][0][RTW89_ETSI][10] = 127,
+	[0][1][2][0][RTW89_MKK][10] = 127,
+	[0][1][2][0][RTW89_IC][10] = 127,
+	[0][1][2][0][RTW89_KCC][10] = 127,
+	[0][1][2][0][RTW89_ACMA][10] = 127,
+	[0][1][2][0][RTW89_CN][10] = 127,
+	[0][1][2][0][RTW89_UK][10] = 127,
+	[0][1][2][0][RTW89_FCC][11] = 127,
+	[0][1][2][0][RTW89_ETSI][11] = 127,
+	[0][1][2][0][RTW89_MKK][11] = 127,
+	[0][1][2][0][RTW89_IC][11] = 127,
+	[0][1][2][0][RTW89_KCC][11] = 127,
+	[0][1][2][0][RTW89_ACMA][11] = 127,
+	[0][1][2][0][RTW89_CN][11] = 127,
+	[0][1][2][0][RTW89_UK][11] = 127,
+	[0][1][2][0][RTW89_FCC][12] = 127,
+	[0][1][2][0][RTW89_ETSI][12] = 127,
+	[0][1][2][0][RTW89_MKK][12] = 127,
+	[0][1][2][0][RTW89_IC][12] = 127,
+	[0][1][2][0][RTW89_KCC][12] = 127,
+	[0][1][2][0][RTW89_ACMA][12] = 127,
+	[0][1][2][0][RTW89_CN][12] = 127,
+	[0][1][2][0][RTW89_UK][12] = 127,
+	[0][1][2][0][RTW89_FCC][13] = 127,
+	[0][1][2][0][RTW89_ETSI][13] = 127,
+	[0][1][2][0][RTW89_MKK][13] = 127,
+	[0][1][2][0][RTW89_IC][13] = 127,
+	[0][1][2][0][RTW89_KCC][13] = 127,
+	[0][1][2][0][RTW89_ACMA][13] = 127,
+	[0][1][2][0][RTW89_CN][13] = 127,
+	[0][1][2][0][RTW89_UK][13] = 127,
+	[0][1][2][1][RTW89_FCC][0] = 127,
+	[0][1][2][1][RTW89_ETSI][0] = 127,
+	[0][1][2][1][RTW89_MKK][0] = 127,
+	[0][1][2][1][RTW89_IC][0] = 127,
+	[0][1][2][1][RTW89_KCC][0] = 127,
+	[0][1][2][1][RTW89_ACMA][0] = 127,
+	[0][1][2][1][RTW89_CN][0] = 127,
+	[0][1][2][1][RTW89_UK][0] = 127,
+	[0][1][2][1][RTW89_FCC][1] = 127,
+	[0][1][2][1][RTW89_ETSI][1] = 127,
+	[0][1][2][1][RTW89_MKK][1] = 127,
+	[0][1][2][1][RTW89_IC][1] = 127,
+	[0][1][2][1][RTW89_KCC][1] = 127,
+	[0][1][2][1][RTW89_ACMA][1] = 127,
+	[0][1][2][1][RTW89_CN][1] = 127,
+	[0][1][2][1][RTW89_UK][1] = 127,
+	[0][1][2][1][RTW89_FCC][2] = 127,
+	[0][1][2][1][RTW89_ETSI][2] = 127,
+	[0][1][2][1][RTW89_MKK][2] = 127,
+	[0][1][2][1][RTW89_IC][2] = 127,
+	[0][1][2][1][RTW89_KCC][2] = 127,
+	[0][1][2][1][RTW89_ACMA][2] = 127,
+	[0][1][2][1][RTW89_CN][2] = 127,
+	[0][1][2][1][RTW89_UK][2] = 127,
+	[0][1][2][1][RTW89_FCC][3] = 127,
+	[0][1][2][1][RTW89_ETSI][3] = 127,
+	[0][1][2][1][RTW89_MKK][3] = 127,
+	[0][1][2][1][RTW89_IC][3] = 127,
+	[0][1][2][1][RTW89_KCC][3] = 127,
+	[0][1][2][1][RTW89_ACMA][3] = 127,
+	[0][1][2][1][RTW89_CN][3] = 127,
+	[0][1][2][1][RTW89_UK][3] = 127,
+	[0][1][2][1][RTW89_FCC][4] = 127,
+	[0][1][2][1][RTW89_ETSI][4] = 127,
+	[0][1][2][1][RTW89_MKK][4] = 127,
+	[0][1][2][1][RTW89_IC][4] = 127,
+	[0][1][2][1][RTW89_KCC][4] = 127,
+	[0][1][2][1][RTW89_ACMA][4] = 127,
+	[0][1][2][1][RTW89_CN][4] = 127,
+	[0][1][2][1][RTW89_UK][4] = 127,
+	[0][1][2][1][RTW89_FCC][5] = 127,
+	[0][1][2][1][RTW89_ETSI][5] = 127,
+	[0][1][2][1][RTW89_MKK][5] = 127,
+	[0][1][2][1][RTW89_IC][5] = 127,
+	[0][1][2][1][RTW89_KCC][5] = 127,
+	[0][1][2][1][RTW89_ACMA][5] = 127,
+	[0][1][2][1][RTW89_CN][5] = 127,
+	[0][1][2][1][RTW89_UK][5] = 127,
+	[0][1][2][1][RTW89_FCC][6] = 127,
+	[0][1][2][1][RTW89_ETSI][6] = 127,
+	[0][1][2][1][RTW89_MKK][6] = 127,
+	[0][1][2][1][RTW89_IC][6] = 127,
+	[0][1][2][1][RTW89_KCC][6] = 127,
+	[0][1][2][1][RTW89_ACMA][6] = 127,
+	[0][1][2][1][RTW89_CN][6] = 127,
+	[0][1][2][1][RTW89_UK][6] = 127,
+	[0][1][2][1][RTW89_FCC][7] = 127,
+	[0][1][2][1][RTW89_ETSI][7] = 127,
+	[0][1][2][1][RTW89_MKK][7] = 127,
+	[0][1][2][1][RTW89_IC][7] = 127,
+	[0][1][2][1][RTW89_KCC][7] = 127,
+	[0][1][2][1][RTW89_ACMA][7] = 127,
+	[0][1][2][1][RTW89_CN][7] = 127,
+	[0][1][2][1][RTW89_UK][7] = 127,
+	[0][1][2][1][RTW89_FCC][8] = 127,
+	[0][1][2][1][RTW89_ETSI][8] = 127,
+	[0][1][2][1][RTW89_MKK][8] = 127,
+	[0][1][2][1][RTW89_IC][8] = 127,
+	[0][1][2][1][RTW89_KCC][8] = 127,
+	[0][1][2][1][RTW89_ACMA][8] = 127,
+	[0][1][2][1][RTW89_CN][8] = 127,
+	[0][1][2][1][RTW89_UK][8] = 127,
+	[0][1][2][1][RTW89_FCC][9] = 127,
+	[0][1][2][1][RTW89_ETSI][9] = 127,
+	[0][1][2][1][RTW89_MKK][9] = 127,
+	[0][1][2][1][RTW89_IC][9] = 127,
+	[0][1][2][1][RTW89_KCC][9] = 127,
+	[0][1][2][1][RTW89_ACMA][9] = 127,
+	[0][1][2][1][RTW89_CN][9] = 127,
+	[0][1][2][1][RTW89_UK][9] = 127,
+	[0][1][2][1][RTW89_FCC][10] = 127,
+	[0][1][2][1][RTW89_ETSI][10] = 127,
+	[0][1][2][1][RTW89_MKK][10] = 127,
+	[0][1][2][1][RTW89_IC][10] = 127,
+	[0][1][2][1][RTW89_KCC][10] = 127,
+	[0][1][2][1][RTW89_ACMA][10] = 127,
+	[0][1][2][1][RTW89_CN][10] = 127,
+	[0][1][2][1][RTW89_UK][10] = 127,
+	[0][1][2][1][RTW89_FCC][11] = 127,
+	[0][1][2][1][RTW89_ETSI][11] = 127,
+	[0][1][2][1][RTW89_MKK][11] = 127,
+	[0][1][2][1][RTW89_IC][11] = 127,
+	[0][1][2][1][RTW89_KCC][11] = 127,
+	[0][1][2][1][RTW89_ACMA][11] = 127,
+	[0][1][2][1][RTW89_CN][11] = 127,
+	[0][1][2][1][RTW89_UK][11] = 127,
+	[0][1][2][1][RTW89_FCC][12] = 127,
+	[0][1][2][1][RTW89_ETSI][12] = 127,
+	[0][1][2][1][RTW89_MKK][12] = 127,
+	[0][1][2][1][RTW89_IC][12] = 127,
+	[0][1][2][1][RTW89_KCC][12] = 127,
+	[0][1][2][1][RTW89_ACMA][12] = 127,
+	[0][1][2][1][RTW89_CN][12] = 127,
+	[0][1][2][1][RTW89_UK][12] = 127,
+	[0][1][2][1][RTW89_FCC][13] = 127,
+	[0][1][2][1][RTW89_ETSI][13] = 127,
+	[0][1][2][1][RTW89_MKK][13] = 127,
+	[0][1][2][1][RTW89_IC][13] = 127,
+	[0][1][2][1][RTW89_KCC][13] = 127,
+	[0][1][2][1][RTW89_ACMA][13] = 127,
+	[0][1][2][1][RTW89_CN][13] = 127,
+	[0][1][2][1][RTW89_UK][13] = 127,
+	[1][0][2][0][RTW89_FCC][0] = 127,
+	[1][0][2][0][RTW89_ETSI][0] = 127,
+	[1][0][2][0][RTW89_MKK][0] = 127,
+	[1][0][2][0][RTW89_IC][0] = 127,
+	[1][0][2][0][RTW89_KCC][0] = 127,
+	[1][0][2][0][RTW89_ACMA][0] = 127,
+	[1][0][2][0][RTW89_CN][0] = 127,
+	[1][0][2][0][RTW89_UK][0] = 127,
+	[1][0][2][0][RTW89_FCC][1] = 127,
+	[1][0][2][0][RTW89_ETSI][1] = 127,
+	[1][0][2][0][RTW89_MKK][1] = 127,
+	[1][0][2][0][RTW89_IC][1] = 127,
+	[1][0][2][0][RTW89_KCC][1] = 127,
+	[1][0][2][0][RTW89_ACMA][1] = 127,
+	[1][0][2][0][RTW89_CN][1] = 127,
+	[1][0][2][0][RTW89_UK][1] = 127,
+	[1][0][2][0][RTW89_FCC][2] = 72,
+	[1][0][2][0][RTW89_ETSI][2] = 58,
+	[1][0][2][0][RTW89_MKK][2] = 80,
+	[1][0][2][0][RTW89_IC][2] = 72,
+	[1][0][2][0][RTW89_KCC][2] = 80,
+	[1][0][2][0][RTW89_ACMA][2] = 58,
+	[1][0][2][0][RTW89_CN][2] = 60,
+	[1][0][2][0][RTW89_UK][2] = 58,
+	[1][0][2][0][RTW89_FCC][3] = 72,
+	[1][0][2][0][RTW89_ETSI][3] = 58,
+	[1][0][2][0][RTW89_MKK][3] = 80,
+	[1][0][2][0][RTW89_IC][3] = 72,
+	[1][0][2][0][RTW89_KCC][3] = 80,
+	[1][0][2][0][RTW89_ACMA][3] = 58,
+	[1][0][2][0][RTW89_CN][3] = 60,
+	[1][0][2][0][RTW89_UK][3] = 58,
+	[1][0][2][0][RTW89_FCC][4] = 76,
+	[1][0][2][0][RTW89_ETSI][4] = 58,
+	[1][0][2][0][RTW89_MKK][4] = 80,
+	[1][0][2][0][RTW89_IC][4] = 76,
+	[1][0][2][0][RTW89_KCC][4] = 80,
+	[1][0][2][0][RTW89_ACMA][4] = 58,
+	[1][0][2][0][RTW89_CN][4] = 60,
+	[1][0][2][0][RTW89_UK][4] = 58,
+	[1][0][2][0][RTW89_FCC][5] = 78,
+	[1][0][2][0][RTW89_ETSI][5] = 58,
+	[1][0][2][0][RTW89_MKK][5] = 80,
+	[1][0][2][0][RTW89_IC][5] = 78,
+	[1][0][2][0][RTW89_KCC][5] = 80,
+	[1][0][2][0][RTW89_ACMA][5] = 58,
+	[1][0][2][0][RTW89_CN][5] = 60,
+	[1][0][2][0][RTW89_UK][5] = 58,
+	[1][0][2][0][RTW89_FCC][6] = 78,
+	[1][0][2][0][RTW89_ETSI][6] = 58,
+	[1][0][2][0][RTW89_MKK][6] = 78,
+	[1][0][2][0][RTW89_IC][6] = 78,
+	[1][0][2][0][RTW89_KCC][6] = 80,
+	[1][0][2][0][RTW89_ACMA][6] = 58,
+	[1][0][2][0][RTW89_CN][6] = 60,
+	[1][0][2][0][RTW89_UK][6] = 58,
+	[1][0][2][0][RTW89_FCC][7] = 78,
+	[1][0][2][0][RTW89_ETSI][7] = 58,
+	[1][0][2][0][RTW89_MKK][7] = 80,
+	[1][0][2][0][RTW89_IC][7] = 78,
+	[1][0][2][0][RTW89_KCC][7] = 80,
+	[1][0][2][0][RTW89_ACMA][7] = 58,
+	[1][0][2][0][RTW89_CN][7] = 60,
+	[1][0][2][0][RTW89_UK][7] = 58,
+	[1][0][2][0][RTW89_FCC][8] = 78,
+	[1][0][2][0][RTW89_ETSI][8] = 58,
+	[1][0][2][0][RTW89_MKK][8] = 80,
+	[1][0][2][0][RTW89_IC][8] = 78,
+	[1][0][2][0][RTW89_KCC][8] = 78,
+	[1][0][2][0][RTW89_ACMA][8] = 58,
+	[1][0][2][0][RTW89_CN][8] = 60,
+	[1][0][2][0][RTW89_UK][8] = 58,
+	[1][0][2][0][RTW89_FCC][9] = 76,
+	[1][0][2][0][RTW89_ETSI][9] = 58,
+	[1][0][2][0][RTW89_MKK][9] = 80,
+	[1][0][2][0][RTW89_IC][9] = 76,
+	[1][0][2][0][RTW89_KCC][9] = 78,
+	[1][0][2][0][RTW89_ACMA][9] = 58,
+	[1][0][2][0][RTW89_CN][9] = 60,
+	[1][0][2][0][RTW89_UK][9] = 58,
+	[1][0][2][0][RTW89_FCC][10] = 70,
+	[1][0][2][0][RTW89_ETSI][10] = 58,
+	[1][0][2][0][RTW89_MKK][10] = 78,
+	[1][0][2][0][RTW89_IC][10] = 70,
+	[1][0][2][0][RTW89_KCC][10] = 78,
+	[1][0][2][0][RTW89_ACMA][10] = 58,
+	[1][0][2][0][RTW89_CN][10] = 60,
+	[1][0][2][0][RTW89_UK][10] = 58,
+	[1][0][2][0][RTW89_FCC][11] = 127,
+	[1][0][2][0][RTW89_ETSI][11] = 127,
+	[1][0][2][0][RTW89_MKK][11] = 127,
+	[1][0][2][0][RTW89_IC][11] = 127,
+	[1][0][2][0][RTW89_KCC][11] = 127,
+	[1][0][2][0][RTW89_ACMA][11] = 127,
+	[1][0][2][0][RTW89_CN][11] = 127,
+	[1][0][2][0][RTW89_UK][11] = 127,
+	[1][0][2][0][RTW89_FCC][12] = 127,
+	[1][0][2][0][RTW89_ETSI][12] = 127,
+	[1][0][2][0][RTW89_MKK][12] = 127,
+	[1][0][2][0][RTW89_IC][12] = 127,
+	[1][0][2][0][RTW89_KCC][12] = 127,
+	[1][0][2][0][RTW89_ACMA][12] = 127,
+	[1][0][2][0][RTW89_CN][12] = 127,
+	[1][0][2][0][RTW89_UK][12] = 127,
+	[1][0][2][0][RTW89_FCC][13] = 127,
+	[1][0][2][0][RTW89_ETSI][13] = 127,
+	[1][0][2][0][RTW89_MKK][13] = 127,
+	[1][0][2][0][RTW89_IC][13] = 127,
+	[1][0][2][0][RTW89_KCC][13] = 127,
+	[1][0][2][0][RTW89_ACMA][13] = 127,
+	[1][0][2][0][RTW89_CN][13] = 127,
+	[1][0][2][0][RTW89_UK][13] = 127,
+	[1][1][2][0][RTW89_FCC][0] = 127,
+	[1][1][2][0][RTW89_ETSI][0] = 127,
+	[1][1][2][0][RTW89_MKK][0] = 127,
+	[1][1][2][0][RTW89_IC][0] = 127,
+	[1][1][2][0][RTW89_KCC][0] = 127,
+	[1][1][2][0][RTW89_ACMA][0] = 127,
+	[1][1][2][0][RTW89_CN][0] = 127,
+	[1][1][2][0][RTW89_UK][0] = 127,
+	[1][1][2][0][RTW89_FCC][1] = 127,
+	[1][1][2][0][RTW89_ETSI][1] = 127,
+	[1][1][2][0][RTW89_MKK][1] = 127,
+	[1][1][2][0][RTW89_IC][1] = 127,
+	[1][1][2][0][RTW89_KCC][1] = 127,
+	[1][1][2][0][RTW89_ACMA][1] = 127,
+	[1][1][2][0][RTW89_CN][1] = 127,
+	[1][1][2][0][RTW89_UK][1] = 127,
+	[1][1][2][0][RTW89_FCC][2] = 127,
+	[1][1][2][0][RTW89_ETSI][2] = 127,
+	[1][1][2][0][RTW89_MKK][2] = 127,
+	[1][1][2][0][RTW89_IC][2] = 127,
+	[1][1][2][0][RTW89_KCC][2] = 127,
+	[1][1][2][0][RTW89_ACMA][2] = 127,
+	[1][1][2][0][RTW89_CN][2] = 127,
+	[1][1][2][0][RTW89_UK][2] = 127,
+	[1][1][2][0][RTW89_FCC][3] = 127,
+	[1][1][2][0][RTW89_ETSI][3] = 127,
+	[1][1][2][0][RTW89_MKK][3] = 127,
+	[1][1][2][0][RTW89_IC][3] = 127,
+	[1][1][2][0][RTW89_KCC][3] = 127,
+	[1][1][2][0][RTW89_ACMA][3] = 127,
+	[1][1][2][0][RTW89_CN][3] = 127,
+	[1][1][2][0][RTW89_UK][3] = 127,
+	[1][1][2][0][RTW89_FCC][4] = 127,
+	[1][1][2][0][RTW89_ETSI][4] = 127,
+	[1][1][2][0][RTW89_MKK][4] = 127,
+	[1][1][2][0][RTW89_IC][4] = 127,
+	[1][1][2][0][RTW89_KCC][4] = 127,
+	[1][1][2][0][RTW89_ACMA][4] = 127,
+	[1][1][2][0][RTW89_CN][4] = 127,
+	[1][1][2][0][RTW89_UK][4] = 127,
+	[1][1][2][0][RTW89_FCC][5] = 127,
+	[1][1][2][0][RTW89_ETSI][5] = 127,
+	[1][1][2][0][RTW89_MKK][5] = 127,
+	[1][1][2][0][RTW89_IC][5] = 127,
+	[1][1][2][0][RTW89_KCC][5] = 127,
+	[1][1][2][0][RTW89_ACMA][5] = 127,
+	[1][1][2][0][RTW89_CN][5] = 127,
+	[1][1][2][0][RTW89_UK][5] = 127,
+	[1][1][2][0][RTW89_FCC][6] = 127,
+	[1][1][2][0][RTW89_ETSI][6] = 127,
+	[1][1][2][0][RTW89_MKK][6] = 127,
+	[1][1][2][0][RTW89_IC][6] = 127,
+	[1][1][2][0][RTW89_KCC][6] = 127,
+	[1][1][2][0][RTW89_ACMA][6] = 127,
+	[1][1][2][0][RTW89_CN][6] = 127,
+	[1][1][2][0][RTW89_UK][6] = 127,
+	[1][1][2][0][RTW89_FCC][7] = 127,
+	[1][1][2][0][RTW89_ETSI][7] = 127,
+	[1][1][2][0][RTW89_MKK][7] = 127,
+	[1][1][2][0][RTW89_IC][7] = 127,
+	[1][1][2][0][RTW89_KCC][7] = 127,
+	[1][1][2][0][RTW89_ACMA][7] = 127,
+	[1][1][2][0][RTW89_CN][7] = 127,
+	[1][1][2][0][RTW89_UK][7] = 127,
+	[1][1][2][0][RTW89_FCC][8] = 127,
+	[1][1][2][0][RTW89_ETSI][8] = 127,
+	[1][1][2][0][RTW89_MKK][8] = 127,
+	[1][1][2][0][RTW89_IC][8] = 127,
+	[1][1][2][0][RTW89_KCC][8] = 127,
+	[1][1][2][0][RTW89_ACMA][8] = 127,
+	[1][1][2][0][RTW89_CN][8] = 127,
+	[1][1][2][0][RTW89_UK][8] = 127,
+	[1][1][2][0][RTW89_FCC][9] = 127,
+	[1][1][2][0][RTW89_ETSI][9] = 127,
+	[1][1][2][0][RTW89_MKK][9] = 127,
+	[1][1][2][0][RTW89_IC][9] = 127,
+	[1][1][2][0][RTW89_KCC][9] = 127,
+	[1][1][2][0][RTW89_ACMA][9] = 127,
+	[1][1][2][0][RTW89_CN][9] = 127,
+	[1][1][2][0][RTW89_UK][9] = 127,
+	[1][1][2][0][RTW89_FCC][10] = 127,
+	[1][1][2][0][RTW89_ETSI][10] = 127,
+	[1][1][2][0][RTW89_MKK][10] = 127,
+	[1][1][2][0][RTW89_IC][10] = 127,
+	[1][1][2][0][RTW89_KCC][10] = 127,
+	[1][1][2][0][RTW89_ACMA][10] = 127,
+	[1][1][2][0][RTW89_CN][10] = 127,
+	[1][1][2][0][RTW89_UK][10] = 127,
+	[1][1][2][0][RTW89_FCC][11] = 127,
+	[1][1][2][0][RTW89_ETSI][11] = 127,
+	[1][1][2][0][RTW89_MKK][11] = 127,
+	[1][1][2][0][RTW89_IC][11] = 127,
+	[1][1][2][0][RTW89_KCC][11] = 127,
+	[1][1][2][0][RTW89_ACMA][11] = 127,
+	[1][1][2][0][RTW89_CN][11] = 127,
+	[1][1][2][0][RTW89_UK][11] = 127,
+	[1][1][2][0][RTW89_FCC][12] = 127,
+	[1][1][2][0][RTW89_ETSI][12] = 127,
+	[1][1][2][0][RTW89_MKK][12] = 127,
+	[1][1][2][0][RTW89_IC][12] = 127,
+	[1][1][2][0][RTW89_KCC][12] = 127,
+	[1][1][2][0][RTW89_ACMA][12] = 127,
+	[1][1][2][0][RTW89_CN][12] = 127,
+	[1][1][2][0][RTW89_UK][12] = 127,
+	[1][1][2][0][RTW89_FCC][13] = 127,
+	[1][1][2][0][RTW89_ETSI][13] = 127,
+	[1][1][2][0][RTW89_MKK][13] = 127,
+	[1][1][2][0][RTW89_IC][13] = 127,
+	[1][1][2][0][RTW89_KCC][13] = 127,
+	[1][1][2][0][RTW89_ACMA][13] = 127,
+	[1][1][2][0][RTW89_CN][13] = 127,
+	[1][1][2][0][RTW89_UK][13] = 127,
+	[1][1][2][1][RTW89_FCC][0] = 127,
+	[1][1][2][1][RTW89_ETSI][0] = 127,
+	[1][1][2][1][RTW89_MKK][0] = 127,
+	[1][1][2][1][RTW89_IC][0] = 127,
+	[1][1][2][1][RTW89_KCC][0] = 127,
+	[1][1][2][1][RTW89_ACMA][0] = 127,
+	[1][1][2][1][RTW89_CN][0] = 127,
+	[1][1][2][1][RTW89_UK][0] = 127,
+	[1][1][2][1][RTW89_FCC][1] = 127,
+	[1][1][2][1][RTW89_ETSI][1] = 127,
+	[1][1][2][1][RTW89_MKK][1] = 127,
+	[1][1][2][1][RTW89_IC][1] = 127,
+	[1][1][2][1][RTW89_KCC][1] = 127,
+	[1][1][2][1][RTW89_ACMA][1] = 127,
+	[1][1][2][1][RTW89_CN][1] = 127,
+	[1][1][2][1][RTW89_UK][1] = 127,
+	[1][1][2][1][RTW89_FCC][2] = 127,
+	[1][1][2][1][RTW89_ETSI][2] = 127,
+	[1][1][2][1][RTW89_MKK][2] = 127,
+	[1][1][2][1][RTW89_IC][2] = 127,
+	[1][1][2][1][RTW89_KCC][2] = 127,
+	[1][1][2][1][RTW89_ACMA][2] = 127,
+	[1][1][2][1][RTW89_CN][2] = 127,
+	[1][1][2][1][RTW89_UK][2] = 127,
+	[1][1][2][1][RTW89_FCC][3] = 127,
+	[1][1][2][1][RTW89_ETSI][3] = 127,
+	[1][1][2][1][RTW89_MKK][3] = 127,
+	[1][1][2][1][RTW89_IC][3] = 127,
+	[1][1][2][1][RTW89_KCC][3] = 127,
+	[1][1][2][1][RTW89_ACMA][3] = 127,
+	[1][1][2][1][RTW89_CN][3] = 127,
+	[1][1][2][1][RTW89_UK][3] = 127,
+	[1][1][2][1][RTW89_FCC][4] = 127,
+	[1][1][2][1][RTW89_ETSI][4] = 127,
+	[1][1][2][1][RTW89_MKK][4] = 127,
+	[1][1][2][1][RTW89_IC][4] = 127,
+	[1][1][2][1][RTW89_KCC][4] = 127,
+	[1][1][2][1][RTW89_ACMA][4] = 127,
+	[1][1][2][1][RTW89_CN][4] = 127,
+	[1][1][2][1][RTW89_UK][4] = 127,
+	[1][1][2][1][RTW89_FCC][5] = 127,
+	[1][1][2][1][RTW89_ETSI][5] = 127,
+	[1][1][2][1][RTW89_MKK][5] = 127,
+	[1][1][2][1][RTW89_IC][5] = 127,
+	[1][1][2][1][RTW89_KCC][5] = 127,
+	[1][1][2][1][RTW89_ACMA][5] = 127,
+	[1][1][2][1][RTW89_CN][5] = 127,
+	[1][1][2][1][RTW89_UK][5] = 127,
+	[1][1][2][1][RTW89_FCC][6] = 127,
+	[1][1][2][1][RTW89_ETSI][6] = 127,
+	[1][1][2][1][RTW89_MKK][6] = 127,
+	[1][1][2][1][RTW89_IC][6] = 127,
+	[1][1][2][1][RTW89_KCC][6] = 127,
+	[1][1][2][1][RTW89_ACMA][6] = 127,
+	[1][1][2][1][RTW89_CN][6] = 127,
+	[1][1][2][1][RTW89_UK][6] = 127,
+	[1][1][2][1][RTW89_FCC][7] = 127,
+	[1][1][2][1][RTW89_ETSI][7] = 127,
+	[1][1][2][1][RTW89_MKK][7] = 127,
+	[1][1][2][1][RTW89_IC][7] = 127,
+	[1][1][2][1][RTW89_KCC][7] = 127,
+	[1][1][2][1][RTW89_ACMA][7] = 127,
+	[1][1][2][1][RTW89_CN][7] = 127,
+	[1][1][2][1][RTW89_UK][7] = 127,
+	[1][1][2][1][RTW89_FCC][8] = 127,
+	[1][1][2][1][RTW89_ETSI][8] = 127,
+	[1][1][2][1][RTW89_MKK][8] = 127,
+	[1][1][2][1][RTW89_IC][8] = 127,
+	[1][1][2][1][RTW89_KCC][8] = 127,
+	[1][1][2][1][RTW89_ACMA][8] = 127,
+	[1][1][2][1][RTW89_CN][8] = 127,
+	[1][1][2][1][RTW89_UK][8] = 127,
+	[1][1][2][1][RTW89_FCC][9] = 127,
+	[1][1][2][1][RTW89_ETSI][9] = 127,
+	[1][1][2][1][RTW89_MKK][9] = 127,
+	[1][1][2][1][RTW89_IC][9] = 127,
+	[1][1][2][1][RTW89_KCC][9] = 127,
+	[1][1][2][1][RTW89_ACMA][9] = 127,
+	[1][1][2][1][RTW89_CN][9] = 127,
+	[1][1][2][1][RTW89_UK][9] = 127,
+	[1][1][2][1][RTW89_FCC][10] = 127,
+	[1][1][2][1][RTW89_ETSI][10] = 127,
+	[1][1][2][1][RTW89_MKK][10] = 127,
+	[1][1][2][1][RTW89_IC][10] = 127,
+	[1][1][2][1][RTW89_KCC][10] = 127,
+	[1][1][2][1][RTW89_ACMA][10] = 127,
+	[1][1][2][1][RTW89_CN][10] = 127,
+	[1][1][2][1][RTW89_UK][10] = 127,
+	[1][1][2][1][RTW89_FCC][11] = 127,
+	[1][1][2][1][RTW89_ETSI][11] = 127,
+	[1][1][2][1][RTW89_MKK][11] = 127,
+	[1][1][2][1][RTW89_IC][11] = 127,
+	[1][1][2][1][RTW89_KCC][11] = 127,
+	[1][1][2][1][RTW89_ACMA][11] = 127,
+	[1][1][2][1][RTW89_CN][11] = 127,
+	[1][1][2][1][RTW89_UK][11] = 127,
+	[1][1][2][1][RTW89_FCC][12] = 127,
+	[1][1][2][1][RTW89_ETSI][12] = 127,
+	[1][1][2][1][RTW89_MKK][12] = 127,
+	[1][1][2][1][RTW89_IC][12] = 127,
+	[1][1][2][1][RTW89_KCC][12] = 127,
+	[1][1][2][1][RTW89_ACMA][12] = 127,
+	[1][1][2][1][RTW89_CN][12] = 127,
+	[1][1][2][1][RTW89_UK][12] = 127,
+	[1][1][2][1][RTW89_FCC][13] = 127,
+	[1][1][2][1][RTW89_ETSI][13] = 127,
+	[1][1][2][1][RTW89_MKK][13] = 127,
+	[1][1][2][1][RTW89_IC][13] = 127,
+	[1][1][2][1][RTW89_KCC][13] = 127,
+	[1][1][2][1][RTW89_ACMA][13] = 127,
+	[1][1][2][1][RTW89_CN][13] = 127,
+	[1][1][2][1][RTW89_UK][13] = 127,
+};
+
+static
+const s8 rtw89_8851b_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
+				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
+				 [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
+	[0][0][1][0][RTW89_WW][0] = 58,
+	[0][0][1][0][RTW89_WW][2] = 58,
+	[0][0][1][0][RTW89_WW][4] = 58,
+	[0][0][1][0][RTW89_WW][6] = 50,
+	[0][0][1][0][RTW89_WW][8] = 58,
+	[0][0][1][0][RTW89_WW][10] = 58,
+	[0][0][1][0][RTW89_WW][12] = 58,
+	[0][0][1][0][RTW89_WW][14] = 58,
+	[0][0][1][0][RTW89_WW][15] = 58,
+	[0][0][1][0][RTW89_WW][17] = 60,
+	[0][0][1][0][RTW89_WW][19] = 60,
+	[0][0][1][0][RTW89_WW][21] = 60,
+	[0][0][1][0][RTW89_WW][23] = 60,
+	[0][0][1][0][RTW89_WW][25] = 60,
+	[0][0][1][0][RTW89_WW][27] = 60,
+	[0][0][1][0][RTW89_WW][29] = 60,
+	[0][0][1][0][RTW89_WW][31] = 60,
+	[0][0][1][0][RTW89_WW][33] = 60,
+	[0][0][1][0][RTW89_WW][35] = 60,
+	[0][0][1][0][RTW89_WW][37] = 74,
+	[0][0][1][0][RTW89_WW][38] = 30,
+	[0][0][1][0][RTW89_WW][40] = 30,
+	[0][0][1][0][RTW89_WW][42] = 30,
+	[0][0][1][0][RTW89_WW][44] = 30,
+	[0][0][1][0][RTW89_WW][46] = 30,
+	[0][0][1][0][RTW89_WW][48] = 72,
+	[0][0][1][0][RTW89_WW][50] = 72,
+	[0][0][1][0][RTW89_WW][52] = 72,
+	[0][1][1][0][RTW89_WW][0] = 0,
+	[0][1][1][0][RTW89_WW][2] = 0,
+	[0][1][1][0][RTW89_WW][4] = 0,
+	[0][1][1][0][RTW89_WW][6] = 0,
+	[0][1][1][0][RTW89_WW][8] = 0,
+	[0][1][1][0][RTW89_WW][10] = 0,
+	[0][1][1][0][RTW89_WW][12] = 0,
+	[0][1][1][0][RTW89_WW][14] = 0,
+	[0][1][1][0][RTW89_WW][15] = 0,
+	[0][1][1][0][RTW89_WW][17] = 0,
+	[0][1][1][0][RTW89_WW][19] = 0,
+	[0][1][1][0][RTW89_WW][21] = 0,
+	[0][1][1][0][RTW89_WW][23] = 0,
+	[0][1][1][0][RTW89_WW][25] = 0,
+	[0][1][1][0][RTW89_WW][27] = 0,
+	[0][1][1][0][RTW89_WW][29] = 0,
+	[0][1][1][0][RTW89_WW][31] = 0,
+	[0][1][1][0][RTW89_WW][33] = 0,
+	[0][1][1][0][RTW89_WW][35] = 0,
+	[0][1][1][0][RTW89_WW][37] = 0,
+	[0][1][1][0][RTW89_WW][38] = 0,
+	[0][1][1][0][RTW89_WW][40] = 0,
+	[0][1][1][0][RTW89_WW][42] = 0,
+	[0][1][1][0][RTW89_WW][44] = 0,
+	[0][1][1][0][RTW89_WW][46] = 0,
+	[0][1][1][0][RTW89_WW][48] = 0,
+	[0][1][1][0][RTW89_WW][50] = 0,
+	[0][1][1][0][RTW89_WW][52] = 0,
+	[0][0][2][0][RTW89_WW][0] = 62,
+	[0][0][2][0][RTW89_WW][2] = 62,
+	[0][0][2][0][RTW89_WW][4] = 62,
+	[0][0][2][0][RTW89_WW][6] = 54,
+	[0][0][2][0][RTW89_WW][8] = 62,
+	[0][0][2][0][RTW89_WW][10] = 62,
+	[0][0][2][0][RTW89_WW][12] = 62,
+	[0][0][2][0][RTW89_WW][14] = 62,
+	[0][0][2][0][RTW89_WW][15] = 60,
+	[0][0][2][0][RTW89_WW][17] = 62,
+	[0][0][2][0][RTW89_WW][19] = 62,
+	[0][0][2][0][RTW89_WW][21] = 62,
+	[0][0][2][0][RTW89_WW][23] = 62,
+	[0][0][2][0][RTW89_WW][25] = 62,
+	[0][0][2][0][RTW89_WW][27] = 62,
+	[0][0][2][0][RTW89_WW][29] = 62,
+	[0][0][2][0][RTW89_WW][31] = 62,
+	[0][0][2][0][RTW89_WW][33] = 62,
+	[0][0][2][0][RTW89_WW][35] = 62,
+	[0][0][2][0][RTW89_WW][37] = 76,
+	[0][0][2][0][RTW89_WW][38] = 30,
+	[0][0][2][0][RTW89_WW][40] = 30,
+	[0][0][2][0][RTW89_WW][42] = 30,
+	[0][0][2][0][RTW89_WW][44] = 30,
+	[0][0][2][0][RTW89_WW][46] = 30,
+	[0][0][2][0][RTW89_WW][48] = 74,
+	[0][0][2][0][RTW89_WW][50] = 76,
+	[0][0][2][0][RTW89_WW][52] = 76,
+	[0][1][2][0][RTW89_WW][0] = 0,
+	[0][1][2][0][RTW89_WW][2] = 0,
+	[0][1][2][0][RTW89_WW][4] = 0,
+	[0][1][2][0][RTW89_WW][6] = 0,
+	[0][1][2][0][RTW89_WW][8] = 0,
+	[0][1][2][0][RTW89_WW][10] = 0,
+	[0][1][2][0][RTW89_WW][12] = 0,
+	[0][1][2][0][RTW89_WW][14] = 0,
+	[0][1][2][0][RTW89_WW][15] = 0,
+	[0][1][2][0][RTW89_WW][17] = 0,
+	[0][1][2][0][RTW89_WW][19] = 0,
+	[0][1][2][0][RTW89_WW][21] = 0,
+	[0][1][2][0][RTW89_WW][23] = 0,
+	[0][1][2][0][RTW89_WW][25] = 0,
+	[0][1][2][0][RTW89_WW][27] = 0,
+	[0][1][2][0][RTW89_WW][29] = 0,
+	[0][1][2][0][RTW89_WW][31] = 0,
+	[0][1][2][0][RTW89_WW][33] = 0,
+	[0][1][2][0][RTW89_WW][35] = 0,
+	[0][1][2][0][RTW89_WW][37] = 0,
+	[0][1][2][0][RTW89_WW][38] = 0,
+	[0][1][2][0][RTW89_WW][40] = 0,
+	[0][1][2][0][RTW89_WW][42] = 0,
+	[0][1][2][0][RTW89_WW][44] = 0,
+	[0][1][2][0][RTW89_WW][46] = 0,
+	[0][1][2][0][RTW89_WW][48] = 0,
+	[0][1][2][0][RTW89_WW][50] = 0,
+	[0][1][2][0][RTW89_WW][52] = 0,
+	[0][1][2][1][RTW89_WW][0] = 0,
+	[0][1][2][1][RTW89_WW][2] = 0,
+	[0][1][2][1][RTW89_WW][4] = 0,
+	[0][1][2][1][RTW89_WW][6] = 0,
+	[0][1][2][1][RTW89_WW][8] = 0,
+	[0][1][2][1][RTW89_WW][10] = 0,
+	[0][1][2][1][RTW89_WW][12] = 0,
+	[0][1][2][1][RTW89_WW][14] = 0,
+	[0][1][2][1][RTW89_WW][15] = 0,
+	[0][1][2][1][RTW89_WW][17] = 0,
+	[0][1][2][1][RTW89_WW][19] = 0,
+	[0][1][2][1][RTW89_WW][21] = 0,
+	[0][1][2][1][RTW89_WW][23] = 0,
+	[0][1][2][1][RTW89_WW][25] = 0,
+	[0][1][2][1][RTW89_WW][27] = 0,
+	[0][1][2][1][RTW89_WW][29] = 0,
+	[0][1][2][1][RTW89_WW][31] = 0,
+	[0][1][2][1][RTW89_WW][33] = 0,
+	[0][1][2][1][RTW89_WW][35] = 0,
+	[0][1][2][1][RTW89_WW][37] = 0,
+	[0][1][2][1][RTW89_WW][38] = 0,
+	[0][1][2][1][RTW89_WW][40] = 0,
+	[0][1][2][1][RTW89_WW][42] = 0,
+	[0][1][2][1][RTW89_WW][44] = 0,
+	[0][1][2][1][RTW89_WW][46] = 0,
+	[0][1][2][1][RTW89_WW][48] = 0,
+	[0][1][2][1][RTW89_WW][50] = 0,
+	[0][1][2][1][RTW89_WW][52] = 0,
+	[1][0][2][0][RTW89_WW][1] = 64,
+	[1][0][2][0][RTW89_WW][5] = 62,
+	[1][0][2][0][RTW89_WW][9] = 64,
+	[1][0][2][0][RTW89_WW][13] = 64,
+	[1][0][2][0][RTW89_WW][16] = 66,
+	[1][0][2][0][RTW89_WW][20] = 66,
+	[1][0][2][0][RTW89_WW][24] = 66,
+	[1][0][2][0][RTW89_WW][28] = 66,
+	[1][0][2][0][RTW89_WW][32] = 66,
+	[1][0][2][0][RTW89_WW][36] = 76,
+	[1][0][2][0][RTW89_WW][39] = 30,
+	[1][0][2][0][RTW89_WW][43] = 30,
+	[1][0][2][0][RTW89_WW][47] = 84,
+	[1][0][2][0][RTW89_WW][51] = 84,
+	[1][1][2][0][RTW89_WW][1] = 0,
+	[1][1][2][0][RTW89_WW][5] = 0,
+	[1][1][2][0][RTW89_WW][9] = 0,
+	[1][1][2][0][RTW89_WW][13] = 0,
+	[1][1][2][0][RTW89_WW][16] = 0,
+	[1][1][2][0][RTW89_WW][20] = 0,
+	[1][1][2][0][RTW89_WW][24] = 0,
+	[1][1][2][0][RTW89_WW][28] = 0,
+	[1][1][2][0][RTW89_WW][32] = 0,
+	[1][1][2][0][RTW89_WW][36] = 0,
+	[1][1][2][0][RTW89_WW][39] = 0,
+	[1][1][2][0][RTW89_WW][43] = 0,
+	[1][1][2][0][RTW89_WW][47] = 0,
+	[1][1][2][0][RTW89_WW][51] = 0,
+	[1][1][2][1][RTW89_WW][1] = 0,
+	[1][1][2][1][RTW89_WW][5] = 0,
+	[1][1][2][1][RTW89_WW][9] = 0,
+	[1][1][2][1][RTW89_WW][13] = 0,
+	[1][1][2][1][RTW89_WW][16] = 0,
+	[1][1][2][1][RTW89_WW][20] = 0,
+	[1][1][2][1][RTW89_WW][24] = 0,
+	[1][1][2][1][RTW89_WW][28] = 0,
+	[1][1][2][1][RTW89_WW][32] = 0,
+	[1][1][2][1][RTW89_WW][36] = 0,
+	[1][1][2][1][RTW89_WW][39] = 0,
+	[1][1][2][1][RTW89_WW][43] = 0,
+	[1][1][2][1][RTW89_WW][47] = 0,
+	[1][1][2][1][RTW89_WW][51] = 0,
+	[2][0][2][0][RTW89_WW][3] = 62,
+	[2][0][2][0][RTW89_WW][11] = 62,
+	[2][0][2][0][RTW89_WW][18] = 64,
+	[2][0][2][0][RTW89_WW][26] = 64,
+	[2][0][2][0][RTW89_WW][34] = 72,
+	[2][0][2][0][RTW89_WW][41] = 30,
+	[2][0][2][0][RTW89_WW][49] = 74,
+	[2][1][2][0][RTW89_WW][3] = 0,
+	[2][1][2][0][RTW89_WW][11] = 0,
+	[2][1][2][0][RTW89_WW][18] = 0,
+	[2][1][2][0][RTW89_WW][26] = 0,
+	[2][1][2][0][RTW89_WW][34] = 0,
+	[2][1][2][0][RTW89_WW][41] = 0,
+	[2][1][2][0][RTW89_WW][49] = 0,
+	[2][1][2][1][RTW89_WW][3] = 0,
+	[2][1][2][1][RTW89_WW][11] = 0,
+	[2][1][2][1][RTW89_WW][18] = 0,
+	[2][1][2][1][RTW89_WW][26] = 0,
+	[2][1][2][1][RTW89_WW][34] = 0,
+	[2][1][2][1][RTW89_WW][41] = 0,
+	[2][1][2][1][RTW89_WW][49] = 0,
+	[3][0][2][0][RTW89_WW][7] = 58,
+	[3][0][2][0][RTW89_WW][22] = 58,
+	[3][0][2][0][RTW89_WW][45] = 0,
+	[3][1][2][0][RTW89_WW][7] = 0,
+	[3][1][2][0][RTW89_WW][22] = 0,
+	[3][1][2][0][RTW89_WW][45] = 0,
+	[3][1][2][1][RTW89_WW][7] = 0,
+	[3][1][2][1][RTW89_WW][22] = 0,
+	[3][1][2][1][RTW89_WW][45] = 0,
+	[0][0][1][0][RTW89_FCC][0] = 80,
+	[0][0][1][0][RTW89_ETSI][0] = 58,
+	[0][0][1][0][RTW89_MKK][0] = 60,
+	[0][0][1][0][RTW89_IC][0] = 62,
+	[0][0][1][0][RTW89_KCC][0] = 74,
+	[0][0][1][0][RTW89_ACMA][0] = 58,
+	[0][0][1][0][RTW89_CN][0] = 60,
+	[0][0][1][0][RTW89_UK][0] = 58,
+	[0][0][1][0][RTW89_FCC][2] = 82,
+	[0][0][1][0][RTW89_ETSI][2] = 58,
+	[0][0][1][0][RTW89_MKK][2] = 60,
+	[0][0][1][0][RTW89_IC][2] = 62,
+	[0][0][1][0][RTW89_KCC][2] = 74,
+	[0][0][1][0][RTW89_ACMA][2] = 58,
+	[0][0][1][0][RTW89_CN][2] = 60,
+	[0][0][1][0][RTW89_UK][2] = 58,
+	[0][0][1][0][RTW89_FCC][4] = 82,
+	[0][0][1][0][RTW89_ETSI][4] = 58,
+	[0][0][1][0][RTW89_MKK][4] = 60,
+	[0][0][1][0][RTW89_IC][4] = 62,
+	[0][0][1][0][RTW89_KCC][4] = 74,
+	[0][0][1][0][RTW89_ACMA][4] = 58,
+	[0][0][1][0][RTW89_CN][4] = 60,
+	[0][0][1][0][RTW89_UK][4] = 58,
+	[0][0][1][0][RTW89_FCC][6] = 82,
+	[0][0][1][0][RTW89_ETSI][6] = 58,
+	[0][0][1][0][RTW89_MKK][6] = 60,
+	[0][0][1][0][RTW89_IC][6] = 62,
+	[0][0][1][0][RTW89_KCC][6] = 50,
+	[0][0][1][0][RTW89_ACMA][6] = 58,
+	[0][0][1][0][RTW89_CN][6] = 60,
+	[0][0][1][0][RTW89_UK][6] = 58,
+	[0][0][1][0][RTW89_FCC][8] = 82,
+	[0][0][1][0][RTW89_ETSI][8] = 58,
+	[0][0][1][0][RTW89_MKK][8] = 60,
+	[0][0][1][0][RTW89_IC][8] = 64,
+	[0][0][1][0][RTW89_KCC][8] = 74,
+	[0][0][1][0][RTW89_ACMA][8] = 58,
+	[0][0][1][0][RTW89_CN][8] = 60,
+	[0][0][1][0][RTW89_UK][8] = 58,
+	[0][0][1][0][RTW89_FCC][10] = 82,
+	[0][0][1][0][RTW89_ETSI][10] = 58,
+	[0][0][1][0][RTW89_MKK][10] = 60,
+	[0][0][1][0][RTW89_IC][10] = 64,
+	[0][0][1][0][RTW89_KCC][10] = 74,
+	[0][0][1][0][RTW89_ACMA][10] = 58,
+	[0][0][1][0][RTW89_CN][10] = 60,
+	[0][0][1][0][RTW89_UK][10] = 58,
+	[0][0][1][0][RTW89_FCC][12] = 82,
+	[0][0][1][0][RTW89_ETSI][12] = 58,
+	[0][0][1][0][RTW89_MKK][12] = 60,
+	[0][0][1][0][RTW89_IC][12] = 64,
+	[0][0][1][0][RTW89_KCC][12] = 76,
+	[0][0][1][0][RTW89_ACMA][12] = 58,
+	[0][0][1][0][RTW89_CN][12] = 60,
+	[0][0][1][0][RTW89_UK][12] = 58,
+	[0][0][1][0][RTW89_FCC][14] = 78,
+	[0][0][1][0][RTW89_ETSI][14] = 58,
+	[0][0][1][0][RTW89_MKK][14] = 60,
+	[0][0][1][0][RTW89_IC][14] = 64,
+	[0][0][1][0][RTW89_KCC][14] = 76,
+	[0][0][1][0][RTW89_ACMA][14] = 58,
+	[0][0][1][0][RTW89_CN][14] = 60,
+	[0][0][1][0][RTW89_UK][14] = 58,
+	[0][0][1][0][RTW89_FCC][15] = 78,
+	[0][0][1][0][RTW89_ETSI][15] = 58,
+	[0][0][1][0][RTW89_MKK][15] = 78,
+	[0][0][1][0][RTW89_IC][15] = 78,
+	[0][0][1][0][RTW89_KCC][15] = 78,
+	[0][0][1][0][RTW89_ACMA][15] = 58,
+	[0][0][1][0][RTW89_CN][15] = 127,
+	[0][0][1][0][RTW89_UK][15] = 58,
+	[0][0][1][0][RTW89_FCC][17] = 82,
+	[0][0][1][0][RTW89_ETSI][17] = 60,
+	[0][0][1][0][RTW89_MKK][17] = 78,
+	[0][0][1][0][RTW89_IC][17] = 82,
+	[0][0][1][0][RTW89_KCC][17] = 78,
+	[0][0][1][0][RTW89_ACMA][17] = 60,
+	[0][0][1][0][RTW89_CN][17] = 127,
+	[0][0][1][0][RTW89_UK][17] = 60,
+	[0][0][1][0][RTW89_FCC][19] = 82,
+	[0][0][1][0][RTW89_ETSI][19] = 60,
+	[0][0][1][0][RTW89_MKK][19] = 78,
+	[0][0][1][0][RTW89_IC][19] = 82,
+	[0][0][1][0][RTW89_KCC][19] = 78,
+	[0][0][1][0][RTW89_ACMA][19] = 60,
+	[0][0][1][0][RTW89_CN][19] = 127,
+	[0][0][1][0][RTW89_UK][19] = 60,
+	[0][0][1][0][RTW89_FCC][21] = 82,
+	[0][0][1][0][RTW89_ETSI][21] = 60,
+	[0][0][1][0][RTW89_MKK][21] = 78,
+	[0][0][1][0][RTW89_IC][21] = 82,
+	[0][0][1][0][RTW89_KCC][21] = 78,
+	[0][0][1][0][RTW89_ACMA][21] = 60,
+	[0][0][1][0][RTW89_CN][21] = 127,
+	[0][0][1][0][RTW89_UK][21] = 60,
+	[0][0][1][0][RTW89_FCC][23] = 82,
+	[0][0][1][0][RTW89_ETSI][23] = 60,
+	[0][0][1][0][RTW89_MKK][23] = 78,
+	[0][0][1][0][RTW89_IC][23] = 82,
+	[0][0][1][0][RTW89_KCC][23] = 78,
+	[0][0][1][0][RTW89_ACMA][23] = 60,
+	[0][0][1][0][RTW89_CN][23] = 127,
+	[0][0][1][0][RTW89_UK][23] = 60,
+	[0][0][1][0][RTW89_FCC][25] = 82,
+	[0][0][1][0][RTW89_ETSI][25] = 60,
+	[0][0][1][0][RTW89_MKK][25] = 78,
+	[0][0][1][0][RTW89_IC][25] = 127,
+	[0][0][1][0][RTW89_KCC][25] = 78,
+	[0][0][1][0][RTW89_ACMA][25] = 127,
+	[0][0][1][0][RTW89_CN][25] = 127,
+	[0][0][1][0][RTW89_UK][25] = 60,
+	[0][0][1][0][RTW89_FCC][27] = 82,
+	[0][0][1][0][RTW89_ETSI][27] = 60,
+	[0][0][1][0][RTW89_MKK][27] = 78,
+	[0][0][1][0][RTW89_IC][27] = 127,
+	[0][0][1][0][RTW89_KCC][27] = 78,
+	[0][0][1][0][RTW89_ACMA][27] = 127,
+	[0][0][1][0][RTW89_CN][27] = 127,
+	[0][0][1][0][RTW89_UK][27] = 60,
+	[0][0][1][0][RTW89_FCC][29] = 82,
+	[0][0][1][0][RTW89_ETSI][29] = 60,
+	[0][0][1][0][RTW89_MKK][29] = 78,
+	[0][0][1][0][RTW89_IC][29] = 127,
+	[0][0][1][0][RTW89_KCC][29] = 78,
+	[0][0][1][0][RTW89_ACMA][29] = 127,
+	[0][0][1][0][RTW89_CN][29] = 127,
+	[0][0][1][0][RTW89_UK][29] = 60,
+	[0][0][1][0][RTW89_FCC][31] = 82,
+	[0][0][1][0][RTW89_ETSI][31] = 60,
+	[0][0][1][0][RTW89_MKK][31] = 78,
+	[0][0][1][0][RTW89_IC][31] = 82,
+	[0][0][1][0][RTW89_KCC][31] = 74,
+	[0][0][1][0][RTW89_ACMA][31] = 60,
+	[0][0][1][0][RTW89_CN][31] = 127,
+	[0][0][1][0][RTW89_UK][31] = 60,
+	[0][0][1][0][RTW89_FCC][33] = 82,
+	[0][0][1][0][RTW89_ETSI][33] = 60,
+	[0][0][1][0][RTW89_MKK][33] = 78,
+	[0][0][1][0][RTW89_IC][33] = 82,
+	[0][0][1][0][RTW89_KCC][33] = 74,
+	[0][0][1][0][RTW89_ACMA][33] = 60,
+	[0][0][1][0][RTW89_CN][33] = 127,
+	[0][0][1][0][RTW89_UK][33] = 60,
+	[0][0][1][0][RTW89_FCC][35] = 72,
+	[0][0][1][0][RTW89_ETSI][35] = 60,
+	[0][0][1][0][RTW89_MKK][35] = 78,
+	[0][0][1][0][RTW89_IC][35] = 72,
+	[0][0][1][0][RTW89_KCC][35] = 74,
+	[0][0][1][0][RTW89_ACMA][35] = 60,
+	[0][0][1][0][RTW89_CN][35] = 127,
+	[0][0][1][0][RTW89_UK][35] = 60,
+	[0][0][1][0][RTW89_FCC][37] = 82,
+	[0][0][1][0][RTW89_ETSI][37] = 127,
+	[0][0][1][0][RTW89_MKK][37] = 78,
+	[0][0][1][0][RTW89_IC][37] = 82,
+	[0][0][1][0][RTW89_KCC][37] = 74,
+	[0][0][1][0][RTW89_ACMA][37] = 78,
+	[0][0][1][0][RTW89_CN][37] = 127,
+	[0][0][1][0][RTW89_UK][37] = 78,
+	[0][0][1][0][RTW89_FCC][38] = 82,
+	[0][0][1][0][RTW89_ETSI][38] = 30,
+	[0][0][1][0][RTW89_MKK][38] = 127,
+	[0][0][1][0][RTW89_IC][38] = 82,
+	[0][0][1][0][RTW89_KCC][38] = 70,
+	[0][0][1][0][RTW89_ACMA][38] = 78,
+	[0][0][1][0][RTW89_CN][38] = 78,
+	[0][0][1][0][RTW89_UK][38] = 58,
+	[0][0][1][0][RTW89_FCC][40] = 82,
+	[0][0][1][0][RTW89_ETSI][40] = 30,
+	[0][0][1][0][RTW89_MKK][40] = 127,
+	[0][0][1][0][RTW89_IC][40] = 82,
+	[0][0][1][0][RTW89_KCC][40] = 76,
+	[0][0][1][0][RTW89_ACMA][40] = 78,
+	[0][0][1][0][RTW89_CN][40] = 78,
+	[0][0][1][0][RTW89_UK][40] = 58,
+	[0][0][1][0][RTW89_FCC][42] = 82,
+	[0][0][1][0][RTW89_ETSI][42] = 30,
+	[0][0][1][0][RTW89_MKK][42] = 127,
+	[0][0][1][0][RTW89_IC][42] = 82,
+	[0][0][1][0][RTW89_KCC][42] = 76,
+	[0][0][1][0][RTW89_ACMA][42] = 78,
+	[0][0][1][0][RTW89_CN][42] = 78,
+	[0][0][1][0][RTW89_UK][42] = 58,
+	[0][0][1][0][RTW89_FCC][44] = 82,
+	[0][0][1][0][RTW89_ETSI][44] = 30,
+	[0][0][1][0][RTW89_MKK][44] = 127,
+	[0][0][1][0][RTW89_IC][44] = 82,
+	[0][0][1][0][RTW89_KCC][44] = 76,
+	[0][0][1][0][RTW89_ACMA][44] = 78,
+	[0][0][1][0][RTW89_CN][44] = 78,
+	[0][0][1][0][RTW89_UK][44] = 58,
+	[0][0][1][0][RTW89_FCC][46] = 82,
+	[0][0][1][0][RTW89_ETSI][46] = 30,
+	[0][0][1][0][RTW89_MKK][46] = 127,
+	[0][0][1][0][RTW89_IC][46] = 82,
+	[0][0][1][0][RTW89_KCC][46] = 76,
+	[0][0][1][0][RTW89_ACMA][46] = 78,
+	[0][0][1][0][RTW89_CN][46] = 78,
+	[0][0][1][0][RTW89_UK][46] = 58,
+	[0][0][1][0][RTW89_FCC][48] = 72,
+	[0][0][1][0][RTW89_ETSI][48] = 127,
+	[0][0][1][0][RTW89_MKK][48] = 127,
+	[0][0][1][0][RTW89_IC][48] = 127,
+	[0][0][1][0][RTW89_KCC][48] = 127,
+	[0][0][1][0][RTW89_ACMA][48] = 127,
+	[0][0][1][0][RTW89_CN][48] = 127,
+	[0][0][1][0][RTW89_UK][48] = 127,
+	[0][0][1][0][RTW89_FCC][50] = 72,
+	[0][0][1][0][RTW89_ETSI][50] = 127,
+	[0][0][1][0][RTW89_MKK][50] = 127,
+	[0][0][1][0][RTW89_IC][50] = 127,
+	[0][0][1][0][RTW89_KCC][50] = 127,
+	[0][0][1][0][RTW89_ACMA][50] = 127,
+	[0][0][1][0][RTW89_CN][50] = 127,
+	[0][0][1][0][RTW89_UK][50] = 127,
+	[0][0][1][0][RTW89_FCC][52] = 72,
+	[0][0][1][0][RTW89_ETSI][52] = 127,
+	[0][0][1][0][RTW89_MKK][52] = 127,
+	[0][0][1][0][RTW89_IC][52] = 127,
+	[0][0][1][0][RTW89_KCC][52] = 127,
+	[0][0][1][0][RTW89_ACMA][52] = 127,
+	[0][0][1][0][RTW89_CN][52] = 127,
+	[0][0][1][0][RTW89_UK][52] = 127,
+	[0][1][1][0][RTW89_FCC][0] = 127,
+	[0][1][1][0][RTW89_ETSI][0] = 127,
+	[0][1][1][0][RTW89_MKK][0] = 127,
+	[0][1][1][0][RTW89_IC][0] = 127,
+	[0][1][1][0][RTW89_KCC][0] = 127,
+	[0][1][1][0][RTW89_ACMA][0] = 127,
+	[0][1][1][0][RTW89_CN][0] = 127,
+	[0][1][1][0][RTW89_UK][0] = 127,
+	[0][1][1][0][RTW89_FCC][2] = 127,
+	[0][1][1][0][RTW89_ETSI][2] = 127,
+	[0][1][1][0][RTW89_MKK][2] = 127,
+	[0][1][1][0][RTW89_IC][2] = 127,
+	[0][1][1][0][RTW89_KCC][2] = 127,
+	[0][1][1][0][RTW89_ACMA][2] = 127,
+	[0][1][1][0][RTW89_CN][2] = 127,
+	[0][1][1][0][RTW89_UK][2] = 127,
+	[0][1][1][0][RTW89_FCC][4] = 127,
+	[0][1][1][0][RTW89_ETSI][4] = 127,
+	[0][1][1][0][RTW89_MKK][4] = 127,
+	[0][1][1][0][RTW89_IC][4] = 127,
+	[0][1][1][0][RTW89_KCC][4] = 127,
+	[0][1][1][0][RTW89_ACMA][4] = 127,
+	[0][1][1][0][RTW89_CN][4] = 127,
+	[0][1][1][0][RTW89_UK][4] = 127,
+	[0][1][1][0][RTW89_FCC][6] = 127,
+	[0][1][1][0][RTW89_ETSI][6] = 127,
+	[0][1][1][0][RTW89_MKK][6] = 127,
+	[0][1][1][0][RTW89_IC][6] = 127,
+	[0][1][1][0][RTW89_KCC][6] = 127,
+	[0][1][1][0][RTW89_ACMA][6] = 127,
+	[0][1][1][0][RTW89_CN][6] = 127,
+	[0][1][1][0][RTW89_UK][6] = 127,
+	[0][1][1][0][RTW89_FCC][8] = 127,
+	[0][1][1][0][RTW89_ETSI][8] = 127,
+	[0][1][1][0][RTW89_MKK][8] = 127,
+	[0][1][1][0][RTW89_IC][8] = 127,
+	[0][1][1][0][RTW89_KCC][8] = 127,
+	[0][1][1][0][RTW89_ACMA][8] = 127,
+	[0][1][1][0][RTW89_CN][8] = 127,
+	[0][1][1][0][RTW89_UK][8] = 127,
+	[0][1][1][0][RTW89_FCC][10] = 127,
+	[0][1][1][0][RTW89_ETSI][10] = 127,
+	[0][1][1][0][RTW89_MKK][10] = 127,
+	[0][1][1][0][RTW89_IC][10] = 127,
+	[0][1][1][0][RTW89_KCC][10] = 127,
+	[0][1][1][0][RTW89_ACMA][10] = 127,
+	[0][1][1][0][RTW89_CN][10] = 127,
+	[0][1][1][0][RTW89_UK][10] = 127,
+	[0][1][1][0][RTW89_FCC][12] = 127,
+	[0][1][1][0][RTW89_ETSI][12] = 127,
+	[0][1][1][0][RTW89_MKK][12] = 127,
+	[0][1][1][0][RTW89_IC][12] = 127,
+	[0][1][1][0][RTW89_KCC][12] = 127,
+	[0][1][1][0][RTW89_ACMA][12] = 127,
+	[0][1][1][0][RTW89_CN][12] = 127,
+	[0][1][1][0][RTW89_UK][12] = 127,
+	[0][1][1][0][RTW89_FCC][14] = 127,
+	[0][1][1][0][RTW89_ETSI][14] = 127,
+	[0][1][1][0][RTW89_MKK][14] = 127,
+	[0][1][1][0][RTW89_IC][14] = 127,
+	[0][1][1][0][RTW89_KCC][14] = 127,
+	[0][1][1][0][RTW89_ACMA][14] = 127,
+	[0][1][1][0][RTW89_CN][14] = 127,
+	[0][1][1][0][RTW89_UK][14] = 127,
+	[0][1][1][0][RTW89_FCC][15] = 127,
+	[0][1][1][0][RTW89_ETSI][15] = 127,
+	[0][1][1][0][RTW89_MKK][15] = 127,
+	[0][1][1][0][RTW89_IC][15] = 127,
+	[0][1][1][0][RTW89_KCC][15] = 127,
+	[0][1][1][0][RTW89_ACMA][15] = 127,
+	[0][1][1][0][RTW89_CN][15] = 127,
+	[0][1][1][0][RTW89_UK][15] = 127,
+	[0][1][1][0][RTW89_FCC][17] = 127,
+	[0][1][1][0][RTW89_ETSI][17] = 127,
+	[0][1][1][0][RTW89_MKK][17] = 127,
+	[0][1][1][0][RTW89_IC][17] = 127,
+	[0][1][1][0][RTW89_KCC][17] = 127,
+	[0][1][1][0][RTW89_ACMA][17] = 127,
+	[0][1][1][0][RTW89_CN][17] = 127,
+	[0][1][1][0][RTW89_UK][17] = 127,
+	[0][1][1][0][RTW89_FCC][19] = 127,
+	[0][1][1][0][RTW89_ETSI][19] = 127,
+	[0][1][1][0][RTW89_MKK][19] = 127,
+	[0][1][1][0][RTW89_IC][19] = 127,
+	[0][1][1][0][RTW89_KCC][19] = 127,
+	[0][1][1][0][RTW89_ACMA][19] = 127,
+	[0][1][1][0][RTW89_CN][19] = 127,
+	[0][1][1][0][RTW89_UK][19] = 127,
+	[0][1][1][0][RTW89_FCC][21] = 127,
+	[0][1][1][0][RTW89_ETSI][21] = 127,
+	[0][1][1][0][RTW89_MKK][21] = 127,
+	[0][1][1][0][RTW89_IC][21] = 127,
+	[0][1][1][0][RTW89_KCC][21] = 127,
+	[0][1][1][0][RTW89_ACMA][21] = 127,
+	[0][1][1][0][RTW89_CN][21] = 127,
+	[0][1][1][0][RTW89_UK][21] = 127,
+	[0][1][1][0][RTW89_FCC][23] = 127,
+	[0][1][1][0][RTW89_ETSI][23] = 127,
+	[0][1][1][0][RTW89_MKK][23] = 127,
+	[0][1][1][0][RTW89_IC][23] = 127,
+	[0][1][1][0][RTW89_KCC][23] = 127,
+	[0][1][1][0][RTW89_ACMA][23] = 127,
+	[0][1][1][0][RTW89_CN][23] = 127,
+	[0][1][1][0][RTW89_UK][23] = 127,
+	[0][1][1][0][RTW89_FCC][25] = 127,
+	[0][1][1][0][RTW89_ETSI][25] = 127,
+	[0][1][1][0][RTW89_MKK][25] = 127,
+	[0][1][1][0][RTW89_IC][25] = 127,
+	[0][1][1][0][RTW89_KCC][25] = 127,
+	[0][1][1][0][RTW89_ACMA][25] = 127,
+	[0][1][1][0][RTW89_CN][25] = 127,
+	[0][1][1][0][RTW89_UK][25] = 127,
+	[0][1][1][0][RTW89_FCC][27] = 127,
+	[0][1][1][0][RTW89_ETSI][27] = 127,
+	[0][1][1][0][RTW89_MKK][27] = 127,
+	[0][1][1][0][RTW89_IC][27] = 127,
+	[0][1][1][0][RTW89_KCC][27] = 127,
+	[0][1][1][0][RTW89_ACMA][27] = 127,
+	[0][1][1][0][RTW89_CN][27] = 127,
+	[0][1][1][0][RTW89_UK][27] = 127,
+	[0][1][1][0][RTW89_FCC][29] = 127,
+	[0][1][1][0][RTW89_ETSI][29] = 127,
+	[0][1][1][0][RTW89_MKK][29] = 127,
+	[0][1][1][0][RTW89_IC][29] = 127,
+	[0][1][1][0][RTW89_KCC][29] = 127,
+	[0][1][1][0][RTW89_ACMA][29] = 127,
+	[0][1][1][0][RTW89_CN][29] = 127,
+	[0][1][1][0][RTW89_UK][29] = 127,
+	[0][1][1][0][RTW89_FCC][31] = 127,
+	[0][1][1][0][RTW89_ETSI][31] = 127,
+	[0][1][1][0][RTW89_MKK][31] = 127,
+	[0][1][1][0][RTW89_IC][31] = 127,
+	[0][1][1][0][RTW89_KCC][31] = 127,
+	[0][1][1][0][RTW89_ACMA][31] = 127,
+	[0][1][1][0][RTW89_CN][31] = 127,
+	[0][1][1][0][RTW89_UK][31] = 127,
+	[0][1][1][0][RTW89_FCC][33] = 127,
+	[0][1][1][0][RTW89_ETSI][33] = 127,
+	[0][1][1][0][RTW89_MKK][33] = 127,
+	[0][1][1][0][RTW89_IC][33] = 127,
+	[0][1][1][0][RTW89_KCC][33] = 127,
+	[0][1][1][0][RTW89_ACMA][33] = 127,
+	[0][1][1][0][RTW89_CN][33] = 127,
+	[0][1][1][0][RTW89_UK][33] = 127,
+	[0][1][1][0][RTW89_FCC][35] = 127,
+	[0][1][1][0][RTW89_ETSI][35] = 127,
+	[0][1][1][0][RTW89_MKK][35] = 127,
+	[0][1][1][0][RTW89_IC][35] = 127,
+	[0][1][1][0][RTW89_KCC][35] = 127,
+	[0][1][1][0][RTW89_ACMA][35] = 127,
+	[0][1][1][0][RTW89_CN][35] = 127,
+	[0][1][1][0][RTW89_UK][35] = 127,
+	[0][1][1][0][RTW89_FCC][37] = 127,
+	[0][1][1][0][RTW89_ETSI][37] = 127,
+	[0][1][1][0][RTW89_MKK][37] = 127,
+	[0][1][1][0][RTW89_IC][37] = 127,
+	[0][1][1][0][RTW89_KCC][37] = 127,
+	[0][1][1][0][RTW89_ACMA][37] = 127,
+	[0][1][1][0][RTW89_CN][37] = 127,
+	[0][1][1][0][RTW89_UK][37] = 127,
+	[0][1][1][0][RTW89_FCC][38] = 127,
+	[0][1][1][0][RTW89_ETSI][38] = 127,
+	[0][1][1][0][RTW89_MKK][38] = 127,
+	[0][1][1][0][RTW89_IC][38] = 127,
+	[0][1][1][0][RTW89_KCC][38] = 127,
+	[0][1][1][0][RTW89_ACMA][38] = 127,
+	[0][1][1][0][RTW89_CN][38] = 127,
+	[0][1][1][0][RTW89_UK][38] = 127,
+	[0][1][1][0][RTW89_FCC][40] = 127,
+	[0][1][1][0][RTW89_ETSI][40] = 127,
+	[0][1][1][0][RTW89_MKK][40] = 127,
+	[0][1][1][0][RTW89_IC][40] = 127,
+	[0][1][1][0][RTW89_KCC][40] = 127,
+	[0][1][1][0][RTW89_ACMA][40] = 127,
+	[0][1][1][0][RTW89_CN][40] = 127,
+	[0][1][1][0][RTW89_UK][40] = 127,
+	[0][1][1][0][RTW89_FCC][42] = 127,
+	[0][1][1][0][RTW89_ETSI][42] = 127,
+	[0][1][1][0][RTW89_MKK][42] = 127,
+	[0][1][1][0][RTW89_IC][42] = 127,
+	[0][1][1][0][RTW89_KCC][42] = 127,
+	[0][1][1][0][RTW89_ACMA][42] = 127,
+	[0][1][1][0][RTW89_CN][42] = 127,
+	[0][1][1][0][RTW89_UK][42] = 127,
+	[0][1][1][0][RTW89_FCC][44] = 127,
+	[0][1][1][0][RTW89_ETSI][44] = 127,
+	[0][1][1][0][RTW89_MKK][44] = 127,
+	[0][1][1][0][RTW89_IC][44] = 127,
+	[0][1][1][0][RTW89_KCC][44] = 127,
+	[0][1][1][0][RTW89_ACMA][44] = 127,
+	[0][1][1][0][RTW89_CN][44] = 127,
+	[0][1][1][0][RTW89_UK][44] = 127,
+	[0][1][1][0][RTW89_FCC][46] = 127,
+	[0][1][1][0][RTW89_ETSI][46] = 127,
+	[0][1][1][0][RTW89_MKK][46] = 127,
+	[0][1][1][0][RTW89_IC][46] = 127,
+	[0][1][1][0][RTW89_KCC][46] = 127,
+	[0][1][1][0][RTW89_ACMA][46] = 127,
+	[0][1][1][0][RTW89_CN][46] = 127,
+	[0][1][1][0][RTW89_UK][46] = 127,
+	[0][1][1][0][RTW89_FCC][48] = 127,
+	[0][1][1][0][RTW89_ETSI][48] = 127,
+	[0][1][1][0][RTW89_MKK][48] = 127,
+	[0][1][1][0][RTW89_IC][48] = 127,
+	[0][1][1][0][RTW89_KCC][48] = 127,
+	[0][1][1][0][RTW89_ACMA][48] = 127,
+	[0][1][1][0][RTW89_CN][48] = 127,
+	[0][1][1][0][RTW89_UK][48] = 127,
+	[0][1][1][0][RTW89_FCC][50] = 127,
+	[0][1][1][0][RTW89_ETSI][50] = 127,
+	[0][1][1][0][RTW89_MKK][50] = 127,
+	[0][1][1][0][RTW89_IC][50] = 127,
+	[0][1][1][0][RTW89_KCC][50] = 127,
+	[0][1][1][0][RTW89_ACMA][50] = 127,
+	[0][1][1][0][RTW89_CN][50] = 127,
+	[0][1][1][0][RTW89_UK][50] = 127,
+	[0][1][1][0][RTW89_FCC][52] = 127,
+	[0][1][1][0][RTW89_ETSI][52] = 127,
+	[0][1][1][0][RTW89_MKK][52] = 127,
+	[0][1][1][0][RTW89_IC][52] = 127,
+	[0][1][1][0][RTW89_KCC][52] = 127,
+	[0][1][1][0][RTW89_ACMA][52] = 127,
+	[0][1][1][0][RTW89_CN][52] = 127,
+	[0][1][1][0][RTW89_UK][52] = 127,
+	[0][0][2][0][RTW89_FCC][0] = 78,
+	[0][0][2][0][RTW89_ETSI][0] = 62,
+	[0][0][2][0][RTW89_MKK][0] = 62,
+	[0][0][2][0][RTW89_IC][0] = 64,
+	[0][0][2][0][RTW89_KCC][0] = 76,
+	[0][0][2][0][RTW89_ACMA][0] = 62,
+	[0][0][2][0][RTW89_CN][0] = 62,
+	[0][0][2][0][RTW89_UK][0] = 62,
+	[0][0][2][0][RTW89_FCC][2] = 82,
+	[0][0][2][0][RTW89_ETSI][2] = 62,
+	[0][0][2][0][RTW89_MKK][2] = 62,
+	[0][0][2][0][RTW89_IC][2] = 64,
+	[0][0][2][0][RTW89_KCC][2] = 76,
+	[0][0][2][0][RTW89_ACMA][2] = 62,
+	[0][0][2][0][RTW89_CN][2] = 62,
+	[0][0][2][0][RTW89_UK][2] = 62,
+	[0][0][2][0][RTW89_FCC][4] = 82,
+	[0][0][2][0][RTW89_ETSI][4] = 62,
+	[0][0][2][0][RTW89_MKK][4] = 62,
+	[0][0][2][0][RTW89_IC][4] = 64,
+	[0][0][2][0][RTW89_KCC][4] = 76,
+	[0][0][2][0][RTW89_ACMA][4] = 62,
+	[0][0][2][0][RTW89_CN][4] = 62,
+	[0][0][2][0][RTW89_UK][4] = 62,
+	[0][0][2][0][RTW89_FCC][6] = 82,
+	[0][0][2][0][RTW89_ETSI][6] = 62,
+	[0][0][2][0][RTW89_MKK][6] = 62,
+	[0][0][2][0][RTW89_IC][6] = 64,
+	[0][0][2][0][RTW89_KCC][6] = 54,
+	[0][0][2][0][RTW89_ACMA][6] = 62,
+	[0][0][2][0][RTW89_CN][6] = 62,
+	[0][0][2][0][RTW89_UK][6] = 62,
+	[0][0][2][0][RTW89_FCC][8] = 82,
+	[0][0][2][0][RTW89_ETSI][8] = 62,
+	[0][0][2][0][RTW89_MKK][8] = 62,
+	[0][0][2][0][RTW89_IC][8] = 64,
+	[0][0][2][0][RTW89_KCC][8] = 76,
+	[0][0][2][0][RTW89_ACMA][8] = 62,
+	[0][0][2][0][RTW89_CN][8] = 62,
+	[0][0][2][0][RTW89_UK][8] = 62,
+	[0][0][2][0][RTW89_FCC][10] = 82,
+	[0][0][2][0][RTW89_ETSI][10] = 62,
+	[0][0][2][0][RTW89_MKK][10] = 62,
+	[0][0][2][0][RTW89_IC][10] = 64,
+	[0][0][2][0][RTW89_KCC][10] = 76,
+	[0][0][2][0][RTW89_ACMA][10] = 62,
+	[0][0][2][0][RTW89_CN][10] = 62,
+	[0][0][2][0][RTW89_UK][10] = 62,
+	[0][0][2][0][RTW89_FCC][12] = 82,
+	[0][0][2][0][RTW89_ETSI][12] = 62,
+	[0][0][2][0][RTW89_MKK][12] = 62,
+	[0][0][2][0][RTW89_IC][12] = 64,
+	[0][0][2][0][RTW89_KCC][12] = 78,
+	[0][0][2][0][RTW89_ACMA][12] = 62,
+	[0][0][2][0][RTW89_CN][12] = 62,
+	[0][0][2][0][RTW89_UK][12] = 62,
+	[0][0][2][0][RTW89_FCC][14] = 76,
+	[0][0][2][0][RTW89_ETSI][14] = 62,
+	[0][0][2][0][RTW89_MKK][14] = 62,
+	[0][0][2][0][RTW89_IC][14] = 64,
+	[0][0][2][0][RTW89_KCC][14] = 78,
+	[0][0][2][0][RTW89_ACMA][14] = 62,
+	[0][0][2][0][RTW89_CN][14] = 62,
+	[0][0][2][0][RTW89_UK][14] = 62,
+	[0][0][2][0][RTW89_FCC][15] = 76,
+	[0][0][2][0][RTW89_ETSI][15] = 60,
+	[0][0][2][0][RTW89_MKK][15] = 78,
+	[0][0][2][0][RTW89_IC][15] = 76,
+	[0][0][2][0][RTW89_KCC][15] = 78,
+	[0][0][2][0][RTW89_ACMA][15] = 60,
+	[0][0][2][0][RTW89_CN][15] = 127,
+	[0][0][2][0][RTW89_UK][15] = 60,
+	[0][0][2][0][RTW89_FCC][17] = 82,
+	[0][0][2][0][RTW89_ETSI][17] = 62,
+	[0][0][2][0][RTW89_MKK][17] = 78,
+	[0][0][2][0][RTW89_IC][17] = 82,
+	[0][0][2][0][RTW89_KCC][17] = 78,
+	[0][0][2][0][RTW89_ACMA][17] = 62,
+	[0][0][2][0][RTW89_CN][17] = 127,
+	[0][0][2][0][RTW89_UK][17] = 62,
+	[0][0][2][0][RTW89_FCC][19] = 82,
+	[0][0][2][0][RTW89_ETSI][19] = 62,
+	[0][0][2][0][RTW89_MKK][19] = 78,
+	[0][0][2][0][RTW89_IC][19] = 82,
+	[0][0][2][0][RTW89_KCC][19] = 78,
+	[0][0][2][0][RTW89_ACMA][19] = 62,
+	[0][0][2][0][RTW89_CN][19] = 127,
+	[0][0][2][0][RTW89_UK][19] = 62,
+	[0][0][2][0][RTW89_FCC][21] = 82,
+	[0][0][2][0][RTW89_ETSI][21] = 62,
+	[0][0][2][0][RTW89_MKK][21] = 78,
+	[0][0][2][0][RTW89_IC][21] = 82,
+	[0][0][2][0][RTW89_KCC][21] = 78,
+	[0][0][2][0][RTW89_ACMA][21] = 62,
+	[0][0][2][0][RTW89_CN][21] = 127,
+	[0][0][2][0][RTW89_UK][21] = 62,
+	[0][0][2][0][RTW89_FCC][23] = 82,
+	[0][0][2][0][RTW89_ETSI][23] = 62,
+	[0][0][2][0][RTW89_MKK][23] = 78,
+	[0][0][2][0][RTW89_IC][23] = 82,
+	[0][0][2][0][RTW89_KCC][23] = 78,
+	[0][0][2][0][RTW89_ACMA][23] = 62,
+	[0][0][2][0][RTW89_CN][23] = 127,
+	[0][0][2][0][RTW89_UK][23] = 62,
+	[0][0][2][0][RTW89_FCC][25] = 82,
+	[0][0][2][0][RTW89_ETSI][25] = 62,
+	[0][0][2][0][RTW89_MKK][25] = 78,
+	[0][0][2][0][RTW89_IC][25] = 127,
+	[0][0][2][0][RTW89_KCC][25] = 78,
+	[0][0][2][0][RTW89_ACMA][25] = 127,
+	[0][0][2][0][RTW89_CN][25] = 127,
+	[0][0][2][0][RTW89_UK][25] = 62,
+	[0][0][2][0][RTW89_FCC][27] = 82,
+	[0][0][2][0][RTW89_ETSI][27] = 62,
+	[0][0][2][0][RTW89_MKK][27] = 78,
+	[0][0][2][0][RTW89_IC][27] = 127,
+	[0][0][2][0][RTW89_KCC][27] = 78,
+	[0][0][2][0][RTW89_ACMA][27] = 127,
+	[0][0][2][0][RTW89_CN][27] = 127,
+	[0][0][2][0][RTW89_UK][27] = 62,
+	[0][0][2][0][RTW89_FCC][29] = 82,
+	[0][0][2][0][RTW89_ETSI][29] = 62,
+	[0][0][2][0][RTW89_MKK][29] = 78,
+	[0][0][2][0][RTW89_IC][29] = 127,
+	[0][0][2][0][RTW89_KCC][29] = 78,
+	[0][0][2][0][RTW89_ACMA][29] = 127,
+	[0][0][2][0][RTW89_CN][29] = 127,
+	[0][0][2][0][RTW89_UK][29] = 62,
+	[0][0][2][0][RTW89_FCC][31] = 82,
+	[0][0][2][0][RTW89_ETSI][31] = 62,
+	[0][0][2][0][RTW89_MKK][31] = 78,
+	[0][0][2][0][RTW89_IC][31] = 82,
+	[0][0][2][0][RTW89_KCC][31] = 74,
+	[0][0][2][0][RTW89_ACMA][31] = 62,
+	[0][0][2][0][RTW89_CN][31] = 127,
+	[0][0][2][0][RTW89_UK][31] = 62,
+	[0][0][2][0][RTW89_FCC][33] = 82,
+	[0][0][2][0][RTW89_ETSI][33] = 62,
+	[0][0][2][0][RTW89_MKK][33] = 78,
+	[0][0][2][0][RTW89_IC][33] = 82,
+	[0][0][2][0][RTW89_KCC][33] = 74,
+	[0][0][2][0][RTW89_ACMA][33] = 62,
+	[0][0][2][0][RTW89_CN][33] = 127,
+	[0][0][2][0][RTW89_UK][33] = 62,
+	[0][0][2][0][RTW89_FCC][35] = 72,
+	[0][0][2][0][RTW89_ETSI][35] = 62,
+	[0][0][2][0][RTW89_MKK][35] = 78,
+	[0][0][2][0][RTW89_IC][35] = 72,
+	[0][0][2][0][RTW89_KCC][35] = 74,
+	[0][0][2][0][RTW89_ACMA][35] = 62,
+	[0][0][2][0][RTW89_CN][35] = 127,
+	[0][0][2][0][RTW89_UK][35] = 62,
+	[0][0][2][0][RTW89_FCC][37] = 82,
+	[0][0][2][0][RTW89_ETSI][37] = 127,
+	[0][0][2][0][RTW89_MKK][37] = 78,
+	[0][0][2][0][RTW89_IC][37] = 82,
+	[0][0][2][0][RTW89_KCC][37] = 76,
+	[0][0][2][0][RTW89_ACMA][37] = 78,
+	[0][0][2][0][RTW89_CN][37] = 127,
+	[0][0][2][0][RTW89_UK][37] = 78,
+	[0][0][2][0][RTW89_FCC][38] = 82,
+	[0][0][2][0][RTW89_ETSI][38] = 30,
+	[0][0][2][0][RTW89_MKK][38] = 127,
+	[0][0][2][0][RTW89_IC][38] = 82,
+	[0][0][2][0][RTW89_KCC][38] = 66,
+	[0][0][2][0][RTW89_ACMA][38] = 78,
+	[0][0][2][0][RTW89_CN][38] = 78,
+	[0][0][2][0][RTW89_UK][38] = 60,
+	[0][0][2][0][RTW89_FCC][40] = 82,
+	[0][0][2][0][RTW89_ETSI][40] = 30,
+	[0][0][2][0][RTW89_MKK][40] = 127,
+	[0][0][2][0][RTW89_IC][40] = 82,
+	[0][0][2][0][RTW89_KCC][40] = 74,
+	[0][0][2][0][RTW89_ACMA][40] = 78,
+	[0][0][2][0][RTW89_CN][40] = 78,
+	[0][0][2][0][RTW89_UK][40] = 60,
+	[0][0][2][0][RTW89_FCC][42] = 82,
+	[0][0][2][0][RTW89_ETSI][42] = 30,
+	[0][0][2][0][RTW89_MKK][42] = 127,
+	[0][0][2][0][RTW89_IC][42] = 82,
+	[0][0][2][0][RTW89_KCC][42] = 74,
+	[0][0][2][0][RTW89_ACMA][42] = 78,
+	[0][0][2][0][RTW89_CN][42] = 78,
+	[0][0][2][0][RTW89_UK][42] = 60,
+	[0][0][2][0][RTW89_FCC][44] = 82,
+	[0][0][2][0][RTW89_ETSI][44] = 30,
+	[0][0][2][0][RTW89_MKK][44] = 127,
+	[0][0][2][0][RTW89_IC][44] = 82,
+	[0][0][2][0][RTW89_KCC][44] = 74,
+	[0][0][2][0][RTW89_ACMA][44] = 78,
+	[0][0][2][0][RTW89_CN][44] = 78,
+	[0][0][2][0][RTW89_UK][44] = 60,
+	[0][0][2][0][RTW89_FCC][46] = 82,
+	[0][0][2][0][RTW89_ETSI][46] = 30,
+	[0][0][2][0][RTW89_MKK][46] = 127,
+	[0][0][2][0][RTW89_IC][46] = 82,
+	[0][0][2][0][RTW89_KCC][46] = 74,
+	[0][0][2][0][RTW89_ACMA][46] = 78,
+	[0][0][2][0][RTW89_CN][46] = 78,
+	[0][0][2][0][RTW89_UK][46] = 60,
+	[0][0][2][0][RTW89_FCC][48] = 74,
+	[0][0][2][0][RTW89_ETSI][48] = 127,
+	[0][0][2][0][RTW89_MKK][48] = 127,
+	[0][0][2][0][RTW89_IC][48] = 127,
+	[0][0][2][0][RTW89_KCC][48] = 127,
+	[0][0][2][0][RTW89_ACMA][48] = 127,
+	[0][0][2][0][RTW89_CN][48] = 127,
+	[0][0][2][0][RTW89_UK][48] = 127,
+	[0][0][2][0][RTW89_FCC][50] = 76,
+	[0][0][2][0][RTW89_ETSI][50] = 127,
+	[0][0][2][0][RTW89_MKK][50] = 127,
+	[0][0][2][0][RTW89_IC][50] = 127,
+	[0][0][2][0][RTW89_KCC][50] = 127,
+	[0][0][2][0][RTW89_ACMA][50] = 127,
+	[0][0][2][0][RTW89_CN][50] = 127,
+	[0][0][2][0][RTW89_UK][50] = 127,
+	[0][0][2][0][RTW89_FCC][52] = 76,
+	[0][0][2][0][RTW89_ETSI][52] = 127,
+	[0][0][2][0][RTW89_MKK][52] = 127,
+	[0][0][2][0][RTW89_IC][52] = 127,
+	[0][0][2][0][RTW89_KCC][52] = 127,
+	[0][0][2][0][RTW89_ACMA][52] = 127,
+	[0][0][2][0][RTW89_CN][52] = 127,
+	[0][0][2][0][RTW89_UK][52] = 127,
+	[0][1][2][0][RTW89_FCC][0] = 127,
+	[0][1][2][0][RTW89_ETSI][0] = 127,
+	[0][1][2][0][RTW89_MKK][0] = 127,
+	[0][1][2][0][RTW89_IC][0] = 127,
+	[0][1][2][0][RTW89_KCC][0] = 127,
+	[0][1][2][0][RTW89_ACMA][0] = 127,
+	[0][1][2][0][RTW89_CN][0] = 127,
+	[0][1][2][0][RTW89_UK][0] = 127,
+	[0][1][2][0][RTW89_FCC][2] = 127,
+	[0][1][2][0][RTW89_ETSI][2] = 127,
+	[0][1][2][0][RTW89_MKK][2] = 127,
+	[0][1][2][0][RTW89_IC][2] = 127,
+	[0][1][2][0][RTW89_KCC][2] = 127,
+	[0][1][2][0][RTW89_ACMA][2] = 127,
+	[0][1][2][0][RTW89_CN][2] = 127,
+	[0][1][2][0][RTW89_UK][2] = 127,
+	[0][1][2][0][RTW89_FCC][4] = 127,
+	[0][1][2][0][RTW89_ETSI][4] = 127,
+	[0][1][2][0][RTW89_MKK][4] = 127,
+	[0][1][2][0][RTW89_IC][4] = 127,
+	[0][1][2][0][RTW89_KCC][4] = 127,
+	[0][1][2][0][RTW89_ACMA][4] = 127,
+	[0][1][2][0][RTW89_CN][4] = 127,
+	[0][1][2][0][RTW89_UK][4] = 127,
+	[0][1][2][0][RTW89_FCC][6] = 127,
+	[0][1][2][0][RTW89_ETSI][6] = 127,
+	[0][1][2][0][RTW89_MKK][6] = 127,
+	[0][1][2][0][RTW89_IC][6] = 127,
+	[0][1][2][0][RTW89_KCC][6] = 127,
+	[0][1][2][0][RTW89_ACMA][6] = 127,
+	[0][1][2][0][RTW89_CN][6] = 127,
+	[0][1][2][0][RTW89_UK][6] = 127,
+	[0][1][2][0][RTW89_FCC][8] = 127,
+	[0][1][2][0][RTW89_ETSI][8] = 127,
+	[0][1][2][0][RTW89_MKK][8] = 127,
+	[0][1][2][0][RTW89_IC][8] = 127,
+	[0][1][2][0][RTW89_KCC][8] = 127,
+	[0][1][2][0][RTW89_ACMA][8] = 127,
+	[0][1][2][0][RTW89_CN][8] = 127,
+	[0][1][2][0][RTW89_UK][8] = 127,
+	[0][1][2][0][RTW89_FCC][10] = 127,
+	[0][1][2][0][RTW89_ETSI][10] = 127,
+	[0][1][2][0][RTW89_MKK][10] = 127,
+	[0][1][2][0][RTW89_IC][10] = 127,
+	[0][1][2][0][RTW89_KCC][10] = 127,
+	[0][1][2][0][RTW89_ACMA][10] = 127,
+	[0][1][2][0][RTW89_CN][10] = 127,
+	[0][1][2][0][RTW89_UK][10] = 127,
+	[0][1][2][0][RTW89_FCC][12] = 127,
+	[0][1][2][0][RTW89_ETSI][12] = 127,
+	[0][1][2][0][RTW89_MKK][12] = 127,
+	[0][1][2][0][RTW89_IC][12] = 127,
+	[0][1][2][0][RTW89_KCC][12] = 127,
+	[0][1][2][0][RTW89_ACMA][12] = 127,
+	[0][1][2][0][RTW89_CN][12] = 127,
+	[0][1][2][0][RTW89_UK][12] = 127,
+	[0][1][2][0][RTW89_FCC][14] = 127,
+	[0][1][2][0][RTW89_ETSI][14] = 127,
+	[0][1][2][0][RTW89_MKK][14] = 127,
+	[0][1][2][0][RTW89_IC][14] = 127,
+	[0][1][2][0][RTW89_KCC][14] = 127,
+	[0][1][2][0][RTW89_ACMA][14] = 127,
+	[0][1][2][0][RTW89_CN][14] = 127,
+	[0][1][2][0][RTW89_UK][14] = 127,
+	[0][1][2][0][RTW89_FCC][15] = 127,
+	[0][1][2][0][RTW89_ETSI][15] = 127,
+	[0][1][2][0][RTW89_MKK][15] = 127,
+	[0][1][2][0][RTW89_IC][15] = 127,
+	[0][1][2][0][RTW89_KCC][15] = 127,
+	[0][1][2][0][RTW89_ACMA][15] = 127,
+	[0][1][2][0][RTW89_CN][15] = 127,
+	[0][1][2][0][RTW89_UK][15] = 127,
+	[0][1][2][0][RTW89_FCC][17] = 127,
+	[0][1][2][0][RTW89_ETSI][17] = 127,
+	[0][1][2][0][RTW89_MKK][17] = 127,
+	[0][1][2][0][RTW89_IC][17] = 127,
+	[0][1][2][0][RTW89_KCC][17] = 127,
+	[0][1][2][0][RTW89_ACMA][17] = 127,
+	[0][1][2][0][RTW89_CN][17] = 127,
+	[0][1][2][0][RTW89_UK][17] = 127,
+	[0][1][2][0][RTW89_FCC][19] = 127,
+	[0][1][2][0][RTW89_ETSI][19] = 127,
+	[0][1][2][0][RTW89_MKK][19] = 127,
+	[0][1][2][0][RTW89_IC][19] = 127,
+	[0][1][2][0][RTW89_KCC][19] = 127,
+	[0][1][2][0][RTW89_ACMA][19] = 127,
+	[0][1][2][0][RTW89_CN][19] = 127,
+	[0][1][2][0][RTW89_UK][19] = 127,
+	[0][1][2][0][RTW89_FCC][21] = 127,
+	[0][1][2][0][RTW89_ETSI][21] = 127,
+	[0][1][2][0][RTW89_MKK][21] = 127,
+	[0][1][2][0][RTW89_IC][21] = 127,
+	[0][1][2][0][RTW89_KCC][21] = 127,
+	[0][1][2][0][RTW89_ACMA][21] = 127,
+	[0][1][2][0][RTW89_CN][21] = 127,
+	[0][1][2][0][RTW89_UK][21] = 127,
+	[0][1][2][0][RTW89_FCC][23] = 127,
+	[0][1][2][0][RTW89_ETSI][23] = 127,
+	[0][1][2][0][RTW89_MKK][23] = 127,
+	[0][1][2][0][RTW89_IC][23] = 127,
+	[0][1][2][0][RTW89_KCC][23] = 127,
+	[0][1][2][0][RTW89_ACMA][23] = 127,
+	[0][1][2][0][RTW89_CN][23] = 127,
+	[0][1][2][0][RTW89_UK][23] = 127,
+	[0][1][2][0][RTW89_FCC][25] = 127,
+	[0][1][2][0][RTW89_ETSI][25] = 127,
+	[0][1][2][0][RTW89_MKK][25] = 127,
+	[0][1][2][0][RTW89_IC][25] = 127,
+	[0][1][2][0][RTW89_KCC][25] = 127,
+	[0][1][2][0][RTW89_ACMA][25] = 127,
+	[0][1][2][0][RTW89_CN][25] = 127,
+	[0][1][2][0][RTW89_UK][25] = 127,
+	[0][1][2][0][RTW89_FCC][27] = 127,
+	[0][1][2][0][RTW89_ETSI][27] = 127,
+	[0][1][2][0][RTW89_MKK][27] = 127,
+	[0][1][2][0][RTW89_IC][27] = 127,
+	[0][1][2][0][RTW89_KCC][27] = 127,
+	[0][1][2][0][RTW89_ACMA][27] = 127,
+	[0][1][2][0][RTW89_CN][27] = 127,
+	[0][1][2][0][RTW89_UK][27] = 127,
+	[0][1][2][0][RTW89_FCC][29] = 127,
+	[0][1][2][0][RTW89_ETSI][29] = 127,
+	[0][1][2][0][RTW89_MKK][29] = 127,
+	[0][1][2][0][RTW89_IC][29] = 127,
+	[0][1][2][0][RTW89_KCC][29] = 127,
+	[0][1][2][0][RTW89_ACMA][29] = 127,
+	[0][1][2][0][RTW89_CN][29] = 127,
+	[0][1][2][0][RTW89_UK][29] = 127,
+	[0][1][2][0][RTW89_FCC][31] = 127,
+	[0][1][2][0][RTW89_ETSI][31] = 127,
+	[0][1][2][0][RTW89_MKK][31] = 127,
+	[0][1][2][0][RTW89_IC][31] = 127,
+	[0][1][2][0][RTW89_KCC][31] = 127,
+	[0][1][2][0][RTW89_ACMA][31] = 127,
+	[0][1][2][0][RTW89_CN][31] = 127,
+	[0][1][2][0][RTW89_UK][31] = 127,
+	[0][1][2][0][RTW89_FCC][33] = 127,
+	[0][1][2][0][RTW89_ETSI][33] = 127,
+	[0][1][2][0][RTW89_MKK][33] = 127,
+	[0][1][2][0][RTW89_IC][33] = 127,
+	[0][1][2][0][RTW89_KCC][33] = 127,
+	[0][1][2][0][RTW89_ACMA][33] = 127,
+	[0][1][2][0][RTW89_CN][33] = 127,
+	[0][1][2][0][RTW89_UK][33] = 127,
+	[0][1][2][0][RTW89_FCC][35] = 127,
+	[0][1][2][0][RTW89_ETSI][35] = 127,
+	[0][1][2][0][RTW89_MKK][35] = 127,
+	[0][1][2][0][RTW89_IC][35] = 127,
+	[0][1][2][0][RTW89_KCC][35] = 127,
+	[0][1][2][0][RTW89_ACMA][35] = 127,
+	[0][1][2][0][RTW89_CN][35] = 127,
+	[0][1][2][0][RTW89_UK][35] = 127,
+	[0][1][2][0][RTW89_FCC][37] = 127,
+	[0][1][2][0][RTW89_ETSI][37] = 127,
+	[0][1][2][0][RTW89_MKK][37] = 127,
+	[0][1][2][0][RTW89_IC][37] = 127,
+	[0][1][2][0][RTW89_KCC][37] = 127,
+	[0][1][2][0][RTW89_ACMA][37] = 127,
+	[0][1][2][0][RTW89_CN][37] = 127,
+	[0][1][2][0][RTW89_UK][37] = 127,
+	[0][1][2][0][RTW89_FCC][38] = 127,
+	[0][1][2][0][RTW89_ETSI][38] = 127,
+	[0][1][2][0][RTW89_MKK][38] = 127,
+	[0][1][2][0][RTW89_IC][38] = 127,
+	[0][1][2][0][RTW89_KCC][38] = 127,
+	[0][1][2][0][RTW89_ACMA][38] = 127,
+	[0][1][2][0][RTW89_CN][38] = 127,
+	[0][1][2][0][RTW89_UK][38] = 127,
+	[0][1][2][0][RTW89_FCC][40] = 127,
+	[0][1][2][0][RTW89_ETSI][40] = 127,
+	[0][1][2][0][RTW89_MKK][40] = 127,
+	[0][1][2][0][RTW89_IC][40] = 127,
+	[0][1][2][0][RTW89_KCC][40] = 127,
+	[0][1][2][0][RTW89_ACMA][40] = 127,
+	[0][1][2][0][RTW89_CN][40] = 127,
+	[0][1][2][0][RTW89_UK][40] = 127,
+	[0][1][2][0][RTW89_FCC][42] = 127,
+	[0][1][2][0][RTW89_ETSI][42] = 127,
+	[0][1][2][0][RTW89_MKK][42] = 127,
+	[0][1][2][0][RTW89_IC][42] = 127,
+	[0][1][2][0][RTW89_KCC][42] = 127,
+	[0][1][2][0][RTW89_ACMA][42] = 127,
+	[0][1][2][0][RTW89_CN][42] = 127,
+	[0][1][2][0][RTW89_UK][42] = 127,
+	[0][1][2][0][RTW89_FCC][44] = 127,
+	[0][1][2][0][RTW89_ETSI][44] = 127,
+	[0][1][2][0][RTW89_MKK][44] = 127,
+	[0][1][2][0][RTW89_IC][44] = 127,
+	[0][1][2][0][RTW89_KCC][44] = 127,
+	[0][1][2][0][RTW89_ACMA][44] = 127,
+	[0][1][2][0][RTW89_CN][44] = 127,
+	[0][1][2][0][RTW89_UK][44] = 127,
+	[0][1][2][0][RTW89_FCC][46] = 127,
+	[0][1][2][0][RTW89_ETSI][46] = 127,
+	[0][1][2][0][RTW89_MKK][46] = 127,
+	[0][1][2][0][RTW89_IC][46] = 127,
+	[0][1][2][0][RTW89_KCC][46] = 127,
+	[0][1][2][0][RTW89_ACMA][46] = 127,
+	[0][1][2][0][RTW89_CN][46] = 127,
+	[0][1][2][0][RTW89_UK][46] = 127,
+	[0][1][2][0][RTW89_FCC][48] = 127,
+	[0][1][2][0][RTW89_ETSI][48] = 127,
+	[0][1][2][0][RTW89_MKK][48] = 127,
+	[0][1][2][0][RTW89_IC][48] = 127,
+	[0][1][2][0][RTW89_KCC][48] = 127,
+	[0][1][2][0][RTW89_ACMA][48] = 127,
+	[0][1][2][0][RTW89_CN][48] = 127,
+	[0][1][2][0][RTW89_UK][48] = 127,
+	[0][1][2][0][RTW89_FCC][50] = 127,
+	[0][1][2][0][RTW89_ETSI][50] = 127,
+	[0][1][2][0][RTW89_MKK][50] = 127,
+	[0][1][2][0][RTW89_IC][50] = 127,
+	[0][1][2][0][RTW89_KCC][50] = 127,
+	[0][1][2][0][RTW89_ACMA][50] = 127,
+	[0][1][2][0][RTW89_CN][50] = 127,
+	[0][1][2][0][RTW89_UK][50] = 127,
+	[0][1][2][0][RTW89_FCC][52] = 127,
+	[0][1][2][0][RTW89_ETSI][52] = 127,
+	[0][1][2][0][RTW89_MKK][52] = 127,
+	[0][1][2][0][RTW89_IC][52] = 127,
+	[0][1][2][0][RTW89_KCC][52] = 127,
+	[0][1][2][0][RTW89_ACMA][52] = 127,
+	[0][1][2][0][RTW89_CN][52] = 127,
+	[0][1][2][0][RTW89_UK][52] = 127,
+	[0][1][2][1][RTW89_FCC][0] = 127,
+	[0][1][2][1][RTW89_ETSI][0] = 127,
+	[0][1][2][1][RTW89_MKK][0] = 127,
+	[0][1][2][1][RTW89_IC][0] = 127,
+	[0][1][2][1][RTW89_KCC][0] = 127,
+	[0][1][2][1][RTW89_ACMA][0] = 127,
+	[0][1][2][1][RTW89_CN][0] = 127,
+	[0][1][2][1][RTW89_UK][0] = 127,
+	[0][1][2][1][RTW89_FCC][2] = 127,
+	[0][1][2][1][RTW89_ETSI][2] = 127,
+	[0][1][2][1][RTW89_MKK][2] = 127,
+	[0][1][2][1][RTW89_IC][2] = 127,
+	[0][1][2][1][RTW89_KCC][2] = 127,
+	[0][1][2][1][RTW89_ACMA][2] = 127,
+	[0][1][2][1][RTW89_CN][2] = 127,
+	[0][1][2][1][RTW89_UK][2] = 127,
+	[0][1][2][1][RTW89_FCC][4] = 127,
+	[0][1][2][1][RTW89_ETSI][4] = 127,
+	[0][1][2][1][RTW89_MKK][4] = 127,
+	[0][1][2][1][RTW89_IC][4] = 127,
+	[0][1][2][1][RTW89_KCC][4] = 127,
+	[0][1][2][1][RTW89_ACMA][4] = 127,
+	[0][1][2][1][RTW89_CN][4] = 127,
+	[0][1][2][1][RTW89_UK][4] = 127,
+	[0][1][2][1][RTW89_FCC][6] = 127,
+	[0][1][2][1][RTW89_ETSI][6] = 127,
+	[0][1][2][1][RTW89_MKK][6] = 127,
+	[0][1][2][1][RTW89_IC][6] = 127,
+	[0][1][2][1][RTW89_KCC][6] = 127,
+	[0][1][2][1][RTW89_ACMA][6] = 127,
+	[0][1][2][1][RTW89_CN][6] = 127,
+	[0][1][2][1][RTW89_UK][6] = 127,
+	[0][1][2][1][RTW89_FCC][8] = 127,
+	[0][1][2][1][RTW89_ETSI][8] = 127,
+	[0][1][2][1][RTW89_MKK][8] = 127,
+	[0][1][2][1][RTW89_IC][8] = 127,
+	[0][1][2][1][RTW89_KCC][8] = 127,
+	[0][1][2][1][RTW89_ACMA][8] = 127,
+	[0][1][2][1][RTW89_CN][8] = 127,
+	[0][1][2][1][RTW89_UK][8] = 127,
+	[0][1][2][1][RTW89_FCC][10] = 127,
+	[0][1][2][1][RTW89_ETSI][10] = 127,
+	[0][1][2][1][RTW89_MKK][10] = 127,
+	[0][1][2][1][RTW89_IC][10] = 127,
+	[0][1][2][1][RTW89_KCC][10] = 127,
+	[0][1][2][1][RTW89_ACMA][10] = 127,
+	[0][1][2][1][RTW89_CN][10] = 127,
+	[0][1][2][1][RTW89_UK][10] = 127,
+	[0][1][2][1][RTW89_FCC][12] = 127,
+	[0][1][2][1][RTW89_ETSI][12] = 127,
+	[0][1][2][1][RTW89_MKK][12] = 127,
+	[0][1][2][1][RTW89_IC][12] = 127,
+	[0][1][2][1][RTW89_KCC][12] = 127,
+	[0][1][2][1][RTW89_ACMA][12] = 127,
+	[0][1][2][1][RTW89_CN][12] = 127,
+	[0][1][2][1][RTW89_UK][12] = 127,
+	[0][1][2][1][RTW89_FCC][14] = 127,
+	[0][1][2][1][RTW89_ETSI][14] = 127,
+	[0][1][2][1][RTW89_MKK][14] = 127,
+	[0][1][2][1][RTW89_IC][14] = 127,
+	[0][1][2][1][RTW89_KCC][14] = 127,
+	[0][1][2][1][RTW89_ACMA][14] = 127,
+	[0][1][2][1][RTW89_CN][14] = 127,
+	[0][1][2][1][RTW89_UK][14] = 127,
+	[0][1][2][1][RTW89_FCC][15] = 127,
+	[0][1][2][1][RTW89_ETSI][15] = 127,
+	[0][1][2][1][RTW89_MKK][15] = 127,
+	[0][1][2][1][RTW89_IC][15] = 127,
+	[0][1][2][1][RTW89_KCC][15] = 127,
+	[0][1][2][1][RTW89_ACMA][15] = 127,
+	[0][1][2][1][RTW89_CN][15] = 127,
+	[0][1][2][1][RTW89_UK][15] = 127,
+	[0][1][2][1][RTW89_FCC][17] = 127,
+	[0][1][2][1][RTW89_ETSI][17] = 127,
+	[0][1][2][1][RTW89_MKK][17] = 127,
+	[0][1][2][1][RTW89_IC][17] = 127,
+	[0][1][2][1][RTW89_KCC][17] = 127,
+	[0][1][2][1][RTW89_ACMA][17] = 127,
+	[0][1][2][1][RTW89_CN][17] = 127,
+	[0][1][2][1][RTW89_UK][17] = 127,
+	[0][1][2][1][RTW89_FCC][19] = 127,
+	[0][1][2][1][RTW89_ETSI][19] = 127,
+	[0][1][2][1][RTW89_MKK][19] = 127,
+	[0][1][2][1][RTW89_IC][19] = 127,
+	[0][1][2][1][RTW89_KCC][19] = 127,
+	[0][1][2][1][RTW89_ACMA][19] = 127,
+	[0][1][2][1][RTW89_CN][19] = 127,
+	[0][1][2][1][RTW89_UK][19] = 127,
+	[0][1][2][1][RTW89_FCC][21] = 127,
+	[0][1][2][1][RTW89_ETSI][21] = 127,
+	[0][1][2][1][RTW89_MKK][21] = 127,
+	[0][1][2][1][RTW89_IC][21] = 127,
+	[0][1][2][1][RTW89_KCC][21] = 127,
+	[0][1][2][1][RTW89_ACMA][21] = 127,
+	[0][1][2][1][RTW89_CN][21] = 127,
+	[0][1][2][1][RTW89_UK][21] = 127,
+	[0][1][2][1][RTW89_FCC][23] = 127,
+	[0][1][2][1][RTW89_ETSI][23] = 127,
+	[0][1][2][1][RTW89_MKK][23] = 127,
+	[0][1][2][1][RTW89_IC][23] = 127,
+	[0][1][2][1][RTW89_KCC][23] = 127,
+	[0][1][2][1][RTW89_ACMA][23] = 127,
+	[0][1][2][1][RTW89_CN][23] = 127,
+	[0][1][2][1][RTW89_UK][23] = 127,
+	[0][1][2][1][RTW89_FCC][25] = 127,
+	[0][1][2][1][RTW89_ETSI][25] = 127,
+	[0][1][2][1][RTW89_MKK][25] = 127,
+	[0][1][2][1][RTW89_IC][25] = 127,
+	[0][1][2][1][RTW89_KCC][25] = 127,
+	[0][1][2][1][RTW89_ACMA][25] = 127,
+	[0][1][2][1][RTW89_CN][25] = 127,
+	[0][1][2][1][RTW89_UK][25] = 127,
+	[0][1][2][1][RTW89_FCC][27] = 127,
+	[0][1][2][1][RTW89_ETSI][27] = 127,
+	[0][1][2][1][RTW89_MKK][27] = 127,
+	[0][1][2][1][RTW89_IC][27] = 127,
+	[0][1][2][1][RTW89_KCC][27] = 127,
+	[0][1][2][1][RTW89_ACMA][27] = 127,
+	[0][1][2][1][RTW89_CN][27] = 127,
+	[0][1][2][1][RTW89_UK][27] = 127,
+	[0][1][2][1][RTW89_FCC][29] = 127,
+	[0][1][2][1][RTW89_ETSI][29] = 127,
+	[0][1][2][1][RTW89_MKK][29] = 127,
+	[0][1][2][1][RTW89_IC][29] = 127,
+	[0][1][2][1][RTW89_KCC][29] = 127,
+	[0][1][2][1][RTW89_ACMA][29] = 127,
+	[0][1][2][1][RTW89_CN][29] = 127,
+	[0][1][2][1][RTW89_UK][29] = 127,
+	[0][1][2][1][RTW89_FCC][31] = 127,
+	[0][1][2][1][RTW89_ETSI][31] = 127,
+	[0][1][2][1][RTW89_MKK][31] = 127,
+	[0][1][2][1][RTW89_IC][31] = 127,
+	[0][1][2][1][RTW89_KCC][31] = 127,
+	[0][1][2][1][RTW89_ACMA][31] = 127,
+	[0][1][2][1][RTW89_CN][31] = 127,
+	[0][1][2][1][RTW89_UK][31] = 127,
+	[0][1][2][1][RTW89_FCC][33] = 127,
+	[0][1][2][1][RTW89_ETSI][33] = 127,
+	[0][1][2][1][RTW89_MKK][33] = 127,
+	[0][1][2][1][RTW89_IC][33] = 127,
+	[0][1][2][1][RTW89_KCC][33] = 127,
+	[0][1][2][1][RTW89_ACMA][33] = 127,
+	[0][1][2][1][RTW89_CN][33] = 127,
+	[0][1][2][1][RTW89_UK][33] = 127,
+	[0][1][2][1][RTW89_FCC][35] = 127,
+	[0][1][2][1][RTW89_ETSI][35] = 127,
+	[0][1][2][1][RTW89_MKK][35] = 127,
+	[0][1][2][1][RTW89_IC][35] = 127,
+	[0][1][2][1][RTW89_KCC][35] = 127,
+	[0][1][2][1][RTW89_ACMA][35] = 127,
+	[0][1][2][1][RTW89_CN][35] = 127,
+	[0][1][2][1][RTW89_UK][35] = 127,
+	[0][1][2][1][RTW89_FCC][37] = 127,
+	[0][1][2][1][RTW89_ETSI][37] = 127,
+	[0][1][2][1][RTW89_MKK][37] = 127,
+	[0][1][2][1][RTW89_IC][37] = 127,
+	[0][1][2][1][RTW89_KCC][37] = 127,
+	[0][1][2][1][RTW89_ACMA][37] = 127,
+	[0][1][2][1][RTW89_CN][37] = 127,
+	[0][1][2][1][RTW89_UK][37] = 127,
+	[0][1][2][1][RTW89_FCC][38] = 127,
+	[0][1][2][1][RTW89_ETSI][38] = 127,
+	[0][1][2][1][RTW89_MKK][38] = 127,
+	[0][1][2][1][RTW89_IC][38] = 127,
+	[0][1][2][1][RTW89_KCC][38] = 127,
+	[0][1][2][1][RTW89_ACMA][38] = 127,
+	[0][1][2][1][RTW89_CN][38] = 127,
+	[0][1][2][1][RTW89_UK][38] = 127,
+	[0][1][2][1][RTW89_FCC][40] = 127,
+	[0][1][2][1][RTW89_ETSI][40] = 127,
+	[0][1][2][1][RTW89_MKK][40] = 127,
+	[0][1][2][1][RTW89_IC][40] = 127,
+	[0][1][2][1][RTW89_KCC][40] = 127,
+	[0][1][2][1][RTW89_ACMA][40] = 127,
+	[0][1][2][1][RTW89_CN][40] = 127,
+	[0][1][2][1][RTW89_UK][40] = 127,
+	[0][1][2][1][RTW89_FCC][42] = 127,
+	[0][1][2][1][RTW89_ETSI][42] = 127,
+	[0][1][2][1][RTW89_MKK][42] = 127,
+	[0][1][2][1][RTW89_IC][42] = 127,
+	[0][1][2][1][RTW89_KCC][42] = 127,
+	[0][1][2][1][RTW89_ACMA][42] = 127,
+	[0][1][2][1][RTW89_CN][42] = 127,
+	[0][1][2][1][RTW89_UK][42] = 127,
+	[0][1][2][1][RTW89_FCC][44] = 127,
+	[0][1][2][1][RTW89_ETSI][44] = 127,
+	[0][1][2][1][RTW89_MKK][44] = 127,
+	[0][1][2][1][RTW89_IC][44] = 127,
+	[0][1][2][1][RTW89_KCC][44] = 127,
+	[0][1][2][1][RTW89_ACMA][44] = 127,
+	[0][1][2][1][RTW89_CN][44] = 127,
+	[0][1][2][1][RTW89_UK][44] = 127,
+	[0][1][2][1][RTW89_FCC][46] = 127,
+	[0][1][2][1][RTW89_ETSI][46] = 127,
+	[0][1][2][1][RTW89_MKK][46] = 127,
+	[0][1][2][1][RTW89_IC][46] = 127,
+	[0][1][2][1][RTW89_KCC][46] = 127,
+	[0][1][2][1][RTW89_ACMA][46] = 127,
+	[0][1][2][1][RTW89_CN][46] = 127,
+	[0][1][2][1][RTW89_UK][46] = 127,
+	[0][1][2][1][RTW89_FCC][48] = 127,
+	[0][1][2][1][RTW89_ETSI][48] = 127,
+	[0][1][2][1][RTW89_MKK][48] = 127,
+	[0][1][2][1][RTW89_IC][48] = 127,
+	[0][1][2][1][RTW89_KCC][48] = 127,
+	[0][1][2][1][RTW89_ACMA][48] = 127,
+	[0][1][2][1][RTW89_CN][48] = 127,
+	[0][1][2][1][RTW89_UK][48] = 127,
+	[0][1][2][1][RTW89_FCC][50] = 127,
+	[0][1][2][1][RTW89_ETSI][50] = 127,
+	[0][1][2][1][RTW89_MKK][50] = 127,
+	[0][1][2][1][RTW89_IC][50] = 127,
+	[0][1][2][1][RTW89_KCC][50] = 127,
+	[0][1][2][1][RTW89_ACMA][50] = 127,
+	[0][1][2][1][RTW89_CN][50] = 127,
+	[0][1][2][1][RTW89_UK][50] = 127,
+	[0][1][2][1][RTW89_FCC][52] = 127,
+	[0][1][2][1][RTW89_ETSI][52] = 127,
+	[0][1][2][1][RTW89_MKK][52] = 127,
+	[0][1][2][1][RTW89_IC][52] = 127,
+	[0][1][2][1][RTW89_KCC][52] = 127,
+	[0][1][2][1][RTW89_ACMA][52] = 127,
+	[0][1][2][1][RTW89_CN][52] = 127,
+	[0][1][2][1][RTW89_UK][52] = 127,
+	[1][0][2][0][RTW89_FCC][1] = 68,
+	[1][0][2][0][RTW89_ETSI][1] = 64,
+	[1][0][2][0][RTW89_MKK][1] = 64,
+	[1][0][2][0][RTW89_IC][1] = 64,
+	[1][0][2][0][RTW89_KCC][1] = 74,
+	[1][0][2][0][RTW89_ACMA][1] = 64,
+	[1][0][2][0][RTW89_CN][1] = 64,
+	[1][0][2][0][RTW89_UK][1] = 64,
+	[1][0][2][0][RTW89_FCC][5] = 82,
+	[1][0][2][0][RTW89_ETSI][5] = 64,
+	[1][0][2][0][RTW89_MKK][5] = 62,
+	[1][0][2][0][RTW89_IC][5] = 64,
+	[1][0][2][0][RTW89_KCC][5] = 66,
+	[1][0][2][0][RTW89_ACMA][5] = 64,
+	[1][0][2][0][RTW89_CN][5] = 64,
+	[1][0][2][0][RTW89_UK][5] = 64,
+	[1][0][2][0][RTW89_FCC][9] = 82,
+	[1][0][2][0][RTW89_ETSI][9] = 64,
+	[1][0][2][0][RTW89_MKK][9] = 64,
+	[1][0][2][0][RTW89_IC][9] = 64,
+	[1][0][2][0][RTW89_KCC][9] = 78,
+	[1][0][2][0][RTW89_ACMA][9] = 64,
+	[1][0][2][0][RTW89_CN][9] = 64,
+	[1][0][2][0][RTW89_UK][9] = 64,
+	[1][0][2][0][RTW89_FCC][13] = 66,
+	[1][0][2][0][RTW89_ETSI][13] = 64,
+	[1][0][2][0][RTW89_MKK][13] = 64,
+	[1][0][2][0][RTW89_IC][13] = 64,
+	[1][0][2][0][RTW89_KCC][13] = 72,
+	[1][0][2][0][RTW89_ACMA][13] = 64,
+	[1][0][2][0][RTW89_CN][13] = 64,
+	[1][0][2][0][RTW89_UK][13] = 64,
+	[1][0][2][0][RTW89_FCC][16] = 66,
+	[1][0][2][0][RTW89_ETSI][16] = 66,
+	[1][0][2][0][RTW89_MKK][16] = 80,
+	[1][0][2][0][RTW89_IC][16] = 66,
+	[1][0][2][0][RTW89_KCC][16] = 74,
+	[1][0][2][0][RTW89_ACMA][16] = 66,
+	[1][0][2][0][RTW89_CN][16] = 127,
+	[1][0][2][0][RTW89_UK][16] = 66,
+	[1][0][2][0][RTW89_FCC][20] = 80,
+	[1][0][2][0][RTW89_ETSI][20] = 66,
+	[1][0][2][0][RTW89_MKK][20] = 80,
+	[1][0][2][0][RTW89_IC][20] = 80,
+	[1][0][2][0][RTW89_KCC][20] = 74,
+	[1][0][2][0][RTW89_ACMA][20] = 66,
+	[1][0][2][0][RTW89_CN][20] = 127,
+	[1][0][2][0][RTW89_UK][20] = 66,
+	[1][0][2][0][RTW89_FCC][24] = 80,
+	[1][0][2][0][RTW89_ETSI][24] = 66,
+	[1][0][2][0][RTW89_MKK][24] = 80,
+	[1][0][2][0][RTW89_IC][24] = 127,
+	[1][0][2][0][RTW89_KCC][24] = 74,
+	[1][0][2][0][RTW89_ACMA][24] = 127,
+	[1][0][2][0][RTW89_CN][24] = 127,
+	[1][0][2][0][RTW89_UK][24] = 66,
+	[1][0][2][0][RTW89_FCC][28] = 80,
+	[1][0][2][0][RTW89_ETSI][28] = 66,
+	[1][0][2][0][RTW89_MKK][28] = 80,
+	[1][0][2][0][RTW89_IC][28] = 127,
+	[1][0][2][0][RTW89_KCC][28] = 74,
+	[1][0][2][0][RTW89_ACMA][28] = 127,
+	[1][0][2][0][RTW89_CN][28] = 127,
+	[1][0][2][0][RTW89_UK][28] = 66,
+	[1][0][2][0][RTW89_FCC][32] = 76,
+	[1][0][2][0][RTW89_ETSI][32] = 66,
+	[1][0][2][0][RTW89_MKK][32] = 80,
+	[1][0][2][0][RTW89_IC][32] = 76,
+	[1][0][2][0][RTW89_KCC][32] = 78,
+	[1][0][2][0][RTW89_ACMA][32] = 66,
+	[1][0][2][0][RTW89_CN][32] = 127,
+	[1][0][2][0][RTW89_UK][32] = 66,
+	[1][0][2][0][RTW89_FCC][36] = 80,
+	[1][0][2][0][RTW89_ETSI][36] = 127,
+	[1][0][2][0][RTW89_MKK][36] = 80,
+	[1][0][2][0][RTW89_IC][36] = 80,
+	[1][0][2][0][RTW89_KCC][36] = 76,
+	[1][0][2][0][RTW89_ACMA][36] = 78,
+	[1][0][2][0][RTW89_CN][36] = 127,
+	[1][0][2][0][RTW89_UK][36] = 80,
+	[1][0][2][0][RTW89_FCC][39] = 84,
+	[1][0][2][0][RTW89_ETSI][39] = 30,
+	[1][0][2][0][RTW89_MKK][39] = 127,
+	[1][0][2][0][RTW89_IC][39] = 84,
+	[1][0][2][0][RTW89_KCC][39] = 68,
+	[1][0][2][0][RTW89_ACMA][39] = 80,
+	[1][0][2][0][RTW89_CN][39] = 70,
+	[1][0][2][0][RTW89_UK][39] = 64,
+	[1][0][2][0][RTW89_FCC][43] = 84,
+	[1][0][2][0][RTW89_ETSI][43] = 30,
+	[1][0][2][0][RTW89_MKK][43] = 127,
+	[1][0][2][0][RTW89_IC][43] = 84,
+	[1][0][2][0][RTW89_KCC][43] = 78,
+	[1][0][2][0][RTW89_ACMA][43] = 80,
+	[1][0][2][0][RTW89_CN][43] = 80,
+	[1][0][2][0][RTW89_UK][43] = 64,
+	[1][0][2][0][RTW89_FCC][47] = 84,
+	[1][0][2][0][RTW89_ETSI][47] = 127,
+	[1][0][2][0][RTW89_MKK][47] = 127,
+	[1][0][2][0][RTW89_IC][47] = 127,
+	[1][0][2][0][RTW89_KCC][47] = 127,
+	[1][0][2][0][RTW89_ACMA][47] = 127,
+	[1][0][2][0][RTW89_CN][47] = 127,
+	[1][0][2][0][RTW89_UK][47] = 127,
+	[1][0][2][0][RTW89_FCC][51] = 84,
+	[1][0][2][0][RTW89_ETSI][51] = 127,
+	[1][0][2][0][RTW89_MKK][51] = 127,
+	[1][0][2][0][RTW89_IC][51] = 127,
+	[1][0][2][0][RTW89_KCC][51] = 127,
+	[1][0][2][0][RTW89_ACMA][51] = 127,
+	[1][0][2][0][RTW89_CN][51] = 127,
+	[1][0][2][0][RTW89_UK][51] = 127,
+	[1][1][2][0][RTW89_FCC][1] = 127,
+	[1][1][2][0][RTW89_ETSI][1] = 127,
+	[1][1][2][0][RTW89_MKK][1] = 127,
+	[1][1][2][0][RTW89_IC][1] = 127,
+	[1][1][2][0][RTW89_KCC][1] = 127,
+	[1][1][2][0][RTW89_ACMA][1] = 127,
+	[1][1][2][0][RTW89_CN][1] = 127,
+	[1][1][2][0][RTW89_UK][1] = 127,
+	[1][1][2][0][RTW89_FCC][5] = 127,
+	[1][1][2][0][RTW89_ETSI][5] = 127,
+	[1][1][2][0][RTW89_MKK][5] = 127,
+	[1][1][2][0][RTW89_IC][5] = 127,
+	[1][1][2][0][RTW89_KCC][5] = 127,
+	[1][1][2][0][RTW89_ACMA][5] = 127,
+	[1][1][2][0][RTW89_CN][5] = 127,
+	[1][1][2][0][RTW89_UK][5] = 127,
+	[1][1][2][0][RTW89_FCC][9] = 127,
+	[1][1][2][0][RTW89_ETSI][9] = 127,
+	[1][1][2][0][RTW89_MKK][9] = 127,
+	[1][1][2][0][RTW89_IC][9] = 127,
+	[1][1][2][0][RTW89_KCC][9] = 127,
+	[1][1][2][0][RTW89_ACMA][9] = 127,
+	[1][1][2][0][RTW89_CN][9] = 127,
+	[1][1][2][0][RTW89_UK][9] = 127,
+	[1][1][2][0][RTW89_FCC][13] = 127,
+	[1][1][2][0][RTW89_ETSI][13] = 127,
+	[1][1][2][0][RTW89_MKK][13] = 127,
+	[1][1][2][0][RTW89_IC][13] = 127,
+	[1][1][2][0][RTW89_KCC][13] = 127,
+	[1][1][2][0][RTW89_ACMA][13] = 127,
+	[1][1][2][0][RTW89_CN][13] = 127,
+	[1][1][2][0][RTW89_UK][13] = 127,
+	[1][1][2][0][RTW89_FCC][16] = 127,
+	[1][1][2][0][RTW89_ETSI][16] = 127,
+	[1][1][2][0][RTW89_MKK][16] = 127,
+	[1][1][2][0][RTW89_IC][16] = 127,
+	[1][1][2][0][RTW89_KCC][16] = 127,
+	[1][1][2][0][RTW89_ACMA][16] = 127,
+	[1][1][2][0][RTW89_CN][16] = 127,
+	[1][1][2][0][RTW89_UK][16] = 127,
+	[1][1][2][0][RTW89_FCC][20] = 127,
+	[1][1][2][0][RTW89_ETSI][20] = 127,
+	[1][1][2][0][RTW89_MKK][20] = 127,
+	[1][1][2][0][RTW89_IC][20] = 127,
+	[1][1][2][0][RTW89_KCC][20] = 127,
+	[1][1][2][0][RTW89_ACMA][20] = 127,
+	[1][1][2][0][RTW89_CN][20] = 127,
+	[1][1][2][0][RTW89_UK][20] = 127,
+	[1][1][2][0][RTW89_FCC][24] = 127,
+	[1][1][2][0][RTW89_ETSI][24] = 127,
+	[1][1][2][0][RTW89_MKK][24] = 127,
+	[1][1][2][0][RTW89_IC][24] = 127,
+	[1][1][2][0][RTW89_KCC][24] = 127,
+	[1][1][2][0][RTW89_ACMA][24] = 127,
+	[1][1][2][0][RTW89_CN][24] = 127,
+	[1][1][2][0][RTW89_UK][24] = 127,
+	[1][1][2][0][RTW89_FCC][28] = 127,
+	[1][1][2][0][RTW89_ETSI][28] = 127,
+	[1][1][2][0][RTW89_MKK][28] = 127,
+	[1][1][2][0][RTW89_IC][28] = 127,
+	[1][1][2][0][RTW89_KCC][28] = 127,
+	[1][1][2][0][RTW89_ACMA][28] = 127,
+	[1][1][2][0][RTW89_CN][28] = 127,
+	[1][1][2][0][RTW89_UK][28] = 127,
+	[1][1][2][0][RTW89_FCC][32] = 127,
+	[1][1][2][0][RTW89_ETSI][32] = 127,
+	[1][1][2][0][RTW89_MKK][32] = 127,
+	[1][1][2][0][RTW89_IC][32] = 127,
+	[1][1][2][0][RTW89_KCC][32] = 127,
+	[1][1][2][0][RTW89_ACMA][32] = 127,
+	[1][1][2][0][RTW89_CN][32] = 127,
+	[1][1][2][0][RTW89_UK][32] = 127,
+	[1][1][2][0][RTW89_FCC][36] = 127,
+	[1][1][2][0][RTW89_ETSI][36] = 127,
+	[1][1][2][0][RTW89_MKK][36] = 127,
+	[1][1][2][0][RTW89_IC][36] = 127,
+	[1][1][2][0][RTW89_KCC][36] = 127,
+	[1][1][2][0][RTW89_ACMA][36] = 127,
+	[1][1][2][0][RTW89_CN][36] = 127,
+	[1][1][2][0][RTW89_UK][36] = 127,
+	[1][1][2][0][RTW89_FCC][39] = 127,
+	[1][1][2][0][RTW89_ETSI][39] = 127,
+	[1][1][2][0][RTW89_MKK][39] = 127,
+	[1][1][2][0][RTW89_IC][39] = 127,
+	[1][1][2][0][RTW89_KCC][39] = 127,
+	[1][1][2][0][RTW89_ACMA][39] = 127,
+	[1][1][2][0][RTW89_CN][39] = 127,
+	[1][1][2][0][RTW89_UK][39] = 127,
+	[1][1][2][0][RTW89_FCC][43] = 127,
+	[1][1][2][0][RTW89_ETSI][43] = 127,
+	[1][1][2][0][RTW89_MKK][43] = 127,
+	[1][1][2][0][RTW89_IC][43] = 127,
+	[1][1][2][0][RTW89_KCC][43] = 127,
+	[1][1][2][0][RTW89_ACMA][43] = 127,
+	[1][1][2][0][RTW89_CN][43] = 127,
+	[1][1][2][0][RTW89_UK][43] = 127,
+	[1][1][2][0][RTW89_FCC][47] = 127,
+	[1][1][2][0][RTW89_ETSI][47] = 127,
+	[1][1][2][0][RTW89_MKK][47] = 127,
+	[1][1][2][0][RTW89_IC][47] = 127,
+	[1][1][2][0][RTW89_KCC][47] = 127,
+	[1][1][2][0][RTW89_ACMA][47] = 127,
+	[1][1][2][0][RTW89_CN][47] = 127,
+	[1][1][2][0][RTW89_UK][47] = 127,
+	[1][1][2][0][RTW89_FCC][51] = 127,
+	[1][1][2][0][RTW89_ETSI][51] = 127,
+	[1][1][2][0][RTW89_MKK][51] = 127,
+	[1][1][2][0][RTW89_IC][51] = 127,
+	[1][1][2][0][RTW89_KCC][51] = 127,
+	[1][1][2][0][RTW89_ACMA][51] = 127,
+	[1][1][2][0][RTW89_CN][51] = 127,
+	[1][1][2][0][RTW89_UK][51] = 127,
+	[1][1][2][1][RTW89_FCC][1] = 127,
+	[1][1][2][1][RTW89_ETSI][1] = 127,
+	[1][1][2][1][RTW89_MKK][1] = 127,
+	[1][1][2][1][RTW89_IC][1] = 127,
+	[1][1][2][1][RTW89_KCC][1] = 127,
+	[1][1][2][1][RTW89_ACMA][1] = 127,
+	[1][1][2][1][RTW89_CN][1] = 127,
+	[1][1][2][1][RTW89_UK][1] = 127,
+	[1][1][2][1][RTW89_FCC][5] = 127,
+	[1][1][2][1][RTW89_ETSI][5] = 127,
+	[1][1][2][1][RTW89_MKK][5] = 127,
+	[1][1][2][1][RTW89_IC][5] = 127,
+	[1][1][2][1][RTW89_KCC][5] = 127,
+	[1][1][2][1][RTW89_ACMA][5] = 127,
+	[1][1][2][1][RTW89_CN][5] = 127,
+	[1][1][2][1][RTW89_UK][5] = 127,
+	[1][1][2][1][RTW89_FCC][9] = 127,
+	[1][1][2][1][RTW89_ETSI][9] = 127,
+	[1][1][2][1][RTW89_MKK][9] = 127,
+	[1][1][2][1][RTW89_IC][9] = 127,
+	[1][1][2][1][RTW89_KCC][9] = 127,
+	[1][1][2][1][RTW89_ACMA][9] = 127,
+	[1][1][2][1][RTW89_CN][9] = 127,
+	[1][1][2][1][RTW89_UK][9] = 127,
+	[1][1][2][1][RTW89_FCC][13] = 127,
+	[1][1][2][1][RTW89_ETSI][13] = 127,
+	[1][1][2][1][RTW89_MKK][13] = 127,
+	[1][1][2][1][RTW89_IC][13] = 127,
+	[1][1][2][1][RTW89_KCC][13] = 127,
+	[1][1][2][1][RTW89_ACMA][13] = 127,
+	[1][1][2][1][RTW89_CN][13] = 127,
+	[1][1][2][1][RTW89_UK][13] = 127,
+	[1][1][2][1][RTW89_FCC][16] = 127,
+	[1][1][2][1][RTW89_ETSI][16] = 127,
+	[1][1][2][1][RTW89_MKK][16] = 127,
+	[1][1][2][1][RTW89_IC][16] = 127,
+	[1][1][2][1][RTW89_KCC][16] = 127,
+	[1][1][2][1][RTW89_ACMA][16] = 127,
+	[1][1][2][1][RTW89_CN][16] = 127,
+	[1][1][2][1][RTW89_UK][16] = 127,
+	[1][1][2][1][RTW89_FCC][20] = 127,
+	[1][1][2][1][RTW89_ETSI][20] = 127,
+	[1][1][2][1][RTW89_MKK][20] = 127,
+	[1][1][2][1][RTW89_IC][20] = 127,
+	[1][1][2][1][RTW89_KCC][20] = 127,
+	[1][1][2][1][RTW89_ACMA][20] = 127,
+	[1][1][2][1][RTW89_CN][20] = 127,
+	[1][1][2][1][RTW89_UK][20] = 127,
+	[1][1][2][1][RTW89_FCC][24] = 127,
+	[1][1][2][1][RTW89_ETSI][24] = 127,
+	[1][1][2][1][RTW89_MKK][24] = 127,
+	[1][1][2][1][RTW89_IC][24] = 127,
+	[1][1][2][1][RTW89_KCC][24] = 127,
+	[1][1][2][1][RTW89_ACMA][24] = 127,
+	[1][1][2][1][RTW89_CN][24] = 127,
+	[1][1][2][1][RTW89_UK][24] = 127,
+	[1][1][2][1][RTW89_FCC][28] = 127,
+	[1][1][2][1][RTW89_ETSI][28] = 127,
+	[1][1][2][1][RTW89_MKK][28] = 127,
+	[1][1][2][1][RTW89_IC][28] = 127,
+	[1][1][2][1][RTW89_KCC][28] = 127,
+	[1][1][2][1][RTW89_ACMA][28] = 127,
+	[1][1][2][1][RTW89_CN][28] = 127,
+	[1][1][2][1][RTW89_UK][28] = 127,
+	[1][1][2][1][RTW89_FCC][32] = 127,
+	[1][1][2][1][RTW89_ETSI][32] = 127,
+	[1][1][2][1][RTW89_MKK][32] = 127,
+	[1][1][2][1][RTW89_IC][32] = 127,
+	[1][1][2][1][RTW89_KCC][32] = 127,
+	[1][1][2][1][RTW89_ACMA][32] = 127,
+	[1][1][2][1][RTW89_CN][32] = 127,
+	[1][1][2][1][RTW89_UK][32] = 127,
+	[1][1][2][1][RTW89_FCC][36] = 127,
+	[1][1][2][1][RTW89_ETSI][36] = 127,
+	[1][1][2][1][RTW89_MKK][36] = 127,
+	[1][1][2][1][RTW89_IC][36] = 127,
+	[1][1][2][1][RTW89_KCC][36] = 127,
+	[1][1][2][1][RTW89_ACMA][36] = 127,
+	[1][1][2][1][RTW89_CN][36] = 127,
+	[1][1][2][1][RTW89_UK][36] = 127,
+	[1][1][2][1][RTW89_FCC][39] = 127,
+	[1][1][2][1][RTW89_ETSI][39] = 127,
+	[1][1][2][1][RTW89_MKK][39] = 127,
+	[1][1][2][1][RTW89_IC][39] = 127,
+	[1][1][2][1][RTW89_KCC][39] = 127,
+	[1][1][2][1][RTW89_ACMA][39] = 127,
+	[1][1][2][1][RTW89_CN][39] = 127,
+	[1][1][2][1][RTW89_UK][39] = 127,
+	[1][1][2][1][RTW89_FCC][43] = 127,
+	[1][1][2][1][RTW89_ETSI][43] = 127,
+	[1][1][2][1][RTW89_MKK][43] = 127,
+	[1][1][2][1][RTW89_IC][43] = 127,
+	[1][1][2][1][RTW89_KCC][43] = 127,
+	[1][1][2][1][RTW89_ACMA][43] = 127,
+	[1][1][2][1][RTW89_CN][43] = 127,
+	[1][1][2][1][RTW89_UK][43] = 127,
+	[1][1][2][1][RTW89_FCC][47] = 127,
+	[1][1][2][1][RTW89_ETSI][47] = 127,
+	[1][1][2][1][RTW89_MKK][47] = 127,
+	[1][1][2][1][RTW89_IC][47] = 127,
+	[1][1][2][1][RTW89_KCC][47] = 127,
+	[1][1][2][1][RTW89_ACMA][47] = 127,
+	[1][1][2][1][RTW89_CN][47] = 127,
+	[1][1][2][1][RTW89_UK][47] = 127,
+	[1][1][2][1][RTW89_FCC][51] = 127,
+	[1][1][2][1][RTW89_ETSI][51] = 127,
+	[1][1][2][1][RTW89_MKK][51] = 127,
+	[1][1][2][1][RTW89_IC][51] = 127,
+	[1][1][2][1][RTW89_KCC][51] = 127,
+	[1][1][2][1][RTW89_ACMA][51] = 127,
+	[1][1][2][1][RTW89_CN][51] = 127,
+	[1][1][2][1][RTW89_UK][51] = 127,
+	[2][0][2][0][RTW89_FCC][3] = 76,
+	[2][0][2][0][RTW89_ETSI][3] = 64,
+	[2][0][2][0][RTW89_MKK][3] = 62,
+	[2][0][2][0][RTW89_IC][3] = 64,
+	[2][0][2][0][RTW89_KCC][3] = 72,
+	[2][0][2][0][RTW89_ACMA][3] = 64,
+	[2][0][2][0][RTW89_CN][3] = 64,
+	[2][0][2][0][RTW89_UK][3] = 64,
+	[2][0][2][0][RTW89_FCC][11] = 64,
+	[2][0][2][0][RTW89_ETSI][11] = 64,
+	[2][0][2][0][RTW89_MKK][11] = 64,
+	[2][0][2][0][RTW89_IC][11] = 62,
+	[2][0][2][0][RTW89_KCC][11] = 72,
+	[2][0][2][0][RTW89_ACMA][11] = 64,
+	[2][0][2][0][RTW89_CN][11] = 64,
+	[2][0][2][0][RTW89_UK][11] = 64,
+	[2][0][2][0][RTW89_FCC][18] = 66,
+	[2][0][2][0][RTW89_ETSI][18] = 64,
+	[2][0][2][0][RTW89_MKK][18] = 72,
+	[2][0][2][0][RTW89_IC][18] = 66,
+	[2][0][2][0][RTW89_KCC][18] = 72,
+	[2][0][2][0][RTW89_ACMA][18] = 64,
+	[2][0][2][0][RTW89_CN][18] = 127,
+	[2][0][2][0][RTW89_UK][18] = 64,
+	[2][0][2][0][RTW89_FCC][26] = 76,
+	[2][0][2][0][RTW89_ETSI][26] = 64,
+	[2][0][2][0][RTW89_MKK][26] = 72,
+	[2][0][2][0][RTW89_IC][26] = 127,
+	[2][0][2][0][RTW89_KCC][26] = 72,
+	[2][0][2][0][RTW89_ACMA][26] = 127,
+	[2][0][2][0][RTW89_CN][26] = 127,
+	[2][0][2][0][RTW89_UK][26] = 64,
+	[2][0][2][0][RTW89_FCC][34] = 76,
+	[2][0][2][0][RTW89_ETSI][34] = 127,
+	[2][0][2][0][RTW89_MKK][34] = 72,
+	[2][0][2][0][RTW89_IC][34] = 76,
+	[2][0][2][0][RTW89_KCC][34] = 72,
+	[2][0][2][0][RTW89_ACMA][34] = 72,
+	[2][0][2][0][RTW89_CN][34] = 127,
+	[2][0][2][0][RTW89_UK][34] = 72,
+	[2][0][2][0][RTW89_FCC][41] = 76,
+	[2][0][2][0][RTW89_ETSI][41] = 30,
+	[2][0][2][0][RTW89_MKK][41] = 127,
+	[2][0][2][0][RTW89_IC][41] = 76,
+	[2][0][2][0][RTW89_KCC][41] = 64,
+	[2][0][2][0][RTW89_ACMA][41] = 72,
+	[2][0][2][0][RTW89_CN][41] = 72,
+	[2][0][2][0][RTW89_UK][41] = 64,
+	[2][0][2][0][RTW89_FCC][49] = 74,
+	[2][0][2][0][RTW89_ETSI][49] = 127,
+	[2][0][2][0][RTW89_MKK][49] = 127,
+	[2][0][2][0][RTW89_IC][49] = 127,
+	[2][0][2][0][RTW89_KCC][49] = 127,
+	[2][0][2][0][RTW89_ACMA][49] = 127,
+	[2][0][2][0][RTW89_CN][49] = 127,
+	[2][0][2][0][RTW89_UK][49] = 127,
+	[2][1][2][0][RTW89_FCC][3] = 127,
+	[2][1][2][0][RTW89_ETSI][3] = 127,
+	[2][1][2][0][RTW89_MKK][3] = 127,
+	[2][1][2][0][RTW89_IC][3] = 127,
+	[2][1][2][0][RTW89_KCC][3] = 127,
+	[2][1][2][0][RTW89_ACMA][3] = 127,
+	[2][1][2][0][RTW89_CN][3] = 127,
+	[2][1][2][0][RTW89_UK][3] = 127,
+	[2][1][2][0][RTW89_FCC][11] = 127,
+	[2][1][2][0][RTW89_ETSI][11] = 127,
+	[2][1][2][0][RTW89_MKK][11] = 127,
+	[2][1][2][0][RTW89_IC][11] = 127,
+	[2][1][2][0][RTW89_KCC][11] = 127,
+	[2][1][2][0][RTW89_ACMA][11] = 127,
+	[2][1][2][0][RTW89_CN][11] = 127,
+	[2][1][2][0][RTW89_UK][11] = 127,
+	[2][1][2][0][RTW89_FCC][18] = 127,
+	[2][1][2][0][RTW89_ETSI][18] = 127,
+	[2][1][2][0][RTW89_MKK][18] = 127,
+	[2][1][2][0][RTW89_IC][18] = 127,
+	[2][1][2][0][RTW89_KCC][18] = 127,
+	[2][1][2][0][RTW89_ACMA][18] = 127,
+	[2][1][2][0][RTW89_CN][18] = 127,
+	[2][1][2][0][RTW89_UK][18] = 127,
+	[2][1][2][0][RTW89_FCC][26] = 127,
+	[2][1][2][0][RTW89_ETSI][26] = 127,
+	[2][1][2][0][RTW89_MKK][26] = 127,
+	[2][1][2][0][RTW89_IC][26] = 127,
+	[2][1][2][0][RTW89_KCC][26] = 127,
+	[2][1][2][0][RTW89_ACMA][26] = 127,
+	[2][1][2][0][RTW89_CN][26] = 127,
+	[2][1][2][0][RTW89_UK][26] = 127,
+	[2][1][2][0][RTW89_FCC][34] = 127,
+	[2][1][2][0][RTW89_ETSI][34] = 127,
+	[2][1][2][0][RTW89_MKK][34] = 127,
+	[2][1][2][0][RTW89_IC][34] = 127,
+	[2][1][2][0][RTW89_KCC][34] = 127,
+	[2][1][2][0][RTW89_ACMA][34] = 127,
+	[2][1][2][0][RTW89_CN][34] = 127,
+	[2][1][2][0][RTW89_UK][34] = 127,
+	[2][1][2][0][RTW89_FCC][41] = 127,
+	[2][1][2][0][RTW89_ETSI][41] = 127,
+	[2][1][2][0][RTW89_MKK][41] = 127,
+	[2][1][2][0][RTW89_IC][41] = 127,
+	[2][1][2][0][RTW89_KCC][41] = 127,
+	[2][1][2][0][RTW89_ACMA][41] = 127,
+	[2][1][2][0][RTW89_CN][41] = 127,
+	[2][1][2][0][RTW89_UK][41] = 127,
+	[2][1][2][0][RTW89_FCC][49] = 127,
+	[2][1][2][0][RTW89_ETSI][49] = 127,
+	[2][1][2][0][RTW89_MKK][49] = 127,
+	[2][1][2][0][RTW89_IC][49] = 127,
+	[2][1][2][0][RTW89_KCC][49] = 127,
+	[2][1][2][0][RTW89_ACMA][49] = 127,
+	[2][1][2][0][RTW89_CN][49] = 127,
+	[2][1][2][0][RTW89_UK][49] = 127,
+	[2][1][2][1][RTW89_FCC][3] = 127,
+	[2][1][2][1][RTW89_ETSI][3] = 127,
+	[2][1][2][1][RTW89_MKK][3] = 127,
+	[2][1][2][1][RTW89_IC][3] = 127,
+	[2][1][2][1][RTW89_KCC][3] = 127,
+	[2][1][2][1][RTW89_ACMA][3] = 127,
+	[2][1][2][1][RTW89_CN][3] = 127,
+	[2][1][2][1][RTW89_UK][3] = 127,
+	[2][1][2][1][RTW89_FCC][11] = 127,
+	[2][1][2][1][RTW89_ETSI][11] = 127,
+	[2][1][2][1][RTW89_MKK][11] = 127,
+	[2][1][2][1][RTW89_IC][11] = 127,
+	[2][1][2][1][RTW89_KCC][11] = 127,
+	[2][1][2][1][RTW89_ACMA][11] = 127,
+	[2][1][2][1][RTW89_CN][11] = 127,
+	[2][1][2][1][RTW89_UK][11] = 127,
+	[2][1][2][1][RTW89_FCC][18] = 127,
+	[2][1][2][1][RTW89_ETSI][18] = 127,
+	[2][1][2][1][RTW89_MKK][18] = 127,
+	[2][1][2][1][RTW89_IC][18] = 127,
+	[2][1][2][1][RTW89_KCC][18] = 127,
+	[2][1][2][1][RTW89_ACMA][18] = 127,
+	[2][1][2][1][RTW89_CN][18] = 127,
+	[2][1][2][1][RTW89_UK][18] = 127,
+	[2][1][2][1][RTW89_FCC][26] = 127,
+	[2][1][2][1][RTW89_ETSI][26] = 127,
+	[2][1][2][1][RTW89_MKK][26] = 127,
+	[2][1][2][1][RTW89_IC][26] = 127,
+	[2][1][2][1][RTW89_KCC][26] = 127,
+	[2][1][2][1][RTW89_ACMA][26] = 127,
+	[2][1][2][1][RTW89_CN][26] = 127,
+	[2][1][2][1][RTW89_UK][26] = 127,
+	[2][1][2][1][RTW89_FCC][34] = 127,
+	[2][1][2][1][RTW89_ETSI][34] = 127,
+	[2][1][2][1][RTW89_MKK][34] = 127,
+	[2][1][2][1][RTW89_IC][34] = 127,
+	[2][1][2][1][RTW89_KCC][34] = 127,
+	[2][1][2][1][RTW89_ACMA][34] = 127,
+	[2][1][2][1][RTW89_CN][34] = 127,
+	[2][1][2][1][RTW89_UK][34] = 127,
+	[2][1][2][1][RTW89_FCC][41] = 127,
+	[2][1][2][1][RTW89_ETSI][41] = 127,
+	[2][1][2][1][RTW89_MKK][41] = 127,
+	[2][1][2][1][RTW89_IC][41] = 127,
+	[2][1][2][1][RTW89_KCC][41] = 127,
+	[2][1][2][1][RTW89_ACMA][41] = 127,
+	[2][1][2][1][RTW89_CN][41] = 127,
+	[2][1][2][1][RTW89_UK][41] = 127,
+	[2][1][2][1][RTW89_FCC][49] = 127,
+	[2][1][2][1][RTW89_ETSI][49] = 127,
+	[2][1][2][1][RTW89_MKK][49] = 127,
+	[2][1][2][1][RTW89_IC][49] = 127,
+	[2][1][2][1][RTW89_KCC][49] = 127,
+	[2][1][2][1][RTW89_ACMA][49] = 127,
+	[2][1][2][1][RTW89_CN][49] = 127,
+	[2][1][2][1][RTW89_UK][49] = 127,
+	[3][0][2][0][RTW89_FCC][7] = 127,
+	[3][0][2][0][RTW89_ETSI][7] = 127,
+	[3][0][2][0][RTW89_MKK][7] = 127,
+	[3][0][2][0][RTW89_IC][7] = 127,
+	[3][0][2][0][RTW89_KCC][7] = 127,
+	[3][0][2][0][RTW89_ACMA][7] = 127,
+	[3][0][2][0][RTW89_CN][7] = 58,
+	[3][0][2][0][RTW89_UK][7] = 127,
+	[3][0][2][0][RTW89_FCC][22] = 127,
+	[3][0][2][0][RTW89_ETSI][22] = 127,
+	[3][0][2][0][RTW89_MKK][22] = 127,
+	[3][0][2][0][RTW89_IC][22] = 127,
+	[3][0][2][0][RTW89_KCC][22] = 127,
+	[3][0][2][0][RTW89_ACMA][22] = 127,
+	[3][0][2][0][RTW89_CN][22] = 58,
+	[3][0][2][0][RTW89_UK][22] = 127,
+	[3][0][2][0][RTW89_FCC][45] = 127,
+	[3][0][2][0][RTW89_ETSI][45] = 127,
+	[3][0][2][0][RTW89_MKK][45] = 127,
+	[3][0][2][0][RTW89_IC][45] = 127,
+	[3][0][2][0][RTW89_KCC][45] = 127,
+	[3][0][2][0][RTW89_ACMA][45] = 127,
+	[3][0][2][0][RTW89_CN][45] = 127,
+	[3][0][2][0][RTW89_UK][45] = 127,
+	[3][1][2][0][RTW89_FCC][7] = 127,
+	[3][1][2][0][RTW89_ETSI][7] = 127,
+	[3][1][2][0][RTW89_MKK][7] = 127,
+	[3][1][2][0][RTW89_IC][7] = 127,
+	[3][1][2][0][RTW89_KCC][7] = 127,
+	[3][1][2][0][RTW89_ACMA][7] = 127,
+	[3][1][2][0][RTW89_CN][7] = 127,
+	[3][1][2][0][RTW89_UK][7] = 127,
+	[3][1][2][0][RTW89_FCC][22] = 127,
+	[3][1][2][0][RTW89_ETSI][22] = 127,
+	[3][1][2][0][RTW89_MKK][22] = 127,
+	[3][1][2][0][RTW89_IC][22] = 127,
+	[3][1][2][0][RTW89_KCC][22] = 127,
+	[3][1][2][0][RTW89_ACMA][22] = 127,
+	[3][1][2][0][RTW89_CN][22] = 127,
+	[3][1][2][0][RTW89_UK][22] = 127,
+	[3][1][2][0][RTW89_FCC][45] = 127,
+	[3][1][2][0][RTW89_ETSI][45] = 127,
+	[3][1][2][0][RTW89_MKK][45] = 127,
+	[3][1][2][0][RTW89_IC][45] = 127,
+	[3][1][2][0][RTW89_KCC][45] = 127,
+	[3][1][2][0][RTW89_ACMA][45] = 127,
+	[3][1][2][0][RTW89_CN][45] = 127,
+	[3][1][2][0][RTW89_UK][45] = 127,
+	[3][1][2][1][RTW89_FCC][7] = 127,
+	[3][1][2][1][RTW89_ETSI][7] = 127,
+	[3][1][2][1][RTW89_MKK][7] = 127,
+	[3][1][2][1][RTW89_IC][7] = 127,
+	[3][1][2][1][RTW89_KCC][7] = 127,
+	[3][1][2][1][RTW89_ACMA][7] = 127,
+	[3][1][2][1][RTW89_CN][7] = 127,
+	[3][1][2][1][RTW89_UK][7] = 127,
+	[3][1][2][1][RTW89_FCC][22] = 127,
+	[3][1][2][1][RTW89_ETSI][22] = 127,
+	[3][1][2][1][RTW89_MKK][22] = 127,
+	[3][1][2][1][RTW89_IC][22] = 127,
+	[3][1][2][1][RTW89_KCC][22] = 127,
+	[3][1][2][1][RTW89_ACMA][22] = 127,
+	[3][1][2][1][RTW89_CN][22] = 127,
+	[3][1][2][1][RTW89_UK][22] = 127,
+	[3][1][2][1][RTW89_FCC][45] = 127,
+	[3][1][2][1][RTW89_ETSI][45] = 127,
+	[3][1][2][1][RTW89_MKK][45] = 127,
+	[3][1][2][1][RTW89_IC][45] = 127,
+	[3][1][2][1][RTW89_KCC][45] = 127,
+	[3][1][2][1][RTW89_ACMA][45] = 127,
+	[3][1][2][1][RTW89_CN][45] = 127,
+	[3][1][2][1][RTW89_UK][45] = 127,
+};
+
+static
+const s8 rtw89_8851b_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
+				    [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
+	[0][0][RTW89_WW][0] = 30,
+	[0][0][RTW89_WW][1] = 30,
+	[0][0][RTW89_WW][2] = 30,
+	[0][0][RTW89_WW][3] = 30,
+	[0][0][RTW89_WW][4] = 30,
+	[0][0][RTW89_WW][5] = 30,
+	[0][0][RTW89_WW][6] = 30,
+	[0][0][RTW89_WW][7] = 30,
+	[0][0][RTW89_WW][8] = 30,
+	[0][0][RTW89_WW][9] = 30,
+	[0][0][RTW89_WW][10] = 30,
+	[0][0][RTW89_WW][11] = 30,
+	[0][0][RTW89_WW][12] = 30,
+	[0][0][RTW89_WW][13] = 0,
+	[0][1][RTW89_WW][0] = 20,
+	[0][1][RTW89_WW][1] = 22,
+	[0][1][RTW89_WW][2] = 22,
+	[0][1][RTW89_WW][3] = 22,
+	[0][1][RTW89_WW][4] = 22,
+	[0][1][RTW89_WW][5] = 22,
+	[0][1][RTW89_WW][6] = 22,
+	[0][1][RTW89_WW][7] = 22,
+	[0][1][RTW89_WW][8] = 22,
+	[0][1][RTW89_WW][9] = 22,
+	[0][1][RTW89_WW][10] = 22,
+	[0][1][RTW89_WW][11] = 22,
+	[0][1][RTW89_WW][12] = 20,
+	[0][1][RTW89_WW][13] = 0,
+	[1][0][RTW89_WW][0] = 42,
+	[1][0][RTW89_WW][1] = 42,
+	[1][0][RTW89_WW][2] = 42,
+	[1][0][RTW89_WW][3] = 42,
+	[1][0][RTW89_WW][4] = 42,
+	[1][0][RTW89_WW][5] = 42,
+	[1][0][RTW89_WW][6] = 42,
+	[1][0][RTW89_WW][7] = 42,
+	[1][0][RTW89_WW][8] = 42,
+	[1][0][RTW89_WW][9] = 42,
+	[1][0][RTW89_WW][10] = 42,
+	[1][0][RTW89_WW][11] = 42,
+	[1][0][RTW89_WW][12] = 34,
+	[1][0][RTW89_WW][13] = 0,
+	[1][1][RTW89_WW][0] = 32,
+	[1][1][RTW89_WW][1] = 32,
+	[1][1][RTW89_WW][2] = 32,
+	[1][1][RTW89_WW][3] = 32,
+	[1][1][RTW89_WW][4] = 32,
+	[1][1][RTW89_WW][5] = 32,
+	[1][1][RTW89_WW][6] = 32,
+	[1][1][RTW89_WW][7] = 32,
+	[1][1][RTW89_WW][8] = 32,
+	[1][1][RTW89_WW][9] = 32,
+	[1][1][RTW89_WW][10] = 32,
+	[1][1][RTW89_WW][11] = 32,
+	[1][1][RTW89_WW][12] = 32,
+	[1][1][RTW89_WW][13] = 0,
+	[2][0][RTW89_WW][0] = 54,
+	[2][0][RTW89_WW][1] = 54,
+	[2][0][RTW89_WW][2] = 54,
+	[2][0][RTW89_WW][3] = 54,
+	[2][0][RTW89_WW][4] = 54,
+	[2][0][RTW89_WW][5] = 54,
+	[2][0][RTW89_WW][6] = 54,
+	[2][0][RTW89_WW][7] = 54,
+	[2][0][RTW89_WW][8] = 54,
+	[2][0][RTW89_WW][9] = 54,
+	[2][0][RTW89_WW][10] = 54,
+	[2][0][RTW89_WW][11] = 54,
+	[2][0][RTW89_WW][12] = 34,
+	[2][0][RTW89_WW][13] = 0,
+	[2][1][RTW89_WW][0] = 44,
+	[2][1][RTW89_WW][1] = 44,
+	[2][1][RTW89_WW][2] = 44,
+	[2][1][RTW89_WW][3] = 44,
+	[2][1][RTW89_WW][4] = 44,
+	[2][1][RTW89_WW][5] = 44,
+	[2][1][RTW89_WW][6] = 44,
+	[2][1][RTW89_WW][7] = 44,
+	[2][1][RTW89_WW][8] = 44,
+	[2][1][RTW89_WW][9] = 44,
+	[2][1][RTW89_WW][10] = 44,
+	[2][1][RTW89_WW][11] = 44,
+	[2][1][RTW89_WW][12] = 42,
+	[2][1][RTW89_WW][13] = 0,
+	[0][0][RTW89_FCC][0] = 62,
+	[0][0][RTW89_ETSI][0] = 30,
+	[0][0][RTW89_MKK][0] = 40,
+	[0][0][RTW89_IC][0] = 62,
+	[0][0][RTW89_KCC][0] = 46,
+	[0][0][RTW89_ACMA][0] = 30,
+	[0][0][RTW89_CN][0] = 32,
+	[0][0][RTW89_UK][0] = 30,
+	[0][0][RTW89_FCC][1] = 62,
+	[0][0][RTW89_ETSI][1] = 30,
+	[0][0][RTW89_MKK][1] = 44,
+	[0][0][RTW89_IC][1] = 62,
+	[0][0][RTW89_KCC][1] = 46,
+	[0][0][RTW89_ACMA][1] = 30,
+	[0][0][RTW89_CN][1] = 32,
+	[0][0][RTW89_UK][1] = 30,
+	[0][0][RTW89_FCC][2] = 66,
+	[0][0][RTW89_ETSI][2] = 30,
+	[0][0][RTW89_MKK][2] = 44,
+	[0][0][RTW89_IC][2] = 66,
+	[0][0][RTW89_KCC][2] = 46,
+	[0][0][RTW89_ACMA][2] = 30,
+	[0][0][RTW89_CN][2] = 32,
+	[0][0][RTW89_UK][2] = 30,
+	[0][0][RTW89_FCC][3] = 70,
+	[0][0][RTW89_ETSI][3] = 30,
+	[0][0][RTW89_MKK][3] = 44,
+	[0][0][RTW89_IC][3] = 70,
+	[0][0][RTW89_KCC][3] = 46,
+	[0][0][RTW89_ACMA][3] = 30,
+	[0][0][RTW89_CN][3] = 32,
+	[0][0][RTW89_UK][3] = 30,
+	[0][0][RTW89_FCC][4] = 70,
+	[0][0][RTW89_ETSI][4] = 30,
+	[0][0][RTW89_MKK][4] = 44,
+	[0][0][RTW89_IC][4] = 70,
+	[0][0][RTW89_KCC][4] = 48,
+	[0][0][RTW89_ACMA][4] = 30,
+	[0][0][RTW89_CN][4] = 32,
+	[0][0][RTW89_UK][4] = 30,
+	[0][0][RTW89_FCC][5] = 84,
+	[0][0][RTW89_ETSI][5] = 30,
+	[0][0][RTW89_MKK][5] = 44,
+	[0][0][RTW89_IC][5] = 84,
+	[0][0][RTW89_KCC][5] = 48,
+	[0][0][RTW89_ACMA][5] = 30,
+	[0][0][RTW89_CN][5] = 32,
+	[0][0][RTW89_UK][5] = 30,
+	[0][0][RTW89_FCC][6] = 66,
+	[0][0][RTW89_ETSI][6] = 30,
+	[0][0][RTW89_MKK][6] = 44,
+	[0][0][RTW89_IC][6] = 66,
+	[0][0][RTW89_KCC][6] = 48,
+	[0][0][RTW89_ACMA][6] = 30,
+	[0][0][RTW89_CN][6] = 32,
+	[0][0][RTW89_UK][6] = 30,
+	[0][0][RTW89_FCC][7] = 66,
+	[0][0][RTW89_ETSI][7] = 30,
+	[0][0][RTW89_MKK][7] = 44,
+	[0][0][RTW89_IC][7] = 66,
+	[0][0][RTW89_KCC][7] = 48,
+	[0][0][RTW89_ACMA][7] = 30,
+	[0][0][RTW89_CN][7] = 32,
+	[0][0][RTW89_UK][7] = 30,
+	[0][0][RTW89_FCC][8] = 62,
+	[0][0][RTW89_ETSI][8] = 30,
+	[0][0][RTW89_MKK][8] = 44,
+	[0][0][RTW89_IC][8] = 62,
+	[0][0][RTW89_KCC][8] = 48,
+	[0][0][RTW89_ACMA][8] = 30,
+	[0][0][RTW89_CN][8] = 32,
+	[0][0][RTW89_UK][8] = 30,
+	[0][0][RTW89_FCC][9] = 58,
+	[0][0][RTW89_ETSI][9] = 30,
+	[0][0][RTW89_MKK][9] = 44,
+	[0][0][RTW89_IC][9] = 58,
+	[0][0][RTW89_KCC][9] = 44,
+	[0][0][RTW89_ACMA][9] = 30,
+	[0][0][RTW89_CN][9] = 32,
+	[0][0][RTW89_UK][9] = 30,
+	[0][0][RTW89_FCC][10] = 58,
+	[0][0][RTW89_ETSI][10] = 30,
+	[0][0][RTW89_MKK][10] = 44,
+	[0][0][RTW89_IC][10] = 58,
+	[0][0][RTW89_KCC][10] = 44,
+	[0][0][RTW89_ACMA][10] = 30,
+	[0][0][RTW89_CN][10] = 32,
+	[0][0][RTW89_UK][10] = 30,
+	[0][0][RTW89_FCC][11] = 54,
+	[0][0][RTW89_ETSI][11] = 30,
+	[0][0][RTW89_MKK][11] = 44,
+	[0][0][RTW89_IC][11] = 54,
+	[0][0][RTW89_KCC][11] = 44,
+	[0][0][RTW89_ACMA][11] = 30,
+	[0][0][RTW89_CN][11] = 32,
+	[0][0][RTW89_UK][11] = 30,
+	[0][0][RTW89_FCC][12] = 36,
+	[0][0][RTW89_ETSI][12] = 30,
+	[0][0][RTW89_MKK][12] = 40,
+	[0][0][RTW89_IC][12] = 36,
+	[0][0][RTW89_KCC][12] = 44,
+	[0][0][RTW89_ACMA][12] = 30,
+	[0][0][RTW89_CN][12] = 32,
+	[0][0][RTW89_UK][12] = 30,
+	[0][0][RTW89_FCC][13] = 127,
+	[0][0][RTW89_ETSI][13] = 127,
+	[0][0][RTW89_MKK][13] = 127,
+	[0][0][RTW89_IC][13] = 127,
+	[0][0][RTW89_KCC][13] = 127,
+	[0][0][RTW89_ACMA][13] = 127,
+	[0][0][RTW89_CN][13] = 127,
+	[0][0][RTW89_UK][13] = 127,
+	[0][1][RTW89_FCC][0] = 127,
+	[0][1][RTW89_ETSI][0] = 127,
+	[0][1][RTW89_MKK][0] = 127,
+	[0][1][RTW89_IC][0] = 127,
+	[0][1][RTW89_KCC][0] = 127,
+	[0][1][RTW89_ACMA][0] = 127,
+	[0][1][RTW89_CN][0] = 20,
+	[0][1][RTW89_UK][0] = 127,
+	[0][1][RTW89_FCC][1] = 127,
+	[0][1][RTW89_ETSI][1] = 127,
+	[0][1][RTW89_MKK][1] = 127,
+	[0][1][RTW89_IC][1] = 127,
+	[0][1][RTW89_KCC][1] = 127,
+	[0][1][RTW89_ACMA][1] = 127,
+	[0][1][RTW89_CN][1] = 22,
+	[0][1][RTW89_UK][1] = 127,
+	[0][1][RTW89_FCC][2] = 127,
+	[0][1][RTW89_ETSI][2] = 127,
+	[0][1][RTW89_MKK][2] = 127,
+	[0][1][RTW89_IC][2] = 127,
+	[0][1][RTW89_KCC][2] = 127,
+	[0][1][RTW89_ACMA][2] = 127,
+	[0][1][RTW89_CN][2] = 22,
+	[0][1][RTW89_UK][2] = 127,
+	[0][1][RTW89_FCC][3] = 127,
+	[0][1][RTW89_ETSI][3] = 127,
+	[0][1][RTW89_MKK][3] = 127,
+	[0][1][RTW89_IC][3] = 127,
+	[0][1][RTW89_KCC][3] = 127,
+	[0][1][RTW89_ACMA][3] = 127,
+	[0][1][RTW89_CN][3] = 22,
+	[0][1][RTW89_UK][3] = 127,
+	[0][1][RTW89_FCC][4] = 127,
+	[0][1][RTW89_ETSI][4] = 127,
+	[0][1][RTW89_MKK][4] = 127,
+	[0][1][RTW89_IC][4] = 127,
+	[0][1][RTW89_KCC][4] = 127,
+	[0][1][RTW89_ACMA][4] = 127,
+	[0][1][RTW89_CN][4] = 22,
+	[0][1][RTW89_UK][4] = 127,
+	[0][1][RTW89_FCC][5] = 127,
+	[0][1][RTW89_ETSI][5] = 127,
+	[0][1][RTW89_MKK][5] = 127,
+	[0][1][RTW89_IC][5] = 127,
+	[0][1][RTW89_KCC][5] = 127,
+	[0][1][RTW89_ACMA][5] = 127,
+	[0][1][RTW89_CN][5] = 22,
+	[0][1][RTW89_UK][5] = 127,
+	[0][1][RTW89_FCC][6] = 127,
+	[0][1][RTW89_ETSI][6] = 127,
+	[0][1][RTW89_MKK][6] = 127,
+	[0][1][RTW89_IC][6] = 127,
+	[0][1][RTW89_KCC][6] = 127,
+	[0][1][RTW89_ACMA][6] = 127,
+	[0][1][RTW89_CN][6] = 22,
+	[0][1][RTW89_UK][6] = 127,
+	[0][1][RTW89_FCC][7] = 127,
+	[0][1][RTW89_ETSI][7] = 127,
+	[0][1][RTW89_MKK][7] = 127,
+	[0][1][RTW89_IC][7] = 127,
+	[0][1][RTW89_KCC][7] = 127,
+	[0][1][RTW89_ACMA][7] = 127,
+	[0][1][RTW89_CN][7] = 22,
+	[0][1][RTW89_UK][7] = 127,
+	[0][1][RTW89_FCC][8] = 127,
+	[0][1][RTW89_ETSI][8] = 127,
+	[0][1][RTW89_MKK][8] = 127,
+	[0][1][RTW89_IC][8] = 127,
+	[0][1][RTW89_KCC][8] = 127,
+	[0][1][RTW89_ACMA][8] = 127,
+	[0][1][RTW89_CN][8] = 22,
+	[0][1][RTW89_UK][8] = 127,
+	[0][1][RTW89_FCC][9] = 127,
+	[0][1][RTW89_ETSI][9] = 127,
+	[0][1][RTW89_MKK][9] = 127,
+	[0][1][RTW89_IC][9] = 127,
+	[0][1][RTW89_KCC][9] = 127,
+	[0][1][RTW89_ACMA][9] = 127,
+	[0][1][RTW89_CN][9] = 22,
+	[0][1][RTW89_UK][9] = 127,
+	[0][1][RTW89_FCC][10] = 127,
+	[0][1][RTW89_ETSI][10] = 127,
+	[0][1][RTW89_MKK][10] = 127,
+	[0][1][RTW89_IC][10] = 127,
+	[0][1][RTW89_KCC][10] = 127,
+	[0][1][RTW89_ACMA][10] = 127,
+	[0][1][RTW89_CN][10] = 22,
+	[0][1][RTW89_UK][10] = 127,
+	[0][1][RTW89_FCC][11] = 127,
+	[0][1][RTW89_ETSI][11] = 127,
+	[0][1][RTW89_MKK][11] = 127,
+	[0][1][RTW89_IC][11] = 127,
+	[0][1][RTW89_KCC][11] = 127,
+	[0][1][RTW89_ACMA][11] = 127,
+	[0][1][RTW89_CN][11] = 22,
+	[0][1][RTW89_UK][11] = 127,
+	[0][1][RTW89_FCC][12] = 127,
+	[0][1][RTW89_ETSI][12] = 127,
+	[0][1][RTW89_MKK][12] = 127,
+	[0][1][RTW89_IC][12] = 127,
+	[0][1][RTW89_KCC][12] = 127,
+	[0][1][RTW89_ACMA][12] = 127,
+	[0][1][RTW89_CN][12] = 20,
+	[0][1][RTW89_UK][12] = 127,
+	[0][1][RTW89_FCC][13] = 127,
+	[0][1][RTW89_ETSI][13] = 127,
+	[0][1][RTW89_MKK][13] = 127,
+	[0][1][RTW89_IC][13] = 127,
+	[0][1][RTW89_KCC][13] = 127,
+	[0][1][RTW89_ACMA][13] = 127,
+	[0][1][RTW89_CN][13] = 127,
+	[0][1][RTW89_UK][13] = 127,
+	[1][0][RTW89_FCC][0] = 70,
+	[1][0][RTW89_ETSI][0] = 42,
+	[1][0][RTW89_MKK][0] = 52,
+	[1][0][RTW89_IC][0] = 70,
+	[1][0][RTW89_KCC][0] = 56,
+	[1][0][RTW89_ACMA][0] = 42,
+	[1][0][RTW89_CN][0] = 42,
+	[1][0][RTW89_UK][0] = 42,
+	[1][0][RTW89_FCC][1] = 70,
+	[1][0][RTW89_ETSI][1] = 42,
+	[1][0][RTW89_MKK][1] = 52,
+	[1][0][RTW89_IC][1] = 70,
+	[1][0][RTW89_KCC][1] = 56,
+	[1][0][RTW89_ACMA][1] = 42,
+	[1][0][RTW89_CN][1] = 44,
+	[1][0][RTW89_UK][1] = 42,
+	[1][0][RTW89_FCC][2] = 74,
+	[1][0][RTW89_ETSI][2] = 42,
+	[1][0][RTW89_MKK][2] = 52,
+	[1][0][RTW89_IC][2] = 74,
+	[1][0][RTW89_KCC][2] = 56,
+	[1][0][RTW89_ACMA][2] = 42,
+	[1][0][RTW89_CN][2] = 44,
+	[1][0][RTW89_UK][2] = 42,
+	[1][0][RTW89_FCC][3] = 76,
+	[1][0][RTW89_ETSI][3] = 42,
+	[1][0][RTW89_MKK][3] = 52,
+	[1][0][RTW89_IC][3] = 76,
+	[1][0][RTW89_KCC][3] = 56,
+	[1][0][RTW89_ACMA][3] = 42,
+	[1][0][RTW89_CN][3] = 44,
+	[1][0][RTW89_UK][3] = 42,
+	[1][0][RTW89_FCC][4] = 76,
+	[1][0][RTW89_ETSI][4] = 42,
+	[1][0][RTW89_MKK][4] = 52,
+	[1][0][RTW89_IC][4] = 76,
+	[1][0][RTW89_KCC][4] = 56,
+	[1][0][RTW89_ACMA][4] = 42,
+	[1][0][RTW89_CN][4] = 44,
+	[1][0][RTW89_UK][4] = 42,
+	[1][0][RTW89_FCC][5] = 82,
+	[1][0][RTW89_ETSI][5] = 42,
+	[1][0][RTW89_MKK][5] = 52,
+	[1][0][RTW89_IC][5] = 82,
+	[1][0][RTW89_KCC][5] = 56,
+	[1][0][RTW89_ACMA][5] = 42,
+	[1][0][RTW89_CN][5] = 44,
+	[1][0][RTW89_UK][5] = 42,
+	[1][0][RTW89_FCC][6] = 74,
+	[1][0][RTW89_ETSI][6] = 42,
+	[1][0][RTW89_MKK][6] = 52,
+	[1][0][RTW89_IC][6] = 74,
+	[1][0][RTW89_KCC][6] = 56,
+	[1][0][RTW89_ACMA][6] = 42,
+	[1][0][RTW89_CN][6] = 44,
+	[1][0][RTW89_UK][6] = 42,
+	[1][0][RTW89_FCC][7] = 74,
+	[1][0][RTW89_ETSI][7] = 42,
+	[1][0][RTW89_MKK][7] = 52,
+	[1][0][RTW89_IC][7] = 74,
+	[1][0][RTW89_KCC][7] = 56,
+	[1][0][RTW89_ACMA][7] = 42,
+	[1][0][RTW89_CN][7] = 44,
+	[1][0][RTW89_UK][7] = 42,
+	[1][0][RTW89_FCC][8] = 74,
+	[1][0][RTW89_ETSI][8] = 42,
+	[1][0][RTW89_MKK][8] = 52,
+	[1][0][RTW89_IC][8] = 74,
+	[1][0][RTW89_KCC][8] = 56,
+	[1][0][RTW89_ACMA][8] = 42,
+	[1][0][RTW89_CN][8] = 44,
+	[1][0][RTW89_UK][8] = 42,
+	[1][0][RTW89_FCC][9] = 70,
+	[1][0][RTW89_ETSI][9] = 42,
+	[1][0][RTW89_MKK][9] = 52,
+	[1][0][RTW89_IC][9] = 70,
+	[1][0][RTW89_KCC][9] = 58,
+	[1][0][RTW89_ACMA][9] = 42,
+	[1][0][RTW89_CN][9] = 44,
+	[1][0][RTW89_UK][9] = 42,
+	[1][0][RTW89_FCC][10] = 70,
+	[1][0][RTW89_ETSI][10] = 42,
+	[1][0][RTW89_MKK][10] = 52,
+	[1][0][RTW89_IC][10] = 70,
+	[1][0][RTW89_KCC][10] = 58,
+	[1][0][RTW89_ACMA][10] = 42,
+	[1][0][RTW89_CN][10] = 44,
+	[1][0][RTW89_UK][10] = 42,
+	[1][0][RTW89_FCC][11] = 66,
+	[1][0][RTW89_ETSI][11] = 42,
+	[1][0][RTW89_MKK][11] = 52,
+	[1][0][RTW89_IC][11] = 66,
+	[1][0][RTW89_KCC][11] = 58,
+	[1][0][RTW89_ACMA][11] = 42,
+	[1][0][RTW89_CN][11] = 44,
+	[1][0][RTW89_UK][11] = 42,
+	[1][0][RTW89_FCC][12] = 34,
+	[1][0][RTW89_ETSI][12] = 42,
+	[1][0][RTW89_MKK][12] = 52,
+	[1][0][RTW89_IC][12] = 34,
+	[1][0][RTW89_KCC][12] = 58,
+	[1][0][RTW89_ACMA][12] = 42,
+	[1][0][RTW89_CN][12] = 42,
+	[1][0][RTW89_UK][12] = 42,
+	[1][0][RTW89_FCC][13] = 127,
+	[1][0][RTW89_ETSI][13] = 127,
+	[1][0][RTW89_MKK][13] = 127,
+	[1][0][RTW89_IC][13] = 127,
+	[1][0][RTW89_KCC][13] = 127,
+	[1][0][RTW89_ACMA][13] = 127,
+	[1][0][RTW89_CN][13] = 127,
+	[1][0][RTW89_UK][13] = 127,
+	[1][1][RTW89_FCC][0] = 127,
+	[1][1][RTW89_ETSI][0] = 127,
+	[1][1][RTW89_MKK][0] = 127,
+	[1][1][RTW89_IC][0] = 127,
+	[1][1][RTW89_KCC][0] = 127,
+	[1][1][RTW89_ACMA][0] = 127,
+	[1][1][RTW89_CN][0] = 32,
+	[1][1][RTW89_UK][0] = 127,
+	[1][1][RTW89_FCC][1] = 127,
+	[1][1][RTW89_ETSI][1] = 127,
+	[1][1][RTW89_MKK][1] = 127,
+	[1][1][RTW89_IC][1] = 127,
+	[1][1][RTW89_KCC][1] = 127,
+	[1][1][RTW89_ACMA][1] = 127,
+	[1][1][RTW89_CN][1] = 32,
+	[1][1][RTW89_UK][1] = 127,
+	[1][1][RTW89_FCC][2] = 127,
+	[1][1][RTW89_ETSI][2] = 127,
+	[1][1][RTW89_MKK][2] = 127,
+	[1][1][RTW89_IC][2] = 127,
+	[1][1][RTW89_KCC][2] = 127,
+	[1][1][RTW89_ACMA][2] = 127,
+	[1][1][RTW89_CN][2] = 32,
+	[1][1][RTW89_UK][2] = 127,
+	[1][1][RTW89_FCC][3] = 127,
+	[1][1][RTW89_ETSI][3] = 127,
+	[1][1][RTW89_MKK][3] = 127,
+	[1][1][RTW89_IC][3] = 127,
+	[1][1][RTW89_KCC][3] = 127,
+	[1][1][RTW89_ACMA][3] = 127,
+	[1][1][RTW89_CN][3] = 32,
+	[1][1][RTW89_UK][3] = 127,
+	[1][1][RTW89_FCC][4] = 127,
+	[1][1][RTW89_ETSI][4] = 127,
+	[1][1][RTW89_MKK][4] = 127,
+	[1][1][RTW89_IC][4] = 127,
+	[1][1][RTW89_KCC][4] = 127,
+	[1][1][RTW89_ACMA][4] = 127,
+	[1][1][RTW89_CN][4] = 32,
+	[1][1][RTW89_UK][4] = 127,
+	[1][1][RTW89_FCC][5] = 127,
+	[1][1][RTW89_ETSI][5] = 127,
+	[1][1][RTW89_MKK][5] = 127,
+	[1][1][RTW89_IC][5] = 127,
+	[1][1][RTW89_KCC][5] = 127,
+	[1][1][RTW89_ACMA][5] = 127,
+	[1][1][RTW89_CN][5] = 32,
+	[1][1][RTW89_UK][5] = 127,
+	[1][1][RTW89_FCC][6] = 127,
+	[1][1][RTW89_ETSI][6] = 127,
+	[1][1][RTW89_MKK][6] = 127,
+	[1][1][RTW89_IC][6] = 127,
+	[1][1][RTW89_KCC][6] = 127,
+	[1][1][RTW89_ACMA][6] = 127,
+	[1][1][RTW89_CN][6] = 32,
+	[1][1][RTW89_UK][6] = 127,
+	[1][1][RTW89_FCC][7] = 127,
+	[1][1][RTW89_ETSI][7] = 127,
+	[1][1][RTW89_MKK][7] = 127,
+	[1][1][RTW89_IC][7] = 127,
+	[1][1][RTW89_KCC][7] = 127,
+	[1][1][RTW89_ACMA][7] = 127,
+	[1][1][RTW89_CN][7] = 32,
+	[1][1][RTW89_UK][7] = 127,
+	[1][1][RTW89_FCC][8] = 127,
+	[1][1][RTW89_ETSI][8] = 127,
+	[1][1][RTW89_MKK][8] = 127,
+	[1][1][RTW89_IC][8] = 127,
+	[1][1][RTW89_KCC][8] = 127,
+	[1][1][RTW89_ACMA][8] = 127,
+	[1][1][RTW89_CN][8] = 32,
+	[1][1][RTW89_UK][8] = 127,
+	[1][1][RTW89_FCC][9] = 127,
+	[1][1][RTW89_ETSI][9] = 127,
+	[1][1][RTW89_MKK][9] = 127,
+	[1][1][RTW89_IC][9] = 127,
+	[1][1][RTW89_KCC][9] = 127,
+	[1][1][RTW89_ACMA][9] = 127,
+	[1][1][RTW89_CN][9] = 32,
+	[1][1][RTW89_UK][9] = 127,
+	[1][1][RTW89_FCC][10] = 127,
+	[1][1][RTW89_ETSI][10] = 127,
+	[1][1][RTW89_MKK][10] = 127,
+	[1][1][RTW89_IC][10] = 127,
+	[1][1][RTW89_KCC][10] = 127,
+	[1][1][RTW89_ACMA][10] = 127,
+	[1][1][RTW89_CN][10] = 32,
+	[1][1][RTW89_UK][10] = 127,
+	[1][1][RTW89_FCC][11] = 127,
+	[1][1][RTW89_ETSI][11] = 127,
+	[1][1][RTW89_MKK][11] = 127,
+	[1][1][RTW89_IC][11] = 127,
+	[1][1][RTW89_KCC][11] = 127,
+	[1][1][RTW89_ACMA][11] = 127,
+	[1][1][RTW89_CN][11] = 32,
+	[1][1][RTW89_UK][11] = 127,
+	[1][1][RTW89_FCC][12] = 127,
+	[1][1][RTW89_ETSI][12] = 127,
+	[1][1][RTW89_MKK][12] = 127,
+	[1][1][RTW89_IC][12] = 127,
+	[1][1][RTW89_KCC][12] = 127,
+	[1][1][RTW89_ACMA][12] = 127,
+	[1][1][RTW89_CN][12] = 32,
+	[1][1][RTW89_UK][12] = 127,
+	[1][1][RTW89_FCC][13] = 127,
+	[1][1][RTW89_ETSI][13] = 127,
+	[1][1][RTW89_MKK][13] = 127,
+	[1][1][RTW89_IC][13] = 127,
+	[1][1][RTW89_KCC][13] = 127,
+	[1][1][RTW89_ACMA][13] = 127,
+	[1][1][RTW89_CN][13] = 127,
+	[1][1][RTW89_UK][13] = 127,
+	[2][0][RTW89_FCC][0] = 76,
+	[2][0][RTW89_ETSI][0] = 54,
+	[2][0][RTW89_MKK][0] = 64,
+	[2][0][RTW89_IC][0] = 76,
+	[2][0][RTW89_KCC][0] = 68,
+	[2][0][RTW89_ACMA][0] = 54,
+	[2][0][RTW89_CN][0] = 56,
+	[2][0][RTW89_UK][0] = 54,
+	[2][0][RTW89_FCC][1] = 76,
+	[2][0][RTW89_ETSI][1] = 54,
+	[2][0][RTW89_MKK][1] = 64,
+	[2][0][RTW89_IC][1] = 76,
+	[2][0][RTW89_KCC][1] = 68,
+	[2][0][RTW89_ACMA][1] = 54,
+	[2][0][RTW89_CN][1] = 56,
+	[2][0][RTW89_UK][1] = 54,
+	[2][0][RTW89_FCC][2] = 78,
+	[2][0][RTW89_ETSI][2] = 54,
+	[2][0][RTW89_MKK][2] = 64,
+	[2][0][RTW89_IC][2] = 78,
+	[2][0][RTW89_KCC][2] = 68,
+	[2][0][RTW89_ACMA][2] = 54,
+	[2][0][RTW89_CN][2] = 56,
+	[2][0][RTW89_UK][2] = 54,
+	[2][0][RTW89_FCC][3] = 78,
+	[2][0][RTW89_ETSI][3] = 54,
+	[2][0][RTW89_MKK][3] = 64,
+	[2][0][RTW89_IC][3] = 78,
+	[2][0][RTW89_KCC][3] = 68,
+	[2][0][RTW89_ACMA][3] = 54,
+	[2][0][RTW89_CN][3] = 56,
+	[2][0][RTW89_UK][3] = 54,
+	[2][0][RTW89_FCC][4] = 78,
+	[2][0][RTW89_ETSI][4] = 54,
+	[2][0][RTW89_MKK][4] = 64,
+	[2][0][RTW89_IC][4] = 78,
+	[2][0][RTW89_KCC][4] = 68,
+	[2][0][RTW89_ACMA][4] = 54,
+	[2][0][RTW89_CN][4] = 56,
+	[2][0][RTW89_UK][4] = 54,
+	[2][0][RTW89_FCC][5] = 82,
+	[2][0][RTW89_ETSI][5] = 54,
+	[2][0][RTW89_MKK][5] = 64,
+	[2][0][RTW89_IC][5] = 82,
+	[2][0][RTW89_KCC][5] = 68,
+	[2][0][RTW89_ACMA][5] = 54,
+	[2][0][RTW89_CN][5] = 56,
+	[2][0][RTW89_UK][5] = 54,
+	[2][0][RTW89_FCC][6] = 74,
+	[2][0][RTW89_ETSI][6] = 54,
+	[2][0][RTW89_MKK][6] = 64,
+	[2][0][RTW89_IC][6] = 74,
+	[2][0][RTW89_KCC][6] = 68,
+	[2][0][RTW89_ACMA][6] = 54,
+	[2][0][RTW89_CN][6] = 56,
+	[2][0][RTW89_UK][6] = 54,
+	[2][0][RTW89_FCC][7] = 74,
+	[2][0][RTW89_ETSI][7] = 54,
+	[2][0][RTW89_MKK][7] = 64,
+	[2][0][RTW89_IC][7] = 74,
+	[2][0][RTW89_KCC][7] = 68,
+	[2][0][RTW89_ACMA][7] = 54,
+	[2][0][RTW89_CN][7] = 56,
+	[2][0][RTW89_UK][7] = 54,
+	[2][0][RTW89_FCC][8] = 74,
+	[2][0][RTW89_ETSI][8] = 54,
+	[2][0][RTW89_MKK][8] = 64,
+	[2][0][RTW89_IC][8] = 74,
+	[2][0][RTW89_KCC][8] = 68,
+	[2][0][RTW89_ACMA][8] = 54,
+	[2][0][RTW89_CN][8] = 56,
+	[2][0][RTW89_UK][8] = 54,
+	[2][0][RTW89_FCC][9] = 72,
+	[2][0][RTW89_ETSI][9] = 54,
+	[2][0][RTW89_MKK][9] = 64,
+	[2][0][RTW89_IC][9] = 72,
+	[2][0][RTW89_KCC][9] = 68,
+	[2][0][RTW89_ACMA][9] = 54,
+	[2][0][RTW89_CN][9] = 56,
+	[2][0][RTW89_UK][9] = 54,
+	[2][0][RTW89_FCC][10] = 72,
+	[2][0][RTW89_ETSI][10] = 54,
+	[2][0][RTW89_MKK][10] = 64,
+	[2][0][RTW89_IC][10] = 72,
+	[2][0][RTW89_KCC][10] = 68,
+	[2][0][RTW89_ACMA][10] = 54,
+	[2][0][RTW89_CN][10] = 56,
+	[2][0][RTW89_UK][10] = 54,
+	[2][0][RTW89_FCC][11] = 64,
+	[2][0][RTW89_ETSI][11] = 54,
+	[2][0][RTW89_MKK][11] = 64,
+	[2][0][RTW89_IC][11] = 64,
+	[2][0][RTW89_KCC][11] = 68,
+	[2][0][RTW89_ACMA][11] = 54,
+	[2][0][RTW89_CN][11] = 56,
+	[2][0][RTW89_UK][11] = 54,
+	[2][0][RTW89_FCC][12] = 34,
+	[2][0][RTW89_ETSI][12] = 54,
+	[2][0][RTW89_MKK][12] = 64,
+	[2][0][RTW89_IC][12] = 34,
+	[2][0][RTW89_KCC][12] = 68,
+	[2][0][RTW89_ACMA][12] = 54,
+	[2][0][RTW89_CN][12] = 56,
+	[2][0][RTW89_UK][12] = 54,
+	[2][0][RTW89_FCC][13] = 127,
+	[2][0][RTW89_ETSI][13] = 127,
+	[2][0][RTW89_MKK][13] = 127,
+	[2][0][RTW89_IC][13] = 127,
+	[2][0][RTW89_KCC][13] = 127,
+	[2][0][RTW89_ACMA][13] = 127,
+	[2][0][RTW89_CN][13] = 127,
+	[2][0][RTW89_UK][13] = 127,
+	[2][1][RTW89_FCC][0] = 127,
+	[2][1][RTW89_ETSI][0] = 127,
+	[2][1][RTW89_MKK][0] = 127,
+	[2][1][RTW89_IC][0] = 127,
+	[2][1][RTW89_KCC][0] = 127,
+	[2][1][RTW89_ACMA][0] = 127,
+	[2][1][RTW89_CN][0] = 44,
+	[2][1][RTW89_UK][0] = 127,
+	[2][1][RTW89_FCC][1] = 127,
+	[2][1][RTW89_ETSI][1] = 127,
+	[2][1][RTW89_MKK][1] = 127,
+	[2][1][RTW89_IC][1] = 127,
+	[2][1][RTW89_KCC][1] = 127,
+	[2][1][RTW89_ACMA][1] = 127,
+	[2][1][RTW89_CN][1] = 44,
+	[2][1][RTW89_UK][1] = 127,
+	[2][1][RTW89_FCC][2] = 127,
+	[2][1][RTW89_ETSI][2] = 127,
+	[2][1][RTW89_MKK][2] = 127,
+	[2][1][RTW89_IC][2] = 127,
+	[2][1][RTW89_KCC][2] = 127,
+	[2][1][RTW89_ACMA][2] = 127,
+	[2][1][RTW89_CN][2] = 44,
+	[2][1][RTW89_UK][2] = 127,
+	[2][1][RTW89_FCC][3] = 127,
+	[2][1][RTW89_ETSI][3] = 127,
+	[2][1][RTW89_MKK][3] = 127,
+	[2][1][RTW89_IC][3] = 127,
+	[2][1][RTW89_KCC][3] = 127,
+	[2][1][RTW89_ACMA][3] = 127,
+	[2][1][RTW89_CN][3] = 44,
+	[2][1][RTW89_UK][3] = 127,
+	[2][1][RTW89_FCC][4] = 127,
+	[2][1][RTW89_ETSI][4] = 127,
+	[2][1][RTW89_MKK][4] = 127,
+	[2][1][RTW89_IC][4] = 127,
+	[2][1][RTW89_KCC][4] = 127,
+	[2][1][RTW89_ACMA][4] = 127,
+	[2][1][RTW89_CN][4] = 44,
+	[2][1][RTW89_UK][4] = 127,
+	[2][1][RTW89_FCC][5] = 127,
+	[2][1][RTW89_ETSI][5] = 127,
+	[2][1][RTW89_MKK][5] = 127,
+	[2][1][RTW89_IC][5] = 127,
+	[2][1][RTW89_KCC][5] = 127,
+	[2][1][RTW89_ACMA][5] = 127,
+	[2][1][RTW89_CN][5] = 44,
+	[2][1][RTW89_UK][5] = 127,
+	[2][1][RTW89_FCC][6] = 127,
+	[2][1][RTW89_ETSI][6] = 127,
+	[2][1][RTW89_MKK][6] = 127,
+	[2][1][RTW89_IC][6] = 127,
+	[2][1][RTW89_KCC][6] = 127,
+	[2][1][RTW89_ACMA][6] = 127,
+	[2][1][RTW89_CN][6] = 44,
+	[2][1][RTW89_UK][6] = 127,
+	[2][1][RTW89_FCC][7] = 127,
+	[2][1][RTW89_ETSI][7] = 127,
+	[2][1][RTW89_MKK][7] = 127,
+	[2][1][RTW89_IC][7] = 127,
+	[2][1][RTW89_KCC][7] = 127,
+	[2][1][RTW89_ACMA][7] = 127,
+	[2][1][RTW89_CN][7] = 44,
+	[2][1][RTW89_UK][7] = 127,
+	[2][1][RTW89_FCC][8] = 127,
+	[2][1][RTW89_ETSI][8] = 127,
+	[2][1][RTW89_MKK][8] = 127,
+	[2][1][RTW89_IC][8] = 127,
+	[2][1][RTW89_KCC][8] = 127,
+	[2][1][RTW89_ACMA][8] = 127,
+	[2][1][RTW89_CN][8] = 44,
+	[2][1][RTW89_UK][8] = 127,
+	[2][1][RTW89_FCC][9] = 127,
+	[2][1][RTW89_ETSI][9] = 127,
+	[2][1][RTW89_MKK][9] = 127,
+	[2][1][RTW89_IC][9] = 127,
+	[2][1][RTW89_KCC][9] = 127,
+	[2][1][RTW89_ACMA][9] = 127,
+	[2][1][RTW89_CN][9] = 44,
+	[2][1][RTW89_UK][9] = 127,
+	[2][1][RTW89_FCC][10] = 127,
+	[2][1][RTW89_ETSI][10] = 127,
+	[2][1][RTW89_MKK][10] = 127,
+	[2][1][RTW89_IC][10] = 127,
+	[2][1][RTW89_KCC][10] = 127,
+	[2][1][RTW89_ACMA][10] = 127,
+	[2][1][RTW89_CN][10] = 44,
+	[2][1][RTW89_UK][10] = 127,
+	[2][1][RTW89_FCC][11] = 127,
+	[2][1][RTW89_ETSI][11] = 127,
+	[2][1][RTW89_MKK][11] = 127,
+	[2][1][RTW89_IC][11] = 127,
+	[2][1][RTW89_KCC][11] = 127,
+	[2][1][RTW89_ACMA][11] = 127,
+	[2][1][RTW89_CN][11] = 44,
+	[2][1][RTW89_UK][11] = 127,
+	[2][1][RTW89_FCC][12] = 127,
+	[2][1][RTW89_ETSI][12] = 127,
+	[2][1][RTW89_MKK][12] = 127,
+	[2][1][RTW89_IC][12] = 127,
+	[2][1][RTW89_KCC][12] = 127,
+	[2][1][RTW89_ACMA][12] = 127,
+	[2][1][RTW89_CN][12] = 42,
+	[2][1][RTW89_UK][12] = 127,
+	[2][1][RTW89_FCC][13] = 127,
+	[2][1][RTW89_ETSI][13] = 127,
+	[2][1][RTW89_MKK][13] = 127,
+	[2][1][RTW89_IC][13] = 127,
+	[2][1][RTW89_KCC][13] = 127,
+	[2][1][RTW89_ACMA][13] = 127,
+	[2][1][RTW89_CN][13] = 127,
+	[2][1][RTW89_UK][13] = 127,
+};
+
+static
+const s8 rtw89_8851b_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM]
+				    [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
+	[0][0][RTW89_WW][0] = 16,
+	[0][0][RTW89_WW][2] = 16,
+	[0][0][RTW89_WW][4] = 16,
+	[0][0][RTW89_WW][6] = 16,
+	[0][0][RTW89_WW][8] = 16,
+	[0][0][RTW89_WW][10] = 16,
+	[0][0][RTW89_WW][12] = 16,
+	[0][0][RTW89_WW][14] = 16,
+	[0][0][RTW89_WW][15] = 24,
+	[0][0][RTW89_WW][17] = 24,
+	[0][0][RTW89_WW][19] = 24,
+	[0][0][RTW89_WW][21] = 24,
+	[0][0][RTW89_WW][23] = 24,
+	[0][0][RTW89_WW][25] = 24,
+	[0][0][RTW89_WW][27] = 24,
+	[0][0][RTW89_WW][29] = 24,
+	[0][0][RTW89_WW][31] = 24,
+	[0][0][RTW89_WW][33] = 24,
+	[0][0][RTW89_WW][35] = 24,
+	[0][0][RTW89_WW][37] = 44,
+	[0][0][RTW89_WW][38] = 24,
+	[0][0][RTW89_WW][40] = 24,
+	[0][0][RTW89_WW][42] = 24,
+	[0][0][RTW89_WW][44] = 24,
+	[0][0][RTW89_WW][46] = 24,
+	[0][0][RTW89_WW][48] = 42,
+	[0][0][RTW89_WW][50] = 42,
+	[0][0][RTW89_WW][52] = 40,
+	[0][1][RTW89_WW][0] = 4,
+	[0][1][RTW89_WW][2] = 4,
+	[0][1][RTW89_WW][4] = 4,
+	[0][1][RTW89_WW][6] = 4,
+	[0][1][RTW89_WW][8] = 4,
+	[0][1][RTW89_WW][10] = 4,
+	[0][1][RTW89_WW][12] = 4,
+	[0][1][RTW89_WW][14] = 4,
+	[0][1][RTW89_WW][15] = 0,
+	[0][1][RTW89_WW][17] = 0,
+	[0][1][RTW89_WW][19] = 0,
+	[0][1][RTW89_WW][21] = 0,
+	[0][1][RTW89_WW][23] = 0,
+	[0][1][RTW89_WW][25] = 0,
+	[0][1][RTW89_WW][27] = 0,
+	[0][1][RTW89_WW][29] = 0,
+	[0][1][RTW89_WW][31] = 0,
+	[0][1][RTW89_WW][33] = 0,
+	[0][1][RTW89_WW][35] = 0,
+	[0][1][RTW89_WW][37] = 0,
+	[0][1][RTW89_WW][38] = 42,
+	[0][1][RTW89_WW][40] = 42,
+	[0][1][RTW89_WW][42] = 42,
+	[0][1][RTW89_WW][44] = 42,
+	[0][1][RTW89_WW][46] = 42,
+	[0][1][RTW89_WW][48] = 0,
+	[0][1][RTW89_WW][50] = 0,
+	[0][1][RTW89_WW][52] = 0,
+	[1][0][RTW89_WW][0] = 26,
+	[1][0][RTW89_WW][2] = 26,
+	[1][0][RTW89_WW][4] = 26,
+	[1][0][RTW89_WW][6] = 26,
+	[1][0][RTW89_WW][8] = 26,
+	[1][0][RTW89_WW][10] = 26,
+	[1][0][RTW89_WW][12] = 26,
+	[1][0][RTW89_WW][14] = 26,
+	[1][0][RTW89_WW][15] = 34,
+	[1][0][RTW89_WW][17] = 34,
+	[1][0][RTW89_WW][19] = 34,
+	[1][0][RTW89_WW][21] = 34,
+	[1][0][RTW89_WW][23] = 34,
+	[1][0][RTW89_WW][25] = 34,
+	[1][0][RTW89_WW][27] = 34,
+	[1][0][RTW89_WW][29] = 34,
+	[1][0][RTW89_WW][31] = 34,
+	[1][0][RTW89_WW][33] = 34,
+	[1][0][RTW89_WW][35] = 34,
+	[1][0][RTW89_WW][37] = 54,
+	[1][0][RTW89_WW][38] = 28,
+	[1][0][RTW89_WW][40] = 28,
+	[1][0][RTW89_WW][42] = 28,
+	[1][0][RTW89_WW][44] = 28,
+	[1][0][RTW89_WW][46] = 28,
+	[1][0][RTW89_WW][48] = 52,
+	[1][0][RTW89_WW][50] = 52,
+	[1][0][RTW89_WW][52] = 52,
+	[1][1][RTW89_WW][0] = 14,
+	[1][1][RTW89_WW][2] = 14,
+	[1][1][RTW89_WW][4] = 14,
+	[1][1][RTW89_WW][6] = 14,
+	[1][1][RTW89_WW][8] = 14,
+	[1][1][RTW89_WW][10] = 14,
+	[1][1][RTW89_WW][12] = 14,
+	[1][1][RTW89_WW][14] = 14,
+	[1][1][RTW89_WW][15] = 0,
+	[1][1][RTW89_WW][17] = 0,
+	[1][1][RTW89_WW][19] = 0,
+	[1][1][RTW89_WW][21] = 0,
+	[1][1][RTW89_WW][23] = 0,
+	[1][1][RTW89_WW][25] = 0,
+	[1][1][RTW89_WW][27] = 0,
+	[1][1][RTW89_WW][29] = 0,
+	[1][1][RTW89_WW][31] = 0,
+	[1][1][RTW89_WW][33] = 0,
+	[1][1][RTW89_WW][35] = 0,
+	[1][1][RTW89_WW][37] = 0,
+	[1][1][RTW89_WW][38] = 54,
+	[1][1][RTW89_WW][40] = 54,
+	[1][1][RTW89_WW][42] = 54,
+	[1][1][RTW89_WW][44] = 54,
+	[1][1][RTW89_WW][46] = 54,
+	[1][1][RTW89_WW][48] = 0,
+	[1][1][RTW89_WW][50] = 0,
+	[1][1][RTW89_WW][52] = 0,
+	[2][0][RTW89_WW][0] = 40,
+	[2][0][RTW89_WW][2] = 40,
+	[2][0][RTW89_WW][4] = 40,
+	[2][0][RTW89_WW][6] = 40,
+	[2][0][RTW89_WW][8] = 40,
+	[2][0][RTW89_WW][10] = 40,
+	[2][0][RTW89_WW][12] = 40,
+	[2][0][RTW89_WW][14] = 40,
+	[2][0][RTW89_WW][15] = 46,
+	[2][0][RTW89_WW][17] = 46,
+	[2][0][RTW89_WW][19] = 46,
+	[2][0][RTW89_WW][21] = 46,
+	[2][0][RTW89_WW][23] = 46,
+	[2][0][RTW89_WW][25] = 46,
+	[2][0][RTW89_WW][27] = 46,
+	[2][0][RTW89_WW][29] = 46,
+	[2][0][RTW89_WW][31] = 46,
+	[2][0][RTW89_WW][33] = 46,
+	[2][0][RTW89_WW][35] = 46,
+	[2][0][RTW89_WW][37] = 66,
+	[2][0][RTW89_WW][38] = 28,
+	[2][0][RTW89_WW][40] = 28,
+	[2][0][RTW89_WW][42] = 28,
+	[2][0][RTW89_WW][44] = 28,
+	[2][0][RTW89_WW][46] = 28,
+	[2][0][RTW89_WW][48] = 64,
+	[2][0][RTW89_WW][50] = 64,
+	[2][0][RTW89_WW][52] = 60,
+	[2][1][RTW89_WW][0] = 28,
+	[2][1][RTW89_WW][2] = 28,
+	[2][1][RTW89_WW][4] = 28,
+	[2][1][RTW89_WW][6] = 28,
+	[2][1][RTW89_WW][8] = 28,
+	[2][1][RTW89_WW][10] = 28,
+	[2][1][RTW89_WW][12] = 28,
+	[2][1][RTW89_WW][14] = 28,
+	[2][1][RTW89_WW][15] = 0,
+	[2][1][RTW89_WW][17] = 0,
+	[2][1][RTW89_WW][19] = 0,
+	[2][1][RTW89_WW][21] = 0,
+	[2][1][RTW89_WW][23] = 0,
+	[2][1][RTW89_WW][25] = 0,
+	[2][1][RTW89_WW][27] = 0,
+	[2][1][RTW89_WW][29] = 0,
+	[2][1][RTW89_WW][31] = 0,
+	[2][1][RTW89_WW][33] = 0,
+	[2][1][RTW89_WW][35] = 0,
+	[2][1][RTW89_WW][37] = 0,
+	[2][1][RTW89_WW][38] = 56,
+	[2][1][RTW89_WW][40] = 56,
+	[2][1][RTW89_WW][42] = 56,
+	[2][1][RTW89_WW][44] = 56,
+	[2][1][RTW89_WW][46] = 56,
+	[2][1][RTW89_WW][48] = 0,
+	[2][1][RTW89_WW][50] = 0,
+	[2][1][RTW89_WW][52] = 0,
+	[0][0][RTW89_FCC][0] = 52,
+	[0][0][RTW89_ETSI][0] = 24,
+	[0][0][RTW89_MKK][0] = 26,
+	[0][0][RTW89_IC][0] = 28,
+	[0][0][RTW89_KCC][0] = 42,
+	[0][0][RTW89_ACMA][0] = 24,
+	[0][0][RTW89_CN][0] = 16,
+	[0][0][RTW89_UK][0] = 24,
+	[0][0][RTW89_FCC][2] = 54,
+	[0][0][RTW89_ETSI][2] = 24,
+	[0][0][RTW89_MKK][2] = 26,
+	[0][0][RTW89_IC][2] = 28,
+	[0][0][RTW89_KCC][2] = 42,
+	[0][0][RTW89_ACMA][2] = 24,
+	[0][0][RTW89_CN][2] = 16,
+	[0][0][RTW89_UK][2] = 24,
+	[0][0][RTW89_FCC][4] = 52,
+	[0][0][RTW89_ETSI][4] = 24,
+	[0][0][RTW89_MKK][4] = 26,
+	[0][0][RTW89_IC][4] = 28,
+	[0][0][RTW89_KCC][4] = 42,
+	[0][0][RTW89_ACMA][4] = 24,
+	[0][0][RTW89_CN][4] = 16,
+	[0][0][RTW89_UK][4] = 24,
+	[0][0][RTW89_FCC][6] = 52,
+	[0][0][RTW89_ETSI][6] = 24,
+	[0][0][RTW89_MKK][6] = 26,
+	[0][0][RTW89_IC][6] = 28,
+	[0][0][RTW89_KCC][6] = 18,
+	[0][0][RTW89_ACMA][6] = 24,
+	[0][0][RTW89_CN][6] = 16,
+	[0][0][RTW89_UK][6] = 24,
+	[0][0][RTW89_FCC][8] = 52,
+	[0][0][RTW89_ETSI][8] = 24,
+	[0][0][RTW89_MKK][8] = 26,
+	[0][0][RTW89_IC][8] = 52,
+	[0][0][RTW89_KCC][8] = 42,
+	[0][0][RTW89_ACMA][8] = 24,
+	[0][0][RTW89_CN][8] = 16,
+	[0][0][RTW89_UK][8] = 24,
+	[0][0][RTW89_FCC][10] = 52,
+	[0][0][RTW89_ETSI][10] = 24,
+	[0][0][RTW89_MKK][10] = 26,
+	[0][0][RTW89_IC][10] = 52,
+	[0][0][RTW89_KCC][10] = 42,
+	[0][0][RTW89_ACMA][10] = 24,
+	[0][0][RTW89_CN][10] = 16,
+	[0][0][RTW89_UK][10] = 24,
+	[0][0][RTW89_FCC][12] = 56,
+	[0][0][RTW89_ETSI][12] = 24,
+	[0][0][RTW89_MKK][12] = 26,
+	[0][0][RTW89_IC][12] = 56,
+	[0][0][RTW89_KCC][12] = 44,
+	[0][0][RTW89_ACMA][12] = 24,
+	[0][0][RTW89_CN][12] = 16,
+	[0][0][RTW89_UK][12] = 24,
+	[0][0][RTW89_FCC][14] = 56,
+	[0][0][RTW89_ETSI][14] = 24,
+	[0][0][RTW89_MKK][14] = 26,
+	[0][0][RTW89_IC][14] = 56,
+	[0][0][RTW89_KCC][14] = 44,
+	[0][0][RTW89_ACMA][14] = 24,
+	[0][0][RTW89_CN][14] = 16,
+	[0][0][RTW89_UK][14] = 24,
+	[0][0][RTW89_FCC][15] = 54,
+	[0][0][RTW89_ETSI][15] = 24,
+	[0][0][RTW89_MKK][15] = 46,
+	[0][0][RTW89_IC][15] = 54,
+	[0][0][RTW89_KCC][15] = 44,
+	[0][0][RTW89_ACMA][15] = 24,
+	[0][0][RTW89_CN][15] = 127,
+	[0][0][RTW89_UK][15] = 24,
+	[0][0][RTW89_FCC][17] = 54,
+	[0][0][RTW89_ETSI][17] = 24,
+	[0][0][RTW89_MKK][17] = 50,
+	[0][0][RTW89_IC][17] = 54,
+	[0][0][RTW89_KCC][17] = 44,
+	[0][0][RTW89_ACMA][17] = 24,
+	[0][0][RTW89_CN][17] = 127,
+	[0][0][RTW89_UK][17] = 24,
+	[0][0][RTW89_FCC][19] = 54,
+	[0][0][RTW89_ETSI][19] = 24,
+	[0][0][RTW89_MKK][19] = 50,
+	[0][0][RTW89_IC][19] = 54,
+	[0][0][RTW89_KCC][19] = 44,
+	[0][0][RTW89_ACMA][19] = 24,
+	[0][0][RTW89_CN][19] = 127,
+	[0][0][RTW89_UK][19] = 24,
+	[0][0][RTW89_FCC][21] = 54,
+	[0][0][RTW89_ETSI][21] = 24,
+	[0][0][RTW89_MKK][21] = 50,
+	[0][0][RTW89_IC][21] = 54,
+	[0][0][RTW89_KCC][21] = 44,
+	[0][0][RTW89_ACMA][21] = 24,
+	[0][0][RTW89_CN][21] = 127,
+	[0][0][RTW89_UK][21] = 24,
+	[0][0][RTW89_FCC][23] = 54,
+	[0][0][RTW89_ETSI][23] = 24,
+	[0][0][RTW89_MKK][23] = 50,
+	[0][0][RTW89_IC][23] = 54,
+	[0][0][RTW89_KCC][23] = 44,
+	[0][0][RTW89_ACMA][23] = 24,
+	[0][0][RTW89_CN][23] = 127,
+	[0][0][RTW89_UK][23] = 24,
+	[0][0][RTW89_FCC][25] = 54,
+	[0][0][RTW89_ETSI][25] = 24,
+	[0][0][RTW89_MKK][25] = 50,
+	[0][0][RTW89_IC][25] = 127,
+	[0][0][RTW89_KCC][25] = 44,
+	[0][0][RTW89_ACMA][25] = 127,
+	[0][0][RTW89_CN][25] = 127,
+	[0][0][RTW89_UK][25] = 24,
+	[0][0][RTW89_FCC][27] = 54,
+	[0][0][RTW89_ETSI][27] = 24,
+	[0][0][RTW89_MKK][27] = 50,
+	[0][0][RTW89_IC][27] = 127,
+	[0][0][RTW89_KCC][27] = 42,
+	[0][0][RTW89_ACMA][27] = 127,
+	[0][0][RTW89_CN][27] = 127,
+	[0][0][RTW89_UK][27] = 24,
+	[0][0][RTW89_FCC][29] = 54,
+	[0][0][RTW89_ETSI][29] = 24,
+	[0][0][RTW89_MKK][29] = 50,
+	[0][0][RTW89_IC][29] = 127,
+	[0][0][RTW89_KCC][29] = 42,
+	[0][0][RTW89_ACMA][29] = 127,
+	[0][0][RTW89_CN][29] = 127,
+	[0][0][RTW89_UK][29] = 24,
+	[0][0][RTW89_FCC][31] = 54,
+	[0][0][RTW89_ETSI][31] = 24,
+	[0][0][RTW89_MKK][31] = 50,
+	[0][0][RTW89_IC][31] = 56,
+	[0][0][RTW89_KCC][31] = 42,
+	[0][0][RTW89_ACMA][31] = 24,
+	[0][0][RTW89_CN][31] = 127,
+	[0][0][RTW89_UK][31] = 24,
+	[0][0][RTW89_FCC][33] = 56,
+	[0][0][RTW89_ETSI][33] = 24,
+	[0][0][RTW89_MKK][33] = 50,
+	[0][0][RTW89_IC][33] = 56,
+	[0][0][RTW89_KCC][33] = 42,
+	[0][0][RTW89_ACMA][33] = 24,
+	[0][0][RTW89_CN][33] = 127,
+	[0][0][RTW89_UK][33] = 24,
+	[0][0][RTW89_FCC][35] = 56,
+	[0][0][RTW89_ETSI][35] = 24,
+	[0][0][RTW89_MKK][35] = 50,
+	[0][0][RTW89_IC][35] = 56,
+	[0][0][RTW89_KCC][35] = 42,
+	[0][0][RTW89_ACMA][35] = 24,
+	[0][0][RTW89_CN][35] = 127,
+	[0][0][RTW89_UK][35] = 24,
+	[0][0][RTW89_FCC][37] = 86,
+	[0][0][RTW89_ETSI][37] = 127,
+	[0][0][RTW89_MKK][37] = 46,
+	[0][0][RTW89_IC][37] = 86,
+	[0][0][RTW89_KCC][37] = 44,
+	[0][0][RTW89_ACMA][37] = 50,
+	[0][0][RTW89_CN][37] = 127,
+	[0][0][RTW89_UK][37] = 52,
+	[0][0][RTW89_FCC][38] = 68,
+	[0][0][RTW89_ETSI][38] = 28,
+	[0][0][RTW89_MKK][38] = 127,
+	[0][0][RTW89_IC][38] = 68,
+	[0][0][RTW89_KCC][38] = 44,
+	[0][0][RTW89_ACMA][38] = 84,
+	[0][0][RTW89_CN][38] = 54,
+	[0][0][RTW89_UK][38] = 24,
+	[0][0][RTW89_FCC][40] = 68,
+	[0][0][RTW89_ETSI][40] = 28,
+	[0][0][RTW89_MKK][40] = 127,
+	[0][0][RTW89_IC][40] = 68,
+	[0][0][RTW89_KCC][40] = 44,
+	[0][0][RTW89_ACMA][40] = 84,
+	[0][0][RTW89_CN][40] = 54,
+	[0][0][RTW89_UK][40] = 24,
+	[0][0][RTW89_FCC][42] = 70,
+	[0][0][RTW89_ETSI][42] = 28,
+	[0][0][RTW89_MKK][42] = 127,
+	[0][0][RTW89_IC][42] = 70,
+	[0][0][RTW89_KCC][42] = 44,
+	[0][0][RTW89_ACMA][42] = 84,
+	[0][0][RTW89_CN][42] = 54,
+	[0][0][RTW89_UK][42] = 24,
+	[0][0][RTW89_FCC][44] = 62,
+	[0][0][RTW89_ETSI][44] = 28,
+	[0][0][RTW89_MKK][44] = 127,
+	[0][0][RTW89_IC][44] = 62,
+	[0][0][RTW89_KCC][44] = 44,
+	[0][0][RTW89_ACMA][44] = 84,
+	[0][0][RTW89_CN][44] = 54,
+	[0][0][RTW89_UK][44] = 24,
+	[0][0][RTW89_FCC][46] = 62,
+	[0][0][RTW89_ETSI][46] = 28,
+	[0][0][RTW89_MKK][46] = 127,
+	[0][0][RTW89_IC][46] = 62,
+	[0][0][RTW89_KCC][46] = 44,
+	[0][0][RTW89_ACMA][46] = 84,
+	[0][0][RTW89_CN][46] = 54,
+	[0][0][RTW89_UK][46] = 24,
+	[0][0][RTW89_FCC][48] = 42,
+	[0][0][RTW89_ETSI][48] = 127,
+	[0][0][RTW89_MKK][48] = 127,
+	[0][0][RTW89_IC][48] = 127,
+	[0][0][RTW89_KCC][48] = 127,
+	[0][0][RTW89_ACMA][48] = 127,
+	[0][0][RTW89_CN][48] = 127,
+	[0][0][RTW89_UK][48] = 127,
+	[0][0][RTW89_FCC][50] = 42,
+	[0][0][RTW89_ETSI][50] = 127,
+	[0][0][RTW89_MKK][50] = 127,
+	[0][0][RTW89_IC][50] = 127,
+	[0][0][RTW89_KCC][50] = 127,
+	[0][0][RTW89_ACMA][50] = 127,
+	[0][0][RTW89_CN][50] = 127,
+	[0][0][RTW89_UK][50] = 127,
+	[0][0][RTW89_FCC][52] = 40,
+	[0][0][RTW89_ETSI][52] = 127,
+	[0][0][RTW89_MKK][52] = 127,
+	[0][0][RTW89_IC][52] = 127,
+	[0][0][RTW89_KCC][52] = 127,
+	[0][0][RTW89_ACMA][52] = 127,
+	[0][0][RTW89_CN][52] = 127,
+	[0][0][RTW89_UK][52] = 127,
+	[0][1][RTW89_FCC][0] = 127,
+	[0][1][RTW89_ETSI][0] = 127,
+	[0][1][RTW89_MKK][0] = 127,
+	[0][1][RTW89_IC][0] = 127,
+	[0][1][RTW89_KCC][0] = 127,
+	[0][1][RTW89_ACMA][0] = 127,
+	[0][1][RTW89_CN][0] = 4,
+	[0][1][RTW89_UK][0] = 127,
+	[0][1][RTW89_FCC][2] = 127,
+	[0][1][RTW89_ETSI][2] = 127,
+	[0][1][RTW89_MKK][2] = 127,
+	[0][1][RTW89_IC][2] = 127,
+	[0][1][RTW89_KCC][2] = 127,
+	[0][1][RTW89_ACMA][2] = 127,
+	[0][1][RTW89_CN][2] = 4,
+	[0][1][RTW89_UK][2] = 127,
+	[0][1][RTW89_FCC][4] = 127,
+	[0][1][RTW89_ETSI][4] = 127,
+	[0][1][RTW89_MKK][4] = 127,
+	[0][1][RTW89_IC][4] = 127,
+	[0][1][RTW89_KCC][4] = 127,
+	[0][1][RTW89_ACMA][4] = 127,
+	[0][1][RTW89_CN][4] = 4,
+	[0][1][RTW89_UK][4] = 127,
+	[0][1][RTW89_FCC][6] = 127,
+	[0][1][RTW89_ETSI][6] = 127,
+	[0][1][RTW89_MKK][6] = 127,
+	[0][1][RTW89_IC][6] = 127,
+	[0][1][RTW89_KCC][6] = 127,
+	[0][1][RTW89_ACMA][6] = 127,
+	[0][1][RTW89_CN][6] = 4,
+	[0][1][RTW89_UK][6] = 127,
+	[0][1][RTW89_FCC][8] = 127,
+	[0][1][RTW89_ETSI][8] = 127,
+	[0][1][RTW89_MKK][8] = 127,
+	[0][1][RTW89_IC][8] = 127,
+	[0][1][RTW89_KCC][8] = 127,
+	[0][1][RTW89_ACMA][8] = 127,
+	[0][1][RTW89_CN][8] = 4,
+	[0][1][RTW89_UK][8] = 127,
+	[0][1][RTW89_FCC][10] = 127,
+	[0][1][RTW89_ETSI][10] = 127,
+	[0][1][RTW89_MKK][10] = 127,
+	[0][1][RTW89_IC][10] = 127,
+	[0][1][RTW89_KCC][10] = 127,
+	[0][1][RTW89_ACMA][10] = 127,
+	[0][1][RTW89_CN][10] = 4,
+	[0][1][RTW89_UK][10] = 127,
+	[0][1][RTW89_FCC][12] = 127,
+	[0][1][RTW89_ETSI][12] = 127,
+	[0][1][RTW89_MKK][12] = 127,
+	[0][1][RTW89_IC][12] = 127,
+	[0][1][RTW89_KCC][12] = 127,
+	[0][1][RTW89_ACMA][12] = 127,
+	[0][1][RTW89_CN][12] = 4,
+	[0][1][RTW89_UK][12] = 127,
+	[0][1][RTW89_FCC][14] = 127,
+	[0][1][RTW89_ETSI][14] = 127,
+	[0][1][RTW89_MKK][14] = 127,
+	[0][1][RTW89_IC][14] = 127,
+	[0][1][RTW89_KCC][14] = 127,
+	[0][1][RTW89_ACMA][14] = 127,
+	[0][1][RTW89_CN][14] = 4,
+	[0][1][RTW89_UK][14] = 127,
+	[0][1][RTW89_FCC][15] = 127,
+	[0][1][RTW89_ETSI][15] = 127,
+	[0][1][RTW89_MKK][15] = 127,
+	[0][1][RTW89_IC][15] = 127,
+	[0][1][RTW89_KCC][15] = 127,
+	[0][1][RTW89_ACMA][15] = 127,
+	[0][1][RTW89_CN][15] = 127,
+	[0][1][RTW89_UK][15] = 127,
+	[0][1][RTW89_FCC][17] = 127,
+	[0][1][RTW89_ETSI][17] = 127,
+	[0][1][RTW89_MKK][17] = 127,
+	[0][1][RTW89_IC][17] = 127,
+	[0][1][RTW89_KCC][17] = 127,
+	[0][1][RTW89_ACMA][17] = 127,
+	[0][1][RTW89_CN][17] = 127,
+	[0][1][RTW89_UK][17] = 127,
+	[0][1][RTW89_FCC][19] = 127,
+	[0][1][RTW89_ETSI][19] = 127,
+	[0][1][RTW89_MKK][19] = 127,
+	[0][1][RTW89_IC][19] = 127,
+	[0][1][RTW89_KCC][19] = 127,
+	[0][1][RTW89_ACMA][19] = 127,
+	[0][1][RTW89_CN][19] = 127,
+	[0][1][RTW89_UK][19] = 127,
+	[0][1][RTW89_FCC][21] = 127,
+	[0][1][RTW89_ETSI][21] = 127,
+	[0][1][RTW89_MKK][21] = 127,
+	[0][1][RTW89_IC][21] = 127,
+	[0][1][RTW89_KCC][21] = 127,
+	[0][1][RTW89_ACMA][21] = 127,
+	[0][1][RTW89_CN][21] = 127,
+	[0][1][RTW89_UK][21] = 127,
+	[0][1][RTW89_FCC][23] = 127,
+	[0][1][RTW89_ETSI][23] = 127,
+	[0][1][RTW89_MKK][23] = 127,
+	[0][1][RTW89_IC][23] = 127,
+	[0][1][RTW89_KCC][23] = 127,
+	[0][1][RTW89_ACMA][23] = 127,
+	[0][1][RTW89_CN][23] = 127,
+	[0][1][RTW89_UK][23] = 127,
+	[0][1][RTW89_FCC][25] = 127,
+	[0][1][RTW89_ETSI][25] = 127,
+	[0][1][RTW89_MKK][25] = 127,
+	[0][1][RTW89_IC][25] = 127,
+	[0][1][RTW89_KCC][25] = 127,
+	[0][1][RTW89_ACMA][25] = 127,
+	[0][1][RTW89_CN][25] = 127,
+	[0][1][RTW89_UK][25] = 127,
+	[0][1][RTW89_FCC][27] = 127,
+	[0][1][RTW89_ETSI][27] = 127,
+	[0][1][RTW89_MKK][27] = 127,
+	[0][1][RTW89_IC][27] = 127,
+	[0][1][RTW89_KCC][27] = 127,
+	[0][1][RTW89_ACMA][27] = 127,
+	[0][1][RTW89_CN][27] = 127,
+	[0][1][RTW89_UK][27] = 127,
+	[0][1][RTW89_FCC][29] = 127,
+	[0][1][RTW89_ETSI][29] = 127,
+	[0][1][RTW89_MKK][29] = 127,
+	[0][1][RTW89_IC][29] = 127,
+	[0][1][RTW89_KCC][29] = 127,
+	[0][1][RTW89_ACMA][29] = 127,
+	[0][1][RTW89_CN][29] = 127,
+	[0][1][RTW89_UK][29] = 127,
+	[0][1][RTW89_FCC][31] = 127,
+	[0][1][RTW89_ETSI][31] = 127,
+	[0][1][RTW89_MKK][31] = 127,
+	[0][1][RTW89_IC][31] = 127,
+	[0][1][RTW89_KCC][31] = 127,
+	[0][1][RTW89_ACMA][31] = 127,
+	[0][1][RTW89_CN][31] = 127,
+	[0][1][RTW89_UK][31] = 127,
+	[0][1][RTW89_FCC][33] = 127,
+	[0][1][RTW89_ETSI][33] = 127,
+	[0][1][RTW89_MKK][33] = 127,
+	[0][1][RTW89_IC][33] = 127,
+	[0][1][RTW89_KCC][33] = 127,
+	[0][1][RTW89_ACMA][33] = 127,
+	[0][1][RTW89_CN][33] = 127,
+	[0][1][RTW89_UK][33] = 127,
+	[0][1][RTW89_FCC][35] = 127,
+	[0][1][RTW89_ETSI][35] = 127,
+	[0][1][RTW89_MKK][35] = 127,
+	[0][1][RTW89_IC][35] = 127,
+	[0][1][RTW89_KCC][35] = 127,
+	[0][1][RTW89_ACMA][35] = 127,
+	[0][1][RTW89_CN][35] = 127,
+	[0][1][RTW89_UK][35] = 127,
+	[0][1][RTW89_FCC][37] = 127,
+	[0][1][RTW89_ETSI][37] = 127,
+	[0][1][RTW89_MKK][37] = 127,
+	[0][1][RTW89_IC][37] = 127,
+	[0][1][RTW89_KCC][37] = 127,
+	[0][1][RTW89_ACMA][37] = 127,
+	[0][1][RTW89_CN][37] = 127,
+	[0][1][RTW89_UK][37] = 127,
+	[0][1][RTW89_FCC][38] = 127,
+	[0][1][RTW89_ETSI][38] = 127,
+	[0][1][RTW89_MKK][38] = 127,
+	[0][1][RTW89_IC][38] = 127,
+	[0][1][RTW89_KCC][38] = 127,
+	[0][1][RTW89_ACMA][38] = 127,
+	[0][1][RTW89_CN][38] = 42,
+	[0][1][RTW89_UK][38] = 127,
+	[0][1][RTW89_FCC][40] = 127,
+	[0][1][RTW89_ETSI][40] = 127,
+	[0][1][RTW89_MKK][40] = 127,
+	[0][1][RTW89_IC][40] = 127,
+	[0][1][RTW89_KCC][40] = 127,
+	[0][1][RTW89_ACMA][40] = 127,
+	[0][1][RTW89_CN][40] = 42,
+	[0][1][RTW89_UK][40] = 127,
+	[0][1][RTW89_FCC][42] = 127,
+	[0][1][RTW89_ETSI][42] = 127,
+	[0][1][RTW89_MKK][42] = 127,
+	[0][1][RTW89_IC][42] = 127,
+	[0][1][RTW89_KCC][42] = 127,
+	[0][1][RTW89_ACMA][42] = 127,
+	[0][1][RTW89_CN][42] = 42,
+	[0][1][RTW89_UK][42] = 127,
+	[0][1][RTW89_FCC][44] = 127,
+	[0][1][RTW89_ETSI][44] = 127,
+	[0][1][RTW89_MKK][44] = 127,
+	[0][1][RTW89_IC][44] = 127,
+	[0][1][RTW89_KCC][44] = 127,
+	[0][1][RTW89_ACMA][44] = 127,
+	[0][1][RTW89_CN][44] = 42,
+	[0][1][RTW89_UK][44] = 127,
+	[0][1][RTW89_FCC][46] = 127,
+	[0][1][RTW89_ETSI][46] = 127,
+	[0][1][RTW89_MKK][46] = 127,
+	[0][1][RTW89_IC][46] = 127,
+	[0][1][RTW89_KCC][46] = 127,
+	[0][1][RTW89_ACMA][46] = 127,
+	[0][1][RTW89_CN][46] = 42,
+	[0][1][RTW89_UK][46] = 127,
+	[0][1][RTW89_FCC][48] = 127,
+	[0][1][RTW89_ETSI][48] = 127,
+	[0][1][RTW89_MKK][48] = 127,
+	[0][1][RTW89_IC][48] = 127,
+	[0][1][RTW89_KCC][48] = 127,
+	[0][1][RTW89_ACMA][48] = 127,
+	[0][1][RTW89_CN][48] = 127,
+	[0][1][RTW89_UK][48] = 127,
+	[0][1][RTW89_FCC][50] = 127,
+	[0][1][RTW89_ETSI][50] = 127,
+	[0][1][RTW89_MKK][50] = 127,
+	[0][1][RTW89_IC][50] = 127,
+	[0][1][RTW89_KCC][50] = 127,
+	[0][1][RTW89_ACMA][50] = 127,
+	[0][1][RTW89_CN][50] = 127,
+	[0][1][RTW89_UK][50] = 127,
+	[0][1][RTW89_FCC][52] = 127,
+	[0][1][RTW89_ETSI][52] = 127,
+	[0][1][RTW89_MKK][52] = 127,
+	[0][1][RTW89_IC][52] = 127,
+	[0][1][RTW89_KCC][52] = 127,
+	[0][1][RTW89_ACMA][52] = 127,
+	[0][1][RTW89_CN][52] = 127,
+	[0][1][RTW89_UK][52] = 127,
+	[1][0][RTW89_FCC][0] = 64,
+	[1][0][RTW89_ETSI][0] = 34,
+	[1][0][RTW89_MKK][0] = 38,
+	[1][0][RTW89_IC][0] = 38,
+	[1][0][RTW89_KCC][0] = 52,
+	[1][0][RTW89_ACMA][0] = 34,
+	[1][0][RTW89_CN][0] = 26,
+	[1][0][RTW89_UK][0] = 34,
+	[1][0][RTW89_FCC][2] = 66,
+	[1][0][RTW89_ETSI][2] = 34,
+	[1][0][RTW89_MKK][2] = 38,
+	[1][0][RTW89_IC][2] = 38,
+	[1][0][RTW89_KCC][2] = 52,
+	[1][0][RTW89_ACMA][2] = 34,
+	[1][0][RTW89_CN][2] = 26,
+	[1][0][RTW89_UK][2] = 34,
+	[1][0][RTW89_FCC][4] = 62,
+	[1][0][RTW89_ETSI][4] = 34,
+	[1][0][RTW89_MKK][4] = 36,
+	[1][0][RTW89_IC][4] = 38,
+	[1][0][RTW89_KCC][4] = 52,
+	[1][0][RTW89_ACMA][4] = 34,
+	[1][0][RTW89_CN][4] = 26,
+	[1][0][RTW89_UK][4] = 34,
+	[1][0][RTW89_FCC][6] = 62,
+	[1][0][RTW89_ETSI][6] = 34,
+	[1][0][RTW89_MKK][6] = 36,
+	[1][0][RTW89_IC][6] = 38,
+	[1][0][RTW89_KCC][6] = 32,
+	[1][0][RTW89_ACMA][6] = 34,
+	[1][0][RTW89_CN][6] = 26,
+	[1][0][RTW89_UK][6] = 34,
+	[1][0][RTW89_FCC][8] = 62,
+	[1][0][RTW89_ETSI][8] = 34,
+	[1][0][RTW89_MKK][8] = 38,
+	[1][0][RTW89_IC][8] = 62,
+	[1][0][RTW89_KCC][8] = 52,
+	[1][0][RTW89_ACMA][8] = 34,
+	[1][0][RTW89_CN][8] = 26,
+	[1][0][RTW89_UK][8] = 34,
+	[1][0][RTW89_FCC][10] = 62,
+	[1][0][RTW89_ETSI][10] = 34,
+	[1][0][RTW89_MKK][10] = 38,
+	[1][0][RTW89_IC][10] = 62,
+	[1][0][RTW89_KCC][10] = 52,
+	[1][0][RTW89_ACMA][10] = 34,
+	[1][0][RTW89_CN][10] = 26,
+	[1][0][RTW89_UK][10] = 34,
+	[1][0][RTW89_FCC][12] = 62,
+	[1][0][RTW89_ETSI][12] = 34,
+	[1][0][RTW89_MKK][12] = 38,
+	[1][0][RTW89_IC][12] = 62,
+	[1][0][RTW89_KCC][12] = 54,
+	[1][0][RTW89_ACMA][12] = 34,
+	[1][0][RTW89_CN][12] = 26,
+	[1][0][RTW89_UK][12] = 34,
+	[1][0][RTW89_FCC][14] = 64,
+	[1][0][RTW89_ETSI][14] = 34,
+	[1][0][RTW89_MKK][14] = 38,
+	[1][0][RTW89_IC][14] = 64,
+	[1][0][RTW89_KCC][14] = 54,
+	[1][0][RTW89_ACMA][14] = 34,
+	[1][0][RTW89_CN][14] = 26,
+	[1][0][RTW89_UK][14] = 34,
+	[1][0][RTW89_FCC][15] = 62,
+	[1][0][RTW89_ETSI][15] = 34,
+	[1][0][RTW89_MKK][15] = 58,
+	[1][0][RTW89_IC][15] = 62,
+	[1][0][RTW89_KCC][15] = 54,
+	[1][0][RTW89_ACMA][15] = 34,
+	[1][0][RTW89_CN][15] = 127,
+	[1][0][RTW89_UK][15] = 34,
+	[1][0][RTW89_FCC][17] = 62,
+	[1][0][RTW89_ETSI][17] = 34,
+	[1][0][RTW89_MKK][17] = 58,
+	[1][0][RTW89_IC][17] = 62,
+	[1][0][RTW89_KCC][17] = 54,
+	[1][0][RTW89_ACMA][17] = 34,
+	[1][0][RTW89_CN][17] = 127,
+	[1][0][RTW89_UK][17] = 34,
+	[1][0][RTW89_FCC][19] = 64,
+	[1][0][RTW89_ETSI][19] = 34,
+	[1][0][RTW89_MKK][19] = 58,
+	[1][0][RTW89_IC][19] = 64,
+	[1][0][RTW89_KCC][19] = 54,
+	[1][0][RTW89_ACMA][19] = 34,
+	[1][0][RTW89_CN][19] = 127,
+	[1][0][RTW89_UK][19] = 34,
+	[1][0][RTW89_FCC][21] = 64,
+	[1][0][RTW89_ETSI][21] = 34,
+	[1][0][RTW89_MKK][21] = 58,
+	[1][0][RTW89_IC][21] = 64,
+	[1][0][RTW89_KCC][21] = 54,
+	[1][0][RTW89_ACMA][21] = 34,
+	[1][0][RTW89_CN][21] = 127,
+	[1][0][RTW89_UK][21] = 34,
+	[1][0][RTW89_FCC][23] = 64,
+	[1][0][RTW89_ETSI][23] = 34,
+	[1][0][RTW89_MKK][23] = 58,
+	[1][0][RTW89_IC][23] = 64,
+	[1][0][RTW89_KCC][23] = 54,
+	[1][0][RTW89_ACMA][23] = 34,
+	[1][0][RTW89_CN][23] = 127,
+	[1][0][RTW89_UK][23] = 34,
+	[1][0][RTW89_FCC][25] = 64,
+	[1][0][RTW89_ETSI][25] = 34,
+	[1][0][RTW89_MKK][25] = 58,
+	[1][0][RTW89_IC][25] = 127,
+	[1][0][RTW89_KCC][25] = 54,
+	[1][0][RTW89_ACMA][25] = 127,
+	[1][0][RTW89_CN][25] = 127,
+	[1][0][RTW89_UK][25] = 34,
+	[1][0][RTW89_FCC][27] = 64,
+	[1][0][RTW89_ETSI][27] = 34,
+	[1][0][RTW89_MKK][27] = 58,
+	[1][0][RTW89_IC][27] = 127,
+	[1][0][RTW89_KCC][27] = 54,
+	[1][0][RTW89_ACMA][27] = 127,
+	[1][0][RTW89_CN][27] = 127,
+	[1][0][RTW89_UK][27] = 34,
+	[1][0][RTW89_FCC][29] = 64,
+	[1][0][RTW89_ETSI][29] = 34,
+	[1][0][RTW89_MKK][29] = 58,
+	[1][0][RTW89_IC][29] = 127,
+	[1][0][RTW89_KCC][29] = 54,
+	[1][0][RTW89_ACMA][29] = 127,
+	[1][0][RTW89_CN][29] = 127,
+	[1][0][RTW89_UK][29] = 34,
+	[1][0][RTW89_FCC][31] = 64,
+	[1][0][RTW89_ETSI][31] = 34,
+	[1][0][RTW89_MKK][31] = 58,
+	[1][0][RTW89_IC][31] = 64,
+	[1][0][RTW89_KCC][31] = 54,
+	[1][0][RTW89_ACMA][31] = 34,
+	[1][0][RTW89_CN][31] = 127,
+	[1][0][RTW89_UK][31] = 34,
+	[1][0][RTW89_FCC][33] = 64,
+	[1][0][RTW89_ETSI][33] = 34,
+	[1][0][RTW89_MKK][33] = 58,
+	[1][0][RTW89_IC][33] = 64,
+	[1][0][RTW89_KCC][33] = 54,
+	[1][0][RTW89_ACMA][33] = 34,
+	[1][0][RTW89_CN][33] = 127,
+	[1][0][RTW89_UK][33] = 34,
+	[1][0][RTW89_FCC][35] = 64,
+	[1][0][RTW89_ETSI][35] = 34,
+	[1][0][RTW89_MKK][35] = 58,
+	[1][0][RTW89_IC][35] = 64,
+	[1][0][RTW89_KCC][35] = 54,
+	[1][0][RTW89_ACMA][35] = 34,
+	[1][0][RTW89_CN][35] = 127,
+	[1][0][RTW89_UK][35] = 34,
+	[1][0][RTW89_FCC][37] = 78,
+	[1][0][RTW89_ETSI][37] = 127,
+	[1][0][RTW89_MKK][37] = 56,
+	[1][0][RTW89_IC][37] = 78,
+	[1][0][RTW89_KCC][37] = 54,
+	[1][0][RTW89_ACMA][37] = 62,
+	[1][0][RTW89_CN][37] = 127,
+	[1][0][RTW89_UK][37] = 62,
+	[1][0][RTW89_FCC][38] = 82,
+	[1][0][RTW89_ETSI][38] = 28,
+	[1][0][RTW89_MKK][38] = 127,
+	[1][0][RTW89_IC][38] = 82,
+	[1][0][RTW89_KCC][38] = 54,
+	[1][0][RTW89_ACMA][38] = 84,
+	[1][0][RTW89_CN][38] = 66,
+	[1][0][RTW89_UK][38] = 34,
+	[1][0][RTW89_FCC][40] = 82,
+	[1][0][RTW89_ETSI][40] = 28,
+	[1][0][RTW89_MKK][40] = 127,
+	[1][0][RTW89_IC][40] = 82,
+	[1][0][RTW89_KCC][40] = 54,
+	[1][0][RTW89_ACMA][40] = 84,
+	[1][0][RTW89_CN][40] = 66,
+	[1][0][RTW89_UK][40] = 34,
+	[1][0][RTW89_FCC][42] = 78,
+	[1][0][RTW89_ETSI][42] = 28,
+	[1][0][RTW89_MKK][42] = 127,
+	[1][0][RTW89_IC][42] = 78,
+	[1][0][RTW89_KCC][42] = 54,
+	[1][0][RTW89_ACMA][42] = 84,
+	[1][0][RTW89_CN][42] = 66,
+	[1][0][RTW89_UK][42] = 34,
+	[1][0][RTW89_FCC][44] = 82,
+	[1][0][RTW89_ETSI][44] = 28,
+	[1][0][RTW89_MKK][44] = 127,
+	[1][0][RTW89_IC][44] = 82,
+	[1][0][RTW89_KCC][44] = 54,
+	[1][0][RTW89_ACMA][44] = 84,
+	[1][0][RTW89_CN][44] = 66,
+	[1][0][RTW89_UK][44] = 34,
+	[1][0][RTW89_FCC][46] = 82,
+	[1][0][RTW89_ETSI][46] = 28,
+	[1][0][RTW89_MKK][46] = 127,
+	[1][0][RTW89_IC][46] = 82,
+	[1][0][RTW89_KCC][46] = 54,
+	[1][0][RTW89_ACMA][46] = 84,
+	[1][0][RTW89_CN][46] = 66,
+	[1][0][RTW89_UK][46] = 34,
+	[1][0][RTW89_FCC][48] = 52,
+	[1][0][RTW89_ETSI][48] = 127,
+	[1][0][RTW89_MKK][48] = 127,
+	[1][0][RTW89_IC][48] = 127,
+	[1][0][RTW89_KCC][48] = 127,
+	[1][0][RTW89_ACMA][48] = 127,
+	[1][0][RTW89_CN][48] = 127,
+	[1][0][RTW89_UK][48] = 127,
+	[1][0][RTW89_FCC][50] = 52,
+	[1][0][RTW89_ETSI][50] = 127,
+	[1][0][RTW89_MKK][50] = 127,
+	[1][0][RTW89_IC][50] = 127,
+	[1][0][RTW89_KCC][50] = 127,
+	[1][0][RTW89_ACMA][50] = 127,
+	[1][0][RTW89_CN][50] = 127,
+	[1][0][RTW89_UK][50] = 127,
+	[1][0][RTW89_FCC][52] = 52,
+	[1][0][RTW89_ETSI][52] = 127,
+	[1][0][RTW89_MKK][52] = 127,
+	[1][0][RTW89_IC][52] = 127,
+	[1][0][RTW89_KCC][52] = 127,
+	[1][0][RTW89_ACMA][52] = 127,
+	[1][0][RTW89_CN][52] = 127,
+	[1][0][RTW89_UK][52] = 127,
+	[1][1][RTW89_FCC][0] = 127,
+	[1][1][RTW89_ETSI][0] = 127,
+	[1][1][RTW89_MKK][0] = 127,
+	[1][1][RTW89_IC][0] = 127,
+	[1][1][RTW89_KCC][0] = 127,
+	[1][1][RTW89_ACMA][0] = 127,
+	[1][1][RTW89_CN][0] = 14,
+	[1][1][RTW89_UK][0] = 127,
+	[1][1][RTW89_FCC][2] = 127,
+	[1][1][RTW89_ETSI][2] = 127,
+	[1][1][RTW89_MKK][2] = 127,
+	[1][1][RTW89_IC][2] = 127,
+	[1][1][RTW89_KCC][2] = 127,
+	[1][1][RTW89_ACMA][2] = 127,
+	[1][1][RTW89_CN][2] = 14,
+	[1][1][RTW89_UK][2] = 127,
+	[1][1][RTW89_FCC][4] = 127,
+	[1][1][RTW89_ETSI][4] = 127,
+	[1][1][RTW89_MKK][4] = 127,
+	[1][1][RTW89_IC][4] = 127,
+	[1][1][RTW89_KCC][4] = 127,
+	[1][1][RTW89_ACMA][4] = 127,
+	[1][1][RTW89_CN][4] = 14,
+	[1][1][RTW89_UK][4] = 127,
+	[1][1][RTW89_FCC][6] = 127,
+	[1][1][RTW89_ETSI][6] = 127,
+	[1][1][RTW89_MKK][6] = 127,
+	[1][1][RTW89_IC][6] = 127,
+	[1][1][RTW89_KCC][6] = 127,
+	[1][1][RTW89_ACMA][6] = 127,
+	[1][1][RTW89_CN][6] = 14,
+	[1][1][RTW89_UK][6] = 127,
+	[1][1][RTW89_FCC][8] = 127,
+	[1][1][RTW89_ETSI][8] = 127,
+	[1][1][RTW89_MKK][8] = 127,
+	[1][1][RTW89_IC][8] = 127,
+	[1][1][RTW89_KCC][8] = 127,
+	[1][1][RTW89_ACMA][8] = 127,
+	[1][1][RTW89_CN][8] = 14,
+	[1][1][RTW89_UK][8] = 127,
+	[1][1][RTW89_FCC][10] = 127,
+	[1][1][RTW89_ETSI][10] = 127,
+	[1][1][RTW89_MKK][10] = 127,
+	[1][1][RTW89_IC][10] = 127,
+	[1][1][RTW89_KCC][10] = 127,
+	[1][1][RTW89_ACMA][10] = 127,
+	[1][1][RTW89_CN][10] = 14,
+	[1][1][RTW89_UK][10] = 127,
+	[1][1][RTW89_FCC][12] = 127,
+	[1][1][RTW89_ETSI][12] = 127,
+	[1][1][RTW89_MKK][12] = 127,
+	[1][1][RTW89_IC][12] = 127,
+	[1][1][RTW89_KCC][12] = 127,
+	[1][1][RTW89_ACMA][12] = 127,
+	[1][1][RTW89_CN][12] = 14,
+	[1][1][RTW89_UK][12] = 127,
+	[1][1][RTW89_FCC][14] = 127,
+	[1][1][RTW89_ETSI][14] = 127,
+	[1][1][RTW89_MKK][14] = 127,
+	[1][1][RTW89_IC][14] = 127,
+	[1][1][RTW89_KCC][14] = 127,
+	[1][1][RTW89_ACMA][14] = 127,
+	[1][1][RTW89_CN][14] = 14,
+	[1][1][RTW89_UK][14] = 127,
+	[1][1][RTW89_FCC][15] = 127,
+	[1][1][RTW89_ETSI][15] = 127,
+	[1][1][RTW89_MKK][15] = 127,
+	[1][1][RTW89_IC][15] = 127,
+	[1][1][RTW89_KCC][15] = 127,
+	[1][1][RTW89_ACMA][15] = 127,
+	[1][1][RTW89_CN][15] = 127,
+	[1][1][RTW89_UK][15] = 127,
+	[1][1][RTW89_FCC][17] = 127,
+	[1][1][RTW89_ETSI][17] = 127,
+	[1][1][RTW89_MKK][17] = 127,
+	[1][1][RTW89_IC][17] = 127,
+	[1][1][RTW89_KCC][17] = 127,
+	[1][1][RTW89_ACMA][17] = 127,
+	[1][1][RTW89_CN][17] = 127,
+	[1][1][RTW89_UK][17] = 127,
+	[1][1][RTW89_FCC][19] = 127,
+	[1][1][RTW89_ETSI][19] = 127,
+	[1][1][RTW89_MKK][19] = 127,
+	[1][1][RTW89_IC][19] = 127,
+	[1][1][RTW89_KCC][19] = 127,
+	[1][1][RTW89_ACMA][19] = 127,
+	[1][1][RTW89_CN][19] = 127,
+	[1][1][RTW89_UK][19] = 127,
+	[1][1][RTW89_FCC][21] = 127,
+	[1][1][RTW89_ETSI][21] = 127,
+	[1][1][RTW89_MKK][21] = 127,
+	[1][1][RTW89_IC][21] = 127,
+	[1][1][RTW89_KCC][21] = 127,
+	[1][1][RTW89_ACMA][21] = 127,
+	[1][1][RTW89_CN][21] = 127,
+	[1][1][RTW89_UK][21] = 127,
+	[1][1][RTW89_FCC][23] = 127,
+	[1][1][RTW89_ETSI][23] = 127,
+	[1][1][RTW89_MKK][23] = 127,
+	[1][1][RTW89_IC][23] = 127,
+	[1][1][RTW89_KCC][23] = 127,
+	[1][1][RTW89_ACMA][23] = 127,
+	[1][1][RTW89_CN][23] = 127,
+	[1][1][RTW89_UK][23] = 127,
+	[1][1][RTW89_FCC][25] = 127,
+	[1][1][RTW89_ETSI][25] = 127,
+	[1][1][RTW89_MKK][25] = 127,
+	[1][1][RTW89_IC][25] = 127,
+	[1][1][RTW89_KCC][25] = 127,
+	[1][1][RTW89_ACMA][25] = 127,
+	[1][1][RTW89_CN][25] = 127,
+	[1][1][RTW89_UK][25] = 127,
+	[1][1][RTW89_FCC][27] = 127,
+	[1][1][RTW89_ETSI][27] = 127,
+	[1][1][RTW89_MKK][27] = 127,
+	[1][1][RTW89_IC][27] = 127,
+	[1][1][RTW89_KCC][27] = 127,
+	[1][1][RTW89_ACMA][27] = 127,
+	[1][1][RTW89_CN][27] = 127,
+	[1][1][RTW89_UK][27] = 127,
+	[1][1][RTW89_FCC][29] = 127,
+	[1][1][RTW89_ETSI][29] = 127,
+	[1][1][RTW89_MKK][29] = 127,
+	[1][1][RTW89_IC][29] = 127,
+	[1][1][RTW89_KCC][29] = 127,
+	[1][1][RTW89_ACMA][29] = 127,
+	[1][1][RTW89_CN][29] = 127,
+	[1][1][RTW89_UK][29] = 127,
+	[1][1][RTW89_FCC][31] = 127,
+	[1][1][RTW89_ETSI][31] = 127,
+	[1][1][RTW89_MKK][31] = 127,
+	[1][1][RTW89_IC][31] = 127,
+	[1][1][RTW89_KCC][31] = 127,
+	[1][1][RTW89_ACMA][31] = 127,
+	[1][1][RTW89_CN][31] = 127,
+	[1][1][RTW89_UK][31] = 127,
+	[1][1][RTW89_FCC][33] = 127,
+	[1][1][RTW89_ETSI][33] = 127,
+	[1][1][RTW89_MKK][33] = 127,
+	[1][1][RTW89_IC][33] = 127,
+	[1][1][RTW89_KCC][33] = 127,
+	[1][1][RTW89_ACMA][33] = 127,
+	[1][1][RTW89_CN][33] = 127,
+	[1][1][RTW89_UK][33] = 127,
+	[1][1][RTW89_FCC][35] = 127,
+	[1][1][RTW89_ETSI][35] = 127,
+	[1][1][RTW89_MKK][35] = 127,
+	[1][1][RTW89_IC][35] = 127,
+	[1][1][RTW89_KCC][35] = 127,
+	[1][1][RTW89_ACMA][35] = 127,
+	[1][1][RTW89_CN][35] = 127,
+	[1][1][RTW89_UK][35] = 127,
+	[1][1][RTW89_FCC][37] = 127,
+	[1][1][RTW89_ETSI][37] = 127,
+	[1][1][RTW89_MKK][37] = 127,
+	[1][1][RTW89_IC][37] = 127,
+	[1][1][RTW89_KCC][37] = 127,
+	[1][1][RTW89_ACMA][37] = 127,
+	[1][1][RTW89_CN][37] = 127,
+	[1][1][RTW89_UK][37] = 127,
+	[1][1][RTW89_FCC][38] = 127,
+	[1][1][RTW89_ETSI][38] = 127,
+	[1][1][RTW89_MKK][38] = 127,
+	[1][1][RTW89_IC][38] = 127,
+	[1][1][RTW89_KCC][38] = 127,
+	[1][1][RTW89_ACMA][38] = 127,
+	[1][1][RTW89_CN][38] = 54,
+	[1][1][RTW89_UK][38] = 127,
+	[1][1][RTW89_FCC][40] = 127,
+	[1][1][RTW89_ETSI][40] = 127,
+	[1][1][RTW89_MKK][40] = 127,
+	[1][1][RTW89_IC][40] = 127,
+	[1][1][RTW89_KCC][40] = 127,
+	[1][1][RTW89_ACMA][40] = 127,
+	[1][1][RTW89_CN][40] = 54,
+	[1][1][RTW89_UK][40] = 127,
+	[1][1][RTW89_FCC][42] = 127,
+	[1][1][RTW89_ETSI][42] = 127,
+	[1][1][RTW89_MKK][42] = 127,
+	[1][1][RTW89_IC][42] = 127,
+	[1][1][RTW89_KCC][42] = 127,
+	[1][1][RTW89_ACMA][42] = 127,
+	[1][1][RTW89_CN][42] = 54,
+	[1][1][RTW89_UK][42] = 127,
+	[1][1][RTW89_FCC][44] = 127,
+	[1][1][RTW89_ETSI][44] = 127,
+	[1][1][RTW89_MKK][44] = 127,
+	[1][1][RTW89_IC][44] = 127,
+	[1][1][RTW89_KCC][44] = 127,
+	[1][1][RTW89_ACMA][44] = 127,
+	[1][1][RTW89_CN][44] = 54,
+	[1][1][RTW89_UK][44] = 127,
+	[1][1][RTW89_FCC][46] = 127,
+	[1][1][RTW89_ETSI][46] = 127,
+	[1][1][RTW89_MKK][46] = 127,
+	[1][1][RTW89_IC][46] = 127,
+	[1][1][RTW89_KCC][46] = 127,
+	[1][1][RTW89_ACMA][46] = 127,
+	[1][1][RTW89_CN][46] = 54,
+	[1][1][RTW89_UK][46] = 127,
+	[1][1][RTW89_FCC][48] = 127,
+	[1][1][RTW89_ETSI][48] = 127,
+	[1][1][RTW89_MKK][48] = 127,
+	[1][1][RTW89_IC][48] = 127,
+	[1][1][RTW89_KCC][48] = 127,
+	[1][1][RTW89_ACMA][48] = 127,
+	[1][1][RTW89_CN][48] = 127,
+	[1][1][RTW89_UK][48] = 127,
+	[1][1][RTW89_FCC][50] = 127,
+	[1][1][RTW89_ETSI][50] = 127,
+	[1][1][RTW89_MKK][50] = 127,
+	[1][1][RTW89_IC][50] = 127,
+	[1][1][RTW89_KCC][50] = 127,
+	[1][1][RTW89_ACMA][50] = 127,
+	[1][1][RTW89_CN][50] = 127,
+	[1][1][RTW89_UK][50] = 127,
+	[1][1][RTW89_FCC][52] = 127,
+	[1][1][RTW89_ETSI][52] = 127,
+	[1][1][RTW89_MKK][52] = 127,
+	[1][1][RTW89_IC][52] = 127,
+	[1][1][RTW89_KCC][52] = 127,
+	[1][1][RTW89_ACMA][52] = 127,
+	[1][1][RTW89_CN][52] = 127,
+	[1][1][RTW89_UK][52] = 127,
+	[2][0][RTW89_FCC][0] = 78,
+	[2][0][RTW89_ETSI][0] = 46,
+	[2][0][RTW89_MKK][0] = 48,
+	[2][0][RTW89_IC][0] = 50,
+	[2][0][RTW89_KCC][0] = 64,
+	[2][0][RTW89_ACMA][0] = 46,
+	[2][0][RTW89_CN][0] = 40,
+	[2][0][RTW89_UK][0] = 46,
+	[2][0][RTW89_FCC][2] = 74,
+	[2][0][RTW89_ETSI][2] = 46,
+	[2][0][RTW89_MKK][2] = 48,
+	[2][0][RTW89_IC][2] = 48,
+	[2][0][RTW89_KCC][2] = 64,
+	[2][0][RTW89_ACMA][2] = 46,
+	[2][0][RTW89_CN][2] = 40,
+	[2][0][RTW89_UK][2] = 46,
+	[2][0][RTW89_FCC][4] = 74,
+	[2][0][RTW89_ETSI][4] = 46,
+	[2][0][RTW89_MKK][4] = 48,
+	[2][0][RTW89_IC][4] = 48,
+	[2][0][RTW89_KCC][4] = 64,
+	[2][0][RTW89_ACMA][4] = 46,
+	[2][0][RTW89_CN][4] = 40,
+	[2][0][RTW89_UK][4] = 46,
+	[2][0][RTW89_FCC][6] = 74,
+	[2][0][RTW89_ETSI][6] = 46,
+	[2][0][RTW89_MKK][6] = 48,
+	[2][0][RTW89_IC][6] = 48,
+	[2][0][RTW89_KCC][6] = 40,
+	[2][0][RTW89_ACMA][6] = 46,
+	[2][0][RTW89_CN][6] = 40,
+	[2][0][RTW89_UK][6] = 46,
+	[2][0][RTW89_FCC][8] = 74,
+	[2][0][RTW89_ETSI][8] = 46,
+	[2][0][RTW89_MKK][8] = 48,
+	[2][0][RTW89_IC][8] = 64,
+	[2][0][RTW89_KCC][8] = 66,
+	[2][0][RTW89_ACMA][8] = 46,
+	[2][0][RTW89_CN][8] = 40,
+	[2][0][RTW89_UK][8] = 46,
+	[2][0][RTW89_FCC][10] = 74,
+	[2][0][RTW89_ETSI][10] = 46,
+	[2][0][RTW89_MKK][10] = 48,
+	[2][0][RTW89_IC][10] = 64,
+	[2][0][RTW89_KCC][10] = 66,
+	[2][0][RTW89_ACMA][10] = 46,
+	[2][0][RTW89_CN][10] = 40,
+	[2][0][RTW89_UK][10] = 46,
+	[2][0][RTW89_FCC][12] = 74,
+	[2][0][RTW89_ETSI][12] = 46,
+	[2][0][RTW89_MKK][12] = 48,
+	[2][0][RTW89_IC][12] = 64,
+	[2][0][RTW89_KCC][12] = 64,
+	[2][0][RTW89_ACMA][12] = 46,
+	[2][0][RTW89_CN][12] = 40,
+	[2][0][RTW89_UK][12] = 46,
+	[2][0][RTW89_FCC][14] = 80,
+	[2][0][RTW89_ETSI][14] = 46,
+	[2][0][RTW89_MKK][14] = 48,
+	[2][0][RTW89_IC][14] = 64,
+	[2][0][RTW89_KCC][14] = 64,
+	[2][0][RTW89_ACMA][14] = 46,
+	[2][0][RTW89_CN][14] = 40,
+	[2][0][RTW89_UK][14] = 46,
+	[2][0][RTW89_FCC][15] = 72,
+	[2][0][RTW89_ETSI][15] = 46,
+	[2][0][RTW89_MKK][15] = 70,
+	[2][0][RTW89_IC][15] = 72,
+	[2][0][RTW89_KCC][15] = 66,
+	[2][0][RTW89_ACMA][15] = 46,
+	[2][0][RTW89_CN][15] = 127,
+	[2][0][RTW89_UK][15] = 46,
+	[2][0][RTW89_FCC][17] = 72,
+	[2][0][RTW89_ETSI][17] = 46,
+	[2][0][RTW89_MKK][17] = 70,
+	[2][0][RTW89_IC][17] = 72,
+	[2][0][RTW89_KCC][17] = 66,
+	[2][0][RTW89_ACMA][17] = 46,
+	[2][0][RTW89_CN][17] = 127,
+	[2][0][RTW89_UK][17] = 46,
+	[2][0][RTW89_FCC][19] = 70,
+	[2][0][RTW89_ETSI][19] = 46,
+	[2][0][RTW89_MKK][19] = 70,
+	[2][0][RTW89_IC][19] = 70,
+	[2][0][RTW89_KCC][19] = 66,
+	[2][0][RTW89_ACMA][19] = 46,
+	[2][0][RTW89_CN][19] = 127,
+	[2][0][RTW89_UK][19] = 46,
+	[2][0][RTW89_FCC][21] = 70,
+	[2][0][RTW89_ETSI][21] = 46,
+	[2][0][RTW89_MKK][21] = 70,
+	[2][0][RTW89_IC][21] = 70,
+	[2][0][RTW89_KCC][21] = 66,
+	[2][0][RTW89_ACMA][21] = 46,
+	[2][0][RTW89_CN][21] = 127,
+	[2][0][RTW89_UK][21] = 46,
+	[2][0][RTW89_FCC][23] = 70,
+	[2][0][RTW89_ETSI][23] = 46,
+	[2][0][RTW89_MKK][23] = 70,
+	[2][0][RTW89_IC][23] = 70,
+	[2][0][RTW89_KCC][23] = 66,
+	[2][0][RTW89_ACMA][23] = 46,
+	[2][0][RTW89_CN][23] = 127,
+	[2][0][RTW89_UK][23] = 46,
+	[2][0][RTW89_FCC][25] = 70,
+	[2][0][RTW89_ETSI][25] = 46,
+	[2][0][RTW89_MKK][25] = 70,
+	[2][0][RTW89_IC][25] = 127,
+	[2][0][RTW89_KCC][25] = 66,
+	[2][0][RTW89_ACMA][25] = 127,
+	[2][0][RTW89_CN][25] = 127,
+	[2][0][RTW89_UK][25] = 46,
+	[2][0][RTW89_FCC][27] = 70,
+	[2][0][RTW89_ETSI][27] = 46,
+	[2][0][RTW89_MKK][27] = 70,
+	[2][0][RTW89_IC][27] = 127,
+	[2][0][RTW89_KCC][27] = 64,
+	[2][0][RTW89_ACMA][27] = 127,
+	[2][0][RTW89_CN][27] = 127,
+	[2][0][RTW89_UK][27] = 46,
+	[2][0][RTW89_FCC][29] = 70,
+	[2][0][RTW89_ETSI][29] = 46,
+	[2][0][RTW89_MKK][29] = 70,
+	[2][0][RTW89_IC][29] = 127,
+	[2][0][RTW89_KCC][29] = 64,
+	[2][0][RTW89_ACMA][29] = 127,
+	[2][0][RTW89_CN][29] = 127,
+	[2][0][RTW89_UK][29] = 46,
+	[2][0][RTW89_FCC][31] = 70,
+	[2][0][RTW89_ETSI][31] = 46,
+	[2][0][RTW89_MKK][31] = 70,
+	[2][0][RTW89_IC][31] = 70,
+	[2][0][RTW89_KCC][31] = 64,
+	[2][0][RTW89_ACMA][31] = 46,
+	[2][0][RTW89_CN][31] = 127,
+	[2][0][RTW89_UK][31] = 46,
+	[2][0][RTW89_FCC][33] = 70,
+	[2][0][RTW89_ETSI][33] = 46,
+	[2][0][RTW89_MKK][33] = 70,
+	[2][0][RTW89_IC][33] = 70,
+	[2][0][RTW89_KCC][33] = 64,
+	[2][0][RTW89_ACMA][33] = 46,
+	[2][0][RTW89_CN][33] = 127,
+	[2][0][RTW89_UK][33] = 46,
+	[2][0][RTW89_FCC][35] = 70,
+	[2][0][RTW89_ETSI][35] = 46,
+	[2][0][RTW89_MKK][35] = 70,
+	[2][0][RTW89_IC][35] = 70,
+	[2][0][RTW89_KCC][35] = 64,
+	[2][0][RTW89_ACMA][35] = 46,
+	[2][0][RTW89_CN][35] = 127,
+	[2][0][RTW89_UK][35] = 46,
+	[2][0][RTW89_FCC][37] = 84,
+	[2][0][RTW89_ETSI][37] = 127,
+	[2][0][RTW89_MKK][37] = 68,
+	[2][0][RTW89_IC][37] = 84,
+	[2][0][RTW89_KCC][37] = 66,
+	[2][0][RTW89_ACMA][37] = 74,
+	[2][0][RTW89_CN][37] = 127,
+	[2][0][RTW89_UK][37] = 74,
+	[2][0][RTW89_FCC][38] = 84,
+	[2][0][RTW89_ETSI][38] = 28,
+	[2][0][RTW89_MKK][38] = 127,
+	[2][0][RTW89_IC][38] = 84,
+	[2][0][RTW89_KCC][38] = 64,
+	[2][0][RTW89_ACMA][38] = 84,
+	[2][0][RTW89_CN][38] = 68,
+	[2][0][RTW89_UK][38] = 46,
+	[2][0][RTW89_FCC][40] = 84,
+	[2][0][RTW89_ETSI][40] = 28,
+	[2][0][RTW89_MKK][40] = 127,
+	[2][0][RTW89_IC][40] = 84,
+	[2][0][RTW89_KCC][40] = 64,
+	[2][0][RTW89_ACMA][40] = 84,
+	[2][0][RTW89_CN][40] = 68,
+	[2][0][RTW89_UK][40] = 46,
+	[2][0][RTW89_FCC][42] = 80,
+	[2][0][RTW89_ETSI][42] = 28,
+	[2][0][RTW89_MKK][42] = 127,
+	[2][0][RTW89_IC][42] = 80,
+	[2][0][RTW89_KCC][42] = 66,
+	[2][0][RTW89_ACMA][42] = 84,
+	[2][0][RTW89_CN][42] = 68,
+	[2][0][RTW89_UK][42] = 46,
+	[2][0][RTW89_FCC][44] = 82,
+	[2][0][RTW89_ETSI][44] = 28,
+	[2][0][RTW89_MKK][44] = 127,
+	[2][0][RTW89_IC][44] = 82,
+	[2][0][RTW89_KCC][44] = 66,
+	[2][0][RTW89_ACMA][44] = 84,
+	[2][0][RTW89_CN][44] = 68,
+	[2][0][RTW89_UK][44] = 46,
+	[2][0][RTW89_FCC][46] = 82,
+	[2][0][RTW89_ETSI][46] = 28,
+	[2][0][RTW89_MKK][46] = 127,
+	[2][0][RTW89_IC][46] = 82,
+	[2][0][RTW89_KCC][46] = 66,
+	[2][0][RTW89_ACMA][46] = 84,
+	[2][0][RTW89_CN][46] = 68,
+	[2][0][RTW89_UK][46] = 46,
+	[2][0][RTW89_FCC][48] = 64,
+	[2][0][RTW89_ETSI][48] = 127,
+	[2][0][RTW89_MKK][48] = 127,
+	[2][0][RTW89_IC][48] = 127,
+	[2][0][RTW89_KCC][48] = 127,
+	[2][0][RTW89_ACMA][48] = 127,
+	[2][0][RTW89_CN][48] = 127,
+	[2][0][RTW89_UK][48] = 127,
+	[2][0][RTW89_FCC][50] = 64,
+	[2][0][RTW89_ETSI][50] = 127,
+	[2][0][RTW89_MKK][50] = 127,
+	[2][0][RTW89_IC][50] = 127,
+	[2][0][RTW89_KCC][50] = 127,
+	[2][0][RTW89_ACMA][50] = 127,
+	[2][0][RTW89_CN][50] = 127,
+	[2][0][RTW89_UK][50] = 127,
+	[2][0][RTW89_FCC][52] = 60,
+	[2][0][RTW89_ETSI][52] = 127,
+	[2][0][RTW89_MKK][52] = 127,
+	[2][0][RTW89_IC][52] = 127,
+	[2][0][RTW89_KCC][52] = 127,
+	[2][0][RTW89_ACMA][52] = 127,
+	[2][0][RTW89_CN][52] = 127,
+	[2][0][RTW89_UK][52] = 127,
+	[2][1][RTW89_FCC][0] = 127,
+	[2][1][RTW89_ETSI][0] = 127,
+	[2][1][RTW89_MKK][0] = 127,
+	[2][1][RTW89_IC][0] = 127,
+	[2][1][RTW89_KCC][0] = 127,
+	[2][1][RTW89_ACMA][0] = 127,
+	[2][1][RTW89_CN][0] = 28,
+	[2][1][RTW89_UK][0] = 127,
+	[2][1][RTW89_FCC][2] = 127,
+	[2][1][RTW89_ETSI][2] = 127,
+	[2][1][RTW89_MKK][2] = 127,
+	[2][1][RTW89_IC][2] = 127,
+	[2][1][RTW89_KCC][2] = 127,
+	[2][1][RTW89_ACMA][2] = 127,
+	[2][1][RTW89_CN][2] = 28,
+	[2][1][RTW89_UK][2] = 127,
+	[2][1][RTW89_FCC][4] = 127,
+	[2][1][RTW89_ETSI][4] = 127,
+	[2][1][RTW89_MKK][4] = 127,
+	[2][1][RTW89_IC][4] = 127,
+	[2][1][RTW89_KCC][4] = 127,
+	[2][1][RTW89_ACMA][4] = 127,
+	[2][1][RTW89_CN][4] = 28,
+	[2][1][RTW89_UK][4] = 127,
+	[2][1][RTW89_FCC][6] = 127,
+	[2][1][RTW89_ETSI][6] = 127,
+	[2][1][RTW89_MKK][6] = 127,
+	[2][1][RTW89_IC][6] = 127,
+	[2][1][RTW89_KCC][6] = 127,
+	[2][1][RTW89_ACMA][6] = 127,
+	[2][1][RTW89_CN][6] = 28,
+	[2][1][RTW89_UK][6] = 127,
+	[2][1][RTW89_FCC][8] = 127,
+	[2][1][RTW89_ETSI][8] = 127,
+	[2][1][RTW89_MKK][8] = 127,
+	[2][1][RTW89_IC][8] = 127,
+	[2][1][RTW89_KCC][8] = 127,
+	[2][1][RTW89_ACMA][8] = 127,
+	[2][1][RTW89_CN][8] = 28,
+	[2][1][RTW89_UK][8] = 127,
+	[2][1][RTW89_FCC][10] = 127,
+	[2][1][RTW89_ETSI][10] = 127,
+	[2][1][RTW89_MKK][10] = 127,
+	[2][1][RTW89_IC][10] = 127,
+	[2][1][RTW89_KCC][10] = 127,
+	[2][1][RTW89_ACMA][10] = 127,
+	[2][1][RTW89_CN][10] = 28,
+	[2][1][RTW89_UK][10] = 127,
+	[2][1][RTW89_FCC][12] = 127,
+	[2][1][RTW89_ETSI][12] = 127,
+	[2][1][RTW89_MKK][12] = 127,
+	[2][1][RTW89_IC][12] = 127,
+	[2][1][RTW89_KCC][12] = 127,
+	[2][1][RTW89_ACMA][12] = 127,
+	[2][1][RTW89_CN][12] = 28,
+	[2][1][RTW89_UK][12] = 127,
+	[2][1][RTW89_FCC][14] = 127,
+	[2][1][RTW89_ETSI][14] = 127,
+	[2][1][RTW89_MKK][14] = 127,
+	[2][1][RTW89_IC][14] = 127,
+	[2][1][RTW89_KCC][14] = 127,
+	[2][1][RTW89_ACMA][14] = 127,
+	[2][1][RTW89_CN][14] = 28,
+	[2][1][RTW89_UK][14] = 127,
+	[2][1][RTW89_FCC][15] = 127,
+	[2][1][RTW89_ETSI][15] = 127,
+	[2][1][RTW89_MKK][15] = 127,
+	[2][1][RTW89_IC][15] = 127,
+	[2][1][RTW89_KCC][15] = 127,
+	[2][1][RTW89_ACMA][15] = 127,
+	[2][1][RTW89_CN][15] = 127,
+	[2][1][RTW89_UK][15] = 127,
+	[2][1][RTW89_FCC][17] = 127,
+	[2][1][RTW89_ETSI][17] = 127,
+	[2][1][RTW89_MKK][17] = 127,
+	[2][1][RTW89_IC][17] = 127,
+	[2][1][RTW89_KCC][17] = 127,
+	[2][1][RTW89_ACMA][17] = 127,
+	[2][1][RTW89_CN][17] = 127,
+	[2][1][RTW89_UK][17] = 127,
+	[2][1][RTW89_FCC][19] = 127,
+	[2][1][RTW89_ETSI][19] = 127,
+	[2][1][RTW89_MKK][19] = 127,
+	[2][1][RTW89_IC][19] = 127,
+	[2][1][RTW89_KCC][19] = 127,
+	[2][1][RTW89_ACMA][19] = 127,
+	[2][1][RTW89_CN][19] = 127,
+	[2][1][RTW89_UK][19] = 127,
+	[2][1][RTW89_FCC][21] = 127,
+	[2][1][RTW89_ETSI][21] = 127,
+	[2][1][RTW89_MKK][21] = 127,
+	[2][1][RTW89_IC][21] = 127,
+	[2][1][RTW89_KCC][21] = 127,
+	[2][1][RTW89_ACMA][21] = 127,
+	[2][1][RTW89_CN][21] = 127,
+	[2][1][RTW89_UK][21] = 127,
+	[2][1][RTW89_FCC][23] = 127,
+	[2][1][RTW89_ETSI][23] = 127,
+	[2][1][RTW89_MKK][23] = 127,
+	[2][1][RTW89_IC][23] = 127,
+	[2][1][RTW89_KCC][23] = 127,
+	[2][1][RTW89_ACMA][23] = 127,
+	[2][1][RTW89_CN][23] = 127,
+	[2][1][RTW89_UK][23] = 127,
+	[2][1][RTW89_FCC][25] = 127,
+	[2][1][RTW89_ETSI][25] = 127,
+	[2][1][RTW89_MKK][25] = 127,
+	[2][1][RTW89_IC][25] = 127,
+	[2][1][RTW89_KCC][25] = 127,
+	[2][1][RTW89_ACMA][25] = 127,
+	[2][1][RTW89_CN][25] = 127,
+	[2][1][RTW89_UK][25] = 127,
+	[2][1][RTW89_FCC][27] = 127,
+	[2][1][RTW89_ETSI][27] = 127,
+	[2][1][RTW89_MKK][27] = 127,
+	[2][1][RTW89_IC][27] = 127,
+	[2][1][RTW89_KCC][27] = 127,
+	[2][1][RTW89_ACMA][27] = 127,
+	[2][1][RTW89_CN][27] = 127,
+	[2][1][RTW89_UK][27] = 127,
+	[2][1][RTW89_FCC][29] = 127,
+	[2][1][RTW89_ETSI][29] = 127,
+	[2][1][RTW89_MKK][29] = 127,
+	[2][1][RTW89_IC][29] = 127,
+	[2][1][RTW89_KCC][29] = 127,
+	[2][1][RTW89_ACMA][29] = 127,
+	[2][1][RTW89_CN][29] = 127,
+	[2][1][RTW89_UK][29] = 127,
+	[2][1][RTW89_FCC][31] = 127,
+	[2][1][RTW89_ETSI][31] = 127,
+	[2][1][RTW89_MKK][31] = 127,
+	[2][1][RTW89_IC][31] = 127,
+	[2][1][RTW89_KCC][31] = 127,
+	[2][1][RTW89_ACMA][31] = 127,
+	[2][1][RTW89_CN][31] = 127,
+	[2][1][RTW89_UK][31] = 127,
+	[2][1][RTW89_FCC][33] = 127,
+	[2][1][RTW89_ETSI][33] = 127,
+	[2][1][RTW89_MKK][33] = 127,
+	[2][1][RTW89_IC][33] = 127,
+	[2][1][RTW89_KCC][33] = 127,
+	[2][1][RTW89_ACMA][33] = 127,
+	[2][1][RTW89_CN][33] = 127,
+	[2][1][RTW89_UK][33] = 127,
+	[2][1][RTW89_FCC][35] = 127,
+	[2][1][RTW89_ETSI][35] = 127,
+	[2][1][RTW89_MKK][35] = 127,
+	[2][1][RTW89_IC][35] = 127,
+	[2][1][RTW89_KCC][35] = 127,
+	[2][1][RTW89_ACMA][35] = 127,
+	[2][1][RTW89_CN][35] = 127,
+	[2][1][RTW89_UK][35] = 127,
+	[2][1][RTW89_FCC][37] = 127,
+	[2][1][RTW89_ETSI][37] = 127,
+	[2][1][RTW89_MKK][37] = 127,
+	[2][1][RTW89_IC][37] = 127,
+	[2][1][RTW89_KCC][37] = 127,
+	[2][1][RTW89_ACMA][37] = 127,
+	[2][1][RTW89_CN][37] = 127,
+	[2][1][RTW89_UK][37] = 127,
+	[2][1][RTW89_FCC][38] = 127,
+	[2][1][RTW89_ETSI][38] = 127,
+	[2][1][RTW89_MKK][38] = 127,
+	[2][1][RTW89_IC][38] = 127,
+	[2][1][RTW89_KCC][38] = 127,
+	[2][1][RTW89_ACMA][38] = 127,
+	[2][1][RTW89_CN][38] = 56,
+	[2][1][RTW89_UK][38] = 127,
+	[2][1][RTW89_FCC][40] = 127,
+	[2][1][RTW89_ETSI][40] = 127,
+	[2][1][RTW89_MKK][40] = 127,
+	[2][1][RTW89_IC][40] = 127,
+	[2][1][RTW89_KCC][40] = 127,
+	[2][1][RTW89_ACMA][40] = 127,
+	[2][1][RTW89_CN][40] = 56,
+	[2][1][RTW89_UK][40] = 127,
+	[2][1][RTW89_FCC][42] = 127,
+	[2][1][RTW89_ETSI][42] = 127,
+	[2][1][RTW89_MKK][42] = 127,
+	[2][1][RTW89_IC][42] = 127,
+	[2][1][RTW89_KCC][42] = 127,
+	[2][1][RTW89_ACMA][42] = 127,
+	[2][1][RTW89_CN][42] = 56,
+	[2][1][RTW89_UK][42] = 127,
+	[2][1][RTW89_FCC][44] = 127,
+	[2][1][RTW89_ETSI][44] = 127,
+	[2][1][RTW89_MKK][44] = 127,
+	[2][1][RTW89_IC][44] = 127,
+	[2][1][RTW89_KCC][44] = 127,
+	[2][1][RTW89_ACMA][44] = 127,
+	[2][1][RTW89_CN][44] = 56,
+	[2][1][RTW89_UK][44] = 127,
+	[2][1][RTW89_FCC][46] = 127,
+	[2][1][RTW89_ETSI][46] = 127,
+	[2][1][RTW89_MKK][46] = 127,
+	[2][1][RTW89_IC][46] = 127,
+	[2][1][RTW89_KCC][46] = 127,
+	[2][1][RTW89_ACMA][46] = 127,
+	[2][1][RTW89_CN][46] = 56,
+	[2][1][RTW89_UK][46] = 127,
+	[2][1][RTW89_FCC][48] = 127,
+	[2][1][RTW89_ETSI][48] = 127,
+	[2][1][RTW89_MKK][48] = 127,
+	[2][1][RTW89_IC][48] = 127,
+	[2][1][RTW89_KCC][48] = 127,
+	[2][1][RTW89_ACMA][48] = 127,
+	[2][1][RTW89_CN][48] = 127,
+	[2][1][RTW89_UK][48] = 127,
+	[2][1][RTW89_FCC][50] = 127,
+	[2][1][RTW89_ETSI][50] = 127,
+	[2][1][RTW89_MKK][50] = 127,
+	[2][1][RTW89_IC][50] = 127,
+	[2][1][RTW89_KCC][50] = 127,
+	[2][1][RTW89_ACMA][50] = 127,
+	[2][1][RTW89_CN][50] = 127,
+	[2][1][RTW89_UK][50] = 127,
+	[2][1][RTW89_FCC][52] = 127,
+	[2][1][RTW89_ETSI][52] = 127,
+	[2][1][RTW89_MKK][52] = 127,
+	[2][1][RTW89_IC][52] = 127,
+	[2][1][RTW89_KCC][52] = 127,
+	[2][1][RTW89_ACMA][52] = 127,
+	[2][1][RTW89_CN][52] = 127,
+	[2][1][RTW89_UK][52] = 127,
+};
+
+static
+const s8 rtw89_8851b_txpwr_lmt_2g_type2[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
+				       [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
+				       [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
+	[0][0][0][0][RTW89_WW][0] = 58,
+	[0][0][0][0][RTW89_WW][1] = 58,
+	[0][0][0][0][RTW89_WW][2] = 58,
+	[0][0][0][0][RTW89_WW][3] = 58,
+	[0][0][0][0][RTW89_WW][4] = 58,
+	[0][0][0][0][RTW89_WW][5] = 58,
+	[0][0][0][0][RTW89_WW][6] = 58,
+	[0][0][0][0][RTW89_WW][7] = 58,
+	[0][0][0][0][RTW89_WW][8] = 58,
+	[0][0][0][0][RTW89_WW][9] = 58,
+	[0][0][0][0][RTW89_WW][10] = 58,
+	[0][0][0][0][RTW89_WW][11] = 58,
+	[0][0][0][0][RTW89_WW][12] = 52,
+	[0][0][0][0][RTW89_WW][13] = 76,
+	[0][1][0][0][RTW89_WW][0] = 0,
+	[0][1][0][0][RTW89_WW][1] = 0,
+	[0][1][0][0][RTW89_WW][2] = 0,
+	[0][1][0][0][RTW89_WW][3] = 0,
+	[0][1][0][0][RTW89_WW][4] = 0,
+	[0][1][0][0][RTW89_WW][5] = 0,
+	[0][1][0][0][RTW89_WW][6] = 0,
+	[0][1][0][0][RTW89_WW][7] = 0,
+	[0][1][0][0][RTW89_WW][8] = 0,
+	[0][1][0][0][RTW89_WW][9] = 0,
+	[0][1][0][0][RTW89_WW][10] = 0,
+	[0][1][0][0][RTW89_WW][11] = 0,
+	[0][1][0][0][RTW89_WW][12] = 0,
+	[0][1][0][0][RTW89_WW][13] = 0,
+	[1][0][0][0][RTW89_WW][0] = 0,
+	[1][0][0][0][RTW89_WW][1] = 0,
+	[1][0][0][0][RTW89_WW][2] = 58,
+	[1][0][0][0][RTW89_WW][3] = 58,
+	[1][0][0][0][RTW89_WW][4] = 58,
+	[1][0][0][0][RTW89_WW][5] = 58,
+	[1][0][0][0][RTW89_WW][6] = 58,
+	[1][0][0][0][RTW89_WW][7] = 58,
+	[1][0][0][0][RTW89_WW][8] = 58,
+	[1][0][0][0][RTW89_WW][9] = 58,
+	[1][0][0][0][RTW89_WW][10] = 58,
+	[1][0][0][0][RTW89_WW][11] = 0,
+	[1][0][0][0][RTW89_WW][12] = 0,
+	[1][0][0][0][RTW89_WW][13] = 0,
+	[1][1][0][0][RTW89_WW][0] = 0,
+	[1][1][0][0][RTW89_WW][1] = 0,
+	[1][1][0][0][RTW89_WW][2] = 0,
+	[1][1][0][0][RTW89_WW][3] = 0,
+	[1][1][0][0][RTW89_WW][4] = 0,
+	[1][1][0][0][RTW89_WW][5] = 0,
+	[1][1][0][0][RTW89_WW][6] = 0,
+	[1][1][0][0][RTW89_WW][7] = 0,
+	[1][1][0][0][RTW89_WW][8] = 0,
+	[1][1][0][0][RTW89_WW][9] = 0,
+	[1][1][0][0][RTW89_WW][10] = 0,
+	[1][1][0][0][RTW89_WW][11] = 0,
+	[1][1][0][0][RTW89_WW][12] = 0,
+	[1][1][0][0][RTW89_WW][13] = 0,
+	[0][0][1][0][RTW89_WW][0] = 58,
+	[0][0][1][0][RTW89_WW][1] = 60,
+	[0][0][1][0][RTW89_WW][2] = 60,
+	[0][0][1][0][RTW89_WW][3] = 60,
+	[0][0][1][0][RTW89_WW][4] = 60,
+	[0][0][1][0][RTW89_WW][5] = 60,
+	[0][0][1][0][RTW89_WW][6] = 60,
+	[0][0][1][0][RTW89_WW][7] = 60,
+	[0][0][1][0][RTW89_WW][8] = 60,
+	[0][0][1][0][RTW89_WW][9] = 60,
+	[0][0][1][0][RTW89_WW][10] = 60,
+	[0][0][1][0][RTW89_WW][11] = 60,
+	[0][0][1][0][RTW89_WW][12] = 58,
+	[0][0][1][0][RTW89_WW][13] = 0,
+	[0][1][1][0][RTW89_WW][0] = 0,
+	[0][1][1][0][RTW89_WW][1] = 0,
+	[0][1][1][0][RTW89_WW][2] = 0,
+	[0][1][1][0][RTW89_WW][3] = 0,
+	[0][1][1][0][RTW89_WW][4] = 0,
+	[0][1][1][0][RTW89_WW][5] = 0,
+	[0][1][1][0][RTW89_WW][6] = 0,
+	[0][1][1][0][RTW89_WW][7] = 0,
+	[0][1][1][0][RTW89_WW][8] = 0,
+	[0][1][1][0][RTW89_WW][9] = 0,
+	[0][1][1][0][RTW89_WW][10] = 0,
+	[0][1][1][0][RTW89_WW][11] = 0,
+	[0][1][1][0][RTW89_WW][12] = 0,
+	[0][1][1][0][RTW89_WW][13] = 0,
+	[0][0][2][0][RTW89_WW][0] = 60,
+	[0][0][2][0][RTW89_WW][1] = 60,
+	[0][0][2][0][RTW89_WW][2] = 60,
+	[0][0][2][0][RTW89_WW][3] = 60,
+	[0][0][2][0][RTW89_WW][4] = 60,
+	[0][0][2][0][RTW89_WW][5] = 60,
+	[0][0][2][0][RTW89_WW][6] = 60,
+	[0][0][2][0][RTW89_WW][7] = 60,
+	[0][0][2][0][RTW89_WW][8] = 60,
+	[0][0][2][0][RTW89_WW][9] = 60,
+	[0][0][2][0][RTW89_WW][10] = 60,
+	[0][0][2][0][RTW89_WW][11] = 60,
+	[0][0][2][0][RTW89_WW][12] = 60,
+	[0][0][2][0][RTW89_WW][13] = 0,
+	[0][1][2][0][RTW89_WW][0] = 0,
+	[0][1][2][0][RTW89_WW][1] = 0,
+	[0][1][2][0][RTW89_WW][2] = 0,
+	[0][1][2][0][RTW89_WW][3] = 0,
+	[0][1][2][0][RTW89_WW][4] = 0,
+	[0][1][2][0][RTW89_WW][5] = 0,
+	[0][1][2][0][RTW89_WW][6] = 0,
+	[0][1][2][0][RTW89_WW][7] = 0,
+	[0][1][2][0][RTW89_WW][8] = 0,
+	[0][1][2][0][RTW89_WW][9] = 0,
+	[0][1][2][0][RTW89_WW][10] = 0,
+	[0][1][2][0][RTW89_WW][11] = 0,
+	[0][1][2][0][RTW89_WW][12] = 0,
+	[0][1][2][0][RTW89_WW][13] = 0,
+	[0][1][2][1][RTW89_WW][0] = 0,
+	[0][1][2][1][RTW89_WW][1] = 0,
+	[0][1][2][1][RTW89_WW][2] = 0,
+	[0][1][2][1][RTW89_WW][3] = 0,
+	[0][1][2][1][RTW89_WW][4] = 0,
+	[0][1][2][1][RTW89_WW][5] = 0,
+	[0][1][2][1][RTW89_WW][6] = 0,
+	[0][1][2][1][RTW89_WW][7] = 0,
+	[0][1][2][1][RTW89_WW][8] = 0,
+	[0][1][2][1][RTW89_WW][9] = 0,
+	[0][1][2][1][RTW89_WW][10] = 0,
+	[0][1][2][1][RTW89_WW][11] = 0,
+	[0][1][2][1][RTW89_WW][12] = 0,
+	[0][1][2][1][RTW89_WW][13] = 0,
+	[1][0][2][0][RTW89_WW][0] = 0,
+	[1][0][2][0][RTW89_WW][1] = 0,
+	[1][0][2][0][RTW89_WW][2] = 58,
+	[1][0][2][0][RTW89_WW][3] = 58,
+	[1][0][2][0][RTW89_WW][4] = 58,
+	[1][0][2][0][RTW89_WW][5] = 58,
+	[1][0][2][0][RTW89_WW][6] = 58,
+	[1][0][2][0][RTW89_WW][7] = 58,
+	[1][0][2][0][RTW89_WW][8] = 58,
+	[1][0][2][0][RTW89_WW][9] = 58,
+	[1][0][2][0][RTW89_WW][10] = 58,
+	[1][0][2][0][RTW89_WW][11] = 0,
+	[1][0][2][0][RTW89_WW][12] = 0,
+	[1][0][2][0][RTW89_WW][13] = 0,
+	[1][1][2][0][RTW89_WW][0] = 0,
+	[1][1][2][0][RTW89_WW][1] = 0,
+	[1][1][2][0][RTW89_WW][2] = 0,
+	[1][1][2][0][RTW89_WW][3] = 0,
+	[1][1][2][0][RTW89_WW][4] = 0,
+	[1][1][2][0][RTW89_WW][5] = 0,
+	[1][1][2][0][RTW89_WW][6] = 0,
+	[1][1][2][0][RTW89_WW][7] = 0,
+	[1][1][2][0][RTW89_WW][8] = 0,
+	[1][1][2][0][RTW89_WW][9] = 0,
+	[1][1][2][0][RTW89_WW][10] = 0,
+	[1][1][2][0][RTW89_WW][11] = 0,
+	[1][1][2][0][RTW89_WW][12] = 0,
+	[1][1][2][0][RTW89_WW][13] = 0,
+	[1][1][2][1][RTW89_WW][0] = 0,
+	[1][1][2][1][RTW89_WW][1] = 0,
+	[1][1][2][1][RTW89_WW][2] = 0,
+	[1][1][2][1][RTW89_WW][3] = 0,
+	[1][1][2][1][RTW89_WW][4] = 0,
+	[1][1][2][1][RTW89_WW][5] = 0,
+	[1][1][2][1][RTW89_WW][6] = 0,
+	[1][1][2][1][RTW89_WW][7] = 0,
+	[1][1][2][1][RTW89_WW][8] = 0,
+	[1][1][2][1][RTW89_WW][9] = 0,
+	[1][1][2][1][RTW89_WW][10] = 0,
+	[1][1][2][1][RTW89_WW][11] = 0,
+	[1][1][2][1][RTW89_WW][12] = 0,
+	[1][1][2][1][RTW89_WW][13] = 0,
+	[0][0][0][0][RTW89_FCC][0] = 82,
+	[0][0][0][0][RTW89_ETSI][0] = 58,
+	[0][0][0][0][RTW89_MKK][0] = 68,
+	[0][0][0][0][RTW89_IC][0] = 82,
+	[0][0][0][0][RTW89_KCC][0] = 68,
+	[0][0][0][0][RTW89_ACMA][0] = 58,
+	[0][0][0][0][RTW89_CN][0] = 60,
+	[0][0][0][0][RTW89_UK][0] = 58,
+	[0][0][0][0][RTW89_FCC][1] = 82,
+	[0][0][0][0][RTW89_ETSI][1] = 58,
+	[0][0][0][0][RTW89_MKK][1] = 68,
+	[0][0][0][0][RTW89_IC][1] = 82,
+	[0][0][0][0][RTW89_KCC][1] = 68,
+	[0][0][0][0][RTW89_ACMA][1] = 58,
+	[0][0][0][0][RTW89_CN][1] = 60,
+	[0][0][0][0][RTW89_UK][1] = 58,
+	[0][0][0][0][RTW89_FCC][2] = 82,
+	[0][0][0][0][RTW89_ETSI][2] = 58,
+	[0][0][0][0][RTW89_MKK][2] = 68,
+	[0][0][0][0][RTW89_IC][2] = 82,
+	[0][0][0][0][RTW89_KCC][2] = 68,
+	[0][0][0][0][RTW89_ACMA][2] = 58,
+	[0][0][0][0][RTW89_CN][2] = 60,
+	[0][0][0][0][RTW89_UK][2] = 58,
+	[0][0][0][0][RTW89_FCC][3] = 82,
+	[0][0][0][0][RTW89_ETSI][3] = 58,
+	[0][0][0][0][RTW89_MKK][3] = 68,
+	[0][0][0][0][RTW89_IC][3] = 82,
+	[0][0][0][0][RTW89_KCC][3] = 68,
+	[0][0][0][0][RTW89_ACMA][3] = 58,
+	[0][0][0][0][RTW89_CN][3] = 60,
+	[0][0][0][0][RTW89_UK][3] = 58,
+	[0][0][0][0][RTW89_FCC][4] = 82,
+	[0][0][0][0][RTW89_ETSI][4] = 58,
+	[0][0][0][0][RTW89_MKK][4] = 68,
+	[0][0][0][0][RTW89_IC][4] = 82,
+	[0][0][0][0][RTW89_KCC][4] = 68,
+	[0][0][0][0][RTW89_ACMA][4] = 58,
+	[0][0][0][0][RTW89_CN][4] = 60,
+	[0][0][0][0][RTW89_UK][4] = 58,
+	[0][0][0][0][RTW89_FCC][5] = 82,
+	[0][0][0][0][RTW89_ETSI][5] = 58,
+	[0][0][0][0][RTW89_MKK][5] = 68,
+	[0][0][0][0][RTW89_IC][5] = 82,
+	[0][0][0][0][RTW89_KCC][5] = 68,
+	[0][0][0][0][RTW89_ACMA][5] = 58,
+	[0][0][0][0][RTW89_CN][5] = 60,
+	[0][0][0][0][RTW89_UK][5] = 58,
+	[0][0][0][0][RTW89_FCC][6] = 82,
+	[0][0][0][0][RTW89_ETSI][6] = 58,
+	[0][0][0][0][RTW89_MKK][6] = 68,
+	[0][0][0][0][RTW89_IC][6] = 82,
+	[0][0][0][0][RTW89_KCC][6] = 68,
+	[0][0][0][0][RTW89_ACMA][6] = 58,
+	[0][0][0][0][RTW89_CN][6] = 60,
+	[0][0][0][0][RTW89_UK][6] = 58,
+	[0][0][0][0][RTW89_FCC][7] = 82,
+	[0][0][0][0][RTW89_ETSI][7] = 58,
+	[0][0][0][0][RTW89_MKK][7] = 68,
+	[0][0][0][0][RTW89_IC][7] = 82,
+	[0][0][0][0][RTW89_KCC][7] = 68,
+	[0][0][0][0][RTW89_ACMA][7] = 58,
+	[0][0][0][0][RTW89_CN][7] = 60,
+	[0][0][0][0][RTW89_UK][7] = 58,
+	[0][0][0][0][RTW89_FCC][8] = 82,
+	[0][0][0][0][RTW89_ETSI][8] = 58,
+	[0][0][0][0][RTW89_MKK][8] = 68,
+	[0][0][0][0][RTW89_IC][8] = 82,
+	[0][0][0][0][RTW89_KCC][8] = 68,
+	[0][0][0][0][RTW89_ACMA][8] = 58,
+	[0][0][0][0][RTW89_CN][8] = 60,
+	[0][0][0][0][RTW89_UK][8] = 58,
+	[0][0][0][0][RTW89_FCC][9] = 82,
+	[0][0][0][0][RTW89_ETSI][9] = 58,
+	[0][0][0][0][RTW89_MKK][9] = 68,
+	[0][0][0][0][RTW89_IC][9] = 82,
+	[0][0][0][0][RTW89_KCC][9] = 68,
+	[0][0][0][0][RTW89_ACMA][9] = 58,
+	[0][0][0][0][RTW89_CN][9] = 60,
+	[0][0][0][0][RTW89_UK][9] = 58,
+	[0][0][0][0][RTW89_FCC][10] = 80,
+	[0][0][0][0][RTW89_ETSI][10] = 58,
+	[0][0][0][0][RTW89_MKK][10] = 68,
+	[0][0][0][0][RTW89_IC][10] = 80,
+	[0][0][0][0][RTW89_KCC][10] = 68,
+	[0][0][0][0][RTW89_ACMA][10] = 58,
+	[0][0][0][0][RTW89_CN][10] = 60,
+	[0][0][0][0][RTW89_UK][10] = 58,
+	[0][0][0][0][RTW89_FCC][11] = 60,
+	[0][0][0][0][RTW89_ETSI][11] = 58,
+	[0][0][0][0][RTW89_MKK][11] = 68,
+	[0][0][0][0][RTW89_IC][11] = 60,
+	[0][0][0][0][RTW89_KCC][11] = 68,
+	[0][0][0][0][RTW89_ACMA][11] = 58,
+	[0][0][0][0][RTW89_CN][11] = 60,
+	[0][0][0][0][RTW89_UK][11] = 58,
+	[0][0][0][0][RTW89_FCC][12] = 52,
+	[0][0][0][0][RTW89_ETSI][12] = 58,
+	[0][0][0][0][RTW89_MKK][12] = 68,
+	[0][0][0][0][RTW89_IC][12] = 52,
+	[0][0][0][0][RTW89_KCC][12] = 68,
+	[0][0][0][0][RTW89_ACMA][12] = 58,
+	[0][0][0][0][RTW89_CN][12] = 60,
+	[0][0][0][0][RTW89_UK][12] = 58,
+	[0][0][0][0][RTW89_FCC][13] = 127,
+	[0][0][0][0][RTW89_ETSI][13] = 127,
+	[0][0][0][0][RTW89_MKK][13] = 76,
+	[0][0][0][0][RTW89_IC][13] = 127,
+	[0][0][0][0][RTW89_KCC][13] = 127,
+	[0][0][0][0][RTW89_ACMA][13] = 127,
+	[0][0][0][0][RTW89_CN][13] = 127,
+	[0][0][0][0][RTW89_UK][13] = 127,
+	[0][1][0][0][RTW89_FCC][0] = 127,
+	[0][1][0][0][RTW89_ETSI][0] = 127,
+	[0][1][0][0][RTW89_MKK][0] = 127,
+	[0][1][0][0][RTW89_IC][0] = 127,
+	[0][1][0][0][RTW89_KCC][0] = 127,
+	[0][1][0][0][RTW89_ACMA][0] = 127,
+	[0][1][0][0][RTW89_CN][0] = 127,
+	[0][1][0][0][RTW89_UK][0] = 127,
+	[0][1][0][0][RTW89_FCC][1] = 127,
+	[0][1][0][0][RTW89_ETSI][1] = 127,
+	[0][1][0][0][RTW89_MKK][1] = 127,
+	[0][1][0][0][RTW89_IC][1] = 127,
+	[0][1][0][0][RTW89_KCC][1] = 127,
+	[0][1][0][0][RTW89_ACMA][1] = 127,
+	[0][1][0][0][RTW89_CN][1] = 127,
+	[0][1][0][0][RTW89_UK][1] = 127,
+	[0][1][0][0][RTW89_FCC][2] = 127,
+	[0][1][0][0][RTW89_ETSI][2] = 127,
+	[0][1][0][0][RTW89_MKK][2] = 127,
+	[0][1][0][0][RTW89_IC][2] = 127,
+	[0][1][0][0][RTW89_KCC][2] = 127,
+	[0][1][0][0][RTW89_ACMA][2] = 127,
+	[0][1][0][0][RTW89_CN][2] = 127,
+	[0][1][0][0][RTW89_UK][2] = 127,
+	[0][1][0][0][RTW89_FCC][3] = 127,
+	[0][1][0][0][RTW89_ETSI][3] = 127,
+	[0][1][0][0][RTW89_MKK][3] = 127,
+	[0][1][0][0][RTW89_IC][3] = 127,
+	[0][1][0][0][RTW89_KCC][3] = 127,
+	[0][1][0][0][RTW89_ACMA][3] = 127,
+	[0][1][0][0][RTW89_CN][3] = 127,
+	[0][1][0][0][RTW89_UK][3] = 127,
+	[0][1][0][0][RTW89_FCC][4] = 127,
+	[0][1][0][0][RTW89_ETSI][4] = 127,
+	[0][1][0][0][RTW89_MKK][4] = 127,
+	[0][1][0][0][RTW89_IC][4] = 127,
+	[0][1][0][0][RTW89_KCC][4] = 127,
+	[0][1][0][0][RTW89_ACMA][4] = 127,
+	[0][1][0][0][RTW89_CN][4] = 127,
+	[0][1][0][0][RTW89_UK][4] = 127,
+	[0][1][0][0][RTW89_FCC][5] = 127,
+	[0][1][0][0][RTW89_ETSI][5] = 127,
+	[0][1][0][0][RTW89_MKK][5] = 127,
+	[0][1][0][0][RTW89_IC][5] = 127,
+	[0][1][0][0][RTW89_KCC][5] = 127,
+	[0][1][0][0][RTW89_ACMA][5] = 127,
+	[0][1][0][0][RTW89_CN][5] = 127,
+	[0][1][0][0][RTW89_UK][5] = 127,
+	[0][1][0][0][RTW89_FCC][6] = 127,
+	[0][1][0][0][RTW89_ETSI][6] = 127,
+	[0][1][0][0][RTW89_MKK][6] = 127,
+	[0][1][0][0][RTW89_IC][6] = 127,
+	[0][1][0][0][RTW89_KCC][6] = 127,
+	[0][1][0][0][RTW89_ACMA][6] = 127,
+	[0][1][0][0][RTW89_CN][6] = 127,
+	[0][1][0][0][RTW89_UK][6] = 127,
+	[0][1][0][0][RTW89_FCC][7] = 127,
+	[0][1][0][0][RTW89_ETSI][7] = 127,
+	[0][1][0][0][RTW89_MKK][7] = 127,
+	[0][1][0][0][RTW89_IC][7] = 127,
+	[0][1][0][0][RTW89_KCC][7] = 127,
+	[0][1][0][0][RTW89_ACMA][7] = 127,
+	[0][1][0][0][RTW89_CN][7] = 127,
+	[0][1][0][0][RTW89_UK][7] = 127,
+	[0][1][0][0][RTW89_FCC][8] = 127,
+	[0][1][0][0][RTW89_ETSI][8] = 127,
+	[0][1][0][0][RTW89_MKK][8] = 127,
+	[0][1][0][0][RTW89_IC][8] = 127,
+	[0][1][0][0][RTW89_KCC][8] = 127,
+	[0][1][0][0][RTW89_ACMA][8] = 127,
+	[0][1][0][0][RTW89_CN][8] = 127,
+	[0][1][0][0][RTW89_UK][8] = 127,
+	[0][1][0][0][RTW89_FCC][9] = 127,
+	[0][1][0][0][RTW89_ETSI][9] = 127,
+	[0][1][0][0][RTW89_MKK][9] = 127,
+	[0][1][0][0][RTW89_IC][9] = 127,
+	[0][1][0][0][RTW89_KCC][9] = 127,
+	[0][1][0][0][RTW89_ACMA][9] = 127,
+	[0][1][0][0][RTW89_CN][9] = 127,
+	[0][1][0][0][RTW89_UK][9] = 127,
+	[0][1][0][0][RTW89_FCC][10] = 127,
+	[0][1][0][0][RTW89_ETSI][10] = 127,
+	[0][1][0][0][RTW89_MKK][10] = 127,
+	[0][1][0][0][RTW89_IC][10] = 127,
+	[0][1][0][0][RTW89_KCC][10] = 127,
+	[0][1][0][0][RTW89_ACMA][10] = 127,
+	[0][1][0][0][RTW89_CN][10] = 127,
+	[0][1][0][0][RTW89_UK][10] = 127,
+	[0][1][0][0][RTW89_FCC][11] = 127,
+	[0][1][0][0][RTW89_ETSI][11] = 127,
+	[0][1][0][0][RTW89_MKK][11] = 127,
+	[0][1][0][0][RTW89_IC][11] = 127,
+	[0][1][0][0][RTW89_KCC][11] = 127,
+	[0][1][0][0][RTW89_ACMA][11] = 127,
+	[0][1][0][0][RTW89_CN][11] = 127,
+	[0][1][0][0][RTW89_UK][11] = 127,
+	[0][1][0][0][RTW89_FCC][12] = 127,
+	[0][1][0][0][RTW89_ETSI][12] = 127,
+	[0][1][0][0][RTW89_MKK][12] = 127,
+	[0][1][0][0][RTW89_IC][12] = 127,
+	[0][1][0][0][RTW89_KCC][12] = 127,
+	[0][1][0][0][RTW89_ACMA][12] = 127,
+	[0][1][0][0][RTW89_CN][12] = 127,
+	[0][1][0][0][RTW89_UK][12] = 127,
+	[0][1][0][0][RTW89_FCC][13] = 127,
+	[0][1][0][0][RTW89_ETSI][13] = 127,
+	[0][1][0][0][RTW89_MKK][13] = 127,
+	[0][1][0][0][RTW89_IC][13] = 127,
+	[0][1][0][0][RTW89_KCC][13] = 127,
+	[0][1][0][0][RTW89_ACMA][13] = 127,
+	[0][1][0][0][RTW89_CN][13] = 127,
+	[0][1][0][0][RTW89_UK][13] = 127,
+	[1][0][0][0][RTW89_FCC][0] = 127,
+	[1][0][0][0][RTW89_ETSI][0] = 127,
+	[1][0][0][0][RTW89_MKK][0] = 127,
+	[1][0][0][0][RTW89_IC][0] = 127,
+	[1][0][0][0][RTW89_KCC][0] = 127,
+	[1][0][0][0][RTW89_ACMA][0] = 127,
+	[1][0][0][0][RTW89_CN][0] = 127,
+	[1][0][0][0][RTW89_UK][0] = 127,
+	[1][0][0][0][RTW89_FCC][1] = 127,
+	[1][0][0][0][RTW89_ETSI][1] = 127,
+	[1][0][0][0][RTW89_MKK][1] = 127,
+	[1][0][0][0][RTW89_IC][1] = 127,
+	[1][0][0][0][RTW89_KCC][1] = 127,
+	[1][0][0][0][RTW89_ACMA][1] = 127,
+	[1][0][0][0][RTW89_CN][1] = 127,
+	[1][0][0][0][RTW89_UK][1] = 127,
+	[1][0][0][0][RTW89_FCC][2] = 127,
+	[1][0][0][0][RTW89_ETSI][2] = 58,
+	[1][0][0][0][RTW89_MKK][2] = 70,
+	[1][0][0][0][RTW89_IC][2] = 127,
+	[1][0][0][0][RTW89_KCC][2] = 68,
+	[1][0][0][0][RTW89_ACMA][2] = 58,
+	[1][0][0][0][RTW89_CN][2] = 60,
+	[1][0][0][0][RTW89_UK][2] = 58,
+	[1][0][0][0][RTW89_FCC][3] = 127,
+	[1][0][0][0][RTW89_ETSI][3] = 58,
+	[1][0][0][0][RTW89_MKK][3] = 76,
+	[1][0][0][0][RTW89_IC][3] = 127,
+	[1][0][0][0][RTW89_KCC][3] = 68,
+	[1][0][0][0][RTW89_ACMA][3] = 58,
+	[1][0][0][0][RTW89_CN][3] = 60,
+	[1][0][0][0][RTW89_UK][3] = 58,
+	[1][0][0][0][RTW89_FCC][4] = 127,
+	[1][0][0][0][RTW89_ETSI][4] = 58,
+	[1][0][0][0][RTW89_MKK][4] = 76,
+	[1][0][0][0][RTW89_IC][4] = 127,
+	[1][0][0][0][RTW89_KCC][4] = 68,
+	[1][0][0][0][RTW89_ACMA][4] = 58,
+	[1][0][0][0][RTW89_CN][4] = 60,
+	[1][0][0][0][RTW89_UK][4] = 58,
+	[1][0][0][0][RTW89_FCC][5] = 127,
+	[1][0][0][0][RTW89_ETSI][5] = 58,
+	[1][0][0][0][RTW89_MKK][5] = 76,
+	[1][0][0][0][RTW89_IC][5] = 127,
+	[1][0][0][0][RTW89_KCC][5] = 68,
+	[1][0][0][0][RTW89_ACMA][5] = 58,
+	[1][0][0][0][RTW89_CN][5] = 60,
+	[1][0][0][0][RTW89_UK][5] = 58,
+	[1][0][0][0][RTW89_FCC][6] = 127,
+	[1][0][0][0][RTW89_ETSI][6] = 58,
+	[1][0][0][0][RTW89_MKK][6] = 76,
+	[1][0][0][0][RTW89_IC][6] = 127,
+	[1][0][0][0][RTW89_KCC][6] = 68,
+	[1][0][0][0][RTW89_ACMA][6] = 58,
+	[1][0][0][0][RTW89_CN][6] = 60,
+	[1][0][0][0][RTW89_UK][6] = 58,
+	[1][0][0][0][RTW89_FCC][7] = 127,
+	[1][0][0][0][RTW89_ETSI][7] = 58,
+	[1][0][0][0][RTW89_MKK][7] = 76,
+	[1][0][0][0][RTW89_IC][7] = 127,
+	[1][0][0][0][RTW89_KCC][7] = 68,
+	[1][0][0][0][RTW89_ACMA][7] = 58,
+	[1][0][0][0][RTW89_CN][7] = 60,
+	[1][0][0][0][RTW89_UK][7] = 58,
+	[1][0][0][0][RTW89_FCC][8] = 127,
+	[1][0][0][0][RTW89_ETSI][8] = 58,
+	[1][0][0][0][RTW89_MKK][8] = 76,
+	[1][0][0][0][RTW89_IC][8] = 127,
+	[1][0][0][0][RTW89_KCC][8] = 68,
+	[1][0][0][0][RTW89_ACMA][8] = 58,
+	[1][0][0][0][RTW89_CN][8] = 60,
+	[1][0][0][0][RTW89_UK][8] = 58,
+	[1][0][0][0][RTW89_FCC][9] = 127,
+	[1][0][0][0][RTW89_ETSI][9] = 58,
+	[1][0][0][0][RTW89_MKK][9] = 76,
+	[1][0][0][0][RTW89_IC][9] = 127,
+	[1][0][0][0][RTW89_KCC][9] = 68,
+	[1][0][0][0][RTW89_ACMA][9] = 58,
+	[1][0][0][0][RTW89_CN][9] = 60,
+	[1][0][0][0][RTW89_UK][9] = 58,
+	[1][0][0][0][RTW89_FCC][10] = 127,
+	[1][0][0][0][RTW89_ETSI][10] = 58,
+	[1][0][0][0][RTW89_MKK][10] = 66,
+	[1][0][0][0][RTW89_IC][10] = 127,
+	[1][0][0][0][RTW89_KCC][10] = 68,
+	[1][0][0][0][RTW89_ACMA][10] = 58,
+	[1][0][0][0][RTW89_CN][10] = 60,
+	[1][0][0][0][RTW89_UK][10] = 58,
+	[1][0][0][0][RTW89_FCC][11] = 127,
+	[1][0][0][0][RTW89_ETSI][11] = 127,
+	[1][0][0][0][RTW89_MKK][11] = 127,
+	[1][0][0][0][RTW89_IC][11] = 127,
+	[1][0][0][0][RTW89_KCC][11] = 127,
+	[1][0][0][0][RTW89_ACMA][11] = 127,
+	[1][0][0][0][RTW89_CN][11] = 127,
+	[1][0][0][0][RTW89_UK][11] = 127,
+	[1][0][0][0][RTW89_FCC][12] = 127,
+	[1][0][0][0][RTW89_ETSI][12] = 127,
+	[1][0][0][0][RTW89_MKK][12] = 127,
+	[1][0][0][0][RTW89_IC][12] = 127,
+	[1][0][0][0][RTW89_KCC][12] = 127,
+	[1][0][0][0][RTW89_ACMA][12] = 127,
+	[1][0][0][0][RTW89_CN][12] = 127,
+	[1][0][0][0][RTW89_UK][12] = 127,
+	[1][0][0][0][RTW89_FCC][13] = 127,
+	[1][0][0][0][RTW89_ETSI][13] = 127,
+	[1][0][0][0][RTW89_MKK][13] = 127,
+	[1][0][0][0][RTW89_IC][13] = 127,
+	[1][0][0][0][RTW89_KCC][13] = 127,
+	[1][0][0][0][RTW89_ACMA][13] = 127,
+	[1][0][0][0][RTW89_CN][13] = 127,
+	[1][0][0][0][RTW89_UK][13] = 127,
+	[1][1][0][0][RTW89_FCC][0] = 127,
+	[1][1][0][0][RTW89_ETSI][0] = 127,
+	[1][1][0][0][RTW89_MKK][0] = 127,
+	[1][1][0][0][RTW89_IC][0] = 127,
+	[1][1][0][0][RTW89_KCC][0] = 127,
+	[1][1][0][0][RTW89_ACMA][0] = 127,
+	[1][1][0][0][RTW89_CN][0] = 127,
+	[1][1][0][0][RTW89_UK][0] = 127,
+	[1][1][0][0][RTW89_FCC][1] = 127,
+	[1][1][0][0][RTW89_ETSI][1] = 127,
+	[1][1][0][0][RTW89_MKK][1] = 127,
+	[1][1][0][0][RTW89_IC][1] = 127,
+	[1][1][0][0][RTW89_KCC][1] = 127,
+	[1][1][0][0][RTW89_ACMA][1] = 127,
+	[1][1][0][0][RTW89_CN][1] = 127,
+	[1][1][0][0][RTW89_UK][1] = 127,
+	[1][1][0][0][RTW89_FCC][2] = 127,
+	[1][1][0][0][RTW89_ETSI][2] = 127,
+	[1][1][0][0][RTW89_MKK][2] = 127,
+	[1][1][0][0][RTW89_IC][2] = 127,
+	[1][1][0][0][RTW89_KCC][2] = 127,
+	[1][1][0][0][RTW89_ACMA][2] = 127,
+	[1][1][0][0][RTW89_CN][2] = 127,
+	[1][1][0][0][RTW89_UK][2] = 127,
+	[1][1][0][0][RTW89_FCC][3] = 127,
+	[1][1][0][0][RTW89_ETSI][3] = 127,
+	[1][1][0][0][RTW89_MKK][3] = 127,
+	[1][1][0][0][RTW89_IC][3] = 127,
+	[1][1][0][0][RTW89_KCC][3] = 127,
+	[1][1][0][0][RTW89_ACMA][3] = 127,
+	[1][1][0][0][RTW89_CN][3] = 127,
+	[1][1][0][0][RTW89_UK][3] = 127,
+	[1][1][0][0][RTW89_FCC][4] = 127,
+	[1][1][0][0][RTW89_ETSI][4] = 127,
+	[1][1][0][0][RTW89_MKK][4] = 127,
+	[1][1][0][0][RTW89_IC][4] = 127,
+	[1][1][0][0][RTW89_KCC][4] = 127,
+	[1][1][0][0][RTW89_ACMA][4] = 127,
+	[1][1][0][0][RTW89_CN][4] = 127,
+	[1][1][0][0][RTW89_UK][4] = 127,
+	[1][1][0][0][RTW89_FCC][5] = 127,
+	[1][1][0][0][RTW89_ETSI][5] = 127,
+	[1][1][0][0][RTW89_MKK][5] = 127,
+	[1][1][0][0][RTW89_IC][5] = 127,
+	[1][1][0][0][RTW89_KCC][5] = 127,
+	[1][1][0][0][RTW89_ACMA][5] = 127,
+	[1][1][0][0][RTW89_CN][5] = 127,
+	[1][1][0][0][RTW89_UK][5] = 127,
+	[1][1][0][0][RTW89_FCC][6] = 127,
+	[1][1][0][0][RTW89_ETSI][6] = 127,
+	[1][1][0][0][RTW89_MKK][6] = 127,
+	[1][1][0][0][RTW89_IC][6] = 127,
+	[1][1][0][0][RTW89_KCC][6] = 127,
+	[1][1][0][0][RTW89_ACMA][6] = 127,
+	[1][1][0][0][RTW89_CN][6] = 127,
+	[1][1][0][0][RTW89_UK][6] = 127,
+	[1][1][0][0][RTW89_FCC][7] = 127,
+	[1][1][0][0][RTW89_ETSI][7] = 127,
+	[1][1][0][0][RTW89_MKK][7] = 127,
+	[1][1][0][0][RTW89_IC][7] = 127,
+	[1][1][0][0][RTW89_KCC][7] = 127,
+	[1][1][0][0][RTW89_ACMA][7] = 127,
+	[1][1][0][0][RTW89_CN][7] = 127,
+	[1][1][0][0][RTW89_UK][7] = 127,
+	[1][1][0][0][RTW89_FCC][8] = 127,
+	[1][1][0][0][RTW89_ETSI][8] = 127,
+	[1][1][0][0][RTW89_MKK][8] = 127,
+	[1][1][0][0][RTW89_IC][8] = 127,
+	[1][1][0][0][RTW89_KCC][8] = 127,
+	[1][1][0][0][RTW89_ACMA][8] = 127,
+	[1][1][0][0][RTW89_CN][8] = 127,
+	[1][1][0][0][RTW89_UK][8] = 127,
+	[1][1][0][0][RTW89_FCC][9] = 127,
+	[1][1][0][0][RTW89_ETSI][9] = 127,
+	[1][1][0][0][RTW89_MKK][9] = 127,
+	[1][1][0][0][RTW89_IC][9] = 127,
+	[1][1][0][0][RTW89_KCC][9] = 127,
+	[1][1][0][0][RTW89_ACMA][9] = 127,
+	[1][1][0][0][RTW89_CN][9] = 127,
+	[1][1][0][0][RTW89_UK][9] = 127,
+	[1][1][0][0][RTW89_FCC][10] = 127,
+	[1][1][0][0][RTW89_ETSI][10] = 127,
+	[1][1][0][0][RTW89_MKK][10] = 127,
+	[1][1][0][0][RTW89_IC][10] = 127,
+	[1][1][0][0][RTW89_KCC][10] = 127,
+	[1][1][0][0][RTW89_ACMA][10] = 127,
+	[1][1][0][0][RTW89_CN][10] = 127,
+	[1][1][0][0][RTW89_UK][10] = 127,
+	[1][1][0][0][RTW89_FCC][11] = 127,
+	[1][1][0][0][RTW89_ETSI][11] = 127,
+	[1][1][0][0][RTW89_MKK][11] = 127,
+	[1][1][0][0][RTW89_IC][11] = 127,
+	[1][1][0][0][RTW89_KCC][11] = 127,
+	[1][1][0][0][RTW89_ACMA][11] = 127,
+	[1][1][0][0][RTW89_CN][11] = 127,
+	[1][1][0][0][RTW89_UK][11] = 127,
+	[1][1][0][0][RTW89_FCC][12] = 127,
+	[1][1][0][0][RTW89_ETSI][12] = 127,
+	[1][1][0][0][RTW89_MKK][12] = 127,
+	[1][1][0][0][RTW89_IC][12] = 127,
+	[1][1][0][0][RTW89_KCC][12] = 127,
+	[1][1][0][0][RTW89_ACMA][12] = 127,
+	[1][1][0][0][RTW89_CN][12] = 127,
+	[1][1][0][0][RTW89_UK][12] = 127,
+	[1][1][0][0][RTW89_FCC][13] = 127,
+	[1][1][0][0][RTW89_ETSI][13] = 127,
+	[1][1][0][0][RTW89_MKK][13] = 127,
+	[1][1][0][0][RTW89_IC][13] = 127,
+	[1][1][0][0][RTW89_KCC][13] = 127,
+	[1][1][0][0][RTW89_ACMA][13] = 127,
+	[1][1][0][0][RTW89_CN][13] = 127,
+	[1][1][0][0][RTW89_UK][13] = 127,
+	[0][0][1][0][RTW89_FCC][0] = 78,
+	[0][0][1][0][RTW89_ETSI][0] = 58,
+	[0][0][1][0][RTW89_MKK][0] = 72,
+	[0][0][1][0][RTW89_IC][0] = 78,
+	[0][0][1][0][RTW89_KCC][0] = 76,
+	[0][0][1][0][RTW89_ACMA][0] = 58,
+	[0][0][1][0][RTW89_CN][0] = 60,
+	[0][0][1][0][RTW89_UK][0] = 58,
+	[0][0][1][0][RTW89_FCC][1] = 78,
+	[0][0][1][0][RTW89_ETSI][1] = 60,
+	[0][0][1][0][RTW89_MKK][1] = 74,
+	[0][0][1][0][RTW89_IC][1] = 78,
+	[0][0][1][0][RTW89_KCC][1] = 76,
+	[0][0][1][0][RTW89_ACMA][1] = 60,
+	[0][0][1][0][RTW89_CN][1] = 60,
+	[0][0][1][0][RTW89_UK][1] = 60,
+	[0][0][1][0][RTW89_FCC][2] = 80,
+	[0][0][1][0][RTW89_ETSI][2] = 60,
+	[0][0][1][0][RTW89_MKK][2] = 74,
+	[0][0][1][0][RTW89_IC][2] = 80,
+	[0][0][1][0][RTW89_KCC][2] = 76,
+	[0][0][1][0][RTW89_ACMA][2] = 60,
+	[0][0][1][0][RTW89_CN][2] = 60,
+	[0][0][1][0][RTW89_UK][2] = 60,
+	[0][0][1][0][RTW89_FCC][3] = 80,
+	[0][0][1][0][RTW89_ETSI][3] = 60,
+	[0][0][1][0][RTW89_MKK][3] = 74,
+	[0][0][1][0][RTW89_IC][3] = 80,
+	[0][0][1][0][RTW89_KCC][3] = 76,
+	[0][0][1][0][RTW89_ACMA][3] = 60,
+	[0][0][1][0][RTW89_CN][3] = 60,
+	[0][0][1][0][RTW89_UK][3] = 60,
+	[0][0][1][0][RTW89_FCC][4] = 80,
+	[0][0][1][0][RTW89_ETSI][4] = 60,
+	[0][0][1][0][RTW89_MKK][4] = 74,
+	[0][0][1][0][RTW89_IC][4] = 80,
+	[0][0][1][0][RTW89_KCC][4] = 76,
+	[0][0][1][0][RTW89_ACMA][4] = 60,
+	[0][0][1][0][RTW89_CN][4] = 60,
+	[0][0][1][0][RTW89_UK][4] = 60,
+	[0][0][1][0][RTW89_FCC][5] = 80,
+	[0][0][1][0][RTW89_ETSI][5] = 60,
+	[0][0][1][0][RTW89_MKK][5] = 74,
+	[0][0][1][0][RTW89_IC][5] = 80,
+	[0][0][1][0][RTW89_KCC][5] = 76,
+	[0][0][1][0][RTW89_ACMA][5] = 60,
+	[0][0][1][0][RTW89_CN][5] = 60,
+	[0][0][1][0][RTW89_UK][5] = 60,
+	[0][0][1][0][RTW89_FCC][6] = 80,
+	[0][0][1][0][RTW89_ETSI][6] = 60,
+	[0][0][1][0][RTW89_MKK][6] = 74,
+	[0][0][1][0][RTW89_IC][6] = 80,
+	[0][0][1][0][RTW89_KCC][6] = 76,
+	[0][0][1][0][RTW89_ACMA][6] = 60,
+	[0][0][1][0][RTW89_CN][6] = 60,
+	[0][0][1][0][RTW89_UK][6] = 60,
+	[0][0][1][0][RTW89_FCC][7] = 80,
+	[0][0][1][0][RTW89_ETSI][7] = 60,
+	[0][0][1][0][RTW89_MKK][7] = 74,
+	[0][0][1][0][RTW89_IC][7] = 80,
+	[0][0][1][0][RTW89_KCC][7] = 76,
+	[0][0][1][0][RTW89_ACMA][7] = 60,
+	[0][0][1][0][RTW89_CN][7] = 60,
+	[0][0][1][0][RTW89_UK][7] = 60,
+	[0][0][1][0][RTW89_FCC][8] = 80,
+	[0][0][1][0][RTW89_ETSI][8] = 60,
+	[0][0][1][0][RTW89_MKK][8] = 74,
+	[0][0][1][0][RTW89_IC][8] = 80,
+	[0][0][1][0][RTW89_KCC][8] = 76,
+	[0][0][1][0][RTW89_ACMA][8] = 60,
+	[0][0][1][0][RTW89_CN][8] = 60,
+	[0][0][1][0][RTW89_UK][8] = 60,
+	[0][0][1][0][RTW89_FCC][9] = 76,
+	[0][0][1][0][RTW89_ETSI][9] = 60,
+	[0][0][1][0][RTW89_MKK][9] = 74,
+	[0][0][1][0][RTW89_IC][9] = 76,
+	[0][0][1][0][RTW89_KCC][9] = 74,
+	[0][0][1][0][RTW89_ACMA][9] = 60,
+	[0][0][1][0][RTW89_CN][9] = 60,
+	[0][0][1][0][RTW89_UK][9] = 60,
+	[0][0][1][0][RTW89_FCC][10] = 76,
+	[0][0][1][0][RTW89_ETSI][10] = 60,
+	[0][0][1][0][RTW89_MKK][10] = 74,
+	[0][0][1][0][RTW89_IC][10] = 76,
+	[0][0][1][0][RTW89_KCC][10] = 74,
+	[0][0][1][0][RTW89_ACMA][10] = 60,
+	[0][0][1][0][RTW89_CN][10] = 60,
+	[0][0][1][0][RTW89_UK][10] = 60,
+	[0][0][1][0][RTW89_FCC][11] = 68,
+	[0][0][1][0][RTW89_ETSI][11] = 60,
+	[0][0][1][0][RTW89_MKK][11] = 74,
+	[0][0][1][0][RTW89_IC][11] = 68,
+	[0][0][1][0][RTW89_KCC][11] = 74,
+	[0][0][1][0][RTW89_ACMA][11] = 60,
+	[0][0][1][0][RTW89_CN][11] = 60,
+	[0][0][1][0][RTW89_UK][11] = 60,
+	[0][0][1][0][RTW89_FCC][12] = 64,
+	[0][0][1][0][RTW89_ETSI][12] = 58,
+	[0][0][1][0][RTW89_MKK][12] = 70,
+	[0][0][1][0][RTW89_IC][12] = 64,
+	[0][0][1][0][RTW89_KCC][12] = 74,
+	[0][0][1][0][RTW89_ACMA][12] = 58,
+	[0][0][1][0][RTW89_CN][12] = 60,
+	[0][0][1][0][RTW89_UK][12] = 58,
+	[0][0][1][0][RTW89_FCC][13] = 127,
+	[0][0][1][0][RTW89_ETSI][13] = 127,
+	[0][0][1][0][RTW89_MKK][13] = 127,
+	[0][0][1][0][RTW89_IC][13] = 127,
+	[0][0][1][0][RTW89_KCC][13] = 127,
+	[0][0][1][0][RTW89_ACMA][13] = 127,
+	[0][0][1][0][RTW89_CN][13] = 127,
+	[0][0][1][0][RTW89_UK][13] = 127,
+	[0][1][1][0][RTW89_FCC][0] = 127,
+	[0][1][1][0][RTW89_ETSI][0] = 127,
+	[0][1][1][0][RTW89_MKK][0] = 127,
+	[0][1][1][0][RTW89_IC][0] = 127,
+	[0][1][1][0][RTW89_KCC][0] = 127,
+	[0][1][1][0][RTW89_ACMA][0] = 127,
+	[0][1][1][0][RTW89_CN][0] = 127,
+	[0][1][1][0][RTW89_UK][0] = 127,
+	[0][1][1][0][RTW89_FCC][1] = 127,
+	[0][1][1][0][RTW89_ETSI][1] = 127,
+	[0][1][1][0][RTW89_MKK][1] = 127,
+	[0][1][1][0][RTW89_IC][1] = 127,
+	[0][1][1][0][RTW89_KCC][1] = 127,
+	[0][1][1][0][RTW89_ACMA][1] = 127,
+	[0][1][1][0][RTW89_CN][1] = 127,
+	[0][1][1][0][RTW89_UK][1] = 127,
+	[0][1][1][0][RTW89_FCC][2] = 127,
+	[0][1][1][0][RTW89_ETSI][2] = 127,
+	[0][1][1][0][RTW89_MKK][2] = 127,
+	[0][1][1][0][RTW89_IC][2] = 127,
+	[0][1][1][0][RTW89_KCC][2] = 127,
+	[0][1][1][0][RTW89_ACMA][2] = 127,
+	[0][1][1][0][RTW89_CN][2] = 127,
+	[0][1][1][0][RTW89_UK][2] = 127,
+	[0][1][1][0][RTW89_FCC][3] = 127,
+	[0][1][1][0][RTW89_ETSI][3] = 127,
+	[0][1][1][0][RTW89_MKK][3] = 127,
+	[0][1][1][0][RTW89_IC][3] = 127,
+	[0][1][1][0][RTW89_KCC][3] = 127,
+	[0][1][1][0][RTW89_ACMA][3] = 127,
+	[0][1][1][0][RTW89_CN][3] = 127,
+	[0][1][1][0][RTW89_UK][3] = 127,
+	[0][1][1][0][RTW89_FCC][4] = 127,
+	[0][1][1][0][RTW89_ETSI][4] = 127,
+	[0][1][1][0][RTW89_MKK][4] = 127,
+	[0][1][1][0][RTW89_IC][4] = 127,
+	[0][1][1][0][RTW89_KCC][4] = 127,
+	[0][1][1][0][RTW89_ACMA][4] = 127,
+	[0][1][1][0][RTW89_CN][4] = 127,
+	[0][1][1][0][RTW89_UK][4] = 127,
+	[0][1][1][0][RTW89_FCC][5] = 127,
+	[0][1][1][0][RTW89_ETSI][5] = 127,
+	[0][1][1][0][RTW89_MKK][5] = 127,
+	[0][1][1][0][RTW89_IC][5] = 127,
+	[0][1][1][0][RTW89_KCC][5] = 127,
+	[0][1][1][0][RTW89_ACMA][5] = 127,
+	[0][1][1][0][RTW89_CN][5] = 127,
+	[0][1][1][0][RTW89_UK][5] = 127,
+	[0][1][1][0][RTW89_FCC][6] = 127,
+	[0][1][1][0][RTW89_ETSI][6] = 127,
+	[0][1][1][0][RTW89_MKK][6] = 127,
+	[0][1][1][0][RTW89_IC][6] = 127,
+	[0][1][1][0][RTW89_KCC][6] = 127,
+	[0][1][1][0][RTW89_ACMA][6] = 127,
+	[0][1][1][0][RTW89_CN][6] = 127,
+	[0][1][1][0][RTW89_UK][6] = 127,
+	[0][1][1][0][RTW89_FCC][7] = 127,
+	[0][1][1][0][RTW89_ETSI][7] = 127,
+	[0][1][1][0][RTW89_MKK][7] = 127,
+	[0][1][1][0][RTW89_IC][7] = 127,
+	[0][1][1][0][RTW89_KCC][7] = 127,
+	[0][1][1][0][RTW89_ACMA][7] = 127,
+	[0][1][1][0][RTW89_CN][7] = 127,
+	[0][1][1][0][RTW89_UK][7] = 127,
+	[0][1][1][0][RTW89_FCC][8] = 127,
+	[0][1][1][0][RTW89_ETSI][8] = 127,
+	[0][1][1][0][RTW89_MKK][8] = 127,
+	[0][1][1][0][RTW89_IC][8] = 127,
+	[0][1][1][0][RTW89_KCC][8] = 127,
+	[0][1][1][0][RTW89_ACMA][8] = 127,
+	[0][1][1][0][RTW89_CN][8] = 127,
+	[0][1][1][0][RTW89_UK][8] = 127,
+	[0][1][1][0][RTW89_FCC][9] = 127,
+	[0][1][1][0][RTW89_ETSI][9] = 127,
+	[0][1][1][0][RTW89_MKK][9] = 127,
+	[0][1][1][0][RTW89_IC][9] = 127,
+	[0][1][1][0][RTW89_KCC][9] = 127,
+	[0][1][1][0][RTW89_ACMA][9] = 127,
+	[0][1][1][0][RTW89_CN][9] = 127,
+	[0][1][1][0][RTW89_UK][9] = 127,
+	[0][1][1][0][RTW89_FCC][10] = 127,
+	[0][1][1][0][RTW89_ETSI][10] = 127,
+	[0][1][1][0][RTW89_MKK][10] = 127,
+	[0][1][1][0][RTW89_IC][10] = 127,
+	[0][1][1][0][RTW89_KCC][10] = 127,
+	[0][1][1][0][RTW89_ACMA][10] = 127,
+	[0][1][1][0][RTW89_CN][10] = 127,
+	[0][1][1][0][RTW89_UK][10] = 127,
+	[0][1][1][0][RTW89_FCC][11] = 127,
+	[0][1][1][0][RTW89_ETSI][11] = 127,
+	[0][1][1][0][RTW89_MKK][11] = 127,
+	[0][1][1][0][RTW89_IC][11] = 127,
+	[0][1][1][0][RTW89_KCC][11] = 127,
+	[0][1][1][0][RTW89_ACMA][11] = 127,
+	[0][1][1][0][RTW89_CN][11] = 127,
+	[0][1][1][0][RTW89_UK][11] = 127,
+	[0][1][1][0][RTW89_FCC][12] = 127,
+	[0][1][1][0][RTW89_ETSI][12] = 127,
+	[0][1][1][0][RTW89_MKK][12] = 127,
+	[0][1][1][0][RTW89_IC][12] = 127,
+	[0][1][1][0][RTW89_KCC][12] = 127,
+	[0][1][1][0][RTW89_ACMA][12] = 127,
+	[0][1][1][0][RTW89_CN][12] = 127,
+	[0][1][1][0][RTW89_UK][12] = 127,
+	[0][1][1][0][RTW89_FCC][13] = 127,
+	[0][1][1][0][RTW89_ETSI][13] = 127,
+	[0][1][1][0][RTW89_MKK][13] = 127,
+	[0][1][1][0][RTW89_IC][13] = 127,
+	[0][1][1][0][RTW89_KCC][13] = 127,
+	[0][1][1][0][RTW89_ACMA][13] = 127,
+	[0][1][1][0][RTW89_CN][13] = 127,
+	[0][1][1][0][RTW89_UK][13] = 127,
+	[0][0][2][0][RTW89_FCC][0] = 78,
+	[0][0][2][0][RTW89_ETSI][0] = 60,
+	[0][0][2][0][RTW89_MKK][0] = 72,
+	[0][0][2][0][RTW89_IC][0] = 78,
+	[0][0][2][0][RTW89_KCC][0] = 76,
+	[0][0][2][0][RTW89_ACMA][0] = 60,
+	[0][0][2][0][RTW89_CN][0] = 60,
+	[0][0][2][0][RTW89_UK][0] = 60,
+	[0][0][2][0][RTW89_FCC][1] = 78,
+	[0][0][2][0][RTW89_ETSI][1] = 60,
+	[0][0][2][0][RTW89_MKK][1] = 76,
+	[0][0][2][0][RTW89_IC][1] = 78,
+	[0][0][2][0][RTW89_KCC][1] = 76,
+	[0][0][2][0][RTW89_ACMA][1] = 60,
+	[0][0][2][0][RTW89_CN][1] = 60,
+	[0][0][2][0][RTW89_UK][1] = 60,
+	[0][0][2][0][RTW89_FCC][2] = 80,
+	[0][0][2][0][RTW89_ETSI][2] = 60,
+	[0][0][2][0][RTW89_MKK][2] = 76,
+	[0][0][2][0][RTW89_IC][2] = 80,
+	[0][0][2][0][RTW89_KCC][2] = 76,
+	[0][0][2][0][RTW89_ACMA][2] = 60,
+	[0][0][2][0][RTW89_CN][2] = 60,
+	[0][0][2][0][RTW89_UK][2] = 60,
+	[0][0][2][0][RTW89_FCC][3] = 80,
+	[0][0][2][0][RTW89_ETSI][3] = 60,
+	[0][0][2][0][RTW89_MKK][3] = 76,
+	[0][0][2][0][RTW89_IC][3] = 80,
+	[0][0][2][0][RTW89_KCC][3] = 76,
+	[0][0][2][0][RTW89_ACMA][3] = 60,
+	[0][0][2][0][RTW89_CN][3] = 60,
+	[0][0][2][0][RTW89_UK][3] = 60,
+	[0][0][2][0][RTW89_FCC][4] = 80,
+	[0][0][2][0][RTW89_ETSI][4] = 60,
+	[0][0][2][0][RTW89_MKK][4] = 76,
+	[0][0][2][0][RTW89_IC][4] = 80,
+	[0][0][2][0][RTW89_KCC][4] = 76,
+	[0][0][2][0][RTW89_ACMA][4] = 60,
+	[0][0][2][0][RTW89_CN][4] = 60,
+	[0][0][2][0][RTW89_UK][4] = 60,
+	[0][0][2][0][RTW89_FCC][5] = 80,
+	[0][0][2][0][RTW89_ETSI][5] = 60,
+	[0][0][2][0][RTW89_MKK][5] = 76,
+	[0][0][2][0][RTW89_IC][5] = 80,
+	[0][0][2][0][RTW89_KCC][5] = 76,
+	[0][0][2][0][RTW89_ACMA][5] = 60,
+	[0][0][2][0][RTW89_CN][5] = 60,
+	[0][0][2][0][RTW89_UK][5] = 60,
+	[0][0][2][0][RTW89_FCC][6] = 80,
+	[0][0][2][0][RTW89_ETSI][6] = 60,
+	[0][0][2][0][RTW89_MKK][6] = 76,
+	[0][0][2][0][RTW89_IC][6] = 80,
+	[0][0][2][0][RTW89_KCC][6] = 76,
+	[0][0][2][0][RTW89_ACMA][6] = 60,
+	[0][0][2][0][RTW89_CN][6] = 60,
+	[0][0][2][0][RTW89_UK][6] = 60,
+	[0][0][2][0][RTW89_FCC][7] = 80,
+	[0][0][2][0][RTW89_ETSI][7] = 60,
+	[0][0][2][0][RTW89_MKK][7] = 76,
+	[0][0][2][0][RTW89_IC][7] = 80,
+	[0][0][2][0][RTW89_KCC][7] = 76,
+	[0][0][2][0][RTW89_ACMA][7] = 60,
+	[0][0][2][0][RTW89_CN][7] = 60,
+	[0][0][2][0][RTW89_UK][7] = 60,
+	[0][0][2][0][RTW89_FCC][8] = 78,
+	[0][0][2][0][RTW89_ETSI][8] = 60,
+	[0][0][2][0][RTW89_MKK][8] = 76,
+	[0][0][2][0][RTW89_IC][8] = 78,
+	[0][0][2][0][RTW89_KCC][8] = 76,
+	[0][0][2][0][RTW89_ACMA][8] = 60,
+	[0][0][2][0][RTW89_CN][8] = 60,
+	[0][0][2][0][RTW89_UK][8] = 60,
+	[0][0][2][0][RTW89_FCC][9] = 74,
+	[0][0][2][0][RTW89_ETSI][9] = 60,
+	[0][0][2][0][RTW89_MKK][9] = 76,
+	[0][0][2][0][RTW89_IC][9] = 74,
+	[0][0][2][0][RTW89_KCC][9] = 76,
+	[0][0][2][0][RTW89_ACMA][9] = 60,
+	[0][0][2][0][RTW89_CN][9] = 60,
+	[0][0][2][0][RTW89_UK][9] = 60,
+	[0][0][2][0][RTW89_FCC][10] = 74,
+	[0][0][2][0][RTW89_ETSI][10] = 60,
+	[0][0][2][0][RTW89_MKK][10] = 76,
+	[0][0][2][0][RTW89_IC][10] = 74,
+	[0][0][2][0][RTW89_KCC][10] = 76,
+	[0][0][2][0][RTW89_ACMA][10] = 60,
+	[0][0][2][0][RTW89_CN][10] = 60,
+	[0][0][2][0][RTW89_UK][10] = 60,
+	[0][0][2][0][RTW89_FCC][11] = 68,
+	[0][0][2][0][RTW89_ETSI][11] = 60,
+	[0][0][2][0][RTW89_MKK][11] = 76,
+	[0][0][2][0][RTW89_IC][11] = 68,
+	[0][0][2][0][RTW89_KCC][11] = 76,
+	[0][0][2][0][RTW89_ACMA][11] = 60,
+	[0][0][2][0][RTW89_CN][11] = 60,
+	[0][0][2][0][RTW89_UK][11] = 60,
+	[0][0][2][0][RTW89_FCC][12] = 68,
+	[0][0][2][0][RTW89_ETSI][12] = 60,
+	[0][0][2][0][RTW89_MKK][12] = 70,
+	[0][0][2][0][RTW89_IC][12] = 68,
+	[0][0][2][0][RTW89_KCC][12] = 76,
+	[0][0][2][0][RTW89_ACMA][12] = 60,
+	[0][0][2][0][RTW89_CN][12] = 60,
+	[0][0][2][0][RTW89_UK][12] = 60,
+	[0][0][2][0][RTW89_FCC][13] = 127,
+	[0][0][2][0][RTW89_ETSI][13] = 127,
+	[0][0][2][0][RTW89_MKK][13] = 127,
+	[0][0][2][0][RTW89_IC][13] = 127,
+	[0][0][2][0][RTW89_KCC][13] = 127,
+	[0][0][2][0][RTW89_ACMA][13] = 127,
+	[0][0][2][0][RTW89_CN][13] = 127,
+	[0][0][2][0][RTW89_UK][13] = 127,
+	[0][1][2][0][RTW89_FCC][0] = 127,
+	[0][1][2][0][RTW89_ETSI][0] = 127,
+	[0][1][2][0][RTW89_MKK][0] = 127,
+	[0][1][2][0][RTW89_IC][0] = 127,
+	[0][1][2][0][RTW89_KCC][0] = 127,
+	[0][1][2][0][RTW89_ACMA][0] = 127,
+	[0][1][2][0][RTW89_CN][0] = 127,
+	[0][1][2][0][RTW89_UK][0] = 127,
+	[0][1][2][0][RTW89_FCC][1] = 127,
+	[0][1][2][0][RTW89_ETSI][1] = 127,
+	[0][1][2][0][RTW89_MKK][1] = 127,
+	[0][1][2][0][RTW89_IC][1] = 127,
+	[0][1][2][0][RTW89_KCC][1] = 127,
+	[0][1][2][0][RTW89_ACMA][1] = 127,
+	[0][1][2][0][RTW89_CN][1] = 127,
+	[0][1][2][0][RTW89_UK][1] = 127,
+	[0][1][2][0][RTW89_FCC][2] = 127,
+	[0][1][2][0][RTW89_ETSI][2] = 127,
+	[0][1][2][0][RTW89_MKK][2] = 127,
+	[0][1][2][0][RTW89_IC][2] = 127,
+	[0][1][2][0][RTW89_KCC][2] = 127,
+	[0][1][2][0][RTW89_ACMA][2] = 127,
+	[0][1][2][0][RTW89_CN][2] = 127,
+	[0][1][2][0][RTW89_UK][2] = 127,
+	[0][1][2][0][RTW89_FCC][3] = 127,
+	[0][1][2][0][RTW89_ETSI][3] = 127,
+	[0][1][2][0][RTW89_MKK][3] = 127,
+	[0][1][2][0][RTW89_IC][3] = 127,
+	[0][1][2][0][RTW89_KCC][3] = 127,
+	[0][1][2][0][RTW89_ACMA][3] = 127,
+	[0][1][2][0][RTW89_CN][3] = 127,
+	[0][1][2][0][RTW89_UK][3] = 127,
+	[0][1][2][0][RTW89_FCC][4] = 127,
+	[0][1][2][0][RTW89_ETSI][4] = 127,
+	[0][1][2][0][RTW89_MKK][4] = 127,
+	[0][1][2][0][RTW89_IC][4] = 127,
+	[0][1][2][0][RTW89_KCC][4] = 127,
+	[0][1][2][0][RTW89_ACMA][4] = 127,
+	[0][1][2][0][RTW89_CN][4] = 127,
+	[0][1][2][0][RTW89_UK][4] = 127,
+	[0][1][2][0][RTW89_FCC][5] = 127,
+	[0][1][2][0][RTW89_ETSI][5] = 127,
+	[0][1][2][0][RTW89_MKK][5] = 127,
+	[0][1][2][0][RTW89_IC][5] = 127,
+	[0][1][2][0][RTW89_KCC][5] = 127,
+	[0][1][2][0][RTW89_ACMA][5] = 127,
+	[0][1][2][0][RTW89_CN][5] = 127,
+	[0][1][2][0][RTW89_UK][5] = 127,
+	[0][1][2][0][RTW89_FCC][6] = 127,
+	[0][1][2][0][RTW89_ETSI][6] = 127,
+	[0][1][2][0][RTW89_MKK][6] = 127,
+	[0][1][2][0][RTW89_IC][6] = 127,
+	[0][1][2][0][RTW89_KCC][6] = 127,
+	[0][1][2][0][RTW89_ACMA][6] = 127,
+	[0][1][2][0][RTW89_CN][6] = 127,
+	[0][1][2][0][RTW89_UK][6] = 127,
+	[0][1][2][0][RTW89_FCC][7] = 127,
+	[0][1][2][0][RTW89_ETSI][7] = 127,
+	[0][1][2][0][RTW89_MKK][7] = 127,
+	[0][1][2][0][RTW89_IC][7] = 127,
+	[0][1][2][0][RTW89_KCC][7] = 127,
+	[0][1][2][0][RTW89_ACMA][7] = 127,
+	[0][1][2][0][RTW89_CN][7] = 127,
+	[0][1][2][0][RTW89_UK][7] = 127,
+	[0][1][2][0][RTW89_FCC][8] = 127,
+	[0][1][2][0][RTW89_ETSI][8] = 127,
+	[0][1][2][0][RTW89_MKK][8] = 127,
+	[0][1][2][0][RTW89_IC][8] = 127,
+	[0][1][2][0][RTW89_KCC][8] = 127,
+	[0][1][2][0][RTW89_ACMA][8] = 127,
+	[0][1][2][0][RTW89_CN][8] = 127,
+	[0][1][2][0][RTW89_UK][8] = 127,
+	[0][1][2][0][RTW89_FCC][9] = 127,
+	[0][1][2][0][RTW89_ETSI][9] = 127,
+	[0][1][2][0][RTW89_MKK][9] = 127,
+	[0][1][2][0][RTW89_IC][9] = 127,
+	[0][1][2][0][RTW89_KCC][9] = 127,
+	[0][1][2][0][RTW89_ACMA][9] = 127,
+	[0][1][2][0][RTW89_CN][9] = 127,
+	[0][1][2][0][RTW89_UK][9] = 127,
+	[0][1][2][0][RTW89_FCC][10] = 127,
+	[0][1][2][0][RTW89_ETSI][10] = 127,
+	[0][1][2][0][RTW89_MKK][10] = 127,
+	[0][1][2][0][RTW89_IC][10] = 127,
+	[0][1][2][0][RTW89_KCC][10] = 127,
+	[0][1][2][0][RTW89_ACMA][10] = 127,
+	[0][1][2][0][RTW89_CN][10] = 127,
+	[0][1][2][0][RTW89_UK][10] = 127,
+	[0][1][2][0][RTW89_FCC][11] = 127,
+	[0][1][2][0][RTW89_ETSI][11] = 127,
+	[0][1][2][0][RTW89_MKK][11] = 127,
+	[0][1][2][0][RTW89_IC][11] = 127,
+	[0][1][2][0][RTW89_KCC][11] = 127,
+	[0][1][2][0][RTW89_ACMA][11] = 127,
+	[0][1][2][0][RTW89_CN][11] = 127,
+	[0][1][2][0][RTW89_UK][11] = 127,
+	[0][1][2][0][RTW89_FCC][12] = 127,
+	[0][1][2][0][RTW89_ETSI][12] = 127,
+	[0][1][2][0][RTW89_MKK][12] = 127,
+	[0][1][2][0][RTW89_IC][12] = 127,
+	[0][1][2][0][RTW89_KCC][12] = 127,
+	[0][1][2][0][RTW89_ACMA][12] = 127,
+	[0][1][2][0][RTW89_CN][12] = 127,
+	[0][1][2][0][RTW89_UK][12] = 127,
+	[0][1][2][0][RTW89_FCC][13] = 127,
+	[0][1][2][0][RTW89_ETSI][13] = 127,
+	[0][1][2][0][RTW89_MKK][13] = 127,
+	[0][1][2][0][RTW89_IC][13] = 127,
+	[0][1][2][0][RTW89_KCC][13] = 127,
+	[0][1][2][0][RTW89_ACMA][13] = 127,
+	[0][1][2][0][RTW89_CN][13] = 127,
+	[0][1][2][0][RTW89_UK][13] = 127,
+	[0][1][2][1][RTW89_FCC][0] = 127,
+	[0][1][2][1][RTW89_ETSI][0] = 127,
+	[0][1][2][1][RTW89_MKK][0] = 127,
+	[0][1][2][1][RTW89_IC][0] = 127,
+	[0][1][2][1][RTW89_KCC][0] = 127,
+	[0][1][2][1][RTW89_ACMA][0] = 127,
+	[0][1][2][1][RTW89_CN][0] = 127,
+	[0][1][2][1][RTW89_UK][0] = 127,
+	[0][1][2][1][RTW89_FCC][1] = 127,
+	[0][1][2][1][RTW89_ETSI][1] = 127,
+	[0][1][2][1][RTW89_MKK][1] = 127,
+	[0][1][2][1][RTW89_IC][1] = 127,
+	[0][1][2][1][RTW89_KCC][1] = 127,
+	[0][1][2][1][RTW89_ACMA][1] = 127,
+	[0][1][2][1][RTW89_CN][1] = 127,
+	[0][1][2][1][RTW89_UK][1] = 127,
+	[0][1][2][1][RTW89_FCC][2] = 127,
+	[0][1][2][1][RTW89_ETSI][2] = 127,
+	[0][1][2][1][RTW89_MKK][2] = 127,
+	[0][1][2][1][RTW89_IC][2] = 127,
+	[0][1][2][1][RTW89_KCC][2] = 127,
+	[0][1][2][1][RTW89_ACMA][2] = 127,
+	[0][1][2][1][RTW89_CN][2] = 127,
+	[0][1][2][1][RTW89_UK][2] = 127,
+	[0][1][2][1][RTW89_FCC][3] = 127,
+	[0][1][2][1][RTW89_ETSI][3] = 127,
+	[0][1][2][1][RTW89_MKK][3] = 127,
+	[0][1][2][1][RTW89_IC][3] = 127,
+	[0][1][2][1][RTW89_KCC][3] = 127,
+	[0][1][2][1][RTW89_ACMA][3] = 127,
+	[0][1][2][1][RTW89_CN][3] = 127,
+	[0][1][2][1][RTW89_UK][3] = 127,
+	[0][1][2][1][RTW89_FCC][4] = 127,
+	[0][1][2][1][RTW89_ETSI][4] = 127,
+	[0][1][2][1][RTW89_MKK][4] = 127,
+	[0][1][2][1][RTW89_IC][4] = 127,
+	[0][1][2][1][RTW89_KCC][4] = 127,
+	[0][1][2][1][RTW89_ACMA][4] = 127,
+	[0][1][2][1][RTW89_CN][4] = 127,
+	[0][1][2][1][RTW89_UK][4] = 127,
+	[0][1][2][1][RTW89_FCC][5] = 127,
+	[0][1][2][1][RTW89_ETSI][5] = 127,
+	[0][1][2][1][RTW89_MKK][5] = 127,
+	[0][1][2][1][RTW89_IC][5] = 127,
+	[0][1][2][1][RTW89_KCC][5] = 127,
+	[0][1][2][1][RTW89_ACMA][5] = 127,
+	[0][1][2][1][RTW89_CN][5] = 127,
+	[0][1][2][1][RTW89_UK][5] = 127,
+	[0][1][2][1][RTW89_FCC][6] = 127,
+	[0][1][2][1][RTW89_ETSI][6] = 127,
+	[0][1][2][1][RTW89_MKK][6] = 127,
+	[0][1][2][1][RTW89_IC][6] = 127,
+	[0][1][2][1][RTW89_KCC][6] = 127,
+	[0][1][2][1][RTW89_ACMA][6] = 127,
+	[0][1][2][1][RTW89_CN][6] = 127,
+	[0][1][2][1][RTW89_UK][6] = 127,
+	[0][1][2][1][RTW89_FCC][7] = 127,
+	[0][1][2][1][RTW89_ETSI][7] = 127,
+	[0][1][2][1][RTW89_MKK][7] = 127,
+	[0][1][2][1][RTW89_IC][7] = 127,
+	[0][1][2][1][RTW89_KCC][7] = 127,
+	[0][1][2][1][RTW89_ACMA][7] = 127,
+	[0][1][2][1][RTW89_CN][7] = 127,
+	[0][1][2][1][RTW89_UK][7] = 127,
+	[0][1][2][1][RTW89_FCC][8] = 127,
+	[0][1][2][1][RTW89_ETSI][8] = 127,
+	[0][1][2][1][RTW89_MKK][8] = 127,
+	[0][1][2][1][RTW89_IC][8] = 127,
+	[0][1][2][1][RTW89_KCC][8] = 127,
+	[0][1][2][1][RTW89_ACMA][8] = 127,
+	[0][1][2][1][RTW89_CN][8] = 127,
+	[0][1][2][1][RTW89_UK][8] = 127,
+	[0][1][2][1][RTW89_FCC][9] = 127,
+	[0][1][2][1][RTW89_ETSI][9] = 127,
+	[0][1][2][1][RTW89_MKK][9] = 127,
+	[0][1][2][1][RTW89_IC][9] = 127,
+	[0][1][2][1][RTW89_KCC][9] = 127,
+	[0][1][2][1][RTW89_ACMA][9] = 127,
+	[0][1][2][1][RTW89_CN][9] = 127,
+	[0][1][2][1][RTW89_UK][9] = 127,
+	[0][1][2][1][RTW89_FCC][10] = 127,
+	[0][1][2][1][RTW89_ETSI][10] = 127,
+	[0][1][2][1][RTW89_MKK][10] = 127,
+	[0][1][2][1][RTW89_IC][10] = 127,
+	[0][1][2][1][RTW89_KCC][10] = 127,
+	[0][1][2][1][RTW89_ACMA][10] = 127,
+	[0][1][2][1][RTW89_CN][10] = 127,
+	[0][1][2][1][RTW89_UK][10] = 127,
+	[0][1][2][1][RTW89_FCC][11] = 127,
+	[0][1][2][1][RTW89_ETSI][11] = 127,
+	[0][1][2][1][RTW89_MKK][11] = 127,
+	[0][1][2][1][RTW89_IC][11] = 127,
+	[0][1][2][1][RTW89_KCC][11] = 127,
+	[0][1][2][1][RTW89_ACMA][11] = 127,
+	[0][1][2][1][RTW89_CN][11] = 127,
+	[0][1][2][1][RTW89_UK][11] = 127,
+	[0][1][2][1][RTW89_FCC][12] = 127,
+	[0][1][2][1][RTW89_ETSI][12] = 127,
+	[0][1][2][1][RTW89_MKK][12] = 127,
+	[0][1][2][1][RTW89_IC][12] = 127,
+	[0][1][2][1][RTW89_KCC][12] = 127,
+	[0][1][2][1][RTW89_ACMA][12] = 127,
+	[0][1][2][1][RTW89_CN][12] = 127,
+	[0][1][2][1][RTW89_UK][12] = 127,
+	[0][1][2][1][RTW89_FCC][13] = 127,
+	[0][1][2][1][RTW89_ETSI][13] = 127,
+	[0][1][2][1][RTW89_MKK][13] = 127,
+	[0][1][2][1][RTW89_IC][13] = 127,
+	[0][1][2][1][RTW89_KCC][13] = 127,
+	[0][1][2][1][RTW89_ACMA][13] = 127,
+	[0][1][2][1][RTW89_CN][13] = 127,
+	[0][1][2][1][RTW89_UK][13] = 127,
+	[1][0][2][0][RTW89_FCC][0] = 127,
+	[1][0][2][0][RTW89_ETSI][0] = 127,
+	[1][0][2][0][RTW89_MKK][0] = 127,
+	[1][0][2][0][RTW89_IC][0] = 127,
+	[1][0][2][0][RTW89_KCC][0] = 127,
+	[1][0][2][0][RTW89_ACMA][0] = 127,
+	[1][0][2][0][RTW89_CN][0] = 127,
+	[1][0][2][0][RTW89_UK][0] = 127,
+	[1][0][2][0][RTW89_FCC][1] = 127,
+	[1][0][2][0][RTW89_ETSI][1] = 127,
+	[1][0][2][0][RTW89_MKK][1] = 127,
+	[1][0][2][0][RTW89_IC][1] = 127,
+	[1][0][2][0][RTW89_KCC][1] = 127,
+	[1][0][2][0][RTW89_ACMA][1] = 127,
+	[1][0][2][0][RTW89_CN][1] = 127,
+	[1][0][2][0][RTW89_UK][1] = 127,
+	[1][0][2][0][RTW89_FCC][2] = 70,
+	[1][0][2][0][RTW89_ETSI][2] = 58,
+	[1][0][2][0][RTW89_MKK][2] = 76,
+	[1][0][2][0][RTW89_IC][2] = 70,
+	[1][0][2][0][RTW89_KCC][2] = 76,
+	[1][0][2][0][RTW89_ACMA][2] = 58,
+	[1][0][2][0][RTW89_CN][2] = 60,
+	[1][0][2][0][RTW89_UK][2] = 58,
+	[1][0][2][0][RTW89_FCC][3] = 70,
+	[1][0][2][0][RTW89_ETSI][3] = 58,
+	[1][0][2][0][RTW89_MKK][3] = 76,
+	[1][0][2][0][RTW89_IC][3] = 70,
+	[1][0][2][0][RTW89_KCC][3] = 76,
+	[1][0][2][0][RTW89_ACMA][3] = 58,
+	[1][0][2][0][RTW89_CN][3] = 60,
+	[1][0][2][0][RTW89_UK][3] = 58,
+	[1][0][2][0][RTW89_FCC][4] = 74,
+	[1][0][2][0][RTW89_ETSI][4] = 58,
+	[1][0][2][0][RTW89_MKK][4] = 76,
+	[1][0][2][0][RTW89_IC][4] = 74,
+	[1][0][2][0][RTW89_KCC][4] = 76,
+	[1][0][2][0][RTW89_ACMA][4] = 58,
+	[1][0][2][0][RTW89_CN][4] = 60,
+	[1][0][2][0][RTW89_UK][4] = 58,
+	[1][0][2][0][RTW89_FCC][5] = 76,
+	[1][0][2][0][RTW89_ETSI][5] = 58,
+	[1][0][2][0][RTW89_MKK][5] = 76,
+	[1][0][2][0][RTW89_IC][5] = 76,
+	[1][0][2][0][RTW89_KCC][5] = 76,
+	[1][0][2][0][RTW89_ACMA][5] = 58,
+	[1][0][2][0][RTW89_CN][5] = 60,
+	[1][0][2][0][RTW89_UK][5] = 58,
+	[1][0][2][0][RTW89_FCC][6] = 76,
+	[1][0][2][0][RTW89_ETSI][6] = 58,
+	[1][0][2][0][RTW89_MKK][6] = 76,
+	[1][0][2][0][RTW89_IC][6] = 76,
+	[1][0][2][0][RTW89_KCC][6] = 76,
+	[1][0][2][0][RTW89_ACMA][6] = 58,
+	[1][0][2][0][RTW89_CN][6] = 60,
+	[1][0][2][0][RTW89_UK][6] = 58,
+	[1][0][2][0][RTW89_FCC][7] = 76,
+	[1][0][2][0][RTW89_ETSI][7] = 58,
+	[1][0][2][0][RTW89_MKK][7] = 76,
+	[1][0][2][0][RTW89_IC][7] = 76,
+	[1][0][2][0][RTW89_KCC][7] = 76,
+	[1][0][2][0][RTW89_ACMA][7] = 58,
+	[1][0][2][0][RTW89_CN][7] = 60,
+	[1][0][2][0][RTW89_UK][7] = 58,
+	[1][0][2][0][RTW89_FCC][8] = 78,
+	[1][0][2][0][RTW89_ETSI][8] = 58,
+	[1][0][2][0][RTW89_MKK][8] = 76,
+	[1][0][2][0][RTW89_IC][8] = 78,
+	[1][0][2][0][RTW89_KCC][8] = 76,
+	[1][0][2][0][RTW89_ACMA][8] = 58,
+	[1][0][2][0][RTW89_CN][8] = 60,
+	[1][0][2][0][RTW89_UK][8] = 58,
+	[1][0][2][0][RTW89_FCC][9] = 74,
+	[1][0][2][0][RTW89_ETSI][9] = 58,
+	[1][0][2][0][RTW89_MKK][9] = 76,
+	[1][0][2][0][RTW89_IC][9] = 74,
+	[1][0][2][0][RTW89_KCC][9] = 76,
+	[1][0][2][0][RTW89_ACMA][9] = 58,
+	[1][0][2][0][RTW89_CN][9] = 60,
+	[1][0][2][0][RTW89_UK][9] = 58,
+	[1][0][2][0][RTW89_FCC][10] = 68,
+	[1][0][2][0][RTW89_ETSI][10] = 58,
+	[1][0][2][0][RTW89_MKK][10] = 76,
+	[1][0][2][0][RTW89_IC][10] = 68,
+	[1][0][2][0][RTW89_KCC][10] = 76,
+	[1][0][2][0][RTW89_ACMA][10] = 58,
+	[1][0][2][0][RTW89_CN][10] = 60,
+	[1][0][2][0][RTW89_UK][10] = 58,
+	[1][0][2][0][RTW89_FCC][11] = 127,
+	[1][0][2][0][RTW89_ETSI][11] = 127,
+	[1][0][2][0][RTW89_MKK][11] = 127,
+	[1][0][2][0][RTW89_IC][11] = 127,
+	[1][0][2][0][RTW89_KCC][11] = 127,
+	[1][0][2][0][RTW89_ACMA][11] = 127,
+	[1][0][2][0][RTW89_CN][11] = 127,
+	[1][0][2][0][RTW89_UK][11] = 127,
+	[1][0][2][0][RTW89_FCC][12] = 127,
+	[1][0][2][0][RTW89_ETSI][12] = 127,
+	[1][0][2][0][RTW89_MKK][12] = 127,
+	[1][0][2][0][RTW89_IC][12] = 127,
+	[1][0][2][0][RTW89_KCC][12] = 127,
+	[1][0][2][0][RTW89_ACMA][12] = 127,
+	[1][0][2][0][RTW89_CN][12] = 127,
+	[1][0][2][0][RTW89_UK][12] = 127,
+	[1][0][2][0][RTW89_FCC][13] = 127,
+	[1][0][2][0][RTW89_ETSI][13] = 127,
+	[1][0][2][0][RTW89_MKK][13] = 127,
+	[1][0][2][0][RTW89_IC][13] = 127,
+	[1][0][2][0][RTW89_KCC][13] = 127,
+	[1][0][2][0][RTW89_ACMA][13] = 127,
+	[1][0][2][0][RTW89_CN][13] = 127,
+	[1][0][2][0][RTW89_UK][13] = 127,
+	[1][1][2][0][RTW89_FCC][0] = 127,
+	[1][1][2][0][RTW89_ETSI][0] = 127,
+	[1][1][2][0][RTW89_MKK][0] = 127,
+	[1][1][2][0][RTW89_IC][0] = 127,
+	[1][1][2][0][RTW89_KCC][0] = 127,
+	[1][1][2][0][RTW89_ACMA][0] = 127,
+	[1][1][2][0][RTW89_CN][0] = 127,
+	[1][1][2][0][RTW89_UK][0] = 127,
+	[1][1][2][0][RTW89_FCC][1] = 127,
+	[1][1][2][0][RTW89_ETSI][1] = 127,
+	[1][1][2][0][RTW89_MKK][1] = 127,
+	[1][1][2][0][RTW89_IC][1] = 127,
+	[1][1][2][0][RTW89_KCC][1] = 127,
+	[1][1][2][0][RTW89_ACMA][1] = 127,
+	[1][1][2][0][RTW89_CN][1] = 127,
+	[1][1][2][0][RTW89_UK][1] = 127,
+	[1][1][2][0][RTW89_FCC][2] = 127,
+	[1][1][2][0][RTW89_ETSI][2] = 127,
+	[1][1][2][0][RTW89_MKK][2] = 127,
+	[1][1][2][0][RTW89_IC][2] = 127,
+	[1][1][2][0][RTW89_KCC][2] = 127,
+	[1][1][2][0][RTW89_ACMA][2] = 127,
+	[1][1][2][0][RTW89_CN][2] = 127,
+	[1][1][2][0][RTW89_UK][2] = 127,
+	[1][1][2][0][RTW89_FCC][3] = 127,
+	[1][1][2][0][RTW89_ETSI][3] = 127,
+	[1][1][2][0][RTW89_MKK][3] = 127,
+	[1][1][2][0][RTW89_IC][3] = 127,
+	[1][1][2][0][RTW89_KCC][3] = 127,
+	[1][1][2][0][RTW89_ACMA][3] = 127,
+	[1][1][2][0][RTW89_CN][3] = 127,
+	[1][1][2][0][RTW89_UK][3] = 127,
+	[1][1][2][0][RTW89_FCC][4] = 127,
+	[1][1][2][0][RTW89_ETSI][4] = 127,
+	[1][1][2][0][RTW89_MKK][4] = 127,
+	[1][1][2][0][RTW89_IC][4] = 127,
+	[1][1][2][0][RTW89_KCC][4] = 127,
+	[1][1][2][0][RTW89_ACMA][4] = 127,
+	[1][1][2][0][RTW89_CN][4] = 127,
+	[1][1][2][0][RTW89_UK][4] = 127,
+	[1][1][2][0][RTW89_FCC][5] = 127,
+	[1][1][2][0][RTW89_ETSI][5] = 127,
+	[1][1][2][0][RTW89_MKK][5] = 127,
+	[1][1][2][0][RTW89_IC][5] = 127,
+	[1][1][2][0][RTW89_KCC][5] = 127,
+	[1][1][2][0][RTW89_ACMA][5] = 127,
+	[1][1][2][0][RTW89_CN][5] = 127,
+	[1][1][2][0][RTW89_UK][5] = 127,
+	[1][1][2][0][RTW89_FCC][6] = 127,
+	[1][1][2][0][RTW89_ETSI][6] = 127,
+	[1][1][2][0][RTW89_MKK][6] = 127,
+	[1][1][2][0][RTW89_IC][6] = 127,
+	[1][1][2][0][RTW89_KCC][6] = 127,
+	[1][1][2][0][RTW89_ACMA][6] = 127,
+	[1][1][2][0][RTW89_CN][6] = 127,
+	[1][1][2][0][RTW89_UK][6] = 127,
+	[1][1][2][0][RTW89_FCC][7] = 127,
+	[1][1][2][0][RTW89_ETSI][7] = 127,
+	[1][1][2][0][RTW89_MKK][7] = 127,
+	[1][1][2][0][RTW89_IC][7] = 127,
+	[1][1][2][0][RTW89_KCC][7] = 127,
+	[1][1][2][0][RTW89_ACMA][7] = 127,
+	[1][1][2][0][RTW89_CN][7] = 127,
+	[1][1][2][0][RTW89_UK][7] = 127,
+	[1][1][2][0][RTW89_FCC][8] = 127,
+	[1][1][2][0][RTW89_ETSI][8] = 127,
+	[1][1][2][0][RTW89_MKK][8] = 127,
+	[1][1][2][0][RTW89_IC][8] = 127,
+	[1][1][2][0][RTW89_KCC][8] = 127,
+	[1][1][2][0][RTW89_ACMA][8] = 127,
+	[1][1][2][0][RTW89_CN][8] = 127,
+	[1][1][2][0][RTW89_UK][8] = 127,
+	[1][1][2][0][RTW89_FCC][9] = 127,
+	[1][1][2][0][RTW89_ETSI][9] = 127,
+	[1][1][2][0][RTW89_MKK][9] = 127,
+	[1][1][2][0][RTW89_IC][9] = 127,
+	[1][1][2][0][RTW89_KCC][9] = 127,
+	[1][1][2][0][RTW89_ACMA][9] = 127,
+	[1][1][2][0][RTW89_CN][9] = 127,
+	[1][1][2][0][RTW89_UK][9] = 127,
+	[1][1][2][0][RTW89_FCC][10] = 127,
+	[1][1][2][0][RTW89_ETSI][10] = 127,
+	[1][1][2][0][RTW89_MKK][10] = 127,
+	[1][1][2][0][RTW89_IC][10] = 127,
+	[1][1][2][0][RTW89_KCC][10] = 127,
+	[1][1][2][0][RTW89_ACMA][10] = 127,
+	[1][1][2][0][RTW89_CN][10] = 127,
+	[1][1][2][0][RTW89_UK][10] = 127,
+	[1][1][2][0][RTW89_FCC][11] = 127,
+	[1][1][2][0][RTW89_ETSI][11] = 127,
+	[1][1][2][0][RTW89_MKK][11] = 127,
+	[1][1][2][0][RTW89_IC][11] = 127,
+	[1][1][2][0][RTW89_KCC][11] = 127,
+	[1][1][2][0][RTW89_ACMA][11] = 127,
+	[1][1][2][0][RTW89_CN][11] = 127,
+	[1][1][2][0][RTW89_UK][11] = 127,
+	[1][1][2][0][RTW89_FCC][12] = 127,
+	[1][1][2][0][RTW89_ETSI][12] = 127,
+	[1][1][2][0][RTW89_MKK][12] = 127,
+	[1][1][2][0][RTW89_IC][12] = 127,
+	[1][1][2][0][RTW89_KCC][12] = 127,
+	[1][1][2][0][RTW89_ACMA][12] = 127,
+	[1][1][2][0][RTW89_CN][12] = 127,
+	[1][1][2][0][RTW89_UK][12] = 127,
+	[1][1][2][0][RTW89_FCC][13] = 127,
+	[1][1][2][0][RTW89_ETSI][13] = 127,
+	[1][1][2][0][RTW89_MKK][13] = 127,
+	[1][1][2][0][RTW89_IC][13] = 127,
+	[1][1][2][0][RTW89_KCC][13] = 127,
+	[1][1][2][0][RTW89_ACMA][13] = 127,
+	[1][1][2][0][RTW89_CN][13] = 127,
+	[1][1][2][0][RTW89_UK][13] = 127,
+	[1][1][2][1][RTW89_FCC][0] = 127,
+	[1][1][2][1][RTW89_ETSI][0] = 127,
+	[1][1][2][1][RTW89_MKK][0] = 127,
+	[1][1][2][1][RTW89_IC][0] = 127,
+	[1][1][2][1][RTW89_KCC][0] = 127,
+	[1][1][2][1][RTW89_ACMA][0] = 127,
+	[1][1][2][1][RTW89_CN][0] = 127,
+	[1][1][2][1][RTW89_UK][0] = 127,
+	[1][1][2][1][RTW89_FCC][1] = 127,
+	[1][1][2][1][RTW89_ETSI][1] = 127,
+	[1][1][2][1][RTW89_MKK][1] = 127,
+	[1][1][2][1][RTW89_IC][1] = 127,
+	[1][1][2][1][RTW89_KCC][1] = 127,
+	[1][1][2][1][RTW89_ACMA][1] = 127,
+	[1][1][2][1][RTW89_CN][1] = 127,
+	[1][1][2][1][RTW89_UK][1] = 127,
+	[1][1][2][1][RTW89_FCC][2] = 127,
+	[1][1][2][1][RTW89_ETSI][2] = 127,
+	[1][1][2][1][RTW89_MKK][2] = 127,
+	[1][1][2][1][RTW89_IC][2] = 127,
+	[1][1][2][1][RTW89_KCC][2] = 127,
+	[1][1][2][1][RTW89_ACMA][2] = 127,
+	[1][1][2][1][RTW89_CN][2] = 127,
+	[1][1][2][1][RTW89_UK][2] = 127,
+	[1][1][2][1][RTW89_FCC][3] = 127,
+	[1][1][2][1][RTW89_ETSI][3] = 127,
+	[1][1][2][1][RTW89_MKK][3] = 127,
+	[1][1][2][1][RTW89_IC][3] = 127,
+	[1][1][2][1][RTW89_KCC][3] = 127,
+	[1][1][2][1][RTW89_ACMA][3] = 127,
+	[1][1][2][1][RTW89_CN][3] = 127,
+	[1][1][2][1][RTW89_UK][3] = 127,
+	[1][1][2][1][RTW89_FCC][4] = 127,
+	[1][1][2][1][RTW89_ETSI][4] = 127,
+	[1][1][2][1][RTW89_MKK][4] = 127,
+	[1][1][2][1][RTW89_IC][4] = 127,
+	[1][1][2][1][RTW89_KCC][4] = 127,
+	[1][1][2][1][RTW89_ACMA][4] = 127,
+	[1][1][2][1][RTW89_CN][4] = 127,
+	[1][1][2][1][RTW89_UK][4] = 127,
+	[1][1][2][1][RTW89_FCC][5] = 127,
+	[1][1][2][1][RTW89_ETSI][5] = 127,
+	[1][1][2][1][RTW89_MKK][5] = 127,
+	[1][1][2][1][RTW89_IC][5] = 127,
+	[1][1][2][1][RTW89_KCC][5] = 127,
+	[1][1][2][1][RTW89_ACMA][5] = 127,
+	[1][1][2][1][RTW89_CN][5] = 127,
+	[1][1][2][1][RTW89_UK][5] = 127,
+	[1][1][2][1][RTW89_FCC][6] = 127,
+	[1][1][2][1][RTW89_ETSI][6] = 127,
+	[1][1][2][1][RTW89_MKK][6] = 127,
+	[1][1][2][1][RTW89_IC][6] = 127,
+	[1][1][2][1][RTW89_KCC][6] = 127,
+	[1][1][2][1][RTW89_ACMA][6] = 127,
+	[1][1][2][1][RTW89_CN][6] = 127,
+	[1][1][2][1][RTW89_UK][6] = 127,
+	[1][1][2][1][RTW89_FCC][7] = 127,
+	[1][1][2][1][RTW89_ETSI][7] = 127,
+	[1][1][2][1][RTW89_MKK][7] = 127,
+	[1][1][2][1][RTW89_IC][7] = 127,
+	[1][1][2][1][RTW89_KCC][7] = 127,
+	[1][1][2][1][RTW89_ACMA][7] = 127,
+	[1][1][2][1][RTW89_CN][7] = 127,
+	[1][1][2][1][RTW89_UK][7] = 127,
+	[1][1][2][1][RTW89_FCC][8] = 127,
+	[1][1][2][1][RTW89_ETSI][8] = 127,
+	[1][1][2][1][RTW89_MKK][8] = 127,
+	[1][1][2][1][RTW89_IC][8] = 127,
+	[1][1][2][1][RTW89_KCC][8] = 127,
+	[1][1][2][1][RTW89_ACMA][8] = 127,
+	[1][1][2][1][RTW89_CN][8] = 127,
+	[1][1][2][1][RTW89_UK][8] = 127,
+	[1][1][2][1][RTW89_FCC][9] = 127,
+	[1][1][2][1][RTW89_ETSI][9] = 127,
+	[1][1][2][1][RTW89_MKK][9] = 127,
+	[1][1][2][1][RTW89_IC][9] = 127,
+	[1][1][2][1][RTW89_KCC][9] = 127,
+	[1][1][2][1][RTW89_ACMA][9] = 127,
+	[1][1][2][1][RTW89_CN][9] = 127,
+	[1][1][2][1][RTW89_UK][9] = 127,
+	[1][1][2][1][RTW89_FCC][10] = 127,
+	[1][1][2][1][RTW89_ETSI][10] = 127,
+	[1][1][2][1][RTW89_MKK][10] = 127,
+	[1][1][2][1][RTW89_IC][10] = 127,
+	[1][1][2][1][RTW89_KCC][10] = 127,
+	[1][1][2][1][RTW89_ACMA][10] = 127,
+	[1][1][2][1][RTW89_CN][10] = 127,
+	[1][1][2][1][RTW89_UK][10] = 127,
+	[1][1][2][1][RTW89_FCC][11] = 127,
+	[1][1][2][1][RTW89_ETSI][11] = 127,
+	[1][1][2][1][RTW89_MKK][11] = 127,
+	[1][1][2][1][RTW89_IC][11] = 127,
+	[1][1][2][1][RTW89_KCC][11] = 127,
+	[1][1][2][1][RTW89_ACMA][11] = 127,
+	[1][1][2][1][RTW89_CN][11] = 127,
+	[1][1][2][1][RTW89_UK][11] = 127,
+	[1][1][2][1][RTW89_FCC][12] = 127,
+	[1][1][2][1][RTW89_ETSI][12] = 127,
+	[1][1][2][1][RTW89_MKK][12] = 127,
+	[1][1][2][1][RTW89_IC][12] = 127,
+	[1][1][2][1][RTW89_KCC][12] = 127,
+	[1][1][2][1][RTW89_ACMA][12] = 127,
+	[1][1][2][1][RTW89_CN][12] = 127,
+	[1][1][2][1][RTW89_UK][12] = 127,
+	[1][1][2][1][RTW89_FCC][13] = 127,
+	[1][1][2][1][RTW89_ETSI][13] = 127,
+	[1][1][2][1][RTW89_MKK][13] = 127,
+	[1][1][2][1][RTW89_IC][13] = 127,
+	[1][1][2][1][RTW89_KCC][13] = 127,
+	[1][1][2][1][RTW89_ACMA][13] = 127,
+	[1][1][2][1][RTW89_CN][13] = 127,
+	[1][1][2][1][RTW89_UK][13] = 127,
+};
+
+static
+const s8 rtw89_8851b_txpwr_lmt_5g_type2[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
+				       [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
+				       [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
+	[0][0][1][0][RTW89_WW][0] = 58,
+	[0][0][1][0][RTW89_WW][2] = 58,
+	[0][0][1][0][RTW89_WW][4] = 58,
+	[0][0][1][0][RTW89_WW][6] = 50,
+	[0][0][1][0][RTW89_WW][8] = 58,
+	[0][0][1][0][RTW89_WW][10] = 58,
+	[0][0][1][0][RTW89_WW][12] = 58,
+	[0][0][1][0][RTW89_WW][14] = 58,
+	[0][0][1][0][RTW89_WW][15] = 58,
+	[0][0][1][0][RTW89_WW][17] = 60,
+	[0][0][1][0][RTW89_WW][19] = 60,
+	[0][0][1][0][RTW89_WW][21] = 60,
+	[0][0][1][0][RTW89_WW][23] = 60,
+	[0][0][1][0][RTW89_WW][25] = 60,
+	[0][0][1][0][RTW89_WW][27] = 60,
+	[0][0][1][0][RTW89_WW][29] = 60,
+	[0][0][1][0][RTW89_WW][31] = 60,
+	[0][0][1][0][RTW89_WW][33] = 60,
+	[0][0][1][0][RTW89_WW][35] = 60,
+	[0][0][1][0][RTW89_WW][37] = 74,
+	[0][0][1][0][RTW89_WW][38] = 30,
+	[0][0][1][0][RTW89_WW][40] = 30,
+	[0][0][1][0][RTW89_WW][42] = 30,
+	[0][0][1][0][RTW89_WW][44] = 30,
+	[0][0][1][0][RTW89_WW][46] = 30,
+	[0][0][1][0][RTW89_WW][48] = 72,
+	[0][0][1][0][RTW89_WW][50] = 72,
+	[0][0][1][0][RTW89_WW][52] = 72,
+	[0][1][1][0][RTW89_WW][0] = 0,
+	[0][1][1][0][RTW89_WW][2] = 0,
+	[0][1][1][0][RTW89_WW][4] = 0,
+	[0][1][1][0][RTW89_WW][6] = 0,
+	[0][1][1][0][RTW89_WW][8] = 0,
+	[0][1][1][0][RTW89_WW][10] = 0,
+	[0][1][1][0][RTW89_WW][12] = 0,
+	[0][1][1][0][RTW89_WW][14] = 0,
+	[0][1][1][0][RTW89_WW][15] = 0,
+	[0][1][1][0][RTW89_WW][17] = 0,
+	[0][1][1][0][RTW89_WW][19] = 0,
+	[0][1][1][0][RTW89_WW][21] = 0,
+	[0][1][1][0][RTW89_WW][23] = 0,
+	[0][1][1][0][RTW89_WW][25] = 0,
+	[0][1][1][0][RTW89_WW][27] = 0,
+	[0][1][1][0][RTW89_WW][29] = 0,
+	[0][1][1][0][RTW89_WW][31] = 0,
+	[0][1][1][0][RTW89_WW][33] = 0,
+	[0][1][1][0][RTW89_WW][35] = 0,
+	[0][1][1][0][RTW89_WW][37] = 0,
+	[0][1][1][0][RTW89_WW][38] = 0,
+	[0][1][1][0][RTW89_WW][40] = 0,
+	[0][1][1][0][RTW89_WW][42] = 0,
+	[0][1][1][0][RTW89_WW][44] = 0,
+	[0][1][1][0][RTW89_WW][46] = 0,
+	[0][1][1][0][RTW89_WW][48] = 0,
+	[0][1][1][0][RTW89_WW][50] = 0,
+	[0][1][1][0][RTW89_WW][52] = 0,
+	[0][0][2][0][RTW89_WW][0] = 62,
+	[0][0][2][0][RTW89_WW][2] = 62,
+	[0][0][2][0][RTW89_WW][4] = 62,
+	[0][0][2][0][RTW89_WW][6] = 54,
+	[0][0][2][0][RTW89_WW][8] = 62,
+	[0][0][2][0][RTW89_WW][10] = 62,
+	[0][0][2][0][RTW89_WW][12] = 62,
+	[0][0][2][0][RTW89_WW][14] = 62,
+	[0][0][2][0][RTW89_WW][15] = 60,
+	[0][0][2][0][RTW89_WW][17] = 62,
+	[0][0][2][0][RTW89_WW][19] = 62,
+	[0][0][2][0][RTW89_WW][21] = 62,
+	[0][0][2][0][RTW89_WW][23] = 62,
+	[0][0][2][0][RTW89_WW][25] = 62,
+	[0][0][2][0][RTW89_WW][27] = 62,
+	[0][0][2][0][RTW89_WW][29] = 62,
+	[0][0][2][0][RTW89_WW][31] = 62,
+	[0][0][2][0][RTW89_WW][33] = 62,
+	[0][0][2][0][RTW89_WW][35] = 62,
+	[0][0][2][0][RTW89_WW][37] = 74,
+	[0][0][2][0][RTW89_WW][38] = 30,
+	[0][0][2][0][RTW89_WW][40] = 30,
+	[0][0][2][0][RTW89_WW][42] = 30,
+	[0][0][2][0][RTW89_WW][44] = 30,
+	[0][0][2][0][RTW89_WW][46] = 30,
+	[0][0][2][0][RTW89_WW][48] = 74,
+	[0][0][2][0][RTW89_WW][50] = 74,
+	[0][0][2][0][RTW89_WW][52] = 74,
+	[0][1][2][0][RTW89_WW][0] = 0,
+	[0][1][2][0][RTW89_WW][2] = 0,
+	[0][1][2][0][RTW89_WW][4] = 0,
+	[0][1][2][0][RTW89_WW][6] = 0,
+	[0][1][2][0][RTW89_WW][8] = 0,
+	[0][1][2][0][RTW89_WW][10] = 0,
+	[0][1][2][0][RTW89_WW][12] = 0,
+	[0][1][2][0][RTW89_WW][14] = 0,
+	[0][1][2][0][RTW89_WW][15] = 0,
+	[0][1][2][0][RTW89_WW][17] = 0,
+	[0][1][2][0][RTW89_WW][19] = 0,
+	[0][1][2][0][RTW89_WW][21] = 0,
+	[0][1][2][0][RTW89_WW][23] = 0,
+	[0][1][2][0][RTW89_WW][25] = 0,
+	[0][1][2][0][RTW89_WW][27] = 0,
+	[0][1][2][0][RTW89_WW][29] = 0,
+	[0][1][2][0][RTW89_WW][31] = 0,
+	[0][1][2][0][RTW89_WW][33] = 0,
+	[0][1][2][0][RTW89_WW][35] = 0,
+	[0][1][2][0][RTW89_WW][37] = 0,
+	[0][1][2][0][RTW89_WW][38] = 0,
+	[0][1][2][0][RTW89_WW][40] = 0,
+	[0][1][2][0][RTW89_WW][42] = 0,
+	[0][1][2][0][RTW89_WW][44] = 0,
+	[0][1][2][0][RTW89_WW][46] = 0,
+	[0][1][2][0][RTW89_WW][48] = 0,
+	[0][1][2][0][RTW89_WW][50] = 0,
+	[0][1][2][0][RTW89_WW][52] = 0,
+	[0][1][2][1][RTW89_WW][0] = 0,
+	[0][1][2][1][RTW89_WW][2] = 0,
+	[0][1][2][1][RTW89_WW][4] = 0,
+	[0][1][2][1][RTW89_WW][6] = 0,
+	[0][1][2][1][RTW89_WW][8] = 0,
+	[0][1][2][1][RTW89_WW][10] = 0,
+	[0][1][2][1][RTW89_WW][12] = 0,
+	[0][1][2][1][RTW89_WW][14] = 0,
+	[0][1][2][1][RTW89_WW][15] = 0,
+	[0][1][2][1][RTW89_WW][17] = 0,
+	[0][1][2][1][RTW89_WW][19] = 0,
+	[0][1][2][1][RTW89_WW][21] = 0,
+	[0][1][2][1][RTW89_WW][23] = 0,
+	[0][1][2][1][RTW89_WW][25] = 0,
+	[0][1][2][1][RTW89_WW][27] = 0,
+	[0][1][2][1][RTW89_WW][29] = 0,
+	[0][1][2][1][RTW89_WW][31] = 0,
+	[0][1][2][1][RTW89_WW][33] = 0,
+	[0][1][2][1][RTW89_WW][35] = 0,
+	[0][1][2][1][RTW89_WW][37] = 0,
+	[0][1][2][1][RTW89_WW][38] = 0,
+	[0][1][2][1][RTW89_WW][40] = 0,
+	[0][1][2][1][RTW89_WW][42] = 0,
+	[0][1][2][1][RTW89_WW][44] = 0,
+	[0][1][2][1][RTW89_WW][46] = 0,
+	[0][1][2][1][RTW89_WW][48] = 0,
+	[0][1][2][1][RTW89_WW][50] = 0,
+	[0][1][2][1][RTW89_WW][52] = 0,
+	[1][0][2][0][RTW89_WW][1] = 64,
+	[1][0][2][0][RTW89_WW][5] = 62,
+	[1][0][2][0][RTW89_WW][9] = 64,
+	[1][0][2][0][RTW89_WW][13] = 64,
+	[1][0][2][0][RTW89_WW][16] = 66,
+	[1][0][2][0][RTW89_WW][20] = 66,
+	[1][0][2][0][RTW89_WW][24] = 66,
+	[1][0][2][0][RTW89_WW][28] = 66,
+	[1][0][2][0][RTW89_WW][32] = 66,
+	[1][0][2][0][RTW89_WW][36] = 76,
+	[1][0][2][0][RTW89_WW][39] = 30,
+	[1][0][2][0][RTW89_WW][43] = 30,
+	[1][0][2][0][RTW89_WW][47] = 80,
+	[1][0][2][0][RTW89_WW][51] = 80,
+	[1][1][2][0][RTW89_WW][1] = 0,
+	[1][1][2][0][RTW89_WW][5] = 0,
+	[1][1][2][0][RTW89_WW][9] = 0,
+	[1][1][2][0][RTW89_WW][13] = 0,
+	[1][1][2][0][RTW89_WW][16] = 0,
+	[1][1][2][0][RTW89_WW][20] = 0,
+	[1][1][2][0][RTW89_WW][24] = 0,
+	[1][1][2][0][RTW89_WW][28] = 0,
+	[1][1][2][0][RTW89_WW][32] = 0,
+	[1][1][2][0][RTW89_WW][36] = 0,
+	[1][1][2][0][RTW89_WW][39] = 0,
+	[1][1][2][0][RTW89_WW][43] = 0,
+	[1][1][2][0][RTW89_WW][47] = 0,
+	[1][1][2][0][RTW89_WW][51] = 0,
+	[1][1][2][1][RTW89_WW][1] = 0,
+	[1][1][2][1][RTW89_WW][5] = 0,
+	[1][1][2][1][RTW89_WW][9] = 0,
+	[1][1][2][1][RTW89_WW][13] = 0,
+	[1][1][2][1][RTW89_WW][16] = 0,
+	[1][1][2][1][RTW89_WW][20] = 0,
+	[1][1][2][1][RTW89_WW][24] = 0,
+	[1][1][2][1][RTW89_WW][28] = 0,
+	[1][1][2][1][RTW89_WW][32] = 0,
+	[1][1][2][1][RTW89_WW][36] = 0,
+	[1][1][2][1][RTW89_WW][39] = 0,
+	[1][1][2][1][RTW89_WW][43] = 0,
+	[1][1][2][1][RTW89_WW][47] = 0,
+	[1][1][2][1][RTW89_WW][51] = 0,
+	[2][0][2][0][RTW89_WW][3] = 62,
+	[2][0][2][0][RTW89_WW][11] = 62,
+	[2][0][2][0][RTW89_WW][18] = 64,
+	[2][0][2][0][RTW89_WW][26] = 64,
+	[2][0][2][0][RTW89_WW][34] = 68,
+	[2][0][2][0][RTW89_WW][41] = 30,
+	[2][0][2][0][RTW89_WW][49] = 72,
+	[2][1][2][0][RTW89_WW][3] = 0,
+	[2][1][2][0][RTW89_WW][11] = 0,
+	[2][1][2][0][RTW89_WW][18] = 0,
+	[2][1][2][0][RTW89_WW][26] = 0,
+	[2][1][2][0][RTW89_WW][34] = 0,
+	[2][1][2][0][RTW89_WW][41] = 0,
+	[2][1][2][0][RTW89_WW][49] = 0,
+	[2][1][2][1][RTW89_WW][3] = 0,
+	[2][1][2][1][RTW89_WW][11] = 0,
+	[2][1][2][1][RTW89_WW][18] = 0,
+	[2][1][2][1][RTW89_WW][26] = 0,
+	[2][1][2][1][RTW89_WW][34] = 0,
+	[2][1][2][1][RTW89_WW][41] = 0,
+	[2][1][2][1][RTW89_WW][49] = 0,
+	[3][0][2][0][RTW89_WW][7] = 58,
+	[3][0][2][0][RTW89_WW][22] = 58,
+	[3][0][2][0][RTW89_WW][45] = 0,
+	[3][1][2][0][RTW89_WW][7] = 0,
+	[3][1][2][0][RTW89_WW][22] = 0,
+	[3][1][2][0][RTW89_WW][45] = 0,
+	[3][1][2][1][RTW89_WW][7] = 0,
+	[3][1][2][1][RTW89_WW][22] = 0,
+	[3][1][2][1][RTW89_WW][45] = 0,
+	[0][0][1][0][RTW89_FCC][0] = 78,
+	[0][0][1][0][RTW89_ETSI][0] = 58,
+	[0][0][1][0][RTW89_MKK][0] = 60,
+	[0][0][1][0][RTW89_IC][0] = 62,
+	[0][0][1][0][RTW89_KCC][0] = 74,
+	[0][0][1][0][RTW89_ACMA][0] = 58,
+	[0][0][1][0][RTW89_CN][0] = 60,
+	[0][0][1][0][RTW89_UK][0] = 58,
+	[0][0][1][0][RTW89_FCC][2] = 78,
+	[0][0][1][0][RTW89_ETSI][2] = 58,
+	[0][0][1][0][RTW89_MKK][2] = 60,
+	[0][0][1][0][RTW89_IC][2] = 62,
+	[0][0][1][0][RTW89_KCC][2] = 74,
+	[0][0][1][0][RTW89_ACMA][2] = 58,
+	[0][0][1][0][RTW89_CN][2] = 60,
+	[0][0][1][0][RTW89_UK][2] = 58,
+	[0][0][1][0][RTW89_FCC][4] = 78,
+	[0][0][1][0][RTW89_ETSI][4] = 58,
+	[0][0][1][0][RTW89_MKK][4] = 60,
+	[0][0][1][0][RTW89_IC][4] = 62,
+	[0][0][1][0][RTW89_KCC][4] = 74,
+	[0][0][1][0][RTW89_ACMA][4] = 58,
+	[0][0][1][0][RTW89_CN][4] = 60,
+	[0][0][1][0][RTW89_UK][4] = 58,
+	[0][0][1][0][RTW89_FCC][6] = 78,
+	[0][0][1][0][RTW89_ETSI][6] = 58,
+	[0][0][1][0][RTW89_MKK][6] = 60,
+	[0][0][1][0][RTW89_IC][6] = 62,
+	[0][0][1][0][RTW89_KCC][6] = 50,
+	[0][0][1][0][RTW89_ACMA][6] = 58,
+	[0][0][1][0][RTW89_CN][6] = 60,
+	[0][0][1][0][RTW89_UK][6] = 58,
+	[0][0][1][0][RTW89_FCC][8] = 78,
+	[0][0][1][0][RTW89_ETSI][8] = 58,
+	[0][0][1][0][RTW89_MKK][8] = 60,
+	[0][0][1][0][RTW89_IC][8] = 62,
+	[0][0][1][0][RTW89_KCC][8] = 74,
+	[0][0][1][0][RTW89_ACMA][8] = 58,
+	[0][0][1][0][RTW89_CN][8] = 60,
+	[0][0][1][0][RTW89_UK][8] = 58,
+	[0][0][1][0][RTW89_FCC][10] = 78,
+	[0][0][1][0][RTW89_ETSI][10] = 58,
+	[0][0][1][0][RTW89_MKK][10] = 60,
+	[0][0][1][0][RTW89_IC][10] = 64,
+	[0][0][1][0][RTW89_KCC][10] = 74,
+	[0][0][1][0][RTW89_ACMA][10] = 58,
+	[0][0][1][0][RTW89_CN][10] = 60,
+	[0][0][1][0][RTW89_UK][10] = 58,
+	[0][0][1][0][RTW89_FCC][12] = 78,
+	[0][0][1][0][RTW89_ETSI][12] = 58,
+	[0][0][1][0][RTW89_MKK][12] = 60,
+	[0][0][1][0][RTW89_IC][12] = 64,
+	[0][0][1][0][RTW89_KCC][12] = 74,
+	[0][0][1][0][RTW89_ACMA][12] = 58,
+	[0][0][1][0][RTW89_CN][12] = 60,
+	[0][0][1][0][RTW89_UK][12] = 58,
+	[0][0][1][0][RTW89_FCC][14] = 76,
+	[0][0][1][0][RTW89_ETSI][14] = 58,
+	[0][0][1][0][RTW89_MKK][14] = 60,
+	[0][0][1][0][RTW89_IC][14] = 62,
+	[0][0][1][0][RTW89_KCC][14] = 74,
+	[0][0][1][0][RTW89_ACMA][14] = 58,
+	[0][0][1][0][RTW89_CN][14] = 60,
+	[0][0][1][0][RTW89_UK][14] = 58,
+	[0][0][1][0][RTW89_FCC][15] = 76,
+	[0][0][1][0][RTW89_ETSI][15] = 58,
+	[0][0][1][0][RTW89_MKK][15] = 74,
+	[0][0][1][0][RTW89_IC][15] = 76,
+	[0][0][1][0][RTW89_KCC][15] = 74,
+	[0][0][1][0][RTW89_ACMA][15] = 58,
+	[0][0][1][0][RTW89_CN][15] = 127,
+	[0][0][1][0][RTW89_UK][15] = 58,
+	[0][0][1][0][RTW89_FCC][17] = 78,
+	[0][0][1][0][RTW89_ETSI][17] = 60,
+	[0][0][1][0][RTW89_MKK][17] = 74,
+	[0][0][1][0][RTW89_IC][17] = 78,
+	[0][0][1][0][RTW89_KCC][17] = 74,
+	[0][0][1][0][RTW89_ACMA][17] = 60,
+	[0][0][1][0][RTW89_CN][17] = 127,
+	[0][0][1][0][RTW89_UK][17] = 60,
+	[0][0][1][0][RTW89_FCC][19] = 78,
+	[0][0][1][0][RTW89_ETSI][19] = 60,
+	[0][0][1][0][RTW89_MKK][19] = 74,
+	[0][0][1][0][RTW89_IC][19] = 78,
+	[0][0][1][0][RTW89_KCC][19] = 74,
+	[0][0][1][0][RTW89_ACMA][19] = 60,
+	[0][0][1][0][RTW89_CN][19] = 127,
+	[0][0][1][0][RTW89_UK][19] = 60,
+	[0][0][1][0][RTW89_FCC][21] = 78,
+	[0][0][1][0][RTW89_ETSI][21] = 60,
+	[0][0][1][0][RTW89_MKK][21] = 74,
+	[0][0][1][0][RTW89_IC][21] = 78,
+	[0][0][1][0][RTW89_KCC][21] = 74,
+	[0][0][1][0][RTW89_ACMA][21] = 60,
+	[0][0][1][0][RTW89_CN][21] = 127,
+	[0][0][1][0][RTW89_UK][21] = 60,
+	[0][0][1][0][RTW89_FCC][23] = 78,
+	[0][0][1][0][RTW89_ETSI][23] = 60,
+	[0][0][1][0][RTW89_MKK][23] = 74,
+	[0][0][1][0][RTW89_IC][23] = 78,
+	[0][0][1][0][RTW89_KCC][23] = 74,
+	[0][0][1][0][RTW89_ACMA][23] = 60,
+	[0][0][1][0][RTW89_CN][23] = 127,
+	[0][0][1][0][RTW89_UK][23] = 60,
+	[0][0][1][0][RTW89_FCC][25] = 78,
+	[0][0][1][0][RTW89_ETSI][25] = 60,
+	[0][0][1][0][RTW89_MKK][25] = 74,
+	[0][0][1][0][RTW89_IC][25] = 127,
+	[0][0][1][0][RTW89_KCC][25] = 74,
+	[0][0][1][0][RTW89_ACMA][25] = 127,
+	[0][0][1][0][RTW89_CN][25] = 127,
+	[0][0][1][0][RTW89_UK][25] = 60,
+	[0][0][1][0][RTW89_FCC][27] = 78,
+	[0][0][1][0][RTW89_ETSI][27] = 60,
+	[0][0][1][0][RTW89_MKK][27] = 74,
+	[0][0][1][0][RTW89_IC][27] = 127,
+	[0][0][1][0][RTW89_KCC][27] = 74,
+	[0][0][1][0][RTW89_ACMA][27] = 127,
+	[0][0][1][0][RTW89_CN][27] = 127,
+	[0][0][1][0][RTW89_UK][27] = 60,
+	[0][0][1][0][RTW89_FCC][29] = 78,
+	[0][0][1][0][RTW89_ETSI][29] = 60,
+	[0][0][1][0][RTW89_MKK][29] = 74,
+	[0][0][1][0][RTW89_IC][29] = 127,
+	[0][0][1][0][RTW89_KCC][29] = 74,
+	[0][0][1][0][RTW89_ACMA][29] = 127,
+	[0][0][1][0][RTW89_CN][29] = 127,
+	[0][0][1][0][RTW89_UK][29] = 60,
+	[0][0][1][0][RTW89_FCC][31] = 78,
+	[0][0][1][0][RTW89_ETSI][31] = 60,
+	[0][0][1][0][RTW89_MKK][31] = 74,
+	[0][0][1][0][RTW89_IC][31] = 78,
+	[0][0][1][0][RTW89_KCC][31] = 74,
+	[0][0][1][0][RTW89_ACMA][31] = 60,
+	[0][0][1][0][RTW89_CN][31] = 127,
+	[0][0][1][0][RTW89_UK][31] = 60,
+	[0][0][1][0][RTW89_FCC][33] = 78,
+	[0][0][1][0][RTW89_ETSI][33] = 60,
+	[0][0][1][0][RTW89_MKK][33] = 74,
+	[0][0][1][0][RTW89_IC][33] = 78,
+	[0][0][1][0][RTW89_KCC][33] = 74,
+	[0][0][1][0][RTW89_ACMA][33] = 60,
+	[0][0][1][0][RTW89_CN][33] = 127,
+	[0][0][1][0][RTW89_UK][33] = 60,
+	[0][0][1][0][RTW89_FCC][35] = 70,
+	[0][0][1][0][RTW89_ETSI][35] = 60,
+	[0][0][1][0][RTW89_MKK][35] = 74,
+	[0][0][1][0][RTW89_IC][35] = 70,
+	[0][0][1][0][RTW89_KCC][35] = 74,
+	[0][0][1][0][RTW89_ACMA][35] = 60,
+	[0][0][1][0][RTW89_CN][35] = 127,
+	[0][0][1][0][RTW89_UK][35] = 60,
+	[0][0][1][0][RTW89_FCC][37] = 78,
+	[0][0][1][0][RTW89_ETSI][37] = 127,
+	[0][0][1][0][RTW89_MKK][37] = 74,
+	[0][0][1][0][RTW89_IC][37] = 78,
+	[0][0][1][0][RTW89_KCC][37] = 74,
+	[0][0][1][0][RTW89_ACMA][37] = 74,
+	[0][0][1][0][RTW89_CN][37] = 127,
+	[0][0][1][0][RTW89_UK][37] = 74,
+	[0][0][1][0][RTW89_FCC][38] = 78,
+	[0][0][1][0][RTW89_ETSI][38] = 30,
+	[0][0][1][0][RTW89_MKK][38] = 127,
+	[0][0][1][0][RTW89_IC][38] = 78,
+	[0][0][1][0][RTW89_KCC][38] = 70,
+	[0][0][1][0][RTW89_ACMA][38] = 74,
+	[0][0][1][0][RTW89_CN][38] = 74,
+	[0][0][1][0][RTW89_UK][38] = 58,
+	[0][0][1][0][RTW89_FCC][40] = 78,
+	[0][0][1][0][RTW89_ETSI][40] = 30,
+	[0][0][1][0][RTW89_MKK][40] = 127,
+	[0][0][1][0][RTW89_IC][40] = 78,
+	[0][0][1][0][RTW89_KCC][40] = 74,
+	[0][0][1][0][RTW89_ACMA][40] = 74,
+	[0][0][1][0][RTW89_CN][40] = 74,
+	[0][0][1][0][RTW89_UK][40] = 58,
+	[0][0][1][0][RTW89_FCC][42] = 78,
+	[0][0][1][0][RTW89_ETSI][42] = 30,
+	[0][0][1][0][RTW89_MKK][42] = 127,
+	[0][0][1][0][RTW89_IC][42] = 78,
+	[0][0][1][0][RTW89_KCC][42] = 74,
+	[0][0][1][0][RTW89_ACMA][42] = 74,
+	[0][0][1][0][RTW89_CN][42] = 74,
+	[0][0][1][0][RTW89_UK][42] = 58,
+	[0][0][1][0][RTW89_FCC][44] = 78,
+	[0][0][1][0][RTW89_ETSI][44] = 30,
+	[0][0][1][0][RTW89_MKK][44] = 127,
+	[0][0][1][0][RTW89_IC][44] = 78,
+	[0][0][1][0][RTW89_KCC][44] = 74,
+	[0][0][1][0][RTW89_ACMA][44] = 74,
+	[0][0][1][0][RTW89_CN][44] = 74,
+	[0][0][1][0][RTW89_UK][44] = 58,
+	[0][0][1][0][RTW89_FCC][46] = 78,
+	[0][0][1][0][RTW89_ETSI][46] = 30,
+	[0][0][1][0][RTW89_MKK][46] = 127,
+	[0][0][1][0][RTW89_IC][46] = 78,
+	[0][0][1][0][RTW89_KCC][46] = 74,
+	[0][0][1][0][RTW89_ACMA][46] = 74,
+	[0][0][1][0][RTW89_CN][46] = 74,
+	[0][0][1][0][RTW89_UK][46] = 58,
+	[0][0][1][0][RTW89_FCC][48] = 72,
+	[0][0][1][0][RTW89_ETSI][48] = 127,
+	[0][0][1][0][RTW89_MKK][48] = 127,
+	[0][0][1][0][RTW89_IC][48] = 127,
+	[0][0][1][0][RTW89_KCC][48] = 127,
+	[0][0][1][0][RTW89_ACMA][48] = 127,
+	[0][0][1][0][RTW89_CN][48] = 127,
+	[0][0][1][0][RTW89_UK][48] = 127,
+	[0][0][1][0][RTW89_FCC][50] = 72,
+	[0][0][1][0][RTW89_ETSI][50] = 127,
+	[0][0][1][0][RTW89_MKK][50] = 127,
+	[0][0][1][0][RTW89_IC][50] = 127,
+	[0][0][1][0][RTW89_KCC][50] = 127,
+	[0][0][1][0][RTW89_ACMA][50] = 127,
+	[0][0][1][0][RTW89_CN][50] = 127,
+	[0][0][1][0][RTW89_UK][50] = 127,
+	[0][0][1][0][RTW89_FCC][52] = 72,
+	[0][0][1][0][RTW89_ETSI][52] = 127,
+	[0][0][1][0][RTW89_MKK][52] = 127,
+	[0][0][1][0][RTW89_IC][52] = 127,
+	[0][0][1][0][RTW89_KCC][52] = 127,
+	[0][0][1][0][RTW89_ACMA][52] = 127,
+	[0][0][1][0][RTW89_CN][52] = 127,
+	[0][0][1][0][RTW89_UK][52] = 127,
+	[0][1][1][0][RTW89_FCC][0] = 127,
+	[0][1][1][0][RTW89_ETSI][0] = 127,
+	[0][1][1][0][RTW89_MKK][0] = 127,
+	[0][1][1][0][RTW89_IC][0] = 127,
+	[0][1][1][0][RTW89_KCC][0] = 127,
+	[0][1][1][0][RTW89_ACMA][0] = 127,
+	[0][1][1][0][RTW89_CN][0] = 127,
+	[0][1][1][0][RTW89_UK][0] = 127,
+	[0][1][1][0][RTW89_FCC][2] = 127,
+	[0][1][1][0][RTW89_ETSI][2] = 127,
+	[0][1][1][0][RTW89_MKK][2] = 127,
+	[0][1][1][0][RTW89_IC][2] = 127,
+	[0][1][1][0][RTW89_KCC][2] = 127,
+	[0][1][1][0][RTW89_ACMA][2] = 127,
+	[0][1][1][0][RTW89_CN][2] = 127,
+	[0][1][1][0][RTW89_UK][2] = 127,
+	[0][1][1][0][RTW89_FCC][4] = 127,
+	[0][1][1][0][RTW89_ETSI][4] = 127,
+	[0][1][1][0][RTW89_MKK][4] = 127,
+	[0][1][1][0][RTW89_IC][4] = 127,
+	[0][1][1][0][RTW89_KCC][4] = 127,
+	[0][1][1][0][RTW89_ACMA][4] = 127,
+	[0][1][1][0][RTW89_CN][4] = 127,
+	[0][1][1][0][RTW89_UK][4] = 127,
+	[0][1][1][0][RTW89_FCC][6] = 127,
+	[0][1][1][0][RTW89_ETSI][6] = 127,
+	[0][1][1][0][RTW89_MKK][6] = 127,
+	[0][1][1][0][RTW89_IC][6] = 127,
+	[0][1][1][0][RTW89_KCC][6] = 127,
+	[0][1][1][0][RTW89_ACMA][6] = 127,
+	[0][1][1][0][RTW89_CN][6] = 127,
+	[0][1][1][0][RTW89_UK][6] = 127,
+	[0][1][1][0][RTW89_FCC][8] = 127,
+	[0][1][1][0][RTW89_ETSI][8] = 127,
+	[0][1][1][0][RTW89_MKK][8] = 127,
+	[0][1][1][0][RTW89_IC][8] = 127,
+	[0][1][1][0][RTW89_KCC][8] = 127,
+	[0][1][1][0][RTW89_ACMA][8] = 127,
+	[0][1][1][0][RTW89_CN][8] = 127,
+	[0][1][1][0][RTW89_UK][8] = 127,
+	[0][1][1][0][RTW89_FCC][10] = 127,
+	[0][1][1][0][RTW89_ETSI][10] = 127,
+	[0][1][1][0][RTW89_MKK][10] = 127,
+	[0][1][1][0][RTW89_IC][10] = 127,
+	[0][1][1][0][RTW89_KCC][10] = 127,
+	[0][1][1][0][RTW89_ACMA][10] = 127,
+	[0][1][1][0][RTW89_CN][10] = 127,
+	[0][1][1][0][RTW89_UK][10] = 127,
+	[0][1][1][0][RTW89_FCC][12] = 127,
+	[0][1][1][0][RTW89_ETSI][12] = 127,
+	[0][1][1][0][RTW89_MKK][12] = 127,
+	[0][1][1][0][RTW89_IC][12] = 127,
+	[0][1][1][0][RTW89_KCC][12] = 127,
+	[0][1][1][0][RTW89_ACMA][12] = 127,
+	[0][1][1][0][RTW89_CN][12] = 127,
+	[0][1][1][0][RTW89_UK][12] = 127,
+	[0][1][1][0][RTW89_FCC][14] = 127,
+	[0][1][1][0][RTW89_ETSI][14] = 127,
+	[0][1][1][0][RTW89_MKK][14] = 127,
+	[0][1][1][0][RTW89_IC][14] = 127,
+	[0][1][1][0][RTW89_KCC][14] = 127,
+	[0][1][1][0][RTW89_ACMA][14] = 127,
+	[0][1][1][0][RTW89_CN][14] = 127,
+	[0][1][1][0][RTW89_UK][14] = 127,
+	[0][1][1][0][RTW89_FCC][15] = 127,
+	[0][1][1][0][RTW89_ETSI][15] = 127,
+	[0][1][1][0][RTW89_MKK][15] = 127,
+	[0][1][1][0][RTW89_IC][15] = 127,
+	[0][1][1][0][RTW89_KCC][15] = 127,
+	[0][1][1][0][RTW89_ACMA][15] = 127,
+	[0][1][1][0][RTW89_CN][15] = 127,
+	[0][1][1][0][RTW89_UK][15] = 127,
+	[0][1][1][0][RTW89_FCC][17] = 127,
+	[0][1][1][0][RTW89_ETSI][17] = 127,
+	[0][1][1][0][RTW89_MKK][17] = 127,
+	[0][1][1][0][RTW89_IC][17] = 127,
+	[0][1][1][0][RTW89_KCC][17] = 127,
+	[0][1][1][0][RTW89_ACMA][17] = 127,
+	[0][1][1][0][RTW89_CN][17] = 127,
+	[0][1][1][0][RTW89_UK][17] = 127,
+	[0][1][1][0][RTW89_FCC][19] = 127,
+	[0][1][1][0][RTW89_ETSI][19] = 127,
+	[0][1][1][0][RTW89_MKK][19] = 127,
+	[0][1][1][0][RTW89_IC][19] = 127,
+	[0][1][1][0][RTW89_KCC][19] = 127,
+	[0][1][1][0][RTW89_ACMA][19] = 127,
+	[0][1][1][0][RTW89_CN][19] = 127,
+	[0][1][1][0][RTW89_UK][19] = 127,
+	[0][1][1][0][RTW89_FCC][21] = 127,
+	[0][1][1][0][RTW89_ETSI][21] = 127,
+	[0][1][1][0][RTW89_MKK][21] = 127,
+	[0][1][1][0][RTW89_IC][21] = 127,
+	[0][1][1][0][RTW89_KCC][21] = 127,
+	[0][1][1][0][RTW89_ACMA][21] = 127,
+	[0][1][1][0][RTW89_CN][21] = 127,
+	[0][1][1][0][RTW89_UK][21] = 127,
+	[0][1][1][0][RTW89_FCC][23] = 127,
+	[0][1][1][0][RTW89_ETSI][23] = 127,
+	[0][1][1][0][RTW89_MKK][23] = 127,
+	[0][1][1][0][RTW89_IC][23] = 127,
+	[0][1][1][0][RTW89_KCC][23] = 127,
+	[0][1][1][0][RTW89_ACMA][23] = 127,
+	[0][1][1][0][RTW89_CN][23] = 127,
+	[0][1][1][0][RTW89_UK][23] = 127,
+	[0][1][1][0][RTW89_FCC][25] = 127,
+	[0][1][1][0][RTW89_ETSI][25] = 127,
+	[0][1][1][0][RTW89_MKK][25] = 127,
+	[0][1][1][0][RTW89_IC][25] = 127,
+	[0][1][1][0][RTW89_KCC][25] = 127,
+	[0][1][1][0][RTW89_ACMA][25] = 127,
+	[0][1][1][0][RTW89_CN][25] = 127,
+	[0][1][1][0][RTW89_UK][25] = 127,
+	[0][1][1][0][RTW89_FCC][27] = 127,
+	[0][1][1][0][RTW89_ETSI][27] = 127,
+	[0][1][1][0][RTW89_MKK][27] = 127,
+	[0][1][1][0][RTW89_IC][27] = 127,
+	[0][1][1][0][RTW89_KCC][27] = 127,
+	[0][1][1][0][RTW89_ACMA][27] = 127,
+	[0][1][1][0][RTW89_CN][27] = 127,
+	[0][1][1][0][RTW89_UK][27] = 127,
+	[0][1][1][0][RTW89_FCC][29] = 127,
+	[0][1][1][0][RTW89_ETSI][29] = 127,
+	[0][1][1][0][RTW89_MKK][29] = 127,
+	[0][1][1][0][RTW89_IC][29] = 127,
+	[0][1][1][0][RTW89_KCC][29] = 127,
+	[0][1][1][0][RTW89_ACMA][29] = 127,
+	[0][1][1][0][RTW89_CN][29] = 127,
+	[0][1][1][0][RTW89_UK][29] = 127,
+	[0][1][1][0][RTW89_FCC][31] = 127,
+	[0][1][1][0][RTW89_ETSI][31] = 127,
+	[0][1][1][0][RTW89_MKK][31] = 127,
+	[0][1][1][0][RTW89_IC][31] = 127,
+	[0][1][1][0][RTW89_KCC][31] = 127,
+	[0][1][1][0][RTW89_ACMA][31] = 127,
+	[0][1][1][0][RTW89_CN][31] = 127,
+	[0][1][1][0][RTW89_UK][31] = 127,
+	[0][1][1][0][RTW89_FCC][33] = 127,
+	[0][1][1][0][RTW89_ETSI][33] = 127,
+	[0][1][1][0][RTW89_MKK][33] = 127,
+	[0][1][1][0][RTW89_IC][33] = 127,
+	[0][1][1][0][RTW89_KCC][33] = 127,
+	[0][1][1][0][RTW89_ACMA][33] = 127,
+	[0][1][1][0][RTW89_CN][33] = 127,
+	[0][1][1][0][RTW89_UK][33] = 127,
+	[0][1][1][0][RTW89_FCC][35] = 127,
+	[0][1][1][0][RTW89_ETSI][35] = 127,
+	[0][1][1][0][RTW89_MKK][35] = 127,
+	[0][1][1][0][RTW89_IC][35] = 127,
+	[0][1][1][0][RTW89_KCC][35] = 127,
+	[0][1][1][0][RTW89_ACMA][35] = 127,
+	[0][1][1][0][RTW89_CN][35] = 127,
+	[0][1][1][0][RTW89_UK][35] = 127,
+	[0][1][1][0][RTW89_FCC][37] = 127,
+	[0][1][1][0][RTW89_ETSI][37] = 127,
+	[0][1][1][0][RTW89_MKK][37] = 127,
+	[0][1][1][0][RTW89_IC][37] = 127,
+	[0][1][1][0][RTW89_KCC][37] = 127,
+	[0][1][1][0][RTW89_ACMA][37] = 127,
+	[0][1][1][0][RTW89_CN][37] = 127,
+	[0][1][1][0][RTW89_UK][37] = 127,
+	[0][1][1][0][RTW89_FCC][38] = 127,
+	[0][1][1][0][RTW89_ETSI][38] = 127,
+	[0][1][1][0][RTW89_MKK][38] = 127,
+	[0][1][1][0][RTW89_IC][38] = 127,
+	[0][1][1][0][RTW89_KCC][38] = 127,
+	[0][1][1][0][RTW89_ACMA][38] = 127,
+	[0][1][1][0][RTW89_CN][38] = 127,
+	[0][1][1][0][RTW89_UK][38] = 127,
+	[0][1][1][0][RTW89_FCC][40] = 127,
+	[0][1][1][0][RTW89_ETSI][40] = 127,
+	[0][1][1][0][RTW89_MKK][40] = 127,
+	[0][1][1][0][RTW89_IC][40] = 127,
+	[0][1][1][0][RTW89_KCC][40] = 127,
+	[0][1][1][0][RTW89_ACMA][40] = 127,
+	[0][1][1][0][RTW89_CN][40] = 127,
+	[0][1][1][0][RTW89_UK][40] = 127,
+	[0][1][1][0][RTW89_FCC][42] = 127,
+	[0][1][1][0][RTW89_ETSI][42] = 127,
+	[0][1][1][0][RTW89_MKK][42] = 127,
+	[0][1][1][0][RTW89_IC][42] = 127,
+	[0][1][1][0][RTW89_KCC][42] = 127,
+	[0][1][1][0][RTW89_ACMA][42] = 127,
+	[0][1][1][0][RTW89_CN][42] = 127,
+	[0][1][1][0][RTW89_UK][42] = 127,
+	[0][1][1][0][RTW89_FCC][44] = 127,
+	[0][1][1][0][RTW89_ETSI][44] = 127,
+	[0][1][1][0][RTW89_MKK][44] = 127,
+	[0][1][1][0][RTW89_IC][44] = 127,
+	[0][1][1][0][RTW89_KCC][44] = 127,
+	[0][1][1][0][RTW89_ACMA][44] = 127,
+	[0][1][1][0][RTW89_CN][44] = 127,
+	[0][1][1][0][RTW89_UK][44] = 127,
+	[0][1][1][0][RTW89_FCC][46] = 127,
+	[0][1][1][0][RTW89_ETSI][46] = 127,
+	[0][1][1][0][RTW89_MKK][46] = 127,
+	[0][1][1][0][RTW89_IC][46] = 127,
+	[0][1][1][0][RTW89_KCC][46] = 127,
+	[0][1][1][0][RTW89_ACMA][46] = 127,
+	[0][1][1][0][RTW89_CN][46] = 127,
+	[0][1][1][0][RTW89_UK][46] = 127,
+	[0][1][1][0][RTW89_FCC][48] = 127,
+	[0][1][1][0][RTW89_ETSI][48] = 127,
+	[0][1][1][0][RTW89_MKK][48] = 127,
+	[0][1][1][0][RTW89_IC][48] = 127,
+	[0][1][1][0][RTW89_KCC][48] = 127,
+	[0][1][1][0][RTW89_ACMA][48] = 127,
+	[0][1][1][0][RTW89_CN][48] = 127,
+	[0][1][1][0][RTW89_UK][48] = 127,
+	[0][1][1][0][RTW89_FCC][50] = 127,
+	[0][1][1][0][RTW89_ETSI][50] = 127,
+	[0][1][1][0][RTW89_MKK][50] = 127,
+	[0][1][1][0][RTW89_IC][50] = 127,
+	[0][1][1][0][RTW89_KCC][50] = 127,
+	[0][1][1][0][RTW89_ACMA][50] = 127,
+	[0][1][1][0][RTW89_CN][50] = 127,
+	[0][1][1][0][RTW89_UK][50] = 127,
+	[0][1][1][0][RTW89_FCC][52] = 127,
+	[0][1][1][0][RTW89_ETSI][52] = 127,
+	[0][1][1][0][RTW89_MKK][52] = 127,
+	[0][1][1][0][RTW89_IC][52] = 127,
+	[0][1][1][0][RTW89_KCC][52] = 127,
+	[0][1][1][0][RTW89_ACMA][52] = 127,
+	[0][1][1][0][RTW89_CN][52] = 127,
+	[0][1][1][0][RTW89_UK][52] = 127,
+	[0][0][2][0][RTW89_FCC][0] = 76,
+	[0][0][2][0][RTW89_ETSI][0] = 62,
+	[0][0][2][0][RTW89_MKK][0] = 62,
+	[0][0][2][0][RTW89_IC][0] = 64,
+	[0][0][2][0][RTW89_KCC][0] = 74,
+	[0][0][2][0][RTW89_ACMA][0] = 62,
+	[0][0][2][0][RTW89_CN][0] = 62,
+	[0][0][2][0][RTW89_UK][0] = 62,
+	[0][0][2][0][RTW89_FCC][2] = 78,
+	[0][0][2][0][RTW89_ETSI][2] = 62,
+	[0][0][2][0][RTW89_MKK][2] = 62,
+	[0][0][2][0][RTW89_IC][2] = 64,
+	[0][0][2][0][RTW89_KCC][2] = 74,
+	[0][0][2][0][RTW89_ACMA][2] = 62,
+	[0][0][2][0][RTW89_CN][2] = 62,
+	[0][0][2][0][RTW89_UK][2] = 62,
+	[0][0][2][0][RTW89_FCC][4] = 78,
+	[0][0][2][0][RTW89_ETSI][4] = 62,
+	[0][0][2][0][RTW89_MKK][4] = 62,
+	[0][0][2][0][RTW89_IC][4] = 64,
+	[0][0][2][0][RTW89_KCC][4] = 74,
+	[0][0][2][0][RTW89_ACMA][4] = 62,
+	[0][0][2][0][RTW89_CN][4] = 62,
+	[0][0][2][0][RTW89_UK][4] = 62,
+	[0][0][2][0][RTW89_FCC][6] = 78,
+	[0][0][2][0][RTW89_ETSI][6] = 62,
+	[0][0][2][0][RTW89_MKK][6] = 62,
+	[0][0][2][0][RTW89_IC][6] = 64,
+	[0][0][2][0][RTW89_KCC][6] = 54,
+	[0][0][2][0][RTW89_ACMA][6] = 62,
+	[0][0][2][0][RTW89_CN][6] = 62,
+	[0][0][2][0][RTW89_UK][6] = 62,
+	[0][0][2][0][RTW89_FCC][8] = 78,
+	[0][0][2][0][RTW89_ETSI][8] = 62,
+	[0][0][2][0][RTW89_MKK][8] = 62,
+	[0][0][2][0][RTW89_IC][8] = 64,
+	[0][0][2][0][RTW89_KCC][8] = 74,
+	[0][0][2][0][RTW89_ACMA][8] = 62,
+	[0][0][2][0][RTW89_CN][8] = 62,
+	[0][0][2][0][RTW89_UK][8] = 62,
+	[0][0][2][0][RTW89_FCC][10] = 78,
+	[0][0][2][0][RTW89_ETSI][10] = 62,
+	[0][0][2][0][RTW89_MKK][10] = 62,
+	[0][0][2][0][RTW89_IC][10] = 64,
+	[0][0][2][0][RTW89_KCC][10] = 74,
+	[0][0][2][0][RTW89_ACMA][10] = 62,
+	[0][0][2][0][RTW89_CN][10] = 62,
+	[0][0][2][0][RTW89_UK][10] = 62,
+	[0][0][2][0][RTW89_FCC][12] = 78,
+	[0][0][2][0][RTW89_ETSI][12] = 62,
+	[0][0][2][0][RTW89_MKK][12] = 62,
+	[0][0][2][0][RTW89_IC][12] = 64,
+	[0][0][2][0][RTW89_KCC][12] = 74,
+	[0][0][2][0][RTW89_ACMA][12] = 62,
+	[0][0][2][0][RTW89_CN][12] = 62,
+	[0][0][2][0][RTW89_UK][12] = 62,
+	[0][0][2][0][RTW89_FCC][14] = 74,
+	[0][0][2][0][RTW89_ETSI][14] = 62,
+	[0][0][2][0][RTW89_MKK][14] = 62,
+	[0][0][2][0][RTW89_IC][14] = 64,
+	[0][0][2][0][RTW89_KCC][14] = 74,
+	[0][0][2][0][RTW89_ACMA][14] = 62,
+	[0][0][2][0][RTW89_CN][14] = 62,
+	[0][0][2][0][RTW89_UK][14] = 62,
+	[0][0][2][0][RTW89_FCC][15] = 74,
+	[0][0][2][0][RTW89_ETSI][15] = 60,
+	[0][0][2][0][RTW89_MKK][15] = 74,
+	[0][0][2][0][RTW89_IC][15] = 74,
+	[0][0][2][0][RTW89_KCC][15] = 74,
+	[0][0][2][0][RTW89_ACMA][15] = 60,
+	[0][0][2][0][RTW89_CN][15] = 127,
+	[0][0][2][0][RTW89_UK][15] = 60,
+	[0][0][2][0][RTW89_FCC][17] = 78,
+	[0][0][2][0][RTW89_ETSI][17] = 62,
+	[0][0][2][0][RTW89_MKK][17] = 74,
+	[0][0][2][0][RTW89_IC][17] = 78,
+	[0][0][2][0][RTW89_KCC][17] = 74,
+	[0][0][2][0][RTW89_ACMA][17] = 62,
+	[0][0][2][0][RTW89_CN][17] = 127,
+	[0][0][2][0][RTW89_UK][17] = 62,
+	[0][0][2][0][RTW89_FCC][19] = 78,
+	[0][0][2][0][RTW89_ETSI][19] = 62,
+	[0][0][2][0][RTW89_MKK][19] = 74,
+	[0][0][2][0][RTW89_IC][19] = 78,
+	[0][0][2][0][RTW89_KCC][19] = 74,
+	[0][0][2][0][RTW89_ACMA][19] = 62,
+	[0][0][2][0][RTW89_CN][19] = 127,
+	[0][0][2][0][RTW89_UK][19] = 62,
+	[0][0][2][0][RTW89_FCC][21] = 78,
+	[0][0][2][0][RTW89_ETSI][21] = 62,
+	[0][0][2][0][RTW89_MKK][21] = 74,
+	[0][0][2][0][RTW89_IC][21] = 78,
+	[0][0][2][0][RTW89_KCC][21] = 74,
+	[0][0][2][0][RTW89_ACMA][21] = 62,
+	[0][0][2][0][RTW89_CN][21] = 127,
+	[0][0][2][0][RTW89_UK][21] = 62,
+	[0][0][2][0][RTW89_FCC][23] = 78,
+	[0][0][2][0][RTW89_ETSI][23] = 62,
+	[0][0][2][0][RTW89_MKK][23] = 74,
+	[0][0][2][0][RTW89_IC][23] = 78,
+	[0][0][2][0][RTW89_KCC][23] = 74,
+	[0][0][2][0][RTW89_ACMA][23] = 62,
+	[0][0][2][0][RTW89_CN][23] = 127,
+	[0][0][2][0][RTW89_UK][23] = 62,
+	[0][0][2][0][RTW89_FCC][25] = 78,
+	[0][0][2][0][RTW89_ETSI][25] = 62,
+	[0][0][2][0][RTW89_MKK][25] = 74,
+	[0][0][2][0][RTW89_IC][25] = 127,
+	[0][0][2][0][RTW89_KCC][25] = 74,
+	[0][0][2][0][RTW89_ACMA][25] = 127,
+	[0][0][2][0][RTW89_CN][25] = 127,
+	[0][0][2][0][RTW89_UK][25] = 62,
+	[0][0][2][0][RTW89_FCC][27] = 78,
+	[0][0][2][0][RTW89_ETSI][27] = 62,
+	[0][0][2][0][RTW89_MKK][27] = 74,
+	[0][0][2][0][RTW89_IC][27] = 127,
+	[0][0][2][0][RTW89_KCC][27] = 74,
+	[0][0][2][0][RTW89_ACMA][27] = 127,
+	[0][0][2][0][RTW89_CN][27] = 127,
+	[0][0][2][0][RTW89_UK][27] = 62,
+	[0][0][2][0][RTW89_FCC][29] = 78,
+	[0][0][2][0][RTW89_ETSI][29] = 62,
+	[0][0][2][0][RTW89_MKK][29] = 74,
+	[0][0][2][0][RTW89_IC][29] = 127,
+	[0][0][2][0][RTW89_KCC][29] = 74,
+	[0][0][2][0][RTW89_ACMA][29] = 127,
+	[0][0][2][0][RTW89_CN][29] = 127,
+	[0][0][2][0][RTW89_UK][29] = 62,
+	[0][0][2][0][RTW89_FCC][31] = 78,
+	[0][0][2][0][RTW89_ETSI][31] = 62,
+	[0][0][2][0][RTW89_MKK][31] = 74,
+	[0][0][2][0][RTW89_IC][31] = 78,
+	[0][0][2][0][RTW89_KCC][31] = 74,
+	[0][0][2][0][RTW89_ACMA][31] = 62,
+	[0][0][2][0][RTW89_CN][31] = 127,
+	[0][0][2][0][RTW89_UK][31] = 62,
+	[0][0][2][0][RTW89_FCC][33] = 78,
+	[0][0][2][0][RTW89_ETSI][33] = 62,
+	[0][0][2][0][RTW89_MKK][33] = 74,
+	[0][0][2][0][RTW89_IC][33] = 78,
+	[0][0][2][0][RTW89_KCC][33] = 74,
+	[0][0][2][0][RTW89_ACMA][33] = 62,
+	[0][0][2][0][RTW89_CN][33] = 127,
+	[0][0][2][0][RTW89_UK][33] = 62,
+	[0][0][2][0][RTW89_FCC][35] = 72,
+	[0][0][2][0][RTW89_ETSI][35] = 62,
+	[0][0][2][0][RTW89_MKK][35] = 74,
+	[0][0][2][0][RTW89_IC][35] = 72,
+	[0][0][2][0][RTW89_KCC][35] = 74,
+	[0][0][2][0][RTW89_ACMA][35] = 62,
+	[0][0][2][0][RTW89_CN][35] = 127,
+	[0][0][2][0][RTW89_UK][35] = 62,
+	[0][0][2][0][RTW89_FCC][37] = 78,
+	[0][0][2][0][RTW89_ETSI][37] = 127,
+	[0][0][2][0][RTW89_MKK][37] = 74,
+	[0][0][2][0][RTW89_IC][37] = 78,
+	[0][0][2][0][RTW89_KCC][37] = 74,
+	[0][0][2][0][RTW89_ACMA][37] = 74,
+	[0][0][2][0][RTW89_CN][37] = 127,
+	[0][0][2][0][RTW89_UK][37] = 74,
+	[0][0][2][0][RTW89_FCC][38] = 78,
+	[0][0][2][0][RTW89_ETSI][38] = 30,
+	[0][0][2][0][RTW89_MKK][38] = 127,
+	[0][0][2][0][RTW89_IC][38] = 78,
+	[0][0][2][0][RTW89_KCC][38] = 66,
+	[0][0][2][0][RTW89_ACMA][38] = 74,
+	[0][0][2][0][RTW89_CN][38] = 74,
+	[0][0][2][0][RTW89_UK][38] = 60,
+	[0][0][2][0][RTW89_FCC][40] = 78,
+	[0][0][2][0][RTW89_ETSI][40] = 30,
+	[0][0][2][0][RTW89_MKK][40] = 127,
+	[0][0][2][0][RTW89_IC][40] = 78,
+	[0][0][2][0][RTW89_KCC][40] = 74,
+	[0][0][2][0][RTW89_ACMA][40] = 74,
+	[0][0][2][0][RTW89_CN][40] = 74,
+	[0][0][2][0][RTW89_UK][40] = 60,
+	[0][0][2][0][RTW89_FCC][42] = 78,
+	[0][0][2][0][RTW89_ETSI][42] = 30,
+	[0][0][2][0][RTW89_MKK][42] = 127,
+	[0][0][2][0][RTW89_IC][42] = 78,
+	[0][0][2][0][RTW89_KCC][42] = 74,
+	[0][0][2][0][RTW89_ACMA][42] = 74,
+	[0][0][2][0][RTW89_CN][42] = 74,
+	[0][0][2][0][RTW89_UK][42] = 60,
+	[0][0][2][0][RTW89_FCC][44] = 78,
+	[0][0][2][0][RTW89_ETSI][44] = 30,
+	[0][0][2][0][RTW89_MKK][44] = 127,
+	[0][0][2][0][RTW89_IC][44] = 78,
+	[0][0][2][0][RTW89_KCC][44] = 74,
+	[0][0][2][0][RTW89_ACMA][44] = 74,
+	[0][0][2][0][RTW89_CN][44] = 74,
+	[0][0][2][0][RTW89_UK][44] = 60,
+	[0][0][2][0][RTW89_FCC][46] = 78,
+	[0][0][2][0][RTW89_ETSI][46] = 30,
+	[0][0][2][0][RTW89_MKK][46] = 127,
+	[0][0][2][0][RTW89_IC][46] = 78,
+	[0][0][2][0][RTW89_KCC][46] = 74,
+	[0][0][2][0][RTW89_ACMA][46] = 74,
+	[0][0][2][0][RTW89_CN][46] = 74,
+	[0][0][2][0][RTW89_UK][46] = 60,
+	[0][0][2][0][RTW89_FCC][48] = 74,
+	[0][0][2][0][RTW89_ETSI][48] = 127,
+	[0][0][2][0][RTW89_MKK][48] = 127,
+	[0][0][2][0][RTW89_IC][48] = 127,
+	[0][0][2][0][RTW89_KCC][48] = 127,
+	[0][0][2][0][RTW89_ACMA][48] = 127,
+	[0][0][2][0][RTW89_CN][48] = 127,
+	[0][0][2][0][RTW89_UK][48] = 127,
+	[0][0][2][0][RTW89_FCC][50] = 74,
+	[0][0][2][0][RTW89_ETSI][50] = 127,
+	[0][0][2][0][RTW89_MKK][50] = 127,
+	[0][0][2][0][RTW89_IC][50] = 127,
+	[0][0][2][0][RTW89_KCC][50] = 127,
+	[0][0][2][0][RTW89_ACMA][50] = 127,
+	[0][0][2][0][RTW89_CN][50] = 127,
+	[0][0][2][0][RTW89_UK][50] = 127,
+	[0][0][2][0][RTW89_FCC][52] = 74,
+	[0][0][2][0][RTW89_ETSI][52] = 127,
+	[0][0][2][0][RTW89_MKK][52] = 127,
+	[0][0][2][0][RTW89_IC][52] = 127,
+	[0][0][2][0][RTW89_KCC][52] = 127,
+	[0][0][2][0][RTW89_ACMA][52] = 127,
+	[0][0][2][0][RTW89_CN][52] = 127,
+	[0][0][2][0][RTW89_UK][52] = 127,
+	[0][1][2][0][RTW89_FCC][0] = 127,
+	[0][1][2][0][RTW89_ETSI][0] = 127,
+	[0][1][2][0][RTW89_MKK][0] = 127,
+	[0][1][2][0][RTW89_IC][0] = 127,
+	[0][1][2][0][RTW89_KCC][0] = 127,
+	[0][1][2][0][RTW89_ACMA][0] = 127,
+	[0][1][2][0][RTW89_CN][0] = 127,
+	[0][1][2][0][RTW89_UK][0] = 127,
+	[0][1][2][0][RTW89_FCC][2] = 127,
+	[0][1][2][0][RTW89_ETSI][2] = 127,
+	[0][1][2][0][RTW89_MKK][2] = 127,
+	[0][1][2][0][RTW89_IC][2] = 127,
+	[0][1][2][0][RTW89_KCC][2] = 127,
+	[0][1][2][0][RTW89_ACMA][2] = 127,
+	[0][1][2][0][RTW89_CN][2] = 127,
+	[0][1][2][0][RTW89_UK][2] = 127,
+	[0][1][2][0][RTW89_FCC][4] = 127,
+	[0][1][2][0][RTW89_ETSI][4] = 127,
+	[0][1][2][0][RTW89_MKK][4] = 127,
+	[0][1][2][0][RTW89_IC][4] = 127,
+	[0][1][2][0][RTW89_KCC][4] = 127,
+	[0][1][2][0][RTW89_ACMA][4] = 127,
+	[0][1][2][0][RTW89_CN][4] = 127,
+	[0][1][2][0][RTW89_UK][4] = 127,
+	[0][1][2][0][RTW89_FCC][6] = 127,
+	[0][1][2][0][RTW89_ETSI][6] = 127,
+	[0][1][2][0][RTW89_MKK][6] = 127,
+	[0][1][2][0][RTW89_IC][6] = 127,
+	[0][1][2][0][RTW89_KCC][6] = 127,
+	[0][1][2][0][RTW89_ACMA][6] = 127,
+	[0][1][2][0][RTW89_CN][6] = 127,
+	[0][1][2][0][RTW89_UK][6] = 127,
+	[0][1][2][0][RTW89_FCC][8] = 127,
+	[0][1][2][0][RTW89_ETSI][8] = 127,
+	[0][1][2][0][RTW89_MKK][8] = 127,
+	[0][1][2][0][RTW89_IC][8] = 127,
+	[0][1][2][0][RTW89_KCC][8] = 127,
+	[0][1][2][0][RTW89_ACMA][8] = 127,
+	[0][1][2][0][RTW89_CN][8] = 127,
+	[0][1][2][0][RTW89_UK][8] = 127,
+	[0][1][2][0][RTW89_FCC][10] = 127,
+	[0][1][2][0][RTW89_ETSI][10] = 127,
+	[0][1][2][0][RTW89_MKK][10] = 127,
+	[0][1][2][0][RTW89_IC][10] = 127,
+	[0][1][2][0][RTW89_KCC][10] = 127,
+	[0][1][2][0][RTW89_ACMA][10] = 127,
+	[0][1][2][0][RTW89_CN][10] = 127,
+	[0][1][2][0][RTW89_UK][10] = 127,
+	[0][1][2][0][RTW89_FCC][12] = 127,
+	[0][1][2][0][RTW89_ETSI][12] = 127,
+	[0][1][2][0][RTW89_MKK][12] = 127,
+	[0][1][2][0][RTW89_IC][12] = 127,
+	[0][1][2][0][RTW89_KCC][12] = 127,
+	[0][1][2][0][RTW89_ACMA][12] = 127,
+	[0][1][2][0][RTW89_CN][12] = 127,
+	[0][1][2][0][RTW89_UK][12] = 127,
+	[0][1][2][0][RTW89_FCC][14] = 127,
+	[0][1][2][0][RTW89_ETSI][14] = 127,
+	[0][1][2][0][RTW89_MKK][14] = 127,
+	[0][1][2][0][RTW89_IC][14] = 127,
+	[0][1][2][0][RTW89_KCC][14] = 127,
+	[0][1][2][0][RTW89_ACMA][14] = 127,
+	[0][1][2][0][RTW89_CN][14] = 127,
+	[0][1][2][0][RTW89_UK][14] = 127,
+	[0][1][2][0][RTW89_FCC][15] = 127,
+	[0][1][2][0][RTW89_ETSI][15] = 127,
+	[0][1][2][0][RTW89_MKK][15] = 127,
+	[0][1][2][0][RTW89_IC][15] = 127,
+	[0][1][2][0][RTW89_KCC][15] = 127,
+	[0][1][2][0][RTW89_ACMA][15] = 127,
+	[0][1][2][0][RTW89_CN][15] = 127,
+	[0][1][2][0][RTW89_UK][15] = 127,
+	[0][1][2][0][RTW89_FCC][17] = 127,
+	[0][1][2][0][RTW89_ETSI][17] = 127,
+	[0][1][2][0][RTW89_MKK][17] = 127,
+	[0][1][2][0][RTW89_IC][17] = 127,
+	[0][1][2][0][RTW89_KCC][17] = 127,
+	[0][1][2][0][RTW89_ACMA][17] = 127,
+	[0][1][2][0][RTW89_CN][17] = 127,
+	[0][1][2][0][RTW89_UK][17] = 127,
+	[0][1][2][0][RTW89_FCC][19] = 127,
+	[0][1][2][0][RTW89_ETSI][19] = 127,
+	[0][1][2][0][RTW89_MKK][19] = 127,
+	[0][1][2][0][RTW89_IC][19] = 127,
+	[0][1][2][0][RTW89_KCC][19] = 127,
+	[0][1][2][0][RTW89_ACMA][19] = 127,
+	[0][1][2][0][RTW89_CN][19] = 127,
+	[0][1][2][0][RTW89_UK][19] = 127,
+	[0][1][2][0][RTW89_FCC][21] = 127,
+	[0][1][2][0][RTW89_ETSI][21] = 127,
+	[0][1][2][0][RTW89_MKK][21] = 127,
+	[0][1][2][0][RTW89_IC][21] = 127,
+	[0][1][2][0][RTW89_KCC][21] = 127,
+	[0][1][2][0][RTW89_ACMA][21] = 127,
+	[0][1][2][0][RTW89_CN][21] = 127,
+	[0][1][2][0][RTW89_UK][21] = 127,
+	[0][1][2][0][RTW89_FCC][23] = 127,
+	[0][1][2][0][RTW89_ETSI][23] = 127,
+	[0][1][2][0][RTW89_MKK][23] = 127,
+	[0][1][2][0][RTW89_IC][23] = 127,
+	[0][1][2][0][RTW89_KCC][23] = 127,
+	[0][1][2][0][RTW89_ACMA][23] = 127,
+	[0][1][2][0][RTW89_CN][23] = 127,
+	[0][1][2][0][RTW89_UK][23] = 127,
+	[0][1][2][0][RTW89_FCC][25] = 127,
+	[0][1][2][0][RTW89_ETSI][25] = 127,
+	[0][1][2][0][RTW89_MKK][25] = 127,
+	[0][1][2][0][RTW89_IC][25] = 127,
+	[0][1][2][0][RTW89_KCC][25] = 127,
+	[0][1][2][0][RTW89_ACMA][25] = 127,
+	[0][1][2][0][RTW89_CN][25] = 127,
+	[0][1][2][0][RTW89_UK][25] = 127,
+	[0][1][2][0][RTW89_FCC][27] = 127,
+	[0][1][2][0][RTW89_ETSI][27] = 127,
+	[0][1][2][0][RTW89_MKK][27] = 127,
+	[0][1][2][0][RTW89_IC][27] = 127,
+	[0][1][2][0][RTW89_KCC][27] = 127,
+	[0][1][2][0][RTW89_ACMA][27] = 127,
+	[0][1][2][0][RTW89_CN][27] = 127,
+	[0][1][2][0][RTW89_UK][27] = 127,
+	[0][1][2][0][RTW89_FCC][29] = 127,
+	[0][1][2][0][RTW89_ETSI][29] = 127,
+	[0][1][2][0][RTW89_MKK][29] = 127,
+	[0][1][2][0][RTW89_IC][29] = 127,
+	[0][1][2][0][RTW89_KCC][29] = 127,
+	[0][1][2][0][RTW89_ACMA][29] = 127,
+	[0][1][2][0][RTW89_CN][29] = 127,
+	[0][1][2][0][RTW89_UK][29] = 127,
+	[0][1][2][0][RTW89_FCC][31] = 127,
+	[0][1][2][0][RTW89_ETSI][31] = 127,
+	[0][1][2][0][RTW89_MKK][31] = 127,
+	[0][1][2][0][RTW89_IC][31] = 127,
+	[0][1][2][0][RTW89_KCC][31] = 127,
+	[0][1][2][0][RTW89_ACMA][31] = 127,
+	[0][1][2][0][RTW89_CN][31] = 127,
+	[0][1][2][0][RTW89_UK][31] = 127,
+	[0][1][2][0][RTW89_FCC][33] = 127,
+	[0][1][2][0][RTW89_ETSI][33] = 127,
+	[0][1][2][0][RTW89_MKK][33] = 127,
+	[0][1][2][0][RTW89_IC][33] = 127,
+	[0][1][2][0][RTW89_KCC][33] = 127,
+	[0][1][2][0][RTW89_ACMA][33] = 127,
+	[0][1][2][0][RTW89_CN][33] = 127,
+	[0][1][2][0][RTW89_UK][33] = 127,
+	[0][1][2][0][RTW89_FCC][35] = 127,
+	[0][1][2][0][RTW89_ETSI][35] = 127,
+	[0][1][2][0][RTW89_MKK][35] = 127,
+	[0][1][2][0][RTW89_IC][35] = 127,
+	[0][1][2][0][RTW89_KCC][35] = 127,
+	[0][1][2][0][RTW89_ACMA][35] = 127,
+	[0][1][2][0][RTW89_CN][35] = 127,
+	[0][1][2][0][RTW89_UK][35] = 127,
+	[0][1][2][0][RTW89_FCC][37] = 127,
+	[0][1][2][0][RTW89_ETSI][37] = 127,
+	[0][1][2][0][RTW89_MKK][37] = 127,
+	[0][1][2][0][RTW89_IC][37] = 127,
+	[0][1][2][0][RTW89_KCC][37] = 127,
+	[0][1][2][0][RTW89_ACMA][37] = 127,
+	[0][1][2][0][RTW89_CN][37] = 127,
+	[0][1][2][0][RTW89_UK][37] = 127,
+	[0][1][2][0][RTW89_FCC][38] = 127,
+	[0][1][2][0][RTW89_ETSI][38] = 127,
+	[0][1][2][0][RTW89_MKK][38] = 127,
+	[0][1][2][0][RTW89_IC][38] = 127,
+	[0][1][2][0][RTW89_KCC][38] = 127,
+	[0][1][2][0][RTW89_ACMA][38] = 127,
+	[0][1][2][0][RTW89_CN][38] = 127,
+	[0][1][2][0][RTW89_UK][38] = 127,
+	[0][1][2][0][RTW89_FCC][40] = 127,
+	[0][1][2][0][RTW89_ETSI][40] = 127,
+	[0][1][2][0][RTW89_MKK][40] = 127,
+	[0][1][2][0][RTW89_IC][40] = 127,
+	[0][1][2][0][RTW89_KCC][40] = 127,
+	[0][1][2][0][RTW89_ACMA][40] = 127,
+	[0][1][2][0][RTW89_CN][40] = 127,
+	[0][1][2][0][RTW89_UK][40] = 127,
+	[0][1][2][0][RTW89_FCC][42] = 127,
+	[0][1][2][0][RTW89_ETSI][42] = 127,
+	[0][1][2][0][RTW89_MKK][42] = 127,
+	[0][1][2][0][RTW89_IC][42] = 127,
+	[0][1][2][0][RTW89_KCC][42] = 127,
+	[0][1][2][0][RTW89_ACMA][42] = 127,
+	[0][1][2][0][RTW89_CN][42] = 127,
+	[0][1][2][0][RTW89_UK][42] = 127,
+	[0][1][2][0][RTW89_FCC][44] = 127,
+	[0][1][2][0][RTW89_ETSI][44] = 127,
+	[0][1][2][0][RTW89_MKK][44] = 127,
+	[0][1][2][0][RTW89_IC][44] = 127,
+	[0][1][2][0][RTW89_KCC][44] = 127,
+	[0][1][2][0][RTW89_ACMA][44] = 127,
+	[0][1][2][0][RTW89_CN][44] = 127,
+	[0][1][2][0][RTW89_UK][44] = 127,
+	[0][1][2][0][RTW89_FCC][46] = 127,
+	[0][1][2][0][RTW89_ETSI][46] = 127,
+	[0][1][2][0][RTW89_MKK][46] = 127,
+	[0][1][2][0][RTW89_IC][46] = 127,
+	[0][1][2][0][RTW89_KCC][46] = 127,
+	[0][1][2][0][RTW89_ACMA][46] = 127,
+	[0][1][2][0][RTW89_CN][46] = 127,
+	[0][1][2][0][RTW89_UK][46] = 127,
+	[0][1][2][0][RTW89_FCC][48] = 127,
+	[0][1][2][0][RTW89_ETSI][48] = 127,
+	[0][1][2][0][RTW89_MKK][48] = 127,
+	[0][1][2][0][RTW89_IC][48] = 127,
+	[0][1][2][0][RTW89_KCC][48] = 127,
+	[0][1][2][0][RTW89_ACMA][48] = 127,
+	[0][1][2][0][RTW89_CN][48] = 127,
+	[0][1][2][0][RTW89_UK][48] = 127,
+	[0][1][2][0][RTW89_FCC][50] = 127,
+	[0][1][2][0][RTW89_ETSI][50] = 127,
+	[0][1][2][0][RTW89_MKK][50] = 127,
+	[0][1][2][0][RTW89_IC][50] = 127,
+	[0][1][2][0][RTW89_KCC][50] = 127,
+	[0][1][2][0][RTW89_ACMA][50] = 127,
+	[0][1][2][0][RTW89_CN][50] = 127,
+	[0][1][2][0][RTW89_UK][50] = 127,
+	[0][1][2][0][RTW89_FCC][52] = 127,
+	[0][1][2][0][RTW89_ETSI][52] = 127,
+	[0][1][2][0][RTW89_MKK][52] = 127,
+	[0][1][2][0][RTW89_IC][52] = 127,
+	[0][1][2][0][RTW89_KCC][52] = 127,
+	[0][1][2][0][RTW89_ACMA][52] = 127,
+	[0][1][2][0][RTW89_CN][52] = 127,
+	[0][1][2][0][RTW89_UK][52] = 127,
+	[0][1][2][1][RTW89_FCC][0] = 127,
+	[0][1][2][1][RTW89_ETSI][0] = 127,
+	[0][1][2][1][RTW89_MKK][0] = 127,
+	[0][1][2][1][RTW89_IC][0] = 127,
+	[0][1][2][1][RTW89_KCC][0] = 127,
+	[0][1][2][1][RTW89_ACMA][0] = 127,
+	[0][1][2][1][RTW89_CN][0] = 127,
+	[0][1][2][1][RTW89_UK][0] = 127,
+	[0][1][2][1][RTW89_FCC][2] = 127,
+	[0][1][2][1][RTW89_ETSI][2] = 127,
+	[0][1][2][1][RTW89_MKK][2] = 127,
+	[0][1][2][1][RTW89_IC][2] = 127,
+	[0][1][2][1][RTW89_KCC][2] = 127,
+	[0][1][2][1][RTW89_ACMA][2] = 127,
+	[0][1][2][1][RTW89_CN][2] = 127,
+	[0][1][2][1][RTW89_UK][2] = 127,
+	[0][1][2][1][RTW89_FCC][4] = 127,
+	[0][1][2][1][RTW89_ETSI][4] = 127,
+	[0][1][2][1][RTW89_MKK][4] = 127,
+	[0][1][2][1][RTW89_IC][4] = 127,
+	[0][1][2][1][RTW89_KCC][4] = 127,
+	[0][1][2][1][RTW89_ACMA][4] = 127,
+	[0][1][2][1][RTW89_CN][4] = 127,
+	[0][1][2][1][RTW89_UK][4] = 127,
+	[0][1][2][1][RTW89_FCC][6] = 127,
+	[0][1][2][1][RTW89_ETSI][6] = 127,
+	[0][1][2][1][RTW89_MKK][6] = 127,
+	[0][1][2][1][RTW89_IC][6] = 127,
+	[0][1][2][1][RTW89_KCC][6] = 127,
+	[0][1][2][1][RTW89_ACMA][6] = 127,
+	[0][1][2][1][RTW89_CN][6] = 127,
+	[0][1][2][1][RTW89_UK][6] = 127,
+	[0][1][2][1][RTW89_FCC][8] = 127,
+	[0][1][2][1][RTW89_ETSI][8] = 127,
+	[0][1][2][1][RTW89_MKK][8] = 127,
+	[0][1][2][1][RTW89_IC][8] = 127,
+	[0][1][2][1][RTW89_KCC][8] = 127,
+	[0][1][2][1][RTW89_ACMA][8] = 127,
+	[0][1][2][1][RTW89_CN][8] = 127,
+	[0][1][2][1][RTW89_UK][8] = 127,
+	[0][1][2][1][RTW89_FCC][10] = 127,
+	[0][1][2][1][RTW89_ETSI][10] = 127,
+	[0][1][2][1][RTW89_MKK][10] = 127,
+	[0][1][2][1][RTW89_IC][10] = 127,
+	[0][1][2][1][RTW89_KCC][10] = 127,
+	[0][1][2][1][RTW89_ACMA][10] = 127,
+	[0][1][2][1][RTW89_CN][10] = 127,
+	[0][1][2][1][RTW89_UK][10] = 127,
+	[0][1][2][1][RTW89_FCC][12] = 127,
+	[0][1][2][1][RTW89_ETSI][12] = 127,
+	[0][1][2][1][RTW89_MKK][12] = 127,
+	[0][1][2][1][RTW89_IC][12] = 127,
+	[0][1][2][1][RTW89_KCC][12] = 127,
+	[0][1][2][1][RTW89_ACMA][12] = 127,
+	[0][1][2][1][RTW89_CN][12] = 127,
+	[0][1][2][1][RTW89_UK][12] = 127,
+	[0][1][2][1][RTW89_FCC][14] = 127,
+	[0][1][2][1][RTW89_ETSI][14] = 127,
+	[0][1][2][1][RTW89_MKK][14] = 127,
+	[0][1][2][1][RTW89_IC][14] = 127,
+	[0][1][2][1][RTW89_KCC][14] = 127,
+	[0][1][2][1][RTW89_ACMA][14] = 127,
+	[0][1][2][1][RTW89_CN][14] = 127,
+	[0][1][2][1][RTW89_UK][14] = 127,
+	[0][1][2][1][RTW89_FCC][15] = 127,
+	[0][1][2][1][RTW89_ETSI][15] = 127,
+	[0][1][2][1][RTW89_MKK][15] = 127,
+	[0][1][2][1][RTW89_IC][15] = 127,
+	[0][1][2][1][RTW89_KCC][15] = 127,
+	[0][1][2][1][RTW89_ACMA][15] = 127,
+	[0][1][2][1][RTW89_CN][15] = 127,
+	[0][1][2][1][RTW89_UK][15] = 127,
+	[0][1][2][1][RTW89_FCC][17] = 127,
+	[0][1][2][1][RTW89_ETSI][17] = 127,
+	[0][1][2][1][RTW89_MKK][17] = 127,
+	[0][1][2][1][RTW89_IC][17] = 127,
+	[0][1][2][1][RTW89_KCC][17] = 127,
+	[0][1][2][1][RTW89_ACMA][17] = 127,
+	[0][1][2][1][RTW89_CN][17] = 127,
+	[0][1][2][1][RTW89_UK][17] = 127,
+	[0][1][2][1][RTW89_FCC][19] = 127,
+	[0][1][2][1][RTW89_ETSI][19] = 127,
+	[0][1][2][1][RTW89_MKK][19] = 127,
+	[0][1][2][1][RTW89_IC][19] = 127,
+	[0][1][2][1][RTW89_KCC][19] = 127,
+	[0][1][2][1][RTW89_ACMA][19] = 127,
+	[0][1][2][1][RTW89_CN][19] = 127,
+	[0][1][2][1][RTW89_UK][19] = 127,
+	[0][1][2][1][RTW89_FCC][21] = 127,
+	[0][1][2][1][RTW89_ETSI][21] = 127,
+	[0][1][2][1][RTW89_MKK][21] = 127,
+	[0][1][2][1][RTW89_IC][21] = 127,
+	[0][1][2][1][RTW89_KCC][21] = 127,
+	[0][1][2][1][RTW89_ACMA][21] = 127,
+	[0][1][2][1][RTW89_CN][21] = 127,
+	[0][1][2][1][RTW89_UK][21] = 127,
+	[0][1][2][1][RTW89_FCC][23] = 127,
+	[0][1][2][1][RTW89_ETSI][23] = 127,
+	[0][1][2][1][RTW89_MKK][23] = 127,
+	[0][1][2][1][RTW89_IC][23] = 127,
+	[0][1][2][1][RTW89_KCC][23] = 127,
+	[0][1][2][1][RTW89_ACMA][23] = 127,
+	[0][1][2][1][RTW89_CN][23] = 127,
+	[0][1][2][1][RTW89_UK][23] = 127,
+	[0][1][2][1][RTW89_FCC][25] = 127,
+	[0][1][2][1][RTW89_ETSI][25] = 127,
+	[0][1][2][1][RTW89_MKK][25] = 127,
+	[0][1][2][1][RTW89_IC][25] = 127,
+	[0][1][2][1][RTW89_KCC][25] = 127,
+	[0][1][2][1][RTW89_ACMA][25] = 127,
+	[0][1][2][1][RTW89_CN][25] = 127,
+	[0][1][2][1][RTW89_UK][25] = 127,
+	[0][1][2][1][RTW89_FCC][27] = 127,
+	[0][1][2][1][RTW89_ETSI][27] = 127,
+	[0][1][2][1][RTW89_MKK][27] = 127,
+	[0][1][2][1][RTW89_IC][27] = 127,
+	[0][1][2][1][RTW89_KCC][27] = 127,
+	[0][1][2][1][RTW89_ACMA][27] = 127,
+	[0][1][2][1][RTW89_CN][27] = 127,
+	[0][1][2][1][RTW89_UK][27] = 127,
+	[0][1][2][1][RTW89_FCC][29] = 127,
+	[0][1][2][1][RTW89_ETSI][29] = 127,
+	[0][1][2][1][RTW89_MKK][29] = 127,
+	[0][1][2][1][RTW89_IC][29] = 127,
+	[0][1][2][1][RTW89_KCC][29] = 127,
+	[0][1][2][1][RTW89_ACMA][29] = 127,
+	[0][1][2][1][RTW89_CN][29] = 127,
+	[0][1][2][1][RTW89_UK][29] = 127,
+	[0][1][2][1][RTW89_FCC][31] = 127,
+	[0][1][2][1][RTW89_ETSI][31] = 127,
+	[0][1][2][1][RTW89_MKK][31] = 127,
+	[0][1][2][1][RTW89_IC][31] = 127,
+	[0][1][2][1][RTW89_KCC][31] = 127,
+	[0][1][2][1][RTW89_ACMA][31] = 127,
+	[0][1][2][1][RTW89_CN][31] = 127,
+	[0][1][2][1][RTW89_UK][31] = 127,
+	[0][1][2][1][RTW89_FCC][33] = 127,
+	[0][1][2][1][RTW89_ETSI][33] = 127,
+	[0][1][2][1][RTW89_MKK][33] = 127,
+	[0][1][2][1][RTW89_IC][33] = 127,
+	[0][1][2][1][RTW89_KCC][33] = 127,
+	[0][1][2][1][RTW89_ACMA][33] = 127,
+	[0][1][2][1][RTW89_CN][33] = 127,
+	[0][1][2][1][RTW89_UK][33] = 127,
+	[0][1][2][1][RTW89_FCC][35] = 127,
+	[0][1][2][1][RTW89_ETSI][35] = 127,
+	[0][1][2][1][RTW89_MKK][35] = 127,
+	[0][1][2][1][RTW89_IC][35] = 127,
+	[0][1][2][1][RTW89_KCC][35] = 127,
+	[0][1][2][1][RTW89_ACMA][35] = 127,
+	[0][1][2][1][RTW89_CN][35] = 127,
+	[0][1][2][1][RTW89_UK][35] = 127,
+	[0][1][2][1][RTW89_FCC][37] = 127,
+	[0][1][2][1][RTW89_ETSI][37] = 127,
+	[0][1][2][1][RTW89_MKK][37] = 127,
+	[0][1][2][1][RTW89_IC][37] = 127,
+	[0][1][2][1][RTW89_KCC][37] = 127,
+	[0][1][2][1][RTW89_ACMA][37] = 127,
+	[0][1][2][1][RTW89_CN][37] = 127,
+	[0][1][2][1][RTW89_UK][37] = 127,
+	[0][1][2][1][RTW89_FCC][38] = 127,
+	[0][1][2][1][RTW89_ETSI][38] = 127,
+	[0][1][2][1][RTW89_MKK][38] = 127,
+	[0][1][2][1][RTW89_IC][38] = 127,
+	[0][1][2][1][RTW89_KCC][38] = 127,
+	[0][1][2][1][RTW89_ACMA][38] = 127,
+	[0][1][2][1][RTW89_CN][38] = 127,
+	[0][1][2][1][RTW89_UK][38] = 127,
+	[0][1][2][1][RTW89_FCC][40] = 127,
+	[0][1][2][1][RTW89_ETSI][40] = 127,
+	[0][1][2][1][RTW89_MKK][40] = 127,
+	[0][1][2][1][RTW89_IC][40] = 127,
+	[0][1][2][1][RTW89_KCC][40] = 127,
+	[0][1][2][1][RTW89_ACMA][40] = 127,
+	[0][1][2][1][RTW89_CN][40] = 127,
+	[0][1][2][1][RTW89_UK][40] = 127,
+	[0][1][2][1][RTW89_FCC][42] = 127,
+	[0][1][2][1][RTW89_ETSI][42] = 127,
+	[0][1][2][1][RTW89_MKK][42] = 127,
+	[0][1][2][1][RTW89_IC][42] = 127,
+	[0][1][2][1][RTW89_KCC][42] = 127,
+	[0][1][2][1][RTW89_ACMA][42] = 127,
+	[0][1][2][1][RTW89_CN][42] = 127,
+	[0][1][2][1][RTW89_UK][42] = 127,
+	[0][1][2][1][RTW89_FCC][44] = 127,
+	[0][1][2][1][RTW89_ETSI][44] = 127,
+	[0][1][2][1][RTW89_MKK][44] = 127,
+	[0][1][2][1][RTW89_IC][44] = 127,
+	[0][1][2][1][RTW89_KCC][44] = 127,
+	[0][1][2][1][RTW89_ACMA][44] = 127,
+	[0][1][2][1][RTW89_CN][44] = 127,
+	[0][1][2][1][RTW89_UK][44] = 127,
+	[0][1][2][1][RTW89_FCC][46] = 127,
+	[0][1][2][1][RTW89_ETSI][46] = 127,
+	[0][1][2][1][RTW89_MKK][46] = 127,
+	[0][1][2][1][RTW89_IC][46] = 127,
+	[0][1][2][1][RTW89_KCC][46] = 127,
+	[0][1][2][1][RTW89_ACMA][46] = 127,
+	[0][1][2][1][RTW89_CN][46] = 127,
+	[0][1][2][1][RTW89_UK][46] = 127,
+	[0][1][2][1][RTW89_FCC][48] = 127,
+	[0][1][2][1][RTW89_ETSI][48] = 127,
+	[0][1][2][1][RTW89_MKK][48] = 127,
+	[0][1][2][1][RTW89_IC][48] = 127,
+	[0][1][2][1][RTW89_KCC][48] = 127,
+	[0][1][2][1][RTW89_ACMA][48] = 127,
+	[0][1][2][1][RTW89_CN][48] = 127,
+	[0][1][2][1][RTW89_UK][48] = 127,
+	[0][1][2][1][RTW89_FCC][50] = 127,
+	[0][1][2][1][RTW89_ETSI][50] = 127,
+	[0][1][2][1][RTW89_MKK][50] = 127,
+	[0][1][2][1][RTW89_IC][50] = 127,
+	[0][1][2][1][RTW89_KCC][50] = 127,
+	[0][1][2][1][RTW89_ACMA][50] = 127,
+	[0][1][2][1][RTW89_CN][50] = 127,
+	[0][1][2][1][RTW89_UK][50] = 127,
+	[0][1][2][1][RTW89_FCC][52] = 127,
+	[0][1][2][1][RTW89_ETSI][52] = 127,
+	[0][1][2][1][RTW89_MKK][52] = 127,
+	[0][1][2][1][RTW89_IC][52] = 127,
+	[0][1][2][1][RTW89_KCC][52] = 127,
+	[0][1][2][1][RTW89_ACMA][52] = 127,
+	[0][1][2][1][RTW89_CN][52] = 127,
+	[0][1][2][1][RTW89_UK][52] = 127,
+	[1][0][2][0][RTW89_FCC][1] = 66,
+	[1][0][2][0][RTW89_ETSI][1] = 64,
+	[1][0][2][0][RTW89_MKK][1] = 64,
+	[1][0][2][0][RTW89_IC][1] = 64,
+	[1][0][2][0][RTW89_KCC][1] = 74,
+	[1][0][2][0][RTW89_ACMA][1] = 64,
+	[1][0][2][0][RTW89_CN][1] = 64,
+	[1][0][2][0][RTW89_UK][1] = 64,
+	[1][0][2][0][RTW89_FCC][5] = 80,
+	[1][0][2][0][RTW89_ETSI][5] = 64,
+	[1][0][2][0][RTW89_MKK][5] = 62,
+	[1][0][2][0][RTW89_IC][5] = 64,
+	[1][0][2][0][RTW89_KCC][5] = 66,
+	[1][0][2][0][RTW89_ACMA][5] = 64,
+	[1][0][2][0][RTW89_CN][5] = 64,
+	[1][0][2][0][RTW89_UK][5] = 64,
+	[1][0][2][0][RTW89_FCC][9] = 80,
+	[1][0][2][0][RTW89_ETSI][9] = 64,
+	[1][0][2][0][RTW89_MKK][9] = 64,
+	[1][0][2][0][RTW89_IC][9] = 64,
+	[1][0][2][0][RTW89_KCC][9] = 76,
+	[1][0][2][0][RTW89_ACMA][9] = 64,
+	[1][0][2][0][RTW89_CN][9] = 64,
+	[1][0][2][0][RTW89_UK][9] = 64,
+	[1][0][2][0][RTW89_FCC][13] = 64,
+	[1][0][2][0][RTW89_ETSI][13] = 64,
+	[1][0][2][0][RTW89_MKK][13] = 64,
+	[1][0][2][0][RTW89_IC][13] = 64,
+	[1][0][2][0][RTW89_KCC][13] = 72,
+	[1][0][2][0][RTW89_ACMA][13] = 64,
+	[1][0][2][0][RTW89_CN][13] = 64,
+	[1][0][2][0][RTW89_UK][13] = 64,
+	[1][0][2][0][RTW89_FCC][16] = 66,
+	[1][0][2][0][RTW89_ETSI][16] = 66,
+	[1][0][2][0][RTW89_MKK][16] = 76,
+	[1][0][2][0][RTW89_IC][16] = 66,
+	[1][0][2][0][RTW89_KCC][16] = 74,
+	[1][0][2][0][RTW89_ACMA][16] = 66,
+	[1][0][2][0][RTW89_CN][16] = 127,
+	[1][0][2][0][RTW89_UK][16] = 66,
+	[1][0][2][0][RTW89_FCC][20] = 80,
+	[1][0][2][0][RTW89_ETSI][20] = 66,
+	[1][0][2][0][RTW89_MKK][20] = 76,
+	[1][0][2][0][RTW89_IC][20] = 80,
+	[1][0][2][0][RTW89_KCC][20] = 74,
+	[1][0][2][0][RTW89_ACMA][20] = 66,
+	[1][0][2][0][RTW89_CN][20] = 127,
+	[1][0][2][0][RTW89_UK][20] = 66,
+	[1][0][2][0][RTW89_FCC][24] = 80,
+	[1][0][2][0][RTW89_ETSI][24] = 66,
+	[1][0][2][0][RTW89_MKK][24] = 76,
+	[1][0][2][0][RTW89_IC][24] = 127,
+	[1][0][2][0][RTW89_KCC][24] = 74,
+	[1][0][2][0][RTW89_ACMA][24] = 127,
+	[1][0][2][0][RTW89_CN][24] = 127,
+	[1][0][2][0][RTW89_UK][24] = 66,
+	[1][0][2][0][RTW89_FCC][28] = 80,
+	[1][0][2][0][RTW89_ETSI][28] = 66,
+	[1][0][2][0][RTW89_MKK][28] = 76,
+	[1][0][2][0][RTW89_IC][28] = 127,
+	[1][0][2][0][RTW89_KCC][28] = 74,
+	[1][0][2][0][RTW89_ACMA][28] = 127,
+	[1][0][2][0][RTW89_CN][28] = 127,
+	[1][0][2][0][RTW89_UK][28] = 66,
+	[1][0][2][0][RTW89_FCC][32] = 74,
+	[1][0][2][0][RTW89_ETSI][32] = 66,
+	[1][0][2][0][RTW89_MKK][32] = 76,
+	[1][0][2][0][RTW89_IC][32] = 74,
+	[1][0][2][0][RTW89_KCC][32] = 76,
+	[1][0][2][0][RTW89_ACMA][32] = 66,
+	[1][0][2][0][RTW89_CN][32] = 127,
+	[1][0][2][0][RTW89_UK][32] = 66,
+	[1][0][2][0][RTW89_FCC][36] = 78,
+	[1][0][2][0][RTW89_ETSI][36] = 127,
+	[1][0][2][0][RTW89_MKK][36] = 76,
+	[1][0][2][0][RTW89_IC][36] = 78,
+	[1][0][2][0][RTW89_KCC][36] = 76,
+	[1][0][2][0][RTW89_ACMA][36] = 76,
+	[1][0][2][0][RTW89_CN][36] = 127,
+	[1][0][2][0][RTW89_UK][36] = 76,
+	[1][0][2][0][RTW89_FCC][39] = 80,
+	[1][0][2][0][RTW89_ETSI][39] = 30,
+	[1][0][2][0][RTW89_MKK][39] = 127,
+	[1][0][2][0][RTW89_IC][39] = 80,
+	[1][0][2][0][RTW89_KCC][39] = 68,
+	[1][0][2][0][RTW89_ACMA][39] = 76,
+	[1][0][2][0][RTW89_CN][39] = 70,
+	[1][0][2][0][RTW89_UK][39] = 64,
+	[1][0][2][0][RTW89_FCC][43] = 80,
+	[1][0][2][0][RTW89_ETSI][43] = 30,
+	[1][0][2][0][RTW89_MKK][43] = 127,
+	[1][0][2][0][RTW89_IC][43] = 80,
+	[1][0][2][0][RTW89_KCC][43] = 76,
+	[1][0][2][0][RTW89_ACMA][43] = 76,
+	[1][0][2][0][RTW89_CN][43] = 76,
+	[1][0][2][0][RTW89_UK][43] = 64,
+	[1][0][2][0][RTW89_FCC][47] = 80,
+	[1][0][2][0][RTW89_ETSI][47] = 127,
+	[1][0][2][0][RTW89_MKK][47] = 127,
+	[1][0][2][0][RTW89_IC][47] = 127,
+	[1][0][2][0][RTW89_KCC][47] = 127,
+	[1][0][2][0][RTW89_ACMA][47] = 127,
+	[1][0][2][0][RTW89_CN][47] = 127,
+	[1][0][2][0][RTW89_UK][47] = 127,
+	[1][0][2][0][RTW89_FCC][51] = 80,
+	[1][0][2][0][RTW89_ETSI][51] = 127,
+	[1][0][2][0][RTW89_MKK][51] = 127,
+	[1][0][2][0][RTW89_IC][51] = 127,
+	[1][0][2][0][RTW89_KCC][51] = 127,
+	[1][0][2][0][RTW89_ACMA][51] = 127,
+	[1][0][2][0][RTW89_CN][51] = 127,
+	[1][0][2][0][RTW89_UK][51] = 127,
+	[1][1][2][0][RTW89_FCC][1] = 127,
+	[1][1][2][0][RTW89_ETSI][1] = 127,
+	[1][1][2][0][RTW89_MKK][1] = 127,
+	[1][1][2][0][RTW89_IC][1] = 127,
+	[1][1][2][0][RTW89_KCC][1] = 127,
+	[1][1][2][0][RTW89_ACMA][1] = 127,
+	[1][1][2][0][RTW89_CN][1] = 127,
+	[1][1][2][0][RTW89_UK][1] = 127,
+	[1][1][2][0][RTW89_FCC][5] = 127,
+	[1][1][2][0][RTW89_ETSI][5] = 127,
+	[1][1][2][0][RTW89_MKK][5] = 127,
+	[1][1][2][0][RTW89_IC][5] = 127,
+	[1][1][2][0][RTW89_KCC][5] = 127,
+	[1][1][2][0][RTW89_ACMA][5] = 127,
+	[1][1][2][0][RTW89_CN][5] = 127,
+	[1][1][2][0][RTW89_UK][5] = 127,
+	[1][1][2][0][RTW89_FCC][9] = 127,
+	[1][1][2][0][RTW89_ETSI][9] = 127,
+	[1][1][2][0][RTW89_MKK][9] = 127,
+	[1][1][2][0][RTW89_IC][9] = 127,
+	[1][1][2][0][RTW89_KCC][9] = 127,
+	[1][1][2][0][RTW89_ACMA][9] = 127,
+	[1][1][2][0][RTW89_CN][9] = 127,
+	[1][1][2][0][RTW89_UK][9] = 127,
+	[1][1][2][0][RTW89_FCC][13] = 127,
+	[1][1][2][0][RTW89_ETSI][13] = 127,
+	[1][1][2][0][RTW89_MKK][13] = 127,
+	[1][1][2][0][RTW89_IC][13] = 127,
+	[1][1][2][0][RTW89_KCC][13] = 127,
+	[1][1][2][0][RTW89_ACMA][13] = 127,
+	[1][1][2][0][RTW89_CN][13] = 127,
+	[1][1][2][0][RTW89_UK][13] = 127,
+	[1][1][2][0][RTW89_FCC][16] = 127,
+	[1][1][2][0][RTW89_ETSI][16] = 127,
+	[1][1][2][0][RTW89_MKK][16] = 127,
+	[1][1][2][0][RTW89_IC][16] = 127,
+	[1][1][2][0][RTW89_KCC][16] = 127,
+	[1][1][2][0][RTW89_ACMA][16] = 127,
+	[1][1][2][0][RTW89_CN][16] = 127,
+	[1][1][2][0][RTW89_UK][16] = 127,
+	[1][1][2][0][RTW89_FCC][20] = 127,
+	[1][1][2][0][RTW89_ETSI][20] = 127,
+	[1][1][2][0][RTW89_MKK][20] = 127,
+	[1][1][2][0][RTW89_IC][20] = 127,
+	[1][1][2][0][RTW89_KCC][20] = 127,
+	[1][1][2][0][RTW89_ACMA][20] = 127,
+	[1][1][2][0][RTW89_CN][20] = 127,
+	[1][1][2][0][RTW89_UK][20] = 127,
+	[1][1][2][0][RTW89_FCC][24] = 127,
+	[1][1][2][0][RTW89_ETSI][24] = 127,
+	[1][1][2][0][RTW89_MKK][24] = 127,
+	[1][1][2][0][RTW89_IC][24] = 127,
+	[1][1][2][0][RTW89_KCC][24] = 127,
+	[1][1][2][0][RTW89_ACMA][24] = 127,
+	[1][1][2][0][RTW89_CN][24] = 127,
+	[1][1][2][0][RTW89_UK][24] = 127,
+	[1][1][2][0][RTW89_FCC][28] = 127,
+	[1][1][2][0][RTW89_ETSI][28] = 127,
+	[1][1][2][0][RTW89_MKK][28] = 127,
+	[1][1][2][0][RTW89_IC][28] = 127,
+	[1][1][2][0][RTW89_KCC][28] = 127,
+	[1][1][2][0][RTW89_ACMA][28] = 127,
+	[1][1][2][0][RTW89_CN][28] = 127,
+	[1][1][2][0][RTW89_UK][28] = 127,
+	[1][1][2][0][RTW89_FCC][32] = 127,
+	[1][1][2][0][RTW89_ETSI][32] = 127,
+	[1][1][2][0][RTW89_MKK][32] = 127,
+	[1][1][2][0][RTW89_IC][32] = 127,
+	[1][1][2][0][RTW89_KCC][32] = 127,
+	[1][1][2][0][RTW89_ACMA][32] = 127,
+	[1][1][2][0][RTW89_CN][32] = 127,
+	[1][1][2][0][RTW89_UK][32] = 127,
+	[1][1][2][0][RTW89_FCC][36] = 127,
+	[1][1][2][0][RTW89_ETSI][36] = 127,
+	[1][1][2][0][RTW89_MKK][36] = 127,
+	[1][1][2][0][RTW89_IC][36] = 127,
+	[1][1][2][0][RTW89_KCC][36] = 127,
+	[1][1][2][0][RTW89_ACMA][36] = 127,
+	[1][1][2][0][RTW89_CN][36] = 127,
+	[1][1][2][0][RTW89_UK][36] = 127,
+	[1][1][2][0][RTW89_FCC][39] = 127,
+	[1][1][2][0][RTW89_ETSI][39] = 127,
+	[1][1][2][0][RTW89_MKK][39] = 127,
+	[1][1][2][0][RTW89_IC][39] = 127,
+	[1][1][2][0][RTW89_KCC][39] = 127,
+	[1][1][2][0][RTW89_ACMA][39] = 127,
+	[1][1][2][0][RTW89_CN][39] = 127,
+	[1][1][2][0][RTW89_UK][39] = 127,
+	[1][1][2][0][RTW89_FCC][43] = 127,
+	[1][1][2][0][RTW89_ETSI][43] = 127,
+	[1][1][2][0][RTW89_MKK][43] = 127,
+	[1][1][2][0][RTW89_IC][43] = 127,
+	[1][1][2][0][RTW89_KCC][43] = 127,
+	[1][1][2][0][RTW89_ACMA][43] = 127,
+	[1][1][2][0][RTW89_CN][43] = 127,
+	[1][1][2][0][RTW89_UK][43] = 127,
+	[1][1][2][0][RTW89_FCC][47] = 127,
+	[1][1][2][0][RTW89_ETSI][47] = 127,
+	[1][1][2][0][RTW89_MKK][47] = 127,
+	[1][1][2][0][RTW89_IC][47] = 127,
+	[1][1][2][0][RTW89_KCC][47] = 127,
+	[1][1][2][0][RTW89_ACMA][47] = 127,
+	[1][1][2][0][RTW89_CN][47] = 127,
+	[1][1][2][0][RTW89_UK][47] = 127,
+	[1][1][2][0][RTW89_FCC][51] = 127,
+	[1][1][2][0][RTW89_ETSI][51] = 127,
+	[1][1][2][0][RTW89_MKK][51] = 127,
+	[1][1][2][0][RTW89_IC][51] = 127,
+	[1][1][2][0][RTW89_KCC][51] = 127,
+	[1][1][2][0][RTW89_ACMA][51] = 127,
+	[1][1][2][0][RTW89_CN][51] = 127,
+	[1][1][2][0][RTW89_UK][51] = 127,
+	[1][1][2][1][RTW89_FCC][1] = 127,
+	[1][1][2][1][RTW89_ETSI][1] = 127,
+	[1][1][2][1][RTW89_MKK][1] = 127,
+	[1][1][2][1][RTW89_IC][1] = 127,
+	[1][1][2][1][RTW89_KCC][1] = 127,
+	[1][1][2][1][RTW89_ACMA][1] = 127,
+	[1][1][2][1][RTW89_CN][1] = 127,
+	[1][1][2][1][RTW89_UK][1] = 127,
+	[1][1][2][1][RTW89_FCC][5] = 127,
+	[1][1][2][1][RTW89_ETSI][5] = 127,
+	[1][1][2][1][RTW89_MKK][5] = 127,
+	[1][1][2][1][RTW89_IC][5] = 127,
+	[1][1][2][1][RTW89_KCC][5] = 127,
+	[1][1][2][1][RTW89_ACMA][5] = 127,
+	[1][1][2][1][RTW89_CN][5] = 127,
+	[1][1][2][1][RTW89_UK][5] = 127,
+	[1][1][2][1][RTW89_FCC][9] = 127,
+	[1][1][2][1][RTW89_ETSI][9] = 127,
+	[1][1][2][1][RTW89_MKK][9] = 127,
+	[1][1][2][1][RTW89_IC][9] = 127,
+	[1][1][2][1][RTW89_KCC][9] = 127,
+	[1][1][2][1][RTW89_ACMA][9] = 127,
+	[1][1][2][1][RTW89_CN][9] = 127,
+	[1][1][2][1][RTW89_UK][9] = 127,
+	[1][1][2][1][RTW89_FCC][13] = 127,
+	[1][1][2][1][RTW89_ETSI][13] = 127,
+	[1][1][2][1][RTW89_MKK][13] = 127,
+	[1][1][2][1][RTW89_IC][13] = 127,
+	[1][1][2][1][RTW89_KCC][13] = 127,
+	[1][1][2][1][RTW89_ACMA][13] = 127,
+	[1][1][2][1][RTW89_CN][13] = 127,
+	[1][1][2][1][RTW89_UK][13] = 127,
+	[1][1][2][1][RTW89_FCC][16] = 127,
+	[1][1][2][1][RTW89_ETSI][16] = 127,
+	[1][1][2][1][RTW89_MKK][16] = 127,
+	[1][1][2][1][RTW89_IC][16] = 127,
+	[1][1][2][1][RTW89_KCC][16] = 127,
+	[1][1][2][1][RTW89_ACMA][16] = 127,
+	[1][1][2][1][RTW89_CN][16] = 127,
+	[1][1][2][1][RTW89_UK][16] = 127,
+	[1][1][2][1][RTW89_FCC][20] = 127,
+	[1][1][2][1][RTW89_ETSI][20] = 127,
+	[1][1][2][1][RTW89_MKK][20] = 127,
+	[1][1][2][1][RTW89_IC][20] = 127,
+	[1][1][2][1][RTW89_KCC][20] = 127,
+	[1][1][2][1][RTW89_ACMA][20] = 127,
+	[1][1][2][1][RTW89_CN][20] = 127,
+	[1][1][2][1][RTW89_UK][20] = 127,
+	[1][1][2][1][RTW89_FCC][24] = 127,
+	[1][1][2][1][RTW89_ETSI][24] = 127,
+	[1][1][2][1][RTW89_MKK][24] = 127,
+	[1][1][2][1][RTW89_IC][24] = 127,
+	[1][1][2][1][RTW89_KCC][24] = 127,
+	[1][1][2][1][RTW89_ACMA][24] = 127,
+	[1][1][2][1][RTW89_CN][24] = 127,
+	[1][1][2][1][RTW89_UK][24] = 127,
+	[1][1][2][1][RTW89_FCC][28] = 127,
+	[1][1][2][1][RTW89_ETSI][28] = 127,
+	[1][1][2][1][RTW89_MKK][28] = 127,
+	[1][1][2][1][RTW89_IC][28] = 127,
+	[1][1][2][1][RTW89_KCC][28] = 127,
+	[1][1][2][1][RTW89_ACMA][28] = 127,
+	[1][1][2][1][RTW89_CN][28] = 127,
+	[1][1][2][1][RTW89_UK][28] = 127,
+	[1][1][2][1][RTW89_FCC][32] = 127,
+	[1][1][2][1][RTW89_ETSI][32] = 127,
+	[1][1][2][1][RTW89_MKK][32] = 127,
+	[1][1][2][1][RTW89_IC][32] = 127,
+	[1][1][2][1][RTW89_KCC][32] = 127,
+	[1][1][2][1][RTW89_ACMA][32] = 127,
+	[1][1][2][1][RTW89_CN][32] = 127,
+	[1][1][2][1][RTW89_UK][32] = 127,
+	[1][1][2][1][RTW89_FCC][36] = 127,
+	[1][1][2][1][RTW89_ETSI][36] = 127,
+	[1][1][2][1][RTW89_MKK][36] = 127,
+	[1][1][2][1][RTW89_IC][36] = 127,
+	[1][1][2][1][RTW89_KCC][36] = 127,
+	[1][1][2][1][RTW89_ACMA][36] = 127,
+	[1][1][2][1][RTW89_CN][36] = 127,
+	[1][1][2][1][RTW89_UK][36] = 127,
+	[1][1][2][1][RTW89_FCC][39] = 127,
+	[1][1][2][1][RTW89_ETSI][39] = 127,
+	[1][1][2][1][RTW89_MKK][39] = 127,
+	[1][1][2][1][RTW89_IC][39] = 127,
+	[1][1][2][1][RTW89_KCC][39] = 127,
+	[1][1][2][1][RTW89_ACMA][39] = 127,
+	[1][1][2][1][RTW89_CN][39] = 127,
+	[1][1][2][1][RTW89_UK][39] = 127,
+	[1][1][2][1][RTW89_FCC][43] = 127,
+	[1][1][2][1][RTW89_ETSI][43] = 127,
+	[1][1][2][1][RTW89_MKK][43] = 127,
+	[1][1][2][1][RTW89_IC][43] = 127,
+	[1][1][2][1][RTW89_KCC][43] = 127,
+	[1][1][2][1][RTW89_ACMA][43] = 127,
+	[1][1][2][1][RTW89_CN][43] = 127,
+	[1][1][2][1][RTW89_UK][43] = 127,
+	[1][1][2][1][RTW89_FCC][47] = 127,
+	[1][1][2][1][RTW89_ETSI][47] = 127,
+	[1][1][2][1][RTW89_MKK][47] = 127,
+	[1][1][2][1][RTW89_IC][47] = 127,
+	[1][1][2][1][RTW89_KCC][47] = 127,
+	[1][1][2][1][RTW89_ACMA][47] = 127,
+	[1][1][2][1][RTW89_CN][47] = 127,
+	[1][1][2][1][RTW89_UK][47] = 127,
+	[1][1][2][1][RTW89_FCC][51] = 127,
+	[1][1][2][1][RTW89_ETSI][51] = 127,
+	[1][1][2][1][RTW89_MKK][51] = 127,
+	[1][1][2][1][RTW89_IC][51] = 127,
+	[1][1][2][1][RTW89_KCC][51] = 127,
+	[1][1][2][1][RTW89_ACMA][51] = 127,
+	[1][1][2][1][RTW89_CN][51] = 127,
+	[1][1][2][1][RTW89_UK][51] = 127,
+	[2][0][2][0][RTW89_FCC][3] = 72,
+	[2][0][2][0][RTW89_ETSI][3] = 64,
+	[2][0][2][0][RTW89_MKK][3] = 62,
+	[2][0][2][0][RTW89_IC][3] = 64,
+	[2][0][2][0][RTW89_KCC][3] = 68,
+	[2][0][2][0][RTW89_ACMA][3] = 64,
+	[2][0][2][0][RTW89_CN][3] = 64,
+	[2][0][2][0][RTW89_UK][3] = 64,
+	[2][0][2][0][RTW89_FCC][11] = 62,
+	[2][0][2][0][RTW89_ETSI][11] = 64,
+	[2][0][2][0][RTW89_MKK][11] = 64,
+	[2][0][2][0][RTW89_IC][11] = 62,
+	[2][0][2][0][RTW89_KCC][11] = 68,
+	[2][0][2][0][RTW89_ACMA][11] = 64,
+	[2][0][2][0][RTW89_CN][11] = 64,
+	[2][0][2][0][RTW89_UK][11] = 64,
+	[2][0][2][0][RTW89_FCC][18] = 66,
+	[2][0][2][0][RTW89_ETSI][18] = 64,
+	[2][0][2][0][RTW89_MKK][18] = 68,
+	[2][0][2][0][RTW89_IC][18] = 66,
+	[2][0][2][0][RTW89_KCC][18] = 68,
+	[2][0][2][0][RTW89_ACMA][18] = 64,
+	[2][0][2][0][RTW89_CN][18] = 127,
+	[2][0][2][0][RTW89_UK][18] = 64,
+	[2][0][2][0][RTW89_FCC][26] = 72,
+	[2][0][2][0][RTW89_ETSI][26] = 64,
+	[2][0][2][0][RTW89_MKK][26] = 68,
+	[2][0][2][0][RTW89_IC][26] = 127,
+	[2][0][2][0][RTW89_KCC][26] = 68,
+	[2][0][2][0][RTW89_ACMA][26] = 127,
+	[2][0][2][0][RTW89_CN][26] = 127,
+	[2][0][2][0][RTW89_UK][26] = 64,
+	[2][0][2][0][RTW89_FCC][34] = 72,
+	[2][0][2][0][RTW89_ETSI][34] = 127,
+	[2][0][2][0][RTW89_MKK][34] = 68,
+	[2][0][2][0][RTW89_IC][34] = 72,
+	[2][0][2][0][RTW89_KCC][34] = 68,
+	[2][0][2][0][RTW89_ACMA][34] = 68,
+	[2][0][2][0][RTW89_CN][34] = 127,
+	[2][0][2][0][RTW89_UK][34] = 68,
+	[2][0][2][0][RTW89_FCC][41] = 72,
+	[2][0][2][0][RTW89_ETSI][41] = 30,
+	[2][0][2][0][RTW89_MKK][41] = 127,
+	[2][0][2][0][RTW89_IC][41] = 72,
+	[2][0][2][0][RTW89_KCC][41] = 64,
+	[2][0][2][0][RTW89_ACMA][41] = 68,
+	[2][0][2][0][RTW89_CN][41] = 68,
+	[2][0][2][0][RTW89_UK][41] = 64,
+	[2][0][2][0][RTW89_FCC][49] = 72,
+	[2][0][2][0][RTW89_ETSI][49] = 127,
+	[2][0][2][0][RTW89_MKK][49] = 127,
+	[2][0][2][0][RTW89_IC][49] = 127,
+	[2][0][2][0][RTW89_KCC][49] = 127,
+	[2][0][2][0][RTW89_ACMA][49] = 127,
+	[2][0][2][0][RTW89_CN][49] = 127,
+	[2][0][2][0][RTW89_UK][49] = 127,
+	[2][1][2][0][RTW89_FCC][3] = 127,
+	[2][1][2][0][RTW89_ETSI][3] = 127,
+	[2][1][2][0][RTW89_MKK][3] = 127,
+	[2][1][2][0][RTW89_IC][3] = 127,
+	[2][1][2][0][RTW89_KCC][3] = 127,
+	[2][1][2][0][RTW89_ACMA][3] = 127,
+	[2][1][2][0][RTW89_CN][3] = 127,
+	[2][1][2][0][RTW89_UK][3] = 127,
+	[2][1][2][0][RTW89_FCC][11] = 127,
+	[2][1][2][0][RTW89_ETSI][11] = 127,
+	[2][1][2][0][RTW89_MKK][11] = 127,
+	[2][1][2][0][RTW89_IC][11] = 127,
+	[2][1][2][0][RTW89_KCC][11] = 127,
+	[2][1][2][0][RTW89_ACMA][11] = 127,
+	[2][1][2][0][RTW89_CN][11] = 127,
+	[2][1][2][0][RTW89_UK][11] = 127,
+	[2][1][2][0][RTW89_FCC][18] = 127,
+	[2][1][2][0][RTW89_ETSI][18] = 127,
+	[2][1][2][0][RTW89_MKK][18] = 127,
+	[2][1][2][0][RTW89_IC][18] = 127,
+	[2][1][2][0][RTW89_KCC][18] = 127,
+	[2][1][2][0][RTW89_ACMA][18] = 127,
+	[2][1][2][0][RTW89_CN][18] = 127,
+	[2][1][2][0][RTW89_UK][18] = 127,
+	[2][1][2][0][RTW89_FCC][26] = 127,
+	[2][1][2][0][RTW89_ETSI][26] = 127,
+	[2][1][2][0][RTW89_MKK][26] = 127,
+	[2][1][2][0][RTW89_IC][26] = 127,
+	[2][1][2][0][RTW89_KCC][26] = 127,
+	[2][1][2][0][RTW89_ACMA][26] = 127,
+	[2][1][2][0][RTW89_CN][26] = 127,
+	[2][1][2][0][RTW89_UK][26] = 127,
+	[2][1][2][0][RTW89_FCC][34] = 127,
+	[2][1][2][0][RTW89_ETSI][34] = 127,
+	[2][1][2][0][RTW89_MKK][34] = 127,
+	[2][1][2][0][RTW89_IC][34] = 127,
+	[2][1][2][0][RTW89_KCC][34] = 127,
+	[2][1][2][0][RTW89_ACMA][34] = 127,
+	[2][1][2][0][RTW89_CN][34] = 127,
+	[2][1][2][0][RTW89_UK][34] = 127,
+	[2][1][2][0][RTW89_FCC][41] = 127,
+	[2][1][2][0][RTW89_ETSI][41] = 127,
+	[2][1][2][0][RTW89_MKK][41] = 127,
+	[2][1][2][0][RTW89_IC][41] = 127,
+	[2][1][2][0][RTW89_KCC][41] = 127,
+	[2][1][2][0][RTW89_ACMA][41] = 127,
+	[2][1][2][0][RTW89_CN][41] = 127,
+	[2][1][2][0][RTW89_UK][41] = 127,
+	[2][1][2][0][RTW89_FCC][49] = 127,
+	[2][1][2][0][RTW89_ETSI][49] = 127,
+	[2][1][2][0][RTW89_MKK][49] = 127,
+	[2][1][2][0][RTW89_IC][49] = 127,
+	[2][1][2][0][RTW89_KCC][49] = 127,
+	[2][1][2][0][RTW89_ACMA][49] = 127,
+	[2][1][2][0][RTW89_CN][49] = 127,
+	[2][1][2][0][RTW89_UK][49] = 127,
+	[2][1][2][1][RTW89_FCC][3] = 127,
+	[2][1][2][1][RTW89_ETSI][3] = 127,
+	[2][1][2][1][RTW89_MKK][3] = 127,
+	[2][1][2][1][RTW89_IC][3] = 127,
+	[2][1][2][1][RTW89_KCC][3] = 127,
+	[2][1][2][1][RTW89_ACMA][3] = 127,
+	[2][1][2][1][RTW89_CN][3] = 127,
+	[2][1][2][1][RTW89_UK][3] = 127,
+	[2][1][2][1][RTW89_FCC][11] = 127,
+	[2][1][2][1][RTW89_ETSI][11] = 127,
+	[2][1][2][1][RTW89_MKK][11] = 127,
+	[2][1][2][1][RTW89_IC][11] = 127,
+	[2][1][2][1][RTW89_KCC][11] = 127,
+	[2][1][2][1][RTW89_ACMA][11] = 127,
+	[2][1][2][1][RTW89_CN][11] = 127,
+	[2][1][2][1][RTW89_UK][11] = 127,
+	[2][1][2][1][RTW89_FCC][18] = 127,
+	[2][1][2][1][RTW89_ETSI][18] = 127,
+	[2][1][2][1][RTW89_MKK][18] = 127,
+	[2][1][2][1][RTW89_IC][18] = 127,
+	[2][1][2][1][RTW89_KCC][18] = 127,
+	[2][1][2][1][RTW89_ACMA][18] = 127,
+	[2][1][2][1][RTW89_CN][18] = 127,
+	[2][1][2][1][RTW89_UK][18] = 127,
+	[2][1][2][1][RTW89_FCC][26] = 127,
+	[2][1][2][1][RTW89_ETSI][26] = 127,
+	[2][1][2][1][RTW89_MKK][26] = 127,
+	[2][1][2][1][RTW89_IC][26] = 127,
+	[2][1][2][1][RTW89_KCC][26] = 127,
+	[2][1][2][1][RTW89_ACMA][26] = 127,
+	[2][1][2][1][RTW89_CN][26] = 127,
+	[2][1][2][1][RTW89_UK][26] = 127,
+	[2][1][2][1][RTW89_FCC][34] = 127,
+	[2][1][2][1][RTW89_ETSI][34] = 127,
+	[2][1][2][1][RTW89_MKK][34] = 127,
+	[2][1][2][1][RTW89_IC][34] = 127,
+	[2][1][2][1][RTW89_KCC][34] = 127,
+	[2][1][2][1][RTW89_ACMA][34] = 127,
+	[2][1][2][1][RTW89_CN][34] = 127,
+	[2][1][2][1][RTW89_UK][34] = 127,
+	[2][1][2][1][RTW89_FCC][41] = 127,
+	[2][1][2][1][RTW89_ETSI][41] = 127,
+	[2][1][2][1][RTW89_MKK][41] = 127,
+	[2][1][2][1][RTW89_IC][41] = 127,
+	[2][1][2][1][RTW89_KCC][41] = 127,
+	[2][1][2][1][RTW89_ACMA][41] = 127,
+	[2][1][2][1][RTW89_CN][41] = 127,
+	[2][1][2][1][RTW89_UK][41] = 127,
+	[2][1][2][1][RTW89_FCC][49] = 127,
+	[2][1][2][1][RTW89_ETSI][49] = 127,
+	[2][1][2][1][RTW89_MKK][49] = 127,
+	[2][1][2][1][RTW89_IC][49] = 127,
+	[2][1][2][1][RTW89_KCC][49] = 127,
+	[2][1][2][1][RTW89_ACMA][49] = 127,
+	[2][1][2][1][RTW89_CN][49] = 127,
+	[2][1][2][1][RTW89_UK][49] = 127,
+	[3][0][2][0][RTW89_FCC][7] = 127,
+	[3][0][2][0][RTW89_ETSI][7] = 127,
+	[3][0][2][0][RTW89_MKK][7] = 127,
+	[3][0][2][0][RTW89_IC][7] = 127,
+	[3][0][2][0][RTW89_KCC][7] = 127,
+	[3][0][2][0][RTW89_ACMA][7] = 127,
+	[3][0][2][0][RTW89_CN][7] = 58,
+	[3][0][2][0][RTW89_UK][7] = 127,
+	[3][0][2][0][RTW89_FCC][22] = 127,
+	[3][0][2][0][RTW89_ETSI][22] = 127,
+	[3][0][2][0][RTW89_MKK][22] = 127,
+	[3][0][2][0][RTW89_IC][22] = 127,
+	[3][0][2][0][RTW89_KCC][22] = 127,
+	[3][0][2][0][RTW89_ACMA][22] = 127,
+	[3][0][2][0][RTW89_CN][22] = 58,
+	[3][0][2][0][RTW89_UK][22] = 127,
+	[3][0][2][0][RTW89_FCC][45] = 127,
+	[3][0][2][0][RTW89_ETSI][45] = 127,
+	[3][0][2][0][RTW89_MKK][45] = 127,
+	[3][0][2][0][RTW89_IC][45] = 127,
+	[3][0][2][0][RTW89_KCC][45] = 127,
+	[3][0][2][0][RTW89_ACMA][45] = 127,
+	[3][0][2][0][RTW89_CN][45] = 127,
+	[3][0][2][0][RTW89_UK][45] = 127,
+	[3][1][2][0][RTW89_FCC][7] = 127,
+	[3][1][2][0][RTW89_ETSI][7] = 127,
+	[3][1][2][0][RTW89_MKK][7] = 127,
+	[3][1][2][0][RTW89_IC][7] = 127,
+	[3][1][2][0][RTW89_KCC][7] = 127,
+	[3][1][2][0][RTW89_ACMA][7] = 127,
+	[3][1][2][0][RTW89_CN][7] = 127,
+	[3][1][2][0][RTW89_UK][7] = 127,
+	[3][1][2][0][RTW89_FCC][22] = 127,
+	[3][1][2][0][RTW89_ETSI][22] = 127,
+	[3][1][2][0][RTW89_MKK][22] = 127,
+	[3][1][2][0][RTW89_IC][22] = 127,
+	[3][1][2][0][RTW89_KCC][22] = 127,
+	[3][1][2][0][RTW89_ACMA][22] = 127,
+	[3][1][2][0][RTW89_CN][22] = 127,
+	[3][1][2][0][RTW89_UK][22] = 127,
+	[3][1][2][0][RTW89_FCC][45] = 127,
+	[3][1][2][0][RTW89_ETSI][45] = 127,
+	[3][1][2][0][RTW89_MKK][45] = 127,
+	[3][1][2][0][RTW89_IC][45] = 127,
+	[3][1][2][0][RTW89_KCC][45] = 127,
+	[3][1][2][0][RTW89_ACMA][45] = 127,
+	[3][1][2][0][RTW89_CN][45] = 127,
+	[3][1][2][0][RTW89_UK][45] = 127,
+	[3][1][2][1][RTW89_FCC][7] = 127,
+	[3][1][2][1][RTW89_ETSI][7] = 127,
+	[3][1][2][1][RTW89_MKK][7] = 127,
+	[3][1][2][1][RTW89_IC][7] = 127,
+	[3][1][2][1][RTW89_KCC][7] = 127,
+	[3][1][2][1][RTW89_ACMA][7] = 127,
+	[3][1][2][1][RTW89_CN][7] = 127,
+	[3][1][2][1][RTW89_UK][7] = 127,
+	[3][1][2][1][RTW89_FCC][22] = 127,
+	[3][1][2][1][RTW89_ETSI][22] = 127,
+	[3][1][2][1][RTW89_MKK][22] = 127,
+	[3][1][2][1][RTW89_IC][22] = 127,
+	[3][1][2][1][RTW89_KCC][22] = 127,
+	[3][1][2][1][RTW89_ACMA][22] = 127,
+	[3][1][2][1][RTW89_CN][22] = 127,
+	[3][1][2][1][RTW89_UK][22] = 127,
+	[3][1][2][1][RTW89_FCC][45] = 127,
+	[3][1][2][1][RTW89_ETSI][45] = 127,
+	[3][1][2][1][RTW89_MKK][45] = 127,
+	[3][1][2][1][RTW89_IC][45] = 127,
+	[3][1][2][1][RTW89_KCC][45] = 127,
+	[3][1][2][1][RTW89_ACMA][45] = 127,
+	[3][1][2][1][RTW89_CN][45] = 127,
+	[3][1][2][1][RTW89_UK][45] = 127,
+};
+
+static
+const s8 rtw89_8851b_txpwr_lmt_ru_2g_type2[RTW89_RU_NUM][RTW89_NTX_NUM]
+					  [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
+	[0][0][RTW89_WW][0] = 30,
+	[0][0][RTW89_WW][1] = 30,
+	[0][0][RTW89_WW][2] = 30,
+	[0][0][RTW89_WW][3] = 30,
+	[0][0][RTW89_WW][4] = 30,
+	[0][0][RTW89_WW][5] = 30,
+	[0][0][RTW89_WW][6] = 30,
+	[0][0][RTW89_WW][7] = 30,
+	[0][0][RTW89_WW][8] = 30,
+	[0][0][RTW89_WW][9] = 30,
+	[0][0][RTW89_WW][10] = 30,
+	[0][0][RTW89_WW][11] = 30,
+	[0][0][RTW89_WW][12] = 30,
+	[0][0][RTW89_WW][13] = 0,
+	[0][1][RTW89_WW][0] = 20,
+	[0][1][RTW89_WW][1] = 22,
+	[0][1][RTW89_WW][2] = 22,
+	[0][1][RTW89_WW][3] = 22,
+	[0][1][RTW89_WW][4] = 22,
+	[0][1][RTW89_WW][5] = 22,
+	[0][1][RTW89_WW][6] = 22,
+	[0][1][RTW89_WW][7] = 22,
+	[0][1][RTW89_WW][8] = 22,
+	[0][1][RTW89_WW][9] = 22,
+	[0][1][RTW89_WW][10] = 22,
+	[0][1][RTW89_WW][11] = 22,
+	[0][1][RTW89_WW][12] = 20,
+	[0][1][RTW89_WW][13] = 0,
+	[1][0][RTW89_WW][0] = 42,
+	[1][0][RTW89_WW][1] = 42,
+	[1][0][RTW89_WW][2] = 42,
+	[1][0][RTW89_WW][3] = 42,
+	[1][0][RTW89_WW][4] = 42,
+	[1][0][RTW89_WW][5] = 42,
+	[1][0][RTW89_WW][6] = 42,
+	[1][0][RTW89_WW][7] = 42,
+	[1][0][RTW89_WW][8] = 42,
+	[1][0][RTW89_WW][9] = 42,
+	[1][0][RTW89_WW][10] = 42,
+	[1][0][RTW89_WW][11] = 42,
+	[1][0][RTW89_WW][12] = 34,
+	[1][0][RTW89_WW][13] = 0,
+	[1][1][RTW89_WW][0] = 32,
+	[1][1][RTW89_WW][1] = 32,
+	[1][1][RTW89_WW][2] = 32,
+	[1][1][RTW89_WW][3] = 32,
+	[1][1][RTW89_WW][4] = 32,
+	[1][1][RTW89_WW][5] = 32,
+	[1][1][RTW89_WW][6] = 32,
+	[1][1][RTW89_WW][7] = 32,
+	[1][1][RTW89_WW][8] = 32,
+	[1][1][RTW89_WW][9] = 32,
+	[1][1][RTW89_WW][10] = 32,
+	[1][1][RTW89_WW][11] = 32,
+	[1][1][RTW89_WW][12] = 32,
+	[1][1][RTW89_WW][13] = 0,
+	[2][0][RTW89_WW][0] = 54,
+	[2][0][RTW89_WW][1] = 54,
+	[2][0][RTW89_WW][2] = 54,
+	[2][0][RTW89_WW][3] = 54,
+	[2][0][RTW89_WW][4] = 54,
+	[2][0][RTW89_WW][5] = 54,
+	[2][0][RTW89_WW][6] = 54,
+	[2][0][RTW89_WW][7] = 54,
+	[2][0][RTW89_WW][8] = 54,
+	[2][0][RTW89_WW][9] = 54,
+	[2][0][RTW89_WW][10] = 54,
+	[2][0][RTW89_WW][11] = 54,
+	[2][0][RTW89_WW][12] = 34,
+	[2][0][RTW89_WW][13] = 0,
+	[2][1][RTW89_WW][0] = 44,
+	[2][1][RTW89_WW][1] = 44,
+	[2][1][RTW89_WW][2] = 44,
+	[2][1][RTW89_WW][3] = 44,
+	[2][1][RTW89_WW][4] = 44,
+	[2][1][RTW89_WW][5] = 44,
+	[2][1][RTW89_WW][6] = 44,
+	[2][1][RTW89_WW][7] = 44,
+	[2][1][RTW89_WW][8] = 44,
+	[2][1][RTW89_WW][9] = 44,
+	[2][1][RTW89_WW][10] = 44,
+	[2][1][RTW89_WW][11] = 44,
+	[2][1][RTW89_WW][12] = 42,
+	[2][1][RTW89_WW][13] = 0,
+	[0][0][RTW89_FCC][0] = 60,
+	[0][0][RTW89_ETSI][0] = 30,
+	[0][0][RTW89_MKK][0] = 40,
+	[0][0][RTW89_IC][0] = 60,
+	[0][0][RTW89_KCC][0] = 46,
+	[0][0][RTW89_ACMA][0] = 30,
+	[0][0][RTW89_CN][0] = 32,
+	[0][0][RTW89_UK][0] = 30,
+	[0][0][RTW89_FCC][1] = 60,
+	[0][0][RTW89_ETSI][1] = 30,
+	[0][0][RTW89_MKK][1] = 44,
+	[0][0][RTW89_IC][1] = 60,
+	[0][0][RTW89_KCC][1] = 46,
+	[0][0][RTW89_ACMA][1] = 30,
+	[0][0][RTW89_CN][1] = 32,
+	[0][0][RTW89_UK][1] = 30,
+	[0][0][RTW89_FCC][2] = 64,
+	[0][0][RTW89_ETSI][2] = 30,
+	[0][0][RTW89_MKK][2] = 44,
+	[0][0][RTW89_IC][2] = 64,
+	[0][0][RTW89_KCC][2] = 46,
+	[0][0][RTW89_ACMA][2] = 30,
+	[0][0][RTW89_CN][2] = 32,
+	[0][0][RTW89_UK][2] = 30,
+	[0][0][RTW89_FCC][3] = 68,
+	[0][0][RTW89_ETSI][3] = 30,
+	[0][0][RTW89_MKK][3] = 44,
+	[0][0][RTW89_IC][3] = 68,
+	[0][0][RTW89_KCC][3] = 46,
+	[0][0][RTW89_ACMA][3] = 30,
+	[0][0][RTW89_CN][3] = 32,
+	[0][0][RTW89_UK][3] = 30,
+	[0][0][RTW89_FCC][4] = 68,
+	[0][0][RTW89_ETSI][4] = 30,
+	[0][0][RTW89_MKK][4] = 44,
+	[0][0][RTW89_IC][4] = 68,
+	[0][0][RTW89_KCC][4] = 48,
+	[0][0][RTW89_ACMA][4] = 30,
+	[0][0][RTW89_CN][4] = 32,
+	[0][0][RTW89_UK][4] = 30,
+	[0][0][RTW89_FCC][5] = 82,
+	[0][0][RTW89_ETSI][5] = 30,
+	[0][0][RTW89_MKK][5] = 44,
+	[0][0][RTW89_IC][5] = 82,
+	[0][0][RTW89_KCC][5] = 48,
+	[0][0][RTW89_ACMA][5] = 30,
+	[0][0][RTW89_CN][5] = 32,
+	[0][0][RTW89_UK][5] = 30,
+	[0][0][RTW89_FCC][6] = 64,
+	[0][0][RTW89_ETSI][6] = 30,
+	[0][0][RTW89_MKK][6] = 44,
+	[0][0][RTW89_IC][6] = 64,
+	[0][0][RTW89_KCC][6] = 48,
+	[0][0][RTW89_ACMA][6] = 30,
+	[0][0][RTW89_CN][6] = 32,
+	[0][0][RTW89_UK][6] = 30,
+	[0][0][RTW89_FCC][7] = 64,
+	[0][0][RTW89_ETSI][7] = 30,
+	[0][0][RTW89_MKK][7] = 44,
+	[0][0][RTW89_IC][7] = 64,
+	[0][0][RTW89_KCC][7] = 48,
+	[0][0][RTW89_ACMA][7] = 30,
+	[0][0][RTW89_CN][7] = 32,
+	[0][0][RTW89_UK][7] = 30,
+	[0][0][RTW89_FCC][8] = 60,
+	[0][0][RTW89_ETSI][8] = 30,
+	[0][0][RTW89_MKK][8] = 44,
+	[0][0][RTW89_IC][8] = 60,
+	[0][0][RTW89_KCC][8] = 48,
+	[0][0][RTW89_ACMA][8] = 30,
+	[0][0][RTW89_CN][8] = 32,
+	[0][0][RTW89_UK][8] = 30,
+	[0][0][RTW89_FCC][9] = 56,
+	[0][0][RTW89_ETSI][9] = 30,
+	[0][0][RTW89_MKK][9] = 44,
+	[0][0][RTW89_IC][9] = 56,
+	[0][0][RTW89_KCC][9] = 44,
+	[0][0][RTW89_ACMA][9] = 30,
+	[0][0][RTW89_CN][9] = 32,
+	[0][0][RTW89_UK][9] = 30,
+	[0][0][RTW89_FCC][10] = 56,
+	[0][0][RTW89_ETSI][10] = 30,
+	[0][0][RTW89_MKK][10] = 44,
+	[0][0][RTW89_IC][10] = 56,
+	[0][0][RTW89_KCC][10] = 44,
+	[0][0][RTW89_ACMA][10] = 30,
+	[0][0][RTW89_CN][10] = 32,
+	[0][0][RTW89_UK][10] = 30,
+	[0][0][RTW89_FCC][11] = 54,
+	[0][0][RTW89_ETSI][11] = 30,
+	[0][0][RTW89_MKK][11] = 44,
+	[0][0][RTW89_IC][11] = 54,
+	[0][0][RTW89_KCC][11] = 44,
+	[0][0][RTW89_ACMA][11] = 30,
+	[0][0][RTW89_CN][11] = 32,
+	[0][0][RTW89_UK][11] = 30,
+	[0][0][RTW89_FCC][12] = 34,
+	[0][0][RTW89_ETSI][12] = 30,
+	[0][0][RTW89_MKK][12] = 40,
+	[0][0][RTW89_IC][12] = 34,
+	[0][0][RTW89_KCC][12] = 44,
+	[0][0][RTW89_ACMA][12] = 30,
+	[0][0][RTW89_CN][12] = 32,
+	[0][0][RTW89_UK][12] = 30,
+	[0][0][RTW89_FCC][13] = 127,
+	[0][0][RTW89_ETSI][13] = 127,
+	[0][0][RTW89_MKK][13] = 127,
+	[0][0][RTW89_IC][13] = 127,
+	[0][0][RTW89_KCC][13] = 127,
+	[0][0][RTW89_ACMA][13] = 127,
+	[0][0][RTW89_CN][13] = 127,
+	[0][0][RTW89_UK][13] = 127,
+	[0][1][RTW89_FCC][0] = 127,
+	[0][1][RTW89_ETSI][0] = 127,
+	[0][1][RTW89_MKK][0] = 127,
+	[0][1][RTW89_IC][0] = 127,
+	[0][1][RTW89_KCC][0] = 127,
+	[0][1][RTW89_ACMA][0] = 127,
+	[0][1][RTW89_CN][0] = 20,
+	[0][1][RTW89_UK][0] = 127,
+	[0][1][RTW89_FCC][1] = 127,
+	[0][1][RTW89_ETSI][1] = 127,
+	[0][1][RTW89_MKK][1] = 127,
+	[0][1][RTW89_IC][1] = 127,
+	[0][1][RTW89_KCC][1] = 127,
+	[0][1][RTW89_ACMA][1] = 127,
+	[0][1][RTW89_CN][1] = 22,
+	[0][1][RTW89_UK][1] = 127,
+	[0][1][RTW89_FCC][2] = 127,
+	[0][1][RTW89_ETSI][2] = 127,
+	[0][1][RTW89_MKK][2] = 127,
+	[0][1][RTW89_IC][2] = 127,
+	[0][1][RTW89_KCC][2] = 127,
+	[0][1][RTW89_ACMA][2] = 127,
+	[0][1][RTW89_CN][2] = 22,
+	[0][1][RTW89_UK][2] = 127,
+	[0][1][RTW89_FCC][3] = 127,
+	[0][1][RTW89_ETSI][3] = 127,
+	[0][1][RTW89_MKK][3] = 127,
+	[0][1][RTW89_IC][3] = 127,
+	[0][1][RTW89_KCC][3] = 127,
+	[0][1][RTW89_ACMA][3] = 127,
+	[0][1][RTW89_CN][3] = 22,
+	[0][1][RTW89_UK][3] = 127,
+	[0][1][RTW89_FCC][4] = 127,
+	[0][1][RTW89_ETSI][4] = 127,
+	[0][1][RTW89_MKK][4] = 127,
+	[0][1][RTW89_IC][4] = 127,
+	[0][1][RTW89_KCC][4] = 127,
+	[0][1][RTW89_ACMA][4] = 127,
+	[0][1][RTW89_CN][4] = 22,
+	[0][1][RTW89_UK][4] = 127,
+	[0][1][RTW89_FCC][5] = 127,
+	[0][1][RTW89_ETSI][5] = 127,
+	[0][1][RTW89_MKK][5] = 127,
+	[0][1][RTW89_IC][5] = 127,
+	[0][1][RTW89_KCC][5] = 127,
+	[0][1][RTW89_ACMA][5] = 127,
+	[0][1][RTW89_CN][5] = 22,
+	[0][1][RTW89_UK][5] = 127,
+	[0][1][RTW89_FCC][6] = 127,
+	[0][1][RTW89_ETSI][6] = 127,
+	[0][1][RTW89_MKK][6] = 127,
+	[0][1][RTW89_IC][6] = 127,
+	[0][1][RTW89_KCC][6] = 127,
+	[0][1][RTW89_ACMA][6] = 127,
+	[0][1][RTW89_CN][6] = 22,
+	[0][1][RTW89_UK][6] = 127,
+	[0][1][RTW89_FCC][7] = 127,
+	[0][1][RTW89_ETSI][7] = 127,
+	[0][1][RTW89_MKK][7] = 127,
+	[0][1][RTW89_IC][7] = 127,
+	[0][1][RTW89_KCC][7] = 127,
+	[0][1][RTW89_ACMA][7] = 127,
+	[0][1][RTW89_CN][7] = 22,
+	[0][1][RTW89_UK][7] = 127,
+	[0][1][RTW89_FCC][8] = 127,
+	[0][1][RTW89_ETSI][8] = 127,
+	[0][1][RTW89_MKK][8] = 127,
+	[0][1][RTW89_IC][8] = 127,
+	[0][1][RTW89_KCC][8] = 127,
+	[0][1][RTW89_ACMA][8] = 127,
+	[0][1][RTW89_CN][8] = 22,
+	[0][1][RTW89_UK][8] = 127,
+	[0][1][RTW89_FCC][9] = 127,
+	[0][1][RTW89_ETSI][9] = 127,
+	[0][1][RTW89_MKK][9] = 127,
+	[0][1][RTW89_IC][9] = 127,
+	[0][1][RTW89_KCC][9] = 127,
+	[0][1][RTW89_ACMA][9] = 127,
+	[0][1][RTW89_CN][9] = 22,
+	[0][1][RTW89_UK][9] = 127,
+	[0][1][RTW89_FCC][10] = 127,
+	[0][1][RTW89_ETSI][10] = 127,
+	[0][1][RTW89_MKK][10] = 127,
+	[0][1][RTW89_IC][10] = 127,
+	[0][1][RTW89_KCC][10] = 127,
+	[0][1][RTW89_ACMA][10] = 127,
+	[0][1][RTW89_CN][10] = 22,
+	[0][1][RTW89_UK][10] = 127,
+	[0][1][RTW89_FCC][11] = 127,
+	[0][1][RTW89_ETSI][11] = 127,
+	[0][1][RTW89_MKK][11] = 127,
+	[0][1][RTW89_IC][11] = 127,
+	[0][1][RTW89_KCC][11] = 127,
+	[0][1][RTW89_ACMA][11] = 127,
+	[0][1][RTW89_CN][11] = 22,
+	[0][1][RTW89_UK][11] = 127,
+	[0][1][RTW89_FCC][12] = 127,
+	[0][1][RTW89_ETSI][12] = 127,
+	[0][1][RTW89_MKK][12] = 127,
+	[0][1][RTW89_IC][12] = 127,
+	[0][1][RTW89_KCC][12] = 127,
+	[0][1][RTW89_ACMA][12] = 127,
+	[0][1][RTW89_CN][12] = 20,
+	[0][1][RTW89_UK][12] = 127,
+	[0][1][RTW89_FCC][13] = 127,
+	[0][1][RTW89_ETSI][13] = 127,
+	[0][1][RTW89_MKK][13] = 127,
+	[0][1][RTW89_IC][13] = 127,
+	[0][1][RTW89_KCC][13] = 127,
+	[0][1][RTW89_ACMA][13] = 127,
+	[0][1][RTW89_CN][13] = 127,
+	[0][1][RTW89_UK][13] = 127,
+	[1][0][RTW89_FCC][0] = 70,
+	[1][0][RTW89_ETSI][0] = 42,
+	[1][0][RTW89_MKK][0] = 52,
+	[1][0][RTW89_IC][0] = 70,
+	[1][0][RTW89_KCC][0] = 56,
+	[1][0][RTW89_ACMA][0] = 42,
+	[1][0][RTW89_CN][0] = 42,
+	[1][0][RTW89_UK][0] = 42,
+	[1][0][RTW89_FCC][1] = 70,
+	[1][0][RTW89_ETSI][1] = 42,
+	[1][0][RTW89_MKK][1] = 52,
+	[1][0][RTW89_IC][1] = 70,
+	[1][0][RTW89_KCC][1] = 56,
+	[1][0][RTW89_ACMA][1] = 42,
+	[1][0][RTW89_CN][1] = 44,
+	[1][0][RTW89_UK][1] = 42,
+	[1][0][RTW89_FCC][2] = 74,
+	[1][0][RTW89_ETSI][2] = 42,
+	[1][0][RTW89_MKK][2] = 52,
+	[1][0][RTW89_IC][2] = 74,
+	[1][0][RTW89_KCC][2] = 56,
+	[1][0][RTW89_ACMA][2] = 42,
+	[1][0][RTW89_CN][2] = 44,
+	[1][0][RTW89_UK][2] = 42,
+	[1][0][RTW89_FCC][3] = 76,
+	[1][0][RTW89_ETSI][3] = 42,
+	[1][0][RTW89_MKK][3] = 52,
+	[1][0][RTW89_IC][3] = 76,
+	[1][0][RTW89_KCC][3] = 56,
+	[1][0][RTW89_ACMA][3] = 42,
+	[1][0][RTW89_CN][3] = 44,
+	[1][0][RTW89_UK][3] = 42,
+	[1][0][RTW89_FCC][4] = 76,
+	[1][0][RTW89_ETSI][4] = 42,
+	[1][0][RTW89_MKK][4] = 52,
+	[1][0][RTW89_IC][4] = 76,
+	[1][0][RTW89_KCC][4] = 56,
+	[1][0][RTW89_ACMA][4] = 42,
+	[1][0][RTW89_CN][4] = 44,
+	[1][0][RTW89_UK][4] = 42,
+	[1][0][RTW89_FCC][5] = 82,
+	[1][0][RTW89_ETSI][5] = 42,
+	[1][0][RTW89_MKK][5] = 52,
+	[1][0][RTW89_IC][5] = 82,
+	[1][0][RTW89_KCC][5] = 56,
+	[1][0][RTW89_ACMA][5] = 42,
+	[1][0][RTW89_CN][5] = 44,
+	[1][0][RTW89_UK][5] = 42,
+	[1][0][RTW89_FCC][6] = 72,
+	[1][0][RTW89_ETSI][6] = 42,
+	[1][0][RTW89_MKK][6] = 52,
+	[1][0][RTW89_IC][6] = 72,
+	[1][0][RTW89_KCC][6] = 56,
+	[1][0][RTW89_ACMA][6] = 42,
+	[1][0][RTW89_CN][6] = 44,
+	[1][0][RTW89_UK][6] = 42,
+	[1][0][RTW89_FCC][7] = 72,
+	[1][0][RTW89_ETSI][7] = 42,
+	[1][0][RTW89_MKK][7] = 52,
+	[1][0][RTW89_IC][7] = 72,
+	[1][0][RTW89_KCC][7] = 56,
+	[1][0][RTW89_ACMA][7] = 42,
+	[1][0][RTW89_CN][7] = 44,
+	[1][0][RTW89_UK][7] = 42,
+	[1][0][RTW89_FCC][8] = 72,
+	[1][0][RTW89_ETSI][8] = 42,
+	[1][0][RTW89_MKK][8] = 52,
+	[1][0][RTW89_IC][8] = 72,
+	[1][0][RTW89_KCC][8] = 56,
+	[1][0][RTW89_ACMA][8] = 42,
+	[1][0][RTW89_CN][8] = 44,
+	[1][0][RTW89_UK][8] = 42,
+	[1][0][RTW89_FCC][9] = 68,
+	[1][0][RTW89_ETSI][9] = 42,
+	[1][0][RTW89_MKK][9] = 52,
+	[1][0][RTW89_IC][9] = 68,
+	[1][0][RTW89_KCC][9] = 58,
+	[1][0][RTW89_ACMA][9] = 42,
+	[1][0][RTW89_CN][9] = 44,
+	[1][0][RTW89_UK][9] = 42,
+	[1][0][RTW89_FCC][10] = 68,
+	[1][0][RTW89_ETSI][10] = 42,
+	[1][0][RTW89_MKK][10] = 52,
+	[1][0][RTW89_IC][10] = 68,
+	[1][0][RTW89_KCC][10] = 58,
+	[1][0][RTW89_ACMA][10] = 42,
+	[1][0][RTW89_CN][10] = 44,
+	[1][0][RTW89_UK][10] = 42,
+	[1][0][RTW89_FCC][11] = 66,
+	[1][0][RTW89_ETSI][11] = 42,
+	[1][0][RTW89_MKK][11] = 52,
+	[1][0][RTW89_IC][11] = 66,
+	[1][0][RTW89_KCC][11] = 58,
+	[1][0][RTW89_ACMA][11] = 42,
+	[1][0][RTW89_CN][11] = 44,
+	[1][0][RTW89_UK][11] = 42,
+	[1][0][RTW89_FCC][12] = 34,
+	[1][0][RTW89_ETSI][12] = 42,
+	[1][0][RTW89_MKK][12] = 52,
+	[1][0][RTW89_IC][12] = 34,
+	[1][0][RTW89_KCC][12] = 58,
+	[1][0][RTW89_ACMA][12] = 42,
+	[1][0][RTW89_CN][12] = 42,
+	[1][0][RTW89_UK][12] = 42,
+	[1][0][RTW89_FCC][13] = 127,
+	[1][0][RTW89_ETSI][13] = 127,
+	[1][0][RTW89_MKK][13] = 127,
+	[1][0][RTW89_IC][13] = 127,
+	[1][0][RTW89_KCC][13] = 127,
+	[1][0][RTW89_ACMA][13] = 127,
+	[1][0][RTW89_CN][13] = 127,
+	[1][0][RTW89_UK][13] = 127,
+	[1][1][RTW89_FCC][0] = 127,
+	[1][1][RTW89_ETSI][0] = 127,
+	[1][1][RTW89_MKK][0] = 127,
+	[1][1][RTW89_IC][0] = 127,
+	[1][1][RTW89_KCC][0] = 127,
+	[1][1][RTW89_ACMA][0] = 127,
+	[1][1][RTW89_CN][0] = 32,
+	[1][1][RTW89_UK][0] = 127,
+	[1][1][RTW89_FCC][1] = 127,
+	[1][1][RTW89_ETSI][1] = 127,
+	[1][1][RTW89_MKK][1] = 127,
+	[1][1][RTW89_IC][1] = 127,
+	[1][1][RTW89_KCC][1] = 127,
+	[1][1][RTW89_ACMA][1] = 127,
+	[1][1][RTW89_CN][1] = 32,
+	[1][1][RTW89_UK][1] = 127,
+	[1][1][RTW89_FCC][2] = 127,
+	[1][1][RTW89_ETSI][2] = 127,
+	[1][1][RTW89_MKK][2] = 127,
+	[1][1][RTW89_IC][2] = 127,
+	[1][1][RTW89_KCC][2] = 127,
+	[1][1][RTW89_ACMA][2] = 127,
+	[1][1][RTW89_CN][2] = 32,
+	[1][1][RTW89_UK][2] = 127,
+	[1][1][RTW89_FCC][3] = 127,
+	[1][1][RTW89_ETSI][3] = 127,
+	[1][1][RTW89_MKK][3] = 127,
+	[1][1][RTW89_IC][3] = 127,
+	[1][1][RTW89_KCC][3] = 127,
+	[1][1][RTW89_ACMA][3] = 127,
+	[1][1][RTW89_CN][3] = 32,
+	[1][1][RTW89_UK][3] = 127,
+	[1][1][RTW89_FCC][4] = 127,
+	[1][1][RTW89_ETSI][4] = 127,
+	[1][1][RTW89_MKK][4] = 127,
+	[1][1][RTW89_IC][4] = 127,
+	[1][1][RTW89_KCC][4] = 127,
+	[1][1][RTW89_ACMA][4] = 127,
+	[1][1][RTW89_CN][4] = 32,
+	[1][1][RTW89_UK][4] = 127,
+	[1][1][RTW89_FCC][5] = 127,
+	[1][1][RTW89_ETSI][5] = 127,
+	[1][1][RTW89_MKK][5] = 127,
+	[1][1][RTW89_IC][5] = 127,
+	[1][1][RTW89_KCC][5] = 127,
+	[1][1][RTW89_ACMA][5] = 127,
+	[1][1][RTW89_CN][5] = 32,
+	[1][1][RTW89_UK][5] = 127,
+	[1][1][RTW89_FCC][6] = 127,
+	[1][1][RTW89_ETSI][6] = 127,
+	[1][1][RTW89_MKK][6] = 127,
+	[1][1][RTW89_IC][6] = 127,
+	[1][1][RTW89_KCC][6] = 127,
+	[1][1][RTW89_ACMA][6] = 127,
+	[1][1][RTW89_CN][6] = 32,
+	[1][1][RTW89_UK][6] = 127,
+	[1][1][RTW89_FCC][7] = 127,
+	[1][1][RTW89_ETSI][7] = 127,
+	[1][1][RTW89_MKK][7] = 127,
+	[1][1][RTW89_IC][7] = 127,
+	[1][1][RTW89_KCC][7] = 127,
+	[1][1][RTW89_ACMA][7] = 127,
+	[1][1][RTW89_CN][7] = 32,
+	[1][1][RTW89_UK][7] = 127,
+	[1][1][RTW89_FCC][8] = 127,
+	[1][1][RTW89_ETSI][8] = 127,
+	[1][1][RTW89_MKK][8] = 127,
+	[1][1][RTW89_IC][8] = 127,
+	[1][1][RTW89_KCC][8] = 127,
+	[1][1][RTW89_ACMA][8] = 127,
+	[1][1][RTW89_CN][8] = 32,
+	[1][1][RTW89_UK][8] = 127,
+	[1][1][RTW89_FCC][9] = 127,
+	[1][1][RTW89_ETSI][9] = 127,
+	[1][1][RTW89_MKK][9] = 127,
+	[1][1][RTW89_IC][9] = 127,
+	[1][1][RTW89_KCC][9] = 127,
+	[1][1][RTW89_ACMA][9] = 127,
+	[1][1][RTW89_CN][9] = 32,
+	[1][1][RTW89_UK][9] = 127,
+	[1][1][RTW89_FCC][10] = 127,
+	[1][1][RTW89_ETSI][10] = 127,
+	[1][1][RTW89_MKK][10] = 127,
+	[1][1][RTW89_IC][10] = 127,
+	[1][1][RTW89_KCC][10] = 127,
+	[1][1][RTW89_ACMA][10] = 127,
+	[1][1][RTW89_CN][10] = 32,
+	[1][1][RTW89_UK][10] = 127,
+	[1][1][RTW89_FCC][11] = 127,
+	[1][1][RTW89_ETSI][11] = 127,
+	[1][1][RTW89_MKK][11] = 127,
+	[1][1][RTW89_IC][11] = 127,
+	[1][1][RTW89_KCC][11] = 127,
+	[1][1][RTW89_ACMA][11] = 127,
+	[1][1][RTW89_CN][11] = 32,
+	[1][1][RTW89_UK][11] = 127,
+	[1][1][RTW89_FCC][12] = 127,
+	[1][1][RTW89_ETSI][12] = 127,
+	[1][1][RTW89_MKK][12] = 127,
+	[1][1][RTW89_IC][12] = 127,
+	[1][1][RTW89_KCC][12] = 127,
+	[1][1][RTW89_ACMA][12] = 127,
+	[1][1][RTW89_CN][12] = 32,
+	[1][1][RTW89_UK][12] = 127,
+	[1][1][RTW89_FCC][13] = 127,
+	[1][1][RTW89_ETSI][13] = 127,
+	[1][1][RTW89_MKK][13] = 127,
+	[1][1][RTW89_IC][13] = 127,
+	[1][1][RTW89_KCC][13] = 127,
+	[1][1][RTW89_ACMA][13] = 127,
+	[1][1][RTW89_CN][13] = 127,
+	[1][1][RTW89_UK][13] = 127,
+	[2][0][RTW89_FCC][0] = 74,
+	[2][0][RTW89_ETSI][0] = 54,
+	[2][0][RTW89_MKK][0] = 64,
+	[2][0][RTW89_IC][0] = 74,
+	[2][0][RTW89_KCC][0] = 68,
+	[2][0][RTW89_ACMA][0] = 54,
+	[2][0][RTW89_CN][0] = 56,
+	[2][0][RTW89_UK][0] = 54,
+	[2][0][RTW89_FCC][1] = 74,
+	[2][0][RTW89_ETSI][1] = 54,
+	[2][0][RTW89_MKK][1] = 64,
+	[2][0][RTW89_IC][1] = 74,
+	[2][0][RTW89_KCC][1] = 68,
+	[2][0][RTW89_ACMA][1] = 54,
+	[2][0][RTW89_CN][1] = 56,
+	[2][0][RTW89_UK][1] = 54,
+	[2][0][RTW89_FCC][2] = 76,
+	[2][0][RTW89_ETSI][2] = 54,
+	[2][0][RTW89_MKK][2] = 64,
+	[2][0][RTW89_IC][2] = 76,
+	[2][0][RTW89_KCC][2] = 68,
+	[2][0][RTW89_ACMA][2] = 54,
+	[2][0][RTW89_CN][2] = 56,
+	[2][0][RTW89_UK][2] = 54,
+	[2][0][RTW89_FCC][3] = 76,
+	[2][0][RTW89_ETSI][3] = 54,
+	[2][0][RTW89_MKK][3] = 64,
+	[2][0][RTW89_IC][3] = 76,
+	[2][0][RTW89_KCC][3] = 68,
+	[2][0][RTW89_ACMA][3] = 54,
+	[2][0][RTW89_CN][3] = 56,
+	[2][0][RTW89_UK][3] = 54,
+	[2][0][RTW89_FCC][4] = 76,
+	[2][0][RTW89_ETSI][4] = 54,
+	[2][0][RTW89_MKK][4] = 64,
+	[2][0][RTW89_IC][4] = 76,
+	[2][0][RTW89_KCC][4] = 68,
+	[2][0][RTW89_ACMA][4] = 54,
+	[2][0][RTW89_CN][4] = 56,
+	[2][0][RTW89_UK][4] = 54,
+	[2][0][RTW89_FCC][5] = 80,
+	[2][0][RTW89_ETSI][5] = 54,
+	[2][0][RTW89_MKK][5] = 64,
+	[2][0][RTW89_IC][5] = 80,
+	[2][0][RTW89_KCC][5] = 68,
+	[2][0][RTW89_ACMA][5] = 54,
+	[2][0][RTW89_CN][5] = 56,
+	[2][0][RTW89_UK][5] = 54,
+	[2][0][RTW89_FCC][6] = 72,
+	[2][0][RTW89_ETSI][6] = 54,
+	[2][0][RTW89_MKK][6] = 64,
+	[2][0][RTW89_IC][6] = 72,
+	[2][0][RTW89_KCC][6] = 68,
+	[2][0][RTW89_ACMA][6] = 54,
+	[2][0][RTW89_CN][6] = 56,
+	[2][0][RTW89_UK][6] = 54,
+	[2][0][RTW89_FCC][7] = 72,
+	[2][0][RTW89_ETSI][7] = 54,
+	[2][0][RTW89_MKK][7] = 64,
+	[2][0][RTW89_IC][7] = 72,
+	[2][0][RTW89_KCC][7] = 68,
+	[2][0][RTW89_ACMA][7] = 54,
+	[2][0][RTW89_CN][7] = 56,
+	[2][0][RTW89_UK][7] = 54,
+	[2][0][RTW89_FCC][8] = 72,
+	[2][0][RTW89_ETSI][8] = 54,
+	[2][0][RTW89_MKK][8] = 64,
+	[2][0][RTW89_IC][8] = 72,
+	[2][0][RTW89_KCC][8] = 68,
+	[2][0][RTW89_ACMA][8] = 54,
+	[2][0][RTW89_CN][8] = 56,
+	[2][0][RTW89_UK][8] = 54,
+	[2][0][RTW89_FCC][9] = 70,
+	[2][0][RTW89_ETSI][9] = 54,
+	[2][0][RTW89_MKK][9] = 64,
+	[2][0][RTW89_IC][9] = 70,
+	[2][0][RTW89_KCC][9] = 68,
+	[2][0][RTW89_ACMA][9] = 54,
+	[2][0][RTW89_CN][9] = 56,
+	[2][0][RTW89_UK][9] = 54,
+	[2][0][RTW89_FCC][10] = 70,
+	[2][0][RTW89_ETSI][10] = 54,
+	[2][0][RTW89_MKK][10] = 64,
+	[2][0][RTW89_IC][10] = 70,
+	[2][0][RTW89_KCC][10] = 68,
+	[2][0][RTW89_ACMA][10] = 54,
+	[2][0][RTW89_CN][10] = 56,
+	[2][0][RTW89_UK][10] = 54,
+	[2][0][RTW89_FCC][11] = 62,
+	[2][0][RTW89_ETSI][11] = 54,
+	[2][0][RTW89_MKK][11] = 64,
+	[2][0][RTW89_IC][11] = 62,
+	[2][0][RTW89_KCC][11] = 68,
+	[2][0][RTW89_ACMA][11] = 54,
+	[2][0][RTW89_CN][11] = 56,
+	[2][0][RTW89_UK][11] = 54,
+	[2][0][RTW89_FCC][12] = 34,
+	[2][0][RTW89_ETSI][12] = 54,
+	[2][0][RTW89_MKK][12] = 64,
+	[2][0][RTW89_IC][12] = 34,
+	[2][0][RTW89_KCC][12] = 68,
+	[2][0][RTW89_ACMA][12] = 54,
+	[2][0][RTW89_CN][12] = 56,
+	[2][0][RTW89_UK][12] = 54,
+	[2][0][RTW89_FCC][13] = 127,
+	[2][0][RTW89_ETSI][13] = 127,
+	[2][0][RTW89_MKK][13] = 127,
+	[2][0][RTW89_IC][13] = 127,
+	[2][0][RTW89_KCC][13] = 127,
+	[2][0][RTW89_ACMA][13] = 127,
+	[2][0][RTW89_CN][13] = 127,
+	[2][0][RTW89_UK][13] = 127,
+	[2][1][RTW89_FCC][0] = 127,
+	[2][1][RTW89_ETSI][0] = 127,
+	[2][1][RTW89_MKK][0] = 127,
+	[2][1][RTW89_IC][0] = 127,
+	[2][1][RTW89_KCC][0] = 127,
+	[2][1][RTW89_ACMA][0] = 127,
+	[2][1][RTW89_CN][0] = 44,
+	[2][1][RTW89_UK][0] = 127,
+	[2][1][RTW89_FCC][1] = 127,
+	[2][1][RTW89_ETSI][1] = 127,
+	[2][1][RTW89_MKK][1] = 127,
+	[2][1][RTW89_IC][1] = 127,
+	[2][1][RTW89_KCC][1] = 127,
+	[2][1][RTW89_ACMA][1] = 127,
+	[2][1][RTW89_CN][1] = 44,
+	[2][1][RTW89_UK][1] = 127,
+	[2][1][RTW89_FCC][2] = 127,
+	[2][1][RTW89_ETSI][2] = 127,
+	[2][1][RTW89_MKK][2] = 127,
+	[2][1][RTW89_IC][2] = 127,
+	[2][1][RTW89_KCC][2] = 127,
+	[2][1][RTW89_ACMA][2] = 127,
+	[2][1][RTW89_CN][2] = 44,
+	[2][1][RTW89_UK][2] = 127,
+	[2][1][RTW89_FCC][3] = 127,
+	[2][1][RTW89_ETSI][3] = 127,
+	[2][1][RTW89_MKK][3] = 127,
+	[2][1][RTW89_IC][3] = 127,
+	[2][1][RTW89_KCC][3] = 127,
+	[2][1][RTW89_ACMA][3] = 127,
+	[2][1][RTW89_CN][3] = 44,
+	[2][1][RTW89_UK][3] = 127,
+	[2][1][RTW89_FCC][4] = 127,
+	[2][1][RTW89_ETSI][4] = 127,
+	[2][1][RTW89_MKK][4] = 127,
+	[2][1][RTW89_IC][4] = 127,
+	[2][1][RTW89_KCC][4] = 127,
+	[2][1][RTW89_ACMA][4] = 127,
+	[2][1][RTW89_CN][4] = 44,
+	[2][1][RTW89_UK][4] = 127,
+	[2][1][RTW89_FCC][5] = 127,
+	[2][1][RTW89_ETSI][5] = 127,
+	[2][1][RTW89_MKK][5] = 127,
+	[2][1][RTW89_IC][5] = 127,
+	[2][1][RTW89_KCC][5] = 127,
+	[2][1][RTW89_ACMA][5] = 127,
+	[2][1][RTW89_CN][5] = 44,
+	[2][1][RTW89_UK][5] = 127,
+	[2][1][RTW89_FCC][6] = 127,
+	[2][1][RTW89_ETSI][6] = 127,
+	[2][1][RTW89_MKK][6] = 127,
+	[2][1][RTW89_IC][6] = 127,
+	[2][1][RTW89_KCC][6] = 127,
+	[2][1][RTW89_ACMA][6] = 127,
+	[2][1][RTW89_CN][6] = 44,
+	[2][1][RTW89_UK][6] = 127,
+	[2][1][RTW89_FCC][7] = 127,
+	[2][1][RTW89_ETSI][7] = 127,
+	[2][1][RTW89_MKK][7] = 127,
+	[2][1][RTW89_IC][7] = 127,
+	[2][1][RTW89_KCC][7] = 127,
+	[2][1][RTW89_ACMA][7] = 127,
+	[2][1][RTW89_CN][7] = 44,
+	[2][1][RTW89_UK][7] = 127,
+	[2][1][RTW89_FCC][8] = 127,
+	[2][1][RTW89_ETSI][8] = 127,
+	[2][1][RTW89_MKK][8] = 127,
+	[2][1][RTW89_IC][8] = 127,
+	[2][1][RTW89_KCC][8] = 127,
+	[2][1][RTW89_ACMA][8] = 127,
+	[2][1][RTW89_CN][8] = 44,
+	[2][1][RTW89_UK][8] = 127,
+	[2][1][RTW89_FCC][9] = 127,
+	[2][1][RTW89_ETSI][9] = 127,
+	[2][1][RTW89_MKK][9] = 127,
+	[2][1][RTW89_IC][9] = 127,
+	[2][1][RTW89_KCC][9] = 127,
+	[2][1][RTW89_ACMA][9] = 127,
+	[2][1][RTW89_CN][9] = 44,
+	[2][1][RTW89_UK][9] = 127,
+	[2][1][RTW89_FCC][10] = 127,
+	[2][1][RTW89_ETSI][10] = 127,
+	[2][1][RTW89_MKK][10] = 127,
+	[2][1][RTW89_IC][10] = 127,
+	[2][1][RTW89_KCC][10] = 127,
+	[2][1][RTW89_ACMA][10] = 127,
+	[2][1][RTW89_CN][10] = 44,
+	[2][1][RTW89_UK][10] = 127,
+	[2][1][RTW89_FCC][11] = 127,
+	[2][1][RTW89_ETSI][11] = 127,
+	[2][1][RTW89_MKK][11] = 127,
+	[2][1][RTW89_IC][11] = 127,
+	[2][1][RTW89_KCC][11] = 127,
+	[2][1][RTW89_ACMA][11] = 127,
+	[2][1][RTW89_CN][11] = 44,
+	[2][1][RTW89_UK][11] = 127,
+	[2][1][RTW89_FCC][12] = 127,
+	[2][1][RTW89_ETSI][12] = 127,
+	[2][1][RTW89_MKK][12] = 127,
+	[2][1][RTW89_IC][12] = 127,
+	[2][1][RTW89_KCC][12] = 127,
+	[2][1][RTW89_ACMA][12] = 127,
+	[2][1][RTW89_CN][12] = 42,
+	[2][1][RTW89_UK][12] = 127,
+	[2][1][RTW89_FCC][13] = 127,
+	[2][1][RTW89_ETSI][13] = 127,
+	[2][1][RTW89_MKK][13] = 127,
+	[2][1][RTW89_IC][13] = 127,
+	[2][1][RTW89_KCC][13] = 127,
+	[2][1][RTW89_ACMA][13] = 127,
+	[2][1][RTW89_CN][13] = 127,
+	[2][1][RTW89_UK][13] = 127,
+};
+
+static
+const s8 rtw89_8851b_txpwr_lmt_ru_5g_type2[RTW89_RU_NUM][RTW89_NTX_NUM]
+					  [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
+	[0][0][RTW89_WW][0] = 16,
+	[0][0][RTW89_WW][2] = 16,
+	[0][0][RTW89_WW][4] = 16,
+	[0][0][RTW89_WW][6] = 16,
+	[0][0][RTW89_WW][8] = 16,
+	[0][0][RTW89_WW][10] = 16,
+	[0][0][RTW89_WW][12] = 16,
+	[0][0][RTW89_WW][14] = 16,
+	[0][0][RTW89_WW][15] = 24,
+	[0][0][RTW89_WW][17] = 24,
+	[0][0][RTW89_WW][19] = 24,
+	[0][0][RTW89_WW][21] = 24,
+	[0][0][RTW89_WW][23] = 24,
+	[0][0][RTW89_WW][25] = 24,
+	[0][0][RTW89_WW][27] = 24,
+	[0][0][RTW89_WW][29] = 24,
+	[0][0][RTW89_WW][31] = 24,
+	[0][0][RTW89_WW][33] = 24,
+	[0][0][RTW89_WW][35] = 24,
+	[0][0][RTW89_WW][37] = 44,
+	[0][0][RTW89_WW][38] = 24,
+	[0][0][RTW89_WW][40] = 24,
+	[0][0][RTW89_WW][42] = 24,
+	[0][0][RTW89_WW][44] = 24,
+	[0][0][RTW89_WW][46] = 24,
+	[0][0][RTW89_WW][48] = 40,
+	[0][0][RTW89_WW][50] = 42,
+	[0][0][RTW89_WW][52] = 38,
+	[0][1][RTW89_WW][0] = 4,
+	[0][1][RTW89_WW][2] = 4,
+	[0][1][RTW89_WW][4] = 4,
+	[0][1][RTW89_WW][6] = 4,
+	[0][1][RTW89_WW][8] = 4,
+	[0][1][RTW89_WW][10] = 4,
+	[0][1][RTW89_WW][12] = 4,
+	[0][1][RTW89_WW][14] = 4,
+	[0][1][RTW89_WW][15] = 0,
+	[0][1][RTW89_WW][17] = 0,
+	[0][1][RTW89_WW][19] = 0,
+	[0][1][RTW89_WW][21] = 0,
+	[0][1][RTW89_WW][23] = 0,
+	[0][1][RTW89_WW][25] = 0,
+	[0][1][RTW89_WW][27] = 0,
+	[0][1][RTW89_WW][29] = 0,
+	[0][1][RTW89_WW][31] = 0,
+	[0][1][RTW89_WW][33] = 0,
+	[0][1][RTW89_WW][35] = 0,
+	[0][1][RTW89_WW][37] = 0,
+	[0][1][RTW89_WW][38] = 42,
+	[0][1][RTW89_WW][40] = 42,
+	[0][1][RTW89_WW][42] = 42,
+	[0][1][RTW89_WW][44] = 42,
+	[0][1][RTW89_WW][46] = 42,
+	[0][1][RTW89_WW][48] = 0,
+	[0][1][RTW89_WW][50] = 0,
+	[0][1][RTW89_WW][52] = 0,
+	[1][0][RTW89_WW][0] = 26,
+	[1][0][RTW89_WW][2] = 26,
+	[1][0][RTW89_WW][4] = 26,
+	[1][0][RTW89_WW][6] = 26,
+	[1][0][RTW89_WW][8] = 26,
+	[1][0][RTW89_WW][10] = 26,
+	[1][0][RTW89_WW][12] = 26,
+	[1][0][RTW89_WW][14] = 26,
+	[1][0][RTW89_WW][15] = 34,
+	[1][0][RTW89_WW][17] = 34,
+	[1][0][RTW89_WW][19] = 34,
+	[1][0][RTW89_WW][21] = 34,
+	[1][0][RTW89_WW][23] = 34,
+	[1][0][RTW89_WW][25] = 34,
+	[1][0][RTW89_WW][27] = 34,
+	[1][0][RTW89_WW][29] = 34,
+	[1][0][RTW89_WW][31] = 34,
+	[1][0][RTW89_WW][33] = 34,
+	[1][0][RTW89_WW][35] = 34,
+	[1][0][RTW89_WW][37] = 54,
+	[1][0][RTW89_WW][38] = 28,
+	[1][0][RTW89_WW][40] = 28,
+	[1][0][RTW89_WW][42] = 28,
+	[1][0][RTW89_WW][44] = 28,
+	[1][0][RTW89_WW][46] = 28,
+	[1][0][RTW89_WW][48] = 52,
+	[1][0][RTW89_WW][50] = 52,
+	[1][0][RTW89_WW][52] = 50,
+	[1][1][RTW89_WW][0] = 14,
+	[1][1][RTW89_WW][2] = 14,
+	[1][1][RTW89_WW][4] = 14,
+	[1][1][RTW89_WW][6] = 14,
+	[1][1][RTW89_WW][8] = 14,
+	[1][1][RTW89_WW][10] = 14,
+	[1][1][RTW89_WW][12] = 14,
+	[1][1][RTW89_WW][14] = 14,
+	[1][1][RTW89_WW][15] = 0,
+	[1][1][RTW89_WW][17] = 0,
+	[1][1][RTW89_WW][19] = 0,
+	[1][1][RTW89_WW][21] = 0,
+	[1][1][RTW89_WW][23] = 0,
+	[1][1][RTW89_WW][25] = 0,
+	[1][1][RTW89_WW][27] = 0,
+	[1][1][RTW89_WW][29] = 0,
+	[1][1][RTW89_WW][31] = 0,
+	[1][1][RTW89_WW][33] = 0,
+	[1][1][RTW89_WW][35] = 0,
+	[1][1][RTW89_WW][37] = 0,
+	[1][1][RTW89_WW][38] = 54,
+	[1][1][RTW89_WW][40] = 54,
+	[1][1][RTW89_WW][42] = 54,
+	[1][1][RTW89_WW][44] = 54,
+	[1][1][RTW89_WW][46] = 54,
+	[1][1][RTW89_WW][48] = 0,
+	[1][1][RTW89_WW][50] = 0,
+	[1][1][RTW89_WW][52] = 0,
+	[2][0][RTW89_WW][0] = 40,
+	[2][0][RTW89_WW][2] = 40,
+	[2][0][RTW89_WW][4] = 40,
+	[2][0][RTW89_WW][6] = 40,
+	[2][0][RTW89_WW][8] = 40,
+	[2][0][RTW89_WW][10] = 40,
+	[2][0][RTW89_WW][12] = 40,
+	[2][0][RTW89_WW][14] = 40,
+	[2][0][RTW89_WW][15] = 46,
+	[2][0][RTW89_WW][17] = 46,
+	[2][0][RTW89_WW][19] = 46,
+	[2][0][RTW89_WW][21] = 46,
+	[2][0][RTW89_WW][23] = 46,
+	[2][0][RTW89_WW][25] = 46,
+	[2][0][RTW89_WW][27] = 46,
+	[2][0][RTW89_WW][29] = 46,
+	[2][0][RTW89_WW][31] = 46,
+	[2][0][RTW89_WW][33] = 46,
+	[2][0][RTW89_WW][35] = 46,
+	[2][0][RTW89_WW][37] = 66,
+	[2][0][RTW89_WW][38] = 28,
+	[2][0][RTW89_WW][40] = 28,
+	[2][0][RTW89_WW][42] = 28,
+	[2][0][RTW89_WW][44] = 28,
+	[2][0][RTW89_WW][46] = 28,
+	[2][0][RTW89_WW][48] = 62,
+	[2][0][RTW89_WW][50] = 62,
+	[2][0][RTW89_WW][52] = 60,
+	[2][1][RTW89_WW][0] = 28,
+	[2][1][RTW89_WW][2] = 28,
+	[2][1][RTW89_WW][4] = 28,
+	[2][1][RTW89_WW][6] = 28,
+	[2][1][RTW89_WW][8] = 28,
+	[2][1][RTW89_WW][10] = 28,
+	[2][1][RTW89_WW][12] = 28,
+	[2][1][RTW89_WW][14] = 28,
+	[2][1][RTW89_WW][15] = 0,
+	[2][1][RTW89_WW][17] = 0,
+	[2][1][RTW89_WW][19] = 0,
+	[2][1][RTW89_WW][21] = 0,
+	[2][1][RTW89_WW][23] = 0,
+	[2][1][RTW89_WW][25] = 0,
+	[2][1][RTW89_WW][27] = 0,
+	[2][1][RTW89_WW][29] = 0,
+	[2][1][RTW89_WW][31] = 0,
+	[2][1][RTW89_WW][33] = 0,
+	[2][1][RTW89_WW][35] = 0,
+	[2][1][RTW89_WW][37] = 0,
+	[2][1][RTW89_WW][38] = 56,
+	[2][1][RTW89_WW][40] = 56,
+	[2][1][RTW89_WW][42] = 56,
+	[2][1][RTW89_WW][44] = 56,
+	[2][1][RTW89_WW][46] = 56,
+	[2][1][RTW89_WW][48] = 0,
+	[2][1][RTW89_WW][50] = 0,
+	[2][1][RTW89_WW][52] = 0,
+	[0][0][RTW89_FCC][0] = 50,
+	[0][0][RTW89_ETSI][0] = 24,
+	[0][0][RTW89_MKK][0] = 26,
+	[0][0][RTW89_IC][0] = 28,
+	[0][0][RTW89_KCC][0] = 42,
+	[0][0][RTW89_ACMA][0] = 24,
+	[0][0][RTW89_CN][0] = 16,
+	[0][0][RTW89_UK][0] = 24,
+	[0][0][RTW89_FCC][2] = 54,
+	[0][0][RTW89_ETSI][2] = 24,
+	[0][0][RTW89_MKK][2] = 26,
+	[0][0][RTW89_IC][2] = 28,
+	[0][0][RTW89_KCC][2] = 42,
+	[0][0][RTW89_ACMA][2] = 24,
+	[0][0][RTW89_CN][2] = 16,
+	[0][0][RTW89_UK][2] = 24,
+	[0][0][RTW89_FCC][4] = 50,
+	[0][0][RTW89_ETSI][4] = 24,
+	[0][0][RTW89_MKK][4] = 26,
+	[0][0][RTW89_IC][4] = 28,
+	[0][0][RTW89_KCC][4] = 42,
+	[0][0][RTW89_ACMA][4] = 24,
+	[0][0][RTW89_CN][4] = 16,
+	[0][0][RTW89_UK][4] = 24,
+	[0][0][RTW89_FCC][6] = 50,
+	[0][0][RTW89_ETSI][6] = 24,
+	[0][0][RTW89_MKK][6] = 26,
+	[0][0][RTW89_IC][6] = 28,
+	[0][0][RTW89_KCC][6] = 18,
+	[0][0][RTW89_ACMA][6] = 24,
+	[0][0][RTW89_CN][6] = 16,
+	[0][0][RTW89_UK][6] = 24,
+	[0][0][RTW89_FCC][8] = 52,
+	[0][0][RTW89_ETSI][8] = 24,
+	[0][0][RTW89_MKK][8] = 26,
+	[0][0][RTW89_IC][8] = 52,
+	[0][0][RTW89_KCC][8] = 42,
+	[0][0][RTW89_ACMA][8] = 24,
+	[0][0][RTW89_CN][8] = 16,
+	[0][0][RTW89_UK][8] = 24,
+	[0][0][RTW89_FCC][10] = 52,
+	[0][0][RTW89_ETSI][10] = 24,
+	[0][0][RTW89_MKK][10] = 26,
+	[0][0][RTW89_IC][10] = 52,
+	[0][0][RTW89_KCC][10] = 42,
+	[0][0][RTW89_ACMA][10] = 24,
+	[0][0][RTW89_CN][10] = 16,
+	[0][0][RTW89_UK][10] = 24,
+	[0][0][RTW89_FCC][12] = 56,
+	[0][0][RTW89_ETSI][12] = 24,
+	[0][0][RTW89_MKK][12] = 26,
+	[0][0][RTW89_IC][12] = 56,
+	[0][0][RTW89_KCC][12] = 44,
+	[0][0][RTW89_ACMA][12] = 24,
+	[0][0][RTW89_CN][12] = 16,
+	[0][0][RTW89_UK][12] = 24,
+	[0][0][RTW89_FCC][14] = 56,
+	[0][0][RTW89_ETSI][14] = 24,
+	[0][0][RTW89_MKK][14] = 26,
+	[0][0][RTW89_IC][14] = 56,
+	[0][0][RTW89_KCC][14] = 44,
+	[0][0][RTW89_ACMA][14] = 24,
+	[0][0][RTW89_CN][14] = 16,
+	[0][0][RTW89_UK][14] = 24,
+	[0][0][RTW89_FCC][15] = 52,
+	[0][0][RTW89_ETSI][15] = 24,
+	[0][0][RTW89_MKK][15] = 46,
+	[0][0][RTW89_IC][15] = 52,
+	[0][0][RTW89_KCC][15] = 44,
+	[0][0][RTW89_ACMA][15] = 24,
+	[0][0][RTW89_CN][15] = 127,
+	[0][0][RTW89_UK][15] = 24,
+	[0][0][RTW89_FCC][17] = 52,
+	[0][0][RTW89_ETSI][17] = 24,
+	[0][0][RTW89_MKK][17] = 50,
+	[0][0][RTW89_IC][17] = 52,
+	[0][0][RTW89_KCC][17] = 44,
+	[0][0][RTW89_ACMA][17] = 24,
+	[0][0][RTW89_CN][17] = 127,
+	[0][0][RTW89_UK][17] = 24,
+	[0][0][RTW89_FCC][19] = 52,
+	[0][0][RTW89_ETSI][19] = 24,
+	[0][0][RTW89_MKK][19] = 50,
+	[0][0][RTW89_IC][19] = 52,
+	[0][0][RTW89_KCC][19] = 44,
+	[0][0][RTW89_ACMA][19] = 24,
+	[0][0][RTW89_CN][19] = 127,
+	[0][0][RTW89_UK][19] = 24,
+	[0][0][RTW89_FCC][21] = 52,
+	[0][0][RTW89_ETSI][21] = 24,
+	[0][0][RTW89_MKK][21] = 50,
+	[0][0][RTW89_IC][21] = 52,
+	[0][0][RTW89_KCC][21] = 44,
+	[0][0][RTW89_ACMA][21] = 24,
+	[0][0][RTW89_CN][21] = 127,
+	[0][0][RTW89_UK][21] = 24,
+	[0][0][RTW89_FCC][23] = 52,
+	[0][0][RTW89_ETSI][23] = 24,
+	[0][0][RTW89_MKK][23] = 50,
+	[0][0][RTW89_IC][23] = 52,
+	[0][0][RTW89_KCC][23] = 44,
+	[0][0][RTW89_ACMA][23] = 24,
+	[0][0][RTW89_CN][23] = 127,
+	[0][0][RTW89_UK][23] = 24,
+	[0][0][RTW89_FCC][25] = 52,
+	[0][0][RTW89_ETSI][25] = 24,
+	[0][0][RTW89_MKK][25] = 50,
+	[0][0][RTW89_IC][25] = 127,
+	[0][0][RTW89_KCC][25] = 44,
+	[0][0][RTW89_ACMA][25] = 127,
+	[0][0][RTW89_CN][25] = 127,
+	[0][0][RTW89_UK][25] = 24,
+	[0][0][RTW89_FCC][27] = 52,
+	[0][0][RTW89_ETSI][27] = 24,
+	[0][0][RTW89_MKK][27] = 50,
+	[0][0][RTW89_IC][27] = 127,
+	[0][0][RTW89_KCC][27] = 42,
+	[0][0][RTW89_ACMA][27] = 127,
+	[0][0][RTW89_CN][27] = 127,
+	[0][0][RTW89_UK][27] = 24,
+	[0][0][RTW89_FCC][29] = 52,
+	[0][0][RTW89_ETSI][29] = 24,
+	[0][0][RTW89_MKK][29] = 50,
+	[0][0][RTW89_IC][29] = 127,
+	[0][0][RTW89_KCC][29] = 42,
+	[0][0][RTW89_ACMA][29] = 127,
+	[0][0][RTW89_CN][29] = 127,
+	[0][0][RTW89_UK][29] = 24,
+	[0][0][RTW89_FCC][31] = 52,
+	[0][0][RTW89_ETSI][31] = 24,
+	[0][0][RTW89_MKK][31] = 50,
+	[0][0][RTW89_IC][31] = 56,
+	[0][0][RTW89_KCC][31] = 42,
+	[0][0][RTW89_ACMA][31] = 24,
+	[0][0][RTW89_CN][31] = 127,
+	[0][0][RTW89_UK][31] = 24,
+	[0][0][RTW89_FCC][33] = 56,
+	[0][0][RTW89_ETSI][33] = 24,
+	[0][0][RTW89_MKK][33] = 50,
+	[0][0][RTW89_IC][33] = 56,
+	[0][0][RTW89_KCC][33] = 42,
+	[0][0][RTW89_ACMA][33] = 24,
+	[0][0][RTW89_CN][33] = 127,
+	[0][0][RTW89_UK][33] = 24,
+	[0][0][RTW89_FCC][35] = 56,
+	[0][0][RTW89_ETSI][35] = 24,
+	[0][0][RTW89_MKK][35] = 50,
+	[0][0][RTW89_IC][35] = 56,
+	[0][0][RTW89_KCC][35] = 42,
+	[0][0][RTW89_ACMA][35] = 24,
+	[0][0][RTW89_CN][35] = 127,
+	[0][0][RTW89_UK][35] = 24,
+	[0][0][RTW89_FCC][37] = 84,
+	[0][0][RTW89_ETSI][37] = 127,
+	[0][0][RTW89_MKK][37] = 46,
+	[0][0][RTW89_IC][37] = 84,
+	[0][0][RTW89_KCC][37] = 44,
+	[0][0][RTW89_ACMA][37] = 50,
+	[0][0][RTW89_CN][37] = 127,
+	[0][0][RTW89_UK][37] = 52,
+	[0][0][RTW89_FCC][38] = 68,
+	[0][0][RTW89_ETSI][38] = 28,
+	[0][0][RTW89_MKK][38] = 127,
+	[0][0][RTW89_IC][38] = 68,
+	[0][0][RTW89_KCC][38] = 44,
+	[0][0][RTW89_ACMA][38] = 84,
+	[0][0][RTW89_CN][38] = 54,
+	[0][0][RTW89_UK][38] = 24,
+	[0][0][RTW89_FCC][40] = 68,
+	[0][0][RTW89_ETSI][40] = 28,
+	[0][0][RTW89_MKK][40] = 127,
+	[0][0][RTW89_IC][40] = 68,
+	[0][0][RTW89_KCC][40] = 44,
+	[0][0][RTW89_ACMA][40] = 84,
+	[0][0][RTW89_CN][40] = 54,
+	[0][0][RTW89_UK][40] = 24,
+	[0][0][RTW89_FCC][42] = 70,
+	[0][0][RTW89_ETSI][42] = 28,
+	[0][0][RTW89_MKK][42] = 127,
+	[0][0][RTW89_IC][42] = 70,
+	[0][0][RTW89_KCC][42] = 44,
+	[0][0][RTW89_ACMA][42] = 84,
+	[0][0][RTW89_CN][42] = 54,
+	[0][0][RTW89_UK][42] = 24,
+	[0][0][RTW89_FCC][44] = 62,
+	[0][0][RTW89_ETSI][44] = 28,
+	[0][0][RTW89_MKK][44] = 127,
+	[0][0][RTW89_IC][44] = 62,
+	[0][0][RTW89_KCC][44] = 44,
+	[0][0][RTW89_ACMA][44] = 84,
+	[0][0][RTW89_CN][44] = 54,
+	[0][0][RTW89_UK][44] = 24,
+	[0][0][RTW89_FCC][46] = 62,
+	[0][0][RTW89_ETSI][46] = 28,
+	[0][0][RTW89_MKK][46] = 127,
+	[0][0][RTW89_IC][46] = 62,
+	[0][0][RTW89_KCC][46] = 44,
+	[0][0][RTW89_ACMA][46] = 84,
+	[0][0][RTW89_CN][46] = 54,
+	[0][0][RTW89_UK][46] = 24,
+	[0][0][RTW89_FCC][48] = 40,
+	[0][0][RTW89_ETSI][48] = 127,
+	[0][0][RTW89_MKK][48] = 127,
+	[0][0][RTW89_IC][48] = 127,
+	[0][0][RTW89_KCC][48] = 127,
+	[0][0][RTW89_ACMA][48] = 127,
+	[0][0][RTW89_CN][48] = 127,
+	[0][0][RTW89_UK][48] = 127,
+	[0][0][RTW89_FCC][50] = 42,
+	[0][0][RTW89_ETSI][50] = 127,
+	[0][0][RTW89_MKK][50] = 127,
+	[0][0][RTW89_IC][50] = 127,
+	[0][0][RTW89_KCC][50] = 127,
+	[0][0][RTW89_ACMA][50] = 127,
+	[0][0][RTW89_CN][50] = 127,
+	[0][0][RTW89_UK][50] = 127,
+	[0][0][RTW89_FCC][52] = 38,
+	[0][0][RTW89_ETSI][52] = 127,
+	[0][0][RTW89_MKK][52] = 127,
+	[0][0][RTW89_IC][52] = 127,
+	[0][0][RTW89_KCC][52] = 127,
+	[0][0][RTW89_ACMA][52] = 127,
+	[0][0][RTW89_CN][52] = 127,
+	[0][0][RTW89_UK][52] = 127,
+	[0][1][RTW89_FCC][0] = 127,
+	[0][1][RTW89_ETSI][0] = 127,
+	[0][1][RTW89_MKK][0] = 127,
+	[0][1][RTW89_IC][0] = 127,
+	[0][1][RTW89_KCC][0] = 127,
+	[0][1][RTW89_ACMA][0] = 127,
+	[0][1][RTW89_CN][0] = 4,
+	[0][1][RTW89_UK][0] = 127,
+	[0][1][RTW89_FCC][2] = 127,
+	[0][1][RTW89_ETSI][2] = 127,
+	[0][1][RTW89_MKK][2] = 127,
+	[0][1][RTW89_IC][2] = 127,
+	[0][1][RTW89_KCC][2] = 127,
+	[0][1][RTW89_ACMA][2] = 127,
+	[0][1][RTW89_CN][2] = 4,
+	[0][1][RTW89_UK][2] = 127,
+	[0][1][RTW89_FCC][4] = 127,
+	[0][1][RTW89_ETSI][4] = 127,
+	[0][1][RTW89_MKK][4] = 127,
+	[0][1][RTW89_IC][4] = 127,
+	[0][1][RTW89_KCC][4] = 127,
+	[0][1][RTW89_ACMA][4] = 127,
+	[0][1][RTW89_CN][4] = 4,
+	[0][1][RTW89_UK][4] = 127,
+	[0][1][RTW89_FCC][6] = 127,
+	[0][1][RTW89_ETSI][6] = 127,
+	[0][1][RTW89_MKK][6] = 127,
+	[0][1][RTW89_IC][6] = 127,
+	[0][1][RTW89_KCC][6] = 127,
+	[0][1][RTW89_ACMA][6] = 127,
+	[0][1][RTW89_CN][6] = 4,
+	[0][1][RTW89_UK][6] = 127,
+	[0][1][RTW89_FCC][8] = 127,
+	[0][1][RTW89_ETSI][8] = 127,
+	[0][1][RTW89_MKK][8] = 127,
+	[0][1][RTW89_IC][8] = 127,
+	[0][1][RTW89_KCC][8] = 127,
+	[0][1][RTW89_ACMA][8] = 127,
+	[0][1][RTW89_CN][8] = 4,
+	[0][1][RTW89_UK][8] = 127,
+	[0][1][RTW89_FCC][10] = 127,
+	[0][1][RTW89_ETSI][10] = 127,
+	[0][1][RTW89_MKK][10] = 127,
+	[0][1][RTW89_IC][10] = 127,
+	[0][1][RTW89_KCC][10] = 127,
+	[0][1][RTW89_ACMA][10] = 127,
+	[0][1][RTW89_CN][10] = 4,
+	[0][1][RTW89_UK][10] = 127,
+	[0][1][RTW89_FCC][12] = 127,
+	[0][1][RTW89_ETSI][12] = 127,
+	[0][1][RTW89_MKK][12] = 127,
+	[0][1][RTW89_IC][12] = 127,
+	[0][1][RTW89_KCC][12] = 127,
+	[0][1][RTW89_ACMA][12] = 127,
+	[0][1][RTW89_CN][12] = 4,
+	[0][1][RTW89_UK][12] = 127,
+	[0][1][RTW89_FCC][14] = 127,
+	[0][1][RTW89_ETSI][14] = 127,
+	[0][1][RTW89_MKK][14] = 127,
+	[0][1][RTW89_IC][14] = 127,
+	[0][1][RTW89_KCC][14] = 127,
+	[0][1][RTW89_ACMA][14] = 127,
+	[0][1][RTW89_CN][14] = 4,
+	[0][1][RTW89_UK][14] = 127,
+	[0][1][RTW89_FCC][15] = 127,
+	[0][1][RTW89_ETSI][15] = 127,
+	[0][1][RTW89_MKK][15] = 127,
+	[0][1][RTW89_IC][15] = 127,
+	[0][1][RTW89_KCC][15] = 127,
+	[0][1][RTW89_ACMA][15] = 127,
+	[0][1][RTW89_CN][15] = 127,
+	[0][1][RTW89_UK][15] = 127,
+	[0][1][RTW89_FCC][17] = 127,
+	[0][1][RTW89_ETSI][17] = 127,
+	[0][1][RTW89_MKK][17] = 127,
+	[0][1][RTW89_IC][17] = 127,
+	[0][1][RTW89_KCC][17] = 127,
+	[0][1][RTW89_ACMA][17] = 127,
+	[0][1][RTW89_CN][17] = 127,
+	[0][1][RTW89_UK][17] = 127,
+	[0][1][RTW89_FCC][19] = 127,
+	[0][1][RTW89_ETSI][19] = 127,
+	[0][1][RTW89_MKK][19] = 127,
+	[0][1][RTW89_IC][19] = 127,
+	[0][1][RTW89_KCC][19] = 127,
+	[0][1][RTW89_ACMA][19] = 127,
+	[0][1][RTW89_CN][19] = 127,
+	[0][1][RTW89_UK][19] = 127,
+	[0][1][RTW89_FCC][21] = 127,
+	[0][1][RTW89_ETSI][21] = 127,
+	[0][1][RTW89_MKK][21] = 127,
+	[0][1][RTW89_IC][21] = 127,
+	[0][1][RTW89_KCC][21] = 127,
+	[0][1][RTW89_ACMA][21] = 127,
+	[0][1][RTW89_CN][21] = 127,
+	[0][1][RTW89_UK][21] = 127,
+	[0][1][RTW89_FCC][23] = 127,
+	[0][1][RTW89_ETSI][23] = 127,
+	[0][1][RTW89_MKK][23] = 127,
+	[0][1][RTW89_IC][23] = 127,
+	[0][1][RTW89_KCC][23] = 127,
+	[0][1][RTW89_ACMA][23] = 127,
+	[0][1][RTW89_CN][23] = 127,
+	[0][1][RTW89_UK][23] = 127,
+	[0][1][RTW89_FCC][25] = 127,
+	[0][1][RTW89_ETSI][25] = 127,
+	[0][1][RTW89_MKK][25] = 127,
+	[0][1][RTW89_IC][25] = 127,
+	[0][1][RTW89_KCC][25] = 127,
+	[0][1][RTW89_ACMA][25] = 127,
+	[0][1][RTW89_CN][25] = 127,
+	[0][1][RTW89_UK][25] = 127,
+	[0][1][RTW89_FCC][27] = 127,
+	[0][1][RTW89_ETSI][27] = 127,
+	[0][1][RTW89_MKK][27] = 127,
+	[0][1][RTW89_IC][27] = 127,
+	[0][1][RTW89_KCC][27] = 127,
+	[0][1][RTW89_ACMA][27] = 127,
+	[0][1][RTW89_CN][27] = 127,
+	[0][1][RTW89_UK][27] = 127,
+	[0][1][RTW89_FCC][29] = 127,
+	[0][1][RTW89_ETSI][29] = 127,
+	[0][1][RTW89_MKK][29] = 127,
+	[0][1][RTW89_IC][29] = 127,
+	[0][1][RTW89_KCC][29] = 127,
+	[0][1][RTW89_ACMA][29] = 127,
+	[0][1][RTW89_CN][29] = 127,
+	[0][1][RTW89_UK][29] = 127,
+	[0][1][RTW89_FCC][31] = 127,
+	[0][1][RTW89_ETSI][31] = 127,
+	[0][1][RTW89_MKK][31] = 127,
+	[0][1][RTW89_IC][31] = 127,
+	[0][1][RTW89_KCC][31] = 127,
+	[0][1][RTW89_ACMA][31] = 127,
+	[0][1][RTW89_CN][31] = 127,
+	[0][1][RTW89_UK][31] = 127,
+	[0][1][RTW89_FCC][33] = 127,
+	[0][1][RTW89_ETSI][33] = 127,
+	[0][1][RTW89_MKK][33] = 127,
+	[0][1][RTW89_IC][33] = 127,
+	[0][1][RTW89_KCC][33] = 127,
+	[0][1][RTW89_ACMA][33] = 127,
+	[0][1][RTW89_CN][33] = 127,
+	[0][1][RTW89_UK][33] = 127,
+	[0][1][RTW89_FCC][35] = 127,
+	[0][1][RTW89_ETSI][35] = 127,
+	[0][1][RTW89_MKK][35] = 127,
+	[0][1][RTW89_IC][35] = 127,
+	[0][1][RTW89_KCC][35] = 127,
+	[0][1][RTW89_ACMA][35] = 127,
+	[0][1][RTW89_CN][35] = 127,
+	[0][1][RTW89_UK][35] = 127,
+	[0][1][RTW89_FCC][37] = 127,
+	[0][1][RTW89_ETSI][37] = 127,
+	[0][1][RTW89_MKK][37] = 127,
+	[0][1][RTW89_IC][37] = 127,
+	[0][1][RTW89_KCC][37] = 127,
+	[0][1][RTW89_ACMA][37] = 127,
+	[0][1][RTW89_CN][37] = 127,
+	[0][1][RTW89_UK][37] = 127,
+	[0][1][RTW89_FCC][38] = 127,
+	[0][1][RTW89_ETSI][38] = 127,
+	[0][1][RTW89_MKK][38] = 127,
+	[0][1][RTW89_IC][38] = 127,
+	[0][1][RTW89_KCC][38] = 127,
+	[0][1][RTW89_ACMA][38] = 127,
+	[0][1][RTW89_CN][38] = 42,
+	[0][1][RTW89_UK][38] = 127,
+	[0][1][RTW89_FCC][40] = 127,
+	[0][1][RTW89_ETSI][40] = 127,
+	[0][1][RTW89_MKK][40] = 127,
+	[0][1][RTW89_IC][40] = 127,
+	[0][1][RTW89_KCC][40] = 127,
+	[0][1][RTW89_ACMA][40] = 127,
+	[0][1][RTW89_CN][40] = 42,
+	[0][1][RTW89_UK][40] = 127,
+	[0][1][RTW89_FCC][42] = 127,
+	[0][1][RTW89_ETSI][42] = 127,
+	[0][1][RTW89_MKK][42] = 127,
+	[0][1][RTW89_IC][42] = 127,
+	[0][1][RTW89_KCC][42] = 127,
+	[0][1][RTW89_ACMA][42] = 127,
+	[0][1][RTW89_CN][42] = 42,
+	[0][1][RTW89_UK][42] = 127,
+	[0][1][RTW89_FCC][44] = 127,
+	[0][1][RTW89_ETSI][44] = 127,
+	[0][1][RTW89_MKK][44] = 127,
+	[0][1][RTW89_IC][44] = 127,
+	[0][1][RTW89_KCC][44] = 127,
+	[0][1][RTW89_ACMA][44] = 127,
+	[0][1][RTW89_CN][44] = 42,
+	[0][1][RTW89_UK][44] = 127,
+	[0][1][RTW89_FCC][46] = 127,
+	[0][1][RTW89_ETSI][46] = 127,
+	[0][1][RTW89_MKK][46] = 127,
+	[0][1][RTW89_IC][46] = 127,
+	[0][1][RTW89_KCC][46] = 127,
+	[0][1][RTW89_ACMA][46] = 127,
+	[0][1][RTW89_CN][46] = 42,
+	[0][1][RTW89_UK][46] = 127,
+	[0][1][RTW89_FCC][48] = 127,
+	[0][1][RTW89_ETSI][48] = 127,
+	[0][1][RTW89_MKK][48] = 127,
+	[0][1][RTW89_IC][48] = 127,
+	[0][1][RTW89_KCC][48] = 127,
+	[0][1][RTW89_ACMA][48] = 127,
+	[0][1][RTW89_CN][48] = 127,
+	[0][1][RTW89_UK][48] = 127,
+	[0][1][RTW89_FCC][50] = 127,
+	[0][1][RTW89_ETSI][50] = 127,
+	[0][1][RTW89_MKK][50] = 127,
+	[0][1][RTW89_IC][50] = 127,
+	[0][1][RTW89_KCC][50] = 127,
+	[0][1][RTW89_ACMA][50] = 127,
+	[0][1][RTW89_CN][50] = 127,
+	[0][1][RTW89_UK][50] = 127,
+	[0][1][RTW89_FCC][52] = 127,
+	[0][1][RTW89_ETSI][52] = 127,
+	[0][1][RTW89_MKK][52] = 127,
+	[0][1][RTW89_IC][52] = 127,
+	[0][1][RTW89_KCC][52] = 127,
+	[0][1][RTW89_ACMA][52] = 127,
+	[0][1][RTW89_CN][52] = 127,
+	[0][1][RTW89_UK][52] = 127,
+	[1][0][RTW89_FCC][0] = 64,
+	[1][0][RTW89_ETSI][0] = 34,
+	[1][0][RTW89_MKK][0] = 38,
+	[1][0][RTW89_IC][0] = 38,
+	[1][0][RTW89_KCC][0] = 52,
+	[1][0][RTW89_ACMA][0] = 34,
+	[1][0][RTW89_CN][0] = 26,
+	[1][0][RTW89_UK][0] = 34,
+	[1][0][RTW89_FCC][2] = 66,
+	[1][0][RTW89_ETSI][2] = 34,
+	[1][0][RTW89_MKK][2] = 38,
+	[1][0][RTW89_IC][2] = 38,
+	[1][0][RTW89_KCC][2] = 52,
+	[1][0][RTW89_ACMA][2] = 34,
+	[1][0][RTW89_CN][2] = 26,
+	[1][0][RTW89_UK][2] = 34,
+	[1][0][RTW89_FCC][4] = 60,
+	[1][0][RTW89_ETSI][4] = 34,
+	[1][0][RTW89_MKK][4] = 36,
+	[1][0][RTW89_IC][4] = 38,
+	[1][0][RTW89_KCC][4] = 52,
+	[1][0][RTW89_ACMA][4] = 34,
+	[1][0][RTW89_CN][4] = 26,
+	[1][0][RTW89_UK][4] = 34,
+	[1][0][RTW89_FCC][6] = 60,
+	[1][0][RTW89_ETSI][6] = 34,
+	[1][0][RTW89_MKK][6] = 36,
+	[1][0][RTW89_IC][6] = 38,
+	[1][0][RTW89_KCC][6] = 32,
+	[1][0][RTW89_ACMA][6] = 34,
+	[1][0][RTW89_CN][6] = 26,
+	[1][0][RTW89_UK][6] = 34,
+	[1][0][RTW89_FCC][8] = 62,
+	[1][0][RTW89_ETSI][8] = 34,
+	[1][0][RTW89_MKK][8] = 38,
+	[1][0][RTW89_IC][8] = 62,
+	[1][0][RTW89_KCC][8] = 52,
+	[1][0][RTW89_ACMA][8] = 34,
+	[1][0][RTW89_CN][8] = 26,
+	[1][0][RTW89_UK][8] = 34,
+	[1][0][RTW89_FCC][10] = 62,
+	[1][0][RTW89_ETSI][10] = 34,
+	[1][0][RTW89_MKK][10] = 38,
+	[1][0][RTW89_IC][10] = 62,
+	[1][0][RTW89_KCC][10] = 52,
+	[1][0][RTW89_ACMA][10] = 34,
+	[1][0][RTW89_CN][10] = 26,
+	[1][0][RTW89_UK][10] = 34,
+	[1][0][RTW89_FCC][12] = 62,
+	[1][0][RTW89_ETSI][12] = 34,
+	[1][0][RTW89_MKK][12] = 38,
+	[1][0][RTW89_IC][12] = 62,
+	[1][0][RTW89_KCC][12] = 54,
+	[1][0][RTW89_ACMA][12] = 34,
+	[1][0][RTW89_CN][12] = 26,
+	[1][0][RTW89_UK][12] = 34,
+	[1][0][RTW89_FCC][14] = 62,
+	[1][0][RTW89_ETSI][14] = 34,
+	[1][0][RTW89_MKK][14] = 38,
+	[1][0][RTW89_IC][14] = 62,
+	[1][0][RTW89_KCC][14] = 54,
+	[1][0][RTW89_ACMA][14] = 34,
+	[1][0][RTW89_CN][14] = 26,
+	[1][0][RTW89_UK][14] = 34,
+	[1][0][RTW89_FCC][15] = 60,
+	[1][0][RTW89_ETSI][15] = 34,
+	[1][0][RTW89_MKK][15] = 58,
+	[1][0][RTW89_IC][15] = 60,
+	[1][0][RTW89_KCC][15] = 54,
+	[1][0][RTW89_ACMA][15] = 34,
+	[1][0][RTW89_CN][15] = 127,
+	[1][0][RTW89_UK][15] = 34,
+	[1][0][RTW89_FCC][17] = 60,
+	[1][0][RTW89_ETSI][17] = 34,
+	[1][0][RTW89_MKK][17] = 58,
+	[1][0][RTW89_IC][17] = 60,
+	[1][0][RTW89_KCC][17] = 54,
+	[1][0][RTW89_ACMA][17] = 34,
+	[1][0][RTW89_CN][17] = 127,
+	[1][0][RTW89_UK][17] = 34,
+	[1][0][RTW89_FCC][19] = 62,
+	[1][0][RTW89_ETSI][19] = 34,
+	[1][0][RTW89_MKK][19] = 58,
+	[1][0][RTW89_IC][19] = 62,
+	[1][0][RTW89_KCC][19] = 54,
+	[1][0][RTW89_ACMA][19] = 34,
+	[1][0][RTW89_CN][19] = 127,
+	[1][0][RTW89_UK][19] = 34,
+	[1][0][RTW89_FCC][21] = 62,
+	[1][0][RTW89_ETSI][21] = 34,
+	[1][0][RTW89_MKK][21] = 58,
+	[1][0][RTW89_IC][21] = 62,
+	[1][0][RTW89_KCC][21] = 54,
+	[1][0][RTW89_ACMA][21] = 34,
+	[1][0][RTW89_CN][21] = 127,
+	[1][0][RTW89_UK][21] = 34,
+	[1][0][RTW89_FCC][23] = 62,
+	[1][0][RTW89_ETSI][23] = 34,
+	[1][0][RTW89_MKK][23] = 58,
+	[1][0][RTW89_IC][23] = 62,
+	[1][0][RTW89_KCC][23] = 54,
+	[1][0][RTW89_ACMA][23] = 34,
+	[1][0][RTW89_CN][23] = 127,
+	[1][0][RTW89_UK][23] = 34,
+	[1][0][RTW89_FCC][25] = 62,
+	[1][0][RTW89_ETSI][25] = 34,
+	[1][0][RTW89_MKK][25] = 58,
+	[1][0][RTW89_IC][25] = 127,
+	[1][0][RTW89_KCC][25] = 54,
+	[1][0][RTW89_ACMA][25] = 127,
+	[1][0][RTW89_CN][25] = 127,
+	[1][0][RTW89_UK][25] = 34,
+	[1][0][RTW89_FCC][27] = 62,
+	[1][0][RTW89_ETSI][27] = 34,
+	[1][0][RTW89_MKK][27] = 58,
+	[1][0][RTW89_IC][27] = 127,
+	[1][0][RTW89_KCC][27] = 54,
+	[1][0][RTW89_ACMA][27] = 127,
+	[1][0][RTW89_CN][27] = 127,
+	[1][0][RTW89_UK][27] = 34,
+	[1][0][RTW89_FCC][29] = 62,
+	[1][0][RTW89_ETSI][29] = 34,
+	[1][0][RTW89_MKK][29] = 58,
+	[1][0][RTW89_IC][29] = 127,
+	[1][0][RTW89_KCC][29] = 54,
+	[1][0][RTW89_ACMA][29] = 127,
+	[1][0][RTW89_CN][29] = 127,
+	[1][0][RTW89_UK][29] = 34,
+	[1][0][RTW89_FCC][31] = 62,
+	[1][0][RTW89_ETSI][31] = 34,
+	[1][0][RTW89_MKK][31] = 58,
+	[1][0][RTW89_IC][31] = 64,
+	[1][0][RTW89_KCC][31] = 54,
+	[1][0][RTW89_ACMA][31] = 34,
+	[1][0][RTW89_CN][31] = 127,
+	[1][0][RTW89_UK][31] = 34,
+	[1][0][RTW89_FCC][33] = 64,
+	[1][0][RTW89_ETSI][33] = 34,
+	[1][0][RTW89_MKK][33] = 58,
+	[1][0][RTW89_IC][33] = 64,
+	[1][0][RTW89_KCC][33] = 54,
+	[1][0][RTW89_ACMA][33] = 34,
+	[1][0][RTW89_CN][33] = 127,
+	[1][0][RTW89_UK][33] = 34,
+	[1][0][RTW89_FCC][35] = 64,
+	[1][0][RTW89_ETSI][35] = 34,
+	[1][0][RTW89_MKK][35] = 58,
+	[1][0][RTW89_IC][35] = 64,
+	[1][0][RTW89_KCC][35] = 54,
+	[1][0][RTW89_ACMA][35] = 34,
+	[1][0][RTW89_CN][35] = 127,
+	[1][0][RTW89_UK][35] = 34,
+	[1][0][RTW89_FCC][37] = 76,
+	[1][0][RTW89_ETSI][37] = 127,
+	[1][0][RTW89_MKK][37] = 56,
+	[1][0][RTW89_IC][37] = 76,
+	[1][0][RTW89_KCC][37] = 54,
+	[1][0][RTW89_ACMA][37] = 62,
+	[1][0][RTW89_CN][37] = 127,
+	[1][0][RTW89_UK][37] = 62,
+	[1][0][RTW89_FCC][38] = 82,
+	[1][0][RTW89_ETSI][38] = 28,
+	[1][0][RTW89_MKK][38] = 127,
+	[1][0][RTW89_IC][38] = 82,
+	[1][0][RTW89_KCC][38] = 54,
+	[1][0][RTW89_ACMA][38] = 84,
+	[1][0][RTW89_CN][38] = 66,
+	[1][0][RTW89_UK][38] = 34,
+	[1][0][RTW89_FCC][40] = 82,
+	[1][0][RTW89_ETSI][40] = 28,
+	[1][0][RTW89_MKK][40] = 127,
+	[1][0][RTW89_IC][40] = 82,
+	[1][0][RTW89_KCC][40] = 54,
+	[1][0][RTW89_ACMA][40] = 84,
+	[1][0][RTW89_CN][40] = 66,
+	[1][0][RTW89_UK][40] = 34,
+	[1][0][RTW89_FCC][42] = 78,
+	[1][0][RTW89_ETSI][42] = 28,
+	[1][0][RTW89_MKK][42] = 127,
+	[1][0][RTW89_IC][42] = 78,
+	[1][0][RTW89_KCC][42] = 54,
+	[1][0][RTW89_ACMA][42] = 84,
+	[1][0][RTW89_CN][42] = 66,
+	[1][0][RTW89_UK][42] = 34,
+	[1][0][RTW89_FCC][44] = 82,
+	[1][0][RTW89_ETSI][44] = 28,
+	[1][0][RTW89_MKK][44] = 127,
+	[1][0][RTW89_IC][44] = 82,
+	[1][0][RTW89_KCC][44] = 54,
+	[1][0][RTW89_ACMA][44] = 84,
+	[1][0][RTW89_CN][44] = 66,
+	[1][0][RTW89_UK][44] = 34,
+	[1][0][RTW89_FCC][46] = 82,
+	[1][0][RTW89_ETSI][46] = 28,
+	[1][0][RTW89_MKK][46] = 127,
+	[1][0][RTW89_IC][46] = 82,
+	[1][0][RTW89_KCC][46] = 54,
+	[1][0][RTW89_ACMA][46] = 84,
+	[1][0][RTW89_CN][46] = 66,
+	[1][0][RTW89_UK][46] = 34,
+	[1][0][RTW89_FCC][48] = 52,
+	[1][0][RTW89_ETSI][48] = 127,
+	[1][0][RTW89_MKK][48] = 127,
+	[1][0][RTW89_IC][48] = 127,
+	[1][0][RTW89_KCC][48] = 127,
+	[1][0][RTW89_ACMA][48] = 127,
+	[1][0][RTW89_CN][48] = 127,
+	[1][0][RTW89_UK][48] = 127,
+	[1][0][RTW89_FCC][50] = 52,
+	[1][0][RTW89_ETSI][50] = 127,
+	[1][0][RTW89_MKK][50] = 127,
+	[1][0][RTW89_IC][50] = 127,
+	[1][0][RTW89_KCC][50] = 127,
+	[1][0][RTW89_ACMA][50] = 127,
+	[1][0][RTW89_CN][50] = 127,
+	[1][0][RTW89_UK][50] = 127,
+	[1][0][RTW89_FCC][52] = 50,
+	[1][0][RTW89_ETSI][52] = 127,
+	[1][0][RTW89_MKK][52] = 127,
+	[1][0][RTW89_IC][52] = 127,
+	[1][0][RTW89_KCC][52] = 127,
+	[1][0][RTW89_ACMA][52] = 127,
+	[1][0][RTW89_CN][52] = 127,
+	[1][0][RTW89_UK][52] = 127,
+	[1][1][RTW89_FCC][0] = 127,
+	[1][1][RTW89_ETSI][0] = 127,
+	[1][1][RTW89_MKK][0] = 127,
+	[1][1][RTW89_IC][0] = 127,
+	[1][1][RTW89_KCC][0] = 127,
+	[1][1][RTW89_ACMA][0] = 127,
+	[1][1][RTW89_CN][0] = 14,
+	[1][1][RTW89_UK][0] = 127,
+	[1][1][RTW89_FCC][2] = 127,
+	[1][1][RTW89_ETSI][2] = 127,
+	[1][1][RTW89_MKK][2] = 127,
+	[1][1][RTW89_IC][2] = 127,
+	[1][1][RTW89_KCC][2] = 127,
+	[1][1][RTW89_ACMA][2] = 127,
+	[1][1][RTW89_CN][2] = 14,
+	[1][1][RTW89_UK][2] = 127,
+	[1][1][RTW89_FCC][4] = 127,
+	[1][1][RTW89_ETSI][4] = 127,
+	[1][1][RTW89_MKK][4] = 127,
+	[1][1][RTW89_IC][4] = 127,
+	[1][1][RTW89_KCC][4] = 127,
+	[1][1][RTW89_ACMA][4] = 127,
+	[1][1][RTW89_CN][4] = 14,
+	[1][1][RTW89_UK][4] = 127,
+	[1][1][RTW89_FCC][6] = 127,
+	[1][1][RTW89_ETSI][6] = 127,
+	[1][1][RTW89_MKK][6] = 127,
+	[1][1][RTW89_IC][6] = 127,
+	[1][1][RTW89_KCC][6] = 127,
+	[1][1][RTW89_ACMA][6] = 127,
+	[1][1][RTW89_CN][6] = 14,
+	[1][1][RTW89_UK][6] = 127,
+	[1][1][RTW89_FCC][8] = 127,
+	[1][1][RTW89_ETSI][8] = 127,
+	[1][1][RTW89_MKK][8] = 127,
+	[1][1][RTW89_IC][8] = 127,
+	[1][1][RTW89_KCC][8] = 127,
+	[1][1][RTW89_ACMA][8] = 127,
+	[1][1][RTW89_CN][8] = 14,
+	[1][1][RTW89_UK][8] = 127,
+	[1][1][RTW89_FCC][10] = 127,
+	[1][1][RTW89_ETSI][10] = 127,
+	[1][1][RTW89_MKK][10] = 127,
+	[1][1][RTW89_IC][10] = 127,
+	[1][1][RTW89_KCC][10] = 127,
+	[1][1][RTW89_ACMA][10] = 127,
+	[1][1][RTW89_CN][10] = 14,
+	[1][1][RTW89_UK][10] = 127,
+	[1][1][RTW89_FCC][12] = 127,
+	[1][1][RTW89_ETSI][12] = 127,
+	[1][1][RTW89_MKK][12] = 127,
+	[1][1][RTW89_IC][12] = 127,
+	[1][1][RTW89_KCC][12] = 127,
+	[1][1][RTW89_ACMA][12] = 127,
+	[1][1][RTW89_CN][12] = 14,
+	[1][1][RTW89_UK][12] = 127,
+	[1][1][RTW89_FCC][14] = 127,
+	[1][1][RTW89_ETSI][14] = 127,
+	[1][1][RTW89_MKK][14] = 127,
+	[1][1][RTW89_IC][14] = 127,
+	[1][1][RTW89_KCC][14] = 127,
+	[1][1][RTW89_ACMA][14] = 127,
+	[1][1][RTW89_CN][14] = 14,
+	[1][1][RTW89_UK][14] = 127,
+	[1][1][RTW89_FCC][15] = 127,
+	[1][1][RTW89_ETSI][15] = 127,
+	[1][1][RTW89_MKK][15] = 127,
+	[1][1][RTW89_IC][15] = 127,
+	[1][1][RTW89_KCC][15] = 127,
+	[1][1][RTW89_ACMA][15] = 127,
+	[1][1][RTW89_CN][15] = 127,
+	[1][1][RTW89_UK][15] = 127,
+	[1][1][RTW89_FCC][17] = 127,
+	[1][1][RTW89_ETSI][17] = 127,
+	[1][1][RTW89_MKK][17] = 127,
+	[1][1][RTW89_IC][17] = 127,
+	[1][1][RTW89_KCC][17] = 127,
+	[1][1][RTW89_ACMA][17] = 127,
+	[1][1][RTW89_CN][17] = 127,
+	[1][1][RTW89_UK][17] = 127,
+	[1][1][RTW89_FCC][19] = 127,
+	[1][1][RTW89_ETSI][19] = 127,
+	[1][1][RTW89_MKK][19] = 127,
+	[1][1][RTW89_IC][19] = 127,
+	[1][1][RTW89_KCC][19] = 127,
+	[1][1][RTW89_ACMA][19] = 127,
+	[1][1][RTW89_CN][19] = 127,
+	[1][1][RTW89_UK][19] = 127,
+	[1][1][RTW89_FCC][21] = 127,
+	[1][1][RTW89_ETSI][21] = 127,
+	[1][1][RTW89_MKK][21] = 127,
+	[1][1][RTW89_IC][21] = 127,
+	[1][1][RTW89_KCC][21] = 127,
+	[1][1][RTW89_ACMA][21] = 127,
+	[1][1][RTW89_CN][21] = 127,
+	[1][1][RTW89_UK][21] = 127,
+	[1][1][RTW89_FCC][23] = 127,
+	[1][1][RTW89_ETSI][23] = 127,
+	[1][1][RTW89_MKK][23] = 127,
+	[1][1][RTW89_IC][23] = 127,
+	[1][1][RTW89_KCC][23] = 127,
+	[1][1][RTW89_ACMA][23] = 127,
+	[1][1][RTW89_CN][23] = 127,
+	[1][1][RTW89_UK][23] = 127,
+	[1][1][RTW89_FCC][25] = 127,
+	[1][1][RTW89_ETSI][25] = 127,
+	[1][1][RTW89_MKK][25] = 127,
+	[1][1][RTW89_IC][25] = 127,
+	[1][1][RTW89_KCC][25] = 127,
+	[1][1][RTW89_ACMA][25] = 127,
+	[1][1][RTW89_CN][25] = 127,
+	[1][1][RTW89_UK][25] = 127,
+	[1][1][RTW89_FCC][27] = 127,
+	[1][1][RTW89_ETSI][27] = 127,
+	[1][1][RTW89_MKK][27] = 127,
+	[1][1][RTW89_IC][27] = 127,
+	[1][1][RTW89_KCC][27] = 127,
+	[1][1][RTW89_ACMA][27] = 127,
+	[1][1][RTW89_CN][27] = 127,
+	[1][1][RTW89_UK][27] = 127,
+	[1][1][RTW89_FCC][29] = 127,
+	[1][1][RTW89_ETSI][29] = 127,
+	[1][1][RTW89_MKK][29] = 127,
+	[1][1][RTW89_IC][29] = 127,
+	[1][1][RTW89_KCC][29] = 127,
+	[1][1][RTW89_ACMA][29] = 127,
+	[1][1][RTW89_CN][29] = 127,
+	[1][1][RTW89_UK][29] = 127,
+	[1][1][RTW89_FCC][31] = 127,
+	[1][1][RTW89_ETSI][31] = 127,
+	[1][1][RTW89_MKK][31] = 127,
+	[1][1][RTW89_IC][31] = 127,
+	[1][1][RTW89_KCC][31] = 127,
+	[1][1][RTW89_ACMA][31] = 127,
+	[1][1][RTW89_CN][31] = 127,
+	[1][1][RTW89_UK][31] = 127,
+	[1][1][RTW89_FCC][33] = 127,
+	[1][1][RTW89_ETSI][33] = 127,
+	[1][1][RTW89_MKK][33] = 127,
+	[1][1][RTW89_IC][33] = 127,
+	[1][1][RTW89_KCC][33] = 127,
+	[1][1][RTW89_ACMA][33] = 127,
+	[1][1][RTW89_CN][33] = 127,
+	[1][1][RTW89_UK][33] = 127,
+	[1][1][RTW89_FCC][35] = 127,
+	[1][1][RTW89_ETSI][35] = 127,
+	[1][1][RTW89_MKK][35] = 127,
+	[1][1][RTW89_IC][35] = 127,
+	[1][1][RTW89_KCC][35] = 127,
+	[1][1][RTW89_ACMA][35] = 127,
+	[1][1][RTW89_CN][35] = 127,
+	[1][1][RTW89_UK][35] = 127,
+	[1][1][RTW89_FCC][37] = 127,
+	[1][1][RTW89_ETSI][37] = 127,
+	[1][1][RTW89_MKK][37] = 127,
+	[1][1][RTW89_IC][37] = 127,
+	[1][1][RTW89_KCC][37] = 127,
+	[1][1][RTW89_ACMA][37] = 127,
+	[1][1][RTW89_CN][37] = 127,
+	[1][1][RTW89_UK][37] = 127,
+	[1][1][RTW89_FCC][38] = 127,
+	[1][1][RTW89_ETSI][38] = 127,
+	[1][1][RTW89_MKK][38] = 127,
+	[1][1][RTW89_IC][38] = 127,
+	[1][1][RTW89_KCC][38] = 127,
+	[1][1][RTW89_ACMA][38] = 127,
+	[1][1][RTW89_CN][38] = 54,
+	[1][1][RTW89_UK][38] = 127,
+	[1][1][RTW89_FCC][40] = 127,
+	[1][1][RTW89_ETSI][40] = 127,
+	[1][1][RTW89_MKK][40] = 127,
+	[1][1][RTW89_IC][40] = 127,
+	[1][1][RTW89_KCC][40] = 127,
+	[1][1][RTW89_ACMA][40] = 127,
+	[1][1][RTW89_CN][40] = 54,
+	[1][1][RTW89_UK][40] = 127,
+	[1][1][RTW89_FCC][42] = 127,
+	[1][1][RTW89_ETSI][42] = 127,
+	[1][1][RTW89_MKK][42] = 127,
+	[1][1][RTW89_IC][42] = 127,
+	[1][1][RTW89_KCC][42] = 127,
+	[1][1][RTW89_ACMA][42] = 127,
+	[1][1][RTW89_CN][42] = 54,
+	[1][1][RTW89_UK][42] = 127,
+	[1][1][RTW89_FCC][44] = 127,
+	[1][1][RTW89_ETSI][44] = 127,
+	[1][1][RTW89_MKK][44] = 127,
+	[1][1][RTW89_IC][44] = 127,
+	[1][1][RTW89_KCC][44] = 127,
+	[1][1][RTW89_ACMA][44] = 127,
+	[1][1][RTW89_CN][44] = 54,
+	[1][1][RTW89_UK][44] = 127,
+	[1][1][RTW89_FCC][46] = 127,
+	[1][1][RTW89_ETSI][46] = 127,
+	[1][1][RTW89_MKK][46] = 127,
+	[1][1][RTW89_IC][46] = 127,
+	[1][1][RTW89_KCC][46] = 127,
+	[1][1][RTW89_ACMA][46] = 127,
+	[1][1][RTW89_CN][46] = 54,
+	[1][1][RTW89_UK][46] = 127,
+	[1][1][RTW89_FCC][48] = 127,
+	[1][1][RTW89_ETSI][48] = 127,
+	[1][1][RTW89_MKK][48] = 127,
+	[1][1][RTW89_IC][48] = 127,
+	[1][1][RTW89_KCC][48] = 127,
+	[1][1][RTW89_ACMA][48] = 127,
+	[1][1][RTW89_CN][48] = 127,
+	[1][1][RTW89_UK][48] = 127,
+	[1][1][RTW89_FCC][50] = 127,
+	[1][1][RTW89_ETSI][50] = 127,
+	[1][1][RTW89_MKK][50] = 127,
+	[1][1][RTW89_IC][50] = 127,
+	[1][1][RTW89_KCC][50] = 127,
+	[1][1][RTW89_ACMA][50] = 127,
+	[1][1][RTW89_CN][50] = 127,
+	[1][1][RTW89_UK][50] = 127,
+	[1][1][RTW89_FCC][52] = 127,
+	[1][1][RTW89_ETSI][52] = 127,
+	[1][1][RTW89_MKK][52] = 127,
+	[1][1][RTW89_IC][52] = 127,
+	[1][1][RTW89_KCC][52] = 127,
+	[1][1][RTW89_ACMA][52] = 127,
+	[1][1][RTW89_CN][52] = 127,
+	[1][1][RTW89_UK][52] = 127,
+	[2][0][RTW89_FCC][0] = 76,
+	[2][0][RTW89_ETSI][0] = 46,
+	[2][0][RTW89_MKK][0] = 48,
+	[2][0][RTW89_IC][0] = 50,
+	[2][0][RTW89_KCC][0] = 64,
+	[2][0][RTW89_ACMA][0] = 46,
+	[2][0][RTW89_CN][0] = 40,
+	[2][0][RTW89_UK][0] = 46,
+	[2][0][RTW89_FCC][2] = 72,
+	[2][0][RTW89_ETSI][2] = 46,
+	[2][0][RTW89_MKK][2] = 48,
+	[2][0][RTW89_IC][2] = 48,
+	[2][0][RTW89_KCC][2] = 64,
+	[2][0][RTW89_ACMA][2] = 46,
+	[2][0][RTW89_CN][2] = 40,
+	[2][0][RTW89_UK][2] = 46,
+	[2][0][RTW89_FCC][4] = 74,
+	[2][0][RTW89_ETSI][4] = 46,
+	[2][0][RTW89_MKK][4] = 48,
+	[2][0][RTW89_IC][4] = 48,
+	[2][0][RTW89_KCC][4] = 64,
+	[2][0][RTW89_ACMA][4] = 46,
+	[2][0][RTW89_CN][4] = 40,
+	[2][0][RTW89_UK][4] = 46,
+	[2][0][RTW89_FCC][6] = 74,
+	[2][0][RTW89_ETSI][6] = 46,
+	[2][0][RTW89_MKK][6] = 48,
+	[2][0][RTW89_IC][6] = 48,
+	[2][0][RTW89_KCC][6] = 40,
+	[2][0][RTW89_ACMA][6] = 46,
+	[2][0][RTW89_CN][6] = 40,
+	[2][0][RTW89_UK][6] = 46,
+	[2][0][RTW89_FCC][8] = 72,
+	[2][0][RTW89_ETSI][8] = 46,
+	[2][0][RTW89_MKK][8] = 48,
+	[2][0][RTW89_IC][8] = 64,
+	[2][0][RTW89_KCC][8] = 66,
+	[2][0][RTW89_ACMA][8] = 46,
+	[2][0][RTW89_CN][8] = 40,
+	[2][0][RTW89_UK][8] = 46,
+	[2][0][RTW89_FCC][10] = 72,
+	[2][0][RTW89_ETSI][10] = 46,
+	[2][0][RTW89_MKK][10] = 48,
+	[2][0][RTW89_IC][10] = 64,
+	[2][0][RTW89_KCC][10] = 66,
+	[2][0][RTW89_ACMA][10] = 46,
+	[2][0][RTW89_CN][10] = 40,
+	[2][0][RTW89_UK][10] = 46,
+	[2][0][RTW89_FCC][12] = 74,
+	[2][0][RTW89_ETSI][12] = 46,
+	[2][0][RTW89_MKK][12] = 48,
+	[2][0][RTW89_IC][12] = 64,
+	[2][0][RTW89_KCC][12] = 64,
+	[2][0][RTW89_ACMA][12] = 46,
+	[2][0][RTW89_CN][12] = 40,
+	[2][0][RTW89_UK][12] = 46,
+	[2][0][RTW89_FCC][14] = 80,
+	[2][0][RTW89_ETSI][14] = 46,
+	[2][0][RTW89_MKK][14] = 48,
+	[2][0][RTW89_IC][14] = 64,
+	[2][0][RTW89_KCC][14] = 64,
+	[2][0][RTW89_ACMA][14] = 46,
+	[2][0][RTW89_CN][14] = 40,
+	[2][0][RTW89_UK][14] = 46,
+	[2][0][RTW89_FCC][15] = 72,
+	[2][0][RTW89_ETSI][15] = 46,
+	[2][0][RTW89_MKK][15] = 70,
+	[2][0][RTW89_IC][15] = 72,
+	[2][0][RTW89_KCC][15] = 66,
+	[2][0][RTW89_ACMA][15] = 46,
+	[2][0][RTW89_CN][15] = 127,
+	[2][0][RTW89_UK][15] = 46,
+	[2][0][RTW89_FCC][17] = 72,
+	[2][0][RTW89_ETSI][17] = 46,
+	[2][0][RTW89_MKK][17] = 70,
+	[2][0][RTW89_IC][17] = 72,
+	[2][0][RTW89_KCC][17] = 66,
+	[2][0][RTW89_ACMA][17] = 46,
+	[2][0][RTW89_CN][17] = 127,
+	[2][0][RTW89_UK][17] = 46,
+	[2][0][RTW89_FCC][19] = 68,
+	[2][0][RTW89_ETSI][19] = 46,
+	[2][0][RTW89_MKK][19] = 70,
+	[2][0][RTW89_IC][19] = 68,
+	[2][0][RTW89_KCC][19] = 66,
+	[2][0][RTW89_ACMA][19] = 46,
+	[2][0][RTW89_CN][19] = 127,
+	[2][0][RTW89_UK][19] = 46,
+	[2][0][RTW89_FCC][21] = 68,
+	[2][0][RTW89_ETSI][21] = 46,
+	[2][0][RTW89_MKK][21] = 70,
+	[2][0][RTW89_IC][21] = 68,
+	[2][0][RTW89_KCC][21] = 66,
+	[2][0][RTW89_ACMA][21] = 46,
+	[2][0][RTW89_CN][21] = 127,
+	[2][0][RTW89_UK][21] = 46,
+	[2][0][RTW89_FCC][23] = 68,
+	[2][0][RTW89_ETSI][23] = 46,
+	[2][0][RTW89_MKK][23] = 70,
+	[2][0][RTW89_IC][23] = 68,
+	[2][0][RTW89_KCC][23] = 66,
+	[2][0][RTW89_ACMA][23] = 46,
+	[2][0][RTW89_CN][23] = 127,
+	[2][0][RTW89_UK][23] = 46,
+	[2][0][RTW89_FCC][25] = 68,
+	[2][0][RTW89_ETSI][25] = 46,
+	[2][0][RTW89_MKK][25] = 70,
+	[2][0][RTW89_IC][25] = 127,
+	[2][0][RTW89_KCC][25] = 66,
+	[2][0][RTW89_ACMA][25] = 127,
+	[2][0][RTW89_CN][25] = 127,
+	[2][0][RTW89_UK][25] = 46,
+	[2][0][RTW89_FCC][27] = 68,
+	[2][0][RTW89_ETSI][27] = 46,
+	[2][0][RTW89_MKK][27] = 70,
+	[2][0][RTW89_IC][27] = 127,
+	[2][0][RTW89_KCC][27] = 64,
+	[2][0][RTW89_ACMA][27] = 127,
+	[2][0][RTW89_CN][27] = 127,
+	[2][0][RTW89_UK][27] = 46,
+	[2][0][RTW89_FCC][29] = 68,
+	[2][0][RTW89_ETSI][29] = 46,
+	[2][0][RTW89_MKK][29] = 70,
+	[2][0][RTW89_IC][29] = 127,
+	[2][0][RTW89_KCC][29] = 64,
+	[2][0][RTW89_ACMA][29] = 127,
+	[2][0][RTW89_CN][29] = 127,
+	[2][0][RTW89_UK][29] = 46,
+	[2][0][RTW89_FCC][31] = 68,
+	[2][0][RTW89_ETSI][31] = 46,
+	[2][0][RTW89_MKK][31] = 70,
+	[2][0][RTW89_IC][31] = 70,
+	[2][0][RTW89_KCC][31] = 64,
+	[2][0][RTW89_ACMA][31] = 46,
+	[2][0][RTW89_CN][31] = 127,
+	[2][0][RTW89_UK][31] = 46,
+	[2][0][RTW89_FCC][33] = 70,
+	[2][0][RTW89_ETSI][33] = 46,
+	[2][0][RTW89_MKK][33] = 70,
+	[2][0][RTW89_IC][33] = 70,
+	[2][0][RTW89_KCC][33] = 64,
+	[2][0][RTW89_ACMA][33] = 46,
+	[2][0][RTW89_CN][33] = 127,
+	[2][0][RTW89_UK][33] = 46,
+	[2][0][RTW89_FCC][35] = 70,
+	[2][0][RTW89_ETSI][35] = 46,
+	[2][0][RTW89_MKK][35] = 70,
+	[2][0][RTW89_IC][35] = 70,
+	[2][0][RTW89_KCC][35] = 64,
+	[2][0][RTW89_ACMA][35] = 46,
+	[2][0][RTW89_CN][35] = 127,
+	[2][0][RTW89_UK][35] = 46,
+	[2][0][RTW89_FCC][37] = 84,
+	[2][0][RTW89_ETSI][37] = 127,
+	[2][0][RTW89_MKK][37] = 68,
+	[2][0][RTW89_IC][37] = 84,
+	[2][0][RTW89_KCC][37] = 66,
+	[2][0][RTW89_ACMA][37] = 74,
+	[2][0][RTW89_CN][37] = 127,
+	[2][0][RTW89_UK][37] = 74,
+	[2][0][RTW89_FCC][38] = 84,
+	[2][0][RTW89_ETSI][38] = 28,
+	[2][0][RTW89_MKK][38] = 127,
+	[2][0][RTW89_IC][38] = 84,
+	[2][0][RTW89_KCC][38] = 64,
+	[2][0][RTW89_ACMA][38] = 84,
+	[2][0][RTW89_CN][38] = 68,
+	[2][0][RTW89_UK][38] = 46,
+	[2][0][RTW89_FCC][40] = 84,
+	[2][0][RTW89_ETSI][40] = 28,
+	[2][0][RTW89_MKK][40] = 127,
+	[2][0][RTW89_IC][40] = 84,
+	[2][0][RTW89_KCC][40] = 64,
+	[2][0][RTW89_ACMA][40] = 84,
+	[2][0][RTW89_CN][40] = 68,
+	[2][0][RTW89_UK][40] = 46,
+	[2][0][RTW89_FCC][42] = 78,
+	[2][0][RTW89_ETSI][42] = 28,
+	[2][0][RTW89_MKK][42] = 127,
+	[2][0][RTW89_IC][42] = 78,
+	[2][0][RTW89_KCC][42] = 66,
+	[2][0][RTW89_ACMA][42] = 84,
+	[2][0][RTW89_CN][42] = 68,
+	[2][0][RTW89_UK][42] = 46,
+	[2][0][RTW89_FCC][44] = 80,
+	[2][0][RTW89_ETSI][44] = 28,
+	[2][0][RTW89_MKK][44] = 127,
+	[2][0][RTW89_IC][44] = 80,
+	[2][0][RTW89_KCC][44] = 66,
+	[2][0][RTW89_ACMA][44] = 84,
+	[2][0][RTW89_CN][44] = 68,
+	[2][0][RTW89_UK][44] = 46,
+	[2][0][RTW89_FCC][46] = 80,
+	[2][0][RTW89_ETSI][46] = 28,
+	[2][0][RTW89_MKK][46] = 127,
+	[2][0][RTW89_IC][46] = 80,
+	[2][0][RTW89_KCC][46] = 66,
+	[2][0][RTW89_ACMA][46] = 84,
+	[2][0][RTW89_CN][46] = 68,
+	[2][0][RTW89_UK][46] = 46,
+	[2][0][RTW89_FCC][48] = 62,
+	[2][0][RTW89_ETSI][48] = 127,
+	[2][0][RTW89_MKK][48] = 127,
+	[2][0][RTW89_IC][48] = 127,
+	[2][0][RTW89_KCC][48] = 127,
+	[2][0][RTW89_ACMA][48] = 127,
+	[2][0][RTW89_CN][48] = 127,
+	[2][0][RTW89_UK][48] = 127,
+	[2][0][RTW89_FCC][50] = 62,
+	[2][0][RTW89_ETSI][50] = 127,
+	[2][0][RTW89_MKK][50] = 127,
+	[2][0][RTW89_IC][50] = 127,
+	[2][0][RTW89_KCC][50] = 127,
+	[2][0][RTW89_ACMA][50] = 127,
+	[2][0][RTW89_CN][50] = 127,
+	[2][0][RTW89_UK][50] = 127,
+	[2][0][RTW89_FCC][52] = 60,
+	[2][0][RTW89_ETSI][52] = 127,
+	[2][0][RTW89_MKK][52] = 127,
+	[2][0][RTW89_IC][52] = 127,
+	[2][0][RTW89_KCC][52] = 127,
+	[2][0][RTW89_ACMA][52] = 127,
+	[2][0][RTW89_CN][52] = 127,
+	[2][0][RTW89_UK][52] = 127,
+	[2][1][RTW89_FCC][0] = 127,
+	[2][1][RTW89_ETSI][0] = 127,
+	[2][1][RTW89_MKK][0] = 127,
+	[2][1][RTW89_IC][0] = 127,
+	[2][1][RTW89_KCC][0] = 127,
+	[2][1][RTW89_ACMA][0] = 127,
+	[2][1][RTW89_CN][0] = 28,
+	[2][1][RTW89_UK][0] = 127,
+	[2][1][RTW89_FCC][2] = 127,
+	[2][1][RTW89_ETSI][2] = 127,
+	[2][1][RTW89_MKK][2] = 127,
+	[2][1][RTW89_IC][2] = 127,
+	[2][1][RTW89_KCC][2] = 127,
+	[2][1][RTW89_ACMA][2] = 127,
+	[2][1][RTW89_CN][2] = 28,
+	[2][1][RTW89_UK][2] = 127,
+	[2][1][RTW89_FCC][4] = 127,
+	[2][1][RTW89_ETSI][4] = 127,
+	[2][1][RTW89_MKK][4] = 127,
+	[2][1][RTW89_IC][4] = 127,
+	[2][1][RTW89_KCC][4] = 127,
+	[2][1][RTW89_ACMA][4] = 127,
+	[2][1][RTW89_CN][4] = 28,
+	[2][1][RTW89_UK][4] = 127,
+	[2][1][RTW89_FCC][6] = 127,
+	[2][1][RTW89_ETSI][6] = 127,
+	[2][1][RTW89_MKK][6] = 127,
+	[2][1][RTW89_IC][6] = 127,
+	[2][1][RTW89_KCC][6] = 127,
+	[2][1][RTW89_ACMA][6] = 127,
+	[2][1][RTW89_CN][6] = 28,
+	[2][1][RTW89_UK][6] = 127,
+	[2][1][RTW89_FCC][8] = 127,
+	[2][1][RTW89_ETSI][8] = 127,
+	[2][1][RTW89_MKK][8] = 127,
+	[2][1][RTW89_IC][8] = 127,
+	[2][1][RTW89_KCC][8] = 127,
+	[2][1][RTW89_ACMA][8] = 127,
+	[2][1][RTW89_CN][8] = 28,
+	[2][1][RTW89_UK][8] = 127,
+	[2][1][RTW89_FCC][10] = 127,
+	[2][1][RTW89_ETSI][10] = 127,
+	[2][1][RTW89_MKK][10] = 127,
+	[2][1][RTW89_IC][10] = 127,
+	[2][1][RTW89_KCC][10] = 127,
+	[2][1][RTW89_ACMA][10] = 127,
+	[2][1][RTW89_CN][10] = 28,
+	[2][1][RTW89_UK][10] = 127,
+	[2][1][RTW89_FCC][12] = 127,
+	[2][1][RTW89_ETSI][12] = 127,
+	[2][1][RTW89_MKK][12] = 127,
+	[2][1][RTW89_IC][12] = 127,
+	[2][1][RTW89_KCC][12] = 127,
+	[2][1][RTW89_ACMA][12] = 127,
+	[2][1][RTW89_CN][12] = 28,
+	[2][1][RTW89_UK][12] = 127,
+	[2][1][RTW89_FCC][14] = 127,
+	[2][1][RTW89_ETSI][14] = 127,
+	[2][1][RTW89_MKK][14] = 127,
+	[2][1][RTW89_IC][14] = 127,
+	[2][1][RTW89_KCC][14] = 127,
+	[2][1][RTW89_ACMA][14] = 127,
+	[2][1][RTW89_CN][14] = 28,
+	[2][1][RTW89_UK][14] = 127,
+	[2][1][RTW89_FCC][15] = 127,
+	[2][1][RTW89_ETSI][15] = 127,
+	[2][1][RTW89_MKK][15] = 127,
+	[2][1][RTW89_IC][15] = 127,
+	[2][1][RTW89_KCC][15] = 127,
+	[2][1][RTW89_ACMA][15] = 127,
+	[2][1][RTW89_CN][15] = 127,
+	[2][1][RTW89_UK][15] = 127,
+	[2][1][RTW89_FCC][17] = 127,
+	[2][1][RTW89_ETSI][17] = 127,
+	[2][1][RTW89_MKK][17] = 127,
+	[2][1][RTW89_IC][17] = 127,
+	[2][1][RTW89_KCC][17] = 127,
+	[2][1][RTW89_ACMA][17] = 127,
+	[2][1][RTW89_CN][17] = 127,
+	[2][1][RTW89_UK][17] = 127,
+	[2][1][RTW89_FCC][19] = 127,
+	[2][1][RTW89_ETSI][19] = 127,
+	[2][1][RTW89_MKK][19] = 127,
+	[2][1][RTW89_IC][19] = 127,
+	[2][1][RTW89_KCC][19] = 127,
+	[2][1][RTW89_ACMA][19] = 127,
+	[2][1][RTW89_CN][19] = 127,
+	[2][1][RTW89_UK][19] = 127,
+	[2][1][RTW89_FCC][21] = 127,
+	[2][1][RTW89_ETSI][21] = 127,
+	[2][1][RTW89_MKK][21] = 127,
+	[2][1][RTW89_IC][21] = 127,
+	[2][1][RTW89_KCC][21] = 127,
+	[2][1][RTW89_ACMA][21] = 127,
+	[2][1][RTW89_CN][21] = 127,
+	[2][1][RTW89_UK][21] = 127,
+	[2][1][RTW89_FCC][23] = 127,
+	[2][1][RTW89_ETSI][23] = 127,
+	[2][1][RTW89_MKK][23] = 127,
+	[2][1][RTW89_IC][23] = 127,
+	[2][1][RTW89_KCC][23] = 127,
+	[2][1][RTW89_ACMA][23] = 127,
+	[2][1][RTW89_CN][23] = 127,
+	[2][1][RTW89_UK][23] = 127,
+	[2][1][RTW89_FCC][25] = 127,
+	[2][1][RTW89_ETSI][25] = 127,
+	[2][1][RTW89_MKK][25] = 127,
+	[2][1][RTW89_IC][25] = 127,
+	[2][1][RTW89_KCC][25] = 127,
+	[2][1][RTW89_ACMA][25] = 127,
+	[2][1][RTW89_CN][25] = 127,
+	[2][1][RTW89_UK][25] = 127,
+	[2][1][RTW89_FCC][27] = 127,
+	[2][1][RTW89_ETSI][27] = 127,
+	[2][1][RTW89_MKK][27] = 127,
+	[2][1][RTW89_IC][27] = 127,
+	[2][1][RTW89_KCC][27] = 127,
+	[2][1][RTW89_ACMA][27] = 127,
+	[2][1][RTW89_CN][27] = 127,
+	[2][1][RTW89_UK][27] = 127,
+	[2][1][RTW89_FCC][29] = 127,
+	[2][1][RTW89_ETSI][29] = 127,
+	[2][1][RTW89_MKK][29] = 127,
+	[2][1][RTW89_IC][29] = 127,
+	[2][1][RTW89_KCC][29] = 127,
+	[2][1][RTW89_ACMA][29] = 127,
+	[2][1][RTW89_CN][29] = 127,
+	[2][1][RTW89_UK][29] = 127,
+	[2][1][RTW89_FCC][31] = 127,
+	[2][1][RTW89_ETSI][31] = 127,
+	[2][1][RTW89_MKK][31] = 127,
+	[2][1][RTW89_IC][31] = 127,
+	[2][1][RTW89_KCC][31] = 127,
+	[2][1][RTW89_ACMA][31] = 127,
+	[2][1][RTW89_CN][31] = 127,
+	[2][1][RTW89_UK][31] = 127,
+	[2][1][RTW89_FCC][33] = 127,
+	[2][1][RTW89_ETSI][33] = 127,
+	[2][1][RTW89_MKK][33] = 127,
+	[2][1][RTW89_IC][33] = 127,
+	[2][1][RTW89_KCC][33] = 127,
+	[2][1][RTW89_ACMA][33] = 127,
+	[2][1][RTW89_CN][33] = 127,
+	[2][1][RTW89_UK][33] = 127,
+	[2][1][RTW89_FCC][35] = 127,
+	[2][1][RTW89_ETSI][35] = 127,
+	[2][1][RTW89_MKK][35] = 127,
+	[2][1][RTW89_IC][35] = 127,
+	[2][1][RTW89_KCC][35] = 127,
+	[2][1][RTW89_ACMA][35] = 127,
+	[2][1][RTW89_CN][35] = 127,
+	[2][1][RTW89_UK][35] = 127,
+	[2][1][RTW89_FCC][37] = 127,
+	[2][1][RTW89_ETSI][37] = 127,
+	[2][1][RTW89_MKK][37] = 127,
+	[2][1][RTW89_IC][37] = 127,
+	[2][1][RTW89_KCC][37] = 127,
+	[2][1][RTW89_ACMA][37] = 127,
+	[2][1][RTW89_CN][37] = 127,
+	[2][1][RTW89_UK][37] = 127,
+	[2][1][RTW89_FCC][38] = 127,
+	[2][1][RTW89_ETSI][38] = 127,
+	[2][1][RTW89_MKK][38] = 127,
+	[2][1][RTW89_IC][38] = 127,
+	[2][1][RTW89_KCC][38] = 127,
+	[2][1][RTW89_ACMA][38] = 127,
+	[2][1][RTW89_CN][38] = 56,
+	[2][1][RTW89_UK][38] = 127,
+	[2][1][RTW89_FCC][40] = 127,
+	[2][1][RTW89_ETSI][40] = 127,
+	[2][1][RTW89_MKK][40] = 127,
+	[2][1][RTW89_IC][40] = 127,
+	[2][1][RTW89_KCC][40] = 127,
+	[2][1][RTW89_ACMA][40] = 127,
+	[2][1][RTW89_CN][40] = 56,
+	[2][1][RTW89_UK][40] = 127,
+	[2][1][RTW89_FCC][42] = 127,
+	[2][1][RTW89_ETSI][42] = 127,
+	[2][1][RTW89_MKK][42] = 127,
+	[2][1][RTW89_IC][42] = 127,
+	[2][1][RTW89_KCC][42] = 127,
+	[2][1][RTW89_ACMA][42] = 127,
+	[2][1][RTW89_CN][42] = 56,
+	[2][1][RTW89_UK][42] = 127,
+	[2][1][RTW89_FCC][44] = 127,
+	[2][1][RTW89_ETSI][44] = 127,
+	[2][1][RTW89_MKK][44] = 127,
+	[2][1][RTW89_IC][44] = 127,
+	[2][1][RTW89_KCC][44] = 127,
+	[2][1][RTW89_ACMA][44] = 127,
+	[2][1][RTW89_CN][44] = 56,
+	[2][1][RTW89_UK][44] = 127,
+	[2][1][RTW89_FCC][46] = 127,
+	[2][1][RTW89_ETSI][46] = 127,
+	[2][1][RTW89_MKK][46] = 127,
+	[2][1][RTW89_IC][46] = 127,
+	[2][1][RTW89_KCC][46] = 127,
+	[2][1][RTW89_ACMA][46] = 127,
+	[2][1][RTW89_CN][46] = 56,
+	[2][1][RTW89_UK][46] = 127,
+	[2][1][RTW89_FCC][48] = 127,
+	[2][1][RTW89_ETSI][48] = 127,
+	[2][1][RTW89_MKK][48] = 127,
+	[2][1][RTW89_IC][48] = 127,
+	[2][1][RTW89_KCC][48] = 127,
+	[2][1][RTW89_ACMA][48] = 127,
+	[2][1][RTW89_CN][48] = 127,
+	[2][1][RTW89_UK][48] = 127,
+	[2][1][RTW89_FCC][50] = 127,
+	[2][1][RTW89_ETSI][50] = 127,
+	[2][1][RTW89_MKK][50] = 127,
+	[2][1][RTW89_IC][50] = 127,
+	[2][1][RTW89_KCC][50] = 127,
+	[2][1][RTW89_ACMA][50] = 127,
+	[2][1][RTW89_CN][50] = 127,
+	[2][1][RTW89_UK][50] = 127,
+	[2][1][RTW89_FCC][52] = 127,
+	[2][1][RTW89_ETSI][52] = 127,
+	[2][1][RTW89_MKK][52] = 127,
+	[2][1][RTW89_IC][52] = 127,
+	[2][1][RTW89_KCC][52] = 127,
+	[2][1][RTW89_ACMA][52] = 127,
+	[2][1][RTW89_CN][52] = 127,
+	[2][1][RTW89_UK][52] = 127,
+};
+
+const struct rtw89_phy_table rtw89_8851b_phy_bb_table = {
+	.regs		= rtw89_8851b_phy_bb_regs,
+	.n_regs		= ARRAY_SIZE(rtw89_8851b_phy_bb_regs),
+	.rf_path	= 0, /* don't care */
+};
+
+const struct rtw89_phy_table rtw89_8851b_phy_bb_gain_table = {
+	.regs		= rtw89_8851b_phy_bb_reg_gain,
+	.n_regs		= ARRAY_SIZE(rtw89_8851b_phy_bb_reg_gain),
+	.rf_path	= 0, /* don't care */
+};
+
+const struct rtw89_phy_table rtw89_8851b_phy_radioa_table = {
+	.regs		= rtw89_8851b_phy_radioa_regs,
+	.n_regs		= ARRAY_SIZE(rtw89_8851b_phy_radioa_regs),
+	.rf_path	= RF_PATH_A,
+	.config		= rtw89_phy_config_rf_reg_v1,
+};
+
+const struct rtw89_phy_table rtw89_8851b_phy_nctl_table = {
+	.regs		= rtw89_8851b_phy_nctl_regs,
+	.n_regs		= ARRAY_SIZE(rtw89_8851b_phy_nctl_regs),
+	.rf_path	= 0, /* don't care */
+};
+
+const struct rtw89_txpwr_table rtw89_8851b_byr_table = {
+	.data = rtw89_8851b_txpwr_byrate,
+	.size = ARRAY_SIZE(rtw89_8851b_txpwr_byrate),
+	.load = rtw89_phy_load_txpwr_byrate,
+};
+
+const struct rtw89_txpwr_track_cfg rtw89_8851b_trk_cfg = {
+	.delta_swingidx_5ga_n = _txpwr_track_delta_swingidx_5ga_n,
+	.delta_swingidx_5ga_p = _txpwr_track_delta_swingidx_5ga_p,
+	.delta_swingidx_2ga_n = _txpwr_track_delta_swingidx_2ga_n,
+	.delta_swingidx_2ga_p = _txpwr_track_delta_swingidx_2ga_p,
+	.delta_swingidx_2g_cck_a_n = _txpwr_track_delta_swingidx_2g_cck_a_n,
+	.delta_swingidx_2g_cck_a_p = _txpwr_track_delta_swingidx_2g_cck_a_p,
+};
+
+const struct rtw89_rfe_parms rtw89_8851b_dflt_parms = {
+	.rule_2ghz = {
+		.lmt = &rtw89_8851b_txpwr_lmt_2g,
+		.lmt_ru = &rtw89_8851b_txpwr_lmt_ru_2g,
+	},
+	.rule_5ghz = {
+		.lmt = &rtw89_8851b_txpwr_lmt_5g,
+		.lmt_ru = &rtw89_8851b_txpwr_lmt_ru_5g,
+	},
+};
+
+static const struct rtw89_rfe_parms rtw89_8851b_rfe_parms_type2 = {
+	.rule_2ghz = {
+		.lmt = &rtw89_8851b_txpwr_lmt_2g_type2,
+		.lmt_ru = &rtw89_8851b_txpwr_lmt_ru_2g_type2,
+	},
+	.rule_5ghz = {
+		.lmt = &rtw89_8851b_txpwr_lmt_5g_type2,
+		.lmt_ru = &rtw89_8851b_txpwr_lmt_ru_5g_type2,
+	},
+};
+
+const struct rtw89_rfe_parms_conf rtw89_8851b_rfe_parms_conf[] = {
+	{
+		.rfe_parms = &rtw89_8851b_rfe_parms_type2,
+		.rfe_type = 2,
+	},
+	{},
+};
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8851b_table.h b/drivers/net/wireless/realtek/rtw89/rtw8851b_table.h
new file mode 100644
index 000000000000..f2e673ba39c8
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw89/rtw8851b_table.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/* Copyright(c) 2022-2023  Realtek Corporation
+ */
+
+#ifndef __RTW89_8851B_TABLE_H__
+#define __RTW89_8851B_TABLE_H__
+
+#include "core.h"
+
+extern const struct rtw89_phy_table rtw89_8851b_phy_bb_table;
+extern const struct rtw89_phy_table rtw89_8851b_phy_bb_gain_table;
+extern const struct rtw89_phy_table rtw89_8851b_phy_radioa_table;
+extern const struct rtw89_phy_table rtw89_8851b_phy_nctl_table;
+extern const struct rtw89_txpwr_table rtw89_8851b_byr_table;
+extern const struct rtw89_txpwr_track_cfg rtw89_8851b_trk_cfg;
+extern const u8 rtw89_8851b_tx_shape[RTW89_BAND_MAX][RTW89_RS_TX_SHAPE_NUM]
+				    [RTW89_REGD_NUM];
+extern const struct rtw89_rfe_parms rtw89_8851b_dflt_parms;
+extern const struct rtw89_rfe_parms_conf rtw89_8851b_rfe_parms_conf[];
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a.c b/drivers/net/wireless/realtek/rtw89/rtw8852a.c
index 9c42b6abd223..d7930efd89b7 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852a.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852a.c
@@ -12,6 +12,11 @@
 #include "rtw8852a_table.h"
 #include "txrx.h"
 
+#define RTW8852A_FW_FORMAT_MAX 0
+#define RTW8852A_FW_BASENAME "rtw89/rtw8852a_fw"
+#define RTW8852A_MODULE_FIRMWARE \
+	RTW8852A_FW_BASENAME ".bin"
+
 static const struct rtw89_hfc_ch_cfg rtw8852a_hfc_chcfg_pcie[] = {
 	{128, 1896, grp_0}, /* ACH 0 */
 	{128, 1896, grp_0}, /* ACH 1 */
@@ -1827,7 +1832,8 @@ rtw8852a_btc_set_wl_txpwr_ctrl(struct rtw89_dev *rtwdev, u32 txpwr_val)
 static
 s8 rtw8852a_btc_get_bt_rssi(struct rtw89_dev *rtwdev, s8 val)
 {
-	return clamp_t(s8, val, -100, 0) + 100;
+	/* +6 for compensate offset */
+	return clamp_t(s8, val + 6, -100, 0) + 100;
 }
 
 static struct rtw89_btc_rf_trx_para rtw89_btc_8852a_rf_ul[] = {
@@ -1947,20 +1953,25 @@ static void rtw8852a_set_wl_lna2(struct rtw89_dev *rtwdev, u8 level)
 
 static void rtw8852a_btc_set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level)
 {
+	struct rtw89_btc *btc = &rtwdev->btc;
+
 	switch (level) {
 	case 0: /* original */
+	default:
 		rtw8852a_bb_ctrl_btc_preagc(rtwdev, false);
-		rtw8852a_set_wl_lna2(rtwdev, 0);
+		btc->dm.wl_lna2 = 0;
 		break;
 	case 1: /* for FDD free-run */
 		rtw8852a_bb_ctrl_btc_preagc(rtwdev, true);
-		rtw8852a_set_wl_lna2(rtwdev, 0);
+		btc->dm.wl_lna2 = 0;
 		break;
 	case 2: /* for BTG Co-Rx*/
 		rtw8852a_bb_ctrl_btc_preagc(rtwdev, false);
-		rtw8852a_set_wl_lna2(rtwdev, 1);
+		btc->dm.wl_lna2 = 1;
 		break;
 	}
+
+	rtw8852a_set_wl_lna2(rtwdev, btc->dm.wl_lna2);
 }
 
 static void rtw8852a_fill_freq_with_ppdu(struct rtw89_dev *rtwdev,
@@ -2054,7 +2065,8 @@ static const struct rtw89_chip_ops rtw8852a_chip_ops = {
 const struct rtw89_chip_info rtw8852a_chip_info = {
 	.chip_id		= RTL8852A,
 	.ops			= &rtw8852a_chip_ops,
-	.fw_name		= "rtw89/rtw8852a_fw.bin",
+	.fw_basename		= RTW8852A_FW_BASENAME,
+	.fw_format_max		= RTW8852A_FW_FORMAT_MAX,
 	.try_ce_fw		= false,
 	.fifo_size		= 458752,
 	.dle_scc_rsvd_size	= 0,
@@ -2074,10 +2086,8 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 				   &rtw89_8852a_phy_radiob_table,},
 	.nctl_table		= &rtw89_8852a_phy_nctl_table,
 	.byr_table		= &rtw89_8852a_byr_table,
-	.txpwr_lmt_2g		= &rtw89_8852a_txpwr_lmt_2g,
-	.txpwr_lmt_5g		= &rtw89_8852a_txpwr_lmt_5g,
-	.txpwr_lmt_ru_2g	= &rtw89_8852a_txpwr_lmt_ru_2g,
-	.txpwr_lmt_ru_5g	= &rtw89_8852a_txpwr_lmt_ru_5g,
+	.dflt_parms		= &rtw89_8852a_dflt_parms,
+	.rfe_parms_conf		= NULL,
 	.txpwr_factor_rf	= 2,
 	.txpwr_factor_mac	= 1,
 	.dig_table		= &rtw89_8852a_phy_dig_table,
@@ -2131,24 +2141,28 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 	.h2c_desc_size		= sizeof(struct rtw89_txwd_body),
 	.txwd_body_size		= sizeof(struct rtw89_txwd_body),
 	.h2c_ctrl_reg		= R_AX_H2CREG_CTRL,
+	.h2c_counter_reg	= {R_AX_UDM1 + 1, B_AX_UDM1_HALMAC_H2C_DEQ_CNT_MASK >> 8},
 	.h2c_regs		= rtw8852a_h2c_regs,
 	.c2h_ctrl_reg		= R_AX_C2HREG_CTRL,
 	.c2h_regs		= rtw8852a_c2h_regs,
+	.c2h_counter_reg	= {R_AX_UDM1 + 1, B_AX_UDM1_HALMAC_C2H_ENQ_CNT_MASK >> 8},
 	.page_regs		= &rtw8852a_page_regs,
 	.cfo_src_fd		= false,
+	.cfo_hw_comp            = false,
 	.dcfo_comp		= &rtw8852a_dcfo_comp,
-	.dcfo_comp_sft		= 3,
+	.dcfo_comp_sft		= 10,
 	.imr_info		= &rtw8852a_imr_info,
 	.rrsr_cfgs		= &rtw8852a_rrsr_cfgs,
 	.bss_clr_map_reg	= R_BSS_CLR_MAP,
 	.dma_ch_mask		= 0,
+	.edcca_lvl_reg		= R_SEG0R_EDCCA_LVL,
 #ifdef CONFIG_PM
 	.wowlan_stub		= &rtw_wowlan_stub_8852a,
 #endif
 };
 EXPORT_SYMBOL(rtw8852a_chip_info);
 
-MODULE_FIRMWARE("rtw89/rtw8852a_fw.bin");
+MODULE_FIRMWARE(RTW8852A_MODULE_FIRMWARE);
 MODULE_AUTHOR("Realtek Corporation");
 MODULE_DESCRIPTION("Realtek 802.11ax wireless 8852A driver");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a_table.c b/drivers/net/wireless/realtek/rtw89/rtw8852a_table.c
index 320bcd4852c6..be54194558ff 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852a_table.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852a_table.c
@@ -43377,6 +43377,7 @@ static const s8 _txpwr_track_delta_swingidx_2g_cck_a_p[] = {
 	0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5,
 	 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10};
 
+static
 const s8 rtw89_8852a_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
 				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
 				 [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
@@ -45566,6 +45567,7 @@ const s8 rtw89_8852a_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
 	[1][1][2][1][RTW89_UK][13] = 127,
 };
 
+static
 const s8 rtw89_8852a_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
 				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
 				 [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
@@ -47898,6 +47900,7 @@ const s8 rtw89_8852a_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
 	[2][1][2][1][RTW89_UK][41] = 40,
 };
 
+static
 const s8 rtw89_8852a_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 				    [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
 	[0][0][RTW89_WW][0] = 32,
@@ -48994,6 +48997,7 @@ const s8 rtw89_8852a_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 	[2][1][RTW89_UK][13] = 127,
 };
 
+static
 const s8 rtw89_8852a_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM]
 				    [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
 	[0][0][RTW89_WW][0] = 22,
@@ -51043,3 +51047,14 @@ const struct rtw89_phy_dig_gain_table rtw89_8852a_phy_dig_table = {
 	.cfg_lna_a = &rtw89_8852a_lna_gain_a_table,
 	.cfg_tia_a = &rtw89_8852a_tia_gain_a_table
 };
+
+const struct rtw89_rfe_parms rtw89_8852a_dflt_parms = {
+	.rule_2ghz = {
+		.lmt = &rtw89_8852a_txpwr_lmt_2g,
+		.lmt_ru = &rtw89_8852a_txpwr_lmt_ru_2g,
+	},
+	.rule_5ghz = {
+		.lmt = &rtw89_8852a_txpwr_lmt_5g,
+		.lmt_ru = &rtw89_8852a_txpwr_lmt_ru_5g,
+	},
+};
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a_table.h b/drivers/net/wireless/realtek/rtw89/rtw8852a_table.h
index 913796506286..41c379b1044d 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852a_table.h
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852a_table.h
@@ -14,15 +14,6 @@ extern const struct rtw89_phy_table rtw89_8852a_phy_nctl_table;
 extern const struct rtw89_txpwr_table rtw89_8852a_byr_table;
 extern const struct rtw89_phy_dig_gain_table rtw89_8852a_phy_dig_table;
 extern const struct rtw89_txpwr_track_cfg rtw89_8852a_trk_cfg;
-extern const s8 rtw89_8852a_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
-					[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
-					[RTW89_REGD_NUM][RTW89_2G_CH_NUM];
-extern const s8 rtw89_8852a_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
-					[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
-					[RTW89_REGD_NUM][RTW89_5G_CH_NUM];
-extern const s8 rtw89_8852a_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
-					   [RTW89_REGD_NUM][RTW89_2G_CH_NUM];
-extern const s8 rtw89_8852a_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM]
-					   [RTW89_REGD_NUM][RTW89_5G_CH_NUM];
+extern const struct rtw89_rfe_parms rtw89_8852a_dflt_parms;
 
 #endif
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852b.c b/drivers/net/wireless/realtek/rtw89/rtw8852b.c
index ee8dba7e0074..eaa2ea0586bc 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852b.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852b.c
@@ -12,6 +12,11 @@
 #include "rtw8852b_table.h"
 #include "txrx.h"
 
+#define RTW8852B_FW_FORMAT_MAX 1
+#define RTW8852B_FW_BASENAME "rtw89/rtw8852b_fw"
+#define RTW8852B_MODULE_FIRMWARE \
+	RTW8852B_FW_BASENAME "-" __stringify(RTW8852B_FW_FORMAT_MAX) ".bin"
+
 static const struct rtw89_hfc_ch_cfg rtw8852b_hfc_chcfg_pcie[] = {
 	{5, 343, grp_0}, /* ACH 0 */
 	{5, 343, grp_0}, /* ACH 1 */
@@ -48,6 +53,10 @@ static const struct rtw89_dle_mem rtw8852b_dle_mem_pcie[] = {
 			   &rtw89_mac_size.ple_size6, &rtw89_mac_size.wde_qt6,
 			   &rtw89_mac_size.wde_qt6, &rtw89_mac_size.ple_qt18,
 			   &rtw89_mac_size.ple_qt58},
+	[RTW89_QTA_WOW] = {RTW89_QTA_WOW, &rtw89_mac_size.wde_size6,
+			   &rtw89_mac_size.ple_size6, &rtw89_mac_size.wde_qt6,
+			   &rtw89_mac_size.wde_qt6, &rtw89_mac_size.ple_qt18,
+			   &rtw89_mac_size.ple_qt_52b_wow},
 	[RTW89_QTA_DLFW] = {RTW89_QTA_DLFW, &rtw89_mac_size.wde_size9,
 			    &rtw89_mac_size.ple_size8, &rtw89_mac_size.wde_qt4,
 			    &rtw89_mac_size.wde_qt4, &rtw89_mac_size.ple_qt13,
@@ -323,7 +332,7 @@ static const struct rtw89_btc_rf_trx_para rtw89_btc_8852b_rf_ul[] = {
 	{255, 0, 0, 7}, /* 2 ->reserved for shared-antenna */
 	{255, 0, 0, 7}, /* 3- >reserved for shared-antenna */
 	{255, 0, 0, 7}, /* 4 ->reserved for shared-antenna */
-	{255, 0, 0, 7}, /* the below id is for non-shared-antenna free-run */
+	{255, 1, 0, 7}, /* the below id is for non-shared-antenna free-run */
 	{6, 1, 0, 7},
 	{13, 1, 0, 7},
 	{13, 1, 0, 7}
@@ -335,7 +344,7 @@ static const struct rtw89_btc_rf_trx_para rtw89_btc_8852b_rf_dl[] = {
 	{255, 0, 0, 7}, /* 2 ->reserved for shared-antenna */
 	{255, 0, 0, 7}, /* 3- >reserved for shared-antenna */
 	{255, 0, 0, 7}, /* 4 ->reserved for shared-antenna */
-	{255, 0, 0, 7}, /* the below id is for non-shared-antenna free-run */
+	{255, 1, 0, 7}, /* the below id is for non-shared-antenna free-run */
 	{255, 1, 0, 7},
 	{255, 1, 0, 7},
 	{255, 1, 0, 7}
@@ -355,7 +364,9 @@ static const struct rtw89_btc_fbtc_mreg rtw89_btc_8852b_mon_reg[] = {
 	RTW89_DEF_FBTC_MREG(REG_MAC, 4, 0xd200),
 	RTW89_DEF_FBTC_MREG(REG_MAC, 4, 0xd220),
 	RTW89_DEF_FBTC_MREG(REG_BB, 4, 0x980),
-	RTW89_DEF_FBTC_MREG(REG_BT_MODEM, 4, 0x178),
+	RTW89_DEF_FBTC_MREG(REG_BB, 4, 0x4738),
+	RTW89_DEF_FBTC_MREG(REG_BB, 4, 0x4688),
+	RTW89_DEF_FBTC_MREG(REG_BB, 4, 0x4694),
 };
 
 static const u8 rtw89_btc_8852b_wl_rssi_thres[BTC_WL_RSSI_THMAX] = {70, 60, 50, 40};
@@ -1284,7 +1295,7 @@ static void rtw8852b_ctrl_cck_en(struct rtw89_dev *rtwdev, bool cck_en)
 static void rtw8852b_5m_mask(struct rtw89_dev *rtwdev, const struct rtw89_chan *chan,
 			     enum rtw89_phy_idx phy_idx)
 {
-	u8 pri_ch = chan->primary_channel;
+	u8 pri_ch = chan->pri_ch_idx;
 	bool mask_5m_low;
 	bool mask_5m_en;
 
@@ -1292,12 +1303,13 @@ static void rtw8852b_5m_mask(struct rtw89_dev *rtwdev, const struct rtw89_chan *
 	case RTW89_CHANNEL_WIDTH_40:
 		/* Prich=1: Mask 5M High, Prich=2: Mask 5M Low */
 		mask_5m_en = true;
-		mask_5m_low = pri_ch == 2;
+		mask_5m_low = pri_ch == RTW89_SC_20_LOWER;
 		break;
 	case RTW89_CHANNEL_WIDTH_80:
 		/* Prich=3: Mask 5M High, Prich=4: Mask 5M Low, Else: Disable */
-		mask_5m_en = pri_ch == 3 || pri_ch == 4;
-		mask_5m_low = pri_ch == 4;
+		mask_5m_en = pri_ch == RTW89_SC_20_UPMOST ||
+			     pri_ch == RTW89_SC_20_LOWEST;
+		mask_5m_low = pri_ch == RTW89_SC_20_LOWEST;
 		break;
 	default:
 		mask_5m_en = false;
@@ -1422,6 +1434,7 @@ static void rtw8852b_set_channel_bb(struct rtw89_dev *rtwdev, const struct rtw89
 {
 	bool cck_en = chan->channel <= 14;
 	u8 pri_ch_idx = chan->pri_ch_idx;
+	u8 band = chan->band_type, chan_idx;
 
 	if (cck_en)
 		rtw8852b_ctrl_sco_cck(rtwdev,  chan->primary_channel);
@@ -1444,8 +1457,8 @@ static void rtw8852b_set_channel_bb(struct rtw89_dev *rtwdev, const struct rtw89
 				       B_BT_DYN_DC_EST_EN_MSK, 0x0);
 		rtw89_phy_write32_mask(rtwdev, R_GNT_BT_WGT_EN, B_GNT_BT_WGT_EN, 0x0);
 	}
-	rtw89_phy_write32_mask(rtwdev, R_MAC_PIN_SEL, B_CH_IDX_SEG0,
-			       chan->primary_channel);
+	chan_idx = rtw89_encode_chan_idx(rtwdev, chan->primary_channel, band);
+	rtw89_phy_write32_mask(rtwdev, R_MAC_PIN_SEL, B_CH_IDX_SEG0, chan_idx);
 	rtw8852b_5m_mask(rtwdev, chan, phy_idx);
 	rtw8852b_bb_set_pop(rtwdev);
 	rtw8852b_bb_reset_all(rtwdev, phy_idx);
@@ -2266,7 +2279,8 @@ do {								\
 static
 s8 rtw8852b_btc_get_bt_rssi(struct rtw89_dev *rtwdev, s8 val)
 {
-	return clamp_t(s8, val, -100, 0) + 100;
+	/* +6 for compensate offset */
+	return clamp_t(s8, val + 6, -100, 0) + 100;
 }
 
 static
@@ -2283,15 +2297,64 @@ static void rtw8852b_btc_wl_s1_standby(struct rtw89_dev *rtwdev, bool state)
 
 	/* set WL standby = Rx for GNT_BT_Tx = 1->0 settle issue */
 	if (state)
-		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x579);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x179);
 	else
 		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x20);
 
 	rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0);
 }
 
+static void rtw8852b_btc_set_wl_lna2(struct rtw89_dev *rtwdev, u8 level)
+{
+	switch (level) {
+	case 0: /* default */
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x1000);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x0);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x1);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x17);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x2);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x3);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x17);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0);
+		break;
+	case 1: /* Fix LNA2=5  */
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x1000);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x0);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x1);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x5);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x2);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x3);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x5);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0);
+		break;
+	}
+}
+
 static void rtw8852b_btc_set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level)
 {
+	struct rtw89_btc *btc = &rtwdev->btc;
+
+	switch (level) {
+	case 0: /* original */
+	default:
+		rtw8852b_bb_ctrl_btc_preagc(rtwdev, false);
+		btc->dm.wl_lna2 = 0;
+		break;
+	case 1: /* for FDD free-run */
+		rtw8852b_bb_ctrl_btc_preagc(rtwdev, true);
+		btc->dm.wl_lna2 = 0;
+		break;
+	case 2: /* for BTG Co-Rx*/
+		rtw8852b_bb_ctrl_btc_preagc(rtwdev, false);
+		btc->dm.wl_lna2 = 1;
+		break;
+	}
+
+	rtw8852b_btc_set_wl_lna2(rtwdev, btc->dm.wl_lna2);
 }
 
 static void rtw8852b_fill_freq_with_ppdu(struct rtw89_dev *rtwdev,
@@ -2299,13 +2362,14 @@ static void rtw8852b_fill_freq_with_ppdu(struct rtw89_dev *rtwdev,
 					 struct ieee80211_rx_status *status)
 {
 	u16 chan = phy_ppdu->chan_idx;
-	u8 band;
+	enum nl80211_band band;
+	u8 ch;
 
 	if (chan == 0)
 		return;
 
-	band = chan <= 14 ? NL80211_BAND_2GHZ : NL80211_BAND_5GHZ;
-	status->freq = ieee80211_channel_to_frequency(chan, band);
+	rtw89_decode_chan_idx(rtwdev, chan, &ch, &band);
+	status->freq = ieee80211_channel_to_frequency(ch, band);
 	status->band = band;
 }
 
@@ -2426,10 +2490,20 @@ static const struct rtw89_chip_ops rtw8852b_chip_ops = {
 	.btc_set_policy		= rtw89_btc_set_policy_v1,
 };
 
+#ifdef CONFIG_PM
+static const struct wiphy_wowlan_support rtw_wowlan_stub_8852b = {
+	.flags = WIPHY_WOWLAN_MAGIC_PKT | WIPHY_WOWLAN_DISCONNECT,
+	.n_patterns = RTW89_MAX_PATTERN_NUM,
+	.pattern_max_len = RTW89_MAX_PATTERN_SIZE,
+	.pattern_min_len = 1,
+};
+#endif
+
 const struct rtw89_chip_info rtw8852b_chip_info = {
 	.chip_id		= RTL8852B,
 	.ops			= &rtw8852b_chip_ops,
-	.fw_name		= "rtw89/rtw8852b_fw.bin",
+	.fw_basename		= RTW8852B_FW_BASENAME,
+	.fw_format_max		= RTW8852B_FW_FORMAT_MAX,
 	.try_ce_fw		= true,
 	.fifo_size		= 196608,
 	.dle_scc_rsvd_size	= 98304,
@@ -2449,10 +2523,8 @@ const struct rtw89_chip_info rtw8852b_chip_info = {
 				   &rtw89_8852b_phy_radiob_table,},
 	.nctl_table		= &rtw89_8852b_phy_nctl_table,
 	.byr_table		= &rtw89_8852b_byr_table,
-	.txpwr_lmt_2g		= &rtw89_8852b_txpwr_lmt_2g,
-	.txpwr_lmt_5g		= &rtw89_8852b_txpwr_lmt_5g,
-	.txpwr_lmt_ru_2g	= &rtw89_8852b_txpwr_lmt_ru_2g,
-	.txpwr_lmt_ru_5g	= &rtw89_8852b_txpwr_lmt_ru_5g,
+	.dflt_parms		= &rtw89_8852b_dflt_parms,
+	.rfe_parms_conf		= NULL,
 	.txpwr_factor_rf	= 2,
 	.txpwr_factor_mac	= 1,
 	.dig_table		= NULL,
@@ -2506,23 +2578,30 @@ const struct rtw89_chip_info rtw8852b_chip_info = {
 	.h2c_desc_size		= sizeof(struct rtw89_txwd_body),
 	.txwd_body_size		= sizeof(struct rtw89_txwd_body),
 	.h2c_ctrl_reg		= R_AX_H2CREG_CTRL,
+	.h2c_counter_reg	= {R_AX_UDM1 + 1, B_AX_UDM1_HALMAC_H2C_DEQ_CNT_MASK >> 8},
 	.h2c_regs		= rtw8852b_h2c_regs,
 	.c2h_ctrl_reg		= R_AX_C2HREG_CTRL,
+	.c2h_counter_reg	= {R_AX_UDM1 + 1, B_AX_UDM1_HALMAC_C2H_ENQ_CNT_MASK >> 8},
 	.c2h_regs		= rtw8852b_c2h_regs,
 	.page_regs		= &rtw8852b_page_regs,
 	.cfo_src_fd		= true,
+	.cfo_hw_comp		= true,
 	.dcfo_comp		= &rtw8852b_dcfo_comp,
-	.dcfo_comp_sft		= 3,
+	.dcfo_comp_sft		= 10,
 	.imr_info		= &rtw8852b_imr_info,
 	.rrsr_cfgs		= &rtw8852b_rrsr_cfgs,
 	.bss_clr_map_reg	= R_BSS_CLR_MAP_V1,
 	.dma_ch_mask		= BIT(RTW89_DMA_ACH4) | BIT(RTW89_DMA_ACH5) |
 				  BIT(RTW89_DMA_ACH6) | BIT(RTW89_DMA_ACH7) |
 				  BIT(RTW89_DMA_B1MG) | BIT(RTW89_DMA_B1HI),
+	.edcca_lvl_reg		= R_SEG0R_EDCCA_LVL_V1,
+#ifdef CONFIG_PM
+	.wowlan_stub		= &rtw_wowlan_stub_8852b,
+#endif
 };
 EXPORT_SYMBOL(rtw8852b_chip_info);
 
-MODULE_FIRMWARE("rtw89/rtw8852b_fw.bin");
+MODULE_FIRMWARE(RTW8852B_MODULE_FIRMWARE);
 MODULE_AUTHOR("Realtek Corporation");
 MODULE_DESCRIPTION("Realtek 802.11ax wireless 8852B driver");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852b_table.c b/drivers/net/wireless/realtek/rtw89/rtw8852b_table.c
index a6734965361f..904cdb9e56fa 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852b_table.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852b_table.c
@@ -14706,6 +14706,7 @@ const u8 rtw89_8852b_tx_shape[RTW89_BAND_MAX][RTW89_RS_TX_SHAPE_NUM]
 	[1][1][RTW89_UKRAINE] = 0,
 };
 
+static
 const s8 rtw89_8852b_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
 				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
 				 [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
@@ -16895,6 +16896,7 @@ const s8 rtw89_8852b_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
 	[1][1][2][1][RTW89_UK][13] = 127,
 };
 
+static
 const s8 rtw89_8852b_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
 				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
 				 [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
@@ -19539,6 +19541,7 @@ const s8 rtw89_8852b_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
 	[2][1][2][1][RTW89_UK][49] = 127,
 };
 
+static
 const s8 rtw89_8852b_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 				    [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
 	[0][0][RTW89_WW][0] = 32,
@@ -20635,6 +20638,7 @@ const s8 rtw89_8852b_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 	[2][1][RTW89_UK][13] = 127,
 };
 
+static
 const s8 rtw89_8852b_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM]
 				    [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
 	[0][0][RTW89_WW][0] = 24,
@@ -22875,3 +22879,14 @@ const struct rtw89_txpwr_track_cfg rtw89_8852b_trk_cfg = {
 	.delta_swingidx_2g_cck_a_n = _txpwr_track_delta_swingidx_2g_cck_a_n,
 	.delta_swingidx_2g_cck_a_p = _txpwr_track_delta_swingidx_2g_cck_a_p,
 };
+
+const struct rtw89_rfe_parms rtw89_8852b_dflt_parms = {
+	.rule_2ghz = {
+		.lmt = &rtw89_8852b_txpwr_lmt_2g,
+		.lmt_ru = &rtw89_8852b_txpwr_lmt_ru_2g,
+	},
+	.rule_5ghz = {
+		.lmt = &rtw89_8852b_txpwr_lmt_5g,
+		.lmt_ru = &rtw89_8852b_txpwr_lmt_ru_5g,
+	},
+};
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852b_table.h b/drivers/net/wireless/realtek/rtw89/rtw8852b_table.h
index 114337ac9fb0..5f4161496a58 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852b_table.h
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852b_table.h
@@ -16,15 +16,6 @@ extern const struct rtw89_txpwr_table rtw89_8852b_byr_table;
 extern const struct rtw89_txpwr_track_cfg rtw89_8852b_trk_cfg;
 extern const u8 rtw89_8852b_tx_shape[RTW89_BAND_MAX][RTW89_RS_TX_SHAPE_NUM]
 				    [RTW89_REGD_NUM];
-extern const s8 rtw89_8852b_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
-					[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
-					[RTW89_REGD_NUM][RTW89_2G_CH_NUM];
-extern const s8 rtw89_8852b_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
-					[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
-					[RTW89_REGD_NUM][RTW89_5G_CH_NUM];
-extern const s8 rtw89_8852b_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
-					   [RTW89_REGD_NUM][RTW89_2G_CH_NUM];
-extern const s8 rtw89_8852b_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM]
-					   [RTW89_REGD_NUM][RTW89_5G_CH_NUM];
+extern const struct rtw89_rfe_parms rtw89_8852b_dflt_parms;
 
 #endif
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c.c b/drivers/net/wireless/realtek/rtw89/rtw8852c.c
index d2dde21d3daf..ceb819a62efc 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852c.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852c.c
@@ -13,6 +13,11 @@
 #include "rtw8852c_table.h"
 #include "util.h"
 
+#define RTW8852C_FW_FORMAT_MAX 0
+#define RTW8852C_FW_BASENAME "rtw89/rtw8852c_fw"
+#define RTW8852C_MODULE_FIRMWARE \
+	RTW8852C_FW_BASENAME ".bin"
+
 static const struct rtw89_hfc_ch_cfg rtw8852c_hfc_chcfg_pcie[] = {
 	{13, 1614, grp_0}, /* ACH 0 */
 	{13, 1614, grp_0}, /* ACH 1 */
@@ -852,76 +857,6 @@ static void rtw8852c_set_gain_error(struct rtw89_dev *rtwdev,
 	}
 }
 
-static
-const u8 rtw8852c_ch_base_table[16] = {1, 0xff,
-				       36, 100, 132, 149, 0xff,
-				       1, 33, 65, 97, 129, 161, 193, 225, 0xff};
-#define RTW8852C_CH_BASE_IDX_2G		0
-#define RTW8852C_CH_BASE_IDX_5G_FIRST	2
-#define RTW8852C_CH_BASE_IDX_5G_LAST	5
-#define RTW8852C_CH_BASE_IDX_6G_FIRST	7
-#define RTW8852C_CH_BASE_IDX_6G_LAST	14
-
-#define RTW8852C_CH_BASE_IDX_MASK	GENMASK(7, 4)
-#define RTW8852C_CH_OFFSET_MASK		GENMASK(3, 0)
-
-static u8 rtw8852c_encode_chan_idx(struct rtw89_dev *rtwdev, u8 central_ch, u8 band)
-{
-	u8 chan_idx;
-	u8 last, first;
-	u8 idx;
-
-	switch (band) {
-	case RTW89_BAND_2G:
-		chan_idx = FIELD_PREP(RTW8852C_CH_BASE_IDX_MASK, RTW8852C_CH_BASE_IDX_2G) |
-			   FIELD_PREP(RTW8852C_CH_OFFSET_MASK, central_ch);
-		return chan_idx;
-	case RTW89_BAND_5G:
-		first = RTW8852C_CH_BASE_IDX_5G_FIRST;
-		last = RTW8852C_CH_BASE_IDX_5G_LAST;
-		break;
-	case RTW89_BAND_6G:
-		first = RTW8852C_CH_BASE_IDX_6G_FIRST;
-		last = RTW8852C_CH_BASE_IDX_6G_LAST;
-		break;
-	default:
-		rtw89_warn(rtwdev, "Unsupported band %d\n", band);
-		return 0;
-	}
-
-	for (idx = last; idx >= first; idx--)
-		if (central_ch >= rtw8852c_ch_base_table[idx])
-			break;
-
-	if (idx < first) {
-		rtw89_warn(rtwdev, "Unknown band %d channel %d\n", band, central_ch);
-		return 0;
-	}
-
-	chan_idx = FIELD_PREP(RTW8852C_CH_BASE_IDX_MASK, idx) |
-		   FIELD_PREP(RTW8852C_CH_OFFSET_MASK,
-			      (central_ch - rtw8852c_ch_base_table[idx]) >> 1);
-	return chan_idx;
-}
-
-static void rtw8852c_decode_chan_idx(struct rtw89_dev *rtwdev, u8 chan_idx,
-				     u8 *ch, enum nl80211_band *band)
-{
-	u8 idx, offset;
-
-	idx = FIELD_GET(RTW8852C_CH_BASE_IDX_MASK, chan_idx);
-	offset = FIELD_GET(RTW8852C_CH_OFFSET_MASK, chan_idx);
-
-	if (idx == RTW8852C_CH_BASE_IDX_2G) {
-		*band = NL80211_BAND_2GHZ;
-		*ch = offset;
-		return;
-	}
-
-	*band = idx <= RTW8852C_CH_BASE_IDX_5G_LAST ? NL80211_BAND_5GHZ : NL80211_BAND_6GHZ;
-	*ch = rtw8852c_ch_base_table[idx] + (offset << 1);
-}
-
 static void rtw8852c_set_gain_offset(struct rtw89_dev *rtwdev,
 				     const struct rtw89_chan *chan,
 				     enum rtw89_phy_idx phy_idx,
@@ -1084,7 +1019,7 @@ static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev,
 		}
 	}
 
-	chan_idx = rtw8852c_encode_chan_idx(rtwdev, chan->primary_channel, band);
+	chan_idx = rtw89_encode_chan_idx(rtwdev, chan->primary_channel, band);
 	rtw89_phy_write32_idx(rtwdev, R_MAC_PIN_SEL, B_CH_IDX_SEG0, chan_idx, phy_idx);
 }
 
@@ -1445,18 +1380,19 @@ static void rtw8852c_5m_mask(struct rtw89_dev *rtwdev,
 			     const struct rtw89_chan *chan,
 			     enum rtw89_phy_idx phy_idx)
 {
-	u8 pri_ch = chan->primary_channel;
+	u8 pri_ch = chan->pri_ch_idx;
 	bool mask_5m_low;
 	bool mask_5m_en;
 
 	switch (chan->band_width) {
 	case RTW89_CHANNEL_WIDTH_40:
 		mask_5m_en = true;
-		mask_5m_low = pri_ch == 2;
+		mask_5m_low = pri_ch == RTW89_SC_20_LOWER;
 		break;
 	case RTW89_CHANNEL_WIDTH_80:
-		mask_5m_en = ((pri_ch == 3) || (pri_ch == 4));
-		mask_5m_low = pri_ch == 4;
+		mask_5m_en = pri_ch == RTW89_SC_20_UPMOST ||
+			     pri_ch == RTW89_SC_20_LOWEST;
+		mask_5m_low = pri_ch == RTW89_SC_20_LOWEST;
 		break;
 	default:
 		mask_5m_en = false;
@@ -2597,7 +2533,8 @@ do {								\
 static
 s8 rtw8852c_btc_get_bt_rssi(struct rtw89_dev *rtwdev, s8 val)
 {
-	return clamp_t(s8, val, -100, 0) + 100;
+	/* +6 for compensate offset */
+	return clamp_t(s8, val + 6, -100, 0) + 100;
 }
 
 static const struct rtw89_btc_rf_trx_para rtw89_btc_8852c_rf_ul[] = {
@@ -2606,7 +2543,7 @@ static const struct rtw89_btc_rf_trx_para rtw89_btc_8852c_rf_ul[] = {
 	{255, 0, 0, 7}, /* 2 ->reserved for shared-antenna */
 	{255, 0, 0, 7}, /* 3- >reserved for shared-antenna */
 	{255, 0, 0, 7}, /* 4 ->reserved for shared-antenna */
-	{255, 0, 0, 7}, /* the below id is for non-shared-antenna free-run */
+	{255, 1, 0, 7}, /* the below id is for non-shared-antenna free-run */
 	{6, 1, 0, 7},
 	{13, 1, 0, 7},
 	{13, 1, 0, 7}
@@ -2618,7 +2555,7 @@ static const struct rtw89_btc_rf_trx_para rtw89_btc_8852c_rf_dl[] = {
 	{255, 0, 0, 7}, /* 2 ->reserved for shared-antenna */
 	{255, 0, 0, 7}, /* 3- >reserved for shared-antenna */
 	{255, 0, 0, 7}, /* 4 ->reserved for shared-antenna */
-	{255, 0, 0, 7}, /* the below id is for non-shared-antenna free-run */
+	{255, 1, 0, 7}, /* the below id is for non-shared-antenna free-run */
 	{255, 1, 0, 7},
 	{255, 1, 0, 7},
 	{255, 1, 0, 7}
@@ -2640,6 +2577,9 @@ static const struct rtw89_btc_fbtc_mreg rtw89_btc_8852c_mon_reg[] = {
 	RTW89_DEF_FBTC_MREG(REG_MAC, 4, 0xd200),
 	RTW89_DEF_FBTC_MREG(REG_MAC, 4, 0xd220),
 	RTW89_DEF_FBTC_MREG(REG_BB, 4, 0x980),
+	RTW89_DEF_FBTC_MREG(REG_BB, 4, 0x4aa4),
+	RTW89_DEF_FBTC_MREG(REG_BB, 4, 0x4778),
+	RTW89_DEF_FBTC_MREG(REG_BB, 4, 0x476c),
 };
 
 static
@@ -2703,20 +2643,25 @@ static void rtw8852c_set_wl_lna2(struct rtw89_dev *rtwdev, u8 level)
 
 static void rtw8852c_btc_set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level)
 {
+	struct rtw89_btc *btc = &rtwdev->btc;
+
 	switch (level) {
 	case 0: /* original */
+	default:
 		rtw8852c_bb_ctrl_btc_preagc(rtwdev, false);
-		rtw8852c_set_wl_lna2(rtwdev, 0);
+		btc->dm.wl_lna2 = 0;
 		break;
 	case 1: /* for FDD free-run */
 		rtw8852c_bb_ctrl_btc_preagc(rtwdev, true);
-		rtw8852c_set_wl_lna2(rtwdev, 0);
+		btc->dm.wl_lna2 = 0;
 		break;
 	case 2: /* for BTG Co-Rx*/
 		rtw8852c_bb_ctrl_btc_preagc(rtwdev, false);
-		rtw8852c_set_wl_lna2(rtwdev, 1);
+		btc->dm.wl_lna2 = 1;
 		break;
 	}
+
+	rtw8852c_set_wl_lna2(rtwdev, btc->dm.wl_lna2);
 }
 
 static void rtw8852c_fill_freq_with_ppdu(struct rtw89_dev *rtwdev,
@@ -2730,7 +2675,7 @@ static void rtw8852c_fill_freq_with_ppdu(struct rtw89_dev *rtwdev,
 	if (chan_idx == 0)
 		return;
 
-	rtw8852c_decode_chan_idx(rtwdev, chan_idx, &ch, &band);
+	rtw89_decode_chan_idx(rtwdev, chan_idx, &ch, &band);
 	status->freq = ieee80211_channel_to_frequency(ch, band);
 	status->band = band;
 }
@@ -2856,7 +2801,8 @@ static const struct rtw89_chip_ops rtw8852c_chip_ops = {
 const struct rtw89_chip_info rtw8852c_chip_info = {
 	.chip_id		= RTL8852C,
 	.ops			= &rtw8852c_chip_ops,
-	.fw_name		= "rtw89/rtw8852c_fw.bin",
+	.fw_basename		= RTW8852C_FW_BASENAME,
+	.fw_format_max		= RTW8852C_FW_FORMAT_MAX,
 	.try_ce_fw		= false,
 	.fifo_size		= 458752,
 	.dle_scc_rsvd_size	= 0,
@@ -2876,12 +2822,8 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 				   &rtw89_8852c_phy_radioa_table,},
 	.nctl_table		= &rtw89_8852c_phy_nctl_table,
 	.byr_table		= &rtw89_8852c_byr_table,
-	.txpwr_lmt_2g		= &rtw89_8852c_txpwr_lmt_2g,
-	.txpwr_lmt_5g		= &rtw89_8852c_txpwr_lmt_5g,
-	.txpwr_lmt_6g		= &rtw89_8852c_txpwr_lmt_6g,
-	.txpwr_lmt_ru_2g	= &rtw89_8852c_txpwr_lmt_ru_2g,
-	.txpwr_lmt_ru_5g	= &rtw89_8852c_txpwr_lmt_ru_5g,
-	.txpwr_lmt_ru_6g	= &rtw89_8852c_txpwr_lmt_ru_6g,
+	.dflt_parms		= &rtw89_8852c_dflt_parms,
+	.rfe_parms_conf		= NULL,
 	.txpwr_factor_rf	= 2,
 	.txpwr_factor_mac	= 1,
 	.dig_table		= NULL,
@@ -2937,24 +2879,28 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 	.h2c_desc_size		= sizeof(struct rtw89_rxdesc_short),
 	.txwd_body_size		= sizeof(struct rtw89_txwd_body_v1),
 	.h2c_ctrl_reg		= R_AX_H2CREG_CTRL_V1,
+	.h2c_counter_reg	= {R_AX_UDM1 + 1, B_AX_UDM1_HALMAC_H2C_DEQ_CNT_MASK >> 8},
 	.h2c_regs		= rtw8852c_h2c_regs,
 	.c2h_ctrl_reg		= R_AX_C2HREG_CTRL_V1,
+	.c2h_counter_reg	= {R_AX_UDM1 + 1, B_AX_UDM1_HALMAC_C2H_ENQ_CNT_MASK >> 8},
 	.c2h_regs		= rtw8852c_c2h_regs,
 	.page_regs		= &rtw8852c_page_regs,
 	.cfo_src_fd		= false,
+	.cfo_hw_comp            = false,
 	.dcfo_comp		= &rtw8852c_dcfo_comp,
-	.dcfo_comp_sft		= 5,
+	.dcfo_comp_sft		= 12,
 	.imr_info		= &rtw8852c_imr_info,
 	.rrsr_cfgs		= &rtw8852c_rrsr_cfgs,
 	.bss_clr_map_reg	= R_BSS_CLR_MAP,
 	.dma_ch_mask		= 0,
+	.edcca_lvl_reg		= R_SEG0R_EDCCA_LVL,
 #ifdef CONFIG_PM
 	.wowlan_stub		= &rtw_wowlan_stub_8852c,
 #endif
 };
 EXPORT_SYMBOL(rtw8852c_chip_info);
 
-MODULE_FIRMWARE("rtw89/rtw8852c_fw.bin");
+MODULE_FIRMWARE(RTW8852C_MODULE_FIRMWARE);
 MODULE_AUTHOR("Realtek Corporation");
 MODULE_DESCRIPTION("Realtek 802.11ax wireless 8852C driver");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c b/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c
index 96c264a057ff..7011e5a6f8fd 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c
@@ -28590,6 +28590,7 @@ const u8 rtw89_8852c_tx_shape[RTW89_BAND_MAX][RTW89_RS_TX_SHAPE_NUM]
 	[2][1][RTW89_KCC] = 0,
 };
 
+static
 const s8 rtw89_8852c_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
 				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
 				 [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
@@ -30107,6 +30108,7 @@ const s8 rtw89_8852c_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
 	[1][1][2][1][RTW89_UK][13] = 127,
 };
 
+static
 const s8 rtw89_8852c_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
 				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
 				 [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
@@ -32020,6 +32022,7 @@ const s8 rtw89_8852c_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
 	[3][1][2][1][RTW89_UK][45] = 127,
 };
 
+static
 const s8 rtw89_8852c_txpwr_lmt_6g[RTW89_6G_BW_NUM][RTW89_NTX_NUM]
 				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
 				 [RTW89_REGD_NUM][RTW89_6G_CH_NUM] = {
@@ -33977,6 +33980,7 @@ const s8 rtw89_8852c_txpwr_lmt_6g[RTW89_6G_BW_NUM][RTW89_NTX_NUM]
 	[3][1][2][1][RTW89_KCC][112] = 127,
 };
 
+static
 const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 				    [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
 	[0][0][RTW89_WW][0] = 32,
@@ -34737,6 +34741,7 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 	[2][1][RTW89_UK][13] = 127,
 };
 
+static
 const s8 rtw89_8852c_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM]
 				    [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
 	[0][0][RTW89_WW][0] = 16,
@@ -36253,6 +36258,7 @@ const s8 rtw89_8852c_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM]
 	[2][1][RTW89_UK][52] = 127,
 };
 
+static
 const s8 rtw89_8852c_txpwr_lmt_ru_6g[RTW89_RU_NUM][RTW89_NTX_NUM]
 				    [RTW89_REGD_NUM][RTW89_6G_CH_NUM] = {
 	[0][0][RTW89_WW][0] = -16,
@@ -37472,3 +37478,18 @@ const struct rtw89_phy_tssi_dbw_table rtw89_8852c_tssi_dbw_table = {
 	.data[RTW89_TSSI_BANDEDGE_MID] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
 	.data[RTW89_TSSI_BANDEDGE_HIGH] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
 };
+
+const struct rtw89_rfe_parms rtw89_8852c_dflt_parms = {
+	.rule_2ghz = {
+		.lmt = &rtw89_8852c_txpwr_lmt_2g,
+		.lmt_ru = &rtw89_8852c_txpwr_lmt_ru_2g,
+	},
+	.rule_5ghz = {
+		.lmt = &rtw89_8852c_txpwr_lmt_5g,
+		.lmt_ru = &rtw89_8852c_txpwr_lmt_ru_5g,
+	},
+	.rule_6ghz = {
+		.lmt = &rtw89_8852c_txpwr_lmt_6g,
+		.lmt_ru = &rtw89_8852c_txpwr_lmt_ru_6g,
+	},
+};
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c_table.h b/drivers/net/wireless/realtek/rtw89/rtw8852c_table.h
index 7d71a92e2d27..6da1849fb1fa 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852c_table.h
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852c_table.h
@@ -17,20 +17,6 @@ extern const struct rtw89_phy_tssi_dbw_table rtw89_8852c_tssi_dbw_table;
 extern const struct rtw89_txpwr_track_cfg rtw89_8852c_trk_cfg;
 extern const u8 rtw89_8852c_tx_shape[RTW89_BAND_MAX][RTW89_RS_TX_SHAPE_NUM]
 				    [RTW89_REGD_NUM];
-extern const s8 rtw89_8852c_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
-					[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
-					[RTW89_REGD_NUM][RTW89_2G_CH_NUM];
-extern const s8 rtw89_8852c_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
-					[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
-					[RTW89_REGD_NUM][RTW89_5G_CH_NUM];
-extern const s8 rtw89_8852c_txpwr_lmt_6g[RTW89_6G_BW_NUM][RTW89_NTX_NUM]
-					[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
-					[RTW89_REGD_NUM][RTW89_6G_CH_NUM];
-extern const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
-					   [RTW89_REGD_NUM][RTW89_2G_CH_NUM];
-extern const s8 rtw89_8852c_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM]
-					   [RTW89_REGD_NUM][RTW89_5G_CH_NUM];
-extern const s8 rtw89_8852c_txpwr_lmt_ru_6g[RTW89_RU_NUM][RTW89_NTX_NUM]
-					   [RTW89_REGD_NUM][RTW89_6G_CH_NUM];
+extern const struct rtw89_rfe_parms rtw89_8852c_dflt_parms;
 
 #endif
diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c
index 61db7189fdab..9e9f6947e7f1 100644
--- a/drivers/net/wireless/realtek/rtw89/ser.c
+++ b/drivers/net/wireless/realtek/rtw89/ser.c
@@ -414,8 +414,11 @@ static void ser_idle_st_hdl(struct rtw89_ser *ser, u8 evt)
 
 static void ser_reset_trx_st_hdl(struct rtw89_ser *ser, u8 evt)
 {
+	struct rtw89_dev *rtwdev = container_of(ser, struct rtw89_dev, ser);
+
 	switch (evt) {
 	case SER_EV_STATE_IN:
+		cancel_delayed_work_sync(&rtwdev->track_work);
 		drv_stop_tx(ser);
 
 		if (hal_stop_dma(ser)) {
@@ -446,6 +449,8 @@ static void ser_reset_trx_st_hdl(struct rtw89_ser *ser, u8 evt)
 		hal_enable_dma(ser);
 		drv_resume_rx(ser);
 		drv_resume_tx(ser);
+		ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->track_work,
+					     RTW89_TRACK_WORK_PERIOD);
 		break;
 
 	default:
diff --git a/drivers/net/wireless/realtek/rtw89/wow.c b/drivers/net/wireless/realtek/rtw89/wow.c
index c78ee2ab732c..2ca8abb70f11 100644
--- a/drivers/net/wireless/realtek/rtw89/wow.c
+++ b/drivers/net/wireless/realtek/rtw89/wow.c
@@ -30,7 +30,7 @@ static void rtw89_wow_enter_lps(struct rtw89_dev *rtwdev)
 	struct ieee80211_vif *wow_vif = rtwdev->wow.wow_vif;
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)wow_vif->drv_priv;
 
-	rtw89_enter_lps(rtwdev, rtwvif);
+	rtw89_enter_lps(rtwdev, rtwvif, false);
 }
 
 static void rtw89_wow_leave_lps(struct rtw89_dev *rtwdev)
@@ -420,14 +420,11 @@ static int rtw89_wow_cfg_wake(struct rtw89_dev *rtwdev, bool wow)
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)wow_vif->drv_priv;
 	struct ieee80211_sta *wow_sta;
 	struct rtw89_sta *rtwsta = NULL;
-	bool is_conn = true;
 	int ret;
 
 	wow_sta = ieee80211_find_sta(wow_vif, rtwvif->bssid);
 	if (wow_sta)
 		rtwsta = (struct rtw89_sta *)wow_sta->drv_priv;
-	else
-		is_conn = false;
 
 	if (wow) {
 		if (rtw_wow->pattern_cnt)
@@ -454,12 +451,6 @@ static int rtw89_wow_cfg_wake(struct rtw89_dev *rtwdev, bool wow)
 		}
 	}
 
-	ret = rtw89_fw_h2c_join_info(rtwdev, rtwvif, rtwsta, !is_conn);
-	if (ret) {
-		rtw89_warn(rtwdev, "failed to send h2c join info\n");
-		return ret;
-	}
-
 	ret = rtw89_fw_h2c_cam(rtwdev, rtwvif, rtwsta, NULL);
 	if (ret) {
 		rtw89_warn(rtwdev, "failed to send h2c cam\n");
diff --git a/drivers/net/wireless/rsi/rsi_91x_mgmt.c b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
index 1b309e47a1f1..7f2c1608f2ce 100644
--- a/drivers/net/wireless/rsi/rsi_91x_mgmt.c
+++ b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
@@ -1127,6 +1127,9 @@ int rsi_set_channel(struct rsi_common *common,
 	rsi_dbg(MGMT_TX_ZONE,
 		"%s: Sending scan req frame\n", __func__);
 
+	if (!channel)
+		return 0;
+
 	skb = dev_alloc_skb(frame_len);
 	if (!skb) {
 		rsi_dbg(ERR_ZONE, "%s: Failed in allocation of skb\n",
@@ -1134,10 +1137,6 @@ int rsi_set_channel(struct rsi_common *common,
 		return -ENOMEM;
 	}
 
-	if (!channel) {
-		dev_kfree_skb(skb);
-		return 0;
-	}
 	memset(skb->data, 0, frame_len);
 	chan_cfg = (struct rsi_chan_config *)skb->data;
 
diff --git a/drivers/net/wireless/silabs/wfx/bus_spi.c b/drivers/net/wireless/silabs/wfx/bus_spi.c
index 7fb1afb8ed31..160b90114aad 100644
--- a/drivers/net/wireless/silabs/wfx/bus_spi.c
+++ b/drivers/net/wireless/silabs/wfx/bus_spi.c
@@ -208,7 +208,7 @@ static int wfx_spi_probe(struct spi_device *func)
 
 	/* Trace below is also displayed by spi_setup() if compiled with DEBUG */
 	dev_dbg(&func->dev, "SPI params: CS=%d, mode=%d bits/word=%d speed=%d\n",
-		func->chip_select, func->mode, func->bits_per_word, func->max_speed_hz);
+		spi_get_chipselect(func, 0), func->mode, func->bits_per_word, func->max_speed_hz);
 	if (func->bits_per_word != 16 && func->bits_per_word != 8)
 		dev_warn(&func->dev, "unusual bits/word value: %d\n", func->bits_per_word);
 	if (func->max_speed_hz > 50000000)
diff --git a/drivers/net/wireless/silabs/wfx/main.c b/drivers/net/wireless/silabs/wfx/main.c
index 6b9864e478ac..0b50f7058bbb 100644
--- a/drivers/net/wireless/silabs/wfx/main.c
+++ b/drivers/net/wireless/silabs/wfx/main.c
@@ -358,13 +358,9 @@ int wfx_probe(struct wfx_dev *wdev)
 
 	wfx_bh_poll_irq(wdev);
 	err = wait_for_completion_timeout(&wdev->firmware_ready, 1 * HZ);
-	if (err <= 0) {
-		if (err == 0) {
-			dev_err(wdev->dev, "timeout while waiting for startup indication\n");
-			err = -ETIMEDOUT;
-		} else if (err == -ERESTARTSYS) {
-			dev_info(wdev->dev, "probe interrupted by user\n");
-		}
+	if (err == 0) {
+		dev_err(wdev->dev, "timeout while waiting for startup indication\n");
+		err = -ETIMEDOUT;
 		goto bh_unregister;
 	}
 
diff --git a/drivers/net/wireless/st/cw1200/cw1200_spi.c b/drivers/net/wireless/st/cw1200/cw1200_spi.c
index fe0d220da44d..c82c0688b549 100644
--- a/drivers/net/wireless/st/cw1200/cw1200_spi.c
+++ b/drivers/net/wireless/st/cw1200/cw1200_spi.c
@@ -378,7 +378,7 @@ static int cw1200_spi_probe(struct spi_device *func)
 	func->mode = SPI_MODE_0;
 
 	pr_info("cw1200_wlan_spi: Probe called (CS %d M %d BPW %d CLK %d)\n",
-		func->chip_select, func->mode, func->bits_per_word,
+		spi_get_chipselect(func, 0), func->mode, func->bits_per_word,
 		func->max_speed_hz);
 
 	if (cw1200_spi_on(plat_data)) {
diff --git a/drivers/net/wireless/virtual/Kconfig b/drivers/net/wireless/virtual/Kconfig
new file mode 100644
index 000000000000..fb3b4b69f26b
--- /dev/null
+++ b/drivers/net/wireless/virtual/Kconfig
@@ -0,0 +1,20 @@
+config MAC80211_HWSIM
+	tristate "Simulated radio testing tool for mac80211"
+	depends on MAC80211
+	help
+	  This driver is a developer testing tool that can be used to test
+	  IEEE 802.11 networking stack (mac80211) functionality. This is not
+	  needed for normal wireless LAN usage and is only for testing. See
+	  Documentation/networking/mac80211_hwsim for more information on how
+	  to use this tool.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called mac80211_hwsim.  If unsure, say N.
+
+config VIRT_WIFI
+	tristate "Wifi wrapper for ethernet drivers"
+	depends on CFG80211
+	help
+	  This option adds support for ethernet connections to appear as if they
+	  are wifi connections through a special rtnetlink device.
+
diff --git a/drivers/net/wireless/virtual/Makefile b/drivers/net/wireless/virtual/Makefile
new file mode 100644
index 000000000000..5773cc6d643e
--- /dev/null
+++ b/drivers/net/wireless/virtual/Makefile
@@ -0,0 +1,3 @@
+obj-$(CONFIG_MAC80211_HWSIM)	+= mac80211_hwsim.o
+
+obj-$(CONFIG_VIRT_WIFI)	+= virt_wifi.o
diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/virtual/mac80211_hwsim.c
index 4cc4eaf80b14..9a8faaf4c6b6 100644
--- a/drivers/net/wireless/mac80211_hwsim.c
+++ b/drivers/net/wireless/virtual/mac80211_hwsim.c
@@ -719,6 +719,11 @@ struct mac80211_hwsim_data {
 	/* RSSI in rx status of the receiver */
 	int rx_rssi;
 
+	/* only used when pmsr capability is supplied */
+	struct cfg80211_pmsr_capabilities pmsr_capa;
+	struct cfg80211_pmsr_request *pmsr_request;
+	struct wireless_dev *pmsr_request_wdev;
+
 	struct mac80211_hwsim_link_data link_data[IEEE80211_MLD_MAX_NUM_LINKS];
 };
 
@@ -747,6 +752,11 @@ struct hwsim_radiotap_ack_hdr {
 	__le16 rt_chbitmask;
 } __packed;
 
+static struct mac80211_hwsim_data *get_hwsim_data_ref_from_addr(const u8 *addr)
+{
+	return rhashtable_lookup_fast(&hwsim_radios_rht, addr, hwsim_rht_params);
+}
+
 /* MAC80211_HWSIM netlink family */
 static struct genl_family hwsim_genl_family;
 
@@ -760,6 +770,104 @@ static const struct genl_multicast_group hwsim_mcgrps[] = {
 
 /* MAC80211_HWSIM netlink policy */
 
+static const struct nla_policy
+hwsim_rate_info_policy[HWSIM_RATE_INFO_ATTR_MAX + 1] = {
+	[HWSIM_RATE_INFO_ATTR_FLAGS] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_MCS] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_LEGACY] = { .type = NLA_U16 },
+	[HWSIM_RATE_INFO_ATTR_NSS] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_BW] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_HE_GI] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_HE_DCM] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_EHT_GI] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC] = { .type = NLA_U8 },
+};
+
+static const struct nla_policy
+hwsim_ftm_result_policy[NL80211_PMSR_FTM_RESP_ATTR_MAX + 1] = {
+	[NL80211_PMSR_FTM_RESP_ATTR_FAIL_REASON] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_BURST_INDEX] = { .type = NLA_U16 },
+	[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_BUSY_RETRY_TIME] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_RESP_ATTR_BURST_DURATION] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_RESP_ATTR_FTMS_PER_BURST] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_RESP_ATTR_RSSI_AVG] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_RSSI_SPREAD] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_TX_RATE] = NLA_POLICY_NESTED(hwsim_rate_info_policy),
+	[NL80211_PMSR_FTM_RESP_ATTR_RX_RATE] = NLA_POLICY_NESTED(hwsim_rate_info_policy),
+	[NL80211_PMSR_FTM_RESP_ATTR_RTT_AVG] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_RTT_VARIANCE] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_RTT_SPREAD] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_DIST_AVG] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_DIST_VARIANCE] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_DIST_SPREAD] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_LCI] = { .type = NLA_STRING },
+	[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC] = { .type = NLA_STRING },
+};
+
+static const struct nla_policy
+hwsim_pmsr_resp_type_policy[NL80211_PMSR_TYPE_MAX + 1] = {
+	[NL80211_PMSR_TYPE_FTM] = NLA_POLICY_NESTED(hwsim_ftm_result_policy),
+};
+
+static const struct nla_policy
+hwsim_pmsr_resp_policy[NL80211_PMSR_RESP_ATTR_MAX + 1] = {
+	[NL80211_PMSR_RESP_ATTR_STATUS] = { .type = NLA_U32 },
+	[NL80211_PMSR_RESP_ATTR_HOST_TIME] = { .type = NLA_U64 },
+	[NL80211_PMSR_RESP_ATTR_AP_TSF] = { .type = NLA_U64 },
+	[NL80211_PMSR_RESP_ATTR_FINAL] = { .type = NLA_FLAG },
+	[NL80211_PMSR_RESP_ATTR_DATA] = NLA_POLICY_NESTED(hwsim_pmsr_resp_type_policy),
+};
+
+static const struct nla_policy
+hwsim_pmsr_peer_result_policy[NL80211_PMSR_PEER_ATTR_MAX + 1] = {
+	[NL80211_PMSR_PEER_ATTR_ADDR] = NLA_POLICY_ETH_ADDR_COMPAT,
+	[NL80211_PMSR_PEER_ATTR_CHAN] = { .type = NLA_REJECT },
+	[NL80211_PMSR_PEER_ATTR_REQ] = { .type = NLA_REJECT },
+	[NL80211_PMSR_PEER_ATTR_RESP] = NLA_POLICY_NESTED(hwsim_pmsr_resp_policy),
+};
+
+static const struct nla_policy
+hwsim_pmsr_peers_result_policy[NL80211_PMSR_ATTR_MAX + 1] = {
+	[NL80211_PMSR_ATTR_MAX_PEERS] = { .type = NLA_REJECT },
+	[NL80211_PMSR_ATTR_REPORT_AP_TSF] = { .type = NLA_REJECT },
+	[NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR] = { .type = NLA_REJECT },
+	[NL80211_PMSR_ATTR_TYPE_CAPA] = { .type = NLA_REJECT },
+	[NL80211_PMSR_ATTR_PEERS] = NLA_POLICY_NESTED_ARRAY(hwsim_pmsr_peer_result_policy),
+};
+
+static const struct nla_policy
+hwsim_ftm_capa_policy[NL80211_PMSR_FTM_CAPA_ATTR_MAX + 1] = {
+	[NL80211_PMSR_FTM_CAPA_ATTR_ASAP] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_NON_ASAP] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_REQ_LCI] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_REQ_CIVICLOC] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_PREAMBLES] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT] = NLA_POLICY_MAX(NLA_U8, 15),
+	[NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST] = NLA_POLICY_MAX(NLA_U8, 31),
+	[NL80211_PMSR_FTM_CAPA_ATTR_TRIGGER_BASED] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_NON_TRIGGER_BASED] = { .type = NLA_FLAG },
+};
+
+static const struct nla_policy
+hwsim_pmsr_capa_type_policy[NL80211_PMSR_TYPE_MAX + 1] = {
+	[NL80211_PMSR_TYPE_FTM] = NLA_POLICY_NESTED(hwsim_ftm_capa_policy),
+};
+
+static const struct nla_policy
+hwsim_pmsr_capa_policy[NL80211_PMSR_ATTR_MAX + 1] = {
+	[NL80211_PMSR_ATTR_MAX_PEERS] = { .type = NLA_U32 },
+	[NL80211_PMSR_ATTR_REPORT_AP_TSF] = { .type = NLA_FLAG },
+	[NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR] = { .type = NLA_FLAG },
+	[NL80211_PMSR_ATTR_TYPE_CAPA] = NLA_POLICY_NESTED(hwsim_pmsr_capa_type_policy),
+	[NL80211_PMSR_ATTR_PEERS] = { .type = NLA_REJECT }, // only for request.
+};
+
 static const struct nla_policy hwsim_genl_policy[HWSIM_ATTR_MAX + 1] = {
 	[HWSIM_ATTR_ADDR_RECEIVER] = NLA_POLICY_ETH_ADDR_COMPAT,
 	[HWSIM_ATTR_ADDR_TRANSMITTER] = NLA_POLICY_ETH_ADDR_COMPAT,
@@ -788,6 +896,8 @@ static const struct nla_policy hwsim_genl_policy[HWSIM_ATTR_MAX + 1] = {
 	[HWSIM_ATTR_IFTYPE_SUPPORT] = { .type = NLA_U32 },
 	[HWSIM_ATTR_CIPHER_SUPPORT] = { .type = NLA_BINARY },
 	[HWSIM_ATTR_MLO_SUPPORT] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_PMSR_SUPPORT] = NLA_POLICY_NESTED(hwsim_pmsr_capa_policy),
+	[HWSIM_ATTR_PMSR_RESULT] = NLA_POLICY_NESTED(hwsim_pmsr_peers_result_policy),
 };
 
 #if IS_REACHABLE(CONFIG_VIRTIO)
@@ -1534,37 +1644,38 @@ static void mac80211_hwsim_add_vendor_rtap(struct sk_buff *skb)
 	 * the values accordingly.
 	 */
 #ifdef HWSIM_RADIOTAP_OUI
-	struct ieee80211_vendor_radiotap *rtap;
+	struct ieee80211_radiotap_vendor_tlv *rtap;
+	static const char vendor_data[8] = "ABCDEFGH";
+
+	// Make sure no padding is needed
+	BUILD_BUG_ON(sizeof(vendor_data) % 4);
+	/* this is last radiotap info before the mac header, so
+	 * skb_reset_mac_header for mac8022 to know the end of
+	 * the radiotap TLV/beginning of the 802.11 header
+	 */
+	skb_reset_mac_header(skb);
 
 	/*
 	 * Note that this code requires the headroom in the SKB
 	 * that was allocated earlier.
 	 */
-	rtap = skb_push(skb, sizeof(*rtap) + 8 + 4);
-	rtap->oui[0] = HWSIM_RADIOTAP_OUI[0];
-	rtap->oui[1] = HWSIM_RADIOTAP_OUI[1];
-	rtap->oui[2] = HWSIM_RADIOTAP_OUI[2];
-	rtap->subns = 127;
-
-	/*
-	 * Radiotap vendor namespaces can (and should) also be
-	 * split into fields by using the standard radiotap
-	 * presence bitmap mechanism. Use just BIT(0) here for
-	 * the presence bitmap.
-	 */
-	rtap->present = BIT(0);
-	/* We have 8 bytes of (dummy) data */
-	rtap->len = 8;
-	/* For testing, also require it to be aligned */
-	rtap->align = 8;
-	/* And also test that padding works, 4 bytes */
-	rtap->pad = 4;
-	/* push the data */
-	memcpy(rtap->data, "ABCDEFGH", 8);
-	/* make sure to clear padding, mac80211 doesn't */
-	memset(rtap->data + 8, 0, 4);
-
-	IEEE80211_SKB_RXCB(skb)->flag |= RX_FLAG_RADIOTAP_VENDOR_DATA;
+	rtap = skb_push(skb, sizeof(*rtap) + sizeof(vendor_data));
+
+	rtap->len = cpu_to_le16(sizeof(*rtap) -
+				sizeof(struct ieee80211_radiotap_tlv) +
+				sizeof(vendor_data));
+	rtap->type = cpu_to_le16(IEEE80211_RADIOTAP_VENDOR_NAMESPACE);
+
+	rtap->content.oui[0] = HWSIM_RADIOTAP_OUI[0];
+	rtap->content.oui[1] = HWSIM_RADIOTAP_OUI[1];
+	rtap->content.oui[2] = HWSIM_RADIOTAP_OUI[2];
+	rtap->content.oui_subtype = 127;
+	/* clear reserved field */
+	rtap->content.reserved = 0;
+	rtap->content.vendor_type = 0;
+	memcpy(rtap->content.data, vendor_data, sizeof(vendor_data));
+
+	IEEE80211_SKB_RXCB(skb)->flag |= RX_FLAG_RADIOTAP_TLV_AT_END;
 #endif
 }
 
@@ -2054,38 +2165,18 @@ static void mac80211_hwsim_tx_frame(struct ieee80211_hw *hw,
 	dev_kfree_skb(skb);
 }
 
-static void mac80211_hwsim_beacon_tx(void *arg, u8 *mac,
-				     struct ieee80211_vif *vif)
+static void __mac80211_hwsim_beacon_tx(struct ieee80211_bss_conf *link_conf,
+				       struct mac80211_hwsim_data *data,
+				       struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       struct sk_buff *skb)
 {
-	struct mac80211_hwsim_link_data *link_data = arg;
-	u32 link_id = link_data->link_id;
-	struct ieee80211_bss_conf *link_conf;
-	struct mac80211_hwsim_data *data =
-		container_of(link_data, struct mac80211_hwsim_data,
-			     link_data[link_id]);
-	struct ieee80211_hw *hw = data->hw;
 	struct ieee80211_tx_info *info;
 	struct ieee80211_rate *txrate;
 	struct ieee80211_mgmt *mgmt;
-	struct sk_buff *skb;
 	/* TODO: get MCS */
 	int bitrate = 100;
 
-	hwsim_check_magic(vif);
-
-	link_conf = rcu_dereference(vif->link_conf[link_id]);
-	if (!link_conf)
-		return;
-
-	if (vif->type != NL80211_IFTYPE_AP &&
-	    vif->type != NL80211_IFTYPE_MESH_POINT &&
-	    vif->type != NL80211_IFTYPE_ADHOC &&
-	    vif->type != NL80211_IFTYPE_OCB)
-		return;
-
-	skb = ieee80211_beacon_get(hw, vif, link_data->link_id);
-	if (skb == NULL)
-		return;
 	info = IEEE80211_SKB_CB(skb);
 	if (ieee80211_hw_check(hw, SUPPORTS_RC_TABLE))
 		ieee80211_get_tx_rates(vif, NULL, skb,
@@ -2115,6 +2206,56 @@ static void mac80211_hwsim_beacon_tx(void *arg, u8 *mac,
 
 	mac80211_hwsim_tx_frame(hw, skb,
 			rcu_dereference(link_conf->chanctx_conf)->def.chan);
+}
+
+static void mac80211_hwsim_beacon_tx(void *arg, u8 *mac,
+				     struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_link_data *link_data = arg;
+	u32 link_id = link_data->link_id;
+	struct ieee80211_bss_conf *link_conf;
+	struct mac80211_hwsim_data *data =
+		container_of(link_data, struct mac80211_hwsim_data,
+			     link_data[link_id]);
+	struct ieee80211_hw *hw = data->hw;
+	struct sk_buff *skb;
+
+	hwsim_check_magic(vif);
+
+	link_conf = rcu_dereference(vif->link_conf[link_id]);
+	if (!link_conf)
+		return;
+
+	if (vif->type != NL80211_IFTYPE_AP &&
+	    vif->type != NL80211_IFTYPE_MESH_POINT &&
+	    vif->type != NL80211_IFTYPE_ADHOC &&
+	    vif->type != NL80211_IFTYPE_OCB)
+		return;
+
+	if (vif->mbssid_tx_vif && vif->mbssid_tx_vif != vif)
+		return;
+
+	if (vif->bss_conf.ema_ap) {
+		struct ieee80211_ema_beacons *ema;
+		u8 i = 0;
+
+		ema = ieee80211_beacon_get_template_ema_list(hw, vif, link_id);
+		if (!ema || !ema->cnt)
+			return;
+
+		for (i = 0; i < ema->cnt; i++) {
+			__mac80211_hwsim_beacon_tx(link_conf, data, hw, vif,
+						   ema->bcn[i].skb);
+			ema->bcn[i].skb = NULL; /* Already freed */
+		}
+		ieee80211_beacon_free_ema_list(ema);
+	} else {
+		skb = ieee80211_beacon_get(hw, vif, link_id);
+		if (!skb)
+			return;
+
+		__mac80211_hwsim_beacon_tx(link_conf, data, hw, vif, skb);
+	}
 
 	while ((skb = ieee80211_get_buffered_bc(hw, vif)) != NULL) {
 		mac80211_hwsim_tx_frame(hw, skb,
@@ -3107,6 +3248,566 @@ static int mac80211_hwsim_change_sta_links(struct ieee80211_hw *hw,
 	return 0;
 }
 
+static int mac80211_hwsim_send_pmsr_ftm_request_peer(struct sk_buff *msg,
+						     struct cfg80211_pmsr_ftm_request_peer *request)
+{
+	struct nlattr *ftm;
+
+	if (!request->requested)
+		return -EINVAL;
+
+	ftm = nla_nest_start(msg, NL80211_PMSR_TYPE_FTM);
+	if (!ftm)
+		return -ENOBUFS;
+
+	if (nla_put_u32(msg, NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE, request->preamble))
+		return -ENOBUFS;
+
+	if (nla_put_u16(msg, NL80211_PMSR_FTM_REQ_ATTR_BURST_PERIOD, request->burst_period))
+		return -ENOBUFS;
+
+	if (request->asap && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_ASAP))
+		return -ENOBUFS;
+
+	if (request->request_lci && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_REQUEST_LCI))
+		return -ENOBUFS;
+
+	if (request->request_civicloc &&
+	    nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_REQUEST_CIVICLOC))
+		return -ENOBUFS;
+
+	if (request->trigger_based && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_TRIGGER_BASED))
+		return -ENOBUFS;
+
+	if (request->non_trigger_based &&
+	    nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED))
+		return -ENOBUFS;
+
+	if (request->lmr_feedback && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_LMR_FEEDBACK))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_NUM_BURSTS_EXP, request->num_bursts_exp))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_BURST_DURATION, request->burst_duration))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST, request->ftms_per_burst))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_NUM_FTMR_RETRIES, request->ftmr_retries))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_BURST_DURATION, request->burst_duration))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR, request->bss_color))
+		return -ENOBUFS;
+
+	nla_nest_end(msg, ftm);
+
+	return 0;
+}
+
+static int mac80211_hwsim_send_pmsr_request_peer(struct sk_buff *msg,
+						 struct cfg80211_pmsr_request_peer *request)
+{
+	struct nlattr *peer, *chandef, *req, *data;
+	int err;
+
+	peer = nla_nest_start(msg, NL80211_PMSR_ATTR_PEERS);
+	if (!peer)
+		return -ENOBUFS;
+
+	if (nla_put(msg, NL80211_PMSR_PEER_ATTR_ADDR, ETH_ALEN,
+		    request->addr))
+		return -ENOBUFS;
+
+	chandef = nla_nest_start(msg, NL80211_PMSR_PEER_ATTR_CHAN);
+	if (!chandef)
+		return -ENOBUFS;
+
+	err = nl80211_send_chandef(msg, &request->chandef);
+	if (err)
+		return err;
+
+	nla_nest_end(msg, chandef);
+
+	req = nla_nest_start(msg, NL80211_PMSR_PEER_ATTR_REQ);
+	if (!req)
+		return -ENOBUFS;
+
+	if (request->report_ap_tsf && nla_put_flag(msg, NL80211_PMSR_REQ_ATTR_GET_AP_TSF))
+		return -ENOBUFS;
+
+	data = nla_nest_start(msg, NL80211_PMSR_REQ_ATTR_DATA);
+	if (!data)
+		return -ENOBUFS;
+
+	err = mac80211_hwsim_send_pmsr_ftm_request_peer(msg, &request->ftm);
+	if (err)
+		return err;
+
+	nla_nest_end(msg, data);
+	nla_nest_end(msg, req);
+	nla_nest_end(msg, peer);
+
+	return 0;
+}
+
+static int mac80211_hwsim_send_pmsr_request(struct sk_buff *msg,
+					    struct cfg80211_pmsr_request *request)
+{
+	struct nlattr *pmsr;
+	int err;
+
+	pmsr = nla_nest_start(msg, NL80211_ATTR_PEER_MEASUREMENTS);
+	if (!pmsr)
+		return -ENOBUFS;
+
+	if (nla_put_u32(msg, NL80211_ATTR_TIMEOUT, request->timeout))
+		return -ENOBUFS;
+
+	if (!is_zero_ether_addr(request->mac_addr)) {
+		if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, request->mac_addr))
+			return -ENOBUFS;
+		if (nla_put(msg, NL80211_ATTR_MAC_MASK, ETH_ALEN, request->mac_addr_mask))
+			return -ENOBUFS;
+	}
+
+	for (int i = 0; i < request->n_peers; i++) {
+		err = mac80211_hwsim_send_pmsr_request_peer(msg, &request->peers[i]);
+		if (err)
+			return err;
+	}
+
+	nla_nest_end(msg, pmsr);
+
+	return 0;
+}
+
+static int mac80211_hwsim_start_pmsr(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct cfg80211_pmsr_request *request)
+{
+	struct mac80211_hwsim_data *data;
+	struct sk_buff *skb = NULL;
+	struct nlattr *pmsr;
+	void *msg_head;
+	u32 _portid;
+	int err = 0;
+
+	data = hw->priv;
+	_portid = READ_ONCE(data->wmediumd);
+	if (!_portid && !hwsim_virtio_enabled)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&data->mutex);
+
+	if (data->pmsr_request) {
+		err = -EBUSY;
+		goto out_free;
+	}
+
+	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+
+	if (!skb) {
+		err = -ENOMEM;
+		goto out_free;
+	}
+
+	msg_head = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0, HWSIM_CMD_START_PMSR);
+
+	if (nla_put(skb, HWSIM_ATTR_ADDR_TRANSMITTER,
+		    ETH_ALEN, data->addresses[1].addr)) {
+		err = -ENOMEM;
+		goto out_free;
+	}
+
+	pmsr = nla_nest_start(skb, HWSIM_ATTR_PMSR_REQUEST);
+	if (!pmsr) {
+		err = -ENOMEM;
+		goto out_free;
+	}
+
+	err = mac80211_hwsim_send_pmsr_request(skb, request);
+	if (err)
+		goto out_free;
+
+	nla_nest_end(skb, pmsr);
+
+	genlmsg_end(skb, msg_head);
+	if (hwsim_virtio_enabled)
+		hwsim_tx_virtio(data, skb);
+	else
+		hwsim_unicast_netgroup(data, skb, _portid);
+
+	data->pmsr_request = request;
+	data->pmsr_request_wdev = ieee80211_vif_to_wdev(vif);
+
+out_free:
+	if (err && skb)
+		nlmsg_free(skb);
+
+	mutex_unlock(&data->mutex);
+	return err;
+}
+
+static void mac80211_hwsim_abort_pmsr(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      struct cfg80211_pmsr_request *request)
+{
+	struct mac80211_hwsim_data *data;
+	struct sk_buff *skb = NULL;
+	struct nlattr *pmsr;
+	void *msg_head;
+	u32 _portid;
+	int err = 0;
+
+	data = hw->priv;
+	_portid = READ_ONCE(data->wmediumd);
+	if (!_portid && !hwsim_virtio_enabled)
+		return;
+
+	mutex_lock(&data->mutex);
+
+	if (data->pmsr_request != request) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!skb) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	msg_head = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0, HWSIM_CMD_ABORT_PMSR);
+
+	if (nla_put(skb, HWSIM_ATTR_ADDR_TRANSMITTER, ETH_ALEN, data->addresses[1].addr))
+		goto out;
+
+	pmsr = nla_nest_start(skb, HWSIM_ATTR_PMSR_REQUEST);
+	if (!pmsr) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	err = mac80211_hwsim_send_pmsr_request(skb, request);
+	if (err)
+		goto out;
+
+	err = nla_nest_end(skb, pmsr);
+	if (err)
+		goto out;
+
+	genlmsg_end(skb, msg_head);
+	if (hwsim_virtio_enabled)
+		hwsim_tx_virtio(data, skb);
+	else
+		hwsim_unicast_netgroup(data, skb, _portid);
+
+out:
+	if (err && skb)
+		nlmsg_free(skb);
+
+	mutex_unlock(&data->mutex);
+}
+
+static int mac80211_hwsim_parse_rate_info(struct nlattr *rateattr,
+					  struct rate_info *rate_info,
+					  struct genl_info *info)
+{
+	struct nlattr *tb[HWSIM_RATE_INFO_ATTR_MAX + 1];
+	int ret;
+
+	ret = nla_parse_nested(tb, HWSIM_RATE_INFO_ATTR_MAX,
+			       rateattr, hwsim_rate_info_policy, info->extack);
+	if (ret)
+		return ret;
+
+	if (tb[HWSIM_RATE_INFO_ATTR_FLAGS])
+		rate_info->flags = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_FLAGS]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_MCS])
+		rate_info->mcs = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_MCS]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_LEGACY])
+		rate_info->legacy = nla_get_u16(tb[HWSIM_RATE_INFO_ATTR_LEGACY]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_NSS])
+		rate_info->nss = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_NSS]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_BW])
+		rate_info->bw = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_BW]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_HE_GI])
+		rate_info->he_gi = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_HE_GI]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_HE_DCM])
+		rate_info->he_dcm = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_HE_DCM]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC])
+		rate_info->he_ru_alloc =
+			nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH])
+		rate_info->n_bonded_ch = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_EHT_GI])
+		rate_info->eht_gi = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_EHT_GI]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC])
+		rate_info->eht_ru_alloc = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC]);
+
+	return 0;
+}
+
+static int mac80211_hwsim_parse_ftm_result(struct nlattr *ftm,
+					   struct cfg80211_pmsr_ftm_result *result,
+					   struct genl_info *info)
+{
+	struct nlattr *tb[NL80211_PMSR_FTM_RESP_ATTR_MAX + 1];
+	int ret;
+
+	ret = nla_parse_nested(tb, NL80211_PMSR_FTM_RESP_ATTR_MAX,
+			       ftm, hwsim_ftm_result_policy, info->extack);
+	if (ret)
+		return ret;
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_FAIL_REASON])
+		result->failure_reason = nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_FAIL_REASON]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_INDEX])
+		result->burst_index = nla_get_u16(tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_INDEX]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS]) {
+		result->num_ftmr_attempts_valid = 1;
+		result->num_ftmr_attempts =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES]) {
+		result->num_ftmr_successes_valid = 1;
+		result->num_ftmr_successes =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_BUSY_RETRY_TIME])
+		result->busy_retry_time =
+			nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_BUSY_RETRY_TIME]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP])
+		result->num_bursts_exp = nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_DURATION])
+		result->burst_duration = nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_DURATION]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_FTMS_PER_BURST])
+		result->ftms_per_burst = nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_FTMS_PER_BURST]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_AVG]) {
+		result->rssi_avg_valid = 1;
+		result->rssi_avg = nla_get_s32(tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_AVG]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_SPREAD]) {
+		result->rssi_spread_valid = 1;
+		result->rssi_spread =
+			nla_get_s32(tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_SPREAD]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_TX_RATE]) {
+		result->tx_rate_valid = 1;
+		ret = mac80211_hwsim_parse_rate_info(tb[NL80211_PMSR_FTM_RESP_ATTR_TX_RATE],
+						     &result->tx_rate, info);
+		if (ret)
+			return ret;
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RX_RATE]) {
+		result->rx_rate_valid = 1;
+		ret = mac80211_hwsim_parse_rate_info(tb[NL80211_PMSR_FTM_RESP_ATTR_RX_RATE],
+						     &result->rx_rate, info);
+		if (ret)
+			return ret;
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_AVG]) {
+		result->rtt_avg_valid = 1;
+		result->rtt_avg =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_AVG]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_VARIANCE]) {
+		result->rtt_variance_valid = 1;
+		result->rtt_variance =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_VARIANCE]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_SPREAD]) {
+		result->rtt_spread_valid = 1;
+		result->rtt_spread =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_SPREAD]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_AVG]) {
+		result->dist_avg_valid = 1;
+		result->dist_avg =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_AVG]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_VARIANCE]) {
+		result->dist_variance_valid = 1;
+		result->dist_variance =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_VARIANCE]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_SPREAD]) {
+		result->dist_spread_valid = 1;
+		result->dist_spread =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_SPREAD]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_LCI]) {
+		result->lci = nla_data(tb[NL80211_PMSR_FTM_RESP_ATTR_LCI]);
+		result->lci_len = nla_len(tb[NL80211_PMSR_FTM_RESP_ATTR_LCI]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC]) {
+		result->civicloc = nla_data(tb[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC]);
+		result->civicloc_len = nla_len(tb[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC]);
+	}
+
+	return 0;
+}
+
+static int mac80211_hwsim_parse_pmsr_resp(struct nlattr *resp,
+					  struct cfg80211_pmsr_result *result,
+					  struct genl_info *info)
+{
+	struct nlattr *tb[NL80211_PMSR_RESP_ATTR_MAX + 1];
+	struct nlattr *pmsr;
+	int rem;
+	int ret;
+
+	ret = nla_parse_nested(tb, NL80211_PMSR_RESP_ATTR_MAX, resp, hwsim_pmsr_resp_policy,
+			       info->extack);
+	if (ret)
+		return ret;
+
+	if (tb[NL80211_PMSR_RESP_ATTR_STATUS])
+		result->status = nla_get_u32(tb[NL80211_PMSR_RESP_ATTR_STATUS]);
+
+	if (tb[NL80211_PMSR_RESP_ATTR_HOST_TIME])
+		result->host_time = nla_get_u64(tb[NL80211_PMSR_RESP_ATTR_HOST_TIME]);
+
+	if (tb[NL80211_PMSR_RESP_ATTR_AP_TSF]) {
+		result->ap_tsf_valid = 1;
+		result->ap_tsf = nla_get_u64(tb[NL80211_PMSR_RESP_ATTR_AP_TSF]);
+	}
+
+	result->final = !!tb[NL80211_PMSR_RESP_ATTR_FINAL];
+
+	if (!tb[NL80211_PMSR_RESP_ATTR_DATA])
+		return 0;
+
+	nla_for_each_nested(pmsr, tb[NL80211_PMSR_RESP_ATTR_DATA], rem) {
+		switch (nla_type(pmsr)) {
+		case NL80211_PMSR_TYPE_FTM:
+			result->type = NL80211_PMSR_TYPE_FTM;
+			ret = mac80211_hwsim_parse_ftm_result(pmsr, &result->ftm, info);
+			if (ret)
+				return ret;
+			break;
+		default:
+			NL_SET_ERR_MSG_ATTR(info->extack, pmsr, "Unknown pmsr resp type");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int mac80211_hwsim_parse_pmsr_result(struct nlattr *peer,
+					    struct cfg80211_pmsr_result *result,
+					    struct genl_info *info)
+{
+	struct nlattr *tb[NL80211_PMSR_PEER_ATTR_MAX + 1];
+	int ret;
+
+	if (!peer)
+		return -EINVAL;
+
+	ret = nla_parse_nested(tb, NL80211_PMSR_PEER_ATTR_MAX, peer,
+			       hwsim_pmsr_peer_result_policy, info->extack);
+	if (ret)
+		return ret;
+
+	if (tb[NL80211_PMSR_PEER_ATTR_ADDR])
+		memcpy(result->addr, nla_data(tb[NL80211_PMSR_PEER_ATTR_ADDR]),
+		       ETH_ALEN);
+
+	if (tb[NL80211_PMSR_PEER_ATTR_RESP]) {
+		ret = mac80211_hwsim_parse_pmsr_resp(tb[NL80211_PMSR_PEER_ATTR_RESP], result, info);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+};
+
+static int hwsim_pmsr_report_nl(struct sk_buff *msg, struct genl_info *info)
+{
+	struct mac80211_hwsim_data *data;
+	struct nlattr *peers, *peer;
+	struct nlattr *reqattr;
+	const u8 *src;
+	int err;
+	int rem;
+
+	if (!info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER])
+		return -EINVAL;
+
+	src = nla_data(info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER]);
+	data = get_hwsim_data_ref_from_addr(src);
+	if (!data)
+		return -EINVAL;
+
+	mutex_lock(&data->mutex);
+	if (!data->pmsr_request) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	reqattr = info->attrs[HWSIM_ATTR_PMSR_RESULT];
+	if (!reqattr) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	peers = nla_find_nested(reqattr, NL80211_PMSR_ATTR_PEERS);
+	if (!peers) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	nla_for_each_nested(peer, peers, rem) {
+		struct cfg80211_pmsr_result result;
+
+		err = mac80211_hwsim_parse_pmsr_result(peer, &result, info);
+		if (err)
+			goto out;
+
+		cfg80211_pmsr_report(data->pmsr_request_wdev,
+				     data->pmsr_request, &result, GFP_KERNEL);
+	}
+
+	cfg80211_pmsr_complete(data->pmsr_request_wdev, data->pmsr_request, GFP_KERNEL);
+
+	err = 0;
+out:
+	data->pmsr_request = NULL;
+	data->pmsr_request_wdev = NULL;
+
+	mutex_unlock(&data->mutex);
+	return err;
+}
+
 #define HWSIM_COMMON_OPS					\
 	.tx = mac80211_hwsim_tx,				\
 	.wake_tx_queue = ieee80211_handle_wake_tx_queue,	\
@@ -3129,7 +3830,9 @@ static int mac80211_hwsim_change_sta_links(struct ieee80211_hw *hw,
 	.flush = mac80211_hwsim_flush,				\
 	.get_et_sset_count = mac80211_hwsim_get_et_sset_count,	\
 	.get_et_stats = mac80211_hwsim_get_et_stats,		\
-	.get_et_strings = mac80211_hwsim_get_et_strings,
+	.get_et_strings = mac80211_hwsim_get_et_strings,	\
+	.start_pmsr = mac80211_hwsim_start_pmsr,		\
+	.abort_pmsr = mac80211_hwsim_abort_pmsr,
 
 #define HWSIM_NON_MLO_OPS					\
 	.sta_add = mac80211_hwsim_sta_add,			\
@@ -3186,6 +3889,7 @@ struct hwsim_new_radio_params {
 	u32 *ciphers;
 	u8 n_ciphers;
 	bool mlo;
+	const struct cfg80211_pmsr_capabilities *pmsr_capa;
 };
 
 static void hwsim_mcast_config_msg(struct sk_buff *mcast_skb,
@@ -4393,6 +5097,9 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 		hw->wiphy->n_cipher_suites = param->n_ciphers;
 	}
 
+	hw->wiphy->mbssid_max_interfaces = 8;
+	hw->wiphy->ema_max_profile_periodicity = 3;
+
 	data->rx_rssi = DEFAULT_RX_RSSI;
 
 	INIT_DELAYED_WORK(&data->roc_start, hw_roc_start);
@@ -4445,6 +5152,10 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 			      NL80211_EXT_FEATURE_MULTICAST_REGISTRATIONS);
 	wiphy_ext_feature_set(hw->wiphy,
 			      NL80211_EXT_FEATURE_BEACON_RATE_LEGACY);
+	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER);
+
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT);
 
 	hw->wiphy->interface_modes = param->iftypes;
 
@@ -4606,6 +5317,11 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 				    data->debugfs,
 				    data, &hwsim_simulate_radar);
 
+	if (param->pmsr_capa) {
+		data->pmsr_capa = *param->pmsr_capa;
+		hw->wiphy->pmsr_capa = &data->pmsr_capa;
+	}
+
 	spin_lock_bh(&hwsim_radio_lock);
 	err = rhashtable_insert_fast(&hwsim_radios_rht, &data->rht,
 				     hwsim_rht_params);
@@ -4715,6 +5431,7 @@ static int mac80211_hwsim_get_radio(struct sk_buff *skb,
 	param.regd = data->regd;
 	param.channels = data->channels;
 	param.hwname = wiphy_name(data->hw->wiphy);
+	param.pmsr_capa = &data->pmsr_capa;
 
 	res = append_radio_msg(skb, data->idx, &param);
 	if (res < 0)
@@ -4766,13 +5483,6 @@ static void hwsim_mon_setup(struct net_device *dev)
 	eth_hw_addr_set(dev, addr);
 }
 
-static struct mac80211_hwsim_data *get_hwsim_data_ref_from_addr(const u8 *addr)
-{
-	return rhashtable_lookup_fast(&hwsim_radios_rht,
-				      addr,
-				      hwsim_rht_params);
-}
-
 static void hwsim_register_wmediumd(struct net *net, u32 portid)
 {
 	struct mac80211_hwsim_data *data;
@@ -5053,6 +5763,79 @@ static bool hwsim_known_ciphers(const u32 *ciphers, int n_ciphers)
 	return true;
 }
 
+static int parse_ftm_capa(const struct nlattr *ftm_capa, struct cfg80211_pmsr_capabilities *out,
+			  struct genl_info *info)
+{
+	struct nlattr *tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX + 1];
+	int ret;
+
+	ret = nla_parse_nested(tb, NL80211_PMSR_FTM_CAPA_ATTR_MAX, ftm_capa, hwsim_ftm_capa_policy,
+			       NULL);
+	if (ret) {
+		NL_SET_ERR_MSG_ATTR(info->extack, ftm_capa, "malformed FTM capability");
+		return -EINVAL;
+	}
+
+	out->ftm.supported = 1;
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_PREAMBLES])
+		out->ftm.preambles = nla_get_u32(tb[NL80211_PMSR_FTM_CAPA_ATTR_PREAMBLES]);
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS])
+		out->ftm.bandwidths = nla_get_u32(tb[NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS]);
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT])
+		out->ftm.max_bursts_exponent =
+			nla_get_u8(tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT]);
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST])
+		out->ftm.max_ftms_per_burst =
+			nla_get_u8(tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST]);
+	out->ftm.asap = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_ASAP];
+	out->ftm.non_asap = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_NON_ASAP];
+	out->ftm.request_lci = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_REQ_LCI];
+	out->ftm.request_civicloc = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_REQ_CIVICLOC];
+	out->ftm.trigger_based = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_TRIGGER_BASED];
+	out->ftm.non_trigger_based = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_NON_TRIGGER_BASED];
+
+	return 0;
+}
+
+static int parse_pmsr_capa(const struct nlattr *pmsr_capa, struct cfg80211_pmsr_capabilities *out,
+			   struct genl_info *info)
+{
+	struct nlattr *tb[NL80211_PMSR_ATTR_MAX + 1];
+	struct nlattr *nla;
+	int size;
+	int ret;
+
+	ret = nla_parse_nested(tb, NL80211_PMSR_ATTR_MAX, pmsr_capa, hwsim_pmsr_capa_policy, NULL);
+	if (ret) {
+		NL_SET_ERR_MSG_ATTR(info->extack, pmsr_capa, "malformed PMSR capability");
+		return -EINVAL;
+	}
+
+	if (tb[NL80211_PMSR_ATTR_MAX_PEERS])
+		out->max_peers = nla_get_u32(tb[NL80211_PMSR_ATTR_MAX_PEERS]);
+	out->report_ap_tsf = !!tb[NL80211_PMSR_ATTR_REPORT_AP_TSF];
+	out->randomize_mac_addr = !!tb[NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR];
+
+	if (!tb[NL80211_PMSR_ATTR_TYPE_CAPA]) {
+		NL_SET_ERR_MSG_ATTR(info->extack, tb[NL80211_PMSR_ATTR_TYPE_CAPA],
+				    "malformed PMSR type");
+		return -EINVAL;
+	}
+
+	nla_for_each_nested(nla, tb[NL80211_PMSR_ATTR_TYPE_CAPA], size) {
+		switch (nla_type(nla)) {
+		case NL80211_PMSR_TYPE_FTM:
+			parse_ftm_capa(nla, out, info);
+			break;
+		default:
+			NL_SET_ERR_MSG_ATTR(info->extack, nla, "unsupported measurement type");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
 static int hwsim_new_radio_nl(struct sk_buff *msg, struct genl_info *info)
 {
 	struct hwsim_new_radio_params param = { 0 };
@@ -5173,8 +5956,25 @@ static int hwsim_new_radio_nl(struct sk_buff *msg, struct genl_info *info)
 		param.hwname = hwname;
 	}
 
+	if (info->attrs[HWSIM_ATTR_PMSR_SUPPORT]) {
+		struct cfg80211_pmsr_capabilities *pmsr_capa;
+
+		pmsr_capa = kmalloc(sizeof(*pmsr_capa), GFP_KERNEL);
+		if (!pmsr_capa) {
+			ret = -ENOMEM;
+			goto out_free;
+		}
+		ret = parse_pmsr_capa(info->attrs[HWSIM_ATTR_PMSR_SUPPORT], pmsr_capa, info);
+		if (ret)
+			goto out_free;
+		param.pmsr_capa = pmsr_capa;
+	}
+
 	ret = mac80211_hwsim_new_radio(info, &param);
+
+out_free:
 	kfree(hwname);
+	kfree(param.pmsr_capa);
 	return ret;
 }
 
@@ -5353,6 +6153,11 @@ static const struct genl_small_ops hwsim_ops[] = {
 		.doit = hwsim_get_radio_nl,
 		.dumpit = hwsim_dump_radio_nl,
 	},
+	{
+		.cmd = HWSIM_CMD_REPORT_PMSR,
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+		.doit = hwsim_pmsr_report_nl,
+	},
 };
 
 static struct genl_family hwsim_genl_family __ro_after_init = {
@@ -5364,7 +6169,7 @@ static struct genl_family hwsim_genl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.small_ops = hwsim_ops,
 	.n_small_ops = ARRAY_SIZE(hwsim_ops),
-	.resv_start_op = HWSIM_CMD_DEL_MAC_ADDR + 1,
+	.resv_start_op = HWSIM_CMD_REPORT_PMSR + 1, // match with __HWSIM_CMD_MAX
 	.mcgrps = hwsim_mcgrps,
 	.n_mcgrps = ARRAY_SIZE(hwsim_mcgrps),
 };
@@ -5533,6 +6338,9 @@ static int hwsim_virtio_handle_cmd(struct sk_buff *skb)
 	case HWSIM_CMD_TX_INFO_FRAME:
 		hwsim_tx_info_frame_received_nl(skb, &info);
 		break;
+	case HWSIM_CMD_REPORT_PMSR:
+		hwsim_pmsr_report_nl(skb, &info);
+		break;
 	default:
 		pr_err_ratelimited("hwsim: invalid cmd: %d\n", gnlh->cmd);
 		return -EPROTO;
@@ -5748,7 +6556,7 @@ static int __init init_mac80211_hwsim(void)
 	if (err)
 		goto out_exit_netlink;
 
-	hwsim_class = class_create(THIS_MODULE, "mac80211_hwsim");
+	hwsim_class = class_create("mac80211_hwsim");
 	if (IS_ERR(hwsim_class)) {
 		err = PTR_ERR(hwsim_class);
 		goto out_exit_virtio;
diff --git a/drivers/net/wireless/mac80211_hwsim.h b/drivers/net/wireless/virtual/mac80211_hwsim.h
index 527799b2de0f..92126f02c58f 100644
--- a/drivers/net/wireless/mac80211_hwsim.h
+++ b/drivers/net/wireless/virtual/mac80211_hwsim.h
@@ -81,6 +81,9 @@ enum hwsim_tx_control_flags {
  *	to this receiver address for a given station.
  * @HWSIM_CMD_DEL_MAC_ADDR: remove the MAC address again, the attributes
  *	are the same as to @HWSIM_CMD_ADD_MAC_ADDR.
+ * @HWSIM_CMD_START_PMSR: request to start peer measurement with the
+ *	%HWSIM_ATTR_PMSR_REQUEST. Result will be sent back asynchronously
+ *	with %HWSIM_CMD_REPORT_PMSR.
  * @__HWSIM_CMD_MAX: enum limit
  */
 enum {
@@ -93,6 +96,9 @@ enum {
 	HWSIM_CMD_GET_RADIO,
 	HWSIM_CMD_ADD_MAC_ADDR,
 	HWSIM_CMD_DEL_MAC_ADDR,
+	HWSIM_CMD_START_PMSR,
+	HWSIM_CMD_ABORT_PMSR,
+	HWSIM_CMD_REPORT_PMSR,
 	__HWSIM_CMD_MAX,
 };
 #define HWSIM_CMD_MAX (_HWSIM_CMD_MAX - 1)
@@ -142,6 +148,12 @@ enum {
  * @HWSIM_ATTR_CIPHER_SUPPORT: u32 array of supported cipher types
  * @HWSIM_ATTR_MLO_SUPPORT: claim MLO support (exact parameters TBD) for
  *	the new radio
+ * @HWSIM_ATTR_PMSR_SUPPORT: nested attribute used with %HWSIM_CMD_CREATE_RADIO
+ *	to provide peer measurement capabilities. (nl80211_peer_measurement_attrs)
+ * @HWSIM_ATTR_PMSR_REQUEST: nested attribute used with %HWSIM_CMD_START_PMSR
+ *	to provide details about peer measurement request (nl80211_peer_measurement_attrs)
+ * @HWSIM_ATTR_PMSR_RESULT: nested attributed used with %HWSIM_CMD_REPORT_PMSR
+ *	to provide peer measurement result (nl80211_peer_measurement_attrs)
  * @__HWSIM_ATTR_MAX: enum limit
  */
 
@@ -173,6 +185,9 @@ enum {
 	HWSIM_ATTR_IFTYPE_SUPPORT,
 	HWSIM_ATTR_CIPHER_SUPPORT,
 	HWSIM_ATTR_MLO_SUPPORT,
+	HWSIM_ATTR_PMSR_SUPPORT,
+	HWSIM_ATTR_PMSR_REQUEST,
+	HWSIM_ATTR_PMSR_RESULT,
 	__HWSIM_ATTR_MAX,
 };
 #define HWSIM_ATTR_MAX (__HWSIM_ATTR_MAX - 1)
@@ -277,4 +292,47 @@ enum {
 	HWSIM_VQ_RX,
 	HWSIM_NUM_VQS,
 };
+
+/**
+ * enum hwsim_rate_info -- bitrate information.
+ *
+ * Information about a receiving or transmitting bitrate
+ * that can be mapped to struct rate_info
+ *
+ * @HWSIM_RATE_INFO_ATTR_FLAGS: bitflag of flags from &enum rate_info_flags
+ * @HWSIM_RATE_INFO_ATTR_MCS: mcs index if struct describes an HT/VHT/HE rate
+ * @HWSIM_RATE_INFO_ATTR_LEGACY: bitrate in 100kbit/s for 802.11abg
+ * @HWSIM_RATE_INFO_ATTR_NSS: number of streams (VHT & HE only)
+ * @HWSIM_RATE_INFO_ATTR_BW: bandwidth (from &enum rate_info_bw)
+ * @HWSIM_RATE_INFO_ATTR_HE_GI: HE guard interval (from &enum nl80211_he_gi)
+ * @HWSIM_RATE_INFO_ATTR_HE_DCM: HE DCM value
+ * @HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC:  HE RU allocation (from &enum nl80211_he_ru_alloc,
+ *	only valid if bw is %RATE_INFO_BW_HE_RU)
+ * @HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH: In case of EDMG the number of bonded channels (1-4)
+ * @HWSIM_RATE_INFO_ATTR_EHT_GI: EHT guard interval (from &enum nl80211_eht_gi)
+ * @HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC: EHT RU allocation (from &enum nl80211_eht_ru_alloc,
+ *	only valid if bw is %RATE_INFO_BW_EHT_RU)
+ * @NUM_HWSIM_RATE_INFO_ATTRS: internal
+ * @HWSIM_RATE_INFO_ATTR_MAX: highest attribute number
+ */
+enum hwsim_rate_info_attributes {
+	__HWSIM_RATE_INFO_ATTR_INVALID,
+
+	HWSIM_RATE_INFO_ATTR_FLAGS,
+	HWSIM_RATE_INFO_ATTR_MCS,
+	HWSIM_RATE_INFO_ATTR_LEGACY,
+	HWSIM_RATE_INFO_ATTR_NSS,
+	HWSIM_RATE_INFO_ATTR_BW,
+	HWSIM_RATE_INFO_ATTR_HE_GI,
+	HWSIM_RATE_INFO_ATTR_HE_DCM,
+	HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC,
+	HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH,
+	HWSIM_RATE_INFO_ATTR_EHT_GI,
+	HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC,
+
+	/* keep last */
+	NUM_HWSIM_RATE_INFO_ATTRS,
+	HWSIM_RATE_INFO_ATTR_MAX = NUM_HWSIM_RATE_INFO_ATTRS - 1
+};
+
 #endif /* __MAC80211_HWSIM_H */
diff --git a/drivers/net/wireless/virt_wifi.c b/drivers/net/wireless/virtual/virt_wifi.c
index ba14d83353a4..ba14d83353a4 100644
--- a/drivers/net/wireless/virt_wifi.c
+++ b/drivers/net/wireless/virtual/virt_wifi.c
diff --git a/drivers/net/wwan/iosm/iosm_ipc_port.c b/drivers/net/wwan/iosm/iosm_ipc_port.c
index b6d81c627277..5d5b4183e14a 100644
--- a/drivers/net/wwan/iosm/iosm_ipc_port.c
+++ b/drivers/net/wwan/iosm/iosm_ipc_port.c
@@ -63,7 +63,8 @@ struct iosm_cdev *ipc_port_init(struct iosm_imem *ipc_imem,
 	ipc_port->ipc_imem = ipc_imem;
 
 	ipc_port->iosm_port = wwan_create_port(ipc_port->dev, port_type,
-					       &ipc_wwan_ctrl_ops, ipc_port);
+					       &ipc_wwan_ctrl_ops, NULL,
+					       ipc_port);
 
 	return ipc_port;
 }
diff --git a/drivers/net/wwan/mhi_wwan_ctrl.c b/drivers/net/wwan/mhi_wwan_ctrl.c
index f7ca52353f40..e9f979d2d851 100644
--- a/drivers/net/wwan/mhi_wwan_ctrl.c
+++ b/drivers/net/wwan/mhi_wwan_ctrl.c
@@ -237,7 +237,7 @@ static int mhi_wwan_ctrl_probe(struct mhi_device *mhi_dev,
 
 	/* Register as a wwan port, id->driver_data contains wwan port type */
 	port = wwan_create_port(&cntrl->mhi_dev->dev, id->driver_data,
-				&wwan_pops, mhiwwan);
+				&wwan_pops, NULL, mhiwwan);
 	if (IS_ERR(port)) {
 		kfree(mhiwwan);
 		return PTR_ERR(port);
diff --git a/drivers/net/wwan/rpmsg_wwan_ctrl.c b/drivers/net/wwan/rpmsg_wwan_ctrl.c
index 31c24420ab2e..86b60aadfa11 100644
--- a/drivers/net/wwan/rpmsg_wwan_ctrl.c
+++ b/drivers/net/wwan/rpmsg_wwan_ctrl.c
@@ -129,7 +129,7 @@ static int rpmsg_wwan_ctrl_probe(struct rpmsg_device *rpdev)
 
 	/* Register as a wwan port, id.driver_data contains wwan port type */
 	port = wwan_create_port(parent, rpdev->id.driver_data,
-				&rpmsg_wwan_pops, rpwwan);
+				&rpmsg_wwan_pops, NULL, rpwwan);
 	if (IS_ERR(port))
 		return PTR_ERR(port);
 
@@ -149,6 +149,7 @@ static const struct rpmsg_device_id rpmsg_wwan_ctrl_id_table[] = {
 	/* RPMSG channels for Qualcomm SoCs with integrated modem */
 	{ .name = "DATA5_CNTL", .driver_data = WWAN_PORT_QMI },
 	{ .name = "DATA4", .driver_data = WWAN_PORT_AT },
+	{ .name = "DATA1", .driver_data = WWAN_PORT_AT },
 	{},
 };
 MODULE_DEVICE_TABLE(rpmsg, rpmsg_wwan_ctrl_id_table);
diff --git a/drivers/net/wwan/t7xx/t7xx_port_wwan.c b/drivers/net/wwan/t7xx/t7xx_port_wwan.c
index 24bd21942403..17389c8f6600 100644
--- a/drivers/net/wwan/t7xx/t7xx_port_wwan.c
+++ b/drivers/net/wwan/t7xx/t7xx_port_wwan.c
@@ -54,13 +54,13 @@ static void t7xx_port_ctrl_stop(struct wwan_port *port)
 static int t7xx_port_ctrl_tx(struct wwan_port *port, struct sk_buff *skb)
 {
 	struct t7xx_port *port_private = wwan_port_get_drvdata(port);
-	size_t len, offset, chunk_len = 0, txq_mtu = CLDMA_MTU;
 	const struct t7xx_port_conf *port_conf;
+	struct sk_buff *cur = skb, *cloned;
 	struct t7xx_fsm_ctl *ctl;
 	enum md_state md_state;
+	int cnt = 0, ret;
 
-	len = skb->len;
-	if (!len || !port_private->chan_enable)
+	if (!port_private->chan_enable)
 		return -EINVAL;
 
 	port_conf = port_private->port_conf;
@@ -72,23 +72,21 @@ static int t7xx_port_ctrl_tx(struct wwan_port *port, struct sk_buff *skb)
 		return -ENODEV;
 	}
 
-	for (offset = 0; offset < len; offset += chunk_len) {
-		struct sk_buff *skb_ccci;
-		int ret;
-
-		chunk_len = min(len - offset, txq_mtu - sizeof(struct ccci_header));
-		skb_ccci = t7xx_port_alloc_skb(chunk_len);
-		if (!skb_ccci)
-			return -ENOMEM;
-
-		skb_put_data(skb_ccci, skb->data + offset, chunk_len);
-		ret = t7xx_port_send_skb(port_private, skb_ccci, 0, 0);
+	while (cur) {
+		cloned = skb_clone(cur, GFP_KERNEL);
+		cloned->len = skb_headlen(cur);
+		ret = t7xx_port_send_skb(port_private, cloned, 0, 0);
 		if (ret) {
-			dev_kfree_skb_any(skb_ccci);
+			dev_kfree_skb(cloned);
 			dev_err(port_private->dev, "Write error on %s port, %d\n",
 				port_conf->name, ret);
-			return ret;
+			return cnt ? cnt + ret : ret;
 		}
+		cnt += cur->len;
+		if (cur == skb)
+			cur = skb_shinfo(skb)->frag_list;
+		else
+			cur = cur->next;
 	}
 
 	dev_kfree_skb(skb);
@@ -154,13 +152,17 @@ static int t7xx_port_wwan_disable_chl(struct t7xx_port *port)
 static void t7xx_port_wwan_md_state_notify(struct t7xx_port *port, unsigned int state)
 {
 	const struct t7xx_port_conf *port_conf = port->port_conf;
+	unsigned int header_len = sizeof(struct ccci_header);
+	struct wwan_port_caps caps;
 
 	if (state != MD_STATE_READY)
 		return;
 
 	if (!port->wwan.wwan_port) {
+		caps.frag_len = CLDMA_MTU - header_len;
+		caps.headroom_len = header_len;
 		port->wwan.wwan_port = wwan_create_port(port->dev, port_conf->port_type,
-							&wwan_ops, port);
+							&wwan_ops, &caps, port);
 		if (IS_ERR(port->wwan.wwan_port))
 			dev_err(port->dev, "Unable to create WWWAN port %s", port_conf->name);
 	}
diff --git a/drivers/net/wwan/wwan_core.c b/drivers/net/wwan/wwan_core.c
index 966d0ccd2276..284ab1f56391 100644
--- a/drivers/net/wwan/wwan_core.c
+++ b/drivers/net/wwan/wwan_core.c
@@ -67,6 +67,8 @@ struct wwan_device {
  * @rxq: Buffer inbound queue
  * @waitqueue: The waitqueue for port fops (read/write/poll)
  * @data_lock: Port specific data access serialization
+ * @headroom_len: SKB reserved headroom size
+ * @frag_len: Length to fragment packet
  * @at_data: AT port specific data
  */
 struct wwan_port {
@@ -79,6 +81,8 @@ struct wwan_port {
 	struct sk_buff_head rxq;
 	wait_queue_head_t waitqueue;
 	struct mutex data_lock;	/* Port specific data access serialization */
+	size_t headroom_len;
+	size_t frag_len;
 	union {
 		struct {
 			struct ktermios termios;
@@ -426,6 +430,7 @@ static int __wwan_port_dev_assign_name(struct wwan_port *port, const char *fmt)
 struct wwan_port *wwan_create_port(struct device *parent,
 				   enum wwan_port_type type,
 				   const struct wwan_port_ops *ops,
+				   struct wwan_port_caps *caps,
 				   void *drvdata)
 {
 	struct wwan_device *wwandev;
@@ -459,6 +464,8 @@ struct wwan_port *wwan_create_port(struct device *parent,
 
 	port->type = type;
 	port->ops = ops;
+	port->frag_len = caps ? caps->frag_len : SIZE_MAX;
+	port->headroom_len = caps ? caps->headroom_len : 0;
 	mutex_init(&port->ops_lock);
 	skb_queue_head_init(&port->rxq);
 	init_waitqueue_head(&port->waitqueue);
@@ -485,6 +492,7 @@ struct wwan_port *wwan_create_port(struct device *parent,
 	if (err)
 		goto error_put_device;
 
+	dev_info(&wwandev->dev, "port %s attached\n", dev_name(&port->dev));
 	return port;
 
 error_put_device:
@@ -510,6 +518,8 @@ void wwan_remove_port(struct wwan_port *port)
 
 	skb_queue_purge(&port->rxq);
 	dev_set_drvdata(&port->dev, NULL);
+
+	dev_info(&wwandev->dev, "port %s disconnected\n", dev_name(&port->dev));
 	device_unregister(&port->dev);
 
 	/* Release related wwan device */
@@ -702,30 +712,53 @@ static ssize_t wwan_port_fops_read(struct file *filp, char __user *buf,
 static ssize_t wwan_port_fops_write(struct file *filp, const char __user *buf,
 				    size_t count, loff_t *offp)
 {
+	struct sk_buff *skb, *head = NULL, *tail = NULL;
 	struct wwan_port *port = filp->private_data;
-	struct sk_buff *skb;
+	size_t frag_len, remain = count;
 	int ret;
 
 	ret = wwan_wait_tx(port, !!(filp->f_flags & O_NONBLOCK));
 	if (ret)
 		return ret;
 
-	skb = alloc_skb(count, GFP_KERNEL);
-	if (!skb)
-		return -ENOMEM;
+	do {
+		frag_len = min(remain, port->frag_len);
+		skb = alloc_skb(frag_len + port->headroom_len, GFP_KERNEL);
+		if (!skb) {
+			ret = -ENOMEM;
+			goto freeskb;
+		}
+		skb_reserve(skb, port->headroom_len);
+
+		if (!head) {
+			head = skb;
+		} else if (!tail) {
+			skb_shinfo(head)->frag_list = skb;
+			tail = skb;
+		} else {
+			tail->next = skb;
+			tail = skb;
+		}
 
-	if (copy_from_user(skb_put(skb, count), buf, count)) {
-		kfree_skb(skb);
-		return -EFAULT;
-	}
+		if (copy_from_user(skb_put(skb, frag_len), buf + count - remain, frag_len)) {
+			ret = -EFAULT;
+			goto freeskb;
+		}
 
-	ret = wwan_port_op_tx(port, skb, !!(filp->f_flags & O_NONBLOCK));
-	if (ret) {
-		kfree_skb(skb);
-		return ret;
-	}
+		if (skb != head) {
+			head->data_len += skb->len;
+			head->len += skb->len;
+			head->truesize += skb->truesize;
+		}
+	} while (remain -= frag_len);
+
+	ret = wwan_port_op_tx(port, head, !!(filp->f_flags & O_NONBLOCK));
+	if (!ret)
+		return count;
 
-	return count;
+freeskb:
+	kfree_skb(head);
+	return ret;
 }
 
 static __poll_t wwan_port_fops_poll(struct file *filp, poll_table *wait)
@@ -1174,7 +1207,7 @@ static int __init wwan_init(void)
 	if (err)
 		return err;
 
-	wwan_class = class_create(THIS_MODULE, "wwan");
+	wwan_class = class_create("wwan");
 	if (IS_ERR(wwan_class)) {
 		err = PTR_ERR(wwan_class);
 		goto unregister;
diff --git a/drivers/net/wwan/wwan_hwsim.c b/drivers/net/wwan/wwan_hwsim.c
index 2397a903d8f5..ff3dd24ddb33 100644
--- a/drivers/net/wwan/wwan_hwsim.c
+++ b/drivers/net/wwan/wwan_hwsim.c
@@ -205,7 +205,7 @@ static struct wwan_hwsim_port *wwan_hwsim_port_new(struct wwan_hwsim_dev *dev)
 
 	port->wwan = wwan_create_port(&dev->dev, WWAN_PORT_AT,
 				      &wwan_hwsim_port_ops,
-				      port);
+				      NULL, port);
 	if (IS_ERR(port->wwan)) {
 		err = PTR_ERR(port->wwan);
 		goto err_free_port;
@@ -511,7 +511,7 @@ static int __init wwan_hwsim_init(void)
 	if (!wwan_wq)
 		return -ENOMEM;
 
-	wwan_hwsim_class = class_create(THIS_MODULE, "wwan_hwsim");
+	wwan_hwsim_class = class_create("wwan_hwsim");
 	if (IS_ERR(wwan_hwsim_class)) {
 		err = PTR_ERR(wwan_hwsim_class);
 		goto err_wq_destroy;
diff --git a/drivers/nfc/nfcmrvl/i2c.c b/drivers/nfc/nfcmrvl/i2c.c
index e74342b0b728..164e2ab859fd 100644
--- a/drivers/nfc/nfcmrvl/i2c.c
+++ b/drivers/nfc/nfcmrvl/i2c.c
@@ -168,7 +168,7 @@ static int nfcmrvl_i2c_parse_dt(struct device_node *node,
 		return ret;
 	}
 
-	if (of_find_property(node, "i2c-int-falling", NULL))
+	if (of_property_read_bool(node, "i2c-int-falling"))
 		pdata->irq_polarity = IRQF_TRIGGER_FALLING;
 	else
 		pdata->irq_polarity = IRQF_TRIGGER_RISING;
diff --git a/drivers/nfc/nfcmrvl/main.c b/drivers/nfc/nfcmrvl/main.c
index 1a5284de4341..141bc4b66dcb 100644
--- a/drivers/nfc/nfcmrvl/main.c
+++ b/drivers/nfc/nfcmrvl/main.c
@@ -261,11 +261,7 @@ int nfcmrvl_parse_dt(struct device_node *node,
 		return reset_n_io;
 	}
 	pdata->reset_n_io = reset_n_io;
-
-	if (of_find_property(node, "hci-muxed", NULL))
-		pdata->hci_muxed = 1;
-	else
-		pdata->hci_muxed = 0;
+	pdata->hci_muxed = of_property_read_bool(node, "hci-muxed");
 
 	return 0;
 }
diff --git a/drivers/nfc/nfcmrvl/nfcmrvl.h b/drivers/nfc/nfcmrvl/nfcmrvl.h
index 165bd0a95190..f61a99e553db 100644
--- a/drivers/nfc/nfcmrvl/nfcmrvl.h
+++ b/drivers/nfc/nfcmrvl/nfcmrvl.h
@@ -8,8 +8,6 @@
 #ifndef _NFCMRVL_H_
 #define _NFCMRVL_H_
 
-#include <linux/platform_data/nfcmrvl.h>
-
 #include "fw_dnld.h"
 
 /* Define private flags: */
@@ -50,6 +48,34 @@ enum nfcmrvl_phy {
 	NFCMRVL_PHY_SPI		= 3,
 };
 
+struct nfcmrvl_platform_data {
+	/*
+	 * Generic
+	 */
+
+	/* GPIO that is wired to RESET_N signal */
+	int reset_n_io;
+	/* Tell if transport is muxed in HCI one */
+	bool hci_muxed;
+
+	/*
+	 * UART specific
+	 */
+
+	/* Tell if UART needs flow control at init */
+	bool flow_control;
+	/* Tell if firmware supports break control for power management */
+	bool break_control;
+
+
+	/*
+	 * I2C specific
+	 */
+
+	unsigned int irq;
+	unsigned int irq_polarity;
+};
+
 struct nfcmrvl_private {
 
 	unsigned long flags;
diff --git a/drivers/nfc/nfcmrvl/uart.c b/drivers/nfc/nfcmrvl/uart.c
index 9c92cbdc42f0..956ae92f7573 100644
--- a/drivers/nfc/nfcmrvl/uart.c
+++ b/drivers/nfc/nfcmrvl/uart.c
@@ -76,15 +76,8 @@ static int nfcmrvl_uart_parse_dt(struct device_node *node,
 		return ret;
 	}
 
-	if (of_find_property(matched_node, "flow-control", NULL))
-		pdata->flow_control = 1;
-	else
-		pdata->flow_control = 0;
-
-	if (of_find_property(matched_node, "break-control", NULL))
-		pdata->break_control = 1;
-	else
-		pdata->break_control = 0;
+	pdata->flow_control = of_property_read_bool(matched_node, "flow-control");
+	pdata->break_control = of_property_read_bool(matched_node, "break-control");
 
 	of_node_put(matched_node);
 
diff --git a/drivers/nfc/nfcsim.c b/drivers/nfc/nfcsim.c
index 85bf8d586c70..44eeb17ae48d 100644
--- a/drivers/nfc/nfcsim.c
+++ b/drivers/nfc/nfcsim.c
@@ -367,11 +367,6 @@ static void nfcsim_debugfs_init_dev(struct nfcsim *dev)
 	}
 
 	dev_dir = debugfs_create_dir(devname, nfcsim_debugfs_root);
-	if (!dev_dir) {
-		NFCSIM_ERR(dev, "Could not create debugfs entries for nfc%d\n",
-			   idx);
-		return;
-	}
 
 	debugfs_create_u8("dropframe", 0664, dev_dir, &dev->dropframe);
 }
diff --git a/drivers/nfc/trf7970a.c b/drivers/nfc/trf7970a.c
index 21d68664fe08..7eb17f46a815 100644
--- a/drivers/nfc/trf7970a.c
+++ b/drivers/nfc/trf7970a.c
@@ -2229,7 +2229,7 @@ static const struct dev_pm_ops trf7970a_pm_ops = {
 			   trf7970a_pm_runtime_resume, NULL)
 };
 
-static const struct of_device_id trf7970a_of_match[] = {
+static const struct of_device_id trf7970a_of_match[] __maybe_unused = {
 	{.compatible = "ti,trf7970a",},
 	{},
 };
diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
index 88ae18b0efa8..d6bbcc7b5b90 100644
--- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
+++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
@@ -1470,8 +1470,7 @@ static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
 	return rc;
 }
 
-static int switchtec_ntb_add(struct device *dev,
-			     struct class_interface *class_intf)
+static int switchtec_ntb_add(struct device *dev)
 {
 	struct switchtec_dev *stdev = to_stdev(dev);
 	struct switchtec_ntb *sndev;
@@ -1541,8 +1540,7 @@ free_and_exit:
 	return rc;
 }
 
-static void switchtec_ntb_remove(struct device *dev,
-				 struct class_interface *class_intf)
+static void switchtec_ntb_remove(struct device *dev)
 {
 	struct switchtec_dev *stdev = to_stdev(dev);
 	struct switchtec_ntb *sndev = stdev->sndev;
diff --git a/drivers/nubus/bus.c b/drivers/nubus/bus.c
index 17fad660032c..72921e4f35f6 100644
--- a/drivers/nubus/bus.c
+++ b/drivers/nubus/bus.c
@@ -14,11 +14,6 @@
 #define to_nubus_board(d)       container_of(d, struct nubus_board, dev)
 #define to_nubus_driver(d)      container_of(d, struct nubus_driver, driver)
 
-static int nubus_bus_match(struct device *dev, struct device_driver *driver)
-{
-	return 1;
-}
-
 static int nubus_device_probe(struct device *dev)
 {
 	struct nubus_driver *ndrv = to_nubus_driver(dev->driver);
@@ -39,7 +34,6 @@ static void nubus_device_remove(struct device *dev)
 
 struct bus_type nubus_bus_type = {
 	.name		= "nubus",
-	.match		= nubus_bus_match,
 	.probe		= nubus_device_probe,
 	.remove		= nubus_device_remove,
 };
diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c
index 4976a0069e9c..954dbc105fc8 100644
--- a/drivers/nvdimm/bus.c
+++ b/drivers/nvdimm/bus.c
@@ -1320,7 +1320,7 @@ int __init nvdimm_bus_init(void)
 		goto err_dimm_chrdev;
 	nvdimm_major = rc;
 
-	nd_class = class_create(THIS_MODULE, "nd");
+	nd_class = class_create("nd");
 	if (IS_ERR(nd_class)) {
 		rc = PTR_ERR(nd_class);
 		goto err_class;
diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c
index b317ce6c4ec3..596bb11eeba5 100644
--- a/drivers/nvme/host/apple.c
+++ b/drivers/nvme/host/apple.c
@@ -209,16 +209,16 @@ static inline struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q)
 {
 	if (q->is_adminq)
 		return container_of(q, struct apple_nvme, adminq);
-	else
-		return container_of(q, struct apple_nvme, ioq);
+
+	return container_of(q, struct apple_nvme, ioq);
 }
 
 static unsigned int apple_nvme_queue_depth(struct apple_nvme_queue *q)
 {
 	if (q->is_adminq)
 		return APPLE_NVME_AQ_DEPTH;
-	else
-		return APPLE_ANS_MAX_QUEUE_DEPTH;
+
+	return APPLE_ANS_MAX_QUEUE_DEPTH;
 }
 
 static void apple_nvme_rtkit_crashed(void *cookie)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index d6a9bac91a4c..ccb6eb1282f8 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -450,8 +450,8 @@ bool nvme_cancel_request(struct request *req, void *data)
 	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 				"Cancelling I/O %d", req->tag);
 
-	/* don't abort one completed request */
-	if (blk_mq_request_completed(req))
+	/* don't abort one completed or idle request */
+	if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT)
 		return true;
 
 	nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
@@ -4819,8 +4819,6 @@ static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
 	u32 aer_notice_type = nvme_aer_subtype(result);
 	bool requeue = true;
 
-	trace_nvme_async_event(ctrl, aer_notice_type);
-
 	switch (aer_notice_type) {
 	case NVME_AER_NOTICE_NS_CHANGED:
 		set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
@@ -4856,7 +4854,6 @@ static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
 
 static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl)
 {
-	trace_nvme_async_event(ctrl, NVME_AER_ERROR);
 	dev_warn(ctrl->device, "resetting controller due to AER\n");
 	nvme_reset_ctrl(ctrl);
 }
@@ -4872,6 +4869,7 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
 	if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
 		return;
 
+	trace_nvme_async_event(ctrl, result);
 	switch (aer_type) {
 	case NVME_AER_NOTICE:
 		requeue = nvme_handle_aen_notice(ctrl, result);
@@ -4889,7 +4887,6 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
 	case NVME_AER_SMART:
 	case NVME_AER_CSS:
 	case NVME_AER_VS:
-		trace_nvme_async_event(ctrl, aer_type);
 		ctrl->aen_result = result;
 		break;
 	default:
@@ -5392,14 +5389,14 @@ static int __init nvme_core_init(void)
 	if (result < 0)
 		goto destroy_delete_wq;
 
-	nvme_class = class_create(THIS_MODULE, "nvme");
+	nvme_class = class_create("nvme");
 	if (IS_ERR(nvme_class)) {
 		result = PTR_ERR(nvme_class);
 		goto unregister_chrdev;
 	}
 	nvme_class->dev_uevent = nvme_class_uevent;
 
-	nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
+	nvme_subsys_class = class_create("nvme-subsystem");
 	if (IS_ERR(nvme_subsys_class)) {
 		result = PTR_ERR(nvme_subsys_class);
 		goto destroy_class;
@@ -5410,7 +5407,7 @@ static int __init nvme_core_init(void)
 	if (result < 0)
 		goto destroy_subsys_class;
 
-	nvme_ns_chr_class = class_create(THIS_MODULE, "nvme-generic");
+	nvme_ns_chr_class = class_create("nvme-generic");
 	if (IS_ERR(nvme_ns_chr_class)) {
 		result = PTR_ERR(nvme_ns_chr_class);
 		goto unregister_generic_ns;
diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index bbaa04a0c502..0069ebff85df 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -1254,7 +1254,7 @@ static int __init nvmf_init(void)
 	if (!nvmf_default_host)
 		return -ENOMEM;
 
-	nvmf_class = class_create(THIS_MODULE, "nvme-fabrics");
+	nvmf_class = class_create("nvme-fabrics");
 	if (IS_ERR(nvmf_class)) {
 		pr_err("couldn't register class nvme-fabrics\n");
 		ret = PTR_ERR(nvmf_class);
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index 456ee42a6133..2ed75923507d 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -3875,7 +3875,6 @@ static const struct attribute_group *nvme_fc_attr_groups[] = {
 static struct class fc_class = {
 	.name = "fc",
 	.dev_groups = nvme_fc_attr_groups,
-	.owner = THIS_MODULE,
 };
 
 static int __init nvme_fc_init_module(void)
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index cd7873de3121..7f25c0fe3a0b 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -5,7 +5,6 @@
  */
 
 #include <linux/acpi.h>
-#include <linux/aer.h>
 #include <linux/async.h>
 #include <linux/blkdev.h>
 #include <linux/blk-mq.h>
@@ -2535,7 +2534,6 @@ static int nvme_pci_enable(struct nvme_dev *dev)
 
 	nvme_map_cmb(dev);
 
-	pci_enable_pcie_error_reporting(pdev);
 	pci_save_state(pdev);
 
 	result = nvme_pci_configure_admin_queue(dev);
@@ -2600,10 +2598,8 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
 	nvme_suspend_io_queues(dev);
 	nvme_suspend_queue(dev, 0);
 	pci_free_irq_vectors(pdev);
-	if (pci_is_enabled(pdev)) {
-		pci_disable_pcie_error_reporting(pdev);
+	if (pci_is_enabled(pdev))
 		pci_disable_device(pdev);
-	}
 	nvme_reap_pending_cqes(dev);
 
 	nvme_cancel_tagset(&dev->ctrl);
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index bbad26b82b56..0eb79696fb73 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -12,7 +12,6 @@
 #include <linux/string.h>
 #include <linux/atomic.h>
 #include <linux/blk-mq.h>
-#include <linux/blk-mq-rdma.h>
 #include <linux/blk-integrity.h>
 #include <linux/types.h>
 #include <linux/list.h>
@@ -464,7 +463,6 @@ static int nvme_rdma_create_cq(struct ib_device *ibdev,
 		struct nvme_rdma_queue *queue)
 {
 	int ret, comp_vector, idx = nvme_rdma_queue_idx(queue);
-	enum ib_poll_context poll_ctx;
 
 	/*
 	 * Spread I/O queues completion vectors according their queue index.
@@ -473,15 +471,12 @@ static int nvme_rdma_create_cq(struct ib_device *ibdev,
 	comp_vector = (idx == 0 ? idx : idx - 1) % ibdev->num_comp_vectors;
 
 	/* Polling queues need direct cq polling context */
-	if (nvme_rdma_poll_queue(queue)) {
-		poll_ctx = IB_POLL_DIRECT;
+	if (nvme_rdma_poll_queue(queue))
 		queue->ib_cq = ib_alloc_cq(ibdev, queue, queue->cq_size,
-					   comp_vector, poll_ctx);
-	} else {
-		poll_ctx = IB_POLL_SOFTIRQ;
+					   comp_vector, IB_POLL_DIRECT);
+	else
 		queue->ib_cq = ib_cq_pool_get(ibdev, queue->cq_size,
-					      comp_vector, poll_ctx);
-	}
+					      comp_vector, IB_POLL_SOFTIRQ);
 
 	if (IS_ERR(queue->ib_cq)) {
 		ret = PTR_ERR(queue->ib_cq);
@@ -2163,10 +2158,8 @@ static void nvme_rdma_map_queues(struct blk_mq_tag_set *set)
 			ctrl->io_queues[HCTX_TYPE_DEFAULT];
 		set->map[HCTX_TYPE_READ].queue_offset = 0;
 	}
-	blk_mq_rdma_map_queues(&set->map[HCTX_TYPE_DEFAULT],
-			ctrl->device->dev, 0);
-	blk_mq_rdma_map_queues(&set->map[HCTX_TYPE_READ],
-			ctrl->device->dev, 0);
+	blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
+	blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
 
 	if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
 		/* map dedicated poll queues only if we have queues left */
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 49c9e7bc9116..bf0230442d57 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -888,6 +888,9 @@ static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
 	size_t consumed = len;
 	int result;
 
+	if (unlikely(!queue->rd_enabled))
+		return -EFAULT;
+
 	while (len) {
 		switch (nvme_tcp_recv_state(queue)) {
 		case NVME_TCP_RECV_PDU:
diff --git a/drivers/nvme/host/trace.h b/drivers/nvme/host/trace.h
index 6f0eaf6a1528..4fb5922ffdac 100644
--- a/drivers/nvme/host/trace.h
+++ b/drivers/nvme/host/trace.h
@@ -127,15 +127,12 @@ TRACE_EVENT(nvme_async_event,
 	),
 	TP_printk("nvme%d: NVME_AEN=%#08x [%s]",
 		__entry->ctrl_id, __entry->result,
-		__print_symbolic(__entry->result,
-		aer_name(NVME_AER_NOTICE_NS_CHANGED),
-		aer_name(NVME_AER_NOTICE_ANA),
-		aer_name(NVME_AER_NOTICE_FW_ACT_STARTING),
-		aer_name(NVME_AER_NOTICE_DISC_CHANGED),
-		aer_name(NVME_AER_ERROR),
-		aer_name(NVME_AER_SMART),
-		aer_name(NVME_AER_CSS),
-		aer_name(NVME_AER_VS))
+		__print_symbolic(__entry->result & 0x7,
+			aer_name(NVME_AER_ERROR),
+			aer_name(NVME_AER_SMART),
+			aer_name(NVME_AER_NOTICE),
+			aer_name(NVME_AER_CSS),
+			aer_name(NVME_AER_VS))
 	)
 );
 
diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index 80099df37314..39cb570f833d 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -668,21 +668,11 @@ out:
 	nvmet_req_complete(req, status);
 }
 
-static bool nvmet_handle_identify_desclist(struct nvmet_req *req)
+static void nvmet_execute_identify_ctrl_nvm(struct nvmet_req *req)
 {
-	switch (req->cmd->identify.csi) {
-	case NVME_CSI_NVM:
-		nvmet_execute_identify_desclist(req);
-		return true;
-	case NVME_CSI_ZNS:
-		if (IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
-			nvmet_execute_identify_desclist(req);
-			return true;
-		}
-		return false;
-	default:
-		return false;
-	}
+	/* Not supported: return zeroes */
+	nvmet_req_complete(req,
+		   nvmet_zero_sgl(req, 0, sizeof(struct nvme_id_ctrl_nvm)));
 }
 
 static void nvmet_execute_identify(struct nvmet_req *req)
@@ -692,54 +682,49 @@ static void nvmet_execute_identify(struct nvmet_req *req)
 
 	switch (req->cmd->identify.cns) {
 	case NVME_ID_CNS_NS:
+		nvmet_execute_identify_ns(req);
+		return;
+	case NVME_ID_CNS_CTRL:
+		nvmet_execute_identify_ctrl(req);
+		return;
+	case NVME_ID_CNS_NS_ACTIVE_LIST:
+		nvmet_execute_identify_nslist(req);
+		return;
+	case NVME_ID_CNS_NS_DESC_LIST:
+		nvmet_execute_identify_desclist(req);
+		return;
+	case NVME_ID_CNS_CS_NS:
 		switch (req->cmd->identify.csi) {
 		case NVME_CSI_NVM:
-			return nvmet_execute_identify_ns(req);
-		default:
+			/* Not supported */
 			break;
-		}
-		break;
-	case NVME_ID_CNS_CS_NS:
-		if (IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
-			switch (req->cmd->identify.csi) {
-			case NVME_CSI_ZNS:
-				return nvmet_execute_identify_cns_cs_ns(req);
-			default:
-				break;
+		case NVME_CSI_ZNS:
+			if (IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
+				nvmet_execute_identify_ns_zns(req);
+				return;
 			}
-		}
-		break;
-	case NVME_ID_CNS_CTRL:
-		switch (req->cmd->identify.csi) {
-		case NVME_CSI_NVM:
-			return nvmet_execute_identify_ctrl(req);
+			break;
 		}
 		break;
 	case NVME_ID_CNS_CS_CTRL:
-		if (IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
-			switch (req->cmd->identify.csi) {
-			case NVME_CSI_ZNS:
-				return nvmet_execute_identify_cns_cs_ctrl(req);
-			default:
-				break;
-			}
-		}
-		break;
-	case NVME_ID_CNS_NS_ACTIVE_LIST:
 		switch (req->cmd->identify.csi) {
 		case NVME_CSI_NVM:
-			return nvmet_execute_identify_nslist(req);
-		default:
+			nvmet_execute_identify_ctrl_nvm(req);
+			return;
+		case NVME_CSI_ZNS:
+			if (IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
+				nvmet_execute_identify_ctrl_zns(req);
+				return;
+			}
 			break;
 		}
 		break;
-	case NVME_ID_CNS_NS_DESC_LIST:
-		if (nvmet_handle_identify_desclist(req) == true)
-			return;
-		break;
 	}
 
-	nvmet_req_cns_error_complete(req);
+	pr_debug("unhandled identify cns %d on qid %d\n",
+	       req->cmd->identify.cns, req->sq->qid);
+	req->error_loc = offsetof(struct nvme_identify, cns);
+	nvmet_req_complete(req, NVME_SC_INVALID_FIELD | NVME_SC_DNR);
 }
 
 /*
diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c
index 5c16372f3b53..e940a7d37a9d 100644
--- a/drivers/nvme/target/fcloop.c
+++ b/drivers/nvme/target/fcloop.c
@@ -614,10 +614,11 @@ fcloop_fcp_recv_work(struct work_struct *work)
 	struct fcloop_fcpreq *tfcp_req =
 		container_of(work, struct fcloop_fcpreq, fcp_rcv_work);
 	struct nvmefc_fcp_req *fcpreq = tfcp_req->fcpreq;
+	unsigned long flags;
 	int ret = 0;
 	bool aborted = false;
 
-	spin_lock_irq(&tfcp_req->reqlock);
+	spin_lock_irqsave(&tfcp_req->reqlock, flags);
 	switch (tfcp_req->inistate) {
 	case INI_IO_START:
 		tfcp_req->inistate = INI_IO_ACTIVE;
@@ -626,11 +627,11 @@ fcloop_fcp_recv_work(struct work_struct *work)
 		aborted = true;
 		break;
 	default:
-		spin_unlock_irq(&tfcp_req->reqlock);
+		spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 		WARN_ON(1);
 		return;
 	}
-	spin_unlock_irq(&tfcp_req->reqlock);
+	spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 
 	if (unlikely(aborted))
 		ret = -ECANCELED;
@@ -655,8 +656,9 @@ fcloop_fcp_abort_recv_work(struct work_struct *work)
 		container_of(work, struct fcloop_fcpreq, abort_rcv_work);
 	struct nvmefc_fcp_req *fcpreq;
 	bool completed = false;
+	unsigned long flags;
 
-	spin_lock_irq(&tfcp_req->reqlock);
+	spin_lock_irqsave(&tfcp_req->reqlock, flags);
 	fcpreq = tfcp_req->fcpreq;
 	switch (tfcp_req->inistate) {
 	case INI_IO_ABORTED:
@@ -665,11 +667,11 @@ fcloop_fcp_abort_recv_work(struct work_struct *work)
 		completed = true;
 		break;
 	default:
-		spin_unlock_irq(&tfcp_req->reqlock);
+		spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 		WARN_ON(1);
 		return;
 	}
-	spin_unlock_irq(&tfcp_req->reqlock);
+	spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 
 	if (unlikely(completed)) {
 		/* remove reference taken in original abort downcall */
@@ -681,9 +683,9 @@ fcloop_fcp_abort_recv_work(struct work_struct *work)
 		nvmet_fc_rcv_fcp_abort(tfcp_req->tport->targetport,
 					&tfcp_req->tgt_fcp_req);
 
-	spin_lock_irq(&tfcp_req->reqlock);
+	spin_lock_irqsave(&tfcp_req->reqlock, flags);
 	tfcp_req->fcpreq = NULL;
-	spin_unlock_irq(&tfcp_req->reqlock);
+	spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 
 	fcloop_call_host_done(fcpreq, tfcp_req, -ECANCELED);
 	/* call_host_done releases reference for abort downcall */
@@ -699,11 +701,12 @@ fcloop_tgt_fcprqst_done_work(struct work_struct *work)
 	struct fcloop_fcpreq *tfcp_req =
 		container_of(work, struct fcloop_fcpreq, tio_done_work);
 	struct nvmefc_fcp_req *fcpreq;
+	unsigned long flags;
 
-	spin_lock_irq(&tfcp_req->reqlock);
+	spin_lock_irqsave(&tfcp_req->reqlock, flags);
 	fcpreq = tfcp_req->fcpreq;
 	tfcp_req->inistate = INI_IO_COMPLETED;
-	spin_unlock_irq(&tfcp_req->reqlock);
+	spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 
 	fcloop_call_host_done(fcpreq, tfcp_req, tfcp_req->status);
 }
@@ -807,13 +810,14 @@ fcloop_fcp_op(struct nvmet_fc_target_port *tgtport,
 	u32 rsplen = 0, xfrlen = 0;
 	int fcp_err = 0, active, aborted;
 	u8 op = tgt_fcpreq->op;
+	unsigned long flags;
 
-	spin_lock_irq(&tfcp_req->reqlock);
+	spin_lock_irqsave(&tfcp_req->reqlock, flags);
 	fcpreq = tfcp_req->fcpreq;
 	active = tfcp_req->active;
 	aborted = tfcp_req->aborted;
 	tfcp_req->active = true;
-	spin_unlock_irq(&tfcp_req->reqlock);
+	spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 
 	if (unlikely(active))
 		/* illegal - call while i/o active */
@@ -821,9 +825,9 @@ fcloop_fcp_op(struct nvmet_fc_target_port *tgtport,
 
 	if (unlikely(aborted)) {
 		/* target transport has aborted i/o prior */
-		spin_lock_irq(&tfcp_req->reqlock);
+		spin_lock_irqsave(&tfcp_req->reqlock, flags);
 		tfcp_req->active = false;
-		spin_unlock_irq(&tfcp_req->reqlock);
+		spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 		tgt_fcpreq->transferred_length = 0;
 		tgt_fcpreq->fcp_error = -ECANCELED;
 		tgt_fcpreq->done(tgt_fcpreq);
@@ -880,9 +884,9 @@ fcloop_fcp_op(struct nvmet_fc_target_port *tgtport,
 		break;
 	}
 
-	spin_lock_irq(&tfcp_req->reqlock);
+	spin_lock_irqsave(&tfcp_req->reqlock, flags);
 	tfcp_req->active = false;
-	spin_unlock_irq(&tfcp_req->reqlock);
+	spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 
 	tgt_fcpreq->transferred_length = xfrlen;
 	tgt_fcpreq->fcp_error = fcp_err;
@@ -896,15 +900,16 @@ fcloop_tgt_fcp_abort(struct nvmet_fc_target_port *tgtport,
 			struct nvmefc_tgt_fcp_req *tgt_fcpreq)
 {
 	struct fcloop_fcpreq *tfcp_req = tgt_fcp_req_to_fcpreq(tgt_fcpreq);
+	unsigned long flags;
 
 	/*
 	 * mark aborted only in case there were 2 threads in transport
 	 * (one doing io, other doing abort) and only kills ops posted
 	 * after the abort request
 	 */
-	spin_lock_irq(&tfcp_req->reqlock);
+	spin_lock_irqsave(&tfcp_req->reqlock, flags);
 	tfcp_req->aborted = true;
-	spin_unlock_irq(&tfcp_req->reqlock);
+	spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 
 	tfcp_req->status = NVME_SC_INTERNAL;
 
@@ -946,6 +951,7 @@ fcloop_fcp_abort(struct nvme_fc_local_port *localport,
 	struct fcloop_ini_fcpreq *inireq = fcpreq->private;
 	struct fcloop_fcpreq *tfcp_req;
 	bool abortio = true;
+	unsigned long flags;
 
 	spin_lock(&inireq->inilock);
 	tfcp_req = inireq->tfcp_req;
@@ -958,7 +964,7 @@ fcloop_fcp_abort(struct nvme_fc_local_port *localport,
 		return;
 
 	/* break initiator/target relationship for io */
-	spin_lock_irq(&tfcp_req->reqlock);
+	spin_lock_irqsave(&tfcp_req->reqlock, flags);
 	switch (tfcp_req->inistate) {
 	case INI_IO_START:
 	case INI_IO_ACTIVE:
@@ -968,11 +974,11 @@ fcloop_fcp_abort(struct nvme_fc_local_port *localport,
 		abortio = false;
 		break;
 	default:
-		spin_unlock_irq(&tfcp_req->reqlock);
+		spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 		WARN_ON(1);
 		return;
 	}
-	spin_unlock_irq(&tfcp_req->reqlock);
+	spin_unlock_irqrestore(&tfcp_req->reqlock, flags);
 
 	if (abortio)
 		/* leave the reference while the work item is scheduled */
@@ -1562,7 +1568,7 @@ static int __init fcloop_init(void)
 {
 	int ret;
 
-	fcloop_class = class_create(THIS_MODULE, "fcloop");
+	fcloop_class = class_create("fcloop");
 	if (IS_ERR(fcloop_class)) {
 		pr_err("couldn't register class fcloop\n");
 		ret = PTR_ERR(fcloop_class);
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 89bedfcd974c..dc60a22646f7 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -581,8 +581,8 @@ bool nvmet_ns_revalidate(struct nvmet_ns *ns);
 u16 blk_to_nvme_status(struct nvmet_req *req, blk_status_t blk_sts);
 
 bool nvmet_bdev_zns_enable(struct nvmet_ns *ns);
-void nvmet_execute_identify_cns_cs_ctrl(struct nvmet_req *req);
-void nvmet_execute_identify_cns_cs_ns(struct nvmet_req *req);
+void nvmet_execute_identify_ctrl_zns(struct nvmet_req *req);
+void nvmet_execute_identify_ns_zns(struct nvmet_req *req);
 void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req);
 void nvmet_bdev_execute_zone_mgmt_send(struct nvmet_req *req);
 void nvmet_bdev_execute_zone_append(struct nvmet_req *req);
@@ -687,14 +687,6 @@ static inline bool nvmet_use_inline_bvec(struct nvmet_req *req)
 	       req->sg_cnt <= NVMET_MAX_INLINE_BIOVEC;
 }
 
-static inline void nvmet_req_cns_error_complete(struct nvmet_req *req)
-{
-	pr_debug("unhandled identify cns %d on qid %d\n",
-	       req->cmd->identify.cns, req->sq->qid);
-	req->error_loc = offsetof(struct nvme_identify, cns);
-	nvmet_req_complete(req, NVME_SC_INVALID_FIELD | NVME_SC_DNR);
-}
-
 static inline void nvmet_req_bio_put(struct nvmet_req *req, struct bio *bio)
 {
 	if (bio != &req->b.inline_bio)
diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
index 66e8f9fd0ca7..ed98df72c76b 100644
--- a/drivers/nvme/target/tcp.c
+++ b/drivers/nvme/target/tcp.c
@@ -20,6 +20,31 @@
 
 #define NVMET_TCP_DEF_INLINE_DATA_SIZE	(4 * PAGE_SIZE)
 
+static int param_store_val(const char *str, int *val, int min, int max)
+{
+	int ret, new_val;
+
+	ret = kstrtoint(str, 10, &new_val);
+	if (ret)
+		return -EINVAL;
+
+	if (new_val < min || new_val > max)
+		return -EINVAL;
+
+	*val = new_val;
+	return 0;
+}
+
+static int set_params(const char *str, const struct kernel_param *kp)
+{
+	return param_store_val(str, kp->arg, 0, INT_MAX);
+}
+
+static const struct kernel_param_ops set_param_ops = {
+	.set	= set_params,
+	.get	= param_get_int,
+};
+
 /* Define the socket priority to use for connections were it is desirable
  * that the NIC consider performing optimized packet processing or filtering.
  * A non-zero value being sufficient to indicate general consideration of any
@@ -27,8 +52,8 @@
  * values that may be unique for some NIC implementations.
  */
 static int so_priority;
-module_param(so_priority, int, 0644);
-MODULE_PARM_DESC(so_priority, "nvmet tcp socket optimize priority");
+device_param_cb(so_priority, &set_param_ops, &so_priority, 0644);
+MODULE_PARM_DESC(so_priority, "nvmet tcp socket optimize priority: Default 0");
 
 /* Define a time period (in usecs) that io_work() shall sample an activated
  * queue before determining it to be idle.  This optional module behavior
@@ -36,9 +61,10 @@ MODULE_PARM_DESC(so_priority, "nvmet tcp socket optimize priority");
  * using advanced interrupt moderation techniques.
  */
 static int idle_poll_period_usecs;
-module_param(idle_poll_period_usecs, int, 0644);
+device_param_cb(idle_poll_period_usecs, &set_param_ops,
+		&idle_poll_period_usecs, 0644);
 MODULE_PARM_DESC(idle_poll_period_usecs,
-		"nvmet tcp io_work poll till idle time period in usecs");
+		"nvmet tcp io_work poll till idle time period in usecs: Default 0");
 
 #define NVMET_TCP_RECV_BUDGET		8
 #define NVMET_TCP_SEND_BUDGET		8
diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c
index 7e4292d88016..5b5c1e481722 100644
--- a/drivers/nvme/target/zns.c
+++ b/drivers/nvme/target/zns.c
@@ -70,7 +70,7 @@ bool nvmet_bdev_zns_enable(struct nvmet_ns *ns)
 	return true;
 }
 
-void nvmet_execute_identify_cns_cs_ctrl(struct nvmet_req *req)
+void nvmet_execute_identify_ctrl_zns(struct nvmet_req *req)
 {
 	u8 zasl = req->sq->ctrl->subsys->zasl;
 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
@@ -95,9 +95,9 @@ out:
 	nvmet_req_complete(req, status);
 }
 
-void nvmet_execute_identify_cns_cs_ns(struct nvmet_req *req)
+void nvmet_execute_identify_ns_zns(struct nvmet_req *req)
 {
-	struct nvme_id_ns_zns *id_zns;
+	struct nvme_id_ns_zns *id_zns = NULL;
 	u64 zsze;
 	u16 status;
 	u32 mar, mor;
@@ -118,16 +118,18 @@ void nvmet_execute_identify_cns_cs_ns(struct nvmet_req *req)
 	if (status)
 		goto done;
 
-	if (!bdev_is_zoned(req->ns->bdev)) {
-		req->error_loc = offsetof(struct nvme_identify, nsid);
-		goto done;
-	}
-
 	if (nvmet_ns_revalidate(req->ns)) {
 		mutex_lock(&req->ns->subsys->lock);
 		nvmet_ns_changed(req->ns->subsys, req->ns->nsid);
 		mutex_unlock(&req->ns->subsys->lock);
 	}
+
+	if (!bdev_is_zoned(req->ns->bdev)) {
+		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
+		req->error_loc = offsetof(struct nvme_identify, nsid);
+		goto out;
+	}
+
 	zsze = (bdev_zone_sectors(req->ns->bdev) << 9) >>
 					req->ns->blksize_shift;
 	id_zns->lbafe[0].zsze = cpu_to_le64(zsze);
@@ -148,8 +150,8 @@ void nvmet_execute_identify_cns_cs_ns(struct nvmet_req *req)
 
 done:
 	status = nvmet_copy_to_sgl(req, 0, id_zns, sizeof(*id_zns));
-	kfree(id_zns);
 out:
+	kfree(id_zns);
 	nvmet_req_complete(req, status);
 }
 
diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig
index 6dec38805041..b291b27048c7 100644
--- a/drivers/nvmem/Kconfig
+++ b/drivers/nvmem/Kconfig
@@ -21,6 +21,10 @@ config NVMEM_SYSFS
 	 This interface is mostly used by userspace applications to
 	 read/write directly into nvmem.
 
+# Layouts
+
+source "drivers/nvmem/layouts/Kconfig"
+
 # Devices
 
 config NVMEM_APPLE_EFUSES
@@ -336,6 +340,7 @@ config NVMEM_U_BOOT_ENV
 	tristate "U-Boot environment variables support"
 	depends on OF && MTD
 	select CRC32
+	select GENERIC_NET_UTILS
 	help
 	  U-Boot stores its setup as environment variables. This driver adds
 	  support for verifying & exporting such data. It also exposes variables
@@ -368,7 +373,7 @@ config NVMEM_VF610_OCOTP
 	  be called nvmem-vf610-ocotp.
 
 config NVMEM_ZYNQMP
-	bool "Xilinx ZYNQMP SoC nvmem firmware support"
+	tristate "Xilinx ZYNQMP SoC nvmem firmware support"
 	depends on ARCH_ZYNQMP
 	help
 	  This is a driver to access hardware related data like
diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile
index 6a1efffa88f0..f82431ec8aef 100644
--- a/drivers/nvmem/Makefile
+++ b/drivers/nvmem/Makefile
@@ -5,6 +5,7 @@
 
 obj-$(CONFIG_NVMEM)		+= nvmem_core.o
 nvmem_core-y			:= core.o
+obj-y				+= layouts/
 
 # Devices
 obj-$(CONFIG_NVMEM_APPLE_EFUSES)	+= nvmem-apple-efuses.o
diff --git a/drivers/nvmem/bcm-ocotp.c b/drivers/nvmem/bcm-ocotp.c
index a128c7f5e351..0c1fa0c4feb2 100644
--- a/drivers/nvmem/bcm-ocotp.c
+++ b/drivers/nvmem/bcm-ocotp.c
@@ -244,7 +244,6 @@ MODULE_DEVICE_TABLE(acpi, bcm_otpc_acpi_ids);
 static int bcm_otpc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct resource *res;
 	struct otpc_priv *priv;
 	struct nvmem_device *nvmem;
 	int err;
@@ -259,8 +258,7 @@ static int bcm_otpc_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	/* Get OTP base address register. */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	priv->base = devm_ioremap_resource(dev, res);
+	priv->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(priv->base)) {
 		dev_err(dev, "unable to map I/O memory\n");
 		return PTR_ERR(priv->base);
diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c
index 22024b830788..342cd380b420 100644
--- a/drivers/nvmem/core.c
+++ b/drivers/nvmem/core.c
@@ -17,6 +17,7 @@
 #include <linux/nvmem-provider.h>
 #include <linux/gpio/consumer.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/slab.h>
 
 struct nvmem_device {
@@ -38,8 +39,8 @@ struct nvmem_device {
 	unsigned int		nkeepout;
 	nvmem_reg_read_t	reg_read;
 	nvmem_reg_write_t	reg_write;
-	nvmem_cell_post_process_t cell_post_process;
 	struct gpio_desc	*wp_gpio;
+	struct nvmem_layout	*layout;
 	void *priv;
 };
 
@@ -49,9 +50,12 @@ struct nvmem_device {
 struct nvmem_cell_entry {
 	const char		*name;
 	int			offset;
+	size_t			raw_len;
 	int			bytes;
 	int			bit_offset;
 	int			nbits;
+	nvmem_cell_post_process_t read_post_process;
+	void			*priv;
 	struct device_node	*np;
 	struct nvmem_device	*nvmem;
 	struct list_head	node;
@@ -74,6 +78,9 @@ static LIST_HEAD(nvmem_lookup_list);
 
 static BLOCKING_NOTIFIER_HEAD(nvmem_notifier);
 
+static DEFINE_SPINLOCK(nvmem_layout_lock);
+static LIST_HEAD(nvmem_layouts);
+
 static int __nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset,
 			    void *val, size_t bytes)
 {
@@ -463,8 +470,11 @@ static int nvmem_cell_info_to_nvmem_cell_entry_nodup(struct nvmem_device *nvmem,
 {
 	cell->nvmem = nvmem;
 	cell->offset = info->offset;
+	cell->raw_len = info->raw_len ?: info->bytes;
 	cell->bytes = info->bytes;
 	cell->name = info->name;
+	cell->read_post_process = info->read_post_process;
+	cell->priv = info->priv;
 
 	cell->bit_offset = info->bit_offset;
 	cell->nbits = info->nbits;
@@ -688,6 +698,7 @@ static int nvmem_validate_keepouts(struct nvmem_device *nvmem)
 
 static int nvmem_add_cells_from_of(struct nvmem_device *nvmem)
 {
+	struct nvmem_layout *layout = nvmem->layout;
 	struct device *dev = &nvmem->dev;
 	struct device_node *child;
 	const __be32 *addr;
@@ -717,6 +728,9 @@ static int nvmem_add_cells_from_of(struct nvmem_device *nvmem)
 
 		info.np = of_node_get(child);
 
+		if (layout && layout->fixup_cell_info)
+			layout->fixup_cell_info(nvmem, layout, &info);
+
 		ret = nvmem_add_one_cell(nvmem, &info);
 		kfree(info.name);
 		if (ret) {
@@ -728,6 +742,108 @@ static int nvmem_add_cells_from_of(struct nvmem_device *nvmem)
 	return 0;
 }
 
+int __nvmem_layout_register(struct nvmem_layout *layout, struct module *owner)
+{
+	layout->owner = owner;
+
+	spin_lock(&nvmem_layout_lock);
+	list_add(&layout->node, &nvmem_layouts);
+	spin_unlock(&nvmem_layout_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(__nvmem_layout_register);
+
+void nvmem_layout_unregister(struct nvmem_layout *layout)
+{
+	spin_lock(&nvmem_layout_lock);
+	list_del(&layout->node);
+	spin_unlock(&nvmem_layout_lock);
+}
+EXPORT_SYMBOL_GPL(nvmem_layout_unregister);
+
+static struct nvmem_layout *nvmem_layout_get(struct nvmem_device *nvmem)
+{
+	struct device_node *layout_np, *np = nvmem->dev.of_node;
+	struct nvmem_layout *l, *layout = ERR_PTR(-EPROBE_DEFER);
+
+	layout_np = of_get_child_by_name(np, "nvmem-layout");
+	if (!layout_np)
+		return NULL;
+
+	/*
+	 * In case the nvmem device was built-in while the layout was built as a
+	 * module, we shall manually request the layout driver loading otherwise
+	 * we'll never have any match.
+	 */
+	of_request_module(layout_np);
+
+	spin_lock(&nvmem_layout_lock);
+
+	list_for_each_entry(l, &nvmem_layouts, node) {
+		if (of_match_node(l->of_match_table, layout_np)) {
+			if (try_module_get(l->owner))
+				layout = l;
+
+			break;
+		}
+	}
+
+	spin_unlock(&nvmem_layout_lock);
+	of_node_put(layout_np);
+
+	return layout;
+}
+
+static void nvmem_layout_put(struct nvmem_layout *layout)
+{
+	if (layout)
+		module_put(layout->owner);
+}
+
+static int nvmem_add_cells_from_layout(struct nvmem_device *nvmem)
+{
+	struct nvmem_layout *layout = nvmem->layout;
+	int ret;
+
+	if (layout && layout->add_cells) {
+		ret = layout->add_cells(&nvmem->dev, nvmem, layout);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_OF)
+/**
+ * of_nvmem_layout_get_container() - Get OF node to layout container.
+ *
+ * @nvmem: nvmem device.
+ *
+ * Return: a node pointer with refcount incremented or NULL if no
+ * container exists. Use of_node_put() on it when done.
+ */
+struct device_node *of_nvmem_layout_get_container(struct nvmem_device *nvmem)
+{
+	return of_get_child_by_name(nvmem->dev.of_node, "nvmem-layout");
+}
+EXPORT_SYMBOL_GPL(of_nvmem_layout_get_container);
+#endif
+
+const void *nvmem_layout_get_match_data(struct nvmem_device *nvmem,
+					struct nvmem_layout *layout)
+{
+	struct device_node __maybe_unused *layout_np;
+	const struct of_device_id *match;
+
+	layout_np = of_nvmem_layout_get_container(nvmem);
+	match = of_match_node(layout->of_match_table, layout_np);
+
+	return match ? match->data : NULL;
+}
+EXPORT_SYMBOL_GPL(nvmem_layout_get_match_data);
+
 /**
  * nvmem_register() - Register a nvmem device for given nvmem_config.
  * Also creates a binary entry in /sys/bus/nvmem/devices/dev-name/nvmem
@@ -790,7 +906,6 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 	nvmem->type = config->type;
 	nvmem->reg_read = config->reg_read;
 	nvmem->reg_write = config->reg_write;
-	nvmem->cell_post_process = config->cell_post_process;
 	nvmem->keepout = config->keepout;
 	nvmem->nkeepout = config->nkeepout;
 	if (config->of_node)
@@ -834,6 +949,19 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 			goto err_put_device;
 	}
 
+	/*
+	 * If the driver supplied a layout by config->layout, the module
+	 * pointer will be NULL and nvmem_layout_put() will be a noop.
+	 */
+	nvmem->layout = config->layout ?: nvmem_layout_get(nvmem);
+	if (IS_ERR(nvmem->layout)) {
+		rval = PTR_ERR(nvmem->layout);
+		nvmem->layout = NULL;
+
+		if (rval == -EPROBE_DEFER)
+			goto err_teardown_compat;
+	}
+
 	if (config->cells) {
 		rval = nvmem_add_cells(nvmem, config->cells, config->ncells);
 		if (rval)
@@ -854,12 +982,18 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 	if (rval)
 		goto err_remove_cells;
 
+	rval = nvmem_add_cells_from_layout(nvmem);
+	if (rval)
+		goto err_remove_cells;
+
 	blocking_notifier_call_chain(&nvmem_notifier, NVMEM_ADD, nvmem);
 
 	return nvmem;
 
 err_remove_cells:
 	nvmem_device_remove_all_cells(nvmem);
+	nvmem_layout_put(nvmem->layout);
+err_teardown_compat:
 	if (config->compat)
 		nvmem_sysfs_remove_compat(nvmem, config);
 err_put_device:
@@ -881,6 +1015,7 @@ static void nvmem_device_release(struct kref *kref)
 		device_remove_bin_file(nvmem->base_dev, &nvmem->eeprom);
 
 	nvmem_device_remove_all_cells(nvmem);
+	nvmem_layout_put(nvmem->layout);
 	device_unregister(&nvmem->dev);
 }
 
@@ -1246,6 +1381,15 @@ struct nvmem_cell *of_nvmem_cell_get(struct device_node *np, const char *id)
 		return ERR_PTR(-EINVAL);
 	}
 
+	/* nvmem layouts produce cells within the nvmem-layout container */
+	if (of_node_name_eq(nvmem_np, "nvmem-layout")) {
+		nvmem_np = of_get_next_parent(nvmem_np);
+		if (!nvmem_np) {
+			of_node_put(cell_np);
+			return ERR_PTR(-EINVAL);
+		}
+	}
+
 	nvmem = __nvmem_device_get(nvmem_np, device_match_of_node);
 	of_node_put(nvmem_np);
 	if (IS_ERR(nvmem)) {
@@ -1418,7 +1562,7 @@ static int __nvmem_cell_read(struct nvmem_device *nvmem,
 {
 	int rc;
 
-	rc = nvmem_reg_read(nvmem, cell->offset, buf, cell->bytes);
+	rc = nvmem_reg_read(nvmem, cell->offset, buf, cell->raw_len);
 
 	if (rc)
 		return rc;
@@ -1427,9 +1571,9 @@ static int __nvmem_cell_read(struct nvmem_device *nvmem,
 	if (cell->bit_offset || cell->nbits)
 		nvmem_shift_read_buffer_in_place(cell, buf);
 
-	if (nvmem->cell_post_process) {
-		rc = nvmem->cell_post_process(nvmem->priv, id, index,
-					      cell->offset, buf, cell->bytes);
+	if (cell->read_post_process) {
+		rc = cell->read_post_process(cell->priv, id, index,
+					     cell->offset, buf, cell->raw_len);
 		if (rc)
 			return rc;
 	}
@@ -1452,14 +1596,15 @@ static int __nvmem_cell_read(struct nvmem_device *nvmem,
  */
 void *nvmem_cell_read(struct nvmem_cell *cell, size_t *len)
 {
-	struct nvmem_device *nvmem = cell->entry->nvmem;
+	struct nvmem_cell_entry *entry = cell->entry;
+	struct nvmem_device *nvmem = entry->nvmem;
 	u8 *buf;
 	int rc;
 
 	if (!nvmem)
 		return ERR_PTR(-EINVAL);
 
-	buf = kzalloc(cell->entry->bytes, GFP_KERNEL);
+	buf = kzalloc(max_t(size_t, entry->raw_len, entry->bytes), GFP_KERNEL);
 	if (!buf)
 		return ERR_PTR(-ENOMEM);
 
@@ -1535,6 +1680,14 @@ static int __nvmem_cell_entry_write(struct nvmem_cell_entry *cell, void *buf, si
 	    (cell->bit_offset == 0 && len != cell->bytes))
 		return -EINVAL;
 
+	/*
+	 * Any cells which have a read_post_process hook are read-only because
+	 * we cannot reverse the operation and it might affect other cells,
+	 * too.
+	 */
+	if (cell->read_post_process)
+		return -EINVAL;
+
 	if (cell->bit_offset || cell->nbits) {
 		buf = nvmem_cell_prepare_write_buffer(cell, buf, len);
 		if (IS_ERR(buf))
@@ -1958,4 +2111,3 @@ module_exit(nvmem_exit);
 MODULE_AUTHOR("Srinivas Kandagatla <srinivas.kandagatla@linaro.org");
 MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com");
 MODULE_DESCRIPTION("nvmem Driver Core");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/nvmem/imx-ocotp.c b/drivers/nvmem/imx-ocotp.c
index e9b52ecb3f72..ac0edb6398f1 100644
--- a/drivers/nvmem/imx-ocotp.c
+++ b/drivers/nvmem/imx-ocotp.c
@@ -225,18 +225,13 @@ read_end:
 static int imx_ocotp_cell_pp(void *context, const char *id, int index,
 			     unsigned int offset, void *data, size_t bytes)
 {
-	struct ocotp_priv *priv = context;
+	u8 *buf = data;
+	int i;
 
 	/* Deal with some post processing of nvmem cell data */
-	if (id && !strcmp(id, "mac-address")) {
-		if (priv->params->reverse_mac_address) {
-			u8 *buf = data;
-			int i;
-
-			for (i = 0; i < bytes/2; i++)
-				swap(buf[i], buf[bytes - i - 1]);
-		}
-	}
+	if (id && !strcmp(id, "mac-address"))
+		for (i = 0; i < bytes / 2; i++)
+			swap(buf[i], buf[bytes - i - 1]);
 
 	return 0;
 }
@@ -488,7 +483,6 @@ static struct nvmem_config imx_ocotp_nvmem_config = {
 	.stride = 1,
 	.reg_read = imx_ocotp_read,
 	.reg_write = imx_ocotp_write,
-	.cell_post_process = imx_ocotp_cell_pp,
 };
 
 static const struct ocotp_params imx6q_params = {
@@ -595,6 +589,17 @@ static const struct of_device_id imx_ocotp_dt_ids[] = {
 };
 MODULE_DEVICE_TABLE(of, imx_ocotp_dt_ids);
 
+static void imx_ocotp_fixup_cell_info(struct nvmem_device *nvmem,
+				      struct nvmem_layout *layout,
+				      struct nvmem_cell_info *cell)
+{
+	cell->read_post_process = imx_ocotp_cell_pp;
+}
+
+struct nvmem_layout imx_ocotp_layout = {
+	.fixup_cell_info = imx_ocotp_fixup_cell_info,
+};
+
 static int imx_ocotp_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -619,6 +624,9 @@ static int imx_ocotp_probe(struct platform_device *pdev)
 	imx_ocotp_nvmem_config.size = 4 * priv->params->nregs;
 	imx_ocotp_nvmem_config.dev = dev;
 	imx_ocotp_nvmem_config.priv = priv;
+	if (priv->params->reverse_mac_address)
+		imx_ocotp_nvmem_config.layout = &imx_ocotp_layout;
+
 	priv->config = &imx_ocotp_nvmem_config;
 
 	clk_prepare_enable(priv->clk);
diff --git a/drivers/nvmem/layouts/Kconfig b/drivers/nvmem/layouts/Kconfig
new file mode 100644
index 000000000000..7ff1ee1c1f05
--- /dev/null
+++ b/drivers/nvmem/layouts/Kconfig
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: GPL-2.0
+
+menu "Layout Types"
+
+config NVMEM_LAYOUT_SL28_VPD
+	tristate "Kontron sl28 VPD layout support"
+	select CRC8
+	help
+	  Say Y here if you want to support the VPD layout of the Kontron
+	  SMARC-sAL28 boards.
+
+	  If unsure, say N.
+
+config NVMEM_LAYOUT_ONIE_TLV
+	tristate "ONIE tlv support"
+	select CRC32
+	help
+	  Say Y here if you want to support the Open Compute Project ONIE
+	  Type-Length-Value standard table.
+
+	  If unsure, say N.
+
+endmenu
diff --git a/drivers/nvmem/layouts/Makefile b/drivers/nvmem/layouts/Makefile
new file mode 100644
index 000000000000..2974bd7d33ed
--- /dev/null
+++ b/drivers/nvmem/layouts/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for nvmem layouts.
+#
+
+obj-$(CONFIG_NVMEM_LAYOUT_SL28_VPD) += sl28vpd.o
+obj-$(CONFIG_NVMEM_LAYOUT_ONIE_TLV) += onie-tlv.o
diff --git a/drivers/nvmem/layouts/onie-tlv.c b/drivers/nvmem/layouts/onie-tlv.c
new file mode 100644
index 000000000000..59fc87ccfcff
--- /dev/null
+++ b/drivers/nvmem/layouts/onie-tlv.c
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ONIE tlv NVMEM cells provider
+ *
+ * Copyright (C) 2022 Open Compute Group ONIE
+ * Author: Miquel Raynal <miquel.raynal@bootlin.com>
+ * Based on the nvmem driver written by: Vadym Kochan <vadym.kochan@plvision.eu>
+ * Inspired by the first layout written by: Rafał Miłecki <rafal@milecki.pl>
+ */
+
+#include <linux/crc32.h>
+#include <linux/etherdevice.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/nvmem-provider.h>
+#include <linux/of.h>
+
+#define ONIE_TLV_MAX_LEN 2048
+#define ONIE_TLV_CRC_FIELD_SZ 6
+#define ONIE_TLV_CRC_SZ 4
+#define ONIE_TLV_HDR_ID	"TlvInfo"
+
+struct onie_tlv_hdr {
+	u8 id[8];
+	u8 version;
+	__be16 data_len;
+} __packed;
+
+struct onie_tlv {
+	u8 type;
+	u8 len;
+} __packed;
+
+static const char *onie_tlv_cell_name(u8 type)
+{
+	switch (type) {
+	case 0x21:
+		return "product-name";
+	case 0x22:
+		return "part-number";
+	case 0x23:
+		return "serial-number";
+	case 0x24:
+		return "mac-address";
+	case 0x25:
+		return "manufacture-date";
+	case 0x26:
+		return "device-version";
+	case 0x27:
+		return "label-revision";
+	case 0x28:
+		return "platform-name";
+	case 0x29:
+		return "onie-version";
+	case 0x2A:
+		return "num-macs";
+	case 0x2B:
+		return "manufacturer";
+	case 0x2C:
+		return "country-code";
+	case 0x2D:
+		return "vendor";
+	case 0x2E:
+		return "diag-version";
+	case 0x2F:
+		return "service-tag";
+	case 0xFD:
+		return "vendor-extension";
+	case 0xFE:
+		return "crc32";
+	default:
+		break;
+	}
+
+	return NULL;
+}
+
+static int onie_tlv_mac_read_cb(void *priv, const char *id, int index,
+				unsigned int offset, void *buf,
+				size_t bytes)
+{
+	eth_addr_add(buf, index);
+
+	return 0;
+}
+
+static nvmem_cell_post_process_t onie_tlv_read_cb(u8 type, u8 *buf)
+{
+	switch (type) {
+	case 0x24:
+		return &onie_tlv_mac_read_cb;
+	default:
+		break;
+	}
+
+	return NULL;
+}
+
+static int onie_tlv_add_cells(struct device *dev, struct nvmem_device *nvmem,
+			      size_t data_len, u8 *data)
+{
+	struct nvmem_cell_info cell = {};
+	struct device_node *layout;
+	struct onie_tlv tlv;
+	unsigned int hdr_len = sizeof(struct onie_tlv_hdr);
+	unsigned int offset = 0;
+	int ret;
+
+	layout = of_nvmem_layout_get_container(nvmem);
+	if (!layout)
+		return -ENOENT;
+
+	while (offset < data_len) {
+		memcpy(&tlv, data + offset, sizeof(tlv));
+		if (offset + tlv.len >= data_len) {
+			dev_err(dev, "Out of bounds field (0x%x bytes at 0x%x)\n",
+				tlv.len, hdr_len + offset);
+			break;
+		}
+
+		cell.name = onie_tlv_cell_name(tlv.type);
+		if (!cell.name)
+			continue;
+
+		cell.offset = hdr_len + offset + sizeof(tlv.type) + sizeof(tlv.len);
+		cell.bytes = tlv.len;
+		cell.np = of_get_child_by_name(layout, cell.name);
+		cell.read_post_process = onie_tlv_read_cb(tlv.type, data + offset + sizeof(tlv));
+
+		ret = nvmem_add_one_cell(nvmem, &cell);
+		if (ret) {
+			of_node_put(layout);
+			return ret;
+		}
+
+		offset += sizeof(tlv) + tlv.len;
+	}
+
+	of_node_put(layout);
+
+	return 0;
+}
+
+static bool onie_tlv_hdr_is_valid(struct device *dev, struct onie_tlv_hdr *hdr)
+{
+	if (memcmp(hdr->id, ONIE_TLV_HDR_ID, sizeof(hdr->id))) {
+		dev_err(dev, "Invalid header\n");
+		return false;
+	}
+
+	if (hdr->version != 0x1) {
+		dev_err(dev, "Invalid version number\n");
+		return false;
+	}
+
+	return true;
+}
+
+static bool onie_tlv_crc_is_valid(struct device *dev, size_t table_len, u8 *table)
+{
+	struct onie_tlv crc_hdr;
+	u32 read_crc, calc_crc;
+	__be32 crc_be;
+
+	memcpy(&crc_hdr, table + table_len - ONIE_TLV_CRC_FIELD_SZ, sizeof(crc_hdr));
+	if (crc_hdr.type != 0xfe || crc_hdr.len != ONIE_TLV_CRC_SZ) {
+		dev_err(dev, "Invalid CRC field\n");
+		return false;
+	}
+
+	/* The table contains a JAMCRC, which is XOR'ed compared to the original
+	 * CRC32 implementation as known in the Ethernet world.
+	 */
+	memcpy(&crc_be, table + table_len - ONIE_TLV_CRC_SZ, ONIE_TLV_CRC_SZ);
+	read_crc = be32_to_cpu(crc_be);
+	calc_crc = crc32(~0, table, table_len - ONIE_TLV_CRC_SZ) ^ 0xFFFFFFFF;
+	if (read_crc != calc_crc) {
+		dev_err(dev, "Invalid CRC read: 0x%08x, expected: 0x%08x\n",
+			read_crc, calc_crc);
+		return false;
+	}
+
+	return true;
+}
+
+static int onie_tlv_parse_table(struct device *dev, struct nvmem_device *nvmem,
+				struct nvmem_layout *layout)
+{
+	struct onie_tlv_hdr hdr;
+	size_t table_len, data_len, hdr_len;
+	u8 *table, *data;
+	int ret;
+
+	ret = nvmem_device_read(nvmem, 0, sizeof(hdr), &hdr);
+	if (ret < 0)
+		return ret;
+
+	if (!onie_tlv_hdr_is_valid(dev, &hdr)) {
+		dev_err(dev, "Invalid ONIE TLV header\n");
+		return -EINVAL;
+	}
+
+	hdr_len = sizeof(hdr.id) + sizeof(hdr.version) + sizeof(hdr.data_len);
+	data_len = be16_to_cpu(hdr.data_len);
+	table_len = hdr_len + data_len;
+	if (table_len > ONIE_TLV_MAX_LEN) {
+		dev_err(dev, "Invalid ONIE TLV data length\n");
+		return -EINVAL;
+	}
+
+	table = devm_kmalloc(dev, table_len, GFP_KERNEL);
+	if (!table)
+		return -ENOMEM;
+
+	ret = nvmem_device_read(nvmem, 0, table_len, table);
+	if (ret != table_len)
+		return ret;
+
+	if (!onie_tlv_crc_is_valid(dev, table_len, table))
+		return -EINVAL;
+
+	data = table + hdr_len;
+	ret = onie_tlv_add_cells(dev, nvmem, data_len, data);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct of_device_id onie_tlv_of_match_table[] = {
+	{ .compatible = "onie,tlv-layout", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, onie_tlv_of_match_table);
+
+static struct nvmem_layout onie_tlv_layout = {
+	.name = "ONIE tlv layout",
+	.of_match_table = onie_tlv_of_match_table,
+	.add_cells = onie_tlv_parse_table,
+};
+module_nvmem_layout_driver(onie_tlv_layout);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
+MODULE_DESCRIPTION("NVMEM layout driver for Onie TLV table parsing");
diff --git a/drivers/nvmem/layouts/sl28vpd.c b/drivers/nvmem/layouts/sl28vpd.c
new file mode 100644
index 000000000000..05671371f631
--- /dev/null
+++ b/drivers/nvmem/layouts/sl28vpd.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/crc8.h>
+#include <linux/etherdevice.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/nvmem-provider.h>
+#include <linux/of.h>
+#include <uapi/linux/if_ether.h>
+
+#define SL28VPD_MAGIC 'V'
+
+struct sl28vpd_header {
+	u8 magic;
+	u8 version;
+} __packed;
+
+struct sl28vpd_v1 {
+	struct sl28vpd_header header;
+	char serial_number[15];
+	u8 base_mac_address[ETH_ALEN];
+	u8 crc8;
+} __packed;
+
+static int sl28vpd_mac_address_pp(void *priv, const char *id, int index,
+				  unsigned int offset, void *buf,
+				  size_t bytes)
+{
+	if (bytes != ETH_ALEN)
+		return -EINVAL;
+
+	if (index < 0)
+		return -EINVAL;
+
+	if (!is_valid_ether_addr(buf))
+		return -EINVAL;
+
+	eth_addr_add(buf, index);
+
+	return 0;
+}
+
+static const struct nvmem_cell_info sl28vpd_v1_entries[] = {
+	{
+		.name = "serial-number",
+		.offset = offsetof(struct sl28vpd_v1, serial_number),
+		.bytes = sizeof_field(struct sl28vpd_v1, serial_number),
+	},
+	{
+		.name = "base-mac-address",
+		.offset = offsetof(struct sl28vpd_v1, base_mac_address),
+		.bytes = sizeof_field(struct sl28vpd_v1, base_mac_address),
+		.read_post_process = sl28vpd_mac_address_pp,
+	},
+};
+
+static int sl28vpd_v1_check_crc(struct device *dev, struct nvmem_device *nvmem)
+{
+	struct sl28vpd_v1 data_v1;
+	u8 table[CRC8_TABLE_SIZE];
+	int ret;
+	u8 crc;
+
+	crc8_populate_msb(table, 0x07);
+
+	ret = nvmem_device_read(nvmem, 0, sizeof(data_v1), &data_v1);
+	if (ret < 0)
+		return ret;
+	else if (ret != sizeof(data_v1))
+		return -EIO;
+
+	crc = crc8(table, (void *)&data_v1, sizeof(data_v1) - 1, 0);
+
+	if (crc != data_v1.crc8) {
+		dev_err(dev,
+			"Checksum is invalid (got %02x, expected %02x).\n",
+			crc, data_v1.crc8);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int sl28vpd_add_cells(struct device *dev, struct nvmem_device *nvmem,
+			     struct nvmem_layout *layout)
+{
+	const struct nvmem_cell_info *pinfo;
+	struct nvmem_cell_info info = {0};
+	struct device_node *layout_np;
+	struct sl28vpd_header hdr;
+	int ret, i;
+
+	/* check header */
+	ret = nvmem_device_read(nvmem, 0, sizeof(hdr), &hdr);
+	if (ret < 0)
+		return ret;
+	else if (ret != sizeof(hdr))
+		return -EIO;
+
+	if (hdr.magic != SL28VPD_MAGIC) {
+		dev_err(dev, "Invalid magic value (%02x)\n", hdr.magic);
+		return -EINVAL;
+	}
+
+	if (hdr.version != 1) {
+		dev_err(dev, "Version %d is unsupported.\n", hdr.version);
+		return -EINVAL;
+	}
+
+	ret = sl28vpd_v1_check_crc(dev, nvmem);
+	if (ret)
+		return ret;
+
+	layout_np = of_nvmem_layout_get_container(nvmem);
+	if (!layout_np)
+		return -ENOENT;
+
+	for (i = 0; i < ARRAY_SIZE(sl28vpd_v1_entries); i++) {
+		pinfo = &sl28vpd_v1_entries[i];
+
+		info.name = pinfo->name;
+		info.offset = pinfo->offset;
+		info.bytes = pinfo->bytes;
+		info.read_post_process = pinfo->read_post_process;
+		info.np = of_get_child_by_name(layout_np, pinfo->name);
+
+		ret = nvmem_add_one_cell(nvmem, &info);
+		if (ret) {
+			of_node_put(layout_np);
+			return ret;
+		}
+	}
+
+	of_node_put(layout_np);
+
+	return 0;
+}
+
+static const struct of_device_id sl28vpd_of_match_table[] = {
+	{ .compatible = "kontron,sl28-vpd" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, sl28vpd_of_match_table);
+
+static struct nvmem_layout sl28vpd_layout = {
+	.name = "sl28-vpd",
+	.of_match_table = sl28vpd_of_match_table,
+	.add_cells = sl28vpd_add_cells,
+};
+module_nvmem_layout_driver(sl28vpd_layout);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Michael Walle <michael@walle.cc>");
+MODULE_DESCRIPTION("NVMEM layout driver for the VPD of Kontron sl28 boards");
diff --git a/drivers/nvmem/mtk-efuse.c b/drivers/nvmem/mtk-efuse.c
index a08e0aedd21c..b36cd0dcc8c7 100644
--- a/drivers/nvmem/mtk-efuse.c
+++ b/drivers/nvmem/mtk-efuse.c
@@ -10,6 +10,11 @@
 #include <linux/io.h>
 #include <linux/nvmem-provider.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
+
+struct mtk_efuse_pdata {
+	bool uses_post_processing;
+};
 
 struct mtk_efuse_priv {
 	void __iomem *base;
@@ -29,6 +34,37 @@ static int mtk_reg_read(void *context,
 	return 0;
 }
 
+static int mtk_efuse_gpu_speedbin_pp(void *context, const char *id, int index,
+				     unsigned int offset, void *data, size_t bytes)
+{
+	u8 *val = data;
+
+	if (val[0] < 8)
+		val[0] = BIT(val[0]);
+
+	return 0;
+}
+
+static void mtk_efuse_fixup_cell_info(struct nvmem_device *nvmem,
+				      struct nvmem_layout *layout,
+				      struct nvmem_cell_info *cell)
+{
+	size_t sz = strlen(cell->name);
+
+	/*
+	 * On some SoCs, the GPU speedbin is not read as bitmask but as
+	 * a number with range [0-7] (max 3 bits): post process to use
+	 * it in OPP tables to describe supported-hw.
+	 */
+	if (cell->nbits <= 3 &&
+	    strncmp(cell->name, "gpu-speedbin", min(sz, strlen("gpu-speedbin"))) == 0)
+		cell->read_post_process = mtk_efuse_gpu_speedbin_pp;
+}
+
+static struct nvmem_layout mtk_efuse_layout = {
+	.fixup_cell_info = mtk_efuse_fixup_cell_info,
+};
+
 static int mtk_efuse_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -36,6 +72,7 @@ static int mtk_efuse_probe(struct platform_device *pdev)
 	struct nvmem_device *nvmem;
 	struct nvmem_config econfig = {};
 	struct mtk_efuse_priv *priv;
+	const struct mtk_efuse_pdata *pdata;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -45,20 +82,32 @@ static int mtk_efuse_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->base))
 		return PTR_ERR(priv->base);
 
+	pdata = device_get_match_data(dev);
 	econfig.stride = 1;
 	econfig.word_size = 1;
 	econfig.reg_read = mtk_reg_read;
 	econfig.size = resource_size(res);
 	econfig.priv = priv;
 	econfig.dev = dev;
+	if (pdata->uses_post_processing)
+		econfig.layout = &mtk_efuse_layout;
 	nvmem = devm_nvmem_register(dev, &econfig);
 
 	return PTR_ERR_OR_ZERO(nvmem);
 }
 
+static const struct mtk_efuse_pdata mtk_mt8186_efuse_pdata = {
+	.uses_post_processing = true,
+};
+
+static const struct mtk_efuse_pdata mtk_efuse_pdata = {
+	.uses_post_processing = false,
+};
+
 static const struct of_device_id mtk_efuse_of_match[] = {
-	{ .compatible = "mediatek,mt8173-efuse",},
-	{ .compatible = "mediatek,efuse",},
+	{ .compatible = "mediatek,mt8173-efuse", .data = &mtk_efuse_pdata },
+	{ .compatible = "mediatek,mt8186-efuse", .data = &mtk_mt8186_efuse_pdata },
+	{ .compatible = "mediatek,efuse", .data = &mtk_efuse_pdata },
 	{/* sentinel */},
 };
 MODULE_DEVICE_TABLE(of, mtk_efuse_of_match);
diff --git a/drivers/nvmem/nintendo-otp.c b/drivers/nvmem/nintendo-otp.c
index 33961b17f9f1..355e7f1fc6d5 100644
--- a/drivers/nvmem/nintendo-otp.c
+++ b/drivers/nvmem/nintendo-otp.c
@@ -76,7 +76,6 @@ static int nintendo_otp_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	const struct of_device_id *of_id =
 		of_match_device(nintendo_otp_of_table, dev);
-	struct resource *res;
 	struct nvmem_device *nvmem;
 	struct nintendo_otp_priv *priv;
 
@@ -92,8 +91,7 @@ static int nintendo_otp_probe(struct platform_device *pdev)
 	if (!priv)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	priv->regs = devm_ioremap_resource(dev, res);
+	priv->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(priv->regs))
 		return PTR_ERR(priv->regs);
 
diff --git a/drivers/nvmem/stm32-romem.c b/drivers/nvmem/stm32-romem.c
index ba779e26937a..38d0bf557129 100644
--- a/drivers/nvmem/stm32-romem.c
+++ b/drivers/nvmem/stm32-romem.c
@@ -268,7 +268,7 @@ static const struct stm32_romem_cfg stm32mp13_bsec_cfg = {
 	.ta = true,
 };
 
-static const struct of_device_id stm32_romem_of_match[] = {
+static const struct of_device_id stm32_romem_of_match[] __maybe_unused = {
 	{ .compatible = "st,stm32f4-otp", }, {
 		.compatible = "st,stm32mp15-bsec",
 		.data = (void *)&stm32mp15_bsec_cfg,
diff --git a/drivers/nvmem/u-boot-env.c b/drivers/nvmem/u-boot-env.c
index 29b1d87a3c51..ee9fd9989b6e 100644
--- a/drivers/nvmem/u-boot-env.c
+++ b/drivers/nvmem/u-boot-env.c
@@ -4,6 +4,8 @@
  */
 
 #include <linux/crc32.h>
+#include <linux/etherdevice.h>
+#include <linux/if_ether.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/mtd/mtd.h>
@@ -70,6 +72,25 @@ static int u_boot_env_read(void *context, unsigned int offset, void *val,
 	return 0;
 }
 
+static int u_boot_env_read_post_process_ethaddr(void *context, const char *id, int index,
+						unsigned int offset, void *buf, size_t bytes)
+{
+	u8 mac[ETH_ALEN];
+
+	if (bytes != 3 * ETH_ALEN - 1)
+		return -EINVAL;
+
+	if (!mac_pton(buf, mac))
+		return -EINVAL;
+
+	if (index)
+		eth_addr_add(mac, index);
+
+	ether_addr_copy(buf, mac);
+
+	return 0;
+}
+
 static int u_boot_env_add_cells(struct u_boot_env *priv, uint8_t *buf,
 				size_t data_offset, size_t data_len)
 {
@@ -101,6 +122,11 @@ static int u_boot_env_add_cells(struct u_boot_env *priv, uint8_t *buf,
 		priv->cells[idx].offset = data_offset + value - data;
 		priv->cells[idx].bytes = strlen(value);
 		priv->cells[idx].np = of_get_child_by_name(dev->of_node, priv->cells[idx].name);
+		if (!strcmp(var, "ethaddr")) {
+			priv->cells[idx].raw_len = strlen(value);
+			priv->cells[idx].bytes = ETH_ALEN;
+			priv->cells[idx].read_post_process = u_boot_env_read_post_process_ethaddr;
+		}
 	}
 
 	if (WARN_ON(idx != priv->ncells))
diff --git a/drivers/nvmem/vf610-ocotp.c b/drivers/nvmem/vf610-ocotp.c
index 5b6cad16892f..ee9c61ae727d 100644
--- a/drivers/nvmem/vf610-ocotp.c
+++ b/drivers/nvmem/vf610-ocotp.c
@@ -219,8 +219,7 @@ static int vf610_ocotp_probe(struct platform_device *pdev)
 	if (!ocotp_dev)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	ocotp_dev->base = devm_ioremap_resource(dev, res);
+	ocotp_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(ocotp_dev->base))
 		return PTR_ERR(ocotp_dev->base);
 
diff --git a/drivers/of/Makefile b/drivers/of/Makefile
index e0360a44306e..eff624854575 100644
--- a/drivers/of/Makefile
+++ b/drivers/of/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-y = base.o device.o platform.o property.o
+obj-y = base.o cpu.o device.o module.o platform.o property.o
 obj-$(CONFIG_OF_KOBJ) += kobj.o
 obj-$(CONFIG_OF_DYNAMIC) += dynamic.o
 obj-$(CONFIG_OF_FLATTREE) += fdt.o
diff --git a/drivers/of/address.c b/drivers/of/address.c
index 4c0b169ef9bf..0d49f8c9ed88 100644
--- a/drivers/of/address.c
+++ b/drivers/of/address.c
@@ -22,11 +22,6 @@
 #define OF_CHECK_ADDR_COUNT(na)	((na) > 0 && (na) <= OF_MAX_ADDR_CELLS)
 #define OF_CHECK_COUNTS(na, ns)	(OF_CHECK_ADDR_COUNT(na) && (ns) > 0)
 
-static struct of_bus *of_match_bus(struct device_node *np);
-static int __of_address_to_resource(struct device_node *dev, int index,
-		int bar_no, struct resource *r);
-static bool of_mmio_is_nonposted(struct device_node *np);
-
 /* Debug utility */
 #ifdef DEBUG
 static void of_dump_addr(const char *s, const __be32 *addr, int na)
@@ -95,11 +90,17 @@ static int of_bus_default_translate(__be32 *addr, u64 offset, int na)
 	return 0;
 }
 
+static unsigned int of_bus_default_flags_get_flags(const __be32 *addr)
+{
+	return of_read_number(addr, 1);
+}
+
 static unsigned int of_bus_default_get_flags(const __be32 *addr)
 {
 	return IORESOURCE_MEM;
 }
 
+
 #ifdef CONFIG_PCI
 static unsigned int of_bus_pci_get_flags(const __be32 *addr)
 {
@@ -195,17 +196,6 @@ static int of_bus_pci_translate(__be32 *addr, u64 offset, int na)
 }
 #endif /* CONFIG_PCI */
 
-int of_pci_address_to_resource(struct device_node *dev, int bar,
-			       struct resource *r)
-{
-
-	if (!IS_ENABLED(CONFIG_PCI))
-		return -ENOSYS;
-
-	return __of_address_to_resource(dev, -1, bar, r);
-}
-EXPORT_SYMBOL_GPL(of_pci_address_to_resource);
-
 /*
  * of_pci_range_to_resource - Create a resource from an of_pci_range
  * @range:	the PCI range that describes the resource
@@ -213,7 +203,7 @@ EXPORT_SYMBOL_GPL(of_pci_address_to_resource);
  * @res:	pointer to a valid resource that will be updated to
  *              reflect the values contained in the range.
  *
- * Returns EINVAL if the range cannot be converted to resource.
+ * Returns -EINVAL if the range cannot be converted to resource.
  *
  * Note that if the range is an IO range, the resource will be converted
  * using pci_address_to_pio() which can fail if it is called too early or
@@ -229,9 +219,6 @@ int of_pci_range_to_resource(struct of_pci_range *range,
 	res->parent = res->child = res->sibling = NULL;
 	res->name = np->full_name;
 
-	if (!IS_ENABLED(CONFIG_PCI))
-		return -ENOSYS;
-
 	if (res->flags & IORESOURCE_IO) {
 		unsigned long port;
 		err = pci_register_io_range(&np->fwnode, range->cpu_addr,
@@ -264,6 +251,34 @@ invalid_range:
 EXPORT_SYMBOL(of_pci_range_to_resource);
 
 /*
+ * of_range_to_resource - Create a resource from a ranges entry
+ * @np:		device node where the range belongs to
+ * @index:	the 'ranges' index to convert to a resource
+ * @res:	pointer to a valid resource that will be updated to
+ *              reflect the values contained in the range.
+ *
+ * Returns ENOENT if the entry is not found or EINVAL if the range cannot be
+ * converted to resource.
+ */
+int of_range_to_resource(struct device_node *np, int index, struct resource *res)
+{
+	int ret, i = 0;
+	struct of_range_parser parser;
+	struct of_range range;
+
+	ret = of_range_parser_init(&parser, np);
+	if (ret)
+		return ret;
+
+	for_each_of_range(&parser, &range)
+		if (i++ == index)
+			return of_pci_range_to_resource(&range, np, res);
+
+	return -ENOENT;
+}
+EXPORT_SYMBOL(of_range_to_resource);
+
+/*
  * ISA bus specific translator
  */
 
@@ -319,6 +334,11 @@ static unsigned int of_bus_isa_get_flags(const __be32 *addr)
 	return flags;
 }
 
+static int of_bus_default_flags_match(struct device_node *np)
+{
+	return of_bus_n_addr_cells(np) == 3;
+}
+
 /*
  * Array of bus specific translators
  */
@@ -348,6 +368,17 @@ static struct of_bus of_busses[] = {
 		.has_flags = true,
 		.get_flags = of_bus_isa_get_flags,
 	},
+	/* Default with flags cell */
+	{
+		.name = "default-flags",
+		.addresses = "reg",
+		.match = of_bus_default_flags_match,
+		.count_cells = of_bus_default_count_cells,
+		.map = of_bus_default_map,
+		.translate = of_bus_default_translate,
+		.has_flags = true,
+		.get_flags = of_bus_default_flags_get_flags,
+	},
 	/* Default */
 	{
 		.name = "default",
@@ -713,6 +744,29 @@ const __be32 *__of_get_address(struct device_node *dev, int index, int bar_no,
 }
 EXPORT_SYMBOL(__of_get_address);
 
+/**
+ * of_property_read_reg - Retrieve the specified "reg" entry index without translating
+ * @np: device tree node for which to retrieve "reg" from
+ * @idx: "reg" entry index to read
+ * @addr: return value for the untranslated address
+ * @size: return value for the entry size
+ *
+ * Returns -EINVAL if "reg" is not found. Returns 0 on success with addr and
+ * size values filled in.
+ */
+int of_property_read_reg(struct device_node *np, int idx, u64 *addr, u64 *size)
+{
+	const __be32 *prop = of_get_address(np, idx, size, NULL);
+
+	if (!prop)
+		return -EINVAL;
+
+	*addr = of_read_number(prop, of_n_addr_cells(np));
+
+	return 0;
+}
+EXPORT_SYMBOL(of_property_read_reg);
+
 static int parser_init(struct of_pci_range_parser *parser,
 			struct device_node *node, const char *name)
 {
@@ -834,126 +888,6 @@ static u64 of_translate_ioport(struct device_node *dev, const __be32 *in_addr,
 	return port;
 }
 
-static int __of_address_to_resource(struct device_node *dev, int index, int bar_no,
-		struct resource *r)
-{
-	u64 taddr;
-	const __be32	*addrp;
-	u64		size;
-	unsigned int	flags;
-	const char	*name = NULL;
-
-	addrp = __of_get_address(dev, index, bar_no, &size, &flags);
-	if (addrp == NULL)
-		return -EINVAL;
-
-	/* Get optional "reg-names" property to add a name to a resource */
-	if (index >= 0)
-		of_property_read_string_index(dev, "reg-names",	index, &name);
-
-	if (flags & IORESOURCE_MEM)
-		taddr = of_translate_address(dev, addrp);
-	else if (flags & IORESOURCE_IO)
-		taddr = of_translate_ioport(dev, addrp, size);
-	else
-		return -EINVAL;
-
-	if (taddr == OF_BAD_ADDR)
-		return -EINVAL;
-	memset(r, 0, sizeof(struct resource));
-
-	if (of_mmio_is_nonposted(dev))
-		flags |= IORESOURCE_MEM_NONPOSTED;
-
-	r->start = taddr;
-	r->end = taddr + size - 1;
-	r->flags = flags;
-	r->name = name ? name : dev->full_name;
-
-	return 0;
-}
-
-/**
- * of_address_to_resource - Translate device tree address and return as resource
- * @dev:	Caller's Device Node
- * @index:	Index into the array
- * @r:		Pointer to resource array
- *
- * Note that if your address is a PIO address, the conversion will fail if
- * the physical address can't be internally converted to an IO token with
- * pci_address_to_pio(), that is because it's either called too early or it
- * can't be matched to any host bridge IO space
- */
-int of_address_to_resource(struct device_node *dev, int index,
-			   struct resource *r)
-{
-	return __of_address_to_resource(dev, index, -1, r);
-}
-EXPORT_SYMBOL_GPL(of_address_to_resource);
-
-/**
- * of_iomap - Maps the memory mapped IO for a given device_node
- * @np:		the device whose io range will be mapped
- * @index:	index of the io range
- *
- * Returns a pointer to the mapped memory
- */
-void __iomem *of_iomap(struct device_node *np, int index)
-{
-	struct resource res;
-
-	if (of_address_to_resource(np, index, &res))
-		return NULL;
-
-	if (res.flags & IORESOURCE_MEM_NONPOSTED)
-		return ioremap_np(res.start, resource_size(&res));
-	else
-		return ioremap(res.start, resource_size(&res));
-}
-EXPORT_SYMBOL(of_iomap);
-
-/*
- * of_io_request_and_map - Requests a resource and maps the memory mapped IO
- *			   for a given device_node
- * @device:	the device whose io range will be mapped
- * @index:	index of the io range
- * @name:	name "override" for the memory region request or NULL
- *
- * Returns a pointer to the requested and mapped memory or an ERR_PTR() encoded
- * error code on failure. Usage example:
- *
- *	base = of_io_request_and_map(node, 0, "foo");
- *	if (IS_ERR(base))
- *		return PTR_ERR(base);
- */
-void __iomem *of_io_request_and_map(struct device_node *np, int index,
-				    const char *name)
-{
-	struct resource res;
-	void __iomem *mem;
-
-	if (of_address_to_resource(np, index, &res))
-		return IOMEM_ERR_PTR(-EINVAL);
-
-	if (!name)
-		name = res.name;
-	if (!request_mem_region(res.start, resource_size(&res), name))
-		return IOMEM_ERR_PTR(-EBUSY);
-
-	if (res.flags & IORESOURCE_MEM_NONPOSTED)
-		mem = ioremap_np(res.start, resource_size(&res));
-	else
-		mem = ioremap(res.start, resource_size(&res));
-
-	if (!mem) {
-		release_mem_region(res.start, resource_size(&res));
-		return IOMEM_ERR_PTR(-ENOMEM);
-	}
-
-	return mem;
-}
-EXPORT_SYMBOL(of_io_request_and_map);
-
 #ifdef CONFIG_HAS_DMA
 /**
  * of_dma_get_range - Get DMA range info and put it into a map array
@@ -1150,3 +1084,136 @@ static bool of_mmio_is_nonposted(struct device_node *np)
 	of_node_put(parent);
 	return nonposted;
 }
+
+static int __of_address_to_resource(struct device_node *dev, int index, int bar_no,
+		struct resource *r)
+{
+	u64 taddr;
+	const __be32	*addrp;
+	u64		size;
+	unsigned int	flags;
+	const char	*name = NULL;
+
+	addrp = __of_get_address(dev, index, bar_no, &size, &flags);
+	if (addrp == NULL)
+		return -EINVAL;
+
+	/* Get optional "reg-names" property to add a name to a resource */
+	if (index >= 0)
+		of_property_read_string_index(dev, "reg-names",	index, &name);
+
+	if (flags & IORESOURCE_MEM)
+		taddr = of_translate_address(dev, addrp);
+	else if (flags & IORESOURCE_IO)
+		taddr = of_translate_ioport(dev, addrp, size);
+	else
+		return -EINVAL;
+
+	if (taddr == OF_BAD_ADDR)
+		return -EINVAL;
+	memset(r, 0, sizeof(struct resource));
+
+	if (of_mmio_is_nonposted(dev))
+		flags |= IORESOURCE_MEM_NONPOSTED;
+
+	r->start = taddr;
+	r->end = taddr + size - 1;
+	r->flags = flags;
+	r->name = name ? name : dev->full_name;
+
+	return 0;
+}
+
+/**
+ * of_address_to_resource - Translate device tree address and return as resource
+ * @dev:	Caller's Device Node
+ * @index:	Index into the array
+ * @r:		Pointer to resource array
+ *
+ * Returns -EINVAL if the range cannot be converted to resource.
+ *
+ * Note that if your address is a PIO address, the conversion will fail if
+ * the physical address can't be internally converted to an IO token with
+ * pci_address_to_pio(), that is because it's either called too early or it
+ * can't be matched to any host bridge IO space
+ */
+int of_address_to_resource(struct device_node *dev, int index,
+			   struct resource *r)
+{
+	return __of_address_to_resource(dev, index, -1, r);
+}
+EXPORT_SYMBOL_GPL(of_address_to_resource);
+
+int of_pci_address_to_resource(struct device_node *dev, int bar,
+			       struct resource *r)
+{
+
+	if (!IS_ENABLED(CONFIG_PCI))
+		return -ENOSYS;
+
+	return __of_address_to_resource(dev, -1, bar, r);
+}
+EXPORT_SYMBOL_GPL(of_pci_address_to_resource);
+
+/**
+ * of_iomap - Maps the memory mapped IO for a given device_node
+ * @np:		the device whose io range will be mapped
+ * @index:	index of the io range
+ *
+ * Returns a pointer to the mapped memory
+ */
+void __iomem *of_iomap(struct device_node *np, int index)
+{
+	struct resource res;
+
+	if (of_address_to_resource(np, index, &res))
+		return NULL;
+
+	if (res.flags & IORESOURCE_MEM_NONPOSTED)
+		return ioremap_np(res.start, resource_size(&res));
+	else
+		return ioremap(res.start, resource_size(&res));
+}
+EXPORT_SYMBOL(of_iomap);
+
+/*
+ * of_io_request_and_map - Requests a resource and maps the memory mapped IO
+ *			   for a given device_node
+ * @device:	the device whose io range will be mapped
+ * @index:	index of the io range
+ * @name:	name "override" for the memory region request or NULL
+ *
+ * Returns a pointer to the requested and mapped memory or an ERR_PTR() encoded
+ * error code on failure. Usage example:
+ *
+ *	base = of_io_request_and_map(node, 0, "foo");
+ *	if (IS_ERR(base))
+ *		return PTR_ERR(base);
+ */
+void __iomem *of_io_request_and_map(struct device_node *np, int index,
+				    const char *name)
+{
+	struct resource res;
+	void __iomem *mem;
+
+	if (of_address_to_resource(np, index, &res))
+		return IOMEM_ERR_PTR(-EINVAL);
+
+	if (!name)
+		name = res.name;
+	if (!request_mem_region(res.start, resource_size(&res), name))
+		return IOMEM_ERR_PTR(-EBUSY);
+
+	if (res.flags & IORESOURCE_MEM_NONPOSTED)
+		mem = ioremap_np(res.start, resource_size(&res));
+	else
+		mem = ioremap(res.start, resource_size(&res));
+
+	if (!mem) {
+		release_mem_region(res.start, resource_size(&res));
+		return IOMEM_ERR_PTR(-ENOMEM);
+	}
+
+	return mem;
+}
+EXPORT_SYMBOL(of_io_request_and_map);
diff --git a/drivers/of/base.c b/drivers/of/base.c
index ac6fde53342f..166fb7d75337 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -287,193 +287,6 @@ const void *of_get_property(const struct device_node *np, const char *name,
 EXPORT_SYMBOL(of_get_property);
 
 /**
- * of_get_cpu_hwid - Get the hardware ID from a CPU device node
- *
- * @cpun: CPU number(logical index) for which device node is required
- * @thread: The local thread number to get the hardware ID for.
- *
- * Return: The hardware ID for the CPU node or ~0ULL if not found.
- */
-u64 of_get_cpu_hwid(struct device_node *cpun, unsigned int thread)
-{
-	const __be32 *cell;
-	int ac, len;
-
-	ac = of_n_addr_cells(cpun);
-	cell = of_get_property(cpun, "reg", &len);
-	if (!cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len))
-		return ~0ULL;
-
-	cell += ac * thread;
-	return of_read_number(cell, ac);
-}
-
-/*
- * arch_match_cpu_phys_id - Match the given logical CPU and physical id
- *
- * @cpu: logical cpu index of a core/thread
- * @phys_id: physical identifier of a core/thread
- *
- * CPU logical to physical index mapping is architecture specific.
- * However this __weak function provides a default match of physical
- * id to logical cpu index. phys_id provided here is usually values read
- * from the device tree which must match the hardware internal registers.
- *
- * Returns true if the physical identifier and the logical cpu index
- * correspond to the same core/thread, false otherwise.
- */
-bool __weak arch_match_cpu_phys_id(int cpu, u64 phys_id)
-{
-	return (u32)phys_id == cpu;
-}
-
-/*
- * Checks if the given "prop_name" property holds the physical id of the
- * core/thread corresponding to the logical cpu 'cpu'. If 'thread' is not
- * NULL, local thread number within the core is returned in it.
- */
-static bool __of_find_n_match_cpu_property(struct device_node *cpun,
-			const char *prop_name, int cpu, unsigned int *thread)
-{
-	const __be32 *cell;
-	int ac, prop_len, tid;
-	u64 hwid;
-
-	ac = of_n_addr_cells(cpun);
-	cell = of_get_property(cpun, prop_name, &prop_len);
-	if (!cell && !ac && arch_match_cpu_phys_id(cpu, 0))
-		return true;
-	if (!cell || !ac)
-		return false;
-	prop_len /= sizeof(*cell) * ac;
-	for (tid = 0; tid < prop_len; tid++) {
-		hwid = of_read_number(cell, ac);
-		if (arch_match_cpu_phys_id(cpu, hwid)) {
-			if (thread)
-				*thread = tid;
-			return true;
-		}
-		cell += ac;
-	}
-	return false;
-}
-
-/*
- * arch_find_n_match_cpu_physical_id - See if the given device node is
- * for the cpu corresponding to logical cpu 'cpu'.  Return true if so,
- * else false.  If 'thread' is non-NULL, the local thread number within the
- * core is returned in it.
- */
-bool __weak arch_find_n_match_cpu_physical_id(struct device_node *cpun,
-					      int cpu, unsigned int *thread)
-{
-	/* Check for non-standard "ibm,ppc-interrupt-server#s" property
-	 * for thread ids on PowerPC. If it doesn't exist fallback to
-	 * standard "reg" property.
-	 */
-	if (IS_ENABLED(CONFIG_PPC) &&
-	    __of_find_n_match_cpu_property(cpun,
-					   "ibm,ppc-interrupt-server#s",
-					   cpu, thread))
-		return true;
-
-	return __of_find_n_match_cpu_property(cpun, "reg", cpu, thread);
-}
-
-/**
- * of_get_cpu_node - Get device node associated with the given logical CPU
- *
- * @cpu: CPU number(logical index) for which device node is required
- * @thread: if not NULL, local thread number within the physical core is
- *          returned
- *
- * The main purpose of this function is to retrieve the device node for the
- * given logical CPU index. It should be used to initialize the of_node in
- * cpu device. Once of_node in cpu device is populated, all the further
- * references can use that instead.
- *
- * CPU logical to physical index mapping is architecture specific and is built
- * before booting secondary cores. This function uses arch_match_cpu_phys_id
- * which can be overridden by architecture specific implementation.
- *
- * Return: A node pointer for the logical cpu with refcount incremented, use
- * of_node_put() on it when done. Returns NULL if not found.
- */
-struct device_node *of_get_cpu_node(int cpu, unsigned int *thread)
-{
-	struct device_node *cpun;
-
-	for_each_of_cpu_node(cpun) {
-		if (arch_find_n_match_cpu_physical_id(cpun, cpu, thread))
-			return cpun;
-	}
-	return NULL;
-}
-EXPORT_SYMBOL(of_get_cpu_node);
-
-/**
- * of_cpu_node_to_id: Get the logical CPU number for a given device_node
- *
- * @cpu_node: Pointer to the device_node for CPU.
- *
- * Return: The logical CPU number of the given CPU device_node or -ENODEV if the
- * CPU is not found.
- */
-int of_cpu_node_to_id(struct device_node *cpu_node)
-{
-	int cpu;
-	bool found = false;
-	struct device_node *np;
-
-	for_each_possible_cpu(cpu) {
-		np = of_cpu_device_node_get(cpu);
-		found = (cpu_node == np);
-		of_node_put(np);
-		if (found)
-			return cpu;
-	}
-
-	return -ENODEV;
-}
-EXPORT_SYMBOL(of_cpu_node_to_id);
-
-/**
- * of_get_cpu_state_node - Get CPU's idle state node at the given index
- *
- * @cpu_node: The device node for the CPU
- * @index: The index in the list of the idle states
- *
- * Two generic methods can be used to describe a CPU's idle states, either via
- * a flattened description through the "cpu-idle-states" binding or via the
- * hierarchical layout, using the "power-domains" and the "domain-idle-states"
- * bindings. This function check for both and returns the idle state node for
- * the requested index.
- *
- * Return: An idle state node if found at @index. The refcount is incremented
- * for it, so call of_node_put() on it when done. Returns NULL if not found.
- */
-struct device_node *of_get_cpu_state_node(struct device_node *cpu_node,
-					  int index)
-{
-	struct of_phandle_args args;
-	int err;
-
-	err = of_parse_phandle_with_args(cpu_node, "power-domains",
-					"#power-domain-cells", 0, &args);
-	if (!err) {
-		struct device_node *state_node =
-			of_parse_phandle(args.np, "domain-idle-states", index);
-
-		of_node_put(args.np);
-		if (state_node)
-			return state_node;
-	}
-
-	return of_parse_phandle(cpu_node, "cpu-idle-states", index);
-}
-EXPORT_SYMBOL(of_get_cpu_state_node);
-
-/**
  * __of_device_is_compatible() - Check if the node matches given constraints
  * @device: pointer to node
  * @compat: required compatible string, NULL or "" for any match
@@ -1208,19 +1021,23 @@ struct device_node *of_find_matching_node_and_match(struct device_node *from,
 EXPORT_SYMBOL(of_find_matching_node_and_match);
 
 /**
- * of_modalias_node - Lookup appropriate modalias for a device node
+ * of_alias_from_compatible - Lookup appropriate alias for a device node
+ *			      depending on compatible
  * @node:	pointer to a device tree node
- * @modalias:	Pointer to buffer that modalias value will be copied into
- * @len:	Length of modalias value
+ * @alias:	Pointer to buffer that alias value will be copied into
+ * @len:	Length of alias value
  *
  * Based on the value of the compatible property, this routine will attempt
- * to choose an appropriate modalias value for a particular device tree node.
+ * to choose an appropriate alias value for a particular device tree node.
  * It does this by stripping the manufacturer prefix (as delimited by a ',')
  * from the first entry in the compatible list property.
  *
+ * Note: The matching on just the "product" side of the compatible is a relic
+ * from I2C and SPI. Please do not add any new user.
+ *
  * Return: This routine returns 0 on success, <0 on failure.
  */
-int of_modalias_node(struct device_node *node, char *modalias, int len)
+int of_alias_from_compatible(const struct device_node *node, char *alias, int len)
 {
 	const char *compatible, *p;
 	int cplen;
@@ -1229,10 +1046,10 @@ int of_modalias_node(struct device_node *node, char *modalias, int len)
 	if (!compatible || strlen(compatible) > cplen)
 		return -ENODEV;
 	p = strchr(compatible, ',');
-	strscpy(modalias, p ? p + 1 : compatible, len);
+	strscpy(alias, p ? p + 1 : compatible, len);
 	return 0;
 }
-EXPORT_SYMBOL_GPL(of_modalias_node);
+EXPORT_SYMBOL_GPL(of_alias_from_compatible);
 
 /**
  * of_find_node_by_phandle - Find a node given a phandle
diff --git a/drivers/of/cpu.c b/drivers/of/cpu.c
new file mode 100644
index 000000000000..d17b2f851082
--- /dev/null
+++ b/drivers/of/cpu.c
@@ -0,0 +1,210 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/cpu.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+
+/**
+ * of_get_cpu_hwid - Get the hardware ID from a CPU device node
+ *
+ * @cpun: CPU number(logical index) for which device node is required
+ * @thread: The local thread number to get the hardware ID for.
+ *
+ * Return: The hardware ID for the CPU node or ~0ULL if not found.
+ */
+u64 of_get_cpu_hwid(struct device_node *cpun, unsigned int thread)
+{
+	const __be32 *cell;
+	int ac, len;
+
+	ac = of_n_addr_cells(cpun);
+	cell = of_get_property(cpun, "reg", &len);
+	if (!cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len))
+		return ~0ULL;
+
+	cell += ac * thread;
+	return of_read_number(cell, ac);
+}
+
+/*
+ * arch_match_cpu_phys_id - Match the given logical CPU and physical id
+ *
+ * @cpu: logical cpu index of a core/thread
+ * @phys_id: physical identifier of a core/thread
+ *
+ * CPU logical to physical index mapping is architecture specific.
+ * However this __weak function provides a default match of physical
+ * id to logical cpu index. phys_id provided here is usually values read
+ * from the device tree which must match the hardware internal registers.
+ *
+ * Returns true if the physical identifier and the logical cpu index
+ * correspond to the same core/thread, false otherwise.
+ */
+bool __weak arch_match_cpu_phys_id(int cpu, u64 phys_id)
+{
+	return (u32)phys_id == cpu;
+}
+
+/*
+ * Checks if the given "prop_name" property holds the physical id of the
+ * core/thread corresponding to the logical cpu 'cpu'. If 'thread' is not
+ * NULL, local thread number within the core is returned in it.
+ */
+static bool __of_find_n_match_cpu_property(struct device_node *cpun,
+			const char *prop_name, int cpu, unsigned int *thread)
+{
+	const __be32 *cell;
+	int ac, prop_len, tid;
+	u64 hwid;
+
+	ac = of_n_addr_cells(cpun);
+	cell = of_get_property(cpun, prop_name, &prop_len);
+	if (!cell && !ac && arch_match_cpu_phys_id(cpu, 0))
+		return true;
+	if (!cell || !ac)
+		return false;
+	prop_len /= sizeof(*cell) * ac;
+	for (tid = 0; tid < prop_len; tid++) {
+		hwid = of_read_number(cell, ac);
+		if (arch_match_cpu_phys_id(cpu, hwid)) {
+			if (thread)
+				*thread = tid;
+			return true;
+		}
+		cell += ac;
+	}
+	return false;
+}
+
+/*
+ * arch_find_n_match_cpu_physical_id - See if the given device node is
+ * for the cpu corresponding to logical cpu 'cpu'.  Return true if so,
+ * else false.  If 'thread' is non-NULL, the local thread number within the
+ * core is returned in it.
+ */
+bool __weak arch_find_n_match_cpu_physical_id(struct device_node *cpun,
+					      int cpu, unsigned int *thread)
+{
+	/* Check for non-standard "ibm,ppc-interrupt-server#s" property
+	 * for thread ids on PowerPC. If it doesn't exist fallback to
+	 * standard "reg" property.
+	 */
+	if (IS_ENABLED(CONFIG_PPC) &&
+	    __of_find_n_match_cpu_property(cpun,
+					   "ibm,ppc-interrupt-server#s",
+					   cpu, thread))
+		return true;
+
+	return __of_find_n_match_cpu_property(cpun, "reg", cpu, thread);
+}
+
+/**
+ * of_get_cpu_node - Get device node associated with the given logical CPU
+ *
+ * @cpu: CPU number(logical index) for which device node is required
+ * @thread: if not NULL, local thread number within the physical core is
+ *          returned
+ *
+ * The main purpose of this function is to retrieve the device node for the
+ * given logical CPU index. It should be used to initialize the of_node in
+ * cpu device. Once of_node in cpu device is populated, all the further
+ * references can use that instead.
+ *
+ * CPU logical to physical index mapping is architecture specific and is built
+ * before booting secondary cores. This function uses arch_match_cpu_phys_id
+ * which can be overridden by architecture specific implementation.
+ *
+ * Return: A node pointer for the logical cpu with refcount incremented, use
+ * of_node_put() on it when done. Returns NULL if not found.
+ */
+struct device_node *of_get_cpu_node(int cpu, unsigned int *thread)
+{
+	struct device_node *cpun;
+
+	for_each_of_cpu_node(cpun) {
+		if (arch_find_n_match_cpu_physical_id(cpun, cpu, thread))
+			return cpun;
+	}
+	return NULL;
+}
+EXPORT_SYMBOL(of_get_cpu_node);
+
+/**
+ * of_cpu_device_node_get: Get the CPU device_node for a given logical CPU number
+ *
+ * @cpu: The logical CPU number
+ *
+ * Return: Pointer to the device_node for CPU with its reference count
+ * incremented of the given logical CPU number or NULL if the CPU device_node
+ * is not found.
+ */
+struct device_node *of_cpu_device_node_get(int cpu)
+{
+	struct device *cpu_dev;
+	cpu_dev = get_cpu_device(cpu);
+	if (!cpu_dev)
+		return of_get_cpu_node(cpu, NULL);
+	return of_node_get(cpu_dev->of_node);
+}
+EXPORT_SYMBOL(of_cpu_device_node_get);
+
+/**
+ * of_cpu_node_to_id: Get the logical CPU number for a given device_node
+ *
+ * @cpu_node: Pointer to the device_node for CPU.
+ *
+ * Return: The logical CPU number of the given CPU device_node or -ENODEV if the
+ * CPU is not found.
+ */
+int of_cpu_node_to_id(struct device_node *cpu_node)
+{
+	int cpu;
+	bool found = false;
+	struct device_node *np;
+
+	for_each_possible_cpu(cpu) {
+		np = of_cpu_device_node_get(cpu);
+		found = (cpu_node == np);
+		of_node_put(np);
+		if (found)
+			return cpu;
+	}
+
+	return -ENODEV;
+}
+EXPORT_SYMBOL(of_cpu_node_to_id);
+
+/**
+ * of_get_cpu_state_node - Get CPU's idle state node at the given index
+ *
+ * @cpu_node: The device node for the CPU
+ * @index: The index in the list of the idle states
+ *
+ * Two generic methods can be used to describe a CPU's idle states, either via
+ * a flattened description through the "cpu-idle-states" binding or via the
+ * hierarchical layout, using the "power-domains" and the "domain-idle-states"
+ * bindings. This function check for both and returns the idle state node for
+ * the requested index.
+ *
+ * Return: An idle state node if found at @index. The refcount is incremented
+ * for it, so call of_node_put() on it when done. Returns NULL if not found.
+ */
+struct device_node *of_get_cpu_state_node(struct device_node *cpu_node,
+					  int index)
+{
+	struct of_phandle_args args;
+	int err;
+
+	err = of_parse_phandle_with_args(cpu_node, "power-domains",
+					"#power-domain-cells", 0, &args);
+	if (!err) {
+		struct device_node *state_node =
+			of_parse_phandle(args.np, "domain-idle-states", index);
+
+		of_node_put(args.np);
+		if (state_node)
+			return state_node;
+	}
+
+	return of_parse_phandle(cpu_node, "cpu-idle-states", index);
+}
+EXPORT_SYMBOL(of_get_cpu_state_node);
diff --git a/drivers/of/device.c b/drivers/of/device.c
index 955bfb3d1a83..0f00f1b80708 100644
--- a/drivers/of/device.c
+++ b/drivers/of/device.c
@@ -1,5 +1,4 @@
 // SPDX-License-Identifier: GPL-2.0
-#include <linux/string.h>
 #include <linux/kernel.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -9,7 +8,6 @@
 #include <linux/dma-direct.h> /* for bus_dma_region */
 #include <linux/dma-map-ops.h>
 #include <linux/init.h>
-#include <linux/module.h>
 #include <linux/mod_devicetable.h>
 #include <linux/slab.h>
 #include <linux/platform_device.h>
@@ -248,68 +246,6 @@ const void *of_device_get_match_data(const struct device *dev)
 }
 EXPORT_SYMBOL(of_device_get_match_data);
 
-static ssize_t of_device_get_modalias(const struct device *dev, char *str, ssize_t len)
-{
-	const char *compat;
-	char *c;
-	struct property *p;
-	ssize_t csize;
-	ssize_t tsize;
-
-	if ((!dev) || (!dev->of_node) || dev->of_node_reused)
-		return -ENODEV;
-
-	/* Name & Type */
-	/* %p eats all alphanum characters, so %c must be used here */
-	csize = snprintf(str, len, "of:N%pOFn%c%s", dev->of_node, 'T',
-			 of_node_get_device_type(dev->of_node));
-	tsize = csize;
-	len -= csize;
-	if (str)
-		str += csize;
-
-	of_property_for_each_string(dev->of_node, "compatible", p, compat) {
-		csize = strlen(compat) + 1;
-		tsize += csize;
-		if (csize > len)
-			continue;
-
-		csize = snprintf(str, len, "C%s", compat);
-		for (c = str; c; ) {
-			c = strchr(c, ' ');
-			if (c)
-				*c++ = '_';
-		}
-		len -= csize;
-		str += csize;
-	}
-
-	return tsize;
-}
-
-int of_device_request_module(struct device *dev)
-{
-	char *str;
-	ssize_t size;
-	int ret;
-
-	size = of_device_get_modalias(dev, NULL, 0);
-	if (size < 0)
-		return size;
-
-	str = kmalloc(size + 1, GFP_KERNEL);
-	if (!str)
-		return -ENOMEM;
-
-	of_device_get_modalias(dev, str, size);
-	str[size] = '\0';
-	ret = request_module(str);
-	kfree(str);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(of_device_request_module);
-
 /**
  * of_device_modalias - Fill buffer with newline terminated modalias string
  * @dev:	Calling device
@@ -318,7 +254,12 @@ EXPORT_SYMBOL_GPL(of_device_request_module);
  */
 ssize_t of_device_modalias(struct device *dev, char *str, ssize_t len)
 {
-	ssize_t sl = of_device_get_modalias(dev, str, len - 2);
+	ssize_t sl;
+
+	if (!dev || !dev->of_node || dev->of_node_reused)
+		return -ENODEV;
+
+	sl = of_modalias(dev->of_node, str, len - 2);
 	if (sl < 0)
 		return sl;
 	if (sl > len - 2)
@@ -383,8 +324,8 @@ int of_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *
 	if (add_uevent_var(env, "MODALIAS="))
 		return -ENOMEM;
 
-	sl = of_device_get_modalias(dev, &env->buf[env->buflen-1],
-				    sizeof(env->buf) - env->buflen);
+	sl = of_modalias(dev->of_node, &env->buf[env->buflen-1],
+			 sizeof(env->buf) - env->buflen);
 	if (sl < 0)
 		return sl;
 	if (sl >= (sizeof(env->buf) - env->buflen))
diff --git a/drivers/of/module.c b/drivers/of/module.c
new file mode 100644
index 000000000000..0e8aa974f0f2
--- /dev/null
+++ b/drivers/of/module.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Linux kernel module helpers.
+ */
+
+#include <linux/of.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+ssize_t of_modalias(const struct device_node *np, char *str, ssize_t len)
+{
+	const char *compat;
+	char *c;
+	struct property *p;
+	ssize_t csize;
+	ssize_t tsize;
+
+	/* Name & Type */
+	/* %p eats all alphanum characters, so %c must be used here */
+	csize = snprintf(str, len, "of:N%pOFn%c%s", np, 'T',
+			 of_node_get_device_type(np));
+	tsize = csize;
+	len -= csize;
+	if (str)
+		str += csize;
+
+	of_property_for_each_string(np, "compatible", p, compat) {
+		csize = strlen(compat) + 1;
+		tsize += csize;
+		if (csize > len)
+			continue;
+
+		csize = snprintf(str, len, "C%s", compat);
+		for (c = str; c; ) {
+			c = strchr(c, ' ');
+			if (c)
+				*c++ = '_';
+		}
+		len -= csize;
+		str += csize;
+	}
+
+	return tsize;
+}
+
+int of_request_module(const struct device_node *np)
+{
+	char *str;
+	ssize_t size;
+	int ret;
+
+	if (!np)
+		return -ENODEV;
+
+	size = of_modalias(np, NULL, 0);
+	if (size < 0)
+		return size;
+
+	/* Reserve an additional byte for the trailing '\0' */
+	size++;
+
+	str = kmalloc(size, GFP_KERNEL);
+	if (!str)
+		return -ENOMEM;
+
+	of_modalias(np, str, size);
+	str[size - 1] = '\0';
+	ret = request_module(str);
+	kfree(str);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(of_request_module);
diff --git a/drivers/of/of_private.h b/drivers/of/of_private.h
index fb6792d381a6..b57f1014e419 100644
--- a/drivers/of/of_private.h
+++ b/drivers/of/of_private.h
@@ -38,6 +38,7 @@ struct alias_prop {
 #define OF_ROOT_NODE_SIZE_CELLS_DEFAULT 1
 
 extern struct mutex of_mutex;
+extern raw_spinlock_t devtree_lock;
 extern struct list_head aliases_lookup;
 extern struct kset *of_kset;
 
diff --git a/drivers/of/unittest-data/tests-address.dtsi b/drivers/of/unittest-data/tests-address.dtsi
index 6604a52bf6cb..bc0029cbf8ea 100644
--- a/drivers/of/unittest-data/tests-address.dtsi
+++ b/drivers/of/unittest-data/tests-address.dtsi
@@ -14,7 +14,7 @@
 			#size-cells = <1>;
 			/* ranges here is to make sure we don't use it for
 			 * dma-ranges translation */
-			ranges = <0x70000000 0x70000000 0x40000000>,
+			ranges = <0x70000000 0x70000000 0x50000000>,
 				 <0x00000000 0xd0000000 0x20000000>;
 			dma-ranges = <0x0 0x20000000 0x40000000>;
 
@@ -43,6 +43,13 @@
 					     <0x42000000 0x0 0xc0000000 0x20000000 0x0 0x10000000>;
 			};
 
+			bus@a0000000 {
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges = <0xf00baa 0x0 0x0 0xa0000000 0x0 0x100000>,
+					 <0xf00bee 0x1 0x0 0xb0000000 0x0 0x200000>;
+			};
+
 		};
 	};
 };
diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c
index b5a7a31d8bd2..2191c0136531 100644
--- a/drivers/of/unittest.c
+++ b/drivers/of/unittest.c
@@ -1008,6 +1008,153 @@ static void __init of_unittest_pci_dma_ranges(void)
 	of_node_put(np);
 }
 
+static void __init of_unittest_bus_ranges(void)
+{
+	struct device_node *np;
+	struct of_range range;
+	struct of_range_parser parser;
+	struct resource res;
+	int ret, count, i = 0;
+
+	np = of_find_node_by_path("/testcase-data/address-tests");
+	if (!np) {
+		pr_err("missing testcase data\n");
+		return;
+	}
+
+	if (of_range_parser_init(&parser, np)) {
+		pr_err("missing ranges property\n");
+		return;
+	}
+
+	ret = of_range_to_resource(np, 1, &res);
+	unittest(!ret, "of_range_to_resource returned error (%d) node %pOF\n",
+		ret, np);
+	unittest(resource_type(&res) == IORESOURCE_MEM,
+		"of_range_to_resource wrong resource type on node %pOF res=%pR\n",
+		np, &res);
+	unittest(res.start == 0xd0000000,
+		"of_range_to_resource wrong resource start address on node %pOF res=%pR\n",
+		np, &res);
+	unittest(resource_size(&res) == 0x20000000,
+		"of_range_to_resource wrong resource start address on node %pOF res=%pR\n",
+		np, &res);
+
+	count = of_range_count(&parser);
+	unittest(count == 2,
+		"of_range_count wrong size on node %pOF count=%d\n",
+		np, count);
+
+	/*
+	 * Get the "ranges" from the device tree
+	 */
+	for_each_of_range(&parser, &range) {
+		unittest(range.flags == IORESOURCE_MEM,
+			"for_each_of_range wrong flags on node %pOF flags=%x (expected %x)\n",
+			np, range.flags, IORESOURCE_MEM);
+		if (!i) {
+			unittest(range.size == 0x50000000,
+				 "for_each_of_range wrong size on node %pOF size=%llx\n",
+				 np, range.size);
+			unittest(range.cpu_addr == 0x70000000,
+				 "for_each_of_range wrong CPU addr (%llx) on node %pOF",
+				 range.cpu_addr, np);
+			unittest(range.bus_addr == 0x70000000,
+				 "for_each_of_range wrong bus addr (%llx) on node %pOF",
+				 range.pci_addr, np);
+		} else {
+			unittest(range.size == 0x20000000,
+				 "for_each_of_range wrong size on node %pOF size=%llx\n",
+				 np, range.size);
+			unittest(range.cpu_addr == 0xd0000000,
+				 "for_each_of_range wrong CPU addr (%llx) on node %pOF",
+				 range.cpu_addr, np);
+			unittest(range.bus_addr == 0x00000000,
+				 "for_each_of_range wrong bus addr (%llx) on node %pOF",
+				 range.pci_addr, np);
+		}
+		i++;
+	}
+
+	of_node_put(np);
+}
+
+static void __init of_unittest_bus_3cell_ranges(void)
+{
+	struct device_node *np;
+	struct of_range range;
+	struct of_range_parser parser;
+	int i = 0;
+
+	np = of_find_node_by_path("/testcase-data/address-tests/bus@a0000000");
+	if (!np) {
+		pr_err("missing testcase data\n");
+		return;
+	}
+
+	if (of_range_parser_init(&parser, np)) {
+		pr_err("missing ranges property\n");
+		return;
+	}
+
+	/*
+	 * Get the "ranges" from the device tree
+	 */
+	for_each_of_range(&parser, &range) {
+		if (!i) {
+			unittest(range.flags == 0xf00baa,
+				 "for_each_of_range wrong flags on node %pOF flags=%x\n",
+				 np, range.flags);
+			unittest(range.size == 0x100000,
+				 "for_each_of_range wrong size on node %pOF size=%llx\n",
+				 np, range.size);
+			unittest(range.cpu_addr == 0xa0000000,
+				 "for_each_of_range wrong CPU addr (%llx) on node %pOF",
+				 range.cpu_addr, np);
+			unittest(range.bus_addr == 0x0,
+				 "for_each_of_range wrong bus addr (%llx) on node %pOF",
+				 range.pci_addr, np);
+		} else {
+			unittest(range.flags == 0xf00bee,
+				 "for_each_of_range wrong flags on node %pOF flags=%x\n",
+				 np, range.flags);
+			unittest(range.size == 0x200000,
+				 "for_each_of_range wrong size on node %pOF size=%llx\n",
+				 np, range.size);
+			unittest(range.cpu_addr == 0xb0000000,
+				 "for_each_of_range wrong CPU addr (%llx) on node %pOF",
+				 range.cpu_addr, np);
+			unittest(range.bus_addr == 0x100000000,
+				 "for_each_of_range wrong bus addr (%llx) on node %pOF",
+				 range.pci_addr, np);
+		}
+		i++;
+	}
+
+	of_node_put(np);
+}
+
+static void __init of_unittest_reg(void)
+{
+	struct device_node *np;
+	int ret;
+	u64 addr, size;
+
+	np = of_find_node_by_path("/testcase-data/address-tests/bus@80000000/device@1000");
+	if (!np) {
+		pr_err("missing testcase data\n");
+		return;
+	}
+
+	ret = of_property_read_reg(np, 0, &addr, &size);
+	unittest(!ret, "of_property_read_reg(%pOF) returned error %d\n",
+		np, ret);
+	unittest(addr == 0x1000, "of_property_read_reg(%pOF) untranslated address (%llx) incorrect\n",
+		np, addr);
+
+	of_node_put(np);
+}
+
 static void __init of_unittest_parse_interrupts(void)
 {
 	struct device_node *np;
@@ -1529,13 +1676,12 @@ static int unittest_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int unittest_remove(struct platform_device *pdev)
+static void unittest_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
 
 	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
-	return 0;
 }
 
 static const struct of_device_id unittest_match[] = {
@@ -1545,7 +1691,7 @@ static const struct of_device_id unittest_match[] = {
 
 static struct platform_driver unittest_driver = {
 	.probe			= unittest_probe,
-	.remove			= unittest_remove,
+	.remove_new		= unittest_remove,
 	.driver = {
 		.name		= "unittest",
 		.of_match_table	= of_match_ptr(unittest_match),
@@ -1626,23 +1772,17 @@ static int unittest_gpio_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int unittest_gpio_remove(struct platform_device *pdev)
+static void unittest_gpio_remove(struct platform_device *pdev)
 {
 	struct unittest_gpio_dev *devptr = platform_get_drvdata(pdev);
 	struct device *dev = &pdev->dev;
 
 	dev_dbg(dev, "%s for node @%pfw\n", __func__, devptr->chip.fwnode);
 
-	if (!devptr)
-		return -EINVAL;
-
 	if (devptr->chip.base != -1)
 		gpiochip_remove(&devptr->chip);
 
-	platform_set_drvdata(pdev, NULL);
 	kfree(devptr);
-
-	return 0;
 }
 
 static const struct of_device_id unittest_gpio_id[] = {
@@ -1652,7 +1792,7 @@ static const struct of_device_id unittest_gpio_id[] = {
 
 static struct platform_driver unittest_gpio_driver = {
 	.probe	= unittest_gpio_probe,
-	.remove	= unittest_gpio_remove,
+	.remove_new = unittest_gpio_remove,
 	.driver	= {
 		.name		= "unittest-gpio",
 		.of_match_table	= of_match_ptr(unittest_gpio_id),
@@ -2490,7 +2630,7 @@ static int unittest_i2c_bus_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int unittest_i2c_bus_remove(struct platform_device *pdev)
+static void unittest_i2c_bus_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
@@ -2498,8 +2638,6 @@ static int unittest_i2c_bus_remove(struct platform_device *pdev)
 
 	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
 	i2c_del_adapter(&std->adap);
-
-	return 0;
 }
 
 static const struct of_device_id unittest_i2c_bus_match[] = {
@@ -2509,7 +2647,7 @@ static const struct of_device_id unittest_i2c_bus_match[] = {
 
 static struct platform_driver unittest_i2c_bus_driver = {
 	.probe			= unittest_i2c_bus_probe,
-	.remove			= unittest_i2c_bus_remove,
+	.remove_new		= unittest_i2c_bus_remove,
 	.driver = {
 		.name		= "unittest-i2c-bus",
 		.of_match_table	= of_match_ptr(unittest_i2c_bus_match),
@@ -3644,6 +3782,9 @@ static int __init of_unittest(void)
 	of_unittest_dma_get_max_cpu_address();
 	of_unittest_parse_dma_ranges();
 	of_unittest_pci_dma_ranges();
+	of_unittest_bus_ranges();
+	of_unittest_bus_3cell_ranges();
+	of_unittest_reg();
 	of_unittest_match_node();
 	of_unittest_platform_populate();
 	of_unittest_overlay();
diff --git a/drivers/opp/core.c b/drivers/opp/core.c
index e87567dbe99f..954c94865cf5 100644
--- a/drivers/opp/core.c
+++ b/drivers/opp/core.c
@@ -935,8 +935,8 @@ static int _set_opp_bw(const struct opp_table *opp_table,
 	return 0;
 }
 
-static int _set_required_opp(struct device *dev, struct device *pd_dev,
-			     struct dev_pm_opp *opp, int i)
+static int _set_performance_state(struct device *dev, struct device *pd_dev,
+				  struct dev_pm_opp *opp, int i)
 {
 	unsigned int pstate = likely(opp) ? opp->required_opps[i]->pstate : 0;
 	int ret;
@@ -953,37 +953,19 @@ static int _set_required_opp(struct device *dev, struct device *pd_dev,
 	return ret;
 }
 
-/* This is only called for PM domain for now */
-static int _set_required_opps(struct device *dev,
-			      struct opp_table *opp_table,
-			      struct dev_pm_opp *opp, bool up)
+static int _opp_set_required_opps_generic(struct device *dev,
+	struct opp_table *opp_table, struct dev_pm_opp *opp, bool scaling_down)
 {
-	struct opp_table **required_opp_tables = opp_table->required_opp_tables;
-	struct device **genpd_virt_devs = opp_table->genpd_virt_devs;
-	int i, ret = 0;
-
-	if (!required_opp_tables)
-		return 0;
-
-	/* required-opps not fully initialized yet */
-	if (lazy_linking_pending(opp_table))
-		return -EBUSY;
-
-	/*
-	 * We only support genpd's OPPs in the "required-opps" for now, as we
-	 * don't know much about other use cases. Error out if the required OPP
-	 * doesn't belong to a genpd.
-	 */
-	if (unlikely(!required_opp_tables[0]->is_genpd)) {
-		dev_err(dev, "required-opps don't belong to a genpd\n");
-		return -ENOENT;
-	}
-
-	/* Single genpd case */
-	if (!genpd_virt_devs)
-		return _set_required_opp(dev, dev, opp, 0);
+	dev_err(dev, "setting required-opps isn't supported for non-genpd devices\n");
+	return -ENOENT;
+}
 
-	/* Multiple genpd case */
+static int _opp_set_required_opps_genpd(struct device *dev,
+	struct opp_table *opp_table, struct dev_pm_opp *opp, bool scaling_down)
+{
+	struct device **genpd_virt_devs =
+		opp_table->genpd_virt_devs ? opp_table->genpd_virt_devs : &dev;
+	int i, ret = 0;
 
 	/*
 	 * Acquire genpd_virt_dev_lock to make sure we don't use a genpd_dev
@@ -992,15 +974,15 @@ static int _set_required_opps(struct device *dev,
 	mutex_lock(&opp_table->genpd_virt_dev_lock);
 
 	/* Scaling up? Set required OPPs in normal order, else reverse */
-	if (up) {
+	if (!scaling_down) {
 		for (i = 0; i < opp_table->required_opp_count; i++) {
-			ret = _set_required_opp(dev, genpd_virt_devs[i], opp, i);
+			ret = _set_performance_state(dev, genpd_virt_devs[i], opp, i);
 			if (ret)
 				break;
 		}
 	} else {
 		for (i = opp_table->required_opp_count - 1; i >= 0; i--) {
-			ret = _set_required_opp(dev, genpd_virt_devs[i], opp, i);
+			ret = _set_performance_state(dev, genpd_virt_devs[i], opp, i);
 			if (ret)
 				break;
 		}
@@ -1011,6 +993,34 @@ static int _set_required_opps(struct device *dev,
 	return ret;
 }
 
+/* This is only called for PM domain for now */
+static int _set_required_opps(struct device *dev, struct opp_table *opp_table,
+			      struct dev_pm_opp *opp, bool up)
+{
+	/* required-opps not fully initialized yet */
+	if (lazy_linking_pending(opp_table))
+		return -EBUSY;
+
+	if (opp_table->set_required_opps)
+		return opp_table->set_required_opps(dev, opp_table, opp, up);
+
+	return 0;
+}
+
+/* Update set_required_opps handler */
+void _update_set_required_opps(struct opp_table *opp_table)
+{
+	/* Already set */
+	if (opp_table->set_required_opps)
+		return;
+
+	/* All required OPPs will belong to genpd or none */
+	if (opp_table->required_opp_tables[0]->is_genpd)
+		opp_table->set_required_opps = _opp_set_required_opps_genpd;
+	else
+		opp_table->set_required_opps = _opp_set_required_opps_generic;
+}
+
 static void _find_current_opp(struct device *dev, struct opp_table *opp_table)
 {
 	struct dev_pm_opp *opp = ERR_PTR(-ENODEV);
diff --git a/drivers/opp/of.c b/drivers/opp/of.c
index e55c6095adf0..8246e9b7afe7 100644
--- a/drivers/opp/of.c
+++ b/drivers/opp/of.c
@@ -13,7 +13,7 @@
 #include <linux/cpu.h>
 #include <linux/errno.h>
 #include <linux/device.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/pm_domain.h>
 #include <linux/slab.h>
 #include <linux/export.h>
@@ -196,6 +196,8 @@ static void _opp_table_alloc_required_tables(struct opp_table *opp_table,
 	/* Let's do the linking later on */
 	if (lazy)
 		list_add(&opp_table->lazy, &lazy_opp_tables);
+	else
+		_update_set_required_opps(opp_table);
 
 	goto put_np;
 
@@ -224,7 +226,7 @@ void _of_init_opp_table(struct opp_table *opp_table, struct device *dev,
 	of_property_read_u32(np, "voltage-tolerance",
 			     &opp_table->voltage_tolerance_v1);
 
-	if (of_find_property(np, "#power-domain-cells", NULL))
+	if (of_property_present(np, "#power-domain-cells"))
 		opp_table->is_genpd = true;
 
 	/* Get OPP table node */
@@ -411,6 +413,7 @@ static void lazy_link_required_opp_table(struct opp_table *new_table)
 
 		/* All required opp-tables found, remove from lazy list */
 		if (!lazy) {
+			_update_set_required_opps(opp_table);
 			list_del_init(&opp_table->lazy);
 
 			list_for_each_entry(opp, &opp_table->opp_list, node)
@@ -536,7 +539,7 @@ static bool _opp_is_supported(struct device *dev, struct opp_table *opp_table,
 		 * an OPP then the OPP should not be enabled as there is
 		 * no way to see if the hardware supports it.
 		 */
-		if (of_find_property(np, "opp-supported-hw", NULL))
+		if (of_property_present(np, "opp-supported-hw"))
 			return false;
 		else
 			return true;
diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h
index 3a6e077df386..2a057c42ddf4 100644
--- a/drivers/opp/opp.h
+++ b/drivers/opp/opp.h
@@ -184,6 +184,7 @@ enum opp_table_access {
  * @enabled: Set to true if the device's resources are enabled/configured.
  * @genpd_performance_state: Device's power domain support performance state.
  * @is_genpd: Marks if the OPP table belongs to a genpd.
+ * @set_required_opps: Helper responsible to set required OPPs.
  * @dentry:	debugfs dentry pointer of the real device directory (not links).
  * @dentry_name: Name of the real dentry.
  *
@@ -234,6 +235,8 @@ struct opp_table {
 	bool enabled;
 	bool genpd_performance_state;
 	bool is_genpd;
+	int (*set_required_opps)(struct device *dev,
+		struct opp_table *opp_table, struct dev_pm_opp *opp, bool scaling_down);
 
 #ifdef CONFIG_DEBUG_FS
 	struct dentry *dentry;
@@ -257,6 +260,7 @@ void _dev_pm_opp_cpumask_remove_table(const struct cpumask *cpumask, int last_cp
 struct opp_table *_add_opp_table_indexed(struct device *dev, int index, bool getclk);
 void _put_opp_list_kref(struct opp_table *opp_table);
 void _required_opps_available(struct dev_pm_opp *opp, int count);
+void _update_set_required_opps(struct opp_table *opp_table);
 
 static inline bool lazy_linking_pending(struct opp_table *opp_table)
 {
diff --git a/drivers/parisc/Kconfig b/drivers/parisc/Kconfig
index 9eb2c1b5de7d..2fc3222d2634 100644
--- a/drivers/parisc/Kconfig
+++ b/drivers/parisc/Kconfig
@@ -4,6 +4,7 @@ menu "Bus options (PCI, PCMCIA, EISA, GSC, ISA)"
 config GSC
 	bool "VSC/GSC/HSC bus support"
 	select HAVE_EISA
+	select HAS_IOPORT
 	default y
 	help
 	  The VSC, GSC and HSC busses were used from the earliest 700-series
diff --git a/drivers/parport/parport_pc.c b/drivers/parport/parport_pc.c
index 88e125e36230..3bacbaf16f42 100644
--- a/drivers/parport/parport_pc.c
+++ b/drivers/parport/parport_pc.c
@@ -305,9 +305,15 @@ static size_t parport_pc_epp_read_data(struct parport *port, void *buf,
 		}
 		return got;
 	}
-	if ((flags & PARPORT_EPP_FAST) && (length > 1)) {
-		if (!(((long)buf | length) & 0x03))
+	if ((length > 1) && ((flags & PARPORT_EPP_FAST_32)
+			   || flags & PARPORT_EPP_FAST_16
+			   || flags & PARPORT_EPP_FAST_8)) {
+		if ((flags & PARPORT_EPP_FAST_32)
+		    && !(((long)buf | length) & 0x03))
 			insl(EPPDATA(port), buf, (length >> 2));
+		else if ((flags & PARPORT_EPP_FAST_16)
+			 && !(((long)buf | length) & 0x01))
+			insw(EPPDATA(port), buf, length >> 1);
 		else
 			insb(EPPDATA(port), buf, length);
 		if (inb(STATUS(port)) & 0x01) {
@@ -334,9 +340,15 @@ static size_t parport_pc_epp_write_data(struct parport *port, const void *buf,
 {
 	size_t written = 0;
 
-	if ((flags & PARPORT_EPP_FAST) && (length > 1)) {
-		if (!(((long)buf | length) & 0x03))
+	if ((length > 1) && ((flags & PARPORT_EPP_FAST_32)
+			   || flags & PARPORT_EPP_FAST_16
+			   || flags & PARPORT_EPP_FAST_8)) {
+		if ((flags & PARPORT_EPP_FAST_32)
+		    && !(((long)buf | length) & 0x03))
 			outsl(EPPDATA(port), buf, (length >> 2));
+		else if ((flags & PARPORT_EPP_FAST_16)
+			 && !(((long)buf | length) & 0x01))
+			outsw(EPPDATA(port), buf, length >> 1);
 		else
 			outsb(EPPDATA(port), buf, length);
 		if (inb(STATUS(port)) & 0x01) {
diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c
index 549c4bd5caec..5bc81cc0a2de 100644
--- a/drivers/pci/bus.c
+++ b/drivers/pci/bus.c
@@ -182,13 +182,13 @@ static int pci_bus_alloc_from_region(struct pci_bus *bus, struct resource *res,
 		void *alignf_data,
 		struct pci_bus_region *region)
 {
-	int i, ret;
 	struct resource *r, avail;
 	resource_size_t max;
+	int ret;
 
 	type_mask |= IORESOURCE_TYPE_BITS;
 
-	pci_bus_for_each_resource(bus, r, i) {
+	pci_bus_for_each_resource(bus, r) {
 		resource_size_t min_used = min;
 
 		if (!r)
@@ -289,9 +289,8 @@ bool pci_bus_clip_resource(struct pci_dev *dev, int idx)
 	struct resource *res = &dev->resource[idx];
 	struct resource orig_res = *res;
 	struct resource *r;
-	int i;
 
-	pci_bus_for_each_resource(bus, r, i) {
+	pci_bus_for_each_resource(bus, r) {
 		resource_size_t start, end;
 
 		if (!r)
diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 42654035654a..8d49bad7f847 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -3,19 +3,6 @@
 menu "PCI controller drivers"
 	depends on PCI
 
-config PCI_MVEBU
-	tristate "Marvell EBU PCIe controller"
-	depends on ARCH_MVEBU || ARCH_DOVE || COMPILE_TEST
-	depends on MVEBU_MBUS
-	depends on ARM
-	depends on OF
-	depends on BROKEN
-	select PCI_BRIDGE_EMUL
-	help
-	 Add support for Marvell EBU PCIe controller. This PCIe controller
-	 is used on 32-bit Marvell ARM SoCs: Dove, Kirkwood, Armada 370,
-	 Armada XP, Armada 375, Armada 38x and Armada 39x.
-
 config PCI_AARDVARK
 	tristate "Aardvark PCIe controller"
 	depends on (ARCH_MVEBU && ARM64) || COMPILE_TEST
@@ -27,120 +14,54 @@ config PCI_AARDVARK
 	 controller is part of the South Bridge of the Marvel Armada
 	 3700 SoC.
 
-config PCIE_XILINX_NWL
-	bool "NWL PCIe Core"
-	depends on ARCH_ZYNQMP || COMPILE_TEST
-	depends on PCI_MSI
-	help
-	 Say 'Y' here if you want kernel support for Xilinx
-	 NWL PCIe controller. The controller can act as Root Port
-	 or End Point. The current option selection will only
-	 support root port enabling.
-
-config PCI_FTPCI100
-	bool "Faraday Technology FTPCI100 PCI controller"
-	depends on OF
-	default ARCH_GEMINI
-
-config PCI_IXP4XX
-	bool "Intel IXP4xx PCI controller"
-	depends on ARM && OF
-	depends on ARCH_IXP4XX || COMPILE_TEST
-	default ARCH_IXP4XX
-	help
-	  Say Y here if you want support for the PCI host controller found
-	  in the Intel IXP4xx XScale-based network processor SoC.
-
-config PCI_TEGRA
-	bool "NVIDIA Tegra PCIe controller"
-	depends on ARCH_TEGRA || COMPILE_TEST
-	depends on PCI_MSI
-	help
-	  Say Y here if you want support for the PCIe host controller found
-	  on NVIDIA Tegra SoCs.
-
-config PCI_RCAR_GEN2
-	bool "Renesas R-Car Gen2 Internal PCI controller"
-	depends on ARCH_RENESAS || COMPILE_TEST
-	depends on ARM
+config PCIE_ALTERA
+	tristate "Altera PCIe controller"
+	depends on ARM || NIOS2 || ARM64 || COMPILE_TEST
 	help
-	  Say Y here if you want internal PCI support on R-Car Gen2 SoC.
-	  There are 3 internal PCI controllers available with a single
-	  built-in EHCI/OHCI host controller present on each one.
+	  Say Y here if you want to enable PCIe controller support on Altera
+	  FPGA.
 
-config PCIE_RCAR_HOST
-	bool "Renesas R-Car PCIe host controller"
-	depends on ARCH_RENESAS || COMPILE_TEST
+config PCIE_ALTERA_MSI
+	tristate "Altera PCIe MSI feature"
+	depends on PCIE_ALTERA
 	depends on PCI_MSI
 	help
-	  Say Y here if you want PCIe controller support on R-Car SoCs in host
-	  mode.
-
-config PCIE_RCAR_EP
-	bool "Renesas R-Car PCIe endpoint controller"
-	depends on ARCH_RENESAS || COMPILE_TEST
-	depends on PCI_ENDPOINT
-	help
-	  Say Y here if you want PCIe controller support on R-Car SoCs in
-	  endpoint mode.
+	  Say Y here if you want PCIe MSI support for the Altera FPGA.
+	  This MSI driver supports Altera MSI to GIC controller IP.
 
-config PCI_HOST_COMMON
-	tristate
-	select PCI_ECAM
+config PCIE_APPLE_MSI_DOORBELL_ADDR
+	hex
+	default 0xfffff000
+	depends on PCIE_APPLE
 
-config PCI_HOST_GENERIC
-	tristate "Generic PCI host controller"
+config PCIE_APPLE
+	tristate "Apple PCIe controller"
+	depends on ARCH_APPLE || COMPILE_TEST
 	depends on OF
-	select PCI_HOST_COMMON
-	select IRQ_DOMAIN
-	help
-	  Say Y here if you want to support a simple generic PCI host
-	  controller, such as the one emulated by kvmtool.
-
-config PCIE_XILINX
-	bool "Xilinx AXI PCIe host bridge support"
-	depends on OF || COMPILE_TEST
 	depends on PCI_MSI
-	help
-	  Say 'Y' here if you want kernel to support the Xilinx AXI PCIe
-	  Host Bridge driver.
-
-config PCIE_XILINX_CPM
-	bool "Xilinx Versal CPM host bridge support"
-	depends on ARCH_ZYNQMP || COMPILE_TEST
 	select PCI_HOST_COMMON
 	help
-	  Say 'Y' here if you want kernel support for the
-	  Xilinx Versal CPM host bridge.
-
-config PCI_XGENE
-	bool "X-Gene PCIe controller"
-	depends on ARM64 || COMPILE_TEST
-	depends on OF || (ACPI && PCI_QUIRKS)
-	help
-	  Say Y here if you want internal PCI support on APM X-Gene SoC.
-	  There are 5 internal PCIe ports available. Each port is GEN3 capable
-	  and have varied lanes from x1 to x8.
-
-config PCI_XGENE_MSI
-	bool "X-Gene v1 PCIe MSI feature"
-	depends on PCI_XGENE
-	depends on PCI_MSI
-	default y
-	help
-	  Say Y here if you want PCIe MSI support for the APM X-Gene v1 SoC.
-	  This MSI driver supports 5 PCIe ports on the APM X-Gene v1 SoC.
+	  Say Y here if you want to enable PCIe controller support on Apple
+	  system-on-chips, like the Apple M1. This is required for the USB
+	  type-A ports, Ethernet, Wi-Fi, and Bluetooth.
 
-config PCI_V3_SEMI
-	bool "V3 Semiconductor PCI controller"
-	depends on OF
-	depends on ARM || COMPILE_TEST
-	default ARCH_INTEGRATOR_AP
+	  If unsure, say Y if you have an Apple Silicon system.
 
 config PCI_VERSATILE
 	bool "ARM Versatile PB PCI controller"
 	depends on ARCH_VERSATILE || COMPILE_TEST
 
+config PCIE_BRCMSTB
+	tristate "Broadcom Brcmstb PCIe controller"
+	depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCMBCA || \
+		   BMIPS_GENERIC || COMPILE_TEST
+	depends on OF
+	depends on PCI_MSI
+	default ARCH_BRCMSTB || BMIPS_GENERIC
+	help
+	  Say Y here to enable PCIe host controller support for
+	  Broadcom STB based SoCs, like the Raspberry Pi 4.
+
 config PCIE_IPROC
 	tristate
 	help
@@ -159,7 +80,7 @@ config PCIE_IPROC_PLATFORM
 	  through the generic platform bus interface
 
 config PCIE_IPROC_BCMA
-	tristate "Broadcom iProc PCIe BCMA bus driver"
+	tristate "Broadcom iProc BCMA PCIe controller"
 	depends on ARM && (ARCH_BCM_IPROC || COMPILE_TEST)
 	select PCIE_IPROC
 	select BCMA
@@ -177,21 +98,6 @@ config PCIE_IPROC_MSI
 	  Say Y here if you want to enable MSI support for Broadcom's iProc
 	  PCIe controller
 
-config PCIE_ALTERA
-	tristate "Altera PCIe controller"
-	depends on ARM || NIOS2 || ARM64 || COMPILE_TEST
-	help
-	  Say Y here if you want to enable PCIe controller support on Altera
-	  FPGA.
-
-config PCIE_ALTERA_MSI
-	tristate "Altera PCIe MSI feature"
-	depends on PCIE_ALTERA
-	depends on PCI_MSI
-	help
-	  Say Y here if you want PCIe MSI support for the Altera FPGA.
-	  This MSI driver supports Altera MSI to GIC controller IP.
-
 config PCI_HOST_THUNDER_PEM
 	bool "Cavium Thunder PCIe controller to off-chip devices"
 	depends on ARM64 || COMPILE_TEST
@@ -208,33 +114,77 @@ config PCI_HOST_THUNDER_ECAM
 	help
 	  Say Y here if you want ECAM support for CN88XX-Pass-1.x Cavium Thunder SoCs.
 
-config PCIE_ROCKCHIP
-	bool
-	depends on PCI
+config PCI_FTPCI100
+	bool "Faraday Technology FTPCI100 PCI controller"
+	depends on OF
+	default ARCH_GEMINI
 
-config PCIE_ROCKCHIP_HOST
-	tristate "Rockchip PCIe host controller"
-	depends on ARCH_ROCKCHIP || COMPILE_TEST
+config PCI_HOST_COMMON
+	tristate
+	select PCI_ECAM
+
+config PCI_HOST_GENERIC
+	tristate "Generic PCI host controller"
 	depends on OF
-	depends on PCI_MSI
-	select MFD_SYSCON
-	select PCIE_ROCKCHIP
+	select PCI_HOST_COMMON
+	select IRQ_DOMAIN
 	help
-	  Say Y here if you want internal PCI support on Rockchip SoC.
-	  There is 1 internal PCIe port available to support GEN2 with
-	  4 slots.
+	  Say Y here if you want to support a simple generic PCI host
+	  controller, such as the one emulated by kvmtool.
 
-config PCIE_ROCKCHIP_EP
-	bool "Rockchip PCIe endpoint controller"
-	depends on ARCH_ROCKCHIP || COMPILE_TEST
+config PCIE_HISI_ERR
+	depends on ACPI_APEI_GHES && (ARM64 || COMPILE_TEST)
+	bool "HiSilicon HIP PCIe controller error handling driver"
+	help
+	  Say Y here if you want error handling support
+	  for the PCIe controller's errors on HiSilicon HIP SoCs
+
+config PCI_IXP4XX
+	bool "Intel IXP4xx PCI controller"
+	depends on ARM && OF
+	depends on ARCH_IXP4XX || COMPILE_TEST
+	default ARCH_IXP4XX
+	help
+	  Say Y here if you want support for the PCI host controller found
+	  in the Intel IXP4xx XScale-based network processor SoC.
+
+config VMD
+	depends on PCI_MSI && X86_64 && !UML
+	tristate "Intel Volume Management Device Driver"
+	help
+	  Adds support for the Intel Volume Management Device (VMD). VMD is a
+	  secondary PCI host bridge that allows PCI Express root ports,
+	  and devices attached to them, to be removed from the default
+	  PCI domain and placed within the VMD domain. This provides
+	  more bus resources than are otherwise possible with a
+	  single domain. If you know your system provides one of these and
+	  has devices attached to it, say Y; if you are not sure, say N.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called vmd.
+
+config PCI_LOONGSON
+	bool "LOONGSON PCIe controller"
+	depends on MACH_LOONGSON64 || COMPILE_TEST
+	depends on OF || ACPI
+	depends on PCI_QUIRKS
+	default MACH_LOONGSON64
+	help
+	  Say Y here if you want to enable PCI controller support on
+	  Loongson systems.
+
+config PCI_MVEBU
+	tristate "Marvell EBU PCIe controller"
+	depends on ARCH_MVEBU || ARCH_DOVE || COMPILE_TEST
+	depends on MVEBU_MBUS
+	depends on ARM
 	depends on OF
-	depends on PCI_ENDPOINT
-	select MFD_SYSCON
-	select PCIE_ROCKCHIP
+	depends on BROKEN
+	select PCI_BRIDGE_EMUL
 	help
-	  Say Y here if you want to support Rockchip PCIe controller in
-	  endpoint mode on Rockchip SoC. There is 1 internal PCIe port
-	  available to support GEN2 with 4 slots.
+	 Add support for Marvell EBU PCIe controller. This PCIe controller
+	 is used on 32-bit Marvell ARM SoCs: Dove, Kirkwood, Armada 370,
+	 Armada XP, Armada 375, Armada 38x and Armada 39x.
 
 config PCIE_MEDIATEK
 	tristate "MediaTek PCIe controller"
@@ -258,91 +208,142 @@ config PCIE_MEDIATEK_GEN3
 	  Say Y here if you want to enable Gen3 PCIe controller support on
 	  MediaTek SoCs.
 
-config VMD
-	depends on PCI_MSI && X86_64 && !UML
-	tristate "Intel Volume Management Device Driver"
+config PCIE_MT7621
+	tristate "MediaTek MT7621 PCIe controller"
+	depends on SOC_MT7621 || COMPILE_TEST
+	select PHY_MT7621_PCI
+	default SOC_MT7621
 	help
-	  Adds support for the Intel Volume Management Device (VMD). VMD is a
-	  secondary PCI host bridge that allows PCI Express root ports,
-	  and devices attached to them, to be removed from the default
-	  PCI domain and placed within the VMD domain. This provides
-	  more bus resources than are otherwise possible with a
-	  single domain. If you know your system provides one of these and
-	  has devices attached to it, say Y; if you are not sure, say N.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called vmd.
+	  This selects a driver for the MediaTek MT7621 PCIe Controller.
 
-config PCIE_BRCMSTB
-	tristate "Broadcom Brcmstb PCIe host controller"
-	depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCMBCA || \
-		   BMIPS_GENERIC || COMPILE_TEST
-	depends on OF
-	depends on PCI_MSI
-	default ARCH_BRCMSTB || BMIPS_GENERIC
+config PCIE_MICROCHIP_HOST
+	bool "Microchip AXI PCIe controller"
+	depends on PCI_MSI && OF
+	select PCI_HOST_COMMON
 	help
-	  Say Y here to enable PCIe host controller support for
-	  Broadcom STB based SoCs, like the Raspberry Pi 4.
+	  Say Y here if you want kernel to support the Microchip AXI PCIe
+	  Host Bridge driver.
 
 config PCI_HYPERV_INTERFACE
-	tristate "Hyper-V PCI Interface"
+	tristate "Microsoft Hyper-V PCI Interface"
 	depends on ((X86 && X86_64) || ARM64) && HYPERV && PCI_MSI
 	help
-	  The Hyper-V PCI Interface is a helper driver allows other drivers to
-	  have a common interface with the Hyper-V PCI frontend driver.
+	  The Hyper-V PCI Interface is a helper driver that allows other
+	  drivers to have a common interface with the Hyper-V PCI frontend
+	  driver.
 
-config PCI_LOONGSON
-	bool "LOONGSON PCI Controller"
-	depends on MACH_LOONGSON64 || COMPILE_TEST
-	depends on OF || ACPI
-	depends on PCI_QUIRKS
-	default MACH_LOONGSON64
+config PCI_TEGRA
+	bool "NVIDIA Tegra PCIe controller"
+	depends on ARCH_TEGRA || COMPILE_TEST
+	depends on PCI_MSI
 	help
-	  Say Y here if you want to enable PCI controller support on
-	  Loongson systems.
+	  Say Y here if you want support for the PCIe host controller found
+	  on NVIDIA Tegra SoCs.
 
-config PCIE_MICROCHIP_HOST
-	bool "Microchip AXI PCIe host bridge support"
-	depends on PCI_MSI && OF
-	select PCI_HOST_COMMON
+config PCIE_RCAR_HOST
+	bool "Renesas R-Car PCIe controller (host mode)"
+	depends on ARCH_RENESAS || COMPILE_TEST
+	depends on PCI_MSI
 	help
-	  Say Y here if you want kernel to support the Microchip AXI PCIe
-	  Host Bridge driver.
+	  Say Y here if you want PCIe controller support on R-Car SoCs in host
+	  mode.
 
-config PCIE_HISI_ERR
-	depends on ACPI_APEI_GHES && (ARM64 || COMPILE_TEST)
-	bool "HiSilicon HIP PCIe controller error handling driver"
+config PCIE_RCAR_EP
+	bool "Renesas R-Car PCIe controller (endpoint mode)"
+	depends on ARCH_RENESAS || COMPILE_TEST
+	depends on PCI_ENDPOINT
 	help
-	  Say Y here if you want error handling support
-	  for the PCIe controller's errors on HiSilicon HIP SoCs
+	  Say Y here if you want PCIe controller support on R-Car SoCs in
+	  endpoint mode.
 
-config PCIE_APPLE_MSI_DOORBELL_ADDR
-	hex
-	default 0xfffff000
-	depends on PCIE_APPLE
+config PCI_RCAR_GEN2
+	bool "Renesas R-Car Gen2 Internal PCI controller"
+	depends on ARCH_RENESAS || COMPILE_TEST
+	depends on ARM
+	help
+	  Say Y here if you want internal PCI support on R-Car Gen2 SoC.
+	  There are 3 internal PCI controllers available with a single
+	  built-in EHCI/OHCI host controller present on each one.
 
-config PCIE_APPLE
-	tristate "Apple PCIe controller"
-	depends on ARCH_APPLE || COMPILE_TEST
+config PCIE_ROCKCHIP
+	bool
+	depends on PCI
+
+config PCIE_ROCKCHIP_HOST
+	tristate "Rockchip PCIe controller (host mode)"
+	depends on ARCH_ROCKCHIP || COMPILE_TEST
 	depends on OF
 	depends on PCI_MSI
-	select PCI_HOST_COMMON
+	select MFD_SYSCON
+	select PCIE_ROCKCHIP
 	help
-	  Say Y here if you want to enable PCIe controller support on Apple
-	  system-on-chips, like the Apple M1. This is required for the USB
-	  type-A ports, Ethernet, Wi-Fi, and Bluetooth.
+	  Say Y here if you want internal PCI support on Rockchip SoC.
+	  There is 1 internal PCIe port available to support GEN2 with
+	  4 slots.
 
-	  If unsure, say Y if you have an Apple Silicon system.
+config PCIE_ROCKCHIP_EP
+	bool "Rockchip PCIe controller (endpoint mode)"
+	depends on ARCH_ROCKCHIP || COMPILE_TEST
+	depends on OF
+	depends on PCI_ENDPOINT
+	select MFD_SYSCON
+	select PCIE_ROCKCHIP
+	help
+	  Say Y here if you want to support Rockchip PCIe controller in
+	  endpoint mode on Rockchip SoC. There is 1 internal PCIe port
+	  available to support GEN2 with 4 slots.
 
-config PCIE_MT7621
-	tristate "MediaTek MT7621 PCIe Controller"
-	depends on SOC_MT7621 || COMPILE_TEST
-	select PHY_MT7621_PCI
-	default SOC_MT7621
+config PCI_V3_SEMI
+	bool "V3 Semiconductor PCI controller"
+	depends on OF
+	depends on ARM || COMPILE_TEST
+	default ARCH_INTEGRATOR_AP
+
+config PCI_XGENE
+	bool "X-Gene PCIe controller"
+	depends on ARM64 || COMPILE_TEST
+	depends on OF || (ACPI && PCI_QUIRKS)
 	help
-	  This selects a driver for the MediaTek MT7621 PCIe Controller.
+	  Say Y here if you want internal PCI support on APM X-Gene SoC.
+	  There are 5 internal PCIe ports available. Each port is GEN3 capable
+	  and have varied lanes from x1 to x8.
+
+config PCI_XGENE_MSI
+	bool "X-Gene v1 PCIe MSI feature"
+	depends on PCI_XGENE
+	depends on PCI_MSI
+	default y
+	help
+	  Say Y here if you want PCIe MSI support for the APM X-Gene v1 SoC.
+	  This MSI driver supports 5 PCIe ports on the APM X-Gene v1 SoC.
+
+config PCIE_XILINX
+	bool "Xilinx AXI PCIe controller"
+	depends on OF
+	depends on PCI_MSI
+	help
+	  Say 'Y' here if you want kernel to support the Xilinx AXI PCIe
+	  Host Bridge driver.
+
+config PCIE_XILINX_NWL
+	bool "Xilinx NWL PCIe controller"
+	depends on ARCH_ZYNQMP || COMPILE_TEST
+	depends on PCI_MSI
+	help
+	 Say 'Y' here if you want kernel support for Xilinx
+	 NWL PCIe controller. The controller can act as Root Port
+	 or End Point. The current option selection will only
+	 support root port enabling.
+
+config PCIE_XILINX_CPM
+	bool "Xilinx Versal CPM PCI controller"
+	depends on ARCH_ZYNQMP || COMPILE_TEST
+	select PCI_HOST_COMMON
+	help
+	  Say 'Y' here if you want kernel support for the
+	  Xilinx Versal CPM host bridge.
 
+source "drivers/pci/controller/cadence/Kconfig"
 source "drivers/pci/controller/dwc/Kconfig"
 source "drivers/pci/controller/mobiveil/Kconfig"
-source "drivers/pci/controller/cadence/Kconfig"
 endmenu
diff --git a/drivers/pci/controller/cadence/Kconfig b/drivers/pci/controller/cadence/Kconfig
index 5d30564190e1..291d12711363 100644
--- a/drivers/pci/controller/cadence/Kconfig
+++ b/drivers/pci/controller/cadence/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
-menu "Cadence PCIe controllers support"
+menu "Cadence-based PCIe controllers"
 	depends on PCI
 
 config PCIE_CADENCE
@@ -22,7 +22,7 @@ config PCIE_CADENCE_PLAT
 	bool
 
 config PCIE_CADENCE_PLAT_HOST
-	bool "Cadence PCIe platform host controller"
+	bool "Cadence platform PCIe controller (host mode)"
 	depends on OF
 	select PCIE_CADENCE_HOST
 	select PCIE_CADENCE_PLAT
@@ -32,7 +32,7 @@ config PCIE_CADENCE_PLAT_HOST
 	  vendors SoCs.
 
 config PCIE_CADENCE_PLAT_EP
-	bool "Cadence PCIe platform endpoint controller"
+	bool "Cadence platform PCIe controller (endpoint mode)"
 	depends on OF
 	depends on PCI_ENDPOINT
 	select PCIE_CADENCE_EP
@@ -46,7 +46,7 @@ config PCI_J721E
 	bool
 
 config PCI_J721E_HOST
-	bool "TI J721E PCIe platform host controller"
+	bool "TI J721E PCIe controller (host mode)"
 	depends on OF
 	select PCIE_CADENCE_HOST
 	select PCI_J721E
@@ -56,7 +56,7 @@ config PCI_J721E_HOST
 	  core.
 
 config PCI_J721E_EP
-	bool "TI J721E PCIe platform endpoint controller"
+	bool "TI J721E PCIe controller (endpoint mode)"
 	depends on OF
 	depends on PCI_ENDPOINT
 	select PCIE_CADENCE_EP
diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
index 434f6a4f4041..ab96da43e0c2 100644
--- a/drivers/pci/controller/dwc/Kconfig
+++ b/drivers/pci/controller/dwc/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
-menu "DesignWare PCI Core Support"
+menu "DesignWare-based PCIe controllers"
 	depends on PCI
 
 config PCIE_DW
@@ -14,88 +14,67 @@ config PCIE_DW_EP
 	bool
 	select PCIE_DW
 
-config PCI_DRA7XX
-	tristate
-
-config PCI_DRA7XX_HOST
-	tristate "TI DRA7xx PCIe controller Host Mode"
-	depends on SOC_DRA7XX || COMPILE_TEST
-	depends on OF && HAS_IOMEM && TI_PIPE3
+config PCIE_AL
+	bool "Amazon Annapurna Labs PCIe controller"
+	depends on OF && (ARM64 || COMPILE_TEST)
 	depends on PCI_MSI
 	select PCIE_DW_HOST
-	select PCI_DRA7XX
-	default y if SOC_DRA7XX
+	select PCI_ECAM
 	help
-	  Enables support for the PCIe controller in the DRA7xx SoC to work in
-	  host mode. There are two instances of PCIe controller in DRA7xx.
-	  This controller can work either as EP or RC. In order to enable
-	  host-specific features PCI_DRA7XX_HOST must be selected and in order
-	  to enable device-specific features PCI_DRA7XX_EP must be selected.
-	  This uses the DesignWare core.
+	  Say Y here to enable support of the Amazon's Annapurna Labs PCIe
+	  controller IP on Amazon SoCs. The PCIe controller uses the DesignWare
+	  core plus Annapurna Labs proprietary hardware wrappers. This is
+	  required only for DT-based platforms. ACPI platforms with the
+	  Annapurna Labs PCIe controller don't need to enable this.
 
-config PCI_DRA7XX_EP
-	tristate "TI DRA7xx PCIe controller Endpoint Mode"
-	depends on SOC_DRA7XX || COMPILE_TEST
-	depends on OF && HAS_IOMEM && TI_PIPE3
-	depends on PCI_ENDPOINT
-	select PCIE_DW_EP
-	select PCI_DRA7XX
+config PCI_MESON
+	tristate "Amlogic Meson PCIe controller"
+	default m if ARCH_MESON
+	depends on PCI_MSI
+	select PCIE_DW_HOST
 	help
-	  Enables support for the PCIe controller in the DRA7xx SoC to work in
-	  endpoint mode. There are two instances of PCIe controller in DRA7xx.
-	  This controller can work either as EP or RC. In order to enable
-	  host-specific features PCI_DRA7XX_HOST must be selected and in order
-	  to enable device-specific features PCI_DRA7XX_EP must be selected.
-	  This uses the DesignWare core.
+	  Say Y here if you want to enable PCI controller support on Amlogic
+	  SoCs. The PCI controller on Amlogic is based on DesignWare hardware
+	  and therefore the driver re-uses the DesignWare core functions to
+	  implement the driver.
 
-config PCIE_DW_PLAT
+config PCIE_ARTPEC6
 	bool
 
-config PCIE_DW_PLAT_HOST
-	bool "Platform bus based DesignWare PCIe Controller - Host mode"
+config PCIE_ARTPEC6_HOST
+	bool "Axis ARTPEC-6 PCIe controller (host mode)"
+	depends on MACH_ARTPEC6 || COMPILE_TEST
 	depends on PCI_MSI
 	select PCIE_DW_HOST
-	select PCIE_DW_PLAT
+	select PCIE_ARTPEC6
 	help
-	  Enables support for the PCIe controller in the Designware IP to
-	  work in host mode. There are two instances of PCIe controller in
-	  Designware IP.
-	  This controller can work either as EP or RC. In order to enable
-	  host-specific features PCIE_DW_PLAT_HOST must be selected and in
-	  order to enable device-specific features PCI_DW_PLAT_EP must be
-	  selected.
+	  Enables support for the PCIe controller in the ARTPEC-6 SoC to work in
+	  host mode. This uses the DesignWare core.
 
-config PCIE_DW_PLAT_EP
-	bool "Platform bus based DesignWare PCIe Controller - Endpoint mode"
-	depends on PCI && PCI_MSI
+config PCIE_ARTPEC6_EP
+	bool "Axis ARTPEC-6 PCIe controller (endpoint mode)"
+	depends on MACH_ARTPEC6 || COMPILE_TEST
 	depends on PCI_ENDPOINT
 	select PCIE_DW_EP
-	select PCIE_DW_PLAT
+	select PCIE_ARTPEC6
 	help
-	  Enables support for the PCIe controller in the Designware IP to
-	  work in endpoint mode. There are two instances of PCIe controller
-	  in Designware IP.
-	  This controller can work either as EP or RC. In order to enable
-	  host-specific features PCIE_DW_PLAT_HOST must be selected and in
-	  order to enable device-specific features PCI_DW_PLAT_EP must be
-	  selected.
+	  Enables support for the PCIe controller in the ARTPEC-6 SoC to work in
+	  endpoint mode. This uses the DesignWare core.
 
-config PCI_EXYNOS
-	tristate "Samsung Exynos PCIe controller"
-	depends on ARCH_EXYNOS || COMPILE_TEST
+config PCIE_BT1
+	tristate "Baikal-T1 PCIe controller"
+	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
 	depends on PCI_MSI
 	select PCIE_DW_HOST
 	help
-	  Enables support for the PCIe controller in the Samsung Exynos SoCs
-	  to work in host mode. The PCI controller is based on the DesignWare
-	  hardware and therefore the driver re-uses the DesignWare core
-	  functions to implement the driver.
+	  Enables support for the PCIe controller in the Baikal-T1 SoC to work
+	  in host mode. It's based on the Synopsys DWC PCIe v4.60a IP-core.
 
 config PCI_IMX6
 	bool
 
 config PCI_IMX6_HOST
-	bool "Freescale i.MX6/7/8 PCIe controller host mode"
+	bool "Freescale i.MX6/7/8 PCIe controller (host mode)"
 	depends on ARCH_MXC || COMPILE_TEST
 	depends on PCI_MSI
 	select PCIE_DW_HOST
@@ -107,7 +86,7 @@ config PCI_IMX6_HOST
 	  DesignWare core functions to implement the driver.
 
 config PCI_IMX6_EP
-	bool "Freescale i.MX6/7/8 PCIe controller endpoint mode"
+	bool "Freescale i.MX6/7/8 PCIe controller (endpoint mode)"
 	depends on ARCH_MXC || COMPILE_TEST
 	depends on PCI_ENDPOINT
 	select PCIE_DW_EP
@@ -118,43 +97,8 @@ config PCI_IMX6_EP
 	  on DesignWare hardware and therefore the driver re-uses the
 	  DesignWare core functions to implement the driver.
 
-config PCIE_SPEAR13XX
-	bool "STMicroelectronics SPEAr PCIe controller"
-	depends on ARCH_SPEAR13XX || COMPILE_TEST
-	depends on PCI_MSI
-	select PCIE_DW_HOST
-	help
-	  Say Y here if you want PCIe support on SPEAr13XX SoCs.
-
-config PCI_KEYSTONE
-	bool
-
-config PCI_KEYSTONE_HOST
-	bool "PCI Keystone Host Mode"
-	depends on ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST
-	depends on PCI_MSI
-	select PCIE_DW_HOST
-	select PCI_KEYSTONE
-	help
-	  Enables support for the PCIe controller in the Keystone SoC to
-	  work in host mode. The PCI controller on Keystone is based on
-	  DesignWare hardware and therefore the driver re-uses the
-	  DesignWare core functions to implement the driver.
-
-config PCI_KEYSTONE_EP
-	bool "PCI Keystone Endpoint Mode"
-	depends on ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST
-	depends on PCI_ENDPOINT
-	select PCIE_DW_EP
-	select PCI_KEYSTONE
-	help
-	  Enables support for the PCIe controller in the Keystone SoC to
-	  work in endpoint mode. The PCI controller on Keystone is based
-	  on DesignWare hardware and therefore the driver re-uses the
-	  DesignWare core functions to implement the driver.
-
 config PCI_LAYERSCAPE
-	bool "Freescale Layerscape PCIe controller - Host mode"
+	bool "Freescale Layerscape PCIe controller (host mode)"
 	depends on OF && (ARM || ARCH_LAYERSCAPE || COMPILE_TEST)
 	depends on PCI_MSI
 	select PCIE_DW_HOST
@@ -167,7 +111,7 @@ config PCI_LAYERSCAPE
 	  controller works in RC mode.
 
 config PCI_LAYERSCAPE_EP
-	bool "Freescale Layerscape PCIe controller - Endpoint mode"
+	bool "Freescale Layerscape PCIe controller (endpoint mode)"
 	depends on OF && (ARM || ARCH_LAYERSCAPE || COMPILE_TEST)
 	depends on PCI_ENDPOINT
 	select PCIE_DW_EP
@@ -180,7 +124,7 @@ config PCI_LAYERSCAPE_EP
 
 config PCI_HISI
 	depends on OF && (ARM64 || COMPILE_TEST)
-	bool "HiSilicon Hip05 and Hip06 SoCs PCIe controllers"
+	bool "HiSilicon Hip05 and Hip06 SoCs PCIe controller"
 	depends on PCI_MSI
 	select PCIE_DW_HOST
 	select PCI_HOST_COMMON
@@ -188,83 +132,26 @@ config PCI_HISI
 	  Say Y here if you want PCIe controller support on HiSilicon
 	  Hip05 and Hip06 SoCs
 
-config PCIE_QCOM
-	bool "Qualcomm PCIe controller"
-	depends on OF && (ARCH_QCOM || COMPILE_TEST)
-	depends on PCI_MSI
-	select PCIE_DW_HOST
-	select CRC8
-	help
-	  Say Y here to enable PCIe controller support on Qualcomm SoCs. The
-	  PCIe controller uses the DesignWare core plus Qualcomm-specific
-	  hardware wrappers.
-
-config PCIE_QCOM_EP
-	tristate "Qualcomm PCIe controller - Endpoint mode"
-	depends on OF && (ARCH_QCOM || COMPILE_TEST)
-	depends on PCI_ENDPOINT
-	select PCIE_DW_EP
-	help
-	  Say Y here to enable support for the PCIe controllers on Qualcomm SoCs
-	  to work in endpoint mode. The PCIe controller uses the DesignWare core
-	  plus Qualcomm-specific hardware wrappers.
-
-config PCIE_ARMADA_8K
-	bool "Marvell Armada-8K PCIe controller"
-	depends on ARCH_MVEBU || COMPILE_TEST
-	depends on PCI_MSI
-	select PCIE_DW_HOST
-	help
-	  Say Y here if you want to enable PCIe controller support on
-	  Armada-8K SoCs. The PCIe controller on Armada-8K is based on
-	  DesignWare hardware and therefore the driver re-uses the
-	  DesignWare core functions to implement the driver.
-
-config PCIE_ARTPEC6
-	bool
-
-config PCIE_ARTPEC6_HOST
-	bool "Axis ARTPEC-6 PCIe controller Host Mode"
-	depends on MACH_ARTPEC6 || COMPILE_TEST
+config PCIE_KIRIN
+	depends on OF && (ARM64 || COMPILE_TEST)
+	tristate "HiSilicon Kirin PCIe controller"
 	depends on PCI_MSI
 	select PCIE_DW_HOST
-	select PCIE_ARTPEC6
-	help
-	  Enables support for the PCIe controller in the ARTPEC-6 SoC to work in
-	  host mode. This uses the DesignWare core.
-
-config PCIE_ARTPEC6_EP
-	bool "Axis ARTPEC-6 PCIe controller Endpoint Mode"
-	depends on MACH_ARTPEC6 || COMPILE_TEST
-	depends on PCI_ENDPOINT
-	select PCIE_DW_EP
-	select PCIE_ARTPEC6
+	select REGMAP_MMIO
 	help
-	  Enables support for the PCIe controller in the ARTPEC-6 SoC to work in
-	  endpoint mode. This uses the DesignWare core.
+	  Say Y here if you want PCIe controller support
+	  on HiSilicon Kirin series SoCs.
 
-config PCIE_BT1
-	tristate "Baikal-T1 PCIe controller"
-	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
+config PCIE_HISI_STB
+	bool "HiSilicon STB PCIe controller"
+	depends on ARCH_HISI || COMPILE_TEST
 	depends on PCI_MSI
 	select PCIE_DW_HOST
 	help
-	  Enables support for the PCIe controller in the Baikal-T1 SoC to work
-	  in host mode. It's based on the Synopsys DWC PCIe v4.60a IP-core.
-
-config PCIE_ROCKCHIP_DW_HOST
-	bool "Rockchip DesignWare PCIe controller"
-	select PCIE_DW
-	select PCIE_DW_HOST
-	depends on PCI_MSI
-	depends on ARCH_ROCKCHIP || COMPILE_TEST
-	depends on OF
-	help
-	  Enables support for the DesignWare PCIe controller in the
-	  Rockchip SoC except RK3399.
+	  Say Y here if you want PCIe controller support on HiSilicon STB SoCs
 
 config PCIE_INTEL_GW
-	bool "Intel Gateway PCIe host controller support"
+	bool "Intel Gateway PCIe controller "
 	depends on OF && (X86 || COMPILE_TEST)
 	depends on PCI_MSI
 	select PCIE_DW_HOST
@@ -278,7 +165,7 @@ config PCIE_KEEMBAY
 	bool
 
 config PCIE_KEEMBAY_HOST
-	bool "Intel Keem Bay PCIe controller - Host mode"
+	bool "Intel Keem Bay PCIe controller (host mode)"
 	depends on ARCH_KEEMBAY || COMPILE_TEST
 	depends on PCI_MSI
 	select PCIE_DW_HOST
@@ -290,7 +177,7 @@ config PCIE_KEEMBAY_HOST
 	  DesignWare core functions.
 
 config PCIE_KEEMBAY_EP
-	bool "Intel Keem Bay PCIe controller - Endpoint mode"
+	bool "Intel Keem Bay PCIe controller (endpoint mode)"
 	depends on ARCH_KEEMBAY || COMPILE_TEST
 	depends on PCI_MSI
 	depends on PCI_ENDPOINT
@@ -302,39 +189,22 @@ config PCIE_KEEMBAY_EP
 	  The PCIe controller is based on DesignWare Hardware and uses
 	  DesignWare core functions.
 
-config PCIE_KIRIN
-	depends on OF && (ARM64 || COMPILE_TEST)
-	tristate "HiSilicon Kirin series SoCs PCIe controllers"
-	depends on PCI_MSI
-	select PCIE_DW_HOST
-	help
-	  Say Y here if you want PCIe controller support
-	  on HiSilicon Kirin series SoCs.
-
-config PCIE_HISI_STB
-	bool "HiSilicon STB SoCs PCIe controllers"
-	depends on ARCH_HISI || COMPILE_TEST
-	depends on PCI_MSI
-	select PCIE_DW_HOST
-	help
-	  Say Y here if you want PCIe controller support on HiSilicon STB SoCs
-
-config PCI_MESON
-	tristate "MESON PCIe controller"
-	default m if ARCH_MESON
+config PCIE_ARMADA_8K
+	bool "Marvell Armada-8K PCIe controller"
+	depends on ARCH_MVEBU || COMPILE_TEST
 	depends on PCI_MSI
 	select PCIE_DW_HOST
 	help
-	  Say Y here if you want to enable PCI controller support on Amlogic
-	  SoCs. The PCI controller on Amlogic is based on DesignWare hardware
-	  and therefore the driver re-uses the DesignWare core functions to
-	  implement the driver.
+	  Say Y here if you want to enable PCIe controller support on
+	  Armada-8K SoCs. The PCIe controller on Armada-8K is based on
+	  DesignWare hardware and therefore the driver re-uses the
+	  DesignWare core functions to implement the driver.
 
 config PCIE_TEGRA194
 	tristate
 
 config PCIE_TEGRA194_HOST
-	tristate "NVIDIA Tegra194 (and later) PCIe controller - Host Mode"
+	tristate "NVIDIA Tegra194 (and later) PCIe controller (host mode)"
 	depends on ARCH_TEGRA_194_SOC || COMPILE_TEST
 	depends on PCI_MSI
 	select PCIE_DW_HOST
@@ -349,7 +219,7 @@ config PCIE_TEGRA194_HOST
 	  selected. This uses the DesignWare core.
 
 config PCIE_TEGRA194_EP
-	tristate "NVIDIA Tegra194 (and later) PCIe controller - Endpoint Mode"
+	tristate "NVIDIA Tegra194 (and later) PCIe controller (endpoint mode)"
 	depends on ARCH_TEGRA_194_SOC || COMPILE_TEST
 	depends on PCI_ENDPOINT
 	select PCIE_DW_EP
@@ -363,17 +233,92 @@ config PCIE_TEGRA194_EP
 	  in order to enable device-specific features PCIE_TEGRA194_EP must be
 	  selected. This uses the DesignWare core.
 
-config PCIE_VISCONTI_HOST
-	bool "Toshiba Visconti PCIe controllers"
-	depends on ARCH_VISCONTI || COMPILE_TEST
+config PCIE_DW_PLAT
+	bool
+
+config PCIE_DW_PLAT_HOST
+	bool "Platform bus based DesignWare PCIe controller (host mode)"
 	depends on PCI_MSI
 	select PCIE_DW_HOST
+	select PCIE_DW_PLAT
 	help
-	  Say Y here if you want PCIe controller support on Toshiba Visconti SoC.
-	  This driver supports TMPV7708 SoC.
+	  Enables support for the PCIe controller in the Designware IP to
+	  work in host mode. There are two instances of PCIe controller in
+	  Designware IP.
+	  This controller can work either as EP or RC. In order to enable
+	  host-specific features PCIE_DW_PLAT_HOST must be selected and in
+	  order to enable device-specific features PCI_DW_PLAT_EP must be
+	  selected.
+
+config PCIE_DW_PLAT_EP
+	bool "Platform bus based DesignWare PCIe controller (endpoint mode)"
+	depends on PCI && PCI_MSI
+	depends on PCI_ENDPOINT
+	select PCIE_DW_EP
+	select PCIE_DW_PLAT
+	help
+	  Enables support for the PCIe controller in the Designware IP to
+	  work in endpoint mode. There are two instances of PCIe controller
+	  in Designware IP.
+	  This controller can work either as EP or RC. In order to enable
+	  host-specific features PCIE_DW_PLAT_HOST must be selected and in
+	  order to enable device-specific features PCI_DW_PLAT_EP must be
+	  selected.
+
+config PCIE_QCOM
+	bool "Qualcomm PCIe controller (host mode)"
+	depends on OF && (ARCH_QCOM || COMPILE_TEST)
+	depends on PCI_MSI
+	select PCIE_DW_HOST
+	select CRC8
+	help
+	  Say Y here to enable PCIe controller support on Qualcomm SoCs. The
+	  PCIe controller uses the DesignWare core plus Qualcomm-specific
+	  hardware wrappers.
+
+config PCIE_QCOM_EP
+	tristate "Qualcomm PCIe controller (endpoint mode)"
+	depends on OF && (ARCH_QCOM || COMPILE_TEST)
+	depends on PCI_ENDPOINT
+	select PCIE_DW_EP
+	help
+	  Say Y here to enable support for the PCIe controllers on Qualcomm SoCs
+	  to work in endpoint mode. The PCIe controller uses the DesignWare core
+	  plus Qualcomm-specific hardware wrappers.
+
+config PCIE_ROCKCHIP_DW_HOST
+	bool "Rockchip DesignWare PCIe controller"
+	select PCIE_DW
+	select PCIE_DW_HOST
+	depends on PCI_MSI
+	depends on ARCH_ROCKCHIP || COMPILE_TEST
+	depends on OF
+	help
+	  Enables support for the DesignWare PCIe controller in the
+	  Rockchip SoC except RK3399.
+
+config PCI_EXYNOS
+	tristate "Samsung Exynos PCIe controller"
+	depends on ARCH_EXYNOS || COMPILE_TEST
+	depends on PCI_MSI
+	select PCIE_DW_HOST
+	help
+	  Enables support for the PCIe controller in the Samsung Exynos SoCs
+	  to work in host mode. The PCI controller is based on the DesignWare
+	  hardware and therefore the driver re-uses the DesignWare core
+	  functions to implement the driver.
+
+config PCIE_FU740
+	bool "SiFive FU740 PCIe controller"
+	depends on PCI_MSI
+	depends on SOC_SIFIVE || COMPILE_TEST
+	select PCIE_DW_HOST
+	help
+	  Say Y here if you want PCIe controller support for the SiFive
+	  FU740.
 
 config PCIE_UNIPHIER
-	bool "Socionext UniPhier PCIe host controllers"
+	bool "Socionext UniPhier PCIe controller (host mode)"
 	depends on ARCH_UNIPHIER || COMPILE_TEST
 	depends on OF && HAS_IOMEM
 	depends on PCI_MSI
@@ -383,7 +328,7 @@ config PCIE_UNIPHIER
 	  This driver supports LD20 and PXs3 SoCs.
 
 config PCIE_UNIPHIER_EP
-	bool "Socionext UniPhier PCIe endpoint controllers"
+	bool "Socionext UniPhier PCIe controller (endpoint mode)"
 	depends on ARCH_UNIPHIER || COMPILE_TEST
 	depends on OF && HAS_IOMEM
 	depends on PCI_ENDPOINT
@@ -392,26 +337,82 @@ config PCIE_UNIPHIER_EP
 	  Say Y here if you want PCIe endpoint controller support on
 	  UniPhier SoCs. This driver supports Pro5 SoC.
 
-config PCIE_AL
-	bool "Amazon Annapurna Labs PCIe controller"
-	depends on OF && (ARM64 || COMPILE_TEST)
+config PCIE_SPEAR13XX
+	bool "STMicroelectronics SPEAr PCIe controller"
+	depends on ARCH_SPEAR13XX || COMPILE_TEST
 	depends on PCI_MSI
 	select PCIE_DW_HOST
-	select PCI_ECAM
 	help
-	  Say Y here to enable support of the Amazon's Annapurna Labs PCIe
-	  controller IP on Amazon SoCs. The PCIe controller uses the DesignWare
-	  core plus Annapurna Labs proprietary hardware wrappers. This is
-	  required only for DT-based platforms. ACPI platforms with the
-	  Annapurna Labs PCIe controller don't need to enable this.
+	  Say Y here if you want PCIe support on SPEAr13XX SoCs.
 
-config PCIE_FU740
-	bool "SiFive FU740 PCIe host controller"
+config PCI_DRA7XX
+	tristate
+
+config PCI_DRA7XX_HOST
+	tristate "TI DRA7xx PCIe controller (host mode)"
+	depends on SOC_DRA7XX || COMPILE_TEST
+	depends on OF && HAS_IOMEM && TI_PIPE3
 	depends on PCI_MSI
-	depends on SOC_SIFIVE || COMPILE_TEST
 	select PCIE_DW_HOST
+	select PCI_DRA7XX
+	default y if SOC_DRA7XX
 	help
-	  Say Y here if you want PCIe controller support for the SiFive
-	  FU740.
+	  Enables support for the PCIe controller in the DRA7xx SoC to work in
+	  host mode. There are two instances of PCIe controller in DRA7xx.
+	  This controller can work either as EP or RC. In order to enable
+	  host-specific features PCI_DRA7XX_HOST must be selected and in order
+	  to enable device-specific features PCI_DRA7XX_EP must be selected.
+	  This uses the DesignWare core.
+
+config PCI_DRA7XX_EP
+	tristate "TI DRA7xx PCIe controller (endpoint mode)"
+	depends on SOC_DRA7XX || COMPILE_TEST
+	depends on OF && HAS_IOMEM && TI_PIPE3
+	depends on PCI_ENDPOINT
+	select PCIE_DW_EP
+	select PCI_DRA7XX
+	help
+	  Enables support for the PCIe controller in the DRA7xx SoC to work in
+	  endpoint mode. There are two instances of PCIe controller in DRA7xx.
+	  This controller can work either as EP or RC. In order to enable
+	  host-specific features PCI_DRA7XX_HOST must be selected and in order
+	  to enable device-specific features PCI_DRA7XX_EP must be selected.
+	  This uses the DesignWare core.
+
+config PCI_KEYSTONE
+	bool
+
+config PCI_KEYSTONE_HOST
+	bool "TI Keystone PCIe controller (host mode)"
+	depends on ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST
+	depends on PCI_MSI
+	select PCIE_DW_HOST
+	select PCI_KEYSTONE
+	help
+	  Enables support for the PCIe controller in the Keystone SoC to
+	  work in host mode. The PCI controller on Keystone is based on
+	  DesignWare hardware and therefore the driver re-uses the
+	  DesignWare core functions to implement the driver.
+
+config PCI_KEYSTONE_EP
+	bool "TI Keystone PCIe controller (endpoint mode)"
+	depends on ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST
+	depends on PCI_ENDPOINT
+	select PCIE_DW_EP
+	select PCI_KEYSTONE
+	help
+	  Enables support for the PCIe controller in the Keystone SoC to
+	  work in endpoint mode. The PCI controller on Keystone is based
+	  on DesignWare hardware and therefore the driver re-uses the
+	  DesignWare core functions to implement the driver.
+
+config PCIE_VISCONTI_HOST
+	bool "Toshiba Visconti PCIe controller"
+	depends on ARCH_VISCONTI || COMPILE_TEST
+	depends on PCI_MSI
+	select PCIE_DW_HOST
+	help
+	  Say Y here if you want PCIe controller support on Toshiba Visconti SoC.
+	  This driver supports TMPV7708 SoC.
 
 endmenu
diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c
index 55a0405b921d..52906f999f2b 100644
--- a/drivers/pci/controller/dwc/pci-imx6.c
+++ b/drivers/pci/controller/dwc/pci-imx6.c
@@ -1566,6 +1566,13 @@ DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_SYNOPSYS, 0xabcd,
 static int __init imx6_pcie_init(void)
 {
 #ifdef CONFIG_ARM
+	struct device_node *np;
+
+	np = of_find_matching_node(NULL, imx6_pcie_of_match);
+	if (!np)
+		return -ENODEV;
+	of_node_put(np);
+
 	/*
 	 * Since probe() can be deferred we need to make sure that
 	 * hook_fault_code is not called after __init memory is freed
diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c
index ad99707b3b99..c640db60edc6 100644
--- a/drivers/pci/controller/dwc/pci-layerscape-ep.c
+++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c
@@ -110,6 +110,7 @@ static const struct ls_pcie_ep_drvdata lx2_ep_drvdata = {
 };
 
 static const struct of_device_id ls_pcie_ep_of_match[] = {
+	{ .compatible = "fsl,ls1028a-pcie-ep", .data = &ls1_ep_drvdata },
 	{ .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata },
 	{ .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata },
 	{ .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata },
diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
index a232b04af048..4ab30892f6ef 100644
--- a/drivers/pci/controller/dwc/pcie-qcom.c
+++ b/drivers/pci/controller/dwc/pcie-qcom.c
@@ -10,6 +10,7 @@
 
 #include <linux/clk.h>
 #include <linux/crc8.h>
+#include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/gpio/consumer.h>
 #include <linux/interconnect.h>
@@ -33,7 +34,44 @@
 #include "../../pci.h"
 #include "pcie-designware.h"
 
-#define PCIE20_PARF_SYS_CTRL			0x00
+/* PARF registers */
+#define PARF_SYS_CTRL				0x00
+#define PARF_PM_CTRL				0x20
+#define PARF_PCS_DEEMPH				0x34
+#define PARF_PCS_SWING				0x38
+#define PARF_PHY_CTRL				0x40
+#define PARF_PHY_REFCLK				0x4c
+#define PARF_CONFIG_BITS			0x50
+#define PARF_DBI_BASE_ADDR			0x168
+#define PARF_SLV_ADDR_SPACE_SIZE_2_3_3		0x16c /* Register offset specific to IP ver 2.3.3 */
+#define PARF_MHI_CLOCK_RESET_CTRL		0x174
+#define PARF_AXI_MSTR_WR_ADDR_HALT		0x178
+#define PARF_AXI_MSTR_WR_ADDR_HALT_V2		0x1a8
+#define PARF_Q2A_FLUSH				0x1ac
+#define PARF_LTSSM				0x1b0
+#define PARF_SID_OFFSET				0x234
+#define PARF_BDF_TRANSLATE_CFG			0x24c
+#define PARF_SLV_ADDR_SPACE_SIZE		0x358
+#define PARF_DEVICE_TYPE			0x1000
+#define PARF_BDF_TO_SID_TABLE_N			0x2000
+
+/* ELBI registers */
+#define ELBI_SYS_CTRL				0x04
+
+/* DBI registers */
+#define AXI_MSTR_RESP_COMP_CTRL0		0x818
+#define AXI_MSTR_RESP_COMP_CTRL1		0x81c
+#define MISC_CONTROL_1_REG			0x8bc
+
+/* MHI registers */
+#define PARF_DEBUG_CNT_PM_LINKST_IN_L2		0xc04
+#define PARF_DEBUG_CNT_PM_LINKST_IN_L1		0xc0c
+#define PARF_DEBUG_CNT_PM_LINKST_IN_L0S		0xc10
+#define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1	0xc84
+#define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2	0xc88
+
+/* PARF_SYS_CTRL register fields */
+#define MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN	BIT(29)
 #define MST_WAKEUP_EN				BIT(13)
 #define SLV_WAKEUP_EN				BIT(12)
 #define MSTR_ACLK_CGC_DIS			BIT(10)
@@ -43,45 +81,63 @@
 #define L23_CLK_RMV_DIS				BIT(2)
 #define L1_CLK_RMV_DIS				BIT(1)
 
-#define PCIE20_PARF_PM_CTRL			0x20
+/* PARF_PM_CTRL register fields */
 #define REQ_NOT_ENTR_L1				BIT(5)
 
-#define PCIE20_PARF_PHY_CTRL			0x40
+/* PARF_PCS_DEEMPH register fields */
+#define PCS_DEEMPH_TX_DEEMPH_GEN1(x)		FIELD_PREP(GENMASK(21, 16), x)
+#define PCS_DEEMPH_TX_DEEMPH_GEN2_3_5DB(x)	FIELD_PREP(GENMASK(13, 8), x)
+#define PCS_DEEMPH_TX_DEEMPH_GEN2_6DB(x)	FIELD_PREP(GENMASK(5, 0), x)
+
+/* PARF_PCS_SWING register fields */
+#define PCS_SWING_TX_SWING_FULL(x)		FIELD_PREP(GENMASK(14, 8), x)
+#define PCS_SWING_TX_SWING_LOW(x)		FIELD_PREP(GENMASK(6, 0), x)
+
+/* PARF_PHY_CTRL register fields */
 #define PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK	GENMASK(20, 16)
-#define PHY_CTRL_PHY_TX0_TERM_OFFSET(x)		((x) << 16)
+#define PHY_CTRL_PHY_TX0_TERM_OFFSET(x)		FIELD_PREP(PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK, x)
+#define PHY_TEST_PWR_DOWN			BIT(0)
 
-#define PCIE20_PARF_PHY_REFCLK			0x4C
+/* PARF_PHY_REFCLK register fields */
 #define PHY_REFCLK_SSP_EN			BIT(16)
 #define PHY_REFCLK_USE_PAD			BIT(12)
 
-#define PCIE20_PARF_DBI_BASE_ADDR		0x168
-#define PCIE20_PARF_SLV_ADDR_SPACE_SIZE		0x16C
-#define PCIE20_PARF_MHI_CLOCK_RESET_CTRL	0x174
+/* PARF_CONFIG_BITS register fields */
+#define PHY_RX0_EQ(x)				FIELD_PREP(GENMASK(26, 24), x)
+
+/* PARF_SLV_ADDR_SPACE_SIZE register value */
+#define SLV_ADDR_SPACE_SZ			0x10000000
+
+/* PARF_MHI_CLOCK_RESET_CTRL register fields */
 #define AHB_CLK_EN				BIT(0)
 #define MSTR_AXI_CLK_EN				BIT(1)
 #define BYPASS					BIT(4)
 
-#define PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT	0x178
-#define PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT_V2	0x1A8
-#define PCIE20_PARF_LTSSM			0x1B0
-#define PCIE20_PARF_SID_OFFSET			0x234
-#define PCIE20_PARF_BDF_TRANSLATE_CFG		0x24C
-#define PCIE20_PARF_DEVICE_TYPE			0x1000
-#define PCIE20_PARF_BDF_TO_SID_TABLE_N		0x2000
+/* PARF_AXI_MSTR_WR_ADDR_HALT register fields */
+#define EN					BIT(31)
+
+/* PARF_LTSSM register fields */
+#define LTSSM_EN				BIT(8)
 
-#define PCIE20_ELBI_SYS_CTRL			0x04
-#define PCIE20_ELBI_SYS_CTRL_LT_ENABLE		BIT(0)
+/* PARF_DEVICE_TYPE register fields */
+#define DEVICE_TYPE_RC				0x4
+
+/* ELBI_SYS_CTRL register fields */
+#define ELBI_SYS_CTRL_LT_ENABLE			BIT(0)
 
-#define PCIE20_AXI_MSTR_RESP_COMP_CTRL0		0x818
+/* AXI_MSTR_RESP_COMP_CTRL0 register fields */
 #define CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K	0x4
 #define CFG_REMOTE_RD_REQ_BRIDGE_SIZE_4K	0x5
-#define PCIE20_AXI_MSTR_RESP_COMP_CTRL1		0x81c
+
+/* AXI_MSTR_RESP_COMP_CTRL1 register fields */
 #define CFG_BRIDGE_SB_INIT			BIT(0)
 
-#define PCIE_CAP_SLOT_POWER_LIMIT_VAL		FIELD_PREP(PCI_EXP_SLTCAP_SPLV, \
-						250)
-#define PCIE_CAP_SLOT_POWER_LIMIT_SCALE		FIELD_PREP(PCI_EXP_SLTCAP_SPLS, \
-						1)
+/* MISC_CONTROL_1_REG register fields */
+#define DBI_RO_WR_EN				1
+
+/* PCI_EXP_SLTCAP register fields */
+#define PCIE_CAP_SLOT_POWER_LIMIT_VAL		FIELD_PREP(PCI_EXP_SLTCAP_SPLV, 250)
+#define PCIE_CAP_SLOT_POWER_LIMIT_SCALE		FIELD_PREP(PCI_EXP_SLTCAP_SPLS, 1)
 #define PCIE_CAP_SLOT_VAL			(PCI_EXP_SLTCAP_ABP | \
 						PCI_EXP_SLTCAP_PCP | \
 						PCI_EXP_SLTCAP_MRLSP | \
@@ -93,103 +149,62 @@
 						PCIE_CAP_SLOT_POWER_LIMIT_VAL | \
 						PCIE_CAP_SLOT_POWER_LIMIT_SCALE)
 
-#define PCIE20_PARF_Q2A_FLUSH			0x1AC
-
-#define PCIE20_MISC_CONTROL_1_REG		0x8BC
-#define DBI_RO_WR_EN				1
-
 #define PERST_DELAY_US				1000
-/* PARF registers */
-#define PCIE20_PARF_PCS_DEEMPH			0x34
-#define PCS_DEEMPH_TX_DEEMPH_GEN1(x)		((x) << 16)
-#define PCS_DEEMPH_TX_DEEMPH_GEN2_3_5DB(x)	((x) << 8)
-#define PCS_DEEMPH_TX_DEEMPH_GEN2_6DB(x)	((x) << 0)
 
-#define PCIE20_PARF_PCS_SWING			0x38
-#define PCS_SWING_TX_SWING_FULL(x)		((x) << 8)
-#define PCS_SWING_TX_SWING_LOW(x)		((x) << 0)
+#define QCOM_PCIE_CRC8_POLYNOMIAL		(BIT(2) | BIT(1) | BIT(0))
 
-#define PCIE20_PARF_CONFIG_BITS		0x50
-#define PHY_RX0_EQ(x)				((x) << 24)
-
-#define PCIE20_v3_PARF_SLV_ADDR_SPACE_SIZE	0x358
-#define SLV_ADDR_SPACE_SZ			0x10000000
-
-#define PCIE20_LNK_CONTROL2_LINK_STATUS2	0xa0
-
-#define DEVICE_TYPE_RC				0x4
-
-#define QCOM_PCIE_2_1_0_MAX_SUPPLY	3
-#define QCOM_PCIE_2_1_0_MAX_CLOCKS	5
-
-#define QCOM_PCIE_CRC8_POLYNOMIAL (BIT(2) | BIT(1) | BIT(0))
+#define QCOM_PCIE_1_0_0_MAX_CLOCKS		4
+struct qcom_pcie_resources_1_0_0 {
+	struct clk_bulk_data clks[QCOM_PCIE_1_0_0_MAX_CLOCKS];
+	struct reset_control *core;
+	struct regulator *vdda;
+};
 
+#define QCOM_PCIE_2_1_0_MAX_CLOCKS		5
+#define QCOM_PCIE_2_1_0_MAX_RESETS		6
+#define QCOM_PCIE_2_1_0_MAX_SUPPLY		3
 struct qcom_pcie_resources_2_1_0 {
 	struct clk_bulk_data clks[QCOM_PCIE_2_1_0_MAX_CLOCKS];
-	struct reset_control *pci_reset;
-	struct reset_control *axi_reset;
-	struct reset_control *ahb_reset;
-	struct reset_control *por_reset;
-	struct reset_control *phy_reset;
-	struct reset_control *ext_reset;
+	struct reset_control_bulk_data resets[QCOM_PCIE_2_1_0_MAX_RESETS];
+	int num_resets;
 	struct regulator_bulk_data supplies[QCOM_PCIE_2_1_0_MAX_SUPPLY];
 };
 
-struct qcom_pcie_resources_1_0_0 {
-	struct clk *iface;
-	struct clk *aux;
-	struct clk *master_bus;
-	struct clk *slave_bus;
-	struct reset_control *core;
-	struct regulator *vdda;
-};
-
-#define QCOM_PCIE_2_3_2_MAX_SUPPLY	2
+#define QCOM_PCIE_2_3_2_MAX_CLOCKS		4
+#define QCOM_PCIE_2_3_2_MAX_SUPPLY		2
 struct qcom_pcie_resources_2_3_2 {
-	struct clk *aux_clk;
-	struct clk *master_clk;
-	struct clk *slave_clk;
-	struct clk *cfg_clk;
+	struct clk_bulk_data clks[QCOM_PCIE_2_3_2_MAX_CLOCKS];
 	struct regulator_bulk_data supplies[QCOM_PCIE_2_3_2_MAX_SUPPLY];
 };
 
-#define QCOM_PCIE_2_4_0_MAX_CLOCKS	4
+#define QCOM_PCIE_2_3_3_MAX_CLOCKS		5
+#define QCOM_PCIE_2_3_3_MAX_RESETS		7
+struct qcom_pcie_resources_2_3_3 {
+	struct clk_bulk_data clks[QCOM_PCIE_2_3_3_MAX_CLOCKS];
+	struct reset_control_bulk_data rst[QCOM_PCIE_2_3_3_MAX_RESETS];
+};
+
+#define QCOM_PCIE_2_4_0_MAX_CLOCKS		4
+#define QCOM_PCIE_2_4_0_MAX_RESETS		12
 struct qcom_pcie_resources_2_4_0 {
 	struct clk_bulk_data clks[QCOM_PCIE_2_4_0_MAX_CLOCKS];
 	int num_clks;
-	struct reset_control *axi_m_reset;
-	struct reset_control *axi_s_reset;
-	struct reset_control *pipe_reset;
-	struct reset_control *axi_m_vmid_reset;
-	struct reset_control *axi_s_xpu_reset;
-	struct reset_control *parf_reset;
-	struct reset_control *phy_reset;
-	struct reset_control *axi_m_sticky_reset;
-	struct reset_control *pipe_sticky_reset;
-	struct reset_control *pwr_reset;
-	struct reset_control *ahb_reset;
-	struct reset_control *phy_ahb_reset;
-};
-
-struct qcom_pcie_resources_2_3_3 {
-	struct clk *iface;
-	struct clk *axi_m_clk;
-	struct clk *axi_s_clk;
-	struct clk *ahb_clk;
-	struct clk *aux_clk;
-	struct reset_control *rst[7];
+	struct reset_control_bulk_data resets[QCOM_PCIE_2_4_0_MAX_RESETS];
+	int num_resets;
 };
 
-/* 6 clocks typically, 7 for sm8250 */
+#define QCOM_PCIE_2_7_0_MAX_CLOCKS		15
+#define QCOM_PCIE_2_7_0_MAX_SUPPLIES		2
 struct qcom_pcie_resources_2_7_0 {
-	struct clk_bulk_data clks[12];
+	struct clk_bulk_data clks[QCOM_PCIE_2_7_0_MAX_CLOCKS];
 	int num_clks;
-	struct regulator_bulk_data supplies[2];
-	struct reset_control *pci_reset;
+	struct regulator_bulk_data supplies[QCOM_PCIE_2_7_0_MAX_SUPPLIES];
+	struct reset_control *rst;
 };
 
+#define QCOM_PCIE_2_9_0_MAX_CLOCKS		5
 struct qcom_pcie_resources_2_9_0 {
-	struct clk_bulk_data clks[5];
+	struct clk_bulk_data clks[QCOM_PCIE_2_9_0_MAX_CLOCKS];
 	struct reset_control *rst;
 };
 
@@ -222,11 +237,14 @@ struct qcom_pcie {
 	struct dw_pcie *pci;
 	void __iomem *parf;			/* DT parf */
 	void __iomem *elbi;			/* DT elbi */
+	void __iomem *mhi;
 	union qcom_pcie_resources res;
 	struct phy *phy;
 	struct gpio_desc *reset;
 	struct icc_path *icc_mem;
 	const struct qcom_pcie_cfg *cfg;
+	struct dentry *debugfs;
+	bool suspended;
 };
 
 #define to_qcom_pcie(x)		dev_get_drvdata((x)->dev)
@@ -261,9 +279,9 @@ static void qcom_pcie_2_1_0_ltssm_enable(struct qcom_pcie *pcie)
 	u32 val;
 
 	/* enable link training */
-	val = readl(pcie->elbi + PCIE20_ELBI_SYS_CTRL);
-	val |= PCIE20_ELBI_SYS_CTRL_LT_ENABLE;
-	writel(val, pcie->elbi + PCIE20_ELBI_SYS_CTRL);
+	val = readl(pcie->elbi + ELBI_SYS_CTRL);
+	val |= ELBI_SYS_CTRL_LT_ENABLE;
+	writel(val, pcie->elbi + ELBI_SYS_CTRL);
 }
 
 static int qcom_pcie_get_resources_2_1_0(struct qcom_pcie *pcie)
@@ -271,6 +289,7 @@ static int qcom_pcie_get_resources_2_1_0(struct qcom_pcie *pcie)
 	struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
+	bool is_apq = of_device_is_compatible(dev->of_node, "qcom,pcie-apq8064");
 	int ret;
 
 	res->supplies[0].supply = "vdda";
@@ -297,28 +316,20 @@ static int qcom_pcie_get_resources_2_1_0(struct qcom_pcie *pcie)
 	if (ret < 0)
 		return ret;
 
-	res->pci_reset = devm_reset_control_get_exclusive(dev, "pci");
-	if (IS_ERR(res->pci_reset))
-		return PTR_ERR(res->pci_reset);
-
-	res->axi_reset = devm_reset_control_get_exclusive(dev, "axi");
-	if (IS_ERR(res->axi_reset))
-		return PTR_ERR(res->axi_reset);
-
-	res->ahb_reset = devm_reset_control_get_exclusive(dev, "ahb");
-	if (IS_ERR(res->ahb_reset))
-		return PTR_ERR(res->ahb_reset);
+	res->resets[0].id = "pci";
+	res->resets[1].id = "axi";
+	res->resets[2].id = "ahb";
+	res->resets[3].id = "por";
+	res->resets[4].id = "phy";
+	res->resets[5].id = "ext";
 
-	res->por_reset = devm_reset_control_get_exclusive(dev, "por");
-	if (IS_ERR(res->por_reset))
-		return PTR_ERR(res->por_reset);
-
-	res->ext_reset = devm_reset_control_get_optional_exclusive(dev, "ext");
-	if (IS_ERR(res->ext_reset))
-		return PTR_ERR(res->ext_reset);
+	/* ext is optional on APQ8016 */
+	res->num_resets = is_apq ? 5 : 6;
+	ret = devm_reset_control_bulk_get_exclusive(dev, res->num_resets, res->resets);
+	if (ret < 0)
+		return ret;
 
-	res->phy_reset = devm_reset_control_get_exclusive(dev, "phy");
-	return PTR_ERR_OR_ZERO(res->phy_reset);
+	return 0;
 }
 
 static void qcom_pcie_deinit_2_1_0(struct qcom_pcie *pcie)
@@ -326,14 +337,9 @@ static void qcom_pcie_deinit_2_1_0(struct qcom_pcie *pcie)
 	struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0;
 
 	clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
-	reset_control_assert(res->pci_reset);
-	reset_control_assert(res->axi_reset);
-	reset_control_assert(res->ahb_reset);
-	reset_control_assert(res->por_reset);
-	reset_control_assert(res->ext_reset);
-	reset_control_assert(res->phy_reset);
+	reset_control_bulk_assert(res->num_resets, res->resets);
 
-	writel(1, pcie->parf + PCIE20_PARF_PHY_CTRL);
+	writel(1, pcie->parf + PARF_PHY_CTRL);
 
 	regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
 }
@@ -346,12 +352,11 @@ static int qcom_pcie_init_2_1_0(struct qcom_pcie *pcie)
 	int ret;
 
 	/* reset the PCIe interface as uboot can leave it undefined state */
-	reset_control_assert(res->pci_reset);
-	reset_control_assert(res->axi_reset);
-	reset_control_assert(res->ahb_reset);
-	reset_control_assert(res->por_reset);
-	reset_control_assert(res->ext_reset);
-	reset_control_assert(res->phy_reset);
+	ret = reset_control_bulk_assert(res->num_resets, res->resets);
+	if (ret < 0) {
+		dev_err(dev, "cannot assert resets\n");
+		return ret;
+	}
 
 	ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies);
 	if (ret < 0) {
@@ -359,58 +364,14 @@ static int qcom_pcie_init_2_1_0(struct qcom_pcie *pcie)
 		return ret;
 	}
 
-	ret = reset_control_deassert(res->ahb_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert ahb reset\n");
-		goto err_deassert_ahb;
-	}
-
-	ret = reset_control_deassert(res->ext_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert ext reset\n");
-		goto err_deassert_ext;
-	}
-
-	ret = reset_control_deassert(res->phy_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert phy reset\n");
-		goto err_deassert_phy;
-	}
-
-	ret = reset_control_deassert(res->pci_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert pci reset\n");
-		goto err_deassert_pci;
-	}
-
-	ret = reset_control_deassert(res->por_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert por reset\n");
-		goto err_deassert_por;
-	}
-
-	ret = reset_control_deassert(res->axi_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert axi reset\n");
-		goto err_deassert_axi;
+	ret = reset_control_bulk_deassert(res->num_resets, res->resets);
+	if (ret < 0) {
+		dev_err(dev, "cannot deassert resets\n");
+		regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
+		return ret;
 	}
 
 	return 0;
-
-err_deassert_axi:
-	reset_control_assert(res->por_reset);
-err_deassert_por:
-	reset_control_assert(res->pci_reset);
-err_deassert_pci:
-	reset_control_assert(res->phy_reset);
-err_deassert_phy:
-	reset_control_assert(res->ext_reset);
-err_deassert_ext:
-	reset_control_assert(res->ahb_reset);
-err_deassert_ahb:
-	regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
-
-	return ret;
 }
 
 static int qcom_pcie_post_init_2_1_0(struct qcom_pcie *pcie)
@@ -423,9 +384,9 @@ static int qcom_pcie_post_init_2_1_0(struct qcom_pcie *pcie)
 	int ret;
 
 	/* enable PCIe clocks and resets */
-	val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
-	val &= ~BIT(0);
-	writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val &= ~PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
 	ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks);
 	if (ret)
@@ -436,37 +397,37 @@ static int qcom_pcie_post_init_2_1_0(struct qcom_pcie *pcie)
 		writel(PCS_DEEMPH_TX_DEEMPH_GEN1(24) |
 			       PCS_DEEMPH_TX_DEEMPH_GEN2_3_5DB(24) |
 			       PCS_DEEMPH_TX_DEEMPH_GEN2_6DB(34),
-		       pcie->parf + PCIE20_PARF_PCS_DEEMPH);
+		       pcie->parf + PARF_PCS_DEEMPH);
 		writel(PCS_SWING_TX_SWING_FULL(120) |
 			       PCS_SWING_TX_SWING_LOW(120),
-		       pcie->parf + PCIE20_PARF_PCS_SWING);
-		writel(PHY_RX0_EQ(4), pcie->parf + PCIE20_PARF_CONFIG_BITS);
+		       pcie->parf + PARF_PCS_SWING);
+		writel(PHY_RX0_EQ(4), pcie->parf + PARF_CONFIG_BITS);
 	}
 
 	if (of_device_is_compatible(node, "qcom,pcie-ipq8064")) {
 		/* set TX termination offset */
-		val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
+		val = readl(pcie->parf + PARF_PHY_CTRL);
 		val &= ~PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK;
 		val |= PHY_CTRL_PHY_TX0_TERM_OFFSET(7);
-		writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
+		writel(val, pcie->parf + PARF_PHY_CTRL);
 	}
 
 	/* enable external reference clock */
-	val = readl(pcie->parf + PCIE20_PARF_PHY_REFCLK);
+	val = readl(pcie->parf + PARF_PHY_REFCLK);
 	/* USE_PAD is required only for ipq806x */
 	if (!of_device_is_compatible(node, "qcom,pcie-apq8064"))
 		val &= ~PHY_REFCLK_USE_PAD;
 	val |= PHY_REFCLK_SSP_EN;
-	writel(val, pcie->parf + PCIE20_PARF_PHY_REFCLK);
+	writel(val, pcie->parf + PARF_PHY_REFCLK);
 
 	/* wait for clock acquisition */
 	usleep_range(1000, 1500);
 
 	/* Set the Max TLP size to 2K, instead of using default of 4K */
 	writel(CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K,
-	       pci->dbi_base + PCIE20_AXI_MSTR_RESP_COMP_CTRL0);
+	       pci->dbi_base + AXI_MSTR_RESP_COMP_CTRL0);
 	writel(CFG_BRIDGE_SB_INIT,
-	       pci->dbi_base + PCIE20_AXI_MSTR_RESP_COMP_CTRL1);
+	       pci->dbi_base + AXI_MSTR_RESP_COMP_CTRL1);
 
 	return 0;
 }
@@ -476,26 +437,20 @@ static int qcom_pcie_get_resources_1_0_0(struct qcom_pcie *pcie)
 	struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
+	int ret;
 
 	res->vdda = devm_regulator_get(dev, "vdda");
 	if (IS_ERR(res->vdda))
 		return PTR_ERR(res->vdda);
 
-	res->iface = devm_clk_get(dev, "iface");
-	if (IS_ERR(res->iface))
-		return PTR_ERR(res->iface);
-
-	res->aux = devm_clk_get(dev, "aux");
-	if (IS_ERR(res->aux))
-		return PTR_ERR(res->aux);
-
-	res->master_bus = devm_clk_get(dev, "master_bus");
-	if (IS_ERR(res->master_bus))
-		return PTR_ERR(res->master_bus);
+	res->clks[0].id = "iface";
+	res->clks[1].id = "aux";
+	res->clks[2].id = "master_bus";
+	res->clks[3].id = "slave_bus";
 
-	res->slave_bus = devm_clk_get(dev, "slave_bus");
-	if (IS_ERR(res->slave_bus))
-		return PTR_ERR(res->slave_bus);
+	ret = devm_clk_bulk_get(dev, ARRAY_SIZE(res->clks), res->clks);
+	if (ret < 0)
+		return ret;
 
 	res->core = devm_reset_control_get_exclusive(dev, "core");
 	return PTR_ERR_OR_ZERO(res->core);
@@ -506,10 +461,7 @@ static void qcom_pcie_deinit_1_0_0(struct qcom_pcie *pcie)
 	struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0;
 
 	reset_control_assert(res->core);
-	clk_disable_unprepare(res->slave_bus);
-	clk_disable_unprepare(res->master_bus);
-	clk_disable_unprepare(res->iface);
-	clk_disable_unprepare(res->aux);
+	clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
 	regulator_disable(res->vdda);
 }
 
@@ -526,46 +478,23 @@ static int qcom_pcie_init_1_0_0(struct qcom_pcie *pcie)
 		return ret;
 	}
 
-	ret = clk_prepare_enable(res->aux);
-	if (ret) {
-		dev_err(dev, "cannot prepare/enable aux clock\n");
-		goto err_res;
-	}
-
-	ret = clk_prepare_enable(res->iface);
-	if (ret) {
-		dev_err(dev, "cannot prepare/enable iface clock\n");
-		goto err_aux;
-	}
-
-	ret = clk_prepare_enable(res->master_bus);
-	if (ret) {
-		dev_err(dev, "cannot prepare/enable master_bus clock\n");
-		goto err_iface;
-	}
-
-	ret = clk_prepare_enable(res->slave_bus);
+	ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks);
 	if (ret) {
-		dev_err(dev, "cannot prepare/enable slave_bus clock\n");
-		goto err_master;
+		dev_err(dev, "cannot prepare/enable clocks\n");
+		goto err_assert_reset;
 	}
 
 	ret = regulator_enable(res->vdda);
 	if (ret) {
 		dev_err(dev, "cannot enable vdda regulator\n");
-		goto err_slave;
+		goto err_disable_clks;
 	}
 
 	return 0;
-err_slave:
-	clk_disable_unprepare(res->slave_bus);
-err_master:
-	clk_disable_unprepare(res->master_bus);
-err_iface:
-	clk_disable_unprepare(res->iface);
-err_aux:
-	clk_disable_unprepare(res->aux);
-err_res:
+
+err_disable_clks:
+	clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
+err_assert_reset:
 	reset_control_assert(res->core);
 
 	return ret;
@@ -574,13 +503,13 @@ err_res:
 static int qcom_pcie_post_init_1_0_0(struct qcom_pcie *pcie)
 {
 	/* change DBI base address */
-	writel(0, pcie->parf + PCIE20_PARF_DBI_BASE_ADDR);
+	writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
 
 	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-		u32 val = readl(pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT);
+		u32 val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT);
 
-		val |= BIT(31);
-		writel(val, pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT);
+		val |= EN;
+		writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT);
 	}
 
 	return 0;
@@ -591,9 +520,9 @@ static void qcom_pcie_2_3_2_ltssm_enable(struct qcom_pcie *pcie)
 	u32 val;
 
 	/* enable link training */
-	val = readl(pcie->parf + PCIE20_PARF_LTSSM);
-	val |= BIT(8);
-	writel(val, pcie->parf + PCIE20_PARF_LTSSM);
+	val = readl(pcie->parf + PARF_LTSSM);
+	val |= LTSSM_EN;
+	writel(val, pcie->parf + PARF_LTSSM);
 }
 
 static int qcom_pcie_get_resources_2_3_2(struct qcom_pcie *pcie)
@@ -610,21 +539,14 @@ static int qcom_pcie_get_resources_2_3_2(struct qcom_pcie *pcie)
 	if (ret)
 		return ret;
 
-	res->aux_clk = devm_clk_get(dev, "aux");
-	if (IS_ERR(res->aux_clk))
-		return PTR_ERR(res->aux_clk);
-
-	res->cfg_clk = devm_clk_get(dev, "cfg");
-	if (IS_ERR(res->cfg_clk))
-		return PTR_ERR(res->cfg_clk);
-
-	res->master_clk = devm_clk_get(dev, "bus_master");
-	if (IS_ERR(res->master_clk))
-		return PTR_ERR(res->master_clk);
+	res->clks[0].id = "aux";
+	res->clks[1].id = "cfg";
+	res->clks[2].id = "bus_master";
+	res->clks[3].id = "bus_slave";
 
-	res->slave_clk = devm_clk_get(dev, "bus_slave");
-	if (IS_ERR(res->slave_clk))
-		return PTR_ERR(res->slave_clk);
+	ret = devm_clk_bulk_get(dev, ARRAY_SIZE(res->clks), res->clks);
+	if (ret < 0)
+		return ret;
 
 	return 0;
 }
@@ -633,11 +555,7 @@ static void qcom_pcie_deinit_2_3_2(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2;
 
-	clk_disable_unprepare(res->slave_clk);
-	clk_disable_unprepare(res->master_clk);
-	clk_disable_unprepare(res->cfg_clk);
-	clk_disable_unprepare(res->aux_clk);
-
+	clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
 	regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
 }
 
@@ -654,43 +572,14 @@ static int qcom_pcie_init_2_3_2(struct qcom_pcie *pcie)
 		return ret;
 	}
 
-	ret = clk_prepare_enable(res->aux_clk);
-	if (ret) {
-		dev_err(dev, "cannot prepare/enable aux clock\n");
-		goto err_aux_clk;
-	}
-
-	ret = clk_prepare_enable(res->cfg_clk);
-	if (ret) {
-		dev_err(dev, "cannot prepare/enable cfg clock\n");
-		goto err_cfg_clk;
-	}
-
-	ret = clk_prepare_enable(res->master_clk);
-	if (ret) {
-		dev_err(dev, "cannot prepare/enable master clock\n");
-		goto err_master_clk;
-	}
-
-	ret = clk_prepare_enable(res->slave_clk);
+	ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks);
 	if (ret) {
-		dev_err(dev, "cannot prepare/enable slave clock\n");
-		goto err_slave_clk;
+		dev_err(dev, "cannot prepare/enable clocks\n");
+		regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
+		return ret;
 	}
 
 	return 0;
-
-err_slave_clk:
-	clk_disable_unprepare(res->master_clk);
-err_master_clk:
-	clk_disable_unprepare(res->cfg_clk);
-err_cfg_clk:
-	clk_disable_unprepare(res->aux_clk);
-
-err_aux_clk:
-	regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
-
-	return ret;
 }
 
 static int qcom_pcie_post_init_2_3_2(struct qcom_pcie *pcie)
@@ -698,25 +587,25 @@ static int qcom_pcie_post_init_2_3_2(struct qcom_pcie *pcie)
 	u32 val;
 
 	/* enable PCIe clocks and resets */
-	val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
-	val &= ~BIT(0);
-	writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val &= ~PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
 	/* change DBI base address */
-	writel(0, pcie->parf + PCIE20_PARF_DBI_BASE_ADDR);
+	writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
 
 	/* MAC PHY_POWERDOWN MUX DISABLE  */
-	val = readl(pcie->parf + PCIE20_PARF_SYS_CTRL);
-	val &= ~BIT(29);
-	writel(val, pcie->parf + PCIE20_PARF_SYS_CTRL);
+	val = readl(pcie->parf + PARF_SYS_CTRL);
+	val &= ~MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN;
+	writel(val, pcie->parf + PARF_SYS_CTRL);
 
-	val = readl(pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL);
-	val |= BIT(4);
-	writel(val, pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL);
+	val = readl(pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
+	val |= BYPASS;
+	writel(val, pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
 
-	val = readl(pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT_V2);
-	val |= BIT(31);
-	writel(val, pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT_V2);
+	val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2);
+	val |= EN;
+	writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2);
 
 	return 0;
 }
@@ -741,65 +630,24 @@ static int qcom_pcie_get_resources_2_4_0(struct qcom_pcie *pcie)
 	if (ret < 0)
 		return ret;
 
-	res->axi_m_reset = devm_reset_control_get_exclusive(dev, "axi_m");
-	if (IS_ERR(res->axi_m_reset))
-		return PTR_ERR(res->axi_m_reset);
-
-	res->axi_s_reset = devm_reset_control_get_exclusive(dev, "axi_s");
-	if (IS_ERR(res->axi_s_reset))
-		return PTR_ERR(res->axi_s_reset);
-
-	if (is_ipq) {
-		/*
-		 * These resources relates to the PHY or are secure clocks, but
-		 * are controlled here for IPQ4019
-		 */
-		res->pipe_reset = devm_reset_control_get_exclusive(dev, "pipe");
-		if (IS_ERR(res->pipe_reset))
-			return PTR_ERR(res->pipe_reset);
-
-		res->axi_m_vmid_reset = devm_reset_control_get_exclusive(dev,
-									 "axi_m_vmid");
-		if (IS_ERR(res->axi_m_vmid_reset))
-			return PTR_ERR(res->axi_m_vmid_reset);
-
-		res->axi_s_xpu_reset = devm_reset_control_get_exclusive(dev,
-									"axi_s_xpu");
-		if (IS_ERR(res->axi_s_xpu_reset))
-			return PTR_ERR(res->axi_s_xpu_reset);
-
-		res->parf_reset = devm_reset_control_get_exclusive(dev, "parf");
-		if (IS_ERR(res->parf_reset))
-			return PTR_ERR(res->parf_reset);
-
-		res->phy_reset = devm_reset_control_get_exclusive(dev, "phy");
-		if (IS_ERR(res->phy_reset))
-			return PTR_ERR(res->phy_reset);
-	}
-
-	res->axi_m_sticky_reset = devm_reset_control_get_exclusive(dev,
-								   "axi_m_sticky");
-	if (IS_ERR(res->axi_m_sticky_reset))
-		return PTR_ERR(res->axi_m_sticky_reset);
-
-	res->pipe_sticky_reset = devm_reset_control_get_exclusive(dev,
-								  "pipe_sticky");
-	if (IS_ERR(res->pipe_sticky_reset))
-		return PTR_ERR(res->pipe_sticky_reset);
-
-	res->pwr_reset = devm_reset_control_get_exclusive(dev, "pwr");
-	if (IS_ERR(res->pwr_reset))
-		return PTR_ERR(res->pwr_reset);
-
-	res->ahb_reset = devm_reset_control_get_exclusive(dev, "ahb");
-	if (IS_ERR(res->ahb_reset))
-		return PTR_ERR(res->ahb_reset);
-
-	if (is_ipq) {
-		res->phy_ahb_reset = devm_reset_control_get_exclusive(dev, "phy_ahb");
-		if (IS_ERR(res->phy_ahb_reset))
-			return PTR_ERR(res->phy_ahb_reset);
-	}
+	res->resets[0].id = "axi_m";
+	res->resets[1].id = "axi_s";
+	res->resets[2].id = "axi_m_sticky";
+	res->resets[3].id = "pipe_sticky";
+	res->resets[4].id = "pwr";
+	res->resets[5].id = "ahb";
+	res->resets[6].id = "pipe";
+	res->resets[7].id = "axi_m_vmid";
+	res->resets[8].id = "axi_s_xpu";
+	res->resets[9].id = "parf";
+	res->resets[10].id = "phy";
+	res->resets[11].id = "phy_ahb";
+
+	res->num_resets = is_ipq ? 12 : 6;
+
+	ret = devm_reset_control_bulk_get_exclusive(dev, res->num_resets, res->resets);
+	if (ret < 0)
+		return ret;
 
 	return 0;
 }
@@ -808,15 +656,7 @@ static void qcom_pcie_deinit_2_4_0(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0;
 
-	reset_control_assert(res->axi_m_reset);
-	reset_control_assert(res->axi_s_reset);
-	reset_control_assert(res->pipe_reset);
-	reset_control_assert(res->pipe_sticky_reset);
-	reset_control_assert(res->phy_reset);
-	reset_control_assert(res->phy_ahb_reset);
-	reset_control_assert(res->axi_m_sticky_reset);
-	reset_control_assert(res->pwr_reset);
-	reset_control_assert(res->ahb_reset);
+	reset_control_bulk_assert(res->num_resets, res->resets);
 	clk_bulk_disable_unprepare(res->num_clks, res->clks);
 }
 
@@ -827,149 +667,29 @@ static int qcom_pcie_init_2_4_0(struct qcom_pcie *pcie)
 	struct device *dev = pci->dev;
 	int ret;
 
-	ret = reset_control_assert(res->axi_m_reset);
-	if (ret) {
-		dev_err(dev, "cannot assert axi master reset\n");
-		return ret;
-	}
-
-	ret = reset_control_assert(res->axi_s_reset);
-	if (ret) {
-		dev_err(dev, "cannot assert axi slave reset\n");
-		return ret;
-	}
-
-	usleep_range(10000, 12000);
-
-	ret = reset_control_assert(res->pipe_reset);
-	if (ret) {
-		dev_err(dev, "cannot assert pipe reset\n");
-		return ret;
-	}
-
-	ret = reset_control_assert(res->pipe_sticky_reset);
-	if (ret) {
-		dev_err(dev, "cannot assert pipe sticky reset\n");
-		return ret;
-	}
-
-	ret = reset_control_assert(res->phy_reset);
-	if (ret) {
-		dev_err(dev, "cannot assert phy reset\n");
-		return ret;
-	}
-
-	ret = reset_control_assert(res->phy_ahb_reset);
-	if (ret) {
-		dev_err(dev, "cannot assert phy ahb reset\n");
+	ret = reset_control_bulk_assert(res->num_resets, res->resets);
+	if (ret < 0) {
+		dev_err(dev, "cannot assert resets\n");
 		return ret;
 	}
 
 	usleep_range(10000, 12000);
 
-	ret = reset_control_assert(res->axi_m_sticky_reset);
-	if (ret) {
-		dev_err(dev, "cannot assert axi master sticky reset\n");
-		return ret;
-	}
-
-	ret = reset_control_assert(res->pwr_reset);
-	if (ret) {
-		dev_err(dev, "cannot assert power reset\n");
-		return ret;
-	}
-
-	ret = reset_control_assert(res->ahb_reset);
-	if (ret) {
-		dev_err(dev, "cannot assert ahb reset\n");
+	ret = reset_control_bulk_deassert(res->num_resets, res->resets);
+	if (ret < 0) {
+		dev_err(dev, "cannot deassert resets\n");
 		return ret;
 	}
 
 	usleep_range(10000, 12000);
 
-	ret = reset_control_deassert(res->phy_ahb_reset);
+	ret = clk_bulk_prepare_enable(res->num_clks, res->clks);
 	if (ret) {
-		dev_err(dev, "cannot deassert phy ahb reset\n");
+		reset_control_bulk_assert(res->num_resets, res->resets);
 		return ret;
 	}
 
-	ret = reset_control_deassert(res->phy_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert phy reset\n");
-		goto err_rst_phy;
-	}
-
-	ret = reset_control_deassert(res->pipe_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert pipe reset\n");
-		goto err_rst_pipe;
-	}
-
-	ret = reset_control_deassert(res->pipe_sticky_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert pipe sticky reset\n");
-		goto err_rst_pipe_sticky;
-	}
-
-	usleep_range(10000, 12000);
-
-	ret = reset_control_deassert(res->axi_m_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert axi master reset\n");
-		goto err_rst_axi_m;
-	}
-
-	ret = reset_control_deassert(res->axi_m_sticky_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert axi master sticky reset\n");
-		goto err_rst_axi_m_sticky;
-	}
-
-	ret = reset_control_deassert(res->axi_s_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert axi slave reset\n");
-		goto err_rst_axi_s;
-	}
-
-	ret = reset_control_deassert(res->pwr_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert power reset\n");
-		goto err_rst_pwr;
-	}
-
-	ret = reset_control_deassert(res->ahb_reset);
-	if (ret) {
-		dev_err(dev, "cannot deassert ahb reset\n");
-		goto err_rst_ahb;
-	}
-
-	usleep_range(10000, 12000);
-
-	ret = clk_bulk_prepare_enable(res->num_clks, res->clks);
-	if (ret)
-		goto err_clks;
-
 	return 0;
-
-err_clks:
-	reset_control_assert(res->ahb_reset);
-err_rst_ahb:
-	reset_control_assert(res->pwr_reset);
-err_rst_pwr:
-	reset_control_assert(res->axi_s_reset);
-err_rst_axi_s:
-	reset_control_assert(res->axi_m_sticky_reset);
-err_rst_axi_m_sticky:
-	reset_control_assert(res->axi_m_reset);
-err_rst_axi_m:
-	reset_control_assert(res->pipe_sticky_reset);
-err_rst_pipe_sticky:
-	reset_control_assert(res->pipe_reset);
-err_rst_pipe:
-	reset_control_assert(res->phy_reset);
-err_rst_phy:
-	reset_control_assert(res->phy_ahb_reset);
-	return ret;
 }
 
 static int qcom_pcie_post_init_2_4_0(struct qcom_pcie *pcie)
@@ -977,25 +697,25 @@ static int qcom_pcie_post_init_2_4_0(struct qcom_pcie *pcie)
 	u32 val;
 
 	/* enable PCIe clocks and resets */
-	val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
-	val &= ~BIT(0);
-	writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val &= ~PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
 	/* change DBI base address */
-	writel(0, pcie->parf + PCIE20_PARF_DBI_BASE_ADDR);
+	writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
 
 	/* MAC PHY_POWERDOWN MUX DISABLE  */
-	val = readl(pcie->parf + PCIE20_PARF_SYS_CTRL);
-	val &= ~BIT(29);
-	writel(val, pcie->parf + PCIE20_PARF_SYS_CTRL);
+	val = readl(pcie->parf + PARF_SYS_CTRL);
+	val &= ~MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN;
+	writel(val, pcie->parf + PARF_SYS_CTRL);
 
-	val = readl(pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL);
-	val |= BIT(4);
-	writel(val, pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL);
+	val = readl(pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
+	val |= BYPASS;
+	writel(val, pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
 
-	val = readl(pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT_V2);
-	val |= BIT(31);
-	writel(val, pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT_V2);
+	val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2);
+	val |= EN;
+	writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2);
 
 	return 0;
 }
@@ -1005,36 +725,29 @@ static int qcom_pcie_get_resources_2_3_3(struct qcom_pcie *pcie)
 	struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
-	int i;
-	const char *rst_names[] = { "axi_m", "axi_s", "pipe",
-				    "axi_m_sticky", "sticky",
-				    "ahb", "sleep", };
-
-	res->iface = devm_clk_get(dev, "iface");
-	if (IS_ERR(res->iface))
-		return PTR_ERR(res->iface);
-
-	res->axi_m_clk = devm_clk_get(dev, "axi_m");
-	if (IS_ERR(res->axi_m_clk))
-		return PTR_ERR(res->axi_m_clk);
-
-	res->axi_s_clk = devm_clk_get(dev, "axi_s");
-	if (IS_ERR(res->axi_s_clk))
-		return PTR_ERR(res->axi_s_clk);
-
-	res->ahb_clk = devm_clk_get(dev, "ahb");
-	if (IS_ERR(res->ahb_clk))
-		return PTR_ERR(res->ahb_clk);
-
-	res->aux_clk = devm_clk_get(dev, "aux");
-	if (IS_ERR(res->aux_clk))
-		return PTR_ERR(res->aux_clk);
-
-	for (i = 0; i < ARRAY_SIZE(rst_names); i++) {
-		res->rst[i] = devm_reset_control_get(dev, rst_names[i]);
-		if (IS_ERR(res->rst[i]))
-			return PTR_ERR(res->rst[i]);
-	}
+	int ret;
+
+	res->clks[0].id = "iface";
+	res->clks[1].id = "axi_m";
+	res->clks[2].id = "axi_s";
+	res->clks[3].id = "ahb";
+	res->clks[4].id = "aux";
+
+	ret = devm_clk_bulk_get(dev, ARRAY_SIZE(res->clks), res->clks);
+	if (ret < 0)
+		return ret;
+
+	res->rst[0].id = "axi_m";
+	res->rst[1].id = "axi_s";
+	res->rst[2].id = "pipe";
+	res->rst[3].id = "axi_m_sticky";
+	res->rst[4].id = "sticky";
+	res->rst[5].id = "ahb";
+	res->rst[6].id = "sleep";
+
+	ret = devm_reset_control_bulk_get_exclusive(dev, ARRAY_SIZE(res->rst), res->rst);
+	if (ret < 0)
+		return ret;
 
 	return 0;
 }
@@ -1043,11 +756,7 @@ static void qcom_pcie_deinit_2_3_3(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3;
 
-	clk_disable_unprepare(res->iface);
-	clk_disable_unprepare(res->axi_m_clk);
-	clk_disable_unprepare(res->axi_s_clk);
-	clk_disable_unprepare(res->ahb_clk);
-	clk_disable_unprepare(res->aux_clk);
+	clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
 }
 
 static int qcom_pcie_init_2_3_3(struct qcom_pcie *pcie)
@@ -1055,25 +764,20 @@ static int qcom_pcie_init_2_3_3(struct qcom_pcie *pcie)
 	struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
-	int i, ret;
+	int ret;
 
-	for (i = 0; i < ARRAY_SIZE(res->rst); i++) {
-		ret = reset_control_assert(res->rst[i]);
-		if (ret) {
-			dev_err(dev, "reset #%d assert failed (%d)\n", i, ret);
-			return ret;
-		}
+	ret = reset_control_bulk_assert(ARRAY_SIZE(res->rst), res->rst);
+	if (ret < 0) {
+		dev_err(dev, "cannot assert resets\n");
+		return ret;
 	}
 
 	usleep_range(2000, 2500);
 
-	for (i = 0; i < ARRAY_SIZE(res->rst); i++) {
-		ret = reset_control_deassert(res->rst[i]);
-		if (ret) {
-			dev_err(dev, "reset #%d deassert failed (%d)\n", i,
-				ret);
-			return ret;
-		}
+	ret = reset_control_bulk_deassert(ARRAY_SIZE(res->rst), res->rst);
+	if (ret < 0) {
+		dev_err(dev, "cannot deassert resets\n");
+		return ret;
 	}
 
 	/*
@@ -1082,53 +786,20 @@ static int qcom_pcie_init_2_3_3(struct qcom_pcie *pcie)
 	 */
 	usleep_range(2000, 2500);
 
-	ret = clk_prepare_enable(res->iface);
-	if (ret) {
-		dev_err(dev, "cannot prepare/enable core clock\n");
-		goto err_clk_iface;
-	}
-
-	ret = clk_prepare_enable(res->axi_m_clk);
-	if (ret) {
-		dev_err(dev, "cannot prepare/enable core clock\n");
-		goto err_clk_axi_m;
-	}
-
-	ret = clk_prepare_enable(res->axi_s_clk);
-	if (ret) {
-		dev_err(dev, "cannot prepare/enable axi slave clock\n");
-		goto err_clk_axi_s;
-	}
-
-	ret = clk_prepare_enable(res->ahb_clk);
-	if (ret) {
-		dev_err(dev, "cannot prepare/enable ahb clock\n");
-		goto err_clk_ahb;
-	}
-
-	ret = clk_prepare_enable(res->aux_clk);
+	ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks);
 	if (ret) {
-		dev_err(dev, "cannot prepare/enable aux clock\n");
-		goto err_clk_aux;
+		dev_err(dev, "cannot prepare/enable clocks\n");
+		goto err_assert_resets;
 	}
 
 	return 0;
 
-err_clk_aux:
-	clk_disable_unprepare(res->ahb_clk);
-err_clk_ahb:
-	clk_disable_unprepare(res->axi_s_clk);
-err_clk_axi_s:
-	clk_disable_unprepare(res->axi_m_clk);
-err_clk_axi_m:
-	clk_disable_unprepare(res->iface);
-err_clk_iface:
+err_assert_resets:
 	/*
 	 * Not checking for failure, will anyway return
 	 * the original failure in 'ret'.
 	 */
-	for (i = 0; i < ARRAY_SIZE(res->rst); i++)
-		reset_control_assert(res->rst[i]);
+	reset_control_bulk_assert(ARRAY_SIZE(res->rst), res->rst);
 
 	return ret;
 }
@@ -1140,22 +811,22 @@ static int qcom_pcie_post_init_2_3_3(struct qcom_pcie *pcie)
 	u32 val;
 
 	writel(SLV_ADDR_SPACE_SZ,
-		pcie->parf + PCIE20_v3_PARF_SLV_ADDR_SPACE_SIZE);
+		pcie->parf + PARF_SLV_ADDR_SPACE_SIZE_2_3_3);
 
-	val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
-	val &= ~BIT(0);
-	writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val &= ~PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
-	writel(0, pcie->parf + PCIE20_PARF_DBI_BASE_ADDR);
+	writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
 
 	writel(MST_WAKEUP_EN | SLV_WAKEUP_EN | MSTR_ACLK_CGC_DIS
 		| SLV_ACLK_CGC_DIS | CORE_CLK_CGC_DIS |
 		AUX_PWR_DET | L23_CLK_RMV_DIS | L1_CLK_RMV_DIS,
-		pcie->parf + PCIE20_PARF_SYS_CTRL);
-	writel(0, pcie->parf + PCIE20_PARF_Q2A_FLUSH);
+		pcie->parf + PARF_SYS_CTRL);
+	writel(0, pcie->parf + PARF_Q2A_FLUSH);
 
 	writel(PCI_COMMAND_MASTER, pci->dbi_base + PCI_COMMAND);
-	writel(DBI_RO_WR_EN, pci->dbi_base + PCIE20_MISC_CONTROL_1_REG);
+	writel(DBI_RO_WR_EN, pci->dbi_base + MISC_CONTROL_1_REG);
 	writel(PCIE_CAP_SLOT_VAL, pci->dbi_base + offset + PCI_EXP_SLTCAP);
 
 	val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP);
@@ -1177,9 +848,9 @@ static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie)
 	unsigned int idx;
 	int ret;
 
-	res->pci_reset = devm_reset_control_get_exclusive(dev, "pci");
-	if (IS_ERR(res->pci_reset))
-		return PTR_ERR(res->pci_reset);
+	res->rst = devm_reset_control_array_get_exclusive(dev);
+	if (IS_ERR(res->rst))
+		return PTR_ERR(res->rst);
 
 	res->supplies[0].supply = "vdda";
 	res->supplies[1].supply = "vddpe-3v3";
@@ -1205,9 +876,12 @@ static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie)
 	res->clks[idx++].id = "ddrss_sf_tbu";
 	res->clks[idx++].id = "aggre0";
 	res->clks[idx++].id = "aggre1";
+	res->clks[idx++].id = "noc_aggr";
 	res->clks[idx++].id = "noc_aggr_4";
 	res->clks[idx++].id = "noc_aggr_south_sf";
 	res->clks[idx++].id = "cnoc_qx";
+	res->clks[idx++].id = "sleep";
+	res->clks[idx++].id = "cnoc_sf_axi";
 
 	num_opt_clks = idx - num_clks;
 	res->num_clks = idx;
@@ -1237,17 +911,17 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie)
 	if (ret < 0)
 		goto err_disable_regulators;
 
-	ret = reset_control_assert(res->pci_reset);
-	if (ret < 0) {
-		dev_err(dev, "cannot assert pci reset\n");
+	ret = reset_control_assert(res->rst);
+	if (ret) {
+		dev_err(dev, "reset assert failed (%d)\n", ret);
 		goto err_disable_clocks;
 	}
 
 	usleep_range(1000, 1500);
 
-	ret = reset_control_deassert(res->pci_reset);
-	if (ret < 0) {
-		dev_err(dev, "cannot deassert pci reset\n");
+	ret = reset_control_deassert(res->rst);
+	if (ret) {
+		dev_err(dev, "reset deassert failed (%d)\n", ret);
 		goto err_disable_clocks;
 	}
 
@@ -1255,35 +929,33 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie)
 	usleep_range(1000, 1500);
 
 	/* configure PCIe to RC mode */
-	writel(DEVICE_TYPE_RC, pcie->parf + PCIE20_PARF_DEVICE_TYPE);
+	writel(DEVICE_TYPE_RC, pcie->parf + PARF_DEVICE_TYPE);
 
 	/* enable PCIe clocks and resets */
-	val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
-	val &= ~BIT(0);
-	writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val &= ~PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
 	/* change DBI base address */
-	writel(0, pcie->parf + PCIE20_PARF_DBI_BASE_ADDR);
+	writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
 
 	/* MAC PHY_POWERDOWN MUX DISABLE  */
-	val = readl(pcie->parf + PCIE20_PARF_SYS_CTRL);
-	val &= ~BIT(29);
-	writel(val, pcie->parf + PCIE20_PARF_SYS_CTRL);
+	val = readl(pcie->parf + PARF_SYS_CTRL);
+	val &= ~MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN;
+	writel(val, pcie->parf + PARF_SYS_CTRL);
 
-	val = readl(pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL);
-	val |= BIT(4);
-	writel(val, pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL);
+	val = readl(pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
+	val |= BYPASS;
+	writel(val, pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
 
 	/* Enable L1 and L1SS */
-	val = readl(pcie->parf + PCIE20_PARF_PM_CTRL);
+	val = readl(pcie->parf + PARF_PM_CTRL);
 	val &= ~REQ_NOT_ENTR_L1;
-	writel(val, pcie->parf + PCIE20_PARF_PM_CTRL);
+	writel(val, pcie->parf + PARF_PM_CTRL);
 
-	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-		val = readl(pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT);
-		val |= BIT(31);
-		writel(val, pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT);
-	}
+	val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2);
+	val |= EN;
+	writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2);
 
 	return 0;
 err_disable_clocks:
@@ -1303,6 +975,76 @@ static void qcom_pcie_deinit_2_7_0(struct qcom_pcie *pcie)
 	regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
 }
 
+static int qcom_pcie_config_sid_1_9_0(struct qcom_pcie *pcie)
+{
+	/* iommu map structure */
+	struct {
+		u32 bdf;
+		u32 phandle;
+		u32 smmu_sid;
+		u32 smmu_sid_len;
+	} *map;
+	void __iomem *bdf_to_sid_base = pcie->parf + PARF_BDF_TO_SID_TABLE_N;
+	struct device *dev = pcie->pci->dev;
+	u8 qcom_pcie_crc8_table[CRC8_TABLE_SIZE];
+	int i, nr_map, size = 0;
+	u32 smmu_sid_base;
+
+	of_get_property(dev->of_node, "iommu-map", &size);
+	if (!size)
+		return 0;
+
+	map = kzalloc(size, GFP_KERNEL);
+	if (!map)
+		return -ENOMEM;
+
+	of_property_read_u32_array(dev->of_node, "iommu-map", (u32 *)map,
+				   size / sizeof(u32));
+
+	nr_map = size / (sizeof(*map));
+
+	crc8_populate_msb(qcom_pcie_crc8_table, QCOM_PCIE_CRC8_POLYNOMIAL);
+
+	/* Registers need to be zero out first */
+	memset_io(bdf_to_sid_base, 0, CRC8_TABLE_SIZE * sizeof(u32));
+
+	/* Extract the SMMU SID base from the first entry of iommu-map */
+	smmu_sid_base = map[0].smmu_sid;
+
+	/* Look for an available entry to hold the mapping */
+	for (i = 0; i < nr_map; i++) {
+		__be16 bdf_be = cpu_to_be16(map[i].bdf);
+		u32 val;
+		u8 hash;
+
+		hash = crc8(qcom_pcie_crc8_table, (u8 *)&bdf_be, sizeof(bdf_be), 0);
+
+		val = readl(bdf_to_sid_base + hash * sizeof(u32));
+
+		/* If the register is already populated, look for next available entry */
+		while (val) {
+			u8 current_hash = hash++;
+			u8 next_mask = 0xff;
+
+			/* If NEXT field is NULL then update it with next hash */
+			if (!(val & next_mask)) {
+				val |= (u32)hash;
+				writel(val, bdf_to_sid_base + current_hash * sizeof(u32));
+			}
+
+			val = readl(bdf_to_sid_base + hash * sizeof(u32));
+		}
+
+		/* BDF [31:16] | SID [15:8] | NEXT [7:0] */
+		val = map[i].bdf << 16 | (map[i].smmu_sid - smmu_sid_base) << 8 | 0;
+		writel(val, bdf_to_sid_base + hash * sizeof(u32));
+	}
+
+	kfree(map);
+
+	return 0;
+}
+
 static int qcom_pcie_get_resources_2_9_0(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0;
@@ -1371,17 +1113,17 @@ static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie)
 	int i;
 
 	writel(SLV_ADDR_SPACE_SZ,
-		pcie->parf + PCIE20_v3_PARF_SLV_ADDR_SPACE_SIZE);
+		pcie->parf + PARF_SLV_ADDR_SPACE_SIZE);
 
-	val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
-	val &= ~BIT(0);
-	writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val &= ~PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
-	writel(0, pcie->parf + PCIE20_PARF_DBI_BASE_ADDR);
+	writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
 
-	writel(DEVICE_TYPE_RC, pcie->parf + PCIE20_PARF_DEVICE_TYPE);
+	writel(DEVICE_TYPE_RC, pcie->parf + PARF_DEVICE_TYPE);
 	writel(BYPASS | MSTR_AXI_CLK_EN | AHB_CLK_EN,
-		pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL);
+		pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
 	writel(GEN3_RELATED_OFF_RXEQ_RGRDLESS_RXTS |
 		GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL,
 		pci->dbi_base + GEN3_RELATED_OFF);
@@ -1389,9 +1131,9 @@ static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie)
 	writel(MST_WAKEUP_EN | SLV_WAKEUP_EN | MSTR_ACLK_CGC_DIS |
 		SLV_ACLK_CGC_DIS | CORE_CLK_CGC_DIS |
 		AUX_PWR_DET | L23_CLK_RMV_DIS | L1_CLK_RMV_DIS,
-		pcie->parf + PCIE20_PARF_SYS_CTRL);
+		pcie->parf + PARF_SYS_CTRL);
 
-	writel(0, pcie->parf + PCIE20_PARF_Q2A_FLUSH);
+	writel(0, pcie->parf + PARF_Q2A_FLUSH);
 
 	dw_pcie_dbi_ro_wr_en(pci);
 	writel(PCIE_CAP_SLOT_VAL, pci->dbi_base + offset + PCI_EXP_SLTCAP);
@@ -1404,7 +1146,7 @@ static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie)
 			PCI_EXP_DEVCTL2);
 
 	for (i = 0; i < 256; i++)
-		writel(0, pcie->parf + PCIE20_PARF_BDF_TO_SID_TABLE_N + (4 * i));
+		writel(0, pcie->parf + PARF_BDF_TO_SID_TABLE_N + (4 * i));
 
 	return 0;
 }
@@ -1417,77 +1159,6 @@ static int qcom_pcie_link_up(struct dw_pcie *pci)
 	return !!(val & PCI_EXP_LNKSTA_DLLLA);
 }
 
-static int qcom_pcie_config_sid_sm8250(struct qcom_pcie *pcie)
-{
-	/* iommu map structure */
-	struct {
-		u32 bdf;
-		u32 phandle;
-		u32 smmu_sid;
-		u32 smmu_sid_len;
-	} *map;
-	void __iomem *bdf_to_sid_base = pcie->parf + PCIE20_PARF_BDF_TO_SID_TABLE_N;
-	struct device *dev = pcie->pci->dev;
-	u8 qcom_pcie_crc8_table[CRC8_TABLE_SIZE];
-	int i, nr_map, size = 0;
-	u32 smmu_sid_base;
-
-	of_get_property(dev->of_node, "iommu-map", &size);
-	if (!size)
-		return 0;
-
-	map = kzalloc(size, GFP_KERNEL);
-	if (!map)
-		return -ENOMEM;
-
-	of_property_read_u32_array(dev->of_node,
-		"iommu-map", (u32 *)map, size / sizeof(u32));
-
-	nr_map = size / (sizeof(*map));
-
-	crc8_populate_msb(qcom_pcie_crc8_table, QCOM_PCIE_CRC8_POLYNOMIAL);
-
-	/* Registers need to be zero out first */
-	memset_io(bdf_to_sid_base, 0, CRC8_TABLE_SIZE * sizeof(u32));
-
-	/* Extract the SMMU SID base from the first entry of iommu-map */
-	smmu_sid_base = map[0].smmu_sid;
-
-	/* Look for an available entry to hold the mapping */
-	for (i = 0; i < nr_map; i++) {
-		__be16 bdf_be = cpu_to_be16(map[i].bdf);
-		u32 val;
-		u8 hash;
-
-		hash = crc8(qcom_pcie_crc8_table, (u8 *)&bdf_be, sizeof(bdf_be),
-			0);
-
-		val = readl(bdf_to_sid_base + hash * sizeof(u32));
-
-		/* If the register is already populated, look for next available entry */
-		while (val) {
-			u8 current_hash = hash++;
-			u8 next_mask = 0xff;
-
-			/* If NEXT field is NULL then update it with next hash */
-			if (!(val & next_mask)) {
-				val |= (u32)hash;
-				writel(val, bdf_to_sid_base + current_hash * sizeof(u32));
-			}
-
-			val = readl(bdf_to_sid_base + hash * sizeof(u32));
-		}
-
-		/* BDF [31:16] | SID [15:8] | NEXT [7:0] */
-		val = map[i].bdf << 16 | (map[i].smmu_sid - smmu_sid_base) << 8 | 0;
-		writel(val, bdf_to_sid_base + hash * sizeof(u32));
-	}
-
-	kfree(map);
-
-	return 0;
-}
-
 static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
@@ -1608,7 +1279,7 @@ static const struct qcom_pcie_ops ops_1_9_0 = {
 	.init = qcom_pcie_init_2_7_0,
 	.deinit = qcom_pcie_deinit_2_7_0,
 	.ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
-	.config_sid = qcom_pcie_config_sid_sm8250,
+	.config_sid = qcom_pcie_config_sid_1_9_0,
 };
 
 /* Qcom IP rev.: 2.9.0  Synopsys IP rev.: 5.00a */
@@ -1725,13 +1396,51 @@ static void qcom_pcie_icc_update(struct qcom_pcie *pcie)
 	}
 }
 
+static int qcom_pcie_link_transition_count(struct seq_file *s, void *data)
+{
+	struct qcom_pcie *pcie = (struct qcom_pcie *)dev_get_drvdata(s->private);
+
+	seq_printf(s, "L0s transition count: %u\n",
+		   readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L0S));
+
+	seq_printf(s, "L1 transition count: %u\n",
+		   readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L1));
+
+	seq_printf(s, "L1.1 transition count: %u\n",
+		   readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1));
+
+	seq_printf(s, "L1.2 transition count: %u\n",
+		   readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2));
+
+	seq_printf(s, "L2 transition count: %u\n",
+		   readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L2));
+
+	return 0;
+}
+
+static void qcom_pcie_init_debugfs(struct qcom_pcie *pcie)
+{
+	struct dw_pcie *pci = pcie->pci;
+	struct device *dev = pci->dev;
+	char *name;
+
+	name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
+	if (!name)
+		return;
+
+	pcie->debugfs = debugfs_create_dir(name, NULL);
+	debugfs_create_devm_seqfile(dev, "link_transition_count", pcie->debugfs,
+				    qcom_pcie_link_transition_count);
+}
+
 static int qcom_pcie_probe(struct platform_device *pdev)
 {
+	const struct qcom_pcie_cfg *pcie_cfg;
 	struct device *dev = &pdev->dev;
+	struct qcom_pcie *pcie;
 	struct dw_pcie_rp *pp;
+	struct resource *res;
 	struct dw_pcie *pci;
-	struct qcom_pcie *pcie;
-	const struct qcom_pcie_cfg *pcie_cfg;
 	int ret;
 
 	pcie_cfg = of_device_get_match_data(dev);
@@ -1779,6 +1488,16 @@ static int qcom_pcie_probe(struct platform_device *pdev)
 		goto err_pm_runtime_put;
 	}
 
+	/* MHI region is optional */
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mhi");
+	if (res) {
+		pcie->mhi = devm_ioremap_resource(dev, res);
+		if (IS_ERR(pcie->mhi)) {
+			ret = PTR_ERR(pcie->mhi);
+			goto err_pm_runtime_put;
+		}
+	}
+
 	pcie->phy = devm_phy_optional_get(dev, "pciephy");
 	if (IS_ERR(pcie->phy)) {
 		ret = PTR_ERR(pcie->phy);
@@ -1809,6 +1528,9 @@ static int qcom_pcie_probe(struct platform_device *pdev)
 
 	qcom_pcie_icc_update(pcie);
 
+	if (pcie->mhi)
+		qcom_pcie_init_debugfs(pcie);
+
 	return 0;
 
 err_phy_exit:
@@ -1820,6 +1542,62 @@ err_pm_runtime_put:
 	return ret;
 }
 
+static int qcom_pcie_suspend_noirq(struct device *dev)
+{
+	struct qcom_pcie *pcie = dev_get_drvdata(dev);
+	int ret;
+
+	/*
+	 * Set minimum bandwidth required to keep data path functional during
+	 * suspend.
+	 */
+	ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1));
+	if (ret) {
+		dev_err(dev, "Failed to set interconnect bandwidth: %d\n", ret);
+		return ret;
+	}
+
+	/*
+	 * Turn OFF the resources only for controllers without active PCIe
+	 * devices. For controllers with active devices, the resources are kept
+	 * ON and the link is expected to be in L0/L1 (sub)states.
+	 *
+	 * Turning OFF the resources for controllers with active PCIe devices
+	 * will trigger access violation during the end of the suspend cycle,
+	 * as kernel tries to access the PCIe devices config space for masking
+	 * MSIs.
+	 *
+	 * Also, it is not desirable to put the link into L2/L3 state as that
+	 * implies VDD supply will be removed and the devices may go into
+	 * powerdown state. This will affect the lifetime of the storage devices
+	 * like NVMe.
+	 */
+	if (!dw_pcie_link_up(pcie->pci)) {
+		qcom_pcie_host_deinit(&pcie->pci->pp);
+		pcie->suspended = true;
+	}
+
+	return 0;
+}
+
+static int qcom_pcie_resume_noirq(struct device *dev)
+{
+	struct qcom_pcie *pcie = dev_get_drvdata(dev);
+	int ret;
+
+	if (pcie->suspended) {
+		ret = qcom_pcie_host_init(&pcie->pci->pp);
+		if (ret)
+			return ret;
+
+		pcie->suspended = false;
+	}
+
+	qcom_pcie_icc_update(pcie);
+
+	return 0;
+}
+
 static const struct of_device_id qcom_pcie_match[] = {
 	{ .compatible = "qcom,pcie-apq8064", .data = &cfg_2_1_0 },
 	{ .compatible = "qcom,pcie-apq8084", .data = &cfg_1_0_0 },
@@ -1836,11 +1614,13 @@ static const struct of_device_id qcom_pcie_match[] = {
 	{ .compatible = "qcom,pcie-sc8180x", .data = &cfg_1_9_0 },
 	{ .compatible = "qcom,pcie-sc8280xp", .data = &cfg_1_9_0 },
 	{ .compatible = "qcom,pcie-sdm845", .data = &cfg_2_7_0 },
+	{ .compatible = "qcom,pcie-sdx55", .data = &cfg_1_9_0 },
 	{ .compatible = "qcom,pcie-sm8150", .data = &cfg_1_9_0 },
 	{ .compatible = "qcom,pcie-sm8250", .data = &cfg_1_9_0 },
 	{ .compatible = "qcom,pcie-sm8350", .data = &cfg_1_9_0 },
 	{ .compatible = "qcom,pcie-sm8450-pcie0", .data = &cfg_1_9_0 },
 	{ .compatible = "qcom,pcie-sm8450-pcie1", .data = &cfg_1_9_0 },
+	{ .compatible = "qcom,pcie-sm8550", .data = &cfg_1_9_0 },
 	{ }
 };
 
@@ -1856,12 +1636,18 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0302, qcom_fixup_class);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1000, qcom_fixup_class);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class);
 
+static const struct dev_pm_ops qcom_pcie_pm_ops = {
+	NOIRQ_SYSTEM_SLEEP_PM_OPS(qcom_pcie_suspend_noirq, qcom_pcie_resume_noirq)
+};
+
 static struct platform_driver qcom_pcie_driver = {
 	.probe = qcom_pcie_probe,
 	.driver = {
 		.name = "qcom-pcie",
 		.suppress_bind_attrs = true,
 		.of_match_table = qcom_pcie_match,
+		.pm = &qcom_pcie_pm_ops,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 builtin_platform_driver(qcom_pcie_driver);
diff --git a/drivers/pci/controller/mobiveil/Kconfig b/drivers/pci/controller/mobiveil/Kconfig
index 1d7a07ba9ccd..58ce034f701a 100644
--- a/drivers/pci/controller/mobiveil/Kconfig
+++ b/drivers/pci/controller/mobiveil/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
-menu "Mobiveil PCIe Core Support"
+menu "Mobiveil-based PCIe controllers"
 	depends on PCI
 
 config PCIE_MOBIVEIL
@@ -11,6 +11,15 @@ config PCIE_MOBIVEIL_HOST
 	depends on PCI_MSI
 	select PCIE_MOBIVEIL
 
+config PCIE_LAYERSCAPE_GEN4
+	bool "Freescale Layerscape Gen4 PCIe controller"
+	depends on ARCH_LAYERSCAPE || COMPILE_TEST
+	depends on PCI_MSI
+	select PCIE_MOBIVEIL_HOST
+	help
+	  Say Y here if you want PCIe Gen4 controller support on
+	  Layerscape SoCs.
+
 config PCIE_MOBIVEIL_PLAT
 	bool "Mobiveil AXI PCIe controller"
 	depends on ARCH_ZYNQMP || COMPILE_TEST
@@ -22,12 +31,4 @@ config PCIE_MOBIVEIL_PLAT
 	  Soft IP. It has up to 8 outbound and inbound windows
 	  for address translation and it is a PCIe Gen4 IP.
 
-config PCIE_LAYERSCAPE_GEN4
-	bool "Freescale Layerscape PCIe Gen4 controller"
-	depends on ARCH_LAYERSCAPE || COMPILE_TEST
-	depends on PCI_MSI
-	select PCIE_MOBIVEIL_HOST
-	help
-	  Say Y here if you want PCIe Gen4 controller support on
-	  Layerscape SoCs.
 endmenu
diff --git a/drivers/pci/controller/pci-ixp4xx.c b/drivers/pci/controller/pci-ixp4xx.c
index 654ac4a82beb..e44252db6085 100644
--- a/drivers/pci/controller/pci-ixp4xx.c
+++ b/drivers/pci/controller/pci-ixp4xx.c
@@ -26,6 +26,7 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/bits.h>
+#include "../pci.h"
 
 /* Register offsets */
 #define IXP4XX_PCI_NP_AD		0x00
@@ -188,12 +189,13 @@ static u32 ixp4xx_config_addr(u8 bus_num, u16 devfn, int where)
 	/* Root bus is always 0 in this hardware */
 	if (bus_num == 0) {
 		/* type 0 */
-		return BIT(32-PCI_SLOT(devfn)) | ((PCI_FUNC(devfn)) << 8) |
-			(where & ~3);
+		return (PCI_CONF1_ADDRESS(0, 0, PCI_FUNC(devfn), where) &
+			~PCI_CONF1_ENABLE) | BIT(32-PCI_SLOT(devfn));
 	} else {
 		/* type 1 */
-		return (bus_num << 16) | ((PCI_SLOT(devfn)) << 11) |
-			((PCI_FUNC(devfn)) << 8) | (where & ~3) | 1;
+		return (PCI_CONF1_ADDRESS(bus_num, PCI_SLOT(devfn),
+					  PCI_FUNC(devfn), where) &
+			~PCI_CONF1_ENABLE) | 1;
 	}
 }
 
diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c
index 74c109f14ff0..79630885b9c8 100644
--- a/drivers/pci/controller/pci-tegra.c
+++ b/drivers/pci/controller/pci-tegra.c
@@ -1375,7 +1375,7 @@ static int tegra_pcie_phys_get(struct tegra_pcie *pcie)
 	struct tegra_pcie_port *port;
 	int err;
 
-	if (!soc->has_gen2 || of_find_property(np, "phys", NULL) != NULL)
+	if (!soc->has_gen2 || of_property_present(np, "phys"))
 		return tegra_pcie_phys_get_legacy(pcie);
 
 	list_for_each_entry(port, &pcie->ports, list) {
@@ -1944,7 +1944,7 @@ static bool of_regulator_bulk_available(struct device_node *np,
 	for (i = 0; i < num_supplies; i++) {
 		snprintf(property, 32, "%s-supply", supplies[i].supply);
 
-		if (of_find_property(np, property, NULL) == NULL)
+		if (!of_property_present(np, property))
 			return false;
 	}
 
diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
index ae5ad05ddc1d..31de7a29192c 100644
--- a/drivers/pci/controller/pcie-mediatek.c
+++ b/drivers/pci/controller/pcie-mediatek.c
@@ -643,7 +643,7 @@ static int mtk_pcie_setup_irq(struct mtk_pcie_port *port,
 		return err;
 	}
 
-	if (of_find_property(dev->of_node, "interrupt-names", NULL))
+	if (of_property_present(dev->of_node, "interrupt-names"))
 		port->irq = platform_get_irq_byname(pdev, "pcie_irq");
 	else
 		port->irq = platform_get_irq(pdev, port->slot);
diff --git a/drivers/pci/controller/pcie-mt7621.c b/drivers/pci/controller/pcie-mt7621.c
index 63a5f4463a9f..a445ec314375 100644
--- a/drivers/pci/controller/pcie-mt7621.c
+++ b/drivers/pci/controller/pcie-mt7621.c
@@ -378,8 +378,8 @@ static int mt7621_pcie_init_ports(struct mt7621_pcie *pcie)
 		u32 slot = port->slot;
 
 		if (!mt7621_pcie_port_is_linkup(port)) {
-			dev_err(dev, "pcie%d no card, disable it (RST & CLK)\n",
-				slot);
+			dev_info(dev, "pcie%d no card, disable it (RST & CLK)\n",
+				 slot);
 			mt7621_control_assert(port);
 			port->enabled = false;
 			num_disabled++;
diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c
index e4faf90feaf5..e80e56b2a842 100644
--- a/drivers/pci/controller/pcie-rcar-host.c
+++ b/drivers/pci/controller/pcie-rcar-host.c
@@ -219,9 +219,9 @@ static int rcar_pcie_config_access(struct rcar_pcie_host *host,
 
 	/* Enable the configuration access */
 	if (pci_is_root_bus(bus->parent))
-		rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR);
+		rcar_pci_write_reg(pcie, PCIECCTLR_CCIE | TYPE0, PCIECCTLR);
 	else
-		rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR);
+		rcar_pci_write_reg(pcie, PCIECCTLR_CCIE | TYPE1, PCIECCTLR);
 
 	/* Check for errors */
 	if (rcar_pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST)
diff --git a/drivers/pci/controller/pcie-rcar.h b/drivers/pci/controller/pcie-rcar.h
index 9bb125db85c6..6799f81392fc 100644
--- a/drivers/pci/controller/pcie-rcar.h
+++ b/drivers/pci/controller/pcie-rcar.h
@@ -11,7 +11,7 @@
 
 #define PCIECAR			0x000010
 #define PCIECCTLR		0x000018
-#define  CONFIG_SEND_ENABLE	BIT(31)
+#define  PCIECCTLR_CCIE		BIT(31)
 #define  TYPE0			(0 << 8)
 #define  TYPE1			BIT(8)
 #define PCIECDR			0x000020
diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c
index 9440d9811eea..46c9a5c3ca14 100644
--- a/drivers/pci/endpoint/pci-epc-core.c
+++ b/drivers/pci/endpoint/pci-epc-core.c
@@ -860,7 +860,7 @@ EXPORT_SYMBOL_GPL(__devm_pci_epc_create);
 
 static int __init pci_epc_init(void)
 {
-	pci_epc_class = class_create(THIS_MODULE, "pci_epc");
+	pci_epc_class = class_create("pci_epc");
 	if (IS_ERR(pci_epc_class)) {
 		pr_err("failed to create pci epc class --> %ld\n",
 		       PTR_ERR(pci_epc_class));
diff --git a/drivers/pci/hotplug/pciehp_pci.c b/drivers/pci/hotplug/pciehp_pci.c
index d17f3bf36f70..ad12515a4a12 100644
--- a/drivers/pci/hotplug/pciehp_pci.c
+++ b/drivers/pci/hotplug/pciehp_pci.c
@@ -63,7 +63,14 @@ int pciehp_configure_device(struct controller *ctrl)
 
 	pci_assign_unassigned_bridge_resources(bridge);
 	pcie_bus_configure_settings(parent);
+
+	/*
+	 * Release reset_lock during driver binding
+	 * to avoid AB-BA deadlock with device_lock.
+	 */
+	up_read(&ctrl->reset_lock);
 	pci_bus_add_devices(parent);
+	down_read_nested(&ctrl->reset_lock, ctrl->depth);
 
  out:
 	pci_unlock_rescan_remove();
@@ -104,7 +111,15 @@ void pciehp_unconfigure_device(struct controller *ctrl, bool presence)
 	list_for_each_entry_safe_reverse(dev, temp, &parent->devices,
 					 bus_list) {
 		pci_dev_get(dev);
+
+		/*
+		 * Release reset_lock during driver unbinding
+		 * to avoid AB-BA deadlock with device_lock.
+		 */
+		up_read(&ctrl->reset_lock);
 		pci_stop_and_remove_bus_device(dev);
+		down_read_nested(&ctrl->reset_lock, ctrl->depth);
+
 		/*
 		 * Ensure that no new Requests will be generated from
 		 * the device.
diff --git a/drivers/pci/hotplug/rpaphp_core.c b/drivers/pci/hotplug/rpaphp_core.c
index 491986197c47..2316de0fd198 100644
--- a/drivers/pci/hotplug/rpaphp_core.c
+++ b/drivers/pci/hotplug/rpaphp_core.c
@@ -278,7 +278,7 @@ int rpaphp_check_drc_props(struct device_node *dn, char *drc_name,
 		return -EINVAL;
 	}
 
-	if (of_find_property(dn->parent, "ibm,drc-info", NULL))
+	if (of_property_present(dn->parent, "ibm,drc-info"))
 		return rpaphp_check_drc_props_v2(dn, drc_name, drc_type,
 						be32_to_cpu(*my_index));
 	else
@@ -440,7 +440,7 @@ int rpaphp_add_slot(struct device_node *dn)
 	if (!of_node_name_eq(dn, "pci"))
 		return 0;
 
-	if (of_find_property(dn, "ibm,drc-info", NULL))
+	if (of_property_present(dn, "ibm,drc-info"))
 		return rpaphp_drc_info_add_slot(dn);
 	else
 		return rpaphp_drc_add_slot(dn);
diff --git a/drivers/pci/hotplug/shpchp_sysfs.c b/drivers/pci/hotplug/shpchp_sysfs.c
index 64beed7a26be..01d47a42da04 100644
--- a/drivers/pci/hotplug/shpchp_sysfs.c
+++ b/drivers/pci/hotplug/shpchp_sysfs.c
@@ -24,16 +24,16 @@
 static ssize_t show_ctrl(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct pci_dev *pdev;
-	int index, busnr;
 	struct resource *res;
 	struct pci_bus *bus;
 	size_t len = 0;
+	int busnr;
 
 	pdev = to_pci_dev(dev);
 	bus = pdev->subordinate;
 
 	len += sysfs_emit_at(buf, len, "Free resources: memory\n");
-	pci_bus_for_each_resource(bus, res, index) {
+	pci_bus_for_each_resource(bus, res) {
 		if (res && (res->flags & IORESOURCE_MEM) &&
 				!(res->flags & IORESOURCE_PREFETCH)) {
 			len += sysfs_emit_at(buf, len,
@@ -43,7 +43,7 @@ static ssize_t show_ctrl(struct device *dev, struct device_attribute *attr, char
 		}
 	}
 	len += sysfs_emit_at(buf, len, "Free resources: prefetchable memory\n");
-	pci_bus_for_each_resource(bus, res, index) {
+	pci_bus_for_each_resource(bus, res) {
 		if (res && (res->flags & IORESOURCE_MEM) &&
 			       (res->flags & IORESOURCE_PREFETCH)) {
 			len += sysfs_emit_at(buf, len,
@@ -53,7 +53,7 @@ static ssize_t show_ctrl(struct device *dev, struct device_attribute *attr, char
 		}
 	}
 	len += sysfs_emit_at(buf, len, "Free resources: IO\n");
-	pci_bus_for_each_resource(bus, res, index) {
+	pci_bus_for_each_resource(bus, res) {
 		if (res && (res->flags & IORESOURCE_IO)) {
 			len += sysfs_emit_at(buf, len,
 					     "start = %8.8llx, length = %8.8llx\n",
diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c
index 1f716624ca56..ef1d8857a51b 100644
--- a/drivers/pci/msi/msi.c
+++ b/drivers/pci/msi/msi.c
@@ -750,8 +750,7 @@ out_disable:
 	return ret;
 }
 
-static bool pci_msix_validate_entries(struct pci_dev *dev, struct msix_entry *entries,
-				      int nvec, int hwsize)
+static bool pci_msix_validate_entries(struct pci_dev *dev, struct msix_entry *entries, int nvec)
 {
 	bool nogap;
 	int i, j;
@@ -762,10 +761,6 @@ static bool pci_msix_validate_entries(struct pci_dev *dev, struct msix_entry *en
 	nogap = pci_msi_domain_supports(dev, MSI_FLAG_MSIX_CONTIGUOUS, DENY_LEGACY);
 
 	for (i = 0; i < nvec; i++) {
-		/* Entry within hardware limit? */
-		if (entries[i].entry >= hwsize)
-			return false;
-
 		/* Check for duplicate entries */
 		for (j = i + 1; j < nvec; j++) {
 			if (entries[i].entry == entries[j].entry)
@@ -805,7 +800,7 @@ int __pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, int
 	if (hwsize < 0)
 		return hwsize;
 
-	if (!pci_msix_validate_entries(dev, entries, nvec, hwsize))
+	if (!pci_msix_validate_entries(dev, entries, nvec))
 		return -EINVAL;
 
 	if (hwsize < nvec) {
diff --git a/drivers/pci/of.c b/drivers/pci/of.c
index 196834ed44fe..2c25f4fa0225 100644
--- a/drivers/pci/of.c
+++ b/drivers/pci/of.c
@@ -16,14 +16,32 @@
 #include "pci.h"
 
 #ifdef CONFIG_PCI
-void pci_set_of_node(struct pci_dev *dev)
+/**
+ * pci_set_of_node - Find and set device's DT device_node
+ * @dev: the PCI device structure to fill
+ *
+ * Returns 0 on success with of_node set or when no device is described in the
+ * DT. Returns -ENODEV if the device is present, but disabled in the DT.
+ */
+int pci_set_of_node(struct pci_dev *dev)
 {
+	struct device_node *node;
+
 	if (!dev->bus->dev.of_node)
-		return;
-	dev->dev.of_node = of_pci_find_child_device(dev->bus->dev.of_node,
-						    dev->devfn);
-	if (dev->dev.of_node)
-		dev->dev.fwnode = &dev->dev.of_node->fwnode;
+		return 0;
+
+	node = of_pci_find_child_device(dev->bus->dev.of_node, dev->devfn);
+	if (!node)
+		return 0;
+
+	if (!of_device_is_available(node)) {
+		of_node_put(node);
+		return -ENODEV;
+	}
+
+	dev->dev.of_node = node;
+	dev->dev.fwnode = &node->fwnode;
+	return 0;
 }
 
 void pci_release_of_node(struct pci_dev *dev)
@@ -447,7 +465,7 @@ static int of_irq_parse_pci(const struct pci_dev *pdev, struct of_phandle_args *
 		return -ENODEV;
 
 	/* Local interrupt-map in the device node? Use it! */
-	if (of_get_property(dn, "interrupt-map", NULL)) {
+	if (of_property_present(dn, "interrupt-map")) {
 		pin = pci_swizzle_interrupt_pin(pdev, pin);
 		ppnode = dn;
 	}
diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index 9e8205572830..6cd98ffca198 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -746,8 +746,7 @@ EXPORT_SYMBOL_GPL(pci_has_p2pmem);
 
 /**
  * pci_p2pmem_find_many - find a peer-to-peer DMA memory device compatible with
- *	the specified list of clients and shortest distance (as determined
- *	by pci_p2pmem_dma())
+ *	the specified list of clients and shortest distance
  * @clients: array of devices to check (NULL-terminated)
  * @num_clients: number of client devices in the list
  *
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c
index 57ddcc59af30..ae9baf801681 100644
--- a/drivers/pci/pci-driver.c
+++ b/drivers/pci/pci-driver.c
@@ -572,7 +572,8 @@ static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
 
 static void pci_pm_bridge_power_up_actions(struct pci_dev *pci_dev)
 {
-	pci_bridge_wait_for_secondary_bus(pci_dev, "resume", PCI_RESET_WAIT);
+	pci_bridge_wait_for_secondary_bus(pci_dev, "resume");
+
 	/*
 	 * When powering on a bridge from D3cold, the whole hierarchy may be
 	 * powered on into D0uninitialized state, resume them to give them a
diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index dd0d9d9bc509..ab32a91f287b 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -428,7 +428,7 @@ static ssize_t msi_bus_store(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR_RW(msi_bus);
 
-static ssize_t rescan_store(struct bus_type *bus, const char *buf, size_t count)
+static ssize_t rescan_store(const struct bus_type *bus, const char *buf, size_t count)
 {
 	unsigned long val;
 	struct pci_bus *b = NULL;
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 7a67611dc5f4..5ede93222bc1 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -64,6 +64,14 @@ struct pci_pme_device {
 
 #define PME_TIMEOUT 1000 /* How long between PME checks */
 
+/*
+ * Devices may extend the 1 sec period through Request Retry Status
+ * completions (PCIe r6.0 sec 2.3.1).  The spec does not provide an upper
+ * limit, but 60 sec ought to be enough for any device to become
+ * responsive.
+ */
+#define PCIE_RESET_READY_POLL_MS 60000 /* msec */
+
 static void pci_dev_d3_sleep(struct pci_dev *dev)
 {
 	unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay);
@@ -779,9 +787,8 @@ struct resource *pci_find_parent_resource(const struct pci_dev *dev,
 {
 	const struct pci_bus *bus = dev->bus;
 	struct resource *r;
-	int i;
 
-	pci_bus_for_each_resource(bus, r, i) {
+	pci_bus_for_each_resource(bus, r) {
 		if (!r)
 			continue;
 		if (resource_contains(r, res)) {
@@ -4939,7 +4946,6 @@ static int pci_bus_max_d3cold_delay(const struct pci_bus *bus)
  * pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible
  * @dev: PCI bridge
  * @reset_type: reset type in human-readable form
- * @timeout: maximum time to wait for devices on secondary bus (milliseconds)
  *
  * Handle necessary delays before access to the devices on the secondary
  * side of the bridge are permitted after D3cold to D0 transition
@@ -4952,8 +4958,7 @@ static int pci_bus_max_d3cold_delay(const struct pci_bus *bus)
  * Return 0 on success or -ENOTTY if the first device on the secondary bus
  * failed to become accessible.
  */
-int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type,
-				      int timeout)
+int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type)
 {
 	struct pci_dev *child;
 	int delay;
@@ -5031,7 +5036,8 @@ int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type,
 		}
 	}
 
-	return pci_dev_wait(child, reset_type, timeout - delay);
+	return pci_dev_wait(child, reset_type,
+			    PCIE_RESET_READY_POLL_MS - delay);
 }
 
 void pci_reset_secondary_bus(struct pci_dev *dev)
@@ -5068,8 +5074,7 @@ int pci_bridge_secondary_bus_reset(struct pci_dev *dev)
 {
 	pcibios_reset_secondary_bus(dev);
 
-	return pci_bridge_wait_for_secondary_bus(dev, "bus reset",
-						 PCIE_RESET_READY_POLL_MS);
+	return pci_bridge_wait_for_secondary_bus(dev, "bus reset");
 }
 EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset);
 
@@ -6679,7 +6684,7 @@ void pci_reassigndev_resource_alignment(struct pci_dev *dev)
 	}
 }
 
-static ssize_t resource_alignment_show(struct bus_type *bus, char *buf)
+static ssize_t resource_alignment_show(const struct bus_type *bus, char *buf)
 {
 	size_t count = 0;
 
@@ -6691,7 +6696,7 @@ static ssize_t resource_alignment_show(struct bus_type *bus, char *buf)
 	return count;
 }
 
-static ssize_t resource_alignment_store(struct bus_type *bus,
+static ssize_t resource_alignment_store(const struct bus_type *bus,
 					const char *buf, size_t count)
 {
 	char *param, *old, *end;
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index d2c08670a20e..67005a0ee381 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -70,12 +70,6 @@ struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev,
  * Reset (PCIe r6.0 sec 5.8).
  */
 #define PCI_RESET_WAIT		1000	/* msec */
-/*
- * Devices may extend the 1 sec period through Request Retry Status completions
- * (PCIe r6.0 sec 2.3.1).  The spec does not provide an upper limit, but 60 sec
- * ought to be enough for any device to become responsive.
- */
-#define PCIE_RESET_READY_POLL_MS 60000	/* msec */
 
 void pci_update_current_state(struct pci_dev *dev, pci_power_t state);
 void pci_refresh_power_state(struct pci_dev *dev);
@@ -100,8 +94,7 @@ void pci_msix_init(struct pci_dev *dev);
 bool pci_bridge_d3_possible(struct pci_dev *dev);
 void pci_bridge_d3_update(struct pci_dev *dev);
 void pci_bridge_reconfigure_ltr(struct pci_dev *dev);
-int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type,
-				      int timeout);
+int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type);
 
 static inline void pci_wakeup_event(struct pci_dev *dev)
 {
@@ -624,7 +617,7 @@ int of_pci_get_max_link_speed(struct device_node *node);
 u32 of_pci_get_slot_power_limit(struct device_node *node,
 				u8 *slot_power_limit_value,
 				u8 *slot_power_limit_scale);
-void pci_set_of_node(struct pci_dev *dev);
+int pci_set_of_node(struct pci_dev *dev);
 void pci_release_of_node(struct pci_dev *dev);
 void pci_set_bus_of_node(struct pci_bus *bus);
 void pci_release_bus_of_node(struct pci_bus *bus);
@@ -662,7 +655,7 @@ of_pci_get_slot_power_limit(struct device_node *node,
 	return 0;
 }
 
-static inline void pci_set_of_node(struct pci_dev *dev) { }
+static inline int pci_set_of_node(struct pci_dev *dev) { return 0; }
 static inline void pci_release_of_node(struct pci_dev *dev) { }
 static inline void pci_set_bus_of_node(struct pci_bus *bus) { }
 static inline void pci_release_bus_of_node(struct pci_bus *bus) { }
diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c
index a5d7c69b764e..3ceed8e3de41 100644
--- a/drivers/pci/pcie/dpc.c
+++ b/drivers/pci/pcie/dpc.c
@@ -170,8 +170,7 @@ pci_ers_result_t dpc_reset_link(struct pci_dev *pdev)
 	pci_write_config_word(pdev, cap + PCI_EXP_DPC_STATUS,
 			      PCI_EXP_DPC_STATUS_TRIGGER);
 
-	if (pci_bridge_wait_for_secondary_bus(pdev, "DPC",
-					      PCIE_RESET_READY_POLL_MS)) {
+	if (pci_bridge_wait_for_secondary_bus(pdev, "DPC")) {
 		clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags);
 		ret = PCI_ERS_RESULT_DISCONNECT;
 	} else {
diff --git a/drivers/pci/pcie/edr.c b/drivers/pci/pcie/edr.c
index a6b9b479b97a..5f4914d313a1 100644
--- a/drivers/pci/pcie/edr.c
+++ b/drivers/pci/pcie/edr.c
@@ -151,9 +151,18 @@ static void edr_handle_event(acpi_handle handle, u32 event, void *data)
 	if (event != ACPI_NOTIFY_DISCONNECT_RECOVER)
 		return;
 
+	/*
+	 * pdev is a Root Port or Downstream Port that is still present and
+	 * has triggered a containment event, e.g., DPC, so its child
+	 * devices have been disconnected (ACPI r6.5, sec 5.6.6).
+	 */
 	pci_info(pdev, "EDR event received\n");
 
-	/* Locate the port which issued EDR event */
+	/*
+	 * Locate the port that experienced the containment event.  pdev
+	 * may be that port or a parent of it (PCI Firmware r3.3, sec
+	 * 4.6.13).
+	 */
 	edev = acpi_dpc_port_get(pdev);
 	if (!edev) {
 		pci_err(pdev, "Firmware failed to locate DPC port\n");
@@ -193,6 +202,7 @@ send_ost:
 	 */
 	if (estate == PCI_ERS_RESULT_RECOVERED) {
 		pci_dbg(edev, "DPC port successfully recovered\n");
+		pcie_clear_device_status(edev);
 		acpi_send_edr_status(pdev, edev, EDR_OST_SUCCESS);
 	} else {
 		pci_dbg(edev, "DPC port recovery failed\n");
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index a3f68b6ba6ac..9123bafab451 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -533,7 +533,7 @@ void pci_read_bridge_bases(struct pci_bus *child)
 	pci_read_bridge_mmio_pref(child);
 
 	if (dev->transparent) {
-		pci_bus_for_each_resource(child->parent, res, i) {
+		pci_bus_for_each_resource(child->parent, res) {
 			if (res && res->flags) {
 				pci_bus_add_resource(child, res,
 						     PCI_SUBTRACTIVE_DECODE);
@@ -1826,7 +1826,7 @@ int pci_setup_device(struct pci_dev *dev)
 	u32 class;
 	u16 cmd;
 	u8 hdr_type;
-	int pos = 0;
+	int err, pos = 0;
 	struct pci_bus_region region;
 	struct resource *res;
 
@@ -1840,10 +1840,10 @@ int pci_setup_device(struct pci_dev *dev)
 	dev->error_state = pci_channel_io_normal;
 	set_pcie_port_type(dev);
 
-	pci_set_of_node(dev);
+	err = pci_set_of_node(dev);
+	if (err)
+		return err;
 	pci_set_acpi_fwnode(dev);
-	if (dev->dev.fwnode && !fwnode_device_is_available(dev->dev.fwnode))
-		return -ENODEV;
 
 	pci_dev_assign_slot(dev);
 
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index 44cab813bf95..f4e2a88729fd 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -1940,6 +1940,19 @@ static void quirk_radeon_pm(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x6741, quirk_radeon_pm);
 
 /*
+ * NVIDIA Ampere-based HDA controllers can wedge the whole device if a bus
+ * reset is performed too soon after transition to D0, extend d3hot_delay
+ * to previous effective default for all NVIDIA HDA controllers.
+ */
+static void quirk_nvidia_hda_pm(struct pci_dev *dev)
+{
+	quirk_d3hot_delay(dev, 20);
+}
+DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
+			      PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8,
+			      quirk_nvidia_hda_pm);
+
+/*
  * Ryzen5/7 XHCI controllers fail upon resume from runtime suspend or s2idle.
  * https://bugzilla.kernel.org/show_bug.cgi?id=205587
  *
diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c
index 22d39e12b236..30a787d45d2e 100644
--- a/drivers/pci/remove.c
+++ b/drivers/pci/remove.c
@@ -5,10 +5,9 @@
 
 static void pci_free_resources(struct pci_dev *dev)
 {
-	int i;
+	struct resource *res;
 
-	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
-		struct resource *res = dev->resource + i;
+	pci_dev_for_each_resource(dev, res) {
 		if (res->parent)
 			release_resource(res);
 	}
diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c
index c690572b10ce..fdeb121e9175 100644
--- a/drivers/pci/setup-bus.c
+++ b/drivers/pci/setup-bus.c
@@ -124,20 +124,17 @@ static resource_size_t get_res_add_align(struct list_head *head,
 	return dev_res ? dev_res->min_align : 0;
 }
 
-
 /* Sort resources by alignment */
 static void pdev_sort_resources(struct pci_dev *dev, struct list_head *head)
 {
+	struct resource *r;
 	int i;
 
-	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
-		struct resource *r;
+	pci_dev_for_each_resource(dev, r, i) {
 		struct pci_dev_resource *dev_res, *tmp;
 		resource_size_t r_align;
 		struct list_head *n;
 
-		r = &dev->resource[i];
-
 		if (r->flags & IORESOURCE_PCI_FIXED)
 			continue;
 
@@ -773,9 +770,8 @@ static struct resource *find_bus_resource_of_type(struct pci_bus *bus,
 						  unsigned long type)
 {
 	struct resource *r, *r_assigned = NULL;
-	int i;
 
-	pci_bus_for_each_resource(bus, r, i) {
+	pci_bus_for_each_resource(bus, r) {
 		if (r == &ioport_resource || r == &iomem_resource)
 			continue;
 		if (r && (r->flags & type_mask) == type && !r->parent)
@@ -895,10 +891,9 @@ static void pbus_size_io(struct pci_bus *bus, resource_size_t min_size,
 
 	min_align = window_alignment(bus, IORESOURCE_IO);
 	list_for_each_entry(dev, &bus->devices, bus_list) {
-		int i;
+		struct resource *r;
 
-		for (i = 0; i < PCI_NUM_RESOURCES; i++) {
-			struct resource *r = &dev->resource[i];
+		pci_dev_for_each_resource(dev, r) {
 			unsigned long r_size;
 
 			if (r->parent || !(r->flags & IORESOURCE_IO))
@@ -1014,10 +1009,10 @@ static int pbus_size_mem(struct pci_bus *bus, unsigned long mask,
 	size = 0;
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct resource *r;
 		int i;
 
-		for (i = 0; i < PCI_NUM_RESOURCES; i++) {
-			struct resource *r = &dev->resource[i];
+		pci_dev_for_each_resource(dev, r, i) {
 			resource_size_t r_size;
 
 			if (r->parent || (r->flags & IORESOURCE_PCI_FIXED) ||
@@ -1208,7 +1203,7 @@ void __pci_bus_size_bridges(struct pci_bus *bus, struct list_head *realloc_head)
 			additional_mmio_pref_size = 0;
 	struct resource *pref;
 	struct pci_host_bridge *host;
-	int hdr_type, i, ret;
+	int hdr_type, ret;
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		struct pci_bus *b = dev->subordinate;
@@ -1232,7 +1227,7 @@ void __pci_bus_size_bridges(struct pci_bus *bus, struct list_head *realloc_head)
 		host = to_pci_host_bridge(bus->bridge);
 		if (!host->size_windows)
 			return;
-		pci_bus_for_each_resource(bus, pref, i)
+		pci_bus_for_each_resource(bus, pref)
 			if (pref && (pref->flags & IORESOURCE_PREFETCH))
 				break;
 		hdr_type = -1;	/* Intentionally invalid - not a PCI device. */
@@ -1337,12 +1332,11 @@ EXPORT_SYMBOL(pci_bus_size_bridges);
 
 static void assign_fixed_resource_on_bus(struct pci_bus *b, struct resource *r)
 {
-	int i;
 	struct resource *parent_r;
 	unsigned long mask = IORESOURCE_IO | IORESOURCE_MEM |
 			     IORESOURCE_PREFETCH;
 
-	pci_bus_for_each_resource(b, parent_r, i) {
+	pci_bus_for_each_resource(b, parent_r) {
 		if (!parent_r)
 			continue;
 
@@ -1358,11 +1352,10 @@ static void assign_fixed_resource_on_bus(struct pci_bus *b, struct resource *r)
  */
 static void pdev_assign_fixed_resources(struct pci_dev *dev)
 {
-	int i;
+	struct resource *r;
 
-	for (i = 0; i <  PCI_NUM_RESOURCES; i++) {
+	pci_dev_for_each_resource(dev, r) {
 		struct pci_bus *b;
-		struct resource *r = &dev->resource[i];
 
 		if (r->parent || !(r->flags & IORESOURCE_PCI_FIXED) ||
 		    !(r->flags & (IORESOURCE_IO | IORESOURCE_MEM)))
@@ -1795,11 +1788,9 @@ static void remove_dev_resources(struct pci_dev *dev, struct resource *io,
 				 struct resource *mmio,
 				 struct resource *mmio_pref)
 {
-	int i;
-
-	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
-		struct resource *res = &dev->resource[i];
+	struct resource *res;
 
+	pci_dev_for_each_resource(dev, res) {
 		if (resource_type(res) == IORESOURCE_IO) {
 			remove_dev_resource(io, dev, res);
 		} else if (resource_type(res) == IORESOURCE_MEM) {
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c
index b492e67c3d87..967f9a758923 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -484,12 +484,10 @@ int pci_enable_resources(struct pci_dev *dev, int mask)
 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
 	old_cmd = cmd;
 
-	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
+	pci_dev_for_each_resource(dev, r, i) {
 		if (!(mask & (1 << i)))
 			continue;
 
-		r = &dev->resource[i];
-
 		if (!(r->flags & (IORESOURCE_IO | IORESOURCE_MEM)))
 			continue;
 		if ((i == PCI_ROM_RESOURCE) &&
diff --git a/drivers/pci/switch/switchtec.c b/drivers/pci/switch/switchtec.c
index 3d6f17ff2429..d837da055921 100644
--- a/drivers/pci/switch/switchtec.c
+++ b/drivers/pci/switch/switchtec.c
@@ -1804,7 +1804,7 @@ static int __init switchtec_init(void)
 	if (rc)
 		return rc;
 
-	switchtec_class = class_create(THIS_MODULE, "switchtec");
+	switchtec_class = class_create("switchtec");
 	if (IS_ERR(switchtec_class)) {
 		rc = PTR_ERR(switchtec_class);
 		goto err_create_class;
diff --git a/drivers/pci/vgaarb.c b/drivers/pci/vgaarb.c
index f80b6ec88dc3..5a696078b382 100644
--- a/drivers/pci/vgaarb.c
+++ b/drivers/pci/vgaarb.c
@@ -548,10 +548,8 @@ static bool vga_is_firmware_default(struct pci_dev *pdev)
 #if defined(CONFIG_X86) || defined(CONFIG_IA64)
 	u64 base = screen_info.lfb_base;
 	u64 size = screen_info.lfb_size;
+	struct resource *r;
 	u64 limit;
-	resource_size_t start, end;
-	unsigned long flags;
-	int i;
 
 	/* Select the device owning the boot framebuffer if there is one */
 
@@ -561,19 +559,14 @@ static bool vga_is_firmware_default(struct pci_dev *pdev)
 	limit = base + size;
 
 	/* Does firmware framebuffer belong to us? */
-	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
-		flags = pci_resource_flags(pdev, i);
-
-		if ((flags & IORESOURCE_MEM) == 0)
+	pci_dev_for_each_resource(pdev, r) {
+		if (resource_type(r) != IORESOURCE_MEM)
 			continue;
 
-		start = pci_resource_start(pdev, i);
-		end  = pci_resource_end(pdev, i);
-
-		if (!start || !end)
+		if (!r->start || !r->end)
 			continue;
 
-		if (base < start || limit >= end)
+		if (base < r->start || limit >= r->end)
 			continue;
 
 		return true;
diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c
index fcd029ca2eb1..83c0ab50676d 100644
--- a/drivers/pci/xen-pcifront.c
+++ b/drivers/pci/xen-pcifront.c
@@ -390,9 +390,7 @@ static int pcifront_claim_resource(struct pci_dev *dev, void *data)
 	int i;
 	struct resource *r;
 
-	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
-		r = &dev->resource[i];
-
+	pci_dev_for_each_resource(dev, r, i) {
 		if (!r->parent && r->start && r->flags) {
 			dev_info(&pdev->xdev->dev, "claiming resource %s/%d\n",
 				pci_name(dev), i);
diff --git a/drivers/pcmcia/cs.c b/drivers/pcmcia/cs.c
index e3224e49c43f..5658745c398f 100644
--- a/drivers/pcmcia/cs.c
+++ b/drivers/pcmcia/cs.c
@@ -824,7 +824,7 @@ static int pcmcia_socket_uevent(const struct device *dev,
 
 static struct completion pcmcia_unload;
 
-static void pcmcia_release_socket_class(struct class *data)
+static void pcmcia_release_socket_class(const struct class *data)
 {
 	complete(&pcmcia_unload);
 }
diff --git a/drivers/pcmcia/ds.c b/drivers/pcmcia/ds.c
index c8087efa5e4a..d500e5dbbc3f 100644
--- a/drivers/pcmcia/ds.c
+++ b/drivers/pcmcia/ds.c
@@ -1335,8 +1335,7 @@ static struct pcmcia_callback pcmcia_bus_callback = {
 	.resume = pcmcia_bus_resume,
 };
 
-static int pcmcia_bus_add_socket(struct device *dev,
-					   struct class_interface *class_intf)
+static int pcmcia_bus_add_socket(struct device *dev)
 {
 	struct pcmcia_socket *socket = dev_get_drvdata(dev);
 	int ret;
@@ -1369,8 +1368,7 @@ static int pcmcia_bus_add_socket(struct device *dev,
 	return 0;
 }
 
-static void pcmcia_bus_remove_socket(struct device *dev,
-				     struct class_interface *class_intf)
+static void pcmcia_bus_remove_socket(struct device *dev)
 {
 	struct pcmcia_socket *socket = dev_get_drvdata(dev);
 
diff --git a/drivers/pcmcia/rsrc_nonstatic.c b/drivers/pcmcia/rsrc_nonstatic.c
index ad1141fddb4c..471e0c5815f3 100644
--- a/drivers/pcmcia/rsrc_nonstatic.c
+++ b/drivers/pcmcia/rsrc_nonstatic.c
@@ -1200,8 +1200,7 @@ static const struct attribute_group rsrc_attributes = {
 	.attrs = pccard_rsrc_attributes,
 };
 
-static int pccard_sysfs_add_rsrc(struct device *dev,
-					   struct class_interface *class_intf)
+static int pccard_sysfs_add_rsrc(struct device *dev)
 {
 	struct pcmcia_socket *s = dev_get_drvdata(dev);
 
@@ -1210,8 +1209,7 @@ static int pccard_sysfs_add_rsrc(struct device *dev,
 	return sysfs_create_group(&dev->kobj, &rsrc_attributes);
 }
 
-static void pccard_sysfs_remove_rsrc(struct device *dev,
-					       struct class_interface *class_intf)
+static void pccard_sysfs_remove_rsrc(struct device *dev)
 {
 	struct pcmcia_socket *s = dev_get_drvdata(dev);
 
diff --git a/drivers/peci/sysfs.c b/drivers/peci/sysfs.c
index db9ef05776e3..c04244075794 100644
--- a/drivers/peci/sysfs.c
+++ b/drivers/peci/sysfs.c
@@ -15,7 +15,7 @@ static int rescan_controller(struct device *dev, void *data)
 	return peci_controller_scan_devices(to_peci_controller(dev));
 }
 
-static ssize_t rescan_store(struct bus_type *bus, const char *buf, size_t count)
+static ssize_t rescan_store(const struct bus_type *bus, const char *buf, size_t count)
 {
 	bool res;
 	int ret;
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 66c259000a44..711f82400086 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -100,6 +100,16 @@ config ARM_SMMU_V3_PMU
 	   through the SMMU and allow the resulting information to be filtered
 	   based on the Stream ID of the corresponding master.
 
+config ARM_PMUV3
+	depends on HW_PERF_EVENTS && ((ARM && CPU_V7) || ARM64)
+	bool "ARM PMUv3 support" if !ARM64
+	default ARM64
+	  help
+	  Say y if you want to use the ARM performance monitor unit (PMU)
+	  version 3. The PMUv3 is the CPU performance monitors on ARMv8
+	  (aarch32 and aarch64) systems that implement the PMUv3
+	  architecture.
+
 config ARM_DSU_PMU
 	tristate "ARM DynamIQ Shared Unit (DSU) PMU"
 	depends on ARM64
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index 13e45da61100..dabc859540ce 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -5,6 +5,7 @@ obj-$(CONFIG_ARM_CMN) += arm-cmn.o
 obj-$(CONFIG_ARM_DSU_PMU) += arm_dsu_pmu.o
 obj-$(CONFIG_ARM_PMU) += arm_pmu.o arm_pmu_platform.o
 obj-$(CONFIG_ARM_PMU_ACPI) += arm_pmu_acpi.o
+obj-$(CONFIG_ARM_PMUV3) += arm_pmuv3.o
 obj-$(CONFIG_ARM_SMMU_V3_PMU) += arm_smmuv3_pmu.o
 obj-$(CONFIG_FSL_IMX8_DDR_PMU) += fsl_imx8_ddr_perf.o
 obj-$(CONFIG_HISI_PMU) += hisilicon/
diff --git a/drivers/perf/alibaba_uncore_drw_pmu.c b/drivers/perf/alibaba_uncore_drw_pmu.c
index a7689fecb49d..5c5be9fc1b15 100644
--- a/drivers/perf/alibaba_uncore_drw_pmu.c
+++ b/drivers/perf/alibaba_uncore_drw_pmu.c
@@ -656,8 +656,7 @@ static int ali_drw_pmu_probe(struct platform_device *pdev)
 	drw_pmu->dev = &pdev->dev;
 	platform_set_drvdata(pdev, drw_pmu);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	drw_pmu->cfg_base = devm_ioremap_resource(&pdev->dev, res);
+	drw_pmu->cfg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(drw_pmu->cfg_base))
 		return PTR_ERR(drw_pmu->cfg_base);
 
diff --git a/drivers/perf/amlogic/meson_ddr_pmu_core.c b/drivers/perf/amlogic/meson_ddr_pmu_core.c
index b84346dbac2c..0b24dee1ed3c 100644
--- a/drivers/perf/amlogic/meson_ddr_pmu_core.c
+++ b/drivers/perf/amlogic/meson_ddr_pmu_core.c
@@ -156,10 +156,14 @@ static int meson_ddr_perf_event_add(struct perf_event *event, int flags)
 	u64 config2 = event->attr.config2;
 	int i;
 
-	for_each_set_bit(i, (const unsigned long *)&config1, sizeof(config1))
+	for_each_set_bit(i,
+			 (const unsigned long *)&config1,
+			 BITS_PER_TYPE(config1))
 		meson_ddr_set_axi_filter(event, i);
 
-	for_each_set_bit(i, (const unsigned long *)&config2, sizeof(config2))
+	for_each_set_bit(i,
+			 (const unsigned long *)&config2,
+			 BITS_PER_TYPE(config2))
 		meson_ddr_set_axi_filter(event, i + 64);
 
 	if (flags & PERF_EF_START)
diff --git a/drivers/perf/amlogic/meson_g12_ddr_pmu.c b/drivers/perf/amlogic/meson_g12_ddr_pmu.c
index a78fdb15e26c..8b643888d503 100644
--- a/drivers/perf/amlogic/meson_g12_ddr_pmu.c
+++ b/drivers/perf/amlogic/meson_g12_ddr_pmu.c
@@ -21,23 +21,23 @@
 #define DMC_QOS_IRQ		BIT(30)
 
 /* DMC bandwidth monitor register address offset */
-#define DMC_MON_G12_CTRL0		(0x20  << 2)
-#define DMC_MON_G12_CTRL1		(0x21  << 2)
-#define DMC_MON_G12_CTRL2		(0x22  << 2)
-#define DMC_MON_G12_CTRL3		(0x23  << 2)
-#define DMC_MON_G12_CTRL4		(0x24  << 2)
-#define DMC_MON_G12_CTRL5		(0x25  << 2)
-#define DMC_MON_G12_CTRL6		(0x26  << 2)
-#define DMC_MON_G12_CTRL7		(0x27  << 2)
-#define DMC_MON_G12_CTRL8		(0x28  << 2)
-
-#define DMC_MON_G12_ALL_REQ_CNT		(0x29  << 2)
-#define DMC_MON_G12_ALL_GRANT_CNT	(0x2a  << 2)
-#define DMC_MON_G12_ONE_GRANT_CNT	(0x2b  << 2)
-#define DMC_MON_G12_SEC_GRANT_CNT	(0x2c  << 2)
-#define DMC_MON_G12_THD_GRANT_CNT	(0x2d  << 2)
-#define DMC_MON_G12_FOR_GRANT_CNT	(0x2e  << 2)
-#define DMC_MON_G12_TIMER		(0x2f  << 2)
+#define DMC_MON_G12_CTRL0		(0x0  << 2)
+#define DMC_MON_G12_CTRL1		(0x1  << 2)
+#define DMC_MON_G12_CTRL2		(0x2  << 2)
+#define DMC_MON_G12_CTRL3		(0x3  << 2)
+#define DMC_MON_G12_CTRL4		(0x4  << 2)
+#define DMC_MON_G12_CTRL5		(0x5  << 2)
+#define DMC_MON_G12_CTRL6		(0x6  << 2)
+#define DMC_MON_G12_CTRL7		(0x7  << 2)
+#define DMC_MON_G12_CTRL8		(0x8  << 2)
+
+#define DMC_MON_G12_ALL_REQ_CNT		(0x9  << 2)
+#define DMC_MON_G12_ALL_GRANT_CNT	(0xa  << 2)
+#define DMC_MON_G12_ONE_GRANT_CNT	(0xb  << 2)
+#define DMC_MON_G12_SEC_GRANT_CNT	(0xc  << 2)
+#define DMC_MON_G12_THD_GRANT_CNT	(0xd  << 2)
+#define DMC_MON_G12_FOR_GRANT_CNT	(0xe  << 2)
+#define DMC_MON_G12_TIMER		(0xf  << 2)
 
 /* Each bit represent a axi line */
 PMU_FORMAT_ATTR(event, "config:0-7");
diff --git a/drivers/perf/apple_m1_cpu_pmu.c b/drivers/perf/apple_m1_cpu_pmu.c
index 979a7c2b4f56..8574c6e58c83 100644
--- a/drivers/perf/apple_m1_cpu_pmu.c
+++ b/drivers/perf/apple_m1_cpu_pmu.c
@@ -559,7 +559,21 @@ static int m1_pmu_fire_init(struct arm_pmu *cpu_pmu)
 	return m1_pmu_init(cpu_pmu);
 }
 
+static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu)
+{
+	cpu_pmu->name = "apple_avalanche_pmu";
+	return m1_pmu_init(cpu_pmu);
+}
+
+static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu)
+{
+	cpu_pmu->name = "apple_blizzard_pmu";
+	return m1_pmu_init(cpu_pmu);
+}
+
 static const struct of_device_id m1_pmu_of_device_ids[] = {
+	{ .compatible = "apple,avalanche-pmu",	.data = m2_pmu_avalanche_init, },
+	{ .compatible = "apple,blizzard-pmu",	.data = m2_pmu_blizzard_init, },
 	{ .compatible = "apple,icestorm-pmu",	.data = m1_pmu_ice_init, },
 	{ .compatible = "apple,firestorm-pmu",	.data = m1_pmu_fire_init, },
 	{ },
@@ -581,4 +595,3 @@ static struct platform_driver m1_pmu_driver = {
 };
 
 module_platform_driver(m1_pmu_driver);
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c
index c9689861be3f..47d359f72957 100644
--- a/drivers/perf/arm-cmn.c
+++ b/drivers/perf/arm-cmn.c
@@ -57,14 +57,12 @@
 #define CMN_INFO_REQ_VC_NUM		GENMASK_ULL(1, 0)
 
 /* XPs also have some local topology info which has uses too */
-#define CMN_MXP__CONNECT_INFO_P0	0x0008
-#define CMN_MXP__CONNECT_INFO_P1	0x0010
-#define CMN_MXP__CONNECT_INFO_P2	0x0028
-#define CMN_MXP__CONNECT_INFO_P3	0x0030
-#define CMN_MXP__CONNECT_INFO_P4	0x0038
-#define CMN_MXP__CONNECT_INFO_P5	0x0040
+#define CMN_MXP__CONNECT_INFO(p)	(0x0008 + 8 * (p))
 #define CMN__CONNECT_INFO_DEVICE_TYPE	GENMASK_ULL(4, 0)
 
+#define CMN_MAX_PORTS			6
+#define CI700_CONNECT_INFO_P2_5_OFFSET	0x10
+
 /* PMU registers occupy the 3rd 4KB page of each node's region */
 #define CMN_PMU_OFFSET			0x2000
 
@@ -166,7 +164,7 @@
 #define CMN_EVENT_BYNODEID(event)	FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config)
 #define CMN_EVENT_NODEID(event)		FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config)
 
-#define CMN_CONFIG_WP_COMBINE		GENMASK_ULL(27, 24)
+#define CMN_CONFIG_WP_COMBINE		GENMASK_ULL(30, 27)
 #define CMN_CONFIG_WP_DEV_SEL		GENMASK_ULL(50, 48)
 #define CMN_CONFIG_WP_CHN_SEL		GENMASK_ULL(55, 51)
 /* Note that we don't yet support the tertiary match group on newer IPs */
@@ -396,6 +394,25 @@ static struct arm_cmn_node *arm_cmn_node(const struct arm_cmn *cmn,
 	return NULL;
 }
 
+static u32 arm_cmn_device_connect_info(const struct arm_cmn *cmn,
+				       const struct arm_cmn_node *xp, int port)
+{
+	int offset = CMN_MXP__CONNECT_INFO(port);
+
+	if (port >= 2) {
+		if (cmn->model & (CMN600 | CMN650))
+			return 0;
+		/*
+		 * CI-700 may have extra ports, but still has the
+		 * mesh_port_connect_info registers in the way.
+		 */
+		if (cmn->model == CI700)
+			offset += CI700_CONNECT_INFO_P2_5_OFFSET;
+	}
+
+	return readl_relaxed(xp->pmu_base - CMN_PMU_OFFSET + offset);
+}
+
 static struct dentry *arm_cmn_debugfs;
 
 #ifdef CONFIG_DEBUG_FS
@@ -469,7 +486,7 @@ static int arm_cmn_map_show(struct seq_file *s, void *data)
 	y = cmn->mesh_y;
 	while (y--) {
 		int xp_base = cmn->mesh_x * y;
-		u8 port[6][CMN_MAX_DIMENSION];
+		u8 port[CMN_MAX_PORTS][CMN_MAX_DIMENSION];
 
 		for (x = 0; x < cmn->mesh_x; x++)
 			seq_puts(s, "--------+");
@@ -477,14 +494,9 @@ static int arm_cmn_map_show(struct seq_file *s, void *data)
 		seq_printf(s, "\n%d    |", y);
 		for (x = 0; x < cmn->mesh_x; x++) {
 			struct arm_cmn_node *xp = cmn->xps + xp_base + x;
-			void __iomem *base = xp->pmu_base - CMN_PMU_OFFSET;
-
-			port[0][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P0);
-			port[1][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P1);
-			port[2][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P2);
-			port[3][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P3);
-			port[4][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P4);
-			port[5][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P5);
+
+			for (p = 0; p < CMN_MAX_PORTS; p++)
+				port[p][x] = arm_cmn_device_connect_info(cmn, xp, p);
 			seq_printf(s, " XP #%-2d |", xp_base + x);
 		}
 
@@ -1546,7 +1558,7 @@ static int arm_cmn_event_init(struct perf_event *event)
 	type = CMN_EVENT_TYPE(event);
 	/* DTC events (i.e. cycles) already have everything they need */
 	if (type == CMN_TYPE_DTC)
-		return 0;
+		return arm_cmn_validate_group(cmn, event);
 
 	eventid = CMN_EVENT_EVENTID(event);
 	/* For watchpoints we need the actual XP node here */
@@ -2083,18 +2095,9 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset)
 		 * from this, since in that case we will see at least one XP
 		 * with port 2 connected, for the HN-D.
 		 */
-		if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P0))
-			xp_ports |= BIT(0);
-		if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P1))
-			xp_ports |= BIT(1);
-		if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P2))
-			xp_ports |= BIT(2);
-		if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P3))
-			xp_ports |= BIT(3);
-		if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P4))
-			xp_ports |= BIT(4);
-		if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P5))
-			xp_ports |= BIT(5);
+		for (int p = 0; p < CMN_MAX_PORTS; p++)
+			if (arm_cmn_device_connect_info(cmn, xp, p))
+				xp_ports |= BIT(p);
 
 		if (cmn->multi_dtm && (xp_ports & 0xc))
 			arm_cmn_init_dtm(dtm++, xp, 1);
diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c
index e31302ab7e37..a3f1c410b417 100644
--- a/drivers/perf/arm_cspmu/arm_cspmu.c
+++ b/drivers/perf/arm_cspmu/arm_cspmu.c
@@ -1078,12 +1078,14 @@ static int arm_cspmu_request_irq(struct arm_cspmu *cspmu)
 static inline int arm_cspmu_find_cpu_container(int cpu, u32 container_uid)
 {
 	u32 acpi_uid;
-	struct device *cpu_dev = get_cpu_device(cpu);
-	struct acpi_device *acpi_dev = ACPI_COMPANION(cpu_dev);
+	struct device *cpu_dev;
+	struct acpi_device *acpi_dev;
 
+	cpu_dev = get_cpu_device(cpu);
 	if (!cpu_dev)
 		return -ENODEV;
 
+	acpi_dev = ACPI_COMPANION(cpu_dev);
 	while (acpi_dev) {
 		if (!strcmp(acpi_device_hid(acpi_dev),
 			    ACPI_PROCESSOR_CONTAINER_HID) &&
diff --git a/drivers/perf/arm_dmc620_pmu.c b/drivers/perf/arm_dmc620_pmu.c
index 54aa4658fb36..5de06f9a4dd3 100644
--- a/drivers/perf/arm_dmc620_pmu.c
+++ b/drivers/perf/arm_dmc620_pmu.c
@@ -655,8 +655,7 @@ static int dmc620_pmu_device_probe(struct platform_device *pdev)
 		.attr_groups	= dmc620_pmu_attr_groups,
 	};
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	dmc620_pmu->base = devm_ioremap_resource(&pdev->dev, res);
+	dmc620_pmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(dmc620_pmu->base))
 		return PTR_ERR(dmc620_pmu->base);
 
diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c
new file mode 100644
index 000000000000..c98e4039386d
--- /dev/null
+++ b/drivers/perf/arm_pmuv3.c
@@ -0,0 +1,1419 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ARMv8 PMUv3 Performance Events handling code.
+ *
+ * Copyright (C) 2012 ARM Limited
+ * Author: Will Deacon <will.deacon@arm.com>
+ *
+ * This code is based heavily on the ARMv7 perf event code.
+ */
+
+#include <asm/irq_regs.h>
+#include <asm/perf_event.h>
+#include <asm/virt.h>
+
+#include <clocksource/arm_arch_timer.h>
+
+#include <linux/acpi.h>
+#include <linux/clocksource.h>
+#include <linux/of.h>
+#include <linux/perf/arm_pmu.h>
+#include <linux/perf/arm_pmuv3.h>
+#include <linux/platform_device.h>
+#include <linux/sched_clock.h>
+#include <linux/smp.h>
+
+#include <asm/arm_pmuv3.h>
+
+/* ARMv8 Cortex-A53 specific event types. */
+#define ARMV8_A53_PERFCTR_PREF_LINEFILL				0xC2
+
+/* ARMv8 Cavium ThunderX specific event types. */
+#define ARMV8_THUNDER_PERFCTR_L1D_CACHE_MISS_ST			0xE9
+#define ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS		0xEA
+#define ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS		0xEB
+#define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS		0xEC
+#define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS		0xED
+
+/*
+ * ARMv8 Architectural defined events, not all of these may
+ * be supported on any given implementation. Unsupported events will
+ * be disabled at run-time based on the PMCEID registers.
+ */
+static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
+	PERF_MAP_ALL_UNSUPPORTED,
+	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
+	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV8_PMUV3_PERFCTR_INST_RETIRED,
+	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE,
+	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
+	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
+	[PERF_COUNT_HW_BUS_CYCLES]		= ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
+	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND]	= ARMV8_PMUV3_PERFCTR_STALL_FRONTEND,
+	[PERF_COUNT_HW_STALLED_CYCLES_BACKEND]	= ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
+};
+
+static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+						[PERF_COUNT_HW_CACHE_OP_MAX]
+						[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	PERF_CACHE_MAP_ALL_UNSUPPORTED,
+
+	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE,
+	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
+
+	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1I_CACHE,
+	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
+
+	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL,
+	[C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1D_TLB,
+
+	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
+	[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1I_TLB,
+
+	[C(LL)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD,
+	[C(LL)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_LL_CACHE_RD,
+
+	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_BR_PRED,
+	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
+};
+
+static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+					      [PERF_COUNT_HW_CACHE_OP_MAX]
+					      [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	PERF_CACHE_MAP_ALL_UNSUPPORTED,
+
+	[C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREF_LINEFILL,
+
+	[C(NODE)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
+	[C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
+};
+
+static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+					      [PERF_COUNT_HW_CACHE_OP_MAX]
+					      [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	PERF_CACHE_MAP_ALL_UNSUPPORTED,
+
+	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
+	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD,
+	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
+	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,
+
+	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
+	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
+
+	[C(NODE)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
+	[C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
+};
+
+static const unsigned armv8_a73_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+					      [PERF_COUNT_HW_CACHE_OP_MAX]
+					      [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	PERF_CACHE_MAP_ALL_UNSUPPORTED,
+
+	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
+	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
+};
+
+static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+						   [PERF_COUNT_HW_CACHE_OP_MAX]
+						   [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	PERF_CACHE_MAP_ALL_UNSUPPORTED,
+
+	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
+	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD,
+	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
+	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV8_THUNDER_PERFCTR_L1D_CACHE_MISS_ST,
+	[C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS,
+	[C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS,
+
+	[C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS,
+	[C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS,
+
+	[C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD,
+	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
+	[C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,
+	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
+};
+
+static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+					      [PERF_COUNT_HW_CACHE_OP_MAX]
+					      [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	PERF_CACHE_MAP_ALL_UNSUPPORTED,
+
+	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
+	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD,
+	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
+	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,
+
+	[C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD,
+	[C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,
+	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
+	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
+
+	[C(NODE)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
+	[C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
+};
+
+static ssize_t
+armv8pmu_events_sysfs_show(struct device *dev,
+			   struct device_attribute *attr, char *page)
+{
+	struct perf_pmu_events_attr *pmu_attr;
+
+	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
+
+	return sprintf(page, "event=0x%04llx\n", pmu_attr->id);
+}
+
+#define ARMV8_EVENT_ATTR(name, config)						\
+	PMU_EVENT_ATTR_ID(name, armv8pmu_events_sysfs_show, config)
+
+static struct attribute *armv8_pmuv3_event_attrs[] = {
+	ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_SW_INCR),
+	ARMV8_EVENT_ATTR(l1i_cache_refill, ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL),
+	ARMV8_EVENT_ATTR(l1i_tlb_refill, ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL),
+	ARMV8_EVENT_ATTR(l1d_cache_refill, ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL),
+	ARMV8_EVENT_ATTR(l1d_cache, ARMV8_PMUV3_PERFCTR_L1D_CACHE),
+	ARMV8_EVENT_ATTR(l1d_tlb_refill, ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL),
+	ARMV8_EVENT_ATTR(ld_retired, ARMV8_PMUV3_PERFCTR_LD_RETIRED),
+	ARMV8_EVENT_ATTR(st_retired, ARMV8_PMUV3_PERFCTR_ST_RETIRED),
+	ARMV8_EVENT_ATTR(inst_retired, ARMV8_PMUV3_PERFCTR_INST_RETIRED),
+	ARMV8_EVENT_ATTR(exc_taken, ARMV8_PMUV3_PERFCTR_EXC_TAKEN),
+	ARMV8_EVENT_ATTR(exc_return, ARMV8_PMUV3_PERFCTR_EXC_RETURN),
+	ARMV8_EVENT_ATTR(cid_write_retired, ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED),
+	ARMV8_EVENT_ATTR(pc_write_retired, ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED),
+	ARMV8_EVENT_ATTR(br_immed_retired, ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED),
+	ARMV8_EVENT_ATTR(br_return_retired, ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED),
+	ARMV8_EVENT_ATTR(unaligned_ldst_retired, ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED),
+	ARMV8_EVENT_ATTR(br_mis_pred, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED),
+	ARMV8_EVENT_ATTR(cpu_cycles, ARMV8_PMUV3_PERFCTR_CPU_CYCLES),
+	ARMV8_EVENT_ATTR(br_pred, ARMV8_PMUV3_PERFCTR_BR_PRED),
+	ARMV8_EVENT_ATTR(mem_access, ARMV8_PMUV3_PERFCTR_MEM_ACCESS),
+	ARMV8_EVENT_ATTR(l1i_cache, ARMV8_PMUV3_PERFCTR_L1I_CACHE),
+	ARMV8_EVENT_ATTR(l1d_cache_wb, ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB),
+	ARMV8_EVENT_ATTR(l2d_cache, ARMV8_PMUV3_PERFCTR_L2D_CACHE),
+	ARMV8_EVENT_ATTR(l2d_cache_refill, ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL),
+	ARMV8_EVENT_ATTR(l2d_cache_wb, ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB),
+	ARMV8_EVENT_ATTR(bus_access, ARMV8_PMUV3_PERFCTR_BUS_ACCESS),
+	ARMV8_EVENT_ATTR(memory_error, ARMV8_PMUV3_PERFCTR_MEMORY_ERROR),
+	ARMV8_EVENT_ATTR(inst_spec, ARMV8_PMUV3_PERFCTR_INST_SPEC),
+	ARMV8_EVENT_ATTR(ttbr_write_retired, ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED),
+	ARMV8_EVENT_ATTR(bus_cycles, ARMV8_PMUV3_PERFCTR_BUS_CYCLES),
+	/* Don't expose the chain event in /sys, since it's useless in isolation */
+	ARMV8_EVENT_ATTR(l1d_cache_allocate, ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE),
+	ARMV8_EVENT_ATTR(l2d_cache_allocate, ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE),
+	ARMV8_EVENT_ATTR(br_retired, ARMV8_PMUV3_PERFCTR_BR_RETIRED),
+	ARMV8_EVENT_ATTR(br_mis_pred_retired, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED),
+	ARMV8_EVENT_ATTR(stall_frontend, ARMV8_PMUV3_PERFCTR_STALL_FRONTEND),
+	ARMV8_EVENT_ATTR(stall_backend, ARMV8_PMUV3_PERFCTR_STALL_BACKEND),
+	ARMV8_EVENT_ATTR(l1d_tlb, ARMV8_PMUV3_PERFCTR_L1D_TLB),
+	ARMV8_EVENT_ATTR(l1i_tlb, ARMV8_PMUV3_PERFCTR_L1I_TLB),
+	ARMV8_EVENT_ATTR(l2i_cache, ARMV8_PMUV3_PERFCTR_L2I_CACHE),
+	ARMV8_EVENT_ATTR(l2i_cache_refill, ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL),
+	ARMV8_EVENT_ATTR(l3d_cache_allocate, ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE),
+	ARMV8_EVENT_ATTR(l3d_cache_refill, ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL),
+	ARMV8_EVENT_ATTR(l3d_cache, ARMV8_PMUV3_PERFCTR_L3D_CACHE),
+	ARMV8_EVENT_ATTR(l3d_cache_wb, ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB),
+	ARMV8_EVENT_ATTR(l2d_tlb_refill, ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL),
+	ARMV8_EVENT_ATTR(l2i_tlb_refill, ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL),
+	ARMV8_EVENT_ATTR(l2d_tlb, ARMV8_PMUV3_PERFCTR_L2D_TLB),
+	ARMV8_EVENT_ATTR(l2i_tlb, ARMV8_PMUV3_PERFCTR_L2I_TLB),
+	ARMV8_EVENT_ATTR(remote_access, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS),
+	ARMV8_EVENT_ATTR(ll_cache, ARMV8_PMUV3_PERFCTR_LL_CACHE),
+	ARMV8_EVENT_ATTR(ll_cache_miss, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS),
+	ARMV8_EVENT_ATTR(dtlb_walk, ARMV8_PMUV3_PERFCTR_DTLB_WALK),
+	ARMV8_EVENT_ATTR(itlb_walk, ARMV8_PMUV3_PERFCTR_ITLB_WALK),
+	ARMV8_EVENT_ATTR(ll_cache_rd, ARMV8_PMUV3_PERFCTR_LL_CACHE_RD),
+	ARMV8_EVENT_ATTR(ll_cache_miss_rd, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD),
+	ARMV8_EVENT_ATTR(remote_access_rd, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS_RD),
+	ARMV8_EVENT_ATTR(l1d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L1D_CACHE_LMISS_RD),
+	ARMV8_EVENT_ATTR(op_retired, ARMV8_PMUV3_PERFCTR_OP_RETIRED),
+	ARMV8_EVENT_ATTR(op_spec, ARMV8_PMUV3_PERFCTR_OP_SPEC),
+	ARMV8_EVENT_ATTR(stall, ARMV8_PMUV3_PERFCTR_STALL),
+	ARMV8_EVENT_ATTR(stall_slot_backend, ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND),
+	ARMV8_EVENT_ATTR(stall_slot_frontend, ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND),
+	ARMV8_EVENT_ATTR(stall_slot, ARMV8_PMUV3_PERFCTR_STALL_SLOT),
+	ARMV8_EVENT_ATTR(sample_pop, ARMV8_SPE_PERFCTR_SAMPLE_POP),
+	ARMV8_EVENT_ATTR(sample_feed, ARMV8_SPE_PERFCTR_SAMPLE_FEED),
+	ARMV8_EVENT_ATTR(sample_filtrate, ARMV8_SPE_PERFCTR_SAMPLE_FILTRATE),
+	ARMV8_EVENT_ATTR(sample_collision, ARMV8_SPE_PERFCTR_SAMPLE_COLLISION),
+	ARMV8_EVENT_ATTR(cnt_cycles, ARMV8_AMU_PERFCTR_CNT_CYCLES),
+	ARMV8_EVENT_ATTR(stall_backend_mem, ARMV8_AMU_PERFCTR_STALL_BACKEND_MEM),
+	ARMV8_EVENT_ATTR(l1i_cache_lmiss, ARMV8_PMUV3_PERFCTR_L1I_CACHE_LMISS),
+	ARMV8_EVENT_ATTR(l2d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L2D_CACHE_LMISS_RD),
+	ARMV8_EVENT_ATTR(l2i_cache_lmiss, ARMV8_PMUV3_PERFCTR_L2I_CACHE_LMISS),
+	ARMV8_EVENT_ATTR(l3d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L3D_CACHE_LMISS_RD),
+	ARMV8_EVENT_ATTR(trb_wrap, ARMV8_PMUV3_PERFCTR_TRB_WRAP),
+	ARMV8_EVENT_ATTR(trb_trig, ARMV8_PMUV3_PERFCTR_TRB_TRIG),
+	ARMV8_EVENT_ATTR(trcextout0, ARMV8_PMUV3_PERFCTR_TRCEXTOUT0),
+	ARMV8_EVENT_ATTR(trcextout1, ARMV8_PMUV3_PERFCTR_TRCEXTOUT1),
+	ARMV8_EVENT_ATTR(trcextout2, ARMV8_PMUV3_PERFCTR_TRCEXTOUT2),
+	ARMV8_EVENT_ATTR(trcextout3, ARMV8_PMUV3_PERFCTR_TRCEXTOUT3),
+	ARMV8_EVENT_ATTR(cti_trigout4, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT4),
+	ARMV8_EVENT_ATTR(cti_trigout5, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT5),
+	ARMV8_EVENT_ATTR(cti_trigout6, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT6),
+	ARMV8_EVENT_ATTR(cti_trigout7, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT7),
+	ARMV8_EVENT_ATTR(ldst_align_lat, ARMV8_PMUV3_PERFCTR_LDST_ALIGN_LAT),
+	ARMV8_EVENT_ATTR(ld_align_lat, ARMV8_PMUV3_PERFCTR_LD_ALIGN_LAT),
+	ARMV8_EVENT_ATTR(st_align_lat, ARMV8_PMUV3_PERFCTR_ST_ALIGN_LAT),
+	ARMV8_EVENT_ATTR(mem_access_checked, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED),
+	ARMV8_EVENT_ATTR(mem_access_checked_rd, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED_RD),
+	ARMV8_EVENT_ATTR(mem_access_checked_wr, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED_WR),
+	NULL,
+};
+
+static umode_t
+armv8pmu_event_attr_is_visible(struct kobject *kobj,
+			       struct attribute *attr, int unused)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct pmu *pmu = dev_get_drvdata(dev);
+	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
+	struct perf_pmu_events_attr *pmu_attr;
+
+	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr);
+
+	if (pmu_attr->id < ARMV8_PMUV3_MAX_COMMON_EVENTS &&
+	    test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap))
+		return attr->mode;
+
+	if (pmu_attr->id >= ARMV8_PMUV3_EXT_COMMON_EVENT_BASE) {
+		u64 id = pmu_attr->id - ARMV8_PMUV3_EXT_COMMON_EVENT_BASE;
+
+		if (id < ARMV8_PMUV3_MAX_COMMON_EVENTS &&
+		    test_bit(id, cpu_pmu->pmceid_ext_bitmap))
+			return attr->mode;
+	}
+
+	return 0;
+}
+
+static const struct attribute_group armv8_pmuv3_events_attr_group = {
+	.name = "events",
+	.attrs = armv8_pmuv3_event_attrs,
+	.is_visible = armv8pmu_event_attr_is_visible,
+};
+
+PMU_FORMAT_ATTR(event, "config:0-15");
+PMU_FORMAT_ATTR(long, "config1:0");
+PMU_FORMAT_ATTR(rdpmc, "config1:1");
+
+static int sysctl_perf_user_access __read_mostly;
+
+static inline bool armv8pmu_event_is_64bit(struct perf_event *event)
+{
+	return event->attr.config1 & 0x1;
+}
+
+static inline bool armv8pmu_event_want_user_access(struct perf_event *event)
+{
+	return event->attr.config1 & 0x2;
+}
+
+static struct attribute *armv8_pmuv3_format_attrs[] = {
+	&format_attr_event.attr,
+	&format_attr_long.attr,
+	&format_attr_rdpmc.attr,
+	NULL,
+};
+
+static const struct attribute_group armv8_pmuv3_format_attr_group = {
+	.name = "format",
+	.attrs = armv8_pmuv3_format_attrs,
+};
+
+static ssize_t slots_show(struct device *dev, struct device_attribute *attr,
+			  char *page)
+{
+	struct pmu *pmu = dev_get_drvdata(dev);
+	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
+	u32 slots = cpu_pmu->reg_pmmir & ARMV8_PMU_SLOTS_MASK;
+
+	return sysfs_emit(page, "0x%08x\n", slots);
+}
+
+static DEVICE_ATTR_RO(slots);
+
+static ssize_t bus_slots_show(struct device *dev, struct device_attribute *attr,
+			      char *page)
+{
+	struct pmu *pmu = dev_get_drvdata(dev);
+	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
+	u32 bus_slots = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_SLOTS_SHIFT)
+			& ARMV8_PMU_BUS_SLOTS_MASK;
+
+	return sysfs_emit(page, "0x%08x\n", bus_slots);
+}
+
+static DEVICE_ATTR_RO(bus_slots);
+
+static ssize_t bus_width_show(struct device *dev, struct device_attribute *attr,
+			      char *page)
+{
+	struct pmu *pmu = dev_get_drvdata(dev);
+	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
+	u32 bus_width = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_WIDTH_SHIFT)
+			& ARMV8_PMU_BUS_WIDTH_MASK;
+	u32 val = 0;
+
+	/* Encoded as Log2(number of bytes), plus one */
+	if (bus_width > 2 && bus_width < 13)
+		val = 1 << (bus_width - 1);
+
+	return sysfs_emit(page, "0x%08x\n", val);
+}
+
+static DEVICE_ATTR_RO(bus_width);
+
+static struct attribute *armv8_pmuv3_caps_attrs[] = {
+	&dev_attr_slots.attr,
+	&dev_attr_bus_slots.attr,
+	&dev_attr_bus_width.attr,
+	NULL,
+};
+
+static const struct attribute_group armv8_pmuv3_caps_attr_group = {
+	.name = "caps",
+	.attrs = armv8_pmuv3_caps_attrs,
+};
+
+/*
+ * Perf Events' indices
+ */
+#define	ARMV8_IDX_CYCLE_COUNTER	0
+#define	ARMV8_IDX_COUNTER0	1
+#define	ARMV8_IDX_CYCLE_COUNTER_USER	32
+
+/*
+ * We unconditionally enable ARMv8.5-PMU long event counter support
+ * (64-bit events) where supported. Indicate if this arm_pmu has long
+ * event counter support.
+ *
+ * On AArch32, long counters make no sense (you can't access the top
+ * bits), so we only enable this on AArch64.
+ */
+static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu)
+{
+	return (IS_ENABLED(CONFIG_ARM64) && is_pmuv3p5(cpu_pmu->pmuver));
+}
+
+static inline bool armv8pmu_event_has_user_read(struct perf_event *event)
+{
+	return event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT;
+}
+
+/*
+ * We must chain two programmable counters for 64 bit events,
+ * except when we have allocated the 64bit cycle counter (for CPU
+ * cycles event) or when user space counter access is enabled.
+ */
+static inline bool armv8pmu_event_is_chained(struct perf_event *event)
+{
+	int idx = event->hw.idx;
+	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
+
+	return !armv8pmu_event_has_user_read(event) &&
+	       armv8pmu_event_is_64bit(event) &&
+	       !armv8pmu_has_long_event(cpu_pmu) &&
+	       (idx != ARMV8_IDX_CYCLE_COUNTER);
+}
+
+/*
+ * ARMv8 low level PMU access
+ */
+
+/*
+ * Perf Event to low level counters mapping
+ */
+#define	ARMV8_IDX_TO_COUNTER(x)	\
+	(((x) - ARMV8_IDX_COUNTER0) & ARMV8_PMU_COUNTER_MASK)
+
+static inline u32 armv8pmu_pmcr_read(void)
+{
+	return read_pmcr();
+}
+
+static inline void armv8pmu_pmcr_write(u32 val)
+{
+	val &= ARMV8_PMU_PMCR_MASK;
+	isb();
+	write_pmcr(val);
+}
+
+static inline int armv8pmu_has_overflowed(u32 pmovsr)
+{
+	return pmovsr & ARMV8_PMU_OVERFLOWED_MASK;
+}
+
+static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx)
+{
+	return pmnc & BIT(ARMV8_IDX_TO_COUNTER(idx));
+}
+
+static inline u64 armv8pmu_read_evcntr(int idx)
+{
+	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
+
+	return read_pmevcntrn(counter);
+}
+
+static inline u64 armv8pmu_read_hw_counter(struct perf_event *event)
+{
+	int idx = event->hw.idx;
+	u64 val = armv8pmu_read_evcntr(idx);
+
+	if (armv8pmu_event_is_chained(event))
+		val = (val << 32) | armv8pmu_read_evcntr(idx - 1);
+	return val;
+}
+
+/*
+ * The cycle counter is always a 64-bit counter. When ARMV8_PMU_PMCR_LP
+ * is set the event counters also become 64-bit counters. Unless the
+ * user has requested a long counter (attr.config1) then we want to
+ * interrupt upon 32-bit overflow - we achieve this by applying a bias.
+ */
+static bool armv8pmu_event_needs_bias(struct perf_event *event)
+{
+	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	int idx = hwc->idx;
+
+	if (armv8pmu_event_is_64bit(event))
+		return false;
+
+	if (armv8pmu_has_long_event(cpu_pmu) ||
+	    idx == ARMV8_IDX_CYCLE_COUNTER)
+		return true;
+
+	return false;
+}
+
+static u64 armv8pmu_bias_long_counter(struct perf_event *event, u64 value)
+{
+	if (armv8pmu_event_needs_bias(event))
+		value |= GENMASK_ULL(63, 32);
+
+	return value;
+}
+
+static u64 armv8pmu_unbias_long_counter(struct perf_event *event, u64 value)
+{
+	if (armv8pmu_event_needs_bias(event))
+		value &= ~GENMASK_ULL(63, 32);
+
+	return value;
+}
+
+static u64 armv8pmu_read_counter(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	int idx = hwc->idx;
+	u64 value;
+
+	if (idx == ARMV8_IDX_CYCLE_COUNTER)
+		value = read_pmccntr();
+	else
+		value = armv8pmu_read_hw_counter(event);
+
+	return  armv8pmu_unbias_long_counter(event, value);
+}
+
+static inline void armv8pmu_write_evcntr(int idx, u64 value)
+{
+	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
+
+	write_pmevcntrn(counter, value);
+}
+
+static inline void armv8pmu_write_hw_counter(struct perf_event *event,
+					     u64 value)
+{
+	int idx = event->hw.idx;
+
+	if (armv8pmu_event_is_chained(event)) {
+		armv8pmu_write_evcntr(idx, upper_32_bits(value));
+		armv8pmu_write_evcntr(idx - 1, lower_32_bits(value));
+	} else {
+		armv8pmu_write_evcntr(idx, value);
+	}
+}
+
+static void armv8pmu_write_counter(struct perf_event *event, u64 value)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	int idx = hwc->idx;
+
+	value = armv8pmu_bias_long_counter(event, value);
+
+	if (idx == ARMV8_IDX_CYCLE_COUNTER)
+		write_pmccntr(value);
+	else
+		armv8pmu_write_hw_counter(event, value);
+}
+
+static inline void armv8pmu_write_evtype(int idx, u32 val)
+{
+	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
+
+	val &= ARMV8_PMU_EVTYPE_MASK;
+	write_pmevtypern(counter, val);
+}
+
+static inline void armv8pmu_write_event_type(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	int idx = hwc->idx;
+
+	/*
+	 * For chained events, the low counter is programmed to count
+	 * the event of interest and the high counter is programmed
+	 * with CHAIN event code with filters set to count at all ELs.
+	 */
+	if (armv8pmu_event_is_chained(event)) {
+		u32 chain_evt = ARMV8_PMUV3_PERFCTR_CHAIN |
+				ARMV8_PMU_INCLUDE_EL2;
+
+		armv8pmu_write_evtype(idx - 1, hwc->config_base);
+		armv8pmu_write_evtype(idx, chain_evt);
+	} else {
+		if (idx == ARMV8_IDX_CYCLE_COUNTER)
+			write_pmccfiltr(hwc->config_base);
+		else
+			armv8pmu_write_evtype(idx, hwc->config_base);
+	}
+}
+
+static u32 armv8pmu_event_cnten_mask(struct perf_event *event)
+{
+	int counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
+	u32 mask = BIT(counter);
+
+	if (armv8pmu_event_is_chained(event))
+		mask |= BIT(counter - 1);
+	return mask;
+}
+
+static inline void armv8pmu_enable_counter(u32 mask)
+{
+	/*
+	 * Make sure event configuration register writes are visible before we
+	 * enable the counter.
+	 * */
+	isb();
+	write_pmcntenset(mask);
+}
+
+static inline void armv8pmu_enable_event_counter(struct perf_event *event)
+{
+	struct perf_event_attr *attr = &event->attr;
+	u32 mask = armv8pmu_event_cnten_mask(event);
+
+	kvm_set_pmu_events(mask, attr);
+
+	/* We rely on the hypervisor switch code to enable guest counters */
+	if (!kvm_pmu_counter_deferred(attr))
+		armv8pmu_enable_counter(mask);
+}
+
+static inline void armv8pmu_disable_counter(u32 mask)
+{
+	write_pmcntenclr(mask);
+	/*
+	 * Make sure the effects of disabling the counter are visible before we
+	 * start configuring the event.
+	 */
+	isb();
+}
+
+static inline void armv8pmu_disable_event_counter(struct perf_event *event)
+{
+	struct perf_event_attr *attr = &event->attr;
+	u32 mask = armv8pmu_event_cnten_mask(event);
+
+	kvm_clr_pmu_events(mask);
+
+	/* We rely on the hypervisor switch code to disable guest counters */
+	if (!kvm_pmu_counter_deferred(attr))
+		armv8pmu_disable_counter(mask);
+}
+
+static inline void armv8pmu_enable_intens(u32 mask)
+{
+	write_pmintenset(mask);
+}
+
+static inline void armv8pmu_enable_event_irq(struct perf_event *event)
+{
+	u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
+	armv8pmu_enable_intens(BIT(counter));
+}
+
+static inline void armv8pmu_disable_intens(u32 mask)
+{
+	write_pmintenclr(mask);
+	isb();
+	/* Clear the overflow flag in case an interrupt is pending. */
+	write_pmovsclr(mask);
+	isb();
+}
+
+static inline void armv8pmu_disable_event_irq(struct perf_event *event)
+{
+	u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
+	armv8pmu_disable_intens(BIT(counter));
+}
+
+static inline u32 armv8pmu_getreset_flags(void)
+{
+	u32 value;
+
+	/* Read */
+	value = read_pmovsclr();
+
+	/* Write to clear flags */
+	value &= ARMV8_PMU_OVSR_MASK;
+	write_pmovsclr(value);
+
+	return value;
+}
+
+static void armv8pmu_disable_user_access(void)
+{
+	write_pmuserenr(0);
+}
+
+static void armv8pmu_enable_user_access(struct arm_pmu *cpu_pmu)
+{
+	int i;
+	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
+
+	/* Clear any unused counters to avoid leaking their contents */
+	for_each_clear_bit(i, cpuc->used_mask, cpu_pmu->num_events) {
+		if (i == ARMV8_IDX_CYCLE_COUNTER)
+			write_pmccntr(0);
+		else
+			armv8pmu_write_evcntr(i, 0);
+	}
+
+	write_pmuserenr(0);
+	write_pmuserenr(ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_CR);
+}
+
+static void armv8pmu_enable_event(struct perf_event *event)
+{
+	/*
+	 * Enable counter and interrupt, and set the counter to count
+	 * the event that we're interested in.
+	 */
+
+	/*
+	 * Disable counter
+	 */
+	armv8pmu_disable_event_counter(event);
+
+	/*
+	 * Set event.
+	 */
+	armv8pmu_write_event_type(event);
+
+	/*
+	 * Enable interrupt for this counter
+	 */
+	armv8pmu_enable_event_irq(event);
+
+	/*
+	 * Enable counter
+	 */
+	armv8pmu_enable_event_counter(event);
+}
+
+static void armv8pmu_disable_event(struct perf_event *event)
+{
+	/*
+	 * Disable counter
+	 */
+	armv8pmu_disable_event_counter(event);
+
+	/*
+	 * Disable interrupt for this counter
+	 */
+	armv8pmu_disable_event_irq(event);
+}
+
+static void armv8pmu_start(struct arm_pmu *cpu_pmu)
+{
+	struct perf_event_context *ctx;
+	int nr_user = 0;
+
+	ctx = perf_cpu_task_ctx();
+	if (ctx)
+		nr_user = ctx->nr_user;
+
+	if (sysctl_perf_user_access && nr_user)
+		armv8pmu_enable_user_access(cpu_pmu);
+	else
+		armv8pmu_disable_user_access();
+
+	/* Enable all counters */
+	armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E);
+}
+
+static void armv8pmu_stop(struct arm_pmu *cpu_pmu)
+{
+	/* Disable all counters */
+	armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
+}
+
+static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)
+{
+	u32 pmovsr;
+	struct perf_sample_data data;
+	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
+	struct pt_regs *regs;
+	int idx;
+
+	/*
+	 * Get and reset the IRQ flags
+	 */
+	pmovsr = armv8pmu_getreset_flags();
+
+	/*
+	 * Did an overflow occur?
+	 */
+	if (!armv8pmu_has_overflowed(pmovsr))
+		return IRQ_NONE;
+
+	/*
+	 * Handle the counter(s) overflow(s)
+	 */
+	regs = get_irq_regs();
+
+	/*
+	 * Stop the PMU while processing the counter overflows
+	 * to prevent skews in group events.
+	 */
+	armv8pmu_stop(cpu_pmu);
+	for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
+		struct perf_event *event = cpuc->events[idx];
+		struct hw_perf_event *hwc;
+
+		/* Ignore if we don't have an event. */
+		if (!event)
+			continue;
+
+		/*
+		 * We have a single interrupt for all counters. Check that
+		 * each counter has overflowed before we process it.
+		 */
+		if (!armv8pmu_counter_has_overflowed(pmovsr, idx))
+			continue;
+
+		hwc = &event->hw;
+		armpmu_event_update(event);
+		perf_sample_data_init(&data, 0, hwc->last_period);
+		if (!armpmu_event_set_period(event))
+			continue;
+
+		/*
+		 * Perf event overflow will queue the processing of the event as
+		 * an irq_work which will be taken care of in the handling of
+		 * IPI_IRQ_WORK.
+		 */
+		if (perf_event_overflow(event, &data, regs))
+			cpu_pmu->disable(event);
+	}
+	armv8pmu_start(cpu_pmu);
+
+	return IRQ_HANDLED;
+}
+
+static int armv8pmu_get_single_idx(struct pmu_hw_events *cpuc,
+				    struct arm_pmu *cpu_pmu)
+{
+	int idx;
+
+	for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; idx++) {
+		if (!test_and_set_bit(idx, cpuc->used_mask))
+			return idx;
+	}
+	return -EAGAIN;
+}
+
+static int armv8pmu_get_chain_idx(struct pmu_hw_events *cpuc,
+				   struct arm_pmu *cpu_pmu)
+{
+	int idx;
+
+	/*
+	 * Chaining requires two consecutive event counters, where
+	 * the lower idx must be even.
+	 */
+	for (idx = ARMV8_IDX_COUNTER0 + 1; idx < cpu_pmu->num_events; idx += 2) {
+		if (!test_and_set_bit(idx, cpuc->used_mask)) {
+			/* Check if the preceding even counter is available */
+			if (!test_and_set_bit(idx - 1, cpuc->used_mask))
+				return idx;
+			/* Release the Odd counter */
+			clear_bit(idx, cpuc->used_mask);
+		}
+	}
+	return -EAGAIN;
+}
+
+static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc,
+				  struct perf_event *event)
+{
+	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT;
+
+	/* Always prefer to place a cycle counter into the cycle counter. */
+	if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) {
+		if (!test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask))
+			return ARMV8_IDX_CYCLE_COUNTER;
+		else if (armv8pmu_event_is_64bit(event) &&
+			   armv8pmu_event_want_user_access(event) &&
+			   !armv8pmu_has_long_event(cpu_pmu))
+				return -EAGAIN;
+	}
+
+	/*
+	 * Otherwise use events counters
+	 */
+	if (armv8pmu_event_is_chained(event))
+		return	armv8pmu_get_chain_idx(cpuc, cpu_pmu);
+	else
+		return armv8pmu_get_single_idx(cpuc, cpu_pmu);
+}
+
+static void armv8pmu_clear_event_idx(struct pmu_hw_events *cpuc,
+				     struct perf_event *event)
+{
+	int idx = event->hw.idx;
+
+	clear_bit(idx, cpuc->used_mask);
+	if (armv8pmu_event_is_chained(event))
+		clear_bit(idx - 1, cpuc->used_mask);
+}
+
+static int armv8pmu_user_event_idx(struct perf_event *event)
+{
+	if (!sysctl_perf_user_access || !armv8pmu_event_has_user_read(event))
+		return 0;
+
+	/*
+	 * We remap the cycle counter index to 32 to
+	 * match the offset applied to the rest of
+	 * the counter indices.
+	 */
+	if (event->hw.idx == ARMV8_IDX_CYCLE_COUNTER)
+		return ARMV8_IDX_CYCLE_COUNTER_USER;
+
+	return event->hw.idx;
+}
+
+/*
+ * Add an event filter to a given event.
+ */
+static int armv8pmu_set_event_filter(struct hw_perf_event *event,
+				     struct perf_event_attr *attr)
+{
+	unsigned long config_base = 0;
+
+	if (attr->exclude_idle)
+		return -EPERM;
+
+	/*
+	 * If we're running in hyp mode, then we *are* the hypervisor.
+	 * Therefore we ignore exclude_hv in this configuration, since
+	 * there's no hypervisor to sample anyway. This is consistent
+	 * with other architectures (x86 and Power).
+	 */
+	if (is_kernel_in_hyp_mode()) {
+		if (!attr->exclude_kernel && !attr->exclude_host)
+			config_base |= ARMV8_PMU_INCLUDE_EL2;
+		if (attr->exclude_guest)
+			config_base |= ARMV8_PMU_EXCLUDE_EL1;
+		if (attr->exclude_host)
+			config_base |= ARMV8_PMU_EXCLUDE_EL0;
+	} else {
+		if (!attr->exclude_hv && !attr->exclude_host)
+			config_base |= ARMV8_PMU_INCLUDE_EL2;
+	}
+
+	/*
+	 * Filter out !VHE kernels and guest kernels
+	 */
+	if (attr->exclude_kernel)
+		config_base |= ARMV8_PMU_EXCLUDE_EL1;
+
+	if (attr->exclude_user)
+		config_base |= ARMV8_PMU_EXCLUDE_EL0;
+
+	/*
+	 * Install the filter into config_base as this is used to
+	 * construct the event type.
+	 */
+	event->config_base = config_base;
+
+	return 0;
+}
+
+static void armv8pmu_reset(void *info)
+{
+	struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
+	u32 pmcr;
+
+	/* The counter and interrupt enable registers are unknown at reset. */
+	armv8pmu_disable_counter(U32_MAX);
+	armv8pmu_disable_intens(U32_MAX);
+
+	/* Clear the counters we flip at guest entry/exit */
+	kvm_clr_pmu_events(U32_MAX);
+
+	/*
+	 * Initialize & Reset PMNC. Request overflow interrupt for
+	 * 64 bit cycle counter but cheat in armv8pmu_write_counter().
+	 */
+	pmcr = ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C | ARMV8_PMU_PMCR_LC;
+
+	/* Enable long event counter support where available */
+	if (armv8pmu_has_long_event(cpu_pmu))
+		pmcr |= ARMV8_PMU_PMCR_LP;
+
+	armv8pmu_pmcr_write(pmcr);
+}
+
+static int __armv8_pmuv3_map_event_id(struct arm_pmu *armpmu,
+				      struct perf_event *event)
+{
+	if (event->attr.type == PERF_TYPE_HARDWARE &&
+	    event->attr.config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) {
+
+		if (test_bit(ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED,
+			     armpmu->pmceid_bitmap))
+			return ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED;
+
+		if (test_bit(ARMV8_PMUV3_PERFCTR_BR_RETIRED,
+			     armpmu->pmceid_bitmap))
+			return ARMV8_PMUV3_PERFCTR_BR_RETIRED;
+
+		return HW_OP_UNSUPPORTED;
+	}
+
+	return armpmu_map_event(event, &armv8_pmuv3_perf_map,
+				&armv8_pmuv3_perf_cache_map,
+				ARMV8_PMU_EVTYPE_EVENT);
+}
+
+static int __armv8_pmuv3_map_event(struct perf_event *event,
+				   const unsigned (*extra_event_map)
+						  [PERF_COUNT_HW_MAX],
+				   const unsigned (*extra_cache_map)
+						  [PERF_COUNT_HW_CACHE_MAX]
+						  [PERF_COUNT_HW_CACHE_OP_MAX]
+						  [PERF_COUNT_HW_CACHE_RESULT_MAX])
+{
+	int hw_event_id;
+	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
+
+	hw_event_id = __armv8_pmuv3_map_event_id(armpmu, event);
+
+	/*
+	 * CHAIN events only work when paired with an adjacent counter, and it
+	 * never makes sense for a user to open one in isolation, as they'll be
+	 * rotated arbitrarily.
+	 */
+	if (hw_event_id == ARMV8_PMUV3_PERFCTR_CHAIN)
+		return -EINVAL;
+
+	if (armv8pmu_event_is_64bit(event))
+		event->hw.flags |= ARMPMU_EVT_64BIT;
+
+	/*
+	 * User events must be allocated into a single counter, and so
+	 * must not be chained.
+	 *
+	 * Most 64-bit events require long counter support, but 64-bit
+	 * CPU_CYCLES events can be placed into the dedicated cycle
+	 * counter when this is free.
+	 */
+	if (armv8pmu_event_want_user_access(event)) {
+		if (!(event->attach_state & PERF_ATTACH_TASK))
+			return -EINVAL;
+		if (armv8pmu_event_is_64bit(event) &&
+		    (hw_event_id != ARMV8_PMUV3_PERFCTR_CPU_CYCLES) &&
+		    !armv8pmu_has_long_event(armpmu))
+			return -EOPNOTSUPP;
+
+		event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT;
+	}
+
+	/* Only expose micro/arch events supported by this PMU */
+	if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS)
+	    && test_bit(hw_event_id, armpmu->pmceid_bitmap)) {
+		return hw_event_id;
+	}
+
+	return armpmu_map_event(event, extra_event_map, extra_cache_map,
+				ARMV8_PMU_EVTYPE_EVENT);
+}
+
+static int armv8_pmuv3_map_event(struct perf_event *event)
+{
+	return __armv8_pmuv3_map_event(event, NULL, NULL);
+}
+
+static int armv8_a53_map_event(struct perf_event *event)
+{
+	return __armv8_pmuv3_map_event(event, NULL, &armv8_a53_perf_cache_map);
+}
+
+static int armv8_a57_map_event(struct perf_event *event)
+{
+	return __armv8_pmuv3_map_event(event, NULL, &armv8_a57_perf_cache_map);
+}
+
+static int armv8_a73_map_event(struct perf_event *event)
+{
+	return __armv8_pmuv3_map_event(event, NULL, &armv8_a73_perf_cache_map);
+}
+
+static int armv8_thunder_map_event(struct perf_event *event)
+{
+	return __armv8_pmuv3_map_event(event, NULL,
+				       &armv8_thunder_perf_cache_map);
+}
+
+static int armv8_vulcan_map_event(struct perf_event *event)
+{
+	return __armv8_pmuv3_map_event(event, NULL,
+				       &armv8_vulcan_perf_cache_map);
+}
+
+struct armv8pmu_probe_info {
+	struct arm_pmu *pmu;
+	bool present;
+};
+
+static void __armv8pmu_probe_pmu(void *info)
+{
+	struct armv8pmu_probe_info *probe = info;
+	struct arm_pmu *cpu_pmu = probe->pmu;
+	u64 pmceid_raw[2];
+	u32 pmceid[2];
+	int pmuver;
+
+	pmuver = read_pmuver();
+	if (!pmuv3_implemented(pmuver))
+		return;
+
+	cpu_pmu->pmuver = pmuver;
+	probe->present = true;
+
+	/* Read the nb of CNTx counters supported from PMNC */
+	cpu_pmu->num_events = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT)
+		& ARMV8_PMU_PMCR_N_MASK;
+
+	/* Add the CPU cycles counter */
+	cpu_pmu->num_events += 1;
+
+	pmceid[0] = pmceid_raw[0] = read_pmceid0();
+	pmceid[1] = pmceid_raw[1] = read_pmceid1();
+
+	bitmap_from_arr32(cpu_pmu->pmceid_bitmap,
+			     pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
+
+	pmceid[0] = pmceid_raw[0] >> 32;
+	pmceid[1] = pmceid_raw[1] >> 32;
+
+	bitmap_from_arr32(cpu_pmu->pmceid_ext_bitmap,
+			     pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
+
+	/* store PMMIR register for sysfs */
+	if (is_pmuv3p4(pmuver) && (pmceid_raw[1] & BIT(31)))
+		cpu_pmu->reg_pmmir = read_pmmir();
+	else
+		cpu_pmu->reg_pmmir = 0;
+}
+
+static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)
+{
+	struct armv8pmu_probe_info probe = {
+		.pmu = cpu_pmu,
+		.present = false,
+	};
+	int ret;
+
+	ret = smp_call_function_any(&cpu_pmu->supported_cpus,
+				    __armv8pmu_probe_pmu,
+				    &probe, 1);
+	if (ret)
+		return ret;
+
+	return probe.present ? 0 : -ENODEV;
+}
+
+static void armv8pmu_disable_user_access_ipi(void *unused)
+{
+	armv8pmu_disable_user_access();
+}
+
+static int armv8pmu_proc_user_access_handler(struct ctl_table *table, int write,
+		void *buffer, size_t *lenp, loff_t *ppos)
+{
+	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+	if (ret || !write || sysctl_perf_user_access)
+		return ret;
+
+	on_each_cpu(armv8pmu_disable_user_access_ipi, NULL, 1);
+	return 0;
+}
+
+static struct ctl_table armv8_pmu_sysctl_table[] = {
+	{
+		.procname       = "perf_user_access",
+		.data		= &sysctl_perf_user_access,
+		.maxlen		= sizeof(unsigned int),
+		.mode           = 0644,
+		.proc_handler	= armv8pmu_proc_user_access_handler,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
+	{ }
+};
+
+static void armv8_pmu_register_sysctl_table(void)
+{
+	static u32 tbl_registered = 0;
+
+	if (!cmpxchg_relaxed(&tbl_registered, 0, 1))
+		register_sysctl("kernel", armv8_pmu_sysctl_table);
+}
+
+static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name,
+			  int (*map_event)(struct perf_event *event),
+			  const struct attribute_group *events,
+			  const struct attribute_group *format,
+			  const struct attribute_group *caps)
+{
+	int ret = armv8pmu_probe_pmu(cpu_pmu);
+	if (ret)
+		return ret;
+
+	cpu_pmu->handle_irq		= armv8pmu_handle_irq;
+	cpu_pmu->enable			= armv8pmu_enable_event;
+	cpu_pmu->disable		= armv8pmu_disable_event;
+	cpu_pmu->read_counter		= armv8pmu_read_counter;
+	cpu_pmu->write_counter		= armv8pmu_write_counter;
+	cpu_pmu->get_event_idx		= armv8pmu_get_event_idx;
+	cpu_pmu->clear_event_idx	= armv8pmu_clear_event_idx;
+	cpu_pmu->start			= armv8pmu_start;
+	cpu_pmu->stop			= armv8pmu_stop;
+	cpu_pmu->reset			= armv8pmu_reset;
+	cpu_pmu->set_event_filter	= armv8pmu_set_event_filter;
+
+	cpu_pmu->pmu.event_idx		= armv8pmu_user_event_idx;
+
+	cpu_pmu->name			= name;
+	cpu_pmu->map_event		= map_event;
+	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = events ?
+			events : &armv8_pmuv3_events_attr_group;
+	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = format ?
+			format : &armv8_pmuv3_format_attr_group;
+	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_CAPS] = caps ?
+			caps : &armv8_pmuv3_caps_attr_group;
+
+	armv8_pmu_register_sysctl_table();
+	return 0;
+}
+
+static int armv8_pmu_init_nogroups(struct arm_pmu *cpu_pmu, char *name,
+				   int (*map_event)(struct perf_event *event))
+{
+	return armv8_pmu_init(cpu_pmu, name, map_event, NULL, NULL, NULL);
+}
+
+#define PMUV3_INIT_SIMPLE(name)						\
+static int name##_pmu_init(struct arm_pmu *cpu_pmu)			\
+{									\
+	return armv8_pmu_init_nogroups(cpu_pmu, #name, armv8_pmuv3_map_event);\
+}
+
+PMUV3_INIT_SIMPLE(armv8_pmuv3)
+
+PMUV3_INIT_SIMPLE(armv8_cortex_a34)
+PMUV3_INIT_SIMPLE(armv8_cortex_a55)
+PMUV3_INIT_SIMPLE(armv8_cortex_a65)
+PMUV3_INIT_SIMPLE(armv8_cortex_a75)
+PMUV3_INIT_SIMPLE(armv8_cortex_a76)
+PMUV3_INIT_SIMPLE(armv8_cortex_a77)
+PMUV3_INIT_SIMPLE(armv8_cortex_a78)
+PMUV3_INIT_SIMPLE(armv9_cortex_a510)
+PMUV3_INIT_SIMPLE(armv9_cortex_a710)
+PMUV3_INIT_SIMPLE(armv8_cortex_x1)
+PMUV3_INIT_SIMPLE(armv9_cortex_x2)
+PMUV3_INIT_SIMPLE(armv8_neoverse_e1)
+PMUV3_INIT_SIMPLE(armv8_neoverse_n1)
+PMUV3_INIT_SIMPLE(armv9_neoverse_n2)
+PMUV3_INIT_SIMPLE(armv8_neoverse_v1)
+
+PMUV3_INIT_SIMPLE(armv8_nvidia_carmel)
+PMUV3_INIT_SIMPLE(armv8_nvidia_denver)
+
+static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu)
+{
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a35",
+				       armv8_a53_map_event);
+}
+
+static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
+{
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a53",
+				       armv8_a53_map_event);
+}
+
+static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)
+{
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a57",
+				       armv8_a57_map_event);
+}
+
+static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu)
+{
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a72",
+				       armv8_a57_map_event);
+}
+
+static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu)
+{
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a73",
+				       armv8_a73_map_event);
+}
+
+static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
+{
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cavium_thunder",
+				       armv8_thunder_map_event);
+}
+
+static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu)
+{
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_brcm_vulcan",
+				       armv8_vulcan_map_event);
+}
+
+static const struct of_device_id armv8_pmu_of_device_ids[] = {
+	{.compatible = "arm,armv8-pmuv3",	.data = armv8_pmuv3_pmu_init},
+	{.compatible = "arm,cortex-a34-pmu",	.data = armv8_cortex_a34_pmu_init},
+	{.compatible = "arm,cortex-a35-pmu",	.data = armv8_a35_pmu_init},
+	{.compatible = "arm,cortex-a53-pmu",	.data = armv8_a53_pmu_init},
+	{.compatible = "arm,cortex-a55-pmu",	.data = armv8_cortex_a55_pmu_init},
+	{.compatible = "arm,cortex-a57-pmu",	.data = armv8_a57_pmu_init},
+	{.compatible = "arm,cortex-a65-pmu",	.data = armv8_cortex_a65_pmu_init},
+	{.compatible = "arm,cortex-a72-pmu",	.data = armv8_a72_pmu_init},
+	{.compatible = "arm,cortex-a73-pmu",	.data = armv8_a73_pmu_init},
+	{.compatible = "arm,cortex-a75-pmu",	.data = armv8_cortex_a75_pmu_init},
+	{.compatible = "arm,cortex-a76-pmu",	.data = armv8_cortex_a76_pmu_init},
+	{.compatible = "arm,cortex-a77-pmu",	.data = armv8_cortex_a77_pmu_init},
+	{.compatible = "arm,cortex-a78-pmu",	.data = armv8_cortex_a78_pmu_init},
+	{.compatible = "arm,cortex-a510-pmu",	.data = armv9_cortex_a510_pmu_init},
+	{.compatible = "arm,cortex-a710-pmu",	.data = armv9_cortex_a710_pmu_init},
+	{.compatible = "arm,cortex-x1-pmu",	.data = armv8_cortex_x1_pmu_init},
+	{.compatible = "arm,cortex-x2-pmu",	.data = armv9_cortex_x2_pmu_init},
+	{.compatible = "arm,neoverse-e1-pmu",	.data = armv8_neoverse_e1_pmu_init},
+	{.compatible = "arm,neoverse-n1-pmu",	.data = armv8_neoverse_n1_pmu_init},
+	{.compatible = "arm,neoverse-n2-pmu",	.data = armv9_neoverse_n2_pmu_init},
+	{.compatible = "arm,neoverse-v1-pmu",	.data = armv8_neoverse_v1_pmu_init},
+	{.compatible = "cavium,thunder-pmu",	.data = armv8_thunder_pmu_init},
+	{.compatible = "brcm,vulcan-pmu",	.data = armv8_vulcan_pmu_init},
+	{.compatible = "nvidia,carmel-pmu",	.data = armv8_nvidia_carmel_pmu_init},
+	{.compatible = "nvidia,denver-pmu",	.data = armv8_nvidia_denver_pmu_init},
+	{},
+};
+
+static int armv8_pmu_device_probe(struct platform_device *pdev)
+{
+	return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids, NULL);
+}
+
+static struct platform_driver armv8_pmu_driver = {
+	.driver		= {
+		.name	= ARMV8_PMU_PDEV_NAME,
+		.of_match_table = armv8_pmu_of_device_ids,
+		.suppress_bind_attrs = true,
+	},
+	.probe		= armv8_pmu_device_probe,
+};
+
+static int __init armv8_pmu_driver_init(void)
+{
+	if (acpi_disabled)
+		return platform_driver_register(&armv8_pmu_driver);
+	else
+		return arm_pmu_acpi_probe(armv8_pmuv3_pmu_init);
+}
+device_initcall(armv8_pmu_driver_init)
+
+void arch_perf_update_userpage(struct perf_event *event,
+			       struct perf_event_mmap_page *userpg, u64 now)
+{
+	struct clock_read_data *rd;
+	unsigned int seq;
+	u64 ns;
+
+	userpg->cap_user_time = 0;
+	userpg->cap_user_time_zero = 0;
+	userpg->cap_user_time_short = 0;
+	userpg->cap_user_rdpmc = armv8pmu_event_has_user_read(event);
+
+	if (userpg->cap_user_rdpmc) {
+		if (event->hw.flags & ARMPMU_EVT_64BIT)
+			userpg->pmc_width = 64;
+		else
+			userpg->pmc_width = 32;
+	}
+
+	do {
+		rd = sched_clock_read_begin(&seq);
+
+		if (rd->read_sched_clock != arch_timer_read_counter)
+			return;
+
+		userpg->time_mult = rd->mult;
+		userpg->time_shift = rd->shift;
+		userpg->time_zero = rd->epoch_ns;
+		userpg->time_cycles = rd->epoch_cyc;
+		userpg->time_mask = rd->sched_clock_mask;
+
+		/*
+		 * Subtract the cycle base, such that software that
+		 * doesn't know about cap_user_time_short still 'works'
+		 * assuming no wraps.
+		 */
+		ns = mul_u64_u32_shr(rd->epoch_cyc, rd->mult, rd->shift);
+		userpg->time_zero -= ns;
+
+	} while (sched_clock_read_retry(seq));
+
+	userpg->time_offset = userpg->time_zero - now;
+
+	/*
+	 * time_shift is not expected to be greater than 31 due to
+	 * the original published conversion algorithm shifting a
+	 * 32-bit value (now specifies a 64-bit value) - refer
+	 * perf_event_mmap_page documentation in perf_event.h.
+	 */
+	if (userpg->time_shift == 32) {
+		userpg->time_shift = 31;
+		userpg->time_mult >>= 1;
+	}
+
+	/*
+	 * Internal timekeeping for enabled/running/stopped times
+	 * is always computed with the sched_clock.
+	 */
+	userpg->cap_user_time = 1;
+	userpg->cap_user_time_zero = 1;
+	userpg->cap_user_time_short = 1;
+}
diff --git a/drivers/perf/hisilicon/hisi_uncore_cpa_pmu.c b/drivers/perf/hisilicon/hisi_uncore_cpa_pmu.c
index 4c67d57217a7..40f1bc9f9b91 100644
--- a/drivers/perf/hisilicon/hisi_uncore_cpa_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_cpa_pmu.c
@@ -316,7 +316,7 @@ static int hisi_cpa_pmu_probe(struct platform_device *pdev)
 	if (!name)
 		return -ENOMEM;
 
-	hisi_pmu_init(cpa_pmu, name, THIS_MODULE);
+	hisi_pmu_init(cpa_pmu, THIS_MODULE);
 
 	/* Power Management should be disabled before using CPA PMU. */
 	hisi_cpa_pmu_disable_pm(cpa_pmu);
diff --git a/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c b/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c
index 8c3ffcbfd4c0..ffb039d05d07 100644
--- a/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c
@@ -499,13 +499,6 @@ static int hisi_ddrc_pmu_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
-				       &ddrc_pmu->node);
-	if (ret) {
-		dev_err(&pdev->dev, "Error %d registering hotplug;\n", ret);
-		return ret;
-	}
-
 	if (ddrc_pmu->identifier >= HISI_PMU_V2)
 		name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
 				      "hisi_sccl%u_ddrc%u_%u",
@@ -516,7 +509,17 @@ static int hisi_ddrc_pmu_probe(struct platform_device *pdev)
 				      "hisi_sccl%u_ddrc%u", ddrc_pmu->sccl_id,
 				      ddrc_pmu->index_id);
 
-	hisi_pmu_init(ddrc_pmu, name, THIS_MODULE);
+	if (!name)
+		return -ENOMEM;
+
+	ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
+				       &ddrc_pmu->node);
+	if (ret) {
+		dev_err(&pdev->dev, "Error %d registering hotplug;\n", ret);
+		return ret;
+	}
+
+	hisi_pmu_init(ddrc_pmu, THIS_MODULE);
 
 	ret = perf_pmu_register(&ddrc_pmu->pmu, name, -1);
 	if (ret) {
diff --git a/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c b/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c
index 806698b9eabf..15caf99e1eef 100644
--- a/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c
@@ -510,6 +510,11 @@ static int hisi_hha_pmu_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_hha%u",
+			      hha_pmu->sccl_id, hha_pmu->index_id);
+	if (!name)
+		return -ENOMEM;
+
 	ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
 				       &hha_pmu->node);
 	if (ret) {
@@ -517,9 +522,7 @@ static int hisi_hha_pmu_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_hha%u",
-			      hha_pmu->sccl_id, hha_pmu->index_id);
-	hisi_pmu_init(hha_pmu, name, THIS_MODULE);
+	hisi_pmu_init(hha_pmu, THIS_MODULE);
 
 	ret = perf_pmu_register(&hha_pmu->pmu, name, -1);
 	if (ret) {
diff --git a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c
index 5b2c35f1658a..794dbcd19b7a 100644
--- a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c
@@ -544,6 +544,11 @@ static int hisi_l3c_pmu_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_l3c%u",
+			      l3c_pmu->sccl_id, l3c_pmu->ccl_id);
+	if (!name)
+		return -ENOMEM;
+
 	ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
 				       &l3c_pmu->node);
 	if (ret) {
@@ -551,13 +556,7 @@ static int hisi_l3c_pmu_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	/*
-	 * CCL_ID is used to identify the L3C in the same SCCL which was
-	 * used _UID by mistake.
-	 */
-	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_l3c%u",
-			      l3c_pmu->sccl_id, l3c_pmu->ccl_id);
-	hisi_pmu_init(l3c_pmu, name, THIS_MODULE);
+	hisi_pmu_init(l3c_pmu, THIS_MODULE);
 
 	ret = perf_pmu_register(&l3c_pmu->pmu, name, -1);
 	if (ret) {
diff --git a/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c
index afe3419f3f6d..71b6687d6696 100644
--- a/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c
@@ -412,7 +412,7 @@ static int hisi_pa_pmu_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	hisi_pmu_init(pa_pmu, name, THIS_MODULE);
+	hisi_pmu_init(pa_pmu, THIS_MODULE);
 	ret = perf_pmu_register(&pa_pmu->pmu, name, -1);
 	if (ret) {
 		dev_err(pa_pmu->dev, "PMU register failed, ret = %d\n", ret);
diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pmu.c
index f1b0f5e1a28f..2823f381930d 100644
--- a/drivers/perf/hisilicon/hisi_uncore_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_pmu.c
@@ -531,12 +531,10 @@ int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 }
 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_offline_cpu);
 
-void hisi_pmu_init(struct hisi_pmu *hisi_pmu, const char *name,
-		   struct module *module)
+void hisi_pmu_init(struct hisi_pmu *hisi_pmu, struct module *module)
 {
 	struct pmu *pmu = &hisi_pmu->pmu;
 
-	pmu->name               = name;
 	pmu->module             = module;
 	pmu->task_ctx_nr        = perf_invalid_context;
 	pmu->event_init         = hisi_uncore_pmu_event_init;
diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.h b/drivers/perf/hisilicon/hisi_uncore_pmu.h
index f8e3cc6903d7..07890a8e96ca 100644
--- a/drivers/perf/hisilicon/hisi_uncore_pmu.h
+++ b/drivers/perf/hisilicon/hisi_uncore_pmu.h
@@ -121,6 +121,5 @@ ssize_t hisi_uncore_pmu_identifier_attr_show(struct device *dev,
 int hisi_uncore_pmu_init_irq(struct hisi_pmu *hisi_pmu,
 			     struct platform_device *pdev);
 
-void hisi_pmu_init(struct hisi_pmu *hisi_pmu, const char *name,
-		   struct module *module);
+void hisi_pmu_init(struct hisi_pmu *hisi_pmu, struct module *module);
 #endif /* __HISI_UNCORE_PMU_H__ */
diff --git a/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c b/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c
index 1e354433776a..6fe534a665ed 100644
--- a/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c
@@ -445,7 +445,7 @@ static int hisi_sllc_pmu_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	hisi_pmu_init(sllc_pmu, name, THIS_MODULE);
+	hisi_pmu_init(sllc_pmu, THIS_MODULE);
 
 	ret = perf_pmu_register(&sllc_pmu->pmu, name, -1);
 	if (ret) {
diff --git a/drivers/perf/qcom_l3_pmu.c b/drivers/perf/qcom_l3_pmu.c
index 346311a05460..2887edb4eb0b 100644
--- a/drivers/perf/qcom_l3_pmu.c
+++ b/drivers/perf/qcom_l3_pmu.c
@@ -763,8 +763,7 @@ static int qcom_l3_cache_pmu_probe(struct platform_device *pdev)
 		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
 	};
 
-	memrc = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	l3pmu->regs = devm_ioremap_resource(&pdev->dev, memrc);
+	l3pmu->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &memrc);
 	if (IS_ERR(l3pmu->regs))
 		return PTR_ERR(l3pmu->regs);
 
diff --git a/drivers/phy/intel/phy-intel-lgm-combo.c b/drivers/phy/intel/phy-intel-lgm-combo.c
index 6010e246d52e..8c764c457c1c 100644
--- a/drivers/phy/intel/phy-intel-lgm-combo.c
+++ b/drivers/phy/intel/phy-intel-lgm-combo.c
@@ -616,4 +616,3 @@ static struct platform_driver intel_cbphy_driver = {
 module_platform_driver(intel_cbphy_driver);
 
 MODULE_DESCRIPTION("Intel Combo-phy driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/mscc/phy-ocelot-serdes.c b/drivers/phy/mscc/phy-ocelot-serdes.c
index 76f596365176..d9443e865a78 100644
--- a/drivers/phy/mscc/phy-ocelot-serdes.c
+++ b/drivers/phy/mscc/phy-ocelot-serdes.c
@@ -494,6 +494,7 @@ static int serdes_probe(struct platform_device *pdev)
 {
 	struct phy_provider *provider;
 	struct serdes_ctrl *ctrl;
+	struct resource *res;
 	unsigned int i;
 	int ret;
 
@@ -503,6 +504,14 @@ static int serdes_probe(struct platform_device *pdev)
 
 	ctrl->dev = &pdev->dev;
 	ctrl->regs = syscon_node_to_regmap(pdev->dev.parent->of_node);
+	if (IS_ERR(ctrl->regs)) {
+		/* Fall back to using IORESOURCE_REG, if possible */
+		res = platform_get_resource(pdev, IORESOURCE_REG, 0);
+		if (res)
+			ctrl->regs = dev_get_regmap(ctrl->dev->parent,
+						    res->name);
+	}
+
 	if (IS_ERR(ctrl->regs))
 		return PTR_ERR(ctrl->regs);
 
diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c
index 9951efc03eaa..6464dcb56d56 100644
--- a/drivers/phy/phy-core.c
+++ b/drivers/phy/phy-core.c
@@ -1233,7 +1233,7 @@ static void phy_release(struct device *dev)
 
 static int __init phy_core_init(void)
 {
-	phy_class = class_create(THIS_MODULE, "phy");
+	phy_class = class_create("phy");
 	if (IS_ERR(phy_class)) {
 		pr_err("failed to create phy class --> %ld\n",
 			PTR_ERR(phy_class));
diff --git a/drivers/pinctrl/actions/pinctrl-s500.c b/drivers/pinctrl/actions/pinctrl-s500.c
index ced778079b76..3bed4b8d08e6 100644
--- a/drivers/pinctrl/actions/pinctrl-s500.c
+++ b/drivers/pinctrl/actions/pinctrl-s500.c
@@ -1724,4 +1724,3 @@ module_exit(s500_pinctrl_exit);
 MODULE_AUTHOR("Actions Semi Inc.");
 MODULE_AUTHOR("Cristian Ciocaltea <cristian.ciocaltea@gmail.com>");
 MODULE_DESCRIPTION("Actions Semi S500 SoC Pinctrl Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/actions/pinctrl-s700.c b/drivers/pinctrl/actions/pinctrl-s700.c
index fd00940a5799..c2b472660e53 100644
--- a/drivers/pinctrl/actions/pinctrl-s700.c
+++ b/drivers/pinctrl/actions/pinctrl-s700.c
@@ -1908,4 +1908,3 @@ module_exit(s700_pinctrl_exit);
 
 MODULE_AUTHOR("Actions Semi Inc.");
 MODULE_DESCRIPTION("Actions Semi S700 Soc Pinctrl Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/actions/pinctrl-s900.c b/drivers/pinctrl/actions/pinctrl-s900.c
index 811249a8011e..8638d3007cd9 100644
--- a/drivers/pinctrl/actions/pinctrl-s900.c
+++ b/drivers/pinctrl/actions/pinctrl-s900.c
@@ -1827,4 +1827,3 @@ module_exit(s900_pinctrl_exit);
 MODULE_AUTHOR("Actions Semi Inc.");
 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
 MODULE_DESCRIPTION("Actions Semi S900 SoC Pinctrl Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/bcm/pinctrl-ns.c b/drivers/pinctrl/bcm/pinctrl-ns.c
index 465cc96814a1..f80630a74d34 100644
--- a/drivers/pinctrl/bcm/pinctrl-ns.c
+++ b/drivers/pinctrl/bcm/pinctrl-ns.c
@@ -299,5 +299,4 @@ static struct platform_driver ns_pinctrl_driver = {
 module_platform_driver(ns_pinctrl_driver);
 
 MODULE_AUTHOR("Rafał Miłecki");
-MODULE_LICENSE("GPL v2");
 MODULE_DEVICE_TABLE(of, ns_pinctrl_of_match_table);
diff --git a/drivers/pinctrl/core.c b/drivers/pinctrl/core.c
index d6e6c751255f..401886c81344 100644
--- a/drivers/pinctrl/core.c
+++ b/drivers/pinctrl/core.c
@@ -30,7 +30,6 @@
 
 #ifdef CONFIG_GPIOLIB
 #include "../gpio/gpiolib.h"
-#include <asm-generic/gpio.h>
 #endif
 
 #include "core.h"
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8188.c b/drivers/pinctrl/mediatek/pinctrl-mt8188.c
index 6a3d0126288e..c067e043e619 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt8188.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt8188.c
@@ -1670,5 +1670,4 @@ static int __init mt8188_pinctrl_init(void)
 
 arch_initcall(mt8188_pinctrl_init);
 
-MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("MediaTek MT8188 Pinctrl Driver");
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8192.c b/drivers/pinctrl/mediatek/pinctrl-mt8192.c
index 9695f4ec6aba..dee1b3aefd36 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt8192.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt8192.c
@@ -1431,5 +1431,4 @@ static int __init mt8192_pinctrl_init(void)
 }
 arch_initcall(mt8192_pinctrl_init);
 
-MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("MediaTek MT8192 Pinctrl Driver");
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8365.c b/drivers/pinctrl/mediatek/pinctrl-mt8365.c
index db4492e9ee67..75a505035e96 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt8365.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt8365.c
@@ -495,6 +495,5 @@ static int __init mtk_pinctrl_init(void)
 }
 arch_initcall(mtk_pinctrl_init);
 
-MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("MediaTek MT8365 Pinctrl Driver");
 MODULE_AUTHOR("Zhiyong Tao <zhiyong.tao@mediatek.com>");
diff --git a/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c b/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c
index ff5bcea172e8..4e12b3768d65 100644
--- a/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c
+++ b/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c
@@ -2046,7 +2046,6 @@ static int __init npcm7xx_pinctrl_register(void)
 }
 arch_initcall(npcm7xx_pinctrl_register);
 
-MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("jordan_hargrave@dell.com");
 MODULE_AUTHOR("tomer.maimon@nuvoton.com");
 MODULE_DESCRIPTION("Nuvoton NPCM7XX Pinctrl and GPIO driver");
diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c
index 9236a132c7ba..0cc00e9dbcf0 100644
--- a/drivers/pinctrl/pinctrl-amd.c
+++ b/drivers/pinctrl/pinctrl-amd.c
@@ -1213,6 +1213,5 @@ static struct platform_driver amd_gpio_driver = {
 
 module_platform_driver(amd_gpio_driver);
 
-MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Ken Xue <Ken.Xue@amd.com>, Jeff Wu <Jeff.Wu@amd.com>");
 MODULE_DESCRIPTION("AMD GPIO pinctrl driver");
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c
index daeb79a9a602..4515f375c5e8 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm.c
@@ -323,6 +323,7 @@ static int msm_config_reg(struct msm_pinctrl *pctrl,
 		break;
 	case PIN_CONFIG_OUTPUT:
 	case PIN_CONFIG_INPUT_ENABLE:
+	case PIN_CONFIG_OUTPUT_ENABLE:
 		*bit = g->oe_bit;
 		*mask = 1;
 		break;
@@ -414,11 +415,9 @@ static int msm_config_group_get(struct pinctrl_dev *pctldev,
 		val = msm_readl_io(pctrl, g);
 		arg = !!(val & BIT(g->in_bit));
 		break;
-	case PIN_CONFIG_INPUT_ENABLE:
-		/* Pin is output */
-		if (arg)
+	case PIN_CONFIG_OUTPUT_ENABLE:
+		if (!arg)
 			return -EINVAL;
-		arg = 1;
 		break;
 	default:
 		return -ENOTSUPP;
@@ -502,9 +501,36 @@ static int msm_config_group_set(struct pinctrl_dev *pctldev,
 			arg = 1;
 			break;
 		case PIN_CONFIG_INPUT_ENABLE:
-			/* disable output */
+			/*
+			 * According to pinctrl documentation this should
+			 * actually be a no-op.
+			 *
+			 * The docs are explicit that "this does not affect
+			 * the pin's ability to drive output" but what we do
+			 * here is to modify the output enable bit. Thus, to
+			 * follow the docs we should remove that.
+			 *
+			 * The docs say that we should enable any relevant
+			 * input buffer, but TLMM there is no input buffer that
+			 * can be enabled/disabled. It's always on.
+			 *
+			 * The points above, explain why this _should_ be a
+			 * no-op. However, for historical reasons and to
+			 * support old device trees, we'll violate the docs
+			 * still affect the output.
+			 *
+			 * It should further be noted that this old historical
+			 * behavior actually overrides arg to 0. That means
+			 * that "input-enable" and "input-disable" in a device
+			 * tree would _both_ disable the output. We'll
+			 * continue to preserve this behavior as well since
+			 * we have no other use for this attribute.
+			 */
 			arg = 0;
 			break;
+		case PIN_CONFIG_OUTPUT_ENABLE:
+			arg = !!arg;
+			break;
 		default:
 			dev_err(pctrl->dev, "Unsupported config parameter: %x\n",
 				param);
diff --git a/drivers/pinctrl/renesas/pinctrl-rza1.c b/drivers/pinctrl/renesas/pinctrl-rza1.c
index 529c0fc4ec06..48173355a040 100644
--- a/drivers/pinctrl/renesas/pinctrl-rza1.c
+++ b/drivers/pinctrl/renesas/pinctrl-rza1.c
@@ -1407,4 +1407,3 @@ core_initcall(rza1_pinctrl_init);
 
 MODULE_AUTHOR("Jacopo Mondi <jacopo+renesas@jmondi.org");
 MODULE_DESCRIPTION("Pin and gpio controller driver for Reneas RZ/A1 SoC");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/renesas/pinctrl-rza2.c b/drivers/pinctrl/renesas/pinctrl-rza2.c
index c0a04f1ee994..40b1326a1077 100644
--- a/drivers/pinctrl/renesas/pinctrl-rza2.c
+++ b/drivers/pinctrl/renesas/pinctrl-rza2.c
@@ -514,4 +514,3 @@ core_initcall(rza2_pinctrl_init);
 
 MODULE_AUTHOR("Chris Brandt <chris.brandt@renesas.com>");
 MODULE_DESCRIPTION("Pin and gpio controller driver for RZ/A2 SoC");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
index 04b31f0c6b34..9511d920565e 100644
--- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c
+++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
@@ -1570,4 +1570,3 @@ core_initcall(rzg2l_pinctrl_init);
 
 MODULE_AUTHOR("Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>");
 MODULE_DESCRIPTION("Pin and gpio controller driver for RZ/G2L family");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/renesas/pinctrl-rzn1.c b/drivers/pinctrl/renesas/pinctrl-rzn1.c
index 9158c1757492..d73741651419 100644
--- a/drivers/pinctrl/renesas/pinctrl-rzn1.c
+++ b/drivers/pinctrl/renesas/pinctrl-rzn1.c
@@ -952,4 +952,3 @@ subsys_initcall(_pinctrl_drv_register);
 
 MODULE_AUTHOR("Phil Edworthy <phil.edworthy@renesas.com>");
 MODULE_DESCRIPTION("Renesas RZ/N1 pinctrl driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/renesas/pinctrl-rzv2m.c b/drivers/pinctrl/renesas/pinctrl-rzv2m.c
index 3b65a71abd9a..e5472293bc7f 100644
--- a/drivers/pinctrl/renesas/pinctrl-rzv2m.c
+++ b/drivers/pinctrl/renesas/pinctrl-rzv2m.c
@@ -1117,4 +1117,3 @@ core_initcall(rzv2m_pinctrl_init);
 
 MODULE_AUTHOR("Phil Edworthy <phil.edworthy@renesas.com>");
 MODULE_DESCRIPTION("Pin and gpio controller driver for RZ/V2M");
-MODULE_LICENSE("GPL");
diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c
index b895c8130bba..8b7949220382 100644
--- a/drivers/platform/chrome/cros_ec.c
+++ b/drivers/platform/chrome/cros_ec.c
@@ -206,7 +206,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 	err = cros_ec_query_all(ec_dev);
 	if (err) {
 		dev_err(dev, "Cannot identify the EC: error %d\n", err);
-		goto destroy_mutex;
+		goto exit;
 	}
 
 	if (ec_dev->irq > 0) {
@@ -218,7 +218,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 		if (err) {
 			dev_err(dev, "Failed to request IRQ %d: %d\n",
 				ec_dev->irq, err);
-			goto destroy_mutex;
+			goto exit;
 		}
 	}
 
@@ -230,7 +230,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 		dev_err(ec_dev->dev,
 			"Failed to create CrOS EC platform device\n");
 		err = PTR_ERR(ec_dev->ec);
-		goto destroy_mutex;
+		goto exit;
 	}
 
 	if (ec_dev->max_passthru) {
@@ -296,7 +296,6 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 exit:
 	platform_device_unregister(ec_dev->ec);
 	platform_device_unregister(ec_dev->pd);
-destroy_mutex:
 	mutex_destroy(&ec_dev->lock);
 	lockdep_unregister_key(&ec_dev->lockdep_key);
 	return err;
@@ -313,8 +312,7 @@ EXPORT_SYMBOL(cros_ec_register);
  */
 void cros_ec_unregister(struct cros_ec_device *ec_dev)
 {
-	if (ec_dev->pd)
-		platform_device_unregister(ec_dev->pd);
+	platform_device_unregister(ec_dev->pd);
 	platform_device_unregister(ec_dev->ec);
 	mutex_destroy(&ec_dev->lock);
 	lockdep_unregister_key(&ec_dev->lockdep_key);
diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c
index a98c529d8c69..c876120e0ebc 100644
--- a/drivers/platform/chrome/cros_ec_debugfs.c
+++ b/drivers/platform/chrome/cros_ec_debugfs.c
@@ -400,24 +400,48 @@ static void cros_ec_cleanup_console_log(struct cros_ec_debugfs *debug_info)
 	}
 }
 
-static int cros_ec_create_panicinfo(struct cros_ec_debugfs *debug_info)
+/*
+ * Returns the size of the panicinfo data fetched from the EC
+ */
+static int cros_ec_get_panicinfo(struct cros_ec_device *ec_dev, uint8_t *data,
+				 int data_size)
 {
-	struct cros_ec_device *ec_dev = debug_info->ec->ec_dev;
 	int ret;
 	struct cros_ec_command *msg;
-	int insize;
 
-	insize = ec_dev->max_response;
+	if (!data || data_size <= 0 || data_size > ec_dev->max_response)
+		return -EINVAL;
 
-	msg = devm_kzalloc(debug_info->ec->dev,
-			sizeof(*msg) + insize, GFP_KERNEL);
+	msg = kzalloc(sizeof(*msg) + data_size, GFP_KERNEL);
 	if (!msg)
 		return -ENOMEM;
 
 	msg->command = EC_CMD_GET_PANIC_INFO;
-	msg->insize = insize;
+	msg->insize = data_size;
 
 	ret = cros_ec_cmd_xfer_status(ec_dev, msg);
+	if (ret < 0)
+		goto free;
+
+	memcpy(data, msg->data, data_size);
+
+free:
+	kfree(msg);
+	return ret;
+}
+
+static int cros_ec_create_panicinfo(struct cros_ec_debugfs *debug_info)
+{
+	struct cros_ec_device *ec_dev = debug_info->ec->ec_dev;
+	int ret;
+	void *data;
+
+	data = devm_kzalloc(debug_info->ec->dev, ec_dev->max_response,
+			    GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	ret = cros_ec_get_panicinfo(ec_dev, data, ec_dev->max_response);
 	if (ret < 0) {
 		ret = 0;
 		goto free;
@@ -427,7 +451,7 @@ static int cros_ec_create_panicinfo(struct cros_ec_debugfs *debug_info)
 	if (ret == 0)
 		goto free;
 
-	debug_info->panicinfo_blob.data = msg->data;
+	debug_info->panicinfo_blob.data = data;
 	debug_info->panicinfo_blob.size = ret;
 
 	debugfs_create_blob("panicinfo", S_IFREG | 0444, debug_info->dir,
@@ -436,7 +460,7 @@ static int cros_ec_create_panicinfo(struct cros_ec_debugfs *debug_info)
 	return 0;
 
 free:
-	devm_kfree(debug_info->ec->dev, msg);
+	devm_kfree(debug_info->ec->dev, data);
 	return ret;
 }
 
diff --git a/drivers/platform/chrome/cros_typec_switch.c b/drivers/platform/chrome/cros_typec_switch.c
index 9ed1605f4071..752720483753 100644
--- a/drivers/platform/chrome/cros_typec_switch.c
+++ b/drivers/platform/chrome/cros_typec_switch.c
@@ -270,6 +270,7 @@ static int cros_typec_register_switches(struct cros_typec_switch_data *sdata)
 
 	return 0;
 err_switch:
+	fwnode_handle_put(fwnode);
 	cros_typec_unregister_switches(sdata);
 	return ret;
 }
diff --git a/drivers/platform/chrome/wilco_ec/debugfs.c b/drivers/platform/chrome/wilco_ec/debugfs.c
index a812788a0bdc..7a13f13b16cd 100644
--- a/drivers/platform/chrome/wilco_ec/debugfs.c
+++ b/drivers/platform/chrome/wilco_ec/debugfs.c
@@ -251,8 +251,6 @@ static int wilco_ec_debugfs_probe(struct platform_device *pdev)
 		return 0;
 	debug_info->ec = ec;
 	debug_info->dir = debugfs_create_dir("wilco_ec", NULL);
-	if (!debug_info->dir)
-		return 0;
 	debugfs_create_file("raw", 0644, debug_info->dir, NULL, &fops_raw);
 	debugfs_create_file("h1_gpio", 0444, debug_info->dir, ec,
 			    &fops_h1_gpio);
diff --git a/drivers/platform/chrome/wilco_ec/event.c b/drivers/platform/chrome/wilco_ec/event.c
index 69ceead8cdaa..a40f60bcefb6 100644
--- a/drivers/platform/chrome/wilco_ec/event.c
+++ b/drivers/platform/chrome/wilco_ec/event.c
@@ -58,7 +58,6 @@
 #define DRV_NAME		EVENT_DEV_NAME
 #define EVENT_DEV_NAME_FMT	(EVENT_DEV_NAME "%d")
 static struct class event_class = {
-	.owner	= THIS_MODULE,
 	.name	= EVENT_CLASS_NAME,
 };
 
diff --git a/drivers/platform/chrome/wilco_ec/telemetry.c b/drivers/platform/chrome/wilco_ec/telemetry.c
index 60da7a29f2ff..54708aa6c700 100644
--- a/drivers/platform/chrome/wilco_ec/telemetry.c
+++ b/drivers/platform/chrome/wilco_ec/telemetry.c
@@ -42,7 +42,6 @@
 #define DRV_NAME		TELEM_DEV_NAME
 #define TELEM_DEV_NAME_FMT	(TELEM_DEV_NAME "%d")
 static struct class telem_class = {
-	.owner	= THIS_MODULE,
 	.name	= TELEM_CLASS_NAME,
 };
 
diff --git a/drivers/platform/mellanox/mlxbf-bootctl.c b/drivers/platform/mellanox/mlxbf-bootctl.c
index 1c7a288b59a5..1bad1d278672 100644
--- a/drivers/platform/mellanox/mlxbf-bootctl.c
+++ b/drivers/platform/mellanox/mlxbf-bootctl.c
@@ -10,6 +10,7 @@
 
 #include <linux/acpi.h>
 #include <linux/arm-smccc.h>
+#include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 
@@ -44,6 +45,10 @@ static const char * const mlxbf_bootctl_lifecycle_states[] = {
 	[3] = "RMA",
 };
 
+/* Mapped pointer for RSH_BOOT_FIFO_DATA and RSH_BOOT_FIFO_COUNT register. */
+static void __iomem *mlxbf_rsh_boot_data;
+static void __iomem *mlxbf_rsh_boot_cnt;
+
 /* ARM SMC call which is atomic and no need for lock. */
 static int mlxbf_bootctl_smc(unsigned int smc_op, int smc_arg)
 {
@@ -244,11 +249,29 @@ static ssize_t secure_boot_fuse_state_show(struct device *dev,
 	return buf_len;
 }
 
+static ssize_t fw_reset_store(struct device *dev,
+			      struct device_attribute *attr,
+			      const char *buf, size_t count)
+{
+	unsigned long key;
+	int err;
+
+	err = kstrtoul(buf, 16, &key);
+	if (err)
+		return err;
+
+	if (mlxbf_bootctl_smc(MLXBF_BOOTCTL_FW_RESET, key) < 0)
+		return -EINVAL;
+
+	return count;
+}
+
 static DEVICE_ATTR_RW(post_reset_wdog);
 static DEVICE_ATTR_RW(reset_action);
 static DEVICE_ATTR_RW(second_reset_action);
 static DEVICE_ATTR_RO(lifecycle_state);
 static DEVICE_ATTR_RO(secure_boot_fuse_state);
+static DEVICE_ATTR_WO(fw_reset);
 
 static struct attribute *mlxbf_bootctl_attrs[] = {
 	&dev_attr_post_reset_wdog.attr,
@@ -256,6 +279,7 @@ static struct attribute *mlxbf_bootctl_attrs[] = {
 	&dev_attr_second_reset_action.attr,
 	&dev_attr_lifecycle_state.attr,
 	&dev_attr_secure_boot_fuse_state.attr,
+	&dev_attr_fw_reset.attr,
 	NULL
 };
 
@@ -268,6 +292,45 @@ static const struct acpi_device_id mlxbf_bootctl_acpi_ids[] = {
 
 MODULE_DEVICE_TABLE(acpi, mlxbf_bootctl_acpi_ids);
 
+static ssize_t mlxbf_bootctl_bootfifo_read(struct file *filp,
+					   struct kobject *kobj,
+					   struct bin_attribute *bin_attr,
+					   char *buf, loff_t pos,
+					   size_t count)
+{
+	unsigned long timeout = msecs_to_jiffies(500);
+	unsigned long expire = jiffies + timeout;
+	u64 data, cnt = 0;
+	char *p = buf;
+
+	while (count >= sizeof(data)) {
+		/* Give up reading if no more data within 500ms. */
+		if (!cnt) {
+			cnt = readq(mlxbf_rsh_boot_cnt);
+			if (!cnt) {
+				if (time_after(jiffies, expire))
+					break;
+				usleep_range(10, 50);
+				continue;
+			}
+		}
+
+		data = readq(mlxbf_rsh_boot_data);
+		memcpy(p, &data, sizeof(data));
+		count -= sizeof(data);
+		p += sizeof(data);
+		cnt--;
+		expire = jiffies + timeout;
+	}
+
+	return p - buf;
+}
+
+static struct bin_attribute mlxbf_bootctl_bootfifo_sysfs_attr = {
+	.attr = { .name = "bootfifo", .mode = 0400 },
+	.read = mlxbf_bootctl_bootfifo_read,
+};
+
 static bool mlxbf_bootctl_guid_match(const guid_t *guid,
 				     const struct arm_smccc_res *res)
 {
@@ -285,6 +348,16 @@ static int mlxbf_bootctl_probe(struct platform_device *pdev)
 	guid_t guid;
 	int ret;
 
+	/* Get the resource of the bootfifo data register. */
+	mlxbf_rsh_boot_data = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(mlxbf_rsh_boot_data))
+		return PTR_ERR(mlxbf_rsh_boot_data);
+
+	/* Get the resource of the bootfifo counter register. */
+	mlxbf_rsh_boot_cnt = devm_platform_ioremap_resource(pdev, 1);
+	if (IS_ERR(mlxbf_rsh_boot_cnt))
+		return PTR_ERR(mlxbf_rsh_boot_cnt);
+
 	/* Ensure we have the UUID we expect for this service. */
 	arm_smccc_smc(MLXBF_BOOTCTL_SIP_SVC_UID, 0, 0, 0, 0, 0, 0, 0, &res);
 	guid_parse(mlxbf_bootctl_svc_uuid_str, &guid);
@@ -302,11 +375,25 @@ static int mlxbf_bootctl_probe(struct platform_device *pdev)
 	if (ret < 0)
 		dev_warn(&pdev->dev, "Unable to reset the EMMC boot mode\n");
 
+	ret = sysfs_create_bin_file(&pdev->dev.kobj,
+				    &mlxbf_bootctl_bootfifo_sysfs_attr);
+	if (ret)
+		pr_err("Unable to create bootfifo sysfs file, error %d\n", ret);
+
+	return ret;
+}
+
+static int mlxbf_bootctl_remove(struct platform_device *pdev)
+{
+	sysfs_remove_bin_file(&pdev->dev.kobj,
+			      &mlxbf_bootctl_bootfifo_sysfs_attr);
+
 	return 0;
 }
 
 static struct platform_driver mlxbf_bootctl_driver = {
 	.probe = mlxbf_bootctl_probe,
+	.remove = mlxbf_bootctl_remove,
 	.driver = {
 		.name = "mlxbf-bootctl",
 		.dev_groups = mlxbf_bootctl_groups,
diff --git a/drivers/platform/mellanox/mlxbf-bootctl.h b/drivers/platform/mellanox/mlxbf-bootctl.h
index 148fdb43b435..b48243f60a59 100644
--- a/drivers/platform/mellanox/mlxbf-bootctl.h
+++ b/drivers/platform/mellanox/mlxbf-bootctl.h
@@ -75,6 +75,12 @@
 
 #define MLXBF_BOOTCTL_GET_DIMM_INFO		0x82000008
 
+/*
+ * Initiate Firmware Reset via TYU. This might be invoked during the reset
+ * flow in isolation mode.
+ */
+#define MLXBF_BOOTCTL_FW_RESET  0x8200000D
+
 /* SMC function IDs for SiP Service queries */
 #define MLXBF_BOOTCTL_SIP_SVC_CALL_COUNT	0x8200ff00
 #define MLXBF_BOOTCTL_SIP_SVC_UID		0x8200ff01
diff --git a/drivers/platform/olpc/olpc-xo175-ec.c b/drivers/platform/olpc/olpc-xo175-ec.c
index 4823bd2819f6..62ccbcb15c74 100644
--- a/drivers/platform/olpc/olpc-xo175-ec.c
+++ b/drivers/platform/olpc/olpc-xo175-ec.c
@@ -746,6 +746,7 @@ static struct spi_driver olpc_xo175_ec_spi_driver = {
 		.of_match_table = olpc_xo175_ec_of_match,
 		.pm = &olpc_xo175_ec_pm_ops,
 	},
+	.id_table	= olpc_xo175_ec_id_table,
 	.probe		= olpc_xo175_ec_probe,
 	.remove		= olpc_xo175_ec_remove,
 };
diff --git a/drivers/platform/surface/surface_aggregator_registry.c b/drivers/platform/surface/surface_aggregator_registry.c
index 296f72d52e6a..0fe5be539652 100644
--- a/drivers/platform/surface/surface_aggregator_registry.c
+++ b/drivers/platform/surface/surface_aggregator_registry.c
@@ -305,7 +305,7 @@ static const struct software_node *ssam_node_group_sp9[] = {
 	&ssam_node_bat_ac,
 	&ssam_node_bat_main,
 	&ssam_node_tmp_pprof,
-	/* TODO: Tablet mode switch (via POS subsystem) */
+	&ssam_node_pos_tablet_switch,
 	&ssam_node_hid_kip_keyboard,
 	&ssam_node_hid_kip_penstash,
 	&ssam_node_hid_kip_touchpad,
diff --git a/drivers/platform/surface/surface_aggregator_tabletsw.c b/drivers/platform/surface/surface_aggregator_tabletsw.c
index 9fed800c7cc0..8f52b62d1c19 100644
--- a/drivers/platform/surface/surface_aggregator_tabletsw.c
+++ b/drivers/platform/surface/surface_aggregator_tabletsw.c
@@ -20,16 +20,23 @@
 
 struct ssam_tablet_sw;
 
+struct ssam_tablet_sw_state {
+	u32 source;
+	u32 state;
+};
+
 struct ssam_tablet_sw_ops {
-	int (*get_state)(struct ssam_tablet_sw *sw, u32 *state);
-	const char *(*state_name)(struct ssam_tablet_sw *sw, u32 state);
-	bool (*state_is_tablet_mode)(struct ssam_tablet_sw *sw, u32 state);
+	int (*get_state)(struct ssam_tablet_sw *sw, struct ssam_tablet_sw_state *state);
+	const char *(*state_name)(struct ssam_tablet_sw *sw,
+				  const struct ssam_tablet_sw_state *state);
+	bool (*state_is_tablet_mode)(struct ssam_tablet_sw *sw,
+				     const struct ssam_tablet_sw_state *state);
 };
 
 struct ssam_tablet_sw {
 	struct ssam_device *sdev;
 
-	u32 state;
+	struct ssam_tablet_sw_state state;
 	struct work_struct update_work;
 	struct input_dev *mode_switch;
 
@@ -45,9 +52,11 @@ struct ssam_tablet_sw_desc {
 
 	struct {
 		u32 (*notify)(struct ssam_event_notifier *nf, const struct ssam_event *event);
-		int (*get_state)(struct ssam_tablet_sw *sw, u32 *state);
-		const char *(*state_name)(struct ssam_tablet_sw *sw, u32 state);
-		bool (*state_is_tablet_mode)(struct ssam_tablet_sw *sw, u32 state);
+		int (*get_state)(struct ssam_tablet_sw *sw, struct ssam_tablet_sw_state *state);
+		const char *(*state_name)(struct ssam_tablet_sw *sw,
+					  const struct ssam_tablet_sw_state *state);
+		bool (*state_is_tablet_mode)(struct ssam_tablet_sw *sw,
+					     const struct ssam_tablet_sw_state *state);
 	} ops;
 
 	struct {
@@ -61,7 +70,7 @@ struct ssam_tablet_sw_desc {
 static ssize_t state_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct ssam_tablet_sw *sw = dev_get_drvdata(dev);
-	const char *state = sw->ops.state_name(sw, sw->state);
+	const char *state = sw->ops.state_name(sw, &sw->state);
 
 	return sysfs_emit(buf, "%s\n", state);
 }
@@ -79,19 +88,19 @@ static const struct attribute_group ssam_tablet_sw_group = {
 static void ssam_tablet_sw_update_workfn(struct work_struct *work)
 {
 	struct ssam_tablet_sw *sw = container_of(work, struct ssam_tablet_sw, update_work);
+	struct ssam_tablet_sw_state state;
 	int tablet, status;
-	u32 state;
 
 	status = sw->ops.get_state(sw, &state);
 	if (status)
 		return;
 
-	if (sw->state == state)
+	if (sw->state.source == state.source && sw->state.state == state.state)
 		return;
 	sw->state = state;
 
 	/* Send SW_TABLET_MODE event. */
-	tablet = sw->ops.state_is_tablet_mode(sw, state);
+	tablet = sw->ops.state_is_tablet_mode(sw, &state);
 	input_report_switch(sw->mode_switch, SW_TABLET_MODE, tablet);
 	input_sync(sw->mode_switch);
 }
@@ -146,7 +155,7 @@ static int ssam_tablet_sw_probe(struct ssam_device *sdev)
 	sw->mode_switch->id.bustype = BUS_HOST;
 	sw->mode_switch->dev.parent = &sdev->dev;
 
-	tablet = sw->ops.state_is_tablet_mode(sw, sw->state);
+	tablet = sw->ops.state_is_tablet_mode(sw, &sw->state);
 	input_set_capability(sw->mode_switch, EV_SW, SW_TABLET_MODE);
 	input_report_switch(sw->mode_switch, SW_TABLET_MODE, tablet);
 
@@ -203,9 +212,10 @@ enum ssam_kip_cover_state {
 	SSAM_KIP_COVER_STATE_FOLDED_BACK   = 0x05,
 };
 
-static const char *ssam_kip_cover_state_name(struct ssam_tablet_sw *sw, u32 state)
+static const char *ssam_kip_cover_state_name(struct ssam_tablet_sw *sw,
+					     const struct ssam_tablet_sw_state *state)
 {
-	switch (state) {
+	switch (state->state) {
 	case SSAM_KIP_COVER_STATE_DISCONNECTED:
 		return "disconnected";
 
@@ -222,14 +232,15 @@ static const char *ssam_kip_cover_state_name(struct ssam_tablet_sw *sw, u32 stat
 		return "folded-back";
 
 	default:
-		dev_warn(&sw->sdev->dev, "unknown KIP cover state: %u\n", state);
+		dev_warn(&sw->sdev->dev, "unknown KIP cover state: %u\n", state->state);
 		return "<unknown>";
 	}
 }
 
-static bool ssam_kip_cover_state_is_tablet_mode(struct ssam_tablet_sw *sw, u32 state)
+static bool ssam_kip_cover_state_is_tablet_mode(struct ssam_tablet_sw *sw,
+						const struct ssam_tablet_sw_state *state)
 {
-	switch (state) {
+	switch (state->state) {
 	case SSAM_KIP_COVER_STATE_DISCONNECTED:
 	case SSAM_KIP_COVER_STATE_FOLDED_CANVAS:
 	case SSAM_KIP_COVER_STATE_FOLDED_BACK:
@@ -240,7 +251,7 @@ static bool ssam_kip_cover_state_is_tablet_mode(struct ssam_tablet_sw *sw, u32 s
 		return false;
 
 	default:
-		dev_warn(&sw->sdev->dev, "unknown KIP cover state: %d\n", sw->state);
+		dev_warn(&sw->sdev->dev, "unknown KIP cover state: %d\n", state->state);
 		return true;
 	}
 }
@@ -252,7 +263,7 @@ SSAM_DEFINE_SYNC_REQUEST_R(__ssam_kip_get_cover_state, u8, {
 	.instance_id     = 0x00,
 });
 
-static int ssam_kip_get_cover_state(struct ssam_tablet_sw *sw, u32 *state)
+static int ssam_kip_get_cover_state(struct ssam_tablet_sw *sw, struct ssam_tablet_sw_state *state)
 {
 	int status;
 	u8 raw;
@@ -263,7 +274,8 @@ static int ssam_kip_get_cover_state(struct ssam_tablet_sw *sw, u32 *state)
 		return status;
 	}
 
-	*state = raw;
+	state->source = 0;	/* Unused for KIP switch. */
+	state->state = raw;
 	return 0;
 }
 
@@ -312,11 +324,24 @@ MODULE_PARM_DESC(tablet_mode_in_slate_state, "Enable tablet mode in slate device
 #define SSAM_EVENT_POS_CID_POSTURE_CHANGED	0x03
 #define SSAM_POS_MAX_SOURCES			4
 
-enum ssam_pos_state {
-	SSAM_POS_POSTURE_LID_CLOSED = 0x00,
-	SSAM_POS_POSTURE_LAPTOP     = 0x01,
-	SSAM_POS_POSTURE_SLATE      = 0x02,
-	SSAM_POS_POSTURE_TABLET     = 0x03,
+enum ssam_pos_source_id {
+	SSAM_POS_SOURCE_COVER = 0x00,
+	SSAM_POS_SOURCE_SLS   = 0x03,
+};
+
+enum ssam_pos_state_cover {
+	SSAM_POS_COVER_DISCONNECTED  = 0x01,
+	SSAM_POS_COVER_CLOSED        = 0x02,
+	SSAM_POS_COVER_LAPTOP        = 0x03,
+	SSAM_POS_COVER_FOLDED_CANVAS = 0x04,
+	SSAM_POS_COVER_FOLDED_BACK   = 0x05,
+};
+
+enum ssam_pos_state_sls {
+	SSAM_POS_SLS_LID_CLOSED = 0x00,
+	SSAM_POS_SLS_LAPTOP     = 0x01,
+	SSAM_POS_SLS_SLATE      = 0x02,
+	SSAM_POS_SLS_TABLET     = 0x03,
 };
 
 struct ssam_sources_list {
@@ -324,42 +349,116 @@ struct ssam_sources_list {
 	__le32 id[SSAM_POS_MAX_SOURCES];
 } __packed;
 
-static const char *ssam_pos_state_name(struct ssam_tablet_sw *sw, u32 state)
+static const char *ssam_pos_state_name_cover(struct ssam_tablet_sw *sw, u32 state)
+{
+	switch (state) {
+	case SSAM_POS_COVER_DISCONNECTED:
+		return "disconnected";
+
+	case SSAM_POS_COVER_CLOSED:
+		return "closed";
+
+	case SSAM_POS_COVER_LAPTOP:
+		return "laptop";
+
+	case SSAM_POS_COVER_FOLDED_CANVAS:
+		return "folded-canvas";
+
+	case SSAM_POS_COVER_FOLDED_BACK:
+		return "folded-back";
+
+	default:
+		dev_warn(&sw->sdev->dev, "unknown device posture for type-cover: %u\n", state);
+		return "<unknown>";
+	}
+}
+
+static const char *ssam_pos_state_name_sls(struct ssam_tablet_sw *sw, u32 state)
 {
 	switch (state) {
-	case SSAM_POS_POSTURE_LID_CLOSED:
+	case SSAM_POS_SLS_LID_CLOSED:
 		return "closed";
 
-	case SSAM_POS_POSTURE_LAPTOP:
+	case SSAM_POS_SLS_LAPTOP:
 		return "laptop";
 
-	case SSAM_POS_POSTURE_SLATE:
+	case SSAM_POS_SLS_SLATE:
 		return "slate";
 
-	case SSAM_POS_POSTURE_TABLET:
+	case SSAM_POS_SLS_TABLET:
 		return "tablet";
 
 	default:
-		dev_warn(&sw->sdev->dev, "unknown device posture: %u\n", state);
+		dev_warn(&sw->sdev->dev, "unknown device posture for SLS: %u\n", state);
 		return "<unknown>";
 	}
 }
 
-static bool ssam_pos_state_is_tablet_mode(struct ssam_tablet_sw *sw, u32 state)
+static const char *ssam_pos_state_name(struct ssam_tablet_sw *sw,
+				       const struct ssam_tablet_sw_state *state)
+{
+	switch (state->source) {
+	case SSAM_POS_SOURCE_COVER:
+		return ssam_pos_state_name_cover(sw, state->state);
+
+	case SSAM_POS_SOURCE_SLS:
+		return ssam_pos_state_name_sls(sw, state->state);
+
+	default:
+		dev_warn(&sw->sdev->dev, "unknown device posture source: %u\n", state->source);
+		return "<unknown>";
+	}
+}
+
+static bool ssam_pos_state_is_tablet_mode_cover(struct ssam_tablet_sw *sw, u32 state)
 {
 	switch (state) {
-	case SSAM_POS_POSTURE_LAPTOP:
-	case SSAM_POS_POSTURE_LID_CLOSED:
+	case SSAM_POS_COVER_DISCONNECTED:
+	case SSAM_POS_COVER_FOLDED_CANVAS:
+	case SSAM_POS_COVER_FOLDED_BACK:
+		return true;
+
+	case SSAM_POS_COVER_CLOSED:
+	case SSAM_POS_COVER_LAPTOP:
 		return false;
 
-	case SSAM_POS_POSTURE_SLATE:
+	default:
+		dev_warn(&sw->sdev->dev, "unknown device posture for type-cover: %u\n", state);
+		return true;
+	}
+}
+
+static bool ssam_pos_state_is_tablet_mode_sls(struct ssam_tablet_sw *sw, u32 state)
+{
+	switch (state) {
+	case SSAM_POS_SLS_LAPTOP:
+	case SSAM_POS_SLS_LID_CLOSED:
+		return false;
+
+	case SSAM_POS_SLS_SLATE:
 		return tablet_mode_in_slate_state;
 
-	case SSAM_POS_POSTURE_TABLET:
+	case SSAM_POS_SLS_TABLET:
 		return true;
 
 	default:
-		dev_warn(&sw->sdev->dev, "unknown device posture: %u\n", state);
+		dev_warn(&sw->sdev->dev, "unknown device posture for SLS: %u\n", state);
+		return true;
+	}
+}
+
+static bool ssam_pos_state_is_tablet_mode(struct ssam_tablet_sw *sw,
+					  const struct ssam_tablet_sw_state *state)
+{
+	switch (state->source) {
+	case SSAM_POS_SOURCE_COVER:
+		return ssam_pos_state_is_tablet_mode_cover(sw, state->state);
+
+	case SSAM_POS_SOURCE_SLS:
+		return ssam_pos_state_is_tablet_mode_sls(sw, state->state);
+
+	default:
+		dev_warn(&sw->sdev->dev, "unknown device posture source: %u\n", state->source);
 		return true;
 	}
 }
@@ -450,9 +549,10 @@ static int ssam_pos_get_posture_for_source(struct ssam_tablet_sw *sw, u32 source
 	return 0;
 }
 
-static int ssam_pos_get_posture(struct ssam_tablet_sw *sw, u32 *state)
+static int ssam_pos_get_posture(struct ssam_tablet_sw *sw, struct ssam_tablet_sw_state *state)
 {
 	u32 source_id;
+	u32 source_state;
 	int status;
 
 	status = ssam_pos_get_source(sw, &source_id);
@@ -461,13 +561,15 @@ static int ssam_pos_get_posture(struct ssam_tablet_sw *sw, u32 *state)
 		return status;
 	}
 
-	status = ssam_pos_get_posture_for_source(sw, source_id, state);
+	status = ssam_pos_get_posture_for_source(sw, source_id, &source_state);
 	if (status) {
 		dev_err(&sw->sdev->dev, "failed to get posture value for source %u: %d\n",
 			source_id, status);
 		return status;
 	}
 
+	state->source = source_id;
+	state->state = source_state;
 	return 0;
 }
 
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index 4a01b315e0a9..22052031c719 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -84,13 +84,6 @@ config MXM_WMI
           MXM is a standard for laptop graphics cards, the WMI interface
 	  is required for switchable nvidia graphics machines
 
-config PEAQ_WMI
-	tristate "PEAQ 2-in-1 WMI hotkey driver"
-	depends on ACPI_WMI
-	depends on INPUT
-	help
-	 Say Y here if you want to support WMI-based hotkeys on PEAQ 2-in-1s.
-
 config NVIDIA_WMI_EC_BACKLIGHT
 	tristate "EC Backlight Driver for Hybrid Graphics Notebook Systems"
 	depends on ACPI_VIDEO
@@ -213,7 +206,6 @@ config APPLE_GMUX
 	depends on ACPI && PCI
 	depends on PNP
 	depends on BACKLIGHT_CLASS_DEVICE
-	depends on BACKLIGHT_APPLE=n || BACKLIGHT_APPLE
 	help
 	  This driver provides support for the gmux device found on many
 	  Apple laptops, which controls the display mux for the hybrid
@@ -469,6 +461,15 @@ config IDEAPAD_LAPTOP
 	  This is a driver for Lenovo IdeaPad netbooks contains drivers for
 	  rfkill switch, hotkey, fan control and backlight control.
 
+config LENOVO_YMC
+	tristate "Lenovo Yoga Tablet Mode Control"
+	depends on ACPI_WMI
+	depends on INPUT
+	select INPUT_SPARSEKMAP
+	help
+	  This driver maps the Tablet Mode Control switch to SW_TABLET_MODE input
+	  events for Lenovo Yoga notebooks.
+
 config SENSORS_HDAPS
 	tristate "Thinkpad Hard Drive Active Protection System (hdaps)"
 	depends on INPUT
@@ -643,6 +644,14 @@ config THINKPAD_LMI
 
 source "drivers/platform/x86/intel/Kconfig"
 
+config MSI_EC
+	tristate "MSI EC Extras"
+	depends on ACPI
+	depends on ACPI_BATTERY
+	help
+	  This driver allows various MSI laptops' functionalities to be
+	  controlled from userspace, including battery charge threshold.
+
 config MSI_LAPTOP
 	tristate "MSI Laptop Extras"
 	depends on ACPI
@@ -978,22 +987,7 @@ config TOUCHSCREEN_DMI
 	  the OS-image for the device. This option supplies the missing info.
 	  Enable this for x86 tablets with Silead or Chipone touchscreens.
 
-config X86_ANDROID_TABLETS
-	tristate "X86 Android tablet support"
-	depends on I2C && SPI && SERIAL_DEV_BUS && ACPI && EFI && GPIOLIB
-	help
-	  X86 tablets which ship with Android as (part of) the factory image
-	  typically have various problems with their DSDTs. The factory kernels
-	  shipped on these devices typically have device addresses and GPIOs
-	  hardcoded in the kernel, rather than specified in their DSDT.
-
-	  With the DSDT containing a random collection of devices which may or
-	  may not actually be present. This driver contains various fixes for
-	  such tablets, including instantiating kernel devices for devices which
-	  are missing from the DSDT.
-
-	  If you have a x86 Android tablet say Y or M here, for a generic x86
-	  distro config say M here.
+source "drivers/platform/x86/x86-android-tablets/Kconfig"
 
 config FW_ATTR_CLASS
 	tristate
diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile
index 1d3d1b02541b..2cafe51ec4d8 100644
--- a/drivers/platform/x86/Makefile
+++ b/drivers/platform/x86/Makefile
@@ -12,7 +12,6 @@ obj-$(CONFIG_WMI_BMOF)		+= wmi-bmof.o
 obj-$(CONFIG_HUAWEI_WMI)		+= huawei-wmi.o
 obj-$(CONFIG_MXM_WMI)			+= mxm-wmi.o
 obj-$(CONFIG_NVIDIA_WMI_EC_BACKLIGHT)	+= nvidia-wmi-ec-backlight.o
-obj-$(CONFIG_PEAQ_WMI)			+= peaq-wmi.o
 obj-$(CONFIG_XIAOMI_WMI)		+= xiaomi-wmi.o
 obj-$(CONFIG_GIGABYTE_WMI)		+= gigabyte-wmi.o
 obj-$(CONFIG_YOGABOOK_WMI)		+= lenovo-yogabook-wmi.o
@@ -63,6 +62,7 @@ obj-$(CONFIG_UV_SYSFS)       += uv_sysfs.o
 # IBM Thinkpad and Lenovo
 obj-$(CONFIG_IBM_RTL)		+= ibm_rtl.o
 obj-$(CONFIG_IDEAPAD_LAPTOP)	+= ideapad-laptop.o
+obj-$(CONFIG_LENOVO_YMC)	+= lenovo-ymc.o
 obj-$(CONFIG_SENSORS_HDAPS)	+= hdaps.o
 obj-$(CONFIG_THINKPAD_ACPI)	+= thinkpad_acpi.o
 obj-$(CONFIG_THINKPAD_LMI)	+= think-lmi.o
@@ -71,6 +71,7 @@ obj-$(CONFIG_THINKPAD_LMI)	+= think-lmi.o
 obj-y				+= intel/
 
 # MSI
+obj-$(CONFIG_MSI_EC)		+= msi-ec.o
 obj-$(CONFIG_MSI_LAPTOP)	+= msi-laptop.o
 obj-$(CONFIG_MSI_WMI)		+= msi-wmi.o
 
@@ -112,7 +113,7 @@ obj-$(CONFIG_SERIAL_MULTI_INSTANTIATE)	+= serial-multi-instantiate.o
 obj-$(CONFIG_MLX_PLATFORM)		+= mlx-platform.o
 obj-$(CONFIG_TOUCHSCREEN_DMI)		+= touchscreen_dmi.o
 obj-$(CONFIG_WIRELESS_HOTKEY)		+= wireless-hotkey.o
-obj-$(CONFIG_X86_ANDROID_TABLETS)	+= x86-android-tablets.o
+obj-$(CONFIG_X86_ANDROID_TABLETS)	+= x86-android-tablets/
 
 # Intel uncore drivers
 obj-$(CONFIG_INTEL_IPS)				+= intel_ips.o
diff --git a/drivers/platform/x86/acer-wmi.c b/drivers/platform/x86/acer-wmi.c
index ee67efdd5499..377a0becd1a1 100644
--- a/drivers/platform/x86/acer-wmi.c
+++ b/drivers/platform/x86/acer-wmi.c
@@ -2258,7 +2258,7 @@ error_mailled:
 	return err;
 }
 
-static int acer_platform_remove(struct platform_device *device)
+static void acer_platform_remove(struct platform_device *device)
 {
 	if (has_cap(ACER_CAP_MAILLED))
 		acer_led_exit();
@@ -2266,7 +2266,6 @@ static int acer_platform_remove(struct platform_device *device)
 		acer_backlight_exit();
 
 	acer_rfkill_exit();
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -2334,7 +2333,7 @@ static struct platform_driver acer_platform_driver = {
 		.pm = &acer_pm,
 	},
 	.probe = acer_platform_probe,
-	.remove = acer_platform_remove,
+	.remove_new = acer_platform_remove,
 	.shutdown = acer_platform_shutdown,
 };
 
diff --git a/drivers/platform/x86/acerhdf.c b/drivers/platform/x86/acerhdf.c
index a48638ad2a8a..74bcb3d13104 100644
--- a/drivers/platform/x86/acerhdf.c
+++ b/drivers/platform/x86/acerhdf.c
@@ -79,7 +79,6 @@ static unsigned int list_supported;
 static unsigned int fanstate = ACERHDF_FAN_AUTO;
 static char force_bios[16];
 static char force_product[16];
-static unsigned int prev_interval;
 static struct thermal_zone_device *thz_dev;
 static struct thermal_cooling_device *cl_dev;
 static struct platform_device *acerhdf_dev;
@@ -341,25 +340,20 @@ static void acerhdf_check_param(struct thermal_zone_device *thermal)
 		pr_err("fanoff temperature (%d) is above fanon temperature (%d), clamping to %d\n",
 		       fanoff, fanon, fanon);
 		fanoff = fanon;
-	};
+	}
 
 	trips[0].temperature = fanon;
 	trips[0].hysteresis  = fanon - fanoff;
 
-	if (kernelmode && prev_interval != interval) {
+	if (kernelmode) {
 		if (interval > ACERHDF_MAX_INTERVAL) {
 			pr_err("interval too high, set to %d\n",
 			       ACERHDF_MAX_INTERVAL);
 			interval = ACERHDF_MAX_INTERVAL;
 		}
+
 		if (verbose)
 			pr_notice("interval changed to: %d\n", interval);
-
-		if (thermal)
-			thermal->polling_delay_jiffies =
-				round_jiffies(msecs_to_jiffies(interval * 1000));
-
-		prev_interval = interval;
 	}
 }
 
@@ -697,13 +691,6 @@ static int __init acerhdf_register_thermal(void)
 	if (ret)
 		return ret;
 
-	if (strcmp(thz_dev->governor->name,
-				acerhdf_zone_params.governor_name)) {
-		pr_err("Didn't get thermal governor %s, perhaps not compiled into thermal subsystem.\n",
-				acerhdf_zone_params.governor_name);
-		return -EINVAL;
-	}
-
 	return 0;
 }
 
@@ -801,5 +788,5 @@ static const struct kernel_param_ops interval_ops = {
 	.get = param_get_uint,
 };
 
-module_param_cb(interval, &interval_ops, &interval, 0600);
+module_param_cb(interval, &interval_ops, &interval, 0000);
 MODULE_PARM_DESC(interval, "Polling interval of temperature check");
diff --git a/drivers/platform/x86/adv_swbutton.c b/drivers/platform/x86/adv_swbutton.c
index 38693b735c87..6b23ba78e028 100644
--- a/drivers/platform/x86/adv_swbutton.c
+++ b/drivers/platform/x86/adv_swbutton.c
@@ -90,14 +90,12 @@ static int adv_swbutton_probe(struct platform_device *device)
 	return 0;
 }
 
-static int adv_swbutton_remove(struct platform_device *device)
+static void adv_swbutton_remove(struct platform_device *device)
 {
 	acpi_handle handle = ACPI_HANDLE(&device->dev);
 
 	acpi_remove_notify_handler(handle, ACPI_DEVICE_NOTIFY,
 				   adv_swbutton_notify);
-
-	return 0;
 }
 
 static const struct acpi_device_id button_device_ids[] = {
@@ -112,7 +110,7 @@ static struct platform_driver adv_swbutton_driver = {
 		.acpi_match_table = button_device_ids,
 	},
 	.probe = adv_swbutton_probe,
-	.remove = adv_swbutton_remove,
+	.remove_new = adv_swbutton_remove,
 };
 module_platform_driver(adv_swbutton_driver);
 
diff --git a/drivers/platform/x86/amd/Kconfig b/drivers/platform/x86/amd/Kconfig
index 2ce8cb2170df..d9685aef0887 100644
--- a/drivers/platform/x86/amd/Kconfig
+++ b/drivers/platform/x86/amd/Kconfig
@@ -7,7 +7,7 @@ source "drivers/platform/x86/amd/pmf/Kconfig"
 
 config AMD_PMC
 	tristate "AMD SoC PMC driver"
-	depends on ACPI && PCI && RTC_CLASS
+	depends on ACPI && PCI && RTC_CLASS && AMD_NB
 	select SERIO
 	help
 	  The driver provides support for AMD Power Management Controller
diff --git a/drivers/platform/x86/amd/hsmp.c b/drivers/platform/x86/amd/hsmp.c
index 521c6a229362..31382ef52efb 100644
--- a/drivers/platform/x86/amd/hsmp.c
+++ b/drivers/platform/x86/amd/hsmp.c
@@ -340,16 +340,14 @@ static int hsmp_pltdrv_probe(struct platform_device *pdev)
 	return misc_register(&hsmp_device);
 }
 
-static int hsmp_pltdrv_remove(struct platform_device *pdev)
+static void hsmp_pltdrv_remove(struct platform_device *pdev)
 {
 	misc_deregister(&hsmp_device);
-
-	return 0;
 }
 
 static struct platform_driver amd_hsmp_driver = {
 	.probe		= hsmp_pltdrv_probe,
-	.remove		= hsmp_pltdrv_remove,
+	.remove_new	= hsmp_pltdrv_remove,
 	.driver		= {
 		.name	= DRIVER_NAME,
 	},
diff --git a/drivers/platform/x86/amd/pmc.c b/drivers/platform/x86/amd/pmc.c
index 2edaae04a691..427905714f79 100644
--- a/drivers/platform/x86/amd/pmc.c
+++ b/drivers/platform/x86/amd/pmc.c
@@ -10,6 +10,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <asm/amd_nb.h>
 #include <linux/acpi.h>
 #include <linux/bitfield.h>
 #include <linux/bits.h>
@@ -37,8 +38,6 @@
 #define AMD_PMC_SCRATCH_REG_YC		0xD14
 
 /* STB Registers */
-#define AMD_PMC_STB_INDEX_ADDRESS	0xF8
-#define AMD_PMC_STB_INDEX_DATA		0xFC
 #define AMD_PMC_STB_PMI_0		0x03E30600
 #define AMD_PMC_STB_S2IDLE_PREPARE	0xC6000001
 #define AMD_PMC_STB_S2IDLE_RESTORE	0xC6000002
@@ -56,8 +55,6 @@
 #define S2D_TELEMETRY_DRAMBYTES_MAX	0x1000000
 
 /* Base address of SMU for mapping physical address to virtual address */
-#define AMD_PMC_SMU_INDEX_ADDRESS	0xB8
-#define AMD_PMC_SMU_INDEX_DATA		0xBC
 #define AMD_PMC_MAPPING_SIZE		0x01000
 #define AMD_PMC_BASE_ADDR_OFFSET	0x10000
 #define AMD_PMC_BASE_ADDR_LO		0x13B102E8
@@ -97,6 +94,7 @@
 #define AMD_CPU_ID_YC			0x14B5
 #define AMD_CPU_ID_CB			0x14D8
 #define AMD_CPU_ID_PS			0x14E8
+#define AMD_CPU_ID_SP			0x14A4
 
 #define PMC_MSG_DELAY_MIN_US		50
 #define RESPONSE_REGISTER_LOOP_MAX	20000
@@ -268,6 +266,7 @@ static int amd_pmc_stb_debugfs_open_v2(struct inode *inode, struct file *filp)
 	dev->msg_port = 0;
 	if (ret) {
 		dev_err(dev->dev, "error: S2D_NUM_SAMPLES not supported : %d\n", ret);
+		kfree(buf);
 		return ret;
 	}
 
@@ -342,33 +341,6 @@ static int amd_pmc_setup_smu_logging(struct amd_pmc_dev *dev)
 	return 0;
 }
 
-static int amd_pmc_idlemask_read(struct amd_pmc_dev *pdev, struct device *dev,
-				 struct seq_file *s)
-{
-	u32 val;
-
-	switch (pdev->cpu_id) {
-	case AMD_CPU_ID_CZN:
-		val = amd_pmc_reg_read(pdev, AMD_PMC_SCRATCH_REG_CZN);
-		break;
-	case AMD_CPU_ID_YC:
-	case AMD_CPU_ID_CB:
-	case AMD_CPU_ID_PS:
-		val = amd_pmc_reg_read(pdev, AMD_PMC_SCRATCH_REG_YC);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	if (dev)
-		dev_dbg(pdev->dev, "SMU idlemask s0i3: 0x%x\n", val);
-
-	if (s)
-		seq_printf(s, "SMU idlemask : 0x%x\n", val);
-
-	return 0;
-}
-
 static int get_metrics_table(struct amd_pmc_dev *pdev, struct smu_metrics *table)
 {
 	if (!pdev->smu_virt_addr) {
@@ -393,9 +365,8 @@ static void amd_pmc_validate_deepest(struct amd_pmc_dev *pdev)
 
 	if (!table.s0i3_last_entry_status)
 		dev_warn(pdev->dev, "Last suspend didn't reach deepest state\n");
-	else
-		dev_dbg(pdev->dev, "Last suspend in deepest state for %lluus\n",
-			 table.timein_s0i3_lastcapture);
+	pm_report_hw_sleep_time(table.s0i3_last_entry_status ?
+				table.timein_s0i3_lastcapture : 0);
 }
 
 static int amd_pmc_get_smu_version(struct amd_pmc_dev *dev)
@@ -403,6 +374,9 @@ static int amd_pmc_get_smu_version(struct amd_pmc_dev *dev)
 	int rc;
 	u32 val;
 
+	if (dev->cpu_id == AMD_CPU_ID_PCO)
+		return -ENODEV;
+
 	rc = amd_pmc_send_cmd(dev, 0, &val, SMU_MSG_GETSMUVERSION, 1);
 	if (rc)
 		return rc;
@@ -449,12 +423,31 @@ static ssize_t smu_program_show(struct device *d, struct device_attribute *attr,
 static DEVICE_ATTR_RO(smu_fw_version);
 static DEVICE_ATTR_RO(smu_program);
 
+static umode_t pmc_attr_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct amd_pmc_dev *pdev = dev_get_drvdata(dev);
+
+	if (pdev->cpu_id == AMD_CPU_ID_PCO)
+		return 0;
+	return 0444;
+}
+
 static struct attribute *pmc_attrs[] = {
 	&dev_attr_smu_fw_version.attr,
 	&dev_attr_smu_program.attr,
 	NULL,
 };
-ATTRIBUTE_GROUPS(pmc);
+
+static struct attribute_group pmc_attr_group = {
+	.attrs = pmc_attrs,
+	.is_visible = pmc_attr_is_visible,
+};
+
+static const struct attribute_group *pmc_groups[] = {
+	&pmc_attr_group,
+	NULL,
+};
 
 static int smu_fw_info_show(struct seq_file *s, void *unused)
 {
@@ -521,28 +514,47 @@ static int s0ix_stats_show(struct seq_file *s, void *unused)
 }
 DEFINE_SHOW_ATTRIBUTE(s0ix_stats);
 
-static int amd_pmc_idlemask_show(struct seq_file *s, void *unused)
+static int amd_pmc_idlemask_read(struct amd_pmc_dev *pdev, struct device *dev,
+				 struct seq_file *s)
 {
-	struct amd_pmc_dev *dev = s->private;
+	u32 val;
 	int rc;
 
-	/* we haven't yet read SMU version */
-	if (!dev->major) {
-		rc = amd_pmc_get_smu_version(dev);
-		if (rc)
-			return rc;
+	switch (pdev->cpu_id) {
+	case AMD_CPU_ID_CZN:
+		/* we haven't yet read SMU version */
+		if (!pdev->major) {
+			rc = amd_pmc_get_smu_version(pdev);
+			if (rc)
+				return rc;
+		}
+		if (pdev->major > 56 || (pdev->major >= 55 && pdev->minor >= 37))
+			val = amd_pmc_reg_read(pdev, AMD_PMC_SCRATCH_REG_CZN);
+		else
+			return -EINVAL;
+		break;
+	case AMD_CPU_ID_YC:
+	case AMD_CPU_ID_CB:
+	case AMD_CPU_ID_PS:
+		val = amd_pmc_reg_read(pdev, AMD_PMC_SCRATCH_REG_YC);
+		break;
+	default:
+		return -EINVAL;
 	}
 
-	if (dev->major > 56 || (dev->major >= 55 && dev->minor >= 37)) {
-		rc = amd_pmc_idlemask_read(dev, NULL, s);
-		if (rc)
-			return rc;
-	} else {
-		seq_puts(s, "Unsupported SMU version for Idlemask\n");
-	}
+	if (dev)
+		dev_dbg(pdev->dev, "SMU idlemask s0i3: 0x%x\n", val);
+
+	if (s)
+		seq_printf(s, "SMU idlemask : 0x%x\n", val);
 
 	return 0;
 }
+
+static int amd_pmc_idlemask_show(struct seq_file *s, void *unused)
+{
+	return amd_pmc_idlemask_read(s->private, NULL, s);
+}
 DEFINE_SHOW_ATTRIBUTE(amd_pmc_idlemask);
 
 static void amd_pmc_dbgfs_unregister(struct amd_pmc_dev *dev)
@@ -812,6 +824,14 @@ static void amd_pmc_s2idle_check(void)
 		dev_err(pdev->dev, "error writing to STB: %d\n", rc);
 }
 
+static int amd_pmc_dump_data(struct amd_pmc_dev *pdev)
+{
+	if (pdev->cpu_id == AMD_CPU_ID_PCO)
+		return -ENODEV;
+
+	return amd_pmc_send_cmd(pdev, 0, NULL, SMU_MSG_LOG_DUMP_DATA, 0);
+}
+
 static void amd_pmc_s2idle_restore(void)
 {
 	struct amd_pmc_dev *pdev = &pmc;
@@ -824,7 +844,7 @@ static void amd_pmc_s2idle_restore(void)
 		dev_err(pdev->dev, "resume failed: %d\n", rc);
 
 	/* Let SMU know that we are looking for stats */
-	amd_pmc_send_cmd(pdev, 0, NULL, SMU_MSG_LOG_DUMP_DATA, 0);
+	amd_pmc_dump_data(pdev);
 
 	rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_S2IDLE_RESTORE);
 	if (rc)
@@ -840,7 +860,7 @@ static struct acpi_s2idle_dev_ops amd_pmc_s2idle_dev_ops = {
 	.restore = amd_pmc_s2idle_restore,
 };
 
-static int __maybe_unused amd_pmc_suspend_handler(struct device *dev)
+static int amd_pmc_suspend_handler(struct device *dev)
 {
 	struct amd_pmc_dev *pdev = dev_get_drvdata(dev);
 
@@ -866,6 +886,7 @@ static const struct pci_device_id pmc_pci_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_RN) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_PCO) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_RV) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_SP) },
 	{ }
 };
 
@@ -902,17 +923,9 @@ static int amd_pmc_write_stb(struct amd_pmc_dev *dev, u32 data)
 {
 	int err;
 
-	err = pci_write_config_dword(dev->rdev, AMD_PMC_STB_INDEX_ADDRESS, AMD_PMC_STB_PMI_0);
+	err = amd_smn_write(0, AMD_PMC_STB_PMI_0, data);
 	if (err) {
-		dev_err(dev->dev, "failed to write addr in stb: 0x%X\n",
-			AMD_PMC_STB_INDEX_ADDRESS);
-		return pcibios_err_to_errno(err);
-	}
-
-	err = pci_write_config_dword(dev->rdev, AMD_PMC_STB_INDEX_DATA, data);
-	if (err) {
-		dev_err(dev->dev, "failed to write data in stb: 0x%X\n",
-			AMD_PMC_STB_INDEX_DATA);
+		dev_err(dev->dev, "failed to write data in stb: 0x%X\n", AMD_PMC_STB_PMI_0);
 		return pcibios_err_to_errno(err);
 	}
 
@@ -923,18 +936,10 @@ static int amd_pmc_read_stb(struct amd_pmc_dev *dev, u32 *buf)
 {
 	int i, err;
 
-	err = pci_write_config_dword(dev->rdev, AMD_PMC_STB_INDEX_ADDRESS, AMD_PMC_STB_PMI_0);
-	if (err) {
-		dev_err(dev->dev, "error writing addr to stb: 0x%X\n",
-			AMD_PMC_STB_INDEX_ADDRESS);
-		return pcibios_err_to_errno(err);
-	}
-
 	for (i = 0; i < FIFO_SIZE; i++) {
-		err = pci_read_config_dword(dev->rdev, AMD_PMC_STB_INDEX_DATA, buf++);
+		err = amd_smn_read(0, AMD_PMC_STB_PMI_0, buf++);
 		if (err) {
-			dev_err(dev->dev, "error reading data from stb: 0x%X\n",
-				AMD_PMC_STB_INDEX_DATA);
+			dev_err(dev->dev, "error reading data from stb: 0x%X\n", AMD_PMC_STB_PMI_0);
 			return pcibios_err_to_errno(err);
 		}
 	}
@@ -960,31 +965,26 @@ static int amd_pmc_probe(struct platform_device *pdev)
 	}
 
 	dev->cpu_id = rdev->device;
-	dev->rdev = rdev;
-	err = pci_write_config_dword(rdev, AMD_PMC_SMU_INDEX_ADDRESS, AMD_PMC_BASE_ADDR_LO);
-	if (err) {
-		dev_err(dev->dev, "error writing to 0x%x\n", AMD_PMC_SMU_INDEX_ADDRESS);
-		err = pcibios_err_to_errno(err);
+
+	if (dev->cpu_id == AMD_CPU_ID_SP) {
+		dev_warn_once(dev->dev, "S0i3 is not supported on this hardware\n");
+		err = -ENODEV;
 		goto err_pci_dev_put;
 	}
 
-	err = pci_read_config_dword(rdev, AMD_PMC_SMU_INDEX_DATA, &val);
+	dev->rdev = rdev;
+	err = amd_smn_read(0, AMD_PMC_BASE_ADDR_LO, &val);
 	if (err) {
+		dev_err(dev->dev, "error reading 0x%x\n", AMD_PMC_BASE_ADDR_LO);
 		err = pcibios_err_to_errno(err);
 		goto err_pci_dev_put;
 	}
 
 	base_addr_lo = val & AMD_PMC_BASE_ADDR_HI_MASK;
 
-	err = pci_write_config_dword(rdev, AMD_PMC_SMU_INDEX_ADDRESS, AMD_PMC_BASE_ADDR_HI);
-	if (err) {
-		dev_err(dev->dev, "error writing to 0x%x\n", AMD_PMC_SMU_INDEX_ADDRESS);
-		err = pcibios_err_to_errno(err);
-		goto err_pci_dev_put;
-	}
-
-	err = pci_read_config_dword(rdev, AMD_PMC_SMU_INDEX_DATA, &val);
+	err = amd_smn_read(0, AMD_PMC_BASE_ADDR_HI, &val);
 	if (err) {
+		dev_err(dev->dev, "error reading 0x%x\n", AMD_PMC_BASE_ADDR_HI);
 		err = pcibios_err_to_errno(err);
 		goto err_pci_dev_put;
 	}
@@ -1015,6 +1015,7 @@ static int amd_pmc_probe(struct platform_device *pdev)
 	}
 
 	amd_pmc_dbgfs_register(dev);
+	pm_report_max_hw_sleep(U64_MAX);
 	return 0;
 
 err_pci_dev_put:
@@ -1022,7 +1023,7 @@ err_pci_dev_put:
 	return err;
 }
 
-static int amd_pmc_remove(struct platform_device *pdev)
+static void amd_pmc_remove(struct platform_device *pdev)
 {
 	struct amd_pmc_dev *dev = platform_get_drvdata(pdev);
 
@@ -1031,7 +1032,6 @@ static int amd_pmc_remove(struct platform_device *pdev)
 	amd_pmc_dbgfs_unregister(dev);
 	pci_dev_put(dev->rdev);
 	mutex_destroy(&dev->lock);
-	return 0;
 }
 
 static const struct acpi_device_id amd_pmc_acpi_ids[] = {
@@ -1054,7 +1054,7 @@ static struct platform_driver amd_pmc_driver = {
 		.pm = pm_sleep_ptr(&amd_pmc_pm),
 	},
 	.probe = amd_pmc_probe,
-	.remove = amd_pmc_remove,
+	.remove_new = amd_pmc_remove,
 };
 module_platform_driver(amd_pmc_driver);
 
diff --git a/drivers/platform/x86/amd/pmf/Kconfig b/drivers/platform/x86/amd/pmf/Kconfig
index 6d89528c3177..d87986adf91e 100644
--- a/drivers/platform/x86/amd/pmf/Kconfig
+++ b/drivers/platform/x86/amd/pmf/Kconfig
@@ -7,6 +7,7 @@ config AMD_PMF
 	tristate "AMD Platform Management Framework"
 	depends on ACPI && PCI
 	depends on POWER_SUPPLY
+	depends on AMD_NB
 	select ACPI_PLATFORM_PROFILE
 	help
 	  This driver provides support for the AMD Platform Management Framework.
diff --git a/drivers/platform/x86/amd/pmf/core.c b/drivers/platform/x86/amd/pmf/core.c
index da23639071d7..d5bb775dadcf 100644
--- a/drivers/platform/x86/amd/pmf/core.c
+++ b/drivers/platform/x86/amd/pmf/core.c
@@ -8,6 +8,7 @@
  * Author: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
  */
 
+#include <asm/amd_nb.h>
 #include <linux/debugfs.h>
 #include <linux/iopoll.h>
 #include <linux/module.h>
@@ -22,8 +23,6 @@
 #define AMD_PMF_REGISTER_ARGUMENT	0xA58
 
 /* Base address of SMU for mapping physical address to virtual address */
-#define AMD_PMF_SMU_INDEX_ADDRESS	0xB8
-#define AMD_PMF_SMU_INDEX_DATA		0xBC
 #define AMD_PMF_MAPPING_SIZE		0x01000
 #define AMD_PMF_BASE_ADDR_OFFSET	0x10000
 #define AMD_PMF_BASE_ADDR_LO		0x13B102E8
@@ -348,30 +347,19 @@ static int amd_pmf_probe(struct platform_device *pdev)
 	}
 
 	dev->cpu_id = rdev->device;
-	err = pci_write_config_dword(rdev, AMD_PMF_SMU_INDEX_ADDRESS, AMD_PMF_BASE_ADDR_LO);
-	if (err) {
-		dev_err(dev->dev, "error writing to 0x%x\n", AMD_PMF_SMU_INDEX_ADDRESS);
-		pci_dev_put(rdev);
-		return pcibios_err_to_errno(err);
-	}
 
-	err = pci_read_config_dword(rdev, AMD_PMF_SMU_INDEX_DATA, &val);
+	err = amd_smn_read(0, AMD_PMF_BASE_ADDR_LO, &val);
 	if (err) {
+		dev_err(dev->dev, "error in reading from 0x%x\n", AMD_PMF_BASE_ADDR_LO);
 		pci_dev_put(rdev);
 		return pcibios_err_to_errno(err);
 	}
 
 	base_addr_lo = val & AMD_PMF_BASE_ADDR_HI_MASK;
 
-	err = pci_write_config_dword(rdev, AMD_PMF_SMU_INDEX_ADDRESS, AMD_PMF_BASE_ADDR_HI);
-	if (err) {
-		dev_err(dev->dev, "error writing to 0x%x\n", AMD_PMF_SMU_INDEX_ADDRESS);
-		pci_dev_put(rdev);
-		return pcibios_err_to_errno(err);
-	}
-
-	err = pci_read_config_dword(rdev, AMD_PMF_SMU_INDEX_DATA, &val);
+	err = amd_smn_read(0, AMD_PMF_BASE_ADDR_HI, &val);
 	if (err) {
+		dev_err(dev->dev, "error in reading from 0x%x\n", AMD_PMF_BASE_ADDR_HI);
 		pci_dev_put(rdev);
 		return pcibios_err_to_errno(err);
 	}
@@ -402,7 +390,7 @@ static int amd_pmf_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int amd_pmf_remove(struct platform_device *pdev)
+static void amd_pmf_remove(struct platform_device *pdev)
 {
 	struct amd_pmf_dev *dev = platform_get_drvdata(pdev);
 
@@ -413,7 +401,6 @@ static int amd_pmf_remove(struct platform_device *pdev)
 	mutex_destroy(&dev->lock);
 	mutex_destroy(&dev->update_mutex);
 	kfree(dev->buf);
-	return 0;
 }
 
 static const struct attribute_group *amd_pmf_driver_groups[] = {
@@ -428,7 +415,7 @@ static struct platform_driver amd_pmf_driver = {
 		.dev_groups = amd_pmf_driver_groups,
 	},
 	.probe = amd_pmf_probe,
-	.remove = amd_pmf_remove,
+	.remove_new = amd_pmf_remove,
 };
 module_platform_driver(amd_pmf_driver);
 
diff --git a/drivers/platform/x86/amilo-rfkill.c b/drivers/platform/x86/amilo-rfkill.c
index 3e313c4d538d..efcf909786a5 100644
--- a/drivers/platform/x86/amilo-rfkill.c
+++ b/drivers/platform/x86/amilo-rfkill.c
@@ -124,11 +124,10 @@ fail:
 	return rc;
 }
 
-static int amilo_rfkill_remove(struct platform_device *device)
+static void amilo_rfkill_remove(struct platform_device *device)
 {
 	rfkill_unregister(amilo_rfkill_dev);
 	rfkill_destroy(amilo_rfkill_dev);
-	return 0;
 }
 
 static struct platform_driver amilo_rfkill_driver = {
@@ -136,7 +135,7 @@ static struct platform_driver amilo_rfkill_driver = {
 		.name	= KBUILD_MODNAME,
 	},
 	.probe	= amilo_rfkill_probe,
-	.remove	= amilo_rfkill_remove,
+	.remove_new = amilo_rfkill_remove,
 };
 
 static int __init amilo_rfkill_init(void)
diff --git a/drivers/platform/x86/apple-gmux.c b/drivers/platform/x86/apple-gmux.c
index 9333f82cfa8a..e02b4aea4f1e 100644
--- a/drivers/platform/x86/apple-gmux.c
+++ b/drivers/platform/x86/apple-gmux.c
@@ -5,6 +5,7 @@
  *  Copyright (C) Canonical Ltd. <seth.forshee@canonical.com>
  *  Copyright (C) 2010-2012 Andreas Heider <andreas@meetr.de>
  *  Copyright (C) 2015 Lukas Wunner <lukas@wunner.de>
+ *  Copyright (C) 2023 Orlando Chamberlain <orlandoch.dev@gmail.com>
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -15,38 +16,51 @@
 #include <linux/backlight.h>
 #include <linux/acpi.h>
 #include <linux/pnp.h>
-#include <linux/apple_bl.h>
 #include <linux/apple-gmux.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/pci.h>
 #include <linux/vga_switcheroo.h>
+#include <linux/debugfs.h>
+#include <acpi/video.h>
 #include <asm/io.h>
 
 /**
  * DOC: Overview
  *
  * gmux is a microcontroller built into the MacBook Pro to support dual GPUs:
- * A `Lattice XP2`_ on pre-retinas, a `Renesas R4F2113`_ on retinas.
+ * A `Lattice XP2`_ on pre-retinas, a `Renesas R4F2113`_ on pre-T2 retinas.
+ *
+ * On T2 Macbooks, the gmux is part of the T2 Coprocessor's SMC. The SMC has
+ * an I2C connection to a `NXP PCAL6524` GPIO expander, which enables/disables
+ * the voltage regulators of the discrete GPU, drives the display panel power,
+ * and has a GPIO to switch the eDP mux. The Intel CPU can interact with
+ * gmux through MMIO, similar to how the main SMC interface is controlled.
  *
  * (The MacPro6,1 2013 also has a gmux, however it is unclear why since it has
  * dual GPUs but no built-in display.)
  *
  * gmux is connected to the LPC bus of the southbridge. Its I/O ports are
  * accessed differently depending on the microcontroller: Driver functions
- * to access a pre-retina gmux are infixed ``_pio_``, those for a retina gmux
- * are infixed ``_index_``.
+ * to access a pre-retina gmux are infixed ``_pio_``, those for a pre-T2
+ * retina gmux are infixed ``_index_``, and those on T2 Macs are infixed
+ * with ``_mmio_``.
  *
  * .. _Lattice XP2:
  *     http://www.latticesemi.com/en/Products/FPGAandCPLD/LatticeXP2.aspx
  * .. _Renesas R4F2113:
  *     http://www.renesas.com/products/mpumcu/h8s/h8s2100/h8s2113/index.jsp
+ * .. _NXP PCAL6524:
+ *     https://www.nxp.com/docs/en/data-sheet/PCAL6524.pdf
  */
 
+struct apple_gmux_config;
+
 struct apple_gmux_data {
+	u8 __iomem *iomem_base;
 	unsigned long iostart;
 	unsigned long iolen;
-	bool indexed;
+	const struct apple_gmux_config *config;
 	struct mutex index_lock;
 
 	struct backlight_device *bdev;
@@ -60,10 +74,26 @@ struct apple_gmux_data {
 	enum vga_switcheroo_client_id switch_state_external;
 	enum vga_switcheroo_state power_state;
 	struct completion powerchange_done;
+
+	/* debugfs data */
+	u8 selected_port;
+	struct dentry *debug_dentry;
 };
 
 static struct apple_gmux_data *apple_gmux_data;
 
+struct apple_gmux_config {
+	u8 (*read8)(struct apple_gmux_data *gmux_data, int port);
+	void (*write8)(struct apple_gmux_data *gmux_data, int port, u8 val);
+	u32 (*read32)(struct apple_gmux_data *gmux_data, int port);
+	void (*write32)(struct apple_gmux_data *gmux_data, int port, u32 val);
+	const struct vga_switcheroo_handler *gmux_handler;
+	enum vga_switcheroo_handler_flags_t handler_flags;
+	unsigned long resource_type;
+	bool read_version_as_u32;
+	char *name;
+};
+
 #define GMUX_INTERRUPT_ENABLE		0xff
 #define GMUX_INTERRUPT_DISABLE		0x00
 
@@ -193,37 +223,98 @@ static void gmux_index_write32(struct apple_gmux_data *gmux_data, int port,
 	mutex_unlock(&gmux_data->index_lock);
 }
 
+static int gmux_mmio_wait(struct apple_gmux_data *gmux_data)
+{
+	int i = 200;
+	u8 gwr = ioread8(gmux_data->iomem_base + GMUX_MMIO_COMMAND_SEND);
+
+	while (i && gwr) {
+		gwr = ioread8(gmux_data->iomem_base + GMUX_MMIO_COMMAND_SEND);
+		udelay(100);
+		i--;
+	}
+
+	return !!i;
+}
+
+static u8 gmux_mmio_read8(struct apple_gmux_data *gmux_data, int port)
+{
+	u8 val;
+
+	mutex_lock(&gmux_data->index_lock);
+	gmux_mmio_wait(gmux_data);
+	iowrite8((port & 0xff), gmux_data->iomem_base + GMUX_MMIO_PORT_SELECT);
+	iowrite8(GMUX_MMIO_READ | sizeof(val),
+		gmux_data->iomem_base + GMUX_MMIO_COMMAND_SEND);
+	gmux_mmio_wait(gmux_data);
+	val = ioread8(gmux_data->iomem_base);
+	mutex_unlock(&gmux_data->index_lock);
+
+	return val;
+}
+
+static void gmux_mmio_write8(struct apple_gmux_data *gmux_data, int port,
+			      u8 val)
+{
+	mutex_lock(&gmux_data->index_lock);
+	gmux_mmio_wait(gmux_data);
+	iowrite8(val, gmux_data->iomem_base);
+
+	iowrite8(port & 0xff, gmux_data->iomem_base + GMUX_MMIO_PORT_SELECT);
+	iowrite8(GMUX_MMIO_WRITE | sizeof(val),
+		gmux_data->iomem_base + GMUX_MMIO_COMMAND_SEND);
+
+	gmux_mmio_wait(gmux_data);
+	mutex_unlock(&gmux_data->index_lock);
+}
+
+static u32 gmux_mmio_read32(struct apple_gmux_data *gmux_data, int port)
+{
+	u32 val;
+
+	mutex_lock(&gmux_data->index_lock);
+	gmux_mmio_wait(gmux_data);
+	iowrite8((port & 0xff), gmux_data->iomem_base + GMUX_MMIO_PORT_SELECT);
+	iowrite8(GMUX_MMIO_READ | sizeof(val),
+		gmux_data->iomem_base + GMUX_MMIO_COMMAND_SEND);
+	gmux_mmio_wait(gmux_data);
+	val = be32_to_cpu(ioread32(gmux_data->iomem_base));
+	mutex_unlock(&gmux_data->index_lock);
+
+	return val;
+}
+
+static void gmux_mmio_write32(struct apple_gmux_data *gmux_data, int port,
+			       u32 val)
+{
+	mutex_lock(&gmux_data->index_lock);
+	iowrite32(cpu_to_be32(val), gmux_data->iomem_base);
+	iowrite8(port & 0xff, gmux_data->iomem_base + GMUX_MMIO_PORT_SELECT);
+	iowrite8(GMUX_MMIO_WRITE | sizeof(val),
+		gmux_data->iomem_base + GMUX_MMIO_COMMAND_SEND);
+	gmux_mmio_wait(gmux_data);
+	mutex_unlock(&gmux_data->index_lock);
+}
+
 static u8 gmux_read8(struct apple_gmux_data *gmux_data, int port)
 {
-	if (gmux_data->indexed)
-		return gmux_index_read8(gmux_data, port);
-	else
-		return gmux_pio_read8(gmux_data, port);
+	return gmux_data->config->read8(gmux_data, port);
 }
 
 static void gmux_write8(struct apple_gmux_data *gmux_data, int port, u8 val)
 {
-	if (gmux_data->indexed)
-		gmux_index_write8(gmux_data, port, val);
-	else
-		gmux_pio_write8(gmux_data, port, val);
+	return gmux_data->config->write8(gmux_data, port, val);
 }
 
 static u32 gmux_read32(struct apple_gmux_data *gmux_data, int port)
 {
-	if (gmux_data->indexed)
-		return gmux_index_read32(gmux_data, port);
-	else
-		return gmux_pio_read32(gmux_data, port);
+	return gmux_data->config->read32(gmux_data, port);
 }
 
 static void gmux_write32(struct apple_gmux_data *gmux_data, int port,
 			     u32 val)
 {
-	if (gmux_data->indexed)
-		gmux_index_write32(gmux_data, port, val);
-	else
-		gmux_pio_write32(gmux_data, port, val);
+	return gmux_data->config->write32(gmux_data, port, val);
 }
 
 /**
@@ -233,8 +324,8 @@ static void gmux_write32(struct apple_gmux_data *gmux_data, int port,
  * the GPU. On dual GPU MacBook Pros by contrast, either GPU may be suspended
  * to conserve energy. Hence the PWM signal needs to be generated by a separate
  * backlight driver which is controlled by gmux. The earliest generation
- * MBP5 2008/09 uses a `TI LP8543`_ backlight driver. All newer models
- * use a `TI LP8545`_.
+ * MBP5 2008/09 uses a `TI LP8543`_ backlight driver. Newer models
+ * use a `TI LP8545`_ or a TI LP8548.
  *
  * .. _TI LP8543: https://www.ti.com/lit/ds/symlink/lp8543.pdf
  * .. _TI LP8545: https://www.ti.com/lit/ds/symlink/lp8545.pdf
@@ -298,8 +389,8 @@ static const struct backlight_ops gmux_bl_ops = {
  * connecting it either to the discrete GPU or the Thunderbolt controller.
  * Oddly enough, while the full port is no longer switchable, AUX and HPD
  * are still switchable by way of an `NXP CBTL03062`_ (on pre-retinas
- * MBP8 2011 and MBP9 2012) or two `TI TS3DS10224`_ (on retinas) under the
- * control of gmux. Since the integrated GPU is missing the main link,
+ * MBP8 2011 and MBP9 2012) or two `TI TS3DS10224`_ (on pre-t2 retinas) under
+ * the control of gmux. Since the integrated GPU is missing the main link,
  * external displays appear to it as phantoms which fail to link-train.
  *
  * gmux receives the HPD signal of all display connectors and sends an
@@ -346,10 +437,10 @@ static void gmux_read_switch_state(struct apple_gmux_data *gmux_data)
 	else
 		gmux_data->switch_state_ddc = VGA_SWITCHEROO_DIS;
 
-	if (gmux_read8(gmux_data, GMUX_PORT_SWITCH_DISPLAY) == 2)
-		gmux_data->switch_state_display = VGA_SWITCHEROO_IGD;
-	else
+	if (gmux_read8(gmux_data, GMUX_PORT_SWITCH_DISPLAY) & 1)
 		gmux_data->switch_state_display = VGA_SWITCHEROO_DIS;
+	else
+		gmux_data->switch_state_display = VGA_SWITCHEROO_IGD;
 
 	if (gmux_read8(gmux_data, GMUX_PORT_SWITCH_EXTERNAL) == 2)
 		gmux_data->switch_state_external = VGA_SWITCHEROO_IGD;
@@ -463,27 +554,79 @@ static enum vga_switcheroo_client_id gmux_get_client_id(struct pci_dev *pdev)
 		return VGA_SWITCHEROO_DIS;
 }
 
-static const struct vga_switcheroo_handler gmux_handler_indexed = {
+static const struct vga_switcheroo_handler gmux_handler_no_ddc = {
 	.switchto = gmux_switchto,
 	.power_state = gmux_set_power_state,
 	.get_client_id = gmux_get_client_id,
 };
 
-static const struct vga_switcheroo_handler gmux_handler_classic = {
+static const struct vga_switcheroo_handler gmux_handler_ddc = {
 	.switchto = gmux_switchto,
 	.switch_ddc = gmux_switch_ddc,
 	.power_state = gmux_set_power_state,
 	.get_client_id = gmux_get_client_id,
 };
 
+static const struct apple_gmux_config apple_gmux_pio = {
+	.read8 = &gmux_pio_read8,
+	.write8 = &gmux_pio_write8,
+	.read32 = &gmux_pio_read32,
+	.write32 = &gmux_pio_write32,
+	.gmux_handler = &gmux_handler_ddc,
+	.handler_flags = VGA_SWITCHEROO_CAN_SWITCH_DDC,
+	.resource_type = IORESOURCE_IO,
+	.read_version_as_u32 = false,
+	.name = "classic"
+};
+
+static const struct apple_gmux_config apple_gmux_index = {
+	.read8 = &gmux_index_read8,
+	.write8 = &gmux_index_write8,
+	.read32 = &gmux_index_read32,
+	.write32 = &gmux_index_write32,
+	.gmux_handler = &gmux_handler_no_ddc,
+	.handler_flags = VGA_SWITCHEROO_NEEDS_EDP_CONFIG,
+	.resource_type = IORESOURCE_IO,
+	.read_version_as_u32 = true,
+	.name = "indexed"
+};
+
+static const struct apple_gmux_config apple_gmux_mmio = {
+	.read8 = &gmux_mmio_read8,
+	.write8 = &gmux_mmio_write8,
+	.read32 = &gmux_mmio_read32,
+	.write32 = &gmux_mmio_write32,
+	.gmux_handler = &gmux_handler_no_ddc,
+	.handler_flags = VGA_SWITCHEROO_NEEDS_EDP_CONFIG,
+	.resource_type = IORESOURCE_MEM,
+	.read_version_as_u32 = true,
+	.name = "T2"
+};
+
+
 /**
  * DOC: Interrupt
  *
  * gmux is also connected to a GPIO pin of the southbridge and thereby is able
- * to trigger an ACPI GPE. On the MBP5 2008/09 it's GPIO pin 22 of the Nvidia
- * MCP79, on all following generations it's GPIO pin 6 of the Intel PCH.
+ * to trigger an ACPI GPE. ACPI name GMGP holds this GPIO pin's number. On the
+ * MBP5 2008/09 it's GPIO pin 22 of the Nvidia MCP79, on following generations
+ * it's GPIO pin 6 of the Intel PCH, on MMIO gmux's it's pin 21.
+ *
  * The GPE merely signals that an interrupt occurred, the actual type of event
  * is identified by reading a gmux register.
+ *
+ * In addition to the GMGP name, gmux's ACPI device also has two methods GMSP
+ * and GMLV. GMLV likely means "GMUX Level", and reads the value of the GPIO,
+ * while GMSP likely means "GMUX Set Polarity", and seems to write to the GPIO's
+ * value. On newer Macbooks (This was introduced with or sometime before the
+ * MacBookPro14,3), the ACPI GPE method differentiates between the OS type: On
+ * Darwin, only a notification is signaled, whereas on other OSes, the GPIO's
+ * value is read and then inverted.
+ *
+ * Because Linux masquerades as Darwin, it ends up in the notification-only code
+ * path. On MMIO gmux's, this seems to lead to us being unable to clear interrupts,
+ * unless we call GMSP(0). Without this, there is a flood of status=0 interrupts
+ * that can't be cleared. This issue seems to be unique to MMIO gmux's.
  */
 
 static inline void gmux_disable_interrupts(struct apple_gmux_data *gmux_data)
@@ -510,6 +653,9 @@ static void gmux_clear_interrupts(struct apple_gmux_data *gmux_data)
 	/* to clear interrupts write back current status */
 	status = gmux_interrupt_get_status(gmux_data);
 	gmux_write8(gmux_data, GMUX_PORT_INTERRUPT_STATUS, status);
+	/* Prevent flood of status=0 interrupts */
+	if (gmux_data->config == &apple_gmux_mmio)
+		acpi_execute_simple_method(gmux_data->dhandle, "GMSP", 0);
 }
 
 static void gmux_notify_handler(acpi_handle device, u32 value, void *context)
@@ -529,6 +675,80 @@ static void gmux_notify_handler(acpi_handle device, u32 value, void *context)
 		complete(&gmux_data->powerchange_done);
 }
 
+/**
+ * DOC: Debugfs Interface
+ *
+ * gmux ports can be accessed from userspace as a debugfs interface. For example:
+ *
+ * # echo 4 > /sys/kernel/debug/apple_gmux/selected_port
+ * # cat /sys/kernel/debug/apple_gmux/selected_port_data | xxd -p
+ * 00000005
+ *
+ * Reads 4 bytes from port 4 (GMUX_PORT_VERSION_MAJOR).
+ *
+ * 1 and 4 byte writes are also allowed.
+ */
+
+static ssize_t gmux_selected_port_data_write(struct file *file,
+		const char __user *userbuf, size_t count, loff_t *ppos)
+{
+	struct apple_gmux_data *gmux_data = file->private_data;
+
+	if (*ppos)
+		return -EINVAL;
+
+	if (count == 1) {
+		u8 data;
+
+		if (copy_from_user(&data, userbuf, 1))
+			return -EFAULT;
+
+		gmux_write8(gmux_data, gmux_data->selected_port, data);
+	} else if (count == 4) {
+		u32 data;
+
+		if (copy_from_user(&data, userbuf, 4))
+			return -EFAULT;
+
+		gmux_write32(gmux_data, gmux_data->selected_port, data);
+	} else
+		return -EINVAL;
+
+	return count;
+}
+
+static ssize_t gmux_selected_port_data_read(struct file *file,
+		char __user *userbuf, size_t count, loff_t *ppos)
+{
+	struct apple_gmux_data *gmux_data = file->private_data;
+	u32 data;
+
+	data = gmux_read32(gmux_data, gmux_data->selected_port);
+
+	return simple_read_from_buffer(userbuf, count, ppos, &data, sizeof(data));
+}
+
+static const struct file_operations gmux_port_data_ops = {
+	.open = simple_open,
+	.write = gmux_selected_port_data_write,
+	.read = gmux_selected_port_data_read
+};
+
+static void gmux_init_debugfs(struct apple_gmux_data *gmux_data)
+{
+	gmux_data->debug_dentry = debugfs_create_dir(KBUILD_MODNAME, NULL);
+
+	debugfs_create_u8("selected_port", 0644, gmux_data->debug_dentry,
+			&gmux_data->selected_port);
+	debugfs_create_file("selected_port_data", 0644, gmux_data->debug_dentry,
+			gmux_data, &gmux_port_data_ops);
+}
+
+static void gmux_fini_debugfs(struct apple_gmux_data *gmux_data)
+{
+	debugfs_remove_recursive(gmux_data->debug_dentry);
+}
+
 static int gmux_suspend(struct device *dev)
 {
 	struct pnp_dev *pnp = to_pnp_dev(dev);
@@ -560,18 +780,19 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id)
 	struct apple_gmux_data *gmux_data;
 	struct resource *res;
 	struct backlight_properties props;
-	struct backlight_device *bdev;
+	struct backlight_device *bdev = NULL;
 	u8 ver_major, ver_minor, ver_release;
+	bool register_bdev = true;
 	int ret = -ENXIO;
 	acpi_status status;
 	unsigned long long gpe;
-	bool indexed = false;
+	enum apple_gmux_type type;
 	u32 version;
 
 	if (apple_gmux_data)
 		return -EBUSY;
 
-	if (!apple_gmux_detect(pnp, &indexed)) {
+	if (!apple_gmux_detect(pnp, &type)) {
 		pr_info("gmux device not present\n");
 		return -ENODEV;
 	}
@@ -581,6 +802,35 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id)
 		return -ENOMEM;
 	pnp_set_drvdata(pnp, gmux_data);
 
+	switch (type) {
+	case APPLE_GMUX_TYPE_MMIO:
+		gmux_data->config = &apple_gmux_mmio;
+		mutex_init(&gmux_data->index_lock);
+
+		res = pnp_get_resource(pnp, IORESOURCE_MEM, 0);
+		gmux_data->iostart = res->start;
+		/* Although the ACPI table only allocates 8 bytes, we need 16. */
+		gmux_data->iolen = 16;
+		if (!request_mem_region(gmux_data->iostart, gmux_data->iolen,
+					"Apple gmux")) {
+			pr_err("gmux I/O already in use\n");
+			goto err_free;
+		}
+		gmux_data->iomem_base = ioremap(gmux_data->iostart, gmux_data->iolen);
+		if (!gmux_data->iomem_base) {
+			pr_err("couldn't remap gmux mmio region");
+			goto err_release;
+		}
+		goto get_version;
+	case APPLE_GMUX_TYPE_INDEXED:
+		gmux_data->config = &apple_gmux_index;
+		mutex_init(&gmux_data->index_lock);
+		break;
+	case APPLE_GMUX_TYPE_PIO:
+		gmux_data->config = &apple_gmux_pio;
+		break;
+	}
+
 	res = pnp_get_resource(pnp, IORESOURCE_IO, 0);
 	gmux_data->iostart = res->start;
 	gmux_data->iolen = resource_size(res);
@@ -591,9 +841,8 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id)
 		goto err_free;
 	}
 
-	if (indexed) {
-		mutex_init(&gmux_data->index_lock);
-		gmux_data->indexed = true;
+get_version:
+	if (gmux_data->config->read_version_as_u32) {
 		version = gmux_read32(gmux_data, GMUX_PORT_VERSION_MAJOR);
 		ver_major = (version >> 24) & 0xff;
 		ver_minor = (version >> 16) & 0xff;
@@ -604,39 +853,36 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id)
 		ver_release = gmux_read8(gmux_data, GMUX_PORT_VERSION_RELEASE);
 	}
 	pr_info("Found gmux version %d.%d.%d [%s]\n", ver_major, ver_minor,
-		ver_release, (gmux_data->indexed ? "indexed" : "classic"));
+		ver_release, gmux_data->config->name);
 
 	memset(&props, 0, sizeof(props));
 	props.type = BACKLIGHT_PLATFORM;
 	props.max_brightness = gmux_read32(gmux_data, GMUX_PORT_MAX_BRIGHTNESS);
 
-	/*
-	 * Currently it's assumed that the maximum brightness is less than
-	 * 2^24 for compatibility with old gmux versions. Cap the max
-	 * brightness at this value, but print a warning if the hardware
-	 * reports something higher so that it can be fixed.
-	 */
-	if (WARN_ON(props.max_brightness > GMUX_MAX_BRIGHTNESS))
-		props.max_brightness = GMUX_MAX_BRIGHTNESS;
-
-	bdev = backlight_device_register("gmux_backlight", &pnp->dev,
-					 gmux_data, &gmux_bl_ops, &props);
-	if (IS_ERR(bdev)) {
-		ret = PTR_ERR(bdev);
-		goto err_release;
-	}
-
-	gmux_data->bdev = bdev;
-	bdev->props.brightness = gmux_get_brightness(bdev);
-	backlight_update_status(bdev);
+#if IS_REACHABLE(CONFIG_ACPI_VIDEO)
+	register_bdev = acpi_video_get_backlight_type() == acpi_backlight_apple_gmux;
+#endif
+	if (register_bdev) {
+		/*
+		 * Currently it's assumed that the maximum brightness is less than
+		 * 2^24 for compatibility with old gmux versions. Cap the max
+		 * brightness at this value, but print a warning if the hardware
+		 * reports something higher so that it can be fixed.
+		 */
+		if (WARN_ON(props.max_brightness > GMUX_MAX_BRIGHTNESS))
+			props.max_brightness = GMUX_MAX_BRIGHTNESS;
+
+		bdev = backlight_device_register("gmux_backlight", &pnp->dev,
+						 gmux_data, &gmux_bl_ops, &props);
+		if (IS_ERR(bdev)) {
+			ret = PTR_ERR(bdev);
+			goto err_unmap;
+		}
 
-	/*
-	 * The backlight situation on Macs is complicated. If the gmux is
-	 * present it's the best choice, because it always works for
-	 * backlight control and supports more levels than other options.
-	 * Disable the other backlight choices.
-	 */
-	apple_bl_unregister();
+		gmux_data->bdev = bdev;
+		bdev->props.brightness = gmux_get_brightness(bdev);
+		backlight_update_status(bdev);
+	}
 
 	gmux_data->power_state = VGA_SWITCHEROO_ON;
 
@@ -690,21 +936,18 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id)
 	/*
 	 * Retina MacBook Pros cannot switch the panel's AUX separately
 	 * and need eDP pre-calibration. They are distinguishable from
-	 * pre-retinas by having an "indexed" gmux.
+	 * pre-retinas by having an "indexed" or "T2" gmux.
 	 *
 	 * Pre-retina MacBook Pros can switch the panel's DDC separately.
 	 */
-	if (gmux_data->indexed)
-		ret = vga_switcheroo_register_handler(&gmux_handler_indexed,
-					      VGA_SWITCHEROO_NEEDS_EDP_CONFIG);
-	else
-		ret = vga_switcheroo_register_handler(&gmux_handler_classic,
-					      VGA_SWITCHEROO_CAN_SWITCH_DDC);
+	ret = vga_switcheroo_register_handler(gmux_data->config->gmux_handler,
+			gmux_data->config->handler_flags);
 	if (ret) {
 		pr_err("Failed to register vga_switcheroo handler\n");
 		goto err_register_handler;
 	}
 
+	gmux_init_debugfs(gmux_data);
 	return 0;
 
 err_register_handler:
@@ -719,8 +962,14 @@ err_enable_gpe:
 					   &gmux_notify_handler);
 err_notify:
 	backlight_device_unregister(bdev);
+err_unmap:
+	if (gmux_data->iomem_base)
+		iounmap(gmux_data->iomem_base);
 err_release:
-	release_region(gmux_data->iostart, gmux_data->iolen);
+	if (gmux_data->config->resource_type == IORESOURCE_MEM)
+		release_mem_region(gmux_data->iostart, gmux_data->iolen);
+	else
+		release_region(gmux_data->iostart, gmux_data->iolen);
 err_free:
 	kfree(gmux_data);
 	return ret;
@@ -730,6 +979,7 @@ static void gmux_remove(struct pnp_dev *pnp)
 {
 	struct apple_gmux_data *gmux_data = pnp_get_drvdata(pnp);
 
+	gmux_fini_debugfs(gmux_data);
 	vga_switcheroo_unregister_handler();
 	gmux_disable_interrupts(gmux_data);
 	if (gmux_data->gpe >= 0) {
@@ -741,11 +991,13 @@ static void gmux_remove(struct pnp_dev *pnp)
 
 	backlight_device_unregister(gmux_data->bdev);
 
-	release_region(gmux_data->iostart, gmux_data->iolen);
+	if (gmux_data->iomem_base) {
+		iounmap(gmux_data->iomem_base);
+		release_mem_region(gmux_data->iostart, gmux_data->iolen);
+	} else
+		release_region(gmux_data->iostart, gmux_data->iolen);
 	apple_gmux_data = NULL;
 	kfree(gmux_data);
-
-	apple_bl_register();
 }
 
 static const struct pnp_device_id gmux_device_ids[] = {
diff --git a/drivers/platform/x86/barco-p50-gpio.c b/drivers/platform/x86/barco-p50-gpio.c
index 8dd672339485..af566f712057 100644
--- a/drivers/platform/x86/barco-p50-gpio.c
+++ b/drivers/platform/x86/barco-p50-gpio.c
@@ -370,7 +370,7 @@ err_leds:
 	return ret;
 }
 
-static int p50_gpio_remove(struct platform_device *pdev)
+static void p50_gpio_remove(struct platform_device *pdev)
 {
 	struct p50_gpio *p50 = platform_get_drvdata(pdev);
 
@@ -378,8 +378,6 @@ static int p50_gpio_remove(struct platform_device *pdev)
 	platform_device_unregister(p50->leds_pdev);
 
 	gpiod_remove_lookup_table(&p50_gpio_led_table);
-
-	return 0;
 }
 
 static struct platform_driver p50_gpio_driver = {
@@ -387,7 +385,7 @@ static struct platform_driver p50_gpio_driver = {
 		.name = DRIVER_NAME,
 	},
 	.probe = p50_gpio_probe,
-	.remove = p50_gpio_remove,
+	.remove_new = p50_gpio_remove,
 };
 
 /* Board setup */
diff --git a/drivers/platform/x86/classmate-laptop.c b/drivers/platform/x86/classmate-laptop.c
index 8b6a14611859..2edaea2492df 100644
--- a/drivers/platform/x86/classmate-laptop.c
+++ b/drivers/platform/x86/classmate-laptop.c
@@ -1134,7 +1134,7 @@ static void cmpc_exit(void)
 module_init(cmpc_init);
 module_exit(cmpc_exit);
 
-static const struct acpi_device_id cmpc_device_ids[] = {
+static const struct acpi_device_id cmpc_device_ids[] __maybe_unused = {
 	{CMPC_ACCEL_HID, 0},
 	{CMPC_ACCEL_HID_V4, 0},
 	{CMPC_TABLET_HID, 0},
diff --git a/drivers/platform/x86/compal-laptop.c b/drivers/platform/x86/compal-laptop.c
index e10d2f64dfad..61c745490d71 100644
--- a/drivers/platform/x86/compal-laptop.c
+++ b/drivers/platform/x86/compal-laptop.c
@@ -1003,12 +1003,12 @@ remove:
 	return err;
 }
 
-static int compal_remove(struct platform_device *pdev)
+static void compal_remove(struct platform_device *pdev)
 {
 	struct compal_data *data;
 
 	if (!extra_features)
-		return 0;
+		return;
 
 	pr_info("Unloading: resetting fan control to motherboard\n");
 	pwm_disable_control();
@@ -1017,8 +1017,6 @@ static int compal_remove(struct platform_device *pdev)
 	power_supply_unregister(data->psy);
 
 	sysfs_remove_group(&pdev->dev.kobj, &compal_platform_attr_group);
-
-	return 0;
 }
 
 static struct platform_driver compal_driver = {
@@ -1026,7 +1024,7 @@ static struct platform_driver compal_driver = {
 		.name = DRIVER_NAME,
 	},
 	.probe	= compal_probe,
-	.remove	= compal_remove,
+	.remove_new = compal_remove,
 };
 
 static int __init compal_init(void)
diff --git a/drivers/platform/x86/dell/dcdbas.c b/drivers/platform/x86/dell/dcdbas.c
index 0ecb7b164750..76787369d7fa 100644
--- a/drivers/platform/x86/dell/dcdbas.c
+++ b/drivers/platform/x86/dell/dcdbas.c
@@ -698,12 +698,10 @@ static int dcdbas_probe(struct platform_device *dev)
 	return 0;
 }
 
-static int dcdbas_remove(struct platform_device *dev)
+static void dcdbas_remove(struct platform_device *dev)
 {
 	unregister_reboot_notifier(&dcdbas_reboot_nb);
 	sysfs_remove_group(&dev->dev.kobj, &dcdbas_attr_group);
-
-	return 0;
 }
 
 static struct platform_driver dcdbas_driver = {
@@ -711,7 +709,7 @@ static struct platform_driver dcdbas_driver = {
 		.name	= DRIVER_NAME,
 	},
 	.probe		= dcdbas_probe,
-	.remove		= dcdbas_remove,
+	.remove_new	= dcdbas_remove,
 };
 
 static const struct platform_device_info dcdbas_dev_info __initconst = {
diff --git a/drivers/platform/x86/dell/dell-laptop.c b/drivers/platform/x86/dell/dell-laptop.c
index e92c3ad06d69..6586438356de 100644
--- a/drivers/platform/x86/dell/dell-laptop.c
+++ b/drivers/platform/x86/dell/dell-laptop.c
@@ -97,6 +97,7 @@ static struct rfkill *bluetooth_rfkill;
 static struct rfkill *wwan_rfkill;
 static bool force_rfkill;
 static bool micmute_led_registered;
+static bool mute_led_registered;
 
 module_param(force_rfkill, bool, 0444);
 MODULE_PARM_DESC(force_rfkill, "enable rfkill on non whitelisted models");
@@ -2177,6 +2178,34 @@ static struct led_classdev micmute_led_cdev = {
 	.default_trigger = "audio-micmute",
 };
 
+static int mute_led_set(struct led_classdev *led_cdev,
+			   enum led_brightness brightness)
+{
+	struct calling_interface_buffer buffer;
+	struct calling_interface_token *token;
+	int state = brightness != LED_OFF;
+
+	if (state == 0)
+		token = dell_smbios_find_token(GLOBAL_MUTE_DISABLE);
+	else
+		token = dell_smbios_find_token(GLOBAL_MUTE_ENABLE);
+
+	if (!token)
+		return -ENODEV;
+
+	dell_fill_request(&buffer, token->location, token->value, 0, 0);
+	dell_send_request(&buffer, CLASS_TOKEN_WRITE, SELECT_TOKEN_STD);
+
+	return 0;
+}
+
+static struct led_classdev mute_led_cdev = {
+	.name = "platform::mute",
+	.max_brightness = 1,
+	.brightness_set_blocking = mute_led_set,
+	.default_trigger = "audio-mute",
+};
+
 static int __init dell_init(void)
 {
 	struct calling_interface_token *token;
@@ -2230,6 +2259,15 @@ static int __init dell_init(void)
 		micmute_led_registered = true;
 	}
 
+	if (dell_smbios_find_token(GLOBAL_MUTE_DISABLE) &&
+	    dell_smbios_find_token(GLOBAL_MUTE_ENABLE)) {
+		mute_led_cdev.brightness = ledtrig_audio_get(LED_AUDIO_MUTE);
+		ret = led_classdev_register(&platform_device->dev, &mute_led_cdev);
+		if (ret < 0)
+			goto fail_backlight;
+		mute_led_registered = true;
+	}
+
 	if (acpi_video_get_backlight_type() != acpi_backlight_vendor)
 		return 0;
 
@@ -2277,6 +2315,8 @@ fail_get_brightness:
 fail_backlight:
 	if (micmute_led_registered)
 		led_classdev_unregister(&micmute_led_cdev);
+	if (mute_led_registered)
+		led_classdev_unregister(&mute_led_cdev);
 fail_led:
 	dell_cleanup_rfkill();
 fail_rfkill:
@@ -2299,6 +2339,8 @@ static void __exit dell_exit(void)
 	backlight_device_unregister(dell_backlight_device);
 	if (micmute_led_registered)
 		led_classdev_unregister(&micmute_led_cdev);
+	if (mute_led_registered)
+		led_classdev_unregister(&mute_led_cdev);
 	dell_cleanup_rfkill();
 	if (platform_device) {
 		platform_device_unregister(platform_device);
diff --git a/drivers/platform/x86/dell/dell-smbios.h b/drivers/platform/x86/dell/dell-smbios.h
index 75fa8ea0476d..eb341bf000c6 100644
--- a/drivers/platform/x86/dell/dell-smbios.h
+++ b/drivers/platform/x86/dell/dell-smbios.h
@@ -34,6 +34,8 @@
 #define KBD_LED_AUTO_100_TOKEN	0x02F6
 #define GLOBAL_MIC_MUTE_ENABLE	0x0364
 #define GLOBAL_MIC_MUTE_DISABLE	0x0365
+#define GLOBAL_MUTE_ENABLE	0x058C
+#define GLOBAL_MUTE_DISABLE	0x058D
 
 struct notifier_block;
 
diff --git a/drivers/platform/x86/dell/dell-smo8800.c b/drivers/platform/x86/dell/dell-smo8800.c
index 8d6b7a83cf24..f7ec17c56833 100644
--- a/drivers/platform/x86/dell/dell-smo8800.c
+++ b/drivers/platform/x86/dell/dell-smo8800.c
@@ -154,14 +154,13 @@ error:
 	return err;
 }
 
-static int smo8800_remove(struct platform_device *device)
+static void smo8800_remove(struct platform_device *device)
 {
 	struct smo8800_device *smo8800 = platform_get_drvdata(device);
 
 	free_irq(smo8800->irq, smo8800);
 	misc_deregister(&smo8800->miscdev);
 	dev_dbg(&device->dev, "device /dev/freefall unregistered\n");
-	return 0;
 }
 
 /* NOTE: Keep this list in sync with drivers/i2c/busses/i2c-i801.c */
@@ -180,7 +179,7 @@ MODULE_DEVICE_TABLE(acpi, smo8800_ids);
 
 static struct platform_driver smo8800_driver = {
 	.probe = smo8800_probe,
-	.remove = smo8800_remove,
+	.remove_new = smo8800_remove,
 	.driver = {
 		.name = DRIVER_NAME,
 		.acpi_match_table = smo8800_ids,
diff --git a/drivers/platform/x86/hp/hp_accel.c b/drivers/platform/x86/hp/hp_accel.c
index 6477591747cf..52535576772a 100644
--- a/drivers/platform/x86/hp/hp_accel.c
+++ b/drivers/platform/x86/hp/hp_accel.c
@@ -342,7 +342,7 @@ static int lis3lv02d_probe(struct platform_device *device)
 	return ret;
 }
 
-static int lis3lv02d_remove(struct platform_device *device)
+static void lis3lv02d_remove(struct platform_device *device)
 {
 	i8042_remove_filter(hp_accel_i8042_filter);
 	lis3lv02d_joystick_disable(&lis3_dev);
@@ -352,7 +352,6 @@ static int lis3lv02d_remove(struct platform_device *device)
 	flush_work(&hpled_led.work);
 
 	lis3lv02d_remove_fs(&lis3_dev);
-	return 0;
 }
 
 static int __maybe_unused lis3lv02d_suspend(struct device *dev)
@@ -373,7 +372,7 @@ static SIMPLE_DEV_PM_OPS(hp_accel_pm, lis3lv02d_suspend, lis3lv02d_resume);
 /* For the HP MDPS aka 3D Driveguard */
 static struct platform_driver lis3lv02d_driver = {
 	.probe	= lis3lv02d_probe,
-	.remove	= lis3lv02d_remove,
+	.remove_new = lis3lv02d_remove,
 	.driver	= {
 		.name	= "hp_accel",
 		.pm	= &hp_accel_pm,
diff --git a/drivers/platform/x86/hp/tc1100-wmi.c b/drivers/platform/x86/hp/tc1100-wmi.c
index ded26213c420..5298b0f6804f 100644
--- a/drivers/platform/x86/hp/tc1100-wmi.c
+++ b/drivers/platform/x86/hp/tc1100-wmi.c
@@ -170,11 +170,9 @@ static int __init tc1100_probe(struct platform_device *device)
 }
 
 
-static int tc1100_remove(struct platform_device *device)
+static void tc1100_remove(struct platform_device *device)
 {
 	sysfs_remove_group(&device->dev.kobj, &tc1100_attribute_group);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -223,7 +221,7 @@ static struct platform_driver tc1100_driver = {
 		.pm = &tc1100_pm_ops,
 #endif
 	},
-	.remove = tc1100_remove,
+	.remove_new = tc1100_remove,
 };
 
 static int __init tc1100_init(void)
diff --git a/drivers/platform/x86/huawei-wmi.c b/drivers/platform/x86/huawei-wmi.c
index 2df1b2d5e3ea..70e5c4c0574d 100644
--- a/drivers/platform/x86/huawei-wmi.c
+++ b/drivers/platform/x86/huawei-wmi.c
@@ -830,7 +830,7 @@ static int huawei_wmi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int huawei_wmi_remove(struct platform_device *pdev)
+static void huawei_wmi_remove(struct platform_device *pdev)
 {
 	const struct wmi_device_id *guid = huawei_wmi_events_id_table;
 
@@ -846,8 +846,6 @@ static int huawei_wmi_remove(struct platform_device *pdev)
 		huawei_wmi_battery_exit(&pdev->dev);
 		huawei_wmi_fn_lock_exit(&pdev->dev);
 	}
-
-	return 0;
 }
 
 static struct platform_driver huawei_wmi_driver = {
@@ -855,7 +853,7 @@ static struct platform_driver huawei_wmi_driver = {
 		.name = "huawei-wmi",
 	},
 	.probe = huawei_wmi_probe,
-	.remove = huawei_wmi_remove,
+	.remove_new = huawei_wmi_remove,
 };
 
 static __init int huawei_wmi_init(void)
diff --git a/drivers/platform/x86/ibm_rtl.c b/drivers/platform/x86/ibm_rtl.c
index 5fc665f7d9b3..2ab7d9ac542d 100644
--- a/drivers/platform/x86/ibm_rtl.c
+++ b/drivers/platform/x86/ibm_rtl.c
@@ -199,16 +199,26 @@ static int rtl_setup_sysfs(void) {
 
 	ret = subsys_system_register(&rtl_subsys, NULL);
 	if (!ret) {
-		for (i = 0; rtl_attributes[i]; i ++)
-			device_create_file(rtl_subsys.dev_root, rtl_attributes[i]);
+		struct device *dev_root = bus_get_dev_root(&rtl_subsys);
+
+		if (dev_root) {
+			for (i = 0; rtl_attributes[i]; i ++)
+				device_create_file(dev_root, rtl_attributes[i]);
+			put_device(dev_root);
+		}
 	}
 	return ret;
 }
 
 static void rtl_teardown_sysfs(void) {
+	struct device *dev_root = bus_get_dev_root(&rtl_subsys);
 	int i;
-	for (i = 0; rtl_attributes[i]; i ++)
-		device_remove_file(rtl_subsys.dev_root, rtl_attributes[i]);
+
+	if (dev_root) {
+		for (i = 0; rtl_attributes[i]; i ++)
+			device_remove_file(dev_root, rtl_attributes[i]);
+		put_device(dev_root);
+	}
 	bus_unregister(&rtl_subsys);
 }
 
diff --git a/drivers/platform/x86/ideapad-laptop.c b/drivers/platform/x86/ideapad-laptop.c
index 959ec3c5f376..d2fee9a3e239 100644
--- a/drivers/platform/x86/ideapad-laptop.c
+++ b/drivers/platform/x86/ideapad-laptop.c
@@ -20,7 +20,6 @@
 #include <linux/init.h>
 #include <linux/input.h>
 #include <linux/input/sparse-keymap.h>
-#include <linux/jiffies.h>
 #include <linux/kernel.h>
 #include <linux/leds.h>
 #include <linux/module.h>
@@ -31,6 +30,7 @@
 #include <linux/sysfs.h>
 #include <linux/types.h>
 #include <linux/wmi.h>
+#include "ideapad-laptop.h"
 
 #include <acpi/video.h>
 
@@ -85,33 +85,6 @@ enum {
 	SALS_FNLOCK_OFF       = 0xf,
 };
 
-enum {
-	VPCCMD_R_VPC1 = 0x10,
-	VPCCMD_R_BL_MAX,
-	VPCCMD_R_BL,
-	VPCCMD_W_BL,
-	VPCCMD_R_WIFI,
-	VPCCMD_W_WIFI,
-	VPCCMD_R_BT,
-	VPCCMD_W_BT,
-	VPCCMD_R_BL_POWER,
-	VPCCMD_R_NOVO,
-	VPCCMD_R_VPC2,
-	VPCCMD_R_TOUCHPAD,
-	VPCCMD_W_TOUCHPAD,
-	VPCCMD_R_CAMERA,
-	VPCCMD_W_CAMERA,
-	VPCCMD_R_3G,
-	VPCCMD_W_3G,
-	VPCCMD_R_ODD, /* 0x21 */
-	VPCCMD_W_FAN,
-	VPCCMD_R_RF,
-	VPCCMD_W_RF,
-	VPCCMD_R_FAN = 0x2B,
-	VPCCMD_R_SPECIAL_BUTTONS = 0x31,
-	VPCCMD_W_BL_POWER = 0x33,
-};
-
 struct ideapad_dytc_priv {
 	enum platform_profile_option current_profile;
 	struct platform_profile_handler pprof;
@@ -227,7 +200,6 @@ static void ideapad_shared_exit(struct ideapad_private *priv)
 /*
  * ACPI Helpers
  */
-#define IDEAPAD_EC_TIMEOUT 200 /* in ms */
 
 static int eval_int(acpi_handle handle, const char *name, unsigned long *res)
 {
@@ -270,116 +242,11 @@ static int exec_sals(acpi_handle handle, unsigned long arg)
 	return exec_simple_method(handle, "SALS", arg);
 }
 
-static int eval_int_with_arg(acpi_handle handle, const char *name, unsigned long arg, unsigned long *res)
-{
-	struct acpi_object_list params;
-	unsigned long long result;
-	union acpi_object in_obj;
-	acpi_status status;
-
-	params.count = 1;
-	params.pointer = &in_obj;
-	in_obj.type = ACPI_TYPE_INTEGER;
-	in_obj.integer.value = arg;
-
-	status = acpi_evaluate_integer(handle, (char *)name, &params, &result);
-	if (ACPI_FAILURE(status))
-		return -EIO;
-
-	if (res)
-		*res = result;
-
-	return 0;
-}
-
 static int eval_dytc(acpi_handle handle, unsigned long cmd, unsigned long *res)
 {
 	return eval_int_with_arg(handle, "DYTC", cmd, res);
 }
 
-static int eval_vpcr(acpi_handle handle, unsigned long cmd, unsigned long *res)
-{
-	return eval_int_with_arg(handle, "VPCR", cmd, res);
-}
-
-static int eval_vpcw(acpi_handle handle, unsigned long cmd, unsigned long data)
-{
-	struct acpi_object_list params;
-	union acpi_object in_obj[2];
-	acpi_status status;
-
-	params.count = 2;
-	params.pointer = in_obj;
-	in_obj[0].type = ACPI_TYPE_INTEGER;
-	in_obj[0].integer.value = cmd;
-	in_obj[1].type = ACPI_TYPE_INTEGER;
-	in_obj[1].integer.value = data;
-
-	status = acpi_evaluate_object(handle, "VPCW", &params, NULL);
-	if (ACPI_FAILURE(status))
-		return -EIO;
-
-	return 0;
-}
-
-static int read_ec_data(acpi_handle handle, unsigned long cmd, unsigned long *data)
-{
-	unsigned long end_jiffies, val;
-	int err;
-
-	err = eval_vpcw(handle, 1, cmd);
-	if (err)
-		return err;
-
-	end_jiffies = jiffies + msecs_to_jiffies(IDEAPAD_EC_TIMEOUT) + 1;
-
-	while (time_before(jiffies, end_jiffies)) {
-		schedule();
-
-		err = eval_vpcr(handle, 1, &val);
-		if (err)
-			return err;
-
-		if (val == 0)
-			return eval_vpcr(handle, 0, data);
-	}
-
-	acpi_handle_err(handle, "timeout in %s\n", __func__);
-
-	return -ETIMEDOUT;
-}
-
-static int write_ec_cmd(acpi_handle handle, unsigned long cmd, unsigned long data)
-{
-	unsigned long end_jiffies, val;
-	int err;
-
-	err = eval_vpcw(handle, 0, data);
-	if (err)
-		return err;
-
-	err = eval_vpcw(handle, 1, cmd);
-	if (err)
-		return err;
-
-	end_jiffies = jiffies + msecs_to_jiffies(IDEAPAD_EC_TIMEOUT) + 1;
-
-	while (time_before(jiffies, end_jiffies)) {
-		schedule();
-
-		err = eval_vpcr(handle, 1, &val);
-		if (err)
-			return err;
-
-		if (val == 0)
-			return 0;
-	}
-
-	acpi_handle_err(handle, "timeout in %s\n", __func__);
-
-	return -ETIMEDOUT;
-}
-
 /*
  * debugfs
  */
@@ -1918,7 +1785,7 @@ input_failed:
 	return err;
 }
 
-static int ideapad_acpi_remove(struct platform_device *pdev)
+static void ideapad_acpi_remove(struct platform_device *pdev)
 {
 	struct ideapad_private *priv = dev_get_drvdata(&pdev->dev);
 	int i;
@@ -1939,8 +1806,6 @@ static int ideapad_acpi_remove(struct platform_device *pdev)
 	ideapad_input_exit(priv);
 	ideapad_debugfs_exit(priv);
 	ideapad_sysfs_exit(priv);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1967,7 +1832,7 @@ MODULE_DEVICE_TABLE(acpi, ideapad_device_ids);
 
 static struct platform_driver ideapad_acpi_driver = {
 	.probe = ideapad_acpi_add,
-	.remove = ideapad_acpi_remove,
+	.remove_new = ideapad_acpi_remove,
 	.driver = {
 		.name   = "ideapad_acpi",
 		.pm     = &ideapad_pm,
diff --git a/drivers/platform/x86/ideapad-laptop.h b/drivers/platform/x86/ideapad-laptop.h
new file mode 100644
index 000000000000..4498a96de597
--- /dev/null
+++ b/drivers/platform/x86/ideapad-laptop.h
@@ -0,0 +1,152 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ *  ideapad-laptop.h - Lenovo IdeaPad ACPI Extras
+ *
+ *  Copyright © 2010 Intel Corporation
+ *  Copyright © 2010 David Woodhouse <dwmw2@infradead.org>
+ */
+
+#ifndef _IDEAPAD_LAPTOP_H_
+#define _IDEAPAD_LAPTOP_H_
+
+#include <linux/acpi.h>
+#include <linux/jiffies.h>
+#include <linux/errno.h>
+
+enum {
+	VPCCMD_R_VPC1 = 0x10,
+	VPCCMD_R_BL_MAX,
+	VPCCMD_R_BL,
+	VPCCMD_W_BL,
+	VPCCMD_R_WIFI,
+	VPCCMD_W_WIFI,
+	VPCCMD_R_BT,
+	VPCCMD_W_BT,
+	VPCCMD_R_BL_POWER,
+	VPCCMD_R_NOVO,
+	VPCCMD_R_VPC2,
+	VPCCMD_R_TOUCHPAD,
+	VPCCMD_W_TOUCHPAD,
+	VPCCMD_R_CAMERA,
+	VPCCMD_W_CAMERA,
+	VPCCMD_R_3G,
+	VPCCMD_W_3G,
+	VPCCMD_R_ODD, /* 0x21 */
+	VPCCMD_W_FAN,
+	VPCCMD_R_RF,
+	VPCCMD_W_RF,
+	VPCCMD_W_YMC = 0x2A,
+	VPCCMD_R_FAN = 0x2B,
+	VPCCMD_R_SPECIAL_BUTTONS = 0x31,
+	VPCCMD_W_BL_POWER = 0x33,
+};
+
+static inline int eval_int_with_arg(acpi_handle handle, const char *name, unsigned long arg, unsigned long *res)
+{
+	struct acpi_object_list params;
+	unsigned long long result;
+	union acpi_object in_obj;
+	acpi_status status;
+
+	params.count = 1;
+	params.pointer = &in_obj;
+	in_obj.type = ACPI_TYPE_INTEGER;
+	in_obj.integer.value = arg;
+
+	status = acpi_evaluate_integer(handle, (char *)name, &params, &result);
+	if (ACPI_FAILURE(status))
+		return -EIO;
+
+	if (res)
+		*res = result;
+
+	return 0;
+}
+
+static inline int eval_vpcr(acpi_handle handle, unsigned long cmd, unsigned long *res)
+{
+	return eval_int_with_arg(handle, "VPCR", cmd, res);
+}
+
+static inline int eval_vpcw(acpi_handle handle, unsigned long cmd, unsigned long data)
+{
+	struct acpi_object_list params;
+	union acpi_object in_obj[2];
+	acpi_status status;
+
+	params.count = 2;
+	params.pointer = in_obj;
+	in_obj[0].type = ACPI_TYPE_INTEGER;
+	in_obj[0].integer.value = cmd;
+	in_obj[1].type = ACPI_TYPE_INTEGER;
+	in_obj[1].integer.value = data;
+
+	status = acpi_evaluate_object(handle, "VPCW", &params, NULL);
+	if (ACPI_FAILURE(status))
+		return -EIO;
+
+	return 0;
+}
+
+#define IDEAPAD_EC_TIMEOUT 200 /* in ms */
+
+static inline int read_ec_data(acpi_handle handle, unsigned long cmd, unsigned long *data)
+{
+	unsigned long end_jiffies, val;
+	int err;
+
+	err = eval_vpcw(handle, 1, cmd);
+	if (err)
+		return err;
+
+	end_jiffies = jiffies + msecs_to_jiffies(IDEAPAD_EC_TIMEOUT) + 1;
+
+	while (time_before(jiffies, end_jiffies)) {
+		schedule();
+
+		err = eval_vpcr(handle, 1, &val);
+		if (err)
+			return err;
+
+		if (val == 0)
+			return eval_vpcr(handle, 0, data);
+	}
+
+	acpi_handle_err(handle, "timeout in %s\n", __func__);
+
+	return -ETIMEDOUT;
+}
+
+static inline int write_ec_cmd(acpi_handle handle, unsigned long cmd, unsigned long data)
+{
+	unsigned long end_jiffies, val;
+	int err;
+
+	err = eval_vpcw(handle, 0, data);
+	if (err)
+		return err;
+
+	err = eval_vpcw(handle, 1, cmd);
+	if (err)
+		return err;
+
+	end_jiffies = jiffies + msecs_to_jiffies(IDEAPAD_EC_TIMEOUT) + 1;
+
+	while (time_before(jiffies, end_jiffies)) {
+		schedule();
+
+		err = eval_vpcr(handle, 1, &val);
+		if (err)
+			return err;
+
+		if (val == 0)
+			return 0;
+	}
+
+	acpi_handle_err(handle, "timeout in %s\n", __func__);
+
+	return -ETIMEDOUT;
+}
+
+#undef IDEAPAD_EC_TIMEOUT
+#endif /* !_IDEAPAD_LAPTOP_H_ */
diff --git a/drivers/platform/x86/intel/Kconfig b/drivers/platform/x86/intel/Kconfig
index bbbd9e54e9ee..e9dc0c021029 100644
--- a/drivers/platform/x86/intel/Kconfig
+++ b/drivers/platform/x86/intel/Kconfig
@@ -80,6 +80,16 @@ config INTEL_BXTWC_PMIC_TMU
 	  This driver enables the alarm wakeup functionality in the TMU unit of
 	  Whiskey Cove PMIC.
 
+config INTEL_BYTCRC_PWRSRC
+	tristate "Intel Bay Trail Crystal Cove power source driver"
+	depends on INTEL_SOC_PMIC
+	help
+	  This option adds a power source driver for Crystal Cove PMICs
+	  on Intel Bay Trail devices.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called intel_bytcrc_pwrsrc.
+
 config INTEL_CHTDC_TI_PWRBTN
 	tristate "Intel Cherry Trail Dollar Cove TI power button driver"
 	depends on INTEL_SOC_PMIC_CHTDC_TI
diff --git a/drivers/platform/x86/intel/Makefile b/drivers/platform/x86/intel/Makefile
index 411df4040427..c1d5fe05e3f3 100644
--- a/drivers/platform/x86/intel/Makefile
+++ b/drivers/platform/x86/intel/Makefile
@@ -38,6 +38,8 @@ intel_bxtwc_tmu-y			:= bxtwc_tmu.o
 obj-$(CONFIG_INTEL_BXTWC_PMIC_TMU)	+= intel_bxtwc_tmu.o
 intel_crystal_cove_charger-y		:= crystal_cove_charger.o
 obj-$(CONFIG_X86_ANDROID_TABLETS)	+= intel_crystal_cove_charger.o
+intel_bytcrc_pwrsrc-y			:= bytcrc_pwrsrc.o
+obj-$(CONFIG_INTEL_BYTCRC_PWRSRC)	+= intel_bytcrc_pwrsrc.o
 intel_chtdc_ti_pwrbtn-y			:= chtdc_ti_pwrbtn.o
 obj-$(CONFIG_INTEL_CHTDC_TI_PWRBTN)	+= intel_chtdc_ti_pwrbtn.o
 intel_chtwc_int33fe-y			:= chtwc_int33fe.o
diff --git a/drivers/platform/x86/intel/bxtwc_tmu.c b/drivers/platform/x86/intel/bxtwc_tmu.c
index 7ccf583649e6..d0e2a3c293b0 100644
--- a/drivers/platform/x86/intel/bxtwc_tmu.c
+++ b/drivers/platform/x86/intel/bxtwc_tmu.c
@@ -89,7 +89,7 @@ static int bxt_wcove_tmu_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int bxt_wcove_tmu_remove(struct platform_device *pdev)
+static void bxt_wcove_tmu_remove(struct platform_device *pdev)
 {
 	struct wcove_tmu *wctmu = platform_get_drvdata(pdev);
 	unsigned int val;
@@ -101,7 +101,6 @@ static int bxt_wcove_tmu_remove(struct platform_device *pdev)
 	regmap_read(wctmu->regmap, BXTWC_MTMUIRQ_REG, &val);
 	regmap_write(wctmu->regmap, BXTWC_MTMUIRQ_REG,
 			val | BXTWC_TMU_ALRM_MASK);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -132,7 +131,7 @@ MODULE_DEVICE_TABLE(platform, bxt_wcove_tmu_id_table);
 
 static struct platform_driver bxt_wcove_tmu_driver = {
 	.probe = bxt_wcove_tmu_probe,
-	.remove = bxt_wcove_tmu_remove,
+	.remove_new = bxt_wcove_tmu_remove,
 	.driver = {
 		.name = "bxt_wcove_tmu",
 		.pm     = &bxtwc_tmu_pm_ops,
diff --git a/drivers/platform/x86/intel/bytcrc_pwrsrc.c b/drivers/platform/x86/intel/bytcrc_pwrsrc.c
new file mode 100644
index 000000000000..8a022b90d12d
--- /dev/null
+++ b/drivers/platform/x86/intel/bytcrc_pwrsrc.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Power-source driver for Bay Trail Crystal Cove PMIC
+ *
+ * Copyright (c) 2023 Hans de Goede <hdegoede@redhat.com>
+ *
+ * Based on intel_crystalcove_pwrsrc.c from Android kernel sources, which is:
+ * Copyright (C) 2013 Intel Corporation
+ */
+
+#include <linux/debugfs.h>
+#include <linux/mfd/intel_soc_pmic.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#define CRYSTALCOVE_SPWRSRC_REG		0x1E
+#define CRYSTALCOVE_RESETSRC0_REG	0x20
+#define CRYSTALCOVE_RESETSRC1_REG	0x21
+#define CRYSTALCOVE_WAKESRC_REG		0x22
+
+struct crc_pwrsrc_data {
+	struct regmap *regmap;
+	struct dentry *debug_dentry;
+	unsigned int resetsrc0;
+	unsigned int resetsrc1;
+	unsigned int wakesrc;
+};
+
+static const char * const pwrsrc_pwrsrc_info[] = {
+	/* bit 0 */ "USB",
+	/* bit 1 */ "DC in",
+	/* bit 2 */ "Battery",
+	NULL,
+};
+
+static const char * const pwrsrc_resetsrc0_info[] = {
+	/* bit 0 */ "SOC reporting a thermal event",
+	/* bit 1 */ "critical PMIC temperature",
+	/* bit 2 */ "critical system temperature",
+	/* bit 3 */ "critical battery temperature",
+	/* bit 4 */ "VSYS under voltage",
+	/* bit 5 */ "VSYS over voltage",
+	/* bit 6 */ "battery removal",
+	NULL,
+};
+
+static const char * const pwrsrc_resetsrc1_info[] = {
+	/* bit 0 */ "VCRIT threshold",
+	/* bit 1 */ "BATID reporting battery removal",
+	/* bit 2 */ "user pressing the power button",
+	NULL,
+};
+
+static const char * const pwrsrc_wakesrc_info[] = {
+	/* bit 0 */ "user pressing the power button",
+	/* bit 1 */ "a battery insertion",
+	/* bit 2 */ "a USB charger insertion",
+	/* bit 3 */ "an adapter insertion",
+	NULL,
+};
+
+static void crc_pwrsrc_log(struct seq_file *seq, const char *prefix,
+			   const char * const *info, unsigned int reg_val)
+{
+	int i;
+
+	for (i = 0; info[i]; i++) {
+		if (reg_val & BIT(i))
+			seq_printf(seq, "%s by %s\n", prefix, info[i]);
+	}
+}
+
+static int pwrsrc_show(struct seq_file *seq, void *unused)
+{
+	struct crc_pwrsrc_data *data = seq->private;
+	unsigned int reg_val;
+	int ret;
+
+	ret = regmap_read(data->regmap, CRYSTALCOVE_SPWRSRC_REG, &reg_val);
+	if (ret)
+		return ret;
+
+	crc_pwrsrc_log(seq, "System powered", pwrsrc_pwrsrc_info, reg_val);
+	return 0;
+}
+
+static int resetsrc_show(struct seq_file *seq, void *unused)
+{
+	struct crc_pwrsrc_data *data = seq->private;
+
+	crc_pwrsrc_log(seq, "Last shutdown caused", pwrsrc_resetsrc0_info, data->resetsrc0);
+	crc_pwrsrc_log(seq, "Last shutdown caused", pwrsrc_resetsrc1_info, data->resetsrc1);
+	return 0;
+}
+
+static int wakesrc_show(struct seq_file *seq, void *unused)
+{
+	struct crc_pwrsrc_data *data = seq->private;
+
+	crc_pwrsrc_log(seq, "Last wake caused", pwrsrc_wakesrc_info, data->wakesrc);
+	return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(pwrsrc);
+DEFINE_SHOW_ATTRIBUTE(resetsrc);
+DEFINE_SHOW_ATTRIBUTE(wakesrc);
+
+static int crc_pwrsrc_read_and_clear(struct crc_pwrsrc_data *data,
+				     unsigned int reg, unsigned int *val)
+{
+	int ret;
+
+	ret = regmap_read(data->regmap, reg, val);
+	if (ret)
+		return ret;
+
+	return regmap_write(data->regmap, reg, *val);
+}
+
+static int crc_pwrsrc_probe(struct platform_device *pdev)
+{
+	struct intel_soc_pmic *pmic = dev_get_drvdata(pdev->dev.parent);
+	struct crc_pwrsrc_data *data;
+	int ret;
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->regmap = pmic->regmap;
+
+	/*
+	 * Read + clear resetsrc0/1 and wakesrc now, so that they get
+	 * cleared even if the debugfs interface is never used.
+	 *
+	 * Properly clearing the wakesrc is important, leaving bit 0 of it
+	 * set turns reboot into poweroff on some tablets.
+	 */
+	ret = crc_pwrsrc_read_and_clear(data, CRYSTALCOVE_RESETSRC0_REG, &data->resetsrc0);
+	if (ret)
+		return ret;
+
+	ret = crc_pwrsrc_read_and_clear(data, CRYSTALCOVE_RESETSRC1_REG, &data->resetsrc1);
+	if (ret)
+		return ret;
+
+	ret = crc_pwrsrc_read_and_clear(data, CRYSTALCOVE_WAKESRC_REG, &data->wakesrc);
+	if (ret)
+		return ret;
+
+	data->debug_dentry = debugfs_create_dir(KBUILD_MODNAME, NULL);
+	debugfs_create_file("pwrsrc", 0444, data->debug_dentry, data, &pwrsrc_fops);
+	debugfs_create_file("resetsrc", 0444, data->debug_dentry, data, &resetsrc_fops);
+	debugfs_create_file("wakesrc", 0444, data->debug_dentry, data, &wakesrc_fops);
+
+	platform_set_drvdata(pdev, data);
+	return 0;
+}
+
+static int crc_pwrsrc_remove(struct platform_device *pdev)
+{
+	struct crc_pwrsrc_data *data = platform_get_drvdata(pdev);
+
+	debugfs_remove_recursive(data->debug_dentry);
+	return 0;
+}
+
+static struct platform_driver crc_pwrsrc_driver = {
+	.probe = crc_pwrsrc_probe,
+	.remove = crc_pwrsrc_remove,
+	.driver = {
+		.name = "crystal_cove_pwrsrc",
+	},
+};
+module_platform_driver(crc_pwrsrc_driver);
+
+MODULE_ALIAS("platform:crystal_cove_pwrsrc");
+MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
+MODULE_DESCRIPTION("Power-source driver for Bay Trail Crystal Cove PMIC");
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/intel/chtdc_ti_pwrbtn.c b/drivers/platform/x86/intel/chtdc_ti_pwrbtn.c
index 9606a994af22..615f8d1a0c68 100644
--- a/drivers/platform/x86/intel/chtdc_ti_pwrbtn.c
+++ b/drivers/platform/x86/intel/chtdc_ti_pwrbtn.c
@@ -67,11 +67,10 @@ static int chtdc_ti_pwrbtn_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int chtdc_ti_pwrbtn_remove(struct platform_device *pdev)
+static void chtdc_ti_pwrbtn_remove(struct platform_device *pdev)
 {
 	dev_pm_clear_wake_irq(&pdev->dev);
 	device_init_wakeup(&pdev->dev, false);
-	return 0;
 }
 
 static const struct platform_device_id chtdc_ti_pwrbtn_id_table[] = {
@@ -85,7 +84,7 @@ static struct platform_driver chtdc_ti_pwrbtn_driver = {
 		.name	= KBUILD_MODNAME,
 	},
 	.probe		= chtdc_ti_pwrbtn_probe,
-	.remove		= chtdc_ti_pwrbtn_remove,
+	.remove_new	= chtdc_ti_pwrbtn_remove,
 	.id_table	= chtdc_ti_pwrbtn_id_table,
 };
 module_platform_driver(chtdc_ti_pwrbtn_driver);
diff --git a/drivers/platform/x86/intel/chtwc_int33fe.c b/drivers/platform/x86/intel/chtwc_int33fe.c
index 2c9a7d52be07..848baecc1bb0 100644
--- a/drivers/platform/x86/intel/chtwc_int33fe.c
+++ b/drivers/platform/x86/intel/chtwc_int33fe.c
@@ -405,7 +405,7 @@ out_remove_nodes:
 	return ret;
 }
 
-static int cht_int33fe_typec_remove(struct platform_device *pdev)
+static void cht_int33fe_typec_remove(struct platform_device *pdev)
 {
 	struct cht_int33fe_data *data = platform_get_drvdata(pdev);
 
@@ -414,8 +414,6 @@ static int cht_int33fe_typec_remove(struct platform_device *pdev)
 	i2c_unregister_device(data->battery_fg);
 
 	cht_int33fe_remove_nodes(data);
-
-	return 0;
 }
 
 static const struct acpi_device_id cht_int33fe_acpi_ids[] = {
@@ -429,7 +427,7 @@ static struct platform_driver cht_int33fe_typec_driver = {
 		.acpi_match_table = ACPI_PTR(cht_int33fe_acpi_ids),
 	},
 	.probe = cht_int33fe_typec_probe,
-	.remove = cht_int33fe_typec_remove,
+	.remove_new = cht_int33fe_typec_remove,
 };
 
 module_platform_driver(cht_int33fe_typec_driver);
diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c
index b6c06b37862e..5632bd3c534a 100644
--- a/drivers/platform/x86/intel/hid.c
+++ b/drivers/platform/x86/intel/hid.c
@@ -720,7 +720,7 @@ err_remove_notify:
 	return err;
 }
 
-static int intel_hid_remove(struct platform_device *device)
+static void intel_hid_remove(struct platform_device *device)
 {
 	acpi_handle handle = ACPI_HANDLE(&device->dev);
 
@@ -728,12 +728,6 @@ static int intel_hid_remove(struct platform_device *device)
 	acpi_remove_notify_handler(handle, ACPI_DEVICE_NOTIFY, notify_handler);
 	intel_hid_set_enable(&device->dev, false);
 	intel_button_array_enable(&device->dev, false);
-
-	/*
-	 * Even if we failed to shut off the event stream, we can still
-	 * safely detach from the device.
-	 */
-	return 0;
 }
 
 static struct platform_driver intel_hid_pl_driver = {
@@ -743,7 +737,7 @@ static struct platform_driver intel_hid_pl_driver = {
 		.pm = &intel_hid_pl_pm_ops,
 	},
 	.probe = intel_hid_probe,
-	.remove = intel_hid_remove,
+	.remove_new = intel_hid_remove,
 };
 
 /*
diff --git a/drivers/platform/x86/intel/ifs/core.c b/drivers/platform/x86/intel/ifs/core.c
index 206a617c2e02..306f886b52d2 100644
--- a/drivers/platform/x86/intel/ifs/core.c
+++ b/drivers/platform/x86/intel/ifs/core.c
@@ -16,27 +16,63 @@
 
 static const struct x86_cpu_id ifs_cpu_ids[] __initconst = {
 	X86_MATCH(SAPPHIRERAPIDS_X),
+	X86_MATCH(EMERALDRAPIDS_X),
 	{}
 };
 MODULE_DEVICE_TABLE(x86cpu, ifs_cpu_ids);
 
-static struct ifs_device ifs_device = {
-	.data = {
-		.integrity_cap_bit = MSR_INTEGRITY_CAPS_PERIODIC_BIST_BIT,
-		.test_num = 0,
+ATTRIBUTE_GROUPS(plat_ifs);
+ATTRIBUTE_GROUPS(plat_ifs_array);
+
+bool *ifs_pkg_auth;
+
+static const struct ifs_test_caps scan_test = {
+	.integrity_cap_bit = MSR_INTEGRITY_CAPS_PERIODIC_BIST_BIT,
+	.test_num = IFS_TYPE_SAF,
+};
+
+static const struct ifs_test_caps array_test = {
+	.integrity_cap_bit = MSR_INTEGRITY_CAPS_ARRAY_BIST_BIT,
+	.test_num = IFS_TYPE_ARRAY_BIST,
+};
+
+static struct ifs_device ifs_devices[] = {
+	[IFS_TYPE_SAF] = {
+		.test_caps = &scan_test,
+		.misc = {
+			.name = "intel_ifs_0",
+			.minor = MISC_DYNAMIC_MINOR,
+			.groups = plat_ifs_groups,
+		},
 	},
-	.misc = {
-		.name = "intel_ifs_0",
-		.nodename = "intel_ifs/0",
-		.minor = MISC_DYNAMIC_MINOR,
+	[IFS_TYPE_ARRAY_BIST] = {
+		.test_caps = &array_test,
+		.misc = {
+			.name = "intel_ifs_1",
+			.minor = MISC_DYNAMIC_MINOR,
+			.groups = plat_ifs_array_groups,
+		},
 	},
 };
 
+#define IFS_NUMTESTS ARRAY_SIZE(ifs_devices)
+
+static void ifs_cleanup(void)
+{
+	int i;
+
+	for (i = 0; i < IFS_NUMTESTS; i++) {
+		if (ifs_devices[i].misc.this_device)
+			misc_deregister(&ifs_devices[i].misc);
+	}
+	kfree(ifs_pkg_auth);
+}
+
 static int __init ifs_init(void)
 {
 	const struct x86_cpu_id *m;
 	u64 msrval;
-	int ret;
+	int i, ret;
 
 	m = x86_match_cpu(ifs_cpu_ids);
 	if (!m)
@@ -51,28 +87,27 @@ static int __init ifs_init(void)
 	if (rdmsrl_safe(MSR_INTEGRITY_CAPS, &msrval))
 		return -ENODEV;
 
-	ifs_device.misc.groups = ifs_get_groups();
-
-	if (!(msrval & BIT(ifs_device.data.integrity_cap_bit)))
-		return -ENODEV;
-
-	ifs_device.data.pkg_auth = kmalloc_array(topology_max_packages(), sizeof(bool), GFP_KERNEL);
-	if (!ifs_device.data.pkg_auth)
+	ifs_pkg_auth = kmalloc_array(topology_max_packages(), sizeof(bool), GFP_KERNEL);
+	if (!ifs_pkg_auth)
 		return -ENOMEM;
 
-	ret = misc_register(&ifs_device.misc);
-	if (ret) {
-		kfree(ifs_device.data.pkg_auth);
-		return ret;
+	for (i = 0; i < IFS_NUMTESTS; i++) {
+		if (!(msrval & BIT(ifs_devices[i].test_caps->integrity_cap_bit)))
+			continue;
+		ret = misc_register(&ifs_devices[i].misc);
+		if (ret)
+			goto err_exit;
 	}
-
 	return 0;
+
+err_exit:
+	ifs_cleanup();
+	return ret;
 }
 
 static void __exit ifs_exit(void)
 {
-	misc_deregister(&ifs_device.misc);
-	kfree(ifs_device.data.pkg_auth);
+	ifs_cleanup();
 }
 
 module_init(ifs_init);
diff --git a/drivers/platform/x86/intel/ifs/ifs.h b/drivers/platform/x86/intel/ifs/ifs.h
index 046e39304fd5..93191855890f 100644
--- a/drivers/platform/x86/intel/ifs/ifs.h
+++ b/drivers/platform/x86/intel/ifs/ifs.h
@@ -17,7 +17,7 @@
  * In Field Scan (IFS) is a hardware feature to run circuit level tests on
  * a CPU core to detect problems that are not caught by parity or ECC checks.
  * Future CPUs will support more than one type of test which will show up
- * with a new platform-device instance-id, for now only .0 is exposed.
+ * with a new platform-device instance-id.
  *
  *
  * IFS Image
@@ -25,7 +25,10 @@
  *
  * Intel provides a firmware file containing the scan tests via
  * github [#f1]_.  Similar to microcode there is a separate file for each
- * family-model-stepping.
+ * family-model-stepping. IFS Images are not applicable for some test types.
+ * Wherever applicable the sysfs directory would provide a "current_batch" file
+ * (see below) for loading the image.
+ *
  *
  * IFS Image Loading
  * -----------------
@@ -35,7 +38,7 @@
  * SHA hashes for the test. Then the tests themselves. Status MSRs provide
  * feedback on the success/failure of these steps.
  *
- * The test files are kept in a fixed location: /lib/firmware/intel/ifs_0/
+ * The test files are kept in a fixed location: /lib/firmware/intel/ifs_<n>/
  * For e.g if there are 3 test files, they would be named in the following
  * fashion:
  * ff-mm-ss-01.scan
@@ -47,7 +50,7 @@
  * (e.g 1, 2 or 3 in the above scenario) into the curent_batch file.
  * To load ff-mm-ss-02.scan, the following command can be used::
  *
- *   # echo 2 > /sys/devices/virtual/misc/intel_ifs_0/current_batch
+ *   # echo 2 > /sys/devices/virtual/misc/intel_ifs_<n>/current_batch
  *
  * The above file can also be read to know the currently loaded image.
  *
@@ -69,16 +72,16 @@
  * to migrate those applications to other cores before running a core test.
  * It may also be necessary to redirect interrupts to other CPUs.
  *
- * In all cases reading the SCAN_STATUS MSR provides details on what
+ * In all cases reading the corresponding test's STATUS MSR provides details on what
  * happened. The driver makes the value of this MSR visible to applications
  * via the "details" file (see below). Interrupted tests may be restarted.
  *
- * The IFS driver provides sysfs interfaces via /sys/devices/virtual/misc/intel_ifs_0/
+ * The IFS driver provides sysfs interfaces via /sys/devices/virtual/misc/intel_ifs_<n>/
  * to control execution:
  *
  * Test a specific core::
  *
- *   # echo <cpu#> > /sys/devices/virtual/misc/intel_ifs_0/run_test
+ *   # echo <cpu#> > /sys/devices/virtual/misc/intel_ifs_<n>/run_test
  *
  * when HT is enabled any of the sibling cpu# can be specified to test
  * its corresponding physical core. Since the tests are per physical core,
@@ -87,21 +90,21 @@
  *
  * For e.g. to test core corresponding to cpu5
  *
- *   # echo 5 > /sys/devices/virtual/misc/intel_ifs_0/run_test
+ *   # echo 5 > /sys/devices/virtual/misc/intel_ifs_<n>/run_test
  *
  * Results of the last test is provided in /sys::
  *
- *   $ cat /sys/devices/virtual/misc/intel_ifs_0/status
+ *   $ cat /sys/devices/virtual/misc/intel_ifs_<n>/status
  *   pass
  *
  * Status can be one of pass, fail, untested
  *
  * Additional details of the last test is provided by the details file::
  *
- *   $ cat /sys/devices/virtual/misc/intel_ifs_0/details
+ *   $ cat /sys/devices/virtual/misc/intel_ifs_<n>/details
  *   0x8081
  *
- * The details file reports the hex value of the SCAN_STATUS MSR.
+ * The details file reports the hex value of the test specific status MSR.
  * Hardware defined error codes are documented in volume 4 of the Intel
  * Software Developer's Manual but the error_code field may contain one of
  * the following driver defined software codes:
@@ -127,6 +130,7 @@
 #include <linux/device.h>
 #include <linux/miscdevice.h>
 
+#define MSR_ARRAY_BIST				0x00000105
 #define MSR_COPY_SCAN_HASHES			0x000002c2
 #define MSR_SCAN_HASHES_STATUS			0x000002c3
 #define MSR_AUTHENTICATE_AND_COPY_CHUNK		0x000002c4
@@ -137,6 +141,9 @@
 #define SCAN_TEST_PASS				1
 #define SCAN_TEST_FAIL				2
 
+#define IFS_TYPE_SAF			0
+#define IFS_TYPE_ARRAY_BIST		1
+
 /* MSR_SCAN_HASHES_STATUS bit fields */
 union ifs_scan_hashes_status {
 	u64	data;
@@ -189,6 +196,17 @@ union ifs_status {
 	};
 };
 
+/* MSR_ARRAY_BIST bit fields */
+union ifs_array {
+	u64	data;
+	struct {
+		u32	array_bitmask;
+		u16	array_bank;
+		u16	rsvd			:15;
+		u16	ctrl_result		:1;
+	};
+};
+
 /*
  * Driver populated error-codes
  * 0xFD: Test timed out before completing all the chunks.
@@ -197,22 +215,22 @@ union ifs_status {
 #define IFS_SW_TIMEOUT				0xFD
 #define IFS_SW_PARTIAL_COMPLETION		0xFE
 
+struct ifs_test_caps {
+	int	integrity_cap_bit;
+	int	test_num;
+};
+
 /**
  * struct ifs_data - attributes related to intel IFS driver
- * @integrity_cap_bit: MSR_INTEGRITY_CAPS bit enumerating this test
  * @loaded_version: stores the currently loaded ifs image version.
- * @pkg_auth: array of bool storing per package auth status
  * @loaded: If a valid test binary has been loaded into the memory
  * @loading_error: Error occurred on another CPU while loading image
  * @valid_chunks: number of chunks which could be validated.
  * @status: it holds simple status pass/fail/untested
  * @scan_details: opaque scan status code from h/w
  * @cur_batch: number indicating the currently loaded test file
- * @test_num: number indicating the test type
  */
 struct ifs_data {
-	int	integrity_cap_bit;
-	bool	*pkg_auth;
 	int	loaded_version;
 	bool	loaded;
 	bool	loading_error;
@@ -220,7 +238,6 @@ struct ifs_data {
 	int	status;
 	u64	scan_details;
 	u32	cur_batch;
-	int	test_num;
 };
 
 struct ifs_work {
@@ -229,7 +246,8 @@ struct ifs_work {
 };
 
 struct ifs_device {
-	struct ifs_data data;
+	const struct ifs_test_caps *test_caps;
+	struct ifs_data rw_data;
 	struct miscdevice misc;
 };
 
@@ -238,11 +256,21 @@ static inline struct ifs_data *ifs_get_data(struct device *dev)
 	struct miscdevice *m = dev_get_drvdata(dev);
 	struct ifs_device *d = container_of(m, struct ifs_device, misc);
 
-	return &d->data;
+	return &d->rw_data;
+}
+
+static inline const struct ifs_test_caps *ifs_get_test_caps(struct device *dev)
+{
+	struct miscdevice *m = dev_get_drvdata(dev);
+	struct ifs_device *d = container_of(m, struct ifs_device, misc);
+
+	return d->test_caps;
 }
 
+extern bool *ifs_pkg_auth;
 int ifs_load_firmware(struct device *dev);
 int do_core_test(int cpu, struct device *dev);
-const struct attribute_group **ifs_get_groups(void);
+extern struct attribute *plat_ifs_attrs[];
+extern struct attribute *plat_ifs_array_attrs[];
 
 #endif
diff --git a/drivers/platform/x86/intel/ifs/load.c b/drivers/platform/x86/intel/ifs/load.c
index c5c24e6fdc43..61dffb4c8a1d 100644
--- a/drivers/platform/x86/intel/ifs/load.c
+++ b/drivers/platform/x86/intel/ifs/load.c
@@ -192,7 +192,7 @@ static int scan_chunks_sanity_check(struct device *dev)
 	struct ifs_work local_work;
 	int curr_pkg, cpu, ret;
 
-	memset(ifsd->pkg_auth, 0, (topology_max_packages() * sizeof(bool)));
+	memset(ifs_pkg_auth, 0, (topology_max_packages() * sizeof(bool)));
 	ret = validate_ifs_metadata(dev);
 	if (ret)
 		return ret;
@@ -204,7 +204,7 @@ static int scan_chunks_sanity_check(struct device *dev)
 	cpus_read_lock();
 	for_each_online_cpu(cpu) {
 		curr_pkg = topology_physical_package_id(cpu);
-		if (ifsd->pkg_auth[curr_pkg])
+		if (ifs_pkg_auth[curr_pkg])
 			continue;
 		reinit_completion(&ifs_done);
 		local_work.dev = dev;
@@ -215,7 +215,7 @@ static int scan_chunks_sanity_check(struct device *dev)
 			ret = -EIO;
 			goto out;
 		}
-		ifsd->pkg_auth[curr_pkg] = 1;
+		ifs_pkg_auth[curr_pkg] = 1;
 	}
 	ret = 0;
 out:
@@ -257,13 +257,14 @@ static int image_sanity_check(struct device *dev, const struct microcode_header_
  */
 int ifs_load_firmware(struct device *dev)
 {
+	const struct ifs_test_caps *test = ifs_get_test_caps(dev);
 	struct ifs_data *ifsd = ifs_get_data(dev);
 	const struct firmware *fw;
 	char scan_path[64];
 	int ret = -EINVAL;
 
 	snprintf(scan_path, sizeof(scan_path), "intel/ifs_%d/%02x-%02x-%02x-%02x.scan",
-		 ifsd->test_num, boot_cpu_data.x86, boot_cpu_data.x86_model,
+		 test->test_num, boot_cpu_data.x86, boot_cpu_data.x86_model,
 		 boot_cpu_data.x86_stepping, ifsd->cur_batch);
 
 	ret = request_firmware_direct(&fw, scan_path, dev);
diff --git a/drivers/platform/x86/intel/ifs/runtest.c b/drivers/platform/x86/intel/ifs/runtest.c
index 0bfd8fcdd7e8..1061eb7ec399 100644
--- a/drivers/platform/x86/intel/ifs/runtest.c
+++ b/drivers/platform/x86/intel/ifs/runtest.c
@@ -229,6 +229,85 @@ static void ifs_test_core(int cpu, struct device *dev)
 	}
 }
 
+#define SPINUNIT 100 /* 100 nsec */
+static atomic_t array_cpus_out;
+
+/*
+ * Simplified cpu sibling rendezvous loop based on microcode loader __wait_for_cpus()
+ */
+static void wait_for_sibling_cpu(atomic_t *t, long long timeout)
+{
+	int cpu = smp_processor_id();
+	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
+	int all_cpus = cpumask_weight(smt_mask);
+
+	atomic_inc(t);
+	while (atomic_read(t) < all_cpus) {
+		if (timeout < SPINUNIT)
+			return;
+		ndelay(SPINUNIT);
+		timeout -= SPINUNIT;
+		touch_nmi_watchdog();
+	}
+}
+
+static int do_array_test(void *data)
+{
+	union ifs_array *command = data;
+	int cpu = smp_processor_id();
+	int first;
+
+	/*
+	 * Only one logical CPU on a core needs to trigger the Array test via MSR write.
+	 */
+	first = cpumask_first(cpu_smt_mask(cpu));
+
+	if (cpu == first) {
+		wrmsrl(MSR_ARRAY_BIST, command->data);
+		/* Pass back the result of the test */
+		rdmsrl(MSR_ARRAY_BIST, command->data);
+	}
+
+	/* Tests complete faster if the sibling is spinning here */
+	wait_for_sibling_cpu(&array_cpus_out, NSEC_PER_SEC);
+
+	return 0;
+}
+
+static void ifs_array_test_core(int cpu, struct device *dev)
+{
+	union ifs_array command = {};
+	bool timed_out = false;
+	struct ifs_data *ifsd;
+	unsigned long timeout;
+
+	ifsd = ifs_get_data(dev);
+
+	command.array_bitmask = ~0U;
+	timeout = jiffies + HZ / 2;
+
+	do {
+		if (time_after(jiffies, timeout)) {
+			timed_out = true;
+			break;
+		}
+		atomic_set(&array_cpus_out, 0);
+		stop_core_cpuslocked(cpu, do_array_test, &command);
+
+		if (command.ctrl_result)
+			break;
+	} while (command.array_bitmask);
+
+	ifsd->scan_details = command.data;
+
+	if (command.ctrl_result)
+		ifsd->status = SCAN_TEST_FAIL;
+	else if (timed_out || command.array_bitmask)
+		ifsd->status = SCAN_NOT_TESTED;
+	else
+		ifsd->status = SCAN_TEST_PASS;
+}
+
 /*
  * Initiate per core test. It wakes up work queue threads on the target cpu and
  * its sibling cpu. Once all sibling threads wake up, the scan test gets executed and
@@ -236,6 +315,8 @@ static void ifs_test_core(int cpu, struct device *dev)
  */
 int do_core_test(int cpu, struct device *dev)
 {
+	const struct ifs_test_caps *test = ifs_get_test_caps(dev);
+	struct ifs_data *ifsd = ifs_get_data(dev);
 	int ret = 0;
 
 	/* Prevent CPUs from being taken offline during the scan test */
@@ -247,7 +328,18 @@ int do_core_test(int cpu, struct device *dev)
 		goto out;
 	}
 
-	ifs_test_core(cpu, dev);
+	switch (test->test_num) {
+	case IFS_TYPE_SAF:
+		if (!ifsd->loaded)
+			return -EPERM;
+		ifs_test_core(cpu, dev);
+		break;
+	case IFS_TYPE_ARRAY_BIST:
+		ifs_array_test_core(cpu, dev);
+		break;
+	default:
+		return -EINVAL;
+	}
 out:
 	cpus_read_unlock();
 	return ret;
diff --git a/drivers/platform/x86/intel/ifs/sysfs.c b/drivers/platform/x86/intel/ifs/sysfs.c
index ee636a76b083..01b7502f46b0 100644
--- a/drivers/platform/x86/intel/ifs/sysfs.c
+++ b/drivers/platform/x86/intel/ifs/sysfs.c
@@ -13,7 +13,7 @@
  * Protects against simultaneous tests on multiple cores, or
  * reloading can file while a test is in progress
  */
-static DEFINE_SEMAPHORE(ifs_sem);
+static DEFINE_SEMAPHORE(ifs_sem, 1);
 
 /*
  * The sysfs interface to check additional details of last test
@@ -64,7 +64,6 @@ static ssize_t run_test_store(struct device *dev,
 			      struct device_attribute *attr,
 			      const char *buf, size_t count)
 {
-	struct ifs_data *ifsd = ifs_get_data(dev);
 	unsigned int cpu;
 	int rc;
 
@@ -75,10 +74,7 @@ static ssize_t run_test_store(struct device *dev,
 	if (down_interruptible(&ifs_sem))
 		return -EINTR;
 
-	if (!ifsd->loaded)
-		rc = -EPERM;
-	else
-		rc = do_core_test(cpu, dev);
+	rc = do_core_test(cpu, dev);
 
 	up(&ifs_sem);
 
@@ -141,7 +137,7 @@ static ssize_t image_version_show(struct device *dev,
 static DEVICE_ATTR_RO(image_version);
 
 /* global scan sysfs attributes */
-static struct attribute *plat_ifs_attrs[] = {
+struct attribute *plat_ifs_attrs[] = {
 	&dev_attr_details.attr,
 	&dev_attr_status.attr,
 	&dev_attr_run_test.attr,
@@ -150,9 +146,10 @@ static struct attribute *plat_ifs_attrs[] = {
 	NULL
 };
 
-ATTRIBUTE_GROUPS(plat_ifs);
-
-const struct attribute_group **ifs_get_groups(void)
-{
-	return plat_ifs_groups;
-}
+/* global array sysfs attributes */
+struct attribute *plat_ifs_array_attrs[] = {
+	&dev_attr_details.attr,
+	&dev_attr_status.attr,
+	&dev_attr_run_test.attr,
+	NULL
+};
diff --git a/drivers/platform/x86/intel/int0002_vgpio.c b/drivers/platform/x86/intel/int0002_vgpio.c
index 97cfbc520a02..b6708bab7c53 100644
--- a/drivers/platform/x86/intel/int0002_vgpio.c
+++ b/drivers/platform/x86/intel/int0002_vgpio.c
@@ -223,11 +223,10 @@ static int int0002_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int int0002_remove(struct platform_device *pdev)
+static void int0002_remove(struct platform_device *pdev)
 {
 	device_init_wakeup(&pdev->dev, false);
 	acpi_unregister_wakeup_handler(int0002_check_wake, NULL);
-	return 0;
 }
 
 static int int0002_suspend(struct device *dev)
@@ -273,7 +272,7 @@ static struct platform_driver int0002_driver = {
 		.pm			= &int0002_pm_ops,
 	},
 	.probe	= int0002_probe,
-	.remove	= int0002_remove,
+	.remove_new = int0002_remove,
 };
 
 module_platform_driver(int0002_driver);
diff --git a/drivers/platform/x86/intel/int1092/intel_sar.c b/drivers/platform/x86/intel/int1092/intel_sar.c
index 352fc4596494..6246c066ade2 100644
--- a/drivers/platform/x86/intel/int1092/intel_sar.c
+++ b/drivers/platform/x86/intel/int1092/intel_sar.c
@@ -292,7 +292,7 @@ r_free:
 	return result;
 }
 
-static int sar_remove(struct platform_device *device)
+static void sar_remove(struct platform_device *device)
 {
 	struct wwan_sar_context *context = dev_get_drvdata(&device->dev);
 	int reg;
@@ -304,12 +304,11 @@ static int sar_remove(struct platform_device *device)
 		kfree(context->config_data[reg].device_mode_info);
 
 	kfree(context);
-	return 0;
 }
 
 static struct platform_driver sar_driver = {
 	.probe = sar_probe,
-	.remove = sar_remove,
+	.remove_new = sar_remove,
 	.driver = {
 		.name = DRVNAME,
 		.acpi_match_table = ACPI_PTR(sar_device_ids)
diff --git a/drivers/platform/x86/intel/int3472/discrete.c b/drivers/platform/x86/intel/int3472/discrete.c
index f064da74f50a..ef020e23e596 100644
--- a/drivers/platform/x86/intel/int3472/discrete.c
+++ b/drivers/platform/x86/intel/int3472/discrete.c
@@ -317,7 +317,7 @@ static int skl_int3472_parse_crs(struct int3472_discrete_device *int3472)
 	return 0;
 }
 
-static int skl_int3472_discrete_remove(struct platform_device *pdev)
+static void skl_int3472_discrete_remove(struct platform_device *pdev)
 {
 	struct int3472_discrete_device *int3472 = platform_get_drvdata(pdev);
 
@@ -326,8 +326,6 @@ static int skl_int3472_discrete_remove(struct platform_device *pdev)
 	skl_int3472_unregister_clock(int3472);
 	skl_int3472_unregister_pled(int3472);
 	skl_int3472_unregister_regulator(int3472);
-
-	return 0;
 }
 
 static int skl_int3472_discrete_probe(struct platform_device *pdev)
@@ -392,7 +390,7 @@ static struct platform_driver int3472_discrete = {
 		.acpi_match_table = int3472_device_id,
 	},
 	.probe = skl_int3472_discrete_probe,
-	.remove = skl_int3472_discrete_remove,
+	.remove_new = skl_int3472_discrete_remove,
 };
 module_platform_driver(int3472_discrete);
 
diff --git a/drivers/platform/x86/intel/mrfld_pwrbtn.c b/drivers/platform/x86/intel/mrfld_pwrbtn.c
index d58fea51747e..549a3f586f3b 100644
--- a/drivers/platform/x86/intel/mrfld_pwrbtn.c
+++ b/drivers/platform/x86/intel/mrfld_pwrbtn.c
@@ -78,13 +78,12 @@ static int mrfld_pwrbtn_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int mrfld_pwrbtn_remove(struct platform_device *pdev)
+static void mrfld_pwrbtn_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 
 	dev_pm_clear_wake_irq(dev);
 	device_init_wakeup(dev, false);
-	return 0;
 }
 
 static const struct platform_device_id mrfld_pwrbtn_id_table[] = {
@@ -98,7 +97,7 @@ static struct platform_driver mrfld_pwrbtn_driver = {
 		.name	= "mrfld_bcove_pwrbtn",
 	},
 	.probe		= mrfld_pwrbtn_probe,
-	.remove		= mrfld_pwrbtn_remove,
+	.remove_new	= mrfld_pwrbtn_remove,
 	.id_table	= mrfld_pwrbtn_id_table,
 };
 module_platform_driver(mrfld_pwrbtn_driver);
diff --git a/drivers/platform/x86/intel/pmc/core.c b/drivers/platform/x86/intel/pmc/core.c
index b9591969e0fa..da6e7206d38b 100644
--- a/drivers/platform/x86/intel/pmc/core.c
+++ b/drivers/platform/x86/intel/pmc/core.c
@@ -1153,6 +1153,8 @@ static int pmc_core_probe(struct platform_device *pdev)
 	pmc_core_do_dmi_quirks(pmcdev);
 
 	pmc_core_dbgfs_register(pmcdev);
+	pm_report_max_hw_sleep(FIELD_MAX(SLP_S0_RES_COUNTER_MASK) *
+			       pmc_core_adjust_slp_s0_step(pmcdev, 1));
 
 	device_initialized = true;
 	dev_info(&pdev->dev, " initialized\n");
@@ -1160,7 +1162,7 @@ static int pmc_core_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int pmc_core_remove(struct platform_device *pdev)
+static void pmc_core_remove(struct platform_device *pdev)
 {
 	struct pmc_dev *pmcdev = platform_get_drvdata(pdev);
 
@@ -1168,7 +1170,6 @@ static int pmc_core_remove(struct platform_device *pdev)
 	platform_set_drvdata(pdev, NULL);
 	mutex_destroy(&pmcdev->lock);
 	iounmap(pmcdev->regbase);
-	return 0;
 }
 
 static bool warn_on_s0ix_failures;
@@ -1179,12 +1180,6 @@ static __maybe_unused int pmc_core_suspend(struct device *dev)
 {
 	struct pmc_dev *pmcdev = dev_get_drvdata(dev);
 
-	pmcdev->check_counters = false;
-
-	/* No warnings on S0ix failures */
-	if (!warn_on_s0ix_failures)
-		return 0;
-
 	/* Check if the syspend will actually use S0ix */
 	if (pm_suspend_via_firmware())
 		return 0;
@@ -1197,7 +1192,6 @@ static __maybe_unused int pmc_core_suspend(struct device *dev)
 	if (pmc_core_dev_state_get(pmcdev, &pmcdev->s0ix_counter))
 		return -EIO;
 
-	pmcdev->check_counters = true;
 	return 0;
 }
 
@@ -1221,6 +1215,8 @@ static inline bool pmc_core_is_s0ix_failed(struct pmc_dev *pmcdev)
 	if (pmc_core_dev_state_get(pmcdev, &s0ix_counter))
 		return false;
 
+	pm_report_hw_sleep_time((u32)(s0ix_counter - pmcdev->s0ix_counter));
+
 	if (s0ix_counter == pmcdev->s0ix_counter)
 		return true;
 
@@ -1233,12 +1229,16 @@ static __maybe_unused int pmc_core_resume(struct device *dev)
 	const struct pmc_bit_map **maps = pmcdev->map->lpm_sts;
 	int offset = pmcdev->map->lpm_status_offset;
 
-	if (!pmcdev->check_counters)
+	/* Check if the syspend used S0ix */
+	if (pm_suspend_via_firmware())
 		return 0;
 
 	if (!pmc_core_is_s0ix_failed(pmcdev))
 		return 0;
 
+	if (!warn_on_s0ix_failures)
+		return 0;
+
 	if (pmc_core_is_pc10_failed(pmcdev)) {
 		/* S0ix failed because of PC10 entry failure */
 		dev_info(dev, "CPU did not enter PC10!!! (PC10 cnt=0x%llx)\n",
@@ -1275,7 +1275,7 @@ static struct platform_driver pmc_core_driver = {
 		.dev_groups = pmc_dev_groups,
 	},
 	.probe = pmc_core_probe,
-	.remove = pmc_core_remove,
+	.remove_new = pmc_core_remove,
 };
 
 module_platform_driver(pmc_core_driver);
diff --git a/drivers/platform/x86/intel/pmc/core.h b/drivers/platform/x86/intel/pmc/core.h
index 810204d758ab..9ca9b9746719 100644
--- a/drivers/platform/x86/intel/pmc/core.h
+++ b/drivers/platform/x86/intel/pmc/core.h
@@ -16,6 +16,8 @@
 #include <linux/bits.h>
 #include <linux/platform_device.h>
 
+#define SLP_S0_RES_COUNTER_MASK			GENMASK(31, 0)
+
 #define PMC_BASE_ADDR_DEFAULT			0xFE000000
 
 /* Sunrise Point Power Management Controller PCI Device ID */
@@ -319,7 +321,6 @@ struct pmc_reg_map {
  * @pmc_xram_read_bit:	flag to indicate whether PMC XRAM shadow registers
  *			used to read MPHY PG and PLL status are available
  * @mutex_lock:		mutex to complete one transcation
- * @check_counters:	On resume, check if counters are getting incremented
  * @pc10_counter:	PC10 residency counter
  * @s0ix_counter:	S0ix residency (step adjusted)
  * @num_lpm_modes:	Count of enabled modes
@@ -338,7 +339,6 @@ struct pmc_dev {
 	int pmc_xram_read_bit;
 	struct mutex lock; /* generic mutex lock for PMC Core */
 
-	bool check_counters; /* Check for counter increments on resume */
 	u64 pc10_counter;
 	u64 s0ix_counter;
 	int num_lpm_modes;
diff --git a/drivers/platform/x86/intel/pmc/mtl.c b/drivers/platform/x86/intel/pmc/mtl.c
index eeb3bd8c2502..e8cc156412ce 100644
--- a/drivers/platform/x86/intel/pmc/mtl.c
+++ b/drivers/platform/x86/intel/pmc/mtl.c
@@ -8,6 +8,7 @@
  *
  */
 
+#include <linux/pci.h>
 #include "core.h"
 
 const struct pmc_reg_map mtl_reg_map = {
@@ -45,8 +46,38 @@ void mtl_core_configure(struct pmc_dev *pmcdev)
 	pmc_core_send_ltr_ignore(pmcdev, 3);
 }
 
+#define MTL_GNA_PCI_DEV	0x7e4c
+#define MTL_IPU_PCI_DEV	0x7d19
+#define MTL_VPU_PCI_DEV	0x7d1d
+static void mtl_set_device_d3(unsigned int device)
+{
+	struct pci_dev *pcidev;
+
+	pcidev = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL);
+	if (pcidev) {
+		if (!device_trylock(&pcidev->dev)) {
+			pci_dev_put(pcidev);
+			return;
+		}
+		if (!pcidev->dev.driver) {
+			dev_info(&pcidev->dev, "Setting to D3hot\n");
+			pci_set_power_state(pcidev, PCI_D3hot);
+		}
+		device_unlock(&pcidev->dev);
+		pci_dev_put(pcidev);
+	}
+}
+
 void mtl_core_init(struct pmc_dev *pmcdev)
 {
 	pmcdev->map = &mtl_reg_map;
 	pmcdev->core_configure = mtl_core_configure;
+
+	/*
+	 * Set power state of select devices that do not have drivers to D3
+	 * so that they do not block Package C entry.
+	 */
+	mtl_set_device_d3(MTL_GNA_PCI_DEV);
+	mtl_set_device_d3(MTL_IPU_PCI_DEV);
+	mtl_set_device_d3(MTL_VPU_PCI_DEV);
 }
diff --git a/drivers/platform/x86/intel/pmt/class.c b/drivers/platform/x86/intel/pmt/class.c
index 46598dcb634a..f32a233470de 100644
--- a/drivers/platform/x86/intel/pmt/class.c
+++ b/drivers/platform/x86/intel/pmt/class.c
@@ -33,7 +33,7 @@ bool intel_pmt_is_early_client_hw(struct device *dev)
 	 */
 	return !!(ivdev->info->quirks & VSEC_QUIRK_EARLY_HW);
 }
-EXPORT_SYMBOL_GPL(intel_pmt_is_early_client_hw);
+EXPORT_SYMBOL_NS_GPL(intel_pmt_is_early_client_hw, INTEL_PMT);
 
 static inline int
 pmt_memcpy64_fromio(void *to, const u64 __iomem *from, size_t count)
@@ -155,7 +155,6 @@ ATTRIBUTE_GROUPS(intel_pmt);
 
 static struct class intel_pmt_class = {
 	.name = "intel_pmt",
-	.owner = THIS_MODULE,
 	.dev_groups = intel_pmt_groups,
 };
 
@@ -327,7 +326,7 @@ int intel_pmt_dev_create(struct intel_pmt_entry *entry, struct intel_pmt_namespa
 	return intel_pmt_dev_register(entry, ns, dev);
 
 }
-EXPORT_SYMBOL_GPL(intel_pmt_dev_create);
+EXPORT_SYMBOL_NS_GPL(intel_pmt_dev_create, INTEL_PMT);
 
 void intel_pmt_dev_destroy(struct intel_pmt_entry *entry,
 			   struct intel_pmt_namespace *ns)
@@ -343,7 +342,7 @@ void intel_pmt_dev_destroy(struct intel_pmt_entry *entry,
 	device_unregister(dev);
 	xa_erase(ns->xa, entry->devid);
 }
-EXPORT_SYMBOL_GPL(intel_pmt_dev_destroy);
+EXPORT_SYMBOL_NS_GPL(intel_pmt_dev_destroy, INTEL_PMT);
 
 static int __init pmt_class_init(void)
 {
diff --git a/drivers/platform/x86/intel/pmt/crashlog.c b/drivers/platform/x86/intel/pmt/crashlog.c
index ace1239bc0a0..bbb3d61d09f4 100644
--- a/drivers/platform/x86/intel/pmt/crashlog.c
+++ b/drivers/platform/x86/intel/pmt/crashlog.c
@@ -328,3 +328,4 @@ module_exit(pmt_crashlog_exit);
 MODULE_AUTHOR("Alexander Duyck <alexander.h.duyck@linux.intel.com>");
 MODULE_DESCRIPTION("Intel PMT Crashlog driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS(INTEL_PMT);
diff --git a/drivers/platform/x86/intel/pmt/telemetry.c b/drivers/platform/x86/intel/pmt/telemetry.c
index 5e4009c05ecf..39cbc87cc28a 100644
--- a/drivers/platform/x86/intel/pmt/telemetry.c
+++ b/drivers/platform/x86/intel/pmt/telemetry.c
@@ -78,7 +78,7 @@ static int pmt_telem_header_decode(struct intel_pmt_entry *entry,
 	 * reserved for future use. They have zero size. Do not fail
 	 * probe for these. Just ignore them.
 	 */
-	if (header->size == 0)
+	if (header->size == 0 || header->access_type == 0xF)
 		return 1;
 
 	return 0;
@@ -160,3 +160,4 @@ module_exit(pmt_telem_exit);
 MODULE_AUTHOR("David E. Box <david.e.box@linux.intel.com>");
 MODULE_DESCRIPTION("Intel PMT Telemetry driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS(INTEL_PMT);
diff --git a/drivers/platform/x86/intel/sdsi.c b/drivers/platform/x86/intel/sdsi.c
index 9e0ea2cdd704..556e7c6dbb05 100644
--- a/drivers/platform/x86/intel/sdsi.c
+++ b/drivers/platform/x86/intel/sdsi.c
@@ -49,7 +49,7 @@
 #define SDSI_MBOX_CMD_SUCCESS		0x40
 #define SDSI_MBOX_CMD_TIMEOUT		0x80
 
-#define MBOX_TIMEOUT_US			2000
+#define MBOX_TIMEOUT_US			500000
 #define MBOX_TIMEOUT_ACQUIRE_US		1000
 #define MBOX_POLLING_PERIOD_US		100
 #define MBOX_ACQUIRE_NUM_RETRIES	5
diff --git a/drivers/platform/x86/intel/speed_select_if/Kconfig b/drivers/platform/x86/intel/speed_select_if/Kconfig
index ce3e3dc076d2..4eb3ad299db0 100644
--- a/drivers/platform/x86/intel/speed_select_if/Kconfig
+++ b/drivers/platform/x86/intel/speed_select_if/Kconfig
@@ -2,8 +2,12 @@ menu "Intel Speed Select Technology interface support"
 	depends on PCI
 	depends on X86_64 || COMPILE_TEST
 
+config INTEL_SPEED_SELECT_TPMI
+	tristate
+
 config INTEL_SPEED_SELECT_INTERFACE
 	tristate "Intel(R) Speed Select Technology interface drivers"
+	select INTEL_SPEED_SELECT_TPMI if INTEL_TPMI
 	help
 	  This config enables the Intel(R) Speed Select Technology interface
 	  drivers. The Intel(R) speed select technology features are non
diff --git a/drivers/platform/x86/intel/speed_select_if/Makefile b/drivers/platform/x86/intel/speed_select_if/Makefile
index 856076206f35..1d878a36d0ab 100644
--- a/drivers/platform/x86/intel/speed_select_if/Makefile
+++ b/drivers/platform/x86/intel/speed_select_if/Makefile
@@ -8,3 +8,5 @@ obj-$(CONFIG_INTEL_SPEED_SELECT_INTERFACE) += isst_if_common.o
 obj-$(CONFIG_INTEL_SPEED_SELECT_INTERFACE) += isst_if_mmio.o
 obj-$(CONFIG_INTEL_SPEED_SELECT_INTERFACE) += isst_if_mbox_pci.o
 obj-$(CONFIG_INTEL_SPEED_SELECT_INTERFACE) += isst_if_mbox_msr.o
+obj-$(CONFIG_INTEL_SPEED_SELECT_TPMI) += isst_tpmi_core.o
+obj-$(CONFIG_INTEL_SPEED_SELECT_TPMI) += isst_tpmi.o
diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c
index 0954a04623ed..e0572a29212e 100644
--- a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c
+++ b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c
@@ -19,9 +19,13 @@
 #include <linux/uaccess.h>
 #include <uapi/linux/isst_if.h>
 
+#include <asm/cpu_device_id.h>
+#include <asm/intel-family.h>
+
 #include "isst_if_common.h"
 
 #define MSR_THREAD_ID_INFO	0x53
+#define MSR_PM_LOGICAL_ID	0x54
 #define MSR_CPU_BUS_NUMBER	0x128
 
 static struct isst_if_cmd_cb punit_callbacks[ISST_IF_DEV_MAX];
@@ -31,6 +35,7 @@ static int punit_msr_white_list[] = {
 	MSR_CONFIG_TDP_CONTROL,
 	MSR_TURBO_RATIO_LIMIT1,
 	MSR_TURBO_RATIO_LIMIT2,
+	MSR_PM_LOGICAL_ID,
 };
 
 struct isst_valid_cmd_ranges {
@@ -73,6 +78,8 @@ struct isst_cmd {
 	u32 param;
 };
 
+static bool isst_hpm_support;
+
 static DECLARE_HASHTABLE(isst_hash, 8);
 static DEFINE_MUTEX(isst_hash_lock);
 
@@ -262,11 +269,13 @@ bool isst_if_mbox_cmd_set_req(struct isst_if_mbox_cmd *cmd)
 }
 EXPORT_SYMBOL_GPL(isst_if_mbox_cmd_set_req);
 
+static int isst_if_api_version;
+
 static int isst_if_get_platform_info(void __user *argp)
 {
 	struct isst_if_platform_info info;
 
-	info.api_version = ISST_IF_API_VERSION;
+	info.api_version = isst_if_api_version;
 	info.driver_version = ISST_IF_DRIVER_VERSION;
 	info.max_cmds_per_ioctl = ISST_IF_CMD_LIMIT;
 	info.mbox_supported = punit_callbacks[ISST_IF_DEV_MBOX].registered;
@@ -409,11 +418,20 @@ static int isst_if_cpu_online(unsigned int cpu)
 		isst_cpu_info[cpu].pci_dev[1] = _isst_if_get_pci_dev(cpu, 1, 30, 1);
 	}
 
+	if (isst_hpm_support) {
+
+		ret = rdmsrl_safe(MSR_PM_LOGICAL_ID, &data);
+		if (!ret)
+			goto set_punit_id;
+	}
+
 	ret = rdmsrl_safe(MSR_THREAD_ID_INFO, &data);
 	if (ret) {
 		isst_cpu_info[cpu].punit_cpu_id = -1;
 		return ret;
 	}
+
+set_punit_id:
 	isst_cpu_info[cpu].punit_cpu_id = data;
 
 	isst_restore_msr_local(cpu);
@@ -588,6 +606,7 @@ static long isst_if_def_ioctl(struct file *file, unsigned int cmd,
 	struct isst_if_cmd_cb cmd_cb;
 	struct isst_if_cmd_cb *cb;
 	long ret = -ENOTTY;
+	int i;
 
 	switch (cmd) {
 	case ISST_IF_GET_PLATFORM_INFO:
@@ -616,6 +635,16 @@ static long isst_if_def_ioctl(struct file *file, unsigned int cmd,
 		ret = isst_if_exec_multi_cmd(argp, &cmd_cb);
 		break;
 	default:
+		for (i = 0; i < ISST_IF_DEV_MAX; ++i) {
+			struct isst_if_cmd_cb *cb = &punit_callbacks[i];
+			int ret;
+
+			if (cb->def_ioctl) {
+				ret = cb->def_ioctl(file, cmd, arg);
+				if (!ret)
+					return ret;
+			}
+		}
 		break;
 	}
 
@@ -691,6 +720,12 @@ static struct miscdevice isst_if_char_driver = {
 	.fops		= &isst_if_char_driver_ops,
 };
 
+static const struct x86_cpu_id hpm_cpu_ids[] = {
+	X86_MATCH_INTEL_FAM6_MODEL(GRANITERAPIDS_X,	NULL),
+	X86_MATCH_INTEL_FAM6_MODEL(SIERRAFOREST_X,	NULL),
+	{}
+};
+
 static int isst_misc_reg(void)
 {
 	mutex_lock(&punit_misc_dev_reg_lock);
@@ -698,6 +733,12 @@ static int isst_misc_reg(void)
 		goto unlock_exit;
 
 	if (!misc_usage_count) {
+		const struct x86_cpu_id *id;
+
+		id = x86_match_cpu(hpm_cpu_ids);
+		if (id)
+			isst_hpm_support = true;
+
 		misc_device_ret = isst_if_cpu_info_init();
 		if (misc_device_ret)
 			goto unlock_exit;
@@ -756,6 +797,10 @@ int isst_if_cdev_register(int device_type, struct isst_if_cmd_cb *cb)
 		mutex_unlock(&punit_misc_dev_open_lock);
 		return -EAGAIN;
 	}
+	if (!cb->api_version)
+		cb->api_version = ISST_IF_API_VERSION;
+	if (cb->api_version > isst_if_api_version)
+		isst_if_api_version = cb->api_version;
 	memcpy(&punit_callbacks[device_type], cb, sizeof(*cb));
 	punit_callbacks[device_type].registered = 1;
 	mutex_unlock(&punit_misc_dev_open_lock);
diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_common.h b/drivers/platform/x86/intel/speed_select_if/isst_if_common.h
index 35ff506b402e..1004f2c9cca8 100644
--- a/drivers/platform/x86/intel/speed_select_if/isst_if_common.h
+++ b/drivers/platform/x86/intel/speed_select_if/isst_if_common.h
@@ -30,7 +30,8 @@
 
 #define ISST_IF_DEV_MBOX	0
 #define ISST_IF_DEV_MMIO	1
-#define ISST_IF_DEV_MAX		2
+#define ISST_IF_DEV_TPMI	2
+#define ISST_IF_DEV_MAX		3
 
 /**
  * struct isst_if_cmd_cb - Used to register a IOCTL handler
@@ -40,6 +41,7 @@
  * @offset:	Offset to the first valid member in command structure.
  *		This will be the offset of the start of the command
  *		after command count field
+ * @api_version: API version supported for this target. 0, if none.
  * @owner:	Registered module owner
  * @cmd_callback: Callback function to handle IOCTL. The callback has the
  *		command pointer with data for command. There is a pointer
@@ -47,6 +49,8 @@
  *		response to user ioctl buffer. The "resume" argument
  *		can be used to avoid storing the command for replay
  *		during system resume
+ * @def_ioctl:	Default IOCTL handler callback, if there is no match in
+ *		the existing list of IOCTL handled by the common handler.
  *
  * This structure is used to register an handler for IOCTL. To avoid
  * code duplication common code handles all the IOCTL command read/write
@@ -57,8 +61,10 @@ struct isst_if_cmd_cb {
 	int registered;
 	int cmd_size;
 	int offset;
+	int api_version;
 	struct module *owner;
 	long (*cmd_callback)(u8 *ptr, int *write_only, int resume);
+	long (*def_ioctl)(struct file *file, unsigned int cmd, unsigned long arg);
 };
 
 /* Internal interface functions */
diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi.c
new file mode 100644
index 000000000000..17972191538a
--- /dev/null
+++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi.c
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * isst_tpmi.c: SST TPMI interface
+ *
+ * Copyright (c) 2023, Intel Corporation.
+ * All Rights Reserved.
+ *
+ */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/module.h>
+#include <linux/intel_tpmi.h>
+
+#include "isst_tpmi_core.h"
+
+static int intel_sst_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id)
+{
+	int ret;
+
+	ret = tpmi_sst_init();
+	if (ret)
+		return ret;
+
+	ret = tpmi_sst_dev_add(auxdev);
+	if (ret)
+		tpmi_sst_exit();
+
+	return ret;
+}
+
+static void intel_sst_remove(struct auxiliary_device *auxdev)
+{
+	tpmi_sst_dev_remove(auxdev);
+	tpmi_sst_exit();
+}
+
+static int intel_sst_suspend(struct device *dev)
+{
+	tpmi_sst_dev_suspend(to_auxiliary_dev(dev));
+
+	return 0;
+}
+
+static int intel_sst_resume(struct device *dev)
+{
+	tpmi_sst_dev_resume(to_auxiliary_dev(dev));
+
+	return 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(intel_sst_pm, intel_sst_suspend, intel_sst_resume);
+
+static const struct auxiliary_device_id intel_sst_id_table[] = {
+	{ .name = "intel_vsec.tpmi-sst" },
+	{}
+};
+MODULE_DEVICE_TABLE(auxiliary, intel_sst_id_table);
+
+static struct auxiliary_driver intel_sst_aux_driver = {
+	.id_table       = intel_sst_id_table,
+	.remove         = intel_sst_remove,
+	.probe          = intel_sst_probe,
+	.driver = {
+		.pm = pm_sleep_ptr(&intel_sst_pm),
+	},
+};
+
+module_auxiliary_driver(intel_sst_aux_driver);
+
+MODULE_IMPORT_NS(INTEL_TPMI_SST);
+MODULE_DESCRIPTION("Intel TPMI SST Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c
new file mode 100644
index 000000000000..664d2ee60385
--- /dev/null
+++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c
@@ -0,0 +1,1440 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * isst_tpmi.c: SST TPMI interface core
+ *
+ * Copyright (c) 2023, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * This information will be useful to understand flows:
+ * In the current generation of platforms, TPMI is supported via OOB
+ * PCI device. This PCI device has one instance per CPU package.
+ * There is a unique TPMI ID for SST. Each TPMI ID also has multiple
+ * entries, representing per power domain information.
+ *
+ * There is one dev file for complete SST information and control same as the
+ * prior generation of hardware. User spaces don't need to know how the
+ * information is presented by the hardware. The TPMI core module implements
+ * the hardware mapping.
+ */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/delay.h>
+#include <linux/intel_tpmi.h>
+#include <linux/fs.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <uapi/linux/isst_if.h>
+
+#include "isst_tpmi_core.h"
+#include "isst_if_common.h"
+
+/* Supported SST hardware version by this driver */
+#define ISST_HEADER_VERSION		1
+
+/*
+ * Used to indicate if value read from MMIO needs to get multiplied
+ * to get to a standard unit or not.
+ */
+#define SST_MUL_FACTOR_NONE    1
+
+/* Define 100 as a scaling factor frequency ratio to frequency conversion */
+#define SST_MUL_FACTOR_FREQ    100
+
+/* All SST regs are 64 bit size */
+#define SST_REG_SIZE   8
+
+/**
+ * struct sst_header -	SST main header
+ * @interface_version:	Version number for this interface
+ * @cap_mask:		Bitmask of the supported sub features. 1=the sub feature is enabled.
+ *			0=disabled.
+ *			Bit[8]= SST_CP enable (1), disable (0)
+ *			bit[9]= SST_PP enable (1), disable (0)
+ *			other bits are reserved for future use
+ * @cp_offset:		Qword (8 bytes) offset to the SST_CP register bank
+ * @pp_offset:		Qword (8 bytes) offset to the SST_PP register bank
+ * @reserved:		Reserved for future use
+ *
+ * This register allows SW to discover SST capability and the offsets to SST-CP
+ * and SST-PP register banks.
+ */
+struct sst_header {
+	u8 interface_version;
+	u8 cap_mask;
+	u8 cp_offset;
+	u8 pp_offset;
+	u32 reserved;
+} __packed;
+
+/**
+ * struct cp_header -	SST-CP (core-power) header
+ * @feature_id:		0=SST-CP, 1=SST-PP, 2=SST-BF, 3=SST-TF
+ * @feature_rev:	Interface Version number for this SST feature
+ * @ratio_unit:		Frequency ratio unit. 00: 100MHz. All others are reserved
+ * @reserved:		Reserved for future use
+ *
+ * This structure is used store SST-CP header. This is packed to the same
+ * format as defined in the specifications.
+ */
+struct cp_header {
+	u64 feature_id :4;
+	u64 feature_rev :8;
+	u64 ratio_unit :2;
+	u64 reserved :50;
+} __packed;
+
+/**
+ * struct pp_header -	SST-PP (Perf profile) header
+ * @feature_id:		0=SST-CP, 1=SST-PP, 2=SST-BF, 3=SST-TF
+ * @feature_rev:	Interface Version number for this SST feature
+ * @level_en_mask:	SST-PP level enable/disable fuse mask
+ * @allowed_level_mask:	Allowed level mask used for dynamic config level switching
+ * @reserved0:		Reserved for future use
+ * @ratio_unit:		Frequency ratio unit. 00: 100MHz. All others are reserved
+ * @block_size:		Size of PP block in Qword unit (8 bytes)
+ * @dynamic_switch:	If set (1), dynamic switching of SST PP is supported
+ * @memory_ratio_unit:	Memory Controller frequency ratio unit. 00: 100MHz, others reserved
+ * @reserved1:		Reserved for future use
+ *
+ * This structure is used store SST-PP header. This is packed to the same
+ * format as defined in the specifications.
+ */
+struct pp_header {
+	u64 feature_id :4;
+	u64 feature_rev :8;
+	u64 level_en_mask :8;
+	u64 allowed_level_mask :8;
+	u64 reserved0 :4;
+	u64 ratio_unit :2;
+	u64 block_size :8;
+	u64 dynamic_switch :1;
+	u64 memory_ratio_unit :2;
+	u64 reserved1 :19;
+} __packed;
+
+/**
+ * struct feature_offset -	Offsets to SST-PP features
+ * @pp_offset:		Qword offset within PP level for the SST_PP register bank
+ * @bf_offset:		Qword offset within PP level for the SST_BF register bank
+ * @tf_offset:		Qword offset within PP level for the SST_TF register bank
+ * @reserved:		Reserved for future use
+ *
+ * This structure is used store offsets for SST features in the register bank.
+ * This is packed to the same format as defined in the specifications.
+ */
+struct feature_offset {
+	u64 pp_offset :8;
+	u64 bf_offset :8;
+	u64 tf_offset :8;
+	u64 reserved :40;
+} __packed;
+
+/**
+ * struct levels_offset -	Offsets to each SST PP level
+ * @sst_pp_level0_offset:	Qword offset to the register block of PP level 0
+ * @sst_pp_level1_offset:	Qword offset to the register block of PP level 1
+ * @sst_pp_level2_offset:	Qword offset to the register block of PP level 2
+ * @sst_pp_level3_offset:	Qword offset to the register block of PP level 3
+ * @sst_pp_level4_offset:	Qword offset to the register block of PP level 4
+ * @reserved:			Reserved for future use
+ *
+ * This structure is used store offsets of SST PP levels in the register bank.
+ * This is packed to the same format as defined in the specifications.
+ */
+struct levels_offset {
+	u64 sst_pp_level0_offset :8;
+	u64 sst_pp_level1_offset :8;
+	u64 sst_pp_level2_offset :8;
+	u64 sst_pp_level3_offset :8;
+	u64 sst_pp_level4_offset :8;
+	u64 reserved :24;
+} __packed;
+
+/**
+ * struct pp_control_offset -	Offsets for SST PP controls
+ * @perf_level:		A SST-PP level that SW intends to switch to
+ * @perf_level_lock:	SST-PP level select lock. 0 - unlocked. 1 - locked till next reset
+ * @resvd0:		Reserved for future use
+ * @current_state:	Bit mask to control the enable(1)/disable(0) state of each feature
+ *			of the current PP level, bit 0 = BF, bit 1 = TF, bit 2-7 = reserved
+ * @reserved:		Reserved for future use
+ *
+ * This structure is used store offsets of SST PP controls in the register bank.
+ * This is packed to the same format as defined in the specifications.
+ */
+struct pp_control_offset {
+	u64 perf_level :3;
+	u64 perf_level_lock :1;
+	u64 resvd0 :4;
+	u64 current_state :8;
+	u64 reserved :48;
+} __packed;
+
+/**
+ * struct pp_status_offset -	Offsets for SST PP status fields
+ * @sst_pp_level:	Returns the current SST-PP level
+ * @sst_pp_lock:	Returns the lock bit setting of perf_level_lock in pp_control_offset
+ * @error_type:		Returns last error of SST-PP level change request. 0: no error,
+ *			1: level change not allowed, others: reserved
+ * @feature_state:	Bit mask to indicate the enable(1)/disable(0) state of each feature of the
+ *			current PP level. bit 0 = BF, bit 1 = TF, bit 2-7 reserved
+ * @reserved0:		Reserved for future use
+ * @feature_error_type: Returns last error of the specific feature. Three error_type bits per
+ *			feature. i.e. ERROR_TYPE[2:0] for BF, ERROR_TYPE[5:3] for TF, etc.
+ *			0x0: no error, 0x1: The specific feature is not supported by the hardware.
+ *			0x2-0x6: Reserved. 0x7: feature state change is not allowed.
+ * @reserved1:		Reserved for future use
+ *
+ * This structure is used store offsets of SST PP status in the register bank.
+ * This is packed to the same format as defined in the specifications.
+ */
+struct pp_status_offset {
+	u64 sst_pp_level :3;
+	u64 sst_pp_lock :1;
+	u64 error_type :4;
+	u64 feature_state :8;
+	u64 reserved0 :16;
+	u64 feature_error_type : 24;
+	u64 reserved1 :8;
+} __packed;
+
+/**
+ * struct perf_level -	Used to store perf level and mmio offset
+ * @mmio_offset:	mmio offset for a perf level
+ * @level:		perf level for this offset
+ *
+ * This structure is used store final mmio offset of each perf level from the
+ * SST base mmio offset.
+ */
+struct perf_level {
+	int mmio_offset;
+	int level;
+};
+
+/**
+ * struct tpmi_per_power_domain_info -	Store per power_domain SST info
+ * @package_id:		Package id for this power_domain
+ * @power_domain_id:	Power domain id, Each entry from the SST-TPMI instance is a power_domain.
+ * @max_level:		Max possible PP level possible for this power_domain
+ * @ratio_unit:		Ratio unit for converting to MHz
+ * @avx_levels:		Number of AVX levels
+ * @pp_block_size:	Block size from PP header
+ * @sst_header:		Store SST header for this power_domain
+ * @cp_header:		Store SST-CP header for this power_domain
+ * @pp_header:		Store SST-PP header for this power_domain
+ * @perf_levels:	Pointer to each perf level to map level to mmio offset
+ * @feature_offsets:	Store feature offsets for each PP-level
+ * @control_offset:	Store the control offset for each PP-level
+ * @status_offset:	Store the status offset for each PP-level
+ * @sst_base:		Mapped SST base IO memory
+ * @auxdev:		Auxiliary device instance enumerated this instance
+ * @saved_sst_cp_control: Save SST-CP control configuration to store restore for suspend/resume
+ * @saved_clos_configs:	Save SST-CP CLOS configuration to store restore for suspend/resume
+ * @saved_clos_assocs:	Save SST-CP CLOS association to store restore for suspend/resume
+ * @saved_pp_control:	Save SST-PP control information to store restore for suspend/resume
+ *
+ * This structure is used store complete SST information for a power_domain. This information
+ * is used to read/write request for any SST IOCTL. Each physical CPU package can have multiple
+ * power_domains. Each power domain describes its own SST information and has its own controls.
+ */
+struct tpmi_per_power_domain_info {
+	int package_id;
+	int power_domain_id;
+	int max_level;
+	int ratio_unit;
+	int avx_levels;
+	int pp_block_size;
+	struct sst_header sst_header;
+	struct cp_header cp_header;
+	struct pp_header pp_header;
+	struct perf_level *perf_levels;
+	struct feature_offset feature_offsets;
+	struct pp_control_offset control_offset;
+	struct pp_status_offset status_offset;
+	void __iomem *sst_base;
+	struct auxiliary_device *auxdev;
+	u64 saved_sst_cp_control;
+	u64 saved_clos_configs[4];
+	u64 saved_clos_assocs[4];
+	u64 saved_pp_control;
+};
+
+/**
+ * struct tpmi_sst_struct -	Store sst info for a package
+ * @package_id:			Package id for this aux device instance
+ * @number_of_power_domains:	Number of power_domains pointed by power_domain_info pointer
+ * @power_domain_info:		Pointer to power domains information
+ *
+ * This structure is used store full SST information for a package.
+ * Each package has a unique OOB PCI device, which enumerates TPMI.
+ * Each Package will have multiple power_domains.
+ */
+struct tpmi_sst_struct {
+	int package_id;
+	int number_of_power_domains;
+	struct tpmi_per_power_domain_info *power_domain_info;
+};
+
+/**
+ * struct tpmi_sst_common_struct -	Store all SST instances
+ * @max_index:		Maximum instances currently present
+ * @sst_inst:		Pointer to per package instance
+ *
+ * Stores every SST Package instance.
+ */
+struct tpmi_sst_common_struct {
+	int max_index;
+	struct tpmi_sst_struct **sst_inst;
+};
+
+/*
+ * Each IOCTL request is processed under this lock. Also used to protect
+ * registration functions and common data structures.
+ */
+static DEFINE_MUTEX(isst_tpmi_dev_lock);
+
+/* Usage count to track, number of TPMI SST instances registered to this core. */
+static int isst_core_usage_count;
+
+/* Stores complete SST information for every package and power_domain */
+static struct tpmi_sst_common_struct isst_common;
+
+#define SST_MAX_AVX_LEVELS	3
+
+#define SST_PP_OFFSET_0		8
+#define SST_PP_OFFSET_1		16
+#define SST_PP_OFFSET_SIZE	8
+
+static int sst_add_perf_profiles(struct auxiliary_device *auxdev,
+				 struct tpmi_per_power_domain_info *pd_info,
+				 int levels)
+{
+	u64 perf_level_offsets;
+	int i;
+
+	pd_info->perf_levels = devm_kcalloc(&auxdev->dev, levels,
+					    sizeof(struct perf_level),
+					    GFP_KERNEL);
+	if (!pd_info->perf_levels)
+		return 0;
+
+	pd_info->ratio_unit = pd_info->pp_header.ratio_unit;
+	pd_info->avx_levels = SST_MAX_AVX_LEVELS;
+	pd_info->pp_block_size = pd_info->pp_header.block_size;
+
+	/* Read PP Offset 0: Get feature offset with PP level */
+	*((u64 *)&pd_info->feature_offsets) = readq(pd_info->sst_base +
+						    pd_info->sst_header.pp_offset +
+						    SST_PP_OFFSET_0);
+
+	perf_level_offsets = readq(pd_info->sst_base + pd_info->sst_header.pp_offset +
+				   SST_PP_OFFSET_1);
+
+	for (i = 0; i < levels; ++i) {
+		u64 offset;
+
+		offset = perf_level_offsets & (0xffULL << (i * SST_PP_OFFSET_SIZE));
+		offset >>= (i * 8);
+		offset &= 0xff;
+		offset *= 8; /* Convert to byte from QWORD offset */
+		pd_info->perf_levels[i].mmio_offset = pd_info->sst_header.pp_offset + offset;
+	}
+
+	return 0;
+}
+
+static int sst_main(struct auxiliary_device *auxdev, struct tpmi_per_power_domain_info *pd_info)
+{
+	int i, mask, levels;
+
+	*((u64 *)&pd_info->sst_header) = readq(pd_info->sst_base);
+	pd_info->sst_header.cp_offset *= 8;
+	pd_info->sst_header.pp_offset *= 8;
+
+	if (pd_info->sst_header.interface_version != ISST_HEADER_VERSION) {
+		dev_err(&auxdev->dev, "SST: Unsupported version:%x\n",
+			pd_info->sst_header.interface_version);
+		return -ENODEV;
+	}
+
+	/* Read SST CP Header */
+	*((u64 *)&pd_info->cp_header) = readq(pd_info->sst_base + pd_info->sst_header.cp_offset);
+
+	/* Read PP header */
+	*((u64 *)&pd_info->pp_header) = readq(pd_info->sst_base + pd_info->sst_header.pp_offset);
+
+	/* Force level_en_mask level 0 */
+	pd_info->pp_header.level_en_mask |= 0x01;
+
+	mask = 0x01;
+	levels = 0;
+	for (i = 0; i < 8; ++i) {
+		if (pd_info->pp_header.level_en_mask & mask)
+			levels = i;
+		mask <<= 1;
+	}
+	pd_info->max_level = levels;
+	sst_add_perf_profiles(auxdev, pd_info, levels + 1);
+
+	return 0;
+}
+
+/*
+ * Map a package and power_domain id to SST information structure unique for a power_domain.
+ * The caller should call under isst_tpmi_dev_lock.
+ */
+static struct tpmi_per_power_domain_info *get_instance(int pkg_id, int power_domain_id)
+{
+	struct tpmi_per_power_domain_info *power_domain_info;
+	struct tpmi_sst_struct *sst_inst;
+
+	if (pkg_id < 0 || pkg_id > isst_common.max_index ||
+	    pkg_id >= topology_max_packages())
+		return NULL;
+
+	sst_inst = isst_common.sst_inst[pkg_id];
+	if (!sst_inst)
+		return NULL;
+
+	if (power_domain_id < 0 || power_domain_id >= sst_inst->number_of_power_domains)
+		return NULL;
+
+	power_domain_info = &sst_inst->power_domain_info[power_domain_id];
+
+	if (power_domain_info && !power_domain_info->sst_base)
+		return NULL;
+
+	return power_domain_info;
+}
+
+static bool disable_dynamic_sst_features(void)
+{
+	u64 value;
+
+	rdmsrl(MSR_PM_ENABLE, value);
+	return !(value & 0x1);
+}
+
+#define _read_cp_info(name_str, name, offset, start, width, mult_factor)\
+{\
+	u64 val, mask;\
+	\
+	val = readq(power_domain_info->sst_base + power_domain_info->sst_header.cp_offset +\
+			(offset));\
+	mask = GENMASK_ULL((start + width - 1), start);\
+	val &= mask; \
+	val >>= start;\
+	name = (val * mult_factor);\
+}
+
+#define _write_cp_info(name_str, name, offset, start, width, div_factor)\
+{\
+	u64 val, mask;\
+	\
+	val = readq(power_domain_info->sst_base +\
+		    power_domain_info->sst_header.cp_offset + (offset));\
+	mask = GENMASK_ULL((start + width - 1), start);\
+	val &= ~mask;\
+	val |= (name / div_factor) << start;\
+	writeq(val, power_domain_info->sst_base + power_domain_info->sst_header.cp_offset +\
+		(offset));\
+}
+
+#define	SST_CP_CONTROL_OFFSET	8
+#define	SST_CP_STATUS_OFFSET	16
+
+#define SST_CP_ENABLE_START		0
+#define SST_CP_ENABLE_WIDTH		1
+
+#define SST_CP_PRIORITY_TYPE_START	1
+#define SST_CP_PRIORITY_TYPE_WIDTH	1
+
+static long isst_if_core_power_state(void __user *argp)
+{
+	struct tpmi_per_power_domain_info *power_domain_info;
+	struct isst_core_power core_power;
+
+	if (disable_dynamic_sst_features())
+		return -EFAULT;
+
+	if (copy_from_user(&core_power, argp, sizeof(core_power)))
+		return -EFAULT;
+
+	power_domain_info = get_instance(core_power.socket_id, core_power.power_domain_id);
+	if (!power_domain_info)
+		return -EINVAL;
+
+	if (core_power.get_set) {
+		_write_cp_info("cp_enable", core_power.enable, SST_CP_CONTROL_OFFSET,
+			       SST_CP_ENABLE_START, SST_CP_ENABLE_WIDTH, SST_MUL_FACTOR_NONE)
+		_write_cp_info("cp_prio_type", core_power.priority_type, SST_CP_CONTROL_OFFSET,
+			       SST_CP_PRIORITY_TYPE_START, SST_CP_PRIORITY_TYPE_WIDTH,
+			       SST_MUL_FACTOR_NONE)
+	} else {
+		/* get */
+		_read_cp_info("cp_enable", core_power.enable, SST_CP_STATUS_OFFSET,
+			      SST_CP_ENABLE_START, SST_CP_ENABLE_WIDTH, SST_MUL_FACTOR_NONE)
+		_read_cp_info("cp_prio_type", core_power.priority_type, SST_CP_STATUS_OFFSET,
+			      SST_CP_PRIORITY_TYPE_START, SST_CP_PRIORITY_TYPE_WIDTH,
+			      SST_MUL_FACTOR_NONE)
+		core_power.supported = !!(power_domain_info->sst_header.cap_mask & BIT(0));
+		if (copy_to_user(argp, &core_power, sizeof(core_power)))
+			return -EFAULT;
+	}
+
+	return 0;
+}
+
+#define SST_CLOS_CONFIG_0_OFFSET	24
+
+#define SST_CLOS_CONFIG_PRIO_START	4
+#define SST_CLOS_CONFIG_PRIO_WIDTH	4
+
+#define SST_CLOS_CONFIG_MIN_START	8
+#define SST_CLOS_CONFIG_MIN_WIDTH	8
+
+#define SST_CLOS_CONFIG_MAX_START	16
+#define SST_CLOS_CONFIG_MAX_WIDTH	8
+
+static long isst_if_clos_param(void __user *argp)
+{
+	struct tpmi_per_power_domain_info *power_domain_info;
+	struct isst_clos_param clos_param;
+
+	if (copy_from_user(&clos_param, argp, sizeof(clos_param)))
+		return -EFAULT;
+
+	power_domain_info = get_instance(clos_param.socket_id, clos_param.power_domain_id);
+	if (!power_domain_info)
+		return -EINVAL;
+
+	if (clos_param.get_set) {
+		_write_cp_info("clos.min_freq", clos_param.min_freq_mhz,
+			       (SST_CLOS_CONFIG_0_OFFSET + clos_param.clos * SST_REG_SIZE),
+			       SST_CLOS_CONFIG_MIN_START, SST_CLOS_CONFIG_MIN_WIDTH,
+			       SST_MUL_FACTOR_FREQ);
+		_write_cp_info("clos.max_freq", clos_param.max_freq_mhz,
+			       (SST_CLOS_CONFIG_0_OFFSET + clos_param.clos * SST_REG_SIZE),
+			       SST_CLOS_CONFIG_MAX_START, SST_CLOS_CONFIG_MAX_WIDTH,
+			       SST_MUL_FACTOR_FREQ);
+		_write_cp_info("clos.prio", clos_param.prop_prio,
+			       (SST_CLOS_CONFIG_0_OFFSET + clos_param.clos * SST_REG_SIZE),
+			       SST_CLOS_CONFIG_PRIO_START, SST_CLOS_CONFIG_PRIO_WIDTH,
+			       SST_MUL_FACTOR_NONE);
+	} else {
+		/* get */
+		_read_cp_info("clos.min_freq", clos_param.min_freq_mhz,
+				(SST_CLOS_CONFIG_0_OFFSET + clos_param.clos * SST_REG_SIZE),
+				SST_CLOS_CONFIG_MIN_START, SST_CLOS_CONFIG_MIN_WIDTH,
+				SST_MUL_FACTOR_FREQ)
+		_read_cp_info("clos.max_freq", clos_param.max_freq_mhz,
+				(SST_CLOS_CONFIG_0_OFFSET + clos_param.clos * SST_REG_SIZE),
+				SST_CLOS_CONFIG_MAX_START, SST_CLOS_CONFIG_MAX_WIDTH,
+				SST_MUL_FACTOR_FREQ)
+		_read_cp_info("clos.prio", clos_param.prop_prio,
+				(SST_CLOS_CONFIG_0_OFFSET + clos_param.clos * SST_REG_SIZE),
+				SST_CLOS_CONFIG_PRIO_START, SST_CLOS_CONFIG_PRIO_WIDTH,
+				SST_MUL_FACTOR_NONE)
+
+		if (copy_to_user(argp, &clos_param, sizeof(clos_param)))
+			return -EFAULT;
+	}
+
+	return 0;
+}
+
+#define SST_CLOS_ASSOC_0_OFFSET		56
+#define SST_CLOS_ASSOC_CPUS_PER_REG	16
+#define SST_CLOS_ASSOC_BITS_PER_CPU	4
+
+static long isst_if_clos_assoc(void __user *argp)
+{
+	struct isst_if_clos_assoc_cmds assoc_cmds;
+	unsigned char __user *ptr;
+	int i;
+
+	/* Each multi command has u16 command count as the first field */
+	if (copy_from_user(&assoc_cmds, argp, sizeof(assoc_cmds)))
+		return -EFAULT;
+
+	if (!assoc_cmds.cmd_count || assoc_cmds.cmd_count > ISST_IF_CMD_LIMIT)
+		return -EINVAL;
+
+	ptr = argp + offsetof(struct isst_if_clos_assoc_cmds, assoc_info);
+	for (i = 0; i < assoc_cmds.cmd_count; ++i) {
+		struct tpmi_per_power_domain_info *power_domain_info;
+		struct isst_if_clos_assoc clos_assoc;
+		int punit_id, punit_cpu_no, pkg_id;
+		struct tpmi_sst_struct *sst_inst;
+		int offset, shift, cpu;
+		u64 val, mask, clos;
+
+		if (copy_from_user(&clos_assoc, ptr, sizeof(clos_assoc)))
+			return -EFAULT;
+
+		if (clos_assoc.socket_id > topology_max_packages())
+			return -EINVAL;
+
+		cpu = clos_assoc.logical_cpu;
+		clos = clos_assoc.clos;
+
+		if (assoc_cmds.punit_cpu_map)
+			punit_cpu_no = cpu;
+		else
+			return -EOPNOTSUPP;
+
+		if (punit_cpu_no < 0)
+			return -EINVAL;
+
+		punit_id = clos_assoc.power_domain_id;
+		pkg_id = clos_assoc.socket_id;
+
+		sst_inst = isst_common.sst_inst[pkg_id];
+
+		if (clos_assoc.power_domain_id > sst_inst->number_of_power_domains)
+			return -EINVAL;
+
+		power_domain_info = &sst_inst->power_domain_info[punit_id];
+
+		offset = SST_CLOS_ASSOC_0_OFFSET +
+				(punit_cpu_no / SST_CLOS_ASSOC_CPUS_PER_REG) * SST_REG_SIZE;
+		shift = punit_cpu_no % SST_CLOS_ASSOC_CPUS_PER_REG;
+		shift *= SST_CLOS_ASSOC_BITS_PER_CPU;
+
+		val = readq(power_domain_info->sst_base +
+				power_domain_info->sst_header.cp_offset + offset);
+		if (assoc_cmds.get_set) {
+			mask = GENMASK_ULL((shift + SST_CLOS_ASSOC_BITS_PER_CPU - 1), shift);
+			val &= ~mask;
+			val |= (clos << shift);
+			writeq(val, power_domain_info->sst_base +
+					power_domain_info->sst_header.cp_offset + offset);
+		} else {
+			val >>= shift;
+			clos_assoc.clos = val & GENMASK(SST_CLOS_ASSOC_BITS_PER_CPU - 1, 0);
+			if (copy_to_user(ptr, &clos_assoc, sizeof(clos_assoc)))
+				return -EFAULT;
+		}
+
+		ptr += sizeof(clos_assoc);
+	}
+
+	return 0;
+}
+
+#define _read_pp_info(name_str, name, offset, start, width, mult_factor)\
+{\
+	u64 val, _mask;\
+	\
+	val = readq(power_domain_info->sst_base + power_domain_info->sst_header.pp_offset +\
+		    (offset));\
+	_mask = GENMASK_ULL((start + width - 1), start);\
+	val &= _mask;\
+	val >>= start;\
+	name = (val * mult_factor);\
+}
+
+#define _write_pp_info(name_str, name, offset, start, width, div_factor)\
+{\
+	u64 val, _mask;\
+	\
+	val = readq(power_domain_info->sst_base + power_domain_info->sst_header.pp_offset +\
+		    (offset));\
+	_mask = GENMASK((start + width - 1), start);\
+	val &= ~_mask;\
+	val |= (name / div_factor) << start;\
+	writeq(val, power_domain_info->sst_base + power_domain_info->sst_header.pp_offset +\
+	      (offset));\
+}
+
+#define _read_bf_level_info(name_str, name, level, offset, start, width, mult_factor)\
+{\
+	u64 val, _mask;\
+	\
+	val = readq(power_domain_info->sst_base +\
+		    power_domain_info->perf_levels[level].mmio_offset +\
+		(power_domain_info->feature_offsets.bf_offset * 8) + (offset));\
+	_mask = GENMASK_ULL((start + width - 1), start);\
+	val &= _mask; \
+	val >>= start;\
+	name = (val * mult_factor);\
+}
+
+#define _read_tf_level_info(name_str, name, level, offset, start, width, mult_factor)\
+{\
+	u64 val, _mask;\
+	\
+	val = readq(power_domain_info->sst_base +\
+		    power_domain_info->perf_levels[level].mmio_offset +\
+		(power_domain_info->feature_offsets.tf_offset * 8) + (offset));\
+	_mask = GENMASK_ULL((start + width - 1), start);\
+	val &= _mask; \
+	val >>= start;\
+	name = (val * mult_factor);\
+}
+
+#define SST_PP_STATUS_OFFSET	32
+
+#define SST_PP_LEVEL_START	0
+#define SST_PP_LEVEL_WIDTH	3
+
+#define SST_PP_LOCK_START	3
+#define SST_PP_LOCK_WIDTH	1
+
+#define SST_PP_FEATURE_STATE_START	8
+#define SST_PP_FEATURE_STATE_WIDTH	8
+
+#define SST_BF_FEATURE_SUPPORTED_START	12
+#define SST_BF_FEATURE_SUPPORTED_WIDTH	1
+
+#define SST_TF_FEATURE_SUPPORTED_START	12
+#define SST_TF_FEATURE_SUPPORTED_WIDTH	1
+
+static int isst_if_get_perf_level(void __user *argp)
+{
+	struct isst_perf_level_info perf_level;
+	struct tpmi_per_power_domain_info *power_domain_info;
+
+	if (copy_from_user(&perf_level, argp, sizeof(perf_level)))
+		return -EFAULT;
+
+	power_domain_info = get_instance(perf_level.socket_id, perf_level.power_domain_id);
+	if (!power_domain_info)
+		return -EINVAL;
+
+	perf_level.max_level = power_domain_info->max_level;
+	perf_level.level_mask = power_domain_info->pp_header.allowed_level_mask;
+	perf_level.feature_rev = power_domain_info->pp_header.feature_rev;
+	_read_pp_info("current_level", perf_level.current_level, SST_PP_STATUS_OFFSET,
+		      SST_PP_LEVEL_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE)
+	_read_pp_info("locked", perf_level.locked, SST_PP_STATUS_OFFSET,
+		      SST_PP_LOCK_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE)
+	_read_pp_info("feature_state", perf_level.feature_state, SST_PP_STATUS_OFFSET,
+		      SST_PP_FEATURE_STATE_START, SST_PP_FEATURE_STATE_WIDTH, SST_MUL_FACTOR_NONE)
+	perf_level.enabled = !!(power_domain_info->sst_header.cap_mask & BIT(1));
+
+	_read_bf_level_info("bf_support", perf_level.sst_bf_support, 0, 0,
+			    SST_BF_FEATURE_SUPPORTED_START, SST_BF_FEATURE_SUPPORTED_WIDTH,
+			    SST_MUL_FACTOR_NONE);
+	_read_tf_level_info("tf_support", perf_level.sst_tf_support, 0, 0,
+			    SST_TF_FEATURE_SUPPORTED_START, SST_TF_FEATURE_SUPPORTED_WIDTH,
+			    SST_MUL_FACTOR_NONE);
+
+	if (copy_to_user(argp, &perf_level, sizeof(perf_level)))
+		return -EFAULT;
+
+	return 0;
+}
+
+#define SST_PP_CONTROL_OFFSET		24
+#define SST_PP_LEVEL_CHANGE_TIME_MS	5
+#define SST_PP_LEVEL_CHANGE_RETRY_COUNT	3
+
+static int isst_if_set_perf_level(void __user *argp)
+{
+	struct isst_perf_level_control perf_level;
+	struct tpmi_per_power_domain_info *power_domain_info;
+	int level, retry = 0;
+
+	if (disable_dynamic_sst_features())
+		return -EFAULT;
+
+	if (copy_from_user(&perf_level, argp, sizeof(perf_level)))
+		return -EFAULT;
+
+	power_domain_info = get_instance(perf_level.socket_id, perf_level.power_domain_id);
+	if (!power_domain_info)
+		return -EINVAL;
+
+	if (!(power_domain_info->pp_header.allowed_level_mask & BIT(perf_level.level)))
+		return -EINVAL;
+
+	_read_pp_info("current_level", level, SST_PP_STATUS_OFFSET,
+		      SST_PP_LEVEL_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE)
+
+	/* If the requested new level is same as the current level, reject */
+	if (perf_level.level == level)
+		return -EINVAL;
+
+	_write_pp_info("perf_level", perf_level.level, SST_PP_CONTROL_OFFSET,
+		       SST_PP_LEVEL_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE)
+
+	/* It is possible that firmware is busy (although unlikely), so retry */
+	do {
+		/* Give time to FW to process */
+		msleep(SST_PP_LEVEL_CHANGE_TIME_MS);
+
+		_read_pp_info("current_level", level, SST_PP_STATUS_OFFSET,
+			      SST_PP_LEVEL_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE)
+
+		/* Check if the new level is active */
+		if (perf_level.level == level)
+			break;
+
+	} while (retry++ < SST_PP_LEVEL_CHANGE_RETRY_COUNT);
+
+	/* If the level change didn't happen, return fault */
+	if (perf_level.level != level)
+		return -EFAULT;
+
+	/* Reset the feature state on level change */
+	_write_pp_info("perf_feature", 0, SST_PP_CONTROL_OFFSET,
+		       SST_PP_FEATURE_STATE_START, SST_PP_FEATURE_STATE_WIDTH,
+		       SST_MUL_FACTOR_NONE)
+
+	/* Give time to FW to process */
+	msleep(SST_PP_LEVEL_CHANGE_TIME_MS);
+
+	return 0;
+}
+
+static int isst_if_set_perf_feature(void __user *argp)
+{
+	struct isst_perf_feature_control perf_feature;
+	struct tpmi_per_power_domain_info *power_domain_info;
+
+	if (disable_dynamic_sst_features())
+		return -EFAULT;
+
+	if (copy_from_user(&perf_feature, argp, sizeof(perf_feature)))
+		return -EFAULT;
+
+	power_domain_info = get_instance(perf_feature.socket_id, perf_feature.power_domain_id);
+	if (!power_domain_info)
+		return -EINVAL;
+
+	_write_pp_info("perf_feature", perf_feature.feature, SST_PP_CONTROL_OFFSET,
+		       SST_PP_FEATURE_STATE_START, SST_PP_FEATURE_STATE_WIDTH,
+		       SST_MUL_FACTOR_NONE)
+
+	return 0;
+}
+
+#define _read_pp_level_info(name_str, name, level, offset, start, width, mult_factor)\
+{\
+	u64 val, _mask;\
+	\
+	val = readq(power_domain_info->sst_base +\
+		    power_domain_info->perf_levels[level].mmio_offset +\
+		(power_domain_info->feature_offsets.pp_offset * 8) + (offset));\
+	_mask = GENMASK_ULL((start + width - 1), start);\
+	val &= _mask; \
+	val >>= start;\
+	name = (val * mult_factor);\
+}
+
+#define SST_PP_INFO_0_OFFSET	0
+#define SST_PP_INFO_1_OFFSET	8
+#define SST_PP_INFO_2_OFFSET	16
+#define SST_PP_INFO_3_OFFSET	24
+
+/* SST_PP_INFO_4_OFFSET to SST_PP_INFO_9_OFFSET are trl levels */
+#define SST_PP_INFO_4_OFFSET	32
+
+#define SST_PP_INFO_10_OFFSET	80
+#define SST_PP_INFO_11_OFFSET	88
+
+#define SST_PP_P1_SSE_START	0
+#define SST_PP_P1_SSE_WIDTH	8
+
+#define SST_PP_P1_AVX2_START	8
+#define SST_PP_P1_AVX2_WIDTH	8
+
+#define SST_PP_P1_AVX512_START	16
+#define SST_PP_P1_AVX512_WIDTH	8
+
+#define SST_PP_P1_AMX_START	24
+#define SST_PP_P1_AMX_WIDTH	8
+
+#define SST_PP_TDP_START	32
+#define SST_PP_TDP_WIDTH	15
+
+#define SST_PP_T_PROCHOT_START	47
+#define SST_PP_T_PROCHOT_WIDTH	8
+
+#define SST_PP_MAX_MEMORY_FREQ_START	55
+#define SST_PP_MAX_MEMORY_FREQ_WIDTH	7
+
+#define SST_PP_COOLING_TYPE_START	62
+#define SST_PP_COOLING_TYPE_WIDTH	2
+
+#define SST_PP_TRL_0_RATIO_0_START	0
+#define SST_PP_TRL_0_RATIO_0_WIDTH	8
+
+#define SST_PP_TRL_CORES_BUCKET_0_START	0
+#define SST_PP_TRL_CORES_BUCKET_0_WIDTH	8
+
+#define SST_PP_CORE_RATIO_P0_START	0
+#define SST_PP_CORE_RATIO_P0_WIDTH	8
+
+#define SST_PP_CORE_RATIO_P1_START	8
+#define SST_PP_CORE_RATIO_P1_WIDTH	8
+
+#define SST_PP_CORE_RATIO_PN_START	16
+#define SST_PP_CORE_RATIO_PN_WIDTH	8
+
+#define SST_PP_CORE_RATIO_PM_START	24
+#define SST_PP_CORE_RATIO_PM_WIDTH	8
+
+#define SST_PP_CORE_RATIO_P0_FABRIC_START	32
+#define SST_PP_CORE_RATIO_P0_FABRIC_WIDTH	8
+
+#define SST_PP_CORE_RATIO_P1_FABRIC_START	40
+#define SST_PP_CORE_RATIO_P1_FABRIC_WIDTH	8
+
+#define SST_PP_CORE_RATIO_PM_FABRIC_START	48
+#define SST_PP_CORE_RATIO_PM_FABRIC_WIDTH	8
+
+static int isst_if_get_perf_level_info(void __user *argp)
+{
+	struct isst_perf_level_data_info perf_level;
+	struct tpmi_per_power_domain_info *power_domain_info;
+	int i, j;
+
+	if (copy_from_user(&perf_level, argp, sizeof(perf_level)))
+		return -EFAULT;
+
+	power_domain_info = get_instance(perf_level.socket_id, perf_level.power_domain_id);
+	if (!power_domain_info)
+		return -EINVAL;
+
+	if (perf_level.level > power_domain_info->max_level)
+		return -EINVAL;
+
+	if (!(power_domain_info->pp_header.level_en_mask & BIT(perf_level.level)))
+		return -EINVAL;
+
+	_read_pp_level_info("tdp_ratio", perf_level.tdp_ratio, perf_level.level,
+			    SST_PP_INFO_0_OFFSET, SST_PP_P1_SSE_START, SST_PP_P1_SSE_WIDTH,
+			    SST_MUL_FACTOR_NONE)
+	_read_pp_level_info("base_freq_mhz", perf_level.base_freq_mhz, perf_level.level,
+			    SST_PP_INFO_0_OFFSET, SST_PP_P1_SSE_START, SST_PP_P1_SSE_WIDTH,
+			    SST_MUL_FACTOR_FREQ)
+	_read_pp_level_info("base_freq_avx2_mhz", perf_level.base_freq_avx2_mhz, perf_level.level,
+			    SST_PP_INFO_0_OFFSET, SST_PP_P1_AVX2_START, SST_PP_P1_AVX2_WIDTH,
+			    SST_MUL_FACTOR_FREQ)
+	_read_pp_level_info("base_freq_avx512_mhz", perf_level.base_freq_avx512_mhz,
+			    perf_level.level, SST_PP_INFO_0_OFFSET, SST_PP_P1_AVX512_START,
+			    SST_PP_P1_AVX512_WIDTH, SST_MUL_FACTOR_FREQ)
+	_read_pp_level_info("base_freq_amx_mhz", perf_level.base_freq_amx_mhz, perf_level.level,
+			    SST_PP_INFO_0_OFFSET, SST_PP_P1_AMX_START, SST_PP_P1_AMX_WIDTH,
+			    SST_MUL_FACTOR_FREQ)
+
+	_read_pp_level_info("thermal_design_power_w", perf_level.thermal_design_power_w,
+			    perf_level.level, SST_PP_INFO_1_OFFSET, SST_PP_TDP_START,
+			    SST_PP_TDP_WIDTH, SST_MUL_FACTOR_NONE)
+	perf_level.thermal_design_power_w /= 8; /* units are in 1/8th watt */
+	_read_pp_level_info("tjunction_max_c", perf_level.tjunction_max_c, perf_level.level,
+			    SST_PP_INFO_1_OFFSET, SST_PP_T_PROCHOT_START, SST_PP_T_PROCHOT_WIDTH,
+			    SST_MUL_FACTOR_NONE)
+	_read_pp_level_info("max_memory_freq_mhz", perf_level.max_memory_freq_mhz,
+			    perf_level.level, SST_PP_INFO_1_OFFSET, SST_PP_MAX_MEMORY_FREQ_START,
+			    SST_PP_MAX_MEMORY_FREQ_WIDTH, SST_MUL_FACTOR_FREQ)
+	_read_pp_level_info("cooling_type", perf_level.cooling_type, perf_level.level,
+			    SST_PP_INFO_1_OFFSET, SST_PP_COOLING_TYPE_START,
+			    SST_PP_COOLING_TYPE_WIDTH, SST_MUL_FACTOR_NONE)
+
+	for (i = 0; i < TRL_MAX_LEVELS; ++i) {
+		for (j = 0; j < TRL_MAX_BUCKETS; ++j)
+			_read_pp_level_info("trl*_bucket*_freq_mhz",
+					    perf_level.trl_freq_mhz[i][j], perf_level.level,
+					    SST_PP_INFO_4_OFFSET + (i * SST_PP_TRL_0_RATIO_0_WIDTH),
+					    j * SST_PP_TRL_0_RATIO_0_WIDTH,
+					    SST_PP_TRL_0_RATIO_0_WIDTH,
+					    SST_MUL_FACTOR_FREQ);
+	}
+
+	for (i = 0; i < TRL_MAX_BUCKETS; ++i)
+		_read_pp_level_info("bucket*_core_count", perf_level.bucket_core_counts[i],
+				    perf_level.level, SST_PP_INFO_10_OFFSET,
+				    SST_PP_TRL_CORES_BUCKET_0_WIDTH * i,
+				    SST_PP_TRL_CORES_BUCKET_0_WIDTH, SST_MUL_FACTOR_NONE)
+
+	perf_level.max_buckets = TRL_MAX_BUCKETS;
+	perf_level.max_trl_levels = TRL_MAX_LEVELS;
+
+	_read_pp_level_info("p0_freq_mhz", perf_level.p0_freq_mhz, perf_level.level,
+			    SST_PP_INFO_11_OFFSET, SST_PP_CORE_RATIO_P0_START,
+			    SST_PP_CORE_RATIO_P0_WIDTH, SST_MUL_FACTOR_FREQ)
+	_read_pp_level_info("p1_freq_mhz", perf_level.p1_freq_mhz, perf_level.level,
+			    SST_PP_INFO_11_OFFSET, SST_PP_CORE_RATIO_P1_START,
+			    SST_PP_CORE_RATIO_P1_WIDTH, SST_MUL_FACTOR_FREQ)
+	_read_pp_level_info("pn_freq_mhz", perf_level.pn_freq_mhz, perf_level.level,
+			    SST_PP_INFO_11_OFFSET, SST_PP_CORE_RATIO_PN_START,
+			    SST_PP_CORE_RATIO_PN_WIDTH, SST_MUL_FACTOR_FREQ)
+	_read_pp_level_info("pm_freq_mhz", perf_level.pm_freq_mhz, perf_level.level,
+			    SST_PP_INFO_11_OFFSET, SST_PP_CORE_RATIO_PM_START,
+			    SST_PP_CORE_RATIO_PM_WIDTH, SST_MUL_FACTOR_FREQ)
+	_read_pp_level_info("p0_fabric_freq_mhz", perf_level.p0_fabric_freq_mhz,
+			    perf_level.level, SST_PP_INFO_11_OFFSET,
+			    SST_PP_CORE_RATIO_P0_FABRIC_START,
+			    SST_PP_CORE_RATIO_P0_FABRIC_WIDTH, SST_MUL_FACTOR_FREQ)
+	_read_pp_level_info("p1_fabric_freq_mhz", perf_level.p1_fabric_freq_mhz,
+			    perf_level.level, SST_PP_INFO_11_OFFSET,
+			    SST_PP_CORE_RATIO_P1_FABRIC_START,
+			    SST_PP_CORE_RATIO_P1_FABRIC_WIDTH, SST_MUL_FACTOR_FREQ)
+	_read_pp_level_info("pm_fabric_freq_mhz", perf_level.pm_fabric_freq_mhz,
+			    perf_level.level, SST_PP_INFO_11_OFFSET,
+			    SST_PP_CORE_RATIO_PM_FABRIC_START,
+			    SST_PP_CORE_RATIO_PM_FABRIC_WIDTH, SST_MUL_FACTOR_FREQ)
+
+	if (copy_to_user(argp, &perf_level, sizeof(perf_level)))
+		return -EFAULT;
+
+	return 0;
+}
+
+#define SST_PP_FUSED_CORE_COUNT_START	0
+#define SST_PP_FUSED_CORE_COUNT_WIDTH	8
+
+#define SST_PP_RSLVD_CORE_COUNT_START	8
+#define SST_PP_RSLVD_CORE_COUNT_WIDTH	8
+
+#define SST_PP_RSLVD_CORE_MASK_START	0
+#define SST_PP_RSLVD_CORE_MASK_WIDTH	64
+
+static int isst_if_get_perf_level_mask(void __user *argp)
+{
+	static struct isst_perf_level_cpu_mask cpumask;
+	struct tpmi_per_power_domain_info *power_domain_info;
+	u64 mask;
+
+	if (copy_from_user(&cpumask, argp, sizeof(cpumask)))
+		return -EFAULT;
+
+	power_domain_info = get_instance(cpumask.socket_id, cpumask.power_domain_id);
+	if (!power_domain_info)
+		return -EINVAL;
+
+	_read_pp_level_info("mask", mask, cpumask.level, SST_PP_INFO_2_OFFSET,
+			    SST_PP_RSLVD_CORE_MASK_START, SST_PP_RSLVD_CORE_MASK_WIDTH,
+			    SST_MUL_FACTOR_NONE)
+
+	cpumask.mask = mask;
+
+	if (!cpumask.punit_cpu_map)
+		return -EOPNOTSUPP;
+
+	if (copy_to_user(argp, &cpumask, sizeof(cpumask)))
+		return -EFAULT;
+
+	return 0;
+}
+
+#define SST_BF_INFO_0_OFFSET	0
+#define SST_BF_INFO_1_OFFSET	8
+
+#define SST_BF_P1_HIGH_START	13
+#define SST_BF_P1_HIGH_WIDTH	8
+
+#define SST_BF_P1_LOW_START	21
+#define SST_BF_P1_LOW_WIDTH	8
+
+#define SST_BF_T_PROHOT_START	38
+#define SST_BF_T_PROHOT_WIDTH	8
+
+#define SST_BF_TDP_START	46
+#define SST_BF_TDP_WIDTH	15
+
+static int isst_if_get_base_freq_info(void __user *argp)
+{
+	static struct isst_base_freq_info base_freq;
+	struct tpmi_per_power_domain_info *power_domain_info;
+
+	if (copy_from_user(&base_freq, argp, sizeof(base_freq)))
+		return -EFAULT;
+
+	power_domain_info = get_instance(base_freq.socket_id, base_freq.power_domain_id);
+	if (!power_domain_info)
+		return -EINVAL;
+
+	if (base_freq.level > power_domain_info->max_level)
+		return -EINVAL;
+
+	_read_bf_level_info("p1_high", base_freq.high_base_freq_mhz, base_freq.level,
+			    SST_BF_INFO_0_OFFSET, SST_BF_P1_HIGH_START, SST_BF_P1_HIGH_WIDTH,
+			    SST_MUL_FACTOR_FREQ)
+	_read_bf_level_info("p1_low", base_freq.low_base_freq_mhz, base_freq.level,
+			    SST_BF_INFO_0_OFFSET, SST_BF_P1_LOW_START, SST_BF_P1_LOW_WIDTH,
+			    SST_MUL_FACTOR_FREQ)
+	_read_bf_level_info("BF-TJ", base_freq.tjunction_max_c, base_freq.level,
+			    SST_BF_INFO_0_OFFSET, SST_BF_T_PROHOT_START, SST_BF_T_PROHOT_WIDTH,
+			    SST_MUL_FACTOR_NONE)
+	_read_bf_level_info("BF-tdp", base_freq.thermal_design_power_w, base_freq.level,
+			    SST_BF_INFO_0_OFFSET, SST_BF_TDP_START, SST_BF_TDP_WIDTH,
+			    SST_MUL_FACTOR_NONE)
+	base_freq.thermal_design_power_w /= 8; /*unit = 1/8th watt*/
+
+	if (copy_to_user(argp, &base_freq, sizeof(base_freq)))
+		return -EFAULT;
+
+	return 0;
+}
+
+#define P1_HI_CORE_MASK_START	0
+#define P1_HI_CORE_MASK_WIDTH	64
+
+static int isst_if_get_base_freq_mask(void __user *argp)
+{
+	static struct isst_perf_level_cpu_mask cpumask;
+	struct tpmi_per_power_domain_info *power_domain_info;
+	u64 mask;
+
+	if (copy_from_user(&cpumask, argp, sizeof(cpumask)))
+		return -EFAULT;
+
+	power_domain_info = get_instance(cpumask.socket_id, cpumask.power_domain_id);
+	if (!power_domain_info)
+		return -EINVAL;
+
+	_read_bf_level_info("BF-cpumask", mask, cpumask.level, SST_BF_INFO_1_OFFSET,
+			    P1_HI_CORE_MASK_START, P1_HI_CORE_MASK_WIDTH,
+			    SST_MUL_FACTOR_NONE)
+
+	cpumask.mask = mask;
+
+	if (!cpumask.punit_cpu_map)
+		return -EOPNOTSUPP;
+
+	if (copy_to_user(argp, &cpumask, sizeof(cpumask)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int isst_if_get_tpmi_instance_count(void __user *argp)
+{
+	struct isst_tpmi_instance_count tpmi_inst;
+	struct tpmi_sst_struct *sst_inst;
+	int i;
+
+	if (copy_from_user(&tpmi_inst, argp, sizeof(tpmi_inst)))
+		return -EFAULT;
+
+	if (tpmi_inst.socket_id >= topology_max_packages())
+		return -EINVAL;
+
+	tpmi_inst.count = isst_common.sst_inst[tpmi_inst.socket_id]->number_of_power_domains;
+
+	sst_inst = isst_common.sst_inst[tpmi_inst.socket_id];
+	tpmi_inst.valid_mask = 0;
+	for (i = 0; i < sst_inst->number_of_power_domains; ++i) {
+		struct tpmi_per_power_domain_info *pd_info;
+
+		pd_info = &sst_inst->power_domain_info[i];
+		if (pd_info->sst_base)
+			tpmi_inst.valid_mask |= BIT(i);
+	}
+
+	if (copy_to_user(argp, &tpmi_inst, sizeof(tpmi_inst)))
+		return -EFAULT;
+
+	return 0;
+}
+
+#define SST_TF_INFO_0_OFFSET	0
+#define SST_TF_INFO_1_OFFSET	8
+#define SST_TF_INFO_2_OFFSET	16
+
+#define SST_TF_MAX_LP_CLIP_RATIOS	TRL_MAX_LEVELS
+
+#define SST_TF_LP_CLIP_RATIO_0_START	16
+#define SST_TF_LP_CLIP_RATIO_0_WIDTH	8
+
+#define SST_TF_RATIO_0_START	0
+#define SST_TF_RATIO_0_WIDTH	8
+
+#define SST_TF_NUM_CORE_0_START 0
+#define SST_TF_NUM_CORE_0_WIDTH 8
+
+static int isst_if_get_turbo_freq_info(void __user *argp)
+{
+	static struct isst_turbo_freq_info turbo_freq;
+	struct tpmi_per_power_domain_info *power_domain_info;
+	int i, j;
+
+	if (copy_from_user(&turbo_freq, argp, sizeof(turbo_freq)))
+		return -EFAULT;
+
+	power_domain_info = get_instance(turbo_freq.socket_id, turbo_freq.power_domain_id);
+	if (!power_domain_info)
+		return -EINVAL;
+
+	if (turbo_freq.level > power_domain_info->max_level)
+		return -EINVAL;
+
+	turbo_freq.max_buckets = TRL_MAX_BUCKETS;
+	turbo_freq.max_trl_levels = TRL_MAX_LEVELS;
+	turbo_freq.max_clip_freqs = SST_TF_MAX_LP_CLIP_RATIOS;
+
+	for (i = 0; i < turbo_freq.max_clip_freqs; ++i)
+		_read_tf_level_info("lp_clip*", turbo_freq.lp_clip_freq_mhz[i],
+				    turbo_freq.level, SST_TF_INFO_0_OFFSET,
+				    SST_TF_LP_CLIP_RATIO_0_START +
+				    (i * SST_TF_LP_CLIP_RATIO_0_WIDTH),
+				    SST_TF_LP_CLIP_RATIO_0_WIDTH, SST_MUL_FACTOR_FREQ)
+
+	for (i = 0; i < TRL_MAX_LEVELS; ++i) {
+		for (j = 0; j < TRL_MAX_BUCKETS; ++j)
+			_read_tf_level_info("cydn*_bucket_*_trl",
+					    turbo_freq.trl_freq_mhz[i][j], turbo_freq.level,
+					    SST_TF_INFO_2_OFFSET + (i * SST_TF_RATIO_0_WIDTH),
+					    j * SST_TF_RATIO_0_WIDTH, SST_TF_RATIO_0_WIDTH,
+					    SST_MUL_FACTOR_FREQ)
+	}
+
+	for (i = 0; i < TRL_MAX_BUCKETS; ++i)
+		_read_tf_level_info("bucket_*_core_count", turbo_freq.bucket_core_counts[i],
+				    turbo_freq.level, SST_TF_INFO_1_OFFSET,
+				    SST_TF_NUM_CORE_0_WIDTH * i, SST_TF_NUM_CORE_0_WIDTH,
+				    SST_MUL_FACTOR_NONE)
+
+	if (copy_to_user(argp, &turbo_freq, sizeof(turbo_freq)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static long isst_if_def_ioctl(struct file *file, unsigned int cmd,
+			      unsigned long arg)
+{
+	void __user *argp = (void __user *)arg;
+	long ret = -ENOTTY;
+
+	mutex_lock(&isst_tpmi_dev_lock);
+	switch (cmd) {
+	case ISST_IF_COUNT_TPMI_INSTANCES:
+		ret = isst_if_get_tpmi_instance_count(argp);
+		break;
+	case ISST_IF_CORE_POWER_STATE:
+		ret = isst_if_core_power_state(argp);
+		break;
+	case ISST_IF_CLOS_PARAM:
+		ret = isst_if_clos_param(argp);
+		break;
+	case ISST_IF_CLOS_ASSOC:
+		ret = isst_if_clos_assoc(argp);
+		break;
+	case ISST_IF_PERF_LEVELS:
+		ret = isst_if_get_perf_level(argp);
+		break;
+	case ISST_IF_PERF_SET_LEVEL:
+		ret = isst_if_set_perf_level(argp);
+		break;
+	case ISST_IF_PERF_SET_FEATURE:
+		ret = isst_if_set_perf_feature(argp);
+		break;
+	case ISST_IF_GET_PERF_LEVEL_INFO:
+		ret = isst_if_get_perf_level_info(argp);
+		break;
+	case ISST_IF_GET_PERF_LEVEL_CPU_MASK:
+		ret = isst_if_get_perf_level_mask(argp);
+		break;
+	case ISST_IF_GET_BASE_FREQ_INFO:
+		ret = isst_if_get_base_freq_info(argp);
+		break;
+	case ISST_IF_GET_BASE_FREQ_CPU_MASK:
+		ret = isst_if_get_base_freq_mask(argp);
+		break;
+	case ISST_IF_GET_TURBO_FREQ_INFO:
+		ret = isst_if_get_turbo_freq_info(argp);
+		break;
+	default:
+		break;
+	}
+	mutex_unlock(&isst_tpmi_dev_lock);
+
+	return ret;
+}
+
+#define TPMI_SST_AUTO_SUSPEND_DELAY_MS	2000
+
+int tpmi_sst_dev_add(struct auxiliary_device *auxdev)
+{
+	struct intel_tpmi_plat_info *plat_info;
+	struct tpmi_sst_struct *tpmi_sst;
+	int i, ret, pkg = 0, inst = 0;
+	int num_resources;
+
+	plat_info = tpmi_get_platform_data(auxdev);
+	if (!plat_info) {
+		dev_err(&auxdev->dev, "No platform info\n");
+		return -EINVAL;
+	}
+
+	pkg = plat_info->package_id;
+	if (pkg >= topology_max_packages()) {
+		dev_err(&auxdev->dev, "Invalid package id :%x\n", pkg);
+		return -EINVAL;
+	}
+
+	if (isst_common.sst_inst[pkg])
+		return -EEXIST;
+
+	num_resources = tpmi_get_resource_count(auxdev);
+
+	if (!num_resources)
+		return -EINVAL;
+
+	tpmi_sst = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_sst), GFP_KERNEL);
+	if (!tpmi_sst)
+		return -ENOMEM;
+
+	tpmi_sst->power_domain_info = devm_kcalloc(&auxdev->dev, num_resources,
+						   sizeof(*tpmi_sst->power_domain_info),
+						   GFP_KERNEL);
+	if (!tpmi_sst->power_domain_info)
+		return -ENOMEM;
+
+	tpmi_sst->number_of_power_domains = num_resources;
+
+	for (i = 0; i < num_resources; ++i) {
+		struct resource *res;
+
+		res = tpmi_get_resource_at_index(auxdev, i);
+		if (!res) {
+			tpmi_sst->power_domain_info[i].sst_base = NULL;
+			continue;
+		}
+
+		tpmi_sst->power_domain_info[i].package_id = pkg;
+		tpmi_sst->power_domain_info[i].power_domain_id = i;
+		tpmi_sst->power_domain_info[i].auxdev = auxdev;
+		tpmi_sst->power_domain_info[i].sst_base = devm_ioremap_resource(&auxdev->dev, res);
+		if (IS_ERR(tpmi_sst->power_domain_info[i].sst_base))
+			return PTR_ERR(tpmi_sst->power_domain_info[i].sst_base);
+
+		ret = sst_main(auxdev, &tpmi_sst->power_domain_info[i]);
+		if (ret) {
+			devm_iounmap(&auxdev->dev, tpmi_sst->power_domain_info[i].sst_base);
+			tpmi_sst->power_domain_info[i].sst_base =  NULL;
+			continue;
+		}
+
+		++inst;
+	}
+
+	if (!inst)
+		return -ENODEV;
+
+	tpmi_sst->package_id = pkg;
+	auxiliary_set_drvdata(auxdev, tpmi_sst);
+
+	mutex_lock(&isst_tpmi_dev_lock);
+	if (isst_common.max_index < pkg)
+		isst_common.max_index = pkg;
+	isst_common.sst_inst[pkg] = tpmi_sst;
+	mutex_unlock(&isst_tpmi_dev_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_add, INTEL_TPMI_SST);
+
+void tpmi_sst_dev_remove(struct auxiliary_device *auxdev)
+{
+	struct tpmi_sst_struct *tpmi_sst = auxiliary_get_drvdata(auxdev);
+
+	mutex_lock(&isst_tpmi_dev_lock);
+	isst_common.sst_inst[tpmi_sst->package_id] = NULL;
+	mutex_unlock(&isst_tpmi_dev_lock);
+}
+EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_remove, INTEL_TPMI_SST);
+
+void tpmi_sst_dev_suspend(struct auxiliary_device *auxdev)
+{
+	struct tpmi_sst_struct *tpmi_sst = auxiliary_get_drvdata(auxdev);
+	struct tpmi_per_power_domain_info *power_domain_info = tpmi_sst->power_domain_info;
+	void __iomem *cp_base;
+
+	cp_base = power_domain_info->sst_base + power_domain_info->sst_header.cp_offset;
+	power_domain_info->saved_sst_cp_control = readq(cp_base + SST_CP_CONTROL_OFFSET);
+
+	memcpy_fromio(power_domain_info->saved_clos_configs, cp_base + SST_CLOS_CONFIG_0_OFFSET,
+		      sizeof(power_domain_info->saved_clos_configs));
+
+	memcpy_fromio(power_domain_info->saved_clos_assocs, cp_base + SST_CLOS_ASSOC_0_OFFSET,
+		      sizeof(power_domain_info->saved_clos_assocs));
+
+	power_domain_info->saved_pp_control = readq(power_domain_info->sst_base +
+						    power_domain_info->sst_header.pp_offset +
+						    SST_PP_CONTROL_OFFSET);
+}
+EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_suspend, INTEL_TPMI_SST);
+
+void tpmi_sst_dev_resume(struct auxiliary_device *auxdev)
+{
+	struct tpmi_sst_struct *tpmi_sst = auxiliary_get_drvdata(auxdev);
+	struct tpmi_per_power_domain_info *power_domain_info = tpmi_sst->power_domain_info;
+	void __iomem *cp_base;
+
+	cp_base = power_domain_info->sst_base + power_domain_info->sst_header.cp_offset;
+	writeq(power_domain_info->saved_sst_cp_control, cp_base + SST_CP_CONTROL_OFFSET);
+
+	memcpy_toio(cp_base + SST_CLOS_CONFIG_0_OFFSET, power_domain_info->saved_clos_configs,
+		    sizeof(power_domain_info->saved_clos_configs));
+
+	memcpy_toio(cp_base + SST_CLOS_ASSOC_0_OFFSET, power_domain_info->saved_clos_assocs,
+		    sizeof(power_domain_info->saved_clos_assocs));
+
+	writeq(power_domain_info->saved_pp_control, power_domain_info->sst_base +
+				power_domain_info->sst_header.pp_offset + SST_PP_CONTROL_OFFSET);
+}
+EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_resume, INTEL_TPMI_SST);
+
+#define ISST_TPMI_API_VERSION	0x02
+
+int tpmi_sst_init(void)
+{
+	struct isst_if_cmd_cb cb;
+	int ret = 0;
+
+	mutex_lock(&isst_tpmi_dev_lock);
+
+	if (isst_core_usage_count) {
+		++isst_core_usage_count;
+		goto init_done;
+	}
+
+	isst_common.sst_inst = kcalloc(topology_max_packages(),
+				       sizeof(*isst_common.sst_inst),
+				       GFP_KERNEL);
+	if (!isst_common.sst_inst) {
+		ret = -ENOMEM;
+		goto init_done;
+	}
+
+	memset(&cb, 0, sizeof(cb));
+	cb.cmd_size = sizeof(struct isst_if_io_reg);
+	cb.offset = offsetof(struct isst_if_io_regs, io_reg);
+	cb.cmd_callback = NULL;
+	cb.api_version = ISST_TPMI_API_VERSION;
+	cb.def_ioctl = isst_if_def_ioctl;
+	cb.owner = THIS_MODULE;
+	ret = isst_if_cdev_register(ISST_IF_DEV_TPMI, &cb);
+	if (ret)
+		kfree(isst_common.sst_inst);
+init_done:
+	mutex_unlock(&isst_tpmi_dev_lock);
+	return ret;
+}
+EXPORT_SYMBOL_NS_GPL(tpmi_sst_init, INTEL_TPMI_SST);
+
+void tpmi_sst_exit(void)
+{
+	mutex_lock(&isst_tpmi_dev_lock);
+	if (isst_core_usage_count)
+		--isst_core_usage_count;
+
+	if (!isst_core_usage_count) {
+		isst_if_cdev_unregister(ISST_IF_DEV_TPMI);
+		kfree(isst_common.sst_inst);
+	}
+	mutex_unlock(&isst_tpmi_dev_lock);
+}
+EXPORT_SYMBOL_NS_GPL(tpmi_sst_exit, INTEL_TPMI_SST);
+
+MODULE_IMPORT_NS(INTEL_TPMI);
+MODULE_IMPORT_NS(INTEL_TPMI_POWER_DOMAIN);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.h b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.h
new file mode 100644
index 000000000000..900b483703f9
--- /dev/null
+++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Intel Speed Select Interface: Drivers Internal defines
+ * Copyright (c) 2023, Intel Corporation.
+ * All rights reserved.
+ *
+ */
+
+#ifndef _ISST_TPMI_CORE_H
+#define _ISST_TPMI_CORE_H
+
+int tpmi_sst_init(void);
+void tpmi_sst_exit(void);
+int tpmi_sst_dev_add(struct auxiliary_device *auxdev);
+void tpmi_sst_dev_remove(struct auxiliary_device *auxdev);
+void tpmi_sst_dev_suspend(struct auxiliary_device *auxdev);
+void tpmi_sst_dev_resume(struct auxiliary_device *auxdev);
+#endif
diff --git a/drivers/platform/x86/intel/telemetry/pltdrv.c b/drivers/platform/x86/intel/telemetry/pltdrv.c
index 405dea87de6b..06311d0e9451 100644
--- a/drivers/platform/x86/intel/telemetry/pltdrv.c
+++ b/drivers/platform/x86/intel/telemetry/pltdrv.c
@@ -1156,15 +1156,14 @@ out:
 	return ret;
 }
 
-static int telemetry_pltdrv_remove(struct platform_device *pdev)
+static void telemetry_pltdrv_remove(struct platform_device *pdev)
 {
 	telemetry_clear_pltdata();
-	return 0;
 }
 
 static struct platform_driver telemetry_soc_driver = {
 	.probe		= telemetry_pltdrv_probe,
-	.remove		= telemetry_pltdrv_remove,
+	.remove_new	= telemetry_pltdrv_remove,
 	.driver		= {
 		.name	= DRIVER_NAME,
 	},
diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
index cb24de9e97dc..1a300e14f350 100644
--- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
+++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
@@ -224,9 +224,15 @@ int uncore_freq_common_init(int (*read_control_freq)(struct uncore_data *data, u
 	uncore_write = write_control_freq;
 	uncore_read_freq = read_freq;
 
-	if (!uncore_root_kobj)
-		uncore_root_kobj = kobject_create_and_add("intel_uncore_frequency",
-							    &cpu_subsys.dev_root->kobj);
+	if (!uncore_root_kobj) {
+		struct device *dev_root = bus_get_dev_root(&cpu_subsys);
+
+		if (dev_root) {
+			uncore_root_kobj = kobject_create_and_add("intel_uncore_frequency",
+								  &dev_root->kobj);
+			put_device(dev_root);
+		}
+	}
 	if (uncore_root_kobj)
 		++uncore_instance_count;
 	mutex_unlock(&uncore_lock);
diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c
index 00ac7e381441..32e2515ee366 100644
--- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c
+++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c
@@ -204,6 +204,13 @@ static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
 	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D,	NULL),
 	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
 	X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, NULL),
+	X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, NULL),
+	X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, NULL),
+	X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, NULL),
+	X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, NULL),
+	X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, NULL),
+	X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE, NULL),
+	X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, NULL),
 	{}
 };
 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_cpu_ids);
diff --git a/drivers/platform/x86/intel/vbtn.c b/drivers/platform/x86/intel/vbtn.c
index c5e4e35c8d20..6fa1735ad7a4 100644
--- a/drivers/platform/x86/intel/vbtn.c
+++ b/drivers/platform/x86/intel/vbtn.c
@@ -325,18 +325,12 @@ static int intel_vbtn_probe(struct platform_device *device)
 	return 0;
 }
 
-static int intel_vbtn_remove(struct platform_device *device)
+static void intel_vbtn_remove(struct platform_device *device)
 {
 	acpi_handle handle = ACPI_HANDLE(&device->dev);
 
 	device_init_wakeup(&device->dev, false);
 	acpi_remove_notify_handler(handle, ACPI_DEVICE_NOTIFY, notify_handler);
-
-	/*
-	 * Even if we failed to shut off the event stream, we can still
-	 * safely detach from the device.
-	 */
-	return 0;
 }
 
 static int intel_vbtn_pm_prepare(struct device *dev)
@@ -377,7 +371,7 @@ static struct platform_driver intel_vbtn_pl_driver = {
 		.pm = &intel_vbtn_pm_ops,
 	},
 	.probe = intel_vbtn_probe,
-	.remove = intel_vbtn_remove,
+	.remove_new = intel_vbtn_remove,
 };
 
 static acpi_status __init
diff --git a/drivers/platform/x86/intel/vsec.c b/drivers/platform/x86/intel/vsec.c
index 2311c16cb975..c1f9e4471b28 100644
--- a/drivers/platform/x86/intel/vsec.c
+++ b/drivers/platform/x86/intel/vsec.c
@@ -67,14 +67,6 @@ enum intel_vsec_id {
 	VSEC_ID_TPMI		= 66,
 };
 
-static enum intel_vsec_id intel_vsec_allow_list[] = {
-	VSEC_ID_TELEMETRY,
-	VSEC_ID_WATCHER,
-	VSEC_ID_CRASHLOG,
-	VSEC_ID_SDSI,
-	VSEC_ID_TPMI,
-};
-
 static const char *intel_vsec_name(enum intel_vsec_id id)
 {
 	switch (id) {
@@ -98,26 +90,19 @@ static const char *intel_vsec_name(enum intel_vsec_id id)
 	}
 }
 
-static bool intel_vsec_allowed(u16 id)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(intel_vsec_allow_list); i++)
-		if (intel_vsec_allow_list[i] == id)
-			return true;
-
-	return false;
-}
-
-static bool intel_vsec_disabled(u16 id, unsigned long quirks)
+static bool intel_vsec_supported(u16 id, unsigned long caps)
 {
 	switch (id) {
+	case VSEC_ID_TELEMETRY:
+		return !!(caps & VSEC_CAP_TELEMETRY);
 	case VSEC_ID_WATCHER:
-		return !!(quirks & VSEC_QUIRK_NO_WATCHER);
-
+		return !!(caps & VSEC_CAP_WATCHER);
 	case VSEC_ID_CRASHLOG:
-		return !!(quirks & VSEC_QUIRK_NO_CRASHLOG);
-
+		return !!(caps & VSEC_CAP_CRASHLOG);
+	case VSEC_ID_SDSI:
+		return !!(caps & VSEC_CAP_SDSI);
+	case VSEC_ID_TPMI:
+		return !!(caps & VSEC_CAP_TPMI);
 	default:
 		return false;
 	}
@@ -169,11 +154,7 @@ int intel_vsec_add_aux(struct pci_dev *pdev, struct device *parent,
 
 	ret = auxiliary_device_init(auxdev);
 	if (ret < 0) {
-		mutex_lock(&vsec_ida_lock);
-		ida_free(intel_vsec_dev->ida, auxdev->id);
-		mutex_unlock(&vsec_ida_lock);
-		kfree(intel_vsec_dev->resource);
-		kfree(intel_vsec_dev);
+		intel_vsec_dev_release(&auxdev->dev);
 		return ret;
 	}
 
@@ -206,7 +187,7 @@ static int intel_vsec_add_dev(struct pci_dev *pdev, struct intel_vsec_header *he
 	unsigned long quirks = info->quirks;
 	int i;
 
-	if (!intel_vsec_allowed(header->id) || intel_vsec_disabled(header->id, quirks))
+	if (!intel_vsec_supported(header->id, info->caps))
 		return -EINVAL;
 
 	if (!header->num_entries) {
@@ -261,14 +242,14 @@ static int intel_vsec_add_dev(struct pci_dev *pdev, struct intel_vsec_header *he
 static bool intel_vsec_walk_header(struct pci_dev *pdev,
 				   struct intel_vsec_platform_info *info)
 {
-	struct intel_vsec_header **header = info->capabilities;
+	struct intel_vsec_header **header = info->headers;
 	bool have_devices = false;
 	int ret;
 
 	for ( ; *header; header++) {
 		ret = intel_vsec_add_dev(pdev, *header, info);
 		if (ret)
-			dev_info(&pdev->dev, "Could not add device for DVSEC id %d\n",
+			dev_info(&pdev->dev, "Could not add device for VSEC id %d\n",
 				 (*header)->id);
 		else
 			have_devices = true;
@@ -403,14 +384,8 @@ static int intel_vsec_pci_probe(struct pci_dev *pdev, const struct pci_device_id
 	return 0;
 }
 
-/* TGL info */
-static const struct intel_vsec_platform_info tgl_info = {
-	.quirks = VSEC_QUIRK_NO_WATCHER | VSEC_QUIRK_NO_CRASHLOG |
-		  VSEC_QUIRK_TABLE_SHIFT | VSEC_QUIRK_EARLY_HW,
-};
-
 /* DG1 info */
-static struct intel_vsec_header dg1_telemetry = {
+static struct intel_vsec_header dg1_header = {
 	.length = 0x10,
 	.id = 2,
 	.num_entries = 1,
@@ -419,19 +394,31 @@ static struct intel_vsec_header dg1_telemetry = {
 	.offset = 0x466000,
 };
 
-static struct intel_vsec_header *dg1_capabilities[] = {
-	&dg1_telemetry,
+static struct intel_vsec_header *dg1_headers[] = {
+	&dg1_header,
 	NULL
 };
 
 static const struct intel_vsec_platform_info dg1_info = {
-	.capabilities = dg1_capabilities,
+	.caps = VSEC_CAP_TELEMETRY,
+	.headers = dg1_headers,
 	.quirks = VSEC_QUIRK_NO_DVSEC | VSEC_QUIRK_EARLY_HW,
 };
 
 /* MTL info */
 static const struct intel_vsec_platform_info mtl_info = {
-	.quirks = VSEC_QUIRK_NO_WATCHER | VSEC_QUIRK_NO_CRASHLOG,
+	.caps = VSEC_CAP_TELEMETRY,
+};
+
+/* OOBMSM info */
+static const struct intel_vsec_platform_info oobmsm_info = {
+	.caps = VSEC_CAP_TELEMETRY | VSEC_CAP_SDSI | VSEC_CAP_TPMI,
+};
+
+/* TGL info */
+static const struct intel_vsec_platform_info tgl_info = {
+	.caps = VSEC_CAP_TELEMETRY,
+	.quirks = VSEC_QUIRK_TABLE_SHIFT | VSEC_QUIRK_EARLY_HW,
 };
 
 #define PCI_DEVICE_ID_INTEL_VSEC_ADL		0x467d
@@ -446,7 +433,7 @@ static const struct pci_device_id intel_vsec_pci_ids[] = {
 	{ PCI_DEVICE_DATA(INTEL, VSEC_DG1, &dg1_info) },
 	{ PCI_DEVICE_DATA(INTEL, VSEC_MTL_M, &mtl_info) },
 	{ PCI_DEVICE_DATA(INTEL, VSEC_MTL_S, &mtl_info) },
-	{ PCI_DEVICE_DATA(INTEL, VSEC_OOBMSM, &(struct intel_vsec_platform_info) {}) },
+	{ PCI_DEVICE_DATA(INTEL, VSEC_OOBMSM, &oobmsm_info) },
 	{ PCI_DEVICE_DATA(INTEL, VSEC_RPL, &tgl_info) },
 	{ PCI_DEVICE_DATA(INTEL, VSEC_TGL, &tgl_info) },
 	{ }
diff --git a/drivers/platform/x86/intel/vsec.h b/drivers/platform/x86/intel/vsec.h
index ae8fe92c5595..0fd042c171ba 100644
--- a/drivers/platform/x86/intel/vsec.h
+++ b/drivers/platform/x86/intel/vsec.h
@@ -5,6 +5,12 @@
 #include <linux/auxiliary_bus.h>
 #include <linux/bits.h>
 
+#define VSEC_CAP_TELEMETRY	BIT(0)
+#define VSEC_CAP_WATCHER	BIT(1)
+#define VSEC_CAP_CRASHLOG	BIT(2)
+#define VSEC_CAP_SDSI		BIT(3)
+#define VSEC_CAP_TPMI		BIT(4)
+
 struct pci_dev;
 struct resource;
 
@@ -27,7 +33,8 @@ enum intel_vsec_quirks {
 
 /* Platform specific data */
 struct intel_vsec_platform_info {
-	struct intel_vsec_header **capabilities;
+	struct intel_vsec_header **headers;
+	unsigned long caps;
 	unsigned long quirks;
 };
 
diff --git a/drivers/platform/x86/intel_scu_ipc.c b/drivers/platform/x86/intel_scu_ipc.c
index e7a3e3402817..6851d10d6582 100644
--- a/drivers/platform/x86/intel_scu_ipc.c
+++ b/drivers/platform/x86/intel_scu_ipc.c
@@ -82,7 +82,6 @@ static DEFINE_MUTEX(ipclock); /* lock used to prevent multiple call to SCU */
 
 static struct class intel_scu_ipc_class = {
 	.name = "intel_scu_ipc",
-	.owner = THIS_MODULE,
 };
 
 /**
diff --git a/drivers/platform/x86/lenovo-ymc.c b/drivers/platform/x86/lenovo-ymc.c
new file mode 100644
index 000000000000..41676188b373
--- /dev/null
+++ b/drivers/platform/x86/lenovo-ymc.c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * lenovo-ymc.c - Lenovo Yoga Mode Control driver
+ *
+ * Copyright © 2022 Gergo Koteles <soyer@irl.hu>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/acpi.h>
+#include <linux/dmi.h>
+#include <linux/input.h>
+#include <linux/input/sparse-keymap.h>
+#include <linux/wmi.h>
+#include "ideapad-laptop.h"
+
+#define LENOVO_YMC_EVENT_GUID	"06129D99-6083-4164-81AD-F092F9D773A6"
+#define LENOVO_YMC_QUERY_GUID	"09B0EE6E-C3FD-4243-8DA1-7911FF80BB8C"
+
+#define LENOVO_YMC_QUERY_INSTANCE 0
+#define LENOVO_YMC_QUERY_METHOD 0x01
+
+static bool ec_trigger __read_mostly;
+module_param(ec_trigger, bool, 0444);
+MODULE_PARM_DESC(ec_trigger, "Enable EC triggering work-around to force emitting tablet mode events");
+
+static const struct dmi_system_id ec_trigger_quirk_dmi_table[] = {
+	{
+		/* Lenovo Yoga 7 14ARB7 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "82QF"),
+		},
+	},
+	{ }
+};
+
+struct lenovo_ymc_private {
+	struct input_dev *input_dev;
+	struct acpi_device *ec_acpi_dev;
+};
+
+static void lenovo_ymc_trigger_ec(struct wmi_device *wdev, struct lenovo_ymc_private *priv)
+{
+	int err;
+
+	if (!priv->ec_acpi_dev)
+		return;
+
+	err = write_ec_cmd(priv->ec_acpi_dev->handle, VPCCMD_W_YMC, 1);
+	if (err)
+		dev_warn(&wdev->dev, "Could not write YMC: %d\n", err);
+}
+
+static const struct key_entry lenovo_ymc_keymap[] = {
+	/* Laptop */
+	{ KE_SW, 0x01, { .sw = { SW_TABLET_MODE, 0 } } },
+	/* Tablet */
+	{ KE_SW, 0x02, { .sw = { SW_TABLET_MODE, 1 } } },
+	/* Drawing Board */
+	{ KE_SW, 0x03, { .sw = { SW_TABLET_MODE, 1 } } },
+	/* Tent */
+	{ KE_SW, 0x04, { .sw = { SW_TABLET_MODE, 1 } } },
+	{ KE_END },
+};
+
+static void lenovo_ymc_notify(struct wmi_device *wdev, union acpi_object *data)
+{
+	struct lenovo_ymc_private *priv = dev_get_drvdata(&wdev->dev);
+	u32 input_val = 0;
+	struct acpi_buffer input = { sizeof(input_val), &input_val };
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *obj;
+	acpi_status status;
+	int code;
+
+	status = wmi_evaluate_method(LENOVO_YMC_QUERY_GUID,
+				LENOVO_YMC_QUERY_INSTANCE,
+				LENOVO_YMC_QUERY_METHOD,
+				&input, &output);
+
+	if (ACPI_FAILURE(status)) {
+		dev_warn(&wdev->dev,
+			"Failed to evaluate query method: %s\n",
+			acpi_format_exception(status));
+		return;
+	}
+
+	obj = output.pointer;
+
+	if (obj->type != ACPI_TYPE_INTEGER) {
+		dev_warn(&wdev->dev,
+			"WMI event data is not an integer\n");
+		goto free_obj;
+	}
+	code = obj->integer.value;
+
+	if (!sparse_keymap_report_event(priv->input_dev, code, 1, true))
+		dev_warn(&wdev->dev, "Unknown key %d pressed\n", code);
+
+free_obj:
+	kfree(obj);
+	lenovo_ymc_trigger_ec(wdev, priv);
+}
+
+static void acpi_dev_put_helper(void *p) { acpi_dev_put(p); }
+
+static int lenovo_ymc_probe(struct wmi_device *wdev, const void *ctx)
+{
+	struct lenovo_ymc_private *priv;
+	struct input_dev *input_dev;
+	int err;
+
+	ec_trigger |= dmi_check_system(ec_trigger_quirk_dmi_table);
+
+	priv = devm_kzalloc(&wdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	if (ec_trigger) {
+		pr_debug("Lenovo YMC enable EC triggering.\n");
+		priv->ec_acpi_dev = acpi_dev_get_first_match_dev("VPC2004", NULL, -1);
+
+		if (!priv->ec_acpi_dev) {
+			dev_err(&wdev->dev, "Could not find EC ACPI device.\n");
+			return -ENODEV;
+		}
+		err = devm_add_action_or_reset(&wdev->dev,
+				acpi_dev_put_helper, priv->ec_acpi_dev);
+		if (err) {
+			dev_err(&wdev->dev,
+				"Could not clean up EC ACPI device: %d\n", err);
+			return err;
+		}
+	}
+
+	input_dev = devm_input_allocate_device(&wdev->dev);
+	if (!input_dev)
+		return -ENOMEM;
+
+	input_dev->name = "Lenovo Yoga Tablet Mode Control switch";
+	input_dev->phys = LENOVO_YMC_EVENT_GUID "/input0";
+	input_dev->id.bustype = BUS_HOST;
+	input_dev->dev.parent = &wdev->dev;
+	err = sparse_keymap_setup(input_dev, lenovo_ymc_keymap, NULL);
+	if (err) {
+		dev_err(&wdev->dev,
+			"Could not set up input device keymap: %d\n", err);
+		return err;
+	}
+
+	err = input_register_device(input_dev);
+	if (err) {
+		dev_err(&wdev->dev,
+			"Could not register input device: %d\n", err);
+		return err;
+	}
+
+	priv->input_dev = input_dev;
+	dev_set_drvdata(&wdev->dev, priv);
+
+	/* Report the state for the first time on probe */
+	lenovo_ymc_trigger_ec(wdev, priv);
+	lenovo_ymc_notify(wdev, NULL);
+	return 0;
+}
+
+static const struct wmi_device_id lenovo_ymc_wmi_id_table[] = {
+	{ .guid_string = LENOVO_YMC_EVENT_GUID },
+	{ }
+};
+MODULE_DEVICE_TABLE(wmi, lenovo_ymc_wmi_id_table);
+
+static struct wmi_driver lenovo_ymc_driver = {
+	.driver = {
+		.name = "lenovo-ymc",
+	},
+	.id_table = lenovo_ymc_wmi_id_table,
+	.probe = lenovo_ymc_probe,
+	.notify = lenovo_ymc_notify,
+};
+
+module_wmi_driver(lenovo_ymc_driver);
+
+MODULE_AUTHOR("Gergo Koteles <soyer@irl.hu>");
+MODULE_DESCRIPTION("Lenovo Yoga Mode Control driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/msi-ec.c b/drivers/platform/x86/msi-ec.c
new file mode 100644
index 000000000000..ff93986e3d35
--- /dev/null
+++ b/drivers/platform/x86/msi-ec.c
@@ -0,0 +1,897 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+/*
+ * msi-ec: MSI laptops' embedded controller driver.
+ *
+ * This driver allows various MSI laptops' functionalities to be
+ * controlled from userspace.
+ *
+ * It contains EC memory configurations for different firmware versions
+ * and exports battery charge thresholds to userspace.
+ *
+ * Copyright (C) 2023 Jose Angel Pastrana <japp0005@red.ujaen.es>
+ * Copyright (C) 2023 Aakash Singh <mail@singhaakash.dev>
+ * Copyright (C) 2023 Nikita Kravets <teackot@gmail.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include "msi-ec.h"
+
+#include <acpi/battery.h>
+#include <linux/acpi.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/seq_file.h>
+#include <linux/string.h>
+
+static const char *const SM_ECO_NAME       = "eco";
+static const char *const SM_COMFORT_NAME   = "comfort";
+static const char *const SM_SPORT_NAME     = "sport";
+static const char *const SM_TURBO_NAME     = "turbo";
+
+static const char *const FM_AUTO_NAME     = "auto";
+static const char *const FM_SILENT_NAME   = "silent";
+static const char *const FM_BASIC_NAME    = "basic";
+static const char *const FM_ADVANCED_NAME = "advanced";
+
+static const char * const ALLOWED_FW_0[] __initconst = {
+	"14C1EMS1.012",
+	"14C1EMS1.101",
+	"14C1EMS1.102",
+	NULL
+};
+
+static struct msi_ec_conf CONF0 __initdata = {
+	.allowed_fw = ALLOWED_FW_0,
+	.charge_control = {
+		.address      = 0xef,
+		.offset_start = 0x8a,
+		.offset_end   = 0x80,
+		.range_min    = 0x8a,
+		.range_max    = 0xe4,
+	},
+	.webcam = {
+		.address       = 0x2e,
+		.block_address = 0x2f,
+		.bit           = 1,
+	},
+	.fn_super_swap = {
+		.address = 0xbf,
+		.bit     = 4,
+	},
+	.cooler_boost = {
+		.address = 0x98,
+		.bit     = 7,
+	},
+	.shift_mode = {
+		.address = 0xf2,
+		.modes = {
+			{ SM_ECO_NAME,     0xc2 },
+			{ SM_COMFORT_NAME, 0xc1 },
+			{ SM_SPORT_NAME,   0xc0 },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.super_battery = {
+		.address = MSI_EC_ADDR_UNKNOWN, // 0xd5 needs testing
+	},
+	.fan_mode = {
+		.address = 0xf4,
+		.modes = {
+			{ FM_AUTO_NAME,     0x0d },
+			{ FM_SILENT_NAME,   0x1d },
+			{ FM_BASIC_NAME,    0x4d },
+			{ FM_ADVANCED_NAME, 0x8d },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.cpu = {
+		.rt_temp_address       = 0x68,
+		.rt_fan_speed_address  = 0x71,
+		.rt_fan_speed_base_min = 0x19,
+		.rt_fan_speed_base_max = 0x37,
+		.bs_fan_speed_address  = 0x89,
+		.bs_fan_speed_base_min = 0x00,
+		.bs_fan_speed_base_max = 0x0f,
+	},
+	.gpu = {
+		.rt_temp_address      = 0x80,
+		.rt_fan_speed_address = 0x89,
+	},
+	.leds = {
+		.micmute_led_address = 0x2b,
+		.mute_led_address    = 0x2c,
+		.bit                 = 2,
+	},
+	.kbd_bl = {
+		.bl_mode_address  = 0x2c, // ?
+		.bl_modes         = { 0x00, 0x08 }, // ?
+		.max_mode         = 1, // ?
+		.bl_state_address = 0xf3,
+		.state_base_value = 0x80,
+		.max_state        = 3,
+	},
+};
+
+static const char * const ALLOWED_FW_1[] __initconst = {
+	"17F2EMS1.103",
+	"17F2EMS1.104",
+	"17F2EMS1.106",
+	"17F2EMS1.107",
+	NULL
+};
+
+static struct msi_ec_conf CONF1 __initdata = {
+	.allowed_fw = ALLOWED_FW_1,
+	.charge_control = {
+		.address      = 0xef,
+		.offset_start = 0x8a,
+		.offset_end   = 0x80,
+		.range_min    = 0x8a,
+		.range_max    = 0xe4,
+	},
+	.webcam = {
+		.address       = 0x2e,
+		.block_address = 0x2f,
+		.bit           = 1,
+	},
+	.fn_super_swap = {
+		.address = 0xbf,
+		.bit     = 4,
+	},
+	.cooler_boost = {
+		.address = 0x98,
+		.bit     = 7,
+	},
+	.shift_mode = {
+		.address = 0xf2,
+		.modes = {
+			{ SM_ECO_NAME,     0xc2 },
+			{ SM_COMFORT_NAME, 0xc1 },
+			{ SM_SPORT_NAME,   0xc0 },
+			{ SM_TURBO_NAME,   0xc4 },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.super_battery = {
+		.address = MSI_EC_ADDR_UNKNOWN,
+	},
+	.fan_mode = {
+		.address = 0xf4,
+		.modes = {
+			{ FM_AUTO_NAME,     0x0d },
+			{ FM_BASIC_NAME,    0x4d },
+			{ FM_ADVANCED_NAME, 0x8d },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.cpu = {
+		.rt_temp_address       = 0x68,
+		.rt_fan_speed_address  = 0x71,
+		.rt_fan_speed_base_min = 0x19,
+		.rt_fan_speed_base_max = 0x37,
+		.bs_fan_speed_address  = 0x89,
+		.bs_fan_speed_base_min = 0x00,
+		.bs_fan_speed_base_max = 0x0f,
+	},
+	.gpu = {
+		.rt_temp_address      = 0x80,
+		.rt_fan_speed_address = 0x89,
+	},
+	.leds = {
+		.micmute_led_address = 0x2b,
+		.mute_led_address    = 0x2c,
+		.bit                 = 2,
+	},
+	.kbd_bl = {
+		.bl_mode_address  = 0x2c, // ?
+		.bl_modes         = { 0x00, 0x08 }, // ?
+		.max_mode         = 1, // ?
+		.bl_state_address = 0xf3,
+		.state_base_value = 0x80,
+		.max_state        = 3,
+	},
+};
+
+static const char * const ALLOWED_FW_2[] __initconst = {
+	"1552EMS1.118",
+	NULL
+};
+
+static struct msi_ec_conf CONF2 __initdata = {
+	.allowed_fw = ALLOWED_FW_2,
+	.charge_control = {
+		.address      = 0xd7,
+		.offset_start = 0x8a,
+		.offset_end   = 0x80,
+		.range_min    = 0x8a,
+		.range_max    = 0xe4,
+	},
+	.webcam = {
+		.address       = 0x2e,
+		.block_address = 0x2f,
+		.bit           = 1,
+	},
+	.fn_super_swap = {
+		.address = 0xe8,
+		.bit     = 4,
+	},
+	.cooler_boost = {
+		.address = 0x98,
+		.bit     = 7,
+	},
+	.shift_mode = {
+		.address = 0xf2,
+		.modes = {
+			{ SM_ECO_NAME,     0xc2 },
+			{ SM_COMFORT_NAME, 0xc1 },
+			{ SM_SPORT_NAME,   0xc0 },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.super_battery = {
+		.address = 0xeb,
+		.mask    = 0x0f,
+	},
+	.fan_mode = {
+		.address = 0xd4,
+		.modes = {
+			{ FM_AUTO_NAME,     0x0d },
+			{ FM_SILENT_NAME,   0x1d },
+			{ FM_BASIC_NAME,    0x4d },
+			{ FM_ADVANCED_NAME, 0x8d },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.cpu = {
+		.rt_temp_address       = 0x68,
+		.rt_fan_speed_address  = 0x71,
+		.rt_fan_speed_base_min = 0x19,
+		.rt_fan_speed_base_max = 0x37,
+		.bs_fan_speed_address  = 0x89,
+		.bs_fan_speed_base_min = 0x00,
+		.bs_fan_speed_base_max = 0x0f,
+	},
+	.gpu = {
+		.rt_temp_address      = 0x80,
+		.rt_fan_speed_address = 0x89,
+	},
+	.leds = {
+		.micmute_led_address = 0x2c,
+		.mute_led_address    = 0x2d,
+		.bit                 = 1,
+	},
+	.kbd_bl = {
+		.bl_mode_address  = 0x2c, // ?
+		.bl_modes         = { 0x00, 0x08 }, // ?
+		.max_mode         = 1, // ?
+		.bl_state_address = 0xd3,
+		.state_base_value = 0x80,
+		.max_state        = 3,
+	},
+};
+
+static const char * const ALLOWED_FW_3[] __initconst = {
+	"1592EMS1.111",
+	"E1592IMS.10C",
+	NULL
+};
+
+static struct msi_ec_conf CONF3 __initdata = {
+	.allowed_fw = ALLOWED_FW_3,
+	.charge_control = {
+		.address      = 0xef,
+		.offset_start = 0x8a,
+		.offset_end   = 0x80,
+		.range_min    = 0x8a,
+		.range_max    = 0xe4,
+	},
+	.webcam = {
+		.address       = 0x2e,
+		.block_address = 0x2f,
+		.bit           = 1,
+	},
+	.fn_super_swap = {
+		.address = 0xe8,
+		.bit     = 4,
+	},
+	.cooler_boost = {
+		.address = 0x98,
+		.bit     = 7,
+	},
+	.shift_mode = {
+		.address = 0xd2,
+		.modes = {
+			{ SM_ECO_NAME,     0xc2 },
+			{ SM_COMFORT_NAME, 0xc1 },
+			{ SM_SPORT_NAME,   0xc0 },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.super_battery = {
+		.address = 0xeb,
+		.mask    = 0x0f,
+	},
+	.fan_mode = {
+		.address = 0xd4,
+		.modes = {
+			{ FM_AUTO_NAME,     0x0d },
+			{ FM_SILENT_NAME,   0x1d },
+			{ FM_BASIC_NAME,    0x4d },
+			{ FM_ADVANCED_NAME, 0x8d },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.cpu = {
+		.rt_temp_address       = 0x68,
+		.rt_fan_speed_address  = 0xc9,
+		.rt_fan_speed_base_min = 0x19,
+		.rt_fan_speed_base_max = 0x37,
+		.bs_fan_speed_address  = 0x89, // ?
+		.bs_fan_speed_base_min = 0x00,
+		.bs_fan_speed_base_max = 0x0f,
+	},
+	.gpu = {
+		.rt_temp_address      = 0x80,
+		.rt_fan_speed_address = 0x89,
+	},
+	.leds = {
+		.micmute_led_address = 0x2b,
+		.mute_led_address    = 0x2c,
+		.bit                 = 1,
+	},
+	.kbd_bl = {
+		.bl_mode_address  = 0x2c, // ?
+		.bl_modes         = { 0x00, 0x08 }, // ?
+		.max_mode         = 1, // ?
+		.bl_state_address = 0xd3,
+		.state_base_value = 0x80,
+		.max_state        = 3,
+	},
+};
+
+static const char * const ALLOWED_FW_4[] __initconst = {
+	"16V4EMS1.114",
+	NULL
+};
+
+static struct msi_ec_conf CONF4 __initdata = {
+	.allowed_fw = ALLOWED_FW_4,
+	.charge_control = {
+		.address      = 0xd7,
+		.offset_start = 0x8a,
+		.offset_end   = 0x80,
+		.range_min    = 0x8a,
+		.range_max    = 0xe4,
+	},
+	.webcam = {
+		.address       = 0x2e,
+		.block_address = 0x2f,
+		.bit           = 1,
+	},
+	.fn_super_swap = {
+		.address = MSI_EC_ADDR_UNKNOWN, // supported, but unknown
+		.bit     = 4,
+	},
+	.cooler_boost = {
+		.address = 0x98,
+		.bit     = 7,
+	},
+	.shift_mode = {
+		.address = 0xd2,
+		.modes = {
+			{ SM_ECO_NAME,     0xc2 },
+			{ SM_COMFORT_NAME, 0xc1 },
+			{ SM_SPORT_NAME,   0xc0 },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.super_battery = { // may be supported, but address is unknown
+		.address = MSI_EC_ADDR_UNKNOWN,
+		.mask    = 0x0f,
+	},
+	.fan_mode = {
+		.address = 0xd4,
+		.modes = {
+			{ FM_AUTO_NAME,     0x0d },
+			{ FM_SILENT_NAME,   0x1d },
+			{ FM_ADVANCED_NAME, 0x8d },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.cpu = {
+		.rt_temp_address       = 0x68, // needs testing
+		.rt_fan_speed_address  = 0x71, // needs testing
+		.rt_fan_speed_base_min = 0x19,
+		.rt_fan_speed_base_max = 0x37,
+		.bs_fan_speed_address  = MSI_EC_ADDR_UNKNOWN,
+		.bs_fan_speed_base_min = 0x00,
+		.bs_fan_speed_base_max = 0x0f,
+	},
+	.gpu = {
+		.rt_temp_address      = 0x80,
+		.rt_fan_speed_address = MSI_EC_ADDR_UNKNOWN,
+	},
+	.leds = {
+		.micmute_led_address = MSI_EC_ADDR_UNKNOWN,
+		.mute_led_address    = MSI_EC_ADDR_UNKNOWN,
+		.bit                 = 1,
+	},
+	.kbd_bl = {
+		.bl_mode_address  = MSI_EC_ADDR_UNKNOWN, // ?
+		.bl_modes         = { 0x00, 0x08 }, // ?
+		.max_mode         = 1, // ?
+		.bl_state_address = MSI_EC_ADDR_UNSUPP, // 0xd3, not functional
+		.state_base_value = 0x80,
+		.max_state        = 3,
+	},
+};
+
+static const char * const ALLOWED_FW_5[] __initconst = {
+	"158LEMS1.103",
+	"158LEMS1.105",
+	"158LEMS1.106",
+	NULL
+};
+
+static struct msi_ec_conf CONF5 __initdata = {
+	.allowed_fw = ALLOWED_FW_5,
+	.charge_control = {
+		.address      = 0xef,
+		.offset_start = 0x8a,
+		.offset_end   = 0x80,
+		.range_min    = 0x8a,
+		.range_max    = 0xe4,
+	},
+	.webcam = {
+		.address       = 0x2e,
+		.block_address = 0x2f,
+		.bit           = 1,
+	},
+	.fn_super_swap = { // todo: reverse
+		.address = 0xbf,
+		.bit     = 4,
+	},
+	.cooler_boost = {
+		.address = 0x98,
+		.bit     = 7,
+	},
+	.shift_mode = {
+		.address = 0xf2,
+		.modes = {
+			{ SM_ECO_NAME,     0xc2 },
+			{ SM_COMFORT_NAME, 0xc1 },
+			{ SM_TURBO_NAME,   0xc4 },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.super_battery = { // unsupported?
+		.address = MSI_EC_ADDR_UNKNOWN,
+		.mask    = 0x0f,
+	},
+	.fan_mode = {
+		.address = 0xf4,
+		.modes = {
+			{ FM_AUTO_NAME,     0x0d },
+			{ FM_SILENT_NAME,   0x1d },
+			{ FM_ADVANCED_NAME, 0x8d },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.cpu = {
+		.rt_temp_address       = 0x68, // needs testing
+		.rt_fan_speed_address  = 0x71, // needs testing
+		.rt_fan_speed_base_min = 0x19,
+		.rt_fan_speed_base_max = 0x37,
+		.bs_fan_speed_address  = MSI_EC_ADDR_UNSUPP,
+		.bs_fan_speed_base_min = 0x00,
+		.bs_fan_speed_base_max = 0x0f,
+	},
+	.gpu = {
+		.rt_temp_address      = MSI_EC_ADDR_UNKNOWN,
+		.rt_fan_speed_address = MSI_EC_ADDR_UNKNOWN,
+	},
+	.leds = {
+		.micmute_led_address = 0x2b,
+		.mute_led_address    = 0x2c,
+		.bit                 = 2,
+	},
+	.kbd_bl = {
+		.bl_mode_address  = MSI_EC_ADDR_UNKNOWN, // ?
+		.bl_modes         = { 0x00, 0x08 }, // ?
+		.max_mode         = 1, // ?
+		.bl_state_address = MSI_EC_ADDR_UNSUPP, // 0xf3, not functional
+		.state_base_value = 0x80,
+		.max_state        = 3,
+	},
+};
+
+static const char * const ALLOWED_FW_6[] __initconst = {
+	"1542EMS1.102",
+	"1542EMS1.104",
+	NULL
+};
+
+static struct msi_ec_conf CONF6 __initdata = {
+	.allowed_fw = ALLOWED_FW_6,
+	.charge_control = {
+		.address      = 0xef,
+		.offset_start = 0x8a,
+		.offset_end   = 0x80,
+		.range_min    = 0x8a,
+		.range_max    = 0xe4,
+	},
+	.webcam = {
+		.address       = 0x2e,
+		.block_address = MSI_EC_ADDR_UNSUPP,
+		.bit           = 1,
+	},
+	.fn_super_swap = {
+		.address = 0xbf, // todo: reverse
+		.bit     = 4,
+	},
+	.cooler_boost = {
+		.address = 0x98,
+		.bit     = 7,
+	},
+	.shift_mode = {
+		.address = 0xf2,
+		.modes = {
+			{ SM_ECO_NAME,     0xc2 },
+			{ SM_COMFORT_NAME, 0xc1 },
+			{ SM_SPORT_NAME,   0xc0 },
+			{ SM_TURBO_NAME,   0xc4 },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.super_battery = {
+		.address = 0xd5,
+		.mask    = 0x0f,
+	},
+	.fan_mode = {
+		.address = 0xf4,
+		.modes = {
+			{ FM_AUTO_NAME,     0x0d },
+			{ FM_SILENT_NAME,   0x1d },
+			{ FM_ADVANCED_NAME, 0x8d },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.cpu = {
+		.rt_temp_address       = 0x68,
+		.rt_fan_speed_address  = 0xc9,
+		.rt_fan_speed_base_min = 0x19,
+		.rt_fan_speed_base_max = 0x37,
+		.bs_fan_speed_address  = MSI_EC_ADDR_UNSUPP,
+		.bs_fan_speed_base_min = 0x00,
+		.bs_fan_speed_base_max = 0x0f,
+	},
+	.gpu = {
+		.rt_temp_address      = 0x80,
+		.rt_fan_speed_address = MSI_EC_ADDR_UNKNOWN,
+	},
+	.leds = {
+		.micmute_led_address = MSI_EC_ADDR_UNSUPP,
+		.mute_led_address    = MSI_EC_ADDR_UNSUPP,
+		.bit                 = 2,
+	},
+	.kbd_bl = {
+		.bl_mode_address  = MSI_EC_ADDR_UNKNOWN, // ?
+		.bl_modes         = { 0x00, 0x08 }, // ?
+		.max_mode         = 1, // ?
+		.bl_state_address = MSI_EC_ADDR_UNSUPP, // 0xf3, not functional
+		.state_base_value = 0x80,
+		.max_state        = 3,
+	},
+};
+
+static const char * const ALLOWED_FW_7[] __initconst = {
+	"17FKEMS1.108",
+	"17FKEMS1.109",
+	"17FKEMS1.10A",
+	NULL
+};
+
+static struct msi_ec_conf CONF7 __initdata = {
+	.allowed_fw = ALLOWED_FW_7,
+	.charge_control = {
+		.address      = 0xef,
+		.offset_start = 0x8a,
+		.offset_end   = 0x80,
+		.range_min    = 0x8a,
+		.range_max    = 0xe4,
+	},
+	.webcam = {
+		.address       = 0x2e,
+		.block_address = MSI_EC_ADDR_UNSUPP,
+		.bit           = 1,
+	},
+	.fn_super_swap = {
+		.address = 0xbf, // needs testing
+		.bit     = 4,
+	},
+	.cooler_boost = {
+		.address = 0x98,
+		.bit     = 7,
+	},
+	.shift_mode = {
+		.address = 0xf2,
+		.modes = {
+			{ SM_ECO_NAME,     0xc2 },
+			{ SM_COMFORT_NAME, 0xc1 },
+			{ SM_SPORT_NAME,   0xc0 },
+			{ SM_TURBO_NAME,   0xc4 },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.super_battery = {
+		.address = MSI_EC_ADDR_UNKNOWN, // 0xd5 but has its own wet of modes
+		.mask    = 0x0f,
+	},
+	.fan_mode = {
+		.address = 0xf4,
+		.modes = {
+			{ FM_AUTO_NAME,     0x0d }, // d may not be relevant
+			{ FM_SILENT_NAME,   0x1d },
+			{ FM_ADVANCED_NAME, 0x8d },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.cpu = {
+		.rt_temp_address       = 0x68,
+		.rt_fan_speed_address  = 0xc9, // needs testing
+		.rt_fan_speed_base_min = 0x19,
+		.rt_fan_speed_base_max = 0x37,
+		.bs_fan_speed_address  = MSI_EC_ADDR_UNSUPP,
+		.bs_fan_speed_base_min = 0x00,
+		.bs_fan_speed_base_max = 0x0f,
+	},
+	.gpu = {
+		.rt_temp_address      = MSI_EC_ADDR_UNKNOWN,
+		.rt_fan_speed_address = MSI_EC_ADDR_UNKNOWN,
+	},
+	.leds = {
+		.micmute_led_address = MSI_EC_ADDR_UNSUPP,
+		.mute_led_address    = 0x2c,
+		.bit                 = 2,
+	},
+	.kbd_bl = {
+		.bl_mode_address  = MSI_EC_ADDR_UNKNOWN, // ?
+		.bl_modes         = { 0x00, 0x08 }, // ?
+		.max_mode         = 1, // ?
+		.bl_state_address = 0xf3,
+		.state_base_value = 0x80,
+		.max_state        = 3,
+	},
+};
+
+static struct msi_ec_conf *CONFIGS[] __initdata = {
+	&CONF0,
+	&CONF1,
+	&CONF2,
+	&CONF3,
+	&CONF4,
+	&CONF5,
+	&CONF6,
+	&CONF7,
+	NULL
+};
+
+static struct msi_ec_conf conf; // current configuration
+
+/*
+ * Helper functions
+ */
+
+static int ec_read_seq(u8 addr, u8 *buf, u8 len)
+{
+	int result;
+
+	for (u8 i = 0; i < len; i++) {
+		result = ec_read(addr + i, buf + i);
+		if (result < 0)
+			return result;
+	}
+
+	return 0;
+}
+
+static int ec_get_firmware_version(u8 buf[MSI_EC_FW_VERSION_LENGTH + 1])
+{
+	int result;
+
+	memset(buf, 0, MSI_EC_FW_VERSION_LENGTH + 1);
+	result = ec_read_seq(MSI_EC_FW_VERSION_ADDRESS,
+			     buf,
+			     MSI_EC_FW_VERSION_LENGTH);
+	if (result < 0)
+		return result;
+
+	return MSI_EC_FW_VERSION_LENGTH + 1;
+}
+
+/*
+ * Sysfs power_supply subsystem
+ */
+
+static ssize_t charge_control_threshold_show(u8 offset,
+					     struct device *device,
+					     struct device_attribute *attr,
+					     char *buf)
+{
+	u8 rdata;
+	int result;
+
+	result = ec_read(conf.charge_control.address, &rdata);
+	if (result < 0)
+		return result;
+
+	return sysfs_emit(buf, "%i\n", rdata - offset);
+}
+
+static ssize_t charge_control_threshold_store(u8 offset,
+					      struct device *dev,
+					      struct device_attribute *attr,
+					      const char *buf, size_t count)
+{
+	u8 wdata;
+	int result;
+
+	result = kstrtou8(buf, 10, &wdata);
+	if (result < 0)
+		return result;
+
+	wdata += offset;
+	if (wdata < conf.charge_control.range_min ||
+	    wdata > conf.charge_control.range_max)
+		return -EINVAL;
+
+	result = ec_write(conf.charge_control.address, wdata);
+	if (result < 0)
+		return result;
+
+	return count;
+}
+
+static ssize_t charge_control_start_threshold_show(struct device *device,
+						   struct device_attribute *attr,
+						   char *buf)
+{
+	return charge_control_threshold_show(conf.charge_control.offset_start,
+					     device, attr, buf);
+}
+
+static ssize_t charge_control_start_threshold_store(struct device *dev,
+						    struct device_attribute *attr,
+						    const char *buf, size_t count)
+{
+	return charge_control_threshold_store(conf.charge_control.offset_start,
+					      dev, attr, buf, count);
+}
+
+static ssize_t charge_control_end_threshold_show(struct device *device,
+						 struct device_attribute *attr,
+						 char *buf)
+{
+	return charge_control_threshold_show(conf.charge_control.offset_end,
+					     device, attr, buf);
+}
+
+static ssize_t charge_control_end_threshold_store(struct device *dev,
+						  struct device_attribute *attr,
+						  const char *buf, size_t count)
+{
+	return charge_control_threshold_store(conf.charge_control.offset_end,
+					      dev, attr, buf, count);
+}
+
+static DEVICE_ATTR_RW(charge_control_start_threshold);
+static DEVICE_ATTR_RW(charge_control_end_threshold);
+
+static struct attribute *msi_battery_attrs[] = {
+	&dev_attr_charge_control_start_threshold.attr,
+	&dev_attr_charge_control_end_threshold.attr,
+	NULL
+};
+
+ATTRIBUTE_GROUPS(msi_battery);
+
+static int msi_battery_add(struct power_supply *battery,
+			   struct acpi_battery_hook *hook)
+{
+	return device_add_groups(&battery->dev, msi_battery_groups);
+}
+
+static int msi_battery_remove(struct power_supply *battery,
+			      struct acpi_battery_hook *hook)
+{
+	device_remove_groups(&battery->dev, msi_battery_groups);
+	return 0;
+}
+
+static struct acpi_battery_hook battery_hook = {
+	.add_battery = msi_battery_add,
+	.remove_battery = msi_battery_remove,
+	.name = MSI_EC_DRIVER_NAME,
+};
+
+/*
+ * Module load/unload
+ */
+
+static const struct dmi_system_id msi_dmi_table[] __initconst __maybe_unused = {
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "MICRO-STAR INT"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Micro-Star International"),
+		},
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(dmi, msi_dmi_table);
+
+static int __init load_configuration(void)
+{
+	int result;
+
+	u8 fw_version[MSI_EC_FW_VERSION_LENGTH + 1];
+
+	/* get firmware version */
+	result = ec_get_firmware_version(fw_version);
+	if (result < 0)
+		return result;
+
+	/* load the suitable configuration, if exists */
+	for (int i = 0; CONFIGS[i]; i++) {
+		if (match_string(CONFIGS[i]->allowed_fw, -1, fw_version) != -EINVAL) {
+			conf = *CONFIGS[i];
+			conf.allowed_fw = NULL;
+			return 0;
+		}
+	}
+
+	/* config not found */
+
+	for (int i = 0; i < MSI_EC_FW_VERSION_LENGTH; i++) {
+		if (!isgraph(fw_version[i])) {
+			pr_warn("Unable to find a valid firmware version!\n");
+			return -EOPNOTSUPP;
+		}
+	}
+
+	pr_warn("Firmware version is not supported: '%s'\n", fw_version);
+	return -EOPNOTSUPP;
+}
+
+static int __init msi_ec_init(void)
+{
+	int result;
+
+	result = load_configuration();
+	if (result < 0)
+		return result;
+
+	battery_hook_register(&battery_hook);
+	return 0;
+}
+
+static void __exit msi_ec_exit(void)
+{
+	battery_hook_unregister(&battery_hook);
+}
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Jose Angel Pastrana <japp0005@red.ujaen.es>");
+MODULE_AUTHOR("Aakash Singh <mail@singhaakash.dev>");
+MODULE_AUTHOR("Nikita Kravets <teackot@gmail.com>");
+MODULE_DESCRIPTION("MSI Embedded Controller");
+
+module_init(msi_ec_init);
+module_exit(msi_ec_exit);
diff --git a/drivers/platform/x86/msi-ec.h b/drivers/platform/x86/msi-ec.h
new file mode 100644
index 000000000000..be3533dc9cc6
--- /dev/null
+++ b/drivers/platform/x86/msi-ec.h
@@ -0,0 +1,122 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+/*
+ * msi-ec: MSI laptops' embedded controller driver.
+ *
+ * Copyright (C) 2023 Jose Angel Pastrana <japp0005@red.ujaen.es>
+ * Copyright (C) 2023 Aakash Singh <mail@singhaakash.dev>
+ * Copyright (C) 2023 Nikita Kravets <teackot@gmail.com>
+ */
+
+#ifndef _MSI_EC_H_
+#define _MSI_EC_H_
+
+#include <linux/types.h>
+
+#define MSI_EC_DRIVER_NAME "msi-ec"
+
+#define MSI_EC_ADDR_UNKNOWN 0xff01 // unknown address
+#define MSI_EC_ADDR_UNSUPP  0xff01 // unsupported parameter
+
+// Firmware info addresses are universal
+#define MSI_EC_FW_VERSION_ADDRESS 0xa0
+#define MSI_EC_FW_DATE_ADDRESS    0xac
+#define MSI_EC_FW_TIME_ADDRESS    0xb4
+#define MSI_EC_FW_VERSION_LENGTH  12
+#define MSI_EC_FW_DATE_LENGTH     8
+#define MSI_EC_FW_TIME_LENGTH     8
+
+struct msi_ec_charge_control_conf {
+	int address;
+	int offset_start;
+	int offset_end;
+	int range_min;
+	int range_max;
+};
+
+struct msi_ec_webcam_conf {
+	int address;
+	int block_address;
+	int bit;
+};
+
+struct msi_ec_fn_super_swap_conf {
+	int address;
+	int bit;
+};
+
+struct msi_ec_cooler_boost_conf {
+	int address;
+	int bit;
+};
+
+#define MSI_EC_MODE_NULL { NULL, 0 }
+struct msi_ec_mode {
+	const char *name;
+	int value;
+};
+
+struct msi_ec_shift_mode_conf {
+	int address;
+	struct msi_ec_mode modes[5]; // fixed size for easier hard coding
+};
+
+struct msi_ec_super_battery_conf {
+	int address;
+	int mask;
+};
+
+struct msi_ec_fan_mode_conf {
+	int address;
+	struct msi_ec_mode modes[5]; // fixed size for easier hard coding
+};
+
+struct msi_ec_cpu_conf {
+	int rt_temp_address;
+	int rt_fan_speed_address; // realtime
+	int rt_fan_speed_base_min;
+	int rt_fan_speed_base_max;
+	int bs_fan_speed_address; // basic
+	int bs_fan_speed_base_min;
+	int bs_fan_speed_base_max;
+};
+
+struct msi_ec_gpu_conf {
+	int rt_temp_address;
+	int rt_fan_speed_address; // realtime
+};
+
+struct msi_ec_led_conf {
+	int micmute_led_address;
+	int mute_led_address;
+	int bit;
+};
+
+#define MSI_EC_KBD_BL_STATE_MASK 0x3
+struct msi_ec_kbd_bl_conf {
+	int bl_mode_address;
+	int bl_modes[2];
+	int max_mode;
+
+	int bl_state_address;
+	int state_base_value;
+	int max_state;
+};
+
+struct msi_ec_conf {
+	const char * const *allowed_fw;
+
+	struct msi_ec_charge_control_conf charge_control;
+	struct msi_ec_webcam_conf         webcam;
+	struct msi_ec_fn_super_swap_conf  fn_super_swap;
+	struct msi_ec_cooler_boost_conf   cooler_boost;
+	struct msi_ec_shift_mode_conf     shift_mode;
+	struct msi_ec_super_battery_conf  super_battery;
+	struct msi_ec_fan_mode_conf       fan_mode;
+	struct msi_ec_cpu_conf            cpu;
+	struct msi_ec_gpu_conf            gpu;
+	struct msi_ec_led_conf            leds;
+	struct msi_ec_kbd_bl_conf         kbd_bl;
+};
+
+#endif // _MSI_EC_H_
diff --git a/drivers/platform/x86/pcengines-apuv2.c b/drivers/platform/x86/pcengines-apuv2.c
index d063d91db9bc..3aa63b18a2e1 100644
--- a/drivers/platform/x86/pcengines-apuv2.c
+++ b/drivers/platform/x86/pcengines-apuv2.c
@@ -291,5 +291,4 @@ MODULE_AUTHOR("Enrico Weigelt, metux IT consult <info@metux.net>");
 MODULE_DESCRIPTION("PC Engines APUv2/APUv3 board GPIO/LEDs/keys driver");
 MODULE_LICENSE("GPL");
 MODULE_DEVICE_TABLE(dmi, apu_gpio_dmi_table);
-MODULE_ALIAS("platform:pcengines-apuv2");
 MODULE_SOFTDEP("pre: platform:" AMD_FCH_GPIO_DRIVER_NAME " platform:leds-gpio platform:gpio_keys_polled");
diff --git a/drivers/platform/x86/peaq-wmi.c b/drivers/platform/x86/peaq-wmi.c
deleted file mode 100644
index cf9c44c20a82..000000000000
--- a/drivers/platform/x86/peaq-wmi.c
+++ /dev/null
@@ -1,128 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * PEAQ 2-in-1 WMI hotkey driver
- * Copyright (C) 2017 Hans de Goede <hdegoede@redhat.com>
- */
-
-#include <linux/acpi.h>
-#include <linux/dmi.h>
-#include <linux/input.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-#define PEAQ_DOLBY_BUTTON_GUID		"ABBC0F6F-8EA1-11D1-00A0-C90629100000"
-#define PEAQ_DOLBY_BUTTON_METHOD_ID	5
-#define PEAQ_POLL_INTERVAL_MS		250
-#define PEAQ_POLL_IGNORE_MS		500
-#define PEAQ_POLL_MAX_MS		1000
-
-MODULE_ALIAS("wmi:"PEAQ_DOLBY_BUTTON_GUID);
-
-static struct input_dev *peaq_poll_dev;
-
-/*
- * The Dolby button (yes really a Dolby button) causes an ACPI variable to get
- * set on both press and release. The WMI method checks and clears that flag.
- * So for a press + release we will get back One from the WMI method either once
- * (if polling after the release) or twice (polling between press and release).
- * We ignore events for 0.5s after the first event to avoid reporting 2 presses.
- */
-static void peaq_wmi_poll(struct input_dev *input_dev)
-{
-	static unsigned long last_event_time;
-	static bool had_events;
-	union acpi_object obj;
-	acpi_status status;
-	u32 dummy = 0;
-
-	struct acpi_buffer input = { sizeof(dummy), &dummy };
-	struct acpi_buffer output = { sizeof(obj), &obj };
-
-	status = wmi_evaluate_method(PEAQ_DOLBY_BUTTON_GUID, 0,
-				     PEAQ_DOLBY_BUTTON_METHOD_ID,
-				     &input, &output);
-	if (ACPI_FAILURE(status))
-		return;
-
-	if (obj.type != ACPI_TYPE_INTEGER) {
-		dev_err(&input_dev->dev,
-			"Error WMBC did not return an integer\n");
-		return;
-	}
-
-	if (!obj.integer.value)
-		return;
-
-	if (had_events && time_before(jiffies, last_event_time +
-					msecs_to_jiffies(PEAQ_POLL_IGNORE_MS)))
-		return;
-
-	input_event(input_dev, EV_KEY, KEY_SOUND, 1);
-	input_sync(input_dev);
-	input_event(input_dev, EV_KEY, KEY_SOUND, 0);
-	input_sync(input_dev);
-
-	last_event_time = jiffies;
-	had_events = true;
-}
-
-/* Some other devices (Shuttle XS35) use the same WMI GUID for other purposes */
-static const struct dmi_system_id peaq_dmi_table[] __initconst = {
-	{
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "PEAQ"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "PEAQ PMM C1010 MD99187"),
-		},
-	},
-	{}
-};
-
-static int __init peaq_wmi_init(void)
-{
-	int err;
-
-	/* WMI GUID is not unique, also check for a DMI match */
-	if (!dmi_check_system(peaq_dmi_table))
-		return -ENODEV;
-
-	if (!wmi_has_guid(PEAQ_DOLBY_BUTTON_GUID))
-		return -ENODEV;
-
-	peaq_poll_dev = input_allocate_device();
-	if (!peaq_poll_dev)
-		return -ENOMEM;
-
-	peaq_poll_dev->name = "PEAQ WMI hotkeys";
-	peaq_poll_dev->phys = "wmi/input0";
-	peaq_poll_dev->id.bustype = BUS_HOST;
-	input_set_capability(peaq_poll_dev, EV_KEY, KEY_SOUND);
-
-	err = input_setup_polling(peaq_poll_dev, peaq_wmi_poll);
-	if (err)
-		goto err_out;
-
-	input_set_poll_interval(peaq_poll_dev, PEAQ_POLL_INTERVAL_MS);
-	input_set_max_poll_interval(peaq_poll_dev, PEAQ_POLL_MAX_MS);
-
-	err = input_register_device(peaq_poll_dev);
-	if (err)
-		goto err_out;
-
-	return 0;
-
-err_out:
-	input_free_device(peaq_poll_dev);
-	return err;
-}
-
-static void __exit peaq_wmi_exit(void)
-{
-	input_unregister_device(peaq_poll_dev);
-}
-
-module_init(peaq_wmi_init);
-module_exit(peaq_wmi_exit);
-
-MODULE_DESCRIPTION("PEAQ 2-in-1 WMI hotkey driver");
-MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/samsung-q10.c b/drivers/platform/x86/samsung-q10.c
index 6eb08b539311..134e2c3d91ca 100644
--- a/drivers/platform/x86/samsung-q10.c
+++ b/drivers/platform/x86/samsung-q10.c
@@ -65,14 +65,12 @@ static int samsungq10_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int samsungq10_remove(struct platform_device *pdev)
+static void samsungq10_remove(struct platform_device *pdev)
 {
 
 	struct backlight_device *bd = platform_get_drvdata(pdev);
 
 	backlight_device_unregister(bd);
-
-	return 0;
 }
 
 static struct platform_driver samsungq10_driver = {
@@ -80,7 +78,7 @@ static struct platform_driver samsungq10_driver = {
 		.name	= KBUILD_MODNAME,
 	},
 	.probe		= samsungq10_probe,
-	.remove		= samsungq10_remove,
+	.remove_new	= samsungq10_remove,
 };
 
 static struct platform_device *samsungq10_device;
diff --git a/drivers/platform/x86/serial-multi-instantiate.c b/drivers/platform/x86/serial-multi-instantiate.c
index 5362f1a7b77c..f3dcbdd72fec 100644
--- a/drivers/platform/x86/serial-multi-instantiate.c
+++ b/drivers/platform/x86/serial-multi-instantiate.c
@@ -139,7 +139,8 @@ static int smi_spi_probe(struct platform_device *pdev, struct smi *smi,
 			goto error;
 		}
 
-		dev_dbg(dev, "SPI device %s using chip select %u", name, spi_dev->chip_select);
+		dev_dbg(dev, "SPI device %s using chip select %u", name,
+			spi_get_chipselect(spi_dev, 0));
 
 		smi->spi_devs[i] = spi_dev;
 		smi->spi_num++;
@@ -265,13 +266,11 @@ static int smi_probe(struct platform_device *pdev)
 	}
 }
 
-static int smi_remove(struct platform_device *pdev)
+static void smi_remove(struct platform_device *pdev)
 {
 	struct smi *smi = platform_get_drvdata(pdev);
 
 	smi_devs_unregister(smi);
-
-	return 0;
 }
 
 static const struct smi_node bsg1160_data = {
@@ -339,7 +338,7 @@ static struct platform_driver smi_driver = {
 		.acpi_match_table = smi_acpi_ids,
 	},
 	.probe = smi_probe,
-	.remove = smi_remove,
+	.remove_new = smi_remove,
 };
 module_platform_driver(smi_driver);
 
diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c
index 537d6a2d0781..9569f11dec8c 100644
--- a/drivers/platform/x86/sony-laptop.c
+++ b/drivers/platform/x86/sony-laptop.c
@@ -3287,7 +3287,7 @@ static void sony_nc_remove(struct acpi_device *device)
 	dprintk(SONY_NC_DRIVER_NAME " removed.\n");
 }
 
-static const struct acpi_device_id sony_device_ids[] = {
+static const struct acpi_device_id sony_device_ids[] __maybe_unused = {
 	{SONY_NC_HID, 0},
 	{SONY_PIC_HID, 0},
 	{"", 0},
diff --git a/drivers/platform/x86/think-lmi.c b/drivers/platform/x86/think-lmi.c
index 78dc82bda4dd..1138f770149d 100644
--- a/drivers/platform/x86/think-lmi.c
+++ b/drivers/platform/x86/think-lmi.c
@@ -862,19 +862,18 @@ static umode_t auth_attr_is_visible(struct kobject *kobj,
 	struct tlmi_pwd_setting *setting = to_tlmi_pwd_setting(kobj);
 
 	/* We only want to display level and index settings on HDD/NVMe */
-	if ((attr == (struct attribute *)&auth_index) ||
-			(attr == (struct attribute *)&auth_level)) {
+	if (attr == &auth_index.attr || attr == &auth_level.attr) {
 		if ((setting == tlmi_priv.pwd_hdd) || (setting == tlmi_priv.pwd_nvme))
 			return attr->mode;
 		return 0;
 	}
 
 	/* We only display certificates on Admin account, if supported */
-	if ((attr == (struct attribute *)&auth_certificate) ||
-			(attr == (struct attribute *)&auth_signature) ||
-			(attr == (struct attribute *)&auth_save_signature) ||
-			(attr == (struct attribute *)&auth_cert_thumb) ||
-			(attr == (struct attribute *)&auth_cert_to_password)) {
+	if (attr == &auth_certificate.attr ||
+	    attr == &auth_signature.attr ||
+	    attr == &auth_save_signature.attr ||
+	    attr == &auth_cert_thumb.attr ||
+	    attr == &auth_cert_to_password.attr) {
 		if ((setting == tlmi_priv.pwd_admin) && tlmi_priv.certificate_support)
 			return attr->mode;
 		return 0;
@@ -1079,33 +1078,6 @@ static const struct attribute_group tlmi_attr_group = {
 	.attrs = tlmi_attrs,
 };
 
-static ssize_t tlmi_attr_show(struct kobject *kobj, struct attribute *attr,
-				    char *buf)
-{
-	struct kobj_attribute *kattr;
-
-	kattr = container_of(attr, struct kobj_attribute, attr);
-	if (kattr->show)
-		return kattr->show(kobj, kattr, buf);
-	return -EIO;
-}
-
-static ssize_t tlmi_attr_store(struct kobject *kobj, struct attribute *attr,
-				     const char *buf, size_t count)
-{
-	struct kobj_attribute *kattr;
-
-	kattr = container_of(attr, struct kobj_attribute, attr);
-	if (kattr->store)
-		return kattr->store(kobj, kattr, buf, count);
-	return -EIO;
-}
-
-static const struct sysfs_ops tlmi_kobj_sysfs_ops = {
-	.show	= tlmi_attr_show,
-	.store	= tlmi_attr_store,
-};
-
 static void tlmi_attr_setting_release(struct kobject *kobj)
 {
 	struct tlmi_attr_setting *setting = to_tlmi_attr_setting(kobj);
@@ -1123,12 +1095,12 @@ static void tlmi_pwd_setting_release(struct kobject *kobj)
 
 static const struct kobj_type tlmi_attr_setting_ktype = {
 	.release        = &tlmi_attr_setting_release,
-	.sysfs_ops	= &tlmi_kobj_sysfs_ops,
+	.sysfs_ops	= &kobj_sysfs_ops,
 };
 
 static const struct kobj_type tlmi_pwd_setting_ktype = {
 	.release        = &tlmi_pwd_setting_release,
-	.sysfs_ops	= &tlmi_kobj_sysfs_ops,
+	.sysfs_ops	= &kobj_sysfs_ops,
 };
 
 static ssize_t pending_reboot_show(struct kobject *kobj, struct kobj_attribute *attr,
@@ -1385,7 +1357,6 @@ static struct tlmi_pwd_setting *tlmi_create_auth(const char *pwd_type,
 
 static int tlmi_analyze(void)
 {
-	acpi_status status;
 	int i, ret;
 
 	if (wmi_has_guid(LENOVO_SET_BIOS_SETTINGS_GUID) &&
@@ -1422,8 +1393,8 @@ static int tlmi_analyze(void)
 		char *p;
 
 		tlmi_priv.setting[i] = NULL;
-		status = tlmi_setting(i, &item, LENOVO_BIOS_SETTING_GUID);
-		if (ACPI_FAILURE(status))
+		ret = tlmi_setting(i, &item, LENOVO_BIOS_SETTING_GUID);
+		if (ret)
 			break;
 		if (!item)
 			break;
diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c
index 7191ff2625b1..6fe82f805ea8 100644
--- a/drivers/platform/x86/thinkpad_acpi.c
+++ b/drivers/platform/x86/thinkpad_acpi.c
@@ -11699,6 +11699,7 @@ static int __init thinkpad_acpi_module_init(void)
 {
 	const struct dmi_system_id *dmi_id;
 	int ret, i;
+	acpi_object_type obj_type;
 
 	tpacpi_lifecycle = TPACPI_LIFE_INIT;
 
@@ -11724,6 +11725,21 @@ static int __init thinkpad_acpi_module_init(void)
 	TPACPI_ACPIHANDLE_INIT(ecrd);
 	TPACPI_ACPIHANDLE_INIT(ecwr);
 
+	/*
+	 * Quirk: in some models (e.g. X380 Yoga), an object named ECRD
+	 * exists, but it is a register, not a method.
+	 */
+	if (ecrd_handle) {
+		acpi_get_type(ecrd_handle, &obj_type);
+		if (obj_type != ACPI_TYPE_METHOD)
+			ecrd_handle = NULL;
+	}
+	if (ecwr_handle) {
+		acpi_get_type(ecwr_handle, &obj_type);
+		if (obj_type != ACPI_TYPE_METHOD)
+			ecwr_handle = NULL;
+	}
+
 	tpacpi_wq = create_singlethread_workqueue(TPACPI_WORKQUEUE_NAME);
 	if (!tpacpi_wq) {
 		thinkpad_acpi_module_exit();
diff --git a/drivers/platform/x86/wmi.c b/drivers/platform/x86/wmi.c
index 4fe7650dd014..d81319a502ef 100644
--- a/drivers/platform/x86/wmi.c
+++ b/drivers/platform/x86/wmi.c
@@ -1369,7 +1369,7 @@ static void acpi_wmi_notify_handler(acpi_handle handle, u32 event,
 		event, 0);
 }
 
-static int acpi_wmi_remove(struct platform_device *device)
+static void acpi_wmi_remove(struct platform_device *device)
 {
 	struct acpi_device *acpi_device = ACPI_COMPANION(&device->dev);
 
@@ -1379,8 +1379,6 @@ static int acpi_wmi_remove(struct platform_device *device)
 				ACPI_ADR_SPACE_EC, &acpi_wmi_ec_space_handler);
 	wmi_free_devices(acpi_device);
 	device_unregister(dev_get_drvdata(&device->dev));
-
-	return 0;
 }
 
 static int acpi_wmi_probe(struct platform_device *device)
@@ -1468,7 +1466,7 @@ static struct platform_driver acpi_wmi_driver = {
 		.acpi_match_table = wmi_device_ids,
 	},
 	.probe = acpi_wmi_probe,
-	.remove = acpi_wmi_remove,
+	.remove_new = acpi_wmi_remove,
 };
 
 static int __init acpi_wmi_init(void)
diff --git a/drivers/platform/x86/x86-android-tablets.c b/drivers/platform/x86/x86-android-tablets.c
deleted file mode 100644
index 111b007656fc..000000000000
--- a/drivers/platform/x86/x86-android-tablets.c
+++ /dev/null
@@ -1,1803 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * DMI based code to deal with broken DSDTs on X86 tablets which ship with
- * Android as (part of) the factory image. The factory kernels shipped on these
- * devices typically have a bunch of things hardcoded, rather than specified
- * in their DSDT.
- *
- * Copyright (C) 2021-2022 Hans de Goede <hdegoede@redhat.com>
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/acpi.h>
-#include <linux/dmi.h>
-#include <linux/efi.h>
-#include <linux/gpio_keys.h>
-#include <linux/gpio/consumer.h>
-#include <linux/gpio/driver.h>
-#include <linux/gpio/machine.h>
-#include <linux/i2c.h>
-#include <linux/input.h>
-#include <linux/irq.h>
-#include <linux/irqdomain.h>
-#include <linux/module.h>
-#include <linux/mod_devicetable.h>
-#include <linux/pinctrl/consumer.h>
-#include <linux/pinctrl/machine.h>
-#include <linux/platform_data/lp855x.h>
-#include <linux/platform_device.h>
-#include <linux/power/bq24190_charger.h>
-#include <linux/reboot.h>
-#include <linux/rmi.h>
-#include <linux/serdev.h>
-#include <linux/spi/spi.h>
-#include <linux/string.h>
-/* For gpio_get_desc() which is EXPORT_SYMBOL_GPL() */
-#include "../../gpio/gpiolib.h"
-#include "../../gpio/gpiolib-acpi.h"
-
-/*
- * Helper code to get Linux IRQ numbers given a description of the IRQ source
- * (either IOAPIC index, or GPIO chip name + pin-number).
- */
-enum x86_acpi_irq_type {
-	X86_ACPI_IRQ_TYPE_NONE,
-	X86_ACPI_IRQ_TYPE_APIC,
-	X86_ACPI_IRQ_TYPE_GPIOINT,
-	X86_ACPI_IRQ_TYPE_PMIC,
-};
-
-struct x86_acpi_irq_data {
-	char *chip;   /* GPIO chip label (GPIOINT) or PMIC ACPI path (PMIC) */
-	enum x86_acpi_irq_type type;
-	enum irq_domain_bus_token domain;
-	int index;
-	int trigger;  /* ACPI_EDGE_SENSITIVE / ACPI_LEVEL_SENSITIVE */
-	int polarity; /* ACPI_ACTIVE_HIGH / ACPI_ACTIVE_LOW / ACPI_ACTIVE_BOTH */
-};
-
-static int gpiochip_find_match_label(struct gpio_chip *gc, void *data)
-{
-	return gc->label && !strcmp(gc->label, data);
-}
-
-static int x86_android_tablet_get_gpiod(char *label, int pin, struct gpio_desc **desc)
-{
-	struct gpio_desc *gpiod;
-	struct gpio_chip *chip;
-
-	chip = gpiochip_find(label, gpiochip_find_match_label);
-	if (!chip) {
-		pr_err("error cannot find GPIO chip %s\n", label);
-		return -ENODEV;
-	}
-
-	gpiod = gpiochip_get_desc(chip, pin);
-	if (IS_ERR(gpiod)) {
-		pr_err("error %ld getting GPIO %s %d\n", PTR_ERR(gpiod), label, pin);
-		return PTR_ERR(gpiod);
-	}
-
-	*desc = gpiod;
-	return 0;
-}
-
-static int x86_acpi_irq_helper_get(const struct x86_acpi_irq_data *data)
-{
-	struct irq_fwspec fwspec = { };
-	struct irq_domain *domain;
-	struct acpi_device *adev;
-	struct gpio_desc *gpiod;
-	unsigned int irq_type;
-	acpi_handle handle;
-	acpi_status status;
-	int irq, ret;
-
-	switch (data->type) {
-	case X86_ACPI_IRQ_TYPE_APIC:
-		/*
-		 * The DSDT may already reference the GSI in a device skipped by
-		 * acpi_quirk_skip_i2c_client_enumeration(). Unregister the GSI
-		 * to avoid EBUSY errors in this case.
-		 */
-		acpi_unregister_gsi(data->index);
-		irq = acpi_register_gsi(NULL, data->index, data->trigger, data->polarity);
-		if (irq < 0)
-			pr_err("error %d getting APIC IRQ %d\n", irq, data->index);
-
-		return irq;
-	case X86_ACPI_IRQ_TYPE_GPIOINT:
-		/* Like acpi_dev_gpio_irq_get(), but without parsing ACPI resources */
-		ret = x86_android_tablet_get_gpiod(data->chip, data->index, &gpiod);
-		if (ret)
-			return ret;
-
-		irq = gpiod_to_irq(gpiod);
-		if (irq < 0) {
-			pr_err("error %d getting IRQ %s %d\n", irq, data->chip, data->index);
-			return irq;
-		}
-
-		irq_type = acpi_dev_get_irq_type(data->trigger, data->polarity);
-		if (irq_type != IRQ_TYPE_NONE && irq_type != irq_get_trigger_type(irq))
-			irq_set_irq_type(irq, irq_type);
-
-		return irq;
-	case X86_ACPI_IRQ_TYPE_PMIC:
-		status = acpi_get_handle(NULL, data->chip, &handle);
-		if (ACPI_FAILURE(status)) {
-			pr_err("error could not get %s handle\n", data->chip);
-			return -ENODEV;
-		}
-
-		adev = acpi_fetch_acpi_dev(handle);
-		if (!adev) {
-			pr_err("error could not get %s adev\n", data->chip);
-			return -ENODEV;
-		}
-
-		fwspec.fwnode = acpi_fwnode_handle(adev);
-		domain = irq_find_matching_fwspec(&fwspec, data->domain);
-		if (!domain) {
-			pr_err("error could not find IRQ domain for %s\n", data->chip);
-			return -ENODEV;
-		}
-
-		return irq_create_mapping(domain, data->index);
-	default:
-		return 0;
-	}
-}
-
-struct x86_i2c_client_info {
-	struct i2c_board_info board_info;
-	char *adapter_path;
-	struct x86_acpi_irq_data irq_data;
-};
-
-struct x86_serdev_info {
-	const char *ctrl_hid;
-	const char *ctrl_uid;
-	const char *ctrl_devname;
-	/*
-	 * ATM the serdev core only supports of or ACPI matching; and sofar all
-	 * Android x86 tablets DSDTs have usable serdev nodes, but sometimes
-	 * under the wrong controller. So we just tie the existing serdev ACPI
-	 * node to the right controller.
-	 */
-	const char *serdev_hid;
-};
-
-struct x86_dev_info {
-	char *invalid_aei_gpiochip;
-	const char * const *modules;
-	const struct software_node *bat_swnode;
-	struct gpiod_lookup_table * const *gpiod_lookup_tables;
-	const struct x86_i2c_client_info *i2c_client_info;
-	const struct platform_device_info *pdev_info;
-	const struct x86_serdev_info *serdev_info;
-	int i2c_client_count;
-	int pdev_count;
-	int serdev_count;
-	int (*init)(void);
-	void (*exit)(void);
-};
-
-/* Generic / shared charger / battery settings */
-static const char * const tusb1211_chg_det_psy[] = { "tusb1211-charger-detect" };
-static const char * const bq24190_psy[] = { "bq24190-charger" };
-static const char * const bq25890_psy[] = { "bq25890-charger-0" };
-
-static const struct property_entry fg_bq24190_supply_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", bq24190_psy),
-	{ }
-};
-
-static const struct software_node fg_bq24190_supply_node = {
-	.properties = fg_bq24190_supply_props,
-};
-
-static const struct property_entry fg_bq25890_supply_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", bq25890_psy),
-	{ }
-};
-
-static const struct software_node fg_bq25890_supply_node = {
-	.properties = fg_bq25890_supply_props,
-};
-
-/* LiPo HighVoltage (max 4.35V) settings used by most devs with a HV bat. */
-static const struct property_entry generic_lipo_hv_4v35_battery_props[] = {
-	PROPERTY_ENTRY_STRING("compatible", "simple-battery"),
-	PROPERTY_ENTRY_STRING("device-chemistry", "lithium-ion"),
-	PROPERTY_ENTRY_U32("precharge-current-microamp", 256000),
-	PROPERTY_ENTRY_U32("charge-term-current-microamp", 128000),
-	PROPERTY_ENTRY_U32("constant-charge-current-max-microamp", 1856000),
-	PROPERTY_ENTRY_U32("constant-charge-voltage-max-microvolt", 4352000),
-	PROPERTY_ENTRY_U32("factory-internal-resistance-micro-ohms", 150000),
-	{ }
-};
-
-static const struct software_node generic_lipo_hv_4v35_battery_node = {
-	.properties = generic_lipo_hv_4v35_battery_props,
-};
-
-/* For enabling the bq24190 5V boost based on id-pin */
-static struct regulator_consumer_supply intel_int3496_consumer = {
-	.supply = "vbus",
-	.dev_name = "intel-int3496",
-};
-
-static const struct regulator_init_data bq24190_vbus_init_data = {
-	.constraints = {
-		.name = "bq24190_vbus",
-		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
-	},
-	.consumer_supplies = &intel_int3496_consumer,
-	.num_consumer_supplies = 1,
-};
-
-static struct bq24190_platform_data bq24190_pdata = {
-	.regulator_init_data = &bq24190_vbus_init_data,
-};
-
-static const char * const bq24190_modules[] __initconst = {
-	"intel_crystal_cove_charger", /* For the bq24190 IRQ */
-	"bq24190_charger",            /* For the Vbus regulator for intel-int3496 */
-	NULL
-};
-
-/* Generic pdevs array and gpio-lookups for micro USB ID pin handling */
-static const struct platform_device_info int3496_pdevs[] __initconst = {
-	{
-		/* For micro USB ID pin handling */
-		.name = "intel-int3496",
-		.id = PLATFORM_DEVID_NONE,
-	},
-};
-
-static struct gpiod_lookup_table int3496_gpo2_pin22_gpios = {
-	.dev_id = "intel-int3496",
-	.table = {
-		GPIO_LOOKUP("INT33FC:02", 22, "id", GPIO_ACTIVE_HIGH),
-		{ }
-	},
-};
-
-/*
- * Advantech MICA-071
- * This is a standard Windows tablet, but it has an extra "quick launch" button
- * which is not described in the ACPI tables in anyway.
- * Use the x86-android-tablets infra to create a gpio-button device for this.
- */
-static struct gpio_keys_button advantech_mica_071_button = {
-	.code = KEY_PROG1,
-	/* .gpio gets filled in by advantech_mica_071_init() */
-	.active_low = true,
-	.desc = "prog1_key",
-	.type = EV_KEY,
-	.wakeup = false,
-	.debounce_interval = 50,
-};
-
-static const struct gpio_keys_platform_data advantech_mica_071_button_pdata __initconst = {
-	.buttons = &advantech_mica_071_button,
-	.nbuttons = 1,
-	.name = "prog1_key",
-};
-
-static const struct platform_device_info advantech_mica_071_pdevs[] __initconst = {
-	{
-		.name = "gpio-keys",
-		.id = PLATFORM_DEVID_AUTO,
-		.data = &advantech_mica_071_button_pdata,
-		.size_data = sizeof(advantech_mica_071_button_pdata),
-	},
-};
-
-static int __init advantech_mica_071_init(void)
-{
-	struct gpio_desc *gpiod;
-	int ret;
-
-	ret = x86_android_tablet_get_gpiod("INT33FC:00", 2, &gpiod);
-	if (ret < 0)
-		return ret;
-	advantech_mica_071_button.gpio = desc_to_gpio(gpiod);
-
-	return 0;
-}
-
-static const struct x86_dev_info advantech_mica_071_info __initconst = {
-	.pdev_info = advantech_mica_071_pdevs,
-	.pdev_count = ARRAY_SIZE(advantech_mica_071_pdevs),
-	.init = advantech_mica_071_init,
-};
-
-/* Asus ME176C and TF103C tablets shared data */
-static struct gpio_keys_button asus_me176c_tf103c_lid = {
-	.code = SW_LID,
-	/* .gpio gets filled in by asus_me176c_tf103c_init() */
-	.active_low = true,
-	.desc = "lid_sw",
-	.type = EV_SW,
-	.wakeup = true,
-	.debounce_interval = 50,
-};
-
-static const struct gpio_keys_platform_data asus_me176c_tf103c_lid_pdata __initconst = {
-	.buttons = &asus_me176c_tf103c_lid,
-	.nbuttons = 1,
-	.name = "lid_sw",
-};
-
-static const struct platform_device_info asus_me176c_tf103c_pdevs[] __initconst = {
-	{
-		.name = "gpio-keys",
-		.id = PLATFORM_DEVID_AUTO,
-		.data = &asus_me176c_tf103c_lid_pdata,
-		.size_data = sizeof(asus_me176c_tf103c_lid_pdata),
-	},
-	{
-		/* For micro USB ID pin handling */
-		.name = "intel-int3496",
-		.id = PLATFORM_DEVID_NONE,
-	},
-};
-
-static int __init asus_me176c_tf103c_init(void)
-{
-	struct gpio_desc *gpiod;
-	int ret;
-
-	ret = x86_android_tablet_get_gpiod("INT33FC:02", 12, &gpiod);
-	if (ret < 0)
-		return ret;
-	asus_me176c_tf103c_lid.gpio = desc_to_gpio(gpiod);
-
-	return 0;
-}
-
-
-/* Asus ME176C tablets have an Android factory img with everything hardcoded */
-static const char * const asus_me176c_accel_mount_matrix[] = {
-	"-1", "0", "0",
-	"0", "1", "0",
-	"0", "0", "1"
-};
-
-static const struct property_entry asus_me176c_accel_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", asus_me176c_accel_mount_matrix),
-	{ }
-};
-
-static const struct software_node asus_me176c_accel_node = {
-	.properties = asus_me176c_accel_props,
-};
-
-static const struct property_entry asus_me176c_bq24190_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", tusb1211_chg_det_psy),
-	PROPERTY_ENTRY_REF("monitored-battery", &generic_lipo_hv_4v35_battery_node),
-	PROPERTY_ENTRY_U32("ti,system-minimum-microvolt", 3600000),
-	PROPERTY_ENTRY_BOOL("omit-battery-class"),
-	PROPERTY_ENTRY_BOOL("disable-reset"),
-	{ }
-};
-
-static const struct software_node asus_me176c_bq24190_node = {
-	.properties = asus_me176c_bq24190_props,
-};
-
-static const struct property_entry asus_me176c_ug3105_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", bq24190_psy),
-	PROPERTY_ENTRY_REF("monitored-battery", &generic_lipo_hv_4v35_battery_node),
-	PROPERTY_ENTRY_U32("upisemi,rsns-microohm", 10000),
-	{ }
-};
-
-static const struct software_node asus_me176c_ug3105_node = {
-	.properties = asus_me176c_ug3105_props,
-};
-
-static const struct x86_i2c_client_info asus_me176c_i2c_clients[] __initconst = {
-	{
-		/* bq24297 battery charger */
-		.board_info = {
-			.type = "bq24190",
-			.addr = 0x6b,
-			.dev_name = "bq24297",
-			.swnode = &asus_me176c_bq24190_node,
-			.platform_data = &bq24190_pdata,
-		},
-		.adapter_path = "\\_SB_.I2C1",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_PMIC,
-			.chip = "\\_SB_.I2C7.PMIC",
-			.domain = DOMAIN_BUS_WAKEUP,
-			.index = 0,
-		},
-	}, {
-		/* ug3105 battery monitor */
-		.board_info = {
-			.type = "ug3105",
-			.addr = 0x70,
-			.dev_name = "ug3105",
-			.swnode = &asus_me176c_ug3105_node,
-		},
-		.adapter_path = "\\_SB_.I2C1",
-	}, {
-		/* ak09911 compass */
-		.board_info = {
-			.type = "ak09911",
-			.addr = 0x0c,
-			.dev_name = "ak09911",
-		},
-		.adapter_path = "\\_SB_.I2C5",
-	}, {
-		/* kxtj21009 accel */
-		.board_info = {
-			.type = "kxtj21009",
-			.addr = 0x0f,
-			.dev_name = "kxtj21009",
-			.swnode = &asus_me176c_accel_node,
-		},
-		.adapter_path = "\\_SB_.I2C5",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_APIC,
-			.index = 0x44,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_LOW,
-		},
-	}, {
-		/* goodix touchscreen */
-		.board_info = {
-			.type = "GDIX1001:00",
-			.addr = 0x14,
-			.dev_name = "goodix_ts",
-		},
-		.adapter_path = "\\_SB_.I2C6",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_APIC,
-			.index = 0x45,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_LOW,
-		},
-	},
-};
-
-static const struct x86_serdev_info asus_me176c_serdevs[] __initconst = {
-	{
-		.ctrl_hid = "80860F0A",
-		.ctrl_uid = "2",
-		.ctrl_devname = "serial0",
-		.serdev_hid = "BCM2E3A",
-	},
-};
-
-static struct gpiod_lookup_table asus_me176c_goodix_gpios = {
-	.dev_id = "i2c-goodix_ts",
-	.table = {
-		GPIO_LOOKUP("INT33FC:00", 60, "reset", GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP("INT33FC:02", 28, "irq", GPIO_ACTIVE_HIGH),
-		{ }
-	},
-};
-
-static struct gpiod_lookup_table * const asus_me176c_gpios[] = {
-	&int3496_gpo2_pin22_gpios,
-	&asus_me176c_goodix_gpios,
-	NULL
-};
-
-static const struct x86_dev_info asus_me176c_info __initconst = {
-	.i2c_client_info = asus_me176c_i2c_clients,
-	.i2c_client_count = ARRAY_SIZE(asus_me176c_i2c_clients),
-	.pdev_info = asus_me176c_tf103c_pdevs,
-	.pdev_count = ARRAY_SIZE(asus_me176c_tf103c_pdevs),
-	.serdev_info = asus_me176c_serdevs,
-	.serdev_count = ARRAY_SIZE(asus_me176c_serdevs),
-	.gpiod_lookup_tables = asus_me176c_gpios,
-	.bat_swnode = &generic_lipo_hv_4v35_battery_node,
-	.modules = bq24190_modules,
-	.invalid_aei_gpiochip = "INT33FC:02",
-	.init = asus_me176c_tf103c_init,
-};
-
-/* Asus TF103C tablets have an Android factory img with everything hardcoded */
-static const char * const asus_tf103c_accel_mount_matrix[] = {
-	"0", "-1", "0",
-	"-1", "0", "0",
-	"0", "0", "1"
-};
-
-static const struct property_entry asus_tf103c_accel_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", asus_tf103c_accel_mount_matrix),
-	{ }
-};
-
-static const struct software_node asus_tf103c_accel_node = {
-	.properties = asus_tf103c_accel_props,
-};
-
-static const struct property_entry asus_tf103c_touchscreen_props[] = {
-	PROPERTY_ENTRY_STRING("compatible", "atmel,atmel_mxt_ts"),
-	{ }
-};
-
-static const struct software_node asus_tf103c_touchscreen_node = {
-	.properties = asus_tf103c_touchscreen_props,
-};
-
-static const struct property_entry asus_tf103c_battery_props[] = {
-	PROPERTY_ENTRY_STRING("compatible", "simple-battery"),
-	PROPERTY_ENTRY_STRING("device-chemistry", "lithium-ion-polymer"),
-	PROPERTY_ENTRY_U32("precharge-current-microamp", 256000),
-	PROPERTY_ENTRY_U32("charge-term-current-microamp", 128000),
-	PROPERTY_ENTRY_U32("constant-charge-current-max-microamp", 2048000),
-	PROPERTY_ENTRY_U32("constant-charge-voltage-max-microvolt", 4208000),
-	PROPERTY_ENTRY_U32("factory-internal-resistance-micro-ohms", 150000),
-	{ }
-};
-
-static const struct software_node asus_tf103c_battery_node = {
-	.properties = asus_tf103c_battery_props,
-};
-
-static const struct property_entry asus_tf103c_bq24190_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", tusb1211_chg_det_psy),
-	PROPERTY_ENTRY_REF("monitored-battery", &asus_tf103c_battery_node),
-	PROPERTY_ENTRY_U32("ti,system-minimum-microvolt", 3600000),
-	PROPERTY_ENTRY_BOOL("omit-battery-class"),
-	PROPERTY_ENTRY_BOOL("disable-reset"),
-	{ }
-};
-
-static const struct software_node asus_tf103c_bq24190_node = {
-	.properties = asus_tf103c_bq24190_props,
-};
-
-static const struct property_entry asus_tf103c_ug3105_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", bq24190_psy),
-	PROPERTY_ENTRY_REF("monitored-battery", &asus_tf103c_battery_node),
-	PROPERTY_ENTRY_U32("upisemi,rsns-microohm", 5000),
-	{ }
-};
-
-static const struct software_node asus_tf103c_ug3105_node = {
-	.properties = asus_tf103c_ug3105_props,
-};
-
-static const struct x86_i2c_client_info asus_tf103c_i2c_clients[] __initconst = {
-	{
-		/* bq24297 battery charger */
-		.board_info = {
-			.type = "bq24190",
-			.addr = 0x6b,
-			.dev_name = "bq24297",
-			.swnode = &asus_tf103c_bq24190_node,
-			.platform_data = &bq24190_pdata,
-		},
-		.adapter_path = "\\_SB_.I2C1",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_PMIC,
-			.chip = "\\_SB_.I2C7.PMIC",
-			.domain = DOMAIN_BUS_WAKEUP,
-			.index = 0,
-		},
-	}, {
-		/* ug3105 battery monitor */
-		.board_info = {
-			.type = "ug3105",
-			.addr = 0x70,
-			.dev_name = "ug3105",
-			.swnode = &asus_tf103c_ug3105_node,
-		},
-		.adapter_path = "\\_SB_.I2C1",
-	}, {
-		/* ak09911 compass */
-		.board_info = {
-			.type = "ak09911",
-			.addr = 0x0c,
-			.dev_name = "ak09911",
-		},
-		.adapter_path = "\\_SB_.I2C5",
-	}, {
-		/* kxtj21009 accel */
-		.board_info = {
-			.type = "kxtj21009",
-			.addr = 0x0f,
-			.dev_name = "kxtj21009",
-			.swnode = &asus_tf103c_accel_node,
-		},
-		.adapter_path = "\\_SB_.I2C5",
-	}, {
-		/* atmel touchscreen */
-		.board_info = {
-			.type = "atmel_mxt_ts",
-			.addr = 0x4a,
-			.dev_name = "atmel_mxt_ts",
-			.swnode = &asus_tf103c_touchscreen_node,
-		},
-		.adapter_path = "\\_SB_.I2C6",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
-			.chip = "INT33FC:02",
-			.index = 28,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_LOW,
-		},
-	},
-};
-
-static struct gpiod_lookup_table * const asus_tf103c_gpios[] = {
-	&int3496_gpo2_pin22_gpios,
-	NULL
-};
-
-static const struct x86_dev_info asus_tf103c_info __initconst = {
-	.i2c_client_info = asus_tf103c_i2c_clients,
-	.i2c_client_count = ARRAY_SIZE(asus_tf103c_i2c_clients),
-	.pdev_info = asus_me176c_tf103c_pdevs,
-	.pdev_count = ARRAY_SIZE(asus_me176c_tf103c_pdevs),
-	.gpiod_lookup_tables = asus_tf103c_gpios,
-	.bat_swnode = &asus_tf103c_battery_node,
-	.modules = bq24190_modules,
-	.invalid_aei_gpiochip = "INT33FC:02",
-	.init = asus_me176c_tf103c_init,
-};
-
-/*
- * When booted with the BIOS set to Android mode the Chuwi Hi8 (CWI509) DSDT
- * contains a whole bunch of bogus ACPI I2C devices and is missing entries
- * for the touchscreen and the accelerometer.
- */
-static const struct property_entry chuwi_hi8_gsl1680_props[] = {
-	PROPERTY_ENTRY_U32("touchscreen-size-x", 1665),
-	PROPERTY_ENTRY_U32("touchscreen-size-y", 1140),
-	PROPERTY_ENTRY_BOOL("touchscreen-swapped-x-y"),
-	PROPERTY_ENTRY_BOOL("silead,home-button"),
-	PROPERTY_ENTRY_STRING("firmware-name", "gsl1680-chuwi-hi8.fw"),
-	{ }
-};
-
-static const struct software_node chuwi_hi8_gsl1680_node = {
-	.properties = chuwi_hi8_gsl1680_props,
-};
-
-static const char * const chuwi_hi8_mount_matrix[] = {
-	"1", "0", "0",
-	"0", "-1", "0",
-	"0", "0", "1"
-};
-
-static const struct property_entry chuwi_hi8_bma250e_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", chuwi_hi8_mount_matrix),
-	{ }
-};
-
-static const struct software_node chuwi_hi8_bma250e_node = {
-	.properties = chuwi_hi8_bma250e_props,
-};
-
-static const struct x86_i2c_client_info chuwi_hi8_i2c_clients[] __initconst = {
-	{
-		/* Silead touchscreen */
-		.board_info = {
-			.type = "gsl1680",
-			.addr = 0x40,
-			.swnode = &chuwi_hi8_gsl1680_node,
-		},
-		.adapter_path = "\\_SB_.I2C4",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_APIC,
-			.index = 0x44,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_HIGH,
-		},
-	}, {
-		/* BMA250E accelerometer */
-		.board_info = {
-			.type = "bma250e",
-			.addr = 0x18,
-			.swnode = &chuwi_hi8_bma250e_node,
-		},
-		.adapter_path = "\\_SB_.I2C3",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
-			.chip = "INT33FC:02",
-			.index = 23,
-			.trigger = ACPI_LEVEL_SENSITIVE,
-			.polarity = ACPI_ACTIVE_HIGH,
-		},
-	},
-};
-
-static int __init chuwi_hi8_init(void)
-{
-	/*
-	 * Avoid the acpi_unregister_gsi() call in x86_acpi_irq_helper_get()
-	 * breaking the touchscreen + logging various errors when the Windows
-	 * BIOS is used.
-	 */
-	if (acpi_dev_present("MSSL0001", NULL, 1))
-		return -ENODEV;
-
-	return 0;
-}
-
-static const struct x86_dev_info chuwi_hi8_info __initconst = {
-	.i2c_client_info = chuwi_hi8_i2c_clients,
-	.i2c_client_count = ARRAY_SIZE(chuwi_hi8_i2c_clients),
-	.init = chuwi_hi8_init,
-};
-
-#define CZC_EC_EXTRA_PORT	0x68
-#define CZC_EC_ANDROID_KEYS	0x63
-
-static int __init czc_p10t_init(void)
-{
-	/*
-	 * The device boots up in "Windows 7" mode, when the home button sends a
-	 * Windows specific key sequence (Left Meta + D) and the second button
-	 * sends an unknown one while also toggling the Radio Kill Switch.
-	 * This is a surprising behavior when the second button is labeled "Back".
-	 *
-	 * The vendor-supplied Android-x86 build switches the device to a "Android"
-	 * mode by writing value 0x63 to the I/O port 0x68. This just seems to just
-	 * set bit 6 on address 0x96 in the EC region; switching the bit directly
-	 * seems to achieve the same result. It uses a "p10t_switcher" to do the
-	 * job. It doesn't seem to be able to do anything else, and no other use
-	 * of the port 0x68 is known.
-	 *
-	 * In the Android mode, the home button sends just a single scancode,
-	 * which can be handled in Linux userspace more reasonably and the back
-	 * button only sends a scancode without toggling the kill switch.
-	 * The scancode can then be mapped either to Back or RF Kill functionality
-	 * in userspace, depending on how the button is labeled on that particular
-	 * model.
-	 */
-	outb(CZC_EC_ANDROID_KEYS, CZC_EC_EXTRA_PORT);
-	return 0;
-}
-
-static const struct x86_dev_info czc_p10t __initconst = {
-	.init = czc_p10t_init,
-};
-
-/* Lenovo Yoga Book X90F / X91F / X91L need manual instantiation of the fg client */
-static const struct x86_i2c_client_info lenovo_yogabook_x9x_i2c_clients[] __initconst = {
-	{
-		/* BQ27542 fuel-gauge */
-		.board_info = {
-			.type = "bq27542",
-			.addr = 0x55,
-			.dev_name = "bq27542",
-			.swnode = &fg_bq25890_supply_node,
-		},
-		.adapter_path = "\\_SB_.PCI0.I2C1",
-	},
-};
-
-static const struct x86_dev_info lenovo_yogabook_x9x_info __initconst = {
-	.i2c_client_info = lenovo_yogabook_x9x_i2c_clients,
-	.i2c_client_count = ARRAY_SIZE(lenovo_yogabook_x9x_i2c_clients),
-};
-
-/* Lenovo Yoga Tablet 2 1050F/L's Android factory img has everything hardcoded */
-static const struct property_entry lenovo_yoga_tab2_830_1050_bq24190_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", tusb1211_chg_det_psy),
-	PROPERTY_ENTRY_REF("monitored-battery", &generic_lipo_hv_4v35_battery_node),
-	PROPERTY_ENTRY_BOOL("omit-battery-class"),
-	PROPERTY_ENTRY_BOOL("disable-reset"),
-	{ }
-};
-
-static const struct software_node lenovo_yoga_tab2_830_1050_bq24190_node = {
-	.properties = lenovo_yoga_tab2_830_1050_bq24190_props,
-};
-
-/* This gets filled by lenovo_yoga_tab2_830_1050_init() */
-static struct rmi_device_platform_data lenovo_yoga_tab2_830_1050_rmi_pdata = { };
-
-static struct lp855x_platform_data lenovo_yoga_tab2_830_1050_lp8557_pdata = {
-	.device_control = 0x86,
-	.initial_brightness = 128,
-};
-
-static const struct x86_i2c_client_info lenovo_yoga_tab2_830_1050_i2c_clients[] __initconst = {
-	{
-		/* bq24292i battery charger */
-		.board_info = {
-			.type = "bq24190",
-			.addr = 0x6b,
-			.dev_name = "bq24292i",
-			.swnode = &lenovo_yoga_tab2_830_1050_bq24190_node,
-			.platform_data = &bq24190_pdata,
-		},
-		.adapter_path = "\\_SB_.I2C1",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
-			.chip = "INT33FC:02",
-			.index = 2,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_HIGH,
-		},
-	}, {
-		/* BQ27541 fuel-gauge */
-		.board_info = {
-			.type = "bq27541",
-			.addr = 0x55,
-			.dev_name = "bq27541",
-			.swnode = &fg_bq24190_supply_node,
-		},
-		.adapter_path = "\\_SB_.I2C1",
-	}, {
-		/* Synaptics RMI touchscreen */
-		.board_info = {
-			.type = "rmi4_i2c",
-			.addr = 0x38,
-			.dev_name = "rmi4_i2c",
-			.platform_data = &lenovo_yoga_tab2_830_1050_rmi_pdata,
-		},
-		.adapter_path = "\\_SB_.I2C6",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_APIC,
-			.index = 0x45,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_HIGH,
-		},
-	}, {
-		/* LP8557 Backlight controller */
-		.board_info = {
-			.type = "lp8557",
-			.addr = 0x2c,
-			.dev_name = "lp8557",
-			.platform_data = &lenovo_yoga_tab2_830_1050_lp8557_pdata,
-		},
-		.adapter_path = "\\_SB_.I2C3",
-	},
-};
-
-static struct gpiod_lookup_table lenovo_yoga_tab2_830_1050_int3496_gpios = {
-	.dev_id = "intel-int3496",
-	.table = {
-		GPIO_LOOKUP("INT33FC:02", 1, "mux", GPIO_ACTIVE_LOW),
-		GPIO_LOOKUP("INT33FC:02", 24, "id", GPIO_ACTIVE_HIGH),
-		{ }
-	},
-};
-
-#define LENOVO_YOGA_TAB2_830_1050_CODEC_NAME "spi-10WM5102:00"
-
-static struct gpiod_lookup_table lenovo_yoga_tab2_830_1050_codec_gpios = {
-	.dev_id = LENOVO_YOGA_TAB2_830_1050_CODEC_NAME,
-	.table = {
-		GPIO_LOOKUP("gpio_crystalcove", 3, "reset", GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP("INT33FC:01", 23, "wlf,ldoena", GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP("arizona", 2, "wlf,spkvdd-ena", GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP("arizona", 4, "wlf,micd-pol", GPIO_ACTIVE_LOW),
-		{ }
-	},
-};
-
-static struct gpiod_lookup_table * const lenovo_yoga_tab2_830_1050_gpios[] = {
-	&lenovo_yoga_tab2_830_1050_int3496_gpios,
-	&lenovo_yoga_tab2_830_1050_codec_gpios,
-	NULL
-};
-
-static int __init lenovo_yoga_tab2_830_1050_init(void);
-static void lenovo_yoga_tab2_830_1050_exit(void);
-
-static struct x86_dev_info lenovo_yoga_tab2_830_1050_info __initdata = {
-	.i2c_client_info = lenovo_yoga_tab2_830_1050_i2c_clients,
-	/* i2c_client_count gets set by lenovo_yoga_tab2_830_1050_init() */
-	.pdev_info = int3496_pdevs,
-	.pdev_count = ARRAY_SIZE(int3496_pdevs),
-	.gpiod_lookup_tables = lenovo_yoga_tab2_830_1050_gpios,
-	.bat_swnode = &generic_lipo_hv_4v35_battery_node,
-	.modules = bq24190_modules,
-	.invalid_aei_gpiochip = "INT33FC:02",
-	.init = lenovo_yoga_tab2_830_1050_init,
-	.exit = lenovo_yoga_tab2_830_1050_exit,
-};
-
-/*
- * The Lenovo Yoga Tablet 2 830 and 1050 (8" vs 10") versions use the same
- * mainboard, but they need some different treatment related to the display:
- * 1. The 830 uses a portrait LCD panel with a landscape touchscreen, requiring
- *    the touchscreen driver to adjust the touch-coords to match the LCD.
- * 2. Both use an TI LP8557 LED backlight controller. On the 1050 the LP8557's
- *    PWM input is connected to the PMIC's PWM output and everything works fine
- *    with the defaults programmed into the LP8557 by the BIOS.
- *    But on the 830 the LP8557's PWM input is connected to a PWM output coming
- *    from the LCD panel's controller. The Android code has a hack in the i915
- *    driver to write the non-standard DSI reg 0x9f with the desired backlight
- *    level to set the duty-cycle of the LCD's PWM output.
- *
- *    To avoid having to have a similar hack in the mainline kernel the LP8557
- *    entry in lenovo_yoga_tab2_830_1050_i2c_clients instead just programs the
- *    LP8557 to directly set the level, ignoring the PWM input. This means that
- *    the LP8557 i2c_client should only be instantiated on the 830.
- */
-static int __init lenovo_yoga_tab2_830_1050_init_display(void)
-{
-	struct gpio_desc *gpiod;
-	int ret;
-
-	/* Use PMIC GPIO 10 bootstrap pin to differentiate 830 vs 1050 */
-	ret = x86_android_tablet_get_gpiod("gpio_crystalcove", 10, &gpiod);
-	if (ret)
-		return ret;
-
-	ret = gpiod_get_value_cansleep(gpiod);
-	if (ret) {
-		pr_info("detected Lenovo Yoga Tablet 2 1050F/L\n");
-		lenovo_yoga_tab2_830_1050_info.i2c_client_count =
-			ARRAY_SIZE(lenovo_yoga_tab2_830_1050_i2c_clients) - 1;
-	} else {
-		pr_info("detected Lenovo Yoga Tablet 2 830F/L\n");
-		lenovo_yoga_tab2_830_1050_rmi_pdata.sensor_pdata.axis_align.swap_axes = true;
-		lenovo_yoga_tab2_830_1050_rmi_pdata.sensor_pdata.axis_align.flip_y = true;
-		lenovo_yoga_tab2_830_1050_info.i2c_client_count =
-			ARRAY_SIZE(lenovo_yoga_tab2_830_1050_i2c_clients);
-	}
-
-	return 0;
-}
-
-/* SUS (INT33FC:02) pin 6 needs to be configured as pmu_clk for the audio codec */
-static const struct pinctrl_map lenovo_yoga_tab2_830_1050_codec_pinctrl_map =
-	PIN_MAP_MUX_GROUP(LENOVO_YOGA_TAB2_830_1050_CODEC_NAME, "codec_32khz_clk",
-			  "INT33FC:02", "pmu_clk2_grp", "pmu_clk");
-
-static struct pinctrl *lenovo_yoga_tab2_830_1050_codec_pinctrl;
-static struct sys_off_handler *lenovo_yoga_tab2_830_1050_sys_off_handler;
-
-static int __init lenovo_yoga_tab2_830_1050_init_codec(void)
-{
-	struct device *codec_dev;
-	struct pinctrl *pinctrl;
-	int ret;
-
-	codec_dev = bus_find_device_by_name(&spi_bus_type, NULL,
-					    LENOVO_YOGA_TAB2_830_1050_CODEC_NAME);
-	if (!codec_dev) {
-		pr_err("error cannot find %s device\n", LENOVO_YOGA_TAB2_830_1050_CODEC_NAME);
-		return -ENODEV;
-	}
-
-	ret = pinctrl_register_mappings(&lenovo_yoga_tab2_830_1050_codec_pinctrl_map, 1);
-	if (ret)
-		goto err_put_device;
-
-	pinctrl = pinctrl_get_select(codec_dev, "codec_32khz_clk");
-	if (IS_ERR(pinctrl)) {
-		ret = dev_err_probe(codec_dev, PTR_ERR(pinctrl), "selecting codec_32khz_clk\n");
-		goto err_unregister_mappings;
-	}
-
-	/* We're done with the codec_dev now */
-	put_device(codec_dev);
-
-	lenovo_yoga_tab2_830_1050_codec_pinctrl = pinctrl;
-	return 0;
-
-err_unregister_mappings:
-	pinctrl_unregister_mappings(&lenovo_yoga_tab2_830_1050_codec_pinctrl_map);
-err_put_device:
-	put_device(codec_dev);
-	return ret;
-}
-
-/*
- * These tablet's DSDT does not set acpi_gbl_reduced_hardware, so acpi_power_off
- * gets used as pm_power_off handler. This causes "poweroff" on these tablets
- * to hang hard. Requiring pressing the powerbutton for 30 seconds *twice*
- * followed by a normal 3 second press to recover. Avoid this by doing an EFI
- * poweroff instead.
- */
-static int lenovo_yoga_tab2_830_1050_power_off(struct sys_off_data *data)
-{
-	efi.reset_system(EFI_RESET_SHUTDOWN, EFI_SUCCESS, 0, NULL);
-
-	return NOTIFY_DONE;
-}
-
-static int __init lenovo_yoga_tab2_830_1050_init(void)
-{
-	int ret;
-
-	ret = lenovo_yoga_tab2_830_1050_init_display();
-	if (ret)
-		return ret;
-
-	ret = lenovo_yoga_tab2_830_1050_init_codec();
-	if (ret)
-		return ret;
-
-	/* SYS_OFF_PRIO_FIRMWARE + 1 so that it runs before acpi_power_off */
-	lenovo_yoga_tab2_830_1050_sys_off_handler =
-		register_sys_off_handler(SYS_OFF_MODE_POWER_OFF, SYS_OFF_PRIO_FIRMWARE + 1,
-					 lenovo_yoga_tab2_830_1050_power_off, NULL);
-	if (IS_ERR(lenovo_yoga_tab2_830_1050_sys_off_handler))
-		return PTR_ERR(lenovo_yoga_tab2_830_1050_sys_off_handler);
-
-	return 0;
-}
-
-static void lenovo_yoga_tab2_830_1050_exit(void)
-{
-	unregister_sys_off_handler(lenovo_yoga_tab2_830_1050_sys_off_handler);
-
-	if (lenovo_yoga_tab2_830_1050_codec_pinctrl) {
-		pinctrl_put(lenovo_yoga_tab2_830_1050_codec_pinctrl);
-		pinctrl_unregister_mappings(&lenovo_yoga_tab2_830_1050_codec_pinctrl_map);
-	}
-}
-
-/* Lenovo Yoga Tab 3 Pro YT3-X90F */
-
-/*
- * There are 2 batteries, with 2 bq27500 fuel-gauges and 2 bq25892 chargers,
- * "bq25890-charger-1" is instantiated from: drivers/i2c/busses/i2c-cht-wc.c.
- */
-static const char * const lenovo_yt3_bq25892_0_suppliers[] = { "cht_wcove_pwrsrc" };
-static const char * const bq25890_1_psy[] = { "bq25890-charger-1" };
-
-static const struct property_entry fg_bq25890_1_supply_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", bq25890_1_psy),
-	{ }
-};
-
-static const struct software_node fg_bq25890_1_supply_node = {
-	.properties = fg_bq25890_1_supply_props,
-};
-
-/* bq25892 charger settings for the flat lipo battery behind the screen */
-static const struct property_entry lenovo_yt3_bq25892_0_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", lenovo_yt3_bq25892_0_suppliers),
-	PROPERTY_ENTRY_STRING("linux,power-supply-name", "bq25892-second-chrg"),
-	PROPERTY_ENTRY_U32("linux,iinlim-percentage", 40),
-	PROPERTY_ENTRY_BOOL("linux,skip-reset"),
-	/* Values taken from Android Factory Image */
-	PROPERTY_ENTRY_U32("ti,charge-current", 2048000),
-	PROPERTY_ENTRY_U32("ti,battery-regulation-voltage", 4352000),
-	PROPERTY_ENTRY_U32("ti,termination-current", 128000),
-	PROPERTY_ENTRY_U32("ti,precharge-current", 128000),
-	PROPERTY_ENTRY_U32("ti,minimum-sys-voltage", 3700000),
-	PROPERTY_ENTRY_U32("ti,boost-voltage", 4998000),
-	PROPERTY_ENTRY_U32("ti,boost-max-current", 500000),
-	PROPERTY_ENTRY_BOOL("ti,use-ilim-pin"),
-	{ }
-};
-
-static const struct software_node lenovo_yt3_bq25892_0_node = {
-	.properties = lenovo_yt3_bq25892_0_props,
-};
-
-static const struct x86_i2c_client_info lenovo_yt3_i2c_clients[] __initconst = {
-	{
-		/* bq27500 fuel-gauge for the flat lipo battery behind the screen */
-		.board_info = {
-			.type = "bq27500",
-			.addr = 0x55,
-			.dev_name = "bq27500_0",
-			.swnode = &fg_bq25890_supply_node,
-		},
-		.adapter_path = "\\_SB_.PCI0.I2C1",
-	}, {
-		/* bq25892 charger for the flat lipo battery behind the screen */
-		.board_info = {
-			.type = "bq25892",
-			.addr = 0x6b,
-			.dev_name = "bq25892_0",
-			.swnode = &lenovo_yt3_bq25892_0_node,
-		},
-		.adapter_path = "\\_SB_.PCI0.I2C1",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
-			.chip = "INT33FF:01",
-			.index = 5,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_LOW,
-		},
-	}, {
-		/* bq27500 fuel-gauge for the round li-ion cells in the hinge */
-		.board_info = {
-			.type = "bq27500",
-			.addr = 0x55,
-			.dev_name = "bq27500_1",
-			.swnode = &fg_bq25890_1_supply_node,
-		},
-		.adapter_path = "\\_SB_.PCI0.I2C2",
-	}
-};
-
-static int __init lenovo_yt3_init(void)
-{
-	struct gpio_desc *gpiod;
-	int ret;
-
-	/*
-	 * The "bq25892_0" charger IC has its /CE (Charge-Enable) and OTG pins
-	 * connected to GPIOs, rather then having them hardwired to the correct
-	 * values as is normally done.
-	 *
-	 * The bq25890_charger driver controls these through I2C, but this only
-	 * works if not overridden by the pins. Set these pins here:
-	 * 1. Set /CE to 0 to allow charging.
-	 * 2. Set OTG to 0 disable V5 boost output since the 5V boost output of
-	 *    the main "bq25892_1" charger is used when necessary.
-	 */
-
-	/* /CE pin */
-	ret = x86_android_tablet_get_gpiod("INT33FF:02", 22, &gpiod);
-	if (ret < 0)
-		return ret;
-
-	/*
-	 * The gpio_desc returned by x86_android_tablet_get_gpiod() is a "raw"
-	 * gpio_desc, that is there is no way to pass lookup-flags like
-	 * GPIO_ACTIVE_LOW. Set the GPIO to 0 here to enable charging since
-	 * the /CE pin is active-low, but not marked as such in the gpio_desc.
-	 */
-	gpiod_set_value(gpiod, 0);
-
-	/* OTG pin */
-	ret = x86_android_tablet_get_gpiod("INT33FF:03", 19, &gpiod);
-	if (ret < 0)
-		return ret;
-
-	gpiod_set_value(gpiod, 0);
-
-	return 0;
-}
-
-static const struct x86_dev_info lenovo_yt3_info __initconst = {
-	.i2c_client_info = lenovo_yt3_i2c_clients,
-	.i2c_client_count = ARRAY_SIZE(lenovo_yt3_i2c_clients),
-	.init = lenovo_yt3_init,
-};
-
-/* Medion Lifetab S10346 tablets have an Android factory img with everything hardcoded */
-static const char * const medion_lifetab_s10346_accel_mount_matrix[] = {
-	"0", "1", "0",
-	"1", "0", "0",
-	"0", "0", "1"
-};
-
-static const struct property_entry medion_lifetab_s10346_accel_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", medion_lifetab_s10346_accel_mount_matrix),
-	{ }
-};
-
-static const struct software_node medion_lifetab_s10346_accel_node = {
-	.properties = medion_lifetab_s10346_accel_props,
-};
-
-/* Note the LCD panel is mounted upside down, this is correctly indicated in the VBT */
-static const struct property_entry medion_lifetab_s10346_touchscreen_props[] = {
-	PROPERTY_ENTRY_BOOL("touchscreen-inverted-x"),
-	PROPERTY_ENTRY_BOOL("touchscreen-swapped-x-y"),
-	{ }
-};
-
-static const struct software_node medion_lifetab_s10346_touchscreen_node = {
-	.properties = medion_lifetab_s10346_touchscreen_props,
-};
-
-static const struct x86_i2c_client_info medion_lifetab_s10346_i2c_clients[] __initconst = {
-	{
-		/* kxtj21009 accel */
-		.board_info = {
-			.type = "kxtj21009",
-			.addr = 0x0f,
-			.dev_name = "kxtj21009",
-			.swnode = &medion_lifetab_s10346_accel_node,
-		},
-		.adapter_path = "\\_SB_.I2C3",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
-			.chip = "INT33FC:02",
-			.index = 23,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_HIGH,
-		},
-	}, {
-		/* goodix touchscreen */
-		.board_info = {
-			.type = "GDIX1001:00",
-			.addr = 0x14,
-			.dev_name = "goodix_ts",
-			.swnode = &medion_lifetab_s10346_touchscreen_node,
-		},
-		.adapter_path = "\\_SB_.I2C4",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_APIC,
-			.index = 0x44,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_LOW,
-		},
-	},
-};
-
-static struct gpiod_lookup_table medion_lifetab_s10346_goodix_gpios = {
-	.dev_id = "i2c-goodix_ts",
-	.table = {
-		GPIO_LOOKUP("INT33FC:01", 26, "reset", GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP("INT33FC:02", 3, "irq", GPIO_ACTIVE_HIGH),
-		{ }
-	},
-};
-
-static struct gpiod_lookup_table * const medion_lifetab_s10346_gpios[] = {
-	&medion_lifetab_s10346_goodix_gpios,
-	NULL
-};
-
-static const struct x86_dev_info medion_lifetab_s10346_info __initconst = {
-	.i2c_client_info = medion_lifetab_s10346_i2c_clients,
-	.i2c_client_count = ARRAY_SIZE(medion_lifetab_s10346_i2c_clients),
-	.gpiod_lookup_tables = medion_lifetab_s10346_gpios,
-};
-
-/* Nextbook Ares 8 tablets have an Android factory img with everything hardcoded */
-static const char * const nextbook_ares8_accel_mount_matrix[] = {
-	"0", "-1", "0",
-	"-1", "0", "0",
-	"0", "0", "1"
-};
-
-static const struct property_entry nextbook_ares8_accel_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", nextbook_ares8_accel_mount_matrix),
-	{ }
-};
-
-static const struct software_node nextbook_ares8_accel_node = {
-	.properties = nextbook_ares8_accel_props,
-};
-
-static const struct property_entry nextbook_ares8_touchscreen_props[] = {
-	PROPERTY_ENTRY_U32("touchscreen-size-x", 800),
-	PROPERTY_ENTRY_U32("touchscreen-size-y", 1280),
-	{ }
-};
-
-static const struct software_node nextbook_ares8_touchscreen_node = {
-	.properties = nextbook_ares8_touchscreen_props,
-};
-
-static const struct x86_i2c_client_info nextbook_ares8_i2c_clients[] __initconst = {
-	{
-		/* Freescale MMA8653FC accel */
-		.board_info = {
-			.type = "mma8653",
-			.addr = 0x1d,
-			.dev_name = "mma8653",
-			.swnode = &nextbook_ares8_accel_node,
-		},
-		.adapter_path = "\\_SB_.I2C3",
-	}, {
-		/* FT5416DQ9 touchscreen controller */
-		.board_info = {
-			.type = "edt-ft5x06",
-			.addr = 0x38,
-			.dev_name = "ft5416",
-			.swnode = &nextbook_ares8_touchscreen_node,
-		},
-		.adapter_path = "\\_SB_.I2C4",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
-			.chip = "INT33FC:02",
-			.index = 3,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_LOW,
-		},
-	},
-};
-
-static struct gpiod_lookup_table nextbook_ares8_int3496_gpios = {
-	.dev_id = "intel-int3496",
-	.table = {
-		GPIO_LOOKUP("INT33FC:02", 1, "mux", GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP("INT33FC:02", 18, "id", GPIO_ACTIVE_HIGH),
-		{ }
-	},
-};
-
-static struct gpiod_lookup_table * const nextbook_ares8_gpios[] = {
-	&nextbook_ares8_int3496_gpios,
-	NULL
-};
-
-static const struct x86_dev_info nextbook_ares8_info __initconst = {
-	.i2c_client_info = nextbook_ares8_i2c_clients,
-	.i2c_client_count = ARRAY_SIZE(nextbook_ares8_i2c_clients),
-	.pdev_info = int3496_pdevs,
-	.pdev_count = ARRAY_SIZE(int3496_pdevs),
-	.gpiod_lookup_tables = nextbook_ares8_gpios,
-	.invalid_aei_gpiochip = "INT33FC:02",
-};
-
-/*
- * Whitelabel (sold as various brands) TM800A550L tablets.
- * These tablet's DSDT contains a whole bunch of bogus ACPI I2C devices
- * (removed through acpi_quirk_skip_i2c_client_enumeration()) and
- * the touchscreen fwnode has the wrong GPIOs.
- */
-static const char * const whitelabel_tm800a550l_accel_mount_matrix[] = {
-	"-1", "0", "0",
-	"0", "1", "0",
-	"0", "0", "1"
-};
-
-static const struct property_entry whitelabel_tm800a550l_accel_props[] = {
-	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", whitelabel_tm800a550l_accel_mount_matrix),
-	{ }
-};
-
-static const struct software_node whitelabel_tm800a550l_accel_node = {
-	.properties = whitelabel_tm800a550l_accel_props,
-};
-
-static const struct property_entry whitelabel_tm800a550l_goodix_props[] = {
-	PROPERTY_ENTRY_STRING("firmware-name", "gt912-tm800a550l.fw"),
-	PROPERTY_ENTRY_STRING("goodix,config-name", "gt912-tm800a550l.cfg"),
-	PROPERTY_ENTRY_U32("goodix,main-clk", 54),
-	{ }
-};
-
-static const struct software_node whitelabel_tm800a550l_goodix_node = {
-	.properties = whitelabel_tm800a550l_goodix_props,
-};
-
-static const struct x86_i2c_client_info whitelabel_tm800a550l_i2c_clients[] __initconst = {
-	{
-		/* goodix touchscreen */
-		.board_info = {
-			.type = "GDIX1001:00",
-			.addr = 0x14,
-			.dev_name = "goodix_ts",
-			.swnode = &whitelabel_tm800a550l_goodix_node,
-		},
-		.adapter_path = "\\_SB_.I2C2",
-		.irq_data = {
-			.type = X86_ACPI_IRQ_TYPE_APIC,
-			.index = 0x44,
-			.trigger = ACPI_EDGE_SENSITIVE,
-			.polarity = ACPI_ACTIVE_HIGH,
-		},
-	}, {
-		/* kxcj91008 accel */
-		.board_info = {
-			.type = "kxcj91008",
-			.addr = 0x0f,
-			.dev_name = "kxcj91008",
-			.swnode = &whitelabel_tm800a550l_accel_node,
-		},
-		.adapter_path = "\\_SB_.I2C3",
-	},
-};
-
-static struct gpiod_lookup_table whitelabel_tm800a550l_goodix_gpios = {
-	.dev_id = "i2c-goodix_ts",
-	.table = {
-		GPIO_LOOKUP("INT33FC:01", 26, "reset", GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP("INT33FC:02", 3, "irq", GPIO_ACTIVE_HIGH),
-		{ }
-	},
-};
-
-static struct gpiod_lookup_table * const whitelabel_tm800a550l_gpios[] = {
-	&whitelabel_tm800a550l_goodix_gpios,
-	NULL
-};
-
-static const struct x86_dev_info whitelabel_tm800a550l_info __initconst = {
-	.i2c_client_info = whitelabel_tm800a550l_i2c_clients,
-	.i2c_client_count = ARRAY_SIZE(whitelabel_tm800a550l_i2c_clients),
-	.gpiod_lookup_tables = whitelabel_tm800a550l_gpios,
-};
-
-/*
- * If the EFI bootloader is not Xiaomi's own signed Android loader, then the
- * Xiaomi Mi Pad 2 X86 tablet sets OSID in the DSDT to 1 (Windows), causing
- * a bunch of devices to be hidden.
- *
- * This takes care of instantiating the hidden devices manually.
- */
-static const struct x86_i2c_client_info xiaomi_mipad2_i2c_clients[] __initconst = {
-	{
-		/* BQ27520 fuel-gauge */
-		.board_info = {
-			.type = "bq27520",
-			.addr = 0x55,
-			.dev_name = "bq27520",
-			.swnode = &fg_bq25890_supply_node,
-		},
-		.adapter_path = "\\_SB_.PCI0.I2C1",
-	}, {
-		/* KTD2026 RGB notification LED controller */
-		.board_info = {
-			.type = "ktd2026",
-			.addr = 0x30,
-			.dev_name = "ktd2026",
-		},
-		.adapter_path = "\\_SB_.PCI0.I2C3",
-	},
-};
-
-static const struct x86_dev_info xiaomi_mipad2_info __initconst = {
-	.i2c_client_info = xiaomi_mipad2_i2c_clients,
-	.i2c_client_count = ARRAY_SIZE(xiaomi_mipad2_i2c_clients),
-};
-
-static const struct dmi_system_id x86_android_tablet_ids[] __initconst = {
-	{
-		/* Advantech MICA-071 */
-		.matches = {
-			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Advantech"),
-			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "MICA-071"),
-		},
-		.driver_data = (void *)&advantech_mica_071_info,
-	},
-	{
-		/* Asus MeMO Pad 7 ME176C */
-		.matches = {
-			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
-			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "ME176C"),
-		},
-		.driver_data = (void *)&asus_me176c_info,
-	},
-	{
-		/* Asus TF103C */
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "TF103C"),
-		},
-		.driver_data = (void *)&asus_tf103c_info,
-	},
-	{
-		/* Chuwi Hi8 (CWI509) */
-		.matches = {
-			DMI_MATCH(DMI_BOARD_VENDOR, "Hampoo"),
-			DMI_MATCH(DMI_BOARD_NAME, "BYT-PA03C"),
-			DMI_MATCH(DMI_SYS_VENDOR, "ilife"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "S806"),
-		},
-		.driver_data = (void *)&chuwi_hi8_info,
-	},
-	{
-		/* CZC P10T */
-		.ident = "CZC ODEON TPC-10 (\"P10T\")",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "CZC"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "ODEON*TPC-10"),
-		},
-		.driver_data = (void *)&czc_p10t,
-	},
-	{
-		/* CZC P10T variant */
-		.ident = "ViewSonic ViewPad 10",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "ViewSonic"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "VPAD10"),
-		},
-		.driver_data = (void *)&czc_p10t,
-	},
-	{
-		/* Lenovo Yoga Book X90F / X91F / X91L */
-		.matches = {
-			/* Non exact match to match all versions */
-			DMI_MATCH(DMI_PRODUCT_NAME, "Lenovo YB1-X9"),
-		},
-		.driver_data = (void *)&lenovo_yogabook_x9x_info,
-	},
-	{
-		/*
-		 * Lenovo Yoga Tablet 2 830F/L or 1050F/L (The 8" and 10"
-		 * Lenovo Yoga Tablet 2 use the same mainboard)
-		 */
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Intel Corp."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "VALLEYVIEW C0 PLATFORM"),
-			DMI_MATCH(DMI_BOARD_NAME, "BYT-T FFD8"),
-			/* Partial match on beginning of BIOS version */
-			DMI_MATCH(DMI_BIOS_VERSION, "BLADE_21"),
-		},
-		.driver_data = (void *)&lenovo_yoga_tab2_830_1050_info,
-	},
-	{
-		/* Lenovo Yoga Tab 3 Pro YT3-X90F */
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Intel Corporation"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "CHERRYVIEW D1 PLATFORM"),
-			DMI_MATCH(DMI_PRODUCT_VERSION, "Blade3-10A-001"),
-		},
-		.driver_data = (void *)&lenovo_yt3_info,
-	},
-	{
-		/* Medion Lifetab S10346 */
-		.matches = {
-			DMI_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
-			DMI_MATCH(DMI_BOARD_NAME, "Aptio CRB"),
-			/* Above strings are much too generic, also match on BIOS date */
-			DMI_MATCH(DMI_BIOS_DATE, "10/22/2015"),
-		},
-		.driver_data = (void *)&medion_lifetab_s10346_info,
-	},
-	{
-		/* Nextbook Ares 8 */
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Insyde"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "M890BAP"),
-		},
-		.driver_data = (void *)&nextbook_ares8_info,
-	},
-	{
-		/* Whitelabel (sold as various brands) TM800A550L */
-		.matches = {
-			DMI_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
-			DMI_MATCH(DMI_BOARD_NAME, "Aptio CRB"),
-			/* Above strings are too generic, also match on BIOS version */
-			DMI_MATCH(DMI_BIOS_VERSION, "ZY-8-BI-PX4S70VTR400-X423B-005-D"),
-		},
-		.driver_data = (void *)&whitelabel_tm800a550l_info,
-	},
-	{
-		/* Xiaomi Mi Pad 2 */
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Xiaomi Inc"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "Mipad2"),
-		},
-		.driver_data = (void *)&xiaomi_mipad2_info,
-	},
-	{ }
-};
-MODULE_DEVICE_TABLE(dmi, x86_android_tablet_ids);
-
-static int i2c_client_count;
-static int pdev_count;
-static int serdev_count;
-static struct i2c_client **i2c_clients;
-static struct platform_device **pdevs;
-static struct serdev_device **serdevs;
-static struct gpiod_lookup_table * const *gpiod_lookup_tables;
-static const struct software_node *bat_swnode;
-static void (*exit_handler)(void);
-
-static __init int x86_instantiate_i2c_client(const struct x86_dev_info *dev_info,
-					     int idx)
-{
-	const struct x86_i2c_client_info *client_info = &dev_info->i2c_client_info[idx];
-	struct i2c_board_info board_info = client_info->board_info;
-	struct i2c_adapter *adap;
-	acpi_handle handle;
-	acpi_status status;
-
-	board_info.irq = x86_acpi_irq_helper_get(&client_info->irq_data);
-	if (board_info.irq < 0)
-		return board_info.irq;
-
-	status = acpi_get_handle(NULL, client_info->adapter_path, &handle);
-	if (ACPI_FAILURE(status)) {
-		pr_err("Error could not get %s handle\n", client_info->adapter_path);
-		return -ENODEV;
-	}
-
-	adap = i2c_acpi_find_adapter_by_handle(handle);
-	if (!adap) {
-		pr_err("error could not get %s adapter\n", client_info->adapter_path);
-		return -ENODEV;
-	}
-
-	i2c_clients[idx] = i2c_new_client_device(adap, &board_info);
-	put_device(&adap->dev);
-	if (IS_ERR(i2c_clients[idx]))
-		return dev_err_probe(&adap->dev, PTR_ERR(i2c_clients[idx]),
-				      "creating I2C-client %d\n", idx);
-
-	return 0;
-}
-
-static __init int x86_instantiate_serdev(const struct x86_serdev_info *info, int idx)
-{
-	struct acpi_device *ctrl_adev, *serdev_adev;
-	struct serdev_device *serdev;
-	struct device *ctrl_dev;
-	int ret = -ENODEV;
-
-	ctrl_adev = acpi_dev_get_first_match_dev(info->ctrl_hid, info->ctrl_uid, -1);
-	if (!ctrl_adev) {
-		pr_err("error could not get %s/%s ctrl adev\n",
-		       info->ctrl_hid, info->ctrl_uid);
-		return -ENODEV;
-	}
-
-	serdev_adev = acpi_dev_get_first_match_dev(info->serdev_hid, NULL, -1);
-	if (!serdev_adev) {
-		pr_err("error could not get %s serdev adev\n", info->serdev_hid);
-		goto put_ctrl_adev;
-	}
-
-	/* get_first_physical_node() returns a weak ref, no need to put() it */
-	ctrl_dev = acpi_get_first_physical_node(ctrl_adev);
-	if (!ctrl_dev)	{
-		pr_err("error could not get %s/%s ctrl physical dev\n",
-		       info->ctrl_hid, info->ctrl_uid);
-		goto put_serdev_adev;
-	}
-
-	/* ctrl_dev now points to the controller's parent, get the controller */
-	ctrl_dev = device_find_child_by_name(ctrl_dev, info->ctrl_devname);
-	if (!ctrl_dev) {
-		pr_err("error could not get %s/%s %s ctrl dev\n",
-		       info->ctrl_hid, info->ctrl_uid, info->ctrl_devname);
-		goto put_serdev_adev;
-	}
-
-	serdev = serdev_device_alloc(to_serdev_controller(ctrl_dev));
-	if (!serdev) {
-		ret = -ENOMEM;
-		goto put_serdev_adev;
-	}
-
-	ACPI_COMPANION_SET(&serdev->dev, serdev_adev);
-	acpi_device_set_enumerated(serdev_adev);
-
-	ret = serdev_device_add(serdev);
-	if (ret) {
-		dev_err(&serdev->dev, "error %d adding serdev\n", ret);
-		serdev_device_put(serdev);
-		goto put_serdev_adev;
-	}
-
-	serdevs[idx] = serdev;
-
-put_serdev_adev:
-	acpi_dev_put(serdev_adev);
-put_ctrl_adev:
-	acpi_dev_put(ctrl_adev);
-	return ret;
-}
-
-static void x86_android_tablet_cleanup(void)
-{
-	int i;
-
-	for (i = 0; i < serdev_count; i++) {
-		if (serdevs[i])
-			serdev_device_remove(serdevs[i]);
-	}
-
-	kfree(serdevs);
-
-	for (i = 0; i < pdev_count; i++)
-		platform_device_unregister(pdevs[i]);
-
-	kfree(pdevs);
-
-	for (i = 0; i < i2c_client_count; i++)
-		i2c_unregister_device(i2c_clients[i]);
-
-	kfree(i2c_clients);
-
-	if (exit_handler)
-		exit_handler();
-
-	for (i = 0; gpiod_lookup_tables && gpiod_lookup_tables[i]; i++)
-		gpiod_remove_lookup_table(gpiod_lookup_tables[i]);
-
-	software_node_unregister(bat_swnode);
-}
-
-static __init int x86_android_tablet_init(void)
-{
-	const struct x86_dev_info *dev_info;
-	const struct dmi_system_id *id;
-	struct gpio_chip *chip;
-	int i, ret = 0;
-
-	id = dmi_first_match(x86_android_tablet_ids);
-	if (!id)
-		return -ENODEV;
-
-	dev_info = id->driver_data;
-
-	/*
-	 * The broken DSDTs on these devices often also include broken
-	 * _AEI (ACPI Event Interrupt) handlers, disable these.
-	 */
-	if (dev_info->invalid_aei_gpiochip) {
-		chip = gpiochip_find(dev_info->invalid_aei_gpiochip,
-				     gpiochip_find_match_label);
-		if (!chip) {
-			pr_err("error cannot find GPIO chip %s\n", dev_info->invalid_aei_gpiochip);
-			return -ENODEV;
-		}
-		acpi_gpiochip_free_interrupts(chip);
-	}
-
-	/*
-	 * Since this runs from module_init() it cannot use -EPROBE_DEFER,
-	 * instead pre-load any modules which are listed as requirements.
-	 */
-	for (i = 0; dev_info->modules && dev_info->modules[i]; i++)
-		request_module(dev_info->modules[i]);
-
-	bat_swnode = dev_info->bat_swnode;
-	if (bat_swnode) {
-		ret = software_node_register(bat_swnode);
-		if (ret)
-			return ret;
-	}
-
-	gpiod_lookup_tables = dev_info->gpiod_lookup_tables;
-	for (i = 0; gpiod_lookup_tables && gpiod_lookup_tables[i]; i++)
-		gpiod_add_lookup_table(gpiod_lookup_tables[i]);
-
-	if (dev_info->init) {
-		ret = dev_info->init();
-		if (ret < 0) {
-			x86_android_tablet_cleanup();
-			return ret;
-		}
-		exit_handler = dev_info->exit;
-	}
-
-	i2c_clients = kcalloc(dev_info->i2c_client_count, sizeof(*i2c_clients), GFP_KERNEL);
-	if (!i2c_clients) {
-		x86_android_tablet_cleanup();
-		return -ENOMEM;
-	}
-
-	i2c_client_count = dev_info->i2c_client_count;
-	for (i = 0; i < i2c_client_count; i++) {
-		ret = x86_instantiate_i2c_client(dev_info, i);
-		if (ret < 0) {
-			x86_android_tablet_cleanup();
-			return ret;
-		}
-	}
-
-	pdevs = kcalloc(dev_info->pdev_count, sizeof(*pdevs), GFP_KERNEL);
-	if (!pdevs) {
-		x86_android_tablet_cleanup();
-		return -ENOMEM;
-	}
-
-	pdev_count = dev_info->pdev_count;
-	for (i = 0; i < pdev_count; i++) {
-		pdevs[i] = platform_device_register_full(&dev_info->pdev_info[i]);
-		if (IS_ERR(pdevs[i])) {
-			x86_android_tablet_cleanup();
-			return PTR_ERR(pdevs[i]);
-		}
-	}
-
-	serdevs = kcalloc(dev_info->serdev_count, sizeof(*serdevs), GFP_KERNEL);
-	if (!serdevs) {
-		x86_android_tablet_cleanup();
-		return -ENOMEM;
-	}
-
-	serdev_count = dev_info->serdev_count;
-	for (i = 0; i < serdev_count; i++) {
-		ret = x86_instantiate_serdev(&dev_info->serdev_info[i], i);
-		if (ret < 0) {
-			x86_android_tablet_cleanup();
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-module_init(x86_android_tablet_init);
-module_exit(x86_android_tablet_cleanup);
-
-MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
-MODULE_DESCRIPTION("X86 Android tablets DSDT fixups driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/x86-android-tablets/Kconfig b/drivers/platform/x86/x86-android-tablets/Kconfig
new file mode 100644
index 000000000000..6603461d4273
--- /dev/null
+++ b/drivers/platform/x86/x86-android-tablets/Kconfig
@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# X86 Android tablet support Kconfig
+#
+
+config X86_ANDROID_TABLETS
+	tristate "X86 Android tablet support"
+	depends on I2C && SPI && SERIAL_DEV_BUS && ACPI && EFI && GPIOLIB && PMIC_OPREGION
+	help
+	  X86 tablets which ship with Android as (part of) the factory image
+	  typically have various problems with their DSDTs. The factory kernels
+	  shipped on these devices typically have device addresses and GPIOs
+	  hardcoded in the kernel, rather than specified in their DSDT.
+
+	  With the DSDT containing a random collection of devices which may or
+	  may not actually be present. This driver contains various fixes for
+	  such tablets, including instantiating kernel devices for devices which
+	  are missing from the DSDT.
+
+	  If you have a x86 Android tablet say Y or M here, for a generic x86
+	  distro config say M here.
diff --git a/drivers/platform/x86/x86-android-tablets/Makefile b/drivers/platform/x86/x86-android-tablets/Makefile
new file mode 100644
index 000000000000..41ece5a37137
--- /dev/null
+++ b/drivers/platform/x86/x86-android-tablets/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# X86 Android tablet support Makefile
+#
+
+obj-$(CONFIG_X86_ANDROID_TABLETS) += x86-android-tablets.o
+
+x86-android-tablets-y := core.o dmi.o shared-psy-info.o \
+			 asus.o lenovo.o other.o
diff --git a/drivers/platform/x86/x86-android-tablets/asus.c b/drivers/platform/x86/x86-android-tablets/asus.c
new file mode 100644
index 000000000000..cfa038b44b43
--- /dev/null
+++ b/drivers/platform/x86/x86-android-tablets/asus.c
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Board info for Asus X86 tablets which ship with Android as the factory image
+ * and which have broken DSDT tables. The factory kernels shipped on these
+ * devices typically have a bunch of things hardcoded, rather than specified
+ * in their DSDT.
+ *
+ * Copyright (C) 2021-2023 Hans de Goede <hdegoede@redhat.com>
+ */
+
+#include <linux/gpio/machine.h>
+#include <linux/input.h>
+#include <linux/platform_device.h>
+
+#include "shared-psy-info.h"
+#include "x86-android-tablets.h"
+
+/* Asus ME176C and TF103C tablets shared data */
+static struct gpiod_lookup_table int3496_gpo2_pin22_gpios = {
+	.dev_id = "intel-int3496",
+	.table = {
+		GPIO_LOOKUP("INT33FC:02", 22, "id", GPIO_ACTIVE_HIGH),
+		{ }
+	},
+};
+
+static struct x86_gpio_button asus_me176c_tf103c_lid = {
+	.button = {
+		.code = SW_LID,
+		.active_low = true,
+		.desc = "lid_sw",
+		.type = EV_SW,
+		.wakeup = true,
+		.debounce_interval = 50,
+	},
+	.chip = "INT33FC:02",
+	.pin = 12,
+};
+
+/* Asus ME176C tablets have an Android factory img with everything hardcoded */
+static const char * const asus_me176c_accel_mount_matrix[] = {
+	"-1", "0", "0",
+	"0", "1", "0",
+	"0", "0", "1"
+};
+
+static const struct property_entry asus_me176c_accel_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", asus_me176c_accel_mount_matrix),
+	{ }
+};
+
+static const struct software_node asus_me176c_accel_node = {
+	.properties = asus_me176c_accel_props,
+};
+
+static const struct property_entry asus_me176c_bq24190_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY_LEN("supplied-from", tusb1211_chg_det_psy, 1),
+	PROPERTY_ENTRY_REF("monitored-battery", &generic_lipo_hv_4v35_battery_node),
+	PROPERTY_ENTRY_U32("ti,system-minimum-microvolt", 3600000),
+	PROPERTY_ENTRY_BOOL("omit-battery-class"),
+	PROPERTY_ENTRY_BOOL("disable-reset"),
+	{ }
+};
+
+static const struct software_node asus_me176c_bq24190_node = {
+	.properties = asus_me176c_bq24190_props,
+};
+
+static const struct property_entry asus_me176c_ug3105_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY_LEN("supplied-from", bq24190_psy, 1),
+	PROPERTY_ENTRY_REF("monitored-battery", &generic_lipo_hv_4v35_battery_node),
+	PROPERTY_ENTRY_U32("upisemi,rsns-microohm", 10000),
+	{ }
+};
+
+static const struct software_node asus_me176c_ug3105_node = {
+	.properties = asus_me176c_ug3105_props,
+};
+
+static const struct x86_i2c_client_info asus_me176c_i2c_clients[] __initconst = {
+	{
+		/* bq24297 battery charger */
+		.board_info = {
+			.type = "bq24190",
+			.addr = 0x6b,
+			.dev_name = "bq24297",
+			.swnode = &asus_me176c_bq24190_node,
+			.platform_data = &bq24190_pdata,
+		},
+		.adapter_path = "\\_SB_.I2C1",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_PMIC,
+			.chip = "\\_SB_.I2C7.PMIC",
+			.domain = DOMAIN_BUS_WAKEUP,
+			.index = 0,
+		},
+	}, {
+		/* ug3105 battery monitor */
+		.board_info = {
+			.type = "ug3105",
+			.addr = 0x70,
+			.dev_name = "ug3105",
+			.swnode = &asus_me176c_ug3105_node,
+		},
+		.adapter_path = "\\_SB_.I2C1",
+	}, {
+		/* ak09911 compass */
+		.board_info = {
+			.type = "ak09911",
+			.addr = 0x0c,
+			.dev_name = "ak09911",
+		},
+		.adapter_path = "\\_SB_.I2C5",
+	}, {
+		/* kxtj21009 accel */
+		.board_info = {
+			.type = "kxtj21009",
+			.addr = 0x0f,
+			.dev_name = "kxtj21009",
+			.swnode = &asus_me176c_accel_node,
+		},
+		.adapter_path = "\\_SB_.I2C5",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_APIC,
+			.index = 0x44,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	}, {
+		/* goodix touchscreen */
+		.board_info = {
+			.type = "GDIX1001:00",
+			.addr = 0x14,
+			.dev_name = "goodix_ts",
+		},
+		.adapter_path = "\\_SB_.I2C6",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_APIC,
+			.index = 0x45,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	},
+};
+
+static const struct x86_serdev_info asus_me176c_serdevs[] __initconst = {
+	{
+		.ctrl_hid = "80860F0A",
+		.ctrl_uid = "2",
+		.ctrl_devname = "serial0",
+		.serdev_hid = "BCM2E3A",
+	},
+};
+
+static struct gpiod_lookup_table asus_me176c_goodix_gpios = {
+	.dev_id = "i2c-goodix_ts",
+	.table = {
+		GPIO_LOOKUP("INT33FC:00", 60, "reset", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("INT33FC:02", 28, "irq", GPIO_ACTIVE_HIGH),
+		{ }
+	},
+};
+
+static struct gpiod_lookup_table * const asus_me176c_gpios[] = {
+	&int3496_gpo2_pin22_gpios,
+	&asus_me176c_goodix_gpios,
+	NULL
+};
+
+const struct x86_dev_info asus_me176c_info __initconst = {
+	.i2c_client_info = asus_me176c_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(asus_me176c_i2c_clients),
+	.pdev_info = int3496_pdevs,
+	.pdev_count = 1,
+	.serdev_info = asus_me176c_serdevs,
+	.serdev_count = ARRAY_SIZE(asus_me176c_serdevs),
+	.gpio_button = &asus_me176c_tf103c_lid,
+	.gpiod_lookup_tables = asus_me176c_gpios,
+	.bat_swnode = &generic_lipo_hv_4v35_battery_node,
+	.modules = bq24190_modules,
+	.invalid_aei_gpiochip = "INT33FC:02",
+};
+
+/* Asus TF103C tablets have an Android factory img with everything hardcoded */
+static const char * const asus_tf103c_accel_mount_matrix[] = {
+	"0", "-1", "0",
+	"-1", "0", "0",
+	"0", "0", "1"
+};
+
+static const struct property_entry asus_tf103c_accel_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", asus_tf103c_accel_mount_matrix),
+	{ }
+};
+
+static const struct software_node asus_tf103c_accel_node = {
+	.properties = asus_tf103c_accel_props,
+};
+
+static const struct property_entry asus_tf103c_touchscreen_props[] = {
+	PROPERTY_ENTRY_STRING("compatible", "atmel,atmel_mxt_ts"),
+	{ }
+};
+
+static const struct software_node asus_tf103c_touchscreen_node = {
+	.properties = asus_tf103c_touchscreen_props,
+};
+
+static const struct property_entry asus_tf103c_battery_props[] = {
+	PROPERTY_ENTRY_STRING("compatible", "simple-battery"),
+	PROPERTY_ENTRY_STRING("device-chemistry", "lithium-ion-polymer"),
+	PROPERTY_ENTRY_U32("precharge-current-microamp", 256000),
+	PROPERTY_ENTRY_U32("charge-term-current-microamp", 128000),
+	PROPERTY_ENTRY_U32("constant-charge-current-max-microamp", 2048000),
+	PROPERTY_ENTRY_U32("constant-charge-voltage-max-microvolt", 4208000),
+	PROPERTY_ENTRY_U32("factory-internal-resistance-micro-ohms", 150000),
+	{ }
+};
+
+static const struct software_node asus_tf103c_battery_node = {
+	.properties = asus_tf103c_battery_props,
+};
+
+static const struct property_entry asus_tf103c_bq24190_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY_LEN("supplied-from", tusb1211_chg_det_psy, 1),
+	PROPERTY_ENTRY_REF("monitored-battery", &asus_tf103c_battery_node),
+	PROPERTY_ENTRY_U32("ti,system-minimum-microvolt", 3600000),
+	PROPERTY_ENTRY_BOOL("omit-battery-class"),
+	PROPERTY_ENTRY_BOOL("disable-reset"),
+	{ }
+};
+
+static const struct software_node asus_tf103c_bq24190_node = {
+	.properties = asus_tf103c_bq24190_props,
+};
+
+static const struct property_entry asus_tf103c_ug3105_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY_LEN("supplied-from", bq24190_psy, 1),
+	PROPERTY_ENTRY_REF("monitored-battery", &asus_tf103c_battery_node),
+	PROPERTY_ENTRY_U32("upisemi,rsns-microohm", 5000),
+	{ }
+};
+
+static const struct software_node asus_tf103c_ug3105_node = {
+	.properties = asus_tf103c_ug3105_props,
+};
+
+static const struct x86_i2c_client_info asus_tf103c_i2c_clients[] __initconst = {
+	{
+		/* bq24297 battery charger */
+		.board_info = {
+			.type = "bq24190",
+			.addr = 0x6b,
+			.dev_name = "bq24297",
+			.swnode = &asus_tf103c_bq24190_node,
+			.platform_data = &bq24190_pdata,
+		},
+		.adapter_path = "\\_SB_.I2C1",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_PMIC,
+			.chip = "\\_SB_.I2C7.PMIC",
+			.domain = DOMAIN_BUS_WAKEUP,
+			.index = 0,
+		},
+	}, {
+		/* ug3105 battery monitor */
+		.board_info = {
+			.type = "ug3105",
+			.addr = 0x70,
+			.dev_name = "ug3105",
+			.swnode = &asus_tf103c_ug3105_node,
+		},
+		.adapter_path = "\\_SB_.I2C1",
+	}, {
+		/* ak09911 compass */
+		.board_info = {
+			.type = "ak09911",
+			.addr = 0x0c,
+			.dev_name = "ak09911",
+		},
+		.adapter_path = "\\_SB_.I2C5",
+	}, {
+		/* kxtj21009 accel */
+		.board_info = {
+			.type = "kxtj21009",
+			.addr = 0x0f,
+			.dev_name = "kxtj21009",
+			.swnode = &asus_tf103c_accel_node,
+		},
+		.adapter_path = "\\_SB_.I2C5",
+	}, {
+		/* atmel touchscreen */
+		.board_info = {
+			.type = "atmel_mxt_ts",
+			.addr = 0x4a,
+			.dev_name = "atmel_mxt_ts",
+			.swnode = &asus_tf103c_touchscreen_node,
+		},
+		.adapter_path = "\\_SB_.I2C6",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FC:02",
+			.index = 28,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	},
+};
+
+static struct gpiod_lookup_table * const asus_tf103c_gpios[] = {
+	&int3496_gpo2_pin22_gpios,
+	NULL
+};
+
+const struct x86_dev_info asus_tf103c_info __initconst = {
+	.i2c_client_info = asus_tf103c_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(asus_tf103c_i2c_clients),
+	.pdev_info = int3496_pdevs,
+	.pdev_count = 1,
+	.gpio_button = &asus_me176c_tf103c_lid,
+	.gpiod_lookup_tables = asus_tf103c_gpios,
+	.bat_swnode = &asus_tf103c_battery_node,
+	.modules = bq24190_modules,
+	.invalid_aei_gpiochip = "INT33FC:02",
+};
diff --git a/drivers/platform/x86/x86-android-tablets/core.c b/drivers/platform/x86/x86-android-tablets/core.c
new file mode 100644
index 000000000000..245167674aa2
--- /dev/null
+++ b/drivers/platform/x86/x86-android-tablets/core.c
@@ -0,0 +1,391 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * DMI based code to deal with broken DSDTs on X86 tablets which ship with
+ * Android as (part of) the factory image. The factory kernels shipped on these
+ * devices typically have a bunch of things hardcoded, rather than specified
+ * in their DSDT.
+ *
+ * Copyright (C) 2021-2023 Hans de Goede <hdegoede@redhat.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/acpi.h>
+#include <linux/dmi.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/machine.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/serdev.h>
+#include <linux/string.h>
+
+#include "x86-android-tablets.h"
+/* For gpiochip_get_desc() which is EXPORT_SYMBOL_GPL() */
+#include "../../../gpio/gpiolib.h"
+#include "../../../gpio/gpiolib-acpi.h"
+
+static int gpiochip_find_match_label(struct gpio_chip *gc, void *data)
+{
+	return gc->label && !strcmp(gc->label, data);
+}
+
+int x86_android_tablet_get_gpiod(const char *label, int pin, struct gpio_desc **desc)
+{
+	struct gpio_desc *gpiod;
+	struct gpio_chip *chip;
+
+	chip = gpiochip_find((void *)label, gpiochip_find_match_label);
+	if (!chip) {
+		pr_err("error cannot find GPIO chip %s\n", label);
+		return -ENODEV;
+	}
+
+	gpiod = gpiochip_get_desc(chip, pin);
+	if (IS_ERR(gpiod)) {
+		pr_err("error %ld getting GPIO %s %d\n", PTR_ERR(gpiod), label, pin);
+		return PTR_ERR(gpiod);
+	}
+
+	*desc = gpiod;
+	return 0;
+}
+
+int x86_acpi_irq_helper_get(const struct x86_acpi_irq_data *data)
+{
+	struct irq_fwspec fwspec = { };
+	struct irq_domain *domain;
+	struct acpi_device *adev;
+	struct gpio_desc *gpiod;
+	unsigned int irq_type;
+	acpi_handle handle;
+	acpi_status status;
+	int irq, ret;
+
+	switch (data->type) {
+	case X86_ACPI_IRQ_TYPE_APIC:
+		/*
+		 * The DSDT may already reference the GSI in a device skipped by
+		 * acpi_quirk_skip_i2c_client_enumeration(). Unregister the GSI
+		 * to avoid EBUSY errors in this case.
+		 */
+		acpi_unregister_gsi(data->index);
+		irq = acpi_register_gsi(NULL, data->index, data->trigger, data->polarity);
+		if (irq < 0)
+			pr_err("error %d getting APIC IRQ %d\n", irq, data->index);
+
+		return irq;
+	case X86_ACPI_IRQ_TYPE_GPIOINT:
+		/* Like acpi_dev_gpio_irq_get(), but without parsing ACPI resources */
+		ret = x86_android_tablet_get_gpiod(data->chip, data->index, &gpiod);
+		if (ret)
+			return ret;
+
+		irq = gpiod_to_irq(gpiod);
+		if (irq < 0) {
+			pr_err("error %d getting IRQ %s %d\n", irq, data->chip, data->index);
+			return irq;
+		}
+
+		irq_type = acpi_dev_get_irq_type(data->trigger, data->polarity);
+		if (irq_type != IRQ_TYPE_NONE && irq_type != irq_get_trigger_type(irq))
+			irq_set_irq_type(irq, irq_type);
+
+		return irq;
+	case X86_ACPI_IRQ_TYPE_PMIC:
+		status = acpi_get_handle(NULL, data->chip, &handle);
+		if (ACPI_FAILURE(status)) {
+			pr_err("error could not get %s handle\n", data->chip);
+			return -ENODEV;
+		}
+
+		adev = acpi_fetch_acpi_dev(handle);
+		if (!adev) {
+			pr_err("error could not get %s adev\n", data->chip);
+			return -ENODEV;
+		}
+
+		fwspec.fwnode = acpi_fwnode_handle(adev);
+		domain = irq_find_matching_fwspec(&fwspec, data->domain);
+		if (!domain) {
+			pr_err("error could not find IRQ domain for %s\n", data->chip);
+			return -ENODEV;
+		}
+
+		return irq_create_mapping(domain, data->index);
+	default:
+		return 0;
+	}
+}
+
+static int i2c_client_count;
+static int pdev_count;
+static int serdev_count;
+static struct i2c_client **i2c_clients;
+static struct platform_device **pdevs;
+static struct serdev_device **serdevs;
+static struct gpiod_lookup_table * const *gpiod_lookup_tables;
+static const struct software_node *bat_swnode;
+static void (*exit_handler)(void);
+
+static __init int x86_instantiate_i2c_client(const struct x86_dev_info *dev_info,
+					     int idx)
+{
+	const struct x86_i2c_client_info *client_info = &dev_info->i2c_client_info[idx];
+	struct i2c_board_info board_info = client_info->board_info;
+	struct i2c_adapter *adap;
+	acpi_handle handle;
+	acpi_status status;
+
+	board_info.irq = x86_acpi_irq_helper_get(&client_info->irq_data);
+	if (board_info.irq < 0)
+		return board_info.irq;
+
+	status = acpi_get_handle(NULL, client_info->adapter_path, &handle);
+	if (ACPI_FAILURE(status)) {
+		pr_err("Error could not get %s handle\n", client_info->adapter_path);
+		return -ENODEV;
+	}
+
+	adap = i2c_acpi_find_adapter_by_handle(handle);
+	if (!adap) {
+		pr_err("error could not get %s adapter\n", client_info->adapter_path);
+		return -ENODEV;
+	}
+
+	i2c_clients[idx] = i2c_new_client_device(adap, &board_info);
+	put_device(&adap->dev);
+	if (IS_ERR(i2c_clients[idx]))
+		return dev_err_probe(&adap->dev, PTR_ERR(i2c_clients[idx]),
+				      "creating I2C-client %d\n", idx);
+
+	return 0;
+}
+
+static __init int x86_instantiate_serdev(const struct x86_serdev_info *info, int idx)
+{
+	struct acpi_device *ctrl_adev, *serdev_adev;
+	struct serdev_device *serdev;
+	struct device *ctrl_dev;
+	int ret = -ENODEV;
+
+	ctrl_adev = acpi_dev_get_first_match_dev(info->ctrl_hid, info->ctrl_uid, -1);
+	if (!ctrl_adev) {
+		pr_err("error could not get %s/%s ctrl adev\n",
+		       info->ctrl_hid, info->ctrl_uid);
+		return -ENODEV;
+	}
+
+	serdev_adev = acpi_dev_get_first_match_dev(info->serdev_hid, NULL, -1);
+	if (!serdev_adev) {
+		pr_err("error could not get %s serdev adev\n", info->serdev_hid);
+		goto put_ctrl_adev;
+	}
+
+	/* get_first_physical_node() returns a weak ref, no need to put() it */
+	ctrl_dev = acpi_get_first_physical_node(ctrl_adev);
+	if (!ctrl_dev)	{
+		pr_err("error could not get %s/%s ctrl physical dev\n",
+		       info->ctrl_hid, info->ctrl_uid);
+		goto put_serdev_adev;
+	}
+
+	/* ctrl_dev now points to the controller's parent, get the controller */
+	ctrl_dev = device_find_child_by_name(ctrl_dev, info->ctrl_devname);
+	if (!ctrl_dev) {
+		pr_err("error could not get %s/%s %s ctrl dev\n",
+		       info->ctrl_hid, info->ctrl_uid, info->ctrl_devname);
+		goto put_serdev_adev;
+	}
+
+	serdev = serdev_device_alloc(to_serdev_controller(ctrl_dev));
+	if (!serdev) {
+		ret = -ENOMEM;
+		goto put_serdev_adev;
+	}
+
+	ACPI_COMPANION_SET(&serdev->dev, serdev_adev);
+	acpi_device_set_enumerated(serdev_adev);
+
+	ret = serdev_device_add(serdev);
+	if (ret) {
+		dev_err(&serdev->dev, "error %d adding serdev\n", ret);
+		serdev_device_put(serdev);
+		goto put_serdev_adev;
+	}
+
+	serdevs[idx] = serdev;
+
+put_serdev_adev:
+	acpi_dev_put(serdev_adev);
+put_ctrl_adev:
+	acpi_dev_put(ctrl_adev);
+	return ret;
+}
+
+static void x86_android_tablet_cleanup(void)
+{
+	int i;
+
+	for (i = 0; i < serdev_count; i++) {
+		if (serdevs[i])
+			serdev_device_remove(serdevs[i]);
+	}
+
+	kfree(serdevs);
+
+	for (i = 0; i < pdev_count; i++)
+		platform_device_unregister(pdevs[i]);
+
+	kfree(pdevs);
+
+	for (i = 0; i < i2c_client_count; i++)
+		i2c_unregister_device(i2c_clients[i]);
+
+	kfree(i2c_clients);
+
+	if (exit_handler)
+		exit_handler();
+
+	for (i = 0; gpiod_lookup_tables && gpiod_lookup_tables[i]; i++)
+		gpiod_remove_lookup_table(gpiod_lookup_tables[i]);
+
+	software_node_unregister(bat_swnode);
+}
+
+static __init int x86_android_tablet_init(void)
+{
+	const struct x86_dev_info *dev_info;
+	const struct dmi_system_id *id;
+	struct gpio_chip *chip;
+	int i, ret = 0;
+
+	id = dmi_first_match(x86_android_tablet_ids);
+	if (!id)
+		return -ENODEV;
+
+	dev_info = id->driver_data;
+
+	/*
+	 * The broken DSDTs on these devices often also include broken
+	 * _AEI (ACPI Event Interrupt) handlers, disable these.
+	 */
+	if (dev_info->invalid_aei_gpiochip) {
+		chip = gpiochip_find(dev_info->invalid_aei_gpiochip,
+				     gpiochip_find_match_label);
+		if (!chip) {
+			pr_err("error cannot find GPIO chip %s\n", dev_info->invalid_aei_gpiochip);
+			return -ENODEV;
+		}
+		acpi_gpiochip_free_interrupts(chip);
+	}
+
+	/*
+	 * Since this runs from module_init() it cannot use -EPROBE_DEFER,
+	 * instead pre-load any modules which are listed as requirements.
+	 */
+	for (i = 0; dev_info->modules && dev_info->modules[i]; i++)
+		request_module(dev_info->modules[i]);
+
+	bat_swnode = dev_info->bat_swnode;
+	if (bat_swnode) {
+		ret = software_node_register(bat_swnode);
+		if (ret)
+			return ret;
+	}
+
+	gpiod_lookup_tables = dev_info->gpiod_lookup_tables;
+	for (i = 0; gpiod_lookup_tables && gpiod_lookup_tables[i]; i++)
+		gpiod_add_lookup_table(gpiod_lookup_tables[i]);
+
+	if (dev_info->init) {
+		ret = dev_info->init();
+		if (ret < 0) {
+			x86_android_tablet_cleanup();
+			return ret;
+		}
+		exit_handler = dev_info->exit;
+	}
+
+	i2c_clients = kcalloc(dev_info->i2c_client_count, sizeof(*i2c_clients), GFP_KERNEL);
+	if (!i2c_clients) {
+		x86_android_tablet_cleanup();
+		return -ENOMEM;
+	}
+
+	i2c_client_count = dev_info->i2c_client_count;
+	for (i = 0; i < i2c_client_count; i++) {
+		ret = x86_instantiate_i2c_client(dev_info, i);
+		if (ret < 0) {
+			x86_android_tablet_cleanup();
+			return ret;
+		}
+	}
+
+	/* + 1 to make space for (optional) gpio_keys_button pdev */
+	pdevs = kcalloc(dev_info->pdev_count + 1, sizeof(*pdevs), GFP_KERNEL);
+	if (!pdevs) {
+		x86_android_tablet_cleanup();
+		return -ENOMEM;
+	}
+
+	pdev_count = dev_info->pdev_count;
+	for (i = 0; i < pdev_count; i++) {
+		pdevs[i] = platform_device_register_full(&dev_info->pdev_info[i]);
+		if (IS_ERR(pdevs[i])) {
+			x86_android_tablet_cleanup();
+			return PTR_ERR(pdevs[i]);
+		}
+	}
+
+	serdevs = kcalloc(dev_info->serdev_count, sizeof(*serdevs), GFP_KERNEL);
+	if (!serdevs) {
+		x86_android_tablet_cleanup();
+		return -ENOMEM;
+	}
+
+	serdev_count = dev_info->serdev_count;
+	for (i = 0; i < serdev_count; i++) {
+		ret = x86_instantiate_serdev(&dev_info->serdev_info[i], i);
+		if (ret < 0) {
+			x86_android_tablet_cleanup();
+			return ret;
+		}
+	}
+
+	if (dev_info->gpio_button) {
+		struct gpio_keys_platform_data pdata = {
+			.buttons = &dev_info->gpio_button->button,
+			.nbuttons = 1,
+		};
+		struct gpio_desc *gpiod;
+
+		/* Get GPIO for the gpio-button */
+		ret = x86_android_tablet_get_gpiod(dev_info->gpio_button->chip,
+						   dev_info->gpio_button->pin, &gpiod);
+		if (ret < 0) {
+			x86_android_tablet_cleanup();
+			return ret;
+		}
+
+		dev_info->gpio_button->button.gpio = desc_to_gpio(gpiod);
+
+		pdevs[pdev_count] = platform_device_register_data(NULL, "gpio-keys",
+								  PLATFORM_DEVID_AUTO,
+								  &pdata, sizeof(pdata));
+		if (IS_ERR(pdevs[pdev_count])) {
+			x86_android_tablet_cleanup();
+			return PTR_ERR(pdevs[pdev_count]);
+		}
+		pdev_count++;
+	}
+
+	return 0;
+}
+
+module_init(x86_android_tablet_init);
+module_exit(x86_android_tablet_cleanup);
+
+MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
+MODULE_DESCRIPTION("X86 Android tablets DSDT fixups driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/x86-android-tablets/dmi.c b/drivers/platform/x86/x86-android-tablets/dmi.c
new file mode 100644
index 000000000000..23e640b7003d
--- /dev/null
+++ b/drivers/platform/x86/x86-android-tablets/dmi.c
@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * DMI based code to deal with broken DSDTs on X86 tablets which ship with
+ * Android as (part of) the factory image. The factory kernels shipped on these
+ * devices typically have a bunch of things hardcoded, rather than specified
+ * in their DSDT.
+ *
+ * Copyright (C) 2021-2023 Hans de Goede <hdegoede@redhat.com>
+ */
+
+#include <linux/dmi.h>
+#include <linux/init.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+
+#include "x86-android-tablets.h"
+
+const struct dmi_system_id x86_android_tablet_ids[] __initconst = {
+	{
+		/* Acer Iconia One 7 B1-750 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Insyde"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "VESPA2"),
+		},
+		.driver_data = (void *)&acer_b1_750_info,
+	},
+	{
+		/* Advantech MICA-071 */
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Advantech"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "MICA-071"),
+		},
+		.driver_data = (void *)&advantech_mica_071_info,
+	},
+	{
+		/* Asus MeMO Pad 7 ME176C */
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "ME176C"),
+		},
+		.driver_data = (void *)&asus_me176c_info,
+	},
+	{
+		/* Asus TF103C */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "TF103C"),
+		},
+		.driver_data = (void *)&asus_tf103c_info,
+	},
+	{
+		/* Chuwi Hi8 (CWI509) */
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "Hampoo"),
+			DMI_MATCH(DMI_BOARD_NAME, "BYT-PA03C"),
+			DMI_MATCH(DMI_SYS_VENDOR, "ilife"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "S806"),
+		},
+		.driver_data = (void *)&chuwi_hi8_info,
+	},
+	{
+		/* CZC P10T */
+		.ident = "CZC ODEON TPC-10 (\"P10T\")",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "CZC"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ODEON*TPC-10"),
+		},
+		.driver_data = (void *)&czc_p10t,
+	},
+	{
+		/* CZC P10T variant */
+		.ident = "ViewSonic ViewPad 10",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ViewSonic"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "VPAD10"),
+		},
+		.driver_data = (void *)&czc_p10t,
+	},
+	{
+		/* Lenovo Yoga Book X90F / X90L */
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Intel Corporation"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "CHERRYVIEW D1 PLATFORM"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "YETI-11"),
+		},
+		.driver_data = (void *)&lenovo_yogabook_x90_info,
+	},
+	{
+		/* Lenovo Yoga Book X91F / X91L */
+		.matches = {
+			/* Non exact match to match F + L versions */
+			DMI_MATCH(DMI_PRODUCT_NAME, "Lenovo YB1-X91"),
+		},
+		.driver_data = (void *)&lenovo_yogabook_x91_info,
+	},
+	{
+		/*
+		 * Lenovo Yoga Tablet 2 830F/L or 1050F/L (The 8" and 10"
+		 * Lenovo Yoga Tablet 2 use the same mainboard)
+		 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Intel Corp."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "VALLEYVIEW C0 PLATFORM"),
+			DMI_MATCH(DMI_BOARD_NAME, "BYT-T FFD8"),
+			/* Partial match on beginning of BIOS version */
+			DMI_MATCH(DMI_BIOS_VERSION, "BLADE_21"),
+		},
+		.driver_data = (void *)&lenovo_yoga_tab2_830_1050_info,
+	},
+	{
+		/* Lenovo Yoga Tab 3 Pro YT3-X90F */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Intel Corporation"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "CHERRYVIEW D1 PLATFORM"),
+			DMI_MATCH(DMI_PRODUCT_VERSION, "Blade3-10A-001"),
+		},
+		.driver_data = (void *)&lenovo_yt3_info,
+	},
+	{
+		/* Medion Lifetab S10346 */
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
+			DMI_MATCH(DMI_BOARD_NAME, "Aptio CRB"),
+			/* Above strings are much too generic, also match on BIOS date */
+			DMI_MATCH(DMI_BIOS_DATE, "10/22/2015"),
+		},
+		.driver_data = (void *)&medion_lifetab_s10346_info,
+	},
+	{
+		/* Nextbook Ares 8 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Insyde"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "M890BAP"),
+		},
+		.driver_data = (void *)&nextbook_ares8_info,
+	},
+	{
+		/* Peaq C1010 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "PEAQ"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "PEAQ PMM C1010 MD99187"),
+		},
+		.driver_data = (void *)&peaq_c1010_info,
+	},
+	{
+		/* Whitelabel (sold as various brands) TM800A550L */
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
+			DMI_MATCH(DMI_BOARD_NAME, "Aptio CRB"),
+			/* Above strings are too generic, also match on BIOS version */
+			DMI_MATCH(DMI_BIOS_VERSION, "ZY-8-BI-PX4S70VTR400-X423B-005-D"),
+		},
+		.driver_data = (void *)&whitelabel_tm800a550l_info,
+	},
+	{
+		/* Xiaomi Mi Pad 2 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Xiaomi Inc"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Mipad2"),
+		},
+		.driver_data = (void *)&xiaomi_mipad2_info,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(dmi, x86_android_tablet_ids);
diff --git a/drivers/platform/x86/x86-android-tablets/lenovo.c b/drivers/platform/x86/x86-android-tablets/lenovo.c
new file mode 100644
index 000000000000..65cfccaa2894
--- /dev/null
+++ b/drivers/platform/x86/x86-android-tablets/lenovo.c
@@ -0,0 +1,679 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Board info for Lenovo X86 tablets which ship with Android as the factory image
+ * and which have broken DSDT tables. The factory kernels shipped on these
+ * devices typically have a bunch of things hardcoded, rather than specified
+ * in their DSDT.
+ *
+ * Copyright (C) 2021-2023 Hans de Goede <hdegoede@redhat.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/efi.h>
+#include <linux/gpio/machine.h>
+#include <linux/mfd/intel_soc_pmic.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/pinctrl/machine.h>
+#include <linux/platform_data/lp855x.h>
+#include <linux/platform_device.h>
+#include <linux/reboot.h>
+#include <linux/rmi.h>
+#include <linux/spi/spi.h>
+
+#include "shared-psy-info.h"
+#include "x86-android-tablets.h"
+
+/*
+ * Various Lenovo models use a TI LP8557 LED backlight controller with its PWM
+ * input connected to a PWM output coming from the LCD panel's controller.
+ * The Android kernels have a hack in the i915 driver to write a non-standard
+ * panel specific DSI register to set the duty-cycle of the LCD's PWM output.
+ *
+ * To avoid having to have a similar hack in the mainline kernel program the
+ * LP8557 to directly set the level and use the lp855x_bl driver for control.
+ */
+static struct lp855x_platform_data lenovo_lp8557_pdata = {
+	.device_control = 0x86,
+	.initial_brightness = 128,
+};
+
+/* Lenovo Yoga Book X90F / X90L's Android factory img has everything hardcoded */
+
+static const struct property_entry lenovo_yb1_x90_wacom_props[] = {
+	PROPERTY_ENTRY_U32("hid-descr-addr", 0x0001),
+	PROPERTY_ENTRY_U32("post-reset-deassert-delay-ms", 150),
+	{ }
+};
+
+static const struct software_node lenovo_yb1_x90_wacom_node = {
+	.properties = lenovo_yb1_x90_wacom_props,
+};
+
+/*
+ * The HiDeep IST940E touchscreen comes up in I2C-HID mode. The native protocol
+ * reports ABS_MT_PRESSURE and ABS_MT_TOUCH_MAJOR which are not reported in HID
+ * mode, so using native mode is preferred.
+ * It could alternatively be used in HID mode by changing the properties to:
+ *	PROPERTY_ENTRY_U32("hid-descr-addr", 0x0020),
+ *	PROPERTY_ENTRY_U32("post-reset-deassert-delay-ms", 120),
+ * and changing board_info.type to "hid-over-i2c".
+ */
+static const struct property_entry lenovo_yb1_x90_hideep_ts_props[] = {
+	PROPERTY_ENTRY_U32("touchscreen-size-x", 1200),
+	PROPERTY_ENTRY_U32("touchscreen-size-y", 1920),
+	PROPERTY_ENTRY_U32("touchscreen-max-pressure", 16384),
+	PROPERTY_ENTRY_BOOL("hideep,force-native-protocol"),
+	{ }
+};
+
+static const struct software_node lenovo_yb1_x90_hideep_ts_node = {
+	.properties = lenovo_yb1_x90_hideep_ts_props,
+};
+
+static const struct x86_i2c_client_info lenovo_yb1_x90_i2c_clients[] __initconst = {
+	{
+		/* BQ27542 fuel-gauge */
+		.board_info = {
+			.type = "bq27542",
+			.addr = 0x55,
+			.dev_name = "bq27542",
+			.swnode = &fg_bq25890_supply_node,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C1",
+	}, {
+		/* Goodix Touchscreen in keyboard half */
+		.board_info = {
+			.type = "GDIX1001:00",
+			.addr = 0x14,
+			.dev_name = "goodix_ts",
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C2",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FF:01",
+			.index = 56,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	}, {
+		/* Wacom Digitizer in keyboard half */
+		.board_info = {
+			.type = "hid-over-i2c",
+			.addr = 0x09,
+			.dev_name = "wacom",
+			.swnode = &lenovo_yb1_x90_wacom_node,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C4",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FF:01",
+			.index = 49,
+			.trigger = ACPI_LEVEL_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	}, {
+		/* LP8557 Backlight controller */
+		.board_info = {
+			.type = "lp8557",
+			.addr = 0x2c,
+			.dev_name = "lp8557",
+			.platform_data = &lenovo_lp8557_pdata,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C4",
+	}, {
+		/* HiDeep IST940E Touchscreen in display half */
+		.board_info = {
+			.type = "hideep_ts",
+			.addr = 0x6c,
+			.dev_name = "hideep_ts",
+			.swnode = &lenovo_yb1_x90_hideep_ts_node,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C6",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FF:03",
+			.index = 77,
+			.trigger = ACPI_LEVEL_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	},
+};
+
+static const struct platform_device_info lenovo_yb1_x90_pdevs[] __initconst = {
+	{
+		.name = "yogabook-touch-kbd-digitizer-switch",
+		.id = PLATFORM_DEVID_NONE,
+	},
+};
+
+static struct gpiod_lookup_table lenovo_yb1_x90_goodix_gpios = {
+	.dev_id = "i2c-goodix_ts",
+	.table = {
+		GPIO_LOOKUP("INT33FF:01", 53, "reset", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("INT33FF:01", 56, "irq", GPIO_ACTIVE_HIGH),
+		{ }
+	},
+};
+
+static struct gpiod_lookup_table lenovo_yb1_x90_hideep_gpios = {
+	.dev_id = "i2c-hideep_ts",
+	.table = {
+		GPIO_LOOKUP("INT33FF:00", 7, "reset", GPIO_ACTIVE_LOW),
+		{ }
+	},
+};
+
+static struct gpiod_lookup_table lenovo_yb1_x90_wacom_gpios = {
+	.dev_id = "i2c-wacom",
+	.table = {
+		GPIO_LOOKUP("INT33FF:00", 82, "reset", GPIO_ACTIVE_LOW),
+		{ }
+	},
+};
+
+static struct gpiod_lookup_table * const lenovo_yb1_x90_gpios[] = {
+	&lenovo_yb1_x90_hideep_gpios,
+	&lenovo_yb1_x90_goodix_gpios,
+	&lenovo_yb1_x90_wacom_gpios,
+	NULL
+};
+
+static int __init lenovo_yb1_x90_init(void)
+{
+	/* Enable the regulators used by the touchscreens */
+
+	/* Vprog3B 3.0V used by the goodix touchscreen in the keyboard half */
+	intel_soc_pmic_exec_mipi_pmic_seq_element(0x6e, 0x9b, 0x02, 0xff);
+
+	/* Vprog4D 3.0V used by the HiDeep touchscreen in the display half */
+	intel_soc_pmic_exec_mipi_pmic_seq_element(0x6e, 0x9f, 0x02, 0xff);
+
+	/* Vprog5A 1.8V used by the HiDeep touchscreen in the display half */
+	intel_soc_pmic_exec_mipi_pmic_seq_element(0x6e, 0xa0, 0x02, 0xff);
+
+	/* Vprog5B 1.8V used by the goodix touchscreen in the keyboard half */
+	intel_soc_pmic_exec_mipi_pmic_seq_element(0x6e, 0xa1, 0x02, 0xff);
+
+	return 0;
+}
+
+const struct x86_dev_info lenovo_yogabook_x90_info __initconst = {
+	.i2c_client_info = lenovo_yb1_x90_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(lenovo_yb1_x90_i2c_clients),
+	.pdev_info = lenovo_yb1_x90_pdevs,
+	.pdev_count = ARRAY_SIZE(lenovo_yb1_x90_pdevs),
+	.gpiod_lookup_tables = lenovo_yb1_x90_gpios,
+	.init = lenovo_yb1_x90_init,
+};
+
+/* Lenovo Yoga Book X91F/L Windows tablet needs manual instantiation of the fg client */
+static const struct x86_i2c_client_info lenovo_yogabook_x91_i2c_clients[] __initconst = {
+	{
+		/* BQ27542 fuel-gauge */
+		.board_info = {
+			.type = "bq27542",
+			.addr = 0x55,
+			.dev_name = "bq27542",
+			.swnode = &fg_bq25890_supply_node,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C1",
+	},
+};
+
+const struct x86_dev_info lenovo_yogabook_x91_info __initconst = {
+	.i2c_client_info = lenovo_yogabook_x91_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(lenovo_yogabook_x91_i2c_clients),
+};
+
+/* Lenovo Yoga Tablet 2 1050F/L's Android factory img has everything hardcoded */
+static const struct property_entry lenovo_yoga_tab2_830_1050_bq24190_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY_LEN("supplied-from", tusb1211_chg_det_psy, 1),
+	PROPERTY_ENTRY_REF("monitored-battery", &generic_lipo_hv_4v35_battery_node),
+	PROPERTY_ENTRY_BOOL("omit-battery-class"),
+	PROPERTY_ENTRY_BOOL("disable-reset"),
+	{ }
+};
+
+static const struct software_node lenovo_yoga_tab2_830_1050_bq24190_node = {
+	.properties = lenovo_yoga_tab2_830_1050_bq24190_props,
+};
+
+static struct x86_gpio_button lenovo_yoga_tab2_830_1050_lid = {
+	.button = {
+		.code = SW_LID,
+		.active_low = true,
+		.desc = "lid_sw",
+		.type = EV_SW,
+		.wakeup = true,
+		.debounce_interval = 50,
+	},
+	.chip = "INT33FC:02",
+	.pin = 26,
+};
+
+/* This gets filled by lenovo_yoga_tab2_830_1050_init() */
+static struct rmi_device_platform_data lenovo_yoga_tab2_830_1050_rmi_pdata = { };
+
+static struct x86_i2c_client_info lenovo_yoga_tab2_830_1050_i2c_clients[] __initdata = {
+	{
+		/*
+		 * This must be the first entry because lenovo_yoga_tab2_830_1050_init()
+		 * may update its swnode. LSM303DA accelerometer + magnetometer.
+		 */
+		.board_info = {
+			.type = "lsm303d",
+			.addr = 0x1d,
+			.dev_name = "lsm303d",
+		},
+		.adapter_path = "\\_SB_.I2C5",
+	}, {
+		/* bq24292i battery charger */
+		.board_info = {
+			.type = "bq24190",
+			.addr = 0x6b,
+			.dev_name = "bq24292i",
+			.swnode = &lenovo_yoga_tab2_830_1050_bq24190_node,
+			.platform_data = &bq24190_pdata,
+		},
+		.adapter_path = "\\_SB_.I2C1",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FC:02",
+			.index = 2,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_HIGH,
+		},
+	}, {
+		/* BQ27541 fuel-gauge */
+		.board_info = {
+			.type = "bq27541",
+			.addr = 0x55,
+			.dev_name = "bq27541",
+			.swnode = &fg_bq24190_supply_node,
+		},
+		.adapter_path = "\\_SB_.I2C1",
+	}, {
+		/* Synaptics RMI touchscreen */
+		.board_info = {
+			.type = "rmi4_i2c",
+			.addr = 0x38,
+			.dev_name = "rmi4_i2c",
+			.platform_data = &lenovo_yoga_tab2_830_1050_rmi_pdata,
+		},
+		.adapter_path = "\\_SB_.I2C6",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_APIC,
+			.index = 0x45,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_HIGH,
+		},
+	}, {
+		/* LP8557 Backlight controller */
+		.board_info = {
+			.type = "lp8557",
+			.addr = 0x2c,
+			.dev_name = "lp8557",
+			.platform_data = &lenovo_lp8557_pdata,
+		},
+		.adapter_path = "\\_SB_.I2C3",
+	},
+};
+
+static struct gpiod_lookup_table lenovo_yoga_tab2_830_1050_int3496_gpios = {
+	.dev_id = "intel-int3496",
+	.table = {
+		GPIO_LOOKUP("INT33FC:02", 1, "mux", GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP("INT33FC:02", 24, "id", GPIO_ACTIVE_HIGH),
+		{ }
+	},
+};
+
+#define LENOVO_YOGA_TAB2_830_1050_CODEC_NAME "spi-10WM5102:00"
+
+static struct gpiod_lookup_table lenovo_yoga_tab2_830_1050_codec_gpios = {
+	.dev_id = LENOVO_YOGA_TAB2_830_1050_CODEC_NAME,
+	.table = {
+		GPIO_LOOKUP("gpio_crystalcove", 3, "reset", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("INT33FC:01", 23, "wlf,ldoena", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("arizona", 2, "wlf,spkvdd-ena", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("arizona", 4, "wlf,micd-pol", GPIO_ACTIVE_LOW),
+		{ }
+	},
+};
+
+static struct gpiod_lookup_table * const lenovo_yoga_tab2_830_1050_gpios[] = {
+	&lenovo_yoga_tab2_830_1050_int3496_gpios,
+	&lenovo_yoga_tab2_830_1050_codec_gpios,
+	NULL
+};
+
+static int __init lenovo_yoga_tab2_830_1050_init(void);
+static void lenovo_yoga_tab2_830_1050_exit(void);
+
+const struct x86_dev_info lenovo_yoga_tab2_830_1050_info __initconst = {
+	.i2c_client_info = lenovo_yoga_tab2_830_1050_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(lenovo_yoga_tab2_830_1050_i2c_clients),
+	.pdev_info = int3496_pdevs,
+	.pdev_count = 1,
+	.gpio_button = &lenovo_yoga_tab2_830_1050_lid,
+	.gpiod_lookup_tables = lenovo_yoga_tab2_830_1050_gpios,
+	.bat_swnode = &generic_lipo_hv_4v35_battery_node,
+	.modules = bq24190_modules,
+	.init = lenovo_yoga_tab2_830_1050_init,
+	.exit = lenovo_yoga_tab2_830_1050_exit,
+};
+
+/*
+ * The Lenovo Yoga Tablet 2 830 and 1050 (8" vs 10") versions use the same
+ * mainboard, but the 830 uses a portrait LCD panel with a landscape touchscreen,
+ * requiring the touchscreen driver to adjust the touch-coords to match the LCD.
+ * And requiring the accelerometer to have a mount-matrix set to correct for
+ * the 90° rotation of the LCD vs the frame.
+ */
+static const char * const lenovo_yoga_tab2_830_lms303d_mount_matrix[] = {
+	"0", "1", "0",
+	"-1", "0", "0",
+	"0", "0", "1"
+};
+
+static const struct property_entry lenovo_yoga_tab2_830_lms303d_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", lenovo_yoga_tab2_830_lms303d_mount_matrix),
+	{ }
+};
+
+static const struct software_node lenovo_yoga_tab2_830_lms303d_node = {
+	.properties = lenovo_yoga_tab2_830_lms303d_props,
+};
+
+static int __init lenovo_yoga_tab2_830_1050_init_touchscreen(void)
+{
+	struct gpio_desc *gpiod;
+	int ret;
+
+	/* Use PMIC GPIO 10 bootstrap pin to differentiate 830 vs 1050 */
+	ret = x86_android_tablet_get_gpiod("gpio_crystalcove", 10, &gpiod);
+	if (ret)
+		return ret;
+
+	ret = gpiod_get_value_cansleep(gpiod);
+	if (ret) {
+		pr_info("detected Lenovo Yoga Tablet 2 1050F/L\n");
+	} else {
+		pr_info("detected Lenovo Yoga Tablet 2 830F/L\n");
+		lenovo_yoga_tab2_830_1050_rmi_pdata.sensor_pdata.axis_align.swap_axes = true;
+		lenovo_yoga_tab2_830_1050_rmi_pdata.sensor_pdata.axis_align.flip_y = true;
+		lenovo_yoga_tab2_830_1050_i2c_clients[0].board_info.swnode =
+			&lenovo_yoga_tab2_830_lms303d_node;
+	}
+
+	return 0;
+}
+
+/* SUS (INT33FC:02) pin 6 needs to be configured as pmu_clk for the audio codec */
+static const struct pinctrl_map lenovo_yoga_tab2_830_1050_codec_pinctrl_map =
+	PIN_MAP_MUX_GROUP(LENOVO_YOGA_TAB2_830_1050_CODEC_NAME, "codec_32khz_clk",
+			  "INT33FC:02", "pmu_clk2_grp", "pmu_clk");
+
+static struct pinctrl *lenovo_yoga_tab2_830_1050_codec_pinctrl;
+static struct sys_off_handler *lenovo_yoga_tab2_830_1050_sys_off_handler;
+
+static int __init lenovo_yoga_tab2_830_1050_init_codec(void)
+{
+	struct device *codec_dev;
+	struct pinctrl *pinctrl;
+	int ret;
+
+	codec_dev = bus_find_device_by_name(&spi_bus_type, NULL,
+					    LENOVO_YOGA_TAB2_830_1050_CODEC_NAME);
+	if (!codec_dev) {
+		pr_err("error cannot find %s device\n", LENOVO_YOGA_TAB2_830_1050_CODEC_NAME);
+		return -ENODEV;
+	}
+
+	ret = pinctrl_register_mappings(&lenovo_yoga_tab2_830_1050_codec_pinctrl_map, 1);
+	if (ret)
+		goto err_put_device;
+
+	pinctrl = pinctrl_get_select(codec_dev, "codec_32khz_clk");
+	if (IS_ERR(pinctrl)) {
+		ret = dev_err_probe(codec_dev, PTR_ERR(pinctrl), "selecting codec_32khz_clk\n");
+		goto err_unregister_mappings;
+	}
+
+	/* We're done with the codec_dev now */
+	put_device(codec_dev);
+
+	lenovo_yoga_tab2_830_1050_codec_pinctrl = pinctrl;
+	return 0;
+
+err_unregister_mappings:
+	pinctrl_unregister_mappings(&lenovo_yoga_tab2_830_1050_codec_pinctrl_map);
+err_put_device:
+	put_device(codec_dev);
+	return ret;
+}
+
+/*
+ * These tablet's DSDT does not set acpi_gbl_reduced_hardware, so acpi_power_off
+ * gets used as pm_power_off handler. This causes "poweroff" on these tablets
+ * to hang hard. Requiring pressing the powerbutton for 30 seconds *twice*
+ * followed by a normal 3 second press to recover. Avoid this by doing an EFI
+ * poweroff instead.
+ */
+static int lenovo_yoga_tab2_830_1050_power_off(struct sys_off_data *data)
+{
+	efi.reset_system(EFI_RESET_SHUTDOWN, EFI_SUCCESS, 0, NULL);
+
+	return NOTIFY_DONE;
+}
+
+static int __init lenovo_yoga_tab2_830_1050_init(void)
+{
+	int ret;
+
+	ret = lenovo_yoga_tab2_830_1050_init_touchscreen();
+	if (ret)
+		return ret;
+
+	ret = lenovo_yoga_tab2_830_1050_init_codec();
+	if (ret)
+		return ret;
+
+	/* SYS_OFF_PRIO_FIRMWARE + 1 so that it runs before acpi_power_off */
+	lenovo_yoga_tab2_830_1050_sys_off_handler =
+		register_sys_off_handler(SYS_OFF_MODE_POWER_OFF, SYS_OFF_PRIO_FIRMWARE + 1,
+					 lenovo_yoga_tab2_830_1050_power_off, NULL);
+	if (IS_ERR(lenovo_yoga_tab2_830_1050_sys_off_handler))
+		return PTR_ERR(lenovo_yoga_tab2_830_1050_sys_off_handler);
+
+	return 0;
+}
+
+static void lenovo_yoga_tab2_830_1050_exit(void)
+{
+	unregister_sys_off_handler(lenovo_yoga_tab2_830_1050_sys_off_handler);
+
+	if (lenovo_yoga_tab2_830_1050_codec_pinctrl) {
+		pinctrl_put(lenovo_yoga_tab2_830_1050_codec_pinctrl);
+		pinctrl_unregister_mappings(&lenovo_yoga_tab2_830_1050_codec_pinctrl_map);
+	}
+}
+
+/* Lenovo Yoga Tab 3 Pro YT3-X90F */
+
+/*
+ * There are 2 batteries, with 2 bq27500 fuel-gauges and 2 bq25892 chargers,
+ * "bq25890-charger-1" is instantiated from: drivers/i2c/busses/i2c-cht-wc.c.
+ */
+static const char * const lenovo_yt3_bq25892_0_suppliers[] = { "cht_wcove_pwrsrc" };
+static const char * const bq25890_1_psy[] = { "bq25890-charger-1" };
+
+static const struct property_entry fg_bq25890_1_supply_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", bq25890_1_psy),
+	{ }
+};
+
+static const struct software_node fg_bq25890_1_supply_node = {
+	.properties = fg_bq25890_1_supply_props,
+};
+
+/* bq25892 charger settings for the flat lipo battery behind the screen */
+static const struct property_entry lenovo_yt3_bq25892_0_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", lenovo_yt3_bq25892_0_suppliers),
+	PROPERTY_ENTRY_STRING("linux,power-supply-name", "bq25892-second-chrg"),
+	PROPERTY_ENTRY_U32("linux,iinlim-percentage", 40),
+	PROPERTY_ENTRY_BOOL("linux,skip-reset"),
+	/* Values taken from Android Factory Image */
+	PROPERTY_ENTRY_U32("ti,charge-current", 2048000),
+	PROPERTY_ENTRY_U32("ti,battery-regulation-voltage", 4352000),
+	PROPERTY_ENTRY_U32("ti,termination-current", 128000),
+	PROPERTY_ENTRY_U32("ti,precharge-current", 128000),
+	PROPERTY_ENTRY_U32("ti,minimum-sys-voltage", 3700000),
+	PROPERTY_ENTRY_U32("ti,boost-voltage", 4998000),
+	PROPERTY_ENTRY_U32("ti,boost-max-current", 500000),
+	PROPERTY_ENTRY_BOOL("ti,use-ilim-pin"),
+	{ }
+};
+
+static const struct software_node lenovo_yt3_bq25892_0_node = {
+	.properties = lenovo_yt3_bq25892_0_props,
+};
+
+static const struct property_entry lenovo_yt3_hideep_ts_props[] = {
+	PROPERTY_ENTRY_U32("touchscreen-size-x", 1600),
+	PROPERTY_ENTRY_U32("touchscreen-size-y", 2560),
+	PROPERTY_ENTRY_U32("touchscreen-max-pressure", 255),
+	{ }
+};
+
+static const struct software_node lenovo_yt3_hideep_ts_node = {
+	.properties = lenovo_yt3_hideep_ts_props,
+};
+
+static const struct x86_i2c_client_info lenovo_yt3_i2c_clients[] __initconst = {
+	{
+		/* bq27500 fuel-gauge for the flat lipo battery behind the screen */
+		.board_info = {
+			.type = "bq27500",
+			.addr = 0x55,
+			.dev_name = "bq27500_0",
+			.swnode = &fg_bq25890_supply_node,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C1",
+	}, {
+		/* bq25892 charger for the flat lipo battery behind the screen */
+		.board_info = {
+			.type = "bq25892",
+			.addr = 0x6b,
+			.dev_name = "bq25892_0",
+			.swnode = &lenovo_yt3_bq25892_0_node,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C1",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FF:01",
+			.index = 5,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	}, {
+		/* bq27500 fuel-gauge for the round li-ion cells in the hinge */
+		.board_info = {
+			.type = "bq27500",
+			.addr = 0x55,
+			.dev_name = "bq27500_1",
+			.swnode = &fg_bq25890_1_supply_node,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C2",
+	}, {
+		/* HiDeep IST520E Touchscreen */
+		.board_info = {
+			.type = "hideep_ts",
+			.addr = 0x6c,
+			.dev_name = "hideep_ts",
+			.swnode = &lenovo_yt3_hideep_ts_node,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C6",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FF:03",
+			.index = 77,
+			.trigger = ACPI_LEVEL_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	}, {
+		/* LP8557 Backlight controller */
+		.board_info = {
+			.type = "lp8557",
+			.addr = 0x2c,
+			.dev_name = "lp8557",
+			.platform_data = &lenovo_lp8557_pdata,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C1",
+	}
+};
+
+static int __init lenovo_yt3_init(void)
+{
+	struct gpio_desc *gpiod;
+	int ret;
+
+	/*
+	 * The "bq25892_0" charger IC has its /CE (Charge-Enable) and OTG pins
+	 * connected to GPIOs, rather then having them hardwired to the correct
+	 * values as is normally done.
+	 *
+	 * The bq25890_charger driver controls these through I2C, but this only
+	 * works if not overridden by the pins. Set these pins here:
+	 * 1. Set /CE to 0 to allow charging.
+	 * 2. Set OTG to 0 disable V5 boost output since the 5V boost output of
+	 *    the main "bq25892_1" charger is used when necessary.
+	 */
+
+	/* /CE pin */
+	ret = x86_android_tablet_get_gpiod("INT33FF:02", 22, &gpiod);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * The gpio_desc returned by x86_android_tablet_get_gpiod() is a "raw"
+	 * gpio_desc, that is there is no way to pass lookup-flags like
+	 * GPIO_ACTIVE_LOW. Set the GPIO to 0 here to enable charging since
+	 * the /CE pin is active-low, but not marked as such in the gpio_desc.
+	 */
+	gpiod_set_value(gpiod, 0);
+
+	/* OTG pin */
+	ret = x86_android_tablet_get_gpiod("INT33FF:03", 19, &gpiod);
+	if (ret < 0)
+		return ret;
+
+	gpiod_set_value(gpiod, 0);
+
+	/* Enable the regulators used by the touchscreen */
+	intel_soc_pmic_exec_mipi_pmic_seq_element(0x6e, 0x9b, 0x02, 0xff);
+	intel_soc_pmic_exec_mipi_pmic_seq_element(0x6e, 0xa0, 0x02, 0xff);
+
+	return 0;
+}
+
+static struct gpiod_lookup_table lenovo_yt3_hideep_gpios = {
+	.dev_id = "i2c-hideep_ts",
+	.table = {
+		GPIO_LOOKUP("INT33FF:00", 7, "reset", GPIO_ACTIVE_LOW),
+		{ }
+	},
+};
+
+static struct gpiod_lookup_table * const lenovo_yt3_gpios[] = {
+	&lenovo_yt3_hideep_gpios,
+	NULL
+};
+
+const struct x86_dev_info lenovo_yt3_info __initconst = {
+	.i2c_client_info = lenovo_yt3_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(lenovo_yt3_i2c_clients),
+	.gpiod_lookup_tables = lenovo_yt3_gpios,
+	.init = lenovo_yt3_init,
+};
diff --git a/drivers/platform/x86/x86-android-tablets/other.c b/drivers/platform/x86/x86-android-tablets/other.c
new file mode 100644
index 000000000000..83cd7e16c84c
--- /dev/null
+++ b/drivers/platform/x86/x86-android-tablets/other.c
@@ -0,0 +1,522 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * DMI based code to deal with broken DSDTs on X86 tablets which ship with
+ * Android as (part of) the factory image. The factory kernels shipped on these
+ * devices typically have a bunch of things hardcoded, rather than specified
+ * in their DSDT.
+ *
+ * Copyright (C) 2021-2023 Hans de Goede <hdegoede@redhat.com>
+ */
+
+#include <linux/acpi.h>
+#include <linux/gpio/machine.h>
+#include <linux/input.h>
+#include <linux/platform_device.h>
+
+#include "shared-psy-info.h"
+#include "x86-android-tablets.h"
+
+/* Acer Iconia One 7 B1-750 has an Android factory img with everything hardcoded */
+static const char * const acer_b1_750_mount_matrix[] = {
+	"-1", "0", "0",
+	"0", "1", "0",
+	"0", "0", "1"
+};
+
+static const struct property_entry acer_b1_750_bma250e_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", acer_b1_750_mount_matrix),
+	{ }
+};
+
+static const struct software_node acer_b1_750_bma250e_node = {
+	.properties = acer_b1_750_bma250e_props,
+};
+
+static const struct x86_i2c_client_info acer_b1_750_i2c_clients[] __initconst = {
+	{
+		/* Novatek NVT-ts touchscreen */
+		.board_info = {
+			.type = "NVT-ts",
+			.addr = 0x34,
+			.dev_name = "NVT-ts",
+		},
+		.adapter_path = "\\_SB_.I2C4",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FC:02",
+			.index = 3,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	}, {
+		/* BMA250E accelerometer */
+		.board_info = {
+			.type = "bma250e",
+			.addr = 0x18,
+			.swnode = &acer_b1_750_bma250e_node,
+		},
+		.adapter_path = "\\_SB_.I2C3",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FC:02",
+			.index = 25,
+			.trigger = ACPI_LEVEL_SENSITIVE,
+			.polarity = ACPI_ACTIVE_HIGH,
+		},
+	},
+};
+
+static struct gpiod_lookup_table acer_b1_750_goodix_gpios = {
+	.dev_id = "i2c-NVT-ts",
+	.table = {
+		GPIO_LOOKUP("INT33FC:01", 26, "reset", GPIO_ACTIVE_LOW),
+		{ }
+	},
+};
+
+static struct gpiod_lookup_table * const acer_b1_750_gpios[] = {
+	&acer_b1_750_goodix_gpios,
+	&int3496_reference_gpios,
+	NULL
+};
+
+const struct x86_dev_info acer_b1_750_info __initconst = {
+	.i2c_client_info = acer_b1_750_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(acer_b1_750_i2c_clients),
+	.pdev_info = int3496_pdevs,
+	.pdev_count = 1,
+	.gpiod_lookup_tables = acer_b1_750_gpios,
+};
+
+/*
+ * Advantech MICA-071
+ * This is a standard Windows tablet, but it has an extra "quick launch" button
+ * which is not described in the ACPI tables in anyway.
+ * Use the x86-android-tablets infra to create a gpio-button device for this.
+ */
+static struct x86_gpio_button advantech_mica_071_button = {
+	.button = {
+		.code = KEY_PROG1,
+		.active_low = true,
+		.desc = "prog1_key",
+		.type = EV_KEY,
+		.wakeup = false,
+		.debounce_interval = 50,
+	},
+	.chip = "INT33FC:00",
+	.pin = 2,
+};
+
+const struct x86_dev_info advantech_mica_071_info __initconst = {
+	.gpio_button = &advantech_mica_071_button,
+};
+
+/*
+ * When booted with the BIOS set to Android mode the Chuwi Hi8 (CWI509) DSDT
+ * contains a whole bunch of bogus ACPI I2C devices and is missing entries
+ * for the touchscreen and the accelerometer.
+ */
+static const struct property_entry chuwi_hi8_gsl1680_props[] = {
+	PROPERTY_ENTRY_U32("touchscreen-size-x", 1665),
+	PROPERTY_ENTRY_U32("touchscreen-size-y", 1140),
+	PROPERTY_ENTRY_BOOL("touchscreen-swapped-x-y"),
+	PROPERTY_ENTRY_BOOL("silead,home-button"),
+	PROPERTY_ENTRY_STRING("firmware-name", "gsl1680-chuwi-hi8.fw"),
+	{ }
+};
+
+static const struct software_node chuwi_hi8_gsl1680_node = {
+	.properties = chuwi_hi8_gsl1680_props,
+};
+
+static const char * const chuwi_hi8_mount_matrix[] = {
+	"1", "0", "0",
+	"0", "-1", "0",
+	"0", "0", "1"
+};
+
+static const struct property_entry chuwi_hi8_bma250e_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", chuwi_hi8_mount_matrix),
+	{ }
+};
+
+static const struct software_node chuwi_hi8_bma250e_node = {
+	.properties = chuwi_hi8_bma250e_props,
+};
+
+static const struct x86_i2c_client_info chuwi_hi8_i2c_clients[] __initconst = {
+	{
+		/* Silead touchscreen */
+		.board_info = {
+			.type = "gsl1680",
+			.addr = 0x40,
+			.swnode = &chuwi_hi8_gsl1680_node,
+		},
+		.adapter_path = "\\_SB_.I2C4",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_APIC,
+			.index = 0x44,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_HIGH,
+		},
+	}, {
+		/* BMA250E accelerometer */
+		.board_info = {
+			.type = "bma250e",
+			.addr = 0x18,
+			.swnode = &chuwi_hi8_bma250e_node,
+		},
+		.adapter_path = "\\_SB_.I2C3",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FC:02",
+			.index = 23,
+			.trigger = ACPI_LEVEL_SENSITIVE,
+			.polarity = ACPI_ACTIVE_HIGH,
+		},
+	},
+};
+
+static int __init chuwi_hi8_init(void)
+{
+	/*
+	 * Avoid the acpi_unregister_gsi() call in x86_acpi_irq_helper_get()
+	 * breaking the touchscreen + logging various errors when the Windows
+	 * BIOS is used.
+	 */
+	if (acpi_dev_present("MSSL0001", NULL, 1))
+		return -ENODEV;
+
+	return 0;
+}
+
+const struct x86_dev_info chuwi_hi8_info __initconst = {
+	.i2c_client_info = chuwi_hi8_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(chuwi_hi8_i2c_clients),
+	.init = chuwi_hi8_init,
+};
+
+#define CZC_EC_EXTRA_PORT	0x68
+#define CZC_EC_ANDROID_KEYS	0x63
+
+static int __init czc_p10t_init(void)
+{
+	/*
+	 * The device boots up in "Windows 7" mode, when the home button sends a
+	 * Windows specific key sequence (Left Meta + D) and the second button
+	 * sends an unknown one while also toggling the Radio Kill Switch.
+	 * This is a surprising behavior when the second button is labeled "Back".
+	 *
+	 * The vendor-supplied Android-x86 build switches the device to a "Android"
+	 * mode by writing value 0x63 to the I/O port 0x68. This just seems to just
+	 * set bit 6 on address 0x96 in the EC region; switching the bit directly
+	 * seems to achieve the same result. It uses a "p10t_switcher" to do the
+	 * job. It doesn't seem to be able to do anything else, and no other use
+	 * of the port 0x68 is known.
+	 *
+	 * In the Android mode, the home button sends just a single scancode,
+	 * which can be handled in Linux userspace more reasonably and the back
+	 * button only sends a scancode without toggling the kill switch.
+	 * The scancode can then be mapped either to Back or RF Kill functionality
+	 * in userspace, depending on how the button is labeled on that particular
+	 * model.
+	 */
+	outb(CZC_EC_ANDROID_KEYS, CZC_EC_EXTRA_PORT);
+	return 0;
+}
+
+const struct x86_dev_info czc_p10t __initconst = {
+	.init = czc_p10t_init,
+};
+
+/* Medion Lifetab S10346 tablets have an Android factory img with everything hardcoded */
+static const char * const medion_lifetab_s10346_accel_mount_matrix[] = {
+	"0", "1", "0",
+	"1", "0", "0",
+	"0", "0", "1"
+};
+
+static const struct property_entry medion_lifetab_s10346_accel_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", medion_lifetab_s10346_accel_mount_matrix),
+	{ }
+};
+
+static const struct software_node medion_lifetab_s10346_accel_node = {
+	.properties = medion_lifetab_s10346_accel_props,
+};
+
+/* Note the LCD panel is mounted upside down, this is correctly indicated in the VBT */
+static const struct property_entry medion_lifetab_s10346_touchscreen_props[] = {
+	PROPERTY_ENTRY_BOOL("touchscreen-inverted-x"),
+	PROPERTY_ENTRY_BOOL("touchscreen-swapped-x-y"),
+	{ }
+};
+
+static const struct software_node medion_lifetab_s10346_touchscreen_node = {
+	.properties = medion_lifetab_s10346_touchscreen_props,
+};
+
+static const struct x86_i2c_client_info medion_lifetab_s10346_i2c_clients[] __initconst = {
+	{
+		/* kxtj21009 accel */
+		.board_info = {
+			.type = "kxtj21009",
+			.addr = 0x0f,
+			.dev_name = "kxtj21009",
+			.swnode = &medion_lifetab_s10346_accel_node,
+		},
+		.adapter_path = "\\_SB_.I2C3",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FC:02",
+			.index = 23,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_HIGH,
+		},
+	}, {
+		/* goodix touchscreen */
+		.board_info = {
+			.type = "GDIX1001:00",
+			.addr = 0x14,
+			.dev_name = "goodix_ts",
+			.swnode = &medion_lifetab_s10346_touchscreen_node,
+		},
+		.adapter_path = "\\_SB_.I2C4",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_APIC,
+			.index = 0x44,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	},
+};
+
+static struct gpiod_lookup_table medion_lifetab_s10346_goodix_gpios = {
+	.dev_id = "i2c-goodix_ts",
+	.table = {
+		GPIO_LOOKUP("INT33FC:01", 26, "reset", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("INT33FC:02", 3, "irq", GPIO_ACTIVE_HIGH),
+		{ }
+	},
+};
+
+static struct gpiod_lookup_table * const medion_lifetab_s10346_gpios[] = {
+	&medion_lifetab_s10346_goodix_gpios,
+	NULL
+};
+
+const struct x86_dev_info medion_lifetab_s10346_info __initconst = {
+	.i2c_client_info = medion_lifetab_s10346_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(medion_lifetab_s10346_i2c_clients),
+	.gpiod_lookup_tables = medion_lifetab_s10346_gpios,
+};
+
+/* Nextbook Ares 8 tablets have an Android factory img with everything hardcoded */
+static const char * const nextbook_ares8_accel_mount_matrix[] = {
+	"0", "-1", "0",
+	"-1", "0", "0",
+	"0", "0", "1"
+};
+
+static const struct property_entry nextbook_ares8_accel_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", nextbook_ares8_accel_mount_matrix),
+	{ }
+};
+
+static const struct software_node nextbook_ares8_accel_node = {
+	.properties = nextbook_ares8_accel_props,
+};
+
+static const struct property_entry nextbook_ares8_touchscreen_props[] = {
+	PROPERTY_ENTRY_U32("touchscreen-size-x", 800),
+	PROPERTY_ENTRY_U32("touchscreen-size-y", 1280),
+	{ }
+};
+
+static const struct software_node nextbook_ares8_touchscreen_node = {
+	.properties = nextbook_ares8_touchscreen_props,
+};
+
+static const struct x86_i2c_client_info nextbook_ares8_i2c_clients[] __initconst = {
+	{
+		/* Freescale MMA8653FC accel */
+		.board_info = {
+			.type = "mma8653",
+			.addr = 0x1d,
+			.dev_name = "mma8653",
+			.swnode = &nextbook_ares8_accel_node,
+		},
+		.adapter_path = "\\_SB_.I2C3",
+	}, {
+		/* FT5416DQ9 touchscreen controller */
+		.board_info = {
+			.type = "edt-ft5x06",
+			.addr = 0x38,
+			.dev_name = "ft5416",
+			.swnode = &nextbook_ares8_touchscreen_node,
+		},
+		.adapter_path = "\\_SB_.I2C4",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_GPIOINT,
+			.chip = "INT33FC:02",
+			.index = 3,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_LOW,
+		},
+	},
+};
+
+static struct gpiod_lookup_table * const nextbook_ares8_gpios[] = {
+	&int3496_reference_gpios,
+	NULL
+};
+
+const struct x86_dev_info nextbook_ares8_info __initconst = {
+	.i2c_client_info = nextbook_ares8_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(nextbook_ares8_i2c_clients),
+	.pdev_info = int3496_pdevs,
+	.pdev_count = 1,
+	.gpiod_lookup_tables = nextbook_ares8_gpios,
+	.invalid_aei_gpiochip = "INT33FC:02",
+};
+
+/*
+ * Peaq C1010
+ * This is a standard Windows tablet, but it has a special Dolby button.
+ * This button has a WMI interface, but that is broken. Instead of trying to
+ * use the broken WMI interface, instantiate a gpio_keys device for this.
+ */
+static struct x86_gpio_button peaq_c1010_button = {
+	.button = {
+		.code = KEY_SOUND,
+		.active_low = true,
+		.desc = "dolby_key",
+		.type = EV_KEY,
+		.wakeup = false,
+		.debounce_interval = 50,
+	},
+	.chip = "INT33FC:00",
+	.pin = 3,
+};
+
+const struct x86_dev_info peaq_c1010_info __initconst = {
+	.gpio_button = &peaq_c1010_button,
+	/*
+	 * Move the ACPI event handler used by the broken WMI interface out of
+	 * the way. This is the only event handler on INT33FC:00.
+	 */
+	.invalid_aei_gpiochip = "INT33FC:00",
+};
+
+/*
+ * Whitelabel (sold as various brands) TM800A550L tablets.
+ * These tablet's DSDT contains a whole bunch of bogus ACPI I2C devices
+ * (removed through acpi_quirk_skip_i2c_client_enumeration()) and
+ * the touchscreen fwnode has the wrong GPIOs.
+ */
+static const char * const whitelabel_tm800a550l_accel_mount_matrix[] = {
+	"-1", "0", "0",
+	"0", "1", "0",
+	"0", "0", "1"
+};
+
+static const struct property_entry whitelabel_tm800a550l_accel_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("mount-matrix", whitelabel_tm800a550l_accel_mount_matrix),
+	{ }
+};
+
+static const struct software_node whitelabel_tm800a550l_accel_node = {
+	.properties = whitelabel_tm800a550l_accel_props,
+};
+
+static const struct property_entry whitelabel_tm800a550l_goodix_props[] = {
+	PROPERTY_ENTRY_STRING("firmware-name", "gt912-tm800a550l.fw"),
+	PROPERTY_ENTRY_STRING("goodix,config-name", "gt912-tm800a550l.cfg"),
+	PROPERTY_ENTRY_U32("goodix,main-clk", 54),
+	{ }
+};
+
+static const struct software_node whitelabel_tm800a550l_goodix_node = {
+	.properties = whitelabel_tm800a550l_goodix_props,
+};
+
+static const struct x86_i2c_client_info whitelabel_tm800a550l_i2c_clients[] __initconst = {
+	{
+		/* goodix touchscreen */
+		.board_info = {
+			.type = "GDIX1001:00",
+			.addr = 0x14,
+			.dev_name = "goodix_ts",
+			.swnode = &whitelabel_tm800a550l_goodix_node,
+		},
+		.adapter_path = "\\_SB_.I2C2",
+		.irq_data = {
+			.type = X86_ACPI_IRQ_TYPE_APIC,
+			.index = 0x44,
+			.trigger = ACPI_EDGE_SENSITIVE,
+			.polarity = ACPI_ACTIVE_HIGH,
+		},
+	}, {
+		/* kxcj91008 accel */
+		.board_info = {
+			.type = "kxcj91008",
+			.addr = 0x0f,
+			.dev_name = "kxcj91008",
+			.swnode = &whitelabel_tm800a550l_accel_node,
+		},
+		.adapter_path = "\\_SB_.I2C3",
+	},
+};
+
+static struct gpiod_lookup_table whitelabel_tm800a550l_goodix_gpios = {
+	.dev_id = "i2c-goodix_ts",
+	.table = {
+		GPIO_LOOKUP("INT33FC:01", 26, "reset", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("INT33FC:02", 3, "irq", GPIO_ACTIVE_HIGH),
+		{ }
+	},
+};
+
+static struct gpiod_lookup_table * const whitelabel_tm800a550l_gpios[] = {
+	&whitelabel_tm800a550l_goodix_gpios,
+	NULL
+};
+
+const struct x86_dev_info whitelabel_tm800a550l_info __initconst = {
+	.i2c_client_info = whitelabel_tm800a550l_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(whitelabel_tm800a550l_i2c_clients),
+	.gpiod_lookup_tables = whitelabel_tm800a550l_gpios,
+};
+
+/*
+ * If the EFI bootloader is not Xiaomi's own signed Android loader, then the
+ * Xiaomi Mi Pad 2 X86 tablet sets OSID in the DSDT to 1 (Windows), causing
+ * a bunch of devices to be hidden.
+ *
+ * This takes care of instantiating the hidden devices manually.
+ */
+static const struct x86_i2c_client_info xiaomi_mipad2_i2c_clients[] __initconst = {
+	{
+		/* BQ27520 fuel-gauge */
+		.board_info = {
+			.type = "bq27520",
+			.addr = 0x55,
+			.dev_name = "bq27520",
+			.swnode = &fg_bq25890_supply_node,
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C1",
+	}, {
+		/* KTD2026 RGB notification LED controller */
+		.board_info = {
+			.type = "ktd2026",
+			.addr = 0x30,
+			.dev_name = "ktd2026",
+		},
+		.adapter_path = "\\_SB_.PCI0.I2C3",
+	},
+};
+
+const struct x86_dev_info xiaomi_mipad2_info __initconst = {
+	.i2c_client_info = xiaomi_mipad2_i2c_clients,
+	.i2c_client_count = ARRAY_SIZE(xiaomi_mipad2_i2c_clients),
+};
diff --git a/drivers/platform/x86/x86-android-tablets/shared-psy-info.c b/drivers/platform/x86/x86-android-tablets/shared-psy-info.c
new file mode 100644
index 000000000000..d2d0aa51bc3f
--- /dev/null
+++ b/drivers/platform/x86/x86-android-tablets/shared-psy-info.c
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Shared psy info for X86 tablets which ship with Android as the factory image
+ * and which have broken DSDT tables. The factory kernels shipped on these
+ * devices typically have a bunch of things hardcoded, rather than specified
+ * in their DSDT.
+ *
+ * Copyright (C) 2021-2023 Hans de Goede <hdegoede@redhat.com>
+ */
+
+#include <linux/gpio/machine.h>
+#include <linux/platform_device.h>
+#include <linux/power/bq24190_charger.h>
+#include <linux/property.h>
+#include <linux/regulator/machine.h>
+
+#include "shared-psy-info.h"
+
+/* Generic / shared charger / battery settings */
+const char * const tusb1211_chg_det_psy[] = { "tusb1211-charger-detect" };
+const char * const bq24190_psy[] = { "bq24190-charger" };
+const char * const bq25890_psy[] = { "bq25890-charger-0" };
+
+static const struct property_entry fg_bq24190_supply_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", bq24190_psy),
+	{ }
+};
+
+const struct software_node fg_bq24190_supply_node = {
+	.properties = fg_bq24190_supply_props,
+};
+
+static const struct property_entry fg_bq25890_supply_props[] = {
+	PROPERTY_ENTRY_STRING_ARRAY("supplied-from", bq25890_psy),
+	{ }
+};
+
+const struct software_node fg_bq25890_supply_node = {
+	.properties = fg_bq25890_supply_props,
+};
+
+/* LiPo HighVoltage (max 4.35V) settings used by most devs with a HV bat. */
+static const struct property_entry generic_lipo_hv_4v35_battery_props[] = {
+	PROPERTY_ENTRY_STRING("compatible", "simple-battery"),
+	PROPERTY_ENTRY_STRING("device-chemistry", "lithium-ion"),
+	PROPERTY_ENTRY_U32("precharge-current-microamp", 256000),
+	PROPERTY_ENTRY_U32("charge-term-current-microamp", 128000),
+	PROPERTY_ENTRY_U32("constant-charge-current-max-microamp", 1856000),
+	PROPERTY_ENTRY_U32("constant-charge-voltage-max-microvolt", 4352000),
+	PROPERTY_ENTRY_U32("factory-internal-resistance-micro-ohms", 150000),
+	{ }
+};
+
+const struct software_node generic_lipo_hv_4v35_battery_node = {
+	.properties = generic_lipo_hv_4v35_battery_props,
+};
+
+/* For enabling the bq24190 5V boost based on id-pin */
+static struct regulator_consumer_supply intel_int3496_consumer = {
+	.supply = "vbus",
+	.dev_name = "intel-int3496",
+};
+
+static const struct regulator_init_data bq24190_vbus_init_data = {
+	.constraints = {
+		.name = "bq24190_vbus",
+		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
+	},
+	.consumer_supplies = &intel_int3496_consumer,
+	.num_consumer_supplies = 1,
+};
+
+struct bq24190_platform_data bq24190_pdata = {
+	.regulator_init_data = &bq24190_vbus_init_data,
+};
+
+const char * const bq24190_modules[] __initconst = {
+	"intel_crystal_cove_charger", /* For the bq24190 IRQ */
+	"bq24190_charger",            /* For the Vbus regulator for intel-int3496 */
+	NULL
+};
+
+/* Generic pdevs array and gpio-lookups for micro USB ID pin handling */
+const struct platform_device_info int3496_pdevs[] __initconst = {
+	{
+		/* For micro USB ID pin handling */
+		.name = "intel-int3496",
+		.id = PLATFORM_DEVID_NONE,
+	},
+};
+
+struct gpiod_lookup_table int3496_reference_gpios = {
+	.dev_id = "intel-int3496",
+	.table = {
+		GPIO_LOOKUP("INT33FC:01", 15, "vbus", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("INT33FC:02", 1, "mux", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("INT33FC:02", 18, "id", GPIO_ACTIVE_HIGH),
+		{ }
+	},
+};
diff --git a/drivers/platform/x86/x86-android-tablets/shared-psy-info.h b/drivers/platform/x86/x86-android-tablets/shared-psy-info.h
new file mode 100644
index 000000000000..c2d2968cddc2
--- /dev/null
+++ b/drivers/platform/x86/x86-android-tablets/shared-psy-info.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * Shared psy info for X86 tablets which ship with Android as the factory image
+ * and which have broken DSDT tables. The factory kernels shipped on these
+ * devices typically have a bunch of things hardcoded, rather than specified
+ * in their DSDT.
+ *
+ * Copyright (C) 2021-2023 Hans de Goede <hdegoede@redhat.com>
+ */
+#ifndef __PDX86_SHARED_PSY_INFO_H
+#define __PDX86_SHARED_PSY_INFO_H
+
+struct bq24190_platform_data;
+struct gpiod_lookup_table;
+struct platform_device_info;
+struct software_node;
+
+extern const char * const tusb1211_chg_det_psy[];
+extern const char * const bq24190_psy[];
+extern const char * const bq25890_psy[];
+
+extern const struct software_node fg_bq24190_supply_node;
+extern const struct software_node fg_bq25890_supply_node;
+extern const struct software_node generic_lipo_hv_4v35_battery_node;
+
+extern struct bq24190_platform_data bq24190_pdata;
+extern const char * const bq24190_modules[];
+
+extern const struct platform_device_info int3496_pdevs[];
+extern struct gpiod_lookup_table int3496_reference_gpios;
+
+#endif
diff --git a/drivers/platform/x86/x86-android-tablets/x86-android-tablets.h b/drivers/platform/x86/x86-android-tablets/x86-android-tablets.h
new file mode 100644
index 000000000000..b6802d75dbdd
--- /dev/null
+++ b/drivers/platform/x86/x86-android-tablets/x86-android-tablets.h
@@ -0,0 +1,108 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * DMI based code to deal with broken DSDTs on X86 tablets which ship with
+ * Android as (part of) the factory image. The factory kernels shipped on these
+ * devices typically have a bunch of things hardcoded, rather than specified
+ * in their DSDT.
+ *
+ * Copyright (C) 2021-2023 Hans de Goede <hdegoede@redhat.com>
+ */
+#ifndef __PDX86_X86_ANDROID_TABLETS_H
+#define __PDX86_X86_ANDROID_TABLETS_H
+
+#include <linux/gpio_keys.h>
+#include <linux/i2c.h>
+#include <linux/irqdomain_defs.h>
+
+struct gpio_desc;
+struct gpiod_lookup_table;
+struct platform_device_info;
+struct software_node;
+
+/*
+ * Helpers to get Linux IRQ numbers given a description of the IRQ source
+ * (either IOAPIC index, or GPIO chip name + pin-number).
+ */
+enum x86_acpi_irq_type {
+	X86_ACPI_IRQ_TYPE_NONE,
+	X86_ACPI_IRQ_TYPE_APIC,
+	X86_ACPI_IRQ_TYPE_GPIOINT,
+	X86_ACPI_IRQ_TYPE_PMIC,
+};
+
+struct x86_acpi_irq_data {
+	char *chip;   /* GPIO chip label (GPIOINT) or PMIC ACPI path (PMIC) */
+	enum x86_acpi_irq_type type;
+	enum irq_domain_bus_token domain;
+	int index;
+	int trigger;  /* ACPI_EDGE_SENSITIVE / ACPI_LEVEL_SENSITIVE */
+	int polarity; /* ACPI_ACTIVE_HIGH / ACPI_ACTIVE_LOW / ACPI_ACTIVE_BOTH */
+};
+
+/* Structs to describe devices to instantiate */
+struct x86_i2c_client_info {
+	struct i2c_board_info board_info;
+	char *adapter_path;
+	struct x86_acpi_irq_data irq_data;
+};
+
+struct x86_serdev_info {
+	const char *ctrl_hid;
+	const char *ctrl_uid;
+	const char *ctrl_devname;
+	/*
+	 * ATM the serdev core only supports of or ACPI matching; and sofar all
+	 * Android x86 tablets DSDTs have usable serdev nodes, but sometimes
+	 * under the wrong controller. So we just tie the existing serdev ACPI
+	 * node to the right controller.
+	 */
+	const char *serdev_hid;
+};
+
+struct x86_gpio_button {
+	struct gpio_keys_button button;
+	const char *chip;
+	int pin;
+};
+
+struct x86_dev_info {
+	char *invalid_aei_gpiochip;
+	const char * const *modules;
+	const struct software_node *bat_swnode;
+	struct gpiod_lookup_table * const *gpiod_lookup_tables;
+	const struct x86_i2c_client_info *i2c_client_info;
+	const struct platform_device_info *pdev_info;
+	const struct x86_serdev_info *serdev_info;
+	struct x86_gpio_button *gpio_button;
+	int i2c_client_count;
+	int pdev_count;
+	int serdev_count;
+	int (*init)(void);
+	void (*exit)(void);
+};
+
+int x86_android_tablet_get_gpiod(const char *label, int pin, struct gpio_desc **desc);
+int x86_acpi_irq_helper_get(const struct x86_acpi_irq_data *data);
+
+/*
+ * Extern declarations of x86_dev_info structs so there can be a single
+ * MODULE_DEVICE_TABLE(dmi, ...), while splitting the board descriptions.
+ */
+extern const struct x86_dev_info acer_b1_750_info;
+extern const struct x86_dev_info advantech_mica_071_info;
+extern const struct x86_dev_info asus_me176c_info;
+extern const struct x86_dev_info asus_tf103c_info;
+extern const struct x86_dev_info chuwi_hi8_info;
+extern const struct x86_dev_info czc_p10t;
+extern const struct x86_dev_info lenovo_yogabook_x90_info;
+extern const struct x86_dev_info lenovo_yogabook_x91_info;
+extern const struct x86_dev_info lenovo_yoga_tab2_830_1050_info;
+extern const struct x86_dev_info lenovo_yt3_info;
+extern const struct x86_dev_info medion_lifetab_s10346_info;
+extern const struct x86_dev_info nextbook_ares8_info;
+extern const struct x86_dev_info peaq_c1010_info;
+extern const struct x86_dev_info whitelabel_tm800a550l_info;
+extern const struct x86_dev_info xiaomi_mipad2_info;
+extern const struct dmi_system_id x86_android_tablet_ids[];
+
+#endif
diff --git a/drivers/platform/x86/xo1-rfkill.c b/drivers/platform/x86/xo1-rfkill.c
index cb3253c10ef3..e64d5646b4c7 100644
--- a/drivers/platform/x86/xo1-rfkill.c
+++ b/drivers/platform/x86/xo1-rfkill.c
@@ -56,12 +56,11 @@ static int xo1_rfkill_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int xo1_rfkill_remove(struct platform_device *pdev)
+static void xo1_rfkill_remove(struct platform_device *pdev)
 {
 	struct rfkill *rfk = platform_get_drvdata(pdev);
 	rfkill_unregister(rfk);
 	rfkill_destroy(rfk);
-	return 0;
 }
 
 static struct platform_driver xo1_rfkill_driver = {
@@ -69,7 +68,7 @@ static struct platform_driver xo1_rfkill_driver = {
 		.name = "xo1-rfkill",
 	},
 	.probe		= xo1_rfkill_probe,
-	.remove		= xo1_rfkill_remove,
+	.remove_new	= xo1_rfkill_remove,
 };
 
 module_platform_driver(xo1_rfkill_driver);
diff --git a/drivers/pnp/quirks.c b/drivers/pnp/quirks.c
index ac98b9919029..6085a1471de2 100644
--- a/drivers/pnp/quirks.c
+++ b/drivers/pnp/quirks.c
@@ -229,8 +229,7 @@ static void quirk_ad1815_mpu_resources(struct pnp_dev *dev)
 static void quirk_system_pci_resources(struct pnp_dev *dev)
 {
 	struct pci_dev *pdev = NULL;
-	struct resource *res;
-	resource_size_t pnp_start, pnp_end, pci_start, pci_end;
+	struct resource *res, *r;
 	int i, j;
 
 	/*
@@ -243,32 +242,26 @@ static void quirk_system_pci_resources(struct pnp_dev *dev)
 	 * so they won't be claimed by the PNP system driver.
 	 */
 	for_each_pci_dev(pdev) {
-		for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
-			unsigned long flags, type;
+		pci_dev_for_each_resource(pdev, r, i) {
+			unsigned long type = resource_type(r);
 
-			flags = pci_resource_flags(pdev, i);
-			type = flags & (IORESOURCE_IO | IORESOURCE_MEM);
-			if (!type || pci_resource_len(pdev, i) == 0)
+			if (!(type == IORESOURCE_IO || type == IORESOURCE_MEM) ||
+			    resource_size(r) == 0)
 				continue;
 
-			if (flags & IORESOURCE_UNSET)
+			if (r->flags & IORESOURCE_UNSET)
 				continue;
 
-			pci_start = pci_resource_start(pdev, i);
-			pci_end = pci_resource_end(pdev, i);
 			for (j = 0;
 			     (res = pnp_get_resource(dev, type, j)); j++) {
 				if (res->start == 0 && res->end == 0)
 					continue;
 
-				pnp_start = res->start;
-				pnp_end = res->end;
-
 				/*
 				 * If the PNP region doesn't overlap the PCI
 				 * region at all, there's no problem.
 				 */
-				if (pnp_end < pci_start || pnp_start > pci_end)
+				if (!resource_overlaps(res, r))
 					continue;
 
 				/*
@@ -278,8 +271,7 @@ static void quirk_system_pci_resources(struct pnp_dev *dev)
 				 * PNP device describes a bridge with PCI
 				 * behind it.
 				 */
-				if (pnp_start <= pci_start &&
-				    pnp_end >= pci_end)
+				if (res->start <= r->start && res->end >= r->end)
 					continue;
 
 				/*
@@ -288,9 +280,8 @@ static void quirk_system_pci_resources(struct pnp_dev *dev)
 				 * driver from requesting its resources.
 				 */
 				dev_warn(&dev->dev,
-					 "disabling %pR because it overlaps "
-					 "%s BAR %d %pR\n", res,
-					 pci_name(pdev), i, &pdev->resource[i]);
+					 "disabling %pR because it overlaps %s BAR %d %pR\n",
+					 res, pci_name(pdev), i, r);
 				res->flags |= IORESOURCE_DISABLED;
 			}
 		}
diff --git a/drivers/power/reset/as3722-poweroff.c b/drivers/power/reset/as3722-poweroff.c
index 661e1c67f82e..80edff1a556f 100644
--- a/drivers/power/reset/as3722-poweroff.c
+++ b/drivers/power/reset/as3722-poweroff.c
@@ -84,4 +84,3 @@ module_platform_driver(as3722_poweroff_driver);
 MODULE_DESCRIPTION("Power off driver for ams AS3722 PMIC Device");
 MODULE_ALIAS("platform:as3722-power-off");
 MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/power/reset/gpio-poweroff.c b/drivers/power/reset/gpio-poweroff.c
index 1c5af2fef142..84b3c3528afa 100644
--- a/drivers/power/reset/gpio-poweroff.c
+++ b/drivers/power/reset/gpio-poweroff.c
@@ -105,5 +105,4 @@ module_platform_driver(gpio_poweroff_driver);
 
 MODULE_AUTHOR("Jamie Lentin <jm@lentin.co.uk>");
 MODULE_DESCRIPTION("GPIO poweroff driver");
-MODULE_LICENSE("GPL v2");
 MODULE_ALIAS("platform:poweroff-gpio");
diff --git a/drivers/power/reset/gpio-restart.c b/drivers/power/reset/gpio-restart.c
index 5466eeea261c..35d981d5e6c8 100644
--- a/drivers/power/reset/gpio-restart.c
+++ b/drivers/power/reset/gpio-restart.c
@@ -139,4 +139,3 @@ module_platform_driver(gpio_restart_driver);
 
 MODULE_AUTHOR("David Riley <davidriley@chromium.org>");
 MODULE_DESCRIPTION("GPIO restart driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/power/reset/keystone-reset.c b/drivers/power/reset/keystone-reset.c
index c720112db704..83a4e1c9bf94 100644
--- a/drivers/power/reset/keystone-reset.c
+++ b/drivers/power/reset/keystone-reset.c
@@ -169,5 +169,4 @@ module_platform_driver(rsctrl_driver);
 
 MODULE_AUTHOR("Ivan Khoronzhuk <ivan.khoronzhuk@ti.com>");
 MODULE_DESCRIPTION("Texas Instruments keystone reset driver");
-MODULE_LICENSE("GPL v2");
 MODULE_ALIAS("platform:" KBUILD_MODNAME);
diff --git a/drivers/power/reset/ltc2952-poweroff.c b/drivers/power/reset/ltc2952-poweroff.c
index 65d9528cc989..eea05921a054 100644
--- a/drivers/power/reset/ltc2952-poweroff.c
+++ b/drivers/power/reset/ltc2952-poweroff.c
@@ -317,4 +317,3 @@ module_platform_driver(ltc2952_poweroff_driver);
 
 MODULE_AUTHOR("René Moll <rene.moll@xsens.com>");
 MODULE_DESCRIPTION("LTC PowerPath power-off driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/power/reset/mt6323-poweroff.c b/drivers/power/reset/mt6323-poweroff.c
index d90e76fcb938..108167f7738b 100644
--- a/drivers/power/reset/mt6323-poweroff.c
+++ b/drivers/power/reset/mt6323-poweroff.c
@@ -97,4 +97,3 @@ module_platform_driver(mt6323_pwrc_driver);
 
 MODULE_DESCRIPTION("Poweroff driver for MT6323 PMIC");
 MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/power/reset/regulator-poweroff.c b/drivers/power/reset/regulator-poweroff.c
index 20701203935f..7f87fbb8b051 100644
--- a/drivers/power/reset/regulator-poweroff.c
+++ b/drivers/power/reset/regulator-poweroff.c
@@ -79,5 +79,4 @@ module_platform_driver(regulator_poweroff_driver);
 
 MODULE_AUTHOR("Michael Klein <michael@fossekall.de>");
 MODULE_DESCRIPTION("Regulator poweroff driver");
-MODULE_LICENSE("GPL v2");
 MODULE_ALIAS("platform:poweroff-regulator");
diff --git a/drivers/power/reset/restart-poweroff.c b/drivers/power/reset/restart-poweroff.c
index 04d4228119b2..28f1822db162 100644
--- a/drivers/power/reset/restart-poweroff.c
+++ b/drivers/power/reset/restart-poweroff.c
@@ -59,5 +59,4 @@ module_platform_driver(restart_poweroff_driver);
 
 MODULE_AUTHOR("Andrew Lunn <andrew@lunn.ch");
 MODULE_DESCRIPTION("restart poweroff driver");
-MODULE_LICENSE("GPL v2");
 MODULE_ALIAS("platform:poweroff-restart");
diff --git a/drivers/power/reset/tps65086-restart.c b/drivers/power/reset/tps65086-restart.c
index 78b89f745a3d..5ec819eac7da 100644
--- a/drivers/power/reset/tps65086-restart.c
+++ b/drivers/power/reset/tps65086-restart.c
@@ -95,4 +95,3 @@ module_platform_driver(tps65086_restart_driver);
 
 MODULE_AUTHOR("Emil Renner Berthing <kernel@esmil.dk>");
 MODULE_DESCRIPTION("TPS65086 restart driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c
index f3d7c1da299f..a980d60c437b 100644
--- a/drivers/power/supply/power_supply_core.c
+++ b/drivers/power/supply/power_supply_core.c
@@ -1142,7 +1142,7 @@ static int power_supply_read_temp(struct thermal_zone_device *tzd,
 	int ret;
 
 	WARN_ON(tzd == NULL);
-	psy = tzd->devdata;
+	psy = thermal_zone_device_priv(tzd);
 	ret = power_supply_get_property(psy, POWER_SUPPLY_PROP_TEMP, &val);
 	if (ret)
 		return ret;
@@ -1462,7 +1462,7 @@ EXPORT_SYMBOL_GPL(power_supply_get_drvdata);
 
 static int __init power_supply_class_init(void)
 {
-	power_supply_class = class_create(THIS_MODULE, "power_supply");
+	power_supply_class = class_create("power_supply");
 
 	if (IS_ERR(power_supply_class))
 		return PTR_ERR(power_supply_class);
@@ -1485,4 +1485,3 @@ MODULE_DESCRIPTION("Universal power supply monitor class");
 MODULE_AUTHOR("Ian Molton <spyro@f2s.com>, "
 	      "Szabolcs Gyurko, "
 	      "Anton Vorontsov <cbou@mail.ru>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/power/supply/wm97xx_battery.c b/drivers/power/supply/wm97xx_battery.c
index a0e1eaa25d93..f4b190adb335 100644
--- a/drivers/power/supply/wm97xx_battery.c
+++ b/drivers/power/supply/wm97xx_battery.c
@@ -271,6 +271,5 @@ static struct platform_driver wm97xx_bat_driver = {
 
 module_platform_driver(wm97xx_bat_driver);
 
-MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Marek Vasut <marek.vasut@gmail.com>");
 MODULE_DESCRIPTION("WM97xx battery driver");
diff --git a/drivers/pps/pps.c b/drivers/pps/pps.c
index 22a65ad4e46e..5d19baae6a38 100644
--- a/drivers/pps/pps.c
+++ b/drivers/pps/pps.c
@@ -456,7 +456,7 @@ static int __init pps_init(void)
 {
 	int err;
 
-	pps_class = class_create(THIS_MODULE, "pps");
+	pps_class = class_create("pps");
 	if (IS_ERR(pps_class)) {
 		pr_err("failed to allocate class\n");
 		return PTR_ERR(pps_class);
diff --git a/drivers/ps3/ps3av.c b/drivers/ps3/ps3av.c
index 516e6d14d32e..f6c9e56bdba7 100644
--- a/drivers/ps3/ps3av.c
+++ b/drivers/ps3/ps3av.c
@@ -11,13 +11,14 @@
 #include <linux/delay.h>
 #include <linux/notifier.h>
 #include <linux/ioctl.h>
-#include <linux/fb.h>
 #include <linux/slab.h>
 
 #include <asm/firmware.h>
 #include <asm/ps3av.h>
 #include <asm/ps3.h>
 
+#include <video/cmdline.h>
+
 #include "vuart.h"
 
 #define BUFSIZE          4096	/* vuart buf size */
@@ -921,6 +922,7 @@ EXPORT_SYMBOL_GPL(ps3av_audio_mute);
 
 static int ps3av_probe(struct ps3_system_bus_device *dev)
 {
+	const char *mode_option;
 	int res;
 	int id;
 
@@ -968,10 +970,9 @@ static int ps3av_probe(struct ps3_system_bus_device *dev)
 
 	ps3av_get_hw_conf(ps3av);
 
-#ifdef CONFIG_FB
-	if (fb_mode_option && !strcmp(fb_mode_option, "safe"))
+	mode_option = video_get_options(NULL);
+	if (mode_option && !strcmp(mode_option, "safe"))
 		safe_mode = 1;
-#endif /* CONFIG_FB */
 	id = ps3av_auto_videomode(&ps3av->av_hw_conf);
 	if (id < 0) {
 		printk(KERN_ERR "%s: invalid id :%d\n", __func__, id);
diff --git a/drivers/ptp/Kconfig b/drivers/ptp/Kconfig
index fe4971b65c64..b00201d81313 100644
--- a/drivers/ptp/Kconfig
+++ b/drivers/ptp/Kconfig
@@ -186,4 +186,18 @@ config PTP_1588_CLOCK_OCP
 
 	  More information is available at http://www.timingcard.com/
 
+config PTP_DFL_TOD
+	tristate "FPGA DFL ToD Driver"
+	depends on FPGA_DFL
+	depends on PTP_1588_CLOCK
+	help
+	  The DFL (Device Feature List) device driver for the Intel ToD
+	  (Time-of-Day) device in FPGA card. The ToD IP within the FPGA
+	  is exposed as PTP Hardware Clock (PHC) device to the Linux PTP
+	  stack to synchronize the system clock to its ToD information
+	  using phc2sys utility of the Linux PTP stack.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called ptp_dfl_tod.
+
 endmenu
diff --git a/drivers/ptp/Makefile b/drivers/ptp/Makefile
index 28a6fe342d3e..553f18bf3c83 100644
--- a/drivers/ptp/Makefile
+++ b/drivers/ptp/Makefile
@@ -18,3 +18,4 @@ obj-$(CONFIG_PTP_1588_CLOCK_IDTCM)	+= ptp_clockmatrix.o
 obj-$(CONFIG_PTP_1588_CLOCK_IDT82P33)	+= ptp_idt82p33.o
 obj-$(CONFIG_PTP_1588_CLOCK_VMW)	+= ptp_vmw.o
 obj-$(CONFIG_PTP_1588_CLOCK_OCP)	+= ptp_ocp.o
+obj-$(CONFIG_PTP_DFL_TOD)		+= ptp_dfl_tod.o
diff --git a/drivers/ptp/ptp_clock.c b/drivers/ptp/ptp_clock.c
index 62d4d29e7c05..790f9250b381 100644
--- a/drivers/ptp/ptp_clock.c
+++ b/drivers/ptp/ptp_clock.c
@@ -460,7 +460,7 @@ static int __init ptp_init(void)
 {
 	int err;
 
-	ptp_class = class_create(THIS_MODULE, "ptp");
+	ptp_class = class_create("ptp");
 	if (IS_ERR(ptp_class)) {
 		pr_err("ptp: failed to allocate class\n");
 		return PTR_ERR(ptp_class);
diff --git a/drivers/ptp/ptp_dfl_tod.c b/drivers/ptp/ptp_dfl_tod.c
new file mode 100644
index 000000000000..f699d541b360
--- /dev/null
+++ b/drivers/ptp/ptp_dfl_tod.c
@@ -0,0 +1,332 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * DFL device driver for Time-of-Day (ToD) private feature
+ *
+ * Copyright (C) 2023 Intel Corporation
+ */
+
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/dfl.h>
+#include <linux/gcd.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/ptp_clock_kernel.h>
+#include <linux/spinlock.h>
+#include <linux/units.h>
+
+#define FME_FEATURE_ID_TOD		0x22
+
+/* ToD clock register space. */
+#define TOD_CLK_FREQ			0x038
+
+/*
+ * The read sequence of ToD timestamp registers: TOD_NANOSEC, TOD_SECONDSL and
+ * TOD_SECONDSH, because there is a hardware snapshot whenever the TOD_NANOSEC
+ * register is read.
+ *
+ * The ToD IP requires writing registers in the reverse order to the read sequence.
+ * The timestamp is corrected when the TOD_NANOSEC register is written, so the
+ * sequence of write TOD registers: TOD_SECONDSH, TOD_SECONDSL and TOD_NANOSEC.
+ */
+#define TOD_SECONDSH			0x100
+#define TOD_SECONDSL			0x104
+#define TOD_NANOSEC			0x108
+#define TOD_PERIOD			0x110
+#define TOD_ADJUST_PERIOD		0x114
+#define TOD_ADJUST_COUNT		0x118
+#define TOD_DRIFT_ADJUST		0x11c
+#define TOD_DRIFT_ADJUST_RATE		0x120
+#define PERIOD_FRAC_OFFSET		16
+#define SECONDS_MSB			GENMASK_ULL(47, 32)
+#define SECONDS_LSB			GENMASK_ULL(31, 0)
+#define TOD_SECONDSH_SEC_MSB		GENMASK_ULL(15, 0)
+
+#define CAL_SECONDS(m, l)		((FIELD_GET(TOD_SECONDSH_SEC_MSB, (m)) << 32) | (l))
+
+#define TOD_PERIOD_MASK		GENMASK_ULL(19, 0)
+#define TOD_PERIOD_MAX			FIELD_MAX(TOD_PERIOD_MASK)
+#define TOD_PERIOD_MIN			0
+#define TOD_DRIFT_ADJUST_MASK		GENMASK_ULL(15, 0)
+#define TOD_DRIFT_ADJUST_FNS_MAX	FIELD_MAX(TOD_DRIFT_ADJUST_MASK)
+#define TOD_DRIFT_ADJUST_RATE_MAX	TOD_DRIFT_ADJUST_FNS_MAX
+#define TOD_ADJUST_COUNT_MASK		GENMASK_ULL(19, 0)
+#define TOD_ADJUST_COUNT_MAX		FIELD_MAX(TOD_ADJUST_COUNT_MASK)
+#define TOD_ADJUST_INTERVAL_US		10
+#define TOD_ADJUST_MS			\
+		(((TOD_PERIOD_MAX >> 16) + 1) * (TOD_ADJUST_COUNT_MAX + 1))
+#define TOD_ADJUST_MS_MAX		(TOD_ADJUST_MS / MICRO)
+#define TOD_ADJUST_MAX_US		(TOD_ADJUST_MS_MAX * USEC_PER_MSEC)
+#define TOD_MAX_ADJ			(500 * MEGA)
+
+struct dfl_tod {
+	struct ptp_clock_info ptp_clock_ops;
+	struct device *dev;
+	struct ptp_clock *ptp_clock;
+
+	/* ToD Clock address space */
+	void __iomem *tod_ctrl;
+
+	/* ToD clock registers protection */
+	spinlock_t tod_lock;
+};
+
+/*
+ * A fine ToD HW clock offset adjustment. To perform the fine offset adjustment, the
+ * adjust_period and adjust_count argument are used to update the TOD_ADJUST_PERIOD
+ * and TOD_ADJUST_COUNT register for in hardware. The dt->tod_lock spinlock must be
+ * held when calling this function.
+ */
+static int fine_adjust_tod_clock(struct dfl_tod *dt, u32 adjust_period,
+				 u32 adjust_count)
+{
+	void __iomem *base = dt->tod_ctrl;
+	u32 val;
+
+	writel(adjust_period, base + TOD_ADJUST_PERIOD);
+	writel(adjust_count, base + TOD_ADJUST_COUNT);
+
+	/* Wait for present offset adjustment update to complete */
+	return readl_poll_timeout_atomic(base + TOD_ADJUST_COUNT, val, !val, TOD_ADJUST_INTERVAL_US,
+				  TOD_ADJUST_MAX_US);
+}
+
+/*
+ * A coarse ToD HW clock offset adjustment. The coarse time adjustment performs by
+ * adding or subtracting the delta value from the current ToD HW clock time.
+ */
+static int coarse_adjust_tod_clock(struct dfl_tod *dt, s64 delta)
+{
+	u32 seconds_msb, seconds_lsb, nanosec;
+	void __iomem *base = dt->tod_ctrl;
+	u64 seconds, now;
+
+	if (delta == 0)
+		return 0;
+
+	nanosec = readl(base + TOD_NANOSEC);
+	seconds_lsb = readl(base + TOD_SECONDSL);
+	seconds_msb = readl(base + TOD_SECONDSH);
+
+	/* Calculate new time */
+	seconds = CAL_SECONDS(seconds_msb, seconds_lsb);
+	now = seconds * NSEC_PER_SEC + nanosec + delta;
+
+	seconds = div_u64_rem(now, NSEC_PER_SEC, &nanosec);
+	seconds_msb = FIELD_GET(SECONDS_MSB, seconds);
+	seconds_lsb = FIELD_GET(SECONDS_LSB, seconds);
+
+	writel(seconds_msb, base + TOD_SECONDSH);
+	writel(seconds_lsb, base + TOD_SECONDSL);
+	writel(nanosec, base + TOD_NANOSEC);
+
+	return 0;
+}
+
+static int dfl_tod_adjust_fine(struct ptp_clock_info *ptp, long scaled_ppm)
+{
+	struct dfl_tod *dt = container_of(ptp, struct dfl_tod, ptp_clock_ops);
+	u32 tod_period, tod_rem, tod_drift_adjust_fns, tod_drift_adjust_rate;
+	void __iomem *base = dt->tod_ctrl;
+	unsigned long flags, rate;
+	u64 ppb;
+
+	/* Get the clock rate from clock frequency register offset */
+	rate = readl(base + TOD_CLK_FREQ);
+
+	/* add GIGA as nominal ppb */
+	ppb = scaled_ppm_to_ppb(scaled_ppm) + GIGA;
+
+	tod_period = div_u64_rem(ppb << PERIOD_FRAC_OFFSET, rate, &tod_rem);
+	if (tod_period > TOD_PERIOD_MAX)
+		return -ERANGE;
+
+	/*
+	 * The drift of ToD adjusted periodically by adding a drift_adjust_fns
+	 * correction value every drift_adjust_rate count of clock cycles.
+	 */
+	tod_drift_adjust_fns = tod_rem / gcd(tod_rem, rate);
+	tod_drift_adjust_rate = rate / gcd(tod_rem, rate);
+
+	while ((tod_drift_adjust_fns > TOD_DRIFT_ADJUST_FNS_MAX) ||
+	       (tod_drift_adjust_rate > TOD_DRIFT_ADJUST_RATE_MAX)) {
+		tod_drift_adjust_fns >>= 1;
+		tod_drift_adjust_rate >>= 1;
+	}
+
+	if (tod_drift_adjust_fns == 0)
+		tod_drift_adjust_rate = 0;
+
+	spin_lock_irqsave(&dt->tod_lock, flags);
+	writel(tod_period, base + TOD_PERIOD);
+	writel(0, base + TOD_ADJUST_PERIOD);
+	writel(0, base + TOD_ADJUST_COUNT);
+	writel(tod_drift_adjust_fns, base + TOD_DRIFT_ADJUST);
+	writel(tod_drift_adjust_rate, base + TOD_DRIFT_ADJUST_RATE);
+	spin_unlock_irqrestore(&dt->tod_lock, flags);
+
+	return 0;
+}
+
+static int dfl_tod_adjust_time(struct ptp_clock_info *ptp, s64 delta)
+{
+	struct dfl_tod *dt = container_of(ptp, struct dfl_tod, ptp_clock_ops);
+	u32 period, diff, rem, rem_period, adj_period;
+	void __iomem *base = dt->tod_ctrl;
+	unsigned long flags;
+	bool neg_adj;
+	u64 count;
+	int ret;
+
+	neg_adj = delta < 0;
+	if (neg_adj)
+		delta = -delta;
+
+	spin_lock_irqsave(&dt->tod_lock, flags);
+
+	/*
+	 * Get the maximum possible value of the Period register offset
+	 * adjustment in nanoseconds scale. This depends on the current
+	 * Period register setting and the maximum and minimum possible
+	 * values of the Period register.
+	 */
+	period = readl(base + TOD_PERIOD);
+
+	if (neg_adj) {
+		diff = (period - TOD_PERIOD_MIN) >> PERIOD_FRAC_OFFSET;
+		adj_period = period - (diff << PERIOD_FRAC_OFFSET);
+		count = div_u64_rem(delta, diff, &rem);
+		rem_period = period - (rem << PERIOD_FRAC_OFFSET);
+	} else {
+		diff = (TOD_PERIOD_MAX - period) >> PERIOD_FRAC_OFFSET;
+		adj_period = period + (diff << PERIOD_FRAC_OFFSET);
+		count = div_u64_rem(delta, diff, &rem);
+		rem_period = period + (rem << PERIOD_FRAC_OFFSET);
+	}
+
+	ret = 0;
+
+	if (count > TOD_ADJUST_COUNT_MAX) {
+		ret = coarse_adjust_tod_clock(dt, delta);
+	} else {
+		/* Adjust the period by count cycles to adjust the time */
+		if (count)
+			ret = fine_adjust_tod_clock(dt, adj_period, count);
+
+		/* If there is a remainder, adjust the period for an additional cycle */
+		if (rem)
+			ret = fine_adjust_tod_clock(dt, rem_period, 1);
+	}
+
+	spin_unlock_irqrestore(&dt->tod_lock, flags);
+
+	return ret;
+}
+
+static int dfl_tod_get_timex(struct ptp_clock_info *ptp, struct timespec64 *ts,
+			     struct ptp_system_timestamp *sts)
+{
+	struct dfl_tod *dt = container_of(ptp, struct dfl_tod, ptp_clock_ops);
+	u32 seconds_msb, seconds_lsb, nanosec;
+	void __iomem *base = dt->tod_ctrl;
+	unsigned long flags;
+	u64 seconds;
+
+	spin_lock_irqsave(&dt->tod_lock, flags);
+	ptp_read_system_prets(sts);
+	nanosec = readl(base + TOD_NANOSEC);
+	seconds_lsb = readl(base + TOD_SECONDSL);
+	seconds_msb = readl(base + TOD_SECONDSH);
+	ptp_read_system_postts(sts);
+	spin_unlock_irqrestore(&dt->tod_lock, flags);
+
+	seconds = CAL_SECONDS(seconds_msb, seconds_lsb);
+
+	ts->tv_nsec = nanosec;
+	ts->tv_sec = seconds;
+
+	return 0;
+}
+
+static int dfl_tod_set_time(struct ptp_clock_info *ptp,
+			    const struct timespec64 *ts)
+{
+	struct dfl_tod *dt = container_of(ptp, struct dfl_tod, ptp_clock_ops);
+	u32 seconds_msb = FIELD_GET(SECONDS_MSB, ts->tv_sec);
+	u32 seconds_lsb = FIELD_GET(SECONDS_LSB, ts->tv_sec);
+	u32 nanosec = FIELD_GET(SECONDS_LSB, ts->tv_nsec);
+	void __iomem *base = dt->tod_ctrl;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dt->tod_lock, flags);
+	writel(seconds_msb, base + TOD_SECONDSH);
+	writel(seconds_lsb, base + TOD_SECONDSL);
+	writel(nanosec, base + TOD_NANOSEC);
+	spin_unlock_irqrestore(&dt->tod_lock, flags);
+
+	return 0;
+}
+
+static struct ptp_clock_info dfl_tod_clock_ops = {
+	.owner = THIS_MODULE,
+	.name = "dfl_tod",
+	.max_adj = TOD_MAX_ADJ,
+	.adjfine = dfl_tod_adjust_fine,
+	.adjtime = dfl_tod_adjust_time,
+	.gettimex64 = dfl_tod_get_timex,
+	.settime64 = dfl_tod_set_time,
+};
+
+static int dfl_tod_probe(struct dfl_device *ddev)
+{
+	struct device *dev = &ddev->dev;
+	struct dfl_tod *dt;
+
+	dt = devm_kzalloc(dev, sizeof(*dt), GFP_KERNEL);
+	if (!dt)
+		return -ENOMEM;
+
+	dt->tod_ctrl = devm_ioremap_resource(dev, &ddev->mmio_res);
+	if (IS_ERR(dt->tod_ctrl))
+		return PTR_ERR(dt->tod_ctrl);
+
+	dt->dev = dev;
+	spin_lock_init(&dt->tod_lock);
+	dev_set_drvdata(dev, dt);
+
+	dt->ptp_clock_ops = dfl_tod_clock_ops;
+
+	dt->ptp_clock = ptp_clock_register(&dt->ptp_clock_ops, dev);
+	if (IS_ERR(dt->ptp_clock))
+		return dev_err_probe(dt->dev, PTR_ERR(dt->ptp_clock),
+				     "Unable to register PTP clock\n");
+
+	return 0;
+}
+
+static void dfl_tod_remove(struct dfl_device *ddev)
+{
+	struct dfl_tod *dt = dev_get_drvdata(&ddev->dev);
+
+	ptp_clock_unregister(dt->ptp_clock);
+}
+
+static const struct dfl_device_id dfl_tod_ids[] = {
+	{ FME_ID, FME_FEATURE_ID_TOD },
+	{ }
+};
+MODULE_DEVICE_TABLE(dfl, dfl_tod_ids);
+
+static struct dfl_driver dfl_tod_driver = {
+	.drv = {
+		.name = "dfl-tod",
+	},
+	.id_table = dfl_tod_ids,
+	.probe = dfl_tod_probe,
+	.remove = dfl_tod_remove,
+};
+module_dfl_driver(dfl_tod_driver);
+
+MODULE_DESCRIPTION("FPGA DFL ToD driver");
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL");
diff --git a/drivers/ptp/ptp_ines.c b/drivers/ptp/ptp_ines.c
index 61f47fb9d997..ed215b458183 100644
--- a/drivers/ptp/ptp_ines.c
+++ b/drivers/ptp/ptp_ines.c
@@ -792,7 +792,7 @@ static struct platform_driver ines_ptp_ctrl_driver = {
 	.remove = ines_ptp_ctrl_remove,
 	.driver = {
 		.name = "ines_ptp_ctrl",
-		.of_match_table = of_match_ptr(ines_ptp_ctrl_of_match),
+		.of_match_table = ines_ptp_ctrl_of_match,
 	},
 };
 module_platform_driver(ines_ptp_ctrl_driver);
diff --git a/drivers/ptp/ptp_kvm_arm.c b/drivers/ptp/ptp_kvm_arm.c
index b7d28c8dfb84..e68e6943167b 100644
--- a/drivers/ptp/ptp_kvm_arm.c
+++ b/drivers/ptp/ptp_kvm_arm.c
@@ -22,6 +22,10 @@ int kvm_arch_ptp_init(void)
 	return 0;
 }
 
+void kvm_arch_ptp_exit(void)
+{
+}
+
 int kvm_arch_ptp_get_clock(struct timespec64 *ts)
 {
 	return kvm_arch_ptp_get_crosststamp(NULL, ts, NULL);
diff --git a/drivers/ptp/ptp_kvm_common.c b/drivers/ptp/ptp_kvm_common.c
index 9141162c4237..2418977989be 100644
--- a/drivers/ptp/ptp_kvm_common.c
+++ b/drivers/ptp/ptp_kvm_common.c
@@ -130,6 +130,7 @@ static struct kvm_ptp_clock kvm_ptp_clock;
 static void __exit ptp_kvm_exit(void)
 {
 	ptp_clock_unregister(kvm_ptp_clock.ptp_clock);
+	kvm_arch_ptp_exit();
 }
 
 static int __init ptp_kvm_init(void)
diff --git a/drivers/ptp/ptp_kvm_x86.c b/drivers/ptp/ptp_kvm_x86.c
index 4991054a2135..902844cc1a17 100644
--- a/drivers/ptp/ptp_kvm_x86.c
+++ b/drivers/ptp/ptp_kvm_x86.c
@@ -14,27 +14,64 @@
 #include <uapi/linux/kvm_para.h>
 #include <linux/ptp_clock_kernel.h>
 #include <linux/ptp_kvm.h>
+#include <linux/set_memory.h>
 
 static phys_addr_t clock_pair_gpa;
-static struct kvm_clock_pairing clock_pair;
+static struct kvm_clock_pairing clock_pair_glbl;
+static struct kvm_clock_pairing *clock_pair;
 
 int kvm_arch_ptp_init(void)
 {
+	struct page *p;
 	long ret;
 
 	if (!kvm_para_available())
 		return -ENODEV;
 
-	clock_pair_gpa = slow_virt_to_phys(&clock_pair);
-	if (!pvclock_get_pvti_cpu0_va())
-		return -ENODEV;
+	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
+		p = alloc_page(GFP_KERNEL | __GFP_ZERO);
+		if (!p)
+			return -ENOMEM;
+
+		clock_pair = page_address(p);
+		ret = set_memory_decrypted((unsigned long)clock_pair, 1);
+		if (ret) {
+			__free_page(p);
+			clock_pair = NULL;
+			goto nofree;
+		}
+	} else {
+		clock_pair = &clock_pair_glbl;
+	}
+
+	clock_pair_gpa = slow_virt_to_phys(clock_pair);
+	if (!pvclock_get_pvti_cpu0_va()) {
+		ret = -ENODEV;
+		goto err;
+	}
 
 	ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING, clock_pair_gpa,
 			     KVM_CLOCK_PAIRING_WALLCLOCK);
-	if (ret == -KVM_ENOSYS)
-		return -ENODEV;
+	if (ret == -KVM_ENOSYS) {
+		ret = -ENODEV;
+		goto err;
+	}
 
 	return ret;
+
+err:
+	kvm_arch_ptp_exit();
+nofree:
+	return ret;
+}
+
+void kvm_arch_ptp_exit(void)
+{
+	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
+		WARN_ON(set_memory_encrypted((unsigned long)clock_pair, 1));
+		free_page((unsigned long)clock_pair);
+		clock_pair = NULL;
+	}
 }
 
 int kvm_arch_ptp_get_clock(struct timespec64 *ts)
@@ -49,8 +86,8 @@ int kvm_arch_ptp_get_clock(struct timespec64 *ts)
 		return -EOPNOTSUPP;
 	}
 
-	ts->tv_sec = clock_pair.sec;
-	ts->tv_nsec = clock_pair.nsec;
+	ts->tv_sec = clock_pair->sec;
+	ts->tv_nsec = clock_pair->nsec;
 
 	return 0;
 }
@@ -81,9 +118,9 @@ int kvm_arch_ptp_get_crosststamp(u64 *cycle, struct timespec64 *tspec,
 			pr_err_ratelimited("clock pairing hypercall ret %lu\n", ret);
 			return -EOPNOTSUPP;
 		}
-		tspec->tv_sec = clock_pair.sec;
-		tspec->tv_nsec = clock_pair.nsec;
-		*cycle = __pvclock_read_cycles(src, clock_pair.tsc);
+		tspec->tv_sec = clock_pair->sec;
+		tspec->tv_nsec = clock_pair->nsec;
+		*cycle = __pvclock_read_cycles(src, clock_pair->tsc);
 	} while (pvclock_read_retry(src, version));
 
 	*cs = &kvm_clock;
diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c
index 4bbaccd543ad..ab8cab4d1560 100644
--- a/drivers/ptp/ptp_ocp.c
+++ b/drivers/ptp/ptp_ocp.c
@@ -34,7 +34,6 @@
 #define PCI_DEVICE_ID_OROLIA_ARTCARD		0xa000
 
 static struct class timecard_class = {
-	.owner		= THIS_MODULE,
 	.name		= "timecard",
 };
 
@@ -662,6 +661,7 @@ static struct ocp_resource ocp_fb_resource[] = {
 				.num_chipselect = 1,
 				.bits_per_word = 8,
 				.num_devices = 1,
+				.force_irq = true,
 				.devices = &(struct spi_board_info) {
 					.modalias = "spi-nor",
 				},
diff --git a/drivers/pwm/sysfs.c b/drivers/pwm/sysfs.c
index e7db8e45001c..1a106ec32939 100644
--- a/drivers/pwm/sysfs.c
+++ b/drivers/pwm/sysfs.c
@@ -475,7 +475,6 @@ static DEFINE_SIMPLE_DEV_PM_OPS(pwm_class_pm_ops, pwm_class_suspend, pwm_class_r
 
 static struct class pwm_class = {
 	.name = "pwm",
-	.owner = THIS_MODULE,
 	.dev_groups = pwm_chip_groups,
 	.pm = pm_sleep_ptr(&pwm_class_pm_ops),
 };
diff --git a/drivers/rapidio/devices/rio_mport_cdev.c b/drivers/rapidio/devices/rio_mport_cdev.c
index 43db495f1986..a115730ebf14 100644
--- a/drivers/rapidio/devices/rio_mport_cdev.c
+++ b/drivers/rapidio/devices/rio_mport_cdev.c
@@ -2536,10 +2536,8 @@ static void mport_cdev_remove(struct mport_dev *md)
 /*
  * mport_add_mport() - Add rio_mport from LDM device struct
  * @dev:		Linux device model struct
- * @class_intf:	Linux class_interface
  */
-static int mport_add_mport(struct device *dev,
-		struct class_interface *class_intf)
+static int mport_add_mport(struct device *dev)
 {
 	struct rio_mport *mport = NULL;
 	struct mport_dev *chdev = NULL;
@@ -2559,8 +2557,7 @@ static int mport_add_mport(struct device *dev,
  * mport_remove_mport() - Remove rio_mport from global list
  * TODO remove device from global mport_dev list
  */
-static void mport_remove_mport(struct device *dev,
-		struct class_interface *class_intf)
+static void mport_remove_mport(struct device *dev)
 {
 	struct rio_mport *mport = NULL;
 	struct mport_dev *chdev;
@@ -2603,7 +2600,7 @@ static int __init mport_init(void)
 	int ret;
 
 	/* Create device class needed by udev */
-	dev_class = class_create(THIS_MODULE, DRV_NAME);
+	dev_class = class_create(DRV_NAME);
 	if (IS_ERR(dev_class)) {
 		rmcd_error("Unable to create " DRV_NAME " class");
 		return PTR_ERR(dev_class);
diff --git a/drivers/rapidio/rio-driver.c b/drivers/rapidio/rio-driver.c
index e60e49769bed..1b3b4c2e015d 100644
--- a/drivers/rapidio/rio-driver.c
+++ b/drivers/rapidio/rio-driver.c
@@ -223,7 +223,6 @@ static int rio_uevent(const struct device *dev, struct kobj_uevent_env *env)
 
 struct class rio_mport_class = {
 	.name		= "rapidio_port",
-	.owner		= THIS_MODULE,
 	.dev_groups	= rio_mport_groups,
 };
 EXPORT_SYMBOL_GPL(rio_mport_class);
diff --git a/drivers/rapidio/rio-sysfs.c b/drivers/rapidio/rio-sysfs.c
index f7679602498e..90d391210533 100644
--- a/drivers/rapidio/rio-sysfs.c
+++ b/drivers/rapidio/rio-sysfs.c
@@ -286,7 +286,7 @@ const struct attribute_group *rio_dev_groups[] = {
 	NULL,
 };
 
-static ssize_t scan_store(struct bus_type *bus, const char *buf, size_t count)
+static ssize_t scan_store(const struct bus_type *bus, const char *buf, size_t count)
 {
 	long val;
 	int rc;
diff --git a/drivers/rapidio/rio_cm.c b/drivers/rapidio/rio_cm.c
index db4c265287ae..49f8d111e546 100644
--- a/drivers/rapidio/rio_cm.c
+++ b/drivers/rapidio/rio_cm.c
@@ -2087,13 +2087,11 @@ static int riocm_cdev_add(dev_t devno)
 /*
  * riocm_add_mport - add new local mport device into channel management core
  * @dev: device object associated with mport
- * @class_intf: class interface
  *
  * When a new mport device is added, CM immediately reserves inbound and
  * outbound RapidIO mailboxes that will be used.
  */
-static int riocm_add_mport(struct device *dev,
-			   struct class_interface *class_intf)
+static int riocm_add_mport(struct device *dev)
 {
 	int rc;
 	int i;
@@ -2166,14 +2164,12 @@ static int riocm_add_mport(struct device *dev,
 /*
  * riocm_remove_mport - remove local mport device from channel management core
  * @dev: device object associated with mport
- * @class_intf: class interface
  *
  * Removes a local mport device from the list of registered devices that provide
  * channel management services. Returns an error if the specified mport is not
  * registered with the CM core.
  */
-static void riocm_remove_mport(struct device *dev,
-			       struct class_interface *class_intf)
+static void riocm_remove_mport(struct device *dev)
 {
 	struct rio_mport *mport = to_rio_mport(dev);
 	struct cm_dev *cm;
@@ -2297,7 +2293,7 @@ static int __init riocm_init(void)
 	int ret;
 
 	/* Create device class needed by udev */
-	dev_class = class_create(THIS_MODULE, DRV_NAME);
+	dev_class = class_create(DRV_NAME);
 	if (IS_ERR(dev_class)) {
 		riocm_error("Cannot create " DRV_NAME " class");
 		return PTR_ERR(dev_class);
diff --git a/drivers/regulator/88pg86x.c b/drivers/regulator/88pg86x.c
index e91d5885c5ef..74275b681f46 100644
--- a/drivers/regulator/88pg86x.c
+++ b/drivers/regulator/88pg86x.c
@@ -101,6 +101,7 @@ MODULE_DEVICE_TABLE(i2c, pg86x_i2c_id);
 static struct i2c_driver pg86x_regulator_driver = {
 	.driver = {
 		.name = "88pg86x",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(pg86x_dt_ids),
 	},
 	.probe_new = pg86x_i2c_probe,
diff --git a/drivers/regulator/88pm800-regulator.c b/drivers/regulator/88pm800-regulator.c
index d08ee81ed1ac..83e8860309dc 100644
--- a/drivers/regulator/88pm800-regulator.c
+++ b/drivers/regulator/88pm800-regulator.c
@@ -274,6 +274,7 @@ static int pm800_regulator_probe(struct platform_device *pdev)
 static struct platform_driver pm800_regulator_driver = {
 	.driver		= {
 		.name	= "88pm80x-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe		= pm800_regulator_probe,
 };
diff --git a/drivers/regulator/88pm8607.c b/drivers/regulator/88pm8607.c
index 1d1c4a7ec3e2..e6c436955e25 100644
--- a/drivers/regulator/88pm8607.c
+++ b/drivers/regulator/88pm8607.c
@@ -383,6 +383,7 @@ MODULE_DEVICE_TABLE(platform, pm8607_regulator_driver_ids);
 static struct platform_driver pm8607_regulator_driver = {
 	.driver		= {
 		.name	= "88pm860x-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe		= pm8607_regulator_probe,
 	.id_table	= pm8607_regulator_driver_ids,
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index aae28d0a489c..e5f3613c15fa 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -1082,6 +1082,16 @@ config REGULATOR_RT4801
 	  This adds support for voltage regulators in Richtek RT4801 Display Bias IC.
 	  The device supports two regulators (DSVP/DSVN).
 
+config REGULATOR_RT4803
+	tristate "Richtek RT4803 boost regualtor"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  This adds support for RT4803 boost converter that integrates the
+	  bypass switch. If the input voltage is low than the required voltage,
+	  RT4803 will enter boost mode. Otherwise, enable internal bypass
+	  switch to enter bypass mode.
+
 config REGULATOR_RT4831
 	tristate "Richtek RT4831 DSV Regulators"
 	depends on MFD_RT4831
@@ -1120,6 +1130,19 @@ config REGULATOR_RT5190A
 	  buck converters, 1 LDO, mute AC OFF depop function, with the general
 	  I2C control interface.
 
+config REGULATOR_RT5739
+	tristate "Richtek RT5739 Regulator"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  This adds support for voltage regulator in Richtek RT5739.
+	  It's a step-down switching voltage regulator. Using a proprietary
+	  architecture with synchronous rectification, it is capable of
+	  delivering 3.5A continuously at over 80% efficiency.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called rt5739.
+
 config REGULATOR_RT5759
 	tristate "Richtek RT5759 Regulator"
 	depends on I2C
diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile
index ee383d8fc835..58dfe0147cd4 100644
--- a/drivers/regulator/Makefile
+++ b/drivers/regulator/Makefile
@@ -130,10 +130,12 @@ obj-$(CONFIG_REGULATOR_RK808)   += rk808-regulator.o
 obj-$(CONFIG_REGULATOR_RN5T618) += rn5t618-regulator.o
 obj-$(CONFIG_REGULATOR_ROHM)	+= rohm-regulator.o
 obj-$(CONFIG_REGULATOR_RT4801)	+= rt4801-regulator.o
+obj-$(CONFIG_REGULATOR_RT4803)	+= rt4803.o
 obj-$(CONFIG_REGULATOR_RT4831)	+= rt4831-regulator.o
 obj-$(CONFIG_REGULATOR_RT5033)	+= rt5033-regulator.o
 obj-$(CONFIG_REGULATOR_RT5120)	+= rt5120-regulator.o
 obj-$(CONFIG_REGULATOR_RT5190A) += rt5190a-regulator.o
+obj-$(CONFIG_REGULATOR_RT5739)	+= rt5739.o
 obj-$(CONFIG_REGULATOR_RT5759)	+= rt5759-regulator.o
 obj-$(CONFIG_REGULATOR_RT6160)	+= rt6160-regulator.o
 obj-$(CONFIG_REGULATOR_RT6190)	+= rt6190-regulator.o
diff --git a/drivers/regulator/aat2870-regulator.c b/drivers/regulator/aat2870-regulator.c
index d6ed5bf9235e..970d86f2bbb8 100644
--- a/drivers/regulator/aat2870-regulator.c
+++ b/drivers/regulator/aat2870-regulator.c
@@ -178,6 +178,7 @@ static int aat2870_regulator_probe(struct platform_device *pdev)
 static struct platform_driver aat2870_regulator_driver = {
 	.driver = {
 		.name	= "aat2870-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe	= aat2870_regulator_probe,
 };
diff --git a/drivers/regulator/ab8500-ext.c b/drivers/regulator/ab8500-ext.c
index 4f26952caa56..b9955aa4e0d1 100644
--- a/drivers/regulator/ab8500-ext.c
+++ b/drivers/regulator/ab8500-ext.c
@@ -446,6 +446,7 @@ static struct platform_driver ab8500_ext_regulator_driver = {
 	.probe = ab8500_ext_regulator_probe,
 	.driver         = {
 		.name   = "ab8500-ext-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/ab8500.c b/drivers/regulator/ab8500.c
index 23a401734a98..6b4a3a3d8385 100644
--- a/drivers/regulator/ab8500.c
+++ b/drivers/regulator/ab8500.c
@@ -1737,6 +1737,7 @@ static struct platform_driver ab8500_regulator_driver = {
 	.probe = ab8500_regulator_probe,
 	.driver         = {
 		.name   = "ab8500-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/act8865-regulator.c b/drivers/regulator/act8865-regulator.c
index 53f2c75cdeb4..5c409ff4aa99 100644
--- a/drivers/regulator/act8865-regulator.c
+++ b/drivers/regulator/act8865-regulator.c
@@ -789,6 +789,7 @@ MODULE_DEVICE_TABLE(i2c, act8865_ids);
 static struct i2c_driver act8865_pmic_driver = {
 	.driver	= {
 		.name	= "act8865",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe_new	= act8865_pmic_probe,
 	.id_table	= act8865_ids,
diff --git a/drivers/regulator/act8945a-regulator.c b/drivers/regulator/act8945a-regulator.c
index 1db1c6423779..e26264529b74 100644
--- a/drivers/regulator/act8945a-regulator.c
+++ b/drivers/regulator/act8945a-regulator.c
@@ -348,6 +348,7 @@ static void act8945a_pmic_shutdown(struct platform_device *pdev)
 static struct platform_driver act8945a_pmic_driver = {
 	.driver = {
 		.name = "act8945a-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.pm = &act8945a_pm,
 	},
 	.probe = act8945a_pmic_probe,
diff --git a/drivers/regulator/ad5398.c b/drivers/regulator/ad5398.c
index 2ba8ac1773d1..c228cf6956d1 100644
--- a/drivers/regulator/ad5398.c
+++ b/drivers/regulator/ad5398.c
@@ -257,6 +257,7 @@ static struct i2c_driver ad5398_driver = {
 	.probe_new = ad5398_probe,
 	.driver		= {
 		.name	= "ad5398",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.id_table	= ad5398_id,
 };
diff --git a/drivers/regulator/anatop-regulator.c b/drivers/regulator/anatop-regulator.c
index f9856d4e295f..700bd0343196 100644
--- a/drivers/regulator/anatop-regulator.c
+++ b/drivers/regulator/anatop-regulator.c
@@ -328,6 +328,7 @@ MODULE_DEVICE_TABLE(of, of_anatop_regulator_match_tbl);
 static struct platform_driver anatop_regulator_driver = {
 	.driver = {
 		.name	= "anatop_regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_anatop_regulator_match_tbl,
 	},
 	.probe	= anatop_regulator_probe,
diff --git a/drivers/regulator/arizona-ldo1.c b/drivers/regulator/arizona-ldo1.c
index ade0bef4569d..b465c0010665 100644
--- a/drivers/regulator/arizona-ldo1.c
+++ b/drivers/regulator/arizona-ldo1.c
@@ -380,6 +380,7 @@ static struct platform_driver arizona_ldo1_driver = {
 	.remove = arizona_ldo1_remove,
 	.driver		= {
 		.name	= "arizona-ldo1",
+		.probe_type = PROBE_FORCE_SYNCHRONOUS,
 	},
 };
 
@@ -388,6 +389,7 @@ static struct platform_driver madera_ldo1_driver = {
 	.remove = arizona_ldo1_remove,
 	.driver		= {
 		.name	= "madera-ldo1",
+		.probe_type = PROBE_FORCE_SYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/arizona-micsupp.c b/drivers/regulator/arizona-micsupp.c
index 596ecd8041cd..e250e5f3fcbc 100644
--- a/drivers/regulator/arizona-micsupp.c
+++ b/drivers/regulator/arizona-micsupp.c
@@ -365,6 +365,7 @@ static struct platform_driver arizona_micsupp_driver = {
 	.probe = arizona_micsupp_probe,
 	.driver		= {
 		.name	= "arizona-micsupp",
+		.probe_type = PROBE_FORCE_SYNCHRONOUS,
 	},
 };
 
@@ -372,6 +373,7 @@ static struct platform_driver madera_micsupp_driver = {
 	.probe = madera_micsupp_probe,
 	.driver		= {
 		.name	= "madera-micsupp",
+		.probe_type = PROBE_FORCE_SYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/as3711-regulator.c b/drivers/regulator/as3711-regulator.c
index b6b9206969ae..0431a732cd78 100644
--- a/drivers/regulator/as3711-regulator.c
+++ b/drivers/regulator/as3711-regulator.c
@@ -243,6 +243,7 @@ static int as3711_regulator_probe(struct platform_device *pdev)
 static struct platform_driver as3711_regulator_driver = {
 	.driver	= {
 		.name	= "as3711-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe		= as3711_regulator_probe,
 };
diff --git a/drivers/regulator/as3722-regulator.c b/drivers/regulator/as3722-regulator.c
index 7bebf9ce6271..da378bfdba40 100644
--- a/drivers/regulator/as3722-regulator.c
+++ b/drivers/regulator/as3722-regulator.c
@@ -831,6 +831,7 @@ MODULE_DEVICE_TABLE(of, of_as3722_regulator_match);
 static struct platform_driver as3722_regulator_driver = {
 	.driver = {
 		.name = "as3722-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_as3722_regulator_match,
 	},
 	.probe = as3722_regulator_probe,
diff --git a/drivers/regulator/atc260x-regulator.c b/drivers/regulator/atc260x-regulator.c
index 485e58b264c0..87e237d740bc 100644
--- a/drivers/regulator/atc260x-regulator.c
+++ b/drivers/regulator/atc260x-regulator.c
@@ -530,6 +530,7 @@ static struct platform_driver atc260x_regulator_driver = {
 	.probe = atc260x_regulator_probe,
 	.driver = {
 		.name = "atc260x-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/axp20x-regulator.c b/drivers/regulator/axp20x-regulator.c
index d260c442b788..943172b19722 100644
--- a/drivers/regulator/axp20x-regulator.c
+++ b/drivers/regulator/axp20x-regulator.c
@@ -1364,6 +1364,7 @@ static struct platform_driver axp20x_regulator_driver = {
 	.probe	= axp20x_regulator_probe,
 	.driver	= {
 		.name		= "axp20x-regulator",
+		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/bcm590xx-regulator.c b/drivers/regulator/bcm590xx-regulator.c
index 65e23fc5f9c3..9f0cda46b015 100644
--- a/drivers/regulator/bcm590xx-regulator.c
+++ b/drivers/regulator/bcm590xx-regulator.c
@@ -354,6 +354,7 @@ static int bcm590xx_probe(struct platform_device *pdev)
 static struct platform_driver bcm590xx_regulator_driver = {
 	.driver = {
 		.name = "bcm590xx-vregs",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = bcm590xx_probe,
 };
diff --git a/drivers/regulator/bd71815-regulator.c b/drivers/regulator/bd71815-regulator.c
index 8b55046eded8..475b1e0110e7 100644
--- a/drivers/regulator/bd71815-regulator.c
+++ b/drivers/regulator/bd71815-regulator.c
@@ -201,10 +201,10 @@ static int buck12_set_hw_dvs_levels(struct device_node *np,
 
 	data = container_of(desc, struct bd71815_regulator, desc);
 
-	if (of_find_property(np, "rohm,dvs-run-voltage", NULL) ||
-	    of_find_property(np, "rohm,dvs-suspend-voltage", NULL) ||
-	    of_find_property(np, "rohm,dvs-lpsr-voltage", NULL) ||
-	    of_find_property(np, "rohm,dvs-snvs-voltage", NULL)) {
+	if (of_property_present(np, "rohm,dvs-run-voltage") ||
+	    of_property_present(np, "rohm,dvs-suspend-voltage") ||
+	    of_property_present(np, "rohm,dvs-lpsr-voltage") ||
+	    of_property_present(np, "rohm,dvs-snvs-voltage")) {
 		ret = regmap_read(cfg->regmap, desc->vsel_reg, &val);
 		if (ret)
 			return ret;
@@ -619,6 +619,7 @@ MODULE_DEVICE_TABLE(platform, bd7181x_pmic_id);
 static struct platform_driver bd7181x_regulator = {
 	.driver = {
 		.name = "bd7181x-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = bd7181x_probe,
 	.id_table = bd7181x_pmic_id,
diff --git a/drivers/regulator/bd71828-regulator.c b/drivers/regulator/bd71828-regulator.c
index ad728f4f2241..f3205dc9d4fc 100644
--- a/drivers/regulator/bd71828-regulator.c
+++ b/drivers/regulator/bd71828-regulator.c
@@ -771,7 +771,8 @@ static int bd71828_probe(struct platform_device *pdev)
 
 static struct platform_driver bd71828_regulator = {
 	.driver = {
-		.name = "bd71828-pmic"
+		.name = "bd71828-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = bd71828_probe,
 };
diff --git a/drivers/regulator/bd718x7-regulator.c b/drivers/regulator/bd718x7-regulator.c
index 894fab0d53d0..b0b9938c20a1 100644
--- a/drivers/regulator/bd718x7-regulator.c
+++ b/drivers/regulator/bd718x7-regulator.c
@@ -1829,6 +1829,7 @@ MODULE_DEVICE_TABLE(platform, bd718x7_pmic_id);
 static struct platform_driver bd718xx_regulator = {
 	.driver = {
 		.name = "bd718xx-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = bd718xx_probe,
 	.id_table = bd718x7_pmic_id,
diff --git a/drivers/regulator/bd9571mwv-regulator.c b/drivers/regulator/bd9571mwv-regulator.c
index ba020a45f238..d469481d8442 100644
--- a/drivers/regulator/bd9571mwv-regulator.c
+++ b/drivers/regulator/bd9571mwv-regulator.c
@@ -353,6 +353,7 @@ MODULE_DEVICE_TABLE(platform, bd9571mwv_regulator_id_table);
 static struct platform_driver bd9571mwv_regulator_driver = {
 	.driver = {
 		.name = "bd9571mwv-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.pm = DEV_PM_OPS,
 	},
 	.probe = bd9571mwv_regulator_probe,
diff --git a/drivers/regulator/bd9576-regulator.c b/drivers/regulator/bd9576-regulator.c
index 02c70768652b..d4ca7b3f4036 100644
--- a/drivers/regulator/bd9576-regulator.c
+++ b/drivers/regulator/bd9576-regulator.c
@@ -1126,6 +1126,7 @@ MODULE_DEVICE_TABLE(platform, bd957x_pmic_id);
 static struct platform_driver bd957x_regulator = {
 	.driver = {
 		.name = "bd957x-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = bd957x_probe,
 	.id_table = bd957x_pmic_id,
diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
index 4fcd36055b02..dc741ac156c3 100644
--- a/drivers/regulator/core.c
+++ b/drivers/regulator/core.c
@@ -207,6 +207,71 @@ static void regulator_unlock(struct regulator_dev *rdev)
 	mutex_unlock(&regulator_nesting_mutex);
 }
 
+/**
+ * regulator_lock_two - lock two regulators
+ * @rdev1:		first regulator
+ * @rdev2:		second regulator
+ * @ww_ctx:		w/w mutex acquire context
+ *
+ * Locks both rdevs using the regulator_ww_class.
+ */
+static void regulator_lock_two(struct regulator_dev *rdev1,
+			       struct regulator_dev *rdev2,
+			       struct ww_acquire_ctx *ww_ctx)
+{
+	struct regulator_dev *held, *contended;
+	int ret;
+
+	ww_acquire_init(ww_ctx, &regulator_ww_class);
+
+	/* Try to just grab both of them */
+	ret = regulator_lock_nested(rdev1, ww_ctx);
+	WARN_ON(ret);
+	ret = regulator_lock_nested(rdev2, ww_ctx);
+	if (ret != -EDEADLOCK) {
+		WARN_ON(ret);
+		goto exit;
+	}
+
+	held = rdev1;
+	contended = rdev2;
+	while (true) {
+		regulator_unlock(held);
+
+		ww_mutex_lock_slow(&contended->mutex, ww_ctx);
+		contended->ref_cnt++;
+		contended->mutex_owner = current;
+		swap(held, contended);
+		ret = regulator_lock_nested(contended, ww_ctx);
+
+		if (ret != -EDEADLOCK) {
+			WARN_ON(ret);
+			break;
+		}
+	}
+
+exit:
+	ww_acquire_done(ww_ctx);
+}
+
+/**
+ * regulator_unlock_two - unlock two regulators
+ * @rdev1:		first regulator
+ * @rdev2:		second regulator
+ * @ww_ctx:		w/w mutex acquire context
+ *
+ * The inverse of regulator_lock_two().
+ */
+
+static void regulator_unlock_two(struct regulator_dev *rdev1,
+				 struct regulator_dev *rdev2,
+				 struct ww_acquire_ctx *ww_ctx)
+{
+	regulator_unlock(rdev2);
+	regulator_unlock(rdev1);
+	ww_acquire_fini(ww_ctx);
+}
+
 static bool regulator_supply_is_couple(struct regulator_dev *rdev)
 {
 	struct regulator_dev *c_rdev;
@@ -334,6 +399,7 @@ static void regulator_lock_dependent(struct regulator_dev *rdev,
 			ww_mutex_lock_slow(&new_contended_rdev->mutex, ww_ctx);
 			old_contended_rdev = new_contended_rdev;
 			old_contended_rdev->ref_cnt++;
+			old_contended_rdev->mutex_owner = current;
 		}
 
 		err = regulator_lock_recursive(rdev,
@@ -1583,9 +1649,6 @@ static int set_machine_constraints(struct regulator_dev *rdev)
 			rdev->constraints->always_on = true;
 	}
 
-	if (rdev->desc->off_on_delay)
-		rdev->last_off = ktime_get_boottime();
-
 	/* If the constraints say the regulator should be on at this point
 	 * and we have control then make sure it is enabled.
 	 */
@@ -1619,6 +1682,8 @@ static int set_machine_constraints(struct regulator_dev *rdev)
 
 		if (rdev->constraints->always_on)
 			rdev->use_count++;
+	} else if (rdev->desc->off_on_delay) {
+		rdev->last_off = ktime_get();
 	}
 
 	print_constraints(rdev);
@@ -1627,8 +1692,8 @@ static int set_machine_constraints(struct regulator_dev *rdev)
 
 /**
  * set_supply - set regulator supply regulator
- * @rdev: regulator name
- * @supply_rdev: supply regulator name
+ * @rdev: regulator (locked)
+ * @supply_rdev: supply regulator (locked))
  *
  * Called by platform initialisation code to set the supply regulator for this
  * regulator. This ensures that a regulators supply will also be enabled by the
@@ -1800,6 +1865,8 @@ static struct regulator *create_regulator(struct regulator_dev *rdev,
 	struct regulator *regulator;
 	int err = 0;
 
+	lockdep_assert_held_once(&rdev->mutex.base);
+
 	if (dev) {
 		char buf[REG_STR_SIZE];
 		int size;
@@ -1827,9 +1894,7 @@ static struct regulator *create_regulator(struct regulator_dev *rdev,
 	regulator->rdev = rdev;
 	regulator->supply_name = supply_name;
 
-	regulator_lock(rdev);
 	list_add(&regulator->list, &rdev->consumer_list);
-	regulator_unlock(rdev);
 
 	if (dev) {
 		regulator->dev = dev;
@@ -1995,6 +2060,7 @@ static int regulator_resolve_supply(struct regulator_dev *rdev)
 {
 	struct regulator_dev *r;
 	struct device *dev = rdev->dev.parent;
+	struct ww_acquire_ctx ww_ctx;
 	int ret = 0;
 
 	/* No supply to resolve? */
@@ -2061,23 +2127,23 @@ static int regulator_resolve_supply(struct regulator_dev *rdev)
 	 * between rdev->supply null check and setting rdev->supply in
 	 * set_supply() from concurrent tasks.
 	 */
-	regulator_lock(rdev);
+	regulator_lock_two(rdev, r, &ww_ctx);
 
 	/* Supply just resolved by a concurrent task? */
 	if (rdev->supply) {
-		regulator_unlock(rdev);
+		regulator_unlock_two(rdev, r, &ww_ctx);
 		put_device(&r->dev);
 		goto out;
 	}
 
 	ret = set_supply(rdev, r);
 	if (ret < 0) {
-		regulator_unlock(rdev);
+		regulator_unlock_two(rdev, r, &ww_ctx);
 		put_device(&r->dev);
 		goto out;
 	}
 
-	regulator_unlock(rdev);
+	regulator_unlock_two(rdev, r, &ww_ctx);
 
 	/*
 	 * In set_machine_constraints() we may have turned this regulator on
@@ -2190,7 +2256,9 @@ struct regulator *_regulator_get(struct device *dev, const char *id,
 		return regulator;
 	}
 
+	regulator_lock(rdev);
 	regulator = create_regulator(rdev, dev, id);
+	regulator_unlock(rdev);
 	if (regulator == NULL) {
 		regulator = ERR_PTR(-ENOMEM);
 		module_put(rdev->owner);
@@ -2668,7 +2736,7 @@ static int _regulator_do_enable(struct regulator_dev *rdev)
 
 	trace_regulator_enable(rdev_get_name(rdev));
 
-	if (rdev->desc->off_on_delay && rdev->last_off) {
+	if (rdev->desc->off_on_delay) {
 		/* if needed, keep a distance of off_on_delay from last time
 		 * this regulator was disabled.
 		 */
@@ -6049,6 +6117,7 @@ static void regulator_summary_lock(struct ww_acquire_ctx *ww_ctx)
 			ww_mutex_lock_slow(&new_contended_rdev->mutex, ww_ctx);
 			old_contended_rdev = new_contended_rdev;
 			old_contended_rdev->ref_cnt++;
+			old_contended_rdev->mutex_owner = current;
 		}
 
 		err = regulator_summary_lock_all(ww_ctx,
diff --git a/drivers/regulator/cpcap-regulator.c b/drivers/regulator/cpcap-regulator.c
index b0c225d98631..1fd79fb17303 100644
--- a/drivers/regulator/cpcap-regulator.c
+++ b/drivers/regulator/cpcap-regulator.c
@@ -553,6 +553,7 @@ static struct platform_driver cpcap_regulator_driver = {
 	.probe		= cpcap_regulator_probe,
 	.driver		= {
 		.name	= "cpcap-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(cpcap_regulator_id_table),
 	},
 };
diff --git a/drivers/regulator/cros-ec-regulator.c b/drivers/regulator/cros-ec-regulator.c
index 1591636f86c3..fb9643ed7a49 100644
--- a/drivers/regulator/cros-ec-regulator.c
+++ b/drivers/regulator/cros-ec-regulator.c
@@ -215,6 +215,7 @@ static struct platform_driver cros_ec_regulator_driver = {
 	.probe		= cros_ec_regulator_probe,
 	.driver		= {
 		.name		= "cros-ec-regulator",
+		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = regulator_cros_ec_of_match,
 	},
 };
diff --git a/drivers/regulator/da903x-regulator.c b/drivers/regulator/da903x-regulator.c
index 770e694824ac..f79337079a45 100644
--- a/drivers/regulator/da903x-regulator.c
+++ b/drivers/regulator/da903x-regulator.c
@@ -471,6 +471,7 @@ static int da903x_regulator_probe(struct platform_device *pdev)
 static struct platform_driver da903x_regulator_driver = {
 	.driver	= {
 		.name	= "da903x-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe		= da903x_regulator_probe,
 };
diff --git a/drivers/regulator/da9052-regulator.c b/drivers/regulator/da9052-regulator.c
index 23fa429ebe76..ab6f5d61b173 100644
--- a/drivers/regulator/da9052-regulator.c
+++ b/drivers/regulator/da9052-regulator.c
@@ -438,6 +438,7 @@ static struct platform_driver da9052_regulator_driver = {
 	.probe = da9052_regulator_probe,
 	.driver = {
 		.name = "da9052-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/da9055-regulator.c b/drivers/regulator/da9055-regulator.c
index 73ff5fc7d8d7..8fd9ac787588 100644
--- a/drivers/regulator/da9055-regulator.c
+++ b/drivers/regulator/da9055-regulator.c
@@ -576,6 +576,7 @@ static struct platform_driver da9055_regulator_driver = {
 	.probe = da9055_regulator_probe,
 	.driver = {
 		.name = "da9055-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/da9062-regulator.c b/drivers/regulator/da9062-regulator.c
index 1a6324001027..c28b061eef02 100644
--- a/drivers/regulator/da9062-regulator.c
+++ b/drivers/regulator/da9062-regulator.c
@@ -1033,6 +1033,7 @@ static int da9062_regulator_probe(struct platform_device *pdev)
 static struct platform_driver da9062_regulator_driver = {
 	.driver = {
 		.name = "da9062-regulators",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = da9062_regulator_probe,
 };
diff --git a/drivers/regulator/da9063-regulator.c b/drivers/regulator/da9063-regulator.c
index 82f52a2a031a..c5dd77be558b 100644
--- a/drivers/regulator/da9063-regulator.c
+++ b/drivers/regulator/da9063-regulator.c
@@ -83,6 +83,9 @@ struct da9063_regulator_info {
 
 	/* DA9063 event detection bit */
 	struct reg_field oc_event;
+
+	/* DA9063 voltage monitor bit */
+	struct reg_field vmon;
 };
 
 /* Macros for LDO */
@@ -148,6 +151,7 @@ struct da9063_regulator {
 	struct regmap_field			*suspend;
 	struct regmap_field			*sleep;
 	struct regmap_field			*suspend_sleep;
+	struct regmap_field			*vmon;
 };
 
 /* Encapsulates all information for the regulators driver */
@@ -203,6 +207,24 @@ static const unsigned int da9063_bmem_bio_merged_limits[] = {
 	4600000, 4800000, 5000000, 5200000, 5400000, 5600000, 5800000, 6000000
 };
 
+static int da9063_set_xvp(struct regulator_dev *rdev, int lim_uV, int severity, bool enable)
+{
+	struct da9063_regulator *regl = rdev_get_drvdata(rdev);
+	struct device *dev = regl->hw->dev;
+
+	dev_dbg(dev, "%s: lim: %d, sev: %d, en: %d\n", regl->desc.name, lim_uV, severity, enable);
+
+	/*
+	 * only support enable and disable.
+	 * the da9063 offers a GPIO (GP_FB2) which is unasserted if an XV happens.
+	 * therefore ignore severity here, as there might be handlers in hardware.
+	 */
+	if (lim_uV)
+		return -EINVAL;
+
+	return regmap_field_write(regl->vmon, enable ? 1 : 0);
+}
+
 static int da9063_buck_set_mode(struct regulator_dev *rdev, unsigned int mode)
 {
 	struct da9063_regulator *regl = rdev_get_drvdata(rdev);
@@ -541,37 +563,41 @@ static int da9063_buck_get_current_limit(struct regulator_dev *rdev)
 }
 
 static const struct regulator_ops da9063_buck_ops = {
-	.enable			= regulator_enable_regmap,
-	.disable		= regulator_disable_regmap,
-	.is_enabled		= regulator_is_enabled_regmap,
-	.get_voltage_sel	= regulator_get_voltage_sel_regmap,
-	.set_voltage_sel	= regulator_set_voltage_sel_regmap,
-	.list_voltage		= regulator_list_voltage_linear,
-	.set_current_limit	= da9063_buck_set_current_limit,
-	.get_current_limit	= da9063_buck_get_current_limit,
-	.set_mode		= da9063_buck_set_mode,
-	.get_mode		= da9063_buck_get_mode,
-	.get_status		= da9063_buck_get_status,
-	.set_suspend_voltage	= da9063_set_suspend_voltage,
-	.set_suspend_enable	= da9063_suspend_enable,
-	.set_suspend_disable	= da9063_suspend_disable,
-	.set_suspend_mode	= da9063_buck_set_suspend_mode,
+	.enable				= regulator_enable_regmap,
+	.disable			= regulator_disable_regmap,
+	.is_enabled			= regulator_is_enabled_regmap,
+	.get_voltage_sel		= regulator_get_voltage_sel_regmap,
+	.set_voltage_sel		= regulator_set_voltage_sel_regmap,
+	.list_voltage			= regulator_list_voltage_linear,
+	.set_current_limit		= da9063_buck_set_current_limit,
+	.get_current_limit		= da9063_buck_get_current_limit,
+	.set_mode			= da9063_buck_set_mode,
+	.get_mode			= da9063_buck_get_mode,
+	.get_status			= da9063_buck_get_status,
+	.set_suspend_voltage		= da9063_set_suspend_voltage,
+	.set_suspend_enable		= da9063_suspend_enable,
+	.set_suspend_disable		= da9063_suspend_disable,
+	.set_suspend_mode		= da9063_buck_set_suspend_mode,
+	.set_over_voltage_protection	= da9063_set_xvp,
+	.set_under_voltage_protection	= da9063_set_xvp,
 };
 
 static const struct regulator_ops da9063_ldo_ops = {
-	.enable			= regulator_enable_regmap,
-	.disable		= regulator_disable_regmap,
-	.is_enabled		= regulator_is_enabled_regmap,
-	.get_voltage_sel	= regulator_get_voltage_sel_regmap,
-	.set_voltage_sel	= regulator_set_voltage_sel_regmap,
-	.list_voltage		= regulator_list_voltage_linear,
-	.set_mode		= da9063_ldo_set_mode,
-	.get_mode		= da9063_ldo_get_mode,
-	.get_status		= da9063_ldo_get_status,
-	.set_suspend_voltage	= da9063_set_suspend_voltage,
-	.set_suspend_enable	= da9063_suspend_enable,
-	.set_suspend_disable	= da9063_suspend_disable,
-	.set_suspend_mode	= da9063_ldo_set_suspend_mode,
+	.enable				= regulator_enable_regmap,
+	.disable			= regulator_disable_regmap,
+	.is_enabled			= regulator_is_enabled_regmap,
+	.get_voltage_sel		= regulator_get_voltage_sel_regmap,
+	.set_voltage_sel		= regulator_set_voltage_sel_regmap,
+	.list_voltage			= regulator_list_voltage_linear,
+	.set_mode			= da9063_ldo_set_mode,
+	.get_mode			= da9063_ldo_get_mode,
+	.get_status			= da9063_ldo_get_status,
+	.set_suspend_voltage		= da9063_set_suspend_voltage,
+	.set_suspend_enable		= da9063_suspend_enable,
+	.set_suspend_disable		= da9063_suspend_disable,
+	.set_suspend_mode		= da9063_ldo_set_suspend_mode,
+	.set_over_voltage_protection	= da9063_set_xvp,
+	.set_under_voltage_protection	= da9063_set_xvp,
 };
 
 /* Info of regulators for DA9063 */
@@ -581,36 +607,42 @@ static const struct da9063_regulator_info da9063_regulator_info[] = {
 			    da9063_buck_a_limits,
 			    DA9063_REG_BUCK_ILIM_C, DA9063_BCORE1_ILIM_MASK),
 		DA9063_BUCK_COMMON_FIELDS(BCORE1),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_4, DA9063_BCORE1_MON_EN),
 	},
 	{
 		DA9063_BUCK(DA9063, BCORE2, 300, 10, 1570,
 			    da9063_buck_a_limits,
 			    DA9063_REG_BUCK_ILIM_C, DA9063_BCORE2_ILIM_MASK),
 		DA9063_BUCK_COMMON_FIELDS(BCORE2),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_4, DA9063_BCORE2_MON_EN),
 	},
 	{
 		DA9063_BUCK(DA9063, BPRO, 530, 10, 1800,
 			    da9063_buck_a_limits,
 			    DA9063_REG_BUCK_ILIM_B, DA9063_BPRO_ILIM_MASK),
 		DA9063_BUCK_COMMON_FIELDS(BPRO),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_4, DA9063_BPRO_MON_EN),
 	},
 	{
 		DA9063_BUCK(DA9063, BMEM, 800, 20, 3340,
 			    da9063_buck_b_limits,
 			    DA9063_REG_BUCK_ILIM_A, DA9063_BMEM_ILIM_MASK),
 		DA9063_BUCK_COMMON_FIELDS(BMEM),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_4, DA9063_BMEM_MON_EN),
 	},
 	{
 		DA9063_BUCK(DA9063, BIO, 800, 20, 3340,
 			    da9063_buck_b_limits,
 			    DA9063_REG_BUCK_ILIM_A, DA9063_BIO_ILIM_MASK),
 		DA9063_BUCK_COMMON_FIELDS(BIO),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_4, DA9063_BIO_MON_EN),
 	},
 	{
 		DA9063_BUCK(DA9063, BPERI, 800, 20, 3340,
 			    da9063_buck_b_limits,
 			    DA9063_REG_BUCK_ILIM_B, DA9063_BPERI_ILIM_MASK),
 		DA9063_BUCK_COMMON_FIELDS(BPERI),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_4, DA9063_BPERI_MON_EN),
 	},
 	{
 		DA9063_BUCK(DA9063, BCORES_MERGED, 300, 10, 1570,
@@ -618,6 +650,7 @@ static const struct da9063_regulator_info da9063_regulator_info[] = {
 			    DA9063_REG_BUCK_ILIM_C, DA9063_BCORE1_ILIM_MASK),
 		/* BCORES_MERGED uses the same register fields as BCORE1 */
 		DA9063_BUCK_COMMON_FIELDS(BCORE1),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_4, DA9063_BCORE1_MON_EN),
 	},
 	{
 		DA9063_BUCK(DA9063, BMEM_BIO_MERGED, 800, 20, 3340,
@@ -625,47 +658,59 @@ static const struct da9063_regulator_info da9063_regulator_info[] = {
 			    DA9063_REG_BUCK_ILIM_A, DA9063_BMEM_ILIM_MASK),
 		/* BMEM_BIO_MERGED uses the same register fields as BMEM */
 		DA9063_BUCK_COMMON_FIELDS(BMEM),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_4, DA9063_BMEM_MON_EN),
 	},
 	{
 		DA9063_LDO(DA9063, LDO3, 900, 20, 3440),
 		.oc_event = BFIELD(DA9063_REG_STATUS_D, DA9063_LDO3_LIM),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_2, DA9063_LDO3_MON_EN),
 	},
 	{
 		DA9063_LDO(DA9063, LDO7, 900, 50, 3600),
 		.oc_event = BFIELD(DA9063_REG_STATUS_D, DA9063_LDO7_LIM),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_2, DA9063_LDO7_MON_EN),
 	},
 	{
 		DA9063_LDO(DA9063, LDO8, 900, 50, 3600),
 		.oc_event = BFIELD(DA9063_REG_STATUS_D, DA9063_LDO8_LIM),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_2, DA9063_LDO8_MON_EN),
 	},
 	{
 		DA9063_LDO(DA9063, LDO9, 950, 50, 3600),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_3, DA9063_LDO9_MON_EN),
 	},
 	{
 		DA9063_LDO(DA9063, LDO11, 900, 50, 3600),
 		.oc_event = BFIELD(DA9063_REG_STATUS_D, DA9063_LDO11_LIM),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_3, DA9063_LDO11_MON_EN),
 	},
 
 	/* The following LDOs are present only on DA9063, not on DA9063L */
 	{
 		DA9063_LDO(DA9063, LDO1, 600, 20, 1860),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_2, DA9063_LDO1_MON_EN),
 	},
 	{
 		DA9063_LDO(DA9063, LDO2, 600, 20, 1860),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_2, DA9063_LDO2_MON_EN),
 	},
 	{
 		DA9063_LDO(DA9063, LDO4, 900, 20, 3440),
 		.oc_event = BFIELD(DA9063_REG_STATUS_D, DA9063_LDO4_LIM),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_2, DA9063_LDO4_MON_EN),
 	},
 	{
 		DA9063_LDO(DA9063, LDO5, 900, 50, 3600),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_2, DA9063_LDO5_MON_EN),
 	},
 	{
 		DA9063_LDO(DA9063, LDO6, 900, 50, 3600),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_2, DA9063_LDO6_MON_EN),
 	},
 
 	{
 		DA9063_LDO(DA9063, LDO10, 900, 50, 3600),
+		.vmon = BFIELD(DA9063_BB_REG_MON_REG_3, DA9063_LDO10_MON_EN),
 	},
 };
 
@@ -726,6 +771,41 @@ static const struct regulator_init_data *da9063_get_regulator_initdata(
 	return NULL;
 }
 
+static int da9063_check_xvp_constraints(struct regulator_config *config)
+{
+	struct da9063_regulator *regl = config->driver_data;
+	const struct regulation_constraints *constr = &config->init_data->constraints;
+	const struct notification_limit *uv_l = &constr->under_voltage_limits;
+	const struct notification_limit *ov_l = &constr->over_voltage_limits;
+
+	/* make sure that only one severity is used to clarify if unchanged, enabled or disabled */
+	if ((!!uv_l->prot + !!uv_l->err + !!uv_l->warn) > 1) {
+		dev_err(config->dev, "%s: at most one voltage monitoring severity allowed!\n",
+			regl->desc.name);
+		return -EINVAL;
+	}
+
+	/* make sure that UV and OV monitoring is set to the same severity and value */
+	if (uv_l->prot != ov_l->prot) {
+		dev_err(config->dev,
+			"%s: protection-microvolt: value must be equal for uv and ov!\n",
+			regl->desc.name);
+		return -EINVAL;
+	}
+	if (uv_l->err != ov_l->err) {
+		dev_err(config->dev, "%s: error-microvolt: value must be equal for uv and ov!\n",
+			regl->desc.name);
+		return -EINVAL;
+	}
+	if (uv_l->warn != ov_l->warn) {
+		dev_err(config->dev, "%s: warn-microvolt: value must be equal for uv and ov!\n",
+			regl->desc.name);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static struct of_regulator_match da9063_matches[] = {
 	[DA9063_ID_BCORE1]           = { .name = "bcore1"           },
 	[DA9063_ID_BCORE2]           = { .name = "bcore2"           },
@@ -932,6 +1012,12 @@ static int da9063_regulator_probe(struct platform_device *pdev)
 			if (IS_ERR(regl->suspend_sleep))
 				return PTR_ERR(regl->suspend_sleep);
 		}
+		if (regl->info->vmon.reg) {
+			regl->vmon = devm_regmap_field_alloc(&pdev->dev,
+				da9063->regmap, regl->info->vmon);
+			if (IS_ERR(regl->vmon))
+				return PTR_ERR(regl->vmon);
+		}
 
 		/* Register regulator */
 		memset(&config, 0, sizeof(config));
@@ -941,6 +1027,11 @@ static int da9063_regulator_probe(struct platform_device *pdev)
 		if (da9063_reg_matches)
 			config.of_node = da9063_reg_matches[id].of_node;
 		config.regmap = da9063->regmap;
+
+		ret = da9063_check_xvp_constraints(&config);
+		if (ret)
+			return ret;
+
 		regl->rdev = devm_regulator_register(&pdev->dev, &regl->desc,
 						     &config);
 		if (IS_ERR(regl->rdev)) {
@@ -971,6 +1062,7 @@ static int da9063_regulator_probe(struct platform_device *pdev)
 static struct platform_driver da9063_regulator_driver = {
 	.driver = {
 		.name = DA9063_DRVNAME_REGULATORS,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = da9063_regulator_probe,
 };
diff --git a/drivers/regulator/da9121-regulator.c b/drivers/regulator/da9121-regulator.c
index d016e049d264..6ce0fdc18b9c 100644
--- a/drivers/regulator/da9121-regulator.c
+++ b/drivers/regulator/da9121-regulator.c
@@ -1194,6 +1194,7 @@ MODULE_DEVICE_TABLE(i2c, da9121_i2c_id);
 static struct i2c_driver da9121_regulator_driver = {
 	.driver = {
 		.name = "da9121",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(da9121_dt_ids),
 	},
 	.probe_new = da9121_i2c_probe,
diff --git a/drivers/regulator/da9210-regulator.c b/drivers/regulator/da9210-regulator.c
index 7493af0b5c04..4332a3b8a672 100644
--- a/drivers/regulator/da9210-regulator.c
+++ b/drivers/regulator/da9210-regulator.c
@@ -221,6 +221,7 @@ MODULE_DEVICE_TABLE(i2c, da9210_i2c_id);
 static struct i2c_driver da9210_regulator_driver = {
 	.driver = {
 		.name = "da9210",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(da9210_dt_ids),
 	},
 	.probe_new = da9210_i2c_probe,
diff --git a/drivers/regulator/da9211-regulator.c b/drivers/regulator/da9211-regulator.c
index 00828f5baa97..a2b4f6f1e34b 100644
--- a/drivers/regulator/da9211-regulator.c
+++ b/drivers/regulator/da9211-regulator.c
@@ -552,6 +552,7 @@ MODULE_DEVICE_TABLE(of, da9211_dt_ids);
 static struct i2c_driver da9211_regulator_driver = {
 	.driver = {
 		.name = "da9211",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(da9211_dt_ids),
 	},
 	.probe_new = da9211_i2c_probe,
diff --git a/drivers/regulator/db8500-prcmu.c b/drivers/regulator/db8500-prcmu.c
index 0ce6ec4933af..34c5e485d0af 100644
--- a/drivers/regulator/db8500-prcmu.c
+++ b/drivers/regulator/db8500-prcmu.c
@@ -479,6 +479,7 @@ static int db8500_regulator_remove(struct platform_device *pdev)
 static struct platform_driver db8500_regulator_driver = {
 	.driver = {
 		.name = "db8500-prcmu-regulators",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = db8500_regulator_probe,
 	.remove = db8500_regulator_remove,
diff --git a/drivers/regulator/dummy.c b/drivers/regulator/dummy.c
index 24e586f93855..5b9b9e4e762d 100644
--- a/drivers/regulator/dummy.c
+++ b/drivers/regulator/dummy.c
@@ -60,6 +60,7 @@ static struct platform_driver dummy_regulator_driver = {
 	.probe		= dummy_regulator_probe,
 	.driver		= {
 		.name		= "reg-dummy",
+		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/fan53555.c b/drivers/regulator/fan53555.c
index 529963a7e4f5..130f3dbe9840 100644
--- a/drivers/regulator/fan53555.c
+++ b/drivers/regulator/fan53555.c
@@ -8,23 +8,27 @@
 // Copyright (c) 2012 Marvell Technology Ltd.
 // Yunfan Zhang <yfzhang@marvell.com>
 
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/param.h>
-#include <linux/err.h>
 #include <linux/platform_device.h>
+#include <linux/regmap.h>
 #include <linux/regulator/driver.h>
+#include <linux/regulator/fan53555.h>
 #include <linux/regulator/machine.h>
 #include <linux/regulator/of_regulator.h>
-#include <linux/of_device.h>
-#include <linux/i2c.h>
 #include <linux/slab.h>
-#include <linux/regmap.h>
-#include <linux/regulator/fan53555.h>
 
 /* Voltage setting */
 #define FAN53555_VSEL0		0x00
 #define FAN53555_VSEL1		0x01
 
+#define RK8602_VSEL0		0x06
+#define RK8602_VSEL1		0x07
+
 #define TCS4525_VSEL0		0x11
 #define TCS4525_VSEL1		0x10
 #define TCS4525_TIME		0x13
@@ -40,31 +44,32 @@
 #define FAN53555_MONITOR	0x05
 
 /* VSEL bit definitions */
-#define VSEL_BUCK_EN	(1 << 7)
-#define VSEL_MODE		(1 << 6)
+#define VSEL_BUCK_EN		BIT(7)
+#define VSEL_MODE		BIT(6)
 /* Chip ID and Verison */
-#define DIE_ID		0x0F	/* ID1 */
-#define DIE_REV		0x0F	/* ID2 */
+#define DIE_ID			0x0F	/* ID1 */
+#define DIE_REV			0x0F	/* ID2 */
 /* Control bit definitions */
-#define CTL_OUTPUT_DISCHG	(1 << 7)
-#define CTL_SLEW_MASK		(0x7 << 4)
-#define CTL_SLEW_SHIFT		4
-#define CTL_RESET			(1 << 2)
+#define CTL_OUTPUT_DISCHG	BIT(7)
+#define CTL_SLEW_MASK		GENMASK(6, 4)
+#define CTL_RESET		BIT(2)
 #define CTL_MODE_VSEL0_MODE	BIT(0)
 #define CTL_MODE_VSEL1_MODE	BIT(1)
 
 #define FAN53555_NVOLTAGES	64	/* Numbers of voltages */
 #define FAN53526_NVOLTAGES	128
+#define RK8602_NVOLTAGES	160
 
-#define TCS_VSEL0_MODE		(1 << 7)
-#define TCS_VSEL1_MODE		(1 << 6)
+#define TCS_VSEL0_MODE		BIT(7)
+#define TCS_VSEL1_MODE		BIT(6)
 
-#define TCS_SLEW_SHIFT		3
-#define TCS_SLEW_MASK		(0x3 < 3)
+#define TCS_SLEW_MASK		GENMASK(4, 3)
 
 enum fan53555_vendor {
 	FAN53526_VENDOR_FAIRCHILD = 0,
 	FAN53555_VENDOR_FAIRCHILD,
+	FAN53555_VENDOR_ROCKCHIP,	/* RK8600, RK8601 */
+	RK8602_VENDOR_ROCKCHIP,		/* RK8602, RK8603 */
 	FAN53555_VENDOR_SILERGY,
 	FAN53526_VENDOR_TCS,
 };
@@ -89,6 +94,14 @@ enum {
 };
 
 enum {
+	RK8600_CHIP_ID_08 = 8,		/* RK8600, RK8601 */
+};
+
+enum {
+	RK8602_CHIP_ID_10 = 10,		/* RK8602, RK8603 */
+};
+
+enum {
 	TCS4525_CHIP_ID_12 = 12,
 };
 
@@ -118,6 +131,8 @@ struct fan53555_device_info {
 	/* Voltage setting register */
 	unsigned int vol_reg;
 	unsigned int sleep_reg;
+	unsigned int en_reg;
+	unsigned int sleep_en_reg;
 	/* Voltage range and step(linear) */
 	unsigned int vsel_min;
 	unsigned int vsel_step;
@@ -160,7 +175,7 @@ static int fan53555_set_suspend_enable(struct regulator_dev *rdev)
 {
 	struct fan53555_device_info *di = rdev_get_drvdata(rdev);
 
-	return regmap_update_bits(rdev->regmap, di->sleep_reg,
+	return regmap_update_bits(rdev->regmap, di->sleep_en_reg,
 				  VSEL_BUCK_EN, VSEL_BUCK_EN);
 }
 
@@ -168,7 +183,7 @@ static int fan53555_set_suspend_disable(struct regulator_dev *rdev)
 {
 	struct fan53555_device_info *di = rdev_get_drvdata(rdev);
 
-	return regmap_update_bits(rdev->regmap, di->sleep_reg,
+	return regmap_update_bits(rdev->regmap, di->sleep_en_reg,
 				  VSEL_BUCK_EN, 0);
 }
 
@@ -318,6 +333,50 @@ static int fan53555_voltages_setup_fairchild(struct fan53555_device_info *di)
 	return 0;
 }
 
+static int fan53555_voltages_setup_rockchip(struct fan53555_device_info *di)
+{
+	/* Init voltage range and step */
+	switch (di->chip_id) {
+	case RK8600_CHIP_ID_08:
+		di->vsel_min = 712500;
+		di->vsel_step = 12500;
+		break;
+	default:
+		dev_err(di->dev,
+			"Chip ID %d not supported!\n", di->chip_id);
+		return -EINVAL;
+	}
+	di->slew_reg = FAN53555_CONTROL;
+	di->slew_mask = CTL_SLEW_MASK;
+	di->ramp_delay_table = slew_rates;
+	di->n_ramp_values = ARRAY_SIZE(slew_rates);
+	di->vsel_count = FAN53555_NVOLTAGES;
+
+	return 0;
+}
+
+static int rk8602_voltages_setup_rockchip(struct fan53555_device_info *di)
+{
+	/* Init voltage range and step */
+	switch (di->chip_id) {
+	case RK8602_CHIP_ID_10:
+		di->vsel_min = 500000;
+		di->vsel_step = 6250;
+		break;
+	default:
+		dev_err(di->dev,
+			"Chip ID %d not supported!\n", di->chip_id);
+		return -EINVAL;
+	}
+	di->slew_reg = FAN53555_CONTROL;
+	di->slew_mask = CTL_SLEW_MASK;
+	di->ramp_delay_table = slew_rates;
+	di->n_ramp_values = ARRAY_SIZE(slew_rates);
+	di->vsel_count = RK8602_NVOLTAGES;
+
+	return 0;
+}
+
 static int fan53555_voltages_setup_silergy(struct fan53555_device_info *di)
 {
 	/* Init voltage range and step */
@@ -378,6 +437,7 @@ static int fan53555_device_setup(struct fan53555_device_info *di,
 	switch (di->vendor) {
 	case FAN53526_VENDOR_FAIRCHILD:
 	case FAN53555_VENDOR_FAIRCHILD:
+	case FAN53555_VENDOR_ROCKCHIP:
 	case FAN53555_VENDOR_SILERGY:
 		switch (pdata->sleep_vsel_id) {
 		case FAN53555_VSEL_ID_0:
@@ -392,6 +452,27 @@ static int fan53555_device_setup(struct fan53555_device_info *di,
 			dev_err(di->dev, "Invalid VSEL ID!\n");
 			return -EINVAL;
 		}
+		di->sleep_en_reg = di->sleep_reg;
+		di->en_reg = di->vol_reg;
+		break;
+	case RK8602_VENDOR_ROCKCHIP:
+		switch (pdata->sleep_vsel_id) {
+		case FAN53555_VSEL_ID_0:
+			di->sleep_reg = RK8602_VSEL0;
+			di->vol_reg = RK8602_VSEL1;
+			di->sleep_en_reg = FAN53555_VSEL0;
+			di->en_reg = FAN53555_VSEL1;
+			break;
+		case FAN53555_VSEL_ID_1:
+			di->sleep_reg = RK8602_VSEL1;
+			di->vol_reg = RK8602_VSEL0;
+			di->sleep_en_reg = FAN53555_VSEL1;
+			di->en_reg = FAN53555_VSEL0;
+			break;
+		default:
+			dev_err(di->dev, "Invalid VSEL ID!\n");
+			return -EINVAL;
+		}
 		break;
 	case FAN53526_VENDOR_TCS:
 		switch (pdata->sleep_vsel_id) {
@@ -407,6 +488,8 @@ static int fan53555_device_setup(struct fan53555_device_info *di,
 			dev_err(di->dev, "Invalid VSEL ID!\n");
 			return -EINVAL;
 		}
+		di->sleep_en_reg = di->sleep_reg;
+		di->en_reg = di->vol_reg;
 		break;
 	default:
 		dev_err(di->dev, "vendor %d not supported!\n", di->vendor);
@@ -428,10 +511,23 @@ static int fan53555_device_setup(struct fan53555_device_info *di,
 		}
 		break;
 	case FAN53555_VENDOR_FAIRCHILD:
+	case FAN53555_VENDOR_ROCKCHIP:
 	case FAN53555_VENDOR_SILERGY:
 		di->mode_reg = di->vol_reg;
 		di->mode_mask = VSEL_MODE;
 		break;
+	case RK8602_VENDOR_ROCKCHIP:
+		di->mode_mask = VSEL_MODE;
+
+		switch (pdata->sleep_vsel_id) {
+		case FAN53555_VSEL_ID_0:
+			di->mode_reg = FAN53555_VSEL1;
+			break;
+		case FAN53555_VSEL_ID_1:
+			di->mode_reg = FAN53555_VSEL0;
+			break;
+		}
+		break;
 	case FAN53526_VENDOR_TCS:
 		di->mode_reg = TCS4525_COMMAND;
 
@@ -457,6 +553,12 @@ static int fan53555_device_setup(struct fan53555_device_info *di,
 	case FAN53555_VENDOR_FAIRCHILD:
 		ret = fan53555_voltages_setup_fairchild(di);
 		break;
+	case FAN53555_VENDOR_ROCKCHIP:
+		ret = fan53555_voltages_setup_rockchip(di);
+		break;
+	case RK8602_VENDOR_ROCKCHIP:
+		ret = rk8602_voltages_setup_rockchip(di);
+		break;
 	case FAN53555_VENDOR_SILERGY:
 		ret = fan53555_voltages_setup_silergy(di);
 		break;
@@ -482,12 +584,12 @@ static int fan53555_regulator_register(struct fan53555_device_info *di,
 	rdesc->ops = &fan53555_regulator_ops;
 	rdesc->type = REGULATOR_VOLTAGE;
 	rdesc->n_voltages = di->vsel_count;
-	rdesc->enable_reg = di->vol_reg;
+	rdesc->enable_reg = di->en_reg;
 	rdesc->enable_mask = VSEL_BUCK_EN;
 	rdesc->min_uV = di->vsel_min;
 	rdesc->uV_step = di->vsel_step;
 	rdesc->vsel_reg = di->vol_reg;
-	rdesc->vsel_mask = di->vsel_count - 1;
+	rdesc->vsel_mask = BIT(fls(di->vsel_count - 1)) - 1;
 	rdesc->ramp_reg = di->slew_reg;
 	rdesc->ramp_mask = di->slew_mask;
 	rdesc->ramp_delay_table = di->ramp_delay_table;
@@ -533,6 +635,12 @@ static const struct of_device_id __maybe_unused fan53555_dt_ids[] = {
 		.compatible = "fcs,fan53555",
 		.data = (void *)FAN53555_VENDOR_FAIRCHILD
 	}, {
+		.compatible = "rockchip,rk8600",
+		.data = (void *)FAN53555_VENDOR_ROCKCHIP
+	}, {
+		.compatible = "rockchip,rk8602",
+		.data = (void *)RK8602_VENDOR_ROCKCHIP
+	}, {
 		.compatible = "silergy,syr827",
 		.data = (void *)FAN53555_VENDOR_SILERGY,
 	}, {
@@ -569,10 +677,9 @@ static int fan53555_regulator_probe(struct i2c_client *client)
 	if (!pdata)
 		pdata = fan53555_parse_dt(&client->dev, np, &di->desc);
 
-	if (!pdata || !pdata->regulator) {
-		dev_err(&client->dev, "Platform data not found!\n");
-		return -ENODEV;
-	}
+	if (!pdata || !pdata->regulator)
+		return dev_err_probe(&client->dev, -ENODEV,
+				     "Platform data not found!\n");
 
 	di->regulator = pdata->regulator;
 	if (client->dev.of_node) {
@@ -581,10 +688,9 @@ static int fan53555_regulator_probe(struct i2c_client *client)
 	} else {
 		/* if no ramp constraint set, get the pdata ramp_delay */
 		if (!di->regulator->constraints.ramp_delay) {
-			if (pdata->slew_rate >= ARRAY_SIZE(slew_rates)) {
-				dev_err(&client->dev, "Invalid slew_rate\n");
-				return -EINVAL;
-			}
+			if (pdata->slew_rate >= ARRAY_SIZE(slew_rates))
+				return dev_err_probe(&client->dev, -EINVAL,
+						     "Invalid slew_rate\n");
 
 			di->regulator->constraints.ramp_delay
 					= slew_rates[pdata->slew_rate];
@@ -594,34 +700,31 @@ static int fan53555_regulator_probe(struct i2c_client *client)
 	}
 
 	regmap = devm_regmap_init_i2c(client, &fan53555_regmap_config);
-	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Failed to allocate regmap!\n");
-		return PTR_ERR(regmap);
-	}
+	if (IS_ERR(regmap))
+		return dev_err_probe(&client->dev, PTR_ERR(regmap),
+				     "Failed to allocate regmap!\n");
+
 	di->dev = &client->dev;
 	i2c_set_clientdata(client, di);
 	/* Get chip ID */
 	ret = regmap_read(regmap, FAN53555_ID1, &val);
-	if (ret < 0) {
-		dev_err(&client->dev, "Failed to get chip ID!\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(&client->dev, ret, "Failed to get chip ID!\n");
+
 	di->chip_id = val & DIE_ID;
 	/* Get chip revision */
 	ret = regmap_read(regmap, FAN53555_ID2, &val);
-	if (ret < 0) {
-		dev_err(&client->dev, "Failed to get chip Rev!\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(&client->dev, ret, "Failed to get chip Rev!\n");
+
 	di->chip_rev = val & DIE_REV;
 	dev_info(&client->dev, "FAN53555 Option[%d] Rev[%d] Detected!\n",
 				di->chip_id, di->chip_rev);
 	/* Device init */
 	ret = fan53555_device_setup(di, pdata);
-	if (ret < 0) {
-		dev_err(&client->dev, "Failed to setup device!\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(&client->dev, ret, "Failed to setup device!\n");
+
 	/* Register regulator */
 	config.dev = di->dev;
 	config.init_data = di->regulator;
@@ -631,9 +734,9 @@ static int fan53555_regulator_probe(struct i2c_client *client)
 
 	ret = fan53555_regulator_register(di, &config);
 	if (ret < 0)
-		dev_err(&client->dev, "Failed to register regulator!\n");
-	return ret;
+		dev_err_probe(&client->dev, ret, "Failed to register regulator!\n");
 
+	return ret;
 }
 
 static const struct i2c_device_id fan53555_id[] = {
@@ -644,6 +747,12 @@ static const struct i2c_device_id fan53555_id[] = {
 		.name = "fan53555",
 		.driver_data = FAN53555_VENDOR_FAIRCHILD
 	}, {
+		.name = "rk8600",
+		.driver_data = FAN53555_VENDOR_ROCKCHIP
+	}, {
+		.name = "rk8602",
+		.driver_data = RK8602_VENDOR_ROCKCHIP
+	}, {
 		.name = "syr827",
 		.driver_data = FAN53555_VENDOR_SILERGY
 	}, {
@@ -663,6 +772,7 @@ MODULE_DEVICE_TABLE(i2c, fan53555_id);
 static struct i2c_driver fan53555_regulator_driver = {
 	.driver = {
 		.name = "fan53555-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(fan53555_dt_ids),
 	},
 	.probe_new = fan53555_regulator_probe,
diff --git a/drivers/regulator/fan53880.c b/drivers/regulator/fan53880.c
index 1d88d5381544..a3bebdee570e 100644
--- a/drivers/regulator/fan53880.c
+++ b/drivers/regulator/fan53880.c
@@ -172,6 +172,7 @@ MODULE_DEVICE_TABLE(i2c, fan53880_i2c_id);
 static struct i2c_driver fan53880_regulator_driver = {
 	.driver = {
 		.name = "fan53880",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= fan53880_dt_ids,
 	},
 	.probe_new = fan53880_i2c_probe,
diff --git a/drivers/regulator/fixed.c b/drivers/regulator/fixed.c
index e6724a229d23..364d1a2683b7 100644
--- a/drivers/regulator/fixed.c
+++ b/drivers/regulator/fixed.c
@@ -151,7 +151,7 @@ of_get_fixed_voltage_config(struct device *dev,
 	of_property_read_u32(np, "startup-delay-us", &config->startup_delay);
 	of_property_read_u32(np, "off-on-delay-us", &config->off_on_delay);
 
-	if (of_find_property(np, "vin-supply", NULL))
+	if (of_property_present(np, "vin-supply"))
 		config->input_supply = "vin";
 
 	return config;
@@ -334,6 +334,7 @@ static struct platform_driver regulator_fixed_voltage_driver = {
 	.probe		= reg_fixed_voltage_probe,
 	.driver		= {
 		.name		= "reg-fixed-voltage",
+		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(fixed_of_match),
 	},
 };
diff --git a/drivers/regulator/gpio-regulator.c b/drivers/regulator/gpio-regulator.c
index 95e61a2f43f5..65927fa2ef16 100644
--- a/drivers/regulator/gpio-regulator.c
+++ b/drivers/regulator/gpio-regulator.c
@@ -220,7 +220,7 @@ of_get_gpio_regulator_config(struct device *dev, struct device_node *np,
 				 regtype);
 	}
 
-	if (of_find_property(np, "vin-supply", NULL))
+	if (of_property_present(np, "vin-supply"))
 		config->input_supply = "vin";
 
 	return config;
@@ -368,6 +368,7 @@ static struct platform_driver gpio_regulator_driver = {
 	.probe		= gpio_regulator_probe,
 	.driver		= {
 		.name		= "gpio-regulator",
+		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(regulator_gpio_of_match),
 	},
 };
diff --git a/drivers/regulator/hi6421-regulator.c b/drivers/regulator/hi6421-regulator.c
index d144a4bdb76d..1b52423598d3 100644
--- a/drivers/regulator/hi6421-regulator.c
+++ b/drivers/regulator/hi6421-regulator.c
@@ -579,6 +579,7 @@ static struct platform_driver hi6421_regulator_driver = {
 	.id_table = hi6421_regulator_table,
 	.driver = {
 		.name	= "hi6421-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe	= hi6421_regulator_probe,
 };
diff --git a/drivers/regulator/hi6421v530-regulator.c b/drivers/regulator/hi6421v530-regulator.c
index 988115f9b594..23924ff0c7b2 100644
--- a/drivers/regulator/hi6421v530-regulator.c
+++ b/drivers/regulator/hi6421v530-regulator.c
@@ -200,6 +200,7 @@ static struct platform_driver hi6421v530_regulator_driver = {
 	.id_table = hi6421v530_regulator_table,
 	.driver = {
 		.name	= "hi6421v530-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe	= hi6421v530_regulator_probe,
 };
diff --git a/drivers/regulator/hi6421v600-regulator.c b/drivers/regulator/hi6421v600-regulator.c
index 4671678f6b19..4e10daa1e689 100644
--- a/drivers/regulator/hi6421v600-regulator.c
+++ b/drivers/regulator/hi6421v600-regulator.c
@@ -284,6 +284,7 @@ static struct platform_driver hi6421_spmi_regulator_driver = {
 	.id_table = hi6421_spmi_regulator_table,
 	.driver = {
 		.name = "hi6421v600-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe	= hi6421_spmi_regulator_probe,
 };
diff --git a/drivers/regulator/hi655x-regulator.c b/drivers/regulator/hi655x-regulator.c
index 556bb73f3329..1d8211f635b7 100644
--- a/drivers/regulator/hi655x-regulator.c
+++ b/drivers/regulator/hi655x-regulator.c
@@ -206,6 +206,7 @@ static struct platform_driver hi655x_regulator_driver = {
 	.id_table = hi655x_regulator_table,
 	.driver = {
 		.name	= "hi655x-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe	= hi655x_regulator_probe,
 };
diff --git a/drivers/regulator/isl6271a-regulator.c b/drivers/regulator/isl6271a-regulator.c
index b23b052eab10..3c37c4de1d82 100644
--- a/drivers/regulator/isl6271a-regulator.c
+++ b/drivers/regulator/isl6271a-regulator.c
@@ -147,6 +147,7 @@ MODULE_DEVICE_TABLE(i2c, isl6271a_id);
 static struct i2c_driver isl6271a_i2c_driver = {
 	.driver = {
 		.name = "isl6271a",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe_new = isl6271a_probe,
 	.id_table = isl6271a_id,
diff --git a/drivers/regulator/isl9305.c b/drivers/regulator/isl9305.c
index cfb765986d0d..90bc8d054304 100644
--- a/drivers/regulator/isl9305.c
+++ b/drivers/regulator/isl9305.c
@@ -195,6 +195,7 @@ MODULE_DEVICE_TABLE(i2c, isl9305_i2c_id);
 static struct i2c_driver isl9305_regulator_driver = {
 	.driver = {
 		.name = "isl9305",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= of_match_ptr(isl9305_dt_ids),
 	},
 	.probe_new = isl9305_i2c_probe,
diff --git a/drivers/regulator/lm363x-regulator.c b/drivers/regulator/lm363x-regulator.c
index 4b9f618b07e9..7531b2c37f95 100644
--- a/drivers/regulator/lm363x-regulator.c
+++ b/drivers/regulator/lm363x-regulator.c
@@ -355,6 +355,7 @@ static struct platform_driver lm363x_regulator_driver = {
 	.probe = lm363x_regulator_probe,
 	.driver = {
 		.name = "lm363x-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/lochnagar-regulator.c b/drivers/regulator/lochnagar-regulator.c
index cb71fa5f43c3..11b358efbc92 100644
--- a/drivers/regulator/lochnagar-regulator.c
+++ b/drivers/regulator/lochnagar-regulator.c
@@ -272,6 +272,7 @@ static int lochnagar_regulator_probe(struct platform_device *pdev)
 static struct platform_driver lochnagar_regulator_driver = {
 	.driver = {
 		.name = "lochnagar-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(lochnagar_of_match),
 	},
 
diff --git a/drivers/regulator/lp3971.c b/drivers/regulator/lp3971.c
index 8be252f81b09..e06f2a092b89 100644
--- a/drivers/regulator/lp3971.c
+++ b/drivers/regulator/lp3971.c
@@ -447,6 +447,7 @@ MODULE_DEVICE_TABLE(i2c, lp3971_i2c_id);
 static struct i2c_driver lp3971_i2c_driver = {
 	.driver = {
 		.name = "LP3971",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe_new = lp3971_i2c_probe,
 	.id_table = lp3971_i2c_id,
diff --git a/drivers/regulator/lp3972.c b/drivers/regulator/lp3972.c
index 27b216bf18fc..edacca8e14af 100644
--- a/drivers/regulator/lp3972.c
+++ b/drivers/regulator/lp3972.c
@@ -545,6 +545,7 @@ MODULE_DEVICE_TABLE(i2c, lp3972_i2c_id);
 static struct i2c_driver lp3972_i2c_driver = {
 	.driver = {
 		.name = "lp3972",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe_new = lp3972_i2c_probe,
 	.id_table = lp3972_i2c_id,
diff --git a/drivers/regulator/lp872x.c b/drivers/regulator/lp872x.c
index c576894c3d52..a8b0969d4f31 100644
--- a/drivers/regulator/lp872x.c
+++ b/drivers/regulator/lp872x.c
@@ -832,8 +832,7 @@ static struct lp872x_platform_data
 		return ERR_PTR(-ENOMEM);
 
 	of_property_read_u8(np, "ti,general-config", &pdata->general_config);
-	if (of_find_property(np, "ti,update-config", NULL))
-		pdata->update_config = true;
+	pdata->update_config = of_property_read_bool(np, "ti,update-config");
 
 	pdata->dvs = devm_kzalloc(dev, sizeof(struct lp872x_dvs), GFP_KERNEL);
 	if (!pdata->dvs)
@@ -928,7 +927,7 @@ static int lp872x_probe(struct i2c_client *cl)
 	return lp872x_regulator_register(lp);
 }
 
-static const struct of_device_id lp872x_dt_ids[] = {
+static const struct of_device_id lp872x_dt_ids[] __maybe_unused = {
 	{ .compatible = "ti,lp8720", },
 	{ .compatible = "ti,lp8725", },
 	{ }
@@ -945,6 +944,7 @@ MODULE_DEVICE_TABLE(i2c, lp872x_ids);
 static struct i2c_driver lp872x_driver = {
 	.driver = {
 		.name = "lp872x",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(lp872x_dt_ids),
 	},
 	.probe_new = lp872x_probe,
diff --git a/drivers/regulator/lp873x-regulator.c b/drivers/regulator/lp873x-regulator.c
index d6e597922cb5..8dfdd1db2070 100644
--- a/drivers/regulator/lp873x-regulator.c
+++ b/drivers/regulator/lp873x-regulator.c
@@ -187,6 +187,7 @@ MODULE_DEVICE_TABLE(platform, lp873x_regulator_id_table);
 static struct platform_driver lp873x_regulator_driver = {
 	.driver = {
 		.name = "lp873x-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = lp873x_regulator_probe,
 	.id_table = lp873x_regulator_id_table,
diff --git a/drivers/regulator/lp8755.c b/drivers/regulator/lp8755.c
index 467dfdcebc91..37b51b94fb5a 100644
--- a/drivers/regulator/lp8755.c
+++ b/drivers/regulator/lp8755.c
@@ -440,6 +440,7 @@ MODULE_DEVICE_TABLE(i2c, lp8755_id);
 static struct i2c_driver lp8755_i2c_driver = {
 	.driver = {
 		   .name = LP8755_NAME,
+		   .probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		   },
 	.probe_new = lp8755_probe,
 	.remove = lp8755_remove,
diff --git a/drivers/regulator/lp87565-regulator.c b/drivers/regulator/lp87565-regulator.c
index d059ae85047a..bdb60d8a7f3d 100644
--- a/drivers/regulator/lp87565-regulator.c
+++ b/drivers/regulator/lp87565-regulator.c
@@ -237,6 +237,7 @@ MODULE_DEVICE_TABLE(platform, lp87565_regulator_id_table);
 static struct platform_driver lp87565_regulator_driver = {
 	.driver = {
 		.name = "lp87565-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = lp87565_regulator_probe,
 	.id_table = lp87565_regulator_id_table,
diff --git a/drivers/regulator/lp8788-buck.c b/drivers/regulator/lp8788-buck.c
index 74b7b496b12d..e97ade09dede 100644
--- a/drivers/regulator/lp8788-buck.c
+++ b/drivers/regulator/lp8788-buck.c
@@ -531,6 +531,7 @@ static struct platform_driver lp8788_buck_driver = {
 	.probe = lp8788_buck_probe,
 	.driver = {
 		.name = LP8788_DEV_BUCK,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/lp8788-ldo.c b/drivers/regulator/lp8788-ldo.c
index 00e9bb92c326..8e45b7b99556 100644
--- a/drivers/regulator/lp8788-ldo.c
+++ b/drivers/regulator/lp8788-ldo.c
@@ -564,6 +564,7 @@ static struct platform_driver lp8788_dldo_driver = {
 	.probe = lp8788_dldo_probe,
 	.driver = {
 		.name = LP8788_DEV_DLDO,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
@@ -611,6 +612,7 @@ static struct platform_driver lp8788_aldo_driver = {
 	.probe = lp8788_aldo_probe,
 	.driver = {
 		.name = LP8788_DEV_ALDO,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/ltc3589.c b/drivers/regulator/ltc3589.c
index 460d34c50fb0..359b534d8c70 100644
--- a/drivers/regulator/ltc3589.c
+++ b/drivers/regulator/ltc3589.c
@@ -474,6 +474,7 @@ MODULE_DEVICE_TABLE(of, ltc3589_of_match);
 static struct i2c_driver ltc3589_driver = {
 	.driver = {
 		.name = DRIVER_NAME,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(ltc3589_of_match),
 	},
 	.probe_new = ltc3589_probe,
diff --git a/drivers/regulator/ltc3676.c b/drivers/regulator/ltc3676.c
index eb3d6bed6d54..a28e6c3460f1 100644
--- a/drivers/regulator/ltc3676.c
+++ b/drivers/regulator/ltc3676.c
@@ -371,6 +371,7 @@ MODULE_DEVICE_TABLE(of, ltc3676_of_match);
 static struct i2c_driver ltc3676_driver = {
 	.driver = {
 		.name = DRIVER_NAME,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(ltc3676_of_match),
 	},
 	.probe_new = ltc3676_regulator_probe,
diff --git a/drivers/regulator/max14577-regulator.c b/drivers/regulator/max14577-regulator.c
index e34face736f4..5e7171b9065a 100644
--- a/drivers/regulator/max14577-regulator.c
+++ b/drivers/regulator/max14577-regulator.c
@@ -241,6 +241,7 @@ MODULE_DEVICE_TABLE(platform, max14577_regulator_id);
 static struct platform_driver max14577_regulator_driver = {
 	.driver = {
 		   .name = "max14577-regulator",
+		   .probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		   },
 	.probe		= max14577_regulator_probe,
 	.id_table	= max14577_regulator_id,
diff --git a/drivers/regulator/max1586.c b/drivers/regulator/max1586.c
index a00aa2e8ff3f..5d8852b2c168 100644
--- a/drivers/regulator/max1586.c
+++ b/drivers/regulator/max1586.c
@@ -292,6 +292,7 @@ static struct i2c_driver max1586_pmic_driver = {
 	.probe_new = max1586_pmic_probe,
 	.driver		= {
 		.name	= "max1586",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(max1586_of_match),
 	},
 	.id_table	= max1586_id,
diff --git a/drivers/regulator/max20086-regulator.c b/drivers/regulator/max20086-regulator.c
index b8bf76c170fe..ace1d582a191 100644
--- a/drivers/regulator/max20086-regulator.c
+++ b/drivers/regulator/max20086-regulator.c
@@ -286,7 +286,7 @@ static const struct i2c_device_id max20086_i2c_id[] = {
 
 MODULE_DEVICE_TABLE(i2c, max20086_i2c_id);
 
-static const struct of_device_id max20086_dt_ids[] = {
+static const struct of_device_id max20086_dt_ids[] __maybe_unused = {
 	{
 		.compatible = "maxim,max20086",
 		.data = &(const struct max20086_chip_info) {
@@ -320,6 +320,7 @@ MODULE_DEVICE_TABLE(of, max20086_dt_ids);
 static struct i2c_driver max20086_regulator_driver = {
 	.driver = {
 		.name = "max20086",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(max20086_dt_ids),
 	},
 	.probe_new = max20086_i2c_probe,
diff --git a/drivers/regulator/max20411-regulator.c b/drivers/regulator/max20411-regulator.c
index 83dacb4ff173..be8169b86a89 100644
--- a/drivers/regulator/max20411-regulator.c
+++ b/drivers/regulator/max20411-regulator.c
@@ -153,6 +153,7 @@ MODULE_DEVICE_TABLE(i2c, max20411_id);
 static struct i2c_driver max20411_i2c_driver = {
 	.driver	= {
 		.name = "max20411",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= of_max20411_match_tbl,
 	},
 	.probe_new = max20411_probe,
diff --git a/drivers/regulator/max597x-regulator.c b/drivers/regulator/max597x-regulator.c
index 648e3641885a..7873a5267555 100644
--- a/drivers/regulator/max597x-regulator.c
+++ b/drivers/regulator/max597x-regulator.c
@@ -501,6 +501,7 @@ static int max597x_regulator_probe(struct platform_device *pdev)
 static struct platform_driver max597x_regulator_driver = {
 	.driver = {
 		.name = "max597x-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = max597x_regulator_probe,
 };
diff --git a/drivers/regulator/max77620-regulator.c b/drivers/regulator/max77620-regulator.c
index 3cf8f085170a..7bc87d8e9f68 100644
--- a/drivers/regulator/max77620-regulator.c
+++ b/drivers/regulator/max77620-regulator.c
@@ -916,6 +916,7 @@ static struct platform_driver max77620_regulator_driver = {
 	.id_table = max77620_regulator_devtype,
 	.driver = {
 		.name = "max77620-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.pm = &max77620_regulator_pm_ops,
 	},
 };
diff --git a/drivers/regulator/max77650-regulator.c b/drivers/regulator/max77650-regulator.c
index ca08f94a368d..f6539b945037 100644
--- a/drivers/regulator/max77650-regulator.c
+++ b/drivers/regulator/max77650-regulator.c
@@ -395,6 +395,7 @@ MODULE_DEVICE_TABLE(of, max77650_regulator_of_match);
 static struct platform_driver max77650_regulator_driver = {
 	.driver = {
 		.name = "max77650-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = max77650_regulator_of_match,
 	},
 	.probe = max77650_regulator_probe,
diff --git a/drivers/regulator/max77686-regulator.c b/drivers/regulator/max77686-regulator.c
index 55a07d3f3ee2..c7b270fd9e0c 100644
--- a/drivers/regulator/max77686-regulator.c
+++ b/drivers/regulator/max77686-regulator.c
@@ -525,6 +525,7 @@ MODULE_DEVICE_TABLE(platform, max77686_pmic_id);
 static struct platform_driver max77686_pmic_driver = {
 	.driver = {
 		.name = "max77686-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = max77686_pmic_probe,
 	.id_table = max77686_pmic_id,
diff --git a/drivers/regulator/max77693-regulator.c b/drivers/regulator/max77693-regulator.c
index 077ecbbfdf76..72a67d0c5f1e 100644
--- a/drivers/regulator/max77693-regulator.c
+++ b/drivers/regulator/max77693-regulator.c
@@ -281,6 +281,7 @@ MODULE_DEVICE_TABLE(platform, max77693_pmic_id);
 static struct platform_driver max77693_pmic_driver = {
 	.driver = {
 		   .name = "max77693-pmic",
+		   .probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		   },
 	.probe = max77693_pmic_probe,
 	.id_table = max77693_pmic_id,
diff --git a/drivers/regulator/max77802-regulator.c b/drivers/regulator/max77802-regulator.c
index befe5f319819..69eb6abd2551 100644
--- a/drivers/regulator/max77802-regulator.c
+++ b/drivers/regulator/max77802-regulator.c
@@ -554,6 +554,7 @@ MODULE_DEVICE_TABLE(platform, max77802_pmic_id);
 static struct platform_driver max77802_pmic_driver = {
 	.driver = {
 		.name = "max77802-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = max77802_pmic_probe,
 	.id_table = max77802_pmic_id,
diff --git a/drivers/regulator/max77826-regulator.c b/drivers/regulator/max77826-regulator.c
index f9e2e884ff54..ea5d4b18b464 100644
--- a/drivers/regulator/max77826-regulator.c
+++ b/drivers/regulator/max77826-regulator.c
@@ -289,6 +289,7 @@ MODULE_DEVICE_TABLE(i2c, max77826_id);
 static struct i2c_driver max77826_regulator_driver = {
 	.driver = {
 		.name = "max77826",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(max77826_of_match),
 	},
 	.probe_new = max77826_i2c_probe,
diff --git a/drivers/regulator/max8649.c b/drivers/regulator/max8649.c
index aed5443d88e1..a517fb4e3669 100644
--- a/drivers/regulator/max8649.c
+++ b/drivers/regulator/max8649.c
@@ -249,6 +249,7 @@ static struct i2c_driver max8649_driver = {
 	.probe_new	= max8649_regulator_probe,
 	.driver		= {
 		.name	= "max8649",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.id_table	= max8649_id,
 };
diff --git a/drivers/regulator/max8660.c b/drivers/regulator/max8660.c
index 711623be8eb5..d6b89f07ae9e 100644
--- a/drivers/regulator/max8660.c
+++ b/drivers/regulator/max8660.c
@@ -506,6 +506,7 @@ static struct i2c_driver max8660_driver = {
 	.probe_new = max8660_probe,
 	.driver		= {
 		.name	= "max8660",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.id_table	= max8660_id,
 };
diff --git a/drivers/regulator/max8893.c b/drivers/regulator/max8893.c
index 1519bf760da7..10ffd77828b7 100644
--- a/drivers/regulator/max8893.c
+++ b/drivers/regulator/max8893.c
@@ -171,6 +171,7 @@ static struct i2c_driver max8893_driver = {
 	.probe_new	= max8893_probe_new,
 	.driver		= {
 		.name	= "max8893",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(max8893_dt_match),
 	},
 	.id_table	= max8893_ids,
diff --git a/drivers/regulator/max8907-regulator.c b/drivers/regulator/max8907-regulator.c
index 1a6fd68f3fb1..e59aa7a3ee54 100644
--- a/drivers/regulator/max8907-regulator.c
+++ b/drivers/regulator/max8907-regulator.c
@@ -372,6 +372,7 @@ static int max8907_regulator_probe(struct platform_device *pdev)
 static struct platform_driver max8907_regulator_driver = {
 	.driver = {
 		   .name = "max8907-regulator",
+		   .probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		   },
 	.probe = max8907_regulator_probe,
 };
diff --git a/drivers/regulator/max8925-regulator.c b/drivers/regulator/max8925-regulator.c
index d953b6b0db77..c1532db0a4ee 100644
--- a/drivers/regulator/max8925-regulator.c
+++ b/drivers/regulator/max8925-regulator.c
@@ -263,6 +263,7 @@ static int max8925_regulator_probe(struct platform_device *pdev)
 static struct platform_driver max8925_regulator_driver = {
 	.driver		= {
 		.name	= "max8925-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe		= max8925_regulator_probe,
 };
diff --git a/drivers/regulator/max8952.c b/drivers/regulator/max8952.c
index 360a33ecc093..8ad8fe7fd263 100644
--- a/drivers/regulator/max8952.c
+++ b/drivers/regulator/max8952.c
@@ -316,6 +316,7 @@ static struct i2c_driver max8952_pmic_driver = {
 	.probe_new	= max8952_pmic_probe,
 	.driver		= {
 		.name	= "max8952",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(max8952_dt_match),
 	},
 	.id_table	= max8952_ids,
diff --git a/drivers/regulator/max8973-regulator.c b/drivers/regulator/max8973-regulator.c
index 7e00a45db26a..a991a884a31b 100644
--- a/drivers/regulator/max8973-regulator.c
+++ b/drivers/regulator/max8973-regulator.c
@@ -436,7 +436,7 @@ static int max8973_init_dcdc(struct max8973_chip *max,
 
 static int max8973_thermal_read_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct max8973_chip *mchip = tz->devdata;
+	struct max8973_chip *mchip = thermal_zone_device_priv(tz);
 	unsigned int val;
 	int ret;
 
@@ -804,6 +804,7 @@ MODULE_DEVICE_TABLE(i2c, max8973_id);
 static struct i2c_driver max8973_i2c_driver = {
 	.driver = {
 		.name = "max8973",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_max8973_match_tbl,
 	},
 	.probe_new = max8973_probe,
diff --git a/drivers/regulator/max8997-regulator.c b/drivers/regulator/max8997-regulator.c
index ba47a5e2fbcb..0b38eaa73597 100644
--- a/drivers/regulator/max8997-regulator.c
+++ b/drivers/regulator/max8997-regulator.c
@@ -943,14 +943,9 @@ static int max8997_pmic_dt_parse_pdata(struct platform_device *pdev,
 	}
 	of_node_put(regulators_np);
 
-	if (of_get_property(pmic_np, "max8997,pmic-buck1-uses-gpio-dvs", NULL))
-		pdata->buck1_gpiodvs = true;
-
-	if (of_get_property(pmic_np, "max8997,pmic-buck2-uses-gpio-dvs", NULL))
-		pdata->buck2_gpiodvs = true;
-
-	if (of_get_property(pmic_np, "max8997,pmic-buck5-uses-gpio-dvs", NULL))
-		pdata->buck5_gpiodvs = true;
+	pdata->buck1_gpiodvs = of_property_read_bool(pmic_np, "max8997,pmic-buck1-uses-gpio-dvs");
+	pdata->buck2_gpiodvs = of_property_read_bool(pmic_np, "max8997,pmic-buck2-uses-gpio-dvs");
+	pdata->buck5_gpiodvs = of_property_read_bool(pmic_np, "max8997,pmic-buck5-uses-gpio-dvs");
 
 	if (pdata->buck1_gpiodvs || pdata->buck2_gpiodvs ||
 						pdata->buck5_gpiodvs) {
@@ -1202,6 +1197,7 @@ MODULE_DEVICE_TABLE(platform, max8997_pmic_id);
 static struct platform_driver max8997_pmic_driver = {
 	.driver = {
 		.name = "max8997-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = max8997_pmic_probe,
 	.id_table = max8997_pmic_id,
diff --git a/drivers/regulator/max8998.c b/drivers/regulator/max8998.c
index ac69bdd398cb..fadb4717384a 100644
--- a/drivers/regulator/max8998.c
+++ b/drivers/regulator/max8998.c
@@ -618,8 +618,7 @@ static int max8998_pmic_dt_parse_pdata(struct max8998_dev *iodev,
 	if (ret)
 		return -EINVAL;
 
-	if (of_find_property(pmic_np, "max8998,pmic-buck-voltage-lock", NULL))
-		pdata->buck_voltage_lock = true;
+	pdata->buck_voltage_lock = of_property_read_bool(pmic_np, "max8998,pmic-buck-voltage-lock");
 
 	ret = of_property_read_u32(pmic_np,
 					"max8998,pmic-buck1-default-dvs-idx",
@@ -804,6 +803,7 @@ MODULE_DEVICE_TABLE(platform, max8998_pmic_id);
 static struct platform_driver max8998_pmic_driver = {
 	.driver = {
 		.name = "max8998-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = max8998_pmic_probe,
 	.id_table = max8998_pmic_id,
diff --git a/drivers/regulator/mc13783-regulator.c b/drivers/regulator/mc13783-regulator.c
index ab558b26cd7c..fb3aa1cec1f2 100644
--- a/drivers/regulator/mc13783-regulator.c
+++ b/drivers/regulator/mc13783-regulator.c
@@ -455,6 +455,7 @@ static int mc13783_regulator_probe(struct platform_device *pdev)
 static struct platform_driver mc13783_regulator_driver = {
 	.driver	= {
 		.name	= "mc13783-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe		= mc13783_regulator_probe,
 };
diff --git a/drivers/regulator/mc13892-regulator.c b/drivers/regulator/mc13892-regulator.c
index 5221f7a9df91..b29cf6ba6f12 100644
--- a/drivers/regulator/mc13892-regulator.c
+++ b/drivers/regulator/mc13892-regulator.c
@@ -629,6 +629,7 @@ err_unlock:
 static struct platform_driver mc13892_regulator_driver = {
 	.driver	= {
 		.name	= "mc13892-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe	= mc13892_regulator_probe,
 };
diff --git a/drivers/regulator/mcp16502.c b/drivers/regulator/mcp16502.c
index abee1b09008d..3a6d79556942 100644
--- a/drivers/regulator/mcp16502.c
+++ b/drivers/regulator/mcp16502.c
@@ -587,6 +587,7 @@ static struct i2c_driver mcp16502_drv = {
 	.probe_new	= mcp16502_probe,
 	.driver		= {
 		.name	= "mcp16502-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= of_match_ptr(mcp16502_ids),
 #ifdef CONFIG_PM
 		.pm = &mcp16502_pm_ops,
diff --git a/drivers/regulator/mp5416.c b/drivers/regulator/mp5416.c
index 82892d71c2c9..91e9019430b8 100644
--- a/drivers/regulator/mp5416.c
+++ b/drivers/regulator/mp5416.c
@@ -237,6 +237,7 @@ MODULE_DEVICE_TABLE(i2c, mp5416_id);
 static struct i2c_driver mp5416_regulator_driver = {
 	.driver = {
 		.name = "mp5416",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(mp5416_of_match),
 	},
 	.probe_new = mp5416_i2c_probe,
diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c
index f2300714d5a9..b968a682f38a 100644
--- a/drivers/regulator/mp8859.c
+++ b/drivers/regulator/mp8859.c
@@ -129,7 +129,7 @@ static int mp8859_i2c_probe(struct i2c_client *i2c)
 	return 0;
 }
 
-static const struct of_device_id mp8859_dt_id[] = {
+static const struct of_device_id mp8859_dt_id[] __maybe_unused = {
 	{.compatible =  "mps,mp8859"},
 	{},
 };
@@ -144,6 +144,7 @@ MODULE_DEVICE_TABLE(i2c, mp8859_i2c_id);
 static struct i2c_driver mp8859_regulator_driver = {
 	.driver = {
 		.name = "mp8859",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(mp8859_dt_id),
 	},
 	.probe_new = mp8859_i2c_probe,
diff --git a/drivers/regulator/mp886x.c b/drivers/regulator/mp886x.c
index 6a0c94c15027..250c27e462f1 100644
--- a/drivers/regulator/mp886x.c
+++ b/drivers/regulator/mp886x.c
@@ -362,6 +362,7 @@ MODULE_DEVICE_TABLE(i2c, mp886x_id);
 static struct i2c_driver mp886x_regulator_driver = {
 	.driver = {
 		.name = "mp886x-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = mp886x_dt_ids,
 	},
 	.probe_new = mp886x_i2c_probe,
diff --git a/drivers/regulator/mpq7920.c b/drivers/regulator/mpq7920.c
index 54c862edf571..544d41b88514 100644
--- a/drivers/regulator/mpq7920.c
+++ b/drivers/regulator/mpq7920.c
@@ -318,6 +318,7 @@ MODULE_DEVICE_TABLE(i2c, mpq7920_id);
 static struct i2c_driver mpq7920_regulator_driver = {
 	.driver = {
 		.name = "mpq7920",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(mpq7920_of_match),
 	},
 	.probe_new = mpq7920_i2c_probe,
diff --git a/drivers/regulator/mt6311-regulator.c b/drivers/regulator/mt6311-regulator.c
index 69e6af3cd505..a9f0c9f725d4 100644
--- a/drivers/regulator/mt6311-regulator.c
+++ b/drivers/regulator/mt6311-regulator.c
@@ -151,6 +151,7 @@ MODULE_DEVICE_TABLE(of, mt6311_dt_ids);
 static struct i2c_driver mt6311_regulator_driver = {
 	.driver = {
 		.name = "mt6311",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(mt6311_dt_ids),
 	},
 	.probe_new = mt6311_i2c_probe,
diff --git a/drivers/regulator/mt6315-regulator.c b/drivers/regulator/mt6315-regulator.c
index 284c229e1aa4..8047081ea2f7 100644
--- a/drivers/regulator/mt6315-regulator.c
+++ b/drivers/regulator/mt6315-regulator.c
@@ -287,6 +287,7 @@ static void mt6315_regulator_shutdown(struct spmi_device *pdev)
 static struct spmi_driver mt6315_regulator_driver = {
 	.driver		= {
 		.name	= "mt6315-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = mt6315_of_match,
 	},
 	.probe = mt6315_regulator_probe,
diff --git a/drivers/regulator/mt6323-regulator.c b/drivers/regulator/mt6323-regulator.c
index ff9016170db3..b43da848a06e 100644
--- a/drivers/regulator/mt6323-regulator.c
+++ b/drivers/regulator/mt6323-regulator.c
@@ -409,6 +409,7 @@ MODULE_DEVICE_TABLE(platform, mt6323_platform_ids);
 static struct platform_driver mt6323_regulator_driver = {
 	.driver = {
 		.name = "mt6323-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = mt6323_regulator_probe,
 	.id_table = mt6323_platform_ids,
diff --git a/drivers/regulator/mt6331-regulator.c b/drivers/regulator/mt6331-regulator.c
index 56be9a3a84ab..0059f88c6fd7 100644
--- a/drivers/regulator/mt6331-regulator.c
+++ b/drivers/regulator/mt6331-regulator.c
@@ -495,6 +495,7 @@ MODULE_DEVICE_TABLE(platform, mt6331_platform_ids);
 static struct platform_driver mt6331_regulator_driver = {
 	.driver = {
 		.name = "mt6331-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = mt6331_regulator_probe,
 	.id_table = mt6331_platform_ids,
diff --git a/drivers/regulator/mt6332-regulator.c b/drivers/regulator/mt6332-regulator.c
index 77a27d8127a3..8d8331a2aca5 100644
--- a/drivers/regulator/mt6332-regulator.c
+++ b/drivers/regulator/mt6332-regulator.c
@@ -410,6 +410,7 @@ MODULE_DEVICE_TABLE(platform, mt6332_platform_ids);
 static struct platform_driver mt6332_regulator_driver = {
 	.driver = {
 		.name = "mt6332-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = mt6332_regulator_probe,
 	.id_table = mt6332_platform_ids,
diff --git a/drivers/regulator/mt6357-regulator.c b/drivers/regulator/mt6357-regulator.c
index b2352b96aed2..c0439a4e0b50 100644
--- a/drivers/regulator/mt6357-regulator.c
+++ b/drivers/regulator/mt6357-regulator.c
@@ -439,6 +439,7 @@ MODULE_DEVICE_TABLE(platform, mt6357_platform_ids);
 static struct platform_driver mt6357_regulator_driver = {
 	.driver = {
 		.name = "mt6357-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = mt6357_regulator_probe,
 	.id_table = mt6357_platform_ids,
diff --git a/drivers/regulator/mt6358-regulator.c b/drivers/regulator/mt6358-regulator.c
index 8a5ce990f1bf..c9e16bd092f6 100644
--- a/drivers/regulator/mt6358-regulator.c
+++ b/drivers/regulator/mt6358-regulator.c
@@ -733,6 +733,7 @@ MODULE_DEVICE_TABLE(platform, mt6358_platform_ids);
 static struct platform_driver mt6358_regulator_driver = {
 	.driver = {
 		.name = "mt6358-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = mt6358_regulator_probe,
 	.id_table = mt6358_platform_ids,
diff --git a/drivers/regulator/mt6359-regulator.c b/drivers/regulator/mt6359-regulator.c
index de3b0462832c..1849566784ab 100644
--- a/drivers/regulator/mt6359-regulator.c
+++ b/drivers/regulator/mt6359-regulator.c
@@ -982,6 +982,7 @@ MODULE_DEVICE_TABLE(platform, mt6359_platform_ids);
 static struct platform_driver mt6359_regulator_driver = {
 	.driver = {
 		.name = "mt6359-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = mt6359_regulator_probe,
 	.id_table = mt6359_platform_ids,
diff --git a/drivers/regulator/mt6360-regulator.c b/drivers/regulator/mt6360-regulator.c
index 4d34be94d166..ad6587a378d0 100644
--- a/drivers/regulator/mt6360-regulator.c
+++ b/drivers/regulator/mt6360-regulator.c
@@ -446,6 +446,7 @@ MODULE_DEVICE_TABLE(platform, mt6360_regulator_id_table);
 static struct platform_driver mt6360_regulator_driver = {
 	.driver = {
 		.name = "mt6360-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = mt6360_regulator_probe,
 	.id_table = mt6360_regulator_id_table,
diff --git a/drivers/regulator/mt6370-regulator.c b/drivers/regulator/mt6370-regulator.c
index e73f5a46cb9a..27cb32b726e0 100644
--- a/drivers/regulator/mt6370-regulator.c
+++ b/drivers/regulator/mt6370-regulator.c
@@ -379,6 +379,7 @@ MODULE_DEVICE_TABLE(platform, mt6370_devid_table);
 static struct platform_driver mt6370_regulator_driver = {
 	.driver = {
 		.name = "mt6370-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.id_table = mt6370_devid_table,
 	.probe = mt6370_regulator_probe,
diff --git a/drivers/regulator/mt6380-regulator.c b/drivers/regulator/mt6380-regulator.c
index 43234296df36..83e50df7f7c3 100644
--- a/drivers/regulator/mt6380-regulator.c
+++ b/drivers/regulator/mt6380-regulator.c
@@ -328,6 +328,7 @@ MODULE_DEVICE_TABLE(of, mt6380_of_match);
 static struct platform_driver mt6380_regulator_driver = {
 	.driver = {
 		.name = "mt6380-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(mt6380_of_match),
 	},
 	.probe = mt6380_regulator_probe,
diff --git a/drivers/regulator/mt6397-regulator.c b/drivers/regulator/mt6397-regulator.c
index b9bf7ade1f8a..92a2d92f84f9 100644
--- a/drivers/regulator/mt6397-regulator.c
+++ b/drivers/regulator/mt6397-regulator.c
@@ -397,7 +397,7 @@ static const struct platform_device_id mt6397_platform_ids[] = {
 };
 MODULE_DEVICE_TABLE(platform, mt6397_platform_ids);
 
-static const struct of_device_id mt6397_of_match[] = {
+static const struct of_device_id mt6397_of_match[] __maybe_unused = {
 	{ .compatible = "mediatek,mt6397-regulator", },
 	{ /* sentinel */ },
 };
@@ -406,6 +406,7 @@ MODULE_DEVICE_TABLE(of, mt6397_of_match);
 static struct platform_driver mt6397_regulator_driver = {
 	.driver = {
 		.name = "mt6397-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(mt6397_of_match),
 	},
 	.probe = mt6397_regulator_probe,
diff --git a/drivers/regulator/mtk-dvfsrc-regulator.c b/drivers/regulator/mtk-dvfsrc-regulator.c
index 234af3a66c77..efca67207a5a 100644
--- a/drivers/regulator/mtk-dvfsrc-regulator.c
+++ b/drivers/regulator/mtk-dvfsrc-regulator.c
@@ -194,6 +194,7 @@ static int dvfsrc_vcore_regulator_probe(struct platform_device *pdev)
 static struct platform_driver mtk_dvfsrc_regulator_driver = {
 	.driver = {
 		.name  = "mtk-dvfsrc-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = dvfsrc_vcore_regulator_probe,
 };
diff --git a/drivers/regulator/palmas-regulator.c b/drivers/regulator/palmas-regulator.c
index 337dd614695e..076966366b60 100644
--- a/drivers/regulator/palmas-regulator.c
+++ b/drivers/regulator/palmas-regulator.c
@@ -1666,6 +1666,7 @@ static int palmas_regulators_probe(struct platform_device *pdev)
 static struct platform_driver palmas_driver = {
 	.driver = {
 		.name = "palmas-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_palmas_match_tbl,
 	},
 	.probe = palmas_regulators_probe,
diff --git a/drivers/regulator/pbias-regulator.c b/drivers/regulator/pbias-regulator.c
index 4eccf12f39de..0c9873e9abdc 100644
--- a/drivers/regulator/pbias-regulator.c
+++ b/drivers/regulator/pbias-regulator.c
@@ -231,6 +231,7 @@ static struct platform_driver pbias_regulator_driver = {
 	.probe		= pbias_regulator_probe,
 	.driver		= {
 		.name		= "pbias-regulator",
+		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(pbias_of_match),
 	},
 };
diff --git a/drivers/regulator/pca9450-regulator.c b/drivers/regulator/pca9450-regulator.c
index c6351fac9f4d..87a746dcb516 100644
--- a/drivers/regulator/pca9450-regulator.c
+++ b/drivers/regulator/pca9450-regulator.c
@@ -872,6 +872,7 @@ MODULE_DEVICE_TABLE(of, pca9450_of_match);
 static struct i2c_driver pca9450_i2c_driver = {
 	.driver = {
 		.name = "nxp-pca9450",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = pca9450_of_match,
 	},
 	.probe_new = pca9450_i2c_probe,
diff --git a/drivers/regulator/pcap-regulator.c b/drivers/regulator/pcap-regulator.c
index 0345f38f6f78..319a88412154 100644
--- a/drivers/regulator/pcap-regulator.c
+++ b/drivers/regulator/pcap-regulator.c
@@ -251,6 +251,7 @@ static int pcap_regulator_probe(struct platform_device *pdev)
 static struct platform_driver pcap_regulator_driver = {
 	.driver = {
 		.name	= "pcap-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe	= pcap_regulator_probe,
 };
diff --git a/drivers/regulator/pcf50633-regulator.c b/drivers/regulator/pcf50633-regulator.c
index f40e3bb303d6..9f08a62c800e 100644
--- a/drivers/regulator/pcf50633-regulator.c
+++ b/drivers/regulator/pcf50633-regulator.c
@@ -101,6 +101,7 @@ static int pcf50633_regulator_probe(struct platform_device *pdev)
 static struct platform_driver pcf50633_regulator_driver = {
 	.driver = {
 		.name = "pcf50633-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = pcf50633_regulator_probe,
 };
diff --git a/drivers/regulator/pf8x00-regulator.c b/drivers/regulator/pf8x00-regulator.c
index 5d319fb81288..99a15c3be396 100644
--- a/drivers/regulator/pf8x00-regulator.c
+++ b/drivers/regulator/pf8x00-regulator.c
@@ -607,6 +607,7 @@ static struct i2c_driver pf8x00_regulator_driver = {
 	.id_table = pf8x00_i2c_id,
 	.driver = {
 		.name = "pf8x00",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = pf8x00_dt_ids,
 	},
 	.probe_new = pf8x00_i2c_probe,
diff --git a/drivers/regulator/pfuze100-regulator.c b/drivers/regulator/pfuze100-regulator.c
index 9ab604289b5c..a9fcf6a41494 100644
--- a/drivers/regulator/pfuze100-regulator.c
+++ b/drivers/regulator/pfuze100-regulator.c
@@ -845,6 +845,7 @@ static int pfuze100_regulator_probe(struct i2c_client *client)
 static struct i2c_driver pfuze_driver = {
 	.driver = {
 		.name = "pfuze100-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = pfuze_dt_ids,
 	},
 	.probe_new = pfuze100_regulator_probe,
diff --git a/drivers/regulator/pv88060-regulator.c b/drivers/regulator/pv88060-regulator.c
index 48238846f45c..f170e0dd1819 100644
--- a/drivers/regulator/pv88060-regulator.c
+++ b/drivers/regulator/pv88060-regulator.c
@@ -376,6 +376,7 @@ MODULE_DEVICE_TABLE(of, pv88060_dt_ids);
 static struct i2c_driver pv88060_regulator_driver = {
 	.driver = {
 		.name = "pv88060",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(pv88060_dt_ids),
 	},
 	.probe_new = pv88060_i2c_probe,
diff --git a/drivers/regulator/pv88080-regulator.c b/drivers/regulator/pv88080-regulator.c
index 15a67c05f519..133b89d5215c 100644
--- a/drivers/regulator/pv88080-regulator.c
+++ b/drivers/regulator/pv88080-regulator.c
@@ -557,6 +557,7 @@ MODULE_DEVICE_TABLE(i2c, pv88080_i2c_id);
 static struct i2c_driver pv88080_regulator_driver = {
 	.driver = {
 		.name = "pv88080",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(pv88080_dt_ids),
 	},
 	.probe_new = pv88080_i2c_probe,
diff --git a/drivers/regulator/pv88090-regulator.c b/drivers/regulator/pv88090-regulator.c
index a80176bdf8ec..1bc33bc10992 100644
--- a/drivers/regulator/pv88090-regulator.c
+++ b/drivers/regulator/pv88090-regulator.c
@@ -397,6 +397,7 @@ MODULE_DEVICE_TABLE(of, pv88090_dt_ids);
 static struct i2c_driver pv88090_regulator_driver = {
 	.driver = {
 		.name = "pv88090",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(pv88090_dt_ids),
 	},
 	.probe_new = pv88090_i2c_probe,
diff --git a/drivers/regulator/pwm-regulator.c b/drivers/regulator/pwm-regulator.c
index b9eeaff1c661..b64d99695b84 100644
--- a/drivers/regulator/pwm-regulator.c
+++ b/drivers/regulator/pwm-regulator.c
@@ -334,7 +334,7 @@ static int pwm_regulator_probe(struct platform_device *pdev)
 
 	memcpy(&drvdata->desc, &pwm_regulator_desc, sizeof(drvdata->desc));
 
-	if (of_find_property(np, "voltage-table", NULL))
+	if (of_property_present(np, "voltage-table"))
 		ret = pwm_regulator_init_table(pdev, drvdata);
 	else
 		ret = pwm_regulator_init_continuous(pdev, drvdata);
@@ -393,6 +393,7 @@ MODULE_DEVICE_TABLE(of, pwm_of_match);
 static struct platform_driver pwm_regulator_driver = {
 	.driver = {
 		.name		= "pwm-regulator",
+		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(pwm_of_match),
 	},
 	.probe = pwm_regulator_probe,
diff --git a/drivers/regulator/qcom-labibb-regulator.c b/drivers/regulator/qcom-labibb-regulator.c
index bcf7140f3bc9..a8698ca61143 100644
--- a/drivers/regulator/qcom-labibb-regulator.c
+++ b/drivers/regulator/qcom-labibb-regulator.c
@@ -894,6 +894,7 @@ static int qcom_labibb_regulator_probe(struct platform_device *pdev)
 static struct platform_driver qcom_labibb_regulator_driver = {
 	.driver	= {
 		.name = "qcom-lab-ibb-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= qcom_labibb_match,
 	},
 	.probe = qcom_labibb_regulator_probe,
diff --git a/drivers/regulator/qcom-rpmh-regulator.c b/drivers/regulator/qcom-rpmh-regulator.c
index ae6021390143..b0a58c62b1e2 100644
--- a/drivers/regulator/qcom-rpmh-regulator.c
+++ b/drivers/regulator/qcom-rpmh-regulator.c
@@ -694,6 +694,16 @@ static const struct rpmh_vreg_hw_data pmic5_pldo_lv = {
 	.of_map_mode = rpmh_regulator_pmic4_ldo_of_map_mode,
 };
 
+static const struct rpmh_vreg_hw_data pmic5_pldo515_mv = {
+	.regulator_type = VRM,
+	.ops = &rpmh_regulator_vrm_drms_ops,
+	.voltage_range = REGULATOR_LINEAR_RANGE(1800000, 0, 187, 8000),
+	.n_voltages = 188,
+	.hpm_min_load_uA = 10000,
+	.pmic_mode_map = pmic_mode_map_pmic5_ldo,
+	.of_map_mode = rpmh_regulator_pmic4_ldo_of_map_mode,
+};
+
 static const struct rpmh_vreg_hw_data pmic5_nldo = {
 	.regulator_type = VRM,
 	.ops = &rpmh_regulator_vrm_drms_ops,
@@ -704,6 +714,16 @@ static const struct rpmh_vreg_hw_data pmic5_nldo = {
 	.of_map_mode = rpmh_regulator_pmic4_ldo_of_map_mode,
 };
 
+static const struct rpmh_vreg_hw_data pmic5_nldo515 = {
+	.regulator_type = VRM,
+	.ops = &rpmh_regulator_vrm_drms_ops,
+	.voltage_range = REGULATOR_LINEAR_RANGE(320000, 0, 210, 8000),
+	.n_voltages = 211,
+	.hpm_min_load_uA = 30000,
+	.pmic_mode_map = pmic_mode_map_pmic5_ldo,
+	.of_map_mode = rpmh_regulator_pmic4_ldo_of_map_mode,
+};
+
 static const struct rpmh_vreg_hw_data pmic5_hfsmps510 = {
 	.regulator_type = VRM,
 	.ops = &rpmh_regulator_vrm_ops,
@@ -749,6 +769,15 @@ static const struct rpmh_vreg_hw_data pmic5_ftsmps525_mv = {
 	.of_map_mode = rpmh_regulator_pmic4_smps_of_map_mode,
 };
 
+static const struct rpmh_vreg_hw_data pmic5_ftsmps527 = {
+	.regulator_type = VRM,
+	.ops = &rpmh_regulator_vrm_ops,
+	.voltage_range = REGULATOR_LINEAR_RANGE(320000, 0, 215, 8000),
+	.n_voltages = 215,
+	.pmic_mode_map = pmic_mode_map_pmic5_smps,
+	.of_map_mode = rpmh_regulator_pmic4_smps_of_map_mode,
+};
+
 static const struct rpmh_vreg_hw_data pmic5_hfsmps515 = {
 	.regulator_type = VRM,
 	.ops = &rpmh_regulator_vrm_ops,
@@ -937,6 +966,28 @@ static const struct rpmh_vreg_init_data pmm8155au_vreg_data[] = {
 	{}
 };
 
+static const struct rpmh_vreg_init_data pmm8654au_vreg_data[] = {
+	RPMH_VREG("smps1",  "smp%s1",  &pmic5_ftsmps527,  "vdd-s1"),
+	RPMH_VREG("smps2",  "smp%s2",  &pmic5_ftsmps527,  "vdd-s2"),
+	RPMH_VREG("smps3",  "smp%s3",  &pmic5_ftsmps527,  "vdd-s3"),
+	RPMH_VREG("smps4",  "smp%s4",  &pmic5_ftsmps527,  "vdd-s4"),
+	RPMH_VREG("smps5",  "smp%s5",  &pmic5_ftsmps527,  "vdd-s5"),
+	RPMH_VREG("smps6",  "smp%s6",  &pmic5_ftsmps527,  "vdd-s6"),
+	RPMH_VREG("smps7",  "smp%s7",  &pmic5_ftsmps527,  "vdd-s7"),
+	RPMH_VREG("smps8",  "smp%s8",  &pmic5_ftsmps527,  "vdd-s8"),
+	RPMH_VREG("smps9",  "smp%s9",  &pmic5_ftsmps527,  "vdd-s9"),
+	RPMH_VREG("ldo1",   "ldo%s1",  &pmic5_nldo515,    "vdd-s9"),
+	RPMH_VREG("ldo2",   "ldo%s2",  &pmic5_nldo515,    "vdd-l2-l3"),
+	RPMH_VREG("ldo3",   "ldo%s3",  &pmic5_nldo515,    "vdd-l2-l3"),
+	RPMH_VREG("ldo4",   "ldo%s4",  &pmic5_nldo515,    "vdd-s9"),
+	RPMH_VREG("ldo5",   "ldo%s5",  &pmic5_nldo515,    "vdd-s9"),
+	RPMH_VREG("ldo6",   "ldo%s6",  &pmic5_nldo515,    "vdd-l6-l7"),
+	RPMH_VREG("ldo7",   "ldo%s7",  &pmic5_nldo515,    "vdd-l6-l7"),
+	RPMH_VREG("ldo8",   "ldo%s8",  &pmic5_pldo515_mv, "vdd-l8-l9"),
+	RPMH_VREG("ldo9",   "ldo%s9",  &pmic5_pldo,       "vdd-l8-l9"),
+	{}
+};
+
 static const struct rpmh_vreg_init_data pm8350_vreg_data[] = {
 	RPMH_VREG("smps1",  "smp%s1",  &pmic5_ftsmps510, "vdd-s1"),
 	RPMH_VREG("smps2",  "smp%s2",  &pmic5_ftsmps510, "vdd-s2"),
@@ -1432,6 +1483,10 @@ static const struct of_device_id __maybe_unused rpmh_regulator_match_table[] = {
 		.data = pmm8155au_vreg_data,
 	},
 	{
+		.compatible = "qcom,pmm8654au-rpmh-regulators",
+		.data = pmm8654au_vreg_data,
+	},
+	{
 		.compatible = "qcom,pmx55-rpmh-regulators",
 		.data = pmx55_vreg_data,
 	},
@@ -1462,6 +1517,7 @@ MODULE_DEVICE_TABLE(of, rpmh_regulator_match_table);
 static struct platform_driver rpmh_regulator_driver = {
 	.driver = {
 		.name = "qcom-rpmh-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= of_match_ptr(rpmh_regulator_match_table),
 	},
 	.probe = rpmh_regulator_probe,
diff --git a/drivers/regulator/qcom_rpm-regulator.c b/drivers/regulator/qcom_rpm-regulator.c
index 3c41b71a1f52..f95bc9208c13 100644
--- a/drivers/regulator/qcom_rpm-regulator.c
+++ b/drivers/regulator/qcom_rpm-regulator.c
@@ -991,6 +991,7 @@ static struct platform_driver rpm_reg_driver = {
 	.probe          = rpm_reg_probe,
 	.driver  = {
 		.name  = "qcom_rpm_reg",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(rpm_of_match),
 	},
 };
diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c
index 9f2b58458841..18189f35db68 100644
--- a/drivers/regulator/qcom_smd-regulator.c
+++ b/drivers/regulator/qcom_smd-regulator.c
@@ -749,7 +749,7 @@ static const struct regulator_desc pms405_pldo600 = {
 	.ops = &rpm_smps_ldo_ops,
 };
 
-static const struct regulator_desc mp5496_smpa2 = {
+static const struct regulator_desc mp5496_smps = {
 	.linear_ranges = (struct linear_range[]) {
 		REGULATOR_LINEAR_RANGE(600000, 0, 127, 12500),
 	},
@@ -794,7 +794,8 @@ struct rpm_regulator_data {
 };
 
 static const struct rpm_regulator_data rpm_mp5496_regulators[] = {
-	{ "s2", QCOM_SMD_RPM_SMPA, 2, &mp5496_smpa2, "s2" },
+	{ "s1", QCOM_SMD_RPM_SMPA, 1, &mp5496_smps, "s1" },
+	{ "s2", QCOM_SMD_RPM_SMPA, 2, &mp5496_smps, "s2" },
 	{ "l2", QCOM_SMD_RPM_LDOA, 2, &mp5496_ldoa2, "l2" },
 	{}
 };
@@ -1440,6 +1441,7 @@ static struct platform_driver rpm_reg_driver = {
 	.probe = rpm_reg_probe,
 	.driver = {
 		.name  = "qcom_rpm_smd_regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = rpm_of_match,
 	},
 };
diff --git a/drivers/regulator/qcom_spmi-regulator.c b/drivers/regulator/qcom_spmi-regulator.c
index 3e312729741e..c95f6e9c7ab5 100644
--- a/drivers/regulator/qcom_spmi-regulator.c
+++ b/drivers/regulator/qcom_spmi-regulator.c
@@ -2484,6 +2484,7 @@ static int qcom_spmi_regulator_probe(struct platform_device *pdev)
 static struct platform_driver qcom_spmi_regulator_driver = {
 	.driver		= {
 		.name	= "qcom-spmi-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = qcom_spmi_regulator_match,
 	},
 	.probe		= qcom_spmi_regulator_probe,
diff --git a/drivers/regulator/qcom_usb_vbus-regulator.c b/drivers/regulator/qcom_usb_vbus-regulator.c
index 2e627c2b6c51..57ec613f4a0a 100644
--- a/drivers/regulator/qcom_usb_vbus-regulator.c
+++ b/drivers/regulator/qcom_usb_vbus-regulator.c
@@ -100,6 +100,7 @@ MODULE_DEVICE_TABLE(of, qcom_usb_vbus_regulator_match);
 static struct platform_driver qcom_usb_vbus_regulator_driver = {
 	.driver		= {
 		.name	= "qcom-usb-vbus-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = qcom_usb_vbus_regulator_match,
 	},
 	.probe		= qcom_usb_vbus_regulator_probe,
diff --git a/drivers/regulator/rc5t583-regulator.c b/drivers/regulator/rc5t583-regulator.c
index 62641b08b88a..a5afca73715d 100644
--- a/drivers/regulator/rc5t583-regulator.c
+++ b/drivers/regulator/rc5t583-regulator.c
@@ -149,6 +149,7 @@ skip_ext_pwr_config:
 static struct platform_driver rc5t583_regulator_driver = {
 	.driver	= {
 		.name	= "rc5t583-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe		= rc5t583_regulator_probe,
 };
diff --git a/drivers/regulator/rk808-regulator.c b/drivers/regulator/rk808-regulator.c
index fa9fc1aa1ae3..3637e81654a8 100644
--- a/drivers/regulator/rk808-regulator.c
+++ b/drivers/regulator/rk808-regulator.c
@@ -1354,7 +1354,8 @@ static int rk808_regulator_probe(struct platform_device *pdev)
 static struct platform_driver rk808_regulator_driver = {
 	.probe = rk808_regulator_probe,
 	.driver = {
-		.name = "rk808-regulator"
+		.name = "rk808-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/rn5t618-regulator.c b/drivers/regulator/rn5t618-regulator.c
index 5c12d57be040..91808e0735b3 100644
--- a/drivers/regulator/rn5t618-regulator.c
+++ b/drivers/regulator/rn5t618-regulator.c
@@ -143,6 +143,7 @@ static struct platform_driver rn5t618_regulator_driver = {
 	.probe = rn5t618_regulator_probe,
 	.driver = {
 		.name	= "rn5t618-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/rpi-panel-attiny-regulator.c b/drivers/regulator/rpi-panel-attiny-regulator.c
index 34514976475e..9afe961a87f1 100644
--- a/drivers/regulator/rpi-panel-attiny-regulator.c
+++ b/drivers/regulator/rpi-panel-attiny-regulator.c
@@ -396,6 +396,7 @@ MODULE_DEVICE_TABLE(of, attiny_dt_ids);
 static struct i2c_driver attiny_regulator_driver = {
 	.driver = {
 		.name = "rpi_touchscreen_attiny",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(attiny_dt_ids),
 	},
 	.probe_new = attiny_i2c_probe,
diff --git a/drivers/regulator/rt4801-regulator.c b/drivers/regulator/rt4801-regulator.c
index 563d79196fdd..be3dc981195c 100644
--- a/drivers/regulator/rt4801-regulator.c
+++ b/drivers/regulator/rt4801-regulator.c
@@ -239,6 +239,7 @@ MODULE_DEVICE_TABLE(of, rt4801_of_id);
 static struct i2c_driver rt4801_driver = {
 	.driver = {
 		.name = "rt4801",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(rt4801_of_id),
 	},
 	.probe_new = rt4801_probe,
diff --git a/drivers/regulator/rt4803.c b/drivers/regulator/rt4803.c
new file mode 100644
index 000000000000..c96fb026dc10
--- /dev/null
+++ b/drivers/regulator/rt4803.c
@@ -0,0 +1,216 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2023 Richtek Technology Corp.
+ *
+ * Author: ChiYuan Huang <cy_huang@richtek.com>
+ */
+
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/of_regulator.h>
+
+#define RT4803_AUTO_MODE	1
+#define RT4803_FPWM_MODE	2
+
+#define RT4803_REG_CONFIG	0x01
+#define RT4803_REG_VSELL	0x02
+#define RT4803_REG_VSELH	0x03
+#define RT4803_REG_ILIM		0x04
+#define RT4803_REG_STAT		0x05
+
+#define RT4803_MODE_MASK	GENMASK(1, 0)
+#define RT4803_VSEL_MASK	GENMASK(4, 0)
+#define RT4803_ILIM_MASK	GENMASK(3, 0)
+#define RT4803_TSD_MASK		BIT(7)
+#define RT4803_HOTDIE_MASK	BIT(6)
+#define RT4803_FAULT_MASK	BIT(1)
+#define RT4803_PGOOD_MASK	BIT(0)
+
+#define RT4803_VOUT_MINUV	2850000
+#define RT4803_VOUT_STEPUV	50000
+#define RT4803_VOUT_NUM		32
+
+static int rt4803_set_mode(struct regulator_dev *rdev, unsigned int mode)
+{
+	struct regmap *regmap = rdev_get_regmap(rdev);
+	unsigned int modeval;
+
+	switch (mode) {
+	case REGULATOR_MODE_NORMAL:
+		modeval = RT4803_AUTO_MODE;
+		break;
+	case REGULATOR_MODE_FAST:
+		modeval = RT4803_FPWM_MODE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	modeval <<= ffs(RT4803_MODE_MASK) - 1;
+
+	return regmap_update_bits(regmap, RT4803_REG_CONFIG, RT4803_MODE_MASK, modeval);
+}
+
+static unsigned int rt4803_get_mode(struct regulator_dev *rdev)
+{
+	struct regmap *regmap = rdev_get_regmap(rdev);
+	unsigned int modeval;
+	int ret;
+
+	ret = regmap_read(regmap, RT4803_REG_CONFIG, &modeval);
+	if (ret)
+		return REGULATOR_MODE_INVALID;
+
+	modeval >>= ffs(RT4803_MODE_MASK) - 1;
+
+	switch (modeval) {
+	case RT4803_AUTO_MODE:
+		return REGULATOR_MODE_NORMAL;
+	case RT4803_FPWM_MODE:
+		return REGULATOR_MODE_FAST;
+	default:
+		return REGULATOR_MODE_INVALID;
+	}
+}
+
+static int rt4803_get_error_flags(struct regulator_dev *rdev, unsigned int *flags)
+{
+	struct regmap *regmap = rdev_get_regmap(rdev);
+	unsigned int state, events = 0;
+	int ret;
+
+	ret = regmap_read(regmap, RT4803_REG_STAT, &state);
+	if (ret)
+		return ret;
+
+	if (state & RT4803_PGOOD_MASK)
+		goto out_error_flag;
+
+	if (state & RT4803_FAULT_MASK)
+		events |= REGULATOR_ERROR_FAIL;
+
+	if (state & RT4803_HOTDIE_MASK)
+		events |= REGULATOR_ERROR_OVER_TEMP_WARN;
+
+	if (state & RT4803_TSD_MASK)
+		events |= REGULATOR_ERROR_OVER_TEMP;
+
+out_error_flag:
+	*flags = events;
+	return 0;
+}
+
+static int rt4803_set_suspend_voltage(struct regulator_dev *rdev, int uV)
+{
+	struct regmap *regmap = rdev_get_regmap(rdev);
+	unsigned int reg, vsel;
+
+	if (rdev->desc->vsel_reg == RT4803_REG_VSELL)
+		reg = RT4803_REG_VSELH;
+	else
+		reg = RT4803_REG_VSELL;
+
+	vsel = (uV - rdev->desc->min_uV) / rdev->desc->uV_step;
+	vsel <<= ffs(RT4803_VSEL_MASK) - 1;
+
+	return regmap_update_bits(regmap, reg, RT4803_VSEL_MASK, vsel);
+}
+
+static const struct regulator_ops rt4803_regulator_ops = {
+	.list_voltage = regulator_list_voltage_linear,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_mode = rt4803_set_mode,
+	.get_mode = rt4803_get_mode,
+	.get_error_flags = rt4803_get_error_flags,
+	.set_suspend_voltage = rt4803_set_suspend_voltage,
+};
+
+static unsigned int rt4803_of_map_mode(unsigned int mode)
+{
+	switch (mode) {
+	case RT4803_AUTO_MODE:
+		return REGULATOR_MODE_NORMAL;
+	case RT4803_FPWM_MODE:
+		return REGULATOR_MODE_FAST;
+	default:
+		return REGULATOR_MODE_INVALID;
+	}
+}
+
+static const struct regmap_config rt4803_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = RT4803_REG_STAT,
+};
+
+static int rt4803_probe(struct i2c_client *i2c)
+{
+	struct device *dev = &i2c->dev;
+	struct regmap *regmap;
+	struct regulator_desc *desc;
+	struct regulator_config cfg = {};
+	struct regulator_dev *rdev;
+	bool vsel_act_high;
+	int ret;
+
+	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	regmap = devm_regmap_init_i2c(i2c, &rt4803_regmap_config);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Failed to init regmap\n");
+
+	/* Always configure the input current limit to max 5A at initial */
+	ret = regmap_update_bits(regmap, RT4803_REG_ILIM, RT4803_ILIM_MASK, 0xff);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to config ILIM to max\n");
+
+	vsel_act_high = device_property_read_bool(dev, "richtek,vsel-active-high");
+
+	desc->name = "rt4803-regulator";
+	desc->type = REGULATOR_VOLTAGE;
+	desc->owner = THIS_MODULE;
+	desc->ops = &rt4803_regulator_ops;
+	desc->min_uV = RT4803_VOUT_MINUV;
+	desc->uV_step = RT4803_VOUT_STEPUV;
+	desc->n_voltages = RT4803_VOUT_NUM;
+	desc->vsel_mask = RT4803_VSEL_MASK;
+	desc->of_map_mode = rt4803_of_map_mode;
+	if (vsel_act_high)
+		desc->vsel_reg = RT4803_REG_VSELH;
+	else
+		desc->vsel_reg = RT4803_REG_VSELL;
+
+	cfg.dev = dev;
+	cfg.of_node = dev_of_node(dev);
+	cfg.init_data = of_get_regulator_init_data(dev, dev_of_node(dev), desc);
+
+	rdev = devm_regulator_register(dev, desc, &cfg);
+	return PTR_ERR_OR_ZERO(rdev);
+}
+
+static const struct of_device_id rt4803_device_match_table[] = {
+	{ .compatible = "richtek,rt4803" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, rt4803_device_match_table);
+
+static struct i2c_driver rt4803_driver = {
+	.driver = {
+		.name = "rt4803",
+		.of_match_table = rt4803_device_match_table,
+	},
+	.probe = rt4803_probe,
+};
+module_i2c_driver(rt4803_driver);
+
+MODULE_DESCRIPTION("Richtek RT4803 voltage regulator driver");
+MODULE_AUTHOR("ChiYuan Huang <cy_huang@richtek.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/rt4831-regulator.c b/drivers/regulator/rt4831-regulator.c
index 2016062cd7ef..97e6f7e2a0ba 100644
--- a/drivers/regulator/rt4831-regulator.c
+++ b/drivers/regulator/rt4831-regulator.c
@@ -194,6 +194,7 @@ MODULE_DEVICE_TABLE(platform, rt4831_regulator_match);
 static struct platform_driver rt4831_regulator_driver = {
 	.driver = {
 		.name = "rt4831-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.id_table = rt4831_regulator_match,
 	.probe = rt4831_regulator_probe,
diff --git a/drivers/regulator/rt5033-regulator.c b/drivers/regulator/rt5033-regulator.c
index da4cf5a6acc2..2ba74f205543 100644
--- a/drivers/regulator/rt5033-regulator.c
+++ b/drivers/regulator/rt5033-regulator.c
@@ -124,6 +124,7 @@ MODULE_DEVICE_TABLE(platform, rt5033_regulator_id);
 static struct platform_driver rt5033_regulator_driver = {
 	.driver = {
 		.name = "rt5033-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe		= rt5033_regulator_probe,
 	.id_table	= rt5033_regulator_id,
diff --git a/drivers/regulator/rt5120-regulator.c b/drivers/regulator/rt5120-regulator.c
index 8173ede09414..a388ac70865f 100644
--- a/drivers/regulator/rt5120-regulator.c
+++ b/drivers/regulator/rt5120-regulator.c
@@ -409,6 +409,7 @@ MODULE_DEVICE_TABLE(platform, rt5120_regulator_dev_table);
 static struct platform_driver rt5120_regulator_driver = {
 	.driver = {
 		.name = "rt5120-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.id_table = rt5120_regulator_dev_table,
 	.probe = rt5120_regulator_probe,
diff --git a/drivers/regulator/rt5190a-regulator.c b/drivers/regulator/rt5190a-regulator.c
index 4a3397b32582..f6c12f87fb8d 100644
--- a/drivers/regulator/rt5190a-regulator.c
+++ b/drivers/regulator/rt5190a-regulator.c
@@ -505,6 +505,7 @@ MODULE_DEVICE_TABLE(of, rt5190a_device_table);
 static struct i2c_driver rt5190a_driver = {
 	.driver = {
 		.name = "rt5190a",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = rt5190a_device_table,
 	},
 	.probe_new = rt5190a_probe,
diff --git a/drivers/regulator/rt5739.c b/drivers/regulator/rt5739.c
new file mode 100644
index 000000000000..74fc5bf6d87e
--- /dev/null
+++ b/drivers/regulator/rt5739.c
@@ -0,0 +1,291 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Device driver for RT5739 regulator
+ *
+ * Copyright (C) 2023 Richtek Technology Corp.
+ *
+ * Author: ChiYuan Huang <cy_huang@richtek.com>
+ */
+
+#include <linux/bits.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/mod_devicetable.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/of_regulator.h>
+
+#define RT5739_AUTO_MODE	0
+#define RT5739_FPWM_MODE	1
+
+#define RT5739_REG_NSEL0	0x00
+#define RT5739_REG_NSEL1	0x01
+#define RT5739_REG_CNTL1	0x02
+#define RT5739_REG_ID1		0x03
+#define RT5739_REG_CNTL2	0x06
+#define RT5739_REG_CNTL4	0x08
+
+#define RT5739_VSEL_MASK	GENMASK(7, 0)
+#define RT5739_MODEVSEL1_MASK	BIT(1)
+#define RT5739_MODEVSEL0_MASK	BIT(0)
+#define RT5739_VID_MASK		GENMASK(7, 5)
+#define RT5739_ACTD_MASK	BIT(7)
+#define RT5739_ENVSEL1_MASK	BIT(1)
+#define RT5739_ENVSEL0_MASK	BIT(0)
+
+#define RT5739_VOLT_MINUV	300000
+#define RT5739_VOLT_MAXUV	1300000
+#define RT5739_VOLT_STPUV	5000
+#define RT5739_N_VOLTS		201
+#define RT5739_I2CRDY_TIMEUS	1000
+
+static int rt5739_set_mode(struct regulator_dev *rdev, unsigned int mode)
+{
+	const struct regulator_desc *desc = rdev->desc;
+	struct regmap *regmap = rdev_get_regmap(rdev);
+	unsigned int mask, val;
+
+	if (desc->vsel_reg == RT5739_REG_NSEL0)
+		mask = RT5739_MODEVSEL0_MASK;
+	else
+		mask = RT5739_MODEVSEL1_MASK;
+
+	switch (mode) {
+	case REGULATOR_MODE_FAST:
+		val = mask;
+		break;
+	case REGULATOR_MODE_NORMAL:
+		val = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(regmap, RT5739_REG_CNTL1, mask, val);
+}
+
+static unsigned int rt5739_get_mode(struct regulator_dev *rdev)
+{
+	const struct regulator_desc *desc = rdev->desc;
+	struct regmap *regmap = rdev_get_regmap(rdev);
+	unsigned int mask, val;
+	int ret;
+
+	if (desc->vsel_reg == RT5739_REG_NSEL0)
+		mask = RT5739_MODEVSEL0_MASK;
+	else
+		mask = RT5739_MODEVSEL1_MASK;
+
+	ret = regmap_read(regmap, RT5739_REG_CNTL1, &val);
+	if (ret)
+		return REGULATOR_MODE_INVALID;
+
+	if (val & mask)
+		return REGULATOR_MODE_FAST;
+
+	return REGULATOR_MODE_NORMAL;
+}
+
+static int rt5739_set_suspend_voltage(struct regulator_dev *rdev, int uV)
+{
+	const struct regulator_desc *desc = rdev->desc;
+	struct regmap *regmap = rdev_get_regmap(rdev);
+	unsigned int reg, vsel;
+
+	if (uV < RT5739_VOLT_MINUV || uV > RT5739_VOLT_MAXUV)
+		return -EINVAL;
+
+	if (desc->vsel_reg == RT5739_REG_NSEL0)
+		reg = RT5739_REG_NSEL1;
+	else
+		reg = RT5739_REG_NSEL0;
+
+	vsel = (uV - RT5739_VOLT_MINUV) / RT5739_VOLT_STPUV;
+	return regmap_write(regmap, reg, vsel);
+}
+
+static int rt5739_set_suspend_enable(struct regulator_dev *rdev)
+{
+	const struct regulator_desc *desc = rdev->desc;
+	struct regmap *regmap = rdev_get_regmap(rdev);
+	unsigned int mask;
+
+	if (desc->vsel_reg == RT5739_REG_NSEL0)
+		mask = RT5739_ENVSEL1_MASK;
+	else
+		mask = RT5739_ENVSEL0_MASK;
+
+	return regmap_update_bits(regmap, desc->enable_reg, mask, mask);
+}
+
+static int rt5739_set_suspend_disable(struct regulator_dev *rdev)
+{
+	const struct regulator_desc *desc = rdev->desc;
+	struct regmap *regmap = rdev_get_regmap(rdev);
+	unsigned int mask;
+
+	if (desc->vsel_reg == RT5739_REG_NSEL0)
+		mask = RT5739_ENVSEL1_MASK;
+	else
+		mask = RT5739_ENVSEL0_MASK;
+
+	return regmap_update_bits(regmap, desc->enable_reg, mask, 0);
+}
+
+static int rt5739_set_suspend_mode(struct regulator_dev *rdev,
+				   unsigned int mode)
+{
+	const struct regulator_desc *desc = rdev->desc;
+	struct regmap *regmap = rdev_get_regmap(rdev);
+	unsigned int mask, val;
+
+	if (desc->vsel_reg == RT5739_REG_NSEL0)
+		mask = RT5739_MODEVSEL1_MASK;
+	else
+		mask = RT5739_MODEVSEL0_MASK;
+
+	switch (mode) {
+	case REGULATOR_MODE_FAST:
+		val = mask;
+		break;
+	case REGULATOR_MODE_NORMAL:
+		val = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(regmap, RT5739_REG_CNTL1, mask, val);
+}
+
+static const struct regulator_ops rt5739_regulator_ops = {
+	.list_voltage = regulator_list_voltage_linear,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.enable	= regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.set_active_discharge = regulator_set_active_discharge_regmap,
+	.set_mode = rt5739_set_mode,
+	.get_mode = rt5739_get_mode,
+	.set_suspend_voltage = rt5739_set_suspend_voltage,
+	.set_suspend_enable = rt5739_set_suspend_enable,
+	.set_suspend_disable = rt5739_set_suspend_disable,
+	.set_suspend_mode = rt5739_set_suspend_mode,
+};
+
+static unsigned int rt5739_of_map_mode(unsigned int mode)
+{
+	switch (mode) {
+	case RT5739_AUTO_MODE:
+		return REGULATOR_MODE_NORMAL;
+	case RT5739_FPWM_MODE:
+		return REGULATOR_MODE_FAST;
+	default:
+		return REGULATOR_MODE_INVALID;
+	}
+}
+
+static void rt5739_init_regulator_desc(struct regulator_desc *desc,
+				       bool vsel_active_high)
+{
+	/* Fixed */
+	desc->name = "rt5739-regulator";
+	desc->owner = THIS_MODULE;
+	desc->ops = &rt5739_regulator_ops;
+	desc->n_voltages = RT5739_N_VOLTS;
+	desc->min_uV = RT5739_VOLT_MINUV;
+	desc->uV_step = RT5739_VOLT_STPUV;
+	desc->vsel_mask = RT5739_VSEL_MASK;
+	desc->enable_reg = RT5739_REG_CNTL2;
+	desc->active_discharge_reg = RT5739_REG_CNTL1;
+	desc->active_discharge_mask = RT5739_ACTD_MASK;
+	desc->active_discharge_on = RT5739_ACTD_MASK;
+	desc->of_map_mode = rt5739_of_map_mode;
+
+	/* Assigned by vsel level */
+	if (vsel_active_high) {
+		desc->vsel_reg = RT5739_REG_NSEL1;
+		desc->enable_mask = RT5739_ENVSEL1_MASK;
+	} else {
+		desc->vsel_reg = RT5739_REG_NSEL0;
+		desc->enable_mask = RT5739_ENVSEL0_MASK;
+	}
+}
+
+static const struct regmap_config rt5739_regmap_config = {
+	.name = "rt5739",
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = RT5739_REG_CNTL4,
+};
+
+static int rt5739_probe(struct i2c_client *i2c)
+{
+	struct device *dev = &i2c->dev;
+	struct regulator_desc *desc;
+	struct regmap *regmap;
+	struct gpio_desc *enable_gpio;
+	struct regulator_config cfg = {};
+	struct regulator_dev *rdev;
+	bool vsel_acth;
+	unsigned int vid;
+	int ret;
+
+	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	enable_gpio = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_HIGH);
+	if (IS_ERR(enable_gpio))
+		return dev_err_probe(dev, PTR_ERR(enable_gpio), "Failed to get 'enable' gpio\n");
+	else if (enable_gpio)
+		usleep_range(RT5739_I2CRDY_TIMEUS, RT5739_I2CRDY_TIMEUS + 1000);
+
+	regmap = devm_regmap_init_i2c(i2c, &rt5739_regmap_config);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Failed to init regmap\n");
+
+	ret = regmap_read(regmap, RT5739_REG_ID1, &vid);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to read VID\n");
+
+	/* RT5739: (VID & MASK) must be 0 */
+	if (vid & RT5739_VID_MASK)
+		return dev_err_probe(dev, -ENODEV, "Incorrect VID (0x%02x)\n", vid);
+
+	vsel_acth = device_property_read_bool(dev, "richtek,vsel-active-high");
+
+	rt5739_init_regulator_desc(desc, vsel_acth);
+
+	cfg.dev = dev;
+	cfg.of_node = dev_of_node(dev);
+	cfg.init_data = of_get_regulator_init_data(dev, dev_of_node(dev), desc);
+	rdev = devm_regulator_register(dev, desc, &cfg);
+	if (IS_ERR(rdev))
+		return dev_err_probe(dev, PTR_ERR(rdev), "Failed to register regulator\n");
+
+	return 0;
+}
+
+static const struct of_device_id rt5739_device_table[] = {
+	{ .compatible = "richtek,rt5739" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, rt5739_device_table);
+
+static struct i2c_driver rt5739_driver = {
+	.driver = {
+		.name = "rt5739",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+		.of_match_table = rt5739_device_table,
+	},
+	.probe_new = rt5739_probe,
+};
+module_i2c_driver(rt5739_driver);
+
+MODULE_AUTHOR("ChiYuan Huang <cy_huang@richtek.com>");
+MODULE_DESCRIPTION("Richtek RT5739 regulator driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/rt5759-regulator.c b/drivers/regulator/rt5759-regulator.c
index 8488417f4b2c..d5a42ad21a9a 100644
--- a/drivers/regulator/rt5759-regulator.c
+++ b/drivers/regulator/rt5759-regulator.c
@@ -359,6 +359,7 @@ MODULE_DEVICE_TABLE(of, rt5759_device_table);
 static struct i2c_driver rt5759_driver = {
 	.driver = {
 		.name = "rt5759",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(rt5759_device_table),
 	},
 	.probe_new = rt5759_probe,
diff --git a/drivers/regulator/rt6160-regulator.c b/drivers/regulator/rt6160-regulator.c
index 5d7b0e7ad69a..8990dac23460 100644
--- a/drivers/regulator/rt6160-regulator.c
+++ b/drivers/regulator/rt6160-regulator.c
@@ -308,6 +308,7 @@ MODULE_DEVICE_TABLE(of, rt6160_of_match_table);
 static struct i2c_driver rt6160_driver = {
 	.driver = {
 		.name = "rt6160",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = rt6160_of_match_table,
 	},
 	.probe_new = rt6160_probe,
diff --git a/drivers/regulator/rt6190-regulator.c b/drivers/regulator/rt6190-regulator.c
index 995e028abdd7..ca91a1f6d3c8 100644
--- a/drivers/regulator/rt6190-regulator.c
+++ b/drivers/regulator/rt6190-regulator.c
@@ -483,6 +483,7 @@ MODULE_DEVICE_TABLE(of, rt6190_of_dev_table);
 static struct i2c_driver rt6190_driver = {
 	.driver = {
 		.name = "rt6190",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = rt6190_of_dev_table,
 		.pm = pm_ptr(&rt6190_dev_pm),
 	},
diff --git a/drivers/regulator/rt6245-regulator.c b/drivers/regulator/rt6245-regulator.c
index cb22a207e9ff..8721d11c7964 100644
--- a/drivers/regulator/rt6245-regulator.c
+++ b/drivers/regulator/rt6245-regulator.c
@@ -243,6 +243,7 @@ MODULE_DEVICE_TABLE(of, rt6245_of_match_table);
 static struct i2c_driver rt6245_driver = {
 	.driver = {
 		.name = "rt6245",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = rt6245_of_match_table,
 	},
 	.probe_new = rt6245_probe,
diff --git a/drivers/regulator/rtmv20-regulator.c b/drivers/regulator/rtmv20-regulator.c
index 2ee334174e2b..7cbb812477e1 100644
--- a/drivers/regulator/rtmv20-regulator.c
+++ b/drivers/regulator/rtmv20-regulator.c
@@ -425,6 +425,7 @@ MODULE_DEVICE_TABLE(of, rtmv20_of_id);
 static struct i2c_driver rtmv20_driver = {
 	.driver = {
 		.name = "rtmv20",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(rtmv20_of_id),
 		.pm = &rtmv20_pm,
 	},
diff --git a/drivers/regulator/rtq2134-regulator.c b/drivers/regulator/rtq2134-regulator.c
index 8e13dea354a2..ee1577dc3cfc 100644
--- a/drivers/regulator/rtq2134-regulator.c
+++ b/drivers/regulator/rtq2134-regulator.c
@@ -363,6 +363,7 @@ MODULE_DEVICE_TABLE(of, rtq2134_device_tables);
 static struct i2c_driver rtq2134_driver = {
 	.driver = {
 		.name = "rtq2134",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = rtq2134_device_tables,
 	},
 	.probe_new = rtq2134_probe,
diff --git a/drivers/regulator/rtq6752-regulator.c b/drivers/regulator/rtq6752-regulator.c
index dfe45fb67353..8559a266a7eb 100644
--- a/drivers/regulator/rtq6752-regulator.c
+++ b/drivers/regulator/rtq6752-regulator.c
@@ -278,6 +278,7 @@ MODULE_DEVICE_TABLE(of, rtq6752_device_table);
 static struct i2c_driver rtq6752_driver = {
 	.driver = {
 		.name = "rtq6752",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = rtq6752_device_table,
 	},
 	.probe_new = rtq6752_probe,
diff --git a/drivers/regulator/s2mpa01.c b/drivers/regulator/s2mpa01.c
index 28b424fe7bea..b147ff6a16b1 100644
--- a/drivers/regulator/s2mpa01.c
+++ b/drivers/regulator/s2mpa01.c
@@ -376,6 +376,7 @@ MODULE_DEVICE_TABLE(platform, s2mpa01_pmic_id);
 static struct platform_driver s2mpa01_pmic_driver = {
 	.driver = {
 		.name = "s2mpa01-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = s2mpa01_pmic_probe,
 	.id_table = s2mpa01_pmic_id,
diff --git a/drivers/regulator/s2mps11.c b/drivers/regulator/s2mps11.c
index ebc67e3ddd4f..570b61420f3a 100644
--- a/drivers/regulator/s2mps11.c
+++ b/drivers/regulator/s2mps11.c
@@ -1238,6 +1238,7 @@ MODULE_DEVICE_TABLE(platform, s2mps11_pmic_id);
 static struct platform_driver s2mps11_pmic_driver = {
 	.driver = {
 		.name = "s2mps11-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = s2mps11_pmic_probe,
 	.id_table = s2mps11_pmic_id,
diff --git a/drivers/regulator/s5m8767.c b/drivers/regulator/s5m8767.c
index 754c6fcc6e64..bfc0e143bf40 100644
--- a/drivers/regulator/s5m8767.c
+++ b/drivers/regulator/s5m8767.c
@@ -605,7 +605,7 @@ static int s5m8767_pmic_dt_parse_pdata(struct platform_device *pdev,
 
 	of_node_put(regulators_np);
 
-	if (of_get_property(pmic_np, "s5m8767,pmic-buck2-uses-gpio-dvs", NULL)) {
+	if (of_property_read_bool(pmic_np, "s5m8767,pmic-buck2-uses-gpio-dvs")) {
 		pdata->buck2_gpiodvs = true;
 
 		if (of_property_read_u32_array(pmic_np,
@@ -616,7 +616,7 @@ static int s5m8767_pmic_dt_parse_pdata(struct platform_device *pdev,
 		}
 	}
 
-	if (of_get_property(pmic_np, "s5m8767,pmic-buck3-uses-gpio-dvs", NULL)) {
+	if (of_property_read_bool(pmic_np, "s5m8767,pmic-buck3-uses-gpio-dvs")) {
 		pdata->buck3_gpiodvs = true;
 
 		if (of_property_read_u32_array(pmic_np,
@@ -627,7 +627,7 @@ static int s5m8767_pmic_dt_parse_pdata(struct platform_device *pdev,
 		}
 	}
 
-	if (of_get_property(pmic_np, "s5m8767,pmic-buck4-uses-gpio-dvs", NULL)) {
+	if (of_property_read_bool(pmic_np, "s5m8767,pmic-buck4-uses-gpio-dvs")) {
 		pdata->buck4_gpiodvs = true;
 
 		if (of_property_read_u32_array(pmic_np,
@@ -661,14 +661,9 @@ static int s5m8767_pmic_dt_parse_pdata(struct platform_device *pdev,
 	if (ret)
 		return -EINVAL;
 
-	if (of_get_property(pmic_np, "s5m8767,pmic-buck2-ramp-enable", NULL))
-		pdata->buck2_ramp_enable = true;
-
-	if (of_get_property(pmic_np, "s5m8767,pmic-buck3-ramp-enable", NULL))
-		pdata->buck3_ramp_enable = true;
-
-	if (of_get_property(pmic_np, "s5m8767,pmic-buck4-ramp-enable", NULL))
-		pdata->buck4_ramp_enable = true;
+	pdata->buck2_ramp_enable = of_property_read_bool(pmic_np, "s5m8767,pmic-buck2-ramp-enable");
+	pdata->buck3_ramp_enable = of_property_read_bool(pmic_np, "s5m8767,pmic-buck3-ramp-enable");
+	pdata->buck4_ramp_enable = of_property_read_bool(pmic_np, "s5m8767,pmic-buck4-ramp-enable");
 
 	if (pdata->buck2_ramp_enable || pdata->buck3_ramp_enable
 			|| pdata->buck4_ramp_enable) {
@@ -1004,6 +999,7 @@ MODULE_DEVICE_TABLE(platform, s5m8767_pmic_id);
 static struct platform_driver s5m8767_pmic_driver = {
 	.driver = {
 		.name = "s5m8767-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = s5m8767_pmic_probe,
 	.id_table = s5m8767_pmic_id,
diff --git a/drivers/regulator/sc2731-regulator.c b/drivers/regulator/sc2731-regulator.c
index 71e5ceb679f4..5447e1a47d15 100644
--- a/drivers/regulator/sc2731-regulator.c
+++ b/drivers/regulator/sc2731-regulator.c
@@ -245,6 +245,7 @@ static int sc2731_regulator_probe(struct platform_device *pdev)
 static struct platform_driver sc2731_regulator_driver = {
 	.driver = {
 		.name = "sc27xx-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = sc2731_regulator_probe,
 };
diff --git a/drivers/regulator/sky81452-regulator.c b/drivers/regulator/sky81452-regulator.c
index 37658affe072..359e83e37d2d 100644
--- a/drivers/regulator/sky81452-regulator.c
+++ b/drivers/regulator/sky81452-regulator.c
@@ -79,6 +79,7 @@ static int sky81452_reg_probe(struct platform_device *pdev)
 static struct platform_driver sky81452_reg_driver = {
 	.driver = {
 		.name = "sky81452-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = sky81452_reg_probe,
 };
diff --git a/drivers/regulator/slg51000-regulator.c b/drivers/regulator/slg51000-regulator.c
index 1b2eee95ad3f..559ae031010f 100644
--- a/drivers/regulator/slg51000-regulator.c
+++ b/drivers/regulator/slg51000-regulator.c
@@ -505,6 +505,7 @@ MODULE_DEVICE_TABLE(i2c, slg51000_i2c_id);
 static struct i2c_driver slg51000_regulator_driver = {
 	.driver = {
 		.name = "slg51000-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe_new = slg51000_i2c_probe,
 	.id_table = slg51000_i2c_id,
diff --git a/drivers/regulator/sm5703-regulator.c b/drivers/regulator/sm5703-regulator.c
index 05ad28fc4da8..702461cf075e 100644
--- a/drivers/regulator/sm5703-regulator.c
+++ b/drivers/regulator/sm5703-regulator.c
@@ -42,6 +42,7 @@ static const int sm5703_buck_voltagemap[] = {
 		.type = REGULATOR_VOLTAGE,				\
 		.id = SM5703_USBLDO ## _id,				\
 		.ops = &sm5703_regulator_ops_fixed,			\
+		.n_voltages = 1,					\
 		.fixed_uV = SM5703_USBLDO_MICROVOLT,			\
 		.enable_reg = SM5703_REG_USBLDO12,			\
 		.enable_mask = SM5703_REG_EN_USBLDO ##_id,		\
@@ -56,6 +57,7 @@ static const int sm5703_buck_voltagemap[] = {
 		.type = REGULATOR_VOLTAGE,				\
 		.id = SM5703_VBUS,					\
 		.ops = &sm5703_regulator_ops_fixed,			\
+		.n_voltages = 1,					\
 		.fixed_uV = SM5703_VBUS_MICROVOLT,			\
 		.enable_reg = SM5703_REG_CNTL,				\
 		.enable_mask = SM5703_OPERATION_MODE_MASK,		\
@@ -155,6 +157,7 @@ MODULE_DEVICE_TABLE(platform, sm5703_regulator_id);
 static struct platform_driver sm5703_regulator_driver = {
 	.driver = {
 		.name = "sm5703-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe	= sm5703_regulator_probe,
 	.id_table	= sm5703_regulator_id,
diff --git a/drivers/regulator/stm32-booster.c b/drivers/regulator/stm32-booster.c
index 3136ea8a35d5..b64dc5a497fa 100644
--- a/drivers/regulator/stm32-booster.c
+++ b/drivers/regulator/stm32-booster.c
@@ -117,6 +117,7 @@ static struct platform_driver stm32_booster_driver = {
 	.probe = stm32_booster_probe,
 	.driver = {
 		.name  = "stm32-booster",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(stm32_booster_of_match),
 	},
 };
diff --git a/drivers/regulator/stm32-pwr.c b/drivers/regulator/stm32-pwr.c
index 2a42acb7c24e..0e101dff6dda 100644
--- a/drivers/regulator/stm32-pwr.c
+++ b/drivers/regulator/stm32-pwr.c
@@ -129,17 +129,16 @@ static const struct regulator_desc stm32_pwr_desc[] = {
 
 static int stm32_pwr_regulator_probe(struct platform_device *pdev)
 {
-	struct device_node *np = pdev->dev.of_node;
 	struct stm32_pwr_reg *priv;
 	void __iomem *base;
 	struct regulator_dev *rdev;
 	struct regulator_config config = { };
 	int i, ret = 0;
 
-	base = of_iomap(np, 0);
-	if (!base) {
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base)) {
 		dev_err(&pdev->dev, "Unable to map IO memory\n");
-		return -ENOMEM;
+		return PTR_ERR(base);
 	}
 
 	config.dev = &pdev->dev;
@@ -176,6 +175,7 @@ static struct platform_driver stm32_pwr_driver = {
 	.probe = stm32_pwr_regulator_probe,
 	.driver = {
 		.name  = "stm32-pwr-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(stm32_pwr_of_match),
 	},
 };
@@ -183,4 +183,3 @@ module_platform_driver(stm32_pwr_driver);
 
 MODULE_DESCRIPTION("STM32MP1 PWR voltage regulator driver");
 MODULE_AUTHOR("Pascal Paillet <p.paillet@st.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/regulator/stm32-vrefbuf.c b/drivers/regulator/stm32-vrefbuf.c
index 7a454b7b6eab..f5ccc7dd309a 100644
--- a/drivers/regulator/stm32-vrefbuf.c
+++ b/drivers/regulator/stm32-vrefbuf.c
@@ -285,6 +285,7 @@ static struct platform_driver stm32_vrefbuf_driver = {
 	.remove = stm32_vrefbuf_remove,
 	.driver = {
 		.name  = "stm32-vrefbuf",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(stm32_vrefbuf_of_match),
 		.pm = &stm32_vrefbuf_pm_ops,
 	},
diff --git a/drivers/regulator/stpmic1_regulator.c b/drivers/regulator/stpmic1_regulator.c
index 2d7597c76e4a..79d1a3eb18d4 100644
--- a/drivers/regulator/stpmic1_regulator.c
+++ b/drivers/regulator/stpmic1_regulator.c
@@ -576,7 +576,7 @@ static int stpmic1_regulator_register(struct platform_device *pdev, int id,
 	}
 
 	/* set mask reset */
-	if (of_get_property(config.of_node, "st,mask-reset", NULL) &&
+	if (of_property_read_bool(config.of_node, "st,mask-reset") &&
 	    cfg->mask_reset_reg != 0) {
 		ret = regmap_update_bits(pmic_dev->regmap,
 					 cfg->mask_reset_reg,
@@ -638,6 +638,7 @@ MODULE_DEVICE_TABLE(of, of_pmic_regulator_match);
 static struct platform_driver stpmic1_regulator_driver = {
 	.driver = {
 		.name = "stpmic1-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(of_pmic_regulator_match),
 	},
 	.probe = stpmic1_regulator_probe,
diff --git a/drivers/regulator/stw481x-vmmc.c b/drivers/regulator/stw481x-vmmc.c
index 127ab43add49..3958d906bd77 100644
--- a/drivers/regulator/stw481x-vmmc.c
+++ b/drivers/regulator/stw481x-vmmc.c
@@ -95,6 +95,7 @@ static const struct of_device_id stw481x_vmmc_match[] = {
 static struct platform_driver stw481x_vmmc_regulator_driver = {
 	.driver = {
 		.name  = "stw481x-vmmc-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = stw481x_vmmc_match,
 	},
 	.probe = stw481x_vmmc_regulator_probe,
diff --git a/drivers/regulator/sy7636a-regulator.c b/drivers/regulator/sy7636a-regulator.c
index 29fc27c2cda0..d1e7ba1fb3e1 100644
--- a/drivers/regulator/sy7636a-regulator.c
+++ b/drivers/regulator/sy7636a-regulator.c
@@ -127,6 +127,7 @@ MODULE_DEVICE_TABLE(platform, sy7636a_regulator_id_table);
 static struct platform_driver sy7636a_regulator_driver = {
 	.driver = {
 		.name = "sy7636a-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = sy7636a_regulator_probe,
 	.id_table = sy7636a_regulator_id_table,
diff --git a/drivers/regulator/sy8106a-regulator.c b/drivers/regulator/sy8106a-regulator.c
index b10bd99768a3..e3c753986309 100644
--- a/drivers/regulator/sy8106a-regulator.c
+++ b/drivers/regulator/sy8106a-regulator.c
@@ -138,6 +138,7 @@ MODULE_DEVICE_TABLE(i2c, sy8106a_i2c_id);
 static struct i2c_driver sy8106a_regulator_driver = {
 	.driver = {
 		.name = "sy8106a",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= sy8106a_i2c_of_match,
 	},
 	.probe_new = sy8106a_i2c_probe,
diff --git a/drivers/regulator/sy8824x.c b/drivers/regulator/sy8824x.c
index 2a81519bdf67..c327ad69f676 100644
--- a/drivers/regulator/sy8824x.c
+++ b/drivers/regulator/sy8824x.c
@@ -233,6 +233,7 @@ MODULE_DEVICE_TABLE(i2c, sy8824_id);
 static struct i2c_driver sy8824_regulator_driver = {
 	.driver = {
 		.name = "sy8824-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = sy8824_dt_ids,
 	},
 	.probe_new = sy8824_i2c_probe,
diff --git a/drivers/regulator/sy8827n.c b/drivers/regulator/sy8827n.c
index 936a94b6df5b..99ca08cc3a6a 100644
--- a/drivers/regulator/sy8827n.c
+++ b/drivers/regulator/sy8827n.c
@@ -187,6 +187,7 @@ MODULE_DEVICE_TABLE(i2c, sy8827n_id);
 static struct i2c_driver sy8827n_regulator_driver = {
 	.driver = {
 		.name = "sy8827n-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = sy8827n_dt_ids,
 	},
 	.probe_new = sy8827n_i2c_probe,
diff --git a/drivers/regulator/ti-abb-regulator.c b/drivers/regulator/ti-abb-regulator.c
index 115345e9fded..86d2d80b4b41 100644
--- a/drivers/regulator/ti-abb-regulator.c
+++ b/drivers/regulator/ti-abb-regulator.c
@@ -865,6 +865,7 @@ static struct platform_driver ti_abb_driver = {
 	.probe = ti_abb_probe,
 	.driver = {
 		   .name = "ti_abb",
+		   .probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		   .of_match_table = of_match_ptr(ti_abb_of_match),
 		   },
 };
diff --git a/drivers/regulator/tps51632-regulator.c b/drivers/regulator/tps51632-regulator.c
index 152c5ad6709c..9bd4e72914ed 100644
--- a/drivers/regulator/tps51632-regulator.c
+++ b/drivers/regulator/tps51632-regulator.c
@@ -351,6 +351,7 @@ MODULE_DEVICE_TABLE(i2c, tps51632_id);
 static struct i2c_driver tps51632_i2c_driver = {
 	.driver = {
 		.name = "tps51632",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(tps51632_of_match),
 	},
 	.probe_new = tps51632_probe,
diff --git a/drivers/regulator/tps6105x-regulator.c b/drivers/regulator/tps6105x-regulator.c
index a6469fe05635..a09c6ae6a0ce 100644
--- a/drivers/regulator/tps6105x-regulator.c
+++ b/drivers/regulator/tps6105x-regulator.c
@@ -93,6 +93,7 @@ static int tps6105x_regulator_probe(struct platform_device *pdev)
 static struct platform_driver tps6105x_regulator_driver = {
 	.driver = {
 		.name  = "tps6105x-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = tps6105x_regulator_probe,
 };
diff --git a/drivers/regulator/tps62360-regulator.c b/drivers/regulator/tps62360-regulator.c
index da1b2b1341ae..65cc08d1a67d 100644
--- a/drivers/regulator/tps62360-regulator.c
+++ b/drivers/regulator/tps62360-regulator.c
@@ -296,17 +296,10 @@ static struct tps62360_regulator_platform_data *
 		return NULL;
 	}
 
-	if (of_find_property(np, "ti,vsel0-state-high", NULL))
-		pdata->vsel0_def_state = 1;
-
-	if (of_find_property(np, "ti,vsel1-state-high", NULL))
-		pdata->vsel1_def_state = 1;
-
-	if (of_find_property(np, "ti,enable-pull-down", NULL))
-		pdata->en_internal_pulldn = true;
-
-	if (of_find_property(np, "ti,enable-vout-discharge", NULL))
-		pdata->en_discharge = true;
+	pdata->vsel0_def_state = of_property_read_bool(np, "ti,vsel0-state-high");
+	pdata->vsel1_def_state = of_property_read_bool(np, "ti,vsel1-state-high");
+	pdata->en_internal_pulldn = of_property_read_bool(np, "ti,enable-pull-down");
+	pdata->en_discharge = of_property_read_bool(np, "ti,enable-vout-discharge");
 
 	return pdata;
 }
@@ -495,6 +488,7 @@ MODULE_DEVICE_TABLE(i2c, tps62360_id);
 static struct i2c_driver tps62360_i2c_driver = {
 	.driver = {
 		.name = "tps62360",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(tps62360_of_match),
 	},
 	.probe_new = tps62360_probe,
diff --git a/drivers/regulator/tps6286x-regulator.c b/drivers/regulator/tps6286x-regulator.c
index 207ac1d1d88d..f92e7649d0a0 100644
--- a/drivers/regulator/tps6286x-regulator.c
+++ b/drivers/regulator/tps6286x-regulator.c
@@ -147,6 +147,7 @@ MODULE_DEVICE_TABLE(i2c, tps6286x_i2c_id);
 static struct i2c_driver tps6286x_regulator_driver = {
 	.driver = {
 		.name = "tps6286x",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(tps6286x_dt_ids),
 	},
 	.probe_new = tps6286x_i2c_probe,
diff --git a/drivers/regulator/tps65023-regulator.c b/drivers/regulator/tps65023-regulator.c
index 680a57ff0837..d87cac63f346 100644
--- a/drivers/regulator/tps65023-regulator.c
+++ b/drivers/regulator/tps65023-regulator.c
@@ -334,6 +334,7 @@ MODULE_DEVICE_TABLE(i2c, tps_65023_id);
 static struct i2c_driver tps_65023_i2c_driver = {
 	.driver = {
 		.name = "tps65023",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(tps65023_of_match),
 	},
 	.probe_new = tps_65023_probe,
diff --git a/drivers/regulator/tps6507x-regulator.c b/drivers/regulator/tps6507x-regulator.c
index b83816ee6867..6655d9c31187 100644
--- a/drivers/regulator/tps6507x-regulator.c
+++ b/drivers/regulator/tps6507x-regulator.c
@@ -438,6 +438,7 @@ static int tps6507x_pmic_probe(struct platform_device *pdev)
 static struct platform_driver tps6507x_pmic_driver = {
 	.driver = {
 		.name = "tps6507x-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = tps6507x_pmic_probe,
 };
diff --git a/drivers/regulator/tps65086-regulator.c b/drivers/regulator/tps65086-regulator.c
index f1bc54c825dd..663789198ba5 100644
--- a/drivers/regulator/tps65086-regulator.c
+++ b/drivers/regulator/tps65086-regulator.c
@@ -235,6 +235,7 @@ MODULE_DEVICE_TABLE(platform, tps65086_regulator_id_table);
 static struct platform_driver tps65086_regulator_driver = {
 	.driver = {
 		.name = "tps65086-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = tps65086_regulator_probe,
 	.id_table = tps65086_regulator_id_table,
diff --git a/drivers/regulator/tps65090-regulator.c b/drivers/regulator/tps65090-regulator.c
index 1d2e04f452d4..8f916ee366e5 100644
--- a/drivers/regulator/tps65090-regulator.c
+++ b/drivers/regulator/tps65090-regulator.c
@@ -511,6 +511,7 @@ static int tps65090_regulator_probe(struct platform_device *pdev)
 static struct platform_driver tps65090_regulator_driver = {
 	.driver	= {
 		.name	= "tps65090-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe		= tps65090_regulator_probe,
 };
diff --git a/drivers/regulator/tps65132-regulator.c b/drivers/regulator/tps65132-regulator.c
index 0edc83089ba2..d4b02ee791d1 100644
--- a/drivers/regulator/tps65132-regulator.c
+++ b/drivers/regulator/tps65132-regulator.c
@@ -270,6 +270,7 @@ MODULE_DEVICE_TABLE(i2c, tps65132_id);
 static struct i2c_driver tps65132_i2c_driver = {
 	.driver = {
 		.name = "tps65132",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe_new = tps65132_probe,
 	.id_table = tps65132_id,
diff --git a/drivers/regulator/tps65217-regulator.c b/drivers/regulator/tps65217-regulator.c
index 6bb5b02e19e2..b167ba22fe16 100644
--- a/drivers/regulator/tps65217-regulator.c
+++ b/drivers/regulator/tps65217-regulator.c
@@ -258,6 +258,7 @@ static int tps65217_regulator_probe(struct platform_device *pdev)
 static struct platform_driver tps65217_regulator_driver = {
 	.driver = {
 		.name = "tps65217-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = tps65217_regulator_probe,
 };
diff --git a/drivers/regulator/tps65218-regulator.c b/drivers/regulator/tps65218-regulator.c
index 48809c3b3abc..13985883e5f0 100644
--- a/drivers/regulator/tps65218-regulator.c
+++ b/drivers/regulator/tps65218-regulator.c
@@ -349,6 +349,7 @@ MODULE_DEVICE_TABLE(platform, tps65218_regulator_id_table);
 static struct platform_driver tps65218_regulator_driver = {
 	.driver = {
 		.name = "tps65218-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = tps65218_regulator_probe,
 	.id_table = tps65218_regulator_id_table,
diff --git a/drivers/regulator/tps65219-regulator.c b/drivers/regulator/tps65219-regulator.c
index 4b5acaa45049..b1719ee990ab 100644
--- a/drivers/regulator/tps65219-regulator.c
+++ b/drivers/regulator/tps65219-regulator.c
@@ -380,6 +380,7 @@ MODULE_DEVICE_TABLE(platform, tps65219_regulator_id_table);
 static struct platform_driver tps65219_regulator_driver = {
 	.driver = {
 		.name = "tps65219-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = tps65219_regulator_probe,
 	.id_table = tps65219_regulator_id_table,
diff --git a/drivers/regulator/tps6524x-regulator.c b/drivers/regulator/tps6524x-regulator.c
index 740aeccdfb1f..3fee7e38c68b 100644
--- a/drivers/regulator/tps6524x-regulator.c
+++ b/drivers/regulator/tps6524x-regulator.c
@@ -628,6 +628,7 @@ static struct spi_driver pmic_driver = {
 	.probe		= pmic_probe,
 	.driver		= {
 		.name	= "tps6524x",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/tps6586x-regulator.c b/drivers/regulator/tps6586x-regulator.c
index 18bf4b885b08..1ab5767590f3 100644
--- a/drivers/regulator/tps6586x-regulator.c
+++ b/drivers/regulator/tps6586x-regulator.c
@@ -520,6 +520,7 @@ static int tps6586x_regulator_probe(struct platform_device *pdev)
 static struct platform_driver tps6586x_regulator_driver = {
 	.driver	= {
 		.name	= "tps6586x-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe		= tps6586x_regulator_probe,
 };
diff --git a/drivers/regulator/tps65910-regulator.c b/drivers/regulator/tps65910-regulator.c
index 06cbe60c990f..2a0965ba1570 100644
--- a/drivers/regulator/tps65910-regulator.c
+++ b/drivers/regulator/tps65910-regulator.c
@@ -1255,6 +1255,7 @@ static void tps65910_shutdown(struct platform_device *pdev)
 static struct platform_driver tps65910_driver = {
 	.driver = {
 		.name = "tps65910-pmic",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = tps65910_probe,
 	.shutdown = tps65910_shutdown,
diff --git a/drivers/regulator/tps65912-regulator.c b/drivers/regulator/tps65912-regulator.c
index 76f90202ae09..7ff7877a2e09 100644
--- a/drivers/regulator/tps65912-regulator.c
+++ b/drivers/regulator/tps65912-regulator.c
@@ -150,6 +150,7 @@ MODULE_DEVICE_TABLE(platform, tps65912_regulator_id_table);
 static struct platform_driver tps65912_regulator_driver = {
 	.driver = {
 		.name = "tps65912-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = tps65912_regulator_probe,
 	.id_table = tps65912_regulator_id_table,
diff --git a/drivers/regulator/tps68470-regulator.c b/drivers/regulator/tps68470-regulator.c
index 4bca7c4128ab..de7db7690f6b 100644
--- a/drivers/regulator/tps68470-regulator.c
+++ b/drivers/regulator/tps68470-regulator.c
@@ -175,6 +175,7 @@ static int tps68470_regulator_probe(struct platform_device *pdev)
 static struct platform_driver tps68470_regulator_driver = {
 	.driver = {
 		.name = "tps68470-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = tps68470_regulator_probe,
 };
diff --git a/drivers/regulator/twl-regulator.c b/drivers/regulator/twl-regulator.c
index e2a20d512152..3e724f5345de 100644
--- a/drivers/regulator/twl-regulator.c
+++ b/drivers/regulator/twl-regulator.c
@@ -656,6 +656,7 @@ static struct platform_driver twlreg_driver = {
 	 */
 	.driver  = {
 		.name  = "twl4030_reg",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(twl_of_match),
 	},
 };
diff --git a/drivers/regulator/twl6030-regulator.c b/drivers/regulator/twl6030-regulator.c
index f3856750944f..f9c695f9bde8 100644
--- a/drivers/regulator/twl6030-regulator.c
+++ b/drivers/regulator/twl6030-regulator.c
@@ -729,7 +729,7 @@ static int twlreg_probe(struct platform_device *pdev)
 		break;
 	}
 
-	if (of_get_property(np, "ti,retain-on-reset", NULL))
+	if (of_property_read_bool(np, "ti,retain-on-reset"))
 		info->flags |= TWL_6030_WARM_RESET;
 
 	config.dev = &pdev->dev;
@@ -765,6 +765,7 @@ static struct platform_driver twlreg_driver = {
 	 */
 	.driver  = {
 		.name  = "twl6030_reg",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(twl_of_match),
 	},
 };
diff --git a/drivers/regulator/uniphier-regulator.c b/drivers/regulator/uniphier-regulator.c
index 39a68b01fc38..7e2785e10dc6 100644
--- a/drivers/regulator/uniphier-regulator.c
+++ b/drivers/regulator/uniphier-regulator.c
@@ -212,6 +212,7 @@ static struct platform_driver uniphier_regulator_driver = {
 	.remove = uniphier_regulator_remove,
 	.driver = {
 		.name  = "uniphier-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = uniphier_regulator_match,
 	},
 };
diff --git a/drivers/regulator/userspace-consumer.c b/drivers/regulator/userspace-consumer.c
index 402c8037cf39..97f075ed68c9 100644
--- a/drivers/regulator/userspace-consumer.c
+++ b/drivers/regulator/userspace-consumer.c
@@ -216,6 +216,7 @@ static struct platform_driver regulator_userspace_consumer_driver = {
 	.remove		= regulator_userspace_consumer_remove,
 	.driver		= {
 		.name		= "reg-userspace-consumer",
+		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= regulator_userspace_consumer_of_match,
 	},
 };
diff --git a/drivers/regulator/vctrl-regulator.c b/drivers/regulator/vctrl-regulator.c
index aac7be3b33f7..85dca90233f6 100644
--- a/drivers/regulator/vctrl-regulator.c
+++ b/drivers/regulator/vctrl-regulator.c
@@ -543,6 +543,7 @@ static struct platform_driver vctrl_driver = {
 	.probe		= vctrl_probe,
 	.driver		= {
 		.name		= "vctrl-regulator",
+		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(vctrl_of_match),
 	},
 };
diff --git a/drivers/regulator/vexpress-regulator.c b/drivers/regulator/vexpress-regulator.c
index 5d39663efcaa..b545dbc70a4d 100644
--- a/drivers/regulator/vexpress-regulator.c
+++ b/drivers/regulator/vexpress-regulator.c
@@ -88,6 +88,7 @@ static struct platform_driver vexpress_regulator_driver = {
 	.probe = vexpress_regulator_probe,
 	.driver	= {
 		.name = DRVNAME,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = vexpress_regulator_of_match,
 	},
 };
diff --git a/drivers/regulator/virtual.c b/drivers/regulator/virtual.c
index 5d32628a5011..d5a160efdae6 100644
--- a/drivers/regulator/virtual.c
+++ b/drivers/regulator/virtual.c
@@ -362,6 +362,7 @@ static struct platform_driver regulator_virtual_consumer_driver = {
 	.remove		= regulator_virtual_remove,
 	.driver		= {
 		.name		= "reg-virt-consumer",
+		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(regulator_virtual_consumer_of_match),
 	},
 };
diff --git a/drivers/regulator/vqmmc-ipq4019-regulator.c b/drivers/regulator/vqmmc-ipq4019-regulator.c
index c4213f096fe5..086da36abc0b 100644
--- a/drivers/regulator/vqmmc-ipq4019-regulator.c
+++ b/drivers/regulator/vqmmc-ipq4019-regulator.c
@@ -89,6 +89,7 @@ static struct platform_driver ipq4019_regulator_driver = {
 	.probe = ipq4019_regulator_probe,
 	.driver = {
 		.name = "vqmmc-ipq4019-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = of_match_ptr(regulator_ipq4019_of_match),
 	},
 };
diff --git a/drivers/regulator/wm831x-dcdc.c b/drivers/regulator/wm831x-dcdc.c
index e43ed4d93f71..834d7c181971 100644
--- a/drivers/regulator/wm831x-dcdc.c
+++ b/drivers/regulator/wm831x-dcdc.c
@@ -505,6 +505,7 @@ static struct platform_driver wm831x_buckv_driver = {
 	.probe = wm831x_buckv_probe,
 	.driver		= {
 		.name	= "wm831x-buckv",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
@@ -632,6 +633,7 @@ static struct platform_driver wm831x_buckp_driver = {
 	.probe = wm831x_buckp_probe,
 	.driver		= {
 		.name	= "wm831x-buckp",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
@@ -747,6 +749,7 @@ static struct platform_driver wm831x_boostp_driver = {
 	.probe = wm831x_boostp_probe,
 	.driver		= {
 		.name	= "wm831x-boostp",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
@@ -822,6 +825,7 @@ static struct platform_driver wm831x_epe_driver = {
 	.probe = wm831x_epe_probe,
 	.driver		= {
 		.name	= "wm831x-epe",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/wm831x-isink.c b/drivers/regulator/wm831x-isink.c
index eade3ae3e333..ed5e191e8896 100644
--- a/drivers/regulator/wm831x-isink.c
+++ b/drivers/regulator/wm831x-isink.c
@@ -189,6 +189,7 @@ static struct platform_driver wm831x_isink_driver = {
 	.probe = wm831x_isink_probe,
 	.driver		= {
 		.name	= "wm831x-isink",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/wm831x-ldo.c b/drivers/regulator/wm831x-ldo.c
index e091b189ecc0..76b89b1cd519 100644
--- a/drivers/regulator/wm831x-ldo.c
+++ b/drivers/regulator/wm831x-ldo.c
@@ -303,6 +303,7 @@ static struct platform_driver wm831x_gp_ldo_driver = {
 	.probe = wm831x_gp_ldo_probe,
 	.driver		= {
 		.name	= "wm831x-ldo",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
@@ -512,6 +513,7 @@ static struct platform_driver wm831x_aldo_driver = {
 	.probe = wm831x_aldo_probe,
 	.driver		= {
 		.name	= "wm831x-aldo",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
@@ -645,6 +647,7 @@ static struct platform_driver wm831x_alive_ldo_driver = {
 	.probe = wm831x_alive_ldo_probe,
 	.driver		= {
 		.name	= "wm831x-alive-ldo",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/wm8350-regulator.c b/drivers/regulator/wm8350-regulator.c
index b1d5aac8917d..1445bafcab40 100644
--- a/drivers/regulator/wm8350-regulator.c
+++ b/drivers/regulator/wm8350-regulator.c
@@ -1309,6 +1309,7 @@ static struct platform_driver wm8350_regulator_driver = {
 	.remove = wm8350_regulator_remove,
 	.driver		= {
 		.name	= "wm8350-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/regulator/wm8400-regulator.c b/drivers/regulator/wm8400-regulator.c
index e9fd13707721..c4a229f66dec 100644
--- a/drivers/regulator/wm8400-regulator.c
+++ b/drivers/regulator/wm8400-regulator.c
@@ -223,6 +223,7 @@ static int wm8400_regulator_probe(struct platform_device *pdev)
 static struct platform_driver wm8400_regulator_driver = {
 	.driver = {
 		.name = "wm8400-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = wm8400_regulator_probe,
 };
diff --git a/drivers/regulator/wm8994-regulator.c b/drivers/regulator/wm8994-regulator.c
index 40befdd9dfa9..2946db448aec 100644
--- a/drivers/regulator/wm8994-regulator.c
+++ b/drivers/regulator/wm8994-regulator.c
@@ -227,6 +227,7 @@ static struct platform_driver wm8994_ldo_driver = {
 	.probe = wm8994_ldo_probe,
 	.driver		= {
 		.name	= "wm8994-ldo",
+		.probe_type = PROBE_FORCE_SYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/remoteproc/da8xx_remoteproc.c b/drivers/remoteproc/da8xx_remoteproc.c
index 98e0be9476a4..768217f0f5cd 100644
--- a/drivers/remoteproc/da8xx_remoteproc.c
+++ b/drivers/remoteproc/da8xx_remoteproc.c
@@ -84,7 +84,7 @@ struct da8xx_rproc {
  */
 static irqreturn_t handle_event(int irq, void *p)
 {
-	struct rproc *rproc = (struct rproc *)p;
+	struct rproc *rproc = p;
 
 	/* Process incoming buffers on all our vrings */
 	rproc_vq_interrupt(rproc, 0);
@@ -104,8 +104,8 @@ static irqreturn_t handle_event(int irq, void *p)
  */
 static irqreturn_t da8xx_rproc_callback(int irq, void *p)
 {
-	struct rproc *rproc = (struct rproc *)p;
-	struct da8xx_rproc *drproc = (struct da8xx_rproc *)rproc->priv;
+	struct rproc *rproc = p;
+	struct da8xx_rproc *drproc = rproc->priv;
 	u32 chipsig;
 
 	chipsig = readl(drproc->chipsig);
@@ -133,7 +133,7 @@ static irqreturn_t da8xx_rproc_callback(int irq, void *p)
 static int da8xx_rproc_start(struct rproc *rproc)
 {
 	struct device *dev = rproc->dev.parent;
-	struct da8xx_rproc *drproc = (struct da8xx_rproc *)rproc->priv;
+	struct da8xx_rproc *drproc = rproc->priv;
 	struct clk *dsp_clk = drproc->dsp_clk;
 	struct reset_control *dsp_reset = drproc->dsp_reset;
 	int ret;
@@ -183,7 +183,7 @@ static int da8xx_rproc_stop(struct rproc *rproc)
 /* kick a virtqueue */
 static void da8xx_rproc_kick(struct rproc *rproc, int vqid)
 {
-	struct da8xx_rproc *drproc = (struct da8xx_rproc *)rproc->priv;
+	struct da8xx_rproc *drproc = rproc->priv;
 
 	/* Interrupt remote proc */
 	writel(SYSCFG_CHIPSIG2, drproc->chipsig);
@@ -360,7 +360,7 @@ free_mem:
 static int da8xx_rproc_remove(struct platform_device *pdev)
 {
 	struct rproc *rproc = platform_get_drvdata(pdev);
-	struct da8xx_rproc *drproc = (struct da8xx_rproc *)rproc->priv;
+	struct da8xx_rproc *drproc = rproc->priv;
 	struct device *dev = &pdev->dev;
 
 	/*
diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c
index 95da1cbefacf..cab06dbf37fb 100644
--- a/drivers/remoteproc/imx_dsp_rproc.c
+++ b/drivers/remoteproc/imx_dsp_rproc.c
@@ -28,6 +28,14 @@
 
 #define DSP_RPROC_CLK_MAX			5
 
+/*
+ * Module parameters
+ */
+static unsigned int no_mailboxes;
+module_param_named(no_mailboxes, no_mailboxes, int, 0644);
+MODULE_PARM_DESC(no_mailboxes,
+		 "There is no mailbox between cores, so ignore remote proc reply after start, default is 0 (off).");
+
 #define REMOTE_IS_READY				BIT(0)
 #define REMOTE_READY_WAIT_MAX_RETRIES		500
 
@@ -172,6 +180,9 @@ static const struct imx_rproc_att imx_dsp_rproc_att_imx8ulp[] = {
 	{ 0x30000000, 0x90000000, 0x10000000, 0},
 };
 
+/* Initialize the mailboxes between cores, if exists */
+static int (*imx_dsp_rproc_mbox_init)(struct imx_dsp_rproc *priv);
+
 /* Reset function for DSP on i.MX8MP */
 static int imx8mp_dsp_reset(struct imx_dsp_rproc *priv)
 {
@@ -492,12 +503,12 @@ static void imx_dsp_rproc_rxdb_callback(struct mbox_client *cl, void *data)
 }
 
 /**
- * imx_dsp_rproc_mbox_init() - request mailbox channels
+ * imx_dsp_rproc_mbox_alloc() - request mailbox channels
  * @priv: private data pointer
  *
  * Request three mailbox channels (tx, rx, rxdb).
  */
-static int imx_dsp_rproc_mbox_init(struct imx_dsp_rproc *priv)
+static int imx_dsp_rproc_mbox_alloc(struct imx_dsp_rproc *priv)
 {
 	struct device *dev = priv->rproc->dev.parent;
 	struct mbox_client *cl;
@@ -519,7 +530,7 @@ static int imx_dsp_rproc_mbox_init(struct imx_dsp_rproc *priv)
 		ret = PTR_ERR(priv->tx_ch);
 		dev_dbg(cl->dev, "failed to request tx mailbox channel: %d\n",
 			ret);
-		goto err_out;
+		return ret;
 	}
 
 	/* Channel for receiving message */
@@ -528,7 +539,7 @@ static int imx_dsp_rproc_mbox_init(struct imx_dsp_rproc *priv)
 		ret = PTR_ERR(priv->rx_ch);
 		dev_dbg(cl->dev, "failed to request rx mailbox channel: %d\n",
 			ret);
-		goto err_out;
+		goto free_channel_tx;
 	}
 
 	cl = &priv->cl_rxdb;
@@ -544,22 +555,30 @@ static int imx_dsp_rproc_mbox_init(struct imx_dsp_rproc *priv)
 		ret = PTR_ERR(priv->rxdb_ch);
 		dev_dbg(cl->dev, "failed to request mbox chan rxdb, ret %d\n",
 			ret);
-		goto err_out;
+		goto free_channel_rx;
 	}
 
 	return 0;
 
-err_out:
-	if (!IS_ERR(priv->tx_ch))
-		mbox_free_channel(priv->tx_ch);
-	if (!IS_ERR(priv->rx_ch))
-		mbox_free_channel(priv->rx_ch);
-	if (!IS_ERR(priv->rxdb_ch))
-		mbox_free_channel(priv->rxdb_ch);
-
+free_channel_rx:
+	mbox_free_channel(priv->rx_ch);
+free_channel_tx:
+	mbox_free_channel(priv->tx_ch);
 	return ret;
 }
 
+/*
+ * imx_dsp_rproc_mbox_no_alloc()
+ *
+ * Empty function for no mailbox between cores
+ *
+ * Always return 0
+ */
+static int imx_dsp_rproc_mbox_no_alloc(struct imx_dsp_rproc *priv)
+{
+	return 0;
+}
+
 static void imx_dsp_rproc_free_mbox(struct imx_dsp_rproc *priv)
 {
 	mbox_free_channel(priv->tx_ch);
@@ -627,15 +646,19 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv)
 
 		rmem = of_reserved_mem_lookup(it.node);
 		if (!rmem) {
+			of_node_put(it.node);
 			dev_err(dev, "unable to acquire memory-region\n");
 			return -EINVAL;
 		}
 
-		if (imx_dsp_rproc_sys_to_da(priv, rmem->base, rmem->size, &da))
+		if (imx_dsp_rproc_sys_to_da(priv, rmem->base, rmem->size, &da)) {
+			of_node_put(it.node);
 			return -EINVAL;
+		}
 
 		cpu_addr = devm_ioremap_wc(dev, rmem->base, rmem->size);
 		if (!cpu_addr) {
+			of_node_put(it.node);
 			dev_err(dev, "failed to map memory %p\n", &rmem->base);
 			return -ENOMEM;
 		}
@@ -644,10 +667,12 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv)
 		mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)rmem->base,
 					   rmem->size, da, NULL, NULL, it.node->name);
 
-		if (mem)
+		if (mem) {
 			rproc_coredump_add_segment(rproc, da, rmem->size);
-		else
+		} else {
+			of_node_put(it.node);
 			return -ENOMEM;
+		}
 
 		rproc_add_carveout(rproc, mem);
 	}
@@ -715,6 +740,191 @@ static void imx_dsp_rproc_kick(struct rproc *rproc, int vqid)
 		dev_err(dev, "%s: failed (%d, err:%d)\n", __func__, vqid, err);
 }
 
+/*
+ * Custom memory copy implementation for i.MX DSP Cores
+ *
+ * The IRAM is part of the HiFi DSP.
+ * According to hw specs only 32-bits writes are allowed.
+ */
+static int imx_dsp_rproc_memcpy(void *dst, const void *src, size_t size)
+{
+	void __iomem *dest = (void __iomem *)dst;
+	const u8 *src_byte = src;
+	const u32 *source = src;
+	u32 affected_mask;
+	int i, q, r;
+	u32 tmp;
+
+	/* destination must be 32bit aligned */
+	if (!IS_ALIGNED((uintptr_t)dest, 4))
+		return -EINVAL;
+
+	q = size / 4;
+	r = size % 4;
+
+	/* copy data in units of 32 bits at a time */
+	for (i = 0; i < q; i++)
+		writel(source[i], dest + i * 4);
+
+	if (r) {
+		affected_mask = GENMASK(8 * r, 0);
+
+		/*
+		 * first read the 32bit data of dest, then change affected
+		 * bytes, and write back to dest.
+		 * For unaffected bytes, it should not be changed
+		 */
+		tmp = readl(dest + q * 4);
+		tmp &= ~affected_mask;
+
+		/* avoid reading after end of source */
+		for (i = 0; i < r; i++)
+			tmp |= (src_byte[q * 4 + i] << (8 * i));
+
+		writel(tmp, dest + q * 4);
+	}
+
+	return 0;
+}
+
+/*
+ * Custom memset implementation for i.MX DSP Cores
+ *
+ * The IRAM is part of the HiFi DSP.
+ * According to hw specs only 32-bits writes are allowed.
+ */
+static int imx_dsp_rproc_memset(void *addr, u8 value, size_t size)
+{
+	void __iomem *tmp_dst = (void __iomem *)addr;
+	u32 tmp_val = value;
+	u32 affected_mask;
+	int q, r;
+	u32 tmp;
+
+	/* destination must be 32bit aligned */
+	if (!IS_ALIGNED((uintptr_t)addr, 4))
+		return -EINVAL;
+
+	tmp_val |= tmp_val << 8;
+	tmp_val |= tmp_val << 16;
+
+	q = size / 4;
+	r = size % 4;
+
+	while (q--)
+		writel(tmp_val, tmp_dst++);
+
+	if (r) {
+		affected_mask = GENMASK(8 * r, 0);
+
+		/*
+		 * first read the 32bit data of addr, then change affected
+		 * bytes, and write back to addr.
+		 * For unaffected bytes, it should not be changed
+		 */
+		tmp = readl(tmp_dst);
+		tmp &= ~affected_mask;
+
+		tmp |= (tmp_val & affected_mask);
+		writel(tmp, tmp_dst);
+	}
+
+	return 0;
+}
+
+/*
+ * imx_dsp_rproc_elf_load_segments() - load firmware segments to memory
+ * @rproc: remote processor which will be booted using these fw segments
+ * @fw: the ELF firmware image
+ *
+ * This function loads the firmware segments to memory, where the remote
+ * processor expects them.
+ *
+ * Return: 0 on success and an appropriate error code otherwise
+ */
+static int imx_dsp_rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)
+{
+	struct device *dev = &rproc->dev;
+	const void *ehdr, *phdr;
+	int i, ret = 0;
+	u16 phnum;
+	const u8 *elf_data = fw->data;
+	u8 class = fw_elf_get_class(fw);
+	u32 elf_phdr_get_size = elf_size_of_phdr(class);
+
+	ehdr = elf_data;
+	phnum = elf_hdr_get_e_phnum(class, ehdr);
+	phdr = elf_data + elf_hdr_get_e_phoff(class, ehdr);
+
+	/* go through the available ELF segments */
+	for (i = 0; i < phnum; i++, phdr += elf_phdr_get_size) {
+		u64 da = elf_phdr_get_p_paddr(class, phdr);
+		u64 memsz = elf_phdr_get_p_memsz(class, phdr);
+		u64 filesz = elf_phdr_get_p_filesz(class, phdr);
+		u64 offset = elf_phdr_get_p_offset(class, phdr);
+		u32 type = elf_phdr_get_p_type(class, phdr);
+		void *ptr;
+
+		if (type != PT_LOAD || !memsz)
+			continue;
+
+		dev_dbg(dev, "phdr: type %d da 0x%llx memsz 0x%llx filesz 0x%llx\n",
+			type, da, memsz, filesz);
+
+		if (filesz > memsz) {
+			dev_err(dev, "bad phdr filesz 0x%llx memsz 0x%llx\n",
+				filesz, memsz);
+			ret = -EINVAL;
+			break;
+		}
+
+		if (offset + filesz > fw->size) {
+			dev_err(dev, "truncated fw: need 0x%llx avail 0x%zx\n",
+				offset + filesz, fw->size);
+			ret = -EINVAL;
+			break;
+		}
+
+		if (!rproc_u64_fit_in_size_t(memsz)) {
+			dev_err(dev, "size (%llx) does not fit in size_t type\n",
+				memsz);
+			ret = -EOVERFLOW;
+			break;
+		}
+
+		/* grab the kernel address for this device address */
+		ptr = rproc_da_to_va(rproc, da, memsz, NULL);
+		if (!ptr) {
+			dev_err(dev, "bad phdr da 0x%llx mem 0x%llx\n", da,
+				memsz);
+			ret = -EINVAL;
+			break;
+		}
+
+		/* put the segment where the remote processor expects it */
+		if (filesz) {
+			ret = imx_dsp_rproc_memcpy(ptr, elf_data + offset, filesz);
+			if (ret) {
+				dev_err(dev, "memory copy failed for da 0x%llx memsz 0x%llx\n",
+					da, memsz);
+				break;
+			}
+		}
+
+		/* zero out remaining memory for this segment */
+		if (memsz > filesz) {
+			ret = imx_dsp_rproc_memset(ptr + filesz, 0, memsz - filesz);
+			if (ret) {
+				dev_err(dev, "memset failed for da 0x%llx memsz 0x%llx\n",
+					da, memsz);
+				break;
+			}
+		}
+	}
+
+	return ret;
+}
+
 static int imx_dsp_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
 {
 	if (rproc_elf_load_rsc_table(rproc, fw))
@@ -729,7 +939,7 @@ static const struct rproc_ops imx_dsp_rproc_ops = {
 	.start		= imx_dsp_rproc_start,
 	.stop		= imx_dsp_rproc_stop,
 	.kick		= imx_dsp_rproc_kick,
-	.load		= rproc_elf_load_segments,
+	.load		= imx_dsp_rproc_elf_load_segments,
 	.parse_fw	= imx_dsp_rproc_parse_fw,
 	.sanity_check	= rproc_elf_sanity_check,
 	.get_boot_addr	= rproc_elf_get_boot_addr,
@@ -903,6 +1113,11 @@ static int imx_dsp_rproc_probe(struct platform_device *pdev)
 	priv->rproc = rproc;
 	priv->dsp_dcfg = dsp_dcfg;
 
+	if (no_mailboxes)
+		imx_dsp_rproc_mbox_init = imx_dsp_rproc_mbox_no_alloc;
+	else
+		imx_dsp_rproc_mbox_init = imx_dsp_rproc_mbox_alloc;
+
 	dev_set_drvdata(dev, rproc);
 
 	INIT_WORK(&priv->rproc_work, imx_dsp_rproc_vq_work);
diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c
index 9fc978e0393c..0ab840dc7e97 100644
--- a/drivers/remoteproc/imx_rproc.c
+++ b/drivers/remoteproc/imx_rproc.c
@@ -541,6 +541,7 @@ static int imx_rproc_prepare(struct rproc *rproc)
 
 		rmem = of_reserved_mem_lookup(it.node);
 		if (!rmem) {
+			of_node_put(it.node);
 			dev_err(priv->dev, "unable to acquire memory-region\n");
 			return -EINVAL;
 		}
@@ -553,10 +554,12 @@ static int imx_rproc_prepare(struct rproc *rproc)
 					   imx_rproc_mem_alloc, imx_rproc_mem_release,
 					   it.node->name);
 
-		if (mem)
+		if (mem) {
 			rproc_coredump_add_segment(rproc, da, rmem->size);
-		else
+		} else {
+			of_node_put(it.node);
 			return -ENOMEM;
+		}
 
 		rproc_add_carveout(rproc, mem);
 	}
diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c
index 0861b76f185f..e1d93e63d7df 100644
--- a/drivers/remoteproc/mtk_scp.c
+++ b/drivers/remoteproc/mtk_scp.c
@@ -74,8 +74,8 @@ static void scp_wdt_handler(struct mtk_scp *scp, u32 scp_to_host)
 
 static void scp_init_ipi_handler(void *data, unsigned int len, void *priv)
 {
-	struct mtk_scp *scp = (struct mtk_scp *)priv;
-	struct scp_run *run = (struct scp_run *)data;
+	struct mtk_scp *scp = priv;
+	struct scp_run *run = data;
 
 	scp->run.signaled = run->signaled;
 	strscpy(scp->run.fw_ver, run->fw_ver, SCP_FW_VER_LEN);
@@ -498,7 +498,7 @@ static int scp_parse_fw(struct rproc *rproc, const struct firmware *fw)
 
 static int scp_start(struct rproc *rproc)
 {
-	struct mtk_scp *scp = (struct mtk_scp *)rproc->priv;
+	struct mtk_scp *scp = rproc->priv;
 	struct device *dev = scp->dev;
 	struct scp_run *run = &scp->run;
 	int ret;
@@ -587,7 +587,7 @@ static void *mt8192_scp_da_to_va(struct mtk_scp *scp, u64 da, size_t len)
 
 static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len, bool *is_iomem)
 {
-	struct mtk_scp *scp = (struct mtk_scp *)rproc->priv;
+	struct mtk_scp *scp = rproc->priv;
 
 	return scp->data->scp_da_to_va(scp, da, len);
 }
@@ -627,7 +627,7 @@ static void mt8195_scp_stop(struct mtk_scp *scp)
 
 static int scp_stop(struct rproc *rproc)
 {
-	struct mtk_scp *scp = (struct mtk_scp *)rproc->priv;
+	struct mtk_scp *scp = rproc->priv;
 	int ret;
 
 	ret = clk_prepare_enable(scp->clk);
@@ -829,7 +829,7 @@ static int scp_probe(struct platform_device *pdev)
 	if (!rproc)
 		return dev_err_probe(dev, -ENOMEM, "unable to allocate remoteproc\n");
 
-	scp = (struct mtk_scp *)rproc->priv;
+	scp = rproc->priv;
 	scp->rproc = rproc;
 	scp->dev = dev;
 	scp->data = of_device_get_match_data(dev);
diff --git a/drivers/remoteproc/mtk_scp_ipi.c b/drivers/remoteproc/mtk_scp_ipi.c
index fc55df649b40..9c7c17b9d181 100644
--- a/drivers/remoteproc/mtk_scp_ipi.c
+++ b/drivers/remoteproc/mtk_scp_ipi.c
@@ -125,7 +125,7 @@ void scp_ipi_lock(struct mtk_scp *scp, u32 id)
 EXPORT_SYMBOL_GPL(scp_ipi_lock);
 
 /**
- * scp_ipi_lock() - Unlock after operations of an IPI ID
+ * scp_ipi_unlock() - Unlock after operations of an IPI ID
  *
  * @scp:	mtk_scp structure
  * @id:		IPI ID
diff --git a/drivers/remoteproc/pru_rproc.c b/drivers/remoteproc/pru_rproc.c
index b76db7fa693d..095f66130f48 100644
--- a/drivers/remoteproc/pru_rproc.c
+++ b/drivers/remoteproc/pru_rproc.c
@@ -657,7 +657,7 @@ static void *pru_d_da_to_va(struct pru_rproc *pru, u32 da, size_t len)
 		swap(dram0, dram1);
 	shrd_ram = pruss->mem_regions[PRUSS_MEM_SHRD_RAM2];
 
-	if (da >= PRU_PDRAM_DA && da + len <= PRU_PDRAM_DA + dram0.size) {
+	if (da + len <= PRU_PDRAM_DA + dram0.size) {
 		offset = da - PRU_PDRAM_DA;
 		va = (__force void *)(dram0.va + offset);
 	} else if (da >= PRU_SDRAM_DA &&
@@ -706,8 +706,7 @@ static void *pru_i_da_to_va(struct pru_rproc *pru, u32 da, size_t len)
 	 */
 	da &= 0xfffff;
 
-	if (da >= PRU_IRAM_DA &&
-	    da + len <= PRU_IRAM_DA + pru->mem_regions[PRU_IOMEM_IRAM].size) {
+	if (da + len <= PRU_IRAM_DA + pru->mem_regions[PRU_IOMEM_IRAM].size) {
 		offset = da - PRU_IRAM_DA;
 		va = (__force void *)(pru->mem_regions[PRU_IOMEM_IRAM].va +
 				      offset);
diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c
index 08d8dad22ca7..d546ab9dc141 100644
--- a/drivers/remoteproc/qcom_q6v5_adsp.c
+++ b/drivers/remoteproc/qcom_q6v5_adsp.c
@@ -321,7 +321,7 @@ reset:
 
 static int adsp_load(struct rproc *rproc, const struct firmware *fw)
 {
-	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	struct qcom_adsp *adsp = rproc->priv;
 	int ret;
 
 	ret = qcom_mdt_load_no_init(adsp->dev, fw, rproc->firmware, 0,
@@ -379,7 +379,7 @@ static int adsp_map_carveout(struct rproc *rproc)
 
 static int adsp_start(struct rproc *rproc)
 {
-	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	struct qcom_adsp *adsp = rproc->priv;
 	int ret;
 	unsigned int val;
 
@@ -469,7 +469,7 @@ static void qcom_adsp_pil_handover(struct qcom_q6v5 *q6v5)
 
 static int adsp_stop(struct rproc *rproc)
 {
-	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	struct qcom_adsp *adsp = rproc->priv;
 	int handover;
 	int ret;
 
@@ -492,7 +492,7 @@ static int adsp_stop(struct rproc *rproc)
 
 static void *adsp_da_to_va(struct rproc *rproc, u64 da, size_t len, bool *is_iomem)
 {
-	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	struct qcom_adsp *adsp = rproc->priv;
 	int offset;
 
 	offset = da - adsp->mem_reloc;
@@ -696,7 +696,7 @@ static int adsp_probe(struct platform_device *pdev)
 	rproc->has_iommu = desc->has_iommu;
 	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE);
 
-	adsp = (struct qcom_adsp *)rproc->priv;
+	adsp = rproc->priv;
 	adsp->dev = &pdev->dev;
 	adsp->rproc = rproc;
 	adsp->info_name = desc->sysmon_name;
diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c
index ab053084f7a2..8e15e4f85de1 100644
--- a/drivers/remoteproc/qcom_q6v5_mss.c
+++ b/drivers/remoteproc/qcom_q6v5_mss.c
@@ -235,8 +235,8 @@ struct q6v5 {
 	bool has_qaccept_regs;
 	bool has_ext_cntl_regs;
 	bool has_vq6;
-	int mpss_perm;
-	int mba_perm;
+	u64 mpss_perm;
+	u64 mba_perm;
 	const char *hexagon_mdt_image;
 	int version;
 };
@@ -414,7 +414,7 @@ static void q6v5_pds_disable(struct q6v5 *qproc, struct device **pds,
 	}
 }
 
-static int q6v5_xfer_mem_ownership(struct q6v5 *qproc, int *current_perm,
+static int q6v5_xfer_mem_ownership(struct q6v5 *qproc, u64 *current_perm,
 				   bool local, bool remote, phys_addr_t addr,
 				   size_t size)
 {
@@ -967,7 +967,7 @@ static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw,
 	unsigned long dma_attrs = DMA_ATTR_FORCE_CONTIGUOUS;
 	dma_addr_t phys;
 	void *metadata;
-	int mdata_perm;
+	u64 mdata_perm;
 	int xferop_ret;
 	size_t size;
 	void *ptr;
@@ -1562,7 +1562,7 @@ static void qcom_q6v5_dump_segment(struct rproc *rproc,
 
 static int q6v5_start(struct rproc *rproc)
 {
-	struct q6v5 *qproc = (struct q6v5 *)rproc->priv;
+	struct q6v5 *qproc = rproc->priv;
 	int xfermemop_ret;
 	int ret;
 
@@ -1604,7 +1604,7 @@ reclaim_mpss:
 
 static int q6v5_stop(struct rproc *rproc)
 {
-	struct q6v5 *qproc = (struct q6v5 *)rproc->priv;
+	struct q6v5 *qproc = rproc->priv;
 	int ret;
 
 	ret = qcom_q6v5_request_stop(&qproc->q6v5, qproc->sysmon);
@@ -1662,7 +1662,7 @@ static int qcom_q6v5_register_dump_segments(struct rproc *rproc,
 
 static unsigned long q6v5_panic(struct rproc *rproc)
 {
-	struct q6v5 *qproc = (struct q6v5 *)rproc->priv;
+	struct q6v5 *qproc = rproc->priv;
 
 	return qcom_q6v5_panic(&qproc->q6v5);
 }
@@ -1977,7 +1977,7 @@ static int q6v5_probe(struct platform_device *pdev)
 	rproc->auto_boot = false;
 	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE);
 
-	qproc = (struct q6v5 *)rproc->priv;
+	qproc = rproc->priv;
 	qproc->dev = &pdev->dev;
 	qproc->rproc = rproc;
 	qproc->hexagon_mdt_image = "modem.mdt";
diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c
index 0871108fb4dc..e34d82b18a67 100644
--- a/drivers/remoteproc/qcom_q6v5_pas.c
+++ b/drivers/remoteproc/qcom_q6v5_pas.c
@@ -94,7 +94,7 @@ struct qcom_adsp {
 	size_t region_assign_size;
 
 	int region_assign_idx;
-	int region_assign_perms;
+	u64 region_assign_perms;
 
 	struct qcom_rproc_glink glink_subdev;
 	struct qcom_rproc_subdev smd_subdev;
@@ -186,7 +186,7 @@ static int adsp_shutdown_poll_decrypt(struct qcom_adsp *adsp)
 
 static int adsp_unprepare(struct rproc *rproc)
 {
-	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	struct qcom_adsp *adsp = rproc->priv;
 
 	/*
 	 * adsp_load() did pass pas_metadata to the SCM driver for storing
@@ -203,7 +203,7 @@ static int adsp_unprepare(struct rproc *rproc)
 
 static int adsp_load(struct rproc *rproc, const struct firmware *fw)
 {
-	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	struct qcom_adsp *adsp = rproc->priv;
 	int ret;
 
 	/* Store firmware handle to be used in adsp_start() */
@@ -244,7 +244,7 @@ release_dtb_firmware:
 
 static int adsp_start(struct rproc *rproc)
 {
-	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	struct qcom_adsp *adsp = rproc->priv;
 	int ret;
 
 	ret = qcom_q6v5_prepare(&adsp->q6v5);
@@ -360,7 +360,7 @@ static void qcom_pas_handover(struct qcom_q6v5 *q6v5)
 
 static int adsp_stop(struct rproc *rproc)
 {
-	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	struct qcom_adsp *adsp = rproc->priv;
 	int handover;
 	int ret;
 
@@ -390,7 +390,7 @@ static int adsp_stop(struct rproc *rproc)
 
 static void *adsp_da_to_va(struct rproc *rproc, u64 da, size_t len, bool *is_iomem)
 {
-	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	struct qcom_adsp *adsp = rproc->priv;
 	int offset;
 
 	offset = da - adsp->mem_reloc;
@@ -405,7 +405,7 @@ static void *adsp_da_to_va(struct rproc *rproc, u64 da, size_t len, bool *is_iom
 
 static unsigned long adsp_panic(struct rproc *rproc)
 {
-	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	struct qcom_adsp *adsp = rproc->priv;
 
 	return qcom_q6v5_panic(&adsp->q6v5);
 }
@@ -683,7 +683,7 @@ static int adsp_probe(struct platform_device *pdev)
 	rproc->auto_boot = desc->auto_boot;
 	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE);
 
-	adsp = (struct qcom_adsp *)rproc->priv;
+	adsp = rproc->priv;
 	adsp->dev = &pdev->dev;
 	adsp->rproc = rproc;
 	adsp->minidump_id = desc->minidump_id;
diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c
index 9d4d04fff8c6..0fc317265064 100644
--- a/drivers/remoteproc/qcom_wcnss.c
+++ b/drivers/remoteproc/qcom_wcnss.c
@@ -154,7 +154,7 @@ static const struct wcnss_data pronto_v3_data = {
 
 static int wcnss_load(struct rproc *rproc, const struct firmware *fw)
 {
-	struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv;
+	struct qcom_wcnss *wcnss = rproc->priv;
 	int ret;
 
 	ret = qcom_mdt_load(wcnss->dev, fw, rproc->firmware, WCNSS_PAS_ID,
@@ -227,7 +227,7 @@ static void wcnss_configure_iris(struct qcom_wcnss *wcnss)
 
 static int wcnss_start(struct rproc *rproc)
 {
-	struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv;
+	struct qcom_wcnss *wcnss = rproc->priv;
 	int ret, i;
 
 	mutex_lock(&wcnss->iris_lock);
@@ -293,7 +293,7 @@ release_iris_lock:
 
 static int wcnss_stop(struct rproc *rproc)
 {
-	struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv;
+	struct qcom_wcnss *wcnss = rproc->priv;
 	int ret;
 
 	if (wcnss->state) {
@@ -320,7 +320,7 @@ static int wcnss_stop(struct rproc *rproc)
 
 static void *wcnss_da_to_va(struct rproc *rproc, u64 da, size_t len, bool *is_iomem)
 {
-	struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv;
+	struct qcom_wcnss *wcnss = rproc->priv;
 	int offset;
 
 	offset = da - wcnss->mem_reloc;
@@ -566,7 +566,7 @@ static int wcnss_probe(struct platform_device *pdev)
 	}
 	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE);
 
-	wcnss = (struct qcom_wcnss *)rproc->priv;
+	wcnss = rproc->priv;
 	wcnss->dev = &pdev->dev;
 	wcnss->rproc = rproc;
 	platform_set_drvdata(pdev, wcnss);
diff --git a/drivers/remoteproc/rcar_rproc.c b/drivers/remoteproc/rcar_rproc.c
index aa86154109c7..1ff2a73ade90 100644
--- a/drivers/remoteproc/rcar_rproc.c
+++ b/drivers/remoteproc/rcar_rproc.c
@@ -62,13 +62,16 @@ static int rcar_rproc_prepare(struct rproc *rproc)
 
 		rmem = of_reserved_mem_lookup(it.node);
 		if (!rmem) {
+			of_node_put(it.node);
 			dev_err(&rproc->dev,
 				"unable to acquire memory-region\n");
 			return -EINVAL;
 		}
 
-		if (rmem->base > U32_MAX)
+		if (rmem->base > U32_MAX) {
+			of_node_put(it.node);
 			return -EINVAL;
+		}
 
 		/* No need to translate pa to da, R-Car use same map */
 		da = rmem->base;
@@ -79,8 +82,10 @@ static int rcar_rproc_prepare(struct rproc *rproc)
 					   rcar_rproc_mem_release,
 					   it.node->name);
 
-		if (!mem)
+		if (!mem) {
+			of_node_put(it.node);
 			return -ENOMEM;
+		}
 
 		rproc_add_carveout(rproc, mem);
 	}
diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
index 80072b6b6283..695cce218e8c 100644
--- a/drivers/remoteproc/remoteproc_core.c
+++ b/drivers/remoteproc/remoteproc_core.c
@@ -2766,5 +2766,4 @@ static void __exit remoteproc_exit(void)
 }
 module_exit(remoteproc_exit);
 
-MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("Generic Remote Processor Framework");
diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c
index a3268d95a50e..3f1b8963639f 100644
--- a/drivers/remoteproc/st_remoteproc.c
+++ b/drivers/remoteproc/st_remoteproc.c
@@ -129,6 +129,7 @@ static int st_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
 	while (of_phandle_iterator_next(&it) == 0) {
 		rmem = of_reserved_mem_lookup(it.node);
 		if (!rmem) {
+			of_node_put(it.node);
 			dev_err(dev, "unable to acquire memory-region\n");
 			return -EINVAL;
 		}
@@ -150,8 +151,10 @@ static int st_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
 							   it.node->name);
 		}
 
-		if (!mem)
+		if (!mem) {
+			of_node_put(it.node);
 			return -ENOMEM;
+		}
 
 		rproc_add_carveout(rproc, mem);
 		index++;
@@ -379,7 +382,7 @@ static int st_rproc_probe(struct platform_device *pdev)
 		clk_set_rate(ddata->clk, ddata->clk_rate);
 	}
 
-	if (of_get_property(np, "mbox-names", NULL)) {
+	if (of_property_present(np, "mbox-names")) {
 		ddata->mbox_client_vq0.dev		= dev;
 		ddata->mbox_client_vq0.tx_done		= NULL;
 		ddata->mbox_client_vq0.tx_block	= false;
diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c
index 7d782ed9e589..8746cbb1f168 100644
--- a/drivers/remoteproc/stm32_rproc.c
+++ b/drivers/remoteproc/stm32_rproc.c
@@ -223,11 +223,13 @@ static int stm32_rproc_prepare(struct rproc *rproc)
 	while (of_phandle_iterator_next(&it) == 0) {
 		rmem = of_reserved_mem_lookup(it.node);
 		if (!rmem) {
+			of_node_put(it.node);
 			dev_err(dev, "unable to acquire memory-region\n");
 			return -EINVAL;
 		}
 
 		if (stm32_rproc_pa_to_da(rproc, rmem->base, &da) < 0) {
+			of_node_put(it.node);
 			dev_err(dev, "memory region not valid %pa\n",
 				&rmem->base);
 			return -EINVAL;
@@ -254,8 +256,10 @@ static int stm32_rproc_prepare(struct rproc *rproc)
 							   it.node->name);
 		}
 
-		if (!mem)
+		if (!mem) {
+			of_node_put(it.node);
 			return -ENOMEM;
+		}
 
 		rproc_add_carveout(rproc, mem);
 		index++;
@@ -287,8 +291,16 @@ static void stm32_rproc_mb_vq_work(struct work_struct *work)
 	struct stm32_mbox *mb = container_of(work, struct stm32_mbox, vq_work);
 	struct rproc *rproc = dev_get_drvdata(mb->client.dev);
 
+	mutex_lock(&rproc->lock);
+
+	if (rproc->state != RPROC_RUNNING)
+		goto unlock_mutex;
+
 	if (rproc_vq_interrupt(rproc, mb->vq_id) == IRQ_NONE)
 		dev_dbg(&rproc->dev, "no message found in vq%d\n", mb->vq_id);
+
+unlock_mutex:
+	mutex_unlock(&rproc->lock);
 }
 
 static void stm32_rproc_mb_callback(struct mbox_client *cl, void *data)
diff --git a/drivers/remoteproc/ti_k3_r5_remoteproc.c b/drivers/remoteproc/ti_k3_r5_remoteproc.c
index 0481926c6975..23fe44d4d7a5 100644
--- a/drivers/remoteproc/ti_k3_r5_remoteproc.c
+++ b/drivers/remoteproc/ti_k3_r5_remoteproc.c
@@ -71,14 +71,16 @@ struct k3_r5_mem {
 /*
  * All cluster mode values are not applicable on all SoCs. The following
  * are the modes supported on various SoCs:
- *   Split mode      : AM65x, J721E, J7200 and AM64x SoCs
- *   LockStep mode   : AM65x, J721E and J7200 SoCs
- *   Single-CPU mode : AM64x SoCs only
+ *   Split mode       : AM65x, J721E, J7200 and AM64x SoCs
+ *   LockStep mode    : AM65x, J721E and J7200 SoCs
+ *   Single-CPU mode  : AM64x SoCs only
+ *   Single-Core mode : AM62x, AM62A SoCs
  */
 enum cluster_mode {
 	CLUSTER_MODE_SPLIT = 0,
 	CLUSTER_MODE_LOCKSTEP,
 	CLUSTER_MODE_SINGLECPU,
+	CLUSTER_MODE_SINGLECORE
 };
 
 /**
@@ -86,11 +88,13 @@ enum cluster_mode {
  * @tcm_is_double: flag to denote the larger unified TCMs in certain modes
  * @tcm_ecc_autoinit: flag to denote the auto-initialization of TCMs for ECC
  * @single_cpu_mode: flag to denote if SoC/IP supports Single-CPU mode
+ * @is_single_core: flag to denote if SoC/IP has only single core R5
  */
 struct k3_r5_soc_data {
 	bool tcm_is_double;
 	bool tcm_ecc_autoinit;
 	bool single_cpu_mode;
+	bool is_single_core;
 };
 
 /**
@@ -838,7 +842,8 @@ static int k3_r5_rproc_configure(struct k3_r5_rproc *kproc)
 
 	core0 = list_first_entry(&cluster->cores, struct k3_r5_core, elem);
 	if (cluster->mode == CLUSTER_MODE_LOCKSTEP ||
-	    cluster->mode == CLUSTER_MODE_SINGLECPU) {
+	    cluster->mode == CLUSTER_MODE_SINGLECPU ||
+	    cluster->mode == CLUSTER_MODE_SINGLECORE) {
 		core = core0;
 	} else {
 		core = kproc->core;
@@ -852,38 +857,34 @@ static int k3_r5_rproc_configure(struct k3_r5_rproc *kproc)
 	dev_dbg(dev, "boot_vector = 0x%llx, cfg = 0x%x ctrl = 0x%x stat = 0x%x\n",
 		boot_vec, cfg, ctrl, stat);
 
-	/* check if only Single-CPU mode is supported on applicable SoCs */
-	if (cluster->soc_data->single_cpu_mode) {
-		single_cpu =
-			!!(stat & PROC_BOOT_STATUS_FLAG_R5_SINGLECORE_ONLY);
-		if (single_cpu && cluster->mode == CLUSTER_MODE_SPLIT) {
-			dev_err(cluster->dev, "split-mode not permitted, force configuring for single-cpu mode\n");
-			cluster->mode = CLUSTER_MODE_SINGLECPU;
-		}
-		goto config;
+	single_cpu = !!(stat & PROC_BOOT_STATUS_FLAG_R5_SINGLECORE_ONLY);
+	lockstep_en = !!(stat & PROC_BOOT_STATUS_FLAG_R5_LOCKSTEP_PERMITTED);
+
+	/* Override to single CPU mode if set in status flag */
+	if (single_cpu && cluster->mode == CLUSTER_MODE_SPLIT) {
+		dev_err(cluster->dev, "split-mode not permitted, force configuring for single-cpu mode\n");
+		cluster->mode = CLUSTER_MODE_SINGLECPU;
 	}
 
-	/* check conventional LockStep vs Split mode configuration */
-	lockstep_en = !!(stat & PROC_BOOT_STATUS_FLAG_R5_LOCKSTEP_PERMITTED);
+	/* Override to split mode if lockstep enable bit is not set in status flag */
 	if (!lockstep_en && cluster->mode == CLUSTER_MODE_LOCKSTEP) {
 		dev_err(cluster->dev, "lockstep mode not permitted, force configuring for split-mode\n");
 		cluster->mode = CLUSTER_MODE_SPLIT;
 	}
 
-config:
 	/* always enable ARM mode and set boot vector to 0 */
 	boot_vec = 0x0;
 	if (core == core0) {
 		clr_cfg = PROC_BOOT_CFG_FLAG_R5_TEINIT;
-		if (cluster->soc_data->single_cpu_mode) {
-			/*
-			 * Single-CPU configuration bit can only be configured
-			 * on Core0 and system firmware will NACK any requests
-			 * with the bit configured, so program it only on
-			 * permitted cores
-			 */
-			if (cluster->mode == CLUSTER_MODE_SINGLECPU)
-				set_cfg = PROC_BOOT_CFG_FLAG_R5_SINGLE_CORE;
+		/*
+		 * Single-CPU configuration bit can only be configured
+		 * on Core0 and system firmware will NACK any requests
+		 * with the bit configured, so program it only on
+		 * permitted cores
+		 */
+		if (cluster->mode == CLUSTER_MODE_SINGLECPU ||
+		    cluster->mode == CLUSTER_MODE_SINGLECORE) {
+			set_cfg = PROC_BOOT_CFG_FLAG_R5_SINGLE_CORE;
 		} else {
 			/*
 			 * LockStep configuration bit is Read-only on Split-mode
@@ -1074,6 +1075,7 @@ static void k3_r5_adjust_tcm_sizes(struct k3_r5_rproc *kproc)
 
 	if (cluster->mode == CLUSTER_MODE_LOCKSTEP ||
 	    cluster->mode == CLUSTER_MODE_SINGLECPU ||
+	    cluster->mode == CLUSTER_MODE_SINGLECORE ||
 	    !cluster->soc_data->tcm_is_double)
 		return;
 
@@ -1108,12 +1110,12 @@ static int k3_r5_rproc_configure_mode(struct k3_r5_rproc *kproc)
 	struct k3_r5_cluster *cluster = kproc->cluster;
 	struct k3_r5_core *core = kproc->core;
 	struct device *cdev = core->dev;
-	bool r_state = false, c_state = false;
+	bool r_state = false, c_state = false, lockstep_en = false, single_cpu = false;
 	u32 ctrl = 0, cfg = 0, stat = 0, halted = 0;
 	u64 boot_vec = 0;
 	u32 atcm_enable, btcm_enable, loczrama;
 	struct k3_r5_core *core0;
-	enum cluster_mode mode;
+	enum cluster_mode mode = cluster->mode;
 	int ret;
 
 	core0 = list_first_entry(&cluster->cores, struct k3_r5_core, elem);
@@ -1147,13 +1149,14 @@ static int k3_r5_rproc_configure_mode(struct k3_r5_rproc *kproc)
 	atcm_enable = cfg & PROC_BOOT_CFG_FLAG_R5_ATCM_EN ?  1 : 0;
 	btcm_enable = cfg & PROC_BOOT_CFG_FLAG_R5_BTCM_EN ?  1 : 0;
 	loczrama = cfg & PROC_BOOT_CFG_FLAG_R5_TCM_RSTBASE ?  1 : 0;
-	if (cluster->soc_data->single_cpu_mode) {
-		mode = cfg & PROC_BOOT_CFG_FLAG_R5_SINGLE_CORE ?
-				CLUSTER_MODE_SINGLECPU : CLUSTER_MODE_SPLIT;
-	} else {
-		mode = cfg & PROC_BOOT_CFG_FLAG_R5_LOCKSTEP ?
-				CLUSTER_MODE_LOCKSTEP : CLUSTER_MODE_SPLIT;
-	}
+	single_cpu = cfg & PROC_BOOT_CFG_FLAG_R5_SINGLE_CORE ? 1 : 0;
+	lockstep_en = cfg & PROC_BOOT_CFG_FLAG_R5_LOCKSTEP ? 1 : 0;
+
+	if (single_cpu && mode != CLUSTER_MODE_SINGLECORE)
+		mode = CLUSTER_MODE_SINGLECPU;
+	if (lockstep_en)
+		mode = CLUSTER_MODE_LOCKSTEP;
+
 	halted = ctrl & PROC_BOOT_CTRL_FLAG_R5_CORE_HALT;
 
 	/*
@@ -1269,9 +1272,12 @@ init_rmem:
 			goto err_add;
 		}
 
-		/* create only one rproc in lockstep mode or single-cpu mode */
+		/* create only one rproc in lockstep, single-cpu or
+		 * single core mode
+		 */
 		if (cluster->mode == CLUSTER_MODE_LOCKSTEP ||
-		    cluster->mode == CLUSTER_MODE_SINGLECPU)
+		    cluster->mode == CLUSTER_MODE_SINGLECPU ||
+		    cluster->mode == CLUSTER_MODE_SINGLECORE)
 			break;
 	}
 
@@ -1700,12 +1706,6 @@ static int k3_r5_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	cluster->dev = dev;
-	/*
-	 * default to most common efuse configurations - Split-mode on AM64x
-	 * and LockStep-mode on all others
-	 */
-	cluster->mode = data->single_cpu_mode ?
-				CLUSTER_MODE_SPLIT : CLUSTER_MODE_LOCKSTEP;
 	cluster->soc_data = data;
 	INIT_LIST_HEAD(&cluster->cores);
 
@@ -1716,9 +1716,37 @@ static int k3_r5_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	if (ret == -EINVAL) {
+		/*
+		 * default to most common efuse configurations - Split-mode on AM64x
+		 * and LockStep-mode on all others
+		 * default to most common efuse configurations -
+		 * Split-mode on AM64x
+		 * Single core on AM62x
+		 * LockStep-mode on all others
+		 */
+		if (!data->is_single_core)
+			cluster->mode = data->single_cpu_mode ?
+					CLUSTER_MODE_SPLIT : CLUSTER_MODE_LOCKSTEP;
+		else
+			cluster->mode = CLUSTER_MODE_SINGLECORE;
+	}
+
+	if  ((cluster->mode == CLUSTER_MODE_SINGLECPU && !data->single_cpu_mode) ||
+	     (cluster->mode == CLUSTER_MODE_SINGLECORE && !data->is_single_core)) {
+		dev_err(dev, "Cluster mode = %d is not supported on this SoC\n", cluster->mode);
+		return -EINVAL;
+	}
+
 	num_cores = of_get_available_child_count(np);
-	if (num_cores != 2) {
-		dev_err(dev, "MCU cluster requires both R5F cores to be enabled, num_cores = %d\n",
+	if (num_cores != 2 && !data->is_single_core) {
+		dev_err(dev, "MCU cluster requires both R5F cores to be enabled but num_cores is set to = %d\n",
+			num_cores);
+		return -ENODEV;
+	}
+
+	if (num_cores != 1 && data->is_single_core) {
+		dev_err(dev, "SoC supports only single core R5 but num_cores is set to %d\n",
 			num_cores);
 		return -ENODEV;
 	}
@@ -1760,18 +1788,28 @@ static const struct k3_r5_soc_data am65_j721e_soc_data = {
 	.tcm_is_double = false,
 	.tcm_ecc_autoinit = false,
 	.single_cpu_mode = false,
+	.is_single_core = false,
 };
 
 static const struct k3_r5_soc_data j7200_j721s2_soc_data = {
 	.tcm_is_double = true,
 	.tcm_ecc_autoinit = true,
 	.single_cpu_mode = false,
+	.is_single_core = false,
 };
 
 static const struct k3_r5_soc_data am64_soc_data = {
 	.tcm_is_double = true,
 	.tcm_ecc_autoinit = true,
 	.single_cpu_mode = true,
+	.is_single_core = false,
+};
+
+static const struct k3_r5_soc_data am62_soc_data = {
+	.tcm_is_double = false,
+	.tcm_ecc_autoinit = true,
+	.single_cpu_mode = false,
+	.is_single_core = true,
 };
 
 static const struct of_device_id k3_r5_of_match[] = {
@@ -1779,6 +1817,7 @@ static const struct of_device_id k3_r5_of_match[] = {
 	{ .compatible = "ti,j721e-r5fss", .data = &am65_j721e_soc_data, },
 	{ .compatible = "ti,j7200-r5fss", .data = &j7200_j721s2_soc_data, },
 	{ .compatible = "ti,am64-r5fss",  .data = &am64_soc_data, },
+	{ .compatible = "ti,am62-r5fss",  .data = &am62_soc_data, },
 	{ .compatible = "ti,j721s2-r5fss",  .data = &j7200_j721s2_soc_data, },
 	{ /* sentinel */ },
 };
diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c
index 2db57d394155..feca6de68da2 100644
--- a/drivers/remoteproc/xlnx_r5_remoteproc.c
+++ b/drivers/remoteproc/xlnx_r5_remoteproc.c
@@ -8,16 +8,23 @@
 #include <linux/dma-mapping.h>
 #include <linux/firmware/xlnx-zynqmp.h>
 #include <linux/kernel.h>
+#include <linux/mailbox_client.h>
+#include <linux/mailbox/zynqmp-ipi-message.h>
 #include <linux/module.h>
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/of_reserved_mem.h>
 #include <linux/platform_device.h>
 #include <linux/remoteproc.h>
-#include <linux/slab.h>
 
 #include "remoteproc_internal.h"
 
+/* IPI buffer MAX length */
+#define IPI_BUF_LEN_MAX	32U
+
+/* RX mailbox client buffer max length */
+#define MBOX_CLIENT_BUF_MAX	(IPI_BUF_LEN_MAX + \
+				 sizeof(struct zynqmp_ipi_message))
 /*
  * settings for RPU cluster mode which
  * reflects possible values of xlnx,cluster-mode dt-property
@@ -43,6 +50,27 @@ struct mem_bank_data {
 	char *bank_name;
 };
 
+/**
+ * struct mbox_info
+ *
+ * @rx_mc_buf: to copy data from mailbox rx channel
+ * @tx_mc_buf: to copy data to mailbox tx channel
+ * @r5_core: this mailbox's corresponding r5_core pointer
+ * @mbox_work: schedule work after receiving data from mailbox
+ * @mbox_cl: mailbox client
+ * @tx_chan: mailbox tx channel
+ * @rx_chan: mailbox rx channel
+ */
+struct mbox_info {
+	unsigned char rx_mc_buf[MBOX_CLIENT_BUF_MAX];
+	unsigned char tx_mc_buf[MBOX_CLIENT_BUF_MAX];
+	struct zynqmp_r5_core *r5_core;
+	struct work_struct mbox_work;
+	struct mbox_client mbox_cl;
+	struct mbox_chan *tx_chan;
+	struct mbox_chan *rx_chan;
+};
+
 /*
  * Hardcoded TCM bank values. This will be removed once TCM bindings are
  * accepted for system-dt specifications and upstreamed in linux kernel
@@ -61,20 +89,18 @@ static const struct mem_bank_data zynqmp_tcm_banks[] = {
  * @np: device node of RPU instance
  * @tcm_bank_count: number TCM banks accessible to this RPU
  * @tcm_banks: array of each TCM bank data
- * @rmem_count: Number of reserved mem regions
- * @rmem: reserved memory region nodes from device tree
  * @rproc: rproc handle
  * @pm_domain_id: RPU CPU power domain id
+ * @ipi: pointer to mailbox information
  */
 struct zynqmp_r5_core {
 	struct device *dev;
 	struct device_node *np;
 	int tcm_bank_count;
 	struct mem_bank_data **tcm_banks;
-	int rmem_count;
-	struct reserved_mem **rmem;
 	struct rproc *rproc;
 	u32 pm_domain_id;
+	struct mbox_info *ipi;
 };
 
 /**
@@ -92,6 +118,178 @@ struct zynqmp_r5_cluster {
 	struct zynqmp_r5_core **r5_cores;
 };
 
+/**
+ * event_notified_idr_cb() - callback for vq_interrupt per notifyid
+ * @id: rproc->notify id
+ * @ptr: pointer to idr private data
+ * @data: data passed to idr_for_each callback
+ *
+ * Pass notification to remoteproc virtio
+ *
+ * Return: 0. having return is to satisfy the idr_for_each() function
+ *          pointer input argument requirement.
+ **/
+static int event_notified_idr_cb(int id, void *ptr, void *data)
+{
+	struct rproc *rproc = data;
+
+	if (rproc_vq_interrupt(rproc, id) == IRQ_NONE)
+		dev_dbg(&rproc->dev, "data not found for vqid=%d\n", id);
+
+	return 0;
+}
+
+/**
+ * handle_event_notified() - remoteproc notification work function
+ * @work: pointer to the work structure
+ *
+ * It checks each registered remoteproc notify IDs.
+ */
+static void handle_event_notified(struct work_struct *work)
+{
+	struct mbox_info *ipi;
+	struct rproc *rproc;
+
+	ipi = container_of(work, struct mbox_info, mbox_work);
+	rproc = ipi->r5_core->rproc;
+
+	/*
+	 * We only use IPI for interrupt. The RPU firmware side may or may
+	 * not write the notifyid when it trigger IPI.
+	 * And thus, we scan through all the registered notifyids and
+	 * find which one is valid to get the message.
+	 * Even if message from firmware is NULL, we attempt to get vqid
+	 */
+	idr_for_each(&rproc->notifyids, event_notified_idr_cb, rproc);
+}
+
+/**
+ * zynqmp_r5_mb_rx_cb() - receive channel mailbox callback
+ * @cl: mailbox client
+ * @msg: message pointer
+ *
+ * Receive data from ipi buffer, ack interrupt and then
+ * it will schedule the R5 notification work.
+ */
+static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *msg)
+{
+	struct zynqmp_ipi_message *ipi_msg, *buf_msg;
+	struct mbox_info *ipi;
+	size_t len;
+
+	ipi = container_of(cl, struct mbox_info, mbox_cl);
+
+	/* copy data from ipi buffer to r5_core */
+	ipi_msg = (struct zynqmp_ipi_message *)msg;
+	buf_msg = (struct zynqmp_ipi_message *)ipi->rx_mc_buf;
+	len = ipi_msg->len;
+	if (len > IPI_BUF_LEN_MAX) {
+		dev_warn(cl->dev, "msg size exceeded than %d\n",
+			 IPI_BUF_LEN_MAX);
+		len = IPI_BUF_LEN_MAX;
+	}
+	buf_msg->len = len;
+	memcpy(buf_msg->data, ipi_msg->data, len);
+
+	/* received and processed interrupt ack */
+	if (mbox_send_message(ipi->rx_chan, NULL) < 0)
+		dev_err(cl->dev, "ack failed to mbox rx_chan\n");
+
+	schedule_work(&ipi->mbox_work);
+}
+
+/**
+ * zynqmp_r5_setup_mbox() - Setup mailboxes related properties
+ *			    this is used for each individual R5 core
+ *
+ * @cdev: child node device
+ *
+ * Function to setup mailboxes related properties
+ * return : NULL if failed else pointer to mbox_info
+ */
+static struct mbox_info *zynqmp_r5_setup_mbox(struct device *cdev)
+{
+	struct mbox_client *mbox_cl;
+	struct mbox_info *ipi;
+
+	ipi = kzalloc(sizeof(*ipi), GFP_KERNEL);
+	if (!ipi)
+		return NULL;
+
+	mbox_cl = &ipi->mbox_cl;
+	mbox_cl->rx_callback = zynqmp_r5_mb_rx_cb;
+	mbox_cl->tx_block = false;
+	mbox_cl->knows_txdone = false;
+	mbox_cl->tx_done = NULL;
+	mbox_cl->dev = cdev;
+
+	/* Request TX and RX channels */
+	ipi->tx_chan = mbox_request_channel_byname(mbox_cl, "tx");
+	if (IS_ERR(ipi->tx_chan)) {
+		ipi->tx_chan = NULL;
+		kfree(ipi);
+		dev_warn(cdev, "mbox tx channel request failed\n");
+		return NULL;
+	}
+
+	ipi->rx_chan = mbox_request_channel_byname(mbox_cl, "rx");
+	if (IS_ERR(ipi->rx_chan)) {
+		mbox_free_channel(ipi->tx_chan);
+		ipi->rx_chan = NULL;
+		ipi->tx_chan = NULL;
+		kfree(ipi);
+		dev_warn(cdev, "mbox rx channel request failed\n");
+		return NULL;
+	}
+
+	INIT_WORK(&ipi->mbox_work, handle_event_notified);
+
+	return ipi;
+}
+
+static void zynqmp_r5_free_mbox(struct mbox_info *ipi)
+{
+	if (!ipi)
+		return;
+
+	if (ipi->tx_chan) {
+		mbox_free_channel(ipi->tx_chan);
+		ipi->tx_chan = NULL;
+	}
+
+	if (ipi->rx_chan) {
+		mbox_free_channel(ipi->rx_chan);
+		ipi->rx_chan = NULL;
+	}
+
+	kfree(ipi);
+}
+
+/*
+ * zynqmp_r5_core_kick() - kick a firmware if mbox is provided
+ * @rproc: r5 core's corresponding rproc structure
+ * @vqid: virtqueue ID
+ */
+static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid)
+{
+	struct zynqmp_r5_core *r5_core = rproc->priv;
+	struct device *dev = r5_core->dev;
+	struct zynqmp_ipi_message *mb_msg;
+	struct mbox_info *ipi;
+	int ret;
+
+	ipi = r5_core->ipi;
+	if (!ipi)
+		return;
+
+	mb_msg = (struct zynqmp_ipi_message *)ipi->tx_mc_buf;
+	memcpy(mb_msg->data, &vqid, sizeof(vqid));
+	mb_msg->len = sizeof(vqid);
+	ret = mbox_send_message(ipi->tx_chan, mb_msg);
+	if (ret < 0)
+		dev_warn(dev, "failed to send message\n");
+}
+
 /*
  * zynqmp_r5_set_mode()
  *
@@ -239,21 +437,29 @@ static int add_mem_regions_carveout(struct rproc *rproc)
 {
 	struct rproc_mem_entry *rproc_mem;
 	struct zynqmp_r5_core *r5_core;
+	struct of_phandle_iterator it;
 	struct reserved_mem *rmem;
-	int i, num_mem_regions;
+	int i = 0;
 
-	r5_core = (struct zynqmp_r5_core *)rproc->priv;
-	num_mem_regions = r5_core->rmem_count;
+	r5_core = rproc->priv;
 
-	for (i = 0; i < num_mem_regions; i++) {
-		rmem = r5_core->rmem[i];
+	/* Register associated reserved memory regions */
+	of_phandle_iterator_init(&it, r5_core->np, "memory-region", NULL, 0);
 
-		if (!strncmp(rmem->name, "vdev0buffer", strlen("vdev0buffer"))) {
+	while (of_phandle_iterator_next(&it) == 0) {
+		rmem = of_reserved_mem_lookup(it.node);
+		if (!rmem) {
+			of_node_put(it.node);
+			dev_err(&rproc->dev, "unable to acquire memory-region\n");
+			return -EINVAL;
+		}
+
+		if (!strcmp(it.node->name, "vdev0buffer")) {
 			/* Init reserved memory for vdev buffer */
 			rproc_mem = rproc_of_resm_mem_entry_init(&rproc->dev, i,
 								 rmem->size,
 								 rmem->base,
-								 rmem->name);
+								 it.node->name);
 		} else {
 			/* Register associated reserved memory regions */
 			rproc_mem = rproc_mem_entry_init(&rproc->dev, NULL,
@@ -261,16 +467,19 @@ static int add_mem_regions_carveout(struct rproc *rproc)
 							 rmem->size, rmem->base,
 							 zynqmp_r5_mem_region_map,
 							 zynqmp_r5_mem_region_unmap,
-							 rmem->name);
+							 it.node->name);
 		}
 
-		if (!rproc_mem)
+		if (!rproc_mem) {
+			of_node_put(it.node);
 			return -ENOMEM;
+		}
 
 		rproc_add_carveout(rproc, rproc_mem);
 
 		dev_dbg(&rproc->dev, "reserved mem carveout %s addr=%llx, size=0x%llx",
-			rmem->name, rmem->base, rmem->size);
+			it.node->name, rmem->base, rmem->size);
+		i++;
 	}
 
 	return 0;
@@ -363,7 +572,7 @@ static int add_tcm_carveout_split_mode(struct rproc *rproc)
 	size_t bank_size;
 	char *bank_name;
 
-	r5_core = (struct zynqmp_r5_core *)rproc->priv;
+	r5_core = rproc->priv;
 	dev = r5_core->dev;
 	num_banks = r5_core->tcm_bank_count;
 
@@ -432,7 +641,7 @@ static int add_tcm_carveout_lockstep_mode(struct rproc *rproc)
 	u32 pm_domain_id;
 	char *bank_name;
 
-	r5_core = (struct zynqmp_r5_core *)rproc->priv;
+	r5_core = rproc->priv;
 	dev = r5_core->dev;
 
 	/* Go through zynqmp banks for r5 node */
@@ -502,7 +711,7 @@ static int add_tcm_banks(struct rproc *rproc)
 	struct zynqmp_r5_core *r5_core;
 	struct device *dev;
 
-	r5_core = (struct zynqmp_r5_core *)rproc->priv;
+	r5_core = rproc->priv;
 	if (!r5_core)
 		return -EINVAL;
 
@@ -595,7 +804,7 @@ static int zynqmp_r5_rproc_unprepare(struct rproc *rproc)
 	u32 pm_domain_id;
 	int i;
 
-	r5_core = (struct zynqmp_r5_core *)rproc->priv;
+	r5_core = rproc->priv;
 
 	for (i = 0; i < r5_core->tcm_bank_count; i++) {
 		pm_domain_id = r5_core->tcm_banks[i]->pm_domain_id;
@@ -617,6 +826,7 @@ static const struct rproc_ops zynqmp_r5_rproc_ops = {
 	.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
 	.sanity_check	= rproc_elf_sanity_check,
 	.get_boot_addr	= rproc_elf_get_boot_addr,
+	.kick		= zynqmp_r5_rproc_kick,
 };
 
 /**
@@ -649,7 +859,7 @@ static struct zynqmp_r5_core *zynqmp_r5_add_rproc_core(struct device *cdev)
 	}
 
 	r5_rproc->auto_boot = false;
-	r5_core = (struct zynqmp_r5_core *)r5_rproc->priv;
+	r5_core = r5_rproc->priv;
 	r5_core->dev = cdev;
 	r5_core->np = dev_of_node(cdev);
 	if (!r5_core->np) {
@@ -726,59 +936,6 @@ static int zynqmp_r5_get_tcm_node(struct zynqmp_r5_cluster *cluster)
 	return 0;
 }
 
-/**
- * zynqmp_r5_get_mem_region_node()
- * parse memory-region property and get reserved mem regions
- *
- * @r5_core: pointer to zynqmp_r5_core type object
- *
- * Return: 0 for success and error code for failure.
- */
-static int zynqmp_r5_get_mem_region_node(struct zynqmp_r5_core *r5_core)
-{
-	struct device_node *np, *rmem_np;
-	struct reserved_mem **rmem;
-	int res_mem_count, i;
-	struct device *dev;
-
-	dev = r5_core->dev;
-	np = r5_core->np;
-
-	res_mem_count = of_property_count_elems_of_size(np, "memory-region",
-							sizeof(phandle));
-	if (res_mem_count <= 0) {
-		dev_warn(dev, "failed to get memory-region property %d\n",
-			 res_mem_count);
-		return 0;
-	}
-
-	rmem = devm_kcalloc(dev, res_mem_count,
-			    sizeof(struct reserved_mem *), GFP_KERNEL);
-	if (!rmem)
-		return -ENOMEM;
-
-	for (i = 0; i < res_mem_count; i++) {
-		rmem_np = of_parse_phandle(np, "memory-region", i);
-		if (!rmem_np)
-			goto release_rmem;
-
-		rmem[i] = of_reserved_mem_lookup(rmem_np);
-		if (!rmem[i]) {
-			of_node_put(rmem_np);
-			goto release_rmem;
-		}
-
-		of_node_put(rmem_np);
-	}
-
-	r5_core->rmem_count = res_mem_count;
-	r5_core->rmem = rmem;
-	return 0;
-
-release_rmem:
-	return -EINVAL;
-}
-
 /*
  * zynqmp_r5_core_init()
  * Create and initialize zynqmp_r5_core type object
@@ -806,10 +963,6 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster,
 	for (i = 0; i < cluster->core_count; i++) {
 		r5_core = cluster->r5_cores[i];
 
-		ret = zynqmp_r5_get_mem_region_node(r5_core);
-		if (ret)
-			dev_warn(dev, "memory-region prop failed %d\n", ret);
-
 		/* Initialize r5 cores with power-domains parsed from dts */
 		ret = of_property_read_u32_index(r5_core->np, "power-domains",
 						 1, &r5_core->pm_domain_id);
@@ -849,6 +1002,7 @@ static int zynqmp_r5_cluster_init(struct zynqmp_r5_cluster *cluster)
 	struct device_node *child;
 	enum rpu_tcm_comb tcm_mode;
 	int core_count, ret, i;
+	struct mbox_info *ipi;
 
 	ret = of_property_read_u32(dev_node, "xlnx,cluster-mode", &cluster_mode);
 
@@ -929,6 +1083,16 @@ static int zynqmp_r5_cluster_init(struct zynqmp_r5_cluster *cluster)
 		}
 
 		/*
+		 * If mailbox nodes are disabled using "status" property then
+		 * setting up mailbox channels will fail.
+		 */
+		ipi = zynqmp_r5_setup_mbox(&child_pdev->dev);
+		if (ipi) {
+			r5_cores[i]->ipi = ipi;
+			ipi->r5_core = r5_cores[i];
+		}
+
+		/*
 		 * If two child nodes are available in dts in lockstep mode,
 		 * then ignore second child node.
 		 */
@@ -965,6 +1129,7 @@ release_r5_cores:
 	while (i >= 0) {
 		put_device(child_devs[i]);
 		if (r5_cores[i]) {
+			zynqmp_r5_free_mbox(r5_cores[i]->ipi);
 			of_reserved_mem_device_release(r5_cores[i]->dev);
 			rproc_del(r5_cores[i]->rproc);
 			rproc_free(r5_cores[i]->rproc);
@@ -978,17 +1143,18 @@ release_r5_cores:
 
 static void zynqmp_r5_cluster_exit(void *data)
 {
-	struct platform_device *pdev = (struct platform_device *)data;
+	struct platform_device *pdev = data;
 	struct zynqmp_r5_cluster *cluster;
 	struct zynqmp_r5_core *r5_core;
 	int i;
 
-	cluster = (struct zynqmp_r5_cluster *)platform_get_drvdata(pdev);
+	cluster = platform_get_drvdata(pdev);
 	if (!cluster)
 		return;
 
 	for (i = 0; i < cluster->core_count; i++) {
 		r5_core = cluster->r5_cores[i];
+		zynqmp_r5_free_mbox(r5_core->ipi);
 		of_reserved_mem_device_release(r5_core->dev);
 		put_device(r5_core->dev);
 		rproc_del(r5_core->rproc);
diff --git a/drivers/reset/reset-lantiq.c b/drivers/reset/reset-lantiq.c
index b936cfe85641..549ba45d8597 100644
--- a/drivers/reset/reset-lantiq.c
+++ b/drivers/reset/reset-lantiq.c
@@ -207,4 +207,3 @@ module_platform_driver(lantiq_rcu_reset_driver);
 
 MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
 MODULE_DESCRIPTION("Lantiq XWAY RCU Reset Controller Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/reset/reset-microchip-sparx5.c b/drivers/reset/reset-microchip-sparx5.c
index f3528dd1d084..ead25942061d 100644
--- a/drivers/reset/reset-microchip-sparx5.c
+++ b/drivers/reset/reset-microchip-sparx5.c
@@ -179,4 +179,3 @@ postcore_initcall(mchp_sparx5_reset_init);
 
 MODULE_DESCRIPTION("Microchip Sparx5 switch reset driver");
 MODULE_AUTHOR("Steen Hegelund <steen.hegelund@microchip.com>");
-MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/reset/reset-mpfs.c b/drivers/reset/reset-mpfs.c
index e003e50590ec..e71ab73092ab 100644
--- a/drivers/reset/reset-mpfs.c
+++ b/drivers/reset/reset-mpfs.c
@@ -153,5 +153,4 @@ module_auxiliary_driver(mpfs_reset_driver);
 
 MODULE_DESCRIPTION("Microchip PolarFire SoC Reset Driver");
 MODULE_AUTHOR("Conor Dooley <conor.dooley@microchip.com>");
-MODULE_LICENSE("GPL");
 MODULE_IMPORT_NS(MCHP_CLK_MPFS);
diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c
index 01d2805fe30f..1beb40a1d3df 100644
--- a/drivers/rpmsg/qcom_glink_native.c
+++ b/drivers/rpmsg/qcom_glink_native.c
@@ -16,6 +16,7 @@
 #include <linux/rpmsg.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
+#include <linux/wait.h>
 #include <linux/workqueue.h>
 #include <linux/mailbox_client.h>
 
@@ -145,7 +146,8 @@ enum {
  * @open_req:	completed once open-request has been received
  * @intent_req_lock: Synchronises multiple intent requests
  * @intent_req_result: Result of intent request
- * @intent_req_comp: Completion for intent_req signalling
+ * @intent_received: flag indicating that an intent has been received
+ * @intent_req_wq: wait queue for intent_req signalling
  */
 struct glink_channel {
 	struct rpmsg_endpoint ept;
@@ -175,8 +177,9 @@ struct glink_channel {
 	struct completion open_req;
 
 	struct mutex intent_req_lock;
-	bool intent_req_result;
-	struct completion intent_req_comp;
+	int intent_req_result;
+	bool intent_received;
+	wait_queue_head_t intent_req_wq;
 };
 
 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
@@ -221,7 +224,7 @@ static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
 
 	init_completion(&channel->open_req);
 	init_completion(&channel->open_ack);
-	init_completion(&channel->intent_req_comp);
+	init_waitqueue_head(&channel->intent_req_wq);
 
 	INIT_LIST_HEAD(&channel->done_intents);
 	INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
@@ -419,14 +422,14 @@ static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
 		return;
 	}
 
-	channel->intent_req_result = granted;
-	complete(&channel->intent_req_comp);
+	WRITE_ONCE(channel->intent_req_result, granted);
+	wake_up_all(&channel->intent_req_wq);
 }
 
 static void qcom_glink_intent_req_abort(struct glink_channel *channel)
 {
-	channel->intent_req_result = 0;
-	complete(&channel->intent_req_comp);
+	WRITE_ONCE(channel->intent_req_result, 0);
+	wake_up_all(&channel->intent_req_wq);
 }
 
 /**
@@ -756,6 +759,11 @@ static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
 		kfree(intent);
 	}
 	spin_unlock_irqrestore(&channel->intent_lock, flags);
+
+	if (reuse) {
+		WRITE_ONCE(channel->intent_received, true);
+		wake_up_all(&channel->intent_req_wq);
+	}
 }
 
 /**
@@ -993,6 +1001,9 @@ static void qcom_glink_handle_intent(struct qcom_glink *glink,
 			dev_err(glink->dev, "failed to store remote intent\n");
 	}
 
+	WRITE_ONCE(channel->intent_received, true);
+	wake_up_all(&channel->intent_req_wq);
+
 	kfree(msg);
 	qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
 }
@@ -1271,7 +1282,8 @@ static int qcom_glink_request_intent(struct qcom_glink *glink,
 
 	mutex_lock(&channel->intent_req_lock);
 
-	reinit_completion(&channel->intent_req_comp);
+	WRITE_ONCE(channel->intent_req_result, -1);
+	WRITE_ONCE(channel->intent_received, false);
 
 	cmd.id = GLINK_CMD_RX_INTENT_REQ;
 	cmd.cid = channel->lcid;
@@ -1281,12 +1293,15 @@ static int qcom_glink_request_intent(struct qcom_glink *glink,
 	if (ret)
 		goto unlock;
 
-	ret = wait_for_completion_timeout(&channel->intent_req_comp, 10 * HZ);
+	ret = wait_event_timeout(channel->intent_req_wq,
+				 READ_ONCE(channel->intent_req_result) >= 0 &&
+				 READ_ONCE(channel->intent_received),
+				 10 * HZ);
 	if (!ret) {
 		dev_err(glink->dev, "intent request timed out\n");
 		ret = -ETIMEDOUT;
 	} else {
-		ret = channel->intent_req_result ? 0 : -ECANCELED;
+		ret = READ_ONCE(channel->intent_req_result) ? 0 : -ECANCELED;
 	}
 
 unlock:
@@ -1309,7 +1324,7 @@ static int __qcom_glink_send(struct glink_channel *channel,
 	int ret;
 	unsigned long flags;
 	int chunk_size = len;
-	int left_size = 0;
+	size_t offset = 0;
 
 	if (!glink->intentless) {
 		while (!intent) {
@@ -1343,47 +1358,29 @@ static int __qcom_glink_send(struct glink_channel *channel,
 		iid = intent->id;
 	}
 
-	if (wait && chunk_size > SZ_8K) {
-		chunk_size = SZ_8K;
-		left_size = len - chunk_size;
-	}
-	req.msg.cmd = cpu_to_le16(GLINK_CMD_TX_DATA);
-	req.msg.param1 = cpu_to_le16(channel->lcid);
-	req.msg.param2 = cpu_to_le32(iid);
-	req.chunk_size = cpu_to_le32(chunk_size);
-	req.left_size = cpu_to_le32(left_size);
-
-	ret = qcom_glink_tx(glink, &req, sizeof(req), data, chunk_size, wait);
-
-	/* Mark intent available if we failed */
-	if (ret && intent) {
-		intent->in_use = false;
-		return ret;
-	}
-
-	while (left_size > 0) {
-		data = (void *)((char *)data + chunk_size);
-		chunk_size = left_size;
-		if (chunk_size > SZ_8K)
+	while (offset < len) {
+		chunk_size = len - offset;
+		if (chunk_size > SZ_8K && wait)
 			chunk_size = SZ_8K;
-		left_size -= chunk_size;
 
-		req.msg.cmd = cpu_to_le16(GLINK_CMD_TX_DATA_CONT);
+		req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT);
 		req.msg.param1 = cpu_to_le16(channel->lcid);
 		req.msg.param2 = cpu_to_le32(iid);
 		req.chunk_size = cpu_to_le32(chunk_size);
-		req.left_size = cpu_to_le32(left_size);
-
-		ret = qcom_glink_tx(glink, &req, sizeof(req), data,
-				    chunk_size, wait);
+		req.left_size = cpu_to_le32(len - offset - chunk_size);
 
-		/* Mark intent available if we failed */
-		if (ret && intent) {
-			intent->in_use = false;
-			break;
+		ret = qcom_glink_tx(glink, &req, sizeof(req), data + offset, chunk_size, wait);
+		if (ret) {
+			/* Mark intent available if we failed */
+			if (intent)
+				intent->in_use = false;
+			return ret;
 		}
+
+		offset += chunk_size;
 	}
-	return ret;
+
+	return 0;
 }
 
 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
diff --git a/drivers/rpmsg/qcom_glink_rpm.c b/drivers/rpmsg/qcom_glink_rpm.c
index f94bb7d4f1ec..357272d7062e 100644
--- a/drivers/rpmsg/qcom_glink_rpm.c
+++ b/drivers/rpmsg/qcom_glink_rpm.c
@@ -361,7 +361,7 @@ static int glink_rpm_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int glink_rpm_remove(struct platform_device *pdev)
+static void glink_rpm_remove(struct platform_device *pdev)
 {
 	struct glink_rpm *rpm = platform_get_drvdata(pdev);
 	struct qcom_glink *glink = rpm->glink;
@@ -371,8 +371,6 @@ static int glink_rpm_remove(struct platform_device *pdev)
 	qcom_glink_native_remove(glink);
 
 	mbox_free_channel(rpm->mbox_chan);
-
-	return 0;
 }
 
 static const struct of_device_id glink_rpm_of_match[] = {
@@ -383,7 +381,7 @@ MODULE_DEVICE_TABLE(of, glink_rpm_of_match);
 
 static struct platform_driver glink_rpm_driver = {
 	.probe = glink_rpm_probe,
-	.remove = glink_rpm_remove,
+	.remove_new = glink_rpm_remove,
 	.driver = {
 		.name = "qcom_glink_rpm",
 		.of_match_table = glink_rpm_of_match,
diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c
index 1044cf03c542..7b9c298aa491 100644
--- a/drivers/rpmsg/qcom_smd.c
+++ b/drivers/rpmsg/qcom_smd.c
@@ -1533,7 +1533,7 @@ static int qcom_smd_remove_device(struct device *dev, void *data)
  * qcom_smd_unregister_edge() - release an edge and its children
  * @edge:      edge reference acquired from qcom_smd_register_edge
  */
-int qcom_smd_unregister_edge(struct qcom_smd_edge *edge)
+void qcom_smd_unregister_edge(struct qcom_smd_edge *edge)
 {
 	int ret;
 
@@ -1547,8 +1547,6 @@ int qcom_smd_unregister_edge(struct qcom_smd_edge *edge)
 
 	mbox_free_channel(edge->mbox_chan);
 	device_unregister(&edge->dev);
-
-	return 0;
 }
 EXPORT_SYMBOL(qcom_smd_unregister_edge);
 
@@ -1572,22 +1570,22 @@ static int qcom_smd_remove_edge(struct device *dev, void *data)
 {
 	struct qcom_smd_edge *edge = to_smd_edge(dev);
 
-	return qcom_smd_unregister_edge(edge);
+	qcom_smd_unregister_edge(edge);
+
+	return 0;
 }
 
 /*
  * Shut down all smd clients by making sure that each edge stops processing
  * events and scanning for new channels, then call destroy on the devices.
  */
-static int qcom_smd_remove(struct platform_device *pdev)
+static void qcom_smd_remove(struct platform_device *pdev)
 {
-	int ret;
-
-	ret = device_for_each_child(&pdev->dev, NULL, qcom_smd_remove_edge);
-	if (ret)
-		dev_warn(&pdev->dev, "can't remove smd device: %d\n", ret);
-
-	return ret;
+	/*
+	 * qcom_smd_remove_edge always returns zero, so there is no need to
+	 * check the return value of device_for_each_child.
+	 */
+	device_for_each_child(&pdev->dev, NULL, qcom_smd_remove_edge);
 }
 
 static const struct of_device_id qcom_smd_of_match[] = {
@@ -1598,7 +1596,7 @@ MODULE_DEVICE_TABLE(of, qcom_smd_of_match);
 
 static struct platform_driver qcom_smd_driver = {
 	.probe = qcom_smd_probe,
-	.remove = qcom_smd_remove,
+	.remove_new = qcom_smd_remove,
 	.driver = {
 		.name = "qcom-smd",
 		.of_match_table = qcom_smd_of_match,
diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c
index a2207c0cf432..5039df757127 100644
--- a/drivers/rpmsg/rpmsg_core.c
+++ b/drivers/rpmsg/rpmsg_core.c
@@ -694,7 +694,7 @@ static int __init rpmsg_init(void)
 {
 	int ret;
 
-	rpmsg_class = class_create(THIS_MODULE, "rpmsg");
+	rpmsg_class = class_create("rpmsg");
 	if (IS_ERR(rpmsg_class)) {
 		pr_err("failed to create rpmsg class\n");
 		return PTR_ERR(rpmsg_class);
diff --git a/drivers/rtc/class.c b/drivers/rtc/class.c
index e5b7b48cffac..edfd942f8c54 100644
--- a/drivers/rtc/class.c
+++ b/drivers/rtc/class.c
@@ -475,7 +475,7 @@ EXPORT_SYMBOL_GPL(devm_rtc_device_register);
 
 static int __init rtc_init(void)
 {
-	rtc_class = class_create(THIS_MODULE, "rtc");
+	rtc_class = class_create("rtc");
 	if (IS_ERR(rtc_class)) {
 		pr_err("couldn't create class\n");
 		return PTR_ERR(rtc_class);
diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c
index a9c2a8d76c45..9fbfce735d56 100644
--- a/drivers/s390/block/dasd.c
+++ b/drivers/s390/block/dasd.c
@@ -73,7 +73,8 @@ static void dasd_profile_init(struct dasd_profile *, struct dentry *);
 static void dasd_profile_exit(struct dasd_profile *);
 static void dasd_hosts_init(struct dentry *, struct dasd_device *);
 static void dasd_hosts_exit(struct dasd_device *);
-
+static int dasd_handle_autoquiesce(struct dasd_device *, struct dasd_ccw_req *,
+				   unsigned int);
 /*
  * SECTION: Operations on the device structure.
  */
@@ -1451,6 +1452,8 @@ int dasd_start_IO(struct dasd_ccw_req *cqr)
 	case -ENODEV:
 		DBF_DEV_EVENT(DBF_WARNING, device, "%s",
 			      "start_IO: -ENODEV device gone, retry");
+		/* this is equivalent to CC=3 for SSCH report this to EER */
+		dasd_handle_autoquiesce(device, cqr, DASD_EER_STARTIO);
 		break;
 	case -EIO:
 		DBF_DEV_EVENT(DBF_WARNING, device, "%s",
@@ -1953,6 +1956,16 @@ static void __dasd_device_process_final_queue(struct dasd_device *device,
 }
 
 /*
+ * check if device should be autoquiesced due to too many timeouts
+ */
+static void __dasd_device_check_autoquiesce_timeout(struct dasd_device *device,
+						    struct dasd_ccw_req *cqr)
+{
+	if ((device->default_retries - cqr->retries) >= device->aq_timeouts)
+		dasd_handle_autoquiesce(device, cqr, DASD_EER_TIMEOUTS);
+}
+
+/*
  * Take a look at the first request on the ccw queue and check
  * if it reached its expire time. If so, terminate the IO.
  */
@@ -1986,6 +1999,7 @@ static void __dasd_device_check_expire(struct dasd_device *device)
 				"remaining\n", cqr, (cqr->expires/HZ),
 				cqr->retries);
 		}
+		__dasd_device_check_autoquiesce_timeout(device, cqr);
 	}
 }
 
@@ -2325,7 +2339,7 @@ static int _dasd_sleep_on(struct dasd_ccw_req *maincqr, int interruptible)
 		/* Non-temporary stop condition will trigger fail fast */
 		if (device->stopped & ~DASD_STOPPED_PENDING &&
 		    test_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags) &&
-		    (!dasd_eer_enabled(device))) {
+		    !dasd_eer_enabled(device) && device->aq_mask == 0) {
 			cqr->status = DASD_CQR_FAILED;
 			cqr->intrc = -ENOLINK;
 			continue;
@@ -2801,20 +2815,18 @@ restart:
 			dasd_log_sense(cqr, &cqr->irb);
 		}
 
-		/* First of all call extended error reporting. */
-		if (dasd_eer_enabled(base) &&
-		    cqr->status == DASD_CQR_FAILED) {
-			dasd_eer_write(base, cqr, DASD_EER_FATALERROR);
-
-			/* restart request  */
+		/*
+		 * First call extended error reporting and check for autoquiesce
+		 */
+		spin_lock_irqsave(get_ccwdev_lock(base->cdev), flags);
+		if (cqr->status == DASD_CQR_FAILED &&
+		    dasd_handle_autoquiesce(base, cqr, DASD_EER_FATALERROR)) {
 			cqr->status = DASD_CQR_FILLED;
 			cqr->retries = 255;
-			spin_lock_irqsave(get_ccwdev_lock(base->cdev), flags);
-			dasd_device_set_stop_bits(base, DASD_STOPPED_QUIESCE);
-			spin_unlock_irqrestore(get_ccwdev_lock(base->cdev),
-					       flags);
+			spin_unlock_irqrestore(get_ccwdev_lock(base->cdev), flags);
 			goto restart;
 		}
+		spin_unlock_irqrestore(get_ccwdev_lock(base->cdev), flags);
 
 		/* Process finished ERP request. */
 		if (cqr->refers) {
@@ -2856,7 +2868,7 @@ static void __dasd_block_start_head(struct dasd_block *block)
 		/* Non-temporary stop condition will trigger fail fast */
 		if (block->base->stopped & ~DASD_STOPPED_PENDING &&
 		    test_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags) &&
-		    (!dasd_eer_enabled(block->base))) {
+		    !dasd_eer_enabled(block->base) && block->base->aq_mask == 0) {
 			cqr->status = DASD_CQR_FAILED;
 			cqr->intrc = -ENOLINK;
 			dasd_schedule_block_bh(block);
@@ -2941,7 +2953,7 @@ static int _dasd_requeue_request(struct dasd_ccw_req *cqr)
 		return 0;
 	spin_lock_irq(&cqr->dq->lock);
 	req = (struct request *) cqr->callback_data;
-	blk_mq_requeue_request(req, false);
+	blk_mq_requeue_request(req, true);
 	spin_unlock_irq(&cqr->dq->lock);
 
 	return 0;
@@ -3670,8 +3682,8 @@ int dasd_generic_last_path_gone(struct dasd_device *device)
 	dev_warn(&device->cdev->dev, "No operational channel path is left "
 		 "for the device\n");
 	DBF_DEV_EVENT(DBF_WARNING, device, "%s", "last path gone");
-	/* First of all call extended error reporting. */
-	dasd_eer_write(device, NULL, DASD_EER_NOPATH);
+	/* First call extended error reporting and check for autoquiesce. */
+	dasd_handle_autoquiesce(device, NULL, DASD_EER_NOPATH);
 
 	if (device->state < DASD_STATE_BASIC)
 		return 0;
@@ -3803,7 +3815,8 @@ void dasd_generic_path_event(struct ccw_device *cdev, int *path_event)
 			 "No verified channel paths remain for the device\n");
 		DBF_DEV_EVENT(DBF_WARNING, device,
 			      "%s", "last verified path gone");
-		dasd_eer_write(device, NULL, DASD_EER_NOPATH);
+		/* First call extended error reporting and check for autoquiesce. */
+		dasd_handle_autoquiesce(device, NULL, DASD_EER_NOPATH);
 		dasd_device_set_stop_bits(device,
 					  DASD_STOPPED_DC_WAIT);
 	}
@@ -3825,7 +3838,8 @@ EXPORT_SYMBOL_GPL(dasd_generic_verify_path);
 void dasd_generic_space_exhaust(struct dasd_device *device,
 				struct dasd_ccw_req *cqr)
 {
-	dasd_eer_write(device, NULL, DASD_EER_NOSPC);
+	/* First call extended error reporting and check for autoquiesce. */
+	dasd_handle_autoquiesce(device, NULL, DASD_EER_NOSPC);
 
 	if (device->state < DASD_STATE_BASIC)
 		return;
@@ -3958,6 +3972,31 @@ void dasd_schedule_requeue(struct dasd_device *device)
 }
 EXPORT_SYMBOL(dasd_schedule_requeue);
 
+static int dasd_handle_autoquiesce(struct dasd_device *device,
+				   struct dasd_ccw_req *cqr,
+				   unsigned int reason)
+{
+	/* in any case write eer message with reason */
+	if (dasd_eer_enabled(device))
+		dasd_eer_write(device, cqr, reason);
+
+	if (!test_bit(reason, &device->aq_mask))
+		return 0;
+
+	/* notify eer about autoquiesce */
+	if (dasd_eer_enabled(device))
+		dasd_eer_write(device, NULL, DASD_EER_AUTOQUIESCE);
+
+	pr_info("%s: The DASD has been put in the quiesce state\n",
+		dev_name(&device->cdev->dev));
+	dasd_device_set_stop_bits(device, DASD_STOPPED_QUIESCE);
+
+	if (device->features & DASD_FEATURE_REQUEUEQUIESCE)
+		dasd_schedule_requeue(device);
+
+	return 1;
+}
+
 static struct dasd_ccw_req *dasd_generic_build_rdc(struct dasd_device *device,
 						   int rdc_buffer_size,
 						   int magic)
diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c
index df17f0f9cb0f..620fab01b710 100644
--- a/drivers/s390/block/dasd_devmap.c
+++ b/drivers/s390/block/dasd_devmap.c
@@ -50,6 +50,7 @@ struct dasd_devmap {
         unsigned short features;
 	struct dasd_device *device;
 	struct dasd_copy_relation *copy;
+	unsigned int aq_mask;
 };
 
 /*
@@ -1476,6 +1477,128 @@ dasd_eer_store(struct device *dev, struct device_attribute *attr,
 static DEVICE_ATTR(eer_enabled, 0644, dasd_eer_show, dasd_eer_store);
 
 /*
+ * aq_mask controls if the DASD should be quiesced on certain triggers
+ * The aq_mask attribute is interpreted as bitmap of the DASD_EER_* triggers.
+ */
+static ssize_t dasd_aq_mask_show(struct device *dev, struct device_attribute *attr,
+				 char *buf)
+{
+	struct dasd_devmap *devmap;
+	unsigned int aq_mask = 0;
+
+	devmap = dasd_find_busid(dev_name(dev));
+	if (!IS_ERR(devmap))
+		aq_mask = devmap->aq_mask;
+
+	return sysfs_emit(buf, "%d\n", aq_mask);
+}
+
+static ssize_t dasd_aq_mask_store(struct device *dev, struct device_attribute *attr,
+				  const char *buf, size_t count)
+{
+	struct dasd_devmap *devmap;
+	unsigned int val;
+
+	if (kstrtouint(buf, 0, &val) || val > DASD_EER_VALID)
+		return -EINVAL;
+
+	devmap = dasd_devmap_from_cdev(to_ccwdev(dev));
+	if (IS_ERR(devmap))
+		return PTR_ERR(devmap);
+
+	spin_lock(&dasd_devmap_lock);
+	devmap->aq_mask = val;
+	if (devmap->device)
+		devmap->device->aq_mask = devmap->aq_mask;
+	spin_unlock(&dasd_devmap_lock);
+
+	return count;
+}
+
+static DEVICE_ATTR(aq_mask, 0644, dasd_aq_mask_show, dasd_aq_mask_store);
+
+/*
+ * aq_requeue controls if requests are returned to the blocklayer on quiesce
+ * or if requests are only not started
+ */
+static ssize_t dasd_aqr_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	struct dasd_devmap *devmap;
+	int flag;
+
+	devmap = dasd_find_busid(dev_name(dev));
+	if (!IS_ERR(devmap))
+		flag = (devmap->features & DASD_FEATURE_REQUEUEQUIESCE) != 0;
+	else
+		flag = (DASD_FEATURE_DEFAULT &
+			DASD_FEATURE_REQUEUEQUIESCE) != 0;
+	return sysfs_emit(buf, "%d\n", flag);
+}
+
+static ssize_t dasd_aqr_store(struct device *dev, struct device_attribute *attr,
+			      const char *buf, size_t count)
+{
+	bool val;
+	int rc;
+
+	if (kstrtobool(buf, &val))
+		return -EINVAL;
+
+	rc = dasd_set_feature(to_ccwdev(dev), DASD_FEATURE_REQUEUEQUIESCE, val);
+
+	return rc ? : count;
+}
+
+static DEVICE_ATTR(aq_requeue, 0644, dasd_aqr_show, dasd_aqr_store);
+
+/*
+ * aq_timeouts controls how much retries have to time out until
+ * a device gets autoquiesced
+ */
+static ssize_t
+dasd_aq_timeouts_show(struct device *dev, struct device_attribute *attr,
+		      char *buf)
+{
+	struct dasd_device *device;
+	int len;
+
+	device = dasd_device_from_cdev(to_ccwdev(dev));
+	if (IS_ERR(device))
+		return -ENODEV;
+	len = sysfs_emit(buf, "%u\n", device->aq_timeouts);
+	dasd_put_device(device);
+	return len;
+}
+
+static ssize_t
+dasd_aq_timeouts_store(struct device *dev, struct device_attribute *attr,
+		       const char *buf, size_t count)
+{
+	struct dasd_device *device;
+	unsigned int val;
+
+	device = dasd_device_from_cdev(to_ccwdev(dev));
+	if (IS_ERR(device))
+		return -ENODEV;
+
+	if ((kstrtouint(buf, 10, &val) != 0) ||
+	    val > DASD_RETRIES_MAX || val == 0) {
+		dasd_put_device(device);
+		return -EINVAL;
+	}
+
+	if (val)
+		device->aq_timeouts = val;
+
+	dasd_put_device(device);
+	return count;
+}
+
+static DEVICE_ATTR(aq_timeouts, 0644, dasd_aq_timeouts_show,
+		   dasd_aq_timeouts_store);
+
+/*
  * expiration time for default requests
  */
 static ssize_t
@@ -2324,6 +2447,9 @@ static struct attribute * dasd_attrs[] = {
 	&dev_attr_copy_pair.attr,
 	&dev_attr_copy_role.attr,
 	&dev_attr_ping.attr,
+	&dev_attr_aq_mask.attr,
+	&dev_attr_aq_requeue.attr,
+	&dev_attr_aq_timeouts.attr,
 	NULL,
 };
 
diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c
index 1a69f97e88fb..ade1369fe5ed 100644
--- a/drivers/s390/block/dasd_eckd.c
+++ b/drivers/s390/block/dasd_eckd.c
@@ -2109,6 +2109,7 @@ dasd_eckd_check_characteristics(struct dasd_device *device)
 	device->default_retries = DASD_RETRIES;
 	device->path_thrhld = DASD_ECKD_PATH_THRHLD;
 	device->path_interval = DASD_ECKD_PATH_INTERVAL;
+	device->aq_timeouts = DASD_RETRIES_MAX;
 
 	if (private->conf.gneq) {
 		value = 1;
diff --git a/drivers/s390/block/dasd_eer.c b/drivers/s390/block/dasd_eer.c
index a4cc772208a6..c956de711cf7 100644
--- a/drivers/s390/block/dasd_eer.c
+++ b/drivers/s390/block/dasd_eer.c
@@ -387,6 +387,7 @@ void dasd_eer_write(struct dasd_device *device, struct dasd_ccw_req *cqr,
 		break;
 	case DASD_EER_NOPATH:
 	case DASD_EER_NOSPC:
+	case DASD_EER_AUTOQUIESCE:
 		dasd_eer_write_standard_trigger(device, NULL, id);
 		break;
 	case DASD_EER_STATECHANGE:
diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h
index 97adc8a7ae6b..33f812f0e515 100644
--- a/drivers/s390/block/dasd_int.h
+++ b/drivers/s390/block/dasd_int.h
@@ -444,22 +444,22 @@ struct dasd_discipline {
 
 extern struct dasd_discipline *dasd_diag_discipline_pointer;
 
-/*
- * Notification numbers for extended error reporting notifications:
- * The DASD_EER_DISABLE notification is sent before a dasd_device (and it's
- * eer pointer) is freed. The error reporting module needs to do all necessary
- * cleanup steps.
- * The DASD_EER_TRIGGER notification sends the actual error reports (triggers).
- */
-#define DASD_EER_DISABLE 0
-#define DASD_EER_TRIGGER 1
+/* Trigger IDs for extended error reporting DASD EER and autoquiesce */
+enum eer_trigger {
+	DASD_EER_FATALERROR = 1,
+	DASD_EER_NOPATH,
+	DASD_EER_STATECHANGE,
+	DASD_EER_PPRCSUSPEND,
+	DASD_EER_NOSPC,
+	DASD_EER_TIMEOUTS,
+	DASD_EER_STARTIO,
+
+	/* enum end marker, only add new trigger above */
+	DASD_EER_MAX,
+	DASD_EER_AUTOQUIESCE = 31, /* internal only */
+};
 
-/* Trigger IDs for extended error reporting DASD_EER_TRIGGER notification */
-#define DASD_EER_FATALERROR  1
-#define DASD_EER_NOPATH      2
-#define DASD_EER_STATECHANGE 3
-#define DASD_EER_PPRCSUSPEND 4
-#define DASD_EER_NOSPC	     5
+#define DASD_EER_VALID ((1U << DASD_EER_MAX) - 1)
 
 /* DASD path handling */
 
@@ -637,6 +637,8 @@ struct dasd_device {
 	struct dasd_format_entry format_entry;
 	struct kset *paths_info;
 	struct dasd_copy_relation *copy;
+	unsigned long aq_mask;
+	unsigned int aq_timeouts;
 };
 
 struct dasd_block {
diff --git a/drivers/s390/char/hmcdrv_dev.c b/drivers/s390/char/hmcdrv_dev.c
index cb8fdf057eca..8d50c894711f 100644
--- a/drivers/s390/char/hmcdrv_dev.c
+++ b/drivers/s390/char/hmcdrv_dev.c
@@ -308,7 +308,7 @@ int hmcdrv_dev_init(void)
 	 * /proc/devices), but not under /dev nor /sys/devices/virtual. So
 	 * we have to create an associated class (see /sys/class).
 	 */
-	hmcdrv_dev_class = class_create(THIS_MODULE, HMCDRV_DEV_CLASS);
+	hmcdrv_dev_class = class_create(HMCDRV_DEV_CLASS);
 
 	if (IS_ERR(hmcdrv_dev_class)) {
 		rc = PTR_ERR(hmcdrv_dev_class);
diff --git a/drivers/s390/char/raw3270.c b/drivers/s390/char/raw3270.c
index 09d7570d3b7d..7115c0f85650 100644
--- a/drivers/s390/char/raw3270.c
+++ b/drivers/s390/char/raw3270.c
@@ -1319,7 +1319,7 @@ static int raw3270_init(void)
 	if (rc == 0) {
 		/* Create attributes for early (= console) device. */
 		mutex_lock(&raw3270_mutex);
-		class3270 = class_create(THIS_MODULE, "3270");
+		class3270 = class_create("3270");
 		list_for_each_entry(rp, &raw3270_devices, list) {
 			get_device(&rp->cdev->dev);
 			raw3270_create_attributes(rp);
diff --git a/drivers/s390/char/tape_class.c b/drivers/s390/char/tape_class.c
index c21dc68e05a0..277a0f903d11 100644
--- a/drivers/s390/char/tape_class.c
+++ b/drivers/s390/char/tape_class.c
@@ -117,7 +117,7 @@ EXPORT_SYMBOL(unregister_tape_dev);
 
 static int __init tape_init(void)
 {
-	tape_class = class_create(THIS_MODULE, "tape390");
+	tape_class = class_create("tape390");
 
 	return 0;
 }
diff --git a/drivers/s390/char/vmlogrdr.c b/drivers/s390/char/vmlogrdr.c
index ed970ecfafdf..6946ba9a9de2 100644
--- a/drivers/s390/char/vmlogrdr.c
+++ b/drivers/s390/char/vmlogrdr.c
@@ -699,7 +699,7 @@ static int vmlogrdr_register_driver(void)
 	if (ret)
 		goto out_iucv;
 
-	vmlogrdr_class = class_create(THIS_MODULE, "vmlogrdr");
+	vmlogrdr_class = class_create("vmlogrdr");
 	if (IS_ERR(vmlogrdr_class)) {
 		ret = PTR_ERR(vmlogrdr_class);
 		vmlogrdr_class = NULL;
diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c
index 131293f7f152..82efdd20ad01 100644
--- a/drivers/s390/char/vmur.c
+++ b/drivers/s390/char/vmur.c
@@ -1022,7 +1022,7 @@ static int __init ur_init(void)
 
 	debug_set_level(vmur_dbf, 6);
 
-	vmur_class = class_create(THIS_MODULE, "vmur");
+	vmur_class = class_create("vmur");
 	if (IS_ERR(vmur_class)) {
 		rc = PTR_ERR(vmur_class);
 		goto fail_free_dbf;
diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index f4cc1720156f..5a99e0b18289 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -1166,12 +1166,12 @@ EXPORT_SYMBOL(ap_parse_mask_str);
  * AP bus attributes.
  */
 
-static ssize_t ap_domain_show(struct bus_type *bus, char *buf)
+static ssize_t ap_domain_show(const struct bus_type *bus, char *buf)
 {
 	return scnprintf(buf, PAGE_SIZE, "%d\n", ap_domain_index);
 }
 
-static ssize_t ap_domain_store(struct bus_type *bus,
+static ssize_t ap_domain_store(const struct bus_type *bus,
 			       const char *buf, size_t count)
 {
 	int domain;
@@ -1193,7 +1193,7 @@ static ssize_t ap_domain_store(struct bus_type *bus,
 
 static BUS_ATTR_RW(ap_domain);
 
-static ssize_t ap_control_domain_mask_show(struct bus_type *bus, char *buf)
+static ssize_t ap_control_domain_mask_show(const struct bus_type *bus, char *buf)
 {
 	if (!ap_qci_info)	/* QCI not supported */
 		return scnprintf(buf, PAGE_SIZE, "not supported\n");
@@ -1208,7 +1208,7 @@ static ssize_t ap_control_domain_mask_show(struct bus_type *bus, char *buf)
 
 static BUS_ATTR_RO(ap_control_domain_mask);
 
-static ssize_t ap_usage_domain_mask_show(struct bus_type *bus, char *buf)
+static ssize_t ap_usage_domain_mask_show(const struct bus_type *bus, char *buf)
 {
 	if (!ap_qci_info)	/* QCI not supported */
 		return scnprintf(buf, PAGE_SIZE, "not supported\n");
@@ -1223,7 +1223,7 @@ static ssize_t ap_usage_domain_mask_show(struct bus_type *bus, char *buf)
 
 static BUS_ATTR_RO(ap_usage_domain_mask);
 
-static ssize_t ap_adapter_mask_show(struct bus_type *bus, char *buf)
+static ssize_t ap_adapter_mask_show(const struct bus_type *bus, char *buf)
 {
 	if (!ap_qci_info)	/* QCI not supported */
 		return scnprintf(buf, PAGE_SIZE, "not supported\n");
@@ -1238,7 +1238,7 @@ static ssize_t ap_adapter_mask_show(struct bus_type *bus, char *buf)
 
 static BUS_ATTR_RO(ap_adapter_mask);
 
-static ssize_t ap_interrupts_show(struct bus_type *bus, char *buf)
+static ssize_t ap_interrupts_show(const struct bus_type *bus, char *buf)
 {
 	return scnprintf(buf, PAGE_SIZE, "%d\n",
 			 ap_irq_flag ? 1 : 0);
@@ -1246,12 +1246,12 @@ static ssize_t ap_interrupts_show(struct bus_type *bus, char *buf)
 
 static BUS_ATTR_RO(ap_interrupts);
 
-static ssize_t config_time_show(struct bus_type *bus, char *buf)
+static ssize_t config_time_show(const struct bus_type *bus, char *buf)
 {
 	return scnprintf(buf, PAGE_SIZE, "%d\n", ap_config_time);
 }
 
-static ssize_t config_time_store(struct bus_type *bus,
+static ssize_t config_time_store(const struct bus_type *bus,
 				 const char *buf, size_t count)
 {
 	int time;
@@ -1265,12 +1265,12 @@ static ssize_t config_time_store(struct bus_type *bus,
 
 static BUS_ATTR_RW(config_time);
 
-static ssize_t poll_thread_show(struct bus_type *bus, char *buf)
+static ssize_t poll_thread_show(const struct bus_type *bus, char *buf)
 {
 	return scnprintf(buf, PAGE_SIZE, "%d\n", ap_poll_kthread ? 1 : 0);
 }
 
-static ssize_t poll_thread_store(struct bus_type *bus,
+static ssize_t poll_thread_store(const struct bus_type *bus,
 				 const char *buf, size_t count)
 {
 	int flag, rc;
@@ -1289,12 +1289,12 @@ static ssize_t poll_thread_store(struct bus_type *bus,
 
 static BUS_ATTR_RW(poll_thread);
 
-static ssize_t poll_timeout_show(struct bus_type *bus, char *buf)
+static ssize_t poll_timeout_show(const struct bus_type *bus, char *buf)
 {
 	return scnprintf(buf, PAGE_SIZE, "%llu\n", poll_timeout);
 }
 
-static ssize_t poll_timeout_store(struct bus_type *bus, const char *buf,
+static ssize_t poll_timeout_store(const struct bus_type *bus, const char *buf,
 				  size_t count)
 {
 	unsigned long long time;
@@ -1318,21 +1318,21 @@ static ssize_t poll_timeout_store(struct bus_type *bus, const char *buf,
 
 static BUS_ATTR_RW(poll_timeout);
 
-static ssize_t ap_max_domain_id_show(struct bus_type *bus, char *buf)
+static ssize_t ap_max_domain_id_show(const struct bus_type *bus, char *buf)
 {
 	return scnprintf(buf, PAGE_SIZE, "%d\n", ap_max_domain_id);
 }
 
 static BUS_ATTR_RO(ap_max_domain_id);
 
-static ssize_t ap_max_adapter_id_show(struct bus_type *bus, char *buf)
+static ssize_t ap_max_adapter_id_show(const struct bus_type *bus, char *buf)
 {
 	return scnprintf(buf, PAGE_SIZE, "%d\n", ap_max_adapter_id);
 }
 
 static BUS_ATTR_RO(ap_max_adapter_id);
 
-static ssize_t apmask_show(struct bus_type *bus, char *buf)
+static ssize_t apmask_show(const struct bus_type *bus, char *buf)
 {
 	int rc;
 
@@ -1393,7 +1393,7 @@ static int apmask_commit(unsigned long *newapm)
 	return 0;
 }
 
-static ssize_t apmask_store(struct bus_type *bus, const char *buf,
+static ssize_t apmask_store(const struct bus_type *bus, const char *buf,
 			    size_t count)
 {
 	int rc, changes = 0;
@@ -1425,7 +1425,7 @@ done:
 
 static BUS_ATTR_RW(apmask);
 
-static ssize_t aqmask_show(struct bus_type *bus, char *buf)
+static ssize_t aqmask_show(const struct bus_type *bus, char *buf)
 {
 	int rc;
 
@@ -1486,7 +1486,7 @@ static int aqmask_commit(unsigned long *newaqm)
 	return 0;
 }
 
-static ssize_t aqmask_store(struct bus_type *bus, const char *buf,
+static ssize_t aqmask_store(const struct bus_type *bus, const char *buf,
 			    size_t count)
 {
 	int rc, changes = 0;
@@ -1518,13 +1518,13 @@ done:
 
 static BUS_ATTR_RW(aqmask);
 
-static ssize_t scans_show(struct bus_type *bus, char *buf)
+static ssize_t scans_show(const struct bus_type *bus, char *buf)
 {
 	return scnprintf(buf, PAGE_SIZE, "%llu\n",
 			 atomic64_read(&ap_scan_bus_count));
 }
 
-static ssize_t scans_store(struct bus_type *bus, const char *buf,
+static ssize_t scans_store(const struct bus_type *bus, const char *buf,
 			   size_t count)
 {
 	AP_DBF_INFO("%s force AP bus rescan\n", __func__);
@@ -1536,7 +1536,7 @@ static ssize_t scans_store(struct bus_type *bus, const char *buf,
 
 static BUS_ATTR_RW(scans);
 
-static ssize_t bindings_show(struct bus_type *bus, char *buf)
+static ssize_t bindings_show(const struct bus_type *bus, char *buf)
 {
 	int rc;
 	unsigned int apqns, n;
diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c
index 6fe05bb82c77..cff2eea88f98 100644
--- a/drivers/s390/crypto/zcrypt_api.c
+++ b/drivers/s390/crypto/zcrypt_api.c
@@ -340,8 +340,8 @@ static const struct attribute_group *zcdn_dev_attr_groups[] = {
 	NULL
 };
 
-static ssize_t zcdn_create_store(struct class *class,
-				 struct class_attribute *attr,
+static ssize_t zcdn_create_store(const struct class *class,
+				 const struct class_attribute *attr,
 				 const char *buf, size_t count)
 {
 	int rc;
@@ -357,8 +357,8 @@ static ssize_t zcdn_create_store(struct class *class,
 static const struct class_attribute class_attr_zcdn_create =
 	__ATTR(create, 0600, NULL, zcdn_create_store);
 
-static ssize_t zcdn_destroy_store(struct class *class,
-				  struct class_attribute *attr,
+static ssize_t zcdn_destroy_store(const struct class *class,
+				  const struct class_attribute *attr,
 				  const char *buf, size_t count)
 {
 	int rc;
@@ -2171,7 +2171,7 @@ static int __init zcdn_init(void)
 	int rc;
 
 	/* create a new class 'zcrypt' */
-	zcrypt_class = class_create(THIS_MODULE, ZCRYPT_NAME);
+	zcrypt_class = class_create(ZCRYPT_NAME);
 	if (IS_ERR(zcrypt_class)) {
 		rc = PTR_ERR(zcrypt_class);
 		goto out_class_create_failed;
diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c
index eb7e13486087..8acb9eba691b 100644
--- a/drivers/s390/net/ism_drv.c
+++ b/drivers/s390/net/ism_drv.c
@@ -11,7 +11,6 @@
 #include <linux/types.h>
 #include <linux/interrupt.h>
 #include <linux/device.h>
-#include <linux/pci.h>
 #include <linux/err.h>
 #include <linux/ctype.h>
 #include <linux/processor.h>
@@ -676,7 +675,6 @@ static int ism_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	return 0;
 
 err_resource:
-	pci_clear_master(pdev);
 	pci_release_mem_regions(pdev);
 err_disable:
 	pci_disable_device(pdev);
@@ -739,7 +737,6 @@ static void ism_remove(struct pci_dev *pdev)
 	ism_dev_exit(ism);
 	mutex_unlock(&ism_dev_list.mutex);
 
-	pci_clear_master(pdev);
 	pci_release_mem_regions(pdev);
 	pci_disable_device(pdev);
 	device_del(&ism->dev);
@@ -842,6 +839,12 @@ static int smcd_move(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx,
 	return ism_move(smcd->priv, dmb_tok, idx, sf, offset, data, size);
 }
 
+static int smcd_supports_v2(void)
+{
+	return SYSTEM_EID.serial_number[0] != '0' ||
+		SYSTEM_EID.type[0] != '0';
+}
+
 static u64 smcd_get_local_gid(struct smcd_dev *smcd)
 {
 	return ism_get_local_gid(smcd->priv);
@@ -869,6 +872,7 @@ static const struct smcd_ops ism_ops = {
 	.reset_vlan_required = smcd_reset_vlan_required,
 	.signal_event = smcd_signal_ieq,
 	.move_data = smcd_move,
+	.supports_v2 = smcd_supports_v2,
 	.get_system_eid = ism_get_seid,
 	.get_local_gid = smcd_get_local_gid,
 	.get_chid = smcd_get_chid,
diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c
index 3dbf4b21d127..b2a8cd792266 100644
--- a/drivers/s390/scsi/zfcp_scsi.c
+++ b/drivers/s390/scsi/zfcp_scsi.c
@@ -418,7 +418,7 @@ static int zfcp_scsi_sysfs_host_reset(struct Scsi_Host *shost, int reset_type)
 
 struct scsi_transport_template *zfcp_scsi_transport_template;
 
-static struct scsi_host_template zfcp_scsi_host_template = {
+static const struct scsi_host_template zfcp_scsi_host_template = {
 	.module			 = THIS_MODULE,
 	.name			 = "zfcp",
 	.queuecommand		 = zfcp_scsi_queuecommand,
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 954fc31b4bc7..02922768b129 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -391,7 +391,7 @@ static void virtio_ccw_drop_indicator(struct virtio_ccw_device *vcdev,
 	ccw_device_dma_free(vcdev->cdev, thinint_area, sizeof(*thinint_area));
 }
 
-static bool virtio_ccw_kvm_notify(struct virtqueue *vq)
+static inline bool virtio_ccw_do_kvm_notify(struct virtqueue *vq, u32 data)
 {
 	struct virtio_ccw_vq_info *info = vq->priv;
 	struct virtio_ccw_device *vcdev;
@@ -402,12 +402,22 @@ static bool virtio_ccw_kvm_notify(struct virtqueue *vq)
 	BUILD_BUG_ON(sizeof(struct subchannel_id) != sizeof(unsigned int));
 	info->cookie = kvm_hypercall3(KVM_S390_VIRTIO_CCW_NOTIFY,
 				      *((unsigned int *)&schid),
-				      vq->index, info->cookie);
+				      data, info->cookie);
 	if (info->cookie < 0)
 		return false;
 	return true;
 }
 
+static bool virtio_ccw_kvm_notify(struct virtqueue *vq)
+{
+	return virtio_ccw_do_kvm_notify(vq, vq->index);
+}
+
+static bool virtio_ccw_kvm_notify_with_data(struct virtqueue *vq)
+{
+	return virtio_ccw_do_kvm_notify(vq, vring_notification_data(vq));
+}
+
 static int virtio_ccw_read_vq_conf(struct virtio_ccw_device *vcdev,
 				   struct ccw1 *ccw, int index)
 {
@@ -495,6 +505,7 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
 					     struct ccw1 *ccw)
 {
 	struct virtio_ccw_device *vcdev = to_vc_device(vdev);
+	bool (*notify)(struct virtqueue *vq);
 	int err;
 	struct virtqueue *vq = NULL;
 	struct virtio_ccw_vq_info *info;
@@ -502,6 +513,11 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
 	unsigned long flags;
 	bool may_reduce;
 
+	if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA))
+		notify = virtio_ccw_kvm_notify_with_data;
+	else
+		notify = virtio_ccw_kvm_notify;
+
 	/* Allocate queue. */
 	info = kzalloc(sizeof(struct virtio_ccw_vq_info), GFP_KERNEL);
 	if (!info) {
@@ -524,7 +540,7 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
 	may_reduce = vcdev->revision > 0;
 	vq = vring_create_virtqueue(i, info->num, KVM_VIRTIO_CCW_RING_ALIGN,
 				    vdev, true, may_reduce, ctx,
-				    virtio_ccw_kvm_notify, callback, name);
+				    notify, callback, name);
 
 	if (!vq) {
 		/* For now, we fail if we can't get the requested size. */
diff --git a/drivers/sbus/char/display7seg.c b/drivers/sbus/char/display7seg.c
index d93595b39afa..5368b6ba2884 100644
--- a/drivers/sbus/char/display7seg.c
+++ b/drivers/sbus/char/display7seg.c
@@ -200,9 +200,8 @@ static int d7s_probe(struct platform_device *op)
 	 */
 	regs = readb(p->regs);
 	opts = of_find_node_by_path("/options");
-	if (opts &&
-	    of_get_property(opts, "d7s-flipped?", NULL))
-		p->flipped = true;
+	if (opts)
+	    p->flipped = of_property_read_bool(opts, "d7s-flipped?");
 
 	if (p->flipped)
 		regs |= D7S_FLIP;
diff --git a/drivers/sbus/char/oradax.c b/drivers/sbus/char/oradax.c
index e300cf26bc2a..aafce8d00000 100644
--- a/drivers/sbus/char/oradax.c
+++ b/drivers/sbus/char/oradax.c
@@ -18,7 +18,7 @@
  * the recommended way for applications to use the coprocessor, and
  * the driver interface is not intended for general use.
  *
- * See Documentation/sparc/oradax/oracle-dax.rst for more details.
+ * See Documentation/arch/sparc/oradax/oracle-dax.rst for more details.
  */
 
 #include <linux/uaccess.h>
@@ -323,7 +323,7 @@ static int __init dax_attach(void)
 		goto done;
 	}
 
-	cl = class_create(THIS_MODULE, DAX_NAME);
+	cl = class_create(DAX_NAME);
 	if (IS_ERR(cl)) {
 		dax_err("class_create failed");
 		ret = PTR_ERR(cl);
diff --git a/drivers/scsi/3w-9xxx.c b/drivers/scsi/3w-9xxx.c
index 6cb9cca9565b..38d20a69ee12 100644
--- a/drivers/scsi/3w-9xxx.c
+++ b/drivers/scsi/3w-9xxx.c
@@ -1976,8 +1976,7 @@ static int twa_slave_configure(struct scsi_device *sdev)
 	return 0;
 } /* End twa_slave_configure() */
 
-/* scsi_host_template initializer */
-static struct scsi_host_template driver_template = {
+static const struct scsi_host_template driver_template = {
 	.module			= THIS_MODULE,
 	.name			= "3ware 9000 Storage Controller",
 	.queuecommand		= twa_scsi_queue,
diff --git a/drivers/scsi/3w-sas.c b/drivers/scsi/3w-sas.c
index f41c93454f0c..55989eaa2d9f 100644
--- a/drivers/scsi/3w-sas.c
+++ b/drivers/scsi/3w-sas.c
@@ -1530,8 +1530,7 @@ static int twl_slave_configure(struct scsi_device *sdev)
 	return 0;
 } /* End twl_slave_configure() */
 
-/* scsi_host_template initializer */
-static struct scsi_host_template driver_template = {
+static const struct scsi_host_template driver_template = {
 	.module			= THIS_MODULE,
 	.name			= "3w-sas",
 	.queuecommand		= twl_scsi_queue,
diff --git a/drivers/scsi/3w-xxxx.c b/drivers/scsi/3w-xxxx.c
index ffdecb12d654..36c34ced0cc1 100644
--- a/drivers/scsi/3w-xxxx.c
+++ b/drivers/scsi/3w-xxxx.c
@@ -2229,7 +2229,7 @@ static int tw_slave_configure(struct scsi_device *sdev)
 	return 0;
 } /* End tw_slave_configure() */
 
-static struct scsi_host_template driver_template = {
+static const struct scsi_host_template driver_template = {
 	.module			= THIS_MODULE,
 	.name			= "3ware Storage Controller",
 	.queuecommand		= tw_scsi_queue,
diff --git a/drivers/scsi/BusLogic.c b/drivers/scsi/BusLogic.c
index f7b7ffda1161..72ceaf650b0d 100644
--- a/drivers/scsi/BusLogic.c
+++ b/drivers/scsi/BusLogic.c
@@ -54,7 +54,7 @@
 #define FAILURE (-1)
 #endif
 
-static struct scsi_host_template blogic_template;
+static const struct scsi_host_template blogic_template;
 
 /*
   blogic_drvr_options_count is a count of the number of BusLogic Driver
@@ -3663,7 +3663,7 @@ static int __init blogic_parseopts(char *options)
   Get it all started
 */
 
-static struct scsi_host_template blogic_template = {
+static const struct scsi_host_template blogic_template = {
 	.module = THIS_MODULE,
 	.proc_name = "BusLogic",
 	.write_info = blogic_write_info,
diff --git a/drivers/scsi/a100u2w.c b/drivers/scsi/a100u2w.c
index d02eb5b213d0..b95147fb18b0 100644
--- a/drivers/scsi/a100u2w.c
+++ b/drivers/scsi/a100u2w.c
@@ -1065,7 +1065,7 @@ static irqreturn_t inia100_intr(int irqno, void *devid)
 	return res;
 }
 
-static struct scsi_host_template inia100_template = {
+static const struct scsi_host_template inia100_template = {
 	.proc_name		= "inia100",
 	.name			= inia100_REVID,
 	.queuecommand		= inia100_queue,
diff --git a/drivers/scsi/a2091.c b/drivers/scsi/a2091.c
index 74312400468b..204448bfd04b 100644
--- a/drivers/scsi/a2091.c
+++ b/drivers/scsi/a2091.c
@@ -180,7 +180,7 @@ static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
 	}
 }
 
-static struct scsi_host_template a2091_scsi_template = {
+static const struct scsi_host_template a2091_scsi_template = {
 	.module			= THIS_MODULE,
 	.name			= "Commodore A2091/A590 SCSI",
 	.show_info		= wd33c93_show_info,
diff --git a/drivers/scsi/a3000.c b/drivers/scsi/a3000.c
index 2c5cb1a02e86..c3028726bbe4 100644
--- a/drivers/scsi/a3000.c
+++ b/drivers/scsi/a3000.c
@@ -197,7 +197,7 @@ static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
 	}
 }
 
-static struct scsi_host_template amiga_a3000_scsi_template = {
+static const struct scsi_host_template amiga_a3000_scsi_template = {
 	.module			= THIS_MODULE,
 	.name			= "Amiga 3000 built-in SCSI",
 	.show_info		= wd33c93_show_info,
diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c
index 5ba5c18b77b4..68f4dbcfff49 100644
--- a/drivers/scsi/aacraid/linit.c
+++ b/drivers/scsi/aacraid/linit.c
@@ -26,7 +26,6 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/slab.h>
 #include <linux/mutex.h>
 #include <linux/spinlock.h>
@@ -1476,7 +1475,7 @@ static const struct file_operations aac_cfg_fops = {
 	.llseek		= noop_llseek,
 };
 
-static struct scsi_host_template aac_driver_template = {
+static const struct scsi_host_template aac_driver_template = {
 	.module				= THIS_MODULE,
 	.name				= "AAC",
 	.proc_name			= AAC_DRIVERNAME,
@@ -1783,7 +1782,6 @@ static int aac_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	aac_scan_host(aac);
 
-	pci_enable_pcie_error_reporting(pdev);
 	pci_save_state(pdev);
 
 	return 0;
@@ -1949,7 +1947,6 @@ static pci_ers_result_t aac_pci_error_detected(struct pci_dev *pdev,
 		scsi_host_complete_all_commands(shost, DID_NO_CONNECT);
 		aac_release_resources(aac);
 
-		pci_disable_pcie_error_reporting(pdev);
 		aac_adapter_ioremap(aac, 0);
 
 		return PCI_ERS_RESULT_NEED_RESET;
diff --git a/drivers/scsi/advansys.c b/drivers/scsi/advansys.c
index f301aec044bb..ab066bb27a57 100644
--- a/drivers/scsi/advansys.c
+++ b/drivers/scsi/advansys.c
@@ -10602,7 +10602,7 @@ static int AdvInitGetConfig(struct pci_dev *pdev, struct Scsi_Host *shost)
 }
 #endif
 
-static struct scsi_host_template advansys_template = {
+static const struct scsi_host_template advansys_template = {
 	.proc_name = DRV_NAME,
 #ifdef CONFIG_PROC_FS
 	.show_info = advansys_show_info,
diff --git a/drivers/scsi/aha152x.c b/drivers/scsi/aha152x.c
index caeebfb67149..055adb349b0e 100644
--- a/drivers/scsi/aha152x.c
+++ b/drivers/scsi/aha152x.c
@@ -400,7 +400,7 @@ MODULE_DEVICE_TABLE(isapnp, id_table);
 
 #endif /* !AHA152X_PCMCIA */
 
-static struct scsi_host_template aha152x_driver_template;
+static const struct scsi_host_template aha152x_driver_template;
 
 /*
  * internal states of the host
@@ -2946,7 +2946,7 @@ static int aha152x_adjust_queue(struct scsi_device *device)
 	return 0;
 }
 
-static struct scsi_host_template aha152x_driver_template = {
+static const struct scsi_host_template aha152x_driver_template = {
 	.module				= THIS_MODULE,
 	.name				= AHA152X_REVID,
 	.proc_name			= "aha152x",
diff --git a/drivers/scsi/aha1542.c b/drivers/scsi/aha1542.c
index 552ca95157da..9503996c6325 100644
--- a/drivers/scsi/aha1542.c
+++ b/drivers/scsi/aha1542.c
@@ -737,7 +737,8 @@ fail:
 }
 
 /* return non-zero on detection */
-static struct Scsi_Host *aha1542_hw_init(struct scsi_host_template *tpnt, struct device *pdev, int indx)
+static struct Scsi_Host *aha1542_hw_init(const struct scsi_host_template *tpnt,
+					 struct device *pdev, int indx)
 {
 	unsigned int base_io = io[indx];
 	struct Scsi_Host *sh;
@@ -1031,7 +1032,7 @@ static int aha1542_exit_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
 	return 0;
 }
 
-static struct scsi_host_template driver_template = {
+static const struct scsi_host_template driver_template = {
 	.module			= THIS_MODULE,
 	.proc_name		= "aha1542",
 	.name			= "Adaptec 1542",
diff --git a/drivers/scsi/aha1740.c b/drivers/scsi/aha1740.c
index 134255751819..3d18945abaf7 100644
--- a/drivers/scsi/aha1740.c
+++ b/drivers/scsi/aha1740.c
@@ -543,7 +543,7 @@ static int aha1740_eh_abort_handler (struct scsi_cmnd *dummy)
 	return SUCCESS;
 }
 
-static struct scsi_host_template aha1740_template = {
+static const struct scsi_host_template aha1740_template = {
 	.module           = THIS_MODULE,
 	.proc_name        = "aha1740",
 	.show_info        = aha1740_show_info,
diff --git a/drivers/scsi/aic94xx/aic94xx_init.c b/drivers/scsi/aic94xx/aic94xx_init.c
index 954d0c5ae2e2..f7f81f6c3fbf 100644
--- a/drivers/scsi/aic94xx/aic94xx_init.c
+++ b/drivers/scsi/aic94xx/aic94xx_init.c
@@ -35,7 +35,7 @@ static struct scsi_transport_template *aic94xx_transport_template;
 static int asd_scan_finished(struct Scsi_Host *, unsigned long);
 static void asd_scan_start(struct Scsi_Host *);
 
-static struct scsi_host_template aic94xx_sht = {
+static const struct scsi_host_template aic94xx_sht = {
 	.module			= THIS_MODULE,
 	/* .name is initialized */
 	.name			= "aic94xx",
diff --git a/drivers/scsi/am53c974.c b/drivers/scsi/am53c974.c
index b69edb473295..fbb29dbb1e50 100644
--- a/drivers/scsi/am53c974.c
+++ b/drivers/scsi/am53c974.c
@@ -371,7 +371,7 @@ static void dc390_check_eeprom(struct esp *esp)
 static int pci_esp_probe_one(struct pci_dev *pdev,
 			      const struct pci_device_id *id)
 {
-	struct scsi_host_template *hostt = &scsi_esp_template;
+	const struct scsi_host_template *hostt = &scsi_esp_template;
 	int err = -ENODEV;
 	struct Scsi_Host *shost;
 	struct esp *esp;
diff --git a/drivers/scsi/arcmsr/arcmsr.h b/drivers/scsi/arcmsr/arcmsr.h
index 07df255c4b1b..ed8d9319862a 100644
--- a/drivers/scsi/arcmsr/arcmsr.h
+++ b/drivers/scsi/arcmsr/arcmsr.h
@@ -45,11 +45,12 @@
 #include <linux/interrupt.h>
 struct device_attribute;
 /*The limit of outstanding scsi command that firmware can handle*/
+#define ARCMSR_NAME			"arcmsr"
 #define ARCMSR_MAX_FREECCB_NUM		1024
 #define ARCMSR_MAX_OUTSTANDING_CMD	1024
 #define ARCMSR_DEFAULT_OUTSTANDING_CMD	128
 #define ARCMSR_MIN_OUTSTANDING_CMD	32
-#define ARCMSR_DRIVER_VERSION		"v1.50.00.05-20210429"
+#define ARCMSR_DRIVER_VERSION		"v1.50.00.13-20230206"
 #define ARCMSR_SCSI_INITIATOR_ID	255
 #define ARCMSR_MAX_XFER_SECTORS		512
 #define ARCMSR_MAX_XFER_SECTORS_B	4096
diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c
index d3fb8a9c1c39..2cd12c7f06c6 100644
--- a/drivers/scsi/arcmsr/arcmsr_hba.c
+++ b/drivers/scsi/arcmsr/arcmsr_hba.c
@@ -57,7 +57,6 @@
 #include <linux/timer.h>
 #include <linux/slab.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/circ_buf.h>
 #include <asm/dma.h>
 #include <asm/io.h>
@@ -152,8 +151,9 @@ static int arcmsr_adjust_disk_queue_depth(struct scsi_device *sdev, int queue_de
 	return scsi_change_queue_depth(sdev, queue_depth);
 }
 
-static struct scsi_host_template arcmsr_scsi_host_template = {
+static const struct scsi_host_template arcmsr_scsi_host_template = {
 	.module			= THIS_MODULE,
+	.proc_name		= ARCMSR_NAME,
 	.name			= "Areca SAS/SATA RAID driver",
 	.info			= arcmsr_info,
 	.queuecommand		= arcmsr_queue_command,
@@ -997,6 +997,8 @@ static int arcmsr_set_dma_mask(struct AdapterControlBlock *acb)
 		if (((acb->adapter_type == ACB_ADAPTER_TYPE_A) && !dma_mask_64) ||
 		    dma_set_mask(&pcidev->dev, DMA_BIT_MASK(64)))
 			goto	dma32;
+		if (acb->adapter_type <= ACB_ADAPTER_TYPE_B)
+			return 0;
 		if (dma_set_coherent_mask(&pcidev->dev, DMA_BIT_MASK(64)) ||
 		    dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(64))) {
 			printk("arcmsr: set DMA 64 mask failed\n");
@@ -1300,20 +1302,13 @@ static uint8_t arcmsr_abort_allcmd(struct AdapterControlBlock *acb)
 	return rtnval;
 }
 
-static void arcmsr_pci_unmap_dma(struct CommandControlBlock *ccb)
-{
-	struct scsi_cmnd *pcmd = ccb->pcmd;
-
-	scsi_dma_unmap(pcmd);
-}
-
 static void arcmsr_ccb_complete(struct CommandControlBlock *ccb)
 {
 	struct AdapterControlBlock *acb = ccb->acb;
 	struct scsi_cmnd *pcmd = ccb->pcmd;
 	unsigned long flags;
 	atomic_dec(&acb->ccboutstandingcount);
-	arcmsr_pci_unmap_dma(ccb);
+	scsi_dma_unmap(ccb->pcmd);
 	ccb->startdone = ARCMSR_CCB_DONE;
 	spin_lock_irqsave(&acb->ccblist_lock, flags);
 	list_add_tail(&ccb->list, &acb->ccb_free_list);
@@ -1597,7 +1592,7 @@ static void arcmsr_remove_scsi_devices(struct AdapterControlBlock *acb)
 		ccb = acb->pccb_pool[i];
 		if (ccb->startdone == ARCMSR_CCB_START) {
 			ccb->pcmd->result = DID_NO_CONNECT << 16;
-			arcmsr_pci_unmap_dma(ccb);
+			scsi_dma_unmap(ccb->pcmd);
 			scsi_done(ccb->pcmd);
 		}
 	}
@@ -2260,8 +2255,11 @@ static void arcmsr_iop2drv_data_wrote_handle(struct AdapterControlBlock *acb)
 
 	spin_lock_irqsave(&acb->rqbuffer_lock, flags);
 	prbuffer = arcmsr_get_iop_rqbuffer(acb);
-	buf_empty_len = (acb->rqbuf_putIndex - acb->rqbuf_getIndex - 1) &
-		(ARCMSR_MAX_QBUFFER - 1);
+	if (acb->rqbuf_putIndex >= acb->rqbuf_getIndex) {
+		buf_empty_len = (ARCMSR_MAX_QBUFFER - 1) -
+		(acb->rqbuf_putIndex - acb->rqbuf_getIndex);
+	} else
+		buf_empty_len = acb->rqbuf_getIndex - acb->rqbuf_putIndex - 1;
 	if (buf_empty_len >= readl(&prbuffer->data_len)) {
 		if (arcmsr_Read_iop_rqbuffer_data(acb, prbuffer) == 0)
 			acb->acb_flags |= ACB_F_IOPDATA_OVERFLOW;
diff --git a/drivers/scsi/arm/acornscsi.c b/drivers/scsi/arm/acornscsi.c
index 7602639da9b3..0b046e4b395c 100644
--- a/drivers/scsi/arm/acornscsi.c
+++ b/drivers/scsi/arm/acornscsi.c
@@ -2780,7 +2780,7 @@ static int acornscsi_show_info(struct seq_file *m, struct Scsi_Host *instance)
     return 0;
 }
 
-static struct scsi_host_template acornscsi_template = {
+static const struct scsi_host_template acornscsi_template = {
 	.module			= THIS_MODULE,
 	.show_info		= acornscsi_show_info,
 	.name			= "AcornSCSI",
diff --git a/drivers/scsi/arm/arxescsi.c b/drivers/scsi/arm/arxescsi.c
index 2527b542bcdd..925d0bd68aa5 100644
--- a/drivers/scsi/arm/arxescsi.c
+++ b/drivers/scsi/arm/arxescsi.c
@@ -238,7 +238,7 @@ arxescsi_show_info(struct seq_file *m, struct Scsi_Host *host)
 	return 0;
 }
 
-static struct scsi_host_template arxescsi_template = {
+static const struct scsi_host_template arxescsi_template = {
 	.show_info			= arxescsi_show_info,
 	.name				= "ARXE SCSI card",
 	.info				= arxescsi_info,
diff --git a/drivers/scsi/arm/cumana_1.c b/drivers/scsi/arm/cumana_1.c
index 5d4f67ba74c0..d1a2a22ffe8c 100644
--- a/drivers/scsi/arm/cumana_1.c
+++ b/drivers/scsi/arm/cumana_1.c
@@ -211,7 +211,7 @@ static void cumanascsi_write(struct NCR5380_hostdata *hostdata,
 
 #include "../NCR5380.c"
 
-static struct scsi_host_template cumanascsi_template = {
+static const struct scsi_host_template cumanascsi_template = {
 	.module			= THIS_MODULE,
 	.name			= "Cumana 16-bit SCSI",
 	.info			= cumanascsi_info,
diff --git a/drivers/scsi/arm/cumana_2.c b/drivers/scsi/arm/cumana_2.c
index d15053f02472..c5d8f4313b31 100644
--- a/drivers/scsi/arm/cumana_2.c
+++ b/drivers/scsi/arm/cumana_2.c
@@ -356,7 +356,7 @@ static int cumanascsi_2_show_info(struct seq_file *m, struct Scsi_Host *host)
 	return 0;
 }
 
-static struct scsi_host_template cumanascsi2_template = {
+static const struct scsi_host_template cumanascsi2_template = {
 	.module				= THIS_MODULE,
 	.show_info			= cumanascsi_2_show_info,
 	.write_info			= cumanascsi_2_set_proc_info,
diff --git a/drivers/scsi/arm/eesox.c b/drivers/scsi/arm/eesox.c
index 6f374af9f45f..b3ec7635bc72 100644
--- a/drivers/scsi/arm/eesox.c
+++ b/drivers/scsi/arm/eesox.c
@@ -473,7 +473,7 @@ static ssize_t eesoxscsi_store_term(struct device *dev, struct device_attribute
 static DEVICE_ATTR(bus_term, S_IRUGO | S_IWUSR,
 		   eesoxscsi_show_term, eesoxscsi_store_term);
 
-static struct scsi_host_template eesox_template = {
+static const struct scsi_host_template eesox_template = {
 	.module				= THIS_MODULE,
 	.show_info			= eesoxscsi_show_info,
 	.write_info			= eesoxscsi_set_proc_info,
diff --git a/drivers/scsi/arm/oak.c b/drivers/scsi/arm/oak.c
index f18a0620c808..d69245007096 100644
--- a/drivers/scsi/arm/oak.c
+++ b/drivers/scsi/arm/oak.c
@@ -100,7 +100,7 @@ printk("reading %p len %d\n", addr, len);
 
 #include "../NCR5380.c"
 
-static struct scsi_host_template oakscsi_template = {
+static const struct scsi_host_template oakscsi_template = {
 	.module			= THIS_MODULE,
 	.name			= "Oak 16-bit SCSI",
 	.info			= oakscsi_info,
diff --git a/drivers/scsi/arm/powertec.c b/drivers/scsi/arm/powertec.c
index 7586d2a03812..3b5991427886 100644
--- a/drivers/scsi/arm/powertec.c
+++ b/drivers/scsi/arm/powertec.c
@@ -279,7 +279,7 @@ powertecscsi_store_term(struct device *dev, struct device_attribute *attr, const
 static DEVICE_ATTR(bus_term, S_IRUGO | S_IWUSR,
 		   powertecscsi_show_term, powertecscsi_store_term);
 
-static struct scsi_host_template powertecscsi_template = {
+static const struct scsi_host_template powertecscsi_template = {
 	.module				= THIS_MODULE,
 	.show_info			= powertecscsi_show_info,
 	.write_info			= powertecscsi_set_proc_info,
diff --git a/drivers/scsi/atp870u.c b/drivers/scsi/atp870u.c
index 7143418d690f..2a748af269c2 100644
--- a/drivers/scsi/atp870u.c
+++ b/drivers/scsi/atp870u.c
@@ -40,7 +40,7 @@
 
 #include "atp870u.h"
 
-static struct scsi_host_template atp870u_template;
+static const struct scsi_host_template atp870u_template;
 static void send_s870(struct atp_unit *dev,unsigned char c);
 static void atp_is(struct atp_unit *dev, unsigned char c, bool wide_chip,
 		   unsigned char lvdmode);
@@ -1726,7 +1726,7 @@ static void atp870u_remove (struct pci_dev *pdev)
 }
 MODULE_LICENSE("GPL");
 
-static struct scsi_host_template atp870u_template = {
+static const struct scsi_host_template atp870u_template = {
      .module			= THIS_MODULE,
      .name			= "atp870u"		/* name */,
      .proc_name			= "atp870u",
diff --git a/drivers/scsi/be2iscsi/be_cmds.c b/drivers/scsi/be2iscsi/be_cmds.c
index 69b1a80e3687..0b59b63bce79 100644
--- a/drivers/scsi/be2iscsi/be_cmds.c
+++ b/drivers/scsi/be2iscsi/be_cmds.c
@@ -1173,7 +1173,6 @@ int be_cmd_iscsi_post_sgl_pages(struct be_ctrl_info *ctrl,
 	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
 	int status;
 	unsigned int curr_pages;
-	u32 internal_page_offset = 0;
 	u32 temp_num_pages = num_pages;
 
 	if (num_pages == 0xff)
@@ -1192,7 +1191,6 @@ int be_cmd_iscsi_post_sgl_pages(struct be_ctrl_info *ctrl,
 		req->page_offset = page_offset;
 		be_cmd_page_addrs_prepare(req->pages, req->num_pages, q_mem);
 		q_mem->dma = q_mem->dma + (req->num_pages * PAGE_SIZE);
-		internal_page_offset += req->num_pages;
 		page_offset += req->num_pages;
 		num_pages -= req->num_pages;
 
diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c
index 50a577ac3bb4..e48f14ad6dfd 100644
--- a/drivers/scsi/be2iscsi/be_main.c
+++ b/drivers/scsi/be2iscsi/be_main.c
@@ -139,7 +139,7 @@ beiscsi_disp_param(_name)\
 beiscsi_change_param(_name, _minval, _maxval, _defval)\
 beiscsi_store_param(_name)\
 beiscsi_init_param(_name, _minval, _maxval, _defval)\
-DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
+static DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
 	      beiscsi_##_name##_disp, beiscsi_##_name##_store)
 
 /*
@@ -155,14 +155,14 @@ BEISCSI_RW_ATTR(log_enable, 0x00,
 		"\t\t\t\tConfiguration Path	: 0x20\n"
 		"\t\t\t\tiSCSI Protocol		: 0x40\n");
 
-DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
-DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL);
-DEVICE_ATTR(beiscsi_fw_ver, S_IRUGO, beiscsi_fw_ver_disp, NULL);
-DEVICE_ATTR(beiscsi_phys_port, S_IRUGO, beiscsi_phys_port_disp, NULL);
-DEVICE_ATTR(beiscsi_active_session_count, S_IRUGO,
-	     beiscsi_active_session_disp, NULL);
-DEVICE_ATTR(beiscsi_free_session_count, S_IRUGO,
-	     beiscsi_free_session_disp, NULL);
+static DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
+static DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL);
+static DEVICE_ATTR(beiscsi_fw_ver, S_IRUGO, beiscsi_fw_ver_disp, NULL);
+static DEVICE_ATTR(beiscsi_phys_port, S_IRUGO, beiscsi_phys_port_disp, NULL);
+static DEVICE_ATTR(beiscsi_active_session_count, S_IRUGO,
+		   beiscsi_active_session_disp, NULL);
+static DEVICE_ATTR(beiscsi_free_session_count, S_IRUGO,
+		   beiscsi_free_session_disp, NULL);
 
 static struct attribute *beiscsi_attrs[] = {
 	&dev_attr_beiscsi_log_enable.attr,
@@ -398,7 +398,7 @@ static const struct pci_device_id beiscsi_pci_id_table[] = {
 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 
 
-static struct scsi_host_template beiscsi_sht = {
+static const struct scsi_host_template beiscsi_sht = {
 	.module = THIS_MODULE,
 	.name = "Emulex 10Gbe open-iscsi Initiator Driver",
 	.proc_name = DRV_NAME,
@@ -5545,13 +5545,6 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
 		goto disable_pci;
 	}
 
-	/* Enable EEH reporting */
-	ret = pci_enable_pcie_error_reporting(pcidev);
-	if (ret)
-		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
-			    "BM_%d : PCIe Error Reporting "
-			    "Enabling Failed\n");
-
 	pci_save_state(pcidev);
 
 	/* Initialize Driver configuration Paramters */
@@ -5736,7 +5729,6 @@ free_hba:
 	pci_disable_msix(phba->pcidev);
 	pci_dev_put(phba->pcidev);
 	iscsi_host_free(phba->shost);
-	pci_disable_pcie_error_reporting(pcidev);
 	pci_set_drvdata(pcidev, NULL);
 disable_pci:
 	pci_release_regions(pcidev);
@@ -5779,7 +5771,6 @@ static void beiscsi_remove(struct pci_dev *pcidev)
 
 	pci_dev_put(phba->pcidev);
 	iscsi_host_free(phba->shost);
-	pci_disable_pcie_error_reporting(pcidev);
 	pci_set_drvdata(pcidev, NULL);
 	pci_release_regions(pcidev);
 	pci_disable_device(pcidev);
diff --git a/drivers/scsi/be2iscsi/be_main.h b/drivers/scsi/be2iscsi/be_main.h
index 98977c0700f1..71c95d144560 100644
--- a/drivers/scsi/be2iscsi/be_main.h
+++ b/drivers/scsi/be2iscsi/be_main.h
@@ -16,7 +16,6 @@
 #include <linux/in.h>
 #include <linux/ctype.h>
 #include <linux/module.h>
-#include <linux/aer.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_device.h>
diff --git a/drivers/scsi/bfa/bfad.c b/drivers/scsi/bfa/bfad.c
index e5aa982ffedc..529b73a83d69 100644
--- a/drivers/scsi/bfa/bfad.c
+++ b/drivers/scsi/bfa/bfad.c
@@ -738,9 +738,6 @@ bfad_pci_init(struct pci_dev *pdev, struct bfad_s *bfad)
 		goto out_release_region;
 	}
 
-	/* Enable PCIE Advanced Error Recovery (AER) if kernel supports */
-	pci_enable_pcie_error_reporting(pdev);
-
 	bfad->pci_bar0_kva = pci_iomap(pdev, 0, pci_resource_len(pdev, 0));
 	bfad->pci_bar2_kva = pci_iomap(pdev, 2, pci_resource_len(pdev, 2));
 
@@ -801,8 +798,6 @@ bfad_pci_uninit(struct pci_dev *pdev, struct bfad_s *bfad)
 	pci_iounmap(pdev, bfad->pci_bar0_kva);
 	pci_iounmap(pdev, bfad->pci_bar2_kva);
 	pci_release_regions(pdev);
-	/* Disable PCIE Advanced Error Recovery (AER) */
-	pci_disable_pcie_error_reporting(pdev);
 	pci_disable_device(pdev);
 }
 
@@ -1562,7 +1557,6 @@ bfad_pci_slot_reset(struct pci_dev *pdev)
 	if (restart_bfa(bfad) == -1)
 		goto out_disable_device;
 
-	pci_enable_pcie_error_reporting(pdev);
 	dev_printk(KERN_WARNING, &pdev->dev,
 		   "slot_reset completed  flags: 0x%x!\n", bfad->bfad_flags);
 
diff --git a/drivers/scsi/bfa/bfad_drv.h b/drivers/scsi/bfa/bfad_drv.h
index eaee7c8bc2d2..7682cfa34265 100644
--- a/drivers/scsi/bfa/bfad_drv.h
+++ b/drivers/scsi/bfa/bfad_drv.h
@@ -30,7 +30,6 @@
 #include <linux/vmalloc.h>
 #include <linux/workqueue.h>
 #include <linux/bitops.h>
-#include <linux/aer.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_tcq.h>
diff --git a/drivers/scsi/bnx2i/bnx2i_iscsi.c b/drivers/scsi/bnx2i/bnx2i_iscsi.c
index a3c800e04a2e..9971f32a663c 100644
--- a/drivers/scsi/bnx2i/bnx2i_iscsi.c
+++ b/drivers/scsi/bnx2i/bnx2i_iscsi.c
@@ -22,7 +22,7 @@
 
 struct scsi_transport_template *bnx2i_scsi_xport_template;
 struct iscsi_transport bnx2i_iscsi_transport;
-static struct scsi_host_template bnx2i_host_template;
+static const struct scsi_host_template bnx2i_host_template;
 
 /*
  * Global endpoint resource info
@@ -2250,7 +2250,7 @@ static umode_t bnx2i_attr_is_visible(int param_type, int param)
  * 'Scsi_Host_Template' structure and 'iscsi_tranport' structure template
  * used while registering with the scsi host and iSCSI transport module.
  */
-static struct scsi_host_template bnx2i_host_template = {
+static const struct scsi_host_template bnx2i_host_template = {
 	.module			= THIS_MODULE,
 	.name			= "QLogic Offload iSCSI Initiator",
 	.proc_name		= "bnx2i",
diff --git a/drivers/scsi/ch.c b/drivers/scsi/ch.c
index 72fe6df78bc5..ac648bb8f7e7 100644
--- a/drivers/scsi/ch.c
+++ b/drivers/scsi/ch.c
@@ -995,7 +995,7 @@ static int __init init_ch_module(void)
 	int rc;
 
 	printk(KERN_INFO "SCSI Media Changer driver v" VERSION " \n");
-        ch_sysfs_class = class_create(THIS_MODULE, "scsi_changer");
+        ch_sysfs_class = class_create("scsi_changer");
         if (IS_ERR(ch_sysfs_class)) {
 		rc = PTR_ERR(ch_sysfs_class);
 		return rc;
diff --git a/drivers/scsi/csiostor/csio_init.c b/drivers/scsi/csiostor/csio_init.c
index ccbded3353bd..0c32faefad7c 100644
--- a/drivers/scsi/csiostor/csio_init.c
+++ b/drivers/scsi/csiostor/csio_init.c
@@ -38,7 +38,6 @@
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/pci.h>
-#include <linux/aer.h>
 #include <linux/mm.h>
 #include <linux/notifier.h>
 #include <linux/kdebug.h>
diff --git a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
index ff9d4287937a..ec6530240707 100644
--- a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
+++ b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
@@ -80,7 +80,7 @@ static struct cxgb3_client t3_client = {
 	.event_handler = cxgb3i_dev_event_handler,
 };
 
-static struct scsi_host_template cxgb3i_host_template = {
+static const struct scsi_host_template cxgb3i_host_template = {
 	.module		= THIS_MODULE,
 	.name		= DRV_MODULE_NAME,
 	.proc_name	= DRV_MODULE_NAME,
diff --git a/drivers/scsi/cxgbi/libcxgbi.c b/drivers/scsi/cxgbi/libcxgbi.c
index af281e271f88..abde60a50cf7 100644
--- a/drivers/scsi/cxgbi/libcxgbi.c
+++ b/drivers/scsi/cxgbi/libcxgbi.c
@@ -337,7 +337,7 @@ void cxgbi_hbas_remove(struct cxgbi_device *cdev)
 EXPORT_SYMBOL_GPL(cxgbi_hbas_remove);
 
 int cxgbi_hbas_add(struct cxgbi_device *cdev, u64 max_lun,
-		unsigned int max_conns, struct scsi_host_template *sht,
+		unsigned int max_conns, const struct scsi_host_template *sht,
 		struct scsi_transport_template *stt)
 {
 	struct cxgbi_hba *chba;
@@ -2314,9 +2314,9 @@ static int cxgbi_sock_tx_queue_up(struct cxgbi_sock *csk, struct sk_buff *skb)
 		frags++;
 
 	if (frags >= SKB_WR_LIST_SIZE) {
-		pr_err("csk 0x%p, frags %u, %u,%u >%lu.\n",
+		pr_err("csk 0x%p, frags %u, %u,%u >%u.\n",
 		       csk, skb_shinfo(skb)->nr_frags, skb->len,
-		       skb->data_len, SKB_WR_LIST_SIZE);
+		       skb->data_len, (unsigned int)SKB_WR_LIST_SIZE);
 		return -EINVAL;
 	}
 
diff --git a/drivers/scsi/cxgbi/libcxgbi.h b/drivers/scsi/cxgbi/libcxgbi.h
index d8fc7beafa20..d92cf1dccc2f 100644
--- a/drivers/scsi/cxgbi/libcxgbi.h
+++ b/drivers/scsi/cxgbi/libcxgbi.h
@@ -591,7 +591,7 @@ struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *, int *);
 struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *,
 						     int *);
 int cxgbi_hbas_add(struct cxgbi_device *, u64, unsigned int,
-			struct scsi_host_template *,
+			const struct scsi_host_template *,
 			struct scsi_transport_template *);
 void cxgbi_hbas_remove(struct cxgbi_device *);
 
diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c
index 395b00b942f7..debd36974119 100644
--- a/drivers/scsi/cxlflash/main.c
+++ b/drivers/scsi/cxlflash/main.c
@@ -3880,7 +3880,7 @@ static int cxlflash_class_init(void)
 
 	cxlflash_major = MAJOR(devno);
 
-	cxlflash_class = class_create(THIS_MODULE, "cxlflash");
+	cxlflash_class = class_create("cxlflash");
 	if (IS_ERR(cxlflash_class)) {
 		rc = PTR_ERR(cxlflash_class);
 		pr_err("%s: class_create failed rc=%d\n", __func__, rc);
diff --git a/drivers/scsi/cxlflash/superpipe.c b/drivers/scsi/cxlflash/superpipe.c
index 22cfc2e1dfb9..e1b55b03e812 100644
--- a/drivers/scsi/cxlflash/superpipe.c
+++ b/drivers/scsi/cxlflash/superpipe.c
@@ -358,7 +358,7 @@ retry:
 	dev_dbg(dev, "%s: %ssending cmd(%02x)\n", __func__,
 		retry_cnt ? "re" : "", scsi_cmd[0]);
 
-	/* Drop the ioctl read semahpore across lengthy call */
+	/* Drop the ioctl read semaphore across lengthy call */
 	up_read(&cfg->ioctl_rwsem);
 	result = scsi_execute_cmd(sdev, scsi_cmd, REQ_OP_DRV_IN, cmd_buf,
 				  CMD_BUFSIZE, to, CMD_RETRIES, &exec_args);
diff --git a/drivers/scsi/cxlflash/vlun.c b/drivers/scsi/cxlflash/vlun.c
index 9caabf550436..cbd5a648a131 100644
--- a/drivers/scsi/cxlflash/vlun.c
+++ b/drivers/scsi/cxlflash/vlun.c
@@ -448,7 +448,7 @@ static int write_same16(struct scsi_device *sdev,
 		put_unaligned_be32(ws_limit < left ? ws_limit : left,
 				   &scsi_cmd[10]);
 
-		/* Drop the ioctl read semahpore across lengthy call */
+		/* Drop the ioctl read semaphore across lengthy call */
 		up_read(&cfg->ioctl_rwsem);
 		result = scsi_execute_cmd(sdev, scsi_cmd, REQ_OP_DRV_OUT,
 					  cmd_buf, CMD_BUFSIZE, to,
diff --git a/drivers/scsi/dc395x.c b/drivers/scsi/dc395x.c
index 670a836a6ba1..c8e86f8a631e 100644
--- a/drivers/scsi/dc395x.c
+++ b/drivers/scsi/dc395x.c
@@ -4541,7 +4541,7 @@ static int dc395x_show_info(struct seq_file *m, struct Scsi_Host *host)
 }
 
 
-static struct scsi_host_template dc395x_driver_template = {
+static const struct scsi_host_template dc395x_driver_template = {
 	.module                 = THIS_MODULE,
 	.proc_name              = DC395X_NAME,
 	.show_info              = dc395x_show_info,
diff --git a/drivers/scsi/dmx3191d.c b/drivers/scsi/dmx3191d.c
index a171ce6b70b2..dfb091d34363 100644
--- a/drivers/scsi/dmx3191d.c
+++ b/drivers/scsi/dmx3191d.c
@@ -39,7 +39,7 @@
 #define DMX3191D_REGION_LEN	8
 
 
-static struct scsi_host_template dmx3191d_driver_template = {
+static const struct scsi_host_template dmx3191d_driver_template = {
 	.module			= THIS_MODULE,
 	.proc_name		= DMX3191D_DRIVER_NAME,
 	.name			= "Domex DMX3191D",
diff --git a/drivers/scsi/elx/efct/efct_lio.c b/drivers/scsi/elx/efct/efct_lio.c
index be4b5c1ee32d..a982b9cf9870 100644
--- a/drivers/scsi/elx/efct/efct_lio.c
+++ b/drivers/scsi/elx/efct/efct_lio.c
@@ -285,11 +285,6 @@ efct_lio_npiv_check_prod_write_protect(struct se_portal_group *se_tpg)
 	return tpg->tpg_attrib.prod_mode_write_protect;
 }
 
-static u32 efct_lio_tpg_get_inst_index(struct se_portal_group *se_tpg)
-{
-	return 1;
-}
-
 static int efct_lio_check_stop_free(struct se_cmd *se_cmd)
 {
 	struct efct_scsi_tgt_io *ocp =
@@ -355,15 +350,6 @@ static void efct_lio_close_session(struct se_session *se_sess)
 	efc_node_post_shutdown(node, NULL);
 }
 
-static u32 efct_lio_sess_get_index(struct se_session *se_sess)
-{
-	return 0;
-}
-
-static void efct_lio_set_default_node_attrs(struct se_node_acl *nacl)
-{
-}
-
 static int efct_lio_get_cmd_state(struct se_cmd *cmd)
 {
 	struct efct_scsi_tgt_io *ocp =
@@ -1607,14 +1593,11 @@ static const struct target_core_fabric_ops efct_lio_ops = {
 	.tpg_check_demo_mode_cache      = efct_lio_check_demo_mode_cache,
 	.tpg_check_demo_mode_write_protect = efct_lio_check_demo_write_protect,
 	.tpg_check_prod_mode_write_protect = efct_lio_check_prod_write_protect,
-	.tpg_get_inst_index		= efct_lio_tpg_get_inst_index,
 	.check_stop_free		= efct_lio_check_stop_free,
 	.aborted_task			= efct_lio_aborted_task,
 	.release_cmd			= efct_lio_release_cmd,
 	.close_session			= efct_lio_close_session,
-	.sess_get_index			= efct_lio_sess_get_index,
 	.write_pending			= efct_lio_write_pending,
-	.set_default_node_attributes	= efct_lio_set_default_node_attrs,
 	.get_cmd_state			= efct_lio_get_cmd_state,
 	.queue_data_in			= efct_lio_queue_data_in,
 	.queue_status			= efct_lio_queue_status,
@@ -1644,14 +1627,11 @@ static const struct target_core_fabric_ops efct_lio_npiv_ops = {
 					efct_lio_npiv_check_demo_write_protect,
 	.tpg_check_prod_mode_write_protect =
 					efct_lio_npiv_check_prod_write_protect,
-	.tpg_get_inst_index		= efct_lio_tpg_get_inst_index,
 	.check_stop_free		= efct_lio_check_stop_free,
 	.aborted_task			= efct_lio_aborted_task,
 	.release_cmd			= efct_lio_release_cmd,
 	.close_session			= efct_lio_close_session,
-	.sess_get_index			= efct_lio_sess_get_index,
 	.write_pending			= efct_lio_write_pending,
-	.set_default_node_attributes	= efct_lio_set_default_node_attrs,
 	.get_cmd_state			= efct_lio_get_cmd_state,
 	.queue_data_in			= efct_lio_queue_data_in,
 	.queue_status			= efct_lio_queue_status,
diff --git a/drivers/scsi/elx/efct/efct_xport.c b/drivers/scsi/elx/efct/efct_xport.c
index 9495cedcc0b9..cf4dced20b8b 100644
--- a/drivers/scsi/elx/efct/efct_xport.c
+++ b/drivers/scsi/elx/efct/efct_xport.c
@@ -10,7 +10,7 @@
 static struct dentry *efct_debugfs_root;
 static atomic_t efct_debugfs_count;
 
-static struct scsi_host_template efct_template = {
+static const struct scsi_host_template efct_template = {
 	.module			= THIS_MODULE,
 	.name			= EFCT_DRIVER_NAME,
 	.supported_mode		= MODE_TARGET,
diff --git a/drivers/scsi/esas2r/esas2r_ioctl.c b/drivers/scsi/esas2r/esas2r_ioctl.c
index e003d923acbf..055d2e87a2c8 100644
--- a/drivers/scsi/esas2r/esas2r_ioctl.c
+++ b/drivers/scsi/esas2r/esas2r_ioctl.c
@@ -56,7 +56,7 @@ dma_addr_t esas2r_buffered_ioctl_addr;
 u32 esas2r_buffered_ioctl_size;
 struct pci_dev *esas2r_buffered_ioctl_pcid;
 
-static DEFINE_SEMAPHORE(buffered_ioctl_semaphore);
+static DEFINE_SEMAPHORE(buffered_ioctl_semaphore, 1);
 typedef int (*BUFFERED_IOCTL_CALLBACK)(struct esas2r_adapter *,
 				       struct esas2r_request *,
 				       struct esas2r_sg_context *,
diff --git a/drivers/scsi/esas2r/esas2r_main.c b/drivers/scsi/esas2r/esas2r_main.c
index d7a2c49ff5ee..f700a16cd885 100644
--- a/drivers/scsi/esas2r/esas2r_main.c
+++ b/drivers/scsi/esas2r/esas2r_main.c
@@ -231,7 +231,7 @@ struct bin_attribute bin_attr_default_nvram = {
 	.write	= NULL
 };
 
-static struct scsi_host_template driver_template = {
+static const struct scsi_host_template driver_template = {
 	.module				= THIS_MODULE,
 	.show_info			= esas2r_show_info,
 	.name				= ESAS2R_LONGNAME,
diff --git a/drivers/scsi/esp_scsi.c b/drivers/scsi/esp_scsi.c
index 64ec6bb84550..97816a0e6240 100644
--- a/drivers/scsi/esp_scsi.c
+++ b/drivers/scsi/esp_scsi.c
@@ -2660,7 +2660,7 @@ static const char *esp_info(struct Scsi_Host *host)
 	return "esp";
 }
 
-struct scsi_host_template scsi_esp_template = {
+const struct scsi_host_template scsi_esp_template = {
 	.module			= THIS_MODULE,
 	.name			= "esp",
 	.info			= esp_info,
diff --git a/drivers/scsi/esp_scsi.h b/drivers/scsi/esp_scsi.h
index c73760d3cf83..00cd7c0ccc76 100644
--- a/drivers/scsi/esp_scsi.h
+++ b/drivers/scsi/esp_scsi.h
@@ -572,7 +572,7 @@ struct esp {
  * 13) Check scsi_esp_register() return value, release all resources
  *     if an error was returned.
  */
-extern struct scsi_host_template scsi_esp_template;
+extern const struct scsi_host_template scsi_esp_template;
 extern int scsi_esp_register(struct esp *);
 
 extern void scsi_esp_unregister(struct esp *);
diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c
index 38774a272e62..f1429f270170 100644
--- a/drivers/scsi/fcoe/fcoe.c
+++ b/drivers/scsi/fcoe/fcoe.c
@@ -260,7 +260,7 @@ static struct fc_function_template fcoe_vport_fc_functions = {
 	.bsg_request = fc_lport_bsg_request,
 };
 
-static struct scsi_host_template fcoe_shost_template = {
+static const struct scsi_host_template fcoe_shost_template = {
 	.module = THIS_MODULE,
 	.name = "FCoE Driver",
 	.proc_name = FCOE_NAME,
diff --git a/drivers/scsi/fcoe/fcoe_sysfs.c b/drivers/scsi/fcoe/fcoe_sysfs.c
index 6260aa5ea6af..e17957f8085c 100644
--- a/drivers/scsi/fcoe/fcoe_sysfs.c
+++ b/drivers/scsi/fcoe/fcoe_sysfs.c
@@ -659,17 +659,17 @@ static const struct device_type fcoe_fcf_device_type = {
 	.release = fcoe_fcf_device_release,
 };
 
-static ssize_t ctlr_create_store(struct bus_type *bus, const char *buf,
+static ssize_t ctlr_create_store(const struct bus_type *bus, const char *buf,
 				 size_t count)
 {
-	return fcoe_ctlr_create_store(bus, buf, count);
+	return fcoe_ctlr_create_store(buf, count);
 }
 static BUS_ATTR_WO(ctlr_create);
 
-static ssize_t ctlr_destroy_store(struct bus_type *bus, const char *buf,
+static ssize_t ctlr_destroy_store(const struct bus_type *bus, const char *buf,
 				  size_t count)
 {
-	return fcoe_ctlr_destroy_store(bus, buf, count);
+	return fcoe_ctlr_destroy_store(buf, count);
 }
 static BUS_ATTR_WO(ctlr_destroy);
 
diff --git a/drivers/scsi/fcoe/fcoe_transport.c b/drivers/scsi/fcoe/fcoe_transport.c
index 62341c6353a7..46b0bf237be1 100644
--- a/drivers/scsi/fcoe/fcoe_transport.c
+++ b/drivers/scsi/fcoe/fcoe_transport.c
@@ -745,8 +745,7 @@ static int libfcoe_device_notification(struct notifier_block *notifier,
 	return NOTIFY_OK;
 }
 
-ssize_t fcoe_ctlr_create_store(struct bus_type *bus,
-			       const char *buf, size_t count)
+ssize_t fcoe_ctlr_create_store(const char *buf, size_t count)
 {
 	struct net_device *netdev = NULL;
 	struct fcoe_transport *ft = NULL;
@@ -808,8 +807,7 @@ out_nodev:
 	return count;
 }
 
-ssize_t fcoe_ctlr_destroy_store(struct bus_type *bus,
-				const char *buf, size_t count)
+ssize_t fcoe_ctlr_destroy_store(const char *buf, size_t count)
 {
 	int rc = -ENODEV;
 	struct net_device *netdev = NULL;
diff --git a/drivers/scsi/fdomain.c b/drivers/scsi/fdomain.c
index 444eac9b2466..504c4e0c5d17 100644
--- a/drivers/scsi/fdomain.c
+++ b/drivers/scsi/fdomain.c
@@ -496,7 +496,7 @@ static int fdomain_biosparam(struct scsi_device *sdev,
 	return 0;
 }
 
-static struct scsi_host_template fdomain_template = {
+static const struct scsi_host_template fdomain_template = {
 	.module			= THIS_MODULE,
 	.name			= "Future Domain TMC-16x0",
 	.proc_name		= "fdomain",
diff --git a/drivers/scsi/fnic/fnic_main.c b/drivers/scsi/fnic/fnic_main.c
index 1077110ab273..984bc5fc55e2 100644
--- a/drivers/scsi/fnic/fnic_main.c
+++ b/drivers/scsi/fnic/fnic_main.c
@@ -95,7 +95,7 @@ static int fnic_slave_alloc(struct scsi_device *sdev)
 	return 0;
 }
 
-static struct scsi_host_template fnic_host_template = {
+static const struct scsi_host_template fnic_host_template = {
 	.module = THIS_MODULE,
 	.name = DRV_NAME,
 	.queuecommand = fnic_queuecommand,
diff --git a/drivers/scsi/fnic/fnic_trace.c b/drivers/scsi/fnic/fnic_trace.c
index e03967463561..f3c3a26a1384 100644
--- a/drivers/scsi/fnic/fnic_trace.c
+++ b/drivers/scsi/fnic/fnic_trace.c
@@ -781,28 +781,21 @@ void copy_and_format_trace_data(struct fc_trace_hdr *tdata,
 				fnic_dbgfs_t *fnic_dbgfs_prt, int *orig_len,
 				u8 rdata_flag)
 {
-	struct tm tm;
 	int j, i = 1, len;
-	char *fc_trace, *fmt;
 	int ethhdr_len = sizeof(struct ethhdr) - 1;
 	int fcoehdr_len = sizeof(struct fcoe_hdr);
 	int fchdr_len = sizeof(struct fc_frame_header);
 	int max_size = fnic_fc_trace_max_pages * PAGE_SIZE * 3;
+	char *fc_trace;
 
 	tdata->frame_type = tdata->frame_type & 0x7F;
 
 	len = *orig_len;
 
-	time64_to_tm(tdata->time_stamp.tv_sec, 0, &tm);
-
-	fmt = "%02d:%02d:%04ld %02d:%02d:%02d.%09lu ns%8x       %c%8x\t";
-	len += scnprintf(fnic_dbgfs_prt->buffer + len,
-		max_size - len,
-		fmt,
-		tm.tm_mon + 1, tm.tm_mday, tm.tm_year + 1900,
-		tm.tm_hour, tm.tm_min, tm.tm_sec,
-		tdata->time_stamp.tv_nsec, tdata->host_no,
-		tdata->frame_type, tdata->frame_len);
+	len += scnprintf(fnic_dbgfs_prt->buffer + len, max_size - len,
+			 "%ptTs.%09lu ns%8x       %c%8x\t",
+			 &tdata->time_stamp.tv_sec, tdata->time_stamp.tv_nsec,
+			 tdata->host_no, tdata->frame_type, tdata->frame_len);
 
 	fc_trace = (char *)FC_TRACE_ADDRESS(tdata);
 
diff --git a/drivers/scsi/g_NCR5380.c b/drivers/scsi/g_NCR5380.c
index 0c768e7d06b9..f6305e3e60f4 100644
--- a/drivers/scsi/g_NCR5380.c
+++ b/drivers/scsi/g_NCR5380.c
@@ -219,7 +219,7 @@ static int hp_c2502_irqs[] = {
 	9, 5, 7, 3, 4, -1
 };
 
-static int generic_NCR5380_init_one(struct scsi_host_template *tpnt,
+static int generic_NCR5380_init_one(const struct scsi_host_template *tpnt,
 			struct device *pdev, int base, int irq, int board)
 {
 	bool is_pmio = base <= 0xffff;
@@ -689,7 +689,7 @@ static int generic_NCR5380_dma_residual(struct NCR5380_hostdata *hostdata)
 
 #include "NCR5380.c"
 
-static struct scsi_host_template driver_template = {
+static const struct scsi_host_template driver_template = {
 	.module			= THIS_MODULE,
 	.proc_name		= DRV_MODULE_NAME,
 	.name			= "Generic NCR5380/NCR53C400 SCSI",
diff --git a/drivers/scsi/gvp11.c b/drivers/scsi/gvp11.c
index 7d56a236a011..d2eddad099a2 100644
--- a/drivers/scsi/gvp11.c
+++ b/drivers/scsi/gvp11.c
@@ -222,7 +222,7 @@ static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
 	}
 }
 
-static struct scsi_host_template gvp11_scsi_template = {
+static const struct scsi_host_template gvp11_scsi_template = {
 	.module			= THIS_MODULE,
 	.name			= "GVP Series II SCSI",
 	.show_info		= wd33c93_show_info,
diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h
index 6f8a52a1b808..fb7c52c119df 100644
--- a/drivers/scsi/hisi_sas/hisi_sas.h
+++ b/drivers/scsi/hisi_sas/hisi_sas.h
@@ -207,6 +207,7 @@ struct hisi_sas_cq {
 	int	rd_point;
 	int	id;
 	int	irq_no;
+	spinlock_t poll_lock;
 };
 
 struct hisi_sas_dq {
@@ -344,7 +345,7 @@ struct hisi_sas_hw {
 					   int delay_ms, int timeout_ms);
 	void (*debugfs_snapshot_regs)(struct hisi_hba *hisi_hba);
 	int complete_hdr_size;
-	struct scsi_host_template *sht;
+	const struct scsi_host_template *sht;
 };
 
 #define HISI_SAS_MAX_DEBUGFS_DUMP (50)
@@ -484,6 +485,8 @@ struct hisi_hba {
 	struct dentry *debugfs_dump_dentry;
 	struct dentry *debugfs_bist_dentry;
 	struct dentry *debugfs_fifo_dentry;
+
+	int iopoll_q_cnt;
 };
 
 /* Generic HW DMA host memory structures */
@@ -653,16 +656,18 @@ extern void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy,
 extern void hisi_sas_phy_bcast(struct hisi_sas_phy *phy);
 extern void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba,
 				    struct sas_task *task,
-				    struct hisi_sas_slot *slot);
+				    struct hisi_sas_slot *slot,
+				    bool need_lock);
 extern void hisi_sas_init_mem(struct hisi_hba *hisi_hba);
 extern void hisi_sas_rst_work_handler(struct work_struct *work);
 extern void hisi_sas_sync_rst_work_handler(struct work_struct *work);
-extern void hisi_sas_sync_irqs(struct hisi_hba *hisi_hba);
 extern void hisi_sas_phy_oob_ready(struct hisi_hba *hisi_hba, int phy_no);
 extern bool hisi_sas_notify_phy_event(struct hisi_sas_phy *phy,
 				enum hisi_sas_phy_event event);
 extern void hisi_sas_release_tasks(struct hisi_hba *hisi_hba);
 extern u8 hisi_sas_get_prog_phy_linkrate_mask(enum sas_linkrate max);
+extern void hisi_sas_sync_cqs(struct hisi_hba *hisi_hba);
+extern void hisi_sas_sync_poll_cqs(struct hisi_hba *hisi_hba);
 extern void hisi_sas_controller_reset_prepare(struct hisi_hba *hisi_hba);
 extern void hisi_sas_controller_reset_done(struct hisi_hba *hisi_hba);
 #endif
diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c
index 8c038ccf1c09..412431c901a7 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_main.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_main.c
@@ -205,7 +205,7 @@ static int hisi_sas_slot_index_alloc(struct hisi_hba *hisi_hba,
 }
 
 void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task,
-			     struct hisi_sas_slot *slot)
+			     struct hisi_sas_slot *slot, bool need_lock)
 {
 	int device_id = slot->device_id;
 	struct hisi_sas_device *sas_dev = &hisi_hba->devices[device_id];
@@ -239,9 +239,13 @@ void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task,
 		}
 	}
 
-	spin_lock(&sas_dev->lock);
-	list_del_init(&slot->entry);
-	spin_unlock(&sas_dev->lock);
+	if (need_lock) {
+		spin_lock(&sas_dev->lock);
+		list_del_init(&slot->entry);
+		spin_unlock(&sas_dev->lock);
+	} else {
+		list_del_init(&slot->entry);
+	}
 
 	memset(slot, 0, offsetof(struct hisi_sas_slot, buf));
 
@@ -529,10 +533,21 @@ static int hisi_sas_queue_command(struct sas_task *task, gfp_t gfp_flags)
 			dq_index = blk_mq_unique_tag_to_hwq(blk_tag);
 			dq = &hisi_hba->dq[dq_index];
 		} else {
-			struct Scsi_Host *shost = hisi_hba->shost;
-			struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT];
-			int queue = qmap->mq_map[raw_smp_processor_id()];
+			int queue;
+
+			if (hisi_hba->iopoll_q_cnt) {
+				/*
+				 * Use interrupt queue (queue 0) to deliver and complete
+				 * internal IOs of libsas or libata when there is at least
+				 * one iopoll queue
+				 */
+				queue = 0;
+			} else {
+				struct Scsi_Host *shost = hisi_hba->shost;
+				struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT];
 
+				queue = qmap->mq_map[raw_smp_processor_id()];
+			}
 			dq = &hisi_hba->dq[queue];
 		}
 		break;
@@ -672,6 +687,55 @@ static struct hisi_sas_device *hisi_sas_alloc_dev(struct domain_device *device)
 	return sas_dev;
 }
 
+static void hisi_sas_sync_poll_cq(struct hisi_sas_cq *cq)
+{
+	/* make sure CQ entries being processed are processed to completion */
+	spin_lock(&cq->poll_lock);
+	spin_unlock(&cq->poll_lock);
+}
+
+static bool hisi_sas_queue_is_poll(struct hisi_sas_cq *cq)
+{
+	struct hisi_hba *hisi_hba = cq->hisi_hba;
+
+	if (cq->id < hisi_hba->queue_count - hisi_hba->iopoll_q_cnt)
+		return false;
+	return true;
+}
+
+static void hisi_sas_sync_cq(struct hisi_sas_cq *cq)
+{
+	if (hisi_sas_queue_is_poll(cq))
+		hisi_sas_sync_poll_cq(cq);
+	else
+		synchronize_irq(cq->irq_no);
+}
+
+void hisi_sas_sync_poll_cqs(struct hisi_hba *hisi_hba)
+{
+	int i;
+
+	for (i = 0; i < hisi_hba->queue_count; i++) {
+		struct hisi_sas_cq *cq = &hisi_hba->cq[i];
+
+		if (hisi_sas_queue_is_poll(cq))
+			hisi_sas_sync_poll_cq(cq);
+	}
+}
+EXPORT_SYMBOL_GPL(hisi_sas_sync_poll_cqs);
+
+void hisi_sas_sync_cqs(struct hisi_hba *hisi_hba)
+{
+	int i;
+
+	for (i = 0; i < hisi_hba->queue_count; i++) {
+		struct hisi_sas_cq *cq = &hisi_hba->cq[i];
+
+		hisi_sas_sync_cq(cq);
+	}
+}
+EXPORT_SYMBOL_GPL(hisi_sas_sync_cqs);
+
 static void hisi_sas_tmf_aborted(struct sas_task *task)
 {
 	struct hisi_sas_slot *slot = task->lldd_task;
@@ -683,10 +747,10 @@ static void hisi_sas_tmf_aborted(struct sas_task *task)
 		struct hisi_sas_cq *cq =
 			   &hisi_hba->cq[slot->dlvry_queue];
 		/*
-		 * sync irq to avoid free'ing task
+		 * sync irq or poll queue to avoid free'ing task
 		 * before using task in IO completion
 		 */
-		synchronize_irq(cq->irq_no);
+		hisi_sas_sync_cq(cq);
 		slot->task = NULL;
 	}
 }
@@ -1021,7 +1085,7 @@ static void hisi_sas_port_notify_formed(struct asd_sas_phy *sas_phy)
 }
 
 static void hisi_sas_do_release_task(struct hisi_hba *hisi_hba, struct sas_task *task,
-				     struct hisi_sas_slot *slot)
+				     struct hisi_sas_slot *slot, bool need_lock)
 {
 	if (task) {
 		unsigned long flags;
@@ -1038,7 +1102,7 @@ static void hisi_sas_do_release_task(struct hisi_hba *hisi_hba, struct sas_task
 		spin_unlock_irqrestore(&task->task_state_lock, flags);
 	}
 
-	hisi_sas_slot_task_free(hisi_hba, task, slot);
+	hisi_sas_slot_task_free(hisi_hba, task, slot, need_lock);
 }
 
 static void hisi_sas_release_task(struct hisi_hba *hisi_hba,
@@ -1047,8 +1111,11 @@ static void hisi_sas_release_task(struct hisi_hba *hisi_hba,
 	struct hisi_sas_slot *slot, *slot2;
 	struct hisi_sas_device *sas_dev = device->lldd_dev;
 
+	spin_lock(&sas_dev->lock);
 	list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry)
-		hisi_sas_do_release_task(hisi_hba, slot->task, slot);
+		hisi_sas_do_release_task(hisi_hba, slot->task, slot, false);
+
+	spin_unlock(&sas_dev->lock);
 }
 
 void hisi_sas_release_tasks(struct hisi_hba *hisi_hba)
@@ -1453,13 +1520,41 @@ void hisi_sas_controller_reset_prepare(struct hisi_hba *hisi_hba)
 }
 EXPORT_SYMBOL_GPL(hisi_sas_controller_reset_prepare);
 
+static void hisi_sas_async_init_wait_phyup(void *data, async_cookie_t cookie)
+{
+	struct hisi_sas_phy *phy = data;
+	struct hisi_hba *hisi_hba = phy->hisi_hba;
+	struct device *dev = hisi_hba->dev;
+	DECLARE_COMPLETION_ONSTACK(completion);
+	int phy_no = phy->sas_phy.id;
+
+	phy->reset_completion = &completion;
+	hisi_sas_phy_enable(hisi_hba, phy_no, 1);
+	if (!wait_for_completion_timeout(&completion,
+					 HISI_SAS_WAIT_PHYUP_TIMEOUT))
+		dev_warn(dev, "phy%d wait phyup timed out\n", phy_no);
+
+	phy->reset_completion = NULL;
+}
+
 void hisi_sas_controller_reset_done(struct hisi_hba *hisi_hba)
 {
 	struct Scsi_Host *shost = hisi_hba->shost;
+	ASYNC_DOMAIN_EXCLUSIVE(async);
+	int phy_no;
 
 	/* Init and wait for PHYs to come up and all libsas event finished. */
-	hisi_hba->hw->phys_init(hisi_hba);
-	msleep(1000);
+	for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
+		struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
+
+		if (!(hisi_hba->phy_state & BIT(phy_no)))
+			continue;
+
+		async_schedule_domain(hisi_sas_async_init_wait_phyup,
+				      phy, &async);
+	}
+
+	async_synchronize_full_domain(&async);
 	hisi_sas_refresh_port_id(hisi_hba);
 	clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
 
@@ -1540,11 +1635,11 @@ static int hisi_sas_abort_task(struct sas_task *task)
 
 		if (slot) {
 			/*
-			 * sync irq to avoid free'ing task
+			 * sync irq or poll queue to avoid free'ing task
 			 * before using task in IO completion
 			 */
 			cq = &hisi_hba->cq[slot->dlvry_queue];
-			synchronize_irq(cq->irq_no);
+			hisi_sas_sync_cq(cq);
 		}
 		spin_unlock_irqrestore(&task->task_state_lock, flags);
 		rc = TMF_RESP_FUNC_COMPLETE;
@@ -1574,7 +1669,7 @@ static int hisi_sas_abort_task(struct sas_task *task)
 		 */
 		if (rc == TMF_RESP_FUNC_COMPLETE && rc2 != TMF_RESP_FUNC_SUCC) {
 			if (task->lldd_task)
-				hisi_sas_do_release_task(hisi_hba, task, slot);
+				hisi_sas_do_release_task(hisi_hba, task, slot, true);
 		}
 	} else if (task->task_proto & SAS_PROTOCOL_SATA ||
 		task->task_proto & SAS_PROTOCOL_STP) {
@@ -1594,7 +1689,7 @@ static int hisi_sas_abort_task(struct sas_task *task)
 			 */
 			if ((sas_dev->dev_status == HISI_SAS_DEV_NCQ_ERR) &&
 			    qc && qc->scsicmd) {
-				hisi_sas_do_release_task(hisi_hba, task, slot);
+				hisi_sas_do_release_task(hisi_hba, task, slot, true);
 				rc = TMF_RESP_FUNC_COMPLETE;
 			} else {
 				rc = hisi_sas_softreset_ata_disk(device);
@@ -1611,10 +1706,10 @@ static int hisi_sas_abort_task(struct sas_task *task)
 		if (((rc < 0) || (rc == TMF_RESP_FUNC_FAILED)) &&
 					task->lldd_task) {
 			/*
-			 * sync irq to avoid free'ing task
+			 * sync irq or poll queue to avoid free'ing task
 			 * before using task in IO completion
 			 */
-			synchronize_irq(cq->irq_no);
+			hisi_sas_sync_cq(cq);
 			slot->task = NULL;
 		}
 	}
@@ -1885,10 +1980,10 @@ static bool hisi_sas_internal_abort_timeout(struct sas_task *task,
 			struct hisi_sas_cq *cq =
 				&hisi_hba->cq[slot->dlvry_queue];
 			/*
-			 * sync irq to avoid free'ing task
+			 * sync irq or poll queue to avoid free'ing task
 			 * before using task in IO completion
 			 */
-			synchronize_irq(cq->irq_no);
+			hisi_sas_sync_cq(cq);
 			slot->task = NULL;
 		}
 
@@ -1992,18 +2087,6 @@ void hisi_sas_phy_bcast(struct hisi_sas_phy *phy)
 }
 EXPORT_SYMBOL_GPL(hisi_sas_phy_bcast);
 
-void hisi_sas_sync_irqs(struct hisi_hba *hisi_hba)
-{
-	int i;
-
-	for (i = 0; i < hisi_hba->cq_nvecs; i++) {
-		struct hisi_sas_cq *cq = &hisi_hba->cq[i];
-
-		synchronize_irq(cq->irq_no);
-	}
-}
-EXPORT_SYMBOL_GPL(hisi_sas_sync_irqs);
-
 int hisi_sas_host_reset(struct Scsi_Host *shost, int reset_type)
 {
 	struct hisi_hba *hisi_hba = shost_priv(shost);
@@ -2101,6 +2184,7 @@ int hisi_sas_alloc(struct hisi_hba *hisi_hba)
 		/* Completion queue structure */
 		cq->id = i;
 		cq->hisi_hba = hisi_hba;
+		spin_lock_init(&cq->poll_lock);
 
 		/* Delivery queue structure */
 		spin_lock_init(&dq->lock);
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
index d643c5a49aa9..0aa8c9c88535 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
@@ -1258,7 +1258,11 @@ static void slot_complete_v1_hw(struct hisi_hba *hisi_hba,
 
 		slot_err_v1_hw(hisi_hba, task, slot);
 		if (unlikely(slot->abort)) {
-			sas_task_abort(task);
+			if (dev_is_sata(device) && task->ata_task.use_ncq)
+				sas_ata_device_link_abort(device, true);
+			else
+				sas_task_abort(task);
+
 			return;
 		}
 		goto out;
@@ -1306,7 +1310,7 @@ static void slot_complete_v1_hw(struct hisi_hba *hisi_hba,
 	}
 
 out:
-	hisi_sas_slot_task_free(hisi_hba, task, slot);
+	hisi_sas_slot_task_free(hisi_hba, task, slot, true);
 
 	if (task->task_done)
 		task->task_done(task);
@@ -1735,7 +1739,7 @@ static struct attribute *host_v1_hw_attrs[] = {
 
 ATTRIBUTE_GROUPS(host_v1_hw);
 
-static struct scsi_host_template sht_v1_hw = {
+static const struct scsi_host_template sht_v1_hw = {
 	.name			= DRV_NAME,
 	.proc_name		= DRV_NAME,
 	.module			= THIS_MODULE,
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
index cded42f4ca44..cd78e4c983aa 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
@@ -2404,7 +2404,11 @@ static void slot_complete_v2_hw(struct hisi_hba *hisi_hba,
 				 error_info[2], error_info[3]);
 
 		if (unlikely(slot->abort)) {
-			sas_task_abort(task);
+			if (dev_is_sata(device) && task->ata_task.use_ncq)
+				sas_ata_device_link_abort(device, true);
+			else
+				sas_task_abort(task);
+
 			return;
 		}
 		goto out;
@@ -2462,7 +2466,7 @@ out:
 	}
 	task->task_state_flags |= SAS_TASK_STATE_DONE;
 	spin_unlock_irqrestore(&task->task_state_lock, flags);
-	hisi_sas_slot_task_free(hisi_hba, task, slot);
+	hisi_sas_slot_task_free(hisi_hba, task, slot, true);
 
 	if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
 		spin_lock_irqsave(&device->done_lock, flags);
@@ -3551,7 +3555,7 @@ static void map_queues_v2_hw(struct Scsi_Host *shost)
 	}
 }
 
-static struct scsi_host_template sht_v2_hw = {
+static const struct scsi_host_template sht_v2_hw = {
 	.name			= DRV_NAME,
 	.proc_name		= DRV_NAME,
 	.module			= THIS_MODULE,
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
index a63279f55d09..12d588454f5d 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
@@ -552,6 +552,11 @@ static int prot_mask;
 module_param(prot_mask, int, 0444);
 MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=0x0 ");
 
+/* the index of iopoll queues are bigger than interrupt queues' */
+static int experimental_iopoll_q_cnt;
+module_param(experimental_iopoll_q_cnt, int, 0444);
+MODULE_PARM_DESC(experimental_iopoll_q_cnt, "number of queues to be used as poll mode, def=0");
+
 static void debugfs_work_handler_v3_hw(struct work_struct *work);
 static void debugfs_snapshot_regs_v3_hw(struct hisi_hba *hisi_hba);
 
@@ -599,6 +604,27 @@ static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
 	readl_poll_timeout_atomic(regs, val, cond, delay_us, timeout_us);\
 })
 
+static void interrupt_enable_v3_hw(struct hisi_hba *hisi_hba)
+{
+	int i;
+
+	for (i = 0; i < hisi_hba->queue_count; i++)
+		hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0);
+
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffc220ff);
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x155555);
+
+	for (i = 0; i < hisi_hba->n_phy; i++) {
+		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xf2057fff);
+		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
+		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
+		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0);
+		hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0);
+	}
+}
+
 static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 {
 	int i, j;
@@ -619,20 +645,14 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC1, 0xffffffff);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC2, 0xffffffff);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
-	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
-	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
-	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffc220ff);
 	hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
 	hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
 	hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
-	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x155555);
 	hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0);
 	hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0);
-	for (i = 0; i < hisi_hba->queue_count; i++)
-		hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0);
-
 	hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
 
+	interrupt_enable_v3_hw(hisi_hba);
 	for (i = 0; i < hisi_hba->n_phy; i++) {
 		enum sas_linkrate max;
 		struct hisi_sas_phy *phy = &hisi_hba->phy[i];
@@ -655,13 +675,8 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
-		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xf2057fff);
-		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
 		hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
-		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
 		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0);
-		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0);
-		hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0);
 		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x1);
 		hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER, 0x7f7a120);
 		hisi_sas_phy_write32(hisi_hba, i, CON_CFG_DRIVER, 0x2a0a01);
@@ -883,6 +898,7 @@ static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
 
 	cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba,
 		CFG_ABT_SET_QUERY_IPTT);
+	spin_lock(&sas_dev->lock);
 	list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) {
 		cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK;
 		cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) |
@@ -890,6 +906,7 @@ static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
 		hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 			cfg_abt_set_query_iptt);
 	}
+	spin_unlock(&sas_dev->lock);
 	cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF);
 	hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 		cfg_abt_set_query_iptt);
@@ -2320,7 +2337,11 @@ static void slot_complete_v3_hw(struct hisi_hba *hisi_hba,
 					error_info[0], error_info[1],
 					error_info[2], error_info[3]);
 			if (unlikely(slot->abort)) {
-				sas_task_abort(task);
+				if (dev_is_sata(device) && task->ata_task.use_ncq)
+					sas_ata_device_link_abort(device, true);
+				else
+					sas_task_abort(task);
+
 				return;
 			}
 			goto out;
@@ -2374,7 +2395,7 @@ out:
 	}
 	task->task_state_flags |= SAS_TASK_STATE_DONE;
 	spin_unlock_irqrestore(&task->task_state_lock, flags);
-	hisi_sas_slot_task_free(hisi_hba, task, slot);
+	hisi_sas_slot_task_free(hisi_hba, task, slot, true);
 
 	if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
 		spin_lock_irqsave(&device->done_lock, flags);
@@ -2391,23 +2412,25 @@ out:
 		task->task_done(task);
 }
 
-static irqreturn_t  cq_thread_v3_hw(int irq_no, void *p)
+static int complete_v3_hw(struct hisi_sas_cq *cq)
 {
-	struct hisi_sas_cq *cq = p;
-	struct hisi_hba *hisi_hba = cq->hisi_hba;
-	struct hisi_sas_slot *slot;
 	struct hisi_sas_complete_v3_hdr *complete_queue;
-	u32 rd_point = cq->rd_point, wr_point;
+	struct hisi_hba *hisi_hba = cq->hisi_hba;
+	u32 rd_point, wr_point;
 	int queue = cq->id;
+	int completed;
 
+	rd_point = cq->rd_point;
 	complete_queue = hisi_hba->complete_hdr[queue];
 
 	wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
 				   (0x14 * queue));
+	completed = (wr_point + HISI_SAS_QUEUE_SLOTS - rd_point) % HISI_SAS_QUEUE_SLOTS;
 
 	while (rd_point != wr_point) {
 		struct hisi_sas_complete_v3_hdr *complete_hdr;
 		struct device *dev = hisi_hba->dev;
+		struct hisi_sas_slot *slot;
 		u32 dw0, dw1, dw3;
 		int iptt;
 
@@ -2451,6 +2474,28 @@ static irqreturn_t  cq_thread_v3_hw(int irq_no, void *p)
 	cq->rd_point = rd_point;
 	hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
 
+	return completed;
+}
+
+static int queue_complete_v3_hw(struct Scsi_Host *shost, unsigned int queue)
+{
+	struct hisi_hba *hisi_hba = shost_priv(shost);
+	struct hisi_sas_cq *cq = &hisi_hba->cq[queue];
+	int completed;
+
+	spin_lock(&cq->poll_lock);
+	completed = complete_v3_hw(cq);
+	spin_unlock(&cq->poll_lock);
+
+	return completed;
+}
+
+static irqreturn_t cq_thread_v3_hw(int irq_no, void *p)
+{
+	struct hisi_sas_cq *cq = p;
+
+	complete_v3_hw(cq);
+
 	return IRQ_HANDLED;
 }
 
@@ -2474,8 +2519,9 @@ static void hisi_sas_v3_free_vectors(void *data)
 
 static int interrupt_preinit_v3_hw(struct hisi_hba *hisi_hba)
 {
-	int vectors;
-	int max_msi = HISI_SAS_MSI_COUNT_V3_HW, min_msi;
+	/* Allocate all MSI vectors to avoid re-insertion issue */
+	int max_msi = HISI_SAS_MSI_COUNT_V3_HW;
+	int vectors, min_msi;
 	struct Scsi_Host *shost = hisi_hba->shost;
 	struct pci_dev *pdev = hisi_hba->pci_dev;
 	struct irq_affinity desc = {
@@ -2492,8 +2538,8 @@ static int interrupt_preinit_v3_hw(struct hisi_hba *hisi_hba)
 		return -ENOENT;
 
 
-	hisi_hba->cq_nvecs = vectors - BASE_VECTORS_V3_HW;
-	shost->nr_hw_queues = hisi_hba->cq_nvecs;
+	hisi_hba->cq_nvecs = vectors - BASE_VECTORS_V3_HW - hisi_hba->iopoll_q_cnt;
+	shost->nr_hw_queues = hisi_hba->cq_nvecs + hisi_hba->iopoll_q_cnt;
 
 	return devm_add_action(&pdev->dev, hisi_sas_v3_free_vectors, pdev);
 }
@@ -2625,7 +2671,7 @@ static int disable_host_v3_hw(struct hisi_hba *hisi_hba)
 	u32 status, reg_val;
 	int rc;
 
-	interrupt_disable_v3_hw(hisi_hba);
+	hisi_sas_sync_poll_cqs(hisi_hba);
 	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
 
 	hisi_sas_stop_phys(hisi_hba);
@@ -2655,6 +2701,7 @@ static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
 	struct device *dev = hisi_hba->dev;
 	int rc;
 
+	interrupt_disable_v3_hw(hisi_hba);
 	rc = disable_host_v3_hw(hisi_hba);
 	if (rc) {
 		dev_err(dev, "soft reset: disable host failed rc=%d\n", rc);
@@ -2823,6 +2870,18 @@ static ssize_t intr_coal_count_v3_hw_store(struct device *dev,
 }
 static DEVICE_ATTR_RW(intr_coal_count_v3_hw);
 
+static ssize_t iopoll_q_cnt_v3_hw_show(struct device *dev,
+					  struct device_attribute
+					  *attr, char *buf)
+{
+	struct Scsi_Host *shost = class_to_shost(dev);
+	struct hisi_hba *hisi_hba = shost_priv(shost);
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n",
+			 hisi_hba->iopoll_q_cnt);
+}
+static DEVICE_ATTR_RO(iopoll_q_cnt_v3_hw);
+
 static int slave_configure_v3_hw(struct scsi_device *sdev)
 {
 	struct Scsi_Host *shost = dev_to_shost(&sdev->sdev_gendev);
@@ -2852,6 +2911,7 @@ static struct attribute *host_v3_hw_attrs[] = {
 	&dev_attr_intr_conv_v3_hw.attr,
 	&dev_attr_intr_coal_ticks_v3_hw.attr,
 	&dev_attr_intr_coal_count_v3_hw.attr,
+	&dev_attr_iopoll_q_cnt_v3_hw.attr,
 	NULL
 };
 
@@ -3038,7 +3098,7 @@ static void debugfs_snapshot_prepare_v3_hw(struct hisi_hba *hisi_hba)
 
 	wait_cmds_complete_timeout_v3_hw(hisi_hba, 100, 5000);
 
-	hisi_sas_sync_irqs(hisi_hba);
+	hisi_sas_sync_cqs(hisi_hba);
 }
 
 static void debugfs_snapshot_restore_v3_hw(struct hisi_hba *hisi_hba)
@@ -3210,12 +3270,34 @@ static int debugfs_set_bist_v3_hw(struct hisi_hba *hisi_hba, bool enable)
 static void hisi_sas_map_queues(struct Scsi_Host *shost)
 {
 	struct hisi_hba *hisi_hba = shost_priv(shost);
-	struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT];
+	struct blk_mq_queue_map *qmap;
+	int i, qoff;
+
+	for (i = 0, qoff = 0; i < shost->nr_maps; i++) {
+		qmap = &shost->tag_set.map[i];
+		if (i == HCTX_TYPE_DEFAULT) {
+			qmap->nr_queues = hisi_hba->cq_nvecs;
+		} else if (i == HCTX_TYPE_POLL) {
+			qmap->nr_queues = hisi_hba->iopoll_q_cnt;
+		} else {
+			qmap->nr_queues = 0;
+			continue;
+		}
 
-	blk_mq_pci_map_queues(qmap, hisi_hba->pci_dev, BASE_VECTORS_V3_HW);
+		/* At least one interrupt hardware queue */
+		if (!qmap->nr_queues)
+			WARN_ON(i == HCTX_TYPE_DEFAULT);
+		qmap->queue_offset = qoff;
+		if (i == HCTX_TYPE_POLL)
+			blk_mq_map_queues(qmap);
+		else
+			blk_mq_pci_map_queues(qmap, hisi_hba->pci_dev,
+					      BASE_VECTORS_V3_HW);
+		qoff += qmap->nr_queues;
+	}
 }
 
-static struct scsi_host_template sht_v3_hw = {
+static const struct scsi_host_template sht_v3_hw = {
 	.name			= DRV_NAME,
 	.proc_name		= DRV_NAME,
 	.module			= THIS_MODULE,
@@ -3244,6 +3326,7 @@ static struct scsi_host_template sht_v3_hw = {
 	.tag_alloc_policy	= BLK_TAG_ALLOC_RR,
 	.host_reset             = hisi_sas_host_reset,
 	.host_tagset		= 1,
+	.mq_poll		= queue_complete_v3_hw,
 };
 
 static const struct hisi_sas_hw hisi_sas_v3_hw = {
@@ -3303,6 +3386,13 @@ hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
 	if (hisi_sas_get_fw_info(hisi_hba) < 0)
 		goto err_out;
 
+	if (experimental_iopoll_q_cnt < 0 ||
+		experimental_iopoll_q_cnt >= hisi_hba->queue_count)
+		dev_err(dev, "iopoll queue count %d cannot exceed or equal 16, using default 0\n",
+			experimental_iopoll_q_cnt);
+	else
+		hisi_hba->iopoll_q_cnt = experimental_iopoll_q_cnt;
+
 	if (hisi_sas_alloc(hisi_hba)) {
 		hisi_sas_free(hisi_hba);
 		goto err_out;
@@ -4858,6 +4948,10 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	shost->max_cmd_len = 16;
 	shost->can_queue = HISI_SAS_UNRESERVED_IPTT;
 	shost->cmd_per_lun = HISI_SAS_UNRESERVED_IPTT;
+	if (hisi_hba->iopoll_q_cnt)
+		shost->nr_maps = 3;
+	else
+		shost->nr_maps = 1;
 
 	sha->sas_ha_name = DRV_NAME;
 	sha->dev = dev;
@@ -4976,6 +5070,7 @@ static void hisi_sas_reset_prepare_v3_hw(struct pci_dev *pdev)
 	set_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags);
 	hisi_sas_controller_reset_prepare(hisi_hba);
 
+	interrupt_disable_v3_hw(hisi_hba);
 	rc = disable_host_v3_hw(hisi_hba);
 	if (rc)
 		dev_err(dev, "FLR: disable host failed rc=%d\n", rc);
@@ -5005,6 +5100,21 @@ enum {
 	hip08,
 };
 
+static void enable_host_v3_hw(struct hisi_hba *hisi_hba)
+{
+	u32 reg_val;
+
+	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
+			 (u32)((1ULL << hisi_hba->queue_count) - 1));
+
+	phys_init_v3_hw(hisi_hba);
+	reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
+				  AM_CTRL_GLOBAL);
+	reg_val &= ~AM_CTRL_SHUTDOWN_REQ_MSK;
+	hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
+			 AM_CTRL_GLOBAL, reg_val);
+}
+
 static int _suspend_v3_hw(struct device *device)
 {
 	struct pci_dev *pdev = to_pci_dev(device);
@@ -5027,14 +5137,20 @@ static int _suspend_v3_hw(struct device *device)
 	scsi_block_requests(shost);
 	set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
 	flush_workqueue(hisi_hba->wq);
+	interrupt_disable_v3_hw(hisi_hba);
+
+#ifdef CONFIG_PM
+	if (atomic_read(&device->power.usage_count)) {
+		dev_err(dev, "PM suspend: host status cannot be suspended\n");
+		rc = -EBUSY;
+		goto err_out;
+	}
+#endif
 
 	rc = disable_host_v3_hw(hisi_hba);
 	if (rc) {
 		dev_err(dev, "PM suspend: disable host failed rc=%d\n", rc);
-		clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
-		clear_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags);
-		scsi_unblock_requests(shost);
-		return rc;
+		goto err_out_recover_host;
 	}
 
 	hisi_sas_init_mem(hisi_hba);
@@ -5045,6 +5161,17 @@ static int _suspend_v3_hw(struct device *device)
 
 	dev_warn(dev, "end of suspending controller\n");
 	return 0;
+
+err_out_recover_host:
+	enable_host_v3_hw(hisi_hba);
+#ifdef CONFIG_PM
+err_out:
+#endif
+	interrupt_enable_v3_hw(hisi_hba);
+	clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
+	clear_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags);
+	scsi_unblock_requests(shost);
+	return rc;
 }
 
 static int _resume_v3_hw(struct device *device)
diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
index 9b6fbbe15d92..f0bc8bbb3938 100644
--- a/drivers/scsi/hosts.c
+++ b/drivers/scsi/hosts.c
@@ -219,7 +219,7 @@ EXPORT_SYMBOL(scsi_remove_host);
 int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
 			   struct device *dma_dev)
 {
-	struct scsi_host_template *sht = shost->hostt;
+	const struct scsi_host_template *sht = shost->hostt;
 	int error = -EINVAL;
 
 	shost_printk(KERN_INFO, shost, "%s\n",
@@ -389,7 +389,7 @@ static struct device_type scsi_host_type = {
  * Return value:
  * 	Pointer to a new Scsi_Host
  **/
-struct Scsi_Host *scsi_host_alloc(struct scsi_host_template *sht, int privsize)
+struct Scsi_Host *scsi_host_alloc(const struct scsi_host_template *sht, int privsize)
 {
 	struct Scsi_Host *shost;
 	int index;
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index f6da34850af9..af18d20f3079 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -967,7 +967,7 @@ ATTRIBUTE_GROUPS(hpsa_shost);
 #define HPSA_NRESERVED_CMDS	(HPSA_CMDS_RESERVED_FOR_DRIVER +\
 				 HPSA_MAX_CONCURRENT_PASSTHRUS)
 
-static struct scsi_host_template hpsa_driver_template = {
+static const struct scsi_host_template hpsa_driver_template = {
 	.module			= THIS_MODULE,
 	.name			= HPSA,
 	.proc_name		= HPSA,
@@ -9108,7 +9108,6 @@ static void hpsa_remove_one(struct pci_dev *pdev)
 
 	free_percpu(h->lockup_detected);		/* init_one 2 */
 	h->lockup_detected = NULL;			/* init_one 2 */
-	/* (void) pci_disable_pcie_error_reporting(pdev); */	/* init_one 1 */
 
 	hpda_free_ctlr_info(h);				/* init_one 1 */
 }
@@ -9476,8 +9475,6 @@ static void hpsa_free_performant_mode(struct ctlr_info *h)
 static int hpsa_put_ctlr_into_performant_mode(struct ctlr_info *h)
 {
 	u32 trans_support;
-	unsigned long transMethod = CFGTBL_Trans_Performant |
-					CFGTBL_Trans_use_short_tags;
 	int i, rc;
 
 	if (hpsa_simple_mode)
@@ -9489,14 +9486,10 @@ static int hpsa_put_ctlr_into_performant_mode(struct ctlr_info *h)
 
 	/* Check for I/O accelerator mode support */
 	if (trans_support & CFGTBL_Trans_io_accel1) {
-		transMethod |= CFGTBL_Trans_io_accel1 |
-				CFGTBL_Trans_enable_directed_msix;
 		rc = hpsa_alloc_ioaccel1_cmd_and_bft(h);
 		if (rc)
 			return rc;
 	} else if (trans_support & CFGTBL_Trans_io_accel2) {
-		transMethod |= CFGTBL_Trans_io_accel2 |
-				CFGTBL_Trans_enable_directed_msix;
 		rc = hpsa_alloc_ioaccel2_cmd_and_bft(h);
 		if (rc)
 			return rc;
diff --git a/drivers/scsi/hptiop.c b/drivers/scsi/hptiop.c
index 7e8903718245..06ccb51bf6a9 100644
--- a/drivers/scsi/hptiop.c
+++ b/drivers/scsi/hptiop.c
@@ -1159,7 +1159,7 @@ static int hptiop_slave_config(struct scsi_device *sdev)
 	return 0;
 }
 
-static struct scsi_host_template driver_template = {
+static const struct scsi_host_template driver_template = {
 	.module                     = THIS_MODULE,
 	.name                       = driver_name,
 	.queuecommand               = hptiop_queuecommand,
diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c
index 1a0c0b7289d2..ce9eb00e2ca0 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc.c
+++ b/drivers/scsi/ibmvscsi/ibmvfc.c
@@ -3611,7 +3611,7 @@ static struct attribute *ibmvfc_host_attrs[] = {
 
 ATTRIBUTE_GROUPS(ibmvfc_host);
 
-static struct scsi_host_template driver_template = {
+static const struct scsi_host_template driver_template = {
 	.module = THIS_MODULE,
 	.name = "IBM POWER Virtual FC Adapter",
 	.proc_name = IBMVFC_NAME,
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
index e8770310a64b..385f812b8793 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
@@ -3698,16 +3698,6 @@ static int ibmvscsis_check_true(struct se_portal_group *se_tpg)
 	return 1;
 }
 
-static int ibmvscsis_check_false(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
-static u32 ibmvscsis_tpg_get_inst_index(struct se_portal_group *se_tpg)
-{
-	return 1;
-}
-
 static int ibmvscsis_check_stop_free(struct se_cmd *se_cmd)
 {
 	return target_put_sess_cmd(se_cmd);
@@ -3726,11 +3716,6 @@ static void ibmvscsis_release_cmd(struct se_cmd *se_cmd)
 	spin_unlock_bh(&vscsi->intr_lock);
 }
 
-static u32 ibmvscsis_sess_get_index(struct se_session *se_sess)
-{
-	return 0;
-}
-
 static int ibmvscsis_write_pending(struct se_cmd *se_cmd)
 {
 	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
@@ -3765,15 +3750,6 @@ static int ibmvscsis_write_pending(struct se_cmd *se_cmd)
 	return 0;
 }
 
-static void ibmvscsis_set_default_node_attrs(struct se_node_acl *nacl)
-{
-}
-
-static int ibmvscsis_get_cmd_state(struct se_cmd *se_cmd)
-{
-	return 0;
-}
-
 static int ibmvscsis_queue_data_in(struct se_cmd *se_cmd)
 {
 	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
@@ -3982,15 +3958,9 @@ static const struct target_core_fabric_ops ibmvscsis_ops = {
 	.tpg_get_default_depth		= ibmvscsis_get_default_depth,
 	.tpg_check_demo_mode		= ibmvscsis_check_true,
 	.tpg_check_demo_mode_cache	= ibmvscsis_check_true,
-	.tpg_check_demo_mode_write_protect = ibmvscsis_check_false,
-	.tpg_check_prod_mode_write_protect = ibmvscsis_check_false,
-	.tpg_get_inst_index		= ibmvscsis_tpg_get_inst_index,
 	.check_stop_free		= ibmvscsis_check_stop_free,
 	.release_cmd			= ibmvscsis_release_cmd,
-	.sess_get_index			= ibmvscsis_sess_get_index,
 	.write_pending			= ibmvscsis_write_pending,
-	.set_default_node_attributes	= ibmvscsis_set_default_node_attrs,
-	.get_cmd_state			= ibmvscsis_get_cmd_state,
 	.queue_data_in			= ibmvscsis_queue_data_in,
 	.queue_status			= ibmvscsis_queue_status,
 	.queue_tm_rsp			= ibmvscsis_queue_tm_rsp,
diff --git a/drivers/scsi/imm.c b/drivers/scsi/imm.c
index 7a499d621c25..07db98161a03 100644
--- a/drivers/scsi/imm.c
+++ b/drivers/scsi/imm.c
@@ -1096,7 +1096,7 @@ static int imm_adjust_queue(struct scsi_device *device)
 	return 0;
 }
 
-static struct scsi_host_template imm_template = {
+static const struct scsi_host_template imm_template = {
 	.module			= THIS_MODULE,
 	.proc_name		= "imm",
 	.show_info		= imm_show_info,
diff --git a/drivers/scsi/initio.c b/drivers/scsi/initio.c
index 375261d67619..2a50fda3a628 100644
--- a/drivers/scsi/initio.c
+++ b/drivers/scsi/initio.c
@@ -2788,7 +2788,7 @@ static void i91uSCBPost(u8 * host_mem, u8 * cblk_mem)
 	initio_release_scb(host, cblk);	/* Release SCB for current channel */
 }
 
-static struct scsi_host_template initio_template = {
+static const struct scsi_host_template initio_template = {
 	.proc_name		= "INI9100U",
 	.name			= "Initio INI-9X00U/UW SCSI device driver",
 	.queuecommand		= i91u_queuecommand,
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index c74053f0b72f..4d3c280a7360 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -6736,7 +6736,7 @@ static const char *ipr_ioa_info(struct Scsi_Host *host)
 	return buffer;
 }
 
-static struct scsi_host_template driver_template = {
+static const struct scsi_host_template driver_template = {
 	.module = THIS_MODULE,
 	.name = "IPR",
 	.info = ipr_ioa_info,
diff --git a/drivers/scsi/isci/init.c b/drivers/scsi/isci/init.c
index e294d5d961eb..ac1e04b86d8f 100644
--- a/drivers/scsi/isci/init.c
+++ b/drivers/scsi/isci/init.c
@@ -149,7 +149,7 @@ static struct attribute *isci_host_attrs[] = {
 
 ATTRIBUTE_GROUPS(isci_host);
 
-static struct scsi_host_template isci_sht = {
+static const struct scsi_host_template isci_sht = {
 
 	.module				= THIS_MODULE,
 	.name				= DRV_NAME,
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 15f452908926..9637d4bc2bc9 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -47,7 +47,7 @@ MODULE_DESCRIPTION("iSCSI/TCP data-path");
 MODULE_LICENSE("GPL");
 
 static struct scsi_transport_template *iscsi_sw_tcp_scsi_transport;
-static struct scsi_host_template iscsi_sw_tcp_sht;
+static const struct scsi_host_template iscsi_sw_tcp_sht;
 static struct iscsi_transport iscsi_sw_tcp_transport;
 
 static unsigned int iscsi_max_lun = ~0;
@@ -1071,7 +1071,7 @@ static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
 	return 0;
 }
 
-static struct scsi_host_template iscsi_sw_tcp_sht = {
+static const struct scsi_host_template iscsi_sw_tcp_sht = {
 	.module			= THIS_MODULE,
 	.name			= "iSCSI Initiator over TCP/IP",
 	.queuecommand           = iscsi_queuecommand,
diff --git a/drivers/scsi/jazz_esp.c b/drivers/scsi/jazz_esp.c
index 60a88a95a8e2..0c842fb29aa0 100644
--- a/drivers/scsi/jazz_esp.c
+++ b/drivers/scsi/jazz_esp.c
@@ -104,7 +104,7 @@ static const struct esp_driver_ops jazz_esp_ops = {
 
 static int esp_jazz_probe(struct platform_device *dev)
 {
-	struct scsi_host_template *tpnt = &scsi_esp_template;
+	const struct scsi_host_template *tpnt = &scsi_esp_template;
 	struct Scsi_Host *host;
 	struct esp *esp;
 	struct resource *res;
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index 127f3d7f19dc..0fda8905eabd 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -2895,7 +2895,7 @@ EXPORT_SYMBOL_GPL(iscsi_host_add);
  * This should be called by partial offload and software iscsi drivers.
  * To access the driver specific memory use the iscsi_host_priv() macro.
  */
-struct Scsi_Host *iscsi_host_alloc(struct scsi_host_template *sht,
+struct Scsi_Host *iscsi_host_alloc(const struct scsi_host_template *sht,
 				   int dd_data_size, bool xmit_can_sleep)
 {
 	struct Scsi_Host *shost;
diff --git a/drivers/scsi/libsas/sas_discover.c b/drivers/scsi/libsas/sas_discover.c
index 72fdb2e5d047..8c6afe724944 100644
--- a/drivers/scsi/libsas/sas_discover.c
+++ b/drivers/scsi/libsas/sas_discover.c
@@ -360,6 +360,33 @@ static void sas_destruct_ports(struct asd_sas_port *port)
 	}
 }
 
+static bool sas_abort_cmd(struct request *req, void *data)
+{
+	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
+	struct domain_device *dev = data;
+
+	if (dev == cmd_to_domain_dev(cmd))
+		blk_abort_request(req);
+	return true;
+}
+
+static void sas_abort_device_scsi_cmds(struct domain_device *dev)
+{
+	struct sas_ha_struct *sas_ha = dev->port->ha;
+	struct Scsi_Host *shost = sas_ha->core.shost;
+
+	if (dev_is_expander(dev->dev_type))
+		return;
+
+	/*
+	 * For removed device with active IOs, the user space applications have
+	 * to spend very long time waiting for the timeout. This is not
+	 * necessary because a removed device will not return the IOs.
+	 * Abort the inflight IOs here so that EH can be quickly kicked in.
+	 */
+	blk_mq_tagset_busy_iter(&shost->tag_set, sas_abort_cmd, dev);
+}
+
 void sas_unregister_dev(struct asd_sas_port *port, struct domain_device *dev)
 {
 	if (!test_bit(SAS_DEV_DESTROY, &dev->state) &&
@@ -372,6 +399,8 @@ void sas_unregister_dev(struct asd_sas_port *port, struct domain_device *dev)
 	}
 
 	if (!test_and_set_bit(SAS_DEV_DESTROY, &dev->state)) {
+		if (test_bit(SAS_DEV_GONE, &dev->state))
+			sas_abort_device_scsi_cmds(dev);
 		sas_rphy_unlink(dev->rphy);
 		list_move_tail(&dev->disco_list_node, &port->destroy_list);
 	}
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index cf55f8e3bd9f..5e3a93d13a91 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -1036,7 +1036,6 @@ struct lpfc_hba {
 #define FCF_TS_INPROG           0x200 /* FCF table scan in progress */
 #define FCF_RR_INPROG           0x400 /* FCF roundrobin flogi in progress */
 #define HBA_FIP_SUPPORT		0x800 /* FIP support in HBA */
-#define HBA_AER_ENABLED		0x1000 /* AER enabled with HBA */
 #define HBA_DEVLOSS_TMO         0x2000 /* HBA in devloss timeout */
 #define HBA_RRQ_ACTIVE		0x4000 /* process the rrq active list */
 #define HBA_IOQ_FLUSH		0x8000 /* FCP/NVME I/O queues being flushed */
@@ -1190,7 +1189,6 @@ struct lpfc_hba {
 #define LPFC_MAX_ENBL_FC4_TYPE LPFC_ENABLE_FCP
 #define LPFC_DEF_ENBL_FC4_TYPE LPFC_ENABLE_FCP
 #endif
-	uint32_t cfg_aer_support;
 	uint32_t cfg_sriov_nr_virtfn;
 	uint32_t cfg_request_firmware_upgrade;
 	uint32_t cfg_suppress_link_up;
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index 22f2e046e8eb..3863a5341782 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -1644,6 +1644,12 @@ lpfc_sli4_pdev_status_reg_wait(struct lpfc_hba *phba)
 	    !bf_get(lpfc_sliport_status_err, &portstat_reg))
 		return -EPERM;
 
+	/* There is no point to wait if the port is in an unrecoverable
+	 * state.
+	 */
+	if (lpfc_sli4_unrecoverable_port(&portstat_reg))
+		return -EIO;
+
 	/* wait for the SLI port firmware ready after firmware reset */
 	for (i = 0; i < LPFC_FW_RESET_MAXIMUM_WAIT_10MS_CNT; i++) {
 		msleep(10);
@@ -4365,13 +4371,22 @@ static DEVICE_ATTR_RW(lpfc_link_speed);
 
 /*
 # lpfc_aer_support: Support PCIe device Advanced Error Reporting (AER)
-#       0  = aer disabled or not supported
 #       1  = aer supported and enabled (default)
-# Value range is [0,1]. Default value is 1.
+# PCIe error reporting is always enabled by the PCI core, so this always
+# shows 1.
+#
+# N.B. Parts of LPFC_ATTR open-coded since some of the underlying
+# infrastructure (phba->cfg_aer_support) is gone.
 */
-LPFC_ATTR(aer_support, 1, 0, 1,
-	"Enable PCIe device AER support");
-lpfc_param_show(aer_support)
+static uint lpfc_aer_support = 1;
+module_param(lpfc_aer_support, uint, S_IRUGO);
+MODULE_PARM_DESC(lpfc_aer_support, "Enable PCIe device AER support");
+static ssize_t
+lpfc_aer_support_show(struct device *dev, struct device_attribute *attr,
+		      char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%d\n", lpfc_aer_support);
+}
 
 /**
  * lpfc_aer_support_store - Set the adapter for aer support
@@ -4382,76 +4397,27 @@ lpfc_param_show(aer_support)
  * @count: unused variable.
  *
  * Description:
- * If the val is 1 and currently the device's AER capability was not
- * enabled, invoke the kernel's enable AER helper routine, trying to
- * enable the device's AER capability. If the helper routine enabling
- * AER returns success, update the device's cfg_aer_support flag to
- * indicate AER is supported by the device; otherwise, if the device
- * AER capability is already enabled to support AER, then do nothing.
- *
- * If the val is 0 and currently the device's AER support was enabled,
- * invoke the kernel's disable AER helper routine. After that, update
- * the device's cfg_aer_support flag to indicate AER is not supported
- * by the device; otherwise, if the device AER capability is already
- * disabled from supporting AER, then do nothing.
+ * PCIe error reporting is enabled by the PCI core, so drivers don't need
+ * to do anything.  Retain this interface for backwards compatibility,
+ * but do nothing.
  *
  * Returns:
- * length of the buf on success if val is in range the intended mode
- * is supported.
- * -EINVAL if val out of range or intended mode is not supported.
+ * length of the buf on success
+ * -EINVAL if val out of range
  **/
 static ssize_t
 lpfc_aer_support_store(struct device *dev, struct device_attribute *attr,
 		       const char *buf, size_t count)
 {
-	struct Scsi_Host *shost = class_to_shost(dev);
-	struct lpfc_vport *vport = (struct lpfc_vport *)shost->hostdata;
-	struct lpfc_hba *phba = vport->phba;
-	int val = 0, rc = -EINVAL;
+	int val = 0;
 
 	if (!isdigit(buf[0]))
 		return -EINVAL;
 	if (sscanf(buf, "%i", &val) != 1)
 		return -EINVAL;
 
-	switch (val) {
-	case 0:
-		if (phba->hba_flag & HBA_AER_ENABLED) {
-			rc = pci_disable_pcie_error_reporting(phba->pcidev);
-			if (!rc) {
-				spin_lock_irq(&phba->hbalock);
-				phba->hba_flag &= ~HBA_AER_ENABLED;
-				spin_unlock_irq(&phba->hbalock);
-				phba->cfg_aer_support = 0;
-				rc = strlen(buf);
-			} else
-				rc = -EPERM;
-		} else {
-			phba->cfg_aer_support = 0;
-			rc = strlen(buf);
-		}
-		break;
-	case 1:
-		if (!(phba->hba_flag & HBA_AER_ENABLED)) {
-			rc = pci_enable_pcie_error_reporting(phba->pcidev);
-			if (!rc) {
-				spin_lock_irq(&phba->hbalock);
-				phba->hba_flag |= HBA_AER_ENABLED;
-				spin_unlock_irq(&phba->hbalock);
-				phba->cfg_aer_support = 1;
-				rc = strlen(buf);
-			} else
-				 rc = -EPERM;
-		} else {
-			phba->cfg_aer_support = 1;
-			rc = strlen(buf);
-		}
-		break;
-	default:
-		rc = -EINVAL;
-		break;
-	}
-	return rc;
+	dev_info_once(dev, "PCIe error reporting automatically enabled by the PCI core; sysfs write ignored\n");
+	return strlen(buf);
 }
 
 static DEVICE_ATTR_RW(lpfc_aer_support);
@@ -4464,16 +4430,16 @@ static DEVICE_ATTR_RW(lpfc_aer_support);
  * @count: unused variable.
  *
  * Description:
- * If the @buf contains 1 and the device currently has the AER support
- * enabled, then invokes the kernel AER helper routine
+ * If the @buf contains 1, invokes the kernel AER helper routine
  * pci_aer_clear_nonfatal_status() to clean up the uncorrectable
  * error status register.
  *
  * Notes:
  *
  * Returns:
- * -EINVAL if the buf does not contain the 1 or the device is not currently
- * enabled with the AER support.
+ * -EINVAL if the buf does not contain 1
+ * -EPERM if the OS cannot clear AER error status, i.e., when platform
+ * firmware owns the AER Capability
  **/
 static ssize_t
 lpfc_aer_cleanup_state(struct device *dev, struct device_attribute *attr,
@@ -4491,8 +4457,7 @@ lpfc_aer_cleanup_state(struct device *dev, struct device_attribute *attr,
 	if (val != 1)
 		return -EINVAL;
 
-	if (phba->hba_flag & HBA_AER_ENABLED)
-		rc = pci_aer_clear_nonfatal_status(phba->pcidev);
+	rc = pci_aer_clear_nonfatal_status(phba->pcidev);
 
 	if (rc == 0)
 		return strlen(buf);
@@ -7277,7 +7242,6 @@ lpfc_get_cfgparam(struct lpfc_hba *phba)
 
 	lpfc_sg_seg_cnt_init(phba, lpfc_sg_seg_cnt);
 	lpfc_hba_queue_depth_init(phba, lpfc_hba_queue_depth);
-	lpfc_aer_support_init(phba, lpfc_aer_support);
 	lpfc_sriov_nr_virtfn_init(phba, lpfc_sriov_nr_virtfn);
 	lpfc_request_firmware_upgrade_init(phba, lpfc_req_fw_upgrade);
 	lpfc_suppress_link_up_init(phba, lpfc_suppress_link_up);
diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c
index 852b025e2fec..9a322a3a2150 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.c
+++ b/drivers/scsi/lpfc/lpfc_bsg.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2009-2015 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -134,8 +134,8 @@ lpfc_free_bsg_buffers(struct lpfc_hba *phba, struct lpfc_dmabuf *mlist)
 	if (mlist) {
 		list_for_each_entry_safe(mlast, next_mlast, &mlist->list,
 					 list) {
-			lpfc_mbuf_free(phba, mlast->virt, mlast->phys);
 			list_del(&mlast->list);
+			lpfc_mbuf_free(phba, mlast->virt, mlast->phys);
 			kfree(mlast);
 		}
 		lpfc_mbuf_free(phba, mlist->virt, mlist->phys);
diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h
index 976fd5ee7f7e..b833b983e69d 100644
--- a/drivers/scsi/lpfc/lpfc_crtn.h
+++ b/drivers/scsi/lpfc/lpfc_crtn.h
@@ -458,6 +458,8 @@ void lpfc_get_cfgparam(struct lpfc_hba *);
 void lpfc_get_vport_cfgparam(struct lpfc_vport *);
 int lpfc_alloc_sysfs_attr(struct lpfc_vport *);
 void lpfc_free_sysfs_attr(struct lpfc_vport *);
+bool lpfc_error_lost_link(struct lpfc_vport *vport, u32 ulp_status,
+			  u32 ulp_word4);
 extern const struct attribute_group *lpfc_hba_groups[];
 extern const struct attribute_group *lpfc_vport_groups[];
 extern struct scsi_host_template lpfc_template;
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index e941a99aa965..f3bdcebe67f5 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -476,8 +476,8 @@ lpfc_free_ct_rsp(struct lpfc_hba *phba, struct lpfc_dmabuf *mlist)
 	struct lpfc_dmabuf *mlast, *next_mlast;
 
 	list_for_each_entry_safe(mlast, next_mlast, &mlist->list, list) {
-		lpfc_mbuf_free(phba, mlast->virt, mlast->phys);
 		list_del(&mlast->list);
+		lpfc_mbuf_free(phba, mlast->virt, mlast->phys);
 		kfree(mlast);
 	}
 	lpfc_mbuf_free(phba, mlist->virt, mlist->phys);
@@ -958,7 +958,7 @@ lpfc_cmpl_ct_cmd_gid_ft(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		lpfc_vport_set_state(vport, FC_VPORT_FAILED);
 		goto out;
 	}
-	if (lpfc_error_lost_link(ulp_status, ulp_word4)) {
+	if (lpfc_error_lost_link(vport, ulp_status, ulp_word4)) {
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
 				 "0226 NS query failed due to link event: "
 				 "ulp_status x%x ulp_word4 x%x fc_flag x%x "
@@ -1181,7 +1181,7 @@ lpfc_cmpl_ct_cmd_gid_pt(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		lpfc_vport_set_state(vport, FC_VPORT_FAILED);
 		goto out;
 	}
-	if (lpfc_error_lost_link(ulp_status, ulp_word4)) {
+	if (lpfc_error_lost_link(vport, ulp_status, ulp_word4)) {
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
 				 "4166 NS query failed due to link event: "
 				 "ulp_status x%x ulp_word4 x%x fc_flag x%x "
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
index f5252e45a48a..bdf34af4ef36 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.c
+++ b/drivers/scsi/lpfc/lpfc_debugfs.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2007-2015 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -2157,10 +2157,13 @@ lpfc_debugfs_lockstat_write(struct file *file, const char __user *buf,
 	char mybuf[64];
 	char *pbuf;
 	int i;
+	size_t bsize;
 
 	memset(mybuf, 0, sizeof(mybuf));
 
-	if (copy_from_user(mybuf, buf, nbytes))
+	bsize = min(nbytes, (sizeof(mybuf) - 1));
+
+	if (copy_from_user(mybuf, buf, bsize))
 		return -EFAULT;
 	pbuf = &mybuf[0];
 
@@ -2181,7 +2184,7 @@ lpfc_debugfs_lockstat_write(struct file *file, const char __user *buf,
 			qp->lock_conflict.wq_access = 0;
 		}
 	}
-	return nbytes;
+	return bsize;
 }
 #endif
 
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index 35b252f1ef73..6a15f879e517 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -1088,7 +1088,7 @@ stop_rr_fcf_flogi:
 			}
 
 			/* Do not register VFI if the driver aborted FLOGI */
-			if (!lpfc_error_lost_link(ulp_status, ulp_word4))
+			if (!lpfc_error_lost_link(vport, ulp_status, ulp_word4))
 				lpfc_issue_reg_vfi(vport);
 
 			lpfc_nlp_put(ndlp);
@@ -1207,7 +1207,7 @@ flogifail:
 	phba->fcf.fcf_flag &= ~FCF_DISCOVERY;
 	spin_unlock_irq(&phba->hbalock);
 
-	if (!lpfc_error_lost_link(ulp_status, ulp_word4)) {
+	if (!lpfc_error_lost_link(vport, ulp_status, ulp_word4)) {
 		/* FLOGI failed, so just use loop map to make discovery list */
 		lpfc_disc_list_loopmap(vport);
 
@@ -2087,7 +2087,7 @@ lpfc_cmpl_els_plogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 					 ulp_word4);
 
 		/* Do not call DSM for lpfc_els_abort'ed ELS cmds */
-		if (!lpfc_error_lost_link(ulp_status, ulp_word4))
+		if (!lpfc_error_lost_link(vport, ulp_status, ulp_word4))
 			lpfc_disc_state_machine(vport, ndlp, cmdiocb,
 						NLP_EVT_CMPL_PLOGI);
 
@@ -2208,14 +2208,15 @@ lpfc_issue_els_plogi(struct lpfc_vport *vport, uint32_t did, uint8_t retry)
 	 * outstanding UNREG_RPI mbox command completes, unless we
 	 * are going offline. This logic does not apply for Fabric DIDs
 	 */
-	if ((ndlp->nlp_flag & NLP_UNREG_INP) &&
+	if ((ndlp->nlp_flag & (NLP_IGNR_REG_CMPL | NLP_UNREG_INP)) &&
 	    ((ndlp->nlp_DID & Fabric_DID_MASK) != Fabric_DID_MASK) &&
 	    !(vport->fc_flag & FC_OFFLINE_MODE)) {
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
 				 "4110 Issue PLOGI x%x deferred "
-				 "on NPort x%x rpi x%x Data: x%px\n",
+				 "on NPort x%x rpi x%x flg x%x Data:"
+				 " x%px\n",
 				 ndlp->nlp_defer_did, ndlp->nlp_DID,
-				 ndlp->nlp_rpi, ndlp);
+				 ndlp->nlp_rpi, ndlp->nlp_flag, ndlp);
 
 		/* We can only defer 1st PLOGI */
 		if (ndlp->nlp_defer_did == NLP_EVT_NOTHING_PENDING)
@@ -2382,7 +2383,7 @@ lpfc_cmpl_els_prli(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 				 ndlp->fc4_prli_sent);
 
 		/* Do not call DSM for lpfc_els_abort'ed ELS cmds */
-		if (!lpfc_error_lost_link(ulp_status, ulp_word4))
+		if (!lpfc_error_lost_link(vport, ulp_status, ulp_word4))
 			lpfc_disc_state_machine(vport, ndlp, cmdiocb,
 						NLP_EVT_CMPL_PRLI);
 
@@ -3037,15 +3038,16 @@ lpfc_cmpl_els_logo(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 				 ndlp->nlp_DID, ulp_status,
 				 ulp_word4);
 
-		if (lpfc_error_lost_link(ulp_status, ulp_word4)) {
+		if (lpfc_error_lost_link(vport, ulp_status, ulp_word4))
 			skip_recovery = 1;
-			goto out;
-		}
 	}
 
 	/* Call state machine. This will unregister the rpi if needed. */
 	lpfc_disc_state_machine(vport, ndlp, cmdiocb, NLP_EVT_CMPL_LOGO);
 
+	if (skip_recovery)
+		goto out;
+
 	/* The driver sets this flag for an NPIV instance that doesn't want to
 	 * log into the remote port.
 	 */
@@ -4928,7 +4930,7 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 
 	if ((cmd == ELS_CMD_FLOGI) &&
 	    (phba->fc_topology != LPFC_TOPOLOGY_LOOP) &&
-	    !lpfc_error_lost_link(ulp_status, ulp_word4)) {
+	    !lpfc_error_lost_link(vport, ulp_status, ulp_word4)) {
 		/* FLOGI retry policy */
 		retry = 1;
 		/* retry FLOGI forever */
@@ -4942,7 +4944,7 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		else if (cmdiocb->retry >= 32)
 			delay = 1000;
 	} else if ((cmd == ELS_CMD_FDISC) &&
-	    !lpfc_error_lost_link(ulp_status, ulp_word4)) {
+	    !lpfc_error_lost_link(vport, ulp_status, ulp_word4)) {
 		/* retry FDISCs every second up to devloss */
 		retry = 1;
 		maxretry = vport->cfg_devloss_tmo;
@@ -5455,18 +5457,20 @@ out:
 	 * these conditions and release the RPI.
 	 */
 	if (phba->sli_rev == LPFC_SLI_REV4 &&
-	    (vport && vport->port_type == LPFC_NPIV_PORT) &&
-	    !(ndlp->fc4_xpt_flags & SCSI_XPT_REGD) &&
-	    ndlp->nlp_flag & NLP_RELEASE_RPI) {
-		if (ndlp->nlp_state !=  NLP_STE_PLOGI_ISSUE &&
-		    ndlp->nlp_state != NLP_STE_REG_LOGIN_ISSUE) {
-			lpfc_sli4_free_rpi(phba, ndlp->nlp_rpi);
-			spin_lock_irq(&ndlp->lock);
-			ndlp->nlp_rpi = LPFC_RPI_ALLOC_ERROR;
-			ndlp->nlp_flag &= ~NLP_RELEASE_RPI;
-			spin_unlock_irq(&ndlp->lock);
-			lpfc_drop_node(vport, ndlp);
+	    vport && vport->port_type == LPFC_NPIV_PORT &&
+	    !(ndlp->fc4_xpt_flags & SCSI_XPT_REGD)) {
+		if (ndlp->nlp_flag & NLP_RELEASE_RPI) {
+			if (ndlp->nlp_state != NLP_STE_PLOGI_ISSUE &&
+			    ndlp->nlp_state != NLP_STE_REG_LOGIN_ISSUE) {
+				lpfc_sli4_free_rpi(phba, ndlp->nlp_rpi);
+				spin_lock_irq(&ndlp->lock);
+				ndlp->nlp_rpi = LPFC_RPI_ALLOC_ERROR;
+				ndlp->nlp_flag &= ~NLP_RELEASE_RPI;
+				spin_unlock_irq(&ndlp->lock);
+			}
 		}
+
+		lpfc_drop_node(vport, ndlp);
 	}
 
 	/* Release the originating I/O reference. */
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index 66cd0b1dbbd0..5ba3a9ad9501 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -5755,8 +5755,8 @@ lpfc_setup_disc_node(struct lpfc_vport *vport, uint32_t did)
 			     (NLP_FCP_TARGET | NLP_NVME_TARGET)))
 				return NULL;
 
-			ndlp->nlp_prev_state = ndlp->nlp_state;
-			lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE);
+			lpfc_disc_state_machine(vport, ndlp, NULL,
+						NLP_EVT_DEVICE_RECOVERY);
 
 			spin_lock_irq(&ndlp->lock);
 			ndlp->nlp_flag |= NLP_NPR_2B_DISC;
@@ -7269,3 +7269,38 @@ lpfc_parse_fcoe_conf(struct lpfc_hba *phba,
 		lpfc_read_fcf_conn_tbl(phba, rec_ptr);
 
 }
+
+/*
+ * lpfc_error_lost_link - IO failure from link event or FW reset check.
+ *
+ * @vport: Pointer to lpfc_vport data structure.
+ * @ulp_status: IO completion status.
+ * @ulp_word4: Reason code for the ulp_status.
+ *
+ * This function evaluates the ulp_status and ulp_word4 values
+ * for specific error values that indicate an internal link fault
+ * or fw reset event for the completing IO.  Callers require this
+ * common data to decide next steps on the IO.
+ *
+ * Return:
+ * false - No link or reset error occurred.
+ * true - A link or reset error occurred.
+ */
+bool
+lpfc_error_lost_link(struct lpfc_vport *vport, u32 ulp_status, u32 ulp_word4)
+{
+	/* Mask off the extra port data to get just the reason code. */
+	u32 rsn_code = IOERR_PARAM_MASK & ulp_word4;
+
+	if (ulp_status == IOSTAT_LOCAL_REJECT &&
+	    (rsn_code == IOERR_SLI_ABORTED ||
+	     rsn_code == IOERR_LINK_DOWN ||
+	     rsn_code == IOERR_SLI_DOWN)) {
+		lpfc_printf_vlog(vport, KERN_WARNING, LOG_SLI | LOG_ELS,
+				 "0408 Report link error true: <x%x:x%x>\n",
+				 ulp_status, ulp_word4);
+		return true;
+	}
+
+	return false;
+}
diff --git a/drivers/scsi/lpfc/lpfc_hw.h b/drivers/scsi/lpfc/lpfc_hw.h
index 5c283936ff08..19b2d2754f32 100644
--- a/drivers/scsi/lpfc/lpfc_hw.h
+++ b/drivers/scsi/lpfc/lpfc_hw.h
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -4435,16 +4435,4 @@ lpfc_is_LC_HBA(unsigned short device)
 		return 0;
 }
 
-/*
- * Determine if failed because of a link event or firmware reset.
- */
-static inline int
-lpfc_error_lost_link(u32 ulp_status, u32 ulp_word4)
-{
-	return (ulp_status == IOSTAT_LOCAL_REJECT &&
-		(ulp_word4 == IOERR_SLI_ABORTED ||
-		 ulp_word4 == IOERR_LINK_DOWN ||
-		 ulp_word4 == IOERR_SLI_DOWN));
-}
-
 #define BPL_ALIGN_SZ 8 /* 8 byte alignment for bpl and mbufs */
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 4f7485958c49..867b4c788f08 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -32,7 +32,6 @@
 #include <linux/spinlock.h>
 #include <linux/sched/clock.h>
 #include <linux/ctype.h>
-#include <linux/aer.h>
 #include <linux/slab.h>
 #include <linux/firmware.h>
 #include <linux/miscdevice.h>
@@ -2148,7 +2147,7 @@ lpfc_handle_eratt_s4(struct lpfc_hba *phba)
 		/* fall through for not able to recover */
 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
 				"3152 Unrecoverable error\n");
-		phba->link_state = LPFC_HBA_ERROR;
+		lpfc_sli4_offline_eratt(phba);
 		break;
 	case LPFC_SLI_INTF_IF_TYPE_1:
 	default:
@@ -9569,8 +9568,7 @@ lpfc_sli4_post_status_check(struct lpfc_hba *phba)
 			/* Final checks.  The port status should be clean. */
 			if (lpfc_readl(phba->sli4_hba.u.if_type2.STATUSregaddr,
 				&reg_data.word0) ||
-				(bf_get(lpfc_sliport_status_err, &reg_data) &&
-				 !bf_get(lpfc_sliport_status_rn, &reg_data))) {
+				lpfc_sli4_unrecoverable_port(&reg_data)) {
 				phba->work_status[0] =
 					readl(phba->sli4_hba.u.if_type2.
 					      ERR1regaddr);
@@ -12026,7 +12024,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba)
 				goto out_iounmap_all;
 		} else {
 			error = -ENOMEM;
-			goto out_iounmap_all;
+			goto out_iounmap_ctrl;
 		}
 	}
 
@@ -12044,7 +12042,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba)
 			dev_err(&pdev->dev,
 			   "ioremap failed for SLI4 HBA dpp registers.\n");
 			error = -ENOMEM;
-			goto out_iounmap_ctrl;
+			goto out_iounmap_all;
 		}
 		phba->pci_bar4_memmap_p = phba->sli4_hba.dpp_regs_memmap_p;
 	}
@@ -12069,9 +12067,11 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba)
 	return 0;
 
 out_iounmap_all:
-	iounmap(phba->sli4_hba.drbl_regs_memmap_p);
+	if (phba->sli4_hba.drbl_regs_memmap_p)
+		iounmap(phba->sli4_hba.drbl_regs_memmap_p);
 out_iounmap_ctrl:
-	iounmap(phba->sli4_hba.ctrl_regs_memmap_p);
+	if (phba->sli4_hba.ctrl_regs_memmap_p)
+		iounmap(phba->sli4_hba.ctrl_regs_memmap_p);
 out_iounmap_conf:
 	iounmap(phba->sli4_hba.conf_regs_memmap_p);
 
@@ -12107,6 +12107,7 @@ lpfc_sli4_pci_mem_unset(struct lpfc_hba *phba)
 			iounmap(phba->sli4_hba.dpp_regs_memmap_p);
 		break;
 	case LPFC_SLI_INTF_IF_TYPE_1:
+		break;
 	default:
 		dev_printk(KERN_ERR, &phba->pcidev->dev,
 			   "FATAL - unsupported SLI4 interface type - %d\n",
diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c
index 152245f7cacc..adda70423c77 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.c
+++ b/drivers/scsi/lpfc/lpfc_nvme.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -2265,6 +2265,7 @@ lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
 			}
 			if (!vport->localport ||
 			    test_bit(HBA_PCI_ERR, &vport->phba->bit_flags) ||
+			    phba->link_state == LPFC_HBA_ERROR ||
 			    vport->load_flag & FC_UNLOADING)
 				return;
 
@@ -2630,7 +2631,8 @@ lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 		 * return values is ignored.  The upcall is a courtesy to the
 		 * transport.
 		 */
-		if (vport->load_flag & FC_UNLOADING)
+		if (vport->load_flag & FC_UNLOADING ||
+		    unlikely(vport->phba->link_state == LPFC_HBA_ERROR))
 			(void)nvme_fc_set_remoteport_devloss(remoteport, 0);
 
 		ret = nvme_fc_unregister_remoteport(remoteport);
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index cf630aa6734e..8693578888f1 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -34,7 +34,6 @@
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_transport_fc.h>
 #include <scsi/fc/fc_fs.h>
-#include <linux/aer.h>
 #include <linux/crash_dump.h>
 #ifdef CONFIG_X86
 #include <asm/set_memory.h>
@@ -5204,13 +5203,9 @@ lpfc_sli_brdrestart_s3(struct lpfc_hba *phba)
 	volatile struct MAILBOX_word0 mb;
 	struct lpfc_sli *psli;
 	void __iomem *to_slim;
-	uint32_t hba_aer_enabled;
 
 	spin_lock_irq(&phba->hbalock);
 
-	/* Take PCIe device Advanced Error Reporting (AER) state */
-	hba_aer_enabled = phba->hba_flag & HBA_AER_ENABLED;
-
 	psli = &phba->sli;
 
 	/* Restart HBA */
@@ -5251,10 +5246,6 @@ lpfc_sli_brdrestart_s3(struct lpfc_hba *phba)
 	/* Give the INITFF and Post time to settle. */
 	mdelay(100);
 
-	/* Reset HBA AER if it was enabled, note hba_flag was reset above */
-	if (hba_aer_enabled)
-		pci_disable_pcie_error_reporting(phba->pcidev);
-
 	lpfc_hba_down_post(phba);
 
 	return 0;
@@ -5273,7 +5264,6 @@ static int
 lpfc_sli_brdrestart_s4(struct lpfc_hba *phba)
 {
 	struct lpfc_sli *psli = &phba->sli;
-	uint32_t hba_aer_enabled;
 	int rc;
 
 	/* Restart HBA */
@@ -5281,9 +5271,6 @@ lpfc_sli_brdrestart_s4(struct lpfc_hba *phba)
 			"0296 Restart HBA Data: x%x x%x\n",
 			phba->pport->port_state, psli->sli_flag);
 
-	/* Take PCIe device Advanced Error Reporting (AER) state */
-	hba_aer_enabled = phba->hba_flag & HBA_AER_ENABLED;
-
 	rc = lpfc_sli4_brdreset(phba);
 	if (rc) {
 		phba->link_state = LPFC_HBA_ERROR;
@@ -5301,10 +5288,6 @@ lpfc_sli_brdrestart_s4(struct lpfc_hba *phba)
 	memset(&psli->lnk_stat_offsets, 0, sizeof(psli->lnk_stat_offsets));
 	psli->stats_start = ktime_get_seconds();
 
-	/* Reset HBA AER if it was enabled, note hba_flag was reset above */
-	if (hba_aer_enabled)
-		pci_disable_pcie_error_reporting(phba->pcidev);
-
 hba_down_queue:
 	lpfc_hba_down_post(phba);
 	lpfc_sli4_queue_destroy(phba);
@@ -5725,25 +5708,6 @@ lpfc_sli_hba_setup(struct lpfc_hba *phba)
 	}
 	phba->fcp_embed_io = 0;	/* SLI4 FC support only */
 
-	/* Enable PCIe device Advanced Error Reporting (AER) if configured */
-	if (phba->cfg_aer_support == 1 && !(phba->hba_flag & HBA_AER_ENABLED)) {
-		rc = pci_enable_pcie_error_reporting(phba->pcidev);
-		if (!rc) {
-			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
-					"2709 This device supports "
-					"Advanced Error Reporting (AER)\n");
-			spin_lock_irq(&phba->hbalock);
-			phba->hba_flag |= HBA_AER_ENABLED;
-			spin_unlock_irq(&phba->hbalock);
-		} else {
-			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
-					"2708 This device does not support "
-					"Advanced Error Reporting (AER): %d\n",
-					rc);
-			phba->cfg_aer_support = 0;
-		}
-	}
-
 	if (phba->sli_rev == 3) {
 		phba->iocb_cmd_size = SLI3_IOCB_CMD_SIZE;
 		phba->iocb_rsp_size = SLI3_IOCB_RSP_SIZE;
@@ -8080,16 +8044,16 @@ int lpfc_rx_monitor_create_ring(struct lpfc_rx_info_monitor *rx_monitor,
 /**
  * lpfc_rx_monitor_destroy_ring - Free ring buffer for rx_monitor
  * @rx_monitor: Pointer to lpfc_rx_info_monitor object
+ *
+ * Called after cancellation of cmf_timer.
  **/
 void lpfc_rx_monitor_destroy_ring(struct lpfc_rx_info_monitor *rx_monitor)
 {
-	spin_lock(&rx_monitor->lock);
 	kfree(rx_monitor->ring);
 	rx_monitor->ring = NULL;
 	rx_monitor->entries = 0;
 	rx_monitor->head_idx = 0;
 	rx_monitor->tail_idx = 0;
-	spin_unlock(&rx_monitor->lock);
 }
 
 /**
@@ -9053,25 +9017,6 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
 	mod_timer(&phba->eratt_poll,
 		  jiffies + msecs_to_jiffies(1000 * phba->eratt_poll_interval));
 
-	/* Enable PCIe device Advanced Error Reporting (AER) if configured */
-	if (phba->cfg_aer_support == 1 && !(phba->hba_flag & HBA_AER_ENABLED)) {
-		rc = pci_enable_pcie_error_reporting(phba->pcidev);
-		if (!rc) {
-			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
-					"2829 This device supports "
-					"Advanced Error Reporting (AER)\n");
-			spin_lock_irq(&phba->hbalock);
-			phba->hba_flag |= HBA_AER_ENABLED;
-			spin_unlock_irq(&phba->hbalock);
-		} else {
-			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
-					"2830 This device does not support "
-					"Advanced Error Reporting (AER)\n");
-			phba->cfg_aer_support = 0;
-		}
-		rc = 0;
-	}
-
 	/*
 	 * The port is ready, set the host's link state to LINK_DOWN
 	 * in preparation for link interrupts.
@@ -9895,7 +9840,8 @@ lpfc_sli4_async_mbox_unblock(struct lpfc_hba *phba)
  * port for twice the regular mailbox command timeout value.
  *
  *      0 - no timeout on waiting for bootstrap mailbox register ready.
- *      MBXERR_ERROR - wait for bootstrap mailbox register timed out.
+ *      MBXERR_ERROR - wait for bootstrap mailbox register timed out or port
+ *                     is in an unrecoverable state.
  **/
 static int
 lpfc_sli4_wait_bmbx_ready(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
@@ -9903,6 +9849,23 @@ lpfc_sli4_wait_bmbx_ready(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 	uint32_t db_ready;
 	unsigned long timeout;
 	struct lpfc_register bmbx_reg;
+	struct lpfc_register portstat_reg = {-1};
+
+	/* Sanity check - there is no point to wait if the port is in an
+	 * unrecoverable state.
+	 */
+	if (bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) >=
+	    LPFC_SLI_INTF_IF_TYPE_2) {
+		if (lpfc_readl(phba->sli4_hba.u.if_type2.STATUSregaddr,
+			       &portstat_reg.word0) ||
+		    lpfc_sli4_unrecoverable_port(&portstat_reg)) {
+			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+					"3858 Skipping bmbx ready because "
+					"Port Status x%x\n",
+					portstat_reg.word0);
+			return MBXERR_ERROR;
+		}
+	}
 
 	timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, mboxq)
 				   * 1000) + jiffies;
@@ -22325,10 +22288,10 @@ lpfc_free_sgl_per_hdwq(struct lpfc_hba *phba,
 	/* Free sgl pool */
 	list_for_each_entry_safe(list_entry, tmp,
 				 buf_list, list_node) {
+		list_del(&list_entry->list_node);
 		dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 			      list_entry->dma_sgl,
 			      list_entry->dma_phys_sgl);
-		list_del(&list_entry->list_node);
 		kfree(list_entry);
 	}
 
@@ -22475,10 +22438,10 @@ lpfc_free_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba,
 	list_for_each_entry_safe(list_entry, tmp,
 				 buf_list,
 				 list_node) {
+		list_del(&list_entry->list_node);
 		dma_pool_free(phba->lpfc_cmd_rsp_buf_pool,
 			      list_entry->fcp_cmnd,
 			      list_entry->fcp_cmd_rsp_dma_handle);
-		list_del(&list_entry->list_node);
 		kfree(list_entry);
 	}
 
diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h
index 3b62c4032c31..2a0864e6d7cd 100644
--- a/drivers/scsi/lpfc/lpfc_sli4.h
+++ b/drivers/scsi/lpfc/lpfc_sli4.h
@@ -1180,3 +1180,22 @@ static inline void *lpfc_sli4_qe(struct lpfc_queue *q, uint16_t idx)
 	return q->q_pgs[idx / q->entry_cnt_per_pg] +
 		(q->entry_size * (idx % q->entry_cnt_per_pg));
 }
+
+/**
+ * lpfc_sli4_unrecoverable_port - Check ERR and RN bits in portstat_reg
+ * @portstat_reg: portstat_reg pointer containing portstat_reg contents
+ *
+ * Description:
+ * Use only for SLI4 interface type-2 or later.  If ERR is set && RN is 0, then
+ * port is deemed unrecoverable.
+ *
+ * Returns:
+ * true		- ERR && !RN
+ * false	- otherwise
+ */
+static inline bool
+lpfc_sli4_unrecoverable_port(struct lpfc_register *portstat_reg)
+{
+	return bf_get(lpfc_sliport_status_err, portstat_reg) &&
+	       !bf_get(lpfc_sliport_status_rn, portstat_reg);
+}
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index 0238208cdd11..c97411b0992e 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -20,7 +20,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "14.2.0.10"
+#define LPFC_DRIVER_VERSION "14.2.0.11"
 #define LPFC_DRIVER_NAME		"lpfc"
 
 /* Used for SLI 2/3 */
diff --git a/drivers/scsi/mac53c94.c b/drivers/scsi/mac53c94.c
index f75928f7773e..6a019132109c 100644
--- a/drivers/scsi/mac53c94.c
+++ b/drivers/scsi/mac53c94.c
@@ -392,7 +392,7 @@ static void set_dma_cmds(struct fsc_state *state, struct scsi_cmnd *cmd)
 	mac53c94_priv(cmd)->this_residual = total;
 }
 
-static struct scsi_host_template mac53c94_template = {
+static const struct scsi_host_template mac53c94_template = {
 	.proc_name	= "53c94",
 	.name		= "53C94",
 	.queuecommand	= mac53c94_queue,
diff --git a/drivers/scsi/mac_esp.c b/drivers/scsi/mac_esp.c
index 6d23ab5aee56..3f0061b00494 100644
--- a/drivers/scsi/mac_esp.c
+++ b/drivers/scsi/mac_esp.c
@@ -289,7 +289,7 @@ static struct esp_driver_ops mac_esp_ops = {
 
 static int esp_mac_probe(struct platform_device *dev)
 {
-	struct scsi_host_template *tpnt = &scsi_esp_template;
+	const struct scsi_host_template *tpnt = &scsi_esp_template;
 	struct Scsi_Host *host;
 	struct esp *esp;
 	int err;
diff --git a/drivers/scsi/megaraid.c b/drivers/scsi/megaraid.c
index bf491af9f0d6..e92f1a73cc9b 100644
--- a/drivers/scsi/megaraid.c
+++ b/drivers/scsi/megaraid.c
@@ -1441,6 +1441,7 @@ mega_cmd_done(adapter_t *adapter, u8 completed[], int nstatus, int status)
 		 */
 		if (cmdid == CMDID_INT_CMDS) {
 			scb = &adapter->int_scb;
+			cmd = scb->cmd;
 
 			list_del_init(&scb->list);
 			scb->state = SCB_FREE;
@@ -4100,7 +4101,7 @@ mega_internal_command(adapter_t *adapter, megacmd_t *mc, mega_passthru *pthru)
 	return rval;
 }
 
-static struct scsi_host_template megaraid_template = {
+static const struct scsi_host_template megaraid_template = {
 	.module				= THIS_MODULE,
 	.name				= "MegaRAID",
 	.proc_name			= "megaraid_legacy",
diff --git a/drivers/scsi/megaraid/megaraid_mbox.c b/drivers/scsi/megaraid/megaraid_mbox.c
index 132de68c14e9..ef2b6380e19a 100644
--- a/drivers/scsi/megaraid/megaraid_mbox.c
+++ b/drivers/scsi/megaraid/megaraid_mbox.c
@@ -325,7 +325,7 @@ ATTRIBUTE_GROUPS(megaraid_sdev);
 /*
  * Scsi host template for megaraid unified driver
  */
-static struct scsi_host_template megaraid_template_g = {
+static const struct scsi_host_template megaraid_template_g = {
 	.module				= THIS_MODULE,
 	.name				= "LSI Logic MegaRAID driver",
 	.proc_name			= "megaraid",
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index c895189375e2..317c944c68e3 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -3505,7 +3505,7 @@ ATTRIBUTE_GROUPS(megaraid_host);
 /*
  * Scsi host template for megaraid_sas driver
  */
-static struct scsi_host_template megasas_template = {
+static const struct scsi_host_template megasas_template = {
 
 	.module = THIS_MODULE,
 	.name = "Avago SAS based MegaRAID driver",
diff --git a/drivers/scsi/mesh.c b/drivers/scsi/mesh.c
index 84b541a57b7b..e276583c590c 100644
--- a/drivers/scsi/mesh.c
+++ b/drivers/scsi/mesh.c
@@ -1830,7 +1830,7 @@ static int mesh_shutdown(struct macio_dev *mdev)
 	return 0;
 }
 
-static struct scsi_host_template mesh_template = {
+static const struct scsi_host_template mesh_template = {
 	.proc_name			= "mesh",
 	.name				= "MESH",
 	.queuecommand			= mesh_queue,
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h b/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h
index 0a2af48915a5..2fc196499c89 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- *  Copyright 2017-2022 Broadcom Inc. All rights reserved.
+ *  Copyright 2017-2023 Broadcom Inc. All rights reserved.
  */
 #ifndef MPI30_CNFG_H
 #define MPI30_CNFG_H     1
@@ -63,8 +63,9 @@
 #define MPI3_PCIE_LINK_PGAD_LINKNUM_MASK                (0x000000ff)
 #define MPI3_SECURITY_PGAD_FORM_MASK                    (0xf0000000)
 #define MPI3_SECURITY_PGAD_FORM_GET_NEXT_SLOT           (0x00000000)
-#define MPI3_SECURITY_PGAD_FORM_SOT_NUM                 (0x10000000)
+#define MPI3_SECURITY_PGAD_FORM_SLOT_NUM		(0x10000000)
 #define MPI3_SECURITY_PGAD_SLOT_GROUP_MASK              (0x0000ff00)
+#define MPI3_SECURITY_PGAD_SLOT_GROUP_SHIFT		(8)
 #define MPI3_SECURITY_PGAD_SLOT_MASK                    (0x000000ff)
 struct mpi3_config_request {
 	__le16             host_tag;
@@ -135,7 +136,6 @@ struct mpi3_config_page_header {
 #define MPI3_SAS_PHYINFO_PHY_POWER_CONDITION_ACTIVE     (0x00000000)
 #define MPI3_SAS_PHYINFO_PHY_POWER_CONDITION_PARTIAL    (0x08000000)
 #define MPI3_SAS_PHYINFO_PHY_POWER_CONDITION_SLUMBER    (0x10000000)
-#define MPI3_SAS_NEG_LINK_RATE_PHYSICAL_SHIFT                 (0)
 #define MPI3_SAS_PHYINFO_REQUESTED_INSIDE_ZPSDS_CHANGED_MASK  (0x04000000)
 #define MPI3_SAS_PHYINFO_REQUESTED_INSIDE_ZPSDS_CHANGED_SHIFT (26)
 #define MPI3_SAS_PHYINFO_INSIDE_ZPSDS_PERSISTENT_MASK         (0x02000000)
@@ -201,6 +201,11 @@ struct mpi3_config_page_header {
 #define MPI3_TEMP_SENSOR_LOCATION_DRAM                  (0x3)
 #define MPI3_MFGPAGE_VENDORID_BROADCOM                  (0x1000)
 #define MPI3_MFGPAGE_DEVID_SAS4116                      (0x00a5)
+#define MPI3_MFGPAGE_DEVID_SAS5116_MPI			(0x00b3)
+#define MPI3_MFGPAGE_DEVID_SAS5116_NVME			(0x00b4)
+#define MPI3_MFGPAGE_DEVID_SAS5116_MPI_MGMT		(0x00b5)
+#define MPI3_MFGPAGE_DEVID_SAS5116_NVME_MGMT		(0x00b6)
+#define MPI3_MFGPAGE_DEVID_SAS5116_PCIE_SWITCH		(0x00b8)
 struct mpi3_man_page0 {
 	struct mpi3_config_page_header         header;
 	u8                                 chip_revision[8];
@@ -466,7 +471,7 @@ struct mpi3_man_page9 {
 
 #define MPI3_MAN9_PAGEVERSION                   (0x00)
 struct mpi3_man10_istwi_ctrlr_entry {
-	__le16     slave_address;
+	__le16     target_address;
 	__le16     flags;
 	u8         scl_low_override;
 	u8         scl_high_override;
@@ -476,8 +481,8 @@ struct mpi3_man10_istwi_ctrlr_entry {
 #define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_SPEED_MASK         (0x000c)
 #define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_SPEED_100K         (0x0000)
 #define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_SPEED_400K         (0x0004)
-#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_SLAVE_ENABLED          (0x0002)
-#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_MASTER_ENABLED         (0x0001)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_TARGET_ENABLED          (0x0002)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_INITIATOR_ENABLED         (0x0001)
 #ifndef MPI3_MAN10_ISTWI_CTRLR_MAX
 #define MPI3_MAN10_ISTWI_CTRLR_MAX          (1)
 #endif
@@ -1160,7 +1165,7 @@ struct mpi3_io_unit_page12 {
 struct mpi3_iounit13_allowed_function {
 	__le16                             sub_function;
 	u8                                 function_code;
-	u8                                 fuction_flags;
+	u8                                 function_flags;
 };
 #define MPI3_IOUNIT13_FUNCTION_FLAGS_ADMIN_BLOCKED                 (0x04)
 #define MPI3_IOUNIT13_FUNCTION_FLAGS_OOB_BLOCKED                   (0x02)
@@ -1176,6 +1181,48 @@ struct mpi3_io_unit_page13 {
 #define MPI3_IOUNIT13_PAGEVERSION                                  (0x00)
 #define MPI3_IOUNIT13_FLAGS_ADMIN_BLOCKED                          (0x0002)
 #define MPI3_IOUNIT13_FLAGS_OOB_BLOCKED                            (0x0001)
+#ifndef MPI3_IOUNIT14_MD_MAX
+#define MPI3_IOUNIT14_MD_MAX                                       (1)
+#endif
+struct mpi3_iounit14_pagemetadata {
+	u8                                 page_type;
+	u8                                 page_number;
+	u8                                 reserved02;
+	u8                                 page_flags;
+};
+#define MPI3_IOUNIT14_PAGEMETADATA_PAGEFLAGS_OOBWRITE_ALLOWED      (0x02)
+#define MPI3_IOUNIT14_PAGEMETADATA_PAGEFLAGS_HOSTWRITE_ALLOWED     (0x01)
+struct mpi3_io_unit_page14 {
+	struct mpi3_config_page_header         header;
+	u8                                 flags;
+	u8                                 reserved09[3];
+	u8                                 num_pages;
+	u8                                 reserved0d[3];
+	struct mpi3_iounit14_pagemetadata      page_metadata[MPI3_IOUNIT14_MD_MAX];
+};
+#define MPI3_IOUNIT14_PAGEVERSION                                  (0x00)
+#define MPI3_IOUNIT14_FLAGS_READONLY                               (0x01)
+#ifndef MPI3_IOUNIT15_PBD_MAX
+#define MPI3_IOUNIT15_PBD_MAX                                       (1)
+#endif
+struct mpi3_io_unit_page15 {
+	struct mpi3_config_page_header         header;
+	u8                                 flags;
+	u8                                 reserved09[3];
+	__le32                             reserved0c;
+	u8                                 power_budgeting_capability;
+	u8                                 reserved11[3];
+	u8                                 num_power_budget_data;
+	u8                                 reserved15[3];
+	__le32                             power_budget_data[MPI3_IOUNIT15_PBD_MAX];
+};
+#define MPI3_IOUNIT15_PAGEVERSION                                   (0x00)
+#define MPI3_IOUNIT15_FLAGS_EPRINIT_INITREQUIRED                    (0x04)
+#define MPI3_IOUNIT15_FLAGS_EPRSUPPORT_MASK                         (0x03)
+#define MPI3_IOUNIT15_FLAGS_EPRSUPPORT_NOT_SUPPORTED                (0x00)
+#define MPI3_IOUNIT15_FLAGS_EPRSUPPORT_WITHOUT_POWER_BRAKE_GPIO     (0x01)
+#define MPI3_IOUNIT15_FLAGS_EPRSUPPORT_WITH_POWER_BRAKE_GPIO        (0x02)
+#define MPI3_IOUNIT15_NUMPOWERBUDGETDATA_POWER_BUDGETING_DISABLED   (0x00)
 struct mpi3_ioc_page0 {
 	struct mpi3_config_page_header         header;
 	__le32                             reserved08;
@@ -1273,6 +1320,7 @@ struct mpi3_driver_page0 {
 #define MPI3_DRIVER0_BSDOPTS_REGISTRATION_MASK              (0x00000003)
 #define MPI3_DRIVER0_BSDOPTS_REGISTRATION_IOC_AND_DEVS      (0x00000000)
 #define MPI3_DRIVER0_BSDOPTS_REGISTRATION_IOC_ONLY          (0x00000001)
+#define MPI3_DRIVER0_BSDOPTS_REGISTRATION_IOC_AND_INTERNAL_DEVS		(0x00000002)
 struct mpi3_driver_page1 {
 	struct mpi3_config_page_header         header;
 	__le32                             flags;
@@ -1340,7 +1388,7 @@ union mpi3_driver2_trigger_element {
 #define MPI3_DRIVER2_TRIGGER_FLAGS_DIAG_FW_RELEASE                            (0x01)
 struct mpi3_driver_page2 {
 	struct mpi3_config_page_header         header;
-	__le64                             master_trigger;
+	__le64                             global_trigger;
 	__le32                             reserved10[3];
 	u8                                 num_triggers;
 	u8                                 reserved1d[3];
@@ -1348,11 +1396,13 @@ struct mpi3_driver_page2 {
 };
 
 #define MPI3_DRIVER2_PAGEVERSION               (0x00)
-#define MPI3_DRIVER2_MASTERTRIGGER_DIAG_TRACE_RELEASE                       (0x8000000000000000ULL)
-#define MPI3_DRIVER2_MASTERTRIGGER_DIAG_FW_RELEASE                          (0x4000000000000000ULL)
-#define MPI3_DRIVER2_MASTERTRIGGER_SNAPDUMP                                 (0x2000000000000000ULL)
-#define MPI3_DRIVER2_MASTERTRIGGER_DEVICE_REMOVAL_ENABLED                   (0x0000000000000004ULL)
-#define MPI3_DRIVER2_MASTERTRIGGER_TASK_MANAGEMENT_ENABLED                  (0x0000000000000002ULL)
+#define MPI3_DRIVER2_GLOBALTRIGGER_DIAG_TRACE_RELEASE                       (0x8000000000000000ULL)
+#define MPI3_DRIVER2_GLOBALTRIGGER_DIAG_FW_RELEASE                          (0x4000000000000000ULL)
+#define MPI3_DRIVER2_GLOBALTRIGGER_SNAPDUMP_ENABLED			    (0x2000000000000000ULL)
+#define MPI3_DRIVER2_GLOBALTRIGGER_POST_DIAG_TRACE_DISABLED                 (0x1000000000000000ULL)
+#define MPI3_DRIVER2_GLOBALTRIGGER_POST_DIAG_FW_DISABLED                    (0x0800000000000000ULL)
+#define MPI3_DRIVER2_GLOBALTRIGGER_DEVICE_REMOVAL_ENABLED                   (0x0000000000000004ULL)
+#define MPI3_DRIVER2_GLOBALTRIGGER_TASK_MANAGEMENT_ENABLED                  (0x0000000000000002ULL)
 struct mpi3_driver_page10 {
 	struct mpi3_config_page_header         header;
 	__le16                             flags;
@@ -1395,6 +1445,12 @@ union mpi3_security_nonce {
 	u8                                 byte[64];
 };
 
+union mpi3_security_root_digest {
+	__le32                             dword[16];
+	__le16                             word[32];
+	u8                                 byte[64];
+};
+
 union mpi3_security0_cert_chain {
 	__le32                             dword[1024];
 	__le16                             word[2048];
@@ -1467,6 +1523,32 @@ struct mpi3_security_page1 {
 };
 
 #define MPI3_SECURITY1_PAGEVERSION               (0x00)
+#ifndef MPI3_SECURITY2_TRUSTED_ROOT_MAX
+#define MPI3_SECURITY2_TRUSTED_ROOT_MAX      1
+#endif
+struct mpi3_security2_trusted_root {
+	u8                                 level;
+	u8                                 hash_algorithm;
+	__le16                             trusted_root_flags;
+	__le32                             reserved04[3];
+	union mpi3_security_root_digest       root_digest;
+};
+#define MPI3_SECURITY2_TRUSTEDROOT_TRUSTEDROOTFLAGS_HASHALGOSOURCE_MASK            (0x0006)
+#define MPI3_SECURITY2_TRUSTEDROOT_TRUSTEDROOTFLAGS_HASHALGOSOURCE_SHIFT           (1)
+#define MPI3_SECURITY2_TRUSTEDROOT_TRUSTEDROOTFLAGS_HASHALGOSOURCE_HA_FIELD        (0x0000)
+#define MPI3_SECURITY2_TRUSTEDROOT_TRUSTEDROOTFLAGS_HASHALGOSOURCE_AKI             (0x0002)
+#define MPI3_SECURITY2_TRUSTEDROOT_TRUSTEDROOTFLAGS_USERPROVISIONED_YES            (0x0001)
+struct mpi3_security_page2 {
+	struct mpi3_config_page_header         header;
+	__le32                             reserved08[2];
+	union mpi3_security_mac               mac;
+	union mpi3_security_nonce             nonce;
+	__le32                             reserved90[3];
+	u8                                 num_roots;
+	u8                                 reserved9d[3];
+	struct mpi3_security2_trusted_root     trusted_root[MPI3_SECURITY2_TRUSTED_ROOT_MAX];
+};
+#define MPI3_SECURITY2_PAGEVERSION               (0x00)
 struct mpi3_sas_io_unit0_phy_data {
 	u8                 io_unit_port;
 	u8                 port_flags;
@@ -2351,6 +2433,10 @@ struct mpi3_device_page0 {
 #define MPI3_DEVICE0_ASTATUS_NVME_MAX                               (0x5f)
 #define MPI3_DEVICE0_ASTATUS_VD_UNKNOWN                             (0x80)
 #define MPI3_DEVICE0_ASTATUS_VD_MAX                                 (0x8f)
+#define MPI3_DEVICE0_FLAGS_MAX_WRITE_SAME_MASK          (0xe000)
+#define MPI3_DEVICE0_FLAGS_MAX_WRITE_SAME_NO_LIMIT      (0x0000)
+#define MPI3_DEVICE0_FLAGS_MAX_WRITE_SAME_256_LB        (0x2000)
+#define MPI3_DEVICE0_FLAGS_MAX_WRITE_SAME_2048_LB       (0x4000)
 #define MPI3_DEVICE0_FLAGS_CONTROLLER_DEV_HANDLE        (0x0080)
 #define MPI3_DEVICE0_FLAGS_IO_THROTTLING_REQUIRED       (0x0010)
 #define MPI3_DEVICE0_FLAGS_HIDDEN                       (0x0008)
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_image.h b/drivers/scsi/mpi3mr/mpi/mpi30_image.h
index 64c58815988a..47035b811902 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_image.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_image.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- *  Copyright 2018-2022 Broadcom Inc. All rights reserved.
+ *  Copyright 2018-2023 Broadcom Inc. All rights reserved.
  */
 #ifndef MPI30_IMAGE_H
 #define MPI30_IMAGE_H     1
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_init.h b/drivers/scsi/mpi3mr/mpi/mpi30_init.h
index 3c03610ecfa6..af86d12c8e49 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_init.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_init.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- *  Copyright 2016-2022 Broadcom Inc. All rights reserved.
+ *  Copyright 2016-2023 Broadcom Inc. All rights reserved.
  */
 #ifndef MPI30_INIT_H
 #define MPI30_INIT_H     1
@@ -56,6 +56,7 @@ struct mpi3_scsi_io_request {
 #define MPI3_SCSIIO_FLAGS_DMAOPERATION_HOST_PI              (0x00010000)
 #define MPI3_SCSIIO_FLAGS_DIVERT_REASON_MASK                (0x000000f0)
 #define MPI3_SCSIIO_FLAGS_DIVERT_REASON_IO_THROTTLING       (0x00000010)
+#define MPI3_SCSIIO_FLAGS_DIVERT_REASON_WRITE_SAME_TOO_LARGE (0x00000020)
 #define MPI3_SCSIIO_FLAGS_DIVERT_REASON_PROD_SPECIFIC       (0x00000080)
 #define MPI3_SCSIIO_METASGL_INDEX                           (3)
 struct mpi3_scsi_io_reply {
@@ -114,4 +115,24 @@ struct mpi3_scsi_io_reply {
 #define MPI3_SCSI_RSP_ARI0_MASK                 (0xff000000)
 #define MPI3_SCSI_RSP_ARI0_SHIFT                (24)
 #define MPI3_SCSI_TASKTAG_UNKNOWN               (0xffff)
+#define MPI3_SCSITASKMGMT_MSGFLAGS_DO_NOT_SEND_TASK_IU      (0x08)
+#define MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK               (0x01)
+#define MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK_SET           (0x02)
+#define MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET             (0x03)
+#define MPI3_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET       (0x05)
+#define MPI3_SCSITASKMGMT_TASKTYPE_CLEAR_TASK_SET           (0x06)
+#define MPI3_SCSITASKMGMT_TASKTYPE_QUERY_TASK               (0x07)
+#define MPI3_SCSITASKMGMT_TASKTYPE_CLEAR_ACA                (0x08)
+#define MPI3_SCSITASKMGMT_TASKTYPE_QUERY_TASK_SET           (0x09)
+#define MPI3_SCSITASKMGMT_TASKTYPE_QUERY_ASYNC_EVENT        (0x0a)
+#define MPI3_SCSITASKMGMT_TASKTYPE_I_T_NEXUS_RESET          (0x0b)
+#define MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE                (0x00)
+#define MPI3_SCSITASKMGMT_RSPCODE_INVALID_FRAME              (0x02)
+#define MPI3_SCSITASKMGMT_RSPCODE_TM_FUNCTION_NOT_SUPPORTED  (0x04)
+#define MPI3_SCSITASKMGMT_RSPCODE_TM_FAILED                  (0x05)
+#define MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED               (0x08)
+#define MPI3_SCSITASKMGMT_RSPCODE_TM_INVALID_LUN             (0x09)
+#define MPI3_SCSITASKMGMT_RSPCODE_TM_OVERLAPPED_TAG          (0x0a)
+#define MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC           (0x80)
+#define MPI3_SCSITASKMGMT_RSPCODE_TM_NVME_DENIED             (0x81)
 #endif
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h b/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h
index 1c6c6730df5c..f5e9c2309ce6 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- *  Copyright 2016-2022 Broadcom Inc. All rights reserved.
+ *  Copyright 2016-2023 Broadcom Inc. All rights reserved.
  */
 #ifndef MPI30_IOC_H
 #define MPI30_IOC_H     1
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_pci.h b/drivers/scsi/mpi3mr/mpi/mpi30_pci.h
index b7a5df01120d..7c15e5851ce4 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_pci.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_pci.h
@@ -1,11 +1,13 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- *  Copyright 2016-2022 Broadcom Inc. All rights reserved.
+ *  Copyright 2016-2023 Broadcom Inc. All rights reserved.
  *
  */
 #ifndef MPI30_PCI_H
 #define MPI30_PCI_H     1
-
+#ifndef MPI3_NVME_ENCAP_CMD_MAX
+#define MPI3_NVME_ENCAP_CMD_MAX               (1)
+#endif
 #define MPI3_NVME_FLAGS_FORCE_ADMIN_ERR_REPLY_MASK      (0x0002)
 #define MPI3_NVME_FLAGS_FORCE_ADMIN_ERR_REPLY_FAIL_ONLY (0x0000)
 #define MPI3_NVME_FLAGS_FORCE_ADMIN_ERR_REPLY_ALL       (0x0002)
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_sas.h b/drivers/scsi/mpi3mr/mpi/mpi30_sas.h
index e587f77ccd68..4a93c67d335f 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_sas.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_sas.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- *  Copyright 2016-2022 Broadcom Inc. All rights reserved.
+ *  Copyright 2016-2023 Broadcom Inc. All rights reserved.
  */
 #ifndef MPI30_SAS_H
 #define MPI30_SAS_H     1
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_transport.h b/drivers/scsi/mpi3mr/mpi/mpi30_transport.h
index 9b76b9632751..441cfc2c7f09 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_transport.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_transport.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- *  Copyright 2016-2022 Broadcom Inc. All rights reserved.
+ *  Copyright 2016-2023 Broadcom Inc. All rights reserved.
  */
 #ifndef MPI30_TRANSPORT_H
 #define MPI30_TRANSPORT_H     1
@@ -18,7 +18,7 @@ union mpi3_version_union {
 
 #define MPI3_VERSION_MAJOR                                              (3)
 #define MPI3_VERSION_MINOR                                              (0)
-#define MPI3_VERSION_UNIT                                               (26)
+#define MPI3_VERSION_UNIT                                               (27)
 #define MPI3_VERSION_DEV                                                (0)
 #define MPI3_DEVHANDLE_INVALID                                          (0xffff)
 struct mpi3_sysif_oper_queue_indexes {
diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h
index 364fb1b5e45a..dfe6b87fe288 100644
--- a/drivers/scsi/mpi3mr/mpi3mr.h
+++ b/drivers/scsi/mpi3mr/mpi3mr.h
@@ -2,7 +2,7 @@
 /*
  * Driver for Broadcom MPI3 Storage Controllers
  *
- * Copyright (C) 2017-2022 Broadcom Inc.
+ * Copyright (C) 2017-2023 Broadcom Inc.
  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
  *
  */
@@ -55,8 +55,8 @@ extern struct list_head mrioc_list;
 extern int prot_mask;
 extern atomic64_t event_counter;
 
-#define MPI3MR_DRIVER_VERSION	"8.2.0.3.0"
-#define MPI3MR_DRIVER_RELDATE	"08-September-2022"
+#define MPI3MR_DRIVER_VERSION	"8.4.1.0.0"
+#define MPI3MR_DRIVER_RELDATE	"16-March-2023"
 
 #define MPI3MR_DRIVER_NAME	"mpi3mr"
 #define MPI3MR_DRIVER_LICENSE	"GPL"
@@ -126,6 +126,7 @@ extern atomic64_t event_counter;
 #define	MPI3MR_RAID_ERRREC_RESET_TIMEOUT	180
 #define MPI3MR_PREPARE_FOR_RESET_TIMEOUT	180
 #define MPI3MR_RESET_ACK_TIMEOUT		30
+#define MPI3MR_MUR_TIMEOUT			120
 
 #define MPI3MR_WATCHDOG_INTERVAL		1000 /* in milli seconds */
 
@@ -652,7 +653,11 @@ union _form_spec_inf {
 	struct tgt_dev_vd vd_inf;
 };
 
-
+enum mpi3mr_dev_state {
+	MPI3MR_DEV_CREATED = 1,
+	MPI3MR_DEV_REMOVE_HS_STARTED = 2,
+	MPI3MR_DEV_DELETED = 3,
+};
 
 /**
  * struct mpi3mr_tgt_dev - target device data structure
@@ -676,6 +681,7 @@ union _form_spec_inf {
  * @enclosure_logical_id: Enclosure logical identifier
  * @dev_spec: Device type specific information
  * @ref_count: Reference count
+ * @state: device state
  */
 struct mpi3mr_tgt_dev {
 	struct list_head list;
@@ -697,6 +703,7 @@ struct mpi3mr_tgt_dev {
 	u64 enclosure_logical_id;
 	union _form_spec_inf dev_spec;
 	struct kref ref_count;
+	enum mpi3mr_dev_state state;
 };
 
 /**
diff --git a/drivers/scsi/mpi3mr/mpi3mr_app.c b/drivers/scsi/mpi3mr/mpi3mr_app.c
index d10c6afb7f9c..08645a99ad6b 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_app.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_app.c
@@ -2,7 +2,7 @@
 /*
  * Driver for Broadcom MPI3 Storage Controllers
  *
- * Copyright (C) 2017-2022 Broadcom Inc.
+ * Copyright (C) 2017-2023 Broadcom Inc.
  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
  *
  */
@@ -922,6 +922,7 @@ err_out:
 /**
  * mpi3mr_bsg_process_mpt_cmds - MPI Pass through BSG handler
  * @job: BSG job reference
+ * @reply_payload_rcv_len: length of payload recvd
  *
  * This function is the top level handler for MPI Pass through
  * command, this does basic validation of the input data buffers,
@@ -1471,6 +1472,7 @@ static int mpi3mr_bsg_request(struct bsg_job *job)
 
 /**
  * mpi3mr_bsg_exit - de-registration from bsg layer
+ * @mrioc: Adapter instance reference
  *
  * This will be called during driver unload and all
  * bsg resources allocated during load will be freed.
@@ -1505,6 +1507,7 @@ static void mpi3mr_bsg_node_release(struct device *dev)
 
 /**
  * mpi3mr_bsg_init -  registration with bsg layer
+ * @mrioc: Adapter instance reference
  *
  * This will be called during driver load and it will
  * register driver with bsg layer
diff --git a/drivers/scsi/mpi3mr/mpi3mr_debug.h b/drivers/scsi/mpi3mr/mpi3mr_debug.h
index ee6edd8322e6..e94f7520d153 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_debug.h
+++ b/drivers/scsi/mpi3mr/mpi3mr_debug.h
@@ -2,7 +2,7 @@
 /*
  * Driver for Broadcom MPI3 Storage Controllers
  *
- * Copyright (C) 2017-2022 Broadcom Inc.
+ * Copyright (C) 2017-2023 Broadcom Inc.
  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
  *
  */
diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c
index d109a4ceb72b..075fa67e95ee 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_fw.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c
@@ -2,7 +2,7 @@
 /*
  * Driver for Broadcom MPI3 Storage Controllers
  *
- * Copyright (C) 2017-2022 Broadcom Inc.
+ * Copyright (C) 2017-2023 Broadcom Inc.
  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
  *
  */
@@ -1098,7 +1098,7 @@ static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
 	ioc_config &= ~MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
 
-	timeout = MPI3MR_RESET_ACK_TIMEOUT * 10;
+	timeout = MPI3MR_MUR_TIMEOUT * 10;
 	do {
 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 		if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)) {
@@ -2620,14 +2620,12 @@ static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
 	mrioc->num_admin_req = mrioc->admin_req_q_sz /
 	    MPI3MR_ADMIN_REQ_FRAME_SZ;
 	mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
-	mrioc->admin_req_base = NULL;
 
 	mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
 	mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
 	    MPI3MR_ADMIN_REPLY_FRAME_SZ;
 	mrioc->admin_reply_ci = 0;
 	mrioc->admin_reply_ephase = 1;
-	mrioc->admin_reply_base = NULL;
 	atomic_set(&mrioc->admin_reply_q_in_use, 0);
 
 	if (!mrioc->admin_req_base) {
diff --git a/drivers/scsi/mpi3mr/mpi3mr_os.c b/drivers/scsi/mpi3mr/mpi3mr_os.c
index 6d55698ea4d1..d627355303d7 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_os.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_os.c
@@ -2,7 +2,7 @@
 /*
  * Driver for Broadcom MPI3 Storage Controllers
  *
- * Copyright (C) 2017-2022 Broadcom Inc.
+ * Copyright (C) 2017-2023 Broadcom Inc.
  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
  *
  */
@@ -652,6 +652,7 @@ static void mpi3mr_tgtdev_add_to_list(struct mpi3mr_ioc *mrioc,
 	mpi3mr_tgtdev_get(tgtdev);
 	INIT_LIST_HEAD(&tgtdev->list);
 	list_add_tail(&tgtdev->list, &mrioc->tgtdev_list);
+	tgtdev->state = MPI3MR_DEV_CREATED;
 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
 }
 
@@ -659,20 +660,25 @@ static void mpi3mr_tgtdev_add_to_list(struct mpi3mr_ioc *mrioc,
  * mpi3mr_tgtdev_del_from_list -Delete tgtdevice from the list
  * @mrioc: Adapter instance reference
  * @tgtdev: Target device
+ * @must_delete: Must delete the target device from the list irrespective
+ * of the device state.
  *
  * Remove the target device from the target device list
  *
  * Return: Nothing.
  */
 static void mpi3mr_tgtdev_del_from_list(struct mpi3mr_ioc *mrioc,
-	struct mpi3mr_tgt_dev *tgtdev)
+	struct mpi3mr_tgt_dev *tgtdev, bool must_delete)
 {
 	unsigned long flags;
 
 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
-	if (!list_empty(&tgtdev->list)) {
-		list_del_init(&tgtdev->list);
-		mpi3mr_tgtdev_put(tgtdev);
+	if ((tgtdev->state == MPI3MR_DEV_REMOVE_HS_STARTED) || (must_delete == true)) {
+		if (!list_empty(&tgtdev->list)) {
+			list_del_init(&tgtdev->list);
+			tgtdev->state = MPI3MR_DEV_DELETED;
+			mpi3mr_tgtdev_put(tgtdev);
+		}
 	}
 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
 }
@@ -1036,7 +1042,7 @@ void mpi3mr_rfresh_tgtdevs(struct mpi3mr_ioc *mrioc)
 			    tgtdev->perst_id);
 			if (tgtdev->host_exposed)
 				mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
-			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev);
+			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev, true);
 			mpi3mr_tgtdev_put(tgtdev);
 		}
 	}
@@ -1281,12 +1287,12 @@ static void mpi3mr_devstatuschg_evt_bh(struct mpi3mr_ioc *mrioc,
 		if (!tgtdev->host_exposed)
 			mpi3mr_report_tgtdev_to_host(mrioc, tgtdev->perst_id);
 	}
-	if (tgtdev->starget && tgtdev->starget->hostdata) {
-		if (delete)
-			mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
-	}
+
+	if (delete)
+		mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
+
 	if (cleanup) {
-		mpi3mr_tgtdev_del_from_list(mrioc, tgtdev);
+		mpi3mr_tgtdev_del_from_list(mrioc, tgtdev, false);
 		mpi3mr_tgtdev_put(tgtdev);
 	}
 
@@ -1604,7 +1610,7 @@ static void mpi3mr_sastopochg_evt_bh(struct mpi3mr_ioc *mrioc,
 		case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING:
 			if (tgtdev->host_exposed)
 				mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
-			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev);
+			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev, false);
 			mpi3mr_tgtdev_put(tgtdev);
 			break;
 		case MPI3_EVENT_SAS_TOPO_PHY_RC_RESPONDING:
@@ -1762,7 +1768,7 @@ static void mpi3mr_pcietopochg_evt_bh(struct mpi3mr_ioc *mrioc,
 		case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
 			if (tgtdev->host_exposed)
 				mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
-			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev);
+			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev, false);
 			mpi3mr_tgtdev_put(tgtdev);
 			break;
 		default:
@@ -2016,12 +2022,18 @@ static int mpi3mr_create_tgtdev(struct mpi3mr_ioc *mrioc,
 	int retval = 0;
 	struct mpi3mr_tgt_dev *tgtdev = NULL;
 	u16 perst_id = 0;
+	unsigned long flags;
 
 	perst_id = le16_to_cpu(dev_pg0->persistent_id);
 	if (perst_id == MPI3_DEVICE0_PERSISTENTID_INVALID)
 		return retval;
 
-	tgtdev = mpi3mr_get_tgtdev_by_perst_id(mrioc, perst_id);
+	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
+	tgtdev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, perst_id);
+	if (tgtdev)
+		tgtdev->state = MPI3MR_DEV_CREATED;
+	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
+
 	if (tgtdev) {
 		mpi3mr_update_tgtdev(mrioc, tgtdev, dev_pg0, true);
 		mpi3mr_tgtdev_put(tgtdev);
@@ -2219,6 +2231,14 @@ static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_ioc *mrioc, u16 handle,
 	u8 retrycount = 5;
 	struct mpi3mr_drv_cmd *drv_cmd = cmdparam;
 	struct delayed_dev_rmhs_node *delayed_dev_rmhs = NULL;
+	struct mpi3mr_tgt_dev *tgtdev = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
+	tgtdev = __mpi3mr_get_tgtdev_by_handle(mrioc, handle);
+	if (tgtdev && (iou_rc == MPI3_CTRL_OP_REMOVE_DEVICE))
+		tgtdev->state = MPI3MR_DEV_REMOVE_HS_STARTED;
+	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
 
 	if (drv_cmd)
 		goto issue_cmd;
@@ -3311,19 +3331,19 @@ static int mpi3mr_get_chain_idx(struct mpi3mr_ioc *mrioc)
 {
 	u8 retry_count = 5;
 	int cmd_idx = -1;
+	unsigned long flags;
 
+	spin_lock_irqsave(&mrioc->chain_buf_lock, flags);
 	do {
-		spin_lock(&mrioc->chain_buf_lock);
 		cmd_idx = find_first_zero_bit(mrioc->chain_bitmap,
 		    mrioc->chain_buf_count);
 		if (cmd_idx < mrioc->chain_buf_count) {
 			set_bit(cmd_idx, mrioc->chain_bitmap);
-			spin_unlock(&mrioc->chain_buf_lock);
 			break;
 		}
-		spin_unlock(&mrioc->chain_buf_lock);
 		cmd_idx = -1;
 	} while (retry_count--);
+	spin_unlock_irqrestore(&mrioc->chain_buf_lock, flags);
 	return cmd_idx;
 }
 
@@ -3996,10 +4016,14 @@ static int mpi3mr_eh_target_reset(struct scsi_cmnd *scmd)
 	stgt_priv_data = sdev_priv_data->tgt_priv_data;
 	dev_handle = stgt_priv_data->dev_handle;
 	if (stgt_priv_data->dev_removed) {
+		struct scmd_priv *cmd_priv = scsi_cmd_priv(scmd);
 		sdev_printk(KERN_INFO, scmd->device,
 		    "%s:target(handle = 0x%04x) is removed, target reset is not issued\n",
 		    mrioc->name, dev_handle);
-		retval = FAILED;
+		if (!cmd_priv->in_lld_scope || cmd_priv->host_tag == MPI3MR_HOSTTAG_INVALID)
+			retval = SUCCESS;
+		else
+			retval = FAILED;
 		goto out;
 	}
 	sdev_printk(KERN_INFO, scmd->device,
@@ -4064,10 +4088,14 @@ static int mpi3mr_eh_dev_reset(struct scsi_cmnd *scmd)
 	stgt_priv_data = sdev_priv_data->tgt_priv_data;
 	dev_handle = stgt_priv_data->dev_handle;
 	if (stgt_priv_data->dev_removed) {
+		struct scmd_priv *cmd_priv = scsi_cmd_priv(scmd);
 		sdev_printk(KERN_INFO, scmd->device,
 		    "%s: device(handle = 0x%04x) is removed, device(LUN) reset is not issued\n",
 		    mrioc->name, dev_handle);
-		retval = FAILED;
+		if (!cmd_priv->in_lld_scope || cmd_priv->host_tag == MPI3MR_HOSTTAG_INVALID)
+			retval = SUCCESS;
+		else
+			retval = FAILED;
 		goto out;
 	}
 	sdev_printk(KERN_INFO, scmd->device,
@@ -4625,13 +4653,24 @@ static int mpi3mr_qcmd(struct Scsi_Host *shost,
 		goto out;
 	}
 
+	stgt_priv_data = sdev_priv_data->tgt_priv_data;
+	dev_handle = stgt_priv_data->dev_handle;
+
+	/* Avoid error handling escalation when device is removed or blocked */
+
+	if (scmd->device->host->shost_state == SHOST_RECOVERY &&
+		scmd->cmnd[0] == TEST_UNIT_READY &&
+		(stgt_priv_data->dev_removed || (dev_handle == MPI3MR_INVALID_DEV_HANDLE))) {
+		scsi_build_sense(scmd, 0, UNIT_ATTENTION, 0x29, 0x07);
+		scsi_done(scmd);
+		goto out;
+	}
+
 	if (mrioc->reset_in_progress) {
 		retval = SCSI_MLQUEUE_HOST_BUSY;
 		goto out;
 	}
 
-	stgt_priv_data = sdev_priv_data->tgt_priv_data;
-
 	if (atomic_read(&stgt_priv_data->block_io)) {
 		if (mrioc->stop_drv_processing) {
 			scmd->result = DID_NO_CONNECT << 16;
@@ -4642,7 +4681,6 @@ static int mpi3mr_qcmd(struct Scsi_Host *shost,
 		goto out;
 	}
 
-	dev_handle = stgt_priv_data->dev_handle;
 	if (dev_handle == MPI3MR_INVALID_DEV_HANDLE) {
 		scmd->result = DID_NO_CONNECT << 16;
 		scsi_done(scmd);
@@ -4758,7 +4796,7 @@ out:
 	return retval;
 }
 
-static struct scsi_host_template mpi3mr_driver_template = {
+static const struct scsi_host_template mpi3mr_driver_template = {
 	.module				= THIS_MODULE,
 	.name				= "MPI3 Storage Controller",
 	.proc_name			= MPI3MR_DRIVER_NAME,
@@ -5111,7 +5149,7 @@ static void mpi3mr_remove(struct pci_dev *pdev)
 	list_for_each_entry_safe(tgtdev, tgtdev_next, &mrioc->tgtdev_list,
 	    list) {
 		mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
-		mpi3mr_tgtdev_del_from_list(mrioc, tgtdev);
+		mpi3mr_tgtdev_del_from_list(mrioc, tgtdev, true);
 		mpi3mr_tgtdev_put(tgtdev);
 	}
 	mpi3mr_stop_watchdog(mrioc);
diff --git a/drivers/scsi/mpi3mr/mpi3mr_transport.c b/drivers/scsi/mpi3mr/mpi3mr_transport.c
index 5748bd9369ff..4d84d5bd173f 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_transport.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_transport.c
@@ -2,7 +2,7 @@
 /*
  * Driver for Broadcom MPI3 Storage Controllers
  *
- * Copyright (C) 2017-2022 Broadcom Inc.
+ * Copyright (C) 2017-2023 Broadcom Inc.
  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
  *
  */
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index 14ae0a9c5d3d..53f5492579cb 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -60,7 +60,6 @@
 #include <linux/ktime.h>
 #include <linux/kthread.h>
 #include <asm/page.h>        /* To get host page size per arch */
-#include <linux/aer.h>
 
 
 #include "mpt3sas_base.h"
@@ -3535,7 +3534,6 @@ mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)
 
 	if (pci_is_enabled(pdev)) {
 		pci_release_selected_regions(ioc->pdev, ioc->bars);
-		pci_disable_pcie_error_reporting(pdev);
 		pci_disable_device(pdev);
 	}
 }
@@ -3615,9 +3613,6 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
 		goto out_fail;
 	}
 
-/* AER (Advanced Error Reporting) hooks */
-	pci_enable_pcie_error_reporting(pdev);
-
 	pci_set_master(pdev);
 
 
@@ -4761,21 +4756,15 @@ _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc)
 	int i = 0;
 	char desc[17] = {0};
 	u32 iounit_pg1_flags;
-	u32 bios_version;
 
-	bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion);
 	strncpy(desc, ioc->manu_pg0.ChipName, 16);
-	ioc_info(ioc, "%s: FWVersion(%02d.%02d.%02d.%02d), ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02d.%02d)\n",
+	ioc_info(ioc, "%s: FWVersion(%02d.%02d.%02d.%02d), ChipRevision(0x%02x)\n",
 		 desc,
 		 (ioc->facts.FWVersion.Word & 0xFF000000) >> 24,
 		 (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16,
 		 (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8,
 		 ioc->facts.FWVersion.Word & 0x000000FF,
-		 ioc->pdev->revision,
-		 (bios_version & 0xFF000000) >> 24,
-		 (bios_version & 0x00FF0000) >> 16,
-		 (bios_version & 0x0000FF00) >> 8,
-		 bios_version & 0x000000FF);
+		 ioc->pdev->revision);
 
 	_base_display_OEMs_branding(ioc);
 
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index 8e24ebcebfe5..c3c1f466fe01 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -52,7 +52,6 @@
 #include <linux/delay.h>
 #include <linux/pci.h>
 #include <linux/interrupt.h>
-#include <linux/aer.h>
 #include <linux/raid_class.h>
 #include <linux/blk-mq-pci.h>
 #include <asm/unaligned.h>
@@ -11926,7 +11925,7 @@ static void scsih_map_queues(struct Scsi_Host *shost)
 }
 
 /* shost template for SAS 2.0 HBA devices */
-static struct scsi_host_template mpt2sas_driver_template = {
+static const struct scsi_host_template mpt2sas_driver_template = {
 	.module				= THIS_MODULE,
 	.name				= "Fusion MPT SAS Host",
 	.proc_name			= MPT2SAS_DRIVER_NAME,
@@ -11964,7 +11963,7 @@ static struct raid_function_template mpt2sas_raid_functions = {
 };
 
 /* shost template for SAS 3.0 HBA devices */
-static struct scsi_host_template mpt3sas_driver_template = {
+static const struct scsi_host_template mpt3sas_driver_template = {
 	.module				= THIS_MODULE,
 	.name				= "Fusion MPT SAS Host",
 	.proc_name			= MPT3SAS_DRIVER_NAME,
@@ -12930,10 +12929,10 @@ _mpt3sas_exit(void)
 	pr_info("mpt3sas version %s unloading\n",
 				MPT3SAS_DRIVER_VERSION);
 
-	mpt3sas_ctl_exit(hbas_to_enumerate);
-
 	pci_unregister_driver(&mpt3sas_driver);
 
+	mpt3sas_ctl_exit(hbas_to_enumerate);
+
 	scsih_exit();
 }
 
diff --git a/drivers/scsi/mvme147.c b/drivers/scsi/mvme147.c
index 472fa043094f..98b99c0f5bc7 100644
--- a/drivers/scsi/mvme147.c
+++ b/drivers/scsi/mvme147.c
@@ -69,7 +69,7 @@ static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
 	m147_pcc->dma_cntrl = 0;
 }
 
-static struct scsi_host_template mvme147_host_template = {
+static const struct scsi_host_template mvme147_host_template = {
 	.module			= THIS_MODULE,
 	.proc_name		= "MVME147",
 	.name			= "MVME147 built-in SCSI",
diff --git a/drivers/scsi/mvsas/mv_init.c b/drivers/scsi/mvsas/mv_init.c
index cfe84473a515..49e2a5e7ce54 100644
--- a/drivers/scsi/mvsas/mv_init.c
+++ b/drivers/scsi/mvsas/mv_init.c
@@ -29,7 +29,7 @@ static const struct attribute_group *mvst_host_groups[];
 
 #define SOC_SAS_NUM 2
 
-static struct scsi_host_template mvs_sht = {
+static const struct scsi_host_template mvs_sht = {
 	.module			= THIS_MODULE,
 	.name			= DRV_NAME,
 	.queuecommand		= sas_queuecommand,
diff --git a/drivers/scsi/mvumi.c b/drivers/scsi/mvumi.c
index 60c65586f30e..73aa7059b556 100644
--- a/drivers/scsi/mvumi.c
+++ b/drivers/scsi/mvumi.c
@@ -2168,7 +2168,7 @@ mvumi_bios_param(struct scsi_device *sdev, struct block_device *bdev,
 	return 0;
 }
 
-static struct scsi_host_template mvumi_template = {
+static const struct scsi_host_template mvumi_template = {
 
 	.module = THIS_MODULE,
 	.name = "Marvell Storage Controller",
diff --git a/drivers/scsi/myrb.c b/drivers/scsi/myrb.c
index e885c1dbf61f..ca2e932dd9b7 100644
--- a/drivers/scsi/myrb.c
+++ b/drivers/scsi/myrb.c
@@ -2203,7 +2203,7 @@ static struct attribute *myrb_shost_attrs[] = {
 
 ATTRIBUTE_GROUPS(myrb_shost);
 
-static struct scsi_host_template myrb_template = {
+static const struct scsi_host_template myrb_template = {
 	.module			= THIS_MODULE,
 	.name			= "DAC960",
 	.proc_name		= "myrb",
diff --git a/drivers/scsi/myrs.c b/drivers/scsi/myrs.c
index 7eb8c39da366..a1eec65a9713 100644
--- a/drivers/scsi/myrs.c
+++ b/drivers/scsi/myrs.c
@@ -1915,7 +1915,7 @@ static void myrs_slave_destroy(struct scsi_device *sdev)
 	kfree(sdev->hostdata);
 }
 
-static struct scsi_host_template myrs_template = {
+static const struct scsi_host_template myrs_template = {
 	.module			= THIS_MODULE,
 	.name			= "DAC960",
 	.proc_name		= "myrs",
diff --git a/drivers/scsi/nsp32.c b/drivers/scsi/nsp32.c
index 75bb0028ed74..b7987019686e 100644
--- a/drivers/scsi/nsp32.c
+++ b/drivers/scsi/nsp32.c
@@ -259,7 +259,7 @@ static void nsp32_dmessage(const char *, int, int,    char *, ...);
 /*
  * max_sectors is currently limited up to 128.
  */
-static struct scsi_host_template nsp32_template = {
+static const struct scsi_host_template nsp32_template = {
 	.proc_name			= "nsp32",
 	.name				= "Workbit NinjaSCSI-32Bi/UDE",
 	.show_info			= nsp32_show_info,
diff --git a/drivers/scsi/pcmcia/sym53c500_cs.c b/drivers/scsi/pcmcia/sym53c500_cs.c
index 5d7dfefd6f6c..278c78d066c4 100644
--- a/drivers/scsi/pcmcia/sym53c500_cs.c
+++ b/drivers/scsi/pcmcia/sym53c500_cs.c
@@ -668,7 +668,7 @@ ATTRIBUTE_GROUPS(SYM53C500_shost);
 /*
 *  scsi_host_template initializer
 */
-static struct scsi_host_template sym53c500_driver_template = {
+static const struct scsi_host_template sym53c500_driver_template = {
      .module			= THIS_MODULE,
      .name			= "SYM53C500",
      .info			= SYM53C500_info,
@@ -702,7 +702,7 @@ SYM53C500_config(struct pcmcia_device *link)
 	int ret;
 	int irq_level, port_base;
 	struct Scsi_Host *host;
-	struct scsi_host_template *tpnt = &sym53c500_driver_template;
+	const struct scsi_host_template *tpnt = &sym53c500_driver_template;
 	struct sym53c500_data *data;
 
 	dev_dbg(&link->dev, "SYM53C500_config\n");
diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c
index 7e589fe3e010..8b9490011e36 100644
--- a/drivers/scsi/pm8001/pm8001_init.c
+++ b/drivers/scsi/pm8001/pm8001_init.c
@@ -96,7 +96,7 @@ static void pm8001_map_queues(struct Scsi_Host *shost)
 /*
  * The main structure which LLDD must register for scsi core.
  */
-static struct scsi_host_template pm8001_sht = {
+static const struct scsi_host_template pm8001_sht = {
 	.module			= THIS_MODULE,
 	.name			= DRV_NAME,
 	.proc_name		= DRV_NAME,
diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c
index 836ddc476764..9415a4819470 100644
--- a/drivers/scsi/pmcraid.c
+++ b/drivers/scsi/pmcraid.c
@@ -3611,7 +3611,7 @@ static struct attribute *pmcraid_host_attrs[] = {
 ATTRIBUTE_GROUPS(pmcraid_host);
 
 /* host template structure for pmcraid driver */
-static struct scsi_host_template pmcraid_host_template = {
+static const struct scsi_host_template pmcraid_host_template = {
 	.module = THIS_MODULE,
 	.name = PMCRAID_DRIVER_NAME,
 	.queuecommand = pmcraid_queuecommand,
@@ -5346,7 +5346,7 @@ static int __init pmcraid_init(void)
 	}
 
 	pmcraid_major = MAJOR(dev);
-	pmcraid_class = class_create(THIS_MODULE, PMCRAID_DEVFILE);
+	pmcraid_class = class_create(PMCRAID_DEVFILE);
 
 	if (IS_ERR(pmcraid_class)) {
 		error = PTR_ERR(pmcraid_class);
diff --git a/drivers/scsi/ppa.c b/drivers/scsi/ppa.c
index c6c1bc608224..909c49541984 100644
--- a/drivers/scsi/ppa.c
+++ b/drivers/scsi/ppa.c
@@ -972,7 +972,7 @@ static int ppa_adjust_queue(struct scsi_device *device)
 	return 0;
 }
 
-static struct scsi_host_template ppa_template = {
+static const struct scsi_host_template ppa_template = {
 	.module			= THIS_MODULE,
 	.proc_name		= "ppa",
 	.show_info		= ppa_show_info,
diff --git a/drivers/scsi/ps3rom.c b/drivers/scsi/ps3rom.c
index 2b80cab70333..90495a832f34 100644
--- a/drivers/scsi/ps3rom.c
+++ b/drivers/scsi/ps3rom.c
@@ -323,7 +323,7 @@ done:
 	return IRQ_HANDLED;
 }
 
-static struct scsi_host_template ps3rom_host_template = {
+static const struct scsi_host_template ps3rom_host_template = {
 	.name =			DEVICE_NAME,
 	.slave_configure =	ps3rom_slave_configure,
 	.queuecommand =		ps3rom_queuecommand,
diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c
index 35e16600fc63..3b64de81ea0d 100644
--- a/drivers/scsi/qedf/qedf_main.c
+++ b/drivers/scsi/qedf/qedf_main.c
@@ -979,7 +979,7 @@ static int qedf_slave_configure(struct scsi_device *sdev)
 	return 0;
 }
 
-static struct scsi_host_template qedf_host_template = {
+static const struct scsi_host_template qedf_host_template = {
 	.module 	= THIS_MODULE,
 	.name 		= QEDF_MODULE_NAME,
 	.this_id 	= -1,
@@ -2224,7 +2224,6 @@ static bool qedf_process_completions(struct qedf_fastpath *fp)
 	u16 prod_idx;
 	struct fcoe_cqe *cqe;
 	struct qedf_io_work *io_work;
-	int num_handled = 0;
 	unsigned int cpu;
 	struct qedf_ioreq *io_req = NULL;
 	u16 xid;
@@ -2247,7 +2246,6 @@ static bool qedf_process_completions(struct qedf_fastpath *fp)
 
 	while (new_cqes) {
 		fp->completions++;
-		num_handled++;
 		cqe = &que->cq[que->cq_cons_idx];
 
 		comp_type = (cqe->cqe_data >> FCOE_CQE_CQE_TYPE_SHIFT) &
diff --git a/drivers/scsi/qedi/qedi_gbl.h b/drivers/scsi/qedi/qedi_gbl.h
index 72942772b198..0e316cc24b19 100644
--- a/drivers/scsi/qedi/qedi_gbl.h
+++ b/drivers/scsi/qedi/qedi_gbl.h
@@ -17,7 +17,7 @@ extern int qedi_do_not_recover;
 
 extern uint qedi_io_tracing;
 
-extern struct scsi_host_template qedi_host_template;
+extern const struct scsi_host_template qedi_host_template;
 extern struct iscsi_transport qedi_iscsi_transport;
 extern const struct qed_iscsi_ops *qedi_ops;
 extern const struct qedi_debugfs_ops qedi_debugfs_ops[];
diff --git a/drivers/scsi/qedi/qedi_iscsi.c b/drivers/scsi/qedi/qedi_iscsi.c
index 31ec429104e2..6ed8ef97642c 100644
--- a/drivers/scsi/qedi/qedi_iscsi.c
+++ b/drivers/scsi/qedi/qedi_iscsi.c
@@ -40,7 +40,7 @@ static int qedi_eh_host_reset(struct scsi_cmnd *cmd)
 	return qedi_recover_all_conns(qedi);
 }
 
-struct scsi_host_template qedi_host_template = {
+const struct scsi_host_template qedi_host_template = {
 	.module = THIS_MODULE,
 	.name = "QLogic QEDI 25/40/100Gb iSCSI Initiator Driver",
 	.proc_name = QEDI_MODULE_NAME,
diff --git a/drivers/scsi/qla1280.c b/drivers/scsi/qla1280.c
index 1e7f4d138e06..6e5e89aaa283 100644
--- a/drivers/scsi/qla1280.c
+++ b/drivers/scsi/qla1280.c
@@ -4115,7 +4115,7 @@ qla1280_get_token(char *str)
 }
 
 
-static struct scsi_host_template qla1280_driver_template = {
+static const struct scsi_host_template qla1280_driver_template = {
 	.module			= THIS_MODULE,
 	.proc_name		= "qla1280",
 	.name			= "Qlogic ISP 1280/12160",
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index ec0e987b71fa..df5e5b7fdcfe 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -22,7 +22,6 @@
 #include <linux/interrupt.h>
 #include <linux/workqueue.h>
 #include <linux/firmware.h>
-#include <linux/aer.h>
 #include <linux/mutex.h>
 #include <linux/btree.h>
 
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 9aba07c7bde4..391c8b3623a6 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -214,7 +214,7 @@ extern void qla2x00_free_exchoffld_buffer(struct qla_hw_data *);
 
 extern int qla81xx_restart_mpi_firmware(scsi_qla_host_t *);
 
-extern struct scsi_qla_host *qla2x00_create_host(struct scsi_host_template *,
+extern struct scsi_qla_host *qla2x00_create_host(const struct scsi_host_template *,
 	struct qla_hw_data *);
 extern void qla2x00_free_host(struct scsi_qla_host *);
 extern void qla2x00_relogin(struct scsi_qla_host *);
diff --git a/drivers/scsi/qla2xxx/qla_mid.c b/drivers/scsi/qla2xxx/qla_mid.c
index 78661b658dcd..b67416951a5f 100644
--- a/drivers/scsi/qla2xxx/qla_mid.c
+++ b/drivers/scsi/qla2xxx/qla_mid.c
@@ -496,7 +496,7 @@ qla24xx_create_vhost(struct fc_vport *fc_vport)
 	scsi_qla_host_t *base_vha = shost_priv(fc_vport->shost);
 	struct qla_hw_data *ha = base_vha->hw;
 	scsi_qla_host_t *vha;
-	struct scsi_host_template *sht = &qla2xxx_driver_template;
+	const struct scsi_host_template *sht = &qla2xxx_driver_template;
 	struct Scsi_Host *host;
 
 	vha = qla2x00_create_host(sht, ha);
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index d0cdbfe771a9..2fa695bf38b7 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -2958,9 +2958,6 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 		ql2xallocfwdump = 0;
 	}
 
-	/* This may fail but that's ok */
-	pci_enable_pcie_error_reporting(pdev);
-
 	ha = kzalloc(sizeof(struct qla_hw_data), GFP_KERNEL);
 	if (!ha) {
 		ql_log_pci(ql_log_fatal, pdev, 0x0009,
@@ -3968,8 +3965,6 @@ qla2x00_remove_one(struct pci_dev *pdev)
 	pci_release_selected_regions(ha->pdev, ha->bars);
 	kfree(ha);
 
-	pci_disable_pcie_error_reporting(pdev);
-
 	pci_disable_device(pdev);
 }
 
@@ -5024,8 +5019,8 @@ qla2x00_mem_free(struct qla_hw_data *ha)
 	ha->vp_map = NULL;
 }
 
-struct scsi_qla_host *qla2x00_create_host(struct scsi_host_template *sht,
-						struct qla_hw_data *ha)
+struct scsi_qla_host *qla2x00_create_host(const struct scsi_host_template *sht,
+					  struct qla_hw_data *ha)
 {
 	struct Scsi_Host *host;
 	struct scsi_qla_host *vha = NULL;
@@ -6845,7 +6840,6 @@ qla2x00_disable_board_on_pci_error(struct work_struct *work)
 	qla2x00_unmap_iobases(ha);
 
 	pci_release_selected_regions(ha->pdev, ha->bars);
-	pci_disable_pcie_error_reporting(pdev);
 	pci_disable_device(pdev);
 
 	/*
diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
index aa0cf5ca6c1c..5258b07687a9 100644
--- a/drivers/scsi/qla2xxx/qla_target.c
+++ b/drivers/scsi/qla2xxx/qla_target.c
@@ -6395,8 +6395,7 @@ int qlt_add_target(struct qla_hw_data *ha, struct scsi_qla_host *base_vha)
 		return -ENOMEM;
 	}
 
-	if (!(base_vha->host->hostt->supported_mode & MODE_TARGET))
-		base_vha->host->hostt->supported_mode |= MODE_TARGET;
+	qla2xxx_driver_template.supported_mode |= MODE_TARGET;
 
 	rc = btree_init64(&tgt->lun_qpair_map);
 	if (rc) {
diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
index 8024322c9c5a..3b5ba4b47b3b 100644
--- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c
+++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
@@ -377,11 +377,6 @@ static void tcm_qla2xxx_close_session(struct se_session *se_sess)
 	tcm_qla2xxx_put_sess(sess);
 }
 
-static u32 tcm_qla2xxx_sess_get_index(struct se_session *se_sess)
-{
-	return 0;
-}
-
 static int tcm_qla2xxx_write_pending(struct se_cmd *se_cmd)
 {
 	struct qla_tgt_cmd *cmd = container_of(se_cmd,
@@ -421,11 +416,6 @@ static int tcm_qla2xxx_write_pending(struct se_cmd *se_cmd)
 	return qlt_rdy_to_xfer(cmd);
 }
 
-static void tcm_qla2xxx_set_default_node_attrs(struct se_node_acl *nacl)
-{
-	return;
-}
-
 static int tcm_qla2xxx_get_cmd_state(struct se_cmd *se_cmd)
 {
 	if (!(se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
@@ -1811,10 +1801,8 @@ static const struct target_core_fabric_ops tcm_qla2xxx_ops = {
 	.check_stop_free		= tcm_qla2xxx_check_stop_free,
 	.release_cmd			= tcm_qla2xxx_release_cmd,
 	.close_session			= tcm_qla2xxx_close_session,
-	.sess_get_index			= tcm_qla2xxx_sess_get_index,
 	.sess_get_initiator_sid		= NULL,
 	.write_pending			= tcm_qla2xxx_write_pending,
-	.set_default_node_attributes	= tcm_qla2xxx_set_default_node_attrs,
 	.get_cmd_state			= tcm_qla2xxx_get_cmd_state,
 	.queue_data_in			= tcm_qla2xxx_queue_data_in,
 	.queue_status			= tcm_qla2xxx_queue_status,
@@ -1852,10 +1840,8 @@ static const struct target_core_fabric_ops tcm_qla2xxx_npiv_ops = {
 	.check_stop_free                = tcm_qla2xxx_check_stop_free,
 	.release_cmd			= tcm_qla2xxx_release_cmd,
 	.close_session			= tcm_qla2xxx_close_session,
-	.sess_get_index			= tcm_qla2xxx_sess_get_index,
 	.sess_get_initiator_sid		= NULL,
 	.write_pending			= tcm_qla2xxx_write_pending,
-	.set_default_node_attributes	= tcm_qla2xxx_set_default_node_attrs,
 	.get_cmd_state			= tcm_qla2xxx_get_cmd_state,
 	.queue_data_in			= tcm_qla2xxx_queue_data_in,
 	.queue_status			= tcm_qla2xxx_queue_status,
diff --git a/drivers/scsi/qla4xxx/ql4_def.h b/drivers/scsi/qla4xxx/ql4_def.h
index 5f82c8afd5e0..5e683ba49fa5 100644
--- a/drivers/scsi/qla4xxx/ql4_def.h
+++ b/drivers/scsi/qla4xxx/ql4_def.h
@@ -23,7 +23,6 @@
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/mutex.h>
-#include <linux/aer.h>
 #include <linux/bsg-lib.h>
 #include <linux/vmalloc.h>
 
diff --git a/drivers/scsi/qla4xxx/ql4_isr.c b/drivers/scsi/qla4xxx/ql4_isr.c
index 6f0e77dc2a34..cf52258ecdde 100644
--- a/drivers/scsi/qla4xxx/ql4_isr.c
+++ b/drivers/scsi/qla4xxx/ql4_isr.c
@@ -472,14 +472,12 @@ static void qla4xxx_mbox_status_entry(struct scsi_qla_host *ha,
  **/
 void qla4xxx_process_response_queue(struct scsi_qla_host *ha)
 {
-	uint32_t count = 0;
 	struct srb *srb = NULL;
 	struct status_entry *sts_entry;
 
 	/* Process all responses from response queue */
 	while ((ha->response_ptr->signature != RESPONSE_PROCESSED)) {
 		sts_entry = (struct status_entry *) ha->response_ptr;
-		count++;
 
 		/* Advance pointers for next entry */
 		if (ha->response_out == (RESPONSE_QUEUE_DEPTH - 1)) {
diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
index 005502125b27..ee6d784c095c 100644
--- a/drivers/scsi/qla4xxx/ql4_os.c
+++ b/drivers/scsi/qla4xxx/ql4_os.c
@@ -8639,8 +8639,6 @@ static int qla4xxx_probe_adapter(struct pci_dev *pdev,
 	ha->host_no = host->host_no;
 	ha->func_num = PCI_FUNC(ha->pdev->devfn);
 
-	pci_enable_pcie_error_reporting(pdev);
-
 	/* Setup Runtime configurable options */
 	if (is_qla8022(ha)) {
 		ha->isp_ops = &qla4_82xx_isp_ops;
@@ -8867,7 +8865,6 @@ probe_failed:
 	qla4xxx_free_adapter(ha);
 
 probe_failed_ioconfig:
-	pci_disable_pcie_error_reporting(pdev);
 	scsi_host_put(ha->host);
 
 probe_disable_device:
@@ -9022,7 +9019,6 @@ static void qla4xxx_remove_adapter(struct pci_dev *pdev)
 
 	scsi_host_put(ha->host);
 
-	pci_disable_pcie_error_reporting(pdev);
 	pci_disable_device(pdev);
 }
 
diff --git a/drivers/scsi/qlogicpti.c b/drivers/scsi/qlogicpti.c
index 8c961ff03fcd..1e8fbd457248 100644
--- a/drivers/scsi/qlogicpti.c
+++ b/drivers/scsi/qlogicpti.c
@@ -513,7 +513,7 @@ static int qlogicpti_load_firmware(struct qlogicpti *qpti)
 		       qpti->qpti_id);
 		err = 1;
 		goto out;
-	}		
+	}
 	sbus_writew(SBUS_CTRL_RESET, qpti->qregs + SBUS_CTRL);
 	sbus_writew((DMA_CTRL_CCLEAR | DMA_CTRL_CIRQ), qpti->qregs + CMD_DMA_CTRL);
 	sbus_writew((DMA_CTRL_CCLEAR | DMA_CTRL_CIRQ), qpti->qregs + DATA_DMA_CTRL);
@@ -563,7 +563,7 @@ static int qlogicpti_load_firmware(struct qlogicpti *qpti)
 		       qpti->qpti_id);
 		err = 1;
 		goto out;
-	}		
+	}
 
 	/* Load it up.. */
 	for (i = 0; i < risc_code_length; i++) {
@@ -1136,7 +1136,7 @@ static struct scsi_cmnd *qlogicpti_intr_handler(struct qlogicpti *qpti)
 
 	if (!(sbus_readw(qpti->qregs + SBUS_STAT) & SBUS_STAT_RINT))
 		return NULL;
-		
+
 	in_ptr = sbus_readw(qpti->qregs + MBOX5);
 	sbus_writew(HCCTRL_CRIRQ, qpti->qregs + HCCTRL);
 	if (sbus_readw(qpti->qregs + SBUS_SEMAPHORE) & SBUS_SEMAPHORE_LCK) {
@@ -1287,7 +1287,7 @@ static int qlogicpti_reset(struct scsi_cmnd *Cmnd)
 	return return_status;
 }
 
-static struct scsi_host_template qpti_template = {
+static const struct scsi_host_template qpti_template = {
 	.module			= THIS_MODULE,
 	.name			= "qlogicpti",
 	.info			= qlogicpti_info,
@@ -1362,9 +1362,8 @@ static int qpti_sbus_probe(struct platform_device *op)
 	fcode = of_get_property(dp, "isp-fcode", NULL);
 	if (fcode && fcode[0])
 		printk("(FCode %s)", fcode);
-	if (of_find_property(dp, "differential", NULL) != NULL)
-		qpti->differential = 1;
-			
+	qpti->differential = of_property_read_bool(dp, "differential");
+
 	printk("\nqlogicpti%d: [%s Wide, using %s interface]\n",
 		qpti->qpti_id,
 		(qpti->ultra ? "Ultra" : "Fast"),
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 8553277effb3..f4fa1035a191 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -250,6 +250,11 @@ static const char *sdebug_version_date = "20210520";
 
 #define SDEB_XA_NOT_IN_USE XA_MARK_1
 
+static struct kmem_cache *queued_cmd_cache;
+
+#define TO_QUEUED_CMD(scmd)  ((void *)(scmd)->host_scribble)
+#define ASSIGN_QUEUED_CMD(scmnd, qc) { (scmnd)->host_scribble = (void *) qc; }
+
 /* Zone types (zbcr05 table 25) */
 enum sdebug_z_type {
 	ZBC_ZTYPE_CNV	= 0x1,
@@ -288,7 +293,6 @@ struct sdebug_dev_info {
 	uuid_t lu_name;
 	struct sdebug_host_info *sdbg_host;
 	unsigned long uas_bm[1];
-	atomic_t num_in_q;
 	atomic_t stopped;	/* 1: by SSU, 2: device start */
 	bool used;
 
@@ -324,9 +328,12 @@ struct sdeb_store_info {
 	void *map_storep;	/* provisioning map */
 };
 
-#define to_sdebug_host(d)	\
+#define dev_to_sdebug_host(d)	\
 	container_of(d, struct sdebug_host_info, dev)
 
+#define shost_to_sdebug_host(shost)	\
+	dev_to_sdebug_host(shost->dma_dev)
+
 enum sdeb_defer_type {SDEB_DEFER_NONE = 0, SDEB_DEFER_HRT = 1,
 		      SDEB_DEFER_WQ = 2, SDEB_DEFER_POLL = 3};
 
@@ -334,13 +341,7 @@ struct sdebug_defer {
 	struct hrtimer hrt;
 	struct execute_work ew;
 	ktime_t cmpl_ts;/* time since boot to complete this cmd */
-	int sqa_idx;	/* index of sdebug_queue array */
-	int qc_idx;	/* index of sdebug_queued_cmd array within sqa_idx */
-	int hc_idx;	/* hostwide tag index */
 	int issuing_cpu;
-	bool init_hrt;
-	bool init_wq;
-	bool init_poll;
 	bool aborted;	/* true when blk_abort_request() already called */
 	enum sdeb_defer_type defer_t;
 };
@@ -349,15 +350,12 @@ struct sdebug_queued_cmd {
 	/* corresponding bit set in in_use_bm[] in owning struct sdebug_queue
 	 * instance indicates this slot is in use.
 	 */
-	struct sdebug_defer *sd_dp;
-	struct scsi_cmnd *a_cmnd;
+	struct sdebug_defer sd_dp;
+	struct scsi_cmnd *scmd;
 };
 
-struct sdebug_queue {
-	struct sdebug_queued_cmd qc_arr[SDEBUG_CANQUEUE];
-	unsigned long in_use_bm[SDEBUG_CANQUEUE_WORDS];
-	spinlock_t qc_lock;
-	atomic_t blocked;	/* to temporarily stop more being queued */
+struct sdebug_scsi_cmd {
+	spinlock_t   lock;
 };
 
 static atomic_t sdebug_cmnd_count;   /* number of incoming commands */
@@ -507,6 +505,8 @@ static int sdebug_add_store(void);
 static void sdebug_erase_store(int idx, struct sdeb_store_info *sip);
 static void sdebug_erase_all_stores(bool apart_from_first);
 
+static void sdebug_free_queued_cmd(struct sdebug_queued_cmd *sqcp);
+
 /*
  * The following are overflow arrays for cdbs that "hit" the same index in
  * the opcode_info_arr array. The most time sensitive (or commonly used) cdb
@@ -754,7 +754,6 @@ static int sdebug_max_luns = DEF_MAX_LUNS;
 static int sdebug_max_queue = SDEBUG_CANQUEUE;	/* per submit queue */
 static unsigned int sdebug_medium_error_start = OPT_MEDIUM_ERR_ADDR;
 static int sdebug_medium_error_count = OPT_MEDIUM_ERR_NUM;
-static atomic_t retired_max_queue;	/* if > 0 then was prior max_queue */
 static int sdebug_ndelay = DEF_NDELAY;	/* if > 0 then unit is nanoseconds */
 static int sdebug_no_lun_0 = DEF_NO_LUN_0;
 static int sdebug_no_uld;
@@ -814,7 +813,7 @@ static int sdebug_cylinders_per;	/* cylinders per surface */
 static int sdebug_sectors_per;		/* sectors per cylinder */
 
 static LIST_HEAD(sdebug_host_list);
-static DEFINE_SPINLOCK(sdebug_host_list_lock);
+static DEFINE_MUTEX(sdebug_host_list_mutex);
 
 static struct xarray per_store_arr;
 static struct xarray *per_store_ap = &per_store_arr;
@@ -841,7 +840,6 @@ static int sdeb_zbc_nr_conv = DEF_ZBC_NR_CONV_ZONES;
 
 static int submit_queues = DEF_SUBMIT_QUEUES;  /* > 1 for multi-queue (mq) */
 static int poll_queues; /* iouring iopoll interface.*/
-static struct sdebug_queue *sdebug_q_arr;  /* ptr to array of submit queues */
 
 static DEFINE_RWLOCK(atomic_rw);
 static DEFINE_RWLOCK(atomic_rw2);
@@ -906,7 +904,7 @@ static void sdebug_max_tgts_luns(void)
 	struct sdebug_host_info *sdbg_host;
 	struct Scsi_Host *hpnt;
 
-	spin_lock(&sdebug_host_list_lock);
+	mutex_lock(&sdebug_host_list_mutex);
 	list_for_each_entry(sdbg_host, &sdebug_host_list, host_list) {
 		hpnt = sdbg_host->shost;
 		if ((hpnt->this_id >= 0) &&
@@ -917,7 +915,7 @@ static void sdebug_max_tgts_luns(void)
 		/* sdebug_max_luns; */
 		hpnt->max_lun = SCSI_W_LUN_REPORT_LUNS + 1;
 	}
-	spin_unlock(&sdebug_host_list_lock);
+	mutex_unlock(&sdebug_host_list_mutex);
 }
 
 enum sdeb_cmd_data {SDEB_IN_DATA = 0, SDEB_IN_CDB = 1};
@@ -1049,30 +1047,27 @@ static void all_config_cdb_len(void)
 	struct Scsi_Host *shost;
 	struct scsi_device *sdev;
 
-	spin_lock(&sdebug_host_list_lock);
+	mutex_lock(&sdebug_host_list_mutex);
 	list_for_each_entry(sdbg_host, &sdebug_host_list, host_list) {
 		shost = sdbg_host->shost;
 		shost_for_each_device(sdev, shost) {
 			config_cdb_len(sdev);
 		}
 	}
-	spin_unlock(&sdebug_host_list_lock);
+	mutex_unlock(&sdebug_host_list_mutex);
 }
 
 static void clear_luns_changed_on_target(struct sdebug_dev_info *devip)
 {
-	struct sdebug_host_info *sdhp;
+	struct sdebug_host_info *sdhp = devip->sdbg_host;
 	struct sdebug_dev_info *dp;
 
-	spin_lock(&sdebug_host_list_lock);
-	list_for_each_entry(sdhp, &sdebug_host_list, host_list) {
-		list_for_each_entry(dp, &sdhp->dev_info_list, dev_list) {
-			if ((devip->sdbg_host == dp->sdbg_host) &&
-			    (devip->target == dp->target))
-				clear_bit(SDEBUG_UA_LUNS_CHANGED, dp->uas_bm);
+	list_for_each_entry(dp, &sdhp->dev_info_list, dev_list) {
+		if ((devip->sdbg_host == dp->sdbg_host) &&
+		    (devip->target == dp->target)) {
+			clear_bit(SDEBUG_UA_LUNS_CHANGED, dp->uas_bm);
 		}
 	}
-	spin_unlock(&sdebug_host_list_lock);
 }
 
 static int make_ua(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
@@ -4899,20 +4894,6 @@ fini:
 	return res;
 }
 
-static struct sdebug_queue *get_queue(struct scsi_cmnd *cmnd)
-{
-	u16 hwq;
-	u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(cmnd));
-
-	hwq = blk_mq_unique_tag_to_hwq(tag);
-
-	pr_debug("tag=%#x, hwq=%d\n", tag, hwq);
-	if (WARN_ON_ONCE(hwq >= submit_queues))
-		hwq = 0;
-
-	return sdebug_q_arr + hwq;
-}
-
 static u32 get_tag(struct scsi_cmnd *cmnd)
 {
 	return blk_mq_unique_tag(scsi_cmd_to_rq(cmnd));
@@ -4921,75 +4902,41 @@ static u32 get_tag(struct scsi_cmnd *cmnd)
 /* Queued (deferred) command completions converge here. */
 static void sdebug_q_cmd_complete(struct sdebug_defer *sd_dp)
 {
-	bool aborted = sd_dp->aborted;
-	int qc_idx;
-	int retiring = 0;
-	unsigned long iflags;
-	struct sdebug_queue *sqp;
-	struct sdebug_queued_cmd *sqcp;
-	struct scsi_cmnd *scp;
-	struct sdebug_dev_info *devip;
+	struct sdebug_queued_cmd *sqcp = container_of(sd_dp, struct sdebug_queued_cmd, sd_dp);
+	unsigned long flags;
+	struct scsi_cmnd *scp = sqcp->scmd;
+	struct sdebug_scsi_cmd *sdsc;
+	bool aborted;
 
-	if (unlikely(aborted))
-		sd_dp->aborted = false;
-	qc_idx = sd_dp->qc_idx;
-	sqp = sdebug_q_arr + sd_dp->sqa_idx;
 	if (sdebug_statistics) {
 		atomic_inc(&sdebug_completions);
 		if (raw_smp_processor_id() != sd_dp->issuing_cpu)
 			atomic_inc(&sdebug_miss_cpus);
 	}
-	if (unlikely((qc_idx < 0) || (qc_idx >= SDEBUG_CANQUEUE))) {
-		pr_err("wild qc_idx=%d\n", qc_idx);
-		return;
-	}
-	spin_lock_irqsave(&sqp->qc_lock, iflags);
-	WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_NONE);
-	sqcp = &sqp->qc_arr[qc_idx];
-	scp = sqcp->a_cmnd;
-	if (unlikely(scp == NULL)) {
-		spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-		pr_err("scp is NULL, sqa_idx=%d, qc_idx=%d, hc_idx=%d\n",
-		       sd_dp->sqa_idx, qc_idx, sd_dp->hc_idx);
-		return;
-	}
-	devip = (struct sdebug_dev_info *)scp->device->hostdata;
-	if (likely(devip))
-		atomic_dec(&devip->num_in_q);
-	else
-		pr_err("devip=NULL\n");
-	if (unlikely(atomic_read(&retired_max_queue) > 0))
-		retiring = 1;
-
-	sqcp->a_cmnd = NULL;
-	if (unlikely(!test_and_clear_bit(qc_idx, sqp->in_use_bm))) {
-		spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-		pr_err("Unexpected completion\n");
-		return;
+
+	if (!scp) {
+		pr_err("scmd=NULL\n");
+		goto out;
 	}
 
-	if (unlikely(retiring)) {	/* user has reduced max_queue */
-		int k, retval;
+	sdsc = scsi_cmd_priv(scp);
+	spin_lock_irqsave(&sdsc->lock, flags);
+	aborted = sd_dp->aborted;
+	if (unlikely(aborted))
+		sd_dp->aborted = false;
+	ASSIGN_QUEUED_CMD(scp, NULL);
+
+	spin_unlock_irqrestore(&sdsc->lock, flags);
 
-		retval = atomic_read(&retired_max_queue);
-		if (qc_idx >= retval) {
-			spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-			pr_err("index %d too large\n", retval);
-			return;
-		}
-		k = find_last_bit(sqp->in_use_bm, retval);
-		if ((k < sdebug_max_queue) || (k == retval))
-			atomic_set(&retired_max_queue, 0);
-		else
-			atomic_set(&retired_max_queue, k + 1);
-	}
-	spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-	if (unlikely(aborted)) {
-		if (sdebug_verbose)
-			pr_info("bypassing scsi_done() due to aborted cmd\n");
-		return;
+	if (aborted) {
+		pr_info("bypassing scsi_done() due to aborted cmd, kicking-off EH\n");
+		blk_abort_request(scsi_cmd_to_rq(scp));
+		goto out;
 	}
+
 	scsi_done(scp); /* callback to mid level */
+out:
+	sdebug_free_queued_cmd(sqcp);
 }
 
 /* When high resolution timer goes off this function is called. */
@@ -5152,7 +5099,6 @@ static struct sdebug_dev_info *sdebug_device_create(
 		} else {
 			devip->zmodel = BLK_ZONED_NONE;
 		}
-		devip->sdbg_host = sdbg_host;
 		devip->create_ts = ktime_get_boottime();
 		atomic_set(&devip->stopped, (sdeb_tur_ms_to_ready > 0 ? 2 : 0));
 		list_add_tail(&devip->dev_list, &sdbg_host->dev_info_list);
@@ -5166,11 +5112,7 @@ static struct sdebug_dev_info *find_build_dev_info(struct scsi_device *sdev)
 	struct sdebug_dev_info *open_devip = NULL;
 	struct sdebug_dev_info *devip;
 
-	sdbg_host = *(struct sdebug_host_info **)shost_priv(sdev->host);
-	if (!sdbg_host) {
-		pr_err("Host info NULL\n");
-		return NULL;
-	}
+	sdbg_host = shost_to_sdebug_host(sdev->host);
 
 	list_for_each_entry(devip, &sdbg_host->dev_info_list, dev_list) {
 		if ((devip->used) && (devip->channel == sdev->channel) &&
@@ -5194,7 +5136,6 @@ static struct sdebug_dev_info *find_build_dev_info(struct scsi_device *sdev)
 	open_devip->target = sdev->id;
 	open_devip->lun = sdev->lun;
 	open_devip->sdbg_host = sdbg_host;
-	atomic_set(&open_devip->num_in_q, 0);
 	set_bit(SDEBUG_UA_POOCCUR, open_devip->uas_bm);
 	open_devip->used = true;
 	return open_devip;
@@ -5245,215 +5186,171 @@ static void scsi_debug_slave_destroy(struct scsi_device *sdp)
 	}
 }
 
-static void stop_qc_helper(struct sdebug_defer *sd_dp,
+/* Returns true if we require the queued memory to be freed by the caller. */
+static bool stop_qc_helper(struct sdebug_defer *sd_dp,
 			   enum sdeb_defer_type defer_t)
 {
-	if (!sd_dp)
-		return;
-	if (defer_t == SDEB_DEFER_HRT)
-		hrtimer_cancel(&sd_dp->hrt);
-	else if (defer_t == SDEB_DEFER_WQ)
-		cancel_work_sync(&sd_dp->ew.work);
-}
-
-/* If @cmnd found deletes its timer or work queue and returns true; else
-   returns false */
-static bool stop_queued_cmnd(struct scsi_cmnd *cmnd)
-{
-	unsigned long iflags;
-	int j, k, qmax, r_qmax;
-	enum sdeb_defer_type l_defer_t;
-	struct sdebug_queue *sqp;
-	struct sdebug_queued_cmd *sqcp;
-	struct sdebug_dev_info *devip;
-	struct sdebug_defer *sd_dp;
+	if (defer_t == SDEB_DEFER_HRT) {
+		int res = hrtimer_try_to_cancel(&sd_dp->hrt);
 
-	for (j = 0, sqp = sdebug_q_arr; j < submit_queues; ++j, ++sqp) {
-		spin_lock_irqsave(&sqp->qc_lock, iflags);
-		qmax = sdebug_max_queue;
-		r_qmax = atomic_read(&retired_max_queue);
-		if (r_qmax > qmax)
-			qmax = r_qmax;
-		for (k = 0; k < qmax; ++k) {
-			if (test_bit(k, sqp->in_use_bm)) {
-				sqcp = &sqp->qc_arr[k];
-				if (cmnd != sqcp->a_cmnd)
-					continue;
-				/* found */
-				devip = (struct sdebug_dev_info *)
-						cmnd->device->hostdata;
-				if (devip)
-					atomic_dec(&devip->num_in_q);
-				sqcp->a_cmnd = NULL;
-				sd_dp = sqcp->sd_dp;
-				if (sd_dp) {
-					l_defer_t = READ_ONCE(sd_dp->defer_t);
-					WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_NONE);
-				} else
-					l_defer_t = SDEB_DEFER_NONE;
-				spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-				stop_qc_helper(sd_dp, l_defer_t);
-				clear_bit(k, sqp->in_use_bm);
-				return true;
-			}
+		switch (res) {
+		case 0: /* Not active, it must have already run */
+		case -1: /* -1 It's executing the CB */
+			return false;
+		case 1: /* Was active, we've now cancelled */
+		default:
+			return true;
 		}
-		spin_unlock_irqrestore(&sqp->qc_lock, iflags);
+	} else if (defer_t == SDEB_DEFER_WQ) {
+		/* Cancel if pending */
+		if (cancel_work_sync(&sd_dp->ew.work))
+			return true;
+		/* Was not pending, so it must have run */
+		return false;
+	} else if (defer_t == SDEB_DEFER_POLL) {
+		return true;
 	}
+
 	return false;
 }
 
-/* Deletes (stops) timers or work queues of all queued commands */
-static void stop_all_queued(void)
+
+static bool scsi_debug_stop_cmnd(struct scsi_cmnd *cmnd)
 {
-	unsigned long iflags;
-	int j, k;
 	enum sdeb_defer_type l_defer_t;
-	struct sdebug_queue *sqp;
-	struct sdebug_queued_cmd *sqcp;
-	struct sdebug_dev_info *devip;
 	struct sdebug_defer *sd_dp;
+	struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmnd);
+	struct sdebug_queued_cmd *sqcp = TO_QUEUED_CMD(cmnd);
 
-	for (j = 0, sqp = sdebug_q_arr; j < submit_queues; ++j, ++sqp) {
-		spin_lock_irqsave(&sqp->qc_lock, iflags);
-		for (k = 0; k < SDEBUG_CANQUEUE; ++k) {
-			if (test_bit(k, sqp->in_use_bm)) {
-				sqcp = &sqp->qc_arr[k];
-				if (sqcp->a_cmnd == NULL)
-					continue;
-				devip = (struct sdebug_dev_info *)
-					sqcp->a_cmnd->device->hostdata;
-				if (devip)
-					atomic_dec(&devip->num_in_q);
-				sqcp->a_cmnd = NULL;
-				sd_dp = sqcp->sd_dp;
-				if (sd_dp) {
-					l_defer_t = READ_ONCE(sd_dp->defer_t);
-					WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_NONE);
-				} else
-					l_defer_t = SDEB_DEFER_NONE;
-				spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-				stop_qc_helper(sd_dp, l_defer_t);
-				clear_bit(k, sqp->in_use_bm);
-				spin_lock_irqsave(&sqp->qc_lock, iflags);
-			}
-		}
-		spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-	}
+	lockdep_assert_held(&sdsc->lock);
+
+	if (!sqcp)
+		return false;
+	sd_dp = &sqcp->sd_dp;
+	l_defer_t = READ_ONCE(sd_dp->defer_t);
+	ASSIGN_QUEUED_CMD(cmnd, NULL);
+
+	if (stop_qc_helper(sd_dp, l_defer_t))
+		sdebug_free_queued_cmd(sqcp);
+
+	return true;
+}
+
+/*
+ * Called from scsi_debug_abort() only, which is for timed-out cmd.
+ */
+static bool scsi_debug_abort_cmnd(struct scsi_cmnd *cmnd)
+{
+	struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmnd);
+	unsigned long flags;
+	bool res;
+
+	spin_lock_irqsave(&sdsc->lock, flags);
+	res = scsi_debug_stop_cmnd(cmnd);
+	spin_unlock_irqrestore(&sdsc->lock, flags);
+
+	return res;
 }
 
-/* Free queued command memory on heap */
-static void free_all_queued(void)
+/*
+ * All we can do is set the cmnd as internally aborted and wait for it to
+ * finish. We cannot call scsi_done() as normal completion path may do that.
+ */
+static bool sdebug_stop_cmnd(struct request *rq, void *data)
 {
-	int j, k;
-	struct sdebug_queue *sqp;
-	struct sdebug_queued_cmd *sqcp;
+	scsi_debug_abort_cmnd(blk_mq_rq_to_pdu(rq));
 
-	for (j = 0, sqp = sdebug_q_arr; j < submit_queues; ++j, ++sqp) {
-		for (k = 0; k < SDEBUG_CANQUEUE; ++k) {
-			sqcp = &sqp->qc_arr[k];
-			kfree(sqcp->sd_dp);
-			sqcp->sd_dp = NULL;
-		}
+	return true;
+}
+
+/* Deletes (stops) timers or work queues of all queued commands */
+static void stop_all_queued(void)
+{
+	struct sdebug_host_info *sdhp;
+
+	mutex_lock(&sdebug_host_list_mutex);
+	list_for_each_entry(sdhp, &sdebug_host_list, host_list) {
+		struct Scsi_Host *shost = sdhp->shost;
+
+		blk_mq_tagset_busy_iter(&shost->tag_set, sdebug_stop_cmnd, NULL);
 	}
+	mutex_unlock(&sdebug_host_list_mutex);
 }
 
 static int scsi_debug_abort(struct scsi_cmnd *SCpnt)
 {
-	bool ok;
+	bool ok = scsi_debug_abort_cmnd(SCpnt);
 
 	++num_aborts;
-	if (SCpnt) {
-		ok = stop_queued_cmnd(SCpnt);
-		if (SCpnt->device && (SDEBUG_OPT_ALL_NOISE & sdebug_opts))
-			sdev_printk(KERN_INFO, SCpnt->device,
-				    "%s: command%s found\n", __func__,
-				    ok ? "" : " not");
-	}
+
+	if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
+		sdev_printk(KERN_INFO, SCpnt->device,
+			    "%s: command%s found\n", __func__,
+			    ok ? "" : " not");
+
 	return SUCCESS;
 }
 
 static int scsi_debug_device_reset(struct scsi_cmnd *SCpnt)
 {
+	struct scsi_device *sdp = SCpnt->device;
+	struct sdebug_dev_info *devip = sdp->hostdata;
+
 	++num_dev_resets;
-	if (SCpnt && SCpnt->device) {
-		struct scsi_device *sdp = SCpnt->device;
-		struct sdebug_dev_info *devip =
-				(struct sdebug_dev_info *)sdp->hostdata;
 
-		if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
-			sdev_printk(KERN_INFO, sdp, "%s\n", __func__);
-		if (devip)
-			set_bit(SDEBUG_UA_POR, devip->uas_bm);
-	}
+	if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
+		sdev_printk(KERN_INFO, sdp, "%s\n", __func__);
+	if (devip)
+		set_bit(SDEBUG_UA_POR, devip->uas_bm);
+
 	return SUCCESS;
 }
 
 static int scsi_debug_target_reset(struct scsi_cmnd *SCpnt)
 {
-	struct sdebug_host_info *sdbg_host;
+	struct scsi_device *sdp = SCpnt->device;
+	struct sdebug_host_info *sdbg_host = shost_to_sdebug_host(sdp->host);
 	struct sdebug_dev_info *devip;
-	struct scsi_device *sdp;
-	struct Scsi_Host *hp;
 	int k = 0;
 
 	++num_target_resets;
-	if (!SCpnt)
-		goto lie;
-	sdp = SCpnt->device;
-	if (!sdp)
-		goto lie;
 	if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
 		sdev_printk(KERN_INFO, sdp, "%s\n", __func__);
-	hp = sdp->host;
-	if (!hp)
-		goto lie;
-	sdbg_host = *(struct sdebug_host_info **)shost_priv(hp);
-	if (sdbg_host) {
-		list_for_each_entry(devip,
-				    &sdbg_host->dev_info_list,
-				    dev_list)
-			if (devip->target == sdp->id) {
-				set_bit(SDEBUG_UA_BUS_RESET, devip->uas_bm);
-				++k;
-			}
+
+	list_for_each_entry(devip, &sdbg_host->dev_info_list, dev_list) {
+		if (devip->target == sdp->id) {
+			set_bit(SDEBUG_UA_BUS_RESET, devip->uas_bm);
+			++k;
+		}
 	}
+
 	if (SDEBUG_OPT_RESET_NOISE & sdebug_opts)
 		sdev_printk(KERN_INFO, sdp,
 			    "%s: %d device(s) found in target\n", __func__, k);
-lie:
+
 	return SUCCESS;
 }
 
 static int scsi_debug_bus_reset(struct scsi_cmnd *SCpnt)
 {
-	struct sdebug_host_info *sdbg_host;
+	struct scsi_device *sdp = SCpnt->device;
+	struct sdebug_host_info *sdbg_host = shost_to_sdebug_host(sdp->host);
 	struct sdebug_dev_info *devip;
-	struct scsi_device *sdp;
-	struct Scsi_Host *hp;
 	int k = 0;
 
 	++num_bus_resets;
-	if (!(SCpnt && SCpnt->device))
-		goto lie;
-	sdp = SCpnt->device;
+
 	if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
 		sdev_printk(KERN_INFO, sdp, "%s\n", __func__);
-	hp = sdp->host;
-	if (hp) {
-		sdbg_host = *(struct sdebug_host_info **)shost_priv(hp);
-		if (sdbg_host) {
-			list_for_each_entry(devip,
-					    &sdbg_host->dev_info_list,
-					    dev_list) {
-				set_bit(SDEBUG_UA_BUS_RESET, devip->uas_bm);
-				++k;
-			}
-		}
+
+	list_for_each_entry(devip, &sdbg_host->dev_info_list, dev_list) {
+		set_bit(SDEBUG_UA_BUS_RESET, devip->uas_bm);
+		++k;
 	}
+
 	if (SDEBUG_OPT_RESET_NOISE & sdebug_opts)
 		sdev_printk(KERN_INFO, sdp,
 			    "%s: %d device(s) found in host\n", __func__, k);
-lie:
 	return SUCCESS;
 }
 
@@ -5464,9 +5361,9 @@ static int scsi_debug_host_reset(struct scsi_cmnd *SCpnt)
 	int k = 0;
 
 	++num_host_resets;
-	if ((SCpnt->device) && (SDEBUG_OPT_ALL_NOISE & sdebug_opts))
+	if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
 		sdev_printk(KERN_INFO, SCpnt->device, "%s\n", __func__);
-	spin_lock(&sdebug_host_list_lock);
+	mutex_lock(&sdebug_host_list_mutex);
 	list_for_each_entry(sdbg_host, &sdebug_host_list, host_list) {
 		list_for_each_entry(devip, &sdbg_host->dev_info_list,
 				    dev_list) {
@@ -5474,7 +5371,7 @@ static int scsi_debug_host_reset(struct scsi_cmnd *SCpnt)
 			++k;
 		}
 	}
-	spin_unlock(&sdebug_host_list_lock);
+	mutex_unlock(&sdebug_host_list_mutex);
 	stop_all_queued();
 	if (SDEBUG_OPT_RESET_NOISE & sdebug_opts)
 		sdev_printk(KERN_INFO, SCpnt->device,
@@ -5537,11 +5434,18 @@ static void sdebug_build_parts(unsigned char *ramp, unsigned long store_size)
 
 static void block_unblock_all_queues(bool block)
 {
-	int j;
-	struct sdebug_queue *sqp;
+	struct sdebug_host_info *sdhp;
 
-	for (j = 0, sqp = sdebug_q_arr; j < submit_queues; ++j, ++sqp)
-		atomic_set(&sqp->blocked, (int)block);
+	lockdep_assert_held(&sdebug_host_list_mutex);
+
+	list_for_each_entry(sdhp, &sdebug_host_list, host_list) {
+		struct Scsi_Host *shost = sdhp->shost;
+
+		if (block)
+			scsi_block_requests(shost);
+		else
+			scsi_unblock_requests(shost);
+	}
 }
 
 /* Adjust (by rounding down) the sdebug_cmnd_count so abs(every_nth)-1
@@ -5554,10 +5458,13 @@ static void tweak_cmnd_count(void)
 	modulo = abs(sdebug_every_nth);
 	if (modulo < 2)
 		return;
+
+	mutex_lock(&sdebug_host_list_mutex);
 	block_unblock_all_queues(true);
 	count = atomic_read(&sdebug_cmnd_count);
 	atomic_set(&sdebug_cmnd_count, (count / modulo) * modulo);
 	block_unblock_all_queues(false);
+	mutex_unlock(&sdebug_host_list_mutex);
 }
 
 static void clear_queue_stats(void)
@@ -5577,6 +5484,33 @@ static bool inject_on_this_cmd(void)
 
 #define INCLUSIVE_TIMING_MAX_NS 1000000		/* 1 millisecond */
 
+
+void sdebug_free_queued_cmd(struct sdebug_queued_cmd *sqcp)
+{
+	if (sqcp)
+		kmem_cache_free(queued_cmd_cache, sqcp);
+}
+
+static struct sdebug_queued_cmd *sdebug_alloc_queued_cmd(struct scsi_cmnd *scmd)
+{
+	struct sdebug_queued_cmd *sqcp;
+	struct sdebug_defer *sd_dp;
+
+	sqcp = kmem_cache_zalloc(queued_cmd_cache, GFP_ATOMIC);
+	if (!sqcp)
+		return NULL;
+
+	sd_dp = &sqcp->sd_dp;
+
+	hrtimer_init(&sd_dp->hrt, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+	sd_dp->hrt.function = sdebug_q_cmd_hrt_complete;
+	INIT_WORK(&sd_dp->ew.work, sdebug_q_cmd_wq_complete);
+
+	sqcp->scmd = scmd;
+
+	return sqcp;
+}
+
 /* Complete the processing of the thread that queued a SCSI command to this
  * driver. It either completes the command by calling cmnd_done() or
  * schedules a hr timer or work queue then returns 0. Returns
@@ -5588,13 +5522,11 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 				    struct sdebug_dev_info *),
 			 int delta_jiff, int ndelay)
 {
-	bool new_sd_dp;
-	bool inject = false;
-	bool polled = scsi_cmd_to_rq(cmnd)->cmd_flags & REQ_POLLED;
-	int k, num_in_q, qdepth;
-	unsigned long iflags;
+	struct request *rq = scsi_cmd_to_rq(cmnd);
+	bool polled = rq->cmd_flags & REQ_POLLED;
+	struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmnd);
+	unsigned long flags;
 	u64 ns_from_boot = 0;
-	struct sdebug_queue *sqp;
 	struct sdebug_queued_cmd *sqcp;
 	struct scsi_device *sdp;
 	struct sdebug_defer *sd_dp;
@@ -5609,66 +5541,30 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 	if (delta_jiff == 0)
 		goto respond_in_thread;
 
-	sqp = get_queue(cmnd);
-	spin_lock_irqsave(&sqp->qc_lock, iflags);
-	if (unlikely(atomic_read(&sqp->blocked))) {
-		spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-		return SCSI_MLQUEUE_HOST_BUSY;
-	}
-	num_in_q = atomic_read(&devip->num_in_q);
-	qdepth = cmnd->device->queue_depth;
-	if (unlikely((qdepth > 0) && (num_in_q >= qdepth))) {
-		if (scsi_result) {
-			spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-			goto respond_in_thread;
-		} else
-			scsi_result = device_qfull_result;
-	} else if (unlikely(sdebug_every_nth &&
-			    (SDEBUG_OPT_RARE_TSF & sdebug_opts) &&
-			    (scsi_result == 0))) {
-		if ((num_in_q == (qdepth - 1)) &&
+
+	if (unlikely(sdebug_every_nth && (SDEBUG_OPT_RARE_TSF & sdebug_opts) &&
+		     (scsi_result == 0))) {
+		int num_in_q = scsi_device_busy(sdp);
+		int qdepth = cmnd->device->queue_depth;
+
+		if ((num_in_q == qdepth) &&
 		    (atomic_inc_return(&sdebug_a_tsf) >=
 		     abs(sdebug_every_nth))) {
 			atomic_set(&sdebug_a_tsf, 0);
-			inject = true;
 			scsi_result = device_qfull_result;
-		}
-	}
 
-	k = find_first_zero_bit(sqp->in_use_bm, sdebug_max_queue);
-	if (unlikely(k >= sdebug_max_queue)) {
-		spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-		if (scsi_result)
-			goto respond_in_thread;
-		scsi_result = device_qfull_result;
-		if (SDEBUG_OPT_Q_NOISE & sdebug_opts)
-			sdev_printk(KERN_INFO, sdp, "%s: max_queue=%d exceeded: TASK SET FULL\n",
-				    __func__, sdebug_max_queue);
-		goto respond_in_thread;
-	}
-	set_bit(k, sqp->in_use_bm);
-	atomic_inc(&devip->num_in_q);
-	sqcp = &sqp->qc_arr[k];
-	sqcp->a_cmnd = cmnd;
-	cmnd->host_scribble = (unsigned char *)sqcp;
-	sd_dp = sqcp->sd_dp;
-	spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-
-	if (!sd_dp) {
-		sd_dp = kzalloc(sizeof(*sd_dp), GFP_ATOMIC);
-		if (!sd_dp) {
-			atomic_dec(&devip->num_in_q);
-			clear_bit(k, sqp->in_use_bm);
-			return SCSI_MLQUEUE_HOST_BUSY;
+			if (unlikely(SDEBUG_OPT_Q_NOISE & sdebug_opts))
+				sdev_printk(KERN_INFO, sdp, "%s: num_in_q=%d +1, <inject> status: TASK SET FULL\n",
+					    __func__, num_in_q);
 		}
-		new_sd_dp = true;
-	} else {
-		new_sd_dp = false;
 	}
 
-	/* Set the hostwide tag */
-	if (sdebug_host_max_queue)
-		sd_dp->hc_idx = get_tag(cmnd);
+	sqcp = sdebug_alloc_queued_cmd(cmnd);
+	if (!sqcp) {
+		pr_err("%s no alloc\n", __func__);
+		return SCSI_MLQUEUE_HOST_BUSY;
+	}
+	sd_dp = &sqcp->sd_dp;
 
 	if (polled)
 		ns_from_boot = ktime_get_boottime_ns();
@@ -5715,14 +5611,8 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 				u64 d = ktime_get_boottime_ns() - ns_from_boot;
 
 				if (kt <= d) {	/* elapsed duration >= kt */
-					spin_lock_irqsave(&sqp->qc_lock, iflags);
-					sqcp->a_cmnd = NULL;
-					atomic_dec(&devip->num_in_q);
-					clear_bit(k, sqp->in_use_bm);
-					spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-					if (new_sd_dp)
-						kfree(sd_dp);
 					/* call scsi_done() from this thread */
+					sdebug_free_queued_cmd(sqcp);
 					scsi_done(cmnd);
 					return 0;
 				}
@@ -5730,72 +5620,54 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 				kt -= d;
 			}
 		}
+		if (sdebug_statistics)
+			sd_dp->issuing_cpu = raw_smp_processor_id();
 		if (polled) {
+			spin_lock_irqsave(&sdsc->lock, flags);
 			sd_dp->cmpl_ts = ktime_add(ns_to_ktime(ns_from_boot), kt);
-			spin_lock_irqsave(&sqp->qc_lock, iflags);
-			if (!sd_dp->init_poll) {
-				sd_dp->init_poll = true;
-				sqcp->sd_dp = sd_dp;
-				sd_dp->sqa_idx = sqp - sdebug_q_arr;
-				sd_dp->qc_idx = k;
-			}
+			ASSIGN_QUEUED_CMD(cmnd, sqcp);
 			WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_POLL);
-			spin_unlock_irqrestore(&sqp->qc_lock, iflags);
+			spin_unlock_irqrestore(&sdsc->lock, flags);
 		} else {
-			if (!sd_dp->init_hrt) {
-				sd_dp->init_hrt = true;
-				sqcp->sd_dp = sd_dp;
-				hrtimer_init(&sd_dp->hrt, CLOCK_MONOTONIC,
-					     HRTIMER_MODE_REL_PINNED);
-				sd_dp->hrt.function = sdebug_q_cmd_hrt_complete;
-				sd_dp->sqa_idx = sqp - sdebug_q_arr;
-				sd_dp->qc_idx = k;
-			}
-			WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_HRT);
 			/* schedule the invocation of scsi_done() for a later time */
+			spin_lock_irqsave(&sdsc->lock, flags);
+			ASSIGN_QUEUED_CMD(cmnd, sqcp);
+			WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_HRT);
 			hrtimer_start(&sd_dp->hrt, kt, HRTIMER_MODE_REL_PINNED);
+			/*
+			 * The completion handler will try to grab sqcp->lock,
+			 * so there is no chance that the completion handler
+			 * will call scsi_done() until we release the lock
+			 * here (so ok to keep referencing sdsc).
+			 */
+			spin_unlock_irqrestore(&sdsc->lock, flags);
 		}
-		if (sdebug_statistics)
-			sd_dp->issuing_cpu = raw_smp_processor_id();
 	} else {	/* jdelay < 0, use work queue */
 		if (unlikely((sdebug_opts & SDEBUG_OPT_CMD_ABORT) &&
-			     atomic_read(&sdeb_inject_pending)))
+			     atomic_read(&sdeb_inject_pending))) {
 			sd_dp->aborted = true;
+			atomic_set(&sdeb_inject_pending, 0);
+			sdev_printk(KERN_INFO, sdp, "abort request tag=%#x\n",
+				    blk_mq_unique_tag_to_tag(get_tag(cmnd)));
+		}
+
+		if (sdebug_statistics)
+			sd_dp->issuing_cpu = raw_smp_processor_id();
 		if (polled) {
+			spin_lock_irqsave(&sdsc->lock, flags);
+			ASSIGN_QUEUED_CMD(cmnd, sqcp);
 			sd_dp->cmpl_ts = ns_to_ktime(ns_from_boot);
-			spin_lock_irqsave(&sqp->qc_lock, iflags);
-			if (!sd_dp->init_poll) {
-				sd_dp->init_poll = true;
-				sqcp->sd_dp = sd_dp;
-				sd_dp->sqa_idx = sqp - sdebug_q_arr;
-				sd_dp->qc_idx = k;
-			}
 			WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_POLL);
-			spin_unlock_irqrestore(&sqp->qc_lock, iflags);
+			spin_unlock_irqrestore(&sdsc->lock, flags);
 		} else {
-			if (!sd_dp->init_wq) {
-				sd_dp->init_wq = true;
-				sqcp->sd_dp = sd_dp;
-				sd_dp->sqa_idx = sqp - sdebug_q_arr;
-				sd_dp->qc_idx = k;
-				INIT_WORK(&sd_dp->ew.work, sdebug_q_cmd_wq_complete);
-			}
+			spin_lock_irqsave(&sdsc->lock, flags);
+			ASSIGN_QUEUED_CMD(cmnd, sqcp);
 			WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_WQ);
 			schedule_work(&sd_dp->ew.work);
-		}
-		if (sdebug_statistics)
-			sd_dp->issuing_cpu = raw_smp_processor_id();
-		if (unlikely(sd_dp->aborted)) {
-			sdev_printk(KERN_INFO, sdp, "abort request tag %d\n",
-				    scsi_cmd_to_rq(cmnd)->tag);
-			blk_abort_request(scsi_cmd_to_rq(cmnd));
-			atomic_set(&sdeb_inject_pending, 0);
-			sd_dp->aborted = false;
+			spin_unlock_irqrestore(&sdsc->lock, flags);
 		}
 	}
-	if (unlikely((SDEBUG_OPT_Q_NOISE & sdebug_opts) && scsi_result == device_qfull_result))
-		sdev_printk(KERN_INFO, sdp, "%s: num_in_q=%d +1, %s%s\n", __func__,
-			    num_in_q, (inject ? "<inject> " : ""), "status: TASK SET FULL");
+
 	return 0;
 
 respond_in_thread:	/* call back to mid-layer using invocation thread */
@@ -5996,14 +5868,39 @@ static int scsi_debug_write_info(struct Scsi_Host *host, char *buffer,
 	return length;
 }
 
+struct sdebug_submit_queue_data {
+	int *first;
+	int *last;
+	int queue_num;
+};
+
+static bool sdebug_submit_queue_iter(struct request *rq, void *opaque)
+{
+	struct sdebug_submit_queue_data *data = opaque;
+	u32 unique_tag = blk_mq_unique_tag(rq);
+	u16 hwq = blk_mq_unique_tag_to_hwq(unique_tag);
+	u16 tag = blk_mq_unique_tag_to_tag(unique_tag);
+	int queue_num = data->queue_num;
+
+	if (hwq != queue_num)
+		return true;
+
+	/* Rely on iter'ing in ascending tag order */
+	if (*data->first == -1)
+		*data->first = *data->last = tag;
+	else
+		*data->last = tag;
+
+	return true;
+}
+
 /* Output seen with 'cat /proc/scsi/scsi_debug/<host_id>'. It will be the
  * same for each scsi_debug host (if more than one). Some of the counters
  * output are not atomics so might be inaccurate in a busy system. */
 static int scsi_debug_show_info(struct seq_file *m, struct Scsi_Host *host)
 {
-	int f, j, l;
-	struct sdebug_queue *sqp;
 	struct sdebug_host_info *sdhp;
+	int j;
 
 	seq_printf(m, "scsi_debug adapter driver, version %s [%s]\n",
 		   SDEBUG_VERSION, sdebug_version_date);
@@ -6031,11 +5928,17 @@ static int scsi_debug_show_info(struct seq_file *m, struct Scsi_Host *host)
 		   atomic_read(&sdeb_mq_poll_count));
 
 	seq_printf(m, "submit_queues=%d\n", submit_queues);
-	for (j = 0, sqp = sdebug_q_arr; j < submit_queues; ++j, ++sqp) {
+	for (j = 0; j < submit_queues; ++j) {
+		int f = -1, l = -1;
+		struct sdebug_submit_queue_data data = {
+			.queue_num = j,
+			.first = &f,
+			.last = &l,
+		};
 		seq_printf(m, "  queue %d:\n", j);
-		f = find_first_bit(sqp->in_use_bm, sdebug_max_queue);
-		if (f != sdebug_max_queue) {
-			l = find_last_bit(sqp->in_use_bm, sdebug_max_queue);
+		blk_mq_tagset_busy_iter(&host->tag_set, sdebug_submit_queue_iter,
+					&data);
+		if (f >= 0) {
 			seq_printf(m, "    in_use_bm BUSY: %s: %d,%d\n",
 				   "first,last bits", f, l);
 		}
@@ -6086,15 +5989,15 @@ static ssize_t delay_store(struct device_driver *ddp, const char *buf,
 	if (count > 0 && sscanf(buf, "%d", &jdelay) == 1) {
 		res = count;
 		if (sdebug_jdelay != jdelay) {
-			int j, k;
-			struct sdebug_queue *sqp;
+			struct sdebug_host_info *sdhp;
 
+			mutex_lock(&sdebug_host_list_mutex);
 			block_unblock_all_queues(true);
-			for (j = 0, sqp = sdebug_q_arr; j < submit_queues;
-			     ++j, ++sqp) {
-				k = find_first_bit(sqp->in_use_bm,
-						   sdebug_max_queue);
-				if (k != sdebug_max_queue) {
+
+			list_for_each_entry(sdhp, &sdebug_host_list, host_list) {
+				struct Scsi_Host *shost = sdhp->shost;
+
+				if (scsi_host_busy(shost)) {
 					res = -EBUSY;   /* queued commands */
 					break;
 				}
@@ -6104,6 +6007,7 @@ static ssize_t delay_store(struct device_driver *ddp, const char *buf,
 				sdebug_ndelay = 0;
 			}
 			block_unblock_all_queues(false);
+			mutex_unlock(&sdebug_host_list_mutex);
 		}
 		return res;
 	}
@@ -6126,25 +6030,27 @@ static ssize_t ndelay_store(struct device_driver *ddp, const char *buf,
 	    (ndelay >= 0) && (ndelay < (1000 * 1000 * 1000))) {
 		res = count;
 		if (sdebug_ndelay != ndelay) {
-			int j, k;
-			struct sdebug_queue *sqp;
+			struct sdebug_host_info *sdhp;
 
+			mutex_lock(&sdebug_host_list_mutex);
 			block_unblock_all_queues(true);
-			for (j = 0, sqp = sdebug_q_arr; j < submit_queues;
-			     ++j, ++sqp) {
-				k = find_first_bit(sqp->in_use_bm,
-						   sdebug_max_queue);
-				if (k != sdebug_max_queue) {
+
+			list_for_each_entry(sdhp, &sdebug_host_list, host_list) {
+				struct Scsi_Host *shost = sdhp->shost;
+
+				if (scsi_host_busy(shost)) {
 					res = -EBUSY;   /* queued commands */
 					break;
 				}
 			}
+
 			if (res > 0) {
 				sdebug_ndelay = ndelay;
 				sdebug_jdelay = ndelay  ? JDELAY_OVERRIDDEN
 							: DEF_JDELAY;
 			}
 			block_unblock_all_queues(false);
+			mutex_unlock(&sdebug_host_list_mutex);
 		}
 		return res;
 	}
@@ -6390,13 +6296,13 @@ static ssize_t lun_format_store(struct device_driver *ddp, const char *buf,
 			struct sdebug_host_info *sdhp;
 			struct sdebug_dev_info *dp;
 
-			spin_lock(&sdebug_host_list_lock);
+			mutex_lock(&sdebug_host_list_mutex);
 			list_for_each_entry(sdhp, &sdebug_host_list, host_list) {
 				list_for_each_entry(dp, &sdhp->dev_info_list, dev_list) {
 					set_bit(SDEBUG_UA_LUNS_CHANGED, dp->uas_bm);
 				}
 			}
-			spin_unlock(&sdebug_host_list_lock);
+			mutex_unlock(&sdebug_host_list_mutex);
 		}
 		return count;
 	}
@@ -6426,7 +6332,7 @@ static ssize_t max_luns_store(struct device_driver *ddp, const char *buf,
 			struct sdebug_host_info *sdhp;
 			struct sdebug_dev_info *dp;
 
-			spin_lock(&sdebug_host_list_lock);
+			mutex_lock(&sdebug_host_list_mutex);
 			list_for_each_entry(sdhp, &sdebug_host_list,
 					    host_list) {
 				list_for_each_entry(dp, &sdhp->dev_info_list,
@@ -6435,7 +6341,7 @@ static ssize_t max_luns_store(struct device_driver *ddp, const char *buf,
 						dp->uas_bm);
 				}
 			}
-			spin_unlock(&sdebug_host_list_lock);
+			mutex_unlock(&sdebug_host_list_mutex);
 		}
 		return count;
 	}
@@ -6452,28 +6358,19 @@ static ssize_t max_queue_show(struct device_driver *ddp, char *buf)
 static ssize_t max_queue_store(struct device_driver *ddp, const char *buf,
 			       size_t count)
 {
-	int j, n, k, a;
-	struct sdebug_queue *sqp;
+	int n;
 
 	if ((count > 0) && (1 == sscanf(buf, "%d", &n)) && (n > 0) &&
 	    (n <= SDEBUG_CANQUEUE) &&
 	    (sdebug_host_max_queue == 0)) {
-		block_unblock_all_queues(true);
-		k = 0;
-		for (j = 0, sqp = sdebug_q_arr; j < submit_queues;
-		     ++j, ++sqp) {
-			a = find_last_bit(sqp->in_use_bm, SDEBUG_CANQUEUE);
-			if (a > k)
-				k = a;
-		}
-		sdebug_max_queue = n;
-		if (k == SDEBUG_CANQUEUE)
-			atomic_set(&retired_max_queue, 0);
-		else if (k >= n)
-			atomic_set(&retired_max_queue, k + 1);
+		mutex_lock(&sdebug_host_list_mutex);
+
+		/* We may only change sdebug_max_queue when we have no shosts */
+		if (list_empty(&sdebug_host_list))
+			sdebug_max_queue = n;
 		else
-			atomic_set(&retired_max_queue, 0);
-		block_unblock_all_queues(false);
+			count = -EBUSY;
+		mutex_unlock(&sdebug_host_list_mutex);
 		return count;
 	}
 	return -EINVAL;
@@ -6542,7 +6439,7 @@ static ssize_t virtual_gb_store(struct device_driver *ddp, const char *buf,
 			struct sdebug_host_info *sdhp;
 			struct sdebug_dev_info *dp;
 
-			spin_lock(&sdebug_host_list_lock);
+			mutex_lock(&sdebug_host_list_mutex);
 			list_for_each_entry(sdhp, &sdebug_host_list,
 					    host_list) {
 				list_for_each_entry(dp, &sdhp->dev_info_list,
@@ -6551,7 +6448,7 @@ static ssize_t virtual_gb_store(struct device_driver *ddp, const char *buf,
 						dp->uas_bm);
 				}
 			}
-			spin_unlock(&sdebug_host_list_lock);
+			mutex_unlock(&sdebug_host_list_mutex);
 		}
 		return count;
 	}
@@ -6901,7 +6798,6 @@ static int __init scsi_debug_init(void)
 
 	ramdisk_lck_a[0] = &atomic_rw;
 	ramdisk_lck_a[1] = &atomic_rw2;
-	atomic_set(&retired_max_queue, 0);
 
 	if (sdebug_ndelay >= 1000 * 1000 * 1000) {
 		pr_warn("ndelay must be less than 1 second, ignored\n");
@@ -6997,13 +6893,6 @@ static int __init scsi_debug_init(void)
 			sdebug_max_queue);
 	}
 
-	sdebug_q_arr = kcalloc(submit_queues, sizeof(struct sdebug_queue),
-			       GFP_KERNEL);
-	if (sdebug_q_arr == NULL)
-		return -ENOMEM;
-	for (k = 0; k < submit_queues; ++k)
-		spin_lock_init(&sdebug_q_arr[k].qc_lock);
-
 	/*
 	 * check for host managed zoned block device specified with
 	 * ptype=0x14 or zbc=XXX.
@@ -7012,10 +6901,8 @@ static int __init scsi_debug_init(void)
 		sdeb_zbc_model = BLK_ZONED_HM;
 	} else if (sdeb_zbc_model_s && *sdeb_zbc_model_s) {
 		k = sdeb_zbc_model_str(sdeb_zbc_model_s);
-		if (k < 0) {
-			ret = k;
-			goto free_q_arr;
-		}
+		if (k < 0)
+			return k;
 		sdeb_zbc_model = k;
 		switch (sdeb_zbc_model) {
 		case BLK_ZONED_NONE:
@@ -7027,8 +6914,7 @@ static int __init scsi_debug_init(void)
 			break;
 		default:
 			pr_err("Invalid ZBC model\n");
-			ret = -EINVAL;
-			goto free_q_arr;
+			return -EINVAL;
 		}
 	}
 	if (sdeb_zbc_model != BLK_ZONED_NONE) {
@@ -7075,17 +6961,14 @@ static int __init scsi_debug_init(void)
 		    sdebug_unmap_granularity <=
 		    sdebug_unmap_alignment) {
 			pr_err("ERR: unmap_granularity <= unmap_alignment\n");
-			ret = -EINVAL;
-			goto free_q_arr;
+			return -EINVAL;
 		}
 	}
 	xa_init_flags(per_store_ap, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
 	if (want_store) {
 		idx = sdebug_add_store();
-		if (idx < 0) {
-			ret = idx;
-			goto free_q_arr;
-		}
+		if (idx < 0)
+			return idx;
 	}
 
 	pseudo_primary = root_device_register("pseudo_0");
@@ -7108,6 +6991,12 @@ static int __init scsi_debug_init(void)
 	hosts_to_add = sdebug_add_host;
 	sdebug_add_host = 0;
 
+	queued_cmd_cache = KMEM_CACHE(sdebug_queued_cmd, SLAB_HWCACHE_ALIGN);
+	if (!queued_cmd_cache) {
+		ret = -ENOMEM;
+		goto driver_unreg;
+	}
+
 	for (k = 0; k < hosts_to_add; k++) {
 		if (want_store && k == 0) {
 			ret = sdebug_add_host_helper(idx);
@@ -7130,14 +7019,14 @@ static int __init scsi_debug_init(void)
 
 	return 0;
 
+driver_unreg:
+	driver_unregister(&sdebug_driverfs_driver);
 bus_unreg:
 	bus_unregister(&pseudo_lld_bus);
 dev_unreg:
 	root_device_unregister(pseudo_primary);
 free_vm:
 	sdebug_erase_store(idx, NULL);
-free_q_arr:
-	kfree(sdebug_q_arr);
 	return ret;
 }
 
@@ -7145,17 +7034,15 @@ static void __exit scsi_debug_exit(void)
 {
 	int k = sdebug_num_hosts;
 
-	stop_all_queued();
 	for (; k; k--)
 		sdebug_do_remove_host(true);
-	free_all_queued();
+	kmem_cache_destroy(queued_cmd_cache);
 	driver_unregister(&sdebug_driverfs_driver);
 	bus_unregister(&pseudo_lld_bus);
 	root_device_unregister(pseudo_primary);
 
 	sdebug_erase_all_stores(false);
 	xa_destroy(per_store_ap);
-	kfree(sdebug_q_arr);
 }
 
 device_initcall(scsi_debug_init);
@@ -7165,7 +7052,7 @@ static void sdebug_release_adapter(struct device *dev)
 {
 	struct sdebug_host_info *sdbg_host;
 
-	sdbg_host = to_sdebug_host(dev);
+	sdbg_host = dev_to_sdebug_host(dev);
 	kfree(sdbg_host);
 }
 
@@ -7311,9 +7198,9 @@ static int sdebug_add_host_helper(int per_host_idx)
 			goto clean;
 	}
 
-	spin_lock(&sdebug_host_list_lock);
+	mutex_lock(&sdebug_host_list_mutex);
 	list_add_tail(&sdbg_host->host_list, &sdebug_host_list);
-	spin_unlock(&sdebug_host_list_lock);
+	mutex_unlock(&sdebug_host_list_mutex);
 
 	sdbg_host->dev.bus = &pseudo_lld_bus;
 	sdbg_host->dev.parent = pseudo_primary;
@@ -7322,9 +7209,9 @@ static int sdebug_add_host_helper(int per_host_idx)
 
 	error = device_register(&sdbg_host->dev);
 	if (error) {
-		spin_lock(&sdebug_host_list_lock);
+		mutex_lock(&sdebug_host_list_mutex);
 		list_del(&sdbg_host->host_list);
-		spin_unlock(&sdebug_host_list_lock);
+		mutex_unlock(&sdebug_host_list_mutex);
 		goto clean;
 	}
 
@@ -7364,7 +7251,7 @@ static void sdebug_do_remove_host(bool the_end)
 	struct sdebug_host_info *sdbg_host = NULL;
 	struct sdebug_host_info *sdbg_host2;
 
-	spin_lock(&sdebug_host_list_lock);
+	mutex_lock(&sdebug_host_list_mutex);
 	if (!list_empty(&sdebug_host_list)) {
 		sdbg_host = list_entry(sdebug_host_list.prev,
 				       struct sdebug_host_info, host_list);
@@ -7389,7 +7276,7 @@ static void sdebug_do_remove_host(bool the_end)
 	}
 	if (sdbg_host)
 		list_del(&sdbg_host->host_list);
-	spin_unlock(&sdebug_host_list_lock);
+	mutex_unlock(&sdebug_host_list_mutex);
 
 	if (!sdbg_host)
 		return;
@@ -7400,16 +7287,13 @@ static void sdebug_do_remove_host(bool the_end)
 
 static int sdebug_change_qdepth(struct scsi_device *sdev, int qdepth)
 {
-	int num_in_q = 0;
-	struct sdebug_dev_info *devip;
+	struct sdebug_dev_info *devip = sdev->hostdata;
 
-	block_unblock_all_queues(true);
-	devip = (struct sdebug_dev_info *)sdev->hostdata;
-	if (NULL == devip) {
-		block_unblock_all_queues(false);
+	if (!devip)
 		return	-ENODEV;
-	}
-	num_in_q = atomic_read(&devip->num_in_q);
+
+	mutex_lock(&sdebug_host_list_mutex);
+	block_unblock_all_queues(true);
 
 	if (qdepth > SDEBUG_CANQUEUE) {
 		qdepth = SDEBUG_CANQUEUE;
@@ -7421,11 +7305,12 @@ static int sdebug_change_qdepth(struct scsi_device *sdev, int qdepth)
 	if (qdepth != sdev->queue_depth)
 		scsi_change_queue_depth(sdev, qdepth);
 
-	if (SDEBUG_OPT_Q_NOISE & sdebug_opts) {
-		sdev_printk(KERN_INFO, sdev, "%s: qdepth=%d, num_in_q=%d\n",
-			    __func__, qdepth, num_in_q);
-	}
 	block_unblock_all_queues(false);
+	mutex_unlock(&sdebug_host_list_mutex);
+
+	if (SDEBUG_OPT_Q_NOISE & sdebug_opts)
+		sdev_printk(KERN_INFO, sdev, "%s: qdepth=%d\n", __func__, qdepth);
+
 	return sdev->queue_depth;
 }
 
@@ -7515,94 +7400,82 @@ static void sdebug_map_queues(struct Scsi_Host *shost)
 	}
 }
 
-static int sdebug_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
+struct sdebug_blk_mq_poll_data {
+	unsigned int queue_num;
+	int *num_entries;
+};
+
+/*
+ * We don't handle aborted commands here, but it does not seem possible to have
+ * aborted polled commands from schedule_resp()
+ */
+static bool sdebug_blk_mq_poll_iter(struct request *rq, void *opaque)
 {
-	bool first;
-	bool retiring = false;
-	int num_entries = 0;
-	unsigned int qc_idx = 0;
-	unsigned long iflags;
-	ktime_t kt_from_boot = ktime_get_boottime();
-	struct sdebug_queue *sqp;
-	struct sdebug_queued_cmd *sqcp;
-	struct scsi_cmnd *scp;
-	struct sdebug_dev_info *devip;
+	struct sdebug_blk_mq_poll_data *data = opaque;
+	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
+	struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmd);
 	struct sdebug_defer *sd_dp;
+	u32 unique_tag = blk_mq_unique_tag(rq);
+	u16 hwq = blk_mq_unique_tag_to_hwq(unique_tag);
+	struct sdebug_queued_cmd *sqcp;
+	unsigned long flags;
+	int queue_num = data->queue_num;
+	ktime_t time;
 
-	sqp = sdebug_q_arr + queue_num;
+	/* We're only interested in one queue for this iteration */
+	if (hwq != queue_num)
+		return true;
 
-	spin_lock_irqsave(&sqp->qc_lock, iflags);
+	/* Subsequent checks would fail if this failed, but check anyway */
+	if (!test_bit(SCMD_STATE_INFLIGHT, &cmd->state))
+		return true;
 
-	qc_idx = find_first_bit(sqp->in_use_bm, sdebug_max_queue);
-	if (qc_idx >= sdebug_max_queue)
-		goto unlock;
+	time = ktime_get_boottime();
 
-	for (first = true; first || qc_idx + 1 < sdebug_max_queue; )   {
-		if (first) {
-			first = false;
-			if (!test_bit(qc_idx, sqp->in_use_bm))
-				continue;
-		} else {
-			qc_idx = find_next_bit(sqp->in_use_bm, sdebug_max_queue, qc_idx + 1);
-		}
-		if (qc_idx >= sdebug_max_queue)
-			break;
+	spin_lock_irqsave(&sdsc->lock, flags);
+	sqcp = TO_QUEUED_CMD(cmd);
+	if (!sqcp) {
+		spin_unlock_irqrestore(&sdsc->lock, flags);
+		return true;
+	}
 
-		sqcp = &sqp->qc_arr[qc_idx];
-		sd_dp = sqcp->sd_dp;
-		if (unlikely(!sd_dp))
-			continue;
-		scp = sqcp->a_cmnd;
-		if (unlikely(scp == NULL)) {
-			pr_err("scp is NULL, queue_num=%d, qc_idx=%u from %s\n",
-			       queue_num, qc_idx, __func__);
-			break;
-		}
-		if (READ_ONCE(sd_dp->defer_t) == SDEB_DEFER_POLL) {
-			if (kt_from_boot < sd_dp->cmpl_ts)
-				continue;
+	sd_dp = &sqcp->sd_dp;
+	if (READ_ONCE(sd_dp->defer_t) != SDEB_DEFER_POLL) {
+		spin_unlock_irqrestore(&sdsc->lock, flags);
+		return true;
+	}
 
-		} else		/* ignoring non REQ_POLLED requests */
-			continue;
-		devip = (struct sdebug_dev_info *)scp->device->hostdata;
-		if (likely(devip))
-			atomic_dec(&devip->num_in_q);
-		else
-			pr_err("devip=NULL from %s\n", __func__);
-		if (unlikely(atomic_read(&retired_max_queue) > 0))
-			retiring = true;
-
-		sqcp->a_cmnd = NULL;
-		if (unlikely(!test_and_clear_bit(qc_idx, sqp->in_use_bm))) {
-			pr_err("Unexpected completion sqp %p queue_num=%d qc_idx=%u from %s\n",
-				sqp, queue_num, qc_idx, __func__);
-			break;
-		}
-		if (unlikely(retiring)) {	/* user has reduced max_queue */
-			int k, retval;
+	if (time < sd_dp->cmpl_ts) {
+		spin_unlock_irqrestore(&sdsc->lock, flags);
+		return true;
+	}
 
-			retval = atomic_read(&retired_max_queue);
-			if (qc_idx >= retval) {
-				pr_err("index %d too large\n", retval);
-				break;
-			}
-			k = find_last_bit(sqp->in_use_bm, retval);
-			if ((k < sdebug_max_queue) || (k == retval))
-				atomic_set(&retired_max_queue, 0);
-			else
-				atomic_set(&retired_max_queue, k + 1);
-		}
-		WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_NONE);
-		spin_unlock_irqrestore(&sqp->qc_lock, iflags);
-		scsi_done(scp); /* callback to mid level */
-		num_entries++;
-		spin_lock_irqsave(&sqp->qc_lock, iflags);
-		if (find_first_bit(sqp->in_use_bm, sdebug_max_queue) >= sdebug_max_queue)
-			break;
+	ASSIGN_QUEUED_CMD(cmd, NULL);
+	spin_unlock_irqrestore(&sdsc->lock, flags);
+
+	if (sdebug_statistics) {
+		atomic_inc(&sdebug_completions);
+		if (raw_smp_processor_id() != sd_dp->issuing_cpu)
+			atomic_inc(&sdebug_miss_cpus);
 	}
 
-unlock:
-	spin_unlock_irqrestore(&sqp->qc_lock, iflags);
+	sdebug_free_queued_cmd(sqcp);
+
+	scsi_done(cmd); /* callback to mid level */
+	(*data->num_entries)++;
+	return true;
+}
+
+static int sdebug_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
+{
+	int num_entries = 0;
+	struct sdebug_blk_mq_poll_data data = {
+		.queue_num = queue_num,
+		.num_entries = &num_entries,
+	};
+
+	blk_mq_tagset_busy_iter(&shost->tag_set, sdebug_blk_mq_poll_iter,
+				&data);
 
 	if (num_entries > 0)
 		atomic_add(num_entries, &sdeb_mq_poll_count);
@@ -7776,6 +7649,16 @@ err_out:
 	return schedule_resp(scp, NULL, DID_NO_CONNECT << 16, NULL, 0, 0);
 }
 
+static int sdebug_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
+{
+	struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmd);
+
+	spin_lock_init(&sdsc->lock);
+
+	return 0;
+}
+
+
 static struct scsi_host_template sdebug_driver_template = {
 	.show_info =		scsi_debug_show_info,
 	.write_info =		scsi_debug_write_info,
@@ -7803,6 +7686,8 @@ static struct scsi_host_template sdebug_driver_template = {
 	.max_segment_size =	-1U,
 	.module =		THIS_MODULE,
 	.track_queue_depth =	1,
+	.cmd_size = sizeof(struct sdebug_scsi_cmd),
+	.init_cmd_priv = sdebug_init_cmd_priv,
 };
 
 static int sdebug_driver_probe(struct device *dev)
@@ -7812,14 +7697,14 @@ static int sdebug_driver_probe(struct device *dev)
 	struct Scsi_Host *hpnt;
 	int hprot;
 
-	sdbg_host = to_sdebug_host(dev);
+	sdbg_host = dev_to_sdebug_host(dev);
 
 	sdebug_driver_template.can_queue = sdebug_max_queue;
 	sdebug_driver_template.cmd_per_lun = sdebug_max_queue;
 	if (!sdebug_clustering)
 		sdebug_driver_template.dma_boundary = PAGE_SIZE - 1;
 
-	hpnt = scsi_host_alloc(&sdebug_driver_template, sizeof(sdbg_host));
+	hpnt = scsi_host_alloc(&sdebug_driver_template, 0);
 	if (NULL == hpnt) {
 		pr_err("scsi_host_alloc failed\n");
 		error = -ENODEV;
@@ -7862,7 +7747,6 @@ static int sdebug_driver_probe(struct device *dev)
 		hpnt->nr_maps = 3;
 
 	sdbg_host->shost = hpnt;
-	*((struct sdebug_host_info **)hpnt->hostdata) = sdbg_host;
 	if ((hpnt->this_id >= 0) && (sdebug_num_tgts > hpnt->this_id))
 		hpnt->max_id = sdebug_num_tgts + 1;
 	else
@@ -7936,7 +7820,7 @@ static void sdebug_driver_remove(struct device *dev)
 	struct sdebug_host_info *sdbg_host;
 	struct sdebug_dev_info *sdbg_devinfo, *tmp;
 
-	sdbg_host = to_sdebug_host(dev);
+	sdbg_host = dev_to_sdebug_host(dev);
 
 	scsi_remove_host(sdbg_host->shost);
 
@@ -7950,15 +7834,8 @@ static void sdebug_driver_remove(struct device *dev)
 	scsi_host_put(sdbg_host->shost);
 }
 
-static int pseudo_lld_bus_match(struct device *dev,
-				struct device_driver *dev_driver)
-{
-	return 1;
-}
-
 static struct bus_type pseudo_lld_bus = {
 	.name = "pseudo",
-	.match = pseudo_lld_bus_match,
 	.probe = sdebug_driver_probe,
 	.remove = sdebug_driver_remove,
 	.drv_groups = sdebug_drv_groups,
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index 2aa2c2aee6e7..3ec8bfd4090f 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -58,7 +58,7 @@
 #define HOST_RESET_SETTLE_TIME  (10)
 
 static int scsi_eh_try_stu(struct scsi_cmnd *scmd);
-static enum scsi_disposition scsi_try_to_abort_cmd(struct scsi_host_template *,
+static enum scsi_disposition scsi_try_to_abort_cmd(const struct scsi_host_template *,
 						   struct scsi_cmnd *);
 
 void scsi_eh_wakeup(struct Scsi_Host *shost)
@@ -699,7 +699,7 @@ EXPORT_SYMBOL_GPL(scsi_check_sense);
 
 static void scsi_handle_queue_ramp_up(struct scsi_device *sdev)
 {
-	struct scsi_host_template *sht = sdev->host->hostt;
+	const struct scsi_host_template *sht = sdev->host->hostt;
 	struct scsi_device *tmp_sdev;
 
 	if (!sht->track_queue_depth ||
@@ -731,7 +731,7 @@ static void scsi_handle_queue_ramp_up(struct scsi_device *sdev)
 
 static void scsi_handle_queue_full(struct scsi_device *sdev)
 {
-	struct scsi_host_template *sht = sdev->host->hostt;
+	const struct scsi_host_template *sht = sdev->host->hostt;
 	struct scsi_device *tmp_sdev;
 
 	if (!sht->track_queue_depth)
@@ -840,7 +840,7 @@ static enum scsi_disposition scsi_try_host_reset(struct scsi_cmnd *scmd)
 	unsigned long flags;
 	enum scsi_disposition rtn;
 	struct Scsi_Host *host = scmd->device->host;
-	struct scsi_host_template *hostt = host->hostt;
+	const struct scsi_host_template *hostt = host->hostt;
 
 	SCSI_LOG_ERROR_RECOVERY(3,
 		shost_printk(KERN_INFO, host, "Snd Host RST\n"));
@@ -870,7 +870,7 @@ static enum scsi_disposition scsi_try_bus_reset(struct scsi_cmnd *scmd)
 	unsigned long flags;
 	enum scsi_disposition rtn;
 	struct Scsi_Host *host = scmd->device->host;
-	struct scsi_host_template *hostt = host->hostt;
+	const struct scsi_host_template *hostt = host->hostt;
 
 	SCSI_LOG_ERROR_RECOVERY(3, scmd_printk(KERN_INFO, scmd,
 		"%s: Snd Bus RST\n", __func__));
@@ -912,7 +912,7 @@ static enum scsi_disposition scsi_try_target_reset(struct scsi_cmnd *scmd)
 	unsigned long flags;
 	enum scsi_disposition rtn;
 	struct Scsi_Host *host = scmd->device->host;
-	struct scsi_host_template *hostt = host->hostt;
+	const struct scsi_host_template *hostt = host->hostt;
 
 	if (!hostt->eh_target_reset_handler)
 		return FAILED;
@@ -941,7 +941,7 @@ static enum scsi_disposition scsi_try_target_reset(struct scsi_cmnd *scmd)
 static enum scsi_disposition scsi_try_bus_device_reset(struct scsi_cmnd *scmd)
 {
 	enum scsi_disposition rtn;
-	struct scsi_host_template *hostt = scmd->device->host->hostt;
+	const struct scsi_host_template *hostt = scmd->device->host->hostt;
 
 	if (!hostt->eh_device_reset_handler)
 		return FAILED;
@@ -970,7 +970,7 @@ static enum scsi_disposition scsi_try_bus_device_reset(struct scsi_cmnd *scmd)
  *    link down on FibreChannel)
  */
 static enum scsi_disposition
-scsi_try_to_abort_cmd(struct scsi_host_template *hostt, struct scsi_cmnd *scmd)
+scsi_try_to_abort_cmd(const struct scsi_host_template *hostt, struct scsi_cmnd *scmd)
 {
 	if (!hostt->eh_abort_handler)
 		return FAILED;
diff --git a/drivers/scsi/scsi_sysctl.c b/drivers/scsi/scsi_sysctl.c
index 7259704a7f52..7f0914ea168f 100644
--- a/drivers/scsi/scsi_sysctl.c
+++ b/drivers/scsi/scsi_sysctl.c
@@ -21,25 +21,11 @@ static struct ctl_table scsi_table[] = {
 	{ }
 };
 
-static struct ctl_table scsi_dir_table[] = {
-	{ .procname	= "scsi",
-	  .mode		= 0555,
-	  .child	= scsi_table },
-	{ }
-};
-
-static struct ctl_table scsi_root_table[] = {
-	{ .procname	= "dev",
-	  .mode		= 0555,
-	  .child	= scsi_dir_table },
-	{ }
-};
-
 static struct ctl_table_header *scsi_table_header;
 
 int __init scsi_init_sysctl(void)
 {
-	scsi_table_header = register_sysctl_table(scsi_root_table);
+	scsi_table_header = register_sysctl("dev/scsi", scsi_table);
 	if (!scsi_table_header)
 		return -ENOMEM;
 	return 0;
diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
index ee28f73af4d4..603e8fcfcb8a 100644
--- a/drivers/scsi/scsi_sysfs.c
+++ b/drivers/scsi/scsi_sysfs.c
@@ -296,7 +296,7 @@ store_host_reset(struct device *dev, struct device_attribute *attr,
 		const char *buf, size_t count)
 {
 	struct Scsi_Host *shost = class_to_shost(dev);
-	struct scsi_host_template *sht = shost->hostt;
+	const struct scsi_host_template *sht = shost->hostt;
 	int ret = -EINVAL;
 	int type;
 
@@ -1025,7 +1025,7 @@ sdev_store_queue_depth(struct device *dev, struct device_attribute *attr,
 {
 	int depth, retval;
 	struct scsi_device *sdev = to_scsi_device(dev);
-	struct scsi_host_template *sht = sdev->host->hostt;
+	const struct scsi_host_template *sht = sdev->host->hostt;
 
 	if (!sht->change_queue_depth)
 		return -EINVAL;
@@ -1606,7 +1606,7 @@ void scsi_sysfs_device_initialize(struct scsi_device *sdev)
 {
 	unsigned long flags;
 	struct Scsi_Host *shost = sdev->host;
-	struct scsi_host_template *hostt = shost->hostt;
+	const struct scsi_host_template *hostt = shost->hostt;
 	struct scsi_target  *starget = sdev->sdev_target;
 
 	device_initialize(&sdev->sdev_gendev);
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index f12e9467ebb4..64ff2629eaf9 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -905,7 +905,7 @@ fc_host_fpin_rcv(struct Scsi_Host *shost, u32 fpin_len, char *fpin_buf,
 {
 	struct fc_els_fpin *fpin = (struct fc_els_fpin *)fpin_buf;
 	struct fc_tlv_desc *tlv;
-	u32 desc_cnt = 0, bytes_remain;
+	u32 bytes_remain;
 	u32 dtag;
 	enum fc_host_event_code event_code =
 		event_acknowledge ? FCH_EVT_LINK_FPIN_ACK : FCH_EVT_LINK_FPIN;
@@ -932,7 +932,6 @@ fc_host_fpin_rcv(struct Scsi_Host *shost, u32 fpin_len, char *fpin_buf,
 			fc_fpin_congn_stats_update(shost, tlv);
 		}
 
-		desc_cnt++;
 		bytes_remain -= FC_TLV_DESC_SZ_FROM_LENGTH(tlv);
 		tlv = fc_tlv_next_desc(tlv);
 	}
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index 4bb87043e6db..1624d528aa1f 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -587,7 +587,6 @@ ATTRIBUTE_GROUPS(sd_disk);
 
 static struct class sd_disk_class = {
 	.name		= "scsi_disk",
-	.owner		= THIS_MODULE,
 	.dev_release	= scsi_disk_release,
 	.dev_groups	= sd_disk_groups,
 };
diff --git a/drivers/scsi/ses.c b/drivers/scsi/ses.c
index b54f2c6c08c3..d7d0c35c58b8 100644
--- a/drivers/scsi/ses.c
+++ b/drivers/scsi/ses.c
@@ -662,8 +662,7 @@ static void ses_match_to_enclosure(struct enclosure_device *edev,
 	}
 }
 
-static int ses_intf_add(struct device *cdev,
-			struct class_interface *intf)
+static int ses_intf_add(struct device *cdev)
 {
 	struct scsi_device *sdev = to_scsi_device(cdev->parent);
 	struct scsi_device *tmp_sdev;
@@ -865,8 +864,7 @@ static void ses_intf_remove_enclosure(struct scsi_device *sdev)
 	enclosure_unregister(edev);
 }
 
-static void ses_intf_remove(struct device *cdev,
-			    struct class_interface *intf)
+static void ses_intf_remove(struct device *cdev)
 {
 	struct scsi_device *sdev = to_scsi_device(cdev->parent);
 
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index a91049213203..037f8c98a6d3 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -96,8 +96,8 @@ static int scatter_elem_sz_prev = SG_SCATTER_SZ;
 
 #define SG_SECTOR_SZ 512
 
-static int sg_add_device(struct device *, struct class_interface *);
-static void sg_remove_device(struct device *, struct class_interface *);
+static int sg_add_device(struct device *);
+static void sg_remove_device(struct device *);
 
 static DEFINE_IDR(sg_index_idr);
 static DEFINE_RWLOCK(sg_index_lock);	/* Also used to lock
@@ -1488,7 +1488,7 @@ out_unlock:
 }
 
 static int
-sg_add_device(struct device *cl_dev, struct class_interface *cl_intf)
+sg_add_device(struct device *cl_dev)
 {
 	struct scsi_device *scsidp = to_scsi_device(cl_dev->parent);
 	Sg_device *sdp = NULL;
@@ -1578,7 +1578,7 @@ sg_device_destroy(struct kref *kref)
 }
 
 static void
-sg_remove_device(struct device *cl_dev, struct class_interface *cl_intf)
+sg_remove_device(struct device *cl_dev)
 {
 	struct scsi_device *scsidp = to_scsi_device(cl_dev->parent);
 	Sg_device *sdp = dev_get_drvdata(cl_dev);
@@ -1677,7 +1677,7 @@ init_sg(void)
 				    SG_MAX_DEVS, "sg");
 	if (rc)
 		return rc;
-        sg_sysfs_class = class_create(THIS_MODULE, "scsi_generic");
+        sg_sysfs_class = class_create("scsi_generic");
         if ( IS_ERR(sg_sysfs_class) ) {
 		rc = PTR_ERR(sg_sysfs_class);
 		goto err_out;
diff --git a/drivers/scsi/sgiwd93.c b/drivers/scsi/sgiwd93.c
index 57d5dff62f63..88e2b5eb9caa 100644
--- a/drivers/scsi/sgiwd93.c
+++ b/drivers/scsi/sgiwd93.c
@@ -204,7 +204,7 @@ static inline void init_hpc_chain(struct ip22_hostdata *hdata)
  * arguments not with pointers.  So this is going to blow up beautyfully
  * on 64-bit systems with memory outside the compat address spaces.
  */
-static struct scsi_host_template sgiwd93_template = {
+static const struct scsi_host_template sgiwd93_template = {
 	.module			= THIS_MODULE,
 	.proc_name		= "SGIWD93",
 	.name			= "SGI WD93",
diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
index 49a8f91810b6..03de97cd72c2 100644
--- a/drivers/scsi/smartpqi/smartpqi_init.c
+++ b/drivers/scsi/smartpqi/smartpqi_init.c
@@ -7403,7 +7403,7 @@ static struct attribute *pqi_sdev_attrs[] = {
 
 ATTRIBUTE_GROUPS(pqi_sdev);
 
-static struct scsi_host_template pqi_driver_template = {
+static const struct scsi_host_template pqi_driver_template = {
 	.module = THIS_MODULE,
 	.name = DRIVER_NAME_SHORT,
 	.proc_name = DRIVER_NAME_SHORT,
diff --git a/drivers/scsi/snic/snic_main.c b/drivers/scsi/snic/snic_main.c
index 174f7811fe50..cc824dcfe7da 100644
--- a/drivers/scsi/snic/snic_main.c
+++ b/drivers/scsi/snic/snic_main.c
@@ -100,7 +100,7 @@ snic_change_queue_depth(struct scsi_device *sdev, int qdepth)
 	return sdev->queue_depth;
 }
 
-static struct scsi_host_template snic_host_template = {
+static const struct scsi_host_template snic_host_template = {
 	.module = THIS_MODULE,
 	.name = SNIC_DRV_NAME,
 	.queuecommand = snic_queuecommand,
diff --git a/drivers/scsi/snic/snic_scsi.c b/drivers/scsi/snic/snic_scsi.c
index 961af6fc21bc..c50ede326cc4 100644
--- a/drivers/scsi/snic/snic_scsi.c
+++ b/drivers/scsi/snic/snic_scsi.c
@@ -487,7 +487,6 @@ snic_process_icmnd_cmpl_status(struct snic *snic,
 			       struct scsi_cmnd *sc)
 {
 	u8 scsi_stat = icmnd_cmpl->scsi_status;
-	u64 xfer_len = 0;
 	int ret = 0;
 
 	/* Mark the IO as complete */
@@ -496,15 +495,11 @@ snic_process_icmnd_cmpl_status(struct snic *snic,
 	if (likely(cmpl_stat == SNIC_STAT_IO_SUCCESS)) {
 		sc->result = (DID_OK << 16) | scsi_stat;
 
-		xfer_len = scsi_bufflen(sc);
-
 		/* Update SCSI Cmd with resid value */
 		scsi_set_resid(sc, le32_to_cpu(icmnd_cmpl->resid));
 
-		if (icmnd_cmpl->flags & SNIC_ICMND_CMPL_UNDR_RUN) {
-			xfer_len -= le32_to_cpu(icmnd_cmpl->resid);
+		if (icmnd_cmpl->flags & SNIC_ICMND_CMPL_UNDR_RUN)
 			atomic64_inc(&snic->s_stats.misc.io_under_run);
-		}
 
 		if (icmnd_cmpl->scsi_status == SAM_STAT_TASK_SET_FULL)
 			atomic64_inc(&snic->s_stats.misc.qfull);
diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c
index 9e51dcd30bfd..12869e6d4ebd 100644
--- a/drivers/scsi/sr.c
+++ b/drivers/scsi/sr.c
@@ -590,20 +590,15 @@ static int sr_open(struct cdrom_device_info *cdi, int purpose)
 {
 	struct scsi_cd *cd = cdi->handle;
 	struct scsi_device *sdev = cd->device;
-	int retval;
 
 	/*
 	 * If the device is in error recovery, wait until it is done.
 	 * If the device is offline, then disallow any access to it.
 	 */
-	retval = -ENXIO;
 	if (!scsi_block_when_processing_errors(sdev))
-		goto error_out;
+		return -ENXIO;
 
 	return 0;
-
-error_out:
-	return retval;	
 }
 
 static void sr_release(struct cdrom_device_info *cdi)
diff --git a/drivers/scsi/stex.c b/drivers/scsi/stex.c
index 8def242675ef..5b230e149c3d 100644
--- a/drivers/scsi/stex.c
+++ b/drivers/scsi/stex.c
@@ -1472,7 +1472,7 @@ static int stex_biosparam(struct scsi_device *sdev,
 	return 0;
 }
 
-static struct scsi_host_template driver_template = {
+static const struct scsi_host_template driver_template = {
 	.module				= THIS_MODULE,
 	.name				= DRV_NAME,
 	.proc_name			= DRV_NAME,
diff --git a/drivers/scsi/sun3x_esp.c b/drivers/scsi/sun3x_esp.c
index d3489ac7ab28..30f67cbf4a7a 100644
--- a/drivers/scsi/sun3x_esp.c
+++ b/drivers/scsi/sun3x_esp.c
@@ -169,7 +169,7 @@ static const struct esp_driver_ops sun3x_esp_ops = {
 
 static int esp_sun3x_probe(struct platform_device *dev)
 {
-	struct scsi_host_template *tpnt = &scsi_esp_template;
+	const struct scsi_host_template *tpnt = &scsi_esp_template;
 	struct Scsi_Host *host;
 	struct esp *esp;
 	struct resource *res;
diff --git a/drivers/scsi/sun_esp.c b/drivers/scsi/sun_esp.c
index 5dc38d35745b..d06e933191a2 100644
--- a/drivers/scsi/sun_esp.c
+++ b/drivers/scsi/sun_esp.c
@@ -146,7 +146,7 @@ static void esp_get_differential(struct esp *esp)
 	struct device_node *dp;
 
 	dp = op->dev.of_node;
-	if (of_find_property(dp, "differential", NULL))
+	if (of_property_read_bool(dp, "differential"))
 		esp->flags |= ESP_FLAG_DIFFERENTIAL;
 	else
 		esp->flags &= ~ESP_FLAG_DIFFERENTIAL;
@@ -451,7 +451,7 @@ static const struct esp_driver_ops sbus_esp_ops = {
 static int esp_sbus_probe_one(struct platform_device *op,
 			      struct platform_device *espdma, int hme)
 {
-	struct scsi_host_template *tpnt = &scsi_esp_template;
+	const struct scsi_host_template *tpnt = &scsi_esp_template;
 	struct Scsi_Host *host;
 	struct esp *esp;
 	int err;
diff --git a/drivers/scsi/sym53c8xx_2/sym_glue.c b/drivers/scsi/sym53c8xx_2/sym_glue.c
index 2e2852bd5860..ee36a9c15d9c 100644
--- a/drivers/scsi/sym53c8xx_2/sym_glue.c
+++ b/drivers/scsi/sym53c8xx_2/sym_glue.c
@@ -1224,7 +1224,7 @@ static void sym_free_resources(struct sym_hcb *np, struct pci_dev *pdev,
  *  If all is OK, install interrupt handling and
  *  start the timer daemon.
  */
-static struct Scsi_Host *sym_attach(struct scsi_host_template *tpnt, int unit,
+static struct Scsi_Host *sym_attach(const struct scsi_host_template *tpnt, int unit,
 				    struct sym_device *dev)
 {
 	struct sym_data *sym_data;
@@ -1625,7 +1625,7 @@ static int sym_detach(struct Scsi_Host *shost, struct pci_dev *pdev)
 /*
  * Driver host template.
  */
-static struct scsi_host_template sym2_template = {
+static const struct scsi_host_template sym2_template = {
 	.module			= THIS_MODULE,
 	.name			= "sym53c8xx",
 	.info			= sym53c8xx_info, 
diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
index c5558c45ab3a..58498da9869a 100644
--- a/drivers/scsi/virtio_scsi.c
+++ b/drivers/scsi/virtio_scsi.c
@@ -746,7 +746,7 @@ static enum scsi_timeout_action virtscsi_eh_timed_out(struct scsi_cmnd *scmnd)
 	return SCSI_EH_RESET_TIMER;
 }
 
-static struct scsi_host_template virtscsi_host_template = {
+static const struct scsi_host_template virtscsi_host_template = {
 	.module = THIS_MODULE,
 	.name = "Virtio SCSI HBA",
 	.proc_name = "virtio_scsi",
diff --git a/drivers/scsi/wd719x.c b/drivers/scsi/wd719x.c
index ff1b22077251..5a380eecfc75 100644
--- a/drivers/scsi/wd719x.c
+++ b/drivers/scsi/wd719x.c
@@ -878,7 +878,7 @@ fail_free_params:
 	return ret;
 }
 
-static struct scsi_host_template wd719x_template = {
+static const struct scsi_host_template wd719x_template = {
 	.module				= THIS_MODULE,
 	.name				= "Western Digital 719x",
 	.cmd_size			= sizeof(struct wd719x_scb),
diff --git a/drivers/scsi/xen-scsifront.c b/drivers/scsi/xen-scsifront.c
index 71a3bb83984c..caae61aa2afe 100644
--- a/drivers/scsi/xen-scsifront.c
+++ b/drivers/scsi/xen-scsifront.c
@@ -770,7 +770,7 @@ static void scsifront_sdev_destroy(struct scsi_device *sdev)
 	}
 }
 
-static struct scsi_host_template scsifront_sht = {
+static const struct scsi_host_template scsifront_sht = {
 	.module			= THIS_MODULE,
 	.name			= "Xen SCSI frontend driver",
 	.queuecommand		= scsifront_queuecommand,
diff --git a/drivers/scsi/zorro_esp.c b/drivers/scsi/zorro_esp.c
index 928c8adf5cb3..56cae22a4242 100644
--- a/drivers/scsi/zorro_esp.c
+++ b/drivers/scsi/zorro_esp.c
@@ -713,7 +713,7 @@ MODULE_DEVICE_TABLE(zorro, zorro_esp_zorro_tbl);
 static int zorro_esp_probe(struct zorro_dev *z,
 				       const struct zorro_device_id *ent)
 {
-	struct scsi_host_template *tpnt = &scsi_esp_template;
+	const struct scsi_host_template *tpnt = &scsi_esp_template;
 	struct Scsi_Host *host;
 	struct esp *esp;
 	const struct zorro_driver_data *zdd;
diff --git a/drivers/sh/intc/userimask.c b/drivers/sh/intc/userimask.c
index f9f043a3d90a..abe9091827cd 100644
--- a/drivers/sh/intc/userimask.c
+++ b/drivers/sh/intc/userimask.c
@@ -61,10 +61,18 @@ static DEVICE_ATTR(userimask, S_IRUSR | S_IWUSR,
 
 static int __init userimask_sysdev_init(void)
 {
+	struct device *dev_root;
+	int ret = 0;
+
 	if (unlikely(!uimask))
 		return -ENXIO;
 
-	return device_create_file(intc_subsys.dev_root, &dev_attr_userimask);
+	dev_root = bus_get_dev_root(&intc_subsys);
+	if (dev_root) {
+		ret = device_create_file(dev_root, &dev_attr_userimask);
+		put_device(dev_root);
+	}
+	return ret;
 }
 late_initcall(userimask_sysdev_init);
 
diff --git a/drivers/soc/amlogic/meson-gx-pwrc-vpu.c b/drivers/soc/amlogic/meson-gx-pwrc-vpu.c
index 312fd9afccb0..5d4f12800d93 100644
--- a/drivers/soc/amlogic/meson-gx-pwrc-vpu.c
+++ b/drivers/soc/amlogic/meson-gx-pwrc-vpu.c
@@ -308,11 +308,9 @@ static int meson_gx_pwrc_vpu_probe(struct platform_device *pdev)
 	}
 
 	rstc = devm_reset_control_array_get_exclusive(&pdev->dev);
-	if (IS_ERR(rstc)) {
-		if (PTR_ERR(rstc) != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "failed to get reset lines\n");
-		return PTR_ERR(rstc);
-	}
+	if (IS_ERR(rstc))
+		return dev_err_probe(&pdev->dev, PTR_ERR(rstc),
+				     "failed to get reset lines\n");
 
 	vpu_clk = devm_clk_get(&pdev->dev, "vpu");
 	if (IS_ERR(vpu_clk)) {
diff --git a/drivers/soc/amlogic/meson-gx-socinfo.c b/drivers/soc/amlogic/meson-gx-socinfo.c
index 165f7548401b..6abb730344ab 100644
--- a/drivers/soc/amlogic/meson-gx-socinfo.c
+++ b/drivers/soc/amlogic/meson-gx-socinfo.c
@@ -174,11 +174,6 @@ static int __init meson_gx_socinfo_init(void)
 		return -ENODEV;
 
 	soc_dev_attr->family = "Amlogic Meson";
-
-	np = of_find_node_by_path("/");
-	of_property_read_string(np, "model", &soc_dev_attr->machine);
-	of_node_put(np);
-
 	soc_dev_attr->revision = kasprintf(GFP_KERNEL, "%x:%x - %x:%x",
 					   socinfo_to_major(socinfo),
 					   socinfo_to_minor(socinfo),
diff --git a/drivers/soc/apple/apple-pmgr-pwrstate.c b/drivers/soc/apple/apple-pmgr-pwrstate.c
index a3e2bc1d2686..d62a776c89a1 100644
--- a/drivers/soc/apple/apple-pmgr-pwrstate.c
+++ b/drivers/soc/apple/apple-pmgr-pwrstate.c
@@ -322,6 +322,5 @@ static struct platform_driver apple_pmgr_ps_driver = {
 
 MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");
 MODULE_DESCRIPTION("PMGR power state driver for Apple SoCs");
-MODULE_LICENSE("GPL v2");
 
 module_platform_driver(apple_pmgr_ps_driver);
diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c
index 35ec35aa500d..d9f19dc99da5 100644
--- a/drivers/soc/apple/rtkit.c
+++ b/drivers/soc/apple/rtkit.c
@@ -55,7 +55,7 @@ enum {
 
 #define APPLE_RTKIT_BUFFER_REQUEST	1
 #define APPLE_RTKIT_BUFFER_REQUEST_SIZE GENMASK_ULL(51, 44)
-#define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK_ULL(41, 0)
+#define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK_ULL(43, 0)
 
 #define APPLE_RTKIT_SYSLOG_TYPE GENMASK_ULL(59, 52)
 
@@ -409,11 +409,17 @@ static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg)
 		rtk->syslog_n_entries, rtk->syslog_msg_size);
 }
 
+static bool should_crop_syslog_char(char c)
+{
+	return c == '\n' || c == '\r' || c == ' ' || c == '\0';
+}
+
 static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg)
 {
 	u8 idx = msg & 0xff;
 	char log_context[24];
 	size_t entry_size = 0x20 + rtk->syslog_msg_size;
+	int msglen;
 
 	if (!rtk->syslog_msg_buffer) {
 		dev_warn(
@@ -446,7 +452,13 @@ static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg)
 			   rtk->syslog_msg_size);
 
 	log_context[sizeof(log_context) - 1] = 0;
-	rtk->syslog_msg_buffer[rtk->syslog_msg_size - 1] = 0;
+
+	msglen = rtk->syslog_msg_size - 1;
+	while (msglen > 0 &&
+		   should_crop_syslog_char(rtk->syslog_msg_buffer[msglen - 1]))
+		msglen--;
+
+	rtk->syslog_msg_buffer[msglen] = 0;
 	dev_info(rtk->dev, "RTKit: syslog message: %s: %s\n", log_context,
 		 rtk->syslog_msg_buffer);
 
diff --git a/drivers/soc/bcm/bcm2835-power.c b/drivers/soc/bcm/bcm2835-power.c
index bf51f03f77d6..1a179d4e011c 100644
--- a/drivers/soc/bcm/bcm2835-power.c
+++ b/drivers/soc/bcm/bcm2835-power.c
@@ -711,4 +711,3 @@ module_platform_driver(bcm2835_power_driver);
 
 MODULE_AUTHOR("Eric Anholt <eric@anholt.net>");
 MODULE_DESCRIPTION("Driver for Broadcom BCM2835 PM power domains and reset");
-MODULE_LICENSE("GPL");
diff --git a/drivers/soc/bcm/brcmstb/Kconfig b/drivers/soc/bcm/brcmstb/Kconfig
index 38e476905d96..c68d0e5267c4 100644
--- a/drivers/soc/bcm/brcmstb/Kconfig
+++ b/drivers/soc/bcm/brcmstb/Kconfig
@@ -4,8 +4,6 @@ if SOC_BRCMSTB
 config BRCMSTB_PM
 	bool "Support suspend/resume for STB platforms"
 	default y
-	depends on PM
-	depends on ARCH_BRCMSTB || BMIPS_GENERIC
-	select ARM_CPU_SUSPEND if ARM
+	depends on PM && BMIPS_GENERIC
 
 endif # SOC_BRCMSTB
diff --git a/drivers/soc/bcm/brcmstb/biuctrl.c b/drivers/soc/bcm/brcmstb/biuctrl.c
index e1d7b4543248..364ddbe365c2 100644
--- a/drivers/soc/bcm/brcmstb/biuctrl.c
+++ b/drivers/soc/bcm/brcmstb/biuctrl.c
@@ -288,6 +288,10 @@ static int __init setup_hifcpubiuctrl_regs(struct device_node *np)
 	if (BRCM_ID(family_id) == 0x7260 && BRCM_REV(family_id) == 0)
 		cpubiuctrl_regs = b53_cpubiuctrl_no_wb_regs;
 out:
+	if (ret && cpubiuctrl_base) {
+		iounmap(cpubiuctrl_base);
+		cpubiuctrl_base = NULL;
+	}
 	return ret;
 }
 
diff --git a/drivers/soc/bcm/brcmstb/pm/Makefile b/drivers/soc/bcm/brcmstb/pm/Makefile
index f849cfa69446..9133a9ee0782 100644
--- a/drivers/soc/bcm/brcmstb/pm/Makefile
+++ b/drivers/soc/bcm/brcmstb/pm/Makefile
@@ -1,3 +1,2 @@
 # SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_ARM)		+= s2-arm.o pm-arm.o
 obj-$(CONFIG_BMIPS_GENERIC)	+= s2-mips.o s3-mips.o pm-mips.o
diff --git a/drivers/soc/bcm/brcmstb/pm/aon_defs.h b/drivers/soc/bcm/brcmstb/pm/aon_defs.h
deleted file mode 100644
index f695262ac930..000000000000
--- a/drivers/soc/bcm/brcmstb/pm/aon_defs.h
+++ /dev/null
@@ -1,105 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Always ON (AON) register interface between bootloader and Linux
- *
- * Copyright © 2014-2017 Broadcom
- */
-
-#ifndef __BRCMSTB_AON_DEFS_H__
-#define __BRCMSTB_AON_DEFS_H__
-
-#include <linux/compiler.h>
-
-/* Magic number in upper 16-bits */
-#define BRCMSTB_S3_MAGIC_MASK                   0xffff0000
-#define BRCMSTB_S3_MAGIC_SHORT                  0x5AFE0000
-
-enum {
-	/* Restore random key for AES memory verification (off = fixed key) */
-	S3_FLAG_LOAD_RANDKEY		= (1 << 0),
-
-	/* Scratch buffer page table is present */
-	S3_FLAG_SCRATCH_BUFFER_TABLE	= (1 << 1),
-
-	/* Skip all memory verification */
-	S3_FLAG_NO_MEM_VERIFY		= (1 << 2),
-
-	/*
-	 * Modification of this bit reserved for bootloader only.
-	 * 1=PSCI started Linux, 0=Direct jump to Linux.
-	 */
-	S3_FLAG_PSCI_BOOT		= (1 << 3),
-
-	/*
-	 * Modification of this bit reserved for bootloader only.
-	 * 1=64 bit boot, 0=32 bit boot.
-	 */
-	S3_FLAG_BOOTED64		= (1 << 4),
-};
-
-#define BRCMSTB_HASH_LEN			(128 / 8) /* 128-bit hash */
-
-#define AON_REG_MAGIC_FLAGS			0x00
-#define AON_REG_CONTROL_LOW			0x04
-#define AON_REG_CONTROL_HIGH			0x08
-#define AON_REG_S3_HASH				0x0c /* hash of S3 params */
-#define AON_REG_CONTROL_HASH_LEN		0x1c
-#define AON_REG_PANIC				0x20
-
-#define BRCMSTB_S3_MAGIC		0x5AFEB007
-#define BRCMSTB_PANIC_MAGIC		0x512E115E
-#define BOOTLOADER_SCRATCH_SIZE		64
-#define BRCMSTB_DTU_STATE_MAP_ENTRIES	(8*1024)
-#define BRCMSTB_DTU_CONFIG_ENTRIES	(512)
-#define BRCMSTB_DTU_COUNT		(2)
-
-#define IMAGE_DESCRIPTORS_BUFSIZE	(2 * 1024)
-#define S3_BOOTLOADER_RESERVED		(S3_FLAG_PSCI_BOOT | S3_FLAG_BOOTED64)
-
-struct brcmstb_bootloader_dtu_table {
-	uint32_t	dtu_state_map[BRCMSTB_DTU_STATE_MAP_ENTRIES];
-	uint32_t	dtu_config[BRCMSTB_DTU_CONFIG_ENTRIES];
-};
-
-/*
- * Bootloader utilizes a custom parameter block left in DRAM for handling S3
- * warm resume
- */
-struct brcmstb_s3_params {
-	/* scratch memory for bootloader */
-	uint8_t scratch[BOOTLOADER_SCRATCH_SIZE];
-
-	uint32_t magic; /* BRCMSTB_S3_MAGIC */
-	uint64_t reentry; /* PA */
-
-	/* descriptors */
-	uint32_t hash[BRCMSTB_HASH_LEN / 4];
-
-	/*
-	 * If 0, then ignore this parameter (there is only one set of
-	 *   descriptors)
-	 *
-	 * If non-0, then a second set of descriptors is stored at:
-	 *
-	 *   descriptors + desc_offset_2
-	 *
-	 * The MAC result of both descriptors is XOR'd and stored in @hash
-	 */
-	uint32_t desc_offset_2;
-
-	/*
-	 * (Physical) address of a brcmstb_bootloader_scratch_table, for
-	 * providing a large DRAM buffer to the bootloader
-	 */
-	uint64_t buffer_table;
-
-	uint32_t spare[70];
-
-	uint8_t descriptors[IMAGE_DESCRIPTORS_BUFSIZE];
-	/*
-	 * Must be last member of struct. See brcmstb_pm_s3_finish() for reason.
-	 */
-	struct brcmstb_bootloader_dtu_table dtu[BRCMSTB_DTU_COUNT];
-} __packed;
-
-#endif /* __BRCMSTB_AON_DEFS_H__ */
diff --git a/drivers/soc/bcm/brcmstb/pm/pm-arm.c b/drivers/soc/bcm/brcmstb/pm/pm-arm.c
deleted file mode 100644
index d681cd24c6e1..000000000000
--- a/drivers/soc/bcm/brcmstb/pm/pm-arm.c
+++ /dev/null
@@ -1,874 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * ARM-specific support for Broadcom STB S2/S3/S5 power management
- *
- * S2: clock gate CPUs and as many peripherals as possible
- * S3: power off all of the chip except the Always ON (AON) island; keep DDR is
- *     self-refresh
- * S5: (a.k.a. S3 cold boot) much like S3, except DDR is powered down, so we
- *     treat this mode like a soft power-off, with wakeup allowed from AON
- *
- * Copyright © 2014-2017 Broadcom
- */
-
-#define pr_fmt(fmt) "brcmstb-pm: " fmt
-
-#include <linux/bitops.h>
-#include <linux/compiler.h>
-#include <linux/delay.h>
-#include <linux/dma-mapping.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <linux/io.h>
-#include <linux/ioport.h>
-#include <linux/kconfig.h>
-#include <linux/kernel.h>
-#include <linux/memblock.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_address.h>
-#include <linux/panic_notifier.h>
-#include <linux/platform_device.h>
-#include <linux/pm.h>
-#include <linux/printk.h>
-#include <linux/proc_fs.h>
-#include <linux/sizes.h>
-#include <linux/slab.h>
-#include <linux/sort.h>
-#include <linux/suspend.h>
-#include <linux/types.h>
-#include <linux/uaccess.h>
-#include <linux/soc/brcmstb/brcmstb.h>
-
-#include <asm/fncpy.h>
-#include <asm/setup.h>
-#include <asm/suspend.h>
-
-#include "pm.h"
-#include "aon_defs.h"
-
-#define SHIMPHY_DDR_PAD_CNTRL		0x8c
-
-/* Method #0 */
-#define SHIMPHY_PAD_PLL_SEQUENCE	BIT(8)
-#define SHIMPHY_PAD_GATE_PLL_S3		BIT(9)
-
-/* Method #1 */
-#define PWRDWN_SEQ_NO_SEQUENCING	0
-#define PWRDWN_SEQ_HOLD_CHANNEL		1
-#define	PWRDWN_SEQ_RESET_PLL		2
-#define PWRDWN_SEQ_POWERDOWN_PLL	3
-
-#define SHIMPHY_PAD_S3_PWRDWN_SEQ_MASK	0x00f00000
-#define SHIMPHY_PAD_S3_PWRDWN_SEQ_SHIFT	20
-
-#define	DDR_FORCE_CKE_RST_N		BIT(3)
-#define	DDR_PHY_RST_N			BIT(2)
-#define	DDR_PHY_CKE			BIT(1)
-
-#define	DDR_PHY_NO_CHANNEL		0xffffffff
-
-#define MAX_NUM_MEMC			3
-
-struct brcmstb_memc {
-	void __iomem *ddr_phy_base;
-	void __iomem *ddr_shimphy_base;
-	void __iomem *ddr_ctrl;
-};
-
-struct brcmstb_pm_control {
-	void __iomem *aon_ctrl_base;
-	void __iomem *aon_sram;
-	struct brcmstb_memc memcs[MAX_NUM_MEMC];
-
-	void __iomem *boot_sram;
-	size_t boot_sram_len;
-
-	bool support_warm_boot;
-	size_t pll_status_offset;
-	int num_memc;
-
-	struct brcmstb_s3_params *s3_params;
-	dma_addr_t s3_params_pa;
-	int s3entry_method;
-	u32 warm_boot_offset;
-	u32 phy_a_standby_ctrl_offs;
-	u32 phy_b_standby_ctrl_offs;
-	bool needs_ddr_pad;
-	struct platform_device *pdev;
-};
-
-enum bsp_initiate_command {
-	BSP_CLOCK_STOP		= 0x00,
-	BSP_GEN_RANDOM_KEY	= 0x4A,
-	BSP_RESTORE_RANDOM_KEY	= 0x55,
-	BSP_GEN_FIXED_KEY	= 0x63,
-};
-
-#define PM_INITIATE		0x01
-#define PM_INITIATE_SUCCESS	0x00
-#define PM_INITIATE_FAIL	0xfe
-
-static struct brcmstb_pm_control ctrl;
-
-noinline int brcmstb_pm_s3_finish(void);
-
-static int (*brcmstb_pm_do_s2_sram)(void __iomem *aon_ctrl_base,
-		void __iomem *ddr_phy_pll_status);
-
-static int brcmstb_init_sram(struct device_node *dn)
-{
-	void __iomem *sram;
-	struct resource res;
-	int ret;
-
-	ret = of_address_to_resource(dn, 0, &res);
-	if (ret)
-		return ret;
-
-	/* Uncached, executable remapping of SRAM */
-	sram = __arm_ioremap_exec(res.start, resource_size(&res), false);
-	if (!sram)
-		return -ENOMEM;
-
-	ctrl.boot_sram = sram;
-	ctrl.boot_sram_len = resource_size(&res);
-
-	return 0;
-}
-
-static const struct of_device_id sram_dt_ids[] = {
-	{ .compatible = "mmio-sram" },
-	{ /* sentinel */ }
-};
-
-static int do_bsp_initiate_command(enum bsp_initiate_command cmd)
-{
-	void __iomem *base = ctrl.aon_ctrl_base;
-	int ret;
-	int timeo = 1000 * 1000; /* 1 second */
-
-	writel_relaxed(0, base + AON_CTRL_PM_INITIATE);
-	(void)readl_relaxed(base + AON_CTRL_PM_INITIATE);
-
-	/* Go! */
-	writel_relaxed((cmd << 1) | PM_INITIATE, base + AON_CTRL_PM_INITIATE);
-
-	/*
-	 * If firmware doesn't support the 'ack', then just assume it's done
-	 * after 10ms. Note that this only works for command 0, BSP_CLOCK_STOP
-	 */
-	if (of_machine_is_compatible("brcm,bcm74371a0")) {
-		(void)readl_relaxed(base + AON_CTRL_PM_INITIATE);
-		mdelay(10);
-		return 0;
-	}
-
-	for (;;) {
-		ret = readl_relaxed(base + AON_CTRL_PM_INITIATE);
-		if (!(ret & PM_INITIATE))
-			break;
-		if (timeo <= 0) {
-			pr_err("error: timeout waiting for BSP (%x)\n", ret);
-			break;
-		}
-		timeo -= 50;
-		udelay(50);
-	}
-
-	return (ret & 0xff) != PM_INITIATE_SUCCESS;
-}
-
-static int brcmstb_pm_handshake(void)
-{
-	void __iomem *base = ctrl.aon_ctrl_base;
-	u32 tmp;
-	int ret;
-
-	/* BSP power handshake, v1 */
-	tmp = readl_relaxed(base + AON_CTRL_HOST_MISC_CMDS);
-	tmp &= ~1UL;
-	writel_relaxed(tmp, base + AON_CTRL_HOST_MISC_CMDS);
-	(void)readl_relaxed(base + AON_CTRL_HOST_MISC_CMDS);
-
-	ret = do_bsp_initiate_command(BSP_CLOCK_STOP);
-	if (ret)
-		pr_err("BSP handshake failed\n");
-
-	/*
-	 * HACK: BSP may have internal race on the CLOCK_STOP command.
-	 * Avoid touching the BSP for a few milliseconds.
-	 */
-	mdelay(3);
-
-	return ret;
-}
-
-static inline void shimphy_set(u32 value, u32 mask)
-{
-	int i;
-
-	if (!ctrl.needs_ddr_pad)
-		return;
-
-	for (i = 0; i < ctrl.num_memc; i++) {
-		u32 tmp;
-
-		tmp = readl_relaxed(ctrl.memcs[i].ddr_shimphy_base +
-			SHIMPHY_DDR_PAD_CNTRL);
-		tmp = value | (tmp & mask);
-		writel_relaxed(tmp, ctrl.memcs[i].ddr_shimphy_base +
-			SHIMPHY_DDR_PAD_CNTRL);
-	}
-	wmb(); /* Complete sequence in order. */
-}
-
-static inline void ddr_ctrl_set(bool warmboot)
-{
-	int i;
-
-	for (i = 0; i < ctrl.num_memc; i++) {
-		u32 tmp;
-
-		tmp = readl_relaxed(ctrl.memcs[i].ddr_ctrl +
-				ctrl.warm_boot_offset);
-		if (warmboot)
-			tmp |= 1;
-		else
-			tmp &= ~1; /* Cold boot */
-		writel_relaxed(tmp, ctrl.memcs[i].ddr_ctrl +
-				ctrl.warm_boot_offset);
-	}
-	/* Complete sequence in order */
-	wmb();
-}
-
-static inline void s3entry_method0(void)
-{
-	shimphy_set(SHIMPHY_PAD_GATE_PLL_S3 | SHIMPHY_PAD_PLL_SEQUENCE,
-		    0xffffffff);
-}
-
-static inline void s3entry_method1(void)
-{
-	/*
-	 * S3 Entry Sequence
-	 * -----------------
-	 * Step 1: SHIMPHY_ADDR_CNTL_0_DDR_PAD_CNTRL [ S3_PWRDWN_SEQ ] = 3
-	 * Step 2: MEMC_DDR_0_WARM_BOOT [ WARM_BOOT ] = 1
-	 */
-	shimphy_set((PWRDWN_SEQ_POWERDOWN_PLL <<
-		    SHIMPHY_PAD_S3_PWRDWN_SEQ_SHIFT),
-		    ~SHIMPHY_PAD_S3_PWRDWN_SEQ_MASK);
-
-	ddr_ctrl_set(true);
-}
-
-static inline void s5entry_method1(void)
-{
-	int i;
-
-	/*
-	 * S5 Entry Sequence
-	 * -----------------
-	 * Step 1: SHIMPHY_ADDR_CNTL_0_DDR_PAD_CNTRL [ S3_PWRDWN_SEQ ] = 3
-	 * Step 2: MEMC_DDR_0_WARM_BOOT [ WARM_BOOT ] = 0
-	 * Step 3: DDR_PHY_CONTROL_REGS_[AB]_0_STANDBY_CONTROL[ CKE ] = 0
-	 *	   DDR_PHY_CONTROL_REGS_[AB]_0_STANDBY_CONTROL[ RST_N ] = 0
-	 */
-	shimphy_set((PWRDWN_SEQ_POWERDOWN_PLL <<
-		    SHIMPHY_PAD_S3_PWRDWN_SEQ_SHIFT),
-		    ~SHIMPHY_PAD_S3_PWRDWN_SEQ_MASK);
-
-	ddr_ctrl_set(false);
-
-	for (i = 0; i < ctrl.num_memc; i++) {
-		u32 tmp;
-
-		/* Step 3: Channel A (RST_N = CKE = 0) */
-		tmp = readl_relaxed(ctrl.memcs[i].ddr_phy_base +
-				  ctrl.phy_a_standby_ctrl_offs);
-		tmp &= ~(DDR_PHY_RST_N | DDR_PHY_RST_N);
-		writel_relaxed(tmp, ctrl.memcs[i].ddr_phy_base +
-			     ctrl.phy_a_standby_ctrl_offs);
-
-		/* Step 3: Channel B? */
-		if (ctrl.phy_b_standby_ctrl_offs != DDR_PHY_NO_CHANNEL) {
-			tmp = readl_relaxed(ctrl.memcs[i].ddr_phy_base +
-					  ctrl.phy_b_standby_ctrl_offs);
-			tmp &= ~(DDR_PHY_RST_N | DDR_PHY_RST_N);
-			writel_relaxed(tmp, ctrl.memcs[i].ddr_phy_base +
-				     ctrl.phy_b_standby_ctrl_offs);
-		}
-	}
-	/* Must complete */
-	wmb();
-}
-
-/*
- * Run a Power Management State Machine (PMSM) shutdown command and put the CPU
- * into a low-power mode
- */
-static void brcmstb_do_pmsm_power_down(unsigned long base_cmd, bool onewrite)
-{
-	void __iomem *base = ctrl.aon_ctrl_base;
-
-	if ((ctrl.s3entry_method == 1) && (base_cmd == PM_COLD_CONFIG))
-		s5entry_method1();
-
-	/* pm_start_pwrdn transition 0->1 */
-	writel_relaxed(base_cmd, base + AON_CTRL_PM_CTRL);
-
-	if (!onewrite) {
-		(void)readl_relaxed(base + AON_CTRL_PM_CTRL);
-
-		writel_relaxed(base_cmd | PM_PWR_DOWN, base + AON_CTRL_PM_CTRL);
-		(void)readl_relaxed(base + AON_CTRL_PM_CTRL);
-	}
-	wfi();
-}
-
-/* Support S5 cold boot out of "poweroff" */
-static void brcmstb_pm_poweroff(void)
-{
-	brcmstb_pm_handshake();
-
-	/* Clear magic S3 warm-boot value */
-	writel_relaxed(0, ctrl.aon_sram + AON_REG_MAGIC_FLAGS);
-	(void)readl_relaxed(ctrl.aon_sram + AON_REG_MAGIC_FLAGS);
-
-	/* Skip wait-for-interrupt signal; just use a countdown */
-	writel_relaxed(0x10, ctrl.aon_ctrl_base + AON_CTRL_PM_CPU_WAIT_COUNT);
-	(void)readl_relaxed(ctrl.aon_ctrl_base + AON_CTRL_PM_CPU_WAIT_COUNT);
-
-	if (ctrl.s3entry_method == 1) {
-		shimphy_set((PWRDWN_SEQ_POWERDOWN_PLL <<
-			     SHIMPHY_PAD_S3_PWRDWN_SEQ_SHIFT),
-			     ~SHIMPHY_PAD_S3_PWRDWN_SEQ_MASK);
-		ddr_ctrl_set(false);
-		brcmstb_do_pmsm_power_down(M1_PM_COLD_CONFIG, true);
-		return; /* We should never actually get here */
-	}
-
-	brcmstb_do_pmsm_power_down(PM_COLD_CONFIG, false);
-}
-
-static void *brcmstb_pm_copy_to_sram(void *fn, size_t len)
-{
-	unsigned int size = ALIGN(len, FNCPY_ALIGN);
-
-	if (ctrl.boot_sram_len < size) {
-		pr_err("standby code will not fit in SRAM\n");
-		return NULL;
-	}
-
-	return fncpy(ctrl.boot_sram, fn, size);
-}
-
-/*
- * S2 suspend/resume picks up where we left off, so we must execute carefully
- * from SRAM, in order to allow DDR to come back up safely before we continue.
- */
-static int brcmstb_pm_s2(void)
-{
-	/* A previous S3 can set a value hazardous to S2, so make sure. */
-	if (ctrl.s3entry_method == 1) {
-		shimphy_set((PWRDWN_SEQ_NO_SEQUENCING <<
-			    SHIMPHY_PAD_S3_PWRDWN_SEQ_SHIFT),
-			    ~SHIMPHY_PAD_S3_PWRDWN_SEQ_MASK);
-		ddr_ctrl_set(false);
-	}
-
-	brcmstb_pm_do_s2_sram = brcmstb_pm_copy_to_sram(&brcmstb_pm_do_s2,
-			brcmstb_pm_do_s2_sz);
-	if (!brcmstb_pm_do_s2_sram)
-		return -EINVAL;
-
-	return brcmstb_pm_do_s2_sram(ctrl.aon_ctrl_base,
-			ctrl.memcs[0].ddr_phy_base +
-			ctrl.pll_status_offset);
-}
-
-/*
- * This function is called on a new stack, so don't allow inlining (which will
- * generate stack references on the old stack). It cannot be made static because
- * it is referenced from brcmstb_pm_s3()
- */
-noinline int brcmstb_pm_s3_finish(void)
-{
-	struct brcmstb_s3_params *params = ctrl.s3_params;
-	dma_addr_t params_pa = ctrl.s3_params_pa;
-	phys_addr_t reentry = virt_to_phys(&cpu_resume_arm);
-	enum bsp_initiate_command cmd;
-	u32 flags;
-
-	/*
-	 * Clear parameter structure, but not DTU area, which has already been
-	 * filled in. We know DTU is a the end, so we can just subtract its
-	 * size.
-	 */
-	memset(params, 0, sizeof(*params) - sizeof(params->dtu));
-
-	flags = readl_relaxed(ctrl.aon_sram + AON_REG_MAGIC_FLAGS);
-
-	flags &= S3_BOOTLOADER_RESERVED;
-	flags |= S3_FLAG_NO_MEM_VERIFY;
-	flags |= S3_FLAG_LOAD_RANDKEY;
-
-	/* Load random / fixed key */
-	if (flags & S3_FLAG_LOAD_RANDKEY)
-		cmd = BSP_GEN_RANDOM_KEY;
-	else
-		cmd = BSP_GEN_FIXED_KEY;
-	if (do_bsp_initiate_command(cmd)) {
-		pr_info("key loading failed\n");
-		return -EIO;
-	}
-
-	params->magic = BRCMSTB_S3_MAGIC;
-	params->reentry = reentry;
-
-	/* No more writes to DRAM */
-	flush_cache_all();
-
-	flags |= BRCMSTB_S3_MAGIC_SHORT;
-
-	writel_relaxed(flags, ctrl.aon_sram + AON_REG_MAGIC_FLAGS);
-	writel_relaxed(lower_32_bits(params_pa),
-		       ctrl.aon_sram + AON_REG_CONTROL_LOW);
-	writel_relaxed(upper_32_bits(params_pa),
-		       ctrl.aon_sram + AON_REG_CONTROL_HIGH);
-
-	switch (ctrl.s3entry_method) {
-	case 0:
-		s3entry_method0();
-		brcmstb_do_pmsm_power_down(PM_WARM_CONFIG, false);
-		break;
-	case 1:
-		s3entry_method1();
-		brcmstb_do_pmsm_power_down(M1_PM_WARM_CONFIG, true);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* Must have been interrupted from wfi()? */
-	return -EINTR;
-}
-
-static int brcmstb_pm_do_s3(unsigned long sp)
-{
-	unsigned long save_sp;
-	int ret;
-
-	asm volatile (
-		"mov	%[save], sp\n"
-		"mov	sp, %[new]\n"
-		"bl	brcmstb_pm_s3_finish\n"
-		"mov	%[ret], r0\n"
-		"mov	%[new], sp\n"
-		"mov	sp, %[save]\n"
-		: [save] "=&r" (save_sp), [ret] "=&r" (ret)
-		: [new] "r" (sp)
-	);
-
-	return ret;
-}
-
-static int brcmstb_pm_s3(void)
-{
-	void __iomem *sp = ctrl.boot_sram + ctrl.boot_sram_len;
-
-	return cpu_suspend((unsigned long)sp, brcmstb_pm_do_s3);
-}
-
-static int brcmstb_pm_standby(bool deep_standby)
-{
-	int ret;
-
-	if (brcmstb_pm_handshake())
-		return -EIO;
-
-	if (deep_standby)
-		ret = brcmstb_pm_s3();
-	else
-		ret = brcmstb_pm_s2();
-	if (ret)
-		pr_err("%s: standby failed\n", __func__);
-
-	return ret;
-}
-
-static int brcmstb_pm_enter(suspend_state_t state)
-{
-	int ret = -EINVAL;
-
-	switch (state) {
-	case PM_SUSPEND_STANDBY:
-		ret = brcmstb_pm_standby(false);
-		break;
-	case PM_SUSPEND_MEM:
-		ret = brcmstb_pm_standby(true);
-		break;
-	}
-
-	return ret;
-}
-
-static int brcmstb_pm_valid(suspend_state_t state)
-{
-	switch (state) {
-	case PM_SUSPEND_STANDBY:
-		return true;
-	case PM_SUSPEND_MEM:
-		return ctrl.support_warm_boot;
-	default:
-		return false;
-	}
-}
-
-static const struct platform_suspend_ops brcmstb_pm_ops = {
-	.enter		= brcmstb_pm_enter,
-	.valid		= brcmstb_pm_valid,
-};
-
-static const struct of_device_id aon_ctrl_dt_ids[] = {
-	{ .compatible = "brcm,brcmstb-aon-ctrl" },
-	{}
-};
-
-struct ddr_phy_ofdata {
-	bool supports_warm_boot;
-	size_t pll_status_offset;
-	int s3entry_method;
-	u32 warm_boot_offset;
-	u32 phy_a_standby_ctrl_offs;
-	u32 phy_b_standby_ctrl_offs;
-};
-
-static struct ddr_phy_ofdata ddr_phy_71_1 = {
-	.supports_warm_boot = true,
-	.pll_status_offset = 0x0c,
-	.s3entry_method = 1,
-	.warm_boot_offset = 0x2c,
-	.phy_a_standby_ctrl_offs = 0x198,
-	.phy_b_standby_ctrl_offs = DDR_PHY_NO_CHANNEL
-};
-
-static struct ddr_phy_ofdata ddr_phy_72_0 = {
-	.supports_warm_boot = true,
-	.pll_status_offset = 0x10,
-	.s3entry_method = 1,
-	.warm_boot_offset = 0x40,
-	.phy_a_standby_ctrl_offs = 0x2a4,
-	.phy_b_standby_ctrl_offs = 0x8a4
-};
-
-static struct ddr_phy_ofdata ddr_phy_225_1 = {
-	.supports_warm_boot = false,
-	.pll_status_offset = 0x4,
-	.s3entry_method = 0
-};
-
-static struct ddr_phy_ofdata ddr_phy_240_1 = {
-	.supports_warm_boot = true,
-	.pll_status_offset = 0x4,
-	.s3entry_method = 0
-};
-
-static const struct of_device_id ddr_phy_dt_ids[] = {
-	{
-		.compatible = "brcm,brcmstb-ddr-phy-v71.1",
-		.data = &ddr_phy_71_1,
-	},
-	{
-		.compatible = "brcm,brcmstb-ddr-phy-v72.0",
-		.data = &ddr_phy_72_0,
-	},
-	{
-		.compatible = "brcm,brcmstb-ddr-phy-v225.1",
-		.data = &ddr_phy_225_1,
-	},
-	{
-		.compatible = "brcm,brcmstb-ddr-phy-v240.1",
-		.data = &ddr_phy_240_1,
-	},
-	{
-		/* Same as v240.1, for the registers we care about */
-		.compatible = "brcm,brcmstb-ddr-phy-v240.2",
-		.data = &ddr_phy_240_1,
-	},
-	{}
-};
-
-struct ddr_seq_ofdata {
-	bool needs_ddr_pad;
-	u32 warm_boot_offset;
-};
-
-static const struct ddr_seq_ofdata ddr_seq_b22 = {
-	.needs_ddr_pad = false,
-	.warm_boot_offset = 0x2c,
-};
-
-static const struct ddr_seq_ofdata ddr_seq = {
-	.needs_ddr_pad = true,
-};
-
-static const struct of_device_id ddr_shimphy_dt_ids[] = {
-	{ .compatible = "brcm,brcmstb-ddr-shimphy-v1.0" },
-	{}
-};
-
-static const struct of_device_id brcmstb_memc_of_match[] = {
-	{
-		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.1",
-		.data = &ddr_seq,
-	},
-	{
-		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.2",
-		.data = &ddr_seq_b22,
-	},
-	{
-		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.3",
-		.data = &ddr_seq_b22,
-	},
-	{
-		.compatible = "brcm,brcmstb-memc-ddr-rev-b.3.0",
-		.data = &ddr_seq_b22,
-	},
-	{
-		.compatible = "brcm,brcmstb-memc-ddr-rev-b.3.1",
-		.data = &ddr_seq_b22,
-	},
-	{
-		.compatible = "brcm,brcmstb-memc-ddr",
-		.data = &ddr_seq,
-	},
-	{},
-};
-
-static void __iomem *brcmstb_ioremap_match(const struct of_device_id *matches,
-					   int index, const void **ofdata)
-{
-	struct device_node *dn;
-	const struct of_device_id *match;
-
-	dn = of_find_matching_node_and_match(NULL, matches, &match);
-	if (!dn)
-		return ERR_PTR(-EINVAL);
-
-	if (ofdata)
-		*ofdata = match->data;
-
-	return of_io_request_and_map(dn, index, dn->full_name);
-}
-/*
- * The AON is a small domain in the SoC that can retain its state across
- * various system wide sleep states and specific reset conditions; the
- * AON DATA RAM is a small RAM of a few words (< 1KB) which can store
- * persistent information across such events.
- *
- * The purpose of the below panic notifier is to help with notifying
- * the bootloader that a panic occurred and so that it should try its
- * best to preserve the DRAM contents holding that buffer for recovery
- * by the kernel as opposed to wiping out DRAM clean again.
- *
- * Reference: comment from Florian Fainelli, at
- * https://lore.kernel.org/lkml/781cafb0-8d06-8b56-907a-5175c2da196a@gmail.com
- */
-static int brcmstb_pm_panic_notify(struct notifier_block *nb,
-		unsigned long action, void *data)
-{
-	writel_relaxed(BRCMSTB_PANIC_MAGIC, ctrl.aon_sram + AON_REG_PANIC);
-
-	return NOTIFY_DONE;
-}
-
-static struct notifier_block brcmstb_pm_panic_nb = {
-	.notifier_call = brcmstb_pm_panic_notify,
-};
-
-static int brcmstb_pm_probe(struct platform_device *pdev)
-{
-	const struct ddr_phy_ofdata *ddr_phy_data;
-	const struct ddr_seq_ofdata *ddr_seq_data;
-	const struct of_device_id *of_id = NULL;
-	struct device_node *dn;
-	void __iomem *base;
-	int ret, i, s;
-
-	/* AON ctrl registers */
-	base = brcmstb_ioremap_match(aon_ctrl_dt_ids, 0, NULL);
-	if (IS_ERR(base)) {
-		pr_err("error mapping AON_CTRL\n");
-		ret = PTR_ERR(base);
-		goto aon_err;
-	}
-	ctrl.aon_ctrl_base = base;
-
-	/* AON SRAM registers */
-	base = brcmstb_ioremap_match(aon_ctrl_dt_ids, 1, NULL);
-	if (IS_ERR(base)) {
-		/* Assume standard offset */
-		ctrl.aon_sram = ctrl.aon_ctrl_base +
-				     AON_CTRL_SYSTEM_DATA_RAM_OFS;
-		s = 0;
-	} else {
-		ctrl.aon_sram = base;
-		s = 1;
-	}
-
-	writel_relaxed(0, ctrl.aon_sram + AON_REG_PANIC);
-
-	/* DDR PHY registers */
-	base = brcmstb_ioremap_match(ddr_phy_dt_ids, 0,
-				     (const void **)&ddr_phy_data);
-	if (IS_ERR(base)) {
-		pr_err("error mapping DDR PHY\n");
-		ret = PTR_ERR(base);
-		goto ddr_phy_err;
-	}
-	ctrl.support_warm_boot = ddr_phy_data->supports_warm_boot;
-	ctrl.pll_status_offset = ddr_phy_data->pll_status_offset;
-	/* Only need DDR PHY 0 for now? */
-	ctrl.memcs[0].ddr_phy_base = base;
-	ctrl.s3entry_method = ddr_phy_data->s3entry_method;
-	ctrl.phy_a_standby_ctrl_offs = ddr_phy_data->phy_a_standby_ctrl_offs;
-	ctrl.phy_b_standby_ctrl_offs = ddr_phy_data->phy_b_standby_ctrl_offs;
-	/*
-	 * Slightly gross to use the phy ver to get a memc,
-	 * offset but that is the only versioned things so far
-	 * we can test for.
-	 */
-	ctrl.warm_boot_offset = ddr_phy_data->warm_boot_offset;
-
-	/* DDR SHIM-PHY registers */
-	for_each_matching_node(dn, ddr_shimphy_dt_ids) {
-		i = ctrl.num_memc;
-		if (i >= MAX_NUM_MEMC) {
-			of_node_put(dn);
-			pr_warn("too many MEMCs (max %d)\n", MAX_NUM_MEMC);
-			break;
-		}
-
-		base = of_io_request_and_map(dn, 0, dn->full_name);
-		if (IS_ERR(base)) {
-			of_node_put(dn);
-			if (!ctrl.support_warm_boot)
-				break;
-
-			pr_err("error mapping DDR SHIMPHY %d\n", i);
-			ret = PTR_ERR(base);
-			goto ddr_shimphy_err;
-		}
-		ctrl.memcs[i].ddr_shimphy_base = base;
-		ctrl.num_memc++;
-	}
-
-	/* Sequencer DRAM Param and Control Registers */
-	i = 0;
-	for_each_matching_node(dn, brcmstb_memc_of_match) {
-		base = of_iomap(dn, 0);
-		if (!base) {
-			of_node_put(dn);
-			pr_err("error mapping DDR Sequencer %d\n", i);
-			ret = -ENOMEM;
-			goto brcmstb_memc_err;
-		}
-
-		of_id = of_match_node(brcmstb_memc_of_match, dn);
-		if (!of_id) {
-			iounmap(base);
-			of_node_put(dn);
-			ret = -EINVAL;
-			goto brcmstb_memc_err;
-		}
-
-		ddr_seq_data = of_id->data;
-		ctrl.needs_ddr_pad = ddr_seq_data->needs_ddr_pad;
-		/* Adjust warm boot offset based on the DDR sequencer */
-		if (ddr_seq_data->warm_boot_offset)
-			ctrl.warm_boot_offset = ddr_seq_data->warm_boot_offset;
-
-		ctrl.memcs[i].ddr_ctrl = base;
-		i++;
-	}
-
-	pr_debug("PM: supports warm boot:%d, method:%d, wboffs:%x\n",
-		ctrl.support_warm_boot, ctrl.s3entry_method,
-		ctrl.warm_boot_offset);
-
-	dn = of_find_matching_node(NULL, sram_dt_ids);
-	if (!dn) {
-		pr_err("SRAM not found\n");
-		ret = -EINVAL;
-		goto brcmstb_memc_err;
-	}
-
-	ret = brcmstb_init_sram(dn);
-	of_node_put(dn);
-	if (ret) {
-		pr_err("error setting up SRAM for PM\n");
-		goto brcmstb_memc_err;
-	}
-
-	ctrl.pdev = pdev;
-
-	ctrl.s3_params = kmalloc(sizeof(*ctrl.s3_params), GFP_KERNEL);
-	if (!ctrl.s3_params) {
-		ret = -ENOMEM;
-		goto s3_params_err;
-	}
-	ctrl.s3_params_pa = dma_map_single(&pdev->dev, ctrl.s3_params,
-					   sizeof(*ctrl.s3_params),
-					   DMA_TO_DEVICE);
-	if (dma_mapping_error(&pdev->dev, ctrl.s3_params_pa)) {
-		pr_err("error mapping DMA memory\n");
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	atomic_notifier_chain_register(&panic_notifier_list,
-				       &brcmstb_pm_panic_nb);
-
-	pm_power_off = brcmstb_pm_poweroff;
-	suspend_set_ops(&brcmstb_pm_ops);
-
-	return 0;
-
-out:
-	kfree(ctrl.s3_params);
-s3_params_err:
-	iounmap(ctrl.boot_sram);
-brcmstb_memc_err:
-	for (i--; i >= 0; i--)
-		iounmap(ctrl.memcs[i].ddr_ctrl);
-ddr_shimphy_err:
-	for (i = 0; i < ctrl.num_memc; i++)
-		iounmap(ctrl.memcs[i].ddr_shimphy_base);
-
-	iounmap(ctrl.memcs[0].ddr_phy_base);
-ddr_phy_err:
-	iounmap(ctrl.aon_ctrl_base);
-	if (s)
-		iounmap(ctrl.aon_sram);
-aon_err:
-	pr_warn("PM: initialization failed with code %d\n", ret);
-
-	return ret;
-}
-
-static struct platform_driver brcmstb_pm_driver = {
-	.driver = {
-		.name	= "brcmstb-pm",
-		.of_match_table = aon_ctrl_dt_ids,
-	},
-};
-
-static int __init brcmstb_pm_init(void)
-{
-	return platform_driver_probe(&brcmstb_pm_driver,
-				     brcmstb_pm_probe);
-}
-module_init(brcmstb_pm_init);
diff --git a/drivers/soc/bcm/brcmstb/pm/s2-arm.S b/drivers/soc/bcm/brcmstb/pm/s2-arm.S
deleted file mode 100644
index 0d693795de27..000000000000
--- a/drivers/soc/bcm/brcmstb/pm/s2-arm.S
+++ /dev/null
@@ -1,69 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright © 2014-2017 Broadcom
- */
-
-#include <linux/linkage.h>
-#include <asm/assembler.h>
-
-#include "pm.h"
-
-	.arch armv7-a
-	.text
-	.align	3
-
-#define AON_CTRL_REG		r10
-#define DDR_PHY_STATUS_REG	r11
-
-/*
- * r0: AON_CTRL base address
- * r1: DDRY PHY PLL status register address
- */
-ENTRY(brcmstb_pm_do_s2)
-	stmfd	sp!, {r4-r11, lr}
-	mov	AON_CTRL_REG, r0
-	mov	DDR_PHY_STATUS_REG, r1
-
-	/* Flush memory transactions */
-	dsb
-
-	/* Cache DDR_PHY_STATUS_REG translation */
-	ldr	r0, [DDR_PHY_STATUS_REG]
-
-	/* power down request */
-	ldr	r0, =PM_S2_COMMAND
-	ldr	r1, =0
-	str	r1, [AON_CTRL_REG, #AON_CTRL_PM_CTRL]
-	ldr	r1, [AON_CTRL_REG, #AON_CTRL_PM_CTRL]
-	str	r0, [AON_CTRL_REG, #AON_CTRL_PM_CTRL]
-	ldr	r0, [AON_CTRL_REG, #AON_CTRL_PM_CTRL]
-
-	/* Wait for interrupt */
-	wfi
-	nop
-
-	/* Bring MEMC back up */
-1:	ldr	r0, [DDR_PHY_STATUS_REG]
-	ands	r0, #1
-	beq	1b
-
-	/* Power-up handshake */
-	ldr	r0, =1
-	str	r0, [AON_CTRL_REG, #AON_CTRL_HOST_MISC_CMDS]
-	ldr	r0, [AON_CTRL_REG, #AON_CTRL_HOST_MISC_CMDS]
-
-	ldr	r0, =0
-	str	r0, [AON_CTRL_REG, #AON_CTRL_PM_CTRL]
-	ldr	r0, [AON_CTRL_REG, #AON_CTRL_PM_CTRL]
-
-	/* Return to caller */
-	ldr	r0, =0
-	ldmfd	sp!, {r4-r11, pc}
-
-	ENDPROC(brcmstb_pm_do_s2)
-
-	/* Place literal pool here */
-	.ltorg
-
-ENTRY(brcmstb_pm_do_s2_sz)
-	.word   . - brcmstb_pm_do_s2
diff --git a/drivers/soc/bcm/raspberrypi-power.c b/drivers/soc/bcm/raspberrypi-power.c
index 068715d6e66d..58175af982a0 100644
--- a/drivers/soc/bcm/raspberrypi-power.c
+++ b/drivers/soc/bcm/raspberrypi-power.c
@@ -243,4 +243,3 @@ builtin_platform_driver(rpi_power_driver);
 MODULE_AUTHOR("Alexander Aring <aar@pengutronix.de>");
 MODULE_AUTHOR("Eric Anholt <eric@anholt.net>");
 MODULE_DESCRIPTION("Raspberry Pi power domain driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/soc/canaan/Kconfig b/drivers/soc/canaan/Kconfig
index 2527cf5757ec..43ced2bf8444 100644
--- a/drivers/soc/canaan/Kconfig
+++ b/drivers/soc/canaan/Kconfig
@@ -3,8 +3,9 @@
 config SOC_K210_SYSCTL
 	bool "Canaan Kendryte K210 SoC system controller"
 	depends on RISCV && SOC_CANAAN && OF
+	depends on COMMON_CLK_K210
 	default SOC_CANAAN
-        select PM
-        select MFD_SYSCON
+	select PM
+	select MFD_SYSCON
 	help
 	  Canaan Kendryte K210 SoC system controller driver.
diff --git a/drivers/soc/fsl/qbman/dpaa_sys.c b/drivers/soc/fsl/qbman/dpaa_sys.c
index 9dd8bb571dbc..33751450047e 100644
--- a/drivers/soc/fsl/qbman/dpaa_sys.c
+++ b/drivers/soc/fsl/qbman/dpaa_sys.c
@@ -39,8 +39,7 @@ int qbman_init_private_mem(struct device *dev, int idx, dma_addr_t *addr,
 {
 	struct device_node *mem_node;
 	struct reserved_mem *rmem;
-	struct property *prop;
-	int len, err;
+	int err;
 	__be32 *res_array;
 
 	mem_node = of_parse_phandle(dev->of_node, "memory-region", idx);
@@ -63,8 +62,9 @@ int qbman_init_private_mem(struct device *dev, int idx, dma_addr_t *addr,
 	 * This is needed because QBMan HW does not allow the base address/
 	 * size to be modified once set.
 	 */
-	prop = of_find_property(mem_node, "reg", &len);
-	if (!prop) {
+	if (!of_property_present(mem_node, "reg")) {
+		struct property *prop;
+
 		prop = devm_kzalloc(dev, sizeof(*prop), GFP_KERNEL);
 		if (!prop)
 			return -ENOMEM;
diff --git a/drivers/soc/fsl/qe/Kconfig b/drivers/soc/fsl/qe/Kconfig
index 357c5800b112..7268c2fbcbc1 100644
--- a/drivers/soc/fsl/qe/Kconfig
+++ b/drivers/soc/fsl/qe/Kconfig
@@ -33,6 +33,29 @@ config UCC
 	bool
 	default y if UCC_FAST || UCC_SLOW
 
+config CPM_TSA
+	tristate "CPM TSA support"
+	depends on OF && HAS_IOMEM
+	depends on CPM1 || COMPILE_TEST
+	help
+	  Freescale CPM Time Slot Assigner (TSA)
+	  controller.
+
+	  This option enables support for this
+	  controller
+
+config CPM_QMC
+	tristate "CPM QMC support"
+	depends on OF && HAS_IOMEM
+	depends on CPM1 || (FSL_SOC && COMPILE_TEST)
+	depends on CPM_TSA
+	help
+	  Freescale CPM QUICC Multichannel Controller
+	  (QMC)
+
+	  This option enables support for this
+	  controller
+
 config QE_TDM
 	bool
 	default y if FSL_UCC_HDLC
diff --git a/drivers/soc/fsl/qe/Makefile b/drivers/soc/fsl/qe/Makefile
index 55a555304f3a..ec8506e13113 100644
--- a/drivers/soc/fsl/qe/Makefile
+++ b/drivers/soc/fsl/qe/Makefile
@@ -4,6 +4,8 @@
 #
 obj-$(CONFIG_QUICC_ENGINE)+= qe.o qe_common.o qe_ic.o qe_io.o
 obj-$(CONFIG_CPM)	+= qe_common.o
+obj-$(CONFIG_CPM_TSA)	+= tsa.o
+obj-$(CONFIG_CPM_QMC)	+= qmc.o
 obj-$(CONFIG_UCC)	+= ucc.o
 obj-$(CONFIG_UCC_SLOW)	+= ucc_slow.o
 obj-$(CONFIG_UCC_FAST)	+= ucc_fast.o
diff --git a/drivers/soc/fsl/qe/gpio.c b/drivers/soc/fsl/qe/gpio.c
index 1c41eb49d5a7..3ef24ba0245b 100644
--- a/drivers/soc/fsl/qe/gpio.c
+++ b/drivers/soc/fsl/qe/gpio.c
@@ -13,7 +13,7 @@
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>	/* for of_mm_gpio_chip */
+#include <linux/gpio/legacy-of-mm-gpiochip.h>
 #include <linux/gpio/consumer.h>
 #include <linux/gpio/driver.h>
 #include <linux/slab.h>
diff --git a/drivers/soc/fsl/qe/qmc.c b/drivers/soc/fsl/qe/qmc.c
new file mode 100644
index 000000000000..b3c292c9a14e
--- /dev/null
+++ b/drivers/soc/fsl/qe/qmc.c
@@ -0,0 +1,1537 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * QMC driver
+ *
+ * Copyright 2022 CS GROUP France
+ *
+ * Author: Herve Codina <herve.codina@bootlin.com>
+ */
+
+#include <soc/fsl/qe/qmc.h>
+#include <linux/dma-mapping.h>
+#include <linux/hdlc.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <soc/fsl/cpm.h>
+#include <sysdev/fsl_soc.h>
+#include "tsa.h"
+
+/* SCC general mode register high (32 bits) */
+#define SCC_GSMRL	0x00
+#define SCC_GSMRL_ENR		(1 << 5)
+#define SCC_GSMRL_ENT		(1 << 4)
+#define SCC_GSMRL_MODE_QMC	(0x0A << 0)
+
+/* SCC general mode register low (32 bits) */
+#define SCC_GSMRH	0x04
+#define   SCC_GSMRH_CTSS	(1 << 7)
+#define   SCC_GSMRH_CDS		(1 << 8)
+#define   SCC_GSMRH_CTSP	(1 << 9)
+#define   SCC_GSMRH_CDP		(1 << 10)
+
+/* SCC event register (16 bits) */
+#define SCC_SCCE	0x10
+#define   SCC_SCCE_IQOV		(1 << 3)
+#define   SCC_SCCE_GINT		(1 << 2)
+#define   SCC_SCCE_GUN		(1 << 1)
+#define   SCC_SCCE_GOV		(1 << 0)
+
+/* SCC mask register (16 bits) */
+#define SCC_SCCM	0x14
+/* Multichannel base pointer (32 bits) */
+#define QMC_GBL_MCBASE		0x00
+/* Multichannel controller state (16 bits) */
+#define QMC_GBL_QMCSTATE	0x04
+/* Maximum receive buffer length (16 bits) */
+#define QMC_GBL_MRBLR		0x06
+/* Tx time-slot assignment table pointer (16 bits) */
+#define QMC_GBL_TX_S_PTR	0x08
+/* Rx pointer (16 bits) */
+#define QMC_GBL_RXPTR		0x0A
+/* Global receive frame threshold (16 bits) */
+#define QMC_GBL_GRFTHR		0x0C
+/* Global receive frame count (16 bits) */
+#define QMC_GBL_GRFCNT		0x0E
+/* Multichannel interrupt base address (32 bits) */
+#define QMC_GBL_INTBASE		0x10
+/* Multichannel interrupt pointer (32 bits) */
+#define QMC_GBL_INTPTR		0x14
+/* Rx time-slot assignment table pointer (16 bits) */
+#define QMC_GBL_RX_S_PTR	0x18
+/* Tx pointer (16 bits) */
+#define QMC_GBL_TXPTR		0x1A
+/* CRC constant (32 bits) */
+#define QMC_GBL_C_MASK32	0x1C
+/* Time slot assignment table Rx (32 x 16 bits) */
+#define QMC_GBL_TSATRX		0x20
+/* Time slot assignment table Tx (32 x 16 bits) */
+#define QMC_GBL_TSATTX		0x60
+/* CRC constant (16 bits) */
+#define QMC_GBL_C_MASK16	0xA0
+
+/* TSA entry (16bit entry in TSATRX and TSATTX) */
+#define QMC_TSA_VALID		(1 << 15)
+#define QMC_TSA_WRAP		(1 << 14)
+#define QMC_TSA_MASK		(0x303F)
+#define QMC_TSA_CHANNEL(x)	((x) << 6)
+
+/* Tx buffer descriptor base address (16 bits, offset from MCBASE) */
+#define QMC_SPE_TBASE	0x00
+
+/* Channel mode register (16 bits) */
+#define QMC_SPE_CHAMR	0x02
+#define   QMC_SPE_CHAMR_MODE_HDLC	(1 << 15)
+#define   QMC_SPE_CHAMR_MODE_TRANSP	((0 << 15) | (1 << 13))
+#define   QMC_SPE_CHAMR_ENT		(1 << 12)
+#define   QMC_SPE_CHAMR_POL		(1 << 8)
+#define   QMC_SPE_CHAMR_HDLC_IDLM	(1 << 13)
+#define   QMC_SPE_CHAMR_HDLC_CRC	(1 << 7)
+#define   QMC_SPE_CHAMR_HDLC_NOF	(0x0f << 0)
+#define   QMC_SPE_CHAMR_TRANSP_RD	(1 << 14)
+#define   QMC_SPE_CHAMR_TRANSP_SYNC	(1 << 10)
+
+/* Tx internal state (32 bits) */
+#define QMC_SPE_TSTATE	0x04
+/* Tx buffer descriptor pointer (16 bits) */
+#define QMC_SPE_TBPTR	0x0C
+/* Zero-insertion state (32 bits) */
+#define QMC_SPE_ZISTATE	0x14
+/* Channel’s interrupt mask flags (16 bits) */
+#define QMC_SPE_INTMSK	0x1C
+/* Rx buffer descriptor base address (16 bits, offset from MCBASE) */
+#define QMC_SPE_RBASE	0x20
+/* HDLC: Maximum frame length register (16 bits) */
+#define QMC_SPE_MFLR	0x22
+/* TRANSPARENT: Transparent maximum receive length (16 bits) */
+#define QMC_SPE_TMRBLR	0x22
+/* Rx internal state (32 bits) */
+#define QMC_SPE_RSTATE	0x24
+/* Rx buffer descriptor pointer (16 bits) */
+#define QMC_SPE_RBPTR	0x2C
+/* Packs 4 bytes to 1 long word before writing to buffer (32 bits) */
+#define QMC_SPE_RPACK	0x30
+/* Zero deletion state (32 bits) */
+#define QMC_SPE_ZDSTATE	0x34
+
+/* Transparent synchronization (16 bits) */
+#define QMC_SPE_TRNSYNC 0x3C
+#define   QMC_SPE_TRNSYNC_RX(x)	((x) << 8)
+#define   QMC_SPE_TRNSYNC_TX(x)	((x) << 0)
+
+/* Interrupt related registers bits */
+#define QMC_INT_V		(1 << 15)
+#define QMC_INT_W		(1 << 14)
+#define QMC_INT_NID		(1 << 13)
+#define QMC_INT_IDL		(1 << 12)
+#define QMC_INT_GET_CHANNEL(x)	(((x) & 0x0FC0) >> 6)
+#define QMC_INT_MRF		(1 << 5)
+#define QMC_INT_UN		(1 << 4)
+#define QMC_INT_RXF		(1 << 3)
+#define QMC_INT_BSY		(1 << 2)
+#define QMC_INT_TXB		(1 << 1)
+#define QMC_INT_RXB		(1 << 0)
+
+/* BD related registers bits */
+#define QMC_BD_RX_E	(1 << 15)
+#define QMC_BD_RX_W	(1 << 13)
+#define QMC_BD_RX_I	(1 << 12)
+#define QMC_BD_RX_L	(1 << 11)
+#define QMC_BD_RX_F	(1 << 10)
+#define QMC_BD_RX_CM	(1 << 9)
+#define QMC_BD_RX_UB	(1 << 7)
+#define QMC_BD_RX_LG	(1 << 5)
+#define QMC_BD_RX_NO	(1 << 4)
+#define QMC_BD_RX_AB	(1 << 3)
+#define QMC_BD_RX_CR	(1 << 2)
+
+#define QMC_BD_TX_R	(1 << 15)
+#define QMC_BD_TX_W	(1 << 13)
+#define QMC_BD_TX_I	(1 << 12)
+#define QMC_BD_TX_L	(1 << 11)
+#define QMC_BD_TX_TC	(1 << 10)
+#define QMC_BD_TX_CM	(1 << 9)
+#define QMC_BD_TX_UB	(1 << 7)
+#define QMC_BD_TX_PAD	(0x0f << 0)
+
+/* Numbers of BDs and interrupt items */
+#define QMC_NB_TXBDS	8
+#define QMC_NB_RXBDS	8
+#define QMC_NB_INTS	128
+
+struct qmc_xfer_desc {
+	union {
+		void (*tx_complete)(void *context);
+		void (*rx_complete)(void *context, size_t length);
+	};
+	void *context;
+};
+
+struct qmc_chan {
+	struct list_head list;
+	unsigned int id;
+	struct qmc *qmc;
+	void *__iomem s_param;
+	enum qmc_mode mode;
+	u64	tx_ts_mask;
+	u64	rx_ts_mask;
+	bool is_reverse_data;
+
+	spinlock_t	tx_lock;
+	cbd_t __iomem *txbds;
+	cbd_t __iomem *txbd_free;
+	cbd_t __iomem *txbd_done;
+	struct qmc_xfer_desc tx_desc[QMC_NB_TXBDS];
+	u64	nb_tx_underrun;
+	bool	is_tx_stopped;
+
+	spinlock_t	rx_lock;
+	cbd_t __iomem *rxbds;
+	cbd_t __iomem *rxbd_free;
+	cbd_t __iomem *rxbd_done;
+	struct qmc_xfer_desc rx_desc[QMC_NB_RXBDS];
+	u64	nb_rx_busy;
+	int	rx_pending;
+	bool	is_rx_halted;
+	bool	is_rx_stopped;
+};
+
+struct qmc {
+	struct device *dev;
+	struct tsa_serial *tsa_serial;
+	void *__iomem scc_regs;
+	void *__iomem scc_pram;
+	void *__iomem dpram;
+	u16 scc_pram_offset;
+	cbd_t __iomem *bd_table;
+	dma_addr_t bd_dma_addr;
+	size_t bd_size;
+	u16 __iomem *int_table;
+	u16 __iomem *int_curr;
+	dma_addr_t int_dma_addr;
+	size_t int_size;
+	struct list_head chan_head;
+	struct qmc_chan *chans[64];
+};
+
+static inline void qmc_write16(void *__iomem addr, u16 val)
+{
+	iowrite16be(val, addr);
+}
+
+static inline u16 qmc_read16(void *__iomem addr)
+{
+	return ioread16be(addr);
+}
+
+static inline void qmc_setbits16(void *__iomem addr, u16 set)
+{
+	qmc_write16(addr, qmc_read16(addr) | set);
+}
+
+static inline void qmc_clrbits16(void *__iomem addr, u16 clr)
+{
+	qmc_write16(addr, qmc_read16(addr) & ~clr);
+}
+
+static inline void qmc_write32(void *__iomem addr, u32 val)
+{
+	iowrite32be(val, addr);
+}
+
+static inline u32 qmc_read32(void *__iomem addr)
+{
+	return ioread32be(addr);
+}
+
+static inline void qmc_setbits32(void *__iomem addr, u32 set)
+{
+	qmc_write32(addr, qmc_read32(addr) | set);
+}
+
+
+int qmc_chan_get_info(struct qmc_chan *chan, struct qmc_chan_info *info)
+{
+	struct tsa_serial_info tsa_info;
+	int ret;
+
+	/* Retrieve info from the TSA related serial */
+	ret = tsa_serial_get_info(chan->qmc->tsa_serial, &tsa_info);
+	if (ret)
+		return ret;
+
+	info->mode = chan->mode;
+	info->rx_fs_rate = tsa_info.rx_fs_rate;
+	info->rx_bit_rate = tsa_info.rx_bit_rate;
+	info->nb_tx_ts = hweight64(chan->tx_ts_mask);
+	info->tx_fs_rate = tsa_info.tx_fs_rate;
+	info->tx_bit_rate = tsa_info.tx_bit_rate;
+	info->nb_rx_ts = hweight64(chan->rx_ts_mask);
+
+	return 0;
+}
+EXPORT_SYMBOL(qmc_chan_get_info);
+
+int qmc_chan_set_param(struct qmc_chan *chan, const struct qmc_chan_param *param)
+{
+	if (param->mode != chan->mode)
+		return -EINVAL;
+
+	switch (param->mode) {
+	case QMC_HDLC:
+		if ((param->hdlc.max_rx_buf_size % 4) ||
+		    (param->hdlc.max_rx_buf_size < 8))
+			return -EINVAL;
+
+		qmc_write16(chan->qmc->scc_pram + QMC_GBL_MRBLR,
+			    param->hdlc.max_rx_buf_size - 8);
+		qmc_write16(chan->s_param + QMC_SPE_MFLR,
+			    param->hdlc.max_rx_frame_size);
+		if (param->hdlc.is_crc32) {
+			qmc_setbits16(chan->s_param + QMC_SPE_CHAMR,
+				      QMC_SPE_CHAMR_HDLC_CRC);
+		} else {
+			qmc_clrbits16(chan->s_param + QMC_SPE_CHAMR,
+				      QMC_SPE_CHAMR_HDLC_CRC);
+		}
+		break;
+
+	case QMC_TRANSPARENT:
+		qmc_write16(chan->s_param + QMC_SPE_TMRBLR,
+			    param->transp.max_rx_buf_size);
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(qmc_chan_set_param);
+
+int qmc_chan_write_submit(struct qmc_chan *chan, dma_addr_t addr, size_t length,
+			  void (*complete)(void *context), void *context)
+{
+	struct qmc_xfer_desc *xfer_desc;
+	unsigned long flags;
+	cbd_t *__iomem bd;
+	u16 ctrl;
+	int ret;
+
+	/*
+	 * R bit  UB bit
+	 *   0       0  : The BD is free
+	 *   1       1  : The BD is in used, waiting for transfer
+	 *   0       1  : The BD is in used, waiting for completion
+	 *   1       0  : Should not append
+	 */
+
+	spin_lock_irqsave(&chan->tx_lock, flags);
+	bd = chan->txbd_free;
+
+	ctrl = qmc_read16(&bd->cbd_sc);
+	if (ctrl & (QMC_BD_TX_R | QMC_BD_TX_UB)) {
+		/* We are full ... */
+		ret = -EBUSY;
+		goto end;
+	}
+
+	qmc_write16(&bd->cbd_datlen, length);
+	qmc_write32(&bd->cbd_bufaddr, addr);
+
+	xfer_desc = &chan->tx_desc[bd - chan->txbds];
+	xfer_desc->tx_complete = complete;
+	xfer_desc->context = context;
+
+	/* Activate the descriptor */
+	ctrl |= (QMC_BD_TX_R | QMC_BD_TX_UB);
+	wmb(); /* Be sure to flush the descriptor before control update */
+	qmc_write16(&bd->cbd_sc, ctrl);
+
+	if (!chan->is_tx_stopped)
+		qmc_setbits16(chan->s_param + QMC_SPE_CHAMR, QMC_SPE_CHAMR_POL);
+
+	if (ctrl & QMC_BD_TX_W)
+		chan->txbd_free = chan->txbds;
+	else
+		chan->txbd_free++;
+
+	ret = 0;
+
+end:
+	spin_unlock_irqrestore(&chan->tx_lock, flags);
+	return ret;
+}
+EXPORT_SYMBOL(qmc_chan_write_submit);
+
+static void qmc_chan_write_done(struct qmc_chan *chan)
+{
+	struct qmc_xfer_desc *xfer_desc;
+	void (*complete)(void *context);
+	unsigned long flags;
+	void *context;
+	cbd_t *__iomem bd;
+	u16 ctrl;
+
+	/*
+	 * R bit  UB bit
+	 *   0       0  : The BD is free
+	 *   1       1  : The BD is in used, waiting for transfer
+	 *   0       1  : The BD is in used, waiting for completion
+	 *   1       0  : Should not append
+	 */
+
+	spin_lock_irqsave(&chan->tx_lock, flags);
+	bd = chan->txbd_done;
+
+	ctrl = qmc_read16(&bd->cbd_sc);
+	while (!(ctrl & QMC_BD_TX_R)) {
+		if (!(ctrl & QMC_BD_TX_UB))
+			goto end;
+
+		xfer_desc = &chan->tx_desc[bd - chan->txbds];
+		complete = xfer_desc->tx_complete;
+		context = xfer_desc->context;
+		xfer_desc->tx_complete = NULL;
+		xfer_desc->context = NULL;
+
+		qmc_write16(&bd->cbd_sc, ctrl & ~QMC_BD_TX_UB);
+
+		if (ctrl & QMC_BD_TX_W)
+			chan->txbd_done = chan->txbds;
+		else
+			chan->txbd_done++;
+
+		if (complete) {
+			spin_unlock_irqrestore(&chan->tx_lock, flags);
+			complete(context);
+			spin_lock_irqsave(&chan->tx_lock, flags);
+		}
+
+		bd = chan->txbd_done;
+		ctrl = qmc_read16(&bd->cbd_sc);
+	}
+
+end:
+	spin_unlock_irqrestore(&chan->tx_lock, flags);
+}
+
+int qmc_chan_read_submit(struct qmc_chan *chan, dma_addr_t addr, size_t length,
+			 void (*complete)(void *context, size_t length), void *context)
+{
+	struct qmc_xfer_desc *xfer_desc;
+	unsigned long flags;
+	cbd_t *__iomem bd;
+	u16 ctrl;
+	int ret;
+
+	/*
+	 * E bit  UB bit
+	 *   0       0  : The BD is free
+	 *   1       1  : The BD is in used, waiting for transfer
+	 *   0       1  : The BD is in used, waiting for completion
+	 *   1       0  : Should not append
+	 */
+
+	spin_lock_irqsave(&chan->rx_lock, flags);
+	bd = chan->rxbd_free;
+
+	ctrl = qmc_read16(&bd->cbd_sc);
+	if (ctrl & (QMC_BD_RX_E | QMC_BD_RX_UB)) {
+		/* We are full ... */
+		ret = -EBUSY;
+		goto end;
+	}
+
+	qmc_write16(&bd->cbd_datlen, 0); /* data length is updated by the QMC */
+	qmc_write32(&bd->cbd_bufaddr, addr);
+
+	xfer_desc = &chan->rx_desc[bd - chan->rxbds];
+	xfer_desc->rx_complete = complete;
+	xfer_desc->context = context;
+
+	/* Activate the descriptor */
+	ctrl |= (QMC_BD_RX_E | QMC_BD_RX_UB);
+	wmb(); /* Be sure to flush data before descriptor activation */
+	qmc_write16(&bd->cbd_sc, ctrl);
+
+	/* Restart receiver if needed */
+	if (chan->is_rx_halted && !chan->is_rx_stopped) {
+		/* Restart receiver */
+		if (chan->mode == QMC_TRANSPARENT)
+			qmc_write32(chan->s_param + QMC_SPE_ZDSTATE, 0x18000080);
+		else
+			qmc_write32(chan->s_param + QMC_SPE_ZDSTATE, 0x00000080);
+		qmc_write32(chan->s_param + QMC_SPE_RSTATE, 0x31000000);
+		chan->is_rx_halted = false;
+	}
+	chan->rx_pending++;
+
+	if (ctrl & QMC_BD_RX_W)
+		chan->rxbd_free = chan->rxbds;
+	else
+		chan->rxbd_free++;
+
+	ret = 0;
+end:
+	spin_unlock_irqrestore(&chan->rx_lock, flags);
+	return ret;
+}
+EXPORT_SYMBOL(qmc_chan_read_submit);
+
+static void qmc_chan_read_done(struct qmc_chan *chan)
+{
+	void (*complete)(void *context, size_t size);
+	struct qmc_xfer_desc *xfer_desc;
+	unsigned long flags;
+	cbd_t *__iomem bd;
+	void *context;
+	u16 datalen;
+	u16 ctrl;
+
+	/*
+	 * E bit  UB bit
+	 *   0       0  : The BD is free
+	 *   1       1  : The BD is in used, waiting for transfer
+	 *   0       1  : The BD is in used, waiting for completion
+	 *   1       0  : Should not append
+	 */
+
+	spin_lock_irqsave(&chan->rx_lock, flags);
+	bd = chan->rxbd_done;
+
+	ctrl = qmc_read16(&bd->cbd_sc);
+	while (!(ctrl & QMC_BD_RX_E)) {
+		if (!(ctrl & QMC_BD_RX_UB))
+			goto end;
+
+		xfer_desc = &chan->rx_desc[bd - chan->rxbds];
+		complete = xfer_desc->rx_complete;
+		context = xfer_desc->context;
+		xfer_desc->rx_complete = NULL;
+		xfer_desc->context = NULL;
+
+		datalen = qmc_read16(&bd->cbd_datlen);
+		qmc_write16(&bd->cbd_sc, ctrl & ~QMC_BD_RX_UB);
+
+		if (ctrl & QMC_BD_RX_W)
+			chan->rxbd_done = chan->rxbds;
+		else
+			chan->rxbd_done++;
+
+		chan->rx_pending--;
+
+		if (complete) {
+			spin_unlock_irqrestore(&chan->rx_lock, flags);
+			complete(context, datalen);
+			spin_lock_irqsave(&chan->rx_lock, flags);
+		}
+
+		bd = chan->rxbd_done;
+		ctrl = qmc_read16(&bd->cbd_sc);
+	}
+
+end:
+	spin_unlock_irqrestore(&chan->rx_lock, flags);
+}
+
+static int qmc_chan_command(struct qmc_chan *chan, u8 qmc_opcode)
+{
+	return cpm_command(chan->id << 2, (qmc_opcode << 4) | 0x0E);
+}
+
+static int qmc_chan_stop_rx(struct qmc_chan *chan)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&chan->rx_lock, flags);
+
+	/* Send STOP RECEIVE command */
+	ret = qmc_chan_command(chan, 0x0);
+	if (ret) {
+		dev_err(chan->qmc->dev, "chan %u: Send STOP RECEIVE failed (%d)\n",
+			chan->id, ret);
+		goto end;
+	}
+
+	chan->is_rx_stopped = true;
+
+end:
+	spin_unlock_irqrestore(&chan->rx_lock, flags);
+	return ret;
+}
+
+static int qmc_chan_stop_tx(struct qmc_chan *chan)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&chan->tx_lock, flags);
+
+	/* Send STOP TRANSMIT command */
+	ret = qmc_chan_command(chan, 0x1);
+	if (ret) {
+		dev_err(chan->qmc->dev, "chan %u: Send STOP TRANSMIT failed (%d)\n",
+			chan->id, ret);
+		goto end;
+	}
+
+	chan->is_tx_stopped = true;
+
+end:
+	spin_unlock_irqrestore(&chan->tx_lock, flags);
+	return ret;
+}
+
+int qmc_chan_stop(struct qmc_chan *chan, int direction)
+{
+	int ret;
+
+	if (direction & QMC_CHAN_READ) {
+		ret = qmc_chan_stop_rx(chan);
+		if (ret)
+			return ret;
+	}
+
+	if (direction & QMC_CHAN_WRITE) {
+		ret = qmc_chan_stop_tx(chan);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(qmc_chan_stop);
+
+static void qmc_chan_start_rx(struct qmc_chan *chan)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&chan->rx_lock, flags);
+
+	/* Restart the receiver */
+	if (chan->mode == QMC_TRANSPARENT)
+		qmc_write32(chan->s_param + QMC_SPE_ZDSTATE, 0x18000080);
+	else
+		qmc_write32(chan->s_param + QMC_SPE_ZDSTATE, 0x00000080);
+	qmc_write32(chan->s_param + QMC_SPE_RSTATE, 0x31000000);
+	chan->is_rx_halted = false;
+
+	chan->is_rx_stopped = false;
+
+	spin_unlock_irqrestore(&chan->rx_lock, flags);
+}
+
+static void qmc_chan_start_tx(struct qmc_chan *chan)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&chan->tx_lock, flags);
+
+	/*
+	 * Enable channel transmitter as it could be disabled if
+	 * qmc_chan_reset() was called.
+	 */
+	qmc_setbits16(chan->s_param + QMC_SPE_CHAMR, QMC_SPE_CHAMR_ENT);
+
+	/* Set the POL bit in the channel mode register */
+	qmc_setbits16(chan->s_param + QMC_SPE_CHAMR, QMC_SPE_CHAMR_POL);
+
+	chan->is_tx_stopped = false;
+
+	spin_unlock_irqrestore(&chan->tx_lock, flags);
+}
+
+int qmc_chan_start(struct qmc_chan *chan, int direction)
+{
+	if (direction & QMC_CHAN_READ)
+		qmc_chan_start_rx(chan);
+
+	if (direction & QMC_CHAN_WRITE)
+		qmc_chan_start_tx(chan);
+
+	return 0;
+}
+EXPORT_SYMBOL(qmc_chan_start);
+
+static void qmc_chan_reset_rx(struct qmc_chan *chan)
+{
+	struct qmc_xfer_desc *xfer_desc;
+	unsigned long flags;
+	cbd_t *__iomem bd;
+	u16 ctrl;
+
+	spin_lock_irqsave(&chan->rx_lock, flags);
+	bd = chan->rxbds;
+	do {
+		ctrl = qmc_read16(&bd->cbd_sc);
+		qmc_write16(&bd->cbd_sc, ctrl & ~(QMC_BD_RX_UB | QMC_BD_RX_E));
+
+		xfer_desc = &chan->rx_desc[bd - chan->rxbds];
+		xfer_desc->rx_complete = NULL;
+		xfer_desc->context = NULL;
+
+		bd++;
+	} while (!(ctrl & QMC_BD_RX_W));
+
+	chan->rxbd_free = chan->rxbds;
+	chan->rxbd_done = chan->rxbds;
+	qmc_write16(chan->s_param + QMC_SPE_RBPTR,
+		    qmc_read16(chan->s_param + QMC_SPE_RBASE));
+
+	chan->rx_pending = 0;
+	chan->is_rx_stopped = false;
+
+	spin_unlock_irqrestore(&chan->rx_lock, flags);
+}
+
+static void qmc_chan_reset_tx(struct qmc_chan *chan)
+{
+	struct qmc_xfer_desc *xfer_desc;
+	unsigned long flags;
+	cbd_t *__iomem bd;
+	u16 ctrl;
+
+	spin_lock_irqsave(&chan->tx_lock, flags);
+
+	/* Disable transmitter. It will be re-enable on qmc_chan_start() */
+	qmc_clrbits16(chan->s_param + QMC_SPE_CHAMR, QMC_SPE_CHAMR_ENT);
+
+	bd = chan->txbds;
+	do {
+		ctrl = qmc_read16(&bd->cbd_sc);
+		qmc_write16(&bd->cbd_sc, ctrl & ~(QMC_BD_TX_UB | QMC_BD_TX_R));
+
+		xfer_desc = &chan->tx_desc[bd - chan->txbds];
+		xfer_desc->tx_complete = NULL;
+		xfer_desc->context = NULL;
+
+		bd++;
+	} while (!(ctrl & QMC_BD_TX_W));
+
+	chan->txbd_free = chan->txbds;
+	chan->txbd_done = chan->txbds;
+	qmc_write16(chan->s_param + QMC_SPE_TBPTR,
+		    qmc_read16(chan->s_param + QMC_SPE_TBASE));
+
+	/* Reset TSTATE and ZISTATE to their initial value */
+	qmc_write32(chan->s_param + QMC_SPE_TSTATE, 0x30000000);
+	qmc_write32(chan->s_param + QMC_SPE_ZISTATE, 0x00000100);
+
+	spin_unlock_irqrestore(&chan->tx_lock, flags);
+}
+
+int qmc_chan_reset(struct qmc_chan *chan, int direction)
+{
+	if (direction & QMC_CHAN_READ)
+		qmc_chan_reset_rx(chan);
+
+	if (direction & QMC_CHAN_WRITE)
+		qmc_chan_reset_tx(chan);
+
+	return 0;
+}
+EXPORT_SYMBOL(qmc_chan_reset);
+
+static int qmc_check_chans(struct qmc *qmc)
+{
+	struct tsa_serial_info info;
+	bool is_one_table = false;
+	struct qmc_chan *chan;
+	u64 tx_ts_mask = 0;
+	u64 rx_ts_mask = 0;
+	u64 tx_ts_assigned_mask;
+	u64 rx_ts_assigned_mask;
+	int ret;
+
+	/* Retrieve info from the TSA related serial */
+	ret = tsa_serial_get_info(qmc->tsa_serial, &info);
+	if (ret)
+		return ret;
+
+	if ((info.nb_tx_ts > 64) || (info.nb_rx_ts > 64)) {
+		dev_err(qmc->dev, "Number of TSA Tx/Rx TS assigned not supported\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * If more than 32 TS are assigned to this serial, one common table is
+	 * used for Tx and Rx and so masks must be equal for all channels.
+	 */
+	if ((info.nb_tx_ts > 32) || (info.nb_rx_ts > 32)) {
+		if (info.nb_tx_ts != info.nb_rx_ts) {
+			dev_err(qmc->dev, "Number of TSA Tx/Rx TS assigned are not equal\n");
+			return -EINVAL;
+		}
+		is_one_table = true;
+	}
+
+	tx_ts_assigned_mask = info.nb_tx_ts == 64 ? U64_MAX : (((u64)1) << info.nb_tx_ts) - 1;
+	rx_ts_assigned_mask = info.nb_rx_ts == 64 ? U64_MAX : (((u64)1) << info.nb_rx_ts) - 1;
+
+	list_for_each_entry(chan, &qmc->chan_head, list) {
+		if (chan->tx_ts_mask > tx_ts_assigned_mask) {
+			dev_err(qmc->dev, "chan %u uses TSA unassigned Tx TS\n", chan->id);
+			return -EINVAL;
+		}
+		if (tx_ts_mask & chan->tx_ts_mask) {
+			dev_err(qmc->dev, "chan %u uses an already used Tx TS\n", chan->id);
+			return -EINVAL;
+		}
+
+		if (chan->rx_ts_mask > rx_ts_assigned_mask) {
+			dev_err(qmc->dev, "chan %u uses TSA unassigned Rx TS\n", chan->id);
+			return -EINVAL;
+		}
+		if (rx_ts_mask & chan->rx_ts_mask) {
+			dev_err(qmc->dev, "chan %u uses an already used Rx TS\n", chan->id);
+			return -EINVAL;
+		}
+
+		if (is_one_table && (chan->tx_ts_mask != chan->rx_ts_mask)) {
+			dev_err(qmc->dev, "chan %u uses different Rx and Tx TS\n", chan->id);
+			return -EINVAL;
+		}
+
+		tx_ts_mask |= chan->tx_ts_mask;
+		rx_ts_mask |= chan->rx_ts_mask;
+	}
+
+	return 0;
+}
+
+static unsigned int qmc_nb_chans(struct qmc *qmc)
+{
+	unsigned int count = 0;
+	struct qmc_chan *chan;
+
+	list_for_each_entry(chan, &qmc->chan_head, list)
+		count++;
+
+	return count;
+}
+
+static int qmc_of_parse_chans(struct qmc *qmc, struct device_node *np)
+{
+	struct device_node *chan_np;
+	struct qmc_chan *chan;
+	const char *mode;
+	u32 chan_id;
+	u64 ts_mask;
+	int ret;
+
+	for_each_available_child_of_node(np, chan_np) {
+		ret = of_property_read_u32(chan_np, "reg", &chan_id);
+		if (ret) {
+			dev_err(qmc->dev, "%pOF: failed to read reg\n", chan_np);
+			of_node_put(chan_np);
+			return ret;
+		}
+		if (chan_id > 63) {
+			dev_err(qmc->dev, "%pOF: Invalid chan_id\n", chan_np);
+			of_node_put(chan_np);
+			return -EINVAL;
+		}
+
+		chan = devm_kzalloc(qmc->dev, sizeof(*chan), GFP_KERNEL);
+		if (!chan) {
+			of_node_put(chan_np);
+			return -ENOMEM;
+		}
+
+		chan->id = chan_id;
+		spin_lock_init(&chan->rx_lock);
+		spin_lock_init(&chan->tx_lock);
+
+		ret = of_property_read_u64(chan_np, "fsl,tx-ts-mask", &ts_mask);
+		if (ret) {
+			dev_err(qmc->dev, "%pOF: failed to read fsl,tx-ts-mask\n",
+				chan_np);
+			of_node_put(chan_np);
+			return ret;
+		}
+		chan->tx_ts_mask = ts_mask;
+
+		ret = of_property_read_u64(chan_np, "fsl,rx-ts-mask", &ts_mask);
+		if (ret) {
+			dev_err(qmc->dev, "%pOF: failed to read fsl,rx-ts-mask\n",
+				chan_np);
+			of_node_put(chan_np);
+			return ret;
+		}
+		chan->rx_ts_mask = ts_mask;
+
+		mode = "transparent";
+		ret = of_property_read_string(chan_np, "fsl,operational-mode", &mode);
+		if (ret && ret != -EINVAL) {
+			dev_err(qmc->dev, "%pOF: failed to read fsl,operational-mode\n",
+				chan_np);
+			of_node_put(chan_np);
+			return ret;
+		}
+		if (!strcmp(mode, "transparent")) {
+			chan->mode = QMC_TRANSPARENT;
+		} else if (!strcmp(mode, "hdlc")) {
+			chan->mode = QMC_HDLC;
+		} else {
+			dev_err(qmc->dev, "%pOF: Invalid fsl,operational-mode (%s)\n",
+				chan_np, mode);
+			of_node_put(chan_np);
+			return -EINVAL;
+		}
+
+		chan->is_reverse_data = of_property_read_bool(chan_np,
+							      "fsl,reverse-data");
+
+		list_add_tail(&chan->list, &qmc->chan_head);
+		qmc->chans[chan->id] = chan;
+	}
+
+	return qmc_check_chans(qmc);
+}
+
+static int qmc_setup_tsa_64rxtx(struct qmc *qmc, const struct tsa_serial_info *info)
+{
+	struct qmc_chan *chan;
+	unsigned int i;
+	u16 val;
+
+	/*
+	 * Use a common Tx/Rx 64 entries table.
+	 * Everything was previously checked, Tx and Rx related stuffs are
+	 * identical -> Used Rx related stuff to build the table
+	 */
+
+	/* Invalidate all entries */
+	for (i = 0; i < 64; i++)
+		qmc_write16(qmc->scc_pram + QMC_GBL_TSATRX + (i * 2), 0x0000);
+
+	/* Set entries based on Rx stuff*/
+	list_for_each_entry(chan, &qmc->chan_head, list) {
+		for (i = 0; i < info->nb_rx_ts; i++) {
+			if (!(chan->rx_ts_mask & (((u64)1) << i)))
+				continue;
+
+			val = QMC_TSA_VALID | QMC_TSA_MASK |
+			      QMC_TSA_CHANNEL(chan->id);
+			qmc_write16(qmc->scc_pram + QMC_GBL_TSATRX + (i * 2), val);
+		}
+	}
+
+	/* Set Wrap bit on last entry */
+	qmc_setbits16(qmc->scc_pram + QMC_GBL_TSATRX + ((info->nb_rx_ts - 1) * 2),
+		      QMC_TSA_WRAP);
+
+	/* Init pointers to the table */
+	val = qmc->scc_pram_offset + QMC_GBL_TSATRX;
+	qmc_write16(qmc->scc_pram + QMC_GBL_RX_S_PTR, val);
+	qmc_write16(qmc->scc_pram + QMC_GBL_RXPTR, val);
+	qmc_write16(qmc->scc_pram + QMC_GBL_TX_S_PTR, val);
+	qmc_write16(qmc->scc_pram + QMC_GBL_TXPTR, val);
+
+	return 0;
+}
+
+static int qmc_setup_tsa_32rx_32tx(struct qmc *qmc, const struct tsa_serial_info *info)
+{
+	struct qmc_chan *chan;
+	unsigned int i;
+	u16 val;
+
+	/*
+	 * Use a Tx 32 entries table and a Rx 32 entries table.
+	 * Everything was previously checked.
+	 */
+
+	/* Invalidate all entries */
+	for (i = 0; i < 32; i++) {
+		qmc_write16(qmc->scc_pram + QMC_GBL_TSATRX + (i * 2), 0x0000);
+		qmc_write16(qmc->scc_pram + QMC_GBL_TSATTX + (i * 2), 0x0000);
+	}
+
+	/* Set entries based on Rx and Tx stuff*/
+	list_for_each_entry(chan, &qmc->chan_head, list) {
+		/* Rx part */
+		for (i = 0; i < info->nb_rx_ts; i++) {
+			if (!(chan->rx_ts_mask & (((u64)1) << i)))
+				continue;
+
+			val = QMC_TSA_VALID | QMC_TSA_MASK |
+			      QMC_TSA_CHANNEL(chan->id);
+			qmc_write16(qmc->scc_pram + QMC_GBL_TSATRX + (i * 2), val);
+		}
+		/* Tx part */
+		for (i = 0; i < info->nb_tx_ts; i++) {
+			if (!(chan->tx_ts_mask & (((u64)1) << i)))
+				continue;
+
+			val = QMC_TSA_VALID | QMC_TSA_MASK |
+			      QMC_TSA_CHANNEL(chan->id);
+			qmc_write16(qmc->scc_pram + QMC_GBL_TSATTX + (i * 2), val);
+		}
+	}
+
+	/* Set Wrap bit on last entries */
+	qmc_setbits16(qmc->scc_pram + QMC_GBL_TSATRX + ((info->nb_rx_ts - 1) * 2),
+		      QMC_TSA_WRAP);
+	qmc_setbits16(qmc->scc_pram + QMC_GBL_TSATTX + ((info->nb_tx_ts - 1) * 2),
+		      QMC_TSA_WRAP);
+
+	/* Init Rx pointers ...*/
+	val = qmc->scc_pram_offset + QMC_GBL_TSATRX;
+	qmc_write16(qmc->scc_pram + QMC_GBL_RX_S_PTR, val);
+	qmc_write16(qmc->scc_pram + QMC_GBL_RXPTR, val);
+
+	/* ... and Tx pointers */
+	val = qmc->scc_pram_offset + QMC_GBL_TSATTX;
+	qmc_write16(qmc->scc_pram + QMC_GBL_TX_S_PTR, val);
+	qmc_write16(qmc->scc_pram + QMC_GBL_TXPTR, val);
+
+	return 0;
+}
+
+static int qmc_setup_tsa(struct qmc *qmc)
+{
+	struct tsa_serial_info info;
+	int ret;
+
+	/* Retrieve info from the TSA related serial */
+	ret = tsa_serial_get_info(qmc->tsa_serial, &info);
+	if (ret)
+		return ret;
+
+	/*
+	 * Setup one common 64 entries table or two 32 entries (one for Tx and
+	 * one for Tx) according to assigned TS numbers.
+	 */
+	return ((info.nb_tx_ts > 32) || (info.nb_rx_ts > 32)) ?
+		qmc_setup_tsa_64rxtx(qmc, &info) :
+		qmc_setup_tsa_32rx_32tx(qmc, &info);
+}
+
+static int qmc_setup_chan_trnsync(struct qmc *qmc, struct qmc_chan *chan)
+{
+	struct tsa_serial_info info;
+	u16 first_rx, last_tx;
+	u16 trnsync;
+	int ret;
+
+	/* Retrieve info from the TSA related serial */
+	ret = tsa_serial_get_info(chan->qmc->tsa_serial, &info);
+	if (ret)
+		return ret;
+
+	/* Find the first Rx TS allocated to the channel */
+	first_rx = chan->rx_ts_mask ? __ffs64(chan->rx_ts_mask) + 1 : 0;
+
+	/* Find the last Tx TS allocated to the channel */
+	last_tx = fls64(chan->tx_ts_mask);
+
+	trnsync = 0;
+	if (info.nb_rx_ts)
+		trnsync |= QMC_SPE_TRNSYNC_RX((first_rx % info.nb_rx_ts) * 2);
+	if (info.nb_tx_ts)
+		trnsync |= QMC_SPE_TRNSYNC_TX((last_tx % info.nb_tx_ts) * 2);
+
+	qmc_write16(chan->s_param + QMC_SPE_TRNSYNC, trnsync);
+
+	dev_dbg(qmc->dev, "chan %u: trnsync=0x%04x, rx %u/%u 0x%llx, tx %u/%u 0x%llx\n",
+		chan->id, trnsync,
+		first_rx, info.nb_rx_ts, chan->rx_ts_mask,
+		last_tx, info.nb_tx_ts, chan->tx_ts_mask);
+
+	return 0;
+}
+
+static int qmc_setup_chan(struct qmc *qmc, struct qmc_chan *chan)
+{
+	unsigned int i;
+	cbd_t __iomem *bd;
+	int ret;
+	u16 val;
+
+	chan->qmc = qmc;
+
+	/* Set channel specific parameter base address */
+	chan->s_param = qmc->dpram + (chan->id * 64);
+	/* 16 bd per channel (8 rx and 8 tx) */
+	chan->txbds = qmc->bd_table + (chan->id * (QMC_NB_TXBDS + QMC_NB_RXBDS));
+	chan->rxbds = qmc->bd_table + (chan->id * (QMC_NB_TXBDS + QMC_NB_RXBDS)) + QMC_NB_TXBDS;
+
+	chan->txbd_free = chan->txbds;
+	chan->txbd_done = chan->txbds;
+	chan->rxbd_free = chan->rxbds;
+	chan->rxbd_done = chan->rxbds;
+
+	/* TBASE and TBPTR*/
+	val = chan->id * (QMC_NB_TXBDS + QMC_NB_RXBDS) * sizeof(cbd_t);
+	qmc_write16(chan->s_param + QMC_SPE_TBASE, val);
+	qmc_write16(chan->s_param + QMC_SPE_TBPTR, val);
+
+	/* RBASE and RBPTR*/
+	val = ((chan->id * (QMC_NB_TXBDS + QMC_NB_RXBDS)) + QMC_NB_TXBDS) * sizeof(cbd_t);
+	qmc_write16(chan->s_param + QMC_SPE_RBASE, val);
+	qmc_write16(chan->s_param + QMC_SPE_RBPTR, val);
+	qmc_write32(chan->s_param + QMC_SPE_TSTATE, 0x30000000);
+	qmc_write32(chan->s_param + QMC_SPE_RSTATE, 0x31000000);
+	qmc_write32(chan->s_param + QMC_SPE_ZISTATE, 0x00000100);
+	if (chan->mode == QMC_TRANSPARENT) {
+		qmc_write32(chan->s_param + QMC_SPE_ZDSTATE, 0x18000080);
+		qmc_write16(chan->s_param + QMC_SPE_TMRBLR, 60);
+		val = QMC_SPE_CHAMR_MODE_TRANSP | QMC_SPE_CHAMR_TRANSP_SYNC;
+		if (chan->is_reverse_data)
+			val |= QMC_SPE_CHAMR_TRANSP_RD;
+		qmc_write16(chan->s_param + QMC_SPE_CHAMR, val);
+		ret = qmc_setup_chan_trnsync(qmc, chan);
+		if (ret)
+			return ret;
+	} else {
+		qmc_write32(chan->s_param + QMC_SPE_ZDSTATE, 0x00000080);
+		qmc_write16(chan->s_param + QMC_SPE_MFLR, 60);
+		qmc_write16(chan->s_param + QMC_SPE_CHAMR,
+			QMC_SPE_CHAMR_MODE_HDLC | QMC_SPE_CHAMR_HDLC_IDLM);
+	}
+
+	/* Do not enable interrupts now. They will be enabled later */
+	qmc_write16(chan->s_param + QMC_SPE_INTMSK, 0x0000);
+
+	/* Init Rx BDs and set Wrap bit on last descriptor */
+	BUILD_BUG_ON(QMC_NB_RXBDS == 0);
+	val = QMC_BD_RX_I;
+	for (i = 0; i < QMC_NB_RXBDS; i++) {
+		bd = chan->rxbds + i;
+		qmc_write16(&bd->cbd_sc, val);
+	}
+	bd = chan->rxbds + QMC_NB_RXBDS - 1;
+	qmc_write16(&bd->cbd_sc, val | QMC_BD_RX_W);
+
+	/* Init Tx BDs and set Wrap bit on last descriptor */
+	BUILD_BUG_ON(QMC_NB_TXBDS == 0);
+	val = QMC_BD_TX_I;
+	if (chan->mode == QMC_HDLC)
+		val |= QMC_BD_TX_L | QMC_BD_TX_TC;
+	for (i = 0; i < QMC_NB_TXBDS; i++) {
+		bd = chan->txbds + i;
+		qmc_write16(&bd->cbd_sc, val);
+	}
+	bd = chan->txbds + QMC_NB_TXBDS - 1;
+	qmc_write16(&bd->cbd_sc, val | QMC_BD_TX_W);
+
+	return 0;
+}
+
+static int qmc_setup_chans(struct qmc *qmc)
+{
+	struct qmc_chan *chan;
+	int ret;
+
+	list_for_each_entry(chan, &qmc->chan_head, list) {
+		ret = qmc_setup_chan(qmc, chan);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int qmc_finalize_chans(struct qmc *qmc)
+{
+	struct qmc_chan *chan;
+	int ret;
+
+	list_for_each_entry(chan, &qmc->chan_head, list) {
+		/* Unmask channel interrupts */
+		if (chan->mode == QMC_HDLC) {
+			qmc_write16(chan->s_param + QMC_SPE_INTMSK,
+				    QMC_INT_NID | QMC_INT_IDL | QMC_INT_MRF |
+				    QMC_INT_UN | QMC_INT_RXF | QMC_INT_BSY |
+				    QMC_INT_TXB | QMC_INT_RXB);
+		} else {
+			qmc_write16(chan->s_param + QMC_SPE_INTMSK,
+				    QMC_INT_UN | QMC_INT_BSY |
+				    QMC_INT_TXB | QMC_INT_RXB);
+		}
+
+		/* Forced stop the channel */
+		ret = qmc_chan_stop(chan, QMC_CHAN_ALL);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int qmc_setup_ints(struct qmc *qmc)
+{
+	unsigned int i;
+	u16 __iomem *last;
+
+	/* Raz all entries */
+	for (i = 0; i < (qmc->int_size / sizeof(u16)); i++)
+		qmc_write16(qmc->int_table + i, 0x0000);
+
+	/* Set Wrap bit on last entry */
+	if (qmc->int_size >= sizeof(u16)) {
+		last = qmc->int_table + (qmc->int_size / sizeof(u16)) - 1;
+		qmc_write16(last, QMC_INT_W);
+	}
+
+	return 0;
+}
+
+static void qmc_irq_gint(struct qmc *qmc)
+{
+	struct qmc_chan *chan;
+	unsigned int chan_id;
+	unsigned long flags;
+	u16 int_entry;
+
+	int_entry = qmc_read16(qmc->int_curr);
+	while (int_entry & QMC_INT_V) {
+		/* Clear all but the Wrap bit */
+		qmc_write16(qmc->int_curr, int_entry & QMC_INT_W);
+
+		chan_id = QMC_INT_GET_CHANNEL(int_entry);
+		chan = qmc->chans[chan_id];
+		if (!chan) {
+			dev_err(qmc->dev, "interrupt on invalid chan %u\n", chan_id);
+			goto int_next;
+		}
+
+		if (int_entry & QMC_INT_TXB)
+			qmc_chan_write_done(chan);
+
+		if (int_entry & QMC_INT_UN) {
+			dev_info(qmc->dev, "intr chan %u, 0x%04x (UN)\n", chan_id,
+				 int_entry);
+			chan->nb_tx_underrun++;
+		}
+
+		if (int_entry & QMC_INT_BSY) {
+			dev_info(qmc->dev, "intr chan %u, 0x%04x (BSY)\n", chan_id,
+				 int_entry);
+			chan->nb_rx_busy++;
+			/* Restart the receiver if needed */
+			spin_lock_irqsave(&chan->rx_lock, flags);
+			if (chan->rx_pending && !chan->is_rx_stopped) {
+				if (chan->mode == QMC_TRANSPARENT)
+					qmc_write32(chan->s_param + QMC_SPE_ZDSTATE, 0x18000080);
+				else
+					qmc_write32(chan->s_param + QMC_SPE_ZDSTATE, 0x00000080);
+				qmc_write32(chan->s_param + QMC_SPE_RSTATE, 0x31000000);
+				chan->is_rx_halted = false;
+			} else {
+				chan->is_rx_halted = true;
+			}
+			spin_unlock_irqrestore(&chan->rx_lock, flags);
+		}
+
+		if (int_entry & QMC_INT_RXB)
+			qmc_chan_read_done(chan);
+
+int_next:
+		if (int_entry & QMC_INT_W)
+			qmc->int_curr = qmc->int_table;
+		else
+			qmc->int_curr++;
+		int_entry = qmc_read16(qmc->int_curr);
+	}
+}
+
+static irqreturn_t qmc_irq_handler(int irq, void *priv)
+{
+	struct qmc *qmc = (struct qmc *)priv;
+	u16 scce;
+
+	scce = qmc_read16(qmc->scc_regs + SCC_SCCE);
+	qmc_write16(qmc->scc_regs + SCC_SCCE, scce);
+
+	if (unlikely(scce & SCC_SCCE_IQOV))
+		dev_info(qmc->dev, "IRQ queue overflow\n");
+
+	if (unlikely(scce & SCC_SCCE_GUN))
+		dev_err(qmc->dev, "Global transmitter underrun\n");
+
+	if (unlikely(scce & SCC_SCCE_GOV))
+		dev_err(qmc->dev, "Global receiver overrun\n");
+
+	/* normal interrupt */
+	if (likely(scce & SCC_SCCE_GINT))
+		qmc_irq_gint(qmc);
+
+	return IRQ_HANDLED;
+}
+
+static int qmc_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	unsigned int nb_chans;
+	struct resource *res;
+	struct qmc *qmc;
+	int irq;
+	int ret;
+
+	qmc = devm_kzalloc(&pdev->dev, sizeof(*qmc), GFP_KERNEL);
+	if (!qmc)
+		return -ENOMEM;
+
+	qmc->dev = &pdev->dev;
+	INIT_LIST_HEAD(&qmc->chan_head);
+
+	qmc->scc_regs = devm_platform_ioremap_resource_byname(pdev, "scc_regs");
+	if (IS_ERR(qmc->scc_regs))
+		return PTR_ERR(qmc->scc_regs);
+
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "scc_pram");
+	if (!res)
+		return -EINVAL;
+	qmc->scc_pram_offset = res->start - get_immrbase();
+	qmc->scc_pram = devm_ioremap_resource(qmc->dev, res);
+	if (IS_ERR(qmc->scc_pram))
+		return PTR_ERR(qmc->scc_pram);
+
+	qmc->dpram  = devm_platform_ioremap_resource_byname(pdev, "dpram");
+	if (IS_ERR(qmc->dpram))
+		return PTR_ERR(qmc->dpram);
+
+	qmc->tsa_serial = devm_tsa_serial_get_byphandle(qmc->dev, np, "fsl,tsa-serial");
+	if (IS_ERR(qmc->tsa_serial)) {
+		return dev_err_probe(qmc->dev, PTR_ERR(qmc->tsa_serial),
+				     "Failed to get TSA serial\n");
+	}
+
+	/* Connect the serial (SCC) to TSA */
+	ret = tsa_serial_connect(qmc->tsa_serial);
+	if (ret) {
+		dev_err(qmc->dev, "Failed to connect TSA serial\n");
+		return ret;
+	}
+
+	/* Parse channels informationss */
+	ret = qmc_of_parse_chans(qmc, np);
+	if (ret)
+		goto err_tsa_serial_disconnect;
+
+	nb_chans = qmc_nb_chans(qmc);
+
+	/* Init GMSR_H and GMSR_L registers */
+	qmc_write32(qmc->scc_regs + SCC_GSMRH,
+		    SCC_GSMRH_CDS | SCC_GSMRH_CTSS | SCC_GSMRH_CDP | SCC_GSMRH_CTSP);
+
+	/* enable QMC mode */
+	qmc_write32(qmc->scc_regs + SCC_GSMRL, SCC_GSMRL_MODE_QMC);
+
+	/*
+	 * Allocate the buffer descriptor table
+	 * 8 rx and 8 tx descriptors per channel
+	 */
+	qmc->bd_size = (nb_chans * (QMC_NB_TXBDS + QMC_NB_RXBDS)) * sizeof(cbd_t);
+	qmc->bd_table = dmam_alloc_coherent(qmc->dev, qmc->bd_size,
+		&qmc->bd_dma_addr, GFP_KERNEL);
+	if (!qmc->bd_table) {
+		dev_err(qmc->dev, "Failed to allocate bd table\n");
+		ret = -ENOMEM;
+		goto err_tsa_serial_disconnect;
+	}
+	memset(qmc->bd_table, 0, qmc->bd_size);
+
+	qmc_write32(qmc->scc_pram + QMC_GBL_MCBASE, qmc->bd_dma_addr);
+
+	/* Allocate the interrupt table */
+	qmc->int_size = QMC_NB_INTS * sizeof(u16);
+	qmc->int_table = dmam_alloc_coherent(qmc->dev, qmc->int_size,
+		&qmc->int_dma_addr, GFP_KERNEL);
+	if (!qmc->int_table) {
+		dev_err(qmc->dev, "Failed to allocate interrupt table\n");
+		ret = -ENOMEM;
+		goto err_tsa_serial_disconnect;
+	}
+	memset(qmc->int_table, 0, qmc->int_size);
+
+	qmc->int_curr = qmc->int_table;
+	qmc_write32(qmc->scc_pram + QMC_GBL_INTBASE, qmc->int_dma_addr);
+	qmc_write32(qmc->scc_pram + QMC_GBL_INTPTR, qmc->int_dma_addr);
+
+	/* Set MRBLR (valid for HDLC only) max MRU + max CRC */
+	qmc_write16(qmc->scc_pram + QMC_GBL_MRBLR, HDLC_MAX_MRU + 4);
+
+	qmc_write16(qmc->scc_pram + QMC_GBL_GRFTHR, 1);
+	qmc_write16(qmc->scc_pram + QMC_GBL_GRFCNT, 1);
+
+	qmc_write32(qmc->scc_pram + QMC_GBL_C_MASK32, 0xDEBB20E3);
+	qmc_write16(qmc->scc_pram + QMC_GBL_C_MASK16, 0xF0B8);
+
+	ret = qmc_setup_tsa(qmc);
+	if (ret)
+		goto err_tsa_serial_disconnect;
+
+	qmc_write16(qmc->scc_pram + QMC_GBL_QMCSTATE, 0x8000);
+
+	ret = qmc_setup_chans(qmc);
+	if (ret)
+		goto err_tsa_serial_disconnect;
+
+	/* Init interrupts table */
+	ret = qmc_setup_ints(qmc);
+	if (ret)
+		goto err_tsa_serial_disconnect;
+
+	/* Disable and clear interrupts,  set the irq handler */
+	qmc_write16(qmc->scc_regs + SCC_SCCM, 0x0000);
+	qmc_write16(qmc->scc_regs + SCC_SCCE, 0x000F);
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		goto err_tsa_serial_disconnect;
+	ret = devm_request_irq(qmc->dev, irq, qmc_irq_handler, 0, "qmc", qmc);
+	if (ret < 0)
+		goto err_tsa_serial_disconnect;
+
+	/* Enable interrupts */
+	qmc_write16(qmc->scc_regs + SCC_SCCM,
+		SCC_SCCE_IQOV | SCC_SCCE_GINT | SCC_SCCE_GUN | SCC_SCCE_GOV);
+
+	ret = qmc_finalize_chans(qmc);
+	if (ret < 0)
+		goto err_disable_intr;
+
+	/* Enable transmiter and receiver */
+	qmc_setbits32(qmc->scc_regs + SCC_GSMRL, SCC_GSMRL_ENR | SCC_GSMRL_ENT);
+
+	platform_set_drvdata(pdev, qmc);
+
+	return 0;
+
+err_disable_intr:
+	qmc_write16(qmc->scc_regs + SCC_SCCM, 0);
+
+err_tsa_serial_disconnect:
+	tsa_serial_disconnect(qmc->tsa_serial);
+	return ret;
+}
+
+static int qmc_remove(struct platform_device *pdev)
+{
+	struct qmc *qmc = platform_get_drvdata(pdev);
+
+	/* Disable transmiter and receiver */
+	qmc_setbits32(qmc->scc_regs + SCC_GSMRL, 0);
+
+	/* Disable interrupts */
+	qmc_write16(qmc->scc_regs + SCC_SCCM, 0);
+
+	/* Disconnect the serial from TSA */
+	tsa_serial_disconnect(qmc->tsa_serial);
+
+	return 0;
+}
+
+static const struct of_device_id qmc_id_table[] = {
+	{ .compatible = "fsl,cpm1-scc-qmc" },
+	{} /* sentinel */
+};
+MODULE_DEVICE_TABLE(of, qmc_id_table);
+
+static struct platform_driver qmc_driver = {
+	.driver = {
+		.name = "fsl-qmc",
+		.of_match_table = of_match_ptr(qmc_id_table),
+	},
+	.probe = qmc_probe,
+	.remove = qmc_remove,
+};
+module_platform_driver(qmc_driver);
+
+struct qmc_chan *qmc_chan_get_byphandle(struct device_node *np, const char *phandle_name)
+{
+	struct of_phandle_args out_args;
+	struct platform_device *pdev;
+	struct qmc_chan *qmc_chan;
+	struct qmc *qmc;
+	int ret;
+
+	ret = of_parse_phandle_with_fixed_args(np, phandle_name, 1, 0,
+					       &out_args);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	if (!of_match_node(qmc_driver.driver.of_match_table, out_args.np)) {
+		of_node_put(out_args.np);
+		return ERR_PTR(-EINVAL);
+	}
+
+	pdev = of_find_device_by_node(out_args.np);
+	of_node_put(out_args.np);
+	if (!pdev)
+		return ERR_PTR(-ENODEV);
+
+	qmc = platform_get_drvdata(pdev);
+	if (!qmc) {
+		platform_device_put(pdev);
+		return ERR_PTR(-EPROBE_DEFER);
+	}
+
+	if (out_args.args_count != 1) {
+		platform_device_put(pdev);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (out_args.args[0] >= ARRAY_SIZE(qmc->chans)) {
+		platform_device_put(pdev);
+		return ERR_PTR(-EINVAL);
+	}
+
+	qmc_chan = qmc->chans[out_args.args[0]];
+	if (!qmc_chan) {
+		platform_device_put(pdev);
+		return ERR_PTR(-ENOENT);
+	}
+
+	return qmc_chan;
+}
+EXPORT_SYMBOL(qmc_chan_get_byphandle);
+
+void qmc_chan_put(struct qmc_chan *chan)
+{
+	put_device(chan->qmc->dev);
+}
+EXPORT_SYMBOL(qmc_chan_put);
+
+static void devm_qmc_chan_release(struct device *dev, void *res)
+{
+	struct qmc_chan **qmc_chan = res;
+
+	qmc_chan_put(*qmc_chan);
+}
+
+struct qmc_chan *devm_qmc_chan_get_byphandle(struct device *dev,
+					     struct device_node *np,
+					     const char *phandle_name)
+{
+	struct qmc_chan *qmc_chan;
+	struct qmc_chan **dr;
+
+	dr = devres_alloc(devm_qmc_chan_release, sizeof(*dr), GFP_KERNEL);
+	if (!dr)
+		return ERR_PTR(-ENOMEM);
+
+	qmc_chan = qmc_chan_get_byphandle(np, phandle_name);
+	if (!IS_ERR(qmc_chan)) {
+		*dr = qmc_chan;
+		devres_add(dev, dr);
+	} else {
+		devres_free(dr);
+	}
+
+	return qmc_chan;
+}
+EXPORT_SYMBOL(devm_qmc_chan_get_byphandle);
+
+MODULE_AUTHOR("Herve Codina <herve.codina@bootlin.com>");
+MODULE_DESCRIPTION("CPM QMC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/fsl/qe/tsa.c b/drivers/soc/fsl/qe/tsa.c
new file mode 100644
index 000000000000..3646153117b3
--- /dev/null
+++ b/drivers/soc/fsl/qe/tsa.c
@@ -0,0 +1,846 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TSA driver
+ *
+ * Copyright 2022 CS GROUP France
+ *
+ * Author: Herve Codina <herve.codina@bootlin.com>
+ */
+
+#include "tsa.h"
+#include <dt-bindings/soc/cpm1-fsl,tsa.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+
+/* TSA SI RAM routing tables entry */
+#define TSA_SIRAM_ENTRY_LAST		(1 << 16)
+#define TSA_SIRAM_ENTRY_BYTE		(1 << 17)
+#define TSA_SIRAM_ENTRY_CNT(x)		(((x) & 0x0f) << 18)
+#define TSA_SIRAM_ENTRY_CSEL_MASK	(0x7 << 22)
+#define TSA_SIRAM_ENTRY_CSEL_NU		(0x0 << 22)
+#define TSA_SIRAM_ENTRY_CSEL_SCC2	(0x2 << 22)
+#define TSA_SIRAM_ENTRY_CSEL_SCC3	(0x3 << 22)
+#define TSA_SIRAM_ENTRY_CSEL_SCC4	(0x4 << 22)
+#define TSA_SIRAM_ENTRY_CSEL_SMC1	(0x5 << 22)
+#define TSA_SIRAM_ENTRY_CSEL_SMC2	(0x6 << 22)
+
+/* SI mode register (32 bits) */
+#define TSA_SIMODE	0x00
+#define   TSA_SIMODE_SMC2			0x80000000
+#define   TSA_SIMODE_SMC1			0x00008000
+#define   TSA_SIMODE_TDMA(x)			((x) << 0)
+#define   TSA_SIMODE_TDMB(x)			((x) << 16)
+#define     TSA_SIMODE_TDM_MASK			0x0fff
+#define     TSA_SIMODE_TDM_SDM_MASK		0x0c00
+#define       TSA_SIMODE_TDM_SDM_NORM		0x0000
+#define       TSA_SIMODE_TDM_SDM_ECHO		0x0400
+#define       TSA_SIMODE_TDM_SDM_INTL_LOOP	0x0800
+#define       TSA_SIMODE_TDM_SDM_LOOP_CTRL	0x0c00
+#define     TSA_SIMODE_TDM_RFSD(x)		((x) << 8)
+#define     TSA_SIMODE_TDM_DSC			0x0080
+#define     TSA_SIMODE_TDM_CRT			0x0040
+#define     TSA_SIMODE_TDM_STZ			0x0020
+#define     TSA_SIMODE_TDM_CE			0x0010
+#define     TSA_SIMODE_TDM_FE			0x0008
+#define     TSA_SIMODE_TDM_GM			0x0004
+#define     TSA_SIMODE_TDM_TFSD(x)		((x) << 0)
+
+/* SI global mode register (8 bits) */
+#define TSA_SIGMR	0x04
+#define TSA_SIGMR_ENB			(1<<3)
+#define TSA_SIGMR_ENA			(1<<2)
+#define TSA_SIGMR_RDM_MASK		0x03
+#define   TSA_SIGMR_RDM_STATIC_TDMA	0x00
+#define   TSA_SIGMR_RDM_DYN_TDMA	0x01
+#define   TSA_SIGMR_RDM_STATIC_TDMAB	0x02
+#define   TSA_SIGMR_RDM_DYN_TDMAB	0x03
+
+/* SI status register (8 bits) */
+#define TSA_SISTR	0x06
+
+/* SI command register (8 bits) */
+#define TSA_SICMR	0x07
+
+/* SI clock route register (32 bits) */
+#define TSA_SICR	0x0C
+#define   TSA_SICR_SCC2(x)		((x) << 8)
+#define   TSA_SICR_SCC3(x)		((x) << 16)
+#define   TSA_SICR_SCC4(x)		((x) << 24)
+#define     TSA_SICR_SCC_MASK		0x0ff
+#define     TSA_SICR_SCC_GRX		(1 << 7)
+#define     TSA_SICR_SCC_SCX_TSA	(1 << 6)
+#define     TSA_SICR_SCC_RXCS_MASK	(0x7 << 3)
+#define       TSA_SICR_SCC_RXCS_BRG1	(0x0 << 3)
+#define       TSA_SICR_SCC_RXCS_BRG2	(0x1 << 3)
+#define       TSA_SICR_SCC_RXCS_BRG3	(0x2 << 3)
+#define       TSA_SICR_SCC_RXCS_BRG4	(0x3 << 3)
+#define       TSA_SICR_SCC_RXCS_CLK15	(0x4 << 3)
+#define       TSA_SICR_SCC_RXCS_CLK26	(0x5 << 3)
+#define       TSA_SICR_SCC_RXCS_CLK37	(0x6 << 3)
+#define       TSA_SICR_SCC_RXCS_CLK48	(0x7 << 3)
+#define     TSA_SICR_SCC_TXCS_MASK	(0x7 << 0)
+#define       TSA_SICR_SCC_TXCS_BRG1	(0x0 << 0)
+#define       TSA_SICR_SCC_TXCS_BRG2	(0x1 << 0)
+#define       TSA_SICR_SCC_TXCS_BRG3	(0x2 << 0)
+#define       TSA_SICR_SCC_TXCS_BRG4	(0x3 << 0)
+#define       TSA_SICR_SCC_TXCS_CLK15	(0x4 << 0)
+#define       TSA_SICR_SCC_TXCS_CLK26	(0x5 << 0)
+#define       TSA_SICR_SCC_TXCS_CLK37	(0x6 << 0)
+#define       TSA_SICR_SCC_TXCS_CLK48	(0x7 << 0)
+
+/* Serial interface RAM pointer register (32 bits) */
+#define TSA_SIRP	0x10
+
+struct tsa_entries_area {
+	void *__iomem entries_start;
+	void *__iomem entries_next;
+	void *__iomem last_entry;
+};
+
+struct tsa_tdm {
+	bool is_enable;
+	struct clk *l1rclk_clk;
+	struct clk *l1rsync_clk;
+	struct clk *l1tclk_clk;
+	struct clk *l1tsync_clk;
+	u32 simode_tdm;
+};
+
+#define TSA_TDMA	0
+#define TSA_TDMB	1
+
+struct tsa {
+	struct device *dev;
+	void *__iomem si_regs;
+	void *__iomem si_ram;
+	resource_size_t si_ram_sz;
+	spinlock_t	lock;
+	int tdms; /* TSA_TDMx ORed */
+	struct tsa_tdm tdm[2]; /* TDMa and TDMb */
+	struct tsa_serial {
+		unsigned int id;
+		struct tsa_serial_info info;
+	} serials[6];
+};
+
+static inline struct tsa *tsa_serial_get_tsa(struct tsa_serial *tsa_serial)
+{
+	/* The serials table is indexed by the serial id */
+	return container_of(tsa_serial, struct tsa, serials[tsa_serial->id]);
+}
+
+static inline void tsa_write32(void *__iomem addr, u32 val)
+{
+	iowrite32be(val, addr);
+}
+
+static inline void tsa_write8(void *__iomem addr, u32 val)
+{
+	iowrite8(val, addr);
+}
+
+static inline u32 tsa_read32(void *__iomem addr)
+{
+	return ioread32be(addr);
+}
+
+static inline void tsa_clrbits32(void *__iomem addr, u32 clr)
+{
+	tsa_write32(addr, tsa_read32(addr) & ~clr);
+}
+
+static inline void tsa_clrsetbits32(void *__iomem addr, u32 clr, u32 set)
+{
+	tsa_write32(addr, (tsa_read32(addr) & ~clr) | set);
+}
+
+int tsa_serial_connect(struct tsa_serial *tsa_serial)
+{
+	struct tsa *tsa = tsa_serial_get_tsa(tsa_serial);
+	unsigned long flags;
+	u32 clear;
+	u32 set;
+
+	switch (tsa_serial->id) {
+	case FSL_CPM_TSA_SCC2:
+		clear = TSA_SICR_SCC2(TSA_SICR_SCC_MASK);
+		set = TSA_SICR_SCC2(TSA_SICR_SCC_SCX_TSA);
+		break;
+	case FSL_CPM_TSA_SCC3:
+		clear = TSA_SICR_SCC3(TSA_SICR_SCC_MASK);
+		set = TSA_SICR_SCC3(TSA_SICR_SCC_SCX_TSA);
+		break;
+	case FSL_CPM_TSA_SCC4:
+		clear = TSA_SICR_SCC4(TSA_SICR_SCC_MASK);
+		set = TSA_SICR_SCC4(TSA_SICR_SCC_SCX_TSA);
+		break;
+	default:
+		dev_err(tsa->dev, "Unsupported serial id %u\n", tsa_serial->id);
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&tsa->lock, flags);
+	tsa_clrsetbits32(tsa->si_regs + TSA_SICR, clear, set);
+	spin_unlock_irqrestore(&tsa->lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL(tsa_serial_connect);
+
+int tsa_serial_disconnect(struct tsa_serial *tsa_serial)
+{
+	struct tsa *tsa = tsa_serial_get_tsa(tsa_serial);
+	unsigned long flags;
+	u32 clear;
+
+	switch (tsa_serial->id) {
+	case FSL_CPM_TSA_SCC2:
+		clear = TSA_SICR_SCC2(TSA_SICR_SCC_MASK);
+		break;
+	case FSL_CPM_TSA_SCC3:
+		clear = TSA_SICR_SCC3(TSA_SICR_SCC_MASK);
+		break;
+	case FSL_CPM_TSA_SCC4:
+		clear = TSA_SICR_SCC4(TSA_SICR_SCC_MASK);
+		break;
+	default:
+		dev_err(tsa->dev, "Unsupported serial id %u\n", tsa_serial->id);
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&tsa->lock, flags);
+	tsa_clrsetbits32(tsa->si_regs + TSA_SICR, clear, 0);
+	spin_unlock_irqrestore(&tsa->lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL(tsa_serial_disconnect);
+
+int tsa_serial_get_info(struct tsa_serial *tsa_serial, struct tsa_serial_info *info)
+{
+	memcpy(info, &tsa_serial->info, sizeof(*info));
+	return 0;
+}
+EXPORT_SYMBOL(tsa_serial_get_info);
+
+static void tsa_init_entries_area(struct tsa *tsa, struct tsa_entries_area *area,
+				  u32 tdms, u32 tdm_id, bool is_rx)
+{
+	resource_size_t quarter;
+	resource_size_t half;
+
+	quarter = tsa->si_ram_sz/4;
+	half = tsa->si_ram_sz/2;
+
+	if (tdms == BIT(TSA_TDMA)) {
+		/* Only TDMA */
+		if (is_rx) {
+			/* First half of si_ram */
+			area->entries_start = tsa->si_ram;
+			area->entries_next = area->entries_start + half;
+			area->last_entry = NULL;
+		} else {
+			/* Second half of si_ram */
+			area->entries_start = tsa->si_ram + half;
+			area->entries_next = area->entries_start + half;
+			area->last_entry = NULL;
+		}
+	} else {
+		/* Only TDMB or both TDMs */
+		if (tdm_id == TSA_TDMA) {
+			if (is_rx) {
+				/* First half of first half of si_ram */
+				area->entries_start = tsa->si_ram;
+				area->entries_next = area->entries_start + quarter;
+				area->last_entry = NULL;
+			} else {
+				/* First half of second half of si_ram */
+				area->entries_start = tsa->si_ram + (2 * quarter);
+				area->entries_next = area->entries_start + quarter;
+				area->last_entry = NULL;
+			}
+		} else {
+			if (is_rx) {
+				/* Second half of first half of si_ram */
+				area->entries_start = tsa->si_ram + quarter;
+				area->entries_next = area->entries_start + quarter;
+				area->last_entry = NULL;
+			} else {
+				/* Second half of second half of si_ram */
+				area->entries_start = tsa->si_ram + (3 * quarter);
+				area->entries_next = area->entries_start + quarter;
+				area->last_entry = NULL;
+			}
+		}
+	}
+}
+
+static const char *tsa_serial_id2name(struct tsa *tsa, u32 serial_id)
+{
+	switch (serial_id) {
+	case FSL_CPM_TSA_NU:	return "Not used";
+	case FSL_CPM_TSA_SCC2:	return "SCC2";
+	case FSL_CPM_TSA_SCC3:	return "SCC3";
+	case FSL_CPM_TSA_SCC4:	return "SCC4";
+	case FSL_CPM_TSA_SMC1:	return "SMC1";
+	case FSL_CPM_TSA_SMC2:	return "SMC2";
+	default:
+		break;
+	}
+	return NULL;
+}
+
+static u32 tsa_serial_id2csel(struct tsa *tsa, u32 serial_id)
+{
+	switch (serial_id) {
+	case FSL_CPM_TSA_SCC2:	return TSA_SIRAM_ENTRY_CSEL_SCC2;
+	case FSL_CPM_TSA_SCC3:	return TSA_SIRAM_ENTRY_CSEL_SCC3;
+	case FSL_CPM_TSA_SCC4:	return TSA_SIRAM_ENTRY_CSEL_SCC4;
+	case FSL_CPM_TSA_SMC1:	return TSA_SIRAM_ENTRY_CSEL_SMC1;
+	case FSL_CPM_TSA_SMC2:	return TSA_SIRAM_ENTRY_CSEL_SMC2;
+	default:
+		break;
+	}
+	return TSA_SIRAM_ENTRY_CSEL_NU;
+}
+
+static int tsa_add_entry(struct tsa *tsa, struct tsa_entries_area *area,
+			 u32 count, u32 serial_id)
+{
+	void *__iomem addr;
+	u32 left;
+	u32 val;
+	u32 cnt;
+	u32 nb;
+
+	addr = area->last_entry ? area->last_entry + 4 : area->entries_start;
+
+	nb = DIV_ROUND_UP(count, 8);
+	if ((addr + (nb * 4)) > area->entries_next) {
+		dev_err(tsa->dev, "si ram area full\n");
+		return -ENOSPC;
+	}
+
+	if (area->last_entry) {
+		/* Clear last flag */
+		tsa_clrbits32(area->last_entry, TSA_SIRAM_ENTRY_LAST);
+	}
+
+	left = count;
+	while (left) {
+		val = TSA_SIRAM_ENTRY_BYTE | tsa_serial_id2csel(tsa, serial_id);
+
+		if (left > 16) {
+			cnt = 16;
+		} else {
+			cnt = left;
+			val |= TSA_SIRAM_ENTRY_LAST;
+			area->last_entry = addr;
+		}
+		val |= TSA_SIRAM_ENTRY_CNT(cnt - 1);
+
+		tsa_write32(addr, val);
+		addr += 4;
+		left -= cnt;
+	}
+
+	return 0;
+}
+
+static int tsa_of_parse_tdm_route(struct tsa *tsa, struct device_node *tdm_np,
+				  u32 tdms, u32 tdm_id, bool is_rx)
+{
+	struct tsa_entries_area area;
+	const char *route_name;
+	u32 serial_id;
+	int len, i;
+	u32 count;
+	const char *serial_name;
+	struct tsa_serial_info *serial_info;
+	struct tsa_tdm *tdm;
+	int ret;
+	u32 ts;
+
+	route_name = is_rx ? "fsl,rx-ts-routes" : "fsl,tx-ts-routes";
+
+	len = of_property_count_u32_elems(tdm_np,  route_name);
+	if (len < 0) {
+		dev_err(tsa->dev, "%pOF: failed to read %s\n", tdm_np, route_name);
+		return len;
+	}
+	if (len % 2 != 0) {
+		dev_err(tsa->dev, "%pOF: wrong %s format\n", tdm_np, route_name);
+		return -EINVAL;
+	}
+
+	tsa_init_entries_area(tsa, &area, tdms, tdm_id, is_rx);
+	ts = 0;
+	for (i = 0; i < len; i += 2) {
+		of_property_read_u32_index(tdm_np, route_name, i, &count);
+		of_property_read_u32_index(tdm_np, route_name, i + 1, &serial_id);
+
+		if (serial_id >= ARRAY_SIZE(tsa->serials)) {
+			dev_err(tsa->dev, "%pOF: invalid serial id (%u)\n",
+				tdm_np, serial_id);
+			return -EINVAL;
+		}
+
+		serial_name = tsa_serial_id2name(tsa, serial_id);
+		if (!serial_name) {
+			dev_err(tsa->dev, "%pOF: unsupported serial id (%u)\n",
+				tdm_np, serial_id);
+			return -EINVAL;
+		}
+
+		dev_dbg(tsa->dev, "tdm_id=%u, %s ts %u..%u -> %s\n",
+			tdm_id, route_name, ts, ts+count-1, serial_name);
+		ts += count;
+
+		ret = tsa_add_entry(tsa, &area, count, serial_id);
+		if (ret)
+			return ret;
+
+		serial_info = &tsa->serials[serial_id].info;
+		tdm = &tsa->tdm[tdm_id];
+		if (is_rx) {
+			serial_info->rx_fs_rate = clk_get_rate(tdm->l1rsync_clk);
+			serial_info->rx_bit_rate = clk_get_rate(tdm->l1rclk_clk);
+			serial_info->nb_rx_ts += count;
+		} else {
+			serial_info->tx_fs_rate = tdm->l1tsync_clk ?
+				clk_get_rate(tdm->l1tsync_clk) :
+				clk_get_rate(tdm->l1rsync_clk);
+			serial_info->tx_bit_rate = tdm->l1tclk_clk ?
+				clk_get_rate(tdm->l1tclk_clk) :
+				clk_get_rate(tdm->l1rclk_clk);
+			serial_info->nb_tx_ts += count;
+		}
+	}
+	return 0;
+}
+
+static inline int tsa_of_parse_tdm_rx_route(struct tsa *tsa,
+					    struct device_node *tdm_np,
+					    u32 tdms, u32 tdm_id)
+{
+	return tsa_of_parse_tdm_route(tsa, tdm_np, tdms, tdm_id, true);
+}
+
+static inline int tsa_of_parse_tdm_tx_route(struct tsa *tsa,
+					    struct device_node *tdm_np,
+					    u32 tdms, u32 tdm_id)
+{
+	return tsa_of_parse_tdm_route(tsa, tdm_np, tdms, tdm_id, false);
+}
+
+static int tsa_of_parse_tdms(struct tsa *tsa, struct device_node *np)
+{
+	struct device_node *tdm_np;
+	struct tsa_tdm *tdm;
+	struct clk *clk;
+	u32 tdm_id, val;
+	int ret;
+	int i;
+
+	tsa->tdms = 0;
+	tsa->tdm[0].is_enable = false;
+	tsa->tdm[1].is_enable = false;
+
+	for_each_available_child_of_node(np, tdm_np) {
+		ret = of_property_read_u32(tdm_np, "reg", &tdm_id);
+		if (ret) {
+			dev_err(tsa->dev, "%pOF: failed to read reg\n", tdm_np);
+			of_node_put(tdm_np);
+			return ret;
+		}
+		switch (tdm_id) {
+		case 0:
+			tsa->tdms |= BIT(TSA_TDMA);
+			break;
+		case 1:
+			tsa->tdms |= BIT(TSA_TDMB);
+			break;
+		default:
+			dev_err(tsa->dev, "%pOF: Invalid tdm_id (%u)\n", tdm_np,
+				tdm_id);
+			of_node_put(tdm_np);
+			return -EINVAL;
+		}
+	}
+
+	for_each_available_child_of_node(np, tdm_np) {
+		ret = of_property_read_u32(tdm_np, "reg", &tdm_id);
+		if (ret) {
+			dev_err(tsa->dev, "%pOF: failed to read reg\n", tdm_np);
+			of_node_put(tdm_np);
+			return ret;
+		}
+
+		tdm = &tsa->tdm[tdm_id];
+		tdm->simode_tdm = TSA_SIMODE_TDM_SDM_NORM;
+
+		val = 0;
+		ret = of_property_read_u32(tdm_np, "fsl,rx-frame-sync-delay-bits",
+					   &val);
+		if (ret && ret != -EINVAL) {
+			dev_err(tsa->dev,
+				"%pOF: failed to read fsl,rx-frame-sync-delay-bits\n",
+				tdm_np);
+			of_node_put(tdm_np);
+			return ret;
+		}
+		if (val > 3) {
+			dev_err(tsa->dev,
+				"%pOF: Invalid fsl,rx-frame-sync-delay-bits (%u)\n",
+				tdm_np, val);
+			of_node_put(tdm_np);
+			return -EINVAL;
+		}
+		tdm->simode_tdm |= TSA_SIMODE_TDM_RFSD(val);
+
+		val = 0;
+		ret = of_property_read_u32(tdm_np, "fsl,tx-frame-sync-delay-bits",
+					   &val);
+		if (ret && ret != -EINVAL) {
+			dev_err(tsa->dev,
+				"%pOF: failed to read fsl,tx-frame-sync-delay-bits\n",
+				tdm_np);
+			of_node_put(tdm_np);
+			return ret;
+		}
+		if (val > 3) {
+			dev_err(tsa->dev,
+				"%pOF: Invalid fsl,tx-frame-sync-delay-bits (%u)\n",
+				tdm_np, val);
+			of_node_put(tdm_np);
+			return -EINVAL;
+		}
+		tdm->simode_tdm |= TSA_SIMODE_TDM_TFSD(val);
+
+		if (of_property_read_bool(tdm_np, "fsl,common-rxtx-pins"))
+			tdm->simode_tdm |= TSA_SIMODE_TDM_CRT;
+
+		if (of_property_read_bool(tdm_np, "fsl,clock-falling-edge"))
+			tdm->simode_tdm |= TSA_SIMODE_TDM_CE;
+
+		if (of_property_read_bool(tdm_np, "fsl,fsync-rising-edge"))
+			tdm->simode_tdm |= TSA_SIMODE_TDM_FE;
+
+		if (of_property_read_bool(tdm_np, "fsl,double-speed-clock"))
+			tdm->simode_tdm |= TSA_SIMODE_TDM_DSC;
+
+		clk = of_clk_get_by_name(tdm_np, "l1rsync");
+		if (IS_ERR(clk)) {
+			ret = PTR_ERR(clk);
+			of_node_put(tdm_np);
+			goto err;
+		}
+		ret = clk_prepare_enable(clk);
+		if (ret) {
+			clk_put(clk);
+			of_node_put(tdm_np);
+			goto err;
+		}
+		tdm->l1rsync_clk = clk;
+
+		clk = of_clk_get_by_name(tdm_np, "l1rclk");
+		if (IS_ERR(clk)) {
+			ret = PTR_ERR(clk);
+			of_node_put(tdm_np);
+			goto err;
+		}
+		ret = clk_prepare_enable(clk);
+		if (ret) {
+			clk_put(clk);
+			of_node_put(tdm_np);
+			goto err;
+		}
+		tdm->l1rclk_clk = clk;
+
+		if (!(tdm->simode_tdm & TSA_SIMODE_TDM_CRT)) {
+			clk = of_clk_get_by_name(tdm_np, "l1tsync");
+			if (IS_ERR(clk)) {
+				ret = PTR_ERR(clk);
+				of_node_put(tdm_np);
+				goto err;
+			}
+			ret = clk_prepare_enable(clk);
+			if (ret) {
+				clk_put(clk);
+				of_node_put(tdm_np);
+				goto err;
+			}
+			tdm->l1tsync_clk = clk;
+
+			clk = of_clk_get_by_name(tdm_np, "l1tclk");
+			if (IS_ERR(clk)) {
+				ret = PTR_ERR(clk);
+				of_node_put(tdm_np);
+				goto err;
+			}
+			ret = clk_prepare_enable(clk);
+			if (ret) {
+				clk_put(clk);
+				of_node_put(tdm_np);
+				goto err;
+			}
+			tdm->l1tclk_clk = clk;
+		}
+
+		ret = tsa_of_parse_tdm_rx_route(tsa, tdm_np, tsa->tdms, tdm_id);
+		if (ret) {
+			of_node_put(tdm_np);
+			goto err;
+		}
+
+		ret = tsa_of_parse_tdm_tx_route(tsa, tdm_np, tsa->tdms, tdm_id);
+		if (ret) {
+			of_node_put(tdm_np);
+			goto err;
+		}
+
+		tdm->is_enable = true;
+	}
+	return 0;
+
+err:
+	for (i = 0; i < 2; i++) {
+		if (tsa->tdm[i].l1rsync_clk) {
+			clk_disable_unprepare(tsa->tdm[i].l1rsync_clk);
+			clk_put(tsa->tdm[i].l1rsync_clk);
+		}
+		if (tsa->tdm[i].l1rclk_clk) {
+			clk_disable_unprepare(tsa->tdm[i].l1rclk_clk);
+			clk_put(tsa->tdm[i].l1rclk_clk);
+		}
+		if (tsa->tdm[i].l1tsync_clk) {
+			clk_disable_unprepare(tsa->tdm[i].l1rsync_clk);
+			clk_put(tsa->tdm[i].l1rsync_clk);
+		}
+		if (tsa->tdm[i].l1tclk_clk) {
+			clk_disable_unprepare(tsa->tdm[i].l1rclk_clk);
+			clk_put(tsa->tdm[i].l1rclk_clk);
+		}
+	}
+	return ret;
+}
+
+static void tsa_init_si_ram(struct tsa *tsa)
+{
+	resource_size_t i;
+
+	/* Fill all entries as the last one */
+	for (i = 0; i < tsa->si_ram_sz; i += 4)
+		tsa_write32(tsa->si_ram + i, TSA_SIRAM_ENTRY_LAST);
+}
+
+static int tsa_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct resource *res;
+	struct tsa *tsa;
+	unsigned int i;
+	u32 val;
+	int ret;
+
+	tsa = devm_kzalloc(&pdev->dev, sizeof(*tsa), GFP_KERNEL);
+	if (!tsa)
+		return -ENOMEM;
+
+	tsa->dev = &pdev->dev;
+
+	for (i = 0; i < ARRAY_SIZE(tsa->serials); i++)
+		tsa->serials[i].id = i;
+
+	spin_lock_init(&tsa->lock);
+
+	tsa->si_regs = devm_platform_ioremap_resource_byname(pdev, "si_regs");
+	if (IS_ERR(tsa->si_regs))
+		return PTR_ERR(tsa->si_regs);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "si_ram");
+	if (!res) {
+		dev_err(tsa->dev, "si_ram resource missing\n");
+		return -EINVAL;
+	}
+	tsa->si_ram_sz = resource_size(res);
+	tsa->si_ram = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(tsa->si_ram))
+		return PTR_ERR(tsa->si_ram);
+
+	tsa_init_si_ram(tsa);
+
+	ret = tsa_of_parse_tdms(tsa, np);
+	if (ret)
+		return ret;
+
+	/* Set SIMODE */
+	val = 0;
+	if (tsa->tdm[0].is_enable)
+		val |= TSA_SIMODE_TDMA(tsa->tdm[0].simode_tdm);
+	if (tsa->tdm[1].is_enable)
+		val |= TSA_SIMODE_TDMB(tsa->tdm[1].simode_tdm);
+
+	tsa_clrsetbits32(tsa->si_regs + TSA_SIMODE,
+			 TSA_SIMODE_TDMA(TSA_SIMODE_TDM_MASK) |
+			 TSA_SIMODE_TDMB(TSA_SIMODE_TDM_MASK),
+			 val);
+
+	/* Set SIGMR */
+	val = (tsa->tdms == BIT(TSA_TDMA)) ?
+		TSA_SIGMR_RDM_STATIC_TDMA : TSA_SIGMR_RDM_STATIC_TDMAB;
+	if (tsa->tdms & BIT(TSA_TDMA))
+		val |= TSA_SIGMR_ENA;
+	if (tsa->tdms & BIT(TSA_TDMB))
+		val |= TSA_SIGMR_ENB;
+	tsa_write8(tsa->si_regs + TSA_SIGMR, val);
+
+	platform_set_drvdata(pdev, tsa);
+
+	return 0;
+}
+
+static int tsa_remove(struct platform_device *pdev)
+{
+	struct tsa *tsa = platform_get_drvdata(pdev);
+	int i;
+
+	for (i = 0; i < 2; i++) {
+		if (tsa->tdm[i].l1rsync_clk) {
+			clk_disable_unprepare(tsa->tdm[i].l1rsync_clk);
+			clk_put(tsa->tdm[i].l1rsync_clk);
+		}
+		if (tsa->tdm[i].l1rclk_clk) {
+			clk_disable_unprepare(tsa->tdm[i].l1rclk_clk);
+			clk_put(tsa->tdm[i].l1rclk_clk);
+		}
+		if (tsa->tdm[i].l1tsync_clk) {
+			clk_disable_unprepare(tsa->tdm[i].l1rsync_clk);
+			clk_put(tsa->tdm[i].l1rsync_clk);
+		}
+		if (tsa->tdm[i].l1tclk_clk) {
+			clk_disable_unprepare(tsa->tdm[i].l1rclk_clk);
+			clk_put(tsa->tdm[i].l1rclk_clk);
+		}
+	}
+	return 0;
+}
+
+static const struct of_device_id tsa_id_table[] = {
+	{ .compatible = "fsl,cpm1-tsa" },
+	{} /* sentinel */
+};
+MODULE_DEVICE_TABLE(of, tsa_id_table);
+
+static struct platform_driver tsa_driver = {
+	.driver = {
+		.name = "fsl-tsa",
+		.of_match_table = of_match_ptr(tsa_id_table),
+	},
+	.probe = tsa_probe,
+	.remove = tsa_remove,
+};
+module_platform_driver(tsa_driver);
+
+struct tsa_serial *tsa_serial_get_byphandle(struct device_node *np,
+					    const char *phandle_name)
+{
+	struct of_phandle_args out_args;
+	struct platform_device *pdev;
+	struct tsa_serial *tsa_serial;
+	struct tsa *tsa;
+	int ret;
+
+	ret = of_parse_phandle_with_fixed_args(np, phandle_name, 1, 0, &out_args);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	if (!of_match_node(tsa_driver.driver.of_match_table, out_args.np)) {
+		of_node_put(out_args.np);
+		return ERR_PTR(-EINVAL);
+	}
+
+	pdev = of_find_device_by_node(out_args.np);
+	of_node_put(out_args.np);
+	if (!pdev)
+		return ERR_PTR(-ENODEV);
+
+	tsa = platform_get_drvdata(pdev);
+	if (!tsa) {
+		platform_device_put(pdev);
+		return ERR_PTR(-EPROBE_DEFER);
+	}
+
+	if (out_args.args_count != 1) {
+		platform_device_put(pdev);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (out_args.args[0] >= ARRAY_SIZE(tsa->serials)) {
+		platform_device_put(pdev);
+		return ERR_PTR(-EINVAL);
+	}
+
+	tsa_serial = &tsa->serials[out_args.args[0]];
+
+	/*
+	 * Be sure that the serial id matches the phandle arg.
+	 * The tsa_serials table is indexed by serial ids. The serial id is set
+	 * during the probe() call and needs to be coherent.
+	 */
+	if (WARN_ON(tsa_serial->id != out_args.args[0])) {
+		platform_device_put(pdev);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return tsa_serial;
+}
+EXPORT_SYMBOL(tsa_serial_get_byphandle);
+
+void tsa_serial_put(struct tsa_serial *tsa_serial)
+{
+	struct tsa *tsa = tsa_serial_get_tsa(tsa_serial);
+
+	put_device(tsa->dev);
+}
+EXPORT_SYMBOL(tsa_serial_put);
+
+static void devm_tsa_serial_release(struct device *dev, void *res)
+{
+	struct tsa_serial **tsa_serial = res;
+
+	tsa_serial_put(*tsa_serial);
+}
+
+struct tsa_serial *devm_tsa_serial_get_byphandle(struct device *dev,
+						 struct device_node *np,
+						 const char *phandle_name)
+{
+	struct tsa_serial *tsa_serial;
+	struct tsa_serial **dr;
+
+	dr = devres_alloc(devm_tsa_serial_release, sizeof(*dr), GFP_KERNEL);
+	if (!dr)
+		return ERR_PTR(-ENOMEM);
+
+	tsa_serial = tsa_serial_get_byphandle(np, phandle_name);
+	if (!IS_ERR(tsa_serial)) {
+		*dr = tsa_serial;
+		devres_add(dev, dr);
+	} else {
+		devres_free(dr);
+	}
+
+	return tsa_serial;
+}
+EXPORT_SYMBOL(devm_tsa_serial_get_byphandle);
+
+MODULE_AUTHOR("Herve Codina <herve.codina@bootlin.com>");
+MODULE_DESCRIPTION("CPM TSA driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/fsl/qe/tsa.h b/drivers/soc/fsl/qe/tsa.h
new file mode 100644
index 000000000000..d9df89b6da3e
--- /dev/null
+++ b/drivers/soc/fsl/qe/tsa.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * TSA management
+ *
+ * Copyright 2022 CS GROUP France
+ *
+ * Author: Herve Codina <herve.codina@bootlin.com>
+ */
+#ifndef __SOC_FSL_TSA_H__
+#define __SOC_FSL_TSA_H__
+
+#include <linux/types.h>
+
+struct device_node;
+struct device;
+struct tsa_serial;
+
+struct tsa_serial *tsa_serial_get_byphandle(struct device_node *np,
+					    const char *phandle_name);
+void tsa_serial_put(struct tsa_serial *tsa_serial);
+struct tsa_serial *devm_tsa_serial_get_byphandle(struct device *dev,
+						 struct device_node *np,
+						 const char *phandle_name);
+
+/* Connect and disconnect the TSA serial */
+int tsa_serial_connect(struct tsa_serial *tsa_serial);
+int tsa_serial_disconnect(struct tsa_serial *tsa_serial);
+
+/* Cell information */
+struct tsa_serial_info {
+	unsigned long rx_fs_rate;
+	unsigned long rx_bit_rate;
+	u8 nb_rx_ts;
+	unsigned long tx_fs_rate;
+	unsigned long tx_bit_rate;
+	u8 nb_tx_ts;
+};
+
+/* Get information */
+int tsa_serial_get_info(struct tsa_serial *tsa_serial, struct tsa_serial_info *info);
+
+#endif /* __SOC_FSL_TSA_H__ */
diff --git a/drivers/soc/fujitsu/a64fx-diag.c b/drivers/soc/fujitsu/a64fx-diag.c
index d87f348427bf..524fbfeb94e3 100644
--- a/drivers/soc/fujitsu/a64fx-diag.c
+++ b/drivers/soc/fujitsu/a64fx-diag.c
@@ -149,6 +149,5 @@ static struct platform_driver a64fx_diag_driver = {
 
 module_platform_driver(a64fx_diag_driver);
 
-MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Hitomi Hasegawa <hasegawa-hitomi@fujitsu.com>");
 MODULE_DESCRIPTION("A64FX diag driver");
diff --git a/drivers/soc/imx/Kconfig b/drivers/soc/imx/Kconfig
index a8742fc58f01..76a4593baf0a 100644
--- a/drivers/soc/imx/Kconfig
+++ b/drivers/soc/imx/Kconfig
@@ -10,7 +10,7 @@ config IMX_GPCV2_PM_DOMAINS
 	default y if SOC_IMX7D
 
 config SOC_IMX8M
-	bool "i.MX8M SoC family support"
+	tristate "i.MX8M SoC family support"
 	depends on ARCH_MXC || COMPILE_TEST
 	default ARCH_MXC && ARM64
 	select SOC_BUS
diff --git a/drivers/soc/imx/imx8m-blk-ctrl.c b/drivers/soc/imx/imx8m-blk-ctrl.c
index 399cb85105a1..afbca0d48c14 100644
--- a/drivers/soc/imx/imx8m-blk-ctrl.c
+++ b/drivers/soc/imx/imx8m-blk-ctrl.c
@@ -38,10 +38,10 @@ struct imx8m_blk_ctrl {
 struct imx8m_blk_ctrl_domain_data {
 	const char *name;
 	const char * const *clk_names;
-	int num_clks;
 	const char * const *path_names;
-	int num_paths;
 	const char *gpc_name;
+	int num_clks;
+	int num_paths;
 	u32 rst_mask;
 	u32 clk_mask;
 
@@ -210,7 +210,7 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev)
 	if (!bc->onecell_data.domains)
 		return -ENOMEM;
 
-	bc->bus_power_dev = genpd_dev_pm_attach_by_name(dev, "bus");
+	bc->bus_power_dev = dev_pm_domain_attach_by_name(dev, "bus");
 	if (IS_ERR(bc->bus_power_dev)) {
 		if (PTR_ERR(bc->bus_power_dev) == -ENODEV)
 			return dev_err_probe(dev, -EPROBE_DEFER,
@@ -310,6 +310,10 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev)
 
 	dev_set_drvdata(dev, bc);
 
+	ret = devm_of_platform_populate(dev);
+	if (ret)
+		goto cleanup_provider;
+
 	return 0;
 
 cleanup_provider:
@@ -891,3 +895,4 @@ static struct platform_driver imx8m_blk_ctrl_driver = {
 	},
 };
 module_platform_driver(imx8m_blk_ctrl_driver);
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/imx/imx8mp-blk-ctrl.c b/drivers/soc/imx/imx8mp-blk-ctrl.c
index a0592db8fa86..870aecc0202a 100644
--- a/drivers/soc/imx/imx8mp-blk-ctrl.c
+++ b/drivers/soc/imx/imx8mp-blk-ctrl.c
@@ -642,7 +642,7 @@ static int imx8mp_blk_ctrl_probe(struct platform_device *pdev)
 	if (!bc->onecell_data.domains)
 		return -ENOMEM;
 
-	bc->bus_power_dev = genpd_dev_pm_attach_by_name(dev, "bus");
+	bc->bus_power_dev = dev_pm_domain_attach_by_name(dev, "bus");
 	if (IS_ERR(bc->bus_power_dev))
 		return dev_err_probe(dev, PTR_ERR(bc->bus_power_dev),
 				     "failed to attach bus power domain\n");
@@ -852,7 +852,7 @@ static const struct of_device_id imx8mp_blk_ctrl_of_match[] = {
 		/* Sentinel */
 	}
 };
-MODULE_DEVICE_TABLE(of, imx8m_blk_ctrl_of_match);
+MODULE_DEVICE_TABLE(of, imx8mp_blk_ctrl_of_match);
 
 static struct platform_driver imx8mp_blk_ctrl_driver = {
 	.probe = imx8mp_blk_ctrl_probe,
@@ -864,3 +864,4 @@ static struct platform_driver imx8mp_blk_ctrl_driver = {
 	},
 };
 module_platform_driver(imx8mp_blk_ctrl_driver);
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/imx/soc-imx8m.c b/drivers/soc/imx/soc-imx8m.c
index 32ed9dc88e45..1dcd243df567 100644
--- a/drivers/soc/imx/soc-imx8m.c
+++ b/drivers/soc/imx/soc-imx8m.c
@@ -242,3 +242,4 @@ free_soc:
 	return ret;
 }
 device_initcall(imx8_soc_init);
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/mediatek/Kconfig b/drivers/soc/mediatek/Kconfig
index d6b83a5508ca..a88cf04fc803 100644
--- a/drivers/soc/mediatek/Kconfig
+++ b/drivers/soc/mediatek/Kconfig
@@ -76,6 +76,7 @@ config MTK_MMSYS
 	tristate "MediaTek MMSYS Support"
 	default ARCH_MEDIATEK
 	depends on HAS_IOMEM
+	depends on MTK_CMDQ || MTK_CMDQ=n
 	help
 	  Say yes here to add support for the MediaTek Multimedia
 	  Subsystem (MMSYS).
diff --git a/drivers/soc/mediatek/mt8173-mmsys.h b/drivers/soc/mediatek/mt8173-mmsys.h
new file mode 100644
index 000000000000..9d24e381271e
--- /dev/null
+++ b/drivers/soc/mediatek/mt8173-mmsys.h
@@ -0,0 +1,95 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef __SOC_MEDIATEK_MT8173_MMSYS_H
+#define __SOC_MEDIATEK_MT8173_MMSYS_H
+
+#define MT8173_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN	0x040
+#define MT8173_DISP_REG_CONFIG_DISP_OVL1_MOUT_EN	0x044
+#define MT8173_DISP_REG_CONFIG_DISP_OD_MOUT_EN		0x048
+#define MT8173_DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN	0x04c
+#define MT8173_DISP_REG_CONFIG_DISP_UFOE_MOUT_EN	0x050
+#define MT8173_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN	0x084
+#define MT8173_DISP_REG_CONFIG_DISP_COLOR1_SEL_IN	0x088
+#define MT8173_DISP_REG_CONFIG_DISP_AAL_SEL_IN		0x08c
+#define MT8173_DISP_REG_CONFIG_DISP_UFOE_SEL_IN		0x0a0
+#define MT8173_DISP_REG_CONFIG_DSI0_SEL_IN		0x0a4
+#define MT8173_DISP_REG_CONFIG_DPI_SEL_IN		0x0ac
+#define MT8173_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN	0x0b0
+#define MT8173_DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN	0x0c8
+#define MT8173_DISP_REG_CONFIG_DISP_COLOR0_SOUT_SEL_IN	0x0bc
+
+#define MT8173_AAL_SEL_IN_MERGE				BIT(0)
+#define MT8173_COLOR0_SEL_IN_OVL0			BIT(0)
+#define MT8173_COLOR0_SOUT_MERGE			BIT(0)
+#define MT8173_DPI0_SEL_IN_MASK				GENMASK(1, 0)
+#define MT8173_DPI0_SEL_IN_RDMA1			BIT(0)
+#define MT8173_DSI0_SEL_IN_UFOE				BIT(0)
+#define MT8173_GAMMA_MOUT_EN_RDMA1			BIT(0)
+#define MT8173_OD0_MOUT_EN_RDMA0			BIT(0)
+#define MT8173_OVL0_MOUT_EN_COLOR0			BIT(0)
+#define MT8173_OVL1_MOUT_EN_COLOR1			BIT(0)
+#define MT8173_UFOE_MOUT_EN_DSI0			BIT(0)
+#define MT8173_UFOE_SEL_IN_RDMA0			BIT(0)
+#define MT8173_RDMA0_SOUT_COLOR0			BIT(0)
+
+static const struct mtk_mmsys_routes mt8173_mmsys_routing_table[] = {
+	{
+		DDP_COMPONENT_OVL0, DDP_COMPONENT_COLOR0,
+		MT8173_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN,
+		MT8173_OVL0_MOUT_EN_COLOR0, MT8173_OVL0_MOUT_EN_COLOR0
+	}, {
+		DDP_COMPONENT_OD0, DDP_COMPONENT_RDMA0,
+		MT8173_DISP_REG_CONFIG_DISP_OD_MOUT_EN,
+		MT8173_OD0_MOUT_EN_RDMA0, MT8173_OD0_MOUT_EN_RDMA0
+	}, {
+		DDP_COMPONENT_UFOE, DDP_COMPONENT_DSI0,
+		MT8173_DISP_REG_CONFIG_DISP_UFOE_MOUT_EN,
+		MT8173_UFOE_MOUT_EN_DSI0, MT8173_UFOE_MOUT_EN_DSI0
+	}, {
+		DDP_COMPONENT_COLOR0, DDP_COMPONENT_AAL0,
+		MT8173_DISP_REG_CONFIG_DISP_COLOR0_SOUT_SEL_IN,
+		MT8173_COLOR0_SOUT_MERGE, 0 /* SOUT to AAL */
+	}, {
+		DDP_COMPONENT_RDMA0, DDP_COMPONENT_UFOE,
+		MT8173_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN,
+		MT8173_RDMA0_SOUT_COLOR0, 0 /* SOUT to UFOE */
+	}, {
+		DDP_COMPONENT_OVL0, DDP_COMPONENT_COLOR0,
+		MT8173_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN,
+		MT8173_COLOR0_SEL_IN_OVL0, MT8173_COLOR0_SEL_IN_OVL0
+	}, {
+		DDP_COMPONENT_AAL0, DDP_COMPONENT_COLOR0,
+		MT8173_DISP_REG_CONFIG_DISP_AAL_SEL_IN,
+		MT8173_AAL_SEL_IN_MERGE, 0 /* SEL_IN from COLOR0 */
+	}, {
+		DDP_COMPONENT_RDMA0, DDP_COMPONENT_UFOE,
+		MT8173_DISP_REG_CONFIG_DISP_UFOE_SEL_IN,
+		MT8173_UFOE_SEL_IN_RDMA0, 0 /* SEL_IN from RDMA0 */
+	}, {
+		DDP_COMPONENT_UFOE, DDP_COMPONENT_DSI0,
+		MT8173_DISP_REG_CONFIG_DSI0_SEL_IN,
+		MT8173_DSI0_SEL_IN_UFOE, 0, /* SEL_IN from UFOE */
+	}, {
+		DDP_COMPONENT_OVL1, DDP_COMPONENT_COLOR1,
+		MT8173_DISP_REG_CONFIG_DISP_OVL1_MOUT_EN,
+		MT8173_OVL1_MOUT_EN_COLOR1, MT8173_OVL1_MOUT_EN_COLOR1
+	}, {
+		DDP_COMPONENT_GAMMA, DDP_COMPONENT_RDMA1,
+		MT8173_DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN,
+		MT8173_GAMMA_MOUT_EN_RDMA1, MT8173_GAMMA_MOUT_EN_RDMA1
+	}, {
+		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DPI0,
+		MT8173_DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN,
+		RDMA1_SOUT_MASK, RDMA1_SOUT_DPI0
+	}, {
+		DDP_COMPONENT_OVL1, DDP_COMPONENT_COLOR1,
+		MT8173_DISP_REG_CONFIG_DISP_COLOR1_SEL_IN,
+		COLOR1_SEL_IN_OVL1, COLOR1_SEL_IN_OVL1
+	}, {
+		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DPI0,
+		MT8173_DISP_REG_CONFIG_DPI_SEL_IN,
+		MT8173_DPI0_SEL_IN_MASK, MT8173_DPI0_SEL_IN_RDMA1
+	}
+};
+
+#endif /* __SOC_MEDIATEK_MT8173_MMSYS_H */
diff --git a/drivers/soc/mediatek/mt8195-mmsys.h b/drivers/soc/mediatek/mt8195-mmsys.h
index a6652ae63431..9be2df2832a4 100644
--- a/drivers/soc/mediatek/mt8195-mmsys.h
+++ b/drivers/soc/mediatek/mt8195-mmsys.h
@@ -146,6 +146,19 @@
 #define MT8195_VDO1_MIXER_SOUT_SEL_IN				0xf68
 #define MT8195_MIXER_SOUT_SEL_IN_FROM_DISP_MIXER			0
 
+/* VPPSYS1 */
+#define MT8195_VPP1_HW_DCM_1ST_DIS0				0x150
+#define MT8195_VPP1_HW_DCM_1ST_DIS1				0x160
+#define MT8195_VPP1_HW_DCM_2ND_DIS0				0x1a0
+#define MT8195_VPP1_HW_DCM_2ND_DIS1				0x1b0
+#define MT8195_SVPP2_BUF_BF_RSZ_SWITCH				0xf48
+#define MT8195_SVPP3_BUF_BF_RSZ_SWITCH				0xf74
+
+/* VPPSYS1 HW DCM client*/
+#define MT8195_SVPP1_MDP_RSZ					BIT(25)
+#define MT8195_SVPP2_MDP_RSZ					BIT(4)
+#define MT8195_SVPP3_MDP_RSZ					BIT(5)
+
 static const struct mtk_mmsys_routes mmsys_mt8195_routing_table[] = {
 	{
 		DDP_COMPONENT_OVL0, DDP_COMPONENT_RDMA0,
diff --git a/drivers/soc/mediatek/mtk-mmsys.c b/drivers/soc/mediatek/mtk-mmsys.c
index eb4c7e57896c..9619faa796e8 100644
--- a/drivers/soc/mediatek/mtk-mmsys.c
+++ b/drivers/soc/mediatek/mtk-mmsys.c
@@ -15,6 +15,7 @@
 
 #include "mtk-mmsys.h"
 #include "mt8167-mmsys.h"
+#include "mt8173-mmsys.h"
 #include "mt8183-mmsys.h"
 #include "mt8186-mmsys.h"
 #include "mt8188-mmsys.h"
@@ -40,6 +41,14 @@ static const struct mtk_mmsys_driver_data mt6779_mmsys_driver_data = {
 	.clk_driver = "clk-mt6779-mm",
 };
 
+static const struct mtk_mmsys_driver_data mt6795_mmsys_driver_data = {
+	.clk_driver = "clk-mt6795-mm",
+	.routes = mt8173_mmsys_routing_table,
+	.num_routes = ARRAY_SIZE(mt8173_mmsys_routing_table),
+	.sw0_rst_offset = MT8183_MMSYS_SW0_RST_B,
+	.num_resets = 64,
+};
+
 static const struct mtk_mmsys_driver_data mt6797_mmsys_driver_data = {
 	.clk_driver = "clk-mt6797-mm",
 };
@@ -52,10 +61,10 @@ static const struct mtk_mmsys_driver_data mt8167_mmsys_driver_data = {
 
 static const struct mtk_mmsys_driver_data mt8173_mmsys_driver_data = {
 	.clk_driver = "clk-mt8173-mm",
-	.routes = mmsys_default_routing_table,
-	.num_routes = ARRAY_SIZE(mmsys_default_routing_table),
+	.routes = mt8173_mmsys_routing_table,
+	.num_routes = ARRAY_SIZE(mt8173_mmsys_routing_table),
 	.sw0_rst_offset = MT8183_MMSYS_SW0_RST_B,
-	.num_resets = 32,
+	.num_resets = 64,
 };
 
 static const struct mtk_mmsys_driver_data mt8183_mmsys_driver_data = {
@@ -121,6 +130,8 @@ static const struct mtk_mmsys_driver_data mt8365_mmsys_driver_data = {
 struct mtk_mmsys {
 	void __iomem *regs;
 	const struct mtk_mmsys_driver_data *data;
+	struct platform_device *clks_pdev;
+	struct platform_device *drm_pdev;
 	spinlock_t lock; /* protects mmsys_sw_rst_b reg */
 	struct reset_controller_dev rcdev;
 	struct cmdq_client_reg cmdq_base;
@@ -129,21 +140,18 @@ struct mtk_mmsys {
 static void mtk_mmsys_update_bits(struct mtk_mmsys *mmsys, u32 offset, u32 mask, u32 val,
 				  struct cmdq_pkt *cmdq_pkt)
 {
+	int ret;
 	u32 tmp;
 
-#if IS_REACHABLE(CONFIG_MTK_CMDQ)
-	if (cmdq_pkt) {
-		if (mmsys->cmdq_base.size == 0) {
-			pr_err("mmsys lose gce property, failed to update mmsys bits with cmdq");
+	if (mmsys->cmdq_base.size && cmdq_pkt) {
+		ret = cmdq_pkt_write_mask(cmdq_pkt, mmsys->cmdq_base.subsys,
+					  mmsys->cmdq_base.offset + offset, val,
+					  mask);
+		if (ret)
+			pr_debug("CMDQ unavailable: using CPU write\n");
+		else
 			return;
-		}
-		cmdq_pkt_write_mask(cmdq_pkt, mmsys->cmdq_base.subsys,
-				    mmsys->cmdq_base.offset + offset, val,
-				    mask);
-		return;
 	}
-#endif
-
 	tmp = readl_relaxed(mmsys->regs + offset);
 	tmp = (tmp & ~mask) | (val & mask);
 	writel_relaxed(tmp, mmsys->regs + offset);
@@ -242,6 +250,50 @@ void mtk_mmsys_ddp_dpi_fmt_config(struct device *dev, u32 val)
 }
 EXPORT_SYMBOL_GPL(mtk_mmsys_ddp_dpi_fmt_config);
 
+void mtk_mmsys_vpp_rsz_merge_config(struct device *dev, u32 id, bool enable,
+				    struct cmdq_pkt *cmdq_pkt)
+{
+	u32 reg;
+
+	switch (id) {
+	case 2:
+		reg = MT8195_SVPP2_BUF_BF_RSZ_SWITCH;
+		break;
+	case 3:
+		reg = MT8195_SVPP3_BUF_BF_RSZ_SWITCH;
+		break;
+	default:
+		dev_err(dev, "Invalid id %d\n", id);
+		return;
+	}
+
+	mtk_mmsys_update_bits(dev_get_drvdata(dev), reg, ~0, enable, cmdq_pkt);
+}
+EXPORT_SYMBOL_GPL(mtk_mmsys_vpp_rsz_merge_config);
+
+void mtk_mmsys_vpp_rsz_dcm_config(struct device *dev, bool enable,
+				  struct cmdq_pkt *cmdq_pkt)
+{
+	u32 client;
+
+	client = MT8195_SVPP1_MDP_RSZ;
+	mtk_mmsys_update_bits(dev_get_drvdata(dev),
+			      MT8195_VPP1_HW_DCM_1ST_DIS0, client,
+			      ((enable) ? client : 0), cmdq_pkt);
+	mtk_mmsys_update_bits(dev_get_drvdata(dev),
+			      MT8195_VPP1_HW_DCM_2ND_DIS0, client,
+			      ((enable) ? client : 0), cmdq_pkt);
+
+	client = MT8195_SVPP2_MDP_RSZ | MT8195_SVPP3_MDP_RSZ;
+	mtk_mmsys_update_bits(dev_get_drvdata(dev),
+			      MT8195_VPP1_HW_DCM_1ST_DIS1, client,
+			      ((enable) ? client : 0), cmdq_pkt);
+	mtk_mmsys_update_bits(dev_get_drvdata(dev),
+			      MT8195_VPP1_HW_DCM_2ND_DIS1, client,
+			      ((enable) ? client : 0), cmdq_pkt);
+}
+EXPORT_SYMBOL_GPL(mtk_mmsys_vpp_rsz_dcm_config);
+
 static int mtk_mmsys_reset_update(struct reset_controller_dev *rcdev, unsigned long id,
 				  bool assert)
 {
@@ -330,11 +382,10 @@ static int mtk_mmsys_probe(struct platform_device *pdev)
 		}
 	}
 
-#if IS_REACHABLE(CONFIG_MTK_CMDQ)
+	/* CMDQ is optional */
 	ret = cmdq_dev_get_client_reg(dev, &mmsys->cmdq_base, 0);
 	if (ret)
 		dev_dbg(dev, "No mediatek,gce-client-reg!\n");
-#endif
 
 	platform_set_drvdata(pdev, mmsys);
 
@@ -342,6 +393,7 @@ static int mtk_mmsys_probe(struct platform_device *pdev)
 					     PLATFORM_DEVID_AUTO, NULL, 0);
 	if (IS_ERR(clks))
 		return PTR_ERR(clks);
+	mmsys->clks_pdev = clks;
 
 	if (mmsys->data->is_vppsys)
 		goto out_probe_done;
@@ -352,78 +404,44 @@ static int mtk_mmsys_probe(struct platform_device *pdev)
 		platform_device_unregister(clks);
 		return PTR_ERR(drm);
 	}
+	mmsys->drm_pdev = drm;
 
 out_probe_done:
 	return 0;
 }
 
+static int mtk_mmsys_remove(struct platform_device *pdev)
+{
+	struct mtk_mmsys *mmsys = platform_get_drvdata(pdev);
+
+	platform_device_unregister(mmsys->drm_pdev);
+	platform_device_unregister(mmsys->clks_pdev);
+
+	return 0;
+}
+
 static const struct of_device_id of_match_mtk_mmsys[] = {
-	{
-		.compatible = "mediatek,mt2701-mmsys",
-		.data = &mt2701_mmsys_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt2712-mmsys",
-		.data = &mt2712_mmsys_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt6779-mmsys",
-		.data = &mt6779_mmsys_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt6797-mmsys",
-		.data = &mt6797_mmsys_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8167-mmsys",
-		.data = &mt8167_mmsys_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8173-mmsys",
-		.data = &mt8173_mmsys_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8183-mmsys",
-		.data = &mt8183_mmsys_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8186-mmsys",
-		.data = &mt8186_mmsys_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8188-vdosys0",
-		.data = &mt8188_vdosys0_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8192-mmsys",
-		.data = &mt8192_mmsys_driver_data,
-	},
-	{	/* deprecated compatible */
-		.compatible = "mediatek,mt8195-mmsys",
-		.data = &mt8195_vdosys0_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8195-vdosys0",
-		.data = &mt8195_vdosys0_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8195-vdosys1",
-		.data = &mt8195_vdosys1_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8195-vppsys0",
-		.data = &mt8195_vppsys0_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8195-vppsys1",
-		.data = &mt8195_vppsys1_driver_data,
-	},
-	{
-		.compatible = "mediatek,mt8365-mmsys",
-		.data = &mt8365_mmsys_driver_data,
-	},
-	{ }
+	{ .compatible = "mediatek,mt2701-mmsys", .data = &mt2701_mmsys_driver_data },
+	{ .compatible = "mediatek,mt2712-mmsys", .data = &mt2712_mmsys_driver_data },
+	{ .compatible = "mediatek,mt6779-mmsys", .data = &mt6779_mmsys_driver_data },
+	{ .compatible = "mediatek,mt6795-mmsys", .data = &mt6795_mmsys_driver_data },
+	{ .compatible = "mediatek,mt6797-mmsys", .data = &mt6797_mmsys_driver_data },
+	{ .compatible = "mediatek,mt8167-mmsys", .data = &mt8167_mmsys_driver_data },
+	{ .compatible = "mediatek,mt8173-mmsys", .data = &mt8173_mmsys_driver_data },
+	{ .compatible = "mediatek,mt8183-mmsys", .data = &mt8183_mmsys_driver_data },
+	{ .compatible = "mediatek,mt8186-mmsys", .data = &mt8186_mmsys_driver_data },
+	{ .compatible = "mediatek,mt8188-vdosys0", .data = &mt8188_vdosys0_driver_data },
+	{ .compatible = "mediatek,mt8192-mmsys", .data = &mt8192_mmsys_driver_data },
+	/* "mediatek,mt8195-mmsys" compatible is deprecated */
+	{ .compatible = "mediatek,mt8195-mmsys", .data = &mt8195_vdosys0_driver_data },
+	{ .compatible = "mediatek,mt8195-vdosys0", .data = &mt8195_vdosys0_driver_data },
+	{ .compatible = "mediatek,mt8195-vdosys1", .data = &mt8195_vdosys1_driver_data },
+	{ .compatible = "mediatek,mt8195-vppsys0", .data = &mt8195_vppsys0_driver_data },
+	{ .compatible = "mediatek,mt8195-vppsys1", .data = &mt8195_vppsys1_driver_data },
+	{ .compatible = "mediatek,mt8365-mmsys", .data = &mt8365_mmsys_driver_data },
+	{ /* sentinel */ }
 };
+MODULE_DEVICE_TABLE(of, of_match_mtk_mmsys);
 
 static struct platform_driver mtk_mmsys_drv = {
 	.driver = {
@@ -431,20 +449,9 @@ static struct platform_driver mtk_mmsys_drv = {
 		.of_match_table = of_match_mtk_mmsys,
 	},
 	.probe = mtk_mmsys_probe,
+	.remove = mtk_mmsys_remove,
 };
-
-static int __init mtk_mmsys_init(void)
-{
-	return platform_driver_register(&mtk_mmsys_drv);
-}
-
-static void __exit mtk_mmsys_exit(void)
-{
-	platform_driver_unregister(&mtk_mmsys_drv);
-}
-
-module_init(mtk_mmsys_init);
-module_exit(mtk_mmsys_exit);
+module_platform_driver(mtk_mmsys_drv);
 
 MODULE_AUTHOR("Yongqiang Niu <yongqiang.niu@mediatek.com>");
 MODULE_DESCRIPTION("MediaTek SoC MMSYS driver");
diff --git a/drivers/soc/mediatek/mtk-mmsys.h b/drivers/soc/mediatek/mtk-mmsys.h
index 56f8cc3a97b7..6725403d2e3a 100644
--- a/drivers/soc/mediatek/mtk-mmsys.h
+++ b/drivers/soc/mediatek/mtk-mmsys.h
@@ -96,7 +96,7 @@ struct mtk_mmsys_driver_data {
 };
 
 /*
- * Routes in mt8173, mt2701, mt2712 are different. That means
+ * Routes in mt2701 and mt2712 are different. That means
  * in the same register address, it controls different input/output
  * selection for each SoC. But, right now, they use the same table as
  * default routes meet their requirements. But we don't have the complete
diff --git a/drivers/soc/mediatek/mtk-mutex.c b/drivers/soc/mediatek/mtk-mutex.c
index c5b1b42303ac..26f3d9a41496 100644
--- a/drivers/soc/mediatek/mtk-mutex.c
+++ b/drivers/soc/mediatek/mtk-mutex.c
@@ -14,6 +14,8 @@
 #include <linux/soc/mediatek/mtk-mutex.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
 
+#define MTK_MUTEX_MAX_HANDLES			10
+
 #define MT2701_MUTEX0_MOD0			0x2c
 #define MT2701_MUTEX0_SOF0			0x30
 #define MT8183_MUTEX0_MOD0			0x30
@@ -23,6 +25,7 @@
 #define DISP_REG_MUTEX(n)			(0x24 + 0x20 * (n))
 #define DISP_REG_MUTEX_RST(n)			(0x28 + 0x20 * (n))
 #define DISP_REG_MUTEX_MOD(mutex_mod_reg, n)	(mutex_mod_reg + 0x20 * (n))
+#define DISP_REG_MUTEX_MOD1(mutex_mod_reg, n)	((mutex_mod_reg) + 0x20 * (n) + 0x4)
 #define DISP_REG_MUTEX_SOF(mutex_sof_reg, n)	(mutex_sof_reg + 0x20 * (n))
 #define DISP_REG_MUTEX_MOD2(n)			(0x34 + 0x20 * (n))
 
@@ -163,6 +166,53 @@
 #define MT8195_MUTEX_MOD_DISP1_DPI1		26
 #define MT8195_MUTEX_MOD_DISP1_DP_INTF0		27
 
+/* VPPSYS0 */
+#define MT8195_MUTEX_MOD_MDP_RDMA0             0
+#define MT8195_MUTEX_MOD_MDP_FG0               1
+#define MT8195_MUTEX_MOD_MDP_STITCH0           2
+#define MT8195_MUTEX_MOD_MDP_HDR0              3
+#define MT8195_MUTEX_MOD_MDP_AAL0              4
+#define MT8195_MUTEX_MOD_MDP_RSZ0              5
+#define MT8195_MUTEX_MOD_MDP_TDSHP0            6
+#define MT8195_MUTEX_MOD_MDP_COLOR0            7
+#define MT8195_MUTEX_MOD_MDP_OVL0              8
+#define MT8195_MUTEX_MOD_MDP_PAD0              9
+#define MT8195_MUTEX_MOD_MDP_TCC0              10
+#define MT8195_MUTEX_MOD_MDP_WROT0             11
+
+/* VPPSYS1 */
+#define MT8195_MUTEX_MOD_MDP_TCC1              3
+#define MT8195_MUTEX_MOD_MDP_RDMA1             4
+#define MT8195_MUTEX_MOD_MDP_RDMA2             5
+#define MT8195_MUTEX_MOD_MDP_RDMA3             6
+#define MT8195_MUTEX_MOD_MDP_FG1               7
+#define MT8195_MUTEX_MOD_MDP_FG2               8
+#define MT8195_MUTEX_MOD_MDP_FG3               9
+#define MT8195_MUTEX_MOD_MDP_HDR1              10
+#define MT8195_MUTEX_MOD_MDP_HDR2              11
+#define MT8195_MUTEX_MOD_MDP_HDR3              12
+#define MT8195_MUTEX_MOD_MDP_AAL1              13
+#define MT8195_MUTEX_MOD_MDP_AAL2              14
+#define MT8195_MUTEX_MOD_MDP_AAL3              15
+#define MT8195_MUTEX_MOD_MDP_RSZ1              16
+#define MT8195_MUTEX_MOD_MDP_RSZ2              17
+#define MT8195_MUTEX_MOD_MDP_RSZ3              18
+#define MT8195_MUTEX_MOD_MDP_TDSHP1            19
+#define MT8195_MUTEX_MOD_MDP_TDSHP2            20
+#define MT8195_MUTEX_MOD_MDP_TDSHP3            21
+#define MT8195_MUTEX_MOD_MDP_MERGE2            22
+#define MT8195_MUTEX_MOD_MDP_MERGE3            23
+#define MT8195_MUTEX_MOD_MDP_COLOR1            24
+#define MT8195_MUTEX_MOD_MDP_COLOR2            25
+#define MT8195_MUTEX_MOD_MDP_COLOR3            26
+#define MT8195_MUTEX_MOD_MDP_OVL1              27
+#define MT8195_MUTEX_MOD_MDP_PAD1              28
+#define MT8195_MUTEX_MOD_MDP_PAD2              29
+#define MT8195_MUTEX_MOD_MDP_PAD3              30
+#define MT8195_MUTEX_MOD_MDP_WROT1             31
+#define MT8195_MUTEX_MOD_MDP_WROT2             32
+#define MT8195_MUTEX_MOD_MDP_WROT3             33
+
 #define MT8365_MUTEX_MOD_DISP_OVL0		7
 #define MT8365_MUTEX_MOD_DISP_OVL0_2L		8
 #define MT8365_MUTEX_MOD_DISP_RDMA0		9
@@ -234,7 +284,7 @@
 #define MT8195_MUTEX_EOF_DPI1			(MT8195_MUTEX_SOF_DPI1 << 7)
 
 struct mtk_mutex {
-	int id;
+	u8 id;
 	bool claimed;
 };
 
@@ -264,7 +314,7 @@ struct mtk_mutex_ctx {
 	struct device			*dev;
 	struct clk			*clk;
 	void __iomem			*regs;
-	struct mtk_mutex		mutex[10];
+	struct mtk_mutex		mutex[MTK_MUTEX_MAX_HANDLES];
 	const struct mtk_mutex_data	*data;
 	phys_addr_t			addr;
 	struct cmdq_client_reg		cmdq_reg;
@@ -443,6 +493,52 @@ static const unsigned int mt8195_mutex_mod[DDP_COMPONENT_ID_MAX] = {
 	[DDP_COMPONENT_DP_INTF1] = MT8195_MUTEX_MOD_DISP1_DP_INTF0,
 };
 
+static const unsigned int mt8195_mutex_table_mod[MUTEX_MOD_IDX_MAX] = {
+	[MUTEX_MOD_IDX_MDP_RDMA0] = MT8195_MUTEX_MOD_MDP_RDMA0,
+	[MUTEX_MOD_IDX_MDP_RDMA1] = MT8195_MUTEX_MOD_MDP_RDMA1,
+	[MUTEX_MOD_IDX_MDP_RDMA2] = MT8195_MUTEX_MOD_MDP_RDMA2,
+	[MUTEX_MOD_IDX_MDP_RDMA3] = MT8195_MUTEX_MOD_MDP_RDMA3,
+	[MUTEX_MOD_IDX_MDP_STITCH0] = MT8195_MUTEX_MOD_MDP_STITCH0,
+	[MUTEX_MOD_IDX_MDP_FG0] = MT8195_MUTEX_MOD_MDP_FG0,
+	[MUTEX_MOD_IDX_MDP_FG1] = MT8195_MUTEX_MOD_MDP_FG1,
+	[MUTEX_MOD_IDX_MDP_FG2] = MT8195_MUTEX_MOD_MDP_FG2,
+	[MUTEX_MOD_IDX_MDP_FG3] = MT8195_MUTEX_MOD_MDP_FG3,
+	[MUTEX_MOD_IDX_MDP_HDR0] = MT8195_MUTEX_MOD_MDP_HDR0,
+	[MUTEX_MOD_IDX_MDP_HDR1] = MT8195_MUTEX_MOD_MDP_HDR1,
+	[MUTEX_MOD_IDX_MDP_HDR2] = MT8195_MUTEX_MOD_MDP_HDR2,
+	[MUTEX_MOD_IDX_MDP_HDR3] = MT8195_MUTEX_MOD_MDP_HDR3,
+	[MUTEX_MOD_IDX_MDP_AAL0] = MT8195_MUTEX_MOD_MDP_AAL0,
+	[MUTEX_MOD_IDX_MDP_AAL1] = MT8195_MUTEX_MOD_MDP_AAL1,
+	[MUTEX_MOD_IDX_MDP_AAL2] = MT8195_MUTEX_MOD_MDP_AAL2,
+	[MUTEX_MOD_IDX_MDP_AAL3] = MT8195_MUTEX_MOD_MDP_AAL3,
+	[MUTEX_MOD_IDX_MDP_RSZ0] = MT8195_MUTEX_MOD_MDP_RSZ0,
+	[MUTEX_MOD_IDX_MDP_RSZ1] = MT8195_MUTEX_MOD_MDP_RSZ1,
+	[MUTEX_MOD_IDX_MDP_RSZ2] = MT8195_MUTEX_MOD_MDP_RSZ2,
+	[MUTEX_MOD_IDX_MDP_RSZ3] = MT8195_MUTEX_MOD_MDP_RSZ3,
+	[MUTEX_MOD_IDX_MDP_MERGE2] = MT8195_MUTEX_MOD_MDP_MERGE2,
+	[MUTEX_MOD_IDX_MDP_MERGE3] = MT8195_MUTEX_MOD_MDP_MERGE3,
+	[MUTEX_MOD_IDX_MDP_TDSHP0] = MT8195_MUTEX_MOD_MDP_TDSHP0,
+	[MUTEX_MOD_IDX_MDP_TDSHP1] = MT8195_MUTEX_MOD_MDP_TDSHP1,
+	[MUTEX_MOD_IDX_MDP_TDSHP2] = MT8195_MUTEX_MOD_MDP_TDSHP2,
+	[MUTEX_MOD_IDX_MDP_TDSHP3] = MT8195_MUTEX_MOD_MDP_TDSHP3,
+	[MUTEX_MOD_IDX_MDP_COLOR0] = MT8195_MUTEX_MOD_MDP_COLOR0,
+	[MUTEX_MOD_IDX_MDP_COLOR1] = MT8195_MUTEX_MOD_MDP_COLOR1,
+	[MUTEX_MOD_IDX_MDP_COLOR2] = MT8195_MUTEX_MOD_MDP_COLOR2,
+	[MUTEX_MOD_IDX_MDP_COLOR3] = MT8195_MUTEX_MOD_MDP_COLOR3,
+	[MUTEX_MOD_IDX_MDP_OVL0] = MT8195_MUTEX_MOD_MDP_OVL0,
+	[MUTEX_MOD_IDX_MDP_OVL1] = MT8195_MUTEX_MOD_MDP_OVL1,
+	[MUTEX_MOD_IDX_MDP_PAD0] = MT8195_MUTEX_MOD_MDP_PAD0,
+	[MUTEX_MOD_IDX_MDP_PAD1] = MT8195_MUTEX_MOD_MDP_PAD1,
+	[MUTEX_MOD_IDX_MDP_PAD2] = MT8195_MUTEX_MOD_MDP_PAD2,
+	[MUTEX_MOD_IDX_MDP_PAD3] = MT8195_MUTEX_MOD_MDP_PAD3,
+	[MUTEX_MOD_IDX_MDP_TCC0] = MT8195_MUTEX_MOD_MDP_TCC0,
+	[MUTEX_MOD_IDX_MDP_TCC1] = MT8195_MUTEX_MOD_MDP_TCC1,
+	[MUTEX_MOD_IDX_MDP_WROT0] = MT8195_MUTEX_MOD_MDP_WROT0,
+	[MUTEX_MOD_IDX_MDP_WROT1] = MT8195_MUTEX_MOD_MDP_WROT1,
+	[MUTEX_MOD_IDX_MDP_WROT2] = MT8195_MUTEX_MOD_MDP_WROT2,
+	[MUTEX_MOD_IDX_MDP_WROT3] = MT8195_MUTEX_MOD_MDP_WROT3,
+};
+
 static const unsigned int mt8365_mutex_mod[DDP_COMPONENT_ID_MAX] = {
 	[DDP_COMPONENT_AAL0] = MT8365_MUTEX_MOD_DISP_AAL,
 	[DDP_COMPONENT_CCORR] = MT8365_MUTEX_MOD_DISP_CCORR,
@@ -603,6 +699,13 @@ static const struct mtk_mutex_data mt8195_mutex_driver_data = {
 	.mutex_sof_reg = MT8183_MUTEX0_SOF0,
 };
 
+static const struct mtk_mutex_data mt8195_vpp_mutex_driver_data = {
+	.mutex_sof = mt8195_mutex_sof,
+	.mutex_mod_reg = MT8183_MUTEX0_MOD0,
+	.mutex_sof_reg = MT8183_MUTEX0_SOF0,
+	.mutex_table_mod = mt8195_mutex_table_mod,
+};
+
 static const struct mtk_mutex_data mt8365_mutex_driver_data = {
 	.mutex_mod = mt8365_mutex_mod,
 	.mutex_sof = mt8183_mutex_sof,
@@ -616,7 +719,7 @@ struct mtk_mutex *mtk_mutex_get(struct device *dev)
 	struct mtk_mutex_ctx *mtx = dev_get_drvdata(dev);
 	int i;
 
-	for (i = 0; i < 10; i++)
+	for (i = 0; i < MTK_MUTEX_MAX_HANDLES; i++)
 		if (!mtx->mutex[i].claimed) {
 			mtx->mutex[i].claimed = true;
 			return &mtx->mutex[i];
@@ -768,23 +871,18 @@ int mtk_mutex_enable_by_cmdq(struct mtk_mutex *mutex, void *pkt)
 {
 	struct mtk_mutex_ctx *mtx = container_of(mutex, struct mtk_mutex_ctx,
 						 mutex[mutex->id]);
-#if IS_REACHABLE(CONFIG_MTK_CMDQ)
 	struct cmdq_pkt *cmdq_pkt = (struct cmdq_pkt *)pkt;
 
 	WARN_ON(&mtx->mutex[mutex->id] != mutex);
 
 	if (!mtx->cmdq_reg.size) {
 		dev_err(mtx->dev, "mediatek,gce-client-reg hasn't been set");
-		return -EINVAL;
+		return -ENODEV;
 	}
 
 	cmdq_pkt_write(cmdq_pkt, mtx->cmdq_reg.subsys,
 		       mtx->addr + DISP_REG_MUTEX_EN(mutex->id), 1);
 	return 0;
-#else
-	dev_err(mtx->dev, "Not support for enable MUTEX by CMDQ");
-	return -ENODEV;
-#endif
 }
 EXPORT_SYMBOL_GPL(mtk_mutex_enable_by_cmdq);
 
@@ -828,7 +926,7 @@ int mtk_mutex_write_mod(struct mtk_mutex *mutex,
 	struct mtk_mutex_ctx *mtx = container_of(mutex, struct mtk_mutex_ctx,
 						 mutex[mutex->id]);
 	unsigned int reg;
-	unsigned int offset;
+	u32 reg_offset, id_offset = 0;
 
 	WARN_ON(&mtx->mutex[mutex->id] != mutex);
 
@@ -838,16 +936,34 @@ int mtk_mutex_write_mod(struct mtk_mutex *mutex,
 		return -EINVAL;
 	}
 
-	offset = DISP_REG_MUTEX_MOD(mtx->data->mutex_mod_reg,
-				    mutex->id);
-	reg = readl_relaxed(mtx->regs + offset);
+	/*
+	 * Some SoCs may have multiple MUTEX_MOD registers as more than 32 mods
+	 * are present, hence requiring multiple 32-bits registers.
+	 *
+	 * The mutex_table_mod fully represents that by defining the number of
+	 * the mod sequentially, later used as a bit number, which can be more
+	 * than 0..31.
+	 *
+	 * In order to retain compatibility with older SoCs, we perform R/W on
+	 * the single 32 bits registers, but this requires us to translate the
+	 * mutex ID bit accordingly.
+	 */
+	if (mtx->data->mutex_table_mod[idx] < 32) {
+		reg_offset = DISP_REG_MUTEX_MOD(mtx->data->mutex_mod_reg,
+						mutex->id);
+	} else {
+		reg_offset = DISP_REG_MUTEX_MOD1(mtx->data->mutex_mod_reg,
+						 mutex->id);
+		id_offset = 32;
+	}
 
+	reg = readl_relaxed(mtx->regs + reg_offset);
 	if (clear)
-		reg &= ~BIT(mtx->data->mutex_table_mod[idx]);
+		reg &= ~BIT(mtx->data->mutex_table_mod[idx] - id_offset);
 	else
-		reg |= BIT(mtx->data->mutex_table_mod[idx]);
+		reg |= BIT(mtx->data->mutex_table_mod[idx] - id_offset);
 
-	writel_relaxed(reg, mtx->regs + offset);
+	writel_relaxed(reg, mtx->regs + reg_offset);
 
 	return 0;
 }
@@ -879,27 +995,21 @@ static int mtk_mutex_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct mtk_mutex_ctx *mtx;
 	struct resource *regs;
-	int i;
-#if IS_REACHABLE(CONFIG_MTK_CMDQ)
-	int ret;
-#endif
+	int i, ret;
 
 	mtx = devm_kzalloc(dev, sizeof(*mtx), GFP_KERNEL);
 	if (!mtx)
 		return -ENOMEM;
 
-	for (i = 0; i < 10; i++)
+	for (i = 0; i < MTK_MUTEX_MAX_HANDLES; i++)
 		mtx->mutex[i].id = i;
 
 	mtx->data = of_device_get_match_data(dev);
 
 	if (!mtx->data->no_clk) {
 		mtx->clk = devm_clk_get(dev, NULL);
-		if (IS_ERR(mtx->clk)) {
-			if (PTR_ERR(mtx->clk) != -EPROBE_DEFER)
-				dev_err(dev, "Failed to get clock\n");
-			return PTR_ERR(mtx->clk);
-		}
+		if (IS_ERR(mtx->clk))
+			return dev_err_probe(dev, PTR_ERR(mtx->clk), "Failed to get clock\n");
 	}
 
 	mtx->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &regs);
@@ -909,11 +1019,10 @@ static int mtk_mutex_probe(struct platform_device *pdev)
 	}
 	mtx->addr = regs->start;
 
-#if IS_REACHABLE(CONFIG_MTK_CMDQ)
+	/* CMDQ is optional */
 	ret = cmdq_dev_get_client_reg(dev, &mtx->cmdq_reg, 0);
 	if (ret)
 		dev_dbg(dev, "No mediatek,gce-client-reg!\n");
-#endif
 
 	platform_set_drvdata(pdev, mtx);
 
@@ -921,31 +1030,20 @@ static int mtk_mutex_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id mutex_driver_dt_match[] = {
-	{ .compatible = "mediatek,mt2701-disp-mutex",
-	  .data = &mt2701_mutex_driver_data},
-	{ .compatible = "mediatek,mt2712-disp-mutex",
-	  .data = &mt2712_mutex_driver_data},
-	{ .compatible = "mediatek,mt6795-disp-mutex",
-	  .data = &mt6795_mutex_driver_data},
-	{ .compatible = "mediatek,mt8167-disp-mutex",
-	  .data = &mt8167_mutex_driver_data},
-	{ .compatible = "mediatek,mt8173-disp-mutex",
-	  .data = &mt8173_mutex_driver_data},
-	{ .compatible = "mediatek,mt8183-disp-mutex",
-	  .data = &mt8183_mutex_driver_data},
-	{ .compatible = "mediatek,mt8186-disp-mutex",
-	  .data = &mt8186_mutex_driver_data},
-	{ .compatible = "mediatek,mt8186-mdp3-mutex",
-	  .data = &mt8186_mdp_mutex_driver_data},
-	{ .compatible = "mediatek,mt8188-disp-mutex",
-	  .data = &mt8188_mutex_driver_data},
-	{ .compatible = "mediatek,mt8192-disp-mutex",
-	  .data = &mt8192_mutex_driver_data},
-	{ .compatible = "mediatek,mt8195-disp-mutex",
-	  .data = &mt8195_mutex_driver_data},
-	{ .compatible = "mediatek,mt8365-disp-mutex",
-	  .data = &mt8365_mutex_driver_data},
-	{},
+	{ .compatible = "mediatek,mt2701-disp-mutex", .data = &mt2701_mutex_driver_data },
+	{ .compatible = "mediatek,mt2712-disp-mutex", .data = &mt2712_mutex_driver_data },
+	{ .compatible = "mediatek,mt6795-disp-mutex", .data = &mt6795_mutex_driver_data },
+	{ .compatible = "mediatek,mt8167-disp-mutex", .data = &mt8167_mutex_driver_data },
+	{ .compatible = "mediatek,mt8173-disp-mutex", .data = &mt8173_mutex_driver_data },
+	{ .compatible = "mediatek,mt8183-disp-mutex", .data = &mt8183_mutex_driver_data },
+	{ .compatible = "mediatek,mt8186-disp-mutex", .data = &mt8186_mutex_driver_data },
+	{ .compatible = "mediatek,mt8186-mdp3-mutex", .data = &mt8186_mdp_mutex_driver_data },
+	{ .compatible = "mediatek,mt8188-disp-mutex", .data = &mt8188_mutex_driver_data },
+	{ .compatible = "mediatek,mt8192-disp-mutex", .data = &mt8192_mutex_driver_data },
+	{ .compatible = "mediatek,mt8195-disp-mutex", .data = &mt8195_mutex_driver_data },
+	{ .compatible = "mediatek,mt8195-vpp-mutex",  .data = &mt8195_vpp_mutex_driver_data },
+	{ .compatible = "mediatek,mt8365-disp-mutex", .data = &mt8365_mutex_driver_data },
+	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, mutex_driver_dt_match);
 
@@ -957,19 +1055,7 @@ static struct platform_driver mtk_mutex_driver = {
 		.of_match_table = mutex_driver_dt_match,
 	},
 };
-
-static int __init mtk_mutex_init(void)
-{
-	return platform_driver_register(&mtk_mutex_driver);
-}
-
-static void __exit mtk_mutex_exit(void)
-{
-	platform_driver_unregister(&mtk_mutex_driver);
-}
-
-module_init(mtk_mutex_init);
-module_exit(mtk_mutex_exit);
+module_platform_driver(mtk_mutex_driver);
 
 MODULE_AUTHOR("Yongqiang Niu <yongqiang.niu@mediatek.com>");
 MODULE_DESCRIPTION("MediaTek SoC MUTEX driver");
diff --git a/drivers/soc/mediatek/mtk-svs.c b/drivers/soc/mediatek/mtk-svs.c
index f26eb2f637d5..81585733c8a9 100644
--- a/drivers/soc/mediatek/mtk-svs.c
+++ b/drivers/soc/mediatek/mtk-svs.c
@@ -7,6 +7,7 @@
 #include <linux/bits.h>
 #include <linux/clk.h>
 #include <linux/completion.h>
+#include <linux/cpu.h>
 #include <linux/cpuidle.h>
 #include <linux/debugfs.h>
 #include <linux/device.h>
@@ -558,7 +559,7 @@ static int svs_adjust_pm_opp_volts(struct svs_bank *svsb)
 	}
 
 	/* Get thermal effect */
-	if (svsb->phase == SVSB_PHASE_MON) {
+	if (!IS_ERR_OR_NULL(svsb->tzd)) {
 		ret = thermal_zone_get_temp(svsb->tzd, &tzone_temp);
 		if (ret || (svsb->temp > SVSB_TEMP_UPPER_BOUND &&
 			    svsb->temp < SVSB_TEMP_LOWER_BOUND)) {
@@ -573,7 +574,8 @@ static int svs_adjust_pm_opp_volts(struct svs_bank *svsb)
 			temp_voffset += svsb->tzone_ltemp_voffset;
 
 		/* 2-line bank update all opp volts when running mon mode */
-		if (svsb->type == SVSB_HIGH || svsb->type == SVSB_LOW) {
+		if (svsb->phase == SVSB_PHASE_MON && (svsb->type == SVSB_HIGH ||
+						      svsb->type == SVSB_LOW)) {
 			opp_start = 0;
 			opp_stop = svsb->opp_count;
 		}
@@ -589,11 +591,6 @@ static int svs_adjust_pm_opp_volts(struct svs_bank *svsb)
 			/* do nothing */
 			goto unlock_mutex;
 		case SVSB_PHASE_INIT02:
-			svsb_volt = max(svsb->volt[i], svsb->vmin);
-			opp_volt = svs_bank_volt_to_opp_volt(svsb_volt,
-							     svsb->volt_step,
-							     svsb->volt_base);
-			break;
 		case SVSB_PHASE_MON:
 			svsb_volt = max(svsb->volt[i] + temp_voffset, svsb->vmin);
 			opp_volt = svs_bank_volt_to_opp_volt(svsb_volt,
@@ -624,6 +621,25 @@ unlock_mutex:
 	return ret;
 }
 
+static void svs_bank_disable_and_restore_default_volts(struct svs_platform *svsp,
+						       struct svs_bank *svsb)
+{
+	unsigned long flags;
+
+	if (svsb->mode_support == SVSB_MODE_ALL_DISABLE)
+		return;
+
+	spin_lock_irqsave(&svs_lock, flags);
+	svsp->pbank = svsb;
+	svs_switch_bank(svsp);
+	svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN);
+	svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS);
+	spin_unlock_irqrestore(&svs_lock, flags);
+
+	svsb->phase = SVSB_PHASE_ERROR;
+	svs_adjust_pm_opp_volts(svsb);
+}
+
 #ifdef CONFIG_DEBUG_FS
 static int svs_dump_debug_show(struct seq_file *m, void *p)
 {
@@ -700,7 +716,6 @@ static ssize_t svs_enable_debug_write(struct file *filp,
 {
 	struct svs_bank *svsb = file_inode(filp)->i_private;
 	struct svs_platform *svsp = dev_get_drvdata(svsb->dev);
-	unsigned long flags;
 	int enabled, ret;
 	char *buf = NULL;
 
@@ -716,16 +731,8 @@ static ssize_t svs_enable_debug_write(struct file *filp,
 		return ret;
 
 	if (!enabled) {
-		spin_lock_irqsave(&svs_lock, flags);
-		svsp->pbank = svsb;
+		svs_bank_disable_and_restore_default_volts(svsp, svsb);
 		svsb->mode_support = SVSB_MODE_ALL_DISABLE;
-		svs_switch_bank(svsp);
-		svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN);
-		svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS);
-		spin_unlock_irqrestore(&svs_lock, flags);
-
-		svsb->phase = SVSB_PHASE_ERROR;
-		svs_adjust_pm_opp_volts(svsb);
 	}
 
 	kfree(buf);
@@ -1508,16 +1515,7 @@ static int svs_init02(struct svs_platform *svsp)
 out_of_init02:
 	for (idx = 0; idx < svsp->bank_max; idx++) {
 		svsb = &svsp->banks[idx];
-
-		spin_lock_irqsave(&svs_lock, flags);
-		svsp->pbank = svsb;
-		svs_switch_bank(svsp);
-		svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN);
-		svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS);
-		spin_unlock_irqrestore(&svs_lock, flags);
-
-		svsb->phase = SVSB_PHASE_ERROR;
-		svs_adjust_pm_opp_volts(svsb);
+		svs_bank_disable_and_restore_default_volts(svsp, svsb);
 	}
 
 	return ret;
@@ -1563,23 +1561,12 @@ static int svs_suspend(struct device *dev)
 {
 	struct svs_platform *svsp = dev_get_drvdata(dev);
 	struct svs_bank *svsb;
-	unsigned long flags;
 	int ret;
 	u32 idx;
 
 	for (idx = 0; idx < svsp->bank_max; idx++) {
 		svsb = &svsp->banks[idx];
-
-		/* This might wait for svs_isr() process */
-		spin_lock_irqsave(&svs_lock, flags);
-		svsp->pbank = svsb;
-		svs_switch_bank(svsp);
-		svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN);
-		svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS);
-		spin_unlock_irqrestore(&svs_lock, flags);
-
-		svsb->phase = SVSB_PHASE_ERROR;
-		svs_adjust_pm_opp_volts(svsb);
+		svs_bank_disable_and_restore_default_volts(svsp, svsb);
 	}
 
 	ret = reset_control_assert(svsp->rst);
@@ -1693,7 +1680,7 @@ static int svs_bank_resource_setup(struct svs_platform *svsp)
 			}
 		}
 
-		if (svsb->mode_support & SVSB_MODE_MON) {
+		if (!IS_ERR_OR_NULL(svsb->tzone_name)) {
 			svsb->tzd = thermal_zone_get_zone_by_name(svsb->tzone_name);
 			if (IS_ERR(svsb->tzd)) {
 				dev_err(svsb->dev, "cannot get \"%s\" thermal zone\n",
@@ -1729,26 +1716,28 @@ static int svs_bank_resource_setup(struct svs_platform *svsp)
 	return 0;
 }
 
-static int svs_thermal_efuse_get_data(struct svs_platform *svsp)
+static int svs_get_efuse_data(struct svs_platform *svsp,
+			      const char *nvmem_cell_name,
+			      u32 **svsp_efuse, size_t *svsp_efuse_max)
 {
 	struct nvmem_cell *cell;
 
-	/* Thermal efuse parsing */
-	cell = nvmem_cell_get(svsp->dev, "t-calibration-data");
-	if (IS_ERR_OR_NULL(cell)) {
-		dev_err(svsp->dev, "no \"t-calibration-data\"? %ld\n", PTR_ERR(cell));
+	cell = nvmem_cell_get(svsp->dev, nvmem_cell_name);
+	if (IS_ERR(cell)) {
+		dev_err(svsp->dev, "no \"%s\"? %ld\n",
+			nvmem_cell_name, PTR_ERR(cell));
 		return PTR_ERR(cell);
 	}
 
-	svsp->tefuse = nvmem_cell_read(cell, &svsp->tefuse_max);
-	if (IS_ERR(svsp->tefuse)) {
-		dev_err(svsp->dev, "cannot read thermal efuse: %ld\n",
-			PTR_ERR(svsp->tefuse));
+	*svsp_efuse = nvmem_cell_read(cell, svsp_efuse_max);
+	if (IS_ERR(*svsp_efuse)) {
+		dev_err(svsp->dev, "cannot read \"%s\" efuse: %ld\n",
+			nvmem_cell_name, PTR_ERR(*svsp_efuse));
 		nvmem_cell_put(cell);
-		return PTR_ERR(svsp->tefuse);
+		return PTR_ERR(*svsp_efuse);
 	}
 
-	svsp->tefuse_max /= sizeof(u32);
+	*svsp_efuse_max /= sizeof(u32);
 	nvmem_cell_put(cell);
 
 	return 0;
@@ -1796,7 +1785,8 @@ static bool svs_mt8192_efuse_parsing(struct svs_platform *svsp)
 		svsb->vmax += svsb->dvt_fixed;
 	}
 
-	ret = svs_thermal_efuse_get_data(svsp);
+	ret = svs_get_efuse_data(svsp, "t-calibration-data",
+				 &svsp->tefuse, &svsp->tefuse_max);
 	if (ret)
 		return false;
 
@@ -1901,7 +1891,8 @@ static bool svs_mt8183_efuse_parsing(struct svs_platform *svsp)
 		}
 	}
 
-	ret = svs_thermal_efuse_get_data(svsp);
+	ret = svs_get_efuse_data(svsp, "t-calibration-data",
+				 &svsp->tefuse, &svsp->tefuse_max);
 	if (ret)
 		return false;
 
@@ -2003,32 +1994,6 @@ remove_mt8183_svsb_mon_mode:
 	return true;
 }
 
-static bool svs_is_efuse_data_correct(struct svs_platform *svsp)
-{
-	struct nvmem_cell *cell;
-
-	/* Get svs efuse by nvmem */
-	cell = nvmem_cell_get(svsp->dev, "svs-calibration-data");
-	if (IS_ERR(cell)) {
-		dev_err(svsp->dev, "no \"svs-calibration-data\"? %ld\n",
-			PTR_ERR(cell));
-		return false;
-	}
-
-	svsp->efuse = nvmem_cell_read(cell, &svsp->efuse_max);
-	if (IS_ERR(svsp->efuse)) {
-		dev_err(svsp->dev, "cannot read svs efuse: %ld\n",
-			PTR_ERR(svsp->efuse));
-		nvmem_cell_put(cell);
-		return false;
-	}
-
-	svsp->efuse_max /= sizeof(u32);
-	nvmem_cell_put(cell);
-
-	return true;
-}
-
 static struct device *svs_get_subsys_device(struct svs_platform *svsp,
 					    const char *node_name)
 {
@@ -2059,11 +2024,6 @@ static struct device *svs_add_device_link(struct svs_platform *svsp,
 	struct device *dev;
 	struct device_link *sup_link;
 
-	if (!node_name) {
-		dev_err(svsp->dev, "node name cannot be null\n");
-		return ERR_PTR(-EINVAL);
-	}
-
 	dev = svs_get_subsys_device(svsp, node_name);
 	if (IS_ERR(dev))
 		return dev;
@@ -2159,6 +2119,7 @@ static struct svs_bank svs_mt8192_banks[] = {
 		.type			= SVSB_LOW,
 		.set_freq_pct		= svs_set_bank_freq_pct_v3,
 		.get_volts		= svs_get_bank_volts_v3,
+		.tzone_name		= "gpu1",
 		.volt_flags		= SVSB_REMOVE_DVTFIXED_VOLT,
 		.mode_support		= SVSB_MODE_INIT02,
 		.opp_count		= MAX_OPP_ENTRIES,
@@ -2176,6 +2137,10 @@ static struct svs_bank svs_mt8192_banks[] = {
 		.core_sel		= 0x0fff0100,
 		.int_st			= BIT(0),
 		.ctl0			= 0x00540003,
+		.tzone_htemp		= 85000,
+		.tzone_htemp_voffset	= 0,
+		.tzone_ltemp		= 25000,
+		.tzone_ltemp_voffset	= 7,
 	},
 	{
 		.sw_id			= SVSB_GPU,
@@ -2364,8 +2329,9 @@ static int svs_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	if (!svs_is_efuse_data_correct(svsp)) {
-		dev_notice(svsp->dev, "efuse data isn't correct\n");
+	ret = svs_get_efuse_data(svsp, "svs-calibration-data",
+				 &svsp->efuse, &svsp->efuse_max);
+	if (ret) {
 		ret = -EPERM;
 		goto svs_probe_free_efuse;
 	}
@@ -2373,19 +2339,19 @@ static int svs_probe(struct platform_device *pdev)
 	if (!svsp_data->efuse_parsing(svsp)) {
 		dev_err(svsp->dev, "efuse data parsing failed\n");
 		ret = -EPERM;
-		goto svs_probe_free_resource;
+		goto svs_probe_free_tefuse;
 	}
 
 	ret = svs_bank_resource_setup(svsp);
 	if (ret) {
 		dev_err(svsp->dev, "svs bank resource setup fail: %d\n", ret);
-		goto svs_probe_free_resource;
+		goto svs_probe_free_tefuse;
 	}
 
 	svsp_irq = platform_get_irq(pdev, 0);
 	if (svsp_irq < 0) {
 		ret = svsp_irq;
-		goto svs_probe_free_resource;
+		goto svs_probe_free_tefuse;
 	}
 
 	svsp->main_clk = devm_clk_get(svsp->dev, "main");
@@ -2393,13 +2359,13 @@ static int svs_probe(struct platform_device *pdev)
 		dev_err(svsp->dev, "failed to get clock: %ld\n",
 			PTR_ERR(svsp->main_clk));
 		ret = PTR_ERR(svsp->main_clk);
-		goto svs_probe_free_resource;
+		goto svs_probe_free_tefuse;
 	}
 
 	ret = clk_prepare_enable(svsp->main_clk);
 	if (ret) {
 		dev_err(svsp->dev, "cannot enable main clk: %d\n", ret);
-		goto svs_probe_free_resource;
+		goto svs_probe_free_tefuse;
 	}
 
 	svsp->base = of_iomap(svsp->dev->of_node, 0);
@@ -2439,7 +2405,7 @@ svs_probe_iounmap:
 svs_probe_clk_disable:
 	clk_disable_unprepare(svsp->main_clk);
 
-svs_probe_free_resource:
+svs_probe_free_tefuse:
 	if (!IS_ERR_OR_NULL(svsp->tefuse))
 		kfree(svsp->tefuse);
 
diff --git a/drivers/soc/microchip/mpfs-sys-controller.c b/drivers/soc/microchip/mpfs-sys-controller.c
index 6e20207b5756..216d9f4ea0ce 100644
--- a/drivers/soc/microchip/mpfs-sys-controller.c
+++ b/drivers/soc/microchip/mpfs-sys-controller.c
@@ -11,12 +11,19 @@
 #include <linux/slab.h>
 #include <linux/kref.h>
 #include <linux/module.h>
+#include <linux/jiffies.h>
 #include <linux/interrupt.h>
 #include <linux/of_platform.h>
 #include <linux/mailbox_client.h>
 #include <linux/platform_device.h>
 #include <soc/microchip/mpfs.h>
 
+/*
+ * This timeout must be long, as some services (example: image authentication)
+ * take significant time to complete
+ */
+#define MPFS_SYS_CTRL_TIMEOUT_MS 30000
+
 static DEFINE_MUTEX(transaction_lock);
 
 struct mpfs_sys_controller {
@@ -28,35 +35,47 @@ struct mpfs_sys_controller {
 
 int mpfs_blocking_transaction(struct mpfs_sys_controller *sys_controller, struct mpfs_mss_msg *msg)
 {
-	int ret, err;
+	unsigned long timeout = msecs_to_jiffies(MPFS_SYS_CTRL_TIMEOUT_MS);
+	int ret;
 
-	err = mutex_lock_interruptible(&transaction_lock);
-	if (err)
-		return err;
+	ret = mutex_lock_interruptible(&transaction_lock);
+	if (ret)
+		return ret;
 
 	reinit_completion(&sys_controller->c);
 
 	ret = mbox_send_message(sys_controller->chan, msg);
-	if (ret >= 0) {
-		if (wait_for_completion_timeout(&sys_controller->c, HZ)) {
-			ret = 0;
-		} else {
-			ret = -ETIMEDOUT;
-			dev_warn(sys_controller->client.dev,
-				 "MPFS sys controller transaction timeout\n");
-		}
+	if (ret < 0) {
+		dev_warn(sys_controller->client.dev, "MPFS sys controller service timeout\n");
+		goto out;
+	}
+
+	/*
+	 * Unfortunately, the system controller will only deliver an interrupt
+	 * if a service succeeds. mbox_send_message() will block until the busy
+	 * flag is gone. If the busy flag is gone but no interrupt has arrived
+	 * to trigger the rx callback then the service can be deemed to have
+	 * failed.
+	 * The caller can then interrogate msg::response::resp_status to
+	 * determine the cause of the failure.
+	 * mbox_send_message() returns positive integers in the success path, so
+	 * ret needs to be cleared if we do get an interrupt.
+	 */
+	if (!wait_for_completion_timeout(&sys_controller->c, timeout)) {
+		ret = -EBADMSG;
+		dev_warn(sys_controller->client.dev, "MPFS sys controller service failed\n");
 	} else {
-		dev_err(sys_controller->client.dev,
-			"mpfs sys controller transaction returned %d\n", ret);
+		ret = 0;
 	}
 
+out:
 	mutex_unlock(&transaction_lock);
 
 	return ret;
 }
 EXPORT_SYMBOL(mpfs_blocking_transaction);
 
-static void rx_callback(struct mbox_client *client, void *msg)
+static void mpfs_sys_controller_rx_callback(struct mbox_client *client, void *msg)
 {
 	struct mpfs_sys_controller *sys_controller =
 		container_of(client, struct mpfs_sys_controller, client);
@@ -66,8 +85,8 @@ static void rx_callback(struct mbox_client *client, void *msg)
 
 static void mpfs_sys_controller_delete(struct kref *kref)
 {
-	struct mpfs_sys_controller *sys_controller = container_of(kref, struct mpfs_sys_controller,
-					       consumers);
+	struct mpfs_sys_controller *sys_controller =
+		container_of(kref, struct mpfs_sys_controller, consumers);
 
 	mbox_free_channel(sys_controller->chan);
 	kfree(sys_controller);
@@ -102,8 +121,9 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	sys_controller->client.dev = dev;
-	sys_controller->client.rx_callback = rx_callback;
+	sys_controller->client.rx_callback = mpfs_sys_controller_rx_callback;
 	sys_controller->client.tx_block = 1U;
+	sys_controller->client.tx_tout = msecs_to_jiffies(MPFS_SYS_CTRL_TIMEOUT_MS);
 
 	sys_controller->chan = mbox_request_channel(&sys_controller->client, 0);
 	if (IS_ERR(sys_controller->chan)) {
diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig
index a8f283086a21..a491718f8064 100644
--- a/drivers/soc/qcom/Kconfig
+++ b/drivers/soc/qcom/Kconfig
@@ -72,7 +72,7 @@ config QCOM_LLCC
 
 config QCOM_KRYO_L2_ACCESSORS
 	bool
-	depends on ARCH_QCOM && ARM64 || COMPILE_TEST
+	depends on (ARCH_QCOM || COMPILE_TEST) && ARM64
 
 config QCOM_MDT_LOADER
 	tristate
@@ -275,4 +275,8 @@ config QCOM_ICC_BWMON
 	  the fixed bandwidth votes from cpufreq (CPU nodes) thus achieve high
 	  memory throughput even with lower CPU frequencies.
 
+config QCOM_INLINE_CRYPTO_ENGINE
+	tristate
+	select QCOM_SCM
+
 endmenu
diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile
index 6e88da899f60..0f43a88b4894 100644
--- a/drivers/soc/qcom/Makefile
+++ b/drivers/soc/qcom/Makefile
@@ -32,3 +32,4 @@ obj-$(CONFIG_QCOM_RPMHPD) += rpmhpd.o
 obj-$(CONFIG_QCOM_RPMPD) += rpmpd.o
 obj-$(CONFIG_QCOM_KRYO_L2_ACCESSORS) +=	kryo-l2-accessors.o
 obj-$(CONFIG_QCOM_ICC_BWMON)	+= icc-bwmon.o
+obj-$(CONFIG_QCOM_INLINE_CRYPTO_ENGINE)	+= ice.o
diff --git a/drivers/soc/qcom/icc-bwmon.c b/drivers/soc/qcom/icc-bwmon.c
index d07be3700db6..fd58c5b69897 100644
--- a/drivers/soc/qcom/icc-bwmon.c
+++ b/drivers/soc/qcom/icc-bwmon.c
@@ -34,14 +34,27 @@
 /* Internal sampling clock frequency */
 #define HW_TIMER_HZ				19200000
 
-#define BWMON_V4_GLOBAL_IRQ_CLEAR		0x008
-#define BWMON_V4_GLOBAL_IRQ_ENABLE		0x00c
+#define BWMON_V4_GLOBAL_IRQ_CLEAR		0x108
+#define BWMON_V4_GLOBAL_IRQ_ENABLE		0x10c
 /*
  * All values here and further are matching regmap fields, so without absolute
  * register offsets.
  */
 #define BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE	BIT(0)
 
+/*
+ * Starting with SDM845, the BWMON4 register space has changed a bit:
+ * the global registers were jammed into the beginning of the monitor region.
+ * To keep the proper offsets, one would have to map <GLOBAL_BASE 0x200> and
+ * <GLOBAL_BASE+0x100 0x300>, which is straight up wrong.
+ * To facilitate for that, while allowing the older, arguably more proper
+ * implementations to work, offset the global registers by -0x100 to avoid
+ * having to map half of the global registers twice.
+ */
+#define BWMON_V4_845_OFFSET			0x100
+#define BWMON_V4_GLOBAL_IRQ_CLEAR_845		(BWMON_V4_GLOBAL_IRQ_CLEAR - BWMON_V4_845_OFFSET)
+#define BWMON_V4_GLOBAL_IRQ_ENABLE_845		(BWMON_V4_GLOBAL_IRQ_ENABLE - BWMON_V4_845_OFFSET)
+
 #define BWMON_V4_IRQ_STATUS			0x100
 #define BWMON_V4_IRQ_CLEAR			0x108
 
@@ -118,9 +131,13 @@
 #define BWMON_NEEDS_FORCE_CLEAR			BIT(1)
 
 enum bwmon_fields {
+	/* Global region fields, keep them at the top */
 	F_GLOBAL_IRQ_CLEAR,
 	F_GLOBAL_IRQ_ENABLE,
-	F_IRQ_STATUS,
+	F_NUM_GLOBAL_FIELDS,
+
+	/* Monitor region fields */
+	F_IRQ_STATUS = F_NUM_GLOBAL_FIELDS,
 	F_IRQ_CLEAR,
 	F_IRQ_ENABLE,
 	F_ENABLE,
@@ -157,6 +174,9 @@ struct icc_bwmon_data {
 
 	const struct regmap_config *regmap_cfg;
 	const struct reg_field *regmap_fields;
+
+	const struct regmap_config *global_regmap_cfg;
+	const struct reg_field *global_regmap_fields;
 };
 
 struct icc_bwmon {
@@ -164,8 +184,8 @@ struct icc_bwmon {
 	const struct icc_bwmon_data *data;
 	int irq;
 
-	struct regmap *regmap;
 	struct regmap_field *regs[F_NUM_FIELDS];
+	struct regmap_field *global_regs[F_NUM_GLOBAL_FIELDS];
 
 	unsigned int max_bw_kbps;
 	unsigned int min_bw_kbps;
@@ -175,8 +195,8 @@ struct icc_bwmon {
 
 /* BWMON v4 */
 static const struct reg_field msm8998_bwmon_reg_fields[] = {
-	[F_GLOBAL_IRQ_CLEAR]	= REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR, 0, 0),
-	[F_GLOBAL_IRQ_ENABLE]	= REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE, 0, 0),
+	[F_GLOBAL_IRQ_CLEAR]	= {},
+	[F_GLOBAL_IRQ_ENABLE]	= {},
 	[F_IRQ_STATUS]		= REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7),
 	[F_IRQ_CLEAR]		= REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7),
 	[F_IRQ_ENABLE]		= REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7),
@@ -202,7 +222,6 @@ static const struct reg_field msm8998_bwmon_reg_fields[] = {
 };
 
 static const struct regmap_range msm8998_bwmon_reg_noread_ranges[] = {
-	regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR, BWMON_V4_GLOBAL_IRQ_CLEAR),
 	regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR),
 	regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR),
 };
@@ -222,16 +241,33 @@ static const struct regmap_access_table msm8998_bwmon_reg_volatile_table = {
 	.n_yes_ranges	= ARRAY_SIZE(msm8998_bwmon_reg_volatile_ranges),
 };
 
+static const struct reg_field msm8998_bwmon_global_reg_fields[] = {
+	[F_GLOBAL_IRQ_CLEAR]	= REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR, 0, 0),
+	[F_GLOBAL_IRQ_ENABLE]	= REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE, 0, 0),
+};
+
+static const struct regmap_range msm8998_bwmon_global_reg_noread_ranges[] = {
+	regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR, BWMON_V4_GLOBAL_IRQ_CLEAR),
+};
+
+static const struct regmap_access_table msm8998_bwmon_global_reg_read_table = {
+	.no_ranges	= msm8998_bwmon_global_reg_noread_ranges,
+	.n_no_ranges	= ARRAY_SIZE(msm8998_bwmon_global_reg_noread_ranges),
+};
+
 /*
  * Fill the cache for non-readable registers only as rest does not really
  * matter and can be read from the device.
  */
 static const struct reg_default msm8998_bwmon_reg_defaults[] = {
-	{ BWMON_V4_GLOBAL_IRQ_CLEAR, 0x0 },
 	{ BWMON_V4_IRQ_CLEAR, 0x0 },
 	{ BWMON_V4_CLEAR, 0x0 },
 };
 
+static const struct reg_default msm8998_bwmon_global_reg_defaults[] = {
+	{ BWMON_V4_GLOBAL_IRQ_CLEAR, 0x0 },
+};
+
 static const struct regmap_config msm8998_bwmon_regmap_cfg = {
 	.reg_bits		= 32,
 	.reg_stride		= 4,
@@ -252,6 +288,93 @@ static const struct regmap_config msm8998_bwmon_regmap_cfg = {
 	.cache_type		= REGCACHE_RBTREE,
 };
 
+static const struct regmap_config msm8998_bwmon_global_regmap_cfg = {
+	.reg_bits		= 32,
+	.reg_stride		= 4,
+	.val_bits		= 32,
+	/*
+	 * No concurrent access expected - driver has one interrupt handler,
+	 * regmap is not shared, no driver or user-space API.
+	 */
+	.disable_locking	= true,
+	.rd_table		= &msm8998_bwmon_global_reg_read_table,
+	.reg_defaults		= msm8998_bwmon_global_reg_defaults,
+	.num_reg_defaults	= ARRAY_SIZE(msm8998_bwmon_global_reg_defaults),
+	/*
+	 * Cache is necessary for using regmap fields with non-readable
+	 * registers.
+	 */
+	.cache_type		= REGCACHE_RBTREE,
+};
+
+static const struct reg_field sdm845_cpu_bwmon_reg_fields[] = {
+	[F_GLOBAL_IRQ_CLEAR]	= REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR_845, 0, 0),
+	[F_GLOBAL_IRQ_ENABLE]	= REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE_845, 0, 0),
+	[F_IRQ_STATUS]		= REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7),
+	[F_IRQ_CLEAR]		= REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7),
+	[F_IRQ_ENABLE]		= REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7),
+	/* F_ENABLE covers entire register to disable other features */
+	[F_ENABLE]		= REG_FIELD(BWMON_V4_ENABLE, 0, 31),
+	[F_CLEAR]		= REG_FIELD(BWMON_V4_CLEAR, 0, 1),
+	[F_SAMPLE_WINDOW]	= REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23),
+	[F_THRESHOLD_HIGH]	= REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11),
+	[F_THRESHOLD_MED]	= REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11),
+	[F_THRESHOLD_LOW]	= REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11),
+	[F_ZONE_ACTIONS_ZONE0]	= REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7),
+	[F_ZONE_ACTIONS_ZONE1]	= REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15),
+	[F_ZONE_ACTIONS_ZONE2]	= REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23),
+	[F_ZONE_ACTIONS_ZONE3]	= REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31),
+	[F_THRESHOLD_COUNT_ZONE0]	= REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7),
+	[F_THRESHOLD_COUNT_ZONE1]	= REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15),
+	[F_THRESHOLD_COUNT_ZONE2]	= REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23),
+	[F_THRESHOLD_COUNT_ZONE3]	= REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31),
+	[F_ZONE0_MAX]		= REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11),
+	[F_ZONE1_MAX]		= REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11),
+	[F_ZONE2_MAX]		= REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11),
+	[F_ZONE3_MAX]		= REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11),
+};
+
+static const struct regmap_range sdm845_cpu_bwmon_reg_noread_ranges[] = {
+	regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR_845, BWMON_V4_GLOBAL_IRQ_CLEAR_845),
+	regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR),
+	regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR),
+};
+
+static const struct regmap_access_table sdm845_cpu_bwmon_reg_read_table = {
+	.no_ranges	= sdm845_cpu_bwmon_reg_noread_ranges,
+	.n_no_ranges	= ARRAY_SIZE(sdm845_cpu_bwmon_reg_noread_ranges),
+};
+
+/*
+ * Fill the cache for non-readable registers only as rest does not really
+ * matter and can be read from the device.
+ */
+static const struct reg_default sdm845_cpu_bwmon_reg_defaults[] = {
+	{ BWMON_V4_GLOBAL_IRQ_CLEAR_845, 0x0 },
+	{ BWMON_V4_IRQ_CLEAR, 0x0 },
+	{ BWMON_V4_CLEAR, 0x0 },
+};
+
+static const struct regmap_config sdm845_cpu_bwmon_regmap_cfg = {
+	.reg_bits		= 32,
+	.reg_stride		= 4,
+	.val_bits		= 32,
+	/*
+	 * No concurrent access expected - driver has one interrupt handler,
+	 * regmap is not shared, no driver or user-space API.
+	 */
+	.disable_locking	= true,
+	.rd_table		= &sdm845_cpu_bwmon_reg_read_table,
+	.volatile_table		= &msm8998_bwmon_reg_volatile_table,
+	.reg_defaults		= sdm845_cpu_bwmon_reg_defaults,
+	.num_reg_defaults	= ARRAY_SIZE(sdm845_cpu_bwmon_reg_defaults),
+	/*
+	 * Cache is necessary for using regmap fields with non-readable
+	 * registers.
+	 */
+	.cache_type		= REGCACHE_RBTREE,
+};
+
 /* BWMON v5 */
 static const struct reg_field sdm845_llcc_bwmon_reg_fields[] = {
 	[F_GLOBAL_IRQ_CLEAR]	= {},
@@ -350,6 +473,13 @@ static void bwmon_clear_counters(struct icc_bwmon *bwmon, bool clear_all)
 
 static void bwmon_clear_irq(struct icc_bwmon *bwmon)
 {
+	struct regmap_field *global_irq_clr;
+
+	if (bwmon->data->global_regmap_fields)
+		global_irq_clr = bwmon->global_regs[F_GLOBAL_IRQ_CLEAR];
+	else
+		global_irq_clr = bwmon->regs[F_GLOBAL_IRQ_CLEAR];
+
 	/*
 	 * Clear zone and global interrupts. The order and barriers are
 	 * important. Quoting downstream Qualcomm msm-4.9 tree:
@@ -370,15 +500,22 @@ static void bwmon_clear_irq(struct icc_bwmon *bwmon)
 	if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR)
 		regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], 0);
 	if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)
-		regmap_field_force_write(bwmon->regs[F_GLOBAL_IRQ_CLEAR],
+		regmap_field_force_write(global_irq_clr,
 					 BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE);
 }
 
 static void bwmon_disable(struct icc_bwmon *bwmon)
 {
+	struct regmap_field *global_irq_en;
+
+	if (bwmon->data->global_regmap_fields)
+		global_irq_en = bwmon->global_regs[F_GLOBAL_IRQ_ENABLE];
+	else
+		global_irq_en = bwmon->regs[F_GLOBAL_IRQ_ENABLE];
+
 	/* Disable interrupts. Strict ordering, see bwmon_clear_irq(). */
 	if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)
-		regmap_field_write(bwmon->regs[F_GLOBAL_IRQ_ENABLE], 0x0);
+		regmap_field_write(global_irq_en, 0x0);
 	regmap_field_write(bwmon->regs[F_IRQ_ENABLE], 0x0);
 
 	/*
@@ -390,10 +527,18 @@ static void bwmon_disable(struct icc_bwmon *bwmon)
 
 static void bwmon_enable(struct icc_bwmon *bwmon, unsigned int irq_enable)
 {
+	struct regmap_field *global_irq_en;
+
+	if (bwmon->data->global_regmap_fields)
+		global_irq_en = bwmon->global_regs[F_GLOBAL_IRQ_ENABLE];
+	else
+		global_irq_en = bwmon->regs[F_GLOBAL_IRQ_ENABLE];
+
 	/* Enable interrupts */
 	if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)
-		regmap_field_write(bwmon->regs[F_GLOBAL_IRQ_ENABLE],
+		regmap_field_write(global_irq_en,
 				   BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE);
+
 	regmap_field_write(bwmon->regs[F_IRQ_ENABLE], irq_enable);
 
 	/* Enable bwmon */
@@ -556,7 +701,9 @@ static int bwmon_init_regmap(struct platform_device *pdev,
 	struct device *dev = &pdev->dev;
 	void __iomem *base;
 	struct regmap *map;
+	int ret;
 
+	/* Map the monitor base */
 	base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(base))
 		return dev_err_probe(dev, PTR_ERR(base),
@@ -567,12 +714,35 @@ static int bwmon_init_regmap(struct platform_device *pdev,
 		return dev_err_probe(dev, PTR_ERR(map),
 				     "failed to initialize regmap\n");
 
+	BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_global_reg_fields) != F_NUM_GLOBAL_FIELDS);
 	BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_reg_fields) != F_NUM_FIELDS);
+	BUILD_BUG_ON(ARRAY_SIZE(sdm845_cpu_bwmon_reg_fields) != F_NUM_FIELDS);
 	BUILD_BUG_ON(ARRAY_SIZE(sdm845_llcc_bwmon_reg_fields) != F_NUM_FIELDS);
 
-	return devm_regmap_field_bulk_alloc(dev, map, bwmon->regs,
+	ret = devm_regmap_field_bulk_alloc(dev, map, bwmon->regs,
 					   bwmon->data->regmap_fields,
 					   F_NUM_FIELDS);
+	if (ret)
+		return ret;
+
+	if (bwmon->data->global_regmap_cfg) {
+		/* Map the global base, if separate */
+		base = devm_platform_ioremap_resource(pdev, 1);
+		if (IS_ERR(base))
+			return dev_err_probe(dev, PTR_ERR(base),
+					     "failed to map bwmon global registers\n");
+
+		map = devm_regmap_init_mmio(dev, base, bwmon->data->global_regmap_cfg);
+		if (IS_ERR(map))
+			return dev_err_probe(dev, PTR_ERR(map),
+					     "failed to initialize global regmap\n");
+
+		ret = devm_regmap_field_bulk_alloc(dev, map, bwmon->global_regs,
+						   bwmon->data->global_regmap_fields,
+						   F_NUM_GLOBAL_FIELDS);
+	}
+
+	return ret;
 }
 
 static int bwmon_probe(struct platform_device *pdev)
@@ -645,6 +815,21 @@ static const struct icc_bwmon_data msm8998_bwmon_data = {
 	.quirks = BWMON_HAS_GLOBAL_IRQ,
 	.regmap_fields = msm8998_bwmon_reg_fields,
 	.regmap_cfg = &msm8998_bwmon_regmap_cfg,
+	.global_regmap_fields = msm8998_bwmon_global_reg_fields,
+	.global_regmap_cfg = &msm8998_bwmon_global_regmap_cfg,
+};
+
+static const struct icc_bwmon_data sdm845_cpu_bwmon_data = {
+	.sample_ms = 4,
+	.count_unit_kb = 64,
+	.default_highbw_kbps = 4800 * 1024, /* 4.8 GBps */
+	.default_medbw_kbps = 512 * 1024, /* 512 MBps */
+	.default_lowbw_kbps = 0,
+	.zone1_thres_count = 16,
+	.zone3_thres_count = 1,
+	.quirks = BWMON_HAS_GLOBAL_IRQ,
+	.regmap_fields = sdm845_cpu_bwmon_reg_fields,
+	.regmap_cfg = &sdm845_cpu_bwmon_regmap_cfg,
 };
 
 static const struct icc_bwmon_data sdm845_llcc_bwmon_data = {
@@ -673,16 +858,18 @@ static const struct icc_bwmon_data sc7280_llcc_bwmon_data = {
 };
 
 static const struct of_device_id bwmon_of_match[] = {
-	{
-		.compatible = "qcom,msm8998-bwmon",
-		.data = &msm8998_bwmon_data
-	}, {
-		.compatible = "qcom,sdm845-llcc-bwmon",
-		.data = &sdm845_llcc_bwmon_data
-	}, {
-		.compatible = "qcom,sc7280-llcc-bwmon",
-		.data = &sc7280_llcc_bwmon_data
-	},
+	/* BWMONv4, separate monitor and global register spaces */
+	{ .compatible = "qcom,msm8998-bwmon", .data = &msm8998_bwmon_data },
+	/* BWMONv4, unified register space */
+	{ .compatible = "qcom,sdm845-bwmon", .data = &sdm845_cpu_bwmon_data },
+	/* BWMONv5 */
+	{ .compatible = "qcom,sdm845-llcc-bwmon", .data = &sdm845_llcc_bwmon_data },
+	{ .compatible = "qcom,sc7280-llcc-bwmon", .data = &sc7280_llcc_bwmon_data },
+
+	/* Compatibles kept for legacy reasons */
+	{ .compatible = "qcom,sc7280-cpu-bwmon", .data = &sdm845_cpu_bwmon_data },
+	{ .compatible = "qcom,sc8280xp-cpu-bwmon", .data = &sdm845_cpu_bwmon_data },
+	{ .compatible = "qcom,sm8550-cpu-bwmon", .data = &sdm845_cpu_bwmon_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, bwmon_of_match);
diff --git a/drivers/soc/qcom/ice.c b/drivers/soc/qcom/ice.c
new file mode 100644
index 000000000000..a6123ea96272
--- /dev/null
+++ b/drivers/soc/qcom/ice.c
@@ -0,0 +1,366 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Qualcomm ICE (Inline Crypto Engine) support.
+ *
+ * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2019, Google LLC
+ * Copyright (c) 2023, Linaro Limited
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/iopoll.h>
+#include <linux/of_platform.h>
+
+#include <linux/firmware/qcom/qcom_scm.h>
+
+#include <soc/qcom/ice.h>
+
+#define AES_256_XTS_KEY_SIZE			64
+
+/* QCOM ICE registers */
+#define QCOM_ICE_REG_VERSION			0x0008
+#define QCOM_ICE_REG_FUSE_SETTING		0x0010
+#define QCOM_ICE_REG_BIST_STATUS		0x0070
+#define QCOM_ICE_REG_ADVANCED_CONTROL		0x1000
+
+/* BIST ("built-in self-test") status flags */
+#define QCOM_ICE_BIST_STATUS_MASK		GENMASK(31, 28)
+
+#define QCOM_ICE_FUSE_SETTING_MASK		0x1
+#define QCOM_ICE_FORCE_HW_KEY0_SETTING_MASK	0x2
+#define QCOM_ICE_FORCE_HW_KEY1_SETTING_MASK	0x4
+
+#define qcom_ice_writel(engine, val, reg)	\
+	writel((val), (engine)->base + (reg))
+
+#define qcom_ice_readl(engine, reg)	\
+	readl((engine)->base + (reg))
+
+struct qcom_ice {
+	struct device *dev;
+	void __iomem *base;
+	struct device_link *link;
+
+	struct clk *core_clk;
+};
+
+static bool qcom_ice_check_supported(struct qcom_ice *ice)
+{
+	u32 regval = qcom_ice_readl(ice, QCOM_ICE_REG_VERSION);
+	struct device *dev = ice->dev;
+	int major = FIELD_GET(GENMASK(31, 24), regval);
+	int minor = FIELD_GET(GENMASK(23, 16), regval);
+	int step = FIELD_GET(GENMASK(15, 0), regval);
+
+	/* For now this driver only supports ICE version 3 and 4. */
+	if (major != 3 && major != 4) {
+		dev_warn(dev, "Unsupported ICE version: v%d.%d.%d\n",
+			 major, minor, step);
+		return false;
+	}
+
+	dev_info(dev, "Found QC Inline Crypto Engine (ICE) v%d.%d.%d\n",
+		 major, minor, step);
+
+	/* If fuses are blown, ICE might not work in the standard way. */
+	regval = qcom_ice_readl(ice, QCOM_ICE_REG_FUSE_SETTING);
+	if (regval & (QCOM_ICE_FUSE_SETTING_MASK |
+		      QCOM_ICE_FORCE_HW_KEY0_SETTING_MASK |
+		      QCOM_ICE_FORCE_HW_KEY1_SETTING_MASK)) {
+		dev_warn(dev, "Fuses are blown; ICE is unusable!\n");
+		return false;
+	}
+
+	return true;
+}
+
+static void qcom_ice_low_power_mode_enable(struct qcom_ice *ice)
+{
+	u32 regval;
+
+	regval = qcom_ice_readl(ice, QCOM_ICE_REG_ADVANCED_CONTROL);
+
+	/* Enable low power mode sequence */
+	regval |= 0x7000;
+	qcom_ice_writel(ice, regval, QCOM_ICE_REG_ADVANCED_CONTROL);
+}
+
+static void qcom_ice_optimization_enable(struct qcom_ice *ice)
+{
+	u32 regval;
+
+	/* ICE Optimizations Enable Sequence */
+	regval = qcom_ice_readl(ice, QCOM_ICE_REG_ADVANCED_CONTROL);
+	regval |= 0xd807100;
+	/* ICE HPG requires delay before writing */
+	udelay(5);
+	qcom_ice_writel(ice, regval, QCOM_ICE_REG_ADVANCED_CONTROL);
+	udelay(5);
+}
+
+/*
+ * Wait until the ICE BIST (built-in self-test) has completed.
+ *
+ * This may be necessary before ICE can be used.
+ * Note that we don't really care whether the BIST passed or failed;
+ * we really just want to make sure that it isn't still running. This is
+ * because (a) the BIST is a FIPS compliance thing that never fails in
+ * practice, (b) ICE is documented to reject crypto requests if the BIST
+ * fails, so we needn't do it in software too, and (c) properly testing
+ * storage encryption requires testing the full storage stack anyway,
+ * and not relying on hardware-level self-tests.
+ */
+static int qcom_ice_wait_bist_status(struct qcom_ice *ice)
+{
+	u32 regval;
+	int err;
+
+	err = readl_poll_timeout(ice->base + QCOM_ICE_REG_BIST_STATUS,
+				 regval, !(regval & QCOM_ICE_BIST_STATUS_MASK),
+				 50, 5000);
+	if (err)
+		dev_err(ice->dev, "Timed out waiting for ICE self-test to complete\n");
+
+	return err;
+}
+
+int qcom_ice_enable(struct qcom_ice *ice)
+{
+	qcom_ice_low_power_mode_enable(ice);
+	qcom_ice_optimization_enable(ice);
+
+	return qcom_ice_wait_bist_status(ice);
+}
+EXPORT_SYMBOL_GPL(qcom_ice_enable);
+
+int qcom_ice_resume(struct qcom_ice *ice)
+{
+	struct device *dev = ice->dev;
+	int err;
+
+	err = clk_prepare_enable(ice->core_clk);
+	if (err) {
+		dev_err(dev, "failed to enable core clock (%d)\n",
+			err);
+		return err;
+	}
+
+	return qcom_ice_wait_bist_status(ice);
+}
+EXPORT_SYMBOL_GPL(qcom_ice_resume);
+
+int qcom_ice_suspend(struct qcom_ice *ice)
+{
+	clk_disable_unprepare(ice->core_clk);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(qcom_ice_suspend);
+
+int qcom_ice_program_key(struct qcom_ice *ice,
+			 u8 algorithm_id, u8 key_size,
+			 const u8 crypto_key[], u8 data_unit_size,
+			 int slot)
+{
+	struct device *dev = ice->dev;
+	union {
+		u8 bytes[AES_256_XTS_KEY_SIZE];
+		u32 words[AES_256_XTS_KEY_SIZE / sizeof(u32)];
+	} key;
+	int i;
+	int err;
+
+	/* Only AES-256-XTS has been tested so far. */
+	if (algorithm_id != QCOM_ICE_CRYPTO_ALG_AES_XTS ||
+	    key_size != QCOM_ICE_CRYPTO_KEY_SIZE_256) {
+		dev_err_ratelimited(dev,
+				    "Unhandled crypto capability; algorithm_id=%d, key_size=%d\n",
+				    algorithm_id, key_size);
+		return -EINVAL;
+	}
+
+	memcpy(key.bytes, crypto_key, AES_256_XTS_KEY_SIZE);
+
+	/* The SCM call requires that the key words are encoded in big endian */
+	for (i = 0; i < ARRAY_SIZE(key.words); i++)
+		__cpu_to_be32s(&key.words[i]);
+
+	err = qcom_scm_ice_set_key(slot, key.bytes, AES_256_XTS_KEY_SIZE,
+				   QCOM_SCM_ICE_CIPHER_AES_256_XTS,
+				   data_unit_size);
+
+	memzero_explicit(&key, sizeof(key));
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(qcom_ice_program_key);
+
+int qcom_ice_evict_key(struct qcom_ice *ice, int slot)
+{
+	return qcom_scm_ice_invalidate_key(slot);
+}
+EXPORT_SYMBOL_GPL(qcom_ice_evict_key);
+
+static struct qcom_ice *qcom_ice_create(struct device *dev,
+					void __iomem *base)
+{
+	struct qcom_ice *engine;
+
+	if (!qcom_scm_is_available())
+		return ERR_PTR(-EPROBE_DEFER);
+
+	if (!qcom_scm_ice_available()) {
+		dev_warn(dev, "ICE SCM interface not found\n");
+		return NULL;
+	}
+
+	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
+	if (!engine)
+		return ERR_PTR(-ENOMEM);
+
+	engine->dev = dev;
+	engine->base = base;
+
+	/*
+	 * Legacy DT binding uses different clk names for each consumer,
+	 * so lets try those first. If none of those are a match, it means
+	 * the we only have one clock and it is part of the dedicated DT node.
+	 * Also, enable the clock before we check what HW version the driver
+	 * supports.
+	 */
+	engine->core_clk = devm_clk_get_optional_enabled(dev, "ice_core_clk");
+	if (!engine->core_clk)
+		engine->core_clk = devm_clk_get_optional_enabled(dev, "ice");
+	if (!engine->core_clk)
+		engine->core_clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(engine->core_clk))
+		return ERR_CAST(engine->core_clk);
+
+	if (!qcom_ice_check_supported(engine))
+		return ERR_PTR(-EOPNOTSUPP);
+
+	dev_dbg(dev, "Registered Qualcomm Inline Crypto Engine\n");
+
+	return engine;
+}
+
+/**
+ * of_qcom_ice_get() - get an ICE instance from a DT node
+ * @dev: device pointer for the consumer device
+ *
+ * This function will provide an ICE instance either by creating one for the
+ * consumer device if its DT node provides the 'ice' reg range and the 'ice'
+ * clock (for legacy DT style). On the other hand, if consumer provides a
+ * phandle via 'qcom,ice' property to an ICE DT, the ICE instance will already
+ * be created and so this function will return that instead.
+ *
+ * Return: ICE pointer on success, NULL if there is no ICE data provided by the
+ * consumer or ERR_PTR() on error.
+ */
+struct qcom_ice *of_qcom_ice_get(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct qcom_ice *ice;
+	struct device_node *node;
+	struct resource *res;
+	void __iomem *base;
+
+	if (!dev || !dev->of_node)
+		return ERR_PTR(-ENODEV);
+
+	/*
+	 * In order to support legacy style devicetree bindings, we need
+	 * to create the ICE instance using the consumer device and the reg
+	 * range called 'ice' it provides.
+	 */
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ice");
+	if (res) {
+		base = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(base))
+			return ERR_CAST(base);
+
+		/* create ICE instance using consumer dev */
+		return qcom_ice_create(&pdev->dev, base);
+	}
+
+	/*
+	 * If the consumer node does not provider an 'ice' reg range
+	 * (legacy DT binding), then it must at least provide a phandle
+	 * to the ICE devicetree node, otherwise ICE is not supported.
+	 */
+	node = of_parse_phandle(dev->of_node, "qcom,ice", 0);
+	if (!node)
+		return NULL;
+
+	pdev = of_find_device_by_node(node);
+	if (!pdev) {
+		dev_err(dev, "Cannot find device node %s\n", node->name);
+		ice = ERR_PTR(-EPROBE_DEFER);
+		goto out;
+	}
+
+	ice = platform_get_drvdata(pdev);
+	if (!ice) {
+		dev_err(dev, "Cannot get ice instance from %s\n",
+			dev_name(&pdev->dev));
+		platform_device_put(pdev);
+		ice = ERR_PTR(-EPROBE_DEFER);
+		goto out;
+	}
+
+	ice->link = device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER);
+	if (!ice->link) {
+		dev_err(&pdev->dev,
+			"Failed to create device link to consumer %s\n",
+			dev_name(dev));
+		platform_device_put(pdev);
+		ice = ERR_PTR(-EINVAL);
+	}
+
+out:
+	of_node_put(node);
+
+	return ice;
+}
+EXPORT_SYMBOL_GPL(of_qcom_ice_get);
+
+static int qcom_ice_probe(struct platform_device *pdev)
+{
+	struct qcom_ice *engine;
+	void __iomem *base;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base)) {
+		dev_warn(&pdev->dev, "ICE registers not found\n");
+		return PTR_ERR(base);
+	}
+
+	engine = qcom_ice_create(&pdev->dev, base);
+	if (IS_ERR(engine))
+		return PTR_ERR(engine);
+
+	platform_set_drvdata(pdev, engine);
+
+	return 0;
+}
+
+static const struct of_device_id qcom_ice_of_match_table[] = {
+	{ .compatible = "qcom,inline-crypto-engine" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, qcom_ice_of_match_table);
+
+static struct platform_driver qcom_ice_driver = {
+	.probe	= qcom_ice_probe,
+	.driver = {
+		.name = "qcom-ice",
+		.of_match_table = qcom_ice_of_match_table,
+	},
+};
+
+module_platform_driver(qcom_ice_driver);
+
+MODULE_DESCRIPTION("Qualcomm Inline Crypto Engine driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c
index 26efe12012a0..67c19ed2219a 100644
--- a/drivers/soc/qcom/llcc-qcom.c
+++ b/drivers/soc/qcom/llcc-qcom.c
@@ -62,8 +62,6 @@
 #define LLCC_TRP_WRSC_CACHEABLE_EN    0x21f2c
 #define LLCC_TRP_ALGO_CFG8	      0x21f30
 
-#define BANK_OFFSET_STRIDE	      0x80000
-
 #define LLCC_VERSION_2_0_0_0          0x02000000
 #define LLCC_VERSION_2_1_0_0          0x02010000
 #define LLCC_VERSION_4_1_0_0          0x04010000
@@ -122,10 +120,11 @@ struct llcc_slice_config {
 
 struct qcom_llcc_config {
 	const struct llcc_slice_config *sct_data;
-	int size;
-	bool need_llcc_cfg;
 	const u32 *reg_offset;
 	const struct llcc_edac_reg_offset *edac_reg_offset;
+	int size;
+	bool need_llcc_cfg;
+	bool no_edac;
 };
 
 enum llcc_reg_offset {
@@ -227,6 +226,14 @@ static const struct llcc_slice_config sm6350_data[] =  {
 	{ LLCC_MODPE,    29,  64, 1, 1, 0xFFF, 0x0, 0, 0, 0, 0, 1, 0 },
 };
 
+static const struct llcc_slice_config sm7150_data[] =  {
+	{ LLCC_CPUSS,    1,  512, 1, 0, 0xF, 0x0, 0, 0, 0, 1, 1 },
+	{ LLCC_MDM,      8,  128, 2, 0, 0xF, 0x0, 0, 0, 0, 1, 0 },
+	{ LLCC_GPUHTW,   11, 256, 1, 1, 0xF, 0x0, 0, 0, 0, 1, 0 },
+	{ LLCC_GPU,      12, 256, 1, 1, 0xF, 0x0, 0, 0, 0, 1, 0 },
+	{ LLCC_NPU,      23, 512, 1, 0, 0xF, 0x0, 0, 0, 0, 1, 0 },
+};
+
 static const struct llcc_slice_config sm8150_data[] =  {
 	{  LLCC_CPUSS,    1, 3072, 1, 1, 0xFFF, 0x0,   0, 0, 0, 1, 1 },
 	{  LLCC_VIDSC0,   2, 512,  2, 1, 0xFFF, 0x0,   0, 0, 0, 1, 0 },
@@ -454,6 +461,7 @@ static const struct qcom_llcc_config sdm845_cfg = {
 	.need_llcc_cfg	= false,
 	.reg_offset	= llcc_v1_reg_offset,
 	.edac_reg_offset = &llcc_v1_edac_reg_offset,
+	.no_edac	= true,
 };
 
 static const struct qcom_llcc_config sm6350_cfg = {
@@ -464,6 +472,14 @@ static const struct qcom_llcc_config sm6350_cfg = {
 	.edac_reg_offset = &llcc_v1_edac_reg_offset,
 };
 
+static const struct qcom_llcc_config sm7150_cfg = {
+	.sct_data       = sm7150_data,
+	.size           = ARRAY_SIZE(sm7150_data),
+	.need_llcc_cfg	= true,
+	.reg_offset	= llcc_v1_reg_offset,
+	.edac_reg_offset = &llcc_v1_edac_reg_offset,
+};
+
 static const struct qcom_llcc_config sm8150_cfg = {
 	.sct_data       = sm8150_data,
 	.size           = ARRAY_SIZE(sm8150_data),
@@ -898,8 +914,8 @@ static int qcom_llcc_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct regmap *qcom_llcc_init_mmio(struct platform_device *pdev,
-		const char *name)
+static struct regmap *qcom_llcc_init_mmio(struct platform_device *pdev, u8 index,
+					  const char *name)
 {
 	void __iomem *base;
 	struct regmap_config llcc_regmap_config = {
@@ -909,7 +925,7 @@ static struct regmap *qcom_llcc_init_mmio(struct platform_device *pdev,
 		.fast_io = true,
 	};
 
-	base = devm_platform_ioremap_resource_byname(pdev, name);
+	base = devm_platform_ioremap_resource(pdev, index);
 	if (IS_ERR(base))
 		return ERR_CAST(base);
 
@@ -927,6 +943,7 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 	const struct llcc_slice_config *llcc_cfg;
 	u32 sz;
 	u32 version;
+	struct regmap *regmap;
 
 	drv_data = devm_kzalloc(dev, sizeof(*drv_data), GFP_KERNEL);
 	if (!drv_data) {
@@ -934,21 +951,51 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 		goto err;
 	}
 
-	drv_data->regmap = qcom_llcc_init_mmio(pdev, "llcc_base");
-	if (IS_ERR(drv_data->regmap)) {
-		ret = PTR_ERR(drv_data->regmap);
+	/* Initialize the first LLCC bank regmap */
+	regmap = qcom_llcc_init_mmio(pdev, 0, "llcc0_base");
+	if (IS_ERR(regmap)) {
+		ret = PTR_ERR(regmap);
+		goto err;
+	}
+
+	cfg = of_device_get_match_data(&pdev->dev);
+
+	ret = regmap_read(regmap, cfg->reg_offset[LLCC_COMMON_STATUS0], &num_banks);
+	if (ret)
+		goto err;
+
+	num_banks &= LLCC_LB_CNT_MASK;
+	num_banks >>= LLCC_LB_CNT_SHIFT;
+	drv_data->num_banks = num_banks;
+
+	drv_data->regmaps = devm_kcalloc(dev, num_banks, sizeof(*drv_data->regmaps), GFP_KERNEL);
+	if (!drv_data->regmaps) {
+		ret = -ENOMEM;
 		goto err;
 	}
 
-	drv_data->bcast_regmap =
-		qcom_llcc_init_mmio(pdev, "llcc_broadcast_base");
+	drv_data->regmaps[0] = regmap;
+
+	/* Initialize rest of LLCC bank regmaps */
+	for (i = 1; i < num_banks; i++) {
+		char *base = kasprintf(GFP_KERNEL, "llcc%d_base", i);
+
+		drv_data->regmaps[i] = qcom_llcc_init_mmio(pdev, i, base);
+		if (IS_ERR(drv_data->regmaps[i])) {
+			ret = PTR_ERR(drv_data->regmaps[i]);
+			kfree(base);
+			goto err;
+		}
+
+		kfree(base);
+	}
+
+	drv_data->bcast_regmap = qcom_llcc_init_mmio(pdev, i, "llcc_broadcast_base");
 	if (IS_ERR(drv_data->bcast_regmap)) {
 		ret = PTR_ERR(drv_data->bcast_regmap);
 		goto err;
 	}
 
-	cfg = of_device_get_match_data(&pdev->dev);
-
 	/* Extract version of the IP */
 	ret = regmap_read(drv_data->bcast_regmap, cfg->reg_offset[LLCC_COMMON_HW_INFO],
 			  &version);
@@ -957,15 +1004,6 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 
 	drv_data->version = version;
 
-	ret = regmap_read(drv_data->regmap, cfg->reg_offset[LLCC_COMMON_STATUS0],
-			  &num_banks);
-	if (ret)
-		goto err;
-
-	num_banks &= LLCC_LB_CNT_MASK;
-	num_banks >>= LLCC_LB_CNT_SHIFT;
-	drv_data->num_banks = num_banks;
-
 	llcc_cfg = cfg->sct_data;
 	sz = cfg->size;
 
@@ -973,16 +1011,6 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 		if (llcc_cfg[i].slice_id > drv_data->max_slices)
 			drv_data->max_slices = llcc_cfg[i].slice_id;
 
-	drv_data->offsets = devm_kcalloc(dev, num_banks, sizeof(u32),
-							GFP_KERNEL);
-	if (!drv_data->offsets) {
-		ret = -ENOMEM;
-		goto err;
-	}
-
-	for (i = 0; i < num_banks; i++)
-		drv_data->offsets[i] = i * BANK_OFFSET_STRIDE;
-
 	drv_data->bitmap = devm_bitmap_zalloc(dev, drv_data->max_slices,
 					      GFP_KERNEL);
 	if (!drv_data->bitmap) {
@@ -1001,7 +1029,14 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 		goto err;
 
 	drv_data->ecc_irq = platform_get_irq_optional(pdev, 0);
-	if (drv_data->ecc_irq >= 0) {
+
+	/*
+	 * On some platforms, the access to EDAC registers will be locked by
+	 * the bootloader. So probing the EDAC driver will result in a crash.
+	 * Hence, disable the creation of EDAC platform device for the
+	 * problematic platforms.
+	 */
+	if (!cfg->no_edac) {
 		llcc_edac = platform_device_register_data(&pdev->dev,
 						"qcom_llcc_edac", -1, drv_data,
 						sizeof(*drv_data));
@@ -1022,6 +1057,7 @@ static const struct of_device_id qcom_llcc_of_match[] = {
 	{ .compatible = "qcom,sc8280xp-llcc", .data = &sc8280xp_cfg },
 	{ .compatible = "qcom,sdm845-llcc", .data = &sdm845_cfg },
 	{ .compatible = "qcom,sm6350-llcc", .data = &sm6350_cfg },
+	{ .compatible = "qcom,sm7150-llcc", .data = &sm7150_cfg },
 	{ .compatible = "qcom,sm8150-llcc", .data = &sm8150_cfg },
 	{ .compatible = "qcom,sm8250-llcc", .data = &sm8250_cfg },
 	{ .compatible = "qcom,sm8350-llcc", .data = &sm8350_cfg },
diff --git a/drivers/soc/qcom/pmic_glink.c b/drivers/soc/qcom/pmic_glink.c
index bb3fb57abcc6..8bf95df0a56a 100644
--- a/drivers/soc/qcom/pmic_glink.c
+++ b/drivers/soc/qcom/pmic_glink.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2022, Linaro Ltd
  */
 #include <linux/auxiliary_bus.h>
+#include <linux/of_device.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/rpmsg.h>
@@ -11,12 +12,23 @@
 #include <linux/soc/qcom/pdr.h>
 #include <linux/soc/qcom/pmic_glink.h>
 
+enum {
+	PMIC_GLINK_CLIENT_BATT = 0,
+	PMIC_GLINK_CLIENT_ALTMODE,
+	PMIC_GLINK_CLIENT_UCSI,
+};
+
+#define PMIC_GLINK_CLIENT_DEFAULT	(BIT(PMIC_GLINK_CLIENT_BATT) |	\
+					 BIT(PMIC_GLINK_CLIENT_ALTMODE))
+
 struct pmic_glink {
 	struct device *dev;
 	struct pdr_handle *pdr;
 
 	struct rpmsg_endpoint *ept;
 
+	unsigned long client_mask;
+
 	struct auxiliary_device altmode_aux;
 	struct auxiliary_device ps_aux;
 	struct auxiliary_device ucsi_aux;
@@ -233,6 +245,7 @@ static struct rpmsg_driver pmic_glink_rpmsg_driver = {
 
 static int pmic_glink_probe(struct platform_device *pdev)
 {
+	const unsigned long *match_data;
 	struct pdr_service *service;
 	struct pmic_glink *pg;
 	int ret;
@@ -249,12 +262,27 @@ static int pmic_glink_probe(struct platform_device *pdev)
 	mutex_init(&pg->client_lock);
 	mutex_init(&pg->state_lock);
 
-	ret = pmic_glink_add_aux_device(pg, &pg->altmode_aux, "altmode");
-	if (ret)
-		return ret;
-	ret = pmic_glink_add_aux_device(pg, &pg->ps_aux, "power-supply");
-	if (ret)
-		goto out_release_altmode_aux;
+	match_data = (unsigned long *)of_device_get_match_data(&pdev->dev);
+	if (match_data)
+		pg->client_mask = *match_data;
+	else
+		pg->client_mask = PMIC_GLINK_CLIENT_DEFAULT;
+
+	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI)) {
+		ret = pmic_glink_add_aux_device(pg, &pg->ucsi_aux, "ucsi");
+		if (ret)
+			return ret;
+	}
+	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE)) {
+		ret = pmic_glink_add_aux_device(pg, &pg->altmode_aux, "altmode");
+		if (ret)
+			goto out_release_ucsi_aux;
+	}
+	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_BATT)) {
+		ret = pmic_glink_add_aux_device(pg, &pg->ps_aux, "power-supply");
+		if (ret)
+			goto out_release_altmode_aux;
+	}
 
 	pg->pdr = pdr_handle_alloc(pmic_glink_pdr_callback, pg);
 	if (IS_ERR(pg->pdr)) {
@@ -278,9 +306,14 @@ static int pmic_glink_probe(struct platform_device *pdev)
 out_release_pdr_handle:
 	pdr_handle_release(pg->pdr);
 out_release_aux_devices:
-	pmic_glink_del_aux_device(pg, &pg->ps_aux);
+	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_BATT))
+		pmic_glink_del_aux_device(pg, &pg->ps_aux);
 out_release_altmode_aux:
-	pmic_glink_del_aux_device(pg, &pg->altmode_aux);
+	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE))
+		pmic_glink_del_aux_device(pg, &pg->altmode_aux);
+out_release_ucsi_aux:
+	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI))
+		pmic_glink_del_aux_device(pg, &pg->ucsi_aux);
 
 	return ret;
 }
@@ -291,8 +324,12 @@ static int pmic_glink_remove(struct platform_device *pdev)
 
 	pdr_handle_release(pg->pdr);
 
-	pmic_glink_del_aux_device(pg, &pg->ps_aux);
-	pmic_glink_del_aux_device(pg, &pg->altmode_aux);
+	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_BATT))
+		pmic_glink_del_aux_device(pg, &pg->ps_aux);
+	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE))
+		pmic_glink_del_aux_device(pg, &pg->altmode_aux);
+	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI))
+		pmic_glink_del_aux_device(pg, &pg->ucsi_aux);
 
 	mutex_lock(&__pmic_glink_lock);
 	__pmic_glink = NULL;
@@ -301,8 +338,14 @@ static int pmic_glink_remove(struct platform_device *pdev)
 	return 0;
 }
 
+/* Do not handle altmode for now on those platforms */
+static const unsigned long pmic_glink_sm8450_client_mask = BIT(PMIC_GLINK_CLIENT_BATT) |
+							   BIT(PMIC_GLINK_CLIENT_UCSI);
+
 static const struct of_device_id pmic_glink_of_match[] = {
-	{ .compatible = "qcom,pmic-glink", },
+	{ .compatible = "qcom,sm8450-pmic-glink", .data = &pmic_glink_sm8450_client_mask },
+	{ .compatible = "qcom,sm8550-pmic-glink", .data = &pmic_glink_sm8450_client_mask },
+	{ .compatible = "qcom,pmic-glink" },
 	{}
 };
 MODULE_DEVICE_TABLE(of, pmic_glink_of_match);
diff --git a/drivers/soc/qcom/qcom_aoss.c b/drivers/soc/qcom/qcom_aoss.c
index 18c856056475..e376c32cc16e 100644
--- a/drivers/soc/qcom/qcom_aoss.c
+++ b/drivers/soc/qcom/qcom_aoss.c
@@ -395,7 +395,7 @@ static int qmp_cooling_devices_register(struct qmp *qmp)
 		return -ENOMEM;
 
 	for_each_available_child_of_node(np, child) {
-		if (!of_find_property(child, "#cooling-cells", NULL))
+		if (!of_property_present(child, "#cooling-cells"))
 			continue;
 		ret = qmp_cooling_device_add(qmp, &qmp->cooling_devs[count++],
 					     child);
diff --git a/drivers/soc/qcom/qcom_gsbi.c b/drivers/soc/qcom/qcom_gsbi.c
index 290bdefbf28a..f1742e5bddb9 100644
--- a/drivers/soc/qcom/qcom_gsbi.c
+++ b/drivers/soc/qcom/qcom_gsbi.c
@@ -114,7 +114,7 @@ struct gsbi_info {
 	struct regmap *tcsr;
 };
 
-static const struct of_device_id tcsr_dt_match[] = {
+static const struct of_device_id tcsr_dt_match[] __maybe_unused = {
 	{ .compatible = "qcom,tcsr-ipq8064", .data = &config_ipq8064},
 	{ .compatible = "qcom,tcsr-apq8064", .data = &config_apq8064},
 	{ .compatible = "qcom,tcsr-msm8960", .data = &config_msm8960},
diff --git a/drivers/soc/qcom/rmtfs_mem.c b/drivers/soc/qcom/rmtfs_mem.c
index 538fa182169a..ce48a9f3b4c8 100644
--- a/drivers/soc/qcom/rmtfs_mem.c
+++ b/drivers/soc/qcom/rmtfs_mem.c
@@ -31,7 +31,7 @@ struct qcom_rmtfs_mem {
 
 	unsigned int client_id;
 
-	unsigned int perms;
+	u64 perms;
 };
 
 static ssize_t qcom_rmtfs_mem_show(struct device *dev,
@@ -126,7 +126,6 @@ static int qcom_rmtfs_mem_release(struct inode *inode, struct file *filp)
 }
 
 static struct class rmtfs_class = {
-	.owner          = THIS_MODULE,
 	.name           = "rmtfs",
 };
 
diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c
index 0f8b2249f889..f93544f6d796 100644
--- a/drivers/soc/qcom/rpmh-rsc.c
+++ b/drivers/soc/qcom/rpmh-rsc.c
@@ -1073,7 +1073,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev)
 	drv->ver.minor = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT);
 	drv->ver.minor >>= MINOR_VER_SHIFT;
 
-	if (drv->ver.major == 3 && drv->ver.minor == 0)
+	if (drv->ver.major == 3 && drv->ver.minor >= 0)
 		drv->regs = rpmh_rsc_reg_offset_ver_3_0;
 	else
 		drv->regs = rpmh_rsc_reg_offset_ver_2_7;
diff --git a/drivers/soc/qcom/rpmpd.c b/drivers/soc/qcom/rpmpd.c
index 337b1ad1cd3b..f8397dcb146c 100644
--- a/drivers/soc/qcom/rpmpd.c
+++ b/drivers/soc/qcom/rpmpd.c
@@ -40,57 +40,6 @@
 
 #define MAX_CORNER_RPMPD_STATE	6
 
-#define DEFINE_RPMPD_PAIR(_platform, _name, _active, r_type, r_key,	\
-			  r_id)						\
-	static struct rpmpd _platform##_##_active;			\
-	static struct rpmpd _platform##_##_name = {			\
-		.pd = {	.name = #_name,	},				\
-		.peer = &_platform##_##_active,				\
-		.res_type = RPMPD_##r_type,				\
-		.res_id = r_id,						\
-		.key = KEY_##r_key,					\
-	};								\
-	static struct rpmpd _platform##_##_active = {			\
-		.pd = { .name = #_active, },				\
-		.peer = &_platform##_##_name,				\
-		.active_only = true,					\
-		.res_type = RPMPD_##r_type,				\
-		.res_id = r_id,						\
-		.key = KEY_##r_key,					\
-	}
-
-#define DEFINE_RPMPD_CORNER(_platform, _name, r_type, r_id)		\
-	static struct rpmpd _platform##_##_name = {			\
-		.pd = { .name = #_name, },				\
-		.res_type = RPMPD_##r_type,				\
-		.res_id = r_id,						\
-		.key = KEY_CORNER,					\
-	}
-
-#define DEFINE_RPMPD_LEVEL(_platform, _name, r_type, r_id)		\
-	static struct rpmpd _platform##_##_name = {			\
-		.pd = { .name = #_name, },				\
-		.res_type = RPMPD_##r_type,				\
-		.res_id = r_id,						\
-		.key = KEY_LEVEL,					\
-	}
-
-#define DEFINE_RPMPD_VFC(_platform, _name, r_type, r_id)		\
-	static struct rpmpd _platform##_##_name = {			\
-		.pd = { .name = #_name, },				\
-		.res_type = RPMPD_##r_type,				\
-		.res_id = r_id,						\
-		.key = KEY_FLOOR_CORNER,				\
-	}
-
-#define DEFINE_RPMPD_VFL(_platform, _name, r_type, r_id)		\
-	static struct rpmpd _platform##_##_name = {			\
-		.pd = { .name = #_name, },				\
-		.res_type = RPMPD_##r_type,				\
-		.res_id = r_id,						\
-		.key = KEY_FLOOR_LEVEL,					\
-	}
-
 struct rpmpd_req {
 	__le32 key;
 	__le32 nbytes;
@@ -99,6 +48,7 @@ struct rpmpd_req {
 
 struct rpmpd {
 	struct generic_pm_domain pd;
+	struct generic_pm_domain *parent;
 	struct rpmpd *peer;
 	const bool active_only;
 	unsigned int corner;
@@ -118,19 +68,459 @@ struct rpmpd_desc {
 
 static DEFINE_MUTEX(rpmpd_lock);
 
-/* mdm9607 RPM Power Domains */
-DEFINE_RPMPD_PAIR(mdm9607, vddcx, vddcx_ao, SMPA, LEVEL, 3);
-DEFINE_RPMPD_VFL(mdm9607, vddcx_vfl, SMPA, 3);
+/* CX */
+static struct rpmpd cx_rwcx0_lvl_ao;
+static struct rpmpd cx_rwcx0_lvl = {
+	.pd = { .name = "cx", },
+	.peer = &cx_rwcx0_lvl_ao,
+	.res_type = RPMPD_RWCX,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd cx_rwcx0_lvl_ao = {
+	.pd = { .name = "cx_ao", },
+	.peer = &cx_rwcx0_lvl,
+	.active_only = true,
+	.res_type = RPMPD_RWCX,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd cx_s1a_corner_ao;
+static struct rpmpd cx_s1a_corner = {
+	.pd = { .name = "cx", },
+	.peer = &cx_s1a_corner_ao,
+	.res_type = RPMPD_SMPA,
+	.res_id = 1,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd cx_s1a_corner_ao = {
+	.pd = { .name = "cx_ao", },
+	.peer = &cx_s1a_corner,
+	.active_only = true,
+	.res_type = RPMPD_SMPA,
+	.res_id = 1,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd cx_s2a_corner_ao;
+static struct rpmpd cx_s2a_corner = {
+	.pd = { .name = "cx", },
+	.peer = &cx_s2a_corner_ao,
+	.res_type = RPMPD_SMPA,
+	.res_id = 2,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd cx_s2a_corner_ao = {
+	.pd = { .name = "cx_ao", },
+	.peer = &cx_s2a_corner,
+	.active_only = true,
+	.res_type = RPMPD_SMPA,
+	.res_id = 2,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd cx_s2a_lvl_ao;
+static struct rpmpd cx_s2a_lvl = {
+	.pd = { .name = "cx", },
+	.peer = &cx_s2a_lvl_ao,
+	.res_type = RPMPD_SMPA,
+	.res_id = 2,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd cx_s2a_lvl_ao = {
+	.pd = { .name = "cx_ao", },
+	.peer = &cx_s2a_lvl,
+	.active_only = true,
+	.res_type = RPMPD_SMPA,
+	.res_id = 2,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd cx_s3a_lvl_ao;
+static struct rpmpd cx_s3a_lvl = {
+	.pd = { .name = "cx", },
+	.peer = &cx_s3a_lvl_ao,
+	.res_type = RPMPD_SMPA,
+	.res_id = 3,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd cx_s3a_lvl_ao = {
+	.pd = { .name = "cx_ao", },
+	.peer = &cx_s3a_lvl,
+	.active_only = true,
+	.res_type = RPMPD_SMPA,
+	.res_id = 3,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd cx_rwcx0_vfl = {
+	.pd = { .name = "cx_vfl", },
+	.res_type = RPMPD_RWCX,
+	.res_id = 0,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+static struct rpmpd cx_rwsc2_vfl = {
+	.pd = { .name = "cx_vfl", },
+	.res_type = RPMPD_RWSC,
+	.res_id = 2,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+static struct rpmpd cx_s1a_vfc = {
+	.pd = { .name = "cx_vfc", },
+	.res_type = RPMPD_SMPA,
+	.res_id = 1,
+	.key = KEY_FLOOR_CORNER,
+};
+
+static struct rpmpd cx_s2a_vfc = {
+	.pd = { .name = "cx_vfc", },
+	.res_type = RPMPD_SMPA,
+	.res_id = 2,
+	.key = KEY_FLOOR_CORNER,
+};
+
+static struct rpmpd cx_s2a_vfl = {
+	.pd = { .name = "cx_vfl", },
+	.res_type = RPMPD_SMPA,
+	.res_id = 2,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+static struct rpmpd cx_s3a_vfl = {
+	.pd = { .name = "cx_vfl", },
+	.res_type = RPMPD_SMPA,
+	.res_id = 3,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+/* G(F)X */
+static struct rpmpd gfx_s2b_corner = {
+	.pd = { .name = "gfx", },
+	.res_type = RPMPD_SMPB,
+	.res_id = 2,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd gfx_s2b_vfc = {
+	.pd = { .name = "gfx_vfc", },
+	.res_type = RPMPD_SMPB,
+	.res_id = 2,
+	.key = KEY_FLOOR_CORNER,
+};
+
+static struct rpmpd mx_rwmx0_lvl;
+static struct rpmpd gx_rwgx0_lvl_ao;
+static struct rpmpd gx_rwgx0_lvl = {
+	.pd = { .name = "gx", },
+	.peer = &gx_rwgx0_lvl_ao,
+	.res_type = RPMPD_RWGX,
+	.parent = &mx_rwmx0_lvl.pd,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd mx_rwmx0_lvl_ao;
+static struct rpmpd gx_rwgx0_lvl_ao = {
+	.pd = { .name = "gx_ao", },
+	.peer = &gx_rwgx0_lvl,
+	.parent = &mx_rwmx0_lvl_ao.pd,
+	.active_only = true,
+	.res_type = RPMPD_RWGX,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+/* MX */
+static struct rpmpd mx_l3a_corner_ao;
+static struct rpmpd mx_l3a_corner = {
+	.pd = { .name = "mx", },
+	.peer = &mx_l3a_corner_ao,
+	.res_type = RPMPD_LDOA,
+	.res_id = 3,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd mx_l3a_corner_ao = {
+	.pd = { .name = "mx_ao", },
+	.peer = &mx_l3a_corner,
+	.active_only = true,
+	.res_type = RPMPD_LDOA,
+	.res_id = 3,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd mx_l12a_lvl_ao;
+static struct rpmpd mx_l12a_lvl = {
+	.pd = { .name = "mx", },
+	.peer = &mx_l12a_lvl_ao,
+	.res_type = RPMPD_LDOA,
+	.res_id = 12,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd mx_l12a_lvl_ao = {
+	.pd = { .name = "mx_ao", },
+	.peer = &mx_l12a_lvl,
+	.active_only = true,
+	.res_type = RPMPD_LDOA,
+	.res_id = 12,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd mx_s2a_corner_ao;
+static struct rpmpd mx_s2a_corner = {
+	.pd = { .name = "mx", },
+	.peer = &mx_s2a_corner_ao,
+	.res_type = RPMPD_SMPA,
+	.res_id = 2,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd mx_s2a_corner_ao = {
+	.pd = { .name = "mx_ao", },
+	.peer = &mx_s2a_corner,
+	.active_only = true,
+	.res_type = RPMPD_SMPA,
+	.res_id = 2,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd mx_rwmx0_lvl_ao;
+static struct rpmpd mx_rwmx0_lvl = {
+	.pd = { .name = "mx", },
+	.peer = &mx_rwmx0_lvl_ao,
+	.res_type = RPMPD_RWMX,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd mx_rwmx0_lvl_ao = {
+	.pd = { .name = "mx_ao", },
+	.peer = &mx_rwmx0_lvl,
+	.active_only = true,
+	.res_type = RPMPD_RWMX,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd mx_s6a_lvl_ao;
+static struct rpmpd mx_s6a_lvl = {
+	.pd = { .name = "mx", },
+	.peer = &mx_s6a_lvl_ao,
+	.res_type = RPMPD_SMPA,
+	.res_id = 6,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd mx_s6a_lvl_ao = {
+	.pd = { .name = "mx_ao", },
+	.peer = &mx_s6a_lvl,
+	.active_only = true,
+	.res_type = RPMPD_SMPA,
+	.res_id = 6,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd mx_s7a_lvl_ao;
+static struct rpmpd mx_s7a_lvl = {
+	.pd = { .name = "mx", },
+	.peer = &mx_s7a_lvl_ao,
+	.res_type = RPMPD_SMPA,
+	.res_id = 7,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd mx_s7a_lvl_ao = {
+	.pd = { .name = "mx_ao", },
+	.peer = &mx_s7a_lvl,
+	.active_only = true,
+	.res_type = RPMPD_SMPA,
+	.res_id = 7,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd mx_l12a_vfl = {
+	.pd = { .name = "mx_vfl", },
+	.res_type = RPMPD_LDOA,
+	.res_id = 12,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+static struct rpmpd mx_rwmx0_vfl = {
+	.pd = { .name = "mx_vfl", },
+	.res_type = RPMPD_RWMX,
+	.res_id = 0,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+static struct rpmpd mx_rwsm6_vfl = {
+	.pd = { .name = "mx_vfl", },
+	.res_type = RPMPD_RWSM,
+	.res_id = 6,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+/* MD */
+static struct rpmpd md_s1a_corner_ao;
+static struct rpmpd md_s1a_corner = {
+	.pd = { .name = "md", },
+	.peer = &md_s1a_corner_ao,
+	.res_type = RPMPD_SMPA,
+	.res_id = 1,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd md_s1a_corner_ao = {
+	.pd = { .name = "md_ao", },
+	.peer = &md_s1a_corner,
+	.active_only = true,
+	.res_type = RPMPD_SMPA,
+	.res_id = 1,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd md_s1a_lvl_ao;
+static struct rpmpd md_s1a_lvl = {
+	.pd = { .name = "md", },
+	.peer = &md_s1a_lvl_ao,
+	.res_type = RPMPD_SMPA,
+	.res_id = 1,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd md_s1a_lvl_ao = {
+	.pd = { .name = "md_ao", },
+	.peer = &md_s1a_lvl,
+	.active_only = true,
+	.res_type = RPMPD_SMPA,
+	.res_id = 1,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd md_s1a_vfc = {
+	.pd = { .name = "md_vfc", },
+	.res_type = RPMPD_SMPA,
+	.res_id = 1,
+	.key = KEY_FLOOR_CORNER,
+};
+
+/* LPI_CX */
+static struct rpmpd lpi_cx_rwlc0_lvl = {
+	.pd = { .name = "lpi_cx", },
+	.res_type = RPMPD_RWLC,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd lpi_cx_rwlc0_vfl = {
+	.pd = { .name = "lpi_cx_vfl", },
+	.res_type = RPMPD_RWLC,
+	.res_id = 0,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+/* LPI_MX */
+static struct rpmpd lpi_mx_rwlm0_lvl = {
+	.pd = { .name = "lpi_mx", },
+	.res_type = RPMPD_RWLM,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd lpi_mx_rwlm0_vfl = {
+	.pd = { .name = "lpi_mx_vfl", },
+	.res_type = RPMPD_RWLM,
+	.res_id = 0,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+/* SSC_CX */
+static struct rpmpd ssc_cx_l26a_corner = {
+	.pd = { .name = "ssc_cx", },
+	.res_type = RPMPD_LDOA,
+	.res_id = 26,
+	.key = KEY_CORNER,
+};
+
+static struct rpmpd ssc_cx_rwlc0_lvl = {
+	.pd = { .name = "ssc_cx", },
+	.res_type = RPMPD_RWLC,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd ssc_cx_rwsc0_lvl = {
+	.pd = { .name = "ssc_cx", },
+	.res_type = RPMPD_RWSC,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd ssc_cx_l26a_vfc = {
+	.pd = { .name = "ssc_cx_vfc", },
+	.res_type = RPMPD_LDOA,
+	.res_id = 26,
+	.key = KEY_FLOOR_CORNER,
+};
+
+static struct rpmpd ssc_cx_rwlc0_vfl = {
+	.pd = { .name = "ssc_cx_vfl", },
+	.res_type = RPMPD_RWLC,
+	.res_id = 0,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+static struct rpmpd ssc_cx_rwsc0_vfl = {
+	.pd = { .name = "ssc_cx_vfl", },
+	.res_type = RPMPD_RWSC,
+	.res_id = 0,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+/* SSC_MX */
+static struct rpmpd ssc_mx_rwlm0_lvl = {
+	.pd = { .name = "ssc_mx", },
+	.res_type = RPMPD_RWLM,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd ssc_mx_rwsm0_lvl = {
+	.pd = { .name = "ssc_mx", },
+	.res_type = RPMPD_RWSM,
+	.res_id = 0,
+	.key = KEY_LEVEL,
+};
+
+static struct rpmpd ssc_mx_rwlm0_vfl = {
+	.pd = { .name = "ssc_mx_vfl", },
+	.res_type = RPMPD_RWLM,
+	.res_id = 0,
+	.key = KEY_FLOOR_LEVEL,
+};
+
+static struct rpmpd ssc_mx_rwsm0_vfl = {
+	.pd = { .name = "ssc_mx_vfl", },
+	.res_type = RPMPD_RWSM,
+	.res_id = 0,
+	.key = KEY_FLOOR_LEVEL,
+};
 
-DEFINE_RPMPD_PAIR(mdm9607, vddmx, vddmx_ao, LDOA, LEVEL, 12);
-DEFINE_RPMPD_VFL(mdm9607, vddmx_vfl, LDOA, 12);
 static struct rpmpd *mdm9607_rpmpds[] = {
-	[MDM9607_VDDCX] =	&mdm9607_vddcx,
-	[MDM9607_VDDCX_AO] =	&mdm9607_vddcx_ao,
-	[MDM9607_VDDCX_VFL] =	&mdm9607_vddcx_vfl,
-	[MDM9607_VDDMX] =	&mdm9607_vddmx,
-	[MDM9607_VDDMX_AO] =	&mdm9607_vddmx_ao,
-	[MDM9607_VDDMX_VFL] =	&mdm9607_vddmx_vfl,
+	[MDM9607_VDDCX] =	&cx_s3a_lvl,
+	[MDM9607_VDDCX_AO] =	&cx_s3a_lvl_ao,
+	[MDM9607_VDDCX_VFL] =	&cx_s3a_vfl,
+	[MDM9607_VDDMX] =	&mx_l12a_lvl,
+	[MDM9607_VDDMX_AO] =	&mx_l12a_lvl_ao,
+	[MDM9607_VDDMX_VFL] =	&mx_l12a_vfl,
 };
 
 static const struct rpmpd_desc mdm9607_desc = {
@@ -139,14 +529,10 @@ static const struct rpmpd_desc mdm9607_desc = {
 	.max_state = RPM_SMD_LEVEL_TURBO,
 };
 
-/* msm8226 RPM Power Domains */
-DEFINE_RPMPD_PAIR(msm8226, vddcx, vddcx_ao, SMPA, CORNER, 1);
-DEFINE_RPMPD_VFC(msm8226, vddcx_vfc, SMPA, 1);
-
 static struct rpmpd *msm8226_rpmpds[] = {
-	[MSM8226_VDDCX] =	&msm8226_vddcx,
-	[MSM8226_VDDCX_AO] =	&msm8226_vddcx_ao,
-	[MSM8226_VDDCX_VFC] =	&msm8226_vddcx_vfc,
+	[MSM8226_VDDCX] =	&cx_s1a_corner,
+	[MSM8226_VDDCX_AO] =	&cx_s1a_corner_ao,
+	[MSM8226_VDDCX_VFC] =	&cx_s1a_vfc,
 };
 
 static const struct rpmpd_desc msm8226_desc = {
@@ -155,24 +541,15 @@ static const struct rpmpd_desc msm8226_desc = {
 	.max_state = MAX_CORNER_RPMPD_STATE,
 };
 
-/* msm8939 RPM Power Domains */
-DEFINE_RPMPD_PAIR(msm8939, vddmd, vddmd_ao, SMPA, CORNER, 1);
-DEFINE_RPMPD_VFC(msm8939, vddmd_vfc, SMPA, 1);
-
-DEFINE_RPMPD_PAIR(msm8939, vddcx, vddcx_ao, SMPA, CORNER, 2);
-DEFINE_RPMPD_VFC(msm8939, vddcx_vfc, SMPA, 2);
-
-DEFINE_RPMPD_PAIR(msm8939, vddmx, vddmx_ao, LDOA, CORNER, 3);
-
 static struct rpmpd *msm8939_rpmpds[] = {
-	[MSM8939_VDDMDCX] =	&msm8939_vddmd,
-	[MSM8939_VDDMDCX_AO] =	&msm8939_vddmd_ao,
-	[MSM8939_VDDMDCX_VFC] =	&msm8939_vddmd_vfc,
-	[MSM8939_VDDCX] =	&msm8939_vddcx,
-	[MSM8939_VDDCX_AO] =	&msm8939_vddcx_ao,
-	[MSM8939_VDDCX_VFC] =	&msm8939_vddcx_vfc,
-	[MSM8939_VDDMX] =	&msm8939_vddmx,
-	[MSM8939_VDDMX_AO] =	&msm8939_vddmx_ao,
+	[MSM8939_VDDMDCX] =	&md_s1a_corner,
+	[MSM8939_VDDMDCX_AO] =	&md_s1a_corner_ao,
+	[MSM8939_VDDMDCX_VFC] =	&md_s1a_vfc,
+	[MSM8939_VDDCX] =	&cx_s2a_corner,
+	[MSM8939_VDDCX_AO] =	&cx_s2a_corner_ao,
+	[MSM8939_VDDCX_VFC] =	&cx_s2a_vfc,
+	[MSM8939_VDDMX] =	&mx_l3a_corner,
+	[MSM8939_VDDMX_AO] =	&mx_l3a_corner_ao,
 };
 
 static const struct rpmpd_desc msm8939_desc = {
@@ -181,18 +558,12 @@ static const struct rpmpd_desc msm8939_desc = {
 	.max_state = MAX_CORNER_RPMPD_STATE,
 };
 
-/* msm8916 RPM Power Domains */
-DEFINE_RPMPD_PAIR(msm8916, vddcx, vddcx_ao, SMPA, CORNER, 1);
-DEFINE_RPMPD_PAIR(msm8916, vddmx, vddmx_ao, LDOA, CORNER, 3);
-
-DEFINE_RPMPD_VFC(msm8916, vddcx_vfc, SMPA, 1);
-
 static struct rpmpd *msm8916_rpmpds[] = {
-	[MSM8916_VDDCX] =	&msm8916_vddcx,
-	[MSM8916_VDDCX_AO] =	&msm8916_vddcx_ao,
-	[MSM8916_VDDCX_VFC] =	&msm8916_vddcx_vfc,
-	[MSM8916_VDDMX] =	&msm8916_vddmx,
-	[MSM8916_VDDMX_AO] =	&msm8916_vddmx_ao,
+	[MSM8916_VDDCX] =	&cx_s1a_corner,
+	[MSM8916_VDDCX_AO] =	&cx_s1a_corner_ao,
+	[MSM8916_VDDCX_VFC] =	&cx_s1a_vfc,
+	[MSM8916_VDDMX] =	&mx_l3a_corner,
+	[MSM8916_VDDMX_AO] =	&mx_l3a_corner_ao,
 };
 
 static const struct rpmpd_desc msm8916_desc = {
@@ -201,21 +572,14 @@ static const struct rpmpd_desc msm8916_desc = {
 	.max_state = MAX_CORNER_RPMPD_STATE,
 };
 
-/* msm8953 RPM Power Domains */
-DEFINE_RPMPD_PAIR(msm8953, vddmd, vddmd_ao, SMPA, LEVEL, 1);
-DEFINE_RPMPD_PAIR(msm8953, vddcx, vddcx_ao, SMPA, LEVEL, 2);
-DEFINE_RPMPD_PAIR(msm8953, vddmx, vddmx_ao, SMPA, LEVEL, 7);
-
-DEFINE_RPMPD_VFL(msm8953, vddcx_vfl, SMPA, 2);
-
 static struct rpmpd *msm8953_rpmpds[] = {
-	[MSM8953_VDDMD] =	&msm8953_vddmd,
-	[MSM8953_VDDMD_AO] =	&msm8953_vddmd_ao,
-	[MSM8953_VDDCX] =	&msm8953_vddcx,
-	[MSM8953_VDDCX_AO] =	&msm8953_vddcx_ao,
-	[MSM8953_VDDCX_VFL] =	&msm8953_vddcx_vfl,
-	[MSM8953_VDDMX] =	&msm8953_vddmx,
-	[MSM8953_VDDMX_AO] =	&msm8953_vddmx_ao,
+	[MSM8953_VDDMD] =	&md_s1a_lvl,
+	[MSM8953_VDDMD_AO] =	&md_s1a_lvl_ao,
+	[MSM8953_VDDCX] =	&cx_s2a_lvl,
+	[MSM8953_VDDCX_AO] =	&cx_s2a_lvl_ao,
+	[MSM8953_VDDCX_VFL] =	&cx_s2a_vfl,
+	[MSM8953_VDDMX] =	&mx_s7a_lvl,
+	[MSM8953_VDDMX_AO] =	&mx_s7a_lvl_ao,
 };
 
 static const struct rpmpd_desc msm8953_desc = {
@@ -224,20 +588,13 @@ static const struct rpmpd_desc msm8953_desc = {
 	.max_state = RPM_SMD_LEVEL_TURBO,
 };
 
-/* msm8976 RPM Power Domains */
-DEFINE_RPMPD_PAIR(msm8976, vddcx, vddcx_ao, SMPA, LEVEL, 2);
-DEFINE_RPMPD_PAIR(msm8976, vddmx, vddmx_ao, SMPA, LEVEL, 6);
-
-DEFINE_RPMPD_VFL(msm8976, vddcx_vfl, RWSC, 2);
-DEFINE_RPMPD_VFL(msm8976, vddmx_vfl, RWSM, 6);
-
 static struct rpmpd *msm8976_rpmpds[] = {
-	[MSM8976_VDDCX] =	&msm8976_vddcx,
-	[MSM8976_VDDCX_AO] =	&msm8976_vddcx_ao,
-	[MSM8976_VDDCX_VFL] =	&msm8976_vddcx_vfl,
-	[MSM8976_VDDMX] =	&msm8976_vddmx,
-	[MSM8976_VDDMX_AO] =	&msm8976_vddmx_ao,
-	[MSM8976_VDDMX_VFL] =	&msm8976_vddmx_vfl,
+	[MSM8976_VDDCX] =	&cx_s2a_lvl,
+	[MSM8976_VDDCX_AO] =	&cx_s2a_lvl_ao,
+	[MSM8976_VDDCX_VFL] =	&cx_rwsc2_vfl,
+	[MSM8976_VDDMX] =	&mx_s6a_lvl,
+	[MSM8976_VDDMX_AO] =	&mx_s6a_lvl_ao,
+	[MSM8976_VDDMX_VFL] =	&mx_rwsm6_vfl,
 };
 
 static const struct rpmpd_desc msm8976_desc = {
@@ -246,23 +603,16 @@ static const struct rpmpd_desc msm8976_desc = {
 	.max_state = RPM_SMD_LEVEL_TURBO_HIGH,
 };
 
-/* msm8994 RPM Power domains */
-DEFINE_RPMPD_PAIR(msm8994, vddcx, vddcx_ao, SMPA, CORNER, 1);
-DEFINE_RPMPD_PAIR(msm8994, vddmx, vddmx_ao, SMPA, CORNER, 2);
-/* Attention! *Some* 8994 boards with pm8004 may use SMPC here! */
-DEFINE_RPMPD_CORNER(msm8994, vddgfx, SMPB, 2);
-
-DEFINE_RPMPD_VFC(msm8994, vddcx_vfc, SMPA, 1);
-DEFINE_RPMPD_VFC(msm8994, vddgfx_vfc, SMPB, 2);
-
 static struct rpmpd *msm8994_rpmpds[] = {
-	[MSM8994_VDDCX] =	&msm8994_vddcx,
-	[MSM8994_VDDCX_AO] =	&msm8994_vddcx_ao,
-	[MSM8994_VDDCX_VFC] =	&msm8994_vddcx_vfc,
-	[MSM8994_VDDMX] =	&msm8994_vddmx,
-	[MSM8994_VDDMX_AO] =	&msm8994_vddmx_ao,
-	[MSM8994_VDDGFX] =	&msm8994_vddgfx,
-	[MSM8994_VDDGFX_VFC] =	&msm8994_vddgfx_vfc,
+	[MSM8994_VDDCX] =	&cx_s1a_corner,
+	[MSM8994_VDDCX_AO] =	&cx_s1a_corner_ao,
+	[MSM8994_VDDCX_VFC] =	&cx_s1a_vfc,
+	[MSM8994_VDDMX] =	&mx_s2a_corner,
+	[MSM8994_VDDMX_AO] =	&mx_s2a_corner_ao,
+
+	/* Attention! *Some* 8994 boards with pm8004 may use SMPC here! */
+	[MSM8994_VDDGFX] =	&gfx_s2b_corner,
+	[MSM8994_VDDGFX_VFC] =	&gfx_s2b_vfc,
 };
 
 static const struct rpmpd_desc msm8994_desc = {
@@ -271,22 +621,14 @@ static const struct rpmpd_desc msm8994_desc = {
 	.max_state = MAX_CORNER_RPMPD_STATE,
 };
 
-/* msm8996 RPM Power domains */
-DEFINE_RPMPD_PAIR(msm8996, vddcx, vddcx_ao, SMPA, CORNER, 1);
-DEFINE_RPMPD_PAIR(msm8996, vddmx, vddmx_ao, SMPA, CORNER, 2);
-DEFINE_RPMPD_CORNER(msm8996, vddsscx, LDOA, 26);
-
-DEFINE_RPMPD_VFC(msm8996, vddcx_vfc, SMPA, 1);
-DEFINE_RPMPD_VFC(msm8996, vddsscx_vfc, LDOA, 26);
-
 static struct rpmpd *msm8996_rpmpds[] = {
-	[MSM8996_VDDCX] =	&msm8996_vddcx,
-	[MSM8996_VDDCX_AO] =	&msm8996_vddcx_ao,
-	[MSM8996_VDDCX_VFC] =	&msm8996_vddcx_vfc,
-	[MSM8996_VDDMX] =	&msm8996_vddmx,
-	[MSM8996_VDDMX_AO] =	&msm8996_vddmx_ao,
-	[MSM8996_VDDSSCX] =	&msm8996_vddsscx,
-	[MSM8996_VDDSSCX_VFC] =	&msm8996_vddsscx_vfc,
+	[MSM8996_VDDCX] =	&cx_s1a_corner,
+	[MSM8996_VDDCX_AO] =	&cx_s1a_corner_ao,
+	[MSM8996_VDDCX_VFC] =	&cx_s1a_vfc,
+	[MSM8996_VDDMX] =	&mx_s2a_corner,
+	[MSM8996_VDDMX_AO] =	&mx_s2a_corner_ao,
+	[MSM8996_VDDSSCX] =	&ssc_cx_l26a_corner,
+	[MSM8996_VDDSSCX_VFC] =	&ssc_cx_l26a_vfc,
 };
 
 static const struct rpmpd_desc msm8996_desc = {
@@ -295,30 +637,17 @@ static const struct rpmpd_desc msm8996_desc = {
 	.max_state = MAX_CORNER_RPMPD_STATE,
 };
 
-/* msm8998 RPM Power domains */
-DEFINE_RPMPD_PAIR(msm8998, vddcx, vddcx_ao, RWCX, LEVEL, 0);
-DEFINE_RPMPD_VFL(msm8998, vddcx_vfl, RWCX, 0);
-
-DEFINE_RPMPD_PAIR(msm8998, vddmx, vddmx_ao, RWMX, LEVEL, 0);
-DEFINE_RPMPD_VFL(msm8998, vddmx_vfl, RWMX, 0);
-
-DEFINE_RPMPD_LEVEL(msm8998, vdd_ssccx, RWSC, 0);
-DEFINE_RPMPD_VFL(msm8998, vdd_ssccx_vfl, RWSC, 0);
-
-DEFINE_RPMPD_LEVEL(msm8998, vdd_sscmx, RWSM, 0);
-DEFINE_RPMPD_VFL(msm8998, vdd_sscmx_vfl, RWSM, 0);
-
 static struct rpmpd *msm8998_rpmpds[] = {
-	[MSM8998_VDDCX] =		&msm8998_vddcx,
-	[MSM8998_VDDCX_AO] =		&msm8998_vddcx_ao,
-	[MSM8998_VDDCX_VFL] =		&msm8998_vddcx_vfl,
-	[MSM8998_VDDMX] =		&msm8998_vddmx,
-	[MSM8998_VDDMX_AO] =		&msm8998_vddmx_ao,
-	[MSM8998_VDDMX_VFL] =		&msm8998_vddmx_vfl,
-	[MSM8998_SSCCX] =		&msm8998_vdd_ssccx,
-	[MSM8998_SSCCX_VFL] =		&msm8998_vdd_ssccx_vfl,
-	[MSM8998_SSCMX] =		&msm8998_vdd_sscmx,
-	[MSM8998_SSCMX_VFL] =		&msm8998_vdd_sscmx_vfl,
+	[MSM8998_VDDCX] =	&cx_rwcx0_lvl,
+	[MSM8998_VDDCX_AO] =	&cx_rwcx0_lvl_ao,
+	[MSM8998_VDDCX_VFL] =	&cx_rwcx0_vfl,
+	[MSM8998_VDDMX] =	&mx_rwmx0_lvl,
+	[MSM8998_VDDMX_AO] =	&mx_rwmx0_lvl_ao,
+	[MSM8998_VDDMX_VFL] =	&mx_rwmx0_vfl,
+	[MSM8998_SSCCX] =	&ssc_cx_rwsc0_lvl,
+	[MSM8998_SSCCX_VFL] =	&ssc_cx_rwsc0_vfl,
+	[MSM8998_SSCMX] =	&ssc_mx_rwsm0_lvl,
+	[MSM8998_SSCMX_VFL] =	&ssc_mx_rwsm0_vfl,
 };
 
 static const struct rpmpd_desc msm8998_desc = {
@@ -327,24 +656,14 @@ static const struct rpmpd_desc msm8998_desc = {
 	.max_state = RPM_SMD_LEVEL_BINNING,
 };
 
-/* qcs404 RPM Power domains */
-DEFINE_RPMPD_PAIR(qcs404, vddmx, vddmx_ao, RWMX, LEVEL, 0);
-DEFINE_RPMPD_VFL(qcs404, vddmx_vfl, RWMX, 0);
-
-DEFINE_RPMPD_LEVEL(qcs404, vdd_lpicx, RWLC, 0);
-DEFINE_RPMPD_VFL(qcs404, vdd_lpicx_vfl, RWLC, 0);
-
-DEFINE_RPMPD_LEVEL(qcs404, vdd_lpimx, RWLM, 0);
-DEFINE_RPMPD_VFL(qcs404, vdd_lpimx_vfl, RWLM, 0);
-
 static struct rpmpd *qcs404_rpmpds[] = {
-	[QCS404_VDDMX] = &qcs404_vddmx,
-	[QCS404_VDDMX_AO] = &qcs404_vddmx_ao,
-	[QCS404_VDDMX_VFL] = &qcs404_vddmx_vfl,
-	[QCS404_LPICX] = &qcs404_vdd_lpicx,
-	[QCS404_LPICX_VFL] = &qcs404_vdd_lpicx_vfl,
-	[QCS404_LPIMX] = &qcs404_vdd_lpimx,
-	[QCS404_LPIMX_VFL] = &qcs404_vdd_lpimx_vfl,
+	[QCS404_VDDMX] =	&mx_rwmx0_lvl,
+	[QCS404_VDDMX_AO] =	&mx_rwmx0_lvl_ao,
+	[QCS404_VDDMX_VFL] =	&mx_rwmx0_vfl,
+	[QCS404_LPICX] =	&lpi_cx_rwlc0_lvl,
+	[QCS404_LPICX_VFL] =	&lpi_cx_rwlc0_vfl,
+	[QCS404_LPIMX] =	&lpi_mx_rwlm0_lvl,
+	[QCS404_LPIMX_VFL] =	&lpi_mx_rwlm0_vfl,
 };
 
 static const struct rpmpd_desc qcs404_desc = {
@@ -353,30 +672,17 @@ static const struct rpmpd_desc qcs404_desc = {
 	.max_state = RPM_SMD_LEVEL_BINNING,
 };
 
-/* sdm660 RPM Power domains */
-DEFINE_RPMPD_PAIR(sdm660, vddcx, vddcx_ao, RWCX, LEVEL, 0);
-DEFINE_RPMPD_VFL(sdm660, vddcx_vfl, RWCX, 0);
-
-DEFINE_RPMPD_PAIR(sdm660, vddmx, vddmx_ao, RWMX, LEVEL, 0);
-DEFINE_RPMPD_VFL(sdm660, vddmx_vfl, RWMX, 0);
-
-DEFINE_RPMPD_LEVEL(sdm660, vdd_ssccx, RWLC, 0);
-DEFINE_RPMPD_VFL(sdm660, vdd_ssccx_vfl, RWLC, 0);
-
-DEFINE_RPMPD_LEVEL(sdm660, vdd_sscmx, RWLM, 0);
-DEFINE_RPMPD_VFL(sdm660, vdd_sscmx_vfl, RWLM, 0);
-
 static struct rpmpd *sdm660_rpmpds[] = {
-	[SDM660_VDDCX] =		&sdm660_vddcx,
-	[SDM660_VDDCX_AO] =		&sdm660_vddcx_ao,
-	[SDM660_VDDCX_VFL] =		&sdm660_vddcx_vfl,
-	[SDM660_VDDMX] =		&sdm660_vddmx,
-	[SDM660_VDDMX_AO] =		&sdm660_vddmx_ao,
-	[SDM660_VDDMX_VFL] =		&sdm660_vddmx_vfl,
-	[SDM660_SSCCX] =		&sdm660_vdd_ssccx,
-	[SDM660_SSCCX_VFL] =		&sdm660_vdd_ssccx_vfl,
-	[SDM660_SSCMX] =		&sdm660_vdd_sscmx,
-	[SDM660_SSCMX_VFL] =		&sdm660_vdd_sscmx_vfl,
+	[SDM660_VDDCX] =	&cx_rwcx0_lvl,
+	[SDM660_VDDCX_AO] =	&cx_rwcx0_lvl_ao,
+	[SDM660_VDDCX_VFL] =	&cx_rwcx0_vfl,
+	[SDM660_VDDMX] =	&mx_rwmx0_lvl,
+	[SDM660_VDDMX_AO] =	&mx_rwmx0_lvl_ao,
+	[SDM660_VDDMX_VFL] =	&mx_rwmx0_vfl,
+	[SDM660_SSCCX] =	&ssc_cx_rwlc0_lvl,
+	[SDM660_SSCCX_VFL] =	&ssc_cx_rwlc0_vfl,
+	[SDM660_SSCMX] =	&ssc_mx_rwlm0_lvl,
+	[SDM660_SSCMX_VFL] =	&ssc_mx_rwlm0_vfl,
 };
 
 static const struct rpmpd_desc sdm660_desc = {
@@ -385,25 +691,15 @@ static const struct rpmpd_desc sdm660_desc = {
 	.max_state = RPM_SMD_LEVEL_TURBO,
 };
 
-/* sm4250/6115 RPM Power domains */
-DEFINE_RPMPD_PAIR(sm6115, vddcx, vddcx_ao, RWCX, LEVEL, 0);
-DEFINE_RPMPD_VFL(sm6115, vddcx_vfl, RWCX, 0);
-
-DEFINE_RPMPD_PAIR(sm6115, vddmx, vddmx_ao, RWMX, LEVEL, 0);
-DEFINE_RPMPD_VFL(sm6115, vddmx_vfl, RWMX, 0);
-
-DEFINE_RPMPD_LEVEL(sm6115, vdd_lpi_cx, RWLC, 0);
-DEFINE_RPMPD_LEVEL(sm6115, vdd_lpi_mx, RWLM, 0);
-
 static struct rpmpd *sm6115_rpmpds[] = {
-	[SM6115_VDDCX] =		&sm6115_vddcx,
-	[SM6115_VDDCX_AO] =		&sm6115_vddcx_ao,
-	[SM6115_VDDCX_VFL] =		&sm6115_vddcx_vfl,
-	[SM6115_VDDMX] =		&sm6115_vddmx,
-	[SM6115_VDDMX_AO] =		&sm6115_vddmx_ao,
-	[SM6115_VDDMX_VFL] =		&sm6115_vddmx_vfl,
-	[SM6115_VDD_LPI_CX] =		&sm6115_vdd_lpi_cx,
-	[SM6115_VDD_LPI_MX] =		&sm6115_vdd_lpi_mx,
+	[SM6115_VDDCX] =	&cx_rwcx0_lvl,
+	[SM6115_VDDCX_AO] =	&cx_rwcx0_lvl_ao,
+	[SM6115_VDDCX_VFL] =	&cx_rwcx0_vfl,
+	[SM6115_VDDMX] =	&mx_rwmx0_lvl,
+	[SM6115_VDDMX_AO] =	&mx_rwmx0_lvl_ao,
+	[SM6115_VDDMX_VFL] =	&mx_rwmx0_vfl,
+	[SM6115_VDD_LPI_CX] =	&lpi_cx_rwlc0_lvl,
+	[SM6115_VDD_LPI_MX] =	&lpi_mx_rwlm0_lvl,
 };
 
 static const struct rpmpd_desc sm6115_desc = {
@@ -412,20 +708,13 @@ static const struct rpmpd_desc sm6115_desc = {
 	.max_state = RPM_SMD_LEVEL_TURBO_NO_CPR,
 };
 
-/* sm6125 RPM Power domains */
-DEFINE_RPMPD_PAIR(sm6125, vddcx, vddcx_ao, RWCX, LEVEL, 0);
-DEFINE_RPMPD_VFL(sm6125, vddcx_vfl, RWCX, 0);
-
-DEFINE_RPMPD_PAIR(sm6125, vddmx, vddmx_ao, RWMX, LEVEL, 0);
-DEFINE_RPMPD_VFL(sm6125, vddmx_vfl, RWMX, 0);
-
 static struct rpmpd *sm6125_rpmpds[] = {
-	[SM6125_VDDCX] =		&sm6125_vddcx,
-	[SM6125_VDDCX_AO] =		&sm6125_vddcx_ao,
-	[SM6125_VDDCX_VFL] =		&sm6125_vddcx_vfl,
-	[SM6125_VDDMX] =		&sm6125_vddmx,
-	[SM6125_VDDMX_AO] =		&sm6125_vddmx_ao,
-	[SM6125_VDDMX_VFL] =		&sm6125_vddmx_vfl,
+	[SM6125_VDDCX] =	&cx_rwcx0_lvl,
+	[SM6125_VDDCX_AO] =	&cx_rwcx0_lvl_ao,
+	[SM6125_VDDCX_VFL] =	&cx_rwcx0_vfl,
+	[SM6125_VDDMX] =	&mx_rwmx0_lvl,
+	[SM6125_VDDMX_AO] =	&mx_rwmx0_lvl_ao,
+	[SM6125_VDDMX_VFL] =	&mx_rwmx0_vfl,
 };
 
 static const struct rpmpd_desc sm6125_desc = {
@@ -434,18 +723,17 @@ static const struct rpmpd_desc sm6125_desc = {
 	.max_state = RPM_SMD_LEVEL_BINNING,
 };
 
-DEFINE_RPMPD_PAIR(sm6375, vddgx, vddgx_ao, RWGX, LEVEL, 0);
 static struct rpmpd *sm6375_rpmpds[] = {
-	[SM6375_VDDCX] = &sm6125_vddcx,
-	[SM6375_VDDCX_AO] = &sm6125_vddcx_ao,
-	[SM6375_VDDCX_VFL] = &sm6125_vddcx_vfl,
-	[SM6375_VDDMX] = &sm6125_vddmx,
-	[SM6375_VDDMX_AO] = &sm6125_vddmx_ao,
-	[SM6375_VDDMX_VFL] = &sm6125_vddmx_vfl,
-	[SM6375_VDDGX] = &sm6375_vddgx,
-	[SM6375_VDDGX_AO] = &sm6375_vddgx_ao,
-	[SM6375_VDD_LPI_CX] = &sm6115_vdd_lpi_cx,
-	[SM6375_VDD_LPI_MX] = &sm6115_vdd_lpi_mx,
+	[SM6375_VDDCX] =	&cx_rwcx0_lvl,
+	[SM6375_VDDCX_AO] =	&cx_rwcx0_lvl_ao,
+	[SM6375_VDDCX_VFL] =	&cx_rwcx0_vfl,
+	[SM6375_VDDMX] =	&mx_rwmx0_lvl,
+	[SM6375_VDDMX_AO] =	&mx_rwmx0_lvl_ao,
+	[SM6375_VDDMX_VFL] =	&mx_rwmx0_vfl,
+	[SM6375_VDDGX] =	&gx_rwgx0_lvl,
+	[SM6375_VDDGX_AO] =	&gx_rwgx0_lvl_ao,
+	[SM6375_VDD_LPI_CX] =	&lpi_cx_rwlc0_lvl,
+	[SM6375_VDD_LPI_MX] =	&lpi_mx_rwlm0_lvl,
 };
 
 static const struct rpmpd_desc sm6375_desc = {
@@ -455,14 +743,14 @@ static const struct rpmpd_desc sm6375_desc = {
 };
 
 static struct rpmpd *qcm2290_rpmpds[] = {
-	[QCM2290_VDDCX] = &sm6115_vddcx,
-	[QCM2290_VDDCX_AO] = &sm6115_vddcx_ao,
-	[QCM2290_VDDCX_VFL] = &sm6115_vddcx_vfl,
-	[QCM2290_VDDMX] = &sm6115_vddmx,
-	[QCM2290_VDDMX_AO] = &sm6115_vddmx_ao,
-	[QCM2290_VDDMX_VFL] = &sm6115_vddmx_vfl,
-	[QCM2290_VDD_LPI_CX] = &sm6115_vdd_lpi_cx,
-	[QCM2290_VDD_LPI_MX] = &sm6115_vdd_lpi_mx,
+	[QCM2290_VDDCX] =	&cx_rwcx0_lvl,
+	[QCM2290_VDDCX_AO] =	&cx_rwcx0_lvl_ao,
+	[QCM2290_VDDCX_VFL] =	&cx_rwcx0_vfl,
+	[QCM2290_VDDMX] =	&mx_rwmx0_lvl,
+	[QCM2290_VDDMX_AO] =	&mx_rwmx0_lvl_ao,
+	[QCM2290_VDDMX_VFL] =	&mx_rwmx0_vfl,
+	[QCM2290_VDD_LPI_CX] =	&lpi_cx_rwlc0_lvl,
+	[QCM2290_VDD_LPI_MX] =	&lpi_mx_rwlm0_lvl,
 };
 
 static const struct rpmpd_desc qcm2290_desc = {
@@ -673,6 +961,15 @@ static int rpmpd_probe(struct platform_device *pdev)
 		data->domains[i] = &rpmpds[i]->pd;
 	}
 
+	/* Add subdomains */
+	for (i = 0; i < num; i++) {
+		if (!rpmpds[i])
+			continue;
+
+		if (rpmpds[i]->parent)
+			pm_genpd_add_subdomain(rpmpds[i]->parent, &rpmpds[i]->pd);
+	}
+
 	return of_genpd_add_provider_onecell(pdev->dev.of_node, data);
 }
 
diff --git a/drivers/soc/qcom/smd-rpm.c b/drivers/soc/qcom/smd-rpm.c
index 523627d5d398..0c1aa809cc4e 100644
--- a/drivers/soc/qcom/smd-rpm.c
+++ b/drivers/soc/qcom/smd-rpm.c
@@ -113,7 +113,7 @@ int qcom_rpm_smd_write(struct qcom_smd_rpm *rpm,
 	if (WARN_ON(size >= 256))
 		return -EINVAL;
 
-	pkt = kmalloc(size, GFP_KERNEL);
+	pkt = kmalloc(size, GFP_ATOMIC);
 	if (!pkt)
 		return -ENOMEM;
 
diff --git a/drivers/soc/qcom/smem.c b/drivers/soc/qcom/smem.c
index 4f163d62942c..6be7ea93c78c 100644
--- a/drivers/soc/qcom/smem.c
+++ b/drivers/soc/qcom/smem.c
@@ -85,7 +85,7 @@
 #define SMEM_GLOBAL_HOST	0xfffe
 
 /* Max number of processors/hosts in a system */
-#define SMEM_HOST_COUNT		15
+#define SMEM_HOST_COUNT		20
 
 /**
   * struct smem_proc_comm - proc_comm communication struct (legacy)
@@ -1045,7 +1045,7 @@ static int qcom_smem_probe(struct platform_device *pdev)
 	int i;
 
 	num_regions = 1;
-	if (of_find_property(pdev->dev.of_node, "qcom,rpm-msg-ram", NULL))
+	if (of_property_present(pdev->dev.of_node, "qcom,rpm-msg-ram"))
 		num_regions++;
 
 	array_size = num_regions * sizeof(struct smem_region);
diff --git a/drivers/soc/qcom/smsm.c b/drivers/soc/qcom/smsm.c
index 3e8994d6110e..c58cfff64856 100644
--- a/drivers/soc/qcom/smsm.c
+++ b/drivers/soc/qcom/smsm.c
@@ -452,11 +452,10 @@ static int smsm_get_size_info(struct qcom_smsm *smsm)
 	} *info;
 
 	info = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_SIZE_INFO, &size);
-	if (IS_ERR(info) && PTR_ERR(info) != -ENOENT) {
-		if (PTR_ERR(info) != -EPROBE_DEFER)
-			dev_err(smsm->dev, "unable to retrieve smsm size info\n");
-		return PTR_ERR(info);
-	} else if (IS_ERR(info) || size != sizeof(*info)) {
+	if (IS_ERR(info) && PTR_ERR(info) != -ENOENT)
+		return dev_err_probe(smsm->dev, PTR_ERR(info),
+				     "unable to retrieve smsm size info\n");
+	else if (IS_ERR(info) || size != sizeof(*info)) {
 		dev_warn(smsm->dev, "no smsm size info, using defaults\n");
 		smsm->num_entries = SMSM_DEFAULT_NUM_ENTRIES;
 		smsm->num_hosts = SMSM_DEFAULT_NUM_HOSTS;
@@ -510,7 +509,7 @@ static int qcom_smsm_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	for_each_child_of_node(pdev->dev.of_node, local_node) {
-		if (of_find_property(local_node, "#qcom,smem-state-cells", NULL))
+		if (of_property_present(local_node, "#qcom,smem-state-cells"))
 			break;
 	}
 	if (!local_node) {
diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c
index e9012ca1a87b..c2e4a57dd666 100644
--- a/drivers/soc/qcom/socinfo.c
+++ b/drivers/soc/qcom/socinfo.c
@@ -109,15 +109,20 @@ static const char *const pmic_models[] = {
 	[32] = "PM8150B",
 	[33] = "PMK8002",
 	[36] = "PM8009",
+	[37] = "PMI632",
 	[38] = "PM8150C",
+	[40] = "PM6150",
 	[41] = "SMB2351",
+	[44] = "PM8008",
 	[45] = "PM6125",
+	[46] = "PM7250B",
 	[47] = "PMK8350",
 	[48] = "PM8350",
 	[49] = "PM8350C",
 	[50] = "PM8350B",
 	[51] = "PMR735A",
 	[52] = "PMR735B",
+	[55] = "PM2250",
 	[58] = "PM8450",
 	[65] = "PM8010",
 };
@@ -405,6 +410,7 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(SA8155) },
 	{ qcom_board_id(SDA439) },
 	{ qcom_board_id(SDA429) },
+	{ qcom_board_id(SM7150) },
 	{ qcom_board_id(IPQ8070) },
 	{ qcom_board_id(IPQ8071) },
 	{ qcom_board_id(QM215) },
@@ -426,6 +432,7 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(QCM2150) },
 	{ qcom_board_id(SDA429W) },
 	{ qcom_board_id(SM8350) },
+	{ qcom_board_id(QCM2290) },
 	{ qcom_board_id(SM6115) },
 	{ qcom_board_id(SC8280XP) },
 	{ qcom_board_id(IPQ6005) },
@@ -441,7 +448,16 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(SC7280) },
 	{ qcom_board_id(SC7180P) },
 	{ qcom_board_id(SM6375) },
+	{ qcom_board_id(IPQ9514) },
+	{ qcom_board_id(IPQ9550) },
+	{ qcom_board_id(IPQ9554) },
+	{ qcom_board_id(IPQ9570) },
+	{ qcom_board_id(IPQ9574) },
 	{ qcom_board_id(SM8550) },
+	{ qcom_board_id(IPQ9510) },
+	{ qcom_board_id(QRB4210) },
+	{ qcom_board_id(QRB2210) },
+	{ qcom_board_id(SA8775P) },
 	{ qcom_board_id(QRU1000) },
 	{ qcom_board_id(QDU1000) },
 	{ qcom_board_id(QDU1010) },
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index 4e8b51ba2266..de31589ed054 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -206,13 +206,6 @@ config ARCH_R8A77990
 	  This enables support for the Renesas R-Car E3 SoC.
 	  This includes different gradings like R-Car E3e.
 
-config ARCH_R8A77950
-	bool "ARM64 Platform support for R-Car H3 ES1.x"
-	select ARCH_RCAR_GEN3
-	select SYSC_R8A7795
-	help
-	  This enables support for the Renesas R-Car H3 SoC (revision 1.x).
-
 config ARCH_R8A77951
 	bool "ARM64 Platform support for R-Car H3 ES2.0+"
 	select ARCH_RCAR_GEN3
diff --git a/drivers/soc/renesas/pwc-rzv2m.c b/drivers/soc/renesas/pwc-rzv2m.c
index c83bdbdabb64..452cee8d68be 100644
--- a/drivers/soc/renesas/pwc-rzv2m.c
+++ b/drivers/soc/renesas/pwc-rzv2m.c
@@ -131,7 +131,7 @@ static struct platform_driver rzv2m_pwc_driver = {
 	.probe = rzv2m_pwc_probe,
 	.driver = {
 		.name = "rzv2m_pwc",
-		.of_match_table = of_match_ptr(rzv2m_pwc_of_match),
+		.of_match_table = rzv2m_pwc_of_match,
 	},
 };
 module_platform_driver(rzv2m_pwc_driver);
diff --git a/drivers/soc/renesas/r8a7795-sysc.c b/drivers/soc/renesas/r8a7795-sysc.c
index 91074411b8cf..cbe1ff0fc583 100644
--- a/drivers/soc/renesas/r8a7795-sysc.c
+++ b/drivers/soc/renesas/r8a7795-sysc.c
@@ -38,8 +38,6 @@ static struct rcar_sysc_area r8a7795_areas[] __initdata = {
 	{ "a3vp",	0x340, 0, R8A7795_PD_A3VP,	R8A7795_PD_ALWAYS_ON },
 	{ "cr7",	0x240, 0, R8A7795_PD_CR7,	R8A7795_PD_ALWAYS_ON },
 	{ "a3vc",	0x380, 0, R8A7795_PD_A3VC,	R8A7795_PD_ALWAYS_ON },
-	/* A2VC0 exists on ES1.x only */
-	{ "a2vc0",	0x3c0, 0, R8A7795_PD_A2VC0,	R8A7795_PD_A3VC },
 	{ "a2vc1",	0x3c0, 1, R8A7795_PD_A2VC1,	R8A7795_PD_A3VC },
 	{ "3dg-a",	0x100, 0, R8A7795_PD_3DG_A,	R8A7795_PD_ALWAYS_ON },
 	{ "3dg-b",	0x100, 1, R8A7795_PD_3DG_B,	R8A7795_PD_3DG_A },
@@ -54,14 +52,10 @@ static struct rcar_sysc_area r8a7795_areas[] __initdata = {
 	 * Fixups for R-Car H3 revisions
 	 */
 
-#define HAS_A2VC0	BIT(0)		/* Power domain A2VC0 is present */
 #define NO_EXTMASK	BIT(1)		/* Missing SYSCEXTMASK register */
 
 static const struct soc_device_attribute r8a7795_quirks_match[] __initconst = {
 	{
-		.soc_id = "r8a7795", .revision = "ES1.*",
-		.data = (void *)(HAS_A2VC0 | NO_EXTMASK),
-	}, {
 		.soc_id = "r8a7795", .revision = "ES2.*",
 		.data = (void *)(NO_EXTMASK),
 	},
@@ -77,10 +71,6 @@ static int __init r8a7795_sysc_init(void)
 	if (attr)
 		quirks = (uintptr_t)attr->data;
 
-	if (!(quirks & HAS_A2VC0))
-		rcar_sysc_nullify(r8a7795_areas, ARRAY_SIZE(r8a7795_areas),
-				  R8A7795_PD_A2VC0);
-
 	if (quirks & NO_EXTMASK)
 		r8a7795_sysc_info.extmask_val = 0;
 
diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c
index 468ebce1ea88..42af7c09f743 100644
--- a/drivers/soc/renesas/renesas-soc.c
+++ b/drivers/soc/renesas/renesas-soc.c
@@ -269,7 +269,7 @@ static const struct renesas_soc soc_shmobile_ag5 __initconst __maybe_unused = {
 };
 
 
-static const struct of_device_id renesas_socs[] __initconst = {
+static const struct of_device_id renesas_socs[] __initconst __maybe_unused = {
 #ifdef CONFIG_ARCH_R7S72100
 	{ .compatible = "renesas,r7s72100",	.data = &soc_rz_a1h },
 #endif
@@ -330,10 +330,8 @@ static const struct of_device_id renesas_socs[] __initconst = {
 #ifdef CONFIG_ARCH_R8A7794
 	{ .compatible = "renesas,r8a7794",	.data = &soc_rcar_e2 },
 #endif
-#if defined(CONFIG_ARCH_R8A77950) || defined(CONFIG_ARCH_R8A77951)
-	{ .compatible = "renesas,r8a7795",	.data = &soc_rcar_h3 },
-#endif
 #ifdef CONFIG_ARCH_R8A77951
+	{ .compatible = "renesas,r8a7795",	.data = &soc_rcar_h3 },
 	{ .compatible = "renesas,r8a779m0",	.data = &soc_rcar_h3 },
 	{ .compatible = "renesas,r8a779m1",	.data = &soc_rcar_h3 },
 	{ .compatible = "renesas,r8a779m8",	.data = &soc_rcar_h3 },
@@ -375,20 +373,20 @@ static const struct of_device_id renesas_socs[] __initconst = {
 #ifdef CONFIG_ARCH_R8A779G0
 	{ .compatible = "renesas,r8a779g0",	.data = &soc_rcar_v4h },
 #endif
-#if defined(CONFIG_ARCH_R9A07G043)
+#ifdef CONFIG_ARCH_R9A07G043
 #ifdef CONFIG_RISCV
 	{ .compatible = "renesas,r9a07g043",	.data = &soc_rz_five },
 #else
 	{ .compatible = "renesas,r9a07g043",	.data = &soc_rz_g2ul },
 #endif
 #endif
-#if defined(CONFIG_ARCH_R9A07G044)
+#ifdef CONFIG_ARCH_R9A07G044
 	{ .compatible = "renesas,r9a07g044",	.data = &soc_rz_g2l },
 #endif
-#if defined(CONFIG_ARCH_R9A07G054)
+#ifdef CONFIG_ARCH_R9A07G054
 	{ .compatible = "renesas,r9a07g054",	.data = &soc_rz_v2l },
 #endif
-#if defined(CONFIG_ARCH_R9A09G011)
+#ifdef CONFIG_ARCH_R9A09G011
 	{ .compatible = "renesas,r9a09g011",	.data = &soc_rz_v2m },
 #endif
 #ifdef CONFIG_ARCH_SH73A0
@@ -471,8 +469,11 @@ static int __init renesas_soc_init(void)
 	}
 
 	soc_dev_attr = kzalloc(sizeof(*soc_dev_attr), GFP_KERNEL);
-	if (!soc_dev_attr)
+	if (!soc_dev_attr) {
+		if (chipid)
+			iounmap(chipid);
 		return -ENOMEM;
+	}
 
 	np = of_find_node_by_path("/");
 	of_property_read_string(np, "model", &soc_dev_attr->machine);
diff --git a/drivers/soc/renesas/rmobile-sysc.c b/drivers/soc/renesas/rmobile-sysc.c
index 204e6135180b..728ebac98e14 100644
--- a/drivers/soc/renesas/rmobile-sysc.c
+++ b/drivers/soc/renesas/rmobile-sysc.c
@@ -343,7 +343,7 @@ static int __init rmobile_init_pm_domains(void)
 			break;
 		}
 
-		fwnode_dev_initialized(&np->fwnode, true);
+		fwnode_dev_initialized(of_fwnode_handle(np), true);
 	}
 
 	put_special_pds();
diff --git a/drivers/soc/sunxi/sunxi_mbus.c b/drivers/soc/sunxi/sunxi_mbus.c
index d90e4a264b6f..1734da357ca2 100644
--- a/drivers/soc/sunxi/sunxi_mbus.c
+++ b/drivers/soc/sunxi/sunxi_mbus.c
@@ -82,7 +82,7 @@ static int sunxi_mbus_notifier(struct notifier_block *nb,
 	 * Older DTs or SoCs who are not clearly understood need to set
 	 * that DMA offset though.
 	 */
-	if (of_find_property(dev->of_node, "interconnects", NULL))
+	if (of_property_present(dev->of_node, "interconnects"))
 		return NOTIFY_DONE;
 
 	ret = dma_direct_set_offset(dev, PHYS_OFFSET, 0, SZ_4G);
diff --git a/drivers/soc/sunxi/sunxi_sram.c b/drivers/soc/sunxi/sunxi_sram.c
index f09918c59042..4c4864cd2342 100644
--- a/drivers/soc/sunxi/sunxi_sram.c
+++ b/drivers/soc/sunxi/sunxi_sram.c
@@ -424,4 +424,3 @@ builtin_platform_driver_probe(sunxi_sram_driver, sunxi_sram_probe);
 
 MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
 MODULE_DESCRIPTION("Allwinner sunXi SRAM Controller Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/soc/tegra/cbb/tegra-cbb.c b/drivers/soc/tegra/cbb/tegra-cbb.c
index a8566b9dd8de..bd96204a68ee 100644
--- a/drivers/soc/tegra/cbb/tegra-cbb.c
+++ b/drivers/soc/tegra/cbb/tegra-cbb.c
@@ -16,7 +16,6 @@
 #include <linux/of_address.h>
 #include <linux/interrupt.h>
 #include <linux/ioport.h>
-#include <linux/version.h>
 #include <soc/tegra/fuse.h>
 #include <soc/tegra/tegra-cbb.h>
 
diff --git a/drivers/soc/tegra/cbb/tegra194-cbb.c b/drivers/soc/tegra/cbb/tegra194-cbb.c
index d4112b683f00..54d7ce05c636 100644
--- a/drivers/soc/tegra/cbb/tegra194-cbb.c
+++ b/drivers/soc/tegra/cbb/tegra194-cbb.c
@@ -23,7 +23,6 @@
 #include <linux/of_address.h>
 #include <linux/interrupt.h>
 #include <linux/ioport.h>
-#include <linux/version.h>
 #include <soc/tegra/fuse.h>
 #include <soc/tegra/tegra-cbb.h>
 
@@ -2191,7 +2190,6 @@ MODULE_DEVICE_TABLE(of, tegra194_cbb_match);
 static int tegra194_cbb_get_bridges(struct tegra194_cbb *cbb, struct device_node *np)
 {
 	struct tegra_cbb *entry;
-	struct resource res;
 	unsigned long flags;
 	unsigned int i;
 	int err;
@@ -2211,8 +2209,7 @@ static int tegra194_cbb_get_bridges(struct tegra194_cbb *cbb, struct device_node
 	spin_unlock_irqrestore(&cbb_lock, flags);
 
 	if (!cbb->bridges) {
-		while (of_address_to_resource(np, cbb->num_bridges, &res) == 0)
-			cbb->num_bridges++;
+		cbb->num_bridges = of_address_count(np);
 
 		cbb->bridges = devm_kcalloc(cbb->base.dev, cbb->num_bridges,
 					    sizeof(*cbb->bridges), GFP_KERNEL);
@@ -2359,4 +2356,3 @@ module_exit(tegra194_cbb_exit);
 
 MODULE_AUTHOR("Sumit Gupta <sumitg@nvidia.com>");
 MODULE_DESCRIPTION("Control Backbone error handling driver for Tegra194");
-MODULE_LICENSE("GPL");
diff --git a/drivers/soc/tegra/cbb/tegra234-cbb.c b/drivers/soc/tegra/cbb/tegra234-cbb.c
index f33d094e5ea6..5d16161b2566 100644
--- a/drivers/soc/tegra/cbb/tegra234-cbb.c
+++ b/drivers/soc/tegra/cbb/tegra234-cbb.c
@@ -24,7 +24,6 @@
 #include <linux/of_address.h>
 #include <linux/interrupt.h>
 #include <linux/ioport.h>
-#include <linux/version.h>
 #include <soc/tegra/fuse.h>
 #include <soc/tegra/tegra-cbb.h>
 
@@ -1174,11 +1173,6 @@ static int tegra234_cbb_probe(struct platform_device *pdev)
 	return tegra_cbb_register(&cbb->base);
 }
 
-static int tegra234_cbb_remove(struct platform_device *pdev)
-{
-	return 0;
-}
-
 static int __maybe_unused tegra234_cbb_resume_noirq(struct device *dev)
 {
 	struct tegra234_cbb *cbb = dev_get_drvdata(dev);
@@ -1196,7 +1190,6 @@ static const struct dev_pm_ops tegra234_cbb_pm = {
 
 static struct platform_driver tegra234_cbb_driver = {
 	.probe = tegra234_cbb_probe,
-	.remove = tegra234_cbb_remove,
 	.driver = {
 		.name = "tegra234-cbb",
 		.of_match_table = tegra234_cbb_dt_ids,
@@ -1218,4 +1211,3 @@ static void __exit tegra234_cbb_exit(void)
 module_exit(tegra234_cbb_exit);
 
 MODULE_DESCRIPTION("Control Backbone 2.0 error handling driver for Tegra234");
-MODULE_LICENSE("GPL");
diff --git a/drivers/soc/tegra/flowctrl.c b/drivers/soc/tegra/flowctrl.c
index 5db919d96aba..221202db3313 100644
--- a/drivers/soc/tegra/flowctrl.c
+++ b/drivers/soc/tegra/flowctrl.c
@@ -156,10 +156,8 @@ void flowctrl_cpu_suspend_exit(unsigned int cpuid)
 static int tegra_flowctrl_probe(struct platform_device *pdev)
 {
 	void __iomem *base = tegra_flowctrl_base;
-	struct resource *res;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	tegra_flowctrl_base = devm_ioremap_resource(&pdev->dev, res);
+	tegra_flowctrl_base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
 	if (IS_ERR(tegra_flowctrl_base))
 		return PTR_ERR(tegra_flowctrl_base);
 
diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c
index f02953f793e9..d7a37f5d4527 100644
--- a/drivers/soc/tegra/fuse/fuse-tegra.c
+++ b/drivers/soc/tegra/fuse/fuse-tegra.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (c) 2013-2022, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2013-2023, NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <linux/clk.h>
@@ -166,7 +166,7 @@ static int tegra_fuse_probe(struct platform_device *pdev)
 	nvmem.nkeepout = fuse->soc->num_keepouts;
 	nvmem.type = NVMEM_TYPE_OTP;
 	nvmem.read_only = true;
-	nvmem.root_only = true;
+	nvmem.root_only = false;
 	nvmem.reg_read = tegra_fuse_read;
 	nvmem.size = fuse->soc->info->size;
 	nvmem.word_size = 4;
diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c
index cf4cfbf9f7c5..5d17799524c9 100644
--- a/drivers/soc/tegra/pmc.c
+++ b/drivers/soc/tegra/pmc.c
@@ -3,7 +3,7 @@
  * drivers/soc/tegra/pmc.c
  *
  * Copyright (c) 2010 Google, Inc
- * Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
  *
  * Author:
  *	Colin Cross <ccross@google.com>
@@ -177,6 +177,7 @@
 /* Tegra186 and later */
 #define WAKE_AOWAKE_CNTRL(x) (0x000 + ((x) << 2))
 #define WAKE_AOWAKE_CNTRL_LEVEL (1 << 3)
+#define WAKE_AOWAKE_CNTRL_SR_CAPTURE_EN (1 << 1)
 #define WAKE_AOWAKE_MASK_W(x) (0x180 + ((x) << 2))
 #define WAKE_AOWAKE_MASK_R(x) (0x300 + ((x) << 2))
 #define WAKE_AOWAKE_STATUS_W(x) (0x30c + ((x) << 2))
@@ -191,6 +192,8 @@
 #define WAKE_AOWAKE_CTRL 0x4f4
 #define  WAKE_AOWAKE_CTRL_INTR_POLARITY BIT(0)
 
+#define SW_WAKE_ID		83 /* wake83 */
+
 /* for secure PMC */
 #define TEGRA_SMC_PMC		0xc2fffe00
 #define  TEGRA_SMC_PMC_READ	0xaa
@@ -355,6 +358,7 @@ struct tegra_pmc_soc {
 	void (*setup_irq_polarity)(struct tegra_pmc *pmc,
 				   struct device_node *np,
 				   bool invert);
+	void (*set_wake_filters)(struct tegra_pmc *pmc);
 	int (*irq_set_wake)(struct irq_data *data, unsigned int on);
 	int (*irq_set_type)(struct irq_data *data, unsigned int type);
 	int (*powergate_set)(struct tegra_pmc *pmc, unsigned int id,
@@ -2416,6 +2420,17 @@ static int tegra210_pmc_irq_set_type(struct irq_data *data, unsigned int type)
 	return 0;
 }
 
+static void tegra186_pmc_set_wake_filters(struct tegra_pmc *pmc)
+{
+	u32 value;
+
+	/* SW Wake (wake83) needs SR_CAPTURE filter to be enabled */
+	value = readl(pmc->wake + WAKE_AOWAKE_CNTRL(SW_WAKE_ID));
+	value |= WAKE_AOWAKE_CNTRL_SR_CAPTURE_EN;
+	writel(value, pmc->wake + WAKE_AOWAKE_CNTRL(SW_WAKE_ID));
+	dev_dbg(pmc->dev, "WAKE_AOWAKE_CNTRL_83 = 0x%x\n", value);
+}
+
 static int tegra186_pmc_irq_set_wake(struct irq_data *data, unsigned int on)
 {
 	struct tegra_pmc *pmc = irq_data_get_irq_chip_data(data);
@@ -3042,6 +3057,10 @@ static int tegra_pmc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, pmc);
 	tegra_pm_init_suspend();
 
+	/* Some wakes require specific filter configuration */
+	if (pmc->soc->set_wake_filters)
+		pmc->soc->set_wake_filters(pmc);
+
 	return 0;
 
 cleanup_powergates:
@@ -3938,6 +3957,7 @@ static const struct tegra_pmc_soc tegra186_pmc_soc = {
 	.regs = &tegra186_pmc_regs,
 	.init = tegra186_pmc_init,
 	.setup_irq_polarity = tegra186_pmc_setup_irq_polarity,
+	.set_wake_filters = tegra186_pmc_set_wake_filters,
 	.irq_set_wake = tegra186_pmc_irq_set_wake,
 	.irq_set_type = tegra186_pmc_irq_set_type,
 	.reset_sources = tegra186_reset_sources,
@@ -4122,6 +4142,7 @@ static const struct tegra_pmc_soc tegra194_pmc_soc = {
 	.regs = &tegra194_pmc_regs,
 	.init = tegra186_pmc_init,
 	.setup_irq_polarity = tegra186_pmc_setup_irq_polarity,
+	.set_wake_filters = tegra186_pmc_set_wake_filters,
 	.irq_set_wake = tegra186_pmc_irq_set_wake,
 	.irq_set_type = tegra186_pmc_irq_set_type,
 	.reset_sources = tegra194_reset_sources,
@@ -4225,7 +4246,9 @@ static const char * const tegra234_reset_sources[] = {
 };
 
 static const struct tegra_wake_event tegra234_wake_events[] = {
+	TEGRA_WAKE_IRQ("pmu", 24, 209),
 	TEGRA_WAKE_GPIO("power", 29, 1, TEGRA234_AON_GPIO(EE, 4)),
+	TEGRA_WAKE_GPIO("mgbe", 56, 0, TEGRA234_MAIN_GPIO(Y, 3)),
 	TEGRA_WAKE_IRQ("rtc", 73, 10),
 };
 
@@ -4247,6 +4270,7 @@ static const struct tegra_pmc_soc tegra234_pmc_soc = {
 	.regs = &tegra234_pmc_regs,
 	.init = tegra186_pmc_init,
 	.setup_irq_polarity = tegra186_pmc_setup_irq_polarity,
+	.set_wake_filters = tegra186_pmc_set_wake_filters,
 	.irq_set_wake = tegra186_pmc_irq_set_wake,
 	.irq_set_type = tegra186_pmc_irq_set_type,
 	.reset_sources = tegra234_reset_sources,
diff --git a/drivers/soc/tegra/powergate-bpmp.c b/drivers/soc/tegra/powergate-bpmp.c
index 8eaf50d0b6af..179ed895c279 100644
--- a/drivers/soc/tegra/powergate-bpmp.c
+++ b/drivers/soc/tegra/powergate-bpmp.c
@@ -286,7 +286,7 @@ remove:
 		tegra_powergate_remove(powergate);
 	}
 
-	kfree(genpd->domains);
+	kfree(domains);
 	return err;
 }
 
diff --git a/drivers/soc/ti/k3-ringacc.c b/drivers/soc/ti/k3-ringacc.c
index e01e4d815230..8f131368a758 100644
--- a/drivers/soc/ti/k3-ringacc.c
+++ b/drivers/soc/ti/k3-ringacc.c
@@ -406,6 +406,11 @@ static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id,
 
 	mutex_lock(&ringacc->req_lock);
 
+	if (!try_module_get(ringacc->dev->driver->owner)) {
+		ret = -EINVAL;
+		goto err_module_get;
+	}
+
 	if (test_bit(fwd_id, ringacc->rings_inuse)) {
 		ret = -EBUSY;
 		goto error;
@@ -421,6 +426,8 @@ static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id,
 	return 0;
 
 error:
+	module_put(ringacc->dev->driver->owner);
+err_module_get:
 	mutex_unlock(&ringacc->req_lock);
 	return ret;
 }
diff --git a/drivers/soc/ti/k3-socinfo.c b/drivers/soc/ti/k3-socinfo.c
index d15764e19d96..ad97e08a25f6 100644
--- a/drivers/soc/ti/k3-socinfo.c
+++ b/drivers/soc/ti/k3-socinfo.c
@@ -43,6 +43,7 @@ static const struct k3_soc_id {
 	{ 0xBB38, "AM64X" },
 	{ 0xBB75, "J721S2"},
 	{ 0xBB7E, "AM62X" },
+	{ 0xBB80, "J784S4" },
 	{ 0xBB8D, "AM62AX" },
 };
 
diff --git a/drivers/soc/ti/knav_dma.c b/drivers/soc/ti/knav_dma.c
index 84afebd355be..0fbc37cd5123 100644
--- a/drivers/soc/ti/knav_dma.c
+++ b/drivers/soc/ti/knav_dma.c
@@ -666,8 +666,8 @@ static int dma_init(struct device_node *cloud, struct device_node *dma_node)
 	dma->rx_priority = DMA_PRIO_DEFAULT;
 	dma->tx_priority = DMA_PRIO_DEFAULT;
 
-	dma->enable_all	= (of_get_property(node, "ti,enable-all", NULL) != NULL);
-	dma->loopback	= (of_get_property(node, "ti,loop-back",  NULL) != NULL);
+	dma->enable_all	= of_property_read_bool(node, "ti,enable-all");
+	dma->loopback	= of_property_read_bool(node, "ti,loop-back");
 
 	ret = of_property_read_u32(node, "ti,rx-retry-timeout", &timeout);
 	if (ret < 0) {
diff --git a/drivers/soc/ti/knav_qmss_acc.c b/drivers/soc/ti/knav_qmss_acc.c
index fde66e28e046..3d388646ed43 100644
--- a/drivers/soc/ti/knav_qmss_acc.c
+++ b/drivers/soc/ti/knav_qmss_acc.c
@@ -521,7 +521,7 @@ int knav_init_acc_range(struct knav_device *kdev,
 
 	info->pdsp = pdsp;
 	channels = range->num_queues;
-	if (of_get_property(node, "multi-queue", NULL)) {
+	if (of_property_read_bool(node, "multi-queue")) {
 		range->flags |= RANGE_MULTI_QUEUE;
 		channels = 1;
 		if (range->queue_base & (32 - 1)) {
diff --git a/drivers/soc/ti/knav_qmss_queue.c b/drivers/soc/ti/knav_qmss_queue.c
index 8fb76908be70..0f252c2549ba 100644
--- a/drivers/soc/ti/knav_qmss_queue.c
+++ b/drivers/soc/ti/knav_qmss_queue.c
@@ -1264,10 +1264,10 @@ static int knav_setup_queue_range(struct knav_device *kdev,
 	if (range->num_irqs)
 		range->flags |= RANGE_HAS_IRQ;
 
-	if (of_get_property(node, "qalloc-by-id", NULL))
+	if (of_property_read_bool(node, "qalloc-by-id"))
 		range->flags |= RANGE_RESERVED;
 
-	if (of_get_property(node, "accumulator", NULL)) {
+	if (of_property_present(node, "accumulator")) {
 		ret = knav_init_acc_range(kdev, node, range);
 		if (ret < 0) {
 			devm_kfree(dev, range);
diff --git a/drivers/soc/ti/omap_prm.c b/drivers/soc/ti/omap_prm.c
index 913b964374a4..ecd9a8bdd7c0 100644
--- a/drivers/soc/ti/omap_prm.c
+++ b/drivers/soc/ti/omap_prm.c
@@ -684,7 +684,7 @@ static int omap_prm_domain_init(struct device *dev, struct omap_prm *prm)
 	const char *name;
 	int error;
 
-	if (!of_find_property(dev->of_node, "#power-domain-cells", NULL))
+	if (!of_property_present(dev->of_node, "#power-domain-cells"))
 		return 0;
 
 	of_node_put(dev->of_node);
diff --git a/drivers/soc/ti/pm33xx.c b/drivers/soc/ti/pm33xx.c
index ce09c42eaed2..f04c21157904 100644
--- a/drivers/soc/ti/pm33xx.c
+++ b/drivers/soc/ti/pm33xx.c
@@ -527,7 +527,7 @@ static int am33xx_pm_probe(struct platform_device *pdev)
 
 	ret = am33xx_pm_alloc_sram();
 	if (ret)
-		return ret;
+		goto err_wkup_m3_ipc_put;
 
 	ret = am33xx_pm_rtc_setup();
 	if (ret)
@@ -572,13 +572,14 @@ err_pm_runtime_put:
 	pm_runtime_put_sync(dev);
 err_pm_runtime_disable:
 	pm_runtime_disable(dev);
-	wkup_m3_ipc_put(m3_ipc);
 err_unsetup_rtc:
 	iounmap(rtc_base_virt);
 	clk_put(rtc_fck);
 err_free_sram:
 	am33xx_pm_free_sram();
 	pm33xx_dev = NULL;
+err_wkup_m3_ipc_put:
+	wkup_m3_ipc_put(m3_ipc);
 	return ret;
 }
 
diff --git a/drivers/soc/ti/smartreflex.c b/drivers/soc/ti/smartreflex.c
index 9d9496e0a94c..da7898239a46 100644
--- a/drivers/soc/ti/smartreflex.c
+++ b/drivers/soc/ti/smartreflex.c
@@ -937,21 +937,8 @@ err_list_del:
 
 static int omap_sr_remove(struct platform_device *pdev)
 {
-	struct omap_sr_data *pdata = pdev->dev.platform_data;
 	struct device *dev = &pdev->dev;
-	struct omap_sr *sr_info;
-
-	if (!pdata) {
-		dev_err(&pdev->dev, "%s: platform data missing\n", __func__);
-		return -EINVAL;
-	}
-
-	sr_info = _sr_lookup(pdata->voltdm);
-	if (IS_ERR(sr_info)) {
-		dev_warn(&pdev->dev, "%s: omap_sr struct not found\n",
-			__func__);
-		return PTR_ERR(sr_info);
-	}
+	struct omap_sr *sr_info = platform_get_drvdata(pdev);
 
 	if (sr_info->autocomp_active)
 		sr_stop_vddautocomp(sr_info);
@@ -965,20 +952,7 @@ static int omap_sr_remove(struct platform_device *pdev)
 
 static void omap_sr_shutdown(struct platform_device *pdev)
 {
-	struct omap_sr_data *pdata = pdev->dev.platform_data;
-	struct omap_sr *sr_info;
-
-	if (!pdata) {
-		dev_err(&pdev->dev, "%s: platform data missing\n", __func__);
-		return;
-	}
-
-	sr_info = _sr_lookup(pdata->voltdm);
-	if (IS_ERR(sr_info)) {
-		dev_warn(&pdev->dev, "%s: omap_sr struct not found\n",
-			__func__);
-		return;
-	}
+	struct omap_sr *sr_info = platform_get_drvdata(pdev);
 
 	if (sr_info->autocomp_active)
 		sr_stop_vddautocomp(sr_info);
diff --git a/drivers/soc/ti/wkup_m3_ipc.c b/drivers/soc/ti/wkup_m3_ipc.c
index 343c58ed5896..c9197912ec24 100644
--- a/drivers/soc/ti/wkup_m3_ipc.c
+++ b/drivers/soc/ti/wkup_m3_ipc.c
@@ -615,7 +615,6 @@ static int wkup_m3_ipc_probe(struct platform_device *pdev)
 	int irq, ret, temp;
 	phandle rproc_phandle;
 	struct rproc *m3_rproc;
-	struct resource *res;
 	struct task_struct *task;
 	struct wkup_m3_ipc *m3_ipc;
 	struct device_node *np = dev->of_node;
@@ -624,8 +623,7 @@ static int wkup_m3_ipc_probe(struct platform_device *pdev)
 	if (!m3_ipc)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	m3_ipc->ipc_mem_base = devm_ioremap_resource(dev, res);
+	m3_ipc->ipc_mem_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(m3_ipc->ipc_mem_base))
 		return PTR_ERR(m3_ipc->ipc_mem_base);
 
@@ -681,7 +679,7 @@ static int wkup_m3_ipc_probe(struct platform_device *pdev)
 			dev_warn(dev, "Invalid VTT GPIO(%d) pin\n", temp);
 	}
 
-	if (of_find_property(np, "ti,set-io-isolation", NULL))
+	if (of_property_read_bool(np, "ti,set-io-isolation"))
 		wkup_m3_set_io_isolation(m3_ipc);
 
 	ret = of_property_read_string(np, "firmware-name",
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index 47bbba04fe3a..3de2ebe8294a 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -79,6 +79,13 @@ config SPI_ALTERA_DFL
 	  Altera SPI master controller.  The SPI master is connected
 	  to a SPI slave to Avalon bridge in a Intel MAX BMC.
 
+config SPI_AMLOGIC_SPIFC_A1
+	tristate "Amlogic A1 SPIFC controller"
+	depends on ARCH_MESON || COMPILE_TEST
+	help
+	  This enables master mode support for the SPIFC (SPI flash
+	  controller) available in Amlogic A1 (A113L SoC).
+
 config SPI_AR934X
 	tristate "Qualcomm Atheros AR934X/QCA95XX SPI controller driver"
 	depends on ATH79 || COMPILE_TEST
@@ -239,7 +246,7 @@ config SPI_CADENCE
 
 config SPI_CADENCE_QUADSPI
 	tristate "Cadence Quad SPI controller"
-	depends on OF && (ARM || ARM64 || X86 || COMPILE_TEST)
+	depends on OF && (ARM || ARM64 || X86 || RISCV || COMPILE_TEST)
 	help
 	  Enable support for the Cadence Quad SPI Flash controller.
 
@@ -276,7 +283,7 @@ config SPI_COLDFIRE_QSPI
 
 config SPI_DAVINCI
 	tristate "Texas Instruments DaVinci/DA8x/OMAP-L/AM1x SoC SPI controller"
-	depends on ARCH_DAVINCI || ARCH_KEYSTONE
+	depends on ARCH_DAVINCI || ARCH_KEYSTONE || COMPILE_TEST
 	select SPI_BITBANG
 	help
 	  SPI master controller for DaVinci/DA8x/OMAP-L/AM1x SPI modules.
@@ -406,7 +413,8 @@ config SPI_HISI_SFC_V3XX
 
 config SPI_NXP_FLEXSPI
 	tristate "NXP Flex SPI controller"
-	depends on ARCH_LAYERSCAPE || HAS_IOMEM
+	depends on ARCH_LAYERSCAPE || ARCH_MXC || COMPILE_TEST
+	depends on HAS_IOMEM
 	help
 	  This enables support for the Flex SPI controller in master mode.
 	  Up to four slave devices can be connected on two buses with two
@@ -777,6 +785,7 @@ config SPI_PXA2XX_PCI
 
 config SPI_ROCKCHIP
 	tristate "Rockchip SPI controller driver"
+	depends on ARCH_ROCKCHIP || COMPILE_TEST
 	help
 	  This selects a driver for Rockchip SPI controller.
 
@@ -818,7 +827,7 @@ config SPI_RSPI
 
 config SPI_QCOM_QSPI
 	tristate "QTI QSPI controller"
-	depends on ARCH_QCOM
+	depends on ARCH_QCOM || COMPILE_TEST
 	help
 	  QSPI(Quad SPI) driver for Qualcomm QSPI controller.
 
diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile
index d87cf75bee6a..28c4817a8a74 100644
--- a/drivers/spi/Makefile
+++ b/drivers/spi/Makefile
@@ -17,6 +17,7 @@ obj-$(CONFIG_SPI_LOOPBACK_TEST)		+= spi-loopback-test.o
 obj-$(CONFIG_SPI_ALTERA)		+= spi-altera-platform.o
 obj-$(CONFIG_SPI_ALTERA_CORE)		+= spi-altera-core.o
 obj-$(CONFIG_SPI_ALTERA_DFL)		+= spi-altera-dfl.o
+obj-$(CONFIG_SPI_AMLOGIC_SPIFC_A1)	+= spi-amlogic-spifc-a1.o
 obj-$(CONFIG_SPI_AR934X)		+= spi-ar934x.o
 obj-$(CONFIG_SPI_ARMADA_3700)		+= spi-armada-3700.o
 obj-$(CONFIG_SPI_ASPEED_SMC)		+= spi-aspeed-smc.o
diff --git a/drivers/spi/atmel-quadspi.c b/drivers/spi/atmel-quadspi.c
index f4632cb07495..3d1252566134 100644
--- a/drivers/spi/atmel-quadspi.c
+++ b/drivers/spi/atmel-quadspi.c
@@ -700,25 +700,33 @@ disable_pclk:
 	return err;
 }
 
-static int atmel_qspi_remove(struct platform_device *pdev)
+static void atmel_qspi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctrl = platform_get_drvdata(pdev);
 	struct atmel_qspi *aq = spi_controller_get_devdata(ctrl);
 	int ret;
 
-	ret = pm_runtime_resume_and_get(&pdev->dev);
-	if (ret < 0)
-		return ret;
-
 	spi_unregister_controller(ctrl);
-	atmel_qspi_write(QSPI_CR_QSPIDIS, aq, QSPI_CR);
+
+	ret = pm_runtime_get_sync(&pdev->dev);
+	if (ret >= 0) {
+		atmel_qspi_write(QSPI_CR_QSPIDIS, aq, QSPI_CR);
+		clk_disable(aq->qspick);
+		clk_disable(aq->pclk);
+	} else {
+		/*
+		 * atmel_qspi_runtime_{suspend,resume} just disable and enable
+		 * the two clks respectively. So after resume failed these are
+		 * off, and we skip hardware access and disabling these clks again.
+		 */
+		dev_warn(&pdev->dev, "Failed to resume device on remove\n");
+	}
+
+	clk_unprepare(aq->qspick);
+	clk_unprepare(aq->pclk);
 
 	pm_runtime_disable(&pdev->dev);
 	pm_runtime_put_noidle(&pdev->dev);
-
-	clk_disable_unprepare(aq->qspick);
-	clk_disable_unprepare(aq->pclk);
-	return 0;
 }
 
 static int __maybe_unused atmel_qspi_suspend(struct device *dev)
@@ -786,7 +794,11 @@ static int __maybe_unused atmel_qspi_runtime_resume(struct device *dev)
 	if (ret)
 		return ret;
 
-	return clk_enable(aq->qspick);
+	ret = clk_enable(aq->qspick);
+	if (ret)
+		clk_disable(aq->pclk);
+
+	return ret;
 }
 
 static const struct dev_pm_ops __maybe_unused atmel_qspi_pm_ops = {
@@ -823,7 +835,7 @@ static struct platform_driver atmel_qspi_driver = {
 		.pm	= pm_ptr(&atmel_qspi_pm_ops),
 	},
 	.probe		= atmel_qspi_probe,
-	.remove		= atmel_qspi_remove,
+	.remove_new	= atmel_qspi_remove,
 };
 module_platform_driver(atmel_qspi_driver);
 
diff --git a/drivers/spi/spi-altera-core.c b/drivers/spi/spi-altera-core.c
index 94fe6bf1b9a6..87e37f48f196 100644
--- a/drivers/spi/spi-altera-core.c
+++ b/drivers/spi/spi-altera-core.c
@@ -80,7 +80,7 @@ static void altera_spi_set_cs(struct spi_device *spi, bool is_high)
 		altr_spi_writel(hw, ALTERA_SPI_TARGET_SEL, 0);
 	} else {
 		altr_spi_writel(hw, ALTERA_SPI_TARGET_SEL,
-				BIT(spi->chip_select));
+				BIT(spi_get_chipselect(spi, 0)));
 		hw->imr |= ALTERA_SPI_CONTROL_SSO_MSK;
 		altr_spi_writel(hw, ALTERA_SPI_CONTROL, hw->imr);
 	}
diff --git a/drivers/spi/spi-amd.c b/drivers/spi/spi-amd.c
index bfc3ab5f39ea..fecead757a3c 100644
--- a/drivers/spi/spi-amd.c
+++ b/drivers/spi/spi-amd.c
@@ -347,7 +347,7 @@ fin_msg:
 	case AMD_SPI_V1:
 		break;
 	case AMD_SPI_V2:
-		amd_spi_clear_chip(amd_spi, message->spi->chip_select);
+		amd_spi_clear_chip(amd_spi, spi_get_chipselect(message->spi, 0));
 		break;
 	default:
 		return -ENODEV;
@@ -364,7 +364,7 @@ static int amd_spi_master_transfer(struct spi_master *master,
 	struct amd_spi *amd_spi = spi_master_get_devdata(master);
 	struct spi_device *spi = msg->spi;
 
-	amd_spi_select_chip(amd_spi, spi->chip_select);
+	amd_spi_select_chip(amd_spi, spi_get_chipselect(spi, 0));
 
 	/*
 	 * Extract spi_transfers from the spi message and
diff --git a/drivers/spi/spi-amlogic-spifc-a1.c b/drivers/spi/spi-amlogic-spifc-a1.c
new file mode 100644
index 000000000000..3c4224c38399
--- /dev/null
+++ b/drivers/spi/spi-amlogic-spifc-a1.c
@@ -0,0 +1,456 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for Amlogic A1 SPI flash controller (SPIFC)
+ *
+ * Copyright (c) 2023, SberDevices. All Rights Reserved.
+ *
+ * Author: Martin Kurbanov <mmkurbanov@sberdevices.ru>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
+#include <linux/types.h>
+
+#define SPIFC_A1_AHB_CTRL_REG		0x0
+#define SPIFC_A1_AHB_BUS_EN		BIT(31)
+
+#define SPIFC_A1_USER_CTRL0_REG		0x200
+#define SPIFC_A1_USER_REQUEST_ENABLE	BIT(31)
+#define SPIFC_A1_USER_REQUEST_FINISH	BIT(30)
+#define SPIFC_A1_USER_DATA_UPDATED	BIT(0)
+
+#define SPIFC_A1_USER_CTRL1_REG		0x204
+#define SPIFC_A1_USER_CMD_ENABLE	BIT(30)
+#define SPIFC_A1_USER_CMD_MODE		GENMASK(29, 28)
+#define SPIFC_A1_USER_CMD_CODE		GENMASK(27, 20)
+#define SPIFC_A1_USER_ADDR_ENABLE	BIT(19)
+#define SPIFC_A1_USER_ADDR_MODE		GENMASK(18, 17)
+#define SPIFC_A1_USER_ADDR_BYTES	GENMASK(16, 15)
+#define SPIFC_A1_USER_DOUT_ENABLE	BIT(14)
+#define SPIFC_A1_USER_DOUT_MODE		GENMASK(11, 10)
+#define SPIFC_A1_USER_DOUT_BYTES	GENMASK(9, 0)
+
+#define SPIFC_A1_USER_CTRL2_REG		0x208
+#define SPIFC_A1_USER_DUMMY_ENABLE	BIT(31)
+#define SPIFC_A1_USER_DUMMY_MODE	GENMASK(30, 29)
+#define SPIFC_A1_USER_DUMMY_CLK_SYCLES	GENMASK(28, 23)
+
+#define SPIFC_A1_USER_CTRL3_REG		0x20c
+#define SPIFC_A1_USER_DIN_ENABLE	BIT(31)
+#define SPIFC_A1_USER_DIN_MODE		GENMASK(28, 27)
+#define SPIFC_A1_USER_DIN_BYTES		GENMASK(25, 16)
+
+#define SPIFC_A1_USER_ADDR_REG		0x210
+
+#define SPIFC_A1_AHB_REQ_CTRL_REG	0x214
+#define SPIFC_A1_AHB_REQ_ENABLE		BIT(31)
+
+#define SPIFC_A1_ACTIMING0_REG		(0x0088 << 2)
+#define SPIFC_A1_TSLCH			GENMASK(31, 30)
+#define SPIFC_A1_TCLSH			GENMASK(29, 28)
+#define SPIFC_A1_TSHWL			GENMASK(20, 16)
+#define SPIFC_A1_TSHSL2			GENMASK(15, 12)
+#define SPIFC_A1_TSHSL1			GENMASK(11, 8)
+#define SPIFC_A1_TWHSL			GENMASK(7, 0)
+
+#define SPIFC_A1_DBUF_CTRL_REG		0x240
+#define SPIFC_A1_DBUF_DIR		BIT(31)
+#define SPIFC_A1_DBUF_AUTO_UPDATE_ADDR	BIT(30)
+#define SPIFC_A1_DBUF_ADDR		GENMASK(7, 0)
+
+#define SPIFC_A1_DBUF_DATA_REG		0x244
+
+#define SPIFC_A1_USER_DBUF_ADDR_REG	0x248
+
+#define SPIFC_A1_BUFFER_SIZE		512
+
+#define SPIFC_A1_MAX_HZ			200000000
+#define SPIFC_A1_MIN_HZ			1000000
+
+#define SPIFC_A1_USER_CMD(op) ( \
+	SPIFC_A1_USER_CMD_ENABLE | \
+	FIELD_PREP(SPIFC_A1_USER_CMD_CODE, (op)->cmd.opcode) | \
+	FIELD_PREP(SPIFC_A1_USER_CMD_MODE, ilog2((op)->cmd.buswidth)))
+
+#define SPIFC_A1_USER_ADDR(op) ( \
+	SPIFC_A1_USER_ADDR_ENABLE | \
+	FIELD_PREP(SPIFC_A1_USER_ADDR_MODE, ilog2((op)->addr.buswidth)) | \
+	FIELD_PREP(SPIFC_A1_USER_ADDR_BYTES, (op)->addr.nbytes - 1))
+
+#define SPIFC_A1_USER_DUMMY(op) ( \
+	SPIFC_A1_USER_DUMMY_ENABLE | \
+	FIELD_PREP(SPIFC_A1_USER_DUMMY_MODE, ilog2((op)->dummy.buswidth)) | \
+	FIELD_PREP(SPIFC_A1_USER_DUMMY_CLK_SYCLES, (op)->dummy.nbytes << 3))
+
+#define SPIFC_A1_TSLCH_VAL	FIELD_PREP(SPIFC_A1_TSLCH, 1)
+#define SPIFC_A1_TCLSH_VAL	FIELD_PREP(SPIFC_A1_TCLSH, 1)
+#define SPIFC_A1_TSHWL_VAL	FIELD_PREP(SPIFC_A1_TSHWL, 7)
+#define SPIFC_A1_TSHSL2_VAL	FIELD_PREP(SPIFC_A1_TSHSL2, 7)
+#define SPIFC_A1_TSHSL1_VAL	FIELD_PREP(SPIFC_A1_TSHSL1, 7)
+#define SPIFC_A1_TWHSL_VAL	FIELD_PREP(SPIFC_A1_TWHSL, 2)
+#define SPIFC_A1_ACTIMING0_VAL	(SPIFC_A1_TSLCH_VAL | SPIFC_A1_TCLSH_VAL | \
+				 SPIFC_A1_TSHWL_VAL | SPIFC_A1_TSHSL2_VAL | \
+				 SPIFC_A1_TSHSL1_VAL | SPIFC_A1_TWHSL_VAL)
+
+struct amlogic_spifc_a1 {
+	struct spi_controller *ctrl;
+	struct clk *clk;
+	struct device *dev;
+	void __iomem *base;
+};
+
+static int amlogic_spifc_a1_request(struct amlogic_spifc_a1 *spifc, bool read)
+{
+	u32 mask = SPIFC_A1_USER_REQUEST_FINISH |
+		   (read ? SPIFC_A1_USER_DATA_UPDATED : 0);
+	u32 val;
+
+	writel(SPIFC_A1_USER_REQUEST_ENABLE,
+	       spifc->base + SPIFC_A1_USER_CTRL0_REG);
+
+	return readl_poll_timeout(spifc->base + SPIFC_A1_USER_CTRL0_REG,
+				  val, (val & mask) == mask, 0,
+				  200 * USEC_PER_MSEC);
+}
+
+static void amlogic_spifc_a1_drain_buffer(struct amlogic_spifc_a1 *spifc,
+					  char *buf, u32 len)
+{
+	u32 data;
+	const u32 count = len / sizeof(data);
+	const u32 pad = len % sizeof(data);
+
+	writel(SPIFC_A1_DBUF_AUTO_UPDATE_ADDR,
+	       spifc->base + SPIFC_A1_DBUF_CTRL_REG);
+	ioread32_rep(spifc->base + SPIFC_A1_DBUF_DATA_REG, buf, count);
+
+	if (pad) {
+		data = readl(spifc->base + SPIFC_A1_DBUF_DATA_REG);
+		memcpy(buf + len - pad, &data, pad);
+	}
+}
+
+static void amlogic_spifc_a1_fill_buffer(struct amlogic_spifc_a1 *spifc,
+					 const char *buf, u32 len)
+{
+	u32 data;
+	const u32 count = len / sizeof(data);
+	const u32 pad = len % sizeof(data);
+
+	writel(SPIFC_A1_DBUF_DIR | SPIFC_A1_DBUF_AUTO_UPDATE_ADDR,
+	       spifc->base + SPIFC_A1_DBUF_CTRL_REG);
+	iowrite32_rep(spifc->base + SPIFC_A1_DBUF_DATA_REG, buf, count);
+
+	if (pad) {
+		memcpy(&data, buf + len - pad, pad);
+		writel(data, spifc->base + SPIFC_A1_DBUF_DATA_REG);
+	}
+}
+
+static void amlogic_spifc_a1_user_init(struct amlogic_spifc_a1 *spifc)
+{
+	writel(0, spifc->base + SPIFC_A1_USER_CTRL0_REG);
+	writel(0, spifc->base + SPIFC_A1_USER_CTRL1_REG);
+	writel(0, spifc->base + SPIFC_A1_USER_CTRL2_REG);
+	writel(0, spifc->base + SPIFC_A1_USER_CTRL3_REG);
+}
+
+static void amlogic_spifc_a1_set_cmd(struct amlogic_spifc_a1 *spifc,
+				     u32 cmd_cfg)
+{
+	u32 val;
+
+	val = readl(spifc->base + SPIFC_A1_USER_CTRL1_REG);
+	val &= ~(SPIFC_A1_USER_CMD_MODE | SPIFC_A1_USER_CMD_CODE);
+	val |= cmd_cfg;
+	writel(val, spifc->base + SPIFC_A1_USER_CTRL1_REG);
+}
+
+static void amlogic_spifc_a1_set_addr(struct amlogic_spifc_a1 *spifc, u32 addr,
+				      u32 addr_cfg)
+{
+	u32 val;
+
+	writel(addr, spifc->base + SPIFC_A1_USER_ADDR_REG);
+
+	val = readl(spifc->base + SPIFC_A1_USER_CTRL1_REG);
+	val &= ~(SPIFC_A1_USER_ADDR_MODE | SPIFC_A1_USER_ADDR_BYTES);
+	val |= addr_cfg;
+	writel(val, spifc->base + SPIFC_A1_USER_CTRL1_REG);
+}
+
+static void amlogic_spifc_a1_set_dummy(struct amlogic_spifc_a1 *spifc,
+				       u32 dummy_cfg)
+{
+	u32 val = readl(spifc->base + SPIFC_A1_USER_CTRL2_REG);
+
+	val &= ~(SPIFC_A1_USER_DUMMY_MODE | SPIFC_A1_USER_DUMMY_CLK_SYCLES);
+	val |= dummy_cfg;
+	writel(val, spifc->base + SPIFC_A1_USER_CTRL2_REG);
+}
+
+static int amlogic_spifc_a1_read(struct amlogic_spifc_a1 *spifc, void *buf,
+				 u32 size, u32 mode)
+{
+	u32 val = readl(spifc->base + SPIFC_A1_USER_CTRL3_REG);
+	int ret;
+
+	val &= ~(SPIFC_A1_USER_DIN_MODE | SPIFC_A1_USER_DIN_BYTES);
+	val |= SPIFC_A1_USER_DIN_ENABLE;
+	val |= FIELD_PREP(SPIFC_A1_USER_DIN_MODE, mode);
+	val |= FIELD_PREP(SPIFC_A1_USER_DIN_BYTES, size);
+	writel(val, spifc->base + SPIFC_A1_USER_CTRL3_REG);
+
+	ret = amlogic_spifc_a1_request(spifc, true);
+	if (!ret)
+		amlogic_spifc_a1_drain_buffer(spifc, buf, size);
+
+	return ret;
+}
+
+static int amlogic_spifc_a1_write(struct amlogic_spifc_a1 *spifc,
+				  const void *buf, u32 size, u32 mode)
+{
+	u32 val;
+
+	amlogic_spifc_a1_fill_buffer(spifc, buf, size);
+
+	val = readl(spifc->base + SPIFC_A1_USER_CTRL1_REG);
+	val &= ~(SPIFC_A1_USER_DOUT_MODE | SPIFC_A1_USER_DOUT_BYTES);
+	val |= FIELD_PREP(SPIFC_A1_USER_DOUT_MODE, mode);
+	val |= FIELD_PREP(SPIFC_A1_USER_DOUT_BYTES, size);
+	val |= SPIFC_A1_USER_DOUT_ENABLE;
+	writel(val, spifc->base + SPIFC_A1_USER_CTRL1_REG);
+
+	return amlogic_spifc_a1_request(spifc, false);
+}
+
+static int amlogic_spifc_a1_exec_op(struct spi_mem *mem,
+				    const struct spi_mem_op *op)
+{
+	struct amlogic_spifc_a1 *spifc =
+		spi_controller_get_devdata(mem->spi->controller);
+	size_t off, nbytes = op->data.nbytes;
+	u32 cmd_cfg, addr_cfg, dummy_cfg, dmode;
+	int ret;
+
+	amlogic_spifc_a1_user_init(spifc);
+
+	cmd_cfg = SPIFC_A1_USER_CMD(op);
+	amlogic_spifc_a1_set_cmd(spifc, cmd_cfg);
+
+	if (op->addr.nbytes) {
+		addr_cfg = SPIFC_A1_USER_ADDR(op);
+		amlogic_spifc_a1_set_addr(spifc, op->addr.val, addr_cfg);
+	}
+
+	if (op->dummy.nbytes) {
+		dummy_cfg = SPIFC_A1_USER_DUMMY(op);
+		amlogic_spifc_a1_set_dummy(spifc, dummy_cfg);
+	}
+
+	if (!op->data.nbytes)
+		return amlogic_spifc_a1_request(spifc, false);
+
+	dmode = ilog2(op->data.buswidth);
+	off = 0;
+
+	do {
+		size_t block_size = min_t(size_t, nbytes, SPIFC_A1_BUFFER_SIZE);
+
+		amlogic_spifc_a1_set_cmd(spifc, cmd_cfg);
+
+		if (op->addr.nbytes)
+			amlogic_spifc_a1_set_addr(spifc, op->addr.val + off,
+						  addr_cfg);
+
+		if (op->dummy.nbytes)
+			amlogic_spifc_a1_set_dummy(spifc, dummy_cfg);
+
+		writel(0, spifc->base + SPIFC_A1_USER_DBUF_ADDR_REG);
+
+		if (op->data.dir == SPI_MEM_DATA_IN)
+			ret = amlogic_spifc_a1_read(spifc,
+						    op->data.buf.in + off,
+						    block_size, dmode);
+		else
+			ret = amlogic_spifc_a1_write(spifc,
+						     op->data.buf.out + off,
+						     block_size, dmode);
+
+		nbytes -= block_size;
+		off += block_size;
+	} while (nbytes != 0 && !ret);
+
+	return ret;
+}
+
+static void amlogic_spifc_a1_hw_init(struct amlogic_spifc_a1 *spifc)
+{
+	u32 regv;
+
+	regv = readl(spifc->base + SPIFC_A1_AHB_REQ_CTRL_REG);
+	regv &= ~(SPIFC_A1_AHB_REQ_ENABLE);
+	writel(regv, spifc->base + SPIFC_A1_AHB_REQ_CTRL_REG);
+
+	regv = readl(spifc->base + SPIFC_A1_AHB_CTRL_REG);
+	regv &= ~(SPIFC_A1_AHB_BUS_EN);
+	writel(regv, spifc->base + SPIFC_A1_AHB_CTRL_REG);
+
+	writel(SPIFC_A1_ACTIMING0_VAL, spifc->base + SPIFC_A1_ACTIMING0_REG);
+
+	writel(0, spifc->base + SPIFC_A1_USER_DBUF_ADDR_REG);
+}
+
+static const struct spi_controller_mem_ops amlogic_spifc_a1_mem_ops = {
+	.exec_op = amlogic_spifc_a1_exec_op,
+};
+
+static int amlogic_spifc_a1_probe(struct platform_device *pdev)
+{
+	struct spi_controller *ctrl;
+	struct amlogic_spifc_a1 *spifc;
+	int ret;
+
+	ctrl = devm_spi_alloc_master(&pdev->dev, sizeof(*spifc));
+	if (!ctrl)
+		return -ENOMEM;
+
+	spifc = spi_controller_get_devdata(ctrl);
+	platform_set_drvdata(pdev, spifc);
+
+	spifc->dev = &pdev->dev;
+	spifc->ctrl = ctrl;
+
+	spifc->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(spifc->base))
+		return PTR_ERR(spifc->base);
+
+	spifc->clk = devm_clk_get_enabled(spifc->dev, NULL);
+	if (IS_ERR(spifc->clk))
+		return dev_err_probe(spifc->dev, PTR_ERR(spifc->clk),
+				     "unable to get clock\n");
+
+	amlogic_spifc_a1_hw_init(spifc);
+
+	pm_runtime_set_autosuspend_delay(spifc->dev, 500);
+	pm_runtime_use_autosuspend(spifc->dev);
+	devm_pm_runtime_enable(spifc->dev);
+
+	ctrl->num_chipselect = 1;
+	ctrl->dev.of_node = pdev->dev.of_node;
+	ctrl->bits_per_word_mask = SPI_BPW_MASK(8);
+	ctrl->auto_runtime_pm = true;
+	ctrl->mem_ops = &amlogic_spifc_a1_mem_ops;
+	ctrl->min_speed_hz = SPIFC_A1_MIN_HZ;
+	ctrl->max_speed_hz = SPIFC_A1_MAX_HZ;
+	ctrl->mode_bits = (SPI_RX_DUAL | SPI_TX_DUAL |
+			   SPI_RX_QUAD | SPI_TX_QUAD);
+
+	ret = devm_spi_register_controller(spifc->dev, ctrl);
+	if (ret)
+		return dev_err_probe(spifc->dev, ret,
+				     "failed to register spi controller\n");
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int amlogic_spifc_a1_suspend(struct device *dev)
+{
+	struct amlogic_spifc_a1 *spifc = dev_get_drvdata(dev);
+	int ret;
+
+	ret = spi_controller_suspend(spifc->ctrl);
+	if (ret)
+		return ret;
+
+	if (!pm_runtime_suspended(dev))
+		clk_disable_unprepare(spifc->clk);
+
+	return 0;
+}
+
+static int amlogic_spifc_a1_resume(struct device *dev)
+{
+	struct amlogic_spifc_a1 *spifc = dev_get_drvdata(dev);
+	int ret = 0;
+
+	if (!pm_runtime_suspended(dev)) {
+		ret = clk_prepare_enable(spifc->clk);
+		if (ret)
+			return ret;
+	}
+
+	amlogic_spifc_a1_hw_init(spifc);
+
+	ret = spi_controller_resume(spifc->ctrl);
+	if (ret)
+		clk_disable_unprepare(spifc->clk);
+
+	return ret;
+}
+#endif /* CONFIG_PM_SLEEP */
+
+#ifdef CONFIG_PM
+static int amlogic_spifc_a1_runtime_suspend(struct device *dev)
+{
+	struct amlogic_spifc_a1 *spifc = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(spifc->clk);
+
+	return 0;
+}
+
+static int amlogic_spifc_a1_runtime_resume(struct device *dev)
+{
+	struct amlogic_spifc_a1 *spifc = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(spifc->clk);
+	if (!ret)
+		amlogic_spifc_a1_hw_init(spifc);
+
+	return ret;
+}
+#endif /* CONFIG_PM */
+
+static const struct dev_pm_ops amlogic_spifc_a1_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(amlogic_spifc_a1_suspend,
+				amlogic_spifc_a1_resume)
+	SET_RUNTIME_PM_OPS(amlogic_spifc_a1_runtime_suspend,
+			   amlogic_spifc_a1_runtime_resume,
+			   NULL)
+};
+
+#ifdef CONFIG_OF
+static const struct of_device_id amlogic_spifc_a1_dt_match[] = {
+	{ .compatible = "amlogic,a1-spifc", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, amlogic_spifc_a1_dt_match);
+#endif /* CONFIG_OF */
+
+static struct platform_driver amlogic_spifc_a1_driver = {
+	.probe	= amlogic_spifc_a1_probe,
+	.driver	= {
+		.name		= "amlogic-spifc-a1",
+		.of_match_table	= of_match_ptr(amlogic_spifc_a1_dt_match),
+		.pm		= &amlogic_spifc_a1_pm_ops,
+	},
+};
+module_platform_driver(amlogic_spifc_a1_driver);
+
+MODULE_AUTHOR("Martin Kurbanov <mmkurbanov@sberdevices.ru>");
+MODULE_DESCRIPTION("Amlogic A1 SPIFC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/spi/spi-ar934x.c b/drivers/spi/spi-ar934x.c
index 4a6ecaa0a9c9..9dcada8c4cb9 100644
--- a/drivers/spi/spi-ar934x.c
+++ b/drivers/spi/spi-ar934x.c
@@ -125,7 +125,7 @@ static int ar934x_spi_transfer_one_message(struct spi_controller *ctlr,
 				iowrite32(reg, sp->base + AR934X_SPI_DATAOUT);
 			}
 
-			reg = AR934X_SPI_SHIFT_VAL(spi->chip_select, term,
+			reg = AR934X_SPI_SHIFT_VAL(spi_get_chipselect(spi, 0), term,
 						   trx_cur * 8);
 			iowrite32(reg, sp->base + AR934X_SPI_REG_SHIFT_CTRL);
 			stat = readl_poll_timeout(
@@ -220,7 +220,7 @@ err_clk_disable:
 	return ret;
 }
 
-static int ar934x_spi_remove(struct platform_device *pdev)
+static void ar934x_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr;
 	struct ar934x_spi *sp;
@@ -230,8 +230,6 @@ static int ar934x_spi_remove(struct platform_device *pdev)
 
 	spi_unregister_controller(ctlr);
 	clk_disable_unprepare(sp->clk);
-
-	return 0;
 }
 
 static struct platform_driver ar934x_spi_driver = {
@@ -240,7 +238,7 @@ static struct platform_driver ar934x_spi_driver = {
 		.of_match_table = ar934x_spi_match,
 	},
 	.probe = ar934x_spi_probe,
-	.remove = ar934x_spi_remove,
+	.remove_new = ar934x_spi_remove,
 };
 
 module_platform_driver(ar934x_spi_driver);
diff --git a/drivers/spi/spi-armada-3700.c b/drivers/spi/spi-armada-3700.c
index 4d554b948d71..a7fb7c94e70e 100644
--- a/drivers/spi/spi-armada-3700.c
+++ b/drivers/spi/spi-armada-3700.c
@@ -437,9 +437,9 @@ static void a3700_spi_set_cs(struct spi_device *spi, bool enable)
 	struct a3700_spi *a3700_spi = spi_controller_get_devdata(spi->controller);
 
 	if (!enable)
-		a3700_spi_activate_cs(a3700_spi, spi->chip_select);
+		a3700_spi_activate_cs(a3700_spi, spi_get_chipselect(spi, 0));
 	else
-		a3700_spi_deactivate_cs(a3700_spi, spi->chip_select);
+		a3700_spi_deactivate_cs(a3700_spi, spi_get_chipselect(spi, 0));
 }
 
 static void a3700_spi_header_set(struct a3700_spi *a3700_spi)
@@ -908,14 +908,12 @@ out:
 	return ret;
 }
 
-static int a3700_spi_remove(struct platform_device *pdev)
+static void a3700_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct a3700_spi *spi = spi_controller_get_devdata(host);
 
 	clk_unprepare(spi->clk);
-
-	return 0;
 }
 
 static struct platform_driver a3700_spi_driver = {
@@ -924,7 +922,7 @@ static struct platform_driver a3700_spi_driver = {
 		.of_match_table = of_match_ptr(a3700_spi_dt_ids),
 	},
 	.probe		= a3700_spi_probe,
-	.remove		= a3700_spi_remove,
+	.remove_new	= a3700_spi_remove,
 };
 
 module_platform_driver(a3700_spi_driver);
diff --git a/drivers/spi/spi-aspeed-smc.c b/drivers/spi/spi-aspeed-smc.c
index 873ff2cf72c9..e75b0d51f06a 100644
--- a/drivers/spi/spi-aspeed-smc.c
+++ b/drivers/spi/spi-aspeed-smc.c
@@ -296,7 +296,7 @@ static const struct aspeed_spi_data ast2400_spi_data;
 static int do_aspeed_spi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
 {
 	struct aspeed_spi *aspi = spi_controller_get_devdata(mem->spi->master);
-	struct aspeed_spi_chip *chip = &aspi->chips[mem->spi->chip_select];
+	struct aspeed_spi_chip *chip = &aspi->chips[spi_get_chipselect(mem->spi, 0)];
 	u32 addr_mode, addr_mode_backup;
 	u32 ctl_val;
 	int ret = 0;
@@ -377,7 +377,8 @@ static const char *aspeed_spi_get_name(struct spi_mem *mem)
 	struct aspeed_spi *aspi = spi_controller_get_devdata(mem->spi->master);
 	struct device *dev = aspi->dev;
 
-	return devm_kasprintf(dev, GFP_KERNEL, "%s.%d", dev_name(dev), mem->spi->chip_select);
+	return devm_kasprintf(dev, GFP_KERNEL, "%s.%d", dev_name(dev),
+			      spi_get_chipselect(mem->spi, 0));
 }
 
 struct aspeed_spi_window {
@@ -553,7 +554,7 @@ static int aspeed_spi_do_calibration(struct aspeed_spi_chip *chip);
 static int aspeed_spi_dirmap_create(struct spi_mem_dirmap_desc *desc)
 {
 	struct aspeed_spi *aspi = spi_controller_get_devdata(desc->mem->spi->master);
-	struct aspeed_spi_chip *chip = &aspi->chips[desc->mem->spi->chip_select];
+	struct aspeed_spi_chip *chip = &aspi->chips[spi_get_chipselect(desc->mem->spi, 0)];
 	struct spi_mem_op *op = &desc->info.op_tmpl;
 	u32 ctl_val;
 	int ret = 0;
@@ -620,7 +621,7 @@ static ssize_t aspeed_spi_dirmap_read(struct spi_mem_dirmap_desc *desc,
 				      u64 offset, size_t len, void *buf)
 {
 	struct aspeed_spi *aspi = spi_controller_get_devdata(desc->mem->spi->master);
-	struct aspeed_spi_chip *chip = &aspi->chips[desc->mem->spi->chip_select];
+	struct aspeed_spi_chip *chip = &aspi->chips[spi_get_chipselect(desc->mem->spi, 0)];
 
 	/* Switch to USER command mode if mapping window is too small */
 	if (chip->ahb_window_size < offset + len) {
@@ -670,7 +671,7 @@ static int aspeed_spi_setup(struct spi_device *spi)
 {
 	struct aspeed_spi *aspi = spi_controller_get_devdata(spi->master);
 	const struct aspeed_spi_data *data = aspi->data;
-	unsigned int cs = spi->chip_select;
+	unsigned int cs = spi_get_chipselect(spi, 0);
 	struct aspeed_spi_chip *chip = &aspi->chips[cs];
 
 	chip->aspi = aspi;
@@ -697,7 +698,7 @@ static int aspeed_spi_setup(struct spi_device *spi)
 static void aspeed_spi_cleanup(struct spi_device *spi)
 {
 	struct aspeed_spi *aspi = spi_controller_get_devdata(spi->master);
-	unsigned int cs = spi->chip_select;
+	unsigned int cs = spi_get_chipselect(spi, 0);
 
 	aspeed_spi_chip_enable(aspi, cs, false);
 
@@ -787,13 +788,12 @@ disable_clk:
 	return ret;
 }
 
-static int aspeed_spi_remove(struct platform_device *pdev)
+static void aspeed_spi_remove(struct platform_device *pdev)
 {
 	struct aspeed_spi *aspi = platform_get_drvdata(pdev);
 
 	aspeed_spi_enable(aspi, false);
 	clk_disable_unprepare(aspi->clk);
-	return 0;
 }
 
 /*
@@ -1201,7 +1201,7 @@ MODULE_DEVICE_TABLE(of, aspeed_spi_matches);
 
 static struct platform_driver aspeed_spi_driver = {
 	.probe			= aspeed_spi_probe,
-	.remove			= aspeed_spi_remove,
+	.remove_new		= aspeed_spi_remove,
 	.driver	= {
 		.name		= DEVICE_NAME,
 		.of_match_table = aspeed_spi_matches,
diff --git a/drivers/spi/spi-at91-usart.c b/drivers/spi/spi-at91-usart.c
index fab9d223e24a..7854d9790fe9 100644
--- a/drivers/spi/spi-at91-usart.c
+++ b/drivers/spi/spi-at91-usart.c
@@ -390,7 +390,7 @@ static int at91_usart_spi_setup(struct spi_device *spi)
 
 	dev_dbg(&spi->dev,
 		"setup: bpw %u mode 0x%x -> mr %d %08x\n",
-		spi->bits_per_word, spi->mode, spi->chip_select, mr);
+		spi->bits_per_word, spi->mode, spi_get_chipselect(spi, 0), mr);
 
 	return 0;
 }
@@ -647,15 +647,13 @@ __maybe_unused static int at91_usart_spi_resume(struct device *dev)
 	return spi_controller_resume(ctrl);
 }
 
-static int at91_usart_spi_remove(struct platform_device *pdev)
+static void at91_usart_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr = platform_get_drvdata(pdev);
 	struct at91_usart_spi *aus = spi_controller_get_devdata(ctlr);
 
 	at91_usart_spi_release_dma(ctlr);
 	clk_disable_unprepare(aus->clk);
-
-	return 0;
 }
 
 static const struct dev_pm_ops at91_usart_spi_pm_ops = {
@@ -670,7 +668,7 @@ static struct platform_driver at91_usart_spi_driver = {
 		.pm = &at91_usart_spi_pm_ops,
 	},
 	.probe = at91_usart_spi_probe,
-	.remove = at91_usart_spi_remove,
+	.remove_new = at91_usart_spi_remove,
 };
 
 module_platform_driver(at91_usart_spi_driver);
diff --git a/drivers/spi/spi-ath79.c b/drivers/spi/spi-ath79.c
index 795e88dbef1b..d3dd21386f12 100644
--- a/drivers/spi/spi-ath79.c
+++ b/drivers/spi/spi-ath79.c
@@ -71,7 +71,7 @@ static void ath79_spi_chipselect(struct spi_device *spi, int is_active)
 {
 	struct ath79_spi *sp = ath79_spidev_to_sp(spi);
 	int cs_high = (spi->mode & SPI_CS_HIGH) ? is_active : !is_active;
-	u32 cs_bit = AR71XX_SPI_IOC_CS(spi->chip_select);
+	u32 cs_bit = AR71XX_SPI_IOC_CS(spi_get_chipselect(spi, 0));
 
 	if (cs_high)
 		sp->ioc_base |= cs_bit;
@@ -140,7 +140,7 @@ static int ath79_exec_mem_op(struct spi_mem *mem,
 	struct ath79_spi *sp = ath79_spidev_to_sp(mem->spi);
 
 	/* Ensures that reading is performed on device connected to hardware cs0 */
-	if (mem->spi->chip_select || mem->spi->cs_gpiod)
+	if (spi_get_chipselect(mem->spi, 0) || spi_get_csgpiod(mem->spi, 0))
 		return -ENOTSUPP;
 
 	/* Only use for fast-read op. */
@@ -237,7 +237,7 @@ err_put_host:
 	return ret;
 }
 
-static int ath79_spi_remove(struct platform_device *pdev)
+static void ath79_spi_remove(struct platform_device *pdev)
 {
 	struct ath79_spi *sp = platform_get_drvdata(pdev);
 
@@ -245,8 +245,6 @@ static int ath79_spi_remove(struct platform_device *pdev)
 	ath79_spi_disable(sp);
 	clk_disable_unprepare(sp->clk);
 	spi_controller_put(sp->bitbang.master);
-
-	return 0;
 }
 
 static void ath79_spi_shutdown(struct platform_device *pdev)
@@ -262,7 +260,7 @@ MODULE_DEVICE_TABLE(of, ath79_spi_of_match);
 
 static struct platform_driver ath79_spi_driver = {
 	.probe		= ath79_spi_probe,
-	.remove		= ath79_spi_remove,
+	.remove_new	= ath79_spi_remove,
 	.shutdown	= ath79_spi_shutdown,
 	.driver		= {
 		.name	= DRV_NAME,
diff --git a/drivers/spi/spi-atmel.c b/drivers/spi/spi-atmel.c
index 5c5678f065f3..7f06305e16cb 100644
--- a/drivers/spi/spi-atmel.c
+++ b/drivers/spi/spi-atmel.c
@@ -327,10 +327,10 @@ static void cs_activate(struct atmel_spi *as, struct spi_device *spi)
 	int chip_select;
 	u32 mr;
 
-	if (spi->cs_gpiod)
+	if (spi_get_csgpiod(spi, 0))
 		chip_select = as->native_cs_for_gpio;
 	else
-		chip_select = spi->chip_select;
+		chip_select = spi_get_chipselect(spi, 0);
 
 	if (atmel_spi_is_v2(as)) {
 		spi_writel(as, CSR0 + 4 * chip_select, asd->csr);
@@ -378,10 +378,10 @@ static void cs_deactivate(struct atmel_spi *as, struct spi_device *spi)
 	int chip_select;
 	u32 mr;
 
-	if (spi->cs_gpiod)
+	if (spi_get_csgpiod(spi, 0))
 		chip_select = as->native_cs_for_gpio;
 	else
-		chip_select = spi->chip_select;
+		chip_select = spi_get_chipselect(spi, 0);
 
 	/* only deactivate *this* device; sometimes transfers to
 	 * another device may be active when this routine is called.
@@ -394,7 +394,7 @@ static void cs_deactivate(struct atmel_spi *as, struct spi_device *spi)
 
 	dev_dbg(&spi->dev, "DEactivate NPCS, mr %08x\n", mr);
 
-	if (!spi->cs_gpiod)
+	if (!spi_get_csgpiod(spi, 0))
 		spi_writel(as, CR, SPI_BIT(LASTXFER));
 }
 
@@ -800,10 +800,10 @@ static int atmel_spi_set_xfer_speed(struct atmel_spi *as,
 	unsigned long		bus_hz;
 	int chip_select;
 
-	if (spi->cs_gpiod)
+	if (spi_get_csgpiod(spi, 0))
 		chip_select = as->native_cs_for_gpio;
 	else
-		chip_select = spi->chip_select;
+		chip_select = spi_get_chipselect(spi, 0);
 
 	/* v1 chips start out at half the peripheral bus speed. */
 	bus_hz = as->spi_clk;
@@ -1189,7 +1189,7 @@ static int atmel_spi_setup(struct spi_device *spi)
 	as = spi_controller_get_devdata(spi->controller);
 
 	/* see notes above re chipselect */
-	if (!spi->cs_gpiod && (spi->mode & SPI_CS_HIGH)) {
+	if (!spi_get_csgpiod(spi, 0) && (spi->mode & SPI_CS_HIGH)) {
 		dev_warn(&spi->dev, "setup: non GPIO CS can't be active-high\n");
 		return -EINVAL;
 	}
@@ -1201,16 +1201,16 @@ static int atmel_spi_setup(struct spi_device *spi)
 	 */
 	initialize_native_cs_for_gpio(as);
 
-	if (spi->cs_gpiod && as->native_cs_free) {
+	if (spi_get_csgpiod(spi, 0) && as->native_cs_free) {
 		dev_err(&spi->dev,
 			"No native CS available to support this GPIO CS\n");
 		return -EBUSY;
 	}
 
-	if (spi->cs_gpiod)
+	if (spi_get_csgpiod(spi, 0))
 		chip_select = as->native_cs_for_gpio;
 	else
-		chip_select = spi->chip_select;
+		chip_select = spi_get_chipselect(spi, 0);
 
 	csr = SPI_BF(BITS, bits - 8);
 	if (spi->mode & SPI_CPOL)
@@ -1218,7 +1218,7 @@ static int atmel_spi_setup(struct spi_device *spi)
 	if (!(spi->mode & SPI_CPHA))
 		csr |= SPI_BIT(NCPHA);
 
-	if (!spi->cs_gpiod)
+	if (!spi_get_csgpiod(spi, 0))
 		csr |= SPI_BIT(CSAAT);
 	csr |= SPI_BF(DLYBS, 0);
 
@@ -1244,7 +1244,7 @@ static int atmel_spi_setup(struct spi_device *spi)
 
 	dev_dbg(&spi->dev,
 		"setup: bpw %u mode 0x%x -> csr%d %08x\n",
-		bits, spi->mode, spi->chip_select, csr);
+		bits, spi->mode, spi_get_chipselect(spi, 0), csr);
 
 	if (!atmel_spi_is_v2(as))
 		spi_writel(as, CSR0 + 4 * chip_select, csr);
@@ -1596,7 +1596,7 @@ out_unmap_regs:
 	return ret;
 }
 
-static int atmel_spi_remove(struct platform_device *pdev)
+static void atmel_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller	*host = platform_get_drvdata(pdev);
 	struct atmel_spi	*as = spi_controller_get_devdata(host);
@@ -1627,8 +1627,6 @@ static int atmel_spi_remove(struct platform_device *pdev)
 
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static int atmel_spi_runtime_suspend(struct device *dev)
@@ -1712,7 +1710,7 @@ static struct platform_driver atmel_spi_driver = {
 		.of_match_table	= atmel_spi_dt_ids,
 	},
 	.probe		= atmel_spi_probe,
-	.remove		= atmel_spi_remove,
+	.remove_new	= atmel_spi_remove,
 };
 module_platform_driver(atmel_spi_driver);
 
diff --git a/drivers/spi/spi-au1550.c b/drivers/spi/spi-au1550.c
index e008761298da..0b57e6afce0f 100644
--- a/drivers/spi/spi-au1550.c
+++ b/drivers/spi/spi-au1550.c
@@ -166,7 +166,7 @@ static void au1550_spi_chipsel(struct spi_device *spi, int value)
 	switch (value) {
 	case BITBANG_CS_INACTIVE:
 		if (hw->pdata->deactivate_cs)
-			hw->pdata->deactivate_cs(hw->pdata, spi->chip_select,
+			hw->pdata->deactivate_cs(hw->pdata, spi_get_chipselect(spi, 0),
 					cspol);
 		break;
 
@@ -211,7 +211,7 @@ static void au1550_spi_chipsel(struct spi_device *spi, int value)
 		} while ((stat & PSC_SPISTAT_DR) == 0);
 
 		if (hw->pdata->activate_cs)
-			hw->pdata->activate_cs(hw->pdata, spi->chip_select,
+			hw->pdata->activate_cs(hw->pdata, spi_get_chipselect(spi, 0),
 					cspol);
 		break;
 	}
@@ -923,7 +923,7 @@ err_nomem:
 	return err;
 }
 
-static int au1550_spi_remove(struct platform_device *pdev)
+static void au1550_spi_remove(struct platform_device *pdev)
 {
 	struct au1550_spi *hw = platform_get_drvdata(pdev);
 
@@ -942,7 +942,6 @@ static int au1550_spi_remove(struct platform_device *pdev)
 	}
 
 	spi_master_put(hw->master);
-	return 0;
 }
 
 /* work with hotplug and coldplug */
@@ -950,7 +949,7 @@ MODULE_ALIAS("platform:au1550-spi");
 
 static struct platform_driver au1550_spi_drv = {
 	.probe = au1550_spi_probe,
-	.remove = au1550_spi_remove,
+	.remove_new = au1550_spi_remove,
 	.driver = {
 		.name = "au1550-spi",
 	},
diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c
index 80c3e38f5c1b..89661f3b0d44 100644
--- a/drivers/spi/spi-axi-spi-engine.c
+++ b/drivers/spi/spi-axi-spi-engine.c
@@ -193,7 +193,7 @@ static void spi_engine_gen_cs(struct spi_engine_program *p, bool dry,
 	unsigned int mask = 0xff;
 
 	if (assert)
-		mask ^= BIT(spi->chip_select);
+		mask ^= BIT(spi_get_chipselect(spi, 0));
 
 	spi_engine_program_add_cmd(p, dry, SPI_ENGINE_CMD_ASSERT(1, mask));
 }
@@ -554,7 +554,7 @@ err_put_master:
 	return ret;
 }
 
-static int spi_engine_remove(struct platform_device *pdev)
+static void spi_engine_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = spi_master_get(platform_get_drvdata(pdev));
 	struct spi_engine *spi_engine = spi_master_get_devdata(master);
@@ -572,8 +572,6 @@ static int spi_engine_remove(struct platform_device *pdev)
 
 	clk_disable_unprepare(spi_engine->ref_clk);
 	clk_disable_unprepare(spi_engine->clk);
-
-	return 0;
 }
 
 static const struct of_device_id spi_engine_match_table[] = {
@@ -584,7 +582,7 @@ MODULE_DEVICE_TABLE(of, spi_engine_match_table);
 
 static struct platform_driver spi_engine_driver = {
 	.probe = spi_engine_probe,
-	.remove = spi_engine_remove,
+	.remove_new = spi_engine_remove,
 	.driver = {
 		.name = "spi-engine",
 		.of_match_table = spi_engine_match_table,
diff --git a/drivers/spi/spi-bcm-qspi.c b/drivers/spi/spi-bcm-qspi.c
index 0eee574d3e1f..6b46a3b67c41 100644
--- a/drivers/spi/spi-bcm-qspi.c
+++ b/drivers/spi/spi-bcm-qspi.c
@@ -986,7 +986,7 @@ static int write_to_hw(struct bcm_qspi *qspi, struct spi_device *spi)
 		if (has_bspi(qspi))
 			mspi_cdram &= ~1;
 		else
-			mspi_cdram |= (~(1 << spi->chip_select) &
+			mspi_cdram |= (~(1 << spi_get_chipselect(spi, 0)) &
 				       MSPI_CDRAM_PCS);
 
 		write_cdram_slot(qspi, slot, mspi_cdram);
@@ -1046,8 +1046,8 @@ static int bcm_qspi_bspi_exec_mem_op(struct spi_device *spi,
 			return -EIO;
 
 	from = op->addr.val;
-	if (!spi->cs_gpiod)
-		bcm_qspi_chip_select(qspi, spi->chip_select);
+	if (!spi_get_csgpiod(spi, 0))
+		bcm_qspi_chip_select(qspi, spi_get_chipselect(spi, 0));
 	bcm_qspi_write(qspi, MSPI, MSPI_WRITE_LOCK, 0);
 
 	/*
@@ -1126,8 +1126,8 @@ static int bcm_qspi_transfer_one(struct spi_master *master,
 	int slots;
 	unsigned long timeo = msecs_to_jiffies(100);
 
-	if (!spi->cs_gpiod)
-		bcm_qspi_chip_select(qspi, spi->chip_select);
+	if (!spi_get_csgpiod(spi, 0))
+		bcm_qspi_chip_select(qspi, spi_get_chipselect(spi, 0));
 	qspi->trans_pos.trans = trans;
 	qspi->trans_pos.byte = 0;
 
@@ -1457,7 +1457,7 @@ static const struct bcm_qspi_data bcm_qspi_spcr3_data = {
 	.has_spcr3_sysclk = true,
 };
 
-static const struct of_device_id bcm_qspi_of_match[] = {
+static const struct of_device_id bcm_qspi_of_match[] __maybe_unused = {
 	{
 		.compatible = "brcm,spi-bcm7445-qspi",
 		.data = &bcm_qspi_rev_data,
diff --git a/drivers/spi/spi-bcm2835.c b/drivers/spi/spi-bcm2835.c
index 747e03228c48..3b253da98c05 100644
--- a/drivers/spi/spi-bcm2835.c
+++ b/drivers/spi/spi-bcm2835.c
@@ -1274,9 +1274,9 @@ static int bcm2835_spi_setup(struct spi_device *spi)
 	 * The SPI core has successfully requested the CS GPIO line from the
 	 * device tree, so we are done.
 	 */
-	if (spi->cs_gpiod)
+	if (spi_get_csgpiod(spi, 0))
 		return 0;
-	if (spi->chip_select > 1) {
+	if (spi_get_chipselect(spi, 0) > 1) {
 		/* error in the case of native CS requested with CS > 1
 		 * officially there is a CS2, but it is not documented
 		 * which GPIO is connected with that...
@@ -1301,18 +1301,19 @@ static int bcm2835_spi_setup(struct spi_device *spi)
 	if (!chip)
 		return 0;
 
-	spi->cs_gpiod = gpiochip_request_own_desc(chip, 8 - spi->chip_select,
-						  DRV_NAME,
-						  GPIO_LOOKUP_FLAGS_DEFAULT,
-						  GPIOD_OUT_LOW);
-	if (IS_ERR(spi->cs_gpiod)) {
-		ret = PTR_ERR(spi->cs_gpiod);
+	spi_set_csgpiod(spi, 0, gpiochip_request_own_desc(chip,
+							  8 - (spi_get_chipselect(spi, 0)),
+							  DRV_NAME,
+							  GPIO_LOOKUP_FLAGS_DEFAULT,
+							  GPIOD_OUT_LOW));
+	if (IS_ERR(spi_get_csgpiod(spi, 0))) {
+		ret = PTR_ERR(spi_get_csgpiod(spi, 0));
 		goto err_cleanup;
 	}
 
 	/* and set up the "mode" and level */
 	dev_info(&spi->dev, "setting up native-CS%i to use GPIO\n",
-		 spi->chip_select);
+		 spi_get_chipselect(spi, 0));
 
 	return 0;
 
@@ -1398,7 +1399,7 @@ out_clk_disable:
 	return err;
 }
 
-static int bcm2835_spi_remove(struct platform_device *pdev)
+static void bcm2835_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr = platform_get_drvdata(pdev);
 	struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
@@ -1414,17 +1415,6 @@ static int bcm2835_spi_remove(struct platform_device *pdev)
 		   BCM2835_SPI_CS_CLEAR_RX | BCM2835_SPI_CS_CLEAR_TX);
 
 	clk_disable_unprepare(bs->clk);
-
-	return 0;
-}
-
-static void bcm2835_spi_shutdown(struct platform_device *pdev)
-{
-	int ret;
-
-	ret = bcm2835_spi_remove(pdev);
-	if (ret)
-		dev_err(&pdev->dev, "failed to shutdown\n");
 }
 
 static const struct of_device_id bcm2835_spi_match[] = {
@@ -1439,8 +1429,8 @@ static struct platform_driver bcm2835_spi_driver = {
 		.of_match_table	= bcm2835_spi_match,
 	},
 	.probe		= bcm2835_spi_probe,
-	.remove		= bcm2835_spi_remove,
-	.shutdown	= bcm2835_spi_shutdown,
+	.remove_new	= bcm2835_spi_remove,
+	.shutdown	= bcm2835_spi_remove,
 };
 module_platform_driver(bcm2835_spi_driver);
 
diff --git a/drivers/spi/spi-bcm2835aux.c b/drivers/spi/spi-bcm2835aux.c
index e28521922330..288f7b994b36 100644
--- a/drivers/spi/spi-bcm2835aux.c
+++ b/drivers/spi/spi-bcm2835aux.c
@@ -448,7 +448,7 @@ static int bcm2835aux_spi_setup(struct spi_device *spi)
 	if (spi->mode & SPI_NO_CS)
 		return 0;
 
-	if (spi->cs_gpiod)
+	if (spi_get_csgpiod(spi, 0))
 		return 0;
 
 	/* for dt-backwards compatibility: only support native on CS0
@@ -465,7 +465,7 @@ static int bcm2835aux_spi_setup(struct spi_device *spi)
 	dev_warn(&spi->dev,
 		 "Native CS is not supported - please configure cs-gpio in device-tree\n");
 
-	if (spi->chip_select == 0)
+	if (spi_get_chipselect(spi, 0) == 0)
 		return 0;
 
 	dev_warn(&spi->dev, "Native CS is not working for cs > 0\n");
@@ -567,7 +567,7 @@ out_clk_disable:
 	return err;
 }
 
-static int bcm2835aux_spi_remove(struct platform_device *pdev)
+static void bcm2835aux_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct bcm2835aux_spi *bs = spi_master_get_devdata(master);
@@ -580,8 +580,6 @@ static int bcm2835aux_spi_remove(struct platform_device *pdev)
 
 	/* disable the HW block by releasing the clock */
 	clk_disable_unprepare(bs->clk);
-
-	return 0;
 }
 
 static const struct of_device_id bcm2835aux_spi_match[] = {
@@ -596,7 +594,7 @@ static struct platform_driver bcm2835aux_spi_driver = {
 		.of_match_table	= bcm2835aux_spi_match,
 	},
 	.probe		= bcm2835aux_spi_probe,
-	.remove		= bcm2835aux_spi_remove,
+	.remove_new	= bcm2835aux_spi_remove,
 };
 module_platform_driver(bcm2835aux_spi_driver);
 
diff --git a/drivers/spi/spi-bcm63xx-hsspi.c b/drivers/spi/spi-bcm63xx-hsspi.c
index cd0a6478f5e7..ee2528dad02d 100644
--- a/drivers/spi/spi-bcm63xx-hsspi.c
+++ b/drivers/spi/spi-bcm63xx-hsspi.c
@@ -349,7 +349,7 @@ static int bcm63xx_hsspi_do_prepend_txrx(struct spi_device *spi,
 					 struct spi_transfer *t)
 {
 	struct bcm63xx_hsspi *bs = spi_master_get_devdata(spi->master);
-	unsigned int chip_select = spi->chip_select;
+	unsigned int chip_select = spi_get_chipselect(spi, 0);
 	u16 opcode = 0, val;
 	const u8 *tx = t->tx_buf;
 	u8 *rx = t->rx_buf;
@@ -441,7 +441,7 @@ static void bcm63xx_hsspi_set_cs(struct bcm63xx_hsspi *bs, unsigned int cs,
 static void bcm63xx_hsspi_set_clk(struct bcm63xx_hsspi *bs,
 				  struct spi_device *spi, int hz)
 {
-	unsigned int profile = spi->chip_select;
+	unsigned int profile = spi_get_chipselect(spi, 0);
 	u32 reg;
 
 	reg = DIV_ROUND_UP(2048, DIV_ROUND_UP(bs->speed_hz, hz));
@@ -468,7 +468,7 @@ static void bcm63xx_hsspi_set_clk(struct bcm63xx_hsspi *bs,
 static int bcm63xx_hsspi_do_txrx(struct spi_device *spi, struct spi_transfer *t)
 {
 	struct bcm63xx_hsspi *bs = spi_master_get_devdata(spi->master);
-	unsigned int chip_select = spi->chip_select;
+	unsigned int chip_select = spi_get_chipselect(spi, 0);
 	u16 opcode = 0, val;
 	int pending = t->len;
 	int step_size = HSSPI_BUFFER_LEN;
@@ -478,7 +478,7 @@ static int bcm63xx_hsspi_do_txrx(struct spi_device *spi, struct spi_transfer *t)
 
 	bcm63xx_hsspi_set_clk(bs, spi, t->speed_hz);
 	if (!t->cs_off)
-		bcm63xx_hsspi_set_cs(bs, spi->chip_select, true);
+		bcm63xx_hsspi_set_cs(bs, spi_get_chipselect(spi, 0), true);
 
 	if (tx && rx)
 		opcode = HSSPI_OP_READ_WRITE;
@@ -545,14 +545,14 @@ static int bcm63xx_hsspi_setup(struct spi_device *spi)
 	u32 reg;
 
 	reg = __raw_readl(bs->regs +
-			  HSSPI_PROFILE_SIGNAL_CTRL_REG(spi->chip_select));
+			  HSSPI_PROFILE_SIGNAL_CTRL_REG(spi_get_chipselect(spi, 0)));
 	reg &= ~(SIGNAL_CTRL_LAUNCH_RISING | SIGNAL_CTRL_LATCH_RISING);
 	if (spi->mode & SPI_CPHA)
 		reg |= SIGNAL_CTRL_LAUNCH_RISING;
 	else
 		reg |= SIGNAL_CTRL_LATCH_RISING;
 	__raw_writel(reg, bs->regs +
-		     HSSPI_PROFILE_SIGNAL_CTRL_REG(spi->chip_select));
+		     HSSPI_PROFILE_SIGNAL_CTRL_REG(spi_get_chipselect(spi, 0)));
 
 	mutex_lock(&bs->bus_mutex);
 	reg = __raw_readl(bs->regs + HSSPI_GLOBAL_CTRL_REG);
@@ -560,16 +560,16 @@ static int bcm63xx_hsspi_setup(struct spi_device *spi)
 	/* only change actual polarities if there is no transfer */
 	if ((reg & GLOBAL_CTRL_CS_POLARITY_MASK) == bs->cs_polarity) {
 		if (spi->mode & SPI_CS_HIGH)
-			reg |= BIT(spi->chip_select);
+			reg |= BIT(spi_get_chipselect(spi, 0));
 		else
-			reg &= ~BIT(spi->chip_select);
+			reg &= ~BIT(spi_get_chipselect(spi, 0));
 		__raw_writel(reg, bs->regs + HSSPI_GLOBAL_CTRL_REG);
 	}
 
 	if (spi->mode & SPI_CS_HIGH)
-		bs->cs_polarity |= BIT(spi->chip_select);
+		bs->cs_polarity |= BIT(spi_get_chipselect(spi, 0));
 	else
-		bs->cs_polarity &= ~BIT(spi->chip_select);
+		bs->cs_polarity &= ~BIT(spi_get_chipselect(spi, 0));
 
 	mutex_unlock(&bs->bus_mutex);
 
@@ -600,7 +600,7 @@ static int bcm63xx_hsspi_do_dummy_cs_txrx(struct spi_device *spi,
 	 * e. At the end restore the polarities again to their default values.
 	 */
 
-	dummy_cs = !spi->chip_select;
+	dummy_cs = !spi_get_chipselect(spi, 0);
 	bcm63xx_hsspi_set_cs(bs, dummy_cs, true);
 
 	list_for_each_entry(t, &msg->transfers, transfer_list) {
@@ -633,22 +633,22 @@ static int bcm63xx_hsspi_do_dummy_cs_txrx(struct spi_device *spi,
 				keep_cs = true;
 			} else {
 				if (!t->cs_off)
-					bcm63xx_hsspi_set_cs(bs, spi->chip_select, false);
+					bcm63xx_hsspi_set_cs(bs, spi_get_chipselect(spi, 0), false);
 
 				spi_transfer_cs_change_delay_exec(msg, t);
 
 				if (!list_next_entry(t, transfer_list)->cs_off)
-					bcm63xx_hsspi_set_cs(bs, spi->chip_select, true);
+					bcm63xx_hsspi_set_cs(bs, spi_get_chipselect(spi, 0), true);
 			}
 		} else if (!list_is_last(&t->transfer_list, &msg->transfers) &&
 			   t->cs_off != list_next_entry(t, transfer_list)->cs_off) {
-			bcm63xx_hsspi_set_cs(bs, spi->chip_select, t->cs_off);
+			bcm63xx_hsspi_set_cs(bs, spi_get_chipselect(spi, 0), t->cs_off);
 		}
 	}
 
 	bcm63xx_hsspi_set_cs(bs, dummy_cs, false);
 	if (status || !keep_cs)
-		bcm63xx_hsspi_set_cs(bs, spi->chip_select, false);
+		bcm63xx_hsspi_set_cs(bs, spi_get_chipselect(spi, 0), false);
 
 	return status;
 }
@@ -878,7 +878,7 @@ out_disable_clk:
 }
 
 
-static int bcm63xx_hsspi_remove(struct platform_device *pdev)
+static void bcm63xx_hsspi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct bcm63xx_hsspi *bs = spi_master_get_devdata(master);
@@ -888,8 +888,6 @@ static int bcm63xx_hsspi_remove(struct platform_device *pdev)
 	clk_disable_unprepare(bs->pll_clk);
 	clk_disable_unprepare(bs->clk);
 	sysfs_remove_group(&pdev->dev.kobj, &bcm63xx_hsspi_group);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -946,7 +944,7 @@ static struct platform_driver bcm63xx_hsspi_driver = {
 		.of_match_table = bcm63xx_hsspi_of_match,
 	},
 	.probe		= bcm63xx_hsspi_probe,
-	.remove		= bcm63xx_hsspi_remove,
+	.remove_new	= bcm63xx_hsspi_remove,
 };
 
 module_platform_driver(bcm63xx_hsspi_driver);
diff --git a/drivers/spi/spi-bcm63xx.c b/drivers/spi/spi-bcm63xx.c
index 3686d78c44a6..9aecb77c3d89 100644
--- a/drivers/spi/spi-bcm63xx.c
+++ b/drivers/spi/spi-bcm63xx.c
@@ -157,16 +157,6 @@ static inline u8 bcm_spi_readb(struct bcm63xx_spi *bs,
 	return readb(bs->regs + bs->reg_offsets[offset]);
 }
 
-static inline u16 bcm_spi_readw(struct bcm63xx_spi *bs,
-				unsigned int offset)
-{
-#ifdef CONFIG_CPU_BIG_ENDIAN
-	return ioread16be(bs->regs + bs->reg_offsets[offset]);
-#else
-	return readw(bs->regs + bs->reg_offsets[offset]);
-#endif
-}
-
 static inline void bcm_spi_writeb(struct bcm63xx_spi *bs,
 				  u8 value, unsigned int offset)
 {
@@ -292,7 +282,7 @@ static int bcm63xx_txrx_bufs(struct spi_device *spi, struct spi_transfer *first,
 	/* Issue the transfer */
 	cmd = SPI_CMD_START_IMMEDIATE;
 	cmd |= (prepend_len << SPI_CMD_PREPEND_BYTE_CNT_SHIFT);
-	cmd |= (spi->chip_select << SPI_CMD_DEVICE_ID_SHIFT);
+	cmd |= (spi_get_chipselect(spi, 0) << SPI_CMD_DEVICE_ID_SHIFT);
 	bcm_spi_writew(bs, cmd, SPI_CMD);
 
 	/* Enable the CMD_DONE interrupt */
@@ -615,7 +605,7 @@ out_err:
 	return ret;
 }
 
-static int bcm63xx_spi_remove(struct platform_device *pdev)
+static void bcm63xx_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct bcm63xx_spi *bs = spi_master_get_devdata(master);
@@ -625,11 +615,8 @@ static int bcm63xx_spi_remove(struct platform_device *pdev)
 
 	/* HW shutdown */
 	clk_disable_unprepare(bs->clk);
-
-	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
 static int bcm63xx_spi_suspend(struct device *dev)
 {
 	struct spi_master *master = dev_get_drvdata(dev);
@@ -656,11 +643,8 @@ static int bcm63xx_spi_resume(struct device *dev)
 
 	return 0;
 }
-#endif
 
-static const struct dev_pm_ops bcm63xx_spi_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(bcm63xx_spi_suspend, bcm63xx_spi_resume)
-};
+static DEFINE_SIMPLE_DEV_PM_OPS(bcm63xx_spi_pm_ops, bcm63xx_spi_suspend, bcm63xx_spi_resume);
 
 static struct platform_driver bcm63xx_spi_driver = {
 	.driver = {
@@ -670,7 +654,7 @@ static struct platform_driver bcm63xx_spi_driver = {
 	},
 	.id_table	= bcm63xx_spi_dev_match,
 	.probe		= bcm63xx_spi_probe,
-	.remove		= bcm63xx_spi_remove,
+	.remove_new	= bcm63xx_spi_remove,
 };
 
 module_platform_driver(bcm63xx_spi_driver);
diff --git a/drivers/spi/spi-bcmbca-hsspi.c b/drivers/spi/spi-bcmbca-hsspi.c
index 3f9e6131ad86..8cbd01619789 100644
--- a/drivers/spi/spi-bcmbca-hsspi.c
+++ b/drivers/spi/spi-bcmbca-hsspi.c
@@ -193,7 +193,7 @@ static void bcmbca_hsspi_set_cs(struct bcmbca_hsspi *bs, unsigned int cs,
 static void bcmbca_hsspi_set_clk(struct bcmbca_hsspi *bs,
 				  struct spi_device *spi, int hz)
 {
-	unsigned int profile = spi->chip_select;
+	unsigned int profile = spi_get_chipselect(spi, 0);
 	u32 reg;
 
 	reg = DIV_ROUND_UP(2048, DIV_ROUND_UP(bs->speed_hz, hz));
@@ -251,7 +251,7 @@ static int bcmbca_hsspi_do_txrx(struct spi_device *spi, struct spi_transfer *t,
 								struct spi_message *msg)
 {
 	struct bcmbca_hsspi *bs = spi_master_get_devdata(spi->master);
-	unsigned int chip_select = spi->chip_select;
+	unsigned int chip_select = spi_get_chipselect(spi, 0);
 	u16 opcode = 0, val;
 	int pending = t->len;
 	int step_size = HSSPI_BUFFER_LEN;
@@ -312,7 +312,7 @@ static int bcmbca_hsspi_do_txrx(struct spi_device *spi, struct spi_transfer *t,
 			    PINGPONG_COMMAND_START_NOW;
 		__raw_writel(reg, bs->regs + HSSPI_PINGPONG_COMMAND_REG(0));
 
-		if (bcmbca_hsspi_wait_cmd(bs, spi->chip_select))
+		if (bcmbca_hsspi_wait_cmd(bs, spi_get_chipselect(spi, 0)))
 			return -ETIMEDOUT;
 
 		pending -= curr_step;
@@ -332,33 +332,33 @@ static int bcmbca_hsspi_setup(struct spi_device *spi)
 	u32 reg;
 
 	reg = __raw_readl(bs->regs +
-			  HSSPI_PROFILE_SIGNAL_CTRL_REG(spi->chip_select));
+			  HSSPI_PROFILE_SIGNAL_CTRL_REG(spi_get_chipselect(spi, 0)));
 	reg &= ~(SIGNAL_CTRL_LAUNCH_RISING | SIGNAL_CTRL_LATCH_RISING);
 	if (spi->mode & SPI_CPHA)
 		reg |= SIGNAL_CTRL_LAUNCH_RISING;
 	else
 		reg |= SIGNAL_CTRL_LATCH_RISING;
 	__raw_writel(reg, bs->regs +
-		     HSSPI_PROFILE_SIGNAL_CTRL_REG(spi->chip_select));
+		     HSSPI_PROFILE_SIGNAL_CTRL_REG(spi_get_chipselect(spi, 0)));
 
 	mutex_lock(&bs->bus_mutex);
 	reg = __raw_readl(bs->regs + HSSPI_GLOBAL_CTRL_REG);
 
 	if (spi->mode & SPI_CS_HIGH)
-		reg |= BIT(spi->chip_select);
+		reg |= BIT(spi_get_chipselect(spi, 0));
 	else
-		reg &= ~BIT(spi->chip_select);
+		reg &= ~BIT(spi_get_chipselect(spi, 0));
 	__raw_writel(reg, bs->regs + HSSPI_GLOBAL_CTRL_REG);
 
 	if (spi->mode & SPI_CS_HIGH)
-		bs->cs_polarity |= BIT(spi->chip_select);
+		bs->cs_polarity |= BIT(spi_get_chipselect(spi, 0));
 	else
-		bs->cs_polarity &= ~BIT(spi->chip_select);
+		bs->cs_polarity &= ~BIT(spi_get_chipselect(spi, 0));
 
 	reg = __raw_readl(bs->spim_ctrl);
-	reg &= ~BIT(spi->chip_select + SPIM_CTRL_CS_OVERRIDE_VAL_SHIFT);
+	reg &= ~BIT(spi_get_chipselect(spi, 0) + SPIM_CTRL_CS_OVERRIDE_VAL_SHIFT);
 	if (spi->mode & SPI_CS_HIGH)
-		reg |= BIT(spi->chip_select + SPIM_CTRL_CS_OVERRIDE_VAL_SHIFT);
+		reg |= BIT(spi_get_chipselect(spi, 0) + SPIM_CTRL_CS_OVERRIDE_VAL_SHIFT);
 	__raw_writel(reg, bs->spim_ctrl);
 
 	mutex_unlock(&bs->bus_mutex);
@@ -388,16 +388,16 @@ static int bcmbca_hsspi_transfer_one(struct spi_master *master,
 				keep_cs = true;
 			} else {
 				if (!t->cs_off)
-					bcmbca_hsspi_set_cs(bs, spi->chip_select, false);
+					bcmbca_hsspi_set_cs(bs, spi_get_chipselect(spi, 0), false);
 
 				spi_transfer_cs_change_delay_exec(msg, t);
 
 				if (!list_next_entry(t, transfer_list)->cs_off)
-					bcmbca_hsspi_set_cs(bs, spi->chip_select, true);
+					bcmbca_hsspi_set_cs(bs, spi_get_chipselect(spi, 0), true);
 			}
 		} else if (!list_is_last(&t->transfer_list, &msg->transfers) &&
 			   t->cs_off != list_next_entry(t, transfer_list)->cs_off) {
-			bcmbca_hsspi_set_cs(bs, spi->chip_select, t->cs_off);
+			bcmbca_hsspi_set_cs(bs, spi_get_chipselect(spi, 0), t->cs_off);
 		}
 
 		msg->actual_length += t->len;
@@ -406,7 +406,7 @@ static int bcmbca_hsspi_transfer_one(struct spi_master *master,
 	mutex_unlock(&bs->msg_mutex);
 
 	if (status || !keep_cs)
-		bcmbca_hsspi_set_cs(bs, spi->chip_select, false);
+		bcmbca_hsspi_set_cs(bs, spi_get_chipselect(spi, 0), false);
 
 	msg->status = status;
 	spi_finalize_current_message(master);
@@ -576,7 +576,7 @@ out_disable_clk:
 	return ret;
 }
 
-static int bcmbca_hsspi_remove(struct platform_device *pdev)
+static void bcmbca_hsspi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct bcmbca_hsspi *bs = spi_master_get_devdata(master);
@@ -586,8 +586,6 @@ static int bcmbca_hsspi_remove(struct platform_device *pdev)
 	clk_disable_unprepare(bs->pll_clk);
 	clk_disable_unprepare(bs->clk);
 	sysfs_remove_group(&pdev->dev.kobj, &bcmbca_hsspi_group);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -644,7 +642,7 @@ static struct platform_driver bcmbca_hsspi_driver = {
 		   .of_match_table = bcmbca_hsspi_of_match,
 		   },
 	.probe = bcmbca_hsspi_probe,
-	.remove = bcmbca_hsspi_remove,
+	.remove_new = bcmbca_hsspi_remove,
 };
 
 module_platform_driver(bcmbca_hsspi_driver);
diff --git a/drivers/spi/spi-brcmstb-qspi.c b/drivers/spi/spi-brcmstb-qspi.c
index de362b35718f..e1b137419f5c 100644
--- a/drivers/spi/spi-brcmstb-qspi.c
+++ b/drivers/spi/spi-brcmstb-qspi.c
@@ -21,16 +21,14 @@ static int brcmstb_qspi_probe(struct platform_device *pdev)
 	return bcm_qspi_probe(pdev, NULL);
 }
 
-static int brcmstb_qspi_remove(struct platform_device *pdev)
+static void brcmstb_qspi_remove(struct platform_device *pdev)
 {
 	bcm_qspi_remove(pdev);
-
-	return 0;
 }
 
 static struct platform_driver brcmstb_qspi_driver = {
 	.probe			= brcmstb_qspi_probe,
-	.remove			= brcmstb_qspi_remove,
+	.remove_new		= brcmstb_qspi_remove,
 	.driver = {
 		.name		= "brcmstb_qspi",
 		.pm		= &bcm_qspi_pm_ops,
diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c
index 64b6a460d739..6ddb2dfc0f00 100644
--- a/drivers/spi/spi-cadence-quadspi.c
+++ b/drivers/spi/spi-cadence-quadspi.c
@@ -791,6 +791,21 @@ failrd:
 	return ret;
 }
 
+static void cqspi_controller_enable(struct cqspi_st *cqspi, bool enable)
+{
+	void __iomem *reg_base = cqspi->iobase;
+	unsigned int reg;
+
+	reg = readl(reg_base + CQSPI_REG_CONFIG);
+
+	if (enable)
+		reg |= CQSPI_REG_CONFIG_ENABLE_MASK;
+	else
+		reg &= ~CQSPI_REG_CONFIG_ENABLE_MASK;
+
+	writel(reg, reg_base + CQSPI_REG_CONFIG);
+}
+
 static int cqspi_versal_indirect_read_dma(struct cqspi_flash_pdata *f_pdata,
 					  u_char *rxbuf, loff_t from_addr,
 					  size_t n_rx)
@@ -815,10 +830,14 @@ static int cqspi_versal_indirect_read_dma(struct cqspi_flash_pdata *f_pdata,
 	if (ret)
 		return ret;
 
+	cqspi_controller_enable(cqspi, 0);
+
 	reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
 	reg |= CQSPI_REG_CONFIG_DMA_MASK;
 	writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
 
+	cqspi_controller_enable(cqspi, 1);
+
 	dma_addr = dma_map_single(dev, rxbuf, bytes_to_dma, DMA_FROM_DEVICE);
 	if (dma_mapping_error(dev, dma_addr)) {
 		dev_err(dev, "dma mapping failed\n");
@@ -863,7 +882,7 @@ static int cqspi_versal_indirect_read_dma(struct cqspi_flash_pdata *f_pdata,
 	reinit_completion(&cqspi->transfer_complete);
 
 	if (!wait_for_completion_timeout(&cqspi->transfer_complete,
-					 msecs_to_jiffies(CQSPI_READ_TIMEOUT_MS))) {
+					 msecs_to_jiffies(max_t(size_t, bytes_to_dma, 500)))) {
 		ret = -ETIMEDOUT;
 		goto failrd;
 	}
@@ -876,10 +895,14 @@ static int cqspi_versal_indirect_read_dma(struct cqspi_flash_pdata *f_pdata,
 	       cqspi->iobase + CQSPI_REG_INDIRECTRD);
 	dma_unmap_single(dev, dma_addr, bytes_to_dma, DMA_FROM_DEVICE);
 
+	cqspi_controller_enable(cqspi, 0);
+
 	reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
 	reg &= ~CQSPI_REG_CONFIG_DMA_MASK;
 	writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
 
+	cqspi_controller_enable(cqspi, 1);
+
 	ret = zynqmp_pm_ospi_mux_select(cqspi->pd_dev_id,
 					PM_OSPI_MUX_SEL_LINEAR);
 	if (ret)
@@ -1182,21 +1205,6 @@ static void cqspi_readdata_capture(struct cqspi_st *cqspi,
 	writel(reg, reg_base + CQSPI_REG_READCAPTURE);
 }
 
-static void cqspi_controller_enable(struct cqspi_st *cqspi, bool enable)
-{
-	void __iomem *reg_base = cqspi->iobase;
-	unsigned int reg;
-
-	reg = readl(reg_base + CQSPI_REG_CONFIG);
-
-	if (enable)
-		reg |= CQSPI_REG_CONFIG_ENABLE_MASK;
-	else
-		reg &= ~CQSPI_REG_CONFIG_ENABLE_MASK;
-
-	writel(reg, reg_base + CQSPI_REG_CONFIG);
-}
-
 static void cqspi_configure(struct cqspi_flash_pdata *f_pdata,
 			    unsigned long sclk)
 {
@@ -1355,7 +1363,7 @@ static int cqspi_mem_process(struct spi_mem *mem, const struct spi_mem_op *op)
 	struct cqspi_st *cqspi = spi_master_get_devdata(mem->spi->master);
 	struct cqspi_flash_pdata *f_pdata;
 
-	f_pdata = &cqspi->f_pdata[mem->spi->chip_select];
+	f_pdata = &cqspi->f_pdata[spi_get_chipselect(mem->spi, 0)];
 	cqspi_configure(f_pdata, mem->spi->max_speed_hz);
 
 	if (op->data.dir == SPI_MEM_DATA_IN && op->data.buf.in) {
@@ -1561,7 +1569,8 @@ static const char *cqspi_get_name(struct spi_mem *mem)
 	struct cqspi_st *cqspi = spi_master_get_devdata(mem->spi->master);
 	struct device *dev = &cqspi->pdev->dev;
 
-	return devm_kasprintf(dev, GFP_KERNEL, "%s.%d", dev_name(dev), mem->spi->chip_select);
+	return devm_kasprintf(dev, GFP_KERNEL, "%s.%d", dev_name(dev),
+			      spi_get_chipselect(mem->spi, 0));
 }
 
 static const struct spi_controller_mem_ops cqspi_mem_ops = {
@@ -1615,7 +1624,7 @@ static int cqspi_setup_flash(struct cqspi_st *cqspi)
 static int cqspi_probe(struct platform_device *pdev)
 {
 	const struct cqspi_driver_platdata *ddata;
-	struct reset_control *rstc, *rstc_ocp;
+	struct reset_control *rstc, *rstc_ocp, *rstc_ref;
 	struct device *dev = &pdev->dev;
 	struct spi_master *master;
 	struct resource *res_ahb;
@@ -1705,6 +1714,17 @@ static int cqspi_probe(struct platform_device *pdev)
 		goto probe_reset_failed;
 	}
 
+	if (of_device_is_compatible(pdev->dev.of_node, "starfive,jh7110-qspi")) {
+		rstc_ref = devm_reset_control_get_optional_exclusive(dev, "rstc_ref");
+		if (IS_ERR(rstc_ref)) {
+			ret = PTR_ERR(rstc_ref);
+			dev_err(dev, "Cannot get QSPI REF reset.\n");
+			goto probe_reset_failed;
+		}
+		reset_control_assert(rstc_ref);
+		reset_control_deassert(rstc_ref);
+	}
+
 	reset_control_assert(rstc);
 	reset_control_deassert(rstc);
 
@@ -1784,7 +1804,7 @@ probe_pm_failed:
 	return ret;
 }
 
-static int cqspi_remove(struct platform_device *pdev)
+static void cqspi_remove(struct platform_device *pdev)
 {
 	struct cqspi_st *cqspi = platform_get_drvdata(pdev);
 
@@ -1798,36 +1818,38 @@ static int cqspi_remove(struct platform_device *pdev)
 
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
 static int cqspi_suspend(struct device *dev)
 {
 	struct cqspi_st *cqspi = dev_get_drvdata(dev);
+	struct spi_master *master = dev_get_drvdata(dev);
+	int ret;
 
+	ret = spi_master_suspend(master);
 	cqspi_controller_enable(cqspi, 0);
-	return 0;
+
+	clk_disable_unprepare(cqspi->clk);
+
+	return ret;
 }
 
 static int cqspi_resume(struct device *dev)
 {
 	struct cqspi_st *cqspi = dev_get_drvdata(dev);
+	struct spi_master *master = dev_get_drvdata(dev);
 
-	cqspi_controller_enable(cqspi, 1);
-	return 0;
-}
+	clk_prepare_enable(cqspi->clk);
+	cqspi_wait_idle(cqspi);
+	cqspi_controller_init(cqspi);
 
-static const struct dev_pm_ops cqspi__dev_pm_ops = {
-	.suspend = cqspi_suspend,
-	.resume = cqspi_resume,
-};
+	cqspi->current_cs = -1;
+	cqspi->sclk = 0;
+
+	return spi_master_resume(master);
+}
 
-#define CQSPI_DEV_PM_OPS	(&cqspi__dev_pm_ops)
-#else
-#define CQSPI_DEV_PM_OPS	NULL
-#endif
+static DEFINE_SIMPLE_DEV_PM_OPS(cqspi_dev_pm_ops, cqspi_suspend, cqspi_resume);
 
 static const struct cqspi_driver_platdata cdns_qspi = {
 	.quirks = CQSPI_DISABLE_DAC_MODE,
@@ -1859,6 +1881,10 @@ static const struct cqspi_driver_platdata versal_ospi = {
 	.get_dma_status = cqspi_get_versal_dma_status,
 };
 
+static const struct cqspi_driver_platdata jh7110_qspi = {
+	.quirks = CQSPI_DISABLE_DAC_MODE,
+};
+
 static const struct of_device_id cqspi_dt_ids[] = {
 	{
 		.compatible = "cdns,qspi-nor",
@@ -1884,6 +1910,10 @@ static const struct of_device_id cqspi_dt_ids[] = {
 		.compatible = "intel,socfpga-qspi",
 		.data = &socfpga_qspi,
 	},
+	{
+		.compatible = "starfive,jh7110-qspi",
+		.data = &jh7110_qspi,
+	},
 	{ /* end of table */ }
 };
 
@@ -1891,10 +1921,10 @@ MODULE_DEVICE_TABLE(of, cqspi_dt_ids);
 
 static struct platform_driver cqspi_platform_driver = {
 	.probe = cqspi_probe,
-	.remove = cqspi_remove,
+	.remove_new = cqspi_remove,
 	.driver = {
 		.name = CQSPI_NAME,
-		.pm = CQSPI_DEV_PM_OPS,
+		.pm = &cqspi_dev_pm_ops,
 		.of_match_table = cqspi_dt_ids,
 	},
 };
diff --git a/drivers/spi/spi-cadence-xspi.c b/drivers/spi/spi-cadence-xspi.c
index 91db3c973167..ce4a3145f065 100644
--- a/drivers/spi/spi-cadence-xspi.c
+++ b/drivers/spi/spi-cadence-xspi.c
@@ -409,8 +409,8 @@ static int cdns_xspi_mem_op(struct cdns_xspi_dev *cdns_xspi,
 {
 	enum spi_mem_data_dir dir = op->data.dir;
 
-	if (cdns_xspi->cur_cs != mem->spi->chip_select)
-		cdns_xspi->cur_cs = mem->spi->chip_select;
+	if (cdns_xspi->cur_cs != spi_get_chipselect(mem->spi, 0))
+		cdns_xspi->cur_cs = spi_get_chipselect(mem->spi, 0);
 
 	return cdns_xspi_send_stig_command(cdns_xspi, op,
 					   (dir != SPI_MEM_NO_DATA));
diff --git a/drivers/spi/spi-cadence.c b/drivers/spi/spi-cadence.c
index 6a7f7df1e776..ac85d5562212 100644
--- a/drivers/spi/spi-cadence.c
+++ b/drivers/spi/spi-cadence.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * Cadence SPI controller driver (master mode only)
+ * Cadence SPI controller driver (master and slave mode)
  *
  * Copyright (C) 2008 - 2014 Xilinx, Inc.
  *
@@ -139,17 +139,21 @@ static inline void cdns_spi_write(struct cdns_spi *xspi, u32 offset, u32 val)
 /**
  * cdns_spi_init_hw - Initialize the hardware and configure the SPI controller
  * @xspi:	Pointer to the cdns_spi structure
+ * @is_slave:	Flag to indicate slave or master mode
+ * * On reset the SPI controller is configured to slave or  master mode.
+ * In master mode baud rate divisor is set to 4, threshold value for TX FIFO
+ * not full interrupt is set to 1 and size of the word to be transferred as 8 bit.
  *
- * On reset the SPI controller is configured to be in master mode, baud rate
- * divisor is set to 4, threshold value for TX FIFO not full interrupt is set
- * to 1 and size of the word to be transferred as 8 bit.
  * This function initializes the SPI controller to disable and clear all the
  * interrupts, enable manual slave select and manual start, deselect all the
  * chip select lines, and enable the SPI controller.
  */
-static void cdns_spi_init_hw(struct cdns_spi *xspi)
+static void cdns_spi_init_hw(struct cdns_spi *xspi, bool is_slave)
 {
-	u32 ctrl_reg = CDNS_SPI_CR_DEFAULT;
+	u32 ctrl_reg = 0;
+
+	if (!is_slave)
+		ctrl_reg |= CDNS_SPI_CR_DEFAULT;
 
 	if (xspi->is_decoded_cs)
 		ctrl_reg |= CDNS_SPI_CR_PERI_SEL;
@@ -173,7 +177,7 @@ static void cdns_spi_init_hw(struct cdns_spi *xspi)
  */
 static void cdns_spi_chipselect(struct spi_device *spi, bool is_high)
 {
-	struct cdns_spi *xspi = spi_master_get_devdata(spi->master);
+	struct cdns_spi *xspi = spi_controller_get_devdata(spi->controller);
 	u32 ctrl_reg;
 
 	ctrl_reg = cdns_spi_read(xspi, CDNS_SPI_CR);
@@ -185,11 +189,11 @@ static void cdns_spi_chipselect(struct spi_device *spi, bool is_high)
 		/* Select the slave */
 		ctrl_reg &= ~CDNS_SPI_CR_SSCTRL;
 		if (!(xspi->is_decoded_cs))
-			ctrl_reg |= ((~(CDNS_SPI_SS0 << spi->chip_select)) <<
+			ctrl_reg |= ((~(CDNS_SPI_SS0 << spi_get_chipselect(spi, 0))) <<
 				     CDNS_SPI_SS_SHIFT) &
 				     CDNS_SPI_CR_SSCTRL;
 		else
-			ctrl_reg |= (spi->chip_select << CDNS_SPI_SS_SHIFT) &
+			ctrl_reg |= (spi_get_chipselect(spi, 0) << CDNS_SPI_SS_SHIFT) &
 				     CDNS_SPI_CR_SSCTRL;
 	}
 
@@ -204,7 +208,7 @@ static void cdns_spi_chipselect(struct spi_device *spi, bool is_high)
  */
 static void cdns_spi_config_clock_mode(struct spi_device *spi)
 {
-	struct cdns_spi *xspi = spi_master_get_devdata(spi->master);
+	struct cdns_spi *xspi = spi_controller_get_devdata(spi->controller);
 	u32 ctrl_reg, new_ctrl_reg;
 
 	new_ctrl_reg = cdns_spi_read(xspi, CDNS_SPI_CR);
@@ -247,7 +251,7 @@ static void cdns_spi_config_clock_mode(struct spi_device *spi)
 static void cdns_spi_config_clock_freq(struct spi_device *spi,
 				       struct spi_transfer *transfer)
 {
-	struct cdns_spi *xspi = spi_master_get_devdata(spi->master);
+	struct cdns_spi *xspi = spi_controller_get_devdata(spi->controller);
 	u32 ctrl_reg, baud_rate_val;
 	unsigned long frequency;
 
@@ -285,7 +289,7 @@ static void cdns_spi_config_clock_freq(struct spi_device *spi,
 static int cdns_spi_setup_transfer(struct spi_device *spi,
 				   struct spi_transfer *transfer)
 {
-	struct cdns_spi *xspi = spi_master_get_devdata(spi->master);
+	struct cdns_spi *xspi = spi_controller_get_devdata(spi->controller);
 
 	cdns_spi_config_clock_freq(spi, transfer);
 
@@ -325,6 +329,25 @@ static void cdns_spi_fill_tx_fifo(struct cdns_spi *xspi)
 }
 
 /**
+ * cdns_spi_read_rx_fifo - Reads the RX FIFO with as many bytes as possible
+ * @xspi:       Pointer to the cdns_spi structure
+ * @count:	Read byte count
+ */
+static void cdns_spi_read_rx_fifo(struct cdns_spi *xspi, unsigned long count)
+{
+	u8 data;
+
+	/* Read out the data from the RX FIFO */
+	while (count > 0) {
+		data = cdns_spi_read(xspi, CDNS_SPI_RXD);
+		if (xspi->rxbuf)
+			*xspi->rxbuf++ = data;
+		xspi->rx_bytes--;
+		count--;
+	}
+}
+
+/**
  * cdns_spi_irq - Interrupt service routine of the SPI controller
  * @irq:	IRQ number
  * @dev_id:	Pointer to the xspi structure
@@ -340,8 +363,8 @@ static void cdns_spi_fill_tx_fifo(struct cdns_spi *xspi)
  */
 static irqreturn_t cdns_spi_irq(int irq, void *dev_id)
 {
-	struct spi_master *master = dev_id;
-	struct cdns_spi *xspi = spi_master_get_devdata(master);
+	struct spi_controller *ctlr = dev_id;
+	struct cdns_spi *xspi = spi_controller_get_devdata(ctlr);
 	irqreturn_t status;
 	u32 intr_status;
 
@@ -355,33 +378,39 @@ static irqreturn_t cdns_spi_irq(int irq, void *dev_id)
 		 * transferred is non-zero
 		 */
 		cdns_spi_write(xspi, CDNS_SPI_IDR, CDNS_SPI_IXR_DEFAULT);
-		spi_finalize_current_transfer(master);
+		spi_finalize_current_transfer(ctlr);
 		status = IRQ_HANDLED;
 	} else if (intr_status & CDNS_SPI_IXR_TXOW) {
-		unsigned long trans_cnt;
-
-		trans_cnt = xspi->rx_bytes - xspi->tx_bytes;
+		int trans_cnt = cdns_spi_read(xspi, CDNS_SPI_THLD);
+		/* Set threshold to one if number of pending are
+		 * less than half fifo
+		 */
+		if (xspi->tx_bytes < xspi->tx_fifo_depth >> 1)
+			cdns_spi_write(xspi, CDNS_SPI_THLD, 1);
 
-		/* Read out the data from the RX FIFO */
 		while (trans_cnt) {
-			u8 data;
-
-			data = cdns_spi_read(xspi, CDNS_SPI_RXD);
-			if (xspi->rxbuf)
-				*xspi->rxbuf++ = data;
-
-			xspi->rx_bytes--;
+			cdns_spi_read_rx_fifo(xspi, 1);
+
+			if (xspi->tx_bytes) {
+				if (xspi->txbuf)
+					cdns_spi_write(xspi, CDNS_SPI_TXD,
+						       *xspi->txbuf++);
+				else
+					cdns_spi_write(xspi, CDNS_SPI_TXD, 0);
+				xspi->tx_bytes--;
+			}
 			trans_cnt--;
 		}
-
-		if (xspi->tx_bytes) {
-			/* There is more data to send */
-			cdns_spi_fill_tx_fifo(xspi);
-		} else {
-			/* Transfer is completed */
+		if (!xspi->tx_bytes) {
+			/* Fixed delay due to controller limitation with
+			 * RX_NEMPTY incorrect status
+			 * Xilinx AR:65885 contains more details
+			 */
+			udelay(10);
+			cdns_spi_read_rx_fifo(xspi, xspi->rx_bytes);
 			cdns_spi_write(xspi, CDNS_SPI_IDR,
 				       CDNS_SPI_IXR_DEFAULT);
-			spi_finalize_current_transfer(master);
+			spi_finalize_current_transfer(ctlr);
 		}
 		status = IRQ_HANDLED;
 	}
@@ -389,37 +418,47 @@ static irqreturn_t cdns_spi_irq(int irq, void *dev_id)
 	return status;
 }
 
-static int cdns_prepare_message(struct spi_master *master,
+static int cdns_prepare_message(struct spi_controller *ctlr,
 				struct spi_message *msg)
 {
-	cdns_spi_config_clock_mode(msg->spi);
+	if (!spi_controller_is_slave(ctlr))
+		cdns_spi_config_clock_mode(msg->spi);
 	return 0;
 }
 
 /**
  * cdns_transfer_one - Initiates the SPI transfer
- * @master:	Pointer to spi_master structure
+ * @ctlr:	Pointer to spi_controller structure
  * @spi:	Pointer to the spi_device structure
  * @transfer:	Pointer to the spi_transfer structure which provides
  *		information about next transfer parameters
  *
- * This function fills the TX FIFO, starts the SPI transfer and
+ * This function in master mode fills the TX FIFO, starts the SPI transfer and
  * returns a positive transfer count so that core will wait for completion.
+ * This function in slave mode fills the TX FIFO and wait for transfer trigger.
  *
  * Return:	Number of bytes transferred in the last transfer
  */
-static int cdns_transfer_one(struct spi_master *master,
+static int cdns_transfer_one(struct spi_controller *ctlr,
 			     struct spi_device *spi,
 			     struct spi_transfer *transfer)
 {
-	struct cdns_spi *xspi = spi_master_get_devdata(master);
+	struct cdns_spi *xspi = spi_controller_get_devdata(ctlr);
 
 	xspi->txbuf = transfer->tx_buf;
 	xspi->rxbuf = transfer->rx_buf;
 	xspi->tx_bytes = transfer->len;
 	xspi->rx_bytes = transfer->len;
 
-	cdns_spi_setup_transfer(spi, transfer);
+	if (!spi_controller_is_slave(ctlr))
+		cdns_spi_setup_transfer(spi, transfer);
+
+	/* Set TX empty threshold to half of FIFO depth
+	 * only if TX bytes are more than half FIFO depth.
+	 */
+	if (xspi->tx_bytes > (xspi->tx_fifo_depth >> 1))
+		cdns_spi_write(xspi, CDNS_SPI_THLD, xspi->tx_fifo_depth >> 1);
+
 	cdns_spi_fill_tx_fifo(xspi);
 	spi_transfer_delay_exec(transfer);
 
@@ -429,16 +468,16 @@ static int cdns_transfer_one(struct spi_master *master,
 
 /**
  * cdns_prepare_transfer_hardware - Prepares hardware for transfer.
- * @master:	Pointer to the spi_master structure which provides
+ * @ctlr:	Pointer to the spi_controller structure which provides
  *		information about the controller.
  *
  * This function enables SPI master controller.
  *
  * Return:	0 always
  */
-static int cdns_prepare_transfer_hardware(struct spi_master *master)
+static int cdns_prepare_transfer_hardware(struct spi_controller *ctlr)
 {
-	struct cdns_spi *xspi = spi_master_get_devdata(master);
+	struct cdns_spi *xspi = spi_controller_get_devdata(ctlr);
 
 	cdns_spi_write(xspi, CDNS_SPI_ER, CDNS_SPI_ER_ENABLE);
 
@@ -447,24 +486,33 @@ static int cdns_prepare_transfer_hardware(struct spi_master *master)
 
 /**
  * cdns_unprepare_transfer_hardware - Relaxes hardware after transfer
- * @master:	Pointer to the spi_master structure which provides
+ * @ctlr:	Pointer to the spi_controller structure which provides
  *		information about the controller.
  *
  * This function disables the SPI master controller when no slave selected.
+ * This function flush out if any pending data in FIFO.
  *
  * Return:	0 always
  */
-static int cdns_unprepare_transfer_hardware(struct spi_master *master)
+static int cdns_unprepare_transfer_hardware(struct spi_controller *ctlr)
 {
-	struct cdns_spi *xspi = spi_master_get_devdata(master);
+	struct cdns_spi *xspi = spi_controller_get_devdata(ctlr);
 	u32 ctrl_reg;
+	unsigned int cnt = xspi->tx_fifo_depth;
+
+	if (spi_controller_is_slave(ctlr)) {
+		while (cnt--)
+			cdns_spi_read(xspi, CDNS_SPI_RXD);
+	}
 
 	/* Disable the SPI if slave is deselected */
 	ctrl_reg = cdns_spi_read(xspi, CDNS_SPI_CR);
 	ctrl_reg = (ctrl_reg & CDNS_SPI_CR_SSCTRL) >>  CDNS_SPI_SS_SHIFT;
-	if (ctrl_reg == CDNS_SPI_NOSS)
+	if (ctrl_reg == CDNS_SPI_NOSS || spi_controller_is_slave(ctlr))
 		cdns_spi_write(xspi, CDNS_SPI_ER, CDNS_SPI_ER_DISABLE);
 
+	/* Reset to default */
+	cdns_spi_write(xspi, CDNS_SPI_THLD, 0x1);
 	return 0;
 }
 
@@ -487,6 +535,27 @@ static void cdns_spi_detect_fifo_depth(struct cdns_spi *xspi)
 }
 
 /**
+ * cdns_slave_abort - Abort slave transfer
+ * @ctlr:	Pointer to the spi_controller structure
+ *
+ * This function abort slave transfer if there any transfer timeout.
+ *
+ * Return:      0 always
+ */
+static int cdns_slave_abort(struct spi_controller *ctlr)
+{
+	struct cdns_spi *xspi = spi_controller_get_devdata(ctlr);
+	u32 intr_status;
+
+	intr_status = cdns_spi_read(xspi, CDNS_SPI_ISR);
+	cdns_spi_write(xspi, CDNS_SPI_ISR, intr_status);
+	cdns_spi_write(xspi, CDNS_SPI_IDR, (CDNS_SPI_IXR_MODF | CDNS_SPI_IXR_RXNEMTY));
+	spi_finalize_current_transfer(ctlr);
+
+	return 0;
+}
+
+/**
  * cdns_spi_probe - Probe method for the SPI driver
  * @pdev:	Pointer to the platform_device structure
  *
@@ -497,71 +566,79 @@ static void cdns_spi_detect_fifo_depth(struct cdns_spi *xspi)
 static int cdns_spi_probe(struct platform_device *pdev)
 {
 	int ret = 0, irq;
-	struct spi_master *master;
+	struct spi_controller *ctlr;
 	struct cdns_spi *xspi;
 	u32 num_cs;
+	bool slave;
 
-	master = spi_alloc_master(&pdev->dev, sizeof(*xspi));
-	if (!master)
+	slave = of_property_read_bool(pdev->dev.of_node, "spi-slave");
+	if (slave)
+		ctlr = spi_alloc_slave(&pdev->dev, sizeof(*xspi));
+	else
+		ctlr = spi_alloc_master(&pdev->dev, sizeof(*xspi));
+
+	if (!ctlr)
 		return -ENOMEM;
 
-	xspi = spi_master_get_devdata(master);
-	master->dev.of_node = pdev->dev.of_node;
-	platform_set_drvdata(pdev, master);
+	xspi = spi_controller_get_devdata(ctlr);
+	ctlr->dev.of_node = pdev->dev.of_node;
+	platform_set_drvdata(pdev, ctlr);
 
 	xspi->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(xspi->regs)) {
 		ret = PTR_ERR(xspi->regs);
-		goto remove_master;
+		goto remove_ctlr;
 	}
 
 	xspi->pclk = devm_clk_get(&pdev->dev, "pclk");
 	if (IS_ERR(xspi->pclk)) {
 		dev_err(&pdev->dev, "pclk clock not found.\n");
 		ret = PTR_ERR(xspi->pclk);
-		goto remove_master;
-	}
-
-	xspi->ref_clk = devm_clk_get(&pdev->dev, "ref_clk");
-	if (IS_ERR(xspi->ref_clk)) {
-		dev_err(&pdev->dev, "ref_clk clock not found.\n");
-		ret = PTR_ERR(xspi->ref_clk);
-		goto remove_master;
+		goto remove_ctlr;
 	}
 
 	ret = clk_prepare_enable(xspi->pclk);
 	if (ret) {
 		dev_err(&pdev->dev, "Unable to enable APB clock.\n");
-		goto remove_master;
+		goto remove_ctlr;
 	}
 
-	ret = clk_prepare_enable(xspi->ref_clk);
-	if (ret) {
-		dev_err(&pdev->dev, "Unable to enable device clock.\n");
-		goto clk_dis_apb;
-	}
+	if (!spi_controller_is_slave(ctlr)) {
+		xspi->ref_clk = devm_clk_get(&pdev->dev, "ref_clk");
+		if (IS_ERR(xspi->ref_clk)) {
+			dev_err(&pdev->dev, "ref_clk clock not found.\n");
+			ret = PTR_ERR(xspi->ref_clk);
+			goto clk_dis_apb;
+		}
 
-	pm_runtime_use_autosuspend(&pdev->dev);
-	pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
-	pm_runtime_get_noresume(&pdev->dev);
-	pm_runtime_set_active(&pdev->dev);
-	pm_runtime_enable(&pdev->dev);
+		ret = clk_prepare_enable(xspi->ref_clk);
+		if (ret) {
+			dev_err(&pdev->dev, "Unable to enable device clock.\n");
+			goto clk_dis_apb;
+		}
 
-	ret = of_property_read_u32(pdev->dev.of_node, "num-cs", &num_cs);
-	if (ret < 0)
-		master->num_chipselect = CDNS_SPI_DEFAULT_NUM_CS;
-	else
-		master->num_chipselect = num_cs;
+		pm_runtime_use_autosuspend(&pdev->dev);
+		pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
+		pm_runtime_get_noresume(&pdev->dev);
+		pm_runtime_set_active(&pdev->dev);
+		pm_runtime_enable(&pdev->dev);
+
+		ret = of_property_read_u32(pdev->dev.of_node, "num-cs", &num_cs);
+		if (ret < 0)
+			ctlr->num_chipselect = CDNS_SPI_DEFAULT_NUM_CS;
+		else
+			ctlr->num_chipselect = num_cs;
 
-	ret = of_property_read_u32(pdev->dev.of_node, "is-decoded-cs",
-				   &xspi->is_decoded_cs);
-	if (ret < 0)
-		xspi->is_decoded_cs = 0;
+		ret = of_property_read_u32(pdev->dev.of_node, "is-decoded-cs",
+					   &xspi->is_decoded_cs);
+		if (ret < 0)
+			xspi->is_decoded_cs = 0;
+	}
 
 	cdns_spi_detect_fifo_depth(xspi);
 
 	/* SPI controller initializations */
-	cdns_spi_init_hw(xspi);
+	cdns_spi_init_hw(xspi, spi_controller_is_slave(ctlr));
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq <= 0) {
@@ -570,48 +647,53 @@ static int cdns_spi_probe(struct platform_device *pdev)
 	}
 
 	ret = devm_request_irq(&pdev->dev, irq, cdns_spi_irq,
-			       0, pdev->name, master);
+			       0, pdev->name, ctlr);
 	if (ret != 0) {
 		ret = -ENXIO;
 		dev_err(&pdev->dev, "request_irq failed\n");
 		goto clk_dis_all;
 	}
 
-	master->use_gpio_descriptors = true;
-	master->prepare_transfer_hardware = cdns_prepare_transfer_hardware;
-	master->prepare_message = cdns_prepare_message;
-	master->transfer_one = cdns_transfer_one;
-	master->unprepare_transfer_hardware = cdns_unprepare_transfer_hardware;
-	master->set_cs = cdns_spi_chipselect;
-	master->auto_runtime_pm = true;
-	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
-
-	xspi->clk_rate = clk_get_rate(xspi->ref_clk);
-	/* Set to default valid value */
-	master->max_speed_hz = xspi->clk_rate / 4;
-	xspi->speed_hz = master->max_speed_hz;
-
-	master->bits_per_word_mask = SPI_BPW_MASK(8);
-
-	pm_runtime_mark_last_busy(&pdev->dev);
-	pm_runtime_put_autosuspend(&pdev->dev);
-
-	ret = spi_register_master(master);
+	ctlr->use_gpio_descriptors = true;
+	ctlr->prepare_transfer_hardware = cdns_prepare_transfer_hardware;
+	ctlr->prepare_message = cdns_prepare_message;
+	ctlr->transfer_one = cdns_transfer_one;
+	ctlr->unprepare_transfer_hardware = cdns_unprepare_transfer_hardware;
+	ctlr->mode_bits = SPI_CPOL | SPI_CPHA;
+	ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
+
+	if (!spi_controller_is_slave(ctlr)) {
+		ctlr->mode_bits |=  SPI_CS_HIGH;
+		ctlr->set_cs = cdns_spi_chipselect;
+		ctlr->auto_runtime_pm = true;
+		xspi->clk_rate = clk_get_rate(xspi->ref_clk);
+		/* Set to default valid value */
+		ctlr->max_speed_hz = xspi->clk_rate / 4;
+		xspi->speed_hz = ctlr->max_speed_hz;
+		pm_runtime_mark_last_busy(&pdev->dev);
+		pm_runtime_put_autosuspend(&pdev->dev);
+	} else {
+		ctlr->mode_bits |= SPI_NO_CS;
+		ctlr->slave_abort = cdns_slave_abort;
+	}
+	ret = spi_register_controller(ctlr);
 	if (ret) {
-		dev_err(&pdev->dev, "spi_register_master failed\n");
+		dev_err(&pdev->dev, "spi_register_controller failed\n");
 		goto clk_dis_all;
 	}
 
 	return ret;
 
 clk_dis_all:
-	pm_runtime_set_suspended(&pdev->dev);
-	pm_runtime_disable(&pdev->dev);
-	clk_disable_unprepare(xspi->ref_clk);
+	if (!spi_controller_is_slave(ctlr)) {
+		pm_runtime_set_suspended(&pdev->dev);
+		pm_runtime_disable(&pdev->dev);
+		clk_disable_unprepare(xspi->ref_clk);
+	}
 clk_dis_apb:
 	clk_disable_unprepare(xspi->pclk);
-remove_master:
-	spi_master_put(master);
+remove_ctlr:
+	spi_controller_put(ctlr);
 	return ret;
 }
 
@@ -625,10 +707,10 @@ remove_master:
  *
  * Return:	0 on success and error value on error
  */
-static int cdns_spi_remove(struct platform_device *pdev)
+static void cdns_spi_remove(struct platform_device *pdev)
 {
-	struct spi_master *master = platform_get_drvdata(pdev);
-	struct cdns_spi *xspi = spi_master_get_devdata(master);
+	struct spi_controller *ctlr = platform_get_drvdata(pdev);
+	struct cdns_spi *xspi = spi_controller_get_devdata(ctlr);
 
 	cdns_spi_write(xspi, CDNS_SPI_ER, CDNS_SPI_ER_DISABLE);
 
@@ -637,9 +719,7 @@ static int cdns_spi_remove(struct platform_device *pdev)
 	pm_runtime_set_suspended(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
-	spi_unregister_master(master);
-
-	return 0;
+	spi_unregister_controller(ctlr);
 }
 
 /**
@@ -653,9 +733,9 @@ static int cdns_spi_remove(struct platform_device *pdev)
  */
 static int __maybe_unused cdns_spi_suspend(struct device *dev)
 {
-	struct spi_master *master = dev_get_drvdata(dev);
+	struct spi_controller *ctlr = dev_get_drvdata(dev);
 
-	return spi_master_suspend(master);
+	return spi_controller_suspend(ctlr);
 }
 
 /**
@@ -668,11 +748,11 @@ static int __maybe_unused cdns_spi_suspend(struct device *dev)
  */
 static int __maybe_unused cdns_spi_resume(struct device *dev)
 {
-	struct spi_master *master = dev_get_drvdata(dev);
-	struct cdns_spi *xspi = spi_master_get_devdata(master);
+	struct spi_controller *ctlr = dev_get_drvdata(dev);
+	struct cdns_spi *xspi = spi_controller_get_devdata(ctlr);
 
-	cdns_spi_init_hw(xspi);
-	return spi_master_resume(master);
+	cdns_spi_init_hw(xspi, spi_controller_is_slave(ctlr));
+	return spi_controller_resume(ctlr);
 }
 
 /**
@@ -685,8 +765,8 @@ static int __maybe_unused cdns_spi_resume(struct device *dev)
  */
 static int __maybe_unused cdns_spi_runtime_resume(struct device *dev)
 {
-	struct spi_master *master = dev_get_drvdata(dev);
-	struct cdns_spi *xspi = spi_master_get_devdata(master);
+	struct spi_controller *ctlr = dev_get_drvdata(dev);
+	struct cdns_spi *xspi = spi_controller_get_devdata(ctlr);
 	int ret;
 
 	ret = clk_prepare_enable(xspi->pclk);
@@ -714,8 +794,8 @@ static int __maybe_unused cdns_spi_runtime_resume(struct device *dev)
  */
 static int __maybe_unused cdns_spi_runtime_suspend(struct device *dev)
 {
-	struct spi_master *master = dev_get_drvdata(dev);
-	struct cdns_spi *xspi = spi_master_get_devdata(master);
+	struct spi_controller *ctlr = dev_get_drvdata(dev);
+	struct cdns_spi *xspi = spi_controller_get_devdata(ctlr);
 
 	clk_disable_unprepare(xspi->ref_clk);
 	clk_disable_unprepare(xspi->pclk);
@@ -739,7 +819,7 @@ MODULE_DEVICE_TABLE(of, cdns_spi_of_match);
 /* cdns_spi_driver - This structure defines the SPI subsystem platform driver */
 static struct platform_driver cdns_spi_driver = {
 	.probe	= cdns_spi_probe,
-	.remove	= cdns_spi_remove,
+	.remove_new = cdns_spi_remove,
 	.driver = {
 		.name = CDNS_SPI_NAME,
 		.of_match_table = cdns_spi_of_match,
diff --git a/drivers/spi/spi-cavium-octeon.c b/drivers/spi/spi-cavium-octeon.c
index 1a2de6ce9064..58060be33106 100644
--- a/drivers/spi/spi-cavium-octeon.c
+++ b/drivers/spi/spi-cavium-octeon.c
@@ -69,15 +69,13 @@ fail:
 	return err;
 }
 
-static int octeon_spi_remove(struct platform_device *pdev)
+static void octeon_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct octeon_spi *p = spi_master_get_devdata(master);
 
 	/* Clear the CSENA* and put everything in a known state. */
 	writeq(0, p->register_base + OCTEON_SPI_CFG(p));
-
-	return 0;
 }
 
 static const struct of_device_id octeon_spi_match[] = {
@@ -92,7 +90,7 @@ static struct platform_driver octeon_spi_driver = {
 		.of_match_table = octeon_spi_match,
 	},
 	.probe		= octeon_spi_probe,
-	.remove		= octeon_spi_remove,
+	.remove_new	= octeon_spi_remove,
 };
 
 module_platform_driver(octeon_spi_driver);
diff --git a/drivers/spi/spi-cavium.c b/drivers/spi/spi-cavium.c
index 6854c3ce423b..dfe224defd6e 100644
--- a/drivers/spi/spi-cavium.c
+++ b/drivers/spi/spi-cavium.c
@@ -57,8 +57,8 @@ static int octeon_spi_do_transfer(struct octeon_spi *p,
 	mpi_cfg.s.cslate = cpha ? 1 : 0;
 	mpi_cfg.s.enable = 1;
 
-	if (spi->chip_select < 4)
-		p->cs_enax |= 1ull << (12 + spi->chip_select);
+	if (spi_get_chipselect(spi, 0) < 4)
+		p->cs_enax |= 1ull << (12 + spi_get_chipselect(spi, 0));
 	mpi_cfg.u64 |= p->cs_enax;
 
 	if (mpi_cfg.u64 != p->last_cfg) {
@@ -78,7 +78,7 @@ static int octeon_spi_do_transfer(struct octeon_spi *p,
 			writeq(d, p->register_base + OCTEON_SPI_DAT0(p) + (8 * i));
 		}
 		mpi_tx.u64 = 0;
-		mpi_tx.s.csid = spi->chip_select;
+		mpi_tx.s.csid = spi_get_chipselect(spi, 0);
 		mpi_tx.s.leavecs = 1;
 		mpi_tx.s.txnum = tx_buf ? OCTEON_SPI_MAX_BYTES : 0;
 		mpi_tx.s.totnum = OCTEON_SPI_MAX_BYTES;
@@ -103,7 +103,7 @@ static int octeon_spi_do_transfer(struct octeon_spi *p,
 	}
 
 	mpi_tx.u64 = 0;
-	mpi_tx.s.csid = spi->chip_select;
+	mpi_tx.s.csid = spi_get_chipselect(spi, 0);
 	if (last_xfer)
 		mpi_tx.s.leavecs = xfer->cs_change;
 	else
diff --git a/drivers/spi/spi-coldfire-qspi.c b/drivers/spi/spi-coldfire-qspi.c
index 263ce9047327..b1bd8a6b5bf9 100644
--- a/drivers/spi/spi-coldfire-qspi.c
+++ b/drivers/spi/spi-coldfire-qspi.c
@@ -290,9 +290,9 @@ static void mcfqspi_set_cs(struct spi_device *spi, bool enable)
 	bool cs_high = spi->mode & SPI_CS_HIGH;
 
 	if (enable)
-		mcfqspi_cs_select(mcfqspi, spi->chip_select, cs_high);
+		mcfqspi_cs_select(mcfqspi, spi_get_chipselect(spi, 0), cs_high);
 	else
-		mcfqspi_cs_deselect(mcfqspi, spi->chip_select, cs_high);
+		mcfqspi_cs_deselect(mcfqspi, spi_get_chipselect(spi, 0), cs_high);
 }
 
 static int mcfqspi_transfer_one(struct spi_master *master,
@@ -324,11 +324,11 @@ static int mcfqspi_transfer_one(struct spi_master *master,
 static int mcfqspi_setup(struct spi_device *spi)
 {
 	mcfqspi_cs_deselect(spi_master_get_devdata(spi->master),
-			    spi->chip_select, spi->mode & SPI_CS_HIGH);
+			    spi_get_chipselect(spi, 0), spi->mode & SPI_CS_HIGH);
 
 	dev_dbg(&spi->dev,
 			"bits per word %d, chip select %d, speed %d KHz\n",
-			spi->bits_per_word, spi->chip_select,
+			spi->bits_per_word, spi_get_chipselect(spi, 0),
 			(MCFQSPI_BUSCLK / mcfqspi_qmr_baud(spi->max_speed_hz))
 			/ 1000);
 
@@ -434,7 +434,7 @@ fail0:
 	return status;
 }
 
-static int mcfqspi_remove(struct platform_device *pdev)
+static void mcfqspi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct mcfqspi *mcfqspi = spi_master_get_devdata(master);
@@ -445,8 +445,6 @@ static int mcfqspi_remove(struct platform_device *pdev)
 
 	mcfqspi_cs_teardown(mcfqspi);
 	clk_disable_unprepare(mcfqspi->clk);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -509,7 +507,7 @@ static struct platform_driver mcfqspi_driver = {
 	.driver.owner	= THIS_MODULE,
 	.driver.pm	= &mcfqspi_pm,
 	.probe		= mcfqspi_probe,
-	.remove		= mcfqspi_remove,
+	.remove_new	= mcfqspi_remove,
 };
 module_platform_driver(mcfqspi_driver);
 
diff --git a/drivers/spi/spi-davinci.c b/drivers/spi/spi-davinci.c
index d112c2cac042..b04811c911e2 100644
--- a/drivers/spi/spi-davinci.c
+++ b/drivers/spi/spi-davinci.c
@@ -199,7 +199,7 @@ static void davinci_spi_chipselect(struct spi_device *spi, int value)
 {
 	struct davinci_spi *dspi;
 	struct davinci_spi_config *spicfg = spi->controller_data;
-	u8 chip_sel = spi->chip_select;
+	u8 chip_sel = spi_get_chipselect(spi, 0);
 	u16 spidat1 = CS_DEFAULT;
 
 	dspi = spi_master_get_devdata(spi->master);
@@ -212,11 +212,11 @@ static void davinci_spi_chipselect(struct spi_device *spi, int value)
 	 * Board specific chip select logic decides the polarity and cs
 	 * line for the controller
 	 */
-	if (spi->cs_gpiod) {
+	if (spi_get_csgpiod(spi, 0)) {
 		if (value == BITBANG_CS_ACTIVE)
-			gpiod_set_value(spi->cs_gpiod, 1);
+			gpiod_set_value(spi_get_csgpiod(spi, 0), 1);
 		else
-			gpiod_set_value(spi->cs_gpiod, 0);
+			gpiod_set_value(spi_get_csgpiod(spi, 0), 0);
 	} else {
 		if (value == BITBANG_CS_ACTIVE) {
 			if (!(spi->mode & SPI_CS_WORD))
@@ -293,11 +293,11 @@ static int davinci_spi_setup_transfer(struct spi_device *spi,
 	if (bits_per_word <= 8) {
 		dspi->get_rx = davinci_spi_rx_buf_u8;
 		dspi->get_tx = davinci_spi_tx_buf_u8;
-		dspi->bytes_per_word[spi->chip_select] = 1;
+		dspi->bytes_per_word[spi_get_chipselect(spi, 0)] = 1;
 	} else {
 		dspi->get_rx = davinci_spi_rx_buf_u16;
 		dspi->get_tx = davinci_spi_tx_buf_u16;
-		dspi->bytes_per_word[spi->chip_select] = 2;
+		dspi->bytes_per_word[spi_get_chipselect(spi, 0)] = 2;
 	}
 
 	if (!hz)
@@ -415,11 +415,11 @@ static int davinci_spi_setup(struct spi_device *spi)
 	dspi = spi_master_get_devdata(spi->master);
 
 	if (!(spi->mode & SPI_NO_CS)) {
-		if (np && spi->cs_gpiod)
+		if (np && spi_get_csgpiod(spi, 0))
 			internal_cs = false;
 
 		if (internal_cs)
-			set_io_bits(dspi->base + SPIPC0, 1 << spi->chip_select);
+			set_io_bits(dspi->base + SPIPC0, 1 << spi_get_chipselect(spi, 0));
 	}
 
 	if (spi->mode & SPI_READY)
@@ -579,7 +579,7 @@ static int davinci_spi_bufs(struct spi_device *spi, struct spi_transfer *t)
 		spicfg = &davinci_spi_default_cfg;
 
 	/* convert len to words based on bits_per_word */
-	data_type = dspi->bytes_per_word[spi->chip_select];
+	data_type = dspi->bytes_per_word[spi_get_chipselect(spi, 0)];
 
 	dspi->tx = t->tx_buf;
 	dspi->rx = t->rx_buf;
@@ -1018,7 +1018,7 @@ err:
  * It will also call spi_bitbang_stop to destroy the work queue which was
  * created by spi_bitbang_start.
  */
-static int davinci_spi_remove(struct platform_device *pdev)
+static void davinci_spi_remove(struct platform_device *pdev)
 {
 	struct davinci_spi *dspi;
 	struct spi_master *master;
@@ -1036,7 +1036,6 @@ static int davinci_spi_remove(struct platform_device *pdev)
 	}
 
 	spi_master_put(master);
-	return 0;
 }
 
 static struct platform_driver davinci_spi_driver = {
@@ -1045,7 +1044,7 @@ static struct platform_driver davinci_spi_driver = {
 		.of_match_table = of_match_ptr(davinci_spi_of_match),
 	},
 	.probe = davinci_spi_probe,
-	.remove = davinci_spi_remove,
+	.remove_new = davinci_spi_remove,
 };
 module_platform_driver(davinci_spi_driver);
 
diff --git a/drivers/spi/spi-dln2.c b/drivers/spi/spi-dln2.c
index 0a1fb2bc9e54..6bd93c47853c 100644
--- a/drivers/spi/spi-dln2.c
+++ b/drivers/spi/spi-dln2.c
@@ -596,12 +596,12 @@ static int dln2_spi_prepare_message(struct spi_master *master,
 	struct dln2_spi *dln2 = spi_master_get_devdata(master);
 	struct spi_device *spi = message->spi;
 
-	if (dln2->cs != spi->chip_select) {
-		ret = dln2_spi_cs_set_one(dln2, spi->chip_select);
+	if (dln2->cs != spi_get_chipselect(spi, 0)) {
+		ret = dln2_spi_cs_set_one(dln2, spi_get_chipselect(spi, 0));
 		if (ret < 0)
 			return ret;
 
-		dln2->cs = spi->chip_select;
+		dln2->cs = spi_get_chipselect(spi, 0);
 	}
 
 	return 0;
@@ -781,7 +781,7 @@ exit_free_master:
 	return ret;
 }
 
-static int dln2_spi_remove(struct platform_device *pdev)
+static void dln2_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct dln2_spi *dln2 = spi_master_get_devdata(master);
@@ -790,8 +790,6 @@ static int dln2_spi_remove(struct platform_device *pdev)
 
 	if (dln2_spi_enable(dln2, false) < 0)
 		dev_err(&pdev->dev, "Failed to disable SPI module\n");
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -873,7 +871,7 @@ static struct platform_driver spi_dln2_driver = {
 		.pm	= &dln2_spi_pm,
 	},
 	.probe		= dln2_spi_probe,
-	.remove		= dln2_spi_remove,
+	.remove_new	= dln2_spi_remove,
 };
 module_platform_driver(spi_dln2_driver);
 
diff --git a/drivers/spi/spi-dw-bt1.c b/drivers/spi/spi-dw-bt1.c
index 3fb89dee595e..5e1c01822967 100644
--- a/drivers/spi/spi-dw-bt1.c
+++ b/drivers/spi/spi-dw-bt1.c
@@ -308,7 +308,7 @@ err_disable_clk:
 	return ret;
 }
 
-static int dw_spi_bt1_remove(struct platform_device *pdev)
+static void dw_spi_bt1_remove(struct platform_device *pdev)
 {
 	struct dw_spi_bt1 *dwsbt1 = platform_get_drvdata(pdev);
 
@@ -317,8 +317,6 @@ static int dw_spi_bt1_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 
 	clk_disable_unprepare(dwsbt1->clk);
-
-	return 0;
 }
 
 static const struct of_device_id dw_spi_bt1_of_match[] = {
@@ -330,7 +328,7 @@ MODULE_DEVICE_TABLE(of, dw_spi_bt1_of_match);
 
 static struct platform_driver dw_spi_bt1_driver = {
 	.probe	= dw_spi_bt1_probe,
-	.remove	= dw_spi_bt1_remove,
+	.remove_new = dw_spi_bt1_remove,
 	.driver	= {
 		.name		= "bt1-sys-ssi",
 		.of_match_table	= dw_spi_bt1_of_match,
diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c
index c3bfb6c84cab..ae3108c70f50 100644
--- a/drivers/spi/spi-dw-core.c
+++ b/drivers/spi/spi-dw-core.c
@@ -103,7 +103,7 @@ void dw_spi_set_cs(struct spi_device *spi, bool enable)
 	 * support active-high or active-low CS level.
 	 */
 	if (cs_high == enable)
-		dw_writel(dws, DW_SPI_SER, BIT(spi->chip_select));
+		dw_writel(dws, DW_SPI_SER, BIT(spi_get_chipselect(spi, 0)));
 	else
 		dw_writel(dws, DW_SPI_SER, 0);
 }
diff --git a/drivers/spi/spi-dw-mmio.c b/drivers/spi/spi-dw-mmio.c
index 26c40ea6dd12..5e6faa98aa85 100644
--- a/drivers/spi/spi-dw-mmio.c
+++ b/drivers/spi/spi-dw-mmio.c
@@ -54,6 +54,20 @@ struct dw_spi_mscc {
 };
 
 /*
+ * Elba SoC does not use ssi, pin override is used for cs 0,1 and
+ * gpios for cs 2,3 as defined in the device tree.
+ *
+ * cs:  |       1               0
+ * bit: |---3-------2-------1-------0
+ *      |  cs1   cs1_ovr   cs0   cs0_ovr
+ */
+#define ELBA_SPICS_REG			0x2468
+#define ELBA_SPICS_OFFSET(cs)		((cs) << 1)
+#define ELBA_SPICS_MASK(cs)		(GENMASK(1, 0) << ELBA_SPICS_OFFSET(cs))
+#define ELBA_SPICS_SET(cs, val)		\
+		((((val) << 1) | BIT(0)) << ELBA_SPICS_OFFSET(cs))
+
+/*
  * The Designware SPI controller (referred to as master in the documentation)
  * automatically deasserts chip select when the tx fifo is empty. The chip
  * selects then needs to be either driven as GPIOs or, for the first 4 using
@@ -65,7 +79,7 @@ static void dw_spi_mscc_set_cs(struct spi_device *spi, bool enable)
 	struct dw_spi *dws = spi_master_get_devdata(spi->master);
 	struct dw_spi_mmio *dwsmmio = container_of(dws, struct dw_spi_mmio, dws);
 	struct dw_spi_mscc *dwsmscc = dwsmmio->priv;
-	u32 cs = spi->chip_select;
+	u32 cs = spi_get_chipselect(spi, 0);
 
 	if (cs < 4) {
 		u32 sw_mode = MSCC_SPI_MST_SW_MODE_SW_PIN_CTRL_MODE;
@@ -138,7 +152,7 @@ static void dw_spi_sparx5_set_cs(struct spi_device *spi, bool enable)
 	struct dw_spi *dws = spi_master_get_devdata(spi->master);
 	struct dw_spi_mmio *dwsmmio = container_of(dws, struct dw_spi_mmio, dws);
 	struct dw_spi_mscc *dwsmscc = dwsmmio->priv;
-	u8 cs = spi->chip_select;
+	u8 cs = spi_get_chipselect(spi, 0);
 
 	if (!enable) {
 		/* CS override drive enable */
@@ -237,6 +251,49 @@ static int dw_spi_canaan_k210_init(struct platform_device *pdev,
 	return 0;
 }
 
+static void dw_spi_elba_override_cs(struct regmap *syscon, int cs, int enable)
+{
+	regmap_update_bits(syscon, ELBA_SPICS_REG, ELBA_SPICS_MASK(cs),
+			   ELBA_SPICS_SET(cs, enable));
+}
+
+static void dw_spi_elba_set_cs(struct spi_device *spi, bool enable)
+{
+	struct dw_spi *dws = spi_master_get_devdata(spi->master);
+	struct dw_spi_mmio *dwsmmio = container_of(dws, struct dw_spi_mmio, dws);
+	struct regmap *syscon = dwsmmio->priv;
+	u8 cs;
+
+	cs = spi->chip_select;
+	if (cs < 2)
+		dw_spi_elba_override_cs(syscon, spi->chip_select, enable);
+
+	/*
+	 * The DW SPI controller needs a native CS bit selected to start
+	 * the serial engine.
+	 */
+	spi->chip_select = 0;
+	dw_spi_set_cs(spi, enable);
+	spi->chip_select = cs;
+}
+
+static int dw_spi_elba_init(struct platform_device *pdev,
+			    struct dw_spi_mmio *dwsmmio)
+{
+	struct regmap *syscon;
+
+	syscon = syscon_regmap_lookup_by_phandle(dev_of_node(&pdev->dev),
+						 "amd,pensando-elba-syscon");
+	if (IS_ERR(syscon))
+		return dev_err_probe(&pdev->dev, PTR_ERR(syscon),
+				     "syscon regmap lookup failed\n");
+
+	dwsmmio->priv = syscon;
+	dwsmmio->dws.set_cs = dw_spi_elba_set_cs;
+
+	return 0;
+}
+
 static int dw_spi_mmio_probe(struct platform_device *pdev)
 {
 	int (*init_func)(struct platform_device *pdev,
@@ -328,7 +385,7 @@ out_clk:
 	return ret;
 }
 
-static int dw_spi_mmio_remove(struct platform_device *pdev)
+static void dw_spi_mmio_remove(struct platform_device *pdev)
 {
 	struct dw_spi_mmio *dwsmmio = platform_get_drvdata(pdev);
 
@@ -337,8 +394,6 @@ static int dw_spi_mmio_remove(struct platform_device *pdev)
 	clk_disable_unprepare(dwsmmio->pclk);
 	clk_disable_unprepare(dwsmmio->clk);
 	reset_control_assert(dwsmmio->rstc);
-
-	return 0;
 }
 
 static const struct of_device_id dw_spi_mmio_of_match[] = {
@@ -352,6 +407,7 @@ static const struct of_device_id dw_spi_mmio_of_match[] = {
 	{ .compatible = "intel,thunderbay-ssi", .data = dw_spi_intel_init},
 	{ .compatible = "microchip,sparx5-spi", dw_spi_mscc_sparx5_init},
 	{ .compatible = "canaan,k210-spi", dw_spi_canaan_k210_init},
+	{ .compatible = "amd,pensando-elba-spi", .data = dw_spi_elba_init},
 	{ /* end of table */}
 };
 MODULE_DEVICE_TABLE(of, dw_spi_mmio_of_match);
@@ -366,7 +422,7 @@ MODULE_DEVICE_TABLE(acpi, dw_spi_mmio_acpi_match);
 
 static struct platform_driver dw_spi_mmio_driver = {
 	.probe		= dw_spi_mmio_probe,
-	.remove		= dw_spi_mmio_remove,
+	.remove_new	= dw_spi_mmio_remove,
 	.driver		= {
 		.name	= DRIVER_NAME,
 		.of_match_table = dw_spi_mmio_of_match,
diff --git a/drivers/spi/spi-ep93xx.c b/drivers/spi/spi-ep93xx.c
index 5896a7b2fade..1615fd22f9a2 100644
--- a/drivers/spi/spi-ep93xx.c
+++ b/drivers/spi/spi-ep93xx.c
@@ -745,14 +745,12 @@ fail_release_master:
 	return error;
 }
 
-static int ep93xx_spi_remove(struct platform_device *pdev)
+static void ep93xx_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct ep93xx_spi *espi = spi_master_get_devdata(master);
 
 	ep93xx_spi_release_dma(espi);
-
-	return 0;
 }
 
 static struct platform_driver ep93xx_spi_driver = {
@@ -760,7 +758,7 @@ static struct platform_driver ep93xx_spi_driver = {
 		.name	= "ep93xx-spi",
 	},
 	.probe		= ep93xx_spi_probe,
-	.remove		= ep93xx_spi_remove,
+	.remove_new	= ep93xx_spi_remove,
 };
 module_platform_driver(ep93xx_spi_driver);
 
diff --git a/drivers/spi/spi-falcon.c b/drivers/spi/spi-falcon.c
index a7d4dffac66b..4c103dff0d44 100644
--- a/drivers/spi/spi-falcon.c
+++ b/drivers/spi/spi-falcon.c
@@ -131,7 +131,7 @@ int falcon_sflash_xfer(struct spi_device *spi, struct spi_transfer *t,
 				 * especially alen and dumlen.
 				 */
 
-				priv->sfcmd = ((spi->chip_select
+				priv->sfcmd = ((spi_get_chipselect(spi, 0)
 						<< SFCMD_CS_OFFSET)
 					       & SFCMD_CS_MASK);
 				priv->sfcmd |= SFCMD_KEEP_CS_KEEP_SELECTED;
diff --git a/drivers/spi/spi-fsi.c b/drivers/spi/spi-fsi.c
index cf1e4f9ebd72..ba3b17d7c9ec 100644
--- a/drivers/spi/spi-fsi.c
+++ b/drivers/spi/spi-fsi.c
@@ -425,7 +425,7 @@ static int fsi_spi_transfer_one_message(struct spi_controller *ctlr,
 					struct spi_message *mesg)
 {
 	int rc;
-	u8 seq_slave = SPI_FSI_SEQUENCE_SEL_SLAVE(mesg->spi->chip_select + 1);
+	u8 seq_slave = SPI_FSI_SEQUENCE_SEL_SLAVE(spi_get_chipselect(mesg->spi, 0) + 1);
 	unsigned int len;
 	struct spi_transfer *transfer;
 	struct fsi_spi *ctx = spi_controller_get_devdata(ctlr);
diff --git a/drivers/spi/spi-fsl-cpm.c b/drivers/spi/spi-fsl-cpm.c
index 17a44d4f5021..38452089e8f3 100644
--- a/drivers/spi/spi-fsl-cpm.c
+++ b/drivers/spi/spi-fsl-cpm.c
@@ -21,6 +21,7 @@
 #include <linux/spi/spi.h>
 #include <linux/types.h>
 #include <linux/platform_device.h>
+#include <linux/byteorder/generic.h>
 
 #include "spi-fsl-cpm.h"
 #include "spi-fsl-lib.h"
@@ -120,6 +121,21 @@ int fsl_spi_cpm_bufs(struct mpc8xxx_spi *mspi,
 		mspi->rx_dma = mspi->dma_dummy_rx;
 		mspi->map_rx_dma = 0;
 	}
+	if (t->bits_per_word == 16 && t->tx_buf) {
+		const u16 *src = t->tx_buf;
+		u16 *dst;
+		int i;
+
+		dst = kmalloc(t->len, GFP_KERNEL);
+		if (!dst)
+			return -ENOMEM;
+
+		for (i = 0; i < t->len >> 1; i++)
+			dst[i] = cpu_to_le16p(src + i);
+
+		mspi->tx = dst;
+		mspi->map_tx_dma = 1;
+	}
 
 	if (mspi->map_tx_dma) {
 		void *nonconst_tx = (void *)mspi->tx; /* shut up gcc */
@@ -173,6 +189,13 @@ void fsl_spi_cpm_bufs_complete(struct mpc8xxx_spi *mspi)
 	if (mspi->map_rx_dma)
 		dma_unmap_single(dev, mspi->rx_dma, t->len, DMA_FROM_DEVICE);
 	mspi->xfer_in_progress = NULL;
+
+	if (t->bits_per_word == 16 && t->rx_buf) {
+		int i;
+
+		for (i = 0; i < t->len; i += 2)
+			le16_to_cpus(t->rx_buf + i);
+	}
 }
 EXPORT_SYMBOL_GPL(fsl_spi_cpm_bufs_complete);
 
diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c
index e419642eb10e..4339485d202c 100644
--- a/drivers/spi/spi-fsl-dspi.c
+++ b/drivers/spi/spi-fsl-dspi.c
@@ -902,19 +902,19 @@ static irqreturn_t dspi_interrupt(int irq, void *dev_id)
 
 static void dspi_assert_cs(struct spi_device *spi, bool *cs)
 {
-	if (!spi->cs_gpiod || *cs)
+	if (!spi_get_csgpiod(spi, 0) || *cs)
 		return;
 
-	gpiod_set_value_cansleep(spi->cs_gpiod, true);
+	gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), true);
 	*cs = true;
 }
 
 static void dspi_deassert_cs(struct spi_device *spi, bool *cs)
 {
-	if (!spi->cs_gpiod || !*cs)
+	if (!spi_get_csgpiod(spi, 0) || !*cs)
 		return;
 
-	gpiod_set_value_cansleep(spi->cs_gpiod, false);
+	gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), false);
 	*cs = false;
 }
 
@@ -938,8 +938,8 @@ static int dspi_transfer_one_message(struct spi_controller *ctlr,
 
 		/* Prepare command word for CMD FIFO */
 		dspi->tx_cmd = SPI_PUSHR_CMD_CTAS(0);
-		if (!spi->cs_gpiod)
-			dspi->tx_cmd |= SPI_PUSHR_CMD_PCS(spi->chip_select);
+		if (!spi_get_csgpiod(spi, 0))
+			dspi->tx_cmd |= SPI_PUSHR_CMD_PCS(spi_get_chipselect(spi, 0));
 
 		if (list_is_last(&dspi->cur_transfer->transfer_list,
 				 &dspi->cur_msg->transfers)) {
@@ -1058,7 +1058,7 @@ static int dspi_setup(struct spi_device *spi)
 			chip->ctar_val |= SPI_CTAR_LSBFE;
 	}
 
-	gpiod_direction_output(spi->cs_gpiod, false);
+	gpiod_direction_output(spi_get_csgpiod(spi, 0), false);
 	dspi_deassert_cs(spi, &cs);
 
 	spi_set_ctldata(spi, chip);
@@ -1068,10 +1068,10 @@ static int dspi_setup(struct spi_device *spi)
 
 static void dspi_cleanup(struct spi_device *spi)
 {
-	struct chip_data *chip = spi_get_ctldata((struct spi_device *)spi);
+	struct chip_data *chip = spi_get_ctldata(spi);
 
 	dev_dbg(&spi->dev, "spi_device %u.%u cleanup\n",
-		spi->controller->bus_num, spi->chip_select);
+		spi->controller->bus_num, spi_get_chipselect(spi, 0));
 
 	kfree(chip);
 }
@@ -1425,7 +1425,7 @@ out_ctlr_put:
 	return ret;
 }
 
-static int dspi_remove(struct platform_device *pdev)
+static void dspi_remove(struct platform_device *pdev)
 {
 	struct fsl_dspi *dspi = platform_get_drvdata(pdev);
 
@@ -1444,8 +1444,6 @@ static int dspi_remove(struct platform_device *pdev)
 	if (dspi->irq)
 		free_irq(dspi->irq, dspi);
 	clk_disable_unprepare(dspi->clk);
-
-	return 0;
 }
 
 static void dspi_shutdown(struct platform_device *pdev)
@@ -1459,7 +1457,7 @@ static struct platform_driver fsl_dspi_driver = {
 	.driver.owner		= THIS_MODULE,
 	.driver.pm		= &dspi_pm,
 	.probe			= dspi_probe,
-	.remove			= dspi_remove,
+	.remove_new		= dspi_remove,
 	.shutdown		= dspi_shutdown,
 };
 module_platform_driver(fsl_dspi_driver);
diff --git a/drivers/spi/spi-fsl-espi.c b/drivers/spi/spi-fsl-espi.c
index f7066bef7b06..b3d2d3db5850 100644
--- a/drivers/spi/spi-fsl-espi.c
+++ b/drivers/spi/spi-fsl-espi.c
@@ -345,7 +345,7 @@ static void fsl_espi_setup_transfer(struct spi_device *spi,
 
 	/* don't write the mode register if the mode doesn't change */
 	if (cs->hw_mode != hw_mode_old)
-		fsl_espi_write_reg(espi, ESPI_SPMODEx(spi->chip_select),
+		fsl_espi_write_reg(espi, ESPI_SPMODEx(spi_get_chipselect(spi, 0)),
 				   cs->hw_mode);
 }
 
@@ -359,7 +359,7 @@ static int fsl_espi_bufs(struct spi_device *spi, struct spi_transfer *t)
 	reinit_completion(&espi->done);
 
 	/* Set SPCOM[CS] and SPCOM[TRANLEN] field */
-	spcom = SPCOM_CS(spi->chip_select);
+	spcom = SPCOM_CS(spi_get_chipselect(spi, 0));
 	spcom |= SPCOM_TRANLEN(t->len - 1);
 
 	/* configure RXSKIP mode */
@@ -492,7 +492,7 @@ static int fsl_espi_setup(struct spi_device *spi)
 
 	pm_runtime_get_sync(espi->dev);
 
-	cs->hw_mode = fsl_espi_read_reg(espi, ESPI_SPMODEx(spi->chip_select));
+	cs->hw_mode = fsl_espi_read_reg(espi, ESPI_SPMODEx(spi_get_chipselect(spi, 0)));
 	/* mask out bits we are going to set */
 	cs->hw_mode &= ~(CSMODE_CP_BEGIN_EDGECLK | CSMODE_CI_INACTIVEHIGH
 			 | CSMODE_REV);
@@ -783,11 +783,9 @@ static int of_fsl_espi_probe(struct platform_device *ofdev)
 	return fsl_espi_probe(dev, &mem, irq, num_cs);
 }
 
-static int of_fsl_espi_remove(struct platform_device *dev)
+static void of_fsl_espi_remove(struct platform_device *dev)
 {
 	pm_runtime_disable(&dev->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -837,7 +835,7 @@ static struct platform_driver fsl_espi_driver = {
 		.pm = &espi_pm,
 	},
 	.probe		= of_fsl_espi_probe,
-	.remove		= of_fsl_espi_remove,
+	.remove_new	= of_fsl_espi_remove,
 };
 module_platform_driver(fsl_espi_driver);
 
diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c
index 34488de55587..f2341ab99556 100644
--- a/drivers/spi/spi-fsl-lpspi.c
+++ b/drivers/spi/spi-fsl-lpspi.c
@@ -425,7 +425,7 @@ static int fsl_lpspi_setup_transfer(struct spi_controller *controller,
 	if (fsl_lpspi->is_only_cs1)
 		fsl_lpspi->config.chip_select = 1;
 	else
-		fsl_lpspi->config.chip_select = spi->chip_select;
+		fsl_lpspi->config.chip_select = spi_get_chipselect(spi, 0);
 
 	if (!fsl_lpspi->config.speed_hz)
 		fsl_lpspi->config.speed_hz = spi->max_speed_hz;
@@ -937,7 +937,7 @@ out_controller_put:
 	return ret;
 }
 
-static int fsl_lpspi_remove(struct platform_device *pdev)
+static void fsl_lpspi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *controller = platform_get_drvdata(pdev);
 	struct fsl_lpspi_data *fsl_lpspi =
@@ -946,7 +946,6 @@ static int fsl_lpspi_remove(struct platform_device *pdev)
 	fsl_lpspi_dma_exit(controller);
 
 	pm_runtime_disable(fsl_lpspi->dev);
-	return 0;
 }
 
 static int __maybe_unused fsl_lpspi_suspend(struct device *dev)
@@ -983,7 +982,7 @@ static struct platform_driver fsl_lpspi_driver = {
 		.pm = &fsl_lpspi_pm_ops,
 	},
 	.probe = fsl_lpspi_probe,
-	.remove = fsl_lpspi_remove,
+	.remove_new = fsl_lpspi_remove,
 };
 module_platform_driver(fsl_lpspi_driver);
 
diff --git a/drivers/spi/spi-fsl-qspi.c b/drivers/spi/spi-fsl-qspi.c
index 85cc71ba624a..8ade61e5ebc0 100644
--- a/drivers/spi/spi-fsl-qspi.c
+++ b/drivers/spi/spi-fsl-qspi.c
@@ -528,7 +528,7 @@ static void fsl_qspi_select_mem(struct fsl_qspi *q, struct spi_device *spi)
 	unsigned long rate = spi->max_speed_hz;
 	int ret;
 
-	if (q->selected == spi->chip_select)
+	if (q->selected == spi_get_chipselect(spi, 0))
 		return;
 
 	if (needs_4x_clock(q))
@@ -544,7 +544,7 @@ static void fsl_qspi_select_mem(struct fsl_qspi *q, struct spi_device *spi)
 	if (ret)
 		return;
 
-	q->selected = spi->chip_select;
+	q->selected = spi_get_chipselect(spi, 0);
 
 	fsl_qspi_invalidate(q);
 }
@@ -823,7 +823,7 @@ static const char *fsl_qspi_get_name(struct spi_mem *mem)
 
 	name = devm_kasprintf(dev, GFP_KERNEL,
 			      "%s-%d", dev_name(q->dev),
-			      mem->spi->chip_select);
+			      spi_get_chipselect(mem->spi, 0));
 
 	if (!name) {
 		dev_err(dev, "failed to get memory for custom flash name\n");
@@ -948,7 +948,7 @@ err_put_ctrl:
 	return ret;
 }
 
-static int fsl_qspi_remove(struct platform_device *pdev)
+static void fsl_qspi_remove(struct platform_device *pdev)
 {
 	struct fsl_qspi *q = platform_get_drvdata(pdev);
 
@@ -959,8 +959,6 @@ static int fsl_qspi_remove(struct platform_device *pdev)
 	fsl_qspi_clk_disable_unprep(q);
 
 	mutex_destroy(&q->lock);
-
-	return 0;
 }
 
 static int fsl_qspi_suspend(struct device *dev)
@@ -1000,7 +998,7 @@ static struct platform_driver fsl_qspi_driver = {
 		.pm =   &fsl_qspi_pm_ops,
 	},
 	.probe          = fsl_qspi_probe,
-	.remove		= fsl_qspi_remove,
+	.remove_new	= fsl_qspi_remove,
 };
 module_platform_driver(fsl_qspi_driver);
 
diff --git a/drivers/spi/spi-fsl-spi.c b/drivers/spi/spi-fsl-spi.c
index 93152144fd2e..106fe60a0a50 100644
--- a/drivers/spi/spi-fsl-spi.c
+++ b/drivers/spi/spi-fsl-spi.c
@@ -145,10 +145,10 @@ static void fsl_spi_grlib_set_shifts(u32 *rx_shift, u32 *tx_shift,
 	}
 }
 
-static int mspi_apply_cpu_mode_quirks(struct spi_mpc8xxx_cs *cs,
-				struct spi_device *spi,
-				struct mpc8xxx_spi *mpc8xxx_spi,
-				int bits_per_word)
+static void mspi_apply_cpu_mode_quirks(struct spi_mpc8xxx_cs *cs,
+				       struct spi_device *spi,
+				       struct mpc8xxx_spi *mpc8xxx_spi,
+				       int bits_per_word)
 {
 	cs->rx_shift = 0;
 	cs->tx_shift = 0;
@@ -161,8 +161,7 @@ static int mspi_apply_cpu_mode_quirks(struct spi_mpc8xxx_cs *cs,
 	} else if (bits_per_word <= 32) {
 		cs->get_rx = mpc8xxx_spi_rx_buf_u32;
 		cs->get_tx = mpc8xxx_spi_tx_buf_u32;
-	} else
-		return -EINVAL;
+	}
 
 	if (mpc8xxx_spi->set_shifts)
 		mpc8xxx_spi->set_shifts(&cs->rx_shift, &cs->tx_shift,
@@ -173,26 +172,6 @@ static int mspi_apply_cpu_mode_quirks(struct spi_mpc8xxx_cs *cs,
 	mpc8xxx_spi->tx_shift = cs->tx_shift;
 	mpc8xxx_spi->get_rx = cs->get_rx;
 	mpc8xxx_spi->get_tx = cs->get_tx;
-
-	return bits_per_word;
-}
-
-static int mspi_apply_qe_mode_quirks(struct spi_mpc8xxx_cs *cs,
-				struct spi_device *spi,
-				int bits_per_word)
-{
-	/* QE uses Little Endian for words > 8
-	 * so transform all words > 8 into 8 bits
-	 * Unfortnatly that doesn't work for LSB so
-	 * reject these for now */
-	/* Note: 32 bits word, LSB works iff
-	 * tfcr/rfcr is set to CPMFCR_GBL */
-	if (spi->mode & SPI_LSB_FIRST &&
-	    bits_per_word > 8)
-		return -EINVAL;
-	if (bits_per_word > 8)
-		return 8; /* pretend its 8 bits */
-	return bits_per_word;
 }
 
 static int fsl_spi_setup_transfer(struct spi_device *spi,
@@ -219,15 +198,7 @@ static int fsl_spi_setup_transfer(struct spi_device *spi,
 		hz = spi->max_speed_hz;
 
 	if (!(mpc8xxx_spi->flags & SPI_CPM_MODE))
-		bits_per_word = mspi_apply_cpu_mode_quirks(cs, spi,
-							   mpc8xxx_spi,
-							   bits_per_word);
-	else if (mpc8xxx_spi->flags & SPI_QE)
-		bits_per_word = mspi_apply_qe_mode_quirks(cs, spi,
-							  bits_per_word);
-
-	if (bits_per_word < 0)
-		return bits_per_word;
+		mspi_apply_cpu_mode_quirks(cs, spi, mpc8xxx_spi, bits_per_word);
 
 	if (bits_per_word == 32)
 		bits_per_word = 0;
@@ -292,18 +263,10 @@ static int fsl_spi_bufs(struct spi_device *spi, struct spi_transfer *t,
 	if (t->bits_per_word)
 		bits_per_word = t->bits_per_word;
 
-	if (bits_per_word > 8) {
-		/* invalid length? */
-		if (len & 1)
-			return -EINVAL;
+	if (bits_per_word > 8)
 		len /= 2;
-	}
-	if (bits_per_word > 16) {
-		/* invalid length? */
-		if (len & 1)
-			return -EINVAL;
+	if (bits_per_word > 16)
 		len /= 2;
-	}
 
 	mpc8xxx_spi->tx = t->tx_buf;
 	mpc8xxx_spi->rx = t->rx_buf;
@@ -359,6 +322,22 @@ static int fsl_spi_prepare_message(struct spi_controller *ctlr,
 				t->bits_per_word = 32;
 			else if ((t->len & 1) == 0)
 				t->bits_per_word = 16;
+		} else {
+			/*
+			 * CPM/QE uses Little Endian for words > 8
+			 * so transform 16 and 32 bits words into 8 bits
+			 * Unfortnatly that doesn't work for LSB so
+			 * reject these for now
+			 * Note: 32 bits word, LSB works iff
+			 * tfcr/rfcr is set to CPMFCR_GBL
+			 */
+			if (m->spi->mode & SPI_LSB_FIRST && t->bits_per_word > 8)
+				return -EINVAL;
+			if (t->bits_per_word == 16 || t->bits_per_word == 32)
+				t->bits_per_word = 8; /* pretend its 8 bits */
+			if (t->bits_per_word == 8 && t->len >= 256 &&
+			    (mpc8xxx_spi->flags & SPI_CPM1))
+				t->bits_per_word = 16;
 		}
 	}
 	return fsl_spi_setup_transfer(m->spi, first);
@@ -503,7 +482,7 @@ static void fsl_spi_grlib_cs_control(struct spi_device *spi, bool on)
 	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(spi->master);
 	struct fsl_spi_reg __iomem *reg_base = mpc8xxx_spi->reg_base;
 	u32 slvsel;
-	u16 cs = spi->chip_select;
+	u16 cs = spi_get_chipselect(spi, 0);
 
 	if (cs < mpc8xxx_spi->native_chipselects) {
 		slvsel = mpc8xxx_spi_read_reg(&reg_base->slvsel);
@@ -592,8 +571,14 @@ static struct spi_master *fsl_spi_probe(struct device *dev,
 	if (mpc8xxx_spi->type == TYPE_GRLIB)
 		fsl_spi_grlib_probe(dev);
 
-	master->bits_per_word_mask =
-		(SPI_BPW_RANGE_MASK(4, 16) | SPI_BPW_MASK(32)) &
+	if (mpc8xxx_spi->flags & SPI_CPM_MODE)
+		master->bits_per_word_mask =
+			(SPI_BPW_RANGE_MASK(4, 8) | SPI_BPW_MASK(16) | SPI_BPW_MASK(32));
+	else
+		master->bits_per_word_mask =
+			(SPI_BPW_RANGE_MASK(4, 16) | SPI_BPW_MASK(32));
+
+	master->bits_per_word_mask &=
 		SPI_BPW_RANGE_MASK(1, mpc8xxx_spi->max_bits_per_word);
 
 	if (mpc8xxx_spi->flags & SPI_QE_CPU_MODE)
@@ -716,13 +701,12 @@ unmap_out:
 	return ret;
 }
 
-static int of_fsl_spi_remove(struct platform_device *ofdev)
+static void of_fsl_spi_remove(struct platform_device *ofdev)
 {
 	struct spi_master *master = platform_get_drvdata(ofdev);
 	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master);
 
 	fsl_spi_cpm_free(mpc8xxx_spi);
-	return 0;
 }
 
 static struct platform_driver of_fsl_spi_driver = {
@@ -731,7 +715,7 @@ static struct platform_driver of_fsl_spi_driver = {
 		.of_match_table = of_fsl_spi_match,
 	},
 	.probe		= of_fsl_spi_probe,
-	.remove		= of_fsl_spi_remove,
+	.remove_new	= of_fsl_spi_remove,
 };
 
 #ifdef CONFIG_MPC832x_RDB
@@ -763,20 +747,18 @@ static int plat_mpc8xxx_spi_probe(struct platform_device *pdev)
 	return PTR_ERR_OR_ZERO(master);
 }
 
-static int plat_mpc8xxx_spi_remove(struct platform_device *pdev)
+static void plat_mpc8xxx_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master);
 
 	fsl_spi_cpm_free(mpc8xxx_spi);
-
-	return 0;
 }
 
 MODULE_ALIAS("platform:mpc8xxx_spi");
 static struct platform_driver mpc8xxx_spi_driver = {
 	.probe = plat_mpc8xxx_spi_probe,
-	.remove = plat_mpc8xxx_spi_remove,
+	.remove_new = plat_mpc8xxx_spi_remove,
 	.driver = {
 		.name = "mpc8xxx_spi",
 	},
diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c
index babb039bcb43..ba7be505ec4e 100644
--- a/drivers/spi/spi-geni-qcom.c
+++ b/drivers/spi/spi-geni-qcom.c
@@ -391,9 +391,9 @@ static int setup_fifo_params(struct spi_device *spi_slv,
 			cpha = CPHA;
 
 		if (spi_slv->mode & SPI_CS_HIGH)
-			demux_output_inv = BIT(spi_slv->chip_select);
+			demux_output_inv = BIT(spi_get_chipselect(spi_slv, 0));
 
-		demux_sel = spi_slv->chip_select;
+		demux_sel = spi_get_chipselect(spi_slv, 0);
 		mas->cur_bits_per_word = spi_slv->bits_per_word;
 
 		spi_setup_word_len(mas, spi_slv->mode, spi_slv->bits_per_word);
@@ -469,7 +469,7 @@ static int setup_gsi_xfer(struct spi_transfer *xfer, struct spi_geni_master *mas
 	peripheral.loopback_en = !!(spi_slv->mode & SPI_LOOP);
 	peripheral.clock_pol_high = !!(spi_slv->mode & SPI_CPOL);
 	peripheral.data_pol_high = !!(spi_slv->mode & SPI_CPHA);
-	peripheral.cs = spi_slv->chip_select;
+	peripheral.cs = spi_get_chipselect(spi_slv, 0);
 	peripheral.pack_en = true;
 	peripheral.word_len = xfer->bits_per_word - MIN_WORD_LEN;
 
@@ -1114,7 +1114,7 @@ spi_geni_probe_runtime_disable:
 	return ret;
 }
 
-static int spi_geni_remove(struct platform_device *pdev)
+static void spi_geni_remove(struct platform_device *pdev)
 {
 	struct spi_master *spi = platform_get_drvdata(pdev);
 	struct spi_geni_master *mas = spi_master_get_devdata(spi);
@@ -1126,7 +1126,6 @@ static int spi_geni_remove(struct platform_device *pdev)
 
 	free_irq(mas->irq, spi);
 	pm_runtime_disable(&pdev->dev);
-	return 0;
 }
 
 static int __maybe_unused spi_geni_runtime_suspend(struct device *dev)
@@ -1208,7 +1207,7 @@ MODULE_DEVICE_TABLE(of, spi_geni_dt_match);
 
 static struct platform_driver spi_geni_driver = {
 	.probe  = spi_geni_probe,
-	.remove = spi_geni_remove,
+	.remove_new = spi_geni_remove,
 	.driver = {
 		.name = "geni_spi",
 		.pm = &spi_geni_pm_ops,
diff --git a/drivers/spi/spi-gpio.c b/drivers/spi/spi-gpio.c
index 9c8c7948044e..092afc7679d4 100644
--- a/drivers/spi/spi-gpio.c
+++ b/drivers/spi/spi-gpio.c
@@ -230,7 +230,7 @@ static void spi_gpio_chipselect(struct spi_device *spi, int is_active)
 
 	/* Drive chip select line, if we have one */
 	if (spi_gpio->cs_gpios) {
-		struct gpio_desc *cs = spi_gpio->cs_gpios[spi->chip_select];
+		struct gpio_desc *cs = spi_gpio->cs_gpios[spi_get_chipselect(spi, 0)];
 
 		/* SPI chip selects are normally active-low */
 		gpiod_set_value_cansleep(cs, (spi->mode & SPI_CS_HIGH) ? is_active : !is_active);
@@ -248,7 +248,7 @@ static int spi_gpio_setup(struct spi_device *spi)
 	 * initialized from the descriptor lookup.
 	 */
 	if (spi_gpio->cs_gpios) {
-		cs = spi_gpio->cs_gpios[spi->chip_select];
+		cs = spi_gpio->cs_gpios[spi_get_chipselect(spi, 0)];
 		if (!spi->controller_state && cs)
 			status = gpiod_direction_output(cs,
 						  !(spi->mode & SPI_CS_HIGH));
diff --git a/drivers/spi/spi-gxp.c b/drivers/spi/spi-gxp.c
index c900c2f39b57..684d63f402f3 100644
--- a/drivers/spi/spi-gxp.c
+++ b/drivers/spi/spi-gxp.c
@@ -201,7 +201,7 @@ static ssize_t gxp_spi_write(struct gxp_spi_chip *chip, const struct spi_mem_op
 static int do_gxp_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
 {
 	struct gxp_spi *spifi = spi_controller_get_devdata(mem->spi->master);
-	struct gxp_spi_chip *chip = &spifi->chips[mem->spi->chip_select];
+	struct gxp_spi_chip *chip = &spifi->chips[spi_get_chipselect(mem->spi, 0)];
 	int ret;
 
 	if (op->data.dir == SPI_MEM_DATA_IN) {
@@ -237,7 +237,7 @@ static const struct spi_controller_mem_ops gxp_spi_mem_ops = {
 static int gxp_spi_setup(struct spi_device *spi)
 {
 	struct gxp_spi *spifi = spi_controller_get_devdata(spi->master);
-	unsigned int cs = spi->chip_select;
+	unsigned int cs = spi_get_chipselect(spi, 0);
 	struct gxp_spi_chip *chip = &spifi->chips[cs];
 
 	chip->spifi = spifi;
diff --git a/drivers/spi/spi-hisi-kunpeng.c b/drivers/spi/spi-hisi-kunpeng.c
index 525cc0143a30..524eadbef87b 100644
--- a/drivers/spi/spi-hisi-kunpeng.c
+++ b/drivers/spi/spi-hisi-kunpeng.c
@@ -523,15 +523,13 @@ static int hisi_spi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int hisi_spi_remove(struct platform_device *pdev)
+static void hisi_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *master = platform_get_drvdata(pdev);
 	struct hisi_spi *hs = spi_controller_get_devdata(master);
 
 	debugfs_remove_recursive(hs->debugfs);
 	spi_unregister_controller(master);
-
-	return 0;
 }
 
 static const struct acpi_device_id hisi_spi_acpi_match[] = {
@@ -542,7 +540,7 @@ MODULE_DEVICE_TABLE(acpi, hisi_spi_acpi_match);
 
 static struct platform_driver hisi_spi_driver = {
 	.probe		= hisi_spi_probe,
-	.remove		= hisi_spi_remove,
+	.remove_new	= hisi_spi_remove,
 	.driver		= {
 		.name	= "hisi-kunpeng-spi",
 		.acpi_match_table = hisi_spi_acpi_match,
diff --git a/drivers/spi/spi-hisi-sfc-v3xx.c b/drivers/spi/spi-hisi-sfc-v3xx.c
index f07d1045a30a..7cbcb065bb44 100644
--- a/drivers/spi/spi-hisi-sfc-v3xx.c
+++ b/drivers/spi/spi-hisi-sfc-v3xx.c
@@ -361,7 +361,7 @@ static int hisi_sfc_v3xx_exec_op(struct spi_mem *mem,
 {
 	struct hisi_sfc_v3xx_host *host;
 	struct spi_device *spi = mem->spi;
-	u8 chip_select = spi->chip_select;
+	u8 chip_select = spi_get_chipselect(spi, 0);
 
 	host = spi_controller_get_devdata(spi->master);
 
diff --git a/drivers/spi/spi-img-spfi.c b/drivers/spi/spi-img-spfi.c
index 257046f843ff..d775f87770e3 100644
--- a/drivers/spi/spi-img-spfi.c
+++ b/drivers/spi/spi-img-spfi.c
@@ -413,15 +413,15 @@ static int img_spfi_prepare(struct spi_master *master, struct spi_message *msg)
 	val = spfi_readl(spfi, SPFI_PORT_STATE);
 	val &= ~(SPFI_PORT_STATE_DEV_SEL_MASK <<
 		 SPFI_PORT_STATE_DEV_SEL_SHIFT);
-	val |= msg->spi->chip_select << SPFI_PORT_STATE_DEV_SEL_SHIFT;
+	val |= spi_get_chipselect(msg->spi, 0) << SPFI_PORT_STATE_DEV_SEL_SHIFT;
 	if (msg->spi->mode & SPI_CPHA)
-		val |= SPFI_PORT_STATE_CK_PHASE(msg->spi->chip_select);
+		val |= SPFI_PORT_STATE_CK_PHASE(spi_get_chipselect(msg->spi, 0));
 	else
-		val &= ~SPFI_PORT_STATE_CK_PHASE(msg->spi->chip_select);
+		val &= ~SPFI_PORT_STATE_CK_PHASE(spi_get_chipselect(msg->spi, 0));
 	if (msg->spi->mode & SPI_CPOL)
-		val |= SPFI_PORT_STATE_CK_POL(msg->spi->chip_select);
+		val |= SPFI_PORT_STATE_CK_POL(spi_get_chipselect(msg->spi, 0));
 	else
-		val &= ~SPFI_PORT_STATE_CK_POL(msg->spi->chip_select);
+		val &= ~SPFI_PORT_STATE_CK_POL(spi_get_chipselect(msg->spi, 0));
 	spfi_writel(spfi, val, SPFI_PORT_STATE);
 
 	return 0;
@@ -450,11 +450,11 @@ static void img_spfi_config(struct spi_master *master, struct spi_device *spi,
 	div = DIV_ROUND_UP(clk_get_rate(spfi->spfi_clk), xfer->speed_hz);
 	div = clamp(512 / (1 << get_count_order(div)), 1, 128);
 
-	val = spfi_readl(spfi, SPFI_DEVICE_PARAMETER(spi->chip_select));
+	val = spfi_readl(spfi, SPFI_DEVICE_PARAMETER(spi_get_chipselect(spi, 0)));
 	val &= ~(SPFI_DEVICE_PARAMETER_BITCLK_MASK <<
 		 SPFI_DEVICE_PARAMETER_BITCLK_SHIFT);
 	val |= div << SPFI_DEVICE_PARAMETER_BITCLK_SHIFT;
-	spfi_writel(spfi, val, SPFI_DEVICE_PARAMETER(spi->chip_select));
+	spfi_writel(spfi, val, SPFI_DEVICE_PARAMETER(spi_get_chipselect(spi, 0)));
 
 	spfi_writel(spfi, xfer->len << SPFI_TRANSACTION_TSIZE_SHIFT,
 		    SPFI_TRANSACTION);
@@ -665,7 +665,7 @@ put_spi:
 	return ret;
 }
 
-static int img_spfi_remove(struct platform_device *pdev)
+static void img_spfi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct img_spfi *spfi = spi_master_get_devdata(master);
@@ -680,8 +680,6 @@ static int img_spfi_remove(struct platform_device *pdev)
 		clk_disable_unprepare(spfi->spfi_clk);
 		clk_disable_unprepare(spfi->sys_clk);
 	}
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -758,7 +756,7 @@ static struct platform_driver img_spfi_driver = {
 		.of_match_table = of_match_ptr(img_spfi_of_match),
 	},
 	.probe = img_spfi_probe,
-	.remove = img_spfi_remove,
+	.remove_new = img_spfi_remove,
 };
 module_platform_driver(img_spfi_driver);
 
diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c
index e4ccd0c329d0..34e5f81ec431 100644
--- a/drivers/spi/spi-imx.c
+++ b/drivers/spi/spi-imx.c
@@ -252,6 +252,18 @@ static bool spi_imx_can_dma(struct spi_controller *controller, struct spi_device
 	return true;
 }
 
+/*
+ * Note the number of natively supported chip selects for MX51 is 4. Some
+ * devices may have less actual SS pins but the register map supports 4. When
+ * using gpio chip selects the cs values passed into the macros below can go
+ * outside the range 0 - 3. We therefore need to limit the cs value to avoid
+ * corrupting bits outside the allocated locations.
+ *
+ * The simplest way to do this is to just mask the cs bits to 2 bits. This
+ * still allows all 4 native chip selects to work as well as gpio chip selects
+ * (which can use any of the 4 chip select configurations).
+ */
+
 #define MX51_ECSPI_CTRL		0x08
 #define MX51_ECSPI_CTRL_ENABLE		(1 <<  0)
 #define MX51_ECSPI_CTRL_XCH		(1 <<  2)
@@ -260,16 +272,16 @@ static bool spi_imx_can_dma(struct spi_controller *controller, struct spi_device
 #define MX51_ECSPI_CTRL_DRCTL(drctl)	((drctl) << 16)
 #define MX51_ECSPI_CTRL_POSTDIV_OFFSET	8
 #define MX51_ECSPI_CTRL_PREDIV_OFFSET	12
-#define MX51_ECSPI_CTRL_CS(cs)		((cs) << 18)
+#define MX51_ECSPI_CTRL_CS(cs)		((cs & 3) << 18)
 #define MX51_ECSPI_CTRL_BL_OFFSET	20
 #define MX51_ECSPI_CTRL_BL_MASK		(0xfff << 20)
 
 #define MX51_ECSPI_CONFIG	0x0c
-#define MX51_ECSPI_CONFIG_SCLKPHA(cs)	(1 << ((cs) +  0))
-#define MX51_ECSPI_CONFIG_SCLKPOL(cs)	(1 << ((cs) +  4))
-#define MX51_ECSPI_CONFIG_SBBCTRL(cs)	(1 << ((cs) +  8))
-#define MX51_ECSPI_CONFIG_SSBPOL(cs)	(1 << ((cs) + 12))
-#define MX51_ECSPI_CONFIG_SCLKCTL(cs)	(1 << ((cs) + 20))
+#define MX51_ECSPI_CONFIG_SCLKPHA(cs)	(1 << ((cs & 3) +  0))
+#define MX51_ECSPI_CONFIG_SCLKPOL(cs)	(1 << ((cs & 3) +  4))
+#define MX51_ECSPI_CONFIG_SBBCTRL(cs)	(1 << ((cs & 3) +  8))
+#define MX51_ECSPI_CONFIG_SSBPOL(cs)	(1 << ((cs & 3) + 12))
+#define MX51_ECSPI_CONFIG_SCLKCTL(cs)	(1 << ((cs & 3) + 20))
 
 #define MX51_ECSPI_INT		0x10
 #define MX51_ECSPI_INT_TEEN		(1 <<  0)
@@ -528,7 +540,7 @@ static int mx51_ecspi_prepare_message(struct spi_imx_data *spi_imx,
 		ctrl |= MX51_ECSPI_CTRL_DRCTL(spi_imx->spi_drctl);
 
 	/* set chip select to use */
-	ctrl |= MX51_ECSPI_CTRL_CS(spi->chip_select);
+	ctrl |= MX51_ECSPI_CTRL_CS(spi_get_chipselect(spi, 0));
 
 	/*
 	 * The ctrl register must be written first, with the EN bit set other
@@ -549,22 +561,22 @@ static int mx51_ecspi_prepare_message(struct spi_imx_data *spi_imx,
 	 * BURST_LENGTH + 1 bits are received
 	 */
 	if (spi_imx->slave_mode && is_imx53_ecspi(spi_imx))
-		cfg &= ~MX51_ECSPI_CONFIG_SBBCTRL(spi->chip_select);
+		cfg &= ~MX51_ECSPI_CONFIG_SBBCTRL(spi_get_chipselect(spi, 0));
 	else
-		cfg |= MX51_ECSPI_CONFIG_SBBCTRL(spi->chip_select);
+		cfg |= MX51_ECSPI_CONFIG_SBBCTRL(spi_get_chipselect(spi, 0));
 
 	if (spi->mode & SPI_CPOL) {
-		cfg |= MX51_ECSPI_CONFIG_SCLKPOL(spi->chip_select);
-		cfg |= MX51_ECSPI_CONFIG_SCLKCTL(spi->chip_select);
+		cfg |= MX51_ECSPI_CONFIG_SCLKPOL(spi_get_chipselect(spi, 0));
+		cfg |= MX51_ECSPI_CONFIG_SCLKCTL(spi_get_chipselect(spi, 0));
 	} else {
-		cfg &= ~MX51_ECSPI_CONFIG_SCLKPOL(spi->chip_select);
-		cfg &= ~MX51_ECSPI_CONFIG_SCLKCTL(spi->chip_select);
+		cfg &= ~MX51_ECSPI_CONFIG_SCLKPOL(spi_get_chipselect(spi, 0));
+		cfg &= ~MX51_ECSPI_CONFIG_SCLKCTL(spi_get_chipselect(spi, 0));
 	}
 
 	if (spi->mode & SPI_CS_HIGH)
-		cfg |= MX51_ECSPI_CONFIG_SSBPOL(spi->chip_select);
+		cfg |= MX51_ECSPI_CONFIG_SSBPOL(spi_get_chipselect(spi, 0));
 	else
-		cfg &= ~MX51_ECSPI_CONFIG_SSBPOL(spi->chip_select);
+		cfg &= ~MX51_ECSPI_CONFIG_SSBPOL(spi_get_chipselect(spi, 0));
 
 	if (cfg == current_cfg)
 		return 0;
@@ -614,9 +626,9 @@ static void mx51_configure_cpha(struct spi_imx_data *spi_imx,
 	cpha ^= flip_cpha;
 
 	if (cpha)
-		cfg |= MX51_ECSPI_CONFIG_SCLKPHA(spi->chip_select);
+		cfg |= MX51_ECSPI_CONFIG_SCLKPHA(spi_get_chipselect(spi, 0));
 	else
-		cfg &= ~MX51_ECSPI_CONFIG_SCLKPHA(spi->chip_select);
+		cfg &= ~MX51_ECSPI_CONFIG_SCLKPHA(spi_get_chipselect(spi, 0));
 
 	writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);
 }
@@ -768,8 +780,8 @@ static int mx31_prepare_transfer(struct spi_imx_data *spi_imx,
 		reg |= MX31_CSPICTRL_POL;
 	if (spi->mode & SPI_CS_HIGH)
 		reg |= MX31_CSPICTRL_SSPOL;
-	if (!spi->cs_gpiod)
-		reg |= (spi->chip_select) <<
+	if (!spi_get_csgpiod(spi, 0))
+		reg |= (spi_get_chipselect(spi, 0)) <<
 			(is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT :
 						  MX31_CSPICTRL_CS_SHIFT);
 
@@ -868,8 +880,8 @@ static int mx21_prepare_transfer(struct spi_imx_data *spi_imx,
 		reg |= MX21_CSPICTRL_POL;
 	if (spi->mode & SPI_CS_HIGH)
 		reg |= MX21_CSPICTRL_SSPOL;
-	if (!spi->cs_gpiod)
-		reg |= spi->chip_select << MX21_CSPICTRL_CS_SHIFT;
+	if (!spi_get_csgpiod(spi, 0))
+		reg |= spi_get_chipselect(spi, 0) << MX21_CSPICTRL_CS_SHIFT;
 
 	writel(reg, spi_imx->base + MXC_CSPICTRL);
 
@@ -1753,8 +1765,7 @@ static int spi_imx_probe(struct platform_device *pdev)
 
 	init_completion(&spi_imx->xfer_done);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	spi_imx->base = devm_ioremap_resource(&pdev->dev, res);
+	spi_imx->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(spi_imx->base)) {
 		ret = PTR_ERR(spi_imx->base);
 		goto out_controller_put;
@@ -1848,7 +1859,7 @@ out_controller_put:
 	return ret;
 }
 
-static int spi_imx_remove(struct platform_device *pdev)
+static void spi_imx_remove(struct platform_device *pdev)
 {
 	struct spi_controller *controller = platform_get_drvdata(pdev);
 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
@@ -1856,21 +1867,17 @@ static int spi_imx_remove(struct platform_device *pdev)
 
 	spi_unregister_controller(controller);
 
-	ret = pm_runtime_resume_and_get(spi_imx->dev);
-	if (ret < 0) {
-		dev_err(spi_imx->dev, "failed to enable clock\n");
-		return ret;
-	}
-
-	writel(0, spi_imx->base + MXC_CSPICTRL);
+	ret = pm_runtime_get_sync(spi_imx->dev);
+	if (ret >= 0)
+		writel(0, spi_imx->base + MXC_CSPICTRL);
+	else
+		dev_warn(spi_imx->dev, "failed to enable clock, skip hw disable\n");
 
 	pm_runtime_dont_use_autosuspend(spi_imx->dev);
 	pm_runtime_put_sync(spi_imx->dev);
 	pm_runtime_disable(spi_imx->dev);
 
 	spi_imx_sdma_exit(spi_imx);
-
-	return 0;
 }
 
 static int __maybe_unused spi_imx_runtime_resume(struct device *dev)
@@ -1932,7 +1939,7 @@ static struct platform_driver spi_imx_driver = {
 		   .pm = &imx_spi_pm,
 	},
 	.probe = spi_imx_probe,
-	.remove = spi_imx_remove,
+	.remove_new = spi_imx_remove,
 };
 module_platform_driver(spi_imx_driver);
 
diff --git a/drivers/spi/spi-ingenic.c b/drivers/spi/spi-ingenic.c
index 713a238bee63..7d4b515a160d 100644
--- a/drivers/spi/spi-ingenic.c
+++ b/drivers/spi/spi-ingenic.c
@@ -263,7 +263,7 @@ static int spi_ingenic_prepare_message(struct spi_controller *ctlr,
 {
 	struct ingenic_spi *priv = spi_controller_get_devdata(ctlr);
 	struct spi_device *spi = message->spi;
-	unsigned int cs = REG_SSICR1_FRMHL << spi->chip_select;
+	unsigned int cs = REG_SSICR1_FRMHL << spi_get_chipselect(spi, 0);
 	unsigned int ssicr0_mask = REG_SSICR0_LOOP | REG_SSICR0_FSEL;
 	unsigned int ssicr1_mask = REG_SSICR1_PHA | REG_SSICR1_POL | cs;
 	unsigned int ssicr0 = 0, ssicr1 = 0;
@@ -282,7 +282,7 @@ static int spi_ingenic_prepare_message(struct spi_controller *ctlr,
 
 	if (spi->mode & SPI_LOOP)
 		ssicr0 |= REG_SSICR0_LOOP;
-	if (spi->chip_select)
+	if (spi_get_chipselect(spi, 0))
 		ssicr0 |= REG_SSICR0_FSEL;
 
 	if (spi->mode & SPI_CPHA)
diff --git a/drivers/spi/spi-intel-pci.c b/drivers/spi/spi-intel-pci.c
index 4d69e320d018..a7381e774b95 100644
--- a/drivers/spi/spi-intel-pci.c
+++ b/drivers/spi/spi-intel-pci.c
@@ -83,6 +83,7 @@ static const struct pci_device_id intel_spi_pci_ids[] = {
 	{ PCI_VDEVICE(INTEL, 0xa2a4), (unsigned long)&cnl_info },
 	{ PCI_VDEVICE(INTEL, 0xa324), (unsigned long)&cnl_info },
 	{ PCI_VDEVICE(INTEL, 0xa3a4), (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0xae23), (unsigned long)&cnl_info },
 	{ },
 };
 MODULE_DEVICE_TABLE(pci, intel_spi_pci_ids);
diff --git a/drivers/spi/spi-intel.c b/drivers/spi/spi-intel.c
index f4679868c49f..bc6d22149e7e 100644
--- a/drivers/spi/spi-intel.c
+++ b/drivers/spi/spi-intel.c
@@ -451,7 +451,7 @@ static u32 intel_spi_chip_addr(const struct intel_spi *ispi,
 	/* Pick up the correct start address */
 	if (!mem)
 		return 0;
-	return mem->spi->chip_select == 1 ? ispi->chip0_size : 0;
+	return (spi_get_chipselect(mem->spi, 0) == 1) ? ispi->chip0_size : 0;
 }
 
 static int intel_spi_read_reg(struct intel_spi *ispi, const struct spi_mem *mem,
diff --git a/drivers/spi/spi-iproc-qspi.c b/drivers/spi/spi-iproc-qspi.c
index 91cf8eb7213c..5980a0dbbccb 100644
--- a/drivers/spi/spi-iproc-qspi.c
+++ b/drivers/spi/spi-iproc-qspi.c
@@ -127,11 +127,9 @@ static int bcm_iproc_probe(struct platform_device *pdev)
 	return bcm_qspi_probe(pdev, soc_intc);
 }
 
-static int bcm_iproc_remove(struct platform_device *pdev)
+static void bcm_iproc_remove(struct platform_device *pdev)
 {
 	bcm_qspi_remove(pdev);
-
-	return 0;
 }
 
 static const struct of_device_id bcm_iproc_of_match[] = {
@@ -143,7 +141,7 @@ MODULE_DEVICE_TABLE(of, bcm_iproc_of_match);
 
 static struct platform_driver bcm_iproc_driver = {
 	.probe			= bcm_iproc_probe,
-	.remove			= bcm_iproc_remove,
+	.remove_new		= bcm_iproc_remove,
 	.driver = {
 		.name		= "bcm_iproc",
 		.pm		= &bcm_qspi_pm_ops,
diff --git a/drivers/spi/spi-jcore.c b/drivers/spi/spi-jcore.c
index 74c8319c29f1..c42a3358e8c9 100644
--- a/drivers/spi/spi-jcore.c
+++ b/drivers/spi/spi-jcore.c
@@ -68,9 +68,9 @@ static void jcore_spi_program(struct jcore_spi *hw)
 static void jcore_spi_chipsel(struct spi_device *spi, bool value)
 {
 	struct jcore_spi *hw = spi_master_get_devdata(spi->master);
-	u32 csbit = 1U << (2 * spi->chip_select);
+	u32 csbit = 1U << (2 * spi_get_chipselect(spi, 0));
 
-	dev_dbg(hw->master->dev.parent, "chipselect %d\n", spi->chip_select);
+	dev_dbg(hw->master->dev.parent, "chipselect %d\n", spi_get_chipselect(spi, 0));
 
 	if (value)
 		hw->cs_reg |= csbit;
diff --git a/drivers/spi/spi-lantiq-ssc.c b/drivers/spi/spi-lantiq-ssc.c
index aae26f62ea87..8d6ecc5d6f70 100644
--- a/drivers/spi/spi-lantiq-ssc.c
+++ b/drivers/spi/spi-lantiq-ssc.c
@@ -388,11 +388,11 @@ static int lantiq_ssc_setup(struct spi_device *spidev)
 {
 	struct spi_master *master = spidev->master;
 	struct lantiq_ssc_spi *spi = spi_master_get_devdata(master);
-	unsigned int cs = spidev->chip_select;
+	unsigned int cs = spi_get_chipselect(spidev, 0);
 	u32 gpocon;
 
 	/* GPIOs are used for CS */
-	if (spidev->cs_gpiod)
+	if (spi_get_csgpiod(spidev, 0))
 		return 0;
 
 	dev_dbg(spi->dev, "using internal chipselect %u\n", cs);
@@ -796,7 +796,7 @@ static void lantiq_ssc_handle_err(struct spi_master *master,
 static void lantiq_ssc_set_cs(struct spi_device *spidev, bool enable)
 {
 	struct lantiq_ssc_spi *spi = spi_master_get_devdata(spidev->master);
-	unsigned int cs = spidev->chip_select;
+	unsigned int cs = spi_get_chipselect(spidev, 0);
 	u32 fgpo;
 
 	if (!!(spidev->mode & SPI_CS_HIGH) == enable)
@@ -1017,7 +1017,7 @@ err_master_put:
 	return err;
 }
 
-static int lantiq_ssc_remove(struct platform_device *pdev)
+static void lantiq_ssc_remove(struct platform_device *pdev)
 {
 	struct lantiq_ssc_spi *spi = platform_get_drvdata(pdev);
 
@@ -1030,13 +1030,11 @@ static int lantiq_ssc_remove(struct platform_device *pdev)
 	destroy_workqueue(spi->wq);
 	clk_disable_unprepare(spi->spi_clk);
 	clk_put(spi->fpi_clk);
-
-	return 0;
 }
 
 static struct platform_driver lantiq_ssc_driver = {
 	.probe = lantiq_ssc_probe,
-	.remove = lantiq_ssc_remove,
+	.remove_new = lantiq_ssc_remove,
 	.driver = {
 		.name = "spi-lantiq-ssc",
 		.of_match_table = lantiq_ssc_match,
diff --git a/drivers/spi/spi-loopback-test.c b/drivers/spi/spi-loopback-test.c
index 313106eb8d40..675a73cf1579 100644
--- a/drivers/spi/spi-loopback-test.c
+++ b/drivers/spi/spi-loopback-test.c
@@ -53,6 +53,12 @@ module_param(no_cs, int, 0);
 MODULE_PARM_DESC(no_cs,
 		 "if set Chip Select (CS) will not be used");
 
+/* run tests only for a specific length */
+static int run_only_iter_len = -1;
+module_param(run_only_iter_len, int, 0);
+MODULE_PARM_DESC(run_only_iter_len,
+		 "only run tests for a length of this number in iterate_len list");
+
 /* run only a specific test */
 static int run_only_test = -1;
 module_param(run_only_test, int, 0);
@@ -1033,6 +1039,8 @@ int spi_test_run_test(struct spi_device *spi, const struct spi_test *test,
 
 	for (idx_len = 0; idx_len < SPI_TEST_MAX_ITERATE &&
 	     (len = test->iterate_len[idx_len]) != -1; idx_len++) {
+		if ((run_only_iter_len > -1) && len != run_only_iter_len)
+			continue;
 		FOR_EACH_ALIGNMENT(tx_align) {
 			FOR_EACH_ALIGNMENT(rx_align) {
 				/* and run the iteration */
diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c
index 701838b6f0c4..edd7430d4c05 100644
--- a/drivers/spi/spi-mem.c
+++ b/drivers/spi/spi-mem.c
@@ -325,7 +325,7 @@ int spi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
 	if (!spi_mem_internal_supports_op(mem, op))
 		return -ENOTSUPP;
 
-	if (ctlr->mem_ops && ctlr->mem_ops->exec_op && !mem->spi->cs_gpiod) {
+	if (ctlr->mem_ops && ctlr->mem_ops->exec_op && !spi_get_csgpiod(mem->spi, 0)) {
 		ret = spi_mem_access_start(mem);
 		if (ret)
 			return ret;
@@ -808,7 +808,7 @@ int spi_mem_poll_status(struct spi_mem *mem,
 	    op->data.dir != SPI_MEM_DATA_IN)
 		return -EINVAL;
 
-	if (ctlr->mem_ops && ctlr->mem_ops->poll_status && !mem->spi->cs_gpiod) {
+	if (ctlr->mem_ops && ctlr->mem_ops->poll_status && !spi_get_csgpiod(mem->spi, 0)) {
 		ret = spi_mem_access_start(mem);
 		if (ret)
 			return ret;
diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c
index d47f2623a60f..141562c882f1 100644
--- a/drivers/spi/spi-meson-spicc.c
+++ b/drivers/spi/spi-meson-spicc.c
@@ -505,7 +505,7 @@ static int meson_spicc_prepare_message(struct spi_master *master,
 		conf |= FIELD_PREP(SPICC_DRCTL_MASK, SPICC_DRCTL_IGNORE);
 
 	/* Select CS */
-	conf |= FIELD_PREP(SPICC_CS_MASK, spi->chip_select);
+	conf |= FIELD_PREP(SPICC_CS_MASK, spi_get_chipselect(spi, 0));
 
 	/* Default 8bit word */
 	conf |= FIELD_PREP(SPICC_BITLENGTH_MASK, 8 - 1);
@@ -910,7 +910,7 @@ out_master:
 	return ret;
 }
 
-static int meson_spicc_remove(struct platform_device *pdev)
+static void meson_spicc_remove(struct platform_device *pdev)
 {
 	struct meson_spicc_device *spicc = platform_get_drvdata(pdev);
 
@@ -921,8 +921,6 @@ static int meson_spicc_remove(struct platform_device *pdev)
 	clk_disable_unprepare(spicc->pclk);
 
 	spi_master_put(spicc->master);
-
-	return 0;
 }
 
 static const struct meson_spicc_data meson_spicc_gx_data = {
@@ -967,7 +965,7 @@ MODULE_DEVICE_TABLE(of, meson_spicc_of_match);
 
 static struct platform_driver meson_spicc_driver = {
 	.probe   = meson_spicc_probe,
-	.remove  = meson_spicc_remove,
+	.remove_new = meson_spicc_remove,
 	.driver  = {
 		.name = "meson-spicc",
 		.of_match_table = of_match_ptr(meson_spicc_of_match),
diff --git a/drivers/spi/spi-meson-spifc.c b/drivers/spi/spi-meson-spifc.c
index c8ed7815c4ba..06626f406f68 100644
--- a/drivers/spi/spi-meson-spifc.c
+++ b/drivers/spi/spi-meson-spifc.c
@@ -355,7 +355,7 @@ out_err:
 	return ret;
 }
 
-static int meson_spifc_remove(struct platform_device *pdev)
+static void meson_spifc_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct meson_spifc *spifc = spi_master_get_devdata(master);
@@ -363,8 +363,6 @@ static int meson_spifc_remove(struct platform_device *pdev)
 	pm_runtime_get_sync(&pdev->dev);
 	clk_disable_unprepare(spifc->clk);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -442,7 +440,7 @@ MODULE_DEVICE_TABLE(of, meson_spifc_dt_match);
 
 static struct platform_driver meson_spifc_driver = {
 	.probe	= meson_spifc_probe,
-	.remove	= meson_spifc_remove,
+	.remove_new = meson_spifc_remove,
 	.driver	= {
 		.name		= "meson-spifc",
 		.of_match_table	= of_match_ptr(meson_spifc_dt_match),
diff --git a/drivers/spi/spi-microchip-core-qspi.c b/drivers/spi/spi-microchip-core-qspi.c
index 19a6a46829f6..4f76ddf97b10 100644
--- a/drivers/spi/spi-microchip-core-qspi.c
+++ b/drivers/spi/spi-microchip-core-qspi.c
@@ -566,7 +566,7 @@ out:
 	return ret;
 }
 
-static int mchp_coreqspi_remove(struct platform_device *pdev)
+static void mchp_coreqspi_remove(struct platform_device *pdev)
 {
 	struct mchp_coreqspi *qspi = platform_get_drvdata(pdev);
 	u32 control = readl_relaxed(qspi->regs + REG_CONTROL);
@@ -575,8 +575,6 @@ static int mchp_coreqspi_remove(struct platform_device *pdev)
 	control &= ~CONTROL_ENABLE;
 	writel_relaxed(control, qspi->regs + REG_CONTROL);
 	clk_disable_unprepare(qspi->clk);
-
-	return 0;
 }
 
 static const struct of_device_id mchp_coreqspi_of_match[] = {
@@ -591,7 +589,7 @@ static struct platform_driver mchp_coreqspi_driver = {
 		.name = "microchip,coreqspi",
 		.of_match_table = mchp_coreqspi_of_match,
 	},
-	.remove = mchp_coreqspi_remove,
+	.remove_new = mchp_coreqspi_remove,
 };
 module_platform_driver(mchp_coreqspi_driver);
 
diff --git a/drivers/spi/spi-microchip-core.c b/drivers/spi/spi-microchip-core.c
index aeaa1da88f39..b59e8a0c5b97 100644
--- a/drivers/spi/spi-microchip-core.c
+++ b/drivers/spi/spi-microchip-core.c
@@ -247,8 +247,8 @@ static void mchp_corespi_set_cs(struct spi_device *spi, bool disable)
 	struct mchp_corespi *corespi = spi_master_get_devdata(spi->master);
 
 	reg = mchp_corespi_read(corespi, REG_SLAVE_SELECT);
-	reg &= ~BIT(spi->chip_select);
-	reg |= !disable << spi->chip_select;
+	reg &= ~BIT(spi_get_chipselect(spi, 0));
+	reg |= !disable << spi_get_chipselect(spi, 0);
 
 	mchp_corespi_write(corespi, REG_SLAVE_SELECT, reg);
 }
@@ -265,7 +265,7 @@ static int mchp_corespi_setup(struct spi_device *spi)
 	 */
 	if (spi->mode & SPI_CS_HIGH) {
 		reg = mchp_corespi_read(corespi, REG_SLAVE_SELECT);
-		reg |= BIT(spi->chip_select);
+		reg |= BIT(spi_get_chipselect(spi, 0));
 		mchp_corespi_write(corespi, REG_SLAVE_SELECT, reg);
 	}
 	return 0;
@@ -566,7 +566,7 @@ static int mchp_corespi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int mchp_corespi_remove(struct platform_device *pdev)
+static void mchp_corespi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master  = platform_get_drvdata(pdev);
 	struct mchp_corespi *spi = spi_master_get_devdata(master);
@@ -574,8 +574,6 @@ static int mchp_corespi_remove(struct platform_device *pdev)
 	mchp_corespi_disable_ints(spi);
 	clk_disable_unprepare(spi->clk);
 	mchp_corespi_disable(spi);
-
-	return 0;
 }
 
 #define MICROCHIP_SPI_PM_OPS (NULL)
@@ -599,7 +597,7 @@ static struct platform_driver mchp_corespi_driver = {
 		.pm = MICROCHIP_SPI_PM_OPS,
 		.of_match_table = of_match_ptr(mchp_corespi_dt_ids),
 	},
-	.remove = mchp_corespi_remove,
+	.remove_new = mchp_corespi_remove,
 };
 module_platform_driver(mchp_corespi_driver);
 MODULE_DESCRIPTION("Microchip coreSPI SPI controller driver");
diff --git a/drivers/spi/spi-mpc512x-psc.c b/drivers/spi/spi-mpc512x-psc.c
index 03630359ce70..99aeef28a477 100644
--- a/drivers/spi/spi-mpc512x-psc.c
+++ b/drivers/spi/spi-mpc512x-psc.c
@@ -14,15 +14,13 @@
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/interrupt.h>
-#include <linux/of_address.h>
-#include <linux/of_irq.h>
-#include <linux/of_platform.h>
 #include <linux/completion.h>
 #include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
 #include <linux/delay.h>
 #include <linux/clk.h>
 #include <linux/spi/spi.h>
-#include <linux/fsl_devices.h>
 #include <asm/mpc52xx_psc.h>
 
 enum {
@@ -51,16 +49,12 @@ enum {
 	__ret; })
 
 struct mpc512x_psc_spi {
-	void (*cs_control)(struct spi_device *spi, bool on);
-
 	/* driver internal data */
 	int type;
 	void __iomem *psc;
 	struct mpc512x_psc_fifo __iomem *fifo;
 	unsigned int irq;
 	u8 bits_per_word;
-	struct clk *clk_mclk;
-	struct clk *clk_ipg;
 	u32 mclk_rate;
 
 	struct completion txisrdone;
@@ -127,27 +121,17 @@ static void mpc512x_psc_spi_activate_cs(struct spi_device *spi)
 	out_be32(psc_addr(mps, ccr), ccr);
 	mps->bits_per_word = cs->bits_per_word;
 
-	if (spi->cs_gpiod) {
-		if (mps->cs_control)
-			/* boardfile override */
-			mps->cs_control(spi, (spi->mode & SPI_CS_HIGH) ? 1 : 0);
-		else
-			/* gpiolib will deal with the inversion */
-			gpiod_set_value(spi->cs_gpiod, 1);
+	if (spi_get_csgpiod(spi, 0)) {
+		/* gpiolib will deal with the inversion */
+		gpiod_set_value(spi_get_csgpiod(spi, 0), 1);
 	}
 }
 
 static void mpc512x_psc_spi_deactivate_cs(struct spi_device *spi)
 {
-	struct mpc512x_psc_spi *mps = spi_master_get_devdata(spi->master);
-
-	if (spi->cs_gpiod) {
-		if (mps->cs_control)
-			/* boardfile override */
-			mps->cs_control(spi, (spi->mode & SPI_CS_HIGH) ? 0 : 1);
-		else
-			/* gpiolib will deal with the inversion */
-			gpiod_set_value(spi->cs_gpiod, 0);
+	if (spi_get_csgpiod(spi, 0)) {
+		/* gpiolib will deal with the inversion */
+		gpiod_set_value(spi_get_csgpiod(spi, 0), 0);
 	}
 }
 
@@ -471,30 +455,22 @@ static irqreturn_t mpc512x_psc_spi_isr(int irq, void *dev_id)
 	return IRQ_NONE;
 }
 
-static int mpc512x_psc_spi_do_probe(struct device *dev, u32 regaddr,
-					      u32 size, unsigned int irq)
+static int mpc512x_psc_spi_of_probe(struct platform_device *pdev)
 {
-	struct fsl_spi_platform_data *pdata = dev_get_platdata(dev);
+	struct device *dev = &pdev->dev;
 	struct mpc512x_psc_spi *mps;
 	struct spi_master *master;
 	int ret;
 	void *tempp;
 	struct clk *clk;
 
-	master = spi_alloc_master(dev, sizeof(*mps));
+	master = devm_spi_alloc_master(dev, sizeof(*mps));
 	if (master == NULL)
 		return -ENOMEM;
 
 	dev_set_drvdata(dev, master);
 	mps = spi_master_get_devdata(master);
-	mps->type = (int)of_device_get_match_data(dev);
-	mps->irq = irq;
-
-	if (pdata) {
-		mps->cs_control = pdata->cs_control;
-		master->bus_num = pdata->bus_num;
-		master->num_chipselect = pdata->max_chipselect;
-	}
+	mps->type = (int)device_get_match_data(dev);
 
 	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_LSB_FIRST;
 	master->setup = mpc512x_psc_spi_setup;
@@ -503,94 +479,41 @@ static int mpc512x_psc_spi_do_probe(struct device *dev, u32 regaddr,
 	master->unprepare_transfer_hardware = mpc512x_psc_spi_unprep_xfer_hw;
 	master->use_gpio_descriptors = true;
 	master->cleanup = mpc512x_psc_spi_cleanup;
-	master->dev.of_node = dev->of_node;
 
-	tempp = devm_ioremap(dev, regaddr, size);
-	if (!tempp) {
-		dev_err(dev, "could not ioremap I/O port range\n");
-		ret = -EFAULT;
-		goto free_master;
-	}
+	device_set_node(&master->dev, dev_fwnode(dev));
+
+	tempp = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
+	if (IS_ERR(tempp))
+		return dev_err_probe(dev, PTR_ERR(tempp), "could not ioremap I/O port range\n");
 	mps->psc = tempp;
 	mps->fifo =
 		(struct mpc512x_psc_fifo *)(tempp + sizeof(struct mpc52xx_psc));
+
+	mps->irq = platform_get_irq(pdev, 0);
+	if (mps->irq < 0)
+		return mps->irq;
+
 	ret = devm_request_irq(dev, mps->irq, mpc512x_psc_spi_isr, IRQF_SHARED,
 				"mpc512x-psc-spi", mps);
 	if (ret)
-		goto free_master;
+		return ret;
 	init_completion(&mps->txisrdone);
 
-	clk = devm_clk_get(dev, "mclk");
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
-		goto free_master;
-	}
-	ret = clk_prepare_enable(clk);
-	if (ret)
-		goto free_master;
-	mps->clk_mclk = clk;
+	clk = devm_clk_get_enabled(dev, "mclk");
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
 	mps->mclk_rate = clk_get_rate(clk);
 
-	clk = devm_clk_get(dev, "ipg");
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
-		goto free_mclk_clock;
-	}
-	ret = clk_prepare_enable(clk);
-	if (ret)
-		goto free_mclk_clock;
-	mps->clk_ipg = clk;
+	clk = devm_clk_get_enabled(dev, "ipg");
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
 
 	ret = mpc512x_psc_spi_port_config(master, mps);
 	if (ret < 0)
-		goto free_ipg_clock;
-
-	ret = devm_spi_register_master(dev, master);
-	if (ret < 0)
-		goto free_ipg_clock;
-
-	return ret;
-
-free_ipg_clock:
-	clk_disable_unprepare(mps->clk_ipg);
-free_mclk_clock:
-	clk_disable_unprepare(mps->clk_mclk);
-free_master:
-	spi_master_put(master);
+		return ret;
 
-	return ret;
-}
-
-static int mpc512x_psc_spi_do_remove(struct device *dev)
-{
-	struct spi_master *master = dev_get_drvdata(dev);
-	struct mpc512x_psc_spi *mps = spi_master_get_devdata(master);
-
-	clk_disable_unprepare(mps->clk_mclk);
-	clk_disable_unprepare(mps->clk_ipg);
-
-	return 0;
-}
-
-static int mpc512x_psc_spi_of_probe(struct platform_device *op)
-{
-	const u32 *regaddr_p;
-	u64 regaddr64, size64;
-
-	regaddr_p = of_get_address(op->dev.of_node, 0, &size64, NULL);
-	if (!regaddr_p) {
-		dev_err(&op->dev, "Invalid PSC address\n");
-		return -EINVAL;
-	}
-	regaddr64 = of_translate_address(op->dev.of_node, regaddr_p);
-
-	return mpc512x_psc_spi_do_probe(&op->dev, (u32) regaddr64, (u32) size64,
-				irq_of_parse_and_map(op->dev.of_node, 0));
-}
-
-static int mpc512x_psc_spi_of_remove(struct platform_device *op)
-{
-	return mpc512x_psc_spi_do_remove(&op->dev);
+	return devm_spi_register_master(dev, master);
 }
 
 static const struct of_device_id mpc512x_psc_spi_of_match[] = {
@@ -603,7 +526,6 @@ MODULE_DEVICE_TABLE(of, mpc512x_psc_spi_of_match);
 
 static struct platform_driver mpc512x_psc_spi_of_driver = {
 	.probe = mpc512x_psc_spi_of_probe,
-	.remove = mpc512x_psc_spi_of_remove,
 	.driver = {
 		.name = "mpc512x-psc-spi",
 		.of_match_table = mpc512x_psc_spi_of_match,
diff --git a/drivers/spi/spi-mpc52xx-psc.c b/drivers/spi/spi-mpc52xx-psc.c
index 609311231e64..9a1a080fb688 100644
--- a/drivers/spi/spi-mpc52xx-psc.c
+++ b/drivers/spi/spi-mpc52xx-psc.c
@@ -11,16 +11,14 @@
 #include <linux/types.h>
 #include <linux/errno.h>
 #include <linux/interrupt.h>
-#include <linux/of_address.h>
-#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
 #include <linux/workqueue.h>
 #include <linux/completion.h>
 #include <linux/io.h>
 #include <linux/delay.h>
 #include <linux/spi/spi.h>
-#include <linux/fsl_devices.h>
 #include <linux/slab.h>
-#include <linux/of_irq.h>
 
 #include <asm/mpc52xx.h>
 #include <asm/mpc52xx_psc.h>
@@ -28,10 +26,6 @@
 #define MCLK 20000000 /* PSC port MClk in hz */
 
 struct mpc52xx_psc_spi {
-	/* fsl_spi_platform data */
-	void (*cs_control)(struct spi_device *spi, bool on);
-	u32 sysclk;
-
 	/* driver internal data */
 	struct mpc52xx_psc __iomem *psc;
 	struct mpc52xx_psc_fifo __iomem *fifo;
@@ -101,17 +95,6 @@ static void mpc52xx_psc_spi_activate_cs(struct spi_device *spi)
 		ccr |= (MCLK / 1000000 - 1) & 0xFF;
 	out_be16((u16 __iomem *)&psc->ccr, ccr);
 	mps->bits_per_word = cs->bits_per_word;
-
-	if (mps->cs_control)
-		mps->cs_control(spi, (spi->mode & SPI_CS_HIGH) ? 1 : 0);
-}
-
-static void mpc52xx_psc_spi_deactivate_cs(struct spi_device *spi)
-{
-	struct mpc52xx_psc_spi *mps = spi_master_get_devdata(spi->master);
-
-	if (mps->cs_control)
-		mps->cs_control(spi, (spi->mode & SPI_CS_HIGH) ? 0 : 1);
 }
 
 #define MPC52xx_PSC_BUFSIZE (MPC52xx_PSC_RFNUM_MASK + 1)
@@ -220,14 +203,9 @@ int mpc52xx_psc_spi_transfer_one_message(struct spi_controller *ctlr,
 		m->actual_length += t->len;
 
 		spi_transfer_delay_exec(t);
-
-		if (cs_change)
-			mpc52xx_psc_spi_deactivate_cs(spi);
 	}
 
 	m->status = status;
-	if (status || !cs_change)
-		mpc52xx_psc_spi_deactivate_cs(spi);
 
 	mpc52xx_psc_spi_transfer_setup(spi, NULL);
 
@@ -269,7 +247,7 @@ static int mpc52xx_psc_spi_port_config(int psc_id, struct mpc52xx_psc_spi *mps)
 	int ret;
 
 	/* default sysclk is 512MHz */
-	mclken_div = (mps->sysclk ? mps->sysclk : 512000000) / MCLK;
+	mclken_div = 512000000 / MCLK;
 	ret = mpc52xx_set_psc_clkdiv(psc_id, mclken_div);
 	if (ret)
 		return ret;
@@ -313,16 +291,15 @@ static irqreturn_t mpc52xx_psc_spi_isr(int irq, void *dev_id)
 	return IRQ_NONE;
 }
 
-/* bus_num is used only for the case dev->platform_data == NULL */
-static int mpc52xx_psc_spi_do_probe(struct device *dev, u32 regaddr,
-				u32 size, unsigned int irq, s16 bus_num)
+static int mpc52xx_psc_spi_of_probe(struct platform_device *pdev)
 {
-	struct fsl_spi_platform_data *pdata = dev_get_platdata(dev);
+	struct device *dev = &pdev->dev;
 	struct mpc52xx_psc_spi *mps;
 	struct spi_master *master;
+	u32 bus_num;
 	int ret;
 
-	master = spi_alloc_master(dev, sizeof(*mps));
+	master = devm_spi_alloc_master(dev, sizeof(*mps));
 	if (master == NULL)
 		return -ENOMEM;
 
@@ -332,104 +309,41 @@ static int mpc52xx_psc_spi_do_probe(struct device *dev, u32 regaddr,
 	/* the spi->mode bits understood by this driver: */
 	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_LSB_FIRST;
 
-	mps->irq = irq;
-	if (pdata == NULL) {
-		dev_warn(dev,
-			 "probe called without platform data, no cs_control function will be called\n");
-		mps->cs_control = NULL;
-		mps->sysclk = 0;
-		master->bus_num = bus_num;
-		master->num_chipselect = 255;
-	} else {
-		mps->cs_control = pdata->cs_control;
-		mps->sysclk = pdata->sysclk;
-		master->bus_num = pdata->bus_num;
-		master->num_chipselect = pdata->max_chipselect;
-	}
+	ret = device_property_read_u32(dev, "cell-index", &bus_num);
+	if (ret || bus_num > 5)
+		return dev_err_probe(dev, ret ? : -EINVAL, "Invalid cell-index property\n");
+	master->bus_num = bus_num + 1;
+
+	master->num_chipselect = 255;
 	master->setup = mpc52xx_psc_spi_setup;
 	master->transfer_one_message = mpc52xx_psc_spi_transfer_one_message;
 	master->cleanup = mpc52xx_psc_spi_cleanup;
-	master->dev.of_node = dev->of_node;
 
-	mps->psc = ioremap(regaddr, size);
-	if (!mps->psc) {
-		dev_err(dev, "could not ioremap I/O port range\n");
-		ret = -EFAULT;
-		goto free_master;
-	}
+	device_set_node(&master->dev, dev_fwnode(dev));
+
+	mps->psc = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
+	if (IS_ERR(mps->psc))
+		return dev_err_probe(dev, PTR_ERR(mps->psc), "could not ioremap I/O port range\n");
+
 	/* On the 5200, fifo regs are immediately ajacent to the psc regs */
 	mps->fifo = ((void __iomem *)mps->psc) + sizeof(struct mpc52xx_psc);
 
-	ret = request_irq(mps->irq, mpc52xx_psc_spi_isr, 0, "mpc52xx-psc-spi",
-				mps);
+	mps->irq = platform_get_irq(pdev, 0);
+	if (mps->irq < 0)
+		return mps->irq;
+
+	ret = devm_request_irq(dev, mps->irq, mpc52xx_psc_spi_isr, 0,
+			       "mpc52xx-psc-spi", mps);
 	if (ret)
-		goto free_master;
+		return ret;
 
 	ret = mpc52xx_psc_spi_port_config(master->bus_num, mps);
-	if (ret < 0) {
-		dev_err(dev, "can't configure PSC! Is it capable of SPI?\n");
-		goto free_irq;
-	}
-
-	init_completion(&mps->done);
-
-	ret = spi_register_master(master);
 	if (ret < 0)
-		goto free_irq;
+		return dev_err_probe(dev, ret, "can't configure PSC! Is it capable of SPI?\n");
 
-	return ret;
-
-free_irq:
-	free_irq(mps->irq, mps);
-free_master:
-	if (mps->psc)
-		iounmap(mps->psc);
-	spi_master_put(master);
-
-	return ret;
-}
-
-static int mpc52xx_psc_spi_of_probe(struct platform_device *op)
-{
-	const u32 *regaddr_p;
-	u64 regaddr64, size64;
-	s16 id = -1;
-
-	regaddr_p = of_get_address(op->dev.of_node, 0, &size64, NULL);
-	if (!regaddr_p) {
-		dev_err(&op->dev, "Invalid PSC address\n");
-		return -EINVAL;
-	}
-	regaddr64 = of_translate_address(op->dev.of_node, regaddr_p);
-
-	/* get PSC id (1..6, used by port_config) */
-	if (op->dev.platform_data == NULL) {
-		const u32 *psc_nump;
-
-		psc_nump = of_get_property(op->dev.of_node, "cell-index", NULL);
-		if (!psc_nump || *psc_nump > 5) {
-			dev_err(&op->dev, "Invalid cell-index property\n");
-			return -EINVAL;
-		}
-		id = *psc_nump + 1;
-	}
-
-	return mpc52xx_psc_spi_do_probe(&op->dev, (u32)regaddr64, (u32)size64,
-				irq_of_parse_and_map(op->dev.of_node, 0), id);
-}
-
-static int mpc52xx_psc_spi_of_remove(struct platform_device *op)
-{
-	struct spi_master *master = spi_master_get(platform_get_drvdata(op));
-	struct mpc52xx_psc_spi *mps = spi_master_get_devdata(master);
-
-	spi_unregister_master(master);
-	free_irq(mps->irq, mps);
-	if (mps->psc)
-		iounmap(mps->psc);
-	spi_master_put(master);
+	init_completion(&mps->done);
 
-	return 0;
+	return devm_spi_register_master(dev, master);
 }
 
 static const struct of_device_id mpc52xx_psc_spi_of_match[] = {
@@ -442,7 +356,6 @@ MODULE_DEVICE_TABLE(of, mpc52xx_psc_spi_of_match);
 
 static struct platform_driver mpc52xx_psc_spi_of_driver = {
 	.probe = mpc52xx_psc_spi_of_probe,
-	.remove = mpc52xx_psc_spi_of_remove,
 	.driver = {
 		.name = "mpc52xx-psc-spi",
 		.of_match_table = mpc52xx_psc_spi_of_match,
diff --git a/drivers/spi/spi-mpc52xx.c b/drivers/spi/spi-mpc52xx.c
index 7b64e64c65cf..ab7df5f64342 100644
--- a/drivers/spi/spi-mpc52xx.c
+++ b/drivers/spi/spi-mpc52xx.c
@@ -101,7 +101,7 @@ static void mpc52xx_spi_chipsel(struct mpc52xx_spi *ms, int value)
 	int cs;
 
 	if (ms->gpio_cs_count > 0) {
-		cs = ms->message->spi->chip_select;
+		cs = spi_get_chipselect(ms->message->spi, 0);
 		gpiod_set_value(ms->gpio_cs[cs], value);
 	} else {
 		out_8(ms->regs + SPI_PORTDATA, value ? 0 : 0x08);
@@ -513,7 +513,7 @@ static int mpc52xx_spi_probe(struct platform_device *op)
 	return rc;
 }
 
-static int mpc52xx_spi_remove(struct platform_device *op)
+static void mpc52xx_spi_remove(struct platform_device *op)
 {
 	struct spi_master *master = spi_master_get(platform_get_drvdata(op));
 	struct mpc52xx_spi *ms = spi_master_get_devdata(master);
@@ -529,8 +529,6 @@ static int mpc52xx_spi_remove(struct platform_device *op)
 	spi_unregister_master(master);
 	iounmap(ms->regs);
 	spi_master_put(master);
-
-	return 0;
 }
 
 static const struct of_device_id mpc52xx_spi_match[] = {
@@ -545,6 +543,6 @@ static struct platform_driver mpc52xx_spi_of_driver = {
 		.of_match_table = mpc52xx_spi_match,
 	},
 	.probe = mpc52xx_spi_probe,
-	.remove = mpc52xx_spi_remove,
+	.remove_new = mpc52xx_spi_remove,
 };
 module_platform_driver(mpc52xx_spi_of_driver);
diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c
index 9eab6c20dbc5..21c321f43766 100644
--- a/drivers/spi/spi-mt65xx.c
+++ b/drivers/spi/spi-mt65xx.c
@@ -421,7 +421,7 @@ static int mtk_spi_hw_init(struct spi_master *master,
 
 	/* pad select */
 	if (mdata->dev_comp->need_pad_sel)
-		writel(mdata->pad_sel[spi->chip_select],
+		writel(mdata->pad_sel[spi_get_chipselect(spi, 0)],
 		       mdata->base + SPI_PAD_SEL_REG);
 
 	/* tick delay */
@@ -735,9 +735,9 @@ static int mtk_spi_setup(struct spi_device *spi)
 	if (!spi->controller_data)
 		spi->controller_data = (void *)&mtk_default_chip_info;
 
-	if (mdata->dev_comp->need_pad_sel && spi->cs_gpiod)
+	if (mdata->dev_comp->need_pad_sel && spi_get_csgpiod(spi, 0))
 		/* CS de-asserted, gpiolib will handle inversion */
-		gpiod_direction_output(spi->cs_gpiod, 0);
+		gpiod_direction_output(spi_get_csgpiod(spi, 0), 0);
 
 	return 0;
 }
diff --git a/drivers/spi/spi-mt7621.c b/drivers/spi/spi-mt7621.c
index c4cc8e2f85e2..3e9d396b33bd 100644
--- a/drivers/spi/spi-mt7621.c
+++ b/drivers/spi/spi-mt7621.c
@@ -76,7 +76,7 @@ static inline void mt7621_spi_write(struct mt7621_spi *rs, u32 reg, u32 val)
 static void mt7621_spi_set_cs(struct spi_device *spi, int enable)
 {
 	struct mt7621_spi *rs = spidev_to_mt7621_spi(spi);
-	int cs = spi->chip_select;
+	int cs = spi_get_chipselect(spi, 0);
 	u32 polar = 0;
 	u32 master;
 
diff --git a/drivers/spi/spi-mtk-nor.c b/drivers/spi/spi-mtk-nor.c
index aad92a58c4b8..baa7a5353987 100644
--- a/drivers/spi/spi-mtk-nor.c
+++ b/drivers/spi/spi-mtk-nor.c
@@ -934,7 +934,7 @@ err_probe:
 	return ret;
 }
 
-static int mtk_nor_remove(struct platform_device *pdev)
+static void mtk_nor_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr = dev_get_drvdata(&pdev->dev);
 	struct mtk_nor *sp = spi_controller_get_devdata(ctlr);
@@ -944,8 +944,6 @@ static int mtk_nor_remove(struct platform_device *pdev)
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 
 	mtk_nor_disable_clk(sp);
-
-	return 0;
 }
 
 static int __maybe_unused mtk_nor_runtime_suspend(struct device *dev)
@@ -999,7 +997,7 @@ static struct platform_driver mtk_nor_driver = {
 		.pm = &mtk_nor_pm_ops,
 	},
 	.probe = mtk_nor_probe,
-	.remove = mtk_nor_remove,
+	.remove_new = mtk_nor_remove,
 };
 
 module_platform_driver(mtk_nor_driver);
diff --git a/drivers/spi/spi-mtk-snfi.c b/drivers/spi/spi-mtk-snfi.c
index f3f95eb37365..bed8317cd205 100644
--- a/drivers/spi/spi-mtk-snfi.c
+++ b/drivers/spi/spi-mtk-snfi.c
@@ -1506,7 +1506,7 @@ release_ecc:
 	return ret;
 }
 
-static int mtk_snand_remove(struct platform_device *pdev)
+static void mtk_snand_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr = platform_get_drvdata(pdev);
 	struct mtk_snand *ms = spi_controller_get_devdata(ctlr);
@@ -1515,12 +1515,11 @@ static int mtk_snand_remove(struct platform_device *pdev)
 	mtk_snand_disable_clk(ms);
 	mtk_ecc_release(ms->ecc);
 	kfree(ms->buf);
-	return 0;
 }
 
 static struct platform_driver mtk_snand_driver = {
 	.probe = mtk_snand_probe,
-	.remove = mtk_snand_remove,
+	.remove_new = mtk_snand_remove,
 	.driver = {
 		.name = "mtk-snand",
 		.of_match_table = mtk_snand_ids,
diff --git a/drivers/spi/spi-mux.c b/drivers/spi/spi-mux.c
index 0709e987bd5a..fa8c1f740c70 100644
--- a/drivers/spi/spi-mux.c
+++ b/drivers/spi/spi-mux.c
@@ -51,22 +51,22 @@ static int spi_mux_select(struct spi_device *spi)
 	struct spi_mux_priv *priv = spi_controller_get_devdata(spi->controller);
 	int ret;
 
-	ret = mux_control_select(priv->mux, spi->chip_select);
+	ret = mux_control_select(priv->mux, spi_get_chipselect(spi, 0));
 	if (ret)
 		return ret;
 
-	if (priv->current_cs == spi->chip_select)
+	if (priv->current_cs == spi_get_chipselect(spi, 0))
 		return 0;
 
 	dev_dbg(&priv->spi->dev, "setting up the mux for cs %d\n",
-		spi->chip_select);
+		spi_get_chipselect(spi, 0));
 
 	/* copy the child device's settings except for the cs */
 	priv->spi->max_speed_hz = spi->max_speed_hz;
 	priv->spi->mode = spi->mode;
 	priv->spi->bits_per_word = spi->bits_per_word;
 
-	priv->current_cs = spi->chip_select;
+	priv->current_cs = spi_get_chipselect(spi, 0);
 
 	return 0;
 }
diff --git a/drivers/spi/spi-mxic.c b/drivers/spi/spi-mxic.c
index a3dba17390eb..00617fd4b2c3 100644
--- a/drivers/spi/spi-mxic.c
+++ b/drivers/spi/spi-mxic.c
@@ -306,8 +306,8 @@ static u32 mxic_spi_prep_hc_cfg(struct spi_device *spi, u32 flags)
 		nio = 2;
 
 	return flags | HC_CFG_NIO(nio) |
-	       HC_CFG_TYPE(spi->chip_select, HC_CFG_TYPE_SPI_NOR) |
-	       HC_CFG_SLV_ACT(spi->chip_select) | HC_CFG_IDLE_SIO_LVL(1);
+	       HC_CFG_TYPE(spi_get_chipselect(spi, 0), HC_CFG_TYPE_SPI_NOR) |
+	       HC_CFG_SLV_ACT(spi_get_chipselect(spi, 0)) | HC_CFG_IDLE_SIO_LVL(1);
 }
 
 static u32 mxic_spi_mem_prep_op_cfg(const struct spi_mem_op *op,
@@ -405,7 +405,7 @@ static ssize_t mxic_spi_mem_dirmap_read(struct spi_mem_dirmap_desc *desc,
 	len = min_t(size_t, len, mxic->linear.size);
 	writel(len, mxic->regs + LRD_RANGE);
 	writel(LMODE_CMD0(desc->info.op_tmpl.cmd.opcode) |
-	       LMODE_SLV_ACT(desc->mem->spi->chip_select) |
+	       LMODE_SLV_ACT(spi_get_chipselect(desc->mem->spi, 0)) |
 	       LMODE_EN,
 	       mxic->regs + LRD_CTRL);
 
@@ -449,7 +449,7 @@ static ssize_t mxic_spi_mem_dirmap_write(struct spi_mem_dirmap_desc *desc,
 	len = min_t(size_t, len, mxic->linear.size);
 	writel(len, mxic->regs + LWR_RANGE);
 	writel(LMODE_CMD0(desc->info.op_tmpl.cmd.opcode) |
-	       LMODE_SLV_ACT(desc->mem->spi->chip_select) |
+	       LMODE_SLV_ACT(spi_get_chipselect(desc->mem->spi, 0)) |
 	       LMODE_EN,
 	       mxic->regs + LWR_CTRL);
 
@@ -524,7 +524,7 @@ static int mxic_spi_mem_exec_op(struct spi_mem *mem,
 	writel(HC_EN_BIT, mxic->regs + HC_EN);
 
 	writel(mxic_spi_mem_prep_op_cfg(op, op->data.nbytes),
-	       mxic->regs + SS_CTRL(mem->spi->chip_select));
+	       mxic->regs + SS_CTRL(spi_get_chipselect(mem->spi, 0)));
 
 	writel(readl(mxic->regs + HC_CFG) | HC_CFG_MAN_CS_ASSERT,
 	       mxic->regs + HC_CFG);
@@ -818,7 +818,7 @@ static int mxic_spi_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int mxic_spi_remove(struct platform_device *pdev)
+static void mxic_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct mxic_spi *mxic = spi_master_get_devdata(master);
@@ -826,8 +826,6 @@ static int mxic_spi_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 	mxic_spi_mem_ecc_remove(mxic);
 	spi_unregister_master(master);
-
-	return 0;
 }
 
 static const struct of_device_id mxic_spi_of_ids[] = {
@@ -838,7 +836,7 @@ MODULE_DEVICE_TABLE(of, mxic_spi_of_ids);
 
 static struct platform_driver mxic_spi_driver = {
 	.probe = mxic_spi_probe,
-	.remove = mxic_spi_remove,
+	.remove_new = mxic_spi_remove,
 	.driver = {
 		.name = "mxic-spi",
 		.of_match_table = mxic_spi_of_ids,
diff --git a/drivers/spi/spi-mxs.c b/drivers/spi/spi-mxs.c
index 55178579f3c6..963a53dd680b 100644
--- a/drivers/spi/spi-mxs.c
+++ b/drivers/spi/spi-mxs.c
@@ -369,7 +369,7 @@ static int mxs_spi_transfer_one(struct spi_master *master,
 	/* Program CS register bits here, it will be used for all transfers. */
 	writel(BM_SSP_CTRL0_WAIT_FOR_CMD | BM_SSP_CTRL0_WAIT_FOR_IRQ,
 	       ssp->base + HW_SSP_CTRL0 + STMP_OFFSET_REG_CLR);
-	writel(mxs_spi_cs_to_reg(m->spi->chip_select),
+	writel(mxs_spi_cs_to_reg(spi_get_chipselect(m->spi, 0)),
 	       ssp->base + HW_SSP_CTRL0 + STMP_OFFSET_REG_SET);
 
 	list_for_each_entry(t, &m->transfers, transfer_list) {
@@ -638,7 +638,7 @@ out_master_free:
 	return ret;
 }
 
-static int mxs_spi_remove(struct platform_device *pdev)
+static void mxs_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master;
 	struct mxs_spi *spi;
@@ -653,13 +653,11 @@ static int mxs_spi_remove(struct platform_device *pdev)
 		mxs_spi_runtime_suspend(&pdev->dev);
 
 	dma_release_channel(ssp->dmach);
-
-	return 0;
 }
 
 static struct platform_driver mxs_spi_driver = {
 	.probe	= mxs_spi_probe,
-	.remove	= mxs_spi_remove,
+	.remove_new = mxs_spi_remove,
 	.driver	= {
 		.name	= DRIVER_NAME,
 		.of_match_table = mxs_spi_dt_ids,
diff --git a/drivers/spi/spi-npcm-fiu.c b/drivers/spi/spi-npcm-fiu.c
index 559d3a5b4062..eb353561509a 100644
--- a/drivers/spi/spi-npcm-fiu.c
+++ b/drivers/spi/spi-npcm-fiu.c
@@ -288,7 +288,7 @@ static ssize_t npcm_fiu_direct_read(struct spi_mem_dirmap_desc *desc,
 {
 	struct npcm_fiu_spi *fiu =
 		spi_controller_get_devdata(desc->mem->spi->master);
-	struct npcm_fiu_chip *chip = &fiu->chip[desc->mem->spi->chip_select];
+	struct npcm_fiu_chip *chip = &fiu->chip[spi_get_chipselect(desc->mem->spi, 0)];
 	void __iomem *src = (void __iomem *)(chip->flash_region_mapped_ptr +
 					     offs);
 	u8 *buf_rx = buf;
@@ -315,7 +315,7 @@ static ssize_t npcm_fiu_direct_write(struct spi_mem_dirmap_desc *desc,
 {
 	struct npcm_fiu_spi *fiu =
 		spi_controller_get_devdata(desc->mem->spi->master);
-	struct npcm_fiu_chip *chip = &fiu->chip[desc->mem->spi->chip_select];
+	struct npcm_fiu_chip *chip = &fiu->chip[spi_get_chipselect(desc->mem->spi, 0)];
 	void __iomem *dst = (void __iomem *)(chip->flash_region_mapped_ptr +
 					     offs);
 	const u8 *buf_tx = buf;
@@ -344,7 +344,7 @@ static int npcm_fiu_uma_read(struct spi_mem *mem,
 
 	regmap_update_bits(fiu->regmap, NPCM_FIU_UMA_CTS,
 			   NPCM_FIU_UMA_CTS_DEV_NUM,
-			   (mem->spi->chip_select <<
+			   (spi_get_chipselect(mem->spi, 0) <<
 			    NPCM_FIU_UMA_CTS_DEV_NUM_SHIFT));
 	regmap_update_bits(fiu->regmap, NPCM_FIU_UMA_CMD,
 			   NPCM_FIU_UMA_CMD_CMD, op->cmd.opcode);
@@ -398,7 +398,7 @@ static int npcm_fiu_uma_write(struct spi_mem *mem,
 
 	regmap_update_bits(fiu->regmap, NPCM_FIU_UMA_CTS,
 			   NPCM_FIU_UMA_CTS_DEV_NUM,
-			   (mem->spi->chip_select <<
+			   (spi_get_chipselect(mem->spi, 0) <<
 			    NPCM_FIU_UMA_CTS_DEV_NUM_SHIFT));
 
 	regmap_update_bits(fiu->regmap, NPCM_FIU_UMA_CMD,
@@ -451,7 +451,7 @@ static int npcm_fiu_manualwrite(struct spi_mem *mem,
 
 	regmap_update_bits(fiu->regmap, NPCM_FIU_UMA_CTS,
 			   NPCM_FIU_UMA_CTS_DEV_NUM,
-			   (mem->spi->chip_select <<
+			   (spi_get_chipselect(mem->spi, 0) <<
 			    NPCM_FIU_UMA_CTS_DEV_NUM_SHIFT));
 	regmap_update_bits(fiu->regmap, NPCM_FIU_UMA_CTS,
 			   NPCM_FIU_UMA_CTS_SW_CS, 0);
@@ -545,7 +545,7 @@ static int npcm_fiu_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
 {
 	struct npcm_fiu_spi *fiu =
 		spi_controller_get_devdata(mem->spi->master);
-	struct npcm_fiu_chip *chip = &fiu->chip[mem->spi->chip_select];
+	struct npcm_fiu_chip *chip = &fiu->chip[spi_get_chipselect(mem->spi, 0)];
 	int ret = 0;
 	u8 *buf;
 
@@ -605,7 +605,7 @@ static int npcm_fiu_dirmap_create(struct spi_mem_dirmap_desc *desc)
 {
 	struct npcm_fiu_spi *fiu =
 		spi_controller_get_devdata(desc->mem->spi->master);
-	struct npcm_fiu_chip *chip = &fiu->chip[desc->mem->spi->chip_select];
+	struct npcm_fiu_chip *chip = &fiu->chip[spi_get_chipselect(desc->mem->spi, 0)];
 	struct regmap *gcr_regmap;
 
 	if (!fiu->res_mem) {
@@ -624,7 +624,7 @@ static int npcm_fiu_dirmap_create(struct spi_mem_dirmap_desc *desc)
 		chip->flash_region_mapped_ptr =
 			devm_ioremap(fiu->dev, (fiu->res_mem->start +
 							(fiu->info->max_map_size *
-						    desc->mem->spi->chip_select)),
+						    spi_get_chipselect(desc->mem->spi, 0))),
 					     (u32)desc->info.length);
 		if (!chip->flash_region_mapped_ptr) {
 			dev_warn(fiu->dev, "Error mapping memory region, direct read disabled\n");
@@ -669,9 +669,9 @@ static int npcm_fiu_setup(struct spi_device *spi)
 	struct npcm_fiu_spi *fiu = spi_controller_get_devdata(ctrl);
 	struct npcm_fiu_chip *chip;
 
-	chip = &fiu->chip[spi->chip_select];
+	chip = &fiu->chip[spi_get_chipselect(spi, 0)];
 	chip->fiu = fiu;
-	chip->chipselect = spi->chip_select;
+	chip->chipselect = spi_get_chipselect(spi, 0);
 	chip->clkrate = spi->max_speed_hz;
 
 	fiu->clkrate = clk_get_rate(fiu->clk);
@@ -762,12 +762,11 @@ static int npcm_fiu_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int npcm_fiu_remove(struct platform_device *pdev)
+static void npcm_fiu_remove(struct platform_device *pdev)
 {
 	struct npcm_fiu_spi *fiu = platform_get_drvdata(pdev);
 
 	clk_disable_unprepare(fiu->clk);
-	return 0;
 }
 
 MODULE_DEVICE_TABLE(of, npcm_fiu_dt_ids);
@@ -779,7 +778,7 @@ static struct platform_driver npcm_fiu_driver = {
 		.of_match_table = npcm_fiu_dt_ids,
 	},
 	.probe      = npcm_fiu_probe,
-	.remove	    = npcm_fiu_remove,
+	.remove_new	    = npcm_fiu_remove,
 };
 module_platform_driver(npcm_fiu_driver);
 
diff --git a/drivers/spi/spi-npcm-pspi.c b/drivers/spi/spi-npcm-pspi.c
index 7f2e4d1b0d43..64585c2a25c5 100644
--- a/drivers/spi/spi-npcm-pspi.c
+++ b/drivers/spi/spi-npcm-pspi.c
@@ -430,15 +430,13 @@ out_master_put:
 	return ret;
 }
 
-static int npcm_pspi_remove(struct platform_device *pdev)
+static void npcm_pspi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct npcm_pspi *priv = spi_master_get_devdata(master);
 
 	npcm_pspi_reset_hw(priv);
 	clk_disable_unprepare(priv->clk);
-
-	return 0;
 }
 
 static const struct of_device_id npcm_pspi_match[] = {
@@ -454,7 +452,7 @@ static struct platform_driver npcm_pspi_driver = {
 		.of_match_table	= npcm_pspi_match,
 	},
 	.probe		= npcm_pspi_probe,
-	.remove		= npcm_pspi_remove,
+	.remove_new	= npcm_pspi_remove,
 };
 module_platform_driver(npcm_pspi_driver);
 
diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c
index 1c1991a26c15..544017655787 100644
--- a/drivers/spi/spi-nxp-fspi.c
+++ b/drivers/spi/spi-nxp-fspi.c
@@ -214,9 +214,15 @@
 
 #define FSPI_DLLACR			0xC0
 #define FSPI_DLLACR_OVRDEN		BIT(8)
+#define FSPI_DLLACR_SLVDLY(x)		((x) << 3)
+#define FSPI_DLLACR_DLLRESET		BIT(1)
+#define FSPI_DLLACR_DLLEN		BIT(0)
 
 #define FSPI_DLLBCR			0xC4
 #define FSPI_DLLBCR_OVRDEN		BIT(8)
+#define FSPI_DLLBCR_SLVDLY(x)		((x) << 3)
+#define FSPI_DLLBCR_DLLRESET		BIT(1)
+#define FSPI_DLLBCR_DLLEN		BIT(0)
 
 #define FSPI_STS0			0xE0
 #define FSPI_STS0_DLPHB(x)		((x) << 8)
@@ -231,6 +237,16 @@
 #define FSPI_STS1_AHB_ERRCD(x)		((x) << 8)
 #define FSPI_STS1_AHB_ERRID(x)		(x)
 
+#define FSPI_STS2			0xE8
+#define FSPI_STS2_BREFLOCK		BIT(17)
+#define FSPI_STS2_BSLVLOCK		BIT(16)
+#define FSPI_STS2_AREFLOCK		BIT(1)
+#define FSPI_STS2_ASLVLOCK		BIT(0)
+#define FSPI_STS2_AB_LOCK		(FSPI_STS2_BREFLOCK | \
+					 FSPI_STS2_BSLVLOCK | \
+					 FSPI_STS2_AREFLOCK | \
+					 FSPI_STS2_ASLVLOCK)
+
 #define FSPI_AHBSPNST			0xEC
 #define FSPI_AHBSPNST_DATLFT(x)		((x) << 16)
 #define FSPI_AHBSPNST_BUFID(x)		((x) << 1)
@@ -615,6 +631,35 @@ static int nxp_fspi_clk_disable_unprep(struct nxp_fspi *f)
 	return 0;
 }
 
+static void nxp_fspi_dll_calibration(struct nxp_fspi *f)
+{
+	int ret;
+
+	/* Reset the DLL, set the DLLRESET to 1 and then set to 0 */
+	fspi_writel(f, FSPI_DLLACR_DLLRESET, f->iobase + FSPI_DLLACR);
+	fspi_writel(f, FSPI_DLLBCR_DLLRESET, f->iobase + FSPI_DLLBCR);
+	fspi_writel(f, 0, f->iobase + FSPI_DLLACR);
+	fspi_writel(f, 0, f->iobase + FSPI_DLLBCR);
+
+	/*
+	 * Enable the DLL calibration mode.
+	 * The delay target for slave delay line is:
+	 *   ((SLVDLYTARGET+1) * 1/32 * clock cycle of reference clock.
+	 * When clock rate > 100MHz, recommend SLVDLYTARGET is 0xF, which
+	 * means half of clock cycle of reference clock.
+	 */
+	fspi_writel(f, FSPI_DLLACR_DLLEN | FSPI_DLLACR_SLVDLY(0xF),
+		    f->iobase + FSPI_DLLACR);
+	fspi_writel(f, FSPI_DLLBCR_DLLEN | FSPI_DLLBCR_SLVDLY(0xF),
+		    f->iobase + FSPI_DLLBCR);
+
+	/* Wait to get REF/SLV lock */
+	ret = fspi_readl_poll_tout(f, f->iobase + FSPI_STS2, FSPI_STS2_AB_LOCK,
+				   0, POLL_TOUT, true);
+	if (ret)
+		dev_warn(f->dev, "DLL lock failed, please fix it!\n");
+}
+
 /*
  * In FlexSPI controller, flash access is based on value of FSPI_FLSHXXCR0
  * register and start base address of the slave device.
@@ -663,7 +708,7 @@ static void nxp_fspi_select_mem(struct nxp_fspi *f, struct spi_device *spi)
 	 * Return, if previously selected slave device is same as current
 	 * requested slave device.
 	 */
-	if (f->selected == spi->chip_select)
+	if (f->selected == spi_get_chipselect(spi, 0))
 		return;
 
 	/* Reset FLSHxxCR0 registers */
@@ -676,9 +721,9 @@ static void nxp_fspi_select_mem(struct nxp_fspi *f, struct spi_device *spi)
 	size_kb = FSPI_FLSHXCR0_SZ(f->memmap_phy_size);
 
 	fspi_writel(f, size_kb, f->iobase + FSPI_FLSHA1CR0 +
-		    4 * spi->chip_select);
+		    4 * spi_get_chipselect(spi, 0));
 
-	dev_dbg(f->dev, "Slave device [CS:%x] selected\n", spi->chip_select);
+	dev_dbg(f->dev, "Slave device [CS:%x] selected\n", spi_get_chipselect(spi, 0));
 
 	nxp_fspi_clk_disable_unprep(f);
 
@@ -690,7 +735,14 @@ static void nxp_fspi_select_mem(struct nxp_fspi *f, struct spi_device *spi)
 	if (ret)
 		return;
 
-	f->selected = spi->chip_select;
+	/*
+	 * If clock rate > 100MHz, then switch from DLL override mode to
+	 * DLL calibration mode.
+	 */
+	if (rate > 100000000)
+		nxp_fspi_dll_calibration(f);
+
+	f->selected = spi_get_chipselect(spi, 0);
 }
 
 static int nxp_fspi_read_ahb(struct nxp_fspi *f, const struct spi_mem_op *op)
@@ -997,7 +1049,11 @@ static int nxp_fspi_default_setup(struct nxp_fspi *f)
 	/* Disable the module */
 	fspi_writel(f, FSPI_MCR0_MDIS, base + FSPI_MCR0);
 
-	/* Reset the DLL register to default value */
+	/*
+	 * Config the DLL register to default value, enable the slave clock delay
+	 * line delay cell override mode, and use 1 fixed delay cell in DLL delay
+	 * chain, this is the suggested setting when clock rate < 100MHz.
+	 */
 	fspi_writel(f, FSPI_DLLACR_OVRDEN, base + FSPI_DLLACR);
 	fspi_writel(f, FSPI_DLLBCR_OVRDEN, base + FSPI_DLLBCR);
 
@@ -1055,7 +1111,7 @@ static const char *nxp_fspi_get_name(struct spi_mem *mem)
 
 	name = devm_kasprintf(dev, GFP_KERNEL,
 			      "%s-%d", dev_name(f->dev),
-			      mem->spi->chip_select);
+			      spi_get_chipselect(mem->spi, 0));
 
 	if (!name) {
 		dev_err(dev, "failed to get memory for custom flash name\n");
@@ -1195,7 +1251,7 @@ err_put_ctrl:
 	return ret;
 }
 
-static int nxp_fspi_remove(struct platform_device *pdev)
+static void nxp_fspi_remove(struct platform_device *pdev)
 {
 	struct nxp_fspi *f = platform_get_drvdata(pdev);
 
@@ -1208,8 +1264,6 @@ static int nxp_fspi_remove(struct platform_device *pdev)
 
 	if (f->ahb_addr)
 		iounmap(f->ahb_addr);
-
-	return 0;
 }
 
 static int nxp_fspi_suspend(struct device *dev)
@@ -1257,7 +1311,7 @@ static struct platform_driver nxp_fspi_driver = {
 		.pm =   &nxp_fspi_pm_ops,
 	},
 	.probe          = nxp_fspi_probe,
-	.remove		= nxp_fspi_remove,
+	.remove_new	= nxp_fspi_remove,
 };
 module_platform_driver(nxp_fspi_driver);
 
diff --git a/drivers/spi/spi-oc-tiny.c b/drivers/spi/spi-oc-tiny.c
index 38c14c4e4e21..3af499838e84 100644
--- a/drivers/spi/spi-oc-tiny.c
+++ b/drivers/spi/spi-oc-tiny.c
@@ -271,14 +271,13 @@ exit:
 	return err;
 }
 
-static int tiny_spi_remove(struct platform_device *pdev)
+static void tiny_spi_remove(struct platform_device *pdev)
 {
 	struct tiny_spi *hw = platform_get_drvdata(pdev);
 	struct spi_master *master = hw->bitbang.master;
 
 	spi_bitbang_stop(&hw->bitbang);
 	spi_master_put(master);
-	return 0;
 }
 
 #ifdef CONFIG_OF
@@ -291,7 +290,7 @@ MODULE_DEVICE_TABLE(of, tiny_spi_match);
 
 static struct platform_driver tiny_spi_driver = {
 	.probe = tiny_spi_probe,
-	.remove = tiny_spi_remove,
+	.remove_new = tiny_spi_remove,
 	.driver = {
 		.name = DRV_NAME,
 		.pm = NULL,
diff --git a/drivers/spi/spi-omap-uwire.c b/drivers/spi/spi-omap-uwire.c
index 20c87163d612..902d2e0c1f2f 100644
--- a/drivers/spi/spi-omap-uwire.c
+++ b/drivers/spi/spi-omap-uwire.c
@@ -179,7 +179,7 @@ static void uwire_chipselect(struct spi_device *spi, int value)
 
 	w = uwire_read_reg(UWIRE_CSR);
 	old_cs = (w >> 10) & 0x03;
-	if (value == BITBANG_CS_INACTIVE || old_cs != spi->chip_select) {
+	if (value == BITBANG_CS_INACTIVE || old_cs != spi_get_chipselect(spi, 0)) {
 		/* Deselect this CS, or the previous CS */
 		w &= ~CS_CMD;
 		uwire_write_reg(UWIRE_CSR, w);
@@ -193,7 +193,7 @@ static void uwire_chipselect(struct spi_device *spi, int value)
 		else
 			uwire_write_reg(UWIRE_SR4, 0);
 
-		w = spi->chip_select << 10;
+		w = spi_get_chipselect(spi, 0) << 10;
 		w |= CS_CMD;
 		uwire_write_reg(UWIRE_CSR, w);
 	}
@@ -210,7 +210,7 @@ static int uwire_txrx(struct spi_device *spi, struct spi_transfer *t)
 	if (!t->tx_buf && !t->rx_buf)
 		return 0;
 
-	w = spi->chip_select << 10;
+	w = spi_get_chipselect(spi, 0) << 10;
 	w |= CS_CMD;
 
 	if (t->tx_buf) {
@@ -408,7 +408,7 @@ static int uwire_setup_transfer(struct spi_device *spi, struct spi_transfer *t)
 		rate /= 8;
 		break;
 	}
-	omap_uwire_configure_mode(spi->chip_select, flags);
+	omap_uwire_configure_mode(spi_get_chipselect(spi, 0), flags);
 	pr_debug("%s: uwire flags %02x, armxor %lu KHz, SCK %lu KHz\n",
 			__func__, flags,
 			clk_get_rate(uwire->ck) / 1000,
@@ -505,7 +505,7 @@ static int uwire_probe(struct platform_device *pdev)
 	return status;
 }
 
-static int uwire_remove(struct platform_device *pdev)
+static void uwire_remove(struct platform_device *pdev)
 {
 	struct uwire_spi	*uwire = platform_get_drvdata(pdev);
 
@@ -513,7 +513,6 @@ static int uwire_remove(struct platform_device *pdev)
 
 	spi_bitbang_stop(&uwire->bitbang);
 	uwire_off(uwire);
-	return 0;
 }
 
 /* work with hotplug and coldplug */
@@ -524,7 +523,7 @@ static struct platform_driver uwire_driver = {
 		.name		= "omap_uwire",
 	},
 	.probe = uwire_probe,
-	.remove = uwire_remove,
+	.remove_new = uwire_remove,
 	// suspend ... unuse ck
 	// resume ... use ck
 };
diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c
index 6ba9b0d7710b..8331e247bf5c 100644
--- a/drivers/spi/spi-omap2-mcspi.c
+++ b/drivers/spi/spi-omap2-mcspi.c
@@ -379,7 +379,7 @@ static void omap2_mcspi_rx_callback(void *data)
 {
 	struct spi_device *spi = data;
 	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
-	struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
+	struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 
 	/* We must disable the DMA RX request */
 	omap2_mcspi_set_dma_req(spi, 1, 0);
@@ -391,7 +391,7 @@ static void omap2_mcspi_tx_callback(void *data)
 {
 	struct spi_device *spi = data;
 	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
-	struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi->chip_select];
+	struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 
 	/* We must disable the DMA TX request */
 	omap2_mcspi_set_dma_req(spi, 0, 0);
@@ -408,7 +408,7 @@ static void omap2_mcspi_tx_dma(struct spi_device *spi,
 	struct dma_async_tx_descriptor *tx;
 
 	mcspi = spi_master_get_devdata(spi->master);
-	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
+	mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 
 	dmaengine_slave_config(mcspi_dma->dma_tx, &cfg);
 
@@ -446,7 +446,7 @@ omap2_mcspi_rx_dma(struct spi_device *spi, struct spi_transfer *xfer,
 	struct dma_async_tx_descriptor *tx;
 
 	mcspi = spi_master_get_devdata(spi->master);
-	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
+	mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 	count = xfer->len;
 
 	/*
@@ -591,7 +591,7 @@ omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer)
 	int			wait_res;
 
 	mcspi = spi_master_get_devdata(spi->master);
-	mcspi_dma = &mcspi->dma_channels[spi->chip_select];
+	mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 
 	if (cs->word_len <= 8) {
 		width = DMA_SLAVE_BUSWIDTH_1_BYTE;
@@ -1062,8 +1062,8 @@ static int omap2_mcspi_setup(struct spi_device *spi)
 		cs = kzalloc(sizeof(*cs), GFP_KERNEL);
 		if (!cs)
 			return -ENOMEM;
-		cs->base = mcspi->base + spi->chip_select * 0x14;
-		cs->phys = mcspi->phys + spi->chip_select * 0x14;
+		cs->base = mcspi->base + spi_get_chipselect(spi, 0) * 0x14;
+		cs->phys = mcspi->phys + spi_get_chipselect(spi, 0) * 0x14;
 		cs->mode = 0;
 		cs->chconf0 = 0;
 		cs->chctrl0 = 0;
@@ -1142,7 +1142,7 @@ static int omap2_mcspi_transfer_one(struct spi_master *master,
 	u32				chconf;
 
 	mcspi = spi_master_get_devdata(master);
-	mcspi_dma = mcspi->dma_channels + spi->chip_select;
+	mcspi_dma = mcspi->dma_channels + spi_get_chipselect(spi, 0);
 	cs = spi->controller_state;
 	cd = spi->controller_data;
 
@@ -1158,7 +1158,7 @@ static int omap2_mcspi_transfer_one(struct spi_master *master,
 
 	omap2_mcspi_set_enable(spi, 0);
 
-	if (spi->cs_gpiod)
+	if (spi_get_csgpiod(spi, 0))
 		omap2_mcspi_set_cs(spi, spi->mode & SPI_CS_HIGH);
 
 	if (par_override ||
@@ -1247,7 +1247,7 @@ out:
 
 	omap2_mcspi_set_enable(spi, 0);
 
-	if (spi->cs_gpiod)
+	if (spi_get_csgpiod(spi, 0))
 		omap2_mcspi_set_cs(spi, !(spi->mode & SPI_CS_HIGH));
 
 	if (mcspi->fifo_depth > 0 && t)
@@ -1289,7 +1289,7 @@ static bool omap2_mcspi_can_dma(struct spi_master *master,
 {
 	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 	struct omap2_mcspi_dma *mcspi_dma =
-		&mcspi->dma_channels[spi->chip_select];
+		&mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 
 	if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx)
 		return false;
@@ -1307,7 +1307,7 @@ static size_t omap2_mcspi_max_xfer_size(struct spi_device *spi)
 {
 	struct omap2_mcspi *mcspi = spi_master_get_devdata(spi->master);
 	struct omap2_mcspi_dma *mcspi_dma =
-		&mcspi->dma_channels[spi->chip_select];
+		&mcspi->dma_channels[spi_get_chipselect(spi, 0)];
 
 	if (mcspi->max_xfer_len && mcspi_dma->dma_rx)
 		return mcspi->max_xfer_len;
@@ -1464,7 +1464,7 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 
 		of_property_read_u32(node, "ti,spi-num-cs", &num_cs);
 		master->num_chipselect = num_cs;
-		if (of_get_property(node, "ti,pindir-d0-out-d1-in", NULL))
+		if (of_property_read_bool(node, "ti,pindir-d0-out-d1-in"))
 			mcspi->pin_dir = MCSPI_PINDIR_D0_OUT_D1_IN;
 	} else {
 		pdata = dev_get_platdata(&pdev->dev);
@@ -1477,8 +1477,7 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 		master->max_transfer_size = omap2_mcspi_max_xfer_size;
 	}
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	mcspi->base = devm_ioremap_resource(&pdev->dev, r);
+	mcspi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
 	if (IS_ERR(mcspi->base)) {
 		status = PTR_ERR(mcspi->base);
 		goto free_master;
@@ -1546,7 +1545,7 @@ free_master:
 	return status;
 }
 
-static int omap2_mcspi_remove(struct platform_device *pdev)
+static void omap2_mcspi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
@@ -1556,8 +1555,6 @@ static int omap2_mcspi_remove(struct platform_device *pdev)
 	pm_runtime_dont_use_autosuspend(mcspi->dev);
 	pm_runtime_put_sync(mcspi->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 /* work with hotplug and coldplug */
@@ -1610,7 +1607,7 @@ static struct platform_driver omap2_mcspi_driver = {
 		.of_match_table = omap_mcspi_of_match,
 	},
 	.probe =	omap2_mcspi_probe,
-	.remove =	omap2_mcspi_remove,
+	.remove_new =	omap2_mcspi_remove,
 };
 
 module_platform_driver(omap2_mcspi_driver);
diff --git a/drivers/spi/spi-orion.c b/drivers/spi/spi-orion.c
index 565cd4c48d7b..ad9e83e34297 100644
--- a/drivers/spi/spi-orion.c
+++ b/drivers/spi/spi-orion.c
@@ -346,7 +346,7 @@ static void orion_spi_set_cs(struct spi_device *spi, bool enable)
 	 * as it is handled by a GPIO, but that doesn't matter. What we need
 	 * is to deassert the old chip select and assert some other chip select.
 	 */
-	val |= ORION_SPI_CS(spi->chip_select);
+	val |= ORION_SPI_CS(spi_get_chipselect(spi, 0));
 
 	/*
 	 * Chip select logic is inverted from spi_set_cs(). For lines using a
@@ -470,7 +470,7 @@ orion_spi_write_read(struct spi_device *spi, struct spi_transfer *xfer)
 	unsigned int count;
 	int word_len;
 	struct orion_spi *orion_spi;
-	int cs = spi->chip_select;
+	int cs = spi_get_chipselect(spi, 0);
 	void __iomem *vaddr;
 
 	word_len = spi->bits_per_word;
@@ -728,8 +728,7 @@ static int orion_spi_probe(struct platform_device *pdev)
 		master->max_speed_hz = devdata->max_hz;
 	master->min_speed_hz = DIV_ROUND_UP(tclk_hz, devdata->max_divisor);
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	spi->base = devm_ioremap_resource(&pdev->dev, r);
+	spi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
 	if (IS_ERR(spi->base)) {
 		status = PTR_ERR(spi->base);
 		goto out_rel_axi_clk;
@@ -805,7 +804,7 @@ out:
 }
 
 
-static int orion_spi_remove(struct platform_device *pdev)
+static void orion_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct orion_spi *spi = spi_master_get_devdata(master);
@@ -816,8 +815,6 @@ static int orion_spi_remove(struct platform_device *pdev)
 
 	spi_unregister_master(master);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 MODULE_ALIAS("platform:" DRIVER_NAME);
@@ -857,7 +854,7 @@ static struct platform_driver orion_spi_driver = {
 		.of_match_table = of_match_ptr(orion_spi_of_match_table),
 	},
 	.probe		= orion_spi_probe,
-	.remove		= orion_spi_remove,
+	.remove_new	= orion_spi_remove,
 };
 
 module_platform_driver(orion_spi_driver);
diff --git a/drivers/spi/spi-pci1xxxx.c b/drivers/spi/spi-pci1xxxx.c
index a31c3b612a43..4445d82409d6 100644
--- a/drivers/spi/spi-pci1xxxx.c
+++ b/drivers/spi/spi-pci1xxxx.c
@@ -58,7 +58,7 @@
 #define VENDOR_ID_MCHP 0x1055
 
 #define SPI_SUSPEND_CONFIG 0x101
-#define SPI_RESUME_CONFIG 0x303
+#define SPI_RESUME_CONFIG 0x203
 
 struct pci1xxxx_spi_internal {
 	u8 hw_inst;
@@ -114,17 +114,14 @@ static void pci1xxxx_spi_set_cs(struct spi_device *spi, bool enable)
 
 	/* Set the DEV_SEL bits of the SPI_MST_CTL_REG */
 	regval = readl(par->reg_base + SPI_MST_CTL_REG_OFFSET(p->hw_inst));
-	if (enable) {
+	if (!enable) {
+		regval |= SPI_FORCE_CE;
 		regval &= ~SPI_MST_CTL_DEVSEL_MASK;
-		regval |= (spi->chip_select << 25);
-		writel(regval,
-		       par->reg_base + SPI_MST_CTL_REG_OFFSET(p->hw_inst));
+		regval |= (spi_get_chipselect(spi, 0) << 25);
 	} else {
-		regval &= ~(spi->chip_select << 25);
-		writel(regval,
-		       par->reg_base + SPI_MST_CTL_REG_OFFSET(p->hw_inst));
-
+		regval &= ~SPI_FORCE_CE;
 	}
+	writel(regval, par->reg_base + SPI_MST_CTL_REG_OFFSET(p->hw_inst));
 }
 
 static u8 pci1xxxx_get_clock_div(u32 hz)
@@ -199,8 +196,9 @@ static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr,
 			else
 				regval &= ~SPI_MST_CTL_MODE_SEL;
 
-			regval |= ((clkdiv << 5) | SPI_FORCE_CE | (len << 8));
+			regval |= (clkdiv << 5);
 			regval &= ~SPI_MST_CTL_CMD_LEN_MASK;
+			regval |= (len << 8);
 			writel(regval, par->reg_base +
 			       SPI_MST_CTL_REG_OFFSET(p->hw_inst));
 			regval = readl(par->reg_base +
@@ -222,10 +220,6 @@ static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr,
 			}
 		}
 	}
-
-	regval = readl(par->reg_base + SPI_MST_CTL_REG_OFFSET(p->hw_inst));
-	regval &= ~SPI_FORCE_CE;
-	writel(regval, par->reg_base + SPI_MST_CTL_REG_OFFSET(p->hw_inst));
 	p->spi_xfer_in_progress = false;
 
 	return 0;
diff --git a/drivers/spi/spi-pic32-sqi.c b/drivers/spi/spi-pic32-sqi.c
index 86ad17597f5f..51dfb49523f3 100644
--- a/drivers/spi/spi-pic32-sqi.c
+++ b/drivers/spi/spi-pic32-sqi.c
@@ -267,7 +267,7 @@ static int pic32_sqi_one_transfer(struct pic32_sqi *sqi,
 	u32 nbits;
 
 	/* Device selection */
-	bd_ctrl = spi->chip_select << BD_DEVSEL_SHIFT;
+	bd_ctrl = spi_get_chipselect(spi, 0) << BD_DEVSEL_SHIFT;
 
 	/* half-duplex: select transfer buffer, direction and lane */
 	if (xfer->rx_buf) {
@@ -678,7 +678,7 @@ err_free_master:
 	return ret;
 }
 
-static int pic32_sqi_remove(struct platform_device *pdev)
+static void pic32_sqi_remove(struct platform_device *pdev)
 {
 	struct pic32_sqi *sqi = platform_get_drvdata(pdev);
 
@@ -689,8 +689,6 @@ static int pic32_sqi_remove(struct platform_device *pdev)
 	/* disable clk */
 	clk_disable_unprepare(sqi->base_clk);
 	clk_disable_unprepare(sqi->sys_clk);
-
-	return 0;
 }
 
 static const struct of_device_id pic32_sqi_of_ids[] = {
@@ -705,7 +703,7 @@ static struct platform_driver pic32_sqi_driver = {
 		.of_match_table = of_match_ptr(pic32_sqi_of_ids),
 	},
 	.probe = pic32_sqi_probe,
-	.remove = pic32_sqi_remove,
+	.remove_new = pic32_sqi_remove,
 };
 
 module_platform_driver(pic32_sqi_driver);
diff --git a/drivers/spi/spi-pic32.c b/drivers/spi/spi-pic32.c
index 7e5c09a7d489..f2af5e653f3d 100644
--- a/drivers/spi/spi-pic32.c
+++ b/drivers/spi/spi-pic32.c
@@ -591,7 +591,7 @@ static int pic32_spi_setup(struct spi_device *spi)
 	 * unreliable/erroneous SPI transactions.
 	 * To avoid that we will always handle /CS by toggling GPIO.
 	 */
-	if (!spi->cs_gpiod)
+	if (!spi_get_csgpiod(spi, 0))
 		return -EINVAL;
 
 	return 0;
@@ -600,7 +600,7 @@ static int pic32_spi_setup(struct spi_device *spi)
 static void pic32_spi_cleanup(struct spi_device *spi)
 {
 	/* de-activate cs-gpio, gpiolib will handle inversion */
-	gpiod_direction_output(spi->cs_gpiod, 0);
+	gpiod_direction_output(spi_get_csgpiod(spi, 0), 0);
 }
 
 static int pic32_spi_dma_prep(struct pic32_spi *pic32s, struct device *dev)
@@ -710,8 +710,7 @@ static int pic32_spi_hw_probe(struct platform_device *pdev,
 	struct resource *mem;
 	int ret;
 
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	pic32s->regs = devm_ioremap_resource(&pdev->dev, mem);
+	pic32s->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &mem);
 	if (IS_ERR(pic32s->regs))
 		return PTR_ERR(pic32s->regs);
 
@@ -844,7 +843,7 @@ err_master:
 	return ret;
 }
 
-static int pic32_spi_remove(struct platform_device *pdev)
+static void pic32_spi_remove(struct platform_device *pdev)
 {
 	struct pic32_spi *pic32s;
 
@@ -852,8 +851,6 @@ static int pic32_spi_remove(struct platform_device *pdev)
 	pic32_spi_disable(pic32s);
 	clk_disable_unprepare(pic32s->clk);
 	pic32_spi_dma_unprep(pic32s);
-
-	return 0;
 }
 
 static const struct of_device_id pic32_spi_of_match[] = {
@@ -868,7 +865,7 @@ static struct platform_driver pic32_spi_driver = {
 		.of_match_table = of_match_ptr(pic32_spi_of_match),
 	},
 	.probe = pic32_spi_probe,
-	.remove = pic32_spi_remove,
+	.remove_new = pic32_spi_remove,
 };
 
 module_platform_driver(pic32_spi_driver);
diff --git a/drivers/spi/spi-pl022.c b/drivers/spi/spi-pl022.c
index a17ff839117f..982407bc5d9f 100644
--- a/drivers/spi/spi-pl022.c
+++ b/drivers/spi/spi-pl022.c
@@ -1587,9 +1587,9 @@ static int pl022_transfer_one_message(struct spi_master *master,
 
 	/* Setup the SPI using the per chip configuration */
 	pl022->cur_chip = spi_get_ctldata(msg->spi);
-	pl022->cur_cs = msg->spi->chip_select;
+	pl022->cur_cs = spi_get_chipselect(msg->spi, 0);
 	/* This is always available but may be set to -ENOENT */
-	pl022->cur_gpiod = msg->spi->cs_gpiod;
+	pl022->cur_gpiod = spi_get_csgpiod(msg->spi, 0);
 
 	restore_state(pl022);
 	flush(pl022);
diff --git a/drivers/spi/spi-ppc4xx.c b/drivers/spi/spi-ppc4xx.c
index d65f047b6c82..d725e915025d 100644
--- a/drivers/spi/spi-ppc4xx.c
+++ b/drivers/spi/spi-ppc4xx.c
@@ -464,7 +464,7 @@ free_master:
 	return ret;
 }
 
-static int spi_ppc4xx_of_remove(struct platform_device *op)
+static void spi_ppc4xx_of_remove(struct platform_device *op)
 {
 	struct spi_master *master = platform_get_drvdata(op);
 	struct ppc4xx_spi *hw = spi_master_get_devdata(master);
@@ -474,7 +474,6 @@ static int spi_ppc4xx_of_remove(struct platform_device *op)
 	free_irq(hw->irqnum, hw);
 	iounmap(hw->regs);
 	spi_master_put(master);
-	return 0;
 }
 
 static const struct of_device_id spi_ppc4xx_of_match[] = {
@@ -486,7 +485,7 @@ MODULE_DEVICE_TABLE(of, spi_ppc4xx_of_match);
 
 static struct platform_driver spi_ppc4xx_of_driver = {
 	.probe = spi_ppc4xx_of_probe,
-	.remove = spi_ppc4xx_of_remove,
+	.remove_new = spi_ppc4xx_of_remove,
 	.driver = {
 		.name = DRIVER_NAME,
 		.of_match_table = spi_ppc4xx_of_match,
diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c
index 32cc82a89ec1..1bab18a0f262 100644
--- a/drivers/spi/spi-pxa2xx.c
+++ b/drivers/spi/spi-pxa2xx.c
@@ -368,7 +368,7 @@ static void lpss_ssp_select_cs(struct spi_device *spi,
 
 	value = __lpss_ssp_read_priv(drv_data, config->reg_cs_ctrl);
 
-	cs = spi->chip_select;
+	cs = spi_get_chipselect(spi, 0);
 	cs <<= config->cs_sel_shift;
 	if (cs != (value & config->cs_sel_mask)) {
 		/*
@@ -429,7 +429,7 @@ static void cs_assert(struct spi_device *spi)
 		spi_controller_get_devdata(spi->controller);
 
 	if (drv_data->ssp_type == CE4100_SSP) {
-		pxa2xx_spi_write(drv_data, SSSR, spi->chip_select);
+		pxa2xx_spi_write(drv_data, SSSR, spi_get_chipselect(spi, 0));
 		return;
 	}
 
@@ -1217,7 +1217,7 @@ static int setup(struct spi_device *spi)
 			return -ENOMEM;
 
 		if (drv_data->ssp_type == CE4100_SSP) {
-			if (spi->chip_select > 4) {
+			if (spi_get_chipselect(spi, 0) > 4) {
 				dev_err(&spi->dev,
 					"failed setup: cs number must not be > 4.\n");
 				kfree(chip);
@@ -1659,7 +1659,7 @@ out_error_controller_alloc:
 	return status;
 }
 
-static int pxa2xx_spi_remove(struct platform_device *pdev)
+static void pxa2xx_spi_remove(struct platform_device *pdev)
 {
 	struct driver_data *drv_data = platform_get_drvdata(pdev);
 	struct ssp_device *ssp = drv_data->ssp;
@@ -1684,8 +1684,6 @@ static int pxa2xx_spi_remove(struct platform_device *pdev)
 
 	/* Release SSP */
 	pxa_ssp_free(ssp);
-
-	return 0;
 }
 
 static int pxa2xx_spi_suspend(struct device *dev)
@@ -1756,7 +1754,7 @@ static const struct acpi_device_id pxa2xx_spi_acpi_match[] = {
 MODULE_DEVICE_TABLE(acpi, pxa2xx_spi_acpi_match);
 #endif
 
-static const struct of_device_id pxa2xx_spi_of_match[] = {
+static const struct of_device_id pxa2xx_spi_of_match[] __maybe_unused = {
 	{ .compatible = "marvell,mmp2-ssp", .data = (void *)MMP2_SSP },
 	{}
 };
@@ -1770,7 +1768,7 @@ static struct platform_driver driver = {
 		.of_match_table = of_match_ptr(pxa2xx_spi_of_match),
 	},
 	.probe = pxa2xx_spi_probe,
-	.remove = pxa2xx_spi_remove,
+	.remove_new = pxa2xx_spi_remove,
 };
 
 static int __init pxa2xx_spi_init(void)
diff --git a/drivers/spi/spi-qcom-qspi.c b/drivers/spi/spi-qcom-qspi.c
index c334dfec4117..fab155389999 100644
--- a/drivers/spi/spi-qcom-qspi.c
+++ b/drivers/spi/spi-qcom-qspi.c
@@ -8,6 +8,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
+#include <linux/pinctrl/consumer.h>
 #include <linux/pm_runtime.h>
 #include <linux/pm_opp.h>
 #include <linux/spi/spi.h>
@@ -311,7 +312,7 @@ static int qcom_qspi_prepare_message(struct spi_master *master,
 
 	mstr_cfg = readl(ctrl->base + MSTR_CONFIG);
 	mstr_cfg &= ~CHIP_SELECT_NUM;
-	if (message->spi->chip_select)
+	if (spi_get_chipselect(message->spi, 0))
 		mstr_cfg |= CHIP_SELECT_NUM;
 
 	mstr_cfg |= FB_CLK_EN | PIN_WPN | PIN_HOLDN | SBL_EN | FULL_CYCLE_MODE;
@@ -552,7 +553,7 @@ static int qcom_qspi_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int qcom_qspi_remove(struct platform_device *pdev)
+static void qcom_qspi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 
@@ -560,8 +561,6 @@ static int qcom_qspi_remove(struct platform_device *pdev)
 	spi_unregister_master(master);
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev)
@@ -581,6 +580,8 @@ static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev)
 		return ret;
 	}
 
+	pinctrl_pm_select_sleep_state(dev);
+
 	return 0;
 }
 
@@ -590,6 +591,8 @@ static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev)
 	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 	int ret;
 
+	pinctrl_pm_select_default_state(dev);
+
 	ret = icc_enable(ctrl->icc_path_cpu_to_qspi);
 	if (ret) {
 		dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",
@@ -655,7 +658,7 @@ static struct platform_driver qcom_qspi_driver = {
 		.of_match_table = qcom_qspi_dt_match,
 	},
 	.probe = qcom_qspi_probe,
-	.remove = qcom_qspi_remove,
+	.remove_new = qcom_qspi_remove,
 };
 module_platform_driver(qcom_qspi_driver);
 
diff --git a/drivers/spi/spi-qup.c b/drivers/spi/spi-qup.c
index 678dc51ef017..944ef6b42bce 100644
--- a/drivers/spi/spi-qup.c
+++ b/drivers/spi/spi-qup.c
@@ -1003,8 +1003,7 @@ static int spi_qup_probe(struct platform_device *pdev)
 	int ret, irq, size;
 
 	dev = &pdev->dev;
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	base = devm_ioremap_resource(dev, res);
+	base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
@@ -1271,29 +1270,31 @@ disable_clk:
 }
 #endif /* CONFIG_PM_SLEEP */
 
-static int spi_qup_remove(struct platform_device *pdev)
+static void spi_qup_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = dev_get_drvdata(&pdev->dev);
 	struct spi_qup *controller = spi_master_get_devdata(master);
 	int ret;
 
-	ret = pm_runtime_resume_and_get(&pdev->dev);
-	if (ret < 0)
-		return ret;
+	ret = pm_runtime_get_sync(&pdev->dev);
 
-	ret = spi_qup_set_state(controller, QUP_STATE_RESET);
-	if (ret)
-		return ret;
+	if (ret >= 0) {
+		ret = spi_qup_set_state(controller, QUP_STATE_RESET);
+		if (ret)
+			dev_warn(&pdev->dev, "failed to reset controller (%pe)\n",
+				 ERR_PTR(ret));
 
-	spi_qup_release_dma(master);
+		clk_disable_unprepare(controller->cclk);
+		clk_disable_unprepare(controller->iclk);
+	} else {
+		dev_warn(&pdev->dev, "failed to resume, skip hw disable (%pe)\n",
+			 ERR_PTR(ret));
+	}
 
-	clk_disable_unprepare(controller->cclk);
-	clk_disable_unprepare(controller->iclk);
+	spi_qup_release_dma(master);
 
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static const struct of_device_id spi_qup_dt_match[] = {
@@ -1318,7 +1319,7 @@ static struct platform_driver spi_qup_driver = {
 		.of_match_table = spi_qup_dt_match,
 	},
 	.probe = spi_qup_probe,
-	.remove = spi_qup_remove,
+	.remove_new = spi_qup_remove,
 };
 module_platform_driver(spi_qup_driver);
 
diff --git a/drivers/spi/spi-rb4xx.c b/drivers/spi/spi-rb4xx.c
index 9f97d18a05c1..5073736d3d1f 100644
--- a/drivers/spi/spi-rb4xx.c
+++ b/drivers/spi/spi-rb4xx.c
@@ -107,7 +107,7 @@ static int rb4xx_transfer_one(struct spi_master *master,
 	 * command set was designed to almost not clash with that of the
 	 * boot flash.
 	 */
-	if (spi->chip_select == 2)
+	if (spi_get_chipselect(spi, 0) == 2)
 		/* MMC */
 		spi_ioc = AR71XX_SPI_IOC_CS0;
 	else
@@ -181,13 +181,11 @@ static int rb4xx_spi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int rb4xx_spi_remove(struct platform_device *pdev)
+static void rb4xx_spi_remove(struct platform_device *pdev)
 {
 	struct rb4xx_spi *rbspi = platform_get_drvdata(pdev);
 
 	clk_disable_unprepare(rbspi->clk);
-
-	return 0;
 }
 
 static const struct of_device_id rb4xx_spi_dt_match[] = {
@@ -198,7 +196,7 @@ MODULE_DEVICE_TABLE(of, rb4xx_spi_dt_match);
 
 static struct platform_driver rb4xx_spi_drv = {
 	.probe = rb4xx_spi_probe,
-	.remove = rb4xx_spi_remove,
+	.remove_new = rb4xx_spi_remove,
 	.driver = {
 		.name = "rb4xx-spi",
 		.of_match_table = of_match_ptr(rb4xx_spi_dt_match),
diff --git a/drivers/spi/spi-rockchip-sfc.c b/drivers/spi/spi-rockchip-sfc.c
index bd87d3c92dd3..583f4187f030 100644
--- a/drivers/spi/spi-rockchip-sfc.c
+++ b/drivers/spi/spi-rockchip-sfc.c
@@ -346,7 +346,7 @@ static int rockchip_sfc_xfer_setup(struct rockchip_sfc *sfc,
 
 	/* set the Controller */
 	ctrl |= SFC_CTRL_PHASE_SEL_NEGETIVE;
-	cmd |= mem->spi->chip_select << SFC_CMD_CS_SHIFT;
+	cmd |= spi_get_chipselect(mem->spi, 0) << SFC_CMD_CS_SHIFT;
 
 	dev_dbg(sfc->dev, "sfc addr.nbytes=%x(x%d) dummy.nbytes=%x(x%d)\n",
 		op->addr.nbytes, op->addr.buswidth,
@@ -558,7 +558,6 @@ static int rockchip_sfc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct spi_master *master;
-	struct resource *res;
 	struct rockchip_sfc *sfc;
 	int ret;
 
@@ -576,8 +575,7 @@ static int rockchip_sfc_probe(struct platform_device *pdev)
 	sfc = spi_master_get_devdata(master);
 	sfc->dev = dev;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	sfc->regbase = devm_ioremap_resource(dev, res);
+	sfc->regbase = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(sfc->regbase))
 		return PTR_ERR(sfc->regbase);
 
@@ -632,7 +630,7 @@ static int rockchip_sfc_probe(struct platform_device *pdev)
 	if (ret) {
 		dev_err(dev, "Failed to request irq\n");
 
-		return ret;
+		goto err_irq;
 	}
 
 	ret = rockchip_sfc_init(sfc);
@@ -656,7 +654,7 @@ err_hclk:
 	return ret;
 }
 
-static int rockchip_sfc_remove(struct platform_device *pdev)
+static void rockchip_sfc_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct rockchip_sfc *sfc = platform_get_drvdata(pdev);
@@ -665,8 +663,6 @@ static int rockchip_sfc_remove(struct platform_device *pdev)
 
 	clk_disable_unprepare(sfc->clk);
 	clk_disable_unprepare(sfc->hclk);
-
-	return 0;
 }
 
 static const struct of_device_id rockchip_sfc_dt_ids[] = {
@@ -681,7 +677,7 @@ static struct platform_driver rockchip_sfc_driver = {
 		.of_match_table = rockchip_sfc_dt_ids,
 	},
 	.probe	= rockchip_sfc_probe,
-	.remove	= rockchip_sfc_remove,
+	.remove_new = rockchip_sfc_remove,
 };
 module_platform_driver(rockchip_sfc_driver);
 
diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c
index 79242dc5272d..143ede958ac1 100644
--- a/drivers/spi/spi-rockchip.c
+++ b/drivers/spi/spi-rockchip.c
@@ -246,28 +246,30 @@ static void rockchip_spi_set_cs(struct spi_device *spi, bool enable)
 	bool cs_asserted = spi->mode & SPI_CS_HIGH ? enable : !enable;
 
 	/* Return immediately for no-op */
-	if (cs_asserted == rs->cs_asserted[spi->chip_select])
+	if (cs_asserted == rs->cs_asserted[spi_get_chipselect(spi, 0)])
 		return;
 
 	if (cs_asserted) {
 		/* Keep things powered as long as CS is asserted */
 		pm_runtime_get_sync(rs->dev);
 
-		if (spi->cs_gpiod)
+		if (spi_get_csgpiod(spi, 0))
 			ROCKCHIP_SPI_SET_BITS(rs->regs + ROCKCHIP_SPI_SER, 1);
 		else
-			ROCKCHIP_SPI_SET_BITS(rs->regs + ROCKCHIP_SPI_SER, BIT(spi->chip_select));
+			ROCKCHIP_SPI_SET_BITS(rs->regs + ROCKCHIP_SPI_SER,
+					      BIT(spi_get_chipselect(spi, 0)));
 	} else {
-		if (spi->cs_gpiod)
+		if (spi_get_csgpiod(spi, 0))
 			ROCKCHIP_SPI_CLR_BITS(rs->regs + ROCKCHIP_SPI_SER, 1);
 		else
-			ROCKCHIP_SPI_CLR_BITS(rs->regs + ROCKCHIP_SPI_SER, BIT(spi->chip_select));
+			ROCKCHIP_SPI_CLR_BITS(rs->regs + ROCKCHIP_SPI_SER,
+					      BIT(spi_get_chipselect(spi, 0)));
 
 		/* Drop reference from when we first asserted CS */
 		pm_runtime_put(rs->dev);
 	}
 
-	rs->cs_asserted[spi->chip_select] = cs_asserted;
+	rs->cs_asserted[spi_get_chipselect(spi, 0)] = cs_asserted;
 }
 
 static void rockchip_spi_handle_err(struct spi_controller *ctlr,
@@ -541,7 +543,7 @@ static int rockchip_spi_config(struct rockchip_spi *rs,
 	if (spi->mode & SPI_LSB_FIRST)
 		cr0 |= CR0_FBM_LSB << CR0_FBM_OFFSET;
 	if (spi->mode & SPI_CS_HIGH)
-		cr0 |= BIT(spi->chip_select) << CR0_SOI_OFFSET;
+		cr0 |= BIT(spi_get_chipselect(spi, 0)) << CR0_SOI_OFFSET;
 
 	if (xfer->rx_buf && xfer->tx_buf)
 		cr0 |= CR0_XFM_TR << CR0_XFM_OFFSET;
@@ -724,7 +726,7 @@ static int rockchip_spi_setup(struct spi_device *spi)
 	struct rockchip_spi *rs = spi_controller_get_devdata(spi->controller);
 	u32 cr0;
 
-	if (!spi->cs_gpiod && (spi->mode & SPI_CS_HIGH) && !rs->cs_high_supported) {
+	if (!spi_get_csgpiod(spi, 0) && (spi->mode & SPI_CS_HIGH) && !rs->cs_high_supported) {
 		dev_warn(&spi->dev, "setup: non GPIO CS can't be active-high\n");
 		return -EINVAL;
 	}
@@ -735,10 +737,10 @@ static int rockchip_spi_setup(struct spi_device *spi)
 
 	cr0 &= ~(0x3 << CR0_SCPH_OFFSET);
 	cr0 |= ((spi->mode & 0x3) << CR0_SCPH_OFFSET);
-	if (spi->mode & SPI_CS_HIGH && spi->chip_select <= 1)
-		cr0 |= BIT(spi->chip_select) << CR0_SOI_OFFSET;
-	else if (spi->chip_select <= 1)
-		cr0 &= ~(BIT(spi->chip_select) << CR0_SOI_OFFSET);
+	if (spi->mode & SPI_CS_HIGH && spi_get_chipselect(spi, 0) <= 1)
+		cr0 |= BIT(spi_get_chipselect(spi, 0)) << CR0_SOI_OFFSET;
+	else if (spi_get_chipselect(spi, 0) <= 1)
+		cr0 &= ~(BIT(spi_get_chipselect(spi, 0)) << CR0_SOI_OFFSET);
 
 	writel_relaxed(cr0, rs->regs + ROCKCHIP_SPI_CTRLR0);
 
@@ -772,11 +774,9 @@ static int rockchip_spi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, ctlr);
 
 	rs = spi_controller_get_devdata(ctlr);
-	ctlr->slave = slave_mode;
 
 	/* Get basic io resource and map it */
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	rs->regs = devm_ioremap_resource(&pdev->dev, mem);
+	rs->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &mem);
 	if (IS_ERR(rs->regs)) {
 		ret =  PTR_ERR(rs->regs);
 		goto err_put_ctlr;
@@ -947,7 +947,7 @@ err_put_ctlr:
 	return ret;
 }
 
-static int rockchip_spi_remove(struct platform_device *pdev)
+static void rockchip_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr = spi_controller_get(platform_get_drvdata(pdev));
 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);
@@ -967,8 +967,6 @@ static int rockchip_spi_remove(struct platform_device *pdev)
 		dma_release_channel(ctlr->dma_rx);
 
 	spi_controller_put(ctlr);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1076,7 +1074,7 @@ static struct platform_driver rockchip_spi_driver = {
 		.of_match_table = of_match_ptr(rockchip_spi_dt_match),
 	},
 	.probe = rockchip_spi_probe,
-	.remove = rockchip_spi_remove,
+	.remove_new = rockchip_spi_remove,
 };
 
 module_platform_driver(rockchip_spi_driver);
diff --git a/drivers/spi/spi-rpc-if.c b/drivers/spi/spi-rpc-if.c
index ec0904faf3a1..2f78124a1b59 100644
--- a/drivers/spi/spi-rpc-if.c
+++ b/drivers/spi/spi-rpc-if.c
@@ -173,15 +173,13 @@ out_disable_rpm:
 	return error;
 }
 
-static int rpcif_spi_remove(struct platform_device *pdev)
+static void rpcif_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr = platform_get_drvdata(pdev);
 	struct rpcif *rpc = spi_controller_get_devdata(ctlr);
 
 	spi_unregister_controller(ctlr);
 	pm_runtime_disable(rpc->dev);
-
-	return 0;
 }
 
 static int __maybe_unused rpcif_spi_suspend(struct device *dev)
@@ -202,7 +200,7 @@ static SIMPLE_DEV_PM_OPS(rpcif_spi_pm_ops, rpcif_spi_suspend, rpcif_spi_resume);
 
 static struct platform_driver rpcif_spi_driver = {
 	.probe	= rpcif_spi_probe,
-	.remove	= rpcif_spi_remove,
+	.remove_new = rpcif_spi_remove,
 	.driver = {
 		.name	= "rpc-if-spi",
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/spi/spi-rspi.c b/drivers/spi/spi-rspi.c
index 411b1307b7fd..08ceebbaf69b 100644
--- a/drivers/spi/spi-rspi.c
+++ b/drivers/spi/spi-rspi.c
@@ -950,7 +950,7 @@ static int rspi_setup(struct spi_device *spi)
 	struct rspi_data *rspi = spi_controller_get_devdata(spi->controller);
 	u8 sslp;
 
-	if (spi->cs_gpiod)
+	if (spi_get_csgpiod(spi, 0))
 		return 0;
 
 	pm_runtime_get_sync(&rspi->pdev->dev);
@@ -958,9 +958,9 @@ static int rspi_setup(struct spi_device *spi)
 
 	sslp = rspi_read8(rspi, RSPI_SSLP);
 	if (spi->mode & SPI_CS_HIGH)
-		sslp |= SSLP_SSLP(spi->chip_select);
+		sslp |= SSLP_SSLP(spi_get_chipselect(spi, 0));
 	else
-		sslp &= ~SSLP_SSLP(spi->chip_select);
+		sslp &= ~SSLP_SSLP(spi_get_chipselect(spi, 0));
 	rspi_write8(rspi, sslp, RSPI_SSLP);
 
 	spin_unlock_irq(&rspi->lock);
@@ -1001,8 +1001,8 @@ static int rspi_prepare_message(struct spi_controller *ctlr,
 		rspi->spcmd |= SPCMD_LSBF;
 
 	/* Configure slave signal to assert */
-	rspi->spcmd |= SPCMD_SSLA(spi->cs_gpiod ? rspi->ctlr->unused_native_cs
-						: spi->chip_select);
+	rspi->spcmd |= SPCMD_SSLA(spi_get_csgpiod(spi, 0) ? rspi->ctlr->unused_native_cs
+						: spi_get_chipselect(spi, 0));
 
 	/* CMOS output mode and MOSI signal from previous transfer */
 	rspi->sppcr = 0;
@@ -1172,14 +1172,12 @@ static void rspi_release_dma(struct spi_controller *ctlr)
 		dma_release_channel(ctlr->dma_rx);
 }
 
-static int rspi_remove(struct platform_device *pdev)
+static void rspi_remove(struct platform_device *pdev)
 {
 	struct rspi_data *rspi = platform_get_drvdata(pdev);
 
 	rspi_release_dma(rspi->ctlr);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static const struct spi_ops rspi_ops = {
@@ -1192,7 +1190,7 @@ static const struct spi_ops rspi_ops = {
 	.num_hw_ss =		2,
 };
 
-static const struct spi_ops rspi_rz_ops = {
+static const struct spi_ops rspi_rz_ops __maybe_unused = {
 	.set_config_register =	rspi_rz_set_config_register,
 	.transfer_one =		rspi_rz_transfer_one,
 	.min_div =		2,
@@ -1202,7 +1200,7 @@ static const struct spi_ops rspi_rz_ops = {
 	.num_hw_ss =		1,
 };
 
-static const struct spi_ops qspi_ops = {
+static const struct spi_ops qspi_ops __maybe_unused = {
 	.set_config_register =	qspi_set_config_register,
 	.transfer_one =		qspi_transfer_one,
 	.extra_mode_bits =	SPI_TX_DUAL | SPI_TX_QUAD |
@@ -1214,8 +1212,7 @@ static const struct spi_ops qspi_ops = {
 	.num_hw_ss =		1,
 };
 
-#ifdef CONFIG_OF
-static const struct of_device_id rspi_of_match[] = {
+static const struct of_device_id rspi_of_match[] __maybe_unused = {
 	/* RSPI on legacy SH */
 	{ .compatible = "renesas,rspi", .data = &rspi_ops },
 	/* RSPI on RZ/A1H */
@@ -1227,6 +1224,7 @@ static const struct of_device_id rspi_of_match[] = {
 
 MODULE_DEVICE_TABLE(of, rspi_of_match);
 
+#ifdef CONFIG_OF
 static void rspi_reset_control_assert(void *data)
 {
 	reset_control_assert(data);
@@ -1440,7 +1438,7 @@ static SIMPLE_DEV_PM_OPS(rspi_pm_ops, rspi_suspend, rspi_resume);
 
 static struct platform_driver rspi_driver = {
 	.probe =	rspi_probe,
-	.remove =	rspi_remove,
+	.remove_new =	rspi_remove,
 	.id_table =	spi_driver_ids,
 	.driver		= {
 		.name = "renesas_spi",
diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c
index 71d324ec9a70..7ac17f0d18a9 100644
--- a/drivers/spi/spi-s3c64xx.c
+++ b/drivers/spi/spi-s3c64xx.c
@@ -891,7 +891,7 @@ static int s3c64xx_spi_setup(struct spi_device *spi)
 
 	/* NULL is fine, we just avoid using the FB delay (=0) */
 	if (IS_ERR(cs)) {
-		dev_err(&spi->dev, "No CS for SPI(%d)\n", spi->chip_select);
+		dev_err(&spi->dev, "No CS for SPI(%d)\n", spi_get_chipselect(spi, 0));
 		return -ENODEV;
 	}
 
@@ -1286,7 +1286,7 @@ err_deref_master:
 	return ret;
 }
 
-static int s3c64xx_spi_remove(struct platform_device *pdev)
+static void s3c64xx_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct s3c64xx_spi_driver_data *sdd = spi_master_get_devdata(master);
@@ -1309,8 +1309,6 @@ static int s3c64xx_spi_remove(struct platform_device *pdev)
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 	pm_runtime_set_suspended(&pdev->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1531,7 +1529,7 @@ static struct platform_driver s3c64xx_spi_driver = {
 		.of_match_table = of_match_ptr(s3c64xx_spi_dt_match),
 	},
 	.probe = s3c64xx_spi_probe,
-	.remove = s3c64xx_spi_remove,
+	.remove_new = s3c64xx_spi_remove,
 	.id_table = s3c64xx_spi_driver_ids,
 };
 MODULE_ALIAS("platform:s3c64xx-spi");
diff --git a/drivers/spi/spi-sc18is602.c b/drivers/spi/spi-sc18is602.c
index 983b3621bc2a..70012333020b 100644
--- a/drivers/spi/spi-sc18is602.c
+++ b/drivers/spi/spi-sc18is602.c
@@ -70,7 +70,7 @@ static int sc18is602_txrx(struct sc18is602 *hw, struct spi_message *msg,
 
 	if (hw->tlen == 0) {
 		/* First byte (I2C command) is chip select */
-		hw->buffer[0] = 1 << msg->spi->chip_select;
+		hw->buffer[0] = 1 << spi_get_chipselect(msg->spi, 0);
 		hw->tlen = 1;
 		hw->rindex = 0;
 	}
@@ -229,7 +229,7 @@ static int sc18is602_setup(struct spi_device *spi)
 	struct sc18is602 *hw = spi_master_get_devdata(spi->master);
 
 	/* SC18IS602 does not support CS2 */
-	if (hw->id == sc18is602 && spi->chip_select == 2)
+	if (hw->id == sc18is602 && (spi_get_chipselect(spi, 0) == 2))
 		return -ENXIO;
 
 	return 0;
@@ -315,7 +315,7 @@ static const struct i2c_device_id sc18is602_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, sc18is602_id);
 
-static const struct of_device_id sc18is602_of_match[] = {
+static const struct of_device_id sc18is602_of_match[] __maybe_unused = {
 	{
 		.compatible = "nxp,sc18is602",
 		.data = (void *)sc18is602
diff --git a/drivers/spi/spi-sh-hspi.c b/drivers/spi/spi-sh-hspi.c
index a62034e2a7cb..d6ffeae66ed3 100644
--- a/drivers/spi/spi-sh-hspi.c
+++ b/drivers/spi/spi-sh-hspi.c
@@ -276,15 +276,13 @@ static int hspi_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int hspi_remove(struct platform_device *pdev)
+static void hspi_remove(struct platform_device *pdev)
 {
 	struct hspi_priv *hspi = platform_get_drvdata(pdev);
 
 	pm_runtime_disable(&pdev->dev);
 
 	clk_put(hspi->clk);
-
-	return 0;
 }
 
 static const struct of_device_id hspi_of_match[] = {
@@ -295,7 +293,7 @@ MODULE_DEVICE_TABLE(of, hspi_of_match);
 
 static struct platform_driver hspi_driver = {
 	.probe = hspi_probe,
-	.remove = hspi_remove,
+	.remove_new = hspi_remove,
 	.driver = {
 		.name = "sh-hspi",
 		.of_match_table = hspi_of_match,
diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c
index 9bca3d076f05..9e90b4f8b357 100644
--- a/drivers/spi/spi-sh-msiof.c
+++ b/drivers/spi/spi-sh-msiof.c
@@ -554,7 +554,7 @@ static int sh_msiof_spi_setup(struct spi_device *spi)
 		spi_controller_get_devdata(spi->controller);
 	u32 clr, set, tmp;
 
-	if (spi->cs_gpiod || spi_controller_is_slave(p->ctlr))
+	if (spi_get_csgpiod(spi, 0) || spi_controller_is_slave(p->ctlr))
 		return 0;
 
 	if (p->native_cs_inited &&
@@ -587,11 +587,11 @@ static int sh_msiof_prepare_message(struct spi_controller *ctlr,
 	u32 ss, cs_high;
 
 	/* Configure pins before asserting CS */
-	if (spi->cs_gpiod) {
+	if (spi_get_csgpiod(spi, 0)) {
 		ss = ctlr->unused_native_cs;
 		cs_high = p->native_cs_high;
 	} else {
-		ss = spi->chip_select;
+		ss = spi_get_chipselect(spi, 0);
 		cs_high = !!(spi->mode & SPI_CS_HIGH);
 	}
 	sh_msiof_spi_set_pin_regs(p, ss, !!(spi->mode & SPI_CPOL),
@@ -1073,7 +1073,7 @@ static const struct sh_msiof_chipdata rcar_gen3_data = {
 	.min_div_pow = 1,
 };
 
-static const struct of_device_id sh_msiof_match[] = {
+static const struct of_device_id sh_msiof_match[] __maybe_unused = {
 	{ .compatible = "renesas,sh-mobile-msiof", .data = &sh_data },
 	{ .compatible = "renesas,msiof-r8a7743",   .data = &rcar_gen2_data },
 	{ .compatible = "renesas,msiof-r8a7745",   .data = &rcar_gen2_data },
@@ -1375,13 +1375,12 @@ static int sh_msiof_spi_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int sh_msiof_spi_remove(struct platform_device *pdev)
+static void sh_msiof_spi_remove(struct platform_device *pdev)
 {
 	struct sh_msiof_spi_priv *p = platform_get_drvdata(pdev);
 
 	sh_msiof_release_dma(p);
 	pm_runtime_disable(&pdev->dev);
-	return 0;
 }
 
 static const struct platform_device_id spi_driver_ids[] = {
@@ -1414,7 +1413,7 @@ static SIMPLE_DEV_PM_OPS(sh_msiof_spi_pm_ops, sh_msiof_spi_suspend,
 
 static struct platform_driver sh_msiof_spi_drv = {
 	.probe		= sh_msiof_spi_probe,
-	.remove		= sh_msiof_spi_remove,
+	.remove_new	= sh_msiof_spi_remove,
 	.id_table	= spi_driver_ids,
 	.driver		= {
 		.name		= "spi_sh_msiof",
diff --git a/drivers/spi/spi-sh-sci.c b/drivers/spi/spi-sh-sci.c
index 8f30531e1418..92ca3f2d61ba 100644
--- a/drivers/spi/spi-sh-sci.c
+++ b/drivers/spi/spi-sh-sci.c
@@ -108,7 +108,7 @@ static void sh_sci_spi_chipselect(struct spi_device *dev, int value)
 	struct sh_sci_spi *sp = spi_master_get_devdata(dev->master);
 
 	if (sp->info->chip_select)
-		(sp->info->chip_select)(sp->info, dev->chip_select, value);
+		(sp->info->chip_select)(sp->info, spi_get_chipselect(dev, 0), value);
 }
 
 static int sh_sci_spi_probe(struct platform_device *dev)
@@ -171,7 +171,7 @@ static int sh_sci_spi_probe(struct platform_device *dev)
 	return ret;
 }
 
-static int sh_sci_spi_remove(struct platform_device *dev)
+static void sh_sci_spi_remove(struct platform_device *dev)
 {
 	struct sh_sci_spi *sp = platform_get_drvdata(dev);
 
@@ -179,12 +179,11 @@ static int sh_sci_spi_remove(struct platform_device *dev)
 	setbits(sp, PIN_INIT, 0);
 	iounmap(sp->membase);
 	spi_master_put(sp->bitbang.master);
-	return 0;
 }
 
 static struct platform_driver sh_sci_spi_drv = {
 	.probe		= sh_sci_spi_probe,
-	.remove		= sh_sci_spi_remove,
+	.remove_new	= sh_sci_spi_remove,
 	.driver		= {
 		.name	= "spi_sh_sci",
 	},
diff --git a/drivers/spi/spi-sh.c b/drivers/spi/spi-sh.c
index 3e72fad99adf..d358a2a9c3f5 100644
--- a/drivers/spi/spi-sh.c
+++ b/drivers/spi/spi-sh.c
@@ -377,14 +377,12 @@ static irqreturn_t spi_sh_irq(int irq, void *_ss)
 	return IRQ_HANDLED;
 }
 
-static int spi_sh_remove(struct platform_device *pdev)
+static void spi_sh_remove(struct platform_device *pdev)
 {
 	struct spi_sh_data *ss = platform_get_drvdata(pdev);
 
 	spi_unregister_master(ss->master);
 	free_irq(ss->irq, ss);
-
-	return 0;
 }
 
 static int spi_sh_probe(struct platform_device *pdev)
@@ -461,7 +459,7 @@ static int spi_sh_probe(struct platform_device *pdev)
 
 static struct platform_driver spi_sh_driver = {
 	.probe = spi_sh_probe,
-	.remove = spi_sh_remove,
+	.remove_new = spi_sh_remove,
 	.driver = {
 		.name = "sh_spi",
 	},
diff --git a/drivers/spi/spi-sifive.c b/drivers/spi/spi-sifive.c
index e29e85cee88a..dae9e097c333 100644
--- a/drivers/spi/spi-sifive.c
+++ b/drivers/spi/spi-sifive.c
@@ -135,13 +135,13 @@ sifive_spi_prepare_message(struct spi_master *master, struct spi_message *msg)
 
 	/* Update the chip select polarity */
 	if (device->mode & SPI_CS_HIGH)
-		spi->cs_inactive &= ~BIT(device->chip_select);
+		spi->cs_inactive &= ~BIT(spi_get_chipselect(device, 0));
 	else
-		spi->cs_inactive |= BIT(device->chip_select);
+		spi->cs_inactive |= BIT(spi_get_chipselect(device, 0));
 	sifive_spi_write(spi, SIFIVE_SPI_REG_CSDEF, spi->cs_inactive);
 
 	/* Select the correct device */
-	sifive_spi_write(spi, SIFIVE_SPI_REG_CSID, device->chip_select);
+	sifive_spi_write(spi, SIFIVE_SPI_REG_CSID, spi_get_chipselect(device, 0));
 
 	/* Set clock mode */
 	sifive_spi_write(spi, SIFIVE_SPI_REG_SCKMODE,
@@ -415,7 +415,7 @@ put_master:
 	return ret;
 }
 
-static int sifive_spi_remove(struct platform_device *pdev)
+static void sifive_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct sifive_spi *spi = spi_master_get_devdata(master);
@@ -423,8 +423,6 @@ static int sifive_spi_remove(struct platform_device *pdev)
 	/* Disable all the interrupts just in case */
 	sifive_spi_write(spi, SIFIVE_SPI_REG_IE, 0);
 	clk_disable_unprepare(spi->clk);
-
-	return 0;
 }
 
 static int sifive_spi_suspend(struct device *dev)
@@ -473,7 +471,7 @@ MODULE_DEVICE_TABLE(of, sifive_spi_of_match);
 
 static struct platform_driver sifive_spi_driver = {
 	.probe = sifive_spi_probe,
-	.remove = sifive_spi_remove,
+	.remove_new = sifive_spi_remove,
 	.driver = {
 		.name = SIFIVE_SPI_DRIVER_NAME,
 		.pm = &sifive_spi_pm_ops,
diff --git a/drivers/spi/spi-slave-mt27xx.c b/drivers/spi/spi-slave-mt27xx.c
index f199a6c4738a..4e4d426bfb43 100644
--- a/drivers/spi/spi-slave-mt27xx.c
+++ b/drivers/spi/spi-slave-mt27xx.c
@@ -474,11 +474,9 @@ err_put_ctlr:
 	return ret;
 }
 
-static int mtk_spi_slave_remove(struct platform_device *pdev)
+static void mtk_spi_slave_remove(struct platform_device *pdev)
 {
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -560,7 +558,7 @@ static struct platform_driver mtk_spi_slave_driver = {
 		.of_match_table = mtk_spi_slave_of_match,
 	},
 	.probe = mtk_spi_slave_probe,
-	.remove = mtk_spi_slave_remove,
+	.remove_new = mtk_spi_slave_remove,
 };
 
 module_platform_driver(mtk_spi_slave_driver);
diff --git a/drivers/spi/spi-sn-f-ospi.c b/drivers/spi/spi-sn-f-ospi.c
index 333b22dfd8db..a2bd9dcde075 100644
--- a/drivers/spi/spi-sn-f-ospi.c
+++ b/drivers/spi/spi-sn-f-ospi.c
@@ -267,7 +267,7 @@ static void f_ospi_config_indir_protocol(struct f_ospi *ospi,
 	int unit;
 
 	/* Set one chip select */
-	writel(BIT(spi->chip_select), ospi->base + OSPI_SSEL);
+	writel(BIT(spi_get_chipselect(spi, 0)), ospi->base + OSPI_SSEL);
 
 	mode = f_ospi_get_mode(ospi, op->cmd.buswidth, 1);
 	prot |= FIELD_PREP(OSPI_PROT_MODE_CODE_MASK, mode);
@@ -561,7 +561,7 @@ static bool f_ospi_supports_op(struct spi_mem *mem,
 	if (!f_ospi_supports_op_width(mem, op))
 		return false;
 
-	return true;
+	return spi_mem_default_supports_op(mem, op);
 }
 
 static int f_ospi_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
@@ -670,15 +670,13 @@ err_put_ctlr:
 	return ret;
 }
 
-static int f_ospi_remove(struct platform_device *pdev)
+static void f_ospi_remove(struct platform_device *pdev)
 {
 	struct f_ospi *ospi = platform_get_drvdata(pdev);
 
 	clk_disable_unprepare(ospi->clk);
 
 	mutex_destroy(&ospi->mlock);
-
-	return 0;
 }
 
 static const struct of_device_id f_ospi_dt_ids[] = {
@@ -693,7 +691,7 @@ static struct platform_driver f_ospi_driver = {
 		.of_match_table = f_ospi_dt_ids,
 	},
 	.probe = f_ospi_probe,
-	.remove = f_ospi_remove,
+	.remove_new = f_ospi_remove,
 };
 module_platform_driver(f_ospi_driver);
 
diff --git a/drivers/spi/spi-sprd-adi.c b/drivers/spi/spi-sprd-adi.c
index 1edbf44c05a7..22e39c4c12c4 100644
--- a/drivers/spi/spi-sprd-adi.c
+++ b/drivers/spi/spi-sprd-adi.c
@@ -541,8 +541,7 @@ static int sprd_adi_probe(struct platform_device *pdev)
 	dev_set_drvdata(&pdev->dev, ctlr);
 	sadi = spi_controller_get_devdata(ctlr);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	sadi->base = devm_ioremap_resource(&pdev->dev, res);
+	sadi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(sadi->base)) {
 		ret = PTR_ERR(sadi->base);
 		goto put_ctlr;
@@ -608,13 +607,12 @@ put_ctlr:
 	return ret;
 }
 
-static int sprd_adi_remove(struct platform_device *pdev)
+static void sprd_adi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr = dev_get_drvdata(&pdev->dev);
 	struct sprd_adi *sadi = spi_controller_get_devdata(ctlr);
 
 	unregister_restart_handler(&sadi->restart_handler);
-	return 0;
 }
 
 static struct sprd_adi_data sc9860_data = {
@@ -660,7 +658,7 @@ static struct platform_driver sprd_adi_driver = {
 		.of_match_table = sprd_adi_of_match,
 	},
 	.probe = sprd_adi_probe,
-	.remove = sprd_adi_remove,
+	.remove_new = sprd_adi_remove,
 };
 module_platform_driver(sprd_adi_driver);
 
diff --git a/drivers/spi/spi-sprd.c b/drivers/spi/spi-sprd.c
index 65b8075da4eb..518c7eaca84e 100644
--- a/drivers/spi/spi-sprd.c
+++ b/drivers/spi/spi-sprd.c
@@ -929,8 +929,7 @@ static int sprd_spi_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ss = spi_controller_get_devdata(sctlr);
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	ss->base = devm_ioremap_resource(&pdev->dev, res);
+	ss->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(ss->base)) {
 		ret = PTR_ERR(ss->base);
 		goto free_controller;
@@ -1002,27 +1001,25 @@ free_controller:
 	return ret;
 }
 
-static int sprd_spi_remove(struct platform_device *pdev)
+static void sprd_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *sctlr = platform_get_drvdata(pdev);
 	struct sprd_spi *ss = spi_controller_get_devdata(sctlr);
 	int ret;
 
-	ret = pm_runtime_resume_and_get(ss->dev);
-	if (ret < 0) {
+	ret = pm_runtime_get_sync(ss->dev);
+	if (ret < 0)
 		dev_err(ss->dev, "failed to resume SPI controller\n");
-		return ret;
-	}
 
 	spi_controller_suspend(sctlr);
 
-	if (ss->dma.enable)
-		sprd_spi_dma_release(ss);
-	clk_disable_unprepare(ss->clk);
+	if (ret >= 0) {
+		if (ss->dma.enable)
+			sprd_spi_dma_release(ss);
+		clk_disable_unprepare(ss->clk);
+	}
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static int __maybe_unused sprd_spi_runtime_suspend(struct device *dev)
@@ -1076,7 +1073,7 @@ static struct platform_driver sprd_spi_driver = {
 		.pm = &sprd_spi_pm_ops,
 	},
 	.probe = sprd_spi_probe,
-	.remove  = sprd_spi_remove,
+	.remove_new = sprd_spi_remove,
 };
 
 module_platform_driver(sprd_spi_driver);
diff --git a/drivers/spi/spi-st-ssc4.c b/drivers/spi/spi-st-ssc4.c
index 843be803696b..7fcff9c539e2 100644
--- a/drivers/spi/spi-st-ssc4.c
+++ b/drivers/spi/spi-st-ssc4.c
@@ -183,7 +183,7 @@ static int spi_st_setup(struct spi_device *spi)
 		return -EINVAL;
 	}
 
-	if (!spi->cs_gpiod) {
+	if (!spi_get_csgpiod(spi, 0)) {
 		dev_err(&spi->dev, "no valid gpio assigned\n");
 		return -EINVAL;
 	}
@@ -366,7 +366,7 @@ put_master:
 	return ret;
 }
 
-static int spi_st_remove(struct platform_device *pdev)
+static void spi_st_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct spi_st *spi_st = spi_master_get_devdata(master);
@@ -376,8 +376,6 @@ static int spi_st_remove(struct platform_device *pdev)
 	clk_disable_unprepare(spi_st->clk);
 
 	pinctrl_pm_select_sleep_state(&pdev->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -451,7 +449,7 @@ static struct platform_driver spi_st_driver = {
 		.of_match_table = of_match_ptr(stm_spi_match),
 	},
 	.probe = spi_st_probe,
-	.remove = spi_st_remove,
+	.remove_new = spi_st_remove,
 };
 module_platform_driver(spi_st_driver);
 
diff --git a/drivers/spi/spi-stm32-qspi.c b/drivers/spi/spi-stm32-qspi.c
index 9131660c1afb..2b6804aa6901 100644
--- a/drivers/spi/spi-stm32-qspi.c
+++ b/drivers/spi/spi-stm32-qspi.c
@@ -359,7 +359,7 @@ static int stm32_qspi_get_mode(u8 buswidth)
 static int stm32_qspi_send(struct spi_device *spi, const struct spi_mem_op *op)
 {
 	struct stm32_qspi *qspi = spi_controller_get_devdata(spi->master);
-	struct stm32_qspi_flash *flash = &qspi->flash[spi->chip_select];
+	struct stm32_qspi_flash *flash = &qspi->flash[spi_get_chipselect(spi, 0)];
 	u32 ccr, cr;
 	int timeout, err = 0, err_poll_status = 0;
 
@@ -564,7 +564,7 @@ static int stm32_qspi_transfer_one_message(struct spi_controller *ctrl,
 	struct spi_mem_op op;
 	int ret = 0;
 
-	if (!spi->cs_gpiod)
+	if (!spi_get_csgpiod(spi, 0))
 		return -EOPNOTSUPP;
 
 	ret = pm_runtime_resume_and_get(qspi->dev);
@@ -573,7 +573,7 @@ static int stm32_qspi_transfer_one_message(struct spi_controller *ctrl,
 
 	mutex_lock(&qspi->lock);
 
-	gpiod_set_value_cansleep(spi->cs_gpiod, true);
+	gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), true);
 
 	list_for_each_entry(transfer, &msg->transfers, transfer_list) {
 		u8 dummy_bytes = 0;
@@ -626,7 +626,7 @@ static int stm32_qspi_transfer_one_message(struct spi_controller *ctrl,
 	}
 
 end_of_transfer:
-	gpiod_set_value_cansleep(spi->cs_gpiod, false);
+	gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), false);
 
 	mutex_unlock(&qspi->lock);
 
@@ -669,8 +669,8 @@ static int stm32_qspi_setup(struct spi_device *spi)
 
 	presc = DIV_ROUND_UP(qspi->clk_rate, spi->max_speed_hz) - 1;
 
-	flash = &qspi->flash[spi->chip_select];
-	flash->cs = spi->chip_select;
+	flash = &qspi->flash[spi_get_chipselect(spi, 0)];
+	flash->cs = spi_get_chipselect(spi, 0);
 	flash->presc = presc;
 
 	mutex_lock(&qspi->lock);
@@ -888,7 +888,7 @@ err_clk_disable:
 	return ret;
 }
 
-static int stm32_qspi_remove(struct platform_device *pdev)
+static void stm32_qspi_remove(struct platform_device *pdev)
 {
 	struct stm32_qspi *qspi = platform_get_drvdata(pdev);
 
@@ -903,8 +903,6 @@ static int stm32_qspi_remove(struct platform_device *pdev)
 	pm_runtime_set_suspended(qspi->dev);
 	pm_runtime_dont_use_autosuspend(qspi->dev);
 	clk_disable_unprepare(qspi->clk);
-
-	return 0;
 }
 
 static int __maybe_unused stm32_qspi_runtime_suspend(struct device *dev)
@@ -968,7 +966,7 @@ MODULE_DEVICE_TABLE(of, stm32_qspi_match);
 
 static struct platform_driver stm32_qspi_driver = {
 	.probe	= stm32_qspi_probe,
-	.remove	= stm32_qspi_remove,
+	.remove_new = stm32_qspi_remove,
 	.driver	= {
 		.name = "stm32-qspi",
 		.of_match_table = stm32_qspi_match,
diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c
index def09cf0dc14..d6598e4116bd 100644
--- a/drivers/spi/spi-stm32.c
+++ b/drivers/spi/spi-stm32.c
@@ -984,9 +984,9 @@ static int stm32_spi_prepare_msg(struct spi_master *master,
 	if (spi->cfg->set_number_of_data) {
 		int ret;
 
-		ret = spi_split_transfers_maxsize(master, msg,
-						  STM32H7_SPI_TSIZE_MAX,
-						  GFP_KERNEL | GFP_DMA);
+		ret = spi_split_transfers_maxwords(master, msg,
+						   STM32H7_SPI_TSIZE_MAX,
+						   GFP_KERNEL | GFP_DMA);
 		if (ret)
 			return ret;
 	}
@@ -1780,8 +1780,7 @@ static int stm32_spi_probe(struct platform_device *pdev)
 		of_match_device(pdev->dev.driver->of_match_table,
 				&pdev->dev)->data;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	spi->base = devm_ioremap_resource(&pdev->dev, res);
+	spi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(spi->base))
 		return PTR_ERR(spi->base);
 
@@ -1922,7 +1921,7 @@ err_clk_disable:
 	return ret;
 }
 
-static int stm32_spi_remove(struct platform_device *pdev)
+static void stm32_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct stm32_spi *spi = spi_master_get_devdata(master);
@@ -1946,8 +1945,6 @@ static int stm32_spi_remove(struct platform_device *pdev)
 
 
 	pinctrl_pm_select_sleep_state(&pdev->dev);
-
-	return 0;
 }
 
 static int __maybe_unused stm32_spi_runtime_suspend(struct device *dev)
@@ -2023,7 +2020,7 @@ static const struct dev_pm_ops stm32_spi_pm_ops = {
 
 static struct platform_driver stm32_spi_driver = {
 	.probe = stm32_spi_probe,
-	.remove = stm32_spi_remove,
+	.remove_new = stm32_spi_remove,
 	.driver = {
 		.name = DRIVER_NAME,
 		.pm = &stm32_spi_pm_ops,
diff --git a/drivers/spi/spi-sun4i.c b/drivers/spi/spi-sun4i.c
index 6000d0761206..b8947265d329 100644
--- a/drivers/spi/spi-sun4i.c
+++ b/drivers/spi/spi-sun4i.c
@@ -167,7 +167,7 @@ static void sun4i_spi_set_cs(struct spi_device *spi, bool enable)
 	reg = sun4i_spi_read(sspi, SUN4I_CTL_REG);
 
 	reg &= ~SUN4I_CTL_CS_MASK;
-	reg |= SUN4I_CTL_CS(spi->chip_select);
+	reg |= SUN4I_CTL_CS(spi_get_chipselect(spi, 0));
 
 	/* We want to control the chip select manually */
 	reg |= SUN4I_CTL_CS_MANUAL;
@@ -516,11 +516,9 @@ err_free_master:
 	return ret;
 }
 
-static int sun4i_spi_remove(struct platform_device *pdev)
+static void sun4i_spi_remove(struct platform_device *pdev)
 {
 	pm_runtime_force_suspend(&pdev->dev);
-
-	return 0;
 }
 
 static const struct of_device_id sun4i_spi_match[] = {
@@ -536,7 +534,7 @@ static const struct dev_pm_ops sun4i_spi_pm_ops = {
 
 static struct platform_driver sun4i_spi_driver = {
 	.probe	= sun4i_spi_probe,
-	.remove	= sun4i_spi_remove,
+	.remove_new = sun4i_spi_remove,
 	.driver	= {
 		.name		= "sun4i-spi",
 		.of_match_table	= sun4i_spi_match,
diff --git a/drivers/spi/spi-sun6i.c b/drivers/spi/spi-sun6i.c
index 23ad052528db..7532c85a352c 100644
--- a/drivers/spi/spi-sun6i.c
+++ b/drivers/spi/spi-sun6i.c
@@ -174,7 +174,7 @@ static void sun6i_spi_set_cs(struct spi_device *spi, bool enable)
 
 	reg = sun6i_spi_read(sspi, SUN6I_TFR_CTL_REG);
 	reg &= ~SUN6I_TFR_CTL_CS_MASK;
-	reg |= SUN6I_TFR_CTL_CS(spi->chip_select);
+	reg |= SUN6I_TFR_CTL_CS(spi_get_chipselect(spi, 0));
 
 	if (enable)
 		reg |= SUN6I_TFR_CTL_CS_LEVEL;
@@ -683,7 +683,7 @@ err_free_master:
 	return ret;
 }
 
-static int sun6i_spi_remove(struct platform_device *pdev)
+static void sun6i_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 
@@ -693,7 +693,6 @@ static int sun6i_spi_remove(struct platform_device *pdev)
 		dma_release_channel(master->dma_tx);
 	if (master->dma_rx)
 		dma_release_channel(master->dma_rx);
-	return 0;
 }
 
 static const struct of_device_id sun6i_spi_match[] = {
@@ -710,7 +709,7 @@ static const struct dev_pm_ops sun6i_spi_pm_ops = {
 
 static struct platform_driver sun6i_spi_driver = {
 	.probe	= sun6i_spi_probe,
-	.remove	= sun6i_spi_remove,
+	.remove_new = sun6i_spi_remove,
 	.driver	= {
 		.name		= "sun6i-spi",
 		.of_match_table	= sun6i_spi_match,
diff --git a/drivers/spi/spi-sunplus-sp7021.c b/drivers/spi/spi-sunplus-sp7021.c
index f1fa88777575..eb8f835a4771 100644
--- a/drivers/spi/spi-sunplus-sp7021.c
+++ b/drivers/spi/spi-sunplus-sp7021.c
@@ -504,14 +504,13 @@ static int sp7021_spi_controller_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int sp7021_spi_controller_remove(struct platform_device *pdev)
+static void sp7021_spi_controller_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr = dev_get_drvdata(&pdev->dev);
 
 	spi_unregister_controller(ctlr);
 	pm_runtime_disable(&pdev->dev);
 	pm_runtime_set_suspended(&pdev->dev);
-	return 0;
 }
 
 static int __maybe_unused sp7021_spi_controller_suspend(struct device *dev)
@@ -564,7 +563,7 @@ MODULE_DEVICE_TABLE(of, sp7021_spi_controller_ids);
 
 static struct platform_driver sp7021_spi_controller_driver = {
 	.probe = sp7021_spi_controller_probe,
-	.remove = sp7021_spi_controller_remove,
+	.remove_new = sp7021_spi_controller_remove,
 	.driver = {
 		.name = "sunplus,sp7021-spi-controller",
 		.of_match_table = sp7021_spi_controller_ids,
diff --git a/drivers/spi/spi-synquacer.c b/drivers/spi/spi-synquacer.c
index dc188f9202c9..aeaf7db022f0 100644
--- a/drivers/spi/spi-synquacer.c
+++ b/drivers/spi/spi-synquacer.c
@@ -250,7 +250,7 @@ static int synquacer_spi_config(struct spi_master *master,
 	}
 
 	mode = spi->mode;
-	cs = spi->chip_select;
+	cs = spi_get_chipselect(spi, 0);
 	speed = xfer->speed_hz;
 	bpw = xfer->bits_per_word;
 
@@ -344,7 +344,7 @@ static int synquacer_spi_config(struct spi_master *master,
 	sspi->bpw = bpw;
 	sspi->mode = mode;
 	sspi->speed = speed;
-	sspi->cs = spi->chip_select;
+	sspi->cs = spi_get_chipselect(spi, 0);
 	sspi->bus_width = bus_width;
 
 	return 0;
@@ -488,7 +488,7 @@ static void synquacer_spi_set_cs(struct spi_device *spi, bool enable)
 	val = readl(sspi->regs + SYNQUACER_HSSPI_REG_DMSTART);
 	val &= ~(SYNQUACER_HSSPI_DMPSEL_CS_MASK <<
 		 SYNQUACER_HSSPI_DMPSEL_CS_SHIFT);
-	val |= spi->chip_select << SYNQUACER_HSSPI_DMPSEL_CS_SHIFT;
+	val |= spi_get_chipselect(spi, 0) << SYNQUACER_HSSPI_DMPSEL_CS_SHIFT;
 
 	if (!enable)
 		val |= SYNQUACER_HSSPI_DMSTOP_STOP;
@@ -735,7 +735,7 @@ put_spi:
 	return ret;
 }
 
-static int synquacer_spi_remove(struct platform_device *pdev)
+static void synquacer_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct synquacer_spi *sspi = spi_master_get_devdata(master);
@@ -743,8 +743,6 @@ static int synquacer_spi_remove(struct platform_device *pdev)
 	pm_runtime_disable(sspi->dev);
 
 	clk_disable_unprepare(sspi->clk);
-
-	return 0;
 }
 
 static int __maybe_unused synquacer_spi_suspend(struct device *dev)
@@ -820,7 +818,7 @@ static struct platform_driver synquacer_spi_driver = {
 		.acpi_match_table = ACPI_PTR(synquacer_hsspi_acpi_ids),
 	},
 	.probe = synquacer_spi_probe,
-	.remove = synquacer_spi_remove,
+	.remove_new = synquacer_spi_remove,
 };
 module_platform_driver(synquacer_spi_driver);
 
diff --git a/drivers/spi/spi-tegra114.c b/drivers/spi/spi-tegra114.c
index d9be80e3e1bc..488df681eaef 100644
--- a/drivers/spi/spi-tegra114.c
+++ b/drivers/spi/spi-tegra114.c
@@ -747,7 +747,7 @@ static int tegra_spi_set_hw_cs_timing(struct spi_device *spi)
 	if (setup_dly && hold_dly) {
 		setup_hold = SPI_SETUP_HOLD(setup_dly - 1, hold_dly - 1);
 		spi_cs_timing = SPI_CS_SETUP_HOLD(tspi->spi_cs_timing1,
-						  spi->chip_select,
+						  spi_get_chipselect(spi, 0),
 						  setup_hold);
 		if (tspi->spi_cs_timing1 != spi_cs_timing) {
 			tspi->spi_cs_timing1 = spi_cs_timing;
@@ -760,9 +760,9 @@ static int tegra_spi_set_hw_cs_timing(struct spi_device *spi)
 		inactive_cycles--;
 	cs_state = inactive_cycles ? 0 : 1;
 	spi_cs_timing = tspi->spi_cs_timing2;
-	SPI_SET_CS_ACTIVE_BETWEEN_PACKETS(spi_cs_timing, spi->chip_select,
+	SPI_SET_CS_ACTIVE_BETWEEN_PACKETS(spi_cs_timing, spi_get_chipselect(spi, 0),
 					  cs_state);
-	SPI_SET_CYCLES_BETWEEN_PACKETS(spi_cs_timing, spi->chip_select,
+	SPI_SET_CYCLES_BETWEEN_PACKETS(spi_cs_timing, spi_get_chipselect(spi, 0),
 				       inactive_cycles);
 	if (tspi->spi_cs_timing2 != spi_cs_timing) {
 		tspi->spi_cs_timing2 = spi_cs_timing;
@@ -831,8 +831,8 @@ static u32 tegra_spi_setup_transfer_one(struct spi_device *spi,
 			tegra_spi_writel(tspi, command1, SPI_COMMAND1);
 
 		/* GPIO based chip select control */
-		if (spi->cs_gpiod)
-			gpiod_set_value(spi->cs_gpiod, 1);
+		if (spi_get_csgpiod(spi, 0))
+			gpiod_set_value(spi_get_csgpiod(spi, 0), 1);
 
 		if (is_single_xfer && !(t->cs_change)) {
 			tspi->use_hw_based_cs = true;
@@ -846,7 +846,7 @@ static u32 tegra_spi_setup_transfer_one(struct spi_device *spi,
 				command1 &= ~SPI_CS_SW_VAL;
 		}
 
-		if (tspi->last_used_cs != spi->chip_select) {
+		if (tspi->last_used_cs != spi_get_chipselect(spi, 0)) {
 			if (cdata && cdata->tx_clk_tap_delay)
 				tx_tap = cdata->tx_clk_tap_delay;
 			if (cdata && cdata->rx_clk_tap_delay)
@@ -855,7 +855,7 @@ static u32 tegra_spi_setup_transfer_one(struct spi_device *spi,
 				   SPI_RX_TAP_DELAY(rx_tap);
 			if (command2 != tspi->def_command2_reg)
 				tegra_spi_writel(tspi, command2, SPI_COMMAND2);
-			tspi->last_used_cs = spi->chip_select;
+			tspi->last_used_cs = spi_get_chipselect(spi, 0);
 		}
 
 	} else {
@@ -896,7 +896,7 @@ static int tegra_spi_start_transfer_one(struct spi_device *spi,
 		command1 |= SPI_TX_EN;
 		tspi->cur_direction |= DATA_DIR_TX;
 	}
-	command1 |= SPI_CS_SEL(spi->chip_select);
+	command1 |= SPI_CS_SEL(spi_get_chipselect(spi, 0));
 	tegra_spi_writel(tspi, command1, SPI_COMMAND1);
 	tspi->command1_reg = command1;
 
@@ -980,14 +980,14 @@ static int tegra_spi_setup(struct spi_device *spi)
 
 	spin_lock_irqsave(&tspi->lock, flags);
 	/* GPIO based chip select control */
-	if (spi->cs_gpiod)
-		gpiod_set_value(spi->cs_gpiod, 0);
+	if (spi_get_csgpiod(spi, 0))
+		gpiod_set_value(spi_get_csgpiod(spi, 0), 0);
 
 	val = tspi->def_command1_reg;
 	if (spi->mode & SPI_CS_HIGH)
-		val &= ~SPI_CS_POL_INACTIVE(spi->chip_select);
+		val &= ~SPI_CS_POL_INACTIVE(spi_get_chipselect(spi, 0));
 	else
-		val |= SPI_CS_POL_INACTIVE(spi->chip_select);
+		val |= SPI_CS_POL_INACTIVE(spi_get_chipselect(spi, 0));
 	tspi->def_command1_reg = val;
 	tegra_spi_writel(tspi, tspi->def_command1_reg, SPI_COMMAND1);
 	spin_unlock_irqrestore(&tspi->lock, flags);
@@ -1002,8 +1002,8 @@ static void tegra_spi_transfer_end(struct spi_device *spi)
 	int cs_val = (spi->mode & SPI_CS_HIGH) ? 0 : 1;
 
 	/* GPIO based chip select control */
-	if (spi->cs_gpiod)
-		gpiod_set_value(spi->cs_gpiod, 0);
+	if (spi_get_csgpiod(spi, 0))
+		gpiod_set_value(spi_get_csgpiod(spi, 0), 0);
 
 	if (!tspi->use_hw_based_cs) {
 		if (cs_val)
@@ -1342,8 +1342,7 @@ static int tegra_spi_probe(struct platform_device *pdev)
 		goto exit_free_master;
 	}
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	tspi->base = devm_ioremap_resource(&pdev->dev, r);
+	tspi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
 	if (IS_ERR(tspi->base)) {
 		ret = PTR_ERR(tspi->base);
 		goto exit_free_master;
@@ -1440,7 +1439,7 @@ exit_free_master:
 	return ret;
 }
 
-static int tegra_spi_remove(struct platform_device *pdev)
+static void tegra_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct tegra_spi_data	*tspi = spi_master_get_devdata(master);
@@ -1456,8 +1455,6 @@ static int tegra_spi_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 	if (!pm_runtime_status_suspended(&pdev->dev))
 		tegra_spi_runtime_suspend(&pdev->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1526,7 +1523,7 @@ static struct platform_driver tegra_spi_driver = {
 		.of_match_table	= tegra_spi_of_match,
 	},
 	.probe =	tegra_spi_probe,
-	.remove =	tegra_spi_remove,
+	.remove_new =	tegra_spi_remove,
 };
 module_platform_driver(tegra_spi_driver);
 
diff --git a/drivers/spi/spi-tegra20-sflash.c b/drivers/spi/spi-tegra20-sflash.c
index 220ee08c4a06..4286310628a2 100644
--- a/drivers/spi/spi-tegra20-sflash.c
+++ b/drivers/spi/spi-tegra20-sflash.c
@@ -280,7 +280,7 @@ static int tegra_sflash_start_transfer_one(struct spi_device *spi,
 			command |= SPI_ACTIVE_SCLK_DRIVE_HIGH;
 		else
 			command |= SPI_ACTIVE_SCLK_DRIVE_LOW;
-		command |= SPI_CS0_EN << spi->chip_select;
+		command |= SPI_CS0_EN << spi_get_chipselect(spi, 0);
 	} else {
 		command = tsd->command_reg;
 		command &= ~SPI_BIT_LENGTH(~0);
@@ -520,7 +520,7 @@ exit_free_master:
 	return ret;
 }
 
-static int tegra_sflash_remove(struct platform_device *pdev)
+static void tegra_sflash_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct tegra_sflash_data	*tsd = spi_master_get_devdata(master);
@@ -530,8 +530,6 @@ static int tegra_sflash_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 	if (!pm_runtime_status_suspended(&pdev->dev))
 		tegra_sflash_runtime_suspend(&pdev->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -598,7 +596,7 @@ static struct platform_driver tegra_sflash_driver = {
 		.of_match_table	= tegra_sflash_of_match,
 	},
 	.probe =	tegra_sflash_probe,
-	.remove =	tegra_sflash_remove,
+	.remove_new =	tegra_sflash_remove,
 };
 module_platform_driver(tegra_sflash_driver);
 
diff --git a/drivers/spi/spi-tegra20-slink.c b/drivers/spi/spi-tegra20-slink.c
index 148043d0c2b8..c2915f7672cc 100644
--- a/drivers/spi/spi-tegra20-slink.c
+++ b/drivers/spi/spi-tegra20-slink.c
@@ -758,9 +758,9 @@ static int tegra_slink_setup(struct spi_device *spi)
 	spin_lock_irqsave(&tspi->lock, flags);
 	val = tspi->def_command_reg;
 	if (spi->mode & SPI_CS_HIGH)
-		val |= cs_pol_bit[spi->chip_select];
+		val |= cs_pol_bit[spi_get_chipselect(spi, 0)];
 	else
-		val &= ~cs_pol_bit[spi->chip_select];
+		val &= ~cs_pol_bit[spi_get_chipselect(spi, 0)];
 	tspi->def_command_reg = val;
 	tegra_slink_writel(tspi, tspi->def_command_reg, SLINK_COMMAND);
 	spin_unlock_irqrestore(&tspi->lock, flags);
@@ -781,7 +781,7 @@ static int tegra_slink_prepare_message(struct spi_master *master,
 	tspi->command_reg |= SLINK_CS_SW | SLINK_CS_VALUE;
 
 	tspi->command2_reg = tspi->def_command2_reg;
-	tspi->command2_reg |= SLINK_SS_EN_CS(spi->chip_select);
+	tspi->command2_reg |= SLINK_SS_EN_CS(spi_get_chipselect(spi, 0));
 
 	tspi->command_reg &= ~SLINK_MODES;
 	if (spi->mode & SPI_CPHA)
@@ -1134,7 +1134,7 @@ exit_free_master:
 	return ret;
 }
 
-static int tegra_slink_remove(struct platform_device *pdev)
+static void tegra_slink_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = spi_master_get(platform_get_drvdata(pdev));
 	struct tegra_slink_data	*tspi = spi_master_get_devdata(master);
@@ -1152,7 +1152,6 @@ static int tegra_slink_remove(struct platform_device *pdev)
 		tegra_slink_deinit_dma_param(tspi, true);
 
 	spi_master_put(master);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1220,7 +1219,7 @@ static struct platform_driver tegra_slink_driver = {
 		.of_match_table	= tegra_slink_of_match,
 	},
 	.probe =	tegra_slink_probe,
-	.remove =	tegra_slink_remove,
+	.remove_new =	tegra_slink_remove,
 };
 module_platform_driver(tegra_slink_driver);
 
diff --git a/drivers/spi/spi-tegra210-quad.c b/drivers/spi/spi-tegra210-quad.c
index 0b9bc3b7f53a..fbd14dd7be44 100644
--- a/drivers/spi/spi-tegra210-quad.c
+++ b/drivers/spi/spi-tegra210-quad.c
@@ -142,6 +142,7 @@
 
 #define QSPI_GLOBAL_CONFIG			0X1a4
 #define QSPI_CMB_SEQ_EN				BIT(0)
+#define QSPI_TPM_WAIT_POLL_EN			BIT(1)
 
 #define QSPI_CMB_SEQ_ADDR			0x1a8
 #define QSPI_ADDRESS_VALUE_SET(X)		(((x) & 0xFFFF) << 0)
@@ -164,6 +165,7 @@
 struct tegra_qspi_soc_data {
 	bool has_dma;
 	bool cmb_xfer_capable;
+	bool supports_tpm;
 	unsigned int cs_count;
 };
 
@@ -829,7 +831,7 @@ static u32 tegra_qspi_setup_transfer_one(struct spi_device *spi, struct spi_tran
 		tegra_qspi_mask_clear_irq(tqspi);
 
 		command1 = tqspi->def_command1_reg;
-		command1 |= QSPI_CS_SEL(spi->chip_select);
+		command1 |= QSPI_CS_SEL(spi_get_chipselect(spi, 0));
 		command1 |= QSPI_BIT_LENGTH(bits_per_word - 1);
 
 		command1 &= ~QSPI_CONTROL_MODE_MASK;
@@ -960,11 +962,11 @@ static int tegra_qspi_setup(struct spi_device *spi)
 
 	/* keep default cs state to inactive */
 	val = tqspi->def_command1_reg;
-	val |= QSPI_CS_SEL(spi->chip_select);
+	val |= QSPI_CS_SEL(spi_get_chipselect(spi, 0));
 	if (spi->mode & SPI_CS_HIGH)
-		val &= ~QSPI_CS_POL_INACTIVE(spi->chip_select);
+		val &= ~QSPI_CS_POL_INACTIVE(spi_get_chipselect(spi, 0));
 	else
-		val |= QSPI_CS_POL_INACTIVE(spi->chip_select);
+		val |= QSPI_CS_POL_INACTIVE(spi_get_chipselect(spi, 0));
 
 	tqspi->def_command1_reg = val;
 	tegra_qspi_writel(tqspi, tqspi->def_command1_reg, QSPI_COMMAND1);
@@ -1065,6 +1067,12 @@ static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi,
 
 	/* Enable Combined sequence mode */
 	val = tegra_qspi_readl(tqspi, QSPI_GLOBAL_CONFIG);
+	if (spi->mode & SPI_TPM_HW_FLOW) {
+		if (tqspi->soc_data->supports_tpm)
+			val |= QSPI_TPM_WAIT_POLL_EN;
+		else
+			return -EIO;
+	}
 	val |= QSPI_CMB_SEQ_EN;
 	tegra_qspi_writel(tqspi, val, QSPI_GLOBAL_CONFIG);
 	/* Process individual transfer list */
@@ -1196,6 +1204,8 @@ static int tegra_qspi_non_combined_seq_xfer(struct tegra_qspi *tqspi,
 	/* Disable Combined sequence mode */
 	val = tegra_qspi_readl(tqspi, QSPI_GLOBAL_CONFIG);
 	val &= ~QSPI_CMB_SEQ_EN;
+	if (tqspi->soc_data->supports_tpm)
+		val &= ~QSPI_TPM_WAIT_POLL_EN;
 	tegra_qspi_writel(tqspi, val, QSPI_GLOBAL_CONFIG);
 	list_for_each_entry(transfer, &msg->transfers, transfer_list) {
 		struct spi_transfer *xfer = transfer;
@@ -1454,24 +1464,28 @@ static irqreturn_t tegra_qspi_isr_thread(int irq, void *context_data)
 static struct tegra_qspi_soc_data tegra210_qspi_soc_data = {
 	.has_dma = true,
 	.cmb_xfer_capable = false,
+	.supports_tpm = false,
 	.cs_count = 1,
 };
 
 static struct tegra_qspi_soc_data tegra186_qspi_soc_data = {
 	.has_dma = true,
 	.cmb_xfer_capable = true,
+	.supports_tpm = false,
 	.cs_count = 1,
 };
 
 static struct tegra_qspi_soc_data tegra234_qspi_soc_data = {
 	.has_dma = false,
 	.cmb_xfer_capable = true,
+	.supports_tpm = true,
 	.cs_count = 1,
 };
 
 static struct tegra_qspi_soc_data tegra241_qspi_soc_data = {
 	.has_dma = false,
 	.cmb_xfer_capable = true,
+	.supports_tpm = true,
 	.cs_count = 4,
 };
 
@@ -1552,8 +1566,7 @@ static int tegra_qspi_probe(struct platform_device *pdev)
 
 	tqspi->soc_data = device_get_match_data(&pdev->dev);
 	master->num_chipselect = tqspi->soc_data->cs_count;
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	tqspi->base = devm_ioremap_resource(&pdev->dev, r);
+	tqspi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
 	if (IS_ERR(tqspi->base))
 		return PTR_ERR(tqspi->base);
 
@@ -1630,7 +1643,7 @@ exit_pm_disable:
 	return ret;
 }
 
-static int tegra_qspi_remove(struct platform_device *pdev)
+static void tegra_qspi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct tegra_qspi *tqspi = spi_master_get_devdata(master);
@@ -1639,8 +1652,6 @@ static int tegra_qspi_remove(struct platform_device *pdev)
 	free_irq(tqspi->irq, tqspi);
 	pm_runtime_force_suspend(&pdev->dev);
 	tegra_qspi_deinit_dma(tqspi);
-
-	return 0;
 }
 
 static int __maybe_unused tegra_qspi_suspend(struct device *dev)
@@ -1714,7 +1725,7 @@ static struct platform_driver tegra_qspi_driver = {
 		.acpi_match_table = ACPI_PTR(tegra_qspi_acpi_match),
 	},
 	.probe =	tegra_qspi_probe,
-	.remove =	tegra_qspi_remove,
+	.remove_new =	tegra_qspi_remove,
 };
 module_platform_driver(tegra_qspi_driver);
 
diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c
index 60086869bcae..5914335ff63d 100644
--- a/drivers/spi/spi-ti-qspi.c
+++ b/drivers/spi/spi-ti-qspi.c
@@ -533,10 +533,10 @@ static void ti_qspi_enable_memory_map(struct spi_device *spi)
 	if (qspi->ctrl_base) {
 		regmap_update_bits(qspi->ctrl_base, qspi->ctrl_reg,
 				   MEM_CS_MASK,
-				   MEM_CS_EN(spi->chip_select));
+				   MEM_CS_EN(spi_get_chipselect(spi, 0)));
 	}
 	qspi->mmap_enabled = true;
-	qspi->current_cs = spi->chip_select;
+	qspi->current_cs = spi_get_chipselect(spi, 0);
 }
 
 static void ti_qspi_disable_memory_map(struct spi_device *spi)
@@ -572,7 +572,7 @@ static void ti_qspi_setup_mmap_read(struct spi_device *spi, u8 opcode,
 	memval |= ((addr_width - 1) << QSPI_SETUP_ADDR_SHIFT |
 		   dummy_bytes << QSPI_SETUP_DUMMY_SHIFT);
 	ti_qspi_write(qspi, memval,
-		      QSPI_SPI_SETUP_REG(spi->chip_select));
+		      QSPI_SPI_SETUP_REG(spi_get_chipselect(spi, 0)));
 }
 
 static int ti_qspi_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
@@ -623,7 +623,7 @@ static int ti_qspi_exec_mem_op(struct spi_mem *mem,
 
 	mutex_lock(&qspi->list_lock);
 
-	if (!qspi->mmap_enabled || qspi->current_cs != mem->spi->chip_select) {
+	if (!qspi->mmap_enabled || qspi->current_cs != spi_get_chipselect(mem->spi, 0)) {
 		ti_qspi_setup_clk(qspi, mem->spi->max_speed_hz);
 		ti_qspi_enable_memory_map(mem->spi);
 	}
@@ -673,11 +673,11 @@ static int ti_qspi_start_transfer_one(struct spi_master *master,
 	qspi->dc = 0;
 
 	if (spi->mode & SPI_CPHA)
-		qspi->dc |= QSPI_CKPHA(spi->chip_select);
+		qspi->dc |= QSPI_CKPHA(spi_get_chipselect(spi, 0));
 	if (spi->mode & SPI_CPOL)
-		qspi->dc |= QSPI_CKPOL(spi->chip_select);
+		qspi->dc |= QSPI_CKPOL(spi_get_chipselect(spi, 0));
 	if (spi->mode & SPI_CS_HIGH)
-		qspi->dc |= QSPI_CSPOL(spi->chip_select);
+		qspi->dc |= QSPI_CSPOL(spi_get_chipselect(spi, 0));
 
 	frame_len_words = 0;
 	list_for_each_entry(t, &m->transfers, transfer_list)
@@ -686,7 +686,7 @@ static int ti_qspi_start_transfer_one(struct spi_master *master,
 
 	/* setup command reg */
 	qspi->cmd = 0;
-	qspi->cmd |= QSPI_EN_CS(spi->chip_select);
+	qspi->cmd |= QSPI_EN_CS(spi_get_chipselect(spi, 0));
 	qspi->cmd |= QSPI_FLEN(frame_len_words);
 
 	ti_qspi_write(qspi, qspi->dc, QSPI_SPI_DC_REG);
diff --git a/drivers/spi/spi-topcliff-pch.c b/drivers/spi/spi-topcliff-pch.c
index cbb60198a7f0..af5846cfe5e9 100644
--- a/drivers/spi/spi-topcliff-pch.c
+++ b/drivers/spi/spi-topcliff-pch.c
@@ -499,7 +499,7 @@ static inline void pch_spi_select_chip(struct pch_spi_data *data,
 				       struct spi_device *pspi)
 {
 	if (data->current_chip != NULL) {
-		if (pspi->chip_select != data->n_curnt_chip) {
+		if (spi_get_chipselect(pspi, 0) != data->n_curnt_chip) {
 			dev_dbg(&pspi->dev, "%s : different slave\n", __func__);
 			data->current_chip = NULL;
 		}
@@ -507,7 +507,7 @@ static inline void pch_spi_select_chip(struct pch_spi_data *data,
 
 	data->current_chip = pspi;
 
-	data->n_curnt_chip = data->current_chip->chip_select;
+	data->n_curnt_chip = spi_get_chipselect(data->current_chip, 0);
 
 	dev_dbg(&pspi->dev, "%s :Invoking pch_spi_setup_transfer\n", __func__);
 	pch_spi_setup_transfer(pspi);
@@ -1396,7 +1396,7 @@ err_pci_iomap:
 	return ret;
 }
 
-static int pch_spi_pd_remove(struct platform_device *plat_dev)
+static void pch_spi_pd_remove(struct platform_device *plat_dev)
 {
 	struct pch_spi_board_data *board_dat = dev_get_platdata(&plat_dev->dev);
 	struct pch_spi_data *data = platform_get_drvdata(plat_dev);
@@ -1434,8 +1434,6 @@ static int pch_spi_pd_remove(struct platform_device *plat_dev)
 
 	pci_iounmap(board_dat->pdev, data->io_remap_addr);
 	spi_unregister_master(data->master);
-
-	return 0;
 }
 #ifdef CONFIG_PM
 static int pch_spi_pd_suspend(struct platform_device *pd_dev,
@@ -1516,7 +1514,7 @@ static struct platform_driver pch_spi_pd_driver = {
 		.name = "pch-spi",
 	},
 	.probe = pch_spi_pd_probe,
-	.remove = pch_spi_pd_remove,
+	.remove_new = pch_spi_pd_remove,
 	.suspend = pch_spi_pd_suspend,
 	.resume = pch_spi_pd_resume
 };
diff --git a/drivers/spi/spi-uniphier.c b/drivers/spi/spi-uniphier.c
index cc0da4822231..f5344527af0b 100644
--- a/drivers/spi/spi-uniphier.c
+++ b/drivers/spi/spi-uniphier.c
@@ -775,7 +775,7 @@ out_master_put:
 	return ret;
 }
 
-static int uniphier_spi_remove(struct platform_device *pdev)
+static void uniphier_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct uniphier_spi_priv *priv = spi_master_get_devdata(master);
@@ -786,8 +786,6 @@ static int uniphier_spi_remove(struct platform_device *pdev)
 		dma_release_channel(master->dma_rx);
 
 	clk_disable_unprepare(priv->clk);
-
-	return 0;
 }
 
 static const struct of_device_id uniphier_spi_match[] = {
@@ -798,7 +796,7 @@ MODULE_DEVICE_TABLE(of, uniphier_spi_match);
 
 static struct platform_driver uniphier_spi_driver = {
 	.probe = uniphier_spi_probe,
-	.remove = uniphier_spi_remove,
+	.remove_new = uniphier_spi_remove,
 	.driver = {
 		.name = "uniphier-spi",
 		.of_match_table = uniphier_spi_match,
diff --git a/drivers/spi/spi-wpcm-fiu.c b/drivers/spi/spi-wpcm-fiu.c
index ab33710d50ac..f15312fdcdaf 100644
--- a/drivers/spi/spi-wpcm-fiu.c
+++ b/drivers/spi/spi-wpcm-fiu.c
@@ -158,7 +158,7 @@ static int wpcm_fiu_normal_exec(struct spi_mem *mem, const struct spi_mem_op *op
 	if (op->data.dir == SPI_MEM_DATA_OUT)
 		wpcm_fiu_set_data(fiu, op->data.buf.out, op->data.nbytes);
 
-	ret = wpcm_fiu_do_uma(fiu, mem->spi->chip_select, op->addr.nbytes == 3,
+	ret = wpcm_fiu_do_uma(fiu, spi_get_chipselect(mem->spi, 0), op->addr.nbytes == 3,
 			      op->data.dir == SPI_MEM_DATA_OUT, op->data.nbytes);
 
 	if (op->data.dir == SPI_MEM_DATA_IN)
@@ -196,7 +196,7 @@ static bool wpcm_fiu_4ba_match(const struct spi_mem_op *op)
 static int wpcm_fiu_4ba_exec(struct spi_mem *mem, const struct spi_mem_op *op)
 {
 	struct wpcm_fiu_spi *fiu = spi_controller_get_devdata(mem->spi->controller);
-	int cs = mem->spi->chip_select;
+	int cs = spi_get_chipselect(mem->spi, 0);
 
 	wpcm_fiu_ects_assert(fiu, cs);
 
@@ -241,7 +241,7 @@ static bool wpcm_fiu_rdid_match(const struct spi_mem_op *op)
 static int wpcm_fiu_rdid_exec(struct spi_mem *mem, const struct spi_mem_op *op)
 {
 	struct wpcm_fiu_spi *fiu = spi_controller_get_devdata(mem->spi->controller);
-	int cs = mem->spi->chip_select;
+	int cs = spi_get_chipselect(mem->spi, 0);
 
 	/* First transfer */
 	wpcm_fiu_set_opcode(fiu, op->cmd.opcode);
@@ -278,7 +278,7 @@ static bool wpcm_fiu_dummy_match(const struct spi_mem_op *op)
 static int wpcm_fiu_dummy_exec(struct spi_mem *mem, const struct spi_mem_op *op)
 {
 	struct wpcm_fiu_spi *fiu = spi_controller_get_devdata(mem->spi->controller);
-	int cs = mem->spi->chip_select;
+	int cs = spi_get_chipselect(mem->spi, 0);
 
 	wpcm_fiu_ects_assert(fiu, cs);
 
@@ -376,7 +376,7 @@ static int wpcm_fiu_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
 static int wpcm_fiu_dirmap_create(struct spi_mem_dirmap_desc *desc)
 {
 	struct wpcm_fiu_spi *fiu = spi_controller_get_devdata(desc->mem->spi->controller);
-	int cs = desc->mem->spi->chip_select;
+	int cs = spi_get_chipselect(desc->mem->spi, 0);
 
 	if (desc->info.op_tmpl.data.dir != SPI_MEM_DATA_IN)
 		return -ENOTSUPP;
@@ -400,7 +400,7 @@ static int wpcm_fiu_dirmap_create(struct spi_mem_dirmap_desc *desc)
 static ssize_t wpcm_fiu_direct_read(struct spi_mem_dirmap_desc *desc, u64 offs, size_t len, void *buf)
 {
 	struct wpcm_fiu_spi *fiu = spi_controller_get_devdata(desc->mem->spi->controller);
-	int cs = desc->mem->spi->chip_select;
+	int cs = spi_get_chipselect(desc->mem->spi, 0);
 
 	if (offs >= MAX_MEMORY_SIZE_PER_CS)
 		return -ENOTSUPP;
diff --git a/drivers/spi/spi-xcomm.c b/drivers/spi/spi-xcomm.c
index 8628241ec99e..5d23411f2a3e 100644
--- a/drivers/spi/spi-xcomm.c
+++ b/drivers/spi/spi-xcomm.c
@@ -58,7 +58,7 @@ static int spi_xcomm_sync_config(struct spi_xcomm *spi_xcomm, unsigned int len)
 static void spi_xcomm_chipselect(struct spi_xcomm *spi_xcomm,
 	struct spi_device *spi, int is_active)
 {
-	unsigned long cs = spi->chip_select;
+	unsigned long cs = spi_get_chipselect(spi, 0);
 	uint16_t chipselect = spi_xcomm->chipselect;
 
 	if (is_active)
diff --git a/drivers/spi/spi-xilinx.c b/drivers/spi/spi-xilinx.c
index 1411548f4255..8e6e3876aa9a 100644
--- a/drivers/spi/spi-xilinx.c
+++ b/drivers/spi/spi-xilinx.c
@@ -213,7 +213,7 @@ static void xilinx_spi_chipselect(struct spi_device *spi, int is_on)
 	 */
 
 	cs = xspi->cs_inactive;
-	cs ^= BIT(spi->chip_select);
+	cs ^= BIT(spi_get_chipselect(spi, 0));
 
 	/* Activate the chip select */
 	xspi->write_fn(cs, xspi->regs + XSPI_SSR_OFFSET);
@@ -228,9 +228,9 @@ static int xilinx_spi_setup_transfer(struct spi_device *spi,
 	struct xilinx_spi *xspi = spi_master_get_devdata(spi->master);
 
 	if (spi->mode & SPI_CS_HIGH)
-		xspi->cs_inactive &= ~BIT(spi->chip_select);
+		xspi->cs_inactive &= ~BIT(spi_get_chipselect(spi, 0));
 	else
-		xspi->cs_inactive |= BIT(spi->chip_select);
+		xspi->cs_inactive |= BIT(spi_get_chipselect(spi, 0));
 
 	return 0;
 }
@@ -440,8 +440,7 @@ static int xilinx_spi_probe(struct platform_device *pdev)
 	xspi->bitbang.txrx_bufs = xilinx_spi_txrx_bufs;
 	init_completion(&xspi->done);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	xspi->regs = devm_ioremap_resource(&pdev->dev, res);
+	xspi->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(xspi->regs))
 		return PTR_ERR(xspi->regs);
 
@@ -504,7 +503,7 @@ static int xilinx_spi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int xilinx_spi_remove(struct platform_device *pdev)
+static void xilinx_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct xilinx_spi *xspi = spi_master_get_devdata(master);
@@ -518,8 +517,6 @@ static int xilinx_spi_remove(struct platform_device *pdev)
 	xspi->write_fn(0, regs_base + XIPIF_V123B_DGIER_OFFSET);
 
 	spi_master_put(xspi->bitbang.master);
-
-	return 0;
 }
 
 /* work with hotplug and coldplug */
@@ -527,7 +524,7 @@ MODULE_ALIAS("platform:" XILINX_SPI_NAME);
 
 static struct platform_driver xilinx_spi_driver = {
 	.probe = xilinx_spi_probe,
-	.remove = xilinx_spi_remove,
+	.remove_new = xilinx_spi_remove,
 	.driver = {
 		.name = XILINX_SPI_NAME,
 		.of_match_table = xilinx_spi_of_match,
diff --git a/drivers/spi/spi-xlp.c b/drivers/spi/spi-xlp.c
index e5707fe5c8f1..3b91cdd5ae21 100644
--- a/drivers/spi/spi-xlp.c
+++ b/drivers/spi/spi-xlp.c
@@ -139,7 +139,7 @@ static int xlp_spi_setup(struct spi_device *spi)
 	int cs;
 
 	xspi = spi_master_get_devdata(spi->master);
-	cs = spi->chip_select;
+	cs = spi_get_chipselect(spi, 0);
 	/*
 	 * The value of fdiv must be between 4 and 65535.
 	 */
@@ -350,7 +350,7 @@ static int xlp_spi_transfer_one(struct spi_master *master,
 	struct xlp_spi_priv *xspi = spi_master_get_devdata(master);
 	int ret = 0;
 
-	xspi->cs = spi->chip_select;
+	xspi->cs = spi_get_chipselect(spi, 0);
 	xspi->dev = spi->dev;
 
 	if (spi_transfer_is_last(master, t))
diff --git a/drivers/spi/spi-xtensa-xtfpga.c b/drivers/spi/spi-xtensa-xtfpga.c
index 2fa7608f94cd..24dc845b940e 100644
--- a/drivers/spi/spi-xtensa-xtfpga.c
+++ b/drivers/spi/spi-xtensa-xtfpga.c
@@ -117,15 +117,13 @@ static int xtfpga_spi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int xtfpga_spi_remove(struct platform_device *pdev)
+static void xtfpga_spi_remove(struct platform_device *pdev)
 {
 	struct spi_master *master = platform_get_drvdata(pdev);
 	struct xtfpga_spi *xspi = spi_master_get_devdata(master);
 
 	spi_bitbang_stop(&xspi->bitbang);
 	spi_master_put(master);
-
-	return 0;
 }
 
 MODULE_ALIAS("platform:" XTFPGA_SPI_NAME);
@@ -140,7 +138,7 @@ MODULE_DEVICE_TABLE(of, xtfpga_spi_of_match);
 
 static struct platform_driver xtfpga_spi_driver = {
 	.probe = xtfpga_spi_probe,
-	.remove = xtfpga_spi_remove,
+	.remove_new = xtfpga_spi_remove,
 	.driver = {
 		.name = XTFPGA_SPI_NAME,
 		.of_match_table = of_match_ptr(xtfpga_spi_of_match),
diff --git a/drivers/spi/spi-zynq-qspi.c b/drivers/spi/spi-zynq-qspi.c
index 78f31b61a2aa..ee1995b91287 100644
--- a/drivers/spi/spi-zynq-qspi.c
+++ b/drivers/spi/spi-zynq-qspi.c
@@ -296,7 +296,7 @@ static void zynq_qspi_chipselect(struct spi_device *spi, bool assert)
 	/* Select the lower (CS0) or upper (CS1) memory */
 	if (ctlr->num_chipselect > 1) {
 		config_reg = zynq_qspi_read(xqspi, ZYNQ_QSPI_LINEAR_CFG_OFFSET);
-		if (!spi->chip_select)
+		if (!spi_get_chipselect(spi, 0))
 			config_reg &= ~ZYNQ_QSPI_LCFG_U_PAGE;
 		else
 			config_reg |= ZYNQ_QSPI_LCFG_U_PAGE;
@@ -741,7 +741,7 @@ remove_master:
  *
  * Return:	0 on success and error value on failure
  */
-static int zynq_qspi_remove(struct platform_device *pdev)
+static void zynq_qspi_remove(struct platform_device *pdev)
 {
 	struct zynq_qspi *xqspi = platform_get_drvdata(pdev);
 
@@ -749,8 +749,6 @@ static int zynq_qspi_remove(struct platform_device *pdev)
 
 	clk_disable_unprepare(xqspi->refclk);
 	clk_disable_unprepare(xqspi->pclk);
-
-	return 0;
 }
 
 static const struct of_device_id zynq_qspi_of_match[] = {
@@ -765,7 +763,7 @@ MODULE_DEVICE_TABLE(of, zynq_qspi_of_match);
  */
 static struct platform_driver zynq_qspi_driver = {
 	.probe = zynq_qspi_probe,
-	.remove = zynq_qspi_remove,
+	.remove_new = zynq_qspi_remove,
 	.driver = {
 		.name = "zynq-qspi",
 		.of_match_table = zynq_qspi_of_match,
diff --git a/drivers/spi/spi-zynqmp-gqspi.c b/drivers/spi/spi-zynqmp-gqspi.c
index 95ff15665d44..fb2ca9b90eab 100644
--- a/drivers/spi/spi-zynqmp-gqspi.c
+++ b/drivers/spi/spi-zynqmp-gqspi.c
@@ -468,7 +468,7 @@ static void zynqmp_qspi_chipselect(struct spi_device *qspi, bool is_high)
 	genfifoentry |= GQSPI_GENFIFO_MODE_SPI;
 
 	if (!is_high) {
-		if (!qspi->chip_select) {
+		if (!spi_get_chipselect(qspi, 0)) {
 			xqspi->genfifobus = GQSPI_GENFIFO_BUS_LOWER;
 			xqspi->genfifocs = GQSPI_GENFIFO_CS_LOWER;
 		} else {
@@ -1364,7 +1364,7 @@ remove_master:
  *
  * Return:	0 Always
  */
-static int zynqmp_qspi_remove(struct platform_device *pdev)
+static void zynqmp_qspi_remove(struct platform_device *pdev)
 {
 	struct zynqmp_qspi *xqspi = platform_get_drvdata(pdev);
 
@@ -1373,15 +1373,13 @@ static int zynqmp_qspi_remove(struct platform_device *pdev)
 	clk_disable_unprepare(xqspi->pclk);
 	pm_runtime_set_suspended(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 MODULE_DEVICE_TABLE(of, zynqmp_qspi_of_match);
 
 static struct platform_driver zynqmp_qspi_driver = {
 	.probe = zynqmp_qspi_probe,
-	.remove = zynqmp_qspi_remove,
+	.remove_new = zynqmp_qspi_remove,
 	.driver = {
 		.name = "zynqmp-qspi",
 		.of_match_table = zynqmp_qspi_of_match,
diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c
index 7bc14fb309a6..9291b2a0e887 100644
--- a/drivers/spi/spi.c
+++ b/drivers/spi/spi.c
@@ -117,24 +117,28 @@ static struct spi_statistics __percpu *spi_alloc_pcpu_stats(struct device *dev)
 	return pcpu_stats;
 }
 
-#define spi_pcpu_stats_totalize(ret, in, field)				\
-do {									\
-	int i;								\
-	ret = 0;							\
-	for_each_possible_cpu(i) {					\
-		const struct spi_statistics *pcpu_stats;		\
-		u64 inc;						\
-		unsigned int start;					\
-		pcpu_stats = per_cpu_ptr(in, i);			\
-		do {							\
-			start = u64_stats_fetch_begin(		\
-					&pcpu_stats->syncp);		\
-			inc = u64_stats_read(&pcpu_stats->field);	\
-		} while (u64_stats_fetch_retry(			\
-					&pcpu_stats->syncp, start));	\
-		ret += inc;						\
-	}								\
-} while (0)
+static ssize_t spi_emit_pcpu_stats(struct spi_statistics __percpu *stat,
+				   char *buf, size_t offset)
+{
+	u64 val = 0;
+	int i;
+
+	for_each_possible_cpu(i) {
+		const struct spi_statistics *pcpu_stats;
+		u64_stats_t *field;
+		unsigned int start;
+		u64 inc;
+
+		pcpu_stats = per_cpu_ptr(stat, i);
+		field = (void *)pcpu_stats + offset;
+		do {
+			start = u64_stats_fetch_begin(&pcpu_stats->syncp);
+			inc = u64_stats_read(field);
+		} while (u64_stats_fetch_retry(&pcpu_stats->syncp, start));
+		val += inc;
+	}
+	return sysfs_emit(buf, "%llu\n", val);
+}
 
 #define SPI_STATISTICS_ATTRS(field, file)				\
 static ssize_t spi_controller_##field##_show(struct device *dev,	\
@@ -165,11 +169,8 @@ static struct device_attribute dev_attr_spi_device_##field = {		\
 static ssize_t spi_statistics_##name##_show(struct spi_statistics __percpu *stat, \
 					    char *buf)			\
 {									\
-	ssize_t len;							\
-	u64 val;							\
-	spi_pcpu_stats_totalize(val, stat, field);			\
-	len = sysfs_emit(buf, "%llu\n", val);				\
-	return len;							\
+	return spi_emit_pcpu_stats(stat, buf,				\
+			offsetof(struct spi_statistics, field));	\
 }									\
 SPI_STATISTICS_ATTRS(name, file)
 
@@ -2354,8 +2355,8 @@ of_register_spi_device(struct spi_controller *ctlr, struct device_node *nc)
 	}
 
 	/* Select device driver */
-	rc = of_modalias_node(nc, spi->modalias,
-				sizeof(spi->modalias));
+	rc = of_alias_from_compatible(nc, spi->modalias,
+				      sizeof(spi->modalias));
 	if (rc < 0) {
 		dev_err(&ctlr->dev, "cannot find modalias for %pOF\n", nc);
 		goto err_out;
@@ -2367,8 +2368,8 @@ of_register_spi_device(struct spi_controller *ctlr, struct device_node *nc)
 
 	/* Store a pointer to the node in the device structure */
 	of_node_get(nc);
-	spi->dev.of_node = nc;
-	spi->dev.fwnode = of_fwnode_handle(nc);
+
+	device_set_node(&spi->dev, of_fwnode_handle(nc));
 
 	/* Register the new device */
 	rc = spi_add_device(spi);
@@ -2776,7 +2777,6 @@ static void spi_controller_release(struct device *dev)
 
 static struct class spi_master_class = {
 	.name		= "spi_master",
-	.owner		= THIS_MODULE,
 	.dev_release	= spi_controller_release,
 	.dev_groups	= spi_master_groups,
 };
@@ -2879,7 +2879,6 @@ static const struct attribute_group *spi_slave_groups[] = {
 
 static struct class spi_slave_class = {
 	.name		= "spi_slave",
-	.owner		= THIS_MODULE,
 	.dev_release	= spi_controller_release,
 	.dev_groups	= spi_slave_groups,
 };
@@ -3075,7 +3074,7 @@ static int spi_controller_check_ops(struct spi_controller *ctlr)
 	 * If ->mem_ops or ->mem_ops->exec_op is NULL, we request that at least
 	 * one of the ->transfer_xxx() method be implemented.
 	 */
-	if (!ctlr->mem_ops || (ctlr->mem_ops && !ctlr->mem_ops->exec_op)) {
+	if (!ctlr->mem_ops || !ctlr->mem_ops->exec_op) {
 		if (!ctlr->transfer && !ctlr->transfer_one &&
 		   !ctlr->transfer_one_message) {
 			return -EINVAL;
@@ -3621,6 +3620,55 @@ int spi_split_transfers_maxsize(struct spi_controller *ctlr,
 }
 EXPORT_SYMBOL_GPL(spi_split_transfers_maxsize);
 
+
+/**
+ * spi_split_transfers_maxwords - split spi transfers into multiple transfers
+ *                                when an individual transfer exceeds a
+ *                                certain number of SPI words
+ * @ctlr:     the @spi_controller for this transfer
+ * @msg:      the @spi_message to transform
+ * @maxwords: the number of words to limit each transfer to
+ * @gfp:      GFP allocation flags
+ *
+ * Return: status of transformation
+ */
+int spi_split_transfers_maxwords(struct spi_controller *ctlr,
+				 struct spi_message *msg,
+				 size_t maxwords,
+				 gfp_t gfp)
+{
+	struct spi_transfer *xfer;
+
+	/*
+	 * Iterate over the transfer_list,
+	 * but note that xfer is advanced to the last transfer inserted
+	 * to avoid checking sizes again unnecessarily (also xfer does
+	 * potentially belong to a different list by the time the
+	 * replacement has happened).
+	 */
+	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
+		size_t maxsize;
+		int ret;
+
+		if (xfer->bits_per_word <= 8)
+			maxsize = maxwords;
+		else if (xfer->bits_per_word <= 16)
+			maxsize = 2 * maxwords;
+		else
+			maxsize = 4 * maxwords;
+
+		if (xfer->len > maxsize) {
+			ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer,
+							   maxsize, gfp);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(spi_split_transfers_maxwords);
+
 /*-------------------------------------------------------------------------*/
 
 /* Core methods for SPI controller protocol drivers.  Some of the
diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c
index 5a038c667401..39d94c850839 100644
--- a/drivers/spi/spidev.c
+++ b/drivers/spi/spidev.c
@@ -393,7 +393,7 @@ spidev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 			struct spi_controller *ctlr = spi->controller;
 
 			if (ctlr->use_gpio_descriptors && ctlr->cs_gpiods &&
-			    ctlr->cs_gpiods[spi->chip_select])
+			    ctlr->cs_gpiods[spi_get_chipselect(spi, 0)])
 				tmp &= ~SPI_CS_HIGH;
 		}
 
@@ -432,7 +432,7 @@ spidev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 			}
 
 			if (ctlr->use_gpio_descriptors && ctlr->cs_gpiods &&
-			    ctlr->cs_gpiods[spi->chip_select])
+			    ctlr->cs_gpiods[spi_get_chipselect(spi, 0)])
 				tmp |= SPI_CS_HIGH;
 
 			tmp |= spi->mode & ~SPI_MODE_MASK;
@@ -805,7 +805,7 @@ static int spidev_probe(struct spi_device *spi)
 		spidev->devt = MKDEV(SPIDEV_MAJOR, minor);
 		dev = device_create(spidev_class, &spi->dev, spidev->devt,
 				    spidev, "spidev%d.%d",
-				    spi->master->bus_num, spi->chip_select);
+				    spi->master->bus_num, spi_get_chipselect(spi, 0));
 		status = PTR_ERR_OR_ZERO(dev);
 	} else {
 		dev_dbg(&spi->dev, "no minor number available!\n");
@@ -877,7 +877,7 @@ static int __init spidev_init(void)
 	if (status < 0)
 		return status;
 
-	spidev_class = class_create(THIS_MODULE, "spidev");
+	spidev_class = class_create("spidev");
 	if (IS_ERR(spidev_class)) {
 		unregister_chrdev(SPIDEV_MAJOR, spidev_spi_driver.driver.name);
 		return PTR_ERR(spidev_class);
diff --git a/drivers/spmi/hisi-spmi-controller.c b/drivers/spmi/hisi-spmi-controller.c
index 5bd23262abd6..9cbd473487cb 100644
--- a/drivers/spmi/hisi-spmi-controller.c
+++ b/drivers/spmi/hisi-spmi-controller.c
@@ -324,13 +324,12 @@ err_put_controller:
 	return ret;
 }
 
-static int spmi_del_controller(struct platform_device *pdev)
+static void spmi_del_controller(struct platform_device *pdev)
 {
 	struct spmi_controller *ctrl = platform_get_drvdata(pdev);
 
 	spmi_controller_remove(ctrl);
 	spmi_controller_put(ctrl);
-	return 0;
 }
 
 static const struct of_device_id spmi_controller_match_table[] = {
@@ -343,7 +342,7 @@ MODULE_DEVICE_TABLE(of, spmi_controller_match_table);
 
 static struct platform_driver spmi_controller_driver = {
 	.probe		= spmi_controller_probe,
-	.remove		= spmi_del_controller,
+	.remove_new	= spmi_del_controller,
 	.driver		= {
 		.name	= "hisi_spmi_controller",
 		.of_match_table = spmi_controller_match_table,
diff --git a/drivers/spmi/spmi-mtk-pmif.c b/drivers/spmi/spmi-mtk-pmif.c
index ad511f2c3324..b3c991e1ea40 100644
--- a/drivers/spmi/spmi-mtk-pmif.c
+++ b/drivers/spmi/spmi-mtk-pmif.c
@@ -503,7 +503,7 @@ err_put_ctrl:
 	return err;
 }
 
-static int mtk_spmi_remove(struct platform_device *pdev)
+static void mtk_spmi_remove(struct platform_device *pdev)
 {
 	struct spmi_controller *ctrl = platform_get_drvdata(pdev);
 	struct pmif *arb = spmi_controller_get_drvdata(ctrl);
@@ -511,7 +511,6 @@ static int mtk_spmi_remove(struct platform_device *pdev)
 	clk_bulk_disable_unprepare(arb->nclks, arb->clks);
 	spmi_controller_remove(ctrl);
 	spmi_controller_put(ctrl);
-	return 0;
 }
 
 static const struct of_device_id mtk_spmi_match_table[] = {
@@ -530,10 +529,10 @@ MODULE_DEVICE_TABLE(of, mtk_spmi_match_table);
 static struct platform_driver mtk_spmi_driver = {
 	.driver		= {
 		.name	= "spmi-mtk",
-		.of_match_table = of_match_ptr(mtk_spmi_match_table),
+		.of_match_table = mtk_spmi_match_table,
 	},
 	.probe		= mtk_spmi_probe,
-	.remove		= mtk_spmi_remove,
+	.remove_new	= mtk_spmi_remove,
 };
 module_platform_driver(mtk_spmi_driver);
 
diff --git a/drivers/spmi/spmi-pmic-arb.c b/drivers/spmi/spmi-pmic-arb.c
index 8b6a42ab816f..dcb675d980d4 100644
--- a/drivers/spmi/spmi-pmic-arb.c
+++ b/drivers/spmi/spmi-pmic-arb.c
@@ -126,7 +126,7 @@ struct apid_data {
 };
 
 /**
- * spmi_pmic_arb - SPMI PMIC Arbiter object
+ * struct spmi_pmic_arb - SPMI PMIC Arbiter object
  *
  * @rd_base:		on v1 "core", on v2 "observer" register base off DT.
  * @wr_base:		on v1 "core", on v2 "chnls"    register base off DT.
@@ -180,7 +180,7 @@ struct spmi_pmic_arb {
 };
 
 /**
- * pmic_arb_ver: version dependent functionality.
+ * struct pmic_arb_ver_ops - version dependent functionality.
  *
  * @ver_str:		version string.
  * @ppid_to_apid:	finds the apid for a given ppid.
@@ -1674,7 +1674,7 @@ err_put_ctrl:
 	return err;
 }
 
-static int spmi_pmic_arb_remove(struct platform_device *pdev)
+static void spmi_pmic_arb_remove(struct platform_device *pdev)
 {
 	struct spmi_controller *ctrl = platform_get_drvdata(pdev);
 	struct spmi_pmic_arb *pmic_arb = spmi_controller_get_drvdata(ctrl);
@@ -1682,7 +1682,6 @@ static int spmi_pmic_arb_remove(struct platform_device *pdev)
 	irq_set_chained_handler_and_data(pmic_arb->irq, NULL, NULL);
 	irq_domain_remove(pmic_arb->domain);
 	spmi_controller_put(ctrl);
-	return 0;
 }
 
 static const struct of_device_id spmi_pmic_arb_match_table[] = {
@@ -1693,7 +1692,7 @@ MODULE_DEVICE_TABLE(of, spmi_pmic_arb_match_table);
 
 static struct platform_driver spmi_pmic_arb_driver = {
 	.probe		= spmi_pmic_arb_probe,
-	.remove		= spmi_pmic_arb_remove,
+	.remove_new	= spmi_pmic_arb_remove,
 	.driver		= {
 		.name	= "spmi_pmic_arb",
 		.of_match_table = spmi_pmic_arb_match_table,
diff --git a/drivers/spmi/spmi.c b/drivers/spmi/spmi.c
index 73551531ed43..7313d4c18a04 100644
--- a/drivers/spmi/spmi.c
+++ b/drivers/spmi/spmi.c
@@ -350,7 +350,8 @@ static void spmi_drv_remove(struct device *dev)
 	const struct spmi_driver *sdrv = to_spmi_driver(dev->driver);
 
 	pm_runtime_get_sync(dev);
-	sdrv->remove(to_spmi_device(dev));
+	if (sdrv->remove)
+		sdrv->remove(to_spmi_device(dev));
 	pm_runtime_put_noidle(dev);
 
 	pm_runtime_disable(dev);
@@ -404,7 +405,7 @@ struct spmi_device *spmi_device_from_of(struct device_node *np)
 EXPORT_SYMBOL_GPL(spmi_device_from_of);
 
 /**
- * spmi_controller_alloc() - Allocate a new SPMI device
+ * spmi_device_alloc() - Allocate a new SPMI device
  * @ctrl:	associated controller
  *
  * Caller is responsible for either calling spmi_device_add() to add the
@@ -582,8 +583,9 @@ void spmi_controller_remove(struct spmi_controller *ctrl)
 EXPORT_SYMBOL_GPL(spmi_controller_remove);
 
 /**
- * spmi_driver_register() - Register client driver with SPMI core
+ * __spmi_driver_register() - Register client driver with SPMI core
  * @sdrv:	client driver to be associated with client-device.
+ * @owner:	module owner
  *
  * This API will register the client driver with the SPMI framework.
  * It is typically called from the driver's module-init function.
diff --git a/drivers/staging/axis-fifo/axis-fifo.c b/drivers/staging/axis-fifo/axis-fifo.c
index dfd2b357f484..7a21f2423204 100644
--- a/drivers/staging/axis-fifo/axis-fifo.c
+++ b/drivers/staging/axis-fifo/axis-fifo.c
@@ -103,17 +103,17 @@
  *           globals
  * ----------------------------
  */
-static int read_timeout = 1000; /* ms to wait before read() times out */
-static int write_timeout = 1000; /* ms to wait before write() times out */
+static long read_timeout = 1000; /* ms to wait before read() times out */
+static long write_timeout = 1000; /* ms to wait before write() times out */
 
 /* ----------------------------
  * module command-line arguments
  * ----------------------------
  */
 
-module_param(read_timeout, int, 0444);
+module_param(read_timeout, long, 0444);
 MODULE_PARM_DESC(read_timeout, "ms to wait before blocking read() timing out; set to -1 for no timeout");
-module_param(write_timeout, int, 0444);
+module_param(write_timeout, long, 0444);
 MODULE_PARM_DESC(write_timeout, "ms to wait before blocking write() timing out; set to -1 for no timeout");
 
 /* ----------------------------
@@ -384,9 +384,7 @@ static ssize_t axis_fifo_read(struct file *f, char __user *buf,
 		mutex_lock(&fifo->read_lock);
 		ret = wait_event_interruptible_timeout(fifo->read_queue,
 			ioread32(fifo->base_addr + XLLF_RDFO_OFFSET),
-				 (read_timeout >= 0) ?
-				  msecs_to_jiffies(read_timeout) :
-				  MAX_SCHEDULE_TIMEOUT);
+			read_timeout);
 
 		if (ret <= 0) {
 			if (ret == 0) {
@@ -528,9 +526,7 @@ static ssize_t axis_fifo_write(struct file *f, const char __user *buf,
 		ret = wait_event_interruptible_timeout(fifo->write_queue,
 			ioread32(fifo->base_addr + XLLF_TDFV_OFFSET)
 				 >= words_to_write,
-				 (write_timeout >= 0) ?
-				  msecs_to_jiffies(write_timeout) :
-				  MAX_SCHEDULE_TIMEOUT);
+			write_timeout);
 
 		if (ret <= 0) {
 			if (ret == 0) {
@@ -920,15 +916,13 @@ err_initial:
 	return rc;
 }
 
-static int axis_fifo_remove(struct platform_device *pdev)
+static void axis_fifo_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct axis_fifo *fifo = dev_get_drvdata(dev);
 
 	misc_deregister(&fifo->miscdev);
 	dev_set_drvdata(dev, NULL);
-
-	return 0;
 }
 
 static const struct of_device_id axis_fifo_of_match[] = {
@@ -943,12 +937,22 @@ static struct platform_driver axis_fifo_driver = {
 		.of_match_table	= axis_fifo_of_match,
 	},
 	.probe		= axis_fifo_probe,
-	.remove		= axis_fifo_remove,
+	.remove_new	= axis_fifo_remove,
 };
 
 static int __init axis_fifo_init(void)
 {
-	pr_info("axis-fifo driver loaded with parameters read_timeout = %i, write_timeout = %i\n",
+	if (read_timeout >= 0)
+		read_timeout = msecs_to_jiffies(read_timeout);
+	else
+		read_timeout = MAX_SCHEDULE_TIMEOUT;
+
+	if (write_timeout >= 0)
+		write_timeout = msecs_to_jiffies(write_timeout);
+	else
+		write_timeout = MAX_SCHEDULE_TIMEOUT;
+
+	pr_info("axis-fifo driver loaded with parameters read_timeout = %li, write_timeout = %li\n",
 		read_timeout, write_timeout);
 	return platform_driver_register(&axis_fifo_driver);
 }
diff --git a/drivers/staging/emxx_udc/emxx_udc.c b/drivers/staging/emxx_udc/emxx_udc.c
index f9765841c4aa..eb63daaca702 100644
--- a/drivers/staging/emxx_udc/emxx_udc.c
+++ b/drivers/staging/emxx_udc/emxx_udc.c
@@ -3137,7 +3137,7 @@ static void nbu2ss_drv_shutdown(struct platform_device *pdev)
 }
 
 /*-------------------------------------------------------------------------*/
-static int nbu2ss_drv_remove(struct platform_device *pdev)
+static void nbu2ss_drv_remove(struct platform_device *pdev)
 {
 	struct nbu2ss_udc	*udc;
 	struct nbu2ss_ep	*ep;
@@ -3154,8 +3154,6 @@ static int nbu2ss_drv_remove(struct platform_device *pdev)
 
 	/* Interrupt Handler - Release */
 	free_irq(vbus_irq, udc);
-
-	return 0;
 }
 
 /*-------------------------------------------------------------------------*/
@@ -3210,7 +3208,7 @@ static int nbu2ss_drv_resume(struct platform_device *pdev)
 static struct platform_driver udc_driver = {
 	.probe		= nbu2ss_drv_probe,
 	.shutdown	= nbu2ss_drv_shutdown,
-	.remove		= nbu2ss_drv_remove,
+	.remove_new	= nbu2ss_drv_remove,
 	.suspend	= nbu2ss_drv_suspend,
 	.resume		= nbu2ss_drv_resume,
 	.driver		= {
diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c
index afaba94d1d1c..3a4abf3bae40 100644
--- a/drivers/staging/fbtft/fbtft-core.c
+++ b/drivers/staging/fbtft/fbtft-core.c
@@ -840,7 +840,7 @@ int fbtft_register_framebuffer(struct fb_info *fb_info)
 		sprintf(text1, ", %zu KiB buffer memory", par->txbuf.len >> 10);
 	if (spi)
 		sprintf(text2, ", spi%d.%d at %d MHz", spi->master->bus_num,
-			spi->chip_select, spi->max_speed_hz / 1000000);
+			spi_get_chipselect(spi, 0), spi->max_speed_hz / 1000000);
 	dev_info(fb_info->dev,
 		 "%s frame buffer, %dx%d, %d KiB video memory%s, fps=%lu%s\n",
 		 fb_info->fix.id, fb_info->var.xres, fb_info->var.yres,
diff --git a/drivers/staging/fieldbus/anybuss/arcx-anybus.c b/drivers/staging/fieldbus/anybuss/arcx-anybus.c
index 9af2e63050d1..f135b9f52c8d 100644
--- a/drivers/staging/fieldbus/anybuss/arcx-anybus.c
+++ b/drivers/staging/fieldbus/anybuss/arcx-anybus.c
@@ -321,7 +321,7 @@ out_reset:
 	return err;
 }
 
-static int controller_remove(struct platform_device *pdev)
+static void controller_remove(struct platform_device *pdev)
 {
 	struct controller_priv *cd = platform_get_drvdata(pdev);
 	int id = cd->class_dev->id;
@@ -329,7 +329,6 @@ static int controller_remove(struct platform_device *pdev)
 	device_unregister(cd->class_dev);
 	ida_simple_remove(&controller_index_ida, id);
 	gpiod_set_value_cansleep(cd->reset_gpiod, 1);
-	return 0;
 }
 
 static const struct of_device_id controller_of_match[] = {
@@ -341,7 +340,7 @@ MODULE_DEVICE_TABLE(of, controller_of_match);
 
 static struct platform_driver controller_driver = {
 	.probe = controller_probe,
-	.remove = controller_remove,
+	.remove_new = controller_remove,
 	.driver		= {
 		.name   = "arcx-anybus-controller",
 		.of_match_table	= of_match_ptr(controller_of_match),
@@ -352,7 +351,7 @@ static int __init controller_init(void)
 {
 	int err;
 
-	controller_class = class_create(THIS_MODULE, "arcx_anybus_controller");
+	controller_class = class_create("arcx_anybus_controller");
 	if (IS_ERR(controller_class))
 		return PTR_ERR(controller_class);
 	err = platform_driver_register(&controller_driver);
diff --git a/drivers/staging/fieldbus/dev_core.c b/drivers/staging/fieldbus/dev_core.c
index 5f54f2674bd1..bf1812d8924f 100644
--- a/drivers/staging/fieldbus/dev_core.c
+++ b/drivers/staging/fieldbus/dev_core.c
@@ -154,7 +154,6 @@ __ATTRIBUTE_GROUPS(fieldbus);
 
 static struct class fieldbus_class = {
 	.name =		"fieldbus_dev",
-	.owner =	THIS_MODULE,
 	.dev_groups =	fieldbus_groups,
 };
 
diff --git a/drivers/staging/gdm724x/gdm_lte.c b/drivers/staging/gdm724x/gdm_lte.c
index 671ee8843c88..5703a9ddb6d0 100644
--- a/drivers/staging/gdm724x/gdm_lte.c
+++ b/drivers/staging/gdm724x/gdm_lte.c
@@ -349,7 +349,7 @@ static s32 gdm_lte_tx_nic_type(struct net_device *dev, struct sk_buff *skb)
 	/* Get ethernet protocol */
 	eth = (struct ethhdr *)skb->data;
 	if (ntohs(eth->h_proto) == ETH_P_8021Q) {
-		vlan_eth = (struct vlan_ethhdr *)skb->data;
+		vlan_eth = skb_vlan_eth_hdr(skb);
 		mac_proto = ntohs(vlan_eth->h_vlan_encapsulated_proto);
 		network_data = skb->data + VLAN_ETH_HLEN;
 		nic_type |= NIC_TYPE_F_VLAN;
@@ -435,7 +435,7 @@ static netdev_tx_t gdm_lte_tx(struct sk_buff *skb, struct net_device *dev)
 	 * driver based on the NIC mac
 	 */
 	if (nic_type & NIC_TYPE_F_VLAN) {
-		struct vlan_ethhdr *vlan_eth = (struct vlan_ethhdr *)skb->data;
+		struct vlan_ethhdr *vlan_eth = skb_vlan_eth_hdr(skb);
 
 		nic->vlan_id = ntohs(vlan_eth->h_vlan_TCI) & VLAN_VID_MASK;
 		data_buf = skb->data + (VLAN_ETH_HLEN - ETH_HLEN);
diff --git a/drivers/staging/greybus/arche-apb-ctrl.c b/drivers/staging/greybus/arche-apb-ctrl.c
index 45afa208d004..8541995008da 100644
--- a/drivers/staging/greybus/arche-apb-ctrl.c
+++ b/drivers/staging/greybus/arche-apb-ctrl.c
@@ -419,13 +419,11 @@ static int arche_apb_ctrl_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int arche_apb_ctrl_remove(struct platform_device *pdev)
+static void arche_apb_ctrl_remove(struct platform_device *pdev)
 {
 	device_remove_file(&pdev->dev, &dev_attr_state);
 	poweroff_seq(pdev);
 	platform_set_drvdata(pdev, NULL);
-
-	return 0;
 }
 
 static int __maybe_unused arche_apb_ctrl_suspend(struct device *dev)
@@ -471,7 +469,7 @@ static const struct of_device_id arche_apb_ctrl_of_match[] = {
 
 static struct platform_driver arche_apb_ctrl_device_driver = {
 	.probe		= arche_apb_ctrl_probe,
-	.remove		= arche_apb_ctrl_remove,
+	.remove_new	= arche_apb_ctrl_remove,
 	.shutdown	= arche_apb_ctrl_shutdown,
 	.driver		= {
 		.name	= "arche-apb-ctrl",
diff --git a/drivers/staging/greybus/arche-platform.c b/drivers/staging/greybus/arche-platform.c
index fcbd5f71eff2..ebe835f25d13 100644
--- a/drivers/staging/greybus/arche-platform.c
+++ b/drivers/staging/greybus/arche-platform.c
@@ -559,7 +559,7 @@ static int arche_remove_child(struct device *dev, void *unused)
 	return 0;
 }
 
-static int arche_platform_remove(struct platform_device *pdev)
+static void arche_platform_remove(struct platform_device *pdev)
 {
 	struct arche_platform_drvdata *arche_pdata = platform_get_drvdata(pdev);
 
@@ -570,8 +570,6 @@ static int arche_platform_remove(struct platform_device *pdev)
 
 	if (usb3613_hub_mode_ctrl(false))
 		dev_warn(arche_pdata->dev, "failed to control hub device\n");
-		/* TODO: Should we do anything more here ?? */
-	return 0;
 }
 
 static __maybe_unused int arche_platform_suspend(struct device *dev)
@@ -631,7 +629,7 @@ MODULE_DEVICE_TABLE(of, arche_combined_id);
 
 static struct platform_driver arche_platform_device_driver = {
 	.probe		= arche_platform_probe,
-	.remove		= arche_platform_remove,
+	.remove_new	= arche_platform_remove,
 	.shutdown	= arche_platform_shutdown,
 	.driver		= {
 		.name	= "arche-platform-ctrl",
diff --git a/drivers/staging/greybus/audio_manager_module.c b/drivers/staging/greybus/audio_manager_module.c
index 0a0f0a394c84..5f9dcbdbc191 100644
--- a/drivers/staging/greybus/audio_manager_module.c
+++ b/drivers/staging/greybus/audio_manager_module.c
@@ -12,8 +12,11 @@
 
 #define to_gb_audio_module_attr(x)	\
 		container_of(x, struct gb_audio_manager_module_attribute, attr)
-#define to_gb_audio_module(x)		\
-		container_of(x, struct gb_audio_manager_module, kobj)
+
+static inline struct gb_audio_manager_module *to_gb_audio_module(struct kobject *kobj)
+{
+	return container_of(kobj, struct gb_audio_manager_module, kobj);
+}
 
 struct gb_audio_manager_module_attribute {
 	struct attribute attr;
@@ -70,9 +73,8 @@ static void gb_audio_module_release(struct kobject *kobj)
 	kfree(module);
 }
 
-static ssize_t gb_audio_module_name_show(
-	struct gb_audio_manager_module *module,
-	struct gb_audio_manager_module_attribute *attr, char *buf)
+static ssize_t gb_audio_module_name_show(struct gb_audio_manager_module *module,
+					 struct gb_audio_manager_module_attribute *attr, char *buf)
 {
 	return sprintf(buf, "%s", module->desc.name);
 }
@@ -80,9 +82,8 @@ static ssize_t gb_audio_module_name_show(
 static struct gb_audio_manager_module_attribute gb_audio_module_name_attribute =
 	__ATTR(name, 0664, gb_audio_module_name_show, NULL);
 
-static ssize_t gb_audio_module_vid_show(
-	struct gb_audio_manager_module *module,
-	struct gb_audio_manager_module_attribute *attr, char *buf)
+static ssize_t gb_audio_module_vid_show(struct gb_audio_manager_module *module,
+					struct gb_audio_manager_module_attribute *attr, char *buf)
 {
 	return sprintf(buf, "%d", module->desc.vid);
 }
@@ -90,9 +91,8 @@ static ssize_t gb_audio_module_vid_show(
 static struct gb_audio_manager_module_attribute gb_audio_module_vid_attribute =
 	__ATTR(vid, 0664, gb_audio_module_vid_show, NULL);
 
-static ssize_t gb_audio_module_pid_show(
-	struct gb_audio_manager_module *module,
-	struct gb_audio_manager_module_attribute *attr, char *buf)
+static ssize_t gb_audio_module_pid_show(struct gb_audio_manager_module *module,
+					struct gb_audio_manager_module_attribute *attr, char *buf)
 {
 	return sprintf(buf, "%d", module->desc.pid);
 }
@@ -100,9 +100,9 @@ static ssize_t gb_audio_module_pid_show(
 static struct gb_audio_manager_module_attribute gb_audio_module_pid_attribute =
 	__ATTR(pid, 0664, gb_audio_module_pid_show, NULL);
 
-static ssize_t gb_audio_module_intf_id_show(
-	struct gb_audio_manager_module *module,
-	struct gb_audio_manager_module_attribute *attr, char *buf)
+static ssize_t gb_audio_module_intf_id_show(struct gb_audio_manager_module *module,
+					    struct gb_audio_manager_module_attribute *attr,
+					    char *buf)
 {
 	return sprintf(buf, "%d", module->desc.intf_id);
 }
@@ -111,9 +111,9 @@ static struct gb_audio_manager_module_attribute
 					gb_audio_module_intf_id_attribute =
 	__ATTR(intf_id, 0664, gb_audio_module_intf_id_show, NULL);
 
-static ssize_t gb_audio_module_ip_devices_show(
-	struct gb_audio_manager_module *module,
-	struct gb_audio_manager_module_attribute *attr, char *buf)
+static ssize_t gb_audio_module_ip_devices_show(struct gb_audio_manager_module *module,
+					       struct gb_audio_manager_module_attribute *attr,
+					       char *buf)
 {
 	return sprintf(buf, "0x%X", module->desc.ip_devices);
 }
@@ -122,9 +122,9 @@ static struct gb_audio_manager_module_attribute
 					gb_audio_module_ip_devices_attribute =
 	__ATTR(ip_devices, 0664, gb_audio_module_ip_devices_show, NULL);
 
-static ssize_t gb_audio_module_op_devices_show(
-	struct gb_audio_manager_module *module,
-	struct gb_audio_manager_module_attribute *attr, char *buf)
+static ssize_t gb_audio_module_op_devices_show(struct gb_audio_manager_module *module,
+					       struct gb_audio_manager_module_attribute *attr,
+					       char *buf)
 {
 	return sprintf(buf, "0x%X", module->desc.op_devices);
 }
@@ -181,10 +181,9 @@ static void send_add_uevent(struct gb_audio_manager_module *module)
 	kobject_uevent_env(&module->kobj, KOBJ_ADD, envp);
 }
 
-int gb_audio_manager_module_create(
-	struct gb_audio_manager_module **module,
-	struct kset *manager_kset,
-	int id, struct gb_audio_manager_module_descriptor *desc)
+int gb_audio_manager_module_create(struct gb_audio_manager_module **module,
+				   struct kset *manager_kset,
+				   int id, struct gb_audio_manager_module_descriptor *desc)
 {
 	int err;
 	struct gb_audio_manager_module *m;
diff --git a/drivers/staging/greybus/audio_topology.c b/drivers/staging/greybus/audio_topology.c
index 62d7674852be..08e6a807c132 100644
--- a/drivers/staging/greybus/audio_topology.c
+++ b/drivers/staging/greybus/audio_topology.c
@@ -24,9 +24,8 @@ struct gbaudio_ctl_pvt {
 	struct gb_audio_ctl_elem_info *info;
 };
 
-static struct gbaudio_module_info *find_gb_module(
-					struct gbaudio_codec_info *codec,
-					char const *name)
+static struct gbaudio_module_info *find_gb_module(struct gbaudio_codec_info *codec,
+						  char const *name)
 {
 	int dev_id;
 	char begin[NAME_SIZE];
diff --git a/drivers/staging/greybus/authentication.c b/drivers/staging/greybus/authentication.c
index 297e69f011c7..7e01790a4659 100644
--- a/drivers/staging/greybus/authentication.c
+++ b/drivers/staging/greybus/authentication.c
@@ -402,7 +402,7 @@ int cap_init(void)
 {
 	int ret;
 
-	cap_class = class_create(THIS_MODULE, "gb_authenticate");
+	cap_class = class_create("gb_authenticate");
 	if (IS_ERR(cap_class))
 		return PTR_ERR(cap_class);
 
diff --git a/drivers/staging/greybus/fw-management.c b/drivers/staging/greybus/fw-management.c
index 3342b84597da..cd9141e4b794 100644
--- a/drivers/staging/greybus/fw-management.c
+++ b/drivers/staging/greybus/fw-management.c
@@ -696,7 +696,7 @@ int fw_mgmt_init(void)
 {
 	int ret;
 
-	fw_mgmt_class = class_create(THIS_MODULE, "gb_fw_mgmt");
+	fw_mgmt_class = class_create("gb_fw_mgmt");
 	if (IS_ERR(fw_mgmt_class))
 		return PTR_ERR(fw_mgmt_class);
 
diff --git a/drivers/staging/greybus/gpio.c b/drivers/staging/greybus/gpio.c
index d729b922a750..2a115a8fc263 100644
--- a/drivers/staging/greybus/gpio.c
+++ b/drivers/staging/greybus/gpio.c
@@ -41,8 +41,11 @@ struct gb_gpio_controller {
 	struct irq_chip		irqc;
 	struct mutex		irq_lock;
 };
-#define gpio_chip_to_gb_gpio_controller(chip) \
-	container_of(chip, struct gb_gpio_controller, chip)
+
+static inline struct gb_gpio_controller *gpio_chip_to_gb_gpio_controller(struct gpio_chip *chip)
+{
+	return container_of(chip, struct gb_gpio_controller, chip);
+}
 
 static struct gpio_chip *irq_data_to_gpio_chip(struct irq_data *d)
 {
diff --git a/drivers/staging/greybus/greybus_authentication.h b/drivers/staging/greybus/greybus_authentication.h
index 7edc7295b7ab..48b4a9794d3c 100644
--- a/drivers/staging/greybus/greybus_authentication.h
+++ b/drivers/staging/greybus/greybus_authentication.h
@@ -41,7 +41,6 @@
 #define CAP_AUTH_RESULT_CR_NO_KEY	0x03
 #define CAP_AUTH_RESULT_CR_SIG_FAIL	0x04
 
-
 /* IOCTL support */
 struct cap_ioc_get_endpoint_uid {
 	__u8			uid[8];
diff --git a/drivers/staging/greybus/loopback.c b/drivers/staging/greybus/loopback.c
index 1a61fce98056..d7b39f3bb652 100644
--- a/drivers/staging/greybus/loopback.c
+++ b/drivers/staging/greybus/loopback.c
@@ -100,7 +100,6 @@ struct gb_loopback {
 
 static struct class loopback_class = {
 	.name		= "gb_loopback",
-	.owner		= THIS_MODULE,
 };
 static DEFINE_IDA(loopback_ida);
 
diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c
index 3fda172239d2..88da1d796f13 100644
--- a/drivers/staging/greybus/pwm.c
+++ b/drivers/staging/greybus/pwm.c
@@ -21,9 +21,11 @@ struct gb_pwm_chip {
 	struct pwm_chip		chip;
 	struct pwm_chip		*pwm;
 };
-#define pwm_chip_to_gb_pwm_chip(chip) \
-	container_of(chip, struct gb_pwm_chip, chip)
 
+static inline struct gb_pwm_chip *pwm_chip_to_gb_pwm_chip(struct pwm_chip *chip)
+{
+	return container_of(chip, struct gb_pwm_chip, chip);
+}
 
 static int gb_pwm_count_operation(struct gb_pwm_chip *pwmc)
 {
diff --git a/drivers/staging/greybus/raw.c b/drivers/staging/greybus/raw.c
index 2a375f407d38..8bca8cb12cc6 100644
--- a/drivers/staging/greybus/raw.c
+++ b/drivers/staging/greybus/raw.c
@@ -340,7 +340,7 @@ static int raw_init(void)
 	dev_t dev;
 	int retval;
 
-	raw_class = class_create(THIS_MODULE, "gb_raw");
+	raw_class = class_create("gb_raw");
 	if (IS_ERR(raw_class)) {
 		retval = PTR_ERR(raw_class);
 		goto error_class;
diff --git a/drivers/staging/greybus/spilib.c b/drivers/staging/greybus/spilib.c
index ad0700a0bb81..efb3bec58e15 100644
--- a/drivers/staging/greybus/spilib.c
+++ b/drivers/staging/greybus/spilib.c
@@ -237,7 +237,7 @@ static struct gb_operation *gb_spi_operation_create(struct gb_spilib *spi,
 	request = operation->request->payload;
 	request->count = cpu_to_le16(count);
 	request->mode = dev->mode;
-	request->chip_select = dev->chip_select;
+	request->chip_select = spi_get_chipselect(dev, 0);
 
 	gb_xfer = &request->transfers[0];
 	tx_data = gb_xfer + count;	/* place tx data after last gb_xfer */
diff --git a/drivers/staging/greybus/tools/.gitignore b/drivers/staging/greybus/tools/.gitignore
deleted file mode 100644
index 1fd364aba774..000000000000
--- a/drivers/staging/greybus/tools/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-loopback_test
diff --git a/drivers/staging/greybus/tools/Android.mk b/drivers/staging/greybus/tools/Android.mk
deleted file mode 100644
index fdadbf611757..000000000000
--- a/drivers/staging/greybus/tools/Android.mk
+++ /dev/null
@@ -1,10 +0,0 @@
-LOCAL_PATH:= $(call my-dir)
-
-include $(CLEAR_VARS)
-
-LOCAL_SRC_FILES:= loopback_test.c
-LOCAL_MODULE_TAGS := optional
-LOCAL_MODULE := gb_loopback_test
-
-include $(BUILD_EXECUTABLE)
-
diff --git a/drivers/staging/greybus/tools/Makefile b/drivers/staging/greybus/tools/Makefile
deleted file mode 100644
index a3bbd73171f2..000000000000
--- a/drivers/staging/greybus/tools/Makefile
+++ /dev/null
@@ -1,33 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-ifeq ($(strip $(V)), 1)
-  Q =
-else
-  Q = @
-endif
-
-CFLAGS	+= -std=gnu99 -Wall -Wextra -g \
-	    -D_GNU_SOURCE \
-	    -Wno-unused-parameter \
-	    -Wmaybe-uninitialized \
-	    -Wredundant-decls \
-	    -Wcast-align \
-	    -Wsign-compare \
-	    -Wno-missing-field-initializers \
-	    -Wno-shift-negative-value
-
-CC	:= $(CROSS_COMPILE)gcc
-
-TOOLS = loopback_test
-
-all: $(TOOLS)
-
-%.o: %.c ../greybus_protocols.h
-	@echo '  TARGET_CC $@'
-	$(Q)$(CC) $(CFLAGS) -c $< -o $@
-
-loopback_%: loopback_%.o
-	@echo '  TARGET_LD $@'
-	$(Q)$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@
-
-clean::
-	rm -f *.o $(TOOLS)
diff --git a/drivers/staging/greybus/tools/README.loopback b/drivers/staging/greybus/tools/README.loopback
deleted file mode 100644
index 070a510cbe7c..000000000000
--- a/drivers/staging/greybus/tools/README.loopback
+++ /dev/null
@@ -1,198 +0,0 @@
-
-
-        1 - LOOPBACK DRIVER
-
-The driver implements the main logic of the loopback test and provides
-sysfs files to configure the test and retrieve the results.
-A user could run a test without the need of the test application given
-that he understands the sysfs interface of the loopback driver.
-
-The loopback kernel driver needs to be loaded and at least one module
-with the loopback feature enabled must be present for the sysfs files to be
-created and for the loopback test application to be able to run.
-
-To load the module:
-# modprobe gb-loopback
-
-
-When the module is probed,  New files are available on the sysfs
-directory of the detected loopback device.
-(typically under "/sys/bus/graybus/devices").
-
-Here is a short summary of the sysfs interface files that should be visible:
-
-* Loopback Configuration Files:
-    async - Use asynchronous operations.
-    iteration_max - Number of tests iterations to perform.
-    size - payload size of the transfer.
-    timeout - The number of microseconds to give an individual
-              asynchronous request before timing out.
-    us_wait - Time to wait between 2 messages
-    type - By writing the test type to this file, the test starts.
-           Valid tests are:
-             0 stop the test
-             2 - ping
-             3 - transfer
-             4 - sink
-
-* Loopback feedback files:
-    error - number of errors that have occurred.
-    iteration_count - Number of iterations performed.
-    requests_completed - Number of requests successfully completed.
-    requests_timedout - Number of requests that have timed out.
-    timeout_max - Max allowed timeout
-    timeout_min - Min allowed timeout.
-
-* Loopback result files:
-    apbridge_unipro_latency_avg
-    apbridge_unipro_latency_max
-    apbridge_unipro_latency_min
-    gpbridge_firmware_latency_avg
-    gpbridge_firmware_latency_max
-    gpbridge_firmware_latency_min
-    requests_per_second_avg
-    requests_per_second_max
-    requests_per_second_min
-    latency_avg
-    latency_max
-    latency_min
-    throughput_avg
-    throughput_max
-    throughput_min
-
-
-
-            2 - LOOPBACK TEST APPLICATION
-
-The loopback test application manages and formats the results provided by
-the loopback kernel module. The purpose of this application
-is to:
-    - Start and manage multiple loopback device tests concurrently.
-    - Calculate the aggregate results for multiple devices.
-    - Gather and format test results (csv or human readable).
-
-The best way to get up to date usage information for the application is
-usually to pass the "-h" parameter.
-Here is the summary of the available options:
-
- Mandatory arguments
-   -t     must be one of the test names - sink, transfer or ping
-   -i     iteration count - the number of iterations to run the test over
- Optional arguments
-   -S     sysfs location - location for greybus 'endo' entries default /sys/bus/greybus/devices/
-   -D     debugfs location - location for loopback debugfs entries default /sys/kernel/debug/gb_loopback/
-   -s     size of data packet to send during test - defaults to zero
-   -m     mask - a bit mask of connections to include example: -m 8 = 4th connection -m 9 = 1st and 4th connection etc
-                 default is zero which means broadcast to all connections
-   -v     verbose output
-   -d     debug output
-   -r     raw data output - when specified the full list of latency values are included in the output CSV
-   -p     porcelain - when specified printout is in a user-friendly non-CSV format. This option suppresses writing to CSV file
-   -a     aggregate - show aggregation of all enabled devies
-   -l     list found loopback devices and exit.
-   -x     Async - Enable async transfers.
-   -o     Timeout - Timeout in microseconds for async operations.
-
-
-
-             3 - REAL WORLD EXAMPLE USAGES
-
- 3.1 - Using the driver sysfs files to run a test on a single device:
-
-* Run a 1000 transfers of a 100 byte packet. Each transfer is started only
-after the previous one finished successfully:
-    echo 0 > /sys/bus/greybus/devices/1-2.17/type
-    echo 0 > /sys/bus/greybus/devices/1-2.17/async
-    echo 2000 > /sys/bus/greybus/devices/1-2.17/us_wait
-    echo 100 > /sys/bus/greybus/devices/1-2.17/size
-    echo 1000 > /sys/bus/greybus/devices/1-2.17/iteration_max
-    echo 0 > /sys/bus/greybus/devices/1-2.17/mask
-    echo 200000 > /sys/bus/greybus/devices/1-2.17/timeout
-    echo 3 > /sys/bus/greybus/devices/1-2.17/type
-
-* Run a 1000 transfers of a 100 byte packet. Transfers are started without
-waiting for the previous one to finish:
-    echo 0 > /sys/bus/greybus/devices/1-2.17/type
-    echo 3 > /sys/bus/greybus/devices/1-2.17/async
-    echo 0 > /sys/bus/greybus/devices/1-2.17/us_wait
-    echo 100 > /sys/bus/greybus/devices/1-2.17/size
-    echo 1000 > /sys/bus/greybus/devices/1-2.17/iteration_max
-    echo 0 > /sys/bus/greybus/devices/1-2.17/mask
-    echo 200000 > /sys/bus/greybus/devices/1-2.17/timeout
-    echo 3 > /sys/bus/greybus/devices/1-2.17/type
-
-* Read the results from sysfs:
-    cat /sys/bus/greybus/devices/1-2.17/requests_per_second_min
-    cat /sys/bus/greybus/devices/1-2.17/requests_per_second_max
-    cat /sys/bus/greybus/devices/1-2.17/requests_per_second_avg
-
-    cat /sys/bus/greybus/devices/1-2.17/latency_min
-    cat /sys/bus/greybus/devices/1-2.17/latency_max
-    cat /sys/bus/greybus/devices/1-2.17/latency_avg
-
-    cat /sys/bus/greybus/devices/1-2.17/apbridge_unipro_latency_min
-    cat /sys/bus/greybus/devices/1-2.17/apbridge_unipro_latency_max
-    cat /sys/bus/greybus/devices/1-2.17/apbridge_unipro_latency_avg
-
-    cat /sys/bus/greybus/devices/1-2.17/gpbridge_firmware_latency_min
-    cat /sys/bus/greybus/devices/1-2.17/gpbridge_firmware_latency_max
-    cat /sys/bus/greybus/devices/1-2.17/gpbridge_firmware_latency_avg
-
-    cat /sys/bus/greybus/devices/1-2.17/error
-    cat /sys/bus/greybus/devices/1-2.17/requests_completed
-    cat /sys/bus/greybus/devices/1-2.17/requests_timedout
-
-
-3.2 - using the test application:
-
-* Run a transfer test 10 iterations of size 100 bytes on all available devices
-    #/loopback_test -t transfer -i 10 -s 100
-    1970-1-1 0:10:7,transfer,1-4.17,100,10,0,443,509,471.700012,66,1963,2256,2124.600098,293,102776,118088,109318.898438,15312,1620,1998,1894.099976,378,56,57,56.799999,1
-    1970-1-1 0:10:7,transfer,1-5.17,100,10,0,399,542,463.399994,143,1845,2505,2175.800049,660,92568,125744,107393.296875,33176,1469,2305,1806.500000,836,56,57,56.799999,1
-
-
-* Show the aggregate results of both devices. ("-a")
-    #/loopback_test -t transfer -i 10 -s 100  -a
-    1970-1-1 0:10:35,transfer,1-4.17,100,10,0,448,580,494.100006,132,1722,2230,2039.400024,508,103936,134560,114515.703125,30624,1513,1980,1806.900024,467,56,57,57.299999,1
-    1970-1-1 0:10:35,transfer,1-5.17,100,10,0,383,558,478.600006,175,1791,2606,2115.199951,815,88856,129456,110919.703125,40600,1457,2246,1773.599976,789,56,57,57.099998,1
-    1970-1-1 0:10:35,transfer,aggregate,100,10,0,383,580,486.000000,197,1722,2606,2077.000000,884,88856,134560,112717.000000,45704,1457,2246,1789.000000,789,56,57,57.000000,1
-
-* Example usage of the mask option to select which devices will
-  run the test (1st, 2nd, or both devices):
-    # /loopback_test -t transfer -i 10 -s 100 -m 1
-    1970-1-1 0:11:56,transfer,1-4.17,100,10,0,514,558,544.900024,44,1791,1943,1836.599976,152,119248,129456,126301.296875,10208,1600,1001609,101613.601562,1000009,56,57,56.900002,1
-    # /loopback_test -t transfer -i 10 -s 100 -m 2
-    1970-1-1 0:12:0,transfer,1-5.17,100,10,0,468,554,539.000000,86,1804,2134,1859.500000,330,108576,128528,124932.500000,19952,1606,1626,1619.300049,20,56,57,57.400002,1
-    # /loopback_test -t transfer -i 10 -s 100 -m 3
-    1970-1-1 0:12:3,transfer,1-4.17,100,10,0,432,510,469.399994,78,1959,2313,2135.800049,354,100224,118320,108785.296875,18096,1610,2024,1893.500000,414,56,57,57.200001,1
-    1970-1-1 0:12:3,transfer,1-5.17,100,10,0,404,542,468.799988,138,1843,2472,2152.500000,629,93728,125744,108646.101562,32016,1504,2247,1853.099976,743,56,57,57.099998,1
-
-* Show output in human readable format ("-p")
-    # /loopback_test -t transfer -i 10 -s 100 -m 3 -p
-
-    1970-1-1 0:12:37
-    test:                       transfer
-    path:                       1-4.17
-    size:                       100
-    iterations:                 10
-    errors:                     0
-    async:                      Disabled
-    requests per-sec:           min=390, max=547, average=469.299988, jitter=157
-    ap-throughput B/s:          min=90480 max=126904 average=108762.101562 jitter=36424
-    ap-latency usec:            min=1826 max=2560 average=2146.000000 jitter=734
-    apbridge-latency usec:      min=1620 max=1982 average=1882.099976 jitter=362
-    gpbridge-latency usec:      min=56 max=57 average=57.099998 jitter=1
-
-
-    1970-1-1 0:12:37
-    test:                       transfer
-    path:                       1-5.17
-    size:                       100
-    iterations:                 10
-    errors:                     0
-    async:                      Disabled
-    requests per-sec:           min=397, max=538, average=461.700012, jitter=141
-    ap-throughput B/s:          min=92104 max=124816 average=106998.898438 jitter=32712
-    ap-latency usec:            min=1856 max=2514 average=2185.699951 jitter=658
-    apbridge-latency usec:      min=1460 max=2296 average=1828.599976 jitter=836
-    gpbridge-latency usec:      min=56 max=57 average=57.099998 jitter=1
diff --git a/drivers/staging/greybus/tools/lbtest b/drivers/staging/greybus/tools/lbtest
deleted file mode 100755
index 47c481239e98..000000000000
--- a/drivers/staging/greybus/tools/lbtest
+++ /dev/null
@@ -1,169 +0,0 @@
-#!/usr/bin/env python
-# SPDX-License-Identifier: BSD-3-Clause
-
-# Copyright (c) 2015 Google, Inc.
-# Copyright (c) 2015 Linaro, Ltd.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-# 3. Neither the name of the copyright holder nor the names of its
-# contributors may be used to endorse or promote products derived from this
-# software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
-# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
-# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
-# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
-# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import print_function
-import csv
-import datetime
-import sys
-import time
-
-dict = {'ping': '2', 'transfer': '3', 'sink': '4'}
-verbose = 1
-
-def abort():
-	sys.exit(1)
-
-def usage():
-	print('Usage: looptest TEST SIZE ITERATIONS PATH\n\n'
-	'  Run TEST for a number of ITERATIONS with operation data SIZE bytes\n'
-	'  TEST may be \'ping\' \'transfer\' or \'sink\'\n'
-	'  SIZE indicates the size of transfer <= greybus max payload bytes\n'
-	'  ITERATIONS indicates the number of times to execute TEST at SIZE bytes\n'
-	'             Note if ITERATIONS is set to zero then this utility will\n'
-	'             initiate an infinite (non terminating) test and exit\n'
-	'             without logging any metrics data\n'
-	'  PATH indicates the sysfs path for the loopback greybus entries e.g.\n'
-	'             /sys/bus/greybus/devices/endo0:1:1:1:1/\n'
-	'Examples:\n'
-	'  looptest transfer 128 10000\n'
-	'  looptest ping 0 128\n'
-	'  looptest sink 2030 32768\n'
-	.format(sys.argv[0]), file=sys.stderr)
-
-	abort()
-
-def read_sysfs_int(path):
-	try:
-		f = open(path, "r");
-		val = f.read();
-		f.close()
-		return int(val)
-	except IOError as e:
-		print("I/O error({0}): {1}".format(e.errno, e.strerror))
-		print("Invalid path %s" % path)
-
-def write_sysfs_val(path, val):
-	try:
-		f = open(path, "r+")
-		f.write(val)
-		f.close()
-	except IOError as e:
-		print("I/O error({0}): {1}".format(e.errno, e.strerror))
-		print("Invalid path %s" % path)
-
-def log_csv(test_name, size, iteration_max, sys_pfx):
-	# file name will test_name_size_iteration_max.csv
-	# every time the same test with the same parameters is run we will then
-	# append to the same CSV with datestamp - representing each test dataset
-	fname = test_name + '_' + size + '_' + str(iteration_max) + '.csv'
-
-	try:
-		# gather data set
-		date = str(datetime.datetime.now())
-		error = read_sysfs_int(sys_pfx + 'error')
-		request_min = read_sysfs_int(sys_pfx + 'requests_per_second_min')
-		request_max = read_sysfs_int(sys_pfx + 'requests_per_second_max')
-		request_avg = read_sysfs_int(sys_pfx + 'requests_per_second_avg')
-		latency_min = read_sysfs_int(sys_pfx + 'latency_min')
-		latency_max = read_sysfs_int(sys_pfx + 'latency_max')
-		latency_avg = read_sysfs_int(sys_pfx + 'latency_avg')
-		throughput_min = read_sysfs_int(sys_pfx + 'throughput_min')
-		throughput_max = read_sysfs_int(sys_pfx + 'throughput_max')
-		throughput_avg = read_sysfs_int(sys_pfx + 'throughput_avg')
-
-		# derive jitter
-		request_jitter = request_max - request_min
-		latency_jitter = latency_max - latency_min
-		throughput_jitter = throughput_max - throughput_min
-
-		# append data set to file
-		with open(fname, 'a') as csvf:
-			row = csv.writer(csvf, delimiter=",", quotechar="'",
-					quoting=csv.QUOTE_MINIMAL)
-			row.writerow([date, test_name, size, iteration_max, error,
-					request_min, request_max, request_avg, request_jitter,
-					latency_min, latency_max, latency_avg, latency_jitter,
-					throughput_min, throughput_max, throughput_avg, throughput_jitter])
-	except IOError as e:
-		print("I/O error({0}): {1}".format(e.errno, e.strerror))
-
-def loopback_run(test_name, size, iteration_max, sys_pfx):
-	test_id = dict[test_name]
-	try:
-		# Terminate any currently running test
-		write_sysfs_val(sys_pfx + 'type', '0')
-		# Set parameter for no wait between messages
-		write_sysfs_val(sys_pfx + 'ms_wait', '0')
-		# Set operation size
-		write_sysfs_val(sys_pfx + 'size', size)
-		# Set iterations
-		write_sysfs_val(sys_pfx + 'iteration_max', str(iteration_max))
-		# Initiate by setting loopback operation type
-		write_sysfs_val(sys_pfx + 'type', test_id)
-		time.sleep(1)
-
-		if iteration_max == 0:
-			print ("Infinite test initiated CSV won't be logged\n")
-			return
-
-		previous = 0
-		err = 0
-		while True:
-			# get current count bail out if it hasn't changed
-			iteration_count = read_sysfs_int(sys_pfx + 'iteration_count')
-			if previous == iteration_count:
-				err = 1
-				break
-			elif iteration_count == iteration_max:
-				break
-			previous = iteration_count
-			if verbose:
-				print('%02d%% complete %d of %d ' %
-					(100 * iteration_count / iteration_max,
-					iteration_count, iteration_max))
-			time.sleep(1)
-		if err:
-			print ('\nError executing test\n')
-		else:
-			log_csv(test_name, size, iteration_max, sys_pfx)
-	except ValueError as ve:
-		print("Error: %s " % format(e.strerror), file=sys.stderr)
-		abort()
-
-def main():
-	if len(sys.argv) < 5:
-		usage()
-
-	if sys.argv[1] in dict.keys():
-		loopback_run(sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4])
-	else:
-		usage()
-if __name__ == '__main__':
-	main()
diff --git a/drivers/staging/greybus/tools/loopback_test.c b/drivers/staging/greybus/tools/loopback_test.c
deleted file mode 100644
index d7ad51ff60c5..000000000000
--- a/drivers/staging/greybus/tools/loopback_test.c
+++ /dev/null
@@ -1,979 +0,0 @@
-// SPDX-License-Identifier: BSD-3-Clause
-/*
- * Loopback test application
- *
- * Copyright 2015 Google Inc.
- * Copyright 2015 Linaro Ltd.
- */
-#include <errno.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <poll.h>
-#include <sys/types.h>
-#include <time.h>
-#include <unistd.h>
-#include <dirent.h>
-#include <signal.h>
-
-#define MAX_NUM_DEVICES 10
-#define MAX_SYSFS_PREFIX 0x80
-#define MAX_SYSFS_PATH	0x200
-#define CSV_MAX_LINE	0x1000
-#define SYSFS_MAX_INT	0x20
-#define MAX_STR_LEN	255
-#define DEFAULT_ASYNC_TIMEOUT 200000
-
-struct dict {
-	char *name;
-	int type;
-};
-
-static struct dict dict[] = {
-	{"ping", 2},
-	{"transfer", 3},
-	{"sink", 4},
-	{NULL,}		/* list termination */
-};
-
-struct loopback_results {
-	float latency_avg;
-	uint32_t latency_max;
-	uint32_t latency_min;
-	uint32_t latency_jitter;
-
-	float request_avg;
-	uint32_t request_max;
-	uint32_t request_min;
-	uint32_t request_jitter;
-
-	float throughput_avg;
-	uint32_t throughput_max;
-	uint32_t throughput_min;
-	uint32_t throughput_jitter;
-
-	float apbridge_unipro_latency_avg;
-	uint32_t apbridge_unipro_latency_max;
-	uint32_t apbridge_unipro_latency_min;
-	uint32_t apbridge_unipro_latency_jitter;
-
-	float gbphy_firmware_latency_avg;
-	uint32_t gbphy_firmware_latency_max;
-	uint32_t gbphy_firmware_latency_min;
-	uint32_t gbphy_firmware_latency_jitter;
-
-	uint32_t error;
-};
-
-struct loopback_device {
-	char name[MAX_STR_LEN];
-	char sysfs_entry[MAX_SYSFS_PATH];
-	char debugfs_entry[MAX_SYSFS_PATH];
-	struct loopback_results results;
-};
-
-struct loopback_test {
-	int verbose;
-	int debug;
-	int raw_data_dump;
-	int porcelain;
-	int mask;
-	int size;
-	int iteration_max;
-	int aggregate_output;
-	int test_id;
-	int device_count;
-	int list_devices;
-	int use_async;
-	int async_timeout;
-	int async_outstanding_operations;
-	int us_wait;
-	int file_output;
-	int stop_all;
-	int poll_count;
-	char test_name[MAX_STR_LEN];
-	char sysfs_prefix[MAX_SYSFS_PREFIX];
-	char debugfs_prefix[MAX_SYSFS_PREFIX];
-	struct timespec poll_timeout;
-	struct loopback_device devices[MAX_NUM_DEVICES];
-	struct loopback_results aggregate_results;
-	struct pollfd fds[MAX_NUM_DEVICES];
-};
-
-struct loopback_test t;
-
-/* Helper macros to calculate the aggregate results for all devices */
-static inline int device_enabled(struct loopback_test *t, int dev_idx);
-
-#define GET_MAX(field)							\
-static int get_##field##_aggregate(struct loopback_test *t)		\
-{									\
-	uint32_t max = 0;						\
-	int i;								\
-	for (i = 0; i < t->device_count; i++) {				\
-		if (!device_enabled(t, i))				\
-			continue;					\
-		if (t->devices[i].results.field > max)			\
-			max = t->devices[i].results.field;		\
-	}								\
-	return max;							\
-}									\
-
-#define GET_MIN(field)							\
-static int get_##field##_aggregate(struct loopback_test *t)		\
-{									\
-	uint32_t min = ~0;						\
-	int i;								\
-	for (i = 0; i < t->device_count; i++) {				\
-		if (!device_enabled(t, i))				\
-			continue;					\
-		if (t->devices[i].results.field < min)			\
-			min = t->devices[i].results.field;		\
-	}								\
-	return min;							\
-}									\
-
-#define GET_AVG(field)							\
-static int get_##field##_aggregate(struct loopback_test *t)		\
-{									\
-	uint32_t val = 0;						\
-	uint32_t count = 0;						\
-	int i;								\
-	for (i = 0; i < t->device_count; i++) {				\
-		if (!device_enabled(t, i))				\
-			continue;					\
-		count++;						\
-		val += t->devices[i].results.field;			\
-	}								\
-	if (count)							\
-		val /= count;						\
-	return val;							\
-}									\
-
-GET_MAX(throughput_max);
-GET_MAX(request_max);
-GET_MAX(latency_max);
-GET_MAX(apbridge_unipro_latency_max);
-GET_MAX(gbphy_firmware_latency_max);
-GET_MIN(throughput_min);
-GET_MIN(request_min);
-GET_MIN(latency_min);
-GET_MIN(apbridge_unipro_latency_min);
-GET_MIN(gbphy_firmware_latency_min);
-GET_AVG(throughput_avg);
-GET_AVG(request_avg);
-GET_AVG(latency_avg);
-GET_AVG(apbridge_unipro_latency_avg);
-GET_AVG(gbphy_firmware_latency_avg);
-
-void abort(void)
-{
-	_exit(1);
-}
-
-void usage(void)
-{
-	fprintf(stderr, "Usage: loopback_test TEST [SIZE] ITERATIONS [SYSPATH] [DBGPATH]\n\n"
-	"  Run TEST for a number of ITERATIONS with operation data SIZE bytes\n"
-	"  TEST may be \'ping\' \'transfer\' or \'sink\'\n"
-	"  SIZE indicates the size of transfer <= greybus max payload bytes\n"
-	"  ITERATIONS indicates the number of times to execute TEST at SIZE bytes\n"
-	"             Note if ITERATIONS is set to zero then this utility will\n"
-	"             initiate an infinite (non terminating) test and exit\n"
-	"             without logging any metrics data\n"
-	"  SYSPATH indicates the sysfs path for the loopback greybus entries e.g.\n"
-	"          /sys/bus/greybus/devices\n"
-	"  DBGPATH indicates the debugfs path for the loopback greybus entries e.g.\n"
-	"          /sys/kernel/debug/gb_loopback/\n"
-	" Mandatory arguments\n"
-	"   -t     must be one of the test names - sink, transfer or ping\n"
-	"   -i     iteration count - the number of iterations to run the test over\n"
-	" Optional arguments\n"
-	"   -S     sysfs location - location for greybus 'endo' entries default /sys/bus/greybus/devices/\n"
-	"   -D     debugfs location - location for loopback debugfs entries default /sys/kernel/debug/gb_loopback/\n"
-	"   -s     size of data packet to send during test - defaults to zero\n"
-	"   -m     mask - a bit mask of connections to include example: -m 8 = 4th connection -m 9 = 1st and 4th connection etc\n"
-	"                 default is zero which means broadcast to all connections\n"
-	"   -v     verbose output\n"
-	"   -d     debug output\n"
-	"   -r     raw data output - when specified the full list of latency values are included in the output CSV\n"
-	"   -p     porcelain - when specified printout is in a user-friendly non-CSV format. This option suppresses writing to CSV file\n"
-	"   -a     aggregate - show aggregation of all enabled devices\n"
-	"   -l     list found loopback devices and exit\n"
-	"   -x     Async - Enable async transfers\n"
-	"   -o     Async Timeout - Timeout in uSec for async operations\n"
-	"   -O     Poll loop time out in seconds(max time a test is expected to last, default: 30sec)\n"
-	"   -c     Max number of outstanding operations for async operations\n"
-	"   -w     Wait in uSec between operations\n"
-	"   -z     Enable output to a CSV file (incompatible with -p)\n"
-	"   -f     When starting new loopback test, stop currently running tests on all devices\n"
-	"Examples:\n"
-	"  Send 10000 transfers with a packet size of 128 bytes to all active connections\n"
-	"  loopback_test -t transfer -s 128 -i 10000 -S /sys/bus/greybus/devices/ -D /sys/kernel/debug/gb_loopback/\n"
-	"  loopback_test -t transfer -s 128 -i 10000 -m 0\n"
-	"  Send 10000 transfers with a packet size of 128 bytes to connection 1 and 4\n"
-	"  loopback_test -t transfer -s 128 -i 10000 -m 9\n"
-	"  loopback_test -t ping -s 0 128 -i -S /sys/bus/greybus/devices/ -D /sys/kernel/debug/gb_loopback/\n"
-	"  loopback_test -t sink -s 2030 -i 32768 -S /sys/bus/greybus/devices/ -D /sys/kernel/debug/gb_loopback/\n");
-	abort();
-}
-
-static inline int device_enabled(struct loopback_test *t, int dev_idx)
-{
-	if (!t->mask || (t->mask & (1 << dev_idx)))
-		return 1;
-
-	return 0;
-}
-
-static void show_loopback_devices(struct loopback_test *t)
-{
-	int i;
-
-	if (t->device_count == 0) {
-		printf("No loopback devices.\n");
-		return;
-	}
-
-	for (i = 0; i < t->device_count; i++)
-		printf("device[%d] = %s\n", i, t->devices[i].name);
-}
-
-int open_sysfs(const char *sys_pfx, const char *node, int flags)
-{
-	int fd;
-	char path[MAX_SYSFS_PATH];
-
-	snprintf(path, sizeof(path), "%s%s", sys_pfx, node);
-	fd = open(path, flags);
-	if (fd < 0) {
-		fprintf(stderr, "unable to open %s\n", path);
-		abort();
-	}
-	return fd;
-}
-
-int read_sysfs_int_fd(int fd, const char *sys_pfx, const char *node)
-{
-	char buf[SYSFS_MAX_INT];
-
-	if (read(fd, buf, sizeof(buf)) < 0) {
-		fprintf(stderr, "unable to read from %s%s %s\n", sys_pfx, node,
-			strerror(errno));
-		close(fd);
-		abort();
-	}
-	return atoi(buf);
-}
-
-float read_sysfs_float_fd(int fd, const char *sys_pfx, const char *node)
-{
-	char buf[SYSFS_MAX_INT];
-
-	if (read(fd, buf, sizeof(buf)) < 0) {
-		fprintf(stderr, "unable to read from %s%s %s\n", sys_pfx, node,
-			strerror(errno));
-		close(fd);
-		abort();
-	}
-	return atof(buf);
-}
-
-int read_sysfs_int(const char *sys_pfx, const char *node)
-{
-	int fd, val;
-
-	fd = open_sysfs(sys_pfx, node, O_RDONLY);
-	val = read_sysfs_int_fd(fd, sys_pfx, node);
-	close(fd);
-	return val;
-}
-
-float read_sysfs_float(const char *sys_pfx, const char *node)
-{
-	int fd;
-	float val;
-
-	fd = open_sysfs(sys_pfx, node, O_RDONLY);
-	val = read_sysfs_float_fd(fd, sys_pfx, node);
-	close(fd);
-	return val;
-}
-
-void write_sysfs_val(const char *sys_pfx, const char *node, int val)
-{
-	int fd, len;
-	char buf[SYSFS_MAX_INT];
-
-	fd = open_sysfs(sys_pfx, node, O_RDWR);
-	len = snprintf(buf, sizeof(buf), "%d", val);
-	if (write(fd, buf, len) < 0) {
-		fprintf(stderr, "unable to write to %s%s %s\n", sys_pfx, node,
-			strerror(errno));
-		close(fd);
-		abort();
-	}
-	close(fd);
-}
-
-static int get_results(struct loopback_test *t)
-{
-	struct loopback_device *d;
-	struct loopback_results *r;
-	int i;
-
-	for (i = 0; i < t->device_count; i++) {
-		if (!device_enabled(t, i))
-			continue;
-
-		d = &t->devices[i];
-		r = &d->results;
-
-		r->error = read_sysfs_int(d->sysfs_entry, "error");
-		r->request_min = read_sysfs_int(d->sysfs_entry, "requests_per_second_min");
-		r->request_max = read_sysfs_int(d->sysfs_entry, "requests_per_second_max");
-		r->request_avg = read_sysfs_float(d->sysfs_entry, "requests_per_second_avg");
-
-		r->latency_min = read_sysfs_int(d->sysfs_entry, "latency_min");
-		r->latency_max = read_sysfs_int(d->sysfs_entry, "latency_max");
-		r->latency_avg = read_sysfs_float(d->sysfs_entry, "latency_avg");
-
-		r->throughput_min = read_sysfs_int(d->sysfs_entry, "throughput_min");
-		r->throughput_max = read_sysfs_int(d->sysfs_entry, "throughput_max");
-		r->throughput_avg = read_sysfs_float(d->sysfs_entry, "throughput_avg");
-
-		r->apbridge_unipro_latency_min =
-			read_sysfs_int(d->sysfs_entry, "apbridge_unipro_latency_min");
-		r->apbridge_unipro_latency_max =
-			read_sysfs_int(d->sysfs_entry, "apbridge_unipro_latency_max");
-		r->apbridge_unipro_latency_avg =
-			read_sysfs_float(d->sysfs_entry, "apbridge_unipro_latency_avg");
-
-		r->gbphy_firmware_latency_min =
-			read_sysfs_int(d->sysfs_entry, "gbphy_firmware_latency_min");
-		r->gbphy_firmware_latency_max =
-			read_sysfs_int(d->sysfs_entry, "gbphy_firmware_latency_max");
-		r->gbphy_firmware_latency_avg =
-			read_sysfs_float(d->sysfs_entry, "gbphy_firmware_latency_avg");
-
-		r->request_jitter = r->request_max - r->request_min;
-		r->latency_jitter = r->latency_max - r->latency_min;
-		r->throughput_jitter = r->throughput_max - r->throughput_min;
-		r->apbridge_unipro_latency_jitter =
-			r->apbridge_unipro_latency_max - r->apbridge_unipro_latency_min;
-		r->gbphy_firmware_latency_jitter =
-			r->gbphy_firmware_latency_max - r->gbphy_firmware_latency_min;
-	}
-
-	/*calculate the aggregate results of all enabled devices */
-	if (t->aggregate_output) {
-		r = &t->aggregate_results;
-
-		r->request_min = get_request_min_aggregate(t);
-		r->request_max = get_request_max_aggregate(t);
-		r->request_avg = get_request_avg_aggregate(t);
-
-		r->latency_min = get_latency_min_aggregate(t);
-		r->latency_max = get_latency_max_aggregate(t);
-		r->latency_avg = get_latency_avg_aggregate(t);
-
-		r->throughput_min = get_throughput_min_aggregate(t);
-		r->throughput_max = get_throughput_max_aggregate(t);
-		r->throughput_avg = get_throughput_avg_aggregate(t);
-
-		r->apbridge_unipro_latency_min =
-			get_apbridge_unipro_latency_min_aggregate(t);
-		r->apbridge_unipro_latency_max =
-			get_apbridge_unipro_latency_max_aggregate(t);
-		r->apbridge_unipro_latency_avg =
-			get_apbridge_unipro_latency_avg_aggregate(t);
-
-		r->gbphy_firmware_latency_min =
-			get_gbphy_firmware_latency_min_aggregate(t);
-		r->gbphy_firmware_latency_max =
-			get_gbphy_firmware_latency_max_aggregate(t);
-		r->gbphy_firmware_latency_avg =
-			get_gbphy_firmware_latency_avg_aggregate(t);
-
-		r->request_jitter = r->request_max - r->request_min;
-		r->latency_jitter = r->latency_max - r->latency_min;
-		r->throughput_jitter = r->throughput_max - r->throughput_min;
-		r->apbridge_unipro_latency_jitter =
-			r->apbridge_unipro_latency_max - r->apbridge_unipro_latency_min;
-		r->gbphy_firmware_latency_jitter =
-			r->gbphy_firmware_latency_max - r->gbphy_firmware_latency_min;
-	}
-
-	return 0;
-}
-
-int format_output(struct loopback_test *t,
-		  struct loopback_results *r,
-		  const char *dev_name,
-		  char *buf, int buf_len,
-		  struct tm *tm)
-{
-	int len = 0;
-
-	memset(buf, 0x00, buf_len);
-	len = snprintf(buf, buf_len, "%u-%u-%u %u:%u:%u",
-		       tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday,
-		       tm->tm_hour, tm->tm_min, tm->tm_sec);
-
-	if (t->porcelain) {
-		len += snprintf(&buf[len], buf_len - len,
-			"\n test:\t\t\t%s\n path:\t\t\t%s\n size:\t\t\t%u\n iterations:\t\t%u\n errors:\t\t%u\n async:\t\t\t%s\n",
-			t->test_name,
-			dev_name,
-			t->size,
-			t->iteration_max,
-			r->error,
-			t->use_async ? "Enabled" : "Disabled");
-
-		len += snprintf(&buf[len], buf_len - len,
-			" requests per-sec:\tmin=%u, max=%u, average=%f, jitter=%u\n",
-			r->request_min,
-			r->request_max,
-			r->request_avg,
-			r->request_jitter);
-
-		len += snprintf(&buf[len], buf_len - len,
-			" ap-throughput B/s:\tmin=%u max=%u average=%f jitter=%u\n",
-			r->throughput_min,
-			r->throughput_max,
-			r->throughput_avg,
-			r->throughput_jitter);
-		len += snprintf(&buf[len], buf_len - len,
-			" ap-latency usec:\tmin=%u max=%u average=%f jitter=%u\n",
-			r->latency_min,
-			r->latency_max,
-			r->latency_avg,
-			r->latency_jitter);
-		len += snprintf(&buf[len], buf_len - len,
-			" apbridge-latency usec:\tmin=%u max=%u average=%f jitter=%u\n",
-			r->apbridge_unipro_latency_min,
-			r->apbridge_unipro_latency_max,
-			r->apbridge_unipro_latency_avg,
-			r->apbridge_unipro_latency_jitter);
-
-		len += snprintf(&buf[len], buf_len - len,
-			" gbphy-latency usec:\tmin=%u max=%u average=%f jitter=%u\n",
-			r->gbphy_firmware_latency_min,
-			r->gbphy_firmware_latency_max,
-			r->gbphy_firmware_latency_avg,
-			r->gbphy_firmware_latency_jitter);
-
-	} else {
-		len += snprintf(&buf[len], buf_len - len, ",%s,%s,%u,%u,%u",
-			t->test_name, dev_name, t->size, t->iteration_max,
-			r->error);
-
-		len += snprintf(&buf[len], buf_len - len, ",%u,%u,%f,%u",
-			r->request_min,
-			r->request_max,
-			r->request_avg,
-			r->request_jitter);
-
-		len += snprintf(&buf[len], buf_len - len, ",%u,%u,%f,%u",
-			r->latency_min,
-			r->latency_max,
-			r->latency_avg,
-			r->latency_jitter);
-
-		len += snprintf(&buf[len], buf_len - len, ",%u,%u,%f,%u",
-			r->throughput_min,
-			r->throughput_max,
-			r->throughput_avg,
-			r->throughput_jitter);
-
-		len += snprintf(&buf[len], buf_len - len, ",%u,%u,%f,%u",
-			r->apbridge_unipro_latency_min,
-			r->apbridge_unipro_latency_max,
-			r->apbridge_unipro_latency_avg,
-			r->apbridge_unipro_latency_jitter);
-
-		len += snprintf(&buf[len], buf_len - len, ",%u,%u,%f,%u",
-			r->gbphy_firmware_latency_min,
-			r->gbphy_firmware_latency_max,
-			r->gbphy_firmware_latency_avg,
-			r->gbphy_firmware_latency_jitter);
-	}
-
-	printf("\n%s\n", buf);
-
-	return len;
-}
-
-static int log_results(struct loopback_test *t)
-{
-	int fd, i, len, ret;
-	struct tm tm;
-	time_t local_time;
-	char file_name[MAX_SYSFS_PATH];
-	char data[CSV_MAX_LINE];
-
-	local_time = time(NULL);
-	tm = *localtime(&local_time);
-
-	/*
-	 * file name will test_name_size_iteration_max.csv
-	 * every time the same test with the same parameters is run we will then
-	 * append to the same CSV with datestamp - representing each test
-	 * dataset.
-	 */
-	if (t->file_output && !t->porcelain) {
-		snprintf(file_name, sizeof(file_name), "%s_%d_%d.csv",
-			 t->test_name, t->size, t->iteration_max);
-
-		fd = open(file_name, O_WRONLY | O_CREAT | O_APPEND, 0644);
-		if (fd < 0) {
-			fprintf(stderr, "unable to open %s for appending\n", file_name);
-			abort();
-		}
-	}
-	for (i = 0; i < t->device_count; i++) {
-		if (!device_enabled(t, i))
-			continue;
-
-		len = format_output(t, &t->devices[i].results,
-				    t->devices[i].name,
-				    data, sizeof(data), &tm);
-		if (t->file_output && !t->porcelain) {
-			ret = write(fd, data, len);
-			if (ret == -1)
-				fprintf(stderr, "unable to write %d bytes to csv.\n", len);
-		}
-	}
-
-	if (t->aggregate_output) {
-		len = format_output(t, &t->aggregate_results, "aggregate",
-				    data, sizeof(data), &tm);
-		if (t->file_output && !t->porcelain) {
-			ret = write(fd, data, len);
-			if (ret == -1)
-				fprintf(stderr, "unable to write %d bytes to csv.\n", len);
-		}
-	}
-
-	if (t->file_output && !t->porcelain)
-		close(fd);
-
-	return 0;
-}
-
-int is_loopback_device(const char *path, const char *node)
-{
-	char file[MAX_SYSFS_PATH];
-
-	snprintf(file, MAX_SYSFS_PATH, "%s%s/iteration_count", path, node);
-	if (access(file, F_OK) == 0)
-		return 1;
-	return 0;
-}
-
-int find_loopback_devices(struct loopback_test *t)
-{
-	struct dirent **namelist;
-	int i, n, ret;
-	unsigned int dev_id;
-	struct loopback_device *d;
-
-	n = scandir(t->sysfs_prefix, &namelist, NULL, alphasort);
-	if (n < 0) {
-		perror("scandir");
-		ret = -ENODEV;
-		goto baddir;
-	}
-
-	/* Don't include '.' and '..' */
-	if (n <= 2) {
-		ret = -ENOMEM;
-		goto done;
-	}
-
-	for (i = 0; i < n; i++) {
-		ret = sscanf(namelist[i]->d_name, "gb_loopback%u", &dev_id);
-		if (ret != 1)
-			continue;
-
-		if (!is_loopback_device(t->sysfs_prefix, namelist[i]->d_name))
-			continue;
-
-		if (t->device_count == MAX_NUM_DEVICES) {
-			fprintf(stderr, "max number of devices reached!\n");
-			break;
-		}
-
-		d = &t->devices[t->device_count++];
-		snprintf(d->name, MAX_STR_LEN, "gb_loopback%u", dev_id);
-
-		snprintf(d->sysfs_entry, MAX_SYSFS_PATH, "%s%s/",
-			 t->sysfs_prefix, d->name);
-
-		snprintf(d->debugfs_entry, MAX_SYSFS_PATH, "%sraw_latency_%s",
-			 t->debugfs_prefix, d->name);
-
-		if (t->debug)
-			printf("add %s %s\n", d->sysfs_entry, d->debugfs_entry);
-	}
-
-	ret = 0;
-done:
-	for (i = 0; i < n; i++)
-		free(namelist[i]);
-	free(namelist);
-baddir:
-	return ret;
-}
-
-static int open_poll_files(struct loopback_test *t)
-{
-	struct loopback_device *dev;
-	char buf[MAX_SYSFS_PATH + MAX_STR_LEN];
-	char dummy;
-	int fds_idx = 0;
-	int i;
-
-	for (i = 0; i < t->device_count; i++) {
-		dev = &t->devices[i];
-
-		if (!device_enabled(t, i))
-			continue;
-
-		snprintf(buf, sizeof(buf), "%s%s", dev->sysfs_entry, "iteration_count");
-		t->fds[fds_idx].fd = open(buf, O_RDONLY);
-		if (t->fds[fds_idx].fd < 0) {
-			fprintf(stderr, "Error opening poll file!\n");
-			goto err;
-		}
-		read(t->fds[fds_idx].fd, &dummy, 1);
-		t->fds[fds_idx].events = POLLERR | POLLPRI;
-		t->fds[fds_idx].revents = 0;
-		fds_idx++;
-	}
-
-	t->poll_count = fds_idx;
-
-	return 0;
-
-err:
-	for (i = 0; i < fds_idx; i++)
-		close(t->fds[i].fd);
-
-	return -1;
-}
-
-static int close_poll_files(struct loopback_test *t)
-{
-	int i;
-
-	for (i = 0; i < t->poll_count; i++)
-		close(t->fds[i].fd);
-
-	return 0;
-}
-static int is_complete(struct loopback_test *t)
-{
-	int iteration_count;
-	int i;
-
-	for (i = 0; i < t->device_count; i++) {
-		if (!device_enabled(t, i))
-			continue;
-
-		iteration_count = read_sysfs_int(t->devices[i].sysfs_entry,
-						 "iteration_count");
-
-		/* at least one device did not finish yet */
-		if (iteration_count != t->iteration_max)
-			return 0;
-	}
-
-	return 1;
-}
-
-static void stop_tests(struct loopback_test *t)
-{
-	int i;
-
-	for (i = 0; i < t->device_count; i++) {
-		if (!device_enabled(t, i))
-			continue;
-		write_sysfs_val(t->devices[i].sysfs_entry, "type", 0);
-	}
-}
-
-static void handler(int sig) { /* do nothing */  }
-
-static int wait_for_complete(struct loopback_test *t)
-{
-	int number_of_events = 0;
-	char dummy;
-	int ret;
-	int i;
-	struct timespec *ts = NULL;
-	struct sigaction sa;
-	sigset_t mask_old, mask;
-
-	sigemptyset(&mask);
-	sigemptyset(&mask_old);
-	sigaddset(&mask, SIGINT);
-	sigprocmask(SIG_BLOCK, &mask, &mask_old);
-
-	sa.sa_handler = handler;
-	sa.sa_flags = 0;
-	sigemptyset(&sa.sa_mask);
-	if (sigaction(SIGINT, &sa, NULL) == -1) {
-		fprintf(stderr, "sigaction error\n");
-		return -1;
-	}
-
-	if (t->poll_timeout.tv_sec != 0)
-		ts = &t->poll_timeout;
-
-	while (1) {
-		ret = ppoll(t->fds, t->poll_count, ts, &mask_old);
-		if (ret <= 0) {
-			stop_tests(t);
-			fprintf(stderr, "Poll exit with errno %d\n", errno);
-			return -1;
-		}
-
-		for (i = 0; i < t->poll_count; i++) {
-			if (t->fds[i].revents & POLLPRI) {
-				/* Dummy read to clear the event */
-				read(t->fds[i].fd, &dummy, 1);
-				number_of_events++;
-			}
-		}
-
-		if (number_of_events == t->poll_count)
-			break;
-	}
-
-	if (!is_complete(t)) {
-		fprintf(stderr, "Iteration count did not finish!\n");
-		return -1;
-	}
-
-	return 0;
-}
-
-static void prepare_devices(struct loopback_test *t)
-{
-	int i;
-
-	/*
-	 * Cancel any running tests on enabled devices. If
-	 * stop_all option is given, stop test on all devices.
-	 */
-	for (i = 0; i < t->device_count; i++)
-		if (t->stop_all || device_enabled(t, i))
-			write_sysfs_val(t->devices[i].sysfs_entry, "type", 0);
-
-	for (i = 0; i < t->device_count; i++) {
-		if (!device_enabled(t, i))
-			continue;
-
-		write_sysfs_val(t->devices[i].sysfs_entry, "us_wait",
-				t->us_wait);
-
-		/* Set operation size */
-		write_sysfs_val(t->devices[i].sysfs_entry, "size", t->size);
-
-		/* Set iterations */
-		write_sysfs_val(t->devices[i].sysfs_entry, "iteration_max",
-				t->iteration_max);
-
-		if (t->use_async) {
-			write_sysfs_val(t->devices[i].sysfs_entry, "async", 1);
-			write_sysfs_val(t->devices[i].sysfs_entry,
-					"timeout", t->async_timeout);
-			write_sysfs_val(t->devices[i].sysfs_entry,
-					"outstanding_operations_max",
-					t->async_outstanding_operations);
-		} else {
-			write_sysfs_val(t->devices[i].sysfs_entry, "async", 0);
-		}
-	}
-}
-
-static int start(struct loopback_test *t)
-{
-	int i;
-
-	/* the test starts by writing test_id to the type file. */
-	for (i = 0; i < t->device_count; i++) {
-		if (!device_enabled(t, i))
-			continue;
-
-		write_sysfs_val(t->devices[i].sysfs_entry, "type", t->test_id);
-	}
-
-	return 0;
-}
-
-void loopback_run(struct loopback_test *t)
-{
-	int i;
-	int ret;
-
-	for (i = 0; dict[i].name != NULL; i++) {
-		if (strstr(dict[i].name, t->test_name))
-			t->test_id = dict[i].type;
-	}
-	if (!t->test_id) {
-		fprintf(stderr, "invalid test %s\n", t->test_name);
-		usage();
-		return;
-	}
-
-	prepare_devices(t);
-
-	ret = open_poll_files(t);
-	if (ret)
-		goto err;
-
-	start(t);
-
-	ret = wait_for_complete(t);
-	close_poll_files(t);
-	if (ret)
-		goto err;
-
-	get_results(t);
-
-	log_results(t);
-
-	return;
-
-err:
-	printf("Error running test\n");
-}
-
-static int sanity_check(struct loopback_test *t)
-{
-	int i;
-
-	if (t->device_count == 0) {
-		fprintf(stderr, "No loopback devices found\n");
-		return -1;
-	}
-
-	for (i = 0; i < MAX_NUM_DEVICES; i++) {
-		if (!device_enabled(t, i))
-			continue;
-
-		if (t->mask && !strcmp(t->devices[i].name, "")) {
-			fprintf(stderr, "Bad device mask %x\n", (1 << i));
-			return -1;
-		}
-	}
-
-	return 0;
-}
-
-int main(int argc, char *argv[])
-{
-	int o, ret;
-	char *sysfs_prefix = "/sys/class/gb_loopback/";
-	char *debugfs_prefix = "/sys/kernel/debug/gb_loopback/";
-
-	memset(&t, 0, sizeof(t));
-
-	while ((o = getopt(argc, argv,
-			   "t:s:i:S:D:m:v::d::r::p::a::l::x::o:O:c:w:z::f::")) != -1) {
-		switch (o) {
-		case 't':
-			snprintf(t.test_name, MAX_STR_LEN, "%s", optarg);
-			break;
-		case 's':
-			t.size = atoi(optarg);
-			break;
-		case 'i':
-			t.iteration_max = atoi(optarg);
-			break;
-		case 'S':
-			snprintf(t.sysfs_prefix, MAX_SYSFS_PREFIX, "%s", optarg);
-			break;
-		case 'D':
-			snprintf(t.debugfs_prefix, MAX_SYSFS_PREFIX, "%s", optarg);
-			break;
-		case 'm':
-			t.mask = atol(optarg);
-			break;
-		case 'v':
-			t.verbose = 1;
-			break;
-		case 'd':
-			t.debug = 1;
-			break;
-		case 'r':
-			t.raw_data_dump = 1;
-			break;
-		case 'p':
-			t.porcelain = 1;
-			break;
-		case 'a':
-			t.aggregate_output = 1;
-			break;
-		case 'l':
-			t.list_devices = 1;
-			break;
-		case 'x':
-			t.use_async = 1;
-			break;
-		case 'o':
-			t.async_timeout = atoi(optarg);
-			break;
-		case 'O':
-			t.poll_timeout.tv_sec = atoi(optarg);
-			break;
-		case 'c':
-			t.async_outstanding_operations = atoi(optarg);
-			break;
-		case 'w':
-			t.us_wait = atoi(optarg);
-			break;
-		case 'z':
-			t.file_output = 1;
-			break;
-		case 'f':
-			t.stop_all = 1;
-			break;
-		default:
-			usage();
-			return -EINVAL;
-		}
-	}
-
-	if (!strcmp(t.sysfs_prefix, ""))
-		snprintf(t.sysfs_prefix, MAX_SYSFS_PREFIX, "%s", sysfs_prefix);
-
-	if (!strcmp(t.debugfs_prefix, ""))
-		snprintf(t.debugfs_prefix, MAX_SYSFS_PREFIX, "%s", debugfs_prefix);
-
-	ret = find_loopback_devices(&t);
-	if (ret)
-		return ret;
-	ret = sanity_check(&t);
-	if (ret)
-		return ret;
-
-	if (t.list_devices) {
-		show_loopback_devices(&t);
-		return 0;
-	}
-
-	if (t.test_name[0] == '\0' || t.iteration_max == 0)
-		usage();
-
-	if (t.async_timeout == 0)
-		t.async_timeout = DEFAULT_ASYNC_TIMEOUT;
-
-	loopback_run(&t);
-
-	return 0;
-}
diff --git a/drivers/staging/greybus/vibrator.c b/drivers/staging/greybus/vibrator.c
index 0e2b188e5ca3..227e18d92a95 100644
--- a/drivers/staging/greybus/vibrator.c
+++ b/drivers/staging/greybus/vibrator.c
@@ -107,7 +107,6 @@ ATTRIBUTE_GROUPS(vibrator);
 
 static struct class vibrator_class = {
 	.name		= "vibrator",
-	.owner		= THIS_MODULE,
 	.dev_groups	= vibrator_groups,
 };
 
diff --git a/drivers/staging/iio/Kconfig b/drivers/staging/iio/Kconfig
index afd05bf3345e..d3968fe2ebb8 100644
--- a/drivers/staging/iio/Kconfig
+++ b/drivers/staging/iio/Kconfig
@@ -10,7 +10,6 @@ source "drivers/staging/iio/adc/Kconfig"
 source "drivers/staging/iio/addac/Kconfig"
 source "drivers/staging/iio/frequency/Kconfig"
 source "drivers/staging/iio/impedance-analyzer/Kconfig"
-source "drivers/staging/iio/meter/Kconfig"
 source "drivers/staging/iio/resolver/Kconfig"
 
 endmenu
diff --git a/drivers/staging/iio/Makefile b/drivers/staging/iio/Makefile
index 5ed56fe57e14..c50f1019f829 100644
--- a/drivers/staging/iio/Makefile
+++ b/drivers/staging/iio/Makefile
@@ -8,5 +8,4 @@ obj-y += adc/
 obj-y += addac/
 obj-y += frequency/
 obj-y += impedance-analyzer/
-obj-y += meter/
 obj-y += resolver/
diff --git a/drivers/staging/iio/meter/Kconfig b/drivers/staging/iio/meter/Kconfig
deleted file mode 100644
index aa6a3e7f6cdb..000000000000
--- a/drivers/staging/iio/meter/Kconfig
+++ /dev/null
@@ -1,37 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# IIO meter drivers configuration
-#
-menu "Active energy metering IC"
-
-config ADE7854
-	tristate "Analog Devices ADE7854/58/68/78 Polyphase Multifunction Energy Metering IC Driver"
-	depends on SPI || I2C
-	help
-	  Say yes here to build support for Analog Devices ADE7854/58/68/78 Polyphase
-	  Multifunction Energy Metering IC Driver.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ade7854.
-
-config ADE7854_I2C
-	tristate "support I2C bus connection"
-	depends on ADE7854 && I2C
-	default y
-	help
-	  Say Y here if you have ADE7854/58/68/78 hooked to an I2C bus.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ade7854-i2c.
-
-config ADE7854_SPI
-	tristate "support SPI bus connection"
-	depends on ADE7854 && SPI
-	default y
-	help
-	  Say Y here if you have ADE7854/58/68/78 hooked to a SPI bus.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ade7854-spi.
-
-endmenu
diff --git a/drivers/staging/iio/meter/Makefile b/drivers/staging/iio/meter/Makefile
deleted file mode 100644
index ed4547e38f3a..000000000000
--- a/drivers/staging/iio/meter/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Makefile for metering ic drivers
-#
-
-obj-$(CONFIG_ADE7854) += ade7854.o
-obj-$(CONFIG_ADE7854_I2C) += ade7854-i2c.o
-obj-$(CONFIG_ADE7854_SPI) += ade7854-spi.o
diff --git a/drivers/staging/iio/meter/ade7854-i2c.c b/drivers/staging/iio/meter/ade7854-i2c.c
deleted file mode 100644
index 572d714eb0dd..000000000000
--- a/drivers/staging/iio/meter/ade7854-i2c.c
+++ /dev/null
@@ -1,153 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * ADE7854/58/68/78 Polyphase Multifunction Energy Metering IC Driver (I2C Bus)
- *
- * Copyright 2010 Analog Devices Inc.
- */
-
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/i2c.h>
-#include <linux/slab.h>
-#include <linux/module.h>
-
-#include <linux/iio/iio.h>
-#include "ade7854.h"
-
-static int ade7854_i2c_write_reg(struct device *dev,
-				 u16 reg_address,
-				 u32 val,
-				 int bits)
-{
-	int ret;
-	int count;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = (reg_address >> 8) & 0xFF;
-	st->tx[1] = reg_address & 0xFF;
-
-	switch (bits) {
-	case 8:
-		st->tx[2] = val & 0xFF;
-		count = 3;
-		break;
-	case 16:
-		st->tx[2] = (val >> 8) & 0xFF;
-		st->tx[3] = val & 0xFF;
-		count = 4;
-		break;
-	case 24:
-		st->tx[2] = (val >> 16) & 0xFF;
-		st->tx[3] = (val >> 8) & 0xFF;
-		st->tx[4] = val & 0xFF;
-		count = 5;
-		break;
-	case 32:
-		st->tx[2] = (val >> 24) & 0xFF;
-		st->tx[3] = (val >> 16) & 0xFF;
-		st->tx[4] = (val >> 8) & 0xFF;
-		st->tx[5] = val & 0xFF;
-		count = 6;
-		break;
-	default:
-		ret = -EINVAL;
-		goto unlock;
-	}
-
-	ret = i2c_master_send(st->i2c, st->tx, count);
-
-unlock:
-	mutex_unlock(&st->buf_lock);
-
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
-static int ade7854_i2c_read_reg(struct device *dev,
-				u16 reg_address,
-				u32 *val,
-				int bits)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	int ret;
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = (reg_address >> 8) & 0xFF;
-	st->tx[1] = reg_address & 0xFF;
-
-	ret = i2c_master_send(st->i2c, st->tx, 2);
-	if (ret < 0)
-		goto unlock;
-
-	ret = i2c_master_recv(st->i2c, st->rx, bits);
-	if (ret < 0)
-		goto unlock;
-
-	switch (bits) {
-	case 8:
-		*val = st->rx[0];
-		break;
-	case 16:
-		*val = (st->rx[0] << 8) | st->rx[1];
-		break;
-	case 24:
-		*val = (st->rx[0] << 16) | (st->rx[1] << 8) | st->rx[2];
-		break;
-	case 32:
-		*val = (st->rx[0] << 24) | (st->rx[1] << 16) |
-			(st->rx[2] << 8) | st->rx[3];
-		break;
-	default:
-		ret = -EINVAL;
-		goto unlock;
-	}
-
-unlock:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static int ade7854_i2c_probe(struct i2c_client *client)
-{
-	struct ade7854_state *st;
-	struct iio_dev *indio_dev;
-
-	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-	st = iio_priv(indio_dev);
-	i2c_set_clientdata(client, indio_dev);
-	st->read_reg = ade7854_i2c_read_reg;
-	st->write_reg = ade7854_i2c_write_reg;
-	st->i2c = client;
-	st->irq = client->irq;
-
-	return ade7854_probe(indio_dev, &client->dev);
-}
-
-static const struct i2c_device_id ade7854_id[] = {
-	{ "ade7854", 0 },
-	{ "ade7858", 0 },
-	{ "ade7868", 0 },
-	{ "ade7878", 0 },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, ade7854_id);
-
-static struct i2c_driver ade7854_i2c_driver = {
-	.driver = {
-		.name = "ade7854",
-	},
-	.probe_new = ade7854_i2c_probe,
-	.id_table = ade7854_id,
-};
-module_i2c_driver(ade7854_i2c_driver);
-
-MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADE7854/58/68/78 Polyphase Multifunction Energy Metering IC I2C Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/iio/meter/ade7854-spi.c b/drivers/staging/iio/meter/ade7854-spi.c
deleted file mode 100644
index f12a6c8b3e88..000000000000
--- a/drivers/staging/iio/meter/ade7854-spi.c
+++ /dev/null
@@ -1,160 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * ADE7854/58/68/78 Polyphase Multifunction Energy Metering IC Driver (SPI Bus)
- *
- * Copyright 2010 Analog Devices Inc.
- */
-
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/spi/spi.h>
-#include <linux/slab.h>
-#include <linux/module.h>
-
-#include <linux/iio/iio.h>
-#include "ade7854.h"
-
-static int ade7854_spi_write_reg(struct device *dev,
-				 u16 reg_address,
-				 u32 val,
-				 int bits)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	struct spi_transfer xfer = {
-		.tx_buf = st->tx,
-		.bits_per_word = 8,
-		.len = 4,
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7854_WRITE_REG;
-	st->tx[1] = (reg_address >> 8) & 0xFF;
-	st->tx[2] = reg_address & 0xFF;
-	switch (bits) {
-	case 8:
-		st->tx[3] = val & 0xFF;
-		break;
-	case 16:
-		xfer.len = 5;
-		st->tx[3] = (val >> 8) & 0xFF;
-		st->tx[4] = val & 0xFF;
-		break;
-	case 24:
-		xfer.len = 6;
-		st->tx[3] = (val >> 16) & 0xFF;
-		st->tx[4] = (val >> 8) & 0xFF;
-		st->tx[5] = val & 0xFF;
-		break;
-	case 32:
-		xfer.len = 7;
-		st->tx[3] = (val >> 24) & 0xFF;
-		st->tx[4] = (val >> 16) & 0xFF;
-		st->tx[5] = (val >> 8) & 0xFF;
-		st->tx[6] = val & 0xFF;
-		break;
-	default:
-		ret = -EINVAL;
-		goto unlock;
-	}
-
-	ret = spi_sync_transfer(st->spi, &xfer, 1);
-unlock:
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7854_spi_read_reg(struct device *dev,
-				u16 reg_address,
-				u32 *val,
-				int bits)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	int ret;
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 3,
-		}, {
-			.rx_buf = st->rx,
-			.bits_per_word = 8,
-			.len = bits,
-		}
-	};
-
-	mutex_lock(&st->buf_lock);
-
-	st->tx[0] = ADE7854_READ_REG;
-	st->tx[1] = (reg_address >> 8) & 0xFF;
-	st->tx[2] = reg_address & 0xFF;
-
-	ret = spi_sync_transfer(st->spi, xfers, ARRAY_SIZE(xfers));
-	if (ret < 0) {
-		dev_err(&st->spi->dev, "problem when reading register 0x%02X",
-			reg_address);
-		goto unlock;
-	}
-
-	switch (bits) {
-	case 8:
-		*val = st->rx[0];
-		break;
-	case 16:
-		*val = be16_to_cpup((const __be16 *)st->rx);
-		break;
-	case 24:
-		*val = (st->rx[0] << 16) | (st->rx[1] << 8) | st->rx[2];
-		break;
-	case 32:
-		*val = be32_to_cpup((const __be32 *)st->rx);
-		break;
-	}
-
-unlock:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static int ade7854_spi_probe(struct spi_device *spi)
-{
-	struct ade7854_state *st;
-	struct iio_dev *indio_dev;
-
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-	st = iio_priv(indio_dev);
-	spi_set_drvdata(spi, indio_dev);
-	st->read_reg = ade7854_spi_read_reg;
-	st->write_reg = ade7854_spi_write_reg;
-	st->irq = spi->irq;
-	st->spi = spi;
-
-	return ade7854_probe(indio_dev, &spi->dev);
-}
-
-static const struct spi_device_id ade7854_id[] = {
-	{ "ade7854", 0 },
-	{ "ade7858", 0 },
-	{ "ade7868", 0 },
-	{ "ade7878", 0 },
-	{ }
-};
-MODULE_DEVICE_TABLE(spi, ade7854_id);
-
-static struct spi_driver ade7854_driver = {
-	.driver = {
-		.name = "ade7854",
-	},
-	.probe = ade7854_spi_probe,
-	.id_table = ade7854_id,
-};
-module_spi_driver(ade7854_driver);
-
-MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADE7854/58/68/78 SPI Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/iio/meter/ade7854.c b/drivers/staging/iio/meter/ade7854.c
deleted file mode 100644
index 68da6ecde6a3..000000000000
--- a/drivers/staging/iio/meter/ade7854.c
+++ /dev/null
@@ -1,556 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * ADE7854/58/68/78 Polyphase Multifunction Energy Metering IC Driver
- *
- * Copyright 2010 Analog Devices Inc.
- */
-
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/delay.h>
-#include <linux/mutex.h>
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
-#include <linux/list.h>
-#include <linux/module.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include "meter.h"
-#include "ade7854.h"
-
-static ssize_t ade7854_read_8bit(struct device *dev,
-				 struct device_attribute *attr,
-				 char *buf)
-{
-	int ret;
-	u32 val = 0;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = st->read_reg(dev, this_attr->address, &val, 8);
-	if (ret < 0)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7854_read_16bit(struct device *dev,
-				  struct device_attribute *attr,
-				  char *buf)
-{
-	int ret;
-	u32 val = 0;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = st->read_reg(dev, this_attr->address, &val, 16);
-	if (ret < 0)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7854_read_24bit(struct device *dev,
-				  struct device_attribute *attr,
-				  char *buf)
-{
-	int ret;
-	u32 val;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = st->read_reg(dev, this_attr->address, &val, 24);
-	if (ret < 0)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7854_read_32bit(struct device *dev,
-				  struct device_attribute *attr,
-				  char *buf)
-{
-	int ret;
-	u32 val = 0;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-
-	ret = st->read_reg(dev, this_attr->address, &val, 32);
-	if (ret < 0)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7854_write_8bit(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf,
-				  size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-
-	int ret;
-	u8 val;
-
-	ret = kstrtou8(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = st->write_reg(dev, this_attr->address, val, 8);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static ssize_t ade7854_write_16bit(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf,
-				   size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-
-	int ret;
-	u16 val;
-
-	ret = kstrtou16(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = st->write_reg(dev, this_attr->address, val, 16);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static ssize_t ade7854_write_24bit(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf,
-				   size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-
-	int ret;
-	u32 val;
-
-	ret = kstrtou32(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = st->write_reg(dev, this_attr->address, val, 24);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static ssize_t ade7854_write_32bit(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf,
-				   size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-
-	int ret;
-	u32 val;
-
-	ret = kstrtou32(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = st->write_reg(dev, this_attr->address, val, 32);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static int ade7854_reset(struct device *dev)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	u32 val;
-
-	st->read_reg(dev, ADE7854_CONFIG, &val, 16);
-	val |= BIT(7); /* Software Chip Reset */
-
-	return st->write_reg(dev, ADE7854_CONFIG, val, 16);
-}
-
-static IIO_DEV_ATTR_AIGAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_AIGAIN);
-static IIO_DEV_ATTR_BIGAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_BIGAIN);
-static IIO_DEV_ATTR_CIGAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_CIGAIN);
-static IIO_DEV_ATTR_NIGAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_NIGAIN);
-static IIO_DEV_ATTR_AVGAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_AVGAIN);
-static IIO_DEV_ATTR_BVGAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_BVGAIN);
-static IIO_DEV_ATTR_CVGAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_CVGAIN);
-static IIO_DEV_ATTR_APPARENT_POWER_A_GAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_AVAGAIN);
-static IIO_DEV_ATTR_APPARENT_POWER_B_GAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_BVAGAIN);
-static IIO_DEV_ATTR_APPARENT_POWER_C_GAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_CVAGAIN);
-static IIO_DEV_ATTR_ACTIVE_POWER_A_OFFSET(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_AWATTOS);
-static IIO_DEV_ATTR_ACTIVE_POWER_B_OFFSET(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_BWATTOS);
-static IIO_DEV_ATTR_ACTIVE_POWER_C_OFFSET(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_CWATTOS);
-static IIO_DEV_ATTR_REACTIVE_POWER_A_GAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_AVARGAIN);
-static IIO_DEV_ATTR_REACTIVE_POWER_B_GAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_BVARGAIN);
-static IIO_DEV_ATTR_REACTIVE_POWER_C_GAIN(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_CVARGAIN);
-static IIO_DEV_ATTR_REACTIVE_POWER_A_OFFSET(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_AVAROS);
-static IIO_DEV_ATTR_REACTIVE_POWER_B_OFFSET(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_BVAROS);
-static IIO_DEV_ATTR_REACTIVE_POWER_C_OFFSET(0644,
-		ade7854_read_24bit,
-		ade7854_write_24bit,
-		ADE7854_CVAROS);
-static IIO_DEV_ATTR_VPEAK(0644,
-		ade7854_read_32bit,
-		ade7854_write_32bit,
-		ADE7854_VPEAK);
-static IIO_DEV_ATTR_IPEAK(0644,
-		ade7854_read_32bit,
-		ade7854_write_32bit,
-		ADE7854_IPEAK);
-static IIO_DEV_ATTR_APHCAL(0644,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_APHCAL);
-static IIO_DEV_ATTR_BPHCAL(0644,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_BPHCAL);
-static IIO_DEV_ATTR_CPHCAL(0644,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_CPHCAL);
-static IIO_DEV_ATTR_CF1DEN(0644,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_CF1DEN);
-static IIO_DEV_ATTR_CF2DEN(0644,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_CF2DEN);
-static IIO_DEV_ATTR_CF3DEN(0644,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_CF3DEN);
-static IIO_DEV_ATTR_LINECYC(0644,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_LINECYC);
-static IIO_DEV_ATTR_SAGCYC(0644,
-		ade7854_read_8bit,
-		ade7854_write_8bit,
-		ADE7854_SAGCYC);
-static IIO_DEV_ATTR_CFCYC(0644,
-		ade7854_read_8bit,
-		ade7854_write_8bit,
-		ADE7854_CFCYC);
-static IIO_DEV_ATTR_PEAKCYC(0644,
-		ade7854_read_8bit,
-		ade7854_write_8bit,
-		ADE7854_PEAKCYC);
-static IIO_DEV_ATTR_CHKSUM(ade7854_read_24bit,
-		ADE7854_CHECKSUM);
-static IIO_DEV_ATTR_ANGLE0(ade7854_read_24bit,
-		ADE7854_ANGLE0);
-static IIO_DEV_ATTR_ANGLE1(ade7854_read_24bit,
-		ADE7854_ANGLE1);
-static IIO_DEV_ATTR_ANGLE2(ade7854_read_24bit,
-		ADE7854_ANGLE2);
-static IIO_DEV_ATTR_AIRMS(0444,
-		ade7854_read_24bit,
-		NULL,
-		ADE7854_AIRMS);
-static IIO_DEV_ATTR_BIRMS(0444,
-		ade7854_read_24bit,
-		NULL,
-		ADE7854_BIRMS);
-static IIO_DEV_ATTR_CIRMS(0444,
-		ade7854_read_24bit,
-		NULL,
-		ADE7854_CIRMS);
-static IIO_DEV_ATTR_NIRMS(0444,
-		ade7854_read_24bit,
-		NULL,
-		ADE7854_NIRMS);
-static IIO_DEV_ATTR_AVRMS(0444,
-		ade7854_read_24bit,
-		NULL,
-		ADE7854_AVRMS);
-static IIO_DEV_ATTR_BVRMS(0444,
-		ade7854_read_24bit,
-		NULL,
-		ADE7854_BVRMS);
-static IIO_DEV_ATTR_CVRMS(0444,
-		ade7854_read_24bit,
-		NULL,
-		ADE7854_CVRMS);
-static IIO_DEV_ATTR_AIRMSOS(0444,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_AIRMSOS);
-static IIO_DEV_ATTR_BIRMSOS(0444,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_BIRMSOS);
-static IIO_DEV_ATTR_CIRMSOS(0444,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_CIRMSOS);
-static IIO_DEV_ATTR_AVRMSOS(0444,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_AVRMSOS);
-static IIO_DEV_ATTR_BVRMSOS(0444,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_BVRMSOS);
-static IIO_DEV_ATTR_CVRMSOS(0444,
-		ade7854_read_16bit,
-		ade7854_write_16bit,
-		ADE7854_CVRMSOS);
-static IIO_DEV_ATTR_VOLT_A(ade7854_read_24bit,
-		ADE7854_VAWV);
-static IIO_DEV_ATTR_VOLT_B(ade7854_read_24bit,
-		ADE7854_VBWV);
-static IIO_DEV_ATTR_VOLT_C(ade7854_read_24bit,
-		ADE7854_VCWV);
-static IIO_DEV_ATTR_CURRENT_A(ade7854_read_24bit,
-		ADE7854_IAWV);
-static IIO_DEV_ATTR_CURRENT_B(ade7854_read_24bit,
-		ADE7854_IBWV);
-static IIO_DEV_ATTR_CURRENT_C(ade7854_read_24bit,
-		ADE7854_ICWV);
-static IIO_DEV_ATTR_AWATTHR(ade7854_read_32bit,
-		ADE7854_AWATTHR);
-static IIO_DEV_ATTR_BWATTHR(ade7854_read_32bit,
-		ADE7854_BWATTHR);
-static IIO_DEV_ATTR_CWATTHR(ade7854_read_32bit,
-		ADE7854_CWATTHR);
-static IIO_DEV_ATTR_AFWATTHR(ade7854_read_32bit,
-		ADE7854_AFWATTHR);
-static IIO_DEV_ATTR_BFWATTHR(ade7854_read_32bit,
-		ADE7854_BFWATTHR);
-static IIO_DEV_ATTR_CFWATTHR(ade7854_read_32bit,
-		ADE7854_CFWATTHR);
-static IIO_DEV_ATTR_AVARHR(ade7854_read_32bit,
-		ADE7854_AVARHR);
-static IIO_DEV_ATTR_BVARHR(ade7854_read_32bit,
-		ADE7854_BVARHR);
-static IIO_DEV_ATTR_CVARHR(ade7854_read_32bit,
-		ADE7854_CVARHR);
-static IIO_DEV_ATTR_AVAHR(ade7854_read_32bit,
-		ADE7854_AVAHR);
-static IIO_DEV_ATTR_BVAHR(ade7854_read_32bit,
-		ADE7854_BVAHR);
-static IIO_DEV_ATTR_CVAHR(ade7854_read_32bit,
-		ADE7854_CVAHR);
-
-static int ade7854_set_irq(struct device *dev, bool enable)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-
-	int ret;
-	u32 irqen;
-
-	ret = st->read_reg(dev, ADE7854_MASK0, &irqen, 32);
-	if (ret < 0)
-		return ret;
-
-	if (enable)
-		irqen |= BIT(17); /* 1: interrupt enabled when all periodical
-				   * (at 8 kHz rate) DSP computations finish.
-				   */
-	else
-		irqen &= ~BIT(17);
-
-	return st->write_reg(dev, ADE7854_MASK0, irqen, 32);
-}
-
-static int ade7854_initial_setup(struct iio_dev *indio_dev)
-{
-	int ret;
-	struct device *dev = &indio_dev->dev;
-
-	/* Disable IRQ */
-	ret = ade7854_set_irq(dev, false);
-	if (ret) {
-		dev_err(dev, "disable irq failed");
-		goto err_ret;
-	}
-
-	ade7854_reset(dev);
-	usleep_range(ADE7854_STARTUP_DELAY, ADE7854_STARTUP_DELAY + 100);
-
-err_ret:
-	return ret;
-}
-
-static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("8000");
-
-static IIO_CONST_ATTR(name, "ade7854");
-
-static struct attribute *ade7854_attributes[] = {
-	&iio_dev_attr_aigain.dev_attr.attr,
-	&iio_dev_attr_bigain.dev_attr.attr,
-	&iio_dev_attr_cigain.dev_attr.attr,
-	&iio_dev_attr_nigain.dev_attr.attr,
-	&iio_dev_attr_avgain.dev_attr.attr,
-	&iio_dev_attr_bvgain.dev_attr.attr,
-	&iio_dev_attr_cvgain.dev_attr.attr,
-	&iio_dev_attr_linecyc.dev_attr.attr,
-	&iio_dev_attr_sagcyc.dev_attr.attr,
-	&iio_dev_attr_cfcyc.dev_attr.attr,
-	&iio_dev_attr_peakcyc.dev_attr.attr,
-	&iio_dev_attr_chksum.dev_attr.attr,
-	&iio_dev_attr_apparent_power_a_gain.dev_attr.attr,
-	&iio_dev_attr_apparent_power_b_gain.dev_attr.attr,
-	&iio_dev_attr_apparent_power_c_gain.dev_attr.attr,
-	&iio_dev_attr_active_power_a_offset.dev_attr.attr,
-	&iio_dev_attr_active_power_b_offset.dev_attr.attr,
-	&iio_dev_attr_active_power_c_offset.dev_attr.attr,
-	&iio_dev_attr_reactive_power_a_gain.dev_attr.attr,
-	&iio_dev_attr_reactive_power_b_gain.dev_attr.attr,
-	&iio_dev_attr_reactive_power_c_gain.dev_attr.attr,
-	&iio_dev_attr_reactive_power_a_offset.dev_attr.attr,
-	&iio_dev_attr_reactive_power_b_offset.dev_attr.attr,
-	&iio_dev_attr_reactive_power_c_offset.dev_attr.attr,
-	&iio_dev_attr_awatthr.dev_attr.attr,
-	&iio_dev_attr_bwatthr.dev_attr.attr,
-	&iio_dev_attr_cwatthr.dev_attr.attr,
-	&iio_dev_attr_afwatthr.dev_attr.attr,
-	&iio_dev_attr_bfwatthr.dev_attr.attr,
-	&iio_dev_attr_cfwatthr.dev_attr.attr,
-	&iio_dev_attr_avarhr.dev_attr.attr,
-	&iio_dev_attr_bvarhr.dev_attr.attr,
-	&iio_dev_attr_cvarhr.dev_attr.attr,
-	&iio_dev_attr_angle0.dev_attr.attr,
-	&iio_dev_attr_angle1.dev_attr.attr,
-	&iio_dev_attr_angle2.dev_attr.attr,
-	&iio_dev_attr_avahr.dev_attr.attr,
-	&iio_dev_attr_bvahr.dev_attr.attr,
-	&iio_dev_attr_cvahr.dev_attr.attr,
-	&iio_const_attr_sampling_frequency_available.dev_attr.attr,
-	&iio_const_attr_name.dev_attr.attr,
-	&iio_dev_attr_vpeak.dev_attr.attr,
-	&iio_dev_attr_ipeak.dev_attr.attr,
-	&iio_dev_attr_aphcal.dev_attr.attr,
-	&iio_dev_attr_bphcal.dev_attr.attr,
-	&iio_dev_attr_cphcal.dev_attr.attr,
-	&iio_dev_attr_cf1den.dev_attr.attr,
-	&iio_dev_attr_cf2den.dev_attr.attr,
-	&iio_dev_attr_cf3den.dev_attr.attr,
-	&iio_dev_attr_airms.dev_attr.attr,
-	&iio_dev_attr_birms.dev_attr.attr,
-	&iio_dev_attr_cirms.dev_attr.attr,
-	&iio_dev_attr_nirms.dev_attr.attr,
-	&iio_dev_attr_avrms.dev_attr.attr,
-	&iio_dev_attr_bvrms.dev_attr.attr,
-	&iio_dev_attr_cvrms.dev_attr.attr,
-	&iio_dev_attr_airmsos.dev_attr.attr,
-	&iio_dev_attr_birmsos.dev_attr.attr,
-	&iio_dev_attr_cirmsos.dev_attr.attr,
-	&iio_dev_attr_avrmsos.dev_attr.attr,
-	&iio_dev_attr_bvrmsos.dev_attr.attr,
-	&iio_dev_attr_cvrmsos.dev_attr.attr,
-	&iio_dev_attr_volt_a.dev_attr.attr,
-	&iio_dev_attr_volt_b.dev_attr.attr,
-	&iio_dev_attr_volt_c.dev_attr.attr,
-	&iio_dev_attr_current_a.dev_attr.attr,
-	&iio_dev_attr_current_b.dev_attr.attr,
-	&iio_dev_attr_current_c.dev_attr.attr,
-	NULL,
-};
-
-static const struct attribute_group ade7854_attribute_group = {
-	.attrs = ade7854_attributes,
-};
-
-static const struct iio_info ade7854_info = {
-	.attrs = &ade7854_attribute_group,
-};
-
-int ade7854_probe(struct iio_dev *indio_dev, struct device *dev)
-{
-	int ret;
-	struct ade7854_state *st = iio_priv(indio_dev);
-	/* setup the industrialio driver allocated elements */
-	mutex_init(&st->buf_lock);
-
-	indio_dev->dev.parent = dev;
-	indio_dev->info = &ade7854_info;
-	indio_dev->modes = INDIO_DIRECT_MODE;
-
-	ret = devm_iio_device_register(dev, indio_dev);
-	if (ret)
-		return ret;
-
-	/* Get the device into a sane initial state */
-	return ade7854_initial_setup(indio_dev);
-}
-EXPORT_SYMBOL(ade7854_probe);
-
-MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADE7854/58/68/78 Polyphase Energy Meter");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/iio/meter/ade7854.h b/drivers/staging/iio/meter/ade7854.h
deleted file mode 100644
index 7a49f8f1016f..000000000000
--- a/drivers/staging/iio/meter/ade7854.h
+++ /dev/null
@@ -1,173 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ADE7854_H
-#define _ADE7854_H
-
-#define ADE7854_AIGAIN    0x4380
-#define ADE7854_AVGAIN    0x4381
-#define ADE7854_BIGAIN    0x4382
-#define ADE7854_BVGAIN    0x4383
-#define ADE7854_CIGAIN    0x4384
-#define ADE7854_CVGAIN    0x4385
-#define ADE7854_NIGAIN    0x4386
-#define ADE7854_AIRMSOS   0x4387
-#define ADE7854_AVRMSOS   0x4388
-#define ADE7854_BIRMSOS   0x4389
-#define ADE7854_BVRMSOS   0x438A
-#define ADE7854_CIRMSOS   0x438B
-#define ADE7854_CVRMSOS   0x438C
-#define ADE7854_NIRMSOS   0x438D
-#define ADE7854_AVAGAIN   0x438E
-#define ADE7854_BVAGAIN   0x438F
-#define ADE7854_CVAGAIN   0x4390
-#define ADE7854_AWGAIN    0x4391
-#define ADE7854_AWATTOS   0x4392
-#define ADE7854_BWGAIN    0x4393
-#define ADE7854_BWATTOS   0x4394
-#define ADE7854_CWGAIN    0x4395
-#define ADE7854_CWATTOS   0x4396
-#define ADE7854_AVARGAIN  0x4397
-#define ADE7854_AVAROS    0x4398
-#define ADE7854_BVARGAIN  0x4399
-#define ADE7854_BVAROS    0x439A
-#define ADE7854_CVARGAIN  0x439B
-#define ADE7854_CVAROS    0x439C
-#define ADE7854_AFWGAIN   0x439D
-#define ADE7854_AFWATTOS  0x439E
-#define ADE7854_BFWGAIN   0x439F
-#define ADE7854_BFWATTOS  0x43A0
-#define ADE7854_CFWGAIN   0x43A1
-#define ADE7854_CFWATTOS  0x43A2
-#define ADE7854_AFVARGAIN 0x43A3
-#define ADE7854_AFVAROS   0x43A4
-#define ADE7854_BFVARGAIN 0x43A5
-#define ADE7854_BFVAROS   0x43A6
-#define ADE7854_CFVARGAIN 0x43A7
-#define ADE7854_CFVAROS   0x43A8
-#define ADE7854_VATHR1    0x43A9
-#define ADE7854_VATHR0    0x43AA
-#define ADE7854_WTHR1     0x43AB
-#define ADE7854_WTHR0     0x43AC
-#define ADE7854_VARTHR1   0x43AD
-#define ADE7854_VARTHR0   0x43AE
-#define ADE7854_RSV       0x43AF
-#define ADE7854_VANOLOAD  0x43B0
-#define ADE7854_APNOLOAD  0x43B1
-#define ADE7854_VARNOLOAD 0x43B2
-#define ADE7854_VLEVEL    0x43B3
-#define ADE7854_DICOEFF   0x43B5
-#define ADE7854_HPFDIS    0x43B6
-#define ADE7854_ISUMLVL   0x43B8
-#define ADE7854_ISUM      0x43BF
-#define ADE7854_AIRMS     0x43C0
-#define ADE7854_AVRMS     0x43C1
-#define ADE7854_BIRMS     0x43C2
-#define ADE7854_BVRMS     0x43C3
-#define ADE7854_CIRMS     0x43C4
-#define ADE7854_CVRMS     0x43C5
-#define ADE7854_NIRMS     0x43C6
-#define ADE7854_RUN       0xE228
-#define ADE7854_AWATTHR   0xE400
-#define ADE7854_BWATTHR   0xE401
-#define ADE7854_CWATTHR   0xE402
-#define ADE7854_AFWATTHR  0xE403
-#define ADE7854_BFWATTHR  0xE404
-#define ADE7854_CFWATTHR  0xE405
-#define ADE7854_AVARHR    0xE406
-#define ADE7854_BVARHR    0xE407
-#define ADE7854_CVARHR    0xE408
-#define ADE7854_AFVARHR   0xE409
-#define ADE7854_BFVARHR   0xE40A
-#define ADE7854_CFVARHR   0xE40B
-#define ADE7854_AVAHR     0xE40C
-#define ADE7854_BVAHR     0xE40D
-#define ADE7854_CVAHR     0xE40E
-#define ADE7854_IPEAK     0xE500
-#define ADE7854_VPEAK     0xE501
-#define ADE7854_STATUS0   0xE502
-#define ADE7854_STATUS1   0xE503
-#define ADE7854_OILVL     0xE507
-#define ADE7854_OVLVL     0xE508
-#define ADE7854_SAGLVL    0xE509
-#define ADE7854_MASK0     0xE50A
-#define ADE7854_MASK1     0xE50B
-#define ADE7854_IAWV      0xE50C
-#define ADE7854_IBWV      0xE50D
-#define ADE7854_ICWV      0xE50E
-#define ADE7854_VAWV      0xE510
-#define ADE7854_VBWV      0xE511
-#define ADE7854_VCWV      0xE512
-#define ADE7854_AWATT     0xE513
-#define ADE7854_BWATT     0xE514
-#define ADE7854_CWATT     0xE515
-#define ADE7854_AVA       0xE519
-#define ADE7854_BVA       0xE51A
-#define ADE7854_CVA       0xE51B
-#define ADE7854_CHECKSUM  0xE51F
-#define ADE7854_VNOM      0xE520
-#define ADE7854_PHSTATUS  0xE600
-#define ADE7854_ANGLE0    0xE601
-#define ADE7854_ANGLE1    0xE602
-#define ADE7854_ANGLE2    0xE603
-#define ADE7854_PERIOD    0xE607
-#define ADE7854_PHNOLOAD  0xE608
-#define ADE7854_LINECYC   0xE60C
-#define ADE7854_ZXTOUT    0xE60D
-#define ADE7854_COMPMODE  0xE60E
-#define ADE7854_GAIN      0xE60F
-#define ADE7854_CFMODE    0xE610
-#define ADE7854_CF1DEN    0xE611
-#define ADE7854_CF2DEN    0xE612
-#define ADE7854_CF3DEN    0xE613
-#define ADE7854_APHCAL    0xE614
-#define ADE7854_BPHCAL    0xE615
-#define ADE7854_CPHCAL    0xE616
-#define ADE7854_PHSIGN    0xE617
-#define ADE7854_CONFIG    0xE618
-#define ADE7854_MMODE     0xE700
-#define ADE7854_ACCMODE   0xE701
-#define ADE7854_LCYCMODE  0xE702
-#define ADE7854_PEAKCYC   0xE703
-#define ADE7854_SAGCYC    0xE704
-#define ADE7854_CFCYC     0xE705
-#define ADE7854_HSDC_CFG  0xE706
-#define ADE7854_CONFIG2   0xEC01
-
-#define ADE7854_READ_REG   0x1
-#define ADE7854_WRITE_REG  0x0
-
-#define ADE7854_MAX_TX    7
-#define ADE7854_MAX_RX    7
-#define ADE7854_STARTUP_DELAY 1000
-
-#define ADE7854_SPI_SLOW	(u32)(300 * 1000)
-#define ADE7854_SPI_BURST	(u32)(1000 * 1000)
-#define ADE7854_SPI_FAST	(u32)(2000 * 1000)
-
-/**
- * struct ade7854_state - device instance specific data
- * @spi:		actual spi_device
- * @read_reg		Wrapper function for I2C and SPI read
- * @write_reg		Wrapper function for I2C and SPI write
- * @indio_dev:		industrial I/O device structure
- * @buf_lock:		mutex to protect tx and rx
- * @tx:			transmit buffer
- * @rx:			receive buffer
- **/
-struct ade7854_state {
-	struct spi_device *spi;
-	struct i2c_client *i2c;
-	int (*read_reg)(struct device *dev, u16 reg_address, u32 *val,
-			int bits);
-	int (*write_reg)(struct device *dev, u16 reg_address, u32 val,
-			 int bits);
-	int irq;
-	struct mutex buf_lock;
-	u8 tx[ADE7854_MAX_TX] __aligned(IIO_DMA_MINALIGN);
-	u8 rx[ADE7854_MAX_RX];
-
-};
-
-int ade7854_probe(struct iio_dev *indio_dev, struct device *dev);
-int ade7854_remove(struct iio_dev *indio_dev);
-
-#endif
diff --git a/drivers/staging/iio/meter/meter.h b/drivers/staging/iio/meter/meter.h
deleted file mode 100644
index 5ed59bf30a25..000000000000
--- a/drivers/staging/iio/meter/meter.h
+++ /dev/null
@@ -1,398 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _METER_H
-#define _METER_H
-
-#include <linux/iio/sysfs.h>
-
-/* metering ic types of attribute */
-
-#define IIO_DEV_ATTR_CURRENT_A_OFFSET(_mode, _show, _store, _addr)	\
-	IIO_DEVICE_ATTR(current_a_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CURRENT_B_OFFSET(_mode, _show, _store, _addr)	\
-	IIO_DEVICE_ATTR(current_b_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CURRENT_C_OFFSET(_mode, _show, _store, _addr)	\
-	IIO_DEVICE_ATTR(current_c_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VOLT_A_OFFSET(_mode, _show, _store, _addr)      \
-	IIO_DEVICE_ATTR(volt_a_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VOLT_B_OFFSET(_mode, _show, _store, _addr)      \
-	IIO_DEVICE_ATTR(volt_b_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VOLT_C_OFFSET(_mode, _show, _store, _addr)      \
-	IIO_DEVICE_ATTR(volt_c_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_REACTIVE_POWER_A_OFFSET(_mode, _show, _store, _addr)   \
-	IIO_DEVICE_ATTR(reactive_power_a_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_REACTIVE_POWER_B_OFFSET(_mode, _show, _store, _addr)   \
-	IIO_DEVICE_ATTR(reactive_power_b_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_REACTIVE_POWER_C_OFFSET(_mode, _show, _store, _addr)   \
-	IIO_DEVICE_ATTR(reactive_power_c_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_ACTIVE_POWER_A_OFFSET(_mode, _show, _store, _addr)     \
-	IIO_DEVICE_ATTR(active_power_a_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_ACTIVE_POWER_B_OFFSET(_mode, _show, _store, _addr)     \
-	IIO_DEVICE_ATTR(active_power_b_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_ACTIVE_POWER_C_OFFSET(_mode, _show, _store, _addr)     \
-	IIO_DEVICE_ATTR(active_power_c_offset, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CURRENT_A_GAIN(_mode, _show, _store, _addr)	\
-	IIO_DEVICE_ATTR(current_a_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CURRENT_B_GAIN(_mode, _show, _store, _addr)	\
-	IIO_DEVICE_ATTR(current_b_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CURRENT_C_GAIN(_mode, _show, _store, _addr)	\
-	IIO_DEVICE_ATTR(current_c_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_APPARENT_POWER_A_GAIN(_mode, _show, _store, _addr)     \
-	IIO_DEVICE_ATTR(apparent_power_a_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_APPARENT_POWER_B_GAIN(_mode, _show, _store, _addr)     \
-	IIO_DEVICE_ATTR(apparent_power_b_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_APPARENT_POWER_C_GAIN(_mode, _show, _store, _addr)     \
-	IIO_DEVICE_ATTR(apparent_power_c_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_ACTIVE_POWER_GAIN(_mode, _show, _store, _addr)     \
-	IIO_DEVICE_ATTR(active_power_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_ACTIVE_POWER_A_GAIN(_mode, _show, _store, _addr)   \
-	IIO_DEVICE_ATTR(active_power_a_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_ACTIVE_POWER_B_GAIN(_mode, _show, _store, _addr)   \
-	IIO_DEVICE_ATTR(active_power_b_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_ACTIVE_POWER_C_GAIN(_mode, _show, _store, _addr)   \
-	IIO_DEVICE_ATTR(active_power_c_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_REACTIVE_POWER_A_GAIN(_mode, _show, _store, _addr)     \
-	IIO_DEVICE_ATTR(reactive_power_a_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_REACTIVE_POWER_B_GAIN(_mode, _show, _store, _addr)     \
-	IIO_DEVICE_ATTR(reactive_power_b_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_REACTIVE_POWER_C_GAIN(_mode, _show, _store, _addr)     \
-	IIO_DEVICE_ATTR(reactive_power_c_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CURRENT_A(_show, _addr)			\
-	IIO_DEVICE_ATTR(current_a, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_CURRENT_B(_show, _addr)			\
-	IIO_DEVICE_ATTR(current_b, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_CURRENT_C(_show, _addr)			\
-	IIO_DEVICE_ATTR(current_c, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_VOLT_A(_show, _addr)			\
-	IIO_DEVICE_ATTR(volt_a, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_VOLT_B(_show, _addr)			\
-	IIO_DEVICE_ATTR(volt_b, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_VOLT_C(_show, _addr)			\
-	IIO_DEVICE_ATTR(volt_c, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_AENERGY(_show, _addr)			\
-	IIO_DEVICE_ATTR(aenergy, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_LENERGY(_show, _addr)			\
-	IIO_DEVICE_ATTR(lenergy, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_RAENERGY(_show, _addr)			\
-	IIO_DEVICE_ATTR(raenergy, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_LAENERGY(_show, _addr)			\
-	IIO_DEVICE_ATTR(laenergy, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_VAENERGY(_show, _addr)			\
-	IIO_DEVICE_ATTR(vaenergy, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_LVAENERGY(_show, _addr)			\
-	IIO_DEVICE_ATTR(lvaenergy, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_RVAENERGY(_show, _addr)			\
-	IIO_DEVICE_ATTR(rvaenergy, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_LVARENERGY(_show, _addr)			\
-	IIO_DEVICE_ATTR(lvarenergy, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_CHKSUM(_show, _addr)                       \
-	IIO_DEVICE_ATTR(chksum, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_ANGLE0(_show, _addr)                       \
-	IIO_DEVICE_ATTR(angle0, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_ANGLE1(_show, _addr)                       \
-	IIO_DEVICE_ATTR(angle1, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_ANGLE2(_show, _addr)                       \
-	IIO_DEVICE_ATTR(angle2, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_AWATTHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(awatthr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_BWATTHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(bwatthr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_CWATTHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(cwatthr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_AFWATTHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(afwatthr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_BFWATTHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(bfwatthr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_CFWATTHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(cfwatthr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_AVARHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(avarhr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_BVARHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(bvarhr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_CVARHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(cvarhr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_AVAHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(avahr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_BVAHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(bvahr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_CVAHR(_show, _addr)			\
-	IIO_DEVICE_ATTR(cvahr, 0444, _show, NULL, _addr)
-
-#define IIO_DEV_ATTR_IOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(ios, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(vos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_PHCAL(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(phcal, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_APHCAL(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(aphcal, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_BPHCAL(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(bphcal, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CPHCAL(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cphcal, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_APOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(apos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_AAPOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(aapos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_BAPOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(bapos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CAPOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(capos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_AVRMSGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(avrmsgain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_BVRMSGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(bvrmsgain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CVRMSGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cvrmsgain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_AIGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(aigain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_BIGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(bigain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CIGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cigain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_NIGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(nigain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_AVGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(avgain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_BVGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(bvgain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CVGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cvgain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_WGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(wgain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_WDIV(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(wdiv, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CFNUM(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cfnum, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CFDEN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cfden, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CF1DEN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cf1den, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CF2DEN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cf2den, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CF3DEN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cf3den, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_IRMS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(irms, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VRMS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(vrms, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_AIRMS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(airms, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_BIRMS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(birms, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CIRMS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cirms, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_NIRMS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(nirms, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_AVRMS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(avrms, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_BVRMS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(bvrms, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CVRMS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cvrms, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_IRMSOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(irmsos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VRMSOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(vrmsos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_AIRMSOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(airmsos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_BIRMSOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(birmsos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CIRMSOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cirmsos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_AVRMSOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(avrmsos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_BVRMSOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(bvrmsos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CVRMSOS(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cvrmsos, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VAGAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(vagain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_PGA_GAIN(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(pga_gain, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VADIV(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(vadiv, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_LINECYC(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(linecyc, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_SAGCYC(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(sagcyc, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_CFCYC(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(cfcyc, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_PEAKCYC(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(peakcyc, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_SAGLVL(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(saglvl, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_IPKLVL(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(ipklvl, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VPKLVL(_mode, _show, _store, _addr)                \
-	IIO_DEVICE_ATTR(vpklvl, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_IPEAK(_mode, _show, _store, _addr)			\
-	IIO_DEVICE_ATTR(ipeak, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_RIPEAK(_mode, _show, _store, _addr)		\
-	IIO_DEVICE_ATTR(ripeak, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VPEAK(_mode, _show, _store, _addr)			\
-	IIO_DEVICE_ATTR(vpeak, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_RVPEAK(_mode, _show, _store, _addr)		\
-	IIO_DEVICE_ATTR(rvpeak, _mode, _show, _store, _addr)
-
-#define IIO_DEV_ATTR_VPERIOD(_mode, _show, _store, _addr)		\
-	IIO_DEVICE_ATTR(vperiod, _mode, _show, _store, _addr)
-
-/* active energy register, AENERGY, is more than half full */
-#define IIO_EVENT_ATTR_AENERGY_HALF_FULL(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(aenergy_half_full, _evlist, _show, _store, _mask)
-
-/* a SAG on the line voltage */
-#define IIO_EVENT_ATTR_LINE_VOLT_SAG(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(line_volt_sag, _evlist, _show, _store, _mask)
-
-/*
- * Indicates the end of energy accumulation over an integer number
- * of half line cycles
- */
-#define IIO_EVENT_ATTR_CYCEND(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(cycend, _evlist, _show, _store, _mask)
-
-/* on the rising and falling edge of the voltage waveform */
-#define IIO_EVENT_ATTR_ZERO_CROSS(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(zero_cross, _evlist, _show, _store, _mask)
-
-/* the active energy register has overflowed */
-#define IIO_EVENT_ATTR_AENERGY_OVERFLOW(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(aenergy_overflow, _evlist, _show, _store, _mask)
-
-/* the apparent energy register has overflowed */
-#define IIO_EVENT_ATTR_VAENERGY_OVERFLOW(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(vaenergy_overflow, _evlist, _show, _store, _mask)
-
-/* the active energy register, VAENERGY, is more than half full */
-#define IIO_EVENT_ATTR_VAENERGY_HALF_FULL(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(vaenergy_half_full, _evlist, _show, _store, _mask)
-
-/* the power has gone from negative to positive */
-#define IIO_EVENT_ATTR_PPOS(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(ppos, _evlist, _show, _store, _mask)
-
-/* the power has gone from positive to negative */
-#define IIO_EVENT_ATTR_PNEG(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(pneg, _evlist, _show, _store, _mask)
-
-/* waveform sample from Channel 1 has exceeded the IPKLVL value */
-#define IIO_EVENT_ATTR_IPKLVL_EXC(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(ipklvl_exc, _evlist, _show, _store, _mask)
-
-/* waveform sample from Channel 2 has exceeded the VPKLVL value */
-#define IIO_EVENT_ATTR_VPKLVL_EXC(_evlist, _show, _store, _mask) \
-	IIO_EVENT_ATTR_SH(vpklvl_exc, _evlist, _show, _store, _mask)
-
-#endif /* _METER_H */
diff --git a/drivers/staging/iio/resolver/ad2s1210.c b/drivers/staging/iio/resolver/ad2s1210.c
index e4cf42438487..06de5823eb8e 100644
--- a/drivers/staging/iio/resolver/ad2s1210.c
+++ b/drivers/staging/iio/resolver/ad2s1210.c
@@ -7,6 +7,7 @@
 #include <linux/types.h>
 #include <linux/mutex.h>
 #include <linux/device.h>
+#include <linux/of.h>
 #include <linux/spi/spi.h>
 #include <linux/slab.h>
 #include <linux/sysfs.h>
@@ -101,7 +102,7 @@ struct ad2s1210_state {
 static const int ad2s1210_mode_vals[4][2] = {
 	[MOD_POS] = { 0, 0 },
 	[MOD_VEL] = { 0, 1 },
-	[MOD_CONFIG] = { 1, 0 },
+	[MOD_CONFIG] = { 1, 1 },
 };
 
 static inline void ad2s1210_set_mode(enum ad2s1210_mode mode,
diff --git a/drivers/staging/ks7010/ks_hostif.c b/drivers/staging/ks7010/ks_hostif.c
index 9429ee155910..af3825578d85 100644
--- a/drivers/staging/ks7010/ks_hostif.c
+++ b/drivers/staging/ks7010/ks_hostif.c
@@ -75,9 +75,8 @@ static void ks_wlan_hw_wakeup_task(struct work_struct *work)
 
 	if (ps_status == PS_SNOOZE) {
 		ks_wlan_hw_wakeup_request(priv);
-		time_left = wait_for_completion_interruptible_timeout(
-				&priv->psstatus.wakeup_wait,
-				msecs_to_jiffies(20));
+		time_left = wait_for_completion_interruptible_timeout(&priv->psstatus.wakeup_wait,
+								      msecs_to_jiffies(20));
 		if (time_left <= 0) {
 			netdev_dbg(priv->net_dev, "wake up timeout or interrupted !!!\n");
 			schedule_work(&priv->wakeup_work);
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
index 0d90683ed227..273155308fe3 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
@@ -3,6 +3,7 @@
  * Support for GalaxyCore GC0310 VGA camera sensor.
  *
  * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
+ * Copyright (c) 2023 Hans de Goede <hdegoede@redhat.com>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License version
@@ -26,234 +27,59 @@
 #include <linux/device.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
+#include <linux/gpio/consumer.h>
+#include <linux/gpio/machine.h>
 #include <linux/i2c.h>
 #include <linux/moduleparam.h>
+#include <linux/pm_runtime.h>
 #include <media/v4l2-device.h>
 #include <linux/io.h>
 #include "../include/linux/atomisp_gmin_platform.h"
 
 #include "gc0310.h"
 
-/* i2c read/write stuff */
-static int gc0310_read_reg(struct i2c_client *client,
-			   u16 data_length, u8 reg, u8 *val)
-{
-	int err;
-	struct i2c_msg msg[2];
-	unsigned char data[1];
-
-	if (!client->adapter) {
-		dev_err(&client->dev, "%s error, no client->adapter\n",
-			__func__);
-		return -ENODEV;
-	}
-
-	if (data_length != GC0310_8BIT) {
-		dev_err(&client->dev, "%s error, invalid data length\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	memset(msg, 0, sizeof(msg));
-
-	msg[0].addr = client->addr;
-	msg[0].flags = 0;
-	msg[0].len = I2C_MSG_LENGTH;
-	msg[0].buf = data;
-
-	/* high byte goes out first */
-	data[0] = (u8)(reg & 0xff);
-
-	msg[1].addr = client->addr;
-	msg[1].len = data_length;
-	msg[1].flags = I2C_M_RD;
-	msg[1].buf = data;
-
-	err = i2c_transfer(client->adapter, msg, 2);
-	if (err != 2) {
-		if (err >= 0)
-			err = -EIO;
-		dev_err(&client->dev,
-			"read from offset 0x%x error %d", reg, err);
-		return err;
-	}
-
-	*val = 0;
-	/* high byte comes first */
-	if (data_length == GC0310_8BIT)
-		*val = (u8)data[0];
-
-	return 0;
-}
-
-static int gc0310_i2c_write(struct i2c_client *client, u16 len, u8 *data)
-{
-	struct i2c_msg msg;
-	const int num_msg = 1;
-	int ret;
-
-	msg.addr = client->addr;
-	msg.flags = 0;
-	msg.len = len;
-	msg.buf = data;
-	ret = i2c_transfer(client->adapter, &msg, 1);
-
-	return ret == num_msg ? 0 : -EIO;
-}
-
-static int gc0310_write_reg(struct i2c_client *client, u16 data_length,
-			    u8 reg, u8 val)
-{
-	int ret;
-	unsigned char data[2] = {0};
-	u8 *wreg = (u8 *)data;
-	const u16 len = data_length + sizeof(u8); /* 8-bit address + data */
-
-	if (data_length != GC0310_8BIT) {
-		dev_err(&client->dev,
-			"%s error, invalid data_length\n", __func__);
-		return -EINVAL;
-	}
-
-	/* high byte goes out first */
-	*wreg = (u8)(reg & 0xff);
-
-	if (data_length == GC0310_8BIT)
-		data[1] = (u8)(val);
-
-	ret = gc0310_i2c_write(client, len, data);
-	if (ret)
-		dev_err(&client->dev,
-			"write error: wrote 0x%x to offset 0x%x error %d",
-			val, reg, ret);
-
-	return ret;
-}
-
 /*
  * gc0310_write_reg_array - Initializes a list of GC0310 registers
  * @client: i2c driver client structure
  * @reglist: list of registers to be written
- *
- * This function initializes a list of registers. When consecutive addresses
- * are found in a row on the list, this function creates a buffer and sends
- * consecutive data in a single i2c_transfer().
- *
- * __gc0310_flush_reg_array, __gc0310_buf_reg_array() and
- * __gc0310_write_reg_is_consecutive() are internal functions to
- * gc0310_write_reg_array_fast() and should be not used anywhere else.
- *
+ * @count: number of register, value pairs in the list
  */
-
-static int __gc0310_flush_reg_array(struct i2c_client *client,
-				    struct gc0310_write_ctrl *ctrl)
-{
-	u16 size;
-
-	if (ctrl->index == 0)
-		return 0;
-
-	size = sizeof(u8) + ctrl->index; /* 8-bit address + data */
-	ctrl->buffer.addr = (u8)(ctrl->buffer.addr);
-	ctrl->index = 0;
-
-	return gc0310_i2c_write(client, size, (u8 *)&ctrl->buffer);
-}
-
-static int __gc0310_buf_reg_array(struct i2c_client *client,
-				  struct gc0310_write_ctrl *ctrl,
-				  const struct gc0310_reg *next)
+static int gc0310_write_reg_array(struct i2c_client *client,
+				  const struct gc0310_reg *reglist, int count)
 {
-	int size;
+	int i, err;
 
-	switch (next->type) {
-	case GC0310_8BIT:
-		size = 1;
-		ctrl->buffer.data[ctrl->index] = (u8)next->val;
-		break;
-	default:
-		return -EINVAL;
+	for (i = 0; i < count; i++) {
+		err = i2c_smbus_write_byte_data(client, reglist[i].reg, reglist[i].val);
+		if (err) {
+			dev_err(&client->dev, "write error: wrote 0x%x to offset 0x%x error %d",
+				reglist[i].val, reglist[i].reg, err);
+			return err;
+		}
 	}
 
-	/* When first item is added, we need to store its starting address */
-	if (ctrl->index == 0)
-		ctrl->buffer.addr = next->reg;
-
-	ctrl->index += size;
-
-	/*
-	 * Buffer cannot guarantee free space for u32? Better flush it to avoid
-	 * possible lack of memory for next item.
-	 */
-	if (ctrl->index + sizeof(u8) >= GC0310_MAX_WRITE_BUF_SIZE)
-		return __gc0310_flush_reg_array(client, ctrl);
-
 	return 0;
 }
 
-static int __gc0310_write_reg_is_consecutive(struct i2c_client *client,
-					     struct gc0310_write_ctrl *ctrl,
-					     const struct gc0310_reg *next)
+static int gc0310_exposure_set(struct gc0310_device *dev, u32 exp)
 {
-	if (ctrl->index == 0)
-		return 1;
-
-	return ctrl->buffer.addr + ctrl->index == next->reg;
-}
-
-static int gc0310_write_reg_array(struct i2c_client *client,
-				  const struct gc0310_reg *reglist)
-{
-	const struct gc0310_reg *next = reglist;
-	struct gc0310_write_ctrl ctrl;
-	int err;
-
-	ctrl.index = 0;
-	for (; next->type != GC0310_TOK_TERM; next++) {
-		switch (next->type & GC0310_TOK_MASK) {
-		case GC0310_TOK_DELAY:
-			err = __gc0310_flush_reg_array(client, &ctrl);
-			if (err)
-				return err;
-			msleep(next->val);
-			break;
-		default:
-			/*
-			 * If next address is not consecutive, data needs to be
-			 * flushed before proceed.
-			 */
-			if (!__gc0310_write_reg_is_consecutive(client, &ctrl,
-							       next)) {
-				err = __gc0310_flush_reg_array(client, &ctrl);
-				if (err)
-					return err;
-			}
-			err = __gc0310_buf_reg_array(client, &ctrl, next);
-			if (err) {
-				dev_err(&client->dev, "%s: write error, aborted\n",
-					__func__);
-				return err;
-			}
-			break;
-		}
-	}
+	struct i2c_client *client = v4l2_get_subdevdata(&dev->sd);
 
-	return __gc0310_flush_reg_array(client, &ctrl);
+	return i2c_smbus_write_word_swapped(client, GC0310_AEC_PK_EXPO_H, exp);
 }
 
-static int gc0310_set_gain(struct v4l2_subdev *sd, int gain)
-
+static int gc0310_gain_set(struct gc0310_device *dev, u32 gain)
 {
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
+	struct i2c_client *client = v4l2_get_subdevdata(&dev->sd);
 	u8 again, dgain;
+	int ret;
 
-	if (gain < 0x20)
-		gain = 0x20;
-	if (gain > 0x80)
-		gain = 0x80;
+	/* Taken from original driver, this never sets dgain lower then 32? */
 
-	if (gain >= 0x20 && gain < 0x40) {
+	/* Change 0 - 95 to 32 - 127 */
+	gain += 32;
+
+	if (gain < 64) {
 		again = 0x0; /* sqrt(2) */
 		dgain = gain;
 	} else {
@@ -261,507 +87,109 @@ static int gc0310_set_gain(struct v4l2_subdev *sd, int gain)
 		dgain = gain / 2;
 	}
 
-	dev_dbg(&client->dev, "gain=0x%x again=0x%x dgain=0x%x\n", gain, again, dgain);
-
-	/* set analog gain */
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-			       GC0310_AGC_ADJ, again);
-	if (ret)
-		return ret;
-
-	/* set digital gain */
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-			       GC0310_DGC_ADJ, dgain);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int __gc0310_set_exposure(struct v4l2_subdev *sd, int coarse_itg,
-				 int gain, int digitgain)
-
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	dev_dbg(&client->dev, "coarse_itg=%d gain=%d digitgain=%d\n", coarse_itg, gain, digitgain);
-
-	/* set exposure */
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-			       GC0310_AEC_PK_EXPO_L,
-			       coarse_itg & 0xff);
+	ret = i2c_smbus_write_byte_data(client, GC0310_AGC_ADJ, again);
 	if (ret)
 		return ret;
 
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-			       GC0310_AEC_PK_EXPO_H,
-			       (coarse_itg >> 8) & 0x0f);
-	if (ret)
-		return ret;
-
-	ret = gc0310_set_gain(sd, gain);
-	if (ret)
-		return ret;
-
-	return ret;
-}
-
-static int gc0310_set_exposure(struct v4l2_subdev *sd, int exposure,
-			       int gain, int digitgain)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	int ret;
-
-	mutex_lock(&dev->input_lock);
-	ret = __gc0310_set_exposure(sd, exposure, gain, digitgain);
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-}
-
-static long gc0310_s_exposure(struct v4l2_subdev *sd,
-			      struct atomisp_exposure *exposure)
-{
-	int exp = exposure->integration_time[0];
-	int gain = exposure->gain[0];
-	int digitgain = exposure->gain[1];
-
-	/* we should not accept the invalid value below. */
-	if (gain == 0) {
-		struct i2c_client *client = v4l2_get_subdevdata(sd);
-
-		v4l2_err(client, "%s: invalid value\n", __func__);
-		return -EINVAL;
-	}
-
-	return gc0310_set_exposure(sd, exp, gain, digitgain);
-}
-
-/* TO DO */
-static int gc0310_v_flip(struct v4l2_subdev *sd, s32 value)
-{
-	return 0;
-}
-
-/* TO DO */
-static int gc0310_h_flip(struct v4l2_subdev *sd, s32 value)
-{
-	return 0;
-}
-
-static long gc0310_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
-{
-	switch (cmd) {
-	case ATOMISP_IOC_S_EXPOSURE:
-		return gc0310_s_exposure(sd, arg);
-	default:
-		return -EINVAL;
-	}
-	return 0;
-}
-
-/* This returns the exposure time being used. This should only be used
- * for filling in EXIF data, not for actual image processing.
- */
-static int gc0310_q_exposure(struct v4l2_subdev *sd, s32 *value)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u8 reg_v;
-	int ret;
-
-	/* get exposure */
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-			      GC0310_AEC_PK_EXPO_L,
-			      &reg_v);
-	if (ret)
-		goto err;
-
-	*value = reg_v;
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-			      GC0310_AEC_PK_EXPO_H,
-			      &reg_v);
-	if (ret)
-		goto err;
-
-	*value = *value + (reg_v << 8);
-err:
-	return ret;
+	return i2c_smbus_write_byte_data(client, GC0310_DGC_ADJ, dgain);
 }
 
 static int gc0310_s_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct gc0310_device *dev =
-	    container_of(ctrl->handler, struct gc0310_device, ctrl_handler);
-	struct i2c_client *client = v4l2_get_subdevdata(&dev->sd);
-	int ret = 0;
+		container_of(ctrl->handler, struct gc0310_device, ctrls.handler);
+	int ret;
+
+	/* Only apply changes to the controls if the device is powered up */
+	if (!pm_runtime_get_if_in_use(dev->sd.dev))
+		return 0;
 
 	switch (ctrl->id) {
-	case V4L2_CID_VFLIP:
-		dev_dbg(&client->dev, "%s: CID_VFLIP:%d.\n",
-			__func__, ctrl->val);
-		ret = gc0310_v_flip(&dev->sd, ctrl->val);
+	case V4L2_CID_EXPOSURE:
+		ret = gc0310_exposure_set(dev, ctrl->val);
 		break;
-	case V4L2_CID_HFLIP:
-		dev_dbg(&client->dev, "%s: CID_HFLIP:%d.\n",
-			__func__, ctrl->val);
-		ret = gc0310_h_flip(&dev->sd, ctrl->val);
+	case V4L2_CID_GAIN:
+		ret = gc0310_gain_set(dev, ctrl->val);
 		break;
 	default:
 		ret = -EINVAL;
-	}
-	return ret;
-}
-
-static int gc0310_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct gc0310_device *dev =
-	    container_of(ctrl->handler, struct gc0310_device, ctrl_handler);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_EXPOSURE_ABSOLUTE:
-		ret = gc0310_q_exposure(&dev->sd, &ctrl->val);
 		break;
-	default:
-		ret = -EINVAL;
 	}
 
+	pm_runtime_put(dev->sd.dev);
 	return ret;
 }
 
 static const struct v4l2_ctrl_ops ctrl_ops = {
 	.s_ctrl = gc0310_s_ctrl,
-	.g_volatile_ctrl = gc0310_g_volatile_ctrl
 };
 
-static const struct v4l2_ctrl_config gc0310_controls[] = {
-	{
-		.ops = &ctrl_ops,
-		.id = V4L2_CID_EXPOSURE_ABSOLUTE,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "exposure",
-		.min = 0x0,
-		.max = 0xffff,
-		.step = 0x01,
-		.def = 0x00,
-		.flags = 0,
-	},
-	{
-		.ops = &ctrl_ops,
-		.id = V4L2_CID_VFLIP,
-		.type = V4L2_CTRL_TYPE_BOOLEAN,
-		.name = "Flip",
-		.min = 0,
-		.max = 1,
-		.step = 1,
-		.def = 0,
-	},
-	{
-		.ops = &ctrl_ops,
-		.id = V4L2_CID_HFLIP,
-		.type = V4L2_CTRL_TYPE_BOOLEAN,
-		.name = "Mirror",
-		.min = 0,
-		.max = 1,
-		.step = 1,
-		.def = 0,
-	},
-};
-
-static int gc0310_init(struct v4l2_subdev *sd)
-{
-	int ret;
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	mutex_lock(&dev->input_lock);
-
-	/* set initial registers */
-	ret  = gc0310_write_reg_array(client, gc0310_reset_register);
-
-	/* restore settings */
-	gc0310_res = gc0310_res_preview;
-	N_RES = N_RES_PREVIEW;
-
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-}
-
-static int power_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret = 0;
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	if (flag) {
-		/* The upstream module driver (written to Crystal
-		 * Cove) had this logic to pulse the rails low first.
-		 * This appears to break things on the MRD7 with the
-		 * X-Powers PMIC...
-		 *
-		 *     ret = dev->platform_data->v1p8_ctrl(sd, 0);
-		 *     ret |= dev->platform_data->v2p8_ctrl(sd, 0);
-		 *     mdelay(50);
-		 */
-		ret |= dev->platform_data->v1p8_ctrl(sd, 1);
-		ret |= dev->platform_data->v2p8_ctrl(sd, 1);
-		usleep_range(10000, 15000);
-	}
-
-	if (!flag || ret) {
-		ret |= dev->platform_data->v1p8_ctrl(sd, 0);
-		ret |= dev->platform_data->v2p8_ctrl(sd, 0);
-	}
-	return ret;
-}
-
-static int gpio_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret;
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	/* GPIO0 == "reset" (active low), GPIO1 == "power down" */
-	if (flag) {
-		/* Pulse reset, then release power down */
-		ret = dev->platform_data->gpio0_ctrl(sd, 0);
-		usleep_range(5000, 10000);
-		ret |= dev->platform_data->gpio0_ctrl(sd, 1);
-		usleep_range(10000, 15000);
-		ret |= dev->platform_data->gpio1_ctrl(sd, 0);
-		usleep_range(10000, 15000);
-	} else {
-		ret = dev->platform_data->gpio1_ctrl(sd, 1);
-		ret |= dev->platform_data->gpio0_ctrl(sd, 0);
-	}
-	return ret;
-}
-
-static int power_up(struct v4l2_subdev *sd)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	if (dev->power_on)
-		return 0; /* Already on */
-
-	/* power control */
-	ret = power_ctrl(sd, 1);
-	if (ret)
-		goto fail_power;
-
-	/* flis clock control */
-	ret = dev->platform_data->flisclk_ctrl(sd, 1);
-	if (ret)
-		goto fail_clk;
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 1);
-	if (ret) {
-		ret = gpio_ctrl(sd, 1);
-		if (ret)
-			goto fail_gpio;
-	}
-
-	msleep(100);
-
-	dev->power_on = true;
-	return 0;
-
-fail_gpio:
-	dev->platform_data->flisclk_ctrl(sd, 0);
-fail_clk:
-	power_ctrl(sd, 0);
-fail_power:
-	dev_err(&client->dev, "sensor power-up failed\n");
-
-	return ret;
-}
-
-static int power_down(struct v4l2_subdev *sd)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	if (!dev->power_on)
-		return 0; /* Already off */
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 0);
-	if (ret) {
-		ret = gpio_ctrl(sd, 0);
-		if (ret)
-			dev_err(&client->dev, "gpio failed 2\n");
-	}
-
-	ret = dev->platform_data->flisclk_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "flisclk failed\n");
-
-	/* power control */
-	ret = power_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "vprog failed.\n");
-
-	dev->power_on = false;
-	return ret;
-}
-
-static int gc0310_s_power(struct v4l2_subdev *sd, int on)
+static struct v4l2_mbus_framefmt *
+gc0310_get_pad_format(struct gc0310_device *dev,
+		      struct v4l2_subdev_state *state,
+		      unsigned int pad, enum v4l2_subdev_format_whence which)
 {
-	int ret;
-
-	if (on == 0)
-		return power_down(sd);
-
-	ret = power_up(sd);
-	if (ret)
-		return ret;
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_format(&dev->sd, state, pad);
 
-	return gc0310_init(sd);
+	return &dev->mode.fmt;
 }
 
-/* TODO: remove it. */
-static int startup(struct v4l2_subdev *sd)
+/* The GC0310 currently only supports 1 fixed fmt */
+static void gc0310_fill_format(struct v4l2_mbus_framefmt *fmt)
 {
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	ret = gc0310_write_reg_array(client, dev->res->regs);
-	if (ret) {
-		dev_err(&client->dev, "gc0310 write register err.\n");
-		return ret;
-	}
-
-	return ret;
+	memset(fmt, 0, sizeof(*fmt));
+	fmt->width = GC0310_NATIVE_WIDTH;
+	fmt->height = GC0310_NATIVE_HEIGHT;
+	fmt->field = V4L2_FIELD_NONE;
+	fmt->code = MEDIA_BUS_FMT_SGRBG8_1X8;
 }
 
 static int gc0310_set_fmt(struct v4l2_subdev *sd,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
-	struct v4l2_mbus_framefmt *fmt = &format->format;
 	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct camera_mipi_info *gc0310_info = NULL;
-	struct gc0310_resolution *res;
-	int ret = 0;
-
-	if (format->pad)
-		return -EINVAL;
+	struct v4l2_mbus_framefmt *fmt;
 
-	if (!fmt)
-		return -EINVAL;
-
-	gc0310_info = v4l2_get_subdev_hostdata(sd);
-	if (!gc0310_info)
-		return -EINVAL;
-
-	mutex_lock(&dev->input_lock);
-
-	res = v4l2_find_nearest_size(gc0310_res_preview,
-				     ARRAY_SIZE(gc0310_res_preview), width,
-				     height, fmt->width, fmt->height);
-	if (!res)
-		res = &gc0310_res_preview[N_RES - 1];
-
-	fmt->width = res->width;
-	fmt->height = res->height;
-	dev->res = res;
-
-	fmt->code = MEDIA_BUS_FMT_SGRBG8_1X8;
-
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		sd_state->pads->try_fmt = *fmt;
-		mutex_unlock(&dev->input_lock);
-		return 0;
-	}
-
-	/* s_power has not been called yet for std v4l2 clients (camorama) */
-	power_up(sd);
-
-	dev_dbg(&client->dev, "%s: before gc0310_write_reg_array %s\n",
-		__func__, dev->res->desc);
-	ret = startup(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc0310 startup err\n");
-		goto err;
-	}
+	fmt = gc0310_get_pad_format(dev, sd_state, format->pad, format->which);
+	gc0310_fill_format(fmt);
 
-err:
-	mutex_unlock(&dev->input_lock);
-	return ret;
+	format->format = *fmt;
+	return 0;
 }
 
 static int gc0310_get_fmt(struct v4l2_subdev *sd,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
-	struct v4l2_mbus_framefmt *fmt = &format->format;
 	struct gc0310_device *dev = to_gc0310_sensor(sd);
+	struct v4l2_mbus_framefmt *fmt;
 
-	if (format->pad)
-		return -EINVAL;
-
-	if (!fmt)
-		return -EINVAL;
-
-	fmt->width = dev->res->width;
-	fmt->height = dev->res->height;
-	fmt->code = MEDIA_BUS_FMT_SGRBG8_1X8;
-
+	fmt = gc0310_get_pad_format(dev, sd_state, format->pad, format->which);
+	format->format = *fmt;
 	return 0;
 }
 
 static int gc0310_detect(struct i2c_client *client)
 {
 	struct i2c_adapter *adapter = client->adapter;
-	u8 high, low;
 	int ret;
-	u16 id;
 
 	if (!i2c_check_functionality(adapter, I2C_FUNC_I2C))
 		return -ENODEV;
 
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-			      GC0310_SC_CMMN_CHIP_ID_H, &high);
-	if (ret) {
-		dev_err(&client->dev, "read sensor_id_high failed\n");
+	ret = i2c_smbus_read_word_swapped(client, GC0310_SC_CMMN_CHIP_ID_H);
+	if (ret < 0) {
+		dev_err(&client->dev, "read sensor_id failed: %d\n", ret);
 		return -ENODEV;
 	}
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-			      GC0310_SC_CMMN_CHIP_ID_L, &low);
-	if (ret) {
-		dev_err(&client->dev, "read sensor_id_low failed\n");
-		return -ENODEV;
-	}
-	id = ((((u16)high) << 8) | (u16)low);
-	dev_dbg(&client->dev, "sensor ID = 0x%x\n", id);
 
-	if (id != GC0310_ID) {
-		dev_err(&client->dev, "sensor ID error, read id = 0x%x, target id = 0x%x\n", id,
-			GC0310_ID);
+	dev_dbg(&client->dev, "sensor ID = 0x%x\n", ret);
+
+	if (ret != GC0310_ID) {
+		dev_err(&client->dev, "sensor ID error, read id = 0x%x, target id = 0x%x\n",
+			ret, GC0310_ID);
 		return -ENODEV;
 	}
 
@@ -774,116 +202,90 @@ static int gc0310_s_stream(struct v4l2_subdev *sd, int enable)
 {
 	struct gc0310_device *dev = to_gc0310_sensor(sd);
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
+	int ret = 0;
 
 	dev_dbg(&client->dev, "%s S enable=%d\n", __func__, enable);
 	mutex_lock(&dev->input_lock);
 
-	if (enable) {
-		/* enable per frame MIPI and sensor ctrl reset  */
-		ret = gc0310_write_reg(client, GC0310_8BIT,
-				       0xFE, 0x30);
-		if (ret) {
-			mutex_unlock(&dev->input_lock);
-			return ret;
-		}
-	}
-
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-			       GC0310_RESET_RELATED, GC0310_REGISTER_PAGE_3);
-	if (ret) {
-		mutex_unlock(&dev->input_lock);
-		return ret;
+	if (dev->is_streaming == enable) {
+		dev_warn(&client->dev, "stream already %s\n", enable ? "started" : "stopped");
+		goto error_unlock;
 	}
 
-	ret = gc0310_write_reg(client, GC0310_8BIT, GC0310_SW_STREAM,
-			       enable ? GC0310_START_STREAMING :
-			       GC0310_STOP_STREAMING);
-	if (ret) {
-		mutex_unlock(&dev->input_lock);
-		return ret;
-	}
+	if (enable) {
+		ret = pm_runtime_get_sync(&client->dev);
+		if (ret < 0)
+			goto error_power_down;
 
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-			       GC0310_RESET_RELATED, GC0310_REGISTER_PAGE_0);
-	if (ret) {
-		mutex_unlock(&dev->input_lock);
-		return ret;
-	}
+		msleep(100);
 
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-static int gc0310_s_config(struct v4l2_subdev *sd,
-			   int irq, void *platform_data)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
+		ret = gc0310_write_reg_array(client, gc0310_reset_register,
+					     ARRAY_SIZE(gc0310_reset_register));
+		if (ret)
+			goto error_power_down;
 
-	if (!platform_data)
-		return -ENODEV;
+		ret = gc0310_write_reg_array(client, gc0310_VGA_30fps,
+					     ARRAY_SIZE(gc0310_VGA_30fps));
+		if (ret)
+			goto error_power_down;
 
-	dev->platform_data =
-	    (struct camera_sensor_platform_data *)platform_data;
+		/* restore value of all ctrls */
+		ret = __v4l2_ctrl_handler_setup(&dev->ctrls.handler);
+		if (ret)
+			goto error_power_down;
 
-	mutex_lock(&dev->input_lock);
-	/* power off the module, then power on it in future
-	 * as first power on by board may not fulfill the
-	 * power on sequqence needed by the module
-	 */
-	dev->power_on = true; /* force power_down() to run */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc0310 power-off err.\n");
-		goto fail_power_off;
+		/* enable per frame MIPI and sensor ctrl reset  */
+		ret = i2c_smbus_write_byte_data(client, 0xFE, 0x30);
+		if (ret)
+			goto error_power_down;
 	}
 
-	ret = power_up(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc0310 power-up err.\n");
-		goto fail_power_on;
-	}
+	ret = i2c_smbus_write_byte_data(client, GC0310_RESET_RELATED, GC0310_REGISTER_PAGE_3);
+	if (ret)
+		goto error_power_down;
 
-	ret = dev->platform_data->csi_cfg(sd, 1);
+	ret = i2c_smbus_write_byte_data(client, GC0310_SW_STREAM,
+					enable ? GC0310_START_STREAMING : GC0310_STOP_STREAMING);
 	if (ret)
-		goto fail_csi_cfg;
+		goto error_power_down;
 
-	/* config & detect sensor */
-	ret = gc0310_detect(client);
-	if (ret) {
-		dev_err(&client->dev, "gc0310_detect err s_config.\n");
-		goto fail_csi_cfg;
-	}
+	ret = i2c_smbus_write_byte_data(client, GC0310_RESET_RELATED, GC0310_REGISTER_PAGE_0);
+	if (ret)
+		goto error_power_down;
 
-	/* turn off sensor, after probed */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc0310 power-off err.\n");
-		goto fail_csi_cfg;
-	}
-	mutex_unlock(&dev->input_lock);
+	if (!enable)
+		pm_runtime_put(&client->dev);
 
+	dev->is_streaming = enable;
+	mutex_unlock(&dev->input_lock);
 	return 0;
 
-fail_csi_cfg:
-	dev->platform_data->csi_cfg(sd, 0);
-fail_power_on:
-	power_down(sd);
-	dev_err(&client->dev, "sensor power-gating failed\n");
-fail_power_off:
+error_power_down:
+	pm_runtime_put(&client->dev);
+	dev->is_streaming = false;
+error_unlock:
 	mutex_unlock(&dev->input_lock);
 	return ret;
 }
 
+static int gc0310_s_config(struct v4l2_subdev *sd)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	int ret;
+
+	ret = pm_runtime_get_sync(&client->dev);
+	if (ret >= 0)
+		ret = gc0310_detect(client);
+
+	pm_runtime_put(&client->dev);
+	return ret;
+}
+
 static int gc0310_g_frame_interval(struct v4l2_subdev *sd,
 				   struct v4l2_subdev_frame_interval *interval)
 {
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
 	interval->interval.numerator = 1;
-	interval->interval.denominator = dev->res->fps;
+	interval->interval.denominator = GC0310_FPS;
 
 	return 0;
 }
@@ -892,7 +294,8 @@ static int gc0310_enum_mbus_code(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_mbus_code_enum *code)
 {
-	if (code->index >= MAX_FMTS)
+	/* We support only a single format */
+	if (code->index)
 		return -EINVAL;
 
 	code->code = MEDIA_BUS_FMT_SGRBG8_1X8;
@@ -903,27 +306,21 @@ static int gc0310_enum_frame_size(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_frame_size_enum *fse)
 {
-	int index = fse->index;
-
-	if (index >= N_RES)
+	/* We support only a single resolution */
+	if (fse->index)
 		return -EINVAL;
 
-	fse->min_width = gc0310_res[index].width;
-	fse->min_height = gc0310_res[index].height;
-	fse->max_width = gc0310_res[index].width;
-	fse->max_height = gc0310_res[index].height;
+	fse->min_width = GC0310_NATIVE_WIDTH;
+	fse->max_width = GC0310_NATIVE_WIDTH;
+	fse->min_height = GC0310_NATIVE_HEIGHT;
+	fse->max_height = GC0310_NATIVE_HEIGHT;
 
 	return 0;
 }
 
 static int gc0310_g_skip_frames(struct v4l2_subdev *sd, u32 *frames)
 {
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	mutex_lock(&dev->input_lock);
-	*frames = dev->res->skip_frames;
-	mutex_unlock(&dev->input_lock);
-
+	*frames = GC0310_SKIP_FRAMES;
 	return 0;
 }
 
@@ -936,11 +333,6 @@ static const struct v4l2_subdev_video_ops gc0310_video_ops = {
 	.g_frame_interval = gc0310_g_frame_interval,
 };
 
-static const struct v4l2_subdev_core_ops gc0310_core_ops = {
-	.s_power = gc0310_s_power,
-	.ioctl = gc0310_ioctl,
-};
-
 static const struct v4l2_subdev_pad_ops gc0310_pad_ops = {
 	.enum_mbus_code = gc0310_enum_mbus_code,
 	.enum_frame_size = gc0310_enum_frame_size,
@@ -949,12 +341,31 @@ static const struct v4l2_subdev_pad_ops gc0310_pad_ops = {
 };
 
 static const struct v4l2_subdev_ops gc0310_ops = {
-	.core = &gc0310_core_ops,
 	.video = &gc0310_video_ops,
 	.pad = &gc0310_pad_ops,
 	.sensor = &gc0310_sensor_ops,
 };
 
+static int gc0310_init_controls(struct gc0310_device *dev)
+{
+	struct v4l2_ctrl_handler *hdl = &dev->ctrls.handler;
+
+	v4l2_ctrl_handler_init(hdl, 2);
+
+	/* Use the same lock for controls as for everything else */
+	hdl->lock = &dev->input_lock;
+	dev->sd.ctrl_handler = hdl;
+
+	dev->ctrls.exposure =
+		v4l2_ctrl_new_std(hdl, &ctrl_ops, V4L2_CID_EXPOSURE, 0, 4095, 1, 1023);
+
+	/* 32 steps at base gain 1 + 64 half steps at base gain 2 */
+	dev->ctrls.gain =
+		v4l2_ctrl_new_std(hdl, &ctrl_ops, V4L2_CID_GAIN, 0, 95, 1, 31);
+
+	return hdl->error;
+}
+
 static void gc0310_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
@@ -962,11 +373,11 @@ static void gc0310_remove(struct i2c_client *client)
 
 	dev_dbg(&client->dev, "gc0310_remove...\n");
 
-	dev->platform_data->csi_cfg(sd, 0);
-
+	atomisp_unregister_subdev(sd);
 	v4l2_device_unregister_subdev(sd);
 	media_entity_cleanup(&dev->sd.entity);
-	v4l2_ctrl_handler_free(&dev->ctrl_handler);
+	v4l2_ctrl_handler_free(&dev->ctrls.handler);
+	pm_runtime_disable(&client->dev);
 	kfree(dev);
 }
 
@@ -974,70 +385,91 @@ static int gc0310_probe(struct i2c_client *client)
 {
 	struct gc0310_device *dev;
 	int ret;
-	void *pdata;
-	unsigned int i;
 
-	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	dev = devm_kzalloc(&client->dev, sizeof(*dev), GFP_KERNEL);
 	if (!dev)
 		return -ENOMEM;
 
-	mutex_init(&dev->input_lock);
+	ret = v4l2_get_acpi_sensor_info(&client->dev, NULL);
+	if (ret)
+		return ret;
 
-	dev->res = &gc0310_res_preview[0];
-	v4l2_i2c_subdev_init(&dev->sd, client, &gc0310_ops);
+	dev->reset = devm_gpiod_get(&client->dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(dev->reset))
+		return dev_err_probe(&client->dev, PTR_ERR(dev->reset),
+				     "getting reset GPIO\n");
 
-	pdata = gmin_camera_platform_data(&dev->sd,
-					  ATOMISP_INPUT_FORMAT_RAW_8,
-					  atomisp_bayer_order_grbg);
-	if (!pdata) {
-		ret = -EINVAL;
-		goto out_free;
-	}
+	dev->powerdown = devm_gpiod_get(&client->dev, "powerdown", GPIOD_OUT_HIGH);
+	if (IS_ERR(dev->powerdown))
+		return dev_err_probe(&client->dev, PTR_ERR(dev->powerdown),
+				     "getting powerdown GPIO\n");
 
-	ret = gc0310_s_config(&dev->sd, client->irq, pdata);
-	if (ret)
-		goto out_free;
+	mutex_init(&dev->input_lock);
+	v4l2_i2c_subdev_init(&dev->sd, client, &gc0310_ops);
+	gc0310_fill_format(&dev->mode.fmt);
 
-	ret = atomisp_register_i2c_module(&dev->sd, pdata, RAW_CAMERA);
-	if (ret)
-		goto out_free;
+	pm_runtime_set_suspended(&client->dev);
+	pm_runtime_enable(&client->dev);
+	pm_runtime_set_autosuspend_delay(&client->dev, 1000);
+	pm_runtime_use_autosuspend(&client->dev);
+
+	ret = gc0310_s_config(&dev->sd);
+	if (ret) {
+		gc0310_remove(client);
+		return ret;
+	}
 
 	dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
 	dev->pad.flags = MEDIA_PAD_FL_SOURCE;
-	dev->format.code = MEDIA_BUS_FMT_SGRBG8_1X8;
 	dev->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
-	ret =
-	    v4l2_ctrl_handler_init(&dev->ctrl_handler,
-				   ARRAY_SIZE(gc0310_controls));
+
+	ret = gc0310_init_controls(dev);
 	if (ret) {
 		gc0310_remove(client);
 		return ret;
 	}
 
-	for (i = 0; i < ARRAY_SIZE(gc0310_controls); i++)
-		v4l2_ctrl_new_custom(&dev->ctrl_handler, &gc0310_controls[i],
-				     NULL);
+	ret = media_entity_pads_init(&dev->sd.entity, 1, &dev->pad);
+	if (ret) {
+		gc0310_remove(client);
+		return ret;
+	}
 
-	if (dev->ctrl_handler.error) {
+	ret = atomisp_register_sensor_no_gmin(&dev->sd, 1, ATOMISP_INPUT_FORMAT_RAW_8,
+					      atomisp_bayer_order_grbg);
+	if (ret) {
 		gc0310_remove(client);
-		return dev->ctrl_handler.error;
+		return ret;
 	}
 
-	/* Use same lock for controls as for everything else. */
-	dev->ctrl_handler.lock = &dev->input_lock;
-	dev->sd.ctrl_handler = &dev->ctrl_handler;
+	return 0;
+}
 
-	ret = media_entity_pads_init(&dev->sd.entity, 1, &dev->pad);
-	if (ret)
-		gc0310_remove(client);
+static int gc0310_suspend(struct device *dev)
+{
+	struct v4l2_subdev *sd = dev_get_drvdata(dev);
+	struct gc0310_device *gc0310_dev = to_gc0310_sensor(sd);
 
-	return ret;
-out_free:
-	v4l2_device_unregister_subdev(&dev->sd);
-	kfree(dev);
-	return ret;
+	gpiod_set_value_cansleep(gc0310_dev->powerdown, 1);
+	gpiod_set_value_cansleep(gc0310_dev->reset, 1);
+	return 0;
+}
+
+static int gc0310_resume(struct device *dev)
+{
+	struct v4l2_subdev *sd = dev_get_drvdata(dev);
+	struct gc0310_device *gc0310_dev = to_gc0310_sensor(sd);
+
+	usleep_range(10000, 15000);
+	gpiod_set_value_cansleep(gc0310_dev->reset, 0);
+	usleep_range(10000, 15000);
+	gpiod_set_value_cansleep(gc0310_dev->powerdown, 0);
+
+	return 0;
 }
 
+static DEFINE_RUNTIME_DEV_PM_OPS(gc0310_pm_ops, gc0310_suspend, gc0310_resume, NULL);
+
 static const struct acpi_device_id gc0310_acpi_match[] = {
 	{"XXGC0310"},
 	{"INT0310"},
@@ -1048,6 +480,7 @@ MODULE_DEVICE_TABLE(acpi, gc0310_acpi_match);
 static struct i2c_driver gc0310_driver = {
 	.driver = {
 		.name = "gc0310",
+		.pm = pm_sleep_ptr(&gc0310_pm_ops),
 		.acpi_match_table = gc0310_acpi_match,
 	},
 	.probe_new = gc0310_probe,
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
index aeb38599fe13..63de214916f5 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
@@ -3,6 +3,7 @@
  * Support for OmniVision OV2680 1080p HD camera sensor.
  *
  * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
+ * Copyright (c) 2023 Hans de Goede <hdegoede@redhat.com>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License version
@@ -418,7 +419,7 @@ static int ov2680_s_stream(struct v4l2_subdev *sd, int enable)
 	if (enable) {
 		ret = pm_runtime_get_sync(sensor->sd.dev);
 		if (ret < 0)
-			goto error_unlock;
+			goto error_power_down;
 
 		ret = ov2680_set_mode(sensor);
 		if (ret)
@@ -446,6 +447,7 @@ static int ov2680_s_stream(struct v4l2_subdev *sd, int enable)
 
 error_power_down:
 	pm_runtime_put(sensor->sd.dev);
+	sensor->is_streaming = false;
 error_unlock:
 	mutex_unlock(&sensor->input_lock);
 	return ret;
@@ -614,21 +616,6 @@ static void ov2680_remove(struct i2c_client *client)
 	pm_runtime_disable(&client->dev);
 }
 
-/*
- * Unlike other sensors which have both a rest and powerdown input pins,
- * the OV2680 only has a powerdown input. But some ACPI tables still list
- * 2 GPIOs for the OV2680 and it is unclear which to use. So try to get
- * up to 2 GPIOs (1 mandatory, 1 optional) and control them in sync.
- */
-static const struct acpi_gpio_params ov2680_first_gpio = { 0, 0, true };
-static const struct acpi_gpio_params ov2680_second_gpio = { 1, 0, true };
-
-static const struct acpi_gpio_mapping ov2680_gpio_mapping[] = {
-	{ "powerdown-gpios", &ov2680_first_gpio, 1 },
-	{ "powerdown-alt-gpios", &ov2680_second_gpio, 1 },
-	{ },
-};
-
 static int ov2680_probe(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
@@ -644,26 +631,24 @@ static int ov2680_probe(struct i2c_client *client)
 	sensor->client = client;
 	v4l2_i2c_subdev_init(&sensor->sd, client, &ov2680_ops);
 
-	ret = devm_acpi_dev_add_driver_gpios(&client->dev, ov2680_gpio_mapping);
+	ret = v4l2_get_acpi_sensor_info(dev, NULL);
 	if (ret)
 		return ret;
 
-	sensor->powerdown = devm_gpiod_get(dev, "powerdown", GPIOD_OUT_HIGH);
+	sensor->powerdown = devm_gpiod_get_optional(dev, "powerdown", GPIOD_OUT_HIGH);
 	if (IS_ERR(sensor->powerdown))
 		return dev_err_probe(dev, PTR_ERR(sensor->powerdown), "getting powerdown GPIO\n");
 
-	sensor->powerdown_alt = devm_gpiod_get_optional(dev, "powerdown-alt", GPIOD_OUT_HIGH);
-	if (IS_ERR(sensor->powerdown_alt))
-		return dev_err_probe(dev, PTR_ERR(sensor->powerdown_alt), "getting powerdown-alt GPIO\n");
-
 	pm_runtime_set_suspended(dev);
 	pm_runtime_enable(dev);
 	pm_runtime_set_autosuspend_delay(dev, 1000);
 	pm_runtime_use_autosuspend(dev);
 
 	ret = ov2680_s_config(&sensor->sd);
-	if (ret)
+	if (ret) {
+		ov2680_remove(client);
 		return ret;
+	}
 
 	sensor->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
 	sensor->pad.flags = MEDIA_PAD_FL_SOURCE;
@@ -699,7 +684,6 @@ static int ov2680_suspend(struct device *dev)
 	struct ov2680_device *sensor = to_ov2680_sensor(sd);
 
 	gpiod_set_value_cansleep(sensor->powerdown, 1);
-	gpiod_set_value_cansleep(sensor->powerdown_alt, 1);
 	return 0;
 }
 
@@ -712,7 +696,6 @@ static int ov2680_resume(struct device *dev)
 	usleep_range(5000, 6000);
 
 	gpiod_set_value_cansleep(sensor->powerdown, 0);
-	gpiod_set_value_cansleep(sensor->powerdown_alt, 0);
 
 	/* according to DS, 20ms is needed between PWDN and i2c access */
 	msleep(20);
diff --git a/drivers/staging/media/atomisp/i2c/gc0310.h b/drivers/staging/media/atomisp/i2c/gc0310.h
index cae480ae6fba..d40406289598 100644
--- a/drivers/staging/media/atomisp/i2c/gc0310.h
+++ b/drivers/staging/media/atomisp/i2c/gc0310.h
@@ -33,34 +33,14 @@
 
 #include "../include/linux/atomisp_platform.h"
 
-/* Defines for register writes and register array processing */
-#define I2C_MSG_LENGTH		1
-#define I2C_RETRY_COUNT		5
+#define GC0310_NATIVE_WIDTH			656
+#define GC0310_NATIVE_HEIGHT			496
 
-#define GC0310_FOCAL_LENGTH_NUM	278	/*2.78mm*/
+#define GC0310_FPS				30
+#define GC0310_SKIP_FRAMES			3
 
-#define MAX_FMTS		1
+#define GC0310_FOCAL_LENGTH_NUM			278 /* 2.78mm */
 
-/*
- * focal length bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define GC0310_FOCAL_LENGTH_DEFAULT 0x1160064
-
-/*
- * current f-number bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define GC0310_F_NUMBER_DEFAULT 0x1a000a
-
-/*
- * f-number range bits definition:
- * bits 31-24: max f-number numerator
- * bits 23-16: max f-number denominator
- * bits 15-8: min f-number numerator
- * bits 7-0: min f-number denominator
- */
-#define GC0310_F_NUMBER_RANGE 0x1a0a1a0a
 #define GC0310_ID	0xa310
 
 #define GC0310_RESET_RELATED		0xFE
@@ -105,84 +85,43 @@
 #define GC0310_START_STREAMING			0x94 /* 8-bit enable */
 #define GC0310_STOP_STREAMING			0x0 /* 8-bit disable */
 
-#define GC0310_BIN_FACTOR_MAX			3
-
-struct regval_list {
-	u16 reg_num;
-	u8 value;
-};
-
-struct gc0310_resolution {
-	u8 *desc;
-	const struct gc0310_reg *regs;
-	int res;
-	int width;
-	int height;
-	int fps;
-	int pix_clk_freq;
-	u32 skip_frames;
-	u16 pixels_per_line;
-	u16 lines_per_frame;
-	bool used;
-};
-
-struct gc0310_format {
-	u8 *desc;
-	u32 pixelformat;
-	struct gc0310_reg *regs;
-};
-
 /*
  * gc0310 device structure.
  */
 struct gc0310_device {
 	struct v4l2_subdev sd;
 	struct media_pad pad;
-	struct v4l2_mbus_framefmt format;
 	struct mutex input_lock;
-	struct v4l2_ctrl_handler ctrl_handler;
+	bool is_streaming;
 
-	struct camera_sensor_platform_data *platform_data;
-	struct gc0310_resolution *res;
-	u8 type;
-	bool power_on;
-};
+	struct gpio_desc *reset;
+	struct gpio_desc *powerdown;
 
-enum gc0310_tok_type {
-	GC0310_8BIT  = 0x0001,
-	GC0310_TOK_TERM   = 0xf000,	/* terminating token for reg list */
-	GC0310_TOK_DELAY  = 0xfe00,	/* delay token for reg list */
-	GC0310_TOK_MASK = 0xfff0
+	struct gc0310_mode {
+		struct v4l2_mbus_framefmt fmt;
+	} mode;
+
+	struct gc0310_ctrls {
+		struct v4l2_ctrl_handler handler;
+		struct v4l2_ctrl *exposure;
+		struct v4l2_ctrl *gain;
+	} ctrls;
 };
 
 /**
  * struct gc0310_reg - MI sensor  register format
- * @type: type of the register
  * @reg: 16-bit offset to register
  * @val: 8/16/32-bit register value
  *
  * Define a structure for sensor register initialization values
  */
 struct gc0310_reg {
-	enum gc0310_tok_type type;
 	u8 reg;
 	u8 val;	/* @set value for read/mod/write, @mask */
 };
 
 #define to_gc0310_sensor(x) container_of(x, struct gc0310_device, sd)
 
-#define GC0310_MAX_WRITE_BUF_SIZE	30
-
-struct gc0310_write_buffer {
-	u8 addr;
-	u8 data[GC0310_MAX_WRITE_BUF_SIZE];
-};
-
-struct gc0310_write_ctrl {
-	int index;
-	struct gc0310_write_buffer buffer;
-};
-
 /*
  * Register settings for various resolution
  */
@@ -190,206 +129,181 @@ static const struct gc0310_reg gc0310_reset_register[] = {
 /////////////////////////////////////////////////
 /////////////////	system reg	/////////////////
 /////////////////////////////////////////////////
-	{GC0310_8BIT, 0xfe, 0xf0},
-	{GC0310_8BIT, 0xfe, 0xf0},
-	{GC0310_8BIT, 0xfe, 0x00},
-
-	{GC0310_8BIT, 0xfc, 0x0e}, //4e
-	{GC0310_8BIT, 0xfc, 0x0e}, //16//4e // [0]apwd [6]regf_clk_gate
-	{GC0310_8BIT, 0xf2, 0x80}, //sync output
-	{GC0310_8BIT, 0xf3, 0x00}, //1f//01 data output
-	{GC0310_8BIT, 0xf7, 0x33}, //f9
-	{GC0310_8BIT, 0xf8, 0x05}, //00
-	{GC0310_8BIT, 0xf9, 0x0e}, // 0x8e //0f
-	{GC0310_8BIT, 0xfa, 0x11},
+	{ 0xfe, 0xf0 },
+	{ 0xfe, 0xf0 },
+	{ 0xfe, 0x00 },
+
+	{ 0xfc, 0x0e }, /* 4e */
+	{ 0xfc, 0x0e }, /* 16//4e // [0]apwd [6]regf_clk_gate */
+	{ 0xf2, 0x80 }, /* sync output */
+	{ 0xf3, 0x00 }, /* 1f//01 data output */
+	{ 0xf7, 0x33 }, /* f9 */
+	{ 0xf8, 0x05 }, /* 00 */
+	{ 0xf9, 0x0e }, /* 0x8e //0f */
+	{ 0xfa, 0x11 },
 
 /////////////////////////////////////////////////
 ///////////////////   MIPI	 ////////////////////
 /////////////////////////////////////////////////
-	{GC0310_8BIT, 0xfe, 0x03},
-	{GC0310_8BIT, 0x01, 0x03}, ///mipi 1lane
-	{GC0310_8BIT, 0x02, 0x22}, // 0x33
-	{GC0310_8BIT, 0x03, 0x94},
-	{GC0310_8BIT, 0x04, 0x01}, // fifo_prog
-	{GC0310_8BIT, 0x05, 0x00}, //fifo_prog
-	{GC0310_8BIT, 0x06, 0x80}, //b0  //YUV ISP data
-	{GC0310_8BIT, 0x11, 0x2a},//1e //LDI set YUV422
-	{GC0310_8BIT, 0x12, 0x90},//00 //04 //00 //04//00 //LWC[7:0]  //
-	{GC0310_8BIT, 0x13, 0x02},//05 //05 //LWC[15:8]
-	{GC0310_8BIT, 0x15, 0x12}, // 0x10 //DPHYY_MODE read_ready
-	{GC0310_8BIT, 0x17, 0x01},
-	{GC0310_8BIT, 0x40, 0x08},
-	{GC0310_8BIT, 0x41, 0x00},
-	{GC0310_8BIT, 0x42, 0x00},
-	{GC0310_8BIT, 0x43, 0x00},
-	{GC0310_8BIT, 0x21, 0x02}, // 0x01
-	{GC0310_8BIT, 0x22, 0x02}, // 0x01
-	{GC0310_8BIT, 0x23, 0x01}, // 0x05 //Nor:0x05 DOU:0x06
-	{GC0310_8BIT, 0x29, 0x00},
-	{GC0310_8BIT, 0x2A, 0x25}, // 0x05 //data zero 0x7a de
-	{GC0310_8BIT, 0x2B, 0x02},
-
-	{GC0310_8BIT, 0xfe, 0x00},
+	{ 0xfe, 0x03 },
+	{ 0x01, 0x03 }, /* mipi 1lane */
+	{ 0x02, 0x22 }, /* 0x33 */
+	{ 0x03, 0x94 },
+	{ 0x04, 0x01 }, /* fifo_prog */
+	{ 0x05, 0x00 }, /* fifo_prog */
+	{ 0x06, 0x80 }, /* b0  //YUV ISP data */
+	{ 0x11, 0x2a }, /* 1e //LDI set YUV422 */
+	{ 0x12, 0x90 }, /* 00 //04 //00 //04//00 //LWC[7:0] */
+	{ 0x13, 0x02 }, /* 05 //05 //LWC[15:8] */
+	{ 0x15, 0x12 }, /* 0x10 //DPHYY_MODE read_ready */
+	{ 0x17, 0x01 },
+	{ 0x40, 0x08 },
+	{ 0x41, 0x00 },
+	{ 0x42, 0x00 },
+	{ 0x43, 0x00 },
+	{ 0x21, 0x02 }, /* 0x01 */
+	{ 0x22, 0x02 }, /* 0x01 */
+	{ 0x23, 0x01 }, /* 0x05 //Nor:0x05 DOU:0x06 */
+	{ 0x29, 0x00 },
+	{ 0x2A, 0x25 }, /* 0x05 //data zero 0x7a de */
+	{ 0x2B, 0x02 },
+
+	{ 0xfe, 0x00 },
 
 /////////////////////////////////////////////////
 /////////////////	CISCTL reg	/////////////////
 /////////////////////////////////////////////////
-	{GC0310_8BIT, 0x00, 0x2f}, //2f//0f//02//01
-	{GC0310_8BIT, 0x01, 0x0f}, //06
-	{GC0310_8BIT, 0x02, 0x04},
-	{GC0310_8BIT, 0x4f, 0x00}, //AEC 0FF
-	{GC0310_8BIT, 0x03, 0x01}, // 0x03 //04
-	{GC0310_8BIT, 0x04, 0xc0}, // 0xe8 //58
-	{GC0310_8BIT, 0x05, 0x00},
-	{GC0310_8BIT, 0x06, 0xb2}, // 0x0a //HB
-	{GC0310_8BIT, 0x07, 0x00},
-	{GC0310_8BIT, 0x08, 0x0c}, // 0x89 //VB
-	{GC0310_8BIT, 0x09, 0x00}, //row start
-	{GC0310_8BIT, 0x0a, 0x00}, //
-	{GC0310_8BIT, 0x0b, 0x00}, //col start
-	{GC0310_8BIT, 0x0c, 0x00},
-	{GC0310_8BIT, 0x0d, 0x01}, //height
-	{GC0310_8BIT, 0x0e, 0xf2}, // 0xf7 //height
-	{GC0310_8BIT, 0x0f, 0x02}, //width
-	{GC0310_8BIT, 0x10, 0x94}, // 0xa0 //height
-	{GC0310_8BIT, 0x17, 0x14},
-	{GC0310_8BIT, 0x18, 0x1a}, //0a//[4]double reset
-	{GC0310_8BIT, 0x19, 0x14}, //AD pipeline
-	{GC0310_8BIT, 0x1b, 0x48},
-	{GC0310_8BIT, 0x1e, 0x6b}, //3b//col bias
-	{GC0310_8BIT, 0x1f, 0x28}, //20//00//08//txlow
-	{GC0310_8BIT, 0x20, 0x89}, //88//0c//[3:2]DA15
-	{GC0310_8BIT, 0x21, 0x49}, //48//[3] txhigh
-	{GC0310_8BIT, 0x22, 0xb0},
-	{GC0310_8BIT, 0x23, 0x04}, //[1:0]vcm_r
-	{GC0310_8BIT, 0x24, 0x16}, //15
-	{GC0310_8BIT, 0x34, 0x20}, //[6:4] rsg high//range
+	{ 0x00, 0x2f }, /* 2f//0f//02//01 */
+	{ 0x01, 0x0f }, /* 06 */
+	{ 0x02, 0x04 },
+	{ 0x4f, 0x00 }, /* AEC 0FF */
+	{ 0x03, 0x01 }, /* 0x03 //04 */
+	{ 0x04, 0xc0 }, /* 0xe8 //58 */
+	{ 0x05, 0x00 },
+	{ 0x06, 0xb2 }, /* 0x0a //HB */
+	{ 0x07, 0x00 },
+	{ 0x08, 0x0c }, /* 0x89 //VB */
+	{ 0x09, 0x00 }, /* row start */
+	{ 0x0a, 0x00 },
+	{ 0x0b, 0x00 }, /* col start */
+	{ 0x0c, 0x00 },
+	{ 0x0d, 0x01 }, /* height */
+	{ 0x0e, 0xf2 }, /* 0xf7 //height */
+	{ 0x0f, 0x02 }, /* width */
+	{ 0x10, 0x94 }, /* 0xa0 //height */
+	{ 0x17, 0x14 },
+	{ 0x18, 0x1a }, /* 0a//[4]double reset */
+	{ 0x19, 0x14 }, /* AD pipeline */
+	{ 0x1b, 0x48 },
+	{ 0x1e, 0x6b }, /* 3b//col bias */
+	{ 0x1f, 0x28 }, /* 20//00//08//txlow */
+	{ 0x20, 0x89 }, /* 88//0c//[3:2]DA15 */
+	{ 0x21, 0x49 }, /* 48//[3] txhigh */
+	{ 0x22, 0xb0 },
+	{ 0x23, 0x04 }, /* [1:0]vcm_r */
+	{ 0x24, 0x16 }, /* 15 */
+	{ 0x34, 0x20 }, /* [6:4] rsg high//range */
 
 /////////////////////////////////////////////////
 ////////////////////   BLK	 ////////////////////
 /////////////////////////////////////////////////
-	{GC0310_8BIT, 0x26, 0x23}, //[1]dark_current_en [0]offset_en
-	{GC0310_8BIT, 0x28, 0xff}, //BLK_limie_value
-	{GC0310_8BIT, 0x29, 0x00}, //global offset
-	{GC0310_8BIT, 0x33, 0x18}, //offset_ratio
-	{GC0310_8BIT, 0x37, 0x20}, //dark_current_ratio
-	{GC0310_8BIT, 0x2a, 0x00},
-	{GC0310_8BIT, 0x2b, 0x00},
-	{GC0310_8BIT, 0x2c, 0x00},
-	{GC0310_8BIT, 0x2d, 0x00},
-	{GC0310_8BIT, 0x2e, 0x00},
-	{GC0310_8BIT, 0x2f, 0x00},
-	{GC0310_8BIT, 0x30, 0x00},
-	{GC0310_8BIT, 0x31, 0x00},
-	{GC0310_8BIT, 0x47, 0x80}, //a7
-	{GC0310_8BIT, 0x4e, 0x66}, //select_row
-	{GC0310_8BIT, 0xa8, 0x02}, //win_width_dark, same with crop_win_width
-	{GC0310_8BIT, 0xa9, 0x80},
+	{ 0x26, 0x23 }, /* [1]dark_current_en [0]offset_en */
+	{ 0x28, 0xff }, /* BLK_limie_value */
+	{ 0x29, 0x00 }, /* global offset */
+	{ 0x33, 0x18 }, /* offset_ratio */
+	{ 0x37, 0x20 }, /* dark_current_ratio */
+	{ 0x2a, 0x00 },
+	{ 0x2b, 0x00 },
+	{ 0x2c, 0x00 },
+	{ 0x2d, 0x00 },
+	{ 0x2e, 0x00 },
+	{ 0x2f, 0x00 },
+	{ 0x30, 0x00 },
+	{ 0x31, 0x00 },
+	{ 0x47, 0x80 }, /* a7 */
+	{ 0x4e, 0x66 }, /* select_row */
+	{ 0xa8, 0x02 }, /* win_width_dark, same with crop_win_width */
+	{ 0xa9, 0x80 },
 
 /////////////////////////////////////////////////
 //////////////////	 ISP reg  ///////////////////
 /////////////////////////////////////////////////
-	{GC0310_8BIT, 0x40, 0x06}, // 0xff //ff //48
-	{GC0310_8BIT, 0x41, 0x00}, // 0x21 //00//[0]curve_en
-	{GC0310_8BIT, 0x42, 0x04}, // 0xcf //0a//[1]awn_en
-	{GC0310_8BIT, 0x44, 0x18}, // 0x18 //02
-	{GC0310_8BIT, 0x46, 0x02}, // 0x03 //sync
-	{GC0310_8BIT, 0x49, 0x03},
-	{GC0310_8BIT, 0x4c, 0x20}, //00[5]pretect exp
-	{GC0310_8BIT, 0x50, 0x01}, //crop enable
-	{GC0310_8BIT, 0x51, 0x00},
-	{GC0310_8BIT, 0x52, 0x00},
-	{GC0310_8BIT, 0x53, 0x00},
-	{GC0310_8BIT, 0x54, 0x01},
-	{GC0310_8BIT, 0x55, 0x01}, //crop window height
-	{GC0310_8BIT, 0x56, 0xf0},
-	{GC0310_8BIT, 0x57, 0x02}, //crop window width
-	{GC0310_8BIT, 0x58, 0x90},
+	{ 0x40, 0x06 }, /* 0xff //ff //48 */
+	{ 0x41, 0x00 }, /* 0x21 //00//[0]curve_en */
+	{ 0x42, 0x04 }, /* 0xcf //0a//[1]awn_en */
+	{ 0x44, 0x18 }, /* 0x18 //02 */
+	{ 0x46, 0x02 }, /* 0x03 //sync */
+	{ 0x49, 0x03 },
+	{ 0x4c, 0x20 }, /* 00[5]pretect exp */
+	{ 0x50, 0x01 }, /* crop enable */
+	{ 0x51, 0x00 },
+	{ 0x52, 0x00 },
+	{ 0x53, 0x00 },
+	{ 0x54, 0x01 },
+	{ 0x55, 0x01 }, /* crop window height */
+	{ 0x56, 0xf0 },
+	{ 0x57, 0x02 }, /* crop window width */
+	{ 0x58, 0x90 },
 
 /////////////////////////////////////////////////
 ///////////////////   GAIN	 ////////////////////
 /////////////////////////////////////////////////
-	{GC0310_8BIT, 0x70, 0x70}, //70 //80//global gain
-	{GC0310_8BIT, 0x71, 0x20}, // pregain gain
-	{GC0310_8BIT, 0x72, 0x40}, // post gain
-	{GC0310_8BIT, 0x5a, 0x84}, //84//analog gain 0
-	{GC0310_8BIT, 0x5b, 0xc9}, //c9
-	{GC0310_8BIT, 0x5c, 0xed}, //ed//not use pga gain highest level
-	{GC0310_8BIT, 0x77, 0x40}, // R gain 0x74 //awb gain
-	{GC0310_8BIT, 0x78, 0x40}, // G gain
-	{GC0310_8BIT, 0x79, 0x40}, // B gain 0x5f
-
-	{GC0310_8BIT, 0x48, 0x00},
-	{GC0310_8BIT, 0xfe, 0x01},
-	{GC0310_8BIT, 0x0a, 0x45}, //[7]col gain mode
-
-	{GC0310_8BIT, 0x3e, 0x40},
-	{GC0310_8BIT, 0x3f, 0x5c},
-	{GC0310_8BIT, 0x40, 0x7b},
-	{GC0310_8BIT, 0x41, 0xbd},
-	{GC0310_8BIT, 0x42, 0xf6},
-	{GC0310_8BIT, 0x43, 0x63},
-	{GC0310_8BIT, 0x03, 0x60},
-	{GC0310_8BIT, 0x44, 0x03},
+	{ 0x70, 0x70 }, /* 70 //80//global gain */
+	{ 0x71, 0x20 }, /* pregain gain */
+	{ 0x72, 0x40 }, /* post gain */
+	{ 0x5a, 0x84 }, /* 84//analog gain 0  */
+	{ 0x5b, 0xc9 }, /* c9 */
+	{ 0x5c, 0xed }, /* ed//not use pga gain highest level */
+	{ 0x77, 0x40 }, /* R gain 0x74 //awb gain */
+	{ 0x78, 0x40 }, /* G gain */
+	{ 0x79, 0x40 }, /* B gain 0x5f */
+
+	{ 0x48, 0x00 },
+	{ 0xfe, 0x01 },
+	{ 0x0a, 0x45 }, /* [7]col gain mode */
+
+	{ 0x3e, 0x40 },
+	{ 0x3f, 0x5c },
+	{ 0x40, 0x7b },
+	{ 0x41, 0xbd },
+	{ 0x42, 0xf6 },
+	{ 0x43, 0x63 },
+	{ 0x03, 0x60 },
+	{ 0x44, 0x03 },
 
 /////////////////////////////////////////////////
 /////////////////	dark sun   //////////////////
 /////////////////////////////////////////////////
-	{GC0310_8BIT, 0xfe, 0x01},
-	{GC0310_8BIT, 0x45, 0xa4}, // 0xf7
-	{GC0310_8BIT, 0x46, 0xf0}, // 0xff //f0//sun value th
-	{GC0310_8BIT, 0x48, 0x03}, //sun mode
-	{GC0310_8BIT, 0x4f, 0x60}, //sun_clamp
-	{GC0310_8BIT, 0xfe, 0x00},
-
-	{GC0310_TOK_TERM, 0, 0},
+	{ 0xfe, 0x01 },
+	{ 0x45, 0xa4 }, /* 0xf7 */
+	{ 0x46, 0xf0 }, /* 0xff //f0//sun value th */
+	{ 0x48, 0x03 }, /* sun mode */
+	{ 0x4f, 0x60 }, /* sun_clamp */
+	{ 0xfe, 0x00 },
 };
 
 static struct gc0310_reg const gc0310_VGA_30fps[] = {
-	{GC0310_8BIT, 0xfe, 0x00},
-	{GC0310_8BIT, 0x0d, 0x01}, //height
-	{GC0310_8BIT, 0x0e, 0xf2}, // 0xf7 //height
-	{GC0310_8BIT, 0x0f, 0x02}, //width
-	{GC0310_8BIT, 0x10, 0x94}, // 0xa0 //height
-
-	{GC0310_8BIT, 0x50, 0x01}, //crop enable
-	{GC0310_8BIT, 0x51, 0x00},
-	{GC0310_8BIT, 0x52, 0x00},
-	{GC0310_8BIT, 0x53, 0x00},
-	{GC0310_8BIT, 0x54, 0x01},
-	{GC0310_8BIT, 0x55, 0x01}, //crop window height
-	{GC0310_8BIT, 0x56, 0xf0},
-	{GC0310_8BIT, 0x57, 0x02}, //crop window width
-	{GC0310_8BIT, 0x58, 0x90},
-
-	{GC0310_8BIT, 0xfe, 0x03},
-	{GC0310_8BIT, 0x12, 0x90},//00 //04 //00 //04//00 //LWC[7:0]  //
-	{GC0310_8BIT, 0x13, 0x02},//05 //05 //LWC[15:8]
-
-	{GC0310_8BIT, 0xfe, 0x00},
-
-	{GC0310_TOK_TERM, 0, 0},
-};
-
-static struct gc0310_resolution gc0310_res_preview[] = {
-	{
-		.desc = "gc0310_VGA_30fps",
-		.width = 656, // 648,
-		.height = 496, // 488,
-		.fps = 30,
-		//.pix_clk_freq = 73,
-		.used = 0,
-#if 0
-		.pixels_per_line = 0x0314,
-		.lines_per_frame = 0x0213,
-#endif
-		.skip_frames = 2,
-		.regs = gc0310_VGA_30fps,
-	},
+	{ 0xfe, 0x00 },
+	{ 0x0d, 0x01 }, /* height */
+	{ 0x0e, 0xf2 }, /* 0xf7 //height */
+	{ 0x0f, 0x02 }, /* width */
+	{ 0x10, 0x94 }, /* 0xa0 //height */
+
+	{ 0x50, 0x01 }, /* crop enable */
+	{ 0x51, 0x00 },
+	{ 0x52, 0x00 },
+	{ 0x53, 0x00 },
+	{ 0x54, 0x01 },
+	{ 0x55, 0x01 }, /* crop window height */
+	{ 0x56, 0xf0 },
+	{ 0x57, 0x02 }, /* crop window width */
+	{ 0x58, 0x90 },
+
+	{ 0xfe, 0x03 },
+	{ 0x12, 0x90 }, /* 00 //04 //00 //04//00 //LWC[7:0]  */
+	{ 0x13, 0x02 }, /* 05 //05 //LWC[15:8] */
+
+	{ 0xfe, 0x00 },
 };
 
-#define N_RES_PREVIEW (ARRAY_SIZE(gc0310_res_preview))
-
-static struct gc0310_resolution *gc0310_res = gc0310_res_preview;
-static unsigned long N_RES = N_RES_PREVIEW;
 #endif
diff --git a/drivers/staging/media/atomisp/i2c/ov2680.h b/drivers/staging/media/atomisp/i2c/ov2680.h
index a37af0a74a53..baf49eb0659e 100644
--- a/drivers/staging/media/atomisp/i2c/ov2680.h
+++ b/drivers/staging/media/atomisp/i2c/ov2680.h
@@ -114,7 +114,6 @@ struct ov2680_device {
 	struct mutex input_lock;
 	struct i2c_client *client;
 	struct gpio_desc *powerdown;
-	struct gpio_desc *powerdown_alt;
 	bool is_streaming;
 
 	struct ov2680_mode {
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp.h b/drivers/staging/media/atomisp/include/linux/atomisp.h
index 63b1bcd35399..c7ec56a1c064 100644
--- a/drivers/staging/media/atomisp/include/linux/atomisp.h
+++ b/drivers/staging/media/atomisp/include/linux/atomisp.h
@@ -299,26 +299,6 @@ struct atomisp_3a_statistics {
 	u32 isp_config_id; /* isp config ID */
 };
 
-/**
- * struct atomisp_cont_capture_conf - continuous capture parameters
- * @num_captures: number of still images to capture
- * @skip_frames: number of frames to skip between 2 captures
- * @offset: offset in ring buffer to start capture
- *
- * For example, to capture 1 frame from past, current, and 1 from future
- * and skip one frame between each capture, parameters would be:
- * num_captures:3
- * skip_frames:1
- * offset:-2
- */
-
-struct atomisp_cont_capture_conf {
-	int num_captures;
-	unsigned int skip_frames;
-	int offset;
-	__u32 reserved[5];
-};
-
 struct atomisp_ae_window {
 	int x_left;
 	int x_right;
@@ -958,9 +938,6 @@ struct atomisp_sensor_ae_bracketing_lut {
 #define ATOMISP_IOC_S_PARAMETERS \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 32, struct atomisp_parameters)
 
-#define ATOMISP_IOC_S_CONT_CAPTURE_CONFIG \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 33, struct atomisp_cont_capture_conf)
-
 #define ATOMISP_IOC_G_METADATA \
 	_IOWR('v', BASE_VIDIOC_PRIVATE + 34, struct atomisp_metadata)
 
@@ -1079,8 +1056,6 @@ struct atomisp_sensor_ae_bracketing_lut {
 #define V4L2_2A_STATUS_AE_READY            BIT(0)
 #define V4L2_2A_STATUS_AWB_READY           BIT(1)
 
-#define V4L2_CID_FMT_AUTO			(V4L2_CID_CAMERA_LASTP1 + 19)
-
 #define V4L2_CID_RUN_MODE			(V4L2_CID_CAMERA_LASTP1 + 20)
 #define ATOMISP_RUN_MODE_VIDEO			1
 #define ATOMISP_RUN_MODE_STILL_CAPTURE		2
@@ -1107,8 +1082,6 @@ struct atomisp_sensor_ae_bracketing_lut {
 /* Lock and unlock raw buffer */
 #define V4L2_CID_ENABLE_RAW_BUFFER_LOCK (V4L2_CID_CAMERA_LASTP1 + 29)
 
-#define V4L2_CID_DEPTH_MODE		(V4L2_CID_CAMERA_LASTP1 + 30)
-
 #define V4L2_CID_EXPOSURE_ZONE_NUM	(V4L2_CID_CAMERA_LASTP1 + 31)
 /* Disable digital zoom */
 #define V4L2_CID_DISABLE_DZ		(V4L2_CID_CAMERA_LASTP1 + 32)
@@ -1127,7 +1100,6 @@ struct atomisp_sensor_ae_bracketing_lut {
 
 #define V4L2_EVENT_ATOMISP_3A_STATS_READY   (V4L2_EVENT_PRIVATE_START + 1)
 #define V4L2_EVENT_ATOMISP_METADATA_READY   (V4L2_EVENT_PRIVATE_START + 2)
-#define V4L2_EVENT_ATOMISP_RAW_BUFFERS_ALLOC_DONE   (V4L2_EVENT_PRIVATE_START + 3)
 #define V4L2_EVENT_ATOMISP_ACC_COMPLETE     (V4L2_EVENT_PRIVATE_START + 4)
 #define V4L2_EVENT_ATOMISP_PAUSE_BUFFER	    (V4L2_EVENT_PRIVATE_START + 5)
 #define V4L2_EVENT_ATOMISP_CSS_RESET	    (V4L2_EVENT_PRIVATE_START + 6)
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
index 539b21d39d3b..e8e965f73fc8 100644
--- a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
+++ b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
@@ -111,12 +111,9 @@ enum atomisp_input_format {
 
 enum intel_v4l2_subdev_type {
 	RAW_CAMERA = 1,
-	SOC_CAMERA = 2,
-	CAMERA_MOTOR = 3,
-	LED_FLASH = 4,
-	XENON_FLASH = 5,
-	FILE_INPUT = 6,
-	TEST_PATTERN = 7,
+	CAMERA_MOTOR = 2,
+	LED_FLASH = 3,
+	TEST_PATTERN = 4,
 };
 
 struct intel_v4l2_subdev_id {
@@ -216,6 +213,8 @@ int atomisp_register_sensor_no_gmin(struct v4l2_subdev *subdev, u32 lanes,
 				    enum atomisp_bayer_order bayer_order);
 void atomisp_unregister_subdev(struct v4l2_subdev *subdev);
 
+int v4l2_get_acpi_sensor_info(struct device *dev, char **module_id_str);
+
 /* API from old platform_camera.h, new CPUID implementation */
 #define __IS_SOC(x) (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && \
 		     boot_cpu_data.x86 == 6 &&                       \
diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.c b/drivers/staging/media/atomisp/pci/atomisp_cmd.c
index 47f18ac5e40e..aa790ae746f3 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_cmd.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.c
@@ -79,8 +79,6 @@ union host {
 	} ptr;
 };
 
-static int atomisp_set_raw_buffer_bitmap(struct atomisp_sub_device *asd, int exp_id);
-
 /*
  * get sensor:dis71430/ov2720 related info from v4l2_subdev->priv data field.
  * subdev->priv is set in mrst.c
@@ -198,19 +196,12 @@ int atomisp_freq_scaling(struct atomisp_device *isp,
 			 enum atomisp_dfs_mode mode,
 			 bool force)
 {
-	struct pci_dev *pdev = to_pci_dev(isp->dev);
-	/* FIXME! Only use subdev[0] status yet */
-	struct atomisp_sub_device *asd = &isp->asd[0];
 	const struct atomisp_dfs_config *dfs;
 	unsigned int new_freq;
 	struct atomisp_freq_scaling_rule curr_rules;
 	int i, ret;
 	unsigned short fps = 0;
 
-	if ((pdev->device & ATOMISP_PCI_DEVICE_SOC_MASK) ==
-	    ATOMISP_PCI_DEVICE_SOC_CHT && ATOMISP_USE_YUVPP(asd))
-		isp->dfs = &dfs_config_cht_soc;
-
 	dfs = isp->dfs;
 
 	if (dfs->lowest_freq == 0 || dfs->max_freq_at_vmin == 0 ||
@@ -230,7 +221,7 @@ int atomisp_freq_scaling(struct atomisp_device *isp,
 		goto done;
 	}
 
-	fps = atomisp_get_sensor_fps(asd);
+	fps = atomisp_get_sensor_fps(&isp->asd);
 	if (fps == 0) {
 		dev_info(isp->dev,
 			 "Sensor didn't report FPS. Using DFS max mode.\n");
@@ -238,22 +229,10 @@ int atomisp_freq_scaling(struct atomisp_device *isp,
 		goto done;
 	}
 
-	curr_rules.width = asd->fmt[asd->capture_pad].fmt.width;
-	curr_rules.height = asd->fmt[asd->capture_pad].fmt.height;
+	curr_rules.width = isp->asd.fmt[isp->asd.capture_pad].fmt.width;
+	curr_rules.height = isp->asd.fmt[isp->asd.capture_pad].fmt.height;
 	curr_rules.fps = fps;
-	curr_rules.run_mode = asd->run_mode->val;
-	/*
-	 * For continuous mode, we need to make the capture setting applied
-	 * since preview mode, because there is no chance to do this when
-	 * starting image capture.
-	 */
-	if (asd->continuous_mode->val) {
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-			curr_rules.run_mode = ATOMISP_RUN_MODE_SDV;
-		else
-			curr_rules.run_mode =
-			    ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE;
-	}
+	curr_rules.run_mode = isp->asd.run_mode->val;
 
 	/* search for the target frequency by looping freq rules*/
 	for (i = 0; i < dfs->dfs_table_size; i++) {
@@ -470,37 +449,13 @@ static void clear_irq_reg(struct atomisp_device *isp)
 	pci_write_config_dword(pdev, PCI_INTERRUPT_CTRL, msg_ret);
 }
 
-static struct atomisp_sub_device *
-__get_asd_from_port(struct atomisp_device *isp, enum mipi_port_id port)
-{
-	int i;
-
-	/* Check which isp subdev to send eof */
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-		struct camera_mipi_info *mipi_info;
-
-		mipi_info = atomisp_to_sensor_mipi_info(
-				isp->inputs[asd->input_curr].camera);
-
-		if (asd->streaming == ATOMISP_DEVICE_STREAMING_ENABLED &&
-		    __get_mipi_port(isp, mipi_info->port) == port) {
-			return asd;
-		}
-	}
-
-	return NULL;
-}
-
 /* interrupt handling function*/
 irqreturn_t atomisp_isr(int irq, void *dev)
 {
 	struct atomisp_device *isp = (struct atomisp_device *)dev;
-	struct atomisp_sub_device *asd;
 	struct atomisp_css_event eof_event;
 	unsigned int irq_infos = 0;
 	unsigned long flags;
-	unsigned int i;
 	int err;
 
 	spin_lock_irqsave(&isp->lock, flags);
@@ -520,18 +475,10 @@ irqreturn_t atomisp_isr(int irq, void *dev)
 	if (!atomisp_streaming_count(isp))
 		goto out_nowake;
 
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-
-		if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-			continue;
-		/*
-		 * Current SOF only support one stream, so the SOF only valid
-		 * either solely one stream is running
-		 */
+	if (isp->asd.streaming == ATOMISP_DEVICE_STREAMING_ENABLED) {
 		if (irq_infos & IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF) {
-			atomic_inc(&asd->sof_count);
-			atomisp_sof_event(asd);
+			atomic_inc(&isp->asd.sof_count);
+			atomisp_sof_event(&isp->asd);
 
 			/* If sequence_temp and sequence are the same
 			 * there where no frames lost so we can increase
@@ -541,14 +488,14 @@ irqreturn_t atomisp_isr(int irq, void *dev)
 			 * NOTE: There is assumption here that ISP will not
 			 * start processing next frame from sensor before old
 			 * one is completely done. */
-			if (atomic_read(&asd->sequence) == atomic_read(
-				&asd->sequence_temp))
-				atomic_set(&asd->sequence_temp,
-					   atomic_read(&asd->sof_count));
+			if (atomic_read(&isp->asd.sequence) ==
+			    atomic_read(&isp->asd.sequence_temp))
+				atomic_set(&isp->asd.sequence_temp,
+					   atomic_read(&isp->asd.sof_count));
 		}
 		if (irq_infos & IA_CSS_IRQ_INFO_EVENTS_READY)
-			atomic_set(&asd->sequence,
-				   atomic_read(&asd->sequence_temp));
+			atomic_set(&isp->asd.sequence,
+				   atomic_read(&isp->asd.sequence_temp));
 	}
 
 	if (irq_infos & IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF) {
@@ -573,21 +520,10 @@ irqreturn_t atomisp_isr(int irq, void *dev)
 	}
 
 	if (irq_infos & IA_CSS_IRQ_INFO_ISYS_EVENTS_READY) {
-		while (ia_css_dequeue_isys_event(&eof_event.event) ==
-		       0) {
-			/* EOF Event does not have the css_pipe returned */
-			asd = __get_asd_from_port(isp, eof_event.event.port);
-			if (!asd) {
-				dev_err(isp->dev, "%s: ISYS event, but no subdev.event:%d",
-					__func__, eof_event.event.type);
-				continue;
-			}
-
-			atomisp_eof_event(asd, eof_event.event.exp_id);
-			dev_dbg_ratelimited(isp->dev,
-					    "%s ISYS event: EOF exp_id %d, asd %d\n",
-					    __func__, eof_event.event.exp_id,
-					    asd->index);
+		while (ia_css_dequeue_isys_event(&eof_event.event) == 0) {
+			atomisp_eof_event(&isp->asd, eof_event.event.exp_id);
+			dev_dbg_ratelimited(isp->dev, "ISYS event: EOF exp_id %d\n",
+					    eof_event.event.exp_id);
 		}
 
 		irq_infos &= ~IA_CSS_IRQ_INFO_ISYS_EVENTS_READY;
@@ -742,19 +678,6 @@ static void atomisp_recover_params_queue(struct atomisp_video_pipe *pipe)
 	atomisp_handle_parameter_and_buffer(pipe);
 }
 
-enum atomisp_metadata_type
-atomisp_get_metadata_type(struct atomisp_sub_device *asd,
-			  enum ia_css_pipe_id pipe_id)
-{
-	if (!asd->continuous_mode->val)
-		return ATOMISP_MAIN_METADATA;
-
-	if (pipe_id == IA_CSS_PIPE_ID_CAPTURE) /* online capture pipe */
-		return ATOMISP_SEC_METADATA;
-	else
-		return ATOMISP_MAIN_METADATA;
-}
-
 void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		      enum ia_css_buffer_type buf_type,
 		      enum ia_css_pipe_id css_pipe_id,
@@ -826,7 +749,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		if (error)
 			break;
 
-		md_type = atomisp_get_metadata_type(asd, css_pipe_id);
+		md_type = ATOMISP_MAIN_METADATA;
 		list_for_each_entry_safe(md_iter, _md_buf_tmp,
 					 &asd->metadata_in_css[md_type], list) {
 			if (md_iter->metadata ==
@@ -883,15 +806,6 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 
 		pipe = vb_to_pipe(&frame->vb.vb2_buf);
 
-		/* FIXME:
-		 * YUVPP doesn't set postview exp_id correctlly in SDV mode.
-		 * This is a WORKAROUND to set exp_id. see HSDES-1503911606.
-		 */
-		if (IS_BYT && buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME &&
-		    asd->continuous_mode->val && ATOMISP_USE_YUVPP(asd))
-			frame->exp_id = (asd->postview_exp_id++) %
-					(ATOMISP_MAX_EXP_ID + 1);
-
 		dev_dbg(isp->dev, "%s: vf frame with exp_id %d is ready\n",
 			__func__, frame->exp_id);
 		if (asd->params.flash_state == ATOMISP_FLASH_ONGOING) {
@@ -908,18 +822,6 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 					__func__);
 		}
 		pipe->frame_config_id[frame->vb.vb2_buf.index] = frame->isp_config_id;
-		if (css_pipe_id == IA_CSS_PIPE_ID_CAPTURE &&
-		    asd->pending_capture_request > 0) {
-			err = atomisp_css_offline_capture_configure(asd,
-				asd->params.offline_parm.num_captures,
-				asd->params.offline_parm.skip_frames,
-				asd->params.offline_parm.offset);
-
-			asd->pending_capture_request--;
-
-			dev_dbg(isp->dev, "Trigger capture again for new buffer. err=%d\n",
-				err);
-		}
 		break;
 	case IA_CSS_BUFFER_TYPE_OUTPUT_FRAME:
 	case IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
@@ -934,15 +836,6 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 
 		pipe = vb_to_pipe(&frame->vb.vb2_buf);
 
-		/* FIXME:
-		 * YUVPP doesn't set preview exp_id correctlly in ZSL mode.
-		 * This is a WORKAROUND to set exp_id. see HSDES-1503911606.
-		 */
-		if (IS_BYT && buf_type == IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME &&
-		    asd->continuous_mode->val && ATOMISP_USE_YUVPP(asd))
-			frame->exp_id = (asd->preview_exp_id++) %
-					(ATOMISP_MAX_EXP_ID + 1);
-
 		dev_dbg(isp->dev, "%s: main frame with exp_id %d is ready\n",
 			__func__, frame->exp_id);
 
@@ -993,35 +886,6 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 
 		asd->params.last_frame_status = asd->frame_status[i];
 
-		if (asd->continuous_mode->val) {
-			if (css_pipe_id == IA_CSS_PIPE_ID_PREVIEW ||
-			    css_pipe_id == IA_CSS_PIPE_ID_VIDEO) {
-				asd->latest_preview_exp_id = frame->exp_id;
-			} else if (css_pipe_id ==
-				   IA_CSS_PIPE_ID_CAPTURE) {
-				if (asd->run_mode->val ==
-				    ATOMISP_RUN_MODE_VIDEO)
-					dev_dbg(isp->dev, "SDV capture raw buffer id: %u\n",
-						frame->exp_id);
-				else
-					dev_dbg(isp->dev, "ZSL capture raw buffer id: %u\n",
-						frame->exp_id);
-			}
-		}
-		/*
-		 * Only after enabled the raw buffer lock
-		 * and in continuous mode.
-		 * in preview/video pipe, each buffer will
-		 * be locked automatically, so record it here.
-		 */
-		if (((css_pipe_id == IA_CSS_PIPE_ID_PREVIEW) ||
-		     (css_pipe_id == IA_CSS_PIPE_ID_VIDEO)) &&
-		    asd->enable_raw_buffer_lock->val &&
-		    asd->continuous_mode->val) {
-			atomisp_set_raw_buffer_bitmap(asd, frame->exp_id);
-			WARN_ON(frame->exp_id > ATOMISP_MAX_EXP_ID);
-		}
-
 		if (asd->params.css_update_params_needed) {
 			atomisp_apply_css_parameters(asd,
 						     &asd->params.css_param);
@@ -1080,43 +944,13 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		atomisp_qbuffers_to_css(asd);
 }
 
-void atomisp_delayed_init_work(struct work_struct *work)
-{
-	struct atomisp_sub_device *asd = container_of(work,
-					 struct atomisp_sub_device,
-					 delayed_init_work);
-	/*
-	 * to SOC camera, use yuvpp pipe and no support continuous mode.
-	 */
-	if (!ATOMISP_USE_YUVPP(asd)) {
-		struct v4l2_event event = {0};
-		struct ia_css_stream *stream;
-
-		stream = asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream;
-
-
-		if (ia_css_alloc_continuous_frame_remain(stream))
-			return;
-
-		ia_css_update_continuous_frames(stream);
-
-		event.type = V4L2_EVENT_ATOMISP_RAW_BUFFERS_ALLOC_DONE;
-		v4l2_event_queue(asd->subdev.devnode, &event);
-	}
-
-	/* signal streamon after delayed init is done */
-	asd->delayed_init = ATOMISP_DELAYED_INIT_DONE;
-	complete(&asd->init_done);
-}
-
 static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout)
 {
 	struct pci_dev *pdev = to_pci_dev(isp->dev);
 	enum ia_css_pipe_id css_pipe_id;
-	bool stream_restart[MAX_STREAM_NUM] = {0};
-	bool depth_mode = false;
-	int i, ret, depth_cnt = 0;
+	bool stream_restart = false;
 	unsigned long flags;
+	int ret;
 
 	lockdep_assert_held(&isp->mutex);
 
@@ -1125,52 +959,37 @@ static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout)
 
 	atomisp_css_irq_enable(isp, IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF, false);
 
-	BUG_ON(isp->num_of_streams > MAX_STREAM_NUM);
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-
-		if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED &&
-		    !asd->stream_prepared)
-			continue;
-
-		depth_cnt++;
-
-		if (asd->delayed_init == ATOMISP_DELAYED_INIT_QUEUED)
-			cancel_work_sync(&asd->delayed_init_work);
-
-		complete(&asd->init_done);
-		asd->delayed_init = ATOMISP_DELAYED_INIT_NOT_QUEUED;
-
-		stream_restart[asd->index] = true;
+	if (isp->asd.streaming == ATOMISP_DEVICE_STREAMING_ENABLED ||
+	    isp->asd.stream_prepared) {
+		stream_restart = true;
 
 		spin_lock_irqsave(&isp->lock, flags);
-		asd->streaming = ATOMISP_DEVICE_STREAMING_STOPPING;
+		isp->asd.streaming = ATOMISP_DEVICE_STREAMING_STOPPING;
 		spin_unlock_irqrestore(&isp->lock, flags);
 
 		/* stream off sensor */
 		ret = v4l2_subdev_call(
-			  isp->inputs[asd->input_curr].
+			  isp->inputs[isp->asd.input_curr].
 			  camera, video, s_stream, 0);
 		if (ret)
 			dev_warn(isp->dev,
 				 "can't stop streaming on sensor!\n");
 
-		atomisp_clear_css_buffer_counters(asd);
+		atomisp_clear_css_buffer_counters(&isp->asd);
 
-		css_pipe_id = atomisp_get_css_pipe_id(asd);
-		atomisp_css_stop(asd, css_pipe_id, true);
+		css_pipe_id = atomisp_get_css_pipe_id(&isp->asd);
+		atomisp_css_stop(&isp->asd, css_pipe_id, true);
 
 		spin_lock_irqsave(&isp->lock, flags);
-		asd->streaming = ATOMISP_DEVICE_STREAMING_DISABLED;
+		isp->asd.streaming = ATOMISP_DEVICE_STREAMING_DISABLED;
 		spin_unlock_irqrestore(&isp->lock, flags);
 
-		asd->preview_exp_id = 1;
-		asd->postview_exp_id = 1;
+		isp->asd.preview_exp_id = 1;
+		isp->asd.postview_exp_id = 1;
 		/* notify HAL the CSS reset */
 		dev_dbg(isp->dev,
-			"send reset event to %s\n", asd->subdev.devnode->name);
-		atomisp_reset_event(asd);
+			"send reset event to %s\n", isp->asd.subdev.devnode->name);
+		atomisp_reset_event(&isp->asd);
 	}
 
 	/* clear irq */
@@ -1186,34 +1005,20 @@ static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout)
 	atomisp_reset(isp);
 	isp->isp_timeout = false;
 
-	if (!isp_timeout) {
-		for (i = 0; i < isp->num_of_streams; i++) {
-			if (isp->asd[i].depth_mode->val)
-				return;
-		}
-	}
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
+	if (stream_restart) {
+		atomisp_css_input_set_mode(&isp->asd, IA_CSS_INPUT_MODE_BUFFERED_SENSOR);
 
-		if (!stream_restart[i])
-			continue;
-
-		if (isp->inputs[asd->input_curr].type != FILE_INPUT)
-			atomisp_css_input_set_mode(asd,
-						   IA_CSS_INPUT_MODE_BUFFERED_SENSOR);
-
-		css_pipe_id = atomisp_get_css_pipe_id(asd);
-		if (atomisp_css_start(asd, css_pipe_id, true)) {
+		css_pipe_id = atomisp_get_css_pipe_id(&isp->asd);
+		if (atomisp_css_start(&isp->asd, css_pipe_id, true)) {
 			dev_warn(isp->dev,
 				 "start SP failed, so do not set streaming to be enable!\n");
 		} else {
 			spin_lock_irqsave(&isp->lock, flags);
-			asd->streaming = ATOMISP_DEVICE_STREAMING_ENABLED;
+			isp->asd.streaming = ATOMISP_DEVICE_STREAMING_ENABLED;
 			spin_unlock_irqrestore(&isp->lock, flags);
 		}
 
-		atomisp_csi2_configure(asd);
+		atomisp_csi2_configure(&isp->asd);
 	}
 
 	atomisp_css_irq_enable(isp, IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF,
@@ -1222,51 +1027,25 @@ static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout)
 	if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_AUTO, true) < 0)
 		dev_dbg(isp->dev, "DFS auto failed while recovering!\n");
 
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd;
-
-		asd = &isp->asd[i];
-
-		if (!stream_restart[i])
-			continue;
-
-		if (asd->continuous_mode->val &&
-		    asd->delayed_init == ATOMISP_DELAYED_INIT_NOT_QUEUED) {
-			reinit_completion(&asd->init_done);
-			asd->delayed_init = ATOMISP_DELAYED_INIT_QUEUED;
-			queue_work(asd->delayed_init_workq,
-				   &asd->delayed_init_work);
-		}
+	if (stream_restart) {
 		/*
 		 * dequeueing buffers is not needed. CSS will recycle
 		 * buffers that it has.
 		 */
-		atomisp_flush_bufs_and_wakeup(asd);
+		atomisp_flush_bufs_and_wakeup(&isp->asd);
 
 		/* Requeue unprocessed per-frame parameters. */
-		atomisp_recover_params_queue(&asd->video_out_capture);
-		atomisp_recover_params_queue(&asd->video_out_preview);
-		atomisp_recover_params_queue(&asd->video_out_video_capture);
-
-		if ((asd->depth_mode->val) &&
-		    (depth_cnt == ATOMISP_DEPTH_SENSOR_STREAMON_COUNT)) {
-			depth_mode = true;
-			continue;
-		}
+		atomisp_recover_params_queue(&isp->asd.video_out_capture);
+		atomisp_recover_params_queue(&isp->asd.video_out_preview);
+		atomisp_recover_params_queue(&isp->asd.video_out_video_capture);
 
 		ret = v4l2_subdev_call(
-			  isp->inputs[asd->input_curr].camera, video,
+			  isp->inputs[isp->asd.input_curr].camera, video,
 			  s_stream, 1);
 		if (ret)
 			dev_warn(isp->dev,
 				 "can't start streaming on sensor!\n");
 	}
-
-	if (depth_mode) {
-		if (atomisp_stream_on_master_slave_sensor(isp, true))
-			dev_warn(isp->dev,
-				 "master slave sensor stream on failed!\n");
-	}
 }
 
 void atomisp_assert_recovery_work(struct work_struct *work)
@@ -1321,10 +1100,6 @@ irqreturn_t atomisp_isr_thread(int irq, void *isp_ptr)
 {
 	struct atomisp_device *isp = isp_ptr;
 	unsigned long flags;
-	bool frame_done_found[MAX_STREAM_NUM] = {0};
-	bool css_pipe_done[MAX_STREAM_NUM] = {0};
-	unsigned int i;
-	struct atomisp_sub_device *asd;
 
 	dev_dbg(isp->dev, ">%s\n", __func__);
 
@@ -1363,15 +1138,11 @@ irqreturn_t atomisp_isr_thread(int irq, void *isp_ptr)
 	 * time, instead, dequue one and process one, then another
 	 */
 	mutex_lock(&isp->mutex);
-	if (atomisp_css_isr_thread(isp, frame_done_found, css_pipe_done))
+	if (atomisp_css_isr_thread(isp))
 		goto out;
 
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-			continue;
-		atomisp_setup_flash(asd);
-	}
+	if (isp->asd.streaming == ATOMISP_DEVICE_STREAMING_ENABLED)
+		atomisp_setup_flash(&isp->asd);
 out:
 	mutex_unlock(&isp->mutex);
 	dev_dbg(isp->dev, "<%s\n", __func__);
@@ -1779,11 +1550,6 @@ int atomisp_formats(struct atomisp_sub_device *asd, int flag,
 void atomisp_free_internal_buffers(struct atomisp_sub_device *asd)
 {
 	atomisp_free_css_parameters(&asd->params.css_param);
-
-	if (asd->raw_output_frame) {
-		ia_css_frame_free(asd->raw_output_frame);
-		asd->raw_output_frame = NULL;
-	}
 }
 
 static void atomisp_update_grid_info(struct atomisp_sub_device *asd,
@@ -2023,8 +1789,6 @@ int atomisp_get_dvs2_bq_resolutions(struct atomisp_sub_device *asd,
 				    struct atomisp_dvs2_bq_resolutions *bq_res)
 {
 	struct ia_css_pipe_config *pipe_cfg = NULL;
-	struct ia_css_stream_config *stream_cfg = NULL;
-	struct ia_css_stream_input_config *input_config = NULL;
 
 	struct ia_css_stream *stream =
 		    asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream;
@@ -2035,9 +1799,6 @@ int atomisp_get_dvs2_bq_resolutions(struct atomisp_sub_device *asd,
 
 	pipe_cfg = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
 		   .pipe_configs[IA_CSS_PIPE_ID_VIDEO];
-	stream_cfg = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		     .stream_config;
-	input_config = &stream_cfg->input_config;
 
 	if (!bq_res)
 		return -EINVAL;
@@ -2049,132 +1810,27 @@ int atomisp_get_dvs2_bq_resolutions(struct atomisp_sub_device *asd,
 	bq_res->envelope_bq.width_bq = 0;
 	bq_res->envelope_bq.height_bq = 0;
 	/* the GDC input resolution */
-	if (!asd->continuous_mode->val) {
-		bq_res->source_bq.width_bq = bq_res->output_bq.width_bq +
-					     pipe_cfg->dvs_envelope.width / 2;
-		bq_res->source_bq.height_bq = bq_res->output_bq.height_bq +
-					      pipe_cfg->dvs_envelope.height / 2;
-		/*
-		 * Bad pixels caused by spatial filter processing
-		 * ISP filter resolution should be given by CSS/FW, but for now
-		 * there is not such API to query, and it is fixed value, so
-		 * hardcoded here.
-		 */
-		bq_res->ispfilter_bq.width_bq = 12 / 2;
-		bq_res->ispfilter_bq.height_bq = 12 / 2;
-		/* spatial filter shift, always 4 pixels */
-		bq_res->gdc_shift_bq.width_bq = 4 / 2;
-		bq_res->gdc_shift_bq.height_bq = 4 / 2;
-
-		if (asd->params.video_dis_en) {
-			bq_res->envelope_bq.width_bq = pipe_cfg->dvs_envelope.width
-						       / 2 - bq_res->ispfilter_bq.width_bq;
-			bq_res->envelope_bq.height_bq = pipe_cfg->dvs_envelope.height
-							/ 2 - bq_res->ispfilter_bq.height_bq;
-		}
-	} else {
-		unsigned int w_padding;
-		unsigned int gdc_effective_input = 0;
-
-		/* For GDC:
-		 * gdc_effective_input = effective_input + envelope
-		 *
-		 * From the comment and formula in BZ1786,
-		 * we see the source_bq should be:
-		 * effective_input / bayer_ds_ratio
-		 */
-		bq_res->source_bq.width_bq =
-		    (input_config->effective_res.width *
-		     pipe_cfg->bayer_ds_out_res.width /
-		     input_config->effective_res.width + 1) / 2;
-		bq_res->source_bq.height_bq =
-		    (input_config->effective_res.height *
-		     pipe_cfg->bayer_ds_out_res.height /
-		     input_config->effective_res.height + 1) / 2;
-
-		if (!asd->params.video_dis_en) {
-			/*
-			 * We adjust the ispfilter_bq to:
-			 * ispfilter_bq = 128/BDS
-			 * we still need firmware team to provide an offical
-			 * formula for SDV.
-			 */
-			bq_res->ispfilter_bq.width_bq = 128 *
-							pipe_cfg->bayer_ds_out_res.width /
-							input_config->effective_res.width / 2;
-			bq_res->ispfilter_bq.height_bq = 128 *
-							 pipe_cfg->bayer_ds_out_res.width /
-							 input_config->effective_res.width / 2;
-
-			if (IS_HWREVISION(asd->isp, ATOMISP_HW_REVISION_ISP2401)) {
-				/* No additional left padding for ISYS2401 */
-				bq_res->gdc_shift_bq.width_bq = 4 / 2;
-				bq_res->gdc_shift_bq.height_bq = 4 / 2;
-			} else {
-				/*
-				 * For the w_padding and gdc_shift_bq cacluation
-				 * Please see the BZ 1786 and 4358 for more info.
-				 * Just test that this formula can work now,
-				 * but we still have no offical formula.
-				 *
-				 * w_padding = ceiling(gdc_effective_input
-				 *             /128, 1) * 128 - effective_width
-				 * gdc_shift_bq = w_padding/BDS/2 + ispfilter_bq/2
-				 */
-				gdc_effective_input =
-				    input_config->effective_res.width +
-				    pipe_cfg->dvs_envelope.width;
-				w_padding = roundup(gdc_effective_input, 128) -
-					    input_config->effective_res.width;
-				w_padding = w_padding *
-					    pipe_cfg->bayer_ds_out_res.width /
-					    input_config->effective_res.width + 1;
-				w_padding = roundup(w_padding / 2, 1);
-
-				bq_res->gdc_shift_bq.width_bq = bq_res->ispfilter_bq.width_bq / 2
-								+ w_padding;
-				bq_res->gdc_shift_bq.height_bq = 4 / 2;
-			}
-		} else {
-			unsigned int dvs_w, dvs_h, dvs_w_max, dvs_h_max;
-
-			bq_res->ispfilter_bq.width_bq = 8 / 2;
-			bq_res->ispfilter_bq.height_bq = 8 / 2;
-
-			if (IS_HWREVISION(asd->isp, ATOMISP_HW_REVISION_ISP2401)) {
-				/* No additional left padding for ISYS2401 */
-				bq_res->gdc_shift_bq.width_bq = 4 / 2;
-				bq_res->gdc_shift_bq.height_bq = 4 / 2;
-			} else {
-				w_padding =
-				    roundup(input_config->effective_res.width, 128) -
-				    input_config->effective_res.width;
-				if (w_padding < 12)
-					w_padding = 12;
-				bq_res->gdc_shift_bq.width_bq = 4 / 2 +
-								((w_padding - 12) *
-								 pipe_cfg->bayer_ds_out_res.width /
-								 input_config->effective_res.width + 1) / 2;
-				bq_res->gdc_shift_bq.height_bq = 4 / 2;
-			}
+	bq_res->source_bq.width_bq = bq_res->output_bq.width_bq +
+				     pipe_cfg->dvs_envelope.width / 2;
+	bq_res->source_bq.height_bq = bq_res->output_bq.height_bq +
+				      pipe_cfg->dvs_envelope.height / 2;
+	/*
+	 * Bad pixels caused by spatial filter processing
+	 * ISP filter resolution should be given by CSS/FW, but for now
+	 * there is not such API to query, and it is fixed value, so
+	 * hardcoded here.
+	 */
+	bq_res->ispfilter_bq.width_bq = 12 / 2;
+	bq_res->ispfilter_bq.height_bq = 12 / 2;
+	/* spatial filter shift, always 4 pixels */
+	bq_res->gdc_shift_bq.width_bq = 4 / 2;
+	bq_res->gdc_shift_bq.height_bq = 4 / 2;
 
-			dvs_w = pipe_cfg->bayer_ds_out_res.width -
-				pipe_cfg->output_info[0].res.width;
-			dvs_h = pipe_cfg->bayer_ds_out_res.height -
-				pipe_cfg->output_info[0].res.height;
-			dvs_w_max = rounddown(
-					pipe_cfg->output_info[0].res.width / 5,
-					ATOM_ISP_STEP_WIDTH);
-			dvs_h_max = rounddown(
-					pipe_cfg->output_info[0].res.height / 5,
-					ATOM_ISP_STEP_HEIGHT);
-			bq_res->envelope_bq.width_bq =
-			    min((dvs_w / 2), (dvs_w_max / 2)) -
-			    bq_res->ispfilter_bq.width_bq;
-			bq_res->envelope_bq.height_bq =
-			    min((dvs_h / 2), (dvs_h_max / 2)) -
-			    bq_res->ispfilter_bq.height_bq;
-		}
+	if (asd->params.video_dis_en) {
+		bq_res->envelope_bq.width_bq = pipe_cfg->dvs_envelope.width / 2 -
+					       bq_res->ispfilter_bq.width_bq;
+		bq_res->envelope_bq.height_bq = pipe_cfg->dvs_envelope.height / 2 -
+						bq_res->ispfilter_bq.height_bq;
 	}
 
 	dev_dbg(asd->isp->dev,
@@ -3640,10 +3296,8 @@ int atomisp_set_parameters(struct video_device *vdev,
 		return -EINVAL;
 	}
 
-	dev_dbg(asd->isp->dev,
-		"%s: set parameter(per_frame_setting %d) for asd%d with isp_config_id %d of %s\n",
-		__func__, arg->per_frame_setting, asd->index,
-		arg->isp_config_id, vdev->name);
+	dev_dbg(asd->isp->dev, "set parameter(per_frame_setting %d) isp_config_id %d of %s\n",
+		arg->per_frame_setting, arg->isp_config_id, vdev->name);
 
 	if (IS_ISP2401) {
 		if (atomisp_is_vf_pipe(pipe) && arg->per_frame_setting) {
@@ -3751,28 +3405,8 @@ int atomisp_param(struct atomisp_sub_device *asd, int flag,
 		}
 
 		/* update dvs envelop info */
-		if (!asd->continuous_mode->val) {
-			config->dvs_envelop.width = vp_cfg->dvs_envelope.width;
-			config->dvs_envelop.height =
-			    vp_cfg->dvs_envelope.height;
-		} else {
-			unsigned int dvs_w, dvs_h, dvs_w_max, dvs_h_max;
-
-			dvs_w = vp_cfg->bayer_ds_out_res.width -
-				vp_cfg->output_info[0].res.width;
-			dvs_h = vp_cfg->bayer_ds_out_res.height -
-				vp_cfg->output_info[0].res.height;
-			dvs_w_max = rounddown(
-					vp_cfg->output_info[0].res.width / 5,
-					ATOM_ISP_STEP_WIDTH);
-			dvs_h_max = rounddown(
-					vp_cfg->output_info[0].res.height / 5,
-					ATOM_ISP_STEP_HEIGHT);
-
-			config->dvs_envelop.width = min(dvs_w, dvs_w_max);
-			config->dvs_envelop.height = min(dvs_h, dvs_h_max);
-		}
-
+		config->dvs_envelop.width = vp_cfg->dvs_envelope.width;
+		config->dvs_envelop.height = vp_cfg->dvs_envelope.height;
 		return 0;
 	}
 
@@ -4254,16 +3888,12 @@ int atomisp_try_fmt(struct video_device *vdev, struct v4l2_pix_format *f,
 	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
 	struct v4l2_subdev_pad_config pad_cfg;
 	struct v4l2_subdev_state pad_state = {
-		.pads = &pad_cfg
-		};
+		.pads = &pad_cfg,
+	};
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_TRY,
 	};
-
-	struct v4l2_mbus_framefmt *snr_mbus_fmt = &format.format;
 	const struct atomisp_format_bridge *fmt;
-	struct atomisp_input_stream_info *stream_info =
-	    (struct atomisp_input_stream_info *)snr_mbus_fmt->reserved;
 	int ret;
 
 	if (!asd) {
@@ -4284,14 +3914,15 @@ int atomisp_try_fmt(struct video_device *vdev, struct v4l2_pix_format *f,
 	if (f->width <= 0 || f->height <= 0)
 		return -EINVAL;
 
-	snr_mbus_fmt->code = fmt->mbus_code;
-	snr_mbus_fmt->width = f->width;
-	snr_mbus_fmt->height = f->height;
+	format.format.code = fmt->mbus_code;
+	format.format.width = f->width;
+	format.format.height = f->height;
 
-	__atomisp_init_stream_info(ATOMISP_INPUT_STREAM_GENERAL, stream_info);
+	__atomisp_init_stream_info(ATOMISP_INPUT_STREAM_GENERAL,
+				   (struct atomisp_input_stream_info *)format.format.reserved);
 
 	dev_dbg(isp->dev, "try_mbus_fmt: asking for %ux%u\n",
-		snr_mbus_fmt->width, snr_mbus_fmt->height);
+		format.format.width, format.format.height);
 
 	ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
 			       pad, set_fmt, &pad_state, &format);
@@ -4299,12 +3930,12 @@ int atomisp_try_fmt(struct video_device *vdev, struct v4l2_pix_format *f,
 		return ret;
 
 	dev_dbg(isp->dev, "try_mbus_fmt: got %ux%u\n",
-		snr_mbus_fmt->width, snr_mbus_fmt->height);
+		format.format.width, format.format.height);
 
-	fmt = atomisp_get_format_bridge_from_mbus(snr_mbus_fmt->code);
+	fmt = atomisp_get_format_bridge_from_mbus(format.format.code);
 	if (!fmt) {
 		dev_err(isp->dev, "unknown sensor format 0x%8.8x\n",
-			snr_mbus_fmt->code);
+			format.format.code);
 		return -EINVAL;
 	}
 
@@ -4318,15 +3949,15 @@ int atomisp_try_fmt(struct video_device *vdev, struct v4l2_pix_format *f,
 	 */
 	if (f->pixelformat == V4L2_PIX_FMT_JPEG ||
 	    f->pixelformat == V4L2_PIX_FMT_CUSTOM_M10MO_RAW) {
-		f->width = snr_mbus_fmt->width;
-		f->height = snr_mbus_fmt->height;
+		f->width = format.format.width;
+		f->height = format.format.height;
 		return 0;
 	}
 
-	if (!res_overflow || (snr_mbus_fmt->width < f->width &&
-			      snr_mbus_fmt->height < f->height)) {
-		f->width = snr_mbus_fmt->width;
-		f->height = snr_mbus_fmt->height;
+	if (!res_overflow || (format.format.width < f->width &&
+			      format.format.height < f->height)) {
+		f->width = format.format.width;
+		f->height = format.format.height;
 		/* Set the flag when resolution requested is
 		 * beyond the max value supported by sensor
 		 */
@@ -4439,43 +4070,6 @@ static inline int atomisp_set_sensor_mipi_to_isp(
 	return 0;
 }
 
-static int __enable_continuous_mode(struct atomisp_sub_device *asd,
-				    bool enable)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	dev_dbg(isp->dev,
-		"continuous mode %d, raw buffers %d, stop preview %d\n",
-		enable, asd->continuous_raw_buffer_size->val,
-		!asd->continuous_viewfinder->val);
-
-	if (!IS_ISP2401)
-		atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_PRIMARY);
-	else
-		atomisp_update_capture_mode(asd);
-
-	/* in case of ANR, force capture pipe to offline mode */
-	atomisp_css_capture_enable_online(asd, ATOMISP_INPUT_STREAM_GENERAL,
-					  asd->params.low_light ? false : !enable);
-	atomisp_css_preview_enable_online(asd, ATOMISP_INPUT_STREAM_GENERAL,
-					  !enable);
-	atomisp_css_enable_continuous(asd, enable);
-	atomisp_css_enable_cvf(asd, asd->continuous_viewfinder->val);
-
-	atomisp_css_continuous_set_num_raw_frames(asd,
-		asd->continuous_raw_buffer_size->val);
-
-	if (!enable) {
-		atomisp_css_enable_raw_binning(asd, false);
-		atomisp_css_input_set_two_pixels_per_clock(asd, false);
-	}
-
-	if (isp->inputs[asd->input_curr].type != FILE_INPUT)
-		atomisp_css_input_set_mode(asd, IA_CSS_INPUT_MODE_BUFFERED_SENSOR);
-
-	return atomisp_update_run_mode(asd);
-}
-
 static int configure_pp_input_nop(struct atomisp_sub_device *asd,
 				  unsigned int width, unsigned int height)
 {
@@ -4517,17 +4111,6 @@ static int css_input_resolution_changed(struct atomisp_sub_device *asd,
 	else
 		atomisp_css_input_set_two_pixels_per_clock(asd, true);
 
-	if (asd->continuous_mode->val) {
-		/* Note for all checks: ffmt includes pad_w+pad_h */
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO ||
-		    (ffmt->width >= 2048 || ffmt->height >= 1536)) {
-			/*
-			 * For preview pipe, enable only if resolution
-			 * is >= 3M for ISP2400.
-			 */
-			atomisp_css_enable_raw_binning(asd, true);
-		}
-	}
 	/*
 	 * If sensor input changed, which means metadata resolution changed
 	 * together. Release all metadata buffers here to let it re-allocated
@@ -4551,7 +4134,6 @@ static int css_input_resolution_changed(struct atomisp_sub_device *asd,
 
 static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 				  struct ia_css_frame_info *output_info,
-				  struct ia_css_frame_info *raw_output_info,
 				  struct v4l2_pix_format *pix,
 				  unsigned int source_pad)
 {
@@ -4592,8 +4174,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 	if (!format)
 		return -EINVAL;
 
-	if (isp->inputs[asd->input_curr].type != TEST_PATTERN &&
-	    isp->inputs[asd->input_curr].type != FILE_INPUT) {
+	if (isp->inputs[asd->input_curr].type != TEST_PATTERN) {
 		mipi_info = atomisp_to_sensor_mipi_info(
 				isp->inputs[asd->input_curr].camera);
 		if (!mipi_info) {
@@ -4622,8 +4203,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 	 * Configure viewfinder also when vfpp is disabled: the
 	 * CSS still requires viewfinder configuration.
 	 */
-	if (asd->fmt_auto->val ||
-	    asd->vfpp->val != ATOMISP_VFPP_ENABLE) {
+	{
 		struct v4l2_rect vf_size = {0};
 		struct v4l2_mbus_framefmt vf_ffmt = {0};
 
@@ -4670,12 +4250,6 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 		}
 	}
 
-	if (asd->continuous_mode->val) {
-		ret = __enable_continuous_mode(asd, true);
-		if (ret)
-			return -EINVAL;
-	}
-
 	atomisp_css_input_set_mode(asd, IA_CSS_INPUT_MODE_BUFFERED_SENSOR);
 
 	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
@@ -4691,49 +4265,15 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 		get_frame_info = atomisp_css_video_get_output_frame_info;
 		pipe_id = IA_CSS_PIPE_ID_VIDEO;
 	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-		if (!asd->continuous_mode->val) {
-			configure_output = atomisp_css_video_configure_output;
-			get_frame_info =
-			    atomisp_css_video_get_output_frame_info;
-			pipe_id = IA_CSS_PIPE_ID_VIDEO;
-		} else {
-			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW ||
-			    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO) {
-				configure_output =
-				    atomisp_css_video_configure_output;
-				get_frame_info =
-				    atomisp_css_video_get_output_frame_info;
-				configure_pp_input =
-				    atomisp_css_video_configure_pp_input;
-				pipe_id = IA_CSS_PIPE_ID_VIDEO;
-			} else {
-				configure_output =
-				    atomisp_css_capture_configure_output;
-				get_frame_info =
-				    atomisp_css_capture_get_output_frame_info;
-				configure_pp_input =
-				    atomisp_css_capture_configure_pp_input;
-				pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-
-				atomisp_update_capture_mode(asd);
-				atomisp_css_capture_enable_online(asd,
-								  ATOMISP_INPUT_STREAM_GENERAL,
-								  false);
-			}
-		}
+		configure_output = atomisp_css_video_configure_output;
+		get_frame_info = atomisp_css_video_get_output_frame_info;
+		pipe_id = IA_CSS_PIPE_ID_VIDEO;
 	} else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW) {
 		configure_output = atomisp_css_preview_configure_output;
 		get_frame_info = atomisp_css_preview_get_output_frame_info;
 		configure_pp_input = atomisp_css_preview_configure_pp_input;
 		pipe_id = IA_CSS_PIPE_ID_PREVIEW;
 	} else {
-		/* CSS doesn't support low light mode on SOC cameras, so disable
-		 * it. FIXME: if this is done elsewhere, it gives corrupted
-		 * colors into thumbnail image.
-		 */
-		if (isp->inputs[asd->input_curr].type == SOC_CAMERA)
-			asd->params.low_light = false;
-
 		if (format->sh_fmt == IA_CSS_FRAME_FORMAT_RAW) {
 			atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_RAW);
 			atomisp_css_enable_dz(asd, false);
@@ -4741,38 +4281,22 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 			atomisp_update_capture_mode(asd);
 		}
 
-		if (!asd->continuous_mode->val)
-			/* in case of ANR, force capture pipe to offline mode */
-			atomisp_css_capture_enable_online(asd, ATOMISP_INPUT_STREAM_GENERAL,
-							  asd->params.low_light ?
-							  false : asd->params.online_process);
+		/* in case of ANR, force capture pipe to offline mode */
+		atomisp_css_capture_enable_online(asd, ATOMISP_INPUT_STREAM_GENERAL,
+						  !asd->params.low_light);
 
 		configure_output = atomisp_css_capture_configure_output;
 		get_frame_info = atomisp_css_capture_get_output_frame_info;
 		configure_pp_input = atomisp_css_capture_configure_pp_input;
 		pipe_id = IA_CSS_PIPE_ID_CAPTURE;
 
-		if (!asd->params.online_process &&
-		    !asd->continuous_mode->val) {
-			ret = atomisp_css_capture_get_output_raw_frame_info(asd,
-				raw_output_info);
-			if (ret)
-				return ret;
-		}
-		if (!asd->continuous_mode->val && asd->run_mode->val
-		    != ATOMISP_RUN_MODE_STILL_CAPTURE) {
+		if (asd->run_mode->val != ATOMISP_RUN_MODE_STILL_CAPTURE) {
 			dev_err(isp->dev,
 				"Need to set the running mode first\n");
 			asd->run_mode->val = ATOMISP_RUN_MODE_STILL_CAPTURE;
 		}
 	}
 
-	/*
-	 * to SOC camera, use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-
 	if (asd->copy_mode)
 		ret = atomisp_css_copy_configure_output(asd, ATOMISP_INPUT_STREAM_GENERAL,
 							pix->width, pix->height,
@@ -4810,16 +4334,6 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 	}
 
 	atomisp_update_grid_info(asd, pipe_id, source_pad);
-
-	/* Free the raw_dump buffer first */
-	ia_css_frame_free(asd->raw_output_frame);
-	asd->raw_output_frame = NULL;
-
-	if (!asd->continuous_mode->val && !asd->params.online_process &&
-	    ia_css_frame_allocate_from_info(&asd->raw_output_frame,
-		    raw_output_info))
-		return -ENOMEM;
-
 	return 0;
 }
 
@@ -4827,12 +4341,6 @@ static void atomisp_get_dis_envelop(struct atomisp_sub_device *asd,
 				    unsigned int width, unsigned int height,
 				    unsigned int *dvs_env_w, unsigned int *dvs_env_h)
 {
-	struct atomisp_device *isp = asd->isp;
-
-	/* if subdev type is SOC camera,we do not need to set DVS */
-	if (isp->inputs[asd->input_curr].type == SOC_CAMERA)
-		asd->params.video_dis_en = false;
-
 	if (asd->params.video_dis_en &&
 	    asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
 		/* envelope is 20% of the output resolution */
@@ -4849,7 +4357,7 @@ static void atomisp_get_dis_envelop(struct atomisp_sub_device *asd,
 }
 
 static void atomisp_check_copy_mode(struct atomisp_sub_device *asd,
-				    int source_pad, struct v4l2_pix_format *f)
+				    int source_pad, const struct v4l2_pix_format *f)
 {
 	struct v4l2_mbus_framefmt *sink, *src;
 
@@ -4872,8 +4380,7 @@ static void atomisp_check_copy_mode(struct atomisp_sub_device *asd,
 	dev_dbg(asd->isp->dev, "copy_mode: %d\n", asd->copy_mode);
 }
 
-static int atomisp_set_fmt_to_snr(struct video_device *vdev,
-				  struct v4l2_pix_format *f, unsigned int pixelformat,
+static int atomisp_set_fmt_to_snr(struct video_device *vdev, const struct v4l2_pix_format *f,
 				  unsigned int padding_w, unsigned int padding_h,
 				  unsigned int dvs_env_w, unsigned int dvs_env_h)
 {
@@ -4882,8 +4389,8 @@ static int atomisp_set_fmt_to_snr(struct video_device *vdev,
 	const struct atomisp_format_bridge *format;
 	struct v4l2_subdev_pad_config pad_cfg;
 	struct v4l2_subdev_state pad_state = {
-		.pads = &pad_cfg
-		};
+		.pads = &pad_cfg,
+	};
 	struct v4l2_subdev_format vformat = {
 		.which = V4L2_SUBDEV_FORMAT_TRY,
 	};
@@ -4906,7 +4413,7 @@ static int atomisp_set_fmt_to_snr(struct video_device *vdev,
 
 	v4l2_fh_init(&fh.vfh, vdev);
 
-	format = atomisp_get_format_bridge(pixelformat);
+	format = atomisp_get_format_bridge(f->pixelformat);
 	if (!format)
 		return -EINVAL;
 
@@ -4981,13 +4488,9 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 	struct atomisp_sub_device *asd = pipe->asd;
 	const struct atomisp_format_bridge *format_bridge;
 	const struct atomisp_format_bridge *snr_format_bridge;
-	struct ia_css_frame_info output_info, raw_output_info;
-	struct v4l2_pix_format snr_fmt;
-	struct v4l2_pix_format backup_fmt, s_fmt;
+	struct ia_css_frame_info output_info;
 	unsigned int dvs_env_w = 0, dvs_env_h = 0;
 	unsigned int padding_w = pad_w, padding_h = pad_h;
-	bool res_overflow = false, crop_needs_override = false;
-	struct v4l2_mbus_framefmt *isp_sink_fmt;
 	struct v4l2_mbus_framefmt isp_source_fmt = {0};
 	struct v4l2_subdev_format vformat = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
@@ -5005,9 +4508,8 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 		return -EINVAL;
 
 	dev_dbg(isp->dev,
-		"setting resolution %ux%u on pad %u for asd%d, bytesperline %u\n",
-		f->fmt.pix.width, f->fmt.pix.height, source_pad,
-		asd->index, f->fmt.pix.bytesperline);
+		"setting resolution %ux%u on pad %u bytesperline %u\n",
+		f->fmt.pix.width, f->fmt.pix.height, source_pad, f->fmt.pix.bytesperline);
 
 	v4l2_fh_init(&fh.vfh, vdev);
 
@@ -5042,105 +4544,7 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 	f->fmt.pix.width = vformat.format.width - padding_w;
 	f->fmt.pix.height = vformat.format.height - padding_h;
 
-	snr_fmt = f->fmt.pix;
-	backup_fmt = snr_fmt;
-
-	/**********************************************************************/
-
-	if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VF ||
-	    (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW
-	     && asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)) {
-		if (asd->fmt_auto->val) {
-			struct v4l2_rect *capture_comp;
-			struct v4l2_rect r = {0};
-
-			r.width = f->fmt.pix.width;
-			r.height = f->fmt.pix.height;
-
-			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW)
-				capture_comp = atomisp_subdev_get_rect(
-						   &asd->subdev, NULL,
-						   V4L2_SUBDEV_FORMAT_ACTIVE,
-						   ATOMISP_SUBDEV_PAD_SOURCE_VIDEO,
-						   V4L2_SEL_TGT_COMPOSE);
-			else
-				capture_comp = atomisp_subdev_get_rect(
-						   &asd->subdev, NULL,
-						   V4L2_SUBDEV_FORMAT_ACTIVE,
-						   ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE,
-						   V4L2_SEL_TGT_COMPOSE);
-
-			if (capture_comp->width < r.width
-			    || capture_comp->height < r.height) {
-				r.width = capture_comp->width;
-				r.height = capture_comp->height;
-			}
-
-			atomisp_subdev_set_selection(
-			    &asd->subdev, fh.state,
-			    V4L2_SUBDEV_FORMAT_ACTIVE, source_pad,
-			    V4L2_SEL_TGT_COMPOSE, 0, &r);
-
-			f->fmt.pix.width = r.width;
-			f->fmt.pix.height = r.height;
-		}
-
-		if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW) {
-			atomisp_css_video_configure_viewfinder(asd,
-							       f->fmt.pix.width, f->fmt.pix.height,
-							       format_bridge->planar ? f->fmt.pix.bytesperline
-							       : f->fmt.pix.bytesperline * 8
-							       / format_bridge->depth,	format_bridge->sh_fmt);
-			atomisp_css_video_get_viewfinder_frame_info(asd,
-				&output_info);
-			asd->copy_mode = false;
-		} else {
-			atomisp_css_capture_configure_viewfinder(asd,
-				f->fmt.pix.width, f->fmt.pix.height,
-				format_bridge->planar ? f->fmt.pix.bytesperline
-				: f->fmt.pix.bytesperline * 8
-				/ format_bridge->depth,	format_bridge->sh_fmt);
-			atomisp_css_capture_get_viewfinder_frame_info(asd,
-				&output_info);
-			asd->copy_mode = false;
-		}
-
-		goto done;
-	}
-	/*
-	 * Check whether main resolution configured smaller
-	 * than snapshot resolution. If so, force main resolution
-	 * to be the same as snapshot resolution
-	 */
-	if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE) {
-		struct v4l2_rect *r;
-
-		r = atomisp_subdev_get_rect(
-			&asd->subdev, NULL,
-			V4L2_SUBDEV_FORMAT_ACTIVE,
-			ATOMISP_SUBDEV_PAD_SOURCE_VF, V4L2_SEL_TGT_COMPOSE);
-
-		if (r->width && r->height
-		    && (r->width > f->fmt.pix.width
-			|| r->height > f->fmt.pix.height))
-			dev_warn(isp->dev,
-				 "Main Resolution config smaller then Vf Resolution. Force to be equal with Vf Resolution.");
-	}
-
-	/* Pipeline configuration done through subdevs. Bail out now. */
-	if (!asd->fmt_auto->val)
-		goto set_fmt_to_isp;
-
-	/* get sensor resolution and format */
-	ret = atomisp_try_fmt(vdev, &snr_fmt, &res_overflow);
-	if (ret) {
-		dev_warn(isp->dev, "Try format failed with error %d\n", ret);
-		return ret;
-	}
-	f->fmt.pix.width = snr_fmt.width;
-	f->fmt.pix.height = snr_fmt.height;
-
-	snr_format_bridge = atomisp_get_format_bridge(snr_fmt.pixelformat);
+	snr_format_bridge = atomisp_get_format_bridge_from_mbus(vformat.format.code);
 	if (!snr_format_bridge) {
 		dev_warn(isp->dev, "Can't find bridge format\n");
 		return -EINVAL;
@@ -5151,10 +4555,6 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 				ATOMISP_SUBDEV_PAD_SINK)->code =
 				    snr_format_bridge->mbus_code;
 
-	isp_sink_fmt = atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-						V4L2_SUBDEV_FORMAT_ACTIVE,
-						ATOMISP_SUBDEV_PAD_SINK);
-
 	isp_source_fmt.code = format_bridge->mbus_code;
 	atomisp_subdev_set_ffmt(&asd->subdev, fh.state,
 				V4L2_SUBDEV_FORMAT_ACTIVE,
@@ -5165,80 +4565,22 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 		padding_h = 0;
 	}
 
-	/* construct resolution supported by isp */
-	if (res_overflow && !asd->continuous_mode->val) {
-		f->fmt.pix.width = rounddown(
-				       clamp_t(u32, f->fmt.pix.width - padding_w,
-					       ATOM_ISP_MIN_WIDTH,
-					       ATOM_ISP_MAX_WIDTH), ATOM_ISP_STEP_WIDTH);
-		f->fmt.pix.height = rounddown(
-					clamp_t(u32, f->fmt.pix.height - padding_h,
-						ATOM_ISP_MIN_HEIGHT,
-						ATOM_ISP_MAX_HEIGHT), ATOM_ISP_STEP_HEIGHT);
-	}
-
 	atomisp_get_dis_envelop(asd, f->fmt.pix.width, f->fmt.pix.height,
 				&dvs_env_w, &dvs_env_h);
 
-	if (asd->continuous_mode->val) {
-		struct v4l2_rect *r;
+	asd->capture_pad = source_pad;
 
-		r = atomisp_subdev_get_rect(
-			&asd->subdev, NULL,
-			V4L2_SUBDEV_FORMAT_ACTIVE,
-			ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE,
-			V4L2_SEL_TGT_COMPOSE);
-		/*
-		 * The ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE should get resolutions
-		 * properly set otherwise, it should not be the capture_pad.
-		 */
-		if (r->width && r->height)
-			asd->capture_pad = ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE;
-		else
-			asd->capture_pad = source_pad;
-	} else {
-		asd->capture_pad = source_pad;
+	ret = atomisp_set_fmt_to_snr(vdev, &f->fmt.pix,
+				     padding_w, padding_h, dvs_env_w, dvs_env_h);
+	if (ret) {
+		dev_warn(isp->dev,
+			 "Set format to sensor failed with %d\n", ret);
+		return -EINVAL;
 	}
-	/*
-	 * set format info to sensor
-	 * In continuous mode, resolution is set only if it is higher than
-	 * existing value. This because preview pipe will be configured after
-	 * capture pipe and usually has lower resolution than capture pipe.
-	 */
-	if (!asd->continuous_mode->val ||
-	    isp_sink_fmt->width < (f->fmt.pix.width + padding_w + dvs_env_w) ||
-	    isp_sink_fmt->height < (f->fmt.pix.height + padding_h +
-				    dvs_env_h)) {
-		/*
-		 * For jpeg or custom raw format the sensor will return constant
-		 * width and height. Because we already had quried try_mbus_fmt,
-		 * f->fmt.pix.width and f->fmt.pix.height has been changed to
-		 * this fixed width and height. So we cannot select the correct
-		 * resolution with that information. So use the original width
-		 * and height while set_mbus_fmt() so actual resolutions are
-		 * being used in while set media bus format.
-		 */
-		s_fmt = f->fmt.pix;
-		if (f->fmt.pix.pixelformat == V4L2_PIX_FMT_JPEG ||
-		    f->fmt.pix.pixelformat == V4L2_PIX_FMT_CUSTOM_M10MO_RAW) {
-			s_fmt.width = backup_fmt.width;
-			s_fmt.height = backup_fmt.height;
-		}
-		ret = atomisp_set_fmt_to_snr(vdev, &s_fmt,
-					     f->fmt.pix.pixelformat, padding_w,
-					     padding_h, dvs_env_w, dvs_env_h);
-		if (ret) {
-			dev_warn(isp->dev,
-				 "Set format to sensor failed with %d\n", ret);
-			return -EINVAL;
-		}
 
-		atomisp_csi_lane_config(isp);
-		crop_needs_override = true;
-	}
+	atomisp_csi_lane_config(isp);
 
-	atomisp_check_copy_mode(asd, source_pad, &backup_fmt);
-	asd->yuvpp_mode = false;			/* Reset variable */
+	atomisp_check_copy_mode(asd, source_pad, &f->fmt.pix);
 
 	isp_sink_crop = *atomisp_subdev_get_rect(&asd->subdev, NULL,
 			V4L2_SUBDEV_FORMAT_ACTIVE,
@@ -5250,27 +4592,10 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 	if (isp_sink_crop.width * 9 / 10 < f->fmt.pix.width ||
 	    isp_sink_crop.height * 9 / 10 < f->fmt.pix.height ||
 	    (atomisp_subdev_format_conversion(asd, source_pad) &&
-	     ((asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-	       !asd->continuous_mode->val) ||
+	     (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO ||
 	      asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER))) {
-		/* for continuous mode, preview size might be smaller than
-		 * still capture size. if preview size still needs crop,
-		 * pick the larger one between crop size of preview and
-		 * still capture.
-		 */
-		if (asd->continuous_mode->val
-		    && source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW
-		    && !crop_needs_override) {
-			isp_sink_crop.width =
-			    max_t(unsigned int, f->fmt.pix.width,
-				  isp_sink_crop.width);
-			isp_sink_crop.height =
-			    max_t(unsigned int, f->fmt.pix.height,
-				  isp_sink_crop.height);
-		} else {
-			isp_sink_crop.width = f->fmt.pix.width;
-			isp_sink_crop.height = f->fmt.pix.height;
-		}
+		isp_sink_crop.width = f->fmt.pix.width;
+		isp_sink_crop.height = f->fmt.pix.height;
 
 		atomisp_subdev_set_selection(&asd->subdev, fh.state,
 					     V4L2_SUBDEV_FORMAT_ACTIVE,
@@ -5314,45 +4639,40 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 		 * which appears to be related by a hardware
 		 * performance limitation.  It's unclear why this
 		 * particular code triggers the issue. */
-		if (crop_needs_override) {
-			if (isp_sink_crop.width * main_compose.height >
-			    isp_sink_crop.height * main_compose.width) {
-				sink_crop.height = isp_sink_crop.height;
-				sink_crop.width = DIV_NEAREST_STEP(
-						      sink_crop.height *
-						      f->fmt.pix.width,
-						      f->fmt.pix.height,
-						      ATOM_ISP_STEP_WIDTH);
-			} else {
-				sink_crop.width = isp_sink_crop.width;
-				sink_crop.height = DIV_NEAREST_STEP(
-						       sink_crop.width *
-						       f->fmt.pix.height,
-						       f->fmt.pix.width,
-						       ATOM_ISP_STEP_HEIGHT);
-			}
-			atomisp_subdev_set_selection(&asd->subdev, fh.state,
-						     V4L2_SUBDEV_FORMAT_ACTIVE,
-						     ATOMISP_SUBDEV_PAD_SINK,
-						     V4L2_SEL_TGT_CROP,
-						     V4L2_SEL_FLAG_KEEP_CONFIG,
-						     &sink_crop);
+		if (isp_sink_crop.width * main_compose.height >
+		    isp_sink_crop.height * main_compose.width) {
+			sink_crop.height = isp_sink_crop.height;
+			sink_crop.width =
+				DIV_NEAREST_STEP(sink_crop.height * f->fmt.pix.width,
+						 f->fmt.pix.height,
+						 ATOM_ISP_STEP_WIDTH);
+		} else {
+			sink_crop.width = isp_sink_crop.width;
+			sink_crop.height =
+				DIV_NEAREST_STEP(sink_crop.width * f->fmt.pix.height,
+						 f->fmt.pix.width,
+						 ATOM_ISP_STEP_HEIGHT);
 		}
 		atomisp_subdev_set_selection(&asd->subdev, fh.state,
 					     V4L2_SUBDEV_FORMAT_ACTIVE,
+					     ATOMISP_SUBDEV_PAD_SINK,
+					     V4L2_SEL_TGT_CROP,
+					     V4L2_SEL_FLAG_KEEP_CONFIG,
+					     &sink_crop);
+
+		atomisp_subdev_set_selection(&asd->subdev, fh.state,
+					     V4L2_SUBDEV_FORMAT_ACTIVE,
 					     source_pad,
 					     V4L2_SEL_TGT_COMPOSE, 0,
 					     &main_compose);
 	}
 
-set_fmt_to_isp:
-	ret = atomisp_set_fmt_to_isp(vdev, &output_info, &raw_output_info,
-				     &f->fmt.pix, source_pad);
+	ret = atomisp_set_fmt_to_isp(vdev, &output_info, &f->fmt.pix, source_pad);
 	if (ret) {
 		dev_warn(isp->dev, "Can't set format on ISP. Error %d\n", ret);
 		return -EINVAL;
 	}
-done:
+
 	pipe->pix.width = f->fmt.pix.width;
 	pipe->pix.height = f->fmt.pix.height;
 	pipe->pix.pixelformat = f->fmt.pix.pixelformat;
@@ -5470,71 +4790,6 @@ out:
 	return ret;
 }
 
-int atomisp_offline_capture_configure(struct atomisp_sub_device *asd,
-				      struct atomisp_cont_capture_conf *cvf_config)
-{
-	struct v4l2_ctrl *c;
-
-	lockdep_assert_held(&asd->isp->mutex);
-
-	/*
-	* In case of M10MO ZSL capture case, we need to issue a separate
-	* capture request to M10MO which will output captured jpeg image
-	*/
-	c = v4l2_ctrl_find(
-		asd->isp->inputs[asd->input_curr].camera->ctrl_handler,
-		V4L2_CID_START_ZSL_CAPTURE);
-	if (c) {
-		int ret;
-
-		dev_dbg(asd->isp->dev, "%s trigger ZSL capture request\n",
-			__func__);
-		/* TODO: use the cvf_config */
-		ret = v4l2_ctrl_s_ctrl(c, 1);
-		if (ret)
-			return ret;
-
-		return v4l2_ctrl_s_ctrl(c, 0);
-	}
-
-	asd->params.offline_parm = *cvf_config;
-
-	if (asd->params.offline_parm.num_captures) {
-		if (asd->streaming == ATOMISP_DEVICE_STREAMING_DISABLED) {
-			unsigned int init_raw_num;
-
-			if (asd->enable_raw_buffer_lock->val) {
-				init_raw_num =
-				    ATOMISP_CSS2_NUM_OFFLINE_INIT_CONTINUOUS_FRAMES_LOCK_EN;
-				if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-				    asd->params.video_dis_en)
-					init_raw_num +=
-					    ATOMISP_CSS2_NUM_DVS_FRAME_DELAY;
-			} else {
-				init_raw_num =
-				    ATOMISP_CSS2_NUM_OFFLINE_INIT_CONTINUOUS_FRAMES;
-			}
-
-			/* TODO: this can be removed once user-space
-			 *       has been updated to use control API */
-			asd->continuous_raw_buffer_size->val =
-			    max_t(int,
-				  asd->continuous_raw_buffer_size->val,
-				  asd->params.offline_parm.
-				  num_captures + init_raw_num);
-			asd->continuous_raw_buffer_size->val =
-			    min_t(int, ATOMISP_CONT_RAW_FRAMES,
-				  asd->continuous_raw_buffer_size->val);
-		}
-		asd->continuous_mode->val = true;
-	} else {
-		asd->continuous_mode->val = false;
-		__enable_continuous_mode(asd, false);
-	}
-
-	return 0;
-}
-
 /*
  * set auto exposure metering window to camera sensor
  */
@@ -5631,53 +4886,6 @@ void atomisp_init_raw_buffer_bitmap(struct atomisp_sub_device *asd)
 	spin_unlock_irqrestore(&asd->raw_buffer_bitmap_lock, flags);
 }
 
-static int atomisp_set_raw_buffer_bitmap(struct atomisp_sub_device *asd, int exp_id)
-{
-	int *bitmap, bit;
-	unsigned long flags;
-
-	if (__checking_exp_id(asd, exp_id))
-		return -EINVAL;
-
-	bitmap = asd->raw_buffer_bitmap + exp_id / 32;
-	bit = exp_id % 32;
-	spin_lock_irqsave(&asd->raw_buffer_bitmap_lock, flags);
-	(*bitmap) |= (1 << bit);
-	asd->raw_buffer_locked_count++;
-	spin_unlock_irqrestore(&asd->raw_buffer_bitmap_lock, flags);
-
-	dev_dbg(asd->isp->dev, "%s: exp_id %d,  raw_buffer_locked_count %d\n",
-		__func__, exp_id, asd->raw_buffer_locked_count);
-
-	/* Check if the raw buffer after next is still locked!!! */
-	exp_id += 2;
-	if (exp_id > ATOMISP_MAX_EXP_ID)
-		exp_id -= ATOMISP_MAX_EXP_ID;
-	bitmap = asd->raw_buffer_bitmap + exp_id / 32;
-	bit = exp_id % 32;
-	if ((*bitmap) & (1 << bit)) {
-		int ret;
-
-		/* WORKAROUND unlock the raw buffer compulsively */
-		ret = atomisp_css_exp_id_unlock(asd, exp_id);
-		if (ret) {
-			dev_err(asd->isp->dev,
-				"%s exp_id is wrapping back to %d but force unlock failed, err %d.\n",
-				__func__, exp_id, ret);
-			return ret;
-		}
-
-		spin_lock_irqsave(&asd->raw_buffer_bitmap_lock, flags);
-		(*bitmap) &= ~(1 << bit);
-		asd->raw_buffer_locked_count--;
-		spin_unlock_irqrestore(&asd->raw_buffer_bitmap_lock, flags);
-		dev_warn(asd->isp->dev,
-			 "%s exp_id is wrapping back to %d but it is still locked so force unlock it, raw_buffer_locked_count %d\n",
-			 __func__, exp_id, asd->raw_buffer_locked_count);
-	}
-	return 0;
-}
-
 static int __is_raw_buffer_locked(struct atomisp_sub_device *asd, int exp_id)
 {
 	int *bitmap, bit;
@@ -5816,9 +5024,7 @@ static int atomisp_get_pipe_id(struct atomisp_video_pipe *pipe)
 		return -EINVAL;
 	}
 
-	if (ATOMISP_USE_YUVPP(asd)) {
-		return IA_CSS_PIPE_ID_YUVPP;
-	} else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
+	if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
 		return IA_CSS_PIPE_ID_VIDEO;
 	} else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) {
 		return IA_CSS_PIPE_ID_CAPTURE;
diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.h b/drivers/staging/media/atomisp/pci/atomisp_cmd.h
index 733b9f8cd06f..399b549bcf83 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_cmd.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.h
@@ -72,7 +72,6 @@ irqreturn_t atomisp_isr_thread(int irq, void *isp_ptr);
 const struct atomisp_format_bridge *get_atomisp_format_bridge_from_mbus(
     u32 mbus_code);
 bool atomisp_is_mbuscode_raw(uint32_t code);
-void atomisp_delayed_init_work(struct work_struct *work);
 
 /* Get internal fmt according to V4L2 fmt */
 bool atomisp_is_viewfinder_support(struct atomisp_device *isp);
@@ -268,9 +267,6 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f);
 int atomisp_set_shading_table(struct atomisp_sub_device *asd,
 			      struct atomisp_shading_table *shading_table);
 
-int atomisp_offline_capture_configure(struct atomisp_sub_device *asd,
-				      struct atomisp_cont_capture_conf *cvf_config);
-
 void atomisp_free_internal_buffers(struct atomisp_sub_device *asd);
 
 int atomisp_s_ae_window(struct atomisp_sub_device *asd,
@@ -317,11 +313,6 @@ void atomisp_init_raw_buffer_bitmap(struct atomisp_sub_device *asd);
 int atomisp_enable_dz_capt_pipe(struct atomisp_sub_device *asd,
 				unsigned int *enable);
 
-/* Function to get metadata type bu pipe id */
-enum atomisp_metadata_type
-atomisp_get_metadata_type(struct atomisp_sub_device *asd,
-			  enum ia_css_pipe_id pipe_id);
-
 u32 atomisp_get_pixel_depth(u32 pixelformat);
 
 /* Function for HAL to inject a fake event to wake up poll thread */
diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat.h b/drivers/staging/media/atomisp/pci/atomisp_compat.h
index 7316eb9f974a..218e8ac276c8 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_compat.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_compat.h
@@ -196,9 +196,6 @@ void atomisp_css_input_set_two_pixels_per_clock(
     struct atomisp_sub_device *asd,
     bool two_ppc);
 
-void atomisp_css_enable_raw_binning(struct atomisp_sub_device *asd,
-				    bool enable);
-
 void atomisp_css_enable_dz(struct atomisp_sub_device *asd, bool enable);
 
 void atomisp_css_capture_set_mode(struct atomisp_sub_device *asd,
@@ -306,10 +303,6 @@ int atomisp_css_copy_get_output_frame_info(
     unsigned int stream_index,
     struct ia_css_frame_info *info);
 
-int atomisp_css_capture_get_output_raw_frame_info(
-    struct atomisp_sub_device *asd,
-    struct ia_css_frame_info *info);
-
 int atomisp_css_preview_get_output_frame_info(
     struct atomisp_sub_device *asd,
     struct ia_css_frame_info *info);
@@ -423,9 +416,7 @@ int atomisp_css_get_dis_stat(struct atomisp_sub_device *asd,
 
 int atomisp_css_update_stream(struct atomisp_sub_device *asd);
 
-int atomisp_css_isr_thread(struct atomisp_device *isp,
-			   bool *frame_done_found,
-			   bool *css_pipe_done);
+int atomisp_css_isr_thread(struct atomisp_device *isp);
 
 bool atomisp_css_valid_sof(struct atomisp_device *isp);
 
diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c
index 61e2e63a0ef1..1dae2a7cfdd9 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c
@@ -158,13 +158,6 @@ static void atomisp_css2_hw_load(hrt_address addr, void *to, uint32_t n)
 	spin_unlock_irqrestore(&mmio_lock, flags);
 }
 
-static int  __printf(1, 0) atomisp_css2_dbg_ftrace_print(const char *fmt,
-							 va_list args)
-{
-	ftrace_vprintk(fmt, args);
-	return 0;
-}
-
 static int  __printf(1, 0) atomisp_vprintk(const char *fmt, va_list args)
 {
 	vprintk(fmt, args);
@@ -597,8 +590,6 @@ static void __apply_additional_pipe_config(
 		.enable_reduced_pipe = true;
 		stream_env->pipe_configs[pipe_id]
 		.enable_dz = false;
-		if (ATOMISP_SOC_CAMERA(asd))
-			stream_env->pipe_configs[pipe_id].enable_dz = true;
 
 		if (asd->params.video_dis_en) {
 			stream_env->pipe_extra_configs[pipe_id]
@@ -612,10 +603,7 @@ static void __apply_additional_pipe_config(
 		break;
 	case IA_CSS_PIPE_ID_YUVPP:
 	case IA_CSS_PIPE_ID_COPY:
-		if (ATOMISP_SOC_CAMERA(asd))
-			stream_env->pipe_configs[pipe_id].enable_dz = true;
-		else
-			stream_env->pipe_configs[pipe_id].enable_dz = false;
+		stream_env->pipe_configs[pipe_id].enable_dz = false;
 		break;
 	default:
 		break;
@@ -658,13 +646,10 @@ static bool is_pipe_valid_to_current_run_mode(struct atomisp_sub_device *asd,
 
 		return false;
 	case ATOMISP_RUN_MODE_PREVIEW:
-		if (!asd->continuous_mode->val) {
-			if (pipe_id == IA_CSS_PIPE_ID_PREVIEW)
-				return true;
+		if (pipe_id == IA_CSS_PIPE_ID_PREVIEW)
+			return true;
 
-			return false;
-		}
-		fallthrough;
+		return false;
 	case ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE:
 		if (pipe_id == IA_CSS_PIPE_ID_CAPTURE ||
 		    pipe_id == IA_CSS_PIPE_ID_PREVIEW)
@@ -672,14 +657,10 @@ static bool is_pipe_valid_to_current_run_mode(struct atomisp_sub_device *asd,
 
 		return false;
 	case ATOMISP_RUN_MODE_VIDEO:
-		if (!asd->continuous_mode->val) {
-			if (pipe_id == IA_CSS_PIPE_ID_VIDEO ||
-			    pipe_id == IA_CSS_PIPE_ID_YUVPP)
-				return true;
-			else
-				return false;
-		}
-		fallthrough;
+		if (pipe_id == IA_CSS_PIPE_ID_VIDEO || pipe_id == IA_CSS_PIPE_ID_YUVPP)
+			return true;
+
+		return false;
 	case ATOMISP_RUN_MODE_SDV:
 		if (pipe_id == IA_CSS_PIPE_ID_CAPTURE ||
 		    pipe_id == IA_CSS_PIPE_ID_VIDEO)
@@ -813,9 +794,6 @@ static inline int __set_css_print_env(struct atomisp_device *isp, int opt)
 	if (opt == 0)
 		isp->css_env.isp_css_env.print_env.debug_print = NULL;
 	else if (opt == 1)
-		isp->css_env.isp_css_env.print_env.debug_print =
-		    atomisp_css2_dbg_ftrace_print;
-	else if (opt == 2)
 		isp->css_env.isp_css_env.print_env.debug_print = atomisp_vprintk;
 	else
 		ret = -EINVAL;
@@ -1745,25 +1723,6 @@ void atomisp_css_input_set_two_pixels_per_clock(
 		.update_pipe[i] = true;
 }
 
-void atomisp_css_enable_raw_binning(struct atomisp_sub_device *asd,
-				    bool enable)
-{
-	struct atomisp_stream_env *stream_env =
-		    &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	unsigned int pipe;
-
-	if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-		pipe = IA_CSS_PIPE_ID_VIDEO;
-	else
-		pipe = IA_CSS_PIPE_ID_PREVIEW;
-
-	stream_env->pipe_extra_configs[pipe].enable_raw_binning = enable;
-	stream_env->update_pipe[pipe] = true;
-	if (enable)
-		stream_env->pipe_configs[pipe].output_info[0].padded_width =
-		    stream_env->stream_config.input_config.effective_res.width;
-}
-
 void atomisp_css_enable_dz(struct atomisp_sub_device *asd, bool enable)
 {
 	int i;
@@ -1894,17 +1853,6 @@ void atomisp_css_enable_continuous(struct atomisp_sub_device *asd,
 		    &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
 	int i;
 
-	/*
-	 * To SOC camera, there is only one YUVPP pipe in any case
-	 * including ZSL/SDV/continuous viewfinder, so always set
-	 * stream_config.continuous to 0.
-	 */
-	if (ATOMISP_USE_YUVPP(asd)) {
-		stream_env->stream_config.continuous = 0;
-		stream_env->stream_config.online = 1;
-		return;
-	}
-
 	if (stream_env->stream_config.continuous != !!enable) {
 		stream_env->stream_config.continuous = !!enable;
 		stream_env->stream_config.pack_raw_pixels = true;
@@ -2122,51 +2070,6 @@ static void __configure_output(struct atomisp_sub_device *asd,
 		pipe_id, width, height, format);
 }
 
-static void __configure_video_preview_output(struct atomisp_sub_device *asd,
-	unsigned int stream_index,
-	unsigned int width, unsigned int height,
-	unsigned int min_width,
-	enum ia_css_frame_format format,
-	enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_stream_env *stream_env =
-		    &asd->stream_env[stream_index];
-	struct ia_css_frame_info *css_output_info;
-	struct ia_css_stream_config *stream_config = &stream_env->stream_config;
-
-	stream_env->pipe_configs[pipe_id].mode =
-	    __pipe_id_to_pipe_mode(asd, pipe_id);
-	stream_env->update_pipe[pipe_id] = true;
-
-	/*
-	 * second_output will be as video main output in SDV mode
-	 * with SOC camera. output will be as video main output in
-	 * normal video mode.
-	 */
-	if (asd->continuous_mode->val)
-		css_output_info = &stream_env->pipe_configs[pipe_id].
-				  output_info[ATOMISP_CSS_OUTPUT_SECOND_INDEX];
-	else
-		css_output_info = &stream_env->pipe_configs[pipe_id].
-				  output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
-
-	css_output_info->res.width = width;
-	css_output_info->res.height = height;
-	css_output_info->format = format;
-	css_output_info->padded_width = min_width;
-
-	/* isp binary 2.2 specific setting*/
-	if (width > stream_config->input_config.effective_res.width ||
-	    height > stream_config->input_config.effective_res.height) {
-		stream_config->input_config.effective_res.width = width;
-		stream_config->input_config.effective_res.height = height;
-	}
-
-	dev_dbg(isp->dev, "configuring pipe[%d] output info w=%d.h=%d.f=%d.\n",
-		pipe_id, width, height, format);
-}
-
 /*
  * For CSS2.1, capture pipe uses capture_pp_in_res to configure yuv
  * downscaling input resolution.
@@ -2470,42 +2373,6 @@ static void __configure_vf_output(struct atomisp_sub_device *asd,
 		pipe_id, width, height, format);
 }
 
-static void __configure_video_vf_output(struct atomisp_sub_device *asd,
-					unsigned int width, unsigned int height,
-					unsigned int min_width,
-					enum ia_css_frame_format format,
-					enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_stream_env *stream_env =
-		    &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	struct ia_css_frame_info *css_output_info;
-
-	stream_env->pipe_configs[pipe_id].mode =
-	    __pipe_id_to_pipe_mode(asd, pipe_id);
-	stream_env->update_pipe[pipe_id] = true;
-
-	/*
-	 * second_vf_output will be as video viewfinder in SDV mode
-	 * with SOC camera. vf_output will be as video viewfinder in
-	 * normal video mode.
-	 */
-	if (asd->continuous_mode->val)
-		css_output_info = &stream_env->pipe_configs[pipe_id].
-				  vf_output_info[ATOMISP_CSS_OUTPUT_SECOND_INDEX];
-	else
-		css_output_info = &stream_env->pipe_configs[pipe_id].
-				  vf_output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
-
-	css_output_info->res.width = width;
-	css_output_info->res.height = height;
-	css_output_info->format = format;
-	css_output_info->padded_width = min_width;
-	dev_dbg(isp->dev,
-		"configuring pipe[%d] vf output info w=%d.h=%d.f=%d.\n",
-		pipe_id, width, height, format);
-}
-
 static int __get_frame_info(struct atomisp_sub_device *asd,
 			    unsigned int stream_index,
 			    struct ia_css_frame_info *info,
@@ -2565,16 +2432,9 @@ static unsigned int atomisp_get_pipe_index(struct atomisp_sub_device *asd,
 	uint16_t source_pad)
 {
 	struct atomisp_device *isp = asd->isp;
-	/*
-	 * to SOC camera, use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		return IA_CSS_PIPE_ID_YUVPP;
 
 	switch (source_pad) {
 	case ATOMISP_SUBDEV_PAD_SOURCE_VIDEO:
-		if (asd->yuvpp_mode)
-			return IA_CSS_PIPE_ID_YUVPP;
 		if (asd->copy_mode)
 			return IA_CSS_PIPE_ID_COPY;
 		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO
@@ -2593,8 +2453,6 @@ static unsigned int atomisp_get_pipe_index(struct atomisp_sub_device *asd,
 		}
 		fallthrough;
 	case ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW:
-		if (asd->yuvpp_mode)
-			return IA_CSS_PIPE_ID_YUVPP;
 		if (asd->copy_mode)
 			return IA_CSS_PIPE_ID_COPY;
 		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
@@ -2617,13 +2475,9 @@ int atomisp_get_css_frame_info(struct atomisp_sub_device *asd,
 	int stream_index;
 	struct atomisp_device *isp = asd->isp;
 
-	if (ATOMISP_SOC_CAMERA(asd)) {
-		stream_index = ATOMISP_INPUT_STREAM_GENERAL;
-	} else {
-		stream_index = (pipe_index == IA_CSS_PIPE_ID_YUVPP) ?
+	stream_index = (pipe_index == IA_CSS_PIPE_ID_YUVPP) ?
 			       ATOMISP_INPUT_STREAM_VIDEO :
 			       ATOMISP_INPUT_STREAM_GENERAL;
-	}
 
 	if (0 != ia_css_pipe_get_info(asd->stream_env[stream_index]
 		.pipes[pipe_index], &info)) {
@@ -2636,12 +2490,7 @@ int atomisp_get_css_frame_info(struct atomisp_sub_device *asd,
 		*frame_info = info.output_info[0];
 		break;
 	case ATOMISP_SUBDEV_PAD_SOURCE_VIDEO:
-		if (ATOMISP_USE_YUVPP(asd) && asd->continuous_mode->val)
-			*frame_info = info.
-				      output_info[ATOMISP_CSS_OUTPUT_SECOND_INDEX];
-		else
-			*frame_info = info.
-				      output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
+		*frame_info = info.output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
 		break;
 	case ATOMISP_SUBDEV_PAD_SOURCE_VF:
 		if (stream_index == ATOMISP_INPUT_STREAM_POSTVIEW)
@@ -2653,15 +2502,7 @@ int atomisp_get_css_frame_info(struct atomisp_sub_device *asd,
 		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
 		    (pipe_index == IA_CSS_PIPE_ID_VIDEO ||
 		     pipe_index == IA_CSS_PIPE_ID_YUVPP))
-			if (ATOMISP_USE_YUVPP(asd) && asd->continuous_mode->val)
-				*frame_info = info.
-					      vf_output_info[ATOMISP_CSS_OUTPUT_SECOND_INDEX];
-			else
-				*frame_info = info.
-					      vf_output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
-		else if (ATOMISP_USE_YUVPP(asd) && asd->continuous_mode->val)
-			*frame_info =
-			    info.output_info[ATOMISP_CSS_OUTPUT_SECOND_INDEX];
+			*frame_info = info.vf_output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
 		else
 			*frame_info =
 			    info.output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
@@ -2727,16 +2568,8 @@ int atomisp_css_preview_configure_output(struct atomisp_sub_device *asd,
 	unsigned int min_width,
 	enum ia_css_frame_format format)
 {
-	/*
-	 * to SOC camera, use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		__configure_video_preview_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width,
-						 height,
-						 min_width, format, IA_CSS_PIPE_ID_YUVPP);
-	else
-		__configure_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width, height,
-				   min_width, format, IA_CSS_PIPE_ID_PREVIEW);
+	__configure_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width, height,
+			   min_width, format, IA_CSS_PIPE_ID_PREVIEW);
 	return 0;
 }
 
@@ -2745,18 +2578,8 @@ int atomisp_css_capture_configure_output(struct atomisp_sub_device *asd,
 	unsigned int min_width,
 	enum ia_css_frame_format format)
 {
-	enum ia_css_pipe_id pipe_id;
-
-	/*
-	 * to SOC camera, use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-	else
-		pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-
 	__configure_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width, height,
-			   min_width, format, pipe_id);
+			   min_width, format, IA_CSS_PIPE_ID_CAPTURE);
 	return 0;
 }
 
@@ -2765,16 +2588,8 @@ int atomisp_css_video_configure_output(struct atomisp_sub_device *asd,
 				       unsigned int min_width,
 				       enum ia_css_frame_format format)
 {
-	/*
-	 * to SOC camera, use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		__configure_video_preview_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width,
-						 height,
-						 min_width, format, IA_CSS_PIPE_ID_YUVPP);
-	else
-		__configure_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width, height,
-				   min_width, format, IA_CSS_PIPE_ID_VIDEO);
+	__configure_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width, height,
+			   min_width, format, IA_CSS_PIPE_ID_VIDEO);
 	return 0;
 }
 
@@ -2784,15 +2599,8 @@ int atomisp_css_video_configure_viewfinder(
     unsigned int min_width,
     enum ia_css_frame_format format)
 {
-	/*
-	 * to SOC camera, video will use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		__configure_video_vf_output(asd, width, height, min_width, format,
-					    IA_CSS_PIPE_ID_YUVPP);
-	else
-		__configure_vf_output(asd, width, height, min_width, format,
-				      IA_CSS_PIPE_ID_VIDEO);
+	__configure_vf_output(asd, width, height, min_width, format,
+			      IA_CSS_PIPE_ID_VIDEO);
 	return 0;
 }
 
@@ -2802,18 +2610,7 @@ int atomisp_css_capture_configure_viewfinder(
     unsigned int min_width,
     enum ia_css_frame_format format)
 {
-	enum ia_css_pipe_id pipe_id;
-
-	/*
-	 * to SOC camera, video will use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-	else
-		pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-
-	__configure_vf_output(asd, width, height, min_width, format,
-			      pipe_id);
+	__configure_vf_output(asd, width, height, min_width, format, IA_CSS_PIPE_ID_CAPTURE);
 	return 0;
 }
 
@@ -2821,45 +2618,16 @@ int atomisp_css_video_get_viewfinder_frame_info(
     struct atomisp_sub_device *asd,
     struct ia_css_frame_info *info)
 {
-	enum ia_css_pipe_id pipe_id;
-	enum frame_info_type frame_type = ATOMISP_CSS_VF_FRAME;
-
-	if (ATOMISP_USE_YUVPP(asd)) {
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-		if (asd->continuous_mode->val)
-			frame_type = ATOMISP_CSS_SECOND_VF_FRAME;
-	} else {
-		pipe_id = IA_CSS_PIPE_ID_VIDEO;
-	}
-
 	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-				frame_type, pipe_id);
+				ATOMISP_CSS_VF_FRAME, IA_CSS_PIPE_ID_VIDEO);
 }
 
 int atomisp_css_capture_get_viewfinder_frame_info(
     struct atomisp_sub_device *asd,
     struct ia_css_frame_info *info)
 {
-	enum ia_css_pipe_id pipe_id;
-
-	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-	else
-		pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-
 	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-				ATOMISP_CSS_VF_FRAME, pipe_id);
-}
-
-int atomisp_css_capture_get_output_raw_frame_info(
-    struct atomisp_sub_device *asd,
-    struct ia_css_frame_info *info)
-{
-	if (ATOMISP_USE_YUVPP(asd))
-		return 0;
-
-	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-				ATOMISP_CSS_RAW_FRAME, IA_CSS_PIPE_ID_CAPTURE);
+				ATOMISP_CSS_VF_FRAME, IA_CSS_PIPE_ID_CAPTURE);
 }
 
 int atomisp_css_copy_get_output_frame_info(
@@ -2875,53 +2643,24 @@ int atomisp_css_preview_get_output_frame_info(
     struct atomisp_sub_device *asd,
     struct ia_css_frame_info *info)
 {
-	enum ia_css_pipe_id pipe_id;
-	enum frame_info_type frame_type = ATOMISP_CSS_OUTPUT_FRAME;
-
-	if (ATOMISP_USE_YUVPP(asd)) {
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-		if (asd->continuous_mode->val)
-			frame_type = ATOMISP_CSS_SECOND_OUTPUT_FRAME;
-	} else {
-		pipe_id = IA_CSS_PIPE_ID_PREVIEW;
-	}
-
 	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-				frame_type, pipe_id);
+				ATOMISP_CSS_OUTPUT_FRAME, IA_CSS_PIPE_ID_PREVIEW);
 }
 
 int atomisp_css_capture_get_output_frame_info(
     struct atomisp_sub_device *asd,
     struct ia_css_frame_info *info)
 {
-	enum ia_css_pipe_id pipe_id;
-
-	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-	else
-		pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-
 	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-				ATOMISP_CSS_OUTPUT_FRAME, pipe_id);
+				ATOMISP_CSS_OUTPUT_FRAME, IA_CSS_PIPE_ID_CAPTURE);
 }
 
 int atomisp_css_video_get_output_frame_info(
     struct atomisp_sub_device *asd,
     struct ia_css_frame_info *info)
 {
-	enum ia_css_pipe_id pipe_id;
-	enum frame_info_type frame_type = ATOMISP_CSS_OUTPUT_FRAME;
-
-	if (ATOMISP_USE_YUVPP(asd)) {
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-		if (asd->continuous_mode->val)
-			frame_type = ATOMISP_CSS_SECOND_OUTPUT_FRAME;
-	} else {
-		pipe_id = IA_CSS_PIPE_ID_VIDEO;
-	}
-
 	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-				frame_type, pipe_id);
+				ATOMISP_CSS_OUTPUT_FRAME, IA_CSS_PIPE_ID_VIDEO);
 }
 
 int atomisp_css_preview_configure_pp_input(
@@ -2930,15 +2669,12 @@ int atomisp_css_preview_configure_pp_input(
 {
 	struct atomisp_stream_env *stream_env =
 		    &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	__configure_preview_pp_input(asd, width, height,
-				     ATOMISP_USE_YUVPP(asd) ?
-				     IA_CSS_PIPE_ID_YUVPP : IA_CSS_PIPE_ID_PREVIEW);
+	__configure_preview_pp_input(asd, width, height, IA_CSS_PIPE_ID_PREVIEW);
 
 	if (width > stream_env->pipe_configs[IA_CSS_PIPE_ID_CAPTURE].
 	    capt_pp_in_res.width)
-		__configure_capture_pp_input(asd, width, height,
-					     ATOMISP_USE_YUVPP(asd) ?
-					     IA_CSS_PIPE_ID_YUVPP : IA_CSS_PIPE_ID_CAPTURE);
+		__configure_capture_pp_input(asd, width, height, IA_CSS_PIPE_ID_CAPTURE);
+
 	return 0;
 }
 
@@ -2946,9 +2682,7 @@ int atomisp_css_capture_configure_pp_input(
     struct atomisp_sub_device *asd,
     unsigned int width, unsigned int height)
 {
-	__configure_capture_pp_input(asd, width, height,
-				     ATOMISP_USE_YUVPP(asd) ?
-				     IA_CSS_PIPE_ID_YUVPP : IA_CSS_PIPE_ID_CAPTURE);
+	__configure_capture_pp_input(asd, width, height, IA_CSS_PIPE_ID_CAPTURE);
 	return 0;
 }
 
@@ -2959,15 +2693,12 @@ int atomisp_css_video_configure_pp_input(
 	struct atomisp_stream_env *stream_env =
 		    &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
 
-	__configure_video_pp_input(asd, width, height,
-				   ATOMISP_USE_YUVPP(asd) ?
-				   IA_CSS_PIPE_ID_YUVPP : IA_CSS_PIPE_ID_VIDEO);
+	__configure_video_pp_input(asd, width, height, IA_CSS_PIPE_ID_VIDEO);
 
 	if (width > stream_env->pipe_configs[IA_CSS_PIPE_ID_CAPTURE].
 	    capt_pp_in_res.width)
-		__configure_capture_pp_input(asd, width, height,
-					     ATOMISP_USE_YUVPP(asd) ?
-					     IA_CSS_PIPE_ID_YUVPP : IA_CSS_PIPE_ID_CAPTURE);
+		__configure_capture_pp_input(asd, width, height, IA_CSS_PIPE_ID_CAPTURE);
+
 	return 0;
 }
 
@@ -3662,41 +3393,33 @@ void atomisp_css_morph_table_free(struct ia_css_morph_table *table)
 	ia_css_morph_table_free(table);
 }
 
-static struct atomisp_sub_device *__get_atomisp_subdev(
-    struct ia_css_pipe *css_pipe,
-    struct atomisp_device *isp,
-    enum atomisp_input_stream_id *stream_id)
+static bool atomisp_css_isr_get_stream_id(struct ia_css_pipe *css_pipe,
+					  struct atomisp_device *isp,
+					  enum atomisp_input_stream_id *stream_id)
 {
-	int i, j, k;
-	struct atomisp_sub_device *asd;
 	struct atomisp_stream_env *stream_env;
+	int i, j;
 
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		if (asd->streaming == ATOMISP_DEVICE_STREAMING_DISABLED)
-			continue;
-		for (j = 0; j < ATOMISP_INPUT_STREAM_NUM; j++) {
-			stream_env = &asd->stream_env[j];
-			for (k = 0; k < IA_CSS_PIPE_ID_NUM; k++) {
-				if (stream_env->pipes[k] &&
-				    stream_env->pipes[k] == css_pipe) {
-					*stream_id = j;
-					return asd;
-				}
+	if (isp->asd.streaming == ATOMISP_DEVICE_STREAMING_DISABLED)
+		return false;
+
+	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
+		stream_env = &isp->asd.stream_env[i];
+		for (j = 0; j < IA_CSS_PIPE_ID_NUM; j++) {
+			if (stream_env->pipes[j] && stream_env->pipes[j] == css_pipe) {
+				*stream_id = i;
+				return true;
 			}
 		}
 	}
 
-	return NULL;
+	return false;
 }
 
-int atomisp_css_isr_thread(struct atomisp_device *isp,
-			   bool *frame_done_found,
-			   bool *css_pipe_done)
+int atomisp_css_isr_thread(struct atomisp_device *isp)
 {
 	enum atomisp_input_stream_id stream_id = 0;
 	struct atomisp_css_event current_event;
-	struct atomisp_sub_device *asd;
 
 	lockdep_assert_held(&isp->mutex);
 
@@ -3722,9 +3445,7 @@ int atomisp_css_isr_thread(struct atomisp_device *isp,
 			continue;
 		}
 
-		asd = __get_atomisp_subdev(current_event.event.pipe,
-					   isp, &stream_id);
-		if (!asd) {
+		if (!atomisp_css_isr_get_stream_id(current_event.event.pipe, isp, &stream_id)) {
 			if (current_event.event.type == IA_CSS_EVENT_TYPE_TIMER)
 				dev_dbg(isp->dev,
 					"event: Timer event.");
@@ -3735,56 +3456,53 @@ int atomisp_css_isr_thread(struct atomisp_device *isp,
 			continue;
 		}
 
-		atomisp_css_temp_pipe_to_pipe_id(asd, &current_event);
+		atomisp_css_temp_pipe_to_pipe_id(&isp->asd, &current_event);
 		switch (current_event.event.type) {
 		case IA_CSS_EVENT_TYPE_OUTPUT_FRAME_DONE:
 			dev_dbg(isp->dev, "event: Output frame done");
-			frame_done_found[asd->index] = true;
-			atomisp_buf_done(asd, 0, IA_CSS_BUFFER_TYPE_OUTPUT_FRAME,
+			atomisp_buf_done(&isp->asd, 0, IA_CSS_BUFFER_TYPE_OUTPUT_FRAME,
 					 current_event.pipe, true, stream_id);
 			break;
 		case IA_CSS_EVENT_TYPE_SECOND_OUTPUT_FRAME_DONE:
 			dev_dbg(isp->dev, "event: Second output frame done");
-			frame_done_found[asd->index] = true;
-			atomisp_buf_done(asd, 0, IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME,
+			atomisp_buf_done(&isp->asd, 0, IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME,
 					 current_event.pipe, true, stream_id);
 			break;
 		case IA_CSS_EVENT_TYPE_3A_STATISTICS_DONE:
 			dev_dbg(isp->dev, "event: 3A stats frame done");
-			atomisp_buf_done(asd, 0,
+			atomisp_buf_done(&isp->asd, 0,
 					 IA_CSS_BUFFER_TYPE_3A_STATISTICS,
 					 current_event.pipe,
 					 false, stream_id);
 			break;
 		case IA_CSS_EVENT_TYPE_METADATA_DONE:
 			dev_dbg(isp->dev, "event: metadata frame done");
-			atomisp_buf_done(asd, 0,
+			atomisp_buf_done(&isp->asd, 0,
 					 IA_CSS_BUFFER_TYPE_METADATA,
 					 current_event.pipe,
 					 false, stream_id);
 			break;
 		case IA_CSS_EVENT_TYPE_VF_OUTPUT_FRAME_DONE:
 			dev_dbg(isp->dev, "event: VF output frame done");
-			atomisp_buf_done(asd, 0,
+			atomisp_buf_done(&isp->asd, 0,
 					 IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME,
 					 current_event.pipe, true, stream_id);
 			break;
 		case IA_CSS_EVENT_TYPE_SECOND_VF_OUTPUT_FRAME_DONE:
 			dev_dbg(isp->dev, "event: second VF output frame done");
-			atomisp_buf_done(asd, 0,
+			atomisp_buf_done(&isp->asd, 0,
 					 IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME,
 					 current_event.pipe, true, stream_id);
 			break;
 		case IA_CSS_EVENT_TYPE_DIS_STATISTICS_DONE:
 			dev_dbg(isp->dev, "event: dis stats frame done");
-			atomisp_buf_done(asd, 0,
+			atomisp_buf_done(&isp->asd, 0,
 					 IA_CSS_BUFFER_TYPE_DIS_STATISTICS,
 					 current_event.pipe,
 					 false, stream_id);
 			break;
 		case IA_CSS_EVENT_TYPE_PIPELINE_DONE:
 			dev_dbg(isp->dev, "event: pipeline done");
-			css_pipe_done[asd->index] = true;
 			break;
 		case IA_CSS_EVENT_TYPE_ACC_STAGE_COMPLETE:
 			dev_warn(isp->dev, "unexpected event: acc stage done");
@@ -3801,23 +3519,17 @@ int atomisp_css_isr_thread(struct atomisp_device *isp,
 
 bool atomisp_css_valid_sof(struct atomisp_device *isp)
 {
-	unsigned int i, j;
-
-	/* Loop for each css stream */
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-		/* Loop for each css vc stream */
-		for (j = 0; j < ATOMISP_INPUT_STREAM_NUM; j++) {
-			if (!asd->stream_env[j].stream)
-				continue;
-
-			dev_dbg(isp->dev,
-				"stream #%d: mode: %d\n", j,
-				asd->stream_env[j].stream_config.mode);
-			if (asd->stream_env[j].stream_config.mode ==
-			    IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
-				return false;
-		}
+	unsigned int i;
+
+	/* Loop for each css vc stream */
+	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
+		if (!isp->asd.stream_env[i].stream)
+			continue;
+
+		dev_dbg(isp->dev, "stream #%d: mode: %d\n",
+			i, isp->asd.stream_env[i].stream_config.mode);
+		if (isp->asd.stream_env[i].stream_config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
+			return false;
 	}
 
 	return true;
diff --git a/drivers/staging/media/atomisp/pci/atomisp_fops.c b/drivers/staging/media/atomisp/pci/atomisp_fops.c
index ce01479bdd68..fa362c8a37e8 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_fops.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_fops.c
@@ -112,8 +112,7 @@ static int atomisp_q_one_metadata_buffer(struct atomisp_sub_device *asd,
 	enum ia_css_pipe_id css_pipe_id)
 {
 	struct atomisp_metadata_buf *metadata_buf;
-	enum atomisp_metadata_type md_type =
-	    atomisp_get_metadata_type(asd, css_pipe_id);
+	enum atomisp_metadata_type md_type = ATOMISP_MAIN_METADATA;
 	struct list_head *metadata_list;
 
 	if (asd->metadata_bufs_in_css[stream_id][css_pipe_id] >=
@@ -347,41 +346,6 @@ static int atomisp_get_css_buf_type(struct atomisp_sub_device *asd,
 				    enum ia_css_pipe_id pipe_id,
 				    uint16_t source_pad)
 {
-	if (ATOMISP_USE_YUVPP(asd)) {
-		/* when run ZSL case */
-		if (asd->continuous_mode->val &&
-		    asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) {
-			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE)
-				return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME;
-			else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW)
-				return IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME;
-			else
-				return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
-		}
-
-		/*when run SDV case*/
-		if (asd->continuous_mode->val &&
-		    asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE)
-				return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME;
-			else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW)
-				return IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME;
-			else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO)
-				return IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME;
-			else
-				return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
-		}
-
-		/*other case: default setting*/
-		if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE ||
-		    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO ||
-		    (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW &&
-		     asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO))
-			return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME;
-		else
-			return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
-	}
-
 	if (pipe_id == IA_CSS_PIPE_ID_COPY ||
 	    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE ||
 	    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO ||
@@ -414,22 +378,10 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		preview_pipe = &asd->video_out_capture;
 		css_preview_pipe_id = IA_CSS_PIPE_ID_CAPTURE;
 	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-		if (asd->continuous_mode->val) {
-			capture_pipe = &asd->video_out_capture;
-			vf_pipe = &asd->video_out_vf;
-			css_capture_pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-		}
 		video_pipe = &asd->video_out_video_capture;
 		preview_pipe = &asd->video_out_preview;
 		css_video_pipe_id = IA_CSS_PIPE_ID_VIDEO;
 		css_preview_pipe_id = IA_CSS_PIPE_ID_VIDEO;
-	} else if (asd->continuous_mode->val) {
-		capture_pipe = &asd->video_out_capture;
-		vf_pipe = &asd->video_out_vf;
-		preview_pipe = &asd->video_out_preview;
-
-		css_preview_pipe_id = IA_CSS_PIPE_ID_PREVIEW;
-		css_capture_pipe_id = IA_CSS_PIPE_ID_CAPTURE;
 	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) {
 		preview_pipe = &asd->video_out_preview;
 		css_preview_pipe_id = IA_CSS_PIPE_ID_PREVIEW;
@@ -447,27 +399,12 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		css_video_pipe_id = IA_CSS_PIPE_ID_COPY;
 	}
 
-	if (asd->yuvpp_mode) {
-		capture_pipe = &asd->video_out_capture;
-		video_pipe   = &asd->video_out_video_capture;
-		preview_pipe = &asd->video_out_preview;
-		css_capture_pipe_id = IA_CSS_PIPE_ID_COPY;
-		css_video_pipe_id   = IA_CSS_PIPE_ID_YUVPP;
-		css_preview_pipe_id = IA_CSS_PIPE_ID_YUVPP;
-	}
-
 	if (capture_pipe) {
 		buf_type = atomisp_get_css_buf_type(
 			       asd, css_capture_pipe_id,
 			       atomisp_subdev_source_pad(&capture_pipe->vdev));
 		input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
 
-		/*
-		 * use yuvpp pipe for SOC camera.
-		 */
-		if (ATOMISP_USE_YUVPP(asd))
-			css_capture_pipe_id = IA_CSS_PIPE_ID_YUVPP;
-
 		atomisp_q_video_buffers_to_css(asd, capture_pipe,
 					       input_stream_id,
 					       buf_type, css_capture_pipe_id);
@@ -482,11 +419,6 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		else
 			input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
 
-		/*
-		 * use yuvpp pipe for SOC camera.
-		 */
-		if (ATOMISP_USE_YUVPP(asd))
-			css_capture_pipe_id = IA_CSS_PIPE_ID_YUVPP;
 		atomisp_q_video_buffers_to_css(asd, vf_pipe,
 					       input_stream_id,
 					       buf_type, css_capture_pipe_id);
@@ -496,22 +428,14 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		buf_type = atomisp_get_css_buf_type(
 			       asd, css_preview_pipe_id,
 			       atomisp_subdev_source_pad(&preview_pipe->vdev));
-		if (ATOMISP_SOC_CAMERA(asd) && css_preview_pipe_id == IA_CSS_PIPE_ID_YUVPP)
-			input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
-		/* else for ext isp use case */
-		else if (css_preview_pipe_id == IA_CSS_PIPE_ID_YUVPP)
+
+		if (css_preview_pipe_id == IA_CSS_PIPE_ID_YUVPP)
 			input_stream_id = ATOMISP_INPUT_STREAM_VIDEO;
 		else if (asd->stream_env[ATOMISP_INPUT_STREAM_PREVIEW].stream)
 			input_stream_id = ATOMISP_INPUT_STREAM_PREVIEW;
 		else
 			input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
 
-		/*
-		 * use yuvpp pipe for SOC camera.
-		 */
-		if (ATOMISP_USE_YUVPP(asd))
-			css_preview_pipe_id = IA_CSS_PIPE_ID_YUVPP;
-
 		atomisp_q_video_buffers_to_css(asd, preview_pipe,
 					       input_stream_id,
 					       buf_type, css_preview_pipe_id);
@@ -526,12 +450,6 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		else
 			input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
 
-		/*
-		 * use yuvpp pipe for SOC camera.
-		 */
-		if (ATOMISP_USE_YUVPP(asd))
-			css_video_pipe_id = IA_CSS_PIPE_ID_YUVPP;
-
 		atomisp_q_video_buffers_to_css(asd, video_pipe,
 					       input_stream_id,
 					       buf_type, css_video_pipe_id);
@@ -545,7 +463,6 @@ static void atomisp_buf_queue(struct vb2_buffer *vb)
 	struct atomisp_video_pipe *pipe = vb_to_pipe(vb);
 	struct ia_css_frame *frame = vb_to_frame(vb);
 	struct atomisp_sub_device *asd = pipe->asd;
-	u16 source_pad = atomisp_subdev_source_pad(&pipe->vdev);
 	unsigned long irqflags;
 	int ret;
 
@@ -593,21 +510,6 @@ static void atomisp_buf_queue(struct vb2_buffer *vb)
 			atomisp_qbuffers_to_css(asd);
 	}
 
-	/*
-	 * Workaround: Due to the design of HALv3,
-	 * sometimes in ZSL or SDV mode HAL needs to
-	 * capture multiple images within one streaming cycle.
-	 * But the capture number cannot be determined by HAL.
-	 * So HAL only sets the capture number to be 1 and queue multiple
-	 * buffers. Atomisp driver needs to check this case and re-trigger
-	 * CSS to do capture when new buffer is queued.
-	 */
-	if (asd->continuous_mode->val && source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE &&
-	    !asd->enable_raw_buffer_lock->val && asd->params.offline_parm.num_captures == 1) {
-		asd->pending_capture_request++;
-		dev_dbg(asd->isp->dev, "Add one pending capture request.\n");
-	}
-
 out_unlock:
 	mutex_unlock(&asd->isp->mutex);
 }
@@ -662,21 +564,15 @@ static void atomisp_subdev_init_struct(struct atomisp_sub_device *asd)
 	asd->params.fpn_en = false;
 	asd->params.xnr_en = false;
 	asd->params.false_color = 0;
-	asd->params.online_process = 1;
 	asd->params.yuv_ds_en = 0;
 	/* s3a grid not enabled for any pipe */
 	asd->params.s3a_enabled_pipe = IA_CSS_PIPE_ID_NUM;
 
-	asd->params.offline_parm.num_captures = 1;
-	asd->params.offline_parm.skip_frames = 0;
-	asd->params.offline_parm.offset = 0;
-	asd->delayed_init = ATOMISP_DELAYED_INIT_NOT_QUEUED;
 	/* Add for channel */
 	asd->input_curr = 0;
 
 	asd->mipi_frame_size = 0;
 	asd->copy_mode = false;
-	asd->yuvpp_mode = false;
 
 	asd->stream_prepared = false;
 	asd->high_speed_mode = false;
@@ -698,12 +594,7 @@ static unsigned int atomisp_subdev_users(struct atomisp_sub_device *asd)
 
 unsigned int atomisp_dev_users(struct atomisp_device *isp)
 {
-	unsigned int i, sum;
-
-	for (i = 0, sum = 0; i < isp->num_of_streams; i++)
-		sum += atomisp_subdev_users(&isp->asd[i]);
-
-	return sum;
+	return atomisp_subdev_users(&isp->asd);
 }
 
 static int atomisp_open(struct file *file)
@@ -815,7 +706,7 @@ static int atomisp_release(struct file *file)
 	 * The sink pad setting can only be cleared when all device nodes
 	 * get released.
 	 */
-	if (asd->fmt_auto->val) {
+	{
 		struct v4l2_mbus_framefmt isp_sink_fmt = { 0 };
 
 		atomisp_subdev_set_ffmt(&asd->subdev, fh.state,
diff --git a/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c b/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
index 7fc7dfa56172..c718a74ea70a 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
@@ -304,7 +304,17 @@ static struct gmin_cfg_var surface3_vars[] = {
 	{},
 };
 
+static struct gmin_cfg_var lenovo_ideapad_miix_310_vars[] = {
+	/* _DSM contains the wrong CsiPort! */
+	{ "OVTI2680:01_CsiPort", "0" },
+	{}
+};
+
 static const struct dmi_system_id gmin_vars[] = {
+	/*
+	 * These DMI IDs were present when the atomisp driver was merged into
+	 * drivers/staging and it is unclear if they are really necessary.
+	 */
 	{
 		.ident = "BYT-T FFD8",
 		.matches = {
@@ -341,6 +351,7 @@ static const struct dmi_system_id gmin_vars[] = {
 		},
 		.driver_data = i8880_vars,
 	},
+	/* Later added DMI ids, these are confirmed to really be necessary! */
 	{
 		.ident = "Surface 3",
 		.matches = {
@@ -348,6 +359,14 @@ static const struct dmi_system_id gmin_vars[] = {
 		},
 		.driver_data = surface3_vars,
 	},
+	{
+		.ident = "Lenovo Ideapad Miix 310",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_VERSION, "MIIX 310-10"),
+		},
+		.driver_data = lenovo_ideapad_miix_310_vars,
+	},
 	{}
 };
 
@@ -1353,37 +1372,22 @@ static int gmin_get_config_var(struct device *maindev,
 			       const char *var,
 			       char *out, size_t *out_len)
 {
+	struct acpi_device *adev = ACPI_COMPANION(maindev);
 	efi_char16_t var16[CFG_VAR_NAME_MAX];
 	const struct dmi_system_id *id;
-	struct device *dev = maindev;
 	char var8[CFG_VAR_NAME_MAX];
 	efi_status_t status;
 	int i, ret;
 
-	/* For sensors, try first to use the _DSM table */
-	if (!is_gmin) {
-		ret = gmin_get_config_dsm_var(maindev, var, out, out_len);
-		if (!ret)
-			return 0;
-	}
-
-	/* Fall-back to other approaches */
-
-	if (!is_gmin && ACPI_COMPANION(dev))
-		dev = &ACPI_COMPANION(dev)->dev;
-
-	if (!is_gmin)
-		ret = snprintf(var8, sizeof(var8), "%s_%s", dev_name(dev), var);
+	if (!is_gmin && adev)
+		ret = snprintf(var8, sizeof(var8), "%s_%s", acpi_dev_name(adev), var);
 	else
 		ret = snprintf(var8, sizeof(var8), "gmin_%s", var);
 
 	if (ret < 0 || ret >= sizeof(var8) - 1)
 		return -EINVAL;
 
-	/* First check a hard-coded list of board-specific variables.
-	 * Some device firmwares lack the ability to set EFI variables at
-	 * runtime.
-	 */
+	/* DMI based quirks override both the _DSM table and EFI variables */
 	id = dmi_first_match(gmin_vars);
 	if (id) {
 		ret = gmin_get_hardcoded_var(maindev, id->driver_data, var8,
@@ -1392,6 +1396,13 @@ static int gmin_get_config_var(struct device *maindev,
 			return 0;
 	}
 
+	/* For sensors, try first to use the _DSM table */
+	if (!is_gmin) {
+		ret = gmin_get_config_dsm_var(maindev, var, out, out_len);
+		if (!ret)
+			return 0;
+	}
+
 	/* Our variable names are ASCII by construction, but EFI names
 	 * are wide chars.  Convert and zero-pad.
 	 */
@@ -1447,3 +1458,243 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0f38, isp_pm_cap_fixup);
 
 MODULE_DESCRIPTION("Ancillary routines for binding ACPI devices");
 MODULE_LICENSE("GPL");
+
+/*
+ * The below helper functions don't really belong here and should eventually be
+ * moved to some place under drivers/media/v4l2-core.
+ */
+#include <linux/platform_data/x86/soc.h>
+
+/*
+ * 79234640-9e10-4fea-a5c1-b5aa8b19756f
+ * This _DSM GUID returns information about the GPIO lines mapped to a sensor.
+ * Function number 1 returns a count of the GPIO lines that are mapped.
+ * Subsequent functions return 32 bit ints encoding information about the GPIO.
+ */
+static const guid_t intel_sensor_gpio_info_guid =
+	GUID_INIT(0x79234640, 0x9e10, 0x4fea,
+		  0xa5, 0xc1, 0xb5, 0xaa, 0x8b, 0x19, 0x75, 0x6f);
+
+/*
+ * 822ace8f-2814-4174-a56b-5f029fe079ee
+ * This _DSM GUID returns a string from the sensor device, which acts as a
+ * module identifier.
+ */
+static const guid_t intel_sensor_module_guid =
+	GUID_INIT(0x822ace8f, 0x2814, 0x4174,
+		  0xa5, 0x6b, 0x5f, 0x02, 0x9f, 0xe0, 0x79, 0xee);
+
+#define INTEL_DSM_TYPE_SHIFT				0
+#define INTEL_DSM_TYPE_MASK				GENMASK(7, 0)
+#define INTEL_DSM_PIN_SHIFT				8
+#define INTEL_DSM_PIN_MASK				GENMASK(15, 8)
+#define INTEL_DSM_SENSOR_ON_VAL_SHIFT			24
+#define INTEL_DSM_SENSOR_ON_VAL_MASK			GENMASK(31, 24)
+
+#define INTEL_DSM_TYPE(x) \
+	(((x) & INTEL_DSM_TYPE_MASK) >> INTEL_DSM_TYPE_SHIFT)
+#define INTEL_DSM_PIN(x) \
+	(((x) & INTEL_DSM_PIN_MASK) >> INTEL_DSM_PIN_SHIFT)
+#define INTEL_DSM_SENSOR_ON_VAL(x) \
+	(((x) & INTEL_DSM_SENSOR_ON_VAL_MASK) >> INTEL_DSM_SENSOR_ON_VAL_SHIFT)
+
+#define V4L2_SENSOR_MAX_ACPI_GPIOS			2u
+
+struct v4l2_acpi_gpio_map {
+	struct acpi_gpio_params params[V4L2_SENSOR_MAX_ACPI_GPIOS];
+	struct acpi_gpio_mapping mapping[V4L2_SENSOR_MAX_ACPI_GPIOS + 1];
+};
+
+struct v4l2_acpi_gpio_parsing_data {
+	struct device *dev;
+	u32 settings[V4L2_SENSOR_MAX_ACPI_GPIOS];
+	unsigned int settings_count;
+	unsigned int res_count;
+	unsigned int map_count;
+	struct v4l2_acpi_gpio_map *map;
+};
+
+/* Note this always returns 1 to continue looping so that res_count is accurate */
+static int v4l2_acpi_handle_gpio_res(struct acpi_resource *ares, void *_data)
+{
+	struct v4l2_acpi_gpio_parsing_data *data = _data;
+	struct acpi_resource_gpio *agpio;
+	const char *name;
+	bool active_low;
+	unsigned int i;
+	u32 settings;
+	u8 pin;
+
+	if (!acpi_gpio_get_io_resource(ares, &agpio))
+		return 1; /* Not a GPIO, continue the loop */
+
+	data->res_count++;
+
+	pin = agpio->pin_table[0];
+	for (i = 0; i < data->settings_count; i++) {
+		if (INTEL_DSM_PIN(data->settings[i]) == pin) {
+			settings = data->settings[i];
+			break;
+		}
+	}
+
+	if (i == data->settings_count) {
+		dev_warn(data->dev, "Could not find DSM GPIO settings for pin %d\n", pin);
+		return 1;
+	}
+
+	switch (INTEL_DSM_TYPE(settings)) {
+	case 0:
+		name = "reset-gpios";
+		break;
+	case 1:
+		name = "powerdown-gpios";
+		break;
+	default:
+		dev_warn(data->dev, "Unknown GPIO type 0x%02lx for pin %d\n",
+			 INTEL_DSM_TYPE(settings), pin);
+		return 1;
+	}
+
+	/*
+	 * Both reset and power-down need to be logical false when the sensor
+	 * is on (sensor should not be in reset and not be powered-down). So
+	 * when the sensor-on-value (which is the physical pin value) is high,
+	 * then the signal is active-low.
+	 */
+	active_low = INTEL_DSM_SENSOR_ON_VAL(settings) ? true : false;
+
+	i = data->map_count;
+	if (i == V4L2_SENSOR_MAX_ACPI_GPIOS)
+		return 1;
+
+	/* res_count is already incremented */
+	data->map->params[i].crs_entry_index = data->res_count - 1;
+	data->map->params[i].active_low = active_low;
+	data->map->mapping[i].name = name;
+	data->map->mapping[i].data = &data->map->params[i];
+	data->map->mapping[i].size = 1;
+	data->map_count++;
+
+	dev_info(data->dev, "%s crs %d %s pin %d active-%s\n", name,
+		 data->res_count - 1, agpio->resource_source.string_ptr,
+		 pin, active_low ? "low" : "high");
+
+	return 1;
+}
+
+/*
+ * Helper function to create an ACPI GPIO lookup table for sensor reset and
+ * powerdown signals on Intel Bay Trail (BYT) and Cherry Trail (CHT) devices,
+ * including setting the correct polarity for the GPIO.
+ *
+ * This uses the "79234640-9e10-4fea-a5c1-b5aa8b19756f" DSM method directly
+ * on the sensor device's ACPI node. This is different from later Intel
+ * hardware which has a separate INT3472 with this info. Since there is
+ * no separate firmware-node to which we can bind to register the GPIO lookups
+ * this unfortunately means that all sensor drivers which may be used on
+ * BYT or CHT hw need to call this function. This also means that this function
+ * may only fail when it is actually called on BYT/CHT hw. In all other cases
+ * it must always succeed.
+ *
+ * Note this code uses the same DSM GUID as the INT3472 discrete.c code
+ * and there is some overlap, but there are enough differences that it is
+ * difficult to share the code.
+ */
+int v4l2_get_acpi_sensor_info(struct device *dev, char **module_id_str)
+{
+	struct acpi_device *adev = ACPI_COMPANION(dev);
+	struct v4l2_acpi_gpio_parsing_data data = { };
+	LIST_HEAD(resource_list);
+	union acpi_object *obj;
+	unsigned int i, j;
+	int ret;
+
+	if (module_id_str)
+		*module_id_str = NULL;
+
+	if (!adev)
+		return 0;
+
+	obj = acpi_evaluate_dsm_typed(adev->handle, &intel_sensor_module_guid,
+				      0x00, 0x01, NULL, ACPI_TYPE_STRING);
+	if (obj) {
+		dev_info(dev, "Sensor module id: '%s'\n", obj->string.pointer);
+		if (module_id_str)
+			*module_id_str = kstrdup(obj->string.pointer, GFP_KERNEL);
+
+		ACPI_FREE(obj);
+	}
+
+	if (!soc_intel_is_byt() && !soc_intel_is_cht())
+		return 0;
+
+	/*
+	 * First get the GPIO-settings count and then get count GPIO-settings
+	 * values. Note the order of these may differ from the order in which
+	 * the GPIOs are listed on the ACPI resources! So we first store them all
+	 * and then enumerate the ACPI resources and match them up by pin number.
+	 */
+	obj = acpi_evaluate_dsm_typed(adev->handle,
+				      &intel_sensor_gpio_info_guid, 0x00, 1,
+				      NULL, ACPI_TYPE_INTEGER);
+	if (!obj)
+		return dev_err_probe(dev, -EIO, "No _DSM entry for GPIO pin count\n");
+
+	data.settings_count = obj->integer.value;
+	ACPI_FREE(obj);
+
+	if (data.settings_count > V4L2_SENSOR_MAX_ACPI_GPIOS)
+		return dev_err_probe(dev, -EIO, "Too many GPIOs %u > %u\n",
+				     data.settings_count, V4L2_SENSOR_MAX_ACPI_GPIOS);
+
+	for (i = 0; i < data.settings_count; i++) {
+		/*
+		 * i + 2 because the index of this _DSM function is 1-based
+		 * and the first function is just a count.
+		 */
+		obj = acpi_evaluate_dsm_typed(adev->handle,
+					      &intel_sensor_gpio_info_guid,
+					      0x00, i + 2,
+					      NULL, ACPI_TYPE_INTEGER);
+		if (!obj)
+			return dev_err_probe(dev, -EIO, "No _DSM entry for GPIO pin %u\n", i);
+
+		data.settings[i] = obj->integer.value;
+		ACPI_FREE(obj);
+	}
+
+	/* Since we match up by pin-number the pin-numbers must be unique */
+	for (i = 0; i < data.settings_count; i++) {
+		for (j = i + 1; j < data.settings_count; j++) {
+			if (INTEL_DSM_PIN(data.settings[i]) !=
+			    INTEL_DSM_PIN(data.settings[j]))
+				continue;
+
+			return dev_err_probe(dev, -EIO, "Duplicate pin number %lu\n",
+					     INTEL_DSM_PIN(data.settings[i]));
+		}
+	}
+
+	/* Use devm_kzalloc() for the mappings + params to auto-free them */
+	data.map = devm_kzalloc(dev, sizeof(*data.map), GFP_KERNEL);
+	if (!data.map)
+		return -ENOMEM;
+
+	/* Now parse the ACPI resources and build the lookup table */
+	data.dev = dev;
+	ret = acpi_dev_get_resources(adev, &resource_list,
+				     v4l2_acpi_handle_gpio_res, &data);
+	if (ret < 0)
+		return ret;
+
+	acpi_dev_free_resource_list(&resource_list);
+
+	if (data.map_count != data.settings_count ||
+	    data.res_count != data.settings_count)
+		dev_warn(dev, "ACPI GPIO resources vs DSM GPIO-info count mismatch (dsm: %d res: %d map %d\n",
+			 data.settings_count, data.res_count, data.map_count);
+
+	return devm_acpi_dev_add_driver_gpios(dev, data.map->mapping);
+}
+EXPORT_SYMBOL_GPL(v4l2_get_acpi_sensor_info);
diff --git a/drivers/staging/media/atomisp/pci/atomisp_internal.h b/drivers/staging/media/atomisp/pci/atomisp_internal.h
index fa38d91420cf..1fac99f4e2b0 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_internal.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_internal.h
@@ -48,8 +48,6 @@
 	(((isp)->media_dev.hw_revision & ATOMISP_HW_REVISION_MASK) == \
 	 ((rev) << ATOMISP_HW_REVISION_SHIFT))
 
-#define MAX_STREAM_NUM	2
-
 #define ATOMISP_PCI_DEVICE_SOC_MASK	0xfff8
 /* MRFLD with 0x1178: ISP freq can burst to 457MHz */
 #define ATOMISP_PCI_DEVICE_SOC_MRFLD	0x1178
@@ -76,9 +74,6 @@
 #define ATOM_RESOLUTION_SUBQCIF_WIDTH	128
 #define ATOM_RESOLUTION_SUBQCIF_HEIGHT	96
 
-#define ATOM_ISP_MAX_WIDTH_TMP	1280
-#define ATOM_ISP_MAX_HEIGHT_TMP	720
-
 #define ATOM_ISP_I2C_BUS_1	4
 #define ATOM_ISP_I2C_BUS_2	5
 
@@ -101,10 +96,6 @@
 #define ATOMISP_METADATA_QUEUE_DEPTH_FOR_HAL	8
 #define ATOMISP_S3A_BUF_QUEUE_DEPTH_FOR_HAL	8
 
-#define ATOMISP_DELAYED_INIT_NOT_QUEUED	0
-#define ATOMISP_DELAYED_INIT_QUEUED	1
-#define ATOMISP_DELAYED_INIT_DONE	2
-
 /*
  * Define how fast CPU should be able to serve ISP interrupts.
  * The bigger the value, the higher risk that the ISP is not
@@ -121,23 +112,6 @@
 #define ATOMISP_CSS_OUTPUT_SECOND_INDEX     1
 #define ATOMISP_CSS_OUTPUT_DEFAULT_INDEX    0
 
-/*
- * ATOMISP_SOC_CAMERA
- * This is to differentiate between ext-isp and soc camera in
- * Moorefield/Baytrail platform.
- */
-#define ATOMISP_SOC_CAMERA(asd)  \
-	(asd->isp->inputs[asd->input_curr].type == SOC_CAMERA)
-
-#define ATOMISP_USE_YUVPP(asd)  \
-	(ATOMISP_SOC_CAMERA(asd) && ATOMISP_CSS_SUPPORT_YUVPP && \
-	!asd->copy_mode)
-
-#define ATOMISP_DEPTH_SENSOR_STREAMON_COUNT 2
-
-#define ATOMISP_DEPTH_DEFAULT_MASTER_SENSOR 0
-#define ATOMISP_DEPTH_DEFAULT_SLAVE_SENSOR 1
-
 /* ISP2401 */
 #define ATOMISP_ION_DEVICE_FD_OFFSET   16
 #define ATOMISP_ION_SHARED_FD_MASK     (0xFFFF)
@@ -205,6 +179,7 @@ struct atomisp_device {
 	struct device *dev;
 	struct v4l2_device v4l2_dev;
 	struct media_device media_dev;
+	struct atomisp_sub_device asd;
 	struct atomisp_platform_data *pdata;
 	void *mmu_l1_base;
 	void __iomem *base;
@@ -214,18 +189,6 @@ struct atomisp_device {
 	struct pm_qos_request pm_qos;
 	s32 max_isr_latency;
 
-	/*
-	 * ISP modules
-	 * Multiple streams are represents by multiple
-	 * atomisp_sub_device instances
-	 */
-	struct atomisp_sub_device *asd;
-	/*
-	 * this will be assigned dyanamically.
-	 * For Merr/BTY(ISP2400), 2 streams are supported.
-	 */
-	unsigned int num_of_streams;
-
 	struct atomisp_mipi_csi2_device csi2_port[ATOMISP_CAMERA_NR_PORTS];
 	struct atomisp_tpg_device tpg;
 
@@ -246,7 +209,7 @@ struct atomisp_device {
 	bool isp_fatal_error;
 	struct work_struct assert_recovery_work;
 
-	spinlock_t lock; /* Protects asd[i].streaming */
+	spinlock_t lock; /* Protects asd.streaming */
 
 	bool need_gfx_throttle;
 
diff --git a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
index d1314bdbf7d5..384f31fc66c5 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
@@ -626,13 +626,7 @@ atomisp_subdev_streaming_count(struct atomisp_sub_device *asd)
 
 unsigned int atomisp_streaming_count(struct atomisp_device *isp)
 {
-	unsigned int i, sum;
-
-	for (i = 0, sum = 0; i < isp->num_of_streams; i++)
-		sum += isp->asd[i].streaming ==
-		       ATOMISP_DEVICE_STREAMING_ENABLED;
-
-	return sum;
+	return isp->asd.streaming == ATOMISP_DEVICE_STREAMING_ENABLED;
 }
 
 /*
@@ -677,18 +671,6 @@ static int atomisp_s_input(struct file *file, void *fh, unsigned int input)
 		return -EINVAL;
 	}
 
-	/*
-	 * check whether the request camera:
-	 * 1: already in use
-	 * 2: if in use, whether it is used by other streams
-	 */
-	if (isp->inputs[input].asd && isp->inputs[input].asd != asd) {
-		dev_err(isp->dev,
-			"%s, camera is already used by stream: %d\n", __func__,
-			isp->inputs[input].asd->index);
-		return -EBUSY;
-	}
-
 	camera = isp->inputs[input].camera;
 	if (!camera) {
 		dev_err(isp->dev, "%s, no camera\n", __func__);
@@ -1112,24 +1094,13 @@ static int atomisp_dqbuf_wrapper(struct file *file, void *fh, struct v4l2_buffer
 	buf->reserved2 = pipe->frame_config_id[buf->index];
 
 	dev_dbg(isp->dev,
-		"dqbuf buffer %d (%s) for asd%d with exp_id %d, isp_config_id %d\n",
-		buf->index, vdev->name, asd->index, buf->reserved >> 16,
-		buf->reserved2);
+		"dqbuf buffer %d (%s) with exp_id %d, isp_config_id %d\n",
+		buf->index, vdev->name, buf->reserved >> 16, buf->reserved2);
 	return 0;
 }
 
 enum ia_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd)
 {
-	if (ATOMISP_USE_YUVPP(asd))
-		return IA_CSS_PIPE_ID_YUVPP;
-
-	if (asd->continuous_mode->val) {
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-			return IA_CSS_PIPE_ID_VIDEO;
-		else
-			return IA_CSS_PIPE_ID_PREVIEW;
-	}
-
 	/*
 	 * Disable vf_pp and run CSS in video mode. This allows using ISP
 	 * scaling but it has one frame delay due to CSS internal buffering.
@@ -1164,60 +1135,12 @@ static unsigned int atomisp_sensor_start_stream(struct atomisp_sub_device *asd)
 
 	if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO ||
 	    (asd->run_mode->val == ATOMISP_RUN_MODE_STILL_CAPTURE &&
-	     !atomisp_is_mbuscode_raw(
-		 asd->fmt[
-		  asd->capture_pad].fmt.code) &&
-	     !asd->continuous_mode->val))
+	     !atomisp_is_mbuscode_raw(asd->fmt[asd->capture_pad].fmt.code)))
 		return 2;
 	else
 		return 1;
 }
 
-int atomisp_stream_on_master_slave_sensor(struct atomisp_device *isp,
-	bool isp_timeout)
-{
-	unsigned int master, slave, delay_slave = 0;
-	int ret;
-
-	master = ATOMISP_DEPTH_DEFAULT_MASTER_SENSOR;
-	slave = ATOMISP_DEPTH_DEFAULT_SLAVE_SENSOR;
-	dev_warn(isp->dev,
-		 "depth mode use default master=%s.slave=%s.\n",
-		 isp->inputs[master].camera->name,
-		 isp->inputs[slave].camera->name);
-
-	ret = v4l2_subdev_call(isp->inputs[master].camera, core,
-			       ioctl, ATOMISP_IOC_G_DEPTH_SYNC_COMP,
-			       &delay_slave);
-	if (ret)
-		dev_warn(isp->dev,
-			 "get depth sensor %s compensation delay failed.\n",
-			 isp->inputs[master].camera->name);
-
-	ret = v4l2_subdev_call(isp->inputs[master].camera,
-			       video, s_stream, 1);
-	if (ret) {
-		dev_err(isp->dev, "depth mode master sensor %s stream-on failed.\n",
-			isp->inputs[master].camera->name);
-		return -EINVAL;
-	}
-
-	if (delay_slave != 0)
-		udelay(delay_slave);
-
-	ret = v4l2_subdev_call(isp->inputs[slave].camera,
-			       video, s_stream, 1);
-	if (ret) {
-		dev_err(isp->dev, "depth mode slave sensor %s stream-on failed.\n",
-			isp->inputs[slave].camera->name);
-		v4l2_subdev_call(isp->inputs[master].camera, video, s_stream, 0);
-
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
 /* Input system HW workaround */
 /* Input system address translation corrupts burst during */
 /* invalidate. SW workaround for this is to set burst length */
@@ -1250,8 +1173,7 @@ int atomisp_start_streaming(struct vb2_queue *vq, unsigned int count)
 
 	mutex_lock(&isp->mutex);
 
-	dev_dbg(isp->dev, "Start stream on pad %d for asd%d\n",
-		atomisp_subdev_source_pad(vdev), asd->index);
+	dev_dbg(isp->dev, "Start stream on pad %d\n", atomisp_subdev_source_pad(vdev));
 
 	ret = atomisp_pipe_check(pipe, false);
 	if (ret)
@@ -1266,57 +1188,7 @@ int atomisp_start_streaming(struct vb2_queue *vq, unsigned int count)
 	 */
 	sensor_start_stream = atomisp_sensor_start_stream(asd);
 
-	/* Reset pending capture request count. */
-	asd->pending_capture_request = 0;
-
 	if (atomisp_subdev_streaming_count(asd) > sensor_start_stream) {
-		/* trigger still capture */
-		if (asd->continuous_mode->val &&
-		    atomisp_subdev_source_pad(vdev)
-		    == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE) {
-			if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-				dev_dbg(isp->dev, "SDV last video raw buffer id: %u\n",
-					asd->latest_preview_exp_id);
-			else
-				dev_dbg(isp->dev, "ZSL last preview raw buffer id: %u\n",
-					asd->latest_preview_exp_id);
-
-			if (asd->delayed_init == ATOMISP_DELAYED_INIT_QUEUED) {
-				flush_work(&asd->delayed_init_work);
-				mutex_unlock(&isp->mutex);
-				ret = wait_for_completion_interruptible(&asd->init_done);
-				mutex_lock(&isp->mutex);
-				if (ret) {
-					ret = -ERESTARTSYS;
-					goto out_unlock;
-				}
-			}
-
-			/* handle per_frame_setting parameter and buffers */
-			atomisp_handle_parameter_and_buffer(pipe);
-
-			/*
-			 * only ZSL/SDV capture request will be here, raise
-			 * the ISP freq to the highest possible to minimize
-			 * the S2S latency.
-			 */
-			atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, false);
-			/*
-			 * When asd->enable_raw_buffer_lock->val is true,
-			 * An extra IOCTL is needed to call
-			 * atomisp_css_exp_id_capture and trigger real capture
-			 */
-			if (!asd->enable_raw_buffer_lock->val) {
-				ret = atomisp_css_offline_capture_configure(asd,
-					asd->params.offline_parm.num_captures,
-					asd->params.offline_parm.skip_frames,
-					asd->params.offline_parm.offset);
-				if (ret) {
-					ret = -EINVAL;
-					goto out_unlock;
-				}
-			}
-		}
 		atomisp_qbuffers_to_css(asd);
 		ret = 0;
 		goto out_unlock;
@@ -1396,19 +1268,6 @@ start_sensor:
 			dev_dbg(isp->dev, "DFS auto mode failed!\n");
 	}
 
-	if (asd->depth_mode->val && atomisp_streaming_count(isp) ==
-	    ATOMISP_DEPTH_SENSOR_STREAMON_COUNT) {
-		ret = atomisp_stream_on_master_slave_sensor(isp, false);
-		if (ret) {
-			dev_err(isp->dev, "master slave sensor stream on failed!\n");
-			goto out_unlock;
-		}
-		goto start_delay_wq;
-	} else if (asd->depth_mode->val && (atomisp_streaming_count(isp) <
-					    ATOMISP_DEPTH_SENSOR_STREAMON_COUNT)) {
-		goto start_delay_wq;
-	}
-
 	/* Enable the CSI interface on ANN B0/K0 */
 	if (isp->media_dev.hw_revision >= ((ATOMISP_HW_REVISION_ISP2401 <<
 					    ATOMISP_HW_REVISION_SHIFT) | ATOMISP_HW_STEPPING_B0)) {
@@ -1427,19 +1286,6 @@ start_sensor:
 		goto out_unlock;
 	}
 
-start_delay_wq:
-	if (asd->continuous_mode->val) {
-		atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-				        V4L2_SUBDEV_FORMAT_ACTIVE,
-				        ATOMISP_SUBDEV_PAD_SINK);
-
-		reinit_completion(&asd->init_done);
-		asd->delayed_init = ATOMISP_DELAYED_INIT_QUEUED;
-		queue_work(asd->delayed_init_workq, &asd->delayed_init_work);
-	} else {
-		asd->delayed_init = ATOMISP_DELAYED_INIT_NOT_QUEUED;
-	}
-
 out_unlock:
 	mutex_unlock(&isp->mutex);
 	return ret;
@@ -1452,16 +1298,15 @@ void atomisp_stop_streaming(struct vb2_queue *vq)
 	struct video_device *vdev = &pipe->vdev;
 	struct atomisp_device *isp = asd->isp;
 	struct pci_dev *pdev = to_pci_dev(isp->dev);
-	bool recreate_streams[MAX_STREAM_NUM] = {0};
 	enum ia_css_pipe_id css_pipe_id;
+	bool recreate_stream = false;
 	bool first_streamoff = false;
 	unsigned long flags;
-	int i, ret;
+	int ret;
 
 	mutex_lock(&isp->mutex);
 
-	dev_dbg(isp->dev, "Stop stream on pad %d for asd%d\n",
-		atomisp_subdev_source_pad(vdev), asd->index);
+	dev_dbg(isp->dev, "Stop stream on pad %d\n", atomisp_subdev_source_pad(vdev));
 
 	/*
 	 * There is no guarantee that the buffers queued to / owned by the ISP
@@ -1479,24 +1324,6 @@ void atomisp_stop_streaming(struct vb2_queue *vq)
 	if (ret == 0)
 		dev_warn(isp->dev, "Warning timeout waiting for CSS to return buffers\n");
 
-	/*
-	 * do only videobuf_streamoff for capture & vf pipes in
-	 * case of continuous capture
-	 */
-	if (asd->continuous_mode->val &&
-	    atomisp_subdev_source_pad(vdev) != ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW &&
-	    atomisp_subdev_source_pad(vdev) != ATOMISP_SUBDEV_PAD_SOURCE_VIDEO) {
-		if (atomisp_subdev_source_pad(vdev) == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE) {
-			/* stop continuous still capture if needed */
-			if (asd->params.offline_parm.num_captures == -1)
-				atomisp_css_offline_capture_configure(asd,
-								      0, 0, 0);
-			atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_AUTO, false);
-		}
-
-		goto out_unlock;
-	}
-
 	if (asd->streaming == ATOMISP_DEVICE_STREAMING_ENABLED)
 		first_streamoff = true;
 
@@ -1513,11 +1340,6 @@ void atomisp_stop_streaming(struct vb2_queue *vq)
 	atomisp_clear_css_buffer_counters(asd);
 	atomisp_css_irq_enable(isp, IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF, false);
 
-	if (asd->delayed_init == ATOMISP_DELAYED_INIT_QUEUED) {
-		cancel_work_sync(&asd->delayed_init_work);
-		asd->delayed_init = ATOMISP_DELAYED_INIT_NOT_QUEUED;
-	}
-
 	css_pipe_id = atomisp_get_css_pipe_id(asd);
 	atomisp_css_stop(asd, css_pipe_id, false);
 
@@ -1566,11 +1388,9 @@ stopsensor:
 	 *
 	 * So force stream destroy here.
 	 */
-	for (i = 0; i < isp->num_of_streams; i++) {
-		if (isp->asd[i].stream_prepared) {
-			atomisp_destroy_pipes_stream_force(&isp->asd[i]);
-			recreate_streams[i] = true;
-		}
+	if (isp->asd.stream_prepared) {
+		atomisp_destroy_pipes_stream_force(&isp->asd);
+		recreate_stream = true;
 	}
 
 	/* disable  PUNIT/ISP acknowlede/handshake - SRSE=3 */
@@ -1578,19 +1398,18 @@ stopsensor:
 			       isp->saved_regs.i_control | MRFLD_PCI_I_CONTROL_SRSE_RESET_MASK);
 	dev_err(isp->dev, "atomisp_reset");
 	atomisp_reset(isp);
-	for (i = 0; i < isp->num_of_streams; i++) {
-		if (recreate_streams[i]) {
-			int ret2;
-
-			ret2 = atomisp_create_pipes_stream(&isp->asd[i]);
-			if (ret2) {
-				dev_err(isp->dev, "%s error re-creating streams: %d\n",
-					__func__, ret2);
-				if (!ret)
-					ret = ret2;
-			}
+
+	if (recreate_stream) {
+		int ret2;
+
+		ret2 = atomisp_create_pipes_stream(&isp->asd);
+		if (ret2) {
+			dev_err(isp->dev, "%s error re-creating streams: %d\n", __func__, ret2);
+			if (!ret)
+				ret = ret2;
 		}
 	}
+
 	isp->isp_timeout = false;
 out_unlock:
 	mutex_unlock(&isp->mutex);
@@ -2304,9 +2123,6 @@ static long atomisp_vidioc_default(struct file *file, void *fh,
 		err = atomisp_set_parameters(vdev, arg);
 		break;
 
-	case ATOMISP_IOC_S_CONT_CAPTURE_CONFIG:
-		err = atomisp_offline_capture_configure(asd, arg);
-		break;
 	case ATOMISP_IOC_G_METADATA:
 		err = atomisp_get_metadata(asd, 0, arg);
 		break;
diff --git a/drivers/staging/media/atomisp/pci/atomisp_ioctl.h b/drivers/staging/media/atomisp/pci/atomisp_ioctl.h
index 59e071f035f9..db6da77df06b 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_ioctl.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_ioctl.h
@@ -53,10 +53,4 @@ unsigned int atomisp_streaming_count(struct atomisp_device *isp);
 long atomisp_compat_ioctl32(struct file *file,
 			    unsigned int cmd, unsigned long arg);
 
-int atomisp_stream_on_master_slave_sensor(struct atomisp_device *isp,
-	bool isp_timeout);
-
-int atomisp_start_streaming(struct vb2_queue *vq, unsigned int count);
-void atomisp_stop_streaming(struct vb2_queue *vq);
-
 #endif /* __ATOMISP_IOCTL_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.c b/drivers/staging/media/atomisp/pci/atomisp_subdev.c
index 9cfb85c61db6..a0acfdb87177 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_subdev.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.c
@@ -189,7 +189,6 @@ static int isp_subdev_subscribe_event(struct v4l2_subdev *sd,
 	    sub->type != V4L2_EVENT_ATOMISP_METADATA_READY &&
 	    sub->type != V4L2_EVENT_ATOMISP_PAUSE_BUFFER &&
 	    sub->type != V4L2_EVENT_ATOMISP_CSS_RESET &&
-	    sub->type != V4L2_EVENT_ATOMISP_RAW_BUFFERS_ALLOC_DONE &&
 	    sub->type != V4L2_EVENT_ATOMISP_ACC_COMPLETE)
 		return -EINVAL;
 
@@ -410,11 +409,6 @@ int atomisp_subdev_set_selection(struct v4l2_subdev *sd,
 			padding_h = 12;
 		}
 
-		if (isp->inputs[isp_sd->input_curr].type == SOC_CAMERA) {
-			padding_w = 0;
-			padding_h = 0;
-		}
-
 		if (atomisp_subdev_format_conversion(isp_sd,
 						     isp_sd->capture_pad)
 		    && crop[pad]->width && crop[pad]->height) {
@@ -422,13 +416,8 @@ int atomisp_subdev_set_selection(struct v4l2_subdev *sd,
 			crop[pad]->height -= padding_h;
 		}
 
-		/* if subdev type is SOC camera,we do not need to set DVS */
-		if (isp->inputs[isp_sd->input_curr].type == SOC_CAMERA)
-			isp_sd->params.video_dis_en = 0;
-
 		if (isp_sd->params.video_dis_en &&
-		    isp_sd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-		    !isp_sd->continuous_mode->val) {
+		    isp_sd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
 			/* This resolution contains 20 % of DVS slack
 			 * (of the desired captured image before
 			 * scaling, or 1 / 6 of what we get from the
@@ -459,8 +448,7 @@ int atomisp_subdev_set_selection(struct v4l2_subdev *sd,
 			break;
 
 		if (isp_sd->params.video_dis_en &&
-		    isp_sd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-		    !isp_sd->continuous_mode->val) {
+		    isp_sd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
 			dvs_w = rounddown(crop[pad]->width / 5,
 					  ATOM_ISP_STEP_WIDTH);
 			dvs_h = rounddown(crop[pad]->height / 5,
@@ -727,11 +715,7 @@ static int __atomisp_update_run_mode(struct atomisp_sub_device *asd)
 	struct v4l2_ctrl *c;
 	s32 mode;
 
-	if (ctrl->val != ATOMISP_RUN_MODE_VIDEO &&
-	    asd->continuous_mode->val)
-		mode = ATOMISP_RUN_MODE_PREVIEW;
-	else
-		mode = ctrl->val;
+	mode = ctrl->val;
 
 	c = v4l2_ctrl_find(
 		isp->inputs[asd->input_curr].camera->ctrl_handler,
@@ -758,23 +742,9 @@ static int s_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct atomisp_sub_device *asd = container_of(
 					     ctrl->handler, struct atomisp_sub_device, ctrl_handler);
-	unsigned int streaming;
-	unsigned long flags;
-
 	switch (ctrl->id) {
 	case V4L2_CID_RUN_MODE:
 		return __atomisp_update_run_mode(asd);
-	case V4L2_CID_DEPTH_MODE:
-		/* Use spinlock instead of mutex to avoid possible locking issues */
-		spin_lock_irqsave(&asd->isp->lock, flags);
-		streaming = asd->streaming;
-		spin_unlock_irqrestore(&asd->isp->lock, flags);
-		if (streaming != ATOMISP_DEVICE_STREAMING_DISABLED) {
-			dev_err(asd->isp->dev,
-				"ISP is streaming, it is not supported to change the depth mode\n");
-			return -EINVAL;
-		}
-		break;
 	}
 
 	return 0;
@@ -784,17 +754,6 @@ static const struct v4l2_ctrl_ops ctrl_ops = {
 	.s_ctrl = &s_ctrl,
 };
 
-static const struct v4l2_ctrl_config ctrl_fmt_auto = {
-	.ops = &ctrl_ops,
-	.id = V4L2_CID_FMT_AUTO,
-	.name = "Automatic format guessing",
-	.type = V4L2_CTRL_TYPE_BOOLEAN,
-	.min = 0,
-	.max = 1,
-	.step = 1,
-	.def = 1,
-};
-
 static const char *const ctrl_run_mode_menu[] = {
 	NULL,
 	"Video",
@@ -831,24 +790,6 @@ static const struct v4l2_ctrl_config ctrl_vfpp = {
 };
 
 /*
- * Control for ISP continuous mode
- *
- * When enabled, capture processing is possible without
- * stopping the preview pipeline. When disabled, ISP needs
- * to be restarted between preview and capture.
- */
-static const struct v4l2_ctrl_config ctrl_continuous_mode = {
-	.ops = &ctrl_ops,
-	.id = V4L2_CID_ATOMISP_CONTINUOUS_MODE,
-	.type = V4L2_CTRL_TYPE_BOOLEAN,
-	.name = "Continuous mode",
-	.min = 0,
-	.max = 1,
-	.step = 1,
-	.def = 0,
-};
-
-/*
  * Control for continuous mode raw buffer size
  *
  * The size of the RAW ringbuffer sets limit on how much
@@ -930,24 +871,6 @@ static const struct v4l2_ctrl_config ctrl_disable_dz = {
 	.def = 0,
 };
 
-/*
- * Control for ISP depth mode
- *
- * When enabled, that means ISP will deal with dual streams and sensors will be
- * in slave/master mode.
- * slave sensor will have no output until master sensor is streamed on.
- */
-static const struct v4l2_ctrl_config ctrl_depth_mode = {
-	.ops = &ctrl_ops,
-	.id = V4L2_CID_DEPTH_MODE,
-	.type = V4L2_CTRL_TYPE_BOOLEAN,
-	.name = "Depth mode",
-	.min = 0,
-	.max = 1,
-	.step = 1,
-	.def = 0,
-};
-
 static int atomisp_init_subdev_pipe(struct atomisp_sub_device *asd,
 				    struct atomisp_video_pipe *pipe, enum v4l2_buf_type buf_type)
 {
@@ -995,7 +918,7 @@ static int isp_subdev_init_entities(struct atomisp_sub_device *asd)
 	int ret;
 
 	v4l2_subdev_init(sd, &isp_subdev_v4l2_ops);
-	sprintf(sd->name, "ATOMISP_SUBDEV_%d", asd->index);
+	sprintf(sd->name, "ATOMISP_SUBDEV");
 	v4l2_set_subdevdata(sd, asd);
 	sd->flags |= V4L2_SUBDEV_FL_HAS_EVENTS | V4L2_SUBDEV_FL_HAS_DEVNODE;
 
@@ -1066,14 +989,10 @@ static int isp_subdev_init_entities(struct atomisp_sub_device *asd)
 	if (ret)
 		return ret;
 
-	asd->fmt_auto = v4l2_ctrl_new_custom(&asd->ctrl_handler,
-					     &ctrl_fmt_auto, NULL);
 	asd->run_mode = v4l2_ctrl_new_custom(&asd->ctrl_handler,
 					     &ctrl_run_mode, NULL);
 	asd->vfpp = v4l2_ctrl_new_custom(&asd->ctrl_handler,
 					 &ctrl_vfpp, NULL);
-	asd->continuous_mode = v4l2_ctrl_new_custom(&asd->ctrl_handler,
-			       &ctrl_continuous_mode, NULL);
 	asd->continuous_viewfinder = v4l2_ctrl_new_custom(&asd->ctrl_handler,
 				     &ctrl_continuous_viewfinder,
 				     NULL);
@@ -1086,10 +1005,6 @@ static int isp_subdev_init_entities(struct atomisp_sub_device *asd)
 	    v4l2_ctrl_new_custom(&asd->ctrl_handler,
 				 &ctrl_enable_raw_buffer_lock,
 				 NULL);
-	asd->depth_mode =
-	    v4l2_ctrl_new_custom(&asd->ctrl_handler,
-				 &ctrl_depth_mode,
-				 NULL);
 	asd->disable_dz =
 	    v4l2_ctrl_new_custom(&asd->ctrl_handler,
 				 &ctrl_disable_dz,
@@ -1103,21 +1018,16 @@ static int isp_subdev_init_entities(struct atomisp_sub_device *asd)
 
 int atomisp_create_pads_links(struct atomisp_device *isp)
 {
-	struct atomisp_sub_device *asd;
-	int i, j, ret = 0;
+	int i, ret;
 
-	isp->num_of_streams = 2;
 	for (i = 0; i < ATOMISP_CAMERA_NR_PORTS; i++) {
-		for (j = 0; j < isp->num_of_streams; j++) {
-			ret =
-			    media_create_pad_link(&isp->csi2_port[i].subdev.
-						  entity, CSI2_PAD_SOURCE,
-						  &isp->asd[j].subdev.entity,
-						  ATOMISP_SUBDEV_PAD_SINK, 0);
-			if (ret < 0)
-				return ret;
-		}
+		ret = media_create_pad_link(&isp->csi2_port[i].subdev.entity,
+					    CSI2_PAD_SOURCE, &isp->asd.subdev.entity,
+					    ATOMISP_SUBDEV_PAD_SINK, 0);
+		if (ret < 0)
+			return ret;
 	}
+
 	for (i = 0; i < isp->input_cnt; i++) {
 		/* Don't create links for the test-pattern-generator */
 		if (isp->inputs[i].type == TEST_PATTERN)
@@ -1132,33 +1042,28 @@ int atomisp_create_pads_links(struct atomisp_device *isp)
 		if (ret < 0)
 			return ret;
 	}
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		ret = media_create_pad_link(&asd->subdev.entity,
-					    ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW,
-					    &asd->video_out_preview.vdev.entity,
-					    0, 0);
-		if (ret < 0)
-			return ret;
-		ret = media_create_pad_link(&asd->subdev.entity,
-					    ATOMISP_SUBDEV_PAD_SOURCE_VF,
-					    &asd->video_out_vf.vdev.entity, 0,
-					    0);
-		if (ret < 0)
-			return ret;
-		ret = media_create_pad_link(&asd->subdev.entity,
-					    ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE,
-					    &asd->video_out_capture.vdev.entity,
-					    0, 0);
-		if (ret < 0)
-			return ret;
-		ret = media_create_pad_link(&asd->subdev.entity,
-					    ATOMISP_SUBDEV_PAD_SOURCE_VIDEO,
-					    &asd->video_out_video_capture.vdev.
-					    entity, 0, 0);
-		if (ret < 0)
-			return ret;
-	}
+
+	ret = media_create_pad_link(&isp->asd.subdev.entity,
+				    ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW,
+				    &isp->asd.video_out_preview.vdev.entity, 0, 0);
+	if (ret < 0)
+		return ret;
+	ret = media_create_pad_link(&isp->asd.subdev.entity,
+				    ATOMISP_SUBDEV_PAD_SOURCE_VF,
+				    &isp->asd.video_out_vf.vdev.entity, 0, 0);
+	if (ret < 0)
+		return ret;
+	ret = media_create_pad_link(&isp->asd.subdev.entity,
+				    ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE,
+				    &isp->asd.video_out_capture.vdev.entity, 0, 0);
+	if (ret < 0)
+		return ret;
+	ret = media_create_pad_link(&isp->asd.subdev.entity,
+				    ATOMISP_SUBDEV_PAD_SOURCE_VIDEO,
+				    &isp->asd.video_out_video_capture.vdev.entity, 0, 0);
+	if (ret < 0)
+		return ret;
+
 	return 0;
 }
 
@@ -1254,29 +1159,13 @@ error:
  */
 int atomisp_subdev_init(struct atomisp_device *isp)
 {
-	struct atomisp_sub_device *asd;
-	int i, ret = 0;
+	int ret;
 
-	/*
-	 * CSS2.0 running ISP2400 support
-	 * multiple streams
-	 */
-	isp->num_of_streams = 2;
-	isp->asd = devm_kzalloc(isp->dev, sizeof(struct atomisp_sub_device) *
-				isp->num_of_streams, GFP_KERNEL);
-	if (!isp->asd)
-		return -ENOMEM;
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		asd->isp = isp;
-		isp_subdev_init_params(asd);
-		asd->index = i;
-		ret = isp_subdev_init_entities(asd);
-		if (ret < 0) {
-			atomisp_subdev_cleanup_entities(asd);
-			break;
-		}
-	}
+	isp->asd.isp = isp;
+	isp_subdev_init_params(&isp->asd);
+	ret = isp_subdev_init_entities(&isp->asd);
+	if (ret < 0)
+		atomisp_subdev_cleanup_entities(&isp->asd);
 
 	return ret;
 }
diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.h b/drivers/staging/media/atomisp/pci/atomisp_subdev.h
index daa6077a83bd..fee663bc415a 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_subdev.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.h
@@ -183,9 +183,6 @@ struct atomisp_css_params {
 };
 
 struct atomisp_subdev_params {
-	/* FIXME: Determines whether raw capture buffer are being passed to
-	 * user space. Unimplemented for now. */
-	int online_process;
 	int yuv_ds_en;
 	unsigned int color_effect;
 	bool gdc_cac_en;
@@ -237,8 +234,6 @@ struct atomisp_subdev_params {
 	enum atomisp_flash_state flash_state;
 	enum atomisp_frame_status last_frame_status;
 
-	/* continuous capture */
-	struct atomisp_cont_capture_conf offline_parm;
 	/* Flag to check if driver needs to update params to css */
 	bool css_update_params_needed;
 };
@@ -264,11 +259,8 @@ struct atomisp_sub_device {
 	/* struct isp_subdev_params params; */
 	struct atomisp_device *isp;
 	struct v4l2_ctrl_handler ctrl_handler;
-	struct v4l2_ctrl *fmt_auto;
 	struct v4l2_ctrl *run_mode;
-	struct v4l2_ctrl *depth_mode;
 	struct v4l2_ctrl *vfpp;
-	struct v4l2_ctrl *continuous_mode;
 	struct v4l2_ctrl *continuous_raw_buffer_size;
 	struct v4l2_ctrl *continuous_viewfinder;
 	struct v4l2_ctrl *enable_raw_buffer_lock;
@@ -307,7 +299,6 @@ struct atomisp_sub_device {
 	spinlock_t dis_stats_lock;
 
 	struct ia_css_frame *vf_frame; /* TODO: needed? */
-	struct ia_css_frame *raw_output_frame;
 	enum atomisp_frame_status frame_status[VIDEO_MAX_FRAME];
 
 	/* This field specifies which camera (v4l2 input) is selected. */
@@ -324,23 +315,11 @@ struct atomisp_sub_device {
 	unsigned int streaming;
 	bool stream_prepared; /* whether css stream is created */
 
-	/* subdev index: will be used to show which subdev is holding the
-	 * resource, like which camera is used by which subdev
-	 */
-	unsigned int index;
-
-	/* delayed memory allocation for css */
-	struct completion init_done;
-	struct workqueue_struct *delayed_init_workq;
-	unsigned int delayed_init;
-	struct work_struct delayed_init_work;
-
 	unsigned int latest_preview_exp_id; /* CSS ZSL/SDV raw buffer id */
 
 	unsigned int mipi_frame_size;
 
 	bool copy_mode; /* CSI2+ use copy mode */
-	bool yuvpp_mode;	/* CSI2+ yuvpp pipe */
 
 	int raw_buffer_bitmap[ATOMISP_MAX_EXP_ID / 32 +
 						 1]; /* Record each Raw Buffer lock status */
@@ -352,7 +331,6 @@ struct atomisp_sub_device {
 
 	struct atomisp_resolution sensor_array_res;
 	bool high_speed_mode; /* Indicate whether now is a high speed mode */
-	int pending_capture_request; /* Indicates the number of pending capture requests. */
 
 	unsigned int preview_exp_id;
 	unsigned int postview_exp_id;
diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
index ba628f7cf385..3f315dabbeeb 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
@@ -64,10 +64,10 @@ module_param(dbg_level, int, 0644);
 MODULE_PARM_DESC(dbg_level, "debug message level (default:0)");
 
 /* log function switch */
-int dbg_func = 2;
+int dbg_func = 1;
 module_param(dbg_func, int, 0644);
 MODULE_PARM_DESC(dbg_func,
-		 "log function switch non/trace_printk/printk (default:printk)");
+		 "log function switch non/printk (default:printk)");
 
 int mipicsi_flag;
 module_param(mipicsi_flag, int, 0644);
@@ -753,8 +753,6 @@ static int atomisp_suspend(struct device *dev)
 {
 	struct atomisp_device *isp = (struct atomisp_device *)
 				     dev_get_drvdata(dev);
-	/* FIXME: only has one isp_subdev at present */
-	struct atomisp_sub_device *asd = &isp->asd[0];
 	unsigned long flags;
 
 	/*
@@ -765,7 +763,7 @@ static int atomisp_suspend(struct device *dev)
 		return -EBUSY;
 
 	spin_lock_irqsave(&isp->lock, flags);
-	if (asd->streaming != ATOMISP_DEVICE_STREAMING_DISABLED) {
+	if (isp->asd.streaming != ATOMISP_DEVICE_STREAMING_DISABLED) {
 		spin_unlock_irqrestore(&isp->lock, flags);
 		dev_err(isp->dev, "atomisp cannot suspend at this time.\n");
 		return -EINVAL;
@@ -842,8 +840,7 @@ int atomisp_csi_lane_config(struct atomisp_device *isp)
 	for (i = 0; i < isp->input_cnt; i++) {
 		struct camera_mipi_info *mipi_info;
 
-		if (isp->inputs[i].type != RAW_CAMERA &&
-		    isp->inputs[i].type != SOC_CAMERA)
+		if (isp->inputs[i].type != RAW_CAMERA)
 			continue;
 
 		mipi_info = atomisp_to_sensor_mipi_info(isp->inputs[i].camera);
@@ -923,8 +920,7 @@ static int atomisp_subdev_probe(struct atomisp_device *isp)
 		int camera_count = 0;
 
 		for (subdevs = pdata->subdevs; subdevs->type; ++subdevs) {
-			if (subdevs->type == RAW_CAMERA ||
-			    subdevs->type == SOC_CAMERA)
+			if (subdevs->type == RAW_CAMERA)
 				camera_count++;
 		}
 		if (camera_count)
@@ -945,9 +941,6 @@ static int atomisp_subdev_probe(struct atomisp_device *isp)
 		case RAW_CAMERA:
 			dev_dbg(isp->dev, "raw_index: %d\n", raw_index);
 			raw_index = isp->input_cnt;
-			fallthrough;
-		case SOC_CAMERA:
-			dev_dbg(isp->dev, "SOC_INDEX: %d\n", isp->input_cnt);
 			if (isp->input_cnt >= ATOM_ISP_MAX_INPUTS) {
 				dev_warn(isp->dev,
 					 "too many atomisp inputs, ignored\n");
@@ -974,7 +967,6 @@ static int atomisp_subdev_probe(struct atomisp_device *isp)
 			isp->motor = subdevs->subdev;
 			break;
 		case LED_FLASH:
-		case XENON_FLASH:
 			if (isp->flash) {
 				dev_warn(isp->dev, "too many atomisp flash devices\n");
 				continue;
@@ -1010,8 +1002,7 @@ static void atomisp_unregister_entities(struct atomisp_device *isp)
 	unsigned int i;
 	struct v4l2_subdev *sd, *next;
 
-	for (i = 0; i < isp->num_of_streams; i++)
-		atomisp_subdev_unregister_entities(&isp->asd[i]);
+	atomisp_subdev_unregister_entities(&isp->asd);
 	atomisp_tpg_unregister_entities(&isp->tpg);
 	for (i = 0; i < ATOMISP_CAMERA_NR_PORTS; i++)
 		atomisp_mipi_csi2_unregister_entities(&isp->csi2_port[i]);
@@ -1070,38 +1061,10 @@ static int atomisp_register_entities(struct atomisp_device *isp)
 		goto tpg_register_failed;
 	}
 
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-
-		ret = atomisp_subdev_register_subdev(asd, &isp->v4l2_dev);
-		if (ret < 0) {
-			dev_err(isp->dev, "atomisp_subdev_register_subdev fail\n");
-			for (; i > 0; i--)
-				atomisp_subdev_unregister_entities(
-				    &isp->asd[i - 1]);
-			goto subdev_register_failed;
-		}
-	}
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-
-		init_completion(&asd->init_done);
-
-		asd->delayed_init_workq =
-		    alloc_workqueue(isp->v4l2_dev.name, WQ_CPU_INTENSIVE,
-				    1);
-		if (!asd->delayed_init_workq) {
-			dev_err(isp->dev,
-				"Failed to initialize delayed init workq\n");
-			ret = -ENOMEM;
-
-			for (; i > 0; i--)
-				destroy_workqueue(isp->asd[i - 1].
-						  delayed_init_workq);
-			goto wq_alloc_failed;
-		}
-		INIT_WORK(&asd->delayed_init_work, atomisp_delayed_init_work);
+	ret = atomisp_subdev_register_subdev(&isp->asd, &isp->v4l2_dev);
+	if (ret < 0) {
+		dev_err(isp->dev, "atomisp_subdev_register_subdev fail\n");
+		goto subdev_register_failed;
 	}
 
 	for (i = 0; i < isp->input_cnt; i++) {
@@ -1126,13 +1089,7 @@ static int atomisp_register_entities(struct atomisp_device *isp)
 	return 0;
 
 link_failed:
-	for (i = 0; i < isp->num_of_streams; i++)
-		destroy_workqueue(isp->asd[i].
-				  delayed_init_workq);
-wq_alloc_failed:
-	for (i = 0; i < isp->num_of_streams; i++)
-		atomisp_subdev_unregister_entities(
-		    &isp->asd[i]);
+	atomisp_subdev_unregister_entities(&isp->asd);
 subdev_register_failed:
 	atomisp_tpg_unregister_entities(&isp->tpg);
 tpg_register_failed:
@@ -1148,13 +1105,11 @@ v4l2_device_failed:
 
 static int atomisp_register_device_nodes(struct atomisp_device *isp)
 {
-	int i, err;
+	int err;
 
-	for (i = 0; i < isp->num_of_streams; i++) {
-		err = atomisp_subdev_register_video_nodes(&isp->asd[i], &isp->v4l2_dev);
-		if (err)
-			return err;
-	}
+	err = atomisp_subdev_register_video_nodes(&isp->asd, &isp->v4l2_dev);
+	if (err)
+		return err;
 
 	err = atomisp_create_pads_links(isp);
 	if (err)
diff --git a/drivers/staging/media/av7110/av7110.c b/drivers/staging/media/av7110/av7110.c
index df81a9b744c2..a5a431c14ea7 100644
--- a/drivers/staging/media/av7110/av7110.c
+++ b/drivers/staging/media/av7110/av7110.c
@@ -1106,9 +1106,11 @@ static int dvb_get_stc(struct dmx_demux *demux, unsigned int num,
 	struct av7110 *av7110;
 
 	/* pointer casting paranoia... */
-	BUG_ON(!demux);
+	if (WARN_ON(!demux))
+		return -EIO;
 	dvbdemux = demux->priv;
-	BUG_ON(!dvbdemux);
+	if (WARN_ON(!dvbdemux))
+		return -EIO;
 	av7110 = dvbdemux->priv;
 
 	dprintk(4, "%p\n", av7110);
diff --git a/drivers/staging/media/av7110/av7110_av.c b/drivers/staging/media/av7110/av7110_av.c
index 0bf513c26b6b..a5c5bebad306 100644
--- a/drivers/staging/media/av7110/av7110_av.c
+++ b/drivers/staging/media/av7110/av7110_av.c
@@ -823,10 +823,10 @@ static int write_ts_to_decoder(struct av7110 *av7110, int type, const u8 *buf, s
 		av7110_ipack_flush(ipack);
 
 	if (buf[3] & ADAPT_FIELD) {
+		if (buf[4] > len - 1 - 4)
+			return 0;
 		len -= buf[4] + 1;
 		buf += buf[4] + 1;
-		if (!len)
-			return 0;
 	}
 
 	av7110_ipack_instant_repack(buf + 4, len - 4, ipack);
diff --git a/drivers/staging/media/av7110/av7110_hw.c b/drivers/staging/media/av7110/av7110_hw.c
index 93ca31e38ddd..a0be37717259 100644
--- a/drivers/staging/media/av7110/av7110_hw.c
+++ b/drivers/staging/media/av7110/av7110_hw.c
@@ -1007,7 +1007,8 @@ static int OSDSetBlock(struct av7110 *av7110, int x0, int y0,
 
 	if (av7110->bmp_state == BMP_LOADING) {
 		/* possible if syscall is repeated by -ERESTARTSYS and if firmware cannot abort */
-		BUG_ON (FW_VERSION(av7110->arm_app) >= 0x261e);
+		if (WARN_ON(FW_VERSION(av7110->arm_app) >= 0x261e))
+			return -EIO;
 		rc = WaitUntilBmpLoaded(av7110);
 		if (rc)
 			return rc;
diff --git a/drivers/staging/media/av7110/av7110_v4l.c b/drivers/staging/media/av7110/av7110_v4l.c
index c89f536f699c..ed2c605808e8 100644
--- a/drivers/staging/media/av7110/av7110_v4l.c
+++ b/drivers/staging/media/av7110/av7110_v4l.c
@@ -213,25 +213,14 @@ static const struct v4l2_audio msp3400_v4l2_audio = {
 	.capability = V4L2_AUDCAP_STEREO
 };
 
-static int av7110_dvb_c_switch(struct saa7146_fh *fh)
+static int av7110_dvb_c_switch(struct saa7146_dev *dev)
 {
-	struct saa7146_dev *dev = fh->dev;
-	struct saa7146_vv *vv = dev->vv_data;
 	struct av7110 *av7110 = (struct av7110*)dev->ext_priv;
 	u16 adswitch;
-	int source, sync, err;
+	int source, sync;
 
 	dprintk(4, "%p\n", av7110);
 
-	if ((vv->video_status & STATUS_OVERLAY) != 0) {
-		vv->ov_suspend = vv->video_fh;
-		err = saa7146_stop_preview(vv->video_fh); /* side effect: video_status is now 0, video_fh is NULL */
-		if (err != 0) {
-			dprintk(2, "suspending video failed\n");
-			vv->ov_suspend = NULL;
-		}
-	}
-
 	if (0 != av7110->current_input) {
 		dprintk(1, "switching to analog TV:\n");
 		adswitch = 1;
@@ -300,17 +289,12 @@ static int av7110_dvb_c_switch(struct saa7146_fh *fh)
 
 	saa7146_set_hps_source_and_sync(dev, source, sync);
 
-	if (vv->ov_suspend != NULL) {
-		saa7146_start_preview(vv->ov_suspend);
-		vv->ov_suspend = NULL;
-	}
-
 	return 0;
 }
 
 static int vidioc_g_tuner(struct file *file, void *fh, struct v4l2_tuner *t)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 	u16 stereo_det;
 	s8 stereo;
@@ -354,7 +338,7 @@ static int vidioc_g_tuner(struct file *file, void *fh, struct v4l2_tuner *t)
 
 static int vidioc_s_tuner(struct file *file, void *fh, const struct v4l2_tuner *t)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 	u16 fm_matrix, src;
 	dprintk(2, "VIDIOC_S_TUNER: %d\n", t->index);
@@ -398,7 +382,7 @@ static int vidioc_s_tuner(struct file *file, void *fh, const struct v4l2_tuner *
 
 static int vidioc_g_frequency(struct file *file, void *fh, struct v4l2_frequency *f)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 
 	dprintk(2, "VIDIOC_G_FREQ: freq:0x%08x\n", f->frequency);
@@ -414,7 +398,7 @@ static int vidioc_g_frequency(struct file *file, void *fh, struct v4l2_frequency
 
 static int vidioc_s_frequency(struct file *file, void *fh, const struct v4l2_frequency *f)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 
 	dprintk(2, "VIDIOC_S_FREQUENCY: freq:0x%08x\n", f->frequency);
@@ -444,7 +428,7 @@ static int vidioc_s_frequency(struct file *file, void *fh, const struct v4l2_fre
 
 static int vidioc_enum_input(struct file *file, void *fh, struct v4l2_input *i)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 
 	dprintk(2, "VIDIOC_ENUMINPUT: %d\n", i->index);
@@ -464,7 +448,7 @@ static int vidioc_enum_input(struct file *file, void *fh, struct v4l2_input *i)
 
 static int vidioc_g_input(struct file *file, void *fh, unsigned int *input)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 
 	*input = av7110->current_input;
@@ -474,7 +458,7 @@ static int vidioc_g_input(struct file *file, void *fh, unsigned int *input)
 
 static int vidioc_s_input(struct file *file, void *fh, unsigned int input)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 
 	dprintk(2, "VIDIOC_S_INPUT: %d\n", input);
@@ -486,7 +470,29 @@ static int vidioc_s_input(struct file *file, void *fh, unsigned int input)
 		return -EINVAL;
 
 	av7110->current_input = input;
-	return av7110_dvb_c_switch(fh);
+	return av7110_dvb_c_switch(dev);
+}
+
+static int vidioc_enum_output(struct file *file, void *fh, struct v4l2_output *o)
+{
+	if (o->index)
+		return -EINVAL;
+	strscpy(o->name, "Video Output", sizeof(o->name));
+	o->type = V4L2_OUTPUT_TYPE_ANALOG;
+	o->std = V4L2_STD_NTSC_M | V4L2_STD_PAL_BG;
+	o->capabilities = V4L2_OUT_CAP_STD;
+	return 0;
+}
+
+static int vidioc_g_output(struct file *file, void *fh, unsigned int *output)
+{
+	*output = 0;
+	return 0;
+}
+
+static int vidioc_s_output(struct file *file, void *fh, unsigned int output)
+{
+	return output ? -EINVAL : 0;
 }
 
 static int vidioc_enumaudio(struct file *file, void *fh, struct v4l2_audio *a)
@@ -500,7 +506,7 @@ static int vidioc_enumaudio(struct file *file, void *fh, struct v4l2_audio *a)
 
 static int vidioc_g_audio(struct file *file, void *fh, struct v4l2_audio *a)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 
 	dprintk(2, "VIDIOC_G_AUDIO: %d\n", a->index);
@@ -514,7 +520,7 @@ static int vidioc_g_audio(struct file *file, void *fh, struct v4l2_audio *a)
 
 static int vidioc_s_audio(struct file *file, void *fh, const struct v4l2_audio *a)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 
 	dprintk(2, "VIDIOC_S_AUDIO: %d\n", a->index);
@@ -526,7 +532,7 @@ static int vidioc_s_audio(struct file *file, void *fh, const struct v4l2_audio *
 static int vidioc_g_sliced_vbi_cap(struct file *file, void *fh,
 					struct v4l2_sliced_vbi_cap *cap)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 
 	dprintk(2, "VIDIOC_G_SLICED_VBI_CAP\n");
@@ -542,7 +548,7 @@ static int vidioc_g_sliced_vbi_cap(struct file *file, void *fh,
 static int vidioc_g_fmt_sliced_vbi_out(struct file *file, void *fh,
 					struct v4l2_format *f)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 
 	dprintk(2, "VIDIOC_G_FMT:\n");
@@ -552,59 +558,58 @@ static int vidioc_g_fmt_sliced_vbi_out(struct file *file, void *fh,
 	if (av7110->wssMode) {
 		f->fmt.sliced.service_set = V4L2_SLICED_WSS_625;
 		f->fmt.sliced.service_lines[0][23] = V4L2_SLICED_WSS_625;
-		f->fmt.sliced.io_size = sizeof(struct v4l2_sliced_vbi_data);
 	}
+	f->fmt.sliced.io_size = sizeof(struct v4l2_sliced_vbi_data);
+	return 0;
+}
+
+static int vidioc_try_fmt_sliced_vbi_out(struct file *file, void *fh,
+					 struct v4l2_format *f)
+{
+	struct saa7146_dev *dev = video_drvdata(file);
+	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
+	bool want_wss = (f->fmt.sliced.service_set & V4L2_SLICED_WSS_625) ||
+		(!f->fmt.sliced.service_set &&
+		 f->fmt.sliced.service_lines[0][23] == V4L2_SLICED_WSS_625);
+
+	dprintk(2, "VIDIOC_G_FMT:\n");
+	if (FW_VERSION(av7110->arm_app) < 0x2623)
+		return -EINVAL;
+	memset(&f->fmt.sliced, 0, sizeof(f->fmt.sliced));
+	if (want_wss) {
+		f->fmt.sliced.service_set = V4L2_SLICED_WSS_625;
+		f->fmt.sliced.service_lines[0][23] = V4L2_SLICED_WSS_625;
+	}
+	f->fmt.sliced.io_size = sizeof(struct v4l2_sliced_vbi_data);
 	return 0;
 }
 
 static int vidioc_s_fmt_sliced_vbi_out(struct file *file, void *fh,
 					struct v4l2_format *f)
 {
-	struct saa7146_dev *dev = ((struct saa7146_fh *)fh)->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110 *)dev->ext_priv;
 
 	dprintk(2, "VIDIOC_S_FMT\n");
-	if (FW_VERSION(av7110->arm_app) < 0x2623)
+	if (vidioc_try_fmt_sliced_vbi_out(file, fh, f))
 		return -EINVAL;
-	if (f->fmt.sliced.service_set != V4L2_SLICED_WSS_625 &&
-	    f->fmt.sliced.service_lines[0][23] != V4L2_SLICED_WSS_625) {
-		memset(&f->fmt.sliced, 0, sizeof(f->fmt.sliced));
+	if (f->fmt.sliced.service_set & V4L2_SLICED_WSS_625) {
+		/* WSS controlled by userspace */
+		av7110->wssMode = 1;
+		av7110->wssData = 0;
+	} else {
 		/* WSS controlled by firmware */
 		av7110->wssMode = 0;
 		av7110->wssData = 0;
 		return av7110_fw_cmd(av7110, COMTYPE_ENCODER,
 				     SetWSSConfig, 1, 0);
-	} else {
-		memset(&f->fmt.sliced, 0, sizeof(f->fmt.sliced));
-		f->fmt.sliced.service_set = V4L2_SLICED_WSS_625;
-		f->fmt.sliced.service_lines[0][23] = V4L2_SLICED_WSS_625;
-		f->fmt.sliced.io_size = sizeof(struct v4l2_sliced_vbi_data);
-		/* WSS controlled by userspace */
-		av7110->wssMode = 1;
-		av7110->wssData = 0;
 	}
 	return 0;
 }
 
-static int av7110_vbi_reset(struct file *file)
-{
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
-	struct av7110 *av7110 = (struct av7110*) dev->ext_priv;
-
-	dprintk(2, "%s\n", __func__);
-	av7110->wssMode = 0;
-	av7110->wssData = 0;
-	if (FW_VERSION(av7110->arm_app) < 0x2623)
-		return 0;
-	else
-		return av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetWSSConfig, 1, 0);
-}
-
 static ssize_t av7110_vbi_write(struct file *file, const char __user *data, size_t count, loff_t *ppos)
 {
-	struct saa7146_fh *fh = file->private_data;
-	struct saa7146_dev *dev = fh->dev;
+	struct saa7146_dev *dev = video_drvdata(file);
 	struct av7110 *av7110 = (struct av7110*) dev->ext_priv;
 	struct v4l2_sliced_vbi_data d;
 	int rc;
@@ -819,13 +824,16 @@ int av7110_init_v4l(struct av7110 *av7110)
 	vv_data->vid_ops.vidioc_s_audio = vidioc_s_audio;
 	vv_data->vid_ops.vidioc_g_fmt_vbi_cap = NULL;
 
-	vv_data->vbi_ops.vidioc_g_tuner = vidioc_g_tuner;
-	vv_data->vbi_ops.vidioc_s_tuner = vidioc_s_tuner;
-	vv_data->vbi_ops.vidioc_g_frequency = vidioc_g_frequency;
-	vv_data->vbi_ops.vidioc_s_frequency = vidioc_s_frequency;
+	vv_data->vbi_ops.vidioc_enum_output = vidioc_enum_output;
+	vv_data->vbi_ops.vidioc_g_output = vidioc_g_output;
+	vv_data->vbi_ops.vidioc_s_output = vidioc_s_output;
+	vv_data->vbi_ops.vidioc_g_parm = NULL;
 	vv_data->vbi_ops.vidioc_g_fmt_vbi_cap = NULL;
+	vv_data->vbi_ops.vidioc_try_fmt_vbi_cap = NULL;
+	vv_data->vbi_ops.vidioc_s_fmt_vbi_cap = NULL;
 	vv_data->vbi_ops.vidioc_g_sliced_vbi_cap = vidioc_g_sliced_vbi_cap;
 	vv_data->vbi_ops.vidioc_g_fmt_sliced_vbi_out = vidioc_g_fmt_sliced_vbi_out;
+	vv_data->vbi_ops.vidioc_try_fmt_sliced_vbi_out = vidioc_try_fmt_sliced_vbi_out;
 	vv_data->vbi_ops.vidioc_s_fmt_sliced_vbi_out = vidioc_s_fmt_sliced_vbi_out;
 
 	if (FW_VERSION(av7110->arm_app) < 0x2623)
@@ -866,7 +874,7 @@ static struct saa7146_standard standard[] = {
 		.h_offset	= 0x48,	.h_pixels	= 708,
 		.v_max_out	= 576,	.h_max_out	= 768,
 	}, {
-		.name	= "NTSC",	.id		= V4L2_STD_NTSC,
+		.name	= "NTSC",	.id		= V4L2_STD_NTSC_M,
 		.v_offset	= 0x10,	.v_field	= 244,
 		.h_offset	= 0x40,	.h_pixels	= 708,
 		.v_max_out	= 480,	.h_max_out	= 640,
@@ -880,7 +888,7 @@ static struct saa7146_standard analog_standard[] = {
 		.h_offset	= 0x08,	.h_pixels	= 708,
 		.v_max_out	= 576,	.h_max_out	= 768,
 	}, {
-		.name	= "NTSC",	.id		= V4L2_STD_NTSC,
+		.name	= "NTSC",	.id		= V4L2_STD_NTSC_M,
 		.v_offset	= 0x10,	.v_field	= 244,
 		.h_offset	= 0x40,	.h_pixels	= 708,
 		.v_max_out	= 480,	.h_max_out	= 640,
@@ -894,7 +902,7 @@ static struct saa7146_standard dvb_standard[] = {
 		.h_offset	= 0x48,	.h_pixels	= 708,
 		.v_max_out	= 576,	.h_max_out	= 768,
 	}, {
-		.name	= "NTSC",	.id		= V4L2_STD_NTSC,
+		.name	= "NTSC",	.id		= V4L2_STD_NTSC_M,
 		.v_offset	= 0x10,	.v_field	= 244,
 		.h_offset	= 0x40,	.h_pixels	= 708,
 		.v_max_out	= 480,	.h_max_out	= 640,
@@ -930,8 +938,6 @@ static struct saa7146_ext_vv av7110_vv_data_st = {
 	.num_stds	= ARRAY_SIZE(standard),
 	.std_callback	= &std_callback,
 
-	.vbi_fops.open	= av7110_vbi_reset,
-	.vbi_fops.release = av7110_vbi_reset,
 	.vbi_fops.write	= av7110_vbi_write,
 };
 
@@ -945,8 +951,6 @@ static struct saa7146_ext_vv av7110_vv_data_c = {
 	.num_stds	= ARRAY_SIZE(standard),
 	.std_callback	= &std_callback,
 
-	.vbi_fops.open	= av7110_vbi_reset,
-	.vbi_fops.release = av7110_vbi_reset,
 	.vbi_fops.write	= av7110_vbi_write,
 };
 
diff --git a/drivers/staging/media/deprecated/atmel/atmel-isc-base.c b/drivers/staging/media/deprecated/atmel/atmel-isc-base.c
index 99e61bbfc9bc..61c5afa58142 100644
--- a/drivers/staging/media/deprecated/atmel/atmel-isc-base.c
+++ b/drivers/staging/media/deprecated/atmel/atmel-isc-base.c
@@ -824,8 +824,10 @@ static int isc_try_configure_pipeline(struct isc_device *isc)
 static void isc_try_fse(struct isc_device *isc,
 			struct v4l2_subdev_state *sd_state)
 {
+	struct v4l2_subdev_frame_size_enum fse = {
+		.which = V4L2_SUBDEV_FORMAT_TRY,
+	};
 	int ret;
-	struct v4l2_subdev_frame_size_enum fse = {};
 
 	/*
 	 * If we do not know yet which format the subdev is using, we cannot
@@ -835,7 +837,6 @@ static void isc_try_fse(struct isc_device *isc,
 		return;
 
 	fse.code = isc->try_config.sd_format->mbus_code;
-	fse.which = V4L2_SUBDEV_FORMAT_TRY;
 
 	ret = v4l2_subdev_call(isc->current_subdev->sd, pad, enum_frame_size,
 			       sd_state, &fse);
@@ -860,8 +861,8 @@ static int isc_try_fmt(struct isc_device *isc, struct v4l2_format *f,
 	struct v4l2_pix_format *pixfmt = &f->fmt.pix;
 	struct v4l2_subdev_pad_config pad_cfg = {};
 	struct v4l2_subdev_state pad_state = {
-		.pads = &pad_cfg
-		};
+		.pads = &pad_cfg,
+	};
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_TRY,
 	};
diff --git a/drivers/staging/media/deprecated/atmel/atmel-sama5d2-isc.c b/drivers/staging/media/deprecated/atmel/atmel-sama5d2-isc.c
index ba0614f981a2..cc86ebcc76af 100644
--- a/drivers/staging/media/deprecated/atmel/atmel-sama5d2-isc.c
+++ b/drivers/staging/media/deprecated/atmel/atmel-sama5d2-isc.c
@@ -389,7 +389,6 @@ static int atmel_isc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct isc_device *isc;
-	struct resource *res;
 	void __iomem *io_base;
 	struct isc_subdev_entity *subdev_entity;
 	int irq;
@@ -403,8 +402,7 @@ static int atmel_isc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, isc);
 	isc->dev = dev;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	io_base = devm_ioremap_resource(dev, res);
+	io_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(io_base))
 		return PTR_ERR(io_base);
 
@@ -580,7 +578,7 @@ unprepare_hclk:
 	return ret;
 }
 
-static int atmel_isc_remove(struct platform_device *pdev)
+static void atmel_isc_remove(struct platform_device *pdev)
 {
 	struct isc_device *isc = platform_get_drvdata(pdev);
 
@@ -594,8 +592,6 @@ static int atmel_isc_remove(struct platform_device *pdev)
 	clk_disable_unprepare(isc->hclock);
 
 	atmel_isc_clk_cleanup(isc);
-
-	return 0;
 }
 
 static int __maybe_unused isc_runtime_suspend(struct device *dev)
@@ -638,7 +634,7 @@ MODULE_DEVICE_TABLE(of, atmel_isc_of_match);
 
 static struct platform_driver atmel_isc_driver = {
 	.probe	= atmel_isc_probe,
-	.remove	= atmel_isc_remove,
+	.remove_new = atmel_isc_remove,
 	.driver	= {
 		.name		= "atmel-sama5d2-isc",
 		.pm		= &atmel_isc_dev_pm_ops,
diff --git a/drivers/staging/media/deprecated/atmel/atmel-sama7g5-isc.c b/drivers/staging/media/deprecated/atmel/atmel-sama7g5-isc.c
index 01ababdfcbd9..68ef3374d25e 100644
--- a/drivers/staging/media/deprecated/atmel/atmel-sama7g5-isc.c
+++ b/drivers/staging/media/deprecated/atmel/atmel-sama7g5-isc.c
@@ -378,7 +378,6 @@ static int microchip_xisc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct isc_device *isc;
-	struct resource *res;
 	void __iomem *io_base;
 	struct isc_subdev_entity *subdev_entity;
 	int irq;
@@ -392,8 +391,7 @@ static int microchip_xisc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, isc);
 	isc->dev = dev;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	io_base = devm_ioremap_resource(dev, res);
+	io_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(io_base))
 		return PTR_ERR(io_base);
 
@@ -549,7 +547,7 @@ unprepare_hclk:
 	return ret;
 }
 
-static int microchip_xisc_remove(struct platform_device *pdev)
+static void microchip_xisc_remove(struct platform_device *pdev)
 {
 	struct isc_device *isc = platform_get_drvdata(pdev);
 
@@ -562,8 +560,6 @@ static int microchip_xisc_remove(struct platform_device *pdev)
 	clk_disable_unprepare(isc->hclock);
 
 	atmel_isc_clk_cleanup(isc);
-
-	return 0;
 }
 
 static int __maybe_unused xisc_runtime_suspend(struct device *dev)
@@ -601,7 +597,7 @@ MODULE_DEVICE_TABLE(of, microchip_xisc_of_match);
 
 static struct platform_driver microchip_xisc_driver = {
 	.probe	= microchip_xisc_probe,
-	.remove	= microchip_xisc_remove,
+	.remove_new = microchip_xisc_remove,
 	.driver	= {
 		.name		= "microchip-sama7g5-xisc",
 		.pm		= &microchip_xisc_dev_pm_ops,
diff --git a/drivers/staging/media/imx/imx-media-capture.c b/drivers/staging/media/imx/imx-media-capture.c
index 93ba09236010..4364df27c6d2 100644
--- a/drivers/staging/media/imx/imx-media-capture.c
+++ b/drivers/staging/media/imx/imx-media-capture.c
@@ -353,12 +353,13 @@ static int capture_legacy_enum_fmt_vid_cap(struct file *file, void *fh,
 {
 	struct capture_priv *priv = video_drvdata(file);
 	const struct imx_media_pixfmt *cc_src;
-	struct v4l2_subdev_format fmt_src;
+	struct v4l2_subdev_format fmt_src = {
+		.pad = priv->src_sd_pad,
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	u32 fourcc;
 	int ret;
 
-	fmt_src.pad = priv->src_sd_pad;
-	fmt_src.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(priv->src_sd, pad, get_fmt, NULL, &fmt_src);
 	if (ret) {
 		dev_err(priv->dev, "failed to get src_sd format\n");
@@ -426,11 +427,12 @@ static int capture_legacy_try_fmt_vid_cap(struct file *file, void *fh,
 					  struct v4l2_format *f)
 {
 	struct capture_priv *priv = video_drvdata(file);
-	struct v4l2_subdev_format fmt_src;
+	struct v4l2_subdev_format fmt_src = {
+		.pad = priv->src_sd_pad,
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	int ret;
 
-	fmt_src.pad = priv->src_sd_pad;
-	fmt_src.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(priv->src_sd, pad, get_fmt, NULL, &fmt_src);
 	if (ret)
 		return ret;
@@ -445,7 +447,10 @@ static int capture_legacy_s_fmt_vid_cap(struct file *file, void *fh,
 					struct v4l2_format *f)
 {
 	struct capture_priv *priv = video_drvdata(file);
-	struct v4l2_subdev_format fmt_src;
+	struct v4l2_subdev_format fmt_src = {
+		.pad = priv->src_sd_pad,
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	const struct imx_media_pixfmt *cc;
 	int ret;
 
@@ -454,8 +459,6 @@ static int capture_legacy_s_fmt_vid_cap(struct file *file, void *fh,
 		return -EBUSY;
 	}
 
-	fmt_src.pad = priv->src_sd_pad;
-	fmt_src.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(priv->src_sd, pad, get_fmt, NULL, &fmt_src);
 	if (ret)
 		return ret;
@@ -501,14 +504,14 @@ static int capture_legacy_g_parm(struct file *file, void *fh,
 				 struct v4l2_streamparm *a)
 {
 	struct capture_priv *priv = video_drvdata(file);
-	struct v4l2_subdev_frame_interval fi;
+	struct v4l2_subdev_frame_interval fi = {
+		.pad = priv->src_sd_pad,
+	};
 	int ret;
 
 	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 
-	memset(&fi, 0, sizeof(fi));
-	fi.pad = priv->src_sd_pad;
 	ret = v4l2_subdev_call(priv->src_sd, video, g_frame_interval, &fi);
 	if (ret < 0)
 		return ret;
@@ -523,14 +526,14 @@ static int capture_legacy_s_parm(struct file *file, void *fh,
 				 struct v4l2_streamparm *a)
 {
 	struct capture_priv *priv = video_drvdata(file);
-	struct v4l2_subdev_frame_interval fi;
+	struct v4l2_subdev_frame_interval fi = {
+		.pad = priv->src_sd_pad,
+	};
 	int ret;
 
 	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 
-	memset(&fi, 0, sizeof(fi));
-	fi.pad = priv->src_sd_pad;
 	fi.interval = a->parm.capture.timeperframe;
 	ret = v4l2_subdev_call(priv->src_sd, video, s_frame_interval, &fi);
 	if (ret < 0)
@@ -670,13 +673,14 @@ static void capture_buf_queue(struct vb2_buffer *vb)
 
 static int capture_validate_fmt(struct capture_priv *priv)
 {
-	struct v4l2_subdev_format fmt_src;
+	struct v4l2_subdev_format fmt_src = {
+		.pad = priv->src_sd_pad,
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	const struct imx_media_pixfmt *cc;
 	int ret;
 
 	/* Retrieve the media bus format on the source subdev. */
-	fmt_src.pad = priv->src_sd_pad;
-	fmt_src.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(priv->src_sd, pad, get_fmt, NULL, &fmt_src);
 	if (ret)
 		return ret;
diff --git a/drivers/staging/media/imx/imx-media-csi.c b/drivers/staging/media/imx/imx-media-csi.c
index 44d87fe30d52..097171bb930d 100644
--- a/drivers/staging/media/imx/imx-media-csi.c
+++ b/drivers/staging/media/imx/imx-media-csi.c
@@ -2041,7 +2041,7 @@ free:
 	return ret;
 }
 
-static int imx_csi_remove(struct platform_device *pdev)
+static void imx_csi_remove(struct platform_device *pdev)
 {
 	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
 	struct csi_priv *priv = sd_to_dev(sd);
@@ -2052,8 +2052,6 @@ static int imx_csi_remove(struct platform_device *pdev)
 	v4l2_async_nf_cleanup(&priv->notifier);
 	v4l2_async_unregister_subdev(sd);
 	media_entity_cleanup(&sd->entity);
-
-	return 0;
 }
 
 static const struct platform_device_id imx_csi_ids[] = {
@@ -2064,7 +2062,7 @@ MODULE_DEVICE_TABLE(platform, imx_csi_ids);
 
 static struct platform_driver imx_csi_driver = {
 	.probe = imx_csi_probe,
-	.remove = imx_csi_remove,
+	.remove_new = imx_csi_remove,
 	.id_table = imx_csi_ids,
 	.driver = {
 		.name = "imx-ipuv3-csi",
diff --git a/drivers/staging/media/imx/imx-media-dev-common.c b/drivers/staging/media/imx/imx-media-dev-common.c
index e6d6ed3b1161..991820a8500f 100644
--- a/drivers/staging/media/imx/imx-media-dev-common.c
+++ b/drivers/staging/media/imx/imx-media-dev-common.c
@@ -19,18 +19,6 @@ static inline struct imx_media_dev *notifier2dev(struct v4l2_async_notifier *n)
 	return container_of(n, struct imx_media_dev, notifier);
 }
 
-/* async subdev bound notifier */
-static int imx_media_subdev_bound(struct v4l2_async_notifier *notifier,
-				  struct v4l2_subdev *sd,
-				  struct v4l2_async_subdev *asd)
-{
-	struct imx_media_dev *imxmd = notifier2dev(notifier);
-
-	dev_dbg(imxmd->md.dev, "subdev %s bound\n", sd->name);
-
-	return 0;
-}
-
 /*
  * Create the missing media links from the CSI-2 receiver.
  * Called after all async subdevs have bound.
@@ -51,7 +39,6 @@ static void imx_media_create_csi2_links(struct imx_media_dev *imxmd)
 	list_for_each_entry(sd, &imxmd->v4l2_dev.subdevs, list) {
 		/* skip if not a CSI or a CSI mux */
 		if (!(sd->grp_id & IMX_MEDIA_GRP_ID_IPU_CSI) &&
-		    !(sd->grp_id & IMX_MEDIA_GRP_ID_CSI) &&
 		    !(sd->grp_id & IMX_MEDIA_GRP_ID_CSI_MUX))
 			continue;
 
@@ -337,7 +324,6 @@ static void imx_media_notify(struct v4l2_subdev *sd, unsigned int notification,
 }
 
 static const struct v4l2_async_notifier_operations imx_media_notifier_ops = {
-	.bound = imx_media_subdev_bound,
 	.complete = imx_media_probe_complete,
 };
 
diff --git a/drivers/staging/media/imx/imx-media-dev.c b/drivers/staging/media/imx/imx-media-dev.c
index f85462214e22..c80113905069 100644
--- a/drivers/staging/media/imx/imx-media-dev.c
+++ b/drivers/staging/media/imx/imx-media-dev.c
@@ -101,7 +101,7 @@ cleanup:
 	return ret;
 }
 
-static int imx_media_remove(struct platform_device *pdev)
+static void imx_media_remove(struct platform_device *pdev)
 {
 	struct imx_media_dev *imxmd =
 		(struct imx_media_dev *)platform_get_drvdata(pdev);
@@ -119,8 +119,6 @@ static int imx_media_remove(struct platform_device *pdev)
 	media_device_unregister(&imxmd->md);
 	v4l2_device_unregister(&imxmd->v4l2_dev);
 	media_device_cleanup(&imxmd->md);
-
-	return 0;
 }
 
 static const struct of_device_id imx_media_dt_ids[] = {
@@ -131,7 +129,7 @@ MODULE_DEVICE_TABLE(of, imx_media_dt_ids);
 
 static struct platform_driver imx_media_pdrv = {
 	.probe		= imx_media_probe,
-	.remove		= imx_media_remove,
+	.remove_new	= imx_media_remove,
 	.driver		= {
 		.name	= "imx-media",
 		.of_match_table	= imx_media_dt_ids,
diff --git a/drivers/staging/media/imx/imx-media-of.c b/drivers/staging/media/imx/imx-media-of.c
index 59f1eb7b62bc..92a99010c150 100644
--- a/drivers/staging/media/imx/imx-media-of.c
+++ b/drivers/staging/media/imx/imx-media-of.c
@@ -16,8 +16,8 @@
 #include <video/imx-ipu-v3.h>
 #include "imx-media.h"
 
-int imx_media_of_add_csi(struct imx_media_dev *imxmd,
-			 struct device_node *csi_np)
+static int imx_media_of_add_csi(struct imx_media_dev *imxmd,
+				struct device_node *csi_np)
 {
 	struct v4l2_async_subdev *asd;
 	int ret = 0;
@@ -41,7 +41,6 @@ int imx_media_of_add_csi(struct imx_media_dev *imxmd,
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(imx_media_of_add_csi);
 
 int imx_media_add_of_subdevs(struct imx_media_dev *imxmd,
 			     struct device_node *np)
diff --git a/drivers/staging/media/imx/imx-media-utils.c b/drivers/staging/media/imx/imx-media-utils.c
index 411e907b68eb..2d712eda2c5d 100644
--- a/drivers/staging/media/imx/imx-media-utils.c
+++ b/drivers/staging/media/imx/imx-media-utils.c
@@ -432,15 +432,15 @@ int imx_media_init_cfg(struct v4l2_subdev *sd,
 		       struct v4l2_subdev_state *sd_state)
 {
 	struct v4l2_mbus_framefmt *mf_try;
-	struct v4l2_subdev_format format;
 	unsigned int pad;
 	int ret;
 
 	for (pad = 0; pad < sd->entity.num_pads; pad++) {
-		memset(&format, 0, sizeof(format));
+		struct v4l2_subdev_format format = {
+			.pad = pad,
+			.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		};
 
-		format.pad = pad;
-		format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &format);
 		if (ret)
 			continue;
@@ -626,36 +626,6 @@ void imx_media_grp_id_to_sd_name(char *sd_name, int sz, u32 grp_id, int ipu_id)
 }
 EXPORT_SYMBOL_GPL(imx_media_grp_id_to_sd_name);
 
-struct v4l2_subdev *
-imx_media_find_subdev_by_fwnode(struct imx_media_dev *imxmd,
-				struct fwnode_handle *fwnode)
-{
-	struct v4l2_subdev *sd;
-
-	list_for_each_entry(sd, &imxmd->v4l2_dev.subdevs, list) {
-		if (sd->fwnode == fwnode)
-			return sd;
-	}
-
-	return NULL;
-}
-EXPORT_SYMBOL_GPL(imx_media_find_subdev_by_fwnode);
-
-struct v4l2_subdev *
-imx_media_find_subdev_by_devname(struct imx_media_dev *imxmd,
-				 const char *devname)
-{
-	struct v4l2_subdev *sd;
-
-	list_for_each_entry(sd, &imxmd->v4l2_dev.subdevs, list) {
-		if (!strcmp(devname, dev_name(sd->dev)))
-			return sd;
-	}
-
-	return NULL;
-}
-EXPORT_SYMBOL_GPL(imx_media_find_subdev_by_devname);
-
 /*
  * Adds a video device to the master video device list. This is called
  * when a video device is registered.
@@ -757,25 +727,6 @@ find_pipeline_entity(struct media_entity *start, u32 grp_id,
 }
 
 /*
- * Find the upstream mipi-csi2 virtual channel reached from the given
- * start entity in the current pipeline.
- * Must be called with mdev->graph_mutex held.
- */
-int imx_media_pipeline_csi2_channel(struct media_entity *start_entity)
-{
-	struct media_pad *pad;
-	int ret = -EPIPE;
-
-	pad = imx_media_pipeline_pad(start_entity, IMX_MEDIA_GRP_ID_CSI2,
-				     0, true);
-	if (pad)
-		ret = pad->index - 1;
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(imx_media_pipeline_csi2_channel);
-
-/*
  * Find a subdev reached upstream from the given start entity in
  * the current pipeline.
  * Must be called with mdev->graph_mutex held.
@@ -795,25 +746,6 @@ imx_media_pipeline_subdev(struct media_entity *start_entity, u32 grp_id,
 EXPORT_SYMBOL_GPL(imx_media_pipeline_subdev);
 
 /*
- * Find a subdev reached upstream from the given start entity in
- * the current pipeline.
- * Must be called with mdev->graph_mutex held.
- */
-struct video_device *
-imx_media_pipeline_video_device(struct media_entity *start_entity,
-				enum v4l2_buf_type buftype, bool upstream)
-{
-	struct media_entity *me;
-
-	me = find_pipeline_entity(start_entity, 0, buftype, upstream);
-	if (!me)
-		return ERR_PTR(-ENODEV);
-
-	return media_entity_to_video_device(me);
-}
-EXPORT_SYMBOL_GPL(imx_media_pipeline_video_device);
-
-/*
  * Turn current pipeline streaming on/off starting from entity.
  */
 int imx_media_pipeline_set_stream(struct imx_media_dev *imxmd,
diff --git a/drivers/staging/media/imx/imx-media.h b/drivers/staging/media/imx/imx-media.h
index 6f9a46573edd..2640cd34dce2 100644
--- a/drivers/staging/media/imx/imx-media.h
+++ b/drivers/staging/media/imx/imx-media.h
@@ -201,24 +201,14 @@ int imx_media_mbus_fmt_to_pix_fmt(struct v4l2_pix_format *pix,
 				  const struct imx_media_pixfmt *cc);
 void imx_media_grp_id_to_sd_name(char *sd_name, int sz,
 				 u32 grp_id, int ipu_id);
-struct v4l2_subdev *
-imx_media_find_subdev_by_fwnode(struct imx_media_dev *imxmd,
-				struct fwnode_handle *fwnode);
-struct v4l2_subdev *
-imx_media_find_subdev_by_devname(struct imx_media_dev *imxmd,
-				 const char *devname);
 void imx_media_add_video_device(struct imx_media_dev *imxmd,
 				struct imx_media_video_dev *vdev);
-int imx_media_pipeline_csi2_channel(struct media_entity *start_entity);
 struct media_pad *
 imx_media_pipeline_pad(struct media_entity *start_entity, u32 grp_id,
 		       enum v4l2_buf_type buftype, bool upstream);
 struct v4l2_subdev *
 imx_media_pipeline_subdev(struct media_entity *start_entity, u32 grp_id,
 			  bool upstream);
-struct video_device *
-imx_media_pipeline_video_device(struct media_entity *start_entity,
-				enum v4l2_buf_type buftype, bool upstream);
 
 struct imx_media_dma_buf {
 	void          *virt;
@@ -261,8 +251,6 @@ void imx_media_unregister_ipu_internal_subdevs(struct imx_media_dev *imxmd);
 /* imx-media-of.c */
 int imx_media_add_of_subdevs(struct imx_media_dev *dev,
 			     struct device_node *np);
-int imx_media_of_add_csi(struct imx_media_dev *imxmd,
-			 struct device_node *csi_np);
 
 /* imx-media-vdic.c */
 struct v4l2_subdev *imx_media_vdic_register(struct v4l2_device *v4l2_dev,
@@ -298,7 +286,6 @@ void imx_media_csc_scaler_device_unregister(struct imx_media_video_dev *vdev);
 
 /* subdev group ids */
 #define IMX_MEDIA_GRP_ID_CSI2          BIT(8)
-#define IMX_MEDIA_GRP_ID_CSI           BIT(9)
 #define IMX_MEDIA_GRP_ID_IPU_CSI_BIT   10
 #define IMX_MEDIA_GRP_ID_IPU_CSI       (0x3 << IMX_MEDIA_GRP_ID_IPU_CSI_BIT)
 #define IMX_MEDIA_GRP_ID_IPU_CSI0      BIT(IMX_MEDIA_GRP_ID_IPU_CSI_BIT)
diff --git a/drivers/staging/media/imx/imx6-mipi-csi2.c b/drivers/staging/media/imx/imx6-mipi-csi2.c
index c4cb558a85c6..c07994ea6e96 100644
--- a/drivers/staging/media/imx/imx6-mipi-csi2.c
+++ b/drivers/staging/media/imx/imx6-mipi-csi2.c
@@ -765,7 +765,7 @@ rmmutex:
 	return ret;
 }
 
-static int csi2_remove(struct platform_device *pdev)
+static void csi2_remove(struct platform_device *pdev)
 {
 	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
 	struct csi2_dev *csi2 = sd_to_dev(sd);
@@ -777,8 +777,6 @@ static int csi2_remove(struct platform_device *pdev)
 	clk_disable_unprepare(csi2->pllref_clk);
 	mutex_destroy(&csi2->lock);
 	media_entity_cleanup(&sd->entity);
-
-	return 0;
 }
 
 static const struct of_device_id csi2_dt_ids[] = {
@@ -793,7 +791,7 @@ static struct platform_driver csi2_driver = {
 		.of_match_table = csi2_dt_ids,
 	},
 	.probe = csi2_probe,
-	.remove = csi2_remove,
+	.remove_new = csi2_remove,
 };
 
 module_platform_driver(csi2_driver);
diff --git a/drivers/staging/media/imx/imx8mq-mipi-csi2.c b/drivers/staging/media/imx/imx8mq-mipi-csi2.c
index 83194328d010..32700cb8bc4d 100644
--- a/drivers/staging/media/imx/imx8mq-mipi-csi2.c
+++ b/drivers/staging/media/imx/imx8mq-mipi-csi2.c
@@ -119,11 +119,8 @@ struct csi_state {
 
 	struct v4l2_mbus_config_mipi_csi2 bus;
 
-	struct mutex lock; /* Protect csi2_fmt, format_mbus, state, hs_settle */
-	const struct csi2_pix_format *csi2_fmt;
-	struct v4l2_mbus_framefmt format_mbus[MIPI_CSI2_PADS_NUM];
+	struct mutex lock; /* Protect state */
 	u32 state;
-	u32 hs_settle;
 
 	struct regmap *phy_gpr;
 	u8 phy_gpr_reg;
@@ -248,23 +245,6 @@ static int imx8mq_mipi_csi_sw_reset(struct csi_state *state)
 	return 0;
 }
 
-static void imx8mq_mipi_csi_system_enable(struct csi_state *state, int on)
-{
-	if (!on) {
-		imx8mq_mipi_csi_write(state, CSI2RX_CFG_DISABLE_DATA_LANES, 0xf);
-		return;
-	}
-
-	regmap_update_bits(state->phy_gpr,
-			   state->phy_gpr_reg,
-			   0x3fff,
-			   GPR_CSI2_1_RX_ENABLE |
-			   GPR_CSI2_1_VID_INTFC_ENB |
-			   GPR_CSI2_1_HSEL |
-			   GPR_CSI2_1_CONT_CLK_MODE |
-			   GPR_CSI2_1_S_PRG_RXHS_SETTLE(state->hs_settle));
-}
-
 static void imx8mq_mipi_csi_set_params(struct csi_state *state)
 {
 	int lanes = state->bus.num_data_lanes;
@@ -304,16 +284,24 @@ static int imx8mq_mipi_csi_clk_get(struct csi_state *state)
 	return devm_clk_bulk_get(state->dev, CSI2_NUM_CLKS, state->clks);
 }
 
-static int imx8mq_mipi_csi_calc_hs_settle(struct csi_state *state)
+static int imx8mq_mipi_csi_calc_hs_settle(struct csi_state *state,
+					  struct v4l2_subdev_state *sd_state,
+					  u32 *hs_settle)
 {
 	s64 link_freq;
 	u32 lane_rate;
 	unsigned long esc_clk_rate;
 	u32 min_ths_settle, max_ths_settle, ths_settle_ns, esc_clk_period_ns;
+	const struct v4l2_mbus_framefmt *fmt;
+	const struct csi2_pix_format *csi2_fmt;
 
 	/* Calculate the line rate from the pixel rate. */
+
+	fmt = v4l2_subdev_get_pad_format(&state->sd, sd_state, MIPI_CSI2_PAD_SINK);
+	csi2_fmt = find_csi2_format(fmt->code);
+
 	link_freq = v4l2_get_link_freq(state->src_sd->ctrl_handler,
-				       state->csi2_fmt->width,
+				       csi2_fmt->width,
 				       state->bus.num_data_lanes * 2);
 	if (link_freq < 0) {
 		dev_err(state->dev, "Unable to obtain link frequency: %d\n",
@@ -354,35 +342,44 @@ static int imx8mq_mipi_csi_calc_hs_settle(struct csi_state *state)
 	max_ths_settle = 140 + 10 * 1000000 / (lane_rate / 1000);
 	ths_settle_ns = (min_ths_settle + max_ths_settle) / 2;
 
-	state->hs_settle = ths_settle_ns / esc_clk_period_ns - 1;
+	*hs_settle = ths_settle_ns / esc_clk_period_ns - 1;
 
 	dev_dbg(state->dev, "lane rate %u Ths_settle %u hs_settle %u\n",
-		lane_rate, ths_settle_ns, state->hs_settle);
+		lane_rate, ths_settle_ns, *hs_settle);
 
 	return 0;
 }
 
-static int imx8mq_mipi_csi_start_stream(struct csi_state *state)
+static int imx8mq_mipi_csi_start_stream(struct csi_state *state,
+					struct v4l2_subdev_state *sd_state)
 {
 	int ret;
+	u32 hs_settle;
 
 	ret = imx8mq_mipi_csi_sw_reset(state);
 	if (ret)
 		return ret;
 
 	imx8mq_mipi_csi_set_params(state);
-	ret = imx8mq_mipi_csi_calc_hs_settle(state);
+	ret = imx8mq_mipi_csi_calc_hs_settle(state, sd_state, &hs_settle);
 	if (ret)
 		return ret;
 
-	imx8mq_mipi_csi_system_enable(state, true);
+	regmap_update_bits(state->phy_gpr,
+			   state->phy_gpr_reg,
+			   0x3fff,
+			   GPR_CSI2_1_RX_ENABLE |
+			   GPR_CSI2_1_VID_INTFC_ENB |
+			   GPR_CSI2_1_HSEL |
+			   GPR_CSI2_1_CONT_CLK_MODE |
+			   GPR_CSI2_1_S_PRG_RXHS_SETTLE(hs_settle));
 
 	return 0;
 }
 
 static void imx8mq_mipi_csi_stop_stream(struct csi_state *state)
 {
-	imx8mq_mipi_csi_system_enable(state, false);
+	imx8mq_mipi_csi_write(state, CSI2RX_CFG_DISABLE_DATA_LANES, 0xf);
 }
 
 /* -----------------------------------------------------------------------------
@@ -397,6 +394,7 @@ static struct csi_state *mipi_sd_to_csi2_state(struct v4l2_subdev *sdev)
 static int imx8mq_mipi_csi_s_stream(struct v4l2_subdev *sd, int enable)
 {
 	struct csi_state *state = mipi_sd_to_csi2_state(sd);
+	struct v4l2_subdev_state *sd_state;
 	int ret = 0;
 
 	if (enable) {
@@ -413,7 +411,10 @@ static int imx8mq_mipi_csi_s_stream(struct v4l2_subdev *sd, int enable)
 			goto unlock;
 		}
 
-		ret = imx8mq_mipi_csi_start_stream(state);
+		sd_state = v4l2_subdev_lock_and_get_active_state(sd);
+		ret = imx8mq_mipi_csi_start_stream(state, sd_state);
+		v4l2_subdev_unlock_state(sd_state);
+
 		if (ret < 0)
 			goto unlock;
 
@@ -437,29 +438,14 @@ unlock:
 	return ret;
 }
 
-static struct v4l2_mbus_framefmt *
-imx8mq_mipi_csi_get_format(struct csi_state *state,
-			   struct v4l2_subdev_state *sd_state,
-			   enum v4l2_subdev_format_whence which,
-			   unsigned int pad)
-{
-	if (which == V4L2_SUBDEV_FORMAT_TRY)
-		return v4l2_subdev_get_try_format(&state->sd, sd_state, pad);
-
-	return &state->format_mbus[pad];
-}
-
 static int imx8mq_mipi_csi_init_cfg(struct v4l2_subdev *sd,
 				    struct v4l2_subdev_state *sd_state)
 {
-	struct csi_state *state = mipi_sd_to_csi2_state(sd);
 	struct v4l2_mbus_framefmt *fmt_sink;
 	struct v4l2_mbus_framefmt *fmt_source;
-	enum v4l2_subdev_format_whence which;
 
-	which = sd_state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE;
-	fmt_sink = imx8mq_mipi_csi_get_format(state, sd_state, which,
-					      MIPI_CSI2_PAD_SINK);
+	fmt_sink = v4l2_subdev_get_pad_format(sd, sd_state, MIPI_CSI2_PAD_SINK);
+	fmt_source = v4l2_subdev_get_pad_format(sd, sd_state, MIPI_CSI2_PAD_SOURCE);
 
 	fmt_sink->code = MEDIA_BUS_FMT_SGBRG10_1X10;
 	fmt_sink->width = MIPI_CSI2_DEF_PIX_WIDTH;
@@ -473,38 +459,15 @@ static int imx8mq_mipi_csi_init_cfg(struct v4l2_subdev *sd,
 		V4L2_MAP_QUANTIZATION_DEFAULT(false, fmt_sink->colorspace,
 					      fmt_sink->ycbcr_enc);
 
-	fmt_source = imx8mq_mipi_csi_get_format(state, sd_state, which,
-						MIPI_CSI2_PAD_SOURCE);
 	*fmt_source = *fmt_sink;
 
 	return 0;
 }
 
-static int imx8mq_mipi_csi_get_fmt(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_state *sd_state,
-				   struct v4l2_subdev_format *sdformat)
-{
-	struct csi_state *state = mipi_sd_to_csi2_state(sd);
-	struct v4l2_mbus_framefmt *fmt;
-
-	fmt = imx8mq_mipi_csi_get_format(state, sd_state, sdformat->which,
-					 sdformat->pad);
-
-	mutex_lock(&state->lock);
-
-	sdformat->format = *fmt;
-
-	mutex_unlock(&state->lock);
-
-	return 0;
-}
-
 static int imx8mq_mipi_csi_enum_mbus_code(struct v4l2_subdev *sd,
 					  struct v4l2_subdev_state *sd_state,
 					  struct v4l2_subdev_mbus_code_enum *code)
 {
-	struct csi_state *state = mipi_sd_to_csi2_state(sd);
-
 	/*
 	 * We can't transcode in any way, the source format is identical
 	 * to the sink format.
@@ -515,8 +478,7 @@ static int imx8mq_mipi_csi_enum_mbus_code(struct v4l2_subdev *sd,
 		if (code->index > 0)
 			return -EINVAL;
 
-		fmt = imx8mq_mipi_csi_get_format(state, sd_state, code->which,
-						 code->pad);
+		fmt = v4l2_subdev_get_pad_format(sd, sd_state, code->pad);
 		code->code = fmt->code;
 		return 0;
 	}
@@ -536,8 +498,7 @@ static int imx8mq_mipi_csi_set_fmt(struct v4l2_subdev *sd,
 				   struct v4l2_subdev_state *sd_state,
 				   struct v4l2_subdev_format *sdformat)
 {
-	struct csi_state *state = mipi_sd_to_csi2_state(sd);
-	struct csi2_pix_format const *csi2_fmt;
+	const struct csi2_pix_format *csi2_fmt;
 	struct v4l2_mbus_framefmt *fmt;
 
 	/*
@@ -545,7 +506,7 @@ static int imx8mq_mipi_csi_set_fmt(struct v4l2_subdev *sd,
 	 * modified.
 	 */
 	if (sdformat->pad == MIPI_CSI2_PAD_SOURCE)
-		return imx8mq_mipi_csi_get_fmt(sd, sd_state, sdformat);
+		return v4l2_subdev_get_fmt(sd, sd_state, sdformat);
 
 	if (sdformat->pad != MIPI_CSI2_PAD_SINK)
 		return -EINVAL;
@@ -554,10 +515,7 @@ static int imx8mq_mipi_csi_set_fmt(struct v4l2_subdev *sd,
 	if (!csi2_fmt)
 		csi2_fmt = &imx8mq_mipi_csi_formats[0];
 
-	fmt = imx8mq_mipi_csi_get_format(state, sd_state, sdformat->which,
-					 sdformat->pad);
-
-	mutex_lock(&state->lock);
+	fmt = v4l2_subdev_get_pad_format(sd, sd_state, sdformat->pad);
 
 	fmt->code = csi2_fmt->code;
 	fmt->width = sdformat->format.width;
@@ -566,16 +524,9 @@ static int imx8mq_mipi_csi_set_fmt(struct v4l2_subdev *sd,
 	sdformat->format = *fmt;
 
 	/* Propagate the format from sink to source. */
-	fmt = imx8mq_mipi_csi_get_format(state, sd_state, sdformat->which,
-					 MIPI_CSI2_PAD_SOURCE);
+	fmt = v4l2_subdev_get_pad_format(sd, sd_state, MIPI_CSI2_PAD_SOURCE);
 	*fmt = sdformat->format;
 
-	/* Store the CSI2 format descriptor for active formats. */
-	if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE)
-		state->csi2_fmt = csi2_fmt;
-
-	mutex_unlock(&state->lock);
-
 	return 0;
 }
 
@@ -586,7 +537,7 @@ static const struct v4l2_subdev_video_ops imx8mq_mipi_csi_video_ops = {
 static const struct v4l2_subdev_pad_ops imx8mq_mipi_csi_pad_ops = {
 	.init_cfg		= imx8mq_mipi_csi_init_cfg,
 	.enum_mbus_code		= imx8mq_mipi_csi_enum_mbus_code,
-	.get_fmt		= imx8mq_mipi_csi_get_fmt,
+	.get_fmt		= v4l2_subdev_get_fmt,
 	.set_fmt		= imx8mq_mipi_csi_set_fmt,
 };
 
@@ -714,6 +665,7 @@ static int imx8mq_mipi_csi_pm_resume(struct device *dev)
 {
 	struct v4l2_subdev *sd = dev_get_drvdata(dev);
 	struct csi_state *state = mipi_sd_to_csi2_state(sd);
+	struct v4l2_subdev_state *sd_state;
 	int ret = 0;
 
 	mutex_lock(&state->lock);
@@ -723,7 +675,9 @@ static int imx8mq_mipi_csi_pm_resume(struct device *dev)
 		ret = imx8mq_mipi_csi_clk_enable(state);
 	}
 	if (state->state & ST_STREAMING) {
-		ret = imx8mq_mipi_csi_start_stream(state);
+		sd_state = v4l2_subdev_lock_and_get_active_state(sd);
+		ret = imx8mq_mipi_csi_start_stream(state, sd_state);
+		v4l2_subdev_unlock_state(sd_state);
 		if (ret)
 			goto unlock;
 	}
@@ -803,6 +757,7 @@ static const struct dev_pm_ops imx8mq_mipi_csi_pm_ops = {
 static int imx8mq_mipi_csi_subdev_init(struct csi_state *state)
 {
 	struct v4l2_subdev *sd = &state->sd;
+	int ret;
 
 	v4l2_subdev_init(sd, &imx8mq_mipi_csi_subdev_ops);
 	sd->owner = THIS_MODULE;
@@ -816,15 +771,22 @@ static int imx8mq_mipi_csi_subdev_init(struct csi_state *state)
 
 	sd->dev = state->dev;
 
-	state->csi2_fmt = &imx8mq_mipi_csi_formats[0];
-	imx8mq_mipi_csi_init_cfg(sd, NULL);
-
 	state->pads[MIPI_CSI2_PAD_SINK].flags = MEDIA_PAD_FL_SINK
 					 | MEDIA_PAD_FL_MUST_CONNECT;
 	state->pads[MIPI_CSI2_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE
 					   | MEDIA_PAD_FL_MUST_CONNECT;
-	return media_entity_pads_init(&sd->entity, MIPI_CSI2_PADS_NUM,
-				      state->pads);
+	ret = media_entity_pads_init(&sd->entity, MIPI_CSI2_PADS_NUM,
+				     state->pads);
+	if (ret)
+		return ret;
+
+	ret = v4l2_subdev_init_finalize(sd);
+	if (ret) {
+		media_entity_cleanup(&sd->entity);
+		return ret;
+	}
+
+	return 0;
 }
 
 static void imx8mq_mipi_csi_release_icc(struct platform_device *pdev)
@@ -950,6 +912,7 @@ cleanup:
 	imx8mq_mipi_csi_runtime_suspend(&pdev->dev);
 
 	media_entity_cleanup(&state->sd.entity);
+	v4l2_subdev_cleanup(&state->sd);
 	v4l2_async_nf_unregister(&state->notifier);
 	v4l2_async_nf_cleanup(&state->notifier);
 	v4l2_async_unregister_subdev(&state->sd);
@@ -961,7 +924,7 @@ mutex:
 	return ret;
 }
 
-static int imx8mq_mipi_csi_remove(struct platform_device *pdev)
+static void imx8mq_mipi_csi_remove(struct platform_device *pdev)
 {
 	struct v4l2_subdev *sd = platform_get_drvdata(pdev);
 	struct csi_state *state = mipi_sd_to_csi2_state(sd);
@@ -973,11 +936,10 @@ static int imx8mq_mipi_csi_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 	imx8mq_mipi_csi_runtime_suspend(&pdev->dev);
 	media_entity_cleanup(&state->sd.entity);
+	v4l2_subdev_cleanup(&state->sd);
 	mutex_destroy(&state->lock);
 	pm_runtime_set_suspended(&pdev->dev);
 	imx8mq_mipi_csi_release_icc(pdev);
-
-	return 0;
 }
 
 static const struct of_device_id imx8mq_mipi_csi_of_match[] = {
@@ -988,7 +950,7 @@ MODULE_DEVICE_TABLE(of, imx8mq_mipi_csi_of_match);
 
 static struct platform_driver imx8mq_mipi_csi_driver = {
 	.probe		= imx8mq_mipi_csi_probe,
-	.remove		= imx8mq_mipi_csi_remove,
+	.remove_new	= imx8mq_mipi_csi_remove,
 	.driver		= {
 		.of_match_table = imx8mq_mipi_csi_of_match,
 		.name		= MIPI_CSI2_DRIVER_NAME,
diff --git a/drivers/staging/media/meson/vdec/vdec.c b/drivers/staging/media/meson/vdec/vdec.c
index 52f224d8def1..5ca4b1200831 100644
--- a/drivers/staging/media/meson/vdec/vdec.c
+++ b/drivers/staging/media/meson/vdec/vdec.c
@@ -1106,19 +1106,17 @@ err_vdev_release:
 	return ret;
 }
 
-static int vdec_remove(struct platform_device *pdev)
+static void vdec_remove(struct platform_device *pdev)
 {
 	struct amvdec_core *core = platform_get_drvdata(pdev);
 
 	video_unregister_device(core->vdev_dec);
 	v4l2_device_unregister(&core->v4l2_dev);
-
-	return 0;
 }
 
 static struct platform_driver meson_vdec_driver = {
 	.probe = vdec_probe,
-	.remove = vdec_remove,
+	.remove_new = vdec_remove,
 	.driver = {
 		.name = "meson-vdec",
 		.of_match_table = vdec_dt_match,
diff --git a/drivers/staging/media/omap4iss/iss.c b/drivers/staging/media/omap4iss/iss.c
index fa2a36d829d3..0c4283bb48ad 100644
--- a/drivers/staging/media/omap4iss/iss.c
+++ b/drivers/staging/media/omap4iss/iss.c
@@ -1323,15 +1323,13 @@ error:
 	return ret;
 }
 
-static int iss_remove(struct platform_device *pdev)
+static void iss_remove(struct platform_device *pdev)
 {
 	struct iss_device *iss = platform_get_drvdata(pdev);
 
 	iss_unregister_entities(iss);
 	media_entity_enum_cleanup(&iss->crashed);
 	iss_cleanup_modules(iss);
-
-	return 0;
 }
 
 static const struct platform_device_id omap4iss_id_table[] = {
@@ -1342,7 +1340,7 @@ MODULE_DEVICE_TABLE(platform, omap4iss_id_table);
 
 static struct platform_driver iss_driver = {
 	.probe		= iss_probe,
-	.remove		= iss_remove,
+	.remove_new	= iss_remove,
 	.id_table	= omap4iss_id_table,
 	.driver = {
 		.name	= "omap4iss",
diff --git a/drivers/staging/media/omap4iss/iss_video.c b/drivers/staging/media/omap4iss/iss_video.c
index 05548eab7daa..22fa4d6cae10 100644
--- a/drivers/staging/media/omap4iss/iss_video.c
+++ b/drivers/staging/media/omap4iss/iss_video.c
@@ -237,7 +237,9 @@ static int
 __iss_video_get_format(struct iss_video *video,
 		       struct v4l2_mbus_framefmt *format)
 {
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *subdev;
 	u32 pad;
 	int ret;
@@ -246,9 +248,7 @@ __iss_video_get_format(struct iss_video *video,
 	if (!subdev)
 		return -EINVAL;
 
-	memset(&fmt, 0, sizeof(fmt));
 	fmt.pad = pad;
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 
 	mutex_lock(&video->mutex);
 	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &fmt);
@@ -610,7 +610,9 @@ static int
 iss_video_try_format(struct file *file, void *fh, struct v4l2_format *format)
 {
 	struct iss_video *video = video_drvdata(file);
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *subdev;
 	u32 pad;
 	int ret;
@@ -625,7 +627,6 @@ iss_video_try_format(struct file *file, void *fh, struct v4l2_format *format)
 	iss_video_pix_to_mbus(&format->fmt.pix, &fmt.format);
 
 	fmt.pad = pad;
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &fmt);
 	if (ret)
 		return ret;
@@ -638,7 +639,9 @@ static int
 iss_video_get_selection(struct file *file, void *fh, struct v4l2_selection *sel)
 {
 	struct iss_video *video = video_drvdata(file);
-	struct v4l2_subdev_format format;
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *subdev;
 	struct v4l2_subdev_selection sdsel = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
@@ -679,7 +682,6 @@ iss_video_get_selection(struct file *file, void *fh, struct v4l2_selection *sel)
 		return ret;
 
 	format.pad = pad;
-	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &format);
 	if (ret < 0)
 		return ret == -ENOIOCTLCMD ? -ENOTTY : ret;
diff --git a/drivers/staging/media/rkvdec/rkvdec.c b/drivers/staging/media/rkvdec/rkvdec.c
index 7bab7586918c..134e2b9fa7d9 100644
--- a/drivers/staging/media/rkvdec/rkvdec.c
+++ b/drivers/staging/media/rkvdec/rkvdec.c
@@ -1062,14 +1062,15 @@ err_disable_runtime_pm:
 	return ret;
 }
 
-static int rkvdec_remove(struct platform_device *pdev)
+static void rkvdec_remove(struct platform_device *pdev)
 {
 	struct rkvdec_dev *rkvdec = platform_get_drvdata(pdev);
 
+	cancel_delayed_work_sync(&rkvdec->watchdog_work);
+
 	rkvdec_v4l2_cleanup(rkvdec);
 	pm_runtime_disable(&pdev->dev);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -1099,7 +1100,7 @@ static const struct dev_pm_ops rkvdec_pm_ops = {
 
 static struct platform_driver rkvdec_driver = {
 	.probe = rkvdec_probe,
-	.remove = rkvdec_remove,
+	.remove_new = rkvdec_remove,
 	.driver = {
 		   .name = "rkvdec",
 		   .of_match_table = of_rkvdec_match,
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c
index a43d5ff66716..8e248d4a0aec 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus.c
@@ -543,10 +543,11 @@ err_v4l2:
 	return ret;
 }
 
-static int cedrus_remove(struct platform_device *pdev)
+static void cedrus_remove(struct platform_device *pdev)
 {
 	struct cedrus_dev *dev = platform_get_drvdata(pdev);
 
+	cancel_delayed_work_sync(&dev->watchdog_work);
 	if (media_devnode_is_registered(dev->mdev.devnode)) {
 		media_device_unregister(&dev->mdev);
 		v4l2_m2m_unregister_media_controller(dev->m2m_dev);
@@ -558,8 +559,6 @@ static int cedrus_remove(struct platform_device *pdev)
 	v4l2_device_unregister(&dev->v4l2_dev);
 
 	cedrus_hw_remove(dev);
-
-	return 0;
 }
 
 static const struct cedrus_variant sun4i_a10_cedrus_variant = {
@@ -706,7 +705,7 @@ static const struct dev_pm_ops cedrus_dev_pm_ops = {
 
 static struct platform_driver cedrus_driver = {
 	.probe		= cedrus_probe,
-	.remove		= cedrus_remove,
+	.remove_new	= cedrus_remove,
 	.driver		= {
 		.name		= CEDRUS_NAME,
 		.of_match_table	= of_match_ptr(cedrus_dt_match),
diff --git a/drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.c b/drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.c
index 7b7947509b69..0dc75adbd9d8 100644
--- a/drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.c
+++ b/drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.c
@@ -493,7 +493,7 @@ error_resources:
 	return ret;
 }
 
-static int sun6i_isp_remove(struct platform_device *platform_dev)
+static void sun6i_isp_remove(struct platform_device *platform_dev)
 {
 	struct sun6i_isp_device *isp_dev = platform_get_drvdata(platform_dev);
 
@@ -503,8 +503,6 @@ static int sun6i_isp_remove(struct platform_device *platform_dev)
 	sun6i_isp_v4l2_cleanup(isp_dev);
 	sun6i_isp_tables_cleanup(isp_dev);
 	sun6i_isp_resources_cleanup(isp_dev);
-
-	return 0;
 }
 
 /*
@@ -540,7 +538,7 @@ MODULE_DEVICE_TABLE(of, sun6i_isp_of_match);
 
 static struct platform_driver sun6i_isp_platform_driver = {
 	.probe	= sun6i_isp_probe,
-	.remove	= sun6i_isp_remove,
+	.remove_new = sun6i_isp_remove,
 	.driver	= {
 		.name		= SUN6I_ISP_NAME,
 		.of_match_table	= of_match_ptr(sun6i_isp_of_match),
diff --git a/drivers/staging/media/tegra-video/csi.c b/drivers/staging/media/tegra-video/csi.c
index 426e653bd55d..36ca639622c9 100644
--- a/drivers/staging/media/tegra-video/csi.c
+++ b/drivers/staging/media/tegra-video/csi.c
@@ -778,14 +778,8 @@ rpm_disable:
 static int tegra_csi_remove(struct platform_device *pdev)
 {
 	struct tegra_csi *csi = platform_get_drvdata(pdev);
-	int err;
 
-	err = host1x_client_unregister(&csi->client);
-	if (err < 0) {
-		dev_err(&pdev->dev,
-			"failed to unregister host1x client: %d\n", err);
-		return err;
-	}
+	host1x_client_unregister(&csi->client);
 
 	pm_runtime_disable(&pdev->dev);
 
diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c
index 11dd142c98c5..2f1aff7e8717 100644
--- a/drivers/staging/media/tegra-video/vi.c
+++ b/drivers/staging/media/tegra-video/vi.c
@@ -493,7 +493,9 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan,
 	const struct tegra_video_format *fmtinfo;
 	static struct lock_class_key key;
 	struct v4l2_subdev *subdev;
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_TRY,
+	};
 	struct v4l2_subdev_state *sd_state;
 	struct v4l2_subdev_frame_size_enum fse = {
 		.which = V4L2_SUBDEV_FORMAT_TRY,
@@ -529,7 +531,6 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan,
 	}
 
 	pix->field = V4L2_FIELD_NONE;
-	fmt.which = V4L2_SUBDEV_FORMAT_TRY;
 	fmt.pad = 0;
 	v4l2_fill_mbus_format(&fmt.format, pix, fmtinfo->code);
 
@@ -590,7 +591,9 @@ static int tegra_channel_set_format(struct file *file, void *fh,
 {
 	struct tegra_vi_channel *chan = video_drvdata(file);
 	const struct tegra_video_format *fmtinfo;
-	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
 	struct v4l2_subdev *subdev;
 	struct v4l2_pix_format *pix = &format->fmt.pix;
 	int ret;
@@ -605,7 +608,6 @@ static int tegra_channel_set_format(struct file *file, void *fh,
 
 	fmtinfo = tegra_get_format_by_fourcc(chan->vi, pix->pixelformat);
 
-	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	fmt.pad = 0;
 	v4l2_fill_mbus_format(&fmt.format, pix, fmtinfo->code);
 	subdev = tegra_channel_get_remote_source_subdev(chan);
@@ -2041,14 +2043,8 @@ rpm_disable:
 static int tegra_vi_remove(struct platform_device *pdev)
 {
 	struct tegra_vi *vi = platform_get_drvdata(pdev);
-	int err;
 
-	err = host1x_client_unregister(&vi->client);
-	if (err < 0) {
-		dev_err(&pdev->dev,
-			"failed to unregister host1x client: %d\n", err);
-		return err;
-	}
+	host1x_client_unregister(&vi->client);
 
 	pm_runtime_disable(&pdev->dev);
 
diff --git a/drivers/staging/most/dim2/dim2.c b/drivers/staging/most/dim2/dim2.c
index 7a5f80e637a0..44d3252d4612 100644
--- a/drivers/staging/most/dim2/dim2.c
+++ b/drivers/staging/most/dim2/dim2.c
@@ -108,7 +108,10 @@ struct dim2_platform_data {
 	u8 fcnt;
 };
 
-#define iface_to_hdm(iface) container_of(iface, struct dim2_hdm, most_iface)
+static inline struct dim2_hdm *iface_to_hdm(struct most_interface *iface)
+{
+	return container_of(iface, struct dim2_hdm, most_iface);
+}
 
 /* Macro to identify a network status message */
 #define PACKET_IS_NET_INFO(p)  \
@@ -775,8 +778,7 @@ static int dim2_probe(struct platform_device *pdev)
 		goto err_free_dev;
 	}
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	dev->io_base = devm_ioremap_resource(&pdev->dev, res);
+	dev->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(dev->io_base)) {
 		ret = PTR_ERR(dev->io_base);
 		goto err_free_dev;
@@ -906,13 +908,11 @@ err_free_dev:
  *
  * Unregister the interface from mostcore
  */
-static int dim2_remove(struct platform_device *pdev)
+static void dim2_remove(struct platform_device *pdev)
 {
 	struct dim2_hdm *dev = platform_get_drvdata(pdev);
 
 	most_deregister_interface(&dev->most_iface);
-
-	return 0;
 }
 
 /* platform specific functions [[ */
@@ -987,7 +987,6 @@ static int rcar_gen2_enable(struct platform_device *pdev)
 		writel(0x04, dev->io_base + 0x600);
 	}
 
-
 	/* BBCR = 0b11 */
 	writel(0x03, dev->io_base + 0x500);
 	writel(0x0002FF02, dev->io_base + 0x508);
@@ -1091,7 +1090,7 @@ MODULE_DEVICE_TABLE(of, dim2_of_match);
 
 static struct platform_driver dim2_driver = {
 	.probe = dim2_probe,
-	.remove = dim2_remove,
+	.remove_new = dim2_remove,
 	.driver = {
 		.name = "hdm_dim2",
 		.of_match_table = dim2_of_match,
diff --git a/drivers/staging/most/dim2/hal.c b/drivers/staging/most/dim2/hal.c
index a5d40b5b138a..6abe3ab2b2cf 100644
--- a/drivers/staging/most/dim2/hal.c
+++ b/drivers/staging/most/dim2/hal.c
@@ -346,9 +346,8 @@ static void dim2_clear_ctram(void)
 		dim2_clear_ctr(ctr_addr);
 }
 
-static void dim2_configure_channel(
-	u8 ch_addr, u8 type, u8 is_tx, u16 dbr_address, u16 hw_buffer_size,
-	u16 packet_length)
+static void dim2_configure_channel(u8 ch_addr, u8 type, u8 is_tx, u16 dbr_address,
+				   u16 hw_buffer_size, u16 packet_length)
 {
 	dim2_configure_cdt(ch_addr, dbr_address, hw_buffer_size, packet_length);
 	dim2_configure_cat(MLB_CAT, ch_addr, type, is_tx ? 1 : 0);
diff --git a/drivers/staging/most/i2c/i2c.c b/drivers/staging/most/i2c/i2c.c
index df53a4c4f850..4e85e681922f 100644
--- a/drivers/staging/most/i2c/i2c.c
+++ b/drivers/staging/most/i2c/i2c.c
@@ -44,7 +44,10 @@ struct hdm_i2c {
 	char name[64];
 };
 
-#define to_hdm(iface) container_of(iface, struct hdm_i2c, most_iface)
+static inline struct hdm_i2c *to_hdm(struct most_interface *iface)
+{
+	return container_of(iface, struct hdm_i2c, most_iface);
+}
 
 static irqreturn_t most_irq_handler(int, void *);
 static void pending_rx_work(struct work_struct *);
diff --git a/drivers/staging/most/video/video.c b/drivers/staging/most/video/video.c
index ffa97ef21ea5..6254a5df2502 100644
--- a/drivers/staging/most/video/video.c
+++ b/drivers/staging/most/video/video.c
@@ -365,8 +365,7 @@ static const struct video_device comp_videodev_template = {
 
 /**************************************************************************/
 
-static struct most_video_dev *get_comp_dev(
-	struct most_interface *iface, int channel_idx)
+static struct most_video_dev *get_comp_dev(struct most_interface *iface, int channel_idx)
 {
 	struct most_video_dev *mdev;
 	unsigned long flags;
diff --git a/drivers/staging/nvec/nvec.c b/drivers/staging/nvec/nvec.c
index b3f114cb00dc..2823cacde130 100644
--- a/drivers/staging/nvec/nvec.c
+++ b/drivers/staging/nvec/nvec.c
@@ -882,7 +882,7 @@ static int tegra_nvec_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int tegra_nvec_remove(struct platform_device *pdev)
+static void tegra_nvec_remove(struct platform_device *pdev)
 {
 	struct nvec_chip *nvec = platform_get_drvdata(pdev);
 
@@ -893,8 +893,6 @@ static int tegra_nvec_remove(struct platform_device *pdev)
 	cancel_work_sync(&nvec->tx_work);
 	/* FIXME: needs check whether nvec is responsible for power off */
 	pm_power_off = NULL;
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -942,7 +940,7 @@ MODULE_DEVICE_TABLE(of, nvidia_nvec_of_match);
 
 static struct platform_driver nvec_device_driver = {
 	.probe   = tegra_nvec_probe,
-	.remove  = tegra_nvec_remove,
+	.remove_new = tegra_nvec_remove,
 	.driver  = {
 		.name = "nvec",
 		.pm = &nvec_pm_ops,
diff --git a/drivers/staging/nvec/nvec_kbd.c b/drivers/staging/nvec/nvec_kbd.c
index 386d619e3ee9..f9a1da952c0a 100644
--- a/drivers/staging/nvec/nvec_kbd.c
+++ b/drivers/staging/nvec/nvec_kbd.c
@@ -161,7 +161,7 @@ static int nvec_kbd_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int nvec_kbd_remove(struct platform_device *pdev)
+static void nvec_kbd_remove(struct platform_device *pdev)
 {
 	struct nvec_chip *nvec = dev_get_drvdata(pdev->dev.parent);
 	char disable_kbd[] = { NVEC_KBD, DISABLE_KBD },
@@ -170,13 +170,11 @@ static int nvec_kbd_remove(struct platform_device *pdev)
 	nvec_write_async(nvec, uncnfg_wake_key_reporting, 3);
 	nvec_write_async(nvec, disable_kbd, 2);
 	nvec_unregister_notifier(nvec, &keys_dev.notifier);
-
-	return 0;
 }
 
 static struct platform_driver nvec_kbd_driver = {
 	.probe  = nvec_kbd_probe,
-	.remove = nvec_kbd_remove,
+	.remove_new = nvec_kbd_remove,
 	.driver = {
 		.name = "nvec-kbd",
 	},
diff --git a/drivers/staging/nvec/nvec_paz00.c b/drivers/staging/nvec/nvec_paz00.c
index 8b4da95081c8..55d59840fca4 100644
--- a/drivers/staging/nvec/nvec_paz00.c
+++ b/drivers/staging/nvec/nvec_paz00.c
@@ -14,9 +14,6 @@
 #include <linux/platform_device.h>
 #include "nvec.h"
 
-#define to_nvec_led(led_cdev) \
-	container_of(led_cdev, struct nvec_led, cdev)
-
 #define NVEC_LED_REQ {'\x0d', '\x10', '\x45', '\x10', '\x00'}
 
 #define NVEC_LED_MAX 8
@@ -29,7 +26,7 @@ struct nvec_led {
 static void nvec_led_brightness_set(struct led_classdev *led_cdev,
 				    enum led_brightness value)
 {
-	struct nvec_led *led = to_nvec_led(led_cdev);
+	struct nvec_led *led = container_of(led_cdev, struct nvec_led, cdev);
 	unsigned char buf[] = NVEC_LED_REQ;
 
 	buf[4] = value;
diff --git a/drivers/staging/nvec/nvec_power.c b/drivers/staging/nvec/nvec_power.c
index b1ef196e1cfe..9943b1fff190 100644
--- a/drivers/staging/nvec/nvec_power.c
+++ b/drivers/staging/nvec/nvec_power.c
@@ -416,7 +416,7 @@ static int nvec_power_probe(struct platform_device *pdev)
 	return PTR_ERR_OR_ZERO(*psy);
 }
 
-static int nvec_power_remove(struct platform_device *pdev)
+static void nvec_power_remove(struct platform_device *pdev)
 {
 	struct nvec_power *power = platform_get_drvdata(pdev);
 
@@ -429,13 +429,11 @@ static int nvec_power_remove(struct platform_device *pdev)
 	case BAT:
 		power_supply_unregister(nvec_bat_psy);
 	}
-
-	return 0;
 }
 
 static struct platform_driver nvec_power_driver = {
 	.probe = nvec_power_probe,
-	.remove = nvec_power_remove,
+	.remove_new = nvec_power_remove,
 	.driver = {
 		   .name = "nvec-power",
 	}
diff --git a/drivers/staging/nvec/nvec_ps2.c b/drivers/staging/nvec/nvec_ps2.c
index 06041c7f7d4f..cb6d71b8dc83 100644
--- a/drivers/staging/nvec/nvec_ps2.c
+++ b/drivers/staging/nvec/nvec_ps2.c
@@ -125,7 +125,7 @@ static int nvec_mouse_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int nvec_mouse_remove(struct platform_device *pdev)
+static void nvec_mouse_remove(struct platform_device *pdev)
 {
 	struct nvec_chip *nvec = dev_get_drvdata(pdev->dev.parent);
 
@@ -133,8 +133,6 @@ static int nvec_mouse_remove(struct platform_device *pdev)
 	ps2_stopstreaming(ps2_dev.ser_dev);
 	nvec_unregister_notifier(nvec, &ps2_dev.notifier);
 	serio_unregister_port(ps2_dev.ser_dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -166,7 +164,7 @@ static SIMPLE_DEV_PM_OPS(nvec_mouse_pm_ops, nvec_mouse_suspend,
 
 static struct platform_driver nvec_mouse_driver = {
 	.probe  = nvec_mouse_probe,
-	.remove = nvec_mouse_remove,
+	.remove_new = nvec_mouse_remove,
 	.driver = {
 		.name = "nvec-mouse",
 		.pm = &nvec_mouse_pm_ops,
diff --git a/drivers/staging/octeon/ethernet.c b/drivers/staging/octeon/ethernet.c
index f662739137b5..9eee28f2940c 100644
--- a/drivers/staging/octeon/ethernet.c
+++ b/drivers/staging/octeon/ethernet.c
@@ -924,7 +924,7 @@ static int cvm_oct_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int cvm_oct_remove(struct platform_device *pdev)
+static void cvm_oct_remove(struct platform_device *pdev)
 {
 	int port;
 
@@ -965,7 +965,6 @@ static int cvm_oct_remove(struct platform_device *pdev)
 	if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL)
 		cvm_oct_mem_empty_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL,
 				      CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, 128);
-	return 0;
 }
 
 static const struct of_device_id cvm_oct_match[] = {
@@ -978,7 +977,7 @@ MODULE_DEVICE_TABLE(of, cvm_oct_match);
 
 static struct platform_driver cvm_oct_driver = {
 	.probe		= cvm_oct_probe,
-	.remove		= cvm_oct_remove,
+	.remove_new	= cvm_oct_remove,
 	.driver		= {
 		.name	= KBUILD_MODNAME,
 		.of_match_table = cvm_oct_match,
diff --git a/drivers/staging/octeon/octeon-stubs.h b/drivers/staging/octeon/octeon-stubs.h
index 7a02e59e283f..3e7b92cd2e35 100644
--- a/drivers/staging/octeon/octeon-stubs.h
+++ b/drivers/staging/octeon/octeon-stubs.h
@@ -1372,9 +1372,7 @@ static inline void cvmx_fau_async_fetch_and_add32(uint64_t scraddr,
 						  int32_t value)
 { }
 
-static inline union cvmx_gmxx_rxx_rx_inbnd cvmx_spi4000_check_speed(
-	int interface,
-	int port)
+static inline union cvmx_gmxx_rxx_rx_inbnd cvmx_spi4000_check_speed(int interface, int port)
 {
 	union cvmx_gmxx_rxx_rx_inbnd r;
 
diff --git a/drivers/staging/pi433/pi433_if.c b/drivers/staging/pi433/pi433_if.c
index b59f6a4cb611..f08fdf06d566 100644
--- a/drivers/staging/pi433/pi433_if.c
+++ b/drivers/staging/pi433/pi433_if.c
@@ -1400,7 +1400,7 @@ static int __init pi433_init(void)
 	if (status < 0)
 		return status;
 
-	pi433_class = class_create(THIS_MODULE, "pi433");
+	pi433_class = class_create("pi433");
 	if (IS_ERR(pi433_class)) {
 		unregister_chrdev(MAJOR(pi433_dev),
 				  pi433_spi_driver.driver.name);
diff --git a/drivers/staging/qlge/qlge_dbg.c b/drivers/staging/qlge/qlge_dbg.c
index 66d28358342f..c7e865f515cf 100644
--- a/drivers/staging/qlge/qlge_dbg.c
+++ b/drivers/staging/qlge/qlge_dbg.c
@@ -351,26 +351,23 @@ static int qlge_get_xgmac_regs(struct qlge_adapter *qdev, u32 *buf,
 		/* We're reading 400 xgmac registers, but we filter out
 		 * several locations that are non-responsive to reads.
 		 */
-		if ((i == 0x00000114) ||
-		    (i == 0x00000118) ||
-			(i == 0x0000013c) ||
-			(i == 0x00000140) ||
-			(i > 0x00000150 && i < 0x000001fc) ||
-			(i > 0x00000278 && i < 0x000002a0) ||
-			(i > 0x000002c0 && i < 0x000002cf) ||
-			(i > 0x000002dc && i < 0x000002f0) ||
-			(i > 0x000003c8 && i < 0x00000400) ||
-			(i > 0x00000400 && i < 0x00000410) ||
-			(i > 0x00000410 && i < 0x00000420) ||
-			(i > 0x00000420 && i < 0x00000430) ||
-			(i > 0x00000430 && i < 0x00000440) ||
-			(i > 0x00000440 && i < 0x00000450) ||
-			(i > 0x00000450 && i < 0x00000500) ||
-			(i > 0x0000054c && i < 0x00000568) ||
-			(i > 0x000005c8 && i < 0x00000600)) {
+		if ((i == 0x00000114) || (i == 0x00000118) ||
+		    (i == 0x0000013c) || (i == 0x00000140) ||
+		    (i > 0x00000150 && i < 0x000001fc) ||
+		    (i > 0x00000278 && i < 0x000002a0) ||
+		    (i > 0x000002c0 && i < 0x000002cf) ||
+		    (i > 0x000002dc && i < 0x000002f0) ||
+		    (i > 0x000003c8 && i < 0x00000400) ||
+		    (i > 0x00000400 && i < 0x00000410) ||
+		    (i > 0x00000410 && i < 0x00000420) ||
+		    (i > 0x00000420 && i < 0x00000430) ||
+		    (i > 0x00000430 && i < 0x00000440) ||
+		    (i > 0x00000440 && i < 0x00000450) ||
+		    (i > 0x00000450 && i < 0x00000500) ||
+		    (i > 0x0000054c && i < 0x00000568) ||
+		    (i > 0x000005c8 && i < 0x00000600)) {
 			if (other_function)
-				status =
-				qlge_read_other_func_xgmac_reg(qdev, i, buf);
+				status = qlge_read_other_func_xgmac_reg(qdev, i, buf);
 			else
 				status = qlge_read_xgmac_reg(qdev, i, buf);
 
diff --git a/drivers/staging/rtl8192e/rtl8192e/Makefile b/drivers/staging/rtl8192e/rtl8192e/Makefile
index 75e6ec510555..a442d79ea71e 100644
--- a/drivers/staging/rtl8192e/rtl8192e/Makefile
+++ b/drivers/staging/rtl8192e/rtl8192e/Makefile
@@ -4,7 +4,7 @@ r8192e_pci-objs :=		\
 	r8192E_phy.o		\
 	r8192E_firmware.o	\
 	r8192E_cmdpkt.o		\
-	r8192E_hwimg.o		\
+	table.o			\
 	r8190P_rtl8256.o	\
 	rtl_cam.o		\
 	rtl_core.o		\
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8190P_def.h b/drivers/staging/rtl8192e/rtl8192e/r8190P_def.h
index ac192254a4bb..385cca79f484 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8190P_def.h
+++ b/drivers/staging/rtl8192e/rtl8192e/r8190P_def.h
@@ -103,21 +103,10 @@ enum rf_optype {
 
 struct bb_reg_definition {
 	u32 rfintfs;
-	u32 rfintfi;
 	u32 rfintfo;
 	u32 rfintfe;
 	u32 rf3wireOffset;
-	u32 rfLSSI_Select;
-	u32 rfTxGainStage;
-	u32 rfHSSIPara1;
 	u32 rfHSSIPara2;
-	u32 rfSwitchControl;
-	u32 rfAGCControl1;
-	u32 rfAGCControl2;
-	u32 rfRxIQImbalance;
-	u32 rfRxAFE;
-	u32 rfTxIQImbalance;
-	u32 rfTxAFE;
 	u32 rfLSSIReadBack;
 	u32 rfLSSIReadBackPi;
 };
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.c b/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.c
index 73a86e1d0701..bb4539e337c8 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.c
+++ b/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.c
@@ -22,9 +22,6 @@ void rtl92e_set_bandwidth(struct net_device *dev,
 	}
 
 	for (eRFPath = 0; eRFPath < priv->num_total_rf_path; eRFPath++) {
-		if (!rtl92e_is_legal_rf_path(dev, eRFPath))
-			continue;
-
 		switch (bandwidth) {
 		case HT_CHANNEL_WIDTH_20:
 			rtl92e_set_rf_reg(dev, (enum rf90_radio_path)eRFPath,
@@ -67,19 +64,14 @@ bool rtl92e_config_rf(struct net_device *dev)
 
 	for (eRFPath = (enum rf90_radio_path)RF90_PATH_A;
 	     eRFPath < priv->num_total_rf_path; eRFPath++) {
-		if (!rtl92e_is_legal_rf_path(dev, eRFPath))
-			continue;
-
 		pPhyReg = &priv->phy_reg_def[eRFPath];
 
 		switch (eRFPath) {
 		case RF90_PATH_A:
-		case RF90_PATH_C:
 			u4RegValue = rtl92e_get_bb_reg(dev, pPhyReg->rfintfs,
 						       bRFSI_RFENV);
 			break;
 		case RF90_PATH_B:
-		case RF90_PATH_D:
 			u4RegValue = rtl92e_get_bb_reg(dev, pPhyReg->rfintfs,
 						       bRFSI_RFENV<<16);
 			break;
@@ -120,12 +112,10 @@ bool rtl92e_config_rf(struct net_device *dev)
 
 		switch (eRFPath) {
 		case RF90_PATH_A:
-		case RF90_PATH_C:
 			rtl92e_set_bb_reg(dev, pPhyReg->rfintfs, bRFSI_RFENV,
 					  u4RegValue);
 			break;
 		case RF90_PATH_B:
-		case RF90_PATH_D:
 			rtl92e_set_bb_reg(dev, pPhyReg->rfintfs,
 					  bRFSI_RFENV<<16, u4RegValue);
 			break;
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c
index 0b5b2ae27f9e..aed53fedeb61 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c
@@ -343,18 +343,11 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev)
 	else
 		priv->tx_pwr_data_read_from_eeprom = false;
 
-	priv->rf_type = RTL819X_DEFAULT_RF_TYPE;
-
 	if (priv->card_8192_version > VERSION_8190_BD) {
 		if (!priv->autoload_fail_flag) {
 			tempval = (rtl92e_eeprom_read(dev,
 						      (EEPROM_RFInd_PowerDiff >> 1))) & 0xff;
 			priv->eeprom_legacy_ht_tx_pwr_diff = tempval & 0xf;
-
-			if (tempval&0x80)
-				priv->rf_type = RF_1T2R;
-			else
-				priv->rf_type = RF_2T4R;
 		} else {
 			priv->eeprom_legacy_ht_tx_pwr_diff = 0x04;
 		}
@@ -433,26 +426,12 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev)
 
 	rtl92e_init_adaptive_rate(dev);
 
-	priv->rf_chip = RF_8256;
-
 	if (priv->reg_chnl_plan == 0xf)
 		priv->chnl_plan = priv->eeprom_chnl_plan;
 	else
 		priv->chnl_plan = priv->reg_chnl_plan;
 
-	if (priv->eeprom_vid == 0x1186 &&  priv->eeprom_did == 0x3304)
-		priv->customer_id =  RT_CID_DLINK;
-
 	switch (priv->eeprom_customer_id) {
-	case EEPROM_CID_DEFAULT:
-		priv->customer_id = RT_CID_DEFAULT;
-		break;
-	case EEPROM_CID_CAMEO:
-		priv->customer_id = RT_CID_819x_CAMEO;
-		break;
-	case  EEPROM_CID_RUNTOP:
-		priv->customer_id = RT_CID_819x_RUNTOP;
-		break;
 	case EEPROM_CID_NetCore:
 		priv->customer_id = RT_CID_819X_NETCORE;
 		break;
@@ -463,20 +442,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev)
 		else
 			priv->chnl_plan = 0x0;
 		break;
-	case EEPROM_CID_Nettronix:
-		priv->customer_id = RT_CID_Nettronix;
-		break;
-	case EEPROM_CID_Pronet:
-		priv->customer_id = RT_CID_PRONET;
-		break;
-	case EEPROM_CID_DLINK:
-		priv->customer_id = RT_CID_DLINK;
-		break;
-
-	case EEPROM_CID_WHQL:
-		break;
-	default:
-		break;
 	}
 
 	if (priv->chnl_plan > CHANNEL_PLAN_LEN - 1)
@@ -512,16 +477,6 @@ static void _rtl92e_hwconfig(struct net_device *dev)
 		regRATR = RATE_ALL_CCK;
 		regRRSR = RATE_ALL_CCK;
 		break;
-	case WIRELESS_MODE_A:
-		regBwOpMode = BW_OPMODE_5G | BW_OPMODE_20MHZ;
-		regRATR = RATE_ALL_OFDM_AG;
-		regRRSR = RATE_ALL_OFDM_AG;
-		break;
-	case WIRELESS_MODE_G:
-		regBwOpMode = BW_OPMODE_20MHZ;
-		regRATR = RATE_ALL_CCK | RATE_ALL_OFDM_AG;
-		regRRSR = RATE_ALL_CCK | RATE_ALL_OFDM_AG;
-		break;
 	case WIRELESS_MODE_AUTO:
 	case WIRELESS_MODE_N_24G:
 		regBwOpMode = BW_OPMODE_20MHZ;
@@ -529,12 +484,7 @@ static void _rtl92e_hwconfig(struct net_device *dev)
 			  RATE_ALL_OFDM_1SS | RATE_ALL_OFDM_2SS;
 		regRRSR = RATE_ALL_CCK | RATE_ALL_OFDM_AG;
 		break;
-	case WIRELESS_MODE_N_5G:
-		regBwOpMode = BW_OPMODE_5G;
-		regRATR = RATE_ALL_OFDM_AG | RATE_ALL_OFDM_1SS |
-			  RATE_ALL_OFDM_2SS;
-		regRRSR = RATE_ALL_OFDM_AG;
-		break;
+	case WIRELESS_MODE_G:
 	default:
 		regBwOpMode = BW_OPMODE_20MHZ;
 		regRATR = RATE_ALL_CCK | RATE_ALL_OFDM_AG;
@@ -547,8 +497,7 @@ static void _rtl92e_hwconfig(struct net_device *dev)
 		u32 ratr_value;
 
 		ratr_value = regRATR;
-		if (priv->rf_type == RF_1T2R)
-			ratr_value &= ~(RATE_ALL_OFDM_2SS);
+		ratr_value &= ~(RATE_ALL_OFDM_2SS);
 		rtl92e_writel(dev, RATR0, ratr_value);
 		rtl92e_writeb(dev, UFWP, 1);
 	}
@@ -701,7 +650,7 @@ start:
 	}
 
 	if (priv->rst_progress == RESET_TYPE_NORESET) {
-		rtStatus = rtl92e_config_phy(dev);
+		rtStatus = rtl92e_config_rf(dev);
 		if (!rtStatus) {
 			netdev_info(dev, "RF Config failed\n");
 			return rtStatus;
@@ -806,7 +755,7 @@ void rtl92e_link_change(struct net_device *dev)
 
 	if (ieee->state == RTLLIB_LINKED) {
 		_rtl92e_net_update(dev);
-		priv->ops->update_ratr_table(dev);
+		rtl92e_update_ratr_table(dev);
 		if ((ieee->pairwise_key_type == KEY_TYPE_WEP40) ||
 		    (ieee->pairwise_key_type == KEY_TYPE_WEP104))
 			rtl92e_enable_hw_security_config(dev);
@@ -1515,9 +1464,7 @@ static void _rtl92e_process_phyinfo(struct r8192_priv *priv, u8 *buffer,
 		return;
 
 	if (!prev_st->bIsCCK && prev_st->bPacketToSelf) {
-		for (rfpath = RF90_PATH_A; rfpath < RF90_PATH_C; rfpath++) {
-			if (!rtl92e_is_legal_rf_path(priv->rtllib->dev, rfpath))
-				continue;
+		for (rfpath = RF90_PATH_A; rfpath < priv->num_total_rf_path; rfpath++) {
 			if (priv->stats.rx_rssi_percentage[rfpath] == 0) {
 				priv->stats.rx_rssi_percentage[rfpath] =
 					 prev_st->RxMIMOSignalStrength[rfpath];
@@ -1895,14 +1842,10 @@ void rtl92e_update_ratr_table(struct net_device *dev)
 		break;
 	case IEEE_N_24G:
 	case IEEE_N_5G:
-		if (ieee->ht_info->peer_mimo_ps == 0) {
+		if (ieee->ht_info->peer_mimo_ps == 0)
 			ratr_value &= 0x0007F007;
-		} else {
-			if (priv->rf_type == RF_1T2R)
-				ratr_value &= 0x000FF007;
-			else
-				ratr_value &= 0x0F81F007;
-		}
+		else
+			ratr_value &= 0x000FF007;
 		break;
 	default:
 		break;
@@ -1970,15 +1913,6 @@ void rtl92e_disable_irq(struct net_device *dev)
 	priv->irq_enabled = 0;
 }
 
-void rtl92e_clear_irq(struct net_device *dev)
-{
-	u32 tmp;
-
-	tmp = rtl92e_readl(dev, ISR);
-	rtl92e_writel(dev, ISR, tmp);
-}
-
-
 void rtl92e_enable_rx(struct net_device *dev)
 {
 	struct r8192_priv *priv = (struct r8192_priv *)rtllib_priv(dev);
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.h b/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.h
index 1713381dc2b4..fa3b71dbb091 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.h
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.h
@@ -18,7 +18,6 @@ void rtl92e_enable_rx(struct net_device *dev);
 void rtl92e_enable_tx(struct net_device *dev);
 void rtl92e_enable_irq(struct net_device *dev);
 void rtl92e_disable_irq(struct net_device *dev);
-void rtl92e_clear_irq(struct net_device *dev);
 void rtl92e_init_variables(struct net_device  *dev);
 void rtl92e_start_beacon(struct net_device *dev);
 void rtl92e_set_reg(struct net_device *dev, u8 variable, u8 *val);
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c
index b011ec8c8a41..ddf998cf2041 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c
@@ -6,7 +6,7 @@
  */
 #include "rtl_core.h"
 #include "r8192E_hw.h"
-#include "r8192E_hwimg.h"
+#include "table.h"
 #include "r8192E_firmware.h"
 #include "r8192E_cmdpkt.h"
 #include <linux/firmware.h>
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_hw.h b/drivers/staging/rtl8192e/rtl8192e/r8192E_hw.h
index 99640c4779f7..f4d4b01630a2 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_hw.h
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_hw.h
@@ -30,15 +30,8 @@ enum baseband_config {
 #define EEPROM_TxPwIndex_CCK			0x2C
 #define EEPROM_TxPwIndex_OFDM_24G	0x3A
 
-#define EEPROM_CID_DEFAULT			0x0
-#define EEPROM_CID_CAMEO				0x1
-#define EEPROM_CID_RUNTOP				0x2
 #define EEPROM_CID_TOSHIBA				0x4
 #define EEPROM_CID_NetCore				0x5
-#define EEPROM_CID_Nettronix			0x6
-#define EEPROM_CID_Pronet				0x7
-#define EEPROM_CID_DLINK				0x8
-#define EEPROM_CID_WHQL					0xFE
 enum _RTL8192PCI_HW {
 	MAC0			= 0x000,
 	MAC4			= 0x004,
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_hwimg.h b/drivers/staging/rtl8192e/rtl8192e/r8192E_hwimg.h
deleted file mode 100644
index 7d63f5a5c1b7..000000000000
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_hwimg.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright(c) 2008 - 2010 Realtek Corporation. All rights reserved.
- *
- * Contact Information: wlanfae <wlanfae@realtek.com>
- */
-#ifndef __INC_HAL8192PciE_FW_IMG_H
-#define __INC_HAL8192PciE_FW_IMG_H
-
-/*Created on  2008/11/18,  3: 7*/
-
-#include <linux/types.h>
-
-#define PHY_REGArrayLengthPciE 1
-extern u32 Rtl8192PciEPHY_REGArray[PHY_REGArrayLengthPciE];
-#define PHY_REG_1T2RArrayLengthPciE 296
-extern u32 Rtl8192PciEPHY_REG_1T2RArray[PHY_REG_1T2RArrayLengthPciE];
-#define RadioA_ArrayLengthPciE 246
-extern u32 Rtl8192PciERadioA_Array[RadioA_ArrayLengthPciE];
-#define RadioB_ArrayLengthPciE 78
-extern u32 Rtl8192PciERadioB_Array[RadioB_ArrayLengthPciE];
-#define RadioC_ArrayLengthPciE 2
-extern u32 Rtl8192PciERadioC_Array[RadioC_ArrayLengthPciE];
-#define RadioD_ArrayLengthPciE 2
-extern u32 Rtl8192PciERadioD_Array[RadioD_ArrayLengthPciE];
-#define MACPHY_ArrayLengthPciE 18
-extern u32 Rtl8192PciEMACPHY_Array[MACPHY_ArrayLengthPciE];
-#define MACPHY_Array_PGLengthPciE 30
-extern u32 Rtl8192PciEMACPHY_Array_PG[MACPHY_Array_PGLengthPciE];
-#define AGCTAB_ArrayLengthPciE 384
-extern u32 Rtl8192PciEAGCTAB_Array[AGCTAB_ArrayLengthPciE];
-
-#endif
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c
index c6cbdea6d5b2..4b0ebe96302e 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c
@@ -12,25 +12,7 @@
 #include "r8192E_phy.h"
 #include "rtl_dm.h"
 
-#include "r8192E_hwimg.h"
-
-static u32 RF_CHANNEL_TABLE_ZEBRA[] = {
-	0,
-	0x085c,
-	0x08dc,
-	0x095c,
-	0x09dc,
-	0x0a5c,
-	0x0adc,
-	0x0b5c,
-	0x0bdc,
-	0x0c5c,
-	0x0cdc,
-	0x0d5c,
-	0x0ddc,
-	0x0e5c,
-	0x0f72,
-};
+#include "table.h"
 
 /*************************Define local function prototype**********************/
 
@@ -47,22 +29,6 @@ static u32 _rtl92e_calculate_bit_shift(u32 dwBitMask)
 	return ffs(dwBitMask) - 1;
 }
 
-u8 rtl92e_is_legal_rf_path(struct net_device *dev, u32 eRFPath)
-{
-	u8 ret = 1;
-	struct r8192_priv *priv = rtllib_priv(dev);
-
-	if (priv->rf_type == RF_2T4R)
-		ret = 0;
-	else if (priv->rf_type == RF_1T2R) {
-		if (eRFPath == RF90_PATH_A || eRFPath == RF90_PATH_B)
-			ret = 1;
-		else if (eRFPath == RF90_PATH_C || eRFPath == RF90_PATH_D)
-			ret = 0;
-	}
-	return ret;
-}
-
 void rtl92e_set_bb_reg(struct net_device *dev, u32 dwRegAddr, u32 dwBitMask,
 		       u32 dwData)
 {
@@ -98,24 +64,20 @@ static u32 _rtl92e_phy_rf_read(struct net_device *dev,
 
 	Offset &= 0x3f;
 
-	if (priv->rf_chip == RF_8256) {
-		rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter4, 0xf00, 0x0);
-		if (Offset >= 31) {
-			priv->rf_reg_0value[eRFPath] |= 0x140;
-			rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset,
-					  bMaskDWord,
-					  (priv->rf_reg_0value[eRFPath] << 16));
-			NewOffset = Offset - 30;
-		} else if (Offset >= 16) {
-			priv->rf_reg_0value[eRFPath] |= 0x100;
-			priv->rf_reg_0value[eRFPath] &= (~0x40);
-			rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset,
-					  bMaskDWord,
-					  (priv->rf_reg_0value[eRFPath] << 16));
-
-			NewOffset = Offset - 15;
-		} else
-			NewOffset = Offset;
+	rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter4, 0xf00, 0x0);
+	if (Offset >= 31) {
+		priv->rf_reg_0value[eRFPath] |= 0x140;
+		rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset,
+				  bMaskDWord,
+				  (priv->rf_reg_0value[eRFPath] << 16));
+		NewOffset = Offset - 30;
+	} else if (Offset >= 16) {
+		priv->rf_reg_0value[eRFPath] |= 0x100;
+		priv->rf_reg_0value[eRFPath] &= (~0x40);
+		rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset,
+				  bMaskDWord,
+				  (priv->rf_reg_0value[eRFPath] << 16));
+		NewOffset = Offset - 15;
 	} else {
 		NewOffset = Offset;
 	}
@@ -129,15 +91,12 @@ static u32 _rtl92e_phy_rf_read(struct net_device *dev,
 	ret = rtl92e_get_bb_reg(dev, pPhyReg->rfLSSIReadBack,
 				bLSSIReadBackData);
 
-	if (priv->rf_chip == RF_8256) {
-		priv->rf_reg_0value[eRFPath] &= 0xebf;
-
-		rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset, bMaskDWord,
-				  (priv->rf_reg_0value[eRFPath] << 16));
+	priv->rf_reg_0value[eRFPath] &= 0xebf;
 
-		rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter4, 0x300, 0x3);
-	}
+	rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset, bMaskDWord,
+			  (priv->rf_reg_0value[eRFPath] << 16));
 
+	rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter4, 0x300, 0x3);
 
 	return ret;
 
@@ -152,24 +111,22 @@ static void _rtl92e_phy_rf_write(struct net_device *dev,
 	struct bb_reg_definition *pPhyReg = &priv->phy_reg_def[eRFPath];
 
 	Offset &= 0x3f;
-	if (priv->rf_chip == RF_8256) {
-		rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter4, 0xf00, 0x0);
 
-		if (Offset >= 31) {
-			priv->rf_reg_0value[eRFPath] |= 0x140;
-			rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset,
-					  bMaskDWord,
-					  (priv->rf_reg_0value[eRFPath] << 16));
-			NewOffset = Offset - 30;
-		} else if (Offset >= 16) {
-			priv->rf_reg_0value[eRFPath] |= 0x100;
-			priv->rf_reg_0value[eRFPath] &= (~0x40);
-			rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset,
-					  bMaskDWord,
-					  (priv->rf_reg_0value[eRFPath] << 16));
-			NewOffset = Offset - 15;
-		} else
-			NewOffset = Offset;
+	rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter4, 0xf00, 0x0);
+
+	if (Offset >= 31) {
+		priv->rf_reg_0value[eRFPath] |= 0x140;
+		rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset,
+				  bMaskDWord,
+				  (priv->rf_reg_0value[eRFPath] << 16));
+		NewOffset = Offset - 30;
+	} else if (Offset >= 16) {
+		priv->rf_reg_0value[eRFPath] |= 0x100;
+		priv->rf_reg_0value[eRFPath] &= (~0x40);
+		rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset,
+				  bMaskDWord,
+				  (priv->rf_reg_0value[eRFPath] << 16));
+		NewOffset = Offset - 15;
 	} else {
 		NewOffset = Offset;
 	}
@@ -181,15 +138,13 @@ static void _rtl92e_phy_rf_write(struct net_device *dev,
 	if (Offset == 0x0)
 		priv->rf_reg_0value[eRFPath] = Data;
 
-	if (priv->rf_chip == RF_8256) {
-		if (Offset != 0) {
-			priv->rf_reg_0value[eRFPath] &= 0xebf;
-			rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset,
-					  bMaskDWord,
-					  (priv->rf_reg_0value[eRFPath] << 16));
-		}
-		rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter4, 0x300, 0x3);
+	if (Offset != 0) {
+		priv->rf_reg_0value[eRFPath] &= 0xebf;
+		rtl92e_set_bb_reg(dev, pPhyReg->rf3wireOffset,
+				  bMaskDWord,
+				  (priv->rf_reg_0value[eRFPath] << 16));
 	}
+	rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter4, 0x300, 0x3);
 }
 
 void rtl92e_set_rf_reg(struct net_device *dev, enum rf90_radio_path eRFPath,
@@ -198,8 +153,6 @@ void rtl92e_set_rf_reg(struct net_device *dev, enum rf90_radio_path eRFPath,
 	struct r8192_priv *priv = rtllib_priv(dev);
 	u32 Original_Value, BitShift, New_Value;
 
-	if (!rtl92e_is_legal_rf_path(dev, eRFPath))
-		return;
 	if (priv->rtllib->rf_power_state != rf_on && !priv->being_init_adapter)
 		return;
 
@@ -235,8 +188,6 @@ u32 rtl92e_get_rf_reg(struct net_device *dev, enum rf90_radio_path eRFPath,
 	u32 Original_Value, Readback_Value, BitShift;
 	struct r8192_priv *priv = rtllib_priv(dev);
 
-	if (!rtl92e_is_legal_rf_path(dev, eRFPath))
-		return 0;
 	if (priv->rtllib->rf_power_state != rf_on && !priv->being_init_adapter)
 		return	0;
 	mutex_lock(&priv->rf_mutex);
@@ -307,12 +258,12 @@ void rtl92e_config_mac(struct net_device *dev)
 	struct r8192_priv *priv = rtllib_priv(dev);
 
 	if (priv->tx_pwr_data_read_from_eeprom) {
-		dwArrayLen = MACPHY_Array_PGLength;
-		pdwArray = Rtl819XMACPHY_Array_PG;
+		dwArrayLen = RTL8192E_MACPHY_ARR_PG_LEN;
+		pdwArray = RTL8192E_MACPHY_ARR_PG;
 
 	} else {
-		dwArrayLen = MACPHY_ArrayLength;
-		pdwArray = Rtl819XMACPHY_Array;
+		dwArrayLen = RTL8192E_MACPHY_ARR_LEN;
+		pdwArray = RTL8192E_MACPHY_ARR;
 	}
 	for (i = 0; i < dwArrayLen; i += 3) {
 		if (pdwArray[i] == 0x318)
@@ -330,17 +281,11 @@ static void _rtl92e_phy_config_bb(struct net_device *dev, u8 ConfigType)
 	u32 *Rtl819XPHY_REGArray_Table = NULL;
 	u32 *Rtl819XAGCTAB_Array_Table = NULL;
 	u16 AGCTAB_ArrayLen, PHY_REGArrayLen = 0;
-	struct r8192_priv *priv = rtllib_priv(dev);
 
-	AGCTAB_ArrayLen = AGCTAB_ArrayLength;
-	Rtl819XAGCTAB_Array_Table = Rtl819XAGCTAB_Array;
-	if (priv->rf_type == RF_2T4R) {
-		PHY_REGArrayLen = PHY_REGArrayLength;
-		Rtl819XPHY_REGArray_Table = Rtl819XPHY_REGArray;
-	} else if (priv->rf_type == RF_1T2R) {
-		PHY_REGArrayLen = PHY_REG_1T2RArrayLength;
-		Rtl819XPHY_REGArray_Table = Rtl819XPHY_REG_1T2RArray;
-	}
+	AGCTAB_ArrayLen = RTL8192E_AGCTAB_ARR_LEN;
+	Rtl819XAGCTAB_Array_Table = RTL8192E_AGCTAB_ARR;
+	PHY_REGArrayLen = RTL8192E_PHY_REG_1T2R_ARR_LEN;
+	Rtl819XPHY_REGArray_Table = RTL8192E_PHY_REG_1T2R_ARR;
 
 	if (ConfigType == BB_CONFIG_PHY_REG) {
 		for (i = 0; i < PHY_REGArrayLen; i += 2) {
@@ -363,89 +308,21 @@ static void _rtl92e_init_bb_rf_reg_def(struct net_device *dev)
 
 	priv->phy_reg_def[RF90_PATH_A].rfintfs = rFPGA0_XAB_RFInterfaceSW;
 	priv->phy_reg_def[RF90_PATH_B].rfintfs = rFPGA0_XAB_RFInterfaceSW;
-	priv->phy_reg_def[RF90_PATH_C].rfintfs = rFPGA0_XCD_RFInterfaceSW;
-	priv->phy_reg_def[RF90_PATH_D].rfintfs = rFPGA0_XCD_RFInterfaceSW;
-
-	priv->phy_reg_def[RF90_PATH_A].rfintfi = rFPGA0_XAB_RFInterfaceRB;
-	priv->phy_reg_def[RF90_PATH_B].rfintfi = rFPGA0_XAB_RFInterfaceRB;
-	priv->phy_reg_def[RF90_PATH_C].rfintfi = rFPGA0_XCD_RFInterfaceRB;
-	priv->phy_reg_def[RF90_PATH_D].rfintfi = rFPGA0_XCD_RFInterfaceRB;
 
 	priv->phy_reg_def[RF90_PATH_A].rfintfo = rFPGA0_XA_RFInterfaceOE;
 	priv->phy_reg_def[RF90_PATH_B].rfintfo = rFPGA0_XB_RFInterfaceOE;
-	priv->phy_reg_def[RF90_PATH_C].rfintfo = rFPGA0_XC_RFInterfaceOE;
-	priv->phy_reg_def[RF90_PATH_D].rfintfo = rFPGA0_XD_RFInterfaceOE;
 
 	priv->phy_reg_def[RF90_PATH_A].rfintfe = rFPGA0_XA_RFInterfaceOE;
 	priv->phy_reg_def[RF90_PATH_B].rfintfe = rFPGA0_XB_RFInterfaceOE;
-	priv->phy_reg_def[RF90_PATH_C].rfintfe = rFPGA0_XC_RFInterfaceOE;
-	priv->phy_reg_def[RF90_PATH_D].rfintfe = rFPGA0_XD_RFInterfaceOE;
 
 	priv->phy_reg_def[RF90_PATH_A].rf3wireOffset = rFPGA0_XA_LSSIParameter;
 	priv->phy_reg_def[RF90_PATH_B].rf3wireOffset = rFPGA0_XB_LSSIParameter;
-	priv->phy_reg_def[RF90_PATH_C].rf3wireOffset = rFPGA0_XC_LSSIParameter;
-	priv->phy_reg_def[RF90_PATH_D].rf3wireOffset = rFPGA0_XD_LSSIParameter;
-
-	priv->phy_reg_def[RF90_PATH_A].rfLSSI_Select = rFPGA0_XAB_RFParameter;
-	priv->phy_reg_def[RF90_PATH_B].rfLSSI_Select = rFPGA0_XAB_RFParameter;
-	priv->phy_reg_def[RF90_PATH_C].rfLSSI_Select = rFPGA0_XCD_RFParameter;
-	priv->phy_reg_def[RF90_PATH_D].rfLSSI_Select = rFPGA0_XCD_RFParameter;
-
-	priv->phy_reg_def[RF90_PATH_A].rfTxGainStage = rFPGA0_TxGainStage;
-	priv->phy_reg_def[RF90_PATH_B].rfTxGainStage = rFPGA0_TxGainStage;
-	priv->phy_reg_def[RF90_PATH_C].rfTxGainStage = rFPGA0_TxGainStage;
-	priv->phy_reg_def[RF90_PATH_D].rfTxGainStage = rFPGA0_TxGainStage;
-
-	priv->phy_reg_def[RF90_PATH_A].rfHSSIPara1 = rFPGA0_XA_HSSIParameter1;
-	priv->phy_reg_def[RF90_PATH_B].rfHSSIPara1 = rFPGA0_XB_HSSIParameter1;
-	priv->phy_reg_def[RF90_PATH_C].rfHSSIPara1 = rFPGA0_XC_HSSIParameter1;
-	priv->phy_reg_def[RF90_PATH_D].rfHSSIPara1 = rFPGA0_XD_HSSIParameter1;
 
 	priv->phy_reg_def[RF90_PATH_A].rfHSSIPara2 = rFPGA0_XA_HSSIParameter2;
 	priv->phy_reg_def[RF90_PATH_B].rfHSSIPara2 = rFPGA0_XB_HSSIParameter2;
-	priv->phy_reg_def[RF90_PATH_C].rfHSSIPara2 = rFPGA0_XC_HSSIParameter2;
-	priv->phy_reg_def[RF90_PATH_D].rfHSSIPara2 = rFPGA0_XD_HSSIParameter2;
-
-	priv->phy_reg_def[RF90_PATH_A].rfSwitchControl = rFPGA0_XAB_SwitchControl;
-	priv->phy_reg_def[RF90_PATH_B].rfSwitchControl = rFPGA0_XAB_SwitchControl;
-	priv->phy_reg_def[RF90_PATH_C].rfSwitchControl = rFPGA0_XCD_SwitchControl;
-	priv->phy_reg_def[RF90_PATH_D].rfSwitchControl = rFPGA0_XCD_SwitchControl;
-
-	priv->phy_reg_def[RF90_PATH_A].rfAGCControl1 = rOFDM0_XAAGCCore1;
-	priv->phy_reg_def[RF90_PATH_B].rfAGCControl1 = rOFDM0_XBAGCCore1;
-	priv->phy_reg_def[RF90_PATH_C].rfAGCControl1 = rOFDM0_XCAGCCore1;
-	priv->phy_reg_def[RF90_PATH_D].rfAGCControl1 = rOFDM0_XDAGCCore1;
-
-	priv->phy_reg_def[RF90_PATH_A].rfAGCControl2 = rOFDM0_XAAGCCore2;
-	priv->phy_reg_def[RF90_PATH_B].rfAGCControl2 = rOFDM0_XBAGCCore2;
-	priv->phy_reg_def[RF90_PATH_C].rfAGCControl2 = rOFDM0_XCAGCCore2;
-	priv->phy_reg_def[RF90_PATH_D].rfAGCControl2 = rOFDM0_XDAGCCore2;
-
-	priv->phy_reg_def[RF90_PATH_A].rfRxIQImbalance = rOFDM0_XARxIQImbalance;
-	priv->phy_reg_def[RF90_PATH_B].rfRxIQImbalance = rOFDM0_XBRxIQImbalance;
-	priv->phy_reg_def[RF90_PATH_C].rfRxIQImbalance = rOFDM0_XCRxIQImbalance;
-	priv->phy_reg_def[RF90_PATH_D].rfRxIQImbalance = rOFDM0_XDRxIQImbalance;
-
-	priv->phy_reg_def[RF90_PATH_A].rfRxAFE = rOFDM0_XARxAFE;
-	priv->phy_reg_def[RF90_PATH_B].rfRxAFE = rOFDM0_XBRxAFE;
-	priv->phy_reg_def[RF90_PATH_C].rfRxAFE = rOFDM0_XCRxAFE;
-	priv->phy_reg_def[RF90_PATH_D].rfRxAFE = rOFDM0_XDRxAFE;
-
-	priv->phy_reg_def[RF90_PATH_A].rfTxIQImbalance = rOFDM0_XATxIQImbalance;
-	priv->phy_reg_def[RF90_PATH_B].rfTxIQImbalance = rOFDM0_XBTxIQImbalance;
-	priv->phy_reg_def[RF90_PATH_C].rfTxIQImbalance = rOFDM0_XCTxIQImbalance;
-	priv->phy_reg_def[RF90_PATH_D].rfTxIQImbalance = rOFDM0_XDTxIQImbalance;
-
-	priv->phy_reg_def[RF90_PATH_A].rfTxAFE = rOFDM0_XATxAFE;
-	priv->phy_reg_def[RF90_PATH_B].rfTxAFE = rOFDM0_XBTxAFE;
-	priv->phy_reg_def[RF90_PATH_C].rfTxAFE = rOFDM0_XCTxAFE;
-	priv->phy_reg_def[RF90_PATH_D].rfTxAFE = rOFDM0_XDTxAFE;
 
 	priv->phy_reg_def[RF90_PATH_A].rfLSSIReadBack = rFPGA0_XA_LSSIReadBack;
 	priv->phy_reg_def[RF90_PATH_B].rfLSSIReadBack = rFPGA0_XB_LSSIReadBack;
-	priv->phy_reg_def[RF90_PATH_C].rfLSSIReadBack = rFPGA0_XC_LSSIReadBack;
-	priv->phy_reg_def[RF90_PATH_D].rfLSSIReadBack = rFPGA0_XD_LSSIReadBack;
-
 }
 
 bool rtl92e_check_bb_and_rf(struct net_device *dev, enum hw90_block CheckBlock,
@@ -534,12 +411,7 @@ static bool _rtl92e_bb_config_para_file(struct net_device *dev)
 	_rtl92e_phy_config_bb(dev, BB_CONFIG_AGC_TAB);
 
 	if (priv->ic_cut  > VERSION_8190_BD) {
-		if (priv->rf_type == RF_2T4R)
-			dwRegValue = priv->antenna_tx_pwr_diff[2] << 8 |
-				      priv->antenna_tx_pwr_diff[1] << 4 |
-				      priv->antenna_tx_pwr_diff[0];
-		else
-			dwRegValue = 0x0;
+		dwRegValue = 0x0;
 		rtl92e_set_bb_reg(dev, rFPGA0_TxGainStage,
 				  (bXBTxAGC|bXCTxAGC|bXDTxAGC), dwRegValue);
 
@@ -590,53 +462,10 @@ void rtl92e_set_tx_power(struct net_device *dev, u8 channel)
 	if (priv->epromtype == EEPROM_93C46) {
 		powerlevel = priv->tx_pwr_level_cck[channel - 1];
 		powerlevelOFDM24G = priv->tx_pwr_level_ofdm_24g[channel - 1];
-	} else if (priv->epromtype == EEPROM_93C56) {
-		if (priv->rf_type == RF_2T4R) {
-			priv->antenna_tx_pwr_diff[2] = 0;
-			priv->antenna_tx_pwr_diff[1] = 0;
-			priv->antenna_tx_pwr_diff[0] = 0;
-
-			rtl92e_set_bb_reg(dev, rFPGA0_TxGainStage,
-					  (bXBTxAGC | bXCTxAGC | bXDTxAGC), 0);
-		}
 	}
-	switch (priv->rf_chip) {
-	case RF_8225:
-		break;
-	case RF_8256:
-		rtl92e_set_cck_tx_power(dev, powerlevel);
-		rtl92e_set_ofdm_tx_power(dev, powerlevelOFDM24G);
-		break;
-	case RF_8258:
-		break;
-	default:
-		netdev_err(dev, "Invalid RF Chip ID.\n");
-		break;
-	}
-}
 
-bool rtl92e_config_phy(struct net_device *dev)
-{
-	struct r8192_priv *priv = rtllib_priv(dev);
-	bool rtStatus = true;
-
-	switch (priv->rf_chip) {
-	case RF_8225:
-		break;
-	case RF_8256:
-		rtStatus = rtl92e_config_rf(dev);
-		break;
-
-	case RF_8258:
-		break;
-	case RF_PSEUDO_11N:
-		break;
-
-	default:
-		netdev_err(dev, "Invalid RF Chip ID.\n");
-		break;
-	}
-	return rtStatus;
+	rtl92e_set_cck_tx_power(dev, powerlevel);
+	rtl92e_set_ofdm_tx_power(dev, powerlevelOFDM24G);
 }
 
 u8 rtl92e_config_rf_path(struct net_device *dev, enum rf90_radio_path eRFPath)
@@ -646,50 +475,26 @@ u8 rtl92e_config_rf_path(struct net_device *dev, enum rf90_radio_path eRFPath)
 
 	switch (eRFPath) {
 	case RF90_PATH_A:
-		for (i = 0; i < RadioA_ArrayLength; i += 2) {
-			if (Rtl819XRadioA_Array[i] == 0xfe) {
+		for (i = 0; i < RTL8192E_RADIO_A_ARR_LEN; i += 2) {
+			if (RTL8192E_RADIO_A_ARR[i] == 0xfe) {
 				msleep(100);
 				continue;
 			}
-			rtl92e_set_rf_reg(dev, eRFPath, Rtl819XRadioA_Array[i],
+			rtl92e_set_rf_reg(dev, eRFPath, RTL8192E_RADIO_A_ARR[i],
 					  bMask12Bits,
-					  Rtl819XRadioA_Array[i+1]);
+					  RTL8192E_RADIO_A_ARR[i + 1]);
 
 		}
 		break;
 	case RF90_PATH_B:
-		for (i = 0; i < RadioB_ArrayLength; i += 2) {
-			if (Rtl819XRadioB_Array[i] == 0xfe) {
+		for (i = 0; i < RTL8192E_RADIO_B_ARR_LEN; i += 2) {
+			if (RTL8192E_RADIO_B_ARR[i] == 0xfe) {
 				msleep(100);
 				continue;
 			}
-			rtl92e_set_rf_reg(dev, eRFPath, Rtl819XRadioB_Array[i],
+			rtl92e_set_rf_reg(dev, eRFPath, RTL8192E_RADIO_B_ARR[i],
 					  bMask12Bits,
-					  Rtl819XRadioB_Array[i+1]);
-
-		}
-		break;
-	case RF90_PATH_C:
-		for (i = 0; i < RadioC_ArrayLength; i += 2) {
-			if (Rtl819XRadioC_Array[i] == 0xfe) {
-				msleep(100);
-				continue;
-			}
-			rtl92e_set_rf_reg(dev, eRFPath, Rtl819XRadioC_Array[i],
-					  bMask12Bits,
-					  Rtl819XRadioC_Array[i+1]);
-
-		}
-		break;
-	case RF90_PATH_D:
-		for (i = 0; i < RadioD_ArrayLength; i += 2) {
-			if (Rtl819XRadioD_Array[i] == 0xfe) {
-				msleep(100);
-				continue;
-			}
-			rtl92e_set_rf_reg(dev, eRFPath, Rtl819XRadioD_Array[i],
-					  bMask12Bits,
-					  Rtl819XRadioD_Array[i+1]);
+					  RTL8192E_RADIO_B_ARR[i + 1]);
 
 		}
 		break;
@@ -707,21 +512,8 @@ static void _rtl92e_set_tx_power_level(struct net_device *dev, u8 channel)
 	u8	powerlevel = priv->tx_pwr_level_cck[channel - 1];
 	u8	powerlevelOFDM24G = priv->tx_pwr_level_ofdm_24g[channel - 1];
 
-	switch (priv->rf_chip) {
-	case RF_8225:
-		break;
-
-	case RF_8256:
-		rtl92e_set_cck_tx_power(dev, powerlevel);
-		rtl92e_set_ofdm_tx_power(dev, powerlevelOFDM24G);
-		break;
-
-	case RF_8258:
-		break;
-	default:
-		netdev_warn(dev, "%s(): Invalid RF Chip ID\n", __func__);
-		break;
-	}
+	rtl92e_set_cck_tx_power(dev, powerlevel);
+	rtl92e_set_ofdm_tx_power(dev, powerlevelOFDM24G);
 }
 
 static u8 _rtl92e_phy_set_sw_chnl_cmd_array(struct net_device *dev,
@@ -786,58 +578,25 @@ static u8 _rtl92e_phy_switch_channel_step(struct net_device *dev, u8 channel,
 						  0, 0, 0);
 
 		RfDependCmdCnt = 0;
-		switch (priv->rf_chip) {
-		case RF_8225:
-			if (!(channel >= 1 && channel <= 14)) {
-				netdev_err(dev,
-					   "Invalid channel requested for 8225: %d\n",
-					   channel);
-				return false;
-			}
-			_rtl92e_phy_set_sw_chnl_cmd_array(dev,
-							  ieee->RfDependCmd,
-							  RfDependCmdCnt++,
-							  MAX_RFDEPENDCMD_CNT,
-							  CmdID_RF_WriteReg,
-							  rZebra1_Channel,
-							  RF_CHANNEL_TABLE_ZEBRA[channel],
-							  10);
-			_rtl92e_phy_set_sw_chnl_cmd_array(dev,
-							  ieee->RfDependCmd,
-							  RfDependCmdCnt++,
-							  MAX_RFDEPENDCMD_CNT,
-							  CmdID_End, 0, 0, 0);
-			break;
 
-		case RF_8256:
-			if (!(channel >= 1 && channel <= 14)) {
-				netdev_err(dev,
-					   "Invalid channel requested for 8256: %d\n",
-					   channel);
-				return false;
-			}
-			_rtl92e_phy_set_sw_chnl_cmd_array(dev,
-							  ieee->RfDependCmd,
-							  RfDependCmdCnt++,
-							  MAX_RFDEPENDCMD_CNT,
-							  CmdID_RF_WriteReg,
-							  rZebra1_Channel,
-							  channel, 10);
-			_rtl92e_phy_set_sw_chnl_cmd_array(dev,
-							  ieee->RfDependCmd,
-							  RfDependCmdCnt++,
-							  MAX_RFDEPENDCMD_CNT,
-							  CmdID_End, 0, 0, 0);
-			break;
-
-		case RF_8258:
-			break;
-
-		default:
-			netdev_warn(dev, "Unknown RF Chip ID\n");
+		if (!(channel >= 1 && channel <= 14)) {
+			netdev_err(dev,
+				   "Invalid channel requested for 8256: %d\n",
+				   channel);
 			return false;
 		}
-
+		_rtl92e_phy_set_sw_chnl_cmd_array(dev,
+						  ieee->RfDependCmd,
+						  RfDependCmdCnt++,
+						  MAX_RFDEPENDCMD_CNT,
+						  CmdID_RF_WriteReg,
+						  rZebra1_Channel,
+						  channel, 10);
+		_rtl92e_phy_set_sw_chnl_cmd_array(dev,
+						  ieee->RfDependCmd,
+						  RfDependCmdCnt++,
+						  MAX_RFDEPENDCMD_CNT,
+						  CmdID_End, 0, 0, 0);
 
 		do {
 			switch (*stage) {
@@ -937,15 +696,6 @@ u8 rtl92e_set_channel(struct net_device *dev, u8 channel)
 
 
 	switch (priv->rtllib->mode) {
-	case WIRELESS_MODE_A:
-	case WIRELESS_MODE_N_5G:
-		if (channel <= 14) {
-			netdev_warn(dev,
-				    "Channel %d not available in 802.11a.\n",
-				    channel);
-			return false;
-		}
-		break;
 	case WIRELESS_MODE_B:
 		if (channel > 14) {
 			netdev_warn(dev,
@@ -1078,10 +828,6 @@ static void _rtl92e_set_bw_mode_work_item(struct net_device *dev)
 	struct r8192_priv *priv = rtllib_priv(dev);
 	u8 regBwOpMode;
 
-	if (priv->rf_chip == RF_PSEUDO_11N) {
-		priv->set_bw_mode_in_progress = false;
-		return;
-	}
 	if (!priv->up) {
 		netdev_err(dev, "%s(): Driver is not initialized\n", __func__);
 		return;
@@ -1147,25 +893,7 @@ static void _rtl92e_set_bw_mode_work_item(struct net_device *dev)
 
 	}
 
-	switch (priv->rf_chip) {
-	case RF_8225:
-		break;
-
-	case RF_8256:
-		rtl92e_set_bandwidth(dev, priv->current_chnl_bw);
-		break;
-
-	case RF_8258:
-		break;
-
-	case RF_PSEUDO_11N:
-		break;
-
-	default:
-		netdev_info(dev, "%s(): Unknown RFChipID: %d\n", __func__,
-			    priv->rf_chip);
-		break;
-	}
+	rtl92e_set_bandwidth(dev, priv->current_chnl_bw);
 
 	atomic_dec(&(priv->rtllib->atm_swbw));
 	priv->set_bw_mode_in_progress = false;
@@ -1291,129 +1019,88 @@ static bool _rtl92e_set_rf_power_state(struct net_device *dev,
 		return false;
 	priv->set_rf_pwr_state_in_progress = true;
 
-	switch (priv->rf_chip) {
-	case RF_8256:
-		switch (rf_power_state) {
-		case rf_on:
-			if ((priv->rtllib->rf_power_state == rf_off) &&
-			     RT_IN_PS_LEVEL(psc, RT_RF_OFF_LEVL_HALT_NIC)) {
-				bool rtstatus;
-				u32 InitilizeCount = 3;
-
-				do {
-					InitilizeCount--;
-					rtstatus = rtl92e_enable_nic(dev);
-				} while (!rtstatus && (InitilizeCount > 0));
-
-				if (!rtstatus) {
-					netdev_err(dev,
-						   "%s(): Failed to initialize Adapter.\n",
-						   __func__);
-					priv->set_rf_pwr_state_in_progress = false;
-					return false;
-				}
-
-				RT_CLEAR_PS_LEVEL(psc,
-						  RT_RF_OFF_LEVL_HALT_NIC);
-			} else {
-				rtl92e_writeb(dev, ANAPAR, 0x37);
-				mdelay(1);
-				rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter1,
-						 0x4, 0x1);
-				priv->hw_rf_off_action = 0;
-
-				rtl92e_set_bb_reg(dev, rFPGA0_XA_RFInterfaceOE,
-						  BIT4, 0x1);
-				rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter4,
-						  0x300, 0x3);
-				rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter1,
-						  0x18, 0x3);
-				rtl92e_set_bb_reg(dev, rOFDM0_TRxPathEnable,
-						  0x3, 0x3);
-				rtl92e_set_bb_reg(dev, rOFDM1_TRxPathEnable,
-						  0x3, 0x3);
-				rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter1,
-						  0x60, 0x3);
-
-			}
-
-			break;
-
-		case rf_sleep:
-			if (priv->rtllib->rf_power_state == rf_off)
-				break;
-
-
-			for (QueueID = 0, i = 0; QueueID < MAX_TX_QUEUE; ) {
-				ring = &priv->tx_ring[QueueID];
-
-				if (skb_queue_len(&ring->queue) == 0) {
-					QueueID++;
-					continue;
-				} else {
-					udelay(10);
-					i++;
-				}
-
-				if (i >= MAX_DOZE_WAITING_TIMES_9x)
-					break;
+	switch (rf_power_state) {
+	case rf_on:
+		if ((priv->rtllib->rf_power_state == rf_off) &&
+		     RT_IN_PS_LEVEL(psc, RT_RF_OFF_LEVL_HALT_NIC)) {
+			bool rtstatus;
+			u32 InitilizeCount = 3;
+
+			do {
+				InitilizeCount--;
+				rtstatus = rtl92e_enable_nic(dev);
+			} while (!rtstatus && (InitilizeCount > 0));
+			if (!rtstatus) {
+				netdev_err(dev,
+					   "%s(): Failed to initialize Adapter.\n",
+					   __func__);
+				priv->set_rf_pwr_state_in_progress = false;
+				return false;
 			}
-			rtl92e_set_rf_off(dev);
+			RT_CLEAR_PS_LEVEL(psc,
+					  RT_RF_OFF_LEVL_HALT_NIC);
+		} else {
+			rtl92e_writeb(dev, ANAPAR, 0x37);
+			mdelay(1);
+			rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter1,
+					 0x4, 0x1);
+			priv->hw_rf_off_action = 0;
+			rtl92e_set_bb_reg(dev, rFPGA0_XA_RFInterfaceOE,
+					  BIT4, 0x1);
+			rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter4,
+					  0x300, 0x3);
+			rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter1,
+					  0x18, 0x3);
+			rtl92e_set_bb_reg(dev, rOFDM0_TRxPathEnable,
+					  0x3, 0x3);
+			rtl92e_set_bb_reg(dev, rOFDM1_TRxPathEnable,
+					  0x3, 0x3);
+			rtl92e_set_bb_reg(dev, rFPGA0_AnalogParameter1,
+					  0x60, 0x3);
+		}
+		break;
+	case rf_sleep:
+		if (priv->rtllib->rf_power_state == rf_off)
 			break;
-
-		case rf_off:
-			for (QueueID = 0, i = 0; QueueID < MAX_TX_QUEUE; ) {
-				ring = &priv->tx_ring[QueueID];
-
-				if (skb_queue_len(&ring->queue) == 0) {
-					QueueID++;
-					continue;
-				} else {
-					udelay(10);
-					i++;
-				}
-
-				if (i >= MAX_DOZE_WAITING_TIMES_9x)
-					break;
+		for (QueueID = 0, i = 0; QueueID < MAX_TX_QUEUE; ) {
+			ring = &priv->tx_ring[QueueID];
+			if (skb_queue_len(&ring->queue) == 0) {
+				QueueID++;
+				continue;
+			} else {
+				udelay(10);
+				i++;
 			}
-
-			if (psc->RegRfPsLevel & RT_RF_OFF_LEVL_HALT_NIC &&
-			    !RT_IN_PS_LEVEL(psc, RT_RF_OFF_LEVL_HALT_NIC)) {
-				rtl92e_disable_nic(dev);
-				RT_SET_PS_LEVEL(psc, RT_RF_OFF_LEVL_HALT_NIC);
-			} else if (!(psc->RegRfPsLevel &
-				   RT_RF_OFF_LEVL_HALT_NIC)) {
-				rtl92e_set_rf_off(dev);
+			if (i >= MAX_DOZE_WAITING_TIMES_9x)
+				break;
+		}
+		rtl92e_set_rf_off(dev);
+		break;
+	case rf_off:
+		for (QueueID = 0, i = 0; QueueID < MAX_TX_QUEUE; ) {
+			ring = &priv->tx_ring[QueueID];
+			if (skb_queue_len(&ring->queue) == 0) {
+				QueueID++;
+				continue;
+			} else {
+				udelay(10);
+				i++;
 			}
-
-			break;
-
-		default:
-			bResult = false;
-			netdev_warn(dev,
-				    "%s(): Unknown state requested: 0x%X.\n",
-				    __func__, rf_power_state);
-			break;
+			if (i >= MAX_DOZE_WAITING_TIMES_9x)
+				break;
 		}
-
+		rtl92e_set_rf_off(dev);
 		break;
-
 	default:
-		netdev_warn(dev, "%s(): Unknown RF type\n", __func__);
+		bResult = false;
+		netdev_warn(dev,
+			    "%s(): Unknown state requested: 0x%X.\n",
+			    __func__, rf_power_state);
 		break;
 	}
 
 	if (bResult) {
 		priv->rtllib->rf_power_state = rf_power_state;
-
-		switch (priv->rf_chip) {
-		case RF_8256:
-			break;
-
-		default:
-			netdev_warn(dev, "%s(): Unknown RF type\n", __func__);
-			break;
-		}
 	}
 
 	priv->set_rf_pwr_state_in_progress = false;
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.h b/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.h
index 75629f5df954..6c4c33ded6a9 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.h
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.h
@@ -9,28 +9,6 @@
 
 #define MAX_DOZE_WAITING_TIMES_9x 64
 
-#define AGCTAB_ArrayLength			AGCTAB_ArrayLengthPciE
-#define MACPHY_ArrayLength			MACPHY_ArrayLengthPciE
-#define RadioA_ArrayLength			RadioA_ArrayLengthPciE
-#define RadioB_ArrayLength			RadioB_ArrayLengthPciE
-#define MACPHY_Array_PGLength			MACPHY_Array_PGLengthPciE
-#define RadioC_ArrayLength			RadioC_ArrayLengthPciE
-#define RadioD_ArrayLength			RadioD_ArrayLengthPciE
-#define PHY_REGArrayLength			PHY_REGArrayLengthPciE
-#define PHY_REG_1T2RArrayLength			PHY_REG_1T2RArrayLengthPciE
-
-#define Rtl819XMACPHY_Array_PG			Rtl8192PciEMACPHY_Array_PG
-#define Rtl819XMACPHY_Array			Rtl8192PciEMACPHY_Array
-#define Rtl819XRadioA_Array			Rtl8192PciERadioA_Array
-#define Rtl819XRadioB_Array			Rtl8192PciERadioB_Array
-#define Rtl819XRadioC_Array			Rtl8192PciERadioC_Array
-#define Rtl819XRadioD_Array			Rtl8192PciERadioD_Array
-#define Rtl819XAGCTAB_Array			Rtl8192PciEAGCTAB_Array
-#define Rtl819XPHY_REGArray			Rtl8192PciEPHY_REGArray
-#define Rtl819XPHY_REG_1T2RArray		Rtl8192PciEPHY_REG_1T2RArray
-
-extern u32 rtl819XAGCTAB_Array[];
-
 enum hw90_block {
 	HW90_BLOCK_MAC = 0,
 	HW90_BLOCK_PHY0 = 1,
@@ -47,15 +25,6 @@ enum rf90_radio_path {
 	RF90_PATH_MAX
 };
 
-#define bMaskByte0                0xff
-#define bMaskByte1                0xff00
-#define bMaskByte2                0xff0000
-#define bMaskByte3                0xff000000
-#define bMaskHWord                0xffff0000
-#define bMaskLWord                0x0000ffff
-#define bMaskDWord                0xffffffff
-
-u8 rtl92e_is_legal_rf_path(struct net_device *dev, u32 eRFPath);
 void rtl92e_set_bb_reg(struct net_device *dev, u32 dwRegAddr,
 		       u32 dwBitMask, u32 dwData);
 u32 rtl92e_get_bb_reg(struct net_device *dev, u32 dwRegAddr, u32 dwBitMask);
@@ -70,7 +39,6 @@ bool rtl92e_check_bb_and_rf(struct net_device *dev,
 bool rtl92e_config_bb(struct net_device *dev);
 void rtl92e_get_tx_power(struct net_device *dev);
 void rtl92e_set_tx_power(struct net_device *dev, u8 channel);
-bool rtl92e_config_phy(struct net_device *dev);
 u8 rtl92e_config_rf_path(struct net_device *dev, enum rf90_radio_path eRFPath);
 
 u8 rtl92e_set_channel(struct net_device *dev, u8 channel);
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_phyreg.h b/drivers/staging/rtl8192e/rtl8192e/r8192E_phyreg.h
index f846f109ed98..e1b30fbdf8cc 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_phyreg.h
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_phyreg.h
@@ -48,41 +48,22 @@
 #define rFPGA0_TxGainStage		0x80c
 #define rFPGA0_RFTiming1		0x810
 #define rFPGA0_RFTiming2		0x814
-#define rFPGA0_XA_HSSIParameter1	0x820
 #define rFPGA0_XA_HSSIParameter2	0x824
-#define rFPGA0_XB_HSSIParameter1	0x828
 #define rFPGA0_XB_HSSIParameter2	0x82c
-#define rFPGA0_XC_HSSIParameter1	0x830
-#define rFPGA0_XC_HSSIParameter2	0x834
-#define rFPGA0_XD_HSSIParameter1	0x838
-#define rFPGA0_XD_HSSIParameter2	0x83c
 #define rFPGA0_XA_LSSIParameter		0x840
 #define rFPGA0_XB_LSSIParameter		0x844
-#define rFPGA0_XC_LSSIParameter		0x848
-#define rFPGA0_XD_LSSIParameter		0x84c
 #define rFPGA0_RFWakeUpParameter	0x850
 #define rFPGA0_RFSleepUpParameter	0x854
-#define rFPGA0_XAB_SwitchControl	0x858
-#define rFPGA0_XCD_SwitchControl	0x85c
 #define rFPGA0_XA_RFInterfaceOE		0x860
 #define rFPGA0_XB_RFInterfaceOE		0x864
-#define rFPGA0_XC_RFInterfaceOE		0x868
-#define rFPGA0_XD_RFInterfaceOE		0x86c
 #define rFPGA0_XAB_RFInterfaceSW	0x870
-#define rFPGA0_XCD_RFInterfaceSW	0x874
-#define rFPGA0_XAB_RFParameter		0x878
-#define rFPGA0_XCD_RFParameter		0x87c
 #define rFPGA0_AnalogParameter1		0x880
 #define rFPGA0_AnalogParameter2		0x884
 #define rFPGA0_AnalogParameter3		0x888
 #define rFPGA0_AnalogParameter4		0x88c
 #define rFPGA0_XA_LSSIReadBack		0x8a0
 #define rFPGA0_XB_LSSIReadBack		0x8a4
-#define rFPGA0_XC_LSSIReadBack		0x8a8
-#define rFPGA0_XD_LSSIReadBack		0x8ac
 #define rFPGA0_PSDReport		0x8b4
-#define rFPGA0_XAB_RFInterfaceRB	0x8e0
-#define rFPGA0_XCD_RFInterfaceRB	0x8e4
 
 /* Page 9 - RF mode & OFDM TxSC */
 #define rFPGA1_RFMOD			0x900
@@ -113,15 +94,6 @@
 #define rOFDM0_TRxPathEnable		0xc04
 #define rOFDM0_TRMuxPar			0xc08
 #define rOFDM0_TRSWIsolation		0xc0c
-/* RxIQ DC offset, Rx digital filter, DC notch filter */
-#define rOFDM0_XARxAFE			0xc10
-#define rOFDM0_XARxIQImbalance		0xc14 /* RxIQ imbalance matrix */
-#define rOFDM0_XBRxAFE			0xc18
-#define rOFDM0_XBRxIQImbalance		0xc1c
-#define rOFDM0_XCRxAFE			0xc20
-#define rOFDM0_XCRxIQImbalance		0xc24
-#define rOFDM0_XDRxAFE			0xc28
-#define rOFDM0_XDRxIQImbalance		0xc2c
 #define rOFDM0_RxDetector1		0xc30 /* PD, BW & SBD */
 #define rOFDM0_RxDetector2		0xc34 /* SBD */
 #define rOFDM0_RxDetector3		0xc38 /* Frame Sync */
@@ -132,25 +104,16 @@
 #define rOFDM0_CCADropThreshold		0xc48
 #define rOFDM0_ECCAThreshold		0xc4c /* Energy CCA */
 #define rOFDM0_XAAGCCore1		0xc50
-#define rOFDM0_XAAGCCore2		0xc54
 #define rOFDM0_XBAGCCore1		0xc58
-#define rOFDM0_XBAGCCore2		0xc5c
 #define rOFDM0_XCAGCCore1		0xc60
-#define rOFDM0_XCAGCCore2		0xc64
 #define rOFDM0_XDAGCCore1		0xc68
-#define rOFDM0_XDAGCCore2		0xc6c
 #define rOFDM0_AGCParameter1		0xc70
 #define rOFDM0_AGCParameter2		0xc74
 #define rOFDM0_AGCRSSITable		0xc78
 #define rOFDM0_HTSTFAGC			0xc7c
 #define rOFDM0_XATxIQImbalance		0xc80
 #define rOFDM0_XATxAFE			0xc84
-#define rOFDM0_XBTxIQImbalance		0xc88
-#define rOFDM0_XBTxAFE			0xc8c
 #define rOFDM0_XCTxIQImbalance		0xc90
-#define rOFDM0_XCTxAFE			0xc94
-#define rOFDM0_XDTxIQImbalance		0xc98
-#define rOFDM0_XDTxAFE			0xc9c
 #define rOFDM0_RxHPParameter		0xce0
 #define rOFDM0_TxPseudoNoiseWgt		0xce4
 #define rOFDM0_FrameSync		0xcf0
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
index 104b16cfa979..27040d1e3230 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
@@ -25,32 +25,10 @@
 int hwwep = 1;
 static char *ifname = "wlan%d";
 
-static const struct rtl819x_ops rtl819xp_ops = {
-	.nic_type			= NIC_8192E,
-	.get_eeprom_size		= rtl92e_get_eeprom_size,
-	.init_adapter_variable		= rtl92e_init_variables,
-	.initialize_adapter		= rtl92e_start_adapter,
-	.link_change			= rtl92e_link_change,
-	.tx_fill_descriptor		= rtl92e_fill_tx_desc,
-	.tx_fill_cmd_descriptor		= rtl92e_fill_tx_cmd_desc,
-	.rx_query_status_descriptor	= rtl92e_get_rx_stats,
-	.rx_command_packet_handler = NULL,
-	.stop_adapter			= rtl92e_stop_adapter,
-	.update_ratr_table		= rtl92e_update_ratr_table,
-	.irq_enable			= rtl92e_enable_irq,
-	.irq_disable			= rtl92e_disable_irq,
-	.irq_clear			= rtl92e_clear_irq,
-	.rx_enable			= rtl92e_enable_rx,
-	.tx_enable			= rtl92e_enable_tx,
-	.interrupt_recognized		= rtl92e_ack_irq,
-	.tx_check_stuck_handler	= rtl92e_is_tx_stuck,
-	.rx_check_stuck_handler	= rtl92e_is_rx_stuck,
-};
-
 static struct pci_device_id rtl8192_pci_id_tbl[] = {
-	{RTL_PCI_DEVICE(0x10ec, 0x8192, rtl819xp_ops)},
-	{RTL_PCI_DEVICE(0x07aa, 0x0044, rtl819xp_ops)},
-	{RTL_PCI_DEVICE(0x07aa, 0x0047, rtl819xp_ops)},
+	{PCI_DEVICE(0x10ec, 0x8192)},
+	{PCI_DEVICE(0x07aa, 0x0044)},
+	{PCI_DEVICE(0x07aa, 0x0047)},
 	{}
 };
 
@@ -255,14 +233,14 @@ void rtl92e_irq_enable(struct net_device *dev)
 
 	priv->irq_enabled = 1;
 
-	priv->ops->irq_enable(dev);
+	rtl92e_enable_irq(dev);
 }
 
 void rtl92e_irq_disable(struct net_device *dev)
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
 
-	priv->ops->irq_disable(dev);
+	rtl92e_disable_irq(dev);
 
 	priv->irq_enabled = 0;
 }
@@ -271,9 +249,6 @@ static void _rtl92e_set_chan(struct net_device *dev, short ch)
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
 
-	if (priv->chan_forced)
-		return;
-
 	priv->chan = ch;
 
 	if (priv->rf_set_chan)
@@ -333,8 +308,7 @@ static const struct rtllib_qos_parameters def_qos_parameters = {
 
 static void _rtl92e_update_beacon(void *data)
 {
-	struct r8192_priv *priv = container_of_work_rsl(data, struct r8192_priv,
-				  update_beacon_wq.work);
+	struct r8192_priv *priv = container_of(data, struct r8192_priv, update_beacon_wq.work);
 	struct net_device *dev = priv->rtllib->dev;
 	struct rtllib_device *ieee = priv->rtllib;
 	struct rtllib_network *net = &ieee->current_network;
@@ -348,8 +322,7 @@ static void _rtl92e_update_beacon(void *data)
 
 static void _rtl92e_qos_activate(void *data)
 {
-	struct r8192_priv *priv = container_of_work_rsl(data, struct r8192_priv,
-				  qos_activate);
+	struct r8192_priv *priv = container_of(data, struct r8192_priv, qos_activate);
 	struct net_device *dev = priv->rtllib->dev;
 	int i;
 
@@ -499,7 +472,7 @@ static void _rtl92e_prepare_beacon(struct tasklet_struct *t)
 	skb_push(pnewskb, priv->rtllib->tx_headroom);
 
 	pdesc = &ring->desc[0];
-	priv->ops->tx_fill_descriptor(dev, pdesc, tcb_desc, pnewskb);
+	rtl92e_fill_tx_desc(dev, pdesc, tcb_desc, pnewskb);
 	__skb_queue_tail(&ring->queue, pnewskb);
 	pdesc->OWN = 1;
 }
@@ -605,8 +578,7 @@ static void _rtl92e_refresh_support_rate(struct r8192_priv *priv)
 {
 	struct rtllib_device *ieee = priv->rtllib;
 
-	if (ieee->mode == WIRELESS_MODE_N_24G ||
-	    ieee->mode == WIRELESS_MODE_N_5G) {
+	if (ieee->mode == WIRELESS_MODE_N_24G) {
 		memcpy(ieee->reg_dot11ht_oper_rate_set,
 		       ieee->reg_ht_supp_rate_set, 16);
 		memcpy(ieee->reg_dot11tx_ht_oper_rate_set,
@@ -617,52 +589,13 @@ static void _rtl92e_refresh_support_rate(struct r8192_priv *priv)
 	}
 }
 
-static u8 _rtl92e_get_supported_wireless_mode(struct net_device *dev)
-{
-	struct r8192_priv *priv = rtllib_priv(dev);
-	u8 ret = 0;
-
-	switch (priv->rf_chip) {
-	case RF_8225:
-	case RF_8256:
-	case RF_6052:
-	case RF_PSEUDO_11N:
-		ret = (WIRELESS_MODE_N_24G | WIRELESS_MODE_G | WIRELESS_MODE_B);
-		break;
-	case RF_8258:
-		ret = (WIRELESS_MODE_A | WIRELESS_MODE_N_5G);
-		break;
-	default:
-		ret = WIRELESS_MODE_B;
-		break;
-	}
-	return ret;
-}
-
 void rtl92e_set_wireless_mode(struct net_device *dev, u8 wireless_mode)
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
-	u8 support_mode = _rtl92e_get_supported_wireless_mode(dev);
-
-	if ((wireless_mode == WIRELESS_MODE_AUTO) ||
-	    ((wireless_mode & support_mode) == 0)) {
-		if (support_mode & WIRELESS_MODE_N_24G) {
-			wireless_mode = WIRELESS_MODE_N_24G;
-		} else if (support_mode & WIRELESS_MODE_N_5G) {
-			wireless_mode = WIRELESS_MODE_N_5G;
-		} else if ((support_mode & WIRELESS_MODE_A)) {
-			wireless_mode = WIRELESS_MODE_A;
-		} else if ((support_mode & WIRELESS_MODE_G)) {
-			wireless_mode = WIRELESS_MODE_G;
-		} else if ((support_mode & WIRELESS_MODE_B)) {
-			wireless_mode = WIRELESS_MODE_B;
-		} else {
-			netdev_info(dev,
-				    "%s(): Unsupported mode requested. Fallback to 802.11b\n",
-				    __func__);
-			wireless_mode = WIRELESS_MODE_B;
-		}
-	}
+	u8 support_mode = (WIRELESS_MODE_N_24G | WIRELESS_MODE_G | WIRELESS_MODE_B);
+
+	if ((wireless_mode == WIRELESS_MODE_AUTO) || ((wireless_mode & support_mode) == 0))
+		wireless_mode = WIRELESS_MODE_N_24G;
 
 	if ((wireless_mode & (WIRELESS_MODE_B | WIRELESS_MODE_G)) ==
 	    (WIRELESS_MODE_G | WIRELESS_MODE_B))
@@ -670,12 +603,11 @@ void rtl92e_set_wireless_mode(struct net_device *dev, u8 wireless_mode)
 
 	priv->rtllib->mode = wireless_mode;
 
-	if ((wireless_mode == WIRELESS_MODE_N_24G) ||
-	    (wireless_mode == WIRELESS_MODE_N_5G)) {
+	if (wireless_mode == WIRELESS_MODE_N_24G)
 		priv->rtllib->ht_info->enable_ht = 1;
-	} else {
+	else
 		priv->rtllib->ht_info->enable_ht = 0;
-	}
+
 	_rtl92e_refresh_support_rate(priv);
 }
 
@@ -692,7 +624,7 @@ static int _rtl92e_sta_up(struct net_device *dev, bool is_silent_reset)
 	priv->rtllib->ieee_up = 1;
 
 	priv->up_first_time = 0;
-	init_status = priv->ops->initialize_adapter(dev);
+	init_status = rtl92e_start_adapter(dev);
 	if (!init_status) {
 		netdev_err(dev, "%s(): Initialization failed!\n", __func__);
 		return -1;
@@ -713,6 +645,7 @@ static int _rtl92e_sta_up(struct net_device *dev, bool is_silent_reset)
 	else
 		netif_wake_queue(dev);
 
+	priv->bfirst_after_down = false;
 	return 0;
 }
 
@@ -725,8 +658,7 @@ static int _rtl92e_sta_down(struct net_device *dev, bool shutdownrf)
 	if (priv->up == 0)
 		return -1;
 
-	if (priv->rtllib->rtllib_ips_leave)
-		priv->rtllib->rtllib_ips_leave(dev);
+	priv->rtllib->rtllib_ips_leave(dev);
 
 	if (priv->rtllib->state == RTLLIB_LINKED)
 		rtl92e_leisure_ps_leave(dev);
@@ -762,7 +694,7 @@ static int _rtl92e_sta_down(struct net_device *dev, bool shutdownrf)
 	}
 	priv->rf_change_in_progress = true;
 	spin_unlock_irqrestore(&priv->rf_ps_lock, flags);
-	priv->ops->stop_adapter(dev, false);
+	rtl92e_stop_adapter(dev, false);
 	spin_lock_irqsave(&priv->rf_ps_lock, flags);
 	priv->rf_change_in_progress = false;
 	spin_unlock_irqrestore(&priv->rf_ps_lock, flags);
@@ -779,7 +711,7 @@ static void _rtl92e_init_priv_handler(struct net_device *dev)
 
 	priv->rtllib->softmac_hard_start_xmit	= _rtl92e_hard_start_xmit;
 	priv->rtllib->set_chan			= _rtl92e_set_chan;
-	priv->rtllib->link_change		= priv->ops->link_change;
+	priv->rtllib->link_change		= rtl92e_link_change;
 	priv->rtllib->softmac_data_hard_start_xmit = _rtl92e_hard_data_xmit;
 	priv->rtllib->check_nic_enough_desc	= _rtl92e_check_nic_enough_desc;
 	priv->rtllib->handle_assoc_response	= _rtl92e_handle_assoc_response;
@@ -854,7 +786,6 @@ static void _rtl92e_init_priv_variable(struct net_device *dev)
 	priv->rtllib->short_slot = 1;
 	priv->promisc = (dev->flags & IFF_PROMISC) ? 1 : 0;
 	priv->bcck_in_ch14 = false;
-	priv->bfsync_processing  = false;
 	priv->cck_present_attn = 0;
 	priv->rfa_txpowertrackingindex = 0;
 	priv->rfc_txpowertrackingindex = 0;
@@ -913,22 +844,15 @@ static void _rtl92e_init_priv_task(struct net_device *dev)
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
 
-	INIT_WORK_RSL(&priv->reset_wq, (void *)_rtl92e_restart, dev);
-	INIT_WORK_RSL(&priv->rtllib->ips_leave_wq, (void *)rtl92e_ips_leave_wq,
-		      dev);
-	INIT_DELAYED_WORK_RSL(&priv->watch_dog_wq,
-			      (void *)_rtl92e_watchdog_wq_cb, dev);
-	INIT_DELAYED_WORK_RSL(&priv->txpower_tracking_wq,
-			      (void *)rtl92e_dm_txpower_tracking_wq, dev);
-	INIT_DELAYED_WORK_RSL(&priv->rfpath_check_wq,
-			      (void *)rtl92e_dm_rf_pathcheck_wq, dev);
-	INIT_DELAYED_WORK_RSL(&priv->update_beacon_wq,
-			      (void *)_rtl92e_update_beacon, dev);
-	INIT_WORK_RSL(&priv->qos_activate, (void *)_rtl92e_qos_activate, dev);
-	INIT_DELAYED_WORK_RSL(&priv->rtllib->hw_wakeup_wq,
-			      (void *)rtl92e_hw_wakeup_wq, dev);
-	INIT_DELAYED_WORK_RSL(&priv->rtllib->hw_sleep_wq,
-			      (void *)rtl92e_hw_sleep_wq, dev);
+	INIT_WORK(&priv->reset_wq, (void *)_rtl92e_restart);
+	INIT_WORK(&priv->rtllib->ips_leave_wq, (void *)rtl92e_ips_leave_wq);
+	INIT_DELAYED_WORK(&priv->watch_dog_wq, (void *)_rtl92e_watchdog_wq_cb);
+	INIT_DELAYED_WORK(&priv->txpower_tracking_wq, (void *)rtl92e_dm_txpower_tracking_wq);
+	INIT_DELAYED_WORK(&priv->rfpath_check_wq, (void *)rtl92e_dm_rf_pathcheck_wq);
+	INIT_DELAYED_WORK(&priv->update_beacon_wq, (void *)_rtl92e_update_beacon);
+	INIT_WORK(&priv->qos_activate, (void *)_rtl92e_qos_activate);
+	INIT_DELAYED_WORK(&priv->rtllib->hw_wakeup_wq, (void *)rtl92e_hw_wakeup_wq);
+	INIT_DELAYED_WORK(&priv->rtllib->hw_sleep_wq, (void *)rtl92e_hw_sleep_wq);
 	tasklet_setup(&priv->irq_rx_tasklet, _rtl92e_irq_rx_tasklet);
 	tasklet_setup(&priv->irq_tx_tasklet, _rtl92e_irq_tx_tasklet);
 	tasklet_setup(&priv->irq_prepare_beacon_tasklet,
@@ -941,13 +865,6 @@ static short _rtl92e_get_channel_map(struct net_device *dev)
 
 	struct r8192_priv *priv = rtllib_priv(dev);
 
-	if ((priv->rf_chip != RF_8225) && (priv->rf_chip != RF_8256) &&
-						(priv->rf_chip != RF_6052)) {
-		netdev_err(dev, "%s: unknown rf chip, can't set channel map\n",
-			   __func__);
-		return -1;
-	}
-
 	if (priv->chnl_plan >= COUNTRY_CODE_MAX) {
 		netdev_info(dev,
 			    "rtl819x_init:Error channel plan! Set to default.\n");
@@ -974,8 +891,8 @@ static short _rtl92e_init(struct net_device *dev)
 	_rtl92e_init_priv_variable(dev);
 	_rtl92e_init_priv_lock(priv);
 	_rtl92e_init_priv_task(dev);
-	priv->ops->get_eeprom_size(dev);
-	priv->ops->init_adapter_variable(dev);
+	rtl92e_get_eeprom_size(dev);
+	rtl92e_init_variables(dev);
 	_rtl92e_get_channel_map(dev);
 
 	rtl92e_dm_init(dev);
@@ -1068,7 +985,7 @@ static enum reset_type _rtl92e_tx_check_stuck(struct net_device *dev)
 	spin_unlock_irqrestore(&priv->irq_th_lock, flags);
 
 	if (bCheckFwTxCnt) {
-		if (priv->ops->tx_check_stuck_handler(dev))
+		if (rtl92e_is_tx_stuck(dev))
 			return RESET_TYPE_SILENT;
 	}
 
@@ -1077,9 +994,7 @@ static enum reset_type _rtl92e_tx_check_stuck(struct net_device *dev)
 
 static enum reset_type _rtl92e_rx_check_stuck(struct net_device *dev)
 {
-	struct r8192_priv *priv = rtllib_priv(dev);
-
-	if (priv->ops->rx_check_stuck_handler(dev))
+	if (rtl92e_is_rx_stuck(dev))
 		return RESET_TYPE_SILENT;
 
 	return RESET_TYPE_NORESET;
@@ -1410,16 +1325,14 @@ static void _rtl92e_watchdog_timer_cb(struct timer_list *t)
  ****************************************************************************/
 void rtl92e_rx_enable(struct net_device *dev)
 {
-	struct r8192_priv *priv = rtllib_priv(dev);
-
-	priv->ops->rx_enable(dev);
+	rtl92e_enable_rx(dev);
 }
 
 void rtl92e_tx_enable(struct net_device *dev)
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
 
-	priv->ops->tx_enable(dev);
+	rtl92e_enable_tx(dev);
 
 	rtllib_reset_queue(priv->rtllib);
 }
@@ -1581,7 +1494,7 @@ static void _rtl92e_tx_cmd(struct net_device *dev, struct sk_buff *skb)
 
 	tcb_desc = (struct cb_desc *)(skb->cb + MAX_DEV_ADDR_SIZE);
 
-	priv->ops->tx_fill_cmd_descriptor(dev, entry, tcb_desc, skb);
+	rtl92e_fill_tx_cmd_desc(dev, entry, tcb_desc, skb);
 
 	__skb_queue_tail(&ring->queue, skb);
 	spin_unlock_irqrestore(&priv->irq_th_lock, flags);
@@ -1640,7 +1553,7 @@ static short _rtl92e_tx(struct net_device *dev, struct sk_buff *skb)
 		if (priv->rtllib->LedControlHandler)
 			priv->rtllib->LedControlHandler(dev, LED_CTL_TX);
 	}
-	priv->ops->tx_fill_descriptor(dev, pdesc, tcb_desc, skb);
+	rtl92e_fill_tx_desc(dev, pdesc, tcb_desc, skb);
 	__skb_queue_tail(&ring->queue, skb);
 	pdesc->OWN = 1;
 	spin_unlock_irqrestore(&priv->irq_th_lock, flags);
@@ -1680,7 +1593,7 @@ static short _rtl92e_alloc_rx_ring(struct net_device *dev)
 			priv->rx_buf[rx_queue_idx][i] = skb;
 			mapping = (dma_addr_t *)skb->cb;
 			*mapping = dma_map_single(&priv->pdev->dev,
-						  skb_tail_pointer_rsl(skb),
+						  skb_tail_pointer(skb),
 						  priv->rxbuffersize, DMA_FROM_DEVICE);
 			if (dma_mapping_error(&priv->pdev->dev, *mapping)) {
 				dev_kfree_skb_any(skb);
@@ -1890,8 +1803,7 @@ static void _rtl92e_rx_normal(struct net_device *dev)
 
 		if (pdesc->OWN)
 			return;
-		if (!priv->ops->rx_query_status_descriptor(dev, &stats,
-		pdesc, skb))
+		if (!rtl92e_get_rx_stats(dev, &stats, pdesc, skb))
 			goto done;
 		new_skb = dev_alloc_skb(priv->rxbuffersize);
 		/* if allocation of new skb failed - drop current packet
@@ -1937,7 +1849,7 @@ static void _rtl92e_rx_normal(struct net_device *dev)
 		priv->rx_buf[rx_queue_idx][priv->rx_idx[rx_queue_idx]] =
 								 skb;
 		*((dma_addr_t *)skb->cb) = dma_map_single(&priv->pdev->dev,
-							  skb_tail_pointer_rsl(skb),
+							  skb_tail_pointer(skb),
 							  priv->rxbuffersize, DMA_FROM_DEVICE);
 		if (dma_mapping_error(&priv->pdev->dev, *((dma_addr_t *)skb->cb))) {
 			dev_kfree_skb_any(skb);
@@ -2062,14 +1974,13 @@ void rtl92e_commit(struct net_device *dev)
 		return;
 	rtllib_softmac_stop_protocol(priv->rtllib, 0, true);
 	rtl92e_irq_disable(dev);
-	priv->ops->stop_adapter(dev, true);
+	rtl92e_stop_adapter(dev, true);
 	_rtl92e_up(dev, false);
 }
 
 static void _rtl92e_restart(void *data)
 {
-	struct r8192_priv *priv = container_of_work_rsl(data, struct r8192_priv,
-				  reset_wq);
+	struct r8192_priv *priv = container_of(data, struct r8192_priv, reset_wq);
 	struct net_device *dev = priv->rtllib->dev;
 
 	mutex_lock(&priv->wx_mutex);
@@ -2118,7 +2029,7 @@ static irqreturn_t _rtl92e_irq(int irq, void *netdev)
 
 	spin_lock_irqsave(&priv->irq_th_lock, flags);
 
-	priv->ops->interrupt_recognized(dev, &inta, &intb);
+	rtl92e_ack_irq(dev, &inta, &intb);
 
 	if (!inta) {
 		spin_unlock_irqrestore(&priv->irq_th_lock, flags);
@@ -2214,10 +2125,8 @@ static int _rtl92e_pci_probe(struct pci_dev *pdev,
 	unsigned long ioaddr = 0;
 	struct net_device *dev = NULL;
 	struct r8192_priv *priv = NULL;
-	struct rtl819x_ops *ops = (struct rtl819x_ops *)(id->driver_data);
 	unsigned long pmem_start, pmem_len, pmem_flags;
 	int err = -ENOMEM;
-	u8 revision_id;
 
 	if (pci_enable_device(pdev)) {
 		dev_err(&pdev->dev, "Failed to enable PCI device");
@@ -2276,13 +2185,6 @@ static int _rtl92e_pci_probe(struct pci_dev *pdev,
 	dev->mem_start = ioaddr;
 	dev->mem_end = ioaddr + pci_resource_len(pdev, 0);
 
-	pci_read_config_byte(pdev, 0x08, &revision_id);
-	/* If the revisionid is 0x10, the device uses rtl8192se. */
-	if (pdev->device == 0x8192 && revision_id == 0x10)
-		goto err_unmap;
-
-	priv->ops = ops;
-
 	if (!rtl92e_check_adapter(pdev, dev))
 		goto err_unmap;
 
@@ -2383,7 +2285,7 @@ bool rtl92e_enable_nic(struct net_device *dev)
 		return false;
 	}
 
-	init_status = priv->ops->initialize_adapter(dev);
+	init_status = rtl92e_start_adapter(dev);
 	if (!init_status) {
 		netdev_warn(dev, "%s(): Initialization failed!\n", __func__);
 		priv->bdisable_nic = false;
@@ -2396,22 +2298,6 @@ bool rtl92e_enable_nic(struct net_device *dev)
 	return init_status;
 }
 
-bool rtl92e_disable_nic(struct net_device *dev)
-{
-	struct r8192_priv *priv = rtllib_priv(dev);
-	u8 tmp_state = 0;
-
-	priv->bdisable_nic = true;
-	tmp_state = priv->rtllib->state;
-	rtllib_softmac_stop_protocol(priv->rtllib, 0, false);
-	priv->rtllib->state = tmp_state;
-	_rtl92e_cancel_deferred_work(priv);
-	rtl92e_irq_disable(dev);
-
-	priv->ops->stop_adapter(dev, false);
-	return true;
-}
-
 module_pci_driver(rtl8192_pci_driver);
 
 void rtl92e_check_rfctrl_gpio_timer(struct timer_list *t)
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_core.h b/drivers/staging/rtl8192e/rtl8192e/rtl_core.h
index fd96eef90c7f..285dac32c074 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_core.h
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_core.h
@@ -52,22 +52,12 @@
 #define DRV_AUTHOR  "<wlanfae@realtek.com>"
 #define DRV_VERSION  "0014.0401.2010"
 
-#define IS_HARDWARE_TYPE_8192SE(_priv)		\
-	(((struct r8192_priv *)rtllib_priv(dev))->card_8192 == NIC_8192SE)
-
-#define RTL_PCI_DEVICE(vend, dev, cfg) \
-	.vendor = (vend), .device = (dev), \
-	.subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, \
-	.driver_data = (kernel_ulong_t)&(cfg)
-
 #define TOTAL_CAM_ENTRY		32
 #define CAM_CONTENT_COUNT	8
 
 #define HAL_HW_PCI_REVISION_ID_8192PCIE		0x01
 #define HAL_HW_PCI_REVISION_ID_8192SE	0x10
 
-#define RTL819X_DEFAULT_RF_TYPE		RF_1T2R
-
 #define RTLLIB_WATCH_DOG_TIME		2000
 
 #define MAX_DEV_ADDR_SIZE		8  /*support till 64 bit bus width OS*/
@@ -134,24 +124,10 @@ enum dcmg_txcmd_op {
 	TXCMD_XXXX_CTRL,
 };
 
-enum rt_rf_type_819xu {
-	RF_TYPE_MIN = 0,
-	RF_8225,
-	RF_8256,
-	RF_8258,
-	RF_6052 = 4,
-	RF_PSEUDO_11N = 5,
-};
-
 enum rt_customer_id {
 	RT_CID_DEFAULT	  = 0,
-	RT_CID_819x_CAMEO       = 6,
-	RT_CID_819x_RUNTOP      = 7,
 	RT_CID_TOSHIBA	  = 9,
 	RT_CID_819X_NETCORE     = 10,
-	RT_CID_Nettronix	= 11,
-	RT_CID_DLINK	    = 12,
-	RT_CID_PRONET	   = 13,
 };
 
 enum reset_type {
@@ -203,41 +179,6 @@ struct rtl8192_tx_ring {
 	struct sk_buff_head queue;
 };
 
-struct rtl819x_ops {
-	enum nic_t nic_type;
-	void (*get_eeprom_size)(struct net_device *dev);
-	void (*init_adapter_variable)(struct net_device *dev);
-	void (*init_before_adapter_start)(struct net_device *dev);
-	bool (*initialize_adapter)(struct net_device *dev);
-	void (*link_change)(struct net_device *dev);
-	void (*tx_fill_descriptor)(struct net_device *dev,
-				   struct tx_desc *tx_desc,
-				   struct cb_desc *cb_desc,
-				   struct sk_buff *skb);
-	void (*tx_fill_cmd_descriptor)(struct net_device *dev,
-				       struct tx_desc_cmd *entry,
-				       struct cb_desc *cb_desc,
-				       struct sk_buff *skb);
-	bool (*rx_query_status_descriptor)(struct net_device *dev,
-					   struct rtllib_rx_stats *stats,
-					   struct rx_desc *pdesc,
-					   struct sk_buff *skb);
-	bool (*rx_command_packet_handler)(struct net_device *dev,
-					  struct sk_buff *skb,
-					  struct rx_desc *pdesc);
-	void (*stop_adapter)(struct net_device *dev, bool reset);
-	void (*update_ratr_table)(struct net_device *dev);
-	void (*irq_enable)(struct net_device *dev);
-	void (*irq_disable)(struct net_device *dev);
-	void (*irq_clear)(struct net_device *dev);
-	void (*rx_enable)(struct net_device *dev);
-	void (*tx_enable)(struct net_device *dev);
-	void (*interrupt_recognized)(struct net_device *dev,
-				     u32 *p_inta, u32 *p_intb);
-	bool (*tx_check_stuck_handler)(struct net_device *dev);
-	bool (*rx_check_stuck_handler)(struct net_device *dev);
-};
-
 struct r8192_priv {
 	struct pci_dev *pdev;
 	struct pci_dev *bridge_pdev;
@@ -255,14 +196,12 @@ struct r8192_priv {
 	struct delayed_work		txpower_tracking_wq;
 	struct delayed_work		rfpath_check_wq;
 	struct delayed_work		gpio_change_rf_wq;
-	struct rtl819x_ops			*ops;
 	struct rtllib_device			*rtllib;
 
 	struct work_struct				reset_wq;
 
 	enum rt_customer_id customer_id;
 
-	enum rt_rf_type_819xu rf_chip;
 	enum ht_channel_width current_chnl_bw;
 	struct bb_reg_definition phy_reg_def[4];
 	struct rate_adaptive rate_adaptive;
@@ -342,7 +281,6 @@ struct r8192_priv {
 	enum nic_t card_8192;
 	u8 card_8192_version;
 
-	u8 rf_type;
 	u8 ic_cut;
 	char nick[IW_ESSID_MAX_SIZE + 1];
 	u8 check_roaming_cnt;
@@ -419,7 +357,6 @@ struct r8192_priv {
 	u8		rfa_txpowertrackingindex_real;
 	u8		rfa_txpowertracking_default;
 	u8		rfc_txpowertrackingindex;
-	u8		rfc_txpowertrackingindex_real;
 	bool		btxpower_tracking;
 	bool		bcck_in_ch14;
 
@@ -438,14 +375,11 @@ struct r8192_priv {
 	bool		bcurrent_turbo_EDCA;
 	bool		bis_cur_rdlstate;
 
-	bool		bfsync_processing;
 	u32		rate_record;
 	u32		rate_count_diff_rec;
 	u32		continue_diff_count;
 	bool		bswitch_fsync;
 	u8		framesync;
-	u8		frame_sync_monitor;
-
 	u32		reset_count;
 
 	enum reset_type rst_progress;
@@ -454,8 +388,6 @@ struct r8192_priv {
 	bool		reset_in_progress;
 	bool		force_reset;
 	bool		force_lps;
-
-	bool		chan_forced;
 };
 
 extern const struct ethtool_ops rtl819x_ethtool_ops;
@@ -495,7 +427,6 @@ u8 rtl92e_rx_db_to_percent(s8 antpower);
 void rtl92e_copy_mpdu_stats(struct rtllib_rx_stats *psrc_stats,
 			    struct rtllib_rx_stats *ptarget_stats);
 bool rtl92e_enable_nic(struct net_device *dev);
-bool rtl92e_disable_nic(struct net_device *dev);
 
 bool rtl92e_set_rf_state(struct net_device *dev,
 			 enum rt_rf_power_state state_to_set,
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c b/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c
index d8408acfc763..56a8ec517c06 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c
@@ -173,7 +173,6 @@ static void _rtl92e_dm_pd_th(struct net_device *dev);
 static void _rtl92e_dm_cs_ratio(struct net_device *dev);
 
 static	void _rtl92e_dm_init_cts_to_self(struct net_device *dev);
-static void _rtl92e_dm_init_wa_broadcom_iot(struct net_device *dev);
 
 static void _rtl92e_dm_check_edca_turbo(struct net_device *dev);
 static void _rtl92e_dm_check_rx_path_selection(struct net_device *dev);
@@ -214,11 +213,8 @@ void rtl92e_dm_init(struct net_device *dev)
 	_rtl92e_dm_init_fsync(dev);
 	_rtl92e_dm_init_rx_path_selection(dev);
 	_rtl92e_dm_init_cts_to_self(dev);
-	if (IS_HARDWARE_TYPE_8192SE(dev))
-		_rtl92e_dm_init_wa_broadcom_iot(dev);
 
-	INIT_DELAYED_WORK_RSL(&priv->gpio_change_rf_wq,
-			      (void *)_rtl92e_dm_check_rf_ctrl_gpio, dev);
+	INIT_DELAYED_WORK(&priv->gpio_change_rf_wq, (void *)_rtl92e_dm_check_rf_ctrl_gpio);
 }
 
 void rtl92e_dm_deinit(struct net_device *dev)
@@ -273,26 +269,14 @@ void rtl92e_init_adaptive_rate(struct net_device *dev)
 		pra->ping_rssi_enable = 0;
 	pra->ping_rssi_thresh_for_ra = 15;
 
-
-	if (priv->rf_type == RF_2T4R) {
-		pra->upper_rssi_threshold_ratr		=	0x8f0f0000;
-		pra->middle_rssi_threshold_ratr		=	0x8f0ff000;
-		pra->low_rssi_threshold_ratr		=	0x8f0ff001;
-		pra->low_rssi_threshold_ratr_40M	=	0x8f0ff005;
-		pra->low_rssi_threshold_ratr_20M	=	0x8f0ff001;
-		pra->ping_rssi_ratr	=	0x0000000d;
-	} else if (priv->rf_type == RF_1T2R) {
-		pra->upper_rssi_threshold_ratr		=	0x000fc000;
-		pra->middle_rssi_threshold_ratr		=	0x000ff000;
-		pra->low_rssi_threshold_ratr		=	0x000ff001;
-		pra->low_rssi_threshold_ratr_40M	=	0x000ff005;
-		pra->low_rssi_threshold_ratr_20M	=	0x000ff001;
-		pra->ping_rssi_ratr	=	0x0000000d;
-	}
-
+	pra->upper_rssi_threshold_ratr		=	0x000fc000;
+	pra->middle_rssi_threshold_ratr		=	0x000ff000;
+	pra->low_rssi_threshold_ratr		=	0x000ff001;
+	pra->low_rssi_threshold_ratr_40M	=	0x000ff005;
+	pra->low_rssi_threshold_ratr_20M	=	0x000ff001;
+	pra->ping_rssi_ratr	=	0x0000000d;
 }
 
-
 static void _rtl92e_dm_check_rate_adaptive(struct net_device *dev)
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
@@ -309,8 +293,7 @@ static void _rtl92e_dm_check_rate_adaptive(struct net_device *dev)
 	if (pra->rate_adaptive_disabled)
 		return;
 
-	if (!(priv->rtllib->mode == WIRELESS_MODE_N_24G ||
-	    priv->rtllib->mode == WIRELESS_MODE_N_5G))
+	if (priv->rtllib->mode != WIRELESS_MODE_N_24G)
 		return;
 
 	if (priv->rtllib->state == RTLLIB_LINKED) {
@@ -391,8 +374,7 @@ static void _rtl92e_dm_check_rate_adaptive(struct net_device *dev)
 			u32 ratr_value;
 
 			ratr_value = targetRATR;
-			if (priv->rf_type == RF_1T2R)
-				ratr_value &= ~(RATE_ALL_OFDM_2SS);
+			ratr_value &= ~(RATE_ALL_OFDM_2SS);
 			rtl92e_writel(dev, RATR0, ratr_value);
 			rtl92e_writeb(dev, UFWP, 1);
 
@@ -490,93 +472,39 @@ static u8	CCKSwingTable_Ch14[CCK_TABLE_LEN][8] = {
 #define		Tssi_Report_Value2			0x13e
 #define		FW_Busy_Flag				0x13f
 
-static void _rtl92e_dm_tx_update_tssi_weak_signal(struct net_device *dev,
-						  u8 RF_Type)
+static void _rtl92e_dm_tx_update_tssi_weak_signal(struct net_device *dev)
 {
 	struct r8192_priv *p = rtllib_priv(dev);
 
-	if (RF_Type == RF_2T4R) {
-		if ((p->rfa_txpowertrackingindex > 0) &&
-		    (p->rfc_txpowertrackingindex > 0)) {
-			p->rfa_txpowertrackingindex--;
-			if (p->rfa_txpowertrackingindex_real > 4) {
-				p->rfa_txpowertrackingindex_real--;
-				rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance,
-						  bMaskDWord,
-						  dm_tx_bb_gain[p->rfa_txpowertrackingindex_real]);
-			}
-
-			p->rfc_txpowertrackingindex--;
-			if (p->rfc_txpowertrackingindex_real > 4) {
-				p->rfc_txpowertrackingindex_real--;
-				rtl92e_set_bb_reg(dev,
-						  rOFDM0_XCTxIQImbalance,
-						  bMaskDWord,
-						  dm_tx_bb_gain[p->rfc_txpowertrackingindex_real]);
-			}
-		} else {
-			rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance,
-					  bMaskDWord,
-					  dm_tx_bb_gain[4]);
+	if (p->rfa_txpowertrackingindex > 0) {
+		p->rfa_txpowertrackingindex--;
+		if (p->rfa_txpowertrackingindex_real > 4) {
+			p->rfa_txpowertrackingindex_real--;
 			rtl92e_set_bb_reg(dev,
-					  rOFDM0_XCTxIQImbalance,
-					  bMaskDWord, dm_tx_bb_gain[4]);
+					  rOFDM0_XATxIQImbalance,
+					  bMaskDWord,
+					  dm_tx_bb_gain[p->rfa_txpowertrackingindex_real]);
 		}
 	} else {
-		if (p->rfa_txpowertrackingindex > 0) {
-			p->rfa_txpowertrackingindex--;
-			if (p->rfa_txpowertrackingindex_real > 4) {
-				p->rfa_txpowertrackingindex_real--;
-				rtl92e_set_bb_reg(dev,
-						  rOFDM0_XATxIQImbalance,
-						  bMaskDWord,
-						  dm_tx_bb_gain[p->rfa_txpowertrackingindex_real]);
-			}
-		} else {
-			rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance,
-					  bMaskDWord, dm_tx_bb_gain[4]);
-		}
+		rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance,
+				  bMaskDWord, dm_tx_bb_gain[4]);
 	}
 }
 
-static void _rtl92e_dm_tx_update_tssi_strong_signal(struct net_device *dev,
-						    u8 RF_Type)
+static void _rtl92e_dm_tx_update_tssi_strong_signal(struct net_device *dev)
 {
 	struct r8192_priv *p = rtllib_priv(dev);
 
-	if (RF_Type == RF_2T4R) {
-		if ((p->rfa_txpowertrackingindex < TX_BB_GAIN_TABLE_LEN - 1) &&
-		    (p->rfc_txpowertrackingindex < TX_BB_GAIN_TABLE_LEN - 1)) {
-			p->rfa_txpowertrackingindex++;
-			p->rfa_txpowertrackingindex_real++;
-			rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance,
-					  bMaskDWord,
-					  dm_tx_bb_gain[p->rfa_txpowertrackingindex_real]);
-			p->rfc_txpowertrackingindex++;
-			p->rfc_txpowertrackingindex_real++;
-			rtl92e_set_bb_reg(dev, rOFDM0_XCTxIQImbalance,
-					  bMaskDWord,
-					  dm_tx_bb_gain[p->rfc_txpowertrackingindex_real]);
-		} else {
-			rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance,
-					  bMaskDWord,
-					  dm_tx_bb_gain[TX_BB_GAIN_TABLE_LEN - 1]);
-			rtl92e_set_bb_reg(dev, rOFDM0_XCTxIQImbalance,
-					  bMaskDWord,
-					  dm_tx_bb_gain[TX_BB_GAIN_TABLE_LEN - 1]);
-		}
+	if (p->rfa_txpowertrackingindex < (TX_BB_GAIN_TABLE_LEN - 1)) {
+		p->rfa_txpowertrackingindex++;
+		p->rfa_txpowertrackingindex_real++;
+		rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance,
+				  bMaskDWord,
+				  dm_tx_bb_gain[p->rfa_txpowertrackingindex_real]);
 	} else {
-		if (p->rfa_txpowertrackingindex < (TX_BB_GAIN_TABLE_LEN - 1)) {
-			p->rfa_txpowertrackingindex++;
-			p->rfa_txpowertrackingindex_real++;
-			rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance,
-					  bMaskDWord,
-					  dm_tx_bb_gain[p->rfa_txpowertrackingindex_real]);
-		} else {
-			rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance,
-					  bMaskDWord,
-					  dm_tx_bb_gain[TX_BB_GAIN_TABLE_LEN - 1]);
-		}
+		rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance,
+				  bMaskDWord,
+				  dm_tx_bb_gain[TX_BB_GAIN_TABLE_LEN - 1]);
 	}
 }
 
@@ -585,10 +513,8 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev)
 	struct r8192_priv *priv = rtllib_priv(dev);
 	bool	viviflag = false;
 	struct dcmd_txcmd tx_cmd;
-	u8	powerlevelOFDM24G;
 	int	i = 0, j = 0, k = 0;
-	u8	RF_Type, tmp_report[5] = {0, 0, 0, 0, 0};
-	u32	Value;
+	u8	tmp_report[5] = {0, 0, 0, 0, 0};
 	u8	Pwr_Flag;
 	u16	Avg_TSSI_Meas, tssi_13dBm, Avg_TSSI_Meas_from_driver = 0;
 	u32	delta = 0;
@@ -597,15 +523,11 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev)
 	rtl92e_writeb(dev, FW_Busy_Flag, 0);
 	priv->rtllib->bdynamic_txpower_enable = false;
 
-	powerlevelOFDM24G = priv->pwr_track >> 24;
-	RF_Type = priv->rf_type;
-	Value = (RF_Type<<8) | powerlevelOFDM24G;
-
 	for (j = 0; j <= 30; j++) {
 
 		tx_cmd.op	= TXCMD_SET_TX_PWR_TRACKING;
 		tx_cmd.length	= 4;
-		tx_cmd.value	= Value;
+		tx_cmd.value	= priv->pwr_track >> 24;
 		rtl92e_send_cmd_pkt(dev, DESC_PACKET_TYPE_NORMAL, (u8 *)&tx_cmd,
 				    sizeof(struct dcmd_txcmd));
 		mdelay(1);
@@ -677,18 +599,12 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev)
 				return;
 			}
 			if (Avg_TSSI_Meas_from_driver < tssi_13dBm - E_FOR_TX_POWER_TRACK)
-				_rtl92e_dm_tx_update_tssi_weak_signal(dev,
-								      RF_Type);
+				_rtl92e_dm_tx_update_tssi_weak_signal(dev);
 			else
-				_rtl92e_dm_tx_update_tssi_strong_signal(dev, RF_Type);
+				_rtl92e_dm_tx_update_tssi_strong_signal(dev);
 
-			if (RF_Type == RF_2T4R) {
-				priv->cck_present_attn_diff
-					= priv->rfa_txpowertrackingindex - priv->rfa_txpowertracking_default;
-			} else {
-				priv->cck_present_attn_diff
-					= priv->rfa_txpowertrackingindex_real - priv->rfa_txpowertracking_default;
-			}
+			priv->cck_present_attn_diff
+				= priv->rfa_txpowertrackingindex_real - priv->rfa_txpowertracking_default;
 
 			if (priv->current_chnl_bw == HT_CHANNEL_WIDTH_20)
 				priv->cck_present_attn =
@@ -897,10 +813,7 @@ static void _rtl92e_dm_check_tx_power_tracking_thermal(struct net_device *dev)
 	static u8	TM_Trigger;
 	u8		TxPowerCheckCnt = 0;
 
-	if (IS_HARDWARE_TYPE_8192SE(dev))
-		TxPowerCheckCnt = 5;
-	else
-		TxPowerCheckCnt = 2;
+	TxPowerCheckCnt = 2;
 	if (!priv->btxpower_tracking)
 		return;
 
@@ -1042,12 +955,10 @@ void rtl92e_dm_restore_state(struct net_device *dev)
 
 	if (priv->rate_adaptive.rate_adaptive_disabled)
 		return;
-	if (!(priv->rtllib->mode == WIRELESS_MODE_N_24G ||
-	      priv->rtllib->mode == WIRELESS_MODE_N_5G))
+	if (priv->rtllib->mode != WIRELESS_MODE_N_24G)
 		return;
 	ratr_value = reg_ratr;
-	if (priv->rf_type == RF_1T2R)
-		ratr_value &= ~(RATE_ALL_OFDM_2SS);
+	ratr_value &= ~(RATE_ALL_OFDM_2SS);
 	rtl92e_writel(dev, RATR0, ratr_value);
 	rtl92e_writeb(dev, UFWP, 1);
 	if (priv->tx_pwr_tracking_init && priv->btxpower_tracking)
@@ -1087,7 +998,6 @@ void rtl92e_dm_backup_state(struct net_device *dev)
 	u32 bit_mask = bMaskByte0;
 
 	priv->bswitch_fsync  = false;
-	priv->bfsync_processing = false;
 
 	if (dm_digtable.dig_algorithm == DIG_ALGO_BY_RSSI)
 		return;
@@ -1619,16 +1529,6 @@ static void _rtl92e_dm_cts_to_self(struct net_device *dev)
 	}
 }
 
-
-static void _rtl92e_dm_init_wa_broadcom_iot(struct net_device *dev)
-{
-	struct r8192_priv *priv = rtllib_priv((struct net_device *)dev);
-	struct rt_hi_throughput *ht_info = priv->rtllib->ht_info;
-
-	ht_info->bWAIotBroadcom = false;
-	ht_info->WAIotTH = WA_IOT_TH_VAL;
-}
-
 static void _rtl92e_dm_check_rf_ctrl_gpio(void *data)
 {
 	struct r8192_priv *priv = container_of_dwork_rsl(data,
@@ -1638,15 +1538,11 @@ static void _rtl92e_dm_check_rf_ctrl_gpio(void *data)
 	enum rt_rf_power_state rf_power_state_to_set;
 	bool bActuallySet = false;
 
-	bActuallySet = false;
-
 	if ((priv->up_first_time == 1) || (priv->being_init_adapter))
 		return;
 
-	if (priv->bfirst_after_down) {
-		priv->bfirst_after_down = true;
+	if (priv->bfirst_after_down)
 		return;
-	}
 
 	tmp1byte = rtl92e_readb(dev, GPI);
 
@@ -1730,9 +1626,6 @@ static void _rtl92e_dm_rx_path_sel_byrssi(struct net_device *dev)
 	static u8 disabled_rf_cnt, cck_Rx_Path_initialized;
 	u8 update_cck_rx_path;
 
-	if (priv->rf_type != RF_2T4R)
-		return;
-
 	if (!cck_Rx_Path_initialized) {
 		dm_rx_path_sel_table.cck_rx_path = (rtl92e_readb(dev, 0xa07)&0xf);
 		cck_Rx_Path_initialized = 1;
@@ -1932,7 +1825,6 @@ static void _rtl92e_dm_init_fsync(struct net_device *dev)
 	priv->rtllib->fsync_firstdiff_ratethreshold = 100;
 	priv->rtllib->fsync_seconddiff_ratethreshold = 200;
 	priv->rtllib->fsync_state = Default_Fsync;
-	priv->frame_sync_monitor = 1;
 
 	timer_setup(&priv->fsync_timer, _rtl92e_dm_fsync_timer_callback, 0);
 }
@@ -2159,12 +2051,10 @@ static void _rtl92e_dm_check_fsync(struct net_device *dev)
 
 			}
 		}
-		if (priv->frame_sync_monitor) {
-			if (reg_c38_State != RegC38_Fsync_AP_BCM) {
-				rtl92e_writeb(dev, rOFDM0_RxDetector3, 0x95);
+		if (reg_c38_State != RegC38_Fsync_AP_BCM) {
+			rtl92e_writeb(dev, rOFDM0_RxDetector3, 0x95);
 
-				reg_c38_State = RegC38_Fsync_AP_BCM;
-			}
+			reg_c38_State = RegC38_Fsync_AP_BCM;
 		}
 	} else {
 		switch (priv->rtllib->fsync_state) {
@@ -2181,50 +2071,40 @@ static void _rtl92e_dm_check_fsync(struct net_device *dev)
 			break;
 		}
 
-		if (priv->frame_sync_monitor) {
-			if (priv->rtllib->state == RTLLIB_LINKED) {
-				if (priv->undecorated_smoothed_pwdb <=
-				    RegC38_TH) {
-					if (reg_c38_State !=
-					    RegC38_NonFsync_Other_AP) {
-						rtl92e_writeb(dev,
-							      rOFDM0_RxDetector3,
-							      0x90);
-
-						reg_c38_State =
-						     RegC38_NonFsync_Other_AP;
-					}
-				} else if (priv->undecorated_smoothed_pwdb >=
-					   (RegC38_TH+5)) {
-					if (reg_c38_State) {
-						rtl92e_writeb(dev,
-							rOFDM0_RxDetector3,
-							priv->framesync);
-						reg_c38_State = RegC38_Default;
-					}
+		if (priv->rtllib->state == RTLLIB_LINKED) {
+			if (priv->undecorated_smoothed_pwdb <=
+			    RegC38_TH) {
+				if (reg_c38_State !=
+				    RegC38_NonFsync_Other_AP) {
+					rtl92e_writeb(dev,
+						      rOFDM0_RxDetector3,
+						      0x90);
+
+					reg_c38_State =
+					     RegC38_NonFsync_Other_AP;
 				}
-			} else {
+			} else if (priv->undecorated_smoothed_pwdb >=
+				   (RegC38_TH+5)) {
 				if (reg_c38_State) {
-					rtl92e_writeb(dev, rOFDM0_RxDetector3,
-						      priv->framesync);
+					rtl92e_writeb(dev,
+						rOFDM0_RxDetector3,
+						priv->framesync);
 					reg_c38_State = RegC38_Default;
 				}
 			}
+		} else {
+			if (reg_c38_State) {
+				rtl92e_writeb(dev, rOFDM0_RxDetector3,
+					      priv->framesync);
+				reg_c38_State = RegC38_Default;
+			}
 		}
 	}
-	if (priv->frame_sync_monitor) {
-		if (priv->reset_count != reset_cnt) {
-			rtl92e_writeb(dev, rOFDM0_RxDetector3,
-				       priv->framesync);
-			reg_c38_State = RegC38_Default;
-			reset_cnt = priv->reset_count;
-		}
-	} else {
-		if (reg_c38_State) {
-			rtl92e_writeb(dev, rOFDM0_RxDetector3,
-				       priv->framesync);
-			reg_c38_State = RegC38_Default;
-		}
+	if (priv->reset_count != reset_cnt) {
+		rtl92e_writeb(dev, rOFDM0_RxDetector3,
+			       priv->framesync);
+		reg_c38_State = RegC38_Default;
+		reset_cnt = priv->reset_count;
 	}
 }
 
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_pci.c b/drivers/staging/rtl8192e/rtl8192e/rtl_pci.c
index 81e1bb856c60..0bc3e013001e 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_pci.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_pci.c
@@ -39,7 +39,7 @@ bool rtl92e_check_adapter(struct pci_dev *pdev, struct net_device *dev)
 	revision_id = pdev->revision;
 	pci_read_config_word(pdev, 0x3C, &irq_line);
 
-	priv->card_8192 = priv->ops->nic_type;
+	priv->card_8192 = NIC_8192E;
 
 	if (device_id == 0x8192) {
 		switch (revision_id) {
@@ -64,10 +64,10 @@ bool rtl92e_check_adapter(struct pci_dev *pdev, struct net_device *dev)
 		}
 	}
 
-	if (priv->ops->nic_type != priv->card_8192) {
+	if (priv->card_8192 != NIC_8192E) {
 		dev_info(&pdev->dev,
 			 "Detect info(%x) and hardware info(%x) not match!\n",
-			 priv->ops->nic_type, priv->card_8192);
+			 NIC_8192E, priv->card_8192);
 		dev_info(&pdev->dev,
 			 "Please select proper driver before install!!!!\n");
 		return false;
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c b/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c
index ef4f736ce325..5a1cd22f5e25 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c
@@ -142,8 +142,7 @@ void rtl92e_ips_leave(struct net_device *dev)
 
 void rtl92e_ips_leave_wq(void *data)
 {
-	struct rtllib_device *ieee = container_of_work_rsl(data,
-				     struct rtllib_device, ips_leave_wq);
+	struct rtllib_device *ieee = container_of(data, struct rtllib_device, ips_leave_wq);
 	struct net_device *dev = ieee->dev;
 	struct r8192_priv *priv = (struct r8192_priv *)rtllib_priv(dev);
 
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c b/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c
index bf0030144e5d..cb28288a618b 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c
@@ -432,15 +432,11 @@ static int _rtl92e_wx_set_scan(struct net_device *dev,
 		if (priv->rtllib->rf_power_state != rf_off) {
 			priv->rtllib->actscanning = true;
 
-			if (ieee->ScanOperationBackupHandler)
-				ieee->ScanOperationBackupHandler(ieee->dev,
-							 SCAN_OPT_BACKUP);
+			ieee->ScanOperationBackupHandler(ieee->dev, SCAN_OPT_BACKUP);
 
 			rtllib_start_scan_syncro(priv->rtllib, 0);
 
-			if (ieee->ScanOperationBackupHandler)
-				ieee->ScanOperationBackupHandler(ieee->dev,
-							 SCAN_OPT_RESTORE);
+			ieee->ScanOperationBackupHandler(ieee->dev, SCAN_OPT_RESTORE);
 		}
 		ret = 0;
 	} else {
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_hwimg.c b/drivers/staging/rtl8192e/rtl8192e/table.c
index e6fce749e65b..0b5cc6049232 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_hwimg.c
+++ b/drivers/staging/rtl8192e/rtl8192e/table.c
@@ -4,11 +4,9 @@
  *
  * Contact Information: wlanfae <wlanfae@realtek.com>
  */
-#include "r8192E_hwimg.h"
+#include "table.h"
 
-u32 Rtl8192PciEPHY_REGArray[PHY_REGArrayLengthPciE] = {0x0,};
-
-u32 Rtl8192PciEPHY_REG_1T2RArray[PHY_REG_1T2RArrayLengthPciE] = {
+u32 RTL8192E_PHY_REG_1T2R_ARR[RTL8192E_PHY_REG_1T2R_ARR_LEN] = {
 	0x800, 0x00000000,
 	0x804, 0x00000001,
 	0x808, 0x0000fc00,
@@ -159,7 +157,7 @@ u32 Rtl8192PciEPHY_REG_1T2RArray[PHY_REG_1T2RArrayLengthPciE] = {
 	0xe1c, 0x12121416,
 };
 
-u32 Rtl8192PciERadioA_Array[RadioA_ArrayLengthPciE] = {
+u32 RTL8192E_RADIO_A_ARR[RTL8192E_RADIO_A_ARR_LEN] = {
 	0x019, 0x00000003,
 	0x000, 0x000000bf,
 	0x001, 0x00000ee0,
@@ -285,7 +283,7 @@ u32 Rtl8192PciERadioA_Array[RadioA_ArrayLengthPciE] = {
 	0x007, 0x00000700,
 };
 
-u32 Rtl8192PciERadioB_Array[RadioB_ArrayLengthPciE] = {
+u32 RTL8192E_RADIO_B_ARR[RTL8192E_RADIO_B_ARR_LEN] = {
 	0x019, 0x00000003,
 	0x000, 0x000000bf,
 	0x001, 0x000006e0,
@@ -327,13 +325,7 @@ u32 Rtl8192PciERadioB_Array[RadioB_ArrayLengthPciE] = {
 	0x007, 0x00000700,
 };
 
-u32 Rtl8192PciERadioC_Array[RadioC_ArrayLengthPciE] = {
-	0x0,  };
-
-u32 Rtl8192PciERadioD_Array[RadioD_ArrayLengthPciE] = {
-	0x0, };
-
-u32 Rtl8192PciEMACPHY_Array[] = {
+u32 RTL8192E_MACPHY_ARR[] = {
 	0x03c, 0xffff0000, 0x00000f0f,
 	0x340, 0xffffffff, 0x161a1a1a,
 	0x344, 0xffffffff, 0x12121416,
@@ -342,7 +334,7 @@ u32 Rtl8192PciEMACPHY_Array[] = {
 	0x318, 0x00000fff, 0x00000100,
 };
 
-u32 Rtl8192PciEMACPHY_Array_PG[] = {
+u32 RTL8192E_MACPHY_ARR_PG[] = {
 	0x03c, 0xffff0000, 0x00000f0f,
 	0xe00, 0xffffffff, 0x06090909,
 	0xe04, 0xffffffff, 0x00030306,
@@ -355,7 +347,7 @@ u32 Rtl8192PciEMACPHY_Array_PG[] = {
 	0x318, 0x00000fff, 0x00000800,
 };
 
-u32 Rtl8192PciEAGCTAB_Array[AGCTAB_ArrayLengthPciE] = {
+u32 RTL8192E_AGCTAB_ARR[RTL8192E_AGCTAB_ARR_LEN] = {
 	0xc78, 0x7d000001,
 	0xc78, 0x7d010001,
 	0xc78, 0x7d020001,
diff --git a/drivers/staging/rtl8192e/rtl8192e/table.h b/drivers/staging/rtl8192e/rtl8192e/table.h
new file mode 100644
index 000000000000..82be44a9d4e8
--- /dev/null
+++ b/drivers/staging/rtl8192e/rtl8192e/table.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright(c) 2008 - 2010 Realtek Corporation. All rights reserved.
+ *
+ * Contact Information: wlanfae <wlanfae@realtek.com>
+ */
+#ifndef __INC_HAL8192PciE_FW_IMG_H
+#define __INC_HAL8192PciE_FW_IMG_H
+
+/*Created on  2008/11/18,  3: 7*/
+
+#include <linux/types.h>
+
+#define RTL8192E_PHY_REG_1T2R_ARR_LEN 296
+extern u32 RTL8192E_PHY_REG_1T2R_ARR[RTL8192E_PHY_REG_1T2R_ARR_LEN];
+#define RTL8192E_RADIO_A_ARR_LEN 246
+extern u32 RTL8192E_RADIO_A_ARR[RTL8192E_RADIO_A_ARR_LEN];
+#define RTL8192E_RADIO_B_ARR_LEN 78
+extern u32 RTL8192E_RADIO_B_ARR[RTL8192E_RADIO_B_ARR_LEN];
+#define RTL8192E_MACPHY_ARR_LEN 18
+extern u32 RTL8192E_MACPHY_ARR[RTL8192E_MACPHY_ARR_LEN];
+#define RTL8192E_MACPHY_ARR_PG_LEN 30
+extern u32 RTL8192E_MACPHY_ARR_PG[RTL8192E_MACPHY_ARR_PG_LEN];
+#define RTL8192E_AGCTAB_ARR_LEN 384
+extern u32 RTL8192E_AGCTAB_ARR[RTL8192E_AGCTAB_ARR_LEN];
+
+#endif
diff --git a/drivers/staging/rtl8192e/rtl819x_HT.h b/drivers/staging/rtl8192e/rtl819x_HT.h
index 22e4f126ed56..f4e9fa849796 100644
--- a/drivers/staging/rtl8192e/rtl819x_HT.h
+++ b/drivers/staging/rtl8192e/rtl819x_HT.h
@@ -162,9 +162,6 @@ struct rt_hi_throughput {
 	u8				IOTPeer;
 	u32 iot_action;
 	u8 iot_ra_func;
-
-	u8	bWAIotBroadcom;
-	u8	WAIotTH;
 } __packed;
 
 struct bss_ht {
diff --git a/drivers/staging/rtl8192e/rtl819x_TSProc.c b/drivers/staging/rtl8192e/rtl819x_TSProc.c
index 68c131afc2ba..c61fdf73c572 100644
--- a/drivers/staging/rtl8192e/rtl819x_TSProc.c
+++ b/drivers/staging/rtl8192e/rtl819x_TSProc.c
@@ -517,6 +517,7 @@ void TsStartAddBaProcess(struct rtllib_device *ieee, struct tx_ts_record *pTxTS)
 			netdev_dbg(ieee->dev, "Immediately Start ADDBA\n");
 			mod_timer(&pTxTS->TsAddBaTimer, jiffies + 10);
 		}
-	} else
+	} else {
 		netdev_dbg(ieee->dev, "BA timer is already added\n");
+	}
 }
diff --git a/drivers/staging/rtl8192e/rtllib.h b/drivers/staging/rtl8192e/rtllib.h
index 1152fbf43383..6e665e866f1f 100644
--- a/drivers/staging/rtl8192e/rtllib.h
+++ b/drivers/staging/rtl8192e/rtllib.h
@@ -62,24 +62,9 @@
 #define IW_CUSTOM_MAX	256	/* In bytes */
 #endif
 
-#define skb_tail_pointer_rsl(skb) skb_tail_pointer(skb)
-
-#define queue_delayed_work_rsl(x, y, z) queue_delayed_work(x, y, z)
-#define INIT_DELAYED_WORK_RSL(x, y, z) INIT_DELAYED_WORK(x, y)
-
-#define queue_work_rsl(x, y) queue_work(x, y)
-#define INIT_WORK_RSL(x, y, z) INIT_WORK(x, y)
-
-#define container_of_work_rsl(x, y, z) container_of(x, y, z)
 #define container_of_dwork_rsl(x, y, z)				\
 	container_of(to_delayed_work(x), y, z)
 
-#define iwe_stream_add_event_rsl(info, start, stop, iwe, len)	\
-	iwe_stream_add_event(info, start, stop, iwe, len)
-
-#define iwe_stream_add_point_rsl(info, start, stop, iwe, p)	\
-	iwe_stream_add_point(info, start, stop, iwe, p)
-
 static inline void *netdev_priv_rsl(struct net_device *dev)
 {
 	return netdev_priv(dev);
@@ -115,7 +100,6 @@ static inline void *netdev_priv_rsl(struct net_device *dev)
 	((psc->CurPsLevel & _PS_FLAG) ? true : false)
 #define	RT_CLEAR_PS_LEVEL(psc, _PS_FLAG)	\
 	(psc->CurPsLevel &= (~(_PS_FLAG)))
-#define	RT_SET_PS_LEVEL(psc, _PS_FLAG)	(psc->CurPsLevel |= _PS_FLAG)
 
 /* defined for skb cb field */
 /* At most 28 byte */
@@ -323,7 +307,6 @@ enum rt_op_mode {
 	RT_OP_MODE_NO_LINK,
 };
 
-
 #define aSifsTime						\
 	 (((priv->rtllib->current_network.mode == IEEE_A)	\
 	|| (priv->rtllib->current_network.mode == IEEE_N_24G)	\
@@ -449,11 +432,6 @@ enum led_ctl_mode {
 	LED_CTL_START_TO_LINK = 8,
 };
 
-enum rt_rf_type_def {
-	RF_1T2R = 0,
-	RF_2T4R,
-};
-
 enum wireless_mode {
 	WIRELESS_MODE_UNKNOWN = 0x00,
 	WIRELESS_MODE_A = 0x01,
@@ -669,7 +647,6 @@ struct rtllib_security {
 	u16 flags;
 } __packed;
 
-
 /* 802.11 data frame from AP
  *       ,-------------------------------------------------------------------.
  * Bytes |  2   |  2   |    6    |    6    |    6    |  2   | 0..2312 |   4  |
@@ -989,6 +966,7 @@ static inline const char *eap_get_type(int type)
 	return ((u32)type >= ARRAY_SIZE(eap_types)) ? "Unknown" :
 		 eap_types[type];
 }
+
 static inline u8 Frame_QoSTID(u8 *buf)
 {
 	struct rtllib_hdr_3addr *hdr;
@@ -1000,7 +978,6 @@ static inline u8 Frame_QoSTID(u8 *buf)
 		    (fc & RTLLIB_FCTL_FROMDS)) ? 30 : 24)))->field.tid;
 }
 
-
 struct eapol {
 	u8 snap[6];
 	u16 ethertype;
@@ -1215,8 +1192,6 @@ struct bandwidth_autoswitch {
 	bool bautoswitch_enable;
 };
 
-
-
 #define REORDER_WIN_SIZE	128
 #define REORDER_ENTRY_NUM	128
 struct rx_reorder_entry {
@@ -1224,6 +1199,7 @@ struct rx_reorder_entry {
 	u16			SeqNum;
 	struct rtllib_rxb *prxb;
 };
+
 enum fsync_state {
 	Default_Fsync,
 	HW_Fsync,
@@ -1260,7 +1236,6 @@ struct rt_pwr_save_ctrl {
 	u8				LPSAwakeIntvl;
 
 	u32				CurPsLevel;
-	u32				RegRfPsLevel;
 };
 
 #define RT_RF_CHANGE_SOURCE u32
@@ -1350,6 +1325,7 @@ struct sw_cam_table {
 	u8				key_index;
 
 };
+
 #define   TOTAL_CAM_ENTRY				32
 struct rate_adaptive {
 	u8				rate_adaptive_disabled;
@@ -1388,7 +1364,6 @@ struct rt_intel_promisc_mode {
 	bool fltr_src_sta_frame;
 };
 
-
 /*************** DRIVER STATUS   *****/
 #define STATUS_SCANNING			0
 /*************** DRIVER STATUS   *****/
@@ -1416,7 +1391,6 @@ struct rtllib_device {
 	size_t assocreq_ies_len, assocresp_ies_len;
 
 	bool	bForcedBgMode;
-	u8 RF_Type;
 
 	u8 hwsec_active;
 	bool is_silent_reset;
@@ -1457,7 +1431,6 @@ struct rtllib_device {
 	struct rx_reorder_entry RxReorderEntry[128];
 	struct list_head		RxReorder_Unused_List;
 
-
 	/* Bookkeeping structures */
 	struct net_device_stats stats;
 	struct rtllib_softmac_stats softmac_stats;
@@ -1754,7 +1727,6 @@ struct rtllib_device {
 				     struct rtllib_assoc_response_frame *resp,
 				     struct rtllib_network *network);
 
-
 	/* check whether Tx hw resource available */
 	short (*check_nic_enough_desc)(struct net_device *dev, int queue_index);
 	void (*SetBWModeHandler)(struct net_device *dev,
@@ -1827,7 +1799,6 @@ struct rtllib_device {
  */
 #define IEEE_SOFTMAC_BEACONS (1<<6)
 
-
 static inline void *rtllib_priv(struct net_device *dev)
 {
 	return ((struct rtllib_device *)netdev_priv(dev))->priv;
@@ -1919,17 +1890,15 @@ static inline int rtllib_is_cck_rate(u8 rate)
 	return 0;
 }
 
-
 /* rtllib.c */
 void free_rtllib(struct net_device *dev);
 struct net_device *alloc_rtllib(int sizeof_priv);
 
 /* rtllib_tx.c */
 
-int rtllib_encrypt_fragment(
-	struct rtllib_device *ieee,
-	struct sk_buff *frag,
-	int hdr_len);
+int rtllib_encrypt_fragment(struct rtllib_device *ieee,
+			    struct sk_buff *frag,
+			    int hdr_len);
 
 netdev_tx_t rtllib_xmit(struct sk_buff *skb,  struct net_device *dev);
 void rtllib_txb_free(struct rtllib_txb *txb);
@@ -2129,7 +2098,6 @@ static inline const char *escape_essid(const char *essid, u8 essid_len)
 /* fun with the built-in rtllib stack... */
 bool rtllib_MgntDisconnect(struct rtllib_device *rtllib, u8 asRsn);
 
-
 /* For the function is more related to hardware setting, it's better to use the
  * ieee handler to refer to it.
  */
diff --git a/drivers/staging/rtl8192e/rtllib_crypt_ccmp.c b/drivers/staging/rtl8192e/rtllib_crypt_ccmp.c
index a8d22da8bc9a..f88096bcb181 100644
--- a/drivers/staging/rtl8192e/rtllib_crypt_ccmp.c
+++ b/drivers/staging/rtl8192e/rtllib_crypt_ccmp.c
@@ -34,9 +34,9 @@ struct rtllib_ccmp_data {
 	u8 tx_pn[CCMP_PN_LEN];
 	u8 rx_pn[CCMP_PN_LEN];
 
-	u32 dot11RSNAStatsCCMPFormatErrors;
-	u32 dot11RSNAStatsCCMPReplays;
-	u32 dot11RSNAStatsCCMPDecryptErrors;
+	u32 dot11rsna_stats_ccmp_format_errors;
+	u32 dot11rsna_stats_ccmp_replays;
+	u32 dot11rsna_stats_ccmp_decrypt_errors;
 
 	int key_idx;
 
@@ -74,7 +74,6 @@ fail:
 	return NULL;
 }
 
-
 static void rtllib_ccmp_deinit(void *priv)
 {
 	struct rtllib_ccmp_data *_priv = priv;
@@ -84,7 +83,6 @@ static void rtllib_ccmp_deinit(void *priv)
 	kfree(priv);
 }
 
-
 static int ccmp_init_iv_and_aad(struct rtllib_hdr_4addr *hdr,
 				u8 *pn, u8 *iv, u8 *aad)
 {
@@ -150,8 +148,6 @@ static int ccmp_init_iv_and_aad(struct rtllib_hdr_4addr *hdr,
 	return aad_len;
 }
 
-
-
 static int rtllib_ccmp_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
 {
 	struct rtllib_ccmp_data *key = priv;
@@ -220,7 +216,6 @@ static int rtllib_ccmp_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
 	return 0;
 }
 
-
 static int rtllib_ccmp_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
 {
 	struct rtllib_ccmp_data *key = priv;
@@ -231,7 +226,7 @@ static int rtllib_ccmp_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
 	u8 pn[6];
 
 	if (skb->len < hdr_len + CCMP_HDR_LEN + CCMP_MIC_LEN) {
-		key->dot11RSNAStatsCCMPFormatErrors++;
+		key->dot11rsna_stats_ccmp_format_errors++;
 		return -1;
 	}
 
@@ -243,7 +238,7 @@ static int rtllib_ccmp_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
 			pr_debug("CCMP: received packet without ExtIV flag from %pM\n",
 				 hdr->addr2);
 		}
-		key->dot11RSNAStatsCCMPFormatErrors++;
+		key->dot11rsna_stats_ccmp_format_errors++;
 		return -2;
 	}
 	keyidx >>= 6;
@@ -268,7 +263,7 @@ static int rtllib_ccmp_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
 	pn[5] = pos[0];
 	pos += 8;
 	if (memcmp(pn, key->rx_pn, CCMP_PN_LEN) <= 0) {
-		key->dot11RSNAStatsCCMPReplays++;
+		key->dot11rsna_stats_ccmp_replays++;
 		return -4;
 	}
 	if (!tcb_desc->bHwSec) {
@@ -301,7 +296,7 @@ static int rtllib_ccmp_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
 				pr_debug("CCMP: decrypt failed: STA= %pM\n",
 					 hdr->addr2);
 			}
-			key->dot11RSNAStatsCCMPDecryptErrors++;
+			key->dot11rsna_stats_ccmp_decrypt_errors++;
 			return -5;
 		}
 
@@ -315,7 +310,6 @@ static int rtllib_ccmp_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
 	return keyidx;
 }
 
-
 static int rtllib_ccmp_set_key(void *key, int len, u8 *seq, void *priv)
 {
 	struct rtllib_ccmp_data *data = priv;
@@ -338,7 +332,7 @@ static int rtllib_ccmp_set_key(void *key, int len, u8 *seq, void *priv)
 			data->rx_pn[5] = seq[0];
 		}
 		if (crypto_aead_setauthsize(data->tfm, CCMP_MIC_LEN) ||
-			crypto_aead_setkey(data->tfm, data->key, CCMP_TK_LEN))
+		    crypto_aead_setkey(data->tfm, data->key, CCMP_TK_LEN))
 			return -1;
 	} else if (len == 0) {
 		data->key_set = 0;
@@ -349,7 +343,6 @@ static int rtllib_ccmp_set_key(void *key, int len, u8 *seq, void *priv)
 	return 0;
 }
 
-
 static int rtllib_ccmp_get_key(void *key, int len, u8 *seq, void *priv)
 {
 	struct rtllib_ccmp_data *data = priv;
@@ -373,7 +366,6 @@ static int rtllib_ccmp_get_key(void *key, int len, u8 *seq, void *priv)
 	return CCMP_TK_LEN;
 }
 
-
 static void rtllib_ccmp_print_stats(struct seq_file *m, void *priv)
 {
 	struct rtllib_ccmp_data *ccmp = priv;
@@ -382,9 +374,9 @@ static void rtllib_ccmp_print_stats(struct seq_file *m, void *priv)
 		   "key[%d] alg=CCMP key_set=%d tx_pn=%pM rx_pn=%pM format_errors=%d replays=%d decrypt_errors=%d\n",
 		   ccmp->key_idx, ccmp->key_set,
 		   ccmp->tx_pn, ccmp->rx_pn,
-		   ccmp->dot11RSNAStatsCCMPFormatErrors,
-		   ccmp->dot11RSNAStatsCCMPReplays,
-		   ccmp->dot11RSNAStatsCCMPDecryptErrors);
+		   ccmp->dot11rsna_stats_ccmp_format_errors,
+		   ccmp->dot11rsna_stats_ccmp_replays,
+		   ccmp->dot11rsna_stats_ccmp_decrypt_errors);
 }
 
 static struct lib80211_crypto_ops rtllib_crypt_ccmp = {
@@ -403,13 +395,11 @@ static struct lib80211_crypto_ops rtllib_crypt_ccmp = {
 	.owner			= THIS_MODULE,
 };
 
-
 static int __init rtllib_crypto_ccmp_init(void)
 {
 	return lib80211_register_crypto_ops(&rtllib_crypt_ccmp);
 }
 
-
 static void __exit rtllib_crypto_ccmp_exit(void)
 {
 	lib80211_unregister_crypto_ops(&rtllib_crypt_ccmp);
diff --git a/drivers/staging/rtl8192e/rtllib_rx.c b/drivers/staging/rtl8192e/rtllib_rx.c
index 669e74a67190..b649d02dc5c8 100644
--- a/drivers/staging/rtl8192e/rtllib_rx.c
+++ b/drivers/staging/rtl8192e/rtllib_rx.c
@@ -154,7 +154,6 @@ rtllib_frag_cache_get(struct rtllib_device *ieee,
 	return skb;
 }
 
-
 /* Called only as a tasklet (software IRQ) */
 static int rtllib_frag_cache_invalidate(struct rtllib_device *ieee,
 					   struct rtllib_hdr_4addr *hdr)
@@ -232,10 +231,12 @@ rtllib_rx_frame_mgmt(struct rtllib_device *ieee, struct sk_buff *skb,
 static unsigned char rfc1042_header[] = {
 	0xaa, 0xaa, 0x03, 0x00, 0x00, 0x00
 };
+
 /* Bridge-Tunnel header (for EtherTypes ETH_P_AARP and ETH_P_IPX) */
 static unsigned char bridge_tunnel_header[] = {
 	0xaa, 0xaa, 0x03, 0x00, 0x00, 0xf8
 };
+
 /* No encapsulation header if EtherType < 0x600 (=length) */
 
 /* Called by rtllib_rx_frame_decrypt */
@@ -318,7 +319,6 @@ rtllib_rx_frame_decrypt(struct rtllib_device *ieee, struct sk_buff *skb,
 	return res;
 }
 
-
 /* Called only as a tasklet (software IRQ), by rtllib_rx */
 static inline int
 rtllib_rx_frame_decrypt_msdu(struct rtllib_device *ieee, struct sk_buff *skb,
@@ -355,9 +355,8 @@ rtllib_rx_frame_decrypt_msdu(struct rtllib_device *ieee, struct sk_buff *skb,
 	return 0;
 }
 
-
 /* this function is stolen from ipw2200 driver*/
-#define IEEE_PACKET_RETRY_TIME (5*HZ)
+#define IEEE_PACKET_RETRY_TIME (5 * HZ)
 static int is_duplicate_packet(struct rtllib_device *ieee,
 				      struct rtllib_hdr_4addr *header)
 {
@@ -887,7 +886,6 @@ static u8 parse_subframe(struct rtllib_device *ieee, struct sk_buff *skb,
 	return rxb->nr_subframes;
 }
 
-
 static size_t rtllib_rx_get_hdrlen(struct rtllib_device *ieee,
 				   struct sk_buff *skb,
 				   struct rtllib_rx_stats *rx_stats)
@@ -938,7 +936,7 @@ static int rtllib_rx_check_duplicate(struct rtllib_device *ieee,
 
 		if (GetTs(ieee, (struct ts_common_info **)&pRxTS, hdr->addr2,
 			(u8)Frame_QoSTID((u8 *)(skb->data)), RX_DIR, true)) {
-			if ((fc & (1<<11)) && (frag == pRxTS->rx_last_frag_num) &&
+			if ((fc & (1 << 11)) && (frag == pRxTS->rx_last_frag_num) &&
 			    (WLAN_GET_SEQ_SEQ(sc) == pRxTS->rx_last_seq_num))
 				return -1;
 			pRxTS->rx_last_frag_num = frag;
@@ -1169,7 +1167,6 @@ static int rtllib_rx_decrypt(struct rtllib_device *ieee, struct sk_buff *skb,
 	if (crypt && !(fc & RTLLIB_FCTL_WEP) && !ieee->open_wep) {
 		if (/*ieee->ieee802_1x &&*/
 		    rtllib_is_eapol_frame(ieee, skb, hdrlen)) {
-
 			/* pass unencrypted EAPOL frames even if encryption is
 			 * configured
 			 */
@@ -1209,13 +1206,11 @@ static void rtllib_rx_check_leave_lps(struct rtllib_device *ieee, u8 unicast,
 				      u8 nr_subframes)
 {
 	if (unicast) {
-
 		if (ieee->state == RTLLIB_LINKED) {
 			if (((ieee->link_detect_info.NumRxUnicastOkInPeriod +
 			    ieee->link_detect_info.NumTxOkInPeriod) > 8) ||
 			    (ieee->link_detect_info.NumRxUnicastOkInPeriod > 2)) {
-				if (ieee->LeisurePSLeave)
-					ieee->LeisurePSLeave(ieee->dev);
+				ieee->LeisurePSLeave(ieee->dev);
 			}
 		}
 	}
@@ -1555,7 +1550,6 @@ static u8 qos_oui[QOS_OUI_LEN] = { 0x00, 0x50, 0xF2 };
 static int rtllib_verify_qos_info(struct rtllib_qos_information_element
 				     *info_element, int sub_type)
 {
-
 	if (info_element->elementID != QOS_ELEMENT_ID)
 		return -1;
 	if (info_element->qui_subtype != sub_type)
@@ -1570,7 +1564,6 @@ static int rtllib_verify_qos_info(struct rtllib_qos_information_element
 	return 0;
 }
 
-
 /* Parse a QoS parameter element */
 static int rtllib_read_qos_param_element(
 			struct rtllib_qos_parameter_info *element_param,
@@ -1600,7 +1593,6 @@ static int rtllib_read_qos_info_element(
 	return rtllib_verify_qos_info(element_info, QOS_OUI_INFO_SUB_TYPE);
 }
 
-
 /* Write QoS parameters from the ac parameters. */
 static int rtllib_qos_convert_ac_to_parameters(struct rtllib_qos_parameter_info *param_elm,
 					       struct rtllib_qos_data *qos_data)
@@ -1624,23 +1616,23 @@ static int rtllib_qos_convert_ac_to_parameters(struct rtllib_qos_parameter_info
 		case 1:
 			/* BIT(0) | BIT(3) */
 			if (acm)
-				qos_data->wmm_acm |= (0x01<<0)|(0x01<<3);
+				qos_data->wmm_acm |= (0x01 << 0) | (0x01 << 3);
 			break;
 		case 2:
 			/* BIT(4) | BIT(5) */
 			if (acm)
-				qos_data->wmm_acm |= (0x01<<4)|(0x01<<5);
+				qos_data->wmm_acm |= (0x01 << 4) | (0x01 << 5);
 			break;
 		case 3:
 			/* BIT(6) | BIT(7) */
 			if (acm)
-				qos_data->wmm_acm |= (0x01<<6)|(0x01<<7);
+				qos_data->wmm_acm |= (0x01 << 6) | (0x01 << 7);
 			break;
 		case 0:
 		default:
 			/* BIT(1) | BIT(2) */
 			if (acm)
-				qos_data->wmm_acm |= (0x01<<1)|(0x01<<2);
+				qos_data->wmm_acm |= (0x01 << 1) | (0x01 << 2);
 			break;
 		}
 
@@ -1844,7 +1836,6 @@ static void rtllib_parse_mife_generic(struct rtllib_device *ieee,
 		}
 	}
 
-
 	if (*tmp_htinfo_len == 0) {
 		if (info_element->len >= 4 &&
 		    info_element->data[0] == 0x00 &&
@@ -1933,7 +1924,6 @@ static void rtllib_parse_mife_generic(struct rtllib_device *ieee,
 	    info_element->data[2] == 0x96)
 		network->cisco_cap_exist = true;
 
-
 	if (info_element->len >= 3 &&
 	    info_element->data[0] == 0x00 &&
 	    info_element->data[1] == 0x0a &&
@@ -1986,7 +1976,7 @@ static void rtllib_parse_mife_generic(struct rtllib_device *ieee,
 	    info_element->data[3] == 0x04) {
 		netdev_dbg(ieee->dev, "MFIE_TYPE_WZC: %d bytes\n",
 			   info_element->len);
-		network->wzc_ie_len = min(info_element->len+2, MAX_WZC_IE_LEN);
+		network->wzc_ie_len = min(info_element->len + 2, MAX_WZC_IE_LEN);
 		memcpy(network->wzc_ie, info_element, network->wzc_ie_len);
 	}
 }
@@ -2143,15 +2133,13 @@ int rtllib_parse_info_param(struct rtllib_device *ieee,
 
 			network->dtim_data = RTLLIB_DTIM_VALID;
 
-
 			if (info_element->data[2] & 1)
 				network->dtim_data |= RTLLIB_DTIM_MBCAST;
 
-			offset = (info_element->data[2] >> 1)*2;
-
+			offset = (info_element->data[2] >> 1) * 2;
 
-			if (ieee->assoc_id < 8*offset ||
-			    ieee->assoc_id > 8*(offset + info_element->len - 3))
+			if (ieee->assoc_id < 8 * offset ||
+			    ieee->assoc_id > 8 * (offset + info_element->len - 3))
 				break;
 
 			offset = (ieee->assoc_id / 8) - offset;
@@ -2204,7 +2192,6 @@ int rtllib_parse_info_param(struct rtllib_device *ieee,
 						 &tmp_htcap_len);
 			break;
 
-
 		case MFIE_TYPE_HT_INFO:
 			netdev_dbg(ieee->dev, "MFIE_TYPE_HT_INFO: %d bytes\n",
 				   info_element->len);
@@ -2367,7 +2354,7 @@ static inline int rtllib_network_init(
 	if (rtllib_is_empty_essid(network->ssid, network->ssid_len))
 		network->flags |= NETWORK_EMPTY_ESSID;
 	stats->signal = 30 + (stats->SignalStrength * 70) / 100;
-	stats->noise = rtllib_translate_todbm((u8)(100-stats->signal)) - 25;
+	stats->noise = rtllib_translate_todbm((u8)(100 - stats->signal)) - 25;
 
 	memcpy(&network->stats, stats, sizeof(network->stats));
 
@@ -2393,7 +2380,6 @@ static inline int is_same_network(struct rtllib_network *src,
 		(dst->capability & WLAN_CAPABILITY_ESS)));
 }
 
-
 static inline void update_network(struct rtllib_device *ieee,
 				  struct rtllib_network *dst,
 				  struct rtllib_network *src)
@@ -2556,22 +2542,22 @@ static inline void rtllib_process_probe_response(
 		   "'%s' ( %pM ): %c%c%c%c %c%c%c%c-%c%c%c%c %c%c%c%c\n",
 		   escape_essid(info_element->data, info_element->len),
 		   beacon->header.addr3,
-		   (le16_to_cpu(beacon->capability) & (1<<0xf)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0xe)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0xd)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0xc)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0xb)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0xa)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0x9)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0x8)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0x7)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0x6)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0x5)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0x4)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0x3)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0x2)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0x1)) ? '1' : '0',
-		   (le16_to_cpu(beacon->capability) & (1<<0x0)) ? '1' : '0');
+		   (le16_to_cpu(beacon->capability) & (1 << 0xf)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0xe)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0xd)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0xc)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0xb)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0xa)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0x9)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0x8)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0x7)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0x6)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0x5)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0x4)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0x3)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0x2)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0x1)) ? '1' : '0',
+		   (le16_to_cpu(beacon->capability) & (1 << 0x0)) ? '1' : '0');
 
 	if (rtllib_network_init(ieee, beacon, network, stats)) {
 		netdev_dbg(ieee->dev, "Dropped '%s' ( %pM) via %s.\n",
@@ -2581,7 +2567,6 @@ static inline void rtllib_process_probe_response(
 		goto free_network;
 	}
 
-
 	if (!rtllib_legal_channel(ieee, network->channel))
 		goto free_network;
 
@@ -2689,9 +2674,7 @@ static inline void rtllib_process_probe_response(
 	    is_same_network(&ieee->current_network, network,
 	    (network->ssid_len ? 1 : 0)) &&
 	    (ieee->state == RTLLIB_LINKED)) {
-		if (ieee->handle_beacon != NULL)
-			ieee->handle_beacon(ieee->dev, beacon,
-					    &ieee->current_network);
+		ieee->handle_beacon(ieee->dev, beacon, &ieee->current_network);
 	}
 free_network:
 	kfree(network);
@@ -2710,7 +2693,6 @@ static void rtllib_rx_mgt(struct rtllib_device *ieee,
 		ieee->last_rx_ps_time = jiffies;
 
 	switch (WLAN_FC_GET_STYPE(le16_to_cpu(header->frame_ctl))) {
-
 	case RTLLIB_STYPE_BEACON:
 		netdev_dbg(ieee->dev, "received BEACON (%d)\n",
 			   WLAN_FC_GET_STYPE(le16_to_cpu(header->frame_ctl)));
diff --git a/drivers/staging/rtl8192e/rtllib_softmac.c b/drivers/staging/rtl8192e/rtllib_softmac.c
index 2552aa089700..b9886e83a6dc 100644
--- a/drivers/staging/rtl8192e/rtllib_softmac.c
+++ b/drivers/staging/rtl8192e/rtllib_softmac.c
@@ -659,8 +659,7 @@ static void rtllib_beacons_stop(struct rtllib_device *ieee)
 
 void rtllib_stop_send_beacons(struct rtllib_device *ieee)
 {
-	if (ieee->stop_send_beacons)
-		ieee->stop_send_beacons(ieee->dev);
+	ieee->stop_send_beacons(ieee->dev);
 	if (ieee->softmac_features & IEEE_SOFTMAC_BEACONS)
 		rtllib_beacons_stop(ieee);
 }
@@ -669,8 +668,7 @@ EXPORT_SYMBOL(rtllib_stop_send_beacons);
 
 void rtllib_start_send_beacons(struct rtllib_device *ieee)
 {
-	if (ieee->start_send_beacons)
-		ieee->start_send_beacons(ieee->dev);
+	ieee->start_send_beacons(ieee->dev);
 	if (ieee->softmac_features & IEEE_SOFTMAC_BEACONS)
 		rtllib_beacons_start(ieee);
 }
@@ -729,8 +727,7 @@ EXPORT_SYMBOL(rtllib_act_scanning);
 /* called with ieee->lock held */
 static void rtllib_start_scan(struct rtllib_device *ieee)
 {
-	if (ieee->rtllib_ips_leave_wq != NULL)
-		ieee->rtllib_ips_leave_wq(ieee->dev);
+	ieee->rtllib_ips_leave_wq(ieee->dev);
 
 	if (IS_DOT11D_ENABLE(ieee)) {
 		if (IS_COUNTRY_IE_VALID(ieee))
@@ -1501,7 +1498,7 @@ static void rtllib_associate_step2(struct rtllib_device *ieee)
 static void rtllib_associate_complete_wq(void *data)
 {
 	struct rtllib_device *ieee = (struct rtllib_device *)
-				     container_of_work_rsl(data,
+				     container_of(data,
 				     struct rtllib_device,
 				     associate_complete_wq);
 	struct rt_pwr_save_ctrl *psc = &ieee->pwr_save_ctrl;
@@ -1575,8 +1572,7 @@ static void rtllib_associate_procedure_wq(void *data)
 				     struct rtllib_device,
 				     associate_procedure_wq);
 	rtllib_stop_scan_syncro(ieee);
-	if (ieee->rtllib_ips_leave != NULL)
-		ieee->rtllib_ips_leave(ieee->dev);
+	ieee->rtllib_ips_leave(ieee->dev);
 	mutex_lock(&ieee->wx_mutex);
 
 	if (ieee->data_hard_stop)
@@ -1585,8 +1581,7 @@ static void rtllib_associate_procedure_wq(void *data)
 	rtllib_stop_scan(ieee);
 	HTSetConnectBwMode(ieee, HT_CHANNEL_WIDTH_20, HT_EXTCHNL_OFFSET_NO_EXT);
 	if (ieee->rf_power_state == rf_off) {
-		if (ieee->rtllib_ips_leave_wq != NULL)
-			ieee->rtllib_ips_leave_wq(ieee->dev);
+		ieee->rtllib_ips_leave_wq(ieee->dev);
 		mutex_unlock(&ieee->wx_mutex);
 		return;
 	}
@@ -2241,10 +2236,8 @@ rtllib_rx_assoc_resp(struct rtllib_device *ieee, struct sk_buff *skb,
 				memcpy(ieee->ht_info->PeerHTInfoBuf,
 				       network->bssht.bd_ht_info_buf,
 				       network->bssht.bd_ht_info_len);
-				if (ieee->handle_assoc_response != NULL)
-					ieee->handle_assoc_response(ieee->dev,
-						 (struct rtllib_assoc_response_frame *)header,
-						 network);
+				ieee->handle_assoc_response(ieee->dev,
+					(struct rtllib_assoc_response_frame *)header, network);
 			}
 			kfree(network);
 
@@ -2856,8 +2849,7 @@ void rtllib_stop_protocol(struct rtllib_device *ieee, u8 shutdown)
 	if (shutdown) {
 		ieee->proto_started = 0;
 		ieee->proto_stoppping = 1;
-		if (ieee->rtllib_ips_leave != NULL)
-			ieee->rtllib_ips_leave(ieee->dev);
+		ieee->rtllib_ips_leave(ieee->dev);
 	}
 
 	rtllib_stop_send_beacons(ieee);
@@ -3004,20 +2996,13 @@ int rtllib_softmac_init(struct rtllib_device *ieee)
 
 	timer_setup(&ieee->beacon_timer, rtllib_send_beacon_cb, 0);
 
-	INIT_DELAYED_WORK_RSL(&ieee->link_change_wq,
-			      (void *)rtllib_link_change_wq, ieee);
-	INIT_DELAYED_WORK_RSL(&ieee->start_ibss_wq,
-			      (void *)rtllib_start_ibss_wq, ieee);
-	INIT_WORK_RSL(&ieee->associate_complete_wq,
-		      (void *)rtllib_associate_complete_wq, ieee);
-	INIT_DELAYED_WORK_RSL(&ieee->associate_procedure_wq,
-			      (void *)rtllib_associate_procedure_wq, ieee);
-	INIT_DELAYED_WORK_RSL(&ieee->softmac_scan_wq,
-			      (void *)rtllib_softmac_scan_wq, ieee);
-	INIT_DELAYED_WORK_RSL(&ieee->associate_retry_wq,
-			      (void *)rtllib_associate_retry_wq, ieee);
-	INIT_WORK_RSL(&ieee->wx_sync_scan_wq, (void *)rtllib_wx_sync_scan_wq,
-		      ieee);
+	INIT_DELAYED_WORK(&ieee->link_change_wq, (void *)rtllib_link_change_wq);
+	INIT_DELAYED_WORK(&ieee->start_ibss_wq, (void *)rtllib_start_ibss_wq);
+	INIT_WORK(&ieee->associate_complete_wq, (void *)rtllib_associate_complete_wq);
+	INIT_DELAYED_WORK(&ieee->associate_procedure_wq, (void *)rtllib_associate_procedure_wq);
+	INIT_DELAYED_WORK(&ieee->softmac_scan_wq, (void *)rtllib_softmac_scan_wq);
+	INIT_DELAYED_WORK(&ieee->associate_retry_wq, (void *)rtllib_associate_retry_wq);
+	INIT_WORK(&ieee->wx_sync_scan_wq, (void *)rtllib_wx_sync_scan_wq);
 
 	mutex_init(&ieee->wx_mutex);
 	mutex_init(&ieee->scan_mutex);
diff --git a/drivers/staging/rtl8192e/rtllib_softmac_wx.c b/drivers/staging/rtl8192e/rtllib_softmac_wx.c
index 06f3d75dc102..1f2fa711e60b 100644
--- a/drivers/staging/rtl8192e/rtllib_softmac_wx.c
+++ b/drivers/staging/rtl8192e/rtllib_softmac_wx.c
@@ -326,8 +326,7 @@ EXPORT_SYMBOL(rtllib_wx_set_mode);
 
 void rtllib_wx_sync_scan_wq(void *data)
 {
-	struct rtllib_device *ieee = container_of_work_rsl(data,
-				     struct rtllib_device, wx_sync_scan_wq);
+	struct rtllib_device *ieee = container_of(data, struct rtllib_device, wx_sync_scan_wq);
 	short chan;
 	enum ht_extchnl_offset chan_offset = 0;
 	enum ht_channel_width bandwidth = 0;
@@ -340,8 +339,7 @@ void rtllib_wx_sync_scan_wq(void *data)
 
 	chan = ieee->current_network.channel;
 
-	if (ieee->LeisurePSLeave)
-		ieee->LeisurePSLeave(ieee->dev);
+	ieee->LeisurePSLeave(ieee->dev);
 	/* notify AP to be in PS mode */
 	rtllib_sta_ps_send_null_frame(ieee, 1);
 	rtllib_sta_ps_send_null_frame(ieee, 1);
@@ -356,8 +354,7 @@ void rtllib_wx_sync_scan_wq(void *data)
 	/* wait for ps packet to be kicked out successfully */
 	msleep(50);
 
-	if (ieee->ScanOperationBackupHandler)
-		ieee->ScanOperationBackupHandler(ieee->dev, SCAN_OPT_BACKUP);
+	ieee->ScanOperationBackupHandler(ieee->dev, SCAN_OPT_BACKUP);
 
 	if (ieee->ht_info->bCurrentHTSupport && ieee->ht_info->enable_ht &&
 	    ieee->ht_info->bCurBW40MHz) {
@@ -382,8 +379,7 @@ void rtllib_wx_sync_scan_wq(void *data)
 		ieee->set_chan(ieee->dev, chan);
 	}
 
-	if (ieee->ScanOperationBackupHandler)
-		ieee->ScanOperationBackupHandler(ieee->dev, SCAN_OPT_RESTORE);
+	ieee->ScanOperationBackupHandler(ieee->dev, SCAN_OPT_RESTORE);
 
 	ieee->state = RTLLIB_LINKED;
 	ieee->link_change(ieee->dev);
diff --git a/drivers/staging/rtl8192e/rtllib_wx.c b/drivers/staging/rtl8192e/rtllib_wx.c
index 217426ee2e92..d6691f3c7c70 100644
--- a/drivers/staging/rtl8192e/rtllib_wx.c
+++ b/drivers/staging/rtl8192e/rtllib_wx.c
@@ -41,8 +41,7 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 	iwe.cmd = SIOCGIWAP;
 	iwe.u.ap_addr.sa_family = ARPHRD_ETHER;
 	ether_addr_copy(iwe.u.ap_addr.sa_data, network->bssid);
-	start = iwe_stream_add_event_rsl(info, start, stop,
-					 &iwe, IW_EV_ADDR_LEN);
+	start = iwe_stream_add_event(info, start, stop, &iwe, IW_EV_ADDR_LEN);
 	/* Remaining entries will be displayed in the order we provide them */
 
 	/* Add the ESSID */
@@ -50,16 +49,13 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 	iwe.u.data.flags = 1;
 	if (network->ssid_len > 0) {
 		iwe.u.data.length = min_t(u8, network->ssid_len, 32);
-		start = iwe_stream_add_point_rsl(info, start, stop, &iwe,
-						 network->ssid);
+		start = iwe_stream_add_point(info, start, stop, &iwe, network->ssid);
 	} else if (network->hidden_ssid_len == 0) {
 		iwe.u.data.length = sizeof("<hidden>");
-		start = iwe_stream_add_point_rsl(info, start, stop,
-						 &iwe, "<hidden>");
+		start = iwe_stream_add_point(info, start, stop, &iwe, "<hidden>");
 	} else {
 		iwe.u.data.length = min_t(u8, network->hidden_ssid_len, 32);
-		start = iwe_stream_add_point_rsl(info, start, stop, &iwe,
-						 network->hidden_ssid);
+		start = iwe_stream_add_point(info, start, stop, &iwe, network->hidden_ssid);
 	}
 	/* Add the protocol name */
 	iwe.cmd = SIOCGIWNAME;
@@ -71,8 +67,7 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 	}
 	*pname = '\0';
 	snprintf(iwe.u.name, IFNAMSIZ, "IEEE802.11%s", proto_name);
-	start = iwe_stream_add_event_rsl(info, start, stop,
-					 &iwe, IW_EV_CHAR_LEN);
+	start = iwe_stream_add_event(info, start, stop, &iwe, IW_EV_CHAR_LEN);
 	/* Add mode */
 	iwe.cmd = SIOCGIWMODE;
 	if (network->capability &
@@ -81,8 +76,7 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 			iwe.u.mode = IW_MODE_MASTER;
 		else
 			iwe.u.mode = IW_MODE_ADHOC;
-		start = iwe_stream_add_event_rsl(info, start, stop,
-						 &iwe, IW_EV_UINT_LEN);
+		start = iwe_stream_add_event(info, start, stop, &iwe, IW_EV_UINT_LEN);
 	}
 
 	/* Add frequency/channel */
@@ -90,8 +84,7 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 	iwe.u.freq.m = network->channel;
 	iwe.u.freq.e = 0;
 	iwe.u.freq.i = 0;
-	start = iwe_stream_add_event_rsl(info, start, stop, &iwe,
-					 IW_EV_FREQ_LEN);
+	start = iwe_stream_add_event(info, start, stop, &iwe, IW_EV_FREQ_LEN);
 
 	/* Add encryption capability */
 	iwe.cmd = SIOCGIWENCODE;
@@ -100,8 +93,7 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 	else
 		iwe.u.data.flags = IW_ENCODE_DISABLED;
 	iwe.u.data.length = 0;
-	start = iwe_stream_add_point_rsl(info, start, stop,
-					 &iwe, network->ssid);
+	start = iwe_stream_add_point(info, start, stop, &iwe, network->ssid);
 	/* Add basic and extended rates */
 	max_rate = 0;
 	p = custom;
@@ -152,12 +144,11 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 	iwe.u.bitrate.disabled = 0;
 	iwe.u.bitrate.fixed = 0;
 	iwe.u.bitrate.value = max_rate * 500000;
-	start = iwe_stream_add_event_rsl(info, start, stop, &iwe, IW_EV_PARAM_LEN);
+	start = iwe_stream_add_event(info, start, stop, &iwe, IW_EV_PARAM_LEN);
 	iwe.cmd = IWEVCUSTOM;
 	iwe.u.data.length = p - custom;
 	if (iwe.u.data.length)
-		start = iwe_stream_add_point_rsl(info, start, stop,
-						 &iwe, custom);
+		start = iwe_stream_add_point(info, start, stop, &iwe, custom);
 	/* Add quality statistics */
 	/* TODO: Fix these values... */
 	iwe.cmd = IWEVQUAL;
@@ -172,13 +163,13 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 	if (!(network->stats.mask & RTLLIB_STATMASK_SIGNAL))
 		iwe.u.qual.updated |= IW_QUAL_QUAL_INVALID;
 	iwe.u.qual.updated = 7;
-	start = iwe_stream_add_event_rsl(info, start, stop, &iwe, IW_EV_QUAL_LEN);
+	start = iwe_stream_add_event(info, start, stop, &iwe, IW_EV_QUAL_LEN);
 
 	iwe.cmd = IWEVCUSTOM;
 	p = custom;
 	iwe.u.data.length = p - custom;
 	if (iwe.u.data.length)
-		start = iwe_stream_add_point_rsl(info, start, stop, &iwe, custom);
+		start = iwe_stream_add_point(info, start, stop, &iwe, custom);
 
 	memset(&iwe, 0, sizeof(iwe));
 	if (network->wpa_ie_len) {
@@ -187,7 +178,7 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 		memcpy(buf, network->wpa_ie, network->wpa_ie_len);
 		iwe.cmd = IWEVGENIE;
 		iwe.u.data.length = network->wpa_ie_len;
-		start = iwe_stream_add_point_rsl(info, start, stop, &iwe, buf);
+		start = iwe_stream_add_point(info, start, stop, &iwe, buf);
 	}
 	memset(&iwe, 0, sizeof(iwe));
 	if (network->rsn_ie_len) {
@@ -196,7 +187,7 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 		memcpy(buf, network->rsn_ie, network->rsn_ie_len);
 		iwe.cmd = IWEVGENIE;
 		iwe.u.data.length = network->rsn_ie_len;
-		start = iwe_stream_add_point_rsl(info, start, stop, &iwe, buf);
+		start = iwe_stream_add_point(info, start, stop, &iwe, buf);
 	}
 
 	/* add info for WZC */
@@ -207,7 +198,7 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 		memcpy(buf, network->wzc_ie, network->wzc_ie_len);
 		iwe.cmd = IWEVGENIE;
 		iwe.u.data.length = network->wzc_ie_len;
-		start = iwe_stream_add_point_rsl(info, start, stop, &iwe, buf);
+		start = iwe_stream_add_point(info, start, stop, &iwe, buf);
 	}
 
 	/* Add EXTRA: Age to display seconds since last beacon/probe response
@@ -220,8 +211,7 @@ static inline char *rtl819x_translate_scan(struct rtllib_device *ieee,
 		      (100 * (jiffies - network->last_scanned)) / HZ);
 	iwe.u.data.length = p - custom;
 	if (iwe.u.data.length)
-		start = iwe_stream_add_point_rsl(info, start, stop,
-						 &iwe, custom);
+		start = iwe_stream_add_point(info, start, stop, &iwe, custom);
 
 	return start;
 }
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
index 5c73e3f8541a..ca09367005e1 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
@@ -1958,43 +1958,6 @@ int ieee80211_parse_info_param(struct ieee80211_device *ieee,
 	return 0;
 }
 
-static inline u8 ieee80211_SignalStrengthTranslate(
-	u8  CurrSS
-	)
-{
-	u8 RetSS;
-
-	// Step 1. Scale mapping.
-	if (CurrSS >= 71 && CurrSS <= 100) {
-		RetSS = 90 + ((CurrSS - 70) / 3);
-	} else if (CurrSS >= 41 && CurrSS <= 70) {
-		RetSS = 78 + ((CurrSS - 40) / 3);
-	} else if (CurrSS >= 31 && CurrSS <= 40) {
-		RetSS = 66 + (CurrSS - 30);
-	} else if (CurrSS >= 21 && CurrSS <= 30) {
-		RetSS = 54 + (CurrSS - 20);
-	} else if (CurrSS >= 5 && CurrSS <= 20) {
-		RetSS = 42 + (((CurrSS - 5) * 2) / 3);
-	} else if (CurrSS == 4) {
-		RetSS = 36;
-	} else if (CurrSS == 3) {
-		RetSS = 27;
-	} else if (CurrSS == 2) {
-		RetSS = 18;
-	} else if (CurrSS == 1) {
-		RetSS = 9;
-	} else {
-		RetSS = CurrSS;
-	}
-	//RT_TRACE(COMP_DBG, DBG_LOUD, ("##### After Mapping:  LastSS: %d, CurrSS: %d, RetSS: %d\n", LastSS, CurrSS, RetSS));
-
-	// Step 2. Smoothing.
-
-	//RT_TRACE(COMP_DBG, DBG_LOUD, ("$$$$$ After Smoothing:  LastSS: %d, CurrSS: %d, RetSS: %d\n", LastSS, CurrSS, RetSS));
-
-	return RetSS;
-}
-
 /* 0-100 index */
 static long ieee80211_translate_todbm(u8 signal_strength_index)
 {
@@ -2095,7 +2058,6 @@ static inline int ieee80211_network_init(
 		network->flags |= NETWORK_EMPTY_ESSID;
 
 	stats->signal = 30 + (stats->SignalStrength * 70) / 100;
-	//stats->signal = ieee80211_SignalStrengthTranslate(stats->signal);
 	stats->noise = ieee80211_translate_todbm((u8)(100 - stats->signal)) - 25;
 
 	memcpy(&network->stats, stats, sizeof(network->stats));
diff --git a/drivers/staging/rtl8192u/r8192U_dm.c b/drivers/staging/rtl8192u/r8192U_dm.c
index 00fc8fd344db..cbae852478ea 100644
--- a/drivers/staging/rtl8192u/r8192U_dm.c
+++ b/drivers/staging/rtl8192u/r8192U_dm.c
@@ -1,19 +1,14 @@
 // SPDX-License-Identifier: GPL-2.0
 /*++
-Copyright-c Realtek Semiconductor Corp. All rights reserved.
-
-Module Name:
-	r8192U_dm.c
-
-Abstract:
-	HW dynamic mechanism.
-
-Major Change History:
-	When		Who				What
-	----------	--------------- -------------------------------
-	2008-05-14	amy                     create version 0 porting from windows code.
-
---*/
+ * Copyright-c Realtek Semiconductor Corp. All rights reserved.
+ *
+ * Module Name:
+ *	r8192U_dm.c
+ *
+ * Abstract:
+ *	HW dynamic mechanism.
+ *--
+ */
 #include "r8192U.h"
 #include "r8192U_dm.h"
 #include "r8192U_hw.h"
@@ -54,12 +49,8 @@ static	void	dm_init_bandwidth_autoswitch(struct net_device *dev);
 static	void	dm_bandwidth_autoswitch(struct net_device *dev);
 
 /* DM --> TX power control */
-/*static	void	dm_initialize_txpower_tracking(struct net_device *dev);*/
-
 static	void	dm_check_txpower_tracking(struct net_device *dev);
 
-/*static	void	dm_txpower_reset_recovery(struct net_device *dev);*/
-
 /* DM --> Dynamic Init Gain by RSSI */
 static	void	dm_dig_init(struct net_device *dev);
 static	void	dm_ctrl_initgain_byrssi(struct net_device *dev);
@@ -74,7 +65,6 @@ static	void dm_init_ctstoself(struct net_device *dev);
 /* DM --> EDCA turbo mode control */
 static	void	dm_check_edca_turbo(struct net_device *dev);
 
-/*static	void	dm_gpio_change_rf(struct net_device *dev);*/
 /* DM --> Check PBC */
 static	void dm_check_pbc_gpio(struct net_device *dev);
 
@@ -121,7 +111,6 @@ void init_hal_dm(struct net_device *dev)
 	/* Initial TX Power Control for near/far range , add by amy 2008/05/15, porting from windows code. */
 	dm_init_dynamic_txpower(dev);
 	init_rate_adaptive(dev);
-	/*dm_initialize_txpower_tracking(dev);*/
 	dm_dig_init(dev);
 	dm_init_edca_turbo(dev);
 	dm_init_bandwidth_autoswitch(dev);
@@ -146,21 +135,6 @@ void dm_CheckRxAggregation(struct net_device *dev)
 	unsigned long		curTxOkCnt = 0;
 	unsigned long		curRxOkCnt = 0;
 
-/*
-	if (pHalData->bForcedUsbRxAggr) {
-		if (pHalData->ForcedUsbRxAggrInfo == 0) {
-			if (pHalData->bCurrentRxAggrEnable) {
-				Adapter->HalFunc.HalUsbRxAggrHandler(Adapter, FALSE);
-			}
-		} else {
-			if (!pHalData->bCurrentRxAggrEnable || (pHalData->ForcedUsbRxAggrInfo != pHalData->LastUsbRxAggrInfoSetting)) {
-				Adapter->HalFunc.HalUsbRxAggrHandler(Adapter, TRUE);
-			}
-		}
-		return;
-	}
-
-*/
 	curTxOkCnt = priv->stats.txbytesunicast - lastTxOkCnt;
 	curRxOkCnt = priv->stats.rxbytesunicast - lastRxOkCnt;
 
@@ -194,10 +168,6 @@ void dm_CheckRxAggregation(struct net_device *dev)
 
 void hal_dm_watchdog(struct net_device *dev)
 {
-	/*struct r8192_priv *priv = ieee80211_priv(dev);*/
-
-	/*static u8	previous_bssid[6] ={0};*/
-
 	/*Add by amy 2008/05/15 ,porting from windows code.*/
 	dm_check_rate_adaptive(dev);
 	dm_dynamic_txpower(dev);
@@ -274,12 +244,8 @@ void init_rate_adaptive(struct net_device *dev)
  * Output:		NONE
  *
  * Return:		NONE
- *
- * Revised History:
- *	When		Who		Remark
- *	05/26/08	amy	Create version 0 porting from windows code.
- *
- *---------------------------------------------------------------------------*/
+ *---------------------------------------------------------------------------
+ */
 static void dm_check_rate_adaptive(struct net_device *dev)
 {
 	struct r8192_priv *priv = ieee80211_priv(dev);
@@ -304,7 +270,6 @@ static void dm_check_rate_adaptive(struct net_device *dev)
 		return;
 
 	if (priv->ieee80211->state == IEEE80211_LINKED) {
-		/*RT_TRACE(COMP_RATE, "dm_CheckRateAdaptive(): \t");*/
 
 		/* Check whether Short GI is enabled */
 		bshort_gi_enabled = (pHTInfo->bCurTxBW40MHz && pHTInfo->bCurShortGI40MHz) ||
@@ -351,36 +316,27 @@ static void dm_check_rate_adaptive(struct net_device *dev)
 					(pra->low_rssi_thresh_for_ra40M):(pra->low_rssi_thresh_for_ra20M);
 		}
 
-		/*DbgPrint("[DM] THresh H/L=%d/%d\n\r", RATR.HighRSSIThreshForRA, RATR.LowRSSIThreshForRA);*/
 		if (priv->undecorated_smoothed_pwdb >= (long)HighRSSIThreshForRA) {
-			/*DbgPrint("[DM] RSSI=%d STA=HIGH\n\r", pHalData->UndecoratedSmoothedPWDB);*/
 			pra->ratr_state = DM_RATR_STA_HIGH;
 			targetRATR = pra->upper_rssi_threshold_ratr;
 		} else if (priv->undecorated_smoothed_pwdb >= (long)LowRSSIThreshForRA) {
-			/*DbgPrint("[DM] RSSI=%d STA=Middle\n\r", pHalData->UndecoratedSmoothedPWDB);*/
 			pra->ratr_state = DM_RATR_STA_MIDDLE;
 			targetRATR = pra->middle_rssi_threshold_ratr;
 		} else {
-			/*DbgPrint("[DM] RSSI=%d STA=LOW\n\r", pHalData->UndecoratedSmoothedPWDB);*/
 			pra->ratr_state = DM_RATR_STA_LOW;
 			targetRATR = pra->low_rssi_threshold_ratr;
 		}
 
 		/* cosa add for test */
 		if (pra->ping_rssi_enable) {
-			/*pHalData->UndecoratedSmoothedPWDB = 19;*/
 			if (priv->undecorated_smoothed_pwdb < (long)(pra->ping_rssi_thresh_for_ra+5)) {
 				if ((priv->undecorated_smoothed_pwdb < (long)pra->ping_rssi_thresh_for_ra) ||
 					ping_rssi_state) {
-					/*DbgPrint("TestRSSI = %d, set RATR to 0x%x\n", pHalData->UndecoratedSmoothedPWDB, pRA->TestRSSIRATR);*/
 					pra->ratr_state = DM_RATR_STA_LOW;
 					targetRATR = pra->ping_rssi_ratr;
 					ping_rssi_state = 1;
 				}
-				/*else
-					DbgPrint("TestRSSI is between the range.\n");*/
 			} else {
-				/*DbgPrint("TestRSSI Recover to 0x%x\n", targetRATR);*/
 				ping_rssi_state = 0;
 			}
 		}
@@ -502,7 +458,6 @@ static void dm_TXPowerTrackingCallback_TSSI(struct net_device *dev)
 	u32						Value;
 	u8						Pwr_Flag;
 	u16						Avg_TSSI_Meas, TSSI_13dBm, Avg_TSSI_Meas_from_driver = 0;
-	/*RT_STATUS				rtStatus = RT_STATUS_SUCCESS;*/
 	bool rtStatus = true;
 	u32						delta = 0;
 
@@ -524,7 +479,6 @@ static void dm_TXPowerTrackingCallback_TSSI(struct net_device *dev)
 		if (rtStatus == RT_STATUS_FAILURE)
 			RT_TRACE(COMP_POWER_TRACKING, "Set configuration with tx cmd queue fail!\n");
 		usleep_range(1000, 2000);
-		/*DbgPrint("hi, vivi, strange\n");*/
 		for (i = 0; i <= 30; i++) {
 			read_nic_byte(dev, 0x1ba, &Pwr_Flag);
 
@@ -570,8 +524,6 @@ static void dm_TXPowerTrackingCallback_TSSI(struct net_device *dev)
 			TSSI_13dBm = priv->TSSI_13dBm;
 			RT_TRACE(COMP_POWER_TRACKING, "TSSI_13dBm = %d\n", TSSI_13dBm);
 
-			/*if (abs(Avg_TSSI_Meas_from_driver - TSSI_13dBm) <= E_FOR_TX_POWER_TRACK)*/
-			/* For MacOS-compatible */
 			if (Avg_TSSI_Meas_from_driver > TSSI_13dBm)
 				delta = Avg_TSSI_Meas_from_driver - TSSI_13dBm;
 			else
@@ -675,7 +627,6 @@ static void dm_TXPowerTrackingCallback_ThermalMeter(struct net_device *dev)
 			}
 		}
 		priv->btxpower_trackingInit = true;
-		/*pHalData->TXPowercount = 0;*/
 		return;
 	}
 
@@ -719,9 +670,6 @@ static void dm_TXPowerTrackingCallback_ThermalMeter(struct net_device *dev)
 		}
 		tmpCCK40Mindex = 0;
 	}
-	/*DbgPrint("%ddb, tmpOFDMindex = %d, tmpCCK20Mindex = %d, tmpCCK40Mindex = %d",
-		((u1Byte)tmpRegA - pHalData->ThermalMeter[0]),
-		tmpOFDMindex, tmpCCK20Mindex, tmpCCK40Mindex);*/
 	if (priv->CurrentChannelBW != HT_CHANNEL_WIDTH_20)	/* 40M */
 		tmpCCKindex = tmpCCK40Mindex;
 	else
@@ -741,7 +689,6 @@ static void dm_TXPowerTrackingCallback_ThermalMeter(struct net_device *dev)
 	}
 
 	if (CCKSwingNeedUpdate) {
-		/*DbgPrint("Update CCK Swing, CCK_index = %d\n", pHalData->CCK_index);*/
 		dm_cck_txpower_adjust(dev, priv->bcck_in_ch14);
 	}
 	if (priv->OFDM_index != tmpOFDMindex) {
@@ -1312,7 +1259,6 @@ static void dm_CheckTXPowerTracking_ThermalMeter(struct net_device *dev)
 {
 	struct r8192_priv *priv = ieee80211_priv(dev);
 	static u8	TM_Trigger;
-	/*DbgPrint("dm_CheckTXPowerTracking()\n");*/
 	if (!priv->btxpower_tracking)
 		return;
 	if (priv->txpower_count  <= 2) {
@@ -1332,7 +1278,6 @@ static void dm_CheckTXPowerTracking_ThermalMeter(struct net_device *dev)
 		TM_Trigger = 1;
 		return;
 	}
-	/*DbgPrint("Schedule TxPowerTrackingWorkItem\n");*/
 	queue_delayed_work(priv->priv_wq, &priv->txpower_tracking_wq, 0);
 	TM_Trigger = 0;
 }
@@ -1340,7 +1285,6 @@ static void dm_CheckTXPowerTracking_ThermalMeter(struct net_device *dev)
 static void dm_check_txpower_tracking(struct net_device *dev)
 {
 	struct r8192_priv *priv = ieee80211_priv(dev);
-	/*static u32 tx_power_track_counter = 0;*/
 
 #ifdef RTL8190P
 	dm_CheckTXPowerTracking_TSSI(dev);
@@ -1425,7 +1369,6 @@ static void dm_CCKTxPowerAdjust_ThermalMeter(struct net_device *dev, bool  bInCH
 		RT_TRACE(COMP_POWER_TRACKING, "CCK not chnl 14, reg 0x%x = 0x%x\n",
 			rCCK0_DebugPort, TempVal);
 	} else {
-		/*priv->CCKTxPowerAdjustCntNotCh14++;	cosa add for debug.*/
 		/* Write 0xa22 0xa23 */
 		TempVal =	CCKSwingTable_Ch14[priv->CCK_index][0] +
 					(CCKSwingTable_Ch14[priv->CCK_index][1]<<8);
@@ -1508,10 +1451,7 @@ void dm_restore_dynamic_mechanism_state(struct net_device *dev)
 			ratr_value = reg_ratr;
 			if (priv->rf_type == RF_1T2R) {	/* 1T2R, Spatial Stream 2 should be disabled */
 				ratr_value &= ~(RATE_ALL_OFDM_2SS);
-				/*DbgPrint("HW_VAR_TATR_0 from 0x%x ==> 0x%x\n", ((pu4Byte)(val))[0], ratr_value);*/
 			}
-			/*DbgPrint("set HW_VAR_TATR_0 = 0x%x\n", ratr_value);*/
-			/*cosa PlatformEFIOWrite4Byte(Adapter, RATR0, ((pu4Byte)(val))[0]);*/
 			write_nic_dword(dev, RATR0, ratr_value);
 			write_nic_byte(dev, UFWP, 1);
 	}
@@ -1533,7 +1473,6 @@ static void dm_bb_initialgain_restore(struct net_device *dev)
 		return;
 
 	/* Disable Initial Gain */
-	/*PHY_SetBBReg(Adapter, UFWP, bMaskLWord, 0x800);*/
 	rtl8192_setBBreg(dev, UFWP, bMaskByte1, 0x8);	/* Only clear byte 1 and rewrite. */
 	rtl8192_setBBreg(dev, rOFDM0_XAAGCCore1, bit_mask, (u32)priv->initgain_backup.xaagccore1);
 	rtl8192_setBBreg(dev, rOFDM0_XBAGCCore1, bit_mask, (u32)priv->initgain_backup.xbagccore1);
@@ -1548,7 +1487,6 @@ static void dm_bb_initialgain_restore(struct net_device *dev)
 	RT_TRACE(COMP_DIG, "dm_BBInitialGainRestore 0xc68 is %x\n", priv->initgain_backup.xdagccore1);
 	RT_TRACE(COMP_DIG, "dm_BBInitialGainRestore 0xa0a is %x\n", priv->initgain_backup.cca);
 	/* Enable Initial Gain */
-	/*PHY_SetBBReg(Adapter, UFWP, bMaskLWord, 0x100);*/
 	rtl8192_setBBreg(dev, UFWP, bMaskByte1, 0x1);	/* Only clear byte 1 and rewrite. */
 
 }	/* dm_BBInitialGainRestore */
@@ -1561,7 +1499,6 @@ static void dm_bb_initialgain_backup(struct net_device *dev)
 	if (dm_digtable.dig_algorithm == DIG_ALGO_BY_RSSI)
 		return;
 
-	/*PHY_SetBBReg(Adapter, UFWP, bMaskLWord, 0x800);*/
 	rtl8192_setBBreg(dev, UFWP, bMaskByte1, 0x8);	/* Only clear byte 1 and rewrite. */
 	priv->initgain_backup.xaagccore1 = (u8)rtl8192_QueryBBReg(dev, rOFDM0_XAAGCCore1, bit_mask);
 	priv->initgain_backup.xbagccore1 = (u8)rtl8192_QueryBBReg(dev, rOFDM0_XBAGCCore1, bit_mask);
@@ -1589,12 +1526,8 @@ static void dm_bb_initialgain_backup(struct net_device *dev)
  * Output:		NONE
  *
  * Return:		NONE
- *
- * Revised History:
- *	When		Who		Remark
- *	05/15/2008	amy		Create Version 0 porting from windows code.
- *
- *---------------------------------------------------------------------------*/
+ *---------------------------------------------------------------------------
+ */
 static void dm_dig_init(struct net_device *dev)
 {
 	struct r8192_priv *priv = ieee80211_priv(dev);
@@ -1634,11 +1567,8 @@ static void dm_dig_init(struct net_device *dev)
  * Output:		NONE
  *
  * Return:		NONE
- *
- * Revised History:
- *	When		Who		Remark
- *	05/27/2008	amy		Create Version 0 porting from windows code.
- *---------------------------------------------------------------------------*/
+ *---------------------------------------------------------------------------
+ */
 static void dm_ctrl_initgain_byrssi(struct net_device *dev)
 {
 	if (!dm_digtable.dig_enable_flag)
@@ -1663,7 +1593,6 @@ static void dm_ctrl_initgain_byrssi_by_driverrssi(
 	if (!dm_digtable.dig_enable_flag)
 		return;
 
-	/*DbgPrint("Dig by Sw Rssi\n");*/
 	if (dm_digtable.dig_algorithm_switch)	/* if switched algorithm, we have to disable FW Dig. */
 		fw_dig = 0;
 
@@ -1680,11 +1609,7 @@ static void dm_ctrl_initgain_byrssi_by_driverrssi(
 	else
 		dm_digtable.cur_connect_state = DIG_DISCONNECT;
 
-	/*DbgPrint("DM_DigTable.PreConnectState = %d, DM_DigTable.CurConnectState = %d\n",
-		DM_DigTable.PreConnectState, DM_DigTable.CurConnectState);*/
-
 	dm_digtable.rssi_val = priv->undecorated_smoothed_pwdb;
-	/*DbgPrint("DM_DigTable.Rssi_val = %d\n", DM_DigTable.Rssi_val);*/
 	dm_initial_gain(dev);
 	dm_pd_th(dev);
 	dm_cs_ratio(dev);
@@ -1720,11 +1645,6 @@ static void dm_ctrl_initgain_byrssi_by_fwfalse_alarm(
 	    (priv->undecorated_smoothed_pwdb < dm_digtable.rssi_high_thresh))
 		return;
 
-	/*DbgPrint("Dig by Fw False Alarm\n");*/
-	/*if (DM_DigTable.Dig_State == DM_STA_DIG_OFF)*/
-	/*DbgPrint("DIG Check\n\r RSSI=%d LOW=%d HIGH=%d STATE=%d",
-	pHalData->UndecoratedSmoothedPWDB, DM_DigTable.RssiLowThresh,
-	DM_DigTable.RssiHighThresh, DM_DigTable.Dig_State);*/
 	/* 1. When RSSI decrease, We have to judge if it is smaller than a threshold
 	 * and then execute the step below.
 	 */
@@ -1757,12 +1677,6 @@ static void dm_ctrl_initgain_byrssi_by_fwfalse_alarm(
 			 * 2008/02/05 MH SD3-Jerry 92U/92E PD_TH are the same.
 			 */
 			write_nic_byte(dev, (rOFDM0_XATxAFE+3), 0x00);
-			/*else if (priv->card_8192 == HARDWARE_TYPE_RTL8190P)
-				write_nic_byte(pAdapter, rOFDM0_RxDetector1, 0x40);
-			else if (pAdapter->HardwareType == HARDWARE_TYPE_RTL8192E)
-			else
-				PlatformEFIOWrite1Byte(pAdapter, rOFDM0_RxDetector1, 0x40);
-			*/
 		} else
 			write_nic_byte(dev, rOFDM0_RxDetector1, 0x42);
 
@@ -1770,7 +1684,6 @@ static void dm_ctrl_initgain_byrssi_by_fwfalse_alarm(
 		write_nic_byte(dev, 0xa0a, 0x08);
 
 		/* 1.5 Higher EDCCA. */
-		/*PlatformEFIOWrite4Byte(pAdapter, rOFDM0_ECCAThreshold, 0x325);*/
 		return;
 	}
 
@@ -1791,7 +1704,6 @@ static void dm_ctrl_initgain_byrssi_by_fwfalse_alarm(
 		reset_cnt = priv->reset_count;
 
 		dm_digtable.dig_state = DM_STA_DIG_ON;
-		/*DbgPrint("DIG ON\n\r");*/
 
 		/* 2.1 Set initial gain.
 		 * 2008/02/26 MH SD3-Jerry suggest to prevent dirty environment.
@@ -1814,13 +1726,6 @@ static void dm_ctrl_initgain_byrssi_by_fwfalse_alarm(
 			 * 2008/02/05 MH SD3-Jerry 92U/92E PD_TH are the same.
 			 */
 			write_nic_byte(dev, (rOFDM0_XATxAFE+3), 0x20);
-			/*
-			else if (priv->card_8192 == HARDWARE_TYPE_RTL8190P)
-				write_nic_byte(dev, rOFDM0_RxDetector1, 0x42);
-			else if (pAdapter->HardwareType == HARDWARE_TYPE_RTL8192E)
-			else
-				PlatformEFIOWrite1Byte(pAdapter, rOFDM0_RxDetector1, 0x42);
-			*/
 		} else
 			write_nic_byte(dev, rOFDM0_RxDetector1, 0x44);
 
@@ -1830,7 +1735,6 @@ static void dm_ctrl_initgain_byrssi_by_fwfalse_alarm(
 		/* 2.4 Lower EDCCA.
 		 * 2008/01/11 MH 90/92 series are the same.
 		 */
-		/*PlatformEFIOWrite4Byte(pAdapter, rOFDM0_ECCAThreshold, 0x346);*/
 
 		/* 2.5 DIG On. */
 		rtl8192_setBBreg(dev, UFWP, bMaskByte1, 0x1);	/*  Only clear byte 1 and rewrite. */
@@ -1850,12 +1754,8 @@ static void dm_ctrl_initgain_byrssi_by_fwfalse_alarm(
  * Output:		NONE
  *
  * Return:		NONE
- *
- * Revised History:
- *	When		Who		Remark
- *	05/28/2008	amy		Create Version 0 porting from windows code.
- *
- *---------------------------------------------------------------------------*/
+ *---------------------------------------------------------------------------
+ */
 static void dm_ctrl_initgain_byrssi_highpwr(
 	struct net_device *dev)
 {
@@ -1881,11 +1781,6 @@ static void dm_ctrl_initgain_byrssi_highpwr(
 		/* 3.1 Higher PD_TH for OFDM for high power state. */
 		if (priv->CurrentChannelBW != HT_CHANNEL_WIDTH_20) {
 			write_nic_byte(dev, (rOFDM0_XATxAFE+3), 0x10);
-
-			/*else if (priv->card_8192 == HARDWARE_TYPE_RTL8190P)
-				write_nic_byte(dev, rOFDM0_RxDetector1, 0x41);
-			*/
-
 		} else
 			write_nic_byte(dev, rOFDM0_RxDetector1, 0x43);
 	} else {
@@ -1899,10 +1794,6 @@ static void dm_ctrl_initgain_byrssi_highpwr(
 			/*  3.2 Recover PD_TH for OFDM for normal power region. */
 			if (priv->CurrentChannelBW != HT_CHANNEL_WIDTH_20) {
 				write_nic_byte(dev, (rOFDM0_XATxAFE+3), 0x20);
-				/*else if (priv->card_8192 == HARDWARE_TYPE_RTL8190P)
-					write_nic_byte(dev, rOFDM0_RxDetector1, 0x42);
-				*/
-
 			} else
 				write_nic_byte(dev, rOFDM0_RxDetector1, 0x44);
 		}
@@ -1944,7 +1835,6 @@ static void dm_initial_gain(
 		dm_digtable.cur_ig_value = priv->DefaultInitialGain[0];
 		dm_digtable.pre_ig_value = 0;
 	}
-	/*DbgPrint("DM_DigTable.CurIGValue = 0x%x, DM_DigTable.PreIGValue = 0x%x\n", DM_DigTable.CurIGValue, DM_DigTable.PreIGValue);*/
 
 	/* if silent reset happened, we should rewrite the values back */
 	if (priv->reset_count != reset_cnt) {
@@ -1960,7 +1850,6 @@ static void dm_initial_gain(
 		if ((dm_digtable.pre_ig_value != dm_digtable.cur_ig_value)
 			|| !initialized || force_write) {
 			initial_gain = (u8)dm_digtable.cur_ig_value;
-			/*DbgPrint("Write initial gain = 0x%x\n", initial_gain);*/
 			/*  Set initial gain. */
 			write_nic_byte(dev, rOFDM0_XAAGCCore1, initial_gain);
 			write_nic_byte(dev, rOFDM0_XBAGCCore1, initial_gain);
@@ -2012,7 +1901,6 @@ static void dm_pd_th(
 	{
 		if ((dm_digtable.prepd_thstate != dm_digtable.curpd_thstate) ||
 		    (initialized <= 3) || force_write) {
-			/*DbgPrint("Write PD_TH state = %d\n", DM_DigTable.CurPD_THState);*/
 			if (dm_digtable.curpd_thstate == DIG_PD_AT_LOW_POWER) {
 				/*  Lower PD_TH for OFDM. */
 				if (priv->CurrentChannelBW != HT_CHANNEL_WIDTH_20) {
@@ -2020,9 +1908,6 @@ static void dm_pd_th(
 					 * 2008/02/05 MH SD3-Jerry 92U/92E PD_TH are the same.
 					 */
 					write_nic_byte(dev, (rOFDM0_XATxAFE+3), 0x00);
-					/*else if (priv->card_8192 == HARDWARE_TYPE_RTL8190P)
-						write_nic_byte(dev, rOFDM0_RxDetector1, 0x40);
-					*/
 				} else
 					write_nic_byte(dev, rOFDM0_RxDetector1, 0x42);
 			} else if (dm_digtable.curpd_thstate == DIG_PD_AT_NORMAL_POWER) {
@@ -2032,18 +1917,12 @@ static void dm_pd_th(
 					 * 2008/02/05 MH SD3-Jerry 92U/92E PD_TH are the same.
 					 */
 					write_nic_byte(dev, (rOFDM0_XATxAFE+3), 0x20);
-					/*else if (priv->card_8192 == HARDWARE_TYPE_RTL8190P)
-						write_nic_byte(dev, rOFDM0_RxDetector1, 0x42);
-					*/
 				} else
 					write_nic_byte(dev, rOFDM0_RxDetector1, 0x44);
 			} else if (dm_digtable.curpd_thstate == DIG_PD_AT_HIGH_POWER) {
 				/* Higher PD_TH for OFDM for high power state. */
 				if (priv->CurrentChannelBW != HT_CHANNEL_WIDTH_20) {
 					write_nic_byte(dev, (rOFDM0_XATxAFE+3), 0x10);
-					/*else if (priv->card_8192 == HARDWARE_TYPE_RTL8190P)
-						write_nic_byte(dev, rOFDM0_RxDetector1, 0x41);
-					*/
 				} else
 					write_nic_byte(dev, rOFDM0_RxDetector1, 0x43);
 			}
@@ -2090,7 +1969,6 @@ static	void dm_cs_ratio(
 	{
 		if ((dm_digtable.precs_ratio_state != dm_digtable.curcs_ratio_state) ||
 		    !initialized || force_write) {
-			/*DbgPrint("Write CS_ratio state = %d\n", DM_DigTable.CurCS_ratioState);*/
 			if (dm_digtable.curcs_ratio_state == DIG_CS_RATIO_LOWER) {
 				/*  Lower CS ratio for CCK. */
 				write_nic_byte(dev, 0xa0a, 0x08);
@@ -2119,7 +1997,6 @@ static void dm_check_edca_turbo(
 {
 	struct r8192_priv *priv = ieee80211_priv(dev);
 	PRT_HIGH_THROUGHPUT	pHTInfo = priv->ieee80211->pHTInfo;
-	/*PSTA_QOS			pStaQos = pMgntInfo->pStaQos;*/
 
 	/* Keep past Tx/Rx packet count for RT-to-RT EDCA turbo. */
 	static unsigned long			lastTxOkCnt;
@@ -2136,20 +2013,16 @@ static void dm_check_edca_turbo(
 	if (priv->ieee80211->pHTInfo->IOTAction & HT_IOT_ACT_DISABLE_EDCA_TURBO)
 		goto dm_CheckEdcaTurbo_EXIT;
 
-	/*printk("========>%s():bis_any_nonbepkts is %d\n", __func__, priv->bis_any_nonbepkts);*/
-	/* Check the status for current condition. */
 	if (!priv->ieee80211->bis_any_nonbepkts) {
 		curTxOkCnt = priv->stats.txbytesunicast - lastTxOkCnt;
 		curRxOkCnt = priv->stats.rxbytesunicast - lastRxOkCnt;
 		/* For RT-AP, we needs to turn it on when Rx>Tx */
 		if (curRxOkCnt > 4*curTxOkCnt) {
-			/*printk("%s():curRxOkCnt > 4*curTxOkCnt\n");*/
 			if (!priv->bis_cur_rdlstate || !priv->bcurrent_turbo_EDCA) {
 				write_nic_dword(dev, EDCAPARA_BE, edca_setting_DL[pHTInfo->IOTPeer]);
 				priv->bis_cur_rdlstate = true;
 			}
 		} else {
-			/*printk("%s():curRxOkCnt < 4*curTxOkCnt\n");*/
 			if (priv->bis_cur_rdlstate || !priv->bcurrent_turbo_EDCA) {
 				write_nic_dword(dev, EDCAPARA_BE, edca_setting_UL[pHTInfo->IOTPeer]);
 				priv->bis_cur_rdlstate = false;
@@ -2249,7 +2122,6 @@ static void dm_ctstoself(struct net_device *dev)
 		curRxOkCnt = priv->stats.rxbytesunicast - lastRxOkCnt;
 		if (curRxOkCnt > 4*curTxOkCnt) { /* downlink, disable CTS to self */
 			pHTInfo->IOTAction &= ~HT_IOT_ACT_FORCED_CTS2SELF;
-			/*DbgPrint("dm_CTSToSelf() ==> CTS to self disabled -- downlink\n");*/
 		} else { /* uplink */
 			pHTInfo->IOTAction |= HT_IOT_ACT_FORCED_CTS2SELF;
 		}
@@ -2269,12 +2141,8 @@ static void dm_ctstoself(struct net_device *dev)
  * Output:		NONE
  *
  * Return:		NONE
- *
- * Revised History:
- *	When		Who		Remark
- *	05/28/2008	amy	Create Version 0 porting from windows code.
- *
- *---------------------------------------------------------------------------*/
+ *---------------------------------------------------------------------------
+ */
 static	void	dm_check_pbc_gpio(struct net_device *dev)
 {
 	struct r8192_priv *priv = ieee80211_priv(dev);
@@ -2303,18 +2171,13 @@ static	void	dm_check_pbc_gpio(struct net_device *dev)
  * Output:		NONE
  *
  * Return:		NONE
- *
- * Revised History:
- *	When		Who		Remark
- *	01/30/2008	MHC		Create Version 0.
- *
- *---------------------------------------------------------------------------*/
+ *---------------------------------------------------------------------------
+ */
 void dm_rf_pathcheck_workitemcallback(struct work_struct *work)
 {
 	struct delayed_work *dwork = to_delayed_work(work);
 	struct r8192_priv *priv = container_of(dwork, struct r8192_priv, rfpath_check_wq);
 	struct net_device *dev = priv->ieee80211->dev;
-	/*bool bactually_set = false;*/
 	u8 rfpath = 0, i;
 
 	/* 2008/01/30 MH After discussing with SD3 Jerry, 0xc04/0xd04 register will
@@ -2378,7 +2241,6 @@ static void dm_rxpath_sel_byrssi(struct net_device *dev)
 
 	if (priv->ieee80211->mode == WIRELESS_MODE_B) {
 		DM_RxPathSelTable.cck_method = CCK_RX_VERSION_2;	/* pure B mode, fixed cck version2 */
-		/*DbgPrint("Pure B mode, use cck rx version2\n");*/
 	}
 
 	/* decide max/sec/min rssi index */
@@ -2531,7 +2393,6 @@ static void dm_rxpath_sel_byrssi(struct net_device *dev)
 			if ((DM_RxPathSelTable.disabled_rf >> i) & 0x1) {	/* disabled rf */
 				if (tmp_max_rssi >= DM_RxPathSelTable.rf_enable_rssi_th[i]) {
 					/* enable the BB Rx path */
-					/*DbgPrint("RF-%d is enabled.\n", 0x1<<i);*/
 					rtl8192_setBBreg(dev, rOFDM0_TRxPathEnable, 0x1<<i, 0x1);	/* 0xc04[3:0] */
 					rtl8192_setBBreg(dev, rOFDM1_TRxPathEnable, 0x1<<i, 0x1);	/* 0xd04[3:0] */
 					DM_RxPathSelTable.rf_enable_rssi_th[i] = 100;
@@ -2552,12 +2413,8 @@ static void dm_rxpath_sel_byrssi(struct net_device *dev)
  * Output:		NONE
  *
  * Return:		NONE
- *
- * Revised History:
- *	When		Who		Remark
- *	05/28/2008	amy		Create Version 0 porting from windows code.
- *
- *---------------------------------------------------------------------------*/
+ *---------------------------------------------------------------------------
+ */
 static void dm_check_rx_path_selection(struct net_device *dev)
 {
 	struct r8192_priv *priv = ieee80211_priv(dev);
@@ -2756,7 +2613,6 @@ void dm_check_fsync(struct net_device *dev)
 #define	RegC38_NonFsync_Other_AP		1
 #define	RegC38_Fsync_AP_BCM			2
 	struct r8192_priv *priv = ieee80211_priv(dev);
-	/*u32			framesyncC34;*/
 	static u8		reg_c38_State = RegC38_Default;
 	static u32	reset_cnt;
 
@@ -2831,14 +2687,12 @@ void dm_check_fsync(struct net_device *dev)
 					if (reg_c38_State) {
 						write_nic_byte(dev, rOFDM0_RxDetector3, priv->framesync);
 						reg_c38_State = RegC38_Default;
-						/*DbgPrint("Fsync is idle, rssi>=40, write 0xc38 = 0x%x\n", pHalData->framesync);*/
 					}
 				}
 			} else {
 				if (reg_c38_State) {
 					write_nic_byte(dev, rOFDM0_RxDetector3, priv->framesync);
 					reg_c38_State = RegC38_Default;
-					/*DbgPrint("Fsync is idle, not connected, write 0xc38 = 0x%x\n", pHalData->framesync);*/
 				}
 			}
 		}
@@ -2848,55 +2702,15 @@ void dm_check_fsync(struct net_device *dev)
 			write_nic_byte(dev, rOFDM0_RxDetector3, priv->framesync);
 			reg_c38_State = RegC38_Default;
 			reset_cnt = priv->reset_count;
-			/*DbgPrint("reg_c38_State = 0 for silent reset.\n");*/
 		}
 	} else {
 		if (reg_c38_State) {
 			write_nic_byte(dev, rOFDM0_RxDetector3, priv->framesync);
 			reg_c38_State = RegC38_Default;
-			/*DbgPrint("framesync no monitor, write 0xc38 = 0x%x\n", pHalData->framesync);*/
 		}
 	}
 }
 
-/*-----------------------------------------------------------------------------
- * Function:	dm_shadow_init()
- *
- * Overview:	Store all NIC MAC/BB register content.
- *
- * Input:		NONE
- *
- * Output:		NONE
- *
- * Return:		NONE
- *
- * Revised History:
- *	When		Who		Remark
- *	05/29/2008	amy		Create Version 0 porting from windows code.
- *
- *---------------------------------------------------------------------------
- */
-void dm_shadow_init(struct net_device *dev)
-{
-	u8	page;
-	u16	offset;
-
-	for (page = 0; page < 5; page++)
-		for (offset = 0; offset < 256; offset++) {
-			read_nic_byte(dev, offset + page * 256, &dm_shadow[page][offset]);
-			/*DbgPrint("P-%d/O-%02x=%02x\r\n", page, offset, DM_Shadow[page][offset]);*/
-		}
-
-	for (page = 8; page < 11; page++)
-		for (offset = 0; offset < 256; offset++)
-			read_nic_byte(dev, offset + page * 256, &dm_shadow[page][offset]);
-
-	for (page = 12; page < 15; page++)
-		for (offset = 0; offset < 256; offset++)
-			read_nic_byte(dev, offset + page * 256, &dm_shadow[page][offset]);
-
-}   /* dm_shadow_init */
-
 /*---------------------------Define function prototype------------------------*/
 /*-----------------------------------------------------------------------------
  * Function:	DM_DynamicTxPower()
@@ -2909,11 +2723,6 @@ void dm_shadow_init(struct net_device *dev)
  * Output:		NONE
  *
  * Return:		NONE
- *
- * Revised History:
- *	When		Who		Remark
- *	03/06/2008	Jacken	Create Version 0.
- *
  *---------------------------------------------------------------------------
  */
 static void dm_init_dynamic_txpower(struct net_device *dev)
@@ -2939,7 +2748,6 @@ static void dm_dynamic_txpower(struct net_device *dev)
 		priv->bDynamicTxLowPower = false;
 		return;
 	}
-	/*printk("priv->ieee80211->current_network.unknown_cap_exist is %d , priv->ieee80211->current_network.broadcom_cap_exist is %d\n", priv->ieee80211->current_network.unknown_cap_exist, priv->ieee80211->current_network.broadcom_cap_exist);*/
 	if ((priv->ieee80211->current_network.atheros_cap_exist) && (priv->ieee80211->mode == IEEE_G)) {
 		txhipower_threshold = TX_POWER_ATHEROAP_THRESH_HIGH;
 		txlowpower_threshold = TX_POWER_ATHEROAP_THRESH_LOW;
@@ -2948,7 +2756,6 @@ static void dm_dynamic_txpower(struct net_device *dev)
 		txlowpower_threshold = TX_POWER_NEAR_FIELD_THRESH_LOW;
 	}
 
-	/*printk("=======>%s(): txhipower_threshold is %d, txlowpower_threshold is %d\n", __func__, txhipower_threshold, txlowpower_threshold);*/
 	RT_TRACE(COMP_TXAGC, "priv->undecorated_smoothed_pwdb = %ld\n", priv->undecorated_smoothed_pwdb);
 
 	if (priv->ieee80211->state == IEEE80211_LINKED) {
@@ -2967,7 +2774,6 @@ static void dm_dynamic_txpower(struct net_device *dev)
 				priv->bDynamicTxLowPower = false;
 		}
 	} else {
-		/*pHalData->bTXPowerCtrlforNearFarRange = !pHalData->bTXPowerCtrlforNearFarRange;*/
 		priv->bDynamicTxHighPower = false;
 		priv->bDynamicTxLowPower = false;
 	}
@@ -2994,14 +2800,10 @@ static void dm_check_txrateandretrycount(struct net_device *dev)
 	struct r8192_priv *priv = ieee80211_priv(dev);
 	struct ieee80211_device *ieee = priv->ieee80211;
 	/* for 11n tx rate */
-	/*priv->stats.CurrentShowTxate = read_nic_byte(dev, CURRENT_TX_RATE_REG);*/
 	read_nic_byte(dev, CURRENT_TX_RATE_REG, &ieee->softmac_stats.CurrentShowTxate);
-	/*printk("=============>tx_rate_reg:%x\n", ieee->softmac_stats.CurrentShowTxate);*/
 	/* for initial tx rate */
-	/*priv->stats.last_packet_rate = read_nic_byte(dev, INITIAL_TX_RATE_REG);*/
 	read_nic_byte(dev, INITIAL_TX_RATE_REG, &ieee->softmac_stats.last_packet_rate);
 	/* for tx retry count */
-	/*priv->stats.txretrycount = read_nic_dword(dev, TX_RETRY_COUNT_REG);*/
 	read_nic_dword(dev, TX_RETRY_COUNT_REG, &ieee->softmac_stats.txretrycount);
 }
 
diff --git a/drivers/staging/rtl8192u/r8192U_dm.h b/drivers/staging/rtl8192u/r8192U_dm.h
index 2159018b4e38..f4eb18216677 100644
--- a/drivers/staging/rtl8192u/r8192U_dm.h
+++ b/drivers/staging/rtl8192u/r8192U_dm.h
@@ -168,7 +168,6 @@ void dm_rf_operation_test_callback(unsigned long data);
 void dm_rf_pathcheck_workitemcallback(struct work_struct *work);
 void dm_fsync_work_callback(struct work_struct *work);
 void dm_cck_txpower_adjust(struct net_device *dev, bool  binch14);
-void dm_shadow_init(struct net_device *dev);
 void dm_initialize_txpower_tracking(struct net_device *dev);
 /*--------------------------Exported Function prototype---------------------*/
 
diff --git a/drivers/staging/rtl8192u/r819xU_phy.c b/drivers/staging/rtl8192u/r819xU_phy.c
index 97f4d89500ae..e6836eacc7aa 100644
--- a/drivers/staging/rtl8192u/r819xU_phy.c
+++ b/drivers/staging/rtl8192u/r819xU_phy.c
@@ -1045,93 +1045,6 @@ static void rtl8192_SetTxPowerLevel(struct net_device *dev, u8 channel)
 }
 
 /******************************************************************************
- * function:  This function sets RF state on or off
- * input:     net_device         *dev
- *            RT_RF_POWER_STATE  eRFPowerState  //Power State to set
- * output:    none
- * return:    none
- * notice:
- *****************************************************************************/
-bool rtl8192_SetRFPowerState(struct net_device *dev,
-			     RT_RF_POWER_STATE eRFPowerState)
-{
-	bool				bResult = true;
-	struct r8192_priv *priv = ieee80211_priv(dev);
-
-	if (eRFPowerState == priv->ieee80211->eRFPowerState)
-		return false;
-
-	if (priv->SetRFPowerStateInProgress)
-		return false;
-
-	priv->SetRFPowerStateInProgress = true;
-
-	switch (priv->rf_chip) {
-	case RF_8256:
-		switch (eRFPowerState) {
-		case eRfOn:
-			/* RF-A, RF-B */
-			/* enable RF-Chip A/B - 0x860[4] */
-			rtl8192_setBBreg(dev, rFPGA0_XA_RFInterfaceOE, BIT(4),
-					 0x1);
-			/* analog to digital on - 0x88c[9:8] */
-			rtl8192_setBBreg(dev, rFPGA0_AnalogParameter4, 0x300,
-					 0x3);
-			/* digital to analog on - 0x880[4:3] */
-			rtl8192_setBBreg(dev, rFPGA0_AnalogParameter1, 0x18,
-					 0x3);
-			/* rx antenna on - 0xc04[1:0] */
-			rtl8192_setBBreg(dev, rOFDM0_TRxPathEnable, 0x3, 0x3);
-			/* rx antenna on - 0xd04[1:0] */
-			rtl8192_setBBreg(dev, rOFDM1_TRxPathEnable, 0x3, 0x3);
-			/* analog to digital part2 on - 0x880[6:5] */
-			rtl8192_setBBreg(dev, rFPGA0_AnalogParameter1, 0x60,
-					 0x3);
-
-			break;
-
-		case eRfSleep:
-
-			break;
-
-		case eRfOff:
-			/* RF-A, RF-B */
-			/* disable RF-Chip A/B - 0x860[4] */
-			rtl8192_setBBreg(dev, rFPGA0_XA_RFInterfaceOE, BIT(4),
-					 0x0);
-			/* analog to digital off, for power save */
-			rtl8192_setBBreg(dev, rFPGA0_AnalogParameter4, 0xf00,
-					 0x0); /* 0x88c[11:8] */
-			/* digital to analog off, for power save - 0x880[4:3] */
-			rtl8192_setBBreg(dev, rFPGA0_AnalogParameter1, 0x18,
-					 0x0);
-			/* rx antenna off - 0xc04[3:0] */
-			rtl8192_setBBreg(dev, rOFDM0_TRxPathEnable, 0xf, 0x0);
-			/* rx antenna off - 0xd04[3:0] */
-			rtl8192_setBBreg(dev, rOFDM1_TRxPathEnable, 0xf, 0x0);
-			/* analog to digital part2 off, for power save */
-			rtl8192_setBBreg(dev, rFPGA0_AnalogParameter1, 0x60,
-					 0x0); /* 0x880[6:5] */
-
-			break;
-
-		default:
-			bResult = false;
-			RT_TRACE(COMP_ERR, "%s(): unknown state to set: 0x%X\n",
-				 __func__, eRFPowerState);
-			break;
-		}
-		break;
-	default:
-		RT_TRACE(COMP_ERR, "Not support rf_chip(%x)\n", priv->rf_chip);
-		break;
-	}
-	priv->SetRFPowerStateInProgress = false;
-
-	return bResult;
-}
-
-/******************************************************************************
  * function:  This function sets command table variable (struct sw_chnl_cmd).
  * input:     sw_chnl_cmd      *CmdTable    //table to be set
  *            u32            CmdTableIdx  //variable index in table to be set
diff --git a/drivers/staging/rtl8192u/r819xU_phy.h b/drivers/staging/rtl8192u/r819xU_phy.h
index 8c2933264407..bafaa6a90c50 100644
--- a/drivers/staging/rtl8192u/r819xU_phy.h
+++ b/drivers/staging/rtl8192u/r819xU_phy.h
@@ -74,8 +74,6 @@ void rtl8192_SetBWMode(struct net_device *dev,
 		       enum ht_extension_chan_offset offset);
 void rtl8192_SwChnl_WorkItem(struct net_device *dev);
 void rtl8192_SetBWModeWorkItem(struct net_device *dev);
-bool rtl8192_SetRFPowerState(struct net_device *dev,
-			     RT_RF_POWER_STATE eRFPowerState);
 void InitialGain819xUsb(struct net_device *dev, u8 Operation);
 
 void InitialGainOperateWorkItemCallBack(struct work_struct *work);
diff --git a/drivers/staging/rtl8712/rtl8712_efuse.h b/drivers/staging/rtl8712/rtl8712_efuse.h
index 2e1ea9d7a295..7a49740212eb 100644
--- a/drivers/staging/rtl8712/rtl8712_efuse.h
+++ b/drivers/staging/rtl8712/rtl8712_efuse.h
@@ -23,6 +23,7 @@ struct PGPKT_STRUCT {
 	u8 word_en;
 	u8 data[PGPKT_DATA_SIZE];
 };
+
 /*--------------------------------------------------------------------------*/
 u8 r8712_efuse_reg_init(struct _adapter *padapter);
 void r8712_efuse_reg_uninit(struct _adapter *padapter);
diff --git a/drivers/staging/rtl8712/rtl871x_mlme.c b/drivers/staging/rtl8712/rtl871x_mlme.c
index 63e12b157001..fccfa0915a02 100644
--- a/drivers/staging/rtl8712/rtl871x_mlme.c
+++ b/drivers/staging/rtl8712/rtl871x_mlme.c
@@ -663,7 +663,7 @@ void r8712_joinbss_event_callback(struct _adapter *adapter, u8 *pbuf)
 		if (!pnetwork)
 			return;
 		memcpy((u8 *)pnetwork + 16, (u8 *)pbuf + 8,
-		       sizeof(struct wlan_network) - 16);
+			sizeof(struct wlan_network) - 16);
 	} else {
 		pnetwork = (struct wlan_network *)pbuf;
 	}
@@ -674,36 +674,36 @@ void r8712_joinbss_event_callback(struct _adapter *adapter, u8 *pbuf)
 	pnetwork->network_type = le32_to_cpu(pnetwork->network_type);
 	pnetwork->network.Length = le32_to_cpu(pnetwork->network.Length);
 	pnetwork->network.Ssid.SsidLength =
-		 le32_to_cpu(pnetwork->network.Ssid.SsidLength);
+		le32_to_cpu(pnetwork->network.Ssid.SsidLength);
 	pnetwork->network.Privacy = le32_to_cpu(pnetwork->network.Privacy);
 	pnetwork->network.Rssi = le32_to_cpu(pnetwork->network.Rssi);
 	pnetwork->network.NetworkTypeInUse =
-		 le32_to_cpu(pnetwork->network.NetworkTypeInUse);
+		le32_to_cpu(pnetwork->network.NetworkTypeInUse);
 	pnetwork->network.Configuration.ATIMWindow =
-		 le32_to_cpu(pnetwork->network.Configuration.ATIMWindow);
+		le32_to_cpu(pnetwork->network.Configuration.ATIMWindow);
 	pnetwork->network.Configuration.BeaconPeriod =
-		 le32_to_cpu(pnetwork->network.Configuration.BeaconPeriod);
+		le32_to_cpu(pnetwork->network.Configuration.BeaconPeriod);
 	pnetwork->network.Configuration.DSConfig =
-		 le32_to_cpu(pnetwork->network.Configuration.DSConfig);
+		le32_to_cpu(pnetwork->network.Configuration.DSConfig);
 	pnetwork->network.Configuration.FHConfig.DwellTime =
-		 le32_to_cpu(pnetwork->network.Configuration.FHConfig.DwellTime);
+		le32_to_cpu(pnetwork->network.Configuration.FHConfig.DwellTime);
 	pnetwork->network.Configuration.FHConfig.HopPattern =
-		 le32_to_cpu(pnetwork->network.Configuration.FHConfig.HopPattern);
+		le32_to_cpu(pnetwork->network.Configuration.FHConfig.HopPattern);
 	pnetwork->network.Configuration.FHConfig.HopSet =
-		 le32_to_cpu(pnetwork->network.Configuration.FHConfig.HopSet);
+		le32_to_cpu(pnetwork->network.Configuration.FHConfig.HopSet);
 	pnetwork->network.Configuration.FHConfig.Length =
-		 le32_to_cpu(pnetwork->network.Configuration.FHConfig.Length);
+		le32_to_cpu(pnetwork->network.Configuration.FHConfig.Length);
 	pnetwork->network.Configuration.Length =
-		 le32_to_cpu(pnetwork->network.Configuration.Length);
+		le32_to_cpu(pnetwork->network.Configuration.Length);
 	pnetwork->network.InfrastructureMode =
-		 le32_to_cpu(pnetwork->network.InfrastructureMode);
+		le32_to_cpu(pnetwork->network.InfrastructureMode);
 	pnetwork->network.IELength = le32_to_cpu(pnetwork->network.IELength);
 #endif
 
 	the_same_macaddr = !memcmp(pnetwork->network.MacAddress,
-				   cur_network->network.MacAddress, ETH_ALEN);
+				cur_network->network.MacAddress, ETH_ALEN);
 	pnetwork->network.Length =
-		 r8712_get_wlan_bssid_ex_sz(&pnetwork->network);
+		r8712_get_wlan_bssid_ex_sz(&pnetwork->network);
 	spin_lock_irqsave(&pmlmepriv->lock, irqL);
 	if (pnetwork->network.Length > sizeof(struct wlan_bssid_ex))
 		goto ignore_joinbss_callback;
@@ -713,24 +713,24 @@ void r8712_joinbss_event_callback(struct _adapter *adapter, u8 *pbuf)
 			if (check_fwstate(pmlmepriv, _FW_LINKED)) {
 				if (the_same_macaddr) {
 					ptarget_wlan =
-					    r8712_find_network(&pmlmepriv->scanned_queue,
-					    cur_network->network.MacAddress);
+						r8712_find_network(&pmlmepriv->scanned_queue,
+						cur_network->network.MacAddress);
 				} else {
 					pcur_wlan =
-					     r8712_find_network(&pmlmepriv->scanned_queue,
-					     cur_network->network.MacAddress);
+						r8712_find_network(&pmlmepriv->scanned_queue,
+						cur_network->network.MacAddress);
 					if (pcur_wlan)
 						pcur_wlan->fixed = false;
 
 					pcur_sta = r8712_get_stainfo(pstapriv,
-					     cur_network->network.MacAddress);
+						cur_network->network.MacAddress);
 					spin_lock_irqsave(&pstapriv->sta_hash_lock, irqL2);
 					r8712_free_stainfo(adapter, pcur_sta);
 					spin_unlock_irqrestore(&(pstapriv->sta_hash_lock), irqL2);
 
 					ptarget_wlan =
-						 r8712_find_network(&pmlmepriv->scanned_queue,
-						 pnetwork->network.MacAddress);
+						r8712_find_network(&pmlmepriv->scanned_queue,
+						pnetwork->network.MacAddress);
 					if (ptarget_wlan)
 						ptarget_wlan->fixed = true;
 				}
@@ -745,7 +745,7 @@ void r8712_joinbss_event_callback(struct _adapter *adapter, u8 *pbuf)
 				if (check_fwstate(pmlmepriv,
 					_FW_UNDER_LINKING))
 					pmlmepriv->fw_state ^=
-						 _FW_UNDER_LINKING;
+						_FW_UNDER_LINKING;
 				goto ignore_joinbss_callback;
 			}
 
@@ -753,16 +753,16 @@ void r8712_joinbss_event_callback(struct _adapter *adapter, u8 *pbuf)
 			if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
 				if (the_same_macaddr) {
 					ptarget_sta =
-						 r8712_get_stainfo(pstapriv,
-						 pnetwork->network.MacAddress);
+						r8712_get_stainfo(pstapriv,
+						pnetwork->network.MacAddress);
 					if (!ptarget_sta)
 						ptarget_sta =
-						 r8712_alloc_stainfo(pstapriv,
-						 pnetwork->network.MacAddress);
+						r8712_alloc_stainfo(pstapriv,
+						pnetwork->network.MacAddress);
 				} else {
 					ptarget_sta =
-						 r8712_alloc_stainfo(pstapriv,
-						 pnetwork->network.MacAddress);
+						r8712_alloc_stainfo(pstapriv,
+						pnetwork->network.MacAddress);
 				}
 				if (ptarget_sta) /*update ptarget_sta*/ {
 					ptarget_sta->aid = pnetwork->join_res;
@@ -773,27 +773,28 @@ void r8712_joinbss_event_callback(struct _adapter *adapter, u8 *pbuf)
 						adapter->securitypriv.busetkipkey = false;
 						adapter->securitypriv.bgrpkey_handshake = false;
 						ptarget_sta->ieee8021x_blocked = true;
-						ptarget_sta->XPrivacy = adapter->
-						securitypriv.PrivacyAlgrthm;
+						ptarget_sta->XPrivacy =
+							adapter->securitypriv.PrivacyAlgrthm;
 						memset((u8 *)&ptarget_sta->x_UncstKey,
-							 0,
-							 sizeof(union Keytype));
+							0,
+							sizeof(union Keytype));
 						memset((u8 *)&ptarget_sta->tkiprxmickey,
-							 0,
-							 sizeof(union Keytype));
+							0,
+							sizeof(union Keytype));
 						memset((u8 *)&ptarget_sta->tkiptxmickey,
-							 0,
-							 sizeof(union Keytype));
-						memset((u8 *)&ptarget_sta->txpn, 0,
-							 sizeof(union pn48));
-						memset((u8 *)&ptarget_sta->rxpn, 0,
-							 sizeof(union pn48));
+							0,
+							sizeof(union Keytype));
+						memset((u8 *)&ptarget_sta->txpn,
+							0,
+							sizeof(union pn48));
+						memset((u8 *)&ptarget_sta->rxpn,
+							0,
+							sizeof(union pn48));
 					}
 				} else {
-					if (check_fwstate(pmlmepriv,
-					    _FW_UNDER_LINKING))
+					if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING))
 						pmlmepriv->fw_state ^=
-							 _FW_UNDER_LINKING;
+							_FW_UNDER_LINKING;
 					goto ignore_joinbss_callback;
 				}
 			}
@@ -815,12 +816,12 @@ void r8712_joinbss_event_callback(struct _adapter *adapter, u8 *pbuf)
 				break;
 			}
 			r8712_update_protection(adapter,
-					  (cur_network->network.IEs) +
-					  sizeof(struct NDIS_802_11_FIXED_IEs),
-					  (cur_network->network.IELength));
+				(cur_network->network.IEs) +
+				sizeof(struct NDIS_802_11_FIXED_IEs),
+				(cur_network->network.IELength));
 			/*TODO: update HT_Capability*/
 			update_ht_cap(adapter, cur_network->network.IEs,
-				      cur_network->network.IELength);
+				cur_network->network.IELength);
 			/*indicate connect*/
 			if (check_fwstate(pmlmepriv, WIFI_STATION_STATE))
 				r8712_indicate_connect(adapter);
@@ -831,7 +832,7 @@ void r8712_joinbss_event_callback(struct _adapter *adapter, u8 *pbuf)
 	} else {
 		if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING)) {
 			mod_timer(&pmlmepriv->assoc_timer,
-				  jiffies + msecs_to_jiffies(1));
+				jiffies + msecs_to_jiffies(1));
 			_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
 		}
 	}
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c
index c6fd6cf741ef..7e2c61c75150 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c
@@ -32,8 +32,6 @@ int	rtw_init_mlme_priv(struct adapter *padapter)
 	INIT_LIST_HEAD(&pmlmepriv->scanned_queue.queue);
 	spin_lock_init(&pmlmepriv->scanned_queue.lock);
 
-	set_scanned_network_val(pmlmepriv, 0);
-
 	memset(&pmlmepriv->assoc_ssid, 0, sizeof(struct ndis_802_11_ssid));
 
 	pbuf = vzalloc(array_size(MAX_BSS_CNT, sizeof(struct wlan_network)));
@@ -161,8 +159,6 @@ struct	wlan_network *rtw_alloc_network(struct	mlme_priv *pmlmepriv)
 	pnetwork->aid = 0;
 	pnetwork->join_res = 0;
 
-	pmlmepriv->num_of_scanned++;
-
 exit:
 	spin_unlock_bh(&free_queue->lock);
 
@@ -198,8 +194,6 @@ void _rtw_free_network(struct	mlme_priv *pmlmepriv, struct wlan_network *pnetwor
 
 	list_add_tail(&(pnetwork->list), &(free_queue->queue));
 
-	pmlmepriv->num_of_scanned--;
-
 	spin_unlock_bh(&free_queue->lock);
 }
 
@@ -220,8 +214,6 @@ void _rtw_free_network_nolock(struct	mlme_priv *pmlmepriv, struct wlan_network *
 
 	list_add_tail(&(pnetwork->list), get_list_head(free_queue));
 
-	pmlmepriv->num_of_scanned--;
-
 	/* spin_unlock_irqrestore(&free_queue->lock, irqL); */
 }
 
@@ -863,7 +855,6 @@ static void free_scanqueue(struct	mlme_priv *pmlmepriv)
 		list_del_init(plist);
 		list_add_tail(plist, &free_queue->queue);
 		plist = ptemp;
-		pmlmepriv->num_of_scanned--;
 	}
 
 	spin_unlock_bh(&free_queue->lock);
@@ -1549,7 +1540,7 @@ void _rtw_join_timeout_handler(struct timer_list *t)
 	if (adapter->bDriverStopped || adapter->bSurpriseRemoved)
 		return;
 
-	spin_lock_irq(&pmlmepriv->lock);
+	spin_lock_bh(&pmlmepriv->lock);
 
 	if (rtw_to_roam(adapter) > 0) { /* join timeout caused by roaming */
 		while (1) {
@@ -1577,7 +1568,7 @@ void _rtw_join_timeout_handler(struct timer_list *t)
 
 	}
 
-	spin_unlock_irq(&pmlmepriv->lock);
+	spin_unlock_bh(&pmlmepriv->lock);
 }
 
 /*
@@ -1590,11 +1581,11 @@ void rtw_scan_timeout_handler(struct timer_list *t)
 						  mlmepriv.scan_to_timer);
 	struct	mlme_priv *pmlmepriv = &adapter->mlmepriv;
 
-	spin_lock_irq(&pmlmepriv->lock);
+	spin_lock_bh(&pmlmepriv->lock);
 
 	_clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY);
 
-	spin_unlock_irq(&pmlmepriv->lock);
+	spin_unlock_bh(&pmlmepriv->lock);
 
 	rtw_indicate_scan_done(adapter, true);
 }
diff --git a/drivers/staging/rtl8723bs/hal/hal_btcoex.c b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
index e36f8c369a04..e26b789b9cdd 100644
--- a/drivers/staging/rtl8723bs/hal/hal_btcoex.c
+++ b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
@@ -1177,14 +1177,6 @@ bool hal_btcoex_IsBtDisabled(struct adapter *padapter)
 		return false;
 }
 
-void hal_btcoex_SetChipType(struct adapter *padapter, u8 chipType)
-{
-	struct hal_com_data *pHalData;
-
-
-	pHalData = GET_HAL_DATA(padapter);
-}
-
 void hal_btcoex_SetPgAntNum(struct adapter *padapter, u8 antNum)
 {
 	struct hal_com_data *pHalData;
diff --git a/drivers/staging/rtl8723bs/hal/hal_com.c b/drivers/staging/rtl8723bs/hal/hal_com.c
index e42556d03bce..852232102433 100644
--- a/drivers/staging/rtl8723bs/hal/hal_com.c
+++ b/drivers/staging/rtl8723bs/hal/hal_com.c
@@ -859,22 +859,6 @@ bool eqNByte(u8 *str1, u8 *str2, u32 num)
 	return true;
 }
 
-/*  */
-/* 	Description: */
-/* 		Translate a character to hex digit. */
-/*  */
-u32 MapCharToHexDigit(char chTmp)
-{
-	if (chTmp >= '0' && chTmp <= '9')
-		return chTmp - '0';
-	else if (chTmp >= 'a' && chTmp <= 'f')
-		return 10 + (chTmp - 'a');
-	else if (chTmp >= 'A' && chTmp <= 'F')
-		return 10 + (chTmp - 'A');
-	else
-		return 0;
-}
-
 bool GetU1ByteIntegerFromStringInDecimal(char *Str, u8 *pInt)
 {
 	u16 i = 0;
@@ -893,45 +877,6 @@ bool GetU1ByteIntegerFromStringInDecimal(char *Str, u8 *pInt)
 	return true;
 }
 
-/*  <20121004, Kordan> For example,
- *  ParseQualifiedString(inString, 0, outString, '[', ']') gets "Kordan" from
- *  a string "Hello [Kordan]".
- *  If RightQualifier does not exist, it will hang in the while loop
- */
-bool ParseQualifiedString(
-	char *In, u32 *Start, char *Out, char LeftQualifier, char RightQualifier
-)
-{
-	u32 i = 0, j = 0;
-	char c = In[(*Start)++];
-
-	if (c != LeftQualifier)
-		return false;
-
-	i = (*Start);
-	while ((c = In[(*Start)++]) != RightQualifier)
-		; /*  find ']' */
-	j = (*Start) - 2;
-	strncpy((char *)Out, (const char *)(In+i), j-i+1);
-
-	return true;
-}
-
-bool isAllSpaceOrTab(u8 *data, u8 size)
-{
-	u8 cnt = 0, NumOfSpaceAndTab = 0;
-
-	while (size > cnt) {
-		if (data[cnt] == ' ' || data[cnt] == '\t' || data[cnt] == '\0')
-			++NumOfSpaceAndTab;
-
-		++cnt;
-	}
-
-	return size == NumOfSpaceAndTab;
-}
-
-
 void rtw_hal_check_rxfifo_full(struct adapter *adapter)
 {
 	struct dvobj_priv *psdpriv = adapter->dvobj;
@@ -952,60 +897,7 @@ void rtw_hal_check_rxfifo_full(struct adapter *adapter)
 	}
 }
 
-void linked_info_dump(struct adapter *padapter, u8 benable)
-{
-	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
-
-	if (padapter->bLinkInfoDump == benable)
-		return;
-
-	if (benable) {
-		pwrctrlpriv->org_power_mgnt = pwrctrlpriv->power_mgnt;/* keep org value */
-		rtw_pm_set_lps(padapter, PS_MODE_ACTIVE);
-
-		pwrctrlpriv->ips_org_mode = pwrctrlpriv->ips_mode;/* keep org value */
-		rtw_pm_set_ips(padapter, IPS_NONE);
-	} else {
-		rtw_pm_set_ips(padapter, pwrctrlpriv->ips_org_mode);
-
-		rtw_pm_set_lps(padapter, pwrctrlpriv->ips_org_mode);
-	}
-	padapter->bLinkInfoDump = benable;
-}
-
 #ifdef DBG_RX_SIGNAL_DISPLAY_RAW_DATA
-void rtw_get_raw_rssi_info(void *sel, struct adapter *padapter)
-{
-	u8 isCCKrate, rf_path;
-	struct hal_com_data *pHalData = GET_HAL_DATA(padapter);
-	struct rx_raw_rssi *psample_pkt_rssi = &padapter->recvpriv.raw_rssi_info;
-
-	netdev_dbg(padapter->pnetdev,
-		   "RxRate = %s, PWDBALL = %d(%%), rx_pwr_all = %d(dBm)\n",
-		   HDATA_RATE(psample_pkt_rssi->data_rate),
-		   psample_pkt_rssi->pwdball, psample_pkt_rssi->pwr_all);
-
-	isCCKrate = psample_pkt_rssi->data_rate <= DESC_RATE11M;
-
-	if (isCCKrate)
-		psample_pkt_rssi->mimo_signal_strength[0] = psample_pkt_rssi->pwdball;
-
-	for (rf_path = 0; rf_path < pHalData->NumTotalRFPath; rf_path++) {
-		netdev_dbg(padapter->pnetdev,
-			   "RF_PATH_%d =>signal_strength:%d(%%), signal_quality:%d(%%)\n",
-			   rf_path,
-			   psample_pkt_rssi->mimo_signal_strength[rf_path],
-			   psample_pkt_rssi->mimo_signal_quality[rf_path]);
-
-		if (!isCCKrate) {
-			netdev_dbg(padapter->pnetdev,
-				   "\trx_ofdm_pwr:%d(dBm), rx_ofdm_snr:%d(dB)\n",
-				   psample_pkt_rssi->ofdm_pwr[rf_path],
-				   psample_pkt_rssi->ofdm_snr[rf_path]);
-		}
-	}
-}
-
 void rtw_dump_raw_rssi_info(struct adapter *padapter)
 {
 	u8 isCCKrate, rf_path;
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
index 0fcae6871108..c3c1b49674d3 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
@@ -2304,7 +2304,6 @@ void Hal_EfuseParseBTCoexistInfo_8723B(
 	}
 
 	hal_btcoex_SetBTCoexist(padapter, pHalData->EEPROMBluetoothCoexist);
-	hal_btcoex_SetChipType(padapter, pHalData->EEPROMBluetoothType);
 	hal_btcoex_SetPgAntNum(padapter, pHalData->EEPROMBluetoothAntNum == Ant_x2 ? 2 : 1);
 	if (pHalData->EEPROMBluetoothAntNum == Ant_x1)
 		hal_btcoex_SetSingleAntPath(padapter, pHalData->ant_path);
diff --git a/drivers/staging/rtl8723bs/include/drv_types.h b/drivers/staging/rtl8723bs/include/drv_types.h
index 82159e1c7f9b..ea6bb44c5e1d 100644
--- a/drivers/staging/rtl8723bs/include/drv_types.h
+++ b/drivers/staging/rtl8723bs/include/drv_types.h
@@ -305,7 +305,11 @@ struct sdio_data intf_data;
 };
 
 #define dvobj_to_pwrctl(dvobj) (&(dvobj->pwrctl_priv))
-#define pwrctl_to_dvobj(pwrctl) container_of(pwrctl, struct dvobj_priv, pwrctl_priv)
+
+static inline struct dvobj_priv *pwrctl_to_dvobj(struct pwrctrl_priv *pwrctl_priv)
+{
+	return container_of(pwrctl_priv, struct dvobj_priv, pwrctl_priv);
+}
 
 static inline struct device *dvobj_to_dev(struct dvobj_priv *dvobj)
 {
diff --git a/drivers/staging/rtl8723bs/include/hal_btcoex.h b/drivers/staging/rtl8723bs/include/hal_btcoex.h
index fb167642da01..525cce3574fe 100644
--- a/drivers/staging/rtl8723bs/include/hal_btcoex.h
+++ b/drivers/staging/rtl8723bs/include/hal_btcoex.h
@@ -23,7 +23,6 @@ struct bt_coexist {
 void hal_btcoex_SetBTCoexist(struct adapter *padapter, u8 bBtExist);
 bool hal_btcoex_IsBtExist(struct adapter *padapter);
 bool hal_btcoex_IsBtDisabled(struct adapter *);
-void hal_btcoex_SetChipType(struct adapter *padapter, u8 chipType);
 void hal_btcoex_SetPgAntNum(struct adapter *padapter, u8 antNum);
 void hal_btcoex_SetSingleAntPath(struct adapter *padapter, u8 singleAntPath);
 
diff --git a/drivers/staging/rtl8723bs/include/hal_com.h b/drivers/staging/rtl8723bs/include/hal_com.h
index 6356b8c2ef81..17d5cfb66a36 100644
--- a/drivers/staging/rtl8723bs/include/hal_com.h
+++ b/drivers/staging/rtl8723bs/include/hal_com.h
@@ -147,18 +147,9 @@ u8 GetHalDefVar(struct adapter *adapter, enum hal_def_variable variable,
 
 bool eqNByte(u8 *str1, u8 *str2, u32 num);
 
-u32 MapCharToHexDigit(char chTmp);
-
-bool ParseQualifiedString(char *In, u32 *Start, char *Out, char LeftQualifier,
-			  char RightQualifier);
-
 bool GetU1ByteIntegerFromStringInDecimal(char *str, u8 *in);
 
-bool isAllSpaceOrTab(u8 *data, u8 size);
-
-void linked_info_dump(struct adapter *padapter, u8 benable);
 #ifdef DBG_RX_SIGNAL_DISPLAY_RAW_DATA
-void rtw_get_raw_rssi_info(void *sel, struct adapter *padapter);
 void rtw_store_phy_info(struct adapter *padapter, union recv_frame *prframe);
 void rtw_dump_raw_rssi_info(struct adapter *padapter);
 #endif
diff --git a/drivers/staging/rtl8723bs/include/ieee80211.h b/drivers/staging/rtl8723bs/include/ieee80211.h
index 9041d8dc5fb1..1098b0209200 100644
--- a/drivers/staging/rtl8723bs/include/ieee80211.h
+++ b/drivers/staging/rtl8723bs/include/ieee80211.h
@@ -174,7 +174,7 @@ struct ieee_param {
 			u8 reserved[32];
 			u8 data[];
 		} wpa_ie;
-	        struct{
+		struct{
 			int command;
 			int reason_code;
 		} mlme;
@@ -271,10 +271,10 @@ struct eapol {
 #define P80211_OUI_LEN 3
 
 struct ieee80211_snap_hdr {
-        u8    dsap;   /* always 0xAA */
-        u8    ssap;   /* always 0xAA */
-        u8    ctrl;   /* always 0x03 */
-        u8    oui[P80211_OUI_LEN];    /* organizational universal id */
+	u8    dsap;   /* always 0xAA */
+	u8    ssap;   /* always 0xAA */
+	u8    ctrl;   /* always 0x03 */
+	u8    oui[P80211_OUI_LEN];    /* organizational universal id */
 } __attribute__ ((packed));
 
 #define SNAP_SIZE sizeof(struct ieee80211_snap_hdr)
@@ -342,12 +342,13 @@ struct ieee80211_snap_hdr {
 #define IEEE80211_OFDM_RATE_48MB_MASK		(1<<10)
 #define IEEE80211_OFDM_RATE_54MB_MASK		(1<<11)
 
-#define IEEE80211_CCK_RATES_MASK	        0x0000000F
+#define IEEE80211_CCK_RATES_MASK		0x0000000F
 #define IEEE80211_CCK_BASIC_RATES_MASK	(IEEE80211_CCK_RATE_1MB_MASK | \
 	IEEE80211_CCK_RATE_2MB_MASK)
-#define IEEE80211_CCK_DEFAULT_RATES_MASK	(IEEE80211_CCK_BASIC_RATES_MASK | \
-        IEEE80211_CCK_RATE_5MB_MASK | \
-        IEEE80211_CCK_RATE_11MB_MASK)
+#define IEEE80211_CCK_DEFAULT_RATES_MASK	\
+	(IEEE80211_CCK_BASIC_RATES_MASK |	\
+	 IEEE80211_CCK_RATE_5MB_MASK |		\
+	 IEEE80211_CCK_RATE_11MB_MASK)
 
 #define IEEE80211_OFDM_RATES_MASK		0x00000FF0
 #define IEEE80211_OFDM_BASIC_RATES_MASK	(IEEE80211_OFDM_RATE_6MB_MASK | \
@@ -359,8 +360,9 @@ struct ieee80211_snap_hdr {
 	IEEE80211_OFDM_RATE_36MB_MASK | \
 	IEEE80211_OFDM_RATE_48MB_MASK | \
 	IEEE80211_OFDM_RATE_54MB_MASK)
-#define IEEE80211_DEFAULT_RATES_MASK (IEEE80211_OFDM_DEFAULT_RATES_MASK | \
-                                IEEE80211_CCK_DEFAULT_RATES_MASK)
+#define IEEE80211_DEFAULT_RATES_MASK		\
+	(IEEE80211_OFDM_DEFAULT_RATES_MASK |	\
+	 IEEE80211_CCK_DEFAULT_RATES_MASK)
 
 #define IEEE80211_NUM_OFDM_RATES	    8
 #define IEEE80211_NUM_CCK_RATES	            4
@@ -509,7 +511,7 @@ join_res:
 
 static inline int is_multicast_mac_addr(const u8 *addr)
 {
-        return ((addr[0] != 0xff) && (0x01 & addr[0]));
+	return ((addr[0] != 0xff) && (0x01 & addr[0]));
 }
 
 static inline int is_broadcast_mac_addr(const u8 *addr)
@@ -612,17 +614,18 @@ enum {
  * @RTW_IEEE80211_CHAN_NO_HT40MINUS: extension channel below this channel
  *      is not permitted.
  */
-  enum rtw_ieee80211_channel_flags {
-          RTW_IEEE80211_CHAN_DISABLED         = 1<<0,
-          RTW_IEEE80211_CHAN_PASSIVE_SCAN     = 1<<1,
-          RTW_IEEE80211_CHAN_NO_IBSS          = 1<<2,
-          RTW_IEEE80211_CHAN_RADAR            = 1<<3,
-          RTW_IEEE80211_CHAN_NO_HT40PLUS      = 1<<4,
-          RTW_IEEE80211_CHAN_NO_HT40MINUS     = 1<<5,
-  };
-
-  #define RTW_IEEE80211_CHAN_NO_HT40 \
-          (RTW_IEEE80211_CHAN_NO_HT40PLUS | RTW_IEEE80211_CHAN_NO_HT40MINUS)
+enum rtw_ieee80211_channel_flags {
+	RTW_IEEE80211_CHAN_DISABLED         = 1<<0,
+	RTW_IEEE80211_CHAN_PASSIVE_SCAN     = 1<<1,
+	RTW_IEEE80211_CHAN_NO_IBSS          = 1<<2,
+	RTW_IEEE80211_CHAN_RADAR            = 1<<3,
+	RTW_IEEE80211_CHAN_NO_HT40PLUS      = 1<<4,
+	RTW_IEEE80211_CHAN_NO_HT40MINUS     = 1<<5,
+};
+
+#define RTW_IEEE80211_CHAN_NO_HT40	  \
+	(RTW_IEEE80211_CHAN_NO_HT40PLUS | \
+	 RTW_IEEE80211_CHAN_NO_HT40MINUS)
 
 /* Represent channel details, subset of ieee80211_channel */
 struct rtw_ieee80211_channel {
diff --git a/drivers/staging/rtl8723bs/include/rtw_mlme.h b/drivers/staging/rtl8723bs/include/rtw_mlme.h
index 1b343b434f4d..fc0b43d38d9a 100644
--- a/drivers/staging/rtl8723bs/include/rtw_mlme.h
+++ b/drivers/staging/rtl8723bs/include/rtw_mlme.h
@@ -303,7 +303,6 @@ struct mlme_priv {
 	struct __queue	free_bss_pool;
 	struct __queue	scanned_queue;
 	u8 *free_bss_buf;
-	u32 num_of_scanned;
 
 	struct ndis_802_11_ssid	assoc_ssid;
 	u8 assoc_bssid[6];
@@ -490,25 +489,6 @@ static inline void _clr_fwstate_(struct mlme_priv *pmlmepriv, signed int state)
 		pmlmepriv->bScanInProcess = false;
 }
 
-/*
- * No Limit on the calling context,
- * therefore set it to be the critical section...
- */
-static inline void clr_fwstate(struct mlme_priv *pmlmepriv, signed int state)
-{
-	spin_lock_bh(&pmlmepriv->lock);
-	if (check_fwstate(pmlmepriv, state) == true)
-		pmlmepriv->fw_state ^= state;
-	spin_unlock_bh(&pmlmepriv->lock);
-}
-
-static inline void set_scanned_network_val(struct mlme_priv *pmlmepriv, signed int val)
-{
-	spin_lock_bh(&pmlmepriv->lock);
-	pmlmepriv->num_of_scanned = val;
-	spin_unlock_bh(&pmlmepriv->lock);
-}
-
 extern u16 rtw_get_capability(struct wlan_bssid_ex *bss);
 extern void rtw_update_scanned_network(struct adapter *adapter, struct wlan_bssid_ex *target);
 extern void rtw_disconnect_hdl_under_linked(struct adapter *adapter, struct sta_info *psta, u8 free_assoc);
diff --git a/drivers/staging/rtl8723bs/include/rtw_recv.h b/drivers/staging/rtl8723bs/include/rtw_recv.h
index 44f67103503a..fef2fd0e8c84 100644
--- a/drivers/staging/rtl8723bs/include/rtw_recv.h
+++ b/drivers/staging/rtl8723bs/include/rtw_recv.h
@@ -398,8 +398,7 @@ static inline u8 *recvframe_pull(union recv_frame *precvframe, signed int sz)
 
 	precvframe->u.hdr.rx_data += sz;
 
-	if (precvframe->u.hdr.rx_data > precvframe->u.hdr.rx_tail)
-	{
+	if (precvframe->u.hdr.rx_data > precvframe->u.hdr.rx_tail) {
 		precvframe->u.hdr.rx_data -= sz;
 		return NULL;
 	}
@@ -425,8 +424,7 @@ static inline u8 *recvframe_put(union recv_frame *precvframe, signed int sz)
 
 	precvframe->u.hdr.rx_tail += sz;
 
-	if (precvframe->u.hdr.rx_tail > precvframe->u.hdr.rx_end)
-	{
+	if (precvframe->u.hdr.rx_tail > precvframe->u.hdr.rx_end) {
 		precvframe->u.hdr.rx_tail = prev_rx_tail;
 		return NULL;
 	}
@@ -451,8 +449,7 @@ static inline u8 *recvframe_pull_tail(union recv_frame *precvframe, signed int s
 
 	precvframe->u.hdr.rx_tail -= sz;
 
-	if (precvframe->u.hdr.rx_tail < precvframe->u.hdr.rx_data)
-	{
+	if (precvframe->u.hdr.rx_tail < precvframe->u.hdr.rx_data) {
 		precvframe->u.hdr.rx_tail += sz;
 		return NULL;
 	}
diff --git a/drivers/staging/rts5208/rtsx.c b/drivers/staging/rts5208/rtsx.c
index 2284a96abcff..db2dd0baa8be 100644
--- a/drivers/staging/rts5208/rtsx.c
+++ b/drivers/staging/rts5208/rtsx.c
@@ -191,7 +191,7 @@ static int device_reset(struct scsi_cmnd *srb)
  * this defines our host template, with which we'll allocate hosts
  */
 
-static struct scsi_host_template rtsx_host_template = {
+static const struct scsi_host_template rtsx_host_template = {
 	/* basic userland interface stuff */
 	.name =				CR_DRIVER_NAME,
 	.proc_name =			CR_DRIVER_NAME,
diff --git a/drivers/staging/rts5208/xd.c b/drivers/staging/rts5208/xd.c
index 42cab93982c0..c0af378ada71 100644
--- a/drivers/staging/rts5208/xd.c
+++ b/drivers/staging/rts5208/xd.c
@@ -31,13 +31,6 @@ static inline void xd_set_err_code(struct rtsx_chip *chip, u8 err_code)
 	xd_card->err_code = err_code;
 }
 
-static inline int xd_check_err_code(struct rtsx_chip *chip, u8 err_code)
-{
-	struct xd_info *xd_card = &chip->xd_card;
-
-	return (xd_card->err_code == err_code);
-}
-
 static int xd_set_init_para(struct rtsx_chip *chip)
 {
 	struct xd_info *xd_card = &chip->xd_card;
diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c
index effc7fcc3703..22ace3168723 100644
--- a/drivers/staging/sm750fb/sm750.c
+++ b/drivers/staging/sm750fb/sm750.c
@@ -989,20 +989,6 @@ release_fb:
 	return err;
 }
 
-static int lynxfb_kick_out_firmware_fb(struct pci_dev *pdev)
-{
-	resource_size_t base = pci_resource_start(pdev, 0);
-	resource_size_t size = pci_resource_len(pdev, 0);
-	bool primary = false;
-
-#ifdef CONFIG_X86
-	primary = pdev->resource[PCI_ROM_RESOURCE].flags &
-					IORESOURCE_ROM_SHADOW;
-#endif
-
-	return aperture_remove_conflicting_devices(base, size, primary, "sm750_fb1");
-}
-
 static int lynxfb_pci_probe(struct pci_dev *pdev,
 			    const struct pci_device_id *ent)
 {
@@ -1011,7 +997,7 @@ static int lynxfb_pci_probe(struct pci_dev *pdev,
 	int fbidx;
 	int err;
 
-	err = lynxfb_kick_out_firmware_fb(pdev);
+	err = aperture_remove_conflicting_pci_devices(pdev, "sm750_fb1");
 	if (err)
 		return err;
 
diff --git a/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c b/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c
index 4f81765912ea..346d00df815a 100644
--- a/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c
+++ b/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c
@@ -1976,7 +1976,7 @@ cleanup_mmal:
 	return ret;
 }
 
-static int bcm2835_mmal_remove(struct platform_device *pdev)
+static void bcm2835_mmal_remove(struct platform_device *pdev)
 {
 	int camera;
 	struct vchiq_mmal_instance *instance = gdev[0]->instance;
@@ -1986,13 +1986,11 @@ static int bcm2835_mmal_remove(struct platform_device *pdev)
 		gdev[camera] = NULL;
 	}
 	vchiq_mmal_finalise(instance);
-
-	return 0;
 }
 
 static struct platform_driver bcm2835_camera_driver = {
 	.probe		= bcm2835_mmal_probe,
-	.remove		= bcm2835_mmal_remove,
+	.remove_new	= bcm2835_mmal_remove,
 	.driver		= {
 		.name	= "bcm2835-camera",
 	},
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
index cddcd3c596c9..90a3958d1f29 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
@@ -149,7 +149,7 @@ static char *g_fragments_base;
 static char *g_free_fragments;
 static struct semaphore g_free_fragments_sema;
 
-static DEFINE_SEMAPHORE(g_free_fragments_mutex);
+static DEFINE_SEMAPHORE(g_free_fragments_mutex, 1);
 
 static int
 vchiq_blocking_bulk_transfer(struct vchiq_instance *instance, unsigned int handle, void *data,
@@ -1849,14 +1849,12 @@ error_exit:
 	return err;
 }
 
-static int vchiq_remove(struct platform_device *pdev)
+static void vchiq_remove(struct platform_device *pdev)
 {
 	platform_device_unregister(bcm2835_audio);
 	platform_device_unregister(bcm2835_camera);
 	vchiq_debugfs_deinit();
 	vchiq_deregister_chrdev();
-
-	return 0;
 }
 
 static struct platform_driver vchiq_driver = {
@@ -1865,7 +1863,7 @@ static struct platform_driver vchiq_driver = {
 		.of_match_table = vchiq_of_match,
 	},
 	.probe = vchiq_probe,
-	.remove = vchiq_remove,
+	.remove_new = vchiq_remove,
 };
 
 static int __init vchiq_driver_init(void)
diff --git a/drivers/staging/vme_user/Kconfig b/drivers/staging/vme_user/Kconfig
index c8eabf8f40f1..d65cc5510649 100644
--- a/drivers/staging/vme_user/Kconfig
+++ b/drivers/staging/vme_user/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 menuconfig VME_BUS
 	bool "VME bridge support"
-	depends on STAGING && PCI
+	depends on PCI
 	help
 	  If you say Y here you get support for the VME bridge Framework.
 
@@ -28,7 +28,6 @@ comment "VME Device Drivers"
 
 config VME_USER
 	tristate "VME user space access driver"
-	depends on STAGING && VME_BUS
 	help
 	  If you say Y here you want to be able to access a limited number of
 	  VME windows in a manner at least semi-compatible with the interface
diff --git a/drivers/staging/vme_user/vme_fake.c b/drivers/staging/vme_user/vme_fake.c
index f5d2c345978a..7c53a8a7b79b 100644
--- a/drivers/staging/vme_user/vme_fake.c
+++ b/drivers/staging/vme_user/vme_fake.c
@@ -329,7 +329,6 @@ err_aspace:
 err_dwidth:
 err_window:
 	return retval;
-
 }
 
 /*
@@ -638,7 +637,6 @@ static noinline_for_stack void fake_vmewrite8(struct fake_driver *bridge,
 	}
 
 	fake_lm_check(bridge, addr, aspace, cycle);
-
 }
 
 static noinline_for_stack void fake_vmewrite16(struct fake_driver *bridge,
@@ -669,7 +667,6 @@ static noinline_for_stack void fake_vmewrite16(struct fake_driver *bridge,
 	}
 
 	fake_lm_check(bridge, addr, aspace, cycle);
-
 }
 
 static noinline_for_stack void fake_vmewrite32(struct fake_driver *bridge,
@@ -700,7 +697,6 @@ static noinline_for_stack void fake_vmewrite32(struct fake_driver *bridge,
 	}
 
 	fake_lm_check(bridge, addr, aspace, cycle);
-
 }
 
 static ssize_t fake_master_write(struct vme_master_resource *image, void *buf,
@@ -1234,7 +1230,6 @@ err_driver:
 	kfree(fake_bridge);
 err_struct:
 	return retval;
-
 }
 
 static void __exit fake_exit(void)
diff --git a/drivers/staging/vme_user/vme_tsi148.c b/drivers/staging/vme_user/vme_tsi148.c
index 482049cfc664..2f5eafd50934 100644
--- a/drivers/staging/vme_user/vme_tsi148.c
+++ b/drivers/staging/vme_user/vme_tsi148.c
@@ -737,7 +737,7 @@ static int tsi148_alloc_resource(struct vme_master_resource *image,
 		return 0;
 
 	if (!image->bus_resource.name) {
-		image->bus_resource.name = kmalloc(VMENAMSIZ+3, GFP_ATOMIC);
+		image->bus_resource.name = kmalloc(VMENAMSIZ + 3, GFP_ATOMIC);
 		if (!image->bus_resource.name) {
 			retval = -ENOMEM;
 			goto err_name;
@@ -983,7 +983,7 @@ static int tsi148_master_set(struct vme_master_resource *image, int enabled,
 		goto err_aspace;
 	}
 
-	temp_ctl &= ~(3<<4);
+	temp_ctl &= ~(3 << 4);
 	if (cycle & VME_SUPER)
 		temp_ctl |= TSI148_LCSR_OTAT_SUP;
 	if (cycle & VME_PROG)
@@ -1023,7 +1023,6 @@ err_gran:
 err_res:
 err_window:
 	return retval;
-
 }
 
 /*
@@ -1741,7 +1740,6 @@ static int tsi148_dma_list_add(struct vme_dma_list *list,
 				  list);
 		prev->descriptor.dnlau = cpu_to_be32(address_high);
 		prev->descriptor.dnlal = cpu_to_be32(address_low);
-
 	}
 
 	return 0;
@@ -1773,7 +1771,6 @@ static int tsi148_dma_busy(struct vme_bridge *tsi148_bridge, int channel)
 		return 0;
 	else
 		return 1;
-
 }
 
 /*
@@ -2187,14 +2184,14 @@ static int tsi148_crcsr_init(struct vme_bridge *tsi148_bridge,
 
 	/* Ensure that the CR/CSR is configured at the correct offset */
 	cbar = ioread32be(bridge->base + TSI148_CBAR);
-	cbar = (cbar & TSI148_CRCSR_CBAR_M)>>3;
+	cbar = (cbar & TSI148_CRCSR_CBAR_M) >> 3;
 
 	vstat = tsi148_slot_get(tsi148_bridge);
 
 	if (cbar != vstat) {
 		cbar = vstat;
 		dev_info(tsi148_bridge->parent, "Setting CR/CSR offset\n");
-		iowrite32be(cbar<<3, bridge->base + TSI148_CBAR);
+		iowrite32be(cbar << 3, bridge->base + TSI148_CBAR);
 	}
 	dev_info(tsi148_bridge->parent, "CR/CSR Offset: %d\n", cbar);
 
@@ -2220,7 +2217,6 @@ static int tsi148_crcsr_init(struct vme_bridge *tsi148_bridge,
 	}
 
 	return 0;
-
 }
 
 static void tsi148_crcsr_exit(struct vme_bridge *tsi148_bridge,
@@ -2530,7 +2526,6 @@ err_driver:
 	kfree(tsi148_bridge);
 err_struct:
 	return retval;
-
 }
 
 static void tsi148_remove(struct pci_dev *pdev)
diff --git a/drivers/staging/vme_user/vme_tsi148.h b/drivers/staging/vme_user/vme_tsi148.h
index b3cb4a089cc8..63f726e1811a 100644
--- a/drivers/staging/vme_user/vme_tsi148.h
+++ b/drivers/staging/vme_user/vme_tsi148.h
@@ -536,22 +536,22 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 	/*
 	 *  PFCS Register Set
 	 */
-#define TSI148_PCFS_CMMD_SERR          (1<<8)	/* SERR_L out pin ssys err */
-#define TSI148_PCFS_CMMD_PERR          (1<<6)	/* PERR_L out pin  parity */
-#define TSI148_PCFS_CMMD_MSTR          (1<<2)	/* PCI bus master */
-#define TSI148_PCFS_CMMD_MEMSP         (1<<1)	/* PCI mem space access  */
-#define TSI148_PCFS_CMMD_IOSP          (1<<0)	/* PCI I/O space enable */
-
-#define TSI148_PCFS_STAT_RCPVE         (1<<15)	/* Detected Parity Error */
-#define TSI148_PCFS_STAT_SIGSE         (1<<14)	/* Signalled System Error */
-#define TSI148_PCFS_STAT_RCVMA         (1<<13)	/* Received Master Abort */
-#define TSI148_PCFS_STAT_RCVTA         (1<<12)	/* Received Target Abort */
-#define TSI148_PCFS_STAT_SIGTA         (1<<11)	/* Signalled Target Abort */
+#define TSI148_PCFS_CMMD_SERR          BIT(8)	/* SERR_L out pin ssys err */
+#define TSI148_PCFS_CMMD_PERR          BIT(6)	/* PERR_L out pin  parity */
+#define TSI148_PCFS_CMMD_MSTR          BIT(2)	/* PCI bus master */
+#define TSI148_PCFS_CMMD_MEMSP         BIT(1)	/* PCI mem space access  */
+#define TSI148_PCFS_CMMD_IOSP          BIT(0)	/* PCI I/O space enable */
+
+#define TSI148_PCFS_STAT_RCPVE         BIT(15)	/* Detected Parity Error */
+#define TSI148_PCFS_STAT_SIGSE         BIT(14)	/* Signalled System Error */
+#define TSI148_PCFS_STAT_RCVMA         BIT(13)	/* Received Master Abort */
+#define TSI148_PCFS_STAT_RCVTA         BIT(12)	/* Received Target Abort */
+#define TSI148_PCFS_STAT_SIGTA         BIT(11)	/* Signalled Target Abort */
 #define TSI148_PCFS_STAT_SELTIM        (3<<9)	/* DELSEL Timing */
-#define TSI148_PCFS_STAT_DPAR          (1<<8)	/* Data Parity Err Reported */
-#define TSI148_PCFS_STAT_FAST          (1<<7)	/* Fast back-to-back Cap */
-#define TSI148_PCFS_STAT_P66M          (1<<5)	/* 66 MHz Capable */
-#define TSI148_PCFS_STAT_CAPL          (1<<4)	/* Capab List - address $34 */
+#define TSI148_PCFS_STAT_DPAR          BIT(8)	/* Data Parity Err Reported */
+#define TSI148_PCFS_STAT_FAST          BIT(7)	/* Fast back-to-back Cap */
+#define TSI148_PCFS_STAT_P66M          BIT(5)	/* 66 MHz Capable */
+#define TSI148_PCFS_STAT_CAPL          BIT(4)	/* Capab List - address $34 */
 
 /*
  *  Revision ID/Class Code Registers   (CRG +$008)
@@ -572,17 +572,17 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
  *  Memory Base Address Lower Reg (CRG + $010)
  */
 #define TSI148_PCFS_MBARL_BASEL_M      (0xFFFFF<<12) /* Base Addr Lower Mask */
-#define TSI148_PCFS_MBARL_PRE          (1<<3)	/* Prefetch */
+#define TSI148_PCFS_MBARL_PRE          BIT(3)	/* Prefetch */
 #define TSI148_PCFS_MBARL_MTYPE_M      (3<<1)	/* Memory Type Mask */
-#define TSI148_PCFS_MBARL_IOMEM        (1<<0)	/* I/O Space Indicator */
+#define TSI148_PCFS_MBARL_IOMEM        BIT(0)	/* I/O Space Indicator */
 
 /*
  *  Message Signaled Interrupt Capabilities Register (CRG + $040)
  */
-#define TSI148_PCFS_MSICAP_64BAC       (1<<7)	/* 64-bit Address Capable */
+#define TSI148_PCFS_MSICAP_64BAC       BIT(7)	/* 64-bit Address Capable */
 #define TSI148_PCFS_MSICAP_MME_M       (7<<4)	/* Multiple Msg Enable Mask */
 #define TSI148_PCFS_MSICAP_MMC_M       (7<<1)	/* Multiple Msg Capable Mask */
-#define TSI148_PCFS_MSICAP_MSIEN       (1<<0)	/* Msg signaled INT Enable */
+#define TSI148_PCFS_MSICAP_MSIEN       BIT(0)	/* Msg signaled INT Enable */
 
 /*
  *  Message Address Lower Register (CRG +$044)
@@ -599,22 +599,22 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
  */
 #define TSI148_PCFS_PCIXCAP_MOST_M     (7<<4)	/* Max outstanding Split Tran */
 #define TSI148_PCFS_PCIXCAP_MMRBC_M    (3<<2)	/* Max Mem Read byte cnt */
-#define TSI148_PCFS_PCIXCAP_ERO        (1<<1)	/* Enable Relaxed Ordering */
-#define TSI148_PCFS_PCIXCAP_DPERE      (1<<0)	/* Data Parity Recover Enable */
+#define TSI148_PCFS_PCIXCAP_ERO        BIT(1)	/* Enable Relaxed Ordering */
+#define TSI148_PCFS_PCIXCAP_DPERE      BIT(0)	/* Data Parity Recover Enable */
 
 /*
  *  PCI-X Status Register (CRG +$054)
  */
-#define TSI148_PCFS_PCIXSTAT_RSCEM     (1<<29)	/* Received Split Comp Error */
+#define TSI148_PCFS_PCIXSTAT_RSCEM     BIT(29)	/* Received Split Comp Error */
 #define TSI148_PCFS_PCIXSTAT_DMCRS_M   (7<<26)	/* max Cumulative Read Size */
 #define TSI148_PCFS_PCIXSTAT_DMOST_M   (7<<23)	/* max outstanding Split Trans
 						 */
 #define TSI148_PCFS_PCIXSTAT_DMMRC_M   (3<<21)	/* max mem read byte count */
-#define TSI148_PCFS_PCIXSTAT_DC        (1<<20)	/* Device Complexity */
-#define TSI148_PCFS_PCIXSTAT_USC       (1<<19)	/* Unexpected Split comp */
-#define TSI148_PCFS_PCIXSTAT_SCD       (1<<18)	/* Split completion discard */
-#define TSI148_PCFS_PCIXSTAT_133C      (1<<17)	/* 133MHz capable */
-#define TSI148_PCFS_PCIXSTAT_64D       (1<<16)	/* 64 bit device */
+#define TSI148_PCFS_PCIXSTAT_DC        BIT(20)	/* Device Complexity */
+#define TSI148_PCFS_PCIXSTAT_USC       BIT(19)	/* Unexpected Split comp */
+#define TSI148_PCFS_PCIXSTAT_SCD       BIT(18)	/* Split completion discard */
+#define TSI148_PCFS_PCIXSTAT_133C      BIT(17)	/* 133MHz capable */
+#define TSI148_PCFS_PCIXSTAT_64D       BIT(16)	/* 64 bit device */
 #define TSI148_PCFS_PCIXSTAT_BN_M      (0xFF<<8)	/* Bus number */
 #define TSI148_PCFS_PCIXSTAT_DN_M      (0x1F<<3)	/* Device number */
 #define TSI148_PCFS_PCIXSTAT_FN_M      (7<<0)	/* Function Number */
@@ -646,23 +646,23 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 /*
  *  Outbound Translation Attribute
  */
-#define TSI148_LCSR_OTAT_EN            (1<<31)	/* Window Enable */
-#define TSI148_LCSR_OTAT_MRPFD         (1<<18)	/* Prefetch Disable */
+#define TSI148_LCSR_OTAT_EN            BIT(31)	/* Window Enable */
+#define TSI148_LCSR_OTAT_MRPFD         BIT(18)	/* Prefetch Disable */
 
 #define TSI148_LCSR_OTAT_PFS_M         (3<<16)	/* Prefetch Size Mask */
 #define TSI148_LCSR_OTAT_PFS_2         (0<<16)	/* 2 Cache Lines P Size */
-#define TSI148_LCSR_OTAT_PFS_4         (1<<16)	/* 4 Cache Lines P Size */
+#define TSI148_LCSR_OTAT_PFS_4         BIT(16)	/* 4 Cache Lines P Size */
 #define TSI148_LCSR_OTAT_PFS_8         (2<<16)	/* 8 Cache Lines P Size */
 #define TSI148_LCSR_OTAT_PFS_16        (3<<16)	/* 16 Cache Lines P Size */
 
 #define TSI148_LCSR_OTAT_2eSSTM_M      (7<<11)	/* 2eSST Xfer Rate Mask */
 #define TSI148_LCSR_OTAT_2eSSTM_160    (0<<11)	/* 160MB/s 2eSST Xfer Rate */
-#define TSI148_LCSR_OTAT_2eSSTM_267    (1<<11)	/* 267MB/s 2eSST Xfer Rate */
+#define TSI148_LCSR_OTAT_2eSSTM_267    BIT(11)	/* 267MB/s 2eSST Xfer Rate */
 #define TSI148_LCSR_OTAT_2eSSTM_320    (2<<11)	/* 320MB/s 2eSST Xfer Rate */
 
 #define TSI148_LCSR_OTAT_TM_M          (7<<8)	/* Xfer Protocol Mask */
 #define TSI148_LCSR_OTAT_TM_SCT        (0<<8)	/* SCT Xfer Protocol */
-#define TSI148_LCSR_OTAT_TM_BLT        (1<<8)	/* BLT Xfer Protocol */
+#define TSI148_LCSR_OTAT_TM_BLT        BIT(8)	/* BLT Xfer Protocol */
 #define TSI148_LCSR_OTAT_TM_MBLT       (2<<8)	/* MBLT Xfer Protocol */
 #define TSI148_LCSR_OTAT_TM_2eVME      (3<<8)	/* 2eVME Xfer Protocol */
 #define TSI148_LCSR_OTAT_TM_2eSST      (4<<8)	/* 2eSST Xfer Protocol */
@@ -670,14 +670,14 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 
 #define TSI148_LCSR_OTAT_DBW_M         (3<<6)	/* Max Data Width */
 #define TSI148_LCSR_OTAT_DBW_16        (0<<6)	/* 16-bit Data Width */
-#define TSI148_LCSR_OTAT_DBW_32        (1<<6)	/* 32-bit Data Width */
+#define TSI148_LCSR_OTAT_DBW_32        BIT(6)	/* 32-bit Data Width */
 
-#define TSI148_LCSR_OTAT_SUP           (1<<5)	/* Supervisory Access */
-#define TSI148_LCSR_OTAT_PGM           (1<<4)	/* Program Access */
+#define TSI148_LCSR_OTAT_SUP           BIT(5)	/* Supervisory Access */
+#define TSI148_LCSR_OTAT_PGM           BIT(4)	/* Program Access */
 
 #define TSI148_LCSR_OTAT_AMODE_M       (0xf<<0)	/* Address Mode Mask */
 #define TSI148_LCSR_OTAT_AMODE_A16     (0<<0)	/* A16 Address Space */
-#define TSI148_LCSR_OTAT_AMODE_A24     (1<<0)	/* A24 Address Space */
+#define TSI148_LCSR_OTAT_AMODE_A24     BIT(0)	/* A24 Address Space */
 #define TSI148_LCSR_OTAT_AMODE_A32     (2<<0)	/* A32 Address Space */
 #define TSI148_LCSR_OTAT_AMODE_A64     (4<<0)	/* A32 Address Space */
 #define TSI148_LCSR_OTAT_AMODE_CRCSR   (5<<0)	/* CR/CSR Address Space */
@@ -689,17 +689,17 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 /*
  *  VME Master Control Register  CRG+$234
  */
-#define TSI148_LCSR_VMCTRL_VSA         (1<<27)	/* VMEbus Stop Ack */
-#define TSI148_LCSR_VMCTRL_VS          (1<<26)	/* VMEbus Stop */
-#define TSI148_LCSR_VMCTRL_DHB         (1<<25)	/* Device Has Bus */
-#define TSI148_LCSR_VMCTRL_DWB         (1<<24)	/* Device Wants Bus */
+#define TSI148_LCSR_VMCTRL_VSA         BIT(27)	/* VMEbus Stop Ack */
+#define TSI148_LCSR_VMCTRL_VS          BIT(26)	/* VMEbus Stop */
+#define TSI148_LCSR_VMCTRL_DHB         BIT(25)	/* Device Has Bus */
+#define TSI148_LCSR_VMCTRL_DWB         BIT(24)	/* Device Wants Bus */
 
-#define TSI148_LCSR_VMCTRL_RMWEN       (1<<20)	/* RMW Enable */
+#define TSI148_LCSR_VMCTRL_RMWEN       BIT(20)	/* RMW Enable */
 
 #define TSI148_LCSR_VMCTRL_ATO_M       (7<<16)	/* Master Access Time-out Mask
 						 */
 #define TSI148_LCSR_VMCTRL_ATO_32      (0<<16)	/* 32 us */
-#define TSI148_LCSR_VMCTRL_ATO_128     (1<<16)	/* 128 us */
+#define TSI148_LCSR_VMCTRL_ATO_128     BIT(16)	/* 128 us */
 #define TSI148_LCSR_VMCTRL_ATO_512     (2<<16)	/* 512 us */
 #define TSI148_LCSR_VMCTRL_ATO_2M      (3<<16)	/* 2 ms */
 #define TSI148_LCSR_VMCTRL_ATO_8M      (4<<16)	/* 8 ms */
@@ -709,7 +709,7 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 
 #define TSI148_LCSR_VMCTRL_VTOFF_M     (7<<12)	/* VMEbus Master Time off */
 #define TSI148_LCSR_VMCTRL_VTOFF_0     (0<<12)	/* 0us */
-#define TSI148_LCSR_VMCTRL_VTOFF_1     (1<<12)	/* 1us */
+#define TSI148_LCSR_VMCTRL_VTOFF_1     BIT(12)	/* 1us */
 #define TSI148_LCSR_VMCTRL_VTOFF_2     (2<<12)	/* 2us */
 #define TSI148_LCSR_VMCTRL_VTOFF_4     (3<<12)	/* 4us */
 #define TSI148_LCSR_VMCTRL_VTOFF_8     (4<<12)	/* 8us */
@@ -719,7 +719,7 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 
 #define TSI148_LCSR_VMCTRL_VTON_M      (7<<8)	/* VMEbus Master Time On */
 #define TSI148_LCSR_VMCTRL_VTON_4      (0<<8)	/* 8us */
-#define TSI148_LCSR_VMCTRL_VTON_8      (1<<8)	/* 8us */
+#define TSI148_LCSR_VMCTRL_VTON_8      BIT(8)	/* 8us */
 #define TSI148_LCSR_VMCTRL_VTON_16     (2<<8)	/* 16us */
 #define TSI148_LCSR_VMCTRL_VTON_32     (3<<8)	/* 32us */
 #define TSI148_LCSR_VMCTRL_VTON_64     (4<<8)	/* 64us */
@@ -730,22 +730,22 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 #define TSI148_LCSR_VMCTRL_VREL_M      (3<<3)	/* VMEbus Master Rel Mode Mask
 						 */
 #define TSI148_LCSR_VMCTRL_VREL_T_D    (0<<3)	/* Time on or Done */
-#define TSI148_LCSR_VMCTRL_VREL_T_R_D  (1<<3)	/* Time on and REQ or Done */
+#define TSI148_LCSR_VMCTRL_VREL_T_R_D  BIT(3)	/* Time on and REQ or Done */
 #define TSI148_LCSR_VMCTRL_VREL_T_B_D  (2<<3)	/* Time on and BCLR or Done */
 #define TSI148_LCSR_VMCTRL_VREL_T_D_R  (3<<3)	/* Time on or Done and REQ */
 
-#define TSI148_LCSR_VMCTRL_VFAIR       (1<<2)	/* VMEbus Master Fair Mode */
+#define TSI148_LCSR_VMCTRL_VFAIR       BIT(2)	/* VMEbus Master Fair Mode */
 #define TSI148_LCSR_VMCTRL_VREQL_M     (3<<0)	/* VMEbus Master Req Level Mask
 						 */
 
 /*
  *  VMEbus Control Register CRG+$238
  */
-#define TSI148_LCSR_VCTRL_LRE          (1<<31)	/* Late Retry Enable */
+#define TSI148_LCSR_VCTRL_LRE          BIT(31)	/* Late Retry Enable */
 
 #define TSI148_LCSR_VCTRL_DLT_M        (0xF<<24)	/* Deadlock Timer */
 #define TSI148_LCSR_VCTRL_DLT_OFF      (0<<24)	/* Deadlock Timer Off */
-#define TSI148_LCSR_VCTRL_DLT_16       (1<<24)	/* 16 VCLKS */
+#define TSI148_LCSR_VCTRL_DLT_16       BIT(24)	/* 16 VCLKS */
 #define TSI148_LCSR_VCTRL_DLT_32       (2<<24)	/* 32 VCLKS */
 #define TSI148_LCSR_VCTRL_DLT_64       (3<<24)	/* 64 VCLKS */
 #define TSI148_LCSR_VCTRL_DLT_128      (4<<24)	/* 128 VCLKS */
@@ -758,22 +758,22 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 #define TSI148_LCSR_VCTRL_DLT_16384    (0xB<<24)	/* 16384 VCLKS */
 #define TSI148_LCSR_VCTRL_DLT_32768    (0xC<<24)	/* 32768 VCLKS */
 
-#define TSI148_LCSR_VCTRL_NERBB        (1<<20)	/* No Early Release of Bus Busy
+#define TSI148_LCSR_VCTRL_NERBB        BIT(20)	/* No Early Release of Bus Busy
 						 */
 
-#define TSI148_LCSR_VCTRL_SRESET       (1<<17)	/* System Reset */
-#define TSI148_LCSR_VCTRL_LRESET       (1<<16)	/* Local Reset */
+#define TSI148_LCSR_VCTRL_SRESET       BIT(17)	/* System Reset */
+#define TSI148_LCSR_VCTRL_LRESET       BIT(16)	/* Local Reset */
 
-#define TSI148_LCSR_VCTRL_SFAILAI      (1<<15)	/* SYSFAIL Auto Slot ID */
+#define TSI148_LCSR_VCTRL_SFAILAI      BIT(15)	/* SYSFAIL Auto Slot ID */
 #define TSI148_LCSR_VCTRL_BID_M        (0x1F<<8)	/* Broadcast ID Mask */
 
-#define TSI148_LCSR_VCTRL_ATOEN        (1<<7)	/* Arbiter Time-out Enable */
-#define TSI148_LCSR_VCTRL_ROBIN        (1<<6)	/* VMEbus Round Robin */
+#define TSI148_LCSR_VCTRL_ATOEN        BIT(7)	/* Arbiter Time-out Enable */
+#define TSI148_LCSR_VCTRL_ROBIN        BIT(6)	/* VMEbus Round Robin */
 
 #define TSI148_LCSR_VCTRL_GTO_M        (7<<0)	/* VMEbus Global Time-out Mask
 						 */
 #define TSI148_LCSR_VCTRL_GTO_8	      (0<<0)	/* 8 us */
-#define TSI148_LCSR_VCTRL_GTO_16	      (1<<0)	/* 16 us */
+#define TSI148_LCSR_VCTRL_GTO_16	      BIT(0)	/* 16 us */
 #define TSI148_LCSR_VCTRL_GTO_32	      (2<<0)	/* 32 us */
 #define TSI148_LCSR_VCTRL_GTO_64	      (3<<0)	/* 64 us */
 #define TSI148_LCSR_VCTRL_GTO_128      (4<<0)	/* 128 us */
@@ -784,48 +784,48 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 /*
  *  VMEbus Status Register  CRG + $23C
  */
-#define TSI148_LCSR_VSTAT_CPURST       (1<<15)	/* Clear power up reset */
-#define TSI148_LCSR_VSTAT_BRDFL        (1<<14)	/* Board fail */
-#define TSI148_LCSR_VSTAT_PURSTS       (1<<12)	/* Power up reset status */
-#define TSI148_LCSR_VSTAT_BDFAILS      (1<<11)	/* Board Fail Status */
-#define TSI148_LCSR_VSTAT_SYSFAILS     (1<<10)	/* System Fail Status */
-#define TSI148_LCSR_VSTAT_ACFAILS      (1<<9)	/* AC fail status */
-#define TSI148_LCSR_VSTAT_SCONS        (1<<8)	/* System Cont Status */
-#define TSI148_LCSR_VSTAT_GAP          (1<<5)	/* Geographic Addr Parity */
+#define TSI148_LCSR_VSTAT_CPURST       BIT(15)	/* Clear power up reset */
+#define TSI148_LCSR_VSTAT_BRDFL        BIT(14)	/* Board fail */
+#define TSI148_LCSR_VSTAT_PURSTS       BIT(12)	/* Power up reset status */
+#define TSI148_LCSR_VSTAT_BDFAILS      BIT(11)	/* Board Fail Status */
+#define TSI148_LCSR_VSTAT_SYSFAILS     BIT(10)	/* System Fail Status */
+#define TSI148_LCSR_VSTAT_ACFAILS      BIT(9)	/* AC fail status */
+#define TSI148_LCSR_VSTAT_SCONS        BIT(8)	/* System Cont Status */
+#define TSI148_LCSR_VSTAT_GAP          BIT(5)	/* Geographic Addr Parity */
 #define TSI148_LCSR_VSTAT_GA_M         (0x1F<<0)  /* Geographic Addr Mask */
 
 /*
  *  PCI Configuration Status Register CRG+$240
  */
-#define TSI148_LCSR_PSTAT_REQ64S       (1<<6)	/* Request 64 status set */
-#define TSI148_LCSR_PSTAT_M66ENS       (1<<5)	/* M66ENS 66Mhz enable */
-#define TSI148_LCSR_PSTAT_FRAMES       (1<<4)	/* Frame Status */
-#define TSI148_LCSR_PSTAT_IRDYS        (1<<3)	/* IRDY status */
-#define TSI148_LCSR_PSTAT_DEVSELS      (1<<2)	/* DEVL status */
-#define TSI148_LCSR_PSTAT_STOPS        (1<<1)	/* STOP status */
-#define TSI148_LCSR_PSTAT_TRDYS        (1<<0)	/* TRDY status */
+#define TSI148_LCSR_PSTAT_REQ64S       BIT(6)	/* Request 64 status set */
+#define TSI148_LCSR_PSTAT_M66ENS       BIT(5)	/* M66ENS 66Mhz enable */
+#define TSI148_LCSR_PSTAT_FRAMES       BIT(4)	/* Frame Status */
+#define TSI148_LCSR_PSTAT_IRDYS        BIT(3)	/* IRDY status */
+#define TSI148_LCSR_PSTAT_DEVSELS      BIT(2)	/* DEVL status */
+#define TSI148_LCSR_PSTAT_STOPS        BIT(1)	/* STOP status */
+#define TSI148_LCSR_PSTAT_TRDYS        BIT(0)	/* TRDY status */
 
 /*
  *  VMEbus Exception Attributes Register  CRG + $268
  */
-#define TSI148_LCSR_VEAT_VES           (1<<31)	/* Status */
-#define TSI148_LCSR_VEAT_VEOF          (1<<30)	/* Overflow */
-#define TSI148_LCSR_VEAT_VESCL         (1<<29)	/* Status Clear */
-#define TSI148_LCSR_VEAT_2EOT          (1<<21)	/* 2e Odd Termination */
-#define TSI148_LCSR_VEAT_2EST          (1<<20)	/* 2e Slave terminated */
-#define TSI148_LCSR_VEAT_BERR          (1<<19)	/* Bus Error */
-#define TSI148_LCSR_VEAT_LWORD         (1<<18)	/* LWORD_ signal state */
-#define TSI148_LCSR_VEAT_WRITE         (1<<17)	/* WRITE_ signal state */
-#define TSI148_LCSR_VEAT_IACK          (1<<16)	/* IACK_ signal state */
-#define TSI148_LCSR_VEAT_DS1           (1<<15)	/* DS1_ signal state */
-#define TSI148_LCSR_VEAT_DS0           (1<<14)	/* DS0_ signal state */
+#define TSI148_LCSR_VEAT_VES           BIT(31)	/* Status */
+#define TSI148_LCSR_VEAT_VEOF          BIT(30)	/* Overflow */
+#define TSI148_LCSR_VEAT_VESCL         BIT(29)	/* Status Clear */
+#define TSI148_LCSR_VEAT_2EOT          BIT(21)	/* 2e Odd Termination */
+#define TSI148_LCSR_VEAT_2EST          BIT(20)	/* 2e Slave terminated */
+#define TSI148_LCSR_VEAT_BERR          BIT(19)	/* Bus Error */
+#define TSI148_LCSR_VEAT_LWORD         BIT(18)	/* LWORD_ signal state */
+#define TSI148_LCSR_VEAT_WRITE         BIT(17)	/* WRITE_ signal state */
+#define TSI148_LCSR_VEAT_IACK          BIT(16)	/* IACK_ signal state */
+#define TSI148_LCSR_VEAT_DS1           BIT(15)	/* DS1_ signal state */
+#define TSI148_LCSR_VEAT_DS0           BIT(14)	/* DS0_ signal state */
 #define TSI148_LCSR_VEAT_AM_M          (0x3F<<8)	/* Address Mode Mask */
 #define TSI148_LCSR_VEAT_XAM_M         (0xFF<<0)	/* Master AMode Mask */
 
 /*
  * VMEbus PCI Error Diagnostics PCI/X Attributes Register  CRG + $280
  */
-#define TSI148_LCSR_EDPAT_EDPCL        (1<<29)
+#define TSI148_LCSR_EDPAT_EDPCL        BIT(29)
 
 /*
  *  Inbound Translation Starting Address Lower
@@ -851,36 +851,36 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 /*
  *  Inbound Translation Attribute
  */
-#define TSI148_LCSR_ITAT_EN            (1<<31)	/* Window Enable */
-#define TSI148_LCSR_ITAT_TH            (1<<18)	/* Prefetch Threshold */
+#define TSI148_LCSR_ITAT_EN            BIT(31)	/* Window Enable */
+#define TSI148_LCSR_ITAT_TH            BIT(18)	/* Prefetch Threshold */
 
 #define TSI148_LCSR_ITAT_VFS_M         (3<<16)	/* Virtual FIFO Size Mask */
 #define TSI148_LCSR_ITAT_VFS_64        (0<<16)	/* 64 bytes Virtual FIFO Size */
-#define TSI148_LCSR_ITAT_VFS_128       (1<<16)	/* 128 bytes Virtual FIFO Sz */
+#define TSI148_LCSR_ITAT_VFS_128       BIT(16)	/* 128 bytes Virtual FIFO Sz */
 #define TSI148_LCSR_ITAT_VFS_256       (2<<16)	/* 256 bytes Virtual FIFO Sz */
 #define TSI148_LCSR_ITAT_VFS_512       (3<<16)	/* 512 bytes Virtual FIFO Sz */
 
 #define TSI148_LCSR_ITAT_2eSSTM_M      (7<<12)	/* 2eSST Xfer Rate Mask */
 #define TSI148_LCSR_ITAT_2eSSTM_160    (0<<12)	/* 160MB/s 2eSST Xfer Rate */
-#define TSI148_LCSR_ITAT_2eSSTM_267    (1<<12)	/* 267MB/s 2eSST Xfer Rate */
+#define TSI148_LCSR_ITAT_2eSSTM_267    BIT(12)	/* 267MB/s 2eSST Xfer Rate */
 #define TSI148_LCSR_ITAT_2eSSTM_320    (2<<12)	/* 320MB/s 2eSST Xfer Rate */
 
-#define TSI148_LCSR_ITAT_2eSSTB        (1<<11)	/* 2eSST Bcast Xfer Protocol */
-#define TSI148_LCSR_ITAT_2eSST         (1<<10)	/* 2eSST Xfer Protocol */
-#define TSI148_LCSR_ITAT_2eVME         (1<<9)	/* 2eVME Xfer Protocol */
-#define TSI148_LCSR_ITAT_MBLT          (1<<8)	/* MBLT Xfer Protocol */
-#define TSI148_LCSR_ITAT_BLT           (1<<7)	/* BLT Xfer Protocol */
+#define TSI148_LCSR_ITAT_2eSSTB        BIT(11)	/* 2eSST Bcast Xfer Protocol */
+#define TSI148_LCSR_ITAT_2eSST         BIT(10)	/* 2eSST Xfer Protocol */
+#define TSI148_LCSR_ITAT_2eVME         BIT(9)	/* 2eVME Xfer Protocol */
+#define TSI148_LCSR_ITAT_MBLT          BIT(8)	/* MBLT Xfer Protocol */
+#define TSI148_LCSR_ITAT_BLT           BIT(7)	/* BLT Xfer Protocol */
 
 #define TSI148_LCSR_ITAT_AS_M          (7<<4)	/* Address Space Mask */
 #define TSI148_LCSR_ITAT_AS_A16        (0<<4)	/* A16 Address Space */
-#define TSI148_LCSR_ITAT_AS_A24        (1<<4)	/* A24 Address Space */
+#define TSI148_LCSR_ITAT_AS_A24        BIT(4)	/* A24 Address Space */
 #define TSI148_LCSR_ITAT_AS_A32        (2<<4)	/* A32 Address Space */
 #define TSI148_LCSR_ITAT_AS_A64        (4<<4)	/* A64 Address Space */
 
-#define TSI148_LCSR_ITAT_SUPR          (1<<3)	/* Supervisor Access */
-#define TSI148_LCSR_ITAT_NPRIV         (1<<2)	/* Non-Priv (User) Access */
-#define TSI148_LCSR_ITAT_PGM           (1<<1)	/* Program Access */
-#define TSI148_LCSR_ITAT_DATA          (1<<0)	/* Data Access */
+#define TSI148_LCSR_ITAT_SUPR          BIT(3)	/* Supervisor Access */
+#define TSI148_LCSR_ITAT_NPRIV         BIT(2)	/* Non-Priv (User) Access */
+#define TSI148_LCSR_ITAT_PGM           BIT(1)	/* Program Access */
+#define TSI148_LCSR_ITAT_DATA          BIT(0)	/* Data Access */
 
 /*
  *  GCSR Base Address Lower Address  CRG +$404
@@ -890,18 +890,18 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 /*
  *  GCSR Attribute Register CRG + $408
  */
-#define TSI148_LCSR_GCSRAT_EN          (1<<7)	/* Enable access to GCSR */
+#define TSI148_LCSR_GCSRAT_EN          BIT(7)	/* Enable access to GCSR */
 
 #define TSI148_LCSR_GCSRAT_AS_M        (7<<4)	/* Address Space Mask */
 #define TSI148_LCSR_GCSRAT_AS_A16       (0<<4)	/* Address Space 16 */
-#define TSI148_LCSR_GCSRAT_AS_A24       (1<<4)	/* Address Space 24 */
+#define TSI148_LCSR_GCSRAT_AS_A24       BIT(4)	/* Address Space 24 */
 #define TSI148_LCSR_GCSRAT_AS_A32       (2<<4)	/* Address Space 32 */
 #define TSI148_LCSR_GCSRAT_AS_A64       (4<<4)	/* Address Space 64 */
 
-#define TSI148_LCSR_GCSRAT_SUPR        (1<<3)	/* Sup set -GCSR decoder */
-#define TSI148_LCSR_GCSRAT_NPRIV       (1<<2)	/* Non-Privliged set - CGSR */
-#define TSI148_LCSR_GCSRAT_PGM         (1<<1)	/* Program set - GCSR decoder */
-#define TSI148_LCSR_GCSRAT_DATA        (1<<0)	/* DATA set GCSR decoder */
+#define TSI148_LCSR_GCSRAT_SUPR        BIT(3)	/* Sup set -GCSR decoder */
+#define TSI148_LCSR_GCSRAT_NPRIV       BIT(2)	/* Non-Privliged set - CGSR */
+#define TSI148_LCSR_GCSRAT_PGM         BIT(1)	/* Program set - GCSR decoder */
+#define TSI148_LCSR_GCSRAT_DATA        BIT(0)	/* DATA set GCSR decoder */
 
 /*
  *  CRG Base Address Lower Address  CRG + $410
@@ -911,18 +911,18 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 /*
  *  CRG Attribute Register  CRG + $414
  */
-#define TSI148_LCSR_CRGAT_EN           (1<<7)	/* Enable PRG Access */
+#define TSI148_LCSR_CRGAT_EN           BIT(7)	/* Enable PRG Access */
 
 #define TSI148_LCSR_CRGAT_AS_M         (7<<4)	/* Address Space */
 #define TSI148_LCSR_CRGAT_AS_A16       (0<<4)	/* Address Space 16 */
-#define TSI148_LCSR_CRGAT_AS_A24       (1<<4)	/* Address Space 24 */
+#define TSI148_LCSR_CRGAT_AS_A24       BIT(4)	/* Address Space 24 */
 #define TSI148_LCSR_CRGAT_AS_A32       (2<<4)	/* Address Space 32 */
 #define TSI148_LCSR_CRGAT_AS_A64       (4<<4)	/* Address Space 64 */
 
-#define TSI148_LCSR_CRGAT_SUPR         (1<<3)	/* Supervisor Access */
-#define TSI148_LCSR_CRGAT_NPRIV        (1<<2)	/* Non-Privliged(User) Access */
-#define TSI148_LCSR_CRGAT_PGM          (1<<1)	/* Program Access */
-#define TSI148_LCSR_CRGAT_DATA         (1<<0)	/* Data Access */
+#define TSI148_LCSR_CRGAT_SUPR         BIT(3)	/* Supervisor Access */
+#define TSI148_LCSR_CRGAT_NPRIV        BIT(2)	/* Non-Privliged(User) Access */
+#define TSI148_LCSR_CRGAT_PGM          BIT(1)	/* Program Access */
+#define TSI148_LCSR_CRGAT_DATA         BIT(0)	/* Data Access */
 
 /*
  *  CR/CSR Offset Lower Register  CRG + $41C
@@ -932,7 +932,7 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 /*
  *  CR/CSR Attribute register  CRG + $420
  */
-#define TSI148_LCSR_CRAT_EN            (1<<7)	/* Enable access to CR/CSR */
+#define TSI148_LCSR_CRAT_EN            BIT(7)	/* Enable access to CR/CSR */
 
 /*
  *  Location Monitor base address lower register  CRG + $428
@@ -942,18 +942,18 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
 /*
  *  Location Monitor Attribute Register  CRG + $42C
  */
-#define TSI148_LCSR_LMAT_EN            (1<<7)	/* Enable Location Monitor */
+#define TSI148_LCSR_LMAT_EN            BIT(7)	/* Enable Location Monitor */
 
 #define TSI148_LCSR_LMAT_AS_M          (7<<4)	/* Address Space MASK  */
 #define TSI148_LCSR_LMAT_AS_A16        (0<<4)	/* A16 */
-#define TSI148_LCSR_LMAT_AS_A24        (1<<4)	/* A24 */
+#define TSI148_LCSR_LMAT_AS_A24        BIT(4)	/* A24 */
 #define TSI148_LCSR_LMAT_AS_A32        (2<<4)	/* A32 */
 #define TSI148_LCSR_LMAT_AS_A64        (4<<4)	/* A64 */
 
-#define TSI148_LCSR_LMAT_SUPR          (1<<3)	/* Supervisor Access */
-#define TSI148_LCSR_LMAT_NPRIV         (1<<2)	/* Non-Priv (User) Access */
-#define TSI148_LCSR_LMAT_PGM           (1<<1)	/* Program Access */
-#define TSI148_LCSR_LMAT_DATA          (1<<0)	/* Data Access  */
+#define TSI148_LCSR_LMAT_SUPR          BIT(3)	/* Supervisor Access */
+#define TSI148_LCSR_LMAT_NPRIV         BIT(2)	/* Non-Priv (User) Access */
+#define TSI148_LCSR_LMAT_PGM           BIT(1)	/* Program Access */
+#define TSI148_LCSR_LMAT_DATA          BIT(0)	/* Data Access  */
 
 /*
  *  Broadcast Pulse Generator Timer Register  CRG + $438
@@ -969,34 +969,34 @@ static const int TSI148_GCSR_MBOX[4] = { TSI148_GCSR_MBOX0,
  *  VMEbus Interrupt Control Register           CRG + $43C
  */
 #define TSI148_LCSR_VICR_CNTS_M        (3<<22)	/* Cntr Source MASK */
-#define TSI148_LCSR_VICR_CNTS_DIS      (1<<22)	/* Cntr Disable */
+#define TSI148_LCSR_VICR_CNTS_DIS      BIT(22)	/* Cntr Disable */
 #define TSI148_LCSR_VICR_CNTS_IRQ1     (2<<22)	/* IRQ1 to Cntr */
 #define TSI148_LCSR_VICR_CNTS_IRQ2     (3<<22)	/* IRQ2 to Cntr */
 
 #define TSI148_LCSR_VICR_EDGIS_M       (3<<20)	/* Edge interrupt MASK */
-#define TSI148_LCSR_VICR_EDGIS_DIS     (1<<20)	/* Edge interrupt Disable */
+#define TSI148_LCSR_VICR_EDGIS_DIS     BIT(20)	/* Edge interrupt Disable */
 #define TSI148_LCSR_VICR_EDGIS_IRQ1    (2<<20)	/* IRQ1 to Edge */
 #define TSI148_LCSR_VICR_EDGIS_IRQ2    (3<<20)	/* IRQ2 to Edge */
 
 #define TSI148_LCSR_VICR_IRQIF_M       (3<<18)	/* IRQ1* Function MASK */
-#define TSI148_LCSR_VICR_IRQIF_NORM    (1<<18)	/* Normal */
+#define TSI148_LCSR_VICR_IRQIF_NORM    BIT(18)	/* Normal */
 #define TSI148_LCSR_VICR_IRQIF_PULSE   (2<<18)	/* Pulse Generator */
 #define TSI148_LCSR_VICR_IRQIF_PROG    (3<<18)	/* Programmable Clock */
 #define TSI148_LCSR_VICR_IRQIF_1U      (4<<18)	/* 1us Clock */
 
 #define TSI148_LCSR_VICR_IRQ2F_M       (3<<16)	/* IRQ2* Function MASK */
-#define TSI148_LCSR_VICR_IRQ2F_NORM    (1<<16)	/* Normal */
+#define TSI148_LCSR_VICR_IRQ2F_NORM    BIT(16)	/* Normal */
 #define TSI148_LCSR_VICR_IRQ2F_PULSE   (2<<16)	/* Pulse Generator */
 #define TSI148_LCSR_VICR_IRQ2F_PROG    (3<<16)	/* Programmable Clock */
 #define TSI148_LCSR_VICR_IRQ2F_1U      (4<<16)	/* 1us Clock */
 
-#define TSI148_LCSR_VICR_BIP           (1<<15)	/* Broadcast Interrupt Pulse */
+#define TSI148_LCSR_VICR_BIP           BIT(15)	/* Broadcast Interrupt Pulse */
 
-#define TSI148_LCSR_VICR_IRQC          (1<<12)	/* VMEbus IRQ Clear */
-#define TSI148_LCSR_VICR_IRQS          (1<<11)	/* VMEbus IRQ Status */
+#define TSI148_LCSR_VICR_IRQC          BIT(12)	/* VMEbus IRQ Clear */
+#define TSI148_LCSR_VICR_IRQS          BIT(11)	/* VMEbus IRQ Status */
 
 #define TSI148_LCSR_VICR_IRQL_M        (7<<8)	/* VMEbus SW IRQ Level Mask */
-#define TSI148_LCSR_VICR_IRQL_1        (1<<8)	/* VMEbus SW IRQ Level 1 */
+#define TSI148_LCSR_VICR_IRQL_1        BIT(8)	/* VMEbus SW IRQ Level 1 */
 #define TSI148_LCSR_VICR_IRQL_2        (2<<8)	/* VMEbus SW IRQ Level 2 */
 #define TSI148_LCSR_VICR_IRQL_3        (3<<8)	/* VMEbus SW IRQ Level 3 */
 #define TSI148_LCSR_VICR_IRQL_4        (4<<8)	/* VMEbus SW IRQ Level 4 */
@@ -1014,29 +1014,29 @@ static const int TSI148_LCSR_VICR_IRQL[8] = { 0, TSI148_LCSR_VICR_IRQL_1,
 /*
  *  Interrupt Enable Register   CRG + $440
  */
-#define TSI148_LCSR_INTEN_DMA1EN       (1<<25)	/* DMAC 1 */
-#define TSI148_LCSR_INTEN_DMA0EN       (1<<24)	/* DMAC 0 */
-#define TSI148_LCSR_INTEN_LM3EN        (1<<23)	/* Location Monitor 3 */
-#define TSI148_LCSR_INTEN_LM2EN        (1<<22)	/* Location Monitor 2 */
-#define TSI148_LCSR_INTEN_LM1EN        (1<<21)	/* Location Monitor 1 */
-#define TSI148_LCSR_INTEN_LM0EN        (1<<20)	/* Location Monitor 0 */
-#define TSI148_LCSR_INTEN_MB3EN        (1<<19)	/* Mail Box 3 */
-#define TSI148_LCSR_INTEN_MB2EN        (1<<18)	/* Mail Box 2 */
-#define TSI148_LCSR_INTEN_MB1EN        (1<<17)	/* Mail Box 1 */
-#define TSI148_LCSR_INTEN_MB0EN        (1<<16)	/* Mail Box 0 */
-#define TSI148_LCSR_INTEN_PERREN       (1<<13)	/* PCI/X Error */
-#define TSI148_LCSR_INTEN_VERREN       (1<<12)	/* VMEbus Error */
-#define TSI148_LCSR_INTEN_VIEEN        (1<<11)	/* VMEbus IRQ Edge */
-#define TSI148_LCSR_INTEN_IACKEN       (1<<10)	/* IACK */
-#define TSI148_LCSR_INTEN_SYSFLEN      (1<<9)	/* System Fail */
-#define TSI148_LCSR_INTEN_ACFLEN       (1<<8)	/* AC Fail */
-#define TSI148_LCSR_INTEN_IRQ7EN       (1<<7)	/* IRQ7 */
-#define TSI148_LCSR_INTEN_IRQ6EN       (1<<6)	/* IRQ6 */
-#define TSI148_LCSR_INTEN_IRQ5EN       (1<<5)	/* IRQ5 */
-#define TSI148_LCSR_INTEN_IRQ4EN       (1<<4)	/* IRQ4 */
-#define TSI148_LCSR_INTEN_IRQ3EN       (1<<3)	/* IRQ3 */
-#define TSI148_LCSR_INTEN_IRQ2EN       (1<<2)	/* IRQ2 */
-#define TSI148_LCSR_INTEN_IRQ1EN       (1<<1)	/* IRQ1 */
+#define TSI148_LCSR_INTEN_DMA1EN       BIT(25)	/* DMAC 1 */
+#define TSI148_LCSR_INTEN_DMA0EN       BIT(24)	/* DMAC 0 */
+#define TSI148_LCSR_INTEN_LM3EN        BIT(23)	/* Location Monitor 3 */
+#define TSI148_LCSR_INTEN_LM2EN        BIT(22)	/* Location Monitor 2 */
+#define TSI148_LCSR_INTEN_LM1EN        BIT(21)	/* Location Monitor 1 */
+#define TSI148_LCSR_INTEN_LM0EN        BIT(20)	/* Location Monitor 0 */
+#define TSI148_LCSR_INTEN_MB3EN        BIT(19)	/* Mail Box 3 */
+#define TSI148_LCSR_INTEN_MB2EN        BIT(18)	/* Mail Box 2 */
+#define TSI148_LCSR_INTEN_MB1EN        BIT(17)	/* Mail Box 1 */
+#define TSI148_LCSR_INTEN_MB0EN        BIT(16)	/* Mail Box 0 */
+#define TSI148_LCSR_INTEN_PERREN       BIT(13)	/* PCI/X Error */
+#define TSI148_LCSR_INTEN_VERREN       BIT(12)	/* VMEbus Error */
+#define TSI148_LCSR_INTEN_VIEEN        BIT(11)	/* VMEbus IRQ Edge */
+#define TSI148_LCSR_INTEN_IACKEN       BIT(10)	/* IACK */
+#define TSI148_LCSR_INTEN_SYSFLEN      BIT(9)	/* System Fail */
+#define TSI148_LCSR_INTEN_ACFLEN       BIT(8)	/* AC Fail */
+#define TSI148_LCSR_INTEN_IRQ7EN       BIT(7)	/* IRQ7 */
+#define TSI148_LCSR_INTEN_IRQ6EN       BIT(6)	/* IRQ6 */
+#define TSI148_LCSR_INTEN_IRQ5EN       BIT(5)	/* IRQ5 */
+#define TSI148_LCSR_INTEN_IRQ4EN       BIT(4)	/* IRQ4 */
+#define TSI148_LCSR_INTEN_IRQ3EN       BIT(3)	/* IRQ3 */
+#define TSI148_LCSR_INTEN_IRQ2EN       BIT(2)	/* IRQ2 */
+#define TSI148_LCSR_INTEN_IRQ1EN       BIT(1)	/* IRQ1 */
 
 static const int TSI148_LCSR_INTEN_LMEN[4] = { TSI148_LCSR_INTEN_LM0EN,
 					TSI148_LCSR_INTEN_LM1EN,
@@ -1054,29 +1054,29 @@ static const int TSI148_LCSR_INTEN_IRQEN[7] = { TSI148_LCSR_INTEN_IRQ1EN,
 /*
  *  Interrupt Enable Out Register CRG + $444
  */
-#define TSI148_LCSR_INTEO_DMA1EO       (1<<25)	/* DMAC 1 */
-#define TSI148_LCSR_INTEO_DMA0EO       (1<<24)	/* DMAC 0 */
-#define TSI148_LCSR_INTEO_LM3EO        (1<<23)	/* Loc Monitor 3 */
-#define TSI148_LCSR_INTEO_LM2EO        (1<<22)	/* Loc Monitor 2 */
-#define TSI148_LCSR_INTEO_LM1EO        (1<<21)	/* Loc Monitor 1 */
-#define TSI148_LCSR_INTEO_LM0EO        (1<<20)	/* Location Monitor 0 */
-#define TSI148_LCSR_INTEO_MB3EO        (1<<19)	/* Mail Box 3 */
-#define TSI148_LCSR_INTEO_MB2EO        (1<<18)	/* Mail Box 2 */
-#define TSI148_LCSR_INTEO_MB1EO        (1<<17)	/* Mail Box 1 */
-#define TSI148_LCSR_INTEO_MB0EO        (1<<16)	/* Mail Box 0 */
-#define TSI148_LCSR_INTEO_PERREO       (1<<13)	/* PCI/X Error */
-#define TSI148_LCSR_INTEO_VERREO       (1<<12)	/* VMEbus Error */
-#define TSI148_LCSR_INTEO_VIEEO        (1<<11)	/* VMEbus IRQ Edge */
-#define TSI148_LCSR_INTEO_IACKEO       (1<<10)	/* IACK */
-#define TSI148_LCSR_INTEO_SYSFLEO      (1<<9)	/* System Fail */
-#define TSI148_LCSR_INTEO_ACFLEO       (1<<8)	/* AC Fail */
-#define TSI148_LCSR_INTEO_IRQ7EO       (1<<7)	/* IRQ7 */
-#define TSI148_LCSR_INTEO_IRQ6EO       (1<<6)	/* IRQ6 */
-#define TSI148_LCSR_INTEO_IRQ5EO       (1<<5)	/* IRQ5 */
-#define TSI148_LCSR_INTEO_IRQ4EO       (1<<4)	/* IRQ4 */
-#define TSI148_LCSR_INTEO_IRQ3EO       (1<<3)	/* IRQ3 */
-#define TSI148_LCSR_INTEO_IRQ2EO       (1<<2)	/* IRQ2 */
-#define TSI148_LCSR_INTEO_IRQ1EO       (1<<1)	/* IRQ1 */
+#define TSI148_LCSR_INTEO_DMA1EO       BIT(25)	/* DMAC 1 */
+#define TSI148_LCSR_INTEO_DMA0EO       BIT(24)	/* DMAC 0 */
+#define TSI148_LCSR_INTEO_LM3EO        BIT(23)	/* Loc Monitor 3 */
+#define TSI148_LCSR_INTEO_LM2EO        BIT(22)	/* Loc Monitor 2 */
+#define TSI148_LCSR_INTEO_LM1EO        BIT(21)	/* Loc Monitor 1 */
+#define TSI148_LCSR_INTEO_LM0EO        BIT(20)	/* Location Monitor 0 */
+#define TSI148_LCSR_INTEO_MB3EO        BIT(19)	/* Mail Box 3 */
+#define TSI148_LCSR_INTEO_MB2EO        BIT(18)	/* Mail Box 2 */
+#define TSI148_LCSR_INTEO_MB1EO        BIT(17)	/* Mail Box 1 */
+#define TSI148_LCSR_INTEO_MB0EO        BIT(16)	/* Mail Box 0 */
+#define TSI148_LCSR_INTEO_PERREO       BIT(13)	/* PCI/X Error */
+#define TSI148_LCSR_INTEO_VERREO       BIT(12)	/* VMEbus Error */
+#define TSI148_LCSR_INTEO_VIEEO        BIT(11)	/* VMEbus IRQ Edge */
+#define TSI148_LCSR_INTEO_IACKEO       BIT(10)	/* IACK */
+#define TSI148_LCSR_INTEO_SYSFLEO      BIT(9)	/* System Fail */
+#define TSI148_LCSR_INTEO_ACFLEO       BIT(8)	/* AC Fail */
+#define TSI148_LCSR_INTEO_IRQ7EO       BIT(7)	/* IRQ7 */
+#define TSI148_LCSR_INTEO_IRQ6EO       BIT(6)	/* IRQ6 */
+#define TSI148_LCSR_INTEO_IRQ5EO       BIT(5)	/* IRQ5 */
+#define TSI148_LCSR_INTEO_IRQ4EO       BIT(4)	/* IRQ4 */
+#define TSI148_LCSR_INTEO_IRQ3EO       BIT(3)	/* IRQ3 */
+#define TSI148_LCSR_INTEO_IRQ2EO       BIT(2)	/* IRQ2 */
+#define TSI148_LCSR_INTEO_IRQ1EO       BIT(1)	/* IRQ1 */
 
 static const int TSI148_LCSR_INTEO_LMEO[4] = { TSI148_LCSR_INTEO_LM0EO,
 					TSI148_LCSR_INTEO_LM1EO,
@@ -1094,29 +1094,29 @@ static const int TSI148_LCSR_INTEO_IRQEO[7] = { TSI148_LCSR_INTEO_IRQ1EO,
 /*
  *  Interrupt Status Register CRG + $448
  */
-#define TSI148_LCSR_INTS_DMA1S         (1<<25)	/* DMA 1 */
-#define TSI148_LCSR_INTS_DMA0S         (1<<24)	/* DMA 0 */
-#define TSI148_LCSR_INTS_LM3S          (1<<23)	/* Location Monitor 3 */
-#define TSI148_LCSR_INTS_LM2S          (1<<22)	/* Location Monitor 2 */
-#define TSI148_LCSR_INTS_LM1S          (1<<21)	/* Location Monitor 1 */
-#define TSI148_LCSR_INTS_LM0S          (1<<20)	/* Location Monitor 0 */
-#define TSI148_LCSR_INTS_MB3S          (1<<19)	/* Mail Box 3 */
-#define TSI148_LCSR_INTS_MB2S          (1<<18)	/* Mail Box 2 */
-#define TSI148_LCSR_INTS_MB1S          (1<<17)	/* Mail Box 1 */
-#define TSI148_LCSR_INTS_MB0S          (1<<16)	/* Mail Box 0 */
-#define TSI148_LCSR_INTS_PERRS         (1<<13)	/* PCI/X Error */
-#define TSI148_LCSR_INTS_VERRS         (1<<12)	/* VMEbus Error */
-#define TSI148_LCSR_INTS_VIES          (1<<11)	/* VMEbus IRQ Edge */
-#define TSI148_LCSR_INTS_IACKS         (1<<10)	/* IACK */
-#define TSI148_LCSR_INTS_SYSFLS        (1<<9)	/* System Fail */
-#define TSI148_LCSR_INTS_ACFLS         (1<<8)	/* AC Fail */
-#define TSI148_LCSR_INTS_IRQ7S         (1<<7)	/* IRQ7 */
-#define TSI148_LCSR_INTS_IRQ6S         (1<<6)	/* IRQ6 */
-#define TSI148_LCSR_INTS_IRQ5S         (1<<5)	/* IRQ5 */
-#define TSI148_LCSR_INTS_IRQ4S         (1<<4)	/* IRQ4 */
-#define TSI148_LCSR_INTS_IRQ3S         (1<<3)	/* IRQ3 */
-#define TSI148_LCSR_INTS_IRQ2S         (1<<2)	/* IRQ2 */
-#define TSI148_LCSR_INTS_IRQ1S         (1<<1)	/* IRQ1 */
+#define TSI148_LCSR_INTS_DMA1S         BIT(25)	/* DMA 1 */
+#define TSI148_LCSR_INTS_DMA0S         BIT(24)	/* DMA 0 */
+#define TSI148_LCSR_INTS_LM3S          BIT(23)	/* Location Monitor 3 */
+#define TSI148_LCSR_INTS_LM2S          BIT(22)	/* Location Monitor 2 */
+#define TSI148_LCSR_INTS_LM1S          BIT(21)	/* Location Monitor 1 */
+#define TSI148_LCSR_INTS_LM0S          BIT(20)	/* Location Monitor 0 */
+#define TSI148_LCSR_INTS_MB3S          BIT(19)	/* Mail Box 3 */
+#define TSI148_LCSR_INTS_MB2S          BIT(18)	/* Mail Box 2 */
+#define TSI148_LCSR_INTS_MB1S          BIT(17)	/* Mail Box 1 */
+#define TSI148_LCSR_INTS_MB0S          BIT(16)	/* Mail Box 0 */
+#define TSI148_LCSR_INTS_PERRS         BIT(13)	/* PCI/X Error */
+#define TSI148_LCSR_INTS_VERRS         BIT(12)	/* VMEbus Error */
+#define TSI148_LCSR_INTS_VIES          BIT(11)	/* VMEbus IRQ Edge */
+#define TSI148_LCSR_INTS_IACKS         BIT(10)	/* IACK */
+#define TSI148_LCSR_INTS_SYSFLS        BIT(9)	/* System Fail */
+#define TSI148_LCSR_INTS_ACFLS         BIT(8)	/* AC Fail */
+#define TSI148_LCSR_INTS_IRQ7S         BIT(7)	/* IRQ7 */
+#define TSI148_LCSR_INTS_IRQ6S         BIT(6)	/* IRQ6 */
+#define TSI148_LCSR_INTS_IRQ5S         BIT(5)	/* IRQ5 */
+#define TSI148_LCSR_INTS_IRQ4S         BIT(4)	/* IRQ4 */
+#define TSI148_LCSR_INTS_IRQ3S         BIT(3)	/* IRQ3 */
+#define TSI148_LCSR_INTS_IRQ2S         BIT(2)	/* IRQ2 */
+#define TSI148_LCSR_INTS_IRQ1S         BIT(1)	/* IRQ1 */
 
 static const int TSI148_LCSR_INTS_LMS[4] = { TSI148_LCSR_INTS_LM0S,
 					TSI148_LCSR_INTS_LM1S,
@@ -1131,22 +1131,22 @@ static const int TSI148_LCSR_INTS_MBS[4] = { TSI148_LCSR_INTS_MB0S,
 /*
  *  Interrupt Clear Register CRG + $44C
  */
-#define TSI148_LCSR_INTC_DMA1C         (1<<25)	/* DMA 1 */
-#define TSI148_LCSR_INTC_DMA0C         (1<<24)	/* DMA 0 */
-#define TSI148_LCSR_INTC_LM3C          (1<<23)	/* Location Monitor 3 */
-#define TSI148_LCSR_INTC_LM2C          (1<<22)	/* Location Monitor 2 */
-#define TSI148_LCSR_INTC_LM1C          (1<<21)	/* Location Monitor 1 */
-#define TSI148_LCSR_INTC_LM0C          (1<<20)	/* Location Monitor 0 */
-#define TSI148_LCSR_INTC_MB3C          (1<<19)	/* Mail Box 3 */
-#define TSI148_LCSR_INTC_MB2C          (1<<18)	/* Mail Box 2 */
-#define TSI148_LCSR_INTC_MB1C          (1<<17)	/* Mail Box 1 */
-#define TSI148_LCSR_INTC_MB0C          (1<<16)	/* Mail Box 0 */
-#define TSI148_LCSR_INTC_PERRC         (1<<13)	/* VMEbus Error */
-#define TSI148_LCSR_INTC_VERRC         (1<<12)	/* VMEbus Access Time-out */
-#define TSI148_LCSR_INTC_VIEC          (1<<11)	/* VMEbus IRQ Edge */
-#define TSI148_LCSR_INTC_IACKC         (1<<10)	/* IACK */
-#define TSI148_LCSR_INTC_SYSFLC        (1<<9)	/* System Fail */
-#define TSI148_LCSR_INTC_ACFLC         (1<<8)	/* AC Fail */
+#define TSI148_LCSR_INTC_DMA1C         BIT(25)	/* DMA 1 */
+#define TSI148_LCSR_INTC_DMA0C         BIT(24)	/* DMA 0 */
+#define TSI148_LCSR_INTC_LM3C          BIT(23)	/* Location Monitor 3 */
+#define TSI148_LCSR_INTC_LM2C          BIT(22)	/* Location Monitor 2 */
+#define TSI148_LCSR_INTC_LM1C          BIT(21)	/* Location Monitor 1 */
+#define TSI148_LCSR_INTC_LM0C          BIT(20)	/* Location Monitor 0 */
+#define TSI148_LCSR_INTC_MB3C          BIT(19)	/* Mail Box 3 */
+#define TSI148_LCSR_INTC_MB2C          BIT(18)	/* Mail Box 2 */
+#define TSI148_LCSR_INTC_MB1C          BIT(17)	/* Mail Box 1 */
+#define TSI148_LCSR_INTC_MB0C          BIT(16)	/* Mail Box 0 */
+#define TSI148_LCSR_INTC_PERRC         BIT(13)	/* VMEbus Error */
+#define TSI148_LCSR_INTC_VERRC         BIT(12)	/* VMEbus Access Time-out */
+#define TSI148_LCSR_INTC_VIEC          BIT(11)	/* VMEbus IRQ Edge */
+#define TSI148_LCSR_INTC_IACKC         BIT(10)	/* IACK */
+#define TSI148_LCSR_INTC_SYSFLC        BIT(9)	/* System Fail */
+#define TSI148_LCSR_INTC_ACFLC         BIT(8)	/* AC Fail */
 
 static const int TSI148_LCSR_INTC_LMC[4] = { TSI148_LCSR_INTC_LM0C,
 					TSI148_LCSR_INTC_LM1C,
@@ -1192,15 +1192,15 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
 /*
  *  DMA Control (0-1) Registers CRG + $500
  */
-#define TSI148_LCSR_DCTL_ABT           (1<<27)	/* Abort */
-#define TSI148_LCSR_DCTL_PAU           (1<<26)	/* Pause */
-#define TSI148_LCSR_DCTL_DGO           (1<<25)	/* DMA Go */
+#define TSI148_LCSR_DCTL_ABT           BIT(27)	/* Abort */
+#define TSI148_LCSR_DCTL_PAU           BIT(26)	/* Pause */
+#define TSI148_LCSR_DCTL_DGO           BIT(25)	/* DMA Go */
 
-#define TSI148_LCSR_DCTL_MOD           (1<<23)	/* Mode */
+#define TSI148_LCSR_DCTL_MOD           BIT(23)	/* Mode */
 
 #define TSI148_LCSR_DCTL_VBKS_M        (7<<12)	/* VMEbus block Size MASK */
 #define TSI148_LCSR_DCTL_VBKS_32       (0<<12)	/* VMEbus block Size 32 */
-#define TSI148_LCSR_DCTL_VBKS_64       (1<<12)	/* VMEbus block Size 64 */
+#define TSI148_LCSR_DCTL_VBKS_64       BIT(12)	/* VMEbus block Size 64 */
 #define TSI148_LCSR_DCTL_VBKS_128      (2<<12)	/* VMEbus block Size 128 */
 #define TSI148_LCSR_DCTL_VBKS_256      (3<<12)	/* VMEbus block Size 256 */
 #define TSI148_LCSR_DCTL_VBKS_512      (4<<12)	/* VMEbus block Size 512 */
@@ -1210,7 +1210,7 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
 
 #define TSI148_LCSR_DCTL_VBOT_M        (7<<8)	/* VMEbus back-off MASK */
 #define TSI148_LCSR_DCTL_VBOT_0        (0<<8)	/* VMEbus back-off  0us */
-#define TSI148_LCSR_DCTL_VBOT_1        (1<<8)	/* VMEbus back-off 1us */
+#define TSI148_LCSR_DCTL_VBOT_1        BIT(8)	/* VMEbus back-off 1us */
 #define TSI148_LCSR_DCTL_VBOT_2        (2<<8)	/* VMEbus back-off 2us */
 #define TSI148_LCSR_DCTL_VBOT_4        (3<<8)	/* VMEbus back-off 4us */
 #define TSI148_LCSR_DCTL_VBOT_8        (4<<8)	/* VMEbus back-off 8us */
@@ -1220,7 +1220,7 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
 
 #define TSI148_LCSR_DCTL_PBKS_M        (7<<4)	/* PCI block size MASK */
 #define TSI148_LCSR_DCTL_PBKS_32       (0<<4)	/* PCI block size 32 bytes */
-#define TSI148_LCSR_DCTL_PBKS_64       (1<<4)	/* PCI block size 64 bytes */
+#define TSI148_LCSR_DCTL_PBKS_64       BIT(4)	/* PCI block size 64 bytes */
 #define TSI148_LCSR_DCTL_PBKS_128      (2<<4)	/* PCI block size 128 bytes */
 #define TSI148_LCSR_DCTL_PBKS_256      (3<<4)	/* PCI block size 256 bytes */
 #define TSI148_LCSR_DCTL_PBKS_512      (4<<4)	/* PCI block size 512 bytes */
@@ -1230,7 +1230,7 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
 
 #define TSI148_LCSR_DCTL_PBOT_M        (7<<0)	/* PCI back off MASK */
 #define TSI148_LCSR_DCTL_PBOT_0        (0<<0)	/* PCI back off 0us */
-#define TSI148_LCSR_DCTL_PBOT_1        (1<<0)	/* PCI back off 1us */
+#define TSI148_LCSR_DCTL_PBOT_1        BIT(0)	/* PCI back off 1us */
 #define TSI148_LCSR_DCTL_PBOT_2        (2<<0)	/* PCI back off 2us */
 #define TSI148_LCSR_DCTL_PBOT_4        (3<<0)	/* PCI back off 3us */
 #define TSI148_LCSR_DCTL_PBOT_8        (4<<0)	/* PCI back off 4us */
@@ -1241,14 +1241,14 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
 /*
  *  DMA Status Registers (0-1)  CRG + $504
  */
-#define TSI148_LCSR_DSTA_SMA           (1<<31)	/* PCI Signalled Master Abt */
-#define TSI148_LCSR_DSTA_RTA           (1<<30)	/* PCI Received Target Abt */
-#define TSI148_LCSR_DSTA_MRC           (1<<29)	/* PCI Max Retry Count */
-#define TSI148_LCSR_DSTA_VBE           (1<<28)	/* VMEbus error */
-#define TSI148_LCSR_DSTA_ABT           (1<<27)	/* Abort */
-#define TSI148_LCSR_DSTA_PAU           (1<<26)	/* Pause */
-#define TSI148_LCSR_DSTA_DON           (1<<25)	/* Done */
-#define TSI148_LCSR_DSTA_BSY           (1<<24)	/* Busy */
+#define TSI148_LCSR_DSTA_SMA           BIT(31)	/* PCI Signalled Master Abt */
+#define TSI148_LCSR_DSTA_RTA           BIT(30)	/* PCI Received Target Abt */
+#define TSI148_LCSR_DSTA_MRC           BIT(29)	/* PCI Max Retry Count */
+#define TSI148_LCSR_DSTA_VBE           BIT(28)	/* VMEbus error */
+#define TSI148_LCSR_DSTA_ABT           BIT(27)	/* Abort */
+#define TSI148_LCSR_DSTA_PAU           BIT(26)	/* Pause */
+#define TSI148_LCSR_DSTA_DON           BIT(25)	/* Done */
+#define TSI148_LCSR_DSTA_BSY           BIT(24)	/* Busy */
 
 /*
  *  DMA Current Link Address Lower (0-1)
@@ -1260,20 +1260,20 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
  */
 #define TSI148_LCSR_DSAT_TYP_M         (3<<28)	/* Source Bus Type */
 #define TSI148_LCSR_DSAT_TYP_PCI       (0<<28)	/* PCI Bus */
-#define TSI148_LCSR_DSAT_TYP_VME       (1<<28)	/* VMEbus */
+#define TSI148_LCSR_DSAT_TYP_VME       BIT(28)	/* VMEbus */
 #define TSI148_LCSR_DSAT_TYP_PAT       (2<<28)	/* Data Pattern */
 
-#define TSI148_LCSR_DSAT_PSZ           (1<<25)	/* Pattern Size */
-#define TSI148_LCSR_DSAT_NIN           (1<<24)	/* No Increment */
+#define TSI148_LCSR_DSAT_PSZ           BIT(25)	/* Pattern Size */
+#define TSI148_LCSR_DSAT_NIN           BIT(24)	/* No Increment */
 
 #define TSI148_LCSR_DSAT_2eSSTM_M      (3<<11)	/* 2eSST Trans Rate Mask */
 #define TSI148_LCSR_DSAT_2eSSTM_160    (0<<11)	/* 160 MB/s */
-#define TSI148_LCSR_DSAT_2eSSTM_267    (1<<11)	/* 267 MB/s */
+#define TSI148_LCSR_DSAT_2eSSTM_267    BIT(11)	/* 267 MB/s */
 #define TSI148_LCSR_DSAT_2eSSTM_320    (2<<11)	/* 320 MB/s */
 
 #define TSI148_LCSR_DSAT_TM_M          (7<<8)	/* Bus Transfer Protocol Mask */
 #define TSI148_LCSR_DSAT_TM_SCT        (0<<8)	/* SCT */
-#define TSI148_LCSR_DSAT_TM_BLT        (1<<8)	/* BLT */
+#define TSI148_LCSR_DSAT_TM_BLT        BIT(8)	/* BLT */
 #define TSI148_LCSR_DSAT_TM_MBLT       (2<<8)	/* MBLT */
 #define TSI148_LCSR_DSAT_TM_2eVME      (3<<8)	/* 2eVME */
 #define TSI148_LCSR_DSAT_TM_2eSST      (4<<8)	/* 2eSST */
@@ -1281,14 +1281,14 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
 
 #define TSI148_LCSR_DSAT_DBW_M         (3<<6)	/* Max Data Width MASK */
 #define TSI148_LCSR_DSAT_DBW_16        (0<<6)	/* 16 Bits */
-#define TSI148_LCSR_DSAT_DBW_32        (1<<6)	/* 32 Bits */
+#define TSI148_LCSR_DSAT_DBW_32        BIT(6)	/* 32 Bits */
 
-#define TSI148_LCSR_DSAT_SUP           (1<<5)	/* Supervisory Mode */
-#define TSI148_LCSR_DSAT_PGM           (1<<4)	/* Program Mode */
+#define TSI148_LCSR_DSAT_SUP           BIT(5)	/* Supervisory Mode */
+#define TSI148_LCSR_DSAT_PGM           BIT(4)	/* Program Mode */
 
 #define TSI148_LCSR_DSAT_AMODE_M       (0xf<<0)	/* Address Space Mask */
 #define TSI148_LCSR_DSAT_AMODE_A16     (0<<0)	/* A16 */
-#define TSI148_LCSR_DSAT_AMODE_A24     (1<<0)	/* A24 */
+#define TSI148_LCSR_DSAT_AMODE_A24     BIT(0)	/* A24 */
 #define TSI148_LCSR_DSAT_AMODE_A32     (2<<0)	/* A32 */
 #define TSI148_LCSR_DSAT_AMODE_A64     (4<<0)	/* A64 */
 #define TSI148_LCSR_DSAT_AMODE_CRCSR   (5<<0)	/* CR/CSR */
@@ -1301,16 +1301,16 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
  *  DMA Destination Attribute Registers (0-1)
  */
 #define TSI148_LCSR_DDAT_TYP_PCI       (0<<28)	/* Destination PCI Bus  */
-#define TSI148_LCSR_DDAT_TYP_VME       (1<<28)	/* Destination VMEbus */
+#define TSI148_LCSR_DDAT_TYP_VME       BIT(28)	/* Destination VMEbus */
 
 #define TSI148_LCSR_DDAT_2eSSTM_M      (3<<11)	/* 2eSST Transfer Rate Mask */
 #define TSI148_LCSR_DDAT_2eSSTM_160    (0<<11)	/* 160 MB/s */
-#define TSI148_LCSR_DDAT_2eSSTM_267    (1<<11)	/* 267 MB/s */
+#define TSI148_LCSR_DDAT_2eSSTM_267    BIT(11)	/* 267 MB/s */
 #define TSI148_LCSR_DDAT_2eSSTM_320    (2<<11)	/* 320 MB/s */
 
 #define TSI148_LCSR_DDAT_TM_M          (7<<8)	/* Bus Transfer Protocol Mask */
 #define TSI148_LCSR_DDAT_TM_SCT        (0<<8)	/* SCT */
-#define TSI148_LCSR_DDAT_TM_BLT        (1<<8)	/* BLT */
+#define TSI148_LCSR_DDAT_TM_BLT        BIT(8)	/* BLT */
 #define TSI148_LCSR_DDAT_TM_MBLT       (2<<8)	/* MBLT */
 #define TSI148_LCSR_DDAT_TM_2eVME      (3<<8)	/* 2eVME */
 #define TSI148_LCSR_DDAT_TM_2eSST      (4<<8)	/* 2eSST */
@@ -1318,14 +1318,14 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
 
 #define TSI148_LCSR_DDAT_DBW_M         (3<<6)	/* Max Data Width MASK */
 #define TSI148_LCSR_DDAT_DBW_16        (0<<6)	/* 16 Bits */
-#define TSI148_LCSR_DDAT_DBW_32        (1<<6)	/* 32 Bits */
+#define TSI148_LCSR_DDAT_DBW_32        BIT(6)	/* 32 Bits */
 
-#define TSI148_LCSR_DDAT_SUP           (1<<5)	/* Supervisory/User Access */
-#define TSI148_LCSR_DDAT_PGM           (1<<4)	/* Program/Data Access */
+#define TSI148_LCSR_DDAT_SUP           BIT(5)	/* Supervisory/User Access */
+#define TSI148_LCSR_DDAT_PGM           BIT(4)	/* Program/Data Access */
 
 #define TSI148_LCSR_DDAT_AMODE_M       (0xf<<0)	/* Address Space Mask */
 #define TSI148_LCSR_DDAT_AMODE_A16      (0<<0)	/* A16 */
-#define TSI148_LCSR_DDAT_AMODE_A24      (1<<0)	/* A24 */
+#define TSI148_LCSR_DDAT_AMODE_A24      BIT(0)	/* A24 */
 #define TSI148_LCSR_DDAT_AMODE_A32      (2<<0)	/* A32 */
 #define TSI148_LCSR_DDAT_AMODE_A64      (4<<0)	/* A64 */
 #define TSI148_LCSR_DDAT_AMODE_CRCSR   (5<<0)	/* CRC/SR */
@@ -1338,7 +1338,7 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
  *  DMA Next Link Address Lower
  */
 #define TSI148_LCSR_DNLAL_DNLAL_M      (0x3FFFFFF<<6)	/* Address Mask */
-#define TSI148_LCSR_DNLAL_LLA          (1<<0)  /* Last Link Address Indicator */
+#define TSI148_LCSR_DNLAL_LLA          BIT(0)  /* Last Link Address Indicator */
 
 /*
  *  DMA 2eSST Broadcast Select
@@ -1352,22 +1352,22 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
 /*
  *  GCSR Control and Status Register  CRG + $604
  */
-#define TSI148_GCSR_GCTRL_LRST         (1<<15)	/* Local Reset */
-#define TSI148_GCSR_GCTRL_SFAILEN      (1<<14)	/* System Fail enable */
-#define TSI148_GCSR_GCTRL_BDFAILS      (1<<13)	/* Board Fail Status */
-#define TSI148_GCSR_GCTRL_SCON         (1<<12)	/* System Copntroller */
-#define TSI148_GCSR_GCTRL_MEN          (1<<11)	/* Module Enable (READY) */
-
-#define TSI148_GCSR_GCTRL_LMI3S        (1<<7)	/* Loc Monitor 3 Int Status */
-#define TSI148_GCSR_GCTRL_LMI2S        (1<<6)	/* Loc Monitor 2 Int Status */
-#define TSI148_GCSR_GCTRL_LMI1S        (1<<5)	/* Loc Monitor 1 Int Status */
-#define TSI148_GCSR_GCTRL_LMI0S        (1<<4)	/* Loc Monitor 0 Int Status */
-#define TSI148_GCSR_GCTRL_MBI3S        (1<<3)	/* Mail box 3 Int Status */
-#define TSI148_GCSR_GCTRL_MBI2S        (1<<2)	/* Mail box 2 Int Status */
-#define TSI148_GCSR_GCTRL_MBI1S        (1<<1)	/* Mail box 1 Int Status */
-#define TSI148_GCSR_GCTRL_MBI0S        (1<<0)	/* Mail box 0 Int Status */
-
-#define TSI148_GCSR_GAP                (1<<5)	/* Geographic Addr Parity */
+#define TSI148_GCSR_GCTRL_LRST         BIT(15)	/* Local Reset */
+#define TSI148_GCSR_GCTRL_SFAILEN      BIT(14)	/* System Fail enable */
+#define TSI148_GCSR_GCTRL_BDFAILS      BIT(13)	/* Board Fail Status */
+#define TSI148_GCSR_GCTRL_SCON         BIT(12)	/* System Copntroller */
+#define TSI148_GCSR_GCTRL_MEN          BIT(11)	/* Module Enable (READY) */
+
+#define TSI148_GCSR_GCTRL_LMI3S        BIT(7)	/* Loc Monitor 3 Int Status */
+#define TSI148_GCSR_GCTRL_LMI2S        BIT(6)	/* Loc Monitor 2 Int Status */
+#define TSI148_GCSR_GCTRL_LMI1S        BIT(5)	/* Loc Monitor 1 Int Status */
+#define TSI148_GCSR_GCTRL_LMI0S        BIT(4)	/* Loc Monitor 0 Int Status */
+#define TSI148_GCSR_GCTRL_MBI3S        BIT(3)	/* Mail box 3 Int Status */
+#define TSI148_GCSR_GCTRL_MBI2S        BIT(2)	/* Mail box 2 Int Status */
+#define TSI148_GCSR_GCTRL_MBI1S        BIT(1)	/* Mail box 1 Int Status */
+#define TSI148_GCSR_GCTRL_MBI0S        BIT(0)	/* Mail box 0 Int Status */
+
+#define TSI148_GCSR_GAP                BIT(5)	/* Geographic Addr Parity */
 #define TSI148_GCSR_GA_M               (0x1F<<0)  /* Geographic Address Mask */
 
 /*
@@ -1377,20 +1377,20 @@ static const int TSI148_LCSR_INTC_MBC[4] = { TSI148_LCSR_INTC_MB0C,
 /*
  *  CR/CSR Bit Clear Register CRG + $FF4
  */
-#define TSI148_CRCSR_CSRBCR_LRSTC      (1<<7)	/* Local Reset Clear */
-#define TSI148_CRCSR_CSRBCR_SFAILC     (1<<6)	/* System Fail Enable Clear */
-#define TSI148_CRCSR_CSRBCR_BDFAILS    (1<<5)	/* Board Fail Status */
-#define TSI148_CRCSR_CSRBCR_MENC       (1<<4)	/* Module Enable Clear */
-#define TSI148_CRCSR_CSRBCR_BERRSC     (1<<3)	/* Bus Error Status Clear */
+#define TSI148_CRCSR_CSRBCR_LRSTC      BIT(7)	/* Local Reset Clear */
+#define TSI148_CRCSR_CSRBCR_SFAILC     BIT(6)	/* System Fail Enable Clear */
+#define TSI148_CRCSR_CSRBCR_BDFAILS    BIT(5)	/* Board Fail Status */
+#define TSI148_CRCSR_CSRBCR_MENC       BIT(4)	/* Module Enable Clear */
+#define TSI148_CRCSR_CSRBCR_BERRSC     BIT(3)	/* Bus Error Status Clear */
 
 /*
  *  CR/CSR Bit Set Register CRG+$FF8
  */
-#define TSI148_CRCSR_CSRBSR_LISTS      (1<<7)	/* Local Reset Clear */
-#define TSI148_CRCSR_CSRBSR_SFAILS     (1<<6)	/* System Fail Enable Clear */
-#define TSI148_CRCSR_CSRBSR_BDFAILS    (1<<5)	/* Board Fail Status */
-#define TSI148_CRCSR_CSRBSR_MENS       (1<<4)	/* Module Enable Clear */
-#define TSI148_CRCSR_CSRBSR_BERRS      (1<<3)	/* Bus Error Status Clear */
+#define TSI148_CRCSR_CSRBSR_LISTS      BIT(7)	/* Local Reset Clear */
+#define TSI148_CRCSR_CSRBSR_SFAILS     BIT(6)	/* System Fail Enable Clear */
+#define TSI148_CRCSR_CSRBSR_BDFAILS    BIT(5)	/* Board Fail Status */
+#define TSI148_CRCSR_CSRBSR_MENS       BIT(4)	/* Module Enable Clear */
+#define TSI148_CRCSR_CSRBSR_BERRS      BIT(3)	/* Bus Error Status Clear */
 
 /*
  *  CR/CSR Base Address Register CRG + FFC
diff --git a/drivers/staging/vme_user/vme_user.c b/drivers/staging/vme_user/vme_user.c
index 4e533c0bfe6d..b9367b575d00 100644
--- a/drivers/staging/vme_user/vme_user.c
+++ b/drivers/staging/vme_user/vme_user.c
@@ -614,7 +614,7 @@ static int vme_user_probe(struct vme_dev *vdev)
 	}
 
 	/* Create sysfs entries - on udev systems this creates the dev files */
-	vme_user_sysfs_class = class_create(THIS_MODULE, driver_name);
+	vme_user_sysfs_class = class_create(driver_name);
 	if (IS_ERR(vme_user_sysfs_class)) {
 		dev_err(&vdev->dev, "Error creating vme_user class.\n");
 		err = PTR_ERR(vme_user_sysfs_class);
diff --git a/drivers/staging/vt6655/baseband.c b/drivers/staging/vt6655/baseband.c
index 6ce41983dcf4..0e135af8316b 100644
--- a/drivers/staging/vt6655/baseband.c
+++ b/drivers/staging/vt6655/baseband.c
@@ -2172,50 +2172,6 @@ bb_software_reset(struct vnt_private *priv)
 }
 
 /*
- * Description: Baseband Power Save Mode ON
- *
- * Parameters:
- *  In:
- *      iobase      - I/O base address
- *  Out:
- *      none
- *
- * Return Value: none
- *
- */
-void
-bb_power_save_mode_on(struct vnt_private *priv)
-{
-	unsigned char by_org_data;
-
-	bb_read_embedded(priv, 0x0D, &by_org_data);
-	by_org_data |= BIT(0);
-	bb_write_embedded(priv, 0x0D, by_org_data);
-}
-
-/*
- * Description: Baseband Power Save Mode OFF
- *
- * Parameters:
- *  In:
- *      iobase      - I/O base address
- *  Out:
- *      none
- *
- * Return Value: none
- *
- */
-void
-bb_power_save_mode_off(struct vnt_private *priv)
-{
-	unsigned char by_org_data;
-
-	bb_read_embedded(priv, 0x0D, &by_org_data);
-	by_org_data &= ~(BIT(0));
-	bb_write_embedded(priv, 0x0D, by_org_data);
-}
-
-/*
  * Description: Set Tx Antenna mode
  *
  * Parameters:
diff --git a/drivers/staging/vt6655/baseband.h b/drivers/staging/vt6655/baseband.h
index 15b2802ed727..e4a02c240a1c 100644
--- a/drivers/staging/vt6655/baseband.h
+++ b/drivers/staging/vt6655/baseband.h
@@ -63,8 +63,6 @@ void bb_set_vga_gain_offset(struct vnt_private *priv, unsigned char by_data);
 /* VT3253 Baseband */
 bool bb_vt3253_init(struct vnt_private *priv);
 void bb_software_reset(struct vnt_private *priv);
-void bb_power_save_mode_on(struct vnt_private *priv);
-void bb_power_save_mode_off(struct vnt_private *priv);
 void bb_set_tx_antenna_mode(struct vnt_private *priv,
 			    unsigned char by_antenna_mode);
 void bb_set_rx_antenna_mode(struct vnt_private *priv,
diff --git a/drivers/staging/vt6656/card.c b/drivers/staging/vt6656/card.c
index 7827e579ef3d..b9dc0d13c00c 100644
--- a/drivers/staging/vt6656/card.c
+++ b/drivers/staging/vt6656/card.c
@@ -11,7 +11,6 @@
  *      vnt_add_basic_rate - Add to BasicRateSet
  *      vnt_ofdm_min_rate - Check if any OFDM rate is in BasicRateSet
  *      vnt_get_tsf_offset - Calculate TSFOffset
- *      vnt_get_current_tsf - Read Current NIC TSF counter
  *      vnt_get_next_tbtt - Calculate Next Beacon TSF counter
  *      vnt_reset_next_tbtt - Set NIC Beacon time
  *      vnt_update_next_tbtt - Sync. NIC Beacon time
@@ -231,26 +230,6 @@ int vnt_adjust_tsf(struct vnt_private *priv, u8 rx_rate,
 }
 
 /*
- * Description: Read NIC TSF counter
- *              Get local TSF counter
- *
- * Parameters:
- *  In:
- *	priv		- The adapter to be read
- *  Out:
- *	current_tsf	- Current TSF counter
- *
- * Return Value: true if success; otherwise false
- *
- */
-bool vnt_get_current_tsf(struct vnt_private *priv, u64 *current_tsf)
-{
-	*current_tsf = priv->current_tsf;
-
-	return true;
-}
-
-/*
  * Description: Clear NIC TSF counter
  *              Clear local TSF counter
  *
diff --git a/drivers/staging/vt6656/card.h b/drivers/staging/vt6656/card.h
index be32c25c95de..eb01f7cc871f 100644
--- a/drivers/staging/vt6656/card.h
+++ b/drivers/staging/vt6656/card.h
@@ -30,7 +30,6 @@ void vnt_update_top_rates(struct vnt_private *priv);
 bool vnt_ofdm_min_rate(struct vnt_private *priv);
 int vnt_adjust_tsf(struct vnt_private *priv, u8 rx_rate,
 		   u64 time_stamp, u64 local_tsf);
-bool vnt_get_current_tsf(struct vnt_private *priv, u64 *current_tsf);
 bool vnt_clear_current_tsf(struct vnt_private *priv);
 int vnt_reset_next_tbtt(struct vnt_private *priv, u16 beacon_interval);
 int vnt_update_next_tbtt(struct vnt_private *priv, u64 tsf,
diff --git a/drivers/staging/wlan-ng/prism2fw.c b/drivers/staging/wlan-ng/prism2fw.c
index 02a2191d5c4d..11658865ca50 100644
--- a/drivers/staging/wlan-ng/prism2fw.c
+++ b/drivers/staging/wlan-ng/prism2fw.c
@@ -689,6 +689,7 @@ static int plugimage(struct imgchunk *fchunk, unsigned int nfchunks,
 	for (i = 0; i < ns3plug; i++) {
 		pstart = s3plug[i].addr;
 		pend = s3plug[i].addr + s3plug[i].len;
+		j = -1;
 		/* find the matching PDR (or filename) */
 		if (s3plug[i].itemcode != 0xffffffffUL) { /* not filename */
 			for (j = 0; j < pda->nrec; j++) {
@@ -696,8 +697,6 @@ static int plugimage(struct imgchunk *fchunk, unsigned int nfchunks,
 				    le16_to_cpu(pda->rec[j]->code))
 					break;
 			}
-		} else {
-			j = -1;
 		}
 		if (j >= pda->nrec && j != -1) { /*  if no matching PDR, fail */
 			pr_warn("warning: Failed to find PDR for plugrec 0x%04x.\n",
@@ -1008,12 +1007,11 @@ static int writeimage(struct wlandevice *wlandev, struct imgchunk *fchunk,
 	rstmsg = kzalloc(sizeof(*rstmsg), GFP_KERNEL);
 	rwrmsg = kzalloc(sizeof(*rwrmsg), GFP_KERNEL);
 	if (!rstmsg || !rwrmsg) {
-		kfree(rstmsg);
-		kfree(rwrmsg);
 		netdev_err(wlandev->netdev,
 			   "%s: no memory for firmware download, aborting download\n",
 			   __func__);
-		return -ENOMEM;
+		result = -ENOMEM;
+		goto free_result;
 	}
 
 	/* Initialize the messages */
diff --git a/drivers/target/Kconfig b/drivers/target/Kconfig
index 72171ea3dd53..92641d39126a 100644
--- a/drivers/target/Kconfig
+++ b/drivers/target/Kconfig
@@ -47,5 +47,6 @@ source "drivers/target/loopback/Kconfig"
 source "drivers/target/tcm_fc/Kconfig"
 source "drivers/target/iscsi/Kconfig"
 source "drivers/target/sbp/Kconfig"
+source "drivers/target/tcm_remote/Kconfig"
 
 endif
diff --git a/drivers/target/Makefile b/drivers/target/Makefile
index 45634747377e..431b84abfb94 100644
--- a/drivers/target/Makefile
+++ b/drivers/target/Makefile
@@ -30,3 +30,4 @@ obj-$(CONFIG_LOOPBACK_TARGET)	+= loopback/
 obj-$(CONFIG_TCM_FC)		+= tcm_fc/
 obj-$(CONFIG_ISCSI_TARGET)	+= iscsi/
 obj-$(CONFIG_SBP_TARGET)	+= sbp/
+obj-$(CONFIG_REMOTE_TARGET)	+= tcm_remote/
diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c
index baf4da7bb3b4..834cce50f9b0 100644
--- a/drivers/target/iscsi/iscsi_target.c
+++ b/drivers/target/iscsi/iscsi_target.c
@@ -26,6 +26,7 @@
 #include <target/target_core_base.h>
 #include <target/target_core_fabric.h>
 
+#include <target/target_core_backend.h>
 #include <target/iscsi/iscsi_target_core.h>
 #include "iscsi_target_parameters.h"
 #include "iscsi_target_seq_pdu_list.h"
@@ -1190,9 +1191,10 @@ int iscsit_setup_scsi_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
 	 * Initialize struct se_cmd descriptor from target_core_mod infrastructure
 	 */
 	__target_init_cmd(&cmd->se_cmd, &iscsi_ops,
-			 conn->sess->se_sess, be32_to_cpu(hdr->data_length),
-			 cmd->data_direction, sam_task_attr,
-			 cmd->sense_buffer + 2, scsilun_to_int(&hdr->lun));
+			  conn->sess->se_sess, be32_to_cpu(hdr->data_length),
+			  cmd->data_direction, sam_task_attr,
+			  cmd->sense_buffer + 2, scsilun_to_int(&hdr->lun),
+			  conn->cmd_cnt);
 
 	pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
 		" ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
@@ -2055,7 +2057,8 @@ iscsit_handle_task_mgt_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
 	__target_init_cmd(&cmd->se_cmd, &iscsi_ops,
 			  conn->sess->se_sess, 0, DMA_NONE,
 			  TCM_SIMPLE_TAG, cmd->sense_buffer + 2,
-			  scsilun_to_int(&hdr->lun));
+			  scsilun_to_int(&hdr->lun),
+			  conn->cmd_cnt);
 
 	target_get_sess_cmd(&cmd->se_cmd, true);
 
@@ -4218,9 +4221,12 @@ static void iscsit_release_commands_from_conn(struct iscsit_conn *conn)
 	list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) {
 		struct se_cmd *se_cmd = &cmd->se_cmd;
 
-		if (se_cmd->se_tfo != NULL) {
-			spin_lock_irq(&se_cmd->t_state_lock);
-			if (se_cmd->transport_state & CMD_T_ABORTED) {
+		if (!se_cmd->se_tfo)
+			continue;
+
+		spin_lock_irq(&se_cmd->t_state_lock);
+		if (se_cmd->transport_state & CMD_T_ABORTED) {
+			if (!(se_cmd->transport_state & CMD_T_TAS))
 				/*
 				 * LIO's abort path owns the cleanup for this,
 				 * so put it back on the list and let
@@ -4228,11 +4234,20 @@ static void iscsit_release_commands_from_conn(struct iscsit_conn *conn)
 				 */
 				list_move_tail(&cmd->i_conn_node,
 					       &conn->conn_cmd_list);
-			} else {
-				se_cmd->transport_state |= CMD_T_FABRIC_STOP;
-			}
+		} else {
+			se_cmd->transport_state |= CMD_T_FABRIC_STOP;
+		}
+
+		if (cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {
+			/*
+			 * We never submitted the cmd to LIO core, so we have
+			 * to tell LIO to perform the completion process.
+			 */
 			spin_unlock_irq(&se_cmd->t_state_lock);
+			target_complete_cmd(&cmd->se_cmd, SAM_STAT_TASK_ABORTED);
+			continue;
 		}
+		spin_unlock_irq(&se_cmd->t_state_lock);
 	}
 	spin_unlock_bh(&conn->cmd_lock);
 
@@ -4243,6 +4258,16 @@ static void iscsit_release_commands_from_conn(struct iscsit_conn *conn)
 		iscsit_free_cmd(cmd, true);
 
 	}
+
+	/*
+	 * Wait on commands that were cleaned up via the aborted_task path.
+	 * LLDs that implement iscsit_wait_conn will already have waited for
+	 * commands.
+	 */
+	if (!conn->conn_transport->iscsit_wait_conn) {
+		target_stop_cmd_counter(conn->cmd_cnt);
+		target_wait_for_cmds(conn->cmd_cnt);
+	}
 }
 
 static void iscsit_stop_timers_for_cmds(
@@ -4517,6 +4542,9 @@ int iscsit_close_session(struct iscsit_session *sess, bool can_sleep)
 	iscsit_stop_time2retain_timer(sess);
 	spin_unlock_bh(&se_tpg->session_lock);
 
+	if (sess->sess_ops->ErrorRecoveryLevel == 2)
+		iscsit_free_connection_recovery_entries(sess);
+
 	/*
 	 * transport_deregister_session_configfs() will clear the
 	 * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context
@@ -4540,9 +4568,6 @@ int iscsit_close_session(struct iscsit_session *sess, bool can_sleep)
 
 	transport_deregister_session(sess->se_sess);
 
-	if (sess->sess_ops->ErrorRecoveryLevel == 2)
-		iscsit_free_connection_recovery_entries(sess);
-
 	iscsit_free_all_ooo_cmdsns(sess);
 
 	spin_lock_bh(&se_tpg->session_lock);
diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c
index 27e448c2d066..274bdd7845ca 100644
--- a/drivers/target/iscsi/iscsi_target_login.c
+++ b/drivers/target/iscsi/iscsi_target_login.c
@@ -1147,8 +1147,14 @@ static struct iscsit_conn *iscsit_alloc_conn(struct iscsi_np *np)
 		goto free_conn_cpumask;
 	}
 
+	conn->cmd_cnt = target_alloc_cmd_counter();
+	if (!conn->cmd_cnt)
+		goto free_conn_allowed_cpumask;
+
 	return conn;
 
+free_conn_allowed_cpumask:
+	free_cpumask_var(conn->allowed_cpumask);
 free_conn_cpumask:
 	free_cpumask_var(conn->conn_cpumask);
 free_conn_ops:
@@ -1162,6 +1168,7 @@ free_conn:
 
 void iscsit_free_conn(struct iscsit_conn *conn)
 {
+	target_free_cmd_counter(conn->cmd_cnt);
 	free_cpumask_var(conn->allowed_cpumask);
 	free_cpumask_var(conn->conn_cpumask);
 	kfree(conn->conn_ops);
diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c
index 139031ccb700..4ec99a55ac30 100644
--- a/drivers/target/loopback/tcm_loop.c
+++ b/drivers/target/loopback/tcm_loop.c
@@ -83,15 +83,8 @@ static int tcm_loop_show_info(struct seq_file *m, struct Scsi_Host *host)
 static int tcm_loop_driver_probe(struct device *);
 static void tcm_loop_driver_remove(struct device *);
 
-static int pseudo_lld_bus_match(struct device *dev,
-				struct device_driver *dev_driver)
-{
-	return 1;
-}
-
 static struct bus_type tcm_loop_lld_bus = {
 	.name			= "tcm_loop_bus",
-	.match			= pseudo_lld_bus_match,
 	.probe			= tcm_loop_driver_probe,
 	.remove			= tcm_loop_driver_remove,
 };
@@ -298,7 +291,7 @@ static int tcm_loop_target_reset(struct scsi_cmnd *sc)
 	return FAILED;
 }
 
-static struct scsi_host_template tcm_loop_driver_template = {
+static const struct scsi_host_template tcm_loop_driver_template = {
 	.show_info		= tcm_loop_show_info,
 	.proc_name		= "tcm_loopback",
 	.name			= "TCM_Loopback",
@@ -480,30 +473,6 @@ static int tcm_loop_check_demo_mode(struct se_portal_group *se_tpg)
 	return 1;
 }
 
-static int tcm_loop_check_demo_mode_cache(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
-/*
- * Allow I_T Nexus full READ-WRITE access without explict Initiator Node ACLs for
- * local virtual Linux/SCSI LLD passthrough into VM hypervisor guest
- */
-static int tcm_loop_check_demo_mode_write_protect(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
-/*
- * Because TCM_Loop does not use explict ACLs and MappedLUNs, this will
- * never be called for TCM_Loop by target_core_fabric_configfs.c code.
- * It has been added here as a nop for target_fabric_tf_ops_check()
- */
-static int tcm_loop_check_prod_mode_write_protect(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
 static int tcm_loop_check_prot_fabric_only(struct se_portal_group *se_tpg)
 {
 	struct tcm_loop_tpg *tl_tpg = container_of(se_tpg, struct tcm_loop_tpg,
@@ -511,21 +480,11 @@ static int tcm_loop_check_prot_fabric_only(struct se_portal_group *se_tpg)
 	return tl_tpg->tl_fabric_prot_type;
 }
 
-static u32 tcm_loop_get_inst_index(struct se_portal_group *se_tpg)
-{
-	return 1;
-}
-
 static u32 tcm_loop_sess_get_index(struct se_session *se_sess)
 {
 	return 1;
 }
 
-static void tcm_loop_set_default_node_attributes(struct se_node_acl *se_acl)
-{
-	return;
-}
-
 static int tcm_loop_get_cmd_state(struct se_cmd *se_cmd)
 {
 	struct tcm_loop_cmd *tl_cmd = container_of(se_cmd,
@@ -1124,18 +1083,11 @@ static const struct target_core_fabric_ops loop_ops = {
 	.tpg_get_wwn			= tcm_loop_get_endpoint_wwn,
 	.tpg_get_tag			= tcm_loop_get_tag,
 	.tpg_check_demo_mode		= tcm_loop_check_demo_mode,
-	.tpg_check_demo_mode_cache	= tcm_loop_check_demo_mode_cache,
-	.tpg_check_demo_mode_write_protect =
-				tcm_loop_check_demo_mode_write_protect,
-	.tpg_check_prod_mode_write_protect =
-				tcm_loop_check_prod_mode_write_protect,
 	.tpg_check_prot_fabric_only	= tcm_loop_check_prot_fabric_only,
-	.tpg_get_inst_index		= tcm_loop_get_inst_index,
 	.check_stop_free		= tcm_loop_check_stop_free,
 	.release_cmd			= tcm_loop_release_cmd,
 	.sess_get_index			= tcm_loop_sess_get_index,
 	.write_pending			= tcm_loop_write_pending,
-	.set_default_node_attributes	= tcm_loop_set_default_node_attributes,
 	.get_cmd_state			= tcm_loop_get_cmd_state,
 	.queue_data_in			= tcm_loop_queue_data_in,
 	.queue_status			= tcm_loop_queue_status,
diff --git a/drivers/target/sbp/sbp_target.c b/drivers/target/sbp/sbp_target.c
index 504670994fb4..2a761bc09193 100644
--- a/drivers/target/sbp/sbp_target.c
+++ b/drivers/target/sbp/sbp_target.c
@@ -1673,11 +1673,6 @@ static int sbp_check_true(struct se_portal_group *se_tpg)
 	return 1;
 }
 
-static int sbp_check_false(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
 static char *sbp_get_fabric_wwn(struct se_portal_group *se_tpg)
 {
 	struct sbp_tpg *tpg = container_of(se_tpg, struct sbp_tpg, se_tpg);
@@ -1692,11 +1687,6 @@ static u16 sbp_get_tag(struct se_portal_group *se_tpg)
 	return tpg->tport_tpgt;
 }
 
-static u32 sbp_tpg_get_inst_index(struct se_portal_group *se_tpg)
-{
-	return 1;
-}
-
 static void sbp_release_cmd(struct se_cmd *se_cmd)
 {
 	struct sbp_target_request *req = container_of(se_cmd,
@@ -1705,11 +1695,6 @@ static void sbp_release_cmd(struct se_cmd *se_cmd)
 	sbp_free_request(req);
 }
 
-static u32 sbp_sess_get_index(struct se_session *se_sess)
-{
-	return 0;
-}
-
 static int sbp_write_pending(struct se_cmd *se_cmd)
 {
 	struct sbp_target_request *req = container_of(se_cmd,
@@ -1733,16 +1718,6 @@ static int sbp_write_pending(struct se_cmd *se_cmd)
 	return 0;
 }
 
-static void sbp_set_default_node_attrs(struct se_node_acl *nacl)
-{
-	return;
-}
-
-static int sbp_get_cmd_state(struct se_cmd *se_cmd)
-{
-	return 0;
-}
-
 static int sbp_queue_data_in(struct se_cmd *se_cmd)
 {
 	struct sbp_target_request *req = container_of(se_cmd,
@@ -2281,14 +2256,8 @@ static const struct target_core_fabric_ops sbp_ops = {
 	.tpg_get_tag			= sbp_get_tag,
 	.tpg_check_demo_mode		= sbp_check_true,
 	.tpg_check_demo_mode_cache	= sbp_check_true,
-	.tpg_check_demo_mode_write_protect = sbp_check_false,
-	.tpg_check_prod_mode_write_protect = sbp_check_false,
-	.tpg_get_inst_index		= sbp_tpg_get_inst_index,
 	.release_cmd			= sbp_release_cmd,
-	.sess_get_index			= sbp_sess_get_index,
 	.write_pending			= sbp_write_pending,
-	.set_default_node_attributes	= sbp_set_default_node_attrs,
-	.get_cmd_state			= sbp_get_cmd_state,
 	.queue_data_in			= sbp_queue_data_in,
 	.queue_status			= sbp_queue_status,
 	.queue_tm_rsp			= sbp_queue_tm_rsp,
diff --git a/drivers/target/target_core_alua.c b/drivers/target/target_core_alua.c
index c8470e7c0e10..3372856319f7 100644
--- a/drivers/target/target_core_alua.c
+++ b/drivers/target/target_core_alua.c
@@ -225,7 +225,7 @@ target_emulate_report_target_port_groups(struct se_cmd *cmd)
 			/*
 			 * Set RELATIVE TARGET PORT IDENTIFIER
 			 */
-			put_unaligned_be16(lun->lun_rtpi, &buf[off]);
+			put_unaligned_be16(lun->lun_tpg->tpg_rtpi, &buf[off]);
 			off += 2;
 			rd_len += 4;
 		}
@@ -399,7 +399,7 @@ target_emulate_set_target_port_groups(struct se_cmd *cmd)
 			spin_lock(&dev->se_port_lock);
 			list_for_each_entry(lun, &dev->dev_sep_list,
 							lun_dev_link) {
-				if (lun->lun_rtpi != rtpi)
+				if (lun->lun_tpg->tpg_rtpi != rtpi)
 					continue;
 
 				// XXX: racy unlock
diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c
index 611b0424e305..74b67c346dfe 100644
--- a/drivers/target/target_core_configfs.c
+++ b/drivers/target/target_core_configfs.c
@@ -335,6 +335,29 @@ EXPORT_SYMBOL(target_undepend_item);
 /*##############################################################################
 // Start functions called by external Target Fabrics Modules
 //############################################################################*/
+static int target_disable_feature(struct se_portal_group *se_tpg)
+{
+	return 0;
+}
+
+static u32 target_default_get_inst_index(struct se_portal_group *se_tpg)
+{
+	return 1;
+}
+
+static u32 target_default_sess_get_index(struct se_session *se_sess)
+{
+	return 0;
+}
+
+static void target_set_default_node_attributes(struct se_node_acl *se_acl)
+{
+}
+
+static int target_default_get_cmd_state(struct se_cmd *se_cmd)
+{
+	return 0;
+}
 
 static int target_fabric_tf_ops_check(const struct target_core_fabric_ops *tfo)
 {
@@ -362,46 +385,14 @@ static int target_fabric_tf_ops_check(const struct target_core_fabric_ops *tfo)
 		pr_err("Missing tfo->tpg_get_tag()\n");
 		return -EINVAL;
 	}
-	if (!tfo->tpg_check_demo_mode) {
-		pr_err("Missing tfo->tpg_check_demo_mode()\n");
-		return -EINVAL;
-	}
-	if (!tfo->tpg_check_demo_mode_cache) {
-		pr_err("Missing tfo->tpg_check_demo_mode_cache()\n");
-		return -EINVAL;
-	}
-	if (!tfo->tpg_check_demo_mode_write_protect) {
-		pr_err("Missing tfo->tpg_check_demo_mode_write_protect()\n");
-		return -EINVAL;
-	}
-	if (!tfo->tpg_check_prod_mode_write_protect) {
-		pr_err("Missing tfo->tpg_check_prod_mode_write_protect()\n");
-		return -EINVAL;
-	}
-	if (!tfo->tpg_get_inst_index) {
-		pr_err("Missing tfo->tpg_get_inst_index()\n");
-		return -EINVAL;
-	}
 	if (!tfo->release_cmd) {
 		pr_err("Missing tfo->release_cmd()\n");
 		return -EINVAL;
 	}
-	if (!tfo->sess_get_index) {
-		pr_err("Missing tfo->sess_get_index()\n");
-		return -EINVAL;
-	}
 	if (!tfo->write_pending) {
 		pr_err("Missing tfo->write_pending()\n");
 		return -EINVAL;
 	}
-	if (!tfo->set_default_node_attributes) {
-		pr_err("Missing tfo->set_default_node_attributes()\n");
-		return -EINVAL;
-	}
-	if (!tfo->get_cmd_state) {
-		pr_err("Missing tfo->get_cmd_state()\n");
-		return -EINVAL;
-	}
 	if (!tfo->queue_data_in) {
 		pr_err("Missing tfo->queue_data_in()\n");
 		return -EINVAL;
@@ -447,8 +438,36 @@ static int target_fabric_tf_ops_check(const struct target_core_fabric_ops *tfo)
 	return 0;
 }
 
+static void target_set_default_ops(struct target_core_fabric_ops *tfo)
+{
+	if (!tfo->tpg_check_demo_mode)
+		tfo->tpg_check_demo_mode = target_disable_feature;
+
+	if (!tfo->tpg_check_demo_mode_cache)
+		tfo->tpg_check_demo_mode_cache = target_disable_feature;
+
+	if (!tfo->tpg_check_demo_mode_write_protect)
+		tfo->tpg_check_demo_mode_write_protect = target_disable_feature;
+
+	if (!tfo->tpg_check_prod_mode_write_protect)
+		tfo->tpg_check_prod_mode_write_protect = target_disable_feature;
+
+	if (!tfo->tpg_get_inst_index)
+		tfo->tpg_get_inst_index = target_default_get_inst_index;
+
+	if (!tfo->sess_get_index)
+		tfo->sess_get_index = target_default_sess_get_index;
+
+	if (!tfo->set_default_node_attributes)
+		tfo->set_default_node_attributes = target_set_default_node_attributes;
+
+	if (!tfo->get_cmd_state)
+		tfo->get_cmd_state = target_default_get_cmd_state;
+}
+
 int target_register_template(const struct target_core_fabric_ops *fo)
 {
+	struct target_core_fabric_ops *tfo;
 	struct target_fabric_configfs *tf;
 	int ret;
 
@@ -461,10 +480,18 @@ int target_register_template(const struct target_core_fabric_ops *fo)
 		pr_err("%s: could not allocate memory!\n", __func__);
 		return -ENOMEM;
 	}
+	tfo = kzalloc(sizeof(struct target_core_fabric_ops), GFP_KERNEL);
+	if (!tfo) {
+		kfree(tf);
+		pr_err("%s: could not allocate memory!\n", __func__);
+		return -ENOMEM;
+	}
+	memcpy(tfo, fo, sizeof(*tfo));
+	target_set_default_ops(tfo);
 
 	INIT_LIST_HEAD(&tf->tf_list);
 	atomic_set(&tf->tf_access_cnt, 0);
-	tf->tf_ops = fo;
+	tf->tf_ops = tfo;
 	target_fabric_setup_cits(tf);
 
 	mutex_lock(&g_tf_lock);
@@ -492,6 +519,7 @@ void target_unregister_template(const struct target_core_fabric_ops *fo)
 			 */
 			rcu_barrier();
 			kfree(t->tf_tpg_base_cit.ct_attrs);
+			kfree(t->tf_ops);
 			kfree(t);
 			return;
 		}
diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index f6e58410ec3f..90f3f4926172 100644
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -223,7 +223,7 @@ struct se_dev_entry *core_get_se_deve_from_rtpi(
 				tpg->se_tpg_tfo->fabric_name);
 			continue;
 		}
-		if (lun->lun_rtpi != rtpi)
+		if (lun->lun_tpg->tpg_rtpi != rtpi)
 			continue;
 
 		kref_get(&deve->pr_kref);
@@ -479,47 +479,6 @@ void core_clear_lun_from_tpg(struct se_lun *lun, struct se_portal_group *tpg)
 	mutex_unlock(&tpg->acl_node_mutex);
 }
 
-int core_alloc_rtpi(struct se_lun *lun, struct se_device *dev)
-{
-	struct se_lun *tmp;
-
-	spin_lock(&dev->se_port_lock);
-	if (dev->export_count == 0x0000ffff) {
-		pr_warn("Reached dev->dev_port_count =="
-				" 0x0000ffff\n");
-		spin_unlock(&dev->se_port_lock);
-		return -ENOSPC;
-	}
-again:
-	/*
-	 * Allocate the next RELATIVE TARGET PORT IDENTIFIER for this struct se_device
-	 * Here is the table from spc4r17 section 7.7.3.8.
-	 *
-	 *    Table 473 -- RELATIVE TARGET PORT IDENTIFIER field
-	 *
-	 * Code      Description
-	 * 0h        Reserved
-	 * 1h        Relative port 1, historically known as port A
-	 * 2h        Relative port 2, historically known as port B
-	 * 3h to FFFFh    Relative port 3 through 65 535
-	 */
-	lun->lun_rtpi = dev->dev_rpti_counter++;
-	if (!lun->lun_rtpi)
-		goto again;
-
-	list_for_each_entry(tmp, &dev->dev_sep_list, lun_dev_link) {
-		/*
-		 * Make sure RELATIVE TARGET PORT IDENTIFIER is unique
-		 * for 16-bit wrap..
-		 */
-		if (lun->lun_rtpi == tmp->lun_rtpi)
-			goto again;
-	}
-	spin_unlock(&dev->se_port_lock);
-
-	return 0;
-}
-
 static void se_release_vpd_for_dev(struct se_device *dev)
 {
 	struct t10_vpd *vpd, *vpd_tmp;
@@ -782,6 +741,7 @@ struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
 	spin_lock_init(&dev->t10_alua.lba_map_lock);
 
 	INIT_WORK(&dev->delayed_cmd_work, target_do_delayed_work);
+	mutex_init(&dev->lun_reset_mutex);
 
 	dev->t10_wwn.t10_dev = dev;
 	/*
diff --git a/drivers/target/target_core_fabric_configfs.c b/drivers/target/target_core_fabric_configfs.c
index 67b18a67317a..b7c637644cd4 100644
--- a/drivers/target/target_core_fabric_configfs.c
+++ b/drivers/target/target_core_fabric_configfs.c
@@ -836,17 +836,49 @@ static ssize_t target_fabric_tpg_base_enable_store(struct config_item *item,
 
 	if (se_tpg->enabled == op)
 		return count;
-
-	ret = se_tpg->se_tpg_tfo->fabric_enable_tpg(se_tpg, op);
+	if (op)
+		ret = target_tpg_enable(se_tpg);
+	else
+		ret = target_tpg_disable(se_tpg);
 	if (ret)
 		return ret;
+	return count;
+}
+static ssize_t target_fabric_tpg_base_rtpi_show(struct config_item *item, char *page)
+{
+	struct se_portal_group *se_tpg = to_tpg(item);
 
-	se_tpg->enabled = op;
+	return sysfs_emit(page, "%#x\n", se_tpg->tpg_rtpi);
+}
+
+static ssize_t target_fabric_tpg_base_rtpi_store(struct config_item *item,
+				   const char *page, size_t count)
+{
+	struct se_portal_group *se_tpg = to_tpg(item);
+	u16 val;
+	int ret;
+
+	ret = kstrtou16(page, 0, &val);
+	if (ret < 0)
+		return ret;
+	if (val == 0)
+		return -EINVAL;
+
+	if (se_tpg->enabled) {
+		pr_info("%s_TPG[%hu] - Can not change RTPI on enabled TPG",
+			se_tpg->se_tpg_tfo->fabric_name,
+			se_tpg->se_tpg_tfo->tpg_get_tag(se_tpg));
+		return -EINVAL;
+	}
+
+	se_tpg->tpg_rtpi = val;
+	se_tpg->rtpi_manual = true;
 
 	return count;
 }
 
 CONFIGFS_ATTR(target_fabric_tpg_base_, enable);
+CONFIGFS_ATTR(target_fabric_tpg_base_, rtpi);
 
 static int
 target_fabric_setup_tpg_base_cit(struct target_fabric_configfs *tf)
@@ -863,8 +895,8 @@ target_fabric_setup_tpg_base_cit(struct target_fabric_configfs *tf)
 	if (tf->tf_ops->fabric_enable_tpg)
 		nr_attrs++;
 
-	if (nr_attrs == 0)
-		goto done;
+	/* + 1 for target_fabric_tpg_base_attr_rtpi */
+	nr_attrs++;
 
 	/* + 1 for final NULL in the array */
 	attrs = kcalloc(nr_attrs + 1, sizeof(*attrs), GFP_KERNEL);
@@ -876,9 +908,10 @@ target_fabric_setup_tpg_base_cit(struct target_fabric_configfs *tf)
 			attrs[i] = tf->tf_ops->tfc_tpg_base_attrs[i];
 
 	if (tf->tf_ops->fabric_enable_tpg)
-		attrs[i] = &target_fabric_tpg_base_attr_enable;
+		attrs[i++] = &target_fabric_tpg_base_attr_enable;
+
+	attrs[i++] = &target_fabric_tpg_base_attr_rtpi;
 
-done:
 	cit->ct_item_ops = &target_fabric_tpg_base_item_ops;
 	cit->ct_attrs = attrs;
 	cit->ct_owner = tf->tf_ops->module;
diff --git a/drivers/target/target_core_internal.h b/drivers/target/target_core_internal.h
index 38a6d08f75b3..408be26d2e9b 100644
--- a/drivers/target/target_core_internal.h
+++ b/drivers/target/target_core_internal.h
@@ -59,7 +59,6 @@ struct target_fabric_configfs {
 extern struct t10_alua_lu_gp *default_lu_gp;
 
 /* target_core_device.c */
-int	core_alloc_rtpi(struct se_lun *lun, struct se_device *dev);
 struct se_dev_entry *core_get_se_deve_from_rtpi(struct se_node_acl *, u16);
 void	target_pr_kref_release(struct kref *);
 void	core_free_device_list_for_node(struct se_node_acl *,
@@ -132,13 +131,14 @@ void core_tpg_remove_lun(struct se_portal_group *, struct se_lun *);
 struct se_node_acl *core_tpg_add_initiator_node_acl(struct se_portal_group *tpg,
 		const char *initiatorname);
 void core_tpg_del_initiator_node_acl(struct se_node_acl *acl);
+int target_tpg_enable(struct se_portal_group *se_tpg);
+int target_tpg_disable(struct se_portal_group *se_tpg);
 
 /* target_core_transport.c */
 int	init_se_kmem_caches(void);
 void	release_se_kmem_caches(void);
 u32	scsi_get_new_index(scsi_index_t);
 void	transport_subsystem_check_init(void);
-void	transport_uninit_session(struct se_session *);
 unsigned char *transport_dump_cmd_direction(struct se_cmd *);
 void	transport_dump_dev_state(struct se_device *, char *, int *);
 void	transport_dump_dev_info(struct se_device *, struct se_lun *,
diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c
index 1493b1d01194..d19ec4e6a4c0 100644
--- a/drivers/target/target_core_pr.c
+++ b/drivers/target/target_core_pr.c
@@ -663,7 +663,7 @@ static struct t10_pr_registration *__core_scsi3_do_alloc_registration(
 	}
 	pr_reg->pr_res_mapped_lun = mapped_lun;
 	pr_reg->pr_aptpl_target_lun = lun->unpacked_lun;
-	pr_reg->tg_pt_sep_rtpi = lun->lun_rtpi;
+	pr_reg->tg_pt_sep_rtpi = lun->lun_tpg->tpg_rtpi;
 	pr_reg->pr_res_key = sa_res_key;
 	pr_reg->pr_reg_all_tg_pt = all_tg_pt;
 	pr_reg->pr_reg_aptpl = aptpl;
@@ -967,7 +967,7 @@ static int __core_scsi3_check_aptpl_registration(
 			rcu_read_unlock();
 
 			pr_reg->pr_reg_nacl = nacl;
-			pr_reg->tg_pt_sep_rtpi = lun->lun_rtpi;
+			pr_reg->tg_pt_sep_rtpi = lun->lun_tpg->tpg_rtpi;
 			list_del(&pr_reg->pr_reg_aptpl_list);
 			spin_unlock(&pr_tmpl->aptpl_reg_lock);
 			/*
@@ -1567,7 +1567,7 @@ core_scsi3_decode_spec_i_port(
 			 */
 			if (tmp_tpg->proto_id != proto_ident)
 				continue;
-			dest_rtpi = tmp_lun->lun_rtpi;
+			dest_rtpi = tmp_lun->lun_tpg->tpg_rtpi;
 
 			iport_ptr = NULL;
 			i_str = target_parse_pr_out_transport_id(tmp_tpg,
@@ -3225,7 +3225,7 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key,
 
 	spin_lock(&dev->se_port_lock);
 	list_for_each_entry(tmp_lun, &dev->dev_sep_list, lun_dev_link) {
-		if (tmp_lun->lun_rtpi != rtpi)
+		if (tmp_lun->lun_tpg->tpg_rtpi != rtpi)
 			continue;
 		dest_se_tpg = tmp_lun->lun_tpg;
 		dest_tf_ops = dest_se_tpg->se_tpg_tfo;
diff --git a/drivers/target/target_core_spc.c b/drivers/target/target_core_spc.c
index fcc7b10a7ae3..89c0d56294cc 100644
--- a/drivers/target/target_core_spc.c
+++ b/drivers/target/target_core_spc.c
@@ -226,7 +226,6 @@ spc_emulate_evpd_83(struct se_cmd *cmd, unsigned char *buf)
 	struct t10_alua_lu_gp_member *lu_gp_mem;
 	struct t10_alua_tg_pt_gp *tg_pt_gp;
 	unsigned char *prod = &dev->t10_wwn.model[0];
-	u32 prod_len;
 	u32 off = 0;
 	u16 len = 0, id_len;
 
@@ -267,10 +266,6 @@ check_t10_vend_desc:
 	 * T10 Vendor Identifier Page, see spc4r17 section 7.7.3.4
 	 */
 	id_len = 8; /* For Vendor field */
-	prod_len = 4; /* For VPD Header */
-	prod_len += 8; /* For Vendor field */
-	prod_len += strlen(prod);
-	prod_len++; /* For : */
 
 	if (dev->dev_flags & DF_EMULATED_VPD_UNIT_SERIAL)
 		id_len += sprintf(&buf[off+12], "%s:%s", prod,
@@ -317,7 +312,7 @@ check_t10_vend_desc:
 		/* Skip over Obsolete field in RTPI payload
 		 * in Table 472 */
 		off += 2;
-		put_unaligned_be16(lun->lun_rtpi, &buf[off]);
+		put_unaligned_be16(lun->lun_tpg->tpg_rtpi, &buf[off]);
 		off += 2;
 		len += 8; /* Header size + Designation descriptor */
 		/*
diff --git a/drivers/target/target_core_stat.c b/drivers/target/target_core_stat.c
index f85ee5b0fd80..c42cbde8a31b 100644
--- a/drivers/target/target_core_stat.c
+++ b/drivers/target/target_core_stat.c
@@ -455,7 +455,7 @@ static ssize_t target_stat_port_indx_show(struct config_item *item, char *page)
 	rcu_read_lock();
 	dev = rcu_dereference(lun->lun_se_dev);
 	if (dev)
-		ret = snprintf(page, PAGE_SIZE, "%u\n", lun->lun_rtpi);
+		ret = snprintf(page, PAGE_SIZE, "%u\n", lun->lun_tpg->tpg_rtpi);
 	rcu_read_unlock();
 	return ret;
 }
@@ -561,7 +561,7 @@ static ssize_t target_stat_tgt_port_indx_show(struct config_item *item,
 	rcu_read_lock();
 	dev = rcu_dereference(lun->lun_se_dev);
 	if (dev)
-		ret = snprintf(page, PAGE_SIZE, "%u\n", lun->lun_rtpi);
+		ret = snprintf(page, PAGE_SIZE, "%u\n", lun->lun_tpg->tpg_rtpi);
 	rcu_read_unlock();
 	return ret;
 }
@@ -579,7 +579,7 @@ static ssize_t target_stat_tgt_port_name_show(struct config_item *item,
 	if (dev)
 		ret = snprintf(page, PAGE_SIZE, "%sPort#%u\n",
 			tpg->se_tpg_tfo->fabric_name,
-			lun->lun_rtpi);
+			lun->lun_tpg->tpg_rtpi);
 	rcu_read_unlock();
 	return ret;
 }
diff --git a/drivers/target/target_core_tmr.c b/drivers/target/target_core_tmr.c
index 2b95b4550a63..4718db628222 100644
--- a/drivers/target/target_core_tmr.c
+++ b/drivers/target/target_core_tmr.c
@@ -188,14 +188,23 @@ static void core_tmr_drain_tmr_list(
 	 * LUN_RESET tmr..
 	 */
 	spin_lock_irqsave(&dev->se_tmr_lock, flags);
-	if (tmr)
-		list_del_init(&tmr->tmr_list);
 	list_for_each_entry_safe(tmr_p, tmr_pp, &dev->dev_tmr_list, tmr_list) {
+		if (tmr_p == tmr)
+			continue;
+
 		cmd = tmr_p->task_cmd;
 		if (!cmd) {
 			pr_err("Unable to locate struct se_cmd for TMR\n");
 			continue;
 		}
+
+		/*
+		 * We only execute one LUN_RESET at a time so we can't wait
+		 * on them below.
+		 */
+		if (tmr_p->function == TMR_LUN_RESET)
+			continue;
+
 		/*
 		 * If this function was called with a valid pr_res_key
 		 * parameter (eg: for PROUT PREEMPT_AND_ABORT service action
@@ -379,14 +388,25 @@ int core_tmr_lun_reset(
 				tmr_nacl->initiatorname);
 		}
 	}
+
+
+	/*
+	 * We only allow one reset or preempt and abort to execute at a time
+	 * to prevent one call from claiming all the cmds causing a second
+	 * call from returning while cmds it should have waited on are still
+	 * running.
+	 */
+	mutex_lock(&dev->lun_reset_mutex);
+
 	pr_debug("LUN_RESET: %s starting for [%s], tas: %d\n",
 		(preempt_and_abort_list) ? "Preempt" : "TMR",
 		dev->transport->name, tas);
-
 	core_tmr_drain_tmr_list(dev, tmr, preempt_and_abort_list);
 	core_tmr_drain_state_list(dev, prout_cmd, tmr_sess, tas,
 				preempt_and_abort_list);
 
+	mutex_unlock(&dev->lun_reset_mutex);
+
 	/*
 	 * Clear any legacy SPC-2 reservation when called during
 	 * LOGICAL UNIT RESET
diff --git a/drivers/target/target_core_tpg.c b/drivers/target/target_core_tpg.c
index 736847c933e5..c0e429e5ef31 100644
--- a/drivers/target/target_core_tpg.c
+++ b/drivers/target/target_core_tpg.c
@@ -31,6 +31,7 @@
 #include "target_core_ua.h"
 
 extern struct se_device *g_lun0_dev;
+static DEFINE_XARRAY_ALLOC(tpg_xa);
 
 /*	__core_tpg_get_initiator_node_acl():
  *
@@ -328,7 +329,7 @@ static void target_shutdown_sessions(struct se_node_acl *acl)
 restart:
 	spin_lock_irqsave(&acl->nacl_sess_lock, flags);
 	list_for_each_entry(sess, &acl->acl_sess_list, sess_acl_list) {
-		if (atomic_read(&sess->stopped))
+		if (sess->cmd_cnt && atomic_read(&sess->cmd_cnt->stopped))
 			continue;
 
 		list_del_init(&sess->sess_acl_list);
@@ -439,6 +440,68 @@ static void core_tpg_lun_ref_release(struct percpu_ref *ref)
 	complete(&lun->lun_shutdown_comp);
 }
 
+static int target_tpg_register_rtpi(struct se_portal_group *se_tpg)
+{
+	u32 val;
+	int ret;
+
+	if (se_tpg->rtpi_manual) {
+		ret = xa_insert(&tpg_xa, se_tpg->tpg_rtpi, se_tpg, GFP_KERNEL);
+		if (ret) {
+			pr_info("%s_TPG[%hu] - Can not set RTPI %#x, it is already busy",
+				se_tpg->se_tpg_tfo->fabric_name,
+				se_tpg->se_tpg_tfo->tpg_get_tag(se_tpg),
+				se_tpg->tpg_rtpi);
+			return -EINVAL;
+		}
+	} else {
+		ret = xa_alloc(&tpg_xa, &val, se_tpg,
+			       XA_LIMIT(1, USHRT_MAX), GFP_KERNEL);
+		if (!ret)
+			se_tpg->tpg_rtpi = val;
+	}
+
+	return ret;
+}
+
+static void target_tpg_deregister_rtpi(struct se_portal_group *se_tpg)
+{
+	if (se_tpg->tpg_rtpi && se_tpg->enabled)
+		xa_erase(&tpg_xa, se_tpg->tpg_rtpi);
+}
+
+int target_tpg_enable(struct se_portal_group *se_tpg)
+{
+	int ret;
+
+	ret = target_tpg_register_rtpi(se_tpg);
+	if (ret)
+		return ret;
+
+	ret = se_tpg->se_tpg_tfo->fabric_enable_tpg(se_tpg, true);
+	if (ret) {
+		target_tpg_deregister_rtpi(se_tpg);
+		return ret;
+	}
+
+	se_tpg->enabled = true;
+
+	return 0;
+}
+
+int target_tpg_disable(struct se_portal_group *se_tpg)
+{
+	int ret;
+
+	target_tpg_deregister_rtpi(se_tpg);
+
+	ret = se_tpg->se_tpg_tfo->fabric_enable_tpg(se_tpg, false);
+	if (!ret)
+		se_tpg->enabled = false;
+
+	return ret;
+}
+
 /* Does not change se_wwn->priv. */
 int core_tpg_register(
 	struct se_wwn *se_wwn,
@@ -535,6 +598,8 @@ int core_tpg_deregister(struct se_portal_group *se_tpg)
 		kfree_rcu(se_tpg->tpg_virt_lun0, rcu_head);
 	}
 
+	target_tpg_deregister_rtpi(se_tpg);
+
 	return 0;
 }
 EXPORT_SYMBOL(core_tpg_deregister);
@@ -578,10 +643,6 @@ int core_tpg_add_lun(
 	if (ret < 0)
 		goto out;
 
-	ret = core_alloc_rtpi(lun, dev);
-	if (ret)
-		goto out_kill_ref;
-
 	if (!(dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA) &&
 	    !(dev->se_hba->hba_flags & HBA_FLAGS_INTERNAL_USE))
 		target_attach_tg_pt_gp(lun, dev->t10_alua.default_tg_pt_gp);
@@ -605,8 +666,6 @@ int core_tpg_add_lun(
 
 	return 0;
 
-out_kill_ref:
-	percpu_ref_exit(&lun->lun_ref);
 out:
 	return ret;
 }
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index 5926316252eb..86adff2a86ed 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -220,12 +220,52 @@ void transport_subsystem_check_init(void)
 	sub_api_initialized = 1;
 }
 
-static void target_release_sess_cmd_refcnt(struct percpu_ref *ref)
+static void target_release_cmd_refcnt(struct percpu_ref *ref)
 {
-	struct se_session *sess = container_of(ref, typeof(*sess), cmd_count);
+	struct target_cmd_counter *cmd_cnt  = container_of(ref,
+							   typeof(*cmd_cnt),
+							   refcnt);
+	wake_up(&cmd_cnt->refcnt_wq);
+}
+
+struct target_cmd_counter *target_alloc_cmd_counter(void)
+{
+	struct target_cmd_counter *cmd_cnt;
+	int rc;
+
+	cmd_cnt = kzalloc(sizeof(*cmd_cnt), GFP_KERNEL);
+	if (!cmd_cnt)
+		return NULL;
+
+	init_completion(&cmd_cnt->stop_done);
+	init_waitqueue_head(&cmd_cnt->refcnt_wq);
+	atomic_set(&cmd_cnt->stopped, 0);
 
-	wake_up(&sess->cmd_count_wq);
+	rc = percpu_ref_init(&cmd_cnt->refcnt, target_release_cmd_refcnt, 0,
+			     GFP_KERNEL);
+	if (rc)
+		goto free_cmd_cnt;
+
+	return cmd_cnt;
+
+free_cmd_cnt:
+	kfree(cmd_cnt);
+	return NULL;
 }
+EXPORT_SYMBOL_GPL(target_alloc_cmd_counter);
+
+void target_free_cmd_counter(struct target_cmd_counter *cmd_cnt)
+{
+	/*
+	 * Drivers like loop do not call target_stop_session during session
+	 * shutdown so we have to drop the ref taken at init time here.
+	 */
+	if (!atomic_read(&cmd_cnt->stopped))
+		percpu_ref_put(&cmd_cnt->refcnt);
+
+	percpu_ref_exit(&cmd_cnt->refcnt);
+}
+EXPORT_SYMBOL_GPL(target_free_cmd_counter);
 
 /**
  * transport_init_session - initialize a session object
@@ -233,32 +273,14 @@ static void target_release_sess_cmd_refcnt(struct percpu_ref *ref)
  *
  * The caller must have zero-initialized @se_sess before calling this function.
  */
-int transport_init_session(struct se_session *se_sess)
+void transport_init_session(struct se_session *se_sess)
 {
 	INIT_LIST_HEAD(&se_sess->sess_list);
 	INIT_LIST_HEAD(&se_sess->sess_acl_list);
 	spin_lock_init(&se_sess->sess_cmd_lock);
-	init_waitqueue_head(&se_sess->cmd_count_wq);
-	init_completion(&se_sess->stop_done);
-	atomic_set(&se_sess->stopped, 0);
-	return percpu_ref_init(&se_sess->cmd_count,
-			       target_release_sess_cmd_refcnt, 0, GFP_KERNEL);
 }
 EXPORT_SYMBOL(transport_init_session);
 
-void transport_uninit_session(struct se_session *se_sess)
-{
-	/*
-	 * Drivers like iscsi and loop do not call target_stop_session
-	 * during session shutdown so we have to drop the ref taken at init
-	 * time here.
-	 */
-	if (!atomic_read(&se_sess->stopped))
-		percpu_ref_put(&se_sess->cmd_count);
-
-	percpu_ref_exit(&se_sess->cmd_count);
-}
-
 /**
  * transport_alloc_session - allocate a session object and initialize it
  * @sup_prot_ops: bitmask that defines which T10-PI modes are supported.
@@ -266,7 +288,6 @@ void transport_uninit_session(struct se_session *se_sess)
 struct se_session *transport_alloc_session(enum target_prot_op sup_prot_ops)
 {
 	struct se_session *se_sess;
-	int ret;
 
 	se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
 	if (!se_sess) {
@@ -274,11 +295,7 @@ struct se_session *transport_alloc_session(enum target_prot_op sup_prot_ops)
 				" se_sess_cache\n");
 		return ERR_PTR(-ENOMEM);
 	}
-	ret = transport_init_session(se_sess);
-	if (ret < 0) {
-		kmem_cache_free(se_sess_cache, se_sess);
-		return ERR_PTR(ret);
-	}
+	transport_init_session(se_sess);
 	se_sess->sup_prot_ops = sup_prot_ops;
 
 	return se_sess;
@@ -444,8 +461,13 @@ target_setup_session(struct se_portal_group *tpg,
 		     int (*callback)(struct se_portal_group *,
 				     struct se_session *, void *))
 {
+	struct target_cmd_counter *cmd_cnt;
 	struct se_session *sess;
+	int rc;
 
+	cmd_cnt = target_alloc_cmd_counter();
+	if (!cmd_cnt)
+		return ERR_PTR(-ENOMEM);
 	/*
 	 * If the fabric driver is using percpu-ida based pre allocation
 	 * of I/O descriptor tags, go ahead and perform that setup now..
@@ -455,29 +477,36 @@ target_setup_session(struct se_portal_group *tpg,
 	else
 		sess = transport_alloc_session(prot_op);
 
-	if (IS_ERR(sess))
-		return sess;
+	if (IS_ERR(sess)) {
+		rc = PTR_ERR(sess);
+		goto free_cnt;
+	}
+	sess->cmd_cnt = cmd_cnt;
 
 	sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg,
 					(unsigned char *)initiatorname);
 	if (!sess->se_node_acl) {
-		transport_free_session(sess);
-		return ERR_PTR(-EACCES);
+		rc = -EACCES;
+		goto free_sess;
 	}
 	/*
 	 * Go ahead and perform any remaining fabric setup that is
 	 * required before transport_register_session().
 	 */
 	if (callback != NULL) {
-		int rc = callback(tpg, sess, private);
-		if (rc) {
-			transport_free_session(sess);
-			return ERR_PTR(rc);
-		}
+		rc = callback(tpg, sess, private);
+		if (rc)
+			goto free_sess;
 	}
 
 	transport_register_session(tpg, sess->se_node_acl, sess, private);
 	return sess;
+
+free_sess:
+	transport_free_session(sess);
+free_cnt:
+	target_free_cmd_counter(cmd_cnt);
+	return ERR_PTR(rc);
 }
 EXPORT_SYMBOL(target_setup_session);
 
@@ -602,7 +631,8 @@ void transport_free_session(struct se_session *se_sess)
 		sbitmap_queue_free(&se_sess->sess_tag_pool);
 		kvfree(se_sess->sess_cmd_map);
 	}
-	transport_uninit_session(se_sess);
+	if (se_sess->cmd_cnt)
+		target_free_cmd_counter(se_sess->cmd_cnt);
 	kmem_cache_free(se_sess_cache, se_sess);
 }
 EXPORT_SYMBOL(transport_free_session);
@@ -1412,14 +1442,12 @@ target_cmd_size_check(struct se_cmd *cmd, unsigned int size)
  *
  * Preserves the value of @cmd->tag.
  */
-void __target_init_cmd(
-	struct se_cmd *cmd,
-	const struct target_core_fabric_ops *tfo,
-	struct se_session *se_sess,
-	u32 data_length,
-	int data_direction,
-	int task_attr,
-	unsigned char *sense_buffer, u64 unpacked_lun)
+void __target_init_cmd(struct se_cmd *cmd,
+		       const struct target_core_fabric_ops *tfo,
+		       struct se_session *se_sess, u32 data_length,
+		       int data_direction, int task_attr,
+		       unsigned char *sense_buffer, u64 unpacked_lun,
+		       struct target_cmd_counter *cmd_cnt)
 {
 	INIT_LIST_HEAD(&cmd->se_delayed_node);
 	INIT_LIST_HEAD(&cmd->se_qf_node);
@@ -1439,6 +1467,7 @@ void __target_init_cmd(
 	cmd->sam_task_attr = task_attr;
 	cmd->sense_buffer = sense_buffer;
 	cmd->orig_fe_lun = unpacked_lun;
+	cmd->cmd_cnt = cmd_cnt;
 
 	if (!(cmd->se_cmd_flags & SCF_USE_CPUID))
 		cmd->cpuid = raw_smp_processor_id();
@@ -1658,7 +1687,8 @@ int target_init_cmd(struct se_cmd *se_cmd, struct se_session *se_sess,
 	 * target_core_fabric_ops->queue_status() callback
 	 */
 	__target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, data_length,
-			  data_dir, task_attr, sense, unpacked_lun);
+			  data_dir, task_attr, sense, unpacked_lun,
+			  se_sess->cmd_cnt);
 
 	/*
 	 * Obtain struct se_cmd->cmd_kref reference. A second kref_get here is
@@ -1953,7 +1983,8 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
 	BUG_ON(!se_tpg);
 
 	__target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
-			  0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun);
+			  0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun,
+			  se_sess->cmd_cnt);
 	/*
 	 * FIXME: Currently expect caller to handle se_cmd->se_tmr_req
 	 * allocation failure.
@@ -2957,7 +2988,6 @@ EXPORT_SYMBOL(transport_generic_free_cmd);
  */
 int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref)
 {
-	struct se_session *se_sess = se_cmd->se_sess;
 	int ret = 0;
 
 	/*
@@ -2970,9 +3000,14 @@ int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref)
 		se_cmd->se_cmd_flags |= SCF_ACK_KREF;
 	}
 
-	if (!percpu_ref_tryget_live(&se_sess->cmd_count))
-		ret = -ESHUTDOWN;
-
+	/*
+	 * Users like xcopy do not use counters since they never do a stop
+	 * and wait.
+	 */
+	if (se_cmd->cmd_cnt) {
+		if (!percpu_ref_tryget_live(&se_cmd->cmd_cnt->refcnt))
+			ret = -ESHUTDOWN;
+	}
 	if (ret && ack_kref)
 		target_put_sess_cmd(se_cmd);
 
@@ -2993,7 +3028,7 @@ static void target_free_cmd_mem(struct se_cmd *cmd)
 static void target_release_cmd_kref(struct kref *kref)
 {
 	struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
-	struct se_session *se_sess = se_cmd->se_sess;
+	struct target_cmd_counter *cmd_cnt = se_cmd->cmd_cnt;
 	struct completion *free_compl = se_cmd->free_compl;
 	struct completion *abrt_compl = se_cmd->abrt_compl;
 
@@ -3004,7 +3039,8 @@ static void target_release_cmd_kref(struct kref *kref)
 	if (abrt_compl)
 		complete(abrt_compl);
 
-	percpu_ref_put(&se_sess->cmd_count);
+	if (cmd_cnt)
+		percpu_ref_put(&cmd_cnt->refcnt);
 }
 
 /**
@@ -3123,46 +3159,67 @@ void target_show_cmd(const char *pfx, struct se_cmd *cmd)
 }
 EXPORT_SYMBOL(target_show_cmd);
 
-static void target_stop_session_confirm(struct percpu_ref *ref)
+static void target_stop_cmd_counter_confirm(struct percpu_ref *ref)
+{
+	struct target_cmd_counter *cmd_cnt = container_of(ref,
+						struct target_cmd_counter,
+						refcnt);
+	complete_all(&cmd_cnt->stop_done);
+}
+
+/**
+ * target_stop_cmd_counter - Stop new IO from being added to the counter.
+ * @cmd_cnt: counter to stop
+ */
+void target_stop_cmd_counter(struct target_cmd_counter *cmd_cnt)
 {
-	struct se_session *se_sess = container_of(ref, struct se_session,
-						  cmd_count);
-	complete_all(&se_sess->stop_done);
+	pr_debug("Stopping command counter.\n");
+	if (!atomic_cmpxchg(&cmd_cnt->stopped, 0, 1))
+		percpu_ref_kill_and_confirm(&cmd_cnt->refcnt,
+					    target_stop_cmd_counter_confirm);
 }
+EXPORT_SYMBOL_GPL(target_stop_cmd_counter);
 
 /**
  * target_stop_session - Stop new IO from being queued on the session.
- * @se_sess:    session to stop
+ * @se_sess: session to stop
  */
 void target_stop_session(struct se_session *se_sess)
 {
-	pr_debug("Stopping session queue.\n");
-	if (atomic_cmpxchg(&se_sess->stopped, 0, 1) == 0)
-		percpu_ref_kill_and_confirm(&se_sess->cmd_count,
-					    target_stop_session_confirm);
+	target_stop_cmd_counter(se_sess->cmd_cnt);
 }
 EXPORT_SYMBOL(target_stop_session);
 
 /**
- * target_wait_for_sess_cmds - Wait for outstanding commands
- * @se_sess:    session to wait for active I/O
+ * target_wait_for_cmds - Wait for outstanding cmds.
+ * @cmd_cnt: counter to wait for active I/O for.
  */
-void target_wait_for_sess_cmds(struct se_session *se_sess)
+void target_wait_for_cmds(struct target_cmd_counter *cmd_cnt)
 {
 	int ret;
 
-	WARN_ON_ONCE(!atomic_read(&se_sess->stopped));
+	WARN_ON_ONCE(!atomic_read(&cmd_cnt->stopped));
 
 	do {
 		pr_debug("Waiting for running cmds to complete.\n");
-		ret = wait_event_timeout(se_sess->cmd_count_wq,
-				percpu_ref_is_zero(&se_sess->cmd_count),
-				180 * HZ);
+		ret = wait_event_timeout(cmd_cnt->refcnt_wq,
+					 percpu_ref_is_zero(&cmd_cnt->refcnt),
+					 180 * HZ);
 	} while (ret <= 0);
 
-	wait_for_completion(&se_sess->stop_done);
+	wait_for_completion(&cmd_cnt->stop_done);
 	pr_debug("Waiting for cmds done.\n");
 }
+EXPORT_SYMBOL_GPL(target_wait_for_cmds);
+
+/**
+ * target_wait_for_sess_cmds - Wait for outstanding commands
+ * @se_sess: session to wait for active I/O
+ */
+void target_wait_for_sess_cmds(struct se_session *se_sess)
+{
+	target_wait_for_cmds(se_sess->cmd_cnt);
+}
 EXPORT_SYMBOL(target_wait_for_sess_cmds);
 
 /*
diff --git a/drivers/target/target_core_xcopy.c b/drivers/target/target_core_xcopy.c
index 49eaee022ef1..91ed015b588c 100644
--- a/drivers/target/target_core_xcopy.c
+++ b/drivers/target/target_core_xcopy.c
@@ -461,8 +461,6 @@ static const struct target_core_fabric_ops xcopy_pt_tfo = {
 
 int target_xcopy_setup_pt(void)
 {
-	int ret;
-
 	xcopy_wq = alloc_workqueue("xcopy_wq", WQ_MEM_RECLAIM, 0);
 	if (!xcopy_wq) {
 		pr_err("Unable to allocate xcopy_wq\n");
@@ -479,9 +477,7 @@ int target_xcopy_setup_pt(void)
 	INIT_LIST_HEAD(&xcopy_pt_nacl.acl_list);
 	INIT_LIST_HEAD(&xcopy_pt_nacl.acl_sess_list);
 	memset(&xcopy_pt_sess, 0, sizeof(struct se_session));
-	ret = transport_init_session(&xcopy_pt_sess);
-	if (ret < 0)
-		goto destroy_wq;
+	transport_init_session(&xcopy_pt_sess);
 
 	xcopy_pt_nacl.se_tpg = &xcopy_pt_tpg;
 	xcopy_pt_nacl.nacl_sess = &xcopy_pt_sess;
@@ -490,19 +486,12 @@ int target_xcopy_setup_pt(void)
 	xcopy_pt_sess.se_node_acl = &xcopy_pt_nacl;
 
 	return 0;
-
-destroy_wq:
-	destroy_workqueue(xcopy_wq);
-	xcopy_wq = NULL;
-	return ret;
 }
 
 void target_xcopy_release_pt(void)
 {
-	if (xcopy_wq) {
+	if (xcopy_wq)
 		destroy_workqueue(xcopy_wq);
-		transport_uninit_session(&xcopy_pt_sess);
-	}
 }
 
 /*
@@ -602,8 +591,8 @@ static int target_xcopy_read_source(
 		(unsigned long long)src_lba, transfer_length_block, src_bytes);
 
 	__target_init_cmd(se_cmd, &xcopy_pt_tfo, &xcopy_pt_sess, src_bytes,
-			  DMA_FROM_DEVICE, 0, &xpt_cmd.sense_buffer[0], 0);
-
+			  DMA_FROM_DEVICE, 0, &xpt_cmd.sense_buffer[0], 0,
+			  NULL);
 	rc = target_xcopy_setup_pt_cmd(&xpt_cmd, xop, src_dev, &cdb[0],
 				remote_port);
 	if (rc < 0) {
@@ -647,8 +636,8 @@ static int target_xcopy_write_destination(
 		(unsigned long long)dst_lba, transfer_length_block, dst_bytes);
 
 	__target_init_cmd(se_cmd, &xcopy_pt_tfo, &xcopy_pt_sess, dst_bytes,
-			  DMA_TO_DEVICE, 0, &xpt_cmd.sense_buffer[0], 0);
-
+			  DMA_TO_DEVICE, 0, &xpt_cmd.sense_buffer[0], 0,
+			  NULL);
 	rc = target_xcopy_setup_pt_cmd(&xpt_cmd, xop, dst_dev, &cdb[0],
 				remote_port);
 	if (rc < 0) {
diff --git a/drivers/target/tcm_fc/tcm_fc.h b/drivers/target/tcm_fc/tcm_fc.h
index 2ff716d8cbdd..00e5573c6296 100644
--- a/drivers/target/tcm_fc/tcm_fc.h
+++ b/drivers/target/tcm_fc/tcm_fc.h
@@ -146,7 +146,6 @@ void ft_release_cmd(struct se_cmd *);
 int ft_queue_status(struct se_cmd *);
 int ft_queue_data_in(struct se_cmd *);
 int ft_write_pending(struct se_cmd *);
-int ft_get_cmd_state(struct se_cmd *);
 void ft_queue_tm_resp(struct se_cmd *);
 void ft_aborted_task(struct se_cmd *);
 
diff --git a/drivers/target/tcm_fc/tfc_cmd.c b/drivers/target/tcm_fc/tfc_cmd.c
index 410b723f9d79..21783cd71c15 100644
--- a/drivers/target/tcm_fc/tfc_cmd.c
+++ b/drivers/target/tcm_fc/tfc_cmd.c
@@ -223,11 +223,6 @@ int ft_write_pending(struct se_cmd *se_cmd)
 	return 0;
 }
 
-int ft_get_cmd_state(struct se_cmd *se_cmd)
-{
-	return 0;
-}
-
 /*
  * FC sequence response handler for follow-on sequences (data) and aborts.
  */
diff --git a/drivers/target/tcm_fc/tfc_conf.c b/drivers/target/tcm_fc/tfc_conf.c
index 1a38c98f681b..6ac3fc1a7d39 100644
--- a/drivers/target/tcm_fc/tfc_conf.c
+++ b/drivers/target/tcm_fc/tfc_conf.c
@@ -398,15 +398,6 @@ static u16 ft_get_tag(struct se_portal_group *se_tpg)
 	return ft_tpg(se_tpg)->index;
 }
 
-static int ft_check_false(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
-static void ft_set_default_node_attr(struct se_node_acl *se_nacl)
-{
-}
-
 static u32 ft_tpg_get_inst_index(struct se_portal_group *se_tpg)
 {
 	return ft_tpg(se_tpg)->index;
@@ -418,10 +409,6 @@ static const struct target_core_fabric_ops ft_fabric_ops = {
 	.node_acl_size =		sizeof(struct ft_node_acl),
 	.tpg_get_wwn =			ft_get_fabric_wwn,
 	.tpg_get_tag =			ft_get_tag,
-	.tpg_check_demo_mode =		ft_check_false,
-	.tpg_check_demo_mode_cache =	ft_check_false,
-	.tpg_check_demo_mode_write_protect = ft_check_false,
-	.tpg_check_prod_mode_write_protect = ft_check_false,
 	.tpg_get_inst_index =		ft_tpg_get_inst_index,
 	.check_stop_free =		ft_check_stop_free,
 	.release_cmd =			ft_release_cmd,
@@ -429,8 +416,6 @@ static const struct target_core_fabric_ops ft_fabric_ops = {
 	.sess_get_index =		ft_sess_get_index,
 	.sess_get_initiator_sid =	NULL,
 	.write_pending =		ft_write_pending,
-	.set_default_node_attributes =	ft_set_default_node_attr,
-	.get_cmd_state =		ft_get_cmd_state,
 	.queue_data_in =		ft_queue_data_in,
 	.queue_status =			ft_queue_status,
 	.queue_tm_rsp =			ft_queue_tm_resp,
diff --git a/drivers/target/tcm_remote/Kconfig b/drivers/target/tcm_remote/Kconfig
new file mode 100644
index 000000000000..e6bebb5fe6f1
--- /dev/null
+++ b/drivers/target/tcm_remote/Kconfig
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config REMOTE_TARGET
+	tristate "TCM Virtual Remote target"
+	depends on SCSI
+	help
+	  Say Y here to enable the TCM Virtual Remote fabric
+	  That fabric is a dummy fabric to tell TCM about configuration
+	  of TPG/ACL/LUN on peer nodes in a cluster.
diff --git a/drivers/target/tcm_remote/Makefile b/drivers/target/tcm_remote/Makefile
new file mode 100644
index 000000000000..5818ffd0b0fa
--- /dev/null
+++ b/drivers/target/tcm_remote/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_REMOTE_TARGET)	+= tcm_remote.o
diff --git a/drivers/target/tcm_remote/tcm_remote.c b/drivers/target/tcm_remote/tcm_remote.c
new file mode 100644
index 000000000000..cb8db2558056
--- /dev/null
+++ b/drivers/target/tcm_remote/tcm_remote.c
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/configfs.h>
+#include <scsi/scsi.h>
+#include <scsi/scsi_tcq.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_cmnd.h>
+
+#include <target/target_core_base.h>
+#include <target/target_core_fabric.h>
+
+#include "tcm_remote.h"
+
+static inline struct tcm_remote_tpg *remote_tpg(struct se_portal_group *se_tpg)
+{
+	return container_of(se_tpg, struct tcm_remote_tpg, remote_se_tpg);
+}
+
+static char *tcm_remote_get_endpoint_wwn(struct se_portal_group *se_tpg)
+{
+	/*
+	 * Return the passed NAA identifier for the Target Port
+	 */
+	return &remote_tpg(se_tpg)->remote_hba->remote_wwn_address[0];
+}
+
+static u16 tcm_remote_get_tag(struct se_portal_group *se_tpg)
+{
+	/*
+	 * This Tag is used when forming SCSI Name identifier in EVPD=1 0x83
+	 * to represent the SCSI Target Port.
+	 */
+	return remote_tpg(se_tpg)->remote_tpgt;
+}
+
+static int tcm_remote_dummy_cmd_fn(struct se_cmd *se_cmd)
+{
+	return 0;
+}
+
+static void tcm_remote_dummy_cmd_void_fn(struct se_cmd *se_cmd)
+{
+
+}
+
+static char *tcm_remote_dump_proto_id(struct tcm_remote_hba *remote_hba)
+{
+	switch (remote_hba->remote_proto_id) {
+	case SCSI_PROTOCOL_SAS:
+		return "SAS";
+	case SCSI_PROTOCOL_SRP:
+		return "SRP";
+	case SCSI_PROTOCOL_FCP:
+		return "FCP";
+	case SCSI_PROTOCOL_ISCSI:
+		return "iSCSI";
+	default:
+		break;
+	}
+
+	return "Unknown";
+}
+
+static int tcm_remote_port_link(
+	struct se_portal_group *se_tpg,
+	struct se_lun *lun)
+{
+	pr_debug("TCM_Remote_ConfigFS: Port Link LUN %lld Successful\n",
+		 lun->unpacked_lun);
+	return 0;
+}
+
+static void tcm_remote_port_unlink(
+	struct se_portal_group *se_tpg,
+	struct se_lun *lun)
+{
+	pr_debug("TCM_Remote_ConfigFS: Port Unlink LUN %lld Successful\n",
+		 lun->unpacked_lun);
+}
+
+static struct se_portal_group *tcm_remote_make_tpg(
+	struct se_wwn *wwn,
+	const char *name)
+{
+	struct tcm_remote_hba *remote_hba = container_of(wwn,
+			struct tcm_remote_hba, remote_hba_wwn);
+	struct tcm_remote_tpg *remote_tpg;
+	unsigned long tpgt;
+	int ret;
+
+	if (strstr(name, "tpgt_") != name) {
+		pr_err("Unable to locate \"tpgt_#\" directory group\n");
+		return ERR_PTR(-EINVAL);
+	}
+	if (kstrtoul(name + 5, 10, &tpgt))
+		return ERR_PTR(-EINVAL);
+
+	if (tpgt >= TL_TPGS_PER_HBA) {
+		pr_err("Passed tpgt: %lu exceeds TL_TPGS_PER_HBA: %u\n",
+		       tpgt, TL_TPGS_PER_HBA);
+		return ERR_PTR(-EINVAL);
+	}
+	remote_tpg = &remote_hba->remote_hba_tpgs[tpgt];
+	remote_tpg->remote_hba = remote_hba;
+	remote_tpg->remote_tpgt = tpgt;
+	/*
+	 * Register the remote_tpg as a emulated TCM Target Endpoint
+	 */
+	ret = core_tpg_register(wwn, &remote_tpg->remote_se_tpg,
+				remote_hba->remote_proto_id);
+	if (ret < 0)
+		return ERR_PTR(-ENOMEM);
+
+	pr_debug("TCM_Remote_ConfigFS: Allocated Emulated %s Target Port %s,t,0x%04lx\n",
+		 tcm_remote_dump_proto_id(remote_hba),
+		 config_item_name(&wwn->wwn_group.cg_item), tpgt);
+	return &remote_tpg->remote_se_tpg;
+}
+
+static void tcm_remote_drop_tpg(struct se_portal_group *se_tpg)
+{
+	struct se_wwn *wwn = se_tpg->se_tpg_wwn;
+	struct tcm_remote_tpg *remote_tpg = container_of(se_tpg,
+				struct tcm_remote_tpg, remote_se_tpg);
+	struct tcm_remote_hba *remote_hba;
+	unsigned short tpgt;
+
+	remote_hba = remote_tpg->remote_hba;
+	tpgt = remote_tpg->remote_tpgt;
+
+	/*
+	 * Deregister the remote_tpg as a emulated TCM Target Endpoint
+	 */
+	core_tpg_deregister(se_tpg);
+
+	remote_tpg->remote_hba = NULL;
+	remote_tpg->remote_tpgt = 0;
+
+	pr_debug("TCM_Remote_ConfigFS: Deallocated Emulated %s Target Port %s,t,0x%04x\n",
+		 tcm_remote_dump_proto_id(remote_hba),
+		 config_item_name(&wwn->wwn_group.cg_item), tpgt);
+}
+
+static struct se_wwn *tcm_remote_make_wwn(
+	struct target_fabric_configfs *tf,
+	struct config_group *group,
+	const char *name)
+{
+	struct tcm_remote_hba *remote_hba;
+	char *ptr;
+	int ret, off = 0;
+
+	remote_hba = kzalloc(sizeof(*remote_hba), GFP_KERNEL);
+	if (!remote_hba)
+		return ERR_PTR(-ENOMEM);
+
+	/*
+	 * Determine the emulated Protocol Identifier and Target Port Name
+	 * based on the incoming configfs directory name.
+	 */
+	ptr = strstr(name, "naa.");
+	if (ptr) {
+		remote_hba->remote_proto_id = SCSI_PROTOCOL_SAS;
+		goto check_len;
+	}
+	ptr = strstr(name, "fc.");
+	if (ptr) {
+		remote_hba->remote_proto_id = SCSI_PROTOCOL_FCP;
+		off = 3; /* Skip over "fc." */
+		goto check_len;
+	}
+	ptr = strstr(name, "0x");
+	if (ptr) {
+		remote_hba->remote_proto_id = SCSI_PROTOCOL_SRP;
+		off = 2; /* Skip over "0x" */
+		goto check_len;
+	}
+	ptr = strstr(name, "iqn.");
+	if (!ptr) {
+		pr_err("Unable to locate prefix for emulated Target Port: %s\n",
+		       name);
+		ret = -EINVAL;
+		goto out;
+	}
+	remote_hba->remote_proto_id = SCSI_PROTOCOL_ISCSI;
+
+check_len:
+	if (strlen(name) >= TL_WWN_ADDR_LEN) {
+		pr_err("Emulated NAA %s Address: %s, exceeds max: %d\n",
+		       name, tcm_remote_dump_proto_id(remote_hba), TL_WWN_ADDR_LEN);
+		ret = -EINVAL;
+		goto out;
+	}
+	snprintf(&remote_hba->remote_wwn_address[0], TL_WWN_ADDR_LEN, "%s", &name[off]);
+
+	pr_debug("TCM_Remote_ConfigFS: Allocated emulated Target %s Address: %s\n",
+		 tcm_remote_dump_proto_id(remote_hba), name);
+	return &remote_hba->remote_hba_wwn;
+out:
+	kfree(remote_hba);
+	return ERR_PTR(ret);
+}
+
+static void tcm_remote_drop_wwn(struct se_wwn *wwn)
+{
+	struct tcm_remote_hba *remote_hba = container_of(wwn,
+				struct tcm_remote_hba, remote_hba_wwn);
+
+	pr_debug("TCM_Remote_ConfigFS: Deallocating emulated Target %s Address: %s\n",
+		 tcm_remote_dump_proto_id(remote_hba),
+		 remote_hba->remote_wwn_address);
+	kfree(remote_hba);
+}
+
+static ssize_t tcm_remote_wwn_version_show(struct config_item *item, char *page)
+{
+	return sprintf(page, "TCM Remote Fabric module %s\n", TCM_REMOTE_VERSION);
+}
+
+CONFIGFS_ATTR_RO(tcm_remote_wwn_, version);
+
+static struct configfs_attribute *tcm_remote_wwn_attrs[] = {
+	&tcm_remote_wwn_attr_version,
+	NULL,
+};
+
+static const struct target_core_fabric_ops remote_ops = {
+	.module				= THIS_MODULE,
+	.fabric_name			= "remote",
+	.tpg_get_wwn			= tcm_remote_get_endpoint_wwn,
+	.tpg_get_tag			= tcm_remote_get_tag,
+	.check_stop_free		= tcm_remote_dummy_cmd_fn,
+	.release_cmd			= tcm_remote_dummy_cmd_void_fn,
+	.write_pending			= tcm_remote_dummy_cmd_fn,
+	.queue_data_in			= tcm_remote_dummy_cmd_fn,
+	.queue_status			= tcm_remote_dummy_cmd_fn,
+	.queue_tm_rsp			= tcm_remote_dummy_cmd_void_fn,
+	.aborted_task			= tcm_remote_dummy_cmd_void_fn,
+	.fabric_make_wwn		= tcm_remote_make_wwn,
+	.fabric_drop_wwn		= tcm_remote_drop_wwn,
+	.fabric_make_tpg		= tcm_remote_make_tpg,
+	.fabric_drop_tpg		= tcm_remote_drop_tpg,
+	.fabric_post_link		= tcm_remote_port_link,
+	.fabric_pre_unlink		= tcm_remote_port_unlink,
+	.tfc_wwn_attrs			= tcm_remote_wwn_attrs,
+};
+
+static int __init tcm_remote_fabric_init(void)
+{
+	return target_register_template(&remote_ops);
+}
+
+static void __exit tcm_remote_fabric_exit(void)
+{
+	target_unregister_template(&remote_ops);
+}
+
+MODULE_DESCRIPTION("TCM virtual remote target");
+MODULE_AUTHOR("Dmitry Bogdanov <d.bogdanov@yadro.com>");
+MODULE_LICENSE("GPL");
+module_init(tcm_remote_fabric_init);
+module_exit(tcm_remote_fabric_exit);
diff --git a/drivers/target/tcm_remote/tcm_remote.h b/drivers/target/tcm_remote/tcm_remote.h
new file mode 100644
index 000000000000..913d1a6eb3a2
--- /dev/null
+++ b/drivers/target/tcm_remote/tcm_remote.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <linux/types.h>
+#include <linux/device.h>
+
+#define TCM_REMOTE_VERSION		"v0.1"
+#define TL_WWN_ADDR_LEN			256
+#define TL_TPGS_PER_HBA			32
+
+struct tcm_remote_tpg {
+	unsigned short remote_tpgt;
+	struct se_portal_group remote_se_tpg;
+	struct tcm_remote_hba *remote_hba;
+};
+
+struct tcm_remote_hba {
+	u8 remote_proto_id;
+	unsigned char remote_wwn_address[TL_WWN_ADDR_LEN];
+	struct tcm_remote_tpg remote_hba_tpgs[TL_TPGS_PER_HBA];
+	struct se_wwn remote_hba_wwn;
+};
diff --git a/drivers/tee/amdtee/call.c b/drivers/tee/amdtee/call.c
index cec6e70f0ac9..e8cd9aaa3467 100644
--- a/drivers/tee/amdtee/call.c
+++ b/drivers/tee/amdtee/call.c
@@ -8,7 +8,7 @@
 #include <linux/tee_drv.h>
 #include <linux/psp-tee.h>
 #include <linux/slab.h>
-#include <linux/psp-sev.h>
+#include <linux/psp.h>
 #include "amdtee_if.h"
 #include "amdtee_private.h"
 
diff --git a/drivers/tee/amdtee/shm_pool.c b/drivers/tee/amdtee/shm_pool.c
index f87f96a291c9..f0303126f199 100644
--- a/drivers/tee/amdtee/shm_pool.c
+++ b/drivers/tee/amdtee/shm_pool.c
@@ -5,7 +5,7 @@
 
 #include <linux/slab.h>
 #include <linux/tee_drv.h>
-#include <linux/psp-sev.h>
+#include <linux/psp.h>
 #include "amdtee_private.h"
 
 static int pool_op_alloc(struct tee_shm_pool *pool, struct tee_shm *shm,
diff --git a/drivers/tee/optee/Kconfig b/drivers/tee/optee/Kconfig
index f121c224e682..70898bbd5809 100644
--- a/drivers/tee/optee/Kconfig
+++ b/drivers/tee/optee/Kconfig
@@ -7,3 +7,20 @@ config OPTEE
 	help
 	  This implements the OP-TEE Trusted Execution Environment (TEE)
 	  driver.
+
+config OPTEE_INSECURE_LOAD_IMAGE
+	bool "Load OP-TEE image as firmware"
+	default n
+	depends on OPTEE && ARM64
+	help
+	  This loads the BL32 image for OP-TEE as firmware when the driver is
+	  probed. This returns -EPROBE_DEFER until the firmware is loadable from
+	  the filesystem which is determined by checking the system_state until
+	  it is in SYSTEM_RUNNING. This also requires enabling the corresponding
+	  option in Trusted Firmware for Arm. The documentation there explains
+	  the security threat associated with enabling this as well as
+	  mitigations at the firmware and platform level.
+	  https://trustedfirmware-a.readthedocs.io/en/latest/threat_model/threat_model.html
+
+	  Additional documentation on kernel security risks are at
+	  Documentation/staging/tee.rst.
diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c
index 290b1bb0e9cd..df5fb5410b72 100644
--- a/drivers/tee/optee/call.c
+++ b/drivers/tee/optee/call.c
@@ -488,7 +488,7 @@ static bool is_normal_memory(pgprot_t p)
 #elif defined(CONFIG_ARM64)
 	return (pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL);
 #else
-#error "Unuspported architecture"
+#error "Unsupported architecture"
 #endif
 }
 
diff --git a/drivers/tee/optee/optee_msg.h b/drivers/tee/optee/optee_msg.h
index 70e9cc2ee96b..e8840a82b983 100644
--- a/drivers/tee/optee/optee_msg.h
+++ b/drivers/tee/optee/optee_msg.h
@@ -241,11 +241,23 @@ struct optee_msg_arg {
  * 384fb3e0-e7f8-11e3-af63-0002a5d5c51b.
  * Represented in 4 32-bit words in OPTEE_MSG_UID_0, OPTEE_MSG_UID_1,
  * OPTEE_MSG_UID_2, OPTEE_MSG_UID_3.
+ *
+ * In the case where the OP-TEE image is loaded by the kernel, this will
+ * initially return an alternate UID to reflect that we are communicating with
+ * the TF-A image loading service at that time instead of OP-TEE. That UID is:
+ * a3fbeab1-1246-315d-c7c4-06b9c03cbea4.
+ * Represented in 4 32-bit words in OPTEE_MSG_IMAGE_LOAD_UID_0,
+ * OPTEE_MSG_IMAGE_LOAD_UID_1, OPTEE_MSG_IMAGE_LOAD_UID_2,
+ * OPTEE_MSG_IMAGE_LOAD_UID_3.
  */
 #define OPTEE_MSG_UID_0			0x384fb3e0
 #define OPTEE_MSG_UID_1			0xe7f811e3
 #define OPTEE_MSG_UID_2			0xaf630002
 #define OPTEE_MSG_UID_3			0xa5d5c51b
+#define OPTEE_MSG_IMAGE_LOAD_UID_0	0xa3fbeab1
+#define OPTEE_MSG_IMAGE_LOAD_UID_1	0x1246315d
+#define OPTEE_MSG_IMAGE_LOAD_UID_2	0xc7c406b9
+#define OPTEE_MSG_IMAGE_LOAD_UID_3	0xc03cbea4
 #define OPTEE_MSG_FUNCID_CALLS_UID	0xFF01
 
 /*
diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h
index 04ae58892608..72685ee0d53f 100644
--- a/drivers/tee/optee/optee_private.h
+++ b/drivers/tee/optee/optee_private.h
@@ -94,11 +94,35 @@ struct optee_supp {
 	struct completion reqs_c;
 };
 
+/*
+ * struct optee_pcpu - per cpu notif private struct passed to work functions
+ * @optee		optee device reference
+ */
+struct optee_pcpu {
+	struct optee *optee;
+};
+
+/*
+ * struct optee_smc - optee smc communication struct
+ * @invoke_fn		handler function to invoke secure monitor
+ * @memremaped_shm	virtual address of memory in shared memory pool
+ * @sec_caps:		secure world capabilities defined by
+ *			OPTEE_SMC_SEC_CAP_* in optee_smc.h
+ * @notif_irq		interrupt used as async notification by OP-TEE or 0
+ * @optee_pcpu		per_cpu optee instance for per cpu work or NULL
+ * @notif_pcpu_wq	workqueue for per cpu asynchronous notification or NULL
+ * @notif_pcpu_work	work for per cpu asynchronous notification
+ * @notif_cpuhp_state   CPU hotplug state assigned for pcpu interrupt management
+ */
 struct optee_smc {
 	optee_invoke_fn *invoke_fn;
 	void *memremaped_shm;
 	u32 sec_caps;
 	unsigned int notif_irq;
+	struct optee_pcpu __percpu *optee_pcpu;
+	struct workqueue_struct *notif_pcpu_wq;
+	struct work_struct notif_pcpu_work;
+	unsigned int notif_cpuhp_state;
 };
 
 /**
diff --git a/drivers/tee/optee/optee_smc.h b/drivers/tee/optee/optee_smc.h
index 73b5e7760d10..7d9fa426505b 100644
--- a/drivers/tee/optee/optee_smc.h
+++ b/drivers/tee/optee/optee_smc.h
@@ -105,6 +105,30 @@ struct optee_smc_call_get_os_revision_result {
 };
 
 /*
+ * Load Trusted OS from optee/tee.bin in the Linux firmware.
+ *
+ * WARNING: Use this cautiously as it could lead to insecure loading of the
+ * Trusted OS.
+ * This SMC instructs EL3 to load a binary and execute it as the Trusted OS.
+ *
+ * Call register usage:
+ * a0 SMC Function ID, OPTEE_SMC_CALL_LOAD_IMAGE
+ * a1 Upper 32bit of a 64bit size for the payload
+ * a2 Lower 32bit of a 64bit size for the payload
+ * a3 Upper 32bit of the physical address for the payload
+ * a4 Lower 32bit of the physical address for the payload
+ *
+ * The payload is in the OP-TEE image format.
+ *
+ * Returns result in a0, 0 on success and an error code otherwise.
+ */
+#define OPTEE_SMC_FUNCID_LOAD_IMAGE 2
+#define OPTEE_SMC_CALL_LOAD_IMAGE \
+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_32, \
+			   ARM_SMCCC_OWNER_TRUSTED_OS_END, \
+			   OPTEE_SMC_FUNCID_LOAD_IMAGE)
+
+/*
  * Call with struct optee_msg_arg as argument
  *
  * When called with OPTEE_SMC_CALL_WITH_RPC_ARG or
diff --git a/drivers/tee/optee/smc_abi.c b/drivers/tee/optee/smc_abi.c
index a1c1fa1a9c28..49702cb08f4f 100644
--- a/drivers/tee/optee/smc_abi.c
+++ b/drivers/tee/optee/smc_abi.c
@@ -7,10 +7,13 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/arm-smccc.h>
+#include <linux/cpuhotplug.h>
 #include <linux/errno.h>
+#include <linux/firmware.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/irqdomain.h>
+#include <linux/kernel.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -52,6 +55,23 @@
  */
 #define OPTEE_MIN_STATIC_POOL_ALIGN    9 /* 512 bytes aligned */
 
+/* SMC ABI considers at most a single TEE firmware */
+static unsigned int pcpu_irq_num;
+
+static int optee_cpuhp_enable_pcpu_irq(unsigned int cpu)
+{
+	enable_percpu_irq(pcpu_irq_num, IRQ_TYPE_NONE);
+
+	return 0;
+}
+
+static int optee_cpuhp_disable_pcpu_irq(unsigned int cpu)
+{
+	disable_percpu_irq(pcpu_irq_num);
+
+	return 0;
+}
+
 /*
  * 1. Convert between struct tee_param and struct optee_msg_param
  *
@@ -991,9 +1011,8 @@ static u32 get_async_notif_value(optee_invoke_fn *invoke_fn, bool *value_valid,
 	return res.a1;
 }
 
-static irqreturn_t notif_irq_handler(int irq, void *dev_id)
+static irqreturn_t irq_handler(struct optee *optee)
 {
-	struct optee *optee = dev_id;
 	bool do_bottom_half = false;
 	bool value_valid;
 	bool value_pending;
@@ -1016,6 +1035,13 @@ static irqreturn_t notif_irq_handler(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t notif_irq_handler(int irq, void *dev_id)
+{
+	struct optee *optee = dev_id;
+
+	return irq_handler(optee);
+}
+
 static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)
 {
 	struct optee *optee = dev_id;
@@ -1025,7 +1051,7 @@ static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)
+static int init_irq(struct optee *optee, u_int irq)
 {
 	int rc;
 
@@ -1040,12 +1066,103 @@ static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)
 	return 0;
 }
 
+static irqreturn_t notif_pcpu_irq_handler(int irq, void *dev_id)
+{
+	struct optee_pcpu *pcpu = dev_id;
+	struct optee *optee = pcpu->optee;
+
+	if (irq_handler(optee) == IRQ_WAKE_THREAD)
+		queue_work(optee->smc.notif_pcpu_wq,
+			   &optee->smc.notif_pcpu_work);
+
+	return IRQ_HANDLED;
+}
+
+static void notif_pcpu_irq_work_fn(struct work_struct *work)
+{
+	struct optee_smc *optee_smc = container_of(work, struct optee_smc,
+						   notif_pcpu_work);
+	struct optee *optee = container_of(optee_smc, struct optee, smc);
+
+	optee_smc_do_bottom_half(optee->ctx);
+}
+
+static int init_pcpu_irq(struct optee *optee, u_int irq)
+{
+	struct optee_pcpu __percpu *optee_pcpu;
+	int cpu, rc;
+
+	optee_pcpu = alloc_percpu(struct optee_pcpu);
+	if (!optee_pcpu)
+		return -ENOMEM;
+
+	for_each_present_cpu(cpu)
+		per_cpu_ptr(optee_pcpu, cpu)->optee = optee;
+
+	rc = request_percpu_irq(irq, notif_pcpu_irq_handler,
+				"optee_pcpu_notification", optee_pcpu);
+	if (rc)
+		goto err_free_pcpu;
+
+	INIT_WORK(&optee->smc.notif_pcpu_work, notif_pcpu_irq_work_fn);
+	optee->smc.notif_pcpu_wq = create_workqueue("optee_pcpu_notification");
+	if (!optee->smc.notif_pcpu_wq) {
+		rc = -EINVAL;
+		goto err_free_pcpu_irq;
+	}
+
+	optee->smc.optee_pcpu = optee_pcpu;
+	optee->smc.notif_irq = irq;
+
+	pcpu_irq_num = irq;
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee/pcpu-notif:starting",
+			       optee_cpuhp_enable_pcpu_irq,
+			       optee_cpuhp_disable_pcpu_irq);
+	if (!rc)
+		rc = -EINVAL;
+	if (rc < 0)
+		goto err_free_pcpu_irq;
+
+	optee->smc.notif_cpuhp_state = rc;
+
+	return 0;
+
+err_free_pcpu_irq:
+	free_percpu_irq(irq, optee_pcpu);
+err_free_pcpu:
+	free_percpu(optee_pcpu);
+
+	return rc;
+}
+
+static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)
+{
+	if (irq_is_percpu_devid(irq))
+		return init_pcpu_irq(optee, irq);
+	else
+		return init_irq(optee, irq);
+}
+
+static void uninit_pcpu_irq(struct optee *optee)
+{
+	cpuhp_remove_state(optee->smc.notif_cpuhp_state);
+
+	destroy_workqueue(optee->smc.notif_pcpu_wq);
+
+	free_percpu_irq(optee->smc.notif_irq, optee->smc.optee_pcpu);
+	free_percpu(optee->smc.optee_pcpu);
+}
+
 static void optee_smc_notif_uninit_irq(struct optee *optee)
 {
 	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
 		optee_smc_stop_async_notif(optee->ctx);
 		if (optee->smc.notif_irq) {
-			free_irq(optee->smc.notif_irq, optee);
+			if (irq_is_percpu_devid(optee->smc.notif_irq))
+				uninit_pcpu_irq(optee);
+			else
+				free_irq(optee->smc.notif_irq, optee);
+
 			irq_dispose_mapping(optee->smc.notif_irq);
 		}
 	}
@@ -1149,6 +1266,22 @@ static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn)
 	return false;
 }
 
+#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
+static bool optee_msg_api_uid_is_optee_image_load(optee_invoke_fn *invoke_fn)
+{
+	struct arm_smccc_res res;
+
+	invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
+
+	if (res.a0 == OPTEE_MSG_IMAGE_LOAD_UID_0 &&
+	    res.a1 == OPTEE_MSG_IMAGE_LOAD_UID_1 &&
+	    res.a2 == OPTEE_MSG_IMAGE_LOAD_UID_2 &&
+	    res.a3 == OPTEE_MSG_IMAGE_LOAD_UID_3)
+		return true;
+	return false;
+}
+#endif
+
 static void optee_msg_get_os_revision(optee_invoke_fn *invoke_fn)
 {
 	union {
@@ -1354,6 +1487,120 @@ static void optee_shutdown(struct platform_device *pdev)
 		optee_disable_shm_cache(optee);
 }
 
+#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
+
+#define OPTEE_FW_IMAGE "optee/tee.bin"
+
+static optee_invoke_fn *cpuhp_invoke_fn;
+
+static int optee_cpuhp_probe(unsigned int cpu)
+{
+	/*
+	 * Invoking a call on a CPU will cause OP-TEE to perform the required
+	 * setup for that CPU. Just invoke the call to get the UID since that
+	 * has no side effects.
+	 */
+	if (optee_msg_api_uid_is_optee_api(cpuhp_invoke_fn))
+		return 0;
+	else
+		return -EINVAL;
+}
+
+static int optee_load_fw(struct platform_device *pdev,
+			 optee_invoke_fn *invoke_fn)
+{
+	const struct firmware *fw = NULL;
+	struct arm_smccc_res res;
+	phys_addr_t data_pa;
+	u8 *data_buf = NULL;
+	u64 data_size;
+	u32 data_pa_high, data_pa_low;
+	u32 data_size_high, data_size_low;
+	int rc;
+	int hp_state;
+
+	if (!optee_msg_api_uid_is_optee_image_load(invoke_fn))
+		return 0;
+
+	rc = request_firmware(&fw, OPTEE_FW_IMAGE, &pdev->dev);
+	if (rc) {
+		/*
+		 * The firmware in the rootfs will not be accessible until we
+		 * are in the SYSTEM_RUNNING state, so return EPROBE_DEFER until
+		 * that point.
+		 */
+		if (system_state < SYSTEM_RUNNING)
+			return -EPROBE_DEFER;
+		goto fw_err;
+	}
+
+	data_size = fw->size;
+	/*
+	 * This uses the GFP_DMA flag to ensure we are allocated memory in the
+	 * 32-bit space since TF-A cannot map memory beyond the 32-bit boundary.
+	 */
+	data_buf = kmalloc(fw->size, GFP_KERNEL | GFP_DMA);
+	if (!data_buf) {
+		rc = -ENOMEM;
+		goto fw_err;
+	}
+	memcpy(data_buf, fw->data, fw->size);
+	data_pa = virt_to_phys(data_buf);
+	reg_pair_from_64(&data_pa_high, &data_pa_low, data_pa);
+	reg_pair_from_64(&data_size_high, &data_size_low, data_size);
+	goto fw_load;
+
+fw_err:
+	pr_warn("image loading failed\n");
+	data_pa_high = 0;
+	data_pa_low = 0;
+	data_size_high = 0;
+	data_size_low = 0;
+
+fw_load:
+	/*
+	 * Always invoke the SMC, even if loading the image fails, to indicate
+	 * to EL3 that we have passed the point where it should allow invoking
+	 * this SMC.
+	 */
+	pr_warn("OP-TEE image loaded from kernel, this can be insecure");
+	invoke_fn(OPTEE_SMC_CALL_LOAD_IMAGE, data_size_high, data_size_low,
+		  data_pa_high, data_pa_low, 0, 0, 0, &res);
+	if (!rc)
+		rc = res.a0;
+	if (fw)
+		release_firmware(fw);
+	kfree(data_buf);
+
+	if (!rc) {
+		/*
+		 * We need to initialize OP-TEE on all other running cores as
+		 * well. Any cores that aren't running yet will get initialized
+		 * when they are brought up by the power management functions in
+		 * TF-A which are registered by the OP-TEE SPD. Due to that we
+		 * can un-register the callback right after registering it.
+		 */
+		cpuhp_invoke_fn = invoke_fn;
+		hp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee:probe",
+					     optee_cpuhp_probe, NULL);
+		if (hp_state < 0) {
+			pr_warn("Failed with CPU hotplug setup for OP-TEE");
+			return -EINVAL;
+		}
+		cpuhp_remove_state(hp_state);
+		cpuhp_invoke_fn = NULL;
+	}
+
+	return rc;
+}
+#else
+static inline int optee_load_fw(struct platform_device *pdev,
+				optee_invoke_fn *invoke_fn)
+{
+	return 0;
+}
+#endif
+
 static int optee_probe(struct platform_device *pdev)
 {
 	optee_invoke_fn *invoke_fn;
@@ -1372,6 +1619,10 @@ static int optee_probe(struct platform_device *pdev)
 	if (IS_ERR(invoke_fn))
 		return PTR_ERR(invoke_fn);
 
+	rc = optee_load_fw(pdev, invoke_fn);
+	if (rc)
+		return rc;
+
 	if (!optee_msg_api_uid_is_optee_api(invoke_fn)) {
 		pr_warn("api uid mismatch\n");
 		return -EINVAL;
diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c
index 452cbb8ad484..0eb342de0b00 100644
--- a/drivers/tee/tee_core.c
+++ b/drivers/tee/tee_core.c
@@ -1226,7 +1226,7 @@ static int __init tee_init(void)
 {
 	int rc;
 
-	tee_class = class_create(THIS_MODULE, "tee");
+	tee_class = class_create("tee");
 	if (IS_ERR(tee_class)) {
 		pr_err("couldn't create class\n");
 		return PTR_ERR(tee_class);
diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c
index b1c6231defad..673cf0359494 100644
--- a/drivers/tee/tee_shm.c
+++ b/drivers/tee/tee_shm.c
@@ -32,7 +32,7 @@ static int shm_get_kernel_pages(unsigned long start, size_t page_count,
 			 is_kmap_addr((void *)start)))
 		return -EINVAL;
 
-	page = virt_to_page(start);
+	page = virt_to_page((void *)start);
 	for (n = 0; n < page_count; n++) {
 		pages[n] = page + n;
 		get_page(pages[n]);
diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile
index eed300e83d48..058664bc3ec0 100644
--- a/drivers/thermal/Makefile
+++ b/drivers/thermal/Makefile
@@ -2,7 +2,7 @@
 #
 # Makefile for sensor chip drivers.
 #
-
+CFLAGS_thermal_core.o		:= -I$(src)
 obj-$(CONFIG_THERMAL)		+= thermal_sys.o
 thermal_sys-y			+= thermal_core.o thermal_sysfs.o
 thermal_sys-y			+= thermal_trip.o thermal_helpers.o
@@ -16,6 +16,7 @@ thermal_sys-$(CONFIG_THERMAL_OF)		+= thermal_of.o
 thermal_sys-$(CONFIG_THERMAL_ACPI)		+= thermal_acpi.o
 
 # governors
+CFLAGS_gov_power_allocator.o			:= -I$(src)
 thermal_sys-$(CONFIG_THERMAL_GOV_FAIR_SHARE)	+= gov_fair_share.o
 thermal_sys-$(CONFIG_THERMAL_GOV_BANG_BANG)	+= gov_bang_bang.o
 thermal_sys-$(CONFIG_THERMAL_GOV_STEP_WISE)	+= gov_step_wise.o
diff --git a/drivers/thermal/amlogic_thermal.c b/drivers/thermal/amlogic_thermal.c
index 9235fda4ec1e..3abc2dcef408 100644
--- a/drivers/thermal/amlogic_thermal.c
+++ b/drivers/thermal/amlogic_thermal.c
@@ -181,7 +181,7 @@ static int amlogic_thermal_disable(struct amlogic_thermal *data)
 static int amlogic_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
 	unsigned int tval;
-	struct amlogic_thermal *pdata = tz->devdata;
+	struct amlogic_thermal *pdata = thermal_zone_device_priv(tz);
 
 	if (!pdata)
 		return -EINVAL;
@@ -262,11 +262,8 @@ static int amlogic_thermal_probe(struct platform_device *pdev)
 		return PTR_ERR(pdata->regmap);
 
 	pdata->clk = devm_clk_get(dev, NULL);
-	if (IS_ERR(pdata->clk)) {
-		if (PTR_ERR(pdata->clk) != -EPROBE_DEFER)
-			dev_err(dev, "failed to get clock\n");
-		return PTR_ERR(pdata->clk);
-	}
+	if (IS_ERR(pdata->clk))
+		return dev_err_probe(dev, PTR_ERR(pdata->clk), "failed to get clock\n");
 
 	pdata->sec_ao_map = syscon_regmap_lookup_by_phandle
 		(pdev->dev.of_node, "amlogic,ao-secure");
@@ -285,7 +282,7 @@ static int amlogic_thermal_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	if (devm_thermal_add_hwmon_sysfs(pdata->tzd))
+	if (devm_thermal_add_hwmon_sysfs(&pdev->dev, pdata->tzd))
 		dev_warn(&pdev->dev, "Failed to add hwmon sysfs attributes\n");
 
 	ret = amlogic_thermal_initialize(pdata);
diff --git a/drivers/thermal/armada_thermal.c b/drivers/thermal/armada_thermal.c
index 2efc222a379b..0e8dfa6a7757 100644
--- a/drivers/thermal/armada_thermal.c
+++ b/drivers/thermal/armada_thermal.c
@@ -360,11 +360,8 @@ static int armada_select_channel(struct armada_thermal_priv *priv, int channel)
 	 * we must absolutely wait for the sensor validity bit to ensure we read
 	 * actual data.
 	 */
-	if (armada_wait_sensor_validity(priv)) {
-		dev_err(priv->dev,
-			"Temperature sensor reading not valid\n");
+	if (armada_wait_sensor_validity(priv))
 		return -EIO;
-	}
 
 	return 0;
 }
@@ -398,15 +395,12 @@ static int armada_read_sensor(struct armada_thermal_priv *priv, int *temp)
 static int armada_get_temp_legacy(struct thermal_zone_device *thermal,
 				  int *temp)
 {
-	struct armada_thermal_priv *priv = thermal->devdata;
+	struct armada_thermal_priv *priv = thermal_zone_device_priv(thermal);
 	int ret;
 
 	/* Valid check */
-	if (!armada_is_valid(priv)) {
-		dev_err(priv->dev,
-			"Temperature sensor reading not valid\n");
+	if (!armada_is_valid(priv))
 		return -EIO;
-	}
 
 	/* Do the actual reading */
 	ret = armada_read_sensor(priv, temp);
@@ -420,7 +414,7 @@ static struct thermal_zone_device_ops legacy_ops = {
 
 static int armada_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct armada_thermal_sensor *sensor = tz->devdata;
+	struct armada_thermal_sensor *sensor = thermal_zone_device_priv(tz);
 	struct armada_thermal_priv *priv = sensor->priv;
 	int ret;
 
diff --git a/drivers/thermal/broadcom/bcm2711_thermal.c b/drivers/thermal/broadcom/bcm2711_thermal.c
index 1f8651d15160..c243a76a3471 100644
--- a/drivers/thermal/broadcom/bcm2711_thermal.c
+++ b/drivers/thermal/broadcom/bcm2711_thermal.c
@@ -33,7 +33,7 @@ struct bcm2711_thermal_priv {
 
 static int bcm2711_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct bcm2711_thermal_priv *priv = tz->devdata;
+	struct bcm2711_thermal_priv *priv = thermal_zone_device_priv(tz);
 	int slope = thermal_zone_get_slope(tz);
 	int offset = thermal_zone_get_offset(tz);
 	u32 val;
@@ -98,7 +98,6 @@ static int bcm2711_thermal_probe(struct platform_device *pdev)
 
 	priv->thermal = thermal;
 
-	thermal->tzp->no_hwmon = false;
 	return thermal_add_hwmon_sysfs(thermal);
 }
 
diff --git a/drivers/thermal/broadcom/bcm2835_thermal.c b/drivers/thermal/broadcom/bcm2835_thermal.c
index 23918bb76ae6..3acc9288b310 100644
--- a/drivers/thermal/broadcom/bcm2835_thermal.c
+++ b/drivers/thermal/broadcom/bcm2835_thermal.c
@@ -90,7 +90,7 @@ static int bcm2835_thermal_temp2adc(int temp, int offset, int slope)
 
 static int bcm2835_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct bcm2835_thermal_data *data = tz->devdata;
+	struct bcm2835_thermal_data *data = thermal_zone_device_priv(tz);
 	u32 val = readl(data->regs + BCM2835_TS_TSENSSTAT);
 
 	if (!(val & BCM2835_TS_TSENSSTAT_VALID))
@@ -267,7 +267,6 @@ static int bcm2835_thermal_probe(struct platform_device *pdev)
 	 * Thermal_zone doesn't enable hwmon as default,
 	 * enable it here
 	 */
-	tz->tzp->no_hwmon = false;
 	err = thermal_add_hwmon_sysfs(tz);
 	if (err)
 		goto err_tz;
@@ -276,7 +275,7 @@ static int bcm2835_thermal_probe(struct platform_device *pdev)
 
 	return 0;
 err_tz:
-	thermal_of_zone_unregister(tz);
+	devm_thermal_of_zone_unregister(&pdev->dev, tz);
 err_clk:
 	clk_disable_unprepare(data->clk);
 
@@ -286,10 +285,8 @@ err_clk:
 static int bcm2835_thermal_remove(struct platform_device *pdev)
 {
 	struct bcm2835_thermal_data *data = platform_get_drvdata(pdev);
-	struct thermal_zone_device *tz = data->tz;
 
 	debugfs_remove_recursive(data->debugfsdir);
-	thermal_of_zone_unregister(tz);
 	clk_disable_unprepare(data->clk);
 
 	return 0;
diff --git a/drivers/thermal/broadcom/brcmstb_thermal.c b/drivers/thermal/broadcom/brcmstb_thermal.c
index 4d02c28331e3..72d1dbe60b8f 100644
--- a/drivers/thermal/broadcom/brcmstb_thermal.c
+++ b/drivers/thermal/broadcom/brcmstb_thermal.c
@@ -152,16 +152,14 @@ static inline u32 avs_tmon_temp_to_code(struct brcmstb_thermal_priv *priv,
 
 static int brcmstb_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct brcmstb_thermal_priv *priv = tz->devdata;
+	struct brcmstb_thermal_priv *priv = thermal_zone_device_priv(tz);
 	u32 val;
 	long t;
 
 	val = __raw_readl(priv->tmon_base + AVS_TMON_STATUS);
 
-	if (!(val & AVS_TMON_STATUS_valid_msk)) {
-		dev_err(priv->dev, "reading not valid\n");
+	if (!(val & AVS_TMON_STATUS_valid_msk))
 		return -EIO;
-	}
 
 	val = (val & AVS_TMON_STATUS_data_msk) >> AVS_TMON_STATUS_data_shift;
 
@@ -262,7 +260,7 @@ static irqreturn_t brcmstb_tmon_irq_thread(int irq, void *data)
 
 static int brcmstb_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct brcmstb_thermal_priv *priv = tz->devdata;
+	struct brcmstb_thermal_priv *priv = thermal_zone_device_priv(tz);
 
 	dev_dbg(priv->dev, "set trips %d <--> %d\n", low, high);
 
diff --git a/drivers/thermal/broadcom/ns-thermal.c b/drivers/thermal/broadcom/ns-thermal.c
index 07a8a3f49bd0..d255aa879fc0 100644
--- a/drivers/thermal/broadcom/ns-thermal.c
+++ b/drivers/thermal/broadcom/ns-thermal.c
@@ -16,7 +16,7 @@
 
 static int ns_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	void __iomem *pvtmon = tz->devdata;
+	void __iomem *pvtmon = thermal_zone_device_priv(tz);
 	int offset = thermal_zone_get_offset(tz);
 	int slope = thermal_zone_get_slope(tz);
 	u32 val;
diff --git a/drivers/thermal/broadcom/sr-thermal.c b/drivers/thermal/broadcom/sr-thermal.c
index 2b93502543ff..747915890022 100644
--- a/drivers/thermal/broadcom/sr-thermal.c
+++ b/drivers/thermal/broadcom/sr-thermal.c
@@ -32,7 +32,7 @@ struct sr_thermal {
 
 static int sr_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct sr_tmon *tmon = tz->devdata;
+	struct sr_tmon *tmon = thermal_zone_device_priv(tz);
 	struct sr_thermal *sr_thermal = tmon->priv;
 
 	*temp = readl(sr_thermal->regs + SR_TMON_TEMP_BASE(tmon->tmon_id));
diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c
index 9f8b438fcf8f..e2cc7bd30862 100644
--- a/drivers/thermal/cpufreq_cooling.c
+++ b/drivers/thermal/cpufreq_cooling.c
@@ -23,7 +23,7 @@
 #include <linux/thermal.h>
 #include <linux/units.h>
 
-#include <trace/events/thermal.h>
+#include "thermal_trace.h"
 
 /*
  * Cooling state <-> CPUFreq frequency
@@ -633,7 +633,7 @@ of_cpufreq_cooling_register(struct cpufreq_policy *policy)
 		return NULL;
 	}
 
-	if (of_find_property(np, "#cooling-cells", NULL)) {
+	if (of_property_present(np, "#cooling-cells")) {
 		struct em_perf_domain *em = em_cpu_get(policy->cpu);
 
 		cdev = __cpufreq_cooling_register(np, policy, em);
diff --git a/drivers/thermal/cpuidle_cooling.c b/drivers/thermal/cpuidle_cooling.c
index 4f41102e8b16..6f6daead485e 100644
--- a/drivers/thermal/cpuidle_cooling.c
+++ b/drivers/thermal/cpuidle_cooling.c
@@ -7,12 +7,13 @@
  */
 #define pr_fmt(fmt) "cpuidle cooling: " fmt
 
+#include <linux/cpu.h>
 #include <linux/cpu_cooling.h>
 #include <linux/cpuidle.h>
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/idle_inject.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/slab.h>
 #include <linux/thermal.h>
 
diff --git a/drivers/thermal/da9062-thermal.c b/drivers/thermal/da9062-thermal.c
index a805a6666c44..2d31b1f73423 100644
--- a/drivers/thermal/da9062-thermal.c
+++ b/drivers/thermal/da9062-thermal.c
@@ -41,6 +41,8 @@
 
 #define DA9062_MILLI_CELSIUS(t)			((t) * 1000)
 
+static unsigned int pp_tmp = DA9062_DEFAULT_POLLING_MS_PERIOD;
+
 struct da9062_thermal_config {
 	const char *name;
 };
@@ -95,7 +97,10 @@ static void da9062_thermal_poll_on(struct work_struct *work)
 		thermal_zone_device_update(thermal->zone,
 					   THERMAL_EVENT_UNSPECIFIED);
 
-		delay = thermal->zone->passive_delay_jiffies;
+		/*
+		 * pp_tmp is between 1s and 10s, so we can round the jiffies
+		 */
+		delay = round_jiffies(msecs_to_jiffies(pp_tmp));
 		queue_delayed_work(system_freezable_wq, &thermal->work, delay);
 		return;
 	}
@@ -123,7 +128,7 @@ static irqreturn_t da9062_thermal_irq_handler(int irq, void *data)
 static int da9062_thermal_get_temp(struct thermal_zone_device *z,
 				   int *temp)
 {
-	struct da9062_thermal *thermal = z->devdata;
+	struct da9062_thermal *thermal = thermal_zone_device_priv(z);
 
 	mutex_lock(&thermal->lock);
 	*temp = thermal->temperature;
@@ -155,7 +160,6 @@ static int da9062_thermal_probe(struct platform_device *pdev)
 {
 	struct da9062 *chip = dev_get_drvdata(pdev->dev.parent);
 	struct da9062_thermal *thermal;
-	unsigned int pp_tmp = DA9062_DEFAULT_POLLING_MS_PERIOD;
 	const struct of_device_id *match;
 	int ret = 0;
 
@@ -208,8 +212,7 @@ static int da9062_thermal_probe(struct platform_device *pdev)
 	}
 
 	dev_dbg(&pdev->dev,
-		"TJUNC temperature polling period set at %d ms\n",
-		jiffies_to_msecs(thermal->zone->passive_delay_jiffies));
+		"TJUNC temperature polling period set at %d ms\n", pp_tmp);
 
 	ret = platform_get_irq_byname(pdev, "THERMAL");
 	if (ret < 0)
diff --git a/drivers/thermal/db8500_thermal.c b/drivers/thermal/db8500_thermal.c
index cb10e280681f..fca5c2c93bf9 100644
--- a/drivers/thermal/db8500_thermal.c
+++ b/drivers/thermal/db8500_thermal.c
@@ -53,6 +53,7 @@ static const unsigned long db8500_thermal_points[] = {
 
 struct db8500_thermal_zone {
 	struct thermal_zone_device *tz;
+	struct device *dev;
 	unsigned long interpolated_temp;
 	unsigned int cur_index;
 };
@@ -60,7 +61,7 @@ struct db8500_thermal_zone {
 /* Callback to get current temperature */
 static int db8500_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct db8500_thermal_zone *th = tz->devdata;
+	struct db8500_thermal_zone *th = thermal_zone_device_priv(tz);
 
 	/*
 	 * TODO: There is no PRCMU interface to get temperature data currently,
@@ -114,7 +115,7 @@ static irqreturn_t prcmu_low_irq_handler(int irq, void *irq_data)
 	idx -= 1;
 
 	db8500_thermal_update_config(th, idx, next_low, next_high);
-	dev_dbg(&th->tz->device,
+	dev_dbg(th->dev,
 		"PRCMU set max %ld, min %ld\n", next_high, next_low);
 
 	thermal_zone_device_update(th->tz, THERMAL_EVENT_UNSPECIFIED);
@@ -136,7 +137,7 @@ static irqreturn_t prcmu_high_irq_handler(int irq, void *irq_data)
 
 		db8500_thermal_update_config(th, idx, next_low, next_high);
 
-		dev_dbg(&th->tz->device,
+		dev_dbg(th->dev,
 			"PRCMU set max %ld, min %ld\n", next_high, next_low);
 	} else if (idx == num_points - 1)
 		/* So we roof out 1 degree over the max point */
@@ -157,6 +158,8 @@ static int db8500_thermal_probe(struct platform_device *pdev)
 	if (!th)
 		return -ENOMEM;
 
+	th->dev = dev;
+
 	low_irq = platform_get_irq_byname(pdev, "IRQ_HOTMON_LOW");
 	if (low_irq < 0)
 		return low_irq;
diff --git a/drivers/thermal/devfreq_cooling.c b/drivers/thermal/devfreq_cooling.c
index 24b474925cd6..262e62ab6cf2 100644
--- a/drivers/thermal/devfreq_cooling.c
+++ b/drivers/thermal/devfreq_cooling.c
@@ -20,7 +20,7 @@
 #include <linux/thermal.h>
 #include <linux/units.h>
 
-#include <trace/events/thermal.h>
+#include "thermal_trace.h"
 
 #define SCALE_ERROR_MITIGATION	100
 
diff --git a/drivers/thermal/dove_thermal.c b/drivers/thermal/dove_thermal.c
index 056622a58d00..9954040d1d2c 100644
--- a/drivers/thermal/dove_thermal.c
+++ b/drivers/thermal/dove_thermal.c
@@ -87,15 +87,12 @@ static int dove_get_temp(struct thermal_zone_device *thermal,
 			  int *temp)
 {
 	unsigned long reg;
-	struct dove_thermal_priv *priv = thermal->devdata;
+	struct dove_thermal_priv *priv = thermal_zone_device_priv(thermal);
 
 	/* Valid check */
 	reg = readl_relaxed(priv->control + PMU_TEMP_DIOD_CTRL1_REG);
-	if ((reg & PMU_TDC1_TEMP_VALID_MASK) == 0x0) {
-		dev_err(&thermal->device,
-			"Temperature sensor reading not valid\n");
+	if ((reg & PMU_TDC1_TEMP_VALID_MASK) == 0x0)
 		return -EIO;
-	}
 
 	/*
 	 * Calculate temperature. According to Marvell internal
diff --git a/drivers/thermal/gov_fair_share.c b/drivers/thermal/gov_fair_share.c
index aad7d5fe3a14..03c2daeb6ee8 100644
--- a/drivers/thermal/gov_fair_share.c
+++ b/drivers/thermal/gov_fair_share.c
@@ -11,7 +11,7 @@
  */
 
 #include <linux/thermal.h>
-#include <trace/events/thermal.h>
+#include "thermal_trace.h"
 
 #include "thermal_core.h"
 
diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c
index 0eaf1527d3e3..8642f1096b91 100644
--- a/drivers/thermal/gov_power_allocator.c
+++ b/drivers/thermal/gov_power_allocator.c
@@ -12,7 +12,7 @@
 #include <linux/thermal.h>
 
 #define CREATE_TRACE_POINTS
-#include <trace/events/thermal_power_allocator.h>
+#include "thermal_trace_ipa.h"
 
 #include "thermal_core.h"
 
diff --git a/drivers/thermal/gov_step_wise.c b/drivers/thermal/gov_step_wise.c
index 31235e169c5a..3d3067804df2 100644
--- a/drivers/thermal/gov_step_wise.c
+++ b/drivers/thermal/gov_step_wise.c
@@ -12,7 +12,7 @@
 
 #include <linux/thermal.h>
 #include <linux/minmax.h>
-#include <trace/events/thermal.h>
+#include "thermal_trace.h"
 
 #include "thermal_core.h"
 
diff --git a/drivers/thermal/hisi_thermal.c b/drivers/thermal/hisi_thermal.c
index 32a7c3cf073d..3f09ef8be41a 100644
--- a/drivers/thermal/hisi_thermal.c
+++ b/drivers/thermal/hisi_thermal.c
@@ -431,14 +431,11 @@ static int hi3660_thermal_probe(struct hisi_thermal_data *data)
 
 static int hisi_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct hisi_thermal_sensor *sensor = tz->devdata;
+	struct hisi_thermal_sensor *sensor = thermal_zone_device_priv(tz);
 	struct hisi_thermal_data *data = sensor->data;
 
 	*temp = data->ops->get_temp(sensor);
 
-	dev_dbg(&data->pdev->dev, "tzd=%p, id=%d, temp=%d, thres=%d\n",
-		sensor->tzd, sensor->id, *temp, sensor->thres_temp);
-
 	return 0;
 }
 
@@ -547,7 +544,6 @@ static int hisi_thermal_probe(struct platform_device *pdev)
 {
 	struct hisi_thermal_data *data;
 	struct device *dev = &pdev->dev;
-	struct resource *res;
 	int i, ret;
 
 	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
@@ -558,8 +554,7 @@ static int hisi_thermal_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, data);
 	data->ops = of_device_get_match_data(dev);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	data->regs = devm_ioremap_resource(dev, res);
+	data->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(data->regs))
 		return PTR_ERR(data->regs);
 
diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c
index 72b5d6f319c1..d8005e9ec992 100644
--- a/drivers/thermal/imx8mm_thermal.c
+++ b/drivers/thermal/imx8mm_thermal.c
@@ -141,7 +141,7 @@ static int imx8mp_tmu_get_temp(void *data, int *temp)
 
 static int tmu_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct tmu_sensor *sensor = tz->devdata;
+	struct tmu_sensor *sensor = thermal_zone_device_priv(tz);
 	struct imx8mm_tmu *tmu = sensor->priv;
 
 	return tmu->socdata->get_temp(sensor, temp);
@@ -282,7 +282,7 @@ static int imx8mm_tmu_probe_set_calib(struct platform_device *pdev,
 	 * strongly recommended to update such old DTs to get correct
 	 * temperature compensation values for each SoC.
 	 */
-	if (!of_find_property(pdev->dev.of_node, "nvmem-cells", NULL)) {
+	if (!of_property_present(pdev->dev.of_node, "nvmem-cells")) {
 		dev_warn(dev,
 			 "No OCOTP nvmem reference found, SoC-specific calibration not loaded. Please update your DT.\n");
 		return 0;
@@ -343,7 +343,7 @@ static int imx8mm_tmu_probe(struct platform_device *pdev)
 		}
 		tmu->sensors[i].hw_id = i;
 
-		if (devm_thermal_add_hwmon_sysfs(tmu->sensors[i].tzd))
+		if (devm_thermal_add_hwmon_sysfs(&pdev->dev, tmu->sensors[i].tzd))
 			dev_warn(&pdev->dev, "failed to add hwmon sysfs attributes\n");
 	}
 
diff --git a/drivers/thermal/imx_sc_thermal.c b/drivers/thermal/imx_sc_thermal.c
index f32e59e74623..839bb9958f60 100644
--- a/drivers/thermal/imx_sc_thermal.c
+++ b/drivers/thermal/imx_sc_thermal.c
@@ -46,7 +46,7 @@ static int imx_sc_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
 	struct imx_sc_msg_misc_get_temp msg;
 	struct imx_sc_rpc_msg *hdr = &msg.hdr;
-	struct imx_sc_sensor *sensor = tz->devdata;
+	struct imx_sc_sensor *sensor = thermal_zone_device_priv(tz);
 	int ret;
 
 	msg.data.req.resource_id = sensor->resource_id;
@@ -58,11 +58,8 @@ static int imx_sc_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 	hdr->size = 2;
 
 	ret = imx_scu_call_rpc(thermal_ipc_handle, &msg, true);
-	if (ret) {
-		dev_err(&sensor->tzd->device, "read temp sensor %d failed, ret %d\n",
-			sensor->resource_id, ret);
+	if (ret)
 		return ret;
-	}
 
 	*temp = msg.data.resp.celsius * 1000 + msg.data.resp.tenths * 100;
 
@@ -119,7 +116,7 @@ static int imx_sc_thermal_probe(struct platform_device *pdev)
 			return ret;
 		}
 
-		if (devm_thermal_add_hwmon_sysfs(sensor->tzd))
+		if (devm_thermal_add_hwmon_sysfs(&pdev->dev, sensor->tzd))
 			dev_warn(&pdev->dev, "failed to add hwmon sysfs attributes\n");
 	}
 
diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c
index fb0d5cab70af..a94ec0a0c9dd 100644
--- a/drivers/thermal/imx_thermal.c
+++ b/drivers/thermal/imx_thermal.c
@@ -252,7 +252,7 @@ static void imx_set_alarm_temp(struct imx_thermal_data *data,
 
 static int imx_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct imx_thermal_data *data = tz->devdata;
+	struct imx_thermal_data *data = thermal_zone_device_priv(tz);
 	const struct thermal_soc_data *soc_data = data->socdata;
 	struct regmap *map = data->tempmon;
 	unsigned int n_meas;
@@ -265,10 +265,8 @@ static int imx_get_temp(struct thermal_zone_device *tz, int *temp)
 
 	regmap_read(map, soc_data->temp_data, &val);
 
-	if ((val & soc_data->temp_valid_mask) == 0) {
-		dev_dbg(&tz->device, "temp measurement never finished\n");
+	if ((val & soc_data->temp_valid_mask) == 0)
 		return -EAGAIN;
-	}
 
 	n_meas = (val & soc_data->temp_value_mask)
 		>> soc_data->temp_value_shift;
@@ -287,13 +285,13 @@ static int imx_get_temp(struct thermal_zone_device *tz, int *temp)
 		if (data->alarm_temp == trips[IMX_TRIP_CRITICAL].temperature &&
 			*temp < trips[IMX_TRIP_PASSIVE].temperature) {
 			imx_set_alarm_temp(data, trips[IMX_TRIP_PASSIVE].temperature);
-			dev_dbg(&tz->device, "thermal alarm off: T < %d\n",
+			dev_dbg(data->dev, "thermal alarm off: T < %d\n",
 				data->alarm_temp / 1000);
 		}
 	}
 
 	if (*temp != data->last_temp) {
-		dev_dbg(&tz->device, "millicelsius: %d\n", *temp);
+		dev_dbg(data->dev, "millicelsius: %d\n", *temp);
 		data->last_temp = *temp;
 	}
 
@@ -311,7 +309,7 @@ static int imx_get_temp(struct thermal_zone_device *tz, int *temp)
 static int imx_change_mode(struct thermal_zone_device *tz,
 			   enum thermal_device_mode mode)
 {
-	struct imx_thermal_data *data = tz->devdata;
+	struct imx_thermal_data *data = thermal_zone_device_priv(tz);
 
 	if (mode == THERMAL_DEVICE_ENABLED) {
 		pm_runtime_get(data->dev);
@@ -332,33 +330,29 @@ static int imx_change_mode(struct thermal_zone_device *tz,
 	return 0;
 }
 
-static int imx_get_crit_temp(struct thermal_zone_device *tz, int *temp)
-{
-	*temp = trips[IMX_TRIP_CRITICAL].temperature;
-
-	return 0;
-}
-
-static int imx_set_trip_temp(struct thermal_zone_device *tz, int trip,
+static int imx_set_trip_temp(struct thermal_zone_device *tz, int trip_id,
 			     int temp)
 {
-	struct imx_thermal_data *data = tz->devdata;
+	struct imx_thermal_data *data = thermal_zone_device_priv(tz);
+	struct thermal_trip trip;
 	int ret;
 
 	ret = pm_runtime_resume_and_get(data->dev);
 	if (ret < 0)
 		return ret;
 
+	ret = __thermal_zone_get_trip(tz, trip_id, &trip);
+	if (ret)
+		return ret;
+
 	/* do not allow changing critical threshold */
-	if (trip == IMX_TRIP_CRITICAL)
+	if (trip.type == THERMAL_TRIP_CRITICAL)
 		return -EPERM;
 
 	/* do not allow passive to be set higher than critical */
 	if (temp < 0 || temp > trips[IMX_TRIP_CRITICAL].temperature)
 		return -EINVAL;
 
-	trips[IMX_TRIP_PASSIVE].temperature = temp;
-
 	imx_set_alarm_temp(data, temp);
 
 	pm_runtime_put(data->dev);
@@ -369,36 +363,16 @@ static int imx_set_trip_temp(struct thermal_zone_device *tz, int trip,
 static int imx_bind(struct thermal_zone_device *tz,
 		    struct thermal_cooling_device *cdev)
 {
-	int ret;
-
-	ret = thermal_zone_bind_cooling_device(tz, IMX_TRIP_PASSIVE, cdev,
-					       THERMAL_NO_LIMIT,
-					       THERMAL_NO_LIMIT,
-					       THERMAL_WEIGHT_DEFAULT);
-	if (ret) {
-		dev_err(&tz->device,
-			"binding zone %s with cdev %s failed:%d\n",
-			tz->type, cdev->type, ret);
-		return ret;
-	}
-
-	return 0;
+	return thermal_zone_bind_cooling_device(tz, IMX_TRIP_PASSIVE, cdev,
+						THERMAL_NO_LIMIT,
+						THERMAL_NO_LIMIT,
+						THERMAL_WEIGHT_DEFAULT);
 }
 
 static int imx_unbind(struct thermal_zone_device *tz,
 		      struct thermal_cooling_device *cdev)
 {
-	int ret;
-
-	ret = thermal_zone_unbind_cooling_device(tz, IMX_TRIP_PASSIVE, cdev);
-	if (ret) {
-		dev_err(&tz->device,
-			"unbinding zone %s with cdev %s failed:%d\n",
-			tz->type, cdev->type, ret);
-		return ret;
-	}
-
-	return 0;
+	return thermal_zone_unbind_cooling_device(tz, IMX_TRIP_PASSIVE, cdev);
 }
 
 static struct thermal_zone_device_ops imx_tz_ops = {
@@ -406,7 +380,6 @@ static struct thermal_zone_device_ops imx_tz_ops = {
 	.unbind = imx_unbind,
 	.get_temp = imx_get_temp,
 	.change_mode = imx_change_mode,
-	.get_crit_temp = imx_get_crit_temp,
 	.set_trip_temp = imx_set_trip_temp,
 };
 
@@ -560,8 +533,7 @@ static irqreturn_t imx_thermal_alarm_irq_thread(int irq, void *dev)
 {
 	struct imx_thermal_data *data = dev;
 
-	dev_dbg(&data->tz->device, "THERMAL ALARM: T > %d\n",
-		data->alarm_temp / 1000);
+	dev_dbg(data->dev, "THERMAL ALARM: T > %d\n", data->alarm_temp / 1000);
 
 	thermal_zone_device_update(data->tz, THERMAL_EVENT_UNSPECIFIED);
 
@@ -594,7 +566,7 @@ static int imx_thermal_register_legacy_cooling(struct imx_thermal_data *data)
 
 	np = of_get_cpu_node(data->policy->cpu, NULL);
 
-	if (!np || !of_find_property(np, "#cooling-cells", NULL)) {
+	if (!np || !of_property_present(np, "#cooling-cells")) {
 		data->cdev = cpufreq_cooling_register(data->policy);
 		if (IS_ERR(data->cdev)) {
 			ret = PTR_ERR(data->cdev);
@@ -671,7 +643,7 @@ static int imx_thermal_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, data);
 
-	if (of_find_property(pdev->dev.of_node, "nvmem-cells", NULL)) {
+	if (of_property_present(pdev->dev.of_node, "nvmem-cells")) {
 		ret = imx_init_from_nvmem_cells(pdev);
 		if (ret)
 			return dev_err_probe(&pdev->dev, ret,
diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
index d0295123cc3e..810231b59dcd 100644
--- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
+++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
@@ -497,7 +497,7 @@ static int int3400_thermal_get_temp(struct thermal_zone_device *thermal,
 static int int3400_thermal_change_mode(struct thermal_zone_device *thermal,
 				       enum thermal_device_mode mode)
 {
-	struct int3400_thermal_priv *priv = thermal->devdata;
+	struct int3400_thermal_priv *priv = thermal_zone_device_priv(thermal);
 	int result = 0;
 
 	if (!priv)
diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c
index 00665967ca52..89cf007146ea 100644
--- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c
+++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c
@@ -14,7 +14,7 @@
 static int int340x_thermal_get_zone_temp(struct thermal_zone_device *zone,
 					 int *temp)
 {
-	struct int34x_thermal_zone *d = zone->devdata;
+	struct int34x_thermal_zone *d = thermal_zone_device_priv(zone);
 	unsigned long long tmp;
 	acpi_status status;
 
@@ -41,7 +41,7 @@ static int int340x_thermal_get_zone_temp(struct thermal_zone_device *zone,
 static int int340x_thermal_set_trip_temp(struct thermal_zone_device *zone,
 					 int trip, int temp)
 {
-	struct int34x_thermal_zone *d = zone->devdata;
+	struct int34x_thermal_zone *d = thermal_zone_device_priv(zone);
 	char name[] = {'P', 'A', 'T', '0' + trip, '\0'};
 	acpi_status status;
 
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
index a1dc18be7609..3ca0a2f5937f 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
@@ -337,7 +337,8 @@ int proc_thermal_mmio_add(struct pci_dev *pdev,
 	}
 
 	if (feature_mask & PROC_THERMAL_FEATURE_FIVR ||
-	    feature_mask & PROC_THERMAL_FEATURE_DVFS) {
+	    feature_mask & PROC_THERMAL_FEATURE_DVFS ||
+	    feature_mask & PROC_THERMAL_FEATURE_DLVR) {
 		ret = proc_thermal_rfim_add(pdev, proc_priv);
 		if (ret) {
 			dev_err(&pdev->dev, "failed to add RFIM interface\n");
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h
index 7d52fcff4937..7acaa8f1b896 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h
@@ -60,6 +60,7 @@ struct rapl_mmio_regs {
 #define PROC_THERMAL_FEATURE_FIVR	0x02
 #define PROC_THERMAL_FEATURE_DVFS	0x04
 #define PROC_THERMAL_FEATURE_MBOX	0x08
+#define PROC_THERMAL_FEATURE_DLVR	0x10
 
 #if IS_ENABLED(CONFIG_PROC_THERMAL_MMIO_RAPL)
 int proc_thermal_rapl_add(struct pci_dev *pdev, struct proc_thermal_device *proc_priv);
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
index d71ee50e7878..0d1e98007270 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
@@ -135,7 +135,7 @@ static irqreturn_t proc_thermal_irq_handler(int irq, void *devid)
 
 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
 {
-	struct proc_thermal_pci *pci_info = tzd->devdata;
+	struct proc_thermal_pci *pci_info = thermal_zone_device_priv(tzd);
 	u32 _temp;
 
 	proc_thermal_mmio_read(pci_info, PROC_THERMAL_MMIO_PKG_TEMP, &_temp);
@@ -146,7 +146,7 @@ static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
 
 static int sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
 {
-	struct proc_thermal_pci *pci_info = tzd->devdata;
+	struct proc_thermal_pci *pci_info = thermal_zone_device_priv(tzd);
 	int tjmax, _temp;
 
 	if (temp <= 0) {
@@ -351,7 +351,7 @@ static SIMPLE_DEV_PM_OPS(proc_thermal_pci_pm, proc_thermal_pci_suspend,
 
 static const struct pci_device_id proc_thermal_pci_ids[] = {
 	{ PCI_DEVICE_DATA(INTEL, ADL_THERMAL, PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_FIVR | PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_MBOX) },
-	{ PCI_DEVICE_DATA(INTEL, MTLP_THERMAL, PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_FIVR | PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_MBOX) },
+	{ PCI_DEVICE_DATA(INTEL, MTLP_THERMAL, PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_FIVR | PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_MBOX | PROC_THERMAL_FEATURE_DLVR) },
 	{ PCI_DEVICE_DATA(INTEL, RPL_THERMAL, PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_FIVR | PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_MBOX) },
 	{ },
 };
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c
index 92ed1213fe37..546b70434004 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c
@@ -39,6 +39,29 @@ static const struct mmio_reg tgl_fivr_mmio_regs[] = {
 	{ 1, 0x5A14, 2, 0x3, 1}, /* fivr_fffc_rev */
 };
 
+static const char * const dlvr_strings[] = {
+	"dlvr_spread_spectrum_pct",
+	"dlvr_control_mode",
+	"dlvr_control_lock",
+	"dlvr_rfim_enable",
+	"dlvr_freq_select",
+	"dlvr_hardware_rev",
+	"dlvr_freq_mhz",
+	"dlvr_pll_busy",
+	NULL
+};
+
+static const struct mmio_reg dlvr_mmio_regs[] = {
+	{ 0, 0x15A08, 5, 0x1F, 0}, /* dlvr_spread_spectrum_pct */
+	{ 0, 0x15A08, 1, 0x1, 5}, /* dlvr_control_mode */
+	{ 0, 0x15A08, 1, 0x1, 6}, /* dlvr_control_lock */
+	{ 0, 0x15A08, 1, 0x1, 7}, /* dlvr_rfim_enable */
+	{ 0, 0x15A08, 12, 0xFFF, 8}, /* dlvr_freq_select */
+	{ 1, 0x15A10, 2, 0x3, 30}, /* dlvr_hardware_rev */
+	{ 1, 0x15A10, 16, 0xFFFF, 0}, /* dlvr_freq_mhz */
+	{ 1, 0x15A10, 1, 0x1, 16}, /* dlvr_pll_busy */
+};
+
 /* These will represent sysfs attribute names */
 static const char * const dvfs_strings[] = {
 	"rfi_restriction_run_busy",
@@ -78,14 +101,16 @@ static ssize_t suffix##_show(struct device *dev,\
 	int ret;\
 \
 	proc_priv = pci_get_drvdata(pdev);\
-	if (table) {\
+	if (table == 1) {\
 		match_strs = (const char **)dvfs_strings;\
 		mmio_regs = adl_dvfs_mmio_regs;\
-	} else { \
+	} else if (table == 2) { \
+		match_strs = (const char **)dlvr_strings;\
+		mmio_regs = dlvr_mmio_regs;\
+	} else {\
 		match_strs = (const char **)fivr_strings;\
 		mmio_regs = tgl_fivr_mmio_regs;\
 	} \
-	\
 	ret = match_string(match_strs, -1, attr->attr.name);\
 	if (ret < 0)\
 		return ret;\
@@ -109,10 +134,13 @@ static ssize_t suffix##_store(struct device *dev,\
 	u32 mask;\
 \
 	proc_priv = pci_get_drvdata(pdev);\
-	if (table) {\
+	if (table == 1) {\
 		match_strs = (const char **)dvfs_strings;\
 		mmio_regs = adl_dvfs_mmio_regs;\
-	} else { \
+	} else if (table == 2) { \
+		match_strs = (const char **)dlvr_strings;\
+		mmio_regs = dlvr_mmio_regs;\
+	} else {\
 		match_strs = (const char **)fivr_strings;\
 		mmio_regs = tgl_fivr_mmio_regs;\
 	} \
@@ -147,6 +175,47 @@ RFIM_STORE(spread_spectrum_clk_enable, 0)
 RFIM_STORE(rfi_vco_ref_code, 0)
 RFIM_STORE(fivr_fffc_rev, 0)
 
+RFIM_SHOW(dlvr_spread_spectrum_pct, 2)
+RFIM_SHOW(dlvr_control_mode, 2)
+RFIM_SHOW(dlvr_control_lock, 2)
+RFIM_SHOW(dlvr_hardware_rev, 2)
+RFIM_SHOW(dlvr_freq_mhz, 2)
+RFIM_SHOW(dlvr_pll_busy, 2)
+RFIM_SHOW(dlvr_freq_select, 2)
+RFIM_SHOW(dlvr_rfim_enable, 2)
+
+RFIM_STORE(dlvr_spread_spectrum_pct, 2)
+RFIM_STORE(dlvr_rfim_enable, 2)
+RFIM_STORE(dlvr_freq_select, 2)
+RFIM_STORE(dlvr_control_mode, 2)
+RFIM_STORE(dlvr_control_lock, 2)
+
+static DEVICE_ATTR_RW(dlvr_spread_spectrum_pct);
+static DEVICE_ATTR_RW(dlvr_control_mode);
+static DEVICE_ATTR_RW(dlvr_control_lock);
+static DEVICE_ATTR_RW(dlvr_freq_select);
+static DEVICE_ATTR_RO(dlvr_hardware_rev);
+static DEVICE_ATTR_RO(dlvr_freq_mhz);
+static DEVICE_ATTR_RO(dlvr_pll_busy);
+static DEVICE_ATTR_RW(dlvr_rfim_enable);
+
+static struct attribute *dlvr_attrs[] = {
+	&dev_attr_dlvr_spread_spectrum_pct.attr,
+	&dev_attr_dlvr_control_mode.attr,
+	&dev_attr_dlvr_control_lock.attr,
+	&dev_attr_dlvr_freq_select.attr,
+	&dev_attr_dlvr_hardware_rev.attr,
+	&dev_attr_dlvr_freq_mhz.attr,
+	&dev_attr_dlvr_pll_busy.attr,
+	&dev_attr_dlvr_rfim_enable.attr,
+	NULL
+};
+
+static const struct attribute_group dlvr_attribute_group = {
+	.attrs = dlvr_attrs,
+	.name = "dlvr"
+};
+
 static DEVICE_ATTR_RW(vco_ref_code_lo);
 static DEVICE_ATTR_RW(vco_ref_code_hi);
 static DEVICE_ATTR_RW(spread_spectrum_pct);
@@ -277,12 +346,22 @@ int proc_thermal_rfim_add(struct pci_dev *pdev, struct proc_thermal_device *proc
 			return ret;
 	}
 
+	if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_DLVR) {
+		ret = sysfs_create_group(&pdev->dev.kobj, &dlvr_attribute_group);
+		if (ret)
+			return ret;
+	}
+
 	if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_DVFS) {
 		ret = sysfs_create_group(&pdev->dev.kobj, &dvfs_attribute_group);
 		if (ret && proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_FIVR) {
 			sysfs_remove_group(&pdev->dev.kobj, &fivr_attribute_group);
 			return ret;
 		}
+		if (ret && proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_DLVR) {
+			sysfs_remove_group(&pdev->dev.kobj, &dlvr_attribute_group);
+			return ret;
+		}
 	}
 
 	return 0;
@@ -296,6 +375,9 @@ void proc_thermal_rfim_remove(struct pci_dev *pdev)
 	if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_FIVR)
 		sysfs_remove_group(&pdev->dev.kobj, &fivr_attribute_group);
 
+	if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_DLVR)
+		sysfs_remove_group(&pdev->dev.kobj, &dlvr_attribute_group);
+
 	if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_DVFS)
 		sysfs_remove_group(&pdev->dev.kobj, &dvfs_attribute_group);
 }
diff --git a/drivers/thermal/intel/intel_pch_thermal.c b/drivers/thermal/intel/intel_pch_thermal.c
index b855d031a855..dce50d239357 100644
--- a/drivers/thermal/intel/intel_pch_thermal.c
+++ b/drivers/thermal/intel/intel_pch_thermal.c
@@ -119,7 +119,7 @@ static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, int trip)
 
 static int pch_thermal_get_temp(struct thermal_zone_device *tzd, int *temp)
 {
-	struct pch_thermal_device *ptd = tzd->devdata;
+	struct pch_thermal_device *ptd = thermal_zone_device_priv(tzd);
 
 	*temp = GET_WPT_TEMP(WPT_TEMP_TSR & readw(ptd->hw_base + WPT_TEMP));
 	return 0;
diff --git a/drivers/thermal/intel/intel_quark_dts_thermal.c b/drivers/thermal/intel/intel_quark_dts_thermal.c
index ffdc95047838..646ca8bd40a9 100644
--- a/drivers/thermal/intel/intel_quark_dts_thermal.c
+++ b/drivers/thermal/intel/intel_quark_dts_thermal.c
@@ -120,7 +120,7 @@ static DEFINE_MUTEX(dts_update_mutex);
 static int soc_dts_enable(struct thermal_zone_device *tzd)
 {
 	u32 out;
-	struct soc_sensor_entry *aux_entry = tzd->devdata;
+	struct soc_sensor_entry *aux_entry = thermal_zone_device_priv(tzd);
 	int ret;
 
 	ret = iosf_mbi_read(QRK_MBI_UNIT_RMU, MBI_REG_READ,
@@ -148,7 +148,7 @@ static int soc_dts_enable(struct thermal_zone_device *tzd)
 static int soc_dts_disable(struct thermal_zone_device *tzd)
 {
 	u32 out;
-	struct soc_sensor_entry *aux_entry = tzd->devdata;
+	struct soc_sensor_entry *aux_entry = thermal_zone_device_priv(tzd);
 	int ret;
 
 	ret = iosf_mbi_read(QRK_MBI_UNIT_RMU, MBI_REG_READ,
@@ -250,7 +250,7 @@ failed:
 static inline int sys_set_trip_temp(struct thermal_zone_device *tzd, int trip,
 				int temp)
 {
-	return update_trip_temp(tzd->devdata, trip, temp);
+	return update_trip_temp(thermal_zone_device_priv(tzd), trip, temp);
 }
 
 static int sys_get_curr_temp(struct thermal_zone_device *tzd,
diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.c b/drivers/thermal/intel/intel_soc_dts_iosf.c
index 8c26f7b2316b..f99dc7e4ae89 100644
--- a/drivers/thermal/intel/intel_soc_dts_iosf.c
+++ b/drivers/thermal/intel/intel_soc_dts_iosf.c
@@ -54,7 +54,7 @@ static int sys_get_trip_temp(struct thermal_zone_device *tzd, int trip,
 	struct intel_soc_dts_sensor_entry *dts;
 	struct intel_soc_dts_sensors *sensors;
 
-	dts = tzd->devdata;
+	dts = thermal_zone_device_priv(tzd);
 	sensors = dts->sensors;
 	mutex_lock(&sensors->dts_update_lock);
 	status = iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ,
@@ -168,7 +168,7 @@ err_restore_ptps:
 static int sys_set_trip_temp(struct thermal_zone_device *tzd, int trip,
 			     int temp)
 {
-	struct intel_soc_dts_sensor_entry *dts = tzd->devdata;
+	struct intel_soc_dts_sensor_entry *dts = thermal_zone_device_priv(tzd);
 	struct intel_soc_dts_sensors *sensors = dts->sensors;
 	int status;
 
@@ -176,7 +176,7 @@ static int sys_set_trip_temp(struct thermal_zone_device *tzd, int trip,
 		return -EINVAL;
 
 	mutex_lock(&sensors->dts_update_lock);
-	status = update_trip_temp(tzd->devdata, trip, temp,
+	status = update_trip_temp(dts, trip, temp,
 				  dts->trip_types[trip]);
 	mutex_unlock(&sensors->dts_update_lock);
 
@@ -186,9 +186,7 @@ static int sys_set_trip_temp(struct thermal_zone_device *tzd, int trip,
 static int sys_get_trip_type(struct thermal_zone_device *tzd,
 			     int trip, enum thermal_trip_type *type)
 {
-	struct intel_soc_dts_sensor_entry *dts;
-
-	dts = tzd->devdata;
+	struct intel_soc_dts_sensor_entry *dts = thermal_zone_device_priv(tzd);
 
 	*type = dts->trip_types[trip];
 
@@ -200,11 +198,10 @@ static int sys_get_curr_temp(struct thermal_zone_device *tzd,
 {
 	int status;
 	u32 out;
-	struct intel_soc_dts_sensor_entry *dts;
+	struct intel_soc_dts_sensor_entry *dts = thermal_zone_device_priv(tzd);
 	struct intel_soc_dts_sensors *sensors;
 	unsigned long raw;
 
-	dts = tzd->devdata;
 	sensors = dts->sensors;
 	status = iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ,
 			       SOC_DTS_OFFSET_TEMP, &out);
diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c
index 1c2de84742df..11a7f8108bbb 100644
--- a/drivers/thermal/intel/x86_pkg_temp_thermal.c
+++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c
@@ -107,7 +107,7 @@ static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu)
 
 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
 {
-	struct zone_device *zonedev = tzd->devdata;
+	struct zone_device *zonedev = thermal_zone_device_priv(tzd);
 	int val;
 
 	val = intel_tcc_get_temp(zonedev->cpu, true);
@@ -122,16 +122,18 @@ static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
 static int
 sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
 {
-	struct zone_device *zonedev = tzd->devdata;
+	struct zone_device *zonedev = thermal_zone_device_priv(tzd);
 	u32 l, h, mask, shift, intr;
-	int tj_max, ret;
+	int tj_max, val, ret;
 
 	tj_max = intel_tcc_get_tjmax(zonedev->cpu);
 	if (tj_max < 0)
 		return tj_max;
 	tj_max *= 1000;
 
-	if (trip >= MAX_NUMBER_OF_TRIPS || temp >= tj_max)
+	val = (tj_max - temp)/1000;
+
+	if (trip >= MAX_NUMBER_OF_TRIPS || val < 0 || val > 0x7f)
 		return -EINVAL;
 
 	ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
@@ -156,7 +158,7 @@ sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
 	if (!temp) {
 		l &= ~intr;
 	} else {
-		l |= (tj_max - temp)/1000 << shift;
+		l |= val << shift;
 		l |= intr;
 	}
 
diff --git a/drivers/thermal/k3_bandgap.c b/drivers/thermal/k3_bandgap.c
index 22c9bcb899c3..791210458606 100644
--- a/drivers/thermal/k3_bandgap.c
+++ b/drivers/thermal/k3_bandgap.c
@@ -141,7 +141,7 @@ static int k3_bgp_read_temp(struct k3_thermal_data *devdata,
 
 static int k3_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct k3_thermal_data *data = tz->devdata;
+	struct k3_thermal_data *data = thermal_zone_device_priv(tz);
 	int ret = 0;
 
 	ret = k3_bgp_read_temp(data, temp);
@@ -222,7 +222,7 @@ static int k3_bandgap_probe(struct platform_device *pdev)
 			goto err_alloc;
 		}
 
-		if (devm_thermal_add_hwmon_sysfs(data[id].tzd))
+		if (devm_thermal_add_hwmon_sysfs(dev, data[id].tzd))
 			dev_warn(dev, "Failed to add hwmon sysfs attributes\n");
 	}
 
diff --git a/drivers/thermal/k3_j72xx_bandgap.c b/drivers/thermal/k3_j72xx_bandgap.c
index 031ea1091909..5be1f09eeb2c 100644
--- a/drivers/thermal/k3_j72xx_bandgap.c
+++ b/drivers/thermal/k3_j72xx_bandgap.c
@@ -248,7 +248,7 @@ static inline int k3_bgp_read_temp(struct k3_thermal_data *devdata,
 /* Get temperature callback function for thermal zone */
 static int k3_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	return k3_bgp_read_temp(tz->devdata, temp);
+	return k3_bgp_read_temp(thermal_zone_device_priv(tz), temp);
 }
 
 static const struct thermal_zone_device_ops k3_of_thermal_ops = {
diff --git a/drivers/thermal/kirkwood_thermal.c b/drivers/thermal/kirkwood_thermal.c
index bec7ec20e79d..668747bd86ef 100644
--- a/drivers/thermal/kirkwood_thermal.c
+++ b/drivers/thermal/kirkwood_thermal.c
@@ -27,17 +27,14 @@ static int kirkwood_get_temp(struct thermal_zone_device *thermal,
 			  int *temp)
 {
 	unsigned long reg;
-	struct kirkwood_thermal_priv *priv = thermal->devdata;
+	struct kirkwood_thermal_priv *priv = thermal_zone_device_priv(thermal);
 
 	reg = readl_relaxed(priv->sensor);
 
 	/* Valid check */
 	if (!((reg >> KIRKWOOD_THERMAL_VALID_OFFSET) &
-	    KIRKWOOD_THERMAL_VALID_MASK)) {
-		dev_err(&thermal->device,
-			"Temperature sensor reading not valid\n");
+	    KIRKWOOD_THERMAL_VALID_MASK))
 		return -EIO;
-	}
 
 	/*
 	 * Calculate temperature. According to Marvell internal
diff --git a/drivers/thermal/max77620_thermal.c b/drivers/thermal/max77620_thermal.c
index 6451a55eb582..61c7622d9945 100644
--- a/drivers/thermal/max77620_thermal.c
+++ b/drivers/thermal/max77620_thermal.c
@@ -46,15 +46,13 @@ struct max77620_therm_info {
 
 static int max77620_thermal_read_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct max77620_therm_info *mtherm = tz->devdata;
+	struct max77620_therm_info *mtherm = thermal_zone_device_priv(tz);
 	unsigned int val;
 	int ret;
 
 	ret = regmap_read(mtherm->rmap, MAX77620_REG_STATLBT, &val);
-	if (ret < 0) {
-		dev_err(mtherm->dev, "Failed to read STATLBT: %d\n", ret);
+	if (ret < 0)
 		return ret;
-	}
 
 	if (val & MAX77620_IRQ_TJALRM2_MASK)
 		*temp = MAX77620_TJALARM2_TEMP;
diff --git a/drivers/thermal/mediatek/auxadc_thermal.c b/drivers/thermal/mediatek/auxadc_thermal.c
index ab730f9552d0..b6bb9eaafb74 100644
--- a/drivers/thermal/mediatek/auxadc_thermal.c
+++ b/drivers/thermal/mediatek/auxadc_thermal.c
@@ -31,6 +31,7 @@
 #define AUXADC_CON2_V		0x010
 #define AUXADC_DATA(channel)	(0x14 + (channel) * 4)
 
+#define APMIXED_SYS_TS_CON0	0x600
 #define APMIXED_SYS_TS_CON1	0x604
 
 /* Thermal Controller Registers */
@@ -281,6 +282,17 @@ enum mtk_thermal_version {
 /* The calibration coefficient of sensor  */
 #define MT7986_CALIBRATION		165
 
+/* MT8365 */
+#define MT8365_TEMP_AUXADC_CHANNEL 11
+#define MT8365_CALIBRATION 164
+#define MT8365_NUM_CONTROLLER 1
+#define MT8365_NUM_BANKS 1
+#define MT8365_NUM_SENSORS 3
+#define MT8365_NUM_SENSORS_PER_ZONE 3
+#define MT8365_TS1 0
+#define MT8365_TS2 1
+#define MT8365_TS3 2
+
 struct mtk_thermal;
 
 struct thermal_bank_cfg {
@@ -307,6 +319,9 @@ struct mtk_thermal_data {
 	bool need_switch_bank;
 	struct thermal_bank_cfg bank_data[MAX_NUM_ZONES];
 	enum mtk_thermal_version version;
+	u32 apmixed_buffer_ctl_reg;
+	u32 apmixed_buffer_ctl_mask;
+	u32 apmixed_buffer_ctl_set;
 };
 
 struct mtk_thermal {
@@ -432,6 +447,24 @@ static const int mt7986_mux_values[MT7986_NUM_SENSORS] = { 0, };
 static const int mt7986_vts_index[MT7986_NUM_SENSORS] = { VTS1 };
 static const int mt7986_tc_offset[MT7986_NUM_CONTROLLER] = { 0x0, };
 
+/* MT8365 thermal sensor data */
+static const int mt8365_bank_data[MT8365_NUM_SENSORS] = {
+	MT8365_TS1, MT8365_TS2, MT8365_TS3
+};
+
+static const int mt8365_msr[MT8365_NUM_SENSORS_PER_ZONE] = {
+	TEMP_MSR0, TEMP_MSR1, TEMP_MSR2
+};
+
+static const int mt8365_adcpnp[MT8365_NUM_SENSORS_PER_ZONE] = {
+	TEMP_ADCPNP0, TEMP_ADCPNP1, TEMP_ADCPNP2
+};
+
+static const int mt8365_mux_values[MT8365_NUM_SENSORS] = { 0, 1, 2 };
+static const int mt8365_tc_offset[MT8365_NUM_CONTROLLER] = { 0 };
+
+static const int mt8365_vts_index[MT8365_NUM_SENSORS] = { VTS1, VTS2, VTS3 };
+
 /*
  * The MT8173 thermal controller has four banks. Each bank can read up to
  * four temperature sensors simultaneously. The MT8173 has a total of 5
@@ -507,6 +540,40 @@ static const struct mtk_thermal_data mt2701_thermal_data = {
 };
 
 /*
+ * The MT8365 thermal controller has one bank, which can read up to
+ * four temperature sensors simultaneously. The MT8365 has a total of 3
+ * temperature sensors.
+ *
+ * The thermal core only gets the maximum temperature of this one bank,
+ * so the bank concept wouldn't be necessary here. However, the SVS (Smart
+ * Voltage Scaling) unit makes its decisions based on the same bank
+ * data.
+ */
+static const struct mtk_thermal_data mt8365_thermal_data = {
+	.auxadc_channel = MT8365_TEMP_AUXADC_CHANNEL,
+	.num_banks = MT8365_NUM_BANKS,
+	.num_sensors = MT8365_NUM_SENSORS,
+	.vts_index = mt8365_vts_index,
+	.cali_val = MT8365_CALIBRATION,
+	.num_controller = MT8365_NUM_CONTROLLER,
+	.controller_offset = mt8365_tc_offset,
+	.need_switch_bank = false,
+	.bank_data = {
+		{
+			.num_sensors = MT8365_NUM_SENSORS,
+			.sensors = mt8365_bank_data
+		},
+	},
+	.msr = mt8365_msr,
+	.adcpnp = mt8365_adcpnp,
+	.sensor_mux_values = mt8365_mux_values,
+	.version = MTK_THERMAL_V1,
+	.apmixed_buffer_ctl_reg = APMIXED_SYS_TS_CON0,
+	.apmixed_buffer_ctl_mask = (u32) ~GENMASK(29, 28),
+	.apmixed_buffer_ctl_set = 0,
+};
+
+/*
  * The MT2712 thermal controller has one bank, which can read up to
  * four temperature sensors simultaneously. The MT2712 has a total of 4
  * temperature sensors.
@@ -560,6 +627,9 @@ static const struct mtk_thermal_data mt7622_thermal_data = {
 	.adcpnp = mt7622_adcpnp,
 	.sensor_mux_values = mt7622_mux_values,
 	.version = MTK_THERMAL_V2,
+	.apmixed_buffer_ctl_reg = APMIXED_SYS_TS_CON1,
+	.apmixed_buffer_ctl_mask = GENMASK(31, 6) | BIT(3),
+	.apmixed_buffer_ctl_set = BIT(0),
 };
 
 /*
@@ -746,14 +816,6 @@ static int mtk_thermal_bank_temperature(struct mtk_thermal_bank *bank)
 			mt, conf->bank_data[bank->id].sensors[i], raw);
 
 
-		/*
-		 * The first read of a sensor often contains very high bogus
-		 * temperature value. Filter these out so that the system does
-		 * not immediately shut down.
-		 */
-		if (temp > 200000)
-			temp = 0;
-
 		if (temp > max)
 			max = temp;
 	}
@@ -763,7 +825,7 @@ static int mtk_thermal_bank_temperature(struct mtk_thermal_bank *bank)
 
 static int mtk_read_temp(struct thermal_zone_device *tz, int *temperature)
 {
-	struct mtk_thermal *mt = tz->devdata;
+	struct mtk_thermal *mt = thermal_zone_device_priv(tz);
 	int i;
 	int tempmax = INT_MIN;
 
@@ -1074,19 +1136,27 @@ static const struct of_device_id mtk_thermal_of_match[] = {
 	{
 		.compatible = "mediatek,mt8183-thermal",
 		.data = (void *)&mt8183_thermal_data,
+	},
+	{
+		.compatible = "mediatek,mt8365-thermal",
+		.data = (void *)&mt8365_thermal_data,
 	}, {
 	},
 };
 MODULE_DEVICE_TABLE(of, mtk_thermal_of_match);
 
-static void mtk_thermal_turn_on_buffer(void __iomem *apmixed_base)
+static void mtk_thermal_turn_on_buffer(struct mtk_thermal *mt,
+				       void __iomem *apmixed_base)
 {
-	int tmp;
+	u32 tmp;
 
-	tmp = readl(apmixed_base + APMIXED_SYS_TS_CON1);
-	tmp &= ~(0x37);
-	tmp |= 0x1;
-	writel(tmp, apmixed_base + APMIXED_SYS_TS_CON1);
+	if (!mt->conf->apmixed_buffer_ctl_reg)
+		return;
+
+	tmp = readl(apmixed_base + mt->conf->apmixed_buffer_ctl_reg);
+	tmp &= mt->conf->apmixed_buffer_ctl_mask;
+	tmp |= mt->conf->apmixed_buffer_ctl_set;
+	writel(tmp, apmixed_base + mt->conf->apmixed_buffer_ctl_reg);
 	udelay(200);
 }
 
@@ -1184,10 +1254,10 @@ static int mtk_thermal_probe(struct platform_device *pdev)
 		goto err_disable_clk_auxadc;
 	}
 
-	if (mt->conf->version != MTK_THERMAL_V1) {
-		mtk_thermal_turn_on_buffer(apmixed_base);
+	mtk_thermal_turn_on_buffer(mt, apmixed_base);
+
+	if (mt->conf->version != MTK_THERMAL_V2)
 		mtk_thermal_release_periodic_ts(mt, auxadc_base);
-	}
 
 	if (mt->conf->version == MTK_THERMAL_V1)
 		mt->raw_to_mcelsius = raw_to_mcelsius_v1;
@@ -1203,6 +1273,9 @@ static int mtk_thermal_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, mt);
 
+	/* Delay for thermal banks to be ready */
+	msleep(30);
+
 	tzdev = devm_thermal_of_zone_register(&pdev->dev, 0, mt,
 					      &mtk_thermal_ops);
 	if (IS_ERR(tzdev)) {
@@ -1210,7 +1283,7 @@ static int mtk_thermal_probe(struct platform_device *pdev)
 		goto err_disable_clk_peri_therm;
 	}
 
-	ret = devm_thermal_add_hwmon_sysfs(tzdev);
+	ret = devm_thermal_add_hwmon_sysfs(&pdev->dev, tzdev);
 	if (ret)
 		dev_warn(&pdev->dev, "error in thermal_add_hwmon_sysfs");
 
diff --git a/drivers/thermal/mediatek/lvts_thermal.c b/drivers/thermal/mediatek/lvts_thermal.c
index 84ba65a27acf..d0a3f95b7884 100644
--- a/drivers/thermal/mediatek/lvts_thermal.c
+++ b/drivers/thermal/mediatek/lvts_thermal.c
@@ -66,7 +66,7 @@
 #define LVTS_MONINT_CONF			0x9FBF7BDE
 
 #define LVTS_INT_SENSOR0			0x0009001F
-#define LVTS_INT_SENSOR1			0X000881F0
+#define LVTS_INT_SENSOR1			0x001203E0
 #define LVTS_INT_SENSOR2			0x00247C00
 #define LVTS_INT_SENSOR3			0x1FC00000
 
@@ -252,7 +252,7 @@ static u32 lvts_temp_to_raw(int temperature)
 
 static int lvts_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct lvts_sensor *lvts_sensor = tz->devdata;
+	struct lvts_sensor *lvts_sensor = thermal_zone_device_priv(tz);
 	void __iomem *msr = lvts_sensor->msr;
 	u32 value;
 
@@ -290,7 +290,7 @@ static int lvts_get_temp(struct thermal_zone_device *tz, int *temp)
 
 static int lvts_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct lvts_sensor *lvts_sensor = tz->devdata;
+	struct lvts_sensor *lvts_sensor = thermal_zone_device_priv(tz);
 	void __iomem *base = lvts_sensor->base;
 	u32 raw_low = lvts_temp_to_raw(low);
 	u32 raw_high = lvts_temp_to_raw(high);
@@ -305,7 +305,8 @@ static int lvts_set_trips(struct thermal_zone_device *tz, int low, int high)
 	 * 14-0 : Raw temperature for threshold
 	 */
 	if (low != -INT_MAX) {
-		dev_dbg(&tz->device, "Setting low limit temperature interrupt: %d\n", low);
+		pr_debug("%s: Setting low limit temperature interrupt: %d\n",
+			 thermal_zone_device_type(tz), low);
 		writel(raw_low, LVTS_H2NTHRE(base));
 	}
 
@@ -318,7 +319,8 @@ static int lvts_set_trips(struct thermal_zone_device *tz, int low, int high)
 	 *
 	 * 14-0 : Raw temperature for threshold
 	 */
-	dev_dbg(&tz->device, "Setting high limit temperature interrupt: %d\n", high);
+	pr_debug("%s: Setting high limit temperature interrupt: %d\n",
+		 thermal_zone_device_type(tz), high);
 	writel(raw_high, LVTS_HTHRE(base));
 
 	return 0;
@@ -393,8 +395,8 @@ static irqreturn_t lvts_ctrl_irq_handler(struct lvts_ctrl *lvts_ctrl)
 	 *                  => 0x1FC00000
 	 * sensor 2 interrupt: 0000 0000 0010 0100 0111 1100 0000 0000
 	 *                  => 0x00247C00
-	 * sensor 1 interrupt: 0000 0000 0001 0001 0000 0011 1110 0000
-	 *                  => 0X000881F0
+	 * sensor 1 interrupt: 0000 0000 0001 0010 0000 0011 1110 0000
+	 *                  => 0X001203E0
 	 * sensor 0 interrupt: 0000 0000 0000 1001 0000 0000 0001 1111
 	 *                  => 0x0009001F
 	 */
@@ -528,29 +530,33 @@ static int lvts_sensor_init(struct device *dev, struct lvts_ctrl *lvts_ctrl,
  * The efuse blob values follows the sensor enumeration per thermal
  * controller. The decoding of the stream is as follow:
  *
- *                        <--?-> <----big0 ???---> <-sensor0-> <-0->
- *                        ------------------------------------------
- * index in the stream: : | 0x0 | 0x1 | 0x2 | 0x3 | 0x4 | 0x5 | 0x6 |
- *                        ------------------------------------------
+ * stream index map for MCU Domain :
  *
- *                        <--sensor1--><-0-> <----big1 ???---> <-sen
- *                        ------------------------------------------
- *                        | 0x7 | 0x8 | 0x9 | 0xA | 0xB | OxC | OxD |
- *                        ------------------------------------------
+ * <-----mcu-tc#0-----> <-----sensor#0-----> <-----sensor#1----->
+ *  0x01 | 0x02 | 0x03 | 0x04 | 0x05 | 0x06 | 0x07 | 0x08 | 0x09
  *
- *                        sor0-> <-0-> <-sensor1-> <-0-> ..........
- *                        ------------------------------------------
- *                        | 0x7 | 0x8 | 0x9 | 0xA | 0xB | OxC | OxD |
- *                        ------------------------------------------
+ * <-----mcu-tc#1-----> <-----sensor#2-----> <-----sensor#3----->
+ *  0x0A | 0x0B | 0x0C | 0x0D | 0x0E | 0x0F | 0x10 | 0x11 | 0x12
  *
- * And so on ...
+ * <-----mcu-tc#2-----> <-----sensor#4-----> <-----sensor#5-----> <-----sensor#6-----> <-----sensor#7----->
+ *  0x13 | 0x14 | 0x15 | 0x16 | 0x17 | 0x18 | 0x19 | 0x1A | 0x1B | 0x1C | 0x1D | 0x1E | 0x1F | 0x20 | 0x21
+ *
+ * stream index map for AP Domain :
+ *
+ * <-----ap--tc#0-----> <-----sensor#0-----> <-----sensor#1----->
+ *  0x22 | 0x23 | 0x24 | 0x25 | 0x26 | 0x27 | 0x28 | 0x29 | 0x2A
+ *
+ * <-----ap--tc#1-----> <-----sensor#2-----> <-----sensor#3----->
+ *  0x2B | 0x2C | 0x2D | 0x2E | 0x2F | 0x30 | 0x31 | 0x32 | 0x33
+ *
+ * <-----ap--tc#2-----> <-----sensor#4-----> <-----sensor#5-----> <-----sensor#6----->
+ *  0x34 | 0x35 | 0x36 | 0x37 | 0x38 | 0x39 | 0x3A | 0x3B | 0x3C | 0x3D | 0x3E | 0x3F
+ *
+ * <-----ap--tc#3-----> <-----sensor#7-----> <-----sensor#8----->
+ *  0x40 | 0x41 | 0x42 | 0x43 | 0x44 | 0x45 | 0x46 | 0x47 | 0x48
  *
  * The data description gives the offset of the calibration data in
  * this bytes stream for each sensor.
- *
- * Each thermal controller can handle up to 4 sensors max, we don't
- * care if there are less as the array of calibration is sized to 4
- * anyway. The unused sensor slot will be zeroed.
  */
 static int lvts_calibration_init(struct device *dev, struct lvts_ctrl *lvts_ctrl,
 					const struct lvts_ctrl_data *lvts_ctrl_data,
@@ -1163,7 +1169,7 @@ static int lvts_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct lvts_ctrl_data mt8195_lvts_data_ctrl[] = {
+static const struct lvts_ctrl_data mt8195_lvts_mcu_data_ctrl[] = {
 	{
 		.cal_offset = { 0x04, 0x07 },
 		.lvts_sensor = {
@@ -1198,13 +1204,63 @@ static const struct lvts_ctrl_data mt8195_lvts_data_ctrl[] = {
 	}
 };
 
+static const struct lvts_ctrl_data mt8195_lvts_ap_data_ctrl[] = {
+		{
+		.cal_offset = { 0x25, 0x28 },
+		.lvts_sensor = {
+			{ .dt_id = MT8195_AP_VPU0 },
+			{ .dt_id = MT8195_AP_VPU1 }
+		},
+		.num_lvts_sensor = 2,
+		.offset = 0x0,
+		.hw_tshut_temp = LVTS_HW_SHUTDOWN_MT8195,
+	},
+	{
+		.cal_offset = { 0x2e, 0x31 },
+		.lvts_sensor = {
+			{ .dt_id = MT8195_AP_GPU0 },
+			{ .dt_id = MT8195_AP_GPU1 }
+		},
+		.num_lvts_sensor = 2,
+		.offset = 0x100,
+		.hw_tshut_temp = LVTS_HW_SHUTDOWN_MT8195,
+	},
+	{
+		.cal_offset = { 0x37, 0x3a, 0x3d },
+		.lvts_sensor = {
+			{ .dt_id = MT8195_AP_VDEC },
+			{ .dt_id = MT8195_AP_IMG },
+			{ .dt_id = MT8195_AP_INFRA },
+		},
+		.num_lvts_sensor = 3,
+		.offset = 0x200,
+		.hw_tshut_temp = LVTS_HW_SHUTDOWN_MT8195,
+	},
+	{
+		.cal_offset = { 0x43, 0x46 },
+		.lvts_sensor = {
+			{ .dt_id = MT8195_AP_CAM0 },
+			{ .dt_id = MT8195_AP_CAM1 }
+		},
+		.num_lvts_sensor = 2,
+		.offset = 0x300,
+		.hw_tshut_temp = LVTS_HW_SHUTDOWN_MT8195,
+	}
+};
+
 static const struct lvts_data mt8195_lvts_mcu_data = {
-	.lvts_ctrl	= mt8195_lvts_data_ctrl,
-	.num_lvts_ctrl	= ARRAY_SIZE(mt8195_lvts_data_ctrl),
+	.lvts_ctrl	= mt8195_lvts_mcu_data_ctrl,
+	.num_lvts_ctrl	= ARRAY_SIZE(mt8195_lvts_mcu_data_ctrl),
+};
+
+static const struct lvts_data mt8195_lvts_ap_data = {
+	.lvts_ctrl	= mt8195_lvts_ap_data_ctrl,
+	.num_lvts_ctrl	= ARRAY_SIZE(mt8195_lvts_ap_data_ctrl),
 };
 
 static const struct of_device_id lvts_of_match[] = {
 	{ .compatible = "mediatek,mt8195-lvts-mcu", .data = &mt8195_lvts_mcu_data },
+	{ .compatible = "mediatek,mt8195-lvts-ap", .data = &mt8195_lvts_ap_data },
 	{},
 };
 MODULE_DEVICE_TABLE(of, lvts_of_match);
diff --git a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
index 31164ade2dd1..5749149ae2e4 100644
--- a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
+++ b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
@@ -360,7 +360,7 @@ static irqreturn_t adc_tm5_gen2_isr(int irq, void *data)
 
 static int adc_tm5_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct adc_tm5_channel *channel = tz->devdata;
+	struct adc_tm5_channel *channel = thermal_zone_device_priv(tz);
 	int ret;
 
 	if (!channel || !channel->iio)
@@ -642,7 +642,7 @@ config_fail:
 
 static int adc_tm5_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct adc_tm5_channel *channel = tz->devdata;
+	struct adc_tm5_channel *channel = thermal_zone_device_priv(tz);
 	struct adc_tm5_chip *chip;
 	int ret;
 
@@ -689,7 +689,7 @@ static int adc_tm5_register_tzd(struct adc_tm5_chip *adc_tm)
 			return PTR_ERR(tzd);
 		}
 		adc_tm->channels[i].tzd = tzd;
-		if (devm_thermal_add_hwmon_sysfs(tzd))
+		if (devm_thermal_add_hwmon_sysfs(adc_tm->dev, tzd))
 			dev_warn(adc_tm->dev,
 				 "Failed to add hwmon sysfs attributes\n");
 	}
diff --git a/drivers/thermal/qcom/qcom-spmi-temp-alarm.c b/drivers/thermal/qcom/qcom-spmi-temp-alarm.c
index 101c75d0e13f..0f88e98428ac 100644
--- a/drivers/thermal/qcom/qcom-spmi-temp-alarm.c
+++ b/drivers/thermal/qcom/qcom-spmi-temp-alarm.c
@@ -187,7 +187,7 @@ static int qpnp_tm_update_temp_no_adc(struct qpnp_tm_chip *chip)
 
 static int qpnp_tm_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct qpnp_tm_chip *chip = tz->devdata;
+	struct qpnp_tm_chip *chip = thermal_zone_device_priv(tz);
 	int ret, mili_celsius;
 
 	if (!temp)
@@ -265,7 +265,7 @@ skip:
 
 static int qpnp_tm_set_trip_temp(struct thermal_zone_device *tz, int trip_id, int temp)
 {
-	struct qpnp_tm_chip *chip = tz->devdata;
+	struct qpnp_tm_chip *chip = thermal_zone_device_priv(tz);
 	struct thermal_trip trip;
 	int ret;
 
@@ -459,7 +459,7 @@ static int qpnp_tm_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	if (devm_thermal_add_hwmon_sysfs(chip->tz_dev))
+	if (devm_thermal_add_hwmon_sysfs(&pdev->dev, chip->tz_dev))
 		dev_warn(&pdev->dev,
 			 "Failed to add hwmon sysfs attributes\n");
 
diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c
index 8020ead2794e..d3218127e617 100644
--- a/drivers/thermal/qcom/tsens.c
+++ b/drivers/thermal/qcom/tsens.c
@@ -673,7 +673,7 @@ static irqreturn_t tsens_combined_irq_thread(int irq, void *data)
 
 static int tsens_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct tsens_sensor *s = tz->devdata;
+	struct tsens_sensor *s = thermal_zone_device_priv(tz);
 	struct tsens_priv *priv = s->priv;
 	struct device *dev = priv->dev;
 	struct tsens_irq_data d;
@@ -1057,7 +1057,7 @@ err_put_device:
 
 static int tsens_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct tsens_sensor *s = tz->devdata;
+	struct tsens_sensor *s = thermal_zone_device_priv(tz);
 	struct tsens_priv *priv = s->priv;
 
 	return priv->ops->get_temp(s, temp);
@@ -1189,7 +1189,7 @@ static int tsens_register(struct tsens_priv *priv)
 		if (priv->ops->enable)
 			priv->ops->enable(priv, i);
 
-		if (devm_thermal_add_hwmon_sysfs(tzd))
+		if (devm_thermal_add_hwmon_sysfs(priv->dev, tzd))
 			dev_warn(priv->dev,
 				 "Failed to add hwmon sysfs attributes\n");
 	}
diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c
index 431c29c0898a..e58756323457 100644
--- a/drivers/thermal/qoriq_thermal.c
+++ b/drivers/thermal/qoriq_thermal.c
@@ -83,7 +83,7 @@ static struct qoriq_tmu_data *qoriq_sensor_to_data(struct qoriq_sensor *s)
 
 static int tmu_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct qoriq_sensor *qsensor = tz->devdata;
+	struct qoriq_sensor *qsensor = thermal_zone_device_priv(tz);
 	struct qoriq_tmu_data *qdata = qoriq_sensor_to_data(qsensor);
 	u32 val;
 	/*
@@ -157,7 +157,7 @@ static int qoriq_tmu_register_tmu_zone(struct device *dev,
 			return ret;
 		}
 
-		if (devm_thermal_add_hwmon_sysfs(tzd))
+		if (devm_thermal_add_hwmon_sysfs(dev, tzd))
 			dev_warn(dev,
 				 "Failed to add hwmon sysfs attributes\n");
 
diff --git a/drivers/thermal/rcar_gen3_thermal.c b/drivers/thermal/rcar_gen3_thermal.c
index d6b5b59c5c53..42a4724d3920 100644
--- a/drivers/thermal/rcar_gen3_thermal.c
+++ b/drivers/thermal/rcar_gen3_thermal.c
@@ -14,7 +14,6 @@
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
-#include <linux/sys_soc.h>
 #include <linux/thermal.h>
 
 #include "thermal_hwmon.h"
@@ -27,7 +26,6 @@
 #define REG_GEN3_IRQTEMP1	0x14
 #define REG_GEN3_IRQTEMP2	0x18
 #define REG_GEN3_IRQTEMP3	0x1C
-#define REG_GEN3_CTSR		0x20
 #define REG_GEN3_THCTR		0x20
 #define REG_GEN3_TEMP		0x28
 #define REG_GEN3_THCODE1	0x50
@@ -46,14 +44,6 @@
 #define IRQ_TEMPD2		BIT(4)
 #define IRQ_TEMPD3		BIT(5)
 
-/* CTSR bits */
-#define CTSR_PONM	BIT(8)
-#define CTSR_AOUT	BIT(7)
-#define CTSR_THBGR	BIT(5)
-#define CTSR_VMEN	BIT(4)
-#define CTSR_VMST	BIT(1)
-#define CTSR_THSST	BIT(0)
-
 /* THCTR bits */
 #define THCTR_PONM	BIT(6)
 #define THCTR_THSST	BIT(0)
@@ -88,8 +78,6 @@ struct rcar_gen3_thermal_priv {
 	struct rcar_gen3_thermal_tsc *tscs[TSC_MAX_NUM];
 	struct thermal_zone_device_ops ops;
 	unsigned int num_tscs;
-	void (*thermal_init)(struct rcar_gen3_thermal_priv *priv,
-			     struct rcar_gen3_thermal_tsc *tsc);
 	int ptat[3];
 };
 
@@ -167,7 +155,7 @@ static int rcar_gen3_thermal_round(int temp)
 
 static int rcar_gen3_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct rcar_gen3_thermal_tsc *tsc = tz->devdata;
+	struct rcar_gen3_thermal_tsc *tsc = thermal_zone_device_priv(tz);
 	int mcelsius, val;
 	int reg;
 
@@ -206,7 +194,7 @@ static int rcar_gen3_thermal_mcelsius_to_temp(struct rcar_gen3_thermal_tsc *tsc,
 
 static int rcar_gen3_thermal_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct rcar_gen3_thermal_tsc *tsc = tz->devdata;
+	struct rcar_gen3_thermal_tsc *tsc = thermal_zone_device_priv(tz);
 	u32 irqmsk = 0;
 
 	if (low != -INT_MAX) {
@@ -248,11 +236,6 @@ static irqreturn_t rcar_gen3_thermal_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static const struct soc_device_attribute r8a7795es1[] = {
-	{ .soc_id = "r8a7795", .revision = "ES1.*" },
-	{ /* sentinel */ }
-};
-
 static bool rcar_gen3_thermal_read_fuses(struct rcar_gen3_thermal_priv *priv)
 {
 	unsigned int i;
@@ -311,34 +294,6 @@ static bool rcar_gen3_thermal_read_fuses(struct rcar_gen3_thermal_priv *priv)
 	return true;
 }
 
-static void rcar_gen3_thermal_init_r8a7795es1(struct rcar_gen3_thermal_priv *priv,
-					      struct rcar_gen3_thermal_tsc *tsc)
-{
-	rcar_gen3_thermal_write(tsc, REG_GEN3_CTSR,  CTSR_THBGR);
-	rcar_gen3_thermal_write(tsc, REG_GEN3_CTSR,  0x0);
-
-	usleep_range(1000, 2000);
-
-	rcar_gen3_thermal_write(tsc, REG_GEN3_CTSR, CTSR_PONM);
-
-	rcar_gen3_thermal_write(tsc, REG_GEN3_IRQCTL, 0x3F);
-	rcar_gen3_thermal_write(tsc, REG_GEN3_IRQMSK, 0);
-	if (priv->ops.set_trips)
-		rcar_gen3_thermal_write(tsc, REG_GEN3_IRQEN,
-					IRQ_TEMPD1 | IRQ_TEMP2);
-
-	rcar_gen3_thermal_write(tsc, REG_GEN3_CTSR,
-				CTSR_PONM | CTSR_AOUT | CTSR_THBGR | CTSR_VMEN);
-
-	usleep_range(100, 200);
-
-	rcar_gen3_thermal_write(tsc, REG_GEN3_CTSR,
-				CTSR_PONM | CTSR_AOUT | CTSR_THBGR | CTSR_VMEN |
-				CTSR_VMST | CTSR_THSST);
-
-	usleep_range(1000, 2000);
-}
-
 static void rcar_gen3_thermal_init(struct rcar_gen3_thermal_priv *priv,
 				   struct rcar_gen3_thermal_tsc *tsc)
 {
@@ -474,9 +429,6 @@ static int rcar_gen3_thermal_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	priv->ops = rcar_gen3_tz_of_ops;
-	priv->thermal_init = rcar_gen3_thermal_init;
-	if (soc_device_match(r8a7795es1))
-		priv->thermal_init = rcar_gen3_thermal_init_r8a7795es1;
 
 	platform_set_drvdata(pdev, priv);
 
@@ -516,7 +468,7 @@ static int rcar_gen3_thermal_probe(struct platform_device *pdev)
 	for (i = 0; i < priv->num_tscs; i++) {
 		struct rcar_gen3_thermal_tsc *tsc = priv->tscs[i];
 
-		priv->thermal_init(priv, tsc);
+		rcar_gen3_thermal_init(priv, tsc);
 		rcar_gen3_thermal_calc_coefs(priv, tsc, *ths_tj_1);
 
 		zone = devm_thermal_of_zone_register(dev, i, tsc, &priv->ops);
@@ -527,7 +479,6 @@ static int rcar_gen3_thermal_probe(struct platform_device *pdev)
 		}
 		tsc->zone = zone;
 
-		tsc->zone->tzp->no_hwmon = false;
 		ret = thermal_add_hwmon_sysfs(tsc->zone);
 		if (ret)
 			goto error_unregister;
@@ -564,7 +515,7 @@ static int __maybe_unused rcar_gen3_thermal_resume(struct device *dev)
 	for (i = 0; i < priv->num_tscs; i++) {
 		struct rcar_gen3_thermal_tsc *tsc = priv->tscs[i];
 
-		priv->thermal_init(priv, tsc);
+		rcar_gen3_thermal_init(priv, tsc);
 	}
 
 	return 0;
diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c
index 436f5f9cf729..b8571f7090aa 100644
--- a/drivers/thermal/rcar_thermal.c
+++ b/drivers/thermal/rcar_thermal.c
@@ -101,7 +101,6 @@ struct rcar_thermal_priv {
 	list_for_each_entry(pos, &common->head, list)
 
 #define MCELSIUS(temp)			((temp) * 1000)
-#define rcar_zone_to_priv(zone)		((zone)->devdata)
 #define rcar_priv_to_dev(priv)		((priv)->common->dev)
 #define rcar_has_irq_support(priv)	((priv)->common->base)
 #define rcar_id_to_shift(priv)		((priv)->id * 8)
@@ -273,7 +272,7 @@ static int rcar_thermal_get_current_temp(struct rcar_thermal_priv *priv,
 
 static int rcar_thermal_get_temp(struct thermal_zone_device *zone, int *temp)
 {
-	struct rcar_thermal_priv *priv = rcar_zone_to_priv(zone);
+	struct rcar_thermal_priv *priv = thermal_zone_device_priv(zone);
 
 	return rcar_thermal_get_current_temp(priv, temp);
 }
@@ -510,11 +509,6 @@ static int rcar_thermal_probe(struct platform_device *pdev)
 		}
 
 		if (chip->use_of_thermal) {
-			/*
-			 * thermal_zone doesn't enable hwmon as default,
-			 * but, enable it here to keep compatible
-			 */
-			priv->zone->tzp->no_hwmon = false;
 			ret = thermal_add_hwmon_sysfs(priv->zone);
 			if (ret)
 				goto error_unregister;
diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c
index 4b7c43f34d1a..77231a9d28ff 100644
--- a/drivers/thermal/rockchip_thermal.c
+++ b/drivers/thermal/rockchip_thermal.c
@@ -40,15 +40,6 @@ enum tshut_polarity {
 };
 
 /*
- * The system has two Temperature Sensors.
- * sensor0 is for CPU, and sensor1 is for GPU.
- */
-enum sensor_id {
-	SENSOR_CPU = 0,
-	SENSOR_GPU,
-};
-
-/*
  * The conversion table has the adc value and temperature.
  * ADC_DECREMENT: the adc value is of diminishing.(e.g. rk3288_code_table)
  * ADC_INCREMENT: the adc value is incremental.(e.g. rk3368_code_table)
@@ -60,12 +51,6 @@ enum adc_sort_mode {
 
 #include "thermal_hwmon.h"
 
-/*
- * The max sensors is two in rockchip SoCs.
- * Two sensors: CPU and GPU sensor.
- */
-#define SOC_MAX_SENSORS	2
-
 /**
  * struct chip_tsadc_table - hold information about chip-specific differences
  * @id: conversion table
@@ -82,7 +67,7 @@ struct chip_tsadc_table {
 
 /**
  * struct rockchip_tsadc_chip - hold the private data of tsadc chip
- * @chn_id: array of sensor ids of chip corresponding to the channel
+ * @chn_offset: the channel offset of the first channel
  * @chn_num: the channel number of tsadc chip
  * @tshut_temp: the hardware-controlled shutdown temperature value
  * @tshut_mode: the hardware-controlled shutdown mode (0:CRU 1:GPIO)
@@ -98,7 +83,7 @@ struct chip_tsadc_table {
  */
 struct rockchip_tsadc_chip {
 	/* The sensor id of chip correspond to the ADC channel */
-	int chn_id[SOC_MAX_SENSORS];
+	int chn_offset;
 	int chn_num;
 
 	/* The hardware-controlled tshut property */
@@ -156,7 +141,7 @@ struct rockchip_thermal_data {
 	struct platform_device *pdev;
 	struct reset_control *reset;
 
-	struct rockchip_thermal_sensor sensors[SOC_MAX_SENSORS];
+	struct rockchip_thermal_sensor *sensors;
 
 	struct clk *clk;
 	struct clk *pclk;
@@ -180,29 +165,49 @@ struct rockchip_thermal_data {
 #define TSADCV2_AUTO_CON			0x04
 #define TSADCV2_INT_EN				0x08
 #define TSADCV2_INT_PD				0x0c
+#define TSADCV3_AUTO_SRC_CON			0x0c
+#define TSADCV3_HT_INT_EN			0x14
+#define TSADCV3_HSHUT_GPIO_INT_EN		0x18
+#define TSADCV3_HSHUT_CRU_INT_EN		0x1c
+#define TSADCV3_INT_PD				0x24
+#define TSADCV3_HSHUT_PD			0x28
 #define TSADCV2_DATA(chn)			(0x20 + (chn) * 0x04)
 #define TSADCV2_COMP_INT(chn)		        (0x30 + (chn) * 0x04)
 #define TSADCV2_COMP_SHUT(chn)		        (0x40 + (chn) * 0x04)
+#define TSADCV3_DATA(chn)			(0x2c + (chn) * 0x04)
+#define TSADCV3_COMP_INT(chn)		        (0x6c + (chn) * 0x04)
+#define TSADCV3_COMP_SHUT(chn)		        (0x10c + (chn) * 0x04)
 #define TSADCV2_HIGHT_INT_DEBOUNCE		0x60
 #define TSADCV2_HIGHT_TSHUT_DEBOUNCE		0x64
+#define TSADCV3_HIGHT_INT_DEBOUNCE		0x14c
+#define TSADCV3_HIGHT_TSHUT_DEBOUNCE		0x150
 #define TSADCV2_AUTO_PERIOD			0x68
 #define TSADCV2_AUTO_PERIOD_HT			0x6c
+#define TSADCV3_AUTO_PERIOD			0x154
+#define TSADCV3_AUTO_PERIOD_HT			0x158
 
 #define TSADCV2_AUTO_EN				BIT(0)
+#define TSADCV2_AUTO_EN_MASK			BIT(16)
 #define TSADCV2_AUTO_SRC_EN(chn)		BIT(4 + (chn))
+#define TSADCV3_AUTO_SRC_EN(chn)		BIT(chn)
+#define TSADCV3_AUTO_SRC_EN_MASK(chn)		BIT(16 + chn)
 #define TSADCV2_AUTO_TSHUT_POLARITY_HIGH	BIT(8)
+#define TSADCV2_AUTO_TSHUT_POLARITY_MASK	BIT(24)
 
 #define TSADCV3_AUTO_Q_SEL_EN			BIT(1)
 
 #define TSADCV2_INT_SRC_EN(chn)			BIT(chn)
+#define TSADCV2_INT_SRC_EN_MASK(chn)		BIT(16 + (chn))
 #define TSADCV2_SHUT_2GPIO_SRC_EN(chn)		BIT(4 + (chn))
 #define TSADCV2_SHUT_2CRU_SRC_EN(chn)		BIT(8 + (chn))
 
 #define TSADCV2_INT_PD_CLEAR_MASK		~BIT(8)
 #define TSADCV3_INT_PD_CLEAR_MASK		~BIT(16)
+#define TSADCV4_INT_PD_CLEAR_MASK		0xffffffff
 
 #define TSADCV2_DATA_MASK			0xfff
 #define TSADCV3_DATA_MASK			0x3ff
+#define TSADCV4_DATA_MASK			0x1ff
 
 #define TSADCV2_HIGHT_INT_DEBOUNCE_COUNT	4
 #define TSADCV2_HIGHT_TSHUT_DEBOUNCE_COUNT	4
@@ -213,6 +218,8 @@ struct rockchip_thermal_data {
 
 #define TSADCV5_AUTO_PERIOD_TIME		1622 /* 2.5ms */
 #define TSADCV5_AUTO_PERIOD_HT_TIME		1622 /* 2.5ms */
+#define TSADCV6_AUTO_PERIOD_TIME		5000 /* 2.5ms */
+#define TSADCV6_AUTO_PERIOD_HT_TIME		5000 /* 2.5ms */
 
 #define TSADCV2_USER_INTER_PD_SOC		0x340 /* 13 clocks */
 #define TSADCV5_USER_INTER_PD_SOC		0xfc0 /* 97us, at least 90us */
@@ -229,6 +236,12 @@ struct rockchip_thermal_data {
 #define RK3568_GRF_TSADC_ANA_REG2		(0x10001 << 2)
 #define RK3568_GRF_TSADC_TSEN			(0x10001 << 8)
 
+#define RK3588_GRF0_TSADC_CON			0x0100
+
+#define RK3588_GRF0_TSADC_TRM			(0xff0077 << 0)
+#define RK3588_GRF0_TSADC_SHUT_2CRU		(0x30003 << 10)
+#define RK3588_GRF0_TSADC_SHUT_2GPIO		(0x70007 << 12)
+
 #define GRF_SARADC_TESTBIT_ON			(0x10001 << 2)
 #define GRF_TSADC_TESTBIT_H_ON			(0x10001 << 2)
 #define GRF_TSADC_VCM_EN_L			(0x10001 << 7)
@@ -523,6 +536,15 @@ static const struct tsadc_table rk3568_code_table[] = {
 	{TSADCV2_DATA_MASK, 125000},
 };
 
+static const struct tsadc_table rk3588_code_table[] = {
+	{0, -40000},
+	{215, -40000},
+	{285, 25000},
+	{350, 85000},
+	{395, 125000},
+	{TSADCV4_DATA_MASK, 125000},
+};
+
 static u32 rk_tsadcv2_temp_to_code(const struct chip_tsadc_table *table,
 				   int temp)
 {
@@ -793,6 +815,25 @@ static void rk_tsadcv7_initialize(struct regmap *grf, void __iomem *regs,
 	}
 }
 
+static void rk_tsadcv8_initialize(struct regmap *grf, void __iomem *regs,
+				  enum tshut_polarity tshut_polarity)
+{
+	writel_relaxed(TSADCV6_AUTO_PERIOD_TIME, regs + TSADCV3_AUTO_PERIOD);
+	writel_relaxed(TSADCV6_AUTO_PERIOD_HT_TIME,
+		       regs + TSADCV3_AUTO_PERIOD_HT);
+	writel_relaxed(TSADCV2_HIGHT_INT_DEBOUNCE_COUNT,
+		       regs + TSADCV3_HIGHT_INT_DEBOUNCE);
+	writel_relaxed(TSADCV2_HIGHT_TSHUT_DEBOUNCE_COUNT,
+		       regs + TSADCV3_HIGHT_TSHUT_DEBOUNCE);
+	if (tshut_polarity == TSHUT_HIGH_ACTIVE)
+		writel_relaxed(TSADCV2_AUTO_TSHUT_POLARITY_HIGH |
+			       TSADCV2_AUTO_TSHUT_POLARITY_MASK,
+			       regs + TSADCV2_AUTO_CON);
+	else
+		writel_relaxed(TSADCV2_AUTO_TSHUT_POLARITY_MASK,
+			       regs + TSADCV2_AUTO_CON);
+}
+
 static void rk_tsadcv2_irq_ack(void __iomem *regs)
 {
 	u32 val;
@@ -809,6 +850,17 @@ static void rk_tsadcv3_irq_ack(void __iomem *regs)
 	writel_relaxed(val & TSADCV3_INT_PD_CLEAR_MASK, regs + TSADCV2_INT_PD);
 }
 
+static void rk_tsadcv4_irq_ack(void __iomem *regs)
+{
+	u32 val;
+
+	val = readl_relaxed(regs + TSADCV3_INT_PD);
+	writel_relaxed(val & TSADCV4_INT_PD_CLEAR_MASK, regs + TSADCV3_INT_PD);
+	val = readl_relaxed(regs + TSADCV3_HSHUT_PD);
+	writel_relaxed(val & TSADCV3_INT_PD_CLEAR_MASK,
+		       regs + TSADCV3_HSHUT_PD);
+}
+
 static void rk_tsadcv2_control(void __iomem *regs, bool enable)
 {
 	u32 val;
@@ -844,6 +896,18 @@ static void rk_tsadcv3_control(void __iomem *regs, bool enable)
 	writel_relaxed(val, regs + TSADCV2_AUTO_CON);
 }
 
+static void rk_tsadcv4_control(void __iomem *regs, bool enable)
+{
+	u32 val;
+
+	if (enable)
+		val = TSADCV2_AUTO_EN | TSADCV2_AUTO_EN_MASK;
+	else
+		val = TSADCV2_AUTO_EN_MASK;
+
+	writel_relaxed(val, regs + TSADCV2_AUTO_CON);
+}
+
 static int rk_tsadcv2_get_temp(const struct chip_tsadc_table *table,
 			       int chn, void __iomem *regs, int *temp)
 {
@@ -854,6 +918,16 @@ static int rk_tsadcv2_get_temp(const struct chip_tsadc_table *table,
 	return rk_tsadcv2_code_to_temp(table, val, temp);
 }
 
+static int rk_tsadcv4_get_temp(const struct chip_tsadc_table *table,
+			       int chn, void __iomem *regs, int *temp)
+{
+	u32 val;
+
+	val = readl_relaxed(regs + TSADCV3_DATA(chn));
+
+	return rk_tsadcv2_code_to_temp(table, val, temp);
+}
+
 static int rk_tsadcv2_alarm_temp(const struct chip_tsadc_table *table,
 				 int chn, void __iomem *regs, int temp)
 {
@@ -888,6 +962,33 @@ static int rk_tsadcv2_alarm_temp(const struct chip_tsadc_table *table,
 	return 0;
 }
 
+static int rk_tsadcv3_alarm_temp(const struct chip_tsadc_table *table,
+				 int chn, void __iomem *regs, int temp)
+{
+	u32 alarm_value;
+
+	/*
+	 * In some cases, some sensors didn't need the trip points, the
+	 * set_trips will pass {-INT_MAX, INT_MAX} to trigger tsadc alarm
+	 * in the end, ignore this case and disable the high temperature
+	 * interrupt.
+	 */
+	if (temp == INT_MAX) {
+		writel_relaxed(TSADCV2_INT_SRC_EN_MASK(chn),
+			       regs + TSADCV3_HT_INT_EN);
+		return 0;
+	}
+	/* Make sure the value is valid */
+	alarm_value = rk_tsadcv2_temp_to_code(table, temp);
+	if (alarm_value == table->data_mask)
+		return -ERANGE;
+	writel_relaxed(alarm_value & table->data_mask,
+		       regs + TSADCV3_COMP_INT(chn));
+	writel_relaxed(TSADCV2_INT_SRC_EN(chn) | TSADCV2_INT_SRC_EN_MASK(chn),
+		       regs + TSADCV3_HT_INT_EN);
+	return 0;
+}
+
 static int rk_tsadcv2_tshut_temp(const struct chip_tsadc_table *table,
 				 int chn, void __iomem *regs, int temp)
 {
@@ -907,6 +1008,25 @@ static int rk_tsadcv2_tshut_temp(const struct chip_tsadc_table *table,
 	return 0;
 }
 
+static int rk_tsadcv3_tshut_temp(const struct chip_tsadc_table *table,
+				 int chn, void __iomem *regs, int temp)
+{
+	u32 tshut_value;
+
+	/* Make sure the value is valid */
+	tshut_value = rk_tsadcv2_temp_to_code(table, temp);
+	if (tshut_value == table->data_mask)
+		return -ERANGE;
+
+	writel_relaxed(tshut_value, regs + TSADCV3_COMP_SHUT(chn));
+
+	/* TSHUT will be valid */
+	writel_relaxed(TSADCV3_AUTO_SRC_EN(chn) | TSADCV3_AUTO_SRC_EN_MASK(chn),
+		       regs + TSADCV3_AUTO_SRC_CON);
+
+	return 0;
+}
+
 static void rk_tsadcv2_tshut_mode(int chn, void __iomem *regs,
 				  enum tshut_mode mode)
 {
@@ -924,9 +1044,25 @@ static void rk_tsadcv2_tshut_mode(int chn, void __iomem *regs,
 	writel_relaxed(val, regs + TSADCV2_INT_EN);
 }
 
+static void rk_tsadcv3_tshut_mode(int chn, void __iomem *regs,
+				  enum tshut_mode mode)
+{
+	u32 val_gpio, val_cru;
+
+	if (mode == TSHUT_MODE_GPIO) {
+		val_gpio = TSADCV2_INT_SRC_EN(chn) | TSADCV2_INT_SRC_EN_MASK(chn);
+		val_cru = TSADCV2_INT_SRC_EN_MASK(chn);
+	} else {
+		val_cru = TSADCV2_INT_SRC_EN(chn) | TSADCV2_INT_SRC_EN_MASK(chn);
+		val_gpio = TSADCV2_INT_SRC_EN_MASK(chn);
+	}
+	writel_relaxed(val_gpio, regs + TSADCV3_HSHUT_GPIO_INT_EN);
+	writel_relaxed(val_cru, regs + TSADCV3_HSHUT_CRU_INT_EN);
+}
+
 static const struct rockchip_tsadc_chip px30_tsadc_data = {
-	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
-	.chn_id[SENSOR_GPU] = 1, /* gpu sensor is channel 1 */
+	/* cpu, gpu */
+	.chn_offset = 0,
 	.chn_num = 2, /* 2 channels for tsadc */
 
 	.tshut_mode = TSHUT_MODE_CRU, /* default TSHUT via CRU */
@@ -949,7 +1085,8 @@ static const struct rockchip_tsadc_chip px30_tsadc_data = {
 };
 
 static const struct rockchip_tsadc_chip rv1108_tsadc_data = {
-	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
+	/* cpu */
+	.chn_offset = 0,
 	.chn_num = 1, /* one channel for tsadc */
 
 	.tshut_mode = TSHUT_MODE_GPIO, /* default TSHUT via GPIO give PMIC */
@@ -973,7 +1110,8 @@ static const struct rockchip_tsadc_chip rv1108_tsadc_data = {
 };
 
 static const struct rockchip_tsadc_chip rk3228_tsadc_data = {
-	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
+	/* cpu */
+	.chn_offset = 0,
 	.chn_num = 1, /* one channel for tsadc */
 
 	.tshut_mode = TSHUT_MODE_GPIO, /* default TSHUT via GPIO give PMIC */
@@ -997,8 +1135,8 @@ static const struct rockchip_tsadc_chip rk3228_tsadc_data = {
 };
 
 static const struct rockchip_tsadc_chip rk3288_tsadc_data = {
-	.chn_id[SENSOR_CPU] = 1, /* cpu sensor is channel 1 */
-	.chn_id[SENSOR_GPU] = 2, /* gpu sensor is channel 2 */
+	/* cpu, gpu */
+	.chn_offset = 1,
 	.chn_num = 2, /* two channels for tsadc */
 
 	.tshut_mode = TSHUT_MODE_GPIO, /* default TSHUT via GPIO give PMIC */
@@ -1022,7 +1160,8 @@ static const struct rockchip_tsadc_chip rk3288_tsadc_data = {
 };
 
 static const struct rockchip_tsadc_chip rk3328_tsadc_data = {
-	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
+	/* cpu */
+	.chn_offset = 0,
 	.chn_num = 1, /* one channels for tsadc */
 
 	.tshut_mode = TSHUT_MODE_CRU, /* default TSHUT via CRU */
@@ -1045,8 +1184,8 @@ static const struct rockchip_tsadc_chip rk3328_tsadc_data = {
 };
 
 static const struct rockchip_tsadc_chip rk3366_tsadc_data = {
-	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
-	.chn_id[SENSOR_GPU] = 1, /* gpu sensor is channel 1 */
+	/* cpu, gpu */
+	.chn_offset = 0,
 	.chn_num = 2, /* two channels for tsadc */
 
 	.tshut_mode = TSHUT_MODE_GPIO, /* default TSHUT via GPIO give PMIC */
@@ -1070,8 +1209,8 @@ static const struct rockchip_tsadc_chip rk3366_tsadc_data = {
 };
 
 static const struct rockchip_tsadc_chip rk3368_tsadc_data = {
-	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
-	.chn_id[SENSOR_GPU] = 1, /* gpu sensor is channel 1 */
+	/* cpu, gpu */
+	.chn_offset = 0,
 	.chn_num = 2, /* two channels for tsadc */
 
 	.tshut_mode = TSHUT_MODE_GPIO, /* default TSHUT via GPIO give PMIC */
@@ -1095,8 +1234,8 @@ static const struct rockchip_tsadc_chip rk3368_tsadc_data = {
 };
 
 static const struct rockchip_tsadc_chip rk3399_tsadc_data = {
-	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
-	.chn_id[SENSOR_GPU] = 1, /* gpu sensor is channel 1 */
+	/* cpu, gpu */
+	.chn_offset = 0,
 	.chn_num = 2, /* two channels for tsadc */
 
 	.tshut_mode = TSHUT_MODE_GPIO, /* default TSHUT via GPIO give PMIC */
@@ -1120,8 +1259,8 @@ static const struct rockchip_tsadc_chip rk3399_tsadc_data = {
 };
 
 static const struct rockchip_tsadc_chip rk3568_tsadc_data = {
-	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
-	.chn_id[SENSOR_GPU] = 1, /* gpu sensor is channel 1 */
+	/* cpu, gpu */
+	.chn_offset = 0,
 	.chn_num = 2, /* two channels for tsadc */
 
 	.tshut_mode = TSHUT_MODE_GPIO, /* default TSHUT via GPIO give PMIC */
@@ -1144,6 +1283,28 @@ static const struct rockchip_tsadc_chip rk3568_tsadc_data = {
 	},
 };
 
+static const struct rockchip_tsadc_chip rk3588_tsadc_data = {
+	/* top, big_core0, big_core1, little_core, center, gpu, npu */
+	.chn_offset = 0,
+	.chn_num = 7, /* seven channels for tsadc */
+	.tshut_mode = TSHUT_MODE_GPIO, /* default TSHUT via GPIO give PMIC */
+	.tshut_polarity = TSHUT_LOW_ACTIVE, /* default TSHUT LOW ACTIVE */
+	.tshut_temp = 95000,
+	.initialize = rk_tsadcv8_initialize,
+	.irq_ack = rk_tsadcv4_irq_ack,
+	.control = rk_tsadcv4_control,
+	.get_temp = rk_tsadcv4_get_temp,
+	.set_alarm_temp = rk_tsadcv3_alarm_temp,
+	.set_tshut_temp = rk_tsadcv3_tshut_temp,
+	.set_tshut_mode = rk_tsadcv3_tshut_mode,
+	.table = {
+		.id = rk3588_code_table,
+		.length = ARRAY_SIZE(rk3588_code_table),
+		.data_mask = TSADCV4_DATA_MASK,
+		.mode = ADC_INCREMENT,
+	},
+};
+
 static const struct of_device_id of_rockchip_thermal_match[] = {
 	{	.compatible = "rockchip,px30-tsadc",
 		.data = (void *)&px30_tsadc_data,
@@ -1180,6 +1341,10 @@ static const struct of_device_id of_rockchip_thermal_match[] = {
 		.compatible = "rockchip,rk3568-tsadc",
 		.data = (void *)&rk3568_tsadc_data,
 	},
+	{
+		.compatible = "rockchip,rk3588-tsadc",
+		.data = (void *)&rk3588_tsadc_data,
+	},
 	{ /* end */ },
 };
 MODULE_DEVICE_TABLE(of, of_rockchip_thermal_match);
@@ -1213,7 +1378,7 @@ static irqreturn_t rockchip_thermal_alarm_irq_thread(int irq, void *dev)
 
 static int rockchip_thermal_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct rockchip_thermal_sensor *sensor = tz->devdata;
+	struct rockchip_thermal_sensor *sensor = thermal_zone_device_priv(tz);
 	struct rockchip_thermal_data *thermal = sensor->thermal;
 	const struct rockchip_tsadc_chip *tsadc = thermal->chip;
 
@@ -1226,16 +1391,13 @@ static int rockchip_thermal_set_trips(struct thermal_zone_device *tz, int low, i
 
 static int rockchip_thermal_get_temp(struct thermal_zone_device *tz, int *out_temp)
 {
-	struct rockchip_thermal_sensor *sensor = tz->devdata;
+	struct rockchip_thermal_sensor *sensor = thermal_zone_device_priv(tz);
 	struct rockchip_thermal_data *thermal = sensor->thermal;
 	const struct rockchip_tsadc_chip *tsadc = sensor->thermal->chip;
 	int retval;
 
 	retval = tsadc->get_temp(&tsadc->table,
 				 sensor->id, thermal->regs, out_temp);
-	dev_dbg(&thermal->pdev->dev, "sensor %d - temp: %d, retval: %d\n",
-		sensor->id, *out_temp, retval);
-
 	return retval;
 }
 
@@ -1353,15 +1515,10 @@ static int rockchip_thermal_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	struct rockchip_thermal_data *thermal;
-	const struct of_device_id *match;
 	int irq;
 	int i;
 	int error;
 
-	match = of_match_node(of_rockchip_thermal_match, np);
-	if (!match)
-		return -ENXIO;
-
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		return -EINVAL;
@@ -1373,57 +1530,40 @@ static int rockchip_thermal_probe(struct platform_device *pdev)
 
 	thermal->pdev = pdev;
 
-	thermal->chip = (const struct rockchip_tsadc_chip *)match->data;
+	thermal->chip = device_get_match_data(&pdev->dev);
 	if (!thermal->chip)
 		return -EINVAL;
 
+	thermal->sensors = devm_kcalloc(&pdev->dev, thermal->chip->chn_num,
+					sizeof(*thermal->sensors), GFP_KERNEL);
+	if (!thermal->sensors)
+		return -ENOMEM;
+
 	thermal->regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
 	if (IS_ERR(thermal->regs))
 		return PTR_ERR(thermal->regs);
 
-	thermal->reset = devm_reset_control_array_get(&pdev->dev, false, false);
-	if (IS_ERR(thermal->reset)) {
-		error = PTR_ERR(thermal->reset);
-		dev_err(&pdev->dev, "failed to get tsadc reset: %d\n", error);
-		return error;
-	}
-
-	thermal->clk = devm_clk_get(&pdev->dev, "tsadc");
-	if (IS_ERR(thermal->clk)) {
-		error = PTR_ERR(thermal->clk);
-		dev_err(&pdev->dev, "failed to get tsadc clock: %d\n", error);
-		return error;
-	}
+	thermal->reset = devm_reset_control_array_get_exclusive(&pdev->dev);
+	if (IS_ERR(thermal->reset))
+		return dev_err_probe(&pdev->dev, PTR_ERR(thermal->reset),
+				     "failed to get tsadc reset.\n");
 
-	thermal->pclk = devm_clk_get(&pdev->dev, "apb_pclk");
-	if (IS_ERR(thermal->pclk)) {
-		error = PTR_ERR(thermal->pclk);
-		dev_err(&pdev->dev, "failed to get apb_pclk clock: %d\n",
-			error);
-		return error;
-	}
+	thermal->clk = devm_clk_get_enabled(&pdev->dev, "tsadc");
+	if (IS_ERR(thermal->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(thermal->clk),
+				     "failed to get tsadc clock.\n");
 
-	error = clk_prepare_enable(thermal->clk);
-	if (error) {
-		dev_err(&pdev->dev, "failed to enable converter clock: %d\n",
-			error);
-		return error;
-	}
-
-	error = clk_prepare_enable(thermal->pclk);
-	if (error) {
-		dev_err(&pdev->dev, "failed to enable pclk: %d\n", error);
-		goto err_disable_clk;
-	}
+	thermal->pclk = devm_clk_get_enabled(&pdev->dev, "apb_pclk");
+	if (IS_ERR(thermal->pclk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(thermal->pclk),
+				     "failed to get apb_pclk clock.\n");
 
 	rockchip_thermal_reset_controller(thermal->reset);
 
 	error = rockchip_configure_from_dt(&pdev->dev, np, thermal);
-	if (error) {
-		dev_err(&pdev->dev, "failed to parse device tree data: %d\n",
-			error);
-		goto err_disable_pclk;
-	}
+	if (error)
+		return dev_err_probe(&pdev->dev, error,
+				"failed to parse device tree data\n");
 
 	thermal->chip->initialize(thermal->grf, thermal->regs,
 				  thermal->tshut_polarity);
@@ -1431,30 +1571,24 @@ static int rockchip_thermal_probe(struct platform_device *pdev)
 	for (i = 0; i < thermal->chip->chn_num; i++) {
 		error = rockchip_thermal_register_sensor(pdev, thermal,
 						&thermal->sensors[i],
-						thermal->chip->chn_id[i]);
-		if (error) {
-			dev_err(&pdev->dev,
-				"failed to register sensor[%d] : error = %d\n",
-				i, error);
-			goto err_disable_pclk;
-		}
+						thermal->chip->chn_offset + i);
+		if (error)
+			return dev_err_probe(&pdev->dev, error,
+				"failed to register sensor[%d].\n", i);
 	}
 
 	error = devm_request_threaded_irq(&pdev->dev, irq, NULL,
 					  &rockchip_thermal_alarm_irq_thread,
 					  IRQF_ONESHOT,
 					  "rockchip_thermal", thermal);
-	if (error) {
-		dev_err(&pdev->dev,
-			"failed to request tsadc irq: %d\n", error);
-		goto err_disable_pclk;
-	}
+	if (error)
+		return dev_err_probe(&pdev->dev, error,
+				     "failed to request tsadc irq.\n");
 
 	thermal->chip->control(thermal->regs, true);
 
 	for (i = 0; i < thermal->chip->chn_num; i++) {
 		rockchip_thermal_toggle_sensor(&thermal->sensors[i], true);
-		thermal->sensors[i].tzd->tzp->no_hwmon = false;
 		error = thermal_add_hwmon_sysfs(thermal->sensors[i].tzd);
 		if (error)
 			dev_warn(&pdev->dev,
@@ -1465,13 +1599,6 @@ static int rockchip_thermal_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, thermal);
 
 	return 0;
-
-err_disable_pclk:
-	clk_disable_unprepare(thermal->pclk);
-err_disable_clk:
-	clk_disable_unprepare(thermal->clk);
-
-	return error;
 }
 
 static int rockchip_thermal_remove(struct platform_device *pdev)
@@ -1488,9 +1615,6 @@ static int rockchip_thermal_remove(struct platform_device *pdev)
 
 	thermal->chip->control(thermal->regs, false);
 
-	clk_disable_unprepare(thermal->pclk);
-	clk_disable_unprepare(thermal->clk);
-
 	return 0;
 }
 
diff --git a/drivers/thermal/rzg2l_thermal.c b/drivers/thermal/rzg2l_thermal.c
index 2e0649f38506..b56981f85306 100644
--- a/drivers/thermal/rzg2l_thermal.c
+++ b/drivers/thermal/rzg2l_thermal.c
@@ -75,7 +75,7 @@ static inline void rzg2l_thermal_write(struct rzg2l_thermal_priv *priv, u32 reg,
 
 static int rzg2l_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct rzg2l_thermal_priv *priv = tz->devdata;
+	struct rzg2l_thermal_priv *priv = thermal_zone_device_priv(tz);
 	u32 result = 0, dsensor, ts_code_ave;
 	int val, i;
 
@@ -216,7 +216,6 @@ static int rzg2l_thermal_probe(struct platform_device *pdev)
 	}
 
 	priv->zone = zone;
-	priv->zone->tzp->no_hwmon = false;
 	ret = thermal_add_hwmon_sysfs(priv->zone);
 	if (ret)
 		goto err;
diff --git a/drivers/thermal/samsung/exynos_tmu.c b/drivers/thermal/samsung/exynos_tmu.c
index 527d1eb0663a..45e5c840d130 100644
--- a/drivers/thermal/samsung/exynos_tmu.c
+++ b/drivers/thermal/samsung/exynos_tmu.c
@@ -645,7 +645,7 @@ static void exynos7_tmu_control(struct platform_device *pdev, bool on)
 
 static int exynos_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct exynos_tmu_data *data = tz->devdata;
+	struct exynos_tmu_data *data = thermal_zone_device_priv(tz);
 	int value, ret = 0;
 
 	if (!data || !data->tmu_read)
@@ -723,7 +723,7 @@ static void exynos4412_tmu_set_emulation(struct exynos_tmu_data *data,
 
 static int exynos_tmu_set_emulation(struct thermal_zone_device *tz, int temp)
 {
-	struct exynos_tmu_data *data = tz->devdata;
+	struct exynos_tmu_data *data = thermal_zone_device_priv(tz);
 	int ret = -EINVAL;
 
 	if (data->soc == SOC_ARCH_EXYNOS4210)
diff --git a/drivers/thermal/spear_thermal.c b/drivers/thermal/spear_thermal.c
index 6a722b10d738..6e78616a576e 100644
--- a/drivers/thermal/spear_thermal.c
+++ b/drivers/thermal/spear_thermal.c
@@ -31,7 +31,7 @@ struct spear_thermal_dev {
 static inline int thermal_get_temp(struct thermal_zone_device *thermal,
 				int *temp)
 {
-	struct spear_thermal_dev *stdev = thermal->devdata;
+	struct spear_thermal_dev *stdev = thermal_zone_device_priv(thermal);
 
 	/*
 	 * Data are ready to be read after 628 usec from POWERDOWN signal
@@ -48,7 +48,7 @@ static struct thermal_zone_device_ops ops = {
 static int __maybe_unused spear_thermal_suspend(struct device *dev)
 {
 	struct thermal_zone_device *spear_thermal = dev_get_drvdata(dev);
-	struct spear_thermal_dev *stdev = spear_thermal->devdata;
+	struct spear_thermal_dev *stdev = thermal_zone_device_priv(spear_thermal);
 	unsigned int actual_mask = 0;
 
 	/* Disable SPEAr Thermal Sensor */
@@ -64,7 +64,7 @@ static int __maybe_unused spear_thermal_suspend(struct device *dev)
 static int __maybe_unused spear_thermal_resume(struct device *dev)
 {
 	struct thermal_zone_device *spear_thermal = dev_get_drvdata(dev);
-	struct spear_thermal_dev *stdev = spear_thermal->devdata;
+	struct spear_thermal_dev *stdev = thermal_zone_device_priv(spear_thermal);
 	unsigned int actual_mask = 0;
 	int ret = 0;
 
@@ -137,7 +137,7 @@ static int spear_thermal_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, spear_thermal);
 
-	dev_info(&spear_thermal->device, "Thermal Sensor Loaded at: 0x%p.\n",
+	dev_info(&pdev->dev, "Thermal Sensor Loaded at: 0x%p.\n",
 			stdev->thermal_base);
 
 	return 0;
@@ -154,7 +154,7 @@ static int spear_thermal_exit(struct platform_device *pdev)
 {
 	unsigned int actual_mask = 0;
 	struct thermal_zone_device *spear_thermal = platform_get_drvdata(pdev);
-	struct spear_thermal_dev *stdev = spear_thermal->devdata;
+	struct spear_thermal_dev *stdev = thermal_zone_device_priv(spear_thermal);
 
 	thermal_zone_device_unregister(spear_thermal);
 
diff --git a/drivers/thermal/sprd_thermal.c b/drivers/thermal/sprd_thermal.c
index ac884514f116..2fb90fdad76e 100644
--- a/drivers/thermal/sprd_thermal.c
+++ b/drivers/thermal/sprd_thermal.c
@@ -206,7 +206,7 @@ static int sprd_thm_temp_to_rawdata(int temp, struct sprd_thermal_sensor *sen)
 
 static int sprd_thm_read_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct sprd_thermal_sensor *sen = tz->devdata;
+	struct sprd_thermal_sensor *sen = thermal_zone_device_priv(tz);
 	u32 data;
 
 	data = readl(sen->data->base + SPRD_THM_TEMP(sen->id)) &
diff --git a/drivers/thermal/st/st_thermal.c b/drivers/thermal/st/st_thermal.c
index 1009f08e64e3..2d3042098445 100644
--- a/drivers/thermal/st/st_thermal.c
+++ b/drivers/thermal/st/st_thermal.c
@@ -108,8 +108,7 @@ static int st_thermal_calibration(struct st_thermal_sensor *sensor)
 /* Callback to get temperature from HW*/
 static int st_thermal_get_temp(struct thermal_zone_device *th, int *temperature)
 {
-	struct st_thermal_sensor *sensor = th->devdata;
-	struct device *dev = sensor->dev;
+	struct st_thermal_sensor *sensor = thermal_zone_device_priv(th);
 	unsigned int temp;
 	unsigned int overflow;
 	int ret;
@@ -127,8 +126,6 @@ static int st_thermal_get_temp(struct thermal_zone_device *th, int *temperature)
 	temp += sensor->cdata->temp_adjust_val;
 	temp = mcelsius(temp);
 
-	dev_dbg(dev, "temperature: %d\n", temp);
-
 	*temperature = temp;
 
 	return 0;
diff --git a/drivers/thermal/st/stm_thermal.c b/drivers/thermal/st/stm_thermal.c
index 735401958f01..903fcf1763f1 100644
--- a/drivers/thermal/st/stm_thermal.c
+++ b/drivers/thermal/st/stm_thermal.c
@@ -303,7 +303,7 @@ static int stm_disable_irq(struct stm_thermal_sensor *sensor)
 
 static int stm_thermal_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct stm_thermal_sensor *sensor = tz->devdata;
+	struct stm_thermal_sensor *sensor = thermal_zone_device_priv(tz);
 	u32 itr1, th;
 	int ret;
 
@@ -351,7 +351,7 @@ static int stm_thermal_set_trips(struct thermal_zone_device *tz, int low, int hi
 /* Callback to get temperature from HW */
 static int stm_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct stm_thermal_sensor *sensor = tz->devdata;
+	struct stm_thermal_sensor *sensor = thermal_zone_device_priv(tz);
 	u32 periods;
 	int freqM, ret;
 
@@ -558,7 +558,6 @@ static int stm_thermal_probe(struct platform_device *pdev)
 	 * Thermal_zone doesn't enable hwmon as default,
 	 * enable it here
 	 */
-	sensor->th_dev->tzp->no_hwmon = false;
 	ret = thermal_add_hwmon_sysfs(sensor->th_dev);
 	if (ret)
 		goto err_tz;
diff --git a/drivers/thermal/sun8i_thermal.c b/drivers/thermal/sun8i_thermal.c
index 497beac63e5d..793ddce72132 100644
--- a/drivers/thermal/sun8i_thermal.c
+++ b/drivers/thermal/sun8i_thermal.c
@@ -110,7 +110,7 @@ static int sun50i_h5_calc_temp(struct ths_device *tmdev,
 
 static int sun8i_ths_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct tsensor *s = tz->devdata;
+	struct tsensor *s = thermal_zone_device_priv(tz);
 	struct ths_device *tmdev = s->tmdev;
 	int val = 0;
 
@@ -475,7 +475,7 @@ static int sun8i_ths_register(struct ths_device *tmdev)
 		if (IS_ERR(tmdev->sensor[i].tzd))
 			return PTR_ERR(tmdev->sensor[i].tzd);
 
-		if (devm_thermal_add_hwmon_sysfs(tmdev->sensor[i].tzd))
+		if (devm_thermal_add_hwmon_sysfs(tmdev->dev, tmdev->sensor[i].tzd))
 			dev_warn(tmdev->dev,
 				 "Failed to add hwmon sysfs attributes\n");
 	}
diff --git a/drivers/thermal/tegra/soctherm.c b/drivers/thermal/tegra/soctherm.c
index 220873298d77..ea66cba09e56 100644
--- a/drivers/thermal/tegra/soctherm.c
+++ b/drivers/thermal/tegra/soctherm.c
@@ -423,7 +423,7 @@ static int translate_temp(u16 val)
 
 static int tegra_thermctl_get_temp(struct thermal_zone_device *tz, int *out_temp)
 {
-	struct tegra_thermctl_zone *zone = tz->devdata;
+	struct tegra_thermctl_zone *zone = thermal_zone_device_priv(tz);
 	u32 val;
 
 	val = readl(zone->reg);
@@ -584,7 +584,7 @@ static int tsensor_group_thermtrip_get(struct tegra_soctherm *ts, int id)
 
 static int tegra_thermctl_set_trip_temp(struct thermal_zone_device *tz, int trip_id, int temp)
 {
-	struct tegra_thermctl_zone *zone = tz->devdata;
+	struct tegra_thermctl_zone *zone = thermal_zone_device_priv(tz);
 	struct tegra_soctherm *ts = zone->ts;
 	struct thermal_trip trip;
 	const struct tegra_tsensor_group *sg = zone->sg;
@@ -658,7 +658,7 @@ static void thermal_irq_disable(struct tegra_thermctl_zone *zn)
 
 static int tegra_thermctl_set_trips(struct thermal_zone_device *tz, int lo, int hi)
 {
-	struct tegra_thermctl_zone *zone = tz->devdata;
+	struct tegra_thermctl_zone *zone = thermal_zone_device_priv(tz);
 	u32 r;
 
 	thermal_irq_disable(zone);
diff --git a/drivers/thermal/tegra/tegra-bpmp-thermal.c b/drivers/thermal/tegra/tegra-bpmp-thermal.c
index 0b7a1a1948cb..a2879d624945 100644
--- a/drivers/thermal/tegra/tegra-bpmp-thermal.c
+++ b/drivers/thermal/tegra/tegra-bpmp-thermal.c
@@ -52,6 +52,8 @@ static int __tegra_bpmp_thermal_get_temp(struct tegra_bpmp_thermal_zone *zone,
 	err = tegra_bpmp_transfer(zone->tegra->bpmp, &msg);
 	if (err)
 		return err;
+	if (msg.rx.ret == -BPMP_EFAULT)
+		return -EAGAIN;
 	if (msg.rx.ret)
 		return -EINVAL;
 
@@ -62,12 +64,14 @@ static int __tegra_bpmp_thermal_get_temp(struct tegra_bpmp_thermal_zone *zone,
 
 static int tegra_bpmp_thermal_get_temp(struct thermal_zone_device *tz, int *out_temp)
 {
-	return __tegra_bpmp_thermal_get_temp(tz->devdata, out_temp);
+	struct tegra_bpmp_thermal_zone *zone = thermal_zone_device_priv(tz);
+
+	return __tegra_bpmp_thermal_get_temp(zone, out_temp);
 }
 
 static int tegra_bpmp_thermal_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct tegra_bpmp_thermal_zone *zone = tz->devdata;
+	struct tegra_bpmp_thermal_zone *zone = thermal_zone_device_priv(tz);
 	struct mrq_thermal_host_to_bpmp_request req;
 	struct tegra_bpmp_message msg;
 	int err;
@@ -207,7 +211,12 @@ static int tegra_bpmp_thermal_probe(struct platform_device *pdev)
 		zone->tegra = tegra;
 
 		err = __tegra_bpmp_thermal_get_temp(zone, &temp);
-		if (err < 0) {
+
+		/*
+		 * Sensors in powergated domains may temporarily fail to be read
+		 * (-EAGAIN), but will become accessible when the domain is powered on.
+		 */
+		if (err < 0 && err != -EAGAIN) {
 			devm_kfree(&pdev->dev, zone);
 			continue;
 		}
diff --git a/drivers/thermal/tegra/tegra30-tsensor.c b/drivers/thermal/tegra/tegra30-tsensor.c
index b3218b71b6d9..cb584a5735ed 100644
--- a/drivers/thermal/tegra/tegra30-tsensor.c
+++ b/drivers/thermal/tegra/tegra30-tsensor.c
@@ -160,7 +160,7 @@ static void devm_tegra_tsensor_hw_disable(void *data)
 
 static int tegra_tsensor_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	const struct tegra_tsensor_channel *tsc = tz->devdata;
+	const struct tegra_tsensor_channel *tsc = thermal_zone_device_priv(tz);
 	const struct tegra_tsensor *ts = tsc->ts;
 	int err, c1, c2, c3, c4, counter;
 	u32 val;
@@ -218,7 +218,7 @@ static int tegra_tsensor_temp_to_counter(const struct tegra_tsensor *ts, int tem
 
 static int tegra_tsensor_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	const struct tegra_tsensor_channel *tsc = tz->devdata;
+	const struct tegra_tsensor_channel *tsc = thermal_zone_device_priv(tz);
 	const struct tegra_tsensor *ts = tsc->ts;
 	u32 val;
 
@@ -359,9 +359,6 @@ static int tegra_tsensor_enable_hw_channel(const struct tegra_tsensor *ts,
 
 	tegra_tsensor_get_hw_channel_trips(tzd, &hot_trip, &crit_trip);
 
-	/* prevent potential racing with tegra_tsensor_set_trips() */
-	mutex_lock(&tzd->lock);
-
 	dev_info_once(ts->dev, "ch%u: PMC emergency shutdown trip set to %dC\n",
 		      id, DIV_ROUND_CLOSEST(crit_trip, 1000));
 
@@ -404,8 +401,6 @@ static int tegra_tsensor_enable_hw_channel(const struct tegra_tsensor *ts,
 	val |= FIELD_PREP(TSENSOR_SENSOR0_CONFIG0_INTR_THERMAL_RST_EN, 1);
 	writel_relaxed(val, tsc->regs + TSENSOR_SENSOR0_CONFIG0);
 
-	mutex_unlock(&tzd->lock);
-
 	err = thermal_zone_device_enable(tzd);
 	if (err) {
 		dev_err(ts->dev, "ch%u: failed to enable zone: %d\n", id, err);
@@ -528,7 +523,7 @@ static int tegra_tsensor_register_channel(struct tegra_tsensor *ts,
 		return 0;
 	}
 
-	if (devm_thermal_add_hwmon_sysfs(tsc->tzd))
+	if (devm_thermal_add_hwmon_sysfs(ts->dev, tsc->tzd))
 		dev_warn(ts->dev, "failed to add hwmon sysfs attributes\n");
 
 	return 0;
@@ -585,6 +580,20 @@ static int tegra_tsensor_probe(struct platform_device *pdev)
 			return err;
 	}
 
+	/*
+	 * Enable the channels before setting the interrupt so
+	 * set_trips() can not be called while we are setting up the
+	 * register TSENSOR_SENSOR0_CONFIG1. With this we close a
+	 * potential race window where we are setting up the TH2 and
+	 * the temperature hits TH1 resulting to an update of the
+	 * TSENSOR_SENSOR0_CONFIG1 register in the ISR.
+	 */
+	for (i = 0; i < ARRAY_SIZE(ts->ch); i++) {
+		err = tegra_tsensor_enable_hw_channel(ts, i);
+		if (err)
+			return err;
+	}
+
 	err = devm_request_threaded_irq(&pdev->dev, irq, NULL,
 					tegra_tsensor_isr, IRQF_ONESHOT,
 					"tegra_tsensor", ts);
@@ -592,12 +601,6 @@ static int tegra_tsensor_probe(struct platform_device *pdev)
 		return dev_err_probe(&pdev->dev, err,
 				     "failed to request interrupt\n");
 
-	for (i = 0; i < ARRAY_SIZE(ts->ch); i++) {
-		err = tegra_tsensor_enable_hw_channel(ts, i);
-		if (err)
-			return err;
-	}
-
 	return 0;
 }
 
diff --git a/drivers/thermal/thermal-generic-adc.c b/drivers/thermal/thermal-generic-adc.c
index 323e273e3298..017b0ce52122 100644
--- a/drivers/thermal/thermal-generic-adc.c
+++ b/drivers/thermal/thermal-generic-adc.c
@@ -54,15 +54,14 @@ static int gadc_thermal_adc_to_temp(struct gadc_thermal_info *gti, int val)
 
 static int gadc_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct gadc_thermal_info *gti = tz->devdata;
+	struct gadc_thermal_info *gti = thermal_zone_device_priv(tz);
 	int val;
 	int ret;
 
 	ret = iio_read_channel_processed(gti->channel, &val);
-	if (ret < 0) {
-		dev_err(gti->dev, "IIO channel read failed %d\n", ret);
+	if (ret < 0)
 		return ret;
-	}
+
 	*temp = gadc_thermal_adc_to_temp(gti, val);
 
 	return 0;
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index 566df4522b88..c5025aca22ee 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -22,7 +22,7 @@
 #include <linux/suspend.h>
 
 #define CREATE_TRACE_POINTS
-#include <trace/events/thermal.h>
+#include "thermal_trace.h"
 
 #include "thermal_core.h"
 #include "thermal_hwmon.h"
@@ -794,68 +794,18 @@ void print_bind_err_msg(struct thermal_zone_device *tz,
 		tz->type, cdev->type, ret);
 }
 
-static void __bind(struct thermal_zone_device *tz, int mask,
-		   struct thermal_cooling_device *cdev,
-		   unsigned long *limits,
-		   unsigned int weight)
-{
-	int i, ret;
-
-	for (i = 0; i < tz->num_trips; i++) {
-		if (mask & (1 << i)) {
-			unsigned long upper, lower;
-
-			upper = THERMAL_NO_LIMIT;
-			lower = THERMAL_NO_LIMIT;
-			if (limits) {
-				lower = limits[i * 2];
-				upper = limits[i * 2 + 1];
-			}
-			ret = thermal_zone_bind_cooling_device(tz, i, cdev,
-							       upper, lower,
-							       weight);
-			if (ret)
-				print_bind_err_msg(tz, cdev, ret);
-		}
-	}
-}
-
 static void bind_cdev(struct thermal_cooling_device *cdev)
 {
-	int i, ret;
-	const struct thermal_zone_params *tzp;
+	int ret;
 	struct thermal_zone_device *pos = NULL;
 
-	mutex_lock(&thermal_list_lock);
-
 	list_for_each_entry(pos, &thermal_tz_list, node) {
-		if (!pos->tzp && !pos->ops->bind)
-			continue;
-
 		if (pos->ops->bind) {
 			ret = pos->ops->bind(pos, cdev);
 			if (ret)
 				print_bind_err_msg(pos, cdev, ret);
-			continue;
-		}
-
-		tzp = pos->tzp;
-		if (!tzp || !tzp->tbp)
-			continue;
-
-		for (i = 0; i < tzp->num_tbps; i++) {
-			if (tzp->tbp[i].cdev || !tzp->tbp[i].match)
-				continue;
-			if (tzp->tbp[i].match(pos, cdev))
-				continue;
-			tzp->tbp[i].cdev = cdev;
-			__bind(pos, tzp->tbp[i].trip_mask, cdev,
-			       tzp->tbp[i].binding_limits,
-			       tzp->tbp[i].weight);
 		}
 	}
-
-	mutex_unlock(&thermal_list_lock);
 }
 
 /**
@@ -933,17 +883,17 @@ __thermal_cooling_device_register(struct device_node *np,
 
 	/* Add 'this' new cdev to the global cdev list */
 	mutex_lock(&thermal_list_lock);
+
 	list_add(&cdev->node, &thermal_cdev_list);
-	mutex_unlock(&thermal_list_lock);
 
 	/* Update binding information for 'this' new cdev */
 	bind_cdev(cdev);
 
-	mutex_lock(&thermal_list_lock);
 	list_for_each_entry(pos, &thermal_tz_list, node)
 		if (atomic_cmpxchg(&pos->need_update, 1, 0))
 			thermal_zone_device_update(pos,
 						   THERMAL_EVENT_UNSPECIFIED);
+
 	mutex_unlock(&thermal_list_lock);
 
 	return cdev;
@@ -1138,16 +1088,6 @@ unlock_list:
 }
 EXPORT_SYMBOL_GPL(thermal_cooling_device_update);
 
-static void __unbind(struct thermal_zone_device *tz, int mask,
-		     struct thermal_cooling_device *cdev)
-{
-	int i;
-
-	for (i = 0; i < tz->num_trips; i++)
-		if (mask & (1 << i))
-			thermal_zone_unbind_cooling_device(tz, i, cdev);
-}
-
 /**
  * thermal_cooling_device_unregister - removes a thermal cooling device
  * @cdev:	the thermal cooling device to remove.
@@ -1157,8 +1097,6 @@ static void __unbind(struct thermal_zone_device *tz, int mask,
  */
 void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev)
 {
-	int i;
-	const struct thermal_zone_params *tzp;
 	struct thermal_zone_device *tz;
 
 	if (!cdev)
@@ -1175,21 +1113,8 @@ void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev)
 
 	/* Unbind all thermal zones associated with 'this' cdev */
 	list_for_each_entry(tz, &thermal_tz_list, node) {
-		if (tz->ops->unbind) {
+		if (tz->ops->unbind)
 			tz->ops->unbind(tz, cdev);
-			continue;
-		}
-
-		if (!tz->tzp || !tz->tzp->tbp)
-			continue;
-
-		tzp = tz->tzp;
-		for (i = 0; i < tzp->num_tbps; i++) {
-			if (tzp->tbp[i].cdev == cdev) {
-				__unbind(tz, tzp->tbp[i].trip_mask, cdev);
-				tzp->tbp[i].cdev = NULL;
-			}
-		}
 	}
 
 	mutex_unlock(&thermal_list_lock);
@@ -1200,41 +1125,20 @@ EXPORT_SYMBOL_GPL(thermal_cooling_device_unregister);
 
 static void bind_tz(struct thermal_zone_device *tz)
 {
-	int i, ret;
+	int ret;
 	struct thermal_cooling_device *pos = NULL;
-	const struct thermal_zone_params *tzp = tz->tzp;
 
-	if (!tzp && !tz->ops->bind)
+	if (!tz->ops->bind)
 		return;
 
 	mutex_lock(&thermal_list_lock);
 
-	/* If there is ops->bind, try to use ops->bind */
-	if (tz->ops->bind) {
-		list_for_each_entry(pos, &thermal_cdev_list, node) {
-			ret = tz->ops->bind(tz, pos);
-			if (ret)
-				print_bind_err_msg(tz, pos, ret);
-		}
-		goto exit;
-	}
-
-	if (!tzp || !tzp->tbp)
-		goto exit;
-
 	list_for_each_entry(pos, &thermal_cdev_list, node) {
-		for (i = 0; i < tzp->num_tbps; i++) {
-			if (tzp->tbp[i].cdev || !tzp->tbp[i].match)
-				continue;
-			if (tzp->tbp[i].match(tz, pos))
-				continue;
-			tzp->tbp[i].cdev = pos;
-			__bind(tz, tzp->tbp[i].trip_mask, pos,
-			       tzp->tbp[i].binding_limits,
-			       tzp->tbp[i].weight);
-		}
+		ret = tz->ops->bind(tz, pos);
+		if (ret)
+			print_bind_err_msg(tz, pos, ret);
 	}
-exit:
+
 	mutex_unlock(&thermal_list_lock);
 }
 
@@ -1352,13 +1256,21 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t
 	if (!tz)
 		return ERR_PTR(-ENOMEM);
 
+	if (tzp) {
+		tz->tzp = kmemdup(tzp, sizeof(*tzp), GFP_KERNEL);
+		if (!tz->tzp) {
+			result = -ENOMEM;
+			goto free_tz;
+		}
+	}
+
 	INIT_LIST_HEAD(&tz->thermal_instances);
 	ida_init(&tz->ida);
 	mutex_init(&tz->lock);
 	id = ida_alloc(&thermal_tz_ida, GFP_KERNEL);
 	if (id < 0) {
 		result = id;
-		goto free_tz;
+		goto free_tzp;
 	}
 
 	tz->id = id;
@@ -1368,7 +1280,6 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t
 		ops->critical = thermal_zone_device_critical;
 
 	tz->ops = ops;
-	tz->tzp = tzp;
 	tz->device.class = thermal_class;
 	tz->devdata = devdata;
 	tz->trips = trips;
@@ -1450,6 +1361,8 @@ release_device:
 	tz = NULL;
 remove_id:
 	ida_free(&thermal_tz_ida, id);
+free_tzp:
+	kfree(tz->tzp);
 free_tz:
 	kfree(tz);
 	return ERR_PTR(result);
@@ -1467,21 +1380,37 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type, int n
 }
 EXPORT_SYMBOL_GPL(thermal_zone_device_register);
 
+void *thermal_zone_device_priv(struct thermal_zone_device *tzd)
+{
+	return tzd->devdata;
+}
+EXPORT_SYMBOL_GPL(thermal_zone_device_priv);
+
+const char *thermal_zone_device_type(struct thermal_zone_device *tzd)
+{
+	return tzd->type;
+}
+EXPORT_SYMBOL_GPL(thermal_zone_device_type);
+
+int thermal_zone_device_id(struct thermal_zone_device *tzd)
+{
+	return tzd->id;
+}
+EXPORT_SYMBOL_GPL(thermal_zone_device_id);
+
 /**
  * thermal_zone_device_unregister - removes the registered thermal zone device
  * @tz: the thermal zone device to remove
  */
 void thermal_zone_device_unregister(struct thermal_zone_device *tz)
 {
-	int i, tz_id;
-	const struct thermal_zone_params *tzp;
+	int tz_id;
 	struct thermal_cooling_device *cdev;
 	struct thermal_zone_device *pos = NULL;
 
 	if (!tz)
 		return;
 
-	tzp = tz->tzp;
 	tz_id = tz->id;
 
 	mutex_lock(&thermal_list_lock);
@@ -1496,22 +1425,9 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz)
 	list_del(&tz->node);
 
 	/* Unbind all cdevs associated with 'this' thermal zone */
-	list_for_each_entry(cdev, &thermal_cdev_list, node) {
-		if (tz->ops->unbind) {
+	list_for_each_entry(cdev, &thermal_cdev_list, node)
+		if (tz->ops->unbind)
 			tz->ops->unbind(tz, cdev);
-			continue;
-		}
-
-		if (!tzp || !tzp->tbp)
-			break;
-
-		for (i = 0; i < tzp->num_tbps; i++) {
-			if (tzp->tbp[i].cdev == cdev) {
-				__unbind(tz, tzp->tbp[i].trip_mask, cdev);
-				tzp->tbp[i].cdev = NULL;
-			}
-		}
-	}
 
 	mutex_unlock(&thermal_list_lock);
 
@@ -1527,6 +1443,8 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz)
 	device_del(&tz->device);
 	mutex_unlock(&tz->lock);
 
+	kfree(tz->tzp);
+
 	put_device(&tz->device);
 
 	thermal_notify_tz_delete(tz_id);
diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c
index 0f648131b0b5..cfba0965a22d 100644
--- a/drivers/thermal/thermal_helpers.c
+++ b/drivers/thermal/thermal_helpers.c
@@ -19,9 +19,8 @@
 #include <linux/string.h>
 #include <linux/sysfs.h>
 
-#include <trace/events/thermal.h>
-
 #include "thermal_core.h"
+#include "thermal_trace.h"
 
 int get_tz_trend(struct thermal_zone_device *tz, int trip)
 {
@@ -107,6 +106,9 @@ int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp)
 			*temp = tz->emul_temperature;
 	}
 
+	if (ret)
+		dev_dbg(&tz->device, "Failed to get temperature: %d\n", ret);
+
 	return ret;
 }
 
diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c
index c594c42bea6d..fbe55509e307 100644
--- a/drivers/thermal/thermal_hwmon.c
+++ b/drivers/thermal/thermal_hwmon.c
@@ -17,6 +17,7 @@
 #include <linux/thermal.h>
 
 #include "thermal_hwmon.h"
+#include "thermal_core.h"
 
 /* hwmon sys I/F */
 /* thermal zone devices with the same type share one hwmon device */
@@ -263,7 +264,7 @@ static void devm_thermal_hwmon_release(struct device *dev, void *res)
 	thermal_remove_hwmon_sysfs(*(struct thermal_zone_device **)res);
 }
 
-int devm_thermal_add_hwmon_sysfs(struct thermal_zone_device *tz)
+int devm_thermal_add_hwmon_sysfs(struct device *dev, struct thermal_zone_device *tz)
 {
 	struct thermal_zone_device **ptr;
 	int ret;
@@ -280,7 +281,7 @@ int devm_thermal_add_hwmon_sysfs(struct thermal_zone_device *tz)
 	}
 
 	*ptr = tz;
-	devres_add(&tz->device, ptr);
+	devres_add(dev, ptr);
 
 	return ret;
 }
diff --git a/drivers/thermal/thermal_hwmon.h b/drivers/thermal/thermal_hwmon.h
index 1a9d65f6a6a8..b429f6e7abdb 100644
--- a/drivers/thermal/thermal_hwmon.h
+++ b/drivers/thermal/thermal_hwmon.h
@@ -17,7 +17,7 @@
 
 #ifdef CONFIG_THERMAL_HWMON
 int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz);
-int devm_thermal_add_hwmon_sysfs(struct thermal_zone_device *tz);
+int devm_thermal_add_hwmon_sysfs(struct device *dev, struct thermal_zone_device *tz);
 void thermal_remove_hwmon_sysfs(struct thermal_zone_device *tz);
 #else
 static inline int
@@ -27,7 +27,7 @@ thermal_add_hwmon_sysfs(struct thermal_zone_device *tz)
 }
 
 static inline int
-devm_thermal_add_hwmon_sysfs(struct thermal_zone_device *tz)
+devm_thermal_add_hwmon_sysfs(struct device *dev, struct thermal_zone_device *tz)
 {
 	return 0;
 }
diff --git a/drivers/thermal/thermal_mmio.c b/drivers/thermal/thermal_mmio.c
index ea616731066c..6845756ad5e7 100644
--- a/drivers/thermal/thermal_mmio.c
+++ b/drivers/thermal/thermal_mmio.c
@@ -23,7 +23,7 @@ static u32 thermal_mmio_readb(void __iomem *mmio_base)
 static int thermal_mmio_get_temperature(struct thermal_zone_device *tz, int *temp)
 {
 	int t;
-	struct thermal_mmio *sensor = tz->devdata;
+	struct thermal_mmio *sensor = thermal_zone_device_priv(tz);
 
 	t = sensor->read_mmio(sensor->mmio_base) & sensor->mask;
 	t *= sensor->factor;
diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c
index ff4d12ef51bc..6fb14e521197 100644
--- a/drivers/thermal/thermal_of.c
+++ b/drivers/thermal/thermal_of.c
@@ -439,7 +439,7 @@ static int thermal_of_unbind(struct thermal_zone_device *tz,
  *
  * @tz: a pointer to the thermal zone structure
  */
-void thermal_of_zone_unregister(struct thermal_zone_device *tz)
+static void thermal_of_zone_unregister(struct thermal_zone_device *tz)
 {
 	struct thermal_trip *trips = tz->trips;
 	struct thermal_zone_params *tzp = tz->tzp;
@@ -451,7 +451,6 @@ void thermal_of_zone_unregister(struct thermal_zone_device *tz)
 	kfree(tzp);
 	kfree(ops);
 }
-EXPORT_SYMBOL_GPL(thermal_of_zone_unregister);
 
 /**
  * thermal_of_zone_register - Register a thermal zone with device node
@@ -473,8 +472,8 @@ EXPORT_SYMBOL_GPL(thermal_of_zone_unregister);
  *	- ENOMEM: if one structure can not be allocated
  *	- Other negative errors are returned by the underlying called functions
  */
-struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data,
-						     const struct thermal_zone_device_ops *ops)
+static struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data,
+							    const struct thermal_zone_device_ops *ops)
 {
 	struct thermal_zone_device *tz;
 	struct thermal_trip *trips;
@@ -550,7 +549,6 @@ out_kfree_of_ops:
 
 	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL_GPL(thermal_of_zone_register);
 
 static void devm_thermal_of_zone_release(struct device *dev, void *res)
 {
diff --git a/drivers/thermal/thermal_trace.h b/drivers/thermal/thermal_trace.h
new file mode 100644
index 000000000000..459c8ce6cf3b
--- /dev/null
+++ b/drivers/thermal/thermal_trace.h
@@ -0,0 +1,205 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM thermal
+
+#if !defined(_TRACE_THERMAL_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_THERMAL_H
+
+#include <linux/devfreq.h>
+#include <linux/thermal.h>
+#include <linux/tracepoint.h>
+
+TRACE_DEFINE_ENUM(THERMAL_TRIP_CRITICAL);
+TRACE_DEFINE_ENUM(THERMAL_TRIP_HOT);
+TRACE_DEFINE_ENUM(THERMAL_TRIP_PASSIVE);
+TRACE_DEFINE_ENUM(THERMAL_TRIP_ACTIVE);
+
+#define show_tzt_type(type)					\
+	__print_symbolic(type,					\
+			 { THERMAL_TRIP_CRITICAL, "CRITICAL"},	\
+			 { THERMAL_TRIP_HOT,      "HOT"},	\
+			 { THERMAL_TRIP_PASSIVE,  "PASSIVE"},	\
+			 { THERMAL_TRIP_ACTIVE,   "ACTIVE"})
+
+TRACE_EVENT(thermal_temperature,
+
+	TP_PROTO(struct thermal_zone_device *tz),
+
+	TP_ARGS(tz),
+
+	TP_STRUCT__entry(
+		__string(thermal_zone, tz->type)
+		__field(int, id)
+		__field(int, temp_prev)
+		__field(int, temp)
+	),
+
+	TP_fast_assign(
+		__assign_str(thermal_zone, tz->type);
+		__entry->id = tz->id;
+		__entry->temp_prev = tz->last_temperature;
+		__entry->temp = tz->temperature;
+	),
+
+	TP_printk("thermal_zone=%s id=%d temp_prev=%d temp=%d",
+		__get_str(thermal_zone), __entry->id, __entry->temp_prev,
+		__entry->temp)
+);
+
+TRACE_EVENT(cdev_update,
+
+	TP_PROTO(struct thermal_cooling_device *cdev, unsigned long target),
+
+	TP_ARGS(cdev, target),
+
+	TP_STRUCT__entry(
+		__string(type, cdev->type)
+		__field(unsigned long, target)
+	),
+
+	TP_fast_assign(
+		__assign_str(type, cdev->type);
+		__entry->target = target;
+	),
+
+	TP_printk("type=%s target=%lu", __get_str(type), __entry->target)
+);
+
+TRACE_EVENT(thermal_zone_trip,
+
+	TP_PROTO(struct thermal_zone_device *tz, int trip,
+		enum thermal_trip_type trip_type),
+
+	TP_ARGS(tz, trip, trip_type),
+
+	TP_STRUCT__entry(
+		__string(thermal_zone, tz->type)
+		__field(int, id)
+		__field(int, trip)
+		__field(enum thermal_trip_type, trip_type)
+	),
+
+	TP_fast_assign(
+		__assign_str(thermal_zone, tz->type);
+		__entry->id = tz->id;
+		__entry->trip = trip;
+		__entry->trip_type = trip_type;
+	),
+
+	TP_printk("thermal_zone=%s id=%d trip=%d trip_type=%s",
+		__get_str(thermal_zone), __entry->id, __entry->trip,
+		show_tzt_type(__entry->trip_type))
+);
+
+#ifdef CONFIG_CPU_THERMAL
+TRACE_EVENT(thermal_power_cpu_get_power_simple,
+	TP_PROTO(int cpu, u32 power),
+
+	TP_ARGS(cpu, power),
+
+	TP_STRUCT__entry(
+		__field(int, cpu)
+		__field(u32, power)
+	),
+
+	TP_fast_assign(
+		__entry->cpu = cpu;
+		__entry->power = power;
+	),
+
+	TP_printk("cpu=%d power=%u", __entry->cpu, __entry->power)
+);
+
+TRACE_EVENT(thermal_power_cpu_limit,
+	TP_PROTO(const struct cpumask *cpus, unsigned int freq,
+		unsigned long cdev_state, u32 power),
+
+	TP_ARGS(cpus, freq, cdev_state, power),
+
+	TP_STRUCT__entry(
+		__bitmask(cpumask, num_possible_cpus())
+		__field(unsigned int,  freq      )
+		__field(unsigned long, cdev_state)
+		__field(u32,           power     )
+	),
+
+	TP_fast_assign(
+		__assign_bitmask(cpumask, cpumask_bits(cpus),
+				num_possible_cpus());
+		__entry->freq = freq;
+		__entry->cdev_state = cdev_state;
+		__entry->power = power;
+	),
+
+	TP_printk("cpus=%s freq=%u cdev_state=%lu power=%u",
+		__get_bitmask(cpumask), __entry->freq, __entry->cdev_state,
+		__entry->power)
+);
+#endif /* CONFIG_CPU_THERMAL */
+
+#ifdef CONFIG_DEVFREQ_THERMAL
+TRACE_EVENT(thermal_power_devfreq_get_power,
+	TP_PROTO(struct thermal_cooling_device *cdev,
+		 struct devfreq_dev_status *status, unsigned long freq,
+		u32 power),
+
+	TP_ARGS(cdev, status,  freq, power),
+
+	TP_STRUCT__entry(
+		__string(type,         cdev->type    )
+		__field(unsigned long, freq          )
+		__field(u32,           busy_time)
+		__field(u32,           total_time)
+		__field(u32,           power)
+	),
+
+	TP_fast_assign(
+		__assign_str(type, cdev->type);
+		__entry->freq = freq;
+		__entry->busy_time = status->busy_time;
+		__entry->total_time = status->total_time;
+		__entry->power = power;
+	),
+
+	TP_printk("type=%s freq=%lu load=%u power=%u",
+		__get_str(type), __entry->freq,
+		__entry->total_time == 0 ? 0 :
+			(100 * __entry->busy_time) / __entry->total_time,
+		__entry->power)
+);
+
+TRACE_EVENT(thermal_power_devfreq_limit,
+	TP_PROTO(struct thermal_cooling_device *cdev, unsigned long freq,
+		unsigned long cdev_state, u32 power),
+
+	TP_ARGS(cdev, freq, cdev_state, power),
+
+	TP_STRUCT__entry(
+		__string(type,         cdev->type)
+		__field(unsigned int,  freq      )
+		__field(unsigned long, cdev_state)
+		__field(u32,           power     )
+	),
+
+	TP_fast_assign(
+		__assign_str(type, cdev->type);
+		__entry->freq = freq;
+		__entry->cdev_state = cdev_state;
+		__entry->power = power;
+	),
+
+	TP_printk("type=%s freq=%u cdev_state=%lu power=%u",
+		__get_str(type), __entry->freq, __entry->cdev_state,
+		__entry->power)
+);
+#endif /* CONFIG_DEVFREQ_THERMAL */
+#endif /* _TRACE_THERMAL_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE thermal_trace
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/drivers/thermal/thermal_trace_ipa.h b/drivers/thermal/thermal_trace_ipa.h
new file mode 100644
index 000000000000..84568db5421b
--- /dev/null
+++ b/drivers/thermal/thermal_trace_ipa.h
@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM thermal_power_allocator
+
+#if !defined(_TRACE_THERMAL_POWER_ALLOCATOR_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_THERMAL_POWER_ALLOCATOR_H
+
+#include <linux/tracepoint.h>
+
+TRACE_EVENT(thermal_power_allocator,
+	TP_PROTO(struct thermal_zone_device *tz, u32 *req_power,
+		 u32 total_req_power, u32 *granted_power,
+		 u32 total_granted_power, size_t num_actors,
+		 u32 power_range, u32 max_allocatable_power,
+		 int current_temp, s32 delta_temp),
+	TP_ARGS(tz, req_power, total_req_power, granted_power,
+		total_granted_power, num_actors, power_range,
+		max_allocatable_power, current_temp, delta_temp),
+	TP_STRUCT__entry(
+		__field(int,           tz_id          )
+		__dynamic_array(u32,   req_power, num_actors    )
+		__field(u32,           total_req_power          )
+		__dynamic_array(u32,   granted_power, num_actors)
+		__field(u32,           total_granted_power      )
+		__field(size_t,        num_actors               )
+		__field(u32,           power_range              )
+		__field(u32,           max_allocatable_power    )
+		__field(int,           current_temp             )
+		__field(s32,           delta_temp               )
+	),
+	TP_fast_assign(
+		__entry->tz_id = tz->id;
+		memcpy(__get_dynamic_array(req_power), req_power,
+			num_actors * sizeof(*req_power));
+		__entry->total_req_power = total_req_power;
+		memcpy(__get_dynamic_array(granted_power), granted_power,
+			num_actors * sizeof(*granted_power));
+		__entry->total_granted_power = total_granted_power;
+		__entry->num_actors = num_actors;
+		__entry->power_range = power_range;
+		__entry->max_allocatable_power = max_allocatable_power;
+		__entry->current_temp = current_temp;
+		__entry->delta_temp = delta_temp;
+	),
+
+	TP_printk("thermal_zone_id=%d req_power={%s} total_req_power=%u granted_power={%s} total_granted_power=%u power_range=%u max_allocatable_power=%u current_temperature=%d delta_temperature=%d",
+		__entry->tz_id,
+		__print_array(__get_dynamic_array(req_power),
+                              __entry->num_actors, 4),
+		__entry->total_req_power,
+		__print_array(__get_dynamic_array(granted_power),
+                              __entry->num_actors, 4),
+		__entry->total_granted_power, __entry->power_range,
+		__entry->max_allocatable_power, __entry->current_temp,
+		__entry->delta_temp)
+);
+
+TRACE_EVENT(thermal_power_allocator_pid,
+	TP_PROTO(struct thermal_zone_device *tz, s32 err, s32 err_integral,
+		 s64 p, s64 i, s64 d, s32 output),
+	TP_ARGS(tz, err, err_integral, p, i, d, output),
+	TP_STRUCT__entry(
+		__field(int, tz_id       )
+		__field(s32, err         )
+		__field(s32, err_integral)
+		__field(s64, p           )
+		__field(s64, i           )
+		__field(s64, d           )
+		__field(s32, output      )
+	),
+	TP_fast_assign(
+		__entry->tz_id = tz->id;
+		__entry->err = err;
+		__entry->err_integral = err_integral;
+		__entry->p = p;
+		__entry->i = i;
+		__entry->d = d;
+		__entry->output = output;
+	),
+
+	TP_printk("thermal_zone_id=%d err=%d err_integral=%d p=%lld i=%lld d=%lld output=%d",
+		  __entry->tz_id, __entry->err, __entry->err_integral,
+		  __entry->p, __entry->i, __entry->d, __entry->output)
+);
+#endif /* _TRACE_THERMAL_POWER_ALLOCATOR_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE thermal_trace_ipa
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
index 8a9055bd376e..6a5335931f4d 100644
--- a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
+++ b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
@@ -23,6 +23,8 @@
 #include "ti-bandgap.h"
 #include "../thermal_hwmon.h"
 
+#define TI_BANDGAP_UPDATE_INTERVAL_MS 250
+
 /* common data structures */
 struct ti_thermal_data {
 	struct cpufreq_policy *policy;
@@ -43,8 +45,8 @@ static void ti_thermal_work(struct work_struct *work)
 
 	thermal_zone_device_update(data->ti_thermal, THERMAL_EVENT_UNSPECIFIED);
 
-	dev_dbg(&data->ti_thermal->device, "updated thermal zone %s\n",
-		data->ti_thermal->type);
+	dev_dbg(data->bgp->dev, "updated thermal zone %s\n",
+		thermal_zone_device_type(data->ti_thermal));
 }
 
 /**
@@ -68,7 +70,7 @@ static inline int ti_thermal_hotspot_temperature(int t, int s, int c)
 static inline int __ti_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
 	struct thermal_zone_device *pcb_tz = NULL;
-	struct ti_thermal_data *data = tz->devdata;
+	struct ti_thermal_data *data = thermal_zone_device_priv(tz);
 	struct ti_bandgap *bgp;
 	const struct ti_temp_sensor *s;
 	int ret, tmp, slope, constant;
@@ -109,7 +111,7 @@ static inline int __ti_thermal_get_temp(struct thermal_zone_device *tz, int *tem
 
 static int __ti_thermal_get_trend(struct thermal_zone_device *tz, int trip, enum thermal_trend *trend)
 {
-	struct ti_thermal_data *data = tz->devdata;
+	struct ti_thermal_data *data = thermal_zone_device_priv(tz);
 	struct ti_bandgap *bgp;
 	int id, tr, ret = 0;
 
@@ -159,7 +161,6 @@ int ti_thermal_expose_sensor(struct ti_bandgap *bgp, int id,
 			     char *domain)
 {
 	struct ti_thermal_data *data;
-	int interval;
 
 	data = ti_bandgap_get_sensor_data(bgp, id);
 
@@ -177,12 +178,11 @@ int ti_thermal_expose_sensor(struct ti_bandgap *bgp, int id,
 		return PTR_ERR(data->ti_thermal);
 	}
 
-	interval = jiffies_to_msecs(data->ti_thermal->polling_delay_jiffies);
-
 	ti_bandgap_set_sensor_data(bgp, id, data);
-	ti_bandgap_write_update_interval(bgp, data->sensor_id, interval);
+	ti_bandgap_write_update_interval(bgp, data->sensor_id,
+					 TI_BANDGAP_UPDATE_INTERVAL_MS);
 
-	if (devm_thermal_add_hwmon_sysfs(data->ti_thermal))
+	if (devm_thermal_add_hwmon_sysfs(bgp->dev, data->ti_thermal))
 		dev_warn(bgp->dev, "failed to add hwmon sysfs attributes\n");
 
 	return 0;
@@ -223,7 +223,7 @@ int ti_thermal_register_cpu_cooling(struct ti_bandgap *bgp, int id)
 	 * using DT, then it must be aware that the cooling device
 	 * loading has to happen via cpufreq driver.
 	 */
-	if (of_find_property(np, "#thermal-sensor-cells", NULL))
+	if (of_property_present(np, "#thermal-sensor-cells"))
 		return 0;
 
 	data = ti_bandgap_get_sensor_data(bgp, id);
diff --git a/drivers/thermal/uniphier_thermal.c b/drivers/thermal/uniphier_thermal.c
index 47801841b3f5..aef6119cc004 100644
--- a/drivers/thermal/uniphier_thermal.c
+++ b/drivers/thermal/uniphier_thermal.c
@@ -187,7 +187,7 @@ static void uniphier_tm_disable_sensor(struct uniphier_tm_dev *tdev)
 
 static int uniphier_tm_get_temp(struct thermal_zone_device *tz, int *out_temp)
 {
-	struct uniphier_tm_dev *tdev = tz->devdata;
+	struct uniphier_tm_dev *tdev = thermal_zone_device_priv(tz);
 	struct regmap *map = tdev->regmap;
 	int ret;
 	u32 temp;
diff --git a/drivers/thunderbolt/acpi.c b/drivers/thunderbolt/acpi.c
index 628225deb8fe..3514bf65b7a4 100644
--- a/drivers/thunderbolt/acpi.c
+++ b/drivers/thunderbolt/acpi.c
@@ -341,7 +341,7 @@ static struct acpi_device *tb_acpi_find_companion(struct device *dev)
 	 */
 	if (tb_is_switch(dev))
 		return tb_acpi_switch_find_companion(tb_to_switch(dev));
-	else if (tb_is_usb4_port_device(dev))
+	if (tb_is_usb4_port_device(dev))
 		return acpi_find_child_by_adr(ACPI_COMPANION(dev->parent),
 					      tb_to_usb4_port_device(dev)->port->port);
 	return NULL;
diff --git a/drivers/thunderbolt/ctl.c b/drivers/thunderbolt/ctl.c
index 6e7d28e8d81a..3a213322ec7a 100644
--- a/drivers/thunderbolt/ctl.c
+++ b/drivers/thunderbolt/ctl.c
@@ -1033,7 +1033,7 @@ static int tb_cfg_get_error(struct tb_ctl *ctl, enum tb_cfg_space space,
 
 	if (res->tb_error == TB_CFG_ERROR_LOCK)
 		return -EACCES;
-	else if (res->tb_error == TB_CFG_ERROR_PORT_NOT_CONNECTED)
+	if (res->tb_error == TB_CFG_ERROR_PORT_NOT_CONNECTED)
 		return -ENOTCONN;
 
 	return -EIO;
diff --git a/drivers/thunderbolt/eeprom.c b/drivers/thunderbolt/eeprom.c
index c90d22f56d4e..0f6099c18a94 100644
--- a/drivers/thunderbolt/eeprom.c
+++ b/drivers/thunderbolt/eeprom.c
@@ -416,7 +416,7 @@ static int tb_drom_parse_entries(struct tb_switch *sw, size_t header_size)
 		if (pos + 1 == drom_size || pos + entry->len > drom_size
 				|| !entry->len) {
 			tb_sw_warn(sw, "DROM buffer overrun\n");
-			return -EILSEQ;
+			return -EIO;
 		}
 
 		switch (entry->type) {
@@ -471,14 +471,13 @@ err:
 
 static int tb_drom_copy_nvm(struct tb_switch *sw, u16 *size)
 {
-	u32 drom_offset;
+	u16 drom_offset;
 	int ret;
 
 	if (!sw->dma_port)
 		return -ENODEV;
 
-	ret = tb_sw_read(sw, &drom_offset, TB_CFG_SWITCH,
-			 sw->cap_plug_events + 12, 1);
+	ret = tb_eeprom_get_drom_offset(sw, &drom_offset);
 	if (ret)
 		return ret;
 
@@ -513,7 +512,7 @@ err_free:
 	return ret;
 }
 
-static int usb4_copy_host_drom(struct tb_switch *sw, u16 *size)
+static int usb4_copy_drom(struct tb_switch *sw, u16 *size)
 {
 	int ret;
 
@@ -536,15 +535,40 @@ static int usb4_copy_host_drom(struct tb_switch *sw, u16 *size)
 	return ret;
 }
 
-static int tb_drom_read_n(struct tb_switch *sw, u16 offset, u8 *val,
-			  size_t count)
+static int tb_drom_bit_bang(struct tb_switch *sw, u16 *size)
 {
-	if (tb_switch_is_usb4(sw))
-		return usb4_switch_drom_read(sw, offset, val, count);
-	return tb_eeprom_read_n(sw, offset, val, count);
+	int ret;
+
+	ret = tb_eeprom_read_n(sw, 14, (u8 *)size, 2);
+	if (ret)
+		return ret;
+
+	*size &= 0x3ff;
+	*size += TB_DROM_DATA_START;
+
+	tb_sw_dbg(sw, "reading DROM (length: %#x)\n", *size);
+	if (*size < sizeof(struct tb_drom_header)) {
+		tb_sw_warn(sw, "DROM too small, aborting\n");
+		return -EIO;
+	}
+
+	sw->drom = kzalloc(*size, GFP_KERNEL);
+	if (!sw->drom)
+		return -ENOMEM;
+
+	ret = tb_eeprom_read_n(sw, 0, sw->drom, *size);
+	if (ret)
+		goto err;
+
+	return 0;
+
+err:
+	kfree(sw->drom);
+	sw->drom = NULL;
+	return ret;
 }
 
-static int tb_drom_parse(struct tb_switch *sw)
+static int tb_drom_parse_v1(struct tb_switch *sw)
 {
 	const struct tb_drom_header *header =
 		(const struct tb_drom_header *)sw->drom;
@@ -555,7 +579,7 @@ static int tb_drom_parse(struct tb_switch *sw)
 		tb_sw_warn(sw,
 			"DROM UID CRC8 mismatch (expected: %#x, got: %#x)\n",
 			header->uid_crc8, crc);
-		return -EILSEQ;
+		return -EIO;
 	}
 	if (!sw->uid)
 		sw->uid = header->uid;
@@ -589,85 +613,14 @@ static int usb4_drom_parse(struct tb_switch *sw)
 	return tb_drom_parse_entries(sw, USB4_DROM_HEADER_SIZE);
 }
 
-/**
- * tb_drom_read() - Copy DROM to sw->drom and parse it
- * @sw: Router whose DROM to read and parse
- *
- * This function reads router DROM and if successful parses the entries and
- * populates the fields in @sw accordingly. Can be called for any router
- * generation.
- *
- * Returns %0 in case of success and negative errno otherwise.
- */
-int tb_drom_read(struct tb_switch *sw)
+static int tb_drom_parse(struct tb_switch *sw, u16 size)
 {
-	u16 size;
-	struct tb_drom_header *header;
-	int res, retries = 1;
-
-	if (sw->drom)
-		return 0;
-
-	if (tb_route(sw) == 0) {
-		/*
-		 * Apple's NHI EFI driver supplies a DROM for the root switch
-		 * in a device property. Use it if available.
-		 */
-		if (tb_drom_copy_efi(sw, &size) == 0)
-			goto parse;
-
-		/* Non-Apple hardware has the DROM as part of NVM */
-		if (tb_drom_copy_nvm(sw, &size) == 0)
-			goto parse;
-
-		/*
-		 * USB4 hosts may support reading DROM through router
-		 * operations.
-		 */
-		if (tb_switch_is_usb4(sw)) {
-			usb4_switch_read_uid(sw, &sw->uid);
-			if (!usb4_copy_host_drom(sw, &size))
-				goto parse;
-		} else {
-			/*
-			 * The root switch contains only a dummy drom
-			 * (header only, no entries). Hardcode the
-			 * configuration here.
-			 */
-			tb_drom_read_uid_only(sw, &sw->uid);
-		}
-
-		return 0;
-	}
-
-	res = tb_drom_read_n(sw, 14, (u8 *) &size, 2);
-	if (res)
-		return res;
-	size &= 0x3ff;
-	size += TB_DROM_DATA_START;
-	tb_sw_dbg(sw, "reading drom (length: %#x)\n", size);
-	if (size < sizeof(*header)) {
-		tb_sw_warn(sw, "drom too small, aborting\n");
-		return -EIO;
-	}
-
-	sw->drom = kzalloc(size, GFP_KERNEL);
-	if (!sw->drom)
-		return -ENOMEM;
-read:
-	res = tb_drom_read_n(sw, 0, sw->drom, size);
-	if (res)
-		goto err;
-
-parse:
-	header = (void *) sw->drom;
+	const struct tb_drom_header *header = (const void *)sw->drom;
+	int ret;
 
 	if (header->data_len + TB_DROM_DATA_START != size) {
-		tb_sw_warn(sw, "drom size mismatch\n");
-		if (retries--) {
-			msleep(100);
-			goto read;
-		}
+		tb_sw_warn(sw, "DROM size mismatch\n");
+		ret = -EIO;
 		goto err;
 	}
 
@@ -675,29 +628,86 @@ parse:
 
 	switch (header->device_rom_revision) {
 	case 3:
-		res = usb4_drom_parse(sw);
+		ret = usb4_drom_parse(sw);
 		break;
 	default:
 		tb_sw_warn(sw, "DROM device_rom_revision %#x unknown\n",
 			   header->device_rom_revision);
 		fallthrough;
 	case 1:
-		res = tb_drom_parse(sw);
+		ret = tb_drom_parse_v1(sw);
 		break;
 	}
 
-	/* If the DROM parsing fails, wait a moment and retry once */
-	if (res == -EILSEQ && retries--) {
+	if (ret) {
 		tb_sw_warn(sw, "parsing DROM failed\n");
-		msleep(100);
-		goto read;
+		goto err;
 	}
 
-	if (!res)
-		return 0;
+	return 0;
 
 err:
 	kfree(sw->drom);
 	sw->drom = NULL;
-	return -EIO;
+
+	return ret;
+}
+
+static int tb_drom_host_read(struct tb_switch *sw)
+{
+	u16 size;
+
+	if (tb_switch_is_usb4(sw)) {
+		usb4_switch_read_uid(sw, &sw->uid);
+		if (!usb4_copy_drom(sw, &size))
+			return tb_drom_parse(sw, size);
+	} else {
+		if (!tb_drom_copy_efi(sw, &size))
+			return tb_drom_parse(sw, size);
+
+		if (!tb_drom_copy_nvm(sw, &size))
+			return tb_drom_parse(sw, size);
+
+		tb_drom_read_uid_only(sw, &sw->uid);
+	}
+
+	return 0;
+}
+
+static int tb_drom_device_read(struct tb_switch *sw)
+{
+	u16 size;
+	int ret;
+
+	if (tb_switch_is_usb4(sw)) {
+		usb4_switch_read_uid(sw, &sw->uid);
+		ret = usb4_copy_drom(sw, &size);
+	} else {
+		ret = tb_drom_bit_bang(sw, &size);
+	}
+
+	if (ret)
+		return ret;
+
+	return tb_drom_parse(sw, size);
+}
+
+/**
+ * tb_drom_read() - Copy DROM to sw->drom and parse it
+ * @sw: Router whose DROM to read and parse
+ *
+ * This function reads router DROM and if successful parses the entries and
+ * populates the fields in @sw accordingly. Can be called for any router
+ * generation.
+ *
+ * Returns %0 in case of success and negative errno otherwise.
+ */
+int tb_drom_read(struct tb_switch *sw)
+{
+	if (sw->drom)
+		return 0;
+
+	if (!tb_route(sw))
+		return tb_drom_host_read(sw);
+	return tb_drom_device_read(sw);
 }
diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c
index cfebec107f3f..d76e923fbc6a 100644
--- a/drivers/thunderbolt/nhi.c
+++ b/drivers/thunderbolt/nhi.c
@@ -526,7 +526,8 @@ static int nhi_alloc_hop(struct tb_nhi *nhi, struct tb_ring *ring)
 			 ring->hop);
 		ret = -EBUSY;
 		goto err_unlock;
-	} else if (!ring->is_tx && nhi->rx_rings[ring->hop]) {
+	}
+	if (!ring->is_tx && nhi->rx_rings[ring->hop]) {
 		dev_warn(&nhi->pdev->dev, "RX hop %d already allocated\n",
 			 ring->hop);
 		ret = -EBUSY;
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index da373ac38285..51e86b5171c7 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -271,9 +271,9 @@ static int nvm_authenticate(struct tb_switch *sw, bool auth_only)
 		}
 		sw->nvm->authenticating = true;
 		return usb4_switch_nvm_authenticate(sw);
-	} else if (auth_only) {
-		return -EOPNOTSUPP;
 	}
+	if (auth_only)
+		return -EOPNOTSUPP;
 
 	sw->nvm->authenticating = true;
 	if (!tb_route(sw)) {
diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c
index a0996cb2893c..485b6e430686 100644
--- a/drivers/thunderbolt/usb4.c
+++ b/drivers/thunderbolt/usb4.c
@@ -9,6 +9,7 @@
 
 #include <linux/delay.h>
 #include <linux/ktime.h>
+#include <linux/units.h>
 
 #include "sb_regs.h"
 #include "tb.h"
@@ -851,7 +852,7 @@ bool usb4_switch_query_dp_resource(struct tb_switch *sw, struct tb_port *in)
 	 */
 	if (ret == -EOPNOTSUPP)
 		return true;
-	else if (ret)
+	if (ret)
 		return false;
 
 	return !status;
@@ -877,7 +878,7 @@ int usb4_switch_alloc_dp_resource(struct tb_switch *sw, struct tb_port *in)
 			     &status);
 	if (ret == -EOPNOTSUPP)
 		return 0;
-	else if (ret)
+	if (ret)
 		return ret;
 
 	return status ? -EBUSY : 0;
@@ -900,7 +901,7 @@ int usb4_switch_dealloc_dp_resource(struct tb_switch *sw, struct tb_port *in)
 			     &status);
 	if (ret == -EOPNOTSUPP)
 		return 0;
-	else if (ret)
+	if (ret)
 		return ret;
 
 	return status ? -EIO : 0;
@@ -1302,6 +1303,20 @@ static int usb4_port_sb_write(struct tb_port *port, enum usb4_sb_target target,
 	return 0;
 }
 
+static int usb4_port_sb_opcode_err_to_errno(u32 val)
+{
+	switch (val) {
+	case 0:
+		return 0;
+	case USB4_SB_OPCODE_ERR:
+		return -EAGAIN;
+	case USB4_SB_OPCODE_ONS:
+		return -EOPNOTSUPP;
+	default:
+		return -EIO;
+	}
+}
+
 static int usb4_port_sb_op(struct tb_port *port, enum usb4_sb_target target,
 			   u8 index, enum usb4_sb_opcode opcode, int timeout_msec)
 {
@@ -1324,21 +1339,8 @@ static int usb4_port_sb_op(struct tb_port *port, enum usb4_sb_target target,
 		if (ret)
 			return ret;
 
-		switch (val) {
-		case 0:
-			return 0;
-
-		case USB4_SB_OPCODE_ERR:
-			return -EAGAIN;
-
-		case USB4_SB_OPCODE_ONS:
-			return -EOPNOTSUPP;
-
-		default:
-			if (val != opcode)
-				return -EIO;
-			break;
-		}
+		if (val != opcode)
+			return usb4_port_sb_opcode_err_to_errno(val);
 	} while (ktime_before(ktime_get(), timeout));
 
 	return -ETIMEDOUT;
@@ -1813,12 +1815,13 @@ int usb4_port_retimer_nvm_authenticate_status(struct tb_port *port, u8 index,
 	if (ret)
 		return ret;
 
-	switch (val) {
+	ret = usb4_port_sb_opcode_err_to_errno(val);
+	switch (ret) {
 	case 0:
 		*status = 0;
 		return 0;
 
-	case USB4_SB_OPCODE_ERR:
+	case -EAGAIN:
 		ret = usb4_port_retimer_read(port, index, USB4_SB_METADATA,
 					     &metadata, sizeof(metadata));
 		if (ret)
@@ -1827,11 +1830,8 @@ int usb4_port_retimer_nvm_authenticate_status(struct tb_port *port, u8 index,
 		*status = metadata & USB4_SB_METADATA_NVM_AUTH_WRITE_MASK;
 		return 0;
 
-	case USB4_SB_OPCODE_ONS:
-		return -EOPNOTSUPP;
-
 	default:
-		return -EIO;
+		return ret;
 	}
 }
 
@@ -1995,7 +1995,7 @@ static unsigned int usb3_bw_to_mbps(u32 bw, u8 scale)
 	unsigned long uframes;
 
 	uframes = bw * 512UL << scale;
-	return DIV_ROUND_CLOSEST(uframes * 8000, 1000 * 1000);
+	return DIV_ROUND_CLOSEST(uframes * 8000, MEGA);
 }
 
 static u32 mbps_to_usb3_bw(unsigned int mbps, u8 scale)
@@ -2003,7 +2003,7 @@ static u32 mbps_to_usb3_bw(unsigned int mbps, u8 scale)
 	unsigned long uframes;
 
 	/* 1 uframe is 1/8 ms (125 us) -> 1 / 8000 s */
-	uframes = ((unsigned long)mbps * 1000 *  1000) / 8000;
+	uframes = ((unsigned long)mbps * MEGA) / 8000;
 	return DIV_ROUND_UP(uframes, 512UL << scale);
 }
 
diff --git a/drivers/thunderbolt/xdomain.c b/drivers/thunderbolt/xdomain.c
index a48335c95d39..e2b54887d331 100644
--- a/drivers/thunderbolt/xdomain.c
+++ b/drivers/thunderbolt/xdomain.c
@@ -1178,9 +1178,8 @@ static int tb_xdomain_get_uuid(struct tb_xdomain *xd)
 		if (xd->state_retries-- > 0) {
 			dev_dbg(&xd->dev, "failed to request UUID, retrying\n");
 			return -EAGAIN;
-		} else {
-			dev_dbg(&xd->dev, "failed to read remote UUID\n");
 		}
+		dev_dbg(&xd->dev, "failed to read remote UUID\n");
 		return ret;
 	}
 
@@ -1367,12 +1366,10 @@ static int tb_xdomain_get_properties(struct tb_xdomain *xd)
 			dev_dbg(&xd->dev,
 				"failed to request remote properties, retrying\n");
 			return -EAGAIN;
-		} else {
-			/* Give up now */
-			dev_err(&xd->dev,
-				"failed read XDomain properties from %pUb\n",
-				xd->remote_uuid);
 		}
+		/* Give up now */
+		dev_err(&xd->dev, "failed read XDomain properties from %pUb\n",
+			xd->remote_uuid);
 
 		return ret;
 	}
@@ -2179,13 +2176,12 @@ static struct tb_xdomain *switch_find_xdomain(struct tb_switch *sw,
 				if (xd->remote_uuid &&
 				    uuid_equal(xd->remote_uuid, lookup->uuid))
 					return xd;
-			} else if (lookup->link &&
-				   lookup->link == xd->link &&
-				   lookup->depth == xd->depth) {
-				return xd;
-			} else if (lookup->route &&
-				   lookup->route == xd->route) {
-				return xd;
+			} else {
+				if (lookup->link && lookup->link == xd->link &&
+				    lookup->depth == xd->depth)
+					return xd;
+				if (lookup->route && lookup->route == xd->route)
+					return xd;
 			}
 		} else if (tb_port_has_remote(port)) {
 			xd = switch_find_xdomain(port->remote->sw, lookup);
diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig
index d35fc068da74..341abaed4ce2 100644
--- a/drivers/tty/Kconfig
+++ b/drivers/tty/Kconfig
@@ -160,7 +160,7 @@ config LEGACY_TIOCSTI
 	  a dangerous legacy operation, and can be disabled on most
 	  systems.
 
-	  Say 'Y here only if you have confirmed that your system's
+	  Say Y here only if you have confirmed that your system's
 	  userspace depends on this functionality to continue operating
 	  normally.
 
@@ -301,6 +301,15 @@ config GOLDFISH_TTY_EARLY_CONSOLE
 	default y if GOLDFISH_TTY=y
 	select SERIAL_EARLYCON
 
+config IPWIRELESS
+	tristate "IPWireless 3G UMTS PCMCIA card support"
+	depends on PCMCIA && NETDEVICES
+	select PPP
+	help
+	  This is a driver for 3G UMTS PCMCIA card from IPWireless company. In
+	  some countries (for example Czech Republic, T-Mobile ISP) this card
+	  is shipped for service called UMTS 4G.
+
 config N_GSM
 	tristate "GSM MUX line discipline support (EXPERIMENTAL)"
 	depends on NET
diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c
index d7515d61659e..c06ad0a0744b 100644
--- a/drivers/tty/amiserial.c
+++ b/drivers/tty/amiserial.c
@@ -347,7 +347,7 @@ static void check_modem_status(struct serial_state *info)
 #if (defined(SERIAL_DEBUG_INTR) || defined(SERIAL_DEBUG_FLOW))
 				printk("CTS tx start...");
 #endif
-				port->tty->hw_stopped = 0;
+				port->tty->hw_stopped = false;
 				info->IER |= UART_IER_THRI;
 				amiga_custom.intena = IF_SETCLR | IF_TBE;
 				mb();
@@ -362,7 +362,7 @@ static void check_modem_status(struct serial_state *info)
 #if (defined(SERIAL_DEBUG_INTR) || defined(SERIAL_DEBUG_FLOW))
 				printk("CTS tx stop...");
 #endif
-				port->tty->hw_stopped = 1;
+				port->tty->hw_stopped = true;
 				info->IER &= ~UART_IER_THRI;
 				/* disable Tx interrupt and remove any pending interrupts */
 				amiga_custom.intena = IF_TBE;
@@ -1197,7 +1197,7 @@ static void rs_set_termios(struct tty_struct *tty, const struct ktermios *old_te
 
 	/* Handle turning off CRTSCTS */
 	if ((old_termios->c_cflag & CRTSCTS) && !C_CRTSCTS(tty)) {
-		tty->hw_stopped = 0;
+		tty->hw_stopped = false;
 		rs_start(tty);
 	}
 
diff --git a/drivers/tty/mxser.c b/drivers/tty/mxser.c
index ef3116e87975..10855e66fda1 100644
--- a/drivers/tty/mxser.c
+++ b/drivers/tty/mxser.c
@@ -553,7 +553,7 @@ static void mxser_handle_cts(struct tty_struct *tty, struct mxser_port *info,
 
 	if (tty->hw_stopped) {
 		if (cts) {
-			tty->hw_stopped = 0;
+			tty->hw_stopped = false;
 
 			if (!mxser_16550A_or_MUST(info))
 				__mxser_start_tx(info);
@@ -563,7 +563,7 @@ static void mxser_handle_cts(struct tty_struct *tty, struct mxser_port *info,
 	} else if (cts)
 		return;
 
-	tty->hw_stopped = 1;
+	tty->hw_stopped = true;
 	if (!mxser_16550A_or_MUST(info))
 		__mxser_stop_tx(info);
 }
@@ -1361,7 +1361,7 @@ static void mxser_set_termios(struct tty_struct *tty,
 	spin_unlock_irqrestore(&info->slock, flags);
 
 	if ((old_termios->c_cflag & CRTSCTS) && !C_CRTSCTS(tty)) {
-		tty->hw_stopped = 0;
+		tty->hw_stopped = false;
 		mxser_start(tty);
 	}
 
diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c
index 7aa05ebed8e1..b411a26cc092 100644
--- a/drivers/tty/n_gsm.c
+++ b/drivers/tty/n_gsm.c
@@ -42,6 +42,7 @@
 #include <linux/ctype.h>
 #include <linux/mm.h>
 #include <linux/math.h>
+#include <linux/nospec.h>
 #include <linux/string.h>
 #include <linux/slab.h>
 #include <linux/poll.h>
@@ -128,6 +129,7 @@ struct gsm_msg {
 
 enum gsm_dlci_state {
 	DLCI_CLOSED,
+	DLCI_WAITING_CONFIG,	/* Waiting for DLCI configuration from user */
 	DLCI_CONFIGURE,		/* Sending PN (for adaption > 1) */
 	DLCI_OPENING,		/* Sending SABM not seen UA */
 	DLCI_OPEN,		/* SABM/UA complete */
@@ -330,6 +332,7 @@ struct gsm_mux {
 	unsigned int t3;	/* Power wake-up timer in seconds. */
 	int n2;			/* Retry count */
 	u8 k;			/* Window size */
+	bool wait_config;	/* Wait for configuration by ioctl before DLCI open */
 	u32 keep_alive;		/* Control channel keep-alive in 10ms */
 
 	/* Statistics (not currently exposed) */
@@ -451,6 +454,7 @@ static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk);
 static struct gsm_msg *gsm_data_alloc(struct gsm_mux *gsm, u8 addr, int len,
 								u8 ctrl);
 static int gsm_send_packet(struct gsm_mux *gsm, struct gsm_msg *msg);
+static struct gsm_dlci *gsm_dlci_alloc(struct gsm_mux *gsm, int addr);
 static void gsmld_write_trigger(struct gsm_mux *gsm);
 static void gsmld_write_task(struct work_struct *work);
 
@@ -2280,6 +2284,7 @@ static void gsm_dlci_begin_open(struct gsm_dlci *dlci)
 
 	switch (dlci->state) {
 	case DLCI_CLOSED:
+	case DLCI_WAITING_CONFIG:
 	case DLCI_CLOSING:
 		dlci->retries = gsm->n2;
 		if (!need_pn) {
@@ -2311,6 +2316,7 @@ static void gsm_dlci_set_opening(struct gsm_dlci *dlci)
 {
 	switch (dlci->state) {
 	case DLCI_CLOSED:
+	case DLCI_WAITING_CONFIG:
 	case DLCI_CLOSING:
 		dlci->state = DLCI_OPENING;
 		break;
@@ -2320,6 +2326,24 @@ static void gsm_dlci_set_opening(struct gsm_dlci *dlci)
 }
 
 /**
+ * gsm_dlci_set_wait_config	-	wait for channel configuration
+ * @dlci: DLCI to configure
+ *
+ * Wait for a DLCI configuration from the application.
+ */
+static void gsm_dlci_set_wait_config(struct gsm_dlci *dlci)
+{
+	switch (dlci->state) {
+	case DLCI_CLOSED:
+	case DLCI_CLOSING:
+		dlci->state = DLCI_WAITING_CONFIG;
+		break;
+	default:
+		break;
+	}
+}
+
+/**
  *	gsm_dlci_begin_close	-	start channel open procedure
  *	@dlci: DLCI to open
  *
@@ -2453,6 +2477,128 @@ static void gsm_kick_timer(struct timer_list *t)
 		pr_info("%s TX queue stalled\n", __func__);
 }
 
+/**
+ * gsm_dlci_copy_config_values	-	copy DLCI configuration
+ * @dlci: source DLCI
+ * @dc: configuration structure to fill
+ */
+static void gsm_dlci_copy_config_values(struct gsm_dlci *dlci, struct gsm_dlci_config *dc)
+{
+	memset(dc, 0, sizeof(*dc));
+	dc->channel = (u32)dlci->addr;
+	dc->adaption = (u32)dlci->adaption;
+	dc->mtu = (u32)dlci->mtu;
+	dc->priority = (u32)dlci->prio;
+	if (dlci->ftype == UIH)
+		dc->i = 1;
+	else
+		dc->i = 2;
+	dc->k = (u32)dlci->k;
+}
+
+/**
+ * gsm_dlci_config	-	configure DLCI from configuration
+ * @dlci: DLCI to configure
+ * @dc: DLCI configuration
+ * @open: open DLCI after configuration?
+ */
+static int gsm_dlci_config(struct gsm_dlci *dlci, struct gsm_dlci_config *dc, int open)
+{
+	struct gsm_mux *gsm;
+	bool need_restart = false;
+	bool need_open = false;
+	unsigned int i;
+
+	/*
+	 * Check that userspace doesn't put stuff in here to prevent breakages
+	 * in the future.
+	 */
+	for (i = 0; i < ARRAY_SIZE(dc->reserved); i++)
+		if (dc->reserved[i])
+			return -EINVAL;
+
+	if (!dlci)
+		return -EINVAL;
+	gsm = dlci->gsm;
+
+	/* Stuff we don't support yet - I frame transport */
+	if (dc->adaption != 1 && dc->adaption != 2)
+		return -EOPNOTSUPP;
+	if (dc->mtu > MAX_MTU || dc->mtu < MIN_MTU || dc->mtu > gsm->mru)
+		return -EINVAL;
+	if (dc->priority >= 64)
+		return -EINVAL;
+	if (dc->i == 0 || dc->i > 2)  /* UIH and UI only */
+		return -EINVAL;
+	if (dc->k > 7)
+		return -EINVAL;
+
+	/*
+	 * See what is needed for reconfiguration
+	 */
+	/* Framing fields */
+	if (dc->adaption != dlci->adaption)
+		need_restart = true;
+	if (dc->mtu != dlci->mtu)
+		need_restart = true;
+	if (dc->i != dlci->ftype)
+		need_restart = true;
+	/* Requires care */
+	if (dc->priority != dlci->prio)
+		need_restart = true;
+
+	if ((open && gsm->wait_config) || need_restart)
+		need_open = true;
+	if (dlci->state == DLCI_WAITING_CONFIG) {
+		need_restart = false;
+		need_open = true;
+	}
+
+	/*
+	 * Close down what is needed, restart and initiate the new
+	 * configuration.
+	 */
+	if (need_restart) {
+		gsm_dlci_begin_close(dlci);
+		wait_event_interruptible(gsm->event, dlci->state == DLCI_CLOSED);
+		if (signal_pending(current))
+			return -EINTR;
+	}
+	/*
+	 * Setup the new configuration values
+	 */
+	dlci->adaption = (int)dc->adaption;
+
+	if (dc->mtu)
+		dlci->mtu = (unsigned int)dc->mtu;
+	else
+		dlci->mtu = gsm->mtu;
+
+	if (dc->priority)
+		dlci->prio = (u8)dc->priority;
+	else
+		dlci->prio = roundup(dlci->addr + 1, 8) - 1;
+
+	if (dc->i == 1)
+		dlci->ftype = UIH;
+	else if (dc->i == 2)
+		dlci->ftype = UI;
+
+	if (dc->k)
+		dlci->k = (u8)dc->k;
+	else
+		dlci->k = gsm->k;
+
+	if (need_open) {
+		if (gsm->initiator)
+			gsm_dlci_begin_open(dlci);
+		else
+			gsm_dlci_set_opening(dlci);
+	}
+
+	return 0;
+}
+
 /*
  *	Allocate/Free DLCI channels
  */
@@ -3078,6 +3224,7 @@ static struct gsm_mux *gsm_alloc_mux(void)
 	gsm->mru = 64;	/* Default to encoding 1 so these should be 64 */
 	gsm->mtu = 64;
 	gsm->dead = true;	/* Avoid early tty opens */
+	gsm->wait_config = false; /* Disabled */
 	gsm->keep_alive = 0;	/* Disabled */
 
 	/* Store the instance to the mux array or abort if no space is
@@ -3130,8 +3277,8 @@ static int gsm_config(struct gsm_mux *gsm, struct gsm_config *c)
 	int need_close = 0;
 	int need_restart = 0;
 
-	/* Stuff we don't support yet - UI or I frame transport, windowing */
-	if ((c->adaption != 1 && c->adaption != 2) || c->k)
+	/* Stuff we don't support yet - UI or I frame transport */
+	if (c->adaption != 1 && c->adaption != 2)
 		return -EOPNOTSUPP;
 	/* Check the MRU/MTU range looks sane */
 	if (c->mru < MIN_MTU || c->mtu < MIN_MTU)
@@ -3218,6 +3365,7 @@ static void gsm_copy_config_ext_values(struct gsm_mux *gsm,
 				       struct gsm_config_ext *ce)
 {
 	memset(ce, 0, sizeof(*ce));
+	ce->wait_config = gsm->wait_config ? 1 : 0;
 	ce->keep_alive = gsm->keep_alive;
 }
 
@@ -3233,7 +3381,12 @@ static int gsm_config_ext(struct gsm_mux *gsm, struct gsm_config_ext *ce)
 		if (ce->reserved[i])
 			return -EINVAL;
 
+	/*
+	 * Setup the new configuration values
+	 */
+	gsm->wait_config = ce->wait_config ? true : false;
 	gsm->keep_alive = ce->keep_alive;
+
 	return 0;
 }
 
@@ -3433,9 +3586,16 @@ static int gsmld_open(struct tty_struct *tty)
 	tty->receive_room = 65536;
 
 	/* Attach the initial passive connection */
-	gsm->encoding = GSM_ADV_OPT;
 	gsmld_attach_gsm(tty, gsm);
 
+	/* The mux will not be activated yet, we wait for correct
+	 * configuration first.
+	 */
+	if (gsm->encoding == GSM_BASIC_OPT)
+		gsm->receive = gsm0_receive;
+	else
+		gsm->receive = gsm1_receive;
+
 	return 0;
 }
 
@@ -3556,8 +3716,10 @@ static int gsmld_ioctl(struct tty_struct *tty, unsigned int cmd,
 {
 	struct gsm_config c;
 	struct gsm_config_ext ce;
+	struct gsm_dlci_config dc;
 	struct gsm_mux *gsm = tty->disc_data;
-	unsigned int base;
+	unsigned int base, addr;
+	struct gsm_dlci *dlci;
 
 	switch (cmd) {
 	case GSMIOC_GETCONF:
@@ -3581,6 +3743,35 @@ static int gsmld_ioctl(struct tty_struct *tty, unsigned int cmd,
 		if (copy_from_user(&ce, (void __user *)arg, sizeof(ce)))
 			return -EFAULT;
 		return gsm_config_ext(gsm, &ce);
+	case GSMIOC_GETCONF_DLCI:
+		if (copy_from_user(&dc, (void __user *)arg, sizeof(dc)))
+			return -EFAULT;
+		if (dc.channel == 0 || dc.channel >= NUM_DLCI)
+			return -EINVAL;
+		addr = array_index_nospec(dc.channel, NUM_DLCI);
+		dlci = gsm->dlci[addr];
+		if (!dlci) {
+			dlci = gsm_dlci_alloc(gsm, addr);
+			if (!dlci)
+				return -ENOMEM;
+		}
+		gsm_dlci_copy_config_values(dlci, &dc);
+		if (copy_to_user((void __user *)arg, &dc, sizeof(dc)))
+			return -EFAULT;
+		return 0;
+	case GSMIOC_SETCONF_DLCI:
+		if (copy_from_user(&dc, (void __user *)arg, sizeof(dc)))
+			return -EFAULT;
+		if (dc.channel == 0 || dc.channel >= NUM_DLCI)
+			return -EINVAL;
+		addr = array_index_nospec(dc.channel, NUM_DLCI);
+		dlci = gsm->dlci[addr];
+		if (!dlci) {
+			dlci = gsm_dlci_alloc(gsm, addr);
+			if (!dlci)
+				return -ENOMEM;
+		}
+		return gsm_dlci_config(dlci, &dc, 0);
 	default:
 		return n_tty_ioctl_helper(tty, cmd, arg);
 	}
@@ -4008,7 +4199,6 @@ static int gsmtty_open(struct tty_struct *tty, struct file *filp)
 {
 	struct gsm_dlci *dlci = tty->driver_data;
 	struct tty_port *port = &dlci->port;
-	struct gsm_mux *gsm = dlci->gsm;
 
 	port->count++;
 	tty_port_tty_set(port, tty);
@@ -4018,10 +4208,15 @@ static int gsmtty_open(struct tty_struct *tty, struct file *filp)
 	   a DM straight back. This is ok as that will have caused a hangup */
 	tty_port_set_initialized(port, true);
 	/* Start sending off SABM messages */
-	if (gsm->initiator)
-		gsm_dlci_begin_open(dlci);
-	else
-		gsm_dlci_set_opening(dlci);
+	if (!dlci->gsm->wait_config) {
+		/* Start sending off SABM messages */
+		if (dlci->gsm->initiator)
+			gsm_dlci_begin_open(dlci);
+		else
+			gsm_dlci_set_opening(dlci);
+	} else {
+		gsm_dlci_set_wait_config(dlci);
+	}
 	/* And wait for virtual carrier */
 	return tty_port_block_til_ready(port, tty, filp);
 }
@@ -4142,6 +4337,7 @@ static int gsmtty_ioctl(struct tty_struct *tty,
 {
 	struct gsm_dlci *dlci = tty->driver_data;
 	struct gsm_netconfig nc;
+	struct gsm_dlci_config dc;
 	int index;
 
 	if (dlci->state == DLCI_CLOSED)
@@ -4165,6 +4361,23 @@ static int gsmtty_ioctl(struct tty_struct *tty,
 		gsm_destroy_network(dlci);
 		mutex_unlock(&dlci->mutex);
 		return 0;
+	case GSMIOC_GETCONF_DLCI:
+		if (copy_from_user(&dc, (void __user *)arg, sizeof(dc)))
+			return -EFAULT;
+		if (dc.channel != dlci->addr)
+			return -EPERM;
+		gsm_dlci_copy_config_values(dlci, &dc);
+		if (copy_to_user((void __user *)arg, &dc, sizeof(dc)))
+			return -EFAULT;
+		return 0;
+	case GSMIOC_SETCONF_DLCI:
+		if (copy_from_user(&dc, (void __user *)arg, sizeof(dc)))
+			return -EFAULT;
+		if (dc.channel >= NUM_DLCI)
+			return -EINVAL;
+		if (dc.channel != 0 && dc.channel != dlci->addr)
+			return -EPERM;
+		return gsm_dlci_config(dlci, &dc, 1);
 	case TIOCMIWAIT:
 		return gsm_wait_modem_change(dlci, (u32)arg);
 	default:
diff --git a/drivers/tty/n_tty.c b/drivers/tty/n_tty.c
index c8f56c9b1a1c..1c9e5d2ea7de 100644
--- a/drivers/tty/n_tty.c
+++ b/drivers/tty/n_tty.c
@@ -28,27 +28,26 @@
  *		EAGAIN
  */
 
-#include <linux/types.h>
-#include <linux/major.h>
+#include <linux/bitmap.h>
+#include <linux/bitops.h>
+#include <linux/ctype.h>
 #include <linux/errno.h>
-#include <linux/signal.h>
+#include <linux/export.h>
 #include <linux/fcntl.h>
+#include <linux/file.h>
+#include <linux/jiffies.h>
+#include <linux/math.h>
+#include <linux/poll.h>
+#include <linux/ratelimit.h>
 #include <linux/sched.h>
-#include <linux/interrupt.h>
-#include <linux/tty.h>
-#include <linux/timer.h>
-#include <linux/ctype.h>
-#include <linux/mm.h>
-#include <linux/string.h>
+#include <linux/signal.h>
 #include <linux/slab.h>
-#include <linux/poll.h>
-#include <linux/bitops.h>
-#include <linux/audit.h>
-#include <linux/file.h>
+#include <linux/string.h>
+#include <linux/tty.h>
+#include <linux/types.h>
 #include <linux/uaccess.h>
-#include <linux/module.h>
-#include <linux/ratelimit.h>
 #include <linux/vmalloc.h>
+
 #include "tty.h"
 
 /*
@@ -625,7 +624,7 @@ static size_t __process_echoes(struct tty_struct *tty)
 		c = echo_buf(ldata, tail);
 		if (c == ECHO_OP_START) {
 			unsigned char op;
-			int no_space_left = 0;
+			bool space_left = true;
 
 			/*
 			 * Since add_echo_byte() is called without holding
@@ -664,7 +663,7 @@ static size_t __process_echoes(struct tty_struct *tty)
 				num_bs = 8 - (num_chars & 7);
 
 				if (num_bs > space) {
-					no_space_left = 1;
+					space_left = false;
 					break;
 				}
 				space -= num_bs;
@@ -690,7 +689,7 @@ static size_t __process_echoes(struct tty_struct *tty)
 			case ECHO_OP_START:
 				/* This is an escaped echo op start code */
 				if (!space) {
-					no_space_left = 1;
+					space_left = false;
 					break;
 				}
 				tty_put_char(tty, ECHO_OP_START);
@@ -710,7 +709,7 @@ static size_t __process_echoes(struct tty_struct *tty)
 				 *
 				 */
 				if (space < 2) {
-					no_space_left = 1;
+					space_left = false;
 					break;
 				}
 				tty_put_char(tty, '^');
@@ -720,7 +719,7 @@ static size_t __process_echoes(struct tty_struct *tty)
 				tail += 2;
 			}
 
-			if (no_space_left)
+			if (!space_left)
 				break;
 		} else {
 			if (O_OPOST(tty)) {
@@ -1177,7 +1176,7 @@ static void n_tty_receive_overrun(struct tty_struct *tty)
 
 	ldata->num_overrun++;
 	if (time_after(jiffies, ldata->overrun_time + HZ) ||
-			time_after(ldata->overrun_time, jiffies)) {
+	    time_after(ldata->overrun_time, jiffies)) {
 		tty_warn(tty, "%d input overrun(s)\n", ldata->num_overrun);
 		ldata->overrun_time = jiffies;
 		ldata->num_overrun = 0;
@@ -1691,7 +1690,7 @@ n_tty_receive_buf_common(struct tty_struct *tty, const unsigned char *cp,
 
 		room = N_TTY_BUF_SIZE - (ldata->read_head - tail);
 		if (I_PARMRK(tty))
-			room = (room + 2) / 3;
+			room = DIV_ROUND_UP(room, 3);
 		room--;
 		if (room <= 0) {
 			overflow = ldata->icanon && ldata->canon_head == tail;
diff --git a/drivers/tty/pty.c b/drivers/tty/pty.c
index 07394fdaf522..2b1c8ab99dba 100644
--- a/drivers/tty/pty.c
+++ b/drivers/tty/pty.c
@@ -931,7 +931,7 @@ static void __init unix98_pty_init(void)
 	if (cdev_add(&ptmx_cdev, MKDEV(TTYAUX_MAJOR, 2), 1) ||
 	    register_chrdev_region(MKDEV(TTYAUX_MAJOR, 2), 1, "/dev/ptmx") < 0)
 		panic("Couldn't register /dev/ptmx driver");
-	device_create(tty_class, NULL, MKDEV(TTYAUX_MAJOR, 2), NULL, "ptmx");
+	device_create(&tty_class, NULL, MKDEV(TTYAUX_MAJOR, 2), NULL, "ptmx");
 }
 
 #else
diff --git a/drivers/tty/serdev/core.c b/drivers/tty/serdev/core.c
index 678014253b7b..e7d663901c07 100644
--- a/drivers/tty/serdev/core.c
+++ b/drivers/tty/serdev/core.c
@@ -366,7 +366,7 @@ int serdev_device_set_parity(struct serdev_device *serdev,
 	struct serdev_controller *ctrl = serdev->ctrl;
 
 	if (!ctrl || !ctrl->ops->set_parity)
-		return -ENOTSUPP;
+		return -EOPNOTSUPP;
 
 	return ctrl->ops->set_parity(ctrl, parity);
 }
@@ -388,7 +388,7 @@ int serdev_device_get_tiocm(struct serdev_device *serdev)
 	struct serdev_controller *ctrl = serdev->ctrl;
 
 	if (!ctrl || !ctrl->ops->get_tiocm)
-		return -ENOTSUPP;
+		return -EOPNOTSUPP;
 
 	return ctrl->ops->get_tiocm(ctrl);
 }
@@ -399,12 +399,23 @@ int serdev_device_set_tiocm(struct serdev_device *serdev, int set, int clear)
 	struct serdev_controller *ctrl = serdev->ctrl;
 
 	if (!ctrl || !ctrl->ops->set_tiocm)
-		return -ENOTSUPP;
+		return -EOPNOTSUPP;
 
 	return ctrl->ops->set_tiocm(ctrl, set, clear);
 }
 EXPORT_SYMBOL_GPL(serdev_device_set_tiocm);
 
+int serdev_device_break_ctl(struct serdev_device *serdev, int break_state)
+{
+	struct serdev_controller *ctrl = serdev->ctrl;
+
+	if (!ctrl || !ctrl->ops->break_ctl)
+		return -EOPNOTSUPP;
+
+	return ctrl->ops->break_ctl(ctrl, break_state);
+}
+EXPORT_SYMBOL_GPL(serdev_device_break_ctl);
+
 static int serdev_drv_probe(struct device *dev)
 {
 	const struct serdev_device_driver *sdrv = to_serdev_device_driver(dev->driver);
diff --git a/drivers/tty/serdev/serdev-ttyport.c b/drivers/tty/serdev/serdev-ttyport.c
index d367803e2044..8033ef19669c 100644
--- a/drivers/tty/serdev/serdev-ttyport.c
+++ b/drivers/tty/serdev/serdev-ttyport.c
@@ -231,7 +231,7 @@ static int ttyport_get_tiocm(struct serdev_controller *ctrl)
 	struct tty_struct *tty = serport->tty;
 
 	if (!tty->ops->tiocmget)
-		return -ENOTSUPP;
+		return -EOPNOTSUPP;
 
 	return tty->ops->tiocmget(tty);
 }
@@ -242,11 +242,22 @@ static int ttyport_set_tiocm(struct serdev_controller *ctrl, unsigned int set, u
 	struct tty_struct *tty = serport->tty;
 
 	if (!tty->ops->tiocmset)
-		return -ENOTSUPP;
+		return -EOPNOTSUPP;
 
 	return tty->ops->tiocmset(tty, set, clear);
 }
 
+static int ttyport_break_ctl(struct serdev_controller *ctrl, unsigned int break_state)
+{
+	struct serport *serport = serdev_controller_get_drvdata(ctrl);
+	struct tty_struct *tty = serport->tty;
+
+	if (!tty->ops->break_ctl)
+		return -EOPNOTSUPP;
+
+	return tty->ops->break_ctl(tty, break_state);
+}
+
 static const struct serdev_controller_ops ctrl_ops = {
 	.write_buf = ttyport_write_buf,
 	.write_flush = ttyport_write_flush,
@@ -259,6 +270,7 @@ static const struct serdev_controller_ops ctrl_ops = {
 	.wait_until_sent = ttyport_wait_until_sent,
 	.get_tiocm = ttyport_get_tiocm,
 	.set_tiocm = ttyport_set_tiocm,
+	.break_ctl = ttyport_break_ctl,
 };
 
 struct device *serdev_tty_port_register(struct tty_port *port,
diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h
index 287153d32536..1e8fe44a7099 100644
--- a/drivers/tty/serial/8250/8250.h
+++ b/drivers/tty/serial/8250/8250.h
@@ -365,6 +365,13 @@ static inline void serial8250_do_prepare_rx_dma(struct uart_8250_port *p)
 	if (dma->prepare_rx_dma)
 		dma->prepare_rx_dma(p);
 }
+
+static inline bool serial8250_tx_dma_running(struct uart_8250_port *p)
+{
+	struct uart_8250_dma *dma = p->dma;
+
+	return dma && dma->tx_running;
+}
 #else
 static inline int serial8250_tx_dma(struct uart_8250_port *p)
 {
@@ -380,6 +387,11 @@ static inline int serial8250_request_dma(struct uart_8250_port *p)
 	return -1;
 }
 static inline void serial8250_release_dma(struct uart_8250_port *p) { }
+
+static inline bool serial8250_tx_dma_running(struct uart_8250_port *p)
+{
+	return false;
+}
 #endif
 
 static inline int ns16550a_goto_highspeed(struct uart_8250_port *up)
diff --git a/drivers/tty/serial/8250/8250_bcm7271.c b/drivers/tty/serial/8250/8250_bcm7271.c
index ed5a94747692..f801b1f5b46c 100644
--- a/drivers/tty/serial/8250/8250_bcm7271.c
+++ b/drivers/tty/serial/8250/8250_bcm7271.c
@@ -1014,14 +1014,16 @@ static int brcmuart_probe(struct platform_device *pdev)
 	/* See if a Baud clock has been specified */
 	baud_mux_clk = of_clk_get_by_name(np, "sw_baud");
 	if (IS_ERR(baud_mux_clk)) {
-		if (PTR_ERR(baud_mux_clk) == -EPROBE_DEFER)
-			return -EPROBE_DEFER;
+		if (PTR_ERR(baud_mux_clk) == -EPROBE_DEFER) {
+			ret = -EPROBE_DEFER;
+			goto release_dma;
+		}
 		dev_dbg(dev, "BAUD MUX clock not specified\n");
 	} else {
 		dev_dbg(dev, "BAUD MUX clock found\n");
 		ret = clk_prepare_enable(baud_mux_clk);
 		if (ret)
-			return ret;
+			goto release_dma;
 		priv->baud_mux_clk = baud_mux_clk;
 		init_real_clk_rates(dev, priv);
 		clk_rate = priv->default_mux_rate;
@@ -1029,7 +1031,8 @@ static int brcmuart_probe(struct platform_device *pdev)
 
 	if (clk_rate == 0) {
 		dev_err(dev, "clock-frequency or clk not defined\n");
-		return -EINVAL;
+		ret = -EINVAL;
+		goto release_dma;
 	}
 
 	dev_dbg(dev, "DMA is %senabled\n", priv->dma_enabled ? "" : "not ");
@@ -1116,7 +1119,9 @@ err1:
 	serial8250_unregister_port(priv->line);
 err:
 	brcmuart_free_bufs(dev, priv);
-	brcmuart_arbitration(priv, 0);
+release_dma:
+	if (priv->dma_enabled)
+		brcmuart_arbitration(priv, 0);
 	return ret;
 }
 
@@ -1128,7 +1133,8 @@ static int brcmuart_remove(struct platform_device *pdev)
 	hrtimer_cancel(&priv->hrt);
 	serial8250_unregister_port(priv->line);
 	brcmuart_free_bufs(&pdev->dev, priv);
-	brcmuart_arbitration(priv, 0);
+	if (priv->dma_enabled)
+		brcmuart_arbitration(priv, 0);
 	return 0;
 }
 
diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c
index ab63c308be0a..13bf535eedcd 100644
--- a/drivers/tty/serial/8250/8250_core.c
+++ b/drivers/tty/serial/8250/8250_core.c
@@ -1158,6 +1158,7 @@ void serial8250_unregister_port(int line)
 		uart->port.type = PORT_UNKNOWN;
 		uart->port.dev = &serial8250_isa_devs->dev;
 		uart->capabilities = 0;
+		serial8250_init_port(uart);
 		serial8250_apply_quirks(uart);
 		uart_add_one_port(&serial8250_reg, &uart->port);
 	} else {
diff --git a/drivers/tty/serial/8250/8250_em.c b/drivers/tty/serial/8250/8250_em.c
index d94c3811a8f7..25a9ecf26be6 100644
--- a/drivers/tty/serial/8250/8250_em.c
+++ b/drivers/tty/serial/8250/8250_em.c
@@ -13,36 +13,49 @@
 #include <linux/serial_reg.h>
 #include <linux/platform_device.h>
 #include <linux/clk.h>
-#include <linux/slab.h>
 
 #include "8250.h"
 
 #define UART_DLL_EM 9
 #define UART_DLM_EM 10
+#define UART_HCR0_EM 11
+
+/*
+ * A high value for UART_FCR_EM avoids overlapping with existing UART_*
+ * register defines. UART_FCR_EM_HW is the real HW register offset.
+ */
+#define UART_FCR_EM 0x10003
+#define UART_FCR_EM_HW 3
+
+#define UART_HCR0_EM_SW_RESET	BIT(7) /* SW Reset */
 
 struct serial8250_em_priv {
-	struct clk *sclk;
 	int line;
 };
 
-static void serial8250_em_serial_out(struct uart_port *p, int offset, int value)
+static void serial8250_em_serial_out_helper(struct uart_port *p, int offset,
+					    int value)
 {
 	switch (offset) {
 	case UART_TX: /* TX @ 0x00 */
 		writeb(value, p->membase);
 		break;
-	case UART_FCR: /* FCR @ 0x0c (+1) */
 	case UART_LCR: /* LCR @ 0x10 (+1) */
 	case UART_MCR: /* MCR @ 0x14 (+1) */
 	case UART_SCR: /* SCR @ 0x20 (+1) */
 		writel(value, p->membase + ((offset + 1) << 2));
 		break;
+	case UART_FCR_EM:
+		writel(value, p->membase + (UART_FCR_EM_HW << 2));
+		break;
 	case UART_IER: /* IER @ 0x04 */
 		value &= 0x0f; /* only 4 valid bits - not Xscale */
 		fallthrough;
 	case UART_DLL_EM: /* DLL @ 0x24 (+9) */
 	case UART_DLM_EM: /* DLM @ 0x28 (+9) */
+	case UART_HCR0_EM: /* HCR0 @ 0x2c */
 		writel(value, p->membase + (offset << 2));
+		break;
 	}
 }
 
@@ -51,20 +64,81 @@ static unsigned int serial8250_em_serial_in(struct uart_port *p, int offset)
 	switch (offset) {
 	case UART_RX: /* RX @ 0x00 */
 		return readb(p->membase);
+	case UART_LCR: /* LCR @ 0x10 (+1) */
 	case UART_MCR: /* MCR @ 0x14 (+1) */
 	case UART_LSR: /* LSR @ 0x18 (+1) */
 	case UART_MSR: /* MSR @ 0x1c (+1) */
 	case UART_SCR: /* SCR @ 0x20 (+1) */
 		return readl(p->membase + ((offset + 1) << 2));
+	case UART_FCR_EM:
+		return readl(p->membase + (UART_FCR_EM_HW << 2));
 	case UART_IER: /* IER @ 0x04 */
 	case UART_IIR: /* IIR @ 0x08 */
 	case UART_DLL_EM: /* DLL @ 0x24 (+9) */
 	case UART_DLM_EM: /* DLM @ 0x28 (+9) */
+	case UART_HCR0_EM: /* HCR0 @ 0x2c */
 		return readl(p->membase + (offset << 2));
 	}
 	return 0;
 }
 
+static void serial8250_em_reg_update(struct uart_port *p, int off, int value)
+{
+	unsigned int ier, fcr, lcr, mcr, hcr0;
+
+	ier = serial8250_em_serial_in(p, UART_IER);
+	fcr = serial8250_em_serial_in(p, UART_FCR_EM);
+	lcr = serial8250_em_serial_in(p, UART_LCR);
+	mcr = serial8250_em_serial_in(p, UART_MCR);
+	hcr0 = serial8250_em_serial_in(p, UART_HCR0_EM);
+
+	serial8250_em_serial_out_helper(p, UART_FCR_EM, fcr |
+							UART_FCR_CLEAR_RCVR |
+							UART_FCR_CLEAR_XMIT);
+	serial8250_em_serial_out_helper(p, UART_HCR0_EM, hcr0 |
+							 UART_HCR0_EM_SW_RESET);
+	serial8250_em_serial_out_helper(p, UART_HCR0_EM, hcr0 &
+							 ~UART_HCR0_EM_SW_RESET);
+
+	switch (off) {
+	case UART_FCR_EM:
+		fcr = value;
+		break;
+	case UART_LCR:
+		lcr = value;
+		break;
+	case UART_MCR:
+		mcr = value;
+		break;
+	}
+
+	serial8250_em_serial_out_helper(p, UART_IER, ier);
+	serial8250_em_serial_out_helper(p, UART_FCR_EM, fcr);
+	serial8250_em_serial_out_helper(p, UART_MCR, mcr);
+	serial8250_em_serial_out_helper(p, UART_LCR, lcr);
+	serial8250_em_serial_out_helper(p, UART_HCR0_EM, hcr0);
+}
+
+static void serial8250_em_serial_out(struct uart_port *p, int offset, int value)
+{
+	switch (offset) {
+	case UART_TX:
+	case UART_SCR:
+	case UART_IER:
+	case UART_DLL_EM:
+	case UART_DLM_EM:
+		serial8250_em_serial_out_helper(p, offset, value);
+		break;
+	case UART_FCR:
+		serial8250_em_reg_update(p, UART_FCR_EM, value);
+		break;
+	case UART_LCR:
+	case UART_MCR:
+		serial8250_em_reg_update(p, offset, value);
+		break;
+	}
+}
+
 static int serial8250_em_serial_dl_read(struct uart_8250_port *up)
 {
 	return serial_in(up, UART_DLL_EM) | serial_in(up, UART_DLM_EM) << 8;
@@ -79,8 +153,10 @@ static void serial8250_em_serial_dl_write(struct uart_8250_port *up, int value)
 static int serial8250_em_probe(struct platform_device *pdev)
 {
 	struct serial8250_em_priv *priv;
+	struct device *dev = &pdev->dev;
 	struct uart_8250_port up;
 	struct resource *regs;
+	struct clk *sclk;
 	int irq, ret;
 
 	irq = platform_get_irq(pdev, 0);
@@ -88,31 +164,26 @@ static int serial8250_em_probe(struct platform_device *pdev)
 		return irq;
 
 	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!regs) {
-		dev_err(&pdev->dev, "missing registers\n");
-		return -EINVAL;
-	}
+	if (!regs)
+		return dev_err_probe(dev, -EINVAL, "missing registers\n");
 
-	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	priv->sclk = devm_clk_get(&pdev->dev, "sclk");
-	if (IS_ERR(priv->sclk)) {
-		dev_err(&pdev->dev, "unable to get clock\n");
-		return PTR_ERR(priv->sclk);
-	}
+	sclk = devm_clk_get_enabled(dev, "sclk");
+	if (IS_ERR(sclk))
+		return dev_err_probe(dev, PTR_ERR(sclk), "unable to get clock\n");
 
 	memset(&up, 0, sizeof(up));
 	up.port.mapbase = regs->start;
 	up.port.irq = irq;
 	up.port.type = PORT_16750;
 	up.port.flags = UPF_FIXED_PORT | UPF_IOREMAP | UPF_FIXED_TYPE;
-	up.port.dev = &pdev->dev;
+	up.port.dev = dev;
 	up.port.private_data = priv;
 
-	clk_prepare_enable(priv->sclk);
-	up.port.uartclk = clk_get_rate(priv->sclk);
+	up.port.uartclk = clk_get_rate(sclk);
 
 	up.port.iotype = UPIO_MEM32;
 	up.port.serial_in = serial8250_em_serial_in;
@@ -121,11 +192,8 @@ static int serial8250_em_probe(struct platform_device *pdev)
 	up.dl_write = serial8250_em_serial_dl_write;
 
 	ret = serial8250_register_8250_port(&up);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "unable to register 8250 port\n");
-		clk_disable_unprepare(priv->sclk);
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "unable to register 8250 port\n");
 
 	priv->line = ret;
 	platform_set_drvdata(pdev, priv);
@@ -137,7 +205,6 @@ static int serial8250_em_remove(struct platform_device *pdev)
 	struct serial8250_em_priv *priv = platform_get_drvdata(pdev);
 
 	serial8250_unregister_port(priv->line);
-	clk_disable_unprepare(priv->sclk);
 	return 0;
 }
 
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index 3ba9c8b93ae6..fe8d79c4ae95 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -15,6 +15,7 @@
 #include <linux/moduleparam.h>
 #include <linux/ioport.h>
 #include <linux/init.h>
+#include <linux/irq.h>
 #include <linux/console.h>
 #include <linux/gpio/consumer.h>
 #include <linux/sysrq.h>
@@ -1932,6 +1933,7 @@ static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir)
 int serial8250_handle_irq(struct uart_port *port, unsigned int iir)
 {
 	struct uart_8250_port *up = up_to_u8250p(port);
+	struct tty_port *tport = &port->state->port;
 	bool skip_rx = false;
 	unsigned long flags;
 	u16 status;
@@ -1957,6 +1959,8 @@ int serial8250_handle_irq(struct uart_port *port, unsigned int iir)
 		skip_rx = true;
 
 	if (status & (UART_LSR_DR | UART_LSR_BI) && !skip_rx) {
+		if (irqd_is_wakeup_set(irq_get_irq_data(port->irq)))
+			pm_wakeup_event(tport->tty->dev, 0);
 		if (!up->dma || handle_rx_dma(up, iir))
 			status = serial8250_rx_chars(up, status);
 	}
@@ -2016,18 +2020,19 @@ static int serial8250_tx_threshold_handle_irq(struct uart_port *port)
 static unsigned int serial8250_tx_empty(struct uart_port *port)
 {
 	struct uart_8250_port *up = up_to_u8250p(port);
+	unsigned int result = 0;
 	unsigned long flags;
-	u16 lsr;
 
 	serial8250_rpm_get(up);
 
 	spin_lock_irqsave(&port->lock, flags);
-	lsr = serial_lsr_in(up);
+	if (!serial8250_tx_dma_running(up) && uart_lsr_tx_empty(serial_lsr_in(up)))
+		result = TIOCSER_TEMT;
 	spin_unlock_irqrestore(&port->lock, flags);
 
 	serial8250_rpm_put(up);
 
-	return uart_lsr_tx_empty(lsr) ? TIOCSER_TEMT : 0;
+	return result;
 }
 
 unsigned int serial8250_do_get_mctrl(struct uart_port *port)
diff --git a/drivers/tty/serial/8250/8250_tegra.c b/drivers/tty/serial/8250/8250_tegra.c
index e7cddeec9d8e..2509e7f74ccf 100644
--- a/drivers/tty/serial/8250/8250_tegra.c
+++ b/drivers/tty/serial/8250/8250_tegra.c
@@ -11,6 +11,7 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/reset.h>
 #include <linux/slab.h>
 
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index 0072892ca7fc..398e5aac2e77 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -248,13 +248,6 @@ config SERIAL_SAMSUNG
 
 	  Choose Y/M here only if you build for such SoC.
 
-config SERIAL_SAMSUNG_UARTS_4
-	bool
-	depends on SERIAL_SAMSUNG
-	default y
-	help
-	  Internal node for the common case of 4 Samsung compatible UARTs
-
 config SERIAL_SAMSUNG_UARTS
 	int
 	depends on SERIAL_SAMSUNG
@@ -958,7 +951,7 @@ config SERIAL_OMAP_CONSOLE
 config SERIAL_SIFIVE
 	tristate "SiFive UART support"
 	depends on OF
-	default SOC_SIFIVE || SOC_CANAAN
+	default ARCH_SIFIVE || ARCH_CANAAN
 	select SERIAL_CORE
 	help
 	  Select this option if you are building a kernel for a device that
@@ -968,7 +961,7 @@ config SERIAL_SIFIVE
 config SERIAL_SIFIVE_CONSOLE
 	bool "Console on SiFive UART"
 	depends on SERIAL_SIFIVE=y
-	default SOC_SIFIVE || SOC_CANAAN
+	default ARCH_SIFIVE || ARCH_CANAAN
 	select SERIAL_CORE_CONSOLE
 	select SERIAL_EARLYCON
 	help
diff --git a/drivers/tty/serial/bcm63xx_uart.c b/drivers/tty/serial/bcm63xx_uart.c
index 62bc7244dc67..55e82d0bf92d 100644
--- a/drivers/tty/serial/bcm63xx_uart.c
+++ b/drivers/tty/serial/bcm63xx_uart.c
@@ -596,6 +596,40 @@ static int bcm_uart_verify_port(struct uart_port *port,
 	return 0;
 }
 
+#ifdef CONFIG_CONSOLE_POLL
+/*
+ * return true when outstanding tx equals fifo size
+ */
+static bool bcm_uart_tx_full(struct uart_port *port)
+{
+	unsigned int val;
+
+	val = bcm_uart_readl(port, UART_MCTL_REG);
+	val = (val & UART_MCTL_TXFIFOFILL_MASK) >> UART_MCTL_TXFIFOFILL_SHIFT;
+	return !(port->fifosize - val);
+}
+
+static int bcm_uart_poll_get_char(struct uart_port *port)
+{
+	unsigned int iestat;
+
+	iestat = bcm_uart_readl(port, UART_IR_REG);
+	if (!(iestat & UART_IR_STAT(UART_IR_RXNOTEMPTY)))
+		return NO_POLL_CHAR;
+
+	return bcm_uart_readl(port, UART_FIFO_REG);
+}
+
+static void bcm_uart_poll_put_char(struct uart_port *port, unsigned char c)
+{
+	while (bcm_uart_tx_full(port)) {
+		cpu_relax();
+	}
+
+	bcm_uart_writel(port, c, UART_FIFO_REG);
+}
+#endif
+
 /* serial core callbacks */
 static const struct uart_ops bcm_uart_ops = {
 	.tx_empty	= bcm_uart_tx_empty,
@@ -614,6 +648,10 @@ static const struct uart_ops bcm_uart_ops = {
 	.request_port	= bcm_uart_request_port,
 	.config_port	= bcm_uart_config_port,
 	.verify_port	= bcm_uart_verify_port,
+#ifdef CONFIG_CONSOLE_POLL
+	.poll_get_char  = bcm_uart_poll_get_char,
+	.poll_put_char  = bcm_uart_poll_put_char,
+#endif
 };
 
 
diff --git a/drivers/tty/serial/cpm_uart/cpm_uart_core.c b/drivers/tty/serial/cpm_uart/cpm_uart_core.c
index 5565f302cb21..349e7da643f0 100644
--- a/drivers/tty/serial/cpm_uart/cpm_uart_core.c
+++ b/drivers/tty/serial/cpm_uart/cpm_uart_core.c
@@ -678,15 +678,14 @@ static int cpm_uart_tx_pump(struct uart_port *port)
 	/* Pick next descriptor and fill from buffer */
 	bdp = pinfo->tx_cur;
 
-	while (!(in_be16(&bdp->cbd_sc) & BD_SC_READY) &&
-	       xmit->tail != xmit->head) {
+	while (!(in_be16(&bdp->cbd_sc) & BD_SC_READY) && !uart_circ_empty(xmit)) {
 		count = 0;
 		p = cpm2cpu_addr(in_be32(&bdp->cbd_bufaddr), pinfo);
 		while (count < pinfo->tx_fifosize) {
 			*p++ = xmit->buf[xmit->tail];
 			uart_xmit_advance(port, 1);
 			count++;
-			if (xmit->head == xmit->tail)
+			if (uart_circ_empty(xmit))
 				break;
 		}
 		out_be16(&bdp->cbd_datlen, count);
diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c
index 074bfed57fc9..c91916e13648 100644
--- a/drivers/tty/serial/fsl_lpuart.c
+++ b/drivers/tty/serial/fsl_lpuart.c
@@ -1296,7 +1296,7 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
 	 * 10ms at any baud rate.
 	 */
 	sport->rx_dma_rng_buf_len = (DMA_RX_TIMEOUT * baud /  bits / 1000) * 2;
-	sport->rx_dma_rng_buf_len = (1 << (fls(sport->rx_dma_rng_buf_len) - 1));
+	sport->rx_dma_rng_buf_len = (1 << fls(sport->rx_dma_rng_buf_len));
 	if (sport->rx_dma_rng_buf_len < 16)
 		sport->rx_dma_rng_buf_len = 16;
 
@@ -1406,12 +1406,12 @@ static int lpuart32_config_rs485(struct uart_port *port, struct ktermios *termio
 			struct lpuart_port, port);
 
 	unsigned long modem = lpuart32_read(&sport->port, UARTMODIR)
-				& ~(UARTMODEM_TXRTSPOL | UARTMODEM_TXRTSE);
+				& ~(UARTMODIR_TXRTSPOL | UARTMODIR_TXRTSE);
 	lpuart32_write(&sport->port, modem, UARTMODIR);
 
 	if (rs485->flags & SER_RS485_ENABLED) {
 		/* Enable auto RS-485 RTS mode */
-		modem |= UARTMODEM_TXRTSE;
+		modem |= UARTMODIR_TXRTSE;
 
 		/*
 		 * The hardware defaults to RTS logic HIGH while transfer.
@@ -1420,9 +1420,9 @@ static int lpuart32_config_rs485(struct uart_port *port, struct ktermios *termio
 		 * Note: UART is assumed to be active high.
 		 */
 		if (rs485->flags & SER_RS485_RTS_ON_SEND)
-			modem |= UARTMODEM_TXRTSPOL;
+			modem |= UARTMODIR_TXRTSPOL;
 		else if (rs485->flags & SER_RS485_RTS_AFTER_SEND)
-			modem &= ~UARTMODEM_TXRTSPOL;
+			modem &= ~UARTMODIR_TXRTSPOL;
 	}
 
 	lpuart32_write(&sport->port, modem, UARTMODIR);
diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c
index 523f296d5747..c5e17569c3ad 100644
--- a/drivers/tty/serial/imx.c
+++ b/drivers/tty/serial/imx.c
@@ -289,20 +289,6 @@ static inline int imx_uart_is_imx1(struct imx_port *sport)
 	return sport->devdata->devtype == IMX1_UART;
 }
 
-static inline int imx_uart_is_imx21(struct imx_port *sport)
-{
-	return sport->devdata->devtype == IMX21_UART;
-}
-
-static inline int imx_uart_is_imx53(struct imx_port *sport)
-{
-	return sport->devdata->devtype == IMX53_UART;
-}
-
-static inline int imx_uart_is_imx6q(struct imx_port *sport)
-{
-	return sport->devdata->devtype == IMX6Q_UART;
-}
 /*
  * Save and restore functions for UCR1, UCR2 and UCR3 registers
  */
@@ -1808,9 +1794,7 @@ imx_uart_set_termios(struct uart_port *port, struct ktermios *termios,
 
 static const char *imx_uart_type(struct uart_port *port)
 {
-	struct imx_port *sport = (struct imx_port *)port;
-
-	return sport->port.type == PORT_IMX ? "IMX" : NULL;
+	return port->type == PORT_IMX ? "IMX" : NULL;
 }
 
 /*
@@ -1818,10 +1802,8 @@ static const char *imx_uart_type(struct uart_port *port)
  */
 static void imx_uart_config_port(struct uart_port *port, int flags)
 {
-	struct imx_port *sport = (struct imx_port *)port;
-
 	if (flags & UART_CONFIG_TYPE)
-		sport->port.type = PORT_IMX;
+		port->type = PORT_IMX;
 }
 
 /*
@@ -1832,20 +1814,19 @@ static void imx_uart_config_port(struct uart_port *port, int flags)
 static int
 imx_uart_verify_port(struct uart_port *port, struct serial_struct *ser)
 {
-	struct imx_port *sport = (struct imx_port *)port;
 	int ret = 0;
 
 	if (ser->type != PORT_UNKNOWN && ser->type != PORT_IMX)
 		ret = -EINVAL;
-	if (sport->port.irq != ser->irq)
+	if (port->irq != ser->irq)
 		ret = -EINVAL;
 	if (ser->io_type != UPIO_MEM)
 		ret = -EINVAL;
-	if (sport->port.uartclk / 16 != ser->baud_base)
+	if (port->uartclk / 16 != ser->baud_base)
 		ret = -EINVAL;
-	if (sport->port.mapbase != (unsigned long)ser->iomem_base)
+	if (port->mapbase != (unsigned long)ser->iomem_base)
 		ret = -EINVAL;
-	if (sport->port.iobase != ser->port)
+	if (port->iobase != ser->port)
 		ret = -EINVAL;
 	if (ser->hub6 != 0)
 		ret = -EINVAL;
@@ -2262,21 +2243,16 @@ static int imx_uart_probe(struct platform_device *pdev)
 	}
 	sport->port.line = ret;
 
-	if (of_get_property(np, "uart-has-rtscts", NULL) ||
-	    of_get_property(np, "fsl,uart-has-rtscts", NULL) /* deprecated */)
-		sport->have_rtscts = 1;
+	sport->have_rtscts = of_property_read_bool(np, "uart-has-rtscts") ||
+		of_property_read_bool(np, "fsl,uart-has-rtscts"); /* deprecated */
 
-	if (of_get_property(np, "fsl,dte-mode", NULL))
-		sport->dte_mode = 1;
+	sport->dte_mode = of_property_read_bool(np, "fsl,dte-mode");
 
-	if (of_get_property(np, "rts-gpios", NULL))
-		sport->have_rtsgpio = 1;
+	sport->have_rtsgpio = of_property_present(np, "rts-gpios");
 
-	if (of_get_property(np, "fsl,inverted-tx", NULL))
-		sport->inverted_tx = 1;
+	sport->inverted_tx = of_property_read_bool(np, "fsl,inverted-tx");
 
-	if (of_get_property(np, "fsl,inverted-rx", NULL))
-		sport->inverted_rx = 1;
+	sport->inverted_rx = of_property_read_bool(np, "fsl,inverted-rx");
 
 	if (!of_property_read_u32_array(np, "fsl,dma-info", dma_buf_conf, 2)) {
 		sport->rx_period_length = dma_buf_conf[0];
diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c
index e9cacfe7e032..9fee722058f4 100644
--- a/drivers/tty/serial/max310x.c
+++ b/drivers/tty/serial/max310x.c
@@ -525,6 +525,11 @@ static bool max310x_reg_precious(struct device *dev, unsigned int reg)
 	return false;
 }
 
+static bool max310x_reg_noinc(struct device *dev, unsigned int reg)
+{
+	return reg == MAX310X_RHR_REG;
+}
+
 static int max310x_set_baud(struct uart_port *port, int baud)
 {
 	unsigned int mode = 0, div = 0, frac = 0, c = 0, F = 0;
@@ -651,14 +656,14 @@ static void max310x_batch_write(struct uart_port *port, u8 *txbuf, unsigned int
 {
 	struct max310x_one *one = to_max310x_port(port);
 
-	regmap_raw_write(one->regmap, MAX310X_THR_REG, txbuf, len);
+	regmap_noinc_write(one->regmap, MAX310X_THR_REG, txbuf, len);
 }
 
 static void max310x_batch_read(struct uart_port *port, u8 *rxbuf, unsigned int len)
 {
 	struct max310x_one *one = to_max310x_port(port);
 
-	regmap_raw_read(one->regmap, MAX310X_RHR_REG, rxbuf, len);
+	regmap_noinc_read(one->regmap, MAX310X_RHR_REG, rxbuf, len);
 }
 
 static void max310x_handle_rx(struct uart_port *port, unsigned int rxlen)
@@ -1468,6 +1473,10 @@ static struct regmap_config regcfg = {
 	.writeable_reg = max310x_reg_writeable,
 	.volatile_reg = max310x_reg_volatile,
 	.precious_reg = max310x_reg_precious,
+	.writeable_noinc_reg = max310x_reg_noinc,
+	.readable_noinc_reg = max310x_reg_noinc,
+	.max_raw_read = MAX310X_FIFO_SIZE,
+	.max_raw_write = MAX310X_FIFO_SIZE,
 };
 
 #ifdef CONFIG_SPI_MASTER
@@ -1553,6 +1562,10 @@ static struct regmap_config regcfg_i2c = {
 	.volatile_reg = max310x_reg_volatile,
 	.precious_reg = max310x_reg_precious,
 	.max_register = MAX310X_I2C_REVID_EXTREG,
+	.writeable_noinc_reg = max310x_reg_noinc,
+	.readable_noinc_reg = max310x_reg_noinc,
+	.max_raw_read = MAX310X_FIFO_SIZE,
+	.max_raw_write = MAX310X_FIFO_SIZE,
 };
 
 static const struct max310x_if_cfg max310x_i2c_if_cfg = {
diff --git a/drivers/tty/serial/meson_uart.c b/drivers/tty/serial/meson_uart.c
index 74110017988a..2501db5a7aaf 100644
--- a/drivers/tty/serial/meson_uart.c
+++ b/drivers/tty/serial/meson_uart.c
@@ -779,7 +779,7 @@ static int meson_uart_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct meson_uart_data s4_uart_data = {
+static struct meson_uart_data meson_g12a_uart_data = {
 	.has_xtal_div2 = true,
 };
 
@@ -789,8 +789,12 @@ static const struct of_device_id meson_uart_dt_match[] = {
 	{ .compatible = "amlogic,meson8b-uart" },
 	{ .compatible = "amlogic,meson-gx-uart" },
 	{
+		.compatible = "amlogic,meson-g12a-uart",
+		.data = (void *)&meson_g12a_uart_data,
+	},
+	{
 		.compatible = "amlogic,meson-s4-uart",
-		.data = (void *)&s4_uart_data,
+		.data = (void *)&meson_g12a_uart_data,
 	},
 	{ /* sentinel */ },
 };
diff --git a/drivers/tty/serial/mxs-auart.c b/drivers/tty/serial/mxs-auart.c
index ef6e7bb6105c..a368f4293967 100644
--- a/drivers/tty/serial/mxs-auart.c
+++ b/drivers/tty/serial/mxs-auart.c
@@ -1587,8 +1587,8 @@ static int mxs_auart_probe(struct platform_device *pdev)
 	}
 	s->port.line = ret;
 
-	if (of_get_property(np, "uart-has-rtscts", NULL) ||
-	    of_get_property(np, "fsl,uart-has-rtscts", NULL) /* deprecated */)
+	if (of_property_read_bool(np, "uart-has-rtscts") ||
+	    of_property_read_bool(np, "fsl,uart-has-rtscts") /* deprecated */)
 		set_bit(MXS_AUART_RTSCTS, &s->flags);
 
 	if (s->port.line >= ARRAY_SIZE(auart_port)) {
diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c
index 28fbc927a546..08dc3e2a729c 100644
--- a/drivers/tty/serial/qcom_geni_serial.c
+++ b/drivers/tty/serial/qcom_geni_serial.c
@@ -854,21 +854,19 @@ static void qcom_geni_serial_stop_tx(struct uart_port *uport)
 }
 
 static void qcom_geni_serial_send_chunk_fifo(struct uart_port *uport,
-					     unsigned int chunk)
+					     unsigned int remaining)
 {
 	struct qcom_geni_serial_port *port = to_dev_port(uport);
 	struct circ_buf *xmit = &uport->state->xmit;
-	unsigned int tx_bytes, c, remaining = chunk;
+	unsigned int tx_bytes;
 	u8 buf[BYTES_PER_FIFO_WORD];
 
 	while (remaining) {
 		memset(buf, 0, sizeof(buf));
 		tx_bytes = min(remaining, BYTES_PER_FIFO_WORD);
 
-		for (c = 0; c < tx_bytes ; c++) {
-			buf[c] = xmit->buf[xmit->tail];
-			uart_xmit_advance(uport, 1);
-		}
+		memcpy(buf, &xmit->buf[xmit->tail], tx_bytes);
+		uart_xmit_advance(uport, tx_bytes);
 
 		iowrite32_rep(uport->membase + SE_GENI_TX_FIFOn, buf, 1);
 
@@ -1535,6 +1533,7 @@ static const struct uart_ops qcom_geni_console_pops = {
 #ifdef CONFIG_CONSOLE_POLL
 	.poll_get_char	= qcom_geni_serial_get_char,
 	.poll_put_char	= qcom_geni_serial_poll_put_char,
+	.poll_init = qcom_geni_serial_port_setup,
 #endif
 	.pm = qcom_geni_serial_pm,
 };
diff --git a/drivers/tty/serial/sb1250-duart.c b/drivers/tty/serial/sb1250-duart.c
index de56f383964e..b6de0dc51f29 100644
--- a/drivers/tty/serial/sb1250-duart.c
+++ b/drivers/tty/serial/sb1250-duart.c
@@ -41,7 +41,7 @@
 #include <asm/sibyte/swarm.h>
 
 
-#if defined(CONFIG_SIBYTE_BCM1x55) || defined(CONFIG_SIBYTE_BCM1x80)
+#if defined(CONFIG_SIBYTE_BCM1x80)
 #include <asm/sibyte/bcm1480_regs.h>
 #include <asm/sibyte/bcm1480_int.h>
 
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index 2bd32c8ece39..54e82f476a2c 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -15,6 +15,7 @@
 #include <linux/init.h>
 #include <linux/console.h>
 #include <linux/gpio/consumer.h>
+#include <linux/kernel.h>
 #include <linux/of.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
@@ -48,9 +49,6 @@ static struct lock_class_key port_lock_key;
  */
 #define RS485_MAX_RTS_DELAY	100 /* msecs */
 
-static void uart_change_speed(struct tty_struct *tty, struct uart_state *state,
-			      const struct ktermios *old_termios);
-static void uart_wait_until_sent(struct tty_struct *tty, int timeout);
 static void uart_change_pm(struct uart_state *state,
 			   enum uart_pm_state pm_state);
 
@@ -137,7 +135,7 @@ static void __uart_start(struct tty_struct *tty)
 	struct uart_state *state = tty->driver_data;
 	struct uart_port *port = state->uart_port;
 
-	if (port && !uart_tx_stopped(port))
+	if (port && !(port->flags & UPF_DEAD) && !uart_tx_stopped(port))
 		port->ops->start_tx(port);
 }
 
@@ -177,6 +175,51 @@ static void uart_port_dtr_rts(struct uart_port *uport, bool active)
 		uart_clear_mctrl(uport, TIOCM_DTR | TIOCM_RTS);
 }
 
+/* Caller holds port mutex */
+static void uart_change_line_settings(struct tty_struct *tty, struct uart_state *state,
+				      const struct ktermios *old_termios)
+{
+	struct uart_port *uport = uart_port_check(state);
+	struct ktermios *termios;
+	bool old_hw_stopped;
+
+	/*
+	 * If we have no tty, termios, or the port does not exist,
+	 * then we can't set the parameters for this port.
+	 */
+	if (!tty || uport->type == PORT_UNKNOWN)
+		return;
+
+	termios = &tty->termios;
+	uport->ops->set_termios(uport, termios, old_termios);
+
+	/*
+	 * Set modem status enables based on termios cflag
+	 */
+	spin_lock_irq(&uport->lock);
+	if (termios->c_cflag & CRTSCTS)
+		uport->status |= UPSTAT_CTS_ENABLE;
+	else
+		uport->status &= ~UPSTAT_CTS_ENABLE;
+
+	if (termios->c_cflag & CLOCAL)
+		uport->status &= ~UPSTAT_DCD_ENABLE;
+	else
+		uport->status |= UPSTAT_DCD_ENABLE;
+
+	/* reset sw-assisted CTS flow control based on (possibly) new mode */
+	old_hw_stopped = uport->hw_stopped;
+	uport->hw_stopped = uart_softcts_mode(uport) &&
+			    !(uport->ops->get_mctrl(uport) & TIOCM_CTS);
+	if (uport->hw_stopped != old_hw_stopped) {
+		if (!old_hw_stopped)
+			uport->ops->stop_tx(uport);
+		else
+			__uart_start(tty);
+	}
+	spin_unlock_irq(&uport->lock);
+}
+
 /*
  * Startup the port.  This will be called once per open.  All calls
  * will be serialised by the per-port mutex.
@@ -232,7 +275,7 @@ static int uart_port_startup(struct tty_struct *tty, struct uart_state *state,
 		/*
 		 * Initialise the hardware port settings.
 		 */
-		uart_change_speed(tty, state, NULL);
+		uart_change_line_settings(tty, state, NULL);
 
 		/*
 		 * Setup the RTS and DTR signals once the
@@ -485,52 +528,6 @@ uart_get_divisor(struct uart_port *port, unsigned int baud)
 }
 EXPORT_SYMBOL(uart_get_divisor);
 
-/* Caller holds port mutex */
-static void uart_change_speed(struct tty_struct *tty, struct uart_state *state,
-			      const struct ktermios *old_termios)
-{
-	struct uart_port *uport = uart_port_check(state);
-	struct ktermios *termios;
-	int hw_stopped;
-
-	/*
-	 * If we have no tty, termios, or the port does not exist,
-	 * then we can't set the parameters for this port.
-	 */
-	if (!tty || uport->type == PORT_UNKNOWN)
-		return;
-
-	termios = &tty->termios;
-	uport->ops->set_termios(uport, termios, old_termios);
-
-	/*
-	 * Set modem status enables based on termios cflag
-	 */
-	spin_lock_irq(&uport->lock);
-	if (termios->c_cflag & CRTSCTS)
-		uport->status |= UPSTAT_CTS_ENABLE;
-	else
-		uport->status &= ~UPSTAT_CTS_ENABLE;
-
-	if (termios->c_cflag & CLOCAL)
-		uport->status &= ~UPSTAT_DCD_ENABLE;
-	else
-		uport->status |= UPSTAT_DCD_ENABLE;
-
-	/* reset sw-assisted CTS flow control based on (possibly) new mode */
-	hw_stopped = uport->hw_stopped;
-	uport->hw_stopped = uart_softcts_mode(uport) &&
-				!(uport->ops->get_mctrl(uport) & TIOCM_CTS);
-	if (uport->hw_stopped) {
-		if (!hw_stopped)
-			uport->ops->stop_tx(uport);
-	} else {
-		if (hw_stopped)
-			__uart_start(tty);
-	}
-	spin_unlock_irq(&uport->lock);
-}
-
 static int uart_put_char(struct tty_struct *tty, unsigned char c)
 {
 	struct uart_state *state = tty->driver_data;
@@ -994,7 +991,7 @@ static int uart_set_info(struct tty_struct *tty, struct tty_port *port,
 				      current->comm,
 				      tty_name(port->tty));
 			}
-			uart_change_speed(tty, state, NULL);
+			uart_change_line_settings(tty, state, NULL);
 		}
 	} else {
 		retval = uart_startup(tty, state, true);
@@ -1491,7 +1488,7 @@ static int uart_set_iso7816_config(struct uart_port *port,
 	 * There are 5 words reserved for future use. Check that userspace
 	 * doesn't put stuff in there to prevent breakages in the future.
 	 */
-	for (i = 0; i < 5; i++)
+	for (i = 0; i < ARRAY_SIZE(iso7816.reserved); i++)
 		if (iso7816.reserved[i])
 			return -EINVAL;
 
@@ -1552,7 +1549,7 @@ uart_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg)
 		goto out;
 
 	/* rs485_config requires more locking than others */
-	if (cmd == TIOCGRS485)
+	if (cmd == TIOCSRS485)
 		down_write(&tty->termios_rwsem);
 
 	mutex_lock(&port->mutex);
@@ -1595,7 +1592,7 @@ uart_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg)
 	}
 out_up:
 	mutex_unlock(&port->mutex);
-	if (cmd == TIOCGRS485)
+	if (cmd == TIOCSRS485)
 		up_write(&tty->termios_rwsem);
 out:
 	return ret;
@@ -1656,15 +1653,15 @@ static void uart_set_termios(struct tty_struct *tty,
 		goto out;
 	}
 
-	uart_change_speed(tty, state, old_termios);
+	uart_change_line_settings(tty, state, old_termios);
 	/* reload cflag from termios; port driver may have overridden flags */
 	cflag = tty->termios.c_cflag;
 
 	/* Handle transition to B0 status */
-	if ((old_termios->c_cflag & CBAUD) && !(cflag & CBAUD))
+	if (((old_termios->c_cflag & CBAUD) != B0) && ((cflag & CBAUD) == B0))
 		uart_clear_mctrl(uport, TIOCM_RTS | TIOCM_DTR);
 	/* Handle transition away from B0 status */
-	else if (!(old_termios->c_cflag & CBAUD) && (cflag & CBAUD)) {
+	else if (((old_termios->c_cflag & CBAUD) == B0) && ((cflag & CBAUD) != B0)) {
 		unsigned int mask = TIOCM_DTR;
 
 		if (!(cflag & CRTSCTS) || !tty_throttled(tty))
@@ -2452,7 +2449,7 @@ int uart_resume_port(struct uart_driver *drv, struct uart_port *uport)
 			ret = ops->startup(uport);
 			if (ret == 0) {
 				if (tty)
-					uart_change_speed(tty, state, NULL);
+					uart_change_line_settings(tty, state, NULL);
 				spin_lock_irq(&uport->lock);
 				if (!(uport->rs485.flags & SER_RS485_ENABLED))
 					ops->set_mctrl(uport, uport->mctrl);
@@ -2593,6 +2590,7 @@ static int uart_poll_init(struct tty_driver *driver, int line, char *options)
 {
 	struct uart_driver *drv = driver->driver_state;
 	struct uart_state *state = drv->state + line;
+	enum uart_pm_state pm_state;
 	struct tty_port *tport;
 	struct uart_port *port;
 	int baud = 9600;
@@ -2610,6 +2608,9 @@ static int uart_poll_init(struct tty_driver *driver, int line, char *options)
 		goto out;
 	}
 
+	pm_state = state->pm_state;
+	uart_change_pm(state, UART_PM_STATE_ON);
+
 	if (port->ops->poll_init) {
 		/*
 		 * We don't set initialized as we only initialized the hw,
@@ -2626,6 +2627,8 @@ static int uart_poll_init(struct tty_driver *driver, int line, char *options)
 		console_list_unlock();
 	}
 out:
+	if (ret)
+		uart_change_pm(state, pm_state);
 	mutex_unlock(&tport->mutex);
 	return ret;
 }
@@ -3305,13 +3308,13 @@ void uart_handle_cts_change(struct uart_port *uport, bool active)
 	if (uart_softcts_mode(uport)) {
 		if (uport->hw_stopped) {
 			if (active) {
-				uport->hw_stopped = 0;
+				uport->hw_stopped = false;
 				uport->ops->start_tx(uport);
 				uart_write_wakeup(uport);
 			}
 		} else {
 			if (!active) {
-				uport->hw_stopped = 1;
+				uport->hw_stopped = true;
 				uport->ops->stop_tx(uport);
 			}
 		}
diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c
index caa09a0c48f4..7c9457962a3d 100644
--- a/drivers/tty/serial/sh-sci.c
+++ b/drivers/tty/serial/sh-sci.c
@@ -588,14 +588,28 @@ static void sci_start_tx(struct uart_port *port)
 
 	if (s->chan_tx && !uart_circ_empty(&s->port.state->xmit) &&
 	    dma_submit_error(s->cookie_tx)) {
+		if (s->cfg->regtype == SCIx_RZ_SCIFA_REGTYPE)
+			/* Switch irq from SCIF to DMA */
+			disable_irq(s->irqs[SCIx_TXI_IRQ]);
+
 		s->cookie_tx = 0;
 		schedule_work(&s->work_tx);
 	}
 #endif
 
-	if (!s->chan_tx || port->type == PORT_SCIFA || port->type == PORT_SCIFB) {
+	if (!s->chan_tx || s->cfg->regtype == SCIx_RZ_SCIFA_REGTYPE ||
+	    port->type == PORT_SCIFA || port->type == PORT_SCIFB) {
 		/* Set TIE (Transmit Interrupt Enable) bit in SCSCR */
 		ctrl = serial_port_in(port, SCSCR);
+
+		/*
+		 * For SCI, TE (transmit enable) must be set after setting TIE
+		 * (transmit interrupt enable) or in the same instruction to start
+		 * the transmit process.
+		 */
+		if (port->type == PORT_SCI)
+			ctrl |= SCSCR_TE;
+
 		serial_port_out(port, SCSCR, ctrl | SCSCR_TIE);
 	}
 }
@@ -833,6 +847,11 @@ static void sci_transmit_chars(struct uart_port *port)
 		} else if (!uart_circ_empty(xmit) && !stopped) {
 			c = xmit->buf[xmit->tail];
 			xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
+		} else if (port->type == PORT_SCI && uart_circ_empty(xmit)) {
+			ctrl = serial_port_in(port, SCSCR);
+			ctrl &= ~SCSCR_TE;
+			serial_port_out(port, SCSCR, ctrl);
+			return;
 		} else {
 			break;
 		}
@@ -846,9 +865,16 @@ static void sci_transmit_chars(struct uart_port *port)
 
 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 		uart_write_wakeup(port);
-	if (uart_circ_empty(xmit))
-		sci_stop_tx(port);
+	if (uart_circ_empty(xmit)) {
+		if (port->type == PORT_SCI) {
+			ctrl = serial_port_in(port, SCSCR);
+			ctrl &= ~SCSCR_TIE;
+			ctrl |= SCSCR_TEIE;
+			serial_port_out(port, SCSCR, ctrl);
+		}
 
+		sci_stop_tx(port);
+	}
 }
 
 static void sci_receive_chars(struct uart_port *port)
@@ -1192,9 +1218,15 @@ static void sci_dma_tx_complete(void *arg)
 		schedule_work(&s->work_tx);
 	} else {
 		s->cookie_tx = -EINVAL;
-		if (port->type == PORT_SCIFA || port->type == PORT_SCIFB) {
+		if (port->type == PORT_SCIFA || port->type == PORT_SCIFB ||
+		    s->cfg->regtype == SCIx_RZ_SCIFA_REGTYPE) {
 			u16 ctrl = serial_port_in(port, SCSCR);
 			serial_port_out(port, SCSCR, ctrl & ~SCSCR_TIE);
+			if (s->cfg->regtype == SCIx_RZ_SCIFA_REGTYPE) {
+				/* Switch irq from DMA to SCIF */
+				dmaengine_pause(s->chan_tx_saved);
+				enable_irq(s->irqs[SCIx_TXI_IRQ]);
+			}
 		}
 	}
 
@@ -1266,9 +1298,13 @@ static void sci_dma_rx_reenable_irq(struct sci_port *s)
 
 	/* Direct new serial port interrupts back to CPU */
 	scr = serial_port_in(port, SCSCR);
-	if (port->type == PORT_SCIFA || port->type == PORT_SCIFB) {
-		scr &= ~SCSCR_RDRQE;
+	if (port->type == PORT_SCIFA || port->type == PORT_SCIFB ||
+	    s->cfg->regtype == SCIx_RZ_SCIFA_REGTYPE) {
 		enable_irq(s->irqs[SCIx_RXI_IRQ]);
+		if (s->cfg->regtype == SCIx_RZ_SCIFA_REGTYPE)
+			scif_set_rtrg(port, s->rx_trigger);
+		else
+			scr &= ~SCSCR_RDRQE;
 	}
 	serial_port_out(port, SCSCR, scr | SCSCR_RIE);
 }
@@ -1505,7 +1541,8 @@ static enum hrtimer_restart sci_dma_rx_timer_fn(struct hrtimer *t)
 			tty_flip_buffer_push(&port->state->port);
 	}
 
-	if (port->type == PORT_SCIFA || port->type == PORT_SCIFB)
+	if (port->type == PORT_SCIFA || port->type == PORT_SCIFB ||
+	    s->cfg->regtype == SCIx_RZ_SCIFA_REGTYPE)
 		sci_dma_rx_submit(s, true);
 
 	sci_dma_rx_reenable_irq(s);
@@ -1531,15 +1568,12 @@ static struct dma_chan *sci_request_dma_chan(struct uart_port *port,
 
 	memset(&cfg, 0, sizeof(cfg));
 	cfg.direction = dir;
-	if (dir == DMA_MEM_TO_DEV) {
-		cfg.dst_addr = port->mapbase +
-			(sci_getreg(port, SCxTDR)->offset << port->regshift);
-		cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
-	} else {
-		cfg.src_addr = port->mapbase +
-			(sci_getreg(port, SCxRDR)->offset << port->regshift);
-		cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
-	}
+	cfg.dst_addr = port->mapbase +
+		(sci_getreg(port, SCxTDR)->offset << port->regshift);
+	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+	cfg.src_addr = port->mapbase +
+		(sci_getreg(port, SCxRDR)->offset << port->regshift);
+	cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 
 	ret = dmaengine_slave_config(chan, &cfg);
 	if (ret) {
@@ -1574,7 +1608,7 @@ static void sci_request_dma(struct uart_port *port)
 	 * Don't request a dma channel if no channel was specified
 	 * in the device tree.
 	 */
-	if (!of_find_property(port->dev->of_node, "dmas", NULL))
+	if (!of_property_present(port->dev->of_node, "dmas"))
 		return;
 
 	chan = sci_request_dma_chan(port, DMA_MEM_TO_DEV);
@@ -1632,7 +1666,8 @@ static void sci_request_dma(struct uart_port *port)
 
 		s->chan_rx_saved = s->chan_rx = chan;
 
-		if (port->type == PORT_SCIFA || port->type == PORT_SCIFB)
+		if (port->type == PORT_SCIFA || port->type == PORT_SCIFB ||
+		    s->cfg->regtype == SCIx_RZ_SCIFA_REGTYPE)
 			sci_dma_rx_submit(s, false);
 	}
 }
@@ -1685,9 +1720,15 @@ static irqreturn_t sci_rx_interrupt(int irq, void *ptr)
 		u16 ssr = serial_port_in(port, SCxSR);
 
 		/* Disable future Rx interrupts */
-		if (port->type == PORT_SCIFA || port->type == PORT_SCIFB) {
-			disable_irq_nosync(irq);
-			scr |= SCSCR_RDRQE;
+		if (port->type == PORT_SCIFA || port->type == PORT_SCIFB ||
+		    s->cfg->regtype == SCIx_RZ_SCIFA_REGTYPE) {
+			disable_irq_nosync(s->irqs[SCIx_RXI_IRQ]);
+			if (s->cfg->regtype == SCIx_RZ_SCIFA_REGTYPE) {
+				scif_set_rtrg(port, 1);
+				scr |= SCSCR_RIE;
+			} else {
+				scr |= SCSCR_RDRQE;
+			}
 		} else {
 			if (sci_dma_rx_submit(s, false) < 0)
 				goto handle_pio;
@@ -1737,6 +1778,24 @@ static irqreturn_t sci_tx_interrupt(int irq, void *ptr)
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t sci_tx_end_interrupt(int irq, void *ptr)
+{
+	struct uart_port *port = ptr;
+	unsigned long flags;
+	unsigned short ctrl;
+
+	if (port->type != PORT_SCI)
+		return sci_tx_interrupt(irq, ptr);
+
+	spin_lock_irqsave(&port->lock, flags);
+	ctrl = serial_port_in(port, SCSCR);
+	ctrl &= ~(SCSCR_TE | SCSCR_TEIE);
+	serial_port_out(port, SCSCR, ctrl);
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	return IRQ_HANDLED;
+}
+
 static irqreturn_t sci_br_interrupt(int irq, void *ptr)
 {
 	struct uart_port *port = ptr;
@@ -1873,7 +1932,7 @@ static const struct sci_irq_desc {
 
 	[SCIx_TEI_IRQ] = {
 		.desc = "tx end",
-		.handler = sci_tx_interrupt,
+		.handler = sci_tx_end_interrupt,
 	},
 
 	/*
@@ -2580,8 +2639,14 @@ done:
 		sci_set_mctrl(port, port->mctrl);
 	}
 
-	scr_val |= SCSCR_RE | SCSCR_TE |
-		   (s->cfg->scscr & ~(SCSCR_CKE1 | SCSCR_CKE0));
+	/*
+	 * For SCI, TE (transmit enable) must be set after setting TIE
+	 * (transmit interrupt enable) or in the same instruction to
+	 * start the transmitting process. So skip setting TE here for SCI.
+	 */
+	if (port->type != PORT_SCI)
+		scr_val |= SCSCR_TE;
+	scr_val |= SCSCR_RE | (s->cfg->scscr & ~(SCSCR_CKE1 | SCSCR_CKE0));
 	serial_port_out(port, SCSCR, scr_val | s->hscif_tot);
 	if ((srr + 1 == 5) &&
 	    (port->type == PORT_SCIFA || port->type == PORT_SCIFB)) {
@@ -3149,7 +3214,7 @@ static int sci_remove(struct platform_device *dev)
 #define SCI_OF_TYPE(data)		((unsigned long)(data) >> 16)
 #define SCI_OF_REGTYPE(data)		((unsigned long)(data) & 0xffff)
 
-static const struct of_device_id of_sci_match[] = {
+static const struct of_device_id of_sci_match[] __maybe_unused = {
 	/* SoC-specific types */
 	{
 		.compatible = "renesas,scif-r7s72100",
diff --git a/drivers/tty/serial/sh-sci.h b/drivers/tty/serial/sh-sci.h
index c0ae78632dda..0b65563c4e9e 100644
--- a/drivers/tty/serial/sh-sci.h
+++ b/drivers/tty/serial/sh-sci.h
@@ -59,6 +59,9 @@ enum {
 #define SCSMR_SRC_19	0x0600	/* Sampling rate 1/19 */
 #define SCSMR_SRC_27	0x0700	/* Sampling rate 1/27 */
 
+/* Serial Control Register, SCI only bits */
+#define SCSCR_TEIE	BIT(2)  /* Transmit End Interrupt Enable */
+
 /* Serial Control Register, SCIFA/SCIFB only bits */
 #define SCSCR_TDRQE	BIT(15)	/* Tx Data Transfer Request Enable */
 #define SCSCR_RDRQE	BIT(14)	/* Rx Data Transfer Request Enable */
diff --git a/drivers/tty/serial/sprd_serial.c b/drivers/tty/serial/sprd_serial.c
index 492a3bdab5ba..b58f51296ace 100644
--- a/drivers/tty/serial/sprd_serial.c
+++ b/drivers/tty/serial/sprd_serial.c
@@ -1250,7 +1250,7 @@ static struct platform_driver sprd_platform_driver = {
 	.remove		= sprd_remove,
 	.driver		= {
 		.name	= "sprd_serial",
-		.of_match_table = of_match_ptr(serial_ids),
+		.of_match_table = serial_ids,
 		.pm	= &sprd_pm_ops,
 	},
 };
diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c
index 767ff9fdb2e5..1e38fc9b10c1 100644
--- a/drivers/tty/serial/stm32-usart.c
+++ b/drivers/tty/serial/stm32-usart.c
@@ -693,8 +693,9 @@ static void stm32_usart_transmit_chars(struct uart_port *port)
 	int ret;
 
 	if (!stm32_port->hw_flow_control &&
-	    port->rs485.flags & SER_RS485_ENABLED) {
-		stm32_port->txdone = false;
+	    port->rs485.flags & SER_RS485_ENABLED &&
+	    (port->x_char ||
+	     !(uart_circ_empty(xmit) || uart_tx_stopped(port)))) {
 		stm32_usart_tc_interrupt_disable(port);
 		stm32_usart_rs485_rts_enable(port);
 	}
@@ -743,7 +744,6 @@ static void stm32_usart_transmit_chars(struct uart_port *port)
 		stm32_usart_tx_interrupt_disable(port);
 		if (!stm32_port->hw_flow_control &&
 		    port->rs485.flags & SER_RS485_ENABLED) {
-			stm32_port->txdone = true;
 			stm32_usart_tc_interrupt_enable(port);
 		}
 	}
diff --git a/drivers/tty/serial/stm32-usart.h b/drivers/tty/serial/stm32-usart.h
index 0ec41a732c88..903285b5aea7 100644
--- a/drivers/tty/serial/stm32-usart.h
+++ b/drivers/tty/serial/stm32-usart.h
@@ -203,7 +203,6 @@ struct stm32_port {
 	bool hw_flow_control;
 	bool swap;		 /* swap RX & TX pins */
 	bool fifoen;
-	bool txdone;
 	int rxftcfg;		/* RX FIFO threshold CFG      */
 	int txftcfg;		/* TX FIFO threshold CFG      */
 	bool wakeup_src;
diff --git a/drivers/tty/serial/sunzilog.c b/drivers/tty/serial/sunzilog.c
index ccb809216e94..0fbeb3dbd843 100644
--- a/drivers/tty/serial/sunzilog.c
+++ b/drivers/tty/serial/sunzilog.c
@@ -1403,7 +1403,7 @@ static int zs_probe(struct platform_device *op)
 	int keyboard_mouse = 0;
 	int err;
 
-	if (of_find_property(op->dev.of_node, "keyboard", NULL))
+	if (of_property_present(op->dev.of_node, "keyboard"))
 		keyboard_mouse = 1;
 
 	/* uarts must come before keyboards/mice */
@@ -1553,7 +1553,7 @@ static int __init sunzilog_init(void)
 
 	for_each_node_by_name(dp, "zs") {
 		num_sunzilog++;
-		if (of_find_property(dp, "keyboard", NULL))
+		if (of_property_present(dp, "keyboard"))
 			num_keybms++;
 	}
 
diff --git a/drivers/tty/serial/ucc_uart.c b/drivers/tty/serial/ucc_uart.c
index 32c7a5b43f8e..404230c1ebb2 100644
--- a/drivers/tty/serial/ucc_uart.c
+++ b/drivers/tty/serial/ucc_uart.c
@@ -366,15 +366,14 @@ static int qe_uart_tx_pump(struct uart_qe_port *qe_port)
 	/* Pick next descriptor and fill from buffer */
 	bdp = qe_port->tx_cur;
 
-	while (!(ioread16be(&bdp->status) & BD_SC_READY) &&
-	       (xmit->tail != xmit->head)) {
+	while (!(ioread16be(&bdp->status) & BD_SC_READY) && !uart_circ_empty(xmit)) {
 		count = 0;
 		p = qe2cpu_addr(be32_to_cpu(bdp->buf), qe_port);
 		while (count < qe_port->tx_fifosize) {
 			*p++ = xmit->buf[xmit->tail];
 			uart_xmit_advance(port, 1);
 			count++;
-			if (xmit->head == xmit->tail)
+			if (uart_circ_empty(xmit))
 				break;
 		}
 
@@ -1179,7 +1178,7 @@ static int soft_uart_init(struct platform_device *ofdev)
 	struct qe_firmware_info *qe_fw_info;
 	int ret;
 
-	if (of_find_property(np, "soft-uart", NULL)) {
+	if (of_property_read_bool(np, "soft-uart")) {
 		dev_dbg(&ofdev->dev, "using Soft-UART mode\n");
 		soft_uart = 1;
 	} else {
diff --git a/drivers/tty/synclink_gt.c b/drivers/tty/synclink_gt.c
index 33f258d6fef9..16e469e581ec 100644
--- a/drivers/tty/synclink_gt.c
+++ b/drivers/tty/synclink_gt.c
@@ -287,7 +287,6 @@ struct slgt_info {
 	unsigned char *tx_buf;
 	int tx_count;
 
-	char *flag_buf;
 	bool drop_rts_on_tx_done;
 	struct	_input_signal_events	input_signal_events;
 
@@ -730,7 +729,7 @@ static void set_termios(struct tty_struct *tty,
 
 	/* Handle turning off CRTSCTS */
 	if ((old_termios->c_cflag & CRTSCTS) && !C_CRTSCTS(tty)) {
-		tty->hw_stopped = 0;
+		tty->hw_stopped = false;
 		tx_release(tty);
 	}
 }
@@ -1953,13 +1952,13 @@ static void cts_change(struct slgt_info *info, unsigned short status)
 		if (info->port.tty) {
 			if (info->port.tty->hw_stopped) {
 				if (info->signals & SerialSignal_CTS) {
-		 			info->port.tty->hw_stopped = 0;
+					info->port.tty->hw_stopped = false;
 					info->pending_bh |= BH_TRANSMIT;
 					return;
 				}
 			} else {
 				if (!(info->signals & SerialSignal_CTS))
-		 			info->port.tty->hw_stopped = 1;
+					info->port.tty->hw_stopped = true;
 			}
 		}
 	}
@@ -3244,13 +3243,7 @@ static int alloc_tmp_rbuf(struct slgt_info *info)
 	info->tmp_rbuf = kmalloc(info->max_frame_size + 5, GFP_KERNEL);
 	if (info->tmp_rbuf == NULL)
 		return -ENOMEM;
-	/* unused flag buffer to satisfy receive_buf calling interface */
-	info->flag_buf = kzalloc(info->max_frame_size + 5, GFP_KERNEL);
-	if (!info->flag_buf) {
-		kfree(info->tmp_rbuf);
-		info->tmp_rbuf = NULL;
-		return -ENOMEM;
-	}
+
 	return 0;
 }
 
@@ -3258,8 +3251,6 @@ static void free_tmp_rbuf(struct slgt_info *info)
 {
 	kfree(info->tmp_rbuf);
 	info->tmp_rbuf = NULL;
-	kfree(info->flag_buf);
-	info->flag_buf = NULL;
 }
 
 /*
@@ -4657,7 +4648,8 @@ check_again:
 				hdlcdev_rx(info,info->tmp_rbuf, framesize);
 			else
 #endif
-				ldisc_receive_buf(tty, info->tmp_rbuf, info->flag_buf, framesize);
+				ldisc_receive_buf(tty, info->tmp_rbuf, NULL,
+						  framesize);
 		}
 	}
 	free_rbufs(info, start, end);
@@ -4691,8 +4683,8 @@ static bool rx_get_buf(struct slgt_info *info)
 	DBGDATA(info, info->rbufs[i].buf, count, "rx");
 	DBGINFO(("rx_get_buf size=%d\n", count));
 	if (count)
-		ldisc_receive_buf(info->port.tty, info->rbufs[i].buf,
-				  info->flag_buf, count);
+		ldisc_receive_buf(info->port.tty, info->rbufs[i].buf, NULL,
+				  count);
 	free_rbufs(info, i, i);
 	return true;
 }
diff --git a/drivers/tty/tty.h b/drivers/tty/tty.h
index f45cd683c02e..1e0d80e98d26 100644
--- a/drivers/tty/tty.h
+++ b/drivers/tty/tty.h
@@ -62,6 +62,8 @@ int __tty_check_change(struct tty_struct *tty, int sig);
 int tty_check_change(struct tty_struct *tty);
 void __stop_tty(struct tty_struct *tty);
 void __start_tty(struct tty_struct *tty);
+void tty_write_unlock(struct tty_struct *tty);
+int tty_write_lock(struct tty_struct *tty, int ndelay);
 void tty_vhangup_session(struct tty_struct *tty);
 void tty_open_proc_set_tty(struct file *filp, struct tty_struct *tty);
 int tty_signal_session_leader(struct tty_struct *tty, int exit_session);
diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c
index 36fb945fdad4..c84be40fb8df 100644
--- a/drivers/tty/tty_io.c
+++ b/drivers/tty/tty_io.c
@@ -933,13 +933,13 @@ static ssize_t tty_read(struct kiocb *iocb, struct iov_iter *to)
 	return i;
 }
 
-static void tty_write_unlock(struct tty_struct *tty)
+void tty_write_unlock(struct tty_struct *tty)
 {
 	mutex_unlock(&tty->atomic_write_lock);
 	wake_up_interruptible_poll(&tty->write_wait, EPOLLOUT);
 }
 
-static int tty_write_lock(struct tty_struct *tty, int ndelay)
+int tty_write_lock(struct tty_struct *tty, int ndelay)
 {
 	if (!mutex_trylock(&tty->atomic_write_lock)) {
 		if (ndelay)
@@ -3070,7 +3070,7 @@ static struct device *tty_get_device(struct tty_struct *tty)
 {
 	dev_t devt = tty_devnum(tty);
 
-	return class_find_device_by_devt(tty_class, devt);
+	return class_find_device_by_devt(&tty_class, devt);
 }
 
 
@@ -3143,8 +3143,6 @@ int tty_put_char(struct tty_struct *tty, unsigned char ch)
 }
 EXPORT_SYMBOL_GPL(tty_put_char);
 
-struct class *tty_class;
-
 static int tty_cdev_add(struct tty_driver *driver, dev_t dev,
 		unsigned int index, unsigned int count)
 {
@@ -3239,7 +3237,7 @@ struct device *tty_register_device_attr(struct tty_driver *driver,
 		return ERR_PTR(-ENOMEM);
 
 	dev->devt = devt;
-	dev->class = tty_class;
+	dev->class = &tty_class;
 	dev->parent = device;
 	dev->release = tty_device_create_release;
 	dev_set_name(dev, "%s", name);
@@ -3294,8 +3292,7 @@ EXPORT_SYMBOL_GPL(tty_register_device_attr);
  */
 void tty_unregister_device(struct tty_driver *driver, unsigned index)
 {
-	device_destroy(tty_class,
-		MKDEV(driver->major, driver->minor_start) + index);
+	device_destroy(&tty_class, MKDEV(driver->major, driver->minor_start) + index);
 	if (!(driver->flags & TTY_DRIVER_DYNAMIC_ALLOC)) {
 		cdev_del(driver->cdevs[index]);
 		driver->cdevs[index] = NULL;
@@ -3510,13 +3507,14 @@ static char *tty_devnode(const struct device *dev, umode_t *mode)
 	return NULL;
 }
 
+const struct class tty_class = {
+	.name		= "tty",
+	.devnode	= tty_devnode,
+};
+
 static int __init tty_class_init(void)
 {
-	tty_class = class_create(THIS_MODULE, "tty");
-	if (IS_ERR(tty_class))
-		return PTR_ERR(tty_class);
-	tty_class->devnode = tty_devnode;
-	return 0;
+	return class_register(&tty_class);
 }
 
 postcore_initcall(tty_class_init);
@@ -3614,42 +3612,24 @@ static struct ctl_table tty_table[] = {
 	{ }
 };
 
-static struct ctl_table tty_dir_table[] = {
-	{
-		.procname	= "tty",
-		.mode		= 0555,
-		.child		= tty_table,
-	},
-	{ }
-};
-
-static struct ctl_table tty_root_table[] = {
-	{
-		.procname	= "dev",
-		.mode		= 0555,
-		.child		= tty_dir_table,
-	},
-	{ }
-};
-
 /*
  * Ok, now we can initialize the rest of the tty devices and can count
  * on memory allocations, interrupts etc..
  */
 int __init tty_init(void)
 {
-	register_sysctl_table(tty_root_table);
+	register_sysctl_init("dev/tty", tty_table);
 	cdev_init(&tty_cdev, &tty_fops);
 	if (cdev_add(&tty_cdev, MKDEV(TTYAUX_MAJOR, 0), 1) ||
 	    register_chrdev_region(MKDEV(TTYAUX_MAJOR, 0), 1, "/dev/tty") < 0)
 		panic("Couldn't register /dev/tty driver\n");
-	device_create(tty_class, NULL, MKDEV(TTYAUX_MAJOR, 0), NULL, "tty");
+	device_create(&tty_class, NULL, MKDEV(TTYAUX_MAJOR, 0), NULL, "tty");
 
 	cdev_init(&console_cdev, &console_fops);
 	if (cdev_add(&console_cdev, MKDEV(TTYAUX_MAJOR, 1), 1) ||
 	    register_chrdev_region(MKDEV(TTYAUX_MAJOR, 1), 1, "/dev/console") < 0)
 		panic("Couldn't register /dev/console driver\n");
-	consdev = device_create_with_groups(tty_class, NULL,
+	consdev = device_create_with_groups(&tty_class, NULL,
 					    MKDEV(TTYAUX_MAJOR, 1), NULL,
 					    cons_dev_groups, "console");
 	if (IS_ERR(consdev))
diff --git a/drivers/tty/tty_ioctl.c b/drivers/tty/tty_ioctl.c
index 12983ce4e43e..2e88b414cf95 100644
--- a/drivers/tty/tty_ioctl.c
+++ b/drivers/tty/tty_ioctl.c
@@ -7,6 +7,7 @@
  * discipline handling modules (like SLIP).
  */
 
+#include <linux/bits.h>
 #include <linux/types.h>
 #include <linux/termios.h>
 #include <linux/errno.h>
@@ -40,10 +41,10 @@
 /*
  * Internal flag options for termios setting behavior
  */
-#define TERMIOS_FLUSH	1
-#define TERMIOS_WAIT	2
-#define TERMIOS_TERMIO	4
-#define TERMIOS_OLD	8
+#define TERMIOS_FLUSH	BIT(0)
+#define TERMIOS_WAIT	BIT(1)
+#define TERMIOS_TERMIO	BIT(2)
+#define TERMIOS_OLD	BIT(3)
 
 
 /**
@@ -500,21 +501,42 @@ static int set_termios(struct tty_struct *tty, void __user *arg, int opt)
 	tmp_termios.c_ispeed = tty_termios_input_baud_rate(&tmp_termios);
 	tmp_termios.c_ospeed = tty_termios_baud_rate(&tmp_termios);
 
-	ld = tty_ldisc_ref(tty);
+	if (opt & (TERMIOS_FLUSH|TERMIOS_WAIT)) {
+retry_write_wait:
+		retval = wait_event_interruptible(tty->write_wait, !tty_chars_in_buffer(tty));
+		if (retval < 0)
+			return retval;
 
-	if (ld != NULL) {
-		if ((opt & TERMIOS_FLUSH) && ld->ops->flush_buffer)
-			ld->ops->flush_buffer(tty);
-		tty_ldisc_deref(ld);
-	}
+		if (tty_write_lock(tty, 0) < 0)
+			goto retry_write_wait;
 
-	if (opt & TERMIOS_WAIT) {
-		tty_wait_until_sent(tty, 0);
-		if (signal_pending(current))
-			return -ERESTARTSYS;
-	}
+		/* Racing writer? */
+		if (tty_chars_in_buffer(tty)) {
+			tty_write_unlock(tty);
+			goto retry_write_wait;
+		}
 
-	tty_set_termios(tty, &tmp_termios);
+		ld = tty_ldisc_ref(tty);
+		if (ld != NULL) {
+			if ((opt & TERMIOS_FLUSH) && ld->ops->flush_buffer)
+				ld->ops->flush_buffer(tty);
+			tty_ldisc_deref(ld);
+		}
+
+		if ((opt & TERMIOS_WAIT) && tty->ops->wait_until_sent) {
+			tty->ops->wait_until_sent(tty, 0);
+			if (signal_pending(current)) {
+				tty_write_unlock(tty);
+				return -ERESTARTSYS;
+			}
+		}
+
+		tty_set_termios(tty, &tmp_termios);
+
+		tty_write_unlock(tty);
+	} else {
+		tty_set_termios(tty, &tmp_termios);
+	}
 
 	/* FIXME: Arguably if tmp_termios == tty->termios AND the
 	   actual requested termios was not tmp_termios then we may
diff --git a/drivers/tty/tty_ldisc.c b/drivers/tty/tty_ldisc.c
index e758f44729e7..3f68e213df1f 100644
--- a/drivers/tty/tty_ldisc.c
+++ b/drivers/tty/tty_ldisc.c
@@ -58,7 +58,6 @@ static struct tty_ldisc_ops *tty_ldiscs[NR_LDISCS];
 int tty_register_ldisc(struct tty_ldisc_ops *new_ldisc)
 {
 	unsigned long flags;
-	int ret = 0;
 
 	if (new_ldisc->num < N_TTY || new_ldisc->num >= NR_LDISCS)
 		return -EINVAL;
@@ -67,7 +66,7 @@ int tty_register_ldisc(struct tty_ldisc_ops *new_ldisc)
 	tty_ldiscs[new_ldisc->num] = new_ldisc;
 	raw_spin_unlock_irqrestore(&tty_ldiscs_lock, flags);
 
-	return ret;
+	return 0;
 }
 EXPORT_SYMBOL(tty_register_ldisc);
 
diff --git a/drivers/tty/vt/vc_screen.c b/drivers/tty/vt/vc_screen.c
index 1dc07f9214d5..498ba9c0ee93 100644
--- a/drivers/tty/vt/vc_screen.c
+++ b/drivers/tty/vt/vc_screen.c
@@ -804,7 +804,7 @@ int __init vcs_init(void)
 
 	if (register_chrdev(VCS_MAJOR, "vcs", &vcs_fops))
 		panic("unable to get major %d for vcs device", VCS_MAJOR);
-	vc_class = class_create(THIS_MODULE, "vc");
+	vc_class = class_create("vc");
 
 	device_create(vc_class, NULL, MKDEV(VCS_MAJOR, 0), NULL, "vcs");
 	device_create(vc_class, NULL, MKDEV(VCS_MAJOR, 64), NULL, "vcsu");
diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c
index 3c2ea9c098f7..1e8e57b45688 100644
--- a/drivers/tty/vt/vt.c
+++ b/drivers/tty/vt/vt.c
@@ -135,10 +135,9 @@ const struct consw *conswitchp;
 #define DEFAULT_CURSOR_BLINK_MS	200
 
 struct vc vc_cons [MAX_NR_CONSOLES];
+EXPORT_SYMBOL(vc_cons);
 
-#ifndef VT_SINGLE_DRIVER
 static const struct consw *con_driver_map[MAX_NR_CONSOLES];
-#endif
 
 static int con_open(struct tty_struct *, struct file *);
 static void vc_init(struct vc_data *vc, unsigned int rows,
@@ -162,6 +161,7 @@ int default_utf8 = true;
 module_param(default_utf8, int, S_IRUGO | S_IWUSR);
 int global_cursor_default = -1;
 module_param(global_cursor_default, int, S_IRUGO | S_IWUSR);
+EXPORT_SYMBOL(global_cursor_default);
 
 static int cur_default = CUR_UNDERLINE;
 module_param(cur_default, int, S_IRUGO | S_IWUSR);
@@ -174,6 +174,7 @@ static int ignore_poke;
 
 int do_poke_blanked_console;
 int console_blanked;
+EXPORT_SYMBOL(console_blanked);
 
 static int vesa_blank_mode; /* 0:none 1:suspendV 2:suspendH 3:powerdown */
 static int vesa_off_interval;
@@ -190,8 +191,10 @@ static DECLARE_WORK(con_driver_unregister_work, con_driver_unregister_callback);
  * saved_* variants are for save/restore around kernel debugger enter/leave
  */
 int fg_console;
+EXPORT_SYMBOL(fg_console);
 int last_console;
 int want_console = -1;
+
 static int saved_fg_console;
 static int saved_last_console;
 static int saved_want_console;
@@ -223,6 +226,7 @@ static int scrollback_delta;
  * the console on our behalf.
  */
 int (*console_blank_hook)(int);
+EXPORT_SYMBOL(console_blank_hook);
 
 static DEFINE_TIMER(console_timer, blank_screen_t);
 static int blank_state;
@@ -639,6 +643,7 @@ void update_region(struct vc_data *vc, unsigned long start, int count)
 		set_cursor(vc);
 	}
 }
+EXPORT_SYMBOL(update_region);
 
 /* Structure of attributes is hardware-dependent */
 
@@ -984,6 +989,7 @@ void redraw_screen(struct vc_data *vc, int is_switch)
 		notify_update(vc);
 	}
 }
+EXPORT_SYMBOL(redraw_screen);
 
 /*
  *	Allocation, freeing and resizing of VTs.
@@ -1000,10 +1006,10 @@ static void visual_init(struct vc_data *vc, int num, int init)
 	if (vc->vc_sw)
 		module_put(vc->vc_sw->owner);
 	vc->vc_sw = conswitchp;
-#ifndef VT_SINGLE_DRIVER
+
 	if (con_driver_map[num])
 		vc->vc_sw = con_driver_map[num];
-#endif
+
 	__module_get(vc->vc_sw->owner);
 	vc->vc_num = num;
 	vc->vc_display_fg = &master_display_fg;
@@ -1305,6 +1311,7 @@ int vc_resize(struct vc_data *vc, unsigned int cols, unsigned int rows)
 {
 	return vc_do_resize(vc->port.tty, vc, cols, rows);
 }
+EXPORT_SYMBOL(vc_resize);
 
 /**
  *	vt_resize		-	resize a VT
@@ -1368,6 +1375,7 @@ enum { EPecma = 0, EPdec, EPeq, EPgt, EPlt};
 
 const unsigned char color_table[] = { 0, 4, 2, 6, 1, 5, 3, 7,
 				       8,12,10,14, 9,13,11,15 };
+EXPORT_SYMBOL(color_table);
 
 /* the default colour table, for VGA+ colour systems */
 unsigned char default_red[] = {
@@ -1375,18 +1383,21 @@ unsigned char default_red[] = {
 	0x55, 0xff, 0x55, 0xff, 0x55, 0xff, 0x55, 0xff
 };
 module_param_array(default_red, byte, NULL, S_IRUGO | S_IWUSR);
+EXPORT_SYMBOL(default_red);
 
 unsigned char default_grn[] = {
 	0x00, 0x00, 0xaa, 0x55, 0x00, 0x00, 0xaa, 0xaa,
 	0x55, 0x55, 0xff, 0xff, 0x55, 0x55, 0xff, 0xff
 };
 module_param_array(default_grn, byte, NULL, S_IRUGO | S_IWUSR);
+EXPORT_SYMBOL(default_grn);
 
 unsigned char default_blu[] = {
 	0x00, 0x00, 0x00, 0x00, 0xaa, 0xaa, 0xaa, 0xaa,
 	0x55, 0x55, 0x55, 0x55, 0xff, 0xff, 0xff, 0xff
 };
 module_param_array(default_blu, byte, NULL, S_IRUGO | S_IWUSR);
+EXPORT_SYMBOL(default_blu);
 
 /*
  * gotoxy() must verify all boundaries, because the arguments
@@ -3143,93 +3154,84 @@ int tioclinux(struct tty_struct *tty, unsigned long arg)
 		return -EFAULT;
 	ret = 0;
 
-	switch (type)
-	{
-		case TIOCL_SETSEL:
-			ret = set_selection_user((struct tiocl_selection
-						 __user *)(p+1), tty);
-			break;
-		case TIOCL_PASTESEL:
-			ret = paste_selection(tty);
-			break;
-		case TIOCL_UNBLANKSCREEN:
-			console_lock();
-			unblank_screen();
-			console_unlock();
-			break;
-		case TIOCL_SELLOADLUT:
-			console_lock();
-			ret = sel_loadlut(p);
-			console_unlock();
-			break;
-		case TIOCL_GETSHIFTSTATE:
+	switch (type) {
+	case TIOCL_SETSEL:
+		return set_selection_user((struct tiocl_selection
+					 __user *)(p+1), tty);
+	case TIOCL_PASTESEL:
+		return paste_selection(tty);
+	case TIOCL_UNBLANKSCREEN:
+		console_lock();
+		unblank_screen();
+		console_unlock();
+		break;
+	case TIOCL_SELLOADLUT:
+		console_lock();
+		ret = sel_loadlut(p);
+		console_unlock();
+		break;
+	case TIOCL_GETSHIFTSTATE:
+		/*
+		 * Make it possible to react to Shift+Mousebutton. Note that
+		 * 'shift_state' is an undocumented kernel-internal variable;
+		 * programs not closely related to the kernel should not use
+		 * this.
+		 */
+		data = vt_get_shift_state();
+		return put_user(data, p);
+	case TIOCL_GETMOUSEREPORTING:
+		console_lock();	/* May be overkill */
+		data = mouse_reporting();
+		console_unlock();
+		return put_user(data, p);
+	case TIOCL_SETVESABLANK:
+		console_lock();
+		ret = set_vesa_blanking(p);
+		console_unlock();
+		break;
+	case TIOCL_GETKMSGREDIRECT:
+		data = vt_get_kmsg_redirect();
+		return put_user(data, p);
+	case TIOCL_SETKMSGREDIRECT:
+		if (!capable(CAP_SYS_ADMIN))
+			return -EPERM;
 
-	/*
-	 * Make it possible to react to Shift+Mousebutton.
-	 * Note that 'shift_state' is an undocumented
-	 * kernel-internal variable; programs not closely
-	 * related to the kernel should not use this.
-	 */
-			data = vt_get_shift_state();
-			ret = put_user(data, p);
-			break;
-		case TIOCL_GETMOUSEREPORTING:
-			console_lock();	/* May be overkill */
-			data = mouse_reporting();
-			console_unlock();
-			ret = put_user(data, p);
-			break;
-		case TIOCL_SETVESABLANK:
-			console_lock();
-			ret = set_vesa_blanking(p);
-			console_unlock();
-			break;
-		case TIOCL_GETKMSGREDIRECT:
-			data = vt_get_kmsg_redirect();
-			ret = put_user(data, p);
-			break;
-		case TIOCL_SETKMSGREDIRECT:
-			if (!capable(CAP_SYS_ADMIN)) {
-				ret = -EPERM;
-			} else {
-				if (get_user(data, p+1))
-					ret = -EFAULT;
-				else
-					vt_kmsg_redirect(data);
-			}
-			break;
-		case TIOCL_GETFGCONSOLE:
-			/* No locking needed as this is a transiently
-			   correct return anyway if the caller hasn't
-			   disabled switching */
-			ret = fg_console;
-			break;
-		case TIOCL_SCROLLCONSOLE:
-			if (get_user(lines, (s32 __user *)(p+4))) {
-				ret = -EFAULT;
-			} else {
-				/* Need the console lock here. Note that lots
-				   of other calls need fixing before the lock
-				   is actually useful ! */
-				console_lock();
-				scrollfront(vc_cons[fg_console].d, lines);
-				console_unlock();
-				ret = 0;
-			}
-			break;
-		case TIOCL_BLANKSCREEN:	/* until explicitly unblanked, not only poked */
-			console_lock();
-			ignore_poke = 1;
-			do_blank_screen(0);
-			console_unlock();
-			break;
-		case TIOCL_BLANKEDSCREEN:
-			ret = console_blanked;
-			break;
-		default:
-			ret = -EINVAL;
-			break;
+		if (get_user(data, p+1))
+			return -EFAULT;
+
+		vt_kmsg_redirect(data);
+
+		break;
+	case TIOCL_GETFGCONSOLE:
+		/*
+		 * No locking needed as this is a transiently correct return
+		 * anyway if the caller hasn't disabled switching.
+		 */
+		return fg_console;
+	case TIOCL_SCROLLCONSOLE:
+		if (get_user(lines, (s32 __user *)(p+4)))
+			return -EFAULT;
+
+		/*
+		 * Needs the console lock here. Note that lots of other calls
+		 * need fixing before the lock is actually useful!
+		 */
+		console_lock();
+		scrollfront(vc_cons[fg_console].d, lines);
+		console_unlock();
+		break;
+	case TIOCL_BLANKSCREEN:	/* until explicitly unblanked, not only poked */
+		console_lock();
+		ignore_poke = 1;
+		do_blank_screen(0);
+		console_unlock();
+		break;
+	case TIOCL_BLANKEDSCREEN:
+		return console_blanked;
+	default:
+		return -EINVAL;
 	}
+
 	return ret;
 }
 
@@ -3539,7 +3541,7 @@ int __init vty_init(const struct file_operations *console_fops)
 	if (cdev_add(&vc0_cdev, MKDEV(TTY_MAJOR, 0), 1) ||
 	    register_chrdev_region(MKDEV(TTY_MAJOR, 0), 1, "/dev/vc/0") < 0)
 		panic("Couldn't register /dev/tty0 driver\n");
-	tty0dev = device_create_with_groups(tty_class, NULL,
+	tty0dev = device_create_with_groups(&tty_class, NULL,
 					    MKDEV(TTY_MAJOR, 0), NULL,
 					    vt_dev_groups, "tty0");
 	if (IS_ERR(tty0dev))
@@ -3571,8 +3573,6 @@ int __init vty_init(const struct file_operations *console_fops)
 	return 0;
 }
 
-#ifndef VT_SINGLE_DRIVER
-
 static struct class *vtconsole_class;
 
 static int do_bind_con_driver(const struct consw *csw, int first, int last,
@@ -4236,12 +4236,13 @@ void give_up_console(const struct consw *csw)
 	do_unregister_con_driver(csw);
 	console_unlock();
 }
+EXPORT_SYMBOL(give_up_console);
 
 static int __init vtconsole_class_init(void)
 {
 	int i;
 
-	vtconsole_class = class_create(THIS_MODULE, "vtconsole");
+	vtconsole_class = class_create("vtconsole");
 	if (IS_ERR(vtconsole_class)) {
 		pr_warn("Unable to create vt console class; errno = %ld\n",
 			PTR_ERR(vtconsole_class));
@@ -4273,8 +4274,6 @@ static int __init vtconsole_class_init(void)
 }
 postcore_initcall(vtconsole_class_init);
 
-#endif
-
 /*
  *	Screen blanking
  */
@@ -4792,23 +4791,3 @@ void vc_scrolldelta_helper(struct vc_data *c, int lines,
 	c->vc_visible_origin = ubase + (from + from_off) % wrap;
 }
 EXPORT_SYMBOL_GPL(vc_scrolldelta_helper);
-
-/*
- *	Visible symbols for modules
- */
-
-EXPORT_SYMBOL(color_table);
-EXPORT_SYMBOL(default_red);
-EXPORT_SYMBOL(default_grn);
-EXPORT_SYMBOL(default_blu);
-EXPORT_SYMBOL(update_region);
-EXPORT_SYMBOL(redraw_screen);
-EXPORT_SYMBOL(vc_resize);
-EXPORT_SYMBOL(fg_console);
-EXPORT_SYMBOL(console_blank_hook);
-EXPORT_SYMBOL(console_blanked);
-EXPORT_SYMBOL(vc_cons);
-EXPORT_SYMBOL(global_cursor_default);
-#ifndef VT_SINGLE_DRIVER
-EXPORT_SYMBOL(give_up_console);
-#endif
diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h
index 529f8507a5e4..d53b93c21a0c 100644
--- a/drivers/ufs/core/ufshcd-priv.h
+++ b/drivers/ufs/core/ufshcd-priv.h
@@ -364,6 +364,7 @@ static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8
 }
 
 static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q)
+	__must_hold(&q->sq_lock)
 {
 	u32 mask = q->max_entries - 1;
 	u32 val;
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index 70b112038792..17d7bb875fee 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -422,7 +422,9 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
 {
 	u64 lba = 0;
 	u8 opcode = 0, group_id = 0;
-	u32 intr, doorbell;
+	u32 doorbell = 0;
+	u32 intr;
+	int hwq_id = -1;
 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
 	struct scsi_cmnd *cmd = lrbp->cmd;
 	struct request *rq = scsi_cmd_to_rq(cmd);
@@ -456,9 +458,16 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
 	}
 
 	intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
-	doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
+
+	if (is_mcq_enabled(hba)) {
+		struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
+
+		hwq_id = hwq->id;
+	} else {
+		doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
+	}
 	trace_ufshcd_command(dev_name(hba->dev), str_t, tag,
-			doorbell, transfer_len, intr, lba, opcode, group_id);
+			doorbell, hwq_id, transfer_len, intr, lba, opcode, group_id);
 }
 
 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
@@ -533,48 +542,66 @@ static void ufshcd_print_evt_hist(struct ufs_hba *hba)
 }
 
 static
-void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
+void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt)
 {
 	const struct ufshcd_lrb *lrbp;
 	int prdt_length;
-	int tag;
 
-	for_each_set_bit(tag, &bitmap, hba->nutrs) {
-		lrbp = &hba->lrb[tag];
+	lrbp = &hba->lrb[tag];
 
-		dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
-				tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000));
-		dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
-				tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000));
-		dev_err(hba->dev,
-			"UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
-			tag, (u64)lrbp->utrd_dma_addr);
-
-		ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
-				sizeof(struct utp_transfer_req_desc));
-		dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
-			(u64)lrbp->ucd_req_dma_addr);
-		ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
-				sizeof(struct utp_upiu_req));
-		dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
-			(u64)lrbp->ucd_rsp_dma_addr);
-		ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
-				sizeof(struct utp_upiu_rsp));
-
-		prdt_length = le16_to_cpu(
-			lrbp->utr_descriptor_ptr->prd_table_length);
-		if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
-			prdt_length /= ufshcd_sg_entry_size(hba);
+	dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
+			tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000));
+	dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
+			tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000));
+	dev_err(hba->dev,
+		"UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
+		tag, (u64)lrbp->utrd_dma_addr);
+
+	ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
+			sizeof(struct utp_transfer_req_desc));
+	dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
+		(u64)lrbp->ucd_req_dma_addr);
+	ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
+			sizeof(struct utp_upiu_req));
+	dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
+		(u64)lrbp->ucd_rsp_dma_addr);
+	ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
+			sizeof(struct utp_upiu_rsp));
+
+	prdt_length = le16_to_cpu(
+		lrbp->utr_descriptor_ptr->prd_table_length);
+	if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
+		prdt_length /= ufshcd_sg_entry_size(hba);
 
-		dev_err(hba->dev,
-			"UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
-			tag, prdt_length,
-			(u64)lrbp->ucd_prdt_dma_addr);
+	dev_err(hba->dev,
+		"UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
+		tag, prdt_length,
+		(u64)lrbp->ucd_prdt_dma_addr);
 
-		if (pr_prdt)
-			ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
-				ufshcd_sg_entry_size(hba) * prdt_length);
-	}
+	if (pr_prdt)
+		ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
+			ufshcd_sg_entry_size(hba) * prdt_length);
+}
+
+static bool ufshcd_print_tr_iter(struct request *req, void *priv)
+{
+	struct scsi_device *sdev = req->q->queuedata;
+	struct Scsi_Host *shost = sdev->host;
+	struct ufs_hba *hba = shost_priv(shost);
+
+	ufshcd_print_tr(hba, req->tag, *(bool *)priv);
+
+	return true;
+}
+
+/**
+ * ufshcd_print_trs_all - print trs for all started requests.
+ * @hba: per-adapter instance.
+ * @pr_prdt: need to print prdt or not.
+ */
+static void ufshcd_print_trs_all(struct ufs_hba *hba, bool pr_prdt)
+{
+	blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt);
 }
 
 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
@@ -1242,7 +1269,7 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
 	struct ufs_pa_layer_attr new_pwr_info;
 
 	if (scale_up) {
-		memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
+		memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info,
 		       sizeof(struct ufs_pa_layer_attr));
 	} else {
 		memcpy(&new_pwr_info, &hba->pwr_info,
@@ -1251,7 +1278,7 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
 		if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
 		    hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
 			/* save the current power mode */
-			memcpy(&hba->clk_scaling.saved_pwr_info.info,
+			memcpy(&hba->clk_scaling.saved_pwr_info,
 				&hba->pwr_info,
 				sizeof(struct ufs_pa_layer_attr));
 
@@ -2208,10 +2235,11 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag,
 
 	if (is_mcq_enabled(hba)) {
 		int utrd_size = sizeof(struct utp_transfer_req_desc);
+		struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr;
+		struct utp_transfer_req_desc *dest = hwq->sqe_base_addr + hwq->sq_tail_slot;
 
 		spin_lock(&hwq->sq_lock);
-		memcpy(hwq->sqe_base_addr + (hwq->sq_tail_slot * utrd_size),
-		       lrbp->utr_descriptor_ptr, utrd_size);
+		memcpy(dest, src, utrd_size);
 		ufshcd_inc_sq_tail(hwq);
 		spin_unlock(&hwq->sq_lock);
 	} else {
@@ -5231,6 +5259,9 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
 	int scsi_status;
 	enum utp_ocs ocs;
 
+	scsi_set_resid(lrbp->cmd,
+		be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count));
+
 	/* overall command status of utrd */
 	ocs = ufshcd_get_tr_ocs(lrbp, cqe);
 
@@ -5321,7 +5352,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
 
 	if ((host_byte(result) != DID_OK) &&
 	    (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
-		ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
+		ufshcd_print_tr(hba, lrbp->task_tag, true);
 	return result;
 }
 
@@ -6405,7 +6436,7 @@ again:
 		ufshcd_print_pwr_info(hba);
 		ufshcd_print_evt_hist(hba);
 		ufshcd_print_tmrs(hba, hba->outstanding_tasks);
-		ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
+		ufshcd_print_trs_all(hba, pr_prdt);
 		spin_lock_irqsave(hba->host->host_lock, flags);
 	}
 
@@ -7434,9 +7465,9 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
 		ufshcd_print_evt_hist(hba);
 		ufshcd_print_host_state(hba);
 		ufshcd_print_pwr_info(hba);
-		ufshcd_print_trs(hba, 1 << tag, true);
+		ufshcd_print_tr(hba, tag, true);
 	} else {
-		ufshcd_print_trs(hba, 1 << tag, false);
+		ufshcd_print_tr(hba, tag, false);
 	}
 	hba->req_abort_count++;
 
@@ -8394,10 +8425,9 @@ static int ufshcd_add_lus(struct ufs_hba *hba)
 
 	/* Initialize devfreq after UFS device is detected */
 	if (ufshcd_is_clkscaling_supported(hba)) {
-		memcpy(&hba->clk_scaling.saved_pwr_info.info,
+		memcpy(&hba->clk_scaling.saved_pwr_info,
 			&hba->pwr_info,
 			sizeof(struct ufs_pa_layer_attr));
-		hba->clk_scaling.saved_pwr_info.is_valid = true;
 		hba->clk_scaling.is_allowed = true;
 
 		ret = ufshcd_devfreq_init(hba);
@@ -8724,7 +8754,7 @@ static struct ufs_hba_variant_params ufs_hba_vps = {
 	.ondemand_data.downdifferential	= 5,
 };
 
-static struct scsi_host_template ufshcd_driver_template = {
+static const struct scsi_host_template ufshcd_driver_template = {
 	.module			= THIS_MODULE,
 	.name			= UFSHCD,
 	.proc_name		= UFSHCD,
@@ -8747,6 +8777,7 @@ static struct scsi_host_template ufshcd_driver_template = {
 	.max_sectors		= (1 << 20) / SECTOR_SIZE, /* 1 MiB */
 	.max_host_blocked	= 1,
 	.track_queue_depth	= 1,
+	.skip_settle_delay	= 1,
 	.sdev_groups		= ufshcd_driver_groups,
 	.rpm_autosuspend_delay	= RPM_AUTOSUSPEND_DELAY_MS,
 };
diff --git a/drivers/ufs/host/ufs-exynos.c b/drivers/ufs/host/ufs-exynos.c
index 7c985fc38db1..0bf5390739e1 100644
--- a/drivers/ufs/host/ufs-exynos.c
+++ b/drivers/ufs/host/ufs-exynos.c
@@ -1761,7 +1761,7 @@ static struct platform_driver exynos_ufs_pltform = {
 	.driver	= {
 		.name	= "exynos-ufshc",
 		.pm	= &exynos_ufs_pm_ops,
-		.of_match_table = of_match_ptr(exynos_ufs_of_match),
+		.of_match_table = exynos_ufs_of_match,
 	},
 };
 module_platform_driver(exynos_ufs_pltform);
diff --git a/drivers/ufs/host/ufs-hisi.c b/drivers/ufs/host/ufs-hisi.c
index 2eed13bc82ca..4c423eba8aa9 100644
--- a/drivers/ufs/host/ufs-hisi.c
+++ b/drivers/ufs/host/ufs-hisi.c
@@ -597,7 +597,7 @@ static struct platform_driver ufs_hisi_pltform = {
 	.driver	= {
 		.name	= "ufshcd-hisi",
 		.pm	= &ufs_hisi_pm_ops,
-		.of_match_table = of_match_ptr(ufs_hisi_of_match),
+		.of_match_table = ufs_hisi_of_match,
 	},
 };
 module_platform_driver(ufs_hisi_pltform);
diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c
index a02cd866e2f8..82d02e7f3b4f 100644
--- a/drivers/ufs/host/ufs-qcom.c
+++ b/drivers/ufs/host/ufs-qcom.c
@@ -1693,7 +1693,7 @@ static int ufs_qcom_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct of_device_id ufs_qcom_of_match[] = {
+static const struct of_device_id ufs_qcom_of_match[] __maybe_unused = {
 	{ .compatible = "qcom,ufshc"},
 	{},
 };
diff --git a/drivers/ufs/host/ufshcd-pci.c b/drivers/ufs/host/ufshcd-pci.c
index 1c91f43e15c8..9c911787f84c 100644
--- a/drivers/ufs/host/ufshcd-pci.c
+++ b/drivers/ufs/host/ufshcd-pci.c
@@ -607,6 +607,7 @@ static const struct pci_device_id ufshcd_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, 0x51FF), (kernel_ulong_t)&ufs_intel_adl_hba_vops },
 	{ PCI_VDEVICE(INTEL, 0x54FF), (kernel_ulong_t)&ufs_intel_adl_hba_vops },
 	{ PCI_VDEVICE(INTEL, 0x7E47), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0xA847), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
 	{ }	/* terminate list */
 };
 
diff --git a/drivers/usb/Kconfig b/drivers/usb/Kconfig
index a871a988829d..7f33bcc315f2 100644
--- a/drivers/usb/Kconfig
+++ b/drivers/usb/Kconfig
@@ -133,35 +133,6 @@ comment "USB port drivers"
 
 if USB
 
-config USB_USS720
-	tristate "USS720 parport driver"
-	depends on PARPORT
-	select PARPORT_NOT_PC
-	help
-	  This driver is for USB parallel port adapters that use the Lucent
-	  Technologies USS-720 chip. These cables are plugged into your USB
-	  port and provide USB compatibility to peripherals designed with
-	  parallel port interfaces.
-
-	  The chip has two modes: automatic mode and manual mode. In automatic
-	  mode, it looks to the computer like a standard USB printer. Only
-	  printers may be connected to the USS-720 in this mode. The generic
-	  USB printer driver ("USB Printer support", above) may be used in
-	  that mode, and you can say N here if you want to use the chip only
-	  in this mode.
-
-	  Manual mode is not limited to printers, any parallel port
-	  device should work. This driver utilizes manual mode.
-	  Note however that some operations are three orders of magnitude
-	  slower than on a PCI/ISA Parallel Port, so timing critical
-	  applications might not work.
-
-	  Say Y here if you own an USS-720 USB->Parport cable and intend to
-	  connect anything other than a printer to it.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called uss720.
-
 source "drivers/usb/serial/Kconfig"
 
 source "drivers/usb/misc/Kconfig"
diff --git a/drivers/usb/Makefile b/drivers/usb/Makefile
index a81e6ef293af..3a9a0dd4be70 100644
--- a/drivers/usb/Makefile
+++ b/drivers/usb/Makefile
@@ -31,7 +31,6 @@ obj-$(CONFIG_USB_FHCI_HCD)	+= host/
 obj-$(CONFIG_USB_XHCI_HCD)	+= host/
 obj-$(CONFIG_USB_SL811_HCD)	+= host/
 obj-$(CONFIG_USB_ISP1362_HCD)	+= host/
-obj-$(CONFIG_USB_U132_HCD)	+= host/
 obj-$(CONFIG_USB_R8A66597_HCD)	+= host/
 obj-$(CONFIG_USB_FSL_USB2)	+= host/
 obj-$(CONFIG_USB_FOTG210_HCD)	+= host/
diff --git a/drivers/usb/cdns3/cdns3-debug.h b/drivers/usb/cdns3/cdns3-debug.h
index a5c6a29e1340..4618cfe85a4f 100644
--- a/drivers/usb/cdns3/cdns3-debug.h
+++ b/drivers/usb/cdns3/cdns3-debug.h
@@ -107,8 +107,7 @@ static inline char *cdns3_decode_ep0_irq(char *str,
  * Prints out all TRBs in the endpoint ring, even those after the Link TRB.
  *.
  */
-static inline char *cdns3_dbg_ring(struct cdns3_endpoint *priv_ep,
-				   struct cdns3_trb *ring, char *str)
+static inline char *cdns3_dbg_ring(struct cdns3_endpoint *priv_ep, char *str)
 {
 	dma_addr_t addr = priv_ep->trb_pool_dma;
 	struct cdns3_trb *trb;
@@ -136,9 +135,6 @@ static inline char *cdns3_dbg_ring(struct cdns3_endpoint *priv_ep,
 		       "\t\tfree trbs: %d, CCS=%d, PCS=%d\n",
 		       priv_ep->free_trbs, priv_ep->ccs, priv_ep->pcs);
 
-	if (trb_per_sector > TRBS_PER_SEGMENT)
-		trb_per_sector = TRBS_PER_SEGMENT;
-
 	if (trb_per_sector > TRBS_PER_SEGMENT) {
 		sprintf(str + ret, "\t\tTransfer ring %d too big\n",
 			trb_per_sector);
@@ -146,7 +142,7 @@ static inline char *cdns3_dbg_ring(struct cdns3_endpoint *priv_ep,
 	}
 
 	for (i = 0; i < trb_per_sector; ++i) {
-		trb = &ring[i];
+		trb = &priv_ep->trb_pool[i];
 		ret += sprintf(str + ret,
 			"\t\t@%pad %08x %08x %08x\n", &addr,
 			le32_to_cpu(trb->buffer),
diff --git a/drivers/usb/cdns3/cdns3-trace.h b/drivers/usb/cdns3/cdns3-trace.h
index 7574b4a62813..40db89e3333c 100644
--- a/drivers/usb/cdns3/cdns3-trace.h
+++ b/drivers/usb/cdns3/cdns3-trace.h
@@ -100,13 +100,12 @@ DECLARE_EVENT_CLASS(cdns3_log_usb_irq,
 	TP_STRUCT__entry(
 		__field(enum usb_device_speed, speed)
 		__field(u32, usb_ists)
-		__dynamic_array(char, str, CDNS3_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->speed = cdns3_get_speed(priv_dev);
 		__entry->usb_ists = usb_ists;
 	),
-	TP_printk("%s", cdns3_decode_usb_irq(__get_str(str), __entry->speed,
+	TP_printk("%s", cdns3_decode_usb_irq(__get_buf(CDNS3_MSG_MAX), __entry->speed,
 					     __entry->usb_ists))
 );
 
@@ -124,7 +123,6 @@ DECLARE_EVENT_CLASS(cdns3_log_epx_irq,
 		__field(u32, ep_traddr)
 		__field(u32, ep_last_sid)
 		__field(u32, use_streams)
-		__dynamic_array(char, str, CDNS3_MSG_MAX)
 	),
 	TP_fast_assign(
 		__assign_str(ep_name, priv_ep->name);
@@ -134,7 +132,7 @@ DECLARE_EVENT_CLASS(cdns3_log_epx_irq,
 		__entry->use_streams = priv_ep->use_streams;
 	),
 	TP_printk("%s, ep_traddr: %08x ep_last_sid: %08x use_streams: %d",
-		  cdns3_decode_epx_irq(__get_str(str),
+		  cdns3_decode_epx_irq(__get_buf(CDNS3_MSG_MAX),
 				       __get_str(ep_name),
 				       __entry->ep_sts),
 		  __entry->ep_traddr,
@@ -153,13 +151,12 @@ DECLARE_EVENT_CLASS(cdns3_log_ep0_irq,
 	TP_STRUCT__entry(
 		__field(int, ep_dir)
 		__field(u32, ep_sts)
-		__dynamic_array(char, str, CDNS3_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->ep_dir = priv_dev->selected_ep;
 		__entry->ep_sts = ep_sts;
 	),
-	TP_printk("%s", cdns3_decode_ep0_irq(__get_str(str),
+	TP_printk("%s", cdns3_decode_ep0_irq(__get_buf(CDNS3_MSG_MAX),
 					     __entry->ep_dir,
 					     __entry->ep_sts))
 );
@@ -178,7 +175,6 @@ DECLARE_EVENT_CLASS(cdns3_log_ctrl,
 		__field(u16, wValue)
 		__field(u16, wIndex)
 		__field(u16, wLength)
-		__dynamic_array(char, str, CDNS3_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->bRequestType = ctrl->bRequestType;
@@ -187,7 +183,7 @@ DECLARE_EVENT_CLASS(cdns3_log_ctrl,
 		__entry->wIndex = le16_to_cpu(ctrl->wIndex);
 		__entry->wLength = le16_to_cpu(ctrl->wLength);
 	),
-	TP_printk("%s", usb_decode_ctrl(__get_str(str), CDNS3_MSG_MAX,
+	TP_printk("%s", usb_decode_ctrl(__get_buf(CDNS3_MSG_MAX), CDNS3_MSG_MAX,
 					__entry->bRequestType,
 					__entry->bRequest, __entry->wValue,
 					__entry->wIndex, __entry->wLength)
@@ -438,22 +434,16 @@ DECLARE_EVENT_CLASS(cdns3_log_ring,
 	TP_PROTO(struct cdns3_endpoint *priv_ep),
 	TP_ARGS(priv_ep),
 	TP_STRUCT__entry(
-		__dynamic_array(u8, ring, TRB_RING_SIZE)
-		__dynamic_array(u8, priv_ep, sizeof(struct cdns3_endpoint))
 		__dynamic_array(char, buffer,
-				(TRBS_PER_SEGMENT * 65) + CDNS3_MSG_MAX)
+				GET_TRBS_PER_SEGMENT(priv_ep->type) > TRBS_PER_SEGMENT ?
+				CDNS3_MSG_MAX :
+				(GET_TRBS_PER_SEGMENT(priv_ep->type) * 65) + CDNS3_MSG_MAX)
 	),
 	TP_fast_assign(
-		memcpy(__get_dynamic_array(priv_ep), priv_ep,
-		       sizeof(struct cdns3_endpoint));
-		memcpy(__get_dynamic_array(ring), priv_ep->trb_pool,
-		       TRB_RING_SIZE);
+		cdns3_dbg_ring(priv_ep, __get_str(buffer));
 	),
 
-	TP_printk("%s",
-		  cdns3_dbg_ring((struct cdns3_endpoint *)__get_str(priv_ep),
-				 (struct cdns3_trb *)__get_str(ring),
-				 __get_str(buffer)))
+	TP_printk("%s", __get_str(buffer))
 );
 
 DEFINE_EVENT(cdns3_log_ring, cdns3_ring,
diff --git a/drivers/usb/cdns3/cdnsp-trace.h b/drivers/usb/cdns3/cdnsp-trace.h
index 5983dfb99653..4b51011eb00b 100644
--- a/drivers/usb/cdns3/cdnsp-trace.h
+++ b/drivers/usb/cdns3/cdnsp-trace.h
@@ -271,7 +271,6 @@ DECLARE_EVENT_CLASS(cdnsp_log_ctrl,
 		__field(u16, wValue)
 		__field(u16, wIndex)
 		__field(u16, wLength)
-		__dynamic_array(char, str, CDNSP_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->bRequestType = ctrl->bRequestType;
@@ -280,7 +279,7 @@ DECLARE_EVENT_CLASS(cdnsp_log_ctrl,
 		__entry->wIndex = le16_to_cpu(ctrl->wIndex);
 		__entry->wLength = le16_to_cpu(ctrl->wLength);
 	),
-	TP_printk("%s", usb_decode_ctrl(__get_str(str), CDNSP_MSG_MAX,
+	TP_printk("%s", usb_decode_ctrl(__get_buf(CDNSP_MSG_MAX), CDNSP_MSG_MAX,
 					__entry->bRequestType,
 					__entry->bRequest, __entry->wValue,
 					__entry->wIndex, __entry->wLength)
@@ -345,7 +344,6 @@ DECLARE_EVENT_CLASS(cdnsp_log_trb,
 		__field(u32, field3)
 		__field(union cdnsp_trb *, trb)
 		__field(dma_addr_t, trb_dma)
-		__dynamic_array(char, str, CDNSP_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->type = ring->type;
@@ -359,7 +357,7 @@ DECLARE_EVENT_CLASS(cdnsp_log_trb,
 
 	),
 	TP_printk("%s: %s trb: %p(%pad)", cdnsp_ring_type_string(__entry->type),
-		  cdnsp_decode_trb(__get_str(str), CDNSP_MSG_MAX,
+		  cdnsp_decode_trb(__get_buf(CDNSP_MSG_MAX), CDNSP_MSG_MAX,
 				   __entry->field0, __entry->field1,
 				   __entry->field2, __entry->field3),
 				   __entry->trb, &__entry->trb_dma
@@ -544,7 +542,6 @@ DECLARE_EVENT_CLASS(cdnsp_log_ep_ctx,
 		__field(u32, info2)
 		__field(u64, deq)
 		__field(u32, tx_info)
-		__dynamic_array(char, str, CDNSP_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->info = le32_to_cpu(ctx->ep_info);
@@ -552,7 +549,7 @@ DECLARE_EVENT_CLASS(cdnsp_log_ep_ctx,
 		__entry->deq = le64_to_cpu(ctx->deq);
 		__entry->tx_info = le32_to_cpu(ctx->tx_info);
 	),
-	TP_printk("%s", cdnsp_decode_ep_context(__get_str(str), CDNSP_MSG_MAX,
+	TP_printk("%s", cdnsp_decode_ep_context(__get_buf(CDNSP_MSG_MAX), CDNSP_MSG_MAX,
 						__entry->info, __entry->info2,
 						__entry->deq, __entry->tx_info)
 	)
@@ -777,7 +774,6 @@ DECLARE_EVENT_CLASS(cdnsp_log_portsc,
 		TP_STRUCT__entry(
 				__field(u32, portnum)
 				__field(u32, portsc)
-				__dynamic_array(char, str, CDNSP_MSG_MAX)
 				),
 		TP_fast_assign(
 				__entry->portnum = portnum;
@@ -785,7 +781,7 @@ DECLARE_EVENT_CLASS(cdnsp_log_portsc,
 				),
 		TP_printk("port-%d: %s",
 			  __entry->portnum,
-			  cdnsp_decode_portsc(__get_str(str), CDNSP_MSG_MAX,
+			  cdnsp_decode_portsc(__get_buf(CDNSP_MSG_MAX), CDNSP_MSG_MAX,
 					      __entry->portsc)
 			)
 );
diff --git a/drivers/usb/chipidea/Makefile b/drivers/usb/chipidea/Makefile
index 6f4a3deced35..71afeab97e83 100644
--- a/drivers/usb/chipidea/Makefile
+++ b/drivers/usb/chipidea/Makefile
@@ -14,5 +14,5 @@ ci_hdrc-$(CONFIG_USB_OTG_FSM)		+= otg_fsm.o
 obj-$(CONFIG_USB_CHIPIDEA_GENERIC)	+= ci_hdrc_usb2.o
 obj-$(CONFIG_USB_CHIPIDEA_MSM)		+= ci_hdrc_msm.o
 obj-$(CONFIG_USB_CHIPIDEA_PCI)		+= ci_hdrc_pci.o
-obj-$(CONFIG_USB_CHIPIDEA_IMX)		+= ci_hdrc_imx.o usbmisc_imx.o
+obj-$(CONFIG_USB_CHIPIDEA_IMX)		+= usbmisc_imx.o ci_hdrc_imx.o
 obj-$(CONFIG_USB_CHIPIDEA_TEGRA)	+= ci_hdrc_tegra.o
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c
index 2eeccf4ec9d6..2855ac303001 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.c
+++ b/drivers/usb/chipidea/ci_hdrc_imx.c
@@ -152,12 +152,12 @@ static struct imx_usbmisc_data *usbmisc_get_init_data(struct device *dev)
 	 * Check the various over current related properties. If over current
 	 * detection is disabled we're not interested in the polarity.
 	 */
-	if (of_find_property(np, "disable-over-current", NULL)) {
+	if (of_property_read_bool(np, "disable-over-current")) {
 		data->disable_oc = 1;
-	} else if (of_find_property(np, "over-current-active-high", NULL)) {
+	} else if (of_property_read_bool(np, "over-current-active-high")) {
 		data->oc_pol_active_low = 0;
 		data->oc_pol_configured = 1;
-	} else if (of_find_property(np, "over-current-active-low", NULL)) {
+	} else if (of_property_read_bool(np, "over-current-active-low")) {
 		data->oc_pol_active_low = 1;
 		data->oc_pol_configured = 1;
 	} else {
diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c
index 281fc51720ce..798cb077867a 100644
--- a/drivers/usb/chipidea/core.c
+++ b/drivers/usb/chipidea/core.c
@@ -753,7 +753,7 @@ static int ci_get_platdata(struct device *dev,
 		return ret;
 	}
 
-	if (of_find_property(dev->of_node, "non-zero-ttctrl-ttha", NULL))
+	if (of_property_read_bool(dev->of_node, "non-zero-ttctrl-ttha"))
 		platdata->flags |= CI_HDRC_SET_NON_ZERO_TTHA;
 
 	ext_id = ERR_PTR(-ENODEV);
@@ -1108,7 +1108,7 @@ static int ci_hdrc_probe(struct platform_device *pdev)
 	ret = ci_usb_phy_init(ci);
 	if (ret) {
 		dev_err(dev, "unable to init phy: %d\n", ret);
-		return ret;
+		goto ulpi_exit;
 	}
 
 	ci->hw_bank.phys = res->start;
diff --git a/drivers/usb/chipidea/debug.c b/drivers/usb/chipidea/debug.c
index bbc610e5bd69..e72c43615d77 100644
--- a/drivers/usb/chipidea/debug.c
+++ b/drivers/usb/chipidea/debug.c
@@ -247,60 +247,6 @@ static int ci_otg_show(struct seq_file *s, void *unused)
 }
 DEFINE_SHOW_ATTRIBUTE(ci_otg);
 
-static int ci_role_show(struct seq_file *s, void *data)
-{
-	struct ci_hdrc *ci = s->private;
-
-	if (ci->role != CI_ROLE_END)
-		seq_printf(s, "%s\n", ci_role(ci)->name);
-
-	return 0;
-}
-
-static ssize_t ci_role_write(struct file *file, const char __user *ubuf,
-			     size_t count, loff_t *ppos)
-{
-	struct seq_file *s = file->private_data;
-	struct ci_hdrc *ci = s->private;
-	enum ci_role role;
-	char buf[8];
-	int ret;
-
-	if (copy_from_user(buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
-		return -EFAULT;
-
-	for (role = CI_ROLE_HOST; role < CI_ROLE_END; role++)
-		if (ci->roles[role] &&
-		    !strncmp(buf, ci->roles[role]->name,
-			     strlen(ci->roles[role]->name)))
-			break;
-
-	if (role == CI_ROLE_END || role == ci->role)
-		return -EINVAL;
-
-	pm_runtime_get_sync(ci->dev);
-	disable_irq(ci->irq);
-	ci_role_stop(ci);
-	ret = ci_role_start(ci, role);
-	enable_irq(ci->irq);
-	pm_runtime_put_sync(ci->dev);
-
-	return ret ? ret : count;
-}
-
-static int ci_role_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ci_role_show, inode->i_private);
-}
-
-static const struct file_operations ci_role_fops = {
-	.open		= ci_role_open,
-	.write		= ci_role_write,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int ci_registers_show(struct seq_file *s, void *unused)
 {
 	struct ci_hdrc *ci = s->private;
@@ -354,7 +300,6 @@ void dbg_create_files(struct ci_hdrc *ci)
 	if (ci_otg_is_fsm_mode(ci))
 		debugfs_create_file("otg", S_IRUGO, dir, ci, &ci_otg_fops);
 
-	debugfs_create_file("role", S_IRUGO | S_IWUSR, dir, ci, &ci_role_fops);
 	debugfs_create_file("registers", S_IRUGO, dir, ci, &ci_registers_fops);
 }
 
diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c
index 1f0951be15ab..c553decb5461 100644
--- a/drivers/usb/class/cdc-wdm.c
+++ b/drivers/usb/class/cdc-wdm.c
@@ -929,7 +929,8 @@ static void wdm_wwan_init(struct wdm_device *desc)
 		return;
 	}
 
-	port = wwan_create_port(&intf->dev, desc->wwanp_type, &wdm_wwan_port_ops, desc);
+	port = wwan_create_port(&intf->dev, desc->wwanp_type, &wdm_wwan_port_ops,
+				NULL, desc);
 	if (IS_ERR(port)) {
 		dev_err(&intf->dev, "%s: Unable to create WWAN port\n",
 			dev_name(intf->usb_dev));
diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c
index a98b2108376a..84d91b1c1eed 100644
--- a/drivers/usb/common/ulpi.c
+++ b/drivers/usb/common/ulpi.c
@@ -90,7 +90,7 @@ static void ulpi_remove(struct device *dev)
 		drv->remove(to_ulpi_dev(dev));
 }
 
-static struct bus_type ulpi_bus = {
+static const struct bus_type ulpi_bus = {
 	.name = "ulpi",
 	.match = ulpi_match,
 	.uevent = ulpi_uevent,
@@ -229,7 +229,7 @@ static int ulpi_read_id(struct ulpi *ulpi)
 	request_module("ulpi:v%04xp%04x", ulpi->id.vendor, ulpi->id.product);
 	return 0;
 err:
-	of_device_request_module(&ulpi->dev);
+	of_request_module(ulpi->dev.of_node);
 	return 0;
 }
 
diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c
index a0e076c6f3a4..f58a0299fb3b 100644
--- a/drivers/usb/core/driver.c
+++ b/drivers/usb/core/driver.c
@@ -2025,7 +2025,7 @@ int usb_disable_usb2_hardware_lpm(struct usb_device *udev)
 
 #endif /* CONFIG_PM */
 
-struct bus_type usb_bus_type = {
+const struct bus_type usb_bus_type = {
 	.name =		"usb",
 	.match =	usb_device_match,
 	.uevent =	usb_uevent,
diff --git a/drivers/usb/core/file.c b/drivers/usb/core/file.c
index da7d88e069e6..c4ed3310e069 100644
--- a/drivers/usb/core/file.c
+++ b/drivers/usb/core/file.c
@@ -88,7 +88,7 @@ static int init_usb_class(void)
 	}
 
 	kref_init(&usb_class->kref);
-	usb_class->class = class_create(THIS_MODULE, "usbmisc");
+	usb_class->class = class_create("usbmisc");
 	if (IS_ERR(usb_class->class)) {
 		result = PTR_ERR(usb_class->class);
 		printk(KERN_ERR "class_create failed for usb devices\n");
diff --git a/drivers/usb/core/message.c b/drivers/usb/core/message.c
index cc404bb7e8f7..b5811620f1de 100644
--- a/drivers/usb/core/message.c
+++ b/drivers/usb/core/message.c
@@ -1908,6 +1908,45 @@ static void __usb_queue_reset_device(struct work_struct *ws)
 	usb_put_intf(iface);	/* Undo _get_ in usb_queue_reset_device() */
 }
 
+/*
+ * Internal function to set the wireless_status sysfs attribute
+ * See usb_set_wireless_status() for more details
+ */
+static void __usb_wireless_status_intf(struct work_struct *ws)
+{
+	struct usb_interface *iface =
+		container_of(ws, struct usb_interface, wireless_status_work);
+
+	device_lock(iface->dev.parent);
+	if (iface->sysfs_files_created)
+		usb_update_wireless_status_attr(iface);
+	device_unlock(iface->dev.parent);
+	usb_put_intf(iface);	/* Undo _get_ in usb_set_wireless_status() */
+}
+
+/**
+ * usb_set_wireless_status - sets the wireless_status struct member
+ * @iface: the interface to modify
+ * @status: the new wireless status
+ *
+ * Set the wireless_status struct member to the new value, and emit
+ * sysfs changes as necessary.
+ *
+ * Returns: 0 on success, -EALREADY if already set.
+ */
+int usb_set_wireless_status(struct usb_interface *iface,
+		enum usb_wireless_status status)
+{
+	if (iface->wireless_status == status)
+		return -EALREADY;
+
+	usb_get_intf(iface);
+	iface->wireless_status = status;
+	schedule_work(&iface->wireless_status_work);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(usb_set_wireless_status);
 
 /*
  * usb_set_configuration - Makes a particular device setting be current
@@ -2100,6 +2139,7 @@ free_interfaces:
 		intf->dev.type = &usb_if_device_type;
 		intf->dev.groups = usb_interface_groups;
 		INIT_WORK(&intf->reset_ws, __usb_queue_reset_device);
+		INIT_WORK(&intf->wireless_status_work, __usb_wireless_status_intf);
 		intf->minor = -1;
 		device_initialize(&intf->dev);
 		pm_runtime_no_callbacks(&intf->dev);
diff --git a/drivers/usb/core/sysfs.c b/drivers/usb/core/sysfs.c
index b63f78e48c74..323dc02becbe 100644
--- a/drivers/usb/core/sysfs.c
+++ b/drivers/usb/core/sysfs.c
@@ -1227,9 +1227,59 @@ static const struct attribute_group intf_assoc_attr_grp = {
 	.is_visible =	intf_assoc_attrs_are_visible,
 };
 
+static ssize_t wireless_status_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct usb_interface *intf;
+
+	intf = to_usb_interface(dev);
+	if (intf->wireless_status == USB_WIRELESS_STATUS_DISCONNECTED)
+		return sysfs_emit(buf, "%s\n", "disconnected");
+	return sysfs_emit(buf, "%s\n", "connected");
+}
+static DEVICE_ATTR_RO(wireless_status);
+
+static struct attribute *intf_wireless_status_attrs[] = {
+	&dev_attr_wireless_status.attr,
+	NULL
+};
+
+static umode_t intf_wireless_status_attr_is_visible(struct kobject *kobj,
+		struct attribute *a, int n)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct usb_interface *intf = to_usb_interface(dev);
+
+	if (a != &dev_attr_wireless_status.attr ||
+	    intf->wireless_status != USB_WIRELESS_STATUS_NA)
+		return a->mode;
+	return 0;
+}
+
+static const struct attribute_group intf_wireless_status_attr_grp = {
+	.attrs =	intf_wireless_status_attrs,
+	.is_visible =	intf_wireless_status_attr_is_visible,
+};
+
+int usb_update_wireless_status_attr(struct usb_interface *intf)
+{
+	struct device *dev = &intf->dev;
+	int ret;
+
+	ret = sysfs_update_group(&dev->kobj, &intf_wireless_status_attr_grp);
+	if (ret < 0)
+		return ret;
+
+	sysfs_notify(&dev->kobj, NULL, "wireless_status");
+	kobject_uevent(&dev->kobj, KOBJ_CHANGE);
+
+	return 0;
+}
+
 const struct attribute_group *usb_interface_groups[] = {
 	&intf_attr_grp,
 	&intf_assoc_attr_grp,
+	&intf_wireless_status_attr_grp,
 	NULL
 };
 
diff --git a/drivers/usb/core/usb-acpi.c b/drivers/usb/core/usb-acpi.c
index 533baa85083c..a34b22537d7c 100644
--- a/drivers/usb/core/usb-acpi.c
+++ b/drivers/usb/core/usb-acpi.c
@@ -81,15 +81,11 @@ int usb_acpi_port_lpm_incapable(struct usb_device *hdev, int index)
 		return -ENODEV;
 	}
 
-	obj = acpi_evaluate_dsm(port_handle, &guid, 0,
-				USB_DSM_DISABLE_U1_U2_FOR_PORT, NULL);
-
-	if (!obj)
-		return -ENODEV;
-
-	if (obj->type != ACPI_TYPE_INTEGER) {
+	obj = acpi_evaluate_dsm_typed(port_handle, &guid, 0,
+				      USB_DSM_DISABLE_U1_U2_FOR_PORT, NULL,
+				      ACPI_TYPE_INTEGER);
+	if (!obj) {
 		dev_dbg(&hdev->dev, "evaluate port-%d _DSM failed\n", port1);
-		ACPI_FREE(obj);
 		return -EINVAL;
 	}
 
diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c
index 34742fbbd84d..901ec732321c 100644
--- a/drivers/usb/core/usb.c
+++ b/drivers/usb/core/usb.c
@@ -207,6 +207,82 @@ int usb_find_common_endpoints_reverse(struct usb_host_interface *alt,
 EXPORT_SYMBOL_GPL(usb_find_common_endpoints_reverse);
 
 /**
+ * usb_find_endpoint() - Given an endpoint address, search for the endpoint's
+ * usb_host_endpoint structure in an interface's current altsetting.
+ * @intf: the interface whose current altsetting should be searched
+ * @ep_addr: the endpoint address (number and direction) to find
+ *
+ * Search the altsetting's list of endpoints for one with the specified address.
+ *
+ * Return: Pointer to the usb_host_endpoint if found, %NULL otherwise.
+ */
+static const struct usb_host_endpoint *usb_find_endpoint(
+		const struct usb_interface *intf, unsigned int ep_addr)
+{
+	int n;
+	const struct usb_host_endpoint *ep;
+
+	n = intf->cur_altsetting->desc.bNumEndpoints;
+	ep = intf->cur_altsetting->endpoint;
+	for (; n > 0; (--n, ++ep)) {
+		if (ep->desc.bEndpointAddress == ep_addr)
+			return ep;
+	}
+	return NULL;
+}
+
+/**
+ * usb_check_bulk_endpoints - Check whether an interface's current altsetting
+ * contains a set of bulk endpoints with the given addresses.
+ * @intf: the interface whose current altsetting should be searched
+ * @ep_addrs: 0-terminated array of the endpoint addresses (number and
+ * direction) to look for
+ *
+ * Search for endpoints with the specified addresses and check their types.
+ *
+ * Return: %true if all the endpoints are found and are bulk, %false otherwise.
+ */
+bool usb_check_bulk_endpoints(
+		const struct usb_interface *intf, const u8 *ep_addrs)
+{
+	const struct usb_host_endpoint *ep;
+
+	for (; *ep_addrs; ++ep_addrs) {
+		ep = usb_find_endpoint(intf, *ep_addrs);
+		if (!ep || !usb_endpoint_xfer_bulk(&ep->desc))
+			return false;
+	}
+	return true;
+}
+EXPORT_SYMBOL_GPL(usb_check_bulk_endpoints);
+
+/**
+ * usb_check_int_endpoints - Check whether an interface's current altsetting
+ * contains a set of interrupt endpoints with the given addresses.
+ * @intf: the interface whose current altsetting should be searched
+ * @ep_addrs: 0-terminated array of the endpoint addresses (number and
+ * direction) to look for
+ *
+ * Search for endpoints with the specified addresses and check their types.
+ *
+ * Return: %true if all the endpoints are found and are interrupt,
+ * %false otherwise.
+ */
+bool usb_check_int_endpoints(
+		const struct usb_interface *intf, const u8 *ep_addrs)
+{
+	const struct usb_host_endpoint *ep;
+
+	for (; *ep_addrs; ++ep_addrs) {
+		ep = usb_find_endpoint(intf, *ep_addrs);
+		if (!ep || !usb_endpoint_xfer_int(&ep->desc))
+			return false;
+	}
+	return true;
+}
+EXPORT_SYMBOL_GPL(usb_check_int_endpoints);
+
+/**
  * usb_find_alt_setting() - Given a configuration, find the alternate setting
  * for the given interface.
  * @config: the configuration to search (not necessarily the current config).
diff --git a/drivers/usb/core/usb.h b/drivers/usb/core/usb.h
index 0eac7d4285d1..ffe3f6818e9c 100644
--- a/drivers/usb/core/usb.h
+++ b/drivers/usb/core/usb.h
@@ -15,6 +15,7 @@ extern int usb_create_sysfs_dev_files(struct usb_device *dev);
 extern void usb_remove_sysfs_dev_files(struct usb_device *dev);
 extern void usb_create_sysfs_intf_files(struct usb_interface *intf);
 extern void usb_remove_sysfs_intf_files(struct usb_interface *intf);
+extern int usb_update_wireless_status_attr(struct usb_interface *intf);
 extern int usb_create_ep_devs(struct device *parent,
 				struct usb_host_endpoint *endpoint,
 				struct usb_device *udev);
@@ -140,7 +141,7 @@ static inline int usb_disable_usb2_hardware_lpm(struct usb_device *udev)
 
 #endif
 
-extern struct bus_type usb_bus_type;
+extern const struct bus_type usb_bus_type;
 extern struct mutex usb_port_peer_mutex;
 extern struct device_type usb_device_type;
 extern struct device_type usb_if_device_type;
diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h
index 40cf2880d7e5..0bb4c0c845bf 100644
--- a/drivers/usb/dwc2/core.h
+++ b/drivers/usb/dwc2/core.h
@@ -1003,6 +1003,7 @@ struct dwc2_hregs_backup {
  * @ctrl_out_desc:	EP0 OUT data phase desc chain pointer
  * @irq:		Interrupt request line number
  * @clk:		Pointer to otg clock
+ * @utmi_clk:		Pointer to utmi_clk clock
  * @reset:		Pointer to dwc2 reset controller
  * @reset_ecc:          Pointer to dwc2 optional reset controller in Stratix10.
  * @regset:		A pointer to a struct debugfs_regset32, which contains
@@ -1065,6 +1066,7 @@ struct dwc2_hsotg {
 	void *priv;
 	int     irq;
 	struct clk *clk;
+	struct clk *utmi_clk;
 	struct reset_control *reset;
 	struct reset_control *reset_ecc;
 
diff --git a/drivers/usb/dwc2/hcd_queue.c b/drivers/usb/dwc2/hcd_queue.c
index 0a1145592fc7..0d4495c6b9f7 100644
--- a/drivers/usb/dwc2/hcd_queue.c
+++ b/drivers/usb/dwc2/hcd_queue.c
@@ -1078,7 +1078,7 @@ static void dwc2_pick_first_frame(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
 	earliest_frame = dwc2_frame_num_inc(frame_number, 1);
 	next_active_frame = earliest_frame;
 
-	/* Get the "no microframe schduler" out of the way... */
+	/* Get the "no microframe scheduler" out of the way... */
 	if (!hsotg->params.uframe_sched) {
 		if (qh->do_split)
 			/* Splits are active at microframe 0 minus 1 */
diff --git a/drivers/usb/dwc2/params.c b/drivers/usb/dwc2/params.c
index 9ed9fd956940..21d16533bd2f 100644
--- a/drivers/usb/dwc2/params.c
+++ b/drivers/usb/dwc2/params.c
@@ -508,8 +508,7 @@ static void dwc2_get_device_properties(struct dwc2_hsotg *hsotg)
 		of_usb_update_otg_caps(hsotg->dev->of_node, &p->otg_caps);
 	}
 
-	if (of_find_property(hsotg->dev->of_node, "disable-over-current", NULL))
-		p->oc_disable = true;
+	p->oc_disable = of_property_read_bool(hsotg->dev->of_node, "disable-over-current");
 }
 
 static void dwc2_check_param_otg_cap(struct dwc2_hsotg *hsotg)
diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c
index d1589ba7d322..5aee284018c0 100644
--- a/drivers/usb/dwc2/platform.c
+++ b/drivers/usb/dwc2/platform.c
@@ -101,10 +101,16 @@ static int __dwc2_lowlevel_hw_enable(struct dwc2_hsotg *hsotg)
 	if (ret)
 		return ret;
 
+	if (hsotg->utmi_clk) {
+		ret = clk_prepare_enable(hsotg->utmi_clk);
+		if (ret)
+			goto err_dis_reg;
+	}
+
 	if (hsotg->clk) {
 		ret = clk_prepare_enable(hsotg->clk);
 		if (ret)
-			return ret;
+			goto err_dis_utmi_clk;
 	}
 
 	if (hsotg->uphy) {
@@ -113,10 +119,29 @@ static int __dwc2_lowlevel_hw_enable(struct dwc2_hsotg *hsotg)
 		ret = hsotg->plat->phy_init(pdev, hsotg->plat->phy_type);
 	} else {
 		ret = phy_init(hsotg->phy);
-		if (ret == 0)
+		if (ret == 0) {
 			ret = phy_power_on(hsotg->phy);
+			if (ret)
+				phy_exit(hsotg->phy);
+		}
 	}
 
+	if (ret)
+		goto err_dis_clk;
+
+	return 0;
+
+err_dis_clk:
+	if (hsotg->clk)
+		clk_disable_unprepare(hsotg->clk);
+
+err_dis_utmi_clk:
+	if (hsotg->utmi_clk)
+		clk_disable_unprepare(hsotg->utmi_clk);
+
+err_dis_reg:
+	regulator_bulk_disable(ARRAY_SIZE(hsotg->supplies), hsotg->supplies);
+
 	return ret;
 }
 
@@ -156,6 +181,9 @@ static int __dwc2_lowlevel_hw_disable(struct dwc2_hsotg *hsotg)
 	if (hsotg->clk)
 		clk_disable_unprepare(hsotg->clk);
 
+	if (hsotg->utmi_clk)
+		clk_disable_unprepare(hsotg->utmi_clk);
+
 	return regulator_bulk_disable(ARRAY_SIZE(hsotg->supplies), hsotg->supplies);
 }
 
@@ -232,6 +260,11 @@ static int dwc2_lowlevel_hw_init(struct dwc2_hsotg *hsotg)
 	if (IS_ERR(hsotg->clk))
 		return dev_err_probe(hsotg->dev, PTR_ERR(hsotg->clk), "cannot get otg clock\n");
 
+	hsotg->utmi_clk = devm_clk_get_optional(hsotg->dev, "utmi");
+	if (IS_ERR(hsotg->utmi_clk))
+		return dev_err_probe(hsotg->dev, PTR_ERR(hsotg->utmi_clk),
+				     "cannot get utmi clock\n");
+
 	/* Regulators */
 	for (i = 0; i < ARRAY_SIZE(hsotg->supplies); i++)
 		hsotg->supplies[i].supply = dwc2_hsotg_supply_names[i];
diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c
index 476b63618511..0beaab932e7d 100644
--- a/drivers/usb/dwc3/core.c
+++ b/drivers/usb/dwc3/core.c
@@ -534,90 +534,6 @@ void dwc3_event_buffers_cleanup(struct dwc3 *dwc)
 	dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), 0);
 }
 
-static int dwc3_alloc_scratch_buffers(struct dwc3 *dwc)
-{
-	if (!dwc->has_hibernation)
-		return 0;
-
-	if (!dwc->nr_scratch)
-		return 0;
-
-	dwc->scratchbuf = kmalloc_array(dwc->nr_scratch,
-			DWC3_SCRATCHBUF_SIZE, GFP_KERNEL);
-	if (!dwc->scratchbuf)
-		return -ENOMEM;
-
-	return 0;
-}
-
-static int dwc3_setup_scratch_buffers(struct dwc3 *dwc)
-{
-	dma_addr_t scratch_addr;
-	u32 param;
-	int ret;
-
-	if (!dwc->has_hibernation)
-		return 0;
-
-	if (!dwc->nr_scratch)
-		return 0;
-
-	 /* should never fall here */
-	if (!WARN_ON(dwc->scratchbuf))
-		return 0;
-
-	scratch_addr = dma_map_single(dwc->sysdev, dwc->scratchbuf,
-			dwc->nr_scratch * DWC3_SCRATCHBUF_SIZE,
-			DMA_BIDIRECTIONAL);
-	if (dma_mapping_error(dwc->sysdev, scratch_addr)) {
-		dev_err(dwc->sysdev, "failed to map scratch buffer\n");
-		ret = -EFAULT;
-		goto err0;
-	}
-
-	dwc->scratch_addr = scratch_addr;
-
-	param = lower_32_bits(scratch_addr);
-
-	ret = dwc3_send_gadget_generic_command(dwc,
-			DWC3_DGCMD_SET_SCRATCHPAD_ADDR_LO, param);
-	if (ret < 0)
-		goto err1;
-
-	param = upper_32_bits(scratch_addr);
-
-	ret = dwc3_send_gadget_generic_command(dwc,
-			DWC3_DGCMD_SET_SCRATCHPAD_ADDR_HI, param);
-	if (ret < 0)
-		goto err1;
-
-	return 0;
-
-err1:
-	dma_unmap_single(dwc->sysdev, dwc->scratch_addr, dwc->nr_scratch *
-			DWC3_SCRATCHBUF_SIZE, DMA_BIDIRECTIONAL);
-
-err0:
-	return ret;
-}
-
-static void dwc3_free_scratch_buffers(struct dwc3 *dwc)
-{
-	if (!dwc->has_hibernation)
-		return;
-
-	if (!dwc->nr_scratch)
-		return;
-
-	 /* should never fall here */
-	if (!WARN_ON(dwc->scratchbuf))
-		return;
-
-	dma_unmap_single(dwc->sysdev, dwc->scratch_addr, dwc->nr_scratch *
-			DWC3_SCRATCHBUF_SIZE, DMA_BIDIRECTIONAL);
-	kfree(dwc->scratchbuf);
-}
-
 static void dwc3_core_num_eps(struct dwc3 *dwc)
 {
 	struct dwc3_hwparams	*parms = &dwc->hwparams;
@@ -800,11 +716,91 @@ static int dwc3_phy_setup(struct dwc3 *dwc)
 	if (dwc->dis_u2_freeclk_exists_quirk || dwc->gfladj_refclk_lpm_sel)
 		reg &= ~DWC3_GUSB2PHYCFG_U2_FREECLK_EXISTS;
 
+	/*
+	 * Some ULPI USB PHY does not support internal VBUS supply, to drive
+	 * the CPEN pin requires the configuration of the ULPI DRVVBUSEXTERNAL
+	 * bit of OTG_CTRL register. Controller configures the USB2 PHY
+	 * ULPIEXTVBUSDRV bit[17] of the GUSB2PHYCFG register to drive vBus
+	 * with an external supply.
+	 */
+	if (dwc->ulpi_ext_vbus_drv)
+		reg |= DWC3_GUSB2PHYCFG_ULPIEXTVBUSDRV;
+
 	dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 
 	return 0;
 }
 
+static int dwc3_phy_init(struct dwc3 *dwc)
+{
+	int ret;
+
+	usb_phy_init(dwc->usb2_phy);
+	usb_phy_init(dwc->usb3_phy);
+
+	ret = phy_init(dwc->usb2_generic_phy);
+	if (ret < 0)
+		goto err_shutdown_usb3_phy;
+
+	ret = phy_init(dwc->usb3_generic_phy);
+	if (ret < 0)
+		goto err_exit_usb2_phy;
+
+	return 0;
+
+err_exit_usb2_phy:
+	phy_exit(dwc->usb2_generic_phy);
+err_shutdown_usb3_phy:
+	usb_phy_shutdown(dwc->usb3_phy);
+	usb_phy_shutdown(dwc->usb2_phy);
+
+	return ret;
+}
+
+static void dwc3_phy_exit(struct dwc3 *dwc)
+{
+	phy_exit(dwc->usb3_generic_phy);
+	phy_exit(dwc->usb2_generic_phy);
+
+	usb_phy_shutdown(dwc->usb3_phy);
+	usb_phy_shutdown(dwc->usb2_phy);
+}
+
+static int dwc3_phy_power_on(struct dwc3 *dwc)
+{
+	int ret;
+
+	usb_phy_set_suspend(dwc->usb2_phy, 0);
+	usb_phy_set_suspend(dwc->usb3_phy, 0);
+
+	ret = phy_power_on(dwc->usb2_generic_phy);
+	if (ret < 0)
+		goto err_suspend_usb3_phy;
+
+	ret = phy_power_on(dwc->usb3_generic_phy);
+	if (ret < 0)
+		goto err_power_off_usb2_phy;
+
+	return 0;
+
+err_power_off_usb2_phy:
+	phy_power_off(dwc->usb2_generic_phy);
+err_suspend_usb3_phy:
+	usb_phy_set_suspend(dwc->usb3_phy, 1);
+	usb_phy_set_suspend(dwc->usb2_phy, 1);
+
+	return ret;
+}
+
+static void dwc3_phy_power_off(struct dwc3 *dwc)
+{
+	phy_power_off(dwc->usb3_generic_phy);
+	phy_power_off(dwc->usb2_generic_phy);
+
+	usb_phy_set_suspend(dwc->usb3_phy, 1);
+	usb_phy_set_suspend(dwc->usb2_phy, 1);
+}
+
 static int dwc3_clk_enable(struct dwc3 *dwc)
 {
 	int ret;
@@ -840,17 +836,8 @@ static void dwc3_clk_disable(struct dwc3 *dwc)
 static void dwc3_core_exit(struct dwc3 *dwc)
 {
 	dwc3_event_buffers_cleanup(dwc);
-
-	usb_phy_set_suspend(dwc->usb2_phy, 1);
-	usb_phy_set_suspend(dwc->usb3_phy, 1);
-	phy_power_off(dwc->usb2_generic_phy);
-	phy_power_off(dwc->usb3_generic_phy);
-
-	usb_phy_shutdown(dwc->usb2_phy);
-	usb_phy_shutdown(dwc->usb3_phy);
-	phy_exit(dwc->usb2_generic_phy);
-	phy_exit(dwc->usb3_generic_phy);
-
+	dwc3_phy_power_off(dwc);
+	dwc3_phy_exit(dwc);
 	dwc3_clk_disable(dwc);
 	reset_control_assert(dwc->reset);
 }
@@ -877,7 +864,6 @@ static bool dwc3_core_is_valid(struct dwc3 *dwc)
 
 static void dwc3_core_setup_global_control(struct dwc3 *dwc)
 {
-	u32 hwparams4 = dwc->hwparams.hwparams4;
 	u32 reg;
 
 	reg = dwc3_readl(dwc->regs, DWC3_GCTL);
@@ -905,9 +891,6 @@ static void dwc3_core_setup_global_control(struct dwc3 *dwc)
 			reg &= ~DWC3_GCTL_DSBLCLKGTNG;
 		break;
 	case DWC3_GHWPARAMS1_EN_PWROPT_HIB:
-		/* enable hibernation here */
-		dwc->nr_scratch = DWC3_GHWPARAMS4_HIBER_SCRATCHBUFS(hwparams4);
-
 		/*
 		 * REVISIT Enabling this bit so that host-mode hibernation
 		 * will work. Device-mode hibernation is not yet implemented.
@@ -1096,7 +1079,7 @@ static int dwc3_core_init(struct dwc3 *dwc)
 
 	ret = dwc3_phy_setup(dwc);
 	if (ret)
-		goto err0;
+		return ret;
 
 	if (!dwc->ulpi_ready) {
 		ret = dwc3_core_ulpi_init(dwc);
@@ -1105,7 +1088,7 @@ static int dwc3_core_init(struct dwc3 *dwc)
 				dwc3_core_soft_reset(dwc);
 				ret = -EPROBE_DEFER;
 			}
-			goto err0;
+			return ret;
 		}
 		dwc->ulpi_ready = true;
 	}
@@ -1113,25 +1096,17 @@ static int dwc3_core_init(struct dwc3 *dwc)
 	if (!dwc->phys_ready) {
 		ret = dwc3_core_get_phy(dwc);
 		if (ret)
-			goto err0a;
+			goto err_exit_ulpi;
 		dwc->phys_ready = true;
 	}
 
-	usb_phy_init(dwc->usb2_phy);
-	usb_phy_init(dwc->usb3_phy);
-	ret = phy_init(dwc->usb2_generic_phy);
-	if (ret < 0)
-		goto err0a;
-
-	ret = phy_init(dwc->usb3_generic_phy);
-	if (ret < 0) {
-		phy_exit(dwc->usb2_generic_phy);
-		goto err0a;
-	}
+	ret = dwc3_phy_init(dwc);
+	if (ret)
+		goto err_exit_ulpi;
 
 	ret = dwc3_core_soft_reset(dwc);
 	if (ret)
-		goto err1;
+		goto err_exit_phy;
 
 	if (hw_mode == DWC3_GHWPARAMS0_MODE_DRD &&
 	    !DWC3_VER_IS_WITHIN(DWC3, ANY, 194A)) {
@@ -1151,10 +1126,6 @@ static int dwc3_core_init(struct dwc3 *dwc)
 	dwc3_core_setup_global_control(dwc);
 	dwc3_core_num_eps(dwc);
 
-	ret = dwc3_setup_scratch_buffers(dwc);
-	if (ret)
-		goto err1;
-
 	/* Set power down scale of suspend_clk */
 	dwc3_set_power_down_clk_scale(dwc);
 
@@ -1166,20 +1137,14 @@ static int dwc3_core_init(struct dwc3 *dwc)
 
 	dwc3_set_incr_burst_type(dwc);
 
-	usb_phy_set_suspend(dwc->usb2_phy, 0);
-	usb_phy_set_suspend(dwc->usb3_phy, 0);
-	ret = phy_power_on(dwc->usb2_generic_phy);
-	if (ret < 0)
-		goto err2;
-
-	ret = phy_power_on(dwc->usb3_generic_phy);
-	if (ret < 0)
-		goto err3;
+	dwc3_phy_power_on(dwc);
+	if (ret)
+		goto err_exit_phy;
 
 	ret = dwc3_event_buffers_setup(dwc);
 	if (ret) {
 		dev_err(dwc->dev, "failed to setup event buffers\n");
-		goto err4;
+		goto err_power_off_phy;
 	}
 
 	/*
@@ -1233,6 +1198,9 @@ static int dwc3_core_init(struct dwc3 *dwc)
 		if (dwc->parkmode_disable_ss_quirk)
 			reg |= DWC3_GUCTL1_PARKMODE_DISABLE_SS;
 
+		if (dwc->parkmode_disable_hs_quirk)
+			reg |= DWC3_GUCTL1_PARKMODE_DISABLE_HS;
+
 		if (DWC3_VER_IS_WITHIN(DWC3, 290A, ANY) &&
 		    (dwc->maximum_speed == USB_SPEED_HIGH ||
 		     dwc->maximum_speed == USB_SPEED_FULL))
@@ -1296,26 +1264,13 @@ static int dwc3_core_init(struct dwc3 *dwc)
 
 	return 0;
 
-err4:
-	phy_power_off(dwc->usb3_generic_phy);
-
-err3:
-	phy_power_off(dwc->usb2_generic_phy);
-
-err2:
-	usb_phy_set_suspend(dwc->usb2_phy, 1);
-	usb_phy_set_suspend(dwc->usb3_phy, 1);
-
-err1:
-	usb_phy_shutdown(dwc->usb2_phy);
-	usb_phy_shutdown(dwc->usb3_phy);
-	phy_exit(dwc->usb2_generic_phy);
-	phy_exit(dwc->usb3_generic_phy);
-
-err0a:
+err_power_off_phy:
+	dwc3_phy_power_off(dwc);
+err_exit_phy:
+	dwc3_phy_exit(dwc);
+err_exit_ulpi:
 	dwc3_ulpi_exit(dwc);
 
-err0:
 	return ret;
 }
 
@@ -1553,8 +1508,12 @@ static void dwc3_get_properties(struct dwc3 *dwc)
 				"snps,dis-tx-ipgap-linecheck-quirk");
 	dwc->resume_hs_terminations = device_property_read_bool(dev,
 				"snps,resume-hs-terminations");
+	dwc->ulpi_ext_vbus_drv = device_property_read_bool(dev,
+				"snps,ulpi-ext-vbus-drv");
 	dwc->parkmode_disable_ss_quirk = device_property_read_bool(dev,
 				"snps,parkmode-disable-ss-quirk");
+	dwc->parkmode_disable_hs_quirk = device_property_read_bool(dev,
+				"snps,parkmode-disable-hs-quirk");
 	dwc->gfladj_refclk_lpm_sel = device_property_read_bool(dev,
 				"snps,gfladj-refclk-lpm-sel-quirk");
 
@@ -1750,16 +1709,72 @@ static struct extcon_dev *dwc3_get_extcon(struct dwc3 *dwc)
 	return edev;
 }
 
+static int dwc3_get_clocks(struct dwc3 *dwc)
+{
+	struct device *dev = dwc->dev;
+
+	if (!dev->of_node)
+		return 0;
+
+	/*
+	 * Clocks are optional, but new DT platforms should support all clocks
+	 * as required by the DT-binding.
+	 * Some devices have different clock names in legacy device trees,
+	 * check for them to retain backwards compatibility.
+	 */
+	dwc->bus_clk = devm_clk_get_optional(dev, "bus_early");
+	if (IS_ERR(dwc->bus_clk)) {
+		return dev_err_probe(dev, PTR_ERR(dwc->bus_clk),
+				"could not get bus clock\n");
+	}
+
+	if (dwc->bus_clk == NULL) {
+		dwc->bus_clk = devm_clk_get_optional(dev, "bus_clk");
+		if (IS_ERR(dwc->bus_clk)) {
+			return dev_err_probe(dev, PTR_ERR(dwc->bus_clk),
+					"could not get bus clock\n");
+		}
+	}
+
+	dwc->ref_clk = devm_clk_get_optional(dev, "ref");
+	if (IS_ERR(dwc->ref_clk)) {
+		return dev_err_probe(dev, PTR_ERR(dwc->ref_clk),
+				"could not get ref clock\n");
+	}
+
+	if (dwc->ref_clk == NULL) {
+		dwc->ref_clk = devm_clk_get_optional(dev, "ref_clk");
+		if (IS_ERR(dwc->ref_clk)) {
+			return dev_err_probe(dev, PTR_ERR(dwc->ref_clk),
+					"could not get ref clock\n");
+		}
+	}
+
+	dwc->susp_clk = devm_clk_get_optional(dev, "suspend");
+	if (IS_ERR(dwc->susp_clk)) {
+		return dev_err_probe(dev, PTR_ERR(dwc->susp_clk),
+				"could not get suspend clock\n");
+	}
+
+	if (dwc->susp_clk == NULL) {
+		dwc->susp_clk = devm_clk_get_optional(dev, "suspend_clk");
+		if (IS_ERR(dwc->susp_clk)) {
+			return dev_err_probe(dev, PTR_ERR(dwc->susp_clk),
+					"could not get suspend clock\n");
+		}
+	}
+
+	return 0;
+}
+
 static int dwc3_probe(struct platform_device *pdev)
 {
 	struct device		*dev = &pdev->dev;
 	struct resource		*res, dwc_res;
+	void __iomem		*regs;
 	struct dwc3		*dwc;
-
 	int			ret;
 
-	void __iomem		*regs;
-
 	dwc = devm_kzalloc(dev, sizeof(*dwc), GFP_KERNEL);
 	if (!dwc)
 		return -ENOMEM;
@@ -1797,77 +1812,25 @@ static int dwc3_probe(struct platform_device *pdev)
 	dwc->reset = devm_reset_control_array_get_optional_shared(dev);
 	if (IS_ERR(dwc->reset)) {
 		ret = PTR_ERR(dwc->reset);
-		goto put_usb_psy;
+		goto err_put_psy;
 	}
 
-	if (dev->of_node) {
-		/*
-		 * Clocks are optional, but new DT platforms should support all
-		 * clocks as required by the DT-binding.
-		 * Some devices have different clock names in legacy device trees,
-		 * check for them to retain backwards compatibility.
-		 */
-		dwc->bus_clk = devm_clk_get_optional(dev, "bus_early");
-		if (IS_ERR(dwc->bus_clk)) {
-			ret = dev_err_probe(dev, PTR_ERR(dwc->bus_clk),
-					    "could not get bus clock\n");
-			goto put_usb_psy;
-		}
-
-		if (dwc->bus_clk == NULL) {
-			dwc->bus_clk = devm_clk_get_optional(dev, "bus_clk");
-			if (IS_ERR(dwc->bus_clk)) {
-				ret = dev_err_probe(dev, PTR_ERR(dwc->bus_clk),
-						    "could not get bus clock\n");
-				goto put_usb_psy;
-			}
-		}
-
-		dwc->ref_clk = devm_clk_get_optional(dev, "ref");
-		if (IS_ERR(dwc->ref_clk)) {
-			ret = dev_err_probe(dev, PTR_ERR(dwc->ref_clk),
-					    "could not get ref clock\n");
-			goto put_usb_psy;
-		}
-
-		if (dwc->ref_clk == NULL) {
-			dwc->ref_clk = devm_clk_get_optional(dev, "ref_clk");
-			if (IS_ERR(dwc->ref_clk)) {
-				ret = dev_err_probe(dev, PTR_ERR(dwc->ref_clk),
-						    "could not get ref clock\n");
-				goto put_usb_psy;
-			}
-		}
-
-		dwc->susp_clk = devm_clk_get_optional(dev, "suspend");
-		if (IS_ERR(dwc->susp_clk)) {
-			ret = dev_err_probe(dev, PTR_ERR(dwc->susp_clk),
-					    "could not get suspend clock\n");
-			goto put_usb_psy;
-		}
-
-		if (dwc->susp_clk == NULL) {
-			dwc->susp_clk = devm_clk_get_optional(dev, "suspend_clk");
-			if (IS_ERR(dwc->susp_clk)) {
-				ret = dev_err_probe(dev, PTR_ERR(dwc->susp_clk),
-						    "could not get suspend clock\n");
-				goto put_usb_psy;
-			}
-		}
-	}
+	ret = dwc3_get_clocks(dwc);
+	if (ret)
+		goto err_put_psy;
 
 	ret = reset_control_deassert(dwc->reset);
 	if (ret)
-		goto put_usb_psy;
+		goto err_put_psy;
 
 	ret = dwc3_clk_enable(dwc);
 	if (ret)
-		goto assert_reset;
+		goto err_assert_reset;
 
 	if (!dwc3_core_is_valid(dwc)) {
 		dev_err(dwc->dev, "this is not a DesignWare USB3 DRD Core\n");
 		ret = -ENODEV;
-		goto disable_clks;
+		goto err_disable_clks;
 	}
 
 	platform_set_drvdata(pdev, dwc);
@@ -1877,19 +1840,17 @@ static int dwc3_probe(struct platform_device *pdev)
 	    DWC3_GHWPARAMS0_AWIDTH(dwc->hwparams.hwparams0) == 64) {
 		ret = dma_set_mask_and_coherent(dwc->sysdev, DMA_BIT_MASK(64));
 		if (ret)
-			goto disable_clks;
+			goto err_disable_clks;
 	}
 
 	spin_lock_init(&dwc->lock);
 	mutex_init(&dwc->mutex);
 
+	pm_runtime_get_noresume(dev);
 	pm_runtime_set_active(dev);
 	pm_runtime_use_autosuspend(dev);
 	pm_runtime_set_autosuspend_delay(dev, DWC3_DEFAULT_AUTOSUSPEND_DELAY);
 	pm_runtime_enable(dev);
-	ret = pm_runtime_get_sync(dev);
-	if (ret < 0)
-		goto err1;
 
 	pm_runtime_forbid(dev);
 
@@ -1897,27 +1858,23 @@ static int dwc3_probe(struct platform_device *pdev)
 	if (ret) {
 		dev_err(dwc->dev, "failed to allocate event buffers\n");
 		ret = -ENOMEM;
-		goto err2;
+		goto err_allow_rpm;
 	}
 
 	dwc->edev = dwc3_get_extcon(dwc);
 	if (IS_ERR(dwc->edev)) {
 		ret = dev_err_probe(dwc->dev, PTR_ERR(dwc->edev), "failed to get extcon\n");
-		goto err3;
+		goto err_free_event_buffers;
 	}
 
 	ret = dwc3_get_dr_mode(dwc);
 	if (ret)
-		goto err3;
-
-	ret = dwc3_alloc_scratch_buffers(dwc);
-	if (ret)
-		goto err3;
+		goto err_free_event_buffers;
 
 	ret = dwc3_core_init(dwc);
 	if (ret) {
 		dev_err_probe(dev, ret, "failed to initialize core\n");
-		goto err4;
+		goto err_free_event_buffers;
 	}
 
 	dwc3_check_params(dwc);
@@ -1925,46 +1882,31 @@ static int dwc3_probe(struct platform_device *pdev)
 
 	ret = dwc3_core_init_mode(dwc);
 	if (ret)
-		goto err5;
+		goto err_exit_debugfs;
 
 	pm_runtime_put(dev);
 
 	return 0;
 
-err5:
+err_exit_debugfs:
 	dwc3_debugfs_exit(dwc);
 	dwc3_event_buffers_cleanup(dwc);
-
-	usb_phy_set_suspend(dwc->usb2_phy, 1);
-	usb_phy_set_suspend(dwc->usb3_phy, 1);
-	phy_power_off(dwc->usb2_generic_phy);
-	phy_power_off(dwc->usb3_generic_phy);
-
-	usb_phy_shutdown(dwc->usb2_phy);
-	usb_phy_shutdown(dwc->usb3_phy);
-	phy_exit(dwc->usb2_generic_phy);
-	phy_exit(dwc->usb3_generic_phy);
-
+	dwc3_phy_power_off(dwc);
+	dwc3_phy_exit(dwc);
 	dwc3_ulpi_exit(dwc);
-
-err4:
-	dwc3_free_scratch_buffers(dwc);
-
-err3:
+err_free_event_buffers:
 	dwc3_free_event_buffers(dwc);
-
-err2:
-	pm_runtime_allow(&pdev->dev);
-
-err1:
-	pm_runtime_put_sync(&pdev->dev);
-	pm_runtime_disable(&pdev->dev);
-
-disable_clks:
+err_allow_rpm:
+	pm_runtime_allow(dev);
+	pm_runtime_disable(dev);
+	pm_runtime_dont_use_autosuspend(dev);
+	pm_runtime_set_suspended(dev);
+	pm_runtime_put_noidle(dev);
+err_disable_clks:
 	dwc3_clk_disable(dwc);
-assert_reset:
+err_assert_reset:
 	reset_control_assert(dwc->reset);
-put_usb_psy:
+err_put_psy:
 	if (dwc->usb_psy)
 		power_supply_put(dwc->usb_psy);
 
@@ -1983,12 +1925,13 @@ static int dwc3_remove(struct platform_device *pdev)
 	dwc3_core_exit(dwc);
 	dwc3_ulpi_exit(dwc);
 
+	pm_runtime_allow(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_set_suspended(&pdev->dev);
 
 	dwc3_free_event_buffers(dwc);
-	dwc3_free_scratch_buffers(dwc);
 
 	if (dwc->usb_psy)
 		power_supply_put(dwc->usb_psy);
diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
index 4743e918dcaf..d56457c02996 100644
--- a/drivers/usb/dwc3/core.h
+++ b/drivers/usb/dwc3/core.h
@@ -263,6 +263,7 @@
 #define DWC3_GUCTL1_DEV_FORCE_20_CLK_FOR_30_CLK	BIT(26)
 #define DWC3_GUCTL1_DEV_L1_EXIT_BY_HW		BIT(24)
 #define DWC3_GUCTL1_PARKMODE_DISABLE_SS		BIT(17)
+#define DWC3_GUCTL1_PARKMODE_DISABLE_HS		BIT(16)
 #define DWC3_GUCTL1_RESUME_OPMODE_HS_HOST	BIT(10)
 
 /* Global Status Register */
@@ -280,6 +281,7 @@
 /* Global USB2 PHY Configuration Register */
 #define DWC3_GUSB2PHYCFG_PHYSOFTRST	BIT(31)
 #define DWC3_GUSB2PHYCFG_U2_FREECLK_EXISTS	BIT(30)
+#define DWC3_GUSB2PHYCFG_ULPIEXTVBUSDRV	BIT(17)
 #define DWC3_GUSB2PHYCFG_SUSPHY		BIT(6)
 #define DWC3_GUSB2PHYCFG_ULPI_UTMI	BIT(4)
 #define DWC3_GUSB2PHYCFG_ENBLSLPM	BIT(8)
@@ -526,6 +528,7 @@
 #define DWC3_DGCMD_SET_ENDPOINT_NRDY	0x0c
 #define DWC3_DGCMD_SET_ENDPOINT_PRIME	0x0d
 #define DWC3_DGCMD_RUN_SOC_BUS_LOOPBACK	0x10
+#define DWC3_DGCMD_DEV_NOTIFICATION	0x07
 
 #define DWC3_DGCMD_STATUS(n)		(((n) >> 12) & 0x0F)
 #define DWC3_DGCMD_CMDACT		BIT(10)
@@ -538,6 +541,8 @@
 #define DWC3_DGCMDPAR_TX_FIFO			BIT(5)
 #define DWC3_DGCMDPAR_LOOPBACK_DIS		(0 << 0)
 #define DWC3_DGCMDPAR_LOOPBACK_ENA		BIT(0)
+#define DWC3_DGCMDPAR_DN_FUNC_WAKE		BIT(0)
+#define DWC3_DGCMDPAR_INTF_SEL(n)		((n) << 4)
 
 /* Device Endpoint Command Register */
 #define DWC3_DEPCMD_PARAM_SHIFT		16
@@ -969,12 +974,10 @@ struct dwc3_scratchpad_array {
  * @drd_work: workqueue used for role swapping
  * @ep0_trb: trb which is used for the ctrl_req
  * @bounce: address of bounce buffer
- * @scratchbuf: address of scratch buffer
  * @setup_buf: used while precessing STD USB requests
  * @ep0_trb_addr: dma address of @ep0_trb
  * @bounce_addr: dma address of @bounce
  * @ep0_usb_req: dummy req used while handling STD USB requests
- * @scratch_addr: dma address of scratchbuf
  * @ep0_in_setup: one control transfer is completed and enter setup phase
  * @lock: for synchronizing
  * @mutex: for mode switching
@@ -999,7 +1002,6 @@ struct dwc3_scratchpad_array {
  * @current_otg_role: current role of operation while using the OTG block
  * @desired_otg_role: desired role of operation while using the OTG block
  * @otg_restart_host: flag that OTG controller needs to restart host
- * @nr_scratch: number of scratch buffers
  * @u1u2: only used on revisions <1.83a for workaround
  * @maximum_speed: maximum speed requested (mainly for testing purposes)
  * @max_ssp_rate: SuperSpeed Plus maximum signaling rate and lane count
@@ -1056,7 +1058,6 @@ struct dwc3_scratchpad_array {
  * @delayed_status: true when gadget driver asks for delayed status
  * @ep0_bounced: true when we used bounce buffer
  * @ep0_expect_in: true when we expect a DATA IN transfer
- * @has_hibernation: true when dwc3 was configured with Hibernation
  * @sysdev_is_parent: true when dwc3 device has a parent driver
  * @has_lpm_erratum: true when core was configured with LPM Erratum. Note that
  *			there's now way for software to detect this in runtime.
@@ -1100,8 +1101,12 @@ struct dwc3_scratchpad_array {
  *			check during HS transmit.
  * @resume_hs_terminations: Set if we enable quirk for fixing improper crc
  *			generation after resume from suspend.
+ * @ulpi_ext_vbus_drv: Set to confiure the upli chip to drives CPEN pin
+ *			VBUS with an external supply.
  * @parkmode_disable_ss_quirk: set if we need to disable all SuperSpeed
  *			instances in park mode.
+ * @parkmode_disable_hs_quirk: set if we need to disable all HishSpeed
+ *			instances in park mode.
  * @tx_de_emphasis_quirk: set if we enable Tx de-emphasis quirk
  * @tx_de_emphasis: Tx de-emphasis value
  *	0	- -6dB de-emphasis
@@ -1110,6 +1115,7 @@ struct dwc3_scratchpad_array {
  *	3	- Reserved
  * @dis_metastability_quirk: set to disable metastability quirk.
  * @dis_split_quirk: set to disable split boundary.
+ * @wakeup_configured: set if the device is configured for remote wakeup.
  * @imod_interval: set the interrupt moderation interval in 250ns
  *			increments or 0 to disable.
  * @max_cfg_eps: current max number of IN eps used across all USB configs.
@@ -1123,11 +1129,9 @@ struct dwc3 {
 	struct work_struct	drd_work;
 	struct dwc3_trb		*ep0_trb;
 	void			*bounce;
-	void			*scratchbuf;
 	u8			*setup_buf;
 	dma_addr_t		ep0_trb_addr;
 	dma_addr_t		bounce_addr;
-	dma_addr_t		scratch_addr;
 	struct dwc3_request	ep0_usb_req;
 	struct completion	ep0_in_setup;
 
@@ -1187,7 +1191,6 @@ struct dwc3 {
 	u32			current_otg_role;
 	u32			desired_otg_role;
 	bool			otg_restart_host;
-	u32			nr_scratch;
 	u32			u1u2;
 	u32			maximum_speed;
 	u32			gadget_max_speed;
@@ -1284,7 +1287,6 @@ struct dwc3 {
 	unsigned		delayed_status:1;
 	unsigned		ep0_bounced:1;
 	unsigned		ep0_expect_in:1;
-	unsigned		has_hibernation:1;
 	unsigned		sysdev_is_parent:1;
 	unsigned		has_lpm_erratum:1;
 	unsigned		is_utmi_l1_suspend:1;
@@ -1317,7 +1319,9 @@ struct dwc3 {
 	unsigned		dis_del_phy_power_chg_quirk:1;
 	unsigned		dis_tx_ipgap_linecheck_quirk:1;
 	unsigned		resume_hs_terminations:1;
+	unsigned		ulpi_ext_vbus_drv:1;
 	unsigned		parkmode_disable_ss_quirk:1;
+	unsigned		parkmode_disable_hs_quirk:1;
 	unsigned		gfladj_refclk_lpm_sel:1;
 
 	unsigned		tx_de_emphasis_quirk:1;
@@ -1327,6 +1331,7 @@ struct dwc3 {
 
 	unsigned		dis_split_quirk:1;
 	unsigned		async_callbacks:1;
+	unsigned		wakeup_configured:1;
 
 	u16			imod_interval;
 
diff --git a/drivers/usb/dwc3/debug.h b/drivers/usb/dwc3/debug.h
index 8bb2c9e3b9ac..09d703852a92 100644
--- a/drivers/usb/dwc3/debug.h
+++ b/drivers/usb/dwc3/debug.h
@@ -72,6 +72,8 @@ dwc3_gadget_generic_cmd_string(u8 cmd)
 		return "Set Endpoint Prime";
 	case DWC3_DGCMD_RUN_SOC_BUS_LOOPBACK:
 		return "Run SoC Bus Loopback Test";
+	case DWC3_DGCMD_DEV_NOTIFICATION:
+		return "Device Notification";
 	default:
 		return "UNKNOWN";
 	}
diff --git a/drivers/usb/dwc3/debugfs.c b/drivers/usb/dwc3/debugfs.c
index 850df0e6bcab..e4a2560b9dc0 100644
--- a/drivers/usb/dwc3/debugfs.c
+++ b/drivers/usb/dwc3/debugfs.c
@@ -88,6 +88,9 @@ static const struct debugfs_reg32 dwc3_regs[] = {
 	dump_register(GPRTBIMAP_HS1),
 	dump_register(GPRTBIMAP_FS0),
 	dump_register(GPRTBIMAP_FS1),
+	dump_register(GUCTL2),
+	dump_register(VER_NUMBER),
+	dump_register(VER_TYPE),
 
 	dump_register(GUSB2PHYCFG(0)),
 	dump_register(GUSB2PHYCFG(1)),
@@ -229,6 +232,8 @@ static const struct debugfs_reg32 dwc3_regs[] = {
 	dump_register(GEVNTCOUNT(0)),
 
 	dump_register(GHWPARAMS8),
+	dump_register(GUCTL3),
+	dump_register(GFLADJ),
 	dump_register(DCFG),
 	dump_register(DCTL),
 	dump_register(DEVTEN),
diff --git a/drivers/usb/dwc3/dwc3-am62.c b/drivers/usb/dwc3/dwc3-am62.c
index 173cf3579c55..cda9458c809b 100644
--- a/drivers/usb/dwc3/dwc3-am62.c
+++ b/drivers/usb/dwc3/dwc3-am62.c
@@ -11,12 +11,14 @@
 #include <linux/platform_device.h>
 #include <linux/mfd/syscon.h>
 #include <linux/of.h>
-#include <linux/of_device.h>
+#include <linux/of_platform.h>
 #include <linux/pm_runtime.h>
 #include <linux/clk.h>
 #include <linux/regmap.h>
 #include <linux/pinctrl/consumer.h>
 
+#include "core.h"
+
 /* USB WRAPPER register offsets */
 #define USBSS_PID			0x0
 #define USBSS_OVERCURRENT_CTRL		0x4
@@ -45,6 +47,10 @@
 #define USBSS_PHY_VBUS_SEL_SHIFT	1
 #define USBSS_PHY_LANE_REVERSE		BIT(0)
 
+/* CORE STAT register bits */
+#define USBSS_CORE_OPERATIONAL_MODE_MASK	GENMASK(13, 12)
+#define USBSS_CORE_OPERATIONAL_MODE_SHIFT	12
+
 /* MODE CONTROL register bits */
 #define USBSS_MODE_VALID	BIT(0)
 
@@ -54,6 +60,13 @@
 #define USBSS_WAKEUP_CFG_SESSVALID_EN	BIT(1)
 #define USBSS_WAKEUP_CFG_VBUSVALID_EN	BIT(0)
 
+#define USBSS_WAKEUP_CFG_ALL	(USBSS_WAKEUP_CFG_VBUSVALID_EN | \
+				 USBSS_WAKEUP_CFG_SESSVALID_EN | \
+				 USBSS_WAKEUP_CFG_LINESTATE_EN | \
+				 USBSS_WAKEUP_CFG_OVERCURRENT_EN)
+
+#define USBSS_WAKEUP_CFG_NONE	0
+
 /* WAKEUP STAT register bits */
 #define USBSS_WAKEUP_STAT_OVERCURRENT	BIT(4)
 #define USBSS_WAKEUP_STAT_LINESTATE	BIT(3)
@@ -97,6 +110,7 @@ struct dwc3_data {
 	struct regmap *syscon;
 	unsigned int offset;
 	unsigned int vbus_divider;
+	u32 wakeup_stat;
 };
 
 static const int dwc3_ti_rate_table[] = {	/* in KHZ */
@@ -233,6 +247,12 @@ static int dwc3_ti_probe(struct platform_device *pdev)
 	reg |= USBSS_MODE_VALID;
 	dwc3_ti_writel(data, USBSS_MODE_CONTROL, reg);
 
+	/* Device has capability to wakeup system from sleep */
+	device_set_wakeup_capable(dev, true);
+	ret = device_wakeup_enable(dev);
+	if (ret)
+		dev_err(dev, "couldn't enable device as a wakeup source: %d\n", ret);
+
 	/* Setting up autosuspend */
 	pm_runtime_set_autosuspend_delay(dev, DWC3_AM62_AUTOSUSPEND_DELAY);
 	pm_runtime_use_autosuspend(dev);
@@ -281,6 +301,27 @@ static int dwc3_ti_remove(struct platform_device *pdev)
 static int dwc3_ti_suspend_common(struct device *dev)
 {
 	struct dwc3_data *data = dev_get_drvdata(dev);
+	u32 reg, current_prtcap_dir;
+
+	if (device_may_wakeup(dev)) {
+		reg = dwc3_ti_readl(data, USBSS_CORE_STAT);
+		current_prtcap_dir = (reg & USBSS_CORE_OPERATIONAL_MODE_MASK)
+				     >> USBSS_CORE_OPERATIONAL_MODE_SHIFT;
+		/* Set wakeup config enable bits */
+		reg = dwc3_ti_readl(data, USBSS_WAKEUP_CONFIG);
+		if (current_prtcap_dir == DWC3_GCTL_PRTCAP_HOST) {
+			reg = USBSS_WAKEUP_CFG_LINESTATE_EN | USBSS_WAKEUP_CFG_OVERCURRENT_EN;
+		} else {
+			reg = USBSS_WAKEUP_CFG_VBUSVALID_EN | USBSS_WAKEUP_CFG_SESSVALID_EN;
+			/*
+			 * Enable LINESTATE wake up only if connected to bus
+			 * and in U2/L3 state else it causes spurious wake-up.
+			 */
+		}
+		dwc3_ti_writel(data, USBSS_WAKEUP_CONFIG, reg);
+		/* clear wakeup status so we know what caused the wake up */
+		dwc3_ti_writel(data, USBSS_WAKEUP_STAT, USBSS_WAKEUP_STAT_CLR);
+	}
 
 	clk_disable_unprepare(data->usb2_refclk);
 
@@ -290,9 +331,18 @@ static int dwc3_ti_suspend_common(struct device *dev)
 static int dwc3_ti_resume_common(struct device *dev)
 {
 	struct dwc3_data *data = dev_get_drvdata(dev);
+	u32 reg;
 
 	clk_prepare_enable(data->usb2_refclk);
 
+	if (device_may_wakeup(dev)) {
+		/* Clear wakeup config enable bits */
+		dwc3_ti_writel(data, USBSS_WAKEUP_CONFIG, USBSS_WAKEUP_CFG_NONE);
+	}
+
+	reg = dwc3_ti_readl(data, USBSS_WAKEUP_STAT);
+	data->wakeup_stat = reg;
+
 	return 0;
 }
 
diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c
index 560793545362..44a04c9b2073 100644
--- a/drivers/usb/dwc3/dwc3-pci.c
+++ b/drivers/usb/dwc3/dwc3-pci.c
@@ -388,107 +388,41 @@ static void dwc3_pci_remove(struct pci_dev *pci)
 }
 
 static const struct pci_device_id dwc3_pci_id_table[] = {
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_BSW),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_BYT),
-	  (kernel_ulong_t) &dwc3_pci_intel_byt_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MRFLD),
-	  (kernel_ulong_t) &dwc3_pci_intel_mrfld_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_CMLLP),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_CMLH),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_SPTLP),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_SPTH),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_BXT),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_BXT_M),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_APL),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_KBP),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_GLK),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_CNPLP),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_CNPH),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_CNPV),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ICLLP),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_EHL),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGPLP),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGPH),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_JSP),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADL),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADL_PCH),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLN),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLN_PCH),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLS),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_RPL),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_RPLS),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTLM),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTLP),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTLS),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTL),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL),
-	  (kernel_ulong_t) &dwc3_pci_intel_swnode, },
-
-	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NL_USB),
-	  (kernel_ulong_t) &dwc3_pci_amd_swnode, },
-
-	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_MR),
-	  (kernel_ulong_t)&dwc3_pci_amd_mr_swnode, },
+	{ PCI_DEVICE_DATA(INTEL, BSW, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, BYT, &dwc3_pci_intel_byt_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, MRFLD, &dwc3_pci_intel_mrfld_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, CMLLP, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, CMLH, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, SPTLP, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, SPTH, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, BXT, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, BXT_M, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, APL, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, KBP, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, GLK, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, CNPLP, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, CNPH, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, CNPV, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, ICLLP, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, EHL, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, TGPLP, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, TGPH, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, JSP, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, ADL, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, ADL_PCH, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, ADLN, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, ADLN_PCH, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, ADLS, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, RPL, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, RPLS, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, MTLM, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, MTLP, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, MTL, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, MTLS, &dwc3_pci_intel_swnode) },
+	{ PCI_DEVICE_DATA(INTEL, TGL, &dwc3_pci_intel_swnode) },
+
+	{ PCI_DEVICE_DATA(AMD, NL_USB, &dwc3_pci_amd_swnode) },
+	{ PCI_DEVICE_DATA(AMD, MR, &dwc3_pci_amd_mr_swnode) },
 
 	{  }	/* Terminating Entry */
 };
diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c
index 61de693461da..953b752a5052 100644
--- a/drivers/usb/dwc3/ep0.c
+++ b/drivers/usb/dwc3/ep0.c
@@ -30,6 +30,8 @@
 static void __dwc3_ep0_do_control_status(struct dwc3 *dwc, struct dwc3_ep *dep);
 static void __dwc3_ep0_do_control_data(struct dwc3 *dwc,
 		struct dwc3_ep *dep, struct dwc3_request *req);
+static int dwc3_ep0_delegate_req(struct dwc3 *dwc,
+				 struct usb_ctrlrequest *ctrl);
 
 static void dwc3_ep0_prepare_one_trb(struct dwc3_ep *dep,
 		dma_addr_t buf_dma, u32 len, u32 type, bool chain)
@@ -356,6 +358,9 @@ static int dwc3_ep0_handle_status(struct dwc3 *dwc,
 				usb_status |= 1 << USB_DEV_STAT_U1_ENABLED;
 			if (reg & DWC3_DCTL_INITU2ENA)
 				usb_status |= 1 << USB_DEV_STAT_U2_ENABLED;
+		} else {
+			usb_status |= dwc->gadget->wakeup_armed <<
+					USB_DEVICE_REMOTE_WAKEUP;
 		}
 
 		break;
@@ -365,7 +370,7 @@ static int dwc3_ep0_handle_status(struct dwc3 *dwc,
 		 * Function Remote Wake Capable	D0
 		 * Function Remote Wakeup	D1
 		 */
-		break;
+		return dwc3_ep0_delegate_req(dwc, ctrl);
 
 	case USB_RECIP_ENDPOINT:
 		dep = dwc3_wIndex_to_dep(dwc, ctrl->wIndex);
@@ -476,6 +481,10 @@ static int dwc3_ep0_handle_device(struct dwc3 *dwc,
 
 	switch (wValue) {
 	case USB_DEVICE_REMOTE_WAKEUP:
+		if (dwc->wakeup_configured)
+			dwc->gadget->wakeup_armed = set;
+		else
+			ret = -EINVAL;
 		break;
 	/*
 	 * 9.4.1 says only for SS, in AddressState only for
@@ -510,13 +519,7 @@ static int dwc3_ep0_handle_intf(struct dwc3 *dwc,
 
 	switch (wValue) {
 	case USB_INTRF_FUNC_SUSPEND:
-		/*
-		 * REVISIT: Ideally we would enable some low power mode here,
-		 * however it's unclear what we should be doing here.
-		 *
-		 * For now, we're not doing anything, just making sure we return
-		 * 0 so USB Command Verifier tests pass without any errors.
-		 */
+		ret = dwc3_ep0_delegate_req(dwc, ctrl);
 		break;
 	default:
 		ret = -EINVAL;
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index cf5b4f49c3ed..c0ca4d12f95d 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -139,6 +139,24 @@ int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
 	return -ETIMEDOUT;
 }
 
+static void dwc3_ep0_reset_state(struct dwc3 *dwc)
+{
+	unsigned int	dir;
+
+	if (dwc->ep0state != EP0_SETUP_PHASE) {
+		dir = !!dwc->ep0_expect_in;
+		if (dwc->ep0state == EP0_DATA_PHASE)
+			dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
+		else
+			dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
+
+		dwc->eps[0]->trb_enqueue = 0;
+		dwc->eps[1]->trb_enqueue = 0;
+
+		dwc3_ep0_stall_and_restart(dwc);
+	}
+}
+
 /**
  * dwc3_ep_inc_trb - increment a trb index.
  * @index: Pointer to the TRB index to increment.
@@ -258,7 +276,7 @@ int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
 	return ret;
 }
 
-static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
+static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async);
 
 /**
  * dwc3_send_gadget_ep_cmd - issue an endpoint command
@@ -325,7 +343,7 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
 
 			fallthrough;
 		case DWC3_LINK_STATE_U3:
-			ret = __dwc3_gadget_wakeup(dwc);
+			ret = __dwc3_gadget_wakeup(dwc, false);
 			dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 					ret);
 			break;
@@ -2273,6 +2291,22 @@ static const struct usb_ep_ops dwc3_gadget_ep_ops = {
 
 /* -------------------------------------------------------------------------- */
 
+static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set)
+{
+	u32 reg;
+
+	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
+		return;
+
+	reg = dwc3_readl(dwc->regs, DWC3_DEVTEN);
+	if (set)
+		reg |= DWC3_DEVTEN_ULSTCNGEN;
+	else
+		reg &= ~DWC3_DEVTEN_ULSTCNGEN;
+
+	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
+}
+
 static int dwc3_gadget_get_frame(struct usb_gadget *g)
 {
 	struct dwc3		*dwc = gadget_to_dwc(g);
@@ -2280,7 +2314,7 @@ static int dwc3_gadget_get_frame(struct usb_gadget *g)
 	return __dwc3_gadget_get_frame(dwc);
 }
 
-static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
+static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async)
 {
 	int			retries;
 
@@ -2311,9 +2345,13 @@ static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
 		return -EINVAL;
 	}
 
+	if (async)
+		dwc3_gadget_enable_linksts_evts(dwc, true);
+
 	ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
 	if (ret < 0) {
 		dev_err(dwc->dev, "failed to put link in Recovery\n");
+		dwc3_gadget_enable_linksts_evts(dwc, false);
 		return ret;
 	}
 
@@ -2325,6 +2363,13 @@ static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 	}
 
+	/*
+	 * Since link status change events are enabled we will receive
+	 * an U0 event when wakeup is successful. So bail out.
+	 */
+	if (async)
+		return 0;
+
 	/* poll until Link State changes to ON */
 	retries = 20000;
 
@@ -2350,13 +2395,77 @@ static int dwc3_gadget_wakeup(struct usb_gadget *g)
 	unsigned long		flags;
 	int			ret;
 
+	if (!dwc->wakeup_configured) {
+		dev_err(dwc->dev, "remote wakeup not configured\n");
+		return -EINVAL;
+	}
+
 	spin_lock_irqsave(&dwc->lock, flags);
-	ret = __dwc3_gadget_wakeup(dwc);
+	if (!dwc->gadget->wakeup_armed) {
+		dev_err(dwc->dev, "not armed for remote wakeup\n");
+		spin_unlock_irqrestore(&dwc->lock, flags);
+		return -EINVAL;
+	}
+	ret = __dwc3_gadget_wakeup(dwc, true);
+
 	spin_unlock_irqrestore(&dwc->lock, flags);
 
 	return ret;
 }
 
+static void dwc3_resume_gadget(struct dwc3 *dwc);
+
+static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id)
+{
+	struct  dwc3		*dwc = gadget_to_dwc(g);
+	unsigned long		flags;
+	int			ret;
+	int			link_state;
+
+	if (!dwc->wakeup_configured) {
+		dev_err(dwc->dev, "remote wakeup not configured\n");
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&dwc->lock, flags);
+	/*
+	 * If the link is in U3, signal for remote wakeup and wait for the
+	 * link to transition to U0 before sending device notification.
+	 */
+	link_state = dwc3_gadget_get_link_state(dwc);
+	if (link_state == DWC3_LINK_STATE_U3) {
+		ret = __dwc3_gadget_wakeup(dwc, false);
+		if (ret) {
+			spin_unlock_irqrestore(&dwc->lock, flags);
+			return -EINVAL;
+		}
+		dwc3_resume_gadget(dwc);
+		dwc->link_state = DWC3_LINK_STATE_U0;
+	}
+
+	ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION,
+					       DWC3_DGCMDPAR_DN_FUNC_WAKE |
+					       DWC3_DGCMDPAR_INTF_SEL(intf_id));
+	if (ret)
+		dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret);
+
+	spin_unlock_irqrestore(&dwc->lock, flags);
+
+	return ret;
+}
+
+static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set)
+{
+	struct dwc3		*dwc = gadget_to_dwc(g);
+	unsigned long		flags;
+
+	spin_lock_irqsave(&dwc->lock, flags);
+	dwc->wakeup_configured = !!set;
+	spin_unlock_irqrestore(&dwc->lock, flags);
+
+	return 0;
+}
+
 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
 		int is_selfpowered)
 {
@@ -2478,7 +2587,7 @@ static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
 }
 
-static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
+static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
 {
 	u32			reg;
 	u32			timeout = 2000;
@@ -2497,17 +2606,11 @@ static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
 			reg &= ~DWC3_DCTL_KEEP_CONNECT;
 		reg |= DWC3_DCTL_RUN_STOP;
 
-		if (dwc->has_hibernation)
-			reg |= DWC3_DCTL_KEEP_CONNECT;
-
 		__dwc3_gadget_set_speed(dwc);
 		dwc->pullups_connected = true;
 	} else {
 		reg &= ~DWC3_DCTL_RUN_STOP;
 
-		if (dwc->has_hibernation && !suspend)
-			reg &= ~DWC3_DCTL_KEEP_CONNECT;
-
 		dwc->pullups_connected = false;
 	}
 
@@ -2532,29 +2635,17 @@ static int __dwc3_gadget_start(struct dwc3 *dwc);
 static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
 {
 	unsigned long flags;
+	int ret;
 
 	spin_lock_irqsave(&dwc->lock, flags);
 	dwc->connected = false;
 
 	/*
-	 * Per databook, when we want to stop the gadget, if a control transfer
-	 * is still in process, complete it and get the core into setup phase.
+	 * Attempt to end pending SETUP status phase, and not wait for the
+	 * function to do so.
 	 */
-	if (dwc->ep0state != EP0_SETUP_PHASE) {
-		int ret;
-
-		if (dwc->delayed_status)
-			dwc3_ep0_send_delayed_status(dwc);
-
-		reinit_completion(&dwc->ep0_in_setup);
-
-		spin_unlock_irqrestore(&dwc->lock, flags);
-		ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
-				msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
-		spin_lock_irqsave(&dwc->lock, flags);
-		if (ret == 0)
-			dev_warn(dwc->dev, "timed out waiting for SETUP phase\n");
-	}
+	if (dwc->delayed_status)
+		dwc3_ep0_send_delayed_status(dwc);
 
 	/*
 	 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
@@ -2564,17 +2655,48 @@ static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
 	 * bit.
 	 */
 	dwc3_stop_active_transfers(dwc);
-	__dwc3_gadget_stop(dwc);
 	spin_unlock_irqrestore(&dwc->lock, flags);
 
 	/*
+	 * Per databook, when we want to stop the gadget, if a control transfer
+	 * is still in process, complete it and get the core into setup phase.
+	 * In case the host is unresponsive to a SETUP transaction, forcefully
+	 * stall the transfer, and move back to the SETUP phase, so that any
+	 * pending endxfers can be executed.
+	 */
+	if (dwc->ep0state != EP0_SETUP_PHASE) {
+		reinit_completion(&dwc->ep0_in_setup);
+
+		ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
+				msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
+		if (ret == 0) {
+			dev_warn(dwc->dev, "wait for SETUP phase timed out\n");
+			spin_lock_irqsave(&dwc->lock, flags);
+			dwc3_ep0_reset_state(dwc);
+			spin_unlock_irqrestore(&dwc->lock, flags);
+		}
+	}
+
+	/*
 	 * Note: if the GEVNTCOUNT indicates events in the event buffer, the
 	 * driver needs to acknowledge them before the controller can halt.
 	 * Simply let the interrupt handler acknowledges and handle the
 	 * remaining event generated by the controller while polling for
 	 * DSTS.DEVCTLHLT.
 	 */
-	return dwc3_gadget_run_stop(dwc, false, false);
+	ret = dwc3_gadget_run_stop(dwc, false);
+
+	/*
+	 * Stop the gadget after controller is halted, so that if needed, the
+	 * events to update EP0 state can still occur while the run/stop
+	 * routine polls for the halted state.  DEVTEN is cleared as part of
+	 * gadget stop.
+	 */
+	spin_lock_irqsave(&dwc->lock, flags);
+	__dwc3_gadget_stop(dwc);
+	spin_unlock_irqrestore(&dwc->lock, flags);
+
+	return ret;
 }
 
 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
@@ -2628,7 +2750,7 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
 
 		dwc3_event_buffers_setup(dwc);
 		__dwc3_gadget_start(dwc);
-		ret = dwc3_gadget_run_stop(dwc, true, false);
+		ret = dwc3_gadget_run_stop(dwc, true);
 	}
 
 	pm_runtime_put(dwc->dev);
@@ -2982,6 +3104,8 @@ static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
 static const struct usb_gadget_ops dwc3_gadget_ops = {
 	.get_frame		= dwc3_gadget_get_frame,
 	.wakeup			= dwc3_gadget_wakeup,
+	.func_wakeup		= dwc3_gadget_func_wakeup,
+	.set_remote_wakeup	= dwc3_gadget_set_remote_wakeup,
 	.set_selfpowered	= dwc3_gadget_set_selfpowered,
 	.pullup			= dwc3_gadget_pullup,
 	.udc_start		= dwc3_gadget_start,
@@ -3827,18 +3951,11 @@ static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
 
 	dwc->gadget->speed = USB_SPEED_UNKNOWN;
 	dwc->setup_packet_pending = false;
+	dwc->gadget->wakeup_armed = false;
+	dwc3_gadget_enable_linksts_evts(dwc, false);
 	usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
 
-	if (dwc->ep0state != EP0_SETUP_PHASE) {
-		unsigned int    dir;
-
-		dir = !!dwc->ep0_expect_in;
-		if (dwc->ep0state == EP0_DATA_PHASE)
-			dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
-		else
-			dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
-		dwc3_ep0_stall_and_restart(dwc);
-	}
+	dwc3_ep0_reset_state(dwc);
 }
 
 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
@@ -3892,20 +4009,7 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
 	 * phase. So ensure that EP0 is in setup phase by issuing a stall
 	 * and restart if EP0 is not in setup phase.
 	 */
-	if (dwc->ep0state != EP0_SETUP_PHASE) {
-		unsigned int	dir;
-
-		dir = !!dwc->ep0_expect_in;
-		if (dwc->ep0state == EP0_DATA_PHASE)
-			dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
-		else
-			dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
-
-		dwc->eps[0]->trb_enqueue = 0;
-		dwc->eps[1]->trb_enqueue = 0;
-
-		dwc3_ep0_stall_and_restart(dwc);
-	}
+	dwc3_ep0_reset_state(dwc);
 
 	/*
 	 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
@@ -3920,6 +4024,8 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
 	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
 	dwc3_gadget_dctl_write_safe(dwc, reg);
 	dwc->test_mode = false;
+	dwc->gadget->wakeup_armed = false;
+	dwc3_gadget_enable_linksts_evts(dwc, false);
 	dwc3_clear_stall_all_ep(dwc);
 
 	/* Reset device address to zero */
@@ -4072,7 +4178,7 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
 	 */
 }
 
-static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
+static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
 {
 	/*
 	 * TODO take core out of low power mode when that's
@@ -4084,6 +4190,8 @@ static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
 		dwc->gadget_driver->resume(dwc->gadget);
 		spin_lock(&dwc->lock);
 	}
+
+	dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
 }
 
 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
@@ -4165,6 +4273,12 @@ static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
 	}
 
 	switch (next) {
+	case DWC3_LINK_STATE_U0:
+		if (dwc->gadget->wakeup_armed) {
+			dwc3_gadget_enable_linksts_evts(dwc, false);
+			dwc3_resume_gadget(dwc);
+		}
+		break;
 	case DWC3_LINK_STATE_U1:
 		if (dwc->speed == USB_SPEED_SUPER)
 			dwc3_suspend_gadget(dwc);
@@ -4195,30 +4309,6 @@ static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
 	dwc->link_state = next;
 }
 
-static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
-		unsigned int evtinfo)
-{
-	unsigned int is_ss = evtinfo & BIT(4);
-
-	/*
-	 * WORKAROUND: DWC3 revision 2.20a with hibernation support
-	 * have a known issue which can cause USB CV TD.9.23 to fail
-	 * randomly.
-	 *
-	 * Because of this issue, core could generate bogus hibernation
-	 * events which SW needs to ignore.
-	 *
-	 * Refers to:
-	 *
-	 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
-	 * Device Fallback from SuperSpeed
-	 */
-	if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
-		return;
-
-	/* enter hibernation here */
-}
-
 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
 		const struct dwc3_event_devt *event)
 {
@@ -4233,29 +4323,18 @@ static void dwc3_gadget_interrupt(struct dwc3 *dwc,
 		dwc3_gadget_conndone_interrupt(dwc);
 		break;
 	case DWC3_DEVICE_EVENT_WAKEUP:
-		dwc3_gadget_wakeup_interrupt(dwc);
+		dwc3_gadget_wakeup_interrupt(dwc, event->event_info);
 		break;
 	case DWC3_DEVICE_EVENT_HIBER_REQ:
-		if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
-					"unexpected hibernation event\n"))
-			break;
-
-		dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
+		dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n");
 		break;
 	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
 		dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
 		break;
 	case DWC3_DEVICE_EVENT_SUSPEND:
 		/* It changed to be suspend event for version 2.30a and above */
-		if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) {
-			/*
-			 * Ignore suspend event until the gadget enters into
-			 * USB_STATE_CONFIGURED state.
-			 */
-			if (dwc->gadget->state >= USB_STATE_CONFIGURED)
-				dwc3_gadget_suspend_interrupt(dwc,
-						event->event_info);
-		}
+		if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
+			dwc3_gadget_suspend_interrupt(dwc, event->event_info);
 		break;
 	case DWC3_DEVICE_EVENT_SOF:
 	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
@@ -4417,11 +4496,6 @@ static int dwc3_gadget_get_irq(struct dwc3 *dwc)
 		goto out;
 
 	irq = platform_get_irq(dwc3_pdev, 0);
-	if (irq > 0)
-		goto out;
-
-	if (!irq)
-		irq = -EINVAL;
 
 out:
 	return irq;
@@ -4493,6 +4567,7 @@ int dwc3_gadget_init(struct dwc3 *dwc)
 	dwc->gadget->sg_supported	= true;
 	dwc->gadget->name		= "dwc3-gadget";
 	dwc->gadget->lpm_capable	= !dwc->usb2_gadget_lpm_disable;
+	dwc->gadget->wakeup_capable	= true;
 
 	/*
 	 * FIXME We might be setting max_speed to <SUPER, however versions
@@ -4584,7 +4659,7 @@ int dwc3_gadget_suspend(struct dwc3 *dwc)
 	if (!dwc->gadget_driver)
 		return 0;
 
-	dwc3_gadget_run_stop(dwc, false, false);
+	dwc3_gadget_run_stop(dwc, false);
 
 	spin_lock_irqsave(&dwc->lock, flags);
 	dwc3_disconnect_gadget(dwc);
@@ -4605,7 +4680,7 @@ int dwc3_gadget_resume(struct dwc3 *dwc)
 	if (ret < 0)
 		goto err0;
 
-	ret = dwc3_gadget_run_stop(dwc, true, false);
+	ret = dwc3_gadget_run_stop(dwc, true);
 	if (ret < 0)
 		goto err1;
 
diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c
index f6f13e7f1ba1..61f57fe5bb78 100644
--- a/drivers/usb/dwc3/host.c
+++ b/drivers/usb/dwc3/host.c
@@ -52,13 +52,8 @@ static int dwc3_host_get_irq(struct dwc3 *dwc)
 		goto out;
 
 	irq = platform_get_irq(dwc3_pdev, 0);
-	if (irq > 0) {
+	if (irq > 0)
 		dwc3_host_fill_xhci_irq_res(dwc, irq, NULL);
-		goto out;
-	}
-
-	if (!irq)
-		irq = -EINVAL;
 
 out:
 	return irq;
diff --git a/drivers/usb/dwc3/trace.h b/drivers/usb/dwc3/trace.h
index 1975aec8d36d..d2997d17cfbe 100644
--- a/drivers/usb/dwc3/trace.h
+++ b/drivers/usb/dwc3/trace.h
@@ -54,14 +54,13 @@ DECLARE_EVENT_CLASS(dwc3_log_event,
 	TP_STRUCT__entry(
 		__field(u32, event)
 		__field(u32, ep0state)
-		__dynamic_array(char, str, DWC3_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->event = event;
 		__entry->ep0state = dwc->ep0state;
 	),
 	TP_printk("event (%08x): %s", __entry->event,
-			dwc3_decode_event(__get_str(str), DWC3_MSG_MAX,
+			dwc3_decode_event(__get_buf(DWC3_MSG_MAX), DWC3_MSG_MAX,
 					__entry->event, __entry->ep0state))
 );
 
@@ -79,7 +78,6 @@ DECLARE_EVENT_CLASS(dwc3_log_ctrl,
 		__field(__u16, wValue)
 		__field(__u16, wIndex)
 		__field(__u16, wLength)
-		__dynamic_array(char, str, DWC3_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->bRequestType = ctrl->bRequestType;
@@ -88,7 +86,7 @@ DECLARE_EVENT_CLASS(dwc3_log_ctrl,
 		__entry->wIndex = le16_to_cpu(ctrl->wIndex);
 		__entry->wLength = le16_to_cpu(ctrl->wLength);
 	),
-	TP_printk("%s", usb_decode_ctrl(__get_str(str), DWC3_MSG_MAX,
+	TP_printk("%s", usb_decode_ctrl(__get_buf(DWC3_MSG_MAX), DWC3_MSG_MAX,
 					__entry->bRequestType,
 					__entry->bRequest, __entry->wValue,
 					__entry->wIndex, __entry->wLength)
diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c
index 5377d873c08e..1b3489149e5e 100644
--- a/drivers/usb/gadget/composite.c
+++ b/drivers/usb/gadget/composite.c
@@ -492,6 +492,46 @@ int usb_interface_id(struct usb_configuration *config,
 }
 EXPORT_SYMBOL_GPL(usb_interface_id);
 
+/**
+ * usb_func_wakeup - sends function wake notification to the host.
+ * @func: function that sends the remote wakeup notification.
+ *
+ * Applicable to devices operating at enhanced superspeed when usb
+ * functions are put in function suspend state and armed for function
+ * remote wakeup. On completion, function wake notification is sent. If
+ * the device is in low power state it tries to bring the device to active
+ * state before sending the wake notification. Since it is a synchronous
+ * call, caller must take care of not calling it in interrupt context.
+ * For devices operating at lower speeds  returns negative errno.
+ *
+ * Returns zero on success, else negative errno.
+ */
+int usb_func_wakeup(struct usb_function *func)
+{
+	struct usb_gadget	*gadget = func->config->cdev->gadget;
+	int			id;
+
+	if (!gadget->ops->func_wakeup)
+		return -EOPNOTSUPP;
+
+	if (!func->func_wakeup_armed) {
+		ERROR(func->config->cdev, "not armed for func remote wakeup\n");
+		return -EINVAL;
+	}
+
+	for (id = 0; id < MAX_CONFIG_INTERFACES; id++)
+		if (func->config->interface[id] == func)
+			break;
+
+	if (id == MAX_CONFIG_INTERFACES) {
+		ERROR(func->config->cdev, "Invalid function\n");
+		return -EINVAL;
+	}
+
+	return gadget->ops->func_wakeup(gadget, id);
+}
+EXPORT_SYMBOL_GPL(usb_func_wakeup);
+
 static u8 encode_bMaxPower(enum usb_device_speed speed,
 		struct usb_configuration *c)
 {
@@ -513,6 +553,19 @@ static u8 encode_bMaxPower(enum usb_device_speed speed,
 		return min(val, 900U) / 8;
 }
 
+void check_remote_wakeup_config(struct usb_gadget *g,
+				struct usb_configuration *c)
+{
+	if (USB_CONFIG_ATT_WAKEUP & c->bmAttributes) {
+		/* Reset the rw bit if gadget is not capable of it */
+		if (!g->wakeup_capable && g->ops->set_remote_wakeup) {
+			WARN(c->cdev, "Clearing wakeup bit for config c.%d\n",
+			     c->bConfigurationValue);
+			c->bmAttributes &= ~USB_CONFIG_ATT_WAKEUP;
+		}
+	}
+}
+
 static int config_buf(struct usb_configuration *config,
 		enum usb_device_speed speed, void *buf, u8 type)
 {
@@ -888,6 +941,9 @@ static void reset_config(struct usb_composite_dev *cdev)
 		if (f->disable)
 			f->disable(f);
 
+		/* Section 9.1.1.6, disable remote wakeup when device is reset */
+		f->func_wakeup_armed = false;
+
 		bitmap_zero(f->endpoints, 32);
 	}
 	cdev->config = NULL;
@@ -994,6 +1050,11 @@ static int set_config(struct usb_composite_dev *cdev,
 		power = min(power, 500U);
 	else
 		power = min(power, 900U);
+
+	if (USB_CONFIG_ATT_WAKEUP & c->bmAttributes)
+		usb_gadget_set_remote_wakeup(gadget, 1);
+	else
+		usb_gadget_set_remote_wakeup(gadget, 0);
 done:
 	if (power <= USB_SELF_POWER_VBUS_MAX_DRAW)
 		usb_gadget_set_selfpowered(gadget);
@@ -1948,9 +2009,20 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl)
 		f = cdev->config->interface[intf];
 		if (!f)
 			break;
-		status = f->get_status ? f->get_status(f) : 0;
-		if (status < 0)
-			break;
+
+		if (f->get_status) {
+			status = f->get_status(f);
+			if (status < 0)
+				break;
+		} else {
+			/* Set D0 and D1 bits based on func wakeup capability */
+			if (f->config->bmAttributes & USB_CONFIG_ATT_WAKEUP) {
+				status |= USB_INTRF_STAT_FUNC_RW_CAP;
+				if (f->func_wakeup_armed)
+					status |= USB_INTRF_STAT_FUNC_RW;
+			}
+		}
+
 		put_unaligned_le16(status & 0x0000ffff, req->buf);
 		break;
 	/*
@@ -1972,8 +2044,44 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl)
 			if (!f)
 				break;
 			value = 0;
-			if (f->func_suspend)
+			if (f->func_suspend) {
 				value = f->func_suspend(f, w_index >> 8);
+			/* SetFeature(FUNCTION_SUSPEND) */
+			} else if (ctrl->bRequest == USB_REQ_SET_FEATURE) {
+				if (!(f->config->bmAttributes &
+				      USB_CONFIG_ATT_WAKEUP) &&
+				     (w_index & USB_INTRF_FUNC_SUSPEND_RW))
+					break;
+
+				f->func_wakeup_armed = !!(w_index &
+							  USB_INTRF_FUNC_SUSPEND_RW);
+
+				if (w_index & USB_INTRF_FUNC_SUSPEND_LP) {
+					if (f->suspend && !f->func_suspended) {
+						f->suspend(f);
+						f->func_suspended = true;
+					}
+				/*
+				 * Handle cases where host sends function resume
+				 * through SetFeature(FUNCTION_SUSPEND) but low power
+				 * bit reset
+				 */
+				} else {
+					if (f->resume && f->func_suspended) {
+						f->resume(f);
+						f->func_suspended = false;
+					}
+				}
+			/* ClearFeature(FUNCTION_SUSPEND) */
+			} else if (ctrl->bRequest == USB_REQ_CLEAR_FEATURE) {
+				f->func_wakeup_armed = false;
+
+				if (f->resume && f->func_suspended) {
+					f->resume(f);
+					f->func_suspended = false;
+				}
+			}
+
 			if (value < 0) {
 				ERROR(cdev,
 				      "func_suspend() returned error %d\n",
@@ -2515,7 +2623,12 @@ void composite_resume(struct usb_gadget *gadget)
 		cdev->driver->resume(cdev);
 	if (cdev->config) {
 		list_for_each_entry(f, &cdev->config->functions, list) {
-			if (f->resume)
+			/*
+			 * Check for func_suspended flag to see if the function is
+			 * in USB3 FUNCTION_SUSPEND state. In this case resume is
+			 * done via FUNCTION_SUSPEND feature selector.
+			 */
+			if (f->resume && !f->func_suspended)
 				f->resume(f);
 		}
 
@@ -2530,6 +2643,10 @@ void composite_resume(struct usb_gadget *gadget)
 			usb_gadget_clear_selfpowered(gadget);
 
 		usb_gadget_vbus_draw(gadget, maxpower);
+	} else {
+		maxpower = CONFIG_USB_GADGET_VBUS_DRAW;
+		maxpower = min(maxpower, 100U);
+		usb_gadget_vbus_draw(gadget, maxpower);
 	}
 
 	cdev->suspended = 0;
diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c
index b9f1136aa0a2..4c639e9ddedc 100644
--- a/drivers/usb/gadget/configfs.c
+++ b/drivers/usb/gadget/configfs.c
@@ -1761,6 +1761,9 @@ static int configfs_composite_bind(struct usb_gadget *gadget,
 		if (gadget_is_otg(gadget))
 			c->descriptors = otg_desc;
 
+		/* Properly configure the bmAttributes wakeup bit */
+		check_remote_wakeup_config(gadget, c);
+
 		cfg = container_of(c, struct config_usb_cfg, c);
 		if (!list_empty(&cfg->string_list)) {
 			i = 0;
diff --git a/drivers/usb/gadget/function/f_ecm.c b/drivers/usb/gadget/function/f_ecm.c
index a7ab30e603e2..c6e63ad77a40 100644
--- a/drivers/usb/gadget/function/f_ecm.c
+++ b/drivers/usb/gadget/function/f_ecm.c
@@ -885,6 +885,26 @@ static struct usb_function_instance *ecm_alloc_inst(void)
 	return &opts->func_inst;
 }
 
+static void ecm_suspend(struct usb_function *f)
+{
+	struct f_ecm *ecm = func_to_ecm(f);
+	struct usb_composite_dev *cdev = ecm->port.func.config->cdev;
+
+	DBG(cdev, "ECM Suspend\n");
+
+	gether_suspend(&ecm->port);
+}
+
+static void ecm_resume(struct usb_function *f)
+{
+	struct f_ecm *ecm = func_to_ecm(f);
+	struct usb_composite_dev *cdev = ecm->port.func.config->cdev;
+
+	DBG(cdev, "ECM Resume\n");
+
+	gether_resume(&ecm->port);
+}
+
 static void ecm_free(struct usb_function *f)
 {
 	struct f_ecm *ecm;
@@ -952,6 +972,8 @@ static struct usb_function *ecm_alloc(struct usb_function_instance *fi)
 	ecm->port.func.setup = ecm_setup;
 	ecm->port.func.disable = ecm_disable;
 	ecm->port.func.free_func = ecm_free;
+	ecm->port.func.suspend = ecm_suspend;
+	ecm->port.func.resume = ecm_resume;
 
 	return &ecm->port.func;
 }
diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index 56cdfb2e4211..a13c946e0663 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -335,8 +335,6 @@ static ssize_t ffs_ep0_write(struct file *file, const char __user *buf,
 	ssize_t ret;
 	char *data;
 
-	ENTER();
-
 	/* Fast check if setup was canceled */
 	if (ffs_setup_state_clear_cancelled(ffs) == FFS_SETUP_CANCELLED)
 		return -EIDRM;
@@ -512,8 +510,6 @@ static ssize_t ffs_ep0_read(struct file *file, char __user *buf,
 	size_t n;
 	int ret;
 
-	ENTER();
-
 	/* Fast check if setup was canceled */
 	if (ffs_setup_state_clear_cancelled(ffs) == FFS_SETUP_CANCELLED)
 		return -EIDRM;
@@ -612,8 +608,6 @@ static int ffs_ep0_open(struct inode *inode, struct file *file)
 {
 	struct ffs_data *ffs = inode->i_private;
 
-	ENTER();
-
 	if (ffs->state == FFS_CLOSING)
 		return -EBUSY;
 
@@ -627,8 +621,6 @@ static int ffs_ep0_release(struct inode *inode, struct file *file)
 {
 	struct ffs_data *ffs = file->private_data;
 
-	ENTER();
-
 	ffs_data_closed(ffs);
 
 	return 0;
@@ -640,8 +632,6 @@ static long ffs_ep0_ioctl(struct file *file, unsigned code, unsigned long value)
 	struct usb_gadget *gadget = ffs->gadget;
 	long ret;
 
-	ENTER();
-
 	if (code == FUNCTIONFS_INTERFACE_REVMAP) {
 		struct ffs_function *func = ffs->func;
 		ret = func ? ffs_func_revmap_intf(func, value) : -ENODEV;
@@ -715,7 +705,6 @@ static void ffs_epfile_io_complete(struct usb_ep *_ep, struct usb_request *req)
 {
 	struct ffs_io_data *io_data = req->context;
 
-	ENTER();
 	if (req->status)
 		io_data->status = req->status;
 	else
@@ -856,8 +845,6 @@ static void ffs_epfile_async_io_complete(struct usb_ep *_ep,
 	struct ffs_io_data *io_data = req->context;
 	struct ffs_data *ffs = io_data->ffs;
 
-	ENTER();
-
 	io_data->status = req->status ? req->status : req->actual;
 	usb_ep_free_request(_ep, req);
 
@@ -1161,8 +1148,6 @@ ffs_epfile_open(struct inode *inode, struct file *file)
 {
 	struct ffs_epfile *epfile = inode->i_private;
 
-	ENTER();
-
 	if (WARN_ON(epfile->ffs->state != FFS_ACTIVE))
 		return -ENODEV;
 
@@ -1179,8 +1164,6 @@ static int ffs_aio_cancel(struct kiocb *kiocb)
 	unsigned long flags;
 	int value;
 
-	ENTER();
-
 	spin_lock_irqsave(&epfile->ffs->eps_lock, flags);
 
 	if (io_data && io_data->ep && io_data->req)
@@ -1198,8 +1181,6 @@ static ssize_t ffs_epfile_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 	struct ffs_io_data io_data, *p = &io_data;
 	ssize_t res;
 
-	ENTER();
-
 	if (!is_sync_kiocb(kiocb)) {
 		p = kzalloc(sizeof(io_data), GFP_KERNEL);
 		if (!p)
@@ -1235,8 +1216,6 @@ static ssize_t ffs_epfile_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 	struct ffs_io_data io_data, *p = &io_data;
 	ssize_t res;
 
-	ENTER();
-
 	if (!is_sync_kiocb(kiocb)) {
 		p = kzalloc(sizeof(io_data), GFP_KERNEL);
 		if (!p)
@@ -1284,8 +1263,6 @@ ffs_epfile_release(struct inode *inode, struct file *file)
 {
 	struct ffs_epfile *epfile = inode->i_private;
 
-	ENTER();
-
 	__ffs_epfile_read_buffer_free(epfile);
 	ffs_data_closed(epfile->ffs);
 
@@ -1299,8 +1276,6 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code,
 	struct ffs_ep *ep;
 	int ret;
 
-	ENTER();
-
 	if (WARN_ON(epfile->ffs->state != FFS_ACTIVE))
 		return -ENODEV;
 
@@ -1399,8 +1374,6 @@ ffs_sb_make_inode(struct super_block *sb, void *data,
 {
 	struct inode *inode;
 
-	ENTER();
-
 	inode = new_inode(sb);
 
 	if (inode) {
@@ -1432,8 +1405,6 @@ static struct dentry *ffs_sb_create_file(struct super_block *sb,
 	struct dentry	*dentry;
 	struct inode	*inode;
 
-	ENTER();
-
 	dentry = d_alloc_name(sb->s_root, name);
 	if (!dentry)
 		return NULL;
@@ -1468,8 +1439,6 @@ static int ffs_sb_fill(struct super_block *sb, struct fs_context *fc)
 	struct inode	*inode;
 	struct ffs_data	*ffs = data->ffs_data;
 
-	ENTER();
-
 	ffs->sb              = sb;
 	data->ffs_data       = NULL;
 	sb->s_fs_info        = ffs;
@@ -1521,8 +1490,6 @@ static int ffs_fs_parse_param(struct fs_context *fc, struct fs_parameter *param)
 	struct fs_parse_result result;
 	int opt;
 
-	ENTER();
-
 	opt = fs_parse(fc, ffs_fs_fs_parameters, param, &result);
 	if (opt < 0)
 		return opt;
@@ -1572,8 +1539,6 @@ static int ffs_fs_get_tree(struct fs_context *fc)
 	struct ffs_data	*ffs;
 	int ret;
 
-	ENTER();
-
 	if (!fc->source)
 		return invalf(fc, "No source specified");
 
@@ -1640,8 +1605,6 @@ static int ffs_fs_init_fs_context(struct fs_context *fc)
 static void
 ffs_fs_kill_sb(struct super_block *sb)
 {
-	ENTER();
-
 	kill_litter_super(sb);
 	if (sb->s_fs_info)
 		ffs_data_closed(sb->s_fs_info);
@@ -1663,8 +1626,6 @@ static int functionfs_init(void)
 {
 	int ret;
 
-	ENTER();
-
 	ret = register_filesystem(&ffs_fs_type);
 	if (!ret)
 		pr_info("file system registered\n");
@@ -1676,8 +1637,6 @@ static int functionfs_init(void)
 
 static void functionfs_cleanup(void)
 {
-	ENTER();
-
 	pr_info("unloading\n");
 	unregister_filesystem(&ffs_fs_type);
 }
@@ -1690,15 +1649,11 @@ static void ffs_data_reset(struct ffs_data *ffs);
 
 static void ffs_data_get(struct ffs_data *ffs)
 {
-	ENTER();
-
 	refcount_inc(&ffs->ref);
 }
 
 static void ffs_data_opened(struct ffs_data *ffs)
 {
-	ENTER();
-
 	refcount_inc(&ffs->ref);
 	if (atomic_add_return(1, &ffs->opened) == 1 &&
 			ffs->state == FFS_DEACTIVATED) {
@@ -1709,8 +1664,6 @@ static void ffs_data_opened(struct ffs_data *ffs)
 
 static void ffs_data_put(struct ffs_data *ffs)
 {
-	ENTER();
-
 	if (refcount_dec_and_test(&ffs->ref)) {
 		pr_info("%s(): freeing\n", __func__);
 		ffs_data_clear(ffs);
@@ -1729,8 +1682,6 @@ static void ffs_data_closed(struct ffs_data *ffs)
 	struct ffs_epfile *epfiles;
 	unsigned long flags;
 
-	ENTER();
-
 	if (atomic_dec_and_test(&ffs->opened)) {
 		if (ffs->no_disconnect) {
 			ffs->state = FFS_DEACTIVATED;
@@ -1765,8 +1716,6 @@ static struct ffs_data *ffs_data_new(const char *dev_name)
 	if (!ffs)
 		return NULL;
 
-	ENTER();
-
 	ffs->io_completion_wq = alloc_ordered_workqueue("%s", 0, dev_name);
 	if (!ffs->io_completion_wq) {
 		kfree(ffs);
@@ -1793,8 +1742,6 @@ static void ffs_data_clear(struct ffs_data *ffs)
 	struct ffs_epfile *epfiles;
 	unsigned long flags;
 
-	ENTER();
-
 	ffs_closed(ffs);
 
 	BUG_ON(ffs->gadget);
@@ -1826,8 +1773,6 @@ static void ffs_data_clear(struct ffs_data *ffs)
 
 static void ffs_data_reset(struct ffs_data *ffs)
 {
-	ENTER();
-
 	ffs_data_clear(ffs);
 
 	ffs->raw_descs_data = NULL;
@@ -1861,8 +1806,6 @@ static int functionfs_bind(struct ffs_data *ffs, struct usb_composite_dev *cdev)
 	struct usb_gadget_strings **lang;
 	int first_id;
 
-	ENTER();
-
 	if (WARN_ON(ffs->state != FFS_ACTIVE
 		 || test_and_set_bit(FFS_FL_BOUND, &ffs->flags)))
 		return -EBADFD;
@@ -1894,8 +1837,6 @@ static int functionfs_bind(struct ffs_data *ffs, struct usb_composite_dev *cdev)
 
 static void functionfs_unbind(struct ffs_data *ffs)
 {
-	ENTER();
-
 	if (!WARN_ON(!ffs->gadget)) {
 		/* dequeue before freeing ep0req */
 		usb_ep_dequeue(ffs->gadget->ep0, ffs->ep0req);
@@ -1914,8 +1855,6 @@ static int ffs_epfiles_create(struct ffs_data *ffs)
 	struct ffs_epfile *epfile, *epfiles;
 	unsigned i, count;
 
-	ENTER();
-
 	count = ffs->eps_count;
 	epfiles = kcalloc(count, sizeof(*epfiles), GFP_KERNEL);
 	if (!epfiles)
@@ -1946,8 +1885,6 @@ static void ffs_epfiles_destroy(struct ffs_epfile *epfiles, unsigned count)
 {
 	struct ffs_epfile *epfile = epfiles;
 
-	ENTER();
-
 	for (; count; --count, ++epfile) {
 		BUG_ON(mutex_is_locked(&epfile->mutex));
 		if (epfile->dentry) {
@@ -2064,8 +2001,6 @@ static int __must_check ffs_do_single_desc(char *data, unsigned len,
 	u8 length;
 	int ret;
 
-	ENTER();
-
 	/* At least two bytes are required: length and type */
 	if (len < 2) {
 		pr_vdebug("descriptor too short\n");
@@ -2204,8 +2139,6 @@ static int __must_check ffs_do_descs(unsigned count, char *data, unsigned len,
 	unsigned long num = 0;
 	int current_class = -1;
 
-	ENTER();
-
 	for (;;) {
 		int ret;
 
@@ -2243,8 +2176,6 @@ static int __ffs_data_do_entity(enum ffs_entity_type type,
 	struct ffs_desc_helper *helper = priv;
 	struct usb_endpoint_descriptor *d;
 
-	ENTER();
-
 	switch (type) {
 	case FFS_DESCRIPTOR:
 		break;
@@ -2292,8 +2223,11 @@ static int __ffs_do_os_desc_header(enum ffs_os_desc_type *next_type,
 	u16 bcd_version = le16_to_cpu(desc->bcdVersion);
 	u16 w_index = le16_to_cpu(desc->wIndex);
 
-	if (bcd_version != 1) {
-		pr_vdebug("unsupported os descriptors version: %d",
+	if (bcd_version == 0x1) {
+		pr_warn("bcdVersion must be 0x0100, stored in Little Endian order. "
+			"Userspace driver should be fixed, accepting 0x0001 for compatibility.\n");
+	} else if (bcd_version != 0x100) {
+		pr_vdebug("unsupported os descriptors version: 0x%x\n",
 			  bcd_version);
 		return -EINVAL;
 	}
@@ -2326,8 +2260,6 @@ static int __must_check ffs_do_single_os_desc(char *data, unsigned len,
 	int ret;
 	const unsigned _len = len;
 
-	ENTER();
-
 	/* loop over all ext compat/ext prop descriptors */
 	while (feature_count--) {
 		ret = entity(type, h, data, len, priv);
@@ -2349,8 +2281,6 @@ static int __must_check ffs_do_os_descs(unsigned count,
 	const unsigned _len = len;
 	unsigned long num = 0;
 
-	ENTER();
-
 	for (num = 0; num < count; ++num) {
 		int ret;
 		enum ffs_os_desc_type type;
@@ -2413,8 +2343,6 @@ static int __ffs_data_do_os_desc(enum ffs_os_desc_type type,
 	struct ffs_data *ffs = priv;
 	u8 length;
 
-	ENTER();
-
 	switch (type) {
 	case FFS_OS_DESC_EXT_COMPAT: {
 		struct usb_ext_compat_desc *d = data;
@@ -2490,8 +2418,6 @@ static int __ffs_data_got_descs(struct ffs_data *ffs,
 	int ret = -EINVAL, i;
 	struct ffs_desc_helper helper;
 
-	ENTER();
-
 	if (get_unaligned_le32(data + 4) != len)
 		goto error;
 
@@ -2622,8 +2548,6 @@ static int __ffs_data_got_strings(struct ffs_data *ffs,
 	const char *data = _data;
 	struct usb_string *s;
 
-	ENTER();
-
 	if (len < 16 ||
 	    get_unaligned_le32(data) != FUNCTIONFS_STRINGS_MAGIC ||
 	    get_unaligned_le32(data + 4) != len)
@@ -3082,8 +3006,6 @@ static inline struct f_fs_opts *ffs_do_functionfs_bind(struct usb_function *f,
 	struct ffs_data *ffs_data;
 	int ret;
 
-	ENTER();
-
 	/*
 	 * Legacy gadget triggers binding in functionfs_ready_callback,
 	 * which already uses locking; taking the same lock here would
@@ -3160,8 +3082,6 @@ static int _ffs_func_bind(struct usb_configuration *c,
 	vla_item_with_sz(d, char, raw_descs, ffs->raw_descs_length);
 	char *vlabuf;
 
-	ENTER();
-
 	/* Has descriptors only for speeds gadget does not support */
 	if (!(full | high | super))
 		return -ENOTSUPP;
@@ -3365,8 +3285,6 @@ static int ffs_func_setup(struct usb_function *f,
 	unsigned long flags;
 	int ret;
 
-	ENTER();
-
 	pr_vdebug("creq->bRequestType = %02x\n", creq->bRequestType);
 	pr_vdebug("creq->bRequest     = %02x\n", creq->bRequest);
 	pr_vdebug("creq->wValue       = %04x\n", le16_to_cpu(creq->wValue));
@@ -3441,13 +3359,11 @@ static bool ffs_func_req_match(struct usb_function *f,
 
 static void ffs_func_suspend(struct usb_function *f)
 {
-	ENTER();
 	ffs_event_add(ffs_func_from_usb(f)->ffs, FUNCTIONFS_SUSPEND);
 }
 
 static void ffs_func_resume(struct usb_function *f)
 {
-	ENTER();
 	ffs_event_add(ffs_func_from_usb(f)->ffs, FUNCTIONFS_RESUME);
 }
 
@@ -3611,7 +3527,6 @@ static void ffs_func_unbind(struct usb_configuration *c,
 	unsigned count = ffs->eps_count;
 	unsigned long flags;
 
-	ENTER();
 	if (ffs->func == func) {
 		ffs_func_eps_disable(func);
 		ffs->func = NULL;
@@ -3651,8 +3566,6 @@ static struct usb_function *ffs_alloc(struct usb_function_instance *fi)
 {
 	struct ffs_function *func;
 
-	ENTER();
-
 	func = kzalloc(sizeof(*func), GFP_KERNEL);
 	if (!func)
 		return ERR_PTR(-ENOMEM);
@@ -3753,7 +3666,6 @@ static int ffs_acquire_dev(const char *dev_name, struct ffs_data *ffs_data)
 	int ret = 0;
 	struct ffs_dev *ffs_dev;
 
-	ENTER();
 	ffs_dev_lock();
 
 	ffs_dev = _ffs_find_dev(dev_name);
@@ -3776,7 +3688,6 @@ static int ffs_acquire_dev(const char *dev_name, struct ffs_data *ffs_data)
 
 static void ffs_release_dev(struct ffs_dev *ffs_dev)
 {
-	ENTER();
 	ffs_dev_lock();
 
 	if (ffs_dev && ffs_dev->mounted) {
@@ -3798,7 +3709,6 @@ static int ffs_ready(struct ffs_data *ffs)
 	struct ffs_dev *ffs_obj;
 	int ret = 0;
 
-	ENTER();
 	ffs_dev_lock();
 
 	ffs_obj = ffs->private_data;
@@ -3831,7 +3741,6 @@ static void ffs_closed(struct ffs_data *ffs)
 	struct f_fs_opts *opts;
 	struct config_item *ci;
 
-	ENTER();
 	ffs_dev_lock();
 
 	ffs_obj = ffs->private_data;
diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c
index a8da3b4a2855..9f6b10134121 100644
--- a/drivers/usb/gadget/function/f_hid.c
+++ b/drivers/usb/gadget/function/f_hid.c
@@ -1325,7 +1325,7 @@ int ghid_setup(struct usb_gadget *g, int count)
 	int status;
 	dev_t dev;
 
-	hidg_class = class_create(THIS_MODULE, "hidg");
+	hidg_class = class_create("hidg");
 	if (IS_ERR(hidg_class)) {
 		status = PTR_ERR(hidg_class);
 		hidg_class = NULL;
diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c
index 4903d761a872..28db3e336e7d 100644
--- a/drivers/usb/gadget/function/f_printer.c
+++ b/drivers/usb/gadget/function/f_printer.c
@@ -1512,7 +1512,7 @@ static int gprinter_setup(int count)
 	int status;
 	dev_t devt;
 
-	usb_gadget_class = class_create(THIS_MODULE, "usb_printer_gadget");
+	usb_gadget_class = class_create("usb_printer_gadget");
 	if (IS_ERR(usb_gadget_class)) {
 		status = PTR_ERR(usb_gadget_class);
 		usb_gadget_class = NULL;
diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c
index 658e2e21fdd0..79ed2e6e576a 100644
--- a/drivers/usb/gadget/function/f_tcm.c
+++ b/drivers/usb/gadget/function/f_tcm.c
@@ -1054,7 +1054,7 @@ static void usbg_cmd_work(struct work_struct *work)
 				  tv_nexus->tvn_se_sess->se_tpg->se_tpg_tfo,
 				  tv_nexus->tvn_se_sess, cmd->data_len, DMA_NONE,
 				  cmd->prio_attr, cmd->sense_iu.sense,
-				  cmd->unpacked_lun);
+				  cmd->unpacked_lun, NULL);
 		goto out;
 	}
 
@@ -1183,7 +1183,7 @@ static void bot_cmd_work(struct work_struct *work)
 				  tv_nexus->tvn_se_sess->se_tpg->se_tpg_tfo,
 				  tv_nexus->tvn_se_sess, cmd->data_len, DMA_NONE,
 				  cmd->prio_attr, cmd->sense_iu.sense,
-				  cmd->unpacked_lun);
+				  cmd->unpacked_lun, NULL);
 		goto out;
 	}
 
@@ -1253,11 +1253,6 @@ static int usbg_check_true(struct se_portal_group *se_tpg)
 	return 1;
 }
 
-static int usbg_check_false(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
 static char *usbg_get_fabric_wwn(struct se_portal_group *se_tpg)
 {
 	struct usbg_tpg *tpg = container_of(se_tpg,
@@ -1274,11 +1269,6 @@ static u16 usbg_get_tag(struct se_portal_group *se_tpg)
 	return tpg->tport_tpgt;
 }
 
-static u32 usbg_tpg_get_inst_index(struct se_portal_group *se_tpg)
-{
-	return 1;
-}
-
 static void usbg_release_cmd(struct se_cmd *se_cmd)
 {
 	struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd,
@@ -1289,20 +1279,6 @@ static void usbg_release_cmd(struct se_cmd *se_cmd)
 	target_free_tag(se_sess, se_cmd);
 }
 
-static u32 usbg_sess_get_index(struct se_session *se_sess)
-{
-	return 0;
-}
-
-static void usbg_set_default_node_attrs(struct se_node_acl *nacl)
-{
-}
-
-static int usbg_get_cmd_state(struct se_cmd *se_cmd)
-{
-	return 0;
-}
-
 static void usbg_queue_tm_rsp(struct se_cmd *se_cmd)
 {
 }
@@ -1691,16 +1667,9 @@ static const struct target_core_fabric_ops usbg_ops = {
 	.tpg_get_wwn			= usbg_get_fabric_wwn,
 	.tpg_get_tag			= usbg_get_tag,
 	.tpg_check_demo_mode		= usbg_check_true,
-	.tpg_check_demo_mode_cache	= usbg_check_false,
-	.tpg_check_demo_mode_write_protect = usbg_check_false,
-	.tpg_check_prod_mode_write_protect = usbg_check_false,
-	.tpg_get_inst_index		= usbg_tpg_get_inst_index,
 	.release_cmd			= usbg_release_cmd,
-	.sess_get_index			= usbg_sess_get_index,
 	.sess_get_initiator_sid		= NULL,
 	.write_pending			= usbg_send_write_request,
-	.set_default_node_attributes	= usbg_set_default_node_attrs,
-	.get_cmd_state			= usbg_get_cmd_state,
 	.queue_data_in			= usbg_send_read_response,
 	.queue_status			= usbg_send_status_response,
 	.queue_tm_rsp			= usbg_queue_tm_rsp,
diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c
index f259975dfba4..6956ad8ba8dd 100644
--- a/drivers/usb/gadget/function/u_ether.c
+++ b/drivers/usb/gadget/function/u_ether.c
@@ -437,6 +437,20 @@ static inline int is_promisc(u16 cdc_filter)
 	return cdc_filter & USB_CDC_PACKET_TYPE_PROMISCUOUS;
 }
 
+static int ether_wakeup_host(struct gether *port)
+{
+	int			ret;
+	struct usb_function	*func = &port->func;
+	struct usb_gadget	*gadget = func->config->cdev->gadget;
+
+	if (func->func_suspended)
+		ret = usb_func_wakeup(func);
+	else
+		ret = usb_gadget_wakeup(gadget);
+
+	return ret;
+}
+
 static netdev_tx_t eth_start_xmit(struct sk_buff *skb,
 					struct net_device *net)
 {
@@ -456,6 +470,15 @@ static netdev_tx_t eth_start_xmit(struct sk_buff *skb,
 		in = NULL;
 		cdc_filter = 0;
 	}
+
+	if (dev->port_usb && dev->port_usb->is_suspend) {
+		DBG(dev, "Port suspended. Triggering wakeup\n");
+		netif_stop_queue(net);
+		spin_unlock_irqrestore(&dev->lock, flags);
+		ether_wakeup_host(dev->port_usb);
+		return NETDEV_TX_BUSY;
+	}
+
 	spin_unlock_irqrestore(&dev->lock, flags);
 
 	if (!in) {
@@ -1014,6 +1037,45 @@ int gether_set_ifname(struct net_device *net, const char *name, int len)
 }
 EXPORT_SYMBOL_GPL(gether_set_ifname);
 
+void gether_suspend(struct gether *link)
+{
+	struct eth_dev *dev = link->ioport;
+	unsigned long flags;
+
+	if (!dev)
+		return;
+
+	if (atomic_read(&dev->tx_qlen)) {
+		/*
+		 * There is a transfer in progress. So we trigger a remote
+		 * wakeup to inform the host.
+		 */
+		ether_wakeup_host(dev->port_usb);
+		return;
+	}
+	spin_lock_irqsave(&dev->lock, flags);
+	link->is_suspend = true;
+	spin_unlock_irqrestore(&dev->lock, flags);
+}
+EXPORT_SYMBOL_GPL(gether_suspend);
+
+void gether_resume(struct gether *link)
+{
+	struct eth_dev *dev = link->ioport;
+	unsigned long flags;
+
+	if (!dev)
+		return;
+
+	if (netif_queue_stopped(dev->net))
+		netif_start_queue(dev->net);
+
+	spin_lock_irqsave(&dev->lock, flags);
+	link->is_suspend = false;
+	spin_unlock_irqrestore(&dev->lock, flags);
+}
+EXPORT_SYMBOL_GPL(gether_resume);
+
 /*
  * gether_cleanup - remove Ethernet-over-USB device
  * Context: may sleep
@@ -1176,6 +1238,7 @@ void gether_disconnect(struct gether *link)
 
 	spin_lock(&dev->lock);
 	dev->port_usb = NULL;
+	link->is_suspend = false;
 	spin_unlock(&dev->lock);
 }
 EXPORT_SYMBOL_GPL(gether_disconnect);
diff --git a/drivers/usb/gadget/function/u_ether.h b/drivers/usb/gadget/function/u_ether.h
index 40144546d1b0..851ee10d6e63 100644
--- a/drivers/usb/gadget/function/u_ether.h
+++ b/drivers/usb/gadget/function/u_ether.h
@@ -79,6 +79,7 @@ struct gether {
 	/* called on network open/close */
 	void				(*open)(struct gether *);
 	void				(*close)(struct gether *);
+	bool				is_suspend;
 };
 
 #define	DEFAULT_FILTER	(USB_CDC_PACKET_TYPE_BROADCAST \
@@ -258,6 +259,9 @@ int gether_set_ifname(struct net_device *net, const char *name, int len);
 
 void gether_cleanup(struct eth_dev *dev);
 
+void gether_suspend(struct gether *link);
+void gether_resume(struct gether *link);
+
 /* connect/disconnect is handled by individual functions */
 struct net_device *gether_connect(struct gether *);
 void gether_disconnect(struct gether *);
diff --git a/drivers/usb/gadget/function/u_fs.h b/drivers/usb/gadget/function/u_fs.h
index f102ec23f3af..4b3365f23fd7 100644
--- a/drivers/usb/gadget/function/u_fs.h
+++ b/drivers/usb/gadget/function/u_fs.h
@@ -32,8 +32,6 @@
 #  define ffs_dump_mem(prefix, ptr, len) do { } while (0)
 #endif /* VERBOSE_DEBUG */
 
-#define ENTER()    pr_vdebug("%s()\n", __func__)
-
 struct f_fs_opts;
 
 struct ffs_dev {
diff --git a/drivers/usb/gadget/function/uvc_configfs.c b/drivers/usb/gadget/function/uvc_configfs.c
index 62b759bb7613..9bf0e985acfa 100644
--- a/drivers/usb/gadget/function/uvc_configfs.c
+++ b/drivers/usb/gadget/function/uvc_configfs.c
@@ -334,6 +334,64 @@ UVCG_DEFAULT_PROCESSING_ATTR(i_processing, iProcessing, 8);
 
 #undef UVCG_DEFAULT_PROCESSING_ATTR
 
+static ssize_t uvcg_default_processing_bm_controls_store(
+	struct config_item *item, const char *page, size_t len)
+{
+	struct config_group *group = to_config_group(item);
+	struct mutex *su_mutex = &group->cg_subsys->su_mutex;
+	struct uvc_processing_unit_descriptor *pd;
+	struct config_item *opts_item;
+	struct f_uvc_opts *opts;
+	u8 *bm_controls, *tmp;
+	unsigned int i;
+	int ret, n = 0;
+
+	mutex_lock(su_mutex);
+
+	opts_item = group->cg_item.ci_parent->ci_parent->ci_parent;
+	opts = to_f_uvc_opts(opts_item);
+	pd = &opts->uvc_processing;
+
+	mutex_lock(&opts->lock);
+	if (opts->refcnt) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	ret = __uvcg_iter_item_entries(page, len, __uvcg_count_item_entries, &n,
+				       sizeof(u8));
+	if (ret)
+		goto unlock;
+
+	if (n > pd->bControlSize) {
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	tmp = bm_controls = kcalloc(n, sizeof(u8), GFP_KERNEL);
+	if (!bm_controls) {
+		ret = -ENOMEM;
+		goto unlock;
+	}
+
+	ret = __uvcg_iter_item_entries(page, len, __uvcg_fill_item_entries, &tmp,
+				       sizeof(u8));
+	if (ret)
+		goto free_mem;
+
+	for (i = 0; i < n; i++)
+		pd->bmControls[i] = bm_controls[i];
+
+	ret = len;
+
+free_mem:
+	kfree(bm_controls);
+unlock:
+	mutex_unlock(&opts->lock);
+	mutex_unlock(su_mutex);
+	return ret;
+}
+
 static ssize_t uvcg_default_processing_bm_controls_show(
 	struct config_item *item, char *page)
 {
@@ -363,7 +421,7 @@ static ssize_t uvcg_default_processing_bm_controls_show(
 	return result;
 }
 
-UVC_ATTR_RO(uvcg_default_processing_, bm_controls, bmControls);
+UVC_ATTR(uvcg_default_processing_, bm_controls, bmControls);
 
 static struct configfs_attribute *uvcg_default_processing_attrs[] = {
 	&uvcg_default_processing_attr_b_unit_id,
@@ -445,6 +503,65 @@ UVCG_DEFAULT_CAMERA_ATTR(w_ocular_focal_length, wOcularFocalLength,
 
 #undef UVCG_DEFAULT_CAMERA_ATTR
 
+static ssize_t uvcg_default_camera_bm_controls_store(
+	struct config_item *item, const char *page, size_t len)
+{
+	struct config_group *group = to_config_group(item);
+	struct mutex *su_mutex = &group->cg_subsys->su_mutex;
+	struct uvc_camera_terminal_descriptor *cd;
+	struct config_item *opts_item;
+	struct f_uvc_opts *opts;
+	u8 *bm_controls, *tmp;
+	unsigned int i;
+	int ret, n = 0;
+
+	mutex_lock(su_mutex);
+
+	opts_item = group->cg_item.ci_parent->ci_parent->ci_parent->
+			ci_parent;
+	opts = to_f_uvc_opts(opts_item);
+	cd = &opts->uvc_camera_terminal;
+
+	mutex_lock(&opts->lock);
+	if (opts->refcnt) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	ret = __uvcg_iter_item_entries(page, len, __uvcg_count_item_entries, &n,
+				       sizeof(u8));
+	if (ret)
+		goto unlock;
+
+	if (n > cd->bControlSize) {
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	tmp = bm_controls = kcalloc(n, sizeof(u8), GFP_KERNEL);
+	if (!bm_controls) {
+		ret = -ENOMEM;
+		goto unlock;
+	}
+
+	ret = __uvcg_iter_item_entries(page, len, __uvcg_fill_item_entries, &tmp,
+				       sizeof(u8));
+	if (ret)
+		goto free_mem;
+
+	for (i = 0; i < n; i++)
+		cd->bmControls[i] = bm_controls[i];
+
+	ret = len;
+
+free_mem:
+	kfree(bm_controls);
+unlock:
+	mutex_unlock(&opts->lock);
+	mutex_unlock(su_mutex);
+	return ret;
+}
+
 static ssize_t uvcg_default_camera_bm_controls_show(
 	struct config_item *item, char *page)
 {
@@ -474,7 +591,7 @@ static ssize_t uvcg_default_camera_bm_controls_show(
 	return result;
 }
 
-UVC_ATTR_RO(uvcg_default_camera_, bm_controls, bmControls);
+UVC_ATTR(uvcg_default_camera_, bm_controls, bmControls);
 
 static struct configfs_attribute *uvcg_default_camera_attrs[] = {
 	&uvcg_default_camera_attr_b_terminal_id,
diff --git a/drivers/usb/gadget/legacy/g_ffs.c b/drivers/usb/gadget/legacy/g_ffs.c
index ae6d8f7092b8..097854683e5b 100644
--- a/drivers/usb/gadget/legacy/g_ffs.c
+++ b/drivers/usb/gadget/legacy/g_ffs.c
@@ -180,8 +180,6 @@ static int __init gfs_init(void)
 	int i;
 	int ret = 0;
 
-	ENTER();
-
 	if (func_num < 2) {
 		gfs_single_func = true;
 		func_num = 1;
@@ -242,8 +240,6 @@ static void __exit gfs_exit(void)
 {
 	int i;
 
-	ENTER();
-
 	if (gfs_registered)
 		usb_composite_unregister(&gfs_driver);
 	gfs_registered = false;
@@ -316,8 +312,6 @@ static int gfs_bind(struct usb_composite_dev *cdev)
 #endif
 	int ret, i;
 
-	ENTER();
-
 	if (missing_funcs)
 		return -ENODEV;
 #if defined CONFIG_USB_FUNCTIONFS_ETH
@@ -445,9 +439,6 @@ static int gfs_unbind(struct usb_composite_dev *cdev)
 {
 	int i;
 
-	ENTER();
-
-
 #ifdef CONFIG_USB_FUNCTIONFS_RNDIS
 	usb_put_function(f_rndis);
 	usb_put_function_instance(fi_rndis);
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/core.c b/drivers/usb/gadget/udc/aspeed-vhub/core.c
index ac3ca24f8b04..86398a04a012 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/core.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/core.c
@@ -21,7 +21,6 @@
 #include <linux/clk.h>
 #include <linux/usb/gadget.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/regmap.h>
 #include <linux/dma-mapping.h>
 
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/dev.c b/drivers/usb/gadget/udc/aspeed-vhub/dev.c
index 4f3bc27c1c62..573109ca5b79 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/dev.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/dev.c
@@ -21,7 +21,6 @@
 #include <linux/clk.h>
 #include <linux/usb/gadget.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/regmap.h>
 #include <linux/dma-mapping.h>
 #include <linux/usb.h>
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/ep0.c b/drivers/usb/gadget/udc/aspeed-vhub/ep0.c
index b4cf46249fea..e9aa74231760 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/ep0.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/ep0.c
@@ -21,7 +21,6 @@
 #include <linux/clk.h>
 #include <linux/usb/gadget.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/regmap.h>
 #include <linux/dma-mapping.h>
 
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/epn.c b/drivers/usb/gadget/udc/aspeed-vhub/epn.c
index 56e55472daa1..148d7ec3ebf4 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/epn.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/epn.c
@@ -21,7 +21,6 @@
 #include <linux/clk.h>
 #include <linux/usb/gadget.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/regmap.h>
 #include <linux/dma-mapping.h>
 
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/hub.c b/drivers/usb/gadget/udc/aspeed-vhub/hub.c
index e2207d014620..a63e4af60a56 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/hub.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/hub.c
@@ -21,7 +21,6 @@
 #include <linux/clk.h>
 #include <linux/usb/gadget.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/regmap.h>
 #include <linux/dma-mapping.h>
 #include <linux/bcd.h>
diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c
index 23b0629a8774..4641153e9706 100644
--- a/drivers/usb/gadget/udc/core.c
+++ b/drivers/usb/gadget/udc/core.c
@@ -26,7 +26,7 @@
 
 static DEFINE_IDA(gadget_id_numbers);
 
-static struct bus_type gadget_bus_type;
+static const struct bus_type gadget_bus_type;
 
 /**
  * struct usb_udc - describes one usb device controller
@@ -37,6 +37,10 @@ static struct bus_type gadget_bus_type;
  * @vbus: for udcs who care about vbus status, this value is real vbus status;
  * for udcs who do not care about vbus status, this value is always true
  * @started: the UDC's started state. True if the UDC had started.
+ * @connect_lock: protects udc->vbus, udc->started, gadget->connect, gadget->deactivate related
+ * functions. usb_gadget_connect_locked, usb_gadget_disconnect_locked,
+ * usb_udc_connect_control_locked, usb_gadget_udc_start_locked, usb_gadget_udc_stop_locked are
+ * called with this lock held.
  *
  * This represents the internal data structure which is used by the UDC-class
  * to hold information about udc driver and gadget together.
@@ -48,6 +52,7 @@ struct usb_udc {
 	struct list_head		list;
 	bool				vbus;
 	bool				started;
+	struct mutex			connect_lock;
 };
 
 static struct class *udc_class;
@@ -514,6 +519,33 @@ out:
 EXPORT_SYMBOL_GPL(usb_gadget_wakeup);
 
 /**
+ * usb_gadget_set_remote_wakeup - configures the device remote wakeup feature.
+ * @gadget:the device being configured for remote wakeup
+ * @set:value to be configured.
+ *
+ * set to one to enable remote wakeup feature and zero to disable it.
+ *
+ * returns zero on success, else negative errno.
+ */
+int usb_gadget_set_remote_wakeup(struct usb_gadget *gadget, int set)
+{
+	int ret = 0;
+
+	if (!gadget->ops->set_remote_wakeup) {
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
+
+	ret = gadget->ops->set_remote_wakeup(gadget, set);
+
+out:
+	trace_usb_gadget_set_remote_wakeup(gadget, ret);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(usb_gadget_set_remote_wakeup);
+
+/**
  * usb_gadget_set_selfpowered - sets the device selfpowered feature.
  * @gadget:the device being declared as self-powered
  *
@@ -660,17 +692,9 @@ out:
 }
 EXPORT_SYMBOL_GPL(usb_gadget_vbus_disconnect);
 
-/**
- * usb_gadget_connect - software-controlled connect to USB host
- * @gadget:the peripheral being connected
- *
- * Enables the D+ (or potentially D-) pullup.  The host will start
- * enumerating this gadget when the pullup is active and a VBUS session
- * is active (the link is powered).
- *
- * Returns zero on success, else negative errno.
- */
-int usb_gadget_connect(struct usb_gadget *gadget)
+/* Internal version of usb_gadget_connect needs to be called with connect_lock held. */
+static int usb_gadget_connect_locked(struct usb_gadget *gadget)
+	__must_hold(&gadget->udc->connect_lock)
 {
 	int ret = 0;
 
@@ -679,10 +703,15 @@ int usb_gadget_connect(struct usb_gadget *gadget)
 		goto out;
 	}
 
-	if (gadget->deactivated) {
+	if (gadget->connected)
+		goto out;
+
+	if (gadget->deactivated || !gadget->udc->started) {
 		/*
 		 * If gadget is deactivated we only save new state.
 		 * Gadget will be connected automatically after activation.
+		 *
+		 * udc first needs to be started before gadget can be pulled up.
 		 */
 		gadget->connected = true;
 		goto out;
@@ -697,22 +726,32 @@ out:
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(usb_gadget_connect);
 
 /**
- * usb_gadget_disconnect - software-controlled disconnect from USB host
- * @gadget:the peripheral being disconnected
- *
- * Disables the D+ (or potentially D-) pullup, which the host may see
- * as a disconnect (when a VBUS session is active).  Not all systems
- * support software pullup controls.
+ * usb_gadget_connect - software-controlled connect to USB host
+ * @gadget:the peripheral being connected
  *
- * Following a successful disconnect, invoke the ->disconnect() callback
- * for the current gadget driver so that UDC drivers don't need to.
+ * Enables the D+ (or potentially D-) pullup.  The host will start
+ * enumerating this gadget when the pullup is active and a VBUS session
+ * is active (the link is powered).
  *
  * Returns zero on success, else negative errno.
  */
-int usb_gadget_disconnect(struct usb_gadget *gadget)
+int usb_gadget_connect(struct usb_gadget *gadget)
+{
+	int ret;
+
+	mutex_lock(&gadget->udc->connect_lock);
+	ret = usb_gadget_connect_locked(gadget);
+	mutex_unlock(&gadget->udc->connect_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(usb_gadget_connect);
+
+/* Internal version of usb_gadget_disconnect needs to be called with connect_lock held. */
+static int usb_gadget_disconnect_locked(struct usb_gadget *gadget)
+	__must_hold(&gadget->udc->connect_lock)
 {
 	int ret = 0;
 
@@ -724,10 +763,12 @@ int usb_gadget_disconnect(struct usb_gadget *gadget)
 	if (!gadget->connected)
 		goto out;
 
-	if (gadget->deactivated) {
+	if (gadget->deactivated || !gadget->udc->started) {
 		/*
 		 * If gadget is deactivated we only save new state.
 		 * Gadget will stay disconnected after activation.
+		 *
+		 * udc should have been started before gadget being pulled down.
 		 */
 		gadget->connected = false;
 		goto out;
@@ -747,6 +788,30 @@ out:
 
 	return ret;
 }
+
+/**
+ * usb_gadget_disconnect - software-controlled disconnect from USB host
+ * @gadget:the peripheral being disconnected
+ *
+ * Disables the D+ (or potentially D-) pullup, which the host may see
+ * as a disconnect (when a VBUS session is active).  Not all systems
+ * support software pullup controls.
+ *
+ * Following a successful disconnect, invoke the ->disconnect() callback
+ * for the current gadget driver so that UDC drivers don't need to.
+ *
+ * Returns zero on success, else negative errno.
+ */
+int usb_gadget_disconnect(struct usb_gadget *gadget)
+{
+	int ret;
+
+	mutex_lock(&gadget->udc->connect_lock);
+	ret = usb_gadget_disconnect_locked(gadget);
+	mutex_unlock(&gadget->udc->connect_lock);
+
+	return ret;
+}
 EXPORT_SYMBOL_GPL(usb_gadget_disconnect);
 
 /**
@@ -767,10 +832,11 @@ int usb_gadget_deactivate(struct usb_gadget *gadget)
 	if (gadget->deactivated)
 		goto out;
 
+	mutex_lock(&gadget->udc->connect_lock);
 	if (gadget->connected) {
-		ret = usb_gadget_disconnect(gadget);
+		ret = usb_gadget_disconnect_locked(gadget);
 		if (ret)
-			goto out;
+			goto unlock;
 
 		/*
 		 * If gadget was being connected before deactivation, we want
@@ -780,6 +846,8 @@ int usb_gadget_deactivate(struct usb_gadget *gadget)
 	}
 	gadget->deactivated = true;
 
+unlock:
+	mutex_unlock(&gadget->udc->connect_lock);
 out:
 	trace_usb_gadget_deactivate(gadget, ret);
 
@@ -803,6 +871,7 @@ int usb_gadget_activate(struct usb_gadget *gadget)
 	if (!gadget->deactivated)
 		goto out;
 
+	mutex_lock(&gadget->udc->connect_lock);
 	gadget->deactivated = false;
 
 	/*
@@ -810,7 +879,8 @@ int usb_gadget_activate(struct usb_gadget *gadget)
 	 * while it was being deactivated, we call usb_gadget_connect().
 	 */
 	if (gadget->connected)
-		ret = usb_gadget_connect(gadget);
+		ret = usb_gadget_connect_locked(gadget);
+	mutex_unlock(&gadget->udc->connect_lock);
 
 out:
 	trace_usb_gadget_activate(gadget, ret);
@@ -1051,12 +1121,13 @@ EXPORT_SYMBOL_GPL(usb_gadget_set_state);
 
 /* ------------------------------------------------------------------------- */
 
-static void usb_udc_connect_control(struct usb_udc *udc)
+/* Acquire connect_lock before calling this function. */
+static void usb_udc_connect_control_locked(struct usb_udc *udc) __must_hold(&udc->connect_lock)
 {
-	if (udc->vbus)
-		usb_gadget_connect(udc->gadget);
+	if (udc->vbus && udc->started)
+		usb_gadget_connect_locked(udc->gadget);
 	else
-		usb_gadget_disconnect(udc->gadget);
+		usb_gadget_disconnect_locked(udc->gadget);
 }
 
 /**
@@ -1072,10 +1143,12 @@ void usb_udc_vbus_handler(struct usb_gadget *gadget, bool status)
 {
 	struct usb_udc *udc = gadget->udc;
 
+	mutex_lock(&udc->connect_lock);
 	if (udc) {
 		udc->vbus = status;
-		usb_udc_connect_control(udc);
+		usb_udc_connect_control_locked(udc);
 	}
+	mutex_unlock(&udc->connect_lock);
 }
 EXPORT_SYMBOL_GPL(usb_udc_vbus_handler);
 
@@ -1097,7 +1170,7 @@ void usb_gadget_udc_reset(struct usb_gadget *gadget,
 EXPORT_SYMBOL_GPL(usb_gadget_udc_reset);
 
 /**
- * usb_gadget_udc_start - tells usb device controller to start up
+ * usb_gadget_udc_start_locked - tells usb device controller to start up
  * @udc: The UDC to be started
  *
  * This call is issued by the UDC Class driver when it's about
@@ -1108,8 +1181,11 @@ EXPORT_SYMBOL_GPL(usb_gadget_udc_reset);
  * necessary to have it powered on.
  *
  * Returns zero on success, else negative errno.
+ *
+ * Caller should acquire connect_lock before invoking this function.
  */
-static inline int usb_gadget_udc_start(struct usb_udc *udc)
+static inline int usb_gadget_udc_start_locked(struct usb_udc *udc)
+	__must_hold(&udc->connect_lock)
 {
 	int ret;
 
@@ -1126,7 +1202,7 @@ static inline int usb_gadget_udc_start(struct usb_udc *udc)
 }
 
 /**
- * usb_gadget_udc_stop - tells usb device controller we don't need it anymore
+ * usb_gadget_udc_stop_locked - tells usb device controller we don't need it anymore
  * @udc: The UDC to be stopped
  *
  * This call is issued by the UDC Class driver after calling
@@ -1135,8 +1211,11 @@ static inline int usb_gadget_udc_start(struct usb_udc *udc)
  * The details are implementation specific, but it can go as
  * far as powering off UDC completely and disable its data
  * line pullups.
+ *
+ * Caller should acquire connect lock before invoking this function.
  */
-static inline void usb_gadget_udc_stop(struct usb_udc *udc)
+static inline void usb_gadget_udc_stop_locked(struct usb_udc *udc)
+	__must_hold(&udc->connect_lock)
 {
 	if (!udc->started) {
 		dev_err(&udc->dev, "UDC had already stopped\n");
@@ -1295,6 +1374,7 @@ int usb_add_gadget(struct usb_gadget *gadget)
 
 	udc->gadget = gadget;
 	gadget->udc = udc;
+	mutex_init(&udc->connect_lock);
 
 	udc->started = false;
 
@@ -1496,11 +1576,15 @@ static int gadget_bind_driver(struct device *dev)
 	if (ret)
 		goto err_bind;
 
-	ret = usb_gadget_udc_start(udc);
-	if (ret)
+	mutex_lock(&udc->connect_lock);
+	ret = usb_gadget_udc_start_locked(udc);
+	if (ret) {
+		mutex_unlock(&udc->connect_lock);
 		goto err_start;
+	}
 	usb_gadget_enable_async_callbacks(udc);
-	usb_udc_connect_control(udc);
+	usb_udc_connect_control_locked(udc);
+	mutex_unlock(&udc->connect_lock);
 
 	kobject_uevent(&udc->dev.kobj, KOBJ_CHANGE);
 	return 0;
@@ -1531,12 +1615,14 @@ static void gadget_unbind_driver(struct device *dev)
 
 	kobject_uevent(&udc->dev.kobj, KOBJ_CHANGE);
 
-	usb_gadget_disconnect(gadget);
+	mutex_lock(&udc->connect_lock);
+	usb_gadget_disconnect_locked(gadget);
 	usb_gadget_disable_async_callbacks(udc);
 	if (gadget->irq)
 		synchronize_irq(gadget->irq);
 	udc->driver->unbind(gadget);
-	usb_gadget_udc_stop(udc);
+	usb_gadget_udc_stop_locked(udc);
+	mutex_unlock(&udc->connect_lock);
 
 	mutex_lock(&udc_lock);
 	driver->is_bound = false;
@@ -1622,11 +1708,15 @@ static ssize_t soft_connect_store(struct device *dev,
 	}
 
 	if (sysfs_streq(buf, "connect")) {
-		usb_gadget_udc_start(udc);
-		usb_gadget_connect(udc->gadget);
+		mutex_lock(&udc->connect_lock);
+		usb_gadget_udc_start_locked(udc);
+		usb_gadget_connect_locked(udc->gadget);
+		mutex_unlock(&udc->connect_lock);
 	} else if (sysfs_streq(buf, "disconnect")) {
-		usb_gadget_disconnect(udc->gadget);
-		usb_gadget_udc_stop(udc);
+		mutex_lock(&udc->connect_lock);
+		usb_gadget_disconnect_locked(udc->gadget);
+		usb_gadget_udc_stop_locked(udc);
+		mutex_unlock(&udc->connect_lock);
 	} else {
 		dev_err(dev, "unsupported command '%s'\n", buf);
 		ret = -EINVAL;
@@ -1747,7 +1837,7 @@ static int usb_udc_uevent(const struct device *dev, struct kobj_uevent_env *env)
 	return 0;
 }
 
-static struct bus_type gadget_bus_type = {
+static const struct bus_type gadget_bus_type = {
 	.name = "gadget",
 	.probe = gadget_bind_driver,
 	.remove = gadget_unbind_driver,
@@ -1758,7 +1848,7 @@ static int __init usb_udc_init(void)
 {
 	int rc;
 
-	udc_class = class_create(THIS_MODULE, "udc");
+	udc_class = class_create("udc");
 	if (IS_ERR(udc_class)) {
 		pr_err("failed to create udc class --> %ld\n",
 				PTR_ERR(udc_class));
diff --git a/drivers/usb/gadget/udc/max3420_udc.c b/drivers/usb/gadget/udc/max3420_udc.c
index ddf0ed3eb4f2..12c519f32bf7 100644
--- a/drivers/usb/gadget/udc/max3420_udc.c
+++ b/drivers/usb/gadget/udc/max3420_udc.c
@@ -1319,7 +1319,7 @@ MODULE_DEVICE_TABLE(of, max3420_udc_of_match);
 static struct spi_driver max3420_driver = {
 	.driver = {
 		.name = "max3420-udc",
-		.of_match_table = of_match_ptr(max3420_udc_of_match),
+		.of_match_table = max3420_udc_of_match,
 	},
 	.probe = max3420_probe,
 	.remove = max3420_remove,
diff --git a/drivers/usb/gadget/udc/mv_udc_core.c b/drivers/usb/gadget/udc/mv_udc_core.c
index b397f3a848cf..08474c08d874 100644
--- a/drivers/usb/gadget/udc/mv_udc_core.c
+++ b/drivers/usb/gadget/udc/mv_udc_core.c
@@ -2229,7 +2229,11 @@ static int mv_udc_probe(struct platform_device *pdev)
 	INIT_LIST_HEAD(&udc->status_req->queue);
 
 	/* allocate a small amount of memory to get valid address */
-	udc->status_req->req.buf = kzalloc(8, GFP_KERNEL);
+	udc->status_req->req.buf = devm_kzalloc(&pdev->dev, 8, GFP_KERNEL);
+	if (!udc->status_req->req.buf) {
+		retval = -ENOMEM;
+		goto err_destroy_dma;
+	}
 	udc->status_req->req.dma = DMA_ADDR_INVALID;
 
 	udc->resume_state = USB_STATE_NOTATTACHED;
diff --git a/drivers/usb/gadget/udc/renesas_usb3.c b/drivers/usb/gadget/udc/renesas_usb3.c
index bee6bceafc4f..aac8bc185afa 100644
--- a/drivers/usb/gadget/udc/renesas_usb3.c
+++ b/drivers/usb/gadget/udc/renesas_usb3.c
@@ -22,7 +22,6 @@
 #include <linux/sizes.h>
 #include <linux/slab.h>
 #include <linux/string.h>
-#include <linux/sys_soc.h>
 #include <linux/uaccess.h>
 #include <linux/usb/ch9.h>
 #include <linux/usb/gadget.h>
@@ -2661,6 +2660,7 @@ static int renesas_usb3_remove(struct platform_device *pdev)
 	debugfs_remove_recursive(usb3->dentry);
 	device_remove_file(&pdev->dev, &dev_attr_role);
 
+	cancel_work_sync(&usb3->role_work);
 	usb_role_switch_unregister(usb3->role_sw);
 
 	usb_del_gadget_udc(&usb3->gadget);
@@ -2781,13 +2781,6 @@ static void renesas_usb3_init_ram(struct renesas_usb3 *usb3, struct device *dev,
 	}
 }
 
-static const struct renesas_usb3_priv renesas_usb3_priv_r8a7795_es1 = {
-	.ramsize_per_ramif = SZ_16K,
-	.num_ramif = 2,
-	.ramsize_per_pipe = SZ_4K,
-	.workaround_for_vbus = true,
-};
-
 static const struct renesas_usb3_priv renesas_usb3_priv_gen3 = {
 	.ramsize_per_ramif = SZ_16K,
 	.num_ramif = 4,
@@ -2829,14 +2822,6 @@ static const struct of_device_id usb3_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, usb3_of_match);
 
-static const struct soc_device_attribute renesas_usb3_quirks_match[] = {
-	{
-		.soc_id = "r8a7795", .revision = "ES1.*",
-		.data = &renesas_usb3_priv_r8a7795_es1,
-	},
-	{ /* sentinel */ }
-};
-
 static const unsigned int renesas_usb3_cable[] = {
 	EXTCON_USB,
 	EXTCON_USB_HOST,
@@ -2854,13 +2839,8 @@ static int renesas_usb3_probe(struct platform_device *pdev)
 	struct renesas_usb3 *usb3;
 	int irq, ret;
 	const struct renesas_usb3_priv *priv;
-	const struct soc_device_attribute *attr;
 
-	attr = soc_device_match(renesas_usb3_quirks_match);
-	if (attr)
-		priv = attr->data;
-	else
-		priv = of_device_get_match_data(&pdev->dev);
+	priv = of_device_get_match_data(&pdev->dev);
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
@@ -3039,7 +3019,7 @@ static struct platform_driver renesas_usb3_driver = {
 	.driver		= {
 		.name =	udc_name,
 		.pm		= &renesas_usb3_pm_ops,
-		.of_match_table = of_match_ptr(usb3_of_match),
+		.of_match_table = usb3_of_match,
 	},
 };
 module_platform_driver(renesas_usb3_driver);
diff --git a/drivers/usb/gadget/udc/renesas_usbf.c b/drivers/usb/gadget/udc/renesas_usbf.c
index cb23e62e8a87..84ac9fe4ce7f 100644
--- a/drivers/usb/gadget/udc/renesas_usbf.c
+++ b/drivers/usb/gadget/udc/renesas_usbf.c
@@ -545,17 +545,6 @@ static inline void usbf_ep_dma_reg_bitclr(struct usbf_ep *ep, uint offset,
 	usbf_ep_dma_reg_writel(ep, offset, tmp);
 }
 
-static inline void usbf_ep_dma_reg_clrset(struct usbf_ep *ep, uint offset,
-					  u32 clr, u32 set)
-{
-	u32 tmp;
-
-	tmp = usbf_ep_dma_reg_readl(ep, offset);
-	tmp &= ~clr;
-	tmp |= set;
-	usbf_ep_dma_reg_writel(ep, offset, tmp);
-}
-
 static void usbf_ep0_send_null(struct usbf_ep *ep0, bool is_data1)
 {
 	u32 set;
diff --git a/drivers/usb/gadget/udc/rzv2m_usb3drd.c b/drivers/usb/gadget/udc/rzv2m_usb3drd.c
index 3c8bbf843038..589c7252e014 100644
--- a/drivers/usb/gadget/udc/rzv2m_usb3drd.c
+++ b/drivers/usb/gadget/udc/rzv2m_usb3drd.c
@@ -6,7 +6,7 @@
  */
 
 #include <linux/io.h>
-#include <linux/of_device.h>
+#include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/reset.h>
@@ -126,7 +126,7 @@ MODULE_DEVICE_TABLE(of, rzv2m_usb3drd_of_match);
 static struct platform_driver rzv2m_usb3drd_driver = {
 	.driver = {
 		.name = "rzv2m-usb3drd",
-		.of_match_table = of_match_ptr(rzv2m_usb3drd_of_match),
+		.of_match_table = rzv2m_usb3drd_of_match,
 	},
 	.probe = rzv2m_usb3drd_probe,
 	.remove = rzv2m_usb3drd_remove,
diff --git a/drivers/usb/gadget/udc/snps_udc_plat.c b/drivers/usb/gadget/udc/snps_udc_plat.c
index 8bbb89c80348..0d3e705655b9 100644
--- a/drivers/usb/gadget/udc/snps_udc_plat.c
+++ b/drivers/usb/gadget/udc/snps_udc_plat.c
@@ -158,7 +158,7 @@ static int udc_plat_probe(struct platform_device *pdev)
 	}
 
 	/* Register for extcon if supported */
-	if (of_get_property(dev->of_node, "extcon", NULL)) {
+	if (of_property_present(dev->of_node, "extcon")) {
 		udc->edev = extcon_get_edev_by_phandle(dev, 0);
 		if (IS_ERR(udc->edev)) {
 			if (PTR_ERR(udc->edev) == -EPROBE_DEFER)
diff --git a/drivers/usb/gadget/udc/tegra-xudc.c b/drivers/usb/gadget/udc/tegra-xudc.c
index 2b71b33725f1..34e9c1df54c7 100644
--- a/drivers/usb/gadget/udc/tegra-xudc.c
+++ b/drivers/usb/gadget/udc/tegra-xudc.c
@@ -2162,15 +2162,14 @@ static int tegra_xudc_gadget_stop(struct usb_gadget *gadget)
 static int tegra_xudc_gadget_vbus_draw(struct usb_gadget *gadget,
 						unsigned int m_a)
 {
-	int ret = 0;
 	struct tegra_xudc *xudc = to_xudc(gadget);
 
 	dev_dbg(xudc->dev, "%s: %u mA\n", __func__, m_a);
 
-	if (xudc->curr_usbphy->chg_type == SDP_TYPE)
-		ret = usb_phy_set_power(xudc->curr_usbphy, m_a);
+	if (xudc->curr_usbphy && xudc->curr_usbphy->chg_type == SDP_TYPE)
+		return usb_phy_set_power(xudc->curr_usbphy, m_a);
 
-	return ret;
+	return 0;
 }
 
 static int tegra_xudc_set_selfpowered(struct usb_gadget *gadget, int is_on)
diff --git a/drivers/usb/gadget/udc/trace.h b/drivers/usb/gadget/udc/trace.h
index abdbcb1bacb0..a5ed26fbc2da 100644
--- a/drivers/usb/gadget/udc/trace.h
+++ b/drivers/usb/gadget/udc/trace.h
@@ -91,6 +91,11 @@ DEFINE_EVENT(udc_log_gadget, usb_gadget_wakeup,
 	TP_ARGS(g, ret)
 );
 
+DEFINE_EVENT(udc_log_gadget, usb_gadget_set_remote_wakeup,
+	TP_PROTO(struct usb_gadget *g, int ret),
+	TP_ARGS(g, ret)
+);
+
 DEFINE_EVENT(udc_log_gadget, usb_gadget_set_selfpowered,
 	TP_PROTO(struct usb_gadget *g, int ret),
 	TP_ARGS(g, ret)
diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig
index eacb603ad1b2..c170672f847e 100644
--- a/drivers/usb/host/Kconfig
+++ b/drivers/usb/host/Kconfig
@@ -622,33 +622,6 @@ config FHCI_DEBUG
 	  Say "y" to see some FHCI debug information and statistics
 	  through debugfs.
 
-config USB_U132_HCD
-	tristate "Elan U132 Adapter Host Controller"
-	depends on USB_FTDI_ELAN
-	help
-	  The U132 adapter is a USB to CardBus adapter specifically designed
-	  for PC cards that contain an OHCI host controller. Typical PC cards
-	  are the Orange Mobile 3G Option GlobeTrotter Fusion card. The U132
-	  adapter will *NOT* work with PC cards that do not contain an OHCI
-	  controller.
-
-	  For those PC cards that contain multiple OHCI controllers only the
-	  first one is used.
-
-	  The driver consists of two modules, the "ftdi-elan" module is a
-	  USB client driver that interfaces to the FTDI chip within ELAN's
-	  USB-to-PCMCIA adapter, and this "u132-hcd" module is a USB host
-	  controller driver that talks to the OHCI controller within the
-	  CardBus cards that are inserted in the U132 adapter.
-
-	  This driver has been tested with a CardBus OHCI USB adapter, and
-	  worked with a USB PEN Drive inserted into the first USB port of
-	  the PCCARD. A rather pointless thing to do, but useful for testing.
-
-	  It is safe to say M here.
-
-	  See also <http://www.elandigitalsystems.com/support/ufaq/u132linux.php>
-
 config USB_SL811_HCD
 	tristate "SL811HS HCD support"
 	depends on HAS_IOMEM
diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile
index 5a13712f367d..be4e5245c52f 100644
--- a/drivers/usb/host/Makefile
+++ b/drivers/usb/host/Makefile
@@ -76,7 +76,6 @@ obj-$(CONFIG_USB_XHCI_MTK)	+= xhci-mtk-hcd.o
 obj-$(CONFIG_USB_XHCI_TEGRA)	+= xhci-tegra.o
 obj-$(CONFIG_USB_SL811_HCD)	+= sl811-hcd.o
 obj-$(CONFIG_USB_SL811_CS)	+= sl811_cs.o
-obj-$(CONFIG_USB_U132_HCD)	+= u132-hcd.o
 obj-$(CONFIG_USB_R8A66597_HCD)	+= r8a66597-hcd.o
 obj-$(CONFIG_USB_FSL_USB2)	+= fsl-mph-dr-of.o
 obj-$(CONFIG_USB_EHCI_FSL)	+= fsl-mph-dr-of.o
diff --git a/drivers/usb/host/ehci-ppc-of.c b/drivers/usb/host/ehci-ppc-of.c
index 62a0a193798c..b3aa464e9d2c 100644
--- a/drivers/usb/host/ehci-ppc-of.c
+++ b/drivers/usb/host/ehci-ppc-of.c
@@ -151,13 +151,13 @@ static int ehci_hcd_ppc_of_probe(struct platform_device *op)
 		of_node_put(np);
 	}
 
-	if (of_get_property(dn, "big-endian", NULL)) {
+	if (of_property_read_bool(dn, "big-endian")) {
 		ehci->big_endian_mmio = 1;
 		ehci->big_endian_desc = 1;
 	}
-	if (of_get_property(dn, "big-endian-regs", NULL))
+	if (of_property_read_bool(dn, "big-endian-regs"))
 		ehci->big_endian_mmio = 1;
-	if (of_get_property(dn, "big-endian-desc", NULL))
+	if (of_property_read_bool(dn, "big-endian-desc"))
 		ehci->big_endian_desc = 1;
 
 	ehci->caps = hcd->regs;
diff --git a/drivers/usb/host/fsl-mph-dr-of.c b/drivers/usb/host/fsl-mph-dr-of.c
index 46c6a152b865..9db909d12354 100644
--- a/drivers/usb/host/fsl-mph-dr-of.c
+++ b/drivers/usb/host/fsl-mph-dr-of.c
@@ -200,19 +200,16 @@ static int fsl_usb2_mph_dr_of_probe(struct platform_device *ofdev)
 	dev_data = get_dr_mode_data(np);
 
 	if (of_device_is_compatible(np, "fsl-usb2-mph")) {
-		if (of_get_property(np, "port0", NULL))
+		if (of_property_present(np, "port0"))
 			pdata->port_enables |= FSL_USB2_PORT0_ENABLED;
 
-		if (of_get_property(np, "port1", NULL))
+		if (of_property_present(np, "port1"))
 			pdata->port_enables |= FSL_USB2_PORT1_ENABLED;
 
 		pdata->operating_mode = FSL_USB2_MPH_HOST;
 	} else {
-		if (of_get_property(np, "fsl,invert-drvvbus", NULL))
-			pdata->invert_drvvbus = 1;
-
-		if (of_get_property(np, "fsl,invert-pwr-fault", NULL))
-			pdata->invert_pwr_fault = 1;
+		pdata->invert_drvvbus = of_property_read_bool(np, "fsl,invert-drvvbus");
+		pdata->invert_pwr_fault = of_property_read_bool(np, "fsl,invert-pwr-fault");
 
 		/* setup mode selected in the device tree */
 		pdata->operating_mode = dev_data->op_mode;
diff --git a/drivers/usb/host/max3421-hcd.c b/drivers/usb/host/max3421-hcd.c
index 28d1524ee2fa..d152d72de126 100644
--- a/drivers/usb/host/max3421-hcd.c
+++ b/drivers/usb/host/max3421-hcd.c
@@ -1951,7 +1951,7 @@ static struct spi_driver max3421_driver = {
 	.remove		= max3421_remove,
 	.driver		= {
 		.name	= "max3421-hcd",
-		.of_match_table = of_match_ptr(max3421_of_match_table),
+		.of_match_table = max3421_of_match_table,
 	},
 };
 
diff --git a/drivers/usb/host/oxu210hp-hcd.c b/drivers/usb/host/oxu210hp-hcd.c
index 3a441310c713..f998d3f1a78a 100644
--- a/drivers/usb/host/oxu210hp-hcd.c
+++ b/drivers/usb/host/oxu210hp-hcd.c
@@ -169,7 +169,7 @@ struct ehci_regs {
 #define FLAG_CF		(1<<0)		/* true: we'll support "high speed" */
 
 	/* PORTSC: offset 0x44 */
-	u32		port_status[0];	/* up to N_PORTS */
+	u32		port_status[];	/* up to N_PORTS */
 /* 31:23 reserved */
 #define PORT_WKOC_E	(1<<22)		/* wake on overcurrent (enable) */
 #define PORT_WKDISC_E	(1<<21)		/* wake on disconnect (enable) */
diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c
index ef08d68b9714..2665832f9add 100644
--- a/drivers/usb/host/pci-quirks.c
+++ b/drivers/usb/host/pci-quirks.c
@@ -207,8 +207,7 @@ EXPORT_SYMBOL_GPL(sb800_prefetch);
 static void usb_amd_find_chipset_info(void)
 {
 	unsigned long flags;
-	struct amd_chipset_info info;
-	info.need_pll_quirk = false;
+	struct amd_chipset_info info = { };
 
 	spin_lock_irqsave(&amd_lock, flags);
 
@@ -218,7 +217,6 @@ static void usb_amd_find_chipset_info(void)
 		spin_unlock_irqrestore(&amd_lock, flags);
 		return;
 	}
-	memset(&info, 0, sizeof(info));
 	spin_unlock_irqrestore(&amd_lock, flags);
 
 	if (!amd_chipset_sb_type_init(&info)) {
diff --git a/drivers/usb/host/u132-hcd.c b/drivers/usb/host/u132-hcd.c
deleted file mode 100644
index 95240c9c45bd..000000000000
--- a/drivers/usb/host/u132-hcd.c
+++ /dev/null
@@ -1,3219 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
-* Host Controller Driver for the Elan Digital Systems U132 adapter
-*
-* Copyright(C) 2006 Elan Digital Systems Limited
-* http://www.elandigitalsystems.com
-*
-* Author and Maintainer - Tony Olech - Elan Digital Systems
-* tony.olech@elandigitalsystems.com
-*
-* This driver was written by Tony Olech(tony.olech@elandigitalsystems.com)
-* based on various USB host drivers in the 2.6.15 linux kernel
-* with constant reference to the 3rd Edition of Linux Device Drivers
-* published by O'Reilly
-*
-* The U132 adapter is a USB to CardBus adapter specifically designed
-* for PC cards that contain an OHCI host controller. Typical PC cards
-* are the Orange Mobile 3G Option GlobeTrotter Fusion card.
-*
-* The U132 adapter will *NOT *work with PC cards that do not contain
-* an OHCI controller. A simple way to test whether a PC card has an
-* OHCI controller as an interface is to insert the PC card directly
-* into a laptop(or desktop) with a CardBus slot and if "lspci" shows
-* a new USB controller and "lsusb -v" shows a new OHCI Host Controller
-* then there is a good chance that the U132 adapter will support the
-* PC card.(you also need the specific client driver for the PC card)
-*
-* Please inform the Author and Maintainer about any PC cards that
-* contain OHCI Host Controller and work when directly connected to
-* an embedded CardBus slot but do not work when they are connected
-* via an ELAN U132 adapter.
-*
-*/
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/delay.h>
-#include <linux/ioport.h>
-#include <linux/pci_ids.h>
-#include <linux/sched.h>
-#include <linux/slab.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/timer.h>
-#include <linux/list.h>
-#include <linux/interrupt.h>
-#include <linux/usb.h>
-#include <linux/usb/hcd.h>
-#include <linux/workqueue.h>
-#include <linux/platform_device.h>
-#include <linux/mutex.h>
-#include <asm/io.h>
-#include <asm/irq.h>
-#include <asm/byteorder.h>
-
-	/* FIXME ohci.h is ONLY for internal use by the OHCI driver.
-	 * If you're going to try stuff like this, you need to split
-	 * out shareable stuff (register declarations?) into its own
-	 * file, maybe name <linux/usb/ohci.h>
-	 */
-
-#include "ohci.h"
-#define OHCI_CONTROL_INIT OHCI_CTRL_CBSR
-#define OHCI_INTR_INIT (OHCI_INTR_MIE | OHCI_INTR_UE | OHCI_INTR_RD | \
-	OHCI_INTR_WDH)
-MODULE_AUTHOR("Tony Olech - Elan Digital Systems Limited");
-MODULE_DESCRIPTION("U132 USB Host Controller Driver");
-MODULE_LICENSE("GPL");
-#define INT_MODULE_PARM(n, v) static int n = v;module_param(n, int, 0444)
-INT_MODULE_PARM(testing, 0);
-/* Some boards misreport power switching/overcurrent*/
-static bool distrust_firmware = true;
-module_param(distrust_firmware, bool, 0);
-MODULE_PARM_DESC(distrust_firmware, "true to distrust firmware power/overcurrent"
-	"t setup");
-static DECLARE_WAIT_QUEUE_HEAD(u132_hcd_wait);
-/*
-* u132_module_lock exists to protect access to global variables
-*
-*/
-static DEFINE_MUTEX(u132_module_lock);
-static int u132_exiting;
-static int u132_instances;
-/*
-* end of the global variables protected by u132_module_lock
-*/
-static struct workqueue_struct *workqueue;
-#define MAX_U132_PORTS 7
-#define MAX_U132_ADDRS 128
-#define MAX_U132_UDEVS 4
-#define MAX_U132_ENDPS 100
-#define MAX_U132_RINGS 4
-static const char *cc_to_text[16] = {
-	"No Error ",
-	"CRC Error ",
-	"Bit Stuff ",
-	"Data Togg ",
-	"Stall ",
-	"DevNotResp ",
-	"PIDCheck ",
-	"UnExpPID ",
-	"DataOver ",
-	"DataUnder ",
-	"(for hw) ",
-	"(for hw) ",
-	"BufferOver ",
-	"BuffUnder ",
-	"(for HCD) ",
-	"(for HCD) "
-};
-struct u132_port {
-	struct u132 *u132;
-	int reset;
-	int enable;
-	int power;
-	int Status;
-};
-struct u132_addr {
-	u8 address;
-};
-struct u132_udev {
-	struct kref kref;
-	struct usb_device *usb_device;
-	u8 enumeration;
-	u8 udev_number;
-	u8 usb_addr;
-	u8 portnumber;
-	u8 endp_number_in[16];
-	u8 endp_number_out[16];
-};
-#define ENDP_QUEUE_SHIFT 3
-#define ENDP_QUEUE_SIZE (1<<ENDP_QUEUE_SHIFT)
-#define ENDP_QUEUE_MASK (ENDP_QUEUE_SIZE-1)
-struct u132_urbq {
-	struct list_head urb_more;
-	struct urb *urb;
-};
-struct u132_spin {
-	spinlock_t slock;
-};
-struct u132_endp {
-	struct kref kref;
-	u8 udev_number;
-	u8 endp_number;
-	u8 usb_addr;
-	u8 usb_endp;
-	struct u132 *u132;
-	struct list_head endp_ring;
-	struct u132_ring *ring;
-	unsigned toggle_bits:2;
-	unsigned active:1;
-	unsigned delayed:1;
-	unsigned input:1;
-	unsigned output:1;
-	unsigned pipetype:2;
-	unsigned dequeueing:1;
-	unsigned edset_flush:1;
-	unsigned spare_bits:14;
-	unsigned long jiffies;
-	struct usb_host_endpoint *hep;
-	struct u132_spin queue_lock;
-	u16 queue_size;
-	u16 queue_last;
-	u16 queue_next;
-	struct urb *urb_list[ENDP_QUEUE_SIZE];
-	struct list_head urb_more;
-	struct delayed_work scheduler;
-};
-struct u132_ring {
-	unsigned in_use:1;
-	unsigned length:7;
-	u8 number;
-	struct u132 *u132;
-	struct u132_endp *curr_endp;
-	struct delayed_work scheduler;
-};
-struct u132 {
-	struct kref kref;
-	struct mutex sw_lock;
-	struct mutex scheduler_lock;
-	struct u132_platform_data *board;
-	struct platform_device *platform_dev;
-	struct u132_ring ring[MAX_U132_RINGS];
-	int sequence_num;
-	int going;
-	int power;
-	int reset;
-	int num_ports;
-	u32 hc_control;
-	u32 hc_fminterval;
-	u32 hc_roothub_status;
-	u32 hc_roothub_a;
-	u32 hc_roothub_portstatus[MAX_ROOT_PORTS];
-	int flags;
-	unsigned long next_statechange;
-	struct delayed_work monitor;
-	int num_endpoints;
-	struct u132_addr addr[MAX_U132_ADDRS];
-	struct u132_udev udev[MAX_U132_UDEVS];
-	struct u132_port port[MAX_U132_PORTS];
-	struct u132_endp *endp[MAX_U132_ENDPS];
-};
-
-/*
-* these cannot be inlines because we need the structure offset!!
-* Does anyone have a better way?????
-*/
-#define ftdi_read_pcimem(pdev, member, data) usb_ftdi_elan_read_pcimem(pdev, \
-	offsetof(struct ohci_regs, member), 0, data);
-#define ftdi_write_pcimem(pdev, member, data) usb_ftdi_elan_write_pcimem(pdev, \
-	offsetof(struct ohci_regs, member), 0, data)
-#define u132_read_pcimem(u132, member, data) \
-	usb_ftdi_elan_read_pcimem(u132->platform_dev, offsetof(struct \
-	ohci_regs, member), 0, data)
-#define u132_write_pcimem(u132, member, data) \
-	usb_ftdi_elan_write_pcimem(u132->platform_dev, offsetof(struct \
-	ohci_regs, member), 0, data)
-static inline struct u132 *udev_to_u132(struct u132_udev *udev)
-{
-	u8 udev_number = udev->udev_number;
-	return container_of(udev, struct u132, udev[udev_number]);
-}
-
-static inline struct u132 *hcd_to_u132(struct usb_hcd *hcd)
-{
-	return (struct u132 *)(hcd->hcd_priv);
-}
-
-static inline struct usb_hcd *u132_to_hcd(struct u132 *u132)
-{
-	return container_of((void *)u132, struct usb_hcd, hcd_priv);
-}
-
-static inline void u132_disable(struct u132 *u132)
-{
-	u132_to_hcd(u132)->state = HC_STATE_HALT;
-}
-
-
-#define kref_to_u132(d) container_of(d, struct u132, kref)
-#define kref_to_u132_endp(d) container_of(d, struct u132_endp, kref)
-#define kref_to_u132_udev(d) container_of(d, struct u132_udev, kref)
-#include "../misc/usb_u132.h"
-static const char hcd_name[] = "u132_hcd";
-#define PORT_C_MASK ((USB_PORT_STAT_C_CONNECTION | USB_PORT_STAT_C_ENABLE | \
-	USB_PORT_STAT_C_SUSPEND | USB_PORT_STAT_C_OVERCURRENT | \
-	USB_PORT_STAT_C_RESET) << 16)
-static void u132_hcd_delete(struct kref *kref)
-{
-	struct u132 *u132 = kref_to_u132(kref);
-	struct platform_device *pdev = u132->platform_dev;
-	struct usb_hcd *hcd = u132_to_hcd(u132);
-	u132->going += 1;
-	mutex_lock(&u132_module_lock);
-	u132_instances -= 1;
-	mutex_unlock(&u132_module_lock);
-	dev_warn(&u132->platform_dev->dev, "FREEING the hcd=%p and thus the u13"
-		"2=%p going=%d pdev=%p\n", hcd, u132, u132->going, pdev);
-	usb_put_hcd(hcd);
-}
-
-static inline void u132_u132_put_kref(struct u132 *u132)
-{
-	kref_put(&u132->kref, u132_hcd_delete);
-}
-
-static inline void u132_u132_init_kref(struct u132 *u132)
-{
-	kref_init(&u132->kref);
-}
-
-static void u132_udev_delete(struct kref *kref)
-{
-	struct u132_udev *udev = kref_to_u132_udev(kref);
-	udev->udev_number = 0;
-	udev->usb_device = NULL;
-	udev->usb_addr = 0;
-	udev->enumeration = 0;
-}
-
-static inline void u132_udev_put_kref(struct u132 *u132, struct u132_udev *udev)
-{
-	kref_put(&udev->kref, u132_udev_delete);
-}
-
-static inline void u132_udev_get_kref(struct u132 *u132, struct u132_udev *udev)
-{
-	kref_get(&udev->kref);
-}
-
-static inline void u132_udev_init_kref(struct u132 *u132,
-	struct u132_udev *udev)
-{
-	kref_init(&udev->kref);
-}
-
-static inline void u132_ring_put_kref(struct u132 *u132, struct u132_ring *ring)
-{
-	kref_put(&u132->kref, u132_hcd_delete);
-}
-
-static void u132_ring_requeue_work(struct u132 *u132, struct u132_ring *ring,
-	unsigned int delta)
-{
-	if (delta > 0) {
-		if (queue_delayed_work(workqueue, &ring->scheduler, delta))
-			return;
-	} else if (queue_delayed_work(workqueue, &ring->scheduler, 0))
-		return;
-	kref_put(&u132->kref, u132_hcd_delete);
-}
-
-static void u132_ring_queue_work(struct u132 *u132, struct u132_ring *ring,
-	unsigned int delta)
-{
-	kref_get(&u132->kref);
-	u132_ring_requeue_work(u132, ring, delta);
-}
-
-static void u132_ring_cancel_work(struct u132 *u132, struct u132_ring *ring)
-{
-	if (cancel_delayed_work(&ring->scheduler))
-		kref_put(&u132->kref, u132_hcd_delete);
-}
-
-static void u132_endp_delete(struct kref *kref)
-{
-	struct u132_endp *endp = kref_to_u132_endp(kref);
-	struct u132 *u132 = endp->u132;
-	u8 usb_addr = endp->usb_addr;
-	u8 usb_endp = endp->usb_endp;
-	u8 address = u132->addr[usb_addr].address;
-	struct u132_udev *udev = &u132->udev[address];
-	u8 endp_number = endp->endp_number;
-	struct usb_host_endpoint *hep = endp->hep;
-	struct u132_ring *ring = endp->ring;
-	struct list_head *head = &endp->endp_ring;
-	ring->length -= 1;
-	if (endp == ring->curr_endp) {
-		if (list_empty(head)) {
-			ring->curr_endp = NULL;
-			list_del(head);
-		} else {
-			struct u132_endp *next_endp = list_entry(head->next,
-				struct u132_endp, endp_ring);
-			ring->curr_endp = next_endp;
-			list_del(head);
-		}
-	} else
-		list_del(head);
-	if (endp->input) {
-		udev->endp_number_in[usb_endp] = 0;
-		u132_udev_put_kref(u132, udev);
-	}
-	if (endp->output) {
-		udev->endp_number_out[usb_endp] = 0;
-		u132_udev_put_kref(u132, udev);
-	}
-	u132->endp[endp_number - 1] = NULL;
-	hep->hcpriv = NULL;
-	kfree(endp);
-	u132_u132_put_kref(u132);
-}
-
-static inline void u132_endp_put_kref(struct u132 *u132, struct u132_endp *endp)
-{
-	kref_put(&endp->kref, u132_endp_delete);
-}
-
-static inline void u132_endp_get_kref(struct u132 *u132, struct u132_endp *endp)
-{
-	kref_get(&endp->kref);
-}
-
-static inline void u132_endp_init_kref(struct u132 *u132,
-	struct u132_endp *endp)
-{
-	kref_init(&endp->kref);
-	kref_get(&u132->kref);
-}
-
-static void u132_endp_queue_work(struct u132 *u132, struct u132_endp *endp,
-	unsigned int delta)
-{
-	if (queue_delayed_work(workqueue, &endp->scheduler, delta))
-		kref_get(&endp->kref);
-}
-
-static void u132_endp_cancel_work(struct u132 *u132, struct u132_endp *endp)
-{
-	if (cancel_delayed_work(&endp->scheduler))
-		kref_put(&endp->kref, u132_endp_delete);
-}
-
-static inline void u132_monitor_put_kref(struct u132 *u132)
-{
-	kref_put(&u132->kref, u132_hcd_delete);
-}
-
-static void u132_monitor_queue_work(struct u132 *u132, unsigned int delta)
-{
-	if (queue_delayed_work(workqueue, &u132->monitor, delta))
-		kref_get(&u132->kref);
-}
-
-static void u132_monitor_requeue_work(struct u132 *u132, unsigned int delta)
-{
-	if (!queue_delayed_work(workqueue, &u132->monitor, delta))
-		kref_put(&u132->kref, u132_hcd_delete);
-}
-
-static void u132_monitor_cancel_work(struct u132 *u132)
-{
-	if (cancel_delayed_work(&u132->monitor))
-		kref_put(&u132->kref, u132_hcd_delete);
-}
-
-static int read_roothub_info(struct u132 *u132)
-{
-	u32 revision;
-	int retval;
-	retval = u132_read_pcimem(u132, revision, &revision);
-	if (retval) {
-		dev_err(&u132->platform_dev->dev, "error %d accessing device co"
-			"ntrol\n", retval);
-		return retval;
-	} else if ((revision & 0xFF) == 0x10) {
-	} else if ((revision & 0xFF) == 0x11) {
-	} else {
-		dev_err(&u132->platform_dev->dev, "device revision is not valid"
-			" %08X\n", revision);
-		return -ENODEV;
-	}
-	retval = u132_read_pcimem(u132, control, &u132->hc_control);
-	if (retval) {
-		dev_err(&u132->platform_dev->dev, "error %d accessing device co"
-			"ntrol\n", retval);
-		return retval;
-	}
-	retval = u132_read_pcimem(u132, roothub.status,
-		&u132->hc_roothub_status);
-	if (retval) {
-		dev_err(&u132->platform_dev->dev, "error %d accessing device re"
-			"g roothub.status\n", retval);
-		return retval;
-	}
-	retval = u132_read_pcimem(u132, roothub.a, &u132->hc_roothub_a);
-	if (retval) {
-		dev_err(&u132->platform_dev->dev, "error %d accessing device re"
-			"g roothub.a\n", retval);
-		return retval;
-	}
-	{
-		int I = u132->num_ports;
-		int i = 0;
-		while (I-- > 0) {
-			retval = u132_read_pcimem(u132, roothub.portstatus[i],
-				&u132->hc_roothub_portstatus[i]);
-			if (retval) {
-				dev_err(&u132->platform_dev->dev, "error %d acc"
-					"essing device roothub.portstatus[%d]\n"
-					, retval, i);
-				return retval;
-			} else
-				i += 1;
-		}
-	}
-	return 0;
-}
-
-static void u132_hcd_monitor_work(struct work_struct *work)
-{
-	struct u132 *u132 = container_of(work, struct u132, monitor.work);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		u132_monitor_put_kref(u132);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed\n");
-		u132_monitor_put_kref(u132);
-		return;
-	} else {
-		int retval;
-		mutex_lock(&u132->sw_lock);
-		retval = read_roothub_info(u132);
-		if (retval) {
-			struct usb_hcd *hcd = u132_to_hcd(u132);
-			u132_disable(u132);
-			u132->going = 1;
-			mutex_unlock(&u132->sw_lock);
-			usb_hc_died(hcd);
-			ftdi_elan_gone_away(u132->platform_dev);
-			u132_monitor_put_kref(u132);
-			return;
-		} else {
-			u132_monitor_requeue_work(u132, 500);
-			mutex_unlock(&u132->sw_lock);
-			return;
-		}
-	}
-}
-
-static void u132_hcd_giveback_urb(struct u132 *u132, struct u132_endp *endp,
-	struct urb *urb, int status)
-{
-	struct u132_ring *ring;
-	unsigned long irqs;
-	struct usb_hcd *hcd = u132_to_hcd(u132);
-	urb->error_count = 0;
-	spin_lock_irqsave(&endp->queue_lock.slock, irqs);
-	usb_hcd_unlink_urb_from_ep(hcd, urb);
-	endp->queue_next += 1;
-	if (ENDP_QUEUE_SIZE > --endp->queue_size) {
-		endp->active = 0;
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-	} else {
-		struct list_head *next = endp->urb_more.next;
-		struct u132_urbq *urbq = list_entry(next, struct u132_urbq,
-			urb_more);
-		list_del(next);
-		endp->urb_list[ENDP_QUEUE_MASK & endp->queue_last++] =
-			urbq->urb;
-		endp->active = 0;
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-		kfree(urbq);
-	}
-	mutex_lock(&u132->scheduler_lock);
-	ring = endp->ring;
-	ring->in_use = 0;
-	u132_ring_cancel_work(u132, ring);
-	u132_ring_queue_work(u132, ring, 0);
-	mutex_unlock(&u132->scheduler_lock);
-	u132_endp_put_kref(u132, endp);
-	usb_hcd_giveback_urb(hcd, urb, status);
-}
-
-static void u132_hcd_forget_urb(struct u132 *u132, struct u132_endp *endp,
-	struct urb *urb, int status)
-{
-	u132_endp_put_kref(u132, endp);
-}
-
-static void u132_hcd_abandon_urb(struct u132 *u132, struct u132_endp *endp,
-	struct urb *urb, int status)
-{
-	unsigned long irqs;
-	struct usb_hcd *hcd = u132_to_hcd(u132);
-	urb->error_count = 0;
-	spin_lock_irqsave(&endp->queue_lock.slock, irqs);
-	usb_hcd_unlink_urb_from_ep(hcd, urb);
-	endp->queue_next += 1;
-	if (ENDP_QUEUE_SIZE > --endp->queue_size) {
-		endp->active = 0;
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-	} else {
-		struct list_head *next = endp->urb_more.next;
-		struct u132_urbq *urbq = list_entry(next, struct u132_urbq,
-			urb_more);
-		list_del(next);
-		endp->urb_list[ENDP_QUEUE_MASK & endp->queue_last++] =
-			urbq->urb;
-		endp->active = 0;
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-		kfree(urbq);
-	}
-	usb_hcd_giveback_urb(hcd, urb, status);
-}
-
-static inline int edset_input(struct u132 *u132, struct u132_ring *ring,
-	struct u132_endp *endp, struct urb *urb, u8 address, u8 toggle_bits,
-	void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-	int toggle_bits, int error_count, int condition_code, int repeat_number,
-	 int halted, int skipped, int actual, int non_null))
-{
-	return usb_ftdi_elan_edset_input(u132->platform_dev, ring->number, endp,
-		 urb, address, endp->usb_endp, toggle_bits, callback);
-}
-
-static inline int edset_setup(struct u132 *u132, struct u132_ring *ring,
-	struct u132_endp *endp, struct urb *urb, u8 address, u8 toggle_bits,
-	void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-	int toggle_bits, int error_count, int condition_code, int repeat_number,
-	 int halted, int skipped, int actual, int non_null))
-{
-	return usb_ftdi_elan_edset_setup(u132->platform_dev, ring->number, endp,
-		 urb, address, endp->usb_endp, toggle_bits, callback);
-}
-
-static inline int edset_single(struct u132 *u132, struct u132_ring *ring,
-	struct u132_endp *endp, struct urb *urb, u8 address, u8 toggle_bits,
-	void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-	int toggle_bits, int error_count, int condition_code, int repeat_number,
-	 int halted, int skipped, int actual, int non_null))
-{
-	return usb_ftdi_elan_edset_single(u132->platform_dev, ring->number,
-		endp, urb, address, endp->usb_endp, toggle_bits, callback);
-}
-
-static inline int edset_output(struct u132 *u132, struct u132_ring *ring,
-	struct u132_endp *endp, struct urb *urb, u8 address, u8 toggle_bits,
-	void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-	int toggle_bits, int error_count, int condition_code, int repeat_number,
-	 int halted, int skipped, int actual, int non_null))
-{
-	return usb_ftdi_elan_edset_output(u132->platform_dev, ring->number,
-		endp, urb, address, endp->usb_endp, toggle_bits, callback);
-}
-
-
-/*
-* must not LOCK sw_lock
-*
-*/
-static void u132_hcd_interrupt_recv(void *data, struct urb *urb, u8 *buf,
-	int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	u8 address = u132->addr[endp->usb_addr].address;
-	struct u132_udev *udev = &u132->udev[address];
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		struct u132_ring *ring = endp->ring;
-		u8 *u = urb->transfer_buffer + urb->actual_length;
-		u8 *b = buf;
-		int L = len;
-
-		while (L-- > 0)
-			*u++ = *b++;
-
-		urb->actual_length += len;
-		if ((condition_code == TD_CC_NOERROR) &&
-			(urb->transfer_buffer_length > urb->actual_length)) {
-			endp->toggle_bits = toggle_bits;
-			usb_settoggle(udev->usb_device, endp->usb_endp, 0,
-				1 & toggle_bits);
-			if (urb->actual_length > 0) {
-				int retval;
-				mutex_unlock(&u132->scheduler_lock);
-				retval = edset_single(u132, ring, endp, urb,
-					address, endp->toggle_bits,
-					u132_hcd_interrupt_recv);
-				if (retval != 0)
-					u132_hcd_giveback_urb(u132, endp, urb,
-						retval);
-			} else {
-				ring->in_use = 0;
-				endp->active = 0;
-				endp->jiffies = jiffies +
-					msecs_to_jiffies(urb->interval);
-				u132_ring_cancel_work(u132, ring);
-				u132_ring_queue_work(u132, ring, 0);
-				mutex_unlock(&u132->scheduler_lock);
-				u132_endp_put_kref(u132, endp);
-			}
-			return;
-		} else if ((condition_code == TD_DATAUNDERRUN) &&
-			((urb->transfer_flags & URB_SHORT_NOT_OK) == 0)) {
-			endp->toggle_bits = toggle_bits;
-			usb_settoggle(udev->usb_device, endp->usb_endp, 0,
-				1 & toggle_bits);
-			mutex_unlock(&u132->scheduler_lock);
-			u132_hcd_giveback_urb(u132, endp, urb, 0);
-			return;
-		} else {
-			if (condition_code == TD_CC_NOERROR) {
-				endp->toggle_bits = toggle_bits;
-				usb_settoggle(udev->usb_device, endp->usb_endp,
-					0, 1 & toggle_bits);
-			} else if (condition_code == TD_CC_STALL) {
-				endp->toggle_bits = 0x2;
-				usb_settoggle(udev->usb_device, endp->usb_endp,
-					0, 0);
-			} else {
-				endp->toggle_bits = 0x2;
-				usb_settoggle(udev->usb_device, endp->usb_endp,
-					0, 0);
-				dev_err(&u132->platform_dev->dev, "urb=%p givin"
-					"g back INTERRUPT %s\n", urb,
-					cc_to_text[condition_code]);
-			}
-			mutex_unlock(&u132->scheduler_lock);
-			u132_hcd_giveback_urb(u132, endp, urb,
-				cc_to_error[condition_code]);
-			return;
-		}
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_bulk_output_sent(void *data, struct urb *urb, u8 *buf,
-	int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	u8 address = u132->addr[endp->usb_addr].address;
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		struct u132_ring *ring = endp->ring;
-		urb->actual_length += len;
-		endp->toggle_bits = toggle_bits;
-		if (urb->transfer_buffer_length > urb->actual_length) {
-			int retval;
-			mutex_unlock(&u132->scheduler_lock);
-			retval = edset_output(u132, ring, endp, urb, address,
-				endp->toggle_bits, u132_hcd_bulk_output_sent);
-			if (retval != 0)
-				u132_hcd_giveback_urb(u132, endp, urb, retval);
-			return;
-		} else {
-			mutex_unlock(&u132->scheduler_lock);
-			u132_hcd_giveback_urb(u132, endp, urb, 0);
-			return;
-		}
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_bulk_input_recv(void *data, struct urb *urb, u8 *buf,
-	int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	u8 address = u132->addr[endp->usb_addr].address;
-	struct u132_udev *udev = &u132->udev[address];
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		struct u132_ring *ring = endp->ring;
-		u8 *u = urb->transfer_buffer + urb->actual_length;
-		u8 *b = buf;
-		int L = len;
-
-		while (L-- > 0)
-			*u++ = *b++;
-
-		urb->actual_length += len;
-		if ((condition_code == TD_CC_NOERROR) &&
-			(urb->transfer_buffer_length > urb->actual_length)) {
-			int retval;
-			endp->toggle_bits = toggle_bits;
-			usb_settoggle(udev->usb_device, endp->usb_endp, 0,
-				1 & toggle_bits);
-			mutex_unlock(&u132->scheduler_lock);
-			retval = usb_ftdi_elan_edset_input(u132->platform_dev,
-				ring->number, endp, urb, address,
-				endp->usb_endp, endp->toggle_bits,
-				u132_hcd_bulk_input_recv);
-			if (retval != 0)
-				u132_hcd_giveback_urb(u132, endp, urb, retval);
-			return;
-		} else if (condition_code == TD_CC_NOERROR) {
-			endp->toggle_bits = toggle_bits;
-			usb_settoggle(udev->usb_device, endp->usb_endp, 0,
-				1 & toggle_bits);
-			mutex_unlock(&u132->scheduler_lock);
-			u132_hcd_giveback_urb(u132, endp, urb,
-				cc_to_error[condition_code]);
-			return;
-		} else if ((condition_code == TD_DATAUNDERRUN) &&
-			((urb->transfer_flags & URB_SHORT_NOT_OK) == 0)) {
-			endp->toggle_bits = toggle_bits;
-			usb_settoggle(udev->usb_device, endp->usb_endp, 0,
-				1 & toggle_bits);
-			mutex_unlock(&u132->scheduler_lock);
-			u132_hcd_giveback_urb(u132, endp, urb, 0);
-			return;
-		} else if (condition_code == TD_DATAUNDERRUN) {
-			endp->toggle_bits = toggle_bits;
-			usb_settoggle(udev->usb_device, endp->usb_endp, 0,
-				1 & toggle_bits);
-			dev_warn(&u132->platform_dev->dev, "urb=%p(SHORT NOT OK"
-				") giving back BULK IN %s\n", urb,
-				cc_to_text[condition_code]);
-			mutex_unlock(&u132->scheduler_lock);
-			u132_hcd_giveback_urb(u132, endp, urb, 0);
-			return;
-		} else if (condition_code == TD_CC_STALL) {
-			endp->toggle_bits = 0x2;
-			usb_settoggle(udev->usb_device, endp->usb_endp, 0, 0);
-			mutex_unlock(&u132->scheduler_lock);
-			u132_hcd_giveback_urb(u132, endp, urb,
-				cc_to_error[condition_code]);
-			return;
-		} else {
-			endp->toggle_bits = 0x2;
-			usb_settoggle(udev->usb_device, endp->usb_endp, 0, 0);
-			dev_err(&u132->platform_dev->dev, "urb=%p giving back B"
-				"ULK IN code=%d %s\n", urb, condition_code,
-				cc_to_text[condition_code]);
-			mutex_unlock(&u132->scheduler_lock);
-			u132_hcd_giveback_urb(u132, endp, urb,
-				cc_to_error[condition_code]);
-			return;
-		}
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_configure_empty_sent(void *data, struct urb *urb, u8 *buf,
-	int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_configure_input_recv(void *data, struct urb *urb, u8 *buf,
-	int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	u8 address = u132->addr[endp->usb_addr].address;
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		struct u132_ring *ring = endp->ring;
-		u8 *u = urb->transfer_buffer;
-		u8 *b = buf;
-		int L = len;
-
-		while (L-- > 0)
-			*u++ = *b++;
-
-		urb->actual_length = len;
-		if ((condition_code == TD_CC_NOERROR) || ((condition_code ==
-			TD_DATAUNDERRUN) && ((urb->transfer_flags &
-			URB_SHORT_NOT_OK) == 0))) {
-			int retval;
-			mutex_unlock(&u132->scheduler_lock);
-			retval = usb_ftdi_elan_edset_empty(u132->platform_dev,
-				ring->number, endp, urb, address,
-				endp->usb_endp, 0x3,
-				u132_hcd_configure_empty_sent);
-			if (retval != 0)
-				u132_hcd_giveback_urb(u132, endp, urb, retval);
-			return;
-		} else if (condition_code == TD_CC_STALL) {
-			mutex_unlock(&u132->scheduler_lock);
-			dev_warn(&u132->platform_dev->dev, "giving back SETUP I"
-				"NPUT STALL urb %p\n", urb);
-			u132_hcd_giveback_urb(u132, endp, urb,
-				cc_to_error[condition_code]);
-			return;
-		} else {
-			mutex_unlock(&u132->scheduler_lock);
-			dev_err(&u132->platform_dev->dev, "giving back SETUP IN"
-				"PUT %s urb %p\n", cc_to_text[condition_code],
-				urb);
-			u132_hcd_giveback_urb(u132, endp, urb,
-				cc_to_error[condition_code]);
-			return;
-		}
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_configure_empty_recv(void *data, struct urb *urb, u8 *buf,
-	int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_configure_setup_sent(void *data, struct urb *urb, u8 *buf,
-	int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	u8 address = u132->addr[endp->usb_addr].address;
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		if (usb_pipein(urb->pipe)) {
-			int retval;
-			struct u132_ring *ring = endp->ring;
-			mutex_unlock(&u132->scheduler_lock);
-			retval = usb_ftdi_elan_edset_input(u132->platform_dev,
-				ring->number, endp, urb, address,
-				endp->usb_endp, 0,
-				u132_hcd_configure_input_recv);
-			if (retval != 0)
-				u132_hcd_giveback_urb(u132, endp, urb, retval);
-			return;
-		} else {
-			int retval;
-			struct u132_ring *ring = endp->ring;
-			mutex_unlock(&u132->scheduler_lock);
-			retval = usb_ftdi_elan_edset_input(u132->platform_dev,
-				ring->number, endp, urb, address,
-				endp->usb_endp, 0,
-				u132_hcd_configure_empty_recv);
-			if (retval != 0)
-				u132_hcd_giveback_urb(u132, endp, urb, retval);
-			return;
-		}
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_enumeration_empty_recv(void *data, struct urb *urb,
-	u8 *buf, int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	u8 address = u132->addr[endp->usb_addr].address;
-	struct u132_udev *udev = &u132->udev[address];
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		u132->addr[0].address = 0;
-		endp->usb_addr = udev->usb_addr;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_enumeration_address_sent(void *data, struct urb *urb,
-	u8 *buf, int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		int retval;
-		struct u132_ring *ring = endp->ring;
-		mutex_unlock(&u132->scheduler_lock);
-		retval = usb_ftdi_elan_edset_input(u132->platform_dev,
-			ring->number, endp, urb, 0, endp->usb_endp, 0,
-			u132_hcd_enumeration_empty_recv);
-		if (retval != 0)
-			u132_hcd_giveback_urb(u132, endp, urb, retval);
-		return;
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_initial_empty_sent(void *data, struct urb *urb, u8 *buf,
-	int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_initial_input_recv(void *data, struct urb *urb, u8 *buf,
-	int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	u8 address = u132->addr[endp->usb_addr].address;
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		int retval;
-		struct u132_ring *ring = endp->ring;
-		u8 *u = urb->transfer_buffer;
-		u8 *b = buf;
-		int L = len;
-
-		while (L-- > 0)
-			*u++ = *b++;
-
-		urb->actual_length = len;
-		mutex_unlock(&u132->scheduler_lock);
-		retval = usb_ftdi_elan_edset_empty(u132->platform_dev,
-			ring->number, endp, urb, address, endp->usb_endp, 0x3,
-			u132_hcd_initial_empty_sent);
-		if (retval != 0)
-			u132_hcd_giveback_urb(u132, endp, urb, retval);
-		return;
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-static void u132_hcd_initial_setup_sent(void *data, struct urb *urb, u8 *buf,
-	int len, int toggle_bits, int error_count, int condition_code,
-	int repeat_number, int halted, int skipped, int actual, int non_null)
-{
-	struct u132_endp *endp = data;
-	struct u132 *u132 = endp->u132;
-	u8 address = u132->addr[endp->usb_addr].address;
-	mutex_lock(&u132->scheduler_lock);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_forget_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (endp->dequeueing) {
-		endp->dequeueing = 0;
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -EINTR);
-		return;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, -ENODEV);
-		return;
-	} else if (!urb->unlinked) {
-		int retval;
-		struct u132_ring *ring = endp->ring;
-		mutex_unlock(&u132->scheduler_lock);
-		retval = usb_ftdi_elan_edset_input(u132->platform_dev,
-			ring->number, endp, urb, address, endp->usb_endp, 0,
-			u132_hcd_initial_input_recv);
-		if (retval != 0)
-			u132_hcd_giveback_urb(u132, endp, urb, retval);
-		return;
-	} else {
-		dev_err(&u132->platform_dev->dev, "CALLBACK called urb=%p "
-				"unlinked=%d\n", urb, urb->unlinked);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_hcd_giveback_urb(u132, endp, urb, 0);
-		return;
-	}
-}
-
-/*
-* this work function is only executed from the work queue
-*
-*/
-static void u132_hcd_ring_work_scheduler(struct work_struct *work)
-{
-	struct u132_ring *ring =
-		container_of(work, struct u132_ring, scheduler.work);
-	struct u132 *u132 = ring->u132;
-	mutex_lock(&u132->scheduler_lock);
-	if (ring->in_use) {
-		mutex_unlock(&u132->scheduler_lock);
-		u132_ring_put_kref(u132, ring);
-		return;
-	} else if (ring->curr_endp) {
-		struct u132_endp *endp, *last_endp = ring->curr_endp;
-		unsigned long wakeup = 0;
-		list_for_each_entry(endp, &last_endp->endp_ring, endp_ring) {
-			if (endp->queue_next == endp->queue_last) {
-			} else if ((endp->delayed == 0)
-				|| time_after_eq(jiffies, endp->jiffies)) {
-				ring->curr_endp = endp;
-				u132_endp_cancel_work(u132, last_endp);
-				u132_endp_queue_work(u132, last_endp, 0);
-				mutex_unlock(&u132->scheduler_lock);
-				u132_ring_put_kref(u132, ring);
-				return;
-			} else {
-				unsigned long delta = endp->jiffies - jiffies;
-				if (delta > wakeup)
-					wakeup = delta;
-			}
-		}
-		if (last_endp->queue_next == last_endp->queue_last) {
-		} else if ((last_endp->delayed == 0) || time_after_eq(jiffies,
-			last_endp->jiffies)) {
-			u132_endp_cancel_work(u132, last_endp);
-			u132_endp_queue_work(u132, last_endp, 0);
-			mutex_unlock(&u132->scheduler_lock);
-			u132_ring_put_kref(u132, ring);
-			return;
-		} else {
-			unsigned long delta = last_endp->jiffies - jiffies;
-			if (delta > wakeup)
-				wakeup = delta;
-		}
-		if (wakeup > 0) {
-			u132_ring_requeue_work(u132, ring, wakeup);
-			mutex_unlock(&u132->scheduler_lock);
-			return;
-		} else {
-			mutex_unlock(&u132->scheduler_lock);
-			u132_ring_put_kref(u132, ring);
-			return;
-		}
-	} else {
-		mutex_unlock(&u132->scheduler_lock);
-		u132_ring_put_kref(u132, ring);
-		return;
-	}
-}
-
-static void u132_hcd_endp_work_scheduler(struct work_struct *work)
-{
-	struct u132_ring *ring;
-	struct u132_endp *endp =
-		container_of(work, struct u132_endp, scheduler.work);
-	struct u132 *u132 = endp->u132;
-	mutex_lock(&u132->scheduler_lock);
-	ring = endp->ring;
-	if (endp->edset_flush) {
-		endp->edset_flush = 0;
-		if (endp->dequeueing)
-			usb_ftdi_elan_edset_flush(u132->platform_dev,
-				ring->number, endp);
-		mutex_unlock(&u132->scheduler_lock);
-		u132_endp_put_kref(u132, endp);
-		return;
-	} else if (endp->active) {
-		mutex_unlock(&u132->scheduler_lock);
-		u132_endp_put_kref(u132, endp);
-		return;
-	} else if (ring->in_use) {
-		mutex_unlock(&u132->scheduler_lock);
-		u132_endp_put_kref(u132, endp);
-		return;
-	} else if (endp->queue_next == endp->queue_last) {
-		mutex_unlock(&u132->scheduler_lock);
-		u132_endp_put_kref(u132, endp);
-		return;
-	} else if (endp->pipetype == PIPE_INTERRUPT) {
-		u8 address = u132->addr[endp->usb_addr].address;
-		if (ring->in_use) {
-			mutex_unlock(&u132->scheduler_lock);
-			u132_endp_put_kref(u132, endp);
-			return;
-		} else {
-			int retval;
-			struct urb *urb = endp->urb_list[ENDP_QUEUE_MASK &
-				endp->queue_next];
-			endp->active = 1;
-			ring->curr_endp = endp;
-			ring->in_use = 1;
-			mutex_unlock(&u132->scheduler_lock);
-			retval = edset_single(u132, ring, endp, urb, address,
-				endp->toggle_bits, u132_hcd_interrupt_recv);
-			if (retval != 0)
-				u132_hcd_giveback_urb(u132, endp, urb, retval);
-			return;
-		}
-	} else if (endp->pipetype == PIPE_CONTROL) {
-		u8 address = u132->addr[endp->usb_addr].address;
-		if (ring->in_use) {
-			mutex_unlock(&u132->scheduler_lock);
-			u132_endp_put_kref(u132, endp);
-			return;
-		} else if (address == 0) {
-			int retval;
-			struct urb *urb = endp->urb_list[ENDP_QUEUE_MASK &
-				endp->queue_next];
-			endp->active = 1;
-			ring->curr_endp = endp;
-			ring->in_use = 1;
-			mutex_unlock(&u132->scheduler_lock);
-			retval = edset_setup(u132, ring, endp, urb, address,
-				0x2, u132_hcd_initial_setup_sent);
-			if (retval != 0)
-				u132_hcd_giveback_urb(u132, endp, urb, retval);
-			return;
-		} else if (endp->usb_addr == 0) {
-			int retval;
-			struct urb *urb = endp->urb_list[ENDP_QUEUE_MASK &
-				endp->queue_next];
-			endp->active = 1;
-			ring->curr_endp = endp;
-			ring->in_use = 1;
-			mutex_unlock(&u132->scheduler_lock);
-			retval = edset_setup(u132, ring, endp, urb, 0, 0x2,
-				u132_hcd_enumeration_address_sent);
-			if (retval != 0)
-				u132_hcd_giveback_urb(u132, endp, urb, retval);
-			return;
-		} else {
-			int retval;
-			struct urb *urb = endp->urb_list[ENDP_QUEUE_MASK &
-				endp->queue_next];
-			address = u132->addr[endp->usb_addr].address;
-			endp->active = 1;
-			ring->curr_endp = endp;
-			ring->in_use = 1;
-			mutex_unlock(&u132->scheduler_lock);
-			retval = edset_setup(u132, ring, endp, urb, address,
-				0x2, u132_hcd_configure_setup_sent);
-			if (retval != 0)
-				u132_hcd_giveback_urb(u132, endp, urb, retval);
-			return;
-		}
-	} else {
-		if (endp->input) {
-			u8 address = u132->addr[endp->usb_addr].address;
-			if (ring->in_use) {
-				mutex_unlock(&u132->scheduler_lock);
-				u132_endp_put_kref(u132, endp);
-				return;
-			} else {
-				int retval;
-				struct urb *urb = endp->urb_list[
-					ENDP_QUEUE_MASK & endp->queue_next];
-				endp->active = 1;
-				ring->curr_endp = endp;
-				ring->in_use = 1;
-				mutex_unlock(&u132->scheduler_lock);
-				retval = edset_input(u132, ring, endp, urb,
-					address, endp->toggle_bits,
-					u132_hcd_bulk_input_recv);
-				if (retval == 0) {
-				} else
-					u132_hcd_giveback_urb(u132, endp, urb,
-						retval);
-				return;
-			}
-		} else {	/* output pipe */
-			u8 address = u132->addr[endp->usb_addr].address;
-			if (ring->in_use) {
-				mutex_unlock(&u132->scheduler_lock);
-				u132_endp_put_kref(u132, endp);
-				return;
-			} else {
-				int retval;
-				struct urb *urb = endp->urb_list[
-					ENDP_QUEUE_MASK & endp->queue_next];
-				endp->active = 1;
-				ring->curr_endp = endp;
-				ring->in_use = 1;
-				mutex_unlock(&u132->scheduler_lock);
-				retval = edset_output(u132, ring, endp, urb,
-					address, endp->toggle_bits,
-					u132_hcd_bulk_output_sent);
-				if (retval == 0) {
-				} else
-					u132_hcd_giveback_urb(u132, endp, urb,
-						retval);
-				return;
-			}
-		}
-	}
-}
-#ifdef CONFIG_PM
-
-static void port_power(struct u132 *u132, int pn, int is_on)
-{
-	u132->port[pn].power = is_on;
-}
-
-#endif
-
-static void u132_power(struct u132 *u132, int is_on)
-{
-	struct usb_hcd *hcd = u132_to_hcd(u132)
-		;	/* hub is inactive unless the port is powered */
-	if (is_on) {
-		if (u132->power)
-			return;
-		u132->power = 1;
-	} else {
-		u132->power = 0;
-		hcd->state = HC_STATE_HALT;
-	}
-}
-
-static int u132_periodic_reinit(struct u132 *u132)
-{
-	int retval;
-	u32 fi = u132->hc_fminterval & 0x03fff;
-	u32 fit;
-	u32 fminterval;
-	retval = u132_read_pcimem(u132, fminterval, &fminterval);
-	if (retval)
-		return retval;
-	fit = fminterval & FIT;
-	retval = u132_write_pcimem(u132, fminterval,
-		(fit ^ FIT) | u132->hc_fminterval);
-	if (retval)
-		return retval;
-	return u132_write_pcimem(u132, periodicstart,
-	       ((9 * fi) / 10) & 0x3fff);
-}
-
-static char *hcfs2string(int state)
-{
-	switch (state) {
-	case OHCI_USB_RESET:
-		return "reset";
-	case OHCI_USB_RESUME:
-		return "resume";
-	case OHCI_USB_OPER:
-		return "operational";
-	case OHCI_USB_SUSPEND:
-		return "suspend";
-	}
-	return "?";
-}
-
-static int u132_init(struct u132 *u132)
-{
-	int retval;
-	u32 control;
-	u132_disable(u132);
-	u132->next_statechange = jiffies;
-	retval = u132_write_pcimem(u132, intrdisable, OHCI_INTR_MIE);
-	if (retval)
-		return retval;
-	retval = u132_read_pcimem(u132, control, &control);
-	if (retval)
-		return retval;
-	if (u132->num_ports == 0) {
-		u32 rh_a = -1;
-		retval = u132_read_pcimem(u132, roothub.a, &rh_a);
-		if (retval)
-			return retval;
-		u132->num_ports = rh_a & RH_A_NDP;
-		retval = read_roothub_info(u132);
-		if (retval)
-			return retval;
-	}
-	if (u132->num_ports > MAX_U132_PORTS)
-		return -EINVAL;
-
-	return 0;
-}
-
-
-/* Start an OHCI controller, set the BUS operational
-* resets USB and controller
-* enable interrupts
-*/
-static int u132_run(struct u132 *u132)
-{
-	int retval;
-	u32 control;
-	u32 status;
-	u32 fminterval;
-	u32 periodicstart;
-	u32 cmdstatus;
-	u32 roothub_a;
-	int mask = OHCI_INTR_INIT;
-	int first = u132->hc_fminterval == 0;
-	int sleep_time = 0;
-	int reset_timeout = 30;	/* ... allow extra time */
-	u132_disable(u132);
-	if (first) {
-		u32 temp;
-		retval = u132_read_pcimem(u132, fminterval, &temp);
-		if (retval)
-			return retval;
-		u132->hc_fminterval = temp & 0x3fff;
-		u132->hc_fminterval |= FSMP(u132->hc_fminterval) << 16;
-	}
-	retval = u132_read_pcimem(u132, control, &u132->hc_control);
-	if (retval)
-		return retval;
-	dev_info(&u132->platform_dev->dev, "resetting from state '%s', control "
-		"= %08X\n", hcfs2string(u132->hc_control & OHCI_CTRL_HCFS),
-		u132->hc_control);
-	switch (u132->hc_control & OHCI_CTRL_HCFS) {
-	case OHCI_USB_OPER:
-		sleep_time = 0;
-		break;
-	case OHCI_USB_SUSPEND:
-	case OHCI_USB_RESUME:
-		u132->hc_control &= OHCI_CTRL_RWC;
-		u132->hc_control |= OHCI_USB_RESUME;
-		sleep_time = 10;
-		break;
-	default:
-		u132->hc_control &= OHCI_CTRL_RWC;
-		u132->hc_control |= OHCI_USB_RESET;
-		sleep_time = 50;
-		break;
-	}
-	retval = u132_write_pcimem(u132, control, u132->hc_control);
-	if (retval)
-		return retval;
-	retval = u132_read_pcimem(u132, control, &control);
-	if (retval)
-		return retval;
-	msleep(sleep_time);
-	retval = u132_read_pcimem(u132, roothub.a, &roothub_a);
-	if (retval)
-		return retval;
-	if (!(roothub_a & RH_A_NPS)) {
-		int temp;	/* power down each port */
-		for (temp = 0; temp < u132->num_ports; temp++) {
-			retval = u132_write_pcimem(u132,
-				roothub.portstatus[temp], RH_PS_LSDA);
-			if (retval)
-				return retval;
-		}
-	}
-	retval = u132_read_pcimem(u132, control, &control);
-	if (retval)
-		return retval;
-retry:
-	retval = u132_read_pcimem(u132, cmdstatus, &status);
-	if (retval)
-		return retval;
-	retval = u132_write_pcimem(u132, cmdstatus, OHCI_HCR);
-	if (retval)
-		return retval;
-extra:	{
-		retval = u132_read_pcimem(u132, cmdstatus, &status);
-		if (retval)
-			return retval;
-		if (0 != (status & OHCI_HCR)) {
-			if (--reset_timeout == 0) {
-				dev_err(&u132->platform_dev->dev, "USB HC reset"
-					" timed out!\n");
-				return -ENODEV;
-			} else {
-				msleep(5);
-				goto extra;
-			}
-		}
-	}
-	if (u132->flags & OHCI_QUIRK_INITRESET) {
-		retval = u132_write_pcimem(u132, control, u132->hc_control);
-		if (retval)
-			return retval;
-		retval = u132_read_pcimem(u132, control, &control);
-		if (retval)
-			return retval;
-	}
-	retval = u132_write_pcimem(u132, ed_controlhead, 0x00000000);
-	if (retval)
-		return retval;
-	retval = u132_write_pcimem(u132, ed_bulkhead, 0x11000000);
-	if (retval)
-		return retval;
-	retval = u132_write_pcimem(u132, hcca, 0x00000000);
-	if (retval)
-		return retval;
-	retval = u132_periodic_reinit(u132);
-	if (retval)
-		return retval;
-	retval = u132_read_pcimem(u132, fminterval, &fminterval);
-	if (retval)
-		return retval;
-	retval = u132_read_pcimem(u132, periodicstart, &periodicstart);
-	if (retval)
-		return retval;
-	if (0 == (fminterval & 0x3fff0000) || 0 == periodicstart) {
-		if (!(u132->flags & OHCI_QUIRK_INITRESET)) {
-			u132->flags |= OHCI_QUIRK_INITRESET;
-			goto retry;
-		} else
-			dev_err(&u132->platform_dev->dev, "init err(%08x %04x)"
-				"\n", fminterval, periodicstart);
-	}			/* start controller operations */
-	u132->hc_control &= OHCI_CTRL_RWC;
-	u132->hc_control |= OHCI_CONTROL_INIT | OHCI_CTRL_BLE | OHCI_USB_OPER;
-	retval = u132_write_pcimem(u132, control, u132->hc_control);
-	if (retval)
-		return retval;
-	retval = u132_write_pcimem(u132, cmdstatus, OHCI_BLF);
-	if (retval)
-		return retval;
-	retval = u132_read_pcimem(u132, cmdstatus, &cmdstatus);
-	if (retval)
-		return retval;
-	retval = u132_read_pcimem(u132, control, &control);
-	if (retval)
-		return retval;
-	u132_to_hcd(u132)->state = HC_STATE_RUNNING;
-	retval = u132_write_pcimem(u132, roothub.status, RH_HS_DRWE);
-	if (retval)
-		return retval;
-	retval = u132_write_pcimem(u132, intrstatus, mask);
-	if (retval)
-		return retval;
-	retval = u132_write_pcimem(u132, intrdisable,
-		OHCI_INTR_MIE | OHCI_INTR_OC | OHCI_INTR_RHSC | OHCI_INTR_FNO |
-		OHCI_INTR_UE | OHCI_INTR_RD | OHCI_INTR_SF | OHCI_INTR_WDH |
-		OHCI_INTR_SO);
-	if (retval)
-		return retval;	/* handle root hub init quirks ... */
-	retval = u132_read_pcimem(u132, roothub.a, &roothub_a);
-	if (retval)
-		return retval;
-	roothub_a &= ~(RH_A_PSM | RH_A_OCPM);
-	if (u132->flags & OHCI_QUIRK_SUPERIO) {
-		roothub_a |= RH_A_NOCP;
-		roothub_a &= ~(RH_A_POTPGT | RH_A_NPS);
-		retval = u132_write_pcimem(u132, roothub.a, roothub_a);
-		if (retval)
-			return retval;
-	} else if ((u132->flags & OHCI_QUIRK_AMD756) || distrust_firmware) {
-		roothub_a |= RH_A_NPS;
-		retval = u132_write_pcimem(u132, roothub.a, roothub_a);
-		if (retval)
-			return retval;
-	}
-	retval = u132_write_pcimem(u132, roothub.status, RH_HS_LPSC);
-	if (retval)
-		return retval;
-	retval = u132_write_pcimem(u132, roothub.b,
-		(roothub_a & RH_A_NPS) ? 0 : RH_B_PPCM);
-	if (retval)
-		return retval;
-	retval = u132_read_pcimem(u132, control, &control);
-	if (retval)
-		return retval;
-	mdelay((roothub_a >> 23) & 0x1fe);
-	u132_to_hcd(u132)->state = HC_STATE_RUNNING;
-	return 0;
-}
-
-static void u132_hcd_stop(struct usb_hcd *hcd)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "u132 device %p(hcd=%p) has b"
-			"een removed %d\n", u132, hcd, u132->going);
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device hcd=%p is being remov"
-			"ed\n", hcd);
-	} else {
-		mutex_lock(&u132->sw_lock);
-		msleep(100);
-		u132_power(u132, 0);
-		mutex_unlock(&u132->sw_lock);
-	}
-}
-
-static int u132_hcd_start(struct usb_hcd *hcd)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed\n");
-		return -ESHUTDOWN;
-	} else if (hcd->self.controller) {
-		int retval;
-		struct platform_device *pdev =
-			to_platform_device(hcd->self.controller);
-		u16 vendor = ((struct u132_platform_data *)
-			dev_get_platdata(&pdev->dev))->vendor;
-		u16 device = ((struct u132_platform_data *)
-			dev_get_platdata(&pdev->dev))->device;
-		mutex_lock(&u132->sw_lock);
-		msleep(10);
-		if (vendor == PCI_VENDOR_ID_AMD && device == 0x740c) {
-			u132->flags = OHCI_QUIRK_AMD756;
-		} else if (vendor == PCI_VENDOR_ID_OPTI && device == 0xc861) {
-			dev_err(&u132->platform_dev->dev, "WARNING: OPTi workar"
-				"ounds unavailable\n");
-		} else if (vendor == PCI_VENDOR_ID_COMPAQ && device == 0xa0f8)
-			u132->flags |= OHCI_QUIRK_ZFMICRO;
-		retval = u132_run(u132);
-		if (retval) {
-			u132_disable(u132);
-			u132->going = 1;
-		}
-		msleep(100);
-		mutex_unlock(&u132->sw_lock);
-		return retval;
-	} else {
-		dev_err(&u132->platform_dev->dev, "platform_device missing\n");
-		return -ENODEV;
-	}
-}
-
-static int u132_hcd_reset(struct usb_hcd *hcd)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed\n");
-		return -ESHUTDOWN;
-	} else {
-		int retval;
-		mutex_lock(&u132->sw_lock);
-		retval = u132_init(u132);
-		if (retval) {
-			u132_disable(u132);
-			u132->going = 1;
-		}
-		mutex_unlock(&u132->sw_lock);
-		return retval;
-	}
-}
-
-static int create_endpoint_and_queue_int(struct u132 *u132,
-	struct u132_udev *udev, struct urb *urb,
-	struct usb_device *usb_dev, u8 usb_addr, u8 usb_endp, u8 address,
-	gfp_t mem_flags)
-{
-	struct u132_ring *ring;
-	unsigned long irqs;
-	int rc;
-	u8 endp_number;
-	struct u132_endp *endp = kmalloc(sizeof(struct u132_endp), mem_flags);
-
-	if (!endp)
-		return -ENOMEM;
-
-	spin_lock_init(&endp->queue_lock.slock);
-	spin_lock_irqsave(&endp->queue_lock.slock, irqs);
-	rc = usb_hcd_link_urb_to_ep(u132_to_hcd(u132), urb);
-	if (rc) {
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-		kfree(endp);
-		return rc;
-	}
-
-	endp_number = ++u132->num_endpoints;
-	urb->ep->hcpriv = u132->endp[endp_number - 1] = endp;
-	INIT_DELAYED_WORK(&endp->scheduler, u132_hcd_endp_work_scheduler);
-	INIT_LIST_HEAD(&endp->urb_more);
-	ring = endp->ring = &u132->ring[0];
-	if (ring->curr_endp) {
-		list_add_tail(&endp->endp_ring, &ring->curr_endp->endp_ring);
-	} else {
-		INIT_LIST_HEAD(&endp->endp_ring);
-		ring->curr_endp = endp;
-	}
-	ring->length += 1;
-	endp->dequeueing = 0;
-	endp->edset_flush = 0;
-	endp->active = 0;
-	endp->delayed = 0;
-	endp->endp_number = endp_number;
-	endp->u132 = u132;
-	endp->hep = urb->ep;
-	endp->pipetype = usb_pipetype(urb->pipe);
-	u132_endp_init_kref(u132, endp);
-	if (usb_pipein(urb->pipe)) {
-		endp->toggle_bits = 0x2;
-		usb_settoggle(udev->usb_device, usb_endp, 0, 0);
-		endp->input = 1;
-		endp->output = 0;
-		udev->endp_number_in[usb_endp] = endp_number;
-		u132_udev_get_kref(u132, udev);
-	} else {
-		endp->toggle_bits = 0x2;
-		usb_settoggle(udev->usb_device, usb_endp, 1, 0);
-		endp->input = 0;
-		endp->output = 1;
-		udev->endp_number_out[usb_endp] = endp_number;
-		u132_udev_get_kref(u132, udev);
-	}
-	urb->hcpriv = u132;
-	endp->delayed = 1;
-	endp->jiffies = jiffies + msecs_to_jiffies(urb->interval);
-	endp->udev_number = address;
-	endp->usb_addr = usb_addr;
-	endp->usb_endp = usb_endp;
-	endp->queue_size = 1;
-	endp->queue_last = 0;
-	endp->queue_next = 0;
-	endp->urb_list[ENDP_QUEUE_MASK & endp->queue_last++] = urb;
-	spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-	u132_endp_queue_work(u132, endp, msecs_to_jiffies(urb->interval));
-	return 0;
-}
-
-static int queue_int_on_old_endpoint(struct u132 *u132,
-	struct u132_udev *udev, struct urb *urb,
-	struct usb_device *usb_dev, struct u132_endp *endp, u8 usb_addr,
-	u8 usb_endp, u8 address)
-{
-	urb->hcpriv = u132;
-	endp->delayed = 1;
-	endp->jiffies = jiffies + msecs_to_jiffies(urb->interval);
-	if (endp->queue_size++ < ENDP_QUEUE_SIZE) {
-		endp->urb_list[ENDP_QUEUE_MASK & endp->queue_last++] = urb;
-	} else {
-		struct u132_urbq *urbq = kmalloc(sizeof(struct u132_urbq),
-			GFP_ATOMIC);
-		if (urbq == NULL) {
-			endp->queue_size -= 1;
-			return -ENOMEM;
-		} else {
-			list_add_tail(&urbq->urb_more, &endp->urb_more);
-			urbq->urb = urb;
-		}
-	}
-	return 0;
-}
-
-static int create_endpoint_and_queue_bulk(struct u132 *u132,
-	struct u132_udev *udev, struct urb *urb,
-	struct usb_device *usb_dev, u8 usb_addr, u8 usb_endp, u8 address,
-	gfp_t mem_flags)
-{
-	int ring_number;
-	struct u132_ring *ring;
-	unsigned long irqs;
-	int rc;
-	u8 endp_number;
-	struct u132_endp *endp = kmalloc(sizeof(struct u132_endp), mem_flags);
-
-	if (!endp)
-		return -ENOMEM;
-
-	spin_lock_init(&endp->queue_lock.slock);
-	spin_lock_irqsave(&endp->queue_lock.slock, irqs);
-	rc = usb_hcd_link_urb_to_ep(u132_to_hcd(u132), urb);
-	if (rc) {
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-		kfree(endp);
-		return rc;
-	}
-
-	endp_number = ++u132->num_endpoints;
-	urb->ep->hcpriv = u132->endp[endp_number - 1] = endp;
-	INIT_DELAYED_WORK(&endp->scheduler, u132_hcd_endp_work_scheduler);
-	INIT_LIST_HEAD(&endp->urb_more);
-	endp->dequeueing = 0;
-	endp->edset_flush = 0;
-	endp->active = 0;
-	endp->delayed = 0;
-	endp->endp_number = endp_number;
-	endp->u132 = u132;
-	endp->hep = urb->ep;
-	endp->pipetype = usb_pipetype(urb->pipe);
-	u132_endp_init_kref(u132, endp);
-	if (usb_pipein(urb->pipe)) {
-		endp->toggle_bits = 0x2;
-		usb_settoggle(udev->usb_device, usb_endp, 0, 0);
-		ring_number = 3;
-		endp->input = 1;
-		endp->output = 0;
-		udev->endp_number_in[usb_endp] = endp_number;
-		u132_udev_get_kref(u132, udev);
-	} else {
-		endp->toggle_bits = 0x2;
-		usb_settoggle(udev->usb_device, usb_endp, 1, 0);
-		ring_number = 2;
-		endp->input = 0;
-		endp->output = 1;
-		udev->endp_number_out[usb_endp] = endp_number;
-		u132_udev_get_kref(u132, udev);
-	}
-	ring = endp->ring = &u132->ring[ring_number - 1];
-	if (ring->curr_endp) {
-		list_add_tail(&endp->endp_ring, &ring->curr_endp->endp_ring);
-	} else {
-		INIT_LIST_HEAD(&endp->endp_ring);
-		ring->curr_endp = endp;
-	}
-	ring->length += 1;
-	urb->hcpriv = u132;
-	endp->udev_number = address;
-	endp->usb_addr = usb_addr;
-	endp->usb_endp = usb_endp;
-	endp->queue_size = 1;
-	endp->queue_last = 0;
-	endp->queue_next = 0;
-	endp->urb_list[ENDP_QUEUE_MASK & endp->queue_last++] = urb;
-	spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-	u132_endp_queue_work(u132, endp, 0);
-	return 0;
-}
-
-static int queue_bulk_on_old_endpoint(struct u132 *u132, struct u132_udev *udev,
-	struct urb *urb,
-	struct usb_device *usb_dev, struct u132_endp *endp, u8 usb_addr,
-	u8 usb_endp, u8 address)
-{
-	urb->hcpriv = u132;
-	if (endp->queue_size++ < ENDP_QUEUE_SIZE) {
-		endp->urb_list[ENDP_QUEUE_MASK & endp->queue_last++] = urb;
-	} else {
-		struct u132_urbq *urbq = kmalloc(sizeof(struct u132_urbq),
-			GFP_ATOMIC);
-		if (urbq == NULL) {
-			endp->queue_size -= 1;
-			return -ENOMEM;
-		} else {
-			list_add_tail(&urbq->urb_more, &endp->urb_more);
-			urbq->urb = urb;
-		}
-	}
-	return 0;
-}
-
-static int create_endpoint_and_queue_control(struct u132 *u132,
-	struct urb *urb,
-	struct usb_device *usb_dev, u8 usb_addr, u8 usb_endp,
-	gfp_t mem_flags)
-{
-	struct u132_ring *ring;
-	unsigned long irqs;
-	int rc;
-	u8 endp_number;
-	struct u132_endp *endp = kmalloc(sizeof(struct u132_endp), mem_flags);
-
-	if (!endp)
-		return -ENOMEM;
-
-	spin_lock_init(&endp->queue_lock.slock);
-	spin_lock_irqsave(&endp->queue_lock.slock, irqs);
-	rc = usb_hcd_link_urb_to_ep(u132_to_hcd(u132), urb);
-	if (rc) {
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-		kfree(endp);
-		return rc;
-	}
-
-	endp_number = ++u132->num_endpoints;
-	urb->ep->hcpriv = u132->endp[endp_number - 1] = endp;
-	INIT_DELAYED_WORK(&endp->scheduler, u132_hcd_endp_work_scheduler);
-	INIT_LIST_HEAD(&endp->urb_more);
-	ring = endp->ring = &u132->ring[0];
-	if (ring->curr_endp) {
-		list_add_tail(&endp->endp_ring, &ring->curr_endp->endp_ring);
-	} else {
-		INIT_LIST_HEAD(&endp->endp_ring);
-		ring->curr_endp = endp;
-	}
-	ring->length += 1;
-	endp->dequeueing = 0;
-	endp->edset_flush = 0;
-	endp->active = 0;
-	endp->delayed = 0;
-	endp->endp_number = endp_number;
-	endp->u132 = u132;
-	endp->hep = urb->ep;
-	u132_endp_init_kref(u132, endp);
-	u132_endp_get_kref(u132, endp);
-	if (usb_addr == 0) {
-		u8 address = u132->addr[usb_addr].address;
-		struct u132_udev *udev = &u132->udev[address];
-		endp->udev_number = address;
-		endp->usb_addr = usb_addr;
-		endp->usb_endp = usb_endp;
-		endp->input = 1;
-		endp->output = 1;
-		endp->pipetype = usb_pipetype(urb->pipe);
-		u132_udev_init_kref(u132, udev);
-		u132_udev_get_kref(u132, udev);
-		udev->endp_number_in[usb_endp] = endp_number;
-		udev->endp_number_out[usb_endp] = endp_number;
-		urb->hcpriv = u132;
-		endp->queue_size = 1;
-		endp->queue_last = 0;
-		endp->queue_next = 0;
-		endp->urb_list[ENDP_QUEUE_MASK & endp->queue_last++] = urb;
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-		u132_endp_queue_work(u132, endp, 0);
-		return 0;
-	} else {		/*(usb_addr > 0) */
-		u8 address = u132->addr[usb_addr].address;
-		struct u132_udev *udev = &u132->udev[address];
-		endp->udev_number = address;
-		endp->usb_addr = usb_addr;
-		endp->usb_endp = usb_endp;
-		endp->input = 1;
-		endp->output = 1;
-		endp->pipetype = usb_pipetype(urb->pipe);
-		u132_udev_get_kref(u132, udev);
-		udev->enumeration = 2;
-		udev->endp_number_in[usb_endp] = endp_number;
-		udev->endp_number_out[usb_endp] = endp_number;
-		urb->hcpriv = u132;
-		endp->queue_size = 1;
-		endp->queue_last = 0;
-		endp->queue_next = 0;
-		endp->urb_list[ENDP_QUEUE_MASK & endp->queue_last++] = urb;
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-		u132_endp_queue_work(u132, endp, 0);
-		return 0;
-	}
-}
-
-static int queue_control_on_old_endpoint(struct u132 *u132,
-	struct urb *urb,
-	struct usb_device *usb_dev, struct u132_endp *endp, u8 usb_addr,
-	u8 usb_endp)
-{
-	if (usb_addr == 0) {
-		if (usb_pipein(urb->pipe)) {
-			urb->hcpriv = u132;
-			if (endp->queue_size++ < ENDP_QUEUE_SIZE) {
-				endp->urb_list[ENDP_QUEUE_MASK &
-					endp->queue_last++] = urb;
-			} else {
-				struct u132_urbq *urbq =
-					kmalloc(sizeof(struct u132_urbq),
-					GFP_ATOMIC);
-				if (urbq == NULL) {
-					endp->queue_size -= 1;
-					return -ENOMEM;
-				} else {
-					list_add_tail(&urbq->urb_more,
-						&endp->urb_more);
-					urbq->urb = urb;
-				}
-			}
-			return 0;
-		} else {	/* usb_pipeout(urb->pipe) */
-			struct u132_addr *addr = &u132->addr[usb_dev->devnum];
-			int I = MAX_U132_UDEVS;
-			int i = 0;
-			while (--I > 0) {
-				struct u132_udev *udev = &u132->udev[++i];
-				if (udev->usb_device) {
-					continue;
-				} else {
-					udev->enumeration = 1;
-					u132->addr[0].address = i;
-					endp->udev_number = i;
-					udev->udev_number = i;
-					udev->usb_addr = usb_dev->devnum;
-					u132_udev_init_kref(u132, udev);
-					udev->endp_number_in[usb_endp] =
-						endp->endp_number;
-					u132_udev_get_kref(u132, udev);
-					udev->endp_number_out[usb_endp] =
-						endp->endp_number;
-					udev->usb_device = usb_dev;
-					((u8 *) (urb->setup_packet))[2] =
-						addr->address = i;
-					u132_udev_get_kref(u132, udev);
-					break;
-				}
-			}
-			if (I == 0) {
-				dev_err(&u132->platform_dev->dev, "run out of d"
-					"evice space\n");
-				return -EINVAL;
-			}
-			urb->hcpriv = u132;
-			if (endp->queue_size++ < ENDP_QUEUE_SIZE) {
-				endp->urb_list[ENDP_QUEUE_MASK &
-					endp->queue_last++] = urb;
-			} else {
-				struct u132_urbq *urbq =
-					kmalloc(sizeof(struct u132_urbq),
-					GFP_ATOMIC);
-				if (urbq == NULL) {
-					endp->queue_size -= 1;
-					return -ENOMEM;
-				} else {
-					list_add_tail(&urbq->urb_more,
-						&endp->urb_more);
-					urbq->urb = urb;
-				}
-			}
-			return 0;
-		}
-	} else {		/*(usb_addr > 0) */
-		u8 address = u132->addr[usb_addr].address;
-		struct u132_udev *udev = &u132->udev[address];
-		urb->hcpriv = u132;
-		if (udev->enumeration != 2)
-			udev->enumeration = 2;
-		if (endp->queue_size++ < ENDP_QUEUE_SIZE) {
-			endp->urb_list[ENDP_QUEUE_MASK & endp->queue_last++] =
-				urb;
-		} else {
-			struct u132_urbq *urbq =
-				kmalloc(sizeof(struct u132_urbq), GFP_ATOMIC);
-			if (urbq == NULL) {
-				endp->queue_size -= 1;
-				return -ENOMEM;
-			} else {
-				list_add_tail(&urbq->urb_more, &endp->urb_more);
-				urbq->urb = urb;
-			}
-		}
-		return 0;
-	}
-}
-
-static int u132_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
-		gfp_t mem_flags)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (irqs_disabled()) {
-		if (gfpflags_allow_blocking(mem_flags)) {
-			printk(KERN_ERR "invalid context for function that might sleep\n");
-			return -EINVAL;
-		}
-	}
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed "
-				"urb=%p\n", urb);
-		return -ESHUTDOWN;
-	} else {
-		u8 usb_addr = usb_pipedevice(urb->pipe);
-		u8 usb_endp = usb_pipeendpoint(urb->pipe);
-		struct usb_device *usb_dev = urb->dev;
-		if (usb_pipetype(urb->pipe) == PIPE_INTERRUPT) {
-			u8 address = u132->addr[usb_addr].address;
-			struct u132_udev *udev = &u132->udev[address];
-			struct u132_endp *endp = urb->ep->hcpriv;
-			urb->actual_length = 0;
-			if (endp) {
-				unsigned long irqs;
-				int retval;
-				spin_lock_irqsave(&endp->queue_lock.slock,
-					irqs);
-				retval = usb_hcd_link_urb_to_ep(hcd, urb);
-				if (retval == 0) {
-					retval = queue_int_on_old_endpoint(
-							u132, udev, urb,
-							usb_dev, endp,
-							usb_addr, usb_endp,
-							address);
-					if (retval)
-						usb_hcd_unlink_urb_from_ep(
-	hcd, urb);
-				}
-				spin_unlock_irqrestore(&endp->queue_lock.slock,
-					irqs);
-				if (retval) {
-					return retval;
-				} else {
-					u132_endp_queue_work(u132, endp,
-						msecs_to_jiffies(urb->interval))
-						;
-					return 0;
-				}
-			} else if (u132->num_endpoints == MAX_U132_ENDPS) {
-				return -EINVAL;
-			} else {	/*(endp == NULL) */
-				return create_endpoint_and_queue_int(u132, udev,
-						urb, usb_dev, usb_addr,
-						usb_endp, address, mem_flags);
-			}
-		} else if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
-			dev_err(&u132->platform_dev->dev, "the hardware does no"
-				"t support PIPE_ISOCHRONOUS\n");
-			return -EINVAL;
-		} else if (usb_pipetype(urb->pipe) == PIPE_BULK) {
-			u8 address = u132->addr[usb_addr].address;
-			struct u132_udev *udev = &u132->udev[address];
-			struct u132_endp *endp = urb->ep->hcpriv;
-			urb->actual_length = 0;
-			if (endp) {
-				unsigned long irqs;
-				int retval;
-				spin_lock_irqsave(&endp->queue_lock.slock,
-					irqs);
-				retval = usb_hcd_link_urb_to_ep(hcd, urb);
-				if (retval == 0) {
-					retval = queue_bulk_on_old_endpoint(
-							u132, udev, urb,
-							usb_dev, endp,
-							usb_addr, usb_endp,
-							address);
-					if (retval)
-						usb_hcd_unlink_urb_from_ep(
-	hcd, urb);
-				}
-				spin_unlock_irqrestore(&endp->queue_lock.slock,
-					irqs);
-				if (retval) {
-					return retval;
-				} else {
-					u132_endp_queue_work(u132, endp, 0);
-					return 0;
-				}
-			} else if (u132->num_endpoints == MAX_U132_ENDPS) {
-				return -EINVAL;
-			} else
-				return create_endpoint_and_queue_bulk(u132,
-					udev, urb, usb_dev, usb_addr,
-					usb_endp, address, mem_flags);
-		} else {
-			struct u132_endp *endp = urb->ep->hcpriv;
-			u16 urb_size = 8;
-			u8 *b = urb->setup_packet;
-			int i = 0;
-			char data[30 * 3 + 4];
-			char *d = data;
-			int m = (sizeof(data) - 1) / 3;
-			int l = 0;
-			data[0] = 0;
-			while (urb_size-- > 0) {
-				if (i > m) {
-				} else if (i++ < m) {
-					int w = sprintf(d, " %02X", *b++);
-					d += w;
-					l += w;
-				} else
-					d += sprintf(d, " ..");
-			}
-			if (endp) {
-				unsigned long irqs;
-				int retval;
-				spin_lock_irqsave(&endp->queue_lock.slock,
-					irqs);
-				retval = usb_hcd_link_urb_to_ep(hcd, urb);
-				if (retval == 0) {
-					retval = queue_control_on_old_endpoint(
-							u132, urb, usb_dev,
-							endp, usb_addr,
-							usb_endp);
-					if (retval)
-						usb_hcd_unlink_urb_from_ep(
-								hcd, urb);
-				}
-				spin_unlock_irqrestore(&endp->queue_lock.slock,
-					irqs);
-				if (retval) {
-					return retval;
-				} else {
-					u132_endp_queue_work(u132, endp, 0);
-					return 0;
-				}
-			} else if (u132->num_endpoints == MAX_U132_ENDPS) {
-				return -EINVAL;
-			} else
-				return create_endpoint_and_queue_control(u132,
-					urb, usb_dev, usb_addr, usb_endp,
-					mem_flags);
-		}
-	}
-}
-
-static int dequeue_from_overflow_chain(struct u132 *u132,
-	struct u132_endp *endp, struct urb *urb)
-{
-	struct u132_urbq *urbq;
-
-	list_for_each_entry(urbq, &endp->urb_more, urb_more) {
-		if (urbq->urb == urb) {
-			struct usb_hcd *hcd = u132_to_hcd(u132);
-			list_del(&urbq->urb_more);
-			endp->queue_size -= 1;
-			urb->error_count = 0;
-			usb_hcd_giveback_urb(hcd, urb, 0);
-			return 0;
-		}
-	}
-	dev_err(&u132->platform_dev->dev, "urb=%p not found in endp[%d]=%p ring"
-		"[%d] %c%c usb_endp=%d usb_addr=%d size=%d next=%04X last=%04X"
-		"\n", urb, endp->endp_number, endp, endp->ring->number,
-		endp->input ? 'I' : ' ', endp->output ? 'O' : ' ',
-		endp->usb_endp, endp->usb_addr, endp->queue_size,
-		endp->queue_next, endp->queue_last);
-	return -EINVAL;
-}
-
-static int u132_endp_urb_dequeue(struct u132 *u132, struct u132_endp *endp,
-		struct urb *urb, int status)
-{
-	unsigned long irqs;
-	int rc;
-
-	spin_lock_irqsave(&endp->queue_lock.slock, irqs);
-	rc = usb_hcd_check_unlink_urb(u132_to_hcd(u132), urb, status);
-	if (rc) {
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-		return rc;
-	}
-	if (endp->queue_size == 0) {
-		dev_err(&u132->platform_dev->dev, "urb=%p not found in endp[%d]"
-			"=%p ring[%d] %c%c usb_endp=%d usb_addr=%d\n", urb,
-			endp->endp_number, endp, endp->ring->number,
-			endp->input ? 'I' : ' ', endp->output ? 'O' : ' ',
-			endp->usb_endp, endp->usb_addr);
-		spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-		return -EINVAL;
-	}
-	if (urb == endp->urb_list[ENDP_QUEUE_MASK & endp->queue_next]) {
-		if (endp->active) {
-			endp->dequeueing = 1;
-			endp->edset_flush = 1;
-			u132_endp_queue_work(u132, endp, 0);
-			spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-			return 0;
-		} else {
-			spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-			u132_hcd_abandon_urb(u132, endp, urb, status);
-			return 0;
-		}
-	} else {
-		u16 queue_list = 0;
-		u16 queue_size = endp->queue_size;
-		u16 queue_scan = endp->queue_next;
-		struct urb **urb_slot = NULL;
-		while (++queue_list < ENDP_QUEUE_SIZE && --queue_size > 0) {
-			if (urb == endp->urb_list[ENDP_QUEUE_MASK &
-				++queue_scan]) {
-				urb_slot = &endp->urb_list[ENDP_QUEUE_MASK &
-					queue_scan];
-				break;
-			}
-		}
-		while (++queue_list < ENDP_QUEUE_SIZE && --queue_size > 0) {
-			*urb_slot = endp->urb_list[ENDP_QUEUE_MASK &
-				++queue_scan];
-			urb_slot = &endp->urb_list[ENDP_QUEUE_MASK &
-				queue_scan];
-		}
-		if (urb_slot) {
-			struct usb_hcd *hcd = u132_to_hcd(u132);
-
-			usb_hcd_unlink_urb_from_ep(hcd, urb);
-			endp->queue_size -= 1;
-			if (list_empty(&endp->urb_more)) {
-				spin_unlock_irqrestore(&endp->queue_lock.slock,
-					irqs);
-			} else {
-				struct list_head *next = endp->urb_more.next;
-				struct u132_urbq *urbq = list_entry(next,
-					struct u132_urbq, urb_more);
-				list_del(next);
-				*urb_slot = urbq->urb;
-				spin_unlock_irqrestore(&endp->queue_lock.slock,
-					irqs);
-				kfree(urbq);
-			}
-			urb->error_count = 0;
-			usb_hcd_giveback_urb(hcd, urb, status);
-			return 0;
-		} else if (list_empty(&endp->urb_more)) {
-			dev_err(&u132->platform_dev->dev, "urb=%p not found in "
-				"endp[%d]=%p ring[%d] %c%c usb_endp=%d usb_addr"
-				"=%d size=%d next=%04X last=%04X\n", urb,
-				endp->endp_number, endp, endp->ring->number,
-				endp->input ? 'I' : ' ',
-				endp->output ? 'O' : ' ', endp->usb_endp,
-				endp->usb_addr, endp->queue_size,
-				endp->queue_next, endp->queue_last);
-			spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-			return -EINVAL;
-		} else {
-			int retval;
-
-			usb_hcd_unlink_urb_from_ep(u132_to_hcd(u132), urb);
-			retval = dequeue_from_overflow_chain(u132, endp,
-				urb);
-			spin_unlock_irqrestore(&endp->queue_lock.slock, irqs);
-			return retval;
-		}
-	}
-}
-
-static int u132_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 2) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else {
-		u8 usb_addr = usb_pipedevice(urb->pipe);
-		u8 usb_endp = usb_pipeendpoint(urb->pipe);
-		u8 address = u132->addr[usb_addr].address;
-		struct u132_udev *udev = &u132->udev[address];
-		if (usb_pipein(urb->pipe)) {
-			u8 endp_number = udev->endp_number_in[usb_endp];
-			struct u132_endp *endp = u132->endp[endp_number - 1];
-			return u132_endp_urb_dequeue(u132, endp, urb, status);
-		} else {
-			u8 endp_number = udev->endp_number_out[usb_endp];
-			struct u132_endp *endp = u132->endp[endp_number - 1];
-			return u132_endp_urb_dequeue(u132, endp, urb, status);
-		}
-	}
-}
-
-static void u132_endpoint_disable(struct usb_hcd *hcd,
-	struct usb_host_endpoint *hep)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 2) {
-		dev_err(&u132->platform_dev->dev, "u132 device %p(hcd=%p hep=%p"
-			") has been removed %d\n", u132, hcd, hep,
-			u132->going);
-	} else {
-		struct u132_endp *endp = hep->hcpriv;
-		if (endp)
-			u132_endp_put_kref(u132, endp);
-	}
-}
-
-static int u132_get_frame(struct usb_hcd *hcd)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed\n");
-		return -ESHUTDOWN;
-	} else {
-		dev_err(&u132->platform_dev->dev, "TODO: u132_get_frame\n");
-		mdelay(100);
-		return 0;
-	}
-}
-
-static int u132_roothub_descriptor(struct u132 *u132,
-	struct usb_hub_descriptor *desc)
-{
-	int retval;
-	u16 temp;
-	u32 rh_a = -1;
-	u32 rh_b = -1;
-	retval = u132_read_pcimem(u132, roothub.a, &rh_a);
-	if (retval)
-		return retval;
-	desc->bDescriptorType = USB_DT_HUB;
-	desc->bPwrOn2PwrGood = (rh_a & RH_A_POTPGT) >> 24;
-	desc->bHubContrCurrent = 0;
-	desc->bNbrPorts = u132->num_ports;
-	temp = 1 + (u132->num_ports / 8);
-	desc->bDescLength = 7 + 2 * temp;
-	temp = HUB_CHAR_COMMON_LPSM | HUB_CHAR_COMMON_OCPM;
-	if (rh_a & RH_A_NPS)
-		temp |= HUB_CHAR_NO_LPSM;
-	if (rh_a & RH_A_PSM)
-		temp |= HUB_CHAR_INDV_PORT_LPSM;
-	if (rh_a & RH_A_NOCP)
-		temp |= HUB_CHAR_NO_OCPM;
-	else if (rh_a & RH_A_OCPM)
-		temp |= HUB_CHAR_INDV_PORT_OCPM;
-	desc->wHubCharacteristics = cpu_to_le16(temp);
-	retval = u132_read_pcimem(u132, roothub.b, &rh_b);
-	if (retval)
-		return retval;
-	memset(desc->u.hs.DeviceRemovable, 0xff,
-			sizeof(desc->u.hs.DeviceRemovable));
-	desc->u.hs.DeviceRemovable[0] = rh_b & RH_B_DR;
-	if (u132->num_ports > 7) {
-		desc->u.hs.DeviceRemovable[1] = (rh_b & RH_B_DR) >> 8;
-		desc->u.hs.DeviceRemovable[2] = 0xff;
-	} else
-		desc->u.hs.DeviceRemovable[1] = 0xff;
-	return 0;
-}
-
-static int u132_roothub_status(struct u132 *u132, __le32 *desc)
-{
-	u32 rh_status = -1;
-	int ret_status = u132_read_pcimem(u132, roothub.status, &rh_status);
-	*desc = cpu_to_le32(rh_status);
-	return ret_status;
-}
-
-static int u132_roothub_portstatus(struct u132 *u132, __le32 *desc, u16 wIndex)
-{
-	if (wIndex == 0 || wIndex > u132->num_ports) {
-		return -EINVAL;
-	} else {
-		int port = wIndex - 1;
-		u32 rh_portstatus = -1;
-		int ret_portstatus = u132_read_pcimem(u132,
-			roothub.portstatus[port], &rh_portstatus);
-		*desc = cpu_to_le32(rh_portstatus);
-		if (*(u16 *) (desc + 2)) {
-			dev_info(&u132->platform_dev->dev, "Port %d Status Chan"
-				"ge = %08X\n", port, *desc);
-		}
-		return ret_portstatus;
-	}
-}
-
-
-/* this timer value might be vendor-specific ... */
-#define PORT_RESET_HW_MSEC 10
-#define PORT_RESET_MSEC 10
-/* wrap-aware logic morphed from <linux/jiffies.h> */
-#define tick_before(t1, t2) ((s16)(((s16)(t1))-((s16)(t2))) < 0)
-static int u132_roothub_portreset(struct u132 *u132, int port_index)
-{
-	int retval;
-	u32 fmnumber;
-	u16 now;
-	u16 reset_done;
-	retval = u132_read_pcimem(u132, fmnumber, &fmnumber);
-	if (retval)
-		return retval;
-	now = fmnumber;
-	reset_done = now + PORT_RESET_MSEC;
-	do {
-		u32 portstat;
-		do {
-			retval = u132_read_pcimem(u132,
-				roothub.portstatus[port_index], &portstat);
-			if (retval)
-				return retval;
-			if (RH_PS_PRS & portstat)
-				continue;
-			else
-				break;
-		} while (tick_before(now, reset_done));
-		if (RH_PS_PRS & portstat)
-			return -ENODEV;
-		if (RH_PS_CCS & portstat) {
-			if (RH_PS_PRSC & portstat) {
-				retval = u132_write_pcimem(u132,
-					roothub.portstatus[port_index],
-					RH_PS_PRSC);
-				if (retval)
-					return retval;
-			}
-		} else
-			break;	/* start the next reset,
-				sleep till it's probably done */
-		retval = u132_write_pcimem(u132, roothub.portstatus[port_index],
-			 RH_PS_PRS);
-		if (retval)
-			return retval;
-		msleep(PORT_RESET_HW_MSEC);
-		retval = u132_read_pcimem(u132, fmnumber, &fmnumber);
-		if (retval)
-			return retval;
-		now = fmnumber;
-	} while (tick_before(now, reset_done));
-	return 0;
-}
-
-static int u132_roothub_setportfeature(struct u132 *u132, u16 wValue,
-	u16 wIndex)
-{
-	if (wIndex == 0 || wIndex > u132->num_ports) {
-		return -EINVAL;
-	} else {
-		int port_index = wIndex - 1;
-		struct u132_port *port = &u132->port[port_index];
-		port->Status &= ~(1 << wValue);
-		switch (wValue) {
-		case USB_PORT_FEAT_SUSPEND:
-			return u132_write_pcimem(u132,
-			       roothub.portstatus[port_index], RH_PS_PSS);
-		case USB_PORT_FEAT_POWER:
-			return u132_write_pcimem(u132,
-			       roothub.portstatus[port_index], RH_PS_PPS);
-		case USB_PORT_FEAT_RESET:
-			return u132_roothub_portreset(u132, port_index);
-		default:
-			return -EPIPE;
-		}
-	}
-}
-
-static int u132_roothub_clearportfeature(struct u132 *u132, u16 wValue,
-	u16 wIndex)
-{
-	if (wIndex == 0 || wIndex > u132->num_ports) {
-		return -EINVAL;
-	} else {
-		int port_index = wIndex - 1;
-		u32 temp;
-		struct u132_port *port = &u132->port[port_index];
-		port->Status &= ~(1 << wValue);
-		switch (wValue) {
-		case USB_PORT_FEAT_ENABLE:
-			temp = RH_PS_CCS;
-			break;
-		case USB_PORT_FEAT_C_ENABLE:
-			temp = RH_PS_PESC;
-			break;
-		case USB_PORT_FEAT_SUSPEND:
-			temp = RH_PS_POCI;
-			if ((u132->hc_control & OHCI_CTRL_HCFS)
-				!= OHCI_USB_OPER) {
-				dev_err(&u132->platform_dev->dev, "TODO resume_"
-					"root_hub\n");
-			}
-			break;
-		case USB_PORT_FEAT_C_SUSPEND:
-			temp = RH_PS_PSSC;
-			break;
-		case USB_PORT_FEAT_POWER:
-			temp = RH_PS_LSDA;
-			break;
-		case USB_PORT_FEAT_C_CONNECTION:
-			temp = RH_PS_CSC;
-			break;
-		case USB_PORT_FEAT_C_OVER_CURRENT:
-			temp = RH_PS_OCIC;
-			break;
-		case USB_PORT_FEAT_C_RESET:
-			temp = RH_PS_PRSC;
-			break;
-		default:
-			return -EPIPE;
-		}
-		return u132_write_pcimem(u132, roothub.portstatus[port_index],
-		       temp);
-	}
-}
-
-
-/* the virtual root hub timer IRQ checks for hub status*/
-static int u132_hub_status_data(struct usb_hcd *hcd, char *buf)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device hcd=%p has been remov"
-			"ed %d\n", hcd, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device hcd=%p is being remov"
-			"ed\n", hcd);
-		return -ESHUTDOWN;
-	} else {
-		int i, changed = 0, length = 1;
-		if (u132->flags & OHCI_QUIRK_AMD756) {
-			if ((u132->hc_roothub_a & RH_A_NDP) > MAX_ROOT_PORTS) {
-				dev_err(&u132->platform_dev->dev, "bogus NDP, r"
-					"ereads as NDP=%d\n",
-					u132->hc_roothub_a & RH_A_NDP);
-				goto done;
-			}
-		}
-		if (u132->hc_roothub_status & (RH_HS_LPSC | RH_HS_OCIC))
-			buf[0] = changed = 1;
-		else
-			buf[0] = 0;
-		if (u132->num_ports > 7) {
-			buf[1] = 0;
-			length++;
-		}
-		for (i = 0; i < u132->num_ports; i++) {
-			if (u132->hc_roothub_portstatus[i] & (RH_PS_CSC |
-				RH_PS_PESC | RH_PS_PSSC | RH_PS_OCIC |
-				RH_PS_PRSC)) {
-				changed = 1;
-				if (i < 7)
-					buf[0] |= 1 << (i + 1);
-				else
-					buf[1] |= 1 << (i - 7);
-				continue;
-			}
-			if (!(u132->hc_roothub_portstatus[i] & RH_PS_CCS))
-				continue;
-
-			if ((u132->hc_roothub_portstatus[i] & RH_PS_PSS))
-				continue;
-		}
-done:
-		return changed ? length : 0;
-	}
-}
-
-static int u132_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
-	u16 wIndex, char *buf, u16 wLength)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed\n");
-		return -ESHUTDOWN;
-	} else {
-		int retval = 0;
-		mutex_lock(&u132->sw_lock);
-		switch (typeReq) {
-		case ClearHubFeature:
-			switch (wValue) {
-			case C_HUB_OVER_CURRENT:
-			case C_HUB_LOCAL_POWER:
-				break;
-			default:
-				goto stall;
-			}
-			break;
-		case SetHubFeature:
-			switch (wValue) {
-			case C_HUB_OVER_CURRENT:
-			case C_HUB_LOCAL_POWER:
-				break;
-			default:
-				goto stall;
-			}
-			break;
-		case ClearPortFeature:{
-				retval = u132_roothub_clearportfeature(u132,
-					wValue, wIndex);
-				if (retval)
-					goto error;
-				break;
-			}
-		case GetHubDescriptor:{
-				retval = u132_roothub_descriptor(u132,
-					(struct usb_hub_descriptor *)buf);
-				if (retval)
-					goto error;
-				break;
-			}
-		case GetHubStatus:{
-				retval = u132_roothub_status(u132,
-					(__le32 *) buf);
-				if (retval)
-					goto error;
-				break;
-			}
-		case GetPortStatus:{
-				retval = u132_roothub_portstatus(u132,
-					(__le32 *) buf, wIndex);
-				if (retval)
-					goto error;
-				break;
-			}
-		case SetPortFeature:{
-				retval = u132_roothub_setportfeature(u132,
-					wValue, wIndex);
-				if (retval)
-					goto error;
-				break;
-			}
-		default:
-			goto stall;
-		error:
-			u132_disable(u132);
-			u132->going = 1;
-			break;
-		stall:
-			retval = -EPIPE;
-			break;
-		}
-		mutex_unlock(&u132->sw_lock);
-		return retval;
-	}
-}
-
-static int u132_start_port_reset(struct usb_hcd *hcd, unsigned port_num)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed\n");
-		return -ESHUTDOWN;
-	} else
-		return 0;
-}
-
-
-#ifdef CONFIG_PM
-static int u132_bus_suspend(struct usb_hcd *hcd)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed\n");
-		return -ESHUTDOWN;
-	} else
-		return 0;
-}
-
-static int u132_bus_resume(struct usb_hcd *hcd)
-{
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed\n");
-		return -ESHUTDOWN;
-	} else
-		return 0;
-}
-
-#else
-#define u132_bus_suspend NULL
-#define u132_bus_resume NULL
-#endif
-static const struct hc_driver u132_hc_driver = {
-	.description = hcd_name,
-	.hcd_priv_size = sizeof(struct u132),
-	.irq = NULL,
-	.flags = HCD_USB11 | HCD_MEMORY,
-	.reset = u132_hcd_reset,
-	.start = u132_hcd_start,
-	.stop = u132_hcd_stop,
-	.urb_enqueue = u132_urb_enqueue,
-	.urb_dequeue = u132_urb_dequeue,
-	.endpoint_disable = u132_endpoint_disable,
-	.get_frame_number = u132_get_frame,
-	.hub_status_data = u132_hub_status_data,
-	.hub_control = u132_hub_control,
-	.bus_suspend = u132_bus_suspend,
-	.bus_resume = u132_bus_resume,
-	.start_port_reset = u132_start_port_reset,
-};
-
-/*
-* This function may be called by the USB core whilst the "usb_all_devices_rwsem"
-* is held for writing, thus this module must not call usb_remove_hcd()
-* synchronously - but instead should immediately stop activity to the
-* device and asynchronously call usb_remove_hcd()
-*/
-static int u132_remove(struct platform_device *pdev)
-{
-	struct usb_hcd *hcd = platform_get_drvdata(pdev);
-	if (hcd) {
-		struct u132 *u132 = hcd_to_u132(hcd);
-		if (u132->going++ > 1) {
-			dev_err(&u132->platform_dev->dev, "already being remove"
-				"d\n");
-			return -ENODEV;
-		} else {
-			int rings = MAX_U132_RINGS;
-			int endps = MAX_U132_ENDPS;
-			dev_err(&u132->platform_dev->dev, "removing device u132"
-				".%d\n", u132->sequence_num);
-			msleep(100);
-			mutex_lock(&u132->sw_lock);
-			u132_monitor_cancel_work(u132);
-			while (rings-- > 0) {
-				struct u132_ring *ring = &u132->ring[rings];
-				u132_ring_cancel_work(u132, ring);
-			}
-			while (endps-- > 0) {
-				struct u132_endp *endp = u132->endp[endps];
-				if (endp)
-					u132_endp_cancel_work(u132, endp);
-			}
-			u132->going += 1;
-			printk(KERN_INFO "removing device u132.%d\n",
-				u132->sequence_num);
-			mutex_unlock(&u132->sw_lock);
-			usb_remove_hcd(hcd);
-			u132_u132_put_kref(u132);
-			return 0;
-		}
-	} else
-		return 0;
-}
-
-static void u132_initialise(struct u132 *u132, struct platform_device *pdev)
-{
-	int rings = MAX_U132_RINGS;
-	int ports = MAX_U132_PORTS;
-	int addrs = MAX_U132_ADDRS;
-	int udevs = MAX_U132_UDEVS;
-	int endps = MAX_U132_ENDPS;
-	u132->board = dev_get_platdata(&pdev->dev);
-	u132->platform_dev = pdev;
-	u132->power = 0;
-	u132->reset = 0;
-	mutex_init(&u132->sw_lock);
-	mutex_init(&u132->scheduler_lock);
-	while (rings-- > 0) {
-		struct u132_ring *ring = &u132->ring[rings];
-		ring->u132 = u132;
-		ring->number = rings + 1;
-		ring->length = 0;
-		ring->curr_endp = NULL;
-		INIT_DELAYED_WORK(&ring->scheduler,
-				  u132_hcd_ring_work_scheduler);
-	}
-	mutex_lock(&u132->sw_lock);
-	INIT_DELAYED_WORK(&u132->monitor, u132_hcd_monitor_work);
-	while (ports-- > 0) {
-		struct u132_port *port = &u132->port[ports];
-		port->u132 = u132;
-		port->reset = 0;
-		port->enable = 0;
-		port->power = 0;
-		port->Status = 0;
-	}
-	while (addrs-- > 0) {
-		struct u132_addr *addr = &u132->addr[addrs];
-		addr->address = 0;
-	}
-	while (udevs-- > 0) {
-		struct u132_udev *udev = &u132->udev[udevs];
-		int i = ARRAY_SIZE(udev->endp_number_in);
-		int o = ARRAY_SIZE(udev->endp_number_out);
-		udev->usb_device = NULL;
-		udev->udev_number = 0;
-		udev->usb_addr = 0;
-		udev->portnumber = 0;
-		while (i-- > 0)
-			udev->endp_number_in[i] = 0;
-
-		while (o-- > 0)
-			udev->endp_number_out[o] = 0;
-
-	}
-	while (endps-- > 0)
-		u132->endp[endps] = NULL;
-
-	mutex_unlock(&u132->sw_lock);
-}
-
-static int u132_probe(struct platform_device *pdev)
-{
-	struct usb_hcd *hcd;
-	int retval;
-	u32 control;
-	u32 rh_a = -1;
-
-	msleep(100);
-	if (u132_exiting > 0)
-		return -ENODEV;
-
-	retval = ftdi_write_pcimem(pdev, intrdisable, OHCI_INTR_MIE);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(pdev, control, &control);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(pdev, roothub.a, &rh_a);
-	if (retval)
-		return retval;
-
-	hcd = usb_create_hcd(&u132_hc_driver, &pdev->dev, dev_name(&pdev->dev));
-	if (!hcd) {
-		printk(KERN_ERR "failed to create the usb hcd struct for U132\n"
-			);
-		ftdi_elan_gone_away(pdev);
-		return -ENOMEM;
-	} else {
-		struct u132 *u132 = hcd_to_u132(hcd);
-		retval = 0;
-		hcd->rsrc_start = 0;
-		mutex_lock(&u132_module_lock);
-		u132->sequence_num = ++u132_instances;
-		mutex_unlock(&u132_module_lock);
-		u132_u132_init_kref(u132);
-		u132_initialise(u132, pdev);
-		hcd->product_desc = "ELAN U132 Host Controller";
-		retval = usb_add_hcd(hcd, 0, 0);
-		if (retval != 0) {
-			dev_err(&u132->platform_dev->dev, "init error %d\n",
-				retval);
-			u132_u132_put_kref(u132);
-			return retval;
-		} else {
-			device_wakeup_enable(hcd->self.controller);
-			u132_monitor_queue_work(u132, 100);
-			return 0;
-		}
-	}
-}
-
-
-#ifdef CONFIG_PM
-/*
- * for this device there's no useful distinction between the controller
- * and its root hub.
- */
-static int u132_suspend(struct platform_device *pdev, pm_message_t state)
-{
-	struct usb_hcd *hcd = platform_get_drvdata(pdev);
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed\n");
-		return -ESHUTDOWN;
-	} else {
-		int retval = 0, ports;
-
-		switch (state.event) {
-		case PM_EVENT_FREEZE:
-			retval = u132_bus_suspend(hcd);
-			break;
-		case PM_EVENT_SUSPEND:
-		case PM_EVENT_HIBERNATE:
-			ports = MAX_U132_PORTS;
-			while (ports-- > 0) {
-				port_power(u132, ports, 0);
-			}
-			break;
-		}
-		return retval;
-	}
-}
-
-static int u132_resume(struct platform_device *pdev)
-{
-	struct usb_hcd *hcd = platform_get_drvdata(pdev);
-	struct u132 *u132 = hcd_to_u132(hcd);
-	if (u132->going > 1) {
-		dev_err(&u132->platform_dev->dev, "device has been removed %d\n"
-			, u132->going);
-		return -ENODEV;
-	} else if (u132->going > 0) {
-		dev_err(&u132->platform_dev->dev, "device is being removed\n");
-		return -ESHUTDOWN;
-	} else {
-		int retval = 0;
-		if (!u132->port[0].power) {
-			int ports = MAX_U132_PORTS;
-			while (ports-- > 0) {
-				port_power(u132, ports, 1);
-			}
-			retval = 0;
-		} else {
-			retval = u132_bus_resume(hcd);
-		}
-		return retval;
-	}
-}
-
-#else
-#define u132_suspend NULL
-#define u132_resume NULL
-#endif
-/*
-* this driver is loaded explicitly by ftdi_u132
-*
-* the platform_driver struct is static because it is per type of module
-*/
-static struct platform_driver u132_platform_driver = {
-	.probe = u132_probe,
-	.remove = u132_remove,
-	.suspend = u132_suspend,
-	.resume = u132_resume,
-	.driver = {
-		   .name = hcd_name,
-		   },
-};
-static int __init u132_hcd_init(void)
-{
-	int retval;
-	u132_instances = 0;
-	u132_exiting = 0;
-	if (usb_disabled())
-		return -ENODEV;
-	workqueue = create_singlethread_workqueue("u132");
-	if (!workqueue)
-		return -ENOMEM;
-	retval = platform_driver_register(&u132_platform_driver);
-	if (retval)
-		destroy_workqueue(workqueue);
-
-	return retval;
-}
-
-
-module_init(u132_hcd_init);
-static void __exit u132_hcd_exit(void)
-{
-	mutex_lock(&u132_module_lock);
-	u132_exiting += 1;
-	mutex_unlock(&u132_module_lock);
-	platform_driver_unregister(&u132_platform_driver);
-	printk(KERN_INFO "u132-hcd driver deregistered\n");
-	wait_event(u132_hcd_wait, u132_instances == 0);
-	destroy_workqueue(workqueue);
-}
-
-
-module_exit(u132_hcd_exit);
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:u132_hcd");
diff --git a/drivers/usb/host/xhci-dbgcap.c b/drivers/usb/host/xhci-dbgcap.c
index f1367b53b260..b40d9238d447 100644
--- a/drivers/usb/host/xhci-dbgcap.c
+++ b/drivers/usb/host/xhci-dbgcap.c
@@ -124,10 +124,10 @@ static void xhci_dbc_init_contexts(struct xhci_dbc *dbc, u32 string_length)
 	/* Set DbC context and info registers: */
 	lo_hi_writeq(dbc->ctx->dma, &dbc->regs->dccp);
 
-	dev_info = cpu_to_le32((DBC_VENDOR_ID << 16) | DBC_PROTOCOL);
+	dev_info = (dbc->idVendor << 16) | dbc->bInterfaceProtocol;
 	writel(dev_info, &dbc->regs->devinfo1);
 
-	dev_info = cpu_to_le32((DBC_DEVICE_REV << 16) | DBC_PRODUCT_ID);
+	dev_info = (dbc->bcdDevice << 16) | dbc->idProduct;
 	writel(dev_info, &dbc->regs->devinfo2);
 }
 
@@ -971,7 +971,186 @@ static ssize_t dbc_store(struct device *dev,
 	return count;
 }
 
+static ssize_t dbc_idVendor_show(struct device *dev,
+			    struct device_attribute *attr,
+			    char *buf)
+{
+	struct xhci_dbc		*dbc;
+	struct xhci_hcd		*xhci;
+
+	xhci = hcd_to_xhci(dev_get_drvdata(dev));
+	dbc = xhci->dbc;
+
+	return sprintf(buf, "%04x\n", dbc->idVendor);
+}
+
+static ssize_t dbc_idVendor_store(struct device *dev,
+			     struct device_attribute *attr,
+			     const char *buf, size_t size)
+{
+	struct xhci_dbc		*dbc;
+	struct xhci_hcd		*xhci;
+	void __iomem		*ptr;
+	u16			value;
+	u32			dev_info;
+
+	if (kstrtou16(buf, 0, &value))
+		return -EINVAL;
+
+	xhci = hcd_to_xhci(dev_get_drvdata(dev));
+	dbc = xhci->dbc;
+	if (dbc->state != DS_DISABLED)
+		return -EBUSY;
+
+	dbc->idVendor = value;
+	ptr = &dbc->regs->devinfo1;
+	dev_info = readl(ptr);
+	dev_info = (dev_info & ~(0xffffu << 16)) | (value << 16);
+	writel(dev_info, ptr);
+
+	return size;
+}
+
+static ssize_t dbc_idProduct_show(struct device *dev,
+			    struct device_attribute *attr,
+			    char *buf)
+{
+	struct xhci_dbc         *dbc;
+	struct xhci_hcd         *xhci;
+
+	xhci = hcd_to_xhci(dev_get_drvdata(dev));
+	dbc = xhci->dbc;
+
+	return sprintf(buf, "%04x\n", dbc->idProduct);
+}
+
+static ssize_t dbc_idProduct_store(struct device *dev,
+			     struct device_attribute *attr,
+			     const char *buf, size_t size)
+{
+	struct xhci_dbc         *dbc;
+	struct xhci_hcd         *xhci;
+	void __iomem		*ptr;
+	u32			dev_info;
+	u16			value;
+
+	if (kstrtou16(buf, 0, &value))
+		return -EINVAL;
+
+	xhci = hcd_to_xhci(dev_get_drvdata(dev));
+	dbc = xhci->dbc;
+	if (dbc->state != DS_DISABLED)
+		return -EBUSY;
+
+	dbc->idProduct = value;
+	ptr = &dbc->regs->devinfo2;
+	dev_info = readl(ptr);
+	dev_info = (dev_info & ~(0xffffu)) | value;
+	writel(dev_info, ptr);
+	return size;
+}
+
+static ssize_t dbc_bcdDevice_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	struct xhci_dbc	*dbc;
+	struct xhci_hcd	*xhci;
+
+	xhci = hcd_to_xhci(dev_get_drvdata(dev));
+	dbc = xhci->dbc;
+
+	return sprintf(buf, "%04x\n", dbc->bcdDevice);
+}
+
+static ssize_t dbc_bcdDevice_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t size)
+{
+	struct xhci_dbc	*dbc;
+	struct xhci_hcd	*xhci;
+	void __iomem *ptr;
+	u32 dev_info;
+	u16 value;
+
+	if (kstrtou16(buf, 0, &value))
+		return -EINVAL;
+
+	xhci = hcd_to_xhci(dev_get_drvdata(dev));
+	dbc = xhci->dbc;
+	if (dbc->state != DS_DISABLED)
+		return -EBUSY;
+
+	dbc->bcdDevice = value;
+	ptr = &dbc->regs->devinfo2;
+	dev_info = readl(ptr);
+	dev_info = (dev_info & ~(0xffffu << 16)) | (value << 16);
+	writel(dev_info, ptr);
+
+	return size;
+}
+
+static ssize_t dbc_bInterfaceProtocol_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct xhci_dbc	*dbc;
+	struct xhci_hcd	*xhci;
+
+	xhci = hcd_to_xhci(dev_get_drvdata(dev));
+	dbc = xhci->dbc;
+
+	return sprintf(buf, "%02x\n", dbc->bInterfaceProtocol);
+}
+
+static ssize_t dbc_bInterfaceProtocol_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t size)
+{
+	struct xhci_dbc *dbc;
+	struct xhci_hcd *xhci;
+	void __iomem *ptr;
+	u32 dev_info;
+	u8 value;
+	int ret;
+
+	/* bInterfaceProtocol is 8 bit, but xhci only supports values 0 and 1 */
+	ret = kstrtou8(buf, 0, &value);
+	if (ret || value > 1)
+		return -EINVAL;
+
+	xhci = hcd_to_xhci(dev_get_drvdata(dev));
+	dbc = xhci->dbc;
+	if (dbc->state != DS_DISABLED)
+		return -EBUSY;
+
+	dbc->bInterfaceProtocol = value;
+	ptr = &dbc->regs->devinfo1;
+	dev_info = readl(ptr);
+	dev_info = (dev_info & ~(0xffu)) | value;
+	writel(dev_info, ptr);
+
+	return size;
+}
+
 static DEVICE_ATTR_RW(dbc);
+static DEVICE_ATTR_RW(dbc_idVendor);
+static DEVICE_ATTR_RW(dbc_idProduct);
+static DEVICE_ATTR_RW(dbc_bcdDevice);
+static DEVICE_ATTR_RW(dbc_bInterfaceProtocol);
+
+static struct attribute *dbc_dev_attributes[] = {
+	&dev_attr_dbc.attr,
+	&dev_attr_dbc_idVendor.attr,
+	&dev_attr_dbc_idProduct.attr,
+	&dev_attr_dbc_bcdDevice.attr,
+	&dev_attr_dbc_bInterfaceProtocol.attr,
+	NULL
+};
+
+static const struct attribute_group dbc_dev_attrib_grp = {
+	.attrs = dbc_dev_attributes,
+};
 
 struct xhci_dbc *
 xhci_alloc_dbc(struct device *dev, void __iomem *base, const struct dbc_driver *driver)
@@ -986,6 +1165,10 @@ xhci_alloc_dbc(struct device *dev, void __iomem *base, const struct dbc_driver *
 	dbc->regs = base;
 	dbc->dev = dev;
 	dbc->driver = driver;
+	dbc->idProduct = DBC_PRODUCT_ID;
+	dbc->idVendor = DBC_VENDOR_ID;
+	dbc->bcdDevice = DBC_DEVICE_REV;
+	dbc->bInterfaceProtocol = DBC_PROTOCOL;
 
 	if (readl(&dbc->regs->control) & DBC_CTRL_DBC_ENABLE)
 		goto err;
@@ -993,7 +1176,7 @@ xhci_alloc_dbc(struct device *dev, void __iomem *base, const struct dbc_driver *
 	INIT_DELAYED_WORK(&dbc->event_work, xhci_dbc_handle_events);
 	spin_lock_init(&dbc->lock);
 
-	ret = device_create_file(dev, &dev_attr_dbc);
+	ret = sysfs_create_group(&dev->kobj, &dbc_dev_attrib_grp);
 	if (ret)
 		goto err;
 
@@ -1012,7 +1195,7 @@ void xhci_dbc_remove(struct xhci_dbc *dbc)
 	xhci_dbc_stop(dbc);
 
 	/* remove sysfs files */
-	device_remove_file(dbc->dev, &dev_attr_dbc);
+	sysfs_remove_group(&dbc->dev->kobj, &dbc_dev_attrib_grp);
 
 	kfree(dbc);
 }
diff --git a/drivers/usb/host/xhci-dbgcap.h b/drivers/usb/host/xhci-dbgcap.h
index ca04192fdab1..51a7ab3ba0ca 100644
--- a/drivers/usb/host/xhci-dbgcap.h
+++ b/drivers/usb/host/xhci-dbgcap.h
@@ -132,6 +132,10 @@ struct xhci_dbc {
 	struct dbc_str_descs		*string;
 	dma_addr_t			string_dma;
 	size_t				string_size;
+	u16				idVendor;
+	u16				idProduct;
+	u16				bcdDevice;
+	u8				bInterfaceProtocol;
 
 	enum dbc_state			state;
 	struct delayed_work		event_work;
diff --git a/drivers/usb/host/xhci-debugfs.c b/drivers/usb/host/xhci-debugfs.c
index 0bc7fe11f749..99baa60ef50f 100644
--- a/drivers/usb/host/xhci-debugfs.c
+++ b/drivers/usb/host/xhci-debugfs.c
@@ -133,6 +133,7 @@ static void xhci_debugfs_regset(struct xhci_hcd *xhci, u32 base,
 	regset->regs = regs;
 	regset->nregs = nregs;
 	regset->base = hcd->regs + base;
+	regset->dev = hcd->self.controller;
 
 	debugfs_create_regset32((const char *)rgs->name, 0444, parent, regset);
 }
diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c
index d0a9467aa5fc..7e106bd804ca 100644
--- a/drivers/usb/host/xhci-mem.c
+++ b/drivers/usb/host/xhci-mem.c
@@ -9,6 +9,7 @@
  */
 
 #include <linux/usb.h>
+#include <linux/overflow.h>
 #include <linux/pci.h>
 #include <linux/slab.h>
 #include <linux/dmapool.h>
@@ -543,14 +544,11 @@ static void xhci_free_stream_ctx(struct xhci_hcd *xhci,
 	size_t size = sizeof(struct xhci_stream_ctx) * num_stream_ctxs;
 
 	if (size > MEDIUM_STREAM_ARRAY_SIZE)
-		dma_free_coherent(dev, size,
-				stream_ctx, dma);
-	else if (size <= SMALL_STREAM_ARRAY_SIZE)
-		return dma_pool_free(xhci->small_streams_pool,
-				stream_ctx, dma);
+		dma_free_coherent(dev, size, stream_ctx, dma);
+	else if (size > SMALL_STREAM_ARRAY_SIZE)
+		dma_pool_free(xhci->medium_streams_pool, stream_ctx, dma);
 	else
-		return dma_pool_free(xhci->medium_streams_pool,
-				stream_ctx, dma);
+		dma_pool_free(xhci->small_streams_pool, stream_ctx, dma);
 }
 
 /*
@@ -568,17 +566,14 @@ static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci,
 		gfp_t mem_flags)
 {
 	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
-	size_t size = sizeof(struct xhci_stream_ctx) * num_stream_ctxs;
+	size_t size = size_mul(sizeof(struct xhci_stream_ctx), num_stream_ctxs);
 
 	if (size > MEDIUM_STREAM_ARRAY_SIZE)
-		return dma_alloc_coherent(dev, size,
-				dma, mem_flags);
-	else if (size <= SMALL_STREAM_ARRAY_SIZE)
-		return dma_pool_alloc(xhci->small_streams_pool,
-				mem_flags, dma);
+		return dma_alloc_coherent(dev, size, dma, mem_flags);
+	if (size > SMALL_STREAM_ARRAY_SIZE)
+		return dma_pool_zalloc(xhci->medium_streams_pool, mem_flags, dma);
 	else
-		return dma_pool_alloc(xhci->medium_streams_pool,
-				mem_flags, dma);
+		return dma_pool_zalloc(xhci->small_streams_pool, mem_flags, dma);
 }
 
 struct xhci_ring *xhci_dma_to_transfer_ring(
@@ -612,8 +607,7 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
 	int ret;
 	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
 
-	xhci_dbg(xhci, "Allocating %u streams and %u "
-			"stream context array entries.\n",
+	xhci_dbg(xhci, "Allocating %u streams and %u stream context array entries.\n",
 			num_streams, num_stream_ctxs);
 	if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) {
 		xhci_dbg(xhci, "Command ring has no reserved TRBs available\n");
@@ -642,8 +636,6 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
 			mem_flags);
 	if (!stream_info->stream_ctx_array)
 		goto cleanup_ring_array;
-	memset(stream_info->stream_ctx_array, 0,
-			sizeof(struct xhci_stream_ctx)*num_stream_ctxs);
 
 	/* Allocate everything needed to free the stream rings later */
 	stream_info->free_streams_command =
@@ -673,8 +665,7 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
 			cur_ring->cycle_state;
 		stream_info->stream_ctx_array[cur_stream].stream_ring =
 			cpu_to_le64(addr);
-		xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n",
-				cur_stream, (unsigned long long) addr);
+		xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n", cur_stream, addr);
 
 		ret = xhci_update_stream_mapping(cur_ring, mem_flags);
 		if (ret) {
@@ -984,16 +975,14 @@ int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id,
 	if (!dev->out_ctx)
 		goto fail;
 
-	xhci_dbg(xhci, "Slot %d output ctx = 0x%llx (dma)\n", slot_id,
-			(unsigned long long)dev->out_ctx->dma);
+	xhci_dbg(xhci, "Slot %d output ctx = 0x%pad (dma)\n", slot_id, &dev->out_ctx->dma);
 
 	/* Allocate the (input) device context for address device command */
 	dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags);
 	if (!dev->in_ctx)
 		goto fail;
 
-	xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id,
-			(unsigned long long)dev->in_ctx->dma);
+	xhci_dbg(xhci, "Slot %d input ctx = 0x%pad (dma)\n", slot_id, &dev->in_ctx->dma);
 
 	/* Initialize the cancellation and bandwidth list for each ep */
 	for (i = 0; i < 31; i++) {
@@ -1400,8 +1389,9 @@ static u32 xhci_get_max_esit_payload(struct usb_device *udev,
 	if ((udev->speed >= USB_SPEED_SUPER_PLUS) &&
 	    USB_SS_SSP_ISOC_COMP(ep->ss_ep_comp.bmAttributes))
 		return le32_to_cpu(ep->ssp_isoc_ep_comp.dwBytesPerInterval);
+
 	/* SuperSpeed or SuperSpeedPlus Isoc ep with less than 48k per esit */
-	else if (udev->speed >= USB_SPEED_SUPER)
+	if (udev->speed >= USB_SPEED_SUPER)
 		return le16_to_cpu(ep->ss_ep_comp.wBytesPerInterval);
 
 	max_packet = usb_endpoint_maxp(&ep->desc);
@@ -1660,7 +1650,7 @@ static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags)
 		goto fail_sp;
 
 	xhci->scratchpad->sp_array = dma_alloc_coherent(dev,
-				     num_sp * sizeof(u64),
+				     size_mul(sizeof(u64), num_sp),
 				     &xhci->scratchpad->sp_dma, flags);
 	if (!xhci->scratchpad->sp_array)
 		goto fail_sp2;
@@ -1685,11 +1675,10 @@ static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags)
 	return 0;
 
  fail_sp4:
-	for (i = i - 1; i >= 0; i--) {
+	while (i--)
 		dma_free_coherent(dev, xhci->page_size,
 				    xhci->scratchpad->sp_buffers[i],
 				    xhci->scratchpad->sp_array[i]);
-	}
 
 	kfree(xhci->scratchpad->sp_buffers);
 
@@ -1799,7 +1788,7 @@ int xhci_alloc_erst(struct xhci_hcd *xhci,
 	struct xhci_segment *seg;
 	struct xhci_erst_entry *entry;
 
-	size = sizeof(struct xhci_erst_entry) * evt_ring->num_segs;
+	size = size_mul(sizeof(struct xhci_erst_entry), evt_ring->num_segs);
 	erst->entries = dma_alloc_coherent(xhci_to_hcd(xhci)->self.sysdev,
 					   size, &erst->erst_dma_addr, flags);
 	if (!erst->entries)
@@ -1830,7 +1819,7 @@ xhci_free_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir)
 	if (!ir)
 		return;
 
-	erst_size = sizeof(struct xhci_erst_entry) * (ir->erst.num_entries);
+	erst_size = sizeof(struct xhci_erst_entry) * ir->erst.num_entries;
 	if (ir->erst.entries)
 		dma_free_coherent(dev, erst_size,
 				  ir->erst.entries,
@@ -1960,8 +1949,7 @@ static void xhci_set_hc_event_deq(struct xhci_hcd *xhci, struct xhci_interrupter
 	deq = xhci_trb_virt_to_dma(ir->event_ring->deq_seg,
 			ir->event_ring->dequeue);
 	if (!deq)
-		xhci_warn(xhci, "WARN something wrong with SW event ring "
-				"dequeue ptr.\n");
+		xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr.\n");
 	/* Update HC event ring dequeue pointer */
 	temp = xhci_read_64(xhci, &ir->ir_set->erst_dequeue);
 	temp &= ERST_PTR_MASK;
@@ -1970,8 +1958,7 @@ static void xhci_set_hc_event_deq(struct xhci_hcd *xhci, struct xhci_interrupter
 	 */
 	temp &= ~ERST_EHB;
 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
-			"// Write event ring dequeue pointer, "
-			"preserving EHB bit");
+		       "// Write event ring dequeue pointer, preserving EHB bit");
 	xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp,
 			&ir->ir_set->erst_dequeue);
 }
@@ -2004,8 +1991,7 @@ static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports,
 	} else if (major_revision <= 0x02) {
 		rhub = &xhci->usb2_rhub;
 	} else {
-		xhci_warn(xhci, "Ignoring unknown port speed, "
-				"Ext Cap %p, revision = 0x%x\n",
+		xhci_warn(xhci, "Ignoring unknown port speed, Ext Cap %p, revision = 0x%x\n",
 				addr, major_revision);
 		/* Ignoring port protocol we can't understand. FIXME */
 		return;
@@ -2020,9 +2006,8 @@ static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports,
 	port_offset = XHCI_EXT_PORT_OFF(temp);
 	port_count = XHCI_EXT_PORT_COUNT(temp);
 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
-			"Ext Cap %p, port offset = %u, "
-			"count = %u, revision = 0x%x",
-			addr, port_offset, port_count, major_revision);
+		       "Ext Cap %p, port offset = %u, count = %u, revision = 0x%x",
+		       addr, port_offset, port_count, major_revision);
 	/* Port count includes the current port offset */
 	if (port_offset == 0 || (port_offset + port_count - 1) > num_ports)
 		/* WTF? "Valid values are ‘1’ to MaxPorts" */
@@ -2079,10 +2064,8 @@ static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports,
 		struct xhci_port *hw_port = &xhci->hw_ports[i];
 		/* Duplicate entry.  Ignore the port if the revisions differ. */
 		if (hw_port->rhub) {
-			xhci_warn(xhci, "Duplicate port entry, Ext Cap %p,"
-					" port %u\n", addr, i);
-			xhci_warn(xhci, "Port was marked as USB %u, "
-					"duplicated as USB %u\n",
+			xhci_warn(xhci, "Duplicate port entry, Ext Cap %p, port %u\n", addr, i);
+			xhci_warn(xhci, "Port was marked as USB %u, duplicated as USB %u\n",
 					hw_port->rhub->maj_rev, major_revision);
 			/* Only adjust the roothub port counts if we haven't
 			 * found a similar duplicate.
@@ -2358,8 +2341,8 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
 		goto fail;
 	xhci->dcbaa->dma = dma;
 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
-			"// Device context base array address = 0x%llx (DMA), %p (virt)",
-			(unsigned long long)xhci->dcbaa->dma, xhci->dcbaa);
+			"// Device context base array address = 0x%pad (DMA), %p (virt)",
+			&xhci->dcbaa->dma, xhci->dcbaa);
 	xhci_write_64(xhci, dma, &xhci->op_regs->dcbaa_ptr);
 
 	/*
@@ -2400,8 +2383,8 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
 		goto fail;
 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
 			"Allocated command ring at %p", xhci->cmd_ring);
-	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%llx",
-			(unsigned long long)xhci->cmd_ring->first_seg->dma);
+	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%pad",
+			&xhci->cmd_ring->first_seg->dma);
 
 	/* Set the address in the Command Ring Control register */
 	val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
@@ -2421,8 +2404,8 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
 	val = readl(&xhci->cap_regs->db_off);
 	val &= DBOFF_MASK;
 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
-			"// Doorbell array is located at offset 0x%x"
-			" from cap regs base addr", val);
+		       "// Doorbell array is located at offset 0x%x from cap regs base addr",
+		       val);
 	xhci->dba = (void __iomem *) xhci->cap_regs + val;
 	/* Set ir_set to interrupt register set 0 */
 
diff --git a/drivers/usb/host/xhci-mtk.c b/drivers/usb/host/xhci-mtk.c
index f7cbb08fc506..90cf40d6d0c3 100644
--- a/drivers/usb/host/xhci-mtk.c
+++ b/drivers/usb/host/xhci-mtk.c
@@ -398,6 +398,7 @@ static int xhci_mtk_clks_get(struct xhci_hcd_mtk *mtk)
 	clks[2].id = "ref_ck";
 	clks[3].id = "mcu_ck";
 	clks[4].id = "dma_ck";
+	clks[5].id = "frmcnt_ck";
 
 	return devm_clk_bulk_get_optional(mtk->dev, BULK_CLKS_NUM, clks);
 }
diff --git a/drivers/usb/host/xhci-mtk.h b/drivers/usb/host/xhci-mtk.h
index 1174a510dd38..faaaf05e36ce 100644
--- a/drivers/usb/host/xhci-mtk.h
+++ b/drivers/usb/host/xhci-mtk.h
@@ -15,7 +15,7 @@
 
 #include "xhci.h"
 
-#define BULK_CLKS_NUM	5
+#define BULK_CLKS_NUM	6
 #define BULK_VREGS_NUM	2
 
 /* support at most 64 ep, use 32 size hash table */
diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c
index 6db07ca419c3..ddb79f23fb3b 100644
--- a/drivers/usb/host/xhci-pci.c
+++ b/drivers/usb/host/xhci-pci.c
@@ -78,14 +78,207 @@ static const char hcd_name[] = "xhci_hcd";
 static struct hc_driver __read_mostly xhci_pci_hc_driver;
 
 static int xhci_pci_setup(struct usb_hcd *hcd);
+static int xhci_pci_run(struct usb_hcd *hcd);
 static int xhci_pci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
 				      struct usb_tt *tt, gfp_t mem_flags);
 
 static const struct xhci_driver_overrides xhci_pci_overrides __initconst = {
 	.reset = xhci_pci_setup,
+	.start = xhci_pci_run,
 	.update_hub_device = xhci_pci_update_hub_device,
 };
 
+static void xhci_msix_sync_irqs(struct xhci_hcd *xhci)
+{
+	struct usb_hcd *hcd = xhci_to_hcd(xhci);
+
+	if (hcd->msix_enabled) {
+		struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
+		int i;
+
+		for (i = 0; i < xhci->msix_count; i++)
+			synchronize_irq(pci_irq_vector(pdev, i));
+	}
+}
+
+/* Free any IRQs and disable MSI-X */
+static void xhci_cleanup_msix(struct xhci_hcd *xhci)
+{
+	struct usb_hcd *hcd = xhci_to_hcd(xhci);
+	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
+
+	if (xhci->quirks & XHCI_PLAT)
+		return;
+
+	/* return if using legacy interrupt */
+	if (hcd->irq > 0)
+		return;
+
+	if (hcd->msix_enabled) {
+		int i;
+
+		for (i = 0; i < xhci->msix_count; i++)
+			free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
+	} else {
+		free_irq(pci_irq_vector(pdev, 0), xhci_to_hcd(xhci));
+	}
+
+	pci_free_irq_vectors(pdev);
+	hcd->msix_enabled = 0;
+}
+
+/*
+ * Set up MSI
+ */
+static int xhci_setup_msi(struct xhci_hcd *xhci)
+{
+	int ret;
+	/*
+	 * TODO:Check with MSI Soc for sysdev
+	 */
+	struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
+
+	ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
+	if (ret < 0) {
+		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
+				"failed to allocate MSI entry");
+		return ret;
+	}
+
+	ret = request_irq(pdev->irq, xhci_msi_irq,
+				0, "xhci_hcd", xhci_to_hcd(xhci));
+	if (ret) {
+		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
+				"disable MSI interrupt");
+		pci_free_irq_vectors(pdev);
+	}
+
+	return ret;
+}
+
+/*
+ * Set up MSI-X
+ */
+static int xhci_setup_msix(struct xhci_hcd *xhci)
+{
+	int i, ret;
+	struct usb_hcd *hcd = xhci_to_hcd(xhci);
+	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
+
+	/*
+	 * calculate number of msi-x vectors supported.
+	 * - HCS_MAX_INTRS: the max number of interrupts the host can handle,
+	 *   with max number of interrupters based on the xhci HCSPARAMS1.
+	 * - num_online_cpus: maximum msi-x vectors per CPUs core.
+	 *   Add additional 1 vector to ensure always available interrupt.
+	 */
+	xhci->msix_count = min(num_online_cpus() + 1,
+				HCS_MAX_INTRS(xhci->hcs_params1));
+
+	ret = pci_alloc_irq_vectors(pdev, xhci->msix_count, xhci->msix_count,
+			PCI_IRQ_MSIX);
+	if (ret < 0) {
+		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
+				"Failed to enable MSI-X");
+		return ret;
+	}
+
+	for (i = 0; i < xhci->msix_count; i++) {
+		ret = request_irq(pci_irq_vector(pdev, i), xhci_msi_irq, 0,
+				"xhci_hcd", xhci_to_hcd(xhci));
+		if (ret)
+			goto disable_msix;
+	}
+
+	hcd->msix_enabled = 1;
+	return ret;
+
+disable_msix:
+	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "disable MSI-X interrupt");
+	while (--i >= 0)
+		free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
+	pci_free_irq_vectors(pdev);
+	return ret;
+}
+
+static int xhci_try_enable_msi(struct usb_hcd *hcd)
+{
+	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
+	struct pci_dev  *pdev;
+	int ret;
+
+	/* The xhci platform device has set up IRQs through usb_add_hcd. */
+	if (xhci->quirks & XHCI_PLAT)
+		return 0;
+
+	pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
+	/*
+	 * Some Fresco Logic host controllers advertise MSI, but fail to
+	 * generate interrupts.  Don't even try to enable MSI.
+	 */
+	if (xhci->quirks & XHCI_BROKEN_MSI)
+		goto legacy_irq;
+
+	/* unregister the legacy interrupt */
+	if (hcd->irq)
+		free_irq(hcd->irq, hcd);
+	hcd->irq = 0;
+
+	ret = xhci_setup_msix(xhci);
+	if (ret)
+		/* fall back to msi*/
+		ret = xhci_setup_msi(xhci);
+
+	if (!ret) {
+		hcd->msi_enabled = 1;
+		return 0;
+	}
+
+	if (!pdev->irq) {
+		xhci_err(xhci, "No msi-x/msi found and no IRQ in BIOS\n");
+		return -EINVAL;
+	}
+
+ legacy_irq:
+	if (!strlen(hcd->irq_descr))
+		snprintf(hcd->irq_descr, sizeof(hcd->irq_descr), "%s:usb%d",
+			 hcd->driver->description, hcd->self.busnum);
+
+	/* fall back to legacy interrupt*/
+	ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,
+			hcd->irq_descr, hcd);
+	if (ret) {
+		xhci_err(xhci, "request interrupt %d failed\n",
+				pdev->irq);
+		return ret;
+	}
+	hcd->irq = pdev->irq;
+	return 0;
+}
+
+static int xhci_pci_run(struct usb_hcd *hcd)
+{
+	int ret;
+
+	if (usb_hcd_is_primary_hcd(hcd)) {
+		ret = xhci_try_enable_msi(hcd);
+		if (ret)
+			return ret;
+	}
+
+	return xhci_run(hcd);
+}
+
+static void xhci_pci_stop(struct usb_hcd *hcd)
+{
+	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
+
+	xhci_stop(hcd);
+
+	if (usb_hcd_is_primary_hcd(hcd))
+		xhci_cleanup_msix(xhci);
+}
+
 /* called after powerup, by probe or system-pm "wakeup" */
 static int xhci_pci_reinit(struct xhci_hcd *xhci, struct pci_dev *pdev)
 {
@@ -535,7 +728,6 @@ static void xhci_pci_remove(struct pci_dev *dev)
 	usb_hcd_pci_remove(dev);
 }
 
-#ifdef CONFIG_PM
 /*
  * In some Intel xHCI controllers, in order to get D3 working,
  * through a vendor specific SSIC CONFIG register at offset 0x883c,
@@ -622,6 +814,10 @@ static int xhci_pci_suspend(struct usb_hcd *hcd, bool do_wakeup)
 		xhci_sparse_control_quirk(hcd);
 
 	ret = xhci_suspend(xhci, do_wakeup);
+
+	/* synchronize irq when using MSI-X */
+	xhci_msix_sync_irqs(xhci);
+
 	if (ret && (xhci->quirks & XHCI_SSIC_PORT_UNUSED))
 		xhci_ssic_port_unused_quirk(hcd, false);
 
@@ -724,12 +920,12 @@ static void xhci_pci_shutdown(struct usb_hcd *hcd)
 	struct pci_dev		*pdev = to_pci_dev(hcd->self.controller);
 
 	xhci_shutdown(hcd);
+	xhci_cleanup_msix(xhci);
 
 	/* Yet another workaround for spurious wakeups at shutdown with HSW */
 	if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
 		pci_set_power_state(pdev, PCI_D3hot);
 }
-#endif /* CONFIG_PM */
 
 /*-------------------------------------------------------------------------*/
 
@@ -771,22 +967,19 @@ static struct pci_driver xhci_pci_driver = {
 	/* suspend and resume implemented later */
 
 	.shutdown = 	usb_hcd_pci_shutdown,
-#ifdef CONFIG_PM
 	.driver = {
-		.pm = &usb_hcd_pci_pm_ops
+		.pm = pm_ptr(&usb_hcd_pci_pm_ops),
 	},
-#endif
 };
 
 static int __init xhci_pci_init(void)
 {
 	xhci_init_driver(&xhci_pci_hc_driver, &xhci_pci_overrides);
-#ifdef CONFIG_PM
-	xhci_pci_hc_driver.pci_suspend = xhci_pci_suspend;
-	xhci_pci_hc_driver.pci_resume = xhci_pci_resume;
-	xhci_pci_hc_driver.pci_poweroff_late = xhci_pci_poweroff_late;
-	xhci_pci_hc_driver.shutdown = xhci_pci_shutdown;
-#endif
+	xhci_pci_hc_driver.pci_suspend = pm_ptr(xhci_pci_suspend);
+	xhci_pci_hc_driver.pci_resume = pm_ptr(xhci_pci_resume);
+	xhci_pci_hc_driver.pci_poweroff_late = pm_ptr(xhci_pci_poweroff_late);
+	xhci_pci_hc_driver.shutdown = pm_ptr(xhci_pci_shutdown);
+	xhci_pci_hc_driver.stop = xhci_pci_stop;
 	return pci_register_driver(&xhci_pci_driver);
 }
 module_init(xhci_pci_init);
diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c
index b9f9625467d6..b0c8e8efc43b 100644
--- a/drivers/usb/host/xhci-plat.c
+++ b/drivers/usb/host/xhci-plat.c
@@ -291,6 +291,21 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s
 			goto dealloc_usb2_hcd;
 		}
 
+		xhci->shared_hcd->usb_phy = devm_usb_get_phy_by_phandle(sysdev,
+			    "usb-phy", 1);
+		if (IS_ERR(xhci->shared_hcd->usb_phy)) {
+			if (PTR_ERR(xhci->shared_hcd->usb_phy) != -ENODEV)
+				dev_err(sysdev, "%s get usb3phy fail (ret=%d)\n",
+					     __func__,
+					    (int)PTR_ERR(xhci->shared_hcd->usb_phy));
+			xhci->shared_hcd->usb_phy = NULL;
+		} else {
+			ret = usb_phy_init(xhci->shared_hcd->usb_phy);
+			if (ret)
+				dev_err(sysdev, "%s init usb3phy fail (ret=%d)\n",
+					    __func__, ret);
+		}
+
 		xhci->shared_hcd->tpl_support = hcd->tpl_support;
 	}
 
@@ -362,10 +377,8 @@ static int xhci_generic_plat_probe(struct platform_device *pdev)
 		if (is_of_node(sysdev->fwnode) ||
 			is_acpi_device_node(sysdev->fwnode))
 			break;
-#ifdef CONFIG_PCI
-		else if (sysdev->bus == &pci_bus_type)
+		else if (dev_is_pci(sysdev))
 			break;
-#endif
 	}
 
 	if (!sysdev)
diff --git a/drivers/usb/host/xhci-rcar.c b/drivers/usb/host/xhci-rcar.c
index 7f18509a1d39..ad966b797b89 100644
--- a/drivers/usb/host/xhci-rcar.c
+++ b/drivers/usb/host/xhci-rcar.c
@@ -12,26 +12,22 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/usb/phy.h>
-#include <linux/sys_soc.h>
 
 #include "xhci.h"
 #include "xhci-plat.h"
 #include "xhci-rzv2m.h"
 
 #define XHCI_RCAR_FIRMWARE_NAME_V1	"r8a779x_usb3_v1.dlmem"
-#define XHCI_RCAR_FIRMWARE_NAME_V2	"r8a779x_usb3_v2.dlmem"
 #define XHCI_RCAR_FIRMWARE_NAME_V3	"r8a779x_usb3_v3.dlmem"
 
 /*
-* - The V3 firmware is for almost all R-Car Gen3 (except r8a7795 ES1.x)
-* - The V2 firmware is for r8a7795 ES1.x.
+* - The V3 firmware is for all R-Car Gen3
 * - The V2 firmware is possible to use on R-Car Gen2. However, the V2 causes
 *   performance degradation. So, this driver continues to use the V1 if R-Car
 *   Gen2.
 * - The V1 firmware is impossible to use on R-Car Gen3.
 */
 MODULE_FIRMWARE(XHCI_RCAR_FIRMWARE_NAME_V1);
-MODULE_FIRMWARE(XHCI_RCAR_FIRMWARE_NAME_V2);
 MODULE_FIRMWARE(XHCI_RCAR_FIRMWARE_NAME_V3);
 
 /*** Register Offset ***/
@@ -78,18 +74,6 @@ MODULE_FIRMWARE(XHCI_RCAR_FIRMWARE_NAME_V3);
 #define RCAR_USB3_RX_POL_VAL	BIT(21)
 #define RCAR_USB3_TX_POL_VAL	BIT(4)
 
-/* For soc_device_attribute */
-#define RCAR_XHCI_FIRMWARE_V2   BIT(0) /* FIRMWARE V2 */
-#define RCAR_XHCI_FIRMWARE_V3   BIT(1) /* FIRMWARE V3 */
-
-static const struct soc_device_attribute rcar_quirks_match[]  = {
-	{
-		.soc_id = "r8a7795", .revision = "ES1.*",
-		.data = (void *)RCAR_XHCI_FIRMWARE_V2,
-	},
-	{ /* sentinel */ }
-};
-
 static void xhci_rcar_start_gen2(struct usb_hcd *hcd)
 {
 	/* LCLK Select */
@@ -135,9 +119,6 @@ static int xhci_rcar_download_firmware(struct usb_hcd *hcd)
 	const struct firmware *fw;
 	int retval, index, j;
 	u32 data, val, temp;
-	u32 quirks = 0;
-	const struct soc_device_attribute *attr;
-	const char *firmware_name;
 
 	/*
 	 * According to the datasheet, "Upon the completion of FW Download,
@@ -146,19 +127,8 @@ static int xhci_rcar_download_firmware(struct usb_hcd *hcd)
 	if (readl(regs + RCAR_USB3_DL_CTRL) & RCAR_USB3_DL_CTRL_FW_SUCCESS)
 		return 0;
 
-	attr = soc_device_match(rcar_quirks_match);
-	if (attr)
-		quirks = (uintptr_t)attr->data;
-
-	if (quirks & RCAR_XHCI_FIRMWARE_V2)
-		firmware_name = XHCI_RCAR_FIRMWARE_NAME_V2;
-	else if (quirks & RCAR_XHCI_FIRMWARE_V3)
-		firmware_name = XHCI_RCAR_FIRMWARE_NAME_V3;
-	else
-		firmware_name = priv->firmware_name;
-
 	/* request R-Car USB3.0 firmware */
-	retval = request_firmware(&fw, firmware_name, dev);
+	retval = request_firmware(&fw, priv->firmware_name, dev);
 	if (retval)
 		return retval;
 
@@ -312,7 +282,7 @@ static struct platform_driver usb_xhci_renesas_driver = {
 	.driver	= {
 		.name = "xhci-renesas-hcd",
 		.pm = &xhci_plat_pm_ops,
-		.of_match_table = of_match_ptr(usb_xhci_of_match),
+		.of_match_table = usb_xhci_of_match,
 	},
 };
 module_platform_driver(usb_xhci_renesas_driver);
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
index eb788c60c1c0..1ad12d5a4857 100644
--- a/drivers/usb/host/xhci-ring.c
+++ b/drivers/usb/host/xhci-ring.c
@@ -3106,6 +3106,7 @@ irqreturn_t xhci_msi_irq(int irq, void *hcd)
 {
 	return xhci_irq(hcd);
 }
+EXPORT_SYMBOL_GPL(xhci_msi_irq);
 
 /****		Endpoint Ring Operations	****/
 
diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c
index a88c39e525c2..c75d93244143 100644
--- a/drivers/usb/host/xhci-tegra.c
+++ b/drivers/usb/host/xhci-tegra.c
@@ -1535,7 +1535,6 @@ static void tegra_xusb_deinit_usb_phy(struct tegra_xusb *tegra)
 
 static int tegra_xusb_probe(struct platform_device *pdev)
 {
-	struct of_phandle_args args;
 	struct tegra_xusb *tegra;
 	struct device_node *np;
 	struct resource *regs;
@@ -1594,15 +1593,13 @@ static int tegra_xusb_probe(struct platform_device *pdev)
 		goto put_padctl;
 	}
 
-	/* Older device-trees don't have padctrl interrupt */
-	err = of_irq_parse_one(np, 0, &args);
-	if (!err) {
-		tegra->padctl_irq = of_irq_get(np, 0);
-		if (tegra->padctl_irq <= 0) {
-			err = (tegra->padctl_irq == 0) ? -ENODEV : tegra->padctl_irq;
-			goto put_padctl;
-		}
-	} else {
+	tegra->padctl_irq = of_irq_get(np, 0);
+	if (tegra->padctl_irq == -EPROBE_DEFER) {
+		err = tegra->padctl_irq;
+		goto put_padctl;
+	} else if (tegra->padctl_irq <= 0) {
+		/* Older device-trees don't have padctrl interrupt */
+		tegra->padctl_irq = 0;
 		dev_dbg(&pdev->dev,
 			"%pOF is missing an interrupt, disabling PM support\n", np);
 	}
diff --git a/drivers/usb/host/xhci-trace.c b/drivers/usb/host/xhci-trace.c
index d0070814d1ea..062662d23241 100644
--- a/drivers/usb/host/xhci-trace.c
+++ b/drivers/usb/host/xhci-trace.c
@@ -12,3 +12,4 @@
 #include "xhci-trace.h"
 
 EXPORT_TRACEPOINT_SYMBOL_GPL(xhci_dbg_quirks);
+EXPORT_TRACEPOINT_SYMBOL_GPL(xhci_dbg_init);
diff --git a/drivers/usb/host/xhci-trace.h b/drivers/usb/host/xhci-trace.h
index 61e93a3540a7..4286dba5b157 100644
--- a/drivers/usb/host/xhci-trace.h
+++ b/drivers/usb/host/xhci-trace.h
@@ -120,7 +120,6 @@ DECLARE_EVENT_CLASS(xhci_log_trb,
 		__field(u32, field1)
 		__field(u32, field2)
 		__field(u32, field3)
-		__dynamic_array(char, str, XHCI_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->type = ring->type;
@@ -130,8 +129,8 @@ DECLARE_EVENT_CLASS(xhci_log_trb,
 		__entry->field3 = le32_to_cpu(trb->field[3]);
 	),
 	TP_printk("%s: %s", xhci_ring_type_string(__entry->type),
-		  xhci_decode_trb(__get_str(str), XHCI_MSG_MAX, __entry->field0, __entry->field1,
-					__entry->field2, __entry->field3)
+		  xhci_decode_trb(__get_buf(XHCI_MSG_MAX), XHCI_MSG_MAX, __entry->field0,
+				  __entry->field1, __entry->field2, __entry->field3)
 	)
 );
 
@@ -322,7 +321,6 @@ DECLARE_EVENT_CLASS(xhci_log_ep_ctx,
 		__field(u32, info2)
 		__field(u64, deq)
 		__field(u32, tx_info)
-		__dynamic_array(char, str, XHCI_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->info = le32_to_cpu(ctx->ep_info);
@@ -330,7 +328,7 @@ DECLARE_EVENT_CLASS(xhci_log_ep_ctx,
 		__entry->deq = le64_to_cpu(ctx->deq);
 		__entry->tx_info = le32_to_cpu(ctx->tx_info);
 	),
-	TP_printk("%s", xhci_decode_ep_context(__get_str(str),
+	TP_printk("%s", xhci_decode_ep_context(__get_buf(XHCI_MSG_MAX),
 		__entry->info, __entry->info2, __entry->deq, __entry->tx_info)
 	)
 );
@@ -368,7 +366,6 @@ DECLARE_EVENT_CLASS(xhci_log_slot_ctx,
 		__field(u32, info2)
 		__field(u32, tt_info)
 		__field(u32, state)
-		__dynamic_array(char, str, XHCI_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->info = le32_to_cpu(ctx->dev_info);
@@ -376,7 +373,7 @@ DECLARE_EVENT_CLASS(xhci_log_slot_ctx,
 		__entry->tt_info = le64_to_cpu(ctx->tt_info);
 		__entry->state = le32_to_cpu(ctx->dev_state);
 	),
-	TP_printk("%s", xhci_decode_slot_context(__get_str(str),
+	TP_printk("%s", xhci_decode_slot_context(__get_buf(XHCI_MSG_MAX),
 			__entry->info, __entry->info2,
 			__entry->tt_info, __entry->state)
 	)
@@ -433,13 +430,12 @@ DECLARE_EVENT_CLASS(xhci_log_ctrl_ctx,
 	TP_STRUCT__entry(
 		__field(u32, drop)
 		__field(u32, add)
-		__dynamic_array(char, str, XHCI_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->drop = le32_to_cpu(ctrl_ctx->drop_flags);
 		__entry->add = le32_to_cpu(ctrl_ctx->add_flags);
 	),
-	TP_printk("%s", xhci_decode_ctrl_ctx(__get_str(str), __entry->drop, __entry->add)
+	TP_printk("%s", xhci_decode_ctrl_ctx(__get_buf(XHCI_MSG_MAX), __entry->drop, __entry->add)
 	)
 );
 
@@ -525,7 +521,6 @@ DECLARE_EVENT_CLASS(xhci_log_portsc,
 		    TP_STRUCT__entry(
 				     __field(u32, portnum)
 				     __field(u32, portsc)
-				     __dynamic_array(char, str, XHCI_MSG_MAX)
 				     ),
 		    TP_fast_assign(
 				   __entry->portnum = portnum;
@@ -533,7 +528,7 @@ DECLARE_EVENT_CLASS(xhci_log_portsc,
 				   ),
 		    TP_printk("port-%d: %s",
 			      __entry->portnum,
-			      xhci_decode_portsc(__get_str(str), __entry->portsc)
+			      xhci_decode_portsc(__get_buf(XHCI_MSG_MAX), __entry->portsc)
 			      )
 );
 
@@ -558,14 +553,13 @@ DECLARE_EVENT_CLASS(xhci_log_doorbell,
 	TP_STRUCT__entry(
 		__field(u32, slot)
 		__field(u32, doorbell)
-		__dynamic_array(char, str, XHCI_MSG_MAX)
 	),
 	TP_fast_assign(
 		__entry->slot = slot;
 		__entry->doorbell = doorbell;
 	),
 	TP_printk("Ring doorbell for %s",
-		  xhci_decode_doorbell(__get_str(str), __entry->slot, __entry->doorbell)
+		  xhci_decode_doorbell(__get_buf(XHCI_MSG_MAX), __entry->slot, __entry->doorbell)
 	)
 );
 
diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
index 6307bae9cddf..78790dc13c5f 100644
--- a/drivers/usb/host/xhci.c
+++ b/drivers/usb/host/xhci.c
@@ -322,192 +322,6 @@ static int xhci_disable_interrupter(struct xhci_interrupter *ir)
 	return 0;
 }
 
-#ifdef CONFIG_USB_PCI
-/*
- * Set up MSI
- */
-static int xhci_setup_msi(struct xhci_hcd *xhci)
-{
-	int ret;
-	/*
-	 * TODO:Check with MSI Soc for sysdev
-	 */
-	struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
-
-	ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
-	if (ret < 0) {
-		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
-				"failed to allocate MSI entry");
-		return ret;
-	}
-
-	ret = request_irq(pdev->irq, xhci_msi_irq,
-				0, "xhci_hcd", xhci_to_hcd(xhci));
-	if (ret) {
-		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
-				"disable MSI interrupt");
-		pci_free_irq_vectors(pdev);
-	}
-
-	return ret;
-}
-
-/*
- * Set up MSI-X
- */
-static int xhci_setup_msix(struct xhci_hcd *xhci)
-{
-	int i, ret;
-	struct usb_hcd *hcd = xhci_to_hcd(xhci);
-	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
-
-	/*
-	 * calculate number of msi-x vectors supported.
-	 * - HCS_MAX_INTRS: the max number of interrupts the host can handle,
-	 *   with max number of interrupters based on the xhci HCSPARAMS1.
-	 * - num_online_cpus: maximum msi-x vectors per CPUs core.
-	 *   Add additional 1 vector to ensure always available interrupt.
-	 */
-	xhci->msix_count = min(num_online_cpus() + 1,
-				HCS_MAX_INTRS(xhci->hcs_params1));
-
-	ret = pci_alloc_irq_vectors(pdev, xhci->msix_count, xhci->msix_count,
-			PCI_IRQ_MSIX);
-	if (ret < 0) {
-		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
-				"Failed to enable MSI-X");
-		return ret;
-	}
-
-	for (i = 0; i < xhci->msix_count; i++) {
-		ret = request_irq(pci_irq_vector(pdev, i), xhci_msi_irq, 0,
-				"xhci_hcd", xhci_to_hcd(xhci));
-		if (ret)
-			goto disable_msix;
-	}
-
-	hcd->msix_enabled = 1;
-	return ret;
-
-disable_msix:
-	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "disable MSI-X interrupt");
-	while (--i >= 0)
-		free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
-	pci_free_irq_vectors(pdev);
-	return ret;
-}
-
-/* Free any IRQs and disable MSI-X */
-static void xhci_cleanup_msix(struct xhci_hcd *xhci)
-{
-	struct usb_hcd *hcd = xhci_to_hcd(xhci);
-	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
-
-	if (xhci->quirks & XHCI_PLAT)
-		return;
-
-	/* return if using legacy interrupt */
-	if (hcd->irq > 0)
-		return;
-
-	if (hcd->msix_enabled) {
-		int i;
-
-		for (i = 0; i < xhci->msix_count; i++)
-			free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
-	} else {
-		free_irq(pci_irq_vector(pdev, 0), xhci_to_hcd(xhci));
-	}
-
-	pci_free_irq_vectors(pdev);
-	hcd->msix_enabled = 0;
-}
-
-static void __maybe_unused xhci_msix_sync_irqs(struct xhci_hcd *xhci)
-{
-	struct usb_hcd *hcd = xhci_to_hcd(xhci);
-
-	if (hcd->msix_enabled) {
-		struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
-		int i;
-
-		for (i = 0; i < xhci->msix_count; i++)
-			synchronize_irq(pci_irq_vector(pdev, i));
-	}
-}
-
-static int xhci_try_enable_msi(struct usb_hcd *hcd)
-{
-	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
-	struct pci_dev  *pdev;
-	int ret;
-
-	/* The xhci platform device has set up IRQs through usb_add_hcd. */
-	if (xhci->quirks & XHCI_PLAT)
-		return 0;
-
-	pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
-	/*
-	 * Some Fresco Logic host controllers advertise MSI, but fail to
-	 * generate interrupts.  Don't even try to enable MSI.
-	 */
-	if (xhci->quirks & XHCI_BROKEN_MSI)
-		goto legacy_irq;
-
-	/* unregister the legacy interrupt */
-	if (hcd->irq)
-		free_irq(hcd->irq, hcd);
-	hcd->irq = 0;
-
-	ret = xhci_setup_msix(xhci);
-	if (ret)
-		/* fall back to msi*/
-		ret = xhci_setup_msi(xhci);
-
-	if (!ret) {
-		hcd->msi_enabled = 1;
-		return 0;
-	}
-
-	if (!pdev->irq) {
-		xhci_err(xhci, "No msi-x/msi found and no IRQ in BIOS\n");
-		return -EINVAL;
-	}
-
- legacy_irq:
-	if (!strlen(hcd->irq_descr))
-		snprintf(hcd->irq_descr, sizeof(hcd->irq_descr), "%s:usb%d",
-			 hcd->driver->description, hcd->self.busnum);
-
-	/* fall back to legacy interrupt*/
-	ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,
-			hcd->irq_descr, hcd);
-	if (ret) {
-		xhci_err(xhci, "request interrupt %d failed\n",
-				pdev->irq);
-		return ret;
-	}
-	hcd->irq = pdev->irq;
-	return 0;
-}
-
-#else
-
-static inline int xhci_try_enable_msi(struct usb_hcd *hcd)
-{
-	return 0;
-}
-
-static inline void xhci_cleanup_msix(struct xhci_hcd *xhci)
-{
-}
-
-static inline void xhci_msix_sync_irqs(struct xhci_hcd *xhci)
-{
-}
-
-#endif
-
 static void compliance_mode_recovery(struct timer_list *t)
 {
 	struct xhci_hcd *xhci;
@@ -705,10 +519,6 @@ int xhci_run(struct usb_hcd *hcd)
 
 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run");
 
-	ret = xhci_try_enable_msi(hcd);
-	if (ret)
-		return ret;
-
 	temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue);
 	temp_64 &= ~ERST_PTR_MASK;
 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
@@ -758,7 +568,7 @@ EXPORT_SYMBOL_GPL(xhci_run);
  * Disable device contexts, disable IRQs, and quiesce the HC.
  * Reset the HC, finish any completed transactions, and cleanup memory.
  */
-static void xhci_stop(struct usb_hcd *hcd)
+void xhci_stop(struct usb_hcd *hcd)
 {
 	u32 temp;
 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
@@ -781,8 +591,6 @@ static void xhci_stop(struct usb_hcd *hcd)
 	xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
 	spin_unlock_irq(&xhci->lock);
 
-	xhci_cleanup_msix(xhci);
-
 	/* Deleting Compliance Mode Recovery Timer */
 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
 			(!(xhci_all_ports_seen_u0(xhci)))) {
@@ -809,6 +617,7 @@ static void xhci_stop(struct usb_hcd *hcd)
 			readl(&xhci->op_regs->status));
 	mutex_unlock(&xhci->mutex);
 }
+EXPORT_SYMBOL_GPL(xhci_stop);
 
 /*
  * Shutdown HC (not bus-specific)
@@ -850,8 +659,6 @@ void xhci_shutdown(struct usb_hcd *hcd)
 
 	spin_unlock_irq(&xhci->lock);
 
-	xhci_cleanup_msix(xhci);
-
 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
 			"xhci_shutdown completed - status = %x",
 			readl(&xhci->op_regs->status));
@@ -1143,10 +950,6 @@ int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup)
 				__func__);
 	}
 
-	/* step 5: remove core well power */
-	/* synchronize irq when using MSI-X */
-	xhci_msix_sync_irqs(xhci);
-
 	return rc;
 }
 EXPORT_SYMBOL_GPL(xhci_suspend);
@@ -1250,7 +1053,6 @@ int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
 		spin_unlock_irq(&xhci->lock);
 		if (retval)
 			return retval;
-		xhci_cleanup_msix(xhci);
 
 		xhci_dbg(xhci, "// Disabling event ring interrupts\n");
 		temp = readl(&xhci->op_regs->status);
diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h
index 786002bb35db..08d721921b7b 100644
--- a/drivers/usb/host/xhci.h
+++ b/drivers/usb/host/xhci.h
@@ -2125,6 +2125,7 @@ int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us);
 int xhci_run(struct usb_hcd *hcd);
 int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks);
 void xhci_shutdown(struct usb_hcd *hcd);
+void xhci_stop(struct usb_hcd *hcd);
 void xhci_init_driver(struct hc_driver *drv,
 		      const struct xhci_driver_overrides *over);
 int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
diff --git a/drivers/usb/image/microtek.c b/drivers/usb/image/microtek.c
index 874ea4b54ced..8c8fa71c69c4 100644
--- a/drivers/usb/image/microtek.c
+++ b/drivers/usb/image/microtek.c
@@ -620,7 +620,7 @@ out:
 
 static DEF_SCSI_QCMD(mts_scsi_queuecommand)
 
-static struct scsi_host_template mts_scsi_host_template = {
+static const struct scsi_host_template mts_scsi_host_template = {
 	.module			= THIS_MODULE,
 	.name			= "microtekX6",
 	.proc_name		= "microtekX6",
diff --git a/drivers/usb/misc/Kconfig b/drivers/usb/misc/Kconfig
index a5f7652db7da..99b15b77dfd5 100644
--- a/drivers/usb/misc/Kconfig
+++ b/drivers/usb/misc/Kconfig
@@ -4,6 +4,35 @@
 #
 comment "USB Miscellaneous drivers"
 
+config USB_USS720
+	tristate "USS720 parport driver"
+	depends on PARPORT
+	select PARPORT_NOT_PC
+	help
+	  This driver is for USB parallel port adapters that use the Lucent
+	  Technologies USS-720 chip. These cables are plugged into your USB
+	  port and provide USB compatibility to peripherals designed with
+	  parallel port interfaces.
+
+	  The chip has two modes: automatic mode and manual mode. In automatic
+	  mode, it looks to the computer like a standard USB printer. Only
+	  printers may be connected to the USS-720 in this mode. The generic
+	  USB printer driver ("USB Printer support", above) may be used in
+	  that mode, and you can say N here if you want to use the chip only
+	  in this mode.
+
+	  Manual mode is not limited to printers, any parallel port
+	  device should work. This driver utilizes manual mode.
+	  Note however that some operations are three orders of magnitude
+	  slower than on a PCI/ISA Parallel Port, so timing critical
+	  applications might not work.
+
+	  Say Y here if you own an USS-720 USB->Parport cable and intend to
+	  connect anything other than a printer to it.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called uss720.
+
 config USB_EMI62
 	tristate "EMI 6|2m USB Audio interface support"
 	help
@@ -108,28 +137,6 @@ config USB_IDMOUSE
 
 	  See also <https://www.fs.tum.de/~echtler/idmouse/>.
 
-config USB_FTDI_ELAN
-	tristate "Elan PCMCIA CardBus Adapter USB Client"
-	help
-	  ELAN's Uxxx series of adapters are USB to PCMCIA CardBus adapters.
-	  Currently only the U132 adapter is available.
-
-	  The U132 is specifically designed for CardBus PC cards that contain
-	  an OHCI host controller. Typical PC cards are the Orange Mobile 3G
-	  Option GlobeTrotter Fusion card. The U132 adapter will *NOT* work
-	  with PC cards that do not contain an OHCI controller. To use a U132
-	  adapter you will need this "ftdi-elan" module as well as the "u132-hcd"
-	  module which is a USB host controller driver that talks to the OHCI
-	  controller within CardBus card that are inserted in the U132 adapter.
-
-	  This driver has been tested with a CardBus OHCI USB adapter, and
-	  worked with a USB PEN Drive inserted into the first USB port of
-	  the PCCARD. A rather pointless thing to do, but useful for testing.
-
-	  See also the USB_U132_HCD entry "Elan U132 Adapter Host Controller"
-
-	  It is safe to say M here.
-
 config USB_APPLEDISPLAY
 	tristate "Apple Cinema Display support"
 	select BACKLIGHT_CLASS_DEVICE
diff --git a/drivers/usb/misc/Makefile b/drivers/usb/misc/Makefile
index 93581baec3a8..1992cc284d8a 100644
--- a/drivers/usb/misc/Makefile
+++ b/drivers/usb/misc/Makefile
@@ -10,7 +10,6 @@ obj-$(CONFIG_USB_CYTHERM)		+= cytherm.o
 obj-$(CONFIG_USB_EMI26)			+= emi26.o
 obj-$(CONFIG_USB_EMI62)			+= emi62.o
 obj-$(CONFIG_USB_EZUSB_FX2)		+= ezusb.o
-obj-$(CONFIG_USB_FTDI_ELAN)		+= ftdi-elan.o
 obj-$(CONFIG_APPLE_MFI_FASTCHARGE)	+= apple-mfi-fastcharge.o
 obj-$(CONFIG_USB_IDMOUSE)		+= idmouse.o
 obj-$(CONFIG_USB_IOWARRIOR)		+= iowarrior.o
diff --git a/drivers/usb/misc/ftdi-elan.c b/drivers/usb/misc/ftdi-elan.c
deleted file mode 100644
index 8ce191e3a4c0..000000000000
--- a/drivers/usb/misc/ftdi-elan.c
+++ /dev/null
@@ -1,2780 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * USB FTDI client driver for Elan Digital Systems's Uxxx adapters
- *
- * Copyright(C) 2006 Elan Digital Systems Limited
- * http://www.elandigitalsystems.com
- *
- * Author and Maintainer - Tony Olech - Elan Digital Systems
- * tony.olech@elandigitalsystems.com
- *
- * This driver was written by Tony Olech(tony.olech@elandigitalsystems.com)
- * based on various USB client drivers in the 2.6.15 linux kernel
- * with constant reference to the 3rd Edition of Linux Device Drivers
- * published by O'Reilly
- *
- * The U132 adapter is a USB to CardBus adapter specifically designed
- * for PC cards that contain an OHCI host controller. Typical PC cards
- * are the Orange Mobile 3G Option GlobeTrotter Fusion card.
- *
- * The U132 adapter will *NOT *work with PC cards that do not contain
- * an OHCI controller. A simple way to test whether a PC card has an
- * OHCI controller as an interface is to insert the PC card directly
- * into a laptop(or desktop) with a CardBus slot and if "lspci" shows
- * a new USB controller and "lsusb -v" shows a new OHCI Host Controller
- * then there is a good chance that the U132 adapter will support the
- * PC card.(you also need the specific client driver for the PC card)
- *
- * Please inform the Author and Maintainer about any PC cards that
- * contain OHCI Host Controller and work when directly connected to
- * an embedded CardBus slot but do not work when they are connected
- * via an ELAN U132 adapter.
- *
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/list.h>
-#include <linux/ioctl.h>
-#include <linux/pci_ids.h>
-#include <linux/slab.h>
-#include <linux/module.h>
-#include <linux/kref.h>
-#include <linux/mutex.h>
-#include <linux/uaccess.h>
-#include <linux/usb.h>
-#include <linux/workqueue.h>
-#include <linux/platform_device.h>
-MODULE_AUTHOR("Tony Olech");
-MODULE_DESCRIPTION("FTDI ELAN driver");
-MODULE_LICENSE("GPL");
-#define INT_MODULE_PARM(n, v) static int n = v;module_param(n, int, 0444)
-static bool distrust_firmware = 1;
-module_param(distrust_firmware, bool, 0);
-MODULE_PARM_DESC(distrust_firmware,
-		 "true to distrust firmware power/overcurrent setup");
-extern struct platform_driver u132_platform_driver;
-/*
- * ftdi_module_lock exists to protect access to global variables
- *
- */
-static struct mutex ftdi_module_lock;
-static int ftdi_instances = 0;
-static struct list_head ftdi_static_list;
-/*
- * end of the global variables protected by ftdi_module_lock
- */
-#include "usb_u132.h"
-#include <asm/io.h>
-#include <linux/usb/hcd.h>
-
-/* FIXME ohci.h is ONLY for internal use by the OHCI driver.
- * If you're going to try stuff like this, you need to split
- * out shareable stuff (register declarations?) into its own
- * file, maybe name <linux/usb/ohci.h>
- */
-
-#include "../host/ohci.h"
-/* Define these values to match your devices*/
-#define USB_FTDI_ELAN_VENDOR_ID 0x0403
-#define USB_FTDI_ELAN_PRODUCT_ID 0xd6ea
-/* table of devices that work with this driver*/
-static const struct usb_device_id ftdi_elan_table[] = {
-	{USB_DEVICE(USB_FTDI_ELAN_VENDOR_ID, USB_FTDI_ELAN_PRODUCT_ID)},
-	{ /* Terminating entry */ }
-};
-
-MODULE_DEVICE_TABLE(usb, ftdi_elan_table);
-/* only the jtag(firmware upgrade device) interface requires
- * a device file and corresponding minor number, but the
- * interface is created unconditionally - I suppose it could
- * be configured or not according to a module parameter.
- * But since we(now) require one interface per device,
- * and since it unlikely that a normal installation would
- * require more than a couple of elan-ftdi devices, 8 seems
- * like a reasonable limit to have here, and if someone
- * really requires more than 8 devices, then they can frig the
- * code and recompile
- */
-#define USB_FTDI_ELAN_MINOR_BASE 192
-#define COMMAND_BITS 5
-#define COMMAND_SIZE (1<<COMMAND_BITS)
-#define COMMAND_MASK (COMMAND_SIZE-1)
-struct u132_command {
-	u8 header;
-	u16 length;
-	u8 address;
-	u8 width;
-	u32 value;
-	int follows;
-	void *buffer;
-};
-#define RESPOND_BITS 5
-#define RESPOND_SIZE (1<<RESPOND_BITS)
-#define RESPOND_MASK (RESPOND_SIZE-1)
-struct u132_respond {
-	u8 header;
-	u8 address;
-	u32 *value;
-	int *result;
-	struct completion wait_completion;
-};
-struct u132_target {
-	void *endp;
-	struct urb *urb;
-	int toggle_bits;
-	int error_count;
-	int condition_code;
-	int repeat_number;
-	int halted;
-	int skipped;
-	int actual;
-	int non_null;
-	int active;
-	int abandoning;
-	void (*callback)(void *endp, struct urb *urb, u8 *buf, int len,
-			 int toggle_bits, int error_count, int condition_code,
-			 int repeat_number, int halted, int skipped, int actual,
-			 int non_null);
-};
-/* Structure to hold all of our device specific stuff*/
-struct usb_ftdi {
-	struct list_head ftdi_list;
-	struct mutex u132_lock;
-	int command_next;
-	int command_head;
-	struct u132_command command[COMMAND_SIZE];
-	int respond_next;
-	int respond_head;
-	struct u132_respond respond[RESPOND_SIZE];
-	struct u132_target target[4];
-	char device_name[16];
-	unsigned synchronized:1;
-	unsigned enumerated:1;
-	unsigned registered:1;
-	unsigned initialized:1;
-	unsigned card_ejected:1;
-	int function;
-	int sequence_num;
-	int disconnected;
-	int gone_away;
-	int stuck_status;
-	int status_queue_delay;
-	struct semaphore sw_lock;
-	struct usb_device *udev;
-	struct usb_interface *interface;
-	struct usb_class_driver *class;
-	struct delayed_work status_work;
-	struct delayed_work command_work;
-	struct delayed_work respond_work;
-	struct u132_platform_data platform_data;
-	struct resource resources[0];
-	struct platform_device platform_dev;
-	unsigned char *bulk_in_buffer;
-	size_t bulk_in_size;
-	size_t bulk_in_last;
-	size_t bulk_in_left;
-	__u8 bulk_in_endpointAddr;
-	__u8 bulk_out_endpointAddr;
-	struct kref kref;
-	u32 controlreg;
-	u8 response[4 + 1024];
-	int expected;
-	int received;
-	int ed_found;
-};
-#define kref_to_usb_ftdi(d) container_of(d, struct usb_ftdi, kref)
-#define platform_device_to_usb_ftdi(d) container_of(d, struct usb_ftdi, \
-						    platform_dev)
-static struct usb_driver ftdi_elan_driver;
-static void ftdi_elan_delete(struct kref *kref)
-{
-	struct usb_ftdi *ftdi = kref_to_usb_ftdi(kref);
-	dev_warn(&ftdi->udev->dev, "FREEING ftdi=%p\n", ftdi);
-	usb_put_dev(ftdi->udev);
-	ftdi->disconnected += 1;
-	mutex_lock(&ftdi_module_lock);
-	list_del_init(&ftdi->ftdi_list);
-	ftdi_instances -= 1;
-	mutex_unlock(&ftdi_module_lock);
-	kfree(ftdi->bulk_in_buffer);
-	ftdi->bulk_in_buffer = NULL;
-	kfree(ftdi);
-}
-
-static void ftdi_elan_put_kref(struct usb_ftdi *ftdi)
-{
-	kref_put(&ftdi->kref, ftdi_elan_delete);
-}
-
-static void ftdi_elan_get_kref(struct usb_ftdi *ftdi)
-{
-	kref_get(&ftdi->kref);
-}
-
-static void ftdi_elan_init_kref(struct usb_ftdi *ftdi)
-{
-	kref_init(&ftdi->kref);
-}
-
-static void ftdi_status_requeue_work(struct usb_ftdi *ftdi, unsigned int delta)
-{
-	if (!schedule_delayed_work(&ftdi->status_work, delta))
-		kref_put(&ftdi->kref, ftdi_elan_delete);
-}
-
-static void ftdi_status_queue_work(struct usb_ftdi *ftdi, unsigned int delta)
-{
-	if (schedule_delayed_work(&ftdi->status_work, delta))
-		kref_get(&ftdi->kref);
-}
-
-static void ftdi_status_cancel_work(struct usb_ftdi *ftdi)
-{
-	if (cancel_delayed_work_sync(&ftdi->status_work))
-		kref_put(&ftdi->kref, ftdi_elan_delete);
-}
-
-static void ftdi_command_requeue_work(struct usb_ftdi *ftdi, unsigned int delta)
-{
-	if (!schedule_delayed_work(&ftdi->command_work, delta))
-		kref_put(&ftdi->kref, ftdi_elan_delete);
-}
-
-static void ftdi_command_queue_work(struct usb_ftdi *ftdi, unsigned int delta)
-{
-	if (schedule_delayed_work(&ftdi->command_work, delta))
-		kref_get(&ftdi->kref);
-}
-
-static void ftdi_command_cancel_work(struct usb_ftdi *ftdi)
-{
-	if (cancel_delayed_work_sync(&ftdi->command_work))
-		kref_put(&ftdi->kref, ftdi_elan_delete);
-}
-
-static void ftdi_response_requeue_work(struct usb_ftdi *ftdi,
-				       unsigned int delta)
-{
-	if (!schedule_delayed_work(&ftdi->respond_work, delta))
-		kref_put(&ftdi->kref, ftdi_elan_delete);
-}
-
-static void ftdi_respond_queue_work(struct usb_ftdi *ftdi, unsigned int delta)
-{
-	if (schedule_delayed_work(&ftdi->respond_work, delta))
-		kref_get(&ftdi->kref);
-}
-
-static void ftdi_response_cancel_work(struct usb_ftdi *ftdi)
-{
-	if (cancel_delayed_work_sync(&ftdi->respond_work))
-		kref_put(&ftdi->kref, ftdi_elan_delete);
-}
-
-void ftdi_elan_gone_away(struct platform_device *pdev)
-{
-	struct usb_ftdi *ftdi = platform_device_to_usb_ftdi(pdev);
-	ftdi->gone_away += 1;
-	ftdi_elan_put_kref(ftdi);
-}
-
-
-EXPORT_SYMBOL_GPL(ftdi_elan_gone_away);
-static void ftdi_release_platform_dev(struct device *dev)
-{
-	dev->parent = NULL;
-}
-
-static void ftdi_elan_do_callback(struct usb_ftdi *ftdi,
-				  struct u132_target *target, u8 *buffer, int length);
-static void ftdi_elan_kick_command_queue(struct usb_ftdi *ftdi);
-static void ftdi_elan_kick_respond_queue(struct usb_ftdi *ftdi);
-static int ftdi_elan_setupOHCI(struct usb_ftdi *ftdi);
-static int ftdi_elan_checkingPCI(struct usb_ftdi *ftdi);
-static int ftdi_elan_enumeratePCI(struct usb_ftdi *ftdi);
-static int ftdi_elan_synchronize(struct usb_ftdi *ftdi);
-static int ftdi_elan_stuck_waiting(struct usb_ftdi *ftdi);
-static int ftdi_elan_command_engine(struct usb_ftdi *ftdi);
-static int ftdi_elan_respond_engine(struct usb_ftdi *ftdi);
-static int ftdi_elan_hcd_init(struct usb_ftdi *ftdi)
-{
-	if (ftdi->platform_dev.dev.parent)
-		return -EBUSY;
-
-	ftdi_elan_get_kref(ftdi);
-	ftdi->platform_data.potpg = 100;
-	ftdi->platform_data.reset = NULL;
-	ftdi->platform_dev.id = ftdi->sequence_num;
-	ftdi->platform_dev.resource = ftdi->resources;
-	ftdi->platform_dev.num_resources = ARRAY_SIZE(ftdi->resources);
-	ftdi->platform_dev.dev.platform_data = &ftdi->platform_data;
-	ftdi->platform_dev.dev.parent = NULL;
-	ftdi->platform_dev.dev.release = ftdi_release_platform_dev;
-	ftdi->platform_dev.dev.dma_mask = NULL;
-	snprintf(ftdi->device_name, sizeof(ftdi->device_name), "u132_hcd");
-	ftdi->platform_dev.name = ftdi->device_name;
-	dev_info(&ftdi->udev->dev, "requesting module '%s'\n", "u132_hcd");
-	request_module("u132_hcd");
-	dev_info(&ftdi->udev->dev, "registering '%s'\n",
-		 ftdi->platform_dev.name);
-
-	return platform_device_register(&ftdi->platform_dev);
-}
-
-static void ftdi_elan_abandon_completions(struct usb_ftdi *ftdi)
-{
-	mutex_lock(&ftdi->u132_lock);
-	while (ftdi->respond_next > ftdi->respond_head) {
-		struct u132_respond *respond = &ftdi->respond[RESPOND_MASK &
-							      ftdi->respond_head++];
-		*respond->result = -ESHUTDOWN;
-		*respond->value = 0;
-		complete(&respond->wait_completion);
-	}
-	mutex_unlock(&ftdi->u132_lock);
-}
-
-static void ftdi_elan_abandon_targets(struct usb_ftdi *ftdi)
-{
-	int ed_number = 4;
-	mutex_lock(&ftdi->u132_lock);
-	while (ed_number-- > 0) {
-		struct u132_target *target = &ftdi->target[ed_number];
-		if (target->active == 1) {
-			target->condition_code = TD_DEVNOTRESP;
-			mutex_unlock(&ftdi->u132_lock);
-			ftdi_elan_do_callback(ftdi, target, NULL, 0);
-			mutex_lock(&ftdi->u132_lock);
-		}
-	}
-	ftdi->received = 0;
-	ftdi->expected = 4;
-	ftdi->ed_found = 0;
-	mutex_unlock(&ftdi->u132_lock);
-}
-
-static void ftdi_elan_flush_targets(struct usb_ftdi *ftdi)
-{
-	int ed_number = 4;
-	mutex_lock(&ftdi->u132_lock);
-	while (ed_number-- > 0) {
-		struct u132_target *target = &ftdi->target[ed_number];
-		target->abandoning = 1;
-	wait_1:if (target->active == 1) {
-			int command_size = ftdi->command_next -
-				ftdi->command_head;
-			if (command_size < COMMAND_SIZE) {
-				struct u132_command *command = &ftdi->command[
-					COMMAND_MASK & ftdi->command_next];
-				command->header = 0x80 | (ed_number << 5) | 0x4;
-				command->length = 0x00;
-				command->address = 0x00;
-				command->width = 0x00;
-				command->follows = 0;
-				command->value = 0;
-				command->buffer = &command->value;
-				ftdi->command_next += 1;
-				ftdi_elan_kick_command_queue(ftdi);
-			} else {
-				mutex_unlock(&ftdi->u132_lock);
-				msleep(100);
-				mutex_lock(&ftdi->u132_lock);
-				goto wait_1;
-			}
-		}
-	wait_2:if (target->active == 1) {
-			int command_size = ftdi->command_next -
-				ftdi->command_head;
-			if (command_size < COMMAND_SIZE) {
-				struct u132_command *command = &ftdi->command[
-					COMMAND_MASK & ftdi->command_next];
-				command->header = 0x90 | (ed_number << 5);
-				command->length = 0x00;
-				command->address = 0x00;
-				command->width = 0x00;
-				command->follows = 0;
-				command->value = 0;
-				command->buffer = &command->value;
-				ftdi->command_next += 1;
-				ftdi_elan_kick_command_queue(ftdi);
-			} else {
-				mutex_unlock(&ftdi->u132_lock);
-				msleep(100);
-				mutex_lock(&ftdi->u132_lock);
-				goto wait_2;
-			}
-		}
-	}
-	ftdi->received = 0;
-	ftdi->expected = 4;
-	ftdi->ed_found = 0;
-	mutex_unlock(&ftdi->u132_lock);
-}
-
-static void ftdi_elan_cancel_targets(struct usb_ftdi *ftdi)
-{
-	int ed_number = 4;
-	mutex_lock(&ftdi->u132_lock);
-	while (ed_number-- > 0) {
-		struct u132_target *target = &ftdi->target[ed_number];
-		target->abandoning = 1;
-	wait:if (target->active == 1) {
-			int command_size = ftdi->command_next -
-				ftdi->command_head;
-			if (command_size < COMMAND_SIZE) {
-				struct u132_command *command = &ftdi->command[
-					COMMAND_MASK & ftdi->command_next];
-				command->header = 0x80 | (ed_number << 5) | 0x4;
-				command->length = 0x00;
-				command->address = 0x00;
-				command->width = 0x00;
-				command->follows = 0;
-				command->value = 0;
-				command->buffer = &command->value;
-				ftdi->command_next += 1;
-				ftdi_elan_kick_command_queue(ftdi);
-			} else {
-				mutex_unlock(&ftdi->u132_lock);
-				msleep(100);
-				mutex_lock(&ftdi->u132_lock);
-				goto wait;
-			}
-		}
-	}
-	ftdi->received = 0;
-	ftdi->expected = 4;
-	ftdi->ed_found = 0;
-	mutex_unlock(&ftdi->u132_lock);
-}
-
-static void ftdi_elan_kick_command_queue(struct usb_ftdi *ftdi)
-{
-	ftdi_command_queue_work(ftdi, 0);
-}
-
-static void ftdi_elan_command_work(struct work_struct *work)
-{
-	struct usb_ftdi *ftdi =
-		container_of(work, struct usb_ftdi, command_work.work);
-
-	if (ftdi->disconnected > 0) {
-		ftdi_elan_put_kref(ftdi);
-		return;
-	} else {
-		int retval = ftdi_elan_command_engine(ftdi);
-		if (retval == -ESHUTDOWN) {
-			ftdi->disconnected += 1;
-		} else if (retval == -ENODEV) {
-			ftdi->disconnected += 1;
-		} else if (retval)
-			dev_err(&ftdi->udev->dev, "command error %d\n", retval);
-		ftdi_command_requeue_work(ftdi, msecs_to_jiffies(10));
-		return;
-	}
-}
-
-static void ftdi_elan_kick_respond_queue(struct usb_ftdi *ftdi)
-{
-	ftdi_respond_queue_work(ftdi, 0);
-}
-
-static void ftdi_elan_respond_work(struct work_struct *work)
-{
-	struct usb_ftdi *ftdi =
-		container_of(work, struct usb_ftdi, respond_work.work);
-	if (ftdi->disconnected > 0) {
-		ftdi_elan_put_kref(ftdi);
-		return;
-	} else {
-		int retval = ftdi_elan_respond_engine(ftdi);
-		if (retval == 0) {
-		} else if (retval == -ESHUTDOWN) {
-			ftdi->disconnected += 1;
-		} else if (retval == -ENODEV) {
-			ftdi->disconnected += 1;
-		} else if (retval == -EILSEQ) {
-			ftdi->disconnected += 1;
-		} else {
-			ftdi->disconnected += 1;
-			dev_err(&ftdi->udev->dev, "respond error %d\n", retval);
-		}
-		if (ftdi->disconnected > 0) {
-			ftdi_elan_abandon_completions(ftdi);
-			ftdi_elan_abandon_targets(ftdi);
-		}
-		ftdi_response_requeue_work(ftdi, msecs_to_jiffies(10));
-		return;
-	}
-}
-
-
-/*
- * the sw_lock is initially held and will be freed
- * after the FTDI has been synchronized
- *
- */
-static void ftdi_elan_status_work(struct work_struct *work)
-{
-	struct usb_ftdi *ftdi =
-		container_of(work, struct usb_ftdi, status_work.work);
-	int work_delay_in_msec = 0;
-	if (ftdi->disconnected > 0) {
-		ftdi_elan_put_kref(ftdi);
-		return;
-	} else if (ftdi->synchronized == 0) {
-		down(&ftdi->sw_lock);
-		if (ftdi_elan_synchronize(ftdi) == 0) {
-			ftdi->synchronized = 1;
-			ftdi_command_queue_work(ftdi, 1);
-			ftdi_respond_queue_work(ftdi, 1);
-			up(&ftdi->sw_lock);
-			work_delay_in_msec = 100;
-		} else {
-			dev_err(&ftdi->udev->dev, "synchronize failed\n");
-			up(&ftdi->sw_lock);
-			work_delay_in_msec = 10 *1000;
-		}
-	} else if (ftdi->stuck_status > 0) {
-		if (ftdi_elan_stuck_waiting(ftdi) == 0) {
-			ftdi->stuck_status = 0;
-			ftdi->synchronized = 0;
-		} else if ((ftdi->stuck_status++ % 60) == 1) {
-			dev_err(&ftdi->udev->dev, "WRONG type of card inserted - please remove\n");
-		} else
-			dev_err(&ftdi->udev->dev, "WRONG type of card inserted - checked %d times\n",
-				ftdi->stuck_status);
-		work_delay_in_msec = 100;
-	} else if (ftdi->enumerated == 0) {
-		if (ftdi_elan_enumeratePCI(ftdi) == 0) {
-			ftdi->enumerated = 1;
-			work_delay_in_msec = 250;
-		} else
-			work_delay_in_msec = 1000;
-	} else if (ftdi->initialized == 0) {
-		if (ftdi_elan_setupOHCI(ftdi) == 0) {
-			ftdi->initialized = 1;
-			work_delay_in_msec = 500;
-		} else {
-			dev_err(&ftdi->udev->dev, "initialized failed - trying again in 10 seconds\n");
-			work_delay_in_msec = 1 *1000;
-		}
-	} else if (ftdi->registered == 0) {
-		work_delay_in_msec = 10;
-		if (ftdi_elan_hcd_init(ftdi) == 0) {
-			ftdi->registered = 1;
-		} else
-			dev_err(&ftdi->udev->dev, "register failed\n");
-		work_delay_in_msec = 250;
-	} else {
-		if (ftdi_elan_checkingPCI(ftdi) == 0) {
-			work_delay_in_msec = 250;
-		} else if (ftdi->controlreg & 0x00400000) {
-			if (ftdi->gone_away > 0) {
-				dev_err(&ftdi->udev->dev, "PCI device eject confirmed platform_dev.dev.parent=%p platform_dev.dev=%p\n",
-					ftdi->platform_dev.dev.parent,
-					&ftdi->platform_dev.dev);
-				platform_device_unregister(&ftdi->platform_dev);
-				ftdi->platform_dev.dev.parent = NULL;
-				ftdi->registered = 0;
-				ftdi->enumerated = 0;
-				ftdi->card_ejected = 0;
-				ftdi->initialized = 0;
-				ftdi->gone_away = 0;
-			} else
-				ftdi_elan_flush_targets(ftdi);
-			work_delay_in_msec = 250;
-		} else {
-			dev_err(&ftdi->udev->dev, "PCI device has disappeared\n");
-			ftdi_elan_cancel_targets(ftdi);
-			work_delay_in_msec = 500;
-			ftdi->enumerated = 0;
-			ftdi->initialized = 0;
-		}
-	}
-	if (ftdi->disconnected > 0) {
-		ftdi_elan_put_kref(ftdi);
-		return;
-	} else {
-		ftdi_status_requeue_work(ftdi,
-					 msecs_to_jiffies(work_delay_in_msec));
-		return;
-	}
-}
-
-
-/*
- * file_operations for the jtag interface
- *
- * the usage count for the device is incremented on open()
- * and decremented on release()
- */
-static int ftdi_elan_open(struct inode *inode, struct file *file)
-{
-	int subminor;
-	struct usb_interface *interface;
-
-	subminor = iminor(inode);
-	interface = usb_find_interface(&ftdi_elan_driver, subminor);
-
-	if (!interface) {
-		pr_err("can't find device for minor %d\n", subminor);
-		return -ENODEV;
-	} else {
-		struct usb_ftdi *ftdi = usb_get_intfdata(interface);
-		if (!ftdi) {
-			return -ENODEV;
-		} else {
-			if (down_interruptible(&ftdi->sw_lock)) {
-				return -EINTR;
-			} else {
-				ftdi_elan_get_kref(ftdi);
-				file->private_data = ftdi;
-				return 0;
-			}
-		}
-	}
-}
-
-static int ftdi_elan_release(struct inode *inode, struct file *file)
-{
-	struct usb_ftdi *ftdi = file->private_data;
-	if (ftdi == NULL)
-		return -ENODEV;
-	up(&ftdi->sw_lock);        /* decrement the count on our device */
-	ftdi_elan_put_kref(ftdi);
-	return 0;
-}
-
-
-/*
- *
- * blocking bulk reads are used to get data from the device
- *
- */
-static ssize_t ftdi_elan_read(struct file *file, char __user *buffer,
-			      size_t count, loff_t *ppos)
-{
-	char data[30 *3 + 4];
-	char *d = data;
-	int m = (sizeof(data) - 1) / 3 - 1;
-	int bytes_read = 0;
-	int retry_on_empty = 10;
-	int retry_on_timeout = 5;
-	struct usb_ftdi *ftdi = file->private_data;
-	if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	}
-	data[0] = 0;
-have:if (ftdi->bulk_in_left > 0) {
-		if (count-- > 0) {
-			char *p = ++ftdi->bulk_in_last + ftdi->bulk_in_buffer;
-			ftdi->bulk_in_left -= 1;
-			if (bytes_read < m) {
-				d += sprintf(d, " %02X", 0x000000FF & *p);
-			} else if (bytes_read > m) {
-			} else
-				d += sprintf(d, " ..");
-			if (copy_to_user(buffer++, p, 1)) {
-				return -EFAULT;
-			} else {
-				bytes_read += 1;
-				goto have;
-			}
-		} else
-			return bytes_read;
-	}
-more:if (count > 0) {
-		int packet_bytes = 0;
-		int retval = usb_bulk_msg(ftdi->udev,
-					  usb_rcvbulkpipe(ftdi->udev, ftdi->bulk_in_endpointAddr),
-					  ftdi->bulk_in_buffer, ftdi->bulk_in_size,
-					  &packet_bytes, 50);
-		if (packet_bytes > 2) {
-			ftdi->bulk_in_left = packet_bytes - 2;
-			ftdi->bulk_in_last = 1;
-			goto have;
-		} else if (retval == -ETIMEDOUT) {
-			if (retry_on_timeout-- > 0) {
-				goto more;
-			} else if (bytes_read > 0) {
-				return bytes_read;
-			} else
-				return retval;
-		} else if (retval == 0) {
-			if (retry_on_empty-- > 0) {
-				goto more;
-			} else
-				return bytes_read;
-		} else
-			return retval;
-	} else
-		return bytes_read;
-}
-
-static void ftdi_elan_write_bulk_callback(struct urb *urb)
-{
-	struct usb_ftdi *ftdi = urb->context;
-	int status = urb->status;
-
-	if (status && !(status == -ENOENT || status == -ECONNRESET ||
-			status == -ESHUTDOWN)) {
-		dev_err(&ftdi->udev->dev,
-			"urb=%p write bulk status received: %d\n", urb, status);
-	}
-	usb_free_coherent(urb->dev, urb->transfer_buffer_length,
-			  urb->transfer_buffer, urb->transfer_dma);
-}
-
-static int fill_buffer_with_all_queued_commands(struct usb_ftdi *ftdi,
-						char *buf, int command_size, int total_size)
-{
-	int ed_commands = 0;
-	int b = 0;
-	int I = command_size;
-	int i = ftdi->command_head;
-	while (I-- > 0) {
-		struct u132_command *command = &ftdi->command[COMMAND_MASK &
-							      i++];
-		int F = command->follows;
-		u8 *f = command->buffer;
-		if (command->header & 0x80) {
-			ed_commands |= 1 << (0x3 & (command->header >> 5));
-		}
-		buf[b++] = command->header;
-		buf[b++] = (command->length >> 0) & 0x00FF;
-		buf[b++] = (command->length >> 8) & 0x00FF;
-		buf[b++] = command->address;
-		buf[b++] = command->width;
-		while (F-- > 0) {
-			buf[b++] = *f++;
-		}
-	}
-	return ed_commands;
-}
-
-static int ftdi_elan_total_command_size(struct usb_ftdi *ftdi, int command_size)
-{
-	int total_size = 0;
-	int I = command_size;
-	int i = ftdi->command_head;
-	while (I-- > 0) {
-		struct u132_command *command = &ftdi->command[COMMAND_MASK &
-							      i++];
-		total_size += 5 + command->follows;
-	}
-	return total_size;
-}
-
-static int ftdi_elan_command_engine(struct usb_ftdi *ftdi)
-{
-	int retval;
-	char *buf;
-	int ed_commands;
-	int total_size;
-	struct urb *urb;
-	int command_size = ftdi->command_next - ftdi->command_head;
-	if (command_size == 0)
-		return 0;
-	total_size = ftdi_elan_total_command_size(ftdi, command_size);
-	urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!urb)
-		return -ENOMEM;
-	buf = usb_alloc_coherent(ftdi->udev, total_size, GFP_KERNEL,
-				 &urb->transfer_dma);
-	if (!buf) {
-		dev_err(&ftdi->udev->dev, "could not get a buffer to write %d commands totaling %d bytes to the Uxxx\n",
-			command_size, total_size);
-		usb_free_urb(urb);
-		return -ENOMEM;
-	}
-	ed_commands = fill_buffer_with_all_queued_commands(ftdi, buf,
-							   command_size, total_size);
-	usb_fill_bulk_urb(urb, ftdi->udev, usb_sndbulkpipe(ftdi->udev,
-							   ftdi->bulk_out_endpointAddr), buf, total_size,
-			  ftdi_elan_write_bulk_callback, ftdi);
-	urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
-	if (ed_commands) {
-		char diag[40 *3 + 4];
-		char *d = diag;
-		int m = total_size;
-		u8 *c = buf;
-		int s = (sizeof(diag) - 1) / 3;
-		diag[0] = 0;
-		while (s-- > 0 && m-- > 0) {
-			if (s > 0 || m == 0) {
-				d += sprintf(d, " %02X", *c++);
-			} else
-				d += sprintf(d, " ..");
-		}
-	}
-	retval = usb_submit_urb(urb, GFP_KERNEL);
-	if (retval) {
-		dev_err(&ftdi->udev->dev, "failed %d to submit urb %p to write %d commands totaling %d bytes to the Uxxx\n",
-			retval, urb, command_size, total_size);
-		usb_free_coherent(ftdi->udev, total_size, buf, urb->transfer_dma);
-		usb_free_urb(urb);
-		return retval;
-	}
-	usb_free_urb(urb);        /* release our reference to this urb,
-				     the USB core will eventually free it entirely */
-	ftdi->command_head += command_size;
-	ftdi_elan_kick_respond_queue(ftdi);
-	return 0;
-}
-
-static void ftdi_elan_do_callback(struct usb_ftdi *ftdi,
-				  struct u132_target *target, u8 *buffer, int length)
-{
-	struct urb *urb = target->urb;
-	int halted = target->halted;
-	int skipped = target->skipped;
-	int actual = target->actual;
-	int non_null = target->non_null;
-	int toggle_bits = target->toggle_bits;
-	int error_count = target->error_count;
-	int condition_code = target->condition_code;
-	int repeat_number = target->repeat_number;
-	void (*callback) (void *, struct urb *, u8 *, int, int, int, int, int,
-			  int, int, int, int) = target->callback;
-	target->active -= 1;
-	target->callback = NULL;
-	(*callback) (target->endp, urb, buffer, length, toggle_bits,
-		     error_count, condition_code, repeat_number, halted, skipped,
-		     actual, non_null);
-}
-
-static char *have_ed_set_response(struct usb_ftdi *ftdi,
-				  struct u132_target *target, u16 ed_length, int ed_number, int ed_type,
-				  char *b)
-{
-	int payload = (ed_length >> 0) & 0x07FF;
-	mutex_lock(&ftdi->u132_lock);
-	target->actual = 0;
-	target->non_null = (ed_length >> 15) & 0x0001;
-	target->repeat_number = (ed_length >> 11) & 0x000F;
-	if (ed_type == 0x02 || ed_type == 0x03) {
-		if (payload == 0 || target->abandoning > 0) {
-			target->abandoning = 0;
-			mutex_unlock(&ftdi->u132_lock);
-			ftdi_elan_do_callback(ftdi, target, 4 + ftdi->response,
-					      payload);
-			ftdi->received = 0;
-			ftdi->expected = 4;
-			ftdi->ed_found = 0;
-			return ftdi->response;
-		} else {
-			ftdi->expected = 4 + payload;
-			ftdi->ed_found = 1;
-			mutex_unlock(&ftdi->u132_lock);
-			return b;
-		}
-	} else {
-		target->abandoning = 0;
-		mutex_unlock(&ftdi->u132_lock);
-		ftdi_elan_do_callback(ftdi, target, 4 + ftdi->response,
-				      payload);
-		ftdi->received = 0;
-		ftdi->expected = 4;
-		ftdi->ed_found = 0;
-		return ftdi->response;
-	}
-}
-
-static char *have_ed_get_response(struct usb_ftdi *ftdi,
-				  struct u132_target *target, u16 ed_length, int ed_number, int ed_type,
-				  char *b)
-{
-	mutex_lock(&ftdi->u132_lock);
-	target->condition_code = TD_DEVNOTRESP;
-	target->actual = (ed_length >> 0) & 0x01FF;
-	target->non_null = (ed_length >> 15) & 0x0001;
-	target->repeat_number = (ed_length >> 11) & 0x000F;
-	mutex_unlock(&ftdi->u132_lock);
-	if (target->active)
-		ftdi_elan_do_callback(ftdi, target, NULL, 0);
-	target->abandoning = 0;
-	ftdi->received = 0;
-	ftdi->expected = 4;
-	ftdi->ed_found = 0;
-	return ftdi->response;
-}
-
-
-/*
- * The engine tries to empty the FTDI fifo
- *
- * all responses found in the fifo data are dispatched thus
- * the response buffer can only ever hold a maximum sized
- * response from the Uxxx.
- *
- */
-static int ftdi_elan_respond_engine(struct usb_ftdi *ftdi)
-{
-	u8 *b = ftdi->response + ftdi->received;
-	int bytes_read = 0;
-	int retry_on_empty = 1;
-	int retry_on_timeout = 3;
-read:{
-		int packet_bytes = 0;
-		int retval = usb_bulk_msg(ftdi->udev,
-					  usb_rcvbulkpipe(ftdi->udev, ftdi->bulk_in_endpointAddr),
-					  ftdi->bulk_in_buffer, ftdi->bulk_in_size,
-					  &packet_bytes, 500);
-		char diag[30 *3 + 4];
-		char *d = diag;
-		int m = packet_bytes;
-		u8 *c = ftdi->bulk_in_buffer;
-		int s = (sizeof(diag) - 1) / 3;
-		diag[0] = 0;
-		while (s-- > 0 && m-- > 0) {
-			if (s > 0 || m == 0) {
-				d += sprintf(d, " %02X", *c++);
-			} else
-				d += sprintf(d, " ..");
-		}
-		if (packet_bytes > 2) {
-			ftdi->bulk_in_left = packet_bytes - 2;
-			ftdi->bulk_in_last = 1;
-			goto have;
-		} else if (retval == -ETIMEDOUT) {
-			if (retry_on_timeout-- > 0) {
-				dev_err(&ftdi->udev->dev, "TIMED OUT with packet_bytes = %d with total %d bytes%s\n",
-					packet_bytes, bytes_read, diag);
-				goto more;
-			} else if (bytes_read > 0) {
-				dev_err(&ftdi->udev->dev, "ONLY %d bytes%s\n",
-					bytes_read, diag);
-				return -ENOMEM;
-			} else {
-				dev_err(&ftdi->udev->dev, "TIMED OUT with packet_bytes = %d with total %d bytes%s\n",
-					packet_bytes, bytes_read, diag);
-				return -ENOMEM;
-			}
-		} else if (retval == -EILSEQ) {
-			dev_err(&ftdi->udev->dev, "error = %d with packet_bytes = %d with total %d bytes%s\n",
-				retval, packet_bytes, bytes_read, diag);
-			return retval;
-		} else if (retval) {
-			dev_err(&ftdi->udev->dev, "error = %d with packet_bytes = %d with total %d bytes%s\n",
-				retval, packet_bytes, bytes_read, diag);
-			return retval;
-		} else {
-			if (retry_on_empty-- > 0) {
-				goto more;
-			} else
-				return 0;
-		}
-	}
-more:{
-		goto read;
-	}
-have:if (ftdi->bulk_in_left > 0) {
-		u8 c = ftdi->bulk_in_buffer[++ftdi->bulk_in_last];
-		bytes_read += 1;
-		ftdi->bulk_in_left -= 1;
-		if (ftdi->received == 0 && c == 0xFF) {
-			goto have;
-		} else
-			*b++ = c;
-		if (++ftdi->received < ftdi->expected) {
-			goto have;
-		} else if (ftdi->ed_found) {
-			int ed_number = (ftdi->response[0] >> 5) & 0x03;
-			u16 ed_length = (ftdi->response[2] << 8) |
-				ftdi->response[1];
-			struct u132_target *target = &ftdi->target[ed_number];
-			int payload = (ed_length >> 0) & 0x07FF;
-			char diag[30 *3 + 4];
-			char *d = diag;
-			int m = payload;
-			u8 *c = 4 + ftdi->response;
-			int s = (sizeof(diag) - 1) / 3;
-			diag[0] = 0;
-			while (s-- > 0 && m-- > 0) {
-				if (s > 0 || m == 0) {
-					d += sprintf(d, " %02X", *c++);
-				} else
-					d += sprintf(d, " ..");
-			}
-			ftdi_elan_do_callback(ftdi, target, 4 + ftdi->response,
-					      payload);
-			ftdi->received = 0;
-			ftdi->expected = 4;
-			ftdi->ed_found = 0;
-			b = ftdi->response;
-			goto have;
-		} else if (ftdi->expected == 8) {
-			u8 buscmd;
-			int respond_head = ftdi->respond_head++;
-			struct u132_respond *respond = &ftdi->respond[
-				RESPOND_MASK & respond_head];
-			u32 data = ftdi->response[7];
-			data <<= 8;
-			data |= ftdi->response[6];
-			data <<= 8;
-			data |= ftdi->response[5];
-			data <<= 8;
-			data |= ftdi->response[4];
-			*respond->value = data;
-			*respond->result = 0;
-			complete(&respond->wait_completion);
-			ftdi->received = 0;
-			ftdi->expected = 4;
-			ftdi->ed_found = 0;
-			b = ftdi->response;
-			buscmd = (ftdi->response[0] >> 0) & 0x0F;
-			if (buscmd == 0x00) {
-			} else if (buscmd == 0x02) {
-			} else if (buscmd == 0x06) {
-			} else if (buscmd == 0x0A) {
-			} else
-				dev_err(&ftdi->udev->dev, "Uxxx unknown(%0X) value = %08X\n",
-					buscmd, data);
-			goto have;
-		} else {
-			if ((ftdi->response[0] & 0x80) == 0x00) {
-				ftdi->expected = 8;
-				goto have;
-			} else {
-				int ed_number = (ftdi->response[0] >> 5) & 0x03;
-				int ed_type = (ftdi->response[0] >> 0) & 0x03;
-				u16 ed_length = (ftdi->response[2] << 8) |
-					ftdi->response[1];
-				struct u132_target *target = &ftdi->target[
-					ed_number];
-				target->halted = (ftdi->response[0] >> 3) &
-					0x01;
-				target->skipped = (ftdi->response[0] >> 2) &
-					0x01;
-				target->toggle_bits = (ftdi->response[3] >> 6)
-					& 0x03;
-				target->error_count = (ftdi->response[3] >> 4)
-					& 0x03;
-				target->condition_code = (ftdi->response[
-								  3] >> 0) & 0x0F;
-				if ((ftdi->response[0] & 0x10) == 0x00) {
-					b = have_ed_set_response(ftdi, target,
-								 ed_length, ed_number, ed_type,
-								 b);
-					goto have;
-				} else {
-					b = have_ed_get_response(ftdi, target,
-								 ed_length, ed_number, ed_type,
-								 b);
-					goto have;
-				}
-			}
-		}
-	} else
-		goto more;
-}
-
-
-/*
- * create a urb, and a buffer for it, and copy the data to the urb
- *
- */
-static ssize_t ftdi_elan_write(struct file *file,
-			       const char __user *user_buffer, size_t count,
-			       loff_t *ppos)
-{
-	int retval = 0;
-	struct urb *urb;
-	char *buf;
-	struct usb_ftdi *ftdi = file->private_data;
-
-	if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	}
-	if (count == 0) {
-		goto exit;
-	}
-	urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!urb) {
-		retval = -ENOMEM;
-		goto error_1;
-	}
-	buf = usb_alloc_coherent(ftdi->udev, count, GFP_KERNEL,
-				 &urb->transfer_dma);
-	if (!buf) {
-		retval = -ENOMEM;
-		goto error_2;
-	}
-	if (copy_from_user(buf, user_buffer, count)) {
-		retval = -EFAULT;
-		goto error_3;
-	}
-	usb_fill_bulk_urb(urb, ftdi->udev, usb_sndbulkpipe(ftdi->udev,
-							   ftdi->bulk_out_endpointAddr), buf, count,
-			  ftdi_elan_write_bulk_callback, ftdi);
-	urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
-	retval = usb_submit_urb(urb, GFP_KERNEL);
-	if (retval) {
-		dev_err(&ftdi->udev->dev,
-			"failed submitting write urb, error %d\n", retval);
-		goto error_3;
-	}
-	usb_free_urb(urb);
-
-exit:
-	return count;
-error_3:
-	usb_free_coherent(ftdi->udev, count, buf, urb->transfer_dma);
-error_2:
-	usb_free_urb(urb);
-error_1:
-	return retval;
-}
-
-static const struct file_operations ftdi_elan_fops = {
-	.owner = THIS_MODULE,
-	.llseek = no_llseek,
-	.read = ftdi_elan_read,
-	.write = ftdi_elan_write,
-	.open = ftdi_elan_open,
-	.release = ftdi_elan_release,
-};
-
-/*
- * usb class driver info in order to get a minor number from the usb core,
- * and to have the device registered with the driver core
- */
-static struct usb_class_driver ftdi_elan_jtag_class = {
-	.name = "ftdi-%d-jtag",
-	.fops = &ftdi_elan_fops,
-	.minor_base = USB_FTDI_ELAN_MINOR_BASE,
-};
-
-/*
- * the following definitions are for the
- * ELAN FPGA state machgine processor that
- * lies on the other side of the FTDI chip
- */
-#define cPCIu132rd 0x0
-#define cPCIu132wr 0x1
-#define cPCIiord 0x2
-#define cPCIiowr 0x3
-#define cPCImemrd 0x6
-#define cPCImemwr 0x7
-#define cPCIcfgrd 0xA
-#define cPCIcfgwr 0xB
-#define cPCInull 0xF
-#define cU132cmd_status 0x0
-#define cU132flash 0x1
-#define cPIDsetup 0x0
-#define cPIDout 0x1
-#define cPIDin 0x2
-#define cPIDinonce 0x3
-#define cCCnoerror 0x0
-#define cCCcrc 0x1
-#define cCCbitstuff 0x2
-#define cCCtoggle 0x3
-#define cCCstall 0x4
-#define cCCnoresp 0x5
-#define cCCbadpid1 0x6
-#define cCCbadpid2 0x7
-#define cCCdataoverrun 0x8
-#define cCCdataunderrun 0x9
-#define cCCbuffoverrun 0xC
-#define cCCbuffunderrun 0xD
-#define cCCnotaccessed 0xF
-static int ftdi_elan_write_reg(struct usb_ftdi *ftdi, u32 data)
-{
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		if (command_size < COMMAND_SIZE) {
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			command->header = 0x00 | cPCIu132wr;
-			command->length = 0x04;
-			command->address = 0x00;
-			command->width = 0x00;
-			command->follows = 4;
-			command->value = data;
-			command->buffer = &command->value;
-			ftdi->command_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			return 0;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-static int ftdi_elan_write_config(struct usb_ftdi *ftdi, int config_offset,
-				  u8 width, u32 data)
-{
-	u8 addressofs = config_offset / 4;
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		if (command_size < COMMAND_SIZE) {
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			command->header = 0x00 | (cPCIcfgwr & 0x0F);
-			command->length = 0x04;
-			command->address = addressofs;
-			command->width = 0x00 | (width & 0x0F);
-			command->follows = 4;
-			command->value = data;
-			command->buffer = &command->value;
-			ftdi->command_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			return 0;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-static int ftdi_elan_write_pcimem(struct usb_ftdi *ftdi, int mem_offset,
-				  u8 width, u32 data)
-{
-	u8 addressofs = mem_offset / 4;
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		if (command_size < COMMAND_SIZE) {
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			command->header = 0x00 | (cPCImemwr & 0x0F);
-			command->length = 0x04;
-			command->address = addressofs;
-			command->width = 0x00 | (width & 0x0F);
-			command->follows = 4;
-			command->value = data;
-			command->buffer = &command->value;
-			ftdi->command_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			return 0;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-int usb_ftdi_elan_write_pcimem(struct platform_device *pdev, int mem_offset,
-			       u8 width, u32 data)
-{
-	struct usb_ftdi *ftdi = platform_device_to_usb_ftdi(pdev);
-	return ftdi_elan_write_pcimem(ftdi, mem_offset, width, data);
-}
-
-
-EXPORT_SYMBOL_GPL(usb_ftdi_elan_write_pcimem);
-static int ftdi_elan_read_reg(struct usb_ftdi *ftdi, u32 *data)
-{
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		int respond_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		respond_size = ftdi->respond_next - ftdi->respond_head;
-		if (command_size < COMMAND_SIZE && respond_size < RESPOND_SIZE)
-		{
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			struct u132_respond *respond = &ftdi->respond[
-				RESPOND_MASK & ftdi->respond_next];
-			int result = -ENODEV;
-			respond->result = &result;
-			respond->header = command->header = 0x00 | cPCIu132rd;
-			command->length = 0x04;
-			respond->address = command->address = cU132cmd_status;
-			command->width = 0x00;
-			command->follows = 0;
-			command->value = 0;
-			command->buffer = NULL;
-			respond->value = data;
-			init_completion(&respond->wait_completion);
-			ftdi->command_next += 1;
-			ftdi->respond_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			wait_for_completion(&respond->wait_completion);
-			return result;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-static int ftdi_elan_read_config(struct usb_ftdi *ftdi, int config_offset,
-				 u8 width, u32 *data)
-{
-	u8 addressofs = config_offset / 4;
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		int respond_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		respond_size = ftdi->respond_next - ftdi->respond_head;
-		if (command_size < COMMAND_SIZE && respond_size < RESPOND_SIZE)
-		{
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			struct u132_respond *respond = &ftdi->respond[
-				RESPOND_MASK & ftdi->respond_next];
-			int result = -ENODEV;
-			respond->result = &result;
-			respond->header = command->header = 0x00 | (cPCIcfgrd &
-								    0x0F);
-			command->length = 0x04;
-			respond->address = command->address = addressofs;
-			command->width = 0x00 | (width & 0x0F);
-			command->follows = 0;
-			command->value = 0;
-			command->buffer = NULL;
-			respond->value = data;
-			init_completion(&respond->wait_completion);
-			ftdi->command_next += 1;
-			ftdi->respond_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			wait_for_completion(&respond->wait_completion);
-			return result;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-static int ftdi_elan_read_pcimem(struct usb_ftdi *ftdi, int mem_offset,
-				 u8 width, u32 *data)
-{
-	u8 addressofs = mem_offset / 4;
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		int respond_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		respond_size = ftdi->respond_next - ftdi->respond_head;
-		if (command_size < COMMAND_SIZE && respond_size < RESPOND_SIZE)
-		{
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			struct u132_respond *respond = &ftdi->respond[
-				RESPOND_MASK & ftdi->respond_next];
-			int result = -ENODEV;
-			respond->result = &result;
-			respond->header = command->header = 0x00 | (cPCImemrd &
-								    0x0F);
-			command->length = 0x04;
-			respond->address = command->address = addressofs;
-			command->width = 0x00 | (width & 0x0F);
-			command->follows = 0;
-			command->value = 0;
-			command->buffer = NULL;
-			respond->value = data;
-			init_completion(&respond->wait_completion);
-			ftdi->command_next += 1;
-			ftdi->respond_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			wait_for_completion(&respond->wait_completion);
-			return result;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-int usb_ftdi_elan_read_pcimem(struct platform_device *pdev, int mem_offset,
-			      u8 width, u32 *data)
-{
-	struct usb_ftdi *ftdi = platform_device_to_usb_ftdi(pdev);
-	if (ftdi->initialized == 0) {
-		return -ENODEV;
-	} else
-		return ftdi_elan_read_pcimem(ftdi, mem_offset, width, data);
-}
-
-
-EXPORT_SYMBOL_GPL(usb_ftdi_elan_read_pcimem);
-static int ftdi_elan_edset_setup(struct usb_ftdi *ftdi, u8 ed_number,
-				 void *endp, struct urb *urb, u8 address, u8 ep_number, u8 toggle_bits,
-				 void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-						   int toggle_bits, int error_count, int condition_code, int repeat_number,
-						   int halted, int skipped, int actual, int non_null))
-{
-	u8 ed = ed_number - 1;
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else if (ftdi->initialized == 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		if (command_size < COMMAND_SIZE) {
-			struct u132_target *target = &ftdi->target[ed];
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			command->header = 0x80 | (ed << 5);
-			command->length = 0x8007;
-			command->address = (toggle_bits << 6) | (ep_number << 2)
-				| (address << 0);
-			command->width = usb_maxpacket(urb->dev, urb->pipe);
-			command->follows = 8;
-			command->value = 0;
-			command->buffer = urb->setup_packet;
-			target->callback = callback;
-			target->endp = endp;
-			target->urb = urb;
-			target->active = 1;
-			ftdi->command_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			return 0;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-int usb_ftdi_elan_edset_setup(struct platform_device *pdev, u8 ed_number,
-			      void *endp, struct urb *urb, u8 address, u8 ep_number, u8 toggle_bits,
-			      void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-						int toggle_bits, int error_count, int condition_code, int repeat_number,
-						int halted, int skipped, int actual, int non_null))
-{
-	struct usb_ftdi *ftdi = platform_device_to_usb_ftdi(pdev);
-	return ftdi_elan_edset_setup(ftdi, ed_number, endp, urb, address,
-				     ep_number, toggle_bits, callback);
-}
-
-
-EXPORT_SYMBOL_GPL(usb_ftdi_elan_edset_setup);
-static int ftdi_elan_edset_input(struct usb_ftdi *ftdi, u8 ed_number,
-				 void *endp, struct urb *urb, u8 address, u8 ep_number, u8 toggle_bits,
-				 void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-						   int toggle_bits, int error_count, int condition_code, int repeat_number,
-						   int halted, int skipped, int actual, int non_null))
-{
-	u8 ed = ed_number - 1;
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else if (ftdi->initialized == 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		if (command_size < COMMAND_SIZE) {
-			struct u132_target *target = &ftdi->target[ed];
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			u32 remaining_length = urb->transfer_buffer_length -
-				urb->actual_length;
-			command->header = 0x82 | (ed << 5);
-			if (remaining_length == 0) {
-				command->length = 0x0000;
-			} else if (remaining_length > 1024) {
-				command->length = 0x8000 | 1023;
-			} else
-				command->length = 0x8000 | (remaining_length -
-							    1);
-			command->address = (toggle_bits << 6) | (ep_number << 2)
-				| (address << 0);
-			command->width = usb_maxpacket(urb->dev, urb->pipe);
-			command->follows = 0;
-			command->value = 0;
-			command->buffer = NULL;
-			target->callback = callback;
-			target->endp = endp;
-			target->urb = urb;
-			target->active = 1;
-			ftdi->command_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			return 0;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-int usb_ftdi_elan_edset_input(struct platform_device *pdev, u8 ed_number,
-			      void *endp, struct urb *urb, u8 address, u8 ep_number, u8 toggle_bits,
-			      void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-						int toggle_bits, int error_count, int condition_code, int repeat_number,
-						int halted, int skipped, int actual, int non_null))
-{
-	struct usb_ftdi *ftdi = platform_device_to_usb_ftdi(pdev);
-	return ftdi_elan_edset_input(ftdi, ed_number, endp, urb, address,
-				     ep_number, toggle_bits, callback);
-}
-
-
-EXPORT_SYMBOL_GPL(usb_ftdi_elan_edset_input);
-static int ftdi_elan_edset_empty(struct usb_ftdi *ftdi, u8 ed_number,
-				 void *endp, struct urb *urb, u8 address, u8 ep_number, u8 toggle_bits,
-				 void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-						   int toggle_bits, int error_count, int condition_code, int repeat_number,
-						   int halted, int skipped, int actual, int non_null))
-{
-	u8 ed = ed_number - 1;
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else if (ftdi->initialized == 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		if (command_size < COMMAND_SIZE) {
-			struct u132_target *target = &ftdi->target[ed];
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			command->header = 0x81 | (ed << 5);
-			command->length = 0x0000;
-			command->address = (toggle_bits << 6) | (ep_number << 2)
-				| (address << 0);
-			command->width = usb_maxpacket(urb->dev, urb->pipe);
-			command->follows = 0;
-			command->value = 0;
-			command->buffer = NULL;
-			target->callback = callback;
-			target->endp = endp;
-			target->urb = urb;
-			target->active = 1;
-			ftdi->command_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			return 0;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-int usb_ftdi_elan_edset_empty(struct platform_device *pdev, u8 ed_number,
-			      void *endp, struct urb *urb, u8 address, u8 ep_number, u8 toggle_bits,
-			      void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-						int toggle_bits, int error_count, int condition_code, int repeat_number,
-						int halted, int skipped, int actual, int non_null))
-{
-	struct usb_ftdi *ftdi = platform_device_to_usb_ftdi(pdev);
-	return ftdi_elan_edset_empty(ftdi, ed_number, endp, urb, address,
-				     ep_number, toggle_bits, callback);
-}
-
-
-EXPORT_SYMBOL_GPL(usb_ftdi_elan_edset_empty);
-static int ftdi_elan_edset_output(struct usb_ftdi *ftdi, u8 ed_number,
-				  void *endp, struct urb *urb, u8 address, u8 ep_number, u8 toggle_bits,
-				  void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-						    int toggle_bits, int error_count, int condition_code, int repeat_number,
-						    int halted, int skipped, int actual, int non_null))
-{
-	u8 ed = ed_number - 1;
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else if (ftdi->initialized == 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		if (command_size < COMMAND_SIZE) {
-			u8 *b;
-			u16 urb_size;
-			int i = 0;
-			char data[30 *3 + 4];
-			char *d = data;
-			int m = (sizeof(data) - 1) / 3 - 1;
-			struct u132_target *target = &ftdi->target[ed];
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			command->header = 0x81 | (ed << 5);
-			command->address = (toggle_bits << 6) | (ep_number << 2)
-				| (address << 0);
-			command->width = usb_maxpacket(urb->dev, urb->pipe);
-			command->follows = min_t(u32, 1024,
-						 urb->transfer_buffer_length -
-						 urb->actual_length);
-			command->value = 0;
-			command->buffer = urb->transfer_buffer +
-				urb->actual_length;
-			command->length = 0x8000 | (command->follows - 1);
-			b = command->buffer;
-			urb_size = command->follows;
-			data[0] = 0;
-			while (urb_size-- > 0) {
-				if (i > m) {
-				} else if (i++ < m) {
-					int w = sprintf(d, " %02X", *b++);
-					d += w;
-				} else
-					d += sprintf(d, " ..");
-			}
-			target->callback = callback;
-			target->endp = endp;
-			target->urb = urb;
-			target->active = 1;
-			ftdi->command_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			return 0;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-int usb_ftdi_elan_edset_output(struct platform_device *pdev, u8 ed_number,
-			       void *endp, struct urb *urb, u8 address, u8 ep_number, u8 toggle_bits,
-			       void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-						 int toggle_bits, int error_count, int condition_code, int repeat_number,
-						 int halted, int skipped, int actual, int non_null))
-{
-	struct usb_ftdi *ftdi = platform_device_to_usb_ftdi(pdev);
-	return ftdi_elan_edset_output(ftdi, ed_number, endp, urb, address,
-				      ep_number, toggle_bits, callback);
-}
-
-
-EXPORT_SYMBOL_GPL(usb_ftdi_elan_edset_output);
-static int ftdi_elan_edset_single(struct usb_ftdi *ftdi, u8 ed_number,
-				  void *endp, struct urb *urb, u8 address, u8 ep_number, u8 toggle_bits,
-				  void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-						    int toggle_bits, int error_count, int condition_code, int repeat_number,
-						    int halted, int skipped, int actual, int non_null))
-{
-	u8 ed = ed_number - 1;
-wait:if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else if (ftdi->initialized == 0) {
-		return -ENODEV;
-	} else {
-		int command_size;
-		mutex_lock(&ftdi->u132_lock);
-		command_size = ftdi->command_next - ftdi->command_head;
-		if (command_size < COMMAND_SIZE) {
-			u32 remaining_length = urb->transfer_buffer_length -
-				urb->actual_length;
-			struct u132_target *target = &ftdi->target[ed];
-			struct u132_command *command = &ftdi->command[
-				COMMAND_MASK & ftdi->command_next];
-			command->header = 0x83 | (ed << 5);
-			if (remaining_length == 0) {
-				command->length = 0x0000;
-			} else if (remaining_length > 1024) {
-				command->length = 0x8000 | 1023;
-			} else
-				command->length = 0x8000 | (remaining_length -
-							    1);
-			command->address = (toggle_bits << 6) | (ep_number << 2)
-				| (address << 0);
-			command->width = usb_maxpacket(urb->dev, urb->pipe);
-			command->follows = 0;
-			command->value = 0;
-			command->buffer = NULL;
-			target->callback = callback;
-			target->endp = endp;
-			target->urb = urb;
-			target->active = 1;
-			ftdi->command_next += 1;
-			ftdi_elan_kick_command_queue(ftdi);
-			mutex_unlock(&ftdi->u132_lock);
-			return 0;
-		} else {
-			mutex_unlock(&ftdi->u132_lock);
-			msleep(100);
-			goto wait;
-		}
-	}
-}
-
-int usb_ftdi_elan_edset_single(struct platform_device *pdev, u8 ed_number,
-			       void *endp, struct urb *urb, u8 address, u8 ep_number, u8 toggle_bits,
-			       void (*callback) (void *endp, struct urb *urb, u8 *buf, int len,
-						 int toggle_bits, int error_count, int condition_code, int repeat_number,
-						 int halted, int skipped, int actual, int non_null))
-{
-	struct usb_ftdi *ftdi = platform_device_to_usb_ftdi(pdev);
-	return ftdi_elan_edset_single(ftdi, ed_number, endp, urb, address,
-				      ep_number, toggle_bits, callback);
-}
-
-
-EXPORT_SYMBOL_GPL(usb_ftdi_elan_edset_single);
-static int ftdi_elan_edset_flush(struct usb_ftdi *ftdi, u8 ed_number,
-				 void *endp)
-{
-	u8 ed = ed_number - 1;
-	if (ftdi->disconnected > 0) {
-		return -ENODEV;
-	} else if (ftdi->initialized == 0) {
-		return -ENODEV;
-	} else {
-		struct u132_target *target = &ftdi->target[ed];
-		mutex_lock(&ftdi->u132_lock);
-		if (target->abandoning > 0) {
-			mutex_unlock(&ftdi->u132_lock);
-			return 0;
-		} else {
-			target->abandoning = 1;
-		wait_1:if (target->active == 1) {
-				int command_size = ftdi->command_next -
-					ftdi->command_head;
-				if (command_size < COMMAND_SIZE) {
-					struct u132_command *command =
-						&ftdi->command[COMMAND_MASK &
-							       ftdi->command_next];
-					command->header = 0x80 | (ed << 5) |
-						0x4;
-					command->length = 0x00;
-					command->address = 0x00;
-					command->width = 0x00;
-					command->follows = 0;
-					command->value = 0;
-					command->buffer = &command->value;
-					ftdi->command_next += 1;
-					ftdi_elan_kick_command_queue(ftdi);
-				} else {
-					mutex_unlock(&ftdi->u132_lock);
-					msleep(100);
-					mutex_lock(&ftdi->u132_lock);
-					goto wait_1;
-				}
-			}
-			mutex_unlock(&ftdi->u132_lock);
-			return 0;
-		}
-	}
-}
-
-int usb_ftdi_elan_edset_flush(struct platform_device *pdev, u8 ed_number,
-			      void *endp)
-{
-	struct usb_ftdi *ftdi = platform_device_to_usb_ftdi(pdev);
-	return ftdi_elan_edset_flush(ftdi, ed_number, endp);
-}
-
-
-EXPORT_SYMBOL_GPL(usb_ftdi_elan_edset_flush);
-static int ftdi_elan_flush_input_fifo(struct usb_ftdi *ftdi)
-{
-	int retry_on_empty = 10;
-	int retry_on_timeout = 5;
-	int retry_on_status = 20;
-more:{
-		int packet_bytes = 0;
-		int retval = usb_bulk_msg(ftdi->udev,
-					  usb_rcvbulkpipe(ftdi->udev, ftdi->bulk_in_endpointAddr),
-					  ftdi->bulk_in_buffer, ftdi->bulk_in_size,
-					  &packet_bytes, 100);
-		if (packet_bytes > 2) {
-			char diag[30 *3 + 4];
-			char *d = diag;
-			int m = (sizeof(diag) - 1) / 3 - 1;
-			char *b = ftdi->bulk_in_buffer;
-			int bytes_read = 0;
-			diag[0] = 0;
-			while (packet_bytes-- > 0) {
-				char c = *b++;
-				if (bytes_read < m) {
-					d += sprintf(d, " %02X",
-						     0x000000FF & c);
-				} else if (bytes_read > m) {
-				} else
-					d += sprintf(d, " ..");
-				bytes_read += 1;
-				continue;
-			}
-			goto more;
-		} else if (packet_bytes > 1) {
-			char s1 = ftdi->bulk_in_buffer[0];
-			char s2 = ftdi->bulk_in_buffer[1];
-			if (s1 == 0x31 && s2 == 0x60) {
-				return 0;
-			} else if (retry_on_status-- > 0) {
-				goto more;
-			} else {
-				dev_err(&ftdi->udev->dev, "STATUS ERROR retry limit reached\n");
-				return -EFAULT;
-			}
-		} else if (packet_bytes > 0) {
-			char b1 = ftdi->bulk_in_buffer[0];
-			dev_err(&ftdi->udev->dev, "only one byte flushed from FTDI = %02X\n",
-				b1);
-			if (retry_on_status-- > 0) {
-				goto more;
-			} else {
-				dev_err(&ftdi->udev->dev, "STATUS ERROR retry limit reached\n");
-				return -EFAULT;
-			}
-		} else if (retval == -ETIMEDOUT) {
-			if (retry_on_timeout-- > 0) {
-				goto more;
-			} else {
-				dev_err(&ftdi->udev->dev, "TIMED OUT retry limit reached\n");
-				return -ENOMEM;
-			}
-		} else if (retval == 0) {
-			if (retry_on_empty-- > 0) {
-				goto more;
-			} else {
-				dev_err(&ftdi->udev->dev, "empty packet retry limit reached\n");
-				return -ENOMEM;
-			}
-		} else {
-			dev_err(&ftdi->udev->dev, "error = %d\n", retval);
-			return retval;
-		}
-	}
-	return -1;
-}
-
-
-/*
- * send the long flush sequence
- *
- */
-static int ftdi_elan_synchronize_flush(struct usb_ftdi *ftdi)
-{
-	int retval;
-	struct urb *urb;
-	char *buf;
-	int I = 257;
-	int i = 0;
-	urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!urb)
-		return -ENOMEM;
-	buf = usb_alloc_coherent(ftdi->udev, I, GFP_KERNEL, &urb->transfer_dma);
-	if (!buf) {
-		dev_err(&ftdi->udev->dev, "could not get a buffer for flush sequence\n");
-		usb_free_urb(urb);
-		return -ENOMEM;
-	}
-	while (I-- > 0)
-		buf[i++] = 0x55;
-	usb_fill_bulk_urb(urb, ftdi->udev, usb_sndbulkpipe(ftdi->udev,
-							   ftdi->bulk_out_endpointAddr), buf, i,
-			  ftdi_elan_write_bulk_callback, ftdi);
-	urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
-	retval = usb_submit_urb(urb, GFP_KERNEL);
-	if (retval) {
-		dev_err(&ftdi->udev->dev, "failed to submit urb containing the flush sequence\n");
-		usb_free_coherent(ftdi->udev, i, buf, urb->transfer_dma);
-		usb_free_urb(urb);
-		return -ENOMEM;
-	}
-	usb_free_urb(urb);
-	return 0;
-}
-
-
-/*
- * send the reset sequence
- *
- */
-static int ftdi_elan_synchronize_reset(struct usb_ftdi *ftdi)
-{
-	int retval;
-	struct urb *urb;
-	char *buf;
-	int I = 4;
-	int i = 0;
-	urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!urb)
-		return -ENOMEM;
-	buf = usb_alloc_coherent(ftdi->udev, I, GFP_KERNEL, &urb->transfer_dma);
-	if (!buf) {
-		dev_err(&ftdi->udev->dev, "could not get a buffer for the reset sequence\n");
-		usb_free_urb(urb);
-		return -ENOMEM;
-	}
-	buf[i++] = 0x55;
-	buf[i++] = 0xAA;
-	buf[i++] = 0x5A;
-	buf[i++] = 0xA5;
-	usb_fill_bulk_urb(urb, ftdi->udev, usb_sndbulkpipe(ftdi->udev,
-							   ftdi->bulk_out_endpointAddr), buf, i,
-			  ftdi_elan_write_bulk_callback, ftdi);
-	urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
-	retval = usb_submit_urb(urb, GFP_KERNEL);
-	if (retval) {
-		dev_err(&ftdi->udev->dev, "failed to submit urb containing the reset sequence\n");
-		usb_free_coherent(ftdi->udev, i, buf, urb->transfer_dma);
-		usb_free_urb(urb);
-		return -ENOMEM;
-	}
-	usb_free_urb(urb);
-	return 0;
-}
-
-static int ftdi_elan_synchronize(struct usb_ftdi *ftdi)
-{
-	int retval;
-	int long_stop = 10;
-	int retry_on_timeout = 5;
-	int retry_on_empty = 10;
-	retval = ftdi_elan_flush_input_fifo(ftdi);
-	if (retval)
-		return retval;
-	ftdi->bulk_in_left = 0;
-	ftdi->bulk_in_last = -1;
-	while (long_stop-- > 0) {
-		int read_stop;
-		int read_stuck;
-		retval = ftdi_elan_synchronize_flush(ftdi);
-		if (retval)
-			return retval;
-		retval = ftdi_elan_flush_input_fifo(ftdi);
-		if (retval)
-			return retval;
-	reset:retval = ftdi_elan_synchronize_reset(ftdi);
-		if (retval)
-			return retval;
-		read_stop = 100;
-		read_stuck = 10;
-	read:{
-			int packet_bytes = 0;
-			retval = usb_bulk_msg(ftdi->udev,
-					      usb_rcvbulkpipe(ftdi->udev,
-							      ftdi->bulk_in_endpointAddr),
-					      ftdi->bulk_in_buffer, ftdi->bulk_in_size,
-					      &packet_bytes, 500);
-			if (packet_bytes > 2) {
-				char diag[30 *3 + 4];
-				char *d = diag;
-				int m = (sizeof(diag) - 1) / 3 - 1;
-				char *b = ftdi->bulk_in_buffer;
-				int bytes_read = 0;
-				unsigned char c = 0;
-				diag[0] = 0;
-				while (packet_bytes-- > 0) {
-					c = *b++;
-					if (bytes_read < m) {
-						d += sprintf(d, " %02X", c);
-					} else if (bytes_read > m) {
-					} else
-						d += sprintf(d, " ..");
-					bytes_read += 1;
-					continue;
-				}
-				if (c == 0x7E) {
-					return 0;
-				} else {
-					if (c == 0x55) {
-						goto read;
-					} else if (read_stop-- > 0) {
-						goto read;
-					} else {
-						dev_err(&ftdi->udev->dev, "retry limit reached\n");
-						continue;
-					}
-				}
-			} else if (packet_bytes > 1) {
-				unsigned char s1 = ftdi->bulk_in_buffer[0];
-				unsigned char s2 = ftdi->bulk_in_buffer[1];
-				if (s1 == 0x31 && s2 == 0x00) {
-					if (read_stuck-- > 0) {
-						goto read;
-					} else
-						goto reset;
-				} else {
-					if (read_stop-- > 0) {
-						goto read;
-					} else {
-						dev_err(&ftdi->udev->dev, "retry limit reached\n");
-						continue;
-					}
-				}
-			} else if (packet_bytes > 0) {
-				if (read_stop-- > 0) {
-					goto read;
-				} else {
-					dev_err(&ftdi->udev->dev, "retry limit reached\n");
-					continue;
-				}
-			} else if (retval == -ETIMEDOUT) {
-				if (retry_on_timeout-- > 0) {
-					goto read;
-				} else {
-					dev_err(&ftdi->udev->dev, "TIMED OUT retry limit reached\n");
-					continue;
-				}
-			} else if (retval == 0) {
-				if (retry_on_empty-- > 0) {
-					goto read;
-				} else {
-					dev_err(&ftdi->udev->dev, "empty packet retry limit reached\n");
-					continue;
-				}
-			} else {
-				dev_err(&ftdi->udev->dev, "error = %d\n",
-					retval);
-				if (read_stop-- > 0) {
-					goto read;
-				} else {
-					dev_err(&ftdi->udev->dev, "retry limit reached\n");
-					continue;
-				}
-			}
-		}
-	}
-	dev_err(&ftdi->udev->dev, "failed to synchronize\n");
-	return -EFAULT;
-}
-
-static int ftdi_elan_stuck_waiting(struct usb_ftdi *ftdi)
-{
-	int retry_on_empty = 10;
-	int retry_on_timeout = 5;
-	int retry_on_status = 50;
-more:{
-		int packet_bytes = 0;
-		int retval = usb_bulk_msg(ftdi->udev,
-					  usb_rcvbulkpipe(ftdi->udev, ftdi->bulk_in_endpointAddr),
-					  ftdi->bulk_in_buffer, ftdi->bulk_in_size,
-					  &packet_bytes, 1000);
-		if (packet_bytes > 2) {
-			char diag[30 *3 + 4];
-			char *d = diag;
-			int m = (sizeof(diag) - 1) / 3 - 1;
-			char *b = ftdi->bulk_in_buffer;
-			int bytes_read = 0;
-			diag[0] = 0;
-			while (packet_bytes-- > 0) {
-				char c = *b++;
-				if (bytes_read < m) {
-					d += sprintf(d, " %02X",
-						     0x000000FF & c);
-				} else if (bytes_read > m) {
-				} else
-					d += sprintf(d, " ..");
-				bytes_read += 1;
-			}
-			goto more;
-		} else if (packet_bytes > 1) {
-			char s1 = ftdi->bulk_in_buffer[0];
-			char s2 = ftdi->bulk_in_buffer[1];
-			if (s1 == 0x31 && s2 == 0x60) {
-				return 0;
-			} else if (retry_on_status-- > 0) {
-				msleep(5);
-				goto more;
-			} else
-				return -EFAULT;
-		} else if (packet_bytes > 0) {
-			char b1 = ftdi->bulk_in_buffer[0];
-			dev_err(&ftdi->udev->dev, "only one byte flushed from FTDI = %02X\n", b1);
-			if (retry_on_status-- > 0) {
-				msleep(5);
-				goto more;
-			} else {
-				dev_err(&ftdi->udev->dev, "STATUS ERROR retry limit reached\n");
-				return -EFAULT;
-			}
-		} else if (retval == -ETIMEDOUT) {
-			if (retry_on_timeout-- > 0) {
-				goto more;
-			} else {
-				dev_err(&ftdi->udev->dev, "TIMED OUT retry limit reached\n");
-				return -ENOMEM;
-			}
-		} else if (retval == 0) {
-			if (retry_on_empty-- > 0) {
-				goto more;
-			} else {
-				dev_err(&ftdi->udev->dev, "empty packet retry limit reached\n");
-				return -ENOMEM;
-			}
-		} else {
-			dev_err(&ftdi->udev->dev, "error = %d\n", retval);
-			return -ENOMEM;
-		}
-	}
-	return -1;
-}
-
-static int ftdi_elan_checkingPCI(struct usb_ftdi *ftdi)
-{
-	int UxxxStatus = ftdi_elan_read_reg(ftdi, &ftdi->controlreg);
-	if (UxxxStatus)
-		return UxxxStatus;
-	if (ftdi->controlreg & 0x00400000) {
-		if (ftdi->card_ejected) {
-		} else {
-			ftdi->card_ejected = 1;
-			dev_err(&ftdi->udev->dev, "CARD EJECTED - controlreg = %08X\n",
-				ftdi->controlreg);
-		}
-		return -ENODEV;
-	} else {
-		u8 fn = ftdi->function - 1;
-		int activePCIfn = fn << 8;
-		u32 pcidata;
-		u32 pciVID;
-		u32 pciPID;
-		int reg = 0;
-		UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-						   &pcidata);
-		if (UxxxStatus)
-			return UxxxStatus;
-		pciVID = pcidata & 0xFFFF;
-		pciPID = (pcidata >> 16) & 0xFFFF;
-		if (pciVID == ftdi->platform_data.vendor && pciPID ==
-		    ftdi->platform_data.device) {
-			return 0;
-		} else {
-			dev_err(&ftdi->udev->dev, "vendor=%04X pciVID=%04X device=%04X pciPID=%04X\n",
-				ftdi->platform_data.vendor, pciVID,
-				ftdi->platform_data.device, pciPID);
-			return -ENODEV;
-		}
-	}
-}
-
-
-#define ftdi_read_pcimem(ftdi, member, data) ftdi_elan_read_pcimem(ftdi, \
-								   offsetof(struct ohci_regs, member), 0, data);
-#define ftdi_write_pcimem(ftdi, member, data) ftdi_elan_write_pcimem(ftdi, \
-								     offsetof(struct ohci_regs, member), 0, data);
-
-#define OHCI_CONTROL_INIT OHCI_CTRL_CBSR
-#define OHCI_INTR_INIT (OHCI_INTR_MIE | OHCI_INTR_UE | OHCI_INTR_RD |	\
-			OHCI_INTR_WDH)
-static int ftdi_elan_check_controller(struct usb_ftdi *ftdi, int quirk)
-{
-	int devices = 0;
-	int retval;
-	u32 hc_control;
-	int num_ports;
-	u32 control;
-	u32 rh_a = -1;
-	u32 status;
-	u32 fminterval;
-	u32 hc_fminterval;
-	u32 periodicstart;
-	u32 cmdstatus;
-	u32 roothub_a;
-	int mask = OHCI_INTR_INIT;
-	int sleep_time = 0;
-	int reset_timeout = 30;        /* ... allow extra time */
-	int temp;
-	retval = ftdi_write_pcimem(ftdi, intrdisable, OHCI_INTR_MIE);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(ftdi, control, &control);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(ftdi, roothub.a, &rh_a);
-	if (retval)
-		return retval;
-	num_ports = rh_a & RH_A_NDP;
-	retval = ftdi_read_pcimem(ftdi, fminterval, &hc_fminterval);
-	if (retval)
-		return retval;
-	hc_fminterval &= 0x3fff;
-	if (hc_fminterval != FI) {
-	}
-	hc_fminterval |= FSMP(hc_fminterval) << 16;
-	retval = ftdi_read_pcimem(ftdi, control, &hc_control);
-	if (retval)
-		return retval;
-	switch (hc_control & OHCI_CTRL_HCFS) {
-	case OHCI_USB_OPER:
-		sleep_time = 0;
-		break;
-	case OHCI_USB_SUSPEND:
-	case OHCI_USB_RESUME:
-		hc_control &= OHCI_CTRL_RWC;
-		hc_control |= OHCI_USB_RESUME;
-		sleep_time = 10;
-		break;
-	default:
-		hc_control &= OHCI_CTRL_RWC;
-		hc_control |= OHCI_USB_RESET;
-		sleep_time = 50;
-		break;
-	}
-	retval = ftdi_write_pcimem(ftdi, control, hc_control);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(ftdi, control, &control);
-	if (retval)
-		return retval;
-	msleep(sleep_time);
-	retval = ftdi_read_pcimem(ftdi, roothub.a, &roothub_a);
-	if (retval)
-		return retval;
-	if (!(roothub_a & RH_A_NPS)) {        /* power down each port */
-		for (temp = 0; temp < num_ports; temp++) {
-			retval = ftdi_write_pcimem(ftdi,
-						   roothub.portstatus[temp], RH_PS_LSDA);
-			if (retval)
-				return retval;
-		}
-	}
-	retval = ftdi_read_pcimem(ftdi, control, &control);
-	if (retval)
-		return retval;
-retry:retval = ftdi_read_pcimem(ftdi, cmdstatus, &status);
-	if (retval)
-		return retval;
-	retval = ftdi_write_pcimem(ftdi, cmdstatus, OHCI_HCR);
-	if (retval)
-		return retval;
-extra:{
-		retval = ftdi_read_pcimem(ftdi, cmdstatus, &status);
-		if (retval)
-			return retval;
-		if (0 != (status & OHCI_HCR)) {
-			if (--reset_timeout == 0) {
-				dev_err(&ftdi->udev->dev, "USB HC reset timed out!\n");
-				return -ENODEV;
-			} else {
-				msleep(5);
-				goto extra;
-			}
-		}
-	}
-	if (quirk & OHCI_QUIRK_INITRESET) {
-		retval = ftdi_write_pcimem(ftdi, control, hc_control);
-		if (retval)
-			return retval;
-		retval = ftdi_read_pcimem(ftdi, control, &control);
-		if (retval)
-			return retval;
-	}
-	retval = ftdi_write_pcimem(ftdi, ed_controlhead, 0x00000000);
-	if (retval)
-		return retval;
-	retval = ftdi_write_pcimem(ftdi, ed_bulkhead, 0x11000000);
-	if (retval)
-		return retval;
-	retval = ftdi_write_pcimem(ftdi, hcca, 0x00000000);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(ftdi, fminterval, &fminterval);
-	if (retval)
-		return retval;
-	retval = ftdi_write_pcimem(ftdi, fminterval,
-				   ((fminterval & FIT) ^ FIT) | hc_fminterval);
-	if (retval)
-		return retval;
-	retval = ftdi_write_pcimem(ftdi, periodicstart,
-				   ((9 *hc_fminterval) / 10) & 0x3fff);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(ftdi, fminterval, &fminterval);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(ftdi, periodicstart, &periodicstart);
-	if (retval)
-		return retval;
-	if (0 == (fminterval & 0x3fff0000) || 0 == periodicstart) {
-		if (!(quirk & OHCI_QUIRK_INITRESET)) {
-			quirk |= OHCI_QUIRK_INITRESET;
-			goto retry;
-		} else
-			dev_err(&ftdi->udev->dev, "init err(%08x %04x)\n",
-				fminterval, periodicstart);
-	}                        /* start controller operations */
-	hc_control &= OHCI_CTRL_RWC;
-	hc_control |= OHCI_CONTROL_INIT | OHCI_CTRL_BLE | OHCI_USB_OPER;
-	retval = ftdi_write_pcimem(ftdi, control, hc_control);
-	if (retval)
-		return retval;
-	retval = ftdi_write_pcimem(ftdi, cmdstatus, OHCI_BLF);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(ftdi, cmdstatus, &cmdstatus);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(ftdi, control, &control);
-	if (retval)
-		return retval;
-	retval = ftdi_write_pcimem(ftdi, roothub.status, RH_HS_DRWE);
-	if (retval)
-		return retval;
-	retval = ftdi_write_pcimem(ftdi, intrstatus, mask);
-	if (retval)
-		return retval;
-	retval = ftdi_write_pcimem(ftdi, intrdisable,
-				   OHCI_INTR_MIE | OHCI_INTR_OC | OHCI_INTR_RHSC | OHCI_INTR_FNO |
-				   OHCI_INTR_UE | OHCI_INTR_RD | OHCI_INTR_SF | OHCI_INTR_WDH |
-				   OHCI_INTR_SO);
-	if (retval)
-		return retval;        /* handle root hub init quirks ... */
-	retval = ftdi_read_pcimem(ftdi, roothub.a, &roothub_a);
-	if (retval)
-		return retval;
-	roothub_a &= ~(RH_A_PSM | RH_A_OCPM);
-	if (quirk & OHCI_QUIRK_SUPERIO) {
-		roothub_a |= RH_A_NOCP;
-		roothub_a &= ~(RH_A_POTPGT | RH_A_NPS);
-		retval = ftdi_write_pcimem(ftdi, roothub.a, roothub_a);
-		if (retval)
-			return retval;
-	} else if ((quirk & OHCI_QUIRK_AMD756) || distrust_firmware) {
-		roothub_a |= RH_A_NPS;
-		retval = ftdi_write_pcimem(ftdi, roothub.a, roothub_a);
-		if (retval)
-			return retval;
-	}
-	retval = ftdi_write_pcimem(ftdi, roothub.status, RH_HS_LPSC);
-	if (retval)
-		return retval;
-	retval = ftdi_write_pcimem(ftdi, roothub.b,
-				   (roothub_a & RH_A_NPS) ? 0 : RH_B_PPCM);
-	if (retval)
-		return retval;
-	retval = ftdi_read_pcimem(ftdi, control, &control);
-	if (retval)
-		return retval;
-	mdelay((roothub_a >> 23) & 0x1fe);
-	for (temp = 0; temp < num_ports; temp++) {
-		u32 portstatus;
-		retval = ftdi_read_pcimem(ftdi, roothub.portstatus[temp],
-					  &portstatus);
-		if (retval)
-			return retval;
-		if (1 & portstatus)
-			devices += 1;
-	}
-	return devices;
-}
-
-static int ftdi_elan_setup_controller(struct usb_ftdi *ftdi, int fn)
-{
-	u32 latence_timer;
-	int UxxxStatus;
-	u32 pcidata;
-	int reg = 0;
-	int activePCIfn = fn << 8;
-	UxxxStatus = ftdi_elan_write_reg(ftdi, 0x0000025FL | 0x2800);
-	if (UxxxStatus)
-		return UxxxStatus;
-	reg = 16;
-	UxxxStatus = ftdi_elan_write_config(ftdi, activePCIfn | reg, 0,
-					    0xFFFFFFFF);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-					   &pcidata);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_write_config(ftdi, activePCIfn | reg, 0,
-					    0xF0000000);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-					   &pcidata);
-	if (UxxxStatus)
-		return UxxxStatus;
-	reg = 12;
-	UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-					   &latence_timer);
-	if (UxxxStatus)
-		return UxxxStatus;
-	latence_timer &= 0xFFFF00FF;
-	latence_timer |= 0x00001600;
-	UxxxStatus = ftdi_elan_write_config(ftdi, activePCIfn | reg, 0x00,
-					    latence_timer);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-					   &pcidata);
-	if (UxxxStatus)
-		return UxxxStatus;
-	reg = 4;
-	UxxxStatus = ftdi_elan_write_config(ftdi, activePCIfn | reg, 0x00,
-					    0x06);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-					   &pcidata);
-	if (UxxxStatus)
-		return UxxxStatus;
-	for (reg = 0; reg <= 0x54; reg += 4) {
-		UxxxStatus = ftdi_elan_read_pcimem(ftdi, reg, 0, &pcidata);
-		if (UxxxStatus)
-			return UxxxStatus;
-	}
-	return 0;
-}
-
-static int ftdi_elan_close_controller(struct usb_ftdi *ftdi, int fn)
-{
-	u32 latence_timer;
-	int UxxxStatus;
-	u32 pcidata;
-	int reg = 0;
-	int activePCIfn = fn << 8;
-	UxxxStatus = ftdi_elan_write_reg(ftdi, 0x0000025FL | 0x2800);
-	if (UxxxStatus)
-		return UxxxStatus;
-	reg = 16;
-	UxxxStatus = ftdi_elan_write_config(ftdi, activePCIfn | reg, 0,
-					    0xFFFFFFFF);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-					   &pcidata);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_write_config(ftdi, activePCIfn | reg, 0,
-					    0x00000000);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-					   &pcidata);
-	if (UxxxStatus)
-		return UxxxStatus;
-	reg = 12;
-	UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-					   &latence_timer);
-	if (UxxxStatus)
-		return UxxxStatus;
-	latence_timer &= 0xFFFF00FF;
-	latence_timer |= 0x00001600;
-	UxxxStatus = ftdi_elan_write_config(ftdi, activePCIfn | reg, 0x00,
-					    latence_timer);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-					   &pcidata);
-	if (UxxxStatus)
-		return UxxxStatus;
-	reg = 4;
-	UxxxStatus = ftdi_elan_write_config(ftdi, activePCIfn | reg, 0x00,
-					    0x00);
-	if (UxxxStatus)
-		return UxxxStatus;
-	return ftdi_elan_read_config(ftdi, activePCIfn | reg, 0, &pcidata);
-}
-
-static int ftdi_elan_found_controller(struct usb_ftdi *ftdi, int fn, int quirk)
-{
-	int result;
-	int UxxxStatus;
-	UxxxStatus = ftdi_elan_setup_controller(ftdi, fn);
-	if (UxxxStatus)
-		return UxxxStatus;
-	result = ftdi_elan_check_controller(ftdi, quirk);
-	UxxxStatus = ftdi_elan_close_controller(ftdi, fn);
-	if (UxxxStatus)
-		return UxxxStatus;
-	return result;
-}
-
-static int ftdi_elan_enumeratePCI(struct usb_ftdi *ftdi)
-{
-	u32 controlreg;
-	u8 sensebits;
-	int UxxxStatus;
-	UxxxStatus = ftdi_elan_read_reg(ftdi, &controlreg);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_write_reg(ftdi, 0x00000000L);
-	if (UxxxStatus)
-		return UxxxStatus;
-	msleep(750);
-	UxxxStatus = ftdi_elan_write_reg(ftdi, 0x00000200L | 0x100);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_write_reg(ftdi, 0x00000200L | 0x500);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_reg(ftdi, &controlreg);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_write_reg(ftdi, 0x0000020CL | 0x000);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_write_reg(ftdi, 0x0000020DL | 0x000);
-	if (UxxxStatus)
-		return UxxxStatus;
-	msleep(250);
-	UxxxStatus = ftdi_elan_write_reg(ftdi, 0x0000020FL | 0x000);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_reg(ftdi, &controlreg);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_write_reg(ftdi, 0x0000025FL | 0x800);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_reg(ftdi, &controlreg);
-	if (UxxxStatus)
-		return UxxxStatus;
-	UxxxStatus = ftdi_elan_read_reg(ftdi, &controlreg);
-	if (UxxxStatus)
-		return UxxxStatus;
-	msleep(1000);
-	sensebits = (controlreg >> 16) & 0x000F;
-	if (0x0D == sensebits)
-		return 0;
-	else
-		return - ENXIO;
-}
-
-static int ftdi_elan_setupOHCI(struct usb_ftdi *ftdi)
-{
-	int UxxxStatus;
-	u32 pcidata;
-	int reg = 0;
-	u8 fn;
-	int activePCIfn = 0;
-	int max_devices = 0;
-	int controllers = 0;
-	int unrecognized = 0;
-	ftdi->function = 0;
-	for (fn = 0; (fn < 4); fn++) {
-		u32 pciVID = 0;
-		u32 pciPID = 0;
-		int devices = 0;
-		activePCIfn = fn << 8;
-		UxxxStatus = ftdi_elan_read_config(ftdi, activePCIfn | reg, 0,
-						   &pcidata);
-		if (UxxxStatus)
-			return UxxxStatus;
-		pciVID = pcidata & 0xFFFF;
-		pciPID = (pcidata >> 16) & 0xFFFF;
-		if ((pciVID == PCI_VENDOR_ID_OPTI) && (pciPID == 0xc861)) {
-			devices = ftdi_elan_found_controller(ftdi, fn, 0);
-			controllers += 1;
-		} else if ((pciVID == PCI_VENDOR_ID_NEC) && (pciPID == 0x0035))
-		{
-			devices = ftdi_elan_found_controller(ftdi, fn, 0);
-			controllers += 1;
-		} else if ((pciVID == PCI_VENDOR_ID_AL) && (pciPID == 0x5237)) {
-			devices = ftdi_elan_found_controller(ftdi, fn, 0);
-			controllers += 1;
-		} else if ((pciVID == PCI_VENDOR_ID_ATT) && (pciPID == 0x5802))
-		{
-			devices = ftdi_elan_found_controller(ftdi, fn, 0);
-			controllers += 1;
-		} else if (pciVID == PCI_VENDOR_ID_AMD && pciPID == 0x740c) {
-			devices = ftdi_elan_found_controller(ftdi, fn,
-							     OHCI_QUIRK_AMD756);
-			controllers += 1;
-		} else if (pciVID == PCI_VENDOR_ID_COMPAQ && pciPID == 0xa0f8) {
-			devices = ftdi_elan_found_controller(ftdi, fn,
-							     OHCI_QUIRK_ZFMICRO);
-			controllers += 1;
-		} else if (0 == pcidata) {
-		} else
-			unrecognized += 1;
-		if (devices > max_devices) {
-			max_devices = devices;
-			ftdi->function = fn + 1;
-			ftdi->platform_data.vendor = pciVID;
-			ftdi->platform_data.device = pciPID;
-		}
-	}
-	if (ftdi->function > 0) {
-		return ftdi_elan_setup_controller(ftdi,	ftdi->function - 1);
-	} else if (controllers > 0) {
-		return -ENXIO;
-	} else if (unrecognized > 0) {
-		return -ENXIO;
-	} else {
-		ftdi->enumerated = 0;
-		return -ENXIO;
-	}
-}
-
-
-/*
- * we use only the first bulk-in and bulk-out endpoints
- */
-static int ftdi_elan_probe(struct usb_interface *interface,
-			   const struct usb_device_id *id)
-{
-	struct usb_host_interface *iface_desc;
-	struct usb_endpoint_descriptor *bulk_in, *bulk_out;
-	int retval;
-	struct usb_ftdi *ftdi;
-
-	ftdi = kzalloc(sizeof(struct usb_ftdi), GFP_KERNEL);
-	if (!ftdi)
-		return -ENOMEM;
-
-	mutex_lock(&ftdi_module_lock);
-	list_add_tail(&ftdi->ftdi_list, &ftdi_static_list);
-	ftdi->sequence_num = ++ftdi_instances;
-	mutex_unlock(&ftdi_module_lock);
-	ftdi_elan_init_kref(ftdi);
-	sema_init(&ftdi->sw_lock, 1);
-	ftdi->udev = usb_get_dev(interface_to_usbdev(interface));
-	ftdi->interface = interface;
-	mutex_init(&ftdi->u132_lock);
-	ftdi->expected = 4;
-
-	iface_desc = interface->cur_altsetting;
-	retval = usb_find_common_endpoints(iface_desc,
-			&bulk_in, &bulk_out, NULL, NULL);
-	if (retval) {
-		dev_err(&ftdi->udev->dev, "Could not find both bulk-in and bulk-out endpoints\n");
-		goto error;
-	}
-
-	ftdi->bulk_in_size = usb_endpoint_maxp(bulk_in);
-	ftdi->bulk_in_endpointAddr = bulk_in->bEndpointAddress;
-	ftdi->bulk_in_buffer = kmalloc(ftdi->bulk_in_size, GFP_KERNEL);
-	if (!ftdi->bulk_in_buffer) {
-		retval = -ENOMEM;
-		goto error;
-	}
-
-	ftdi->bulk_out_endpointAddr = bulk_out->bEndpointAddress;
-
-	dev_info(&ftdi->udev->dev, "interface %d has I=%02X O=%02X\n",
-		 iface_desc->desc.bInterfaceNumber, ftdi->bulk_in_endpointAddr,
-		 ftdi->bulk_out_endpointAddr);
-	usb_set_intfdata(interface, ftdi);
-	if (iface_desc->desc.bInterfaceNumber == 0 &&
-	    ftdi->bulk_in_endpointAddr == 0x81 &&
-	    ftdi->bulk_out_endpointAddr == 0x02) {
-		retval = usb_register_dev(interface, &ftdi_elan_jtag_class);
-		if (retval) {
-			dev_err(&ftdi->udev->dev, "Not able to get a minor for this device\n");
-			usb_set_intfdata(interface, NULL);
-			retval = -ENOMEM;
-			goto error;
-		} else {
-			ftdi->class = &ftdi_elan_jtag_class;
-			dev_info(&ftdi->udev->dev, "USB FDTI=%p JTAG interface %d now attached to ftdi%d\n",
-				 ftdi, iface_desc->desc.bInterfaceNumber,
-				 interface->minor);
-			return 0;
-		}
-	} else if (iface_desc->desc.bInterfaceNumber == 1 &&
-		   ftdi->bulk_in_endpointAddr == 0x83 &&
-		   ftdi->bulk_out_endpointAddr == 0x04) {
-		ftdi->class = NULL;
-		dev_info(&ftdi->udev->dev, "USB FDTI=%p ELAN interface %d now activated\n",
-			 ftdi, iface_desc->desc.bInterfaceNumber);
-		INIT_DELAYED_WORK(&ftdi->status_work, ftdi_elan_status_work);
-		INIT_DELAYED_WORK(&ftdi->command_work, ftdi_elan_command_work);
-		INIT_DELAYED_WORK(&ftdi->respond_work, ftdi_elan_respond_work);
-		ftdi_status_queue_work(ftdi, msecs_to_jiffies(3 *1000));
-		return 0;
-	} else {
-		dev_err(&ftdi->udev->dev,
-			"Could not find ELAN's U132 device\n");
-		retval = -ENODEV;
-		goto error;
-	}
-error:if (ftdi) {
-		ftdi_elan_put_kref(ftdi);
-	}
-	return retval;
-}
-
-static void ftdi_elan_disconnect(struct usb_interface *interface)
-{
-	struct usb_ftdi *ftdi = usb_get_intfdata(interface);
-	ftdi->disconnected += 1;
-	if (ftdi->class) {
-		int minor = interface->minor;
-		struct usb_class_driver *class = ftdi->class;
-		usb_set_intfdata(interface, NULL);
-		usb_deregister_dev(interface, class);
-		dev_info(&ftdi->udev->dev, "USB FTDI U132 jtag interface on minor %d now disconnected\n",
-			 minor);
-	} else {
-		ftdi_status_cancel_work(ftdi);
-		ftdi_command_cancel_work(ftdi);
-		ftdi_response_cancel_work(ftdi);
-		ftdi_elan_abandon_completions(ftdi);
-		ftdi_elan_abandon_targets(ftdi);
-		if (ftdi->registered) {
-			platform_device_unregister(&ftdi->platform_dev);
-			ftdi->synchronized = 0;
-			ftdi->enumerated = 0;
-			ftdi->initialized = 0;
-			ftdi->registered = 0;
-		}
-		ftdi->disconnected += 1;
-		usb_set_intfdata(interface, NULL);
-		dev_info(&ftdi->udev->dev, "USB FTDI U132 host controller interface now disconnected\n");
-	}
-	ftdi_elan_put_kref(ftdi);
-}
-
-static struct usb_driver ftdi_elan_driver = {
-	.name = "ftdi-elan",
-	.probe = ftdi_elan_probe,
-	.disconnect = ftdi_elan_disconnect,
-	.id_table = ftdi_elan_table,
-};
-static int __init ftdi_elan_init(void)
-{
-	int result;
-	pr_info("driver %s\n", ftdi_elan_driver.name);
-	mutex_init(&ftdi_module_lock);
-	INIT_LIST_HEAD(&ftdi_static_list);
-	result = usb_register(&ftdi_elan_driver);
-	if (result) {
-		pr_err("usb_register failed. Error number %d\n", result);
-	}
-	return result;
-
-}
-
-static void __exit ftdi_elan_exit(void)
-{
-	struct usb_ftdi *ftdi;
-	struct usb_ftdi *temp;
-	usb_deregister(&ftdi_elan_driver);
-	pr_info("ftdi_u132 driver deregistered\n");
-	list_for_each_entry_safe(ftdi, temp, &ftdi_static_list, ftdi_list) {
-		ftdi_status_cancel_work(ftdi);
-		ftdi_command_cancel_work(ftdi);
-		ftdi_response_cancel_work(ftdi);
-	}
-}
-
-
-module_init(ftdi_elan_init);
-module_exit(ftdi_elan_exit);
diff --git a/drivers/usb/misc/sisusbvga/sisusbvga.c b/drivers/usb/misc/sisusbvga/sisusbvga.c
index 654a79fd3231..febf34f9f049 100644
--- a/drivers/usb/misc/sisusbvga/sisusbvga.c
+++ b/drivers/usb/misc/sisusbvga/sisusbvga.c
@@ -2778,6 +2778,20 @@ static int sisusb_probe(struct usb_interface *intf,
 	struct usb_device *dev = interface_to_usbdev(intf);
 	struct sisusb_usb_data *sisusb;
 	int retval = 0, i;
+	static const u8 ep_addresses[] = {
+		SISUSB_EP_GFX_IN | USB_DIR_IN,
+		SISUSB_EP_GFX_OUT | USB_DIR_OUT,
+		SISUSB_EP_GFX_BULK_OUT | USB_DIR_OUT,
+		SISUSB_EP_GFX_LBULK_OUT | USB_DIR_OUT,
+		SISUSB_EP_BRIDGE_IN | USB_DIR_IN,
+		SISUSB_EP_BRIDGE_OUT | USB_DIR_OUT,
+		0};
+
+	/* Are the expected endpoints present? */
+	if (!usb_check_bulk_endpoints(intf, ep_addresses)) {
+		dev_err(&intf->dev, "Invalid USB2VGA device\n");
+		return -EINVAL;
+	}
 
 	dev_info(&dev->dev, "USB2VGA dongle found at address %d\n",
 			dev->devnum);
diff --git a/drivers/usb/misc/usb251xb.c b/drivers/usb/misc/usb251xb.c
index e3abe67a155d..ce1da80d3365 100644
--- a/drivers/usb/misc/usb251xb.c
+++ b/drivers/usb/misc/usb251xb.c
@@ -377,7 +377,6 @@ out_err:
 	return err;
 }
 
-#ifdef CONFIG_OF
 static void usb251xb_get_ports_field(struct usb251xb *hub,
 				    const char *prop_name, u8 port_cnt,
 				    bool ds_only, u8 *fld)
@@ -410,10 +409,7 @@ static int usb251xb_get_ofdata(struct usb251xb *hub,
 		return -ENODEV;
 	}
 
-	if (of_get_property(np, "skip-config", NULL))
-		hub->skip_config = 1;
-	else
-		hub->skip_config = 0;
+	hub->skip_config = of_property_read_bool(np, "skip-config");
 
 	hub->gpio_reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(hub->gpio_reset))
@@ -431,40 +427,40 @@ static int usb251xb_get_ofdata(struct usb251xb *hub,
 		hub->device_id = USB251XB_DEF_DEVICE_ID;
 
 	hub->conf_data1 = USB251XB_DEF_CONFIG_DATA_1;
-	if (of_get_property(np, "self-powered", NULL)) {
+	if (of_property_read_bool(np, "self-powered")) {
 		hub->conf_data1 |= BIT(7);
 
 		/* Configure Over-Current sens when self-powered */
 		hub->conf_data1 &= ~BIT(2);
-		if (of_get_property(np, "ganged-sensing", NULL))
+		if (of_property_read_bool(np, "ganged-sensing"))
 			hub->conf_data1 &= ~BIT(1);
-		else if (of_get_property(np, "individual-sensing", NULL))
+		else if (of_property_read_bool(np, "individual-sensing"))
 			hub->conf_data1 |= BIT(1);
-	} else if (of_get_property(np, "bus-powered", NULL)) {
+	} else if (of_property_read_bool(np, "bus-powered")) {
 		hub->conf_data1 &= ~BIT(7);
 
 		/* Disable Over-Current sense when bus-powered */
 		hub->conf_data1 |= BIT(2);
 	}
 
-	if (of_get_property(np, "disable-hi-speed", NULL))
+	if (of_property_read_bool(np, "disable-hi-speed"))
 		hub->conf_data1 |= BIT(5);
 
-	if (of_get_property(np, "multi-tt", NULL))
+	if (of_property_read_bool(np, "multi-tt"))
 		hub->conf_data1 |= BIT(4);
-	else if (of_get_property(np, "single-tt", NULL))
+	else if (of_property_read_bool(np, "single-tt"))
 		hub->conf_data1 &= ~BIT(4);
 
-	if (of_get_property(np, "disable-eop", NULL))
+	if (of_property_read_bool(np, "disable-eop"))
 		hub->conf_data1 |= BIT(3);
 
-	if (of_get_property(np, "individual-port-switching", NULL))
+	if (of_property_read_bool(np, "individual-port-switching"))
 		hub->conf_data1 |= BIT(0);
-	else if (of_get_property(np, "ganged-port-switching", NULL))
+	else if (of_property_read_bool(np, "ganged-port-switching"))
 		hub->conf_data1 &= ~BIT(0);
 
 	hub->conf_data2 = USB251XB_DEF_CONFIG_DATA_2;
-	if (of_get_property(np, "dynamic-power-switching", NULL))
+	if (of_property_read_bool(np, "dynamic-power-switching"))
 		hub->conf_data2 |= BIT(7);
 
 	if (!of_property_read_u32(np, "oc-delay-us", &property_u32)) {
@@ -487,17 +483,17 @@ static int usb251xb_get_ofdata(struct usb251xb *hub,
 		}
 	}
 
-	if (of_get_property(np, "compound-device", NULL))
+	if (of_property_read_bool(np, "compound-device"))
 		hub->conf_data2 |= BIT(3);
 
 	hub->conf_data3 = USB251XB_DEF_CONFIG_DATA_3;
-	if (of_get_property(np, "port-mapping-mode", NULL))
+	if (of_property_read_bool(np, "port-mapping-mode"))
 		hub->conf_data3 |= BIT(3);
 
 	if (data->led_support && of_get_property(np, "led-usb-mode", NULL))
 		hub->conf_data3 &= ~BIT(1);
 
-	if (of_get_property(np, "string-support", NULL))
+	if (of_property_read_bool(np, "string-support"))
 		hub->conf_data3 |= BIT(0);
 
 	hub->non_rem_dev = USB251XB_DEF_NON_REMOVABLE_DEVICES;
@@ -626,13 +622,6 @@ static const struct of_device_id usb251xb_of_match[] = {
 	}
 };
 MODULE_DEVICE_TABLE(of, usb251xb_of_match);
-#else /* CONFIG_OF */
-static int usb251xb_get_ofdata(struct usb251xb *hub,
-			       const struct usb251xb_data *data)
-{
-	return 0;
-}
-#endif /* CONFIG_OF */
 
 static void usb251xb_regulator_disable_action(void *data)
 {
@@ -754,7 +743,7 @@ MODULE_DEVICE_TABLE(i2c, usb251xb_id);
 static struct i2c_driver usb251xb_i2c_driver = {
 	.driver = {
 		.name = DRIVER_NAME,
-		.of_match_table = of_match_ptr(usb251xb_of_match),
+		.of_match_table = usb251xb_of_match,
 		.pm = &usb251xb_pm_ops,
 	},
 	.probe_new = usb251xb_i2c_probe,
diff --git a/drivers/usb/misc/usb3503.c b/drivers/usb/misc/usb3503.c
index bd47c4437ca4..c6cfd1edaf76 100644
--- a/drivers/usb/misc/usb3503.c
+++ b/drivers/usb/misc/usb3503.c
@@ -46,34 +46,18 @@ struct usb3503 {
 	struct device		*dev;
 	struct clk		*clk;
 	u8	port_off_mask;
+	struct gpio_desc	*bypass;
 	struct gpio_desc	*intn;
 	struct gpio_desc 	*reset;
 	struct gpio_desc 	*connect;
 	bool	secondary_ref_clk;
 };
 
-static int usb3503_reset(struct usb3503 *hub, int state)
-{
-	if (!state && hub->connect)
-		gpiod_set_value_cansleep(hub->connect, 0);
-
-	if (hub->reset)
-		gpiod_set_value_cansleep(hub->reset, !state);
-
-	/* Wait T_HUBINIT == 4ms for hub logic to stabilize */
-	if (state)
-		usleep_range(4000, 10000);
-
-	return 0;
-}
-
 static int usb3503_connect(struct usb3503 *hub)
 {
 	struct device *dev = hub->dev;
 	int err;
 
-	usb3503_reset(hub, 1);
-
 	if (hub->regmap) {
 		/* SP_ILOCK: set connect_n, config_n for config */
 		err = regmap_write(hub->regmap, USB3503_SP_ILOCK,
@@ -126,25 +110,46 @@ static int usb3503_connect(struct usb3503 *hub)
 static int usb3503_switch_mode(struct usb3503 *hub, enum usb3503_mode mode)
 {
 	struct device *dev = hub->dev;
-	int err = 0;
+	int rst, bypass, conn;
 
 	switch (mode) {
 	case USB3503_MODE_HUB:
-		err = usb3503_connect(hub);
+		conn = 1;
+		rst = 0;
+		bypass = 0;
 		break;
-
 	case USB3503_MODE_STANDBY:
-		usb3503_reset(hub, 0);
+		conn = 0;
+		rst = 1;
+		bypass = 1;
 		dev_info(dev, "switched to STANDBY mode\n");
 		break;
-
+	case USB3503_MODE_BYPASS:
+		conn = 0;
+		rst = 0;
+		bypass = 1;
+		break;
 	default:
 		dev_err(dev, "unknown mode is requested\n");
-		err = -EINVAL;
-		break;
+		return -EINVAL;
 	}
 
-	return err;
+	if (!conn && hub->connect)
+		gpiod_set_value_cansleep(hub->connect, 0);
+
+	if (hub->reset)
+		gpiod_set_value_cansleep(hub->reset, rst);
+
+	if (hub->bypass)
+		gpiod_set_value_cansleep(hub->bypass, bypass);
+
+	if (conn) {
+		/* Wait T_HUBINIT == 4ms for hub logic to stabilize */
+		usleep_range(4000, 10000);
+		return usb3503_connect(hub);
+	}
+
+	return 0;
 }
 
 static const struct regmap_config usb3503_regmap_config = {
@@ -253,6 +258,14 @@ static int usb3503_probe(struct usb3503 *hub)
 	if (hub->connect)
 		gpiod_set_consumer_name(hub->connect, "usb3503 connect");
 
+	hub->bypass = devm_gpiod_get_optional(dev, "bypass", GPIOD_OUT_HIGH);
+	if (IS_ERR(hub->bypass)) {
+		err = PTR_ERR(hub->bypass);
+		goto err_clk;
+	}
+	if (hub->bypass)
+		gpiod_set_consumer_name(hub->bypass, "usb3503 bypass");
+
 	hub->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(hub->reset)) {
 		err = PTR_ERR(hub->reset);
@@ -388,6 +401,7 @@ MODULE_DEVICE_TABLE(i2c, usb3503_id);
 static const struct of_device_id usb3503_of_match[] = {
 	{ .compatible = "smsc,usb3503", },
 	{ .compatible = "smsc,usb3503a", },
+	{ .compatible = "smsc,usb3803", },
 	{},
 };
 MODULE_DEVICE_TABLE(of, usb3503_of_match);
diff --git a/drivers/usb/mon/mon_bin.c b/drivers/usb/mon/mon_bin.c
index abb1cd35d8a6..952c56789258 100644
--- a/drivers/usb/mon/mon_bin.c
+++ b/drivers/usb/mon/mon_bin.c
@@ -1379,7 +1379,7 @@ int __init mon_bin_init(void)
 {
 	int rc;
 
-	mon_bin_class = class_create(THIS_MODULE, "usbmon");
+	mon_bin_class = class_create("usbmon");
 	if (IS_ERR(mon_bin_class)) {
 		rc = PTR_ERR(mon_bin_class);
 		goto err_class;
diff --git a/drivers/usb/mtu3/mtu3.h b/drivers/usb/mtu3/mtu3.h
index 2d7b57e07eee..b4a7662dded5 100644
--- a/drivers/usb/mtu3/mtu3.h
+++ b/drivers/usb/mtu3/mtu3.h
@@ -90,7 +90,7 @@ struct mtu3_request;
  */
 #define EP0_RESPONSE_BUF  6
 
-#define BULK_CLKS_CNT	4
+#define BULK_CLKS_CNT	6
 
 /* device operated link and speed got from DEVICE_CONF register */
 enum mtu3_speed {
diff --git a/drivers/usb/mtu3/mtu3_dr.c b/drivers/usb/mtu3/mtu3_dr.c
index 9b8aded3d95e..8191b7ed3852 100644
--- a/drivers/usb/mtu3/mtu3_dr.c
+++ b/drivers/usb/mtu3/mtu3_dr.c
@@ -294,6 +294,7 @@ static int ssusb_role_sw_register(struct otg_switch_mtk *otg_sx)
 	role_sx_desc.get = ssusb_role_sw_get;
 	role_sx_desc.fwnode = dev_fwnode(dev);
 	role_sx_desc.driver_data = ssusb;
+	role_sx_desc.allow_userspace_control = true;
 	otg_sx->role_sw = usb_role_switch_register(dev, &role_sx_desc);
 	if (IS_ERR(otg_sx->role_sw))
 		return PTR_ERR(otg_sx->role_sw);
diff --git a/drivers/usb/mtu3/mtu3_gadget.c b/drivers/usb/mtu3/mtu3_gadget.c
index c0264d5426bf..ad0eeac4332d 100644
--- a/drivers/usb/mtu3/mtu3_gadget.c
+++ b/drivers/usb/mtu3/mtu3_gadget.c
@@ -23,7 +23,6 @@ __acquires(mep->mtu->lock)
 		req->status = status;
 
 	trace_mtu3_req_complete(mreq);
-	spin_unlock(&mtu->lock);
 
 	/* ep0 makes use of PIO, needn't unmap it */
 	if (mep->epnum)
@@ -32,6 +31,7 @@ __acquires(mep->mtu->lock)
 	dev_dbg(mtu->dev, "%s complete req: %p, sts %d, %d/%d\n",
 		mep->name, req, req->status, req->actual, req->length);
 
+	spin_unlock(&mtu->lock);
 	usb_gadget_giveback_request(&mep->ep, req);
 	spin_lock(&mtu->lock);
 }
diff --git a/drivers/usb/mtu3/mtu3_host.c b/drivers/usb/mtu3/mtu3_host.c
index f3903367a6a0..177d2caf887c 100644
--- a/drivers/usb/mtu3/mtu3_host.c
+++ b/drivers/usb/mtu3/mtu3_host.c
@@ -11,7 +11,7 @@
 #include <linux/irq.h>
 #include <linux/kernel.h>
 #include <linux/mfd/syscon.h>
-#include <linux/of_device.h>
+#include <linux/of_platform.h>
 #include <linux/regmap.h>
 
 #include "mtu3.h"
diff --git a/drivers/usb/mtu3/mtu3_plat.c b/drivers/usb/mtu3/mtu3_plat.c
index d78ae52b4e26..6f264b129243 100644
--- a/drivers/usb/mtu3/mtu3_plat.c
+++ b/drivers/usb/mtu3/mtu3_plat.c
@@ -234,6 +234,8 @@ static int get_ssusb_rscs(struct platform_device *pdev, struct ssusb_mtk *ssusb)
 	clks[1].id = "ref_ck";
 	clks[2].id = "mcu_ck";
 	clks[3].id = "dma_ck";
+	clks[4].id = "xhci_ck";
+	clks[5].id = "frmcnt_ck";
 	ret = devm_clk_bulk_get_optional(dev, BULK_CLKS_CNT, clks);
 	if (ret)
 		return ret;
diff --git a/drivers/usb/mtu3/mtu3_qmu.c b/drivers/usb/mtu3/mtu3_qmu.c
index a2fdab8b63b2..3d77408e3133 100644
--- a/drivers/usb/mtu3/mtu3_qmu.c
+++ b/drivers/usb/mtu3/mtu3_qmu.c
@@ -210,6 +210,7 @@ static struct qmu_gpd *advance_enq_gpd(struct mtu3_gpd_ring *ring)
 	return ring->enqueue;
 }
 
+/* @dequeue may be NULL if ring is unallocated or freed */
 static struct qmu_gpd *advance_deq_gpd(struct mtu3_gpd_ring *ring)
 {
 	if (ring->dequeue < ring->end)
@@ -221,7 +222,7 @@ static struct qmu_gpd *advance_deq_gpd(struct mtu3_gpd_ring *ring)
 }
 
 /* check if a ring is emtpy */
-static int gpd_ring_empty(struct mtu3_gpd_ring *ring)
+static bool gpd_ring_empty(struct mtu3_gpd_ring *ring)
 {
 	struct qmu_gpd *enq = ring->enqueue;
 	struct qmu_gpd *next;
@@ -467,6 +468,37 @@ static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)
 }
 
 /*
+ * when rx error happens (except zlperr), QMU will stop, and RQCPR saves
+ * the GPD encountered error, Done irq will arise after resuming QMU again.
+ */
+static void qmu_error_rx(struct mtu3 *mtu, u8 epnum)
+{
+	struct mtu3_ep *mep = mtu->out_eps + epnum;
+	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
+	struct qmu_gpd *gpd_current = NULL;
+	struct mtu3_request *mreq;
+	dma_addr_t cur_gpd_dma;
+
+	cur_gpd_dma = read_rxq_cur_addr(mtu->mac_base, epnum);
+	gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
+
+	mreq = next_request(mep);
+	if (!mreq || mreq->gpd != gpd_current) {
+		dev_err(mtu->dev, "no correct RX req is found\n");
+		return;
+	}
+
+	mreq->request.status = -EAGAIN;
+
+	/* by pass the current GDP */
+	gpd_current->dw0_info |= cpu_to_le32(GPD_FLAGS_BPS | GPD_FLAGS_HWO);
+	mtu3_qmu_resume(mep);
+
+	dev_dbg(mtu->dev, "%s EP%d, current=%p, req=%p\n",
+		__func__, epnum, gpd_current, mreq);
+}
+
+/*
  * NOTE: request list maybe is already empty as following case:
  * queue_tx --> qmu_interrupt(clear interrupt pending, schedule tasklet)-->
  * queue_tx --> process_tasklet(meanwhile, the second one is transferred,
@@ -491,7 +523,7 @@ static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)
 	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
 		__func__, epnum, gpd, gpd_current, ring->enqueue);
 
-	while (gpd != gpd_current && !GET_GPD_HWO(gpd)) {
+	while (gpd && gpd != gpd_current && !GET_GPD_HWO(gpd)) {
 
 		mreq = next_request(mep);
 
@@ -530,7 +562,7 @@ static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)
 	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
 		__func__, epnum, gpd, gpd_current, ring->enqueue);
 
-	while (gpd != gpd_current && !GET_GPD_HWO(gpd)) {
+	while (gpd && gpd != gpd_current && !GET_GPD_HWO(gpd)) {
 
 		mreq = next_request(mep);
 
@@ -571,14 +603,18 @@ static void qmu_exception_isr(struct mtu3 *mtu, u32 qmu_status)
 
 	if ((qmu_status & RXQ_CSERR_INT) || (qmu_status & RXQ_LENERR_INT)) {
 		errval = mtu3_readl(mbase, U3D_RQERRIR0);
+		mtu3_writel(mbase, U3D_RQERRIR0, errval);
+
 		for (i = 1; i < mtu->num_eps; i++) {
 			if (errval & QMU_RX_CS_ERR(i))
 				dev_err(mtu->dev, "Rx %d CS error!\n", i);
 
 			if (errval & QMU_RX_LEN_ERR(i))
 				dev_err(mtu->dev, "RX %d Length error\n", i);
+
+			if (errval & (QMU_RX_CS_ERR(i) | QMU_RX_LEN_ERR(i)))
+				qmu_error_rx(mtu, i);
 		}
-		mtu3_writel(mbase, U3D_RQERRIR0, errval);
 	}
 
 	if (qmu_status & RXQ_ZLPERR_INT) {
diff --git a/drivers/usb/musb/Kconfig b/drivers/usb/musb/Kconfig
index 3a1f4bcea80c..9a8cf3de0617 100644
--- a/drivers/usb/musb/Kconfig
+++ b/drivers/usb/musb/Kconfig
@@ -113,7 +113,7 @@ config USB_MUSB_MEDIATEK
 
 config USB_MUSB_POLARFIRE_SOC
 	tristate "Microchip PolarFire SoC platforms"
-	depends on SOC_MICROCHIP_POLARFIRE || COMPILE_TEST
+	depends on ARCH_MICROCHIP_POLARFIRE || COMPILE_TEST
 	depends on NOP_USB_XCEIV
 	select USB_MUSB_DUAL_ROLE
 	help
diff --git a/drivers/usb/musb/da8xx.c b/drivers/usb/musb/da8xx.c
index d47e5c94587b..912e32b78ac6 100644
--- a/drivers/usb/musb/da8xx.c
+++ b/drivers/usb/musb/da8xx.c
@@ -576,14 +576,12 @@ static int da8xx_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int da8xx_remove(struct platform_device *pdev)
+static void da8xx_remove(struct platform_device *pdev)
 {
 	struct da8xx_glue		*glue = platform_get_drvdata(pdev);
 
 	platform_device_unregister(glue->musb);
 	usb_phy_generic_unregister(glue->usb_phy);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -626,7 +624,7 @@ MODULE_DEVICE_TABLE(of, da8xx_id_table);
 
 static struct platform_driver da8xx_driver = {
 	.probe		= da8xx_probe,
-	.remove		= da8xx_remove,
+	.remove_new	= da8xx_remove,
 	.driver		= {
 		.name	= "musb-da8xx",
 		.pm = &da8xx_pm_ops,
diff --git a/drivers/usb/musb/jz4740.c b/drivers/usb/musb/jz4740.c
index c7b1d2a394d9..5aabdd7e2511 100644
--- a/drivers/usb/musb/jz4740.c
+++ b/drivers/usb/musb/jz4740.c
@@ -308,14 +308,12 @@ err_platform_device_put:
 	return ret;
 }
 
-static int jz4740_remove(struct platform_device *pdev)
+static void jz4740_remove(struct platform_device *pdev)
 {
 	struct jz4740_glue *glue = platform_get_drvdata(pdev);
 
 	platform_device_unregister(glue->pdev);
 	clk_disable_unprepare(glue->clk);
-
-	return 0;
 }
 
 static const struct of_device_id jz4740_musb_of_match[] = {
@@ -327,7 +325,7 @@ MODULE_DEVICE_TABLE(of, jz4740_musb_of_match);
 
 static struct platform_driver jz4740_driver = {
 	.probe		= jz4740_probe,
-	.remove		= jz4740_remove,
+	.remove_new	= jz4740_remove,
 	.driver		= {
 		.name	= "musb-jz4740",
 		.of_match_table = jz4740_musb_of_match,
diff --git a/drivers/usb/musb/mediatek.c b/drivers/usb/musb/mediatek.c
index 27b9bd258340..598ee5c0bf34 100644
--- a/drivers/usb/musb/mediatek.c
+++ b/drivers/usb/musb/mediatek.c
@@ -508,15 +508,13 @@ err_unregister_usb_phy:
 	return ret;
 }
 
-static int mtk_musb_remove(struct platform_device *pdev)
+static void mtk_musb_remove(struct platform_device *pdev)
 {
 	struct mtk_glue *glue = platform_get_drvdata(pdev);
 	struct platform_device *usb_phy = glue->usb_phy;
 
 	platform_device_unregister(glue->musb_pdev);
 	usb_phy_generic_unregister(usb_phy);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
@@ -529,7 +527,7 @@ MODULE_DEVICE_TABLE(of, mtk_musb_match);
 
 static struct platform_driver mtk_musb_driver = {
 	.probe = mtk_musb_probe,
-	.remove = mtk_musb_remove,
+	.remove_new = mtk_musb_remove,
 	.driver = {
 		   .name = "musb-mtk",
 		   .of_match_table = of_match_ptr(mtk_musb_match),
diff --git a/drivers/usb/musb/mpfs.c b/drivers/usb/musb/mpfs.c
index cea2e8108867..24b98716f7fc 100644
--- a/drivers/usb/musb/mpfs.c
+++ b/drivers/usb/musb/mpfs.c
@@ -235,15 +235,13 @@ err_phy_release:
 	return ret;
 }
 
-static int mpfs_remove(struct platform_device *pdev)
+static void mpfs_remove(struct platform_device *pdev)
 {
 	struct mpfs_glue *glue = platform_get_drvdata(pdev);
 
 	clk_disable_unprepare(glue->clk);
 	platform_device_unregister(glue->musb);
 	usb_phy_generic_unregister(pdev);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
@@ -256,7 +254,7 @@ MODULE_DEVICE_TABLE(of, mpfs_id_table);
 
 static struct platform_driver mpfs_musb_driver = {
 	.probe = mpfs_probe,
-	.remove = mpfs_remove,
+	.remove_new = mpfs_remove,
 	.driver = {
 		.name = "mpfs-musb",
 		.of_match_table = of_match_ptr(mpfs_id_table)
diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c
index 648bb6021c5e..d162afbbe19f 100644
--- a/drivers/usb/musb/musb_core.c
+++ b/drivers/usb/musb/musb_core.c
@@ -2621,7 +2621,7 @@ static int musb_probe(struct platform_device *pdev)
 	return musb_init_controller(dev, irq, base);
 }
 
-static int musb_remove(struct platform_device *pdev)
+static void musb_remove(struct platform_device *pdev)
 {
 	struct device	*dev = &pdev->dev;
 	struct musb	*musb = dev_to_musb(dev);
@@ -2657,7 +2657,6 @@ static int musb_remove(struct platform_device *pdev)
 	usb_phy_shutdown(musb->xceiv);
 	musb_free(musb);
 	device_init_wakeup(dev, 0);
-	return 0;
 }
 
 #ifdef	CONFIG_PM
@@ -2955,7 +2954,7 @@ static struct platform_driver musb_driver = {
 		.dev_groups	= musb_groups,
 	},
 	.probe		= musb_probe,
-	.remove		= musb_remove,
+	.remove_new	= musb_remove,
 };
 
 module_platform_driver(musb_driver);
diff --git a/drivers/usb/musb/musb_dsps.c b/drivers/usb/musb/musb_dsps.c
index f75cde0f2b43..9119b1d51370 100644
--- a/drivers/usb/musb/musb_dsps.c
+++ b/drivers/usb/musb/musb_dsps.c
@@ -915,7 +915,7 @@ err:
 	return ret;
 }
 
-static int dsps_remove(struct platform_device *pdev)
+static void dsps_remove(struct platform_device *pdev)
 {
 	struct dsps_glue *glue = platform_get_drvdata(pdev);
 
@@ -923,8 +923,6 @@ static int dsps_remove(struct platform_device *pdev)
 
 	pm_runtime_disable(&pdev->dev);
 	iounmap(glue->usbss_base);
-
-	return 0;
 }
 
 static const struct dsps_musb_wrapper am33xx_driver_data = {
@@ -1036,7 +1034,7 @@ static SIMPLE_DEV_PM_OPS(dsps_pm_ops, dsps_suspend, dsps_resume);
 
 static struct platform_driver dsps_usbss_driver = {
 	.probe		= dsps_probe,
-	.remove         = dsps_remove,
+	.remove_new     = dsps_remove,
 	.driver         = {
 		.name   = "musb-dsps",
 		.pm	= &dsps_pm_ops,
diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c
index 44a21ec865fb..b4a4c1df4e0d 100644
--- a/drivers/usb/musb/omap2430.c
+++ b/drivers/usb/musb/omap2430.c
@@ -334,7 +334,7 @@ static int omap2430_probe(struct platform_device *pdev)
 	 * Legacy SoCs using omap_device get confused if node is moved
 	 * because of interconnect properties mixed into the node.
 	 */
-	if (of_get_property(np, "ti,hwmods", NULL)) {
+	if (of_property_present(np, "ti,hwmods")) {
 		dev_warn(&pdev->dev, "please update to probe with ti-sysc\n");
 		populate_irqs = true;
 	} else {
@@ -471,14 +471,12 @@ err0:
 	return ret;
 }
 
-static int omap2430_remove(struct platform_device *pdev)
+static void omap2430_remove(struct platform_device *pdev)
 {
 	struct omap2430_glue *glue = platform_get_drvdata(pdev);
 
 	platform_device_unregister(glue->musb);
 	pm_runtime_disable(glue->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -610,7 +608,7 @@ MODULE_DEVICE_TABLE(of, omap2430_id_table);
 
 static struct platform_driver omap2430_driver = {
 	.probe		= omap2430_probe,
-	.remove		= omap2430_remove,
+	.remove_new	= omap2430_remove,
 	.driver		= {
 		.name	= "musb-omap2430",
 		.pm	= DEV_PM_OPS,
diff --git a/drivers/usb/musb/sunxi.c b/drivers/usb/musb/sunxi.c
index 9b622cd9b2bd..c5c6c4e09300 100644
--- a/drivers/usb/musb/sunxi.c
+++ b/drivers/usb/musb/sunxi.c
@@ -805,15 +805,13 @@ err_unregister_usb_phy:
 	return ret;
 }
 
-static int sunxi_musb_remove(struct platform_device *pdev)
+static void sunxi_musb_remove(struct platform_device *pdev)
 {
 	struct sunxi_glue *glue = platform_get_drvdata(pdev);
 	struct platform_device *usb_phy = glue->usb_phy;
 
 	platform_device_unregister(glue->musb_pdev);
 	usb_phy_generic_unregister(usb_phy);
-
-	return 0;
 }
 
 static const struct sunxi_musb_cfg sun4i_a10_musb_cfg = {
@@ -862,7 +860,7 @@ MODULE_DEVICE_TABLE(of, sunxi_musb_match);
 
 static struct platform_driver sunxi_musb_driver = {
 	.probe = sunxi_musb_probe,
-	.remove = sunxi_musb_remove,
+	.remove_new = sunxi_musb_remove,
 	.driver = {
 		.name = "musb-sunxi",
 		.of_match_table = sunxi_musb_match,
diff --git a/drivers/usb/musb/tusb6010.c b/drivers/usb/musb/tusb6010.c
index 5609b4e84d40..a1f29dbc62e6 100644
--- a/drivers/usb/musb/tusb6010.c
+++ b/drivers/usb/musb/tusb6010.c
@@ -1258,19 +1258,17 @@ static int tusb_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int tusb_remove(struct platform_device *pdev)
+static void tusb_remove(struct platform_device *pdev)
 {
 	struct tusb6010_glue		*glue = platform_get_drvdata(pdev);
 
 	platform_device_unregister(glue->musb);
 	usb_phy_generic_unregister(glue->phy);
-
-	return 0;
 }
 
 static struct platform_driver tusb_driver = {
 	.probe		= tusb_probe,
-	.remove		= tusb_remove,
+	.remove_new	= tusb_remove,
 	.driver		= {
 		.name	= "musb-tusb",
 	},
diff --git a/drivers/usb/musb/ux500.c b/drivers/usb/musb/ux500.c
index 8ea62c344328..c8d9d2a1d2f0 100644
--- a/drivers/usb/musb/ux500.c
+++ b/drivers/usb/musb/ux500.c
@@ -303,14 +303,12 @@ err0:
 	return ret;
 }
 
-static int ux500_remove(struct platform_device *pdev)
+static void ux500_remove(struct platform_device *pdev)
 {
 	struct ux500_glue	*glue = platform_get_drvdata(pdev);
 
 	platform_device_unregister(glue->musb);
 	clk_disable_unprepare(glue->clk);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -357,7 +355,7 @@ MODULE_DEVICE_TABLE(of, ux500_match);
 
 static struct platform_driver ux500_driver = {
 	.probe		= ux500_probe,
-	.remove		= ux500_remove,
+	.remove_new	= ux500_remove,
 	.driver		= {
 		.name	= "musb-ux500",
 		.pm	= &ux500_pm_ops,
diff --git a/drivers/usb/phy/phy-ab8500-usb.c b/drivers/usb/phy/phy-ab8500-usb.c
index 4c52ba96f17e..408f47e39025 100644
--- a/drivers/usb/phy/phy-ab8500-usb.c
+++ b/drivers/usb/phy/phy-ab8500-usb.c
@@ -965,7 +965,7 @@ static int ab8500_usb_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int ab8500_usb_remove(struct platform_device *pdev)
+static void ab8500_usb_remove(struct platform_device *pdev)
 {
 	struct ab8500_usb *ab = platform_get_drvdata(pdev);
 
@@ -977,8 +977,6 @@ static int ab8500_usb_remove(struct platform_device *pdev)
 		ab8500_usb_host_phy_dis(ab);
 	else if (ab->mode == USB_PERIPHERAL)
 		ab8500_usb_peri_phy_dis(ab);
-
-	return 0;
 }
 
 static const struct platform_device_id ab8500_usb_devtype[] = {
@@ -989,7 +987,7 @@ MODULE_DEVICE_TABLE(platform, ab8500_usb_devtype);
 
 static struct platform_driver ab8500_usb_driver = {
 	.probe		= ab8500_usb_probe,
-	.remove		= ab8500_usb_remove,
+	.remove_new	= ab8500_usb_remove,
 	.id_table	= ab8500_usb_devtype,
 	.driver		= {
 		.name	= "abx5x0-usb",
diff --git a/drivers/usb/phy/phy-am335x.c b/drivers/usb/phy/phy-am335x.c
index 8524475d942d..e39665cf4b4a 100644
--- a/drivers/usb/phy/phy-am335x.c
+++ b/drivers/usb/phy/phy-am335x.c
@@ -82,12 +82,11 @@ static int am335x_phy_probe(struct platform_device *pdev)
 	return usb_add_phy_dev(&am_phy->usb_phy_gen.phy);
 }
 
-static int am335x_phy_remove(struct platform_device *pdev)
+static void am335x_phy_remove(struct platform_device *pdev)
 {
 	struct am335x_phy *am_phy = platform_get_drvdata(pdev);
 
 	usb_remove_phy(&am_phy->usb_phy_gen.phy);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -134,7 +133,7 @@ MODULE_DEVICE_TABLE(of, am335x_phy_ids);
 
 static struct platform_driver am335x_phy_driver = {
 	.probe          = am335x_phy_probe,
-	.remove         = am335x_phy_remove,
+	.remove_new     = am335x_phy_remove,
 	.driver         = {
 		.name   = "am335x-phy-driver",
 		.pm = &am335x_pm_ops,
diff --git a/drivers/usb/phy/phy-fsl-usb.c b/drivers/usb/phy/phy-fsl-usb.c
index 972704262b02..79617bb0a70e 100644
--- a/drivers/usb/phy/phy-fsl-usb.c
+++ b/drivers/usb/phy/phy-fsl-usb.c
@@ -983,7 +983,7 @@ static int fsl_otg_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int fsl_otg_remove(struct platform_device *pdev)
+static void fsl_otg_remove(struct platform_device *pdev)
 {
 	struct fsl_usb2_platform_data *pdata = dev_get_platdata(&pdev->dev);
 
@@ -998,13 +998,11 @@ static int fsl_otg_remove(struct platform_device *pdev)
 
 	if (pdata->exit)
 		pdata->exit(pdev);
-
-	return 0;
 }
 
 struct platform_driver fsl_otg_driver = {
 	.probe = fsl_otg_probe,
-	.remove = fsl_otg_remove,
+	.remove_new = fsl_otg_remove,
 	.driver = {
 		.name = driver_name,
 		.owner = THIS_MODULE,
diff --git a/drivers/usb/phy/phy-generic.c b/drivers/usb/phy/phy-generic.c
index c1309ea24a52..770081b828a4 100644
--- a/drivers/usb/phy/phy-generic.c
+++ b/drivers/usb/phy/phy-generic.c
@@ -330,13 +330,11 @@ static int usb_phy_generic_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int usb_phy_generic_remove(struct platform_device *pdev)
+static void usb_phy_generic_remove(struct platform_device *pdev)
 {
 	struct usb_phy_generic *nop = platform_get_drvdata(pdev);
 
 	usb_remove_phy(&nop->phy);
-
-	return 0;
 }
 
 static const struct of_device_id nop_xceiv_dt_ids[] = {
@@ -348,7 +346,7 @@ MODULE_DEVICE_TABLE(of, nop_xceiv_dt_ids);
 
 static struct platform_driver usb_phy_generic_driver = {
 	.probe		= usb_phy_generic_probe,
-	.remove		= usb_phy_generic_remove,
+	.remove_new	= usb_phy_generic_remove,
 	.driver		= {
 		.name	= "usb_phy_generic",
 		.of_match_table = nop_xceiv_dt_ids,
diff --git a/drivers/usb/phy/phy-gpio-vbus-usb.c b/drivers/usb/phy/phy-gpio-vbus-usb.c
index 12dfeff7de3d..817c242a76ca 100644
--- a/drivers/usb/phy/phy-gpio-vbus-usb.c
+++ b/drivers/usb/phy/phy-gpio-vbus-usb.c
@@ -325,7 +325,7 @@ static int gpio_vbus_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int gpio_vbus_remove(struct platform_device *pdev)
+static void gpio_vbus_remove(struct platform_device *pdev)
 {
 	struct gpio_vbus_data *gpio_vbus = platform_get_drvdata(pdev);
 
@@ -333,8 +333,6 @@ static int gpio_vbus_remove(struct platform_device *pdev)
 	cancel_delayed_work_sync(&gpio_vbus->work);
 
 	usb_remove_phy(&gpio_vbus->phy);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -386,7 +384,7 @@ static struct platform_driver gpio_vbus_driver = {
 		.of_match_table = gpio_vbus_of_match,
 	},
 	.probe		= gpio_vbus_probe,
-	.remove		= gpio_vbus_remove,
+	.remove_new	= gpio_vbus_remove,
 };
 
 module_platform_driver(gpio_vbus_driver);
diff --git a/drivers/usb/phy/phy-keystone.c b/drivers/usb/phy/phy-keystone.c
index f75912279b39..bd9a98ad1b30 100644
--- a/drivers/usb/phy/phy-keystone.c
+++ b/drivers/usb/phy/phy-keystone.c
@@ -88,13 +88,11 @@ static int keystone_usbphy_probe(struct platform_device *pdev)
 	return usb_add_phy_dev(&k_phy->usb_phy_gen.phy);
 }
 
-static int keystone_usbphy_remove(struct platform_device *pdev)
+static void keystone_usbphy_remove(struct platform_device *pdev)
 {
 	struct keystone_usbphy *k_phy = platform_get_drvdata(pdev);
 
 	usb_remove_phy(&k_phy->usb_phy_gen.phy);
-
-	return 0;
 }
 
 static const struct of_device_id keystone_usbphy_ids[] = {
@@ -105,7 +103,7 @@ MODULE_DEVICE_TABLE(of, keystone_usbphy_ids);
 
 static struct platform_driver keystone_usbphy_driver = {
 	.probe          = keystone_usbphy_probe,
-	.remove         = keystone_usbphy_remove,
+	.remove_new     = keystone_usbphy_remove,
 	.driver         = {
 		.name   = "keystone-usbphy",
 		.of_match_table = keystone_usbphy_ids,
diff --git a/drivers/usb/phy/phy-mv-usb.c b/drivers/usb/phy/phy-mv-usb.c
index 86503b7d695c..df7c27474a75 100644
--- a/drivers/usb/phy/phy-mv-usb.c
+++ b/drivers/usb/phy/phy-mv-usb.c
@@ -644,7 +644,7 @@ static const struct attribute_group *mv_otg_groups[] = {
 	NULL,
 };
 
-static int mv_otg_remove(struct platform_device *pdev)
+static void mv_otg_remove(struct platform_device *pdev)
 {
 	struct mv_otg *mvotg = platform_get_drvdata(pdev);
 
@@ -654,8 +654,6 @@ static int mv_otg_remove(struct platform_device *pdev)
 	mv_otg_disable(mvotg);
 
 	usb_remove_phy(&mvotg->phy);
-
-	return 0;
 }
 
 static int mv_otg_probe(struct platform_device *pdev)
@@ -869,7 +867,7 @@ static int mv_otg_resume(struct platform_device *pdev)
 
 static struct platform_driver mv_otg_driver = {
 	.probe = mv_otg_probe,
-	.remove = mv_otg_remove,
+	.remove_new = mv_otg_remove,
 	.driver = {
 		   .name = driver_name,
 		   .dev_groups = mv_otg_groups,
diff --git a/drivers/usb/phy/phy-mxs-usb.c b/drivers/usb/phy/phy-mxs-usb.c
index d2836ef5d15c..e1a2b2ea098b 100644
--- a/drivers/usb/phy/phy-mxs-usb.c
+++ b/drivers/usb/phy/phy-mxs-usb.c
@@ -733,7 +733,7 @@ static int mxs_phy_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	/* Some SoCs don't have anatop registers */
-	if (of_get_property(np, "fsl,anatop", NULL)) {
+	if (of_property_present(np, "fsl,anatop")) {
 		mxs_phy->regmap_anatop = syscon_regmap_lookup_by_phandle
 			(np, "fsl,anatop");
 		if (IS_ERR(mxs_phy->regmap_anatop)) {
@@ -801,13 +801,11 @@ static int mxs_phy_probe(struct platform_device *pdev)
 	return usb_add_phy_dev(&mxs_phy->phy);
 }
 
-static int mxs_phy_remove(struct platform_device *pdev)
+static void mxs_phy_remove(struct platform_device *pdev)
 {
 	struct mxs_phy *mxs_phy = platform_get_drvdata(pdev);
 
 	usb_remove_phy(&mxs_phy->phy);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -853,7 +851,7 @@ static SIMPLE_DEV_PM_OPS(mxs_phy_pm, mxs_phy_system_suspend,
 
 static struct platform_driver mxs_phy_driver = {
 	.probe = mxs_phy_probe,
-	.remove = mxs_phy_remove,
+	.remove_new = mxs_phy_remove,
 	.driver = {
 		.name = DRIVER_NAME,
 		.of_match_table = mxs_phy_dt_ids,
diff --git a/drivers/usb/phy/phy-tahvo.c b/drivers/usb/phy/phy-tahvo.c
index f2d2cc586c5b..47562d49dfc1 100644
--- a/drivers/usb/phy/phy-tahvo.c
+++ b/drivers/usb/phy/phy-tahvo.c
@@ -412,7 +412,7 @@ err_disable_clk:
 	return ret;
 }
 
-static int tahvo_usb_remove(struct platform_device *pdev)
+static void tahvo_usb_remove(struct platform_device *pdev)
 {
 	struct tahvo_usb *tu = platform_get_drvdata(pdev);
 
@@ -420,13 +420,11 @@ static int tahvo_usb_remove(struct platform_device *pdev)
 	usb_remove_phy(&tu->phy);
 	if (!IS_ERR(tu->ick))
 		clk_disable(tu->ick);
-
-	return 0;
 }
 
 static struct platform_driver tahvo_usb_driver = {
 	.probe		= tahvo_usb_probe,
-	.remove		= tahvo_usb_remove,
+	.remove_new	= tahvo_usb_remove,
 	.driver		= {
 		.name	= "tahvo-usb",
 		.dev_groups = tahvo_groups,
diff --git a/drivers/usb/phy/phy-tegra-usb.c b/drivers/usb/phy/phy-tegra-usb.c
index f0240107edb1..8b2ff3a8882d 100644
--- a/drivers/usb/phy/phy-tegra-usb.c
+++ b/drivers/usb/phy/phy-tegra-usb.c
@@ -1375,7 +1375,7 @@ static int tegra_usb_phy_probe(struct platform_device *pdev)
 	tegra_phy->is_legacy_phy =
 		of_property_read_bool(np, "nvidia,has-legacy-mode");
 
-	if (of_find_property(np, "dr_mode", NULL))
+	if (of_property_present(np, "dr_mode"))
 		tegra_phy->mode = usb_get_dr_mode(&pdev->dev);
 	else
 		tegra_phy->mode = USB_DR_MODE_HOST;
@@ -1486,18 +1486,16 @@ static int tegra_usb_phy_probe(struct platform_device *pdev)
 	return usb_add_phy_dev(&tegra_phy->u_phy);
 }
 
-static int tegra_usb_phy_remove(struct platform_device *pdev)
+static void tegra_usb_phy_remove(struct platform_device *pdev)
 {
 	struct tegra_usb_phy *tegra_phy = platform_get_drvdata(pdev);
 
 	usb_remove_phy(&tegra_phy->u_phy);
-
-	return 0;
 }
 
 static struct platform_driver tegra_usb_phy_driver = {
 	.probe		= tegra_usb_phy_probe,
-	.remove		= tegra_usb_phy_remove,
+	.remove_new	= tegra_usb_phy_remove,
 	.driver		= {
 		.name	= "tegra-phy",
 		.of_match_table = tegra_usb_phy_id_table,
diff --git a/drivers/usb/phy/phy-twl6030-usb.c b/drivers/usb/phy/phy-twl6030-usb.c
index ab3c38a7d8ac..c3ce6b1054f1 100644
--- a/drivers/usb/phy/phy-twl6030-usb.c
+++ b/drivers/usb/phy/phy-twl6030-usb.c
@@ -409,7 +409,7 @@ err_put_regulator:
 	return status;
 }
 
-static int twl6030_usb_remove(struct platform_device *pdev)
+static void twl6030_usb_remove(struct platform_device *pdev)
 {
 	struct twl6030_usb *twl = platform_get_drvdata(pdev);
 
@@ -422,8 +422,6 @@ static int twl6030_usb_remove(struct platform_device *pdev)
 	free_irq(twl->irq2, twl);
 	regulator_put(twl->usb3v3);
 	cancel_work_sync(&twl->set_vbus_work);
-
-	return 0;
 }
 
 static const struct of_device_id twl6030_usb_id_table[] = {
@@ -434,7 +432,7 @@ MODULE_DEVICE_TABLE(of, twl6030_usb_id_table);
 
 static struct platform_driver twl6030_usb_driver = {
 	.probe		= twl6030_usb_probe,
-	.remove		= twl6030_usb_remove,
+	.remove_new	= twl6030_usb_remove,
 	.driver		= {
 		.name	= "twl6030_usb",
 		.of_match_table = of_match_ptr(twl6030_usb_id_table),
diff --git a/drivers/usb/renesas_usbhs/common.c b/drivers/usb/renesas_usbhs/common.c
index 96f3939a65e2..fa34efabcccf 100644
--- a/drivers/usb/renesas_usbhs/common.c
+++ b/drivers/usb/renesas_usbhs/common.c
@@ -823,7 +823,7 @@ static struct platform_driver renesas_usbhs_driver = {
 	.driver		= {
 		.name	= "renesas_usbhs",
 		.pm	= &usbhsc_pm_ops,
-		.of_match_table = of_match_ptr(usbhs_of_match),
+		.of_match_table = usbhs_of_match,
 	},
 	.probe		= usbhs_probe,
 	.remove		= usbhs_remove,
diff --git a/drivers/usb/roles/class.c b/drivers/usb/roles/class.c
index 56814ef80c24..0395bd5dbd3e 100644
--- a/drivers/usb/roles/class.c
+++ b/drivers/usb/roles/class.c
@@ -392,7 +392,7 @@ EXPORT_SYMBOL_GPL(usb_role_switch_get_drvdata);
 
 static int __init usb_roles_init(void)
 {
-	role_class = class_create(THIS_MODULE, "usb_role");
+	role_class = class_create("usb_role");
 	return PTR_ERR_OR_ZERO(role_class);
 }
 subsys_initcall(usb_roles_init);
diff --git a/drivers/usb/serial/bus.c b/drivers/usb/serial/bus.c
index 9e38142acd38..3eb8dc3a1a8f 100644
--- a/drivers/usb/serial/bus.c
+++ b/drivers/usb/serial/bus.c
@@ -144,7 +144,7 @@ static void free_dynids(struct usb_serial_driver *drv)
 	spin_unlock(&drv->dynids.lock);
 }
 
-struct bus_type usb_serial_bus_type = {
+const struct bus_type usb_serial_bus_type = {
 	.name =		"usb-serial",
 	.match =	usb_serial_device_match,
 	.probe =	usb_serial_device_probe,
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index f31cc3c76329..644a55447fd7 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -595,6 +595,11 @@ static void option_instat_callback(struct urb *urb);
 #define SIERRA_VENDOR_ID			0x1199
 #define SIERRA_PRODUCT_EM9191			0x90d3
 
+/* UNISOC (Spreadtrum) products */
+#define UNISOC_VENDOR_ID			0x1782
+/* TOZED LT70-C based on UNISOC SL8563 uses UNISOC's vendor ID */
+#define TOZED_PRODUCT_LT70C			0x4055
+
 /* Device flags */
 
 /* Highest interface number which can be used with NCTRL() and RSVD() */
@@ -2225,6 +2230,7 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE_AND_INTERFACE_INFO(OPPO_VENDOR_ID, OPPO_PRODUCT_R11, 0xff, 0xff, 0x30) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(SIERRA_VENDOR_ID, SIERRA_PRODUCT_EM9191, 0xff, 0xff, 0x30) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(SIERRA_VENDOR_ID, SIERRA_PRODUCT_EM9191, 0xff, 0, 0) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(UNISOC_VENDOR_ID, TOZED_PRODUCT_LT70C, 0xff, 0, 0) },
 	{ } /* Terminating entry */
 };
 MODULE_DEVICE_TABLE(usb, option_ids);
diff --git a/drivers/usb/serial/quatech2.c b/drivers/usb/serial/quatech2.c
index 6fca40ace83a..fee581409bf6 100644
--- a/drivers/usb/serial/quatech2.c
+++ b/drivers/usb/serial/quatech2.c
@@ -176,14 +176,6 @@ static inline int qt2_control_msg(struct usb_device *dev,
 			       NULL, 0, QT2_USB_TIMEOUT);
 }
 
-static inline int qt2_setdevice(struct usb_device *dev, u8 *data)
-{
-	u16 x = ((u16) (data[1] << 8) | (u16) (data[0]));
-
-	return qt2_control_msg(dev, QT_SET_GET_DEVICE, x, 0);
-}
-
-
 static inline int qt2_getregister(struct usb_device *dev,
 				  u8 uart,
 				  u8 reg,
diff --git a/drivers/usb/storage/uas.c b/drivers/usb/storage/uas.c
index de3836412bf3..2583ee9815c5 100644
--- a/drivers/usb/storage/uas.c
+++ b/drivers/usb/storage/uas.c
@@ -894,7 +894,7 @@ static int uas_slave_configure(struct scsi_device *sdev)
 	return 0;
 }
 
-static struct scsi_host_template uas_host_template = {
+static const struct scsi_host_template uas_host_template = {
 	.module = THIS_MODULE,
 	.name = "uas",
 	.queuecommand = uas_queuecommand,
diff --git a/drivers/usb/storage/usb.c b/drivers/usb/storage/usb.c
index ed7c6ad96a74..7b36a3334fb3 100644
--- a/drivers/usb/storage/usb.c
+++ b/drivers/usb/storage/usb.c
@@ -937,7 +937,7 @@ int usb_stor_probe1(struct us_data **pus,
 		struct usb_interface *intf,
 		const struct usb_device_id *id,
 		const struct us_unusual_dev *unusual_dev,
-		struct scsi_host_template *sht)
+		const struct scsi_host_template *sht)
 {
 	struct Scsi_Host *host;
 	struct us_data *us;
diff --git a/drivers/usb/storage/usb.h b/drivers/usb/storage/usb.h
index 0451fac1adce..fd3f32670873 100644
--- a/drivers/usb/storage/usb.h
+++ b/drivers/usb/storage/usb.h
@@ -187,7 +187,7 @@ extern int usb_stor_probe1(struct us_data **pus,
 		struct usb_interface *intf,
 		const struct usb_device_id *id,
 		const struct us_unusual_dev *unusual_dev,
-		struct scsi_host_template *sht);
+		const struct scsi_host_template *sht);
 extern int usb_stor_probe2(struct us_data *us);
 extern void usb_stor_disconnect(struct usb_interface *intf);
 
diff --git a/drivers/usb/typec/bus.c b/drivers/usb/typec/bus.c
index 098f0efaa58d..fe5b9a2e61f5 100644
--- a/drivers/usb/typec/bus.c
+++ b/drivers/usb/typec/bus.c
@@ -431,7 +431,7 @@ static void typec_remove(struct device *dev)
 	adev->ops = NULL;
 }
 
-struct bus_type typec_bus = {
+const struct bus_type typec_bus = {
 	.name = "typec",
 	.dev_groups = typec_groups,
 	.match = typec_match,
diff --git a/drivers/usb/typec/bus.h b/drivers/usb/typec/bus.h
index c89168857417..643b8c81786d 100644
--- a/drivers/usb/typec/bus.h
+++ b/drivers/usb/typec/bus.h
@@ -28,7 +28,7 @@ struct altmode {
 
 #define to_altmode(d) container_of(d, struct altmode, adev)
 
-extern struct bus_type typec_bus;
+extern const struct bus_type typec_bus;
 extern const struct device_type typec_altmode_dev_type;
 
 #define is_typec_altmode(_dev_) (_dev_->type == &typec_altmode_dev_type)
diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c
index cc3182f70673..349cc2030c90 100644
--- a/drivers/usb/typec/class.c
+++ b/drivers/usb/typec/class.c
@@ -22,7 +22,6 @@ static DEFINE_IDA(typec_index_ida);
 
 struct class typec_class = {
 	.name = "typec",
-	.owner = THIS_MODULE,
 };
 
 /* ------------------------------------------------------------------------- */
diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c
index 746ef3a75b76..8bbeb9b1e439 100644
--- a/drivers/usb/typec/hd3ss3220.c
+++ b/drivers/usb/typec/hd3ss3220.c
@@ -290,7 +290,7 @@ MODULE_DEVICE_TABLE(of, dev_ids);
 static struct i2c_driver hd3ss3220_driver = {
 	.driver = {
 		.name = "hd3ss3220",
-		.of_match_table = of_match_ptr(dev_ids),
+		.of_match_table = dev_ids,
 	},
 	.probe_new = hd3ss3220_probe,
 	.remove =  hd3ss3220_remove,
diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c
index c7177ddd4f12..d9eaf9a0b0bf 100644
--- a/drivers/usb/typec/mux.c
+++ b/drivers/usb/typec/mux.c
@@ -514,5 +514,4 @@ EXPORT_SYMBOL_GPL(typec_mux_get_drvdata);
 
 struct class typec_mux_class = {
 	.name = "typec_mux",
-	.owner = THIS_MODULE,
 };
diff --git a/drivers/usb/typec/pd.c b/drivers/usb/typec/pd.c
index 59c537a5e600..0bcde1ff4d39 100644
--- a/drivers/usb/typec/pd.c
+++ b/drivers/usb/typec/pd.c
@@ -15,7 +15,6 @@ static DEFINE_IDA(pd_ida);
 
 static struct class pd_class = {
 	.name = "usb_power_delivery",
-	.owner = THIS_MODULE,
 };
 
 #define to_pdo(o) container_of(o, struct pdo, dev)
diff --git a/drivers/usb/typec/retimer.c b/drivers/usb/typec/retimer.c
index 0481e82f6bbc..4a7d1b5c4d86 100644
--- a/drivers/usb/typec/retimer.c
+++ b/drivers/usb/typec/retimer.c
@@ -157,5 +157,4 @@ EXPORT_SYMBOL_GPL(typec_retimer_get_drvdata);
 
 struct class retimer_class = {
 	.name = "retimer",
-	.owner = THIS_MODULE,
 };
diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c
index 1ffce00d94b4..62ba53357612 100644
--- a/drivers/usb/typec/tcpm/fusb302.c
+++ b/drivers/usb/typec/tcpm/fusb302.c
@@ -190,7 +190,7 @@ static void fusb302_log(struct fusb302_chip *chip, const char *fmt, ...)
 
 static int fusb302_debug_show(struct seq_file *s, void *v)
 {
-	struct fusb302_chip *chip = (struct fusb302_chip *)s->private;
+	struct fusb302_chip *chip = s->private;
 	int tail;
 
 	mutex_lock(&chip->logbuffer_lock);
@@ -1813,7 +1813,7 @@ static int fusb302_pm_resume(struct device *dev)
 	return 0;
 }
 
-static const struct of_device_id fusb302_dt_match[] = {
+static const struct of_device_id fusb302_dt_match[] __maybe_unused = {
 	{.compatible = "fcs,fusb302"},
 	{},
 };
diff --git a/drivers/usb/typec/tcpm/tcpci_mt6360.c b/drivers/usb/typec/tcpm/tcpci_mt6360.c
index 1b7c31278ebb..6fa8fd5c8041 100644
--- a/drivers/usb/typec/tcpm/tcpci_mt6360.c
+++ b/drivers/usb/typec/tcpm/tcpci_mt6360.c
@@ -43,12 +43,6 @@ struct mt6360_tcpc_info {
 	int irq;
 };
 
-static inline int mt6360_tcpc_read16(struct regmap *regmap,
-				     unsigned int reg, u16 *val)
-{
-	return regmap_raw_read(regmap, reg, val, sizeof(u16));
-}
-
 static inline int mt6360_tcpc_write16(struct regmap *regmap,
 				      unsigned int reg, u16 val)
 {
diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c
index 1ee774c263f0..3c6b0c8e2d3a 100644
--- a/drivers/usb/typec/tcpm/tcpm.c
+++ b/drivers/usb/typec/tcpm/tcpm.c
@@ -737,7 +737,7 @@ static void tcpm_log_source_caps(struct tcpm_port *port)
 
 static int tcpm_debug_show(struct seq_file *s, void *v)
 {
-	struct tcpm_port *port = (struct tcpm_port *)s->private;
+	struct tcpm_port *port = s->private;
 	int tail;
 
 	mutex_lock(&port->logbuffer_lock);
@@ -1523,7 +1523,21 @@ static bool svdm_consume_svids(struct tcpm_port *port, const u32 *p, int cnt)
 		pmdata->svids[pmdata->nsvids++] = svid;
 		tcpm_log(port, "SVID %d: 0x%x", pmdata->nsvids, svid);
 	}
-	return true;
+
+	/*
+	 * PD3.0 Spec 6.4.4.3.2: The SVIDs are returned 2 per VDO (see Table
+	 * 6-43), and can be returned maximum 6 VDOs per response (see Figure
+	 * 6-19). If the Respondersupports 12 or more SVID then the Discover
+	 * SVIDs Command Shall be executed multiple times until a Discover
+	 * SVIDs VDO is returned ending either with a SVID value of 0x0000 in
+	 * the last part of the last VDO or with a VDO containing two SVIDs
+	 * with values of 0x0000.
+	 *
+	 * However, some odd dockers support SVIDs less than 12 but without
+	 * 0x0000 in the last VDO, so we need to break the Discover SVIDs
+	 * request and return false here.
+	 */
+	return cnt == 7;
 abort:
 	tcpm_log(port, "SVID_DISCOVERY_MAX(%d) too low!", SVID_DISCOVERY_MAX);
 	return false;
@@ -6577,6 +6591,8 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc)
 	port->port_type = port->typec_caps.type;
 
 	port->role_sw = usb_role_switch_get(port->dev);
+	if (!port->role_sw)
+		port->role_sw = fwnode_usb_role_switch_get(tcpc->fwnode);
 	if (IS_ERR(port->role_sw)) {
 		err = PTR_ERR(port->role_sw);
 		goto out_destroy_wq;
diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c
index 485b90c13078..8b075ca82ef6 100644
--- a/drivers/usb/typec/tipd/core.c
+++ b/drivers/usb/typec/tipd/core.c
@@ -16,6 +16,7 @@
 #include <linux/usb/typec.h>
 #include <linux/usb/typec_altmode.h>
 #include <linux/usb/role.h>
+#include <linux/workqueue.h>
 
 #include "tps6598x.h"
 #include "trace.h"
@@ -97,6 +98,8 @@ struct tps6598x {
 
 	int wakeup;
 	u16 pwr_status;
+	struct delayed_work	wq_poll;
+	irq_handler_t irq_handler;
 };
 
 static enum power_supply_property tps6598x_psy_props[] = {
@@ -177,16 +180,6 @@ static inline int tps6598x_read64(struct tps6598x *tps, u8 reg, u64 *val)
 	return tps6598x_block_read(tps, reg, val, sizeof(u64));
 }
 
-static inline int tps6598x_write16(struct tps6598x *tps, u8 reg, u16 val)
-{
-	return tps6598x_block_write(tps, reg, &val, sizeof(u16));
-}
-
-static inline int tps6598x_write32(struct tps6598x *tps, u8 reg, u32 val)
-{
-	return tps6598x_block_write(tps, reg, &val, sizeof(u32));
-}
-
 static inline int tps6598x_write64(struct tps6598x *tps, u8 reg, u64 val)
 {
 	return tps6598x_block_write(tps, reg, &val, sizeof(u64));
@@ -568,6 +561,18 @@ err_unlock:
 	return IRQ_NONE;
 }
 
+/* Time interval for Polling */
+#define POLL_INTERVAL	500 /* msecs */
+static void tps6598x_poll_work(struct work_struct *work)
+{
+	struct tps6598x *tps = container_of(to_delayed_work(work),
+					    struct tps6598x, wq_poll);
+
+	tps->irq_handler(0, tps);
+	queue_delayed_work(system_power_efficient_wq,
+			   &tps->wq_poll, msecs_to_jiffies(POLL_INTERVAL));
+}
+
 static int tps6598x_check_mode(struct tps6598x *tps)
 {
 	char mode[5] = { };
@@ -746,6 +751,7 @@ static int tps6598x_probe(struct i2c_client *client)
 			TPS_REG_INT_PLUG_EVENT;
 	}
 
+	tps->irq_handler = irq_handler;
 	/* Make sure the controller has application firmware running */
 	ret = tps6598x_check_mode(tps);
 	if (ret)
@@ -837,10 +843,18 @@ static int tps6598x_probe(struct i2c_client *client)
 			dev_err(&client->dev, "failed to register partner\n");
 	}
 
-	ret = devm_request_threaded_irq(&client->dev, client->irq, NULL,
-					irq_handler,
-					IRQF_SHARED | IRQF_ONESHOT,
-					dev_name(&client->dev), tps);
+	if (client->irq) {
+		ret = devm_request_threaded_irq(&client->dev, client->irq, NULL,
+						irq_handler,
+						IRQF_SHARED | IRQF_ONESHOT,
+						dev_name(&client->dev), tps);
+	} else {
+		dev_warn(tps->dev, "Unable to find the interrupt, switching to polling\n");
+		INIT_DELAYED_WORK(&tps->wq_poll, tps6598x_poll_work);
+		queue_delayed_work(system_power_efficient_wq, &tps->wq_poll,
+				   msecs_to_jiffies(POLL_INTERVAL));
+	}
+
 	if (ret)
 		goto err_disconnect;
 
@@ -848,7 +862,7 @@ static int tps6598x_probe(struct i2c_client *client)
 	fwnode_handle_put(fwnode);
 
 	tps->wakeup = device_property_read_bool(tps->dev, "wakeup-source");
-	if (tps->wakeup) {
+	if (tps->wakeup && client->irq) {
 		device_init_wakeup(&client->dev, true);
 		enable_irq_wake(client->irq);
 	}
@@ -887,6 +901,9 @@ static int __maybe_unused tps6598x_suspend(struct device *dev)
 		enable_irq_wake(client->irq);
 	}
 
+	if (!client->irq)
+		cancel_delayed_work_sync(&tps->wq_poll);
+
 	return 0;
 }
 
@@ -900,6 +917,10 @@ static int __maybe_unused tps6598x_resume(struct device *dev)
 		enable_irq(client->irq);
 	}
 
+	if (client->irq)
+		queue_delayed_work(system_power_efficient_wq, &tps->wq_poll,
+				   msecs_to_jiffies(POLL_INTERVAL));
+
 	return 0;
 }
 
diff --git a/drivers/usb/typec/ucsi/Kconfig b/drivers/usb/typec/ucsi/Kconfig
index 8f9c4b9f31f7..b3bb0191987e 100644
--- a/drivers/usb/typec/ucsi/Kconfig
+++ b/drivers/usb/typec/ucsi/Kconfig
@@ -58,4 +58,14 @@ config UCSI_STM32G0
 	  To compile the driver as a module, choose M here: the module will be
 	  called ucsi_stm32g0.
 
+config UCSI_PMIC_GLINK
+	tristate "UCSI Qualcomm PMIC GLINK Interface Driver"
+	depends on QCOM_PMIC_GLINK
+	help
+	  This driver enables UCSI support on platforms that expose UCSI
+	  interface as PMIC GLINK device.
+
+	  To compile the driver as a module, choose M here: the module will be
+	  called ucsi_glink.
+
 endif
diff --git a/drivers/usb/typec/ucsi/Makefile b/drivers/usb/typec/ucsi/Makefile
index 480d533d762f..77f09e136956 100644
--- a/drivers/usb/typec/ucsi/Makefile
+++ b/drivers/usb/typec/ucsi/Makefile
@@ -18,3 +18,4 @@ endif
 obj-$(CONFIG_UCSI_ACPI)			+= ucsi_acpi.o
 obj-$(CONFIG_UCSI_CCG)			+= ucsi_ccg.o
 obj-$(CONFIG_UCSI_STM32G0)		+= ucsi_stm32g0.o
+obj-$(CONFIG_UCSI_PMIC_GLINK)		+= ucsi_glink.o
diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
index 8d1baf28df55..2b472ec01dc4 100644
--- a/drivers/usb/typec/ucsi/ucsi.c
+++ b/drivers/usb/typec/ucsi/ucsi.c
@@ -1444,11 +1444,13 @@ static void ucsi_init_work(struct work_struct *work)
 
 	ret = ucsi_init(ucsi);
 	if (ret)
-		dev_err(ucsi->dev, "PPM init failed (%d)\n", ret);
+		dev_err_probe(ucsi->dev, ret, "PPM init failed\n");
 
 	if (ret == -EPROBE_DEFER) {
-		if (ucsi->work_count++ > UCSI_ROLE_SWITCH_WAIT_COUNT)
+		if (ucsi->work_count++ > UCSI_ROLE_SWITCH_WAIT_COUNT) {
+			dev_err(ucsi->dev, "PPM init failed, stop trying\n");
 			return;
+		}
 
 		queue_delayed_work(system_long_wq, &ucsi->work,
 				   UCSI_ROLE_SWITCH_INTERVAL);
diff --git a/drivers/usb/typec/ucsi/ucsi_acpi.c b/drivers/usb/typec/ucsi/ucsi_acpi.c
index 62206a6b8ea7..217355f1f9b9 100644
--- a/drivers/usb/typec/ucsi/ucsi_acpi.c
+++ b/drivers/usb/typec/ucsi/ucsi_acpi.c
@@ -9,6 +9,7 @@
 #include <linux/platform_device.h>
 #include <linux/module.h>
 #include <linux/acpi.h>
+#include <linux/dmi.h>
 
 #include "ucsi.h"
 
@@ -23,6 +24,7 @@ struct ucsi_acpi {
 	struct completion complete;
 	unsigned long flags;
 	guid_t guid;
+	u64 cmd;
 };
 
 static int ucsi_acpi_dsm(struct ucsi_acpi *ua, int func)
@@ -62,6 +64,7 @@ static int ucsi_acpi_async_write(struct ucsi *ucsi, unsigned int offset,
 	struct ucsi_acpi *ua = ucsi_get_drvdata(ucsi);
 
 	memcpy(ua->base + offset, val, val_len);
+	ua->cmd = *(u64 *)val;
 
 	return ucsi_acpi_dsm(ua, UCSI_DSM_FUNC_WRITE);
 }
@@ -93,13 +96,46 @@ static const struct ucsi_operations ucsi_acpi_ops = {
 	.async_write = ucsi_acpi_async_write
 };
 
+static int
+ucsi_zenbook_read(struct ucsi *ucsi, unsigned int offset, void *val, size_t val_len)
+{
+	struct ucsi_acpi *ua = ucsi_get_drvdata(ucsi);
+	int ret;
+
+	if (offset == UCSI_VERSION || UCSI_COMMAND(ua->cmd) == UCSI_PPM_RESET) {
+		ret = ucsi_acpi_dsm(ua, UCSI_DSM_FUNC_READ);
+		if (ret)
+			return ret;
+	}
+
+	memcpy(val, ua->base + offset, val_len);
+
+	return 0;
+}
+
+static const struct ucsi_operations ucsi_zenbook_ops = {
+	.read = ucsi_zenbook_read,
+	.sync_write = ucsi_acpi_sync_write,
+	.async_write = ucsi_acpi_async_write
+};
+
+static const struct dmi_system_id zenbook_dmi_id[] = {
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ZenBook UX325UA_UM325UA"),
+		},
+	},
+	{ }
+};
+
 static void ucsi_acpi_notify(acpi_handle handle, u32 event, void *data)
 {
 	struct ucsi_acpi *ua = data;
 	u32 cci;
 	int ret;
 
-	ret = ucsi_acpi_read(ua->ucsi, UCSI_CCI, &cci, sizeof(cci));
+	ret = ua->ucsi->ops->read(ua->ucsi, UCSI_CCI, &cci, sizeof(cci));
 	if (ret)
 		return;
 
@@ -114,6 +150,7 @@ static void ucsi_acpi_notify(acpi_handle handle, u32 event, void *data)
 static int ucsi_acpi_probe(struct platform_device *pdev)
 {
 	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
+	const struct ucsi_operations *ops = &ucsi_acpi_ops;
 	struct ucsi_acpi *ua;
 	struct resource *res;
 	acpi_status status;
@@ -143,7 +180,10 @@ static int ucsi_acpi_probe(struct platform_device *pdev)
 	init_completion(&ua->complete);
 	ua->dev = &pdev->dev;
 
-	ua->ucsi = ucsi_create(&pdev->dev, &ucsi_acpi_ops);
+	if (dmi_check_system(zenbook_dmi_id))
+		ops = &ucsi_zenbook_ops;
+
+	ua->ucsi = ucsi_create(&pdev->dev, ops);
 	if (IS_ERR(ua->ucsi))
 		return PTR_ERR(ua->ucsi);
 
diff --git a/drivers/usb/typec/ucsi/ucsi_glink.c b/drivers/usb/typec/ucsi/ucsi_glink.c
new file mode 100644
index 000000000000..b454a5159896
--- /dev/null
+++ b/drivers/usb/typec/ucsi/ucsi_glink.c
@@ -0,0 +1,345 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2023, Linaro Ltd
+ */
+#include <linux/auxiliary_bus.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/mutex.h>
+#include <linux/property.h>
+#include <linux/soc/qcom/pdr.h>
+#include <linux/soc/qcom/pmic_glink.h>
+#include "ucsi.h"
+
+#define UCSI_BUF_SIZE                   48
+
+#define MSG_TYPE_REQ_RESP               1
+#define UCSI_BUF_SIZE                   48
+
+#define UC_NOTIFY_RECEIVER_UCSI         0x0
+#define UC_UCSI_READ_BUF_REQ            0x11
+#define UC_UCSI_WRITE_BUF_REQ           0x12
+#define UC_UCSI_USBC_NOTIFY_IND         0x13
+
+struct ucsi_read_buf_req_msg {
+	struct pmic_glink_hdr   hdr;
+};
+
+struct ucsi_read_buf_resp_msg {
+	struct pmic_glink_hdr   hdr;
+	u8                      buf[UCSI_BUF_SIZE];
+	u32                     ret_code;
+};
+
+struct ucsi_write_buf_req_msg {
+	struct pmic_glink_hdr   hdr;
+	u8                      buf[UCSI_BUF_SIZE];
+	u32                     reserved;
+};
+
+struct ucsi_write_buf_resp_msg {
+	struct pmic_glink_hdr   hdr;
+	u32                     ret_code;
+};
+
+struct ucsi_notify_ind_msg {
+	struct pmic_glink_hdr   hdr;
+	u32                     notification;
+	u32                     receiver;
+	u32                     reserved;
+};
+
+struct pmic_glink_ucsi {
+	struct device *dev;
+
+	struct pmic_glink_client *client;
+
+	struct ucsi *ucsi;
+	struct completion read_ack;
+	struct completion write_ack;
+	struct completion sync_ack;
+	bool sync_pending;
+	struct mutex lock;	/* protects concurrent access to PMIC Glink interface */
+
+	int sync_val;
+
+	struct work_struct notify_work;
+	struct work_struct register_work;
+
+	u8 read_buf[UCSI_BUF_SIZE];
+};
+
+static int pmic_glink_ucsi_read(struct ucsi *__ucsi, unsigned int offset,
+				void *val, size_t val_len)
+{
+	struct pmic_glink_ucsi *ucsi = ucsi_get_drvdata(__ucsi);
+	struct ucsi_read_buf_req_msg req = {};
+	unsigned long left;
+	int ret;
+
+	req.hdr.owner = PMIC_GLINK_OWNER_USBC;
+	req.hdr.type = MSG_TYPE_REQ_RESP;
+	req.hdr.opcode = UC_UCSI_READ_BUF_REQ;
+
+	mutex_lock(&ucsi->lock);
+	memset(ucsi->read_buf, 0, sizeof(ucsi->read_buf));
+	reinit_completion(&ucsi->read_ack);
+
+	ret = pmic_glink_send(ucsi->client, &req, sizeof(req));
+	if (ret < 0) {
+		dev_err(ucsi->dev, "failed to send UCSI read request: %d\n", ret);
+		goto out_unlock;
+	}
+
+	left = wait_for_completion_timeout(&ucsi->read_ack, 5 * HZ);
+	if (!left) {
+		dev_err(ucsi->dev, "timeout waiting for UCSI read response\n");
+		ret = -ETIMEDOUT;
+		goto out_unlock;
+	}
+
+	memcpy(val, &ucsi->read_buf[offset], val_len);
+	ret = 0;
+
+out_unlock:
+	mutex_unlock(&ucsi->lock);
+
+	return ret;
+}
+
+static int pmic_glink_ucsi_locked_write(struct pmic_glink_ucsi *ucsi, unsigned int offset,
+					const void *val, size_t val_len)
+{
+	struct ucsi_write_buf_req_msg req = {};
+	unsigned long left;
+	int ret;
+
+	req.hdr.owner = PMIC_GLINK_OWNER_USBC;
+	req.hdr.type = MSG_TYPE_REQ_RESP;
+	req.hdr.opcode = UC_UCSI_WRITE_BUF_REQ;
+	memcpy(&req.buf[offset], val, val_len);
+
+	reinit_completion(&ucsi->write_ack);
+
+	ret = pmic_glink_send(ucsi->client, &req, sizeof(req));
+	if (ret < 0) {
+		dev_err(ucsi->dev, "failed to send UCSI write request: %d\n", ret);
+		return ret;
+	}
+
+	left = wait_for_completion_timeout(&ucsi->write_ack, 5 * HZ);
+	if (!left) {
+		dev_err(ucsi->dev, "timeout waiting for UCSI write response\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int pmic_glink_ucsi_async_write(struct ucsi *__ucsi, unsigned int offset,
+				       const void *val, size_t val_len)
+{
+	struct pmic_glink_ucsi *ucsi = ucsi_get_drvdata(__ucsi);
+	int ret;
+
+	mutex_lock(&ucsi->lock);
+	ret = pmic_glink_ucsi_locked_write(ucsi, offset, val, val_len);
+	mutex_unlock(&ucsi->lock);
+
+	return ret;
+}
+
+static int pmic_glink_ucsi_sync_write(struct ucsi *__ucsi, unsigned int offset,
+				      const void *val, size_t val_len)
+{
+	struct pmic_glink_ucsi *ucsi = ucsi_get_drvdata(__ucsi);
+	unsigned long left;
+	int ret;
+
+	/* TOFIX: Downstream forces recipient to CON when UCSI_GET_ALTERNATE_MODES command */
+
+	mutex_lock(&ucsi->lock);
+	ucsi->sync_val = 0;
+	reinit_completion(&ucsi->sync_ack);
+	ucsi->sync_pending = true;
+	ret = pmic_glink_ucsi_locked_write(ucsi, offset, val, val_len);
+	mutex_unlock(&ucsi->lock);
+
+	left = wait_for_completion_timeout(&ucsi->sync_ack, 5 * HZ);
+	if (!left) {
+		dev_err(ucsi->dev, "timeout waiting for UCSI sync write response\n");
+		ret = -ETIMEDOUT;
+	} else if (ucsi->sync_val) {
+		dev_err(ucsi->dev, "sync write returned: %d\n", ucsi->sync_val);
+	}
+
+	ucsi->sync_pending = false;
+
+	return ret;
+}
+
+static const struct ucsi_operations pmic_glink_ucsi_ops = {
+	.read = pmic_glink_ucsi_read,
+	.sync_write = pmic_glink_ucsi_sync_write,
+	.async_write = pmic_glink_ucsi_async_write
+};
+
+static void pmic_glink_ucsi_read_ack(struct pmic_glink_ucsi *ucsi, const void *data, int len)
+{
+	const struct ucsi_read_buf_resp_msg *resp = data;
+
+	if (resp->ret_code)
+		return;
+
+	memcpy(ucsi->read_buf, resp->buf, UCSI_BUF_SIZE);
+	complete(&ucsi->read_ack);
+}
+
+static void pmic_glink_ucsi_write_ack(struct pmic_glink_ucsi *ucsi, const void *data, int len)
+{
+	const struct ucsi_write_buf_resp_msg *resp = data;
+
+	if (resp->ret_code)
+		return;
+
+	ucsi->sync_val = resp->ret_code;
+	complete(&ucsi->write_ack);
+}
+
+static void pmic_glink_ucsi_notify(struct work_struct *work)
+{
+	struct pmic_glink_ucsi *ucsi = container_of(work, struct pmic_glink_ucsi, notify_work);
+	unsigned int con_num;
+	u32 cci;
+	int ret;
+
+	ret = pmic_glink_ucsi_read(ucsi->ucsi, UCSI_CCI, &cci, sizeof(cci));
+	if (ret) {
+		dev_err(ucsi->dev, "failed to read CCI on notification\n");
+		return;
+	}
+
+	con_num = UCSI_CCI_CONNECTOR(cci);
+	if (con_num)
+		ucsi_connector_change(ucsi->ucsi, con_num);
+
+	if (ucsi->sync_pending && cci & UCSI_CCI_BUSY) {
+		ucsi->sync_val = -EBUSY;
+		complete(&ucsi->sync_ack);
+	} else if (ucsi->sync_pending &&
+		   (cci & (UCSI_CCI_ACK_COMPLETE | UCSI_CCI_COMMAND_COMPLETE))) {
+		complete(&ucsi->sync_ack);
+	}
+}
+
+static void pmic_glink_ucsi_register(struct work_struct *work)
+{
+	struct pmic_glink_ucsi *ucsi = container_of(work, struct pmic_glink_ucsi, register_work);
+
+	ucsi_register(ucsi->ucsi);
+}
+
+static void pmic_glink_ucsi_callback(const void *data, size_t len, void *priv)
+{
+	struct pmic_glink_ucsi *ucsi = priv;
+	const struct pmic_glink_hdr *hdr = data;
+
+	switch (hdr->opcode) {
+	case UC_UCSI_READ_BUF_REQ:
+		pmic_glink_ucsi_read_ack(ucsi, data, len);
+		break;
+	case UC_UCSI_WRITE_BUF_REQ:
+		pmic_glink_ucsi_write_ack(ucsi, data, len);
+		break;
+	case UC_UCSI_USBC_NOTIFY_IND:
+		schedule_work(&ucsi->notify_work);
+		break;
+	};
+}
+
+static void pmic_glink_ucsi_pdr_notify(void *priv, int state)
+{
+	struct pmic_glink_ucsi *ucsi = priv;
+
+	if (state == SERVREG_SERVICE_STATE_UP)
+		schedule_work(&ucsi->register_work);
+	else if (state == SERVREG_SERVICE_STATE_DOWN)
+		ucsi_unregister(ucsi->ucsi);
+}
+
+static void pmic_glink_ucsi_destroy(void *data)
+{
+	struct pmic_glink_ucsi *ucsi = data;
+
+	/* Protect to make sure we're not in a middle of a transaction from a glink callback */
+	mutex_lock(&ucsi->lock);
+	ucsi_destroy(ucsi->ucsi);
+	mutex_unlock(&ucsi->lock);
+}
+
+static int pmic_glink_ucsi_probe(struct auxiliary_device *adev,
+				 const struct auxiliary_device_id *id)
+{
+	struct pmic_glink_ucsi *ucsi;
+	struct device *dev = &adev->dev;
+	int ret;
+
+	ucsi = devm_kzalloc(dev, sizeof(*ucsi), GFP_KERNEL);
+	if (!ucsi)
+		return -ENOMEM;
+
+	ucsi->dev = dev;
+	dev_set_drvdata(dev, ucsi);
+
+	INIT_WORK(&ucsi->notify_work, pmic_glink_ucsi_notify);
+	INIT_WORK(&ucsi->register_work, pmic_glink_ucsi_register);
+	init_completion(&ucsi->read_ack);
+	init_completion(&ucsi->write_ack);
+	init_completion(&ucsi->sync_ack);
+	mutex_init(&ucsi->lock);
+
+	ucsi->ucsi = ucsi_create(dev, &pmic_glink_ucsi_ops);
+	if (IS_ERR(ucsi->ucsi))
+		return PTR_ERR(ucsi->ucsi);
+
+	/* Make sure we destroy *after* pmic_glink unregister */
+	ret = devm_add_action_or_reset(dev, pmic_glink_ucsi_destroy, ucsi);
+	if (ret)
+		return ret;
+
+	ucsi_set_drvdata(ucsi->ucsi, ucsi);
+
+	ucsi->client = devm_pmic_glink_register_client(dev,
+						       PMIC_GLINK_OWNER_USBC,
+						       pmic_glink_ucsi_callback,
+						       pmic_glink_ucsi_pdr_notify,
+						       ucsi);
+	return PTR_ERR_OR_ZERO(ucsi->client);
+}
+
+static void pmic_glink_ucsi_remove(struct auxiliary_device *adev)
+{
+	struct pmic_glink_ucsi *ucsi = dev_get_drvdata(&adev->dev);
+
+	/* Unregister first to stop having read & writes */
+	ucsi_unregister(ucsi->ucsi);
+}
+
+static const struct auxiliary_device_id pmic_glink_ucsi_id_table[] = {
+	{ .name = "pmic_glink.ucsi", },
+	{},
+};
+MODULE_DEVICE_TABLE(auxiliary, pmic_glink_ucsi_id_table);
+
+static struct auxiliary_driver pmic_glink_ucsi_driver = {
+	.name = "pmic_glink_ucsi",
+	.probe = pmic_glink_ucsi_probe,
+	.remove = pmic_glink_ucsi_remove,
+	.id_table = pmic_glink_ucsi_id_table,
+};
+
+module_auxiliary_driver(pmic_glink_ucsi_driver);
+
+MODULE_DESCRIPTION("Qualcomm PMIC GLINK UCSI driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c
index 195963b82b63..e29e32b306ad 100644
--- a/drivers/vdpa/mlx5/net/mlx5_vnet.c
+++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c
@@ -778,12 +778,28 @@ static bool vq_is_tx(u16 idx)
 	return idx % 2;
 }
 
-static u16 get_features_12_3(u64 features)
+enum {
+	MLX5_VIRTIO_NET_F_MRG_RXBUF = 2,
+	MLX5_VIRTIO_NET_F_HOST_ECN = 4,
+	MLX5_VIRTIO_NET_F_GUEST_ECN = 6,
+	MLX5_VIRTIO_NET_F_GUEST_TSO6 = 7,
+	MLX5_VIRTIO_NET_F_GUEST_TSO4 = 8,
+	MLX5_VIRTIO_NET_F_GUEST_CSUM = 9,
+	MLX5_VIRTIO_NET_F_CSUM = 10,
+	MLX5_VIRTIO_NET_F_HOST_TSO6 = 11,
+	MLX5_VIRTIO_NET_F_HOST_TSO4 = 12,
+};
+
+static u16 get_features(u64 features)
 {
-	return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) |
-	       (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) |
-	       (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) |
-	       (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6);
+	return (!!(features & BIT_ULL(VIRTIO_NET_F_MRG_RXBUF)) << MLX5_VIRTIO_NET_F_MRG_RXBUF) |
+	       (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_ECN)) << MLX5_VIRTIO_NET_F_HOST_ECN) |
+	       (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_ECN)) << MLX5_VIRTIO_NET_F_GUEST_ECN) |
+	       (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_TSO6)) << MLX5_VIRTIO_NET_F_GUEST_TSO6) |
+	       (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_TSO4)) << MLX5_VIRTIO_NET_F_GUEST_TSO4) |
+	       (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << MLX5_VIRTIO_NET_F_CSUM) |
+	       (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << MLX5_VIRTIO_NET_F_HOST_TSO6) |
+	       (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << MLX5_VIRTIO_NET_F_HOST_TSO4);
 }
 
 static bool counters_supported(const struct mlx5_vdpa_dev *mvdev)
@@ -797,6 +813,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
 	int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in);
 	u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {};
 	void *obj_context;
+	u16 mlx_features;
 	void *cmd_hdr;
 	void *vq_ctx;
 	void *in;
@@ -812,6 +829,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
 		goto err_alloc;
 	}
 
+	mlx_features = get_features(ndev->mvdev.actual_features);
 	cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr);
 
 	MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
@@ -822,7 +840,9 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
 	MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx);
 	MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx);
 	MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3,
-		 get_features_12_3(ndev->mvdev.actual_features));
+		 mlx_features >> 3);
+	MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_2_0,
+		 mlx_features & 7);
 	vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context);
 	MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev));
 
@@ -2171,23 +2191,27 @@ static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx)
 	return MLX5_VDPA_DATAVQ_GROUP;
 }
 
-enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9,
-	MLX5_VIRTIO_NET_F_CSUM = 1 << 10,
-	MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11,
-	MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12,
-};
-
 static u64 mlx_to_vritio_features(u16 dev_features)
 {
 	u64 result = 0;
 
-	if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM)
+	if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_MRG_RXBUF))
+		result |= BIT_ULL(VIRTIO_NET_F_MRG_RXBUF);
+	if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_HOST_ECN))
+		result |= BIT_ULL(VIRTIO_NET_F_HOST_ECN);
+	if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_GUEST_ECN))
+		result |= BIT_ULL(VIRTIO_NET_F_GUEST_ECN);
+	if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_GUEST_TSO6))
+		result |= BIT_ULL(VIRTIO_NET_F_GUEST_TSO6);
+	if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_GUEST_TSO4))
+		result |= BIT_ULL(VIRTIO_NET_F_GUEST_TSO4);
+	if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_GUEST_CSUM))
 		result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM);
-	if (dev_features & MLX5_VIRTIO_NET_F_CSUM)
+	if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_CSUM))
 		result |= BIT_ULL(VIRTIO_NET_F_CSUM);
-	if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6)
+	if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_HOST_TSO6))
 		result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6);
-	if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4)
+	if (dev_features & BIT_ULL(MLX5_VIRTIO_NET_F_HOST_TSO4))
 		result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4);
 
 	return result;
@@ -2298,6 +2322,113 @@ static void update_cvq_info(struct mlx5_vdpa_dev *mvdev)
 	}
 }
 
+static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport)
+{
+	u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {};
+	u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {};
+	int err;
+
+	MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE);
+	MLX5_SET(query_vport_state_in, in, op_mod, opmod);
+	MLX5_SET(query_vport_state_in, in, vport_number, vport);
+	if (vport)
+		MLX5_SET(query_vport_state_in, in, other_vport, 1);
+
+	err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out);
+	if (err)
+		return 0;
+
+	return MLX5_GET(query_vport_state_out, out, state);
+}
+
+static bool get_link_state(struct mlx5_vdpa_dev *mvdev)
+{
+	if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) ==
+	    VPORT_STATE_UP)
+		return true;
+
+	return false;
+}
+
+static void update_carrier(struct work_struct *work)
+{
+	struct mlx5_vdpa_wq_ent *wqent;
+	struct mlx5_vdpa_dev *mvdev;
+	struct mlx5_vdpa_net *ndev;
+
+	wqent = container_of(work, struct mlx5_vdpa_wq_ent, work);
+	mvdev = wqent->mvdev;
+	ndev = to_mlx5_vdpa_ndev(mvdev);
+	if (get_link_state(mvdev))
+		ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP);
+	else
+		ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP);
+
+	if (ndev->config_cb.callback)
+		ndev->config_cb.callback(ndev->config_cb.private);
+
+	kfree(wqent);
+}
+
+static int queue_link_work(struct mlx5_vdpa_net *ndev)
+{
+	struct mlx5_vdpa_wq_ent *wqent;
+
+	wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
+	if (!wqent)
+		return -ENOMEM;
+
+	wqent->mvdev = &ndev->mvdev;
+	INIT_WORK(&wqent->work, update_carrier);
+	queue_work(ndev->mvdev.wq, &wqent->work);
+	return 0;
+}
+
+static int event_handler(struct notifier_block *nb, unsigned long event, void *param)
+{
+	struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb);
+	struct mlx5_eqe *eqe = param;
+	int ret = NOTIFY_DONE;
+
+	if (event == MLX5_EVENT_TYPE_PORT_CHANGE) {
+		switch (eqe->sub_type) {
+		case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
+		case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
+			if (queue_link_work(ndev))
+				return NOTIFY_DONE;
+
+			ret = NOTIFY_OK;
+			break;
+		default:
+			return NOTIFY_DONE;
+		}
+		return ret;
+	}
+	return ret;
+}
+
+static void register_link_notifier(struct mlx5_vdpa_net *ndev)
+{
+	if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_STATUS)))
+		return;
+
+	ndev->nb.notifier_call = event_handler;
+	mlx5_notifier_register(ndev->mvdev.mdev, &ndev->nb);
+	ndev->nb_registered = true;
+	queue_link_work(ndev);
+}
+
+static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
+{
+	if (!ndev->nb_registered)
+		return;
+
+	ndev->nb_registered = false;
+	mlx5_notifier_unregister(ndev->mvdev.mdev, &ndev->nb);
+	if (ndev->mvdev.wq)
+		flush_workqueue(ndev->mvdev.wq);
+}
+
 static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
 {
 	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -2567,10 +2698,11 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
 				mlx5_vdpa_warn(mvdev, "failed to setup control VQ vring\n");
 				goto err_setup;
 			}
+			register_link_notifier(ndev);
 			err = setup_driver(mvdev);
 			if (err) {
 				mlx5_vdpa_warn(mvdev, "failed to setup driver\n");
-				goto err_setup;
+				goto err_driver;
 			}
 		} else {
 			mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n");
@@ -2582,6 +2714,8 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
 	up_write(&ndev->reslock);
 	return;
 
+err_driver:
+	unregister_link_notifier(ndev);
 err_setup:
 	mlx5_vdpa_destroy_mr(&ndev->mvdev);
 	ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED;
@@ -2607,6 +2741,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
 	mlx5_vdpa_info(mvdev, "performing device reset\n");
 
 	down_write(&ndev->reslock);
+	unregister_link_notifier(ndev);
 	teardown_driver(ndev);
 	clear_vqs_ready(ndev);
 	mlx5_vdpa_destroy_mr(&ndev->mvdev);
@@ -2861,9 +2996,7 @@ static int mlx5_vdpa_suspend(struct vdpa_device *vdev)
 	mlx5_vdpa_info(mvdev, "suspending device\n");
 
 	down_write(&ndev->reslock);
-	ndev->nb_registered = false;
-	mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
-	flush_workqueue(ndev->mvdev.wq);
+	unregister_link_notifier(ndev);
 	for (i = 0; i < ndev->cur_num_vqs; i++) {
 		mvq = &ndev->vqs[i];
 		suspend_vq(ndev, mvq);
@@ -3000,84 +3133,6 @@ struct mlx5_vdpa_mgmtdev {
 	struct mlx5_vdpa_net *ndev;
 };
 
-static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport)
-{
-	u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {};
-	int err;
-
-	MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE);
-	MLX5_SET(query_vport_state_in, in, op_mod, opmod);
-	MLX5_SET(query_vport_state_in, in, vport_number, vport);
-	if (vport)
-		MLX5_SET(query_vport_state_in, in, other_vport, 1);
-
-	err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out);
-	if (err)
-		return 0;
-
-	return MLX5_GET(query_vport_state_out, out, state);
-}
-
-static bool get_link_state(struct mlx5_vdpa_dev *mvdev)
-{
-	if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) ==
-	    VPORT_STATE_UP)
-		return true;
-
-	return false;
-}
-
-static void update_carrier(struct work_struct *work)
-{
-	struct mlx5_vdpa_wq_ent *wqent;
-	struct mlx5_vdpa_dev *mvdev;
-	struct mlx5_vdpa_net *ndev;
-
-	wqent = container_of(work, struct mlx5_vdpa_wq_ent, work);
-	mvdev = wqent->mvdev;
-	ndev = to_mlx5_vdpa_ndev(mvdev);
-	if (get_link_state(mvdev))
-		ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP);
-	else
-		ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP);
-
-	if (ndev->nb_registered && ndev->config_cb.callback)
-		ndev->config_cb.callback(ndev->config_cb.private);
-
-	kfree(wqent);
-}
-
-static int event_handler(struct notifier_block *nb, unsigned long event, void *param)
-{
-	struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb);
-	struct mlx5_eqe *eqe = param;
-	int ret = NOTIFY_DONE;
-	struct mlx5_vdpa_wq_ent *wqent;
-
-	if (event == MLX5_EVENT_TYPE_PORT_CHANGE) {
-		if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_STATUS)))
-			return NOTIFY_DONE;
-		switch (eqe->sub_type) {
-		case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
-		case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
-			wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
-			if (!wqent)
-				return NOTIFY_DONE;
-
-			wqent->mvdev = &ndev->mvdev;
-			INIT_WORK(&wqent->work, update_carrier);
-			queue_work(ndev->mvdev.wq, &wqent->work);
-			ret = NOTIFY_OK;
-			break;
-		default:
-			return NOTIFY_DONE;
-		}
-		return ret;
-	}
-	return ret;
-}
-
 static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu)
 {
 	int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
@@ -3127,6 +3182,8 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
 			return -EINVAL;
 		}
 		device_features &= add_config->device_features;
+	} else {
+		device_features &= ~BIT_ULL(VIRTIO_NET_F_MRG_RXBUF);
 	}
 	if (!(device_features & BIT_ULL(VIRTIO_F_VERSION_1) &&
 	      device_features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) {
@@ -3258,9 +3315,6 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
 		goto err_res2;
 	}
 
-	ndev->nb.notifier_call = event_handler;
-	mlx5_notifier_register(mdev, &ndev->nb);
-	ndev->nb_registered = true;
 	mvdev->vdev.mdev = &mgtdev->mgtdev;
 	err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1);
 	if (err)
@@ -3294,10 +3348,7 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *
 
 	mlx5_vdpa_remove_debugfs(ndev->debugfs);
 	ndev->debugfs = NULL;
-	if (ndev->nb_registered) {
-		ndev->nb_registered = false;
-		mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
-	}
+	unregister_link_notifier(ndev);
 	wq = mvdev->wq;
 	mvdev->wq = NULL;
 	destroy_workqueue(wq);
diff --git a/drivers/vdpa/solidrun/Makefile b/drivers/vdpa/solidrun/Makefile
index c0aa3415bf7b..9116252cd5fa 100644
--- a/drivers/vdpa/solidrun/Makefile
+++ b/drivers/vdpa/solidrun/Makefile
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_SNET_VDPA) += snet_vdpa.o
 snet_vdpa-$(CONFIG_SNET_VDPA) += snet_main.o
+snet_vdpa-$(CONFIG_SNET_VDPA) += snet_ctrl.o
 ifdef CONFIG_HWMON
 snet_vdpa-$(CONFIG_SNET_VDPA) += snet_hwmon.o
 endif
diff --git a/drivers/vdpa/solidrun/snet_ctrl.c b/drivers/vdpa/solidrun/snet_ctrl.c
new file mode 100644
index 000000000000..3858738643b4
--- /dev/null
+++ b/drivers/vdpa/solidrun/snet_ctrl.c
@@ -0,0 +1,330 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * SolidRun DPU driver for control plane
+ *
+ * Copyright (C) 2022-2023 SolidRun
+ *
+ * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
+ *
+ */
+
+#include <linux/iopoll.h>
+
+#include "snet_vdpa.h"
+
+enum snet_ctrl_opcodes {
+	SNET_CTRL_OP_DESTROY = 1,
+	SNET_CTRL_OP_READ_VQ_STATE,
+	SNET_CTRL_OP_SUSPEND,
+};
+
+#define SNET_CTRL_TIMEOUT	        2000000
+
+#define SNET_CTRL_DATA_SIZE_MASK	0x0000FFFF
+#define SNET_CTRL_IN_PROCESS_MASK	0x00010000
+#define SNET_CTRL_CHUNK_RDY_MASK	0x00020000
+#define SNET_CTRL_ERROR_MASK		0x0FFC0000
+
+#define SNET_VAL_TO_ERR(val)		(-(((val) & SNET_CTRL_ERROR_MASK) >> 18))
+#define SNET_EMPTY_CTRL(val)		(((val) & SNET_CTRL_ERROR_MASK) || \
+						!((val) & SNET_CTRL_IN_PROCESS_MASK))
+#define SNET_DATA_READY(val)		((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))
+
+/* Control register used to read data from the DPU */
+struct snet_ctrl_reg_ctrl {
+	/* Chunk size in 4B words */
+	u16 data_size;
+	/* We are in the middle of a command */
+	u16 in_process:1;
+	/* A data chunk is ready and can be consumed */
+	u16 chunk_ready:1;
+	/* Error code */
+	u16 error:10;
+	/* Saved for future usage */
+	u16 rsvd:4;
+};
+
+/* Opcode register */
+struct snet_ctrl_reg_op {
+	u16 opcode;
+	/* Only if VQ index is relevant for the command */
+	u16 vq_idx;
+};
+
+struct snet_ctrl_regs {
+	struct snet_ctrl_reg_op op;
+	struct snet_ctrl_reg_ctrl ctrl;
+	u32 rsvd;
+	u32 data[];
+};
+
+static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet)
+{
+	return snet->bar + snet->psnet->cfg.ctrl_off;
+}
+
+static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs)
+{
+	u32 val;
+
+	return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_EMPTY_CTRL(val), 10,
+				  SNET_CTRL_TIMEOUT);
+}
+
+static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs)
+{
+	u32 val;
+
+	return readx_poll_timeout(ioread32, &regs->op, val, !val, 10, SNET_CTRL_TIMEOUT);
+}
+
+static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs)
+{
+	u32 val;
+
+	return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_DATA_READY(val), 10,
+				  SNET_CTRL_TIMEOUT);
+}
+
+static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx)
+{
+	return ioread32(&ctrl_regs->data[word_idx]);
+}
+
+static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs)
+{
+	return ioread32(&ctrl_regs->ctrl);
+}
+
+static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
+{
+	iowrite32(val, &ctrl_regs->ctrl);
+}
+
+static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
+{
+	iowrite32(val, &ctrl_regs->op);
+}
+
+static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs)
+{
+	/* Wait until the DPU finishes completely.
+	 * It will clear the opcode register.
+	 */
+	return snet_wait_for_empty_op(ctrl_regs);
+}
+
+/* Reading ctrl from the DPU:
+ * buf_size must be 4B aligned
+ *
+ * Steps:
+ *
+ * (1) Verify that the DPU is not in the middle of another operation by
+ *     reading the in_process and error bits in the control register.
+ * (2) Write the request opcode and the VQ idx in the opcode register
+ *     and write the buffer size in the control register.
+ * (3) Start readind chunks of data, chunk_ready bit indicates that a
+ *     data chunk is available, we signal that we read the data by clearing the bit.
+ * (4) Detect that the transfer is completed when the in_process bit
+ *     in the control register is cleared or when the an error appears.
+ */
+static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer,
+				   u32 buf_size)
+{
+	struct pci_dev *pdev = snet->pdev;
+	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
+	u32 *bfr_ptr = (u32 *)buffer;
+	u32 val;
+	u16 buf_words;
+	int ret;
+	u16 words, i, tot_words = 0;
+
+	/* Supported for config 2+ */
+	if (!SNET_CFG_VER(snet, 2))
+		return -EOPNOTSUPP;
+
+	if (!IS_ALIGNED(buf_size, 4))
+		return -EINVAL;
+
+	mutex_lock(&snet->ctrl_lock);
+
+	buf_words = buf_size / 4;
+
+	/* Make sure control register is empty */
+	ret = snet_wait_for_empty_ctrl(regs);
+	if (ret) {
+		SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
+		goto exit;
+	}
+
+	/* We need to write the buffer size in the control register, and the opcode + vq index in
+	 * the opcode register.
+	 * We use a spinlock to serialize the writes.
+	 */
+	spin_lock(&snet->ctrl_spinlock);
+
+	snet_write_ctrl(regs, buf_words);
+	snet_write_op(regs, opcode | (vq_idx << 16));
+
+	spin_unlock(&snet->ctrl_spinlock);
+
+	while (buf_words != tot_words) {
+		ret = snet_wait_for_data(regs);
+		if (ret) {
+			SNET_WARN(pdev, "Timeout waiting for control data\n");
+			goto exit;
+		}
+
+		val = snet_read_ctrl(regs);
+
+		/* Error? */
+		if (val & SNET_CTRL_ERROR_MASK) {
+			ret = SNET_VAL_TO_ERR(val);
+			SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret);
+			goto exit;
+		}
+
+		words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words);
+
+		for (i = 0; i < words; i++) {
+			*bfr_ptr = snet_read32_word(regs, i);
+			bfr_ptr++;
+		}
+
+		tot_words += words;
+
+		/* Is the job completed? */
+		if (!(val & SNET_CTRL_IN_PROCESS_MASK))
+			break;
+
+		/* Clear the chunk ready bit and continue */
+		val &= ~SNET_CTRL_CHUNK_RDY_MASK;
+		snet_write_ctrl(regs, val);
+	}
+
+	ret = snet_wait_for_dpu_completion(regs);
+	if (ret)
+		SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n");
+
+exit:
+	mutex_unlock(&snet->ctrl_lock);
+	return ret;
+}
+
+/* Send a control message to the DPU using the old mechanism
+ * used with config version 1.
+ */
+static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode)
+{
+	struct pci_dev *pdev = snet->pdev;
+	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
+	int ret;
+
+	mutex_lock(&snet->ctrl_lock);
+
+	/* Old mechanism uses just 1 register, the opcode register.
+	 * Make sure that the opcode register is empty, and that the DPU isn't
+	 * processing an old message.
+	 */
+	ret = snet_wait_for_empty_op(regs);
+	if (ret) {
+		SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n");
+		goto exit;
+	}
+
+	/* Write the message */
+	snet_write_op(regs, opcode);
+
+	/* DPU ACKs the message by clearing the opcode register */
+	ret = snet_wait_for_empty_op(regs);
+	if (ret)
+		SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n");
+
+exit:
+	mutex_unlock(&snet->ctrl_lock);
+	return ret;
+}
+
+/* Send a control message to the DPU.
+ * A control message is a message without payload.
+ */
+static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx)
+{
+	struct pci_dev *pdev = snet->pdev;
+	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
+	u32 val;
+	int ret;
+
+	/* If config version is not 2+, use the old mechanism */
+	if (!SNET_CFG_VER(snet, 2))
+		return snet_send_ctrl_msg_old(snet, opcode);
+
+	mutex_lock(&snet->ctrl_lock);
+
+	/* Make sure control register is empty */
+	ret = snet_wait_for_empty_ctrl(regs);
+	if (ret) {
+		SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
+		goto exit;
+	}
+
+	/* We need to clear the control register and write the opcode + vq index in the opcode
+	 * register.
+	 * We use a spinlock to serialize the writes.
+	 */
+	spin_lock(&snet->ctrl_spinlock);
+
+	snet_write_ctrl(regs, 0);
+	snet_write_op(regs, opcode | (vq_idx << 16));
+
+	spin_unlock(&snet->ctrl_spinlock);
+
+	/* The DPU ACKs control messages by setting the chunk ready bit
+	 * without data.
+	 */
+	ret = snet_wait_for_data(regs);
+	if (ret) {
+		SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n");
+		goto exit;
+	}
+
+	/* Check for errors */
+	val = snet_read_ctrl(regs);
+	ret = SNET_VAL_TO_ERR(val);
+
+	/* Clear the chunk ready bit */
+	val &= ~SNET_CTRL_CHUNK_RDY_MASK;
+	snet_write_ctrl(regs, val);
+
+	ret = snet_wait_for_dpu_completion(regs);
+	if (ret)
+		SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n",
+			  ret);
+
+exit:
+	mutex_unlock(&snet->ctrl_lock);
+	return ret;
+}
+
+void snet_ctrl_clear(struct snet *snet)
+{
+	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
+
+	snet_write_op(regs, 0);
+}
+
+int snet_destroy_dev(struct snet *snet)
+{
+	return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0);
+}
+
+int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state)
+{
+	return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state,
+				       sizeof(*state));
+}
+
+int snet_suspend_dev(struct snet *snet)
+{
+	return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0);
+}
diff --git a/drivers/vdpa/solidrun/snet_hwmon.c b/drivers/vdpa/solidrun/snet_hwmon.c
index e695e36ff753..42c87387a0f1 100644
--- a/drivers/vdpa/solidrun/snet_hwmon.c
+++ b/drivers/vdpa/solidrun/snet_hwmon.c
@@ -2,7 +2,7 @@
 /*
  * SolidRun DPU driver for control plane
  *
- * Copyright (C) 2022 SolidRun
+ * Copyright (C) 2022-2023 SolidRun
  *
  * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
  *
diff --git a/drivers/vdpa/solidrun/snet_main.c b/drivers/vdpa/solidrun/snet_main.c
index 68de727398ed..cdcd84ce4f5a 100644
--- a/drivers/vdpa/solidrun/snet_main.c
+++ b/drivers/vdpa/solidrun/snet_main.c
@@ -2,7 +2,7 @@
 /*
  * SolidRun DPU driver for control plane
  *
- * Copyright (C) 2022 SolidRun
+ * Copyright (C) 2022-2023 SolidRun
  *
  * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
  *
@@ -16,14 +16,12 @@
 /* SNET signature */
 #define SNET_SIGNATURE          0xD0D06363
 /* Max. config version that we can work with */
-#define SNET_CFG_VERSION        0x1
+#define SNET_CFG_VERSION        0x2
 /* Queue align */
 #define SNET_QUEUE_ALIGNMENT    PAGE_SIZE
 /* Kick value to notify that new data is available */
 #define SNET_KICK_VAL           0x1
 #define SNET_CONFIG_OFF         0x0
-/* ACK timeout for a message */
-#define SNET_ACK_TIMEOUT	2000000
 /* How long we are willing to wait for a SNET device */
 #define SNET_DETECT_TIMEOUT	5000000
 /* How long should we wait for the DPU to read our config */
@@ -32,61 +30,16 @@
 #define SNET_GENERAL_CFG_LEN	36
 #define SNET_GENERAL_CFG_VQ_LEN	40
 
-enum snet_msg {
-	SNET_MSG_DESTROY = 1,
-};
-
 static struct snet *vdpa_to_snet(struct vdpa_device *vdpa)
 {
 	return container_of(vdpa, struct snet, vdpa);
 }
 
-static int snet_wait_for_msg_ack(struct snet *snet)
-{
-	struct pci_dev *pdev = snet->pdev;
-	int ret;
-	u32 val;
-
-	/* The DPU will clear the messages offset once messages
-	 * are processed.
-	 */
-	ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.msg_off,
-				 val, !val, 10, SNET_ACK_TIMEOUT);
-	if (ret)
-		SNET_WARN(pdev, "Timeout waiting for message ACK\n");
-
-	return ret;
-}
-
-/* Sends a message to the DPU.
- * If blocking is set, the function will return once the
- * message was processed by the DPU (or timeout).
- */
-static int snet_send_msg(struct snet *snet, u32 msg, bool blocking)
-{
-	int ret = 0;
-
-	/* Make sure the DPU acked last message before issuing a new one */
-	ret = snet_wait_for_msg_ack(snet);
-	if (ret)
-		return ret;
-
-	/* Write the message */
-	snet_write32(snet, snet->psnet->cfg.msg_off, msg);
-
-	if (blocking)
-		ret = snet_wait_for_msg_ack(snet);
-	else /* If non-blocking, flush the write by issuing a read */
-		snet_read32(snet, snet->psnet->cfg.msg_off);
-
-	return ret;
-}
-
 static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data)
 {
 	struct snet *snet = data;
 	/* Call callback if any */
-	if (snet->cb.callback)
+	if (likely(snet->cb.callback))
 		return snet->cb.callback(snet->cb.private);
 
 	return IRQ_HANDLED;
@@ -96,7 +49,7 @@ static irqreturn_t snet_vq_irq_hndlr(int irq, void *data)
 {
 	struct snet_vq *vq = data;
 	/* Call callback if any */
-	if (vq->cb.callback)
+	if (likely(vq->cb.callback))
 		return vq->cb.callback(vq->cb.private);
 
 	return IRQ_HANDLED;
@@ -153,12 +106,24 @@ static void snet_kick_vq(struct vdpa_device *vdev, u16 idx)
 {
 	struct snet *snet = vdpa_to_snet(vdev);
 	/* not ready - ignore */
-	if (!snet->vqs[idx]->ready)
+	if (unlikely(!snet->vqs[idx]->ready))
 		return;
 
 	iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
 }
 
+static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data)
+{
+	struct snet *snet = vdpa_to_snet(vdev);
+	u16 idx = data & 0xFFFF;
+
+	/* not ready - ignore */
+	if (unlikely(!snet->vqs[idx]->ready))
+		return;
+
+	iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
+}
+
 static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb)
 {
 	struct snet *snet = vdpa_to_snet(vdev);
@@ -181,33 +146,48 @@ static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx)
 	return snet->vqs[idx]->ready;
 }
 
-static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state)
+static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state)
 {
-	struct snet *snet = vdpa_to_snet(vdev);
-	/* Setting the VQ state is not supported.
-	 * If the asked state is the same as the initial one
-	 * we can ignore it.
-	 */
 	if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) {
 		const struct vdpa_vq_state_packed *p = &state->packed;
 
 		if (p->last_avail_counter == 1 && p->last_used_counter == 1 &&
 		    p->last_avail_idx == 0 && p->last_used_idx == 0)
-			return 0;
+			return true;
 	} else {
 		const struct vdpa_vq_state_split *s = &state->split;
 
 		if (s->avail_index == 0)
-			return 0;
+			return true;
 	}
 
+	return false;
+}
+
+static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state)
+{
+	struct snet *snet = vdpa_to_snet(vdev);
+
+	/* We can set any state for config version 2+ */
+	if (SNET_CFG_VER(snet, 2)) {
+		memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state));
+		return 0;
+	}
+
+	/* Older config - we can't set the VQ state.
+	 * Return 0 only if this is the initial state we use in the DPU.
+	 */
+	if (snet_vq_state_is_initial(snet, state))
+		return 0;
+
 	return -EOPNOTSUPP;
 }
 
 static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state)
 {
-	/* Not supported */
-	return -EOPNOTSUPP;
+	struct snet *snet = vdpa_to_snet(vdev);
+
+	return snet_read_vq_state(snet, idx, state);
 }
 
 static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx)
@@ -232,9 +212,9 @@ static int snet_reset_dev(struct snet *snet)
 	if (!snet->status)
 		return 0;
 
-	/* If DPU started, send a destroy message */
+	/* If DPU started, destroy it */
 	if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK)
-		ret = snet_send_msg(snet, SNET_MSG_DESTROY, true);
+		ret = snet_destroy_dev(snet);
 
 	/* Clear VQs */
 	for (i = 0; i < snet->cfg->vq_num; i++) {
@@ -258,7 +238,7 @@ static int snet_reset_dev(struct snet *snet)
 	snet->dpu_ready = false;
 
 	if (ret)
-		SNET_WARN(pdev, "Incomplete reset to SNET[%u] device\n", snet->sid);
+		SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret);
 	else
 		SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid);
 
@@ -356,7 +336,7 @@ static int snet_write_conf(struct snet *snet)
 	 * |             DESC AREA                |
 	 * |            DEVICE AREA               |
 	 * |            DRIVER AREA               |
-	 * |             RESERVED                 |
+	 * |    VQ STATE (CFG 2+)     |   RSVD    |
 	 *
 	 * Magic number should be written last, this is the DPU indication that the data is ready
 	 */
@@ -391,12 +371,15 @@ static int snet_write_conf(struct snet *snet)
 		off += 8;
 		snet_write64(snet, off, snet->vqs[i]->driver_area);
 		off += 8;
+		/* Write VQ state if config version is 2+ */
+		if (SNET_CFG_VER(snet, 2))
+			snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state);
+		off += 4;
+
 		/* Ignore reserved */
-		off += 8;
+		off += 4;
 	}
 
-	/* Clear snet messages address for this device */
-	snet_write32(snet, snet->psnet->cfg.msg_off, 0);
 	/* Write magic number - data is ready */
 	snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE);
 
@@ -512,10 +495,25 @@ static void snet_set_config(struct vdpa_device *vdev, unsigned int offset,
 		iowrite8(*buf_ptr++, cfg_ptr + i);
 }
 
+static int snet_suspend(struct vdpa_device *vdev)
+{
+	struct snet *snet = vdpa_to_snet(vdev);
+	int ret;
+
+	ret = snet_suspend_dev(snet);
+	if (ret)
+		SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret);
+	else
+		SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid);
+
+	return ret;
+}
+
 static const struct vdpa_config_ops snet_config_ops = {
 	.set_vq_address         = snet_set_vq_address,
 	.set_vq_num             = snet_set_vq_num,
 	.kick_vq                = snet_kick_vq,
+	.kick_vq_with_data	= snet_kick_vq_with_data,
 	.set_vq_cb              = snet_set_vq_cb,
 	.set_vq_ready           = snet_set_vq_ready,
 	.get_vq_ready           = snet_get_vq_ready,
@@ -537,6 +535,7 @@ static const struct vdpa_config_ops snet_config_ops = {
 	.set_status             = snet_set_status,
 	.get_config             = snet_get_config,
 	.set_config             = snet_set_config,
+	.suspend		= snet_suspend,
 };
 
 static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet)
@@ -697,7 +696,7 @@ static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet)
 	off += 4;
 	cfg->hwmon_off = psnet_read32(psnet, off);
 	off += 4;
-	cfg->msg_off = psnet_read32(psnet, off);
+	cfg->ctrl_off = psnet_read32(psnet, off);
 	off += 4;
 	cfg->flags = psnet_read32(psnet, off);
 	off += 4;
@@ -997,6 +996,10 @@ static int snet_vdpa_probe_vf(struct pci_dev *pdev)
 		goto free_irqs;
 	}
 
+	/* Init control mutex and spinlock */
+	mutex_init(&snet->ctrl_lock);
+	spin_lock_init(&snet->ctrl_spinlock);
+
 	/* Save pci device pointer */
 	snet->pdev = pdev;
 	snet->psnet = psnet;
@@ -1013,6 +1016,9 @@ static int snet_vdpa_probe_vf(struct pci_dev *pdev)
 	/* Create a VirtIO config pointer */
 	snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off;
 
+	/* Clear control registers */
+	snet_ctrl_clear(snet);
+
 	pci_set_master(pdev);
 	pci_set_drvdata(pdev, snet);
 
diff --git a/drivers/vdpa/solidrun/snet_vdpa.h b/drivers/vdpa/solidrun/snet_vdpa.h
index b7f34169053f..3c78d4e7d485 100644
--- a/drivers/vdpa/solidrun/snet_vdpa.h
+++ b/drivers/vdpa/solidrun/snet_vdpa.h
@@ -2,7 +2,7 @@
 /*
  * SolidRun DPU driver for control plane
  *
- * Copyright (C) 2022 SolidRun
+ * Copyright (C) 2022-2023 SolidRun
  *
  * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
  *
@@ -20,10 +20,15 @@
 #define SNET_INFO(pdev, fmt, ...) dev_info(&(pdev)->dev, "%s"fmt, "snet_vdpa: ", ##__VA_ARGS__)
 #define SNET_DBG(pdev, fmt, ...) dev_dbg(&(pdev)->dev, "%s"fmt, "snet_vdpa: ", ##__VA_ARGS__)
 #define SNET_HAS_FEATURE(s, f) ((s)->negotiated_features & BIT_ULL(f))
+/* Check if negotiated config version is at least @ver */
+#define SNET_CFG_VER(snet, ver) ((snet)->psnet->negotiated_cfg_ver >= (ver))
+
 /* VQ struct */
 struct snet_vq {
 	/* VQ callback */
 	struct vdpa_callback cb;
+	/* VQ state received from bus */
+	struct vdpa_vq_state vq_state;
 	/* desc base address */
 	u64 desc_area;
 	/* device base address */
@@ -51,6 +56,10 @@ struct snet {
 	struct vdpa_device vdpa;
 	/* Config callback */
 	struct vdpa_callback cb;
+	/* To lock the control mechanism */
+	struct mutex ctrl_lock;
+	/* Spinlock to protect critical parts in the control mechanism */
+	spinlock_t ctrl_spinlock;
 	/* array of virqueues */
 	struct snet_vq **vqs;
 	/* Used features */
@@ -117,8 +126,8 @@ struct snet_cfg {
 	u32 kick_off;
 	/* Offset in PCI BAR for HW monitoring */
 	u32 hwmon_off;
-	/* Offset in PCI BAR for SNET messages */
-	u32 msg_off;
+	/* Offset in PCI BAR for Control mechanism */
+	u32 ctrl_off;
 	/* Config general flags - enum snet_cfg_flags */
 	u32 flags;
 	/* Reserved for future usage */
@@ -191,4 +200,9 @@ static inline void snet_write64(struct snet *snet, u32 off, u64 val)
 void psnet_create_hwmon(struct pci_dev *pdev);
 #endif
 
+void snet_ctrl_clear(struct snet *snet);
+int snet_destroy_dev(struct snet *snet);
+int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state);
+int snet_suspend_dev(struct snet *snet);
+
 #endif //_SNET_VDPA_H_
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index eea23c630f7c..d343af4fa60e 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -11,8 +11,8 @@
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/kernel.h>
+#include <linux/kthread.h>
 #include <linux/slab.h>
-#include <linux/sched.h>
 #include <linux/dma-map-ops.h>
 #include <linux/vringh.h>
 #include <linux/vdpa.h>
@@ -35,10 +35,44 @@ module_param(max_iotlb_entries, int, 0444);
 MODULE_PARM_DESC(max_iotlb_entries,
 		 "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)");
 
+static bool use_va = true;
+module_param(use_va, bool, 0444);
+MODULE_PARM_DESC(use_va, "Enable/disable the device's ability to use VA");
+
 #define VDPASIM_QUEUE_ALIGN PAGE_SIZE
 #define VDPASIM_QUEUE_MAX 256
 #define VDPASIM_VENDOR_ID 0
 
+struct vdpasim_mm_work {
+	struct kthread_work work;
+	struct vdpasim *vdpasim;
+	struct mm_struct *mm_to_bind;
+	int ret;
+};
+
+static void vdpasim_mm_work_fn(struct kthread_work *work)
+{
+	struct vdpasim_mm_work *mm_work =
+		container_of(work, struct vdpasim_mm_work, work);
+	struct vdpasim *vdpasim = mm_work->vdpasim;
+
+	mm_work->ret = 0;
+
+	//TODO: should we attach the cgroup of the mm owner?
+	vdpasim->mm_bound = mm_work->mm_to_bind;
+}
+
+static void vdpasim_worker_change_mm_sync(struct vdpasim *vdpasim,
+					  struct vdpasim_mm_work *mm_work)
+{
+	struct kthread_work *work = &mm_work->work;
+
+	kthread_init_work(work, vdpasim_mm_work_fn);
+	kthread_queue_work(vdpasim->worker, work);
+
+	kthread_flush_work(work);
+}
+
 static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa)
 {
 	return container_of(vdpa, struct vdpasim, vdpa);
@@ -59,13 +93,20 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
 {
 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
 	uint16_t last_avail_idx = vq->vring.last_avail_idx;
-
-	vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, true,
-			  (struct vring_desc *)(uintptr_t)vq->desc_addr,
-			  (struct vring_avail *)
-			  (uintptr_t)vq->driver_addr,
-			  (struct vring_used *)
-			  (uintptr_t)vq->device_addr);
+	struct vring_desc *desc = (struct vring_desc *)
+				  (uintptr_t)vq->desc_addr;
+	struct vring_avail *avail = (struct vring_avail *)
+				    (uintptr_t)vq->driver_addr;
+	struct vring_used *used = (struct vring_used *)
+				  (uintptr_t)vq->device_addr;
+
+	if (use_va && vdpasim->mm_bound) {
+		vringh_init_iotlb_va(&vq->vring, vdpasim->features, vq->num,
+				     true, desc, avail, used);
+	} else {
+		vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num,
+				  true, desc, avail, used);
+	}
 
 	vq->vring.last_avail_idx = last_avail_idx;
 
@@ -127,6 +168,25 @@ static void vdpasim_do_reset(struct vdpasim *vdpasim)
 static const struct vdpa_config_ops vdpasim_config_ops;
 static const struct vdpa_config_ops vdpasim_batch_config_ops;
 
+static void vdpasim_work_fn(struct kthread_work *work)
+{
+	struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
+	struct mm_struct *mm = vdpasim->mm_bound;
+
+	if (use_va && mm) {
+		if (!mmget_not_zero(mm))
+			return;
+		kthread_use_mm(mm);
+	}
+
+	vdpasim->dev_attr.work_fn(vdpasim);
+
+	if (use_va && mm) {
+		kthread_unuse_mm(mm);
+		mmput(mm);
+	}
+}
+
 struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
 			       const struct vdpa_dev_set_config *config)
 {
@@ -155,7 +215,7 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
 	vdpa = __vdpa_alloc_device(NULL, ops,
 				   dev_attr->ngroups, dev_attr->nas,
 				   dev_attr->alloc_size,
-				   dev_attr->name, false);
+				   dev_attr->name, use_va);
 	if (IS_ERR(vdpa)) {
 		ret = PTR_ERR(vdpa);
 		goto err_alloc;
@@ -163,11 +223,17 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
 
 	vdpasim = vdpa_to_sim(vdpa);
 	vdpasim->dev_attr = *dev_attr;
-	INIT_WORK(&vdpasim->work, dev_attr->work_fn);
-	spin_lock_init(&vdpasim->lock);
+	dev = &vdpasim->vdpa.dev;
+
+	kthread_init_work(&vdpasim->work, vdpasim_work_fn);
+	vdpasim->worker = kthread_create_worker(0, "vDPA sim worker: %s",
+						dev_attr->name);
+	if (IS_ERR(vdpasim->worker))
+		goto err_iommu;
+
+	mutex_init(&vdpasim->mutex);
 	spin_lock_init(&vdpasim->iommu_lock);
 
-	dev = &vdpasim->vdpa.dev;
 	dev->dma_mask = &dev->coherent_dma_mask;
 	if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))
 		goto err_iommu;
@@ -195,10 +261,6 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
 	for (i = 0; i < vdpasim->dev_attr.nas; i++)
 		vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);
 
-	vdpasim->buffer = kvmalloc(dev_attr->buffer_size, GFP_KERNEL);
-	if (!vdpasim->buffer)
-		goto err_iommu;
-
 	for (i = 0; i < dev_attr->nvqs; i++)
 		vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],
 				 &vdpasim->iommu_lock);
@@ -214,6 +276,12 @@ err_alloc:
 }
 EXPORT_SYMBOL_GPL(vdpasim_create);
 
+void vdpasim_schedule_work(struct vdpasim *vdpasim)
+{
+	kthread_queue_work(vdpasim->worker, &vdpasim->work);
+}
+EXPORT_SYMBOL_GPL(vdpasim_schedule_work);
+
 static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx,
 				  u64 desc_area, u64 driver_area,
 				  u64 device_area)
@@ -248,7 +316,7 @@ static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
 	}
 
 	if (vq->ready)
-		schedule_work(&vdpasim->work);
+		vdpasim_schedule_work(vdpasim);
 }
 
 static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
@@ -267,13 +335,13 @@ static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready)
 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
 	bool old_ready;
 
-	spin_lock(&vdpasim->lock);
+	mutex_lock(&vdpasim->mutex);
 	old_ready = vq->ready;
 	vq->ready = ready;
 	if (vq->ready && !old_ready) {
 		vdpasim_queue_ready(vdpasim, idx);
 	}
-	spin_unlock(&vdpasim->lock);
+	mutex_unlock(&vdpasim->mutex);
 }
 
 static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
@@ -291,9 +359,9 @@ static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx,
 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
 	struct vringh *vrh = &vq->vring;
 
-	spin_lock(&vdpasim->lock);
+	mutex_lock(&vdpasim->mutex);
 	vrh->last_avail_idx = state->split.avail_index;
-	spin_unlock(&vdpasim->lock);
+	mutex_unlock(&vdpasim->mutex);
 
 	return 0;
 }
@@ -390,9 +458,9 @@ static u8 vdpasim_get_status(struct vdpa_device *vdpa)
 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
 	u8 status;
 
-	spin_lock(&vdpasim->lock);
+	mutex_lock(&vdpasim->mutex);
 	status = vdpasim->status;
-	spin_unlock(&vdpasim->lock);
+	mutex_unlock(&vdpasim->mutex);
 
 	return status;
 }
@@ -401,19 +469,19 @@ static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status)
 {
 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
 
-	spin_lock(&vdpasim->lock);
+	mutex_lock(&vdpasim->mutex);
 	vdpasim->status = status;
-	spin_unlock(&vdpasim->lock);
+	mutex_unlock(&vdpasim->mutex);
 }
 
 static int vdpasim_reset(struct vdpa_device *vdpa)
 {
 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
 
-	spin_lock(&vdpasim->lock);
+	mutex_lock(&vdpasim->mutex);
 	vdpasim->status = 0;
 	vdpasim_do_reset(vdpasim);
-	spin_unlock(&vdpasim->lock);
+	mutex_unlock(&vdpasim->mutex);
 
 	return 0;
 }
@@ -422,9 +490,9 @@ static int vdpasim_suspend(struct vdpa_device *vdpa)
 {
 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
 
-	spin_lock(&vdpasim->lock);
+	mutex_lock(&vdpasim->mutex);
 	vdpasim->running = false;
-	spin_unlock(&vdpasim->lock);
+	mutex_unlock(&vdpasim->mutex);
 
 	return 0;
 }
@@ -434,7 +502,7 @@ static int vdpasim_resume(struct vdpa_device *vdpa)
 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
 	int i;
 
-	spin_lock(&vdpasim->lock);
+	mutex_lock(&vdpasim->mutex);
 	vdpasim->running = true;
 
 	if (vdpasim->pending_kick) {
@@ -445,7 +513,7 @@ static int vdpasim_resume(struct vdpa_device *vdpa)
 		vdpasim->pending_kick = false;
 	}
 
-	spin_unlock(&vdpasim->lock);
+	mutex_unlock(&vdpasim->mutex);
 
 	return 0;
 }
@@ -517,14 +585,14 @@ static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group,
 
 	iommu = &vdpasim->iommu[asid];
 
-	spin_lock(&vdpasim->lock);
+	mutex_lock(&vdpasim->mutex);
 
 	for (i = 0; i < vdpasim->dev_attr.nvqs; i++)
 		if (vdpasim_get_vq_group(vdpa, i) == group)
 			vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu,
 					 &vdpasim->iommu_lock);
 
-	spin_unlock(&vdpasim->lock);
+	mutex_unlock(&vdpasim->mutex);
 
 	return 0;
 }
@@ -563,6 +631,30 @@ err:
 	return ret;
 }
 
+static int vdpasim_bind_mm(struct vdpa_device *vdpa, struct mm_struct *mm)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_mm_work mm_work;
+
+	mm_work.vdpasim = vdpasim;
+	mm_work.mm_to_bind = mm;
+
+	vdpasim_worker_change_mm_sync(vdpasim, &mm_work);
+
+	return mm_work.ret;
+}
+
+static void vdpasim_unbind_mm(struct vdpa_device *vdpa)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_mm_work mm_work;
+
+	mm_work.vdpasim = vdpasim;
+	mm_work.mm_to_bind = NULL;
+
+	vdpasim_worker_change_mm_sync(vdpasim, &mm_work);
+}
+
 static int vdpasim_dma_map(struct vdpa_device *vdpa, unsigned int asid,
 			   u64 iova, u64 size,
 			   u64 pa, u32 perm, void *opaque)
@@ -610,14 +702,16 @@ static void vdpasim_free(struct vdpa_device *vdpa)
 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
 	int i;
 
-	cancel_work_sync(&vdpasim->work);
+	kthread_cancel_work_sync(&vdpasim->work);
+	kthread_destroy_worker(vdpasim->worker);
 
 	for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {
 		vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov);
 		vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov);
 	}
 
-	kvfree(vdpasim->buffer);
+	vdpasim->dev_attr.free(vdpasim);
+
 	for (i = 0; i < vdpasim->dev_attr.nas; i++)
 		vhost_iotlb_reset(&vdpasim->iommu[i]);
 	kfree(vdpasim->iommu);
@@ -658,6 +752,8 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
 	.set_group_asid         = vdpasim_set_group_asid,
 	.dma_map                = vdpasim_dma_map,
 	.dma_unmap              = vdpasim_dma_unmap,
+	.bind_mm		= vdpasim_bind_mm,
+	.unbind_mm		= vdpasim_unbind_mm,
 	.free                   = vdpasim_free,
 };
 
@@ -692,6 +788,8 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
 	.get_iova_range         = vdpasim_get_iova_range,
 	.set_group_asid         = vdpasim_set_group_asid,
 	.set_map                = vdpasim_set_map,
+	.bind_mm		= vdpasim_bind_mm,
+	.unbind_mm		= vdpasim_unbind_mm,
 	.free                   = vdpasim_free,
 };
 
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h
index 144858636c10..bb137e479763 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
@@ -39,33 +39,34 @@ struct vdpasim_dev_attr {
 	u64 supported_features;
 	size_t alloc_size;
 	size_t config_size;
-	size_t buffer_size;
 	int nvqs;
 	u32 id;
 	u32 ngroups;
 	u32 nas;
 
-	work_func_t work_fn;
+	void (*work_fn)(struct vdpasim *vdpasim);
 	void (*get_config)(struct vdpasim *vdpasim, void *config);
 	void (*set_config)(struct vdpasim *vdpasim, const void *config);
 	int (*get_stats)(struct vdpasim *vdpasim, u16 idx,
 			 struct sk_buff *msg,
 			 struct netlink_ext_ack *extack);
+	void (*free)(struct vdpasim *vdpasim);
 };
 
 /* State of each vdpasim device */
 struct vdpasim {
 	struct vdpa_device vdpa;
 	struct vdpasim_virtqueue *vqs;
-	struct work_struct work;
+	struct kthread_worker *worker;
+	struct kthread_work work;
+	struct mm_struct *mm_bound;
 	struct vdpasim_dev_attr dev_attr;
-	/* spinlock to synchronize virtqueue state */
-	spinlock_t lock;
+	/* mutex to synchronize virtqueue state */
+	struct mutex mutex;
 	/* virtio config according to device type */
 	void *config;
 	struct vhost_iotlb *iommu;
 	bool *iommu_pt;
-	void *buffer;
 	u32 status;
 	u32 generation;
 	u64 features;
@@ -78,6 +79,7 @@ struct vdpasim {
 
 struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr,
 			       const struct vdpa_dev_set_config *config);
+void vdpasim_schedule_work(struct vdpasim *vdpasim);
 
 /* TODO: cross-endian support */
 static inline bool vdpasim_is_little_endian(struct vdpasim *vdpasim)
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c
index 5117959bed8a..00d7d72713be 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c
@@ -11,7 +11,6 @@
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/kernel.h>
-#include <linux/sched.h>
 #include <linux/blkdev.h>
 #include <linux/vringh.h>
 #include <linux/vdpa.h>
@@ -44,8 +43,39 @@
 #define VDPASIM_BLK_AS_NUM	1
 #define VDPASIM_BLK_GROUP_NUM	1
 
+struct vdpasim_blk {
+	struct vdpasim vdpasim;
+	void *buffer;
+	bool shared_backend;
+};
+
+static struct vdpasim_blk *sim_to_blk(struct vdpasim *vdpasim)
+{
+	return container_of(vdpasim, struct vdpasim_blk, vdpasim);
+}
+
 static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim";
 
+static bool shared_backend;
+module_param(shared_backend, bool, 0444);
+MODULE_PARM_DESC(shared_backend, "Enable the shared backend between virtio-blk devices");
+
+static void *shared_buffer;
+/* mutex to synchronize shared_buffer access */
+static DEFINE_MUTEX(shared_buffer_mutex);
+
+static void vdpasim_blk_buffer_lock(struct vdpasim_blk *blk)
+{
+	if (blk->shared_backend)
+		mutex_lock(&shared_buffer_mutex);
+}
+
+static void vdpasim_blk_buffer_unlock(struct vdpasim_blk *blk)
+{
+	if (blk->shared_backend)
+		mutex_unlock(&shared_buffer_mutex);
+}
+
 static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector,
 				    u64 num_sectors, u64 max_sectors)
 {
@@ -79,6 +109,7 @@ static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector,
 static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
 				   struct vdpasim_virtqueue *vq)
 {
+	struct vdpasim_blk *blk = sim_to_blk(vdpasim);
 	size_t pushed = 0, to_pull, to_push;
 	struct virtio_blk_outhdr hdr;
 	bool handled = false;
@@ -144,9 +175,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
 			break;
 		}
 
+		vdpasim_blk_buffer_lock(blk);
 		bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov,
-					      vdpasim->buffer + offset,
-					      to_push);
+					      blk->buffer + offset, to_push);
+		vdpasim_blk_buffer_unlock(blk);
 		if (bytes < 0) {
 			dev_dbg(&vdpasim->vdpa.dev,
 				"vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
@@ -166,9 +198,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
 			break;
 		}
 
+		vdpasim_blk_buffer_lock(blk);
 		bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov,
-					      vdpasim->buffer + offset,
-					      to_pull);
+					      blk->buffer + offset, to_pull);
+		vdpasim_blk_buffer_unlock(blk);
 		if (bytes < 0) {
 			dev_dbg(&vdpasim->vdpa.dev,
 				"vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
@@ -248,8 +281,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
 		}
 
 		if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
-			memset(vdpasim->buffer + offset, 0,
+			vdpasim_blk_buffer_lock(blk);
+			memset(blk->buffer + offset, 0,
 			       num_sectors << SECTOR_SHIFT);
+			vdpasim_blk_buffer_unlock(blk);
 		}
 
 		break;
@@ -286,13 +321,12 @@ err:
 	return handled;
 }
 
-static void vdpasim_blk_work(struct work_struct *work)
+static void vdpasim_blk_work(struct vdpasim *vdpasim)
 {
-	struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
 	bool reschedule = false;
 	int i;
 
-	spin_lock(&vdpasim->lock);
+	mutex_lock(&vdpasim->mutex);
 
 	if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
 		goto out;
@@ -323,10 +357,10 @@ static void vdpasim_blk_work(struct work_struct *work)
 		}
 	}
 out:
-	spin_unlock(&vdpasim->lock);
+	mutex_unlock(&vdpasim->mutex);
 
 	if (reschedule)
-		schedule_work(&vdpasim->work);
+		vdpasim_schedule_work(vdpasim);
 }
 
 static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config)
@@ -355,6 +389,14 @@ static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config)
 
 }
 
+static void vdpasim_blk_free(struct vdpasim *vdpasim)
+{
+	struct vdpasim_blk *blk = sim_to_blk(vdpasim);
+
+	if (!blk->shared_backend)
+		kvfree(blk->buffer);
+}
+
 static void vdpasim_blk_mgmtdev_release(struct device *dev)
 {
 }
@@ -368,6 +410,7 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
 			       const struct vdpa_dev_set_config *config)
 {
 	struct vdpasim_dev_attr dev_attr = {};
+	struct vdpasim_blk *blk;
 	struct vdpasim *simdev;
 	int ret;
 
@@ -378,16 +421,30 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
 	dev_attr.nvqs = VDPASIM_BLK_VQ_NUM;
 	dev_attr.ngroups = VDPASIM_BLK_GROUP_NUM;
 	dev_attr.nas = VDPASIM_BLK_AS_NUM;
-	dev_attr.alloc_size = sizeof(struct vdpasim);
+	dev_attr.alloc_size = sizeof(struct vdpasim_blk);
 	dev_attr.config_size = sizeof(struct virtio_blk_config);
 	dev_attr.get_config = vdpasim_blk_get_config;
 	dev_attr.work_fn = vdpasim_blk_work;
-	dev_attr.buffer_size = VDPASIM_BLK_CAPACITY << SECTOR_SHIFT;
+	dev_attr.free = vdpasim_blk_free;
 
 	simdev = vdpasim_create(&dev_attr, config);
 	if (IS_ERR(simdev))
 		return PTR_ERR(simdev);
 
+	blk = sim_to_blk(simdev);
+	blk->shared_backend = shared_backend;
+
+	if (blk->shared_backend) {
+		blk->buffer = shared_buffer;
+	} else {
+		blk->buffer = kvmalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT,
+				       GFP_KERNEL);
+		if (!blk->buffer) {
+			ret = -ENOMEM;
+			goto put_dev;
+		}
+	}
+
 	ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_BLK_VQ_NUM);
 	if (ret)
 		goto put_dev;
@@ -437,6 +494,15 @@ static int __init vdpasim_blk_init(void)
 	if (ret)
 		goto parent_err;
 
+	if (shared_backend) {
+		shared_buffer = kvmalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT,
+					 GFP_KERNEL);
+		if (!shared_buffer) {
+			ret = -ENOMEM;
+			goto parent_err;
+		}
+	}
+
 	return 0;
 
 parent_err:
@@ -446,6 +512,7 @@ parent_err:
 
 static void __exit vdpasim_blk_exit(void)
 {
+	kvfree(shared_buffer);
 	vdpa_mgmtdev_unregister(&mgmt_dev);
 	device_unregister(&vdpasim_blk_mgmtdev);
 }
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c
index dfe2ce341803..cfe962911804 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c
@@ -11,7 +11,6 @@
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/kernel.h>
-#include <linux/sched.h>
 #include <linux/etherdevice.h>
 #include <linux/vringh.h>
 #include <linux/vdpa.h>
@@ -59,6 +58,7 @@ struct vdpasim_net{
 	struct vdpasim_dataq_stats tx_stats;
 	struct vdpasim_dataq_stats rx_stats;
 	struct vdpasim_cq_stats cq_stats;
+	void *buffer;
 };
 
 static struct vdpasim_net *sim_to_net(struct vdpasim *vdpasim)
@@ -88,14 +88,15 @@ static bool receive_filter(struct vdpasim *vdpasim, size_t len)
 	size_t hdr_len = modern ? sizeof(struct virtio_net_hdr_v1) :
 				  sizeof(struct virtio_net_hdr);
 	struct virtio_net_config *vio_config = vdpasim->config;
+	struct vdpasim_net *net = sim_to_net(vdpasim);
 
 	if (len < ETH_ALEN + hdr_len)
 		return false;
 
-	if (is_broadcast_ether_addr(vdpasim->buffer + hdr_len) ||
-	    is_multicast_ether_addr(vdpasim->buffer + hdr_len))
+	if (is_broadcast_ether_addr(net->buffer + hdr_len) ||
+	    is_multicast_ether_addr(net->buffer + hdr_len))
 		return true;
-	if (!strncmp(vdpasim->buffer + hdr_len, vio_config->mac, ETH_ALEN))
+	if (!strncmp(net->buffer + hdr_len, vio_config->mac, ETH_ALEN))
 		return true;
 
 	return false;
@@ -192,9 +193,8 @@ static void vdpasim_handle_cvq(struct vdpasim *vdpasim)
 	u64_stats_update_end(&net->cq_stats.syncp);
 }
 
-static void vdpasim_net_work(struct work_struct *work)
+static void vdpasim_net_work(struct vdpasim *vdpasim)
 {
-	struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
 	struct vdpasim_virtqueue *txq = &vdpasim->vqs[1];
 	struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0];
 	struct vdpasim_net *net = sim_to_net(vdpasim);
@@ -203,7 +203,7 @@ static void vdpasim_net_work(struct work_struct *work)
 	u64 rx_drops = 0, rx_overruns = 0, rx_errors = 0, tx_errors = 0;
 	int err;
 
-	spin_lock(&vdpasim->lock);
+	mutex_lock(&vdpasim->mutex);
 
 	if (!vdpasim->running)
 		goto out;
@@ -227,8 +227,7 @@ static void vdpasim_net_work(struct work_struct *work)
 
 		++tx_pkts;
 		read = vringh_iov_pull_iotlb(&txq->vring, &txq->out_iov,
-					     vdpasim->buffer,
-					     PAGE_SIZE);
+					     net->buffer, PAGE_SIZE);
 
 		tx_bytes += read;
 
@@ -247,7 +246,7 @@ static void vdpasim_net_work(struct work_struct *work)
 		}
 
 		write = vringh_iov_push_iotlb(&rxq->vring, &rxq->in_iov,
-					      vdpasim->buffer, read);
+					      net->buffer, read);
 		if (write <= 0) {
 			++rx_errors;
 			break;
@@ -260,13 +259,13 @@ static void vdpasim_net_work(struct work_struct *work)
 		vdpasim_net_complete(rxq, write);
 
 		if (tx_pkts > 4) {
-			schedule_work(&vdpasim->work);
+			vdpasim_schedule_work(vdpasim);
 			goto out;
 		}
 	}
 
 out:
-	spin_unlock(&vdpasim->lock);
+	mutex_unlock(&vdpasim->mutex);
 
 	u64_stats_update_begin(&net->tx_stats.syncp);
 	net->tx_stats.pkts += tx_pkts;
@@ -429,6 +428,13 @@ static void vdpasim_net_setup_config(struct vdpasim *vdpasim,
 		vio_config->mtu = cpu_to_vdpasim16(vdpasim, 1500);
 }
 
+static void vdpasim_net_free(struct vdpasim *vdpasim)
+{
+	struct vdpasim_net *net = sim_to_net(vdpasim);
+
+	kvfree(net->buffer);
+}
+
 static void vdpasim_net_mgmtdev_release(struct device *dev)
 {
 }
@@ -458,7 +464,7 @@ static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
 	dev_attr.get_config = vdpasim_net_get_config;
 	dev_attr.work_fn = vdpasim_net_work;
 	dev_attr.get_stats = vdpasim_net_get_stats;
-	dev_attr.buffer_size = PAGE_SIZE;
+	dev_attr.free = vdpasim_net_free;
 
 	simdev = vdpasim_create(&dev_attr, config);
 	if (IS_ERR(simdev))
@@ -472,6 +478,12 @@ static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
 	u64_stats_init(&net->rx_stats.syncp);
 	u64_stats_init(&net->cq_stats.syncp);
 
+	net->buffer = kvmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!net->buffer) {
+		ret = -ENOMEM;
+		goto reg_err;
+	}
+
 	/*
 	 * Initialization must be completed before this call, since it can
 	 * connect the device to the vDPA bus, so requests can arrive after
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index 0c3b48616a9f..de97e38c3b82 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -37,10 +37,15 @@
 #define DRV_LICENSE  "GPL v2"
 
 #define VDUSE_DEV_MAX (1U << MINORBITS)
+#define VDUSE_MAX_BOUNCE_SIZE (1024 * 1024 * 1024)
+#define VDUSE_MIN_BOUNCE_SIZE (1024 * 1024)
 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)
-#define VDUSE_IOVA_SIZE (128 * 1024 * 1024)
+/* 128 MB reserved for virtqueue creation */
+#define VDUSE_IOVA_SIZE (VDUSE_MAX_BOUNCE_SIZE + 128 * 1024 * 1024)
 #define VDUSE_MSG_DEFAULT_TIMEOUT 30
 
+#define IRQ_UNBOUND -1
+
 struct vduse_virtqueue {
 	u16 index;
 	u16 num_max;
@@ -57,6 +62,9 @@ struct vduse_virtqueue {
 	struct vdpa_callback cb;
 	struct work_struct inject;
 	struct work_struct kick;
+	int irq_effective_cpu;
+	struct cpumask irq_affinity;
+	struct kobject kobj;
 };
 
 struct vduse_dev;
@@ -76,7 +84,7 @@ struct vduse_umem {
 struct vduse_dev {
 	struct vduse_vdpa *vdev;
 	struct device *dev;
-	struct vduse_virtqueue *vqs;
+	struct vduse_virtqueue **vqs;
 	struct vduse_iova_domain *domain;
 	char *name;
 	struct mutex lock;
@@ -106,6 +114,8 @@ struct vduse_dev {
 	u32 vq_align;
 	struct vduse_umem *umem;
 	struct mutex mem_lock;
+	unsigned int bounce_size;
+	struct mutex domain_lock;
 };
 
 struct vduse_dev_msg {
@@ -128,6 +138,7 @@ static struct class *vduse_class;
 static struct cdev vduse_ctrl_cdev;
 static struct cdev vduse_cdev;
 static struct workqueue_struct *vduse_irq_wq;
+static struct workqueue_struct *vduse_irq_bound_wq;
 
 static u32 allowed_device_id[] = {
 	VIRTIO_ID_BLOCK,
@@ -419,7 +430,7 @@ static void vduse_dev_reset(struct vduse_dev *dev)
 	struct vduse_iova_domain *domain = dev->domain;
 
 	/* The coherent mappings are handled in vduse_dev_free_coherent() */
-	if (domain->bounce_map)
+	if (domain && domain->bounce_map)
 		vduse_domain_reset_bounce_map(domain);
 
 	down_write(&dev->rwsem);
@@ -434,7 +445,7 @@ static void vduse_dev_reset(struct vduse_dev *dev)
 	flush_work(&dev->inject);
 
 	for (i = 0; i < dev->vq_num; i++) {
-		struct vduse_virtqueue *vq = &dev->vqs[i];
+		struct vduse_virtqueue *vq = dev->vqs[i];
 
 		vq->ready = false;
 		vq->desc_addr = 0;
@@ -453,6 +464,7 @@ static void vduse_dev_reset(struct vduse_dev *dev)
 		spin_lock(&vq->irq_lock);
 		vq->cb.callback = NULL;
 		vq->cb.private = NULL;
+		vq->cb.trigger = NULL;
 		spin_unlock(&vq->irq_lock);
 		flush_work(&vq->inject);
 		flush_work(&vq->kick);
@@ -466,7 +478,7 @@ static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
 				u64 device_area)
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
-	struct vduse_virtqueue *vq = &dev->vqs[idx];
+	struct vduse_virtqueue *vq = dev->vqs[idx];
 
 	vq->desc_addr = desc_area;
 	vq->driver_addr = driver_area;
@@ -500,7 +512,7 @@ static void vduse_vq_kick_work(struct work_struct *work)
 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
-	struct vduse_virtqueue *vq = &dev->vqs[idx];
+	struct vduse_virtqueue *vq = dev->vqs[idx];
 
 	if (!eventfd_signal_allowed()) {
 		schedule_work(&vq->kick);
@@ -513,18 +525,19 @@ static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
 			      struct vdpa_callback *cb)
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
-	struct vduse_virtqueue *vq = &dev->vqs[idx];
+	struct vduse_virtqueue *vq = dev->vqs[idx];
 
 	spin_lock(&vq->irq_lock);
 	vq->cb.callback = cb->callback;
 	vq->cb.private = cb->private;
+	vq->cb.trigger = cb->trigger;
 	spin_unlock(&vq->irq_lock);
 }
 
 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
-	struct vduse_virtqueue *vq = &dev->vqs[idx];
+	struct vduse_virtqueue *vq = dev->vqs[idx];
 
 	vq->num = num;
 }
@@ -533,7 +546,7 @@ static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
 					u16 idx, bool ready)
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
-	struct vduse_virtqueue *vq = &dev->vqs[idx];
+	struct vduse_virtqueue *vq = dev->vqs[idx];
 
 	vq->ready = ready;
 }
@@ -541,7 +554,7 @@ static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
-	struct vduse_virtqueue *vq = &dev->vqs[idx];
+	struct vduse_virtqueue *vq = dev->vqs[idx];
 
 	return vq->ready;
 }
@@ -550,7 +563,7 @@ static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,
 				const struct vdpa_vq_state *state)
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
-	struct vduse_virtqueue *vq = &dev->vqs[idx];
+	struct vduse_virtqueue *vq = dev->vqs[idx];
 
 	if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
 		vq->state.packed.last_avail_counter =
@@ -569,7 +582,7 @@ static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,
 				struct vdpa_vq_state *state)
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
-	struct vduse_virtqueue *vq = &dev->vqs[idx];
+	struct vduse_virtqueue *vq = dev->vqs[idx];
 
 	if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
 		return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);
@@ -624,8 +637,8 @@ static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
 	int i;
 
 	for (i = 0; i < dev->vq_num; i++)
-		if (num_max < dev->vqs[i].num_max)
-			num_max = dev->vqs[i].num_max;
+		if (num_max < dev->vqs[i]->num_max)
+			num_max = dev->vqs[i]->num_max;
 
 	return num_max;
 }
@@ -708,6 +721,23 @@ static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)
 	return dev->generation;
 }
 
+static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx,
+				      const struct cpumask *cpu_mask)
+{
+	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
+
+	cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask);
+	return 0;
+}
+
+static const struct cpumask *
+vduse_vdpa_get_vq_affinity(struct vdpa_device *vdpa, u16 idx)
+{
+	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
+
+	return &dev->vqs[idx]->irq_affinity;
+}
+
 static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
 				unsigned int asid,
 				struct vhost_iotlb *iotlb)
@@ -758,6 +788,8 @@ static const struct vdpa_config_ops vduse_vdpa_config_ops = {
 	.get_config		= vduse_vdpa_get_config,
 	.set_config		= vduse_vdpa_set_config,
 	.get_generation		= vduse_vdpa_get_generation,
+	.set_vq_affinity	= vduse_vdpa_set_vq_affinity,
+	.get_vq_affinity	= vduse_vdpa_get_vq_affinity,
 	.reset			= vduse_vdpa_reset,
 	.set_map		= vduse_vdpa_set_map,
 	.free			= vduse_vdpa_free,
@@ -863,7 +895,7 @@ static int vduse_kickfd_setup(struct vduse_dev *dev,
 		return -EINVAL;
 
 	index = array_index_nospec(eventfd->index, dev->vq_num);
-	vq = &dev->vqs[index];
+	vq = dev->vqs[index];
 	if (eventfd->fd >= 0) {
 		ctx = eventfd_ctx_fdget(eventfd->fd);
 		if (IS_ERR(ctx))
@@ -889,7 +921,7 @@ static bool vduse_dev_is_ready(struct vduse_dev *dev)
 	int i;
 
 	for (i = 0; i < dev->vq_num; i++)
-		if (!dev->vqs[i].num_max)
+		if (!dev->vqs[i]->num_max)
 			return false;
 
 	return true;
@@ -916,8 +948,26 @@ static void vduse_vq_irq_inject(struct work_struct *work)
 	spin_unlock_irq(&vq->irq_lock);
 }
 
+static bool vduse_vq_signal_irqfd(struct vduse_virtqueue *vq)
+{
+	bool signal = false;
+
+	if (!vq->cb.trigger)
+		return false;
+
+	spin_lock_irq(&vq->irq_lock);
+	if (vq->ready && vq->cb.trigger) {
+		eventfd_signal(vq->cb.trigger, 1);
+		signal = true;
+	}
+	spin_unlock_irq(&vq->irq_lock);
+
+	return signal;
+}
+
 static int vduse_dev_queue_irq_work(struct vduse_dev *dev,
-				    struct work_struct *irq_work)
+				    struct work_struct *irq_work,
+				    int irq_effective_cpu)
 {
 	int ret = -EINVAL;
 
@@ -926,7 +976,11 @@ static int vduse_dev_queue_irq_work(struct vduse_dev *dev,
 		goto unlock;
 
 	ret = 0;
-	queue_work(vduse_irq_wq, irq_work);
+	if (irq_effective_cpu == IRQ_UNBOUND)
+		queue_work(vduse_irq_wq, irq_work);
+	else
+		queue_work_on(irq_effective_cpu,
+			      vduse_irq_bound_wq, irq_work);
 unlock:
 	up_read(&dev->rwsem);
 
@@ -944,6 +998,9 @@ static int vduse_dev_dereg_umem(struct vduse_dev *dev,
 		goto unlock;
 
 	ret = -EINVAL;
+	if (!dev->domain)
+		goto unlock;
+
 	if (dev->umem->iova != iova || size != dev->domain->bounce_size)
 		goto unlock;
 
@@ -970,7 +1027,7 @@ static int vduse_dev_reg_umem(struct vduse_dev *dev,
 	unsigned long npages, lock_limit;
 	int ret;
 
-	if (!dev->domain->bounce_map ||
+	if (!dev->domain || !dev->domain->bounce_map ||
 	    size != dev->domain->bounce_size ||
 	    iova != 0 || uaddr & ~PAGE_MASK)
 		return -EINVAL;
@@ -1029,6 +1086,22 @@ unlock:
 	return ret;
 }
 
+static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq)
+{
+	int curr_cpu = vq->irq_effective_cpu;
+
+	while (true) {
+		curr_cpu = cpumask_next(curr_cpu, &vq->irq_affinity);
+		if (cpu_online(curr_cpu))
+			break;
+
+		if (curr_cpu >= nr_cpu_ids)
+			curr_cpu = IRQ_UNBOUND;
+	}
+
+	vq->irq_effective_cpu = curr_cpu;
+}
+
 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 			    unsigned long arg)
 {
@@ -1044,7 +1117,6 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 		struct vduse_iotlb_entry entry;
 		struct vhost_iotlb_map *map;
 		struct vdpa_map_file *map_file;
-		struct vduse_iova_domain *domain = dev->domain;
 		struct file *f = NULL;
 
 		ret = -EFAULT;
@@ -1055,8 +1127,13 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 		if (entry.start > entry.last)
 			break;
 
-		spin_lock(&domain->iotlb_lock);
-		map = vhost_iotlb_itree_first(domain->iotlb,
+		mutex_lock(&dev->domain_lock);
+		if (!dev->domain) {
+			mutex_unlock(&dev->domain_lock);
+			break;
+		}
+		spin_lock(&dev->domain->iotlb_lock);
+		map = vhost_iotlb_itree_first(dev->domain->iotlb,
 					      entry.start, entry.last);
 		if (map) {
 			map_file = (struct vdpa_map_file *)map->opaque;
@@ -1066,7 +1143,8 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 			entry.last = map->last;
 			entry.perm = map->perm;
 		}
-		spin_unlock(&domain->iotlb_lock);
+		spin_unlock(&dev->domain->iotlb_lock);
+		mutex_unlock(&dev->domain_lock);
 		ret = -EINVAL;
 		if (!f)
 			break;
@@ -1111,7 +1189,7 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 		break;
 	}
 	case VDUSE_DEV_INJECT_CONFIG_IRQ:
-		ret = vduse_dev_queue_irq_work(dev, &dev->inject);
+		ret = vduse_dev_queue_irq_work(dev, &dev->inject, IRQ_UNBOUND);
 		break;
 	case VDUSE_VQ_SETUP: {
 		struct vduse_vq_config config;
@@ -1130,7 +1208,7 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 			break;
 
 		index = array_index_nospec(config.index, dev->vq_num);
-		dev->vqs[index].num_max = config.max_size;
+		dev->vqs[index]->num_max = config.max_size;
 		ret = 0;
 		break;
 	}
@@ -1148,7 +1226,7 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 			break;
 
 		index = array_index_nospec(vq_info.index, dev->vq_num);
-		vq = &dev->vqs[index];
+		vq = dev->vqs[index];
 		vq_info.desc_addr = vq->desc_addr;
 		vq_info.driver_addr = vq->driver_addr;
 		vq_info.device_addr = vq->device_addr;
@@ -1197,8 +1275,14 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 		if (index >= dev->vq_num)
 			break;
 
+		ret = 0;
 		index = array_index_nospec(index, dev->vq_num);
-		ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject);
+		if (!vduse_vq_signal_irqfd(dev->vqs[index])) {
+			vduse_vq_update_effective_cpu(dev->vqs[index]);
+			ret = vduse_dev_queue_irq_work(dev,
+						&dev->vqs[index]->inject,
+						dev->vqs[index]->irq_effective_cpu);
+		}
 		break;
 	}
 	case VDUSE_IOTLB_REG_UMEM: {
@@ -1213,8 +1297,10 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 				 sizeof(umem.reserved)))
 			break;
 
+		mutex_lock(&dev->domain_lock);
 		ret = vduse_dev_reg_umem(dev, umem.iova,
 					 umem.uaddr, umem.size);
+		mutex_unlock(&dev->domain_lock);
 		break;
 	}
 	case VDUSE_IOTLB_DEREG_UMEM: {
@@ -1228,15 +1314,15 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 		if (!is_mem_zero((const char *)umem.reserved,
 				 sizeof(umem.reserved)))
 			break;
-
+		mutex_lock(&dev->domain_lock);
 		ret = vduse_dev_dereg_umem(dev, umem.iova,
 					   umem.size);
+		mutex_unlock(&dev->domain_lock);
 		break;
 	}
 	case VDUSE_IOTLB_GET_INFO: {
 		struct vduse_iova_info info;
 		struct vhost_iotlb_map *map;
-		struct vduse_iova_domain *domain = dev->domain;
 
 		ret = -EFAULT;
 		if (copy_from_user(&info, argp, sizeof(info)))
@@ -1250,18 +1336,24 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 				 sizeof(info.reserved)))
 			break;
 
-		spin_lock(&domain->iotlb_lock);
-		map = vhost_iotlb_itree_first(domain->iotlb,
+		mutex_lock(&dev->domain_lock);
+		if (!dev->domain) {
+			mutex_unlock(&dev->domain_lock);
+			break;
+		}
+		spin_lock(&dev->domain->iotlb_lock);
+		map = vhost_iotlb_itree_first(dev->domain->iotlb,
 					      info.start, info.last);
 		if (map) {
 			info.start = map->start;
 			info.last = map->last;
 			info.capability = 0;
-			if (domain->bounce_map && map->start == 0 &&
-			    map->last == domain->bounce_size - 1)
+			if (dev->domain->bounce_map && map->start == 0 &&
+			    map->last == dev->domain->bounce_size - 1)
 				info.capability |= VDUSE_IOVA_CAP_UMEM;
 		}
-		spin_unlock(&domain->iotlb_lock);
+		spin_unlock(&dev->domain->iotlb_lock);
+		mutex_unlock(&dev->domain_lock);
 		if (!map)
 			break;
 
@@ -1284,7 +1376,10 @@ static int vduse_dev_release(struct inode *inode, struct file *file)
 {
 	struct vduse_dev *dev = file->private_data;
 
-	vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size);
+	mutex_lock(&dev->domain_lock);
+	if (dev->domain)
+		vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size);
+	mutex_unlock(&dev->domain_lock);
 	spin_lock(&dev->msg_lock);
 	/* Make sure the inflight messages can processed after reconncection */
 	list_splice_init(&dev->recv_list, &dev->send_list);
@@ -1339,6 +1434,151 @@ static const struct file_operations vduse_dev_fops = {
 	.llseek		= noop_llseek,
 };
 
+static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)
+{
+	return sprintf(buf, "%*pb\n", cpumask_pr_args(&vq->irq_affinity));
+}
+
+static ssize_t irq_cb_affinity_store(struct vduse_virtqueue *vq,
+				     const char *buf, size_t count)
+{
+	cpumask_var_t new_value;
+	int ret;
+
+	if (!zalloc_cpumask_var(&new_value, GFP_KERNEL))
+		return -ENOMEM;
+
+	ret = cpumask_parse(buf, new_value);
+	if (ret)
+		goto free_mask;
+
+	ret = -EINVAL;
+	if (!cpumask_intersects(new_value, cpu_online_mask))
+		goto free_mask;
+
+	cpumask_copy(&vq->irq_affinity, new_value);
+	ret = count;
+free_mask:
+	free_cpumask_var(new_value);
+	return ret;
+}
+
+struct vq_sysfs_entry {
+	struct attribute attr;
+	ssize_t (*show)(struct vduse_virtqueue *vq, char *buf);
+	ssize_t (*store)(struct vduse_virtqueue *vq, const char *buf,
+			 size_t count);
+};
+
+static struct vq_sysfs_entry irq_cb_affinity_attr = __ATTR_RW(irq_cb_affinity);
+
+static struct attribute *vq_attrs[] = {
+	&irq_cb_affinity_attr.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(vq);
+
+static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr,
+			    char *buf)
+{
+	struct vduse_virtqueue *vq = container_of(kobj,
+					struct vduse_virtqueue, kobj);
+	struct vq_sysfs_entry *entry = container_of(attr,
+					struct vq_sysfs_entry, attr);
+
+	if (!entry->show)
+		return -EIO;
+
+	return entry->show(vq, buf);
+}
+
+static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr,
+			     const char *buf, size_t count)
+{
+	struct vduse_virtqueue *vq = container_of(kobj,
+					struct vduse_virtqueue, kobj);
+	struct vq_sysfs_entry *entry = container_of(attr,
+					struct vq_sysfs_entry, attr);
+
+	if (!entry->store)
+		return -EIO;
+
+	return entry->store(vq, buf, count);
+}
+
+static const struct sysfs_ops vq_sysfs_ops = {
+	.show = vq_attr_show,
+	.store = vq_attr_store,
+};
+
+static void vq_release(struct kobject *kobj)
+{
+	struct vduse_virtqueue *vq = container_of(kobj,
+					struct vduse_virtqueue, kobj);
+	kfree(vq);
+}
+
+static const struct kobj_type vq_type = {
+	.release	= vq_release,
+	.sysfs_ops	= &vq_sysfs_ops,
+	.default_groups	= vq_groups,
+};
+
+static void vduse_dev_deinit_vqs(struct vduse_dev *dev)
+{
+	int i;
+
+	if (!dev->vqs)
+		return;
+
+	for (i = 0; i < dev->vq_num; i++)
+		kobject_put(&dev->vqs[i]->kobj);
+	kfree(dev->vqs);
+}
+
+static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num)
+{
+	int ret, i;
+
+	dev->vq_align = vq_align;
+	dev->vq_num = vq_num;
+	dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
+	if (!dev->vqs)
+		return -ENOMEM;
+
+	for (i = 0; i < vq_num; i++) {
+		dev->vqs[i] = kzalloc(sizeof(*dev->vqs[i]), GFP_KERNEL);
+		if (!dev->vqs[i]) {
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		dev->vqs[i]->index = i;
+		dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND;
+		INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject);
+		INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work);
+		spin_lock_init(&dev->vqs[i]->kick_lock);
+		spin_lock_init(&dev->vqs[i]->irq_lock);
+		cpumask_setall(&dev->vqs[i]->irq_affinity);
+
+		kobject_init(&dev->vqs[i]->kobj, &vq_type);
+		ret = kobject_add(&dev->vqs[i]->kobj,
+				  &dev->dev->kobj, "vq%d", i);
+		if (ret) {
+			kfree(dev->vqs[i]);
+			goto err;
+		}
+	}
+
+	return 0;
+err:
+	while (i--)
+		kobject_put(&dev->vqs[i]->kobj);
+	kfree(dev->vqs);
+	dev->vqs = NULL;
+	return ret;
+}
+
 static struct vduse_dev *vduse_dev_create(void)
 {
 	struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);
@@ -1348,6 +1588,7 @@ static struct vduse_dev *vduse_dev_create(void)
 
 	mutex_init(&dev->lock);
 	mutex_init(&dev->mem_lock);
+	mutex_init(&dev->domain_lock);
 	spin_lock_init(&dev->msg_lock);
 	INIT_LIST_HEAD(&dev->send_list);
 	INIT_LIST_HEAD(&dev->recv_list);
@@ -1396,8 +1637,9 @@ static int vduse_destroy_dev(char *name)
 	device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
 	idr_remove(&vduse_idr, dev->minor);
 	kvfree(dev->config);
-	kfree(dev->vqs);
-	vduse_domain_destroy(dev->domain);
+	vduse_dev_deinit_vqs(dev);
+	if (dev->domain)
+		vduse_domain_destroy(dev->domain);
 	kfree(dev->name);
 	vduse_dev_destroy(dev);
 	module_put(THIS_MODULE);
@@ -1476,8 +1718,48 @@ static ssize_t msg_timeout_store(struct device *device,
 
 static DEVICE_ATTR_RW(msg_timeout);
 
+static ssize_t bounce_size_show(struct device *device,
+				struct device_attribute *attr, char *buf)
+{
+	struct vduse_dev *dev = dev_get_drvdata(device);
+
+	return sysfs_emit(buf, "%u\n", dev->bounce_size);
+}
+
+static ssize_t bounce_size_store(struct device *device,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	struct vduse_dev *dev = dev_get_drvdata(device);
+	unsigned int bounce_size;
+	int ret;
+
+	ret = -EPERM;
+	mutex_lock(&dev->domain_lock);
+	if (dev->domain)
+		goto unlock;
+
+	ret = kstrtouint(buf, 10, &bounce_size);
+	if (ret < 0)
+		goto unlock;
+
+	ret = -EINVAL;
+	if (bounce_size > VDUSE_MAX_BOUNCE_SIZE ||
+	    bounce_size < VDUSE_MIN_BOUNCE_SIZE)
+		goto unlock;
+
+	dev->bounce_size = bounce_size & PAGE_MASK;
+	ret = count;
+unlock:
+	mutex_unlock(&dev->domain_lock);
+	return ret;
+}
+
+static DEVICE_ATTR_RW(bounce_size);
+
 static struct attribute *vduse_dev_attrs[] = {
 	&dev_attr_msg_timeout.attr,
+	&dev_attr_bounce_size.attr,
 	NULL
 };
 
@@ -1486,7 +1768,7 @@ ATTRIBUTE_GROUPS(vduse_dev);
 static int vduse_create_dev(struct vduse_dev_config *config,
 			    void *config_buf, u64 api_version)
 {
-	int i, ret;
+	int ret;
 	struct vduse_dev *dev;
 
 	ret = -EEXIST;
@@ -1506,26 +1788,9 @@ static int vduse_create_dev(struct vduse_dev_config *config,
 	if (!dev->name)
 		goto err_str;
 
-	dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
-					  VDUSE_BOUNCE_SIZE);
-	if (!dev->domain)
-		goto err_domain;
-
+	dev->bounce_size = VDUSE_BOUNCE_SIZE;
 	dev->config = config_buf;
 	dev->config_size = config->config_size;
-	dev->vq_align = config->vq_align;
-	dev->vq_num = config->vq_num;
-	dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
-	if (!dev->vqs)
-		goto err_vqs;
-
-	for (i = 0; i < dev->vq_num; i++) {
-		dev->vqs[i].index = i;
-		INIT_WORK(&dev->vqs[i].inject, vduse_vq_irq_inject);
-		INIT_WORK(&dev->vqs[i].kick, vduse_vq_kick_work);
-		spin_lock_init(&dev->vqs[i].kick_lock);
-		spin_lock_init(&dev->vqs[i].irq_lock);
-	}
 
 	ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
 	if (ret < 0)
@@ -1540,16 +1805,19 @@ static int vduse_create_dev(struct vduse_dev_config *config,
 		ret = PTR_ERR(dev->dev);
 		goto err_dev;
 	}
+
+	ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num);
+	if (ret)
+		goto err_vqs;
+
 	__module_get(THIS_MODULE);
 
 	return 0;
+err_vqs:
+	device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
 err_dev:
 	idr_remove(&vduse_idr, dev->minor);
 err_idr:
-	kfree(dev->vqs);
-err_vqs:
-	vduse_domain_destroy(dev->domain);
-err_domain:
 	kfree(dev->name);
 err_str:
 	vduse_dev_destroy(dev);
@@ -1716,9 +1984,23 @@ static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
 	if (ret)
 		return ret;
 
+	mutex_lock(&dev->domain_lock);
+	if (!dev->domain)
+		dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
+						  dev->bounce_size);
+	mutex_unlock(&dev->domain_lock);
+	if (!dev->domain) {
+		put_device(&dev->vdev->vdpa.dev);
+		return -ENOMEM;
+	}
+
 	ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);
 	if (ret) {
 		put_device(&dev->vdev->vdpa.dev);
+		mutex_lock(&dev->domain_lock);
+		vduse_domain_destroy(dev->domain);
+		dev->domain = NULL;
+		mutex_unlock(&dev->domain_lock);
 		return ret;
 	}
 
@@ -1793,7 +2075,7 @@ static int vduse_init(void)
 	int ret;
 	struct device *dev;
 
-	vduse_class = class_create(THIS_MODULE, "vduse");
+	vduse_class = class_create("vduse");
 	if (IS_ERR(vduse_class))
 		return PTR_ERR(vduse_class);
 
@@ -1824,12 +2106,15 @@ static int vduse_init(void)
 	if (ret)
 		goto err_cdev;
 
+	ret = -ENOMEM;
 	vduse_irq_wq = alloc_workqueue("vduse-irq",
 				WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);
-	if (!vduse_irq_wq) {
-		ret = -ENOMEM;
+	if (!vduse_irq_wq)
 		goto err_wq;
-	}
+
+	vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", WQ_HIGHPRI, 0);
+	if (!vduse_irq_bound_wq)
+		goto err_bound_wq;
 
 	ret = vduse_domain_init();
 	if (ret)
@@ -1843,6 +2128,8 @@ static int vduse_init(void)
 err_mgmtdev:
 	vduse_domain_exit();
 err_domain:
+	destroy_workqueue(vduse_irq_bound_wq);
+err_bound_wq:
 	destroy_workqueue(vduse_irq_wq);
 err_wq:
 	cdev_del(&vduse_cdev);
@@ -1862,6 +2149,7 @@ static void vduse_exit(void)
 {
 	vduse_mgmtdev_exit();
 	vduse_domain_exit();
+	destroy_workqueue(vduse_irq_bound_wq);
 	destroy_workqueue(vduse_irq_wq);
 	cdev_del(&vduse_cdev);
 	device_destroy(vduse_class, vduse_major);
diff --git a/drivers/vfio/group.c b/drivers/vfio/group.c
index 27d5ba7cf9dc..fc75c1000d74 100644
--- a/drivers/vfio/group.c
+++ b/drivers/vfio/group.c
@@ -878,7 +878,7 @@ int __init vfio_group_init(void)
 		return ret;
 
 	/* /dev/vfio/$GROUP */
-	vfio.class = class_create(THIS_MODULE, "vfio");
+	vfio.class = class_create("vfio");
 	if (IS_ERR(vfio.class)) {
 		ret = PTR_ERR(vfio.class);
 		goto err_group_class;
diff --git a/drivers/vfio/iommufd.c b/drivers/vfio/iommufd.c
index db4efbd56042..88b00c501015 100644
--- a/drivers/vfio/iommufd.c
+++ b/drivers/vfio/iommufd.c
@@ -32,13 +32,6 @@ int vfio_iommufd_bind(struct vfio_device *vdev, struct iommufd_ctx *ictx)
 		return 0;
 	}
 
-	/*
-	 * If the driver doesn't provide this op then it means the device does
-	 * not do DMA at all. So nothing to do.
-	 */
-	if (!vdev->ops->bind_iommufd)
-		return 0;
-
 	ret = vdev->ops->bind_iommufd(vdev, ictx, &device_id);
 	if (ret)
 		return ret;
@@ -119,7 +112,8 @@ EXPORT_SYMBOL_GPL(vfio_iommufd_physical_attach_ioas);
 /*
  * The emulated standard ops mean that vfio_device is going to use the
  * "mdev path" and will call vfio_pin_pages()/vfio_dma_rw(). Drivers using this
- * ops set should call vfio_register_emulated_iommu_dev().
+ * ops set should call vfio_register_emulated_iommu_dev(). Drivers that do
+ * not call vfio_pin_pages()/vfio_dma_rw() have no need to provide dma_unmap.
  */
 
 static void vfio_emulated_unmap(void *data, unsigned long iova,
@@ -127,7 +121,8 @@ static void vfio_emulated_unmap(void *data, unsigned long iova,
 {
 	struct vfio_device *vdev = data;
 
-	vdev->ops->dma_unmap(vdev, iova, length);
+	if (vdev->ops->dma_unmap)
+		vdev->ops->dma_unmap(vdev, iova, length);
 }
 
 static const struct iommufd_access_ops vfio_user_ops = {
@@ -138,10 +133,14 @@ static const struct iommufd_access_ops vfio_user_ops = {
 int vfio_iommufd_emulated_bind(struct vfio_device *vdev,
 			       struct iommufd_ctx *ictx, u32 *out_device_id)
 {
+	struct iommufd_access *user;
+
 	lockdep_assert_held(&vdev->dev_set->lock);
 
-	vdev->iommufd_ictx = ictx;
-	iommufd_ctx_get(ictx);
+	user = iommufd_access_create(ictx, &vfio_user_ops, vdev, out_device_id);
+	if (IS_ERR(user))
+		return PTR_ERR(user);
+	vdev->iommufd_access = user;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(vfio_iommufd_emulated_bind);
@@ -152,24 +151,24 @@ void vfio_iommufd_emulated_unbind(struct vfio_device *vdev)
 
 	if (vdev->iommufd_access) {
 		iommufd_access_destroy(vdev->iommufd_access);
+		vdev->iommufd_attached = false;
 		vdev->iommufd_access = NULL;
 	}
-	iommufd_ctx_put(vdev->iommufd_ictx);
-	vdev->iommufd_ictx = NULL;
 }
 EXPORT_SYMBOL_GPL(vfio_iommufd_emulated_unbind);
 
 int vfio_iommufd_emulated_attach_ioas(struct vfio_device *vdev, u32 *pt_id)
 {
-	struct iommufd_access *user;
+	int rc;
 
 	lockdep_assert_held(&vdev->dev_set->lock);
 
-	user = iommufd_access_create(vdev->iommufd_ictx, *pt_id, &vfio_user_ops,
-				     vdev);
-	if (IS_ERR(user))
-		return PTR_ERR(user);
-	vdev->iommufd_access = user;
+	if (vdev->iommufd_attached)
+		return -EBUSY;
+	rc = iommufd_access_attach(vdev->iommufd_access, *pt_id);
+	if (rc)
+		return rc;
+	vdev->iommufd_attached = true;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(vfio_iommufd_emulated_attach_ioas);
diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c
index 43bd6b76e2b6..f0ca33b2e1df 100644
--- a/drivers/vfio/vfio_main.c
+++ b/drivers/vfio/vfio_main.c
@@ -255,8 +255,9 @@ static int __vfio_register_dev(struct vfio_device *device,
 {
 	int ret;
 
-	if (WARN_ON(device->ops->bind_iommufd &&
-		    (!device->ops->unbind_iommufd ||
+	if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) &&
+		    (!device->ops->bind_iommufd ||
+		     !device->ops->unbind_iommufd ||
 		     !device->ops->attach_ioas)))
 		return -EINVAL;
 
@@ -1408,7 +1409,7 @@ static int __init vfio_init(void)
 		goto err_virqfd;
 
 	/* /sys/class/vfio-dev/vfioX */
-	vfio.device_class = class_create(THIS_MODULE, "vfio-dev");
+	vfio.device_class = class_create("vfio-dev");
 	if (IS_ERR(vfio.device_class)) {
 		ret = PTR_ERR(vfio.device_class);
 		goto err_dev_class;
diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig
index 587fbae06182..b455d9ab6f3d 100644
--- a/drivers/vhost/Kconfig
+++ b/drivers/vhost/Kconfig
@@ -13,9 +13,14 @@ config VHOST_RING
 	  This option is selected by any driver which needs to access
 	  the host side of a virtio ring.
 
+config VHOST_TASK
+	bool
+	default n
+
 config VHOST
 	tristate
 	select VHOST_IOTLB
+	select VHOST_TASK
 	help
 	  This option is selected by any driver which needs to access
 	  the core of vhost.
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index 32d0be968103..bb10fa4bb4f6 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -229,7 +229,10 @@ struct vhost_scsi_ctx {
 	struct iov_iter out_iter;
 };
 
-/* Global spinlock to protect vhost_scsi TPG list for vhost IOCTL access */
+/*
+ * Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO
+ * configfs management operations.
+ */
 static DEFINE_MUTEX(vhost_scsi_mutex);
 static LIST_HEAD(vhost_scsi_list);
 
@@ -291,11 +294,6 @@ static int vhost_scsi_check_true(struct se_portal_group *se_tpg)
 	return 1;
 }
 
-static int vhost_scsi_check_false(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
 static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg)
 {
 	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
@@ -320,11 +318,6 @@ static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg)
 	return tpg->tv_fabric_prot_type;
 }
 
-static u32 vhost_scsi_tpg_get_inst_index(struct se_portal_group *se_tpg)
-{
-	return 1;
-}
-
 static void vhost_scsi_release_cmd_res(struct se_cmd *se_cmd)
 {
 	struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd,
@@ -372,11 +365,6 @@ static void vhost_scsi_release_cmd(struct se_cmd *se_cmd)
 	}
 }
 
-static u32 vhost_scsi_sess_get_index(struct se_session *se_sess)
-{
-	return 0;
-}
-
 static int vhost_scsi_write_pending(struct se_cmd *se_cmd)
 {
 	/* Go ahead and process the write immediately */
@@ -384,16 +372,6 @@ static int vhost_scsi_write_pending(struct se_cmd *se_cmd)
 	return 0;
 }
 
-static void vhost_scsi_set_default_node_attrs(struct se_node_acl *nacl)
-{
-	return;
-}
-
-static int vhost_scsi_get_cmd_state(struct se_cmd *se_cmd)
-{
-	return 0;
-}
-
 static int vhost_scsi_queue_data_in(struct se_cmd *se_cmd)
 {
 	transport_generic_free_cmd(se_cmd, 0);
@@ -665,7 +643,7 @@ vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)
 {
 	int sgl_count = 0;
 
-	if (!iter || !iter->iov) {
+	if (!iter || !iter_iov(iter)) {
 		pr_err("%s: iter->iov is NULL, but expected bytes: %zu"
 		       " present\n", __func__, bytes);
 		return -EINVAL;
@@ -1526,7 +1504,7 @@ out:
  * vhost_scsi_tpg with an active struct vhost_scsi_nexus
  *
  *  The lock nesting rule is:
- *    vhost_scsi_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex
+ *    vs->dev.mutex -> vhost_scsi_mutex -> tpg->tv_tpg_mutex -> vq->mutex
  */
 static int
 vhost_scsi_set_endpoint(struct vhost_scsi *vs,
@@ -1540,7 +1518,6 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
 	int index, ret, i, len;
 	bool match = false;
 
-	mutex_lock(&vhost_scsi_mutex);
 	mutex_lock(&vs->dev.mutex);
 
 	/* Verify that ring has been setup correctly. */
@@ -1561,6 +1538,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
 	if (vs->vs_tpg)
 		memcpy(vs_tpg, vs->vs_tpg, len);
 
+	mutex_lock(&vhost_scsi_mutex);
 	list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {
 		mutex_lock(&tpg->tv_tpg_mutex);
 		if (!tpg->tpg_nexus) {
@@ -1576,6 +1554,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
 		if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
 			if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {
 				mutex_unlock(&tpg->tv_tpg_mutex);
+				mutex_unlock(&vhost_scsi_mutex);
 				ret = -EEXIST;
 				goto undepend;
 			}
@@ -1590,6 +1569,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
 			if (ret) {
 				pr_warn("target_depend_item() failed: %d\n", ret);
 				mutex_unlock(&tpg->tv_tpg_mutex);
+				mutex_unlock(&vhost_scsi_mutex);
 				goto undepend;
 			}
 			tpg->tv_tpg_vhost_count++;
@@ -1599,6 +1579,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
 		}
 		mutex_unlock(&tpg->tv_tpg_mutex);
 	}
+	mutex_unlock(&vhost_scsi_mutex);
 
 	if (match) {
 		memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
@@ -1654,7 +1635,6 @@ undepend:
 	kfree(vs_tpg);
 out:
 	mutex_unlock(&vs->dev.mutex);
-	mutex_unlock(&vhost_scsi_mutex);
 	return ret;
 }
 
@@ -1670,7 +1650,6 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
 	int index, ret, i;
 	u8 target;
 
-	mutex_lock(&vhost_scsi_mutex);
 	mutex_lock(&vs->dev.mutex);
 	/* Verify that ring has been setup correctly. */
 	for (index = 0; index < vs->dev.nvqs; ++index) {
@@ -1691,11 +1670,10 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
 		if (!tpg)
 			continue;
 
-		mutex_lock(&tpg->tv_tpg_mutex);
 		tv_tport = tpg->tport;
 		if (!tv_tport) {
 			ret = -ENODEV;
-			goto err_tpg;
+			goto err_dev;
 		}
 
 		if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
@@ -1704,35 +1682,51 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
 				tv_tport->tport_name, tpg->tport_tpgt,
 				t->vhost_wwpn, t->vhost_tpgt);
 			ret = -EINVAL;
-			goto err_tpg;
+			goto err_dev;
 		}
+		match = true;
+	}
+	if (!match)
+		goto free_vs_tpg;
+
+	/* Prevent new cmds from starting and accessing the tpgs/sessions */
+	for (i = 0; i < vs->dev.nvqs; i++) {
+		vq = &vs->vqs[i].vq;
+		mutex_lock(&vq->mutex);
+		vhost_vq_set_backend(vq, NULL);
+		mutex_unlock(&vq->mutex);
+	}
+	/* Make sure cmds are not running before tearing them down. */
+	vhost_scsi_flush(vs);
+
+	for (i = 0; i < vs->dev.nvqs; i++) {
+		vq = &vs->vqs[i].vq;
+		vhost_scsi_destroy_vq_cmds(vq);
+	}
+
+	/*
+	 * We can now release our hold on the tpg and sessions and userspace
+	 * can free them after this point.
+	 */
+	for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
+		target = i;
+		tpg = vs->vs_tpg[target];
+		if (!tpg)
+			continue;
+
+		mutex_lock(&tpg->tv_tpg_mutex);
+
 		tpg->tv_tpg_vhost_count--;
 		tpg->vhost_scsi = NULL;
 		vs->vs_tpg[target] = NULL;
-		match = true;
+
 		mutex_unlock(&tpg->tv_tpg_mutex);
-		/*
-		 * Release se_tpg->tpg_group.cg_item configfs dependency now
-		 * to allow vhost-scsi WWPN se_tpg->tpg_group shutdown to occur.
-		 */
+
 		se_tpg = &tpg->se_tpg;
 		target_undepend_item(&se_tpg->tpg_group.cg_item);
 	}
-	if (match) {
-		for (i = 0; i < vs->dev.nvqs; i++) {
-			vq = &vs->vqs[i].vq;
-			mutex_lock(&vq->mutex);
-			vhost_vq_set_backend(vq, NULL);
-			mutex_unlock(&vq->mutex);
-		}
-		/* Make sure cmds are not running before tearing them down. */
-		vhost_scsi_flush(vs);
 
-		for (i = 0; i < vs->dev.nvqs; i++) {
-			vq = &vs->vqs[i].vq;
-			vhost_scsi_destroy_vq_cmds(vq);
-		}
-	}
+free_vs_tpg:
 	/*
 	 * Act as synchronize_rcu to make sure access to
 	 * old vs->vs_tpg is finished.
@@ -1742,14 +1736,10 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
 	vs->vs_tpg = NULL;
 	WARN_ON(vs->vs_events_nr);
 	mutex_unlock(&vs->dev.mutex);
-	mutex_unlock(&vhost_scsi_mutex);
 	return 0;
 
-err_tpg:
-	mutex_unlock(&tpg->tv_tpg_mutex);
 err_dev:
 	mutex_unlock(&vs->dev.mutex);
-	mutex_unlock(&vhost_scsi_mutex);
 	return ret;
 }
 
@@ -1990,8 +1980,6 @@ vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
 	if (!vs)
 		return;
 
-	mutex_lock(&vs->dev.mutex);
-
 	if (plug)
 		reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
 	else
@@ -1999,11 +1987,18 @@ vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
 
 	vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 	mutex_lock(&vq->mutex);
+	/*
+	 * We can't queue events if the backend has been cleared, because
+	 * we could end up queueing an event after the flush.
+	 */
+	if (!vhost_vq_get_backend(vq))
+		goto unlock;
+
 	if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))
 		vhost_scsi_send_evt(vs, tpg, lun,
 				   VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
+unlock:
 	mutex_unlock(&vq->mutex);
-	mutex_unlock(&vs->dev.mutex);
 }
 
 static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
@@ -2022,15 +2017,10 @@ static int vhost_scsi_port_link(struct se_portal_group *se_tpg,
 	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 				struct vhost_scsi_tpg, se_tpg);
 
-	mutex_lock(&vhost_scsi_mutex);
-
 	mutex_lock(&tpg->tv_tpg_mutex);
 	tpg->tv_tpg_port_count++;
-	mutex_unlock(&tpg->tv_tpg_mutex);
-
 	vhost_scsi_hotplug(tpg, lun);
-
-	mutex_unlock(&vhost_scsi_mutex);
+	mutex_unlock(&tpg->tv_tpg_mutex);
 
 	return 0;
 }
@@ -2041,15 +2031,10 @@ static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,
 	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 				struct vhost_scsi_tpg, se_tpg);
 
-	mutex_lock(&vhost_scsi_mutex);
-
 	mutex_lock(&tpg->tv_tpg_mutex);
 	tpg->tv_tpg_port_count--;
-	mutex_unlock(&tpg->tv_tpg_mutex);
-
 	vhost_scsi_hotunplug(tpg, lun);
-
-	mutex_unlock(&vhost_scsi_mutex);
+	mutex_unlock(&tpg->tv_tpg_mutex);
 }
 
 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(
@@ -2435,17 +2420,11 @@ static const struct target_core_fabric_ops vhost_scsi_ops = {
 	.tpg_get_tag			= vhost_scsi_get_tpgt,
 	.tpg_check_demo_mode		= vhost_scsi_check_true,
 	.tpg_check_demo_mode_cache	= vhost_scsi_check_true,
-	.tpg_check_demo_mode_write_protect = vhost_scsi_check_false,
-	.tpg_check_prod_mode_write_protect = vhost_scsi_check_false,
 	.tpg_check_prot_fabric_only	= vhost_scsi_check_prot_fabric_only,
-	.tpg_get_inst_index		= vhost_scsi_tpg_get_inst_index,
 	.release_cmd			= vhost_scsi_release_cmd,
 	.check_stop_free		= vhost_scsi_check_stop_free,
-	.sess_get_index			= vhost_scsi_sess_get_index,
 	.sess_get_initiator_sid		= NULL,
 	.write_pending			= vhost_scsi_write_pending,
-	.set_default_node_attributes	= vhost_scsi_set_default_node_attrs,
-	.get_cmd_state			= vhost_scsi_get_cmd_state,
 	.queue_data_in			= vhost_scsi_queue_data_in,
 	.queue_status			= vhost_scsi_queue_status,
 	.queue_tm_rsp			= vhost_scsi_queue_tm_rsp,
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 7be9d9d8f01c..8c1aefc865f0 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -219,6 +219,28 @@ static int vhost_vdpa_reset(struct vhost_vdpa *v)
 	return vdpa_reset(vdpa);
 }
 
+static long vhost_vdpa_bind_mm(struct vhost_vdpa *v)
+{
+	struct vdpa_device *vdpa = v->vdpa;
+	const struct vdpa_config_ops *ops = vdpa->config;
+
+	if (!vdpa->use_va || !ops->bind_mm)
+		return 0;
+
+	return ops->bind_mm(vdpa, v->vdev.mm);
+}
+
+static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v)
+{
+	struct vdpa_device *vdpa = v->vdpa;
+	const struct vdpa_config_ops *ops = vdpa->config;
+
+	if (!vdpa->use_va || !ops->unbind_mm)
+		return;
+
+	ops->unbind_mm(vdpa);
+}
+
 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
 {
 	struct vdpa_device *vdpa = v->vdpa;
@@ -599,9 +621,11 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
 		if (vq->call_ctx.ctx) {
 			cb.callback = vhost_vdpa_virtqueue_cb;
 			cb.private = vq;
+			cb.trigger = vq->call_ctx.ctx;
 		} else {
 			cb.callback = NULL;
 			cb.private = NULL;
+			cb.trigger = NULL;
 		}
 		ops->set_vq_cb(vdpa, idx, &cb);
 		vhost_vdpa_setup_vq_irq(v, idx);
@@ -716,6 +740,17 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
 		break;
 	}
 
+	if (r)
+		goto out;
+
+	switch (cmd) {
+	case VHOST_SET_OWNER:
+		r = vhost_vdpa_bind_mm(v);
+		if (r)
+			vhost_dev_reset_owner(d, NULL);
+		break;
+	}
+out:
 	mutex_unlock(&d->mutex);
 	return r;
 }
@@ -851,11 +886,7 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v,
 		if (!v->in_batch)
 			ops->set_map(vdpa, asid, iotlb);
 	}
-	/* If we are in the middle of batch processing, delay the free
-	 * of AS until BATCH_END.
-	 */
-	if (!v->in_batch && !iotlb->nmaps)
-		vhost_vdpa_remove_as(v, asid);
+
 }
 
 static int vhost_vdpa_va_map(struct vhost_vdpa *v,
@@ -1112,8 +1143,6 @@ static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
 		if (v->in_batch && ops->set_map)
 			ops->set_map(vdpa, asid, iotlb);
 		v->in_batch = false;
-		if (!iotlb->nmaps)
-			vhost_vdpa_remove_as(v, asid);
 		break;
 	default:
 		r = -EINVAL;
@@ -1140,7 +1169,7 @@ static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
 	struct vdpa_device *vdpa = v->vdpa;
 	const struct vdpa_config_ops *ops = vdpa->config;
 	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
-	struct bus_type *bus;
+	const struct bus_type *bus;
 	int ret;
 
 	/* Device want to do DMA by itself */
@@ -1287,6 +1316,7 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep)
 	vhost_vdpa_clean_irq(v);
 	vhost_vdpa_reset(v);
 	vhost_dev_stop(&v->vdev);
+	vhost_vdpa_unbind_mm(v);
 	vhost_vdpa_config_put(v);
 	vhost_vdpa_cleanup(v);
 	mutex_unlock(&d->mutex);
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index f11bdbe4c2c5..10bf35a3db6e 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -22,11 +22,11 @@
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 #include <linux/kthread.h>
-#include <linux/cgroup.h>
 #include <linux/module.h>
 #include <linux/sort.h>
 #include <linux/sched/mm.h>
 #include <linux/sched/signal.h>
+#include <linux/sched/vhost_task.h>
 #include <linux/interval_tree_generic.h>
 #include <linux/nospec.h>
 #include <linux/kcov.h>
@@ -255,8 +255,8 @@ void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
 		 * sure it was not in the list.
 		 * test_and_set_bit() implies a memory barrier.
 		 */
-		llist_add(&work->node, &dev->work_list);
-		wake_up_process(dev->worker);
+		llist_add(&work->node, &dev->worker->work_list);
+		wake_up_process(dev->worker->vtsk->task);
 	}
 }
 EXPORT_SYMBOL_GPL(vhost_work_queue);
@@ -264,7 +264,7 @@ EXPORT_SYMBOL_GPL(vhost_work_queue);
 /* A lockless hint for busy polling code to exit the loop */
 bool vhost_has_work(struct vhost_dev *dev)
 {
-	return !llist_empty(&dev->work_list);
+	return dev->worker && !llist_empty(&dev->worker->work_list);
 }
 EXPORT_SYMBOL_GPL(vhost_has_work);
 
@@ -335,22 +335,20 @@ static void vhost_vq_reset(struct vhost_dev *dev,
 
 static int vhost_worker(void *data)
 {
-	struct vhost_dev *dev = data;
+	struct vhost_worker *worker = data;
 	struct vhost_work *work, *work_next;
 	struct llist_node *node;
 
-	kthread_use_mm(dev->mm);
-
 	for (;;) {
 		/* mb paired w/ kthread_stop */
 		set_current_state(TASK_INTERRUPTIBLE);
 
-		if (kthread_should_stop()) {
+		if (vhost_task_should_stop(worker->vtsk)) {
 			__set_current_state(TASK_RUNNING);
 			break;
 		}
 
-		node = llist_del_all(&dev->work_list);
+		node = llist_del_all(&worker->work_list);
 		if (!node)
 			schedule();
 
@@ -360,14 +358,14 @@ static int vhost_worker(void *data)
 		llist_for_each_entry_safe(work, work_next, node, node) {
 			clear_bit(VHOST_WORK_QUEUED, &work->flags);
 			__set_current_state(TASK_RUNNING);
-			kcov_remote_start_common(dev->kcov_handle);
+			kcov_remote_start_common(worker->kcov_handle);
 			work->fn(work);
 			kcov_remote_stop();
 			if (need_resched())
 				schedule();
 		}
 	}
-	kthread_unuse_mm(dev->mm);
+
 	return 0;
 }
 
@@ -436,8 +434,7 @@ static size_t vhost_get_avail_size(struct vhost_virtqueue *vq,
 	size_t event __maybe_unused =
 	       vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
 
-	return sizeof(*vq->avail) +
-	       sizeof(*vq->avail->ring) * num + event;
+	return size_add(struct_size(vq->avail, ring, num), event);
 }
 
 static size_t vhost_get_used_size(struct vhost_virtqueue *vq,
@@ -446,8 +443,7 @@ static size_t vhost_get_used_size(struct vhost_virtqueue *vq,
 	size_t event __maybe_unused =
 	       vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
 
-	return sizeof(*vq->used) +
-	       sizeof(*vq->used->ring) * num + event;
+	return size_add(struct_size(vq->used, ring, num), event);
 }
 
 static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,
@@ -479,7 +475,6 @@ void vhost_dev_init(struct vhost_dev *dev,
 	dev->byte_weight = byte_weight;
 	dev->use_worker = use_worker;
 	dev->msg_handler = msg_handler;
-	init_llist_head(&dev->work_list);
 	init_waitqueue_head(&dev->wait);
 	INIT_LIST_HEAD(&dev->read_list);
 	INIT_LIST_HEAD(&dev->pending_list);
@@ -509,31 +504,6 @@ long vhost_dev_check_owner(struct vhost_dev *dev)
 }
 EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
 
-struct vhost_attach_cgroups_struct {
-	struct vhost_work work;
-	struct task_struct *owner;
-	int ret;
-};
-
-static void vhost_attach_cgroups_work(struct vhost_work *work)
-{
-	struct vhost_attach_cgroups_struct *s;
-
-	s = container_of(work, struct vhost_attach_cgroups_struct, work);
-	s->ret = cgroup_attach_task_all(s->owner, current);
-}
-
-static int vhost_attach_cgroups(struct vhost_dev *dev)
-{
-	struct vhost_attach_cgroups_struct attach;
-
-	attach.owner = current;
-	vhost_work_init(&attach.work, vhost_attach_cgroups_work);
-	vhost_work_queue(dev, &attach.work);
-	vhost_dev_flush(dev);
-	return attach.ret;
-}
-
 /* Caller should have device mutex */
 bool vhost_dev_has_owner(struct vhost_dev *dev)
 {
@@ -571,10 +541,54 @@ static void vhost_detach_mm(struct vhost_dev *dev)
 	dev->mm = NULL;
 }
 
+static void vhost_worker_free(struct vhost_dev *dev)
+{
+	struct vhost_worker *worker = dev->worker;
+
+	if (!worker)
+		return;
+
+	dev->worker = NULL;
+	WARN_ON(!llist_empty(&worker->work_list));
+	vhost_task_stop(worker->vtsk);
+	kfree(worker);
+}
+
+static int vhost_worker_create(struct vhost_dev *dev)
+{
+	struct vhost_worker *worker;
+	struct vhost_task *vtsk;
+	char name[TASK_COMM_LEN];
+	int ret;
+
+	worker = kzalloc(sizeof(*worker), GFP_KERNEL_ACCOUNT);
+	if (!worker)
+		return -ENOMEM;
+
+	dev->worker = worker;
+	worker->kcov_handle = kcov_common_handle();
+	init_llist_head(&worker->work_list);
+	snprintf(name, sizeof(name), "vhost-%d", current->pid);
+
+	vtsk = vhost_task_create(vhost_worker, worker, name);
+	if (!vtsk) {
+		ret = -ENOMEM;
+		goto free_worker;
+	}
+
+	worker->vtsk = vtsk;
+	vhost_task_start(vtsk);
+	return 0;
+
+free_worker:
+	kfree(worker);
+	dev->worker = NULL;
+	return ret;
+}
+
 /* Caller should have device mutex */
 long vhost_dev_set_owner(struct vhost_dev *dev)
 {
-	struct task_struct *worker;
 	int err;
 
 	/* Is there an owner already? */
@@ -585,36 +599,21 @@ long vhost_dev_set_owner(struct vhost_dev *dev)
 
 	vhost_attach_mm(dev);
 
-	dev->kcov_handle = kcov_common_handle();
 	if (dev->use_worker) {
-		worker = kthread_create(vhost_worker, dev,
-					"vhost-%d", current->pid);
-		if (IS_ERR(worker)) {
-			err = PTR_ERR(worker);
-			goto err_worker;
-		}
-
-		dev->worker = worker;
-		wake_up_process(worker); /* avoid contributing to loadavg */
-
-		err = vhost_attach_cgroups(dev);
+		err = vhost_worker_create(dev);
 		if (err)
-			goto err_cgroup;
+			goto err_worker;
 	}
 
 	err = vhost_dev_alloc_iovecs(dev);
 	if (err)
-		goto err_cgroup;
+		goto err_iovecs;
 
 	return 0;
-err_cgroup:
-	if (dev->worker) {
-		kthread_stop(dev->worker);
-		dev->worker = NULL;
-	}
+err_iovecs:
+	vhost_worker_free(dev);
 err_worker:
 	vhost_detach_mm(dev);
-	dev->kcov_handle = 0;
 err_mm:
 	return err;
 }
@@ -705,12 +704,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev)
 	dev->iotlb = NULL;
 	vhost_clear_msg(dev);
 	wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
-	WARN_ON(!llist_empty(&dev->work_list));
-	if (dev->worker) {
-		kthread_stop(dev->worker);
-		dev->worker = NULL;
-		dev->kcov_handle = 0;
-	}
+	vhost_worker_free(dev);
 	vhost_detach_mm(dev);
 }
 EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
@@ -1831,7 +1825,7 @@ EXPORT_SYMBOL_GPL(vhost_dev_ioctl);
 
 /* TODO: This is really inefficient.  We need something like get_user()
  * (instruction directly accesses the data, with an exception table entry
- * returning -EFAULT). See Documentation/x86/exception-tables.rst.
+ * returning -EFAULT). See Documentation/arch/x86/exception-tables.rst.
  */
 static int set_bit_to_user(int nr, void __user *addr)
 {
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 1647b750169c..0308638cdeee 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -16,6 +16,7 @@
 #include <linux/irqbypass.h>
 
 struct vhost_work;
+struct vhost_task;
 typedef void (*vhost_work_fn_t)(struct vhost_work *work);
 
 #define VHOST_WORK_QUEUED 1
@@ -25,6 +26,12 @@ struct vhost_work {
 	unsigned long		flags;
 };
 
+struct vhost_worker {
+	struct vhost_task	*vtsk;
+	struct llist_head	work_list;
+	u64			kcov_handle;
+};
+
 /* Poll a file (eventfd or socket) */
 /* Note: there's nothing vhost specific about this structure. */
 struct vhost_poll {
@@ -147,8 +154,7 @@ struct vhost_dev {
 	struct vhost_virtqueue **vqs;
 	int nvqs;
 	struct eventfd_ctx *log_ctx;
-	struct llist_head work_list;
-	struct task_struct *worker;
+	struct vhost_worker *worker;
 	struct vhost_iotlb *umem;
 	struct vhost_iotlb *iotlb;
 	spinlock_t iotlb_lock;
@@ -158,7 +164,6 @@ struct vhost_dev {
 	int iov_limit;
 	int weight;
 	int byte_weight;
-	u64 kcov_handle;
 	bool use_worker;
 	int (*msg_handler)(struct vhost_dev *dev, u32 asid,
 			   struct vhost_iotlb_msg *msg);
diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c
index a1e27da54481..955d938eb663 100644
--- a/drivers/vhost/vringh.c
+++ b/drivers/vhost/vringh.c
@@ -636,9 +636,9 @@ static inline int xfer_to_user(const struct vringh *vrh,
  * @features: the feature bits for this ring.
  * @num: the number of elements.
  * @weak_barriers: true if we only need memory barriers, not I/O.
- * @desc: the userpace descriptor pointer.
- * @avail: the userpace avail pointer.
- * @used: the userpace used pointer.
+ * @desc: the userspace descriptor pointer.
+ * @avail: the userspace avail pointer.
+ * @used: the userspace used pointer.
  *
  * Returns an error if num is invalid: you should check pointers
  * yourself!
@@ -911,9 +911,9 @@ static inline int kern_xfer(const struct vringh *vrh, void *dst,
  * @features: the feature bits for this ring.
  * @num: the number of elements.
  * @weak_barriers: true if we only need memory barriers, not I/O.
- * @desc: the userpace descriptor pointer.
- * @avail: the userpace avail pointer.
- * @used: the userpace used pointer.
+ * @desc: the userspace descriptor pointer.
+ * @avail: the userspace avail pointer.
+ * @used: the userspace used pointer.
  *
  * Returns an error if num is invalid.
  */
@@ -1094,10 +1094,17 @@ EXPORT_SYMBOL(vringh_need_notify_kern);
 
 #if IS_REACHABLE(CONFIG_VHOST_IOTLB)
 
+struct iotlb_vec {
+	union {
+		struct iovec *iovec;
+		struct bio_vec *bvec;
+	} iov;
+	size_t count;
+};
+
 static int iotlb_translate(const struct vringh *vrh,
 			   u64 addr, u64 len, u64 *translated,
-			   struct bio_vec iov[],
-			   int iov_size, u32 perm)
+			   struct iotlb_vec *ivec, u32 perm)
 {
 	struct vhost_iotlb_map *map;
 	struct vhost_iotlb *iotlb = vrh->iotlb;
@@ -1107,9 +1114,11 @@ static int iotlb_translate(const struct vringh *vrh,
 	spin_lock(vrh->iotlb_lock);
 
 	while (len > s) {
-		u64 size, pa, pfn;
+		uintptr_t io_addr;
+		size_t io_len;
+		u64 size;
 
-		if (unlikely(ret >= iov_size)) {
+		if (unlikely(ret >= ivec->count)) {
 			ret = -ENOBUFS;
 			break;
 		}
@@ -1124,10 +1133,22 @@ static int iotlb_translate(const struct vringh *vrh,
 		}
 
 		size = map->size - addr + map->start;
-		pa = map->addr + addr - map->start;
-		pfn = pa >> PAGE_SHIFT;
-		bvec_set_page(&iov[ret], pfn_to_page(pfn), min(len - s, size),
-			      pa & (PAGE_SIZE - 1));
+		io_len = min(len - s, size);
+		io_addr = map->addr - map->start + addr;
+
+		if (vrh->use_va) {
+			struct iovec *iovec = ivec->iov.iovec;
+
+			iovec[ret].iov_len = io_len;
+			iovec[ret].iov_base = (void __user *)io_addr;
+		} else {
+			u64 pfn = io_addr >> PAGE_SHIFT;
+			struct bio_vec *bvec = ivec->iov.bvec;
+
+			bvec_set_page(&bvec[ret], pfn_to_page(pfn), io_len,
+				      io_addr & (PAGE_SIZE - 1));
+		}
+
 		s += size;
 		addr += size;
 		++ret;
@@ -1141,26 +1162,41 @@ static int iotlb_translate(const struct vringh *vrh,
 	return ret;
 }
 
+#define IOTLB_IOV_STRIDE 16
+
 static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
 				  void *src, size_t len)
 {
+	struct iotlb_vec ivec;
+	union {
+		struct iovec iovec[IOTLB_IOV_STRIDE];
+		struct bio_vec bvec[IOTLB_IOV_STRIDE];
+	} iov;
 	u64 total_translated = 0;
 
+	ivec.iov.iovec = iov.iovec;
+	ivec.count = IOTLB_IOV_STRIDE;
+
 	while (total_translated < len) {
-		struct bio_vec iov[16];
 		struct iov_iter iter;
 		u64 translated;
 		int ret;
 
 		ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
 				      len - total_translated, &translated,
-				      iov, ARRAY_SIZE(iov), VHOST_MAP_RO);
+				      &ivec, VHOST_MAP_RO);
 		if (ret == -ENOBUFS)
-			ret = ARRAY_SIZE(iov);
+			ret = IOTLB_IOV_STRIDE;
 		else if (ret < 0)
 			return ret;
 
-		iov_iter_bvec(&iter, ITER_SOURCE, iov, ret, translated);
+		if (vrh->use_va) {
+			iov_iter_init(&iter, ITER_SOURCE, ivec.iov.iovec, ret,
+				      translated);
+		} else {
+			iov_iter_bvec(&iter, ITER_SOURCE, ivec.iov.bvec, ret,
+				      translated);
+		}
 
 		ret = copy_from_iter(dst, translated, &iter);
 		if (ret < 0)
@@ -1177,23 +1213,36 @@ static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
 static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
 				void *src, size_t len)
 {
+	struct iotlb_vec ivec;
+	union {
+		struct iovec iovec[IOTLB_IOV_STRIDE];
+		struct bio_vec bvec[IOTLB_IOV_STRIDE];
+	} iov;
 	u64 total_translated = 0;
 
+	ivec.iov.iovec = iov.iovec;
+	ivec.count = IOTLB_IOV_STRIDE;
+
 	while (total_translated < len) {
-		struct bio_vec iov[16];
 		struct iov_iter iter;
 		u64 translated;
 		int ret;
 
 		ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
 				      len - total_translated, &translated,
-				      iov, ARRAY_SIZE(iov), VHOST_MAP_WO);
+				      &ivec, VHOST_MAP_WO);
 		if (ret == -ENOBUFS)
-			ret = ARRAY_SIZE(iov);
+			ret = IOTLB_IOV_STRIDE;
 		else if (ret < 0)
 			return ret;
 
-		iov_iter_bvec(&iter, ITER_DEST, iov, ret, translated);
+		if (vrh->use_va) {
+			iov_iter_init(&iter, ITER_DEST, ivec.iov.iovec, ret,
+				      translated);
+		} else {
+			iov_iter_bvec(&iter, ITER_DEST, ivec.iov.bvec, ret,
+				      translated);
+		}
 
 		ret = copy_to_iter(src, translated, &iter);
 		if (ret < 0)
@@ -1210,20 +1259,36 @@ static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
 static inline int getu16_iotlb(const struct vringh *vrh,
 			       u16 *val, const __virtio16 *p)
 {
-	struct bio_vec iov;
-	void *kaddr, *from;
+	struct iotlb_vec ivec;
+	union {
+		struct iovec iovec[1];
+		struct bio_vec bvec[1];
+	} iov;
+	__virtio16 tmp;
 	int ret;
 
+	ivec.iov.iovec = iov.iovec;
+	ivec.count = 1;
+
 	/* Atomic read is needed for getu16 */
-	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
-			      &iov, 1, VHOST_MAP_RO);
+	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
+			      NULL, &ivec, VHOST_MAP_RO);
 	if (ret < 0)
 		return ret;
 
-	kaddr = kmap_atomic(iov.bv_page);
-	from = kaddr + iov.bv_offset;
-	*val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from));
-	kunmap_atomic(kaddr);
+	if (vrh->use_va) {
+		ret = __get_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base);
+		if (ret)
+			return ret;
+	} else {
+		void *kaddr = kmap_local_page(ivec.iov.bvec[0].bv_page);
+		void *from = kaddr + ivec.iov.bvec[0].bv_offset;
+
+		tmp = READ_ONCE(*(__virtio16 *)from);
+		kunmap_local(kaddr);
+	}
+
+	*val = vringh16_to_cpu(vrh, tmp);
 
 	return 0;
 }
@@ -1231,20 +1296,36 @@ static inline int getu16_iotlb(const struct vringh *vrh,
 static inline int putu16_iotlb(const struct vringh *vrh,
 			       __virtio16 *p, u16 val)
 {
-	struct bio_vec iov;
-	void *kaddr, *to;
+	struct iotlb_vec ivec;
+	union {
+		struct iovec iovec;
+		struct bio_vec bvec;
+	} iov;
+	__virtio16 tmp;
 	int ret;
 
+	ivec.iov.iovec = &iov.iovec;
+	ivec.count = 1;
+
 	/* Atomic write is needed for putu16 */
-	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
-			      &iov, 1, VHOST_MAP_WO);
+	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
+			      NULL, &ivec, VHOST_MAP_RO);
 	if (ret < 0)
 		return ret;
 
-	kaddr = kmap_atomic(iov.bv_page);
-	to = kaddr + iov.bv_offset;
-	WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val));
-	kunmap_atomic(kaddr);
+	tmp = cpu_to_vringh16(vrh, val);
+
+	if (vrh->use_va) {
+		ret = __put_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base);
+		if (ret)
+			return ret;
+	} else {
+		void *kaddr = kmap_local_page(ivec.iov.bvec[0].bv_page);
+		void *to = kaddr + ivec.iov.bvec[0].bv_offset;
+
+		WRITE_ONCE(*(__virtio16 *)to, tmp);
+		kunmap_local(kaddr);
+	}
 
 	return 0;
 }
@@ -1306,9 +1387,9 @@ static inline int putused_iotlb(const struct vringh *vrh,
  * @features: the feature bits for this ring.
  * @num: the number of elements.
  * @weak_barriers: true if we only need memory barriers, not I/O.
- * @desc: the userpace descriptor pointer.
- * @avail: the userpace avail pointer.
- * @used: the userpace used pointer.
+ * @desc: the userspace descriptor pointer.
+ * @avail: the userspace avail pointer.
+ * @used: the userspace used pointer.
  *
  * Returns an error if num is invalid.
  */
@@ -1318,12 +1399,40 @@ int vringh_init_iotlb(struct vringh *vrh, u64 features,
 		      struct vring_avail *avail,
 		      struct vring_used *used)
 {
+	vrh->use_va = false;
+
 	return vringh_init_kern(vrh, features, num, weak_barriers,
 				desc, avail, used);
 }
 EXPORT_SYMBOL(vringh_init_iotlb);
 
 /**
+ * vringh_init_iotlb_va - initialize a vringh for a ring with IOTLB containing
+ *                        user VA.
+ * @vrh: the vringh to initialize.
+ * @features: the feature bits for this ring.
+ * @num: the number of elements.
+ * @weak_barriers: true if we only need memory barriers, not I/O.
+ * @desc: the userspace descriptor pointer.
+ * @avail: the userspace avail pointer.
+ * @used: the userspace used pointer.
+ *
+ * Returns an error if num is invalid.
+ */
+int vringh_init_iotlb_va(struct vringh *vrh, u64 features,
+			 unsigned int num, bool weak_barriers,
+			 struct vring_desc *desc,
+			 struct vring_avail *avail,
+			 struct vring_used *used)
+{
+	vrh->use_va = true;
+
+	return vringh_init_kern(vrh, features, num, weak_barriers,
+				desc, avail, used);
+}
+EXPORT_SYMBOL(vringh_init_iotlb_va);
+
+/**
  * vringh_set_iotlb - initialize a vringh for a ring with IOTLB.
  * @vrh: the vring
  * @iotlb: iotlb associated with this vring
diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
index c8e6087769a1..6578db78f0ae 100644
--- a/drivers/vhost/vsock.c
+++ b/drivers/vhost/vsock.c
@@ -439,6 +439,7 @@ static struct virtio_transport vhost_transport = {
 		.notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue,
 		.notify_buffer_size       = virtio_transport_notify_buffer_size,
 
+		.read_skb = virtio_transport_read_skb,
 	},
 
 	.send_pkt = vhost_transport_send_pkt,
diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig
index 6d2fde6c5d11..bf05363d8906 100644
--- a/drivers/video/Kconfig
+++ b/drivers/video/Kconfig
@@ -11,6 +11,9 @@ config APERTURE_HELPERS
 	  Support tracking and hand-over of aperture ownership. Required
 	  by graphics drivers for firmware-provided framebuffers.
 
+config VIDEO_CMDLINE
+	bool
+
 config VIDEO_NOMODESET
 	bool
 	default n
diff --git a/drivers/video/Makefile b/drivers/video/Makefile
index a50eb528ed3c..831c9fa57a6c 100644
--- a/drivers/video/Makefile
+++ b/drivers/video/Makefile
@@ -2,6 +2,7 @@
 
 obj-$(CONFIG_APERTURE_HELPERS)    += aperture.o
 obj-$(CONFIG_VGASTATE)            += vgastate.o
+obj-$(CONFIG_VIDEO_CMDLINE)       += cmdline.o
 obj-$(CONFIG_VIDEO_NOMODESET)     += nomodeset.o
 obj-$(CONFIG_HDMI)                += hdmi.o
 
diff --git a/drivers/video/aperture.c b/drivers/video/aperture.c
index 41e77de1ea82..b009468ffdff 100644
--- a/drivers/video/aperture.c
+++ b/drivers/video/aperture.c
@@ -20,7 +20,7 @@
  * driver can be active at any given time. Many systems load a generic
  * graphics drivers, such as EFI-GOP or VESA, early during the boot process.
  * During later boot stages, they replace the generic driver with a dedicated,
- * hardware-specific driver. To take over the device the dedicated driver
+ * hardware-specific driver. To take over the device, the dedicated driver
  * first has to remove the generic driver. Aperture functions manage
  * ownership of framebuffer memory and hand-over between drivers.
  *
@@ -76,7 +76,7 @@
  * generic EFI or VESA drivers, have to register themselves as owners of their
  * framebuffer apertures. Ownership of the framebuffer memory is achieved
  * by calling devm_aperture_acquire_for_platform_device(). If successful, the
- * driveris the owner of the framebuffer range. The function fails if the
+ * driver is the owner of the framebuffer range. The function fails if the
  * framebuffer is already owned by another driver. See below for an example.
  *
  * .. code-block:: c
@@ -126,7 +126,7 @@
  * et al for the registered framebuffer range, the aperture helpers call
  * platform_device_unregister() and the generic driver unloads itself. The
  * generic driver also has to provide a remove function to make this work.
- * Once hot unplugged fro mhardware, it may not access the device's
+ * Once hot unplugged from hardware, it may not access the device's
  * registers, framebuffer memory, ROM, etc afterwards.
  */
 
@@ -203,7 +203,7 @@ static void aperture_detach_platform_device(struct device *dev)
 
 	/*
 	 * Remove the device from the device hierarchy. This is the right thing
-	 * to do for firmware-based DRM drivers, such as EFI, VESA or VGA. After
+	 * to do for firmware-based fb drivers, such as EFI, VESA or VGA. After
 	 * the new driver takes over the hardware, the firmware device's state
 	 * will be lost.
 	 *
diff --git a/drivers/video/backlight/Kconfig b/drivers/video/backlight/Kconfig
index 4c33e971c0f0..51387b1ef012 100644
--- a/drivers/video/backlight/Kconfig
+++ b/drivers/video/backlight/Kconfig
@@ -285,6 +285,7 @@ config BACKLIGHT_MT6370
 config BACKLIGHT_APPLE
 	tristate "Apple Backlight Driver"
 	depends on X86 && ACPI
+	depends on ACPI_VIDEO=n || ACPI_VIDEO
 	help
 	  If you have an Intel-based Apple say Y to enable a driver for its
 	  backlight.
diff --git a/drivers/video/backlight/apple_bl.c b/drivers/video/backlight/apple_bl.c
index e9e7acb577bf..aaa824437a2a 100644
--- a/drivers/video/backlight/apple_bl.c
+++ b/drivers/video/backlight/apple_bl.c
@@ -24,7 +24,7 @@
 #include <linux/pci.h>
 #include <linux/acpi.h>
 #include <linux/atomic.h>
-#include <linux/apple_bl.h>
+#include <acpi/video.h>
 
 static struct backlight_device *apple_backlight_device;
 
@@ -215,32 +215,21 @@ static struct acpi_driver apple_bl_driver = {
 	},
 };
 
-static atomic_t apple_bl_registered = ATOMIC_INIT(0);
-
-int apple_bl_register(void)
-{
-	if (atomic_xchg(&apple_bl_registered, 1) == 0)
-		return acpi_bus_register_driver(&apple_bl_driver);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(apple_bl_register);
-
-void apple_bl_unregister(void)
-{
-	if (atomic_xchg(&apple_bl_registered, 0) == 1)
-		acpi_bus_unregister_driver(&apple_bl_driver);
-}
-EXPORT_SYMBOL_GPL(apple_bl_unregister);
-
 static int __init apple_bl_init(void)
 {
-	return apple_bl_register();
+	/*
+	 * Use ACPI video detection code to see if this driver should register
+	 * or if another driver, e.g. the apple-gmux driver should be used.
+	 */
+	if (acpi_video_get_backlight_type() != acpi_backlight_vendor)
+		return -ENODEV;
+
+	return acpi_bus_register_driver(&apple_bl_driver);
 }
 
 static void __exit apple_bl_exit(void)
 {
-	apple_bl_unregister();
+	acpi_bus_unregister_driver(&apple_bl_driver);
 }
 
 module_init(apple_bl_init);
diff --git a/drivers/video/backlight/backlight.c b/drivers/video/backlight/backlight.c
index 6eea72aa8dbf..9a885d398c22 100644
--- a/drivers/video/backlight/backlight.c
+++ b/drivers/video/backlight/backlight.c
@@ -751,7 +751,7 @@ static void __exit backlight_class_exit(void)
 
 static int __init backlight_class_init(void)
 {
-	backlight_class = class_create(THIS_MODULE, "backlight");
+	backlight_class = class_create("backlight");
 	if (IS_ERR(backlight_class)) {
 		pr_warn("Unable to create backlight class; errno = %ld\n",
 			PTR_ERR(backlight_class));
diff --git a/drivers/video/backlight/lcd.c b/drivers/video/backlight/lcd.c
index db56e465aaff..77c5cb2a44e2 100644
--- a/drivers/video/backlight/lcd.c
+++ b/drivers/video/backlight/lcd.c
@@ -323,7 +323,7 @@ static void __exit lcd_class_exit(void)
 
 static int __init lcd_class_init(void)
 {
-	lcd_class = class_create(THIS_MODULE, "lcd");
+	lcd_class = class_create("lcd");
 	if (IS_ERR(lcd_class)) {
 		pr_warn("Unable to create backlight class; errno = %ld\n",
 			PTR_ERR(lcd_class));
diff --git a/drivers/video/cmdline.c b/drivers/video/cmdline.c
new file mode 100644
index 000000000000..d3d257489c3d
--- /dev/null
+++ b/drivers/video/cmdline.c
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Based on the fbdev code in drivers/video/fbdev/core/fb_cmdline:
+ *
+ *  Copyright (C) 2014 Intel Corp
+ *  Copyright (C) 1994 Martin Schaller
+ *
+ *	2001 - Documented with DocBook
+ *	- Brad Douglas <brad@neruo.com>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file COPYING in the main directory of this archive
+ * for more details.
+ *
+ * Authors:
+ *    Daniel Vetter <daniel.vetter@ffwll.ch>
+ */
+
+#include <linux/fb.h> /* for FB_MAX */
+#include <linux/init.h>
+
+#include <video/cmdline.h>
+
+/*
+ * FB_MAX is the maximum number of framebuffer devices and also
+ * the maximum number of video= parameters. Although not directly
+ * related to each other, it makes sense to keep it that way.
+ */
+static const char *video_options[FB_MAX] __read_mostly;
+static const char *video_option __read_mostly;
+static int video_of_only __read_mostly;
+
+static const char *__video_get_option_string(const char *name)
+{
+	const char *options = NULL;
+	size_t name_len = 0;
+
+	if (name)
+		name_len = strlen(name);
+
+	if (name_len) {
+		unsigned int i;
+		const char *opt;
+
+		for (i = 0; i < ARRAY_SIZE(video_options); ++i) {
+			if (!video_options[i])
+				continue;
+			if (video_options[i][0] == '\0')
+				continue;
+			opt = video_options[i];
+			if (!strncmp(opt, name, name_len) && opt[name_len] == ':')
+				options = opt + name_len + 1;
+		}
+	}
+
+	/* No match, return global options */
+	if (!options)
+		options = video_option;
+
+	return options;
+}
+
+/**
+ * video_get_options - get kernel boot parameters
+ * @name:	name of the output as it would appear in the boot parameter
+ *		line (video=<name>:<options>)
+ *
+ * Looks up the video= options for the given name. Names are connector
+ * names with DRM, or driver names with fbdev. If no video option for
+ * the name has been specified, the function returns the global video=
+ * setting. A @name of NULL always returns the global video setting.
+ *
+ * Returns:
+ * The string of video options for the given name, or NULL if no video
+ * option has been specified.
+ */
+const char *video_get_options(const char *name)
+{
+	return __video_get_option_string(name);
+}
+EXPORT_SYMBOL(video_get_options);
+
+bool __video_get_options(const char *name, const char **options, bool is_of)
+{
+	bool enabled = true;
+	const char *opt = NULL;
+
+	if (video_of_only && !is_of)
+		enabled = false;
+
+	opt = __video_get_option_string(name);
+
+	if (options)
+		*options = opt;
+
+	return enabled;
+}
+EXPORT_SYMBOL(__video_get_options);
+
+/*
+ * Process command line options for video adapters. This function is
+ * a __setup and __init function. It only stores the options. Drivers
+ * have to call video_get_options() as necessary.
+ */
+static int __init video_setup(char *options)
+{
+	if (!options || !*options)
+		goto out;
+
+	if (!strncmp(options, "ofonly", 6)) {
+		video_of_only = true;
+		goto out;
+	}
+
+	if (strchr(options, ':')) {
+		/* named */
+		size_t i;
+
+		for (i = 0; i < ARRAY_SIZE(video_options); i++) {
+			if (!video_options[i]) {
+				video_options[i] = options;
+				break;
+			}
+		}
+	} else {
+		/* global */
+		video_option = options;
+	}
+
+out:
+	return 1;
+}
+__setup("video=", video_setup);
diff --git a/drivers/video/fbdev/68328fb.c b/drivers/video/fbdev/68328fb.c
index 41df61b37a18..3ccf46f8ffd0 100644
--- a/drivers/video/fbdev/68328fb.c
+++ b/drivers/video/fbdev/68328fb.c
@@ -94,6 +94,7 @@ static int mc68x328fb_pan_display(struct fb_var_screeninfo *var,
 static int mc68x328fb_mmap(struct fb_info *info, struct vm_area_struct *vma);
 
 static const struct fb_ops mc68x328fb_ops = {
+	.owner		= THIS_MODULE,
 	.fb_check_var	= mc68x328fb_check_var,
 	.fb_set_par	= mc68x328fb_set_par,
 	.fb_setcolreg	= mc68x328fb_setcolreg,
diff --git a/drivers/video/fbdev/Kconfig b/drivers/video/fbdev/Kconfig
index ff3646c30d0d..96e91570cdd3 100644
--- a/drivers/video/fbdev/Kconfig
+++ b/drivers/video/fbdev/Kconfig
@@ -3,16 +3,13 @@
 # fbdev configuration
 #
 
-config FB_CMDLINE
-	bool
-
 config FB_NOTIFY
 	bool
 
 menuconfig FB
 	tristate "Support for frame buffer devices"
-	select FB_CMDLINE
 	select FB_NOTIFY
+	select VIDEO_CMDLINE
 	help
 	  The frame buffer device provides an abstraction for the graphics
 	  hardware. It represents the frame buffer of some video hardware and
diff --git a/drivers/video/fbdev/aty/radeon_base.c b/drivers/video/fbdev/aty/radeon_base.c
index 657064227de8..972c4bbedfa3 100644
--- a/drivers/video/fbdev/aty/radeon_base.c
+++ b/drivers/video/fbdev/aty/radeon_base.c
@@ -2238,14 +2238,6 @@ static const struct bin_attribute edid2_attr = {
 	.read	= radeon_show_edid2,
 };
 
-static int radeon_kick_out_firmware_fb(struct pci_dev *pdev)
-{
-	resource_size_t base = pci_resource_start(pdev, 0);
-	resource_size_t size = pci_resource_len(pdev, 0);
-
-	return aperture_remove_conflicting_devices(base, size, false, KBUILD_MODNAME);
-}
-
 static int radeonfb_pci_register(struct pci_dev *pdev,
 				 const struct pci_device_id *ent)
 {
@@ -2296,7 +2288,7 @@ static int radeonfb_pci_register(struct pci_dev *pdev,
 	rinfo->fb_base_phys = pci_resource_start (pdev, 0);
 	rinfo->mmio_base_phys = pci_resource_start (pdev, 2);
 
-	ret = radeon_kick_out_firmware_fb(pdev);
+	ret = aperture_remove_conflicting_pci_devices(pdev, KBUILD_MODNAME);
 	if (ret)
 		goto err_release_fb;
 
diff --git a/drivers/video/fbdev/cg14.c b/drivers/video/fbdev/cg14.c
index 6a745eb46ca1..a028ede39c12 100644
--- a/drivers/video/fbdev/cg14.c
+++ b/drivers/video/fbdev/cg14.c
@@ -569,7 +569,7 @@ out_err:
 	return err;
 }
 
-static int cg14_remove(struct platform_device *op)
+static void cg14_remove(struct platform_device *op)
 {
 	struct fb_info *info = dev_get_drvdata(&op->dev);
 	struct cg14_par *par = info->par;
@@ -580,8 +580,6 @@ static int cg14_remove(struct platform_device *op)
 	cg14_unmap_regs(op, info, par);
 
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static const struct of_device_id cg14_match[] = {
@@ -598,7 +596,7 @@ static struct platform_driver cg14_driver = {
 		.of_match_table = cg14_match,
 	},
 	.probe		= cg14_probe,
-	.remove		= cg14_remove,
+	.remove_new	= cg14_remove,
 };
 
 static int __init cg14_init(void)
diff --git a/drivers/video/fbdev/cg3.c b/drivers/video/fbdev/cg3.c
index 3a37fff4df36..6335cd364c74 100644
--- a/drivers/video/fbdev/cg3.c
+++ b/drivers/video/fbdev/cg3.c
@@ -434,7 +434,7 @@ out_err:
 	return err;
 }
 
-static int cg3_remove(struct platform_device *op)
+static void cg3_remove(struct platform_device *op)
 {
 	struct fb_info *info = dev_get_drvdata(&op->dev);
 	struct cg3_par *par = info->par;
@@ -446,8 +446,6 @@ static int cg3_remove(struct platform_device *op)
 	of_iounmap(&op->resource[0], info->screen_base, info->fix.smem_len);
 
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static const struct of_device_id cg3_match[] = {
@@ -467,7 +465,7 @@ static struct platform_driver cg3_driver = {
 		.of_match_table = cg3_match,
 	},
 	.probe		= cg3_probe,
-	.remove		= cg3_remove,
+	.remove_new	= cg3_remove,
 };
 
 static int __init cg3_init(void)
diff --git a/drivers/video/fbdev/cg6.c b/drivers/video/fbdev/cg6.c
index 97ef43c25974..6884572efea1 100644
--- a/drivers/video/fbdev/cg6.c
+++ b/drivers/video/fbdev/cg6.c
@@ -828,7 +828,7 @@ out_err:
 	return err;
 }
 
-static int cg6_remove(struct platform_device *op)
+static void cg6_remove(struct platform_device *op)
 {
 	struct fb_info *info = dev_get_drvdata(&op->dev);
 	struct cg6_par *par = info->par;
@@ -839,8 +839,6 @@ static int cg6_remove(struct platform_device *op)
 	cg6_unmap_regs(op, info, par);
 
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static const struct of_device_id cg6_match[] = {
@@ -860,7 +858,7 @@ static struct platform_driver cg6_driver = {
 		.of_match_table = cg6_match,
 	},
 	.probe		= cg6_probe,
-	.remove		= cg6_remove,
+	.remove_new	= cg6_remove,
 };
 
 static int __init cg6_init(void)
diff --git a/drivers/video/fbdev/clps711x-fb.c b/drivers/video/fbdev/clps711x-fb.c
index c8bfc608bd9c..ac0d058152a3 100644
--- a/drivers/video/fbdev/clps711x-fb.c
+++ b/drivers/video/fbdev/clps711x-fb.c
@@ -350,7 +350,7 @@ out_fb_release:
 	return ret;
 }
 
-static int clps711x_fb_remove(struct platform_device *pdev)
+static void clps711x_fb_remove(struct platform_device *pdev)
 {
 	struct fb_info *info = platform_get_drvdata(pdev);
 	struct clps711x_fb_info *cfb = info->par;
@@ -360,8 +360,6 @@ static int clps711x_fb_remove(struct platform_device *pdev)
 	unregister_framebuffer(info);
 	fb_dealloc_cmap(&info->cmap);
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static const struct of_device_id clps711x_fb_dt_ids[] = {
@@ -376,7 +374,7 @@ static struct platform_driver clps711x_fb_driver = {
 		.of_match_table	= clps711x_fb_dt_ids,
 	},
 	.probe	= clps711x_fb_probe,
-	.remove	= clps711x_fb_remove,
+	.remove_new = clps711x_fb_remove,
 };
 module_platform_driver(clps711x_fb_driver);
 
diff --git a/drivers/video/fbdev/cobalt_lcdfb.c b/drivers/video/fbdev/cobalt_lcdfb.c
index 5f8b6324d2e8..feeace079425 100644
--- a/drivers/video/fbdev/cobalt_lcdfb.c
+++ b/drivers/video/fbdev/cobalt_lcdfb.c
@@ -324,7 +324,7 @@ static int cobalt_lcdfb_probe(struct platform_device *dev)
 	return 0;
 }
 
-static int cobalt_lcdfb_remove(struct platform_device *dev)
+static void cobalt_lcdfb_remove(struct platform_device *dev)
 {
 	struct fb_info *info;
 
@@ -333,13 +333,11 @@ static int cobalt_lcdfb_remove(struct platform_device *dev)
 		unregister_framebuffer(info);
 		framebuffer_release(info);
 	}
-
-	return 0;
 }
 
 static struct platform_driver cobalt_lcdfb_driver = {
 	.probe	= cobalt_lcdfb_probe,
-	.remove	= cobalt_lcdfb_remove,
+	.remove_new = cobalt_lcdfb_remove,
 	.driver	= {
 		.name	= "cobalt-lcd",
 	},
diff --git a/drivers/video/fbdev/core/Makefile b/drivers/video/fbdev/core/Makefile
index 26cbc965497c..08fabce76b74 100644
--- a/drivers/video/fbdev/core/Makefile
+++ b/drivers/video/fbdev/core/Makefile
@@ -1,9 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_FB_CMDLINE)          += fb_cmdline.o
 obj-$(CONFIG_FB_NOTIFY)           += fb_notify.o
 obj-$(CONFIG_FB)                  += fb.o
 fb-y                              := fbmem.o fbmon.o fbcmap.o fbsysfs.o \
-                                     modedb.o fbcvt.o
+                                     modedb.o fbcvt.o fb_cmdline.o
 fb-$(CONFIG_FB_DEFERRED_IO)       += fb_defio.o
 
 ifeq ($(CONFIG_FRAMEBUFFER_CONSOLE),y)
diff --git a/drivers/video/fbdev/core/fb_cmdline.c b/drivers/video/fbdev/core/fb_cmdline.c
index 3b5bd666b952..4d1634c492ec 100644
--- a/drivers/video/fbdev/core/fb_cmdline.c
+++ b/drivers/video/fbdev/core/fb_cmdline.c
@@ -12,16 +12,14 @@
  * for more details.
  *
  * Authors:
- *    Vetter <danie.vetter@ffwll.ch>
+ *    Daniel Vetter <daniel.vetter@ffwll.ch>
  */
-#include <linux/init.h>
-#include <linux/fb.h>
 
-static char *video_options[FB_MAX] __read_mostly;
-static int ofonly __read_mostly;
+#include <linux/export.h>
+#include <linux/fb.h>
+#include <linux/string.h>
 
-const char *fb_mode_option;
-EXPORT_SYMBOL_GPL(fb_mode_option);
+#include <video/cmdline.h>
 
 /**
  * fb_get_options - get kernel boot parameters
@@ -30,78 +28,34 @@ EXPORT_SYMBOL_GPL(fb_mode_option);
  *          (video=<name>:<options>)
  * @option: the option will be stored here
  *
+ * The caller owns the string returned in @option and is
+ * responsible for releasing the memory.
+ *
  * NOTE: Needed to maintain backwards compatibility
  */
 int fb_get_options(const char *name, char **option)
 {
-	char *opt, *options = NULL;
-	int retval = 0;
-	int name_len = strlen(name), i;
-
-	if (name_len && ofonly && strncmp(name, "offb", 4))
-		retval = 1;
+	const char *options = NULL;
+	bool is_of = false;
+	bool enabled;
 
-	if (name_len && !retval) {
-		for (i = 0; i < FB_MAX; i++) {
-			if (video_options[i] == NULL)
-				continue;
-			if (!video_options[i][0])
-				continue;
-			opt = video_options[i];
-			if (!strncmp(name, opt, name_len) &&
-			    opt[name_len] == ':')
-				options = opt + name_len + 1;
-		}
-	}
-	/* No match, pass global option */
-	if (!options && option && fb_mode_option)
-		options = kstrdup(fb_mode_option, GFP_KERNEL);
-	if (options && !strncmp(options, "off", 3))
-		retval = 1;
-
-	if (option)
-		*option = options;
-
-	return retval;
-}
-EXPORT_SYMBOL(fb_get_options);
+	if (name)
+		is_of = strncmp(name, "offb", 4);
 
-/**
- *	video_setup - process command line options
- *	@options: string of options
- *
- *	Process command line options for frame buffer subsystem.
- *
- *	NOTE: This function is a __setup and __init function.
- *            It only stores the options.  Drivers have to call
- *            fb_get_options() as necessary.
- */
-static int __init video_setup(char *options)
-{
-	if (!options || !*options)
-		goto out;
+	enabled = __video_get_options(name, &options, is_of);
 
-	if (!strncmp(options, "ofonly", 6)) {
-		ofonly = 1;
-		goto out;
+	if (options) {
+		if (!strncmp(options, "off", 3))
+			enabled = false;
 	}
 
-	if (strchr(options, ':')) {
-		/* named */
-		int i;
-
-		for (i = 0; i < FB_MAX; i++) {
-			if (video_options[i] == NULL) {
-				video_options[i] = options;
-				break;
-			}
-		}
-	} else {
-		/* global */
-		fb_mode_option = options;
+	if (option) {
+		if (options)
+			*option = kstrdup(options, GFP_KERNEL);
+		else
+			*option = NULL;
 	}
 
-out:
-	return 1;
+	return enabled ? 0 : 1; // 0 on success, 1 otherwise
 }
-__setup("video=", video_setup);
+EXPORT_SYMBOL(fb_get_options);
diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c
index 3fd95a79e4c3..e808dc86001c 100644
--- a/drivers/video/fbdev/core/fbmem.c
+++ b/drivers/video/fbdev/core/fbmem.c
@@ -1751,7 +1751,7 @@ fbmem_init(void)
 		goto err_chrdev;
 	}
 
-	fb_class = class_create(THIS_MODULE, "graphics");
+	fb_class = class_create("graphics");
 	if (IS_ERR(fb_class)) {
 		ret = PTR_ERR(fb_class);
 		pr_warn("Unable to create fb class; errno = %d\n", ret);
diff --git a/drivers/video/fbdev/core/modedb.c b/drivers/video/fbdev/core/modedb.c
index 6473e0dfe146..23cf8eba785d 100644
--- a/drivers/video/fbdev/core/modedb.c
+++ b/drivers/video/fbdev/core/modedb.c
@@ -620,6 +620,7 @@ int fb_find_mode(struct fb_var_screeninfo *var,
 		 const struct fb_videomode *default_mode,
 		 unsigned int default_bpp)
 {
+	char *mode_option_buf = NULL;
 	int i;
 
 	/* Set up defaults */
@@ -635,8 +636,10 @@ int fb_find_mode(struct fb_var_screeninfo *var,
 		default_bpp = 8;
 
 	/* Did the user specify a video mode? */
-	if (!mode_option)
-		mode_option = fb_mode_option;
+	if (!mode_option) {
+		fb_get_options(NULL, &mode_option_buf);
+		mode_option = mode_option_buf;
+	}
 	if (mode_option) {
 		const char *name = mode_option;
 		unsigned int namelen = strlen(name);
@@ -715,6 +718,7 @@ int fb_find_mode(struct fb_var_screeninfo *var,
 			res_specified = 1;
 		}
 done:
+		kfree(mode_option_buf);
 		if (cvt) {
 			struct fb_videomode cvt_mode;
 			int ret;
diff --git a/drivers/video/fbdev/da8xx-fb.c b/drivers/video/fbdev/da8xx-fb.c
index cd07e401b326..60cd1286370f 100644
--- a/drivers/video/fbdev/da8xx-fb.c
+++ b/drivers/video/fbdev/da8xx-fb.c
@@ -1064,7 +1064,7 @@ static void lcd_da8xx_cpufreq_deregister(struct da8xx_fb_par *par)
 }
 #endif
 
-static int fb_remove(struct platform_device *dev)
+static void fb_remove(struct platform_device *dev)
 {
 	struct fb_info *info = platform_get_drvdata(dev);
 	struct da8xx_fb_par *par = info->par;
@@ -1091,8 +1091,6 @@ static int fb_remove(struct platform_device *dev)
 	pm_runtime_put_sync(&dev->dev);
 	pm_runtime_disable(&dev->dev);
 	framebuffer_release(info);
-
-	return 0;
 }
 
 /*
@@ -1657,7 +1655,7 @@ static SIMPLE_DEV_PM_OPS(fb_pm_ops, fb_suspend, fb_resume);
 
 static struct platform_driver da8xx_fb_driver = {
 	.probe = fb_probe,
-	.remove = fb_remove,
+	.remove_new = fb_remove,
 	.driver = {
 		   .name = DRIVER_NAME,
 		   .pm	= &fb_pm_ops,
diff --git a/drivers/video/fbdev/efifb.c b/drivers/video/fbdev/efifb.c
index a5779fb453a2..3d7be69ab593 100644
--- a/drivers/video/fbdev/efifb.c
+++ b/drivers/video/fbdev/efifb.c
@@ -621,15 +621,13 @@ err_release_mem:
 	return err;
 }
 
-static int efifb_remove(struct platform_device *pdev)
+static void efifb_remove(struct platform_device *pdev)
 {
 	struct fb_info *info = platform_get_drvdata(pdev);
 
 	/* efifb_destroy takes care of info cleanup */
 	unregister_framebuffer(info);
 	sysfs_remove_groups(&pdev->dev.kobj, efifb_groups);
-
-	return 0;
 }
 
 static struct platform_driver efifb_driver = {
@@ -637,7 +635,7 @@ static struct platform_driver efifb_driver = {
 		.name = "efi-framebuffer",
 	},
 	.probe = efifb_probe,
-	.remove = efifb_remove,
+	.remove_new = efifb_remove,
 };
 
 builtin_platform_driver(efifb_driver);
diff --git a/drivers/video/fbdev/ep93xx-fb.c b/drivers/video/fbdev/ep93xx-fb.c
index 305f1587bd89..94fe52928be2 100644
--- a/drivers/video/fbdev/ep93xx-fb.c
+++ b/drivers/video/fbdev/ep93xx-fb.c
@@ -573,7 +573,7 @@ failed_cmap:
 	return err;
 }
 
-static int ep93xxfb_remove(struct platform_device *pdev)
+static void ep93xxfb_remove(struct platform_device *pdev)
 {
 	struct fb_info *info = platform_get_drvdata(pdev);
 	struct ep93xx_fbi *fbi = info->par;
@@ -587,13 +587,11 @@ static int ep93xxfb_remove(struct platform_device *pdev)
 		fbi->mach_info->teardown(pdev);
 
 	kfree(info);
-
-	return 0;
 }
 
 static struct platform_driver ep93xxfb_driver = {
 	.probe		= ep93xxfb_probe,
-	.remove		= ep93xxfb_remove,
+	.remove_new	= ep93xxfb_remove,
 	.driver = {
 		.name	= "ep93xx-fb",
 	},
diff --git a/drivers/video/fbdev/ffb.c b/drivers/video/fbdev/ffb.c
index 7cba3969a970..c6d3111dcbb0 100644
--- a/drivers/video/fbdev/ffb.c
+++ b/drivers/video/fbdev/ffb.c
@@ -1023,7 +1023,7 @@ out_err:
 	return err;
 }
 
-static int ffb_remove(struct platform_device *op)
+static void ffb_remove(struct platform_device *op)
 {
 	struct fb_info *info = dev_get_drvdata(&op->dev);
 	struct ffb_par *par = info->par;
@@ -1035,8 +1035,6 @@ static int ffb_remove(struct platform_device *op)
 	of_iounmap(&op->resource[1], par->dac, sizeof(struct ffb_dac));
 
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static const struct of_device_id ffb_match[] = {
@@ -1056,7 +1054,7 @@ static struct platform_driver ffb_driver = {
 		.of_match_table = ffb_match,
 	},
 	.probe		= ffb_probe,
-	.remove		= ffb_remove,
+	.remove_new	= ffb_remove,
 };
 
 static int __init ffb_init(void)
diff --git a/drivers/video/fbdev/fsl-diu-fb.c b/drivers/video/fbdev/fsl-diu-fb.c
index e332017c6af6..730a07d23fa9 100644
--- a/drivers/video/fbdev/fsl-diu-fb.c
+++ b/drivers/video/fbdev/fsl-diu-fb.c
@@ -1823,7 +1823,7 @@ error:
 	return ret;
 }
 
-static int fsl_diu_remove(struct platform_device *pdev)
+static void fsl_diu_remove(struct platform_device *pdev)
 {
 	struct fsl_diu_data *data;
 	int i;
@@ -1837,8 +1837,6 @@ static int fsl_diu_remove(struct platform_device *pdev)
 		uninstall_fb(&data->fsl_diu_info[i]);
 
 	iounmap(data->diu_reg);
-
-	return 0;
 }
 
 #ifndef MODULE
@@ -1885,7 +1883,7 @@ static struct platform_driver fsl_diu_driver = {
 		.of_match_table = fsl_diu_match,
 	},
 	.probe  	= fsl_diu_probe,
-	.remove 	= fsl_diu_remove,
+	.remove_new 	= fsl_diu_remove,
 	.suspend	= fsl_diu_suspend,
 	.resume		= fsl_diu_resume,
 };
diff --git a/drivers/video/fbdev/gbefb.c b/drivers/video/fbdev/gbefb.c
index 000b4aa44241..3f141e21b7e0 100644
--- a/drivers/video/fbdev/gbefb.c
+++ b/drivers/video/fbdev/gbefb.c
@@ -1233,7 +1233,7 @@ out_release_framebuffer:
 	return ret;
 }
 
-static int gbefb_remove(struct platform_device* p_dev)
+static void gbefb_remove(struct platform_device* p_dev)
 {
 	struct fb_info *info = platform_get_drvdata(p_dev);
 	struct gbefb_par *par = info->par;
@@ -1243,13 +1243,11 @@ static int gbefb_remove(struct platform_device* p_dev)
 	arch_phys_wc_del(par->wc_cookie);
 	release_mem_region(GBE_BASE, sizeof(struct sgi_gbe));
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static struct platform_driver gbefb_driver = {
 	.probe = gbefb_probe,
-	.remove = gbefb_remove,
+	.remove_new = gbefb_remove,
 	.driver	= {
 		.name = "gbefb",
 		.dev_groups	= gbefb_groups,
diff --git a/drivers/video/fbdev/goldfishfb.c b/drivers/video/fbdev/goldfishfb.c
index 2b885cd046fe..6fa2108fd912 100644
--- a/drivers/video/fbdev/goldfishfb.c
+++ b/drivers/video/fbdev/goldfishfb.c
@@ -283,7 +283,7 @@ err_fb_alloc_failed:
 	return ret;
 }
 
-static int goldfish_fb_remove(struct platform_device *pdev)
+static void goldfish_fb_remove(struct platform_device *pdev)
 {
 	size_t framesize;
 	struct goldfish_fb *fb = platform_get_drvdata(pdev);
@@ -296,7 +296,6 @@ static int goldfish_fb_remove(struct platform_device *pdev)
 						fb->fb.fix.smem_start);
 	iounmap(fb->reg_base);
 	kfree(fb);
-	return 0;
 }
 
 static const struct of_device_id goldfish_fb_of_match[] = {
@@ -315,7 +314,7 @@ MODULE_DEVICE_TABLE(acpi, goldfish_fb_acpi_match);
 
 static struct platform_driver goldfish_fb_driver = {
 	.probe		= goldfish_fb_probe,
-	.remove		= goldfish_fb_remove,
+	.remove_new	= goldfish_fb_remove,
 	.driver = {
 		.name = "goldfish_fb",
 		.of_match_table = goldfish_fb_of_match,
diff --git a/drivers/video/fbdev/grvga.c b/drivers/video/fbdev/grvga.c
index 24818b276241..9aa15be29ea9 100644
--- a/drivers/video/fbdev/grvga.c
+++ b/drivers/video/fbdev/grvga.c
@@ -504,7 +504,7 @@ free_fb:
 	return retval;
 }
 
-static int grvga_remove(struct platform_device *device)
+static void grvga_remove(struct platform_device *device)
 {
 	struct fb_info *info = dev_get_drvdata(&device->dev);
 	struct grvga_par *par;
@@ -524,8 +524,6 @@ static int grvga_remove(struct platform_device *device)
 
 		framebuffer_release(info);
 	}
-
-	return 0;
 }
 
 static struct of_device_id svgactrl_of_match[] = {
@@ -545,7 +543,7 @@ static struct platform_driver grvga_driver = {
 		.of_match_table = svgactrl_of_match,
 	},
 	.probe		= grvga_probe,
-	.remove		= grvga_remove,
+	.remove_new	= grvga_remove,
 };
 
 module_platform_driver(grvga_driver);
diff --git a/drivers/video/fbdev/hecubafb.c b/drivers/video/fbdev/hecubafb.c
index eb1eaadc1bbb..5a149458d3b4 100644
--- a/drivers/video/fbdev/hecubafb.c
+++ b/drivers/video/fbdev/hecubafb.c
@@ -279,7 +279,7 @@ err_videomem_alloc:
 	return retval;
 }
 
-static int hecubafb_remove(struct platform_device *dev)
+static void hecubafb_remove(struct platform_device *dev)
 {
 	struct fb_info *info = platform_get_drvdata(dev);
 
@@ -293,12 +293,11 @@ static int hecubafb_remove(struct platform_device *dev)
 		module_put(par->board->owner);
 		framebuffer_release(info);
 	}
-	return 0;
 }
 
 static struct platform_driver hecubafb_driver = {
 	.probe	= hecubafb_probe,
-	.remove = hecubafb_remove,
+	.remove_new = hecubafb_remove,
 	.driver	= {
 		.name	= "hecubafb",
 	},
diff --git a/drivers/video/fbdev/hgafb.c b/drivers/video/fbdev/hgafb.c
index bd3d07aa4f0e..20bdab738ab7 100644
--- a/drivers/video/fbdev/hgafb.c
+++ b/drivers/video/fbdev/hgafb.c
@@ -595,7 +595,7 @@ static int hgafb_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int hgafb_remove(struct platform_device *pdev)
+static void hgafb_remove(struct platform_device *pdev)
 {
 	struct fb_info *info = platform_get_drvdata(pdev);
 
@@ -614,13 +614,11 @@ static int hgafb_remove(struct platform_device *pdev)
 
 	if (release_io_port)
 		release_region(0x3bf, 1);
-
-	return 0;
 }
 
 static struct platform_driver hgafb_driver = {
 	.probe = hgafb_probe,
-	.remove = hgafb_remove,
+	.remove_new = hgafb_remove,
 	.driver = {
 		.name = "hgafb",
 	},
diff --git a/drivers/video/fbdev/hitfb.c b/drivers/video/fbdev/hitfb.c
index bbb0f1d953cc..3033f5056976 100644
--- a/drivers/video/fbdev/hitfb.c
+++ b/drivers/video/fbdev/hitfb.c
@@ -415,15 +415,13 @@ err_fb:
 	return ret;
 }
 
-static int hitfb_remove(struct platform_device *dev)
+static void hitfb_remove(struct platform_device *dev)
 {
 	struct fb_info *info = platform_get_drvdata(dev);
 
 	unregister_framebuffer(info);
 	fb_dealloc_cmap(&info->cmap);
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static int hitfb_suspend(struct device *dev)
@@ -460,7 +458,7 @@ static const struct dev_pm_ops hitfb_dev_pm_ops = {
 
 static struct platform_driver hitfb_driver = {
 	.probe		= hitfb_probe,
-	.remove		= hitfb_remove,
+	.remove_new	= hitfb_remove,
 	.driver		= {
 		.name	= "hitfb",
 		.pm	= &hitfb_dev_pm_ops,
diff --git a/drivers/video/fbdev/imxfb.c b/drivers/video/fbdev/imxfb.c
index 51fde1b2a793..adf36690c342 100644
--- a/drivers/video/fbdev/imxfb.c
+++ b/drivers/video/fbdev/imxfb.c
@@ -1051,7 +1051,7 @@ failed_init:
 	return ret;
 }
 
-static int imxfb_remove(struct platform_device *pdev)
+static void imxfb_remove(struct platform_device *pdev)
 {
 	struct fb_info *info = platform_get_drvdata(pdev);
 	struct imxfb_info *fbi = info->par;
@@ -1064,8 +1064,6 @@ static int imxfb_remove(struct platform_device *pdev)
 		    fbi->map_dma);
 	kfree(info->pseudo_palette);
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static int __maybe_unused imxfb_suspend(struct device *dev)
@@ -1097,7 +1095,7 @@ static struct platform_driver imxfb_driver = {
 		.pm	= &imxfb_pm_ops,
 	},
 	.probe		= imxfb_probe,
-	.remove		= imxfb_remove,
+	.remove_new	= imxfb_remove,
 	.id_table	= imxfb_devtype,
 };
 module_platform_driver(imxfb_driver);
diff --git a/drivers/video/fbdev/leo.c b/drivers/video/fbdev/leo.c
index 3eb0f3583f4f..3ffc0a725f89 100644
--- a/drivers/video/fbdev/leo.c
+++ b/drivers/video/fbdev/leo.c
@@ -637,7 +637,7 @@ out_err:
 	return err;
 }
 
-static int leo_remove(struct platform_device *op)
+static void leo_remove(struct platform_device *op)
 {
 	struct fb_info *info = dev_get_drvdata(&op->dev);
 	struct leo_par *par = info->par;
@@ -648,8 +648,6 @@ static int leo_remove(struct platform_device *op)
 	leo_unmap_regs(op, info, par);
 
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static const struct of_device_id leo_match[] = {
@@ -666,7 +664,7 @@ static struct platform_driver leo_driver = {
 		.of_match_table = leo_match,
 	},
 	.probe		= leo_probe,
-	.remove		= leo_remove,
+	.remove_new	= leo_remove,
 };
 
 static int __init leo_init(void)
diff --git a/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c b/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c
index a236fc910148..b5c8fcab9940 100644
--- a/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c
+++ b/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c
@@ -784,7 +784,7 @@ fbrel:
 	return ret;
 }
 
-static int of_platform_mb862xx_remove(struct platform_device *ofdev)
+static void of_platform_mb862xx_remove(struct platform_device *ofdev)
 {
 	struct fb_info *fbi = dev_get_drvdata(&ofdev->dev);
 	struct mb862xxfb_par *par = fbi->par;
@@ -814,7 +814,6 @@ static int of_platform_mb862xx_remove(struct platform_device *ofdev)
 
 	release_mem_region(par->res->start, res_size);
 	framebuffer_release(fbi);
-	return 0;
 }
 
 /*
@@ -838,7 +837,7 @@ static struct platform_driver of_platform_mb862xxfb_driver = {
 		.of_match_table = of_platform_mb862xx_tbl,
 	},
 	.probe		= of_platform_mb862xx_probe,
-	.remove		= of_platform_mb862xx_remove,
+	.remove_new	= of_platform_mb862xx_remove,
 };
 #endif
 
diff --git a/drivers/video/fbdev/metronomefb.c b/drivers/video/fbdev/metronomefb.c
index 9fd4bb85d735..bbdbf463f0c8 100644
--- a/drivers/video/fbdev/metronomefb.c
+++ b/drivers/video/fbdev/metronomefb.c
@@ -744,7 +744,7 @@ err:
 	return retval;
 }
 
-static int metronomefb_remove(struct platform_device *dev)
+static void metronomefb_remove(struct platform_device *dev)
 {
 	struct fb_info *info = platform_get_drvdata(dev);
 
@@ -761,12 +761,11 @@ static int metronomefb_remove(struct platform_device *dev)
 		dev_dbg(&dev->dev, "calling release\n");
 		framebuffer_release(info);
 	}
-	return 0;
 }
 
 static struct platform_driver metronomefb_driver = {
 	.probe	= metronomefb_probe,
-	.remove = metronomefb_remove,
+	.remove_new = metronomefb_remove,
 	.driver	= {
 		.name	= "metronomefb",
 	},
diff --git a/drivers/video/fbdev/mmp/hw/mmp_ctrl.c b/drivers/video/fbdev/mmp/hw/mmp_ctrl.c
index a9df8ee79810..51fbf02a0343 100644
--- a/drivers/video/fbdev/mmp/hw/mmp_ctrl.c
+++ b/drivers/video/fbdev/mmp/hw/mmp_ctrl.c
@@ -514,9 +514,9 @@ static int mmphw_probe(struct platform_device *pdev)
 	/* get clock */
 	ctrl->clk = devm_clk_get(ctrl->dev, mi->clk_name);
 	if (IS_ERR(ctrl->clk)) {
+		ret = PTR_ERR(ctrl->clk);
 		dev_err_probe(ctrl->dev, ret,
 			      "unable to get clk %s\n", mi->clk_name);
-		ret = -ENOENT;
 		goto failed;
 	}
 	clk_prepare_enable(ctrl->clk);
diff --git a/drivers/video/fbdev/mx3fb.c b/drivers/video/fbdev/mx3fb.c
index 76771e126d0a..63c186e0364a 100644
--- a/drivers/video/fbdev/mx3fb.c
+++ b/drivers/video/fbdev/mx3fb.c
@@ -1616,7 +1616,7 @@ eremap:
 	return ret;
 }
 
-static int mx3fb_remove(struct platform_device *dev)
+static void mx3fb_remove(struct platform_device *dev)
 {
 	struct mx3fb_data *mx3fb = platform_get_drvdata(dev);
 	struct fb_info *fbi = mx3fb->fbi;
@@ -1632,7 +1632,6 @@ static int mx3fb_remove(struct platform_device *dev)
 	dmaengine_put();
 
 	iounmap(mx3fb->reg_base);
-	return 0;
 }
 
 static struct platform_driver mx3fb_driver = {
@@ -1640,7 +1639,7 @@ static struct platform_driver mx3fb_driver = {
 		.name = MX3FB_NAME,
 	},
 	.probe = mx3fb_probe,
-	.remove = mx3fb_remove,
+	.remove_new = mx3fb_remove,
 	.suspend = mx3fb_suspend,
 	.resume = mx3fb_resume,
 };
diff --git a/drivers/video/fbdev/ocfb.c b/drivers/video/fbdev/ocfb.c
index da7e1457e58f..7ebe794583e1 100644
--- a/drivers/video/fbdev/ocfb.c
+++ b/drivers/video/fbdev/ocfb.c
@@ -370,7 +370,7 @@ err_dma_free:
 	return ret;
 }
 
-static int ocfb_remove(struct platform_device *pdev)
+static void ocfb_remove(struct platform_device *pdev)
 {
 	struct ocfb_dev *fbdev = platform_get_drvdata(pdev);
 
@@ -383,8 +383,6 @@ static int ocfb_remove(struct platform_device *pdev)
 	ocfb_writereg(fbdev, OCFB_CTRL, 0);
 
 	platform_set_drvdata(pdev, NULL);
-
-	return 0;
 }
 
 static const struct of_device_id ocfb_match[] = {
@@ -395,7 +393,7 @@ MODULE_DEVICE_TABLE(of, ocfb_match);
 
 static struct platform_driver ocfb_driver = {
 	.probe  = ocfb_probe,
-	.remove	= ocfb_remove,
+	.remove_new = ocfb_remove,
 	.driver = {
 		.name = "ocfb_fb",
 		.of_match_table = ocfb_match,
diff --git a/drivers/video/fbdev/offb.c b/drivers/video/fbdev/offb.c
index b97d251d894b..0065a77b6dbc 100644
--- a/drivers/video/fbdev/offb.c
+++ b/drivers/video/fbdev/offb.c
@@ -658,14 +658,12 @@ static void offb_init_nodriver(struct platform_device *parent, struct device_nod
 	}
 }
 
-static int offb_remove(struct platform_device *pdev)
+static void offb_remove(struct platform_device *pdev)
 {
 	struct fb_info *info = platform_get_drvdata(pdev);
 
 	if (info)
 		unregister_framebuffer(info);
-
-	return 0;
 }
 
 static int offb_probe_bootx_noscreen(struct platform_device *pdev)
@@ -680,7 +678,7 @@ static struct platform_driver offb_driver_bootx_noscreen = {
 		.name = "bootx-noscreen",
 	},
 	.probe = offb_probe_bootx_noscreen,
-	.remove = offb_remove,
+	.remove_new = offb_remove,
 };
 
 static int offb_probe_display(struct platform_device *pdev)
@@ -702,7 +700,7 @@ static struct platform_driver offb_driver_display = {
 		.of_match_table = offb_of_match_display,
 	},
 	.probe = offb_probe_display,
-	.remove = offb_remove,
+	.remove_new = offb_remove,
 };
 
 static int __init offb_init(void)
diff --git a/drivers/video/fbdev/omap/omapfb_main.c b/drivers/video/fbdev/omap/omapfb_main.c
index 18736079843d..ad65554b33c3 100644
--- a/drivers/video/fbdev/omap/omapfb_main.c
+++ b/drivers/video/fbdev/omap/omapfb_main.c
@@ -1799,7 +1799,7 @@ void omapfb_register_panel(struct lcd_panel *panel)
 EXPORT_SYMBOL_GPL(omapfb_register_panel);
 
 /* Called when the device is being detached from the driver */
-static int omapfb_remove(struct platform_device *pdev)
+static void omapfb_remove(struct platform_device *pdev)
 {
 	struct omapfb_device *fbdev = platform_get_drvdata(pdev);
 	enum omapfb_state saved_state = fbdev->state;
@@ -1811,8 +1811,6 @@ static int omapfb_remove(struct platform_device *pdev)
 
 	platform_device_unregister(&omapdss_device);
 	fbdev->dssdev = NULL;
-
-	return 0;
 }
 
 /* PM suspend */
@@ -1837,7 +1835,7 @@ static int omapfb_resume(struct platform_device *pdev)
 
 static struct platform_driver omapfb_driver = {
 	.probe		= omapfb_probe,
-	.remove		= omapfb_remove,
+	.remove_new	= omapfb_remove,
 	.suspend	= omapfb_suspend,
 	.resume		= omapfb_resume,
 	.driver		= {
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/core.c b/drivers/video/fbdev/omap2/omapfb/dss/core.c
index 37858be8be83..5fbd8885bad8 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/core.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/core.c
@@ -171,13 +171,11 @@ static int __init omap_dss_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int omap_dss_remove(struct platform_device *pdev)
+static void omap_dss_remove(struct platform_device *pdev)
 {
 	unregister_pm_notifier(&omap_dss_pm_notif_block);
 
 	dss_uninitialize_debugfs();
-
-	return 0;
 }
 
 static void omap_dss_shutdown(struct platform_device *pdev)
@@ -187,7 +185,7 @@ static void omap_dss_shutdown(struct platform_device *pdev)
 }
 
 static struct platform_driver omap_dss_driver = {
-	.remove         = omap_dss_remove,
+	.remove_new     = omap_dss_remove,
 	.shutdown	= omap_dss_shutdown,
 	.driver         = {
 		.name   = "omapdss",
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc.c b/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
index 92fb6b7e1f68..21fef9db90d2 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
@@ -4017,10 +4017,9 @@ static int dispc_probe(struct platform_device *pdev)
 	return component_add(&pdev->dev, &dispc_component_ops);
 }
 
-static int dispc_remove(struct platform_device *pdev)
+static void dispc_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &dispc_component_ops);
-	return 0;
 }
 
 static int dispc_runtime_suspend(struct device *dev)
@@ -4073,7 +4072,7 @@ static const struct of_device_id dispc_of_match[] = {
 
 static struct platform_driver omap_dispchw_driver = {
 	.probe		= dispc_probe,
-	.remove         = dispc_remove,
+	.remove_new     = dispc_remove,
 	.driver         = {
 		.name   = "omapdss_dispc",
 		.pm	= &dispc_pm_ops,
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dpi.c b/drivers/video/fbdev/omap2/omapfb/dss/dpi.c
index 99ce6e955a46..7c1b7d89389a 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dpi.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dpi.c
@@ -810,15 +810,14 @@ static int dpi_probe(struct platform_device *pdev)
 	return component_add(&pdev->dev, &dpi_component_ops);
 }
 
-static int dpi_remove(struct platform_device *pdev)
+static void dpi_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &dpi_component_ops);
-	return 0;
 }
 
 static struct platform_driver omap_dpi_driver = {
 	.probe		= dpi_probe,
-	.remove		= dpi_remove,
+	.remove_new	= dpi_remove,
 	.driver         = {
 		.name   = "omapdss_dpi",
 		.suppress_bind_attrs = true,
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dsi.c b/drivers/video/fbdev/omap2/omapfb/dss/dsi.c
index 7cddb7b8ae34..b7eb17a16ec4 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dsi.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dsi.c
@@ -5495,10 +5495,9 @@ static int dsi_probe(struct platform_device *pdev)
 	return component_add(&pdev->dev, &dsi_component_ops);
 }
 
-static int dsi_remove(struct platform_device *pdev)
+static void dsi_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &dsi_component_ops);
-	return 0;
 }
 
 static int dsi_runtime_suspend(struct device *dev)
@@ -5565,7 +5564,7 @@ static const struct of_device_id dsi_of_match[] = {
 
 static struct platform_driver omap_dsihw_driver = {
 	.probe		= dsi_probe,
-	.remove		= dsi_remove,
+	.remove_new	= dsi_remove,
 	.driver         = {
 		.name   = "omapdss_dsi",
 		.pm	= &dsi_pm_ops,
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss.c b/drivers/video/fbdev/omap2/omapfb/dss/dss.c
index 335e0af4eec1..d814e4baa4b3 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dss.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss.c
@@ -1224,10 +1224,9 @@ static int dss_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int dss_remove(struct platform_device *pdev)
+static void dss_remove(struct platform_device *pdev)
 {
 	component_master_del(&pdev->dev, &dss_component_ops);
-	return 0;
 }
 
 static int dss_runtime_suspend(struct device *dev)
@@ -1279,7 +1278,7 @@ MODULE_DEVICE_TABLE(of, dss_of_match);
 
 static struct platform_driver omap_dsshw_driver = {
 	.probe		= dss_probe,
-	.remove		= dss_remove,
+	.remove_new	= dss_remove,
 	.driver         = {
 		.name   = "omapdss_dss",
 		.pm	= &dss_pm_ops,
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
index 0f39612e002e..f05b4e35a842 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
@@ -756,10 +756,9 @@ static int hdmi4_probe(struct platform_device *pdev)
 	return component_add(&pdev->dev, &hdmi4_component_ops);
 }
 
-static int hdmi4_remove(struct platform_device *pdev)
+static void hdmi4_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &hdmi4_component_ops);
-	return 0;
 }
 
 static int hdmi_runtime_suspend(struct device *dev)
@@ -792,7 +791,7 @@ static const struct of_device_id hdmi_of_match[] = {
 
 static struct platform_driver omapdss_hdmihw_driver = {
 	.probe		= hdmi4_probe,
-	.remove		= hdmi4_remove,
+	.remove_new	= hdmi4_remove,
 	.driver         = {
 		.name   = "omapdss_hdmi",
 		.pm	= &hdmi_pm_ops,
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
index bfccc2cb917a..03292945b1d4 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
@@ -797,10 +797,9 @@ static int hdmi5_probe(struct platform_device *pdev)
 	return component_add(&pdev->dev, &hdmi5_component_ops);
 }
 
-static int hdmi5_remove(struct platform_device *pdev)
+static void hdmi5_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &hdmi5_component_ops);
-	return 0;
 }
 
 static int hdmi_runtime_suspend(struct device *dev)
@@ -834,7 +833,7 @@ static const struct of_device_id hdmi_of_match[] = {
 
 static struct platform_driver omapdss_hdmihw_driver = {
 	.probe		= hdmi5_probe,
-	.remove		= hdmi5_remove,
+	.remove_new	= hdmi5_remove,
 	.driver         = {
 		.name   = "omapdss_hdmi5",
 		.pm	= &hdmi_pm_ops,
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/sdi.c b/drivers/video/fbdev/omap2/omapfb/dss/sdi.c
index 002f07f5480f..d527931b2b16 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/sdi.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/sdi.c
@@ -375,15 +375,14 @@ static int sdi_probe(struct platform_device *pdev)
 	return component_add(&pdev->dev, &sdi_component_ops);
 }
 
-static int sdi_remove(struct platform_device *pdev)
+static void sdi_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &sdi_component_ops);
-	return 0;
 }
 
 static struct platform_driver omap_sdi_driver = {
 	.probe		= sdi_probe,
-	.remove         = sdi_remove,
+	.remove_new     = sdi_remove,
 	.driver         = {
 		.name   = "omapdss_sdi",
 		.suppress_bind_attrs = true,
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/venc.c b/drivers/video/fbdev/omap2/omapfb/dss/venc.c
index 78a7309d25dd..c9d40e28a06f 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/venc.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/venc.c
@@ -880,10 +880,9 @@ static int venc_probe(struct platform_device *pdev)
 	return component_add(&pdev->dev, &venc_component_ops);
 }
 
-static int venc_remove(struct platform_device *pdev)
+static void venc_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &venc_component_ops);
-	return 0;
 }
 
 static int venc_runtime_suspend(struct device *dev)
@@ -922,7 +921,7 @@ static const struct of_device_id venc_of_match[] = {
 
 static struct platform_driver omap_venchw_driver = {
 	.probe		= venc_probe,
-	.remove		= venc_remove,
+	.remove_new	= venc_remove,
 	.driver         = {
 		.name   = "omapdss_venc",
 		.pm	= &venc_pm_ops,
diff --git a/drivers/video/fbdev/omap2/omapfb/omapfb-main.c b/drivers/video/fbdev/omap2/omapfb/omapfb-main.c
index 5ccddcfce722..c0538069eb48 100644
--- a/drivers/video/fbdev/omap2/omapfb/omapfb-main.c
+++ b/drivers/video/fbdev/omap2/omapfb/omapfb-main.c
@@ -2599,7 +2599,7 @@ err0:
 	return r;
 }
 
-static int omapfb_remove(struct platform_device *pdev)
+static void omapfb_remove(struct platform_device *pdev)
 {
 	struct omapfb2_device *fbdev = platform_get_drvdata(pdev);
 
@@ -2610,13 +2610,11 @@ static int omapfb_remove(struct platform_device *pdev)
 	omapfb_free_resources(fbdev);
 
 	omapdss_compat_uninit();
-
-	return 0;
 }
 
 static struct platform_driver omapfb_driver = {
 	.probe		= omapfb_probe,
-	.remove         = omapfb_remove,
+	.remove_new     = omapfb_remove,
 	.driver         = {
 		.name   = "omapfb",
 	},
diff --git a/drivers/video/fbdev/p9100.c b/drivers/video/fbdev/p9100.c
index 4e88a0a195ad..3e44f9516318 100644
--- a/drivers/video/fbdev/p9100.c
+++ b/drivers/video/fbdev/p9100.c
@@ -327,7 +327,7 @@ out_err:
 	return err;
 }
 
-static int p9100_remove(struct platform_device *op)
+static void p9100_remove(struct platform_device *op)
 {
 	struct fb_info *info = dev_get_drvdata(&op->dev);
 	struct p9100_par *par = info->par;
@@ -339,8 +339,6 @@ static int p9100_remove(struct platform_device *op)
 	of_iounmap(&op->resource[2], info->screen_base, info->fix.smem_len);
 
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static const struct of_device_id p9100_match[] = {
@@ -357,7 +355,7 @@ static struct platform_driver p9100_driver = {
 		.of_match_table = p9100_match,
 	},
 	.probe		= p9100_probe,
-	.remove		= p9100_remove,
+	.remove_new	= p9100_remove,
 };
 
 static int __init p9100_init(void)
diff --git a/drivers/video/fbdev/platinumfb.c b/drivers/video/fbdev/platinumfb.c
index 5b9e26ea6449..82f019f0a0d6 100644
--- a/drivers/video/fbdev/platinumfb.c
+++ b/drivers/video/fbdev/platinumfb.c
@@ -636,7 +636,7 @@ static int platinumfb_probe(struct platform_device* odev)
 	return rc;
 }
 
-static int platinumfb_remove(struct platform_device* odev)
+static void platinumfb_remove(struct platform_device* odev)
 {
 	struct fb_info		*info = dev_get_drvdata(&odev->dev);
 	struct fb_info_platinum	*pinfo = info->par;
@@ -654,8 +654,6 @@ static int platinumfb_remove(struct platform_device* odev)
 	release_mem_region(pinfo->cmap_regs_phys, 0x1000);
 
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static struct of_device_id platinumfb_match[] = 
@@ -673,7 +671,7 @@ static struct platform_driver platinum_driver =
 		.of_match_table = platinumfb_match,
 	},
 	.probe		= platinumfb_probe,
-	.remove		= platinumfb_remove,
+	.remove_new	= platinumfb_remove,
 };
 
 static int __init platinumfb_init(void)
diff --git a/drivers/video/fbdev/ps3fb.c b/drivers/video/fbdev/ps3fb.c
index 2fe08b67eda7..98aaa330a193 100644
--- a/drivers/video/fbdev/ps3fb.c
+++ b/drivers/video/fbdev/ps3fb.c
@@ -936,6 +936,7 @@ static irqreturn_t ps3fb_vsync_interrupt(int irq, void *ptr)
 
 
 static const struct fb_ops ps3fb_ops = {
+	.owner		= THIS_MODULE,
 	.fb_open	= ps3fb_open,
 	.fb_release	= ps3fb_release,
 	.fb_read        = fb_sys_read,
diff --git a/drivers/video/fbdev/pxa168fb.c b/drivers/video/fbdev/pxa168fb.c
index d5d0bbd39213..79f338463092 100644
--- a/drivers/video/fbdev/pxa168fb.c
+++ b/drivers/video/fbdev/pxa168fb.c
@@ -765,14 +765,14 @@ failed_free_info:
 	return ret;
 }
 
-static int pxa168fb_remove(struct platform_device *pdev)
+static void pxa168fb_remove(struct platform_device *pdev)
 {
 	struct pxa168fb_info *fbi = platform_get_drvdata(pdev);
 	struct fb_info *info;
 	unsigned int data;
 
 	if (!fbi)
-		return 0;
+		return;
 
 	/* disable DMA transfer */
 	data = readl(fbi->reg_base + LCD_SPU_DMA_CTRL0);
@@ -794,8 +794,6 @@ static int pxa168fb_remove(struct platform_device *pdev)
 	clk_disable_unprepare(fbi->clk);
 
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static struct platform_driver pxa168fb_driver = {
@@ -803,7 +801,7 @@ static struct platform_driver pxa168fb_driver = {
 		.name	= "pxa168-fb",
 	},
 	.probe		= pxa168fb_probe,
-	.remove		= pxa168fb_remove,
+	.remove_new	= pxa168fb_remove,
 };
 
 module_platform_driver(pxa168fb_driver);
diff --git a/drivers/video/fbdev/pxa3xx-gcu.c b/drivers/video/fbdev/pxa3xx-gcu.c
index d16729215423..43c80316d84b 100644
--- a/drivers/video/fbdev/pxa3xx-gcu.c
+++ b/drivers/video/fbdev/pxa3xx-gcu.c
@@ -675,7 +675,7 @@ err_free_dma:
 	return ret;
 }
 
-static int pxa3xx_gcu_remove(struct platform_device *pdev)
+static void pxa3xx_gcu_remove(struct platform_device *pdev)
 {
 	struct pxa3xx_gcu_priv *priv = platform_get_drvdata(pdev);
 	struct device *dev = &pdev->dev;
@@ -685,8 +685,6 @@ static int pxa3xx_gcu_remove(struct platform_device *pdev)
 	dma_free_coherent(dev, SHARED_SIZE, priv->shared, priv->shared_phys);
 	clk_disable_unprepare(priv->clk);
 	pxa3xx_gcu_free_buffers(dev, priv);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
@@ -699,7 +697,7 @@ MODULE_DEVICE_TABLE(of, pxa3xx_gcu_of_match);
 
 static struct platform_driver pxa3xx_gcu_driver = {
 	.probe	  = pxa3xx_gcu_probe,
-	.remove	 = pxa3xx_gcu_remove,
+	.remove_new	 = pxa3xx_gcu_remove,
 	.driver	 = {
 		.name   = DRV_NAME,
 		.of_match_table = of_match_ptr(pxa3xx_gcu_of_match),
diff --git a/drivers/video/fbdev/pxafb.c b/drivers/video/fbdev/pxafb.c
index c46ed78298ae..2a8b1dea3a67 100644
--- a/drivers/video/fbdev/pxafb.c
+++ b/drivers/video/fbdev/pxafb.c
@@ -2396,13 +2396,13 @@ failed:
 	return ret;
 }
 
-static int pxafb_remove(struct platform_device *dev)
+static void pxafb_remove(struct platform_device *dev)
 {
 	struct pxafb_info *fbi = platform_get_drvdata(dev);
 	struct fb_info *info;
 
 	if (!fbi)
-		return 0;
+		return;
 
 	info = &fbi->fb;
 
@@ -2418,8 +2418,6 @@ static int pxafb_remove(struct platform_device *dev)
 
 	dma_free_coherent(&dev->dev, fbi->dma_buff_size, fbi->dma_buff,
 			  fbi->dma_buff_phys);
-
-	return 0;
 }
 
 static const struct of_device_id pxafb_of_dev_id[] = {
@@ -2432,7 +2430,7 @@ MODULE_DEVICE_TABLE(of, pxafb_of_dev_id);
 
 static struct platform_driver pxafb_driver = {
 	.probe		= pxafb_probe,
-	.remove 	= pxafb_remove,
+	.remove_new 	= pxafb_remove,
 	.driver		= {
 		.name	= "pxa2xx-fb",
 		.of_match_table = pxafb_of_dev_id,
diff --git a/drivers/video/fbdev/s1d13xxxfb.c b/drivers/video/fbdev/s1d13xxxfb.c
index d1b5f965bc96..8f2edccdba46 100644
--- a/drivers/video/fbdev/s1d13xxxfb.c
+++ b/drivers/video/fbdev/s1d13xxxfb.c
@@ -748,13 +748,12 @@ static void __s1d13xxxfb_remove(struct platform_device *pdev)
 			   resource_size(&pdev->resource[1]));
 }
 
-static int s1d13xxxfb_remove(struct platform_device *pdev)
+static void s1d13xxxfb_remove(struct platform_device *pdev)
 {
 	struct fb_info *info = platform_get_drvdata(pdev);
 
 	unregister_framebuffer(info);
 	__s1d13xxxfb_remove(pdev);
-	return 0;
 }
 
 static int s1d13xxxfb_probe(struct platform_device *pdev)
@@ -995,7 +994,7 @@ static int s1d13xxxfb_resume(struct platform_device *dev)
 
 static struct platform_driver s1d13xxxfb_driver = {
 	.probe		= s1d13xxxfb_probe,
-	.remove		= s1d13xxxfb_remove,
+	.remove_new	= s1d13xxxfb_remove,
 #ifdef CONFIG_PM
 	.suspend	= s1d13xxxfb_suspend,
 	.resume		= s1d13xxxfb_resume,
diff --git a/drivers/video/fbdev/s3c-fb.c b/drivers/video/fbdev/s3c-fb.c
index 3abbc5737c3b..1ce707e4cfd0 100644
--- a/drivers/video/fbdev/s3c-fb.c
+++ b/drivers/video/fbdev/s3c-fb.c
@@ -1507,7 +1507,7 @@ err_bus_clk:
  * Shutdown and then release all the resources that the driver allocated
  * on initialisation.
  */
-static int s3c_fb_remove(struct platform_device *pdev)
+static void s3c_fb_remove(struct platform_device *pdev)
 {
 	struct s3c_fb *sfb = platform_get_drvdata(pdev);
 	int win;
@@ -1525,8 +1525,6 @@ static int s3c_fb_remove(struct platform_device *pdev)
 
 	pm_runtime_put_sync(sfb->dev);
 	pm_runtime_disable(sfb->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -1794,7 +1792,7 @@ static const struct dev_pm_ops s3cfb_pm_ops = {
 
 static struct platform_driver s3c_fb_driver = {
 	.probe		= s3c_fb_probe,
-	.remove		= s3c_fb_remove,
+	.remove_new	= s3c_fb_remove,
 	.id_table	= s3c_fb_driver_ids,
 	.driver		= {
 		.name	= "s3c-fb",
diff --git a/drivers/video/fbdev/sh7760fb.c b/drivers/video/fbdev/sh7760fb.c
index 5978a8921232..768011bdb430 100644
--- a/drivers/video/fbdev/sh7760fb.c
+++ b/drivers/video/fbdev/sh7760fb.c
@@ -554,7 +554,7 @@ out_fb:
 	return ret;
 }
 
-static int sh7760fb_remove(struct platform_device *dev)
+static void sh7760fb_remove(struct platform_device *dev)
 {
 	struct fb_info *info = platform_get_drvdata(dev);
 	struct sh7760fb_par *par = info->par;
@@ -568,8 +568,6 @@ static int sh7760fb_remove(struct platform_device *dev)
 	iounmap(par->base);
 	release_mem_region(par->ioarea->start, resource_size(par->ioarea));
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static struct platform_driver sh7760_lcdc_driver = {
@@ -577,7 +575,7 @@ static struct platform_driver sh7760_lcdc_driver = {
 		   .name = "sh7760-lcdc",
 		   },
 	.probe = sh7760fb_probe,
-	.remove = sh7760fb_remove,
+	.remove_new = sh7760fb_remove,
 };
 
 module_platform_driver(sh7760_lcdc_driver);
diff --git a/drivers/video/fbdev/sh_mobile_lcdcfb.c b/drivers/video/fbdev/sh_mobile_lcdcfb.c
index ad9323ed8e2e..093f035d1246 100644
--- a/drivers/video/fbdev/sh_mobile_lcdcfb.c
+++ b/drivers/video/fbdev/sh_mobile_lcdcfb.c
@@ -2249,7 +2249,7 @@ static const struct fb_videomode default_720p = {
 	.sync = FB_SYNC_VERT_HIGH_ACT | FB_SYNC_HOR_HIGH_ACT,
 };
 
-static int sh_mobile_lcdc_remove(struct platform_device *pdev)
+static void sh_mobile_lcdc_remove(struct platform_device *pdev)
 {
 	struct sh_mobile_lcdc_priv *priv = platform_get_drvdata(pdev);
 	unsigned int i;
@@ -2305,7 +2305,6 @@ static int sh_mobile_lcdc_remove(struct platform_device *pdev)
 	if (priv->irq)
 		free_irq(priv->irq, priv);
 	kfree(priv);
-	return 0;
 }
 
 static int sh_mobile_lcdc_check_interface(struct sh_mobile_lcdc_chan *ch)
@@ -2656,7 +2655,7 @@ static struct platform_driver sh_mobile_lcdc_driver = {
 		.pm		= &sh_mobile_lcdc_dev_pm_ops,
 	},
 	.probe		= sh_mobile_lcdc_probe,
-	.remove		= sh_mobile_lcdc_remove,
+	.remove_new	= sh_mobile_lcdc_remove,
 };
 
 module_platform_driver(sh_mobile_lcdc_driver);
diff --git a/drivers/video/fbdev/simplefb.c b/drivers/video/fbdev/simplefb.c
index 10d71879d340..e4a13871bca6 100644
--- a/drivers/video/fbdev/simplefb.c
+++ b/drivers/video/fbdev/simplefb.c
@@ -538,14 +538,12 @@ error_release_mem_region:
 	return ret;
 }
 
-static int simplefb_remove(struct platform_device *pdev)
+static void simplefb_remove(struct platform_device *pdev)
 {
 	struct fb_info *info = platform_get_drvdata(pdev);
 
 	/* simplefb_destroy takes care of info cleanup */
 	unregister_framebuffer(info);
-
-	return 0;
 }
 
 static const struct of_device_id simplefb_of_match[] = {
@@ -560,7 +558,7 @@ static struct platform_driver simplefb_driver = {
 		.of_match_table = simplefb_of_match,
 	},
 	.probe = simplefb_probe,
-	.remove = simplefb_remove,
+	.remove_new = simplefb_remove,
 };
 
 module_platform_driver(simplefb_driver);
diff --git a/drivers/video/fbdev/sm501fb.c b/drivers/video/fbdev/sm501fb.c
index 1f3cbe723def..e0d29be1565b 100644
--- a/drivers/video/fbdev/sm501fb.c
+++ b/drivers/video/fbdev/sm501fb.c
@@ -2045,7 +2045,7 @@ err_alloc:
 /*
  *  Cleanup
  */
-static int sm501fb_remove(struct platform_device *pdev)
+static void sm501fb_remove(struct platform_device *pdev)
 {
 	struct sm501fb_info *info = platform_get_drvdata(pdev);
 	struct fb_info	   *fbinfo_crt = info->fb[0];
@@ -2064,8 +2064,6 @@ static int sm501fb_remove(struct platform_device *pdev)
 
 	framebuffer_release(fbinfo_pnl);
 	framebuffer_release(fbinfo_crt);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -2209,7 +2207,7 @@ static int sm501fb_resume(struct platform_device *pdev)
 
 static struct platform_driver sm501fb_driver = {
 	.probe		= sm501fb_probe,
-	.remove		= sm501fb_remove,
+	.remove_new	= sm501fb_remove,
 	.suspend	= sm501fb_suspend,
 	.resume		= sm501fb_resume,
 	.driver		= {
diff --git a/drivers/video/fbdev/tcx.c b/drivers/video/fbdev/tcx.c
index f2eaf6e7fff6..fc3ac2301b45 100644
--- a/drivers/video/fbdev/tcx.c
+++ b/drivers/video/fbdev/tcx.c
@@ -486,7 +486,7 @@ out_err:
 	return err;
 }
 
-static int tcx_remove(struct platform_device *op)
+static void tcx_remove(struct platform_device *op)
 {
 	struct fb_info *info = dev_get_drvdata(&op->dev);
 	struct tcx_par *par = info->par;
@@ -497,8 +497,6 @@ static int tcx_remove(struct platform_device *op)
 	tcx_unmap_regs(op, info, par);
 
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static const struct of_device_id tcx_match[] = {
@@ -515,7 +513,7 @@ static struct platform_driver tcx_driver = {
 		.of_match_table = tcx_match,
 	},
 	.probe		= tcx_probe,
-	.remove		= tcx_remove,
+	.remove_new	= tcx_remove,
 };
 
 static int __init tcx_init(void)
diff --git a/drivers/video/fbdev/uvesafb.c b/drivers/video/fbdev/uvesafb.c
index f09f483c219b..78d85dae8ec8 100644
--- a/drivers/video/fbdev/uvesafb.c
+++ b/drivers/video/fbdev/uvesafb.c
@@ -1774,7 +1774,7 @@ out:
 	return err;
 }
 
-static int uvesafb_remove(struct platform_device *dev)
+static void uvesafb_remove(struct platform_device *dev)
 {
 	struct fb_info *info = platform_get_drvdata(dev);
 	struct uvesafb_par *par = info->par;
@@ -1793,13 +1793,11 @@ static int uvesafb_remove(struct platform_device *dev)
 	kfree(par->vbe_state_saved);
 
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static struct platform_driver uvesafb_driver = {
 	.probe  = uvesafb_probe,
-	.remove = uvesafb_remove,
+	.remove_new = uvesafb_remove,
 	.driver = {
 		.name = "uvesafb",
 	},
diff --git a/drivers/video/fbdev/vesafb.c b/drivers/video/fbdev/vesafb.c
index 3f8bdfcf51f0..7451c607dc50 100644
--- a/drivers/video/fbdev/vesafb.c
+++ b/drivers/video/fbdev/vesafb.c
@@ -485,7 +485,7 @@ err_release_region:
 	return err;
 }
 
-static int vesafb_remove(struct platform_device *pdev)
+static void vesafb_remove(struct platform_device *pdev)
 {
 	struct fb_info *info = platform_get_drvdata(pdev);
 
@@ -494,8 +494,6 @@ static int vesafb_remove(struct platform_device *pdev)
 
 	/* vesafb_destroy takes care of info cleanup */
 	unregister_framebuffer(info);
-
-	return 0;
 }
 
 static struct platform_driver vesafb_driver = {
@@ -503,7 +501,7 @@ static struct platform_driver vesafb_driver = {
 		.name = "vesa-framebuffer",
 	},
 	.probe = vesafb_probe,
-	.remove = vesafb_remove,
+	.remove_new = vesafb_remove,
 };
 
 module_platform_driver(vesafb_driver);
diff --git a/drivers/video/fbdev/vfb.c b/drivers/video/fbdev/vfb.c
index 95d3c59867d0..a94573997d15 100644
--- a/drivers/video/fbdev/vfb.c
+++ b/drivers/video/fbdev/vfb.c
@@ -79,6 +79,7 @@ static int vfb_mmap(struct fb_info *info,
 		    struct vm_area_struct *vma);
 
 static const struct fb_ops vfb_ops = {
+	.owner		= THIS_MODULE,
 	.fb_read        = fb_sys_read,
 	.fb_write       = fb_sys_write,
 	.fb_check_var	= vfb_check_var,
@@ -479,7 +480,7 @@ err:
 	return retval;
 }
 
-static int vfb_remove(struct platform_device *dev)
+static void vfb_remove(struct platform_device *dev)
 {
 	struct fb_info *info = platform_get_drvdata(dev);
 
@@ -489,12 +490,11 @@ static int vfb_remove(struct platform_device *dev)
 		fb_dealloc_cmap(&info->cmap);
 		framebuffer_release(info);
 	}
-	return 0;
 }
 
 static struct platform_driver vfb_driver = {
 	.probe	= vfb_probe,
-	.remove = vfb_remove,
+	.remove_new = vfb_remove,
 	.driver = {
 		.name	= "vfb",
 	},
diff --git a/drivers/video/fbdev/vga16fb.c b/drivers/video/fbdev/vga16fb.c
index 1a8ffdb2be26..34d00347ad58 100644
--- a/drivers/video/fbdev/vga16fb.c
+++ b/drivers/video/fbdev/vga16fb.c
@@ -1401,14 +1401,12 @@ static int vga16fb_probe(struct platform_device *dev)
 	return ret;
 }
 
-static int vga16fb_remove(struct platform_device *dev)
+static void vga16fb_remove(struct platform_device *dev)
 {
 	struct fb_info *info = platform_get_drvdata(dev);
 
 	if (info)
 		unregister_framebuffer(info);
-
-	return 0;
 }
 
 static const struct platform_device_id vga16fb_driver_id_table[] = {
@@ -1420,7 +1418,7 @@ MODULE_DEVICE_TABLE(platform, vga16fb_driver_id_table);
 
 static struct platform_driver vga16fb_driver = {
 	.probe = vga16fb_probe,
-	.remove = vga16fb_remove,
+	.remove_new = vga16fb_remove,
 	.driver = {
 		.name = "vga16fb",
 	},
diff --git a/drivers/video/fbdev/via/via-gpio.c b/drivers/video/fbdev/via/via-gpio.c
index febb2aadd822..f1b670397c02 100644
--- a/drivers/video/fbdev/via/via-gpio.c
+++ b/drivers/video/fbdev/via/via-gpio.c
@@ -262,7 +262,7 @@ static int viafb_gpio_probe(struct platform_device *platdev)
 }
 
 
-static int viafb_gpio_remove(struct platform_device *platdev)
+static void viafb_gpio_remove(struct platform_device *platdev)
 {
 	unsigned long flags;
 	int i;
@@ -285,7 +285,6 @@ static int viafb_gpio_remove(struct platform_device *platdev)
 		viafb_gpio_disable(viafb_gpio_config.active_gpios[i]);
 	viafb_gpio_config.gpio_chip.ngpio = 0;
 	spin_unlock_irqrestore(&viafb_gpio_config.vdev->reg_lock, flags);
-	return 0;
 }
 
 static struct platform_driver via_gpio_driver = {
@@ -293,7 +292,7 @@ static struct platform_driver via_gpio_driver = {
 		.name = "viafb-gpio",
 	},
 	.probe = viafb_gpio_probe,
-	.remove = viafb_gpio_remove,
+	.remove_new = viafb_gpio_remove,
 };
 
 int viafb_gpio_init(void)
diff --git a/drivers/video/fbdev/via/via_i2c.c b/drivers/video/fbdev/via/via_i2c.c
index c7e63ab47c39..c35e530e0ec9 100644
--- a/drivers/video/fbdev/via/via_i2c.c
+++ b/drivers/video/fbdev/via/via_i2c.c
@@ -246,7 +246,7 @@ static int viafb_i2c_probe(struct platform_device *platdev)
 	return 0;
 }
 
-static int viafb_i2c_remove(struct platform_device *platdev)
+static void viafb_i2c_remove(struct platform_device *platdev)
 {
 	int i;
 
@@ -259,7 +259,6 @@ static int viafb_i2c_remove(struct platform_device *platdev)
 		if (i2c_stuff->is_active)
 			i2c_del_adapter(&i2c_stuff->adapter);
 	}
-	return 0;
 }
 
 static struct platform_driver via_i2c_driver = {
@@ -267,7 +266,7 @@ static struct platform_driver via_i2c_driver = {
 		.name = "viafb-i2c",
 	},
 	.probe = viafb_i2c_probe,
-	.remove = viafb_i2c_remove,
+	.remove_new = viafb_i2c_remove,
 };
 
 int viafb_i2c_init(void)
diff --git a/drivers/video/fbdev/vt8500lcdfb.c b/drivers/video/fbdev/vt8500lcdfb.c
index c61476247ba8..31d4e85b220c 100644
--- a/drivers/video/fbdev/vt8500lcdfb.c
+++ b/drivers/video/fbdev/vt8500lcdfb.c
@@ -439,7 +439,7 @@ failed_free_res:
 	return ret;
 }
 
-static int vt8500lcd_remove(struct platform_device *pdev)
+static void vt8500lcd_remove(struct platform_device *pdev)
 {
 	struct vt8500lcd_info *fbi = platform_get_drvdata(pdev);
 	struct resource *res;
@@ -462,8 +462,6 @@ static int vt8500lcd_remove(struct platform_device *pdev)
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	release_mem_region(res->start, resource_size(res));
-
-	return 0;
 }
 
 static const struct of_device_id via_dt_ids[] = {
@@ -473,7 +471,7 @@ static const struct of_device_id via_dt_ids[] = {
 
 static struct platform_driver vt8500lcd_driver = {
 	.probe		= vt8500lcd_probe,
-	.remove		= vt8500lcd_remove,
+	.remove_new	= vt8500lcd_remove,
 	.driver		= {
 		.name	= "vt8500-lcd",
 		.of_match_table = of_match_ptr(via_dt_ids),
diff --git a/drivers/video/fbdev/wm8505fb.c b/drivers/video/fbdev/wm8505fb.c
index 96a6f7623e19..10a8b1250103 100644
--- a/drivers/video/fbdev/wm8505fb.c
+++ b/drivers/video/fbdev/wm8505fb.c
@@ -372,7 +372,7 @@ static int wm8505fb_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int wm8505fb_remove(struct platform_device *pdev)
+static void wm8505fb_remove(struct platform_device *pdev)
 {
 	struct wm8505fb_info *fbi = platform_get_drvdata(pdev);
 
@@ -382,8 +382,6 @@ static int wm8505fb_remove(struct platform_device *pdev)
 
 	if (fbi->fb.cmap.len)
 		fb_dealloc_cmap(&fbi->fb.cmap);
-
-	return 0;
 }
 
 static const struct of_device_id wmt_dt_ids[] = {
@@ -393,7 +391,7 @@ static const struct of_device_id wmt_dt_ids[] = {
 
 static struct platform_driver wm8505fb_driver = {
 	.probe		= wm8505fb_probe,
-	.remove		= wm8505fb_remove,
+	.remove_new	= wm8505fb_remove,
 	.driver		= {
 		.name	= DRIVER_NAME,
 		.of_match_table = wmt_dt_ids,
@@ -405,5 +403,4 @@ module_platform_driver(wm8505fb_driver);
 
 MODULE_AUTHOR("Ed Spiridonov <edo.rus@gmail.com>");
 MODULE_DESCRIPTION("Framebuffer driver for WMT WM8505");
-MODULE_LICENSE("GPL v2");
 MODULE_DEVICE_TABLE(of, wmt_dt_ids);
diff --git a/drivers/video/fbdev/wmt_ge_rops.c b/drivers/video/fbdev/wmt_ge_rops.c
index 42255d27a1db..3ed143457d22 100644
--- a/drivers/video/fbdev/wmt_ge_rops.c
+++ b/drivers/video/fbdev/wmt_ge_rops.c
@@ -145,10 +145,9 @@ static int wmt_ge_rops_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int wmt_ge_rops_remove(struct platform_device *pdev)
+static void wmt_ge_rops_remove(struct platform_device *pdev)
 {
 	iounmap(regbase);
-	return 0;
 }
 
 static const struct of_device_id wmt_dt_ids[] = {
@@ -158,7 +157,7 @@ static const struct of_device_id wmt_dt_ids[] = {
 
 static struct platform_driver wmt_ge_rops_driver = {
 	.probe		= wmt_ge_rops_probe,
-	.remove		= wmt_ge_rops_remove,
+	.remove_new	= wmt_ge_rops_remove,
 	.driver		= {
 		.name	= "wmt_ge_rops",
 		.of_match_table = wmt_dt_ids,
@@ -170,5 +169,4 @@ module_platform_driver(wmt_ge_rops_driver);
 MODULE_AUTHOR("Alexey Charkov <alchark@gmail.com>");
 MODULE_DESCRIPTION("Accelerators for raster operations using "
 		   "WonderMedia Graphics Engine");
-MODULE_LICENSE("GPL v2");
 MODULE_DEVICE_TABLE(of, wmt_dt_ids);
diff --git a/drivers/video/fbdev/xilinxfb.c b/drivers/video/fbdev/xilinxfb.c
index 7911354827dc..2aa3a528277f 100644
--- a/drivers/video/fbdev/xilinxfb.c
+++ b/drivers/video/fbdev/xilinxfb.c
@@ -474,11 +474,9 @@ static int xilinxfb_of_probe(struct platform_device *pdev)
 	return xilinxfb_assign(pdev, drvdata, &pdata);
 }
 
-static int xilinxfb_of_remove(struct platform_device *op)
+static void xilinxfb_of_remove(struct platform_device *op)
 {
 	xilinxfb_release(&op->dev);
-
-	return 0;
 }
 
 /* Match table for of_platform binding */
@@ -494,7 +492,7 @@ MODULE_DEVICE_TABLE(of, xilinxfb_of_match);
 
 static struct platform_driver xilinxfb_of_driver = {
 	.probe = xilinxfb_of_probe,
-	.remove = xilinxfb_of_remove,
+	.remove_new = xilinxfb_of_remove,
 	.driver = {
 		.name = DRIVER_NAME,
 		.of_match_table = xilinxfb_of_match,
diff --git a/drivers/virt/fsl_hypervisor.c b/drivers/virt/fsl_hypervisor.c
index f8b4389d60d9..e92e2ceb12a4 100644
--- a/drivers/virt/fsl_hypervisor.c
+++ b/drivers/virt/fsl_hypervisor.c
@@ -796,7 +796,7 @@ static int has_fsl_hypervisor(void)
 	if (!node)
 		return 0;
 
-	ret = of_find_property(node, "fsl,hv-version", NULL) != NULL;
+	ret = of_property_present(node, "fsl,hv-version");
 
 	of_node_put(node);
 
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index 3ff746e3f24a..a46a4a29e929 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -61,6 +61,7 @@
 #include <linux/io.h>
 #include <linux/list.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm.h>
 #include <linux/slab.h>
@@ -285,6 +286,16 @@ static bool vm_notify(struct virtqueue *vq)
 	return true;
 }
 
+static bool vm_notify_with_data(struct virtqueue *vq)
+{
+	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);
+	u32 data = vring_notification_data(vq);
+
+	writel(data, vm_dev->base + VIRTIO_MMIO_QUEUE_NOTIFY);
+
+	return true;
+}
+
 /* Notify all virtqueues on an interrupt. */
 static irqreturn_t vm_interrupt(int irq, void *opaque)
 {
@@ -363,12 +374,18 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in
 				  const char *name, bool ctx)
 {
 	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
+	bool (*notify)(struct virtqueue *vq);
 	struct virtio_mmio_vq_info *info;
 	struct virtqueue *vq;
 	unsigned long flags;
 	unsigned int num;
 	int err;
 
+	if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA))
+		notify = vm_notify_with_data;
+	else
+		notify = vm_notify;
+
 	if (!name)
 		return NULL;
 
@@ -397,7 +414,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in
 
 	/* Create the vring */
 	vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev,
-				 true, true, ctx, vm_notify, callback, name);
+				 true, true, ctx, notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
index 9e496e288cfa..d6bb68ba84e5 100644
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -288,6 +288,15 @@ static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
 	return vp_modern_config_vector(&vp_dev->mdev, vector);
 }
 
+static bool vp_notify_with_data(struct virtqueue *vq)
+{
+	u32 data = vring_notification_data(vq);
+
+	iowrite32(data, (void __iomem *)vq->priv);
+
+	return true;
+}
+
 static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
 				  struct virtio_pci_vq_info *info,
 				  unsigned int index,
@@ -298,10 +307,16 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
 {
 
 	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
+	bool (*notify)(struct virtqueue *vq);
 	struct virtqueue *vq;
 	u16 num;
 	int err;
 
+	if (__virtio_test_bit(&vp_dev->vdev, VIRTIO_F_NOTIFICATION_DATA))
+		notify = vp_notify_with_data;
+	else
+		notify = vp_notify;
+
 	if (index >= vp_modern_get_num_queues(mdev))
 		return ERR_PTR(-EINVAL);
 
@@ -310,18 +325,13 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
 	if (!num || vp_modern_get_queue_enable(mdev, index))
 		return ERR_PTR(-ENOENT);
 
-	if (!is_power_of_2(num)) {
-		dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num);
-		return ERR_PTR(-EINVAL);
-	}
-
 	info->msix_vector = msix_vec;
 
 	/* create the vring */
 	vq = vring_create_virtqueue(index, num,
 				    SMP_CACHE_BYTES, &vp_dev->vdev,
 				    true, true, ctx,
-				    vp_notify, callback, name);
+				    notify, callback, name);
 	if (!vq)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 41144b5246a8..c5310eaf8b46 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -231,10 +231,10 @@ static void vring_free(struct virtqueue *_vq);
  * Helpers.
  */
 
-#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
+#define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq)
 
-static inline bool virtqueue_use_indirect(struct vring_virtqueue *vq,
-					  unsigned int total_sg)
+static bool virtqueue_use_indirect(const struct vring_virtqueue *vq,
+				   unsigned int total_sg)
 {
 	/*
 	 * If the host supports indirect descriptor tables, and we have multiple
@@ -269,7 +269,7 @@ static inline bool virtqueue_use_indirect(struct vring_virtqueue *vq,
  * unconditionally on data path.
  */
 
-static bool vring_use_dma_api(struct virtio_device *vdev)
+static bool vring_use_dma_api(const struct virtio_device *vdev)
 {
 	if (!virtio_has_dma_quirk(vdev))
 		return true;
@@ -289,7 +289,7 @@ static bool vring_use_dma_api(struct virtio_device *vdev)
 	return false;
 }
 
-size_t virtio_max_dma_size(struct virtio_device *vdev)
+size_t virtio_max_dma_size(const struct virtio_device *vdev)
 {
 	size_t max_segment_size = SIZE_MAX;
 
@@ -349,7 +349,7 @@ static void vring_free_queue(struct virtio_device *vdev, size_t size,
  * making all of the arch DMA ops work on the vring device itself
  * is a mess.
  */
-static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
+static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
 {
 	return vq->dma_dev;
 }
@@ -423,7 +423,7 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
  */
 
 static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq,
-					   struct vring_desc *desc)
+					   const struct vring_desc *desc)
 {
 	u16 flags;
 
@@ -784,7 +784,7 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
 	}
 }
 
-static inline bool more_used_split(const struct vring_virtqueue *vq)
+static bool more_used_split(const struct vring_virtqueue *vq)
 {
 	return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
 			vq->split.vring.used->idx);
@@ -854,6 +854,14 @@ static void virtqueue_disable_cb_split(struct virtqueue *_vq)
 
 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
+
+		/*
+		 * If device triggered an event already it won't trigger one again:
+		 * no need to disable.
+		 */
+		if (vq->event_triggered)
+			return;
+
 		if (vq->event)
 			/* TODO: this is a hack. Figure out a cleaner value to write. */
 			vring_used_event(&vq->split.vring) = 0x0;
@@ -1172,18 +1180,18 @@ err:
 /*
  * Packed ring specific functions - *_packed().
  */
-static inline bool packed_used_wrap_counter(u16 last_used_idx)
+static bool packed_used_wrap_counter(u16 last_used_idx)
 {
 	return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR));
 }
 
-static inline u16 packed_last_used(u16 last_used_idx)
+static u16 packed_last_used(u16 last_used_idx)
 {
 	return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR));
 }
 
 static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
-				     struct vring_desc_extra *extra)
+				     const struct vring_desc_extra *extra)
 {
 	u16 flags;
 
@@ -1206,7 +1214,7 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
 }
 
 static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
-				   struct vring_packed_desc *desc)
+				    const struct vring_packed_desc *desc)
 {
 	u16 flags;
 
@@ -1612,7 +1620,7 @@ static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
 	return avail == used && used == used_wrap_counter;
 }
 
-static inline bool more_used_packed(const struct vring_virtqueue *vq)
+static bool more_used_packed(const struct vring_virtqueue *vq)
 {
 	u16 last_used;
 	u16 last_used_idx;
@@ -1699,6 +1707,14 @@ static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
 
 	if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
 		vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
+
+		/*
+		 * If device triggered an event already it won't trigger one again:
+		 * no need to disable.
+		 */
+		if (vq->event_triggered)
+			return;
+
 		vq->packed.vring.driver->flags =
 			cpu_to_le16(vq->packed.event_flags_shadow);
 	}
@@ -2330,12 +2346,6 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	/* If device triggered an event already it won't trigger one again:
-	 * no need to disable.
-	 */
-	if (vq->event_triggered)
-		return;
-
 	if (vq->packed_ring)
 		virtqueue_disable_cb_packed(_vq);
 	else
@@ -2752,6 +2762,23 @@ void vring_del_virtqueue(struct virtqueue *_vq)
 }
 EXPORT_SYMBOL_GPL(vring_del_virtqueue);
 
+u32 vring_notification_data(struct virtqueue *_vq)
+{
+	struct vring_virtqueue *vq = to_vvq(_vq);
+	u16 next;
+
+	if (vq->packed_ring)
+		next = (vq->packed.next_avail_idx &
+				~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) |
+			vq->packed.avail_wrap_counter <<
+				VRING_PACKED_EVENT_F_WRAP_CTR;
+	else
+		next = vq->split.avail_idx_shadow;
+
+	return next << 16 | _vq->index;
+}
+EXPORT_SYMBOL_GPL(vring_notification_data);
+
 /* Manipulates transport-specific feature bits. */
 void vring_transport_features(struct virtio_device *vdev)
 {
@@ -2771,6 +2798,8 @@ void vring_transport_features(struct virtio_device *vdev)
 			break;
 		case VIRTIO_F_ORDER_PLATFORM:
 			break;
+		case VIRTIO_F_NOTIFICATION_DATA:
+			break;
 		default:
 			/* We don't understand this bit. */
 			__virtio_clear_bit(vdev, i);
@@ -2786,10 +2815,10 @@ EXPORT_SYMBOL_GPL(vring_transport_features);
  * Returns the size of the vring.  This is mainly used for boasting to
  * userspace.  Unlike other operations, this need not be serialized.
  */
-unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
+unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq)
 {
 
-	struct vring_virtqueue *vq = to_vvq(_vq);
+	const struct vring_virtqueue *vq = to_vvq(_vq);
 
 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
 }
@@ -2819,9 +2848,9 @@ void __virtqueue_unbreak(struct virtqueue *_vq)
 }
 EXPORT_SYMBOL_GPL(__virtqueue_unbreak);
 
-bool virtqueue_is_broken(struct virtqueue *_vq)
+bool virtqueue_is_broken(const struct virtqueue *_vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
+	const struct vring_virtqueue *vq = to_vvq(_vq);
 
 	return READ_ONCE(vq->broken);
 }
@@ -2868,9 +2897,9 @@ void __virtio_unbreak_device(struct virtio_device *dev)
 }
 EXPORT_SYMBOL_GPL(__virtio_unbreak_device);
 
-dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
+dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
+	const struct vring_virtqueue *vq = to_vvq(_vq);
 
 	BUG_ON(!vq->we_own_ring);
 
@@ -2881,9 +2910,9 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
 }
 EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
 
-dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
+dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
+	const struct vring_virtqueue *vq = to_vvq(_vq);
 
 	BUG_ON(!vq->we_own_ring);
 
@@ -2895,9 +2924,9 @@ dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
 }
 EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
 
-dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
+dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
+	const struct vring_virtqueue *vq = to_vvq(_vq);
 
 	BUG_ON(!vq->we_own_ring);
 
@@ -2910,7 +2939,7 @@ dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
 EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
 
 /* Only available for split ring */
-const struct vring *virtqueue_get_vring(struct virtqueue *vq)
+const struct vring *virtqueue_get_vring(const struct virtqueue *vq)
 {
 	return &to_vvq(vq)->split.vring;
 }
diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
index d7f5af62ddaa..eb6aee8c06b2 100644
--- a/drivers/virtio/virtio_vdpa.c
+++ b/drivers/virtio/virtio_vdpa.c
@@ -13,6 +13,7 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/uuid.h>
+#include <linux/group_cpus.h>
 #include <linux/virtio.h>
 #include <linux/vdpa.h>
 #include <linux/virtio_config.h>
@@ -112,6 +113,17 @@ static bool virtio_vdpa_notify(struct virtqueue *vq)
 	return true;
 }
 
+static bool virtio_vdpa_notify_with_data(struct virtqueue *vq)
+{
+	struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
+	const struct vdpa_config_ops *ops = vdpa->config;
+	u32 data = vring_notification_data(vq);
+
+	ops->kick_vq_with_data(vdpa, data);
+
+	return true;
+}
+
 static irqreturn_t virtio_vdpa_config_cb(void *private)
 {
 	struct virtio_vdpa_device *vd_dev = private;
@@ -138,6 +150,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
 	struct device *dma_dev;
 	const struct vdpa_config_ops *ops = vdpa->config;
 	struct virtio_vdpa_vq_info *info;
+	bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify;
 	struct vdpa_callback cb;
 	struct virtqueue *vq;
 	u64 desc_addr, driver_addr, device_addr;
@@ -154,6 +167,14 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
 	if (index >= vdpa->nvqs)
 		return ERR_PTR(-ENOENT);
 
+	/* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */
+	if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) {
+		if (ops->kick_vq_with_data)
+			notify = virtio_vdpa_notify_with_data;
+		else
+			__virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA);
+	}
+
 	/* Queue shouldn't already be set up. */
 	if (ops->get_vq_ready(vdpa, index))
 		return ERR_PTR(-ENOENT);
@@ -183,8 +204,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
 		dma_dev = vdpa_get_dma_dev(vdpa);
 	vq = vring_create_virtqueue_dma(index, max_num, align, vdev,
 					true, may_reduce_num, ctx,
-					virtio_vdpa_notify, callback,
-					name, dma_dev);
+					notify, callback, name, dma_dev);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
@@ -195,6 +215,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
 	/* Setup virtqueue callback */
 	cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
 	cb.private = info;
+	cb.trigger = NULL;
 	ops->set_vq_cb(vdpa, index, &cb);
 	ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
 
@@ -272,6 +293,66 @@ static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
 		virtio_vdpa_del_vq(vq);
 }
 
+static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs)
+{
+	affd->nr_sets = 1;
+	affd->set_size[0] = affvecs;
+}
+
+static struct cpumask *
+create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)
+{
+	unsigned int affvecs = 0, curvec, usedvecs, i;
+	struct cpumask *masks = NULL;
+
+	if (nvecs > affd->pre_vectors + affd->post_vectors)
+		affvecs = nvecs - affd->pre_vectors - affd->post_vectors;
+
+	if (!affd->calc_sets)
+		affd->calc_sets = default_calc_sets;
+
+	affd->calc_sets(affd, affvecs);
+
+	if (!affvecs)
+		return NULL;
+
+	masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
+	if (!masks)
+		return NULL;
+
+	/* Fill out vectors at the beginning that don't need affinity */
+	for (curvec = 0; curvec < affd->pre_vectors; curvec++)
+		cpumask_setall(&masks[curvec]);
+
+	for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) {
+		unsigned int this_vecs = affd->set_size[i];
+		int j;
+		struct cpumask *result = group_cpus_evenly(this_vecs);
+
+		if (!result) {
+			kfree(masks);
+			return NULL;
+		}
+
+		for (j = 0; j < this_vecs; j++)
+			cpumask_copy(&masks[curvec + j], &result[j]);
+		kfree(result);
+
+		curvec += this_vecs;
+		usedvecs += this_vecs;
+	}
+
+	/* Fill out vectors at the end that don't need affinity */
+	if (usedvecs >= affvecs)
+		curvec = affd->pre_vectors + affvecs;
+	else
+		curvec = affd->pre_vectors + usedvecs;
+	for (; curvec < nvecs; curvec++)
+		cpumask_setall(&masks[curvec]);
+
+	return masks;
+}
+
 static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
 				struct virtqueue *vqs[],
 				vq_callback_t *callbacks[],
@@ -282,9 +363,15 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
 	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
 	const struct vdpa_config_ops *ops = vdpa->config;
+	struct irq_affinity default_affd = { 0 };
+	struct cpumask *masks;
 	struct vdpa_callback cb;
 	int i, err, queue_idx = 0;
 
+	masks = create_affinity_masks(nvqs, desc ? desc : &default_affd);
+	if (!masks)
+		return -ENOMEM;
+
 	for (i = 0; i < nvqs; ++i) {
 		if (!names[i]) {
 			vqs[i] = NULL;
@@ -298,6 +385,7 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
 			err = PTR_ERR(vqs[i]);
 			goto err_setup_vq;
 		}
+		ops->set_vq_affinity(vdpa, i, &masks[i]);
 	}
 
 	cb.callback = virtio_vdpa_config_cb;
@@ -337,6 +425,32 @@ static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
 	return dev_name(&vdpa->dev);
 }
 
+static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq,
+				       const struct cpumask *cpu_mask)
+{
+	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
+	struct vdpa_device *vdpa = vd_dev->vdpa;
+	const struct vdpa_config_ops *ops = vdpa->config;
+	unsigned int index = vq->index;
+
+	if (ops->set_vq_affinity)
+		return ops->set_vq_affinity(vdpa, index, cpu_mask);
+
+	return 0;
+}
+
+static const struct cpumask *
+virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index)
+{
+	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
+	const struct vdpa_config_ops *ops = vdpa->config;
+
+	if (ops->get_vq_affinity)
+		return ops->get_vq_affinity(vdpa, index);
+
+	return NULL;
+}
+
 static const struct virtio_config_ops virtio_vdpa_config_ops = {
 	.get		= virtio_vdpa_get,
 	.set		= virtio_vdpa_set,
@@ -349,6 +463,8 @@ static const struct virtio_config_ops virtio_vdpa_config_ops = {
 	.get_features	= virtio_vdpa_get_features,
 	.finalize_features = virtio_vdpa_finalize_features,
 	.bus_name	= virtio_vdpa_bus_name,
+	.set_vq_affinity = virtio_vdpa_set_vq_affinity,
+	.get_vq_affinity = virtio_vdpa_get_vq_affinity,
 };
 
 static void virtio_vdpa_release_dev(struct device *_d)
diff --git a/drivers/w1/masters/Kconfig b/drivers/w1/masters/Kconfig
index 692cac3ff0ee..ad316573288a 100644
--- a/drivers/w1/masters/Kconfig
+++ b/drivers/w1/masters/Kconfig
@@ -53,7 +53,7 @@ config W1_MASTER_GPIO
 
 config HDQ_MASTER_OMAP
 	tristate "OMAP HDQ driver"
-	depends on ARCH_OMAP
+	depends on ARCH_OMAP || COMPILE_TEST
 	help
 	  Say Y here if you want support for the 1-wire or HDQ Interface
 	  on an OMAP processor.
diff --git a/drivers/w1/masters/ds2482.c b/drivers/w1/masters/ds2482.c
index 3d8b51316bef..c1de8a92e144 100644
--- a/drivers/w1/masters/ds2482.c
+++ b/drivers/w1/masters/ds2482.c
@@ -15,7 +15,6 @@
 #include <linux/slab.h>
 #include <linux/i2c.h>
 #include <linux/delay.h>
-#include <asm/delay.h>
 
 #include <linux/w1.h>
 
@@ -36,7 +35,7 @@ MODULE_PARM_DESC(active_pullup, "Active pullup (apply to all buses): " \
 
 /* extra configurations - e.g. 1WS */
 static int extra_config;
-module_param(extra_config, int, S_IRUGO | S_IWUSR);
+module_param(extra_config, int, 0644);
 MODULE_PARM_DESC(extra_config, "Extra Configuration settings 1=APU,2=PPM,3=SPU,8=1WS");
 
 /*
@@ -78,10 +77,8 @@ MODULE_PARM_DESC(extra_config, "Extra Configuration settings 1=APU,2=PPM,3=SPU,8
  * To set the channel, write the value at the index of the channel.
  * Read and compare against the corresponding value to verify the change.
  */
-static const u8 ds2482_chan_wr[8] =
-	{ 0xF0, 0xE1, 0xD2, 0xC3, 0xB4, 0xA5, 0x96, 0x87 };
-static const u8 ds2482_chan_rd[8] =
-	{ 0xB8, 0xB1, 0xAA, 0xA3, 0x9C, 0x95, 0x8E, 0x87 };
+static const u8 ds2482_chan_wr[8] = { 0xF0, 0xE1, 0xD2, 0xC3, 0xB4, 0xA5, 0x96, 0x87 };
+static const u8 ds2482_chan_rd[8] = { 0xB8, 0xB1, 0xAA, 0xA3, 0x9C, 0x95, 0x8E, 0x87 };
 
 
 /*
@@ -454,7 +451,8 @@ static int ds2482_probe(struct i2c_client *client)
 				     I2C_FUNC_SMBUS_BYTE))
 		return -ENODEV;
 
-	if (!(data = kzalloc(sizeof(struct ds2482_data), GFP_KERNEL))) {
+	data = kzalloc(sizeof(struct ds2482_data), GFP_KERNEL);
+	if (!data) {
 		err = -ENOMEM;
 		goto exit;
 	}
@@ -544,6 +542,7 @@ static void ds2482_remove(struct i2c_client *client)
  */
 static const struct i2c_device_id ds2482_id[] = {
 	{ "ds2482", 0 },
+	{ "ds2484", 0 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ds2482_id);
diff --git a/drivers/w1/masters/ds2490.c b/drivers/w1/masters/ds2490.c
index 0eb560fc0153..5f5b97e24700 100644
--- a/drivers/w1/masters/ds2490.c
+++ b/drivers/w1/masters/ds2490.c
@@ -304,6 +304,7 @@ static void ds_reset_device(struct ds_device *dev)
 	if (dev->spu_sleep) {
 		/* lower 4 bits are 0, see ds_set_pullup */
 		u8 del = dev->spu_sleep>>4;
+
 		if (ds_send_control(dev, COMM_SET_DURATION | COMM_IM, del))
 			dev_err(&dev->udev->dev,
 				"%s: Error setting duration\n", __func__);
@@ -731,7 +732,8 @@ static void ds9490r_search(void *data, struct w1_master *master,
 			break;
 
 		if (st.data_in_buffer_status) {
-			/* Bulk in can receive partial ids, but when it does
+			/*
+			 * Bulk in can receive partial ids, but when it does
 			 * they fail crc and will be discarded anyway.
 			 * That has only been seen when status in buffer
 			 * is 0 and bulk is read anyway, so don't read
@@ -743,8 +745,10 @@ static void ds9490r_search(void *data, struct w1_master *master,
 				break;
 			for (i = 0; i < err/8; ++i) {
 				found_ids[found++] = buf[i];
-				/* can't know if there will be a discrepancy
-				 * value after until the next id */
+				/*
+				 * can't know if there will be a discrepancy
+				 * value after until the next id
+				 */
 				if (found == search_limit) {
 					master->search_id = buf[i];
 					break;
@@ -760,7 +764,8 @@ static void ds9490r_search(void *data, struct w1_master *master,
 	if (found <= search_limit) {
 		master->search_id = 0;
 	} else if (!test_bit(W1_WARN_MAX_COUNT, &master->flags)) {
-		/* Only max_slave_count will be scanned in a search,
+		/*
+		 * Only max_slave_count will be scanned in a search,
 		 * but it will start where it left off next search
 		 * until all ids are identified and then it will start
 		 * over.  A continued search will report the previous
diff --git a/drivers/w1/masters/matrox_w1.c b/drivers/w1/masters/matrox_w1.c
index ee716c715710..2852cd2dc67c 100644
--- a/drivers/w1/masters/matrox_w1.c
+++ b/drivers/w1/masters/matrox_w1.c
@@ -7,7 +7,7 @@
 
 #include <asm/types.h>
 #include <linux/atomic.h>
-#include <asm/io.h>
+#include <linux/io.h>
 
 #include <linux/delay.h>
 #include <linux/kernel.h>
@@ -39,8 +39,7 @@
 #define MATROX_GET_DATA			0x2B
 #define MATROX_CURSOR_CTL		0x06
 
-struct matrox_device
-{
+struct matrox_device {
 	void __iomem *base_addr;
 	void __iomem *port_index;
 	void __iomem *port_data;
@@ -64,7 +63,7 @@ struct matrox_device
  *
  * Port mapping.
  */
-static __inline__ u8 matrox_w1_read_reg(struct matrox_device *dev, u8 reg)
+static inline u8 matrox_w1_read_reg(struct matrox_device *dev, u8 reg)
 {
 	u8 ret;
 
@@ -75,7 +74,7 @@ static __inline__ u8 matrox_w1_read_reg(struct matrox_device *dev, u8 reg)
 	return ret;
 }
 
-static __inline__ void matrox_w1_write_reg(struct matrox_device *dev, u8 reg, u8 val)
+static inline void matrox_w1_write_reg(struct matrox_device *dev, u8 reg, u8 val)
 {
 	writeb(reg, dev->port_index);
 	writeb(val, dev->port_data);
@@ -123,13 +122,8 @@ static int matrox_w1_probe(struct pci_dev *pdev, const struct pci_device_id *ent
 
 	dev = kzalloc(sizeof(struct matrox_device) +
 		       sizeof(struct w1_bus_master), GFP_KERNEL);
-	if (!dev) {
-		dev_err(&pdev->dev,
-			"%s: Failed to create new matrox_device object.\n",
-			__func__);
+	if (!dev)
 		return -ENOMEM;
-	}
-
 
 	dev->bus_master = (struct w1_bus_master *)(dev + 1);
 
diff --git a/drivers/w1/masters/omap_hdq.c b/drivers/w1/masters/omap_hdq.c
index bf2ec59c1f9d..6a39b71eb718 100644
--- a/drivers/w1/masters/omap_hdq.c
+++ b/drivers/w1/masters/omap_hdq.c
@@ -1,12 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
- * drivers/w1/masters/omap_hdq.c
- *
  * Copyright (C) 2007,2012 Texas Instruments, Inc.
- *
- * This file is licensed under the terms of the GNU General Public License
- * version 2. This program is licensed "as is" without any warranty of any
- * kind, whether express or implied.
- *
  */
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -48,7 +42,7 @@
 static DECLARE_WAIT_QUEUE_HEAD(hdq_wait_queue);
 
 static int w1_id;
-module_param(w1_id, int, S_IRUSR);
+module_param(w1_id, int, 0400);
 MODULE_PARM_DESC(w1_id, "1-wire id for the slave detection in HDQ mode");
 
 struct hdq_data {
@@ -579,10 +573,8 @@ static int omap_hdq_probe(struct platform_device *pdev)
 	const char *mode;
 
 	hdq_data = devm_kzalloc(dev, sizeof(*hdq_data), GFP_KERNEL);
-	if (!hdq_data) {
-		dev_dbg(&pdev->dev, "unable to allocate memory\n");
+	if (!hdq_data)
 		return -ENOMEM;
-	}
 
 	hdq_data->dev = dev;
 	platform_set_drvdata(pdev, hdq_data);
diff --git a/drivers/w1/masters/w1-gpio.c b/drivers/w1/masters/w1-gpio.c
index d4632aace402..e45acb6d916e 100644
--- a/drivers/w1/masters/w1-gpio.c
+++ b/drivers/w1/masters/w1-gpio.c
@@ -87,7 +87,7 @@ static int w1_gpio_probe(struct platform_device *pdev)
 		 * driver it high/low like we are in full control of the line and
 		 * open drain will happen transparently.
 		 */
-		if (of_get_property(np, "linux,open-drain", NULL))
+		if (of_property_present(np, "linux,open-drain"))
 			gflags = GPIOD_OUT_LOW;
 
 		pdev->dev.platform_data = pdata;
@@ -101,10 +101,8 @@ static int w1_gpio_probe(struct platform_device *pdev)
 
 	master = devm_kzalloc(dev, sizeof(struct w1_bus_master),
 			GFP_KERNEL);
-	if (!master) {
-		dev_err(dev, "Out of memory\n");
+	if (!master)
 		return -ENOMEM;
-	}
 
 	pdata->gpiod = devm_gpiod_get_index(dev, NULL, 0, gflags);
 	if (IS_ERR(pdata->gpiod)) {
diff --git a/drivers/w1/slaves/w1_ds2406.c b/drivers/w1/slaves/w1_ds2406.c
index 6c269af73c80..2f5926859b8b 100644
--- a/drivers/w1/slaves/w1_ds2406.c
+++ b/drivers/w1/slaves/w1_ds2406.c
@@ -27,11 +27,11 @@ static ssize_t w1_f12_read_state(
 	struct bin_attribute *bin_attr,
 	char *buf, loff_t off, size_t count)
 {
-	u8 w1_buf[6]={W1_F12_FUNC_READ_STATUS, 7, 0, 0, 0, 0};
+	u8 w1_buf[6] = {W1_F12_FUNC_READ_STATUS, 7, 0, 0, 0, 0};
 	struct w1_slave *sl = kobj_to_w1_slave(kobj);
-	u16 crc=0;
+	u16 crc = 0;
 	int i;
-	ssize_t rtnval=1;
+	ssize_t rtnval = 1;
 
 	if (off != 0)
 		return 0;
@@ -47,12 +47,12 @@ static ssize_t w1_f12_read_state(
 
 	w1_write_block(sl->master, w1_buf, 3);
 	w1_read_block(sl->master, w1_buf+3, 3);
-	for (i=0; i<6; i++)
-		crc=crc16_byte(crc, w1_buf[i]);
-	if (crc==0xb001) /* good read? */
-		*buf=((w1_buf[3]>>5)&3)|0x30;
+	for (i = 0; i < 6; i++)
+		crc = crc16_byte(crc, w1_buf[i]);
+	if (crc == 0xb001) /* good read? */
+		*buf = ((w1_buf[3]>>5)&3)|0x30;
 	else
-		rtnval=-EIO;
+		rtnval = -EIO;
 
 	mutex_unlock(&sl->master->bus_mutex);
 
@@ -65,10 +65,10 @@ static ssize_t w1_f12_write_output(
 	char *buf, loff_t off, size_t count)
 {
 	struct w1_slave *sl = kobj_to_w1_slave(kobj);
-	u8 w1_buf[6]={W1_F12_FUNC_WRITE_STATUS, 7, 0, 0, 0, 0};
-	u16 crc=0;
+	u8 w1_buf[6] = {W1_F12_FUNC_WRITE_STATUS, 7, 0, 0, 0, 0};
+	u16 crc = 0;
 	int i;
-	ssize_t rtnval=1;
+	ssize_t rtnval = 1;
 
 	if (count != 1 || off != 0)
 		return -EFAULT;
@@ -83,12 +83,12 @@ static ssize_t w1_f12_write_output(
 	w1_buf[3] = (((*buf)&3)<<5)|0x1F;
 	w1_write_block(sl->master, w1_buf, 4);
 	w1_read_block(sl->master, w1_buf+4, 2);
-	for (i=0; i<6; i++)
-		crc=crc16_byte(crc, w1_buf[i]);
-	if (crc==0xb001) /* good read? */
+	for (i = 0; i < 6; i++)
+		crc = crc16_byte(crc, w1_buf[i]);
+	if (crc == 0xb001) /* good read? */
 		w1_write_8(sl->master, 0xFF);
 	else
-		rtnval=-EIO;
+		rtnval = -EIO;
 
 	mutex_unlock(&sl->master->bus_mutex);
 	return rtnval;
@@ -99,7 +99,7 @@ static struct bin_attribute w1_f12_sysfs_bin_files[NB_SYSFS_BIN_FILES] = {
 	{
 		.attr = {
 			.name = "state",
-			.mode = S_IRUGO,
+			.mode = 0444,
 		},
 		.size = 1,
 		.read = w1_f12_read_state,
@@ -107,7 +107,7 @@ static struct bin_attribute w1_f12_sysfs_bin_files[NB_SYSFS_BIN_FILES] = {
 	{
 		.attr = {
 			.name = "output",
-			.mode = S_IRUGO | S_IWUSR | S_IWGRP,
+			.mode = 0664,
 		},
 		.size = 1,
 		.write = w1_f12_write_output,
@@ -133,6 +133,7 @@ static int w1_f12_add_slave(struct w1_slave *sl)
 static void w1_f12_remove_slave(struct w1_slave *sl)
 {
 	int i;
+
 	for (i = NB_SYSFS_BIN_FILES - 1; i >= 0; --i)
 		sysfs_remove_bin_file(&sl->dev.kobj,
 			&(w1_f12_sysfs_bin_files[i]));
diff --git a/drivers/w1/slaves/w1_ds2408.c b/drivers/w1/slaves/w1_ds2408.c
index ad102c577122..56f822a1dfdb 100644
--- a/drivers/w1/slaves/w1_ds2408.c
+++ b/drivers/w1/slaves/w1_ds2408.c
@@ -35,12 +35,12 @@
 
 #define W1_F29_SUCCESS_CONFIRM_BYTE        0xAA
 
-static int _read_reg(struct w1_slave *sl, u8 address, unsigned char* buf)
+static int _read_reg(struct w1_slave *sl, u8 address, unsigned char *buf)
 {
 	u8 wrbuf[3];
-	dev_dbg(&sl->dev,
-			"Reading with slave: %p, reg addr: %0#4x, buff addr: %p",
-			sl, (unsigned int)address, buf);
+
+	dev_dbg(&sl->dev, "Reading with slave: %p, reg addr: %0#4x, buff addr: %p",
+		sl, (unsigned int)address, buf);
 
 	if (!buf)
 		return -EINVAL;
@@ -206,7 +206,7 @@ out:
 }
 
 
-/**
+/*
  * Writing to the activity file resets the activity latches.
  */
 static ssize_t activity_write(struct file *filp, struct kobject *kobj,
@@ -292,7 +292,7 @@ static int w1_f29_disable_test_mode(struct w1_slave *sl)
 {
 	int res;
 	u8 magic[10] = {0x96, };
-	u64 rn = le64_to_cpu(*((u64*)&sl->reg_num));
+	u64 rn = le64_to_cpu(*((u64 *)&sl->reg_num));
 
 	memcpy(&magic[1], &rn, 8);
 	magic[9] = 0x3C;
diff --git a/drivers/w1/slaves/w1_ds2413.c b/drivers/w1/slaves/w1_ds2413.c
index c8cfac555b48..739009806467 100644
--- a/drivers/w1/slaves/w1_ds2413.c
+++ b/drivers/w1/slaves/w1_ds2413.c
@@ -99,8 +99,10 @@ static ssize_t output_write(struct file *filp, struct kobject *kobj,
 	if (w1_reset_select_slave(sl))
 		goto out;
 
-	/* according to the DS2413 datasheet the most significant 6 bits
-	   should be set to "1"s, so do it now */
+	/*
+	 * according to the DS2413 datasheet the most significant 6 bits
+	 * should be set to "1"s, so do it now
+	 */
 	*buf = *buf | 0xFC;
 
 	while (retries--) {
@@ -126,7 +128,7 @@ out:
 	return bytes_written;
 }
 
-static BIN_ATTR(output, S_IRUGO | S_IWUSR | S_IWGRP, NULL, output_write, 1);
+static BIN_ATTR(output, 0664, NULL, output_write, 1);
 
 static struct bin_attribute *w1_f3a_bin_attrs[] = {
 	&bin_attr_state,
diff --git a/drivers/w1/slaves/w1_ds2433.c b/drivers/w1/slaves/w1_ds2433.c
index 0f72df15a024..9f21fd98f799 100644
--- a/drivers/w1/slaves/w1_ds2433.c
+++ b/drivers/w1/slaves/w1_ds2433.c
@@ -42,7 +42,7 @@ struct w1_f23_data {
 	u32	validcrc;
 };
 
-/**
+/*
  * Check the file size bounds and adjusts count as needed.
  * This would not be needed if the file size didn't reset to 0 after a write.
  */
@@ -98,7 +98,8 @@ static ssize_t eeprom_read(struct file *filp, struct kobject *kobj,
 	u8 wrbuf[3];
 #endif
 
-	if ((count = w1_f23_fix_count(off, count, W1_EEPROM_SIZE)) == 0)
+	count = w1_f23_fix_count(off, count, W1_EEPROM_SIZE);
+	if (!count)
 		return 0;
 
 	mutex_lock(&sl->master->bus_mutex);
@@ -115,7 +116,7 @@ static ssize_t eeprom_read(struct file *filp, struct kobject *kobj,
 	}
 	memcpy(buf, &data->memory[off], count);
 
-#else 	/* CONFIG_W1_SLAVE_DS2433_CRC */
+#else	/* CONFIG_W1_SLAVE_DS2433_CRC */
 
 	/* read directly from the EEPROM */
 	if (w1_reset_select_slave(sl)) {
@@ -138,16 +139,17 @@ out_up:
 }
 
 /**
- * Writes to the scratchpad and reads it back for verification.
+ * w1_f23_write() - Writes to the scratchpad and reads it back for verification.
+ * @sl:		The slave structure
+ * @addr:	Address for the write
+ * @len:	length must be <= (W1_PAGE_SIZE - (addr & W1_PAGE_MASK))
+ * @data:	The data to write
+ *
  * Then copies the scratchpad to EEPROM.
  * The data must be on one page.
  * The master must be locked.
  *
- * @param sl	The slave structure
- * @param addr	Address for the write
- * @param len   length must be <= (W1_PAGE_SIZE - (addr & W1_PAGE_MASK))
- * @param data	The data to write
- * @return	0=Success -1=failure
+ * Return:	0=Success, -1=failure
  */
 static int w1_f23_write(struct w1_slave *sl, int addr, int len, const u8 *data)
 {
@@ -207,7 +209,8 @@ static ssize_t eeprom_write(struct file *filp, struct kobject *kobj,
 	struct w1_slave *sl = kobj_to_w1_slave(kobj);
 	int addr, len, idx;
 
-	if ((count = w1_f23_fix_count(off, count, W1_EEPROM_SIZE)) == 0)
+	count = w1_f23_fix_count(off, count, W1_EEPROM_SIZE);
+	if (!count)
 		return 0;
 
 #ifdef CONFIG_W1_SLAVE_DS2433_CRC
diff --git a/drivers/w1/slaves/w1_ds2780.c b/drivers/w1/slaves/w1_ds2780.c
index 9dcb5a54f7fc..3cde1bb1886b 100644
--- a/drivers/w1/slaves/w1_ds2780.c
+++ b/drivers/w1/slaves/w1_ds2780.c
@@ -91,6 +91,7 @@ static ssize_t w1_slave_read(struct file *filp, struct kobject *kobj,
 			     loff_t off, size_t count)
 {
 	struct device *dev = kobj_to_dev(kobj);
+
 	return w1_ds2780_io(dev, buf, off, count, 0);
 }
 
diff --git a/drivers/w1/slaves/w1_ds2781.c b/drivers/w1/slaves/w1_ds2781.c
index 2cb7c020b607..e418484b4a49 100644
--- a/drivers/w1/slaves/w1_ds2781.c
+++ b/drivers/w1/slaves/w1_ds2781.c
@@ -88,6 +88,7 @@ static ssize_t w1_slave_read(struct file *filp, struct kobject *kobj,
 			     loff_t off, size_t count)
 {
 	struct device *dev = kobj_to_dev(kobj);
+
 	return w1_ds2781_io(dev, buf, off, count, 0);
 }
 
diff --git a/drivers/w1/slaves/w1_ds2805.c b/drivers/w1/slaves/w1_ds2805.c
index 6b5d12ba1b65..4c1a2c515317 100644
--- a/drivers/w1/slaves/w1_ds2805.c
+++ b/drivers/w1/slaves/w1_ds2805.c
@@ -264,7 +264,7 @@ out_up:
 static struct bin_attribute w1_f0d_bin_attr = {
 	.attr = {
 		.name = "eeprom",
-		.mode = S_IRUGO | S_IWUSR,
+		.mode = 0644,
 	},
 	.size = W1_F0D_EEPROM_SIZE,
 	.read = w1_f0d_read_bin,
diff --git a/drivers/w1/slaves/w1_ds28e04.c b/drivers/w1/slaves/w1_ds28e04.c
index 6cef6e2edb89..2854b8b9e93f 100644
--- a/drivers/w1/slaves/w1_ds28e04.c
+++ b/drivers/w1/slaves/w1_ds28e04.c
@@ -53,7 +53,7 @@ struct w1_f1C_data {
 	u32	validcrc;
 };
 
-/**
+/*
  * Check the file size bounds and adjusts count as needed.
  * This would not be needed if the file size didn't reset to 0 after a write.
  */
@@ -146,16 +146,17 @@ out_up:
 }
 
 /**
- * Writes to the scratchpad and reads it back for verification.
+ * w1_f1C_write() - Writes to the scratchpad and reads it back for verification.
+ * @sl:		The slave structure
+ * @addr:	Address for the write
+ * @len:	length must be <= (W1_PAGE_SIZE - (addr & W1_PAGE_MASK))
+ * @data:	The data to write
+ *
  * Then copies the scratchpad to EEPROM.
  * The data must be on one page.
  * The master must be locked.
  *
- * @param sl	The slave structure
- * @param addr	Address for the write
- * @param len   length must be <= (W1_PAGE_SIZE - (addr & W1_PAGE_MASK))
- * @param data	The data to write
- * @return	0=Success -1=failure
+ * Return:	0=Success, -1=failure
  */
 static int w1_f1C_write(struct w1_slave *sl, int addr, int len, const u8 *data)
 {
@@ -197,8 +198,10 @@ static int w1_f1C_write(struct w1_slave *sl, int addr, int len, const u8 *data)
 	wrbuf[3] = es;
 
 	for (i = 0; i < sizeof(wrbuf); ++i) {
-		/* issue 10ms strong pullup (or delay) on the last byte
-		   for writing the data from the scratchpad to EEPROM */
+		/*
+		 * issue 10ms strong pullup (or delay) on the last byte
+		 * for writing the data from the scratchpad to EEPROM
+		 */
 		if (w1_strong_pullup && i == sizeof(wrbuf)-1)
 			w1_next_pullup(sl->master, tm);
 
diff --git a/drivers/w1/slaves/w1_ds28e17.c b/drivers/w1/slaves/w1_ds28e17.c
index aed10b72fc99..52261b54d842 100644
--- a/drivers/w1/slaves/w1_ds28e17.c
+++ b/drivers/w1/slaves/w1_ds28e17.c
@@ -31,12 +31,12 @@ MODULE_ALIAS("w1-family-" __stringify(W1_FAMILY_DS28E17));
 
 /* Default I2C speed to be set when a DS28E17 is detected. */
 static int i2c_speed = 100;
-module_param_named(speed, i2c_speed, int, (S_IRUSR | S_IWUSR));
+module_param_named(speed, i2c_speed, int, 0600);
 MODULE_PARM_DESC(speed, "Default I2C speed to be set when a DS28E17 is detected");
 
 /* Default I2C stretch value to be set when a DS28E17 is detected. */
 static char i2c_stretch = 1;
-module_param_named(stretch, i2c_stretch, byte, (S_IRUSR | S_IWUSR));
+module_param_named(stretch, i2c_stretch, byte, 0600);
 MODULE_PARM_DESC(stretch, "Default I2C stretch value to be set when a DS28E17 is detected");
 
 /* DS28E17 device command codes. */
@@ -59,7 +59,7 @@ MODULE_PARM_DESC(stretch, "Default I2C stretch value to be set when a DS28E17 is
 /*
  * Maximum number of I2C bytes to transfer within one CRC16 protected onewire
  * command.
- * */
+ */
 #define W1_F19_WRITE_DATA_LIMIT 255
 
 /* Maximum number of I2C bytes to read with one onewire command. */
diff --git a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c
index 0122e8796879..12a6f020b6b5 100644
--- a/drivers/watchdog/watchdog_dev.c
+++ b/drivers/watchdog/watchdog_dev.c
@@ -1005,7 +1005,6 @@ static struct miscdevice watchdog_miscdev = {
 
 static struct class watchdog_class = {
 	.name =		"watchdog",
-	.owner =	THIS_MODULE,
 	.dev_groups =	wdt_groups,
 };
 
diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index 617a7f4f07a8..586a1673459e 100644
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -97,24 +97,6 @@ static struct ctl_table balloon_table[] = {
 	{ }
 };
 
-static struct ctl_table balloon_root[] = {
-	{
-		.procname	= "balloon",
-		.mode		= 0555,
-		.child		= balloon_table,
-	},
-	{ }
-};
-
-static struct ctl_table xen_root[] = {
-	{
-		.procname	= "xen",
-		.mode		= 0555,
-		.child		= balloon_root,
-	},
-	{ }
-};
-
 #else
 #define xen_hotplug_unpopulated 0
 #endif
@@ -747,7 +729,7 @@ static int __init balloon_init(void)
 #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
 	set_online_page_callback(&xen_online_page);
 	register_memory_notifier(&xen_memory_nb);
-	register_sysctl_table(xen_root);
+	register_sysctl_init("xen/balloon", balloon_table);
 #endif
 
 	balloon_add_regions();
diff --git a/drivers/xen/pcpu.c b/drivers/xen/pcpu.c
index fd3a644b0855..b3e3d1bb37f3 100644
--- a/drivers/xen/pcpu.c
+++ b/drivers/xen/pcpu.c
@@ -58,6 +58,7 @@ struct pcpu {
 	struct list_head list;
 	struct device dev;
 	uint32_t cpu_id;
+	uint32_t acpi_id;
 	uint32_t flags;
 };
 
@@ -249,6 +250,7 @@ static struct pcpu *create_and_register_pcpu(struct xenpf_pcpuinfo *info)
 
 	INIT_LIST_HEAD(&pcpu->list);
 	pcpu->cpu_id = info->xen_cpuid;
+	pcpu->acpi_id = info->acpi_id;
 	pcpu->flags = info->flags;
 
 	/* Need hold on xen_pcpu_lock before pcpu list manipulations */
@@ -381,3 +383,21 @@ err1:
 	return ret;
 }
 arch_initcall(xen_pcpu_init);
+
+#ifdef CONFIG_ACPI
+bool __init xen_processor_present(uint32_t acpi_id)
+{
+	const struct pcpu *pcpu;
+	bool online = false;
+
+	mutex_lock(&xen_pcpu_lock);
+	list_for_each_entry(pcpu, &xen_pcpus, list)
+		if (pcpu->acpi_id == acpi_id) {
+			online = pcpu->flags & XEN_PCPU_FLAGS_ONLINE;
+			break;
+		}
+	mutex_unlock(&xen_pcpu_lock);
+
+	return online;
+}
+#endif
diff --git a/drivers/xen/xen-scsiback.c b/drivers/xen/xen-scsiback.c
index 954188b0b858..a7e7e02c415d 100644
--- a/drivers/xen/xen-scsiback.c
+++ b/drivers/xen/xen-scsiback.c
@@ -1406,11 +1406,6 @@ static void scsiback_drop_tport(struct se_wwn *wwn)
 	kfree(tport);
 }
 
-static u32 scsiback_tpg_get_inst_index(struct se_portal_group *se_tpg)
-{
-	return 1;
-}
-
 static int scsiback_check_stop_free(struct se_cmd *se_cmd)
 {
 	return transport_generic_free_cmd(se_cmd, 0);
@@ -1421,11 +1416,6 @@ static void scsiback_release_cmd(struct se_cmd *se_cmd)
 	target_free_tag(se_cmd->se_sess, se_cmd);
 }
 
-static u32 scsiback_sess_get_index(struct se_session *se_sess)
-{
-	return 0;
-}
-
 static int scsiback_write_pending(struct se_cmd *se_cmd)
 {
 	/* Go ahead and process the write immediately */
@@ -1434,15 +1424,6 @@ static int scsiback_write_pending(struct se_cmd *se_cmd)
 	return 0;
 }
 
-static void scsiback_set_default_node_attrs(struct se_node_acl *nacl)
-{
-}
-
-static int scsiback_get_cmd_state(struct se_cmd *se_cmd)
-{
-	return 0;
-}
-
 static int scsiback_queue_data_in(struct se_cmd *se_cmd)
 {
 	struct vscsibk_pend *pending_req = container_of(se_cmd,
@@ -1822,11 +1803,6 @@ static int scsiback_check_true(struct se_portal_group *se_tpg)
 	return 1;
 }
 
-static int scsiback_check_false(struct se_portal_group *se_tpg)
-{
-	return 0;
-}
-
 static const struct target_core_fabric_ops scsiback_ops = {
 	.module				= THIS_MODULE,
 	.fabric_name			= "xen-pvscsi",
@@ -1834,16 +1810,10 @@ static const struct target_core_fabric_ops scsiback_ops = {
 	.tpg_get_tag			= scsiback_get_tag,
 	.tpg_check_demo_mode		= scsiback_check_true,
 	.tpg_check_demo_mode_cache	= scsiback_check_true,
-	.tpg_check_demo_mode_write_protect = scsiback_check_false,
-	.tpg_check_prod_mode_write_protect = scsiback_check_false,
-	.tpg_get_inst_index		= scsiback_tpg_get_inst_index,
 	.check_stop_free		= scsiback_check_stop_free,
 	.release_cmd			= scsiback_release_cmd,
-	.sess_get_index			= scsiback_sess_get_index,
 	.sess_get_initiator_sid		= NULL,
 	.write_pending			= scsiback_write_pending,
-	.set_default_node_attributes	= scsiback_set_default_node_attrs,
-	.get_cmd_state			= scsiback_get_cmd_state,
 	.queue_data_in			= scsiback_queue_data_in,
 	.queue_status			= scsiback_queue_status,
 	.queue_tm_rsp			= scsiback_queue_tm_rsp,